| 20260506 08:40:22.426 | INFO | /opt/pacbio/bin/smrt-basecaller --process-log-interval=5000 --process-log=NONE --config=traceSaver.roi=[[2048,768,1,64],[2048,2304,1,64],[2048,3840,1,64],[2048,5376,1,64]]@bazWriter.samplingParams.filenames[0]=/data/nrta/0/m84031_260506_033729_s1/pb_internal/baz/m84031_260506_033729_s1_basic_preview.baz@bazWriter.samplingParams.filenames[1]=/data/nrta/0/m84031_260506_033729_s1/pb_internal/baz/m84031_260506_033729_s1_full_preview.baz@bazWriter.samplingParams.presets[0]=BasicPreview@bazWriter.samplingParams.presets[1]=FullPreview@realTimeMetrics.jsonOutputFile=/data/nrta/0/m84031_260506_033729_s1/pb_internal/m84031_260506_033729_s1.rtmetrics.json@/data/icc/m84031_260506_033729_s1/pb_internal/m84031_260506_033729_s1.basecaller.config.json@source.WXIPCDataSourceConfig.sraIndex=0@maskFraction=1.0000000@bazIO.MaxMiBToWrite=12874602@dataSource.darkCalFileName=/data/icc/m84031_260506_033729_s1/pb_internal/m84031_260506_033729_s1.darkcal.h5@dataSource.imagePsfKernel=[[0.0008,0.003,0.0166,0.0035,0.0011],[0.0027,0.013,0.0727,0.014,0.0032],[0.0118,0.0545,0.5944,0.0653,0.0124],[0.0032,0.0131,0.0727,0.0141,0.0031],[0.001,0.0029,0.0168,0.0034,0.0009]]@system.analyzerHardware=A100 --statusfd=2 --outputbazfile=/data/nrta/0/m84031_260506_033729_s1/pb_internal/baz/m84031_260506_033729_s1.bazfilelist.txt --outputtrcfile=/data/icc/m84031_260506_033729_s1/pb_internal/m84031_260506_033729_s1.trc.h5 --numWorkerThreads=0 --nop=0 --maxFrames=8640000 --log-level=INFO --log-file=/data/icc/m84031_260506_033729_s1/metadata/m84031_260506_033729_s1.basecaller.log,/var/log/pacbio/basecaller/m84031_260506_033729_s1.basecaller --log-directory-max-size=1073741820 | 20260506 08:40:22.426 | INFO | Version 26.1.0 | 20260506 08:40:22.426 | INFO | git branch: release/kestrel-13.5 | 20260506 08:40:22.426 | INFO | git commit hash: 2dc54f5a66f39b94662202d5514157d553e25f7d | 20260506 08:40:22.426 | INFO | git commit date: 2026-01-13 11:49:20 +0100 | 20260506 08:40:22.427 | INFO | { | 20260506 08:40:22.427 | INFO | "algorithm" : | 20260506 08:40:22.427 | INFO | { | 20260506 08:40:22.427 | INFO | "Metrics" : | 20260506 08:40:22.427 | INFO | { | 20260506 08:40:22.427 | INFO | "Method" : "Gpu", | 20260506 08:40:22.427 | INFO | "framesPerHFMetricBlock" : 4096, | 20260506 08:40:22.427 | INFO | "generateFeedbackMetrics" : true, | 20260506 08:40:22.427 | INFO | "realtimeActivityLabels" : true, | 20260506 08:40:22.427 | INFO | "sandwichTolerance" : 0 | 20260506 08:40:22.427 | INFO | }, | 20260506 08:40:22.427 | INFO | "baselineStatsAggregatorConfig" : | 20260506 08:40:22.427 | INFO | { | 20260506 08:40:22.427 | INFO | "Method" : "Gpu" | 20260506 08:40:22.427 | INFO | }, | 20260506 08:40:22.427 | INFO | "baselinerConfig" : | 20260506 08:40:22.427 | INFO | { | 20260506 08:40:22.427 | INFO | "Filter" : "MultiScaleLarge", | 20260506 08:40:22.427 | INFO | "JumpSuppression" : 0, | 20260506 08:40:22.427 | INFO | "MeanBiasAdjust" : 0, | 20260506 08:40:22.427 | INFO | "MeanEmaScaleStrides" : 0, | 20260506 08:40:22.427 | INFO | "Method" : "DeviceMultiScale", | 20260506 08:40:22.427 | INFO | "SigmaBiasAdjust" : 0, | 20260506 08:40:22.427 | INFO | "SigmaEmaScaleStrides" : 512 | 20260506 08:40:22.427 | INFO | }, | 20260506 08:40:22.427 | INFO | "dmeConfig" : | 20260506 08:40:22.427 | INFO | { | 20260506 08:40:22.427 | INFO | "AddPrelimEstimateConfidence" : true, | 20260506 08:40:22.427 | INFO | "AmplitudeFeedbackDampingFactor" : 0.89999997615814209, | 20260506 08:40:22.427 | INFO | "AmplitudeFeedbackInitSampleCount" : 2000, | 20260506 08:40:22.427 | INFO | "AnalogMixFractionThreshold" : | 20260506 08:40:22.427 | INFO | [ | 20260506 08:40:22.427 | INFO | 0.013000000268220901, | 20260506 08:40:22.427 | INFO | 0.039000000804662704 | 20260506 08:40:22.427 | INFO | ], | 20260506 08:40:22.427 | INFO | "ConfidenceHalfLife" : | 20260506 08:40:22.427 | INFO | [ | 20260506 08:40:22.427 | INFO | 4096, | 20260506 08:40:22.427 | INFO | 12000 | 20260506 08:40:22.427 | INFO | ], | 20260506 08:40:22.427 | INFO | "ConfidenceHalfLifePauseEnhance" : 0, | 20260506 08:40:22.427 | INFO | "ConvergeCoeff" : 2.9999999242136255e-05, | 20260506 08:40:22.427 | INFO | "EmIterationLimit" : 20, | 20260506 08:40:22.427 | INFO | "EnableAmplitudeRatioFeedback" : true, | 20260506 08:40:22.427 | INFO | "EnableFeedbackMultiloadAdjust" : false, | 20260506 08:40:22.427 | INFO | "EnablePulseVarianceFeedback" : true, | 20260506 08:40:22.428 | INFO | "FeedbackExcessNoiseCoeffDampFactor" : 19, | 20260506 08:40:22.428 | INFO | "FirstAttemptsIterMultiple" : 2, | 20260506 08:40:22.428 | INFO | "GTestStatFactor" : -1, | 20260506 08:40:22.428 | INFO | "GofLogChiSqrThresh1" : 111, | 20260506 08:40:22.428 | INFO | "GofLogChiSqrThresh2" : 8, | 20260506 08:40:22.428 | INFO | "HybridAtol" : 8, | 20260506 08:40:22.428 | INFO | "HybridRtol" : 48, | 20260506 08:40:22.428 | INFO | "IterateToLimit" : false, | 20260506 08:40:22.428 | INFO | "LaserPowerChangeReduceConfidence" : -1, | 20260506 08:40:22.428 | INFO | "MaxFeedbackVarRatio" : 2.5, | 20260506 08:40:22.428 | INFO | "Method" : "EmDevice", | 20260506 08:40:22.428 | INFO | "MinAnalogSnrThresh0" : 2, | 20260506 08:40:22.428 | INFO | "MinAnalogSnrThresh1" : 4, | 20260506 08:40:22.428 | INFO | "MinFeedbackBasesOverThreshold" : 4, | 20260506 08:40:22.428 | INFO | "MinFramesForEstimate" : 4000, | 20260506 08:40:22.428 | INFO | "ModelUpdateWeightMax" : 0.5, | 20260506 08:40:22.428 | INFO | "NumBinsMin" : 500, | 20260506 08:40:22.428 | INFO | "NumFramesPreAccumStats" : 1000, | 20260506 08:40:22.428 | INFO | "PulseAmpRegularization" : 0, | 20260506 08:40:22.428 | INFO | "PureUpdate" : false, | 20260506 08:40:22.428 | INFO | "RtalFeedbackLabelHistorySize" : 5, | 20260506 08:40:22.428 | INFO | "ScaleSnrConfTol" : 1, | 20260506 08:40:22.428 | INFO | "SimModel" : | 20260506 08:40:22.428 | INFO | { | 20260506 08:40:22.428 | INFO | "baselineMean" : 200, | 20260506 08:40:22.428 | INFO | "baselineVar" : 33, | 20260506 08:40:22.428 | INFO | "useSimulatedBaselineParams" : false | 20260506 08:40:22.428 | INFO | }, | 20260506 08:40:22.428 | INFO | "SnrDropThresh" : 1, | 20260506 08:40:22.428 | INFO | "SuccessConfidenceThresh" : 0.10000000149011612, | 20260506 08:40:22.428 | INFO | "ThreshAnalogSNR" : 2.1489999294281006, | 20260506 08:40:22.428 | INFO | "ThreshFractionFeedbackPulses" : 0.12999999523162842, | 20260506 08:40:22.428 | INFO | "XsnArrayCVRatios" : | 20260506 08:40:22.428 | INFO | [ | 20260506 08:40:22.428 | INFO | 0, | 20260506 08:40:22.428 | INFO | 0, | 20260506 08:40:22.428 | INFO | 0, | 20260506 08:40:22.428 | INFO | 0 | 20260506 08:40:22.428 | INFO | ], | 20260506 08:40:22.428 | INFO | "XsnCoeffLstSqWeightPower" : 2, | 20260506 08:40:22.428 | INFO | "XsnCoeffRatioCeil" : 10, | 20260506 08:40:22.428 | INFO | "XsnCoeffRatioFloor" : 0.0099999997764825821 | 20260506 08:40:22.428 | INFO | }, | 20260506 08:40:22.428 | INFO | "frameLabelerConfig" : | 20260506 08:40:22.428 | INFO | { | 20260506 08:40:22.428 | INFO | "Method" : "Device", | 20260506 08:40:22.428 | INFO | "roi" : | 20260506 08:40:22.428 | INFO | { | 20260506 08:40:22.428 | INFO | "filterType" : "Default", | 20260506 08:40:22.428 | INFO | "lowerThreshold" : 2, | 20260506 08:40:22.428 | INFO | "upperThreshold" : 6 | 20260506 08:40:22.428 | INFO | }, | 20260506 08:40:22.428 | INFO | "viterbi" : | 20260506 08:40:22.428 | INFO | { | 20260506 08:40:22.428 | INFO | "alpha" : 1, | 20260506 08:40:22.428 | INFO | "beta" : 1, | 20260506 08:40:22.428 | INFO | "gamma" : 1 | 20260506 08:40:22.428 | INFO | } | 20260506 08:40:22.428 | INFO | }, | 20260506 08:40:22.428 | INFO | "modelEstimationMode" : "DynamicEstimations", | 20260506 08:40:22.428 | INFO | "pulseAccumConfig" : | 20260506 08:40:22.428 | INFO | { | 20260506 08:40:22.428 | INFO | "Method" : "GpuPulses", | 20260506 08:40:22.428 | INFO | "XspAmpThresh" : 0.69999999999999996, | 20260506 08:40:22.428 | INFO | "XspWidthThresh" : 3.5, | 20260506 08:40:22.428 | INFO | "maxPulsePerSec" : 20, | 20260506 08:40:22.428 | INFO | "simConfig" : | 20260506 08:40:22.428 | INFO | { | 20260506 08:40:22.428 | INFO | "basecalls" : | 20260506 08:40:22.428 | INFO | [ | 20260506 08:40:22.428 | INFO | "A", | 20260506 08:40:22.428 | INFO | "C", | 20260506 08:40:22.428 | INFO | "G", | 20260506 08:40:22.428 | INFO | "T" | 20260506 08:40:22.428 | INFO | ], | 20260506 08:40:22.428 | INFO | "ipdVars" : [], | 20260506 08:40:22.428 | INFO | "ipds" : | 20260506 08:40:22.428 | INFO | [ | 20260506 08:40:22.428 | INFO | 6 | 20260506 08:40:22.428 | INFO | ], | 20260506 08:40:22.428 | INFO | "maxSignals" : | 20260506 08:40:22.428 | INFO | [ | 20260506 08:40:22.428 | INFO | 21, | 20260506 08:40:22.428 | INFO | 11, | 20260506 08:40:22.428 | INFO | 17, | 20260506 08:40:22.428 | INFO | 9 | 20260506 08:40:22.428 | INFO | ], | 20260506 08:40:22.428 | INFO | "maxSignalsVars" : [], | 20260506 08:40:22.428 | INFO | "meanSignals" : | 20260506 08:40:22.428 | INFO | [ | 20260506 08:40:22.428 | INFO | 20, | 20260506 08:40:22.428 | INFO | 10, | 20260506 08:40:22.428 | INFO | 16, | 20260506 08:40:22.428 | INFO | 8 | 20260506 08:40:22.428 | INFO | ], | 20260506 08:40:22.428 | INFO | "meanSignalsVars" : [], | 20260506 08:40:22.429 | INFO | "midSignals" : | 20260506 08:40:22.429 | INFO | [ | 20260506 08:40:22.429 | INFO | 21, | 20260506 08:40:22.429 | INFO | 11, | 20260506 08:40:22.429 | INFO | 17, | 20260506 08:40:22.429 | INFO | 9 | 20260506 08:40:22.429 | INFO | ], | 20260506 08:40:22.429 | INFO | "midSignalsVars" : [], | 20260506 08:40:22.429 | INFO | "pwVars" : [], | 20260506 08:40:22.429 | INFO | "pws" : | 20260506 08:40:22.429 | INFO | [ | 20260506 08:40:22.429 | INFO | 5 | 20260506 08:40:22.429 | INFO | ], | 20260506 08:40:22.429 | INFO | "seed" : 0 | 20260506 08:40:22.429 | INFO | } | 20260506 08:40:22.429 | INFO | }, | 20260506 08:40:22.429 | INFO | "pulseSerlzConfig" : | 20260506 08:40:22.429 | INFO | { | 20260506 08:40:22.429 | INFO | "Method" : "GpuPulses", | 20260506 08:40:22.429 | INFO | "PulseCountCoeff" : 2 | 20260506 08:40:22.429 | INFO | }, | 20260506 08:40:22.429 | INFO | "simulatedFaults" : | 20260506 08:40:22.429 | INFO | { | 20260506 08:40:22.429 | INFO | "negativeDyeSpectrumCounter" : 0 | 20260506 08:40:22.429 | INFO | }, | 20260506 08:40:22.429 | INFO | "staticDetModelConfig" : | 20260506 08:40:22.429 | INFO | { | 20260506 08:40:22.429 | INFO | "baselineMean" : 0, | 20260506 08:40:22.429 | INFO | "baselineVariance" : 33 | 20260506 08:40:22.429 | INFO | }, | 20260506 08:40:22.429 | INFO | "traceHistogramConfig" : | 20260506 08:40:22.429 | INFO | { | 20260506 08:40:22.429 | INFO | "BaselineStatMinFrameCount" : 50, | 20260506 08:40:22.429 | INFO | "BinSizeCoeff" : 0.20000000298023224, | 20260506 08:40:22.429 | INFO | "BinSizeLowBoundCoeff" : 1, | 20260506 08:40:22.429 | INFO | "FallBackBaselineSigma" : 10, | 20260506 08:40:22.429 | INFO | "Method" : "Gpu" | 20260506 08:40:22.429 | INFO | } | 20260506 08:40:22.429 | INFO | }, | 20260506 08:40:22.429 | INFO | "bazIO" : | 20260506 08:40:22.429 | INFO | { | 20260506 08:40:22.429 | INFO | "BufferSizeMiBytes" : 4, | 20260506 08:40:22.429 | INFO | "MaxMiBPerSecondWrite" : 450, | 20260506 08:40:22.429 | INFO | "MaxMiBToWrite" : 12874602, | 20260506 08:40:22.429 | INFO | "SyncSizeMiBytes" : 32, | 20260506 08:40:22.429 | INFO | "compression" : | 20260506 08:40:22.429 | INFO | { | 20260506 08:40:22.429 | INFO | "blockSize" : 131072, | 20260506 08:40:22.429 | INFO | "enabled" : true, | 20260506 08:40:22.429 | INFO | "level" : 1 | 20260506 08:40:22.429 | INFO | } | 20260506 08:40:22.429 | INFO | }, | 20260506 08:40:22.429 | INFO | "bazWriter" : | 20260506 08:40:22.429 | INFO | { | 20260506 08:40:22.429 | INFO | "maxChunksQueued" : 2, | 20260506 08:40:22.429 | INFO | "samplingParams" : | 20260506 08:40:22.429 | INFO | { | 20260506 08:40:22.429 | INFO | "filenames" : | 20260506 08:40:22.429 | INFO | [ | 20260506 08:40:22.429 | INFO | "/data/nrta/0/m84031_260506_033729_s1/pb_internal/baz/m84031_260506_033729_s1_basic_preview.baz", | 20260506 08:40:22.429 | INFO | "/data/nrta/0/m84031_260506_033729_s1/pb_internal/baz/m84031_260506_033729_s1_full_preview.baz" | 20260506 08:40:22.429 | INFO | ], | 20260506 08:40:22.429 | INFO | "presets" : | 20260506 08:40:22.429 | INFO | [ | 20260506 08:40:22.429 | INFO | "BasicPreview", | 20260506 08:40:22.429 | INFO | "FullPreview" | 20260506 08:40:22.429 | INFO | ], | 20260506 08:40:22.429 | INFO | "roiList" : | 20260506 08:40:22.429 | INFO | [ | 20260506 08:40:22.429 | INFO | [ | 20260506 08:40:22.429 | INFO | [ | 20260506 08:40:22.429 | INFO | 128, | 20260506 08:40:22.429 | INFO | 128, | 20260506 08:40:22.429 | INFO | 1, | 20260506 08:40:22.429 | INFO | 64 | 20260506 08:40:22.429 | INFO | ], | 20260506 08:40:22.429 | INFO | [ | 20260506 08:40:22.429 | INFO | 384, | 20260506 08:40:22.429 | INFO | 128, | 20260506 08:40:22.429 | INFO | 1, | 20260506 08:40:22.429 | INFO | 64 | 20260506 08:40:22.429 | INFO | ], | 20260506 08:40:22.429 | INFO | [ | 20260506 08:40:22.429 | INFO | 640, | 20260506 08:40:22.429 | INFO | 128, | 20260506 08:40:22.429 | INFO | 1, | 20260506 08:40:22.429 | INFO | 64 | 20260506 08:40:22.429 | INFO | ], | 20260506 08:40:22.429 | INFO | [ | 20260506 08:40:22.429 | INFO | 896, | 20260506 08:40:22.429 | INFO | 128, | 20260506 08:40:22.429 | INFO | 1, | 20260506 08:40:22.429 | INFO | 64 | 20260506 08:40:22.429 | INFO | ], | 20260506 08:40:22.430 | INFO | [ | 20260506 08:40:22.430 | INFO | 1152, | 20260506 08:40:22.430 | INFO | 128, | 20260506 08:40:22.430 | INFO | 1, | 20260506 08:40:22.430 | INFO | 64 | 20260506 08:40:22.430 | INFO | ], | 20260506 08:40:22.430 | INFO | [ | 20260506 08:40:22.430 | INFO | 1408, | 20260506 08:40:22.430 | INFO | 128, | 20260506 08:40:22.430 | INFO | 1, | 20260506 08:40:22.430 | INFO | 64 | 20260506 08:40:22.430 | INFO | ], | 20260506 08:40:22.430 | INFO | [ | 20260506 08:40:22.430 | INFO | 1664, | 20260506 08:40:22.430 | INFO | 128, | 20260506 08:40:22.430 | INFO | 1, | 20260506 08:40:22.430 | INFO | 64 | 20260506 08:40:22.430 | INFO | ], | 20260506 08:40:22.430 | INFO | [ | 20260506 08:40:22.430 | INFO | 1920, | 20260506 08:40:22.430 | INFO | 128, | 20260506 08:40:22.430 | INFO | 1, | 20260506 08:40:22.430 | INFO | 64 | 20260506 08:40:22.430 | INFO | ], | 20260506 08:40:22.430 | INFO | [ | 20260506 08:40:22.430 | INFO | 2176, | 20260506 08:40:22.430 | INFO | 128, | 20260506 08:40:22.430 | INFO | 1, | 20260506 08:40:22.430 | INFO | 64 | 20260506 08:40:22.430 | INFO | ], | 20260506 08:40:22.430 | INFO | [ | 20260506 08:40:22.430 | INFO | 2432, | 20260506 08:40:22.430 | INFO | 128, | 20260506 08:40:22.430 | INFO | 1, | 20260506 08:40:22.430 | INFO | 64 | 20260506 08:40:22.430 | INFO | ], | 20260506 08:40:22.430 | INFO | [ | 20260506 08:40:22.430 | INFO | 2688, | 20260506 08:40:22.430 | INFO | 128, | 20260506 08:40:22.430 | INFO | 1, | 20260506 08:40:22.430 | INFO | 64 | 20260506 08:40:22.430 | INFO | ], | 20260506 08:40:22.430 | INFO | [ | 20260506 08:40:22.430 | INFO | 2944, | 20260506 08:40:22.430 | INFO | 128, | 20260506 08:40:22.430 | INFO | 1, | 20260506 08:40:22.430 | INFO | 64 | 20260506 08:40:22.430 | INFO | ], | 20260506 08:40:22.430 | INFO | [ | 20260506 08:40:22.430 | INFO | 3200, | 20260506 08:40:22.430 | INFO | 128, | 20260506 08:40:22.430 | INFO | 1, | 20260506 08:40:22.430 | INFO | 64 | 20260506 08:40:22.430 | INFO | ], | 20260506 08:40:22.430 | INFO | [ | 20260506 08:40:22.430 | INFO | 3456, | 20260506 08:40:22.430 | INFO | 128, | 20260506 08:40:22.430 | INFO | 1, | 20260506 08:40:22.430 | INFO | 64 | 20260506 08:40:22.430 | INFO | ], | 20260506 08:40:22.430 | INFO | [ | 20260506 08:40:22.430 | INFO | 3712, | 20260506 08:40:22.430 | INFO | 128, | 20260506 08:40:22.430 | INFO | 1, | 20260506 08:40:22.430 | INFO | 64 | 20260506 08:40:22.430 | INFO | ], | 20260506 08:40:22.430 | INFO | [ | 20260506 08:40:22.430 | INFO | 3968, | 20260506 08:40:22.430 | INFO | 128, | 20260506 08:40:22.430 | INFO | 1, | 20260506 08:40:22.430 | INFO | 64 | 20260506 08:40:22.430 | INFO | ], | 20260506 08:40:22.430 | INFO | [ | 20260506 08:40:22.430 | INFO | 128, | 20260506 08:40:22.430 | INFO | 384, | 20260506 08:40:22.430 | INFO | 1, | 20260506 08:40:22.430 | INFO | 64 | 20260506 08:40:22.430 | INFO | ], | 20260506 08:40:22.430 | INFO | [ | 20260506 08:40:22.430 | INFO | 384, | 20260506 08:40:22.430 | INFO | 384, | 20260506 08:40:22.430 | INFO | 1, | 20260506 08:40:22.430 | INFO | 64 | 20260506 08:40:22.430 | INFO | ], | 20260506 08:40:22.430 | INFO | [ | 20260506 08:40:22.430 | INFO | 640, | 20260506 08:40:22.430 | INFO | 384, | 20260506 08:40:22.430 | INFO | 1, | 20260506 08:40:22.430 | INFO | 64 | 20260506 08:40:22.430 | INFO | ], | 20260506 08:40:22.430 | INFO | [ | 20260506 08:40:22.430 | INFO | 896, | 20260506 08:40:22.430 | INFO | 384, | 20260506 08:40:22.430 | INFO | 1, | 20260506 08:40:22.431 | INFO | 64 | 20260506 08:40:22.431 | INFO | ], | 20260506 08:40:22.431 | INFO | [ | 20260506 08:40:22.431 | INFO | 1152, | 20260506 08:40:22.431 | INFO | 384, | 20260506 08:40:22.431 | INFO | 1, | 20260506 08:40:22.431 | INFO | 64 | 20260506 08:40:22.431 | INFO | ], | 20260506 08:40:22.431 | INFO | [ | 20260506 08:40:22.431 | INFO | 1408, | 20260506 08:40:22.431 | INFO | 384, | 20260506 08:40:22.431 | INFO | 1, | 20260506 08:40:22.431 | INFO | 64 | 20260506 08:40:22.431 | INFO | ], | 20260506 08:40:22.431 | INFO | [ | 20260506 08:40:22.431 | INFO | 1664, | 20260506 08:40:22.431 | INFO | 384, | 20260506 08:40:22.431 | INFO | 1, | 20260506 08:40:22.431 | INFO | 64 | 20260506 08:40:22.431 | INFO | ], | 20260506 08:40:22.431 | INFO | [ | 20260506 08:40:22.431 | INFO | 1920, | 20260506 08:40:22.431 | INFO | 384, | 20260506 08:40:22.431 | INFO | 1, | 20260506 08:40:22.431 | INFO | 64 | 20260506 08:40:22.431 | INFO | ], | 20260506 08:40:22.431 | INFO | [ | 20260506 08:40:22.431 | INFO | 2176, | 20260506 08:40:22.431 | INFO | 384, | 20260506 08:40:22.431 | INFO | 1, | 20260506 08:40:22.431 | INFO | 64 | 20260506 08:40:22.431 | INFO | ], | 20260506 08:40:22.431 | INFO | [ | 20260506 08:40:22.431 | INFO | 2432, | 20260506 08:40:22.431 | INFO | 384, | 20260506 08:40:22.431 | INFO | 1, | 20260506 08:40:22.431 | INFO | 64 | 20260506 08:40:22.431 | INFO | ], | 20260506 08:40:22.431 | INFO | [ | 20260506 08:40:22.431 | INFO | 2688, | 20260506 08:40:22.431 | INFO | 384, | 20260506 08:40:22.431 | INFO | 1, | 20260506 08:40:22.431 | INFO | 64 | 20260506 08:40:22.431 | INFO | ], | 20260506 08:40:22.431 | INFO | [ | 20260506 08:40:22.431 | INFO | 2944, | 20260506 08:40:22.431 | INFO | 384, | 20260506 08:40:22.431 | INFO | 1, | 20260506 08:40:22.431 | INFO | 64 | 20260506 08:40:22.431 | INFO | ], | 20260506 08:40:22.431 | INFO | [ | 20260506 08:40:22.431 | INFO | 3200, | 20260506 08:40:22.431 | INFO | 384, | 20260506 08:40:22.431 | INFO | 1, | 20260506 08:40:22.431 | INFO | 64 | 20260506 08:40:22.431 | INFO | ], | 20260506 08:40:22.431 | INFO | [ | 20260506 08:40:22.431 | INFO | 3456, | 20260506 08:40:22.431 | INFO | 384, | 20260506 08:40:22.431 | INFO | 1, | 20260506 08:40:22.431 | INFO | 64 | 20260506 08:40:22.431 | INFO | ], | 20260506 08:40:22.431 | INFO | [ | 20260506 08:40:22.431 | INFO | 3712, | 20260506 08:40:22.431 | INFO | 384, | 20260506 08:40:22.431 | INFO | 1, | 20260506 08:40:22.431 | INFO | 64 | 20260506 08:40:22.431 | INFO | ], | 20260506 08:40:22.431 | INFO | [ | 20260506 08:40:22.431 | INFO | 3968, | 20260506 08:40:22.431 | INFO | 384, | 20260506 08:40:22.431 | INFO | 1, | 20260506 08:40:22.431 | INFO | 64 | 20260506 08:40:22.431 | INFO | ], | 20260506 08:40:22.431 | INFO | [ | 20260506 08:40:22.431 | INFO | 128, | 20260506 08:40:22.431 | INFO | 640, | 20260506 08:40:22.431 | INFO | 1, | 20260506 08:40:22.431 | INFO | 64 | 20260506 08:40:22.431 | INFO | ], | 20260506 08:40:22.431 | INFO | [ | 20260506 08:40:22.431 | INFO | 384, | 20260506 08:40:22.431 | INFO | 640, | 20260506 08:40:22.431 | INFO | 1, | 20260506 08:40:22.431 | INFO | 64 | 20260506 08:40:22.431 | INFO | ], | 20260506 08:40:22.431 | INFO | [ | 20260506 08:40:22.431 | INFO | 640, | 20260506 08:40:22.431 | INFO | 640, | 20260506 08:40:22.431 | INFO | 1, | 20260506 08:40:22.431 | INFO | 64 | 20260506 08:40:22.431 | INFO | ], | 20260506 08:40:22.431 | INFO | [ | 20260506 08:40:22.431 | INFO | 896, | 20260506 08:40:22.432 | INFO | 640, | 20260506 08:40:22.432 | INFO | 1, | 20260506 08:40:22.432 | INFO | 64 | 20260506 08:40:22.432 | INFO | ], | 20260506 08:40:22.432 | INFO | [ | 20260506 08:40:22.432 | INFO | 1152, | 20260506 08:40:22.432 | INFO | 640, | 20260506 08:40:22.432 | INFO | 1, | 20260506 08:40:22.432 | INFO | 64 | 20260506 08:40:22.432 | INFO | ], | 20260506 08:40:22.432 | INFO | [ | 20260506 08:40:22.432 | INFO | 1408, | 20260506 08:40:22.432 | INFO | 640, | 20260506 08:40:22.432 | INFO | 1, | 20260506 08:40:22.432 | INFO | 64 | 20260506 08:40:22.432 | INFO | ], | 20260506 08:40:22.432 | INFO | [ | 20260506 08:40:22.432 | INFO | 1664, | 20260506 08:40:22.432 | INFO | 640, | 20260506 08:40:22.432 | INFO | 1, | 20260506 08:40:22.432 | INFO | 64 | 20260506 08:40:22.432 | INFO | ], | 20260506 08:40:22.432 | INFO | [ | 20260506 08:40:22.432 | INFO | 1920, | 20260506 08:40:22.432 | INFO | 640, | 20260506 08:40:22.432 | INFO | 1, | 20260506 08:40:22.432 | INFO | 64 | 20260506 08:40:22.432 | INFO | ], | 20260506 08:40:22.432 | INFO | [ | 20260506 08:40:22.432 | INFO | 2176, | 20260506 08:40:22.432 | INFO | 640, | 20260506 08:40:22.432 | INFO | 1, | 20260506 08:40:22.432 | INFO | 64 | 20260506 08:40:22.432 | INFO | ], | 20260506 08:40:22.432 | INFO | [ | 20260506 08:40:22.432 | INFO | 2432, | 20260506 08:40:22.432 | INFO | 640, | 20260506 08:40:22.432 | INFO | 1, | 20260506 08:40:22.432 | INFO | 64 | 20260506 08:40:22.432 | INFO | ], | 20260506 08:40:22.432 | INFO | [ | 20260506 08:40:22.432 | INFO | 2688, | 20260506 08:40:22.432 | INFO | 640, | 20260506 08:40:22.432 | INFO | 1, | 20260506 08:40:22.432 | INFO | 64 | 20260506 08:40:22.432 | INFO | ], | 20260506 08:40:22.432 | INFO | [ | 20260506 08:40:22.432 | INFO | 2944, | 20260506 08:40:22.432 | INFO | 640, | 20260506 08:40:22.432 | INFO | 1, | 20260506 08:40:22.432 | INFO | 64 | 20260506 08:40:22.432 | INFO | ], | 20260506 08:40:22.432 | INFO | [ | 20260506 08:40:22.432 | INFO | 3200, | 20260506 08:40:22.432 | INFO | 640, | 20260506 08:40:22.432 | INFO | 1, | 20260506 08:40:22.432 | INFO | 64 | 20260506 08:40:22.432 | INFO | ], | 20260506 08:40:22.432 | INFO | [ | 20260506 08:40:22.432 | INFO | 3456, | 20260506 08:40:22.432 | INFO | 640, | 20260506 08:40:22.432 | INFO | 1, | 20260506 08:40:22.432 | INFO | 64 | 20260506 08:40:22.432 | INFO | ], | 20260506 08:40:22.432 | INFO | [ | 20260506 08:40:22.432 | INFO | 3712, | 20260506 08:40:22.432 | INFO | 640, | 20260506 08:40:22.432 | INFO | 1, | 20260506 08:40:22.432 | INFO | 64 | 20260506 08:40:22.432 | INFO | ], | 20260506 08:40:22.432 | INFO | [ | 20260506 08:40:22.432 | INFO | 3968, | 20260506 08:40:22.432 | INFO | 640, | 20260506 08:40:22.432 | INFO | 1, | 20260506 08:40:22.432 | INFO | 64 | 20260506 08:40:22.432 | INFO | ], | 20260506 08:40:22.432 | INFO | [ | 20260506 08:40:22.432 | INFO | 128, | 20260506 08:40:22.432 | INFO | 896, | 20260506 08:40:22.432 | INFO | 1, | 20260506 08:40:22.432 | INFO | 64 | 20260506 08:40:22.432 | INFO | ], | 20260506 08:40:22.432 | INFO | [ | 20260506 08:40:22.432 | INFO | 384, | 20260506 08:40:22.432 | INFO | 896, | 20260506 08:40:22.432 | INFO | 1, | 20260506 08:40:22.432 | INFO | 64 | 20260506 08:40:22.432 | INFO | ], | 20260506 08:40:22.432 | INFO | [ | 20260506 08:40:22.432 | INFO | 640, | 20260506 08:40:22.432 | INFO | 896, | 20260506 08:40:22.432 | INFO | 1, | 20260506 08:40:22.432 | INFO | 64 | 20260506 08:40:22.433 | INFO | ], | 20260506 08:40:22.433 | INFO | [ | 20260506 08:40:22.433 | INFO | 896, | 20260506 08:40:22.433 | INFO | 896, | 20260506 08:40:22.433 | INFO | 1, | 20260506 08:40:22.433 | INFO | 64 | 20260506 08:40:22.433 | INFO | ], | 20260506 08:40:22.433 | INFO | [ | 20260506 08:40:22.433 | INFO | 1152, | 20260506 08:40:22.433 | INFO | 896, | 20260506 08:40:22.433 | INFO | 1, | 20260506 08:40:22.433 | INFO | 64 | 20260506 08:40:22.433 | INFO | ], | 20260506 08:40:22.433 | INFO | [ | 20260506 08:40:22.433 | INFO | 1408, | 20260506 08:40:22.433 | INFO | 896, | 20260506 08:40:22.433 | INFO | 1, | 20260506 08:40:22.433 | INFO | 64 | 20260506 08:40:22.433 | INFO | ], | 20260506 08:40:22.433 | INFO | [ | 20260506 08:40:22.433 | INFO | 1664, | 20260506 08:40:22.433 | INFO | 896, | 20260506 08:40:22.433 | INFO | 1, | 20260506 08:40:22.433 | INFO | 64 | 20260506 08:40:22.433 | INFO | ], | 20260506 08:40:22.433 | INFO | [ | 20260506 08:40:22.433 | INFO | 1920, | 20260506 08:40:22.433 | INFO | 896, | 20260506 08:40:22.433 | INFO | 1, | 20260506 08:40:22.433 | INFO | 64 | 20260506 08:40:22.433 | INFO | ], | 20260506 08:40:22.433 | INFO | [ | 20260506 08:40:22.433 | INFO | 2176, | 20260506 08:40:22.433 | INFO | 896, | 20260506 08:40:22.433 | INFO | 1, | 20260506 08:40:22.433 | INFO | 64 | 20260506 08:40:22.433 | INFO | ], | 20260506 08:40:22.433 | INFO | [ | 20260506 08:40:22.433 | INFO | 2432, | 20260506 08:40:22.433 | INFO | 896, | 20260506 08:40:22.433 | INFO | 1, | 20260506 08:40:22.433 | INFO | 64 | 20260506 08:40:22.433 | INFO | ], | 20260506 08:40:22.433 | INFO | [ | 20260506 08:40:22.433 | INFO | 2688, | 20260506 08:40:22.433 | INFO | 896, | 20260506 08:40:22.433 | INFO | 1, | 20260506 08:40:22.433 | INFO | 64 | 20260506 08:40:22.433 | INFO | ], | 20260506 08:40:22.433 | INFO | [ | 20260506 08:40:22.433 | INFO | 2944, | 20260506 08:40:22.433 | INFO | 896, | 20260506 08:40:22.433 | INFO | 1, | 20260506 08:40:22.433 | INFO | 64 | 20260506 08:40:22.433 | INFO | ], | 20260506 08:40:22.433 | INFO | [ | 20260506 08:40:22.433 | INFO | 3200, | 20260506 08:40:22.433 | INFO | 896, | 20260506 08:40:22.433 | INFO | 1, | 20260506 08:40:22.433 | INFO | 64 | 20260506 08:40:22.433 | INFO | ], | 20260506 08:40:22.433 | INFO | [ | 20260506 08:40:22.433 | INFO | 3456, | 20260506 08:40:22.433 | INFO | 896, | 20260506 08:40:22.433 | INFO | 1, | 20260506 08:40:22.433 | INFO | 64 | 20260506 08:40:22.433 | INFO | ], | 20260506 08:40:22.433 | INFO | [ | 20260506 08:40:22.433 | INFO | 3712, | 20260506 08:40:22.433 | INFO | 896, | 20260506 08:40:22.433 | INFO | 1, | 20260506 08:40:22.433 | INFO | 64 | 20260506 08:40:22.433 | INFO | ], | 20260506 08:40:22.433 | INFO | [ | 20260506 08:40:22.433 | INFO | 3968, | 20260506 08:40:22.433 | INFO | 896, | 20260506 08:40:22.433 | INFO | 1, | 20260506 08:40:22.433 | INFO | 64 | 20260506 08:40:22.433 | INFO | ], | 20260506 08:40:22.433 | INFO | [ | 20260506 08:40:22.433 | INFO | 128, | 20260506 08:40:22.433 | INFO | 1152, | 20260506 08:40:22.433 | INFO | 1, | 20260506 08:40:22.433 | INFO | 64 | 20260506 08:40:22.433 | INFO | ], | 20260506 08:40:22.433 | INFO | [ | 20260506 08:40:22.433 | INFO | 384, | 20260506 08:40:22.433 | INFO | 1152, | 20260506 08:40:22.433 | INFO | 1, | 20260506 08:40:22.433 | INFO | 64 | 20260506 08:40:22.433 | INFO | ], | 20260506 08:40:22.433 | INFO | [ | 20260506 08:40:22.433 | INFO | 640, | 20260506 08:40:22.433 | INFO | 1152, | 20260506 08:40:22.433 | INFO | 1, | 20260506 08:40:22.433 | INFO | 64 | 20260506 08:40:22.434 | INFO | ], | 20260506 08:40:22.434 | INFO | [ | 20260506 08:40:22.434 | INFO | 896, | 20260506 08:40:22.434 | INFO | 1152, | 20260506 08:40:22.434 | INFO | 1, | 20260506 08:40:22.434 | INFO | 64 | 20260506 08:40:22.434 | INFO | ], | 20260506 08:40:22.434 | INFO | [ | 20260506 08:40:22.434 | INFO | 1152, | 20260506 08:40:22.434 | INFO | 1152, | 20260506 08:40:22.434 | INFO | 1, | 20260506 08:40:22.434 | INFO | 64 | 20260506 08:40:22.434 | INFO | ], | 20260506 08:40:22.434 | INFO | [ | 20260506 08:40:22.434 | INFO | 1408, | 20260506 08:40:22.434 | INFO | 1152, | 20260506 08:40:22.434 | INFO | 1, | 20260506 08:40:22.434 | INFO | 64 | 20260506 08:40:22.434 | INFO | ], | 20260506 08:40:22.434 | INFO | [ | 20260506 08:40:22.434 | INFO | 1664, | 20260506 08:40:22.434 | INFO | 1152, | 20260506 08:40:22.434 | INFO | 1, | 20260506 08:40:22.434 | INFO | 64 | 20260506 08:40:22.434 | INFO | ], | 20260506 08:40:22.434 | INFO | [ | 20260506 08:40:22.434 | INFO | 1920, | 20260506 08:40:22.434 | INFO | 1152, | 20260506 08:40:22.434 | INFO | 1, | 20260506 08:40:22.434 | INFO | 64 | 20260506 08:40:22.434 | INFO | ], | 20260506 08:40:22.434 | INFO | [ | 20260506 08:40:22.434 | INFO | 2176, | 20260506 08:40:22.434 | INFO | 1152, | 20260506 08:40:22.434 | INFO | 1, | 20260506 08:40:22.434 | INFO | 64 | 20260506 08:40:22.434 | INFO | ], | 20260506 08:40:22.434 | INFO | [ | 20260506 08:40:22.434 | INFO | 2432, | 20260506 08:40:22.434 | INFO | 1152, | 20260506 08:40:22.434 | INFO | 1, | 20260506 08:40:22.434 | INFO | 64 | 20260506 08:40:22.434 | INFO | ], | 20260506 08:40:22.434 | INFO | [ | 20260506 08:40:22.434 | INFO | 2688, | 20260506 08:40:22.434 | INFO | 1152, | 20260506 08:40:22.434 | INFO | 1, | 20260506 08:40:22.434 | INFO | 64 | 20260506 08:40:22.434 | INFO | ], | 20260506 08:40:22.434 | INFO | [ | 20260506 08:40:22.434 | INFO | 2944, | 20260506 08:40:22.434 | INFO | 1152, | 20260506 08:40:22.434 | INFO | 1, | 20260506 08:40:22.434 | INFO | 64 | 20260506 08:40:22.434 | INFO | ], | 20260506 08:40:22.434 | INFO | [ | 20260506 08:40:22.434 | INFO | 3200, | 20260506 08:40:22.434 | INFO | 1152, | 20260506 08:40:22.434 | INFO | 1, | 20260506 08:40:22.434 | INFO | 64 | 20260506 08:40:22.434 | INFO | ], | 20260506 08:40:22.434 | INFO | [ | 20260506 08:40:22.434 | INFO | 3456, | 20260506 08:40:22.434 | INFO | 1152, | 20260506 08:40:22.434 | INFO | 1, | 20260506 08:40:22.434 | INFO | 64 | 20260506 08:40:22.434 | INFO | ], | 20260506 08:40:22.434 | INFO | [ | 20260506 08:40:22.434 | INFO | 3712, | 20260506 08:40:22.434 | INFO | 1152, | 20260506 08:40:22.434 | INFO | 1, | 20260506 08:40:22.434 | INFO | 64 | 20260506 08:40:22.434 | INFO | ], | 20260506 08:40:22.434 | INFO | [ | 20260506 08:40:22.434 | INFO | 3968, | 20260506 08:40:22.434 | INFO | 1152, | 20260506 08:40:22.434 | INFO | 1, | 20260506 08:40:22.434 | INFO | 64 | 20260506 08:40:22.434 | INFO | ], | 20260506 08:40:22.434 | INFO | [ | 20260506 08:40:22.434 | INFO | 128, | 20260506 08:40:22.434 | INFO | 1408, | 20260506 08:40:22.434 | INFO | 1, | 20260506 08:40:22.434 | INFO | 64 | 20260506 08:40:22.434 | INFO | ], | 20260506 08:40:22.434 | INFO | [ | 20260506 08:40:22.434 | INFO | 384, | 20260506 08:40:22.434 | INFO | 1408, | 20260506 08:40:22.434 | INFO | 1, | 20260506 08:40:22.434 | INFO | 64 | 20260506 08:40:22.434 | INFO | ], | 20260506 08:40:22.434 | INFO | [ | 20260506 08:40:22.434 | INFO | 640, | 20260506 08:40:22.434 | INFO | 1408, | 20260506 08:40:22.434 | INFO | 1, | 20260506 08:40:22.434 | INFO | 64 | 20260506 08:40:22.435 | INFO | ], | 20260506 08:40:22.435 | INFO | [ | 20260506 08:40:22.435 | INFO | 896, | 20260506 08:40:22.435 | INFO | 1408, | 20260506 08:40:22.435 | INFO | 1, | 20260506 08:40:22.435 | INFO | 64 | 20260506 08:40:22.435 | INFO | ], | 20260506 08:40:22.435 | INFO | [ | 20260506 08:40:22.435 | INFO | 1152, | 20260506 08:40:22.435 | INFO | 1408, | 20260506 08:40:22.435 | INFO | 1, | 20260506 08:40:22.435 | INFO | 64 | 20260506 08:40:22.435 | INFO | ], | 20260506 08:40:22.435 | INFO | [ | 20260506 08:40:22.435 | INFO | 1408, | 20260506 08:40:22.435 | INFO | 1408, | 20260506 08:40:22.435 | INFO | 1, | 20260506 08:40:22.435 | INFO | 64 | 20260506 08:40:22.435 | INFO | ], | 20260506 08:40:22.435 | INFO | [ | 20260506 08:40:22.435 | INFO | 1664, | 20260506 08:40:22.435 | INFO | 1408, | 20260506 08:40:22.435 | INFO | 1, | 20260506 08:40:22.435 | INFO | 64 | 20260506 08:40:22.435 | INFO | ], | 20260506 08:40:22.435 | INFO | [ | 20260506 08:40:22.435 | INFO | 1920, | 20260506 08:40:22.435 | INFO | 1408, | 20260506 08:40:22.435 | INFO | 1, | 20260506 08:40:22.435 | INFO | 64 | 20260506 08:40:22.435 | INFO | ], | 20260506 08:40:22.435 | INFO | [ | 20260506 08:40:22.435 | INFO | 2176, | 20260506 08:40:22.435 | INFO | 1408, | 20260506 08:40:22.435 | INFO | 1, | 20260506 08:40:22.435 | INFO | 64 | 20260506 08:40:22.435 | INFO | ], | 20260506 08:40:22.435 | INFO | [ | 20260506 08:40:22.435 | INFO | 2432, | 20260506 08:40:22.435 | INFO | 1408, | 20260506 08:40:22.435 | INFO | 1, | 20260506 08:40:22.435 | INFO | 64 | 20260506 08:40:22.435 | INFO | ], | 20260506 08:40:22.435 | INFO | [ | 20260506 08:40:22.435 | INFO | 2688, | 20260506 08:40:22.435 | INFO | 1408, | 20260506 08:40:22.435 | INFO | 1, | 20260506 08:40:22.435 | INFO | 64 | 20260506 08:40:22.435 | INFO | ], | 20260506 08:40:22.435 | INFO | [ | 20260506 08:40:22.435 | INFO | 2944, | 20260506 08:40:22.435 | INFO | 1408, | 20260506 08:40:22.435 | INFO | 1, | 20260506 08:40:22.435 | INFO | 64 | 20260506 08:40:22.435 | INFO | ], | 20260506 08:40:22.435 | INFO | [ | 20260506 08:40:22.435 | INFO | 3200, | 20260506 08:40:22.435 | INFO | 1408, | 20260506 08:40:22.435 | INFO | 1, | 20260506 08:40:22.435 | INFO | 64 | 20260506 08:40:22.435 | INFO | ], | 20260506 08:40:22.435 | INFO | [ | 20260506 08:40:22.435 | INFO | 3456, | 20260506 08:40:22.435 | INFO | 1408, | 20260506 08:40:22.435 | INFO | 1, | 20260506 08:40:22.435 | INFO | 64 | 20260506 08:40:22.435 | INFO | ], | 20260506 08:40:22.435 | INFO | [ | 20260506 08:40:22.435 | INFO | 3712, | 20260506 08:40:22.435 | INFO | 1408, | 20260506 08:40:22.435 | INFO | 1, | 20260506 08:40:22.435 | INFO | 64 | 20260506 08:40:22.435 | INFO | ], | 20260506 08:40:22.435 | INFO | [ | 20260506 08:40:22.435 | INFO | 3968, | 20260506 08:40:22.435 | INFO | 1408, | 20260506 08:40:22.435 | INFO | 1, | 20260506 08:40:22.435 | INFO | 64 | 20260506 08:40:22.435 | INFO | ], | 20260506 08:40:22.435 | INFO | [ | 20260506 08:40:22.435 | INFO | 128, | 20260506 08:40:22.435 | INFO | 1664, | 20260506 08:40:22.435 | INFO | 1, | 20260506 08:40:22.435 | INFO | 64 | 20260506 08:40:22.435 | INFO | ], | 20260506 08:40:22.435 | INFO | [ | 20260506 08:40:22.435 | INFO | 384, | 20260506 08:40:22.435 | INFO | 1664, | 20260506 08:40:22.435 | INFO | 1, | 20260506 08:40:22.435 | INFO | 64 | 20260506 08:40:22.435 | INFO | ], | 20260506 08:40:22.435 | INFO | [ | 20260506 08:40:22.435 | INFO | 640, | 20260506 08:40:22.435 | INFO | 1664, | 20260506 08:40:22.435 | INFO | 1, | 20260506 08:40:22.436 | INFO | 64 | 20260506 08:40:22.436 | INFO | ], | 20260506 08:40:22.436 | INFO | [ | 20260506 08:40:22.436 | INFO | 896, | 20260506 08:40:22.436 | INFO | 1664, | 20260506 08:40:22.436 | INFO | 1, | 20260506 08:40:22.436 | INFO | 64 | 20260506 08:40:22.436 | INFO | ], | 20260506 08:40:22.436 | INFO | [ | 20260506 08:40:22.436 | INFO | 1152, | 20260506 08:40:22.436 | INFO | 1664, | 20260506 08:40:22.436 | INFO | 1, | 20260506 08:40:22.436 | INFO | 64 | 20260506 08:40:22.436 | INFO | ], | 20260506 08:40:22.436 | INFO | [ | 20260506 08:40:22.436 | INFO | 1408, | 20260506 08:40:22.436 | INFO | 1664, | 20260506 08:40:22.436 | INFO | 1, | 20260506 08:40:22.436 | INFO | 64 | 20260506 08:40:22.436 | INFO | ], | 20260506 08:40:22.436 | INFO | [ | 20260506 08:40:22.436 | INFO | 1664, | 20260506 08:40:22.436 | INFO | 1664, | 20260506 08:40:22.436 | INFO | 1, | 20260506 08:40:22.436 | INFO | 64 | 20260506 08:40:22.436 | INFO | ], | 20260506 08:40:22.436 | INFO | [ | 20260506 08:40:22.436 | INFO | 1920, | 20260506 08:40:22.436 | INFO | 1664, | 20260506 08:40:22.436 | INFO | 1, | 20260506 08:40:22.436 | INFO | 64 | 20260506 08:40:22.436 | INFO | ], | 20260506 08:40:22.436 | INFO | [ | 20260506 08:40:22.436 | INFO | 2176, | 20260506 08:40:22.436 | INFO | 1664, | 20260506 08:40:22.436 | INFO | 1, | 20260506 08:40:22.436 | INFO | 64 | 20260506 08:40:22.436 | INFO | ], | 20260506 08:40:22.436 | INFO | [ | 20260506 08:40:22.436 | INFO | 2432, | 20260506 08:40:22.436 | INFO | 1664, | 20260506 08:40:22.436 | INFO | 1, | 20260506 08:40:22.436 | INFO | 64 | 20260506 08:40:22.436 | INFO | ], | 20260506 08:40:22.436 | INFO | [ | 20260506 08:40:22.436 | INFO | 2688, | 20260506 08:40:22.436 | INFO | 1664, | 20260506 08:40:22.436 | INFO | 1, | 20260506 08:40:22.436 | INFO | 64 | 20260506 08:40:22.436 | INFO | ], | 20260506 08:40:22.436 | INFO | [ | 20260506 08:40:22.436 | INFO | 2944, | 20260506 08:40:22.436 | INFO | 1664, | 20260506 08:40:22.436 | INFO | 1, | 20260506 08:40:22.436 | INFO | 64 | 20260506 08:40:22.436 | INFO | ], | 20260506 08:40:22.436 | INFO | [ | 20260506 08:40:22.436 | INFO | 3200, | 20260506 08:40:22.436 | INFO | 1664, | 20260506 08:40:22.436 | INFO | 1, | 20260506 08:40:22.436 | INFO | 64 | 20260506 08:40:22.436 | INFO | ], | 20260506 08:40:22.436 | INFO | [ | 20260506 08:40:22.436 | INFO | 3456, | 20260506 08:40:22.436 | INFO | 1664, | 20260506 08:40:22.436 | INFO | 1, | 20260506 08:40:22.436 | INFO | 64 | 20260506 08:40:22.436 | INFO | ], | 20260506 08:40:22.436 | INFO | [ | 20260506 08:40:22.436 | INFO | 3712, | 20260506 08:40:22.436 | INFO | 1664, | 20260506 08:40:22.436 | INFO | 1, | 20260506 08:40:22.436 | INFO | 64 | 20260506 08:40:22.436 | INFO | ], | 20260506 08:40:22.436 | INFO | [ | 20260506 08:40:22.436 | INFO | 3968, | 20260506 08:40:22.436 | INFO | 1664, | 20260506 08:40:22.436 | INFO | 1, | 20260506 08:40:22.436 | INFO | 64 | 20260506 08:40:22.436 | INFO | ], | 20260506 08:40:22.436 | INFO | [ | 20260506 08:40:22.436 | INFO | 128, | 20260506 08:40:22.436 | INFO | 1920, | 20260506 08:40:22.436 | INFO | 1, | 20260506 08:40:22.436 | INFO | 64 | 20260506 08:40:22.436 | INFO | ], | 20260506 08:40:22.436 | INFO | [ | 20260506 08:40:22.436 | INFO | 384, | 20260506 08:40:22.436 | INFO | 1920, | 20260506 08:40:22.436 | INFO | 1, | 20260506 08:40:22.436 | INFO | 64 | 20260506 08:40:22.436 | INFO | ], | 20260506 08:40:22.436 | INFO | [ | 20260506 08:40:22.436 | INFO | 640, | 20260506 08:40:22.436 | INFO | 1920, | 20260506 08:40:22.437 | INFO | 1, | 20260506 08:40:22.437 | INFO | 64 | 20260506 08:40:22.437 | INFO | ], | 20260506 08:40:22.437 | INFO | [ | 20260506 08:40:22.437 | INFO | 896, | 20260506 08:40:22.437 | INFO | 1920, | 20260506 08:40:22.437 | INFO | 1, | 20260506 08:40:22.437 | INFO | 64 | 20260506 08:40:22.437 | INFO | ], | 20260506 08:40:22.437 | INFO | [ | 20260506 08:40:22.437 | INFO | 1152, | 20260506 08:40:22.437 | INFO | 1920, | 20260506 08:40:22.437 | INFO | 1, | 20260506 08:40:22.437 | INFO | 64 | 20260506 08:40:22.437 | INFO | ], | 20260506 08:40:22.437 | INFO | [ | 20260506 08:40:22.437 | INFO | 1408, | 20260506 08:40:22.437 | INFO | 1920, | 20260506 08:40:22.437 | INFO | 1, | 20260506 08:40:22.437 | INFO | 64 | 20260506 08:40:22.437 | INFO | ], | 20260506 08:40:22.437 | INFO | [ | 20260506 08:40:22.437 | INFO | 1664, | 20260506 08:40:22.437 | INFO | 1920, | 20260506 08:40:22.437 | INFO | 1, | 20260506 08:40:22.437 | INFO | 64 | 20260506 08:40:22.437 | INFO | ], | 20260506 08:40:22.437 | INFO | [ | 20260506 08:40:22.437 | INFO | 1920, | 20260506 08:40:22.437 | INFO | 1920, | 20260506 08:40:22.437 | INFO | 1, | 20260506 08:40:22.437 | INFO | 64 | 20260506 08:40:22.437 | INFO | ], | 20260506 08:40:22.437 | INFO | [ | 20260506 08:40:22.437 | INFO | 2176, | 20260506 08:40:22.437 | INFO | 1920, | 20260506 08:40:22.437 | INFO | 1, | 20260506 08:40:22.437 | INFO | 64 | 20260506 08:40:22.437 | INFO | ], | 20260506 08:40:22.437 | INFO | [ | 20260506 08:40:22.437 | INFO | 2432, | 20260506 08:40:22.437 | INFO | 1920, | 20260506 08:40:22.437 | INFO | 1, | 20260506 08:40:22.437 | INFO | 64 | 20260506 08:40:22.437 | INFO | ], | 20260506 08:40:22.437 | INFO | [ | 20260506 08:40:22.437 | INFO | 2688, | 20260506 08:40:22.437 | INFO | 1920, | 20260506 08:40:22.437 | INFO | 1, | 20260506 08:40:22.437 | INFO | 64 | 20260506 08:40:22.437 | INFO | ], | 20260506 08:40:22.437 | INFO | [ | 20260506 08:40:22.437 | INFO | 2944, | 20260506 08:40:22.437 | INFO | 1920, | 20260506 08:40:22.437 | INFO | 1, | 20260506 08:40:22.437 | INFO | 64 | 20260506 08:40:22.437 | INFO | ], | 20260506 08:40:22.437 | INFO | [ | 20260506 08:40:22.437 | INFO | 3200, | 20260506 08:40:22.437 | INFO | 1920, | 20260506 08:40:22.437 | INFO | 1, | 20260506 08:40:22.437 | INFO | 64 | 20260506 08:40:22.437 | INFO | ], | 20260506 08:40:22.437 | INFO | [ | 20260506 08:40:22.437 | INFO | 3456, | 20260506 08:40:22.437 | INFO | 1920, | 20260506 08:40:22.437 | INFO | 1, | 20260506 08:40:22.437 | INFO | 64 | 20260506 08:40:22.437 | INFO | ], | 20260506 08:40:22.437 | INFO | [ | 20260506 08:40:22.437 | INFO | 3712, | 20260506 08:40:22.437 | INFO | 1920, | 20260506 08:40:22.437 | INFO | 1, | 20260506 08:40:22.437 | INFO | 64 | 20260506 08:40:22.437 | INFO | ], | 20260506 08:40:22.437 | INFO | [ | 20260506 08:40:22.437 | INFO | 3968, | 20260506 08:40:22.437 | INFO | 1920, | 20260506 08:40:22.437 | INFO | 1, | 20260506 08:40:22.437 | INFO | 64 | 20260506 08:40:22.437 | INFO | ], | 20260506 08:40:22.437 | INFO | [ | 20260506 08:40:22.437 | INFO | 128, | 20260506 08:40:22.437 | INFO | 2176, | 20260506 08:40:22.437 | INFO | 1, | 20260506 08:40:22.437 | INFO | 64 | 20260506 08:40:22.437 | INFO | ], | 20260506 08:40:22.437 | INFO | [ | 20260506 08:40:22.437 | INFO | 384, | 20260506 08:40:22.437 | INFO | 2176, | 20260506 08:40:22.437 | INFO | 1, | 20260506 08:40:22.437 | INFO | 64 | 20260506 08:40:22.437 | INFO | ], | 20260506 08:40:22.437 | INFO | [ | 20260506 08:40:22.438 | INFO | 640, | 20260506 08:40:22.438 | INFO | 2176, | 20260506 08:40:22.438 | INFO | 1, | 20260506 08:40:22.438 | INFO | 64 | 20260506 08:40:22.438 | INFO | ], | 20260506 08:40:22.438 | INFO | [ | 20260506 08:40:22.438 | INFO | 896, | 20260506 08:40:22.438 | INFO | 2176, | 20260506 08:40:22.438 | INFO | 1, | 20260506 08:40:22.438 | INFO | 64 | 20260506 08:40:22.438 | INFO | ], | 20260506 08:40:22.438 | INFO | [ | 20260506 08:40:22.438 | INFO | 1152, | 20260506 08:40:22.438 | INFO | 2176, | 20260506 08:40:22.438 | INFO | 1, | 20260506 08:40:22.438 | INFO | 64 | 20260506 08:40:22.438 | INFO | ], | 20260506 08:40:22.438 | INFO | [ | 20260506 08:40:22.438 | INFO | 1408, | 20260506 08:40:22.438 | INFO | 2176, | 20260506 08:40:22.438 | INFO | 1, | 20260506 08:40:22.438 | INFO | 64 | 20260506 08:40:22.438 | INFO | ], | 20260506 08:40:22.438 | INFO | [ | 20260506 08:40:22.438 | INFO | 1664, | 20260506 08:40:22.438 | INFO | 2176, | 20260506 08:40:22.438 | INFO | 1, | 20260506 08:40:22.438 | INFO | 64 | 20260506 08:40:22.438 | INFO | ], | 20260506 08:40:22.438 | INFO | [ | 20260506 08:40:22.438 | INFO | 1920, | 20260506 08:40:22.438 | INFO | 2176, | 20260506 08:40:22.438 | INFO | 1, | 20260506 08:40:22.438 | INFO | 64 | 20260506 08:40:22.438 | INFO | ], | 20260506 08:40:22.438 | INFO | [ | 20260506 08:40:22.438 | INFO | 2176, | 20260506 08:40:22.438 | INFO | 2176, | 20260506 08:40:22.438 | INFO | 1, | 20260506 08:40:22.438 | INFO | 64 | 20260506 08:40:22.438 | INFO | ], | 20260506 08:40:22.438 | INFO | [ | 20260506 08:40:22.438 | INFO | 2432, | 20260506 08:40:22.438 | INFO | 2176, | 20260506 08:40:22.438 | INFO | 1, | 20260506 08:40:22.438 | INFO | 64 | 20260506 08:40:22.438 | INFO | ], | 20260506 08:40:22.438 | INFO | [ | 20260506 08:40:22.438 | INFO | 2688, | 20260506 08:40:22.438 | INFO | 2176, | 20260506 08:40:22.438 | INFO | 1, | 20260506 08:40:22.438 | INFO | 64 | 20260506 08:40:22.438 | INFO | ], | 20260506 08:40:22.438 | INFO | [ | 20260506 08:40:22.438 | INFO | 2944, | 20260506 08:40:22.438 | INFO | 2176, | 20260506 08:40:22.438 | INFO | 1, | 20260506 08:40:22.438 | INFO | 64 | 20260506 08:40:22.438 | INFO | ], | 20260506 08:40:22.438 | INFO | [ | 20260506 08:40:22.438 | INFO | 3200, | 20260506 08:40:22.438 | INFO | 2176, | 20260506 08:40:22.438 | INFO | 1, | 20260506 08:40:22.438 | INFO | 64 | 20260506 08:40:22.438 | INFO | ], | 20260506 08:40:22.438 | INFO | [ | 20260506 08:40:22.438 | INFO | 3456, | 20260506 08:40:22.438 | INFO | 2176, | 20260506 08:40:22.438 | INFO | 1, | 20260506 08:40:22.438 | INFO | 64 | 20260506 08:40:22.438 | INFO | ], | 20260506 08:40:22.438 | INFO | [ | 20260506 08:40:22.438 | INFO | 3712, | 20260506 08:40:22.438 | INFO | 2176, | 20260506 08:40:22.438 | INFO | 1, | 20260506 08:40:22.438 | INFO | 64 | 20260506 08:40:22.438 | INFO | ], | 20260506 08:40:22.438 | INFO | [ | 20260506 08:40:22.438 | INFO | 3968, | 20260506 08:40:22.438 | INFO | 2176, | 20260506 08:40:22.438 | INFO | 1, | 20260506 08:40:22.438 | INFO | 64 | 20260506 08:40:22.438 | INFO | ], | 20260506 08:40:22.438 | INFO | [ | 20260506 08:40:22.438 | INFO | 128, | 20260506 08:40:22.438 | INFO | 2432, | 20260506 08:40:22.438 | INFO | 1, | 20260506 08:40:22.438 | INFO | 64 | 20260506 08:40:22.438 | INFO | ], | 20260506 08:40:22.438 | INFO | [ | 20260506 08:40:22.438 | INFO | 384, | 20260506 08:40:22.438 | INFO | 2432, | 20260506 08:40:22.438 | INFO | 1, | 20260506 08:40:22.438 | INFO | 64 | 20260506 08:40:22.439 | INFO | ], | 20260506 08:40:22.439 | INFO | [ | 20260506 08:40:22.439 | INFO | 640, | 20260506 08:40:22.439 | INFO | 2432, | 20260506 08:40:22.439 | INFO | 1, | 20260506 08:40:22.439 | INFO | 64 | 20260506 08:40:22.439 | INFO | ], | 20260506 08:40:22.439 | INFO | [ | 20260506 08:40:22.439 | INFO | 896, | 20260506 08:40:22.439 | INFO | 2432, | 20260506 08:40:22.439 | INFO | 1, | 20260506 08:40:22.439 | INFO | 64 | 20260506 08:40:22.439 | INFO | ], | 20260506 08:40:22.439 | INFO | [ | 20260506 08:40:22.439 | INFO | 1152, | 20260506 08:40:22.439 | INFO | 2432, | 20260506 08:40:22.439 | INFO | 1, | 20260506 08:40:22.439 | INFO | 64 | 20260506 08:40:22.439 | INFO | ], | 20260506 08:40:22.439 | INFO | [ | 20260506 08:40:22.439 | INFO | 1408, | 20260506 08:40:22.439 | INFO | 2432, | 20260506 08:40:22.439 | INFO | 1, | 20260506 08:40:22.439 | INFO | 64 | 20260506 08:40:22.439 | INFO | ], | 20260506 08:40:22.439 | INFO | [ | 20260506 08:40:22.439 | INFO | 1664, | 20260506 08:40:22.439 | INFO | 2432, | 20260506 08:40:22.439 | INFO | 1, | 20260506 08:40:22.439 | INFO | 64 | 20260506 08:40:22.439 | INFO | ], | 20260506 08:40:22.439 | INFO | [ | 20260506 08:40:22.439 | INFO | 1920, | 20260506 08:40:22.439 | INFO | 2432, | 20260506 08:40:22.439 | INFO | 1, | 20260506 08:40:22.439 | INFO | 64 | 20260506 08:40:22.439 | INFO | ], | 20260506 08:40:22.439 | INFO | [ | 20260506 08:40:22.439 | INFO | 2176, | 20260506 08:40:22.439 | INFO | 2432, | 20260506 08:40:22.439 | INFO | 1, | 20260506 08:40:22.439 | INFO | 64 | 20260506 08:40:22.439 | INFO | ], | 20260506 08:40:22.439 | INFO | [ | 20260506 08:40:22.439 | INFO | 2432, | 20260506 08:40:22.439 | INFO | 2432, | 20260506 08:40:22.439 | INFO | 1, | 20260506 08:40:22.439 | INFO | 64 | 20260506 08:40:22.439 | INFO | ], | 20260506 08:40:22.439 | INFO | [ | 20260506 08:40:22.439 | INFO | 2688, | 20260506 08:40:22.439 | INFO | 2432, | 20260506 08:40:22.439 | INFO | 1, | 20260506 08:40:22.439 | INFO | 64 | 20260506 08:40:22.439 | INFO | ], | 20260506 08:40:22.439 | INFO | [ | 20260506 08:40:22.439 | INFO | 2944, | 20260506 08:40:22.439 | INFO | 2432, | 20260506 08:40:22.439 | INFO | 1, | 20260506 08:40:22.439 | INFO | 64 | 20260506 08:40:22.439 | INFO | ], | 20260506 08:40:22.439 | INFO | [ | 20260506 08:40:22.439 | INFO | 3200, | 20260506 08:40:22.439 | INFO | 2432, | 20260506 08:40:22.439 | INFO | 1, | 20260506 08:40:22.439 | INFO | 64 | 20260506 08:40:22.439 | INFO | ], | 20260506 08:40:22.439 | INFO | [ | 20260506 08:40:22.439 | INFO | 3456, | 20260506 08:40:22.439 | INFO | 2432, | 20260506 08:40:22.439 | INFO | 1, | 20260506 08:40:22.439 | INFO | 64 | 20260506 08:40:22.439 | INFO | ], | 20260506 08:40:22.439 | INFO | [ | 20260506 08:40:22.439 | INFO | 3712, | 20260506 08:40:22.439 | INFO | 2432, | 20260506 08:40:22.439 | INFO | 1, | 20260506 08:40:22.439 | INFO | 64 | 20260506 08:40:22.439 | INFO | ], | 20260506 08:40:22.439 | INFO | [ | 20260506 08:40:22.439 | INFO | 3968, | 20260506 08:40:22.439 | INFO | 2432, | 20260506 08:40:22.439 | INFO | 1, | 20260506 08:40:22.439 | INFO | 64 | 20260506 08:40:22.439 | INFO | ], | 20260506 08:40:22.439 | INFO | [ | 20260506 08:40:22.439 | INFO | 128, | 20260506 08:40:22.439 | INFO | 2688, | 20260506 08:40:22.439 | INFO | 1, | 20260506 08:40:22.439 | INFO | 64 | 20260506 08:40:22.439 | INFO | ], | 20260506 08:40:22.439 | INFO | [ | 20260506 08:40:22.439 | INFO | 384, | 20260506 08:40:22.440 | INFO | 2688, | 20260506 08:40:22.440 | INFO | 1, | 20260506 08:40:22.440 | INFO | 64 | 20260506 08:40:22.440 | INFO | ], | 20260506 08:40:22.440 | INFO | [ | 20260506 08:40:22.440 | INFO | 640, | 20260506 08:40:22.440 | INFO | 2688, | 20260506 08:40:22.440 | INFO | 1, | 20260506 08:40:22.440 | INFO | 64 | 20260506 08:40:22.440 | INFO | ], | 20260506 08:40:22.440 | INFO | [ | 20260506 08:40:22.440 | INFO | 896, | 20260506 08:40:22.440 | INFO | 2688, | 20260506 08:40:22.440 | INFO | 1, | 20260506 08:40:22.440 | INFO | 64 | 20260506 08:40:22.440 | INFO | ], | 20260506 08:40:22.440 | INFO | [ | 20260506 08:40:22.440 | INFO | 1152, | 20260506 08:40:22.440 | INFO | 2688, | 20260506 08:40:22.440 | INFO | 1, | 20260506 08:40:22.440 | INFO | 64 | 20260506 08:40:22.440 | INFO | ], | 20260506 08:40:22.440 | INFO | [ | 20260506 08:40:22.440 | INFO | 1408, | 20260506 08:40:22.440 | INFO | 2688, | 20260506 08:40:22.440 | INFO | 1, | 20260506 08:40:22.440 | INFO | 64 | 20260506 08:40:22.440 | INFO | ], | 20260506 08:40:22.440 | INFO | [ | 20260506 08:40:22.440 | INFO | 1664, | 20260506 08:40:22.440 | INFO | 2688, | 20260506 08:40:22.440 | INFO | 1, | 20260506 08:40:22.440 | INFO | 64 | 20260506 08:40:22.440 | INFO | ], | 20260506 08:40:22.440 | INFO | [ | 20260506 08:40:22.440 | INFO | 1920, | 20260506 08:40:22.440 | INFO | 2688, | 20260506 08:40:22.440 | INFO | 1, | 20260506 08:40:22.440 | INFO | 64 | 20260506 08:40:22.440 | INFO | ], | 20260506 08:40:22.440 | INFO | [ | 20260506 08:40:22.440 | INFO | 2176, | 20260506 08:40:22.440 | INFO | 2688, | 20260506 08:40:22.440 | INFO | 1, | 20260506 08:40:22.440 | INFO | 64 | 20260506 08:40:22.440 | INFO | ], | 20260506 08:40:22.440 | INFO | [ | 20260506 08:40:22.440 | INFO | 2432, | 20260506 08:40:22.440 | INFO | 2688, | 20260506 08:40:22.440 | INFO | 1, | 20260506 08:40:22.440 | INFO | 64 | 20260506 08:40:22.440 | INFO | ], | 20260506 08:40:22.440 | INFO | [ | 20260506 08:40:22.440 | INFO | 2688, | 20260506 08:40:22.440 | INFO | 2688, | 20260506 08:40:22.440 | INFO | 1, | 20260506 08:40:22.440 | INFO | 64 | 20260506 08:40:22.440 | INFO | ], | 20260506 08:40:22.440 | INFO | [ | 20260506 08:40:22.440 | INFO | 2944, | 20260506 08:40:22.440 | INFO | 2688, | 20260506 08:40:22.440 | INFO | 1, | 20260506 08:40:22.440 | INFO | 64 | 20260506 08:40:22.440 | INFO | ], | 20260506 08:40:22.440 | INFO | [ | 20260506 08:40:22.440 | INFO | 3200, | 20260506 08:40:22.440 | INFO | 2688, | 20260506 08:40:22.440 | INFO | 1, | 20260506 08:40:22.440 | INFO | 64 | 20260506 08:40:22.440 | INFO | ], | 20260506 08:40:22.440 | INFO | [ | 20260506 08:40:22.440 | INFO | 3456, | 20260506 08:40:22.440 | INFO | 2688, | 20260506 08:40:22.440 | INFO | 1, | 20260506 08:40:22.440 | INFO | 64 | 20260506 08:40:22.440 | INFO | ], | 20260506 08:40:22.440 | INFO | [ | 20260506 08:40:22.440 | INFO | 3712, | 20260506 08:40:22.440 | INFO | 2688, | 20260506 08:40:22.440 | INFO | 1, | 20260506 08:40:22.440 | INFO | 64 | 20260506 08:40:22.440 | INFO | ], | 20260506 08:40:22.440 | INFO | [ | 20260506 08:40:22.440 | INFO | 3968, | 20260506 08:40:22.440 | INFO | 2688, | 20260506 08:40:22.440 | INFO | 1, | 20260506 08:40:22.440 | INFO | 64 | 20260506 08:40:22.440 | INFO | ], | 20260506 08:40:22.440 | INFO | [ | 20260506 08:40:22.440 | INFO | 128, | 20260506 08:40:22.440 | INFO | 2944, | 20260506 08:40:22.440 | INFO | 1, | 20260506 08:40:22.440 | INFO | 64 | 20260506 08:40:22.440 | INFO | ], | 20260506 08:40:22.440 | INFO | [ | 20260506 08:40:22.441 | INFO | 384, | 20260506 08:40:22.441 | INFO | 2944, | 20260506 08:40:22.441 | INFO | 1, | 20260506 08:40:22.441 | INFO | 64 | 20260506 08:40:22.441 | INFO | ], | 20260506 08:40:22.441 | INFO | [ | 20260506 08:40:22.441 | INFO | 640, | 20260506 08:40:22.441 | INFO | 2944, | 20260506 08:40:22.441 | INFO | 1, | 20260506 08:40:22.441 | INFO | 64 | 20260506 08:40:22.441 | INFO | ], | 20260506 08:40:22.441 | INFO | [ | 20260506 08:40:22.441 | INFO | 896, | 20260506 08:40:22.441 | INFO | 2944, | 20260506 08:40:22.441 | INFO | 1, | 20260506 08:40:22.441 | INFO | 64 | 20260506 08:40:22.441 | INFO | ], | 20260506 08:40:22.441 | INFO | [ | 20260506 08:40:22.441 | INFO | 1152, | 20260506 08:40:22.441 | INFO | 2944, | 20260506 08:40:22.441 | INFO | 1, | 20260506 08:40:22.441 | INFO | 64 | 20260506 08:40:22.441 | INFO | ], | 20260506 08:40:22.441 | INFO | [ | 20260506 08:40:22.441 | INFO | 1408, | 20260506 08:40:22.441 | INFO | 2944, | 20260506 08:40:22.441 | INFO | 1, | 20260506 08:40:22.441 | INFO | 64 | 20260506 08:40:22.441 | INFO | ], | 20260506 08:40:22.441 | INFO | [ | 20260506 08:40:22.441 | INFO | 1664, | 20260506 08:40:22.441 | INFO | 2944, | 20260506 08:40:22.441 | INFO | 1, | 20260506 08:40:22.441 | INFO | 64 | 20260506 08:40:22.441 | INFO | ], | 20260506 08:40:22.441 | INFO | [ | 20260506 08:40:22.441 | INFO | 1920, | 20260506 08:40:22.441 | INFO | 2944, | 20260506 08:40:22.441 | INFO | 1, | 20260506 08:40:22.441 | INFO | 64 | 20260506 08:40:22.441 | INFO | ], | 20260506 08:40:22.441 | INFO | [ | 20260506 08:40:22.441 | INFO | 2176, | 20260506 08:40:22.441 | INFO | 2944, | 20260506 08:40:22.441 | INFO | 1, | 20260506 08:40:22.441 | INFO | 64 | 20260506 08:40:22.441 | INFO | ], | 20260506 08:40:22.441 | INFO | [ | 20260506 08:40:22.441 | INFO | 2432, | 20260506 08:40:22.441 | INFO | 2944, | 20260506 08:40:22.441 | INFO | 1, | 20260506 08:40:22.441 | INFO | 64 | 20260506 08:40:22.441 | INFO | ], | 20260506 08:40:22.441 | INFO | [ | 20260506 08:40:22.441 | INFO | 2688, | 20260506 08:40:22.441 | INFO | 2944, | 20260506 08:40:22.441 | INFO | 1, | 20260506 08:40:22.441 | INFO | 64 | 20260506 08:40:22.441 | INFO | ], | 20260506 08:40:22.441 | INFO | [ | 20260506 08:40:22.441 | INFO | 2944, | 20260506 08:40:22.441 | INFO | 2944, | 20260506 08:40:22.441 | INFO | 1, | 20260506 08:40:22.441 | INFO | 64 | 20260506 08:40:22.441 | INFO | ], | 20260506 08:40:22.441 | INFO | [ | 20260506 08:40:22.441 | INFO | 3200, | 20260506 08:40:22.441 | INFO | 2944, | 20260506 08:40:22.441 | INFO | 1, | 20260506 08:40:22.441 | INFO | 64 | 20260506 08:40:22.441 | INFO | ], | 20260506 08:40:22.441 | INFO | [ | 20260506 08:40:22.441 | INFO | 3456, | 20260506 08:40:22.441 | INFO | 2944, | 20260506 08:40:22.441 | INFO | 1, | 20260506 08:40:22.441 | INFO | 64 | 20260506 08:40:22.441 | INFO | ], | 20260506 08:40:22.441 | INFO | [ | 20260506 08:40:22.441 | INFO | 3712, | 20260506 08:40:22.441 | INFO | 2944, | 20260506 08:40:22.441 | INFO | 1, | 20260506 08:40:22.441 | INFO | 64 | 20260506 08:40:22.441 | INFO | ], | 20260506 08:40:22.441 | INFO | [ | 20260506 08:40:22.441 | INFO | 3968, | 20260506 08:40:22.441 | INFO | 2944, | 20260506 08:40:22.441 | INFO | 1, | 20260506 08:40:22.441 | INFO | 64 | 20260506 08:40:22.441 | INFO | ], | 20260506 08:40:22.441 | INFO | [ | 20260506 08:40:22.441 | INFO | 128, | 20260506 08:40:22.441 | INFO | 3200, | 20260506 08:40:22.441 | INFO | 1, | 20260506 08:40:22.441 | INFO | 64 | 20260506 08:40:22.441 | INFO | ], | 20260506 08:40:22.442 | INFO | [ | 20260506 08:40:22.442 | INFO | 384, | 20260506 08:40:22.442 | INFO | 3200, | 20260506 08:40:22.442 | INFO | 1, | 20260506 08:40:22.442 | INFO | 64 | 20260506 08:40:22.442 | INFO | ], | 20260506 08:40:22.442 | INFO | [ | 20260506 08:40:22.442 | INFO | 640, | 20260506 08:40:22.442 | INFO | 3200, | 20260506 08:40:22.442 | INFO | 1, | 20260506 08:40:22.442 | INFO | 64 | 20260506 08:40:22.442 | INFO | ], | 20260506 08:40:22.442 | INFO | [ | 20260506 08:40:22.442 | INFO | 896, | 20260506 08:40:22.442 | INFO | 3200, | 20260506 08:40:22.442 | INFO | 1, | 20260506 08:40:22.442 | INFO | 64 | 20260506 08:40:22.442 | INFO | ], | 20260506 08:40:22.442 | INFO | [ | 20260506 08:40:22.442 | INFO | 1152, | 20260506 08:40:22.442 | INFO | 3200, | 20260506 08:40:22.442 | INFO | 1, | 20260506 08:40:22.442 | INFO | 64 | 20260506 08:40:22.442 | INFO | ], | 20260506 08:40:22.442 | INFO | [ | 20260506 08:40:22.442 | INFO | 1408, | 20260506 08:40:22.442 | INFO | 3200, | 20260506 08:40:22.442 | INFO | 1, | 20260506 08:40:22.442 | INFO | 64 | 20260506 08:40:22.442 | INFO | ], | 20260506 08:40:22.442 | INFO | [ | 20260506 08:40:22.442 | INFO | 1664, | 20260506 08:40:22.442 | INFO | 3200, | 20260506 08:40:22.442 | INFO | 1, | 20260506 08:40:22.442 | INFO | 64 | 20260506 08:40:22.442 | INFO | ], | 20260506 08:40:22.442 | INFO | [ | 20260506 08:40:22.442 | INFO | 1920, | 20260506 08:40:22.442 | INFO | 3200, | 20260506 08:40:22.442 | INFO | 1, | 20260506 08:40:22.442 | INFO | 64 | 20260506 08:40:22.442 | INFO | ], | 20260506 08:40:22.442 | INFO | [ | 20260506 08:40:22.442 | INFO | 2176, | 20260506 08:40:22.442 | INFO | 3200, | 20260506 08:40:22.442 | INFO | 1, | 20260506 08:40:22.442 | INFO | 64 | 20260506 08:40:22.442 | INFO | ], | 20260506 08:40:22.442 | INFO | [ | 20260506 08:40:22.442 | INFO | 2432, | 20260506 08:40:22.442 | INFO | 3200, | 20260506 08:40:22.442 | INFO | 1, | 20260506 08:40:22.442 | INFO | 64 | 20260506 08:40:22.442 | INFO | ], | 20260506 08:40:22.442 | INFO | [ | 20260506 08:40:22.442 | INFO | 2688, | 20260506 08:40:22.442 | INFO | 3200, | 20260506 08:40:22.442 | INFO | 1, | 20260506 08:40:22.442 | INFO | 64 | 20260506 08:40:22.442 | INFO | ], | 20260506 08:40:22.442 | INFO | [ | 20260506 08:40:22.442 | INFO | 2944, | 20260506 08:40:22.442 | INFO | 3200, | 20260506 08:40:22.442 | INFO | 1, | 20260506 08:40:22.442 | INFO | 64 | 20260506 08:40:22.442 | INFO | ], | 20260506 08:40:22.442 | INFO | [ | 20260506 08:40:22.442 | INFO | 3200, | 20260506 08:40:22.442 | INFO | 3200, | 20260506 08:40:22.442 | INFO | 1, | 20260506 08:40:22.442 | INFO | 64 | 20260506 08:40:22.442 | INFO | ], | 20260506 08:40:22.442 | INFO | [ | 20260506 08:40:22.442 | INFO | 3456, | 20260506 08:40:22.442 | INFO | 3200, | 20260506 08:40:22.442 | INFO | 1, | 20260506 08:40:22.442 | INFO | 64 | 20260506 08:40:22.442 | INFO | ], | 20260506 08:40:22.442 | INFO | [ | 20260506 08:40:22.442 | INFO | 3712, | 20260506 08:40:22.442 | INFO | 3200, | 20260506 08:40:22.442 | INFO | 1, | 20260506 08:40:22.442 | INFO | 64 | 20260506 08:40:22.442 | INFO | ], | 20260506 08:40:22.442 | INFO | [ | 20260506 08:40:22.442 | INFO | 3968, | 20260506 08:40:22.442 | INFO | 3200, | 20260506 08:40:22.442 | INFO | 1, | 20260506 08:40:22.442 | INFO | 64 | 20260506 08:40:22.442 | INFO | ], | 20260506 08:40:22.442 | INFO | [ | 20260506 08:40:22.442 | INFO | 128, | 20260506 08:40:22.442 | INFO | 3456, | 20260506 08:40:22.442 | INFO | 1, | 20260506 08:40:22.442 | INFO | 64 | 20260506 08:40:22.443 | INFO | ], | 20260506 08:40:22.443 | INFO | [ | 20260506 08:40:22.443 | INFO | 384, | 20260506 08:40:22.443 | INFO | 3456, | 20260506 08:40:22.443 | INFO | 1, | 20260506 08:40:22.443 | INFO | 64 | 20260506 08:40:22.443 | INFO | ], | 20260506 08:40:22.443 | INFO | [ | 20260506 08:40:22.443 | INFO | 640, | 20260506 08:40:22.443 | INFO | 3456, | 20260506 08:40:22.443 | INFO | 1, | 20260506 08:40:22.443 | INFO | 64 | 20260506 08:40:22.443 | INFO | ], | 20260506 08:40:22.443 | INFO | [ | 20260506 08:40:22.443 | INFO | 896, | 20260506 08:40:22.443 | INFO | 3456, | 20260506 08:40:22.443 | INFO | 1, | 20260506 08:40:22.443 | INFO | 64 | 20260506 08:40:22.443 | INFO | ], | 20260506 08:40:22.443 | INFO | [ | 20260506 08:40:22.443 | INFO | 1152, | 20260506 08:40:22.443 | INFO | 3456, | 20260506 08:40:22.443 | INFO | 1, | 20260506 08:40:22.443 | INFO | 64 | 20260506 08:40:22.443 | INFO | ], | 20260506 08:40:22.443 | INFO | [ | 20260506 08:40:22.443 | INFO | 1408, | 20260506 08:40:22.443 | INFO | 3456, | 20260506 08:40:22.443 | INFO | 1, | 20260506 08:40:22.443 | INFO | 64 | 20260506 08:40:22.443 | INFO | ], | 20260506 08:40:22.443 | INFO | [ | 20260506 08:40:22.443 | INFO | 1664, | 20260506 08:40:22.443 | INFO | 3456, | 20260506 08:40:22.443 | INFO | 1, | 20260506 08:40:22.443 | INFO | 64 | 20260506 08:40:22.443 | INFO | ], | 20260506 08:40:22.443 | INFO | [ | 20260506 08:40:22.443 | INFO | 1920, | 20260506 08:40:22.443 | INFO | 3456, | 20260506 08:40:22.443 | INFO | 1, | 20260506 08:40:22.443 | INFO | 64 | 20260506 08:40:22.443 | INFO | ], | 20260506 08:40:22.443 | INFO | [ | 20260506 08:40:22.443 | INFO | 2176, | 20260506 08:40:22.443 | INFO | 3456, | 20260506 08:40:22.443 | INFO | 1, | 20260506 08:40:22.443 | INFO | 64 | 20260506 08:40:22.443 | INFO | ], | 20260506 08:40:22.443 | INFO | [ | 20260506 08:40:22.443 | INFO | 2432, | 20260506 08:40:22.443 | INFO | 3456, | 20260506 08:40:22.443 | INFO | 1, | 20260506 08:40:22.443 | INFO | 64 | 20260506 08:40:22.443 | INFO | ], | 20260506 08:40:22.443 | INFO | [ | 20260506 08:40:22.443 | INFO | 2688, | 20260506 08:40:22.443 | INFO | 3456, | 20260506 08:40:22.443 | INFO | 1, | 20260506 08:40:22.443 | INFO | 64 | 20260506 08:40:22.443 | INFO | ], | 20260506 08:40:22.443 | INFO | [ | 20260506 08:40:22.443 | INFO | 2944, | 20260506 08:40:22.443 | INFO | 3456, | 20260506 08:40:22.443 | INFO | 1, | 20260506 08:40:22.443 | INFO | 64 | 20260506 08:40:22.443 | INFO | ], | 20260506 08:40:22.443 | INFO | [ | 20260506 08:40:22.443 | INFO | 3200, | 20260506 08:40:22.443 | INFO | 3456, | 20260506 08:40:22.443 | INFO | 1, | 20260506 08:40:22.443 | INFO | 64 | 20260506 08:40:22.443 | INFO | ], | 20260506 08:40:22.443 | INFO | [ | 20260506 08:40:22.443 | INFO | 3456, | 20260506 08:40:22.443 | INFO | 3456, | 20260506 08:40:22.443 | INFO | 1, | 20260506 08:40:22.443 | INFO | 64 | 20260506 08:40:22.443 | INFO | ], | 20260506 08:40:22.443 | INFO | [ | 20260506 08:40:22.443 | INFO | 3712, | 20260506 08:40:22.443 | INFO | 3456, | 20260506 08:40:22.443 | INFO | 1, | 20260506 08:40:22.443 | INFO | 64 | 20260506 08:40:22.443 | INFO | ], | 20260506 08:40:22.443 | INFO | [ | 20260506 08:40:22.443 | INFO | 3968, | 20260506 08:40:22.443 | INFO | 3456, | 20260506 08:40:22.443 | INFO | 1, | 20260506 08:40:22.443 | INFO | 64 | 20260506 08:40:22.443 | INFO | ], | 20260506 08:40:22.443 | INFO | [ | 20260506 08:40:22.443 | INFO | 128, | 20260506 08:40:22.443 | INFO | 3712, | 20260506 08:40:22.444 | INFO | 1, | 20260506 08:40:22.444 | INFO | 64 | 20260506 08:40:22.444 | INFO | ], | 20260506 08:40:22.444 | INFO | [ | 20260506 08:40:22.444 | INFO | 384, | 20260506 08:40:22.444 | INFO | 3712, | 20260506 08:40:22.444 | INFO | 1, | 20260506 08:40:22.444 | INFO | 64 | 20260506 08:40:22.444 | INFO | ], | 20260506 08:40:22.444 | INFO | [ | 20260506 08:40:22.444 | INFO | 640, | 20260506 08:40:22.444 | INFO | 3712, | 20260506 08:40:22.444 | INFO | 1, | 20260506 08:40:22.444 | INFO | 64 | 20260506 08:40:22.444 | INFO | ], | 20260506 08:40:22.444 | INFO | [ | 20260506 08:40:22.444 | INFO | 896, | 20260506 08:40:22.444 | INFO | 3712, | 20260506 08:40:22.444 | INFO | 1, | 20260506 08:40:22.444 | INFO | 64 | 20260506 08:40:22.444 | INFO | ], | 20260506 08:40:22.444 | INFO | [ | 20260506 08:40:22.444 | INFO | 1152, | 20260506 08:40:22.444 | INFO | 3712, | 20260506 08:40:22.444 | INFO | 1, | 20260506 08:40:22.444 | INFO | 64 | 20260506 08:40:22.444 | INFO | ], | 20260506 08:40:22.444 | INFO | [ | 20260506 08:40:22.444 | INFO | 1408, | 20260506 08:40:22.444 | INFO | 3712, | 20260506 08:40:22.444 | INFO | 1, | 20260506 08:40:22.444 | INFO | 64 | 20260506 08:40:22.444 | INFO | ], | 20260506 08:40:22.444 | INFO | [ | 20260506 08:40:22.444 | INFO | 1664, | 20260506 08:40:22.444 | INFO | 3712, | 20260506 08:40:22.444 | INFO | 1, | 20260506 08:40:22.444 | INFO | 64 | 20260506 08:40:22.444 | INFO | ], | 20260506 08:40:22.444 | INFO | [ | 20260506 08:40:22.444 | INFO | 1920, | 20260506 08:40:22.444 | INFO | 3712, | 20260506 08:40:22.444 | INFO | 1, | 20260506 08:40:22.444 | INFO | 64 | 20260506 08:40:22.444 | INFO | ], | 20260506 08:40:22.444 | INFO | [ | 20260506 08:40:22.444 | INFO | 2176, | 20260506 08:40:22.444 | INFO | 3712, | 20260506 08:40:22.444 | INFO | 1, | 20260506 08:40:22.444 | INFO | 64 | 20260506 08:40:22.444 | INFO | ], | 20260506 08:40:22.444 | INFO | [ | 20260506 08:40:22.444 | INFO | 2432, | 20260506 08:40:22.444 | INFO | 3712, | 20260506 08:40:22.444 | INFO | 1, | 20260506 08:40:22.444 | INFO | 64 | 20260506 08:40:22.444 | INFO | ], | 20260506 08:40:22.444 | INFO | [ | 20260506 08:40:22.444 | INFO | 2688, | 20260506 08:40:22.444 | INFO | 3712, | 20260506 08:40:22.444 | INFO | 1, | 20260506 08:40:22.444 | INFO | 64 | 20260506 08:40:22.444 | INFO | ], | 20260506 08:40:22.444 | INFO | [ | 20260506 08:40:22.444 | INFO | 2944, | 20260506 08:40:22.444 | INFO | 3712, | 20260506 08:40:22.444 | INFO | 1, | 20260506 08:40:22.444 | INFO | 64 | 20260506 08:40:22.444 | INFO | ], | 20260506 08:40:22.444 | INFO | [ | 20260506 08:40:22.444 | INFO | 3200, | 20260506 08:40:22.444 | INFO | 3712, | 20260506 08:40:22.444 | INFO | 1, | 20260506 08:40:22.444 | INFO | 64 | 20260506 08:40:22.444 | INFO | ], | 20260506 08:40:22.444 | INFO | [ | 20260506 08:40:22.444 | INFO | 3456, | 20260506 08:40:22.444 | INFO | 3712, | 20260506 08:40:22.444 | INFO | 1, | 20260506 08:40:22.444 | INFO | 64 | 20260506 08:40:22.444 | INFO | ], | 20260506 08:40:22.444 | INFO | [ | 20260506 08:40:22.444 | INFO | 3712, | 20260506 08:40:22.444 | INFO | 3712, | 20260506 08:40:22.444 | INFO | 1, | 20260506 08:40:22.444 | INFO | 64 | 20260506 08:40:22.444 | INFO | ], | 20260506 08:40:22.444 | INFO | [ | 20260506 08:40:22.444 | INFO | 3968, | 20260506 08:40:22.444 | INFO | 3712, | 20260506 08:40:22.444 | INFO | 1, | 20260506 08:40:22.444 | INFO | 64 | 20260506 08:40:22.444 | INFO | ], | 20260506 08:40:22.445 | INFO | [ | 20260506 08:40:22.445 | INFO | 128, | 20260506 08:40:22.445 | INFO | 3968, | 20260506 08:40:22.445 | INFO | 1, | 20260506 08:40:22.445 | INFO | 64 | 20260506 08:40:22.445 | INFO | ], | 20260506 08:40:22.445 | INFO | [ | 20260506 08:40:22.445 | INFO | 384, | 20260506 08:40:22.445 | INFO | 3968, | 20260506 08:40:22.445 | INFO | 1, | 20260506 08:40:22.445 | INFO | 64 | 20260506 08:40:22.445 | INFO | ], | 20260506 08:40:22.445 | INFO | [ | 20260506 08:40:22.445 | INFO | 640, | 20260506 08:40:22.445 | INFO | 3968, | 20260506 08:40:22.445 | INFO | 1, | 20260506 08:40:22.445 | INFO | 64 | 20260506 08:40:22.445 | INFO | ], | 20260506 08:40:22.445 | INFO | [ | 20260506 08:40:22.445 | INFO | 896, | 20260506 08:40:22.445 | INFO | 3968, | 20260506 08:40:22.445 | INFO | 1, | 20260506 08:40:22.445 | INFO | 64 | 20260506 08:40:22.445 | INFO | ], | 20260506 08:40:22.445 | INFO | [ | 20260506 08:40:22.445 | INFO | 1152, | 20260506 08:40:22.445 | INFO | 3968, | 20260506 08:40:22.445 | INFO | 1, | 20260506 08:40:22.445 | INFO | 64 | 20260506 08:40:22.445 | INFO | ], | 20260506 08:40:22.445 | INFO | [ | 20260506 08:40:22.445 | INFO | 1408, | 20260506 08:40:22.445 | INFO | 3968, | 20260506 08:40:22.445 | INFO | 1, | 20260506 08:40:22.445 | INFO | 64 | 20260506 08:40:22.445 | INFO | ], | 20260506 08:40:22.445 | INFO | [ | 20260506 08:40:22.445 | INFO | 1664, | 20260506 08:40:22.445 | INFO | 3968, | 20260506 08:40:22.445 | INFO | 1, | 20260506 08:40:22.445 | INFO | 64 | 20260506 08:40:22.445 | INFO | ], | 20260506 08:40:22.445 | INFO | [ | 20260506 08:40:22.445 | INFO | 1920, | 20260506 08:40:22.445 | INFO | 3968, | 20260506 08:40:22.445 | INFO | 1, | 20260506 08:40:22.445 | INFO | 64 | 20260506 08:40:22.445 | INFO | ], | 20260506 08:40:22.445 | INFO | [ | 20260506 08:40:22.445 | INFO | 2176, | 20260506 08:40:22.445 | INFO | 3968, | 20260506 08:40:22.445 | INFO | 1, | 20260506 08:40:22.445 | INFO | 64 | 20260506 08:40:22.445 | INFO | ], | 20260506 08:40:22.445 | INFO | [ | 20260506 08:40:22.445 | INFO | 2432, | 20260506 08:40:22.445 | INFO | 3968, | 20260506 08:40:22.445 | INFO | 1, | 20260506 08:40:22.445 | INFO | 64 | 20260506 08:40:22.445 | INFO | ], | 20260506 08:40:22.445 | INFO | [ | 20260506 08:40:22.445 | INFO | 2688, | 20260506 08:40:22.445 | INFO | 3968, | 20260506 08:40:22.445 | INFO | 1, | 20260506 08:40:22.445 | INFO | 64 | 20260506 08:40:22.445 | INFO | ], | 20260506 08:40:22.445 | INFO | [ | 20260506 08:40:22.445 | INFO | 2944, | 20260506 08:40:22.445 | INFO | 3968, | 20260506 08:40:22.445 | INFO | 1, | 20260506 08:40:22.445 | INFO | 64 | 20260506 08:40:22.445 | INFO | ], | 20260506 08:40:22.445 | INFO | [ | 20260506 08:40:22.445 | INFO | 3200, | 20260506 08:40:22.445 | INFO | 3968, | 20260506 08:40:22.445 | INFO | 1, | 20260506 08:40:22.445 | INFO | 64 | 20260506 08:40:22.445 | INFO | ], | 20260506 08:40:22.445 | INFO | [ | 20260506 08:40:22.445 | INFO | 3456, | 20260506 08:40:22.445 | INFO | 3968, | 20260506 08:40:22.445 | INFO | 1, | 20260506 08:40:22.445 | INFO | 64 | 20260506 08:40:22.445 | INFO | ], | 20260506 08:40:22.445 | INFO | [ | 20260506 08:40:22.445 | INFO | 3712, | 20260506 08:40:22.445 | INFO | 3968, | 20260506 08:40:22.445 | INFO | 1, | 20260506 08:40:22.445 | INFO | 64 | 20260506 08:40:22.445 | INFO | ], | 20260506 08:40:22.445 | INFO | [ | 20260506 08:40:22.445 | INFO | 3968, | 20260506 08:40:22.445 | INFO | 3968, | 20260506 08:40:22.445 | INFO | 1, | 20260506 08:40:22.446 | INFO | 64 | 20260506 08:40:22.446 | INFO | ], | 20260506 08:40:22.446 | INFO | [ | 20260506 08:40:22.446 | INFO | 128, | 20260506 08:40:22.446 | INFO | 4224, | 20260506 08:40:22.446 | INFO | 1, | 20260506 08:40:22.446 | INFO | 64 | 20260506 08:40:22.446 | INFO | ], | 20260506 08:40:22.446 | INFO | [ | 20260506 08:40:22.446 | INFO | 384, | 20260506 08:40:22.446 | INFO | 4224, | 20260506 08:40:22.446 | INFO | 1, | 20260506 08:40:22.446 | INFO | 64 | 20260506 08:40:22.446 | INFO | ], | 20260506 08:40:22.446 | INFO | [ | 20260506 08:40:22.446 | INFO | 640, | 20260506 08:40:22.446 | INFO | 4224, | 20260506 08:40:22.446 | INFO | 1, | 20260506 08:40:22.446 | INFO | 64 | 20260506 08:40:22.446 | INFO | ], | 20260506 08:40:22.446 | INFO | [ | 20260506 08:40:22.446 | INFO | 896, | 20260506 08:40:22.446 | INFO | 4224, | 20260506 08:40:22.446 | INFO | 1, | 20260506 08:40:22.446 | INFO | 64 | 20260506 08:40:22.446 | INFO | ], | 20260506 08:40:22.446 | INFO | [ | 20260506 08:40:22.446 | INFO | 1152, | 20260506 08:40:22.446 | INFO | 4224, | 20260506 08:40:22.446 | INFO | 1, | 20260506 08:40:22.446 | INFO | 64 | 20260506 08:40:22.446 | INFO | ], | 20260506 08:40:22.446 | INFO | [ | 20260506 08:40:22.446 | INFO | 1408, | 20260506 08:40:22.446 | INFO | 4224, | 20260506 08:40:22.446 | INFO | 1, | 20260506 08:40:22.446 | INFO | 64 | 20260506 08:40:22.446 | INFO | ], | 20260506 08:40:22.446 | INFO | [ | 20260506 08:40:22.446 | INFO | 1664, | 20260506 08:40:22.446 | INFO | 4224, | 20260506 08:40:22.446 | INFO | 1, | 20260506 08:40:22.446 | INFO | 64 | 20260506 08:40:22.446 | INFO | ], | 20260506 08:40:22.446 | INFO | [ | 20260506 08:40:22.446 | INFO | 1920, | 20260506 08:40:22.446 | INFO | 4224, | 20260506 08:40:22.446 | INFO | 1, | 20260506 08:40:22.446 | INFO | 64 | 20260506 08:40:22.446 | INFO | ], | 20260506 08:40:22.446 | INFO | [ | 20260506 08:40:22.446 | INFO | 2176, | 20260506 08:40:22.446 | INFO | 4224, | 20260506 08:40:22.446 | INFO | 1, | 20260506 08:40:22.446 | INFO | 64 | 20260506 08:40:22.446 | INFO | ], | 20260506 08:40:22.446 | INFO | [ | 20260506 08:40:22.446 | INFO | 2432, | 20260506 08:40:22.446 | INFO | 4224, | 20260506 08:40:22.446 | INFO | 1, | 20260506 08:40:22.446 | INFO | 64 | 20260506 08:40:22.446 | INFO | ], | 20260506 08:40:22.446 | INFO | [ | 20260506 08:40:22.446 | INFO | 2688, | 20260506 08:40:22.446 | INFO | 4224, | 20260506 08:40:22.446 | INFO | 1, | 20260506 08:40:22.446 | INFO | 64 | 20260506 08:40:22.446 | INFO | ], | 20260506 08:40:22.446 | INFO | [ | 20260506 08:40:22.446 | INFO | 2944, | 20260506 08:40:22.446 | INFO | 4224, | 20260506 08:40:22.446 | INFO | 1, | 20260506 08:40:22.446 | INFO | 64 | 20260506 08:40:22.446 | INFO | ], | 20260506 08:40:22.446 | INFO | [ | 20260506 08:40:22.446 | INFO | 3200, | 20260506 08:40:22.446 | INFO | 4224, | 20260506 08:40:22.446 | INFO | 1, | 20260506 08:40:22.446 | INFO | 64 | 20260506 08:40:22.446 | INFO | ], | 20260506 08:40:22.446 | INFO | [ | 20260506 08:40:22.446 | INFO | 3456, | 20260506 08:40:22.446 | INFO | 4224, | 20260506 08:40:22.446 | INFO | 1, | 20260506 08:40:22.446 | INFO | 64 | 20260506 08:40:22.446 | INFO | ], | 20260506 08:40:22.446 | INFO | [ | 20260506 08:40:22.446 | INFO | 3712, | 20260506 08:40:22.446 | INFO | 4224, | 20260506 08:40:22.446 | INFO | 1, | 20260506 08:40:22.446 | INFO | 64 | 20260506 08:40:22.446 | INFO | ], | 20260506 08:40:22.446 | INFO | [ | 20260506 08:40:22.446 | INFO | 3968, | 20260506 08:40:22.446 | INFO | 4224, | 20260506 08:40:22.447 | INFO | 1, | 20260506 08:40:22.447 | INFO | 64 | 20260506 08:40:22.447 | INFO | ], | 20260506 08:40:22.447 | INFO | [ | 20260506 08:40:22.447 | INFO | 128, | 20260506 08:40:22.447 | INFO | 4480, | 20260506 08:40:22.447 | INFO | 1, | 20260506 08:40:22.447 | INFO | 64 | 20260506 08:40:22.447 | INFO | ], | 20260506 08:40:22.447 | INFO | [ | 20260506 08:40:22.447 | INFO | 384, | 20260506 08:40:22.447 | INFO | 4480, | 20260506 08:40:22.447 | INFO | 1, | 20260506 08:40:22.447 | INFO | 64 | 20260506 08:40:22.447 | INFO | ], | 20260506 08:40:22.447 | INFO | [ | 20260506 08:40:22.447 | INFO | 640, | 20260506 08:40:22.447 | INFO | 4480, | 20260506 08:40:22.447 | INFO | 1, | 20260506 08:40:22.447 | INFO | 64 | 20260506 08:40:22.447 | INFO | ], | 20260506 08:40:22.447 | INFO | [ | 20260506 08:40:22.447 | INFO | 896, | 20260506 08:40:22.447 | INFO | 4480, | 20260506 08:40:22.447 | INFO | 1, | 20260506 08:40:22.447 | INFO | 64 | 20260506 08:40:22.447 | INFO | ], | 20260506 08:40:22.447 | INFO | [ | 20260506 08:40:22.447 | INFO | 1152, | 20260506 08:40:22.447 | INFO | 4480, | 20260506 08:40:22.447 | INFO | 1, | 20260506 08:40:22.447 | INFO | 64 | 20260506 08:40:22.447 | INFO | ], | 20260506 08:40:22.447 | INFO | [ | 20260506 08:40:22.447 | INFO | 1408, | 20260506 08:40:22.447 | INFO | 4480, | 20260506 08:40:22.447 | INFO | 1, | 20260506 08:40:22.447 | INFO | 64 | 20260506 08:40:22.447 | INFO | ], | 20260506 08:40:22.447 | INFO | [ | 20260506 08:40:22.447 | INFO | 1664, | 20260506 08:40:22.447 | INFO | 4480, | 20260506 08:40:22.447 | INFO | 1, | 20260506 08:40:22.447 | INFO | 64 | 20260506 08:40:22.447 | INFO | ], | 20260506 08:40:22.447 | INFO | [ | 20260506 08:40:22.447 | INFO | 1920, | 20260506 08:40:22.447 | INFO | 4480, | 20260506 08:40:22.447 | INFO | 1, | 20260506 08:40:22.447 | INFO | 64 | 20260506 08:40:22.447 | INFO | ], | 20260506 08:40:22.447 | INFO | [ | 20260506 08:40:22.447 | INFO | 2176, | 20260506 08:40:22.447 | INFO | 4480, | 20260506 08:40:22.447 | INFO | 1, | 20260506 08:40:22.447 | INFO | 64 | 20260506 08:40:22.447 | INFO | ], | 20260506 08:40:22.447 | INFO | [ | 20260506 08:40:22.447 | INFO | 2432, | 20260506 08:40:22.447 | INFO | 4480, | 20260506 08:40:22.447 | INFO | 1, | 20260506 08:40:22.447 | INFO | 64 | 20260506 08:40:22.447 | INFO | ], | 20260506 08:40:22.447 | INFO | [ | 20260506 08:40:22.447 | INFO | 2688, | 20260506 08:40:22.447 | INFO | 4480, | 20260506 08:40:22.447 | INFO | 1, | 20260506 08:40:22.447 | INFO | 64 | 20260506 08:40:22.447 | INFO | ], | 20260506 08:40:22.447 | INFO | [ | 20260506 08:40:22.447 | INFO | 2944, | 20260506 08:40:22.447 | INFO | 4480, | 20260506 08:40:22.447 | INFO | 1, | 20260506 08:40:22.447 | INFO | 64 | 20260506 08:40:22.447 | INFO | ], | 20260506 08:40:22.447 | INFO | [ | 20260506 08:40:22.447 | INFO | 3200, | 20260506 08:40:22.447 | INFO | 4480, | 20260506 08:40:22.447 | INFO | 1, | 20260506 08:40:22.447 | INFO | 64 | 20260506 08:40:22.447 | INFO | ], | 20260506 08:40:22.447 | INFO | [ | 20260506 08:40:22.447 | INFO | 3456, | 20260506 08:40:22.447 | INFO | 4480, | 20260506 08:40:22.447 | INFO | 1, | 20260506 08:40:22.447 | INFO | 64 | 20260506 08:40:22.447 | INFO | ], | 20260506 08:40:22.447 | INFO | [ | 20260506 08:40:22.447 | INFO | 3712, | 20260506 08:40:22.447 | INFO | 4480, | 20260506 08:40:22.447 | INFO | 1, | 20260506 08:40:22.447 | INFO | 64 | 20260506 08:40:22.447 | INFO | ], | 20260506 08:40:22.447 | INFO | [ | 20260506 08:40:22.447 | INFO | 3968, | 20260506 08:40:22.448 | INFO | 4480, | 20260506 08:40:22.448 | INFO | 1, | 20260506 08:40:22.448 | INFO | 64 | 20260506 08:40:22.448 | INFO | ], | 20260506 08:40:22.448 | INFO | [ | 20260506 08:40:22.448 | INFO | 128, | 20260506 08:40:22.448 | INFO | 4736, | 20260506 08:40:22.448 | INFO | 1, | 20260506 08:40:22.448 | INFO | 64 | 20260506 08:40:22.448 | INFO | ], | 20260506 08:40:22.448 | INFO | [ | 20260506 08:40:22.448 | INFO | 384, | 20260506 08:40:22.448 | INFO | 4736, | 20260506 08:40:22.448 | INFO | 1, | 20260506 08:40:22.448 | INFO | 64 | 20260506 08:40:22.448 | INFO | ], | 20260506 08:40:22.448 | INFO | [ | 20260506 08:40:22.448 | INFO | 640, | 20260506 08:40:22.448 | INFO | 4736, | 20260506 08:40:22.448 | INFO | 1, | 20260506 08:40:22.448 | INFO | 64 | 20260506 08:40:22.448 | INFO | ], | 20260506 08:40:22.448 | INFO | [ | 20260506 08:40:22.448 | INFO | 896, | 20260506 08:40:22.448 | INFO | 4736, | 20260506 08:40:22.448 | INFO | 1, | 20260506 08:40:22.448 | INFO | 64 | 20260506 08:40:22.448 | INFO | ], | 20260506 08:40:22.448 | INFO | [ | 20260506 08:40:22.448 | INFO | 1152, | 20260506 08:40:22.448 | INFO | 4736, | 20260506 08:40:22.448 | INFO | 1, | 20260506 08:40:22.448 | INFO | 64 | 20260506 08:40:22.448 | INFO | ], | 20260506 08:40:22.448 | INFO | [ | 20260506 08:40:22.448 | INFO | 1408, | 20260506 08:40:22.448 | INFO | 4736, | 20260506 08:40:22.448 | INFO | 1, | 20260506 08:40:22.448 | INFO | 64 | 20260506 08:40:22.448 | INFO | ], | 20260506 08:40:22.448 | INFO | [ | 20260506 08:40:22.448 | INFO | 1664, | 20260506 08:40:22.448 | INFO | 4736, | 20260506 08:40:22.448 | INFO | 1, | 20260506 08:40:22.448 | INFO | 64 | 20260506 08:40:22.448 | INFO | ], | 20260506 08:40:22.448 | INFO | [ | 20260506 08:40:22.448 | INFO | 1920, | 20260506 08:40:22.448 | INFO | 4736, | 20260506 08:40:22.448 | INFO | 1, | 20260506 08:40:22.448 | INFO | 64 | 20260506 08:40:22.448 | INFO | ], | 20260506 08:40:22.448 | INFO | [ | 20260506 08:40:22.448 | INFO | 2176, | 20260506 08:40:22.448 | INFO | 4736, | 20260506 08:40:22.448 | INFO | 1, | 20260506 08:40:22.448 | INFO | 64 | 20260506 08:40:22.448 | INFO | ], | 20260506 08:40:22.448 | INFO | [ | 20260506 08:40:22.448 | INFO | 2432, | 20260506 08:40:22.448 | INFO | 4736, | 20260506 08:40:22.448 | INFO | 1, | 20260506 08:40:22.448 | INFO | 64 | 20260506 08:40:22.448 | INFO | ], | 20260506 08:40:22.448 | INFO | [ | 20260506 08:40:22.448 | INFO | 2688, | 20260506 08:40:22.448 | INFO | 4736, | 20260506 08:40:22.448 | INFO | 1, | 20260506 08:40:22.448 | INFO | 64 | 20260506 08:40:22.448 | INFO | ], | 20260506 08:40:22.448 | INFO | [ | 20260506 08:40:22.448 | INFO | 2944, | 20260506 08:40:22.448 | INFO | 4736, | 20260506 08:40:22.448 | INFO | 1, | 20260506 08:40:22.448 | INFO | 64 | 20260506 08:40:22.448 | INFO | ], | 20260506 08:40:22.448 | INFO | [ | 20260506 08:40:22.448 | INFO | 3200, | 20260506 08:40:22.448 | INFO | 4736, | 20260506 08:40:22.448 | INFO | 1, | 20260506 08:40:22.448 | INFO | 64 | 20260506 08:40:22.448 | INFO | ], | 20260506 08:40:22.448 | INFO | [ | 20260506 08:40:22.448 | INFO | 3456, | 20260506 08:40:22.448 | INFO | 4736, | 20260506 08:40:22.448 | INFO | 1, | 20260506 08:40:22.448 | INFO | 64 | 20260506 08:40:22.448 | INFO | ], | 20260506 08:40:22.448 | INFO | [ | 20260506 08:40:22.448 | INFO | 3712, | 20260506 08:40:22.448 | INFO | 4736, | 20260506 08:40:22.448 | INFO | 1, | 20260506 08:40:22.448 | INFO | 64 | 20260506 08:40:22.449 | INFO | ], | 20260506 08:40:22.449 | INFO | [ | 20260506 08:40:22.449 | INFO | 3968, | 20260506 08:40:22.449 | INFO | 4736, | 20260506 08:40:22.449 | INFO | 1, | 20260506 08:40:22.449 | INFO | 64 | 20260506 08:40:22.449 | INFO | ], | 20260506 08:40:22.449 | INFO | [ | 20260506 08:40:22.449 | INFO | 128, | 20260506 08:40:22.449 | INFO | 4992, | 20260506 08:40:22.449 | INFO | 1, | 20260506 08:40:22.449 | INFO | 64 | 20260506 08:40:22.449 | INFO | ], | 20260506 08:40:22.449 | INFO | [ | 20260506 08:40:22.449 | INFO | 384, | 20260506 08:40:22.449 | INFO | 4992, | 20260506 08:40:22.449 | INFO | 1, | 20260506 08:40:22.449 | INFO | 64 | 20260506 08:40:22.449 | INFO | ], | 20260506 08:40:22.449 | INFO | [ | 20260506 08:40:22.449 | INFO | 640, | 20260506 08:40:22.449 | INFO | 4992, | 20260506 08:40:22.449 | INFO | 1, | 20260506 08:40:22.449 | INFO | 64 | 20260506 08:40:22.449 | INFO | ], | 20260506 08:40:22.449 | INFO | [ | 20260506 08:40:22.449 | INFO | 896, | 20260506 08:40:22.449 | INFO | 4992, | 20260506 08:40:22.449 | INFO | 1, | 20260506 08:40:22.449 | INFO | 64 | 20260506 08:40:22.449 | INFO | ], | 20260506 08:40:22.449 | INFO | [ | 20260506 08:40:22.449 | INFO | 1152, | 20260506 08:40:22.449 | INFO | 4992, | 20260506 08:40:22.449 | INFO | 1, | 20260506 08:40:22.449 | INFO | 64 | 20260506 08:40:22.449 | INFO | ], | 20260506 08:40:22.449 | INFO | [ | 20260506 08:40:22.449 | INFO | 1408, | 20260506 08:40:22.449 | INFO | 4992, | 20260506 08:40:22.449 | INFO | 1, | 20260506 08:40:22.449 | INFO | 64 | 20260506 08:40:22.449 | INFO | ], | 20260506 08:40:22.449 | INFO | [ | 20260506 08:40:22.449 | INFO | 1664, | 20260506 08:40:22.449 | INFO | 4992, | 20260506 08:40:22.449 | INFO | 1, | 20260506 08:40:22.449 | INFO | 64 | 20260506 08:40:22.449 | INFO | ], | 20260506 08:40:22.449 | INFO | [ | 20260506 08:40:22.449 | INFO | 1920, | 20260506 08:40:22.449 | INFO | 4992, | 20260506 08:40:22.449 | INFO | 1, | 20260506 08:40:22.449 | INFO | 64 | 20260506 08:40:22.449 | INFO | ], | 20260506 08:40:22.449 | INFO | [ | 20260506 08:40:22.449 | INFO | 2176, | 20260506 08:40:22.449 | INFO | 4992, | 20260506 08:40:22.449 | INFO | 1, | 20260506 08:40:22.449 | INFO | 64 | 20260506 08:40:22.449 | INFO | ], | 20260506 08:40:22.449 | INFO | [ | 20260506 08:40:22.449 | INFO | 2432, | 20260506 08:40:22.449 | INFO | 4992, | 20260506 08:40:22.449 | INFO | 1, | 20260506 08:40:22.449 | INFO | 64 | 20260506 08:40:22.449 | INFO | ], | 20260506 08:40:22.449 | INFO | [ | 20260506 08:40:22.449 | INFO | 2688, | 20260506 08:40:22.449 | INFO | 4992, | 20260506 08:40:22.449 | INFO | 1, | 20260506 08:40:22.449 | INFO | 64 | 20260506 08:40:22.449 | INFO | ], | 20260506 08:40:22.449 | INFO | [ | 20260506 08:40:22.449 | INFO | 2944, | 20260506 08:40:22.449 | INFO | 4992, | 20260506 08:40:22.449 | INFO | 1, | 20260506 08:40:22.449 | INFO | 64 | 20260506 08:40:22.449 | INFO | ], | 20260506 08:40:22.449 | INFO | [ | 20260506 08:40:22.449 | INFO | 3200, | 20260506 08:40:22.449 | INFO | 4992, | 20260506 08:40:22.449 | INFO | 1, | 20260506 08:40:22.449 | INFO | 64 | 20260506 08:40:22.449 | INFO | ], | 20260506 08:40:22.449 | INFO | [ | 20260506 08:40:22.449 | INFO | 3456, | 20260506 08:40:22.449 | INFO | 4992, | 20260506 08:40:22.449 | INFO | 1, | 20260506 08:40:22.449 | INFO | 64 | 20260506 08:40:22.449 | INFO | ], | 20260506 08:40:22.449 | INFO | [ | 20260506 08:40:22.449 | INFO | 3712, | 20260506 08:40:22.449 | INFO | 4992, | 20260506 08:40:22.450 | INFO | 1, | 20260506 08:40:22.450 | INFO | 64 | 20260506 08:40:22.450 | INFO | ], | 20260506 08:40:22.450 | INFO | [ | 20260506 08:40:22.450 | INFO | 3968, | 20260506 08:40:22.450 | INFO | 4992, | 20260506 08:40:22.450 | INFO | 1, | 20260506 08:40:22.450 | INFO | 64 | 20260506 08:40:22.450 | INFO | ], | 20260506 08:40:22.450 | INFO | [ | 20260506 08:40:22.450 | INFO | 128, | 20260506 08:40:22.450 | INFO | 5248, | 20260506 08:40:22.450 | INFO | 1, | 20260506 08:40:22.450 | INFO | 64 | 20260506 08:40:22.450 | INFO | ], | 20260506 08:40:22.450 | INFO | [ | 20260506 08:40:22.450 | INFO | 384, | 20260506 08:40:22.450 | INFO | 5248, | 20260506 08:40:22.450 | INFO | 1, | 20260506 08:40:22.450 | INFO | 64 | 20260506 08:40:22.450 | INFO | ], | 20260506 08:40:22.450 | INFO | [ | 20260506 08:40:22.450 | INFO | 640, | 20260506 08:40:22.450 | INFO | 5248, | 20260506 08:40:22.450 | INFO | 1, | 20260506 08:40:22.450 | INFO | 64 | 20260506 08:40:22.450 | INFO | ], | 20260506 08:40:22.450 | INFO | [ | 20260506 08:40:22.450 | INFO | 896, | 20260506 08:40:22.450 | INFO | 5248, | 20260506 08:40:22.450 | INFO | 1, | 20260506 08:40:22.450 | INFO | 64 | 20260506 08:40:22.450 | INFO | ], | 20260506 08:40:22.450 | INFO | [ | 20260506 08:40:22.450 | INFO | 1152, | 20260506 08:40:22.450 | INFO | 5248, | 20260506 08:40:22.450 | INFO | 1, | 20260506 08:40:22.450 | INFO | 64 | 20260506 08:40:22.450 | INFO | ], | 20260506 08:40:22.450 | INFO | [ | 20260506 08:40:22.450 | INFO | 1408, | 20260506 08:40:22.450 | INFO | 5248, | 20260506 08:40:22.450 | INFO | 1, | 20260506 08:40:22.450 | INFO | 64 | 20260506 08:40:22.450 | INFO | ], | 20260506 08:40:22.450 | INFO | [ | 20260506 08:40:22.450 | INFO | 1664, | 20260506 08:40:22.450 | INFO | 5248, | 20260506 08:40:22.450 | INFO | 1, | 20260506 08:40:22.450 | INFO | 64 | 20260506 08:40:22.450 | INFO | ], | 20260506 08:40:22.450 | INFO | [ | 20260506 08:40:22.450 | INFO | 1920, | 20260506 08:40:22.450 | INFO | 5248, | 20260506 08:40:22.450 | INFO | 1, | 20260506 08:40:22.450 | INFO | 64 | 20260506 08:40:22.450 | INFO | ], | 20260506 08:40:22.450 | INFO | [ | 20260506 08:40:22.450 | INFO | 2176, | 20260506 08:40:22.450 | INFO | 5248, | 20260506 08:40:22.450 | INFO | 1, | 20260506 08:40:22.450 | INFO | 64 | 20260506 08:40:22.450 | INFO | ], | 20260506 08:40:22.450 | INFO | [ | 20260506 08:40:22.450 | INFO | 2432, | 20260506 08:40:22.450 | INFO | 5248, | 20260506 08:40:22.450 | INFO | 1, | 20260506 08:40:22.450 | INFO | 64 | 20260506 08:40:22.450 | INFO | ], | 20260506 08:40:22.450 | INFO | [ | 20260506 08:40:22.450 | INFO | 2688, | 20260506 08:40:22.450 | INFO | 5248, | 20260506 08:40:22.450 | INFO | 1, | 20260506 08:40:22.450 | INFO | 64 | 20260506 08:40:22.450 | INFO | ], | 20260506 08:40:22.450 | INFO | [ | 20260506 08:40:22.450 | INFO | 2944, | 20260506 08:40:22.450 | INFO | 5248, | 20260506 08:40:22.450 | INFO | 1, | 20260506 08:40:22.450 | INFO | 64 | 20260506 08:40:22.450 | INFO | ], | 20260506 08:40:22.450 | INFO | [ | 20260506 08:40:22.450 | INFO | 3200, | 20260506 08:40:22.450 | INFO | 5248, | 20260506 08:40:22.450 | INFO | 1, | 20260506 08:40:22.450 | INFO | 64 | 20260506 08:40:22.450 | INFO | ], | 20260506 08:40:22.450 | INFO | [ | 20260506 08:40:22.450 | INFO | 3456, | 20260506 08:40:22.450 | INFO | 5248, | 20260506 08:40:22.450 | INFO | 1, | 20260506 08:40:22.451 | INFO | 64 | 20260506 08:40:22.451 | INFO | ], | 20260506 08:40:22.451 | INFO | [ | 20260506 08:40:22.451 | INFO | 3712, | 20260506 08:40:22.451 | INFO | 5248, | 20260506 08:40:22.451 | INFO | 1, | 20260506 08:40:22.451 | INFO | 64 | 20260506 08:40:22.451 | INFO | ], | 20260506 08:40:22.451 | INFO | [ | 20260506 08:40:22.451 | INFO | 3968, | 20260506 08:40:22.451 | INFO | 5248, | 20260506 08:40:22.451 | INFO | 1, | 20260506 08:40:22.451 | INFO | 64 | 20260506 08:40:22.451 | INFO | ], | 20260506 08:40:22.451 | INFO | [ | 20260506 08:40:22.451 | INFO | 128, | 20260506 08:40:22.451 | INFO | 5504, | 20260506 08:40:22.451 | INFO | 1, | 20260506 08:40:22.451 | INFO | 64 | 20260506 08:40:22.451 | INFO | ], | 20260506 08:40:22.451 | INFO | [ | 20260506 08:40:22.451 | INFO | 384, | 20260506 08:40:22.451 | INFO | 5504, | 20260506 08:40:22.451 | INFO | 1, | 20260506 08:40:22.451 | INFO | 64 | 20260506 08:40:22.451 | INFO | ], | 20260506 08:40:22.451 | INFO | [ | 20260506 08:40:22.451 | INFO | 640, | 20260506 08:40:22.451 | INFO | 5504, | 20260506 08:40:22.451 | INFO | 1, | 20260506 08:40:22.451 | INFO | 64 | 20260506 08:40:22.451 | INFO | ], | 20260506 08:40:22.451 | INFO | [ | 20260506 08:40:22.451 | INFO | 896, | 20260506 08:40:22.451 | INFO | 5504, | 20260506 08:40:22.451 | INFO | 1, | 20260506 08:40:22.451 | INFO | 64 | 20260506 08:40:22.451 | INFO | ], | 20260506 08:40:22.451 | INFO | [ | 20260506 08:40:22.451 | INFO | 1152, | 20260506 08:40:22.451 | INFO | 5504, | 20260506 08:40:22.451 | INFO | 1, | 20260506 08:40:22.451 | INFO | 64 | 20260506 08:40:22.451 | INFO | ], | 20260506 08:40:22.451 | INFO | [ | 20260506 08:40:22.451 | INFO | 1408, | 20260506 08:40:22.451 | INFO | 5504, | 20260506 08:40:22.451 | INFO | 1, | 20260506 08:40:22.451 | INFO | 64 | 20260506 08:40:22.451 | INFO | ], | 20260506 08:40:22.451 | INFO | [ | 20260506 08:40:22.451 | INFO | 1664, | 20260506 08:40:22.451 | INFO | 5504, | 20260506 08:40:22.451 | INFO | 1, | 20260506 08:40:22.451 | INFO | 64 | 20260506 08:40:22.451 | INFO | ], | 20260506 08:40:22.451 | INFO | [ | 20260506 08:40:22.451 | INFO | 1920, | 20260506 08:40:22.451 | INFO | 5504, | 20260506 08:40:22.451 | INFO | 1, | 20260506 08:40:22.451 | INFO | 64 | 20260506 08:40:22.451 | INFO | ], | 20260506 08:40:22.451 | INFO | [ | 20260506 08:40:22.451 | INFO | 2176, | 20260506 08:40:22.451 | INFO | 5504, | 20260506 08:40:22.451 | INFO | 1, | 20260506 08:40:22.451 | INFO | 64 | 20260506 08:40:22.451 | INFO | ], | 20260506 08:40:22.451 | INFO | [ | 20260506 08:40:22.451 | INFO | 2432, | 20260506 08:40:22.451 | INFO | 5504, | 20260506 08:40:22.451 | INFO | 1, | 20260506 08:40:22.451 | INFO | 64 | 20260506 08:40:22.451 | INFO | ], | 20260506 08:40:22.451 | INFO | [ | 20260506 08:40:22.451 | INFO | 2688, | 20260506 08:40:22.451 | INFO | 5504, | 20260506 08:40:22.451 | INFO | 1, | 20260506 08:40:22.451 | INFO | 64 | 20260506 08:40:22.451 | INFO | ], | 20260506 08:40:22.451 | INFO | [ | 20260506 08:40:22.451 | INFO | 2944, | 20260506 08:40:22.451 | INFO | 5504, | 20260506 08:40:22.451 | INFO | 1, | 20260506 08:40:22.451 | INFO | 64 | 20260506 08:40:22.451 | INFO | ], | 20260506 08:40:22.451 | INFO | [ | 20260506 08:40:22.451 | INFO | 3200, | 20260506 08:40:22.451 | INFO | 5504, | 20260506 08:40:22.451 | INFO | 1, | 20260506 08:40:22.451 | INFO | 64 | 20260506 08:40:22.451 | INFO | ], | 20260506 08:40:22.451 | INFO | [ | 20260506 08:40:22.451 | INFO | 3456, | 20260506 08:40:22.451 | INFO | 5504, | 20260506 08:40:22.452 | INFO | 1, | 20260506 08:40:22.452 | INFO | 64 | 20260506 08:40:22.452 | INFO | ], | 20260506 08:40:22.452 | INFO | [ | 20260506 08:40:22.452 | INFO | 3712, | 20260506 08:40:22.452 | INFO | 5504, | 20260506 08:40:22.452 | INFO | 1, | 20260506 08:40:22.452 | INFO | 64 | 20260506 08:40:22.452 | INFO | ], | 20260506 08:40:22.452 | INFO | [ | 20260506 08:40:22.452 | INFO | 3968, | 20260506 08:40:22.452 | INFO | 5504, | 20260506 08:40:22.452 | INFO | 1, | 20260506 08:40:22.452 | INFO | 64 | 20260506 08:40:22.452 | INFO | ], | 20260506 08:40:22.452 | INFO | [ | 20260506 08:40:22.452 | INFO | 128, | 20260506 08:40:22.452 | INFO | 5760, | 20260506 08:40:22.452 | INFO | 1, | 20260506 08:40:22.452 | INFO | 64 | 20260506 08:40:22.452 | INFO | ], | 20260506 08:40:22.452 | INFO | [ | 20260506 08:40:22.452 | INFO | 384, | 20260506 08:40:22.452 | INFO | 5760, | 20260506 08:40:22.452 | INFO | 1, | 20260506 08:40:22.452 | INFO | 64 | 20260506 08:40:22.452 | INFO | ], | 20260506 08:40:22.452 | INFO | [ | 20260506 08:40:22.452 | INFO | 640, | 20260506 08:40:22.452 | INFO | 5760, | 20260506 08:40:22.452 | INFO | 1, | 20260506 08:40:22.452 | INFO | 64 | 20260506 08:40:22.452 | INFO | ], | 20260506 08:40:22.452 | INFO | [ | 20260506 08:40:22.452 | INFO | 896, | 20260506 08:40:22.452 | INFO | 5760, | 20260506 08:40:22.452 | INFO | 1, | 20260506 08:40:22.452 | INFO | 64 | 20260506 08:40:22.452 | INFO | ], | 20260506 08:40:22.452 | INFO | [ | 20260506 08:40:22.452 | INFO | 1152, | 20260506 08:40:22.452 | INFO | 5760, | 20260506 08:40:22.452 | INFO | 1, | 20260506 08:40:22.452 | INFO | 64 | 20260506 08:40:22.452 | INFO | ], | 20260506 08:40:22.452 | INFO | [ | 20260506 08:40:22.452 | INFO | 1408, | 20260506 08:40:22.452 | INFO | 5760, | 20260506 08:40:22.452 | INFO | 1, | 20260506 08:40:22.452 | INFO | 64 | 20260506 08:40:22.452 | INFO | ], | 20260506 08:40:22.452 | INFO | [ | 20260506 08:40:22.452 | INFO | 1664, | 20260506 08:40:22.452 | INFO | 5760, | 20260506 08:40:22.452 | INFO | 1, | 20260506 08:40:22.452 | INFO | 64 | 20260506 08:40:22.452 | INFO | ], | 20260506 08:40:22.452 | INFO | [ | 20260506 08:40:22.452 | INFO | 1920, | 20260506 08:40:22.452 | INFO | 5760, | 20260506 08:40:22.452 | INFO | 1, | 20260506 08:40:22.452 | INFO | 64 | 20260506 08:40:22.452 | INFO | ], | 20260506 08:40:22.452 | INFO | [ | 20260506 08:40:22.452 | INFO | 2176, | 20260506 08:40:22.452 | INFO | 5760, | 20260506 08:40:22.452 | INFO | 1, | 20260506 08:40:22.452 | INFO | 64 | 20260506 08:40:22.452 | INFO | ], | 20260506 08:40:22.452 | INFO | [ | 20260506 08:40:22.452 | INFO | 2432, | 20260506 08:40:22.452 | INFO | 5760, | 20260506 08:40:22.452 | INFO | 1, | 20260506 08:40:22.452 | INFO | 64 | 20260506 08:40:22.452 | INFO | ], | 20260506 08:40:22.452 | INFO | [ | 20260506 08:40:22.452 | INFO | 2688, | 20260506 08:40:22.452 | INFO | 5760, | 20260506 08:40:22.452 | INFO | 1, | 20260506 08:40:22.452 | INFO | 64 | 20260506 08:40:22.452 | INFO | ], | 20260506 08:40:22.452 | INFO | [ | 20260506 08:40:22.452 | INFO | 2944, | 20260506 08:40:22.452 | INFO | 5760, | 20260506 08:40:22.452 | INFO | 1, | 20260506 08:40:22.452 | INFO | 64 | 20260506 08:40:22.452 | INFO | ], | 20260506 08:40:22.452 | INFO | [ | 20260506 08:40:22.452 | INFO | 3200, | 20260506 08:40:22.452 | INFO | 5760, | 20260506 08:40:22.452 | INFO | 1, | 20260506 08:40:22.457 | INFO | 64 | 20260506 08:40:22.457 | INFO | ], | 20260506 08:40:22.457 | INFO | [ | 20260506 08:40:22.457 | INFO | 3456, | 20260506 08:40:22.457 | INFO | 5760, | 20260506 08:40:22.457 | INFO | 1, | 20260506 08:40:22.457 | INFO | 64 | 20260506 08:40:22.457 | INFO | ], | 20260506 08:40:22.457 | INFO | [ | 20260506 08:40:22.457 | INFO | 3712, | 20260506 08:40:22.457 | INFO | 5760, | 20260506 08:40:22.457 | INFO | 1, | 20260506 08:40:22.457 | INFO | 64 | 20260506 08:40:22.457 | INFO | ], | 20260506 08:40:22.457 | INFO | [ | 20260506 08:40:22.457 | INFO | 3968, | 20260506 08:40:22.457 | INFO | 5760, | 20260506 08:40:22.457 | INFO | 1, | 20260506 08:40:22.457 | INFO | 64 | 20260506 08:40:22.457 | INFO | ], | 20260506 08:40:22.457 | INFO | [ | 20260506 08:40:22.457 | INFO | 128, | 20260506 08:40:22.457 | INFO | 6016, | 20260506 08:40:22.457 | INFO | 1, | 20260506 08:40:22.457 | INFO | 64 | 20260506 08:40:22.457 | INFO | ], | 20260506 08:40:22.457 | INFO | [ | 20260506 08:40:22.457 | INFO | 384, | 20260506 08:40:22.457 | INFO | 6016, | 20260506 08:40:22.457 | INFO | 1, | 20260506 08:40:22.457 | INFO | 64 | 20260506 08:40:22.457 | INFO | ], | 20260506 08:40:22.457 | INFO | [ | 20260506 08:40:22.457 | INFO | 640, | 20260506 08:40:22.457 | INFO | 6016, | 20260506 08:40:22.457 | INFO | 1, | 20260506 08:40:22.457 | INFO | 64 | 20260506 08:40:22.457 | INFO | ], | 20260506 08:40:22.457 | INFO | [ | 20260506 08:40:22.457 | INFO | 896, | 20260506 08:40:22.457 | INFO | 6016, | 20260506 08:40:22.457 | INFO | 1, | 20260506 08:40:22.457 | INFO | 64 | 20260506 08:40:22.457 | INFO | ], | 20260506 08:40:22.457 | INFO | [ | 20260506 08:40:22.457 | INFO | 1152, | 20260506 08:40:22.457 | INFO | 6016, | 20260506 08:40:22.457 | INFO | 1, | 20260506 08:40:22.457 | INFO | 64 | 20260506 08:40:22.457 | INFO | ], | 20260506 08:40:22.457 | INFO | [ | 20260506 08:40:22.457 | INFO | 1408, | 20260506 08:40:22.457 | INFO | 6016, | 20260506 08:40:22.457 | INFO | 1, | 20260506 08:40:22.457 | INFO | 64 | 20260506 08:40:22.457 | INFO | ], | 20260506 08:40:22.457 | INFO | [ | 20260506 08:40:22.457 | INFO | 1664, | 20260506 08:40:22.457 | INFO | 6016, | 20260506 08:40:22.457 | INFO | 1, | 20260506 08:40:22.457 | INFO | 64 | 20260506 08:40:22.457 | INFO | ], | 20260506 08:40:22.457 | INFO | [ | 20260506 08:40:22.457 | INFO | 1920, | 20260506 08:40:22.457 | INFO | 6016, | 20260506 08:40:22.457 | INFO | 1, | 20260506 08:40:22.457 | INFO | 64 | 20260506 08:40:22.457 | INFO | ], | 20260506 08:40:22.457 | INFO | [ | 20260506 08:40:22.457 | INFO | 2176, | 20260506 08:40:22.457 | INFO | 6016, | 20260506 08:40:22.457 | INFO | 1, | 20260506 08:40:22.457 | INFO | 64 | 20260506 08:40:22.457 | INFO | ], | 20260506 08:40:22.457 | INFO | [ | 20260506 08:40:22.457 | INFO | 2432, | 20260506 08:40:22.457 | INFO | 6016, | 20260506 08:40:22.457 | INFO | 1, | 20260506 08:40:22.457 | INFO | 64 | 20260506 08:40:22.457 | INFO | ], | 20260506 08:40:22.457 | INFO | [ | 20260506 08:40:22.457 | INFO | 2688, | 20260506 08:40:22.457 | INFO | 6016, | 20260506 08:40:22.457 | INFO | 1, | 20260506 08:40:22.457 | INFO | 64 | 20260506 08:40:22.457 | INFO | ], | 20260506 08:40:22.457 | INFO | [ | 20260506 08:40:22.457 | INFO | 2944, | 20260506 08:40:22.457 | INFO | 6016, | 20260506 08:40:22.457 | INFO | 1, | 20260506 08:40:22.457 | INFO | 64 | 20260506 08:40:22.458 | INFO | ], | 20260506 08:40:22.458 | INFO | [ | 20260506 08:40:22.458 | INFO | 3200, | 20260506 08:40:22.458 | INFO | 6016, | 20260506 08:40:22.458 | INFO | 1, | 20260506 08:40:22.458 | INFO | 64 | 20260506 08:40:22.458 | INFO | ], | 20260506 08:40:22.458 | INFO | [ | 20260506 08:40:22.458 | INFO | 3456, | 20260506 08:40:22.458 | INFO | 6016, | 20260506 08:40:22.458 | INFO | 1, | 20260506 08:40:22.458 | INFO | 64 | 20260506 08:40:22.458 | INFO | ], | 20260506 08:40:22.458 | INFO | [ | 20260506 08:40:22.458 | INFO | 3712, | 20260506 08:40:22.458 | INFO | 6016, | 20260506 08:40:22.458 | INFO | 1, | 20260506 08:40:22.458 | INFO | 64 | 20260506 08:40:22.458 | INFO | ], | 20260506 08:40:22.458 | INFO | [ | 20260506 08:40:22.458 | INFO | 3968, | 20260506 08:40:22.458 | INFO | 6016, | 20260506 08:40:22.458 | INFO | 1, | 20260506 08:40:22.458 | INFO | 64 | 20260506 08:40:22.458 | INFO | ] | 20260506 08:40:22.458 | INFO | ], | 20260506 08:40:22.458 | INFO | [ | 20260506 08:40:22.458 | INFO | [ | 20260506 08:40:22.458 | INFO | 409, | 20260506 08:40:22.458 | INFO | 384, | 20260506 08:40:22.458 | INFO | 1, | 20260506 08:40:22.458 | INFO | 64 | 20260506 08:40:22.458 | INFO | ], | 20260506 08:40:22.458 | INFO | [ | 20260506 08:40:22.458 | INFO | 1228, | 20260506 08:40:22.458 | INFO | 384, | 20260506 08:40:22.458 | INFO | 1, | 20260506 08:40:22.458 | INFO | 64 | 20260506 08:40:22.458 | INFO | ], | 20260506 08:40:22.458 | INFO | [ | 20260506 08:40:22.458 | INFO | 2047, | 20260506 08:40:22.458 | INFO | 384, | 20260506 08:40:22.458 | INFO | 1, | 20260506 08:40:22.458 | INFO | 64 | 20260506 08:40:22.458 | INFO | ], | 20260506 08:40:22.458 | INFO | [ | 20260506 08:40:22.458 | INFO | 2866, | 20260506 08:40:22.458 | INFO | 384, | 20260506 08:40:22.458 | INFO | 1, | 20260506 08:40:22.458 | INFO | 64 | 20260506 08:40:22.458 | INFO | ], | 20260506 08:40:22.458 | INFO | [ | 20260506 08:40:22.458 | INFO | 3685, | 20260506 08:40:22.458 | INFO | 384, | 20260506 08:40:22.458 | INFO | 1, | 20260506 08:40:22.458 | INFO | 64 | 20260506 08:40:22.458 | INFO | ], | 20260506 08:40:22.458 | INFO | [ | 20260506 08:40:22.458 | INFO | 409, | 20260506 08:40:22.458 | INFO | 1152, | 20260506 08:40:22.458 | INFO | 1, | 20260506 08:40:22.458 | INFO | 64 | 20260506 08:40:22.458 | INFO | ], | 20260506 08:40:22.458 | INFO | [ | 20260506 08:40:22.458 | INFO | 1228, | 20260506 08:40:22.458 | INFO | 1152, | 20260506 08:40:22.458 | INFO | 1, | 20260506 08:40:22.458 | INFO | 64 | 20260506 08:40:22.458 | INFO | ], | 20260506 08:40:22.458 | INFO | [ | 20260506 08:40:22.458 | INFO | 2047, | 20260506 08:40:22.458 | INFO | 1152, | 20260506 08:40:22.458 | INFO | 1, | 20260506 08:40:22.458 | INFO | 64 | 20260506 08:40:22.458 | INFO | ], | 20260506 08:40:22.458 | INFO | [ | 20260506 08:40:22.458 | INFO | 2866, | 20260506 08:40:22.458 | INFO | 1152, | 20260506 08:40:22.458 | INFO | 1, | 20260506 08:40:22.458 | INFO | 64 | 20260506 08:40:22.458 | INFO | ], | 20260506 08:40:22.458 | INFO | [ | 20260506 08:40:22.458 | INFO | 3685, | 20260506 08:40:22.458 | INFO | 1152, | 20260506 08:40:22.458 | INFO | 1, | 20260506 08:40:22.458 | INFO | 64 | 20260506 08:40:22.458 | INFO | ], | 20260506 08:40:22.458 | INFO | [ | 20260506 08:40:22.458 | INFO | 409, | 20260506 08:40:22.458 | INFO | 1920, | 20260506 08:40:22.458 | INFO | 1, | 20260506 08:40:22.458 | INFO | 64 | 20260506 08:40:22.458 | INFO | ], | 20260506 08:40:22.458 | INFO | [ | 20260506 08:40:22.458 | INFO | 1228, | 20260506 08:40:22.458 | INFO | 1920, | 20260506 08:40:22.459 | INFO | 1, | 20260506 08:40:22.459 | INFO | 64 | 20260506 08:40:22.459 | INFO | ], | 20260506 08:40:22.459 | INFO | [ | 20260506 08:40:22.459 | INFO | 2047, | 20260506 08:40:22.459 | INFO | 1920, | 20260506 08:40:22.459 | INFO | 1, | 20260506 08:40:22.459 | INFO | 64 | 20260506 08:40:22.459 | INFO | ], | 20260506 08:40:22.459 | INFO | [ | 20260506 08:40:22.459 | INFO | 2866, | 20260506 08:40:22.459 | INFO | 1920, | 20260506 08:40:22.459 | INFO | 1, | 20260506 08:40:22.459 | INFO | 64 | 20260506 08:40:22.459 | INFO | ], | 20260506 08:40:22.459 | INFO | [ | 20260506 08:40:22.459 | INFO | 3685, | 20260506 08:40:22.459 | INFO | 1920, | 20260506 08:40:22.459 | INFO | 1, | 20260506 08:40:22.459 | INFO | 64 | 20260506 08:40:22.459 | INFO | ], | 20260506 08:40:22.459 | INFO | [ | 20260506 08:40:22.459 | INFO | 409, | 20260506 08:40:22.459 | INFO | 2688, | 20260506 08:40:22.459 | INFO | 1, | 20260506 08:40:22.459 | INFO | 64 | 20260506 08:40:22.459 | INFO | ], | 20260506 08:40:22.459 | INFO | [ | 20260506 08:40:22.459 | INFO | 1228, | 20260506 08:40:22.459 | INFO | 2688, | 20260506 08:40:22.459 | INFO | 1, | 20260506 08:40:22.459 | INFO | 64 | 20260506 08:40:22.459 | INFO | ], | 20260506 08:40:22.459 | INFO | [ | 20260506 08:40:22.459 | INFO | 2047, | 20260506 08:40:22.459 | INFO | 2688, | 20260506 08:40:22.459 | INFO | 1, | 20260506 08:40:22.459 | INFO | 64 | 20260506 08:40:22.459 | INFO | ], | 20260506 08:40:22.459 | INFO | [ | 20260506 08:40:22.459 | INFO | 2866, | 20260506 08:40:22.459 | INFO | 2688, | 20260506 08:40:22.459 | INFO | 1, | 20260506 08:40:22.459 | INFO | 64 | 20260506 08:40:22.459 | INFO | ], | 20260506 08:40:22.459 | INFO | [ | 20260506 08:40:22.459 | INFO | 3685, | 20260506 08:40:22.459 | INFO | 2688, | 20260506 08:40:22.459 | INFO | 1, | 20260506 08:40:22.459 | INFO | 64 | 20260506 08:40:22.459 | INFO | ], | 20260506 08:40:22.459 | INFO | [ | 20260506 08:40:22.459 | INFO | 409, | 20260506 08:40:22.459 | INFO | 3456, | 20260506 08:40:22.459 | INFO | 1, | 20260506 08:40:22.459 | INFO | 64 | 20260506 08:40:22.459 | INFO | ], | 20260506 08:40:22.459 | INFO | [ | 20260506 08:40:22.459 | INFO | 1228, | 20260506 08:40:22.459 | INFO | 3456, | 20260506 08:40:22.459 | INFO | 1, | 20260506 08:40:22.459 | INFO | 64 | 20260506 08:40:22.459 | INFO | ], | 20260506 08:40:22.459 | INFO | [ | 20260506 08:40:22.459 | INFO | 2047, | 20260506 08:40:22.459 | INFO | 3456, | 20260506 08:40:22.459 | INFO | 1, | 20260506 08:40:22.459 | INFO | 64 | 20260506 08:40:22.459 | INFO | ], | 20260506 08:40:22.459 | INFO | [ | 20260506 08:40:22.459 | INFO | 2866, | 20260506 08:40:22.459 | INFO | 3456, | 20260506 08:40:22.459 | INFO | 1, | 20260506 08:40:22.459 | INFO | 64 | 20260506 08:40:22.459 | INFO | ], | 20260506 08:40:22.459 | INFO | [ | 20260506 08:40:22.459 | INFO | 3685, | 20260506 08:40:22.459 | INFO | 3456, | 20260506 08:40:22.459 | INFO | 1, | 20260506 08:40:22.459 | INFO | 64 | 20260506 08:40:22.459 | INFO | ], | 20260506 08:40:22.459 | INFO | [ | 20260506 08:40:22.459 | INFO | 409, | 20260506 08:40:22.459 | INFO | 4224, | 20260506 08:40:22.459 | INFO | 1, | 20260506 08:40:22.459 | INFO | 64 | 20260506 08:40:22.459 | INFO | ], | 20260506 08:40:22.459 | INFO | [ | 20260506 08:40:22.459 | INFO | 1228, | 20260506 08:40:22.459 | INFO | 4224, | 20260506 08:40:22.459 | INFO | 1, | 20260506 08:40:22.459 | INFO | 64 | 20260506 08:40:22.459 | INFO | ], | 20260506 08:40:22.459 | INFO | [ | 20260506 08:40:22.459 | INFO | 2047, | 20260506 08:40:22.460 | INFO | 4224, | 20260506 08:40:22.460 | INFO | 1, | 20260506 08:40:22.460 | INFO | 64 | 20260506 08:40:22.460 | INFO | ], | 20260506 08:40:22.460 | INFO | [ | 20260506 08:40:22.460 | INFO | 2866, | 20260506 08:40:22.460 | INFO | 4224, | 20260506 08:40:22.460 | INFO | 1, | 20260506 08:40:22.460 | INFO | 64 | 20260506 08:40:22.460 | INFO | ], | 20260506 08:40:22.460 | INFO | [ | 20260506 08:40:22.460 | INFO | 3685, | 20260506 08:40:22.460 | INFO | 4224, | 20260506 08:40:22.460 | INFO | 1, | 20260506 08:40:22.460 | INFO | 64 | 20260506 08:40:22.460 | INFO | ], | 20260506 08:40:22.460 | INFO | [ | 20260506 08:40:22.460 | INFO | 409, | 20260506 08:40:22.460 | INFO | 4992, | 20260506 08:40:22.460 | INFO | 1, | 20260506 08:40:22.460 | INFO | 64 | 20260506 08:40:22.460 | INFO | ], | 20260506 08:40:22.460 | INFO | [ | 20260506 08:40:22.460 | INFO | 1228, | 20260506 08:40:22.460 | INFO | 4992, | 20260506 08:40:22.460 | INFO | 1, | 20260506 08:40:22.460 | INFO | 64 | 20260506 08:40:22.460 | INFO | ], | 20260506 08:40:22.460 | INFO | [ | 20260506 08:40:22.460 | INFO | 2047, | 20260506 08:40:22.460 | INFO | 4992, | 20260506 08:40:22.460 | INFO | 1, | 20260506 08:40:22.460 | INFO | 64 | 20260506 08:40:22.460 | INFO | ], | 20260506 08:40:22.460 | INFO | [ | 20260506 08:40:22.460 | INFO | 2866, | 20260506 08:40:22.460 | INFO | 4992, | 20260506 08:40:22.460 | INFO | 1, | 20260506 08:40:22.460 | INFO | 64 | 20260506 08:40:22.460 | INFO | ], | 20260506 08:40:22.460 | INFO | [ | 20260506 08:40:22.460 | INFO | 3685, | 20260506 08:40:22.460 | INFO | 4992, | 20260506 08:40:22.460 | INFO | 1, | 20260506 08:40:22.460 | INFO | 64 | 20260506 08:40:22.460 | INFO | ], | 20260506 08:40:22.460 | INFO | [ | 20260506 08:40:22.460 | INFO | 409, | 20260506 08:40:22.460 | INFO | 5760, | 20260506 08:40:22.460 | INFO | 1, | 20260506 08:40:22.460 | INFO | 64 | 20260506 08:40:22.460 | INFO | ], | 20260506 08:40:22.460 | INFO | [ | 20260506 08:40:22.460 | INFO | 1228, | 20260506 08:40:22.460 | INFO | 5760, | 20260506 08:40:22.460 | INFO | 1, | 20260506 08:40:22.460 | INFO | 64 | 20260506 08:40:22.460 | INFO | ], | 20260506 08:40:22.460 | INFO | [ | 20260506 08:40:22.460 | INFO | 2047, | 20260506 08:40:22.460 | INFO | 5760, | 20260506 08:40:22.460 | INFO | 1, | 20260506 08:40:22.460 | INFO | 64 | 20260506 08:40:22.460 | INFO | ], | 20260506 08:40:22.460 | INFO | [ | 20260506 08:40:22.460 | INFO | 2866, | 20260506 08:40:22.460 | INFO | 5760, | 20260506 08:40:22.460 | INFO | 1, | 20260506 08:40:22.460 | INFO | 64 | 20260506 08:40:22.460 | INFO | ], | 20260506 08:40:22.460 | INFO | [ | 20260506 08:40:22.460 | INFO | 3685, | 20260506 08:40:22.460 | INFO | 5760, | 20260506 08:40:22.460 | INFO | 1, | 20260506 08:40:22.460 | INFO | 64 | 20260506 08:40:22.460 | INFO | ] | 20260506 08:40:22.460 | INFO | ] | 20260506 08:40:22.460 | INFO | ] | 20260506 08:40:22.460 | INFO | } | 20260506 08:40:22.460 | INFO | }, | 20260506 08:40:22.460 | INFO | "dataSource" : | 20260506 08:40:22.460 | INFO | { | 20260506 08:40:22.460 | INFO | "crosstalkFilterKernel" : [], | 20260506 08:40:22.460 | INFO | "darkCalFileName" : "/data/icc/m84031_260506_033729_s1/pb_internal/m84031_260506_033729_s1.darkcal.h5", | 20260506 08:40:22.460 | INFO | "imagePsfKernel" : | 20260506 08:40:22.460 | INFO | [ | 20260506 08:40:22.460 | INFO | [ | 20260506 08:40:22.461 | INFO | 0.00080000000000000004, | 20260506 08:40:22.461 | INFO | 0.0030000000000000001, | 20260506 08:40:22.461 | INFO | 0.0166, | 20260506 08:40:22.461 | INFO | 0.0035000000000000001, | 20260506 08:40:22.461 | INFO | 0.0011000000000000001 | 20260506 08:40:22.461 | INFO | ], | 20260506 08:40:22.461 | INFO | [ | 20260506 08:40:22.461 | INFO | 0.0027000000000000001, | 20260506 08:40:22.461 | INFO | 0.012999999999999999, | 20260506 08:40:22.461 | INFO | 0.072700000000000001, | 20260506 08:40:22.461 | INFO | 0.014, | 20260506 08:40:22.461 | INFO | 0.0032000000000000002 | 20260506 08:40:22.461 | INFO | ], | 20260506 08:40:22.461 | INFO | [ | 20260506 08:40:22.461 | INFO | 0.0118, | 20260506 08:40:22.461 | INFO | 0.0545, | 20260506 08:40:22.461 | INFO | 0.59440000000000004, | 20260506 08:40:22.461 | INFO | 0.065299999999999997, | 20260506 08:40:22.461 | INFO | 0.0124 | 20260506 08:40:22.461 | INFO | ], | 20260506 08:40:22.461 | INFO | [ | 20260506 08:40:22.461 | INFO | 0.0032000000000000002, | 20260506 08:40:22.461 | INFO | 0.013100000000000001, | 20260506 08:40:22.461 | INFO | 0.072700000000000001, | 20260506 08:40:22.461 | INFO | 0.0141, | 20260506 08:40:22.461 | INFO | 0.0030999999999999999 | 20260506 08:40:22.461 | INFO | ], | 20260506 08:40:22.461 | INFO | [ | 20260506 08:40:22.461 | INFO | 0.001, | 20260506 08:40:22.461 | INFO | 0.0028999999999999998, | 20260506 08:40:22.461 | INFO | 0.016799999999999999, | 20260506 08:40:22.461 | INFO | 0.0033999999999999998, | 20260506 08:40:22.461 | INFO | 0.00089999999999999998 | 20260506 08:40:22.461 | INFO | ] | 20260506 08:40:22.461 | INFO | ] | 20260506 08:40:22.461 | INFO | }, | 20260506 08:40:22.461 | INFO | "internalMode" : false, | 20260506 08:40:22.461 | INFO | "layout" : | 20260506 08:40:22.461 | INFO | { | 20260506 08:40:22.461 | INFO | "framesPerChunk" : 512, | 20260506 08:40:22.461 | INFO | "lanesPerPool" : 4096, | 20260506 08:40:22.461 | INFO | "zmwsPerLane" : 64 | 20260506 08:40:22.461 | INFO | }, | 20260506 08:40:22.461 | INFO | "maskFraction" : 1, | 20260506 08:40:22.461 | INFO | "monitoringReportInterval" : 8192, | 20260506 08:40:22.461 | INFO | "multipleBazFiles" : true, | 20260506 08:40:22.461 | INFO | "prelimHQ" : | 20260506 08:40:22.461 | INFO | { | 20260506 08:40:22.461 | INFO | "enablePreHQ" : true, | 20260506 08:40:22.461 | INFO | "expectedFrameRate" : 100, | 20260506 08:40:22.461 | INFO | "expectedPulsesPerZmw" : 164, | 20260506 08:40:22.461 | INFO | "fracMultiBlocks" : 0.20000000298023224, | 20260506 08:40:22.461 | INFO | "fracSingleBlocks" : 0.5, | 20260506 08:40:22.461 | INFO | "hqBufferGrowthFactor" : 2, | 20260506 08:40:22.461 | INFO | "hqThrottleFraction" : 1, | 20260506 08:40:22.461 | INFO | "maxPreHQBuffers" : 1, | 20260506 08:40:22.461 | INFO | "maxSustainedPulseRate" : 4, | 20260506 08:40:22.461 | INFO | "preHQBufferGrowthFactor" : 1, | 20260506 08:40:22.461 | INFO | "requiredActivityLabels" : 10, | 20260506 08:40:22.461 | INFO | "zmwOutputStride" : 1 | 20260506 08:40:22.461 | INFO | }, | 20260506 08:40:22.461 | INFO | "realTimeMetrics" : | 20260506 08:40:22.461 | INFO | { | 20260506 08:40:22.461 | INFO | "csvOutputFile" : "", | 20260506 08:40:22.461 | INFO | "diagOutputFile" : "", | 20260506 08:40:22.461 | INFO | "jsonOutputFile" : "/data/nrta/0/m84031_260506_033729_s1/pb_internal/m84031_260506_033729_s1.rtmetrics.json", | 20260506 08:40:22.461 | INFO | "minConfidenceScore" : 0, | 20260506 08:40:22.461 | INFO | "regions" : | 20260506 08:40:22.461 | INFO | [ | 20260506 08:40:22.461 | INFO | { | 20260506 08:40:22.461 | INFO | "featuresForFilter" : | 20260506 08:40:22.461 | INFO | [ | 20260506 08:40:22.461 | INFO | "PorSequencing" | 20260506 08:40:22.461 | INFO | ], | 20260506 08:40:22.461 | INFO | "medianIntraLaneStride" : 8, | 20260506 08:40:22.461 | INFO | "metrics" : | 20260506 08:40:22.461 | INFO | [ | 20260506 08:40:22.461 | INFO | "PulseRate", | 20260506 08:40:22.461 | INFO | "PulseWidth", | 20260506 08:40:22.461 | INFO | "BaseRate", | 20260506 08:40:22.461 | INFO | "BaseWidth", | 20260506 08:40:22.461 | INFO | "Baseline", | 20260506 08:40:22.461 | INFO | "BaselineStd", | 20260506 08:40:22.461 | INFO | "Pkmid", | 20260506 08:40:22.461 | INFO | "Snr" | 20260506 08:40:22.461 | INFO | ], | 20260506 08:40:22.461 | INFO | "minSampleSize" : 1000, | 20260506 08:40:22.461 | INFO | "name" : "FullChip", | 20260506 08:40:22.461 | INFO | "roi" : | 20260506 08:40:22.461 | INFO | [ | 20260506 08:40:22.461 | INFO | [ | 20260506 08:40:22.461 | INFO | 0, | 20260506 08:40:22.461 | INFO | 0, | 20260506 08:40:22.461 | INFO | 4096, | 20260506 08:40:22.461 | INFO | 6144 | 20260506 08:40:22.462 | INFO | ] | 20260506 08:40:22.462 | INFO | ], | 20260506 08:40:22.462 | INFO | "useSingleActivityLabels" : true | 20260506 08:40:22.462 | INFO | }, | 20260506 08:40:22.462 | INFO | { | 20260506 08:40:22.462 | INFO | "featuresForFilter" : | 20260506 08:40:22.462 | INFO | [ | 20260506 08:40:22.462 | INFO | "PorSequencing" | 20260506 08:40:22.462 | INFO | ], | 20260506 08:40:22.462 | INFO | "medianIntraLaneStride" : 1, | 20260506 08:40:22.462 | INFO | "metrics" : | 20260506 08:40:22.462 | INFO | [ | 20260506 08:40:22.462 | INFO | "Baseline", | 20260506 08:40:22.462 | INFO | "BaselineStd", | 20260506 08:40:22.462 | INFO | "Pkmid", | 20260506 08:40:22.462 | INFO | "Snr" | 20260506 08:40:22.462 | INFO | ], | 20260506 08:40:22.462 | INFO | "minSampleSize" : 1000, | 20260506 08:40:22.462 | INFO | "name" : "TopStrip", | 20260506 08:40:22.462 | INFO | "roi" : | 20260506 08:40:22.462 | INFO | [ | 20260506 08:40:22.462 | INFO | [ | 20260506 08:40:22.462 | INFO | 200, | 20260506 08:40:22.462 | INFO | 64, | 20260506 08:40:22.462 | INFO | 64, | 20260506 08:40:22.462 | INFO | 6016 | 20260506 08:40:22.462 | INFO | ] | 20260506 08:40:22.462 | INFO | ], | 20260506 08:40:22.462 | INFO | "useSingleActivityLabels" : true | 20260506 08:40:22.462 | INFO | }, | 20260506 08:40:22.462 | INFO | { | 20260506 08:40:22.462 | INFO | "featuresForFilter" : | 20260506 08:40:22.462 | INFO | [ | 20260506 08:40:22.462 | INFO | "PorSequencing" | 20260506 08:40:22.462 | INFO | ], | 20260506 08:40:22.462 | INFO | "medianIntraLaneStride" : 1, | 20260506 08:40:22.462 | INFO | "metrics" : | 20260506 08:40:22.462 | INFO | [ | 20260506 08:40:22.462 | INFO | "Baseline", | 20260506 08:40:22.462 | INFO | "BaselineStd", | 20260506 08:40:22.462 | INFO | "Pkmid", | 20260506 08:40:22.462 | INFO | "Snr" | 20260506 08:40:22.462 | INFO | ], | 20260506 08:40:22.462 | INFO | "minSampleSize" : 1000, | 20260506 08:40:22.462 | INFO | "name" : "MidStrip", | 20260506 08:40:22.462 | INFO | "roi" : | 20260506 08:40:22.462 | INFO | [ | 20260506 08:40:22.462 | INFO | [ | 20260506 08:40:22.462 | INFO | 2016, | 20260506 08:40:22.462 | INFO | 64, | 20260506 08:40:22.462 | INFO | 64, | 20260506 08:40:22.462 | INFO | 6016 | 20260506 08:40:22.462 | INFO | ] | 20260506 08:40:22.462 | INFO | ], | 20260506 08:40:22.462 | INFO | "useSingleActivityLabels" : true | 20260506 08:40:22.462 | INFO | }, | 20260506 08:40:22.462 | INFO | { | 20260506 08:40:22.462 | INFO | "featuresForFilter" : | 20260506 08:40:22.462 | INFO | [ | 20260506 08:40:22.462 | INFO | "PorSequencing" | 20260506 08:40:22.462 | INFO | ], | 20260506 08:40:22.462 | INFO | "medianIntraLaneStride" : 1, | 20260506 08:40:22.462 | INFO | "metrics" : | 20260506 08:40:22.462 | INFO | [ | 20260506 08:40:22.462 | INFO | "Baseline", | 20260506 08:40:22.462 | INFO | "BaselineStd", | 20260506 08:40:22.462 | INFO | "Pkmid", | 20260506 08:40:22.462 | INFO | "Snr" | 20260506 08:40:22.462 | INFO | ], | 20260506 08:40:22.462 | INFO | "minSampleSize" : 1000, | 20260506 08:40:22.462 | INFO | "name" : "BotStrip", | 20260506 08:40:22.462 | INFO | "roi" : | 20260506 08:40:22.462 | INFO | [ | 20260506 08:40:22.462 | INFO | [ | 20260506 08:40:22.462 | INFO | 3832, | 20260506 08:40:22.462 | INFO | 64, | 20260506 08:40:22.462 | INFO | 64, | 20260506 08:40:22.462 | INFO | 6016 | 20260506 08:40:22.462 | INFO | ] | 20260506 08:40:22.462 | INFO | ], | 20260506 08:40:22.462 | INFO | "useSingleActivityLabels" : true | 20260506 08:40:22.462 | INFO | }, | 20260506 08:40:22.462 | INFO | { | 20260506 08:40:22.462 | INFO | "featuresForFilter" : | 20260506 08:40:22.462 | INFO | [ | 20260506 08:40:22.462 | INFO | "LaserPower2p0x" | 20260506 08:40:22.462 | INFO | ], | 20260506 08:40:22.462 | INFO | "medianIntraLaneStride" : 1, | 20260506 08:40:22.462 | INFO | "metrics" : | 20260506 08:40:22.462 | INFO | [ | 20260506 08:40:22.462 | INFO | "Baseline", | 20260506 08:40:22.463 | INFO | "BaselineStd", | 20260506 08:40:22.463 | INFO | "Pkmid", | 20260506 08:40:22.463 | INFO | "Snr" | 20260506 08:40:22.463 | INFO | ], | 20260506 08:40:22.463 | INFO | "minSampleSize" : 1000, | 20260506 08:40:22.463 | INFO | "name" : "FullChip_LP2P0X", | 20260506 08:40:22.463 | INFO | "roi" : | 20260506 08:40:22.463 | INFO | [ | 20260506 08:40:22.463 | INFO | [ | 20260506 08:40:22.463 | INFO | 0, | 20260506 08:40:22.463 | INFO | 0, | 20260506 08:40:22.463 | INFO | 4096, | 20260506 08:40:22.463 | INFO | 6144 | 20260506 08:40:22.463 | INFO | ] | 20260506 08:40:22.463 | INFO | ], | 20260506 08:40:22.463 | INFO | "useSingleActivityLabels" : true | 20260506 08:40:22.463 | INFO | }, | 20260506 08:40:22.463 | INFO | { | 20260506 08:40:22.463 | INFO | "featuresForFilter" : | 20260506 08:40:22.463 | INFO | [ | 20260506 08:40:22.463 | INFO | "LaserPower1p5x" | 20260506 08:40:22.463 | INFO | ], | 20260506 08:40:22.463 | INFO | "medianIntraLaneStride" : 1, | 20260506 08:40:22.463 | INFO | "metrics" : | 20260506 08:40:22.463 | INFO | [ | 20260506 08:40:22.463 | INFO | "Baseline", | 20260506 08:40:22.463 | INFO | "BaselineStd", | 20260506 08:40:22.463 | INFO | "Pkmid", | 20260506 08:40:22.463 | INFO | "Snr" | 20260506 08:40:22.463 | INFO | ], | 20260506 08:40:22.463 | INFO | "minSampleSize" : 1000, | 20260506 08:40:22.463 | INFO | "name" : "FullChip_LP1P5X", | 20260506 08:40:22.463 | INFO | "roi" : | 20260506 08:40:22.463 | INFO | [ | 20260506 08:40:22.463 | INFO | [ | 20260506 08:40:22.463 | INFO | 0, | 20260506 08:40:22.463 | INFO | 0, | 20260506 08:40:22.463 | INFO | 4096, | 20260506 08:40:22.463 | INFO | 6144 | 20260506 08:40:22.463 | INFO | ] | 20260506 08:40:22.463 | INFO | ], | 20260506 08:40:22.463 | INFO | "useSingleActivityLabels" : true | 20260506 08:40:22.463 | INFO | }, | 20260506 08:40:22.463 | INFO | { | 20260506 08:40:22.463 | INFO | "featuresForFilter" : | 20260506 08:40:22.463 | INFO | [ | 20260506 08:40:22.463 | INFO | "LaserPower0p5x" | 20260506 08:40:22.463 | INFO | ], | 20260506 08:40:22.463 | INFO | "medianIntraLaneStride" : 1, | 20260506 08:40:22.463 | INFO | "metrics" : | 20260506 08:40:22.463 | INFO | [ | 20260506 08:40:22.463 | INFO | "Baseline", | 20260506 08:40:22.463 | INFO | "BaselineStd", | 20260506 08:40:22.463 | INFO | "Pkmid", | 20260506 08:40:22.463 | INFO | "Snr" | 20260506 08:40:22.463 | INFO | ], | 20260506 08:40:22.463 | INFO | "minSampleSize" : 1000, | 20260506 08:40:22.463 | INFO | "name" : "FullChip_LP0P5X", | 20260506 08:40:22.463 | INFO | "roi" : | 20260506 08:40:22.463 | INFO | [ | 20260506 08:40:22.463 | INFO | [ | 20260506 08:40:22.463 | INFO | 0, | 20260506 08:40:22.463 | INFO | 0, | 20260506 08:40:22.463 | INFO | 4096, | 20260506 08:40:22.463 | INFO | 6144 | 20260506 08:40:22.463 | INFO | ] | 20260506 08:40:22.463 | INFO | ], | 20260506 08:40:22.463 | INFO | "useSingleActivityLabels" : true | 20260506 08:40:22.463 | INFO | }, | 20260506 08:40:22.463 | INFO | { | 20260506 08:40:22.463 | INFO | "featuresForFilter" : | 20260506 08:40:22.463 | INFO | [ | 20260506 08:40:22.463 | INFO | "LaserPower0p0x" | 20260506 08:40:22.463 | INFO | ], | 20260506 08:40:22.463 | INFO | "medianIntraLaneStride" : 1, | 20260506 08:40:22.463 | INFO | "metrics" : | 20260506 08:40:22.463 | INFO | [ | 20260506 08:40:22.463 | INFO | "Baseline", | 20260506 08:40:22.463 | INFO | "BaselineStd", | 20260506 08:40:22.463 | INFO | "Pkmid", | 20260506 08:40:22.463 | INFO | "Snr" | 20260506 08:40:22.463 | INFO | ], | 20260506 08:40:22.463 | INFO | "minSampleSize" : 1000, | 20260506 08:40:22.463 | INFO | "name" : "FullChip_LP0P0X", | 20260506 08:40:22.463 | INFO | "roi" : | 20260506 08:40:22.463 | INFO | [ | 20260506 08:40:22.463 | INFO | [ | 20260506 08:40:22.463 | INFO | 0, | 20260506 08:40:22.463 | INFO | 0, | 20260506 08:40:22.463 | INFO | 4096, | 20260506 08:40:22.463 | INFO | 6144 | 20260506 08:40:22.463 | INFO | ] | 20260506 08:40:22.463 | INFO | ], | 20260506 08:40:22.463 | INFO | "useSingleActivityLabels" : true | 20260506 08:40:22.464 | INFO | }, | 20260506 08:40:22.464 | INFO | { | 20260506 08:40:22.464 | INFO | "featuresForFilter" : | 20260506 08:40:22.464 | INFO | [ | 20260506 08:40:22.464 | INFO | "LaserScatter" | 20260506 08:40:22.464 | INFO | ], | 20260506 08:40:22.464 | INFO | "medianIntraLaneStride" : 1, | 20260506 08:40:22.464 | INFO | "metrics" : | 20260506 08:40:22.464 | INFO | [ | 20260506 08:40:22.464 | INFO | "Baseline", | 20260506 08:40:22.464 | INFO | "BaselineStd" | 20260506 08:40:22.464 | INFO | ], | 20260506 08:40:22.464 | INFO | "minSampleSize" : 1000, | 20260506 08:40:22.464 | INFO | "name" : "FullChip_LS", | 20260506 08:40:22.464 | INFO | "roi" : | 20260506 08:40:22.464 | INFO | [ | 20260506 08:40:22.464 | INFO | [ | 20260506 08:40:22.464 | INFO | 0, | 20260506 08:40:22.464 | INFO | 0, | 20260506 08:40:22.464 | INFO | 4096, | 20260506 08:40:22.464 | INFO | 6144 | 20260506 08:40:22.464 | INFO | ] | 20260506 08:40:22.464 | INFO | ], | 20260506 08:40:22.464 | INFO | "useSingleActivityLabels" : false | 20260506 08:40:22.464 | INFO | } | 20260506 08:40:22.464 | INFO | ] | 20260506 08:40:22.464 | INFO | }, | 20260506 08:40:22.464 | INFO | "reportChecking" : | 20260506 08:40:22.464 | INFO | { | 20260506 08:40:22.464 | INFO | "activeFractionThreshold" : 0.97999999999999998, | 20260506 08:40:22.464 | INFO | "growingQueueSizeThreshold" : 2, | 20260506 08:40:22.464 | INFO | "queueSizeThreshold" : 20, | 20260506 08:40:22.464 | INFO | "totalTimeMsThreshold" : 1000000 | 20260506 08:40:22.464 | INFO | }, | 20260506 08:40:22.464 | INFO | "smrtMask" : | 20260506 08:40:22.464 | INFO | { | 20260506 08:40:22.464 | INFO | "smrtMaskScoreFilePath" : "" | 20260506 08:40:22.464 | INFO | }, | 20260506 08:40:22.464 | INFO | "source" : | 20260506 08:40:22.464 | INFO | { | 20260506 08:40:22.464 | INFO | "WXIPCDataSourceConfig" : | 20260506 08:40:22.464 | INFO | { | 20260506 08:40:22.464 | INFO | "acqConfig" : | 20260506 08:40:22.464 | INFO | { | 20260506 08:40:22.464 | INFO | "A" : | 20260506 08:40:22.464 | INFO | { | 20260506 08:40:22.464 | INFO | "baseLabel" : "A", | 20260506 08:40:22.464 | INFO | "excessNoiseCV" : 0.10000000149011612, | 20260506 08:40:22.464 | INFO | "interPulseDistance" : 0.079999998211860657, | 20260506 08:40:22.464 | INFO | "ipd2SlowStepRatio" : 0, | 20260506 08:40:22.464 | INFO | "pulseWidth" : 0.16599999368190765, | 20260506 08:40:22.464 | INFO | "pw2SlowStepRatio" : 3.2000000476837158, | 20260506 08:40:22.464 | INFO | "relAmplitude" : 0.67000001668930054 | 20260506 08:40:22.464 | INFO | }, | 20260506 08:40:22.464 | INFO | "C" : | 20260506 08:40:22.464 | INFO | { | 20260506 08:40:22.464 | INFO | "baseLabel" : "C", | 20260506 08:40:22.464 | INFO | "excessNoiseCV" : 0.10000000149011612, | 20260506 08:40:22.464 | INFO | "interPulseDistance" : 0.070000000298023224, | 20260506 08:40:22.464 | INFO | "ipd2SlowStepRatio" : 0, | 20260506 08:40:22.464 | INFO | "pulseWidth" : 0.20900000631809235, | 20260506 08:40:22.464 | INFO | "pw2SlowStepRatio" : 3.2000000476837158, | 20260506 08:40:22.464 | INFO | "relAmplitude" : 1 | 20260506 08:40:22.464 | INFO | }, | 20260506 08:40:22.464 | INFO | "G" : | 20260506 08:40:22.464 | INFO | { | 20260506 08:40:22.464 | INFO | "baseLabel" : "G", | 20260506 08:40:22.464 | INFO | "excessNoiseCV" : 0.10000000149011612, | 20260506 08:40:22.464 | INFO | "interPulseDistance" : 0.070000000298023224, | 20260506 08:40:22.464 | INFO | "ipd2SlowStepRatio" : 0, | 20260506 08:40:22.464 | INFO | "pulseWidth" : 0.19300000369548798, | 20260506 08:40:22.464 | INFO | "pw2SlowStepRatio" : 3.2000000476837158, | 20260506 08:40:22.464 | INFO | "relAmplitude" : 0.25999999046325684 | 20260506 08:40:22.464 | INFO | }, | 20260506 08:40:22.464 | INFO | "T" : | 20260506 08:40:22.464 | INFO | { | 20260506 08:40:22.464 | INFO | "baseLabel" : "T", | 20260506 08:40:22.464 | INFO | "excessNoiseCV" : 0.10000000149011612, | 20260506 08:40:22.464 | INFO | "interPulseDistance" : 0.079999998211860657, | 20260506 08:40:22.464 | INFO | "ipd2SlowStepRatio" : 0, | 20260506 08:40:22.464 | INFO | "pulseWidth" : 0.16300000250339508, | 20260506 08:40:22.464 | INFO | "pw2SlowStepRatio" : 3.2000000476837158, | 20260506 08:40:22.464 | INFO | "relAmplitude" : 0.44499999284744263 | 20260506 08:40:22.464 | INFO | }, | 20260506 08:40:22.464 | INFO | "chipLayoutName" : "KestrelRTO2", | 20260506 08:40:22.464 | INFO | "photoelectronSensitivity" : 6.25, | 20260506 08:40:22.464 | INFO | "refSnr" : 16.700000762939453 | 20260506 08:40:22.464 | INFO | }, | 20260506 08:40:22.464 | INFO | "autoConnect" : true, | 20260506 08:40:22.464 | INFO | "canonicalFrameRate" : 100, | 20260506 08:40:22.464 | INFO | "chunkValidationLevel" : 0, | 20260506 08:40:22.464 | INFO | "daemonUrl" : "http://localhost:23602", | 20260506 08:40:22.464 | INFO | "dataMode" : "FRAME", | 20260506 08:40:22.464 | INFO | "dataPath" : "Normal", | 20260506 08:40:22.464 | INFO | "loopback" : false, | 20260506 08:40:22.464 | INFO | "maskCardinalityFraction" : 1, | 20260506 08:40:22.465 | INFO | "maskEncoding" : "COMPRESSEDBASE64_V2", | 20260506 08:40:22.465 | INFO | "maxPopLoops" : 10, | 20260506 08:40:22.465 | INFO | "pedestal" : -32768, | 20260506 08:40:22.465 | INFO | "pointerReplacementSafetyFactor" : 0.80000000000000004, | 20260506 08:40:22.465 | INFO | "primerScaler" : 1, | 20260506 08:40:22.465 | INFO | "simConfig" : | 20260506 08:40:22.465 | INFO | { | 20260506 08:40:22.465 | INFO | "numCols" : 0, | 20260506 08:40:22.465 | INFO | "numRows" : 0, | 20260506 08:40:22.465 | INFO | "numSubSensors" : 2, | 20260506 08:40:22.465 | INFO | "platformOverride" : "DONT_CARE", | 20260506 08:40:22.465 | INFO | "regionNumSlotsPerTrack" : 128, | 20260506 08:40:22.465 | INFO | "regionNumTracks" : 8, | 20260506 08:40:22.465 | INFO | "regionSize" : 1024 | 20260506 08:40:22.465 | INFO | }, | 20260506 08:40:22.465 | INFO | "simulatedFrameRate" : 100, | 20260506 08:40:22.465 | INFO | "simulatedInputFile" : "CONSTANT/123", | 20260506 08:40:22.465 | INFO | "simulationCondensePixels" : true, | 20260506 08:40:22.465 | INFO | "simulationNumRowsToReplicate" : 20, | 20260506 08:40:22.465 | INFO | "simulationRoi" : [], | 20260506 08:40:22.465 | INFO | "sleepDebug" : 0, | 20260506 08:40:22.465 | INFO | "smrtMaskMode" : "DEFAULT", | 20260506 08:40:22.465 | INFO | "sraIndex" : 0, | 20260506 08:40:22.465 | INFO | "tilePoolFactor" : 3, | 20260506 08:40:22.465 | INFO | "timeouts" : | 20260506 08:40:22.465 | INFO | { | 20260506 08:40:22.465 | INFO | "armedTimeoutSeconds" : 400, | 20260506 08:40:22.465 | INFO | "doneTimeoutSeconds" : 10, | 20260506 08:40:22.465 | INFO | "firstChunkTimeoutSeconds" : 400, | 20260506 08:40:22.465 | INFO | "httpTimeoutSeconds" : 10, | 20260506 08:40:22.465 | INFO | "notifiedTimeoutSeconds" : 15, | 20260506 08:40:22.465 | INFO | "resetTimeoutSeconds" : 15, | 20260506 08:40:22.465 | INFO | "subsequentChunkTimeoutSeconds" : -1, | 20260506 08:40:22.465 | INFO | "wxDaemonTimeoutSeconds" : 60 | 20260506 08:40:22.465 | INFO | }, | 20260506 08:40:22.465 | INFO | "verbosity" : 0 | 20260506 08:40:22.465 | INFO | } | 20260506 08:40:22.465 | INFO | }, | 20260506 08:40:22.465 | INFO | "system" : | 20260506 08:40:22.465 | INFO | { | 20260506 08:40:22.465 | INFO | "analyzerHardware" : "A100", | 20260506 08:40:22.465 | INFO | "basecallerBatchConcurrency" : 1, | 20260506 08:40:22.465 | INFO | "basecallerConcurrency" : 3, | 20260506 08:40:22.465 | INFO | "basecallerDeviceConcurrency" : 3, | 20260506 08:40:22.465 | INFO | "basecallerGpuDevices" : | 20260506 08:40:22.465 | INFO | [ | 20260506 08:40:22.465 | INFO | 0 | 20260506 08:40:22.465 | INFO | ], | 20260506 08:40:22.465 | INFO | "ioConcurrency" : 5, | 20260506 08:40:22.465 | INFO | "maxPermGpuDataMB" : 10000, | 20260506 08:40:22.465 | INFO | "numWorkerThreads" : 12, | 20260506 08:40:22.465 | INFO | "nvidiaSmiRunTimeSeconds" : 30, | 20260506 08:40:22.465 | INFO | "serialGpuCompute" : true | 20260506 08:40:22.465 | INFO | }, | 20260506 08:40:22.465 | INFO | "traceSaver" : | 20260506 08:40:22.465 | INFO | { | 20260506 08:40:22.465 | INFO | "bufferedChunksAllowed" : 5, | 20260506 08:40:22.465 | INFO | "frameChunking" : 512, | 20260506 08:40:22.465 | INFO | "outFormat" : "Natural", | 20260506 08:40:22.465 | INFO | "roi" : | 20260506 08:40:22.465 | INFO | [ | 20260506 08:40:22.465 | INFO | [ | 20260506 08:40:22.465 | INFO | 2048, | 20260506 08:40:22.465 | INFO | 768, | 20260506 08:40:22.465 | INFO | 1, | 20260506 08:40:22.465 | INFO | 64 | 20260506 08:40:22.465 | INFO | ], | 20260506 08:40:22.465 | INFO | [ | 20260506 08:40:22.465 | INFO | 2048, | 20260506 08:40:22.465 | INFO | 2304, | 20260506 08:40:22.465 | INFO | 1, | 20260506 08:40:22.465 | INFO | 64 | 20260506 08:40:22.465 | INFO | ], | 20260506 08:40:22.465 | INFO | [ | 20260506 08:40:22.465 | INFO | 2048, | 20260506 08:40:22.465 | INFO | 3840, | 20260506 08:40:22.465 | INFO | 1, | 20260506 08:40:22.465 | INFO | 64 | 20260506 08:40:22.465 | INFO | ], | 20260506 08:40:22.465 | INFO | [ | 20260506 08:40:22.465 | INFO | 2048, | 20260506 08:40:22.465 | INFO | 5376, | 20260506 08:40:22.465 | INFO | 1, | 20260506 08:40:22.465 | INFO | 64 | 20260506 08:40:22.465 | INFO | ] | 20260506 08:40:22.465 | INFO | ], | 20260506 08:40:22.465 | INFO | "zmwChunking" : 2048 | 20260506 08:40:22.465 | INFO | } | 20260506 08:40:22.465 | INFO | } | 20260506 08:40:22.465 | INFO | Output Target: /data/nrta/0/m84031_260506_033729_s1/pb_internal/baz/m84031_260506_033729_s1.bazfilelist.txt | 20260506 08:40:22.513 | INFO | Found 1 CUDA devices | 20260506 08:40:22.513 | INFO | CUDA GPU device: 0 | 20260506 08:40:22.513 | INFO | Device Name:NVIDIA A100 80GB PCIe | 20260506 08:40:22.513 | INFO | UUID:GPU-ce26be3f-889e-2795-9302-85bf090c57ae | 20260506 08:40:22.513 | INFO | Global memory:85095874560 | 20260506 08:40:22.513 | INFO | sharedMemPerBlock:49152 | 20260506 08:40:22.513 | INFO | major/minor:8/0 | 20260506 08:40:22.513 | INFO | No message | 20260506 08:40:22.513 | INFO | CpuSchedulerCanary: starting | 20260506 08:40:22.513 | INFO | Starting CPU scheduling canary | 20260506 08:40:22.518 | INFO | Waiting for wx-daemon. Current boot state:"Daemon ready" | 20260506 08:40:22.519 | INFO | Connecting to shared memory key: 0x50420051 at 0x00500000000 | 20260506 08:40:26.647 | WARN | Precalculated crosstalk filter will not be used, one of the dimensions is zero | 20260506 08:40:26.647 | INFO | PSF will be passed to the DataSource for processing. | 20260506 08:40:26.648 | INFO | WXIPCDataSource expectedMaskedZmws:25165824 | 20260506 08:40:26.648 | INFO | WXIPCDataSource expectedSensorZmws:25165824 | 20260506 08:40:26.648 | WARN | PSF:[ [ 0.00079984 0.0029994 0.0165967 0.0034993 0.00109978] [ 0.00269946 0.0129974 0.0726855 0.0139972 0.00319936] [ 0.0117976 0.0544891 0.594281 0.0652869 0.0123975] [ 0.00319936 0.0130974 0.0726855 0.0140972 0.00309938] [ 0.0009998 0.00289942 0.0167966 0.00339932 0.00089982] ] | 20260506 08:40:26.648 | WARN | bNorm:0.811438 | 20260506 08:40:26.648 | WARN | PSF is not that close to the identity PSF. Norm of deviation = 0.811438 | 20260506 08:40:26.649 | INFO | Based on platform 6 the smrtMaskMode default is set to DISABLED | 20260506 08:40:26.649 | INFO | Connecting to wx-daemon http://localhost:23602 | 20260506 08:40:26.649 | INFO | platform:Kestrel | 20260506 08:40:26.650 | INFO | wxShortVersion: 26.1.0 wxLongVersion: 26.1.0.a8206dc561f8e1cdf6683a2ad782923bb7106e43 wxIpcVersion: 3 wxPersonality: 1.8.2-0 dataSourceShortVersion: 26.1.0 dataSourceLongVersion: 26.1.0.0 dataSourceIpcVersion: 3 | 20260506 08:40:26.651 | INFO | Waiting for wx-daemon. Current boot state:"Daemon ready" | 20260506 08:40:26.752 | INFO | Connecting IPC | 20260506 08:40:26.753 | INFO | Reconfiguring:{ "cameraConfig" : 0, "cardinalityFraction" : 1, "crosstalkKernel" : [ [ 0.00014065283436721698, 0.00024733493649707156, -0.02646965601969033, 1.2938141044737791e-05, -0.000209726572126589 ], [ 2.8209306317858476e-05, -0.011227287975635325, -0.16249877620543987, -0.009973250094704407, -0.00028787427166992058 ], [ -0.019849015808807709, -0.11953626080922616, 1.7245260129463982, -0.14535734185743521, -0.018293397259077079 ], [ -0.0011299207645029945, -0.011351514333903183, -0.16237503050695248, -0.010276039796743052, -7.8888800056879383e-05 ], [ -0.00019585354582084264, 0.00063817726959822143, -0.026998977274235791, 0.00028844750081735671, 0.00022703896098751053 ] ], "darkCalFileName" : "/data/icc/m84031_260506_033729_s1/pb_internal/m84031_260506_033729_s1.darkcal.h5", "encoding" : "UINT8", "expectedZmws" : 0, "layoutDims" : [ 4096, 512, 64 ], "layoutType" : "BLOCK_LAYOUT_DENSE", "memoryPoolIndex" : -1, "mode" : "AUTO", "mosi_socket_address" : "tcp://127.0.0.1:45617", "numFrames" : 8640000, "pedestal" : -32768, "requireAuroraUp" : true, "smrtMaskType" : "" } | 20260506 08:40:26.754 | INFO | Reconfigure returned:OK "ok" | 20260506 08:40:26.855 | INFO | Connecting to SRA:0 using tcp://127.0.0.1:40621 | 20260506 08:40:26.856 | INFO | wx-daemon requests 192 pointers | 20260506 08:40:26.856 | INFO | wxipc is sending requests 192 pointers | 20260506 08:40:26.858 | INFO | Pushing 192 free packets, with packetSize 134217728 | 20260506 08:40:26.858 | INFO | Done, Pushed 192 free packets | 20260506 08:40:26.858 | INFO | Sent first free packet to tcp://127.0.0.1:40621 | 20260506 08:40:26.858 | INFO | bursted 192 free packets | 20260506 08:40:26.859 | INFO | Reconfigure_continue returned:OK "ok" | 20260506 08:40:33.297 | INFO | Polled wx-daemon for pedestal and cached 20 | 20260506 08:40:33.297 | INFO | Chip layout: KestrelRTO2 | 20260506 08:40:33.297 | INFO | Frame rate: 100 | 20260506 08:40:33.297 | INFO | Photo-electron sensitivity: 6.25 | 20260506 08:40:33.297 | INFO | Reference SNR: 16.7 | 20260506 08:40:33.297 | INFO | Analog properties: [C, RA 1, exnCV 0.1, ipd 0.07, pw 0.209, pw2ssr 3.2, ipd2ssr 0] | 20260506 08:40:33.297 | INFO | Analog properties: [A, RA 0.67, exnCV 0.1, ipd 0.08, pw 0.166, pw2ssr 3.2, ipd2ssr 0] | 20260506 08:40:33.297 | INFO | Analog properties: [T, RA 0.445, exnCV 0.1, ipd 0.08, pw 0.163, pw2ssr 3.2, ipd2ssr 0] | 20260506 08:40:33.297 | INFO | Analog properties: [G, RA 0.26, exnCV 0.1, ipd 0.07, pw 0.193, pw2ssr 3.2, ipd2ssr 0] | 20260506 08:40:33.297 | INFO | Number of analysis zmwLanes (after smrt masking) = 393216 | 20260506 08:40:33.297 | INFO | Number of analysis chunks = 16875 | 20260506 08:40:33.297 | INFO | Instantiating TrivialRepacker | 20260506 08:40:33.297 | WARN | WXIPCDataSource returning dummy instrument name WXIPCDataSource | 20260506 08:40:33.300 | INFO | OOM score went from -1 to 666 at CreateTracePrepper checkpoint 1 | 20260506 08:40:33.300 | INFO | memory usage change: vm:26471823 RSS:25864 shared:22463 text:4978 data:23695 at CreateTracePrepper checkpoint 1 | 20260506 08:40:33.301 | INFO | Added 2 lanes to the ROI | 20260506 08:40:33.301 | INFO | Added 2 lanes to the ROI | 20260506 08:40:33.301 | INFO | Added 2 lanes to the ROI | 20260506 08:40:33.301 | INFO | Added 2 lanes to the ROI | 20260506 08:40:33.975 | INFO | Creating cached UnitCellProperties for dataSource 0x474ee90 | 20260506 08:40:34.460 | INFO | Done creating cached UnitCellProperties for dataSource 0x474ee90 | 20260506 08:40:34.460 | INFO | memory usage change: vm:26618841 RSS:173438 shared:22512 text:4978 data:170713 at SmrtBasecaller::CreateTraceSaver checkpoint 2 | 20260506 08:40:34.463 | INFO | TraceSaverBody created | 20260506 08:40:34.483 | INFO | memory usage change: vm:26570370 RSS:124784 shared:22612 text:4978 data:122242 at SmrtBasecaller::RunAnalyzer checkpoint 1 | 20260506 08:40:34.483 | INFO | Resource: 0, Device: 0, Assigning 3 CudaStreams | 20260506 08:40:34.483 | INFO | TraceHistogramAccumulator: BinSizeCoeff = 0.2. | 20260506 08:40:34.483 | INFO | TraceHistogramAccumulator: BaselineStatMinFrameCount = 50. | 20260506 08:40:34.483 | INFO | TraceHistogramAccumulator: FallBackBaselineSigma = 10. | 20260506 08:40:34.483 | INFO | DetectionModelEstimator: NumFramesPreAccumStats = 1000. | 20260506 08:40:34.483 | INFO | DetectionModelEstimator: MinFramesForEstimate= 4000. | 20260506 08:40:34.483 | INFO | Pulse signal variance shot noise coefficient = 1.77377. | 20260506 08:40:35.044 | INFO | OOM score went from 666 to 667 at BasecallerBody checkpoint 3 | 20260506 08:40:35.044 | INFO | memory usage change: vm:68517105 RSS:338968 shared:232897 text:4978 data:125937 at BasecallerBody checkpoint 3 | 20260506 08:40:52.437 | INFO | OOM score went from 667 to 684 at SmrtBasecaller::RunAnalyzer checkpoint 2 | 20260506 08:40:52.437 | INFO | memory usage change: vm:75587989 RSS:6877390 shared:6770117 text:4978 data:127125 at SmrtBasecaller::RunAnalyzer checkpoint 2 | 20260506 08:40:52.438 | INFO | CreateRealTimeMetrics: 9 regions | 20260506 08:40:52.438 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260506 08:40:52.472 | INFO | Added 196608 lanes to the ROI | 20260506 08:40:52.529 | INFO | Added 196608 lanes to the ROI | 20260506 08:40:52.648 | INFO | memory usage change: vm:75593765 RSS:6883132 shared:6770145 text:4978 data:132901 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260506 08:40:52.689 | INFO | memory usage change: vm:75618342 RSS:6907750 shared:6770145 text:4978 data:157478 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260506 08:40:52.762 | INFO | memory usage change: vm:75642919 RSS:6932302 shared:6770145 text:4978 data:182055 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260506 08:40:52.772 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:200,64,64,6016 | 20260506 08:40:52.772 | INFO | memory usage change: vm:75618342 RSS:6907759 shared:6770145 text:4978 data:157478 at SmrtBasecaller::SelectedBasecallerLanesWithinROI | 20260506 08:40:52.773 | INFO | Added 3008 lanes to the ROI | 20260506 08:40:52.774 | INFO | Added 3008 lanes to the ROI | 20260506 08:40:52.775 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:2016,64,64,6016 | 20260506 08:40:52.776 | INFO | Added 3008 lanes to the ROI | 20260506 08:40:52.776 | INFO | Added 3008 lanes to the ROI | 20260506 08:40:52.778 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:3832,64,64,6016 | 20260506 08:40:52.778 | INFO | Added 3008 lanes to the ROI | 20260506 08:40:52.779 | INFO | Added 3008 lanes to the ROI | 20260506 08:40:52.780 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260506 08:40:52.807 | INFO | Added 196608 lanes to the ROI | 20260506 08:40:52.856 | INFO | Added 196608 lanes to the ROI | 20260506 08:40:52.970 | INFO | memory usage change: vm:75619873 RSS:6909276 shared:6770145 text:4978 data:159009 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260506 08:40:53.012 | INFO | memory usage change: vm:75644450 RSS:6933828 shared:6770145 text:4978 data:183586 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260506 08:40:53.085 | INFO | memory usage change: vm:75669027 RSS:6958380 shared:6770145 text:4978 data:208163 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260506 08:40:53.094 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260506 08:40:53.094 | INFO | memory usage change: vm:75644450 RSS:6933867 shared:6770145 text:4978 data:183586 at SmrtBasecaller::SelectedBasecallerLanesWithinROI | 20260506 08:40:53.122 | INFO | Added 196608 lanes to the ROI | 20260506 08:40:53.171 | INFO | Added 196608 lanes to the ROI | 20260506 08:40:53.282 | INFO | memory usage change: vm:75645218 RSS:6934592 shared:6770145 text:4978 data:184354 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260506 08:40:53.323 | INFO | memory usage change: vm:75669795 RSS:6959210 shared:6770145 text:4978 data:208931 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260506 08:40:53.397 | INFO | memory usage change: vm:75694372 RSS:6983762 shared:6770145 text:4978 data:233508 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260506 08:40:53.407 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260506 08:40:53.407 | INFO | memory usage change: vm:75669795 RSS:6959212 shared:6770145 text:4978 data:208931 at SmrtBasecaller::SelectedBasecallerLanesWithinROI | 20260506 08:40:53.430 | INFO | Added 196608 lanes to the ROI | 20260506 08:40:53.477 | INFO | Added 196608 lanes to the ROI | 20260506 08:40:53.595 | INFO | memory usage change: vm:75670179 RSS:6959541 shared:6770145 text:4978 data:209315 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260506 08:40:53.636 | INFO | memory usage change: vm:75694756 RSS:6984159 shared:6770145 text:4978 data:233892 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260506 08:40:53.709 | INFO | memory usage change: vm:75719333 RSS:7008711 shared:6770145 text:4978 data:258469 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260506 08:40:53.719 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260506 08:40:53.719 | INFO | memory usage change: vm:75694756 RSS:6984173 shared:6770145 text:4978 data:233892 at SmrtBasecaller::SelectedBasecallerLanesWithinROI | 20260506 08:40:53.742 | INFO | Added 196608 lanes to the ROI | 20260506 08:40:53.785 | INFO | Added 196608 lanes to the ROI | 20260506 08:40:53.890 | INFO | memory usage change: vm:75695140 RSS:6984502 shared:6770145 text:4978 data:234276 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260506 08:40:53.931 | INFO | memory usage change: vm:75719717 RSS:7009120 shared:6770145 text:4978 data:258853 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260506 08:40:54.004 | INFO | memory usage change: vm:75744294 RSS:7033672 shared:6770145 text:4978 data:283430 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260506 08:40:54.014 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1109 region.roi:0,0,4096,6144 | 20260506 08:40:54.014 | INFO | memory usage change: vm:75719717 RSS:7009134 shared:6770145 text:4978 data:258853 at SmrtBasecaller::SelectedBasecallerLanesWithinROI | 20260506 08:40:54.045 | INFO | Added 196608 lanes to the ROI | 20260506 08:40:54.090 | INFO | Added 196608 lanes to the ROI | 20260506 08:40:54.195 | INFO | memory usage change: vm:75720101 RSS:7009463 shared:6770145 text:4978 data:259237 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 2 | 20260506 08:40:54.236 | INFO | memory usage change: vm:75744678 RSS:7034081 shared:6770145 text:4978 data:283814 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 4 | 20260506 08:40:54.309 | INFO | memory usage change: vm:75769255 RSS:7058633 shared:6770145 text:4978 data:308391 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 7 | 20260506 08:40:54.318 | INFO | SmrtBasecaller::CreateRealTimeMetrics line 1139: properties size:9 | 20260506 08:40:54.318 | INFO | memory usage change: vm:75744678 RSS:7034095 shared:6770145 text:4978 data:283814 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 8 | 20260506 08:40:54.510 | INFO | memory usage change: vm:75799976 RSS:7089336 shared:6770145 text:4978 data:339112 at SmrtBasecaller::CreateRealTimeMetrics checkpoint 9 | 20260506 08:40:55.500 | INFO | memory usage change: vm:75641920 RSS:6931401 shared:6770209 text:4978 data:181056 at SmrtBasecaller::RunAnalyzer checkpoint 4 | 20260506 08:40:58.637 | INFO | OOM score went from 684 to 687 at SmrtBasecaller::RunAnalyzer checkpoint 6 | 20260506 08:40:58.637 | INFO | memory usage change: vm:80072365 RSS:8385989 shared:6770225 text:4978 data:4611501 at SmrtBasecaller::RunAnalyzer checkpoint 6 | 20260506 08:40:58.637 | INFO | AddNode CreateBazSaver started | 20260506 08:40:59.056 | INFO | OOM score went from 687 to 688 at SmrtBasecaller::CreateBazSaver checkpoint 3 | 20260506 08:40:59.056 | INFO | memory usage change: vm:80154454 RSS:8449516 shared:6770225 text:4978 data:4693590 at SmrtBasecaller::CreateBazSaver checkpoint 3 | 20260506 08:41:01.828 | INFO | memory usage change: vm:80351144 RSS:8812809 shared:6770225 text:4978 data:4890280 at SmrtBasecaller::CreateBazSaver checkpoint 4 | 20260506 08:41:01.829 | INFO | Enabling real-time activity labels in BAZ file. | 20260506 08:41:01.829 | INFO | Opening 96 BAZ files for writing with filename prefix: /data/nrta/0/m84031_260506_033729_s1/pb_internal/baz/m84031_260506_033729_s1 zmws: 25165824 | 20260506 08:41:01.829 | INFO | Generating BAZ file list to file: /data/nrta/0/m84031_260506_033729_s1/pb_internal/baz/m84031_260506_033729_s1.bazfilelist.txt | 20260506 08:41:04.884 | INFO | Opened 1 baz files so far | 20260506 08:41:07.364 | INFO | Opened 2 baz files so far | 20260506 08:41:07.466 | INFO | Opened 3 baz files so far | 20260506 08:41:07.656 | INFO | Opened 4 baz files so far | 20260506 08:41:07.666 | INFO | Opened 5 baz files so far | 20260506 08:41:07.674 | INFO | Opened 6 baz files so far | 20260506 08:41:07.677 | INFO | Opened 7 baz files so far | 20260506 08:41:07.678 | INFO | Opened 8 baz files so far | 20260506 08:41:07.680 | INFO | Opened 9 baz files so far | 20260506 08:41:07.685 | INFO | Opened 10 baz files so far | 20260506 08:41:07.687 | INFO | Opened 11 baz files so far | 20260506 08:41:07.695 | INFO | Opened 12 baz files so far | 20260506 08:41:07.701 | INFO | Opened 13 baz files so far | 20260506 08:41:09.953 | INFO | Opened 14 baz files so far | 20260506 08:41:09.978 | INFO | Opened 15 baz files so far | 20260506 08:41:10.002 | INFO | Opened 16 baz files so far | 20260506 08:41:10.352 | INFO | Opened 17 baz files so far | 20260506 08:41:10.378 | INFO | Opened 18 baz files so far | 20260506 08:41:10.378 | INFO | Opened 19 baz files so far | 20260506 08:41:10.381 | INFO | Opened 20 baz files so far | 20260506 08:41:10.382 | INFO | Opened 21 baz files so far | 20260506 08:41:10.382 | INFO | Opened 22 baz files so far | 20260506 08:41:10.385 | INFO | Opened 23 baz files so far | 20260506 08:41:10.385 | INFO | Opened 24 baz files so far | 20260506 08:41:10.395 | INFO | Opened 25 baz files so far | 20260506 08:41:12.484 | INFO | Opened 26 baz files so far | 20260506 08:41:12.589 | INFO | Opened 27 baz files so far | 20260506 08:41:12.615 | INFO | Opened 28 baz files so far | 20260506 08:41:13.007 | INFO | Opened 29 baz files so far | 20260506 08:41:13.020 | INFO | Opened 30 baz files so far | 20260506 08:41:13.025 | INFO | Opened 31 baz files so far | 20260506 08:41:13.026 | INFO | Opened 32 baz files so far | 20260506 08:41:13.028 | INFO | Opened 33 baz files so far | 20260506 08:41:13.029 | INFO | Opened 34 baz files so far | 20260506 08:41:13.030 | INFO | Opened 35 baz files so far | 20260506 08:41:13.032 | INFO | Opened 36 baz files so far | 20260506 08:41:13.057 | INFO | Opened 37 baz files so far | 20260506 08:41:14.925 | INFO | Opened 38 baz files so far | 20260506 08:41:14.945 | INFO | Opened 39 baz files so far | 20260506 08:41:15.162 | INFO | Opened 40 baz files so far | 20260506 08:41:15.703 | INFO | Opened 41 baz files so far | 20260506 08:41:15.719 | INFO | Opened 42 baz files so far | 20260506 08:41:15.732 | INFO | Opened 43 baz files so far | 20260506 08:41:15.733 | INFO | Opened 44 baz files so far | 20260506 08:41:15.748 | INFO | Opened 45 baz files so far | 20260506 08:41:15.756 | INFO | Opened 46 baz files so far | 20260506 08:41:15.759 | INFO | Opened 47 baz files so far | 20260506 08:41:15.762 | INFO | Opened 48 baz files so far | 20260506 08:41:15.766 | INFO | Opened 49 baz files so far | 20260506 08:41:17.277 | INFO | Opened 50 baz files so far | 20260506 08:41:17.323 | INFO | Opened 51 baz files so far | 20260506 08:41:17.534 | INFO | Opened 52 baz files so far | 20260506 08:41:18.324 | INFO | Opened 53 baz files so far | 20260506 08:41:18.324 | INFO | Opened 54 baz files so far | 20260506 08:41:18.326 | INFO | Opened 55 baz files so far | 20260506 08:41:18.327 | INFO | Opened 56 baz files so far | 20260506 08:41:18.328 | INFO | Opened 57 baz files so far | 20260506 08:41:18.328 | INFO | Opened 58 baz files so far | 20260506 08:41:18.329 | INFO | Opened 59 baz files so far | 20260506 08:41:18.339 | INFO | Opened 60 baz files so far | 20260506 08:41:18.360 | INFO | Opened 61 baz files so far | 20260506 08:41:19.512 | INFO | Opened 62 baz files so far | 20260506 08:41:19.856 | INFO | Opened 63 baz files so far | 20260506 08:41:19.932 | INFO | Opened 64 baz files so far | 20260506 08:41:21.009 | INFO | Opened 65 baz files so far | 20260506 08:41:21.010 | INFO | Opened 66 baz files so far | 20260506 08:41:21.014 | INFO | Opened 67 baz files so far | 20260506 08:41:21.014 | INFO | Opened 68 baz files so far | 20260506 08:41:21.016 | INFO | Opened 69 baz files so far | 20260506 08:41:21.018 | INFO | Opened 70 baz files so far | 20260506 08:41:21.018 | INFO | Opened 71 baz files so far | 20260506 08:41:21.022 | INFO | Opened 72 baz files so far | 20260506 08:41:21.040 | INFO | Opened 73 baz files so far | 20260506 08:41:21.834 | INFO | Opened 74 baz files so far | 20260506 08:41:22.278 | INFO | Opened 75 baz files so far | 20260506 08:41:22.335 | INFO | Opened 76 baz files so far | 20260506 08:41:23.428 | INFO | Opened 77 baz files so far | 20260506 08:41:23.626 | INFO | Opened 78 baz files so far | 20260506 08:41:23.674 | INFO | Opened 79 baz files so far | 20260506 08:41:23.677 | INFO | Opened 80 baz files so far | 20260506 08:41:23.677 | INFO | Opened 81 baz files so far | 20260506 08:41:23.681 | INFO | Opened 82 baz files so far | 20260506 08:41:23.682 | INFO | Opened 83 baz files so far | 20260506 08:41:23.683 | INFO | Opened 84 baz files so far | 20260506 08:41:23.698 | INFO | Opened 85 baz files so far | 20260506 08:41:24.129 | INFO | Opened 86 baz files so far | 20260506 08:41:24.759 | INFO | Opened 87 baz files so far | 20260506 08:41:24.791 | INFO | Opened 88 baz files so far | 20260506 08:41:25.938 | INFO | Opened 89 baz files so far | 20260506 08:41:26.029 | INFO | Opened 90 baz files so far | 20260506 08:41:26.241 | INFO | Opened 91 baz files so far | 20260506 08:41:26.251 | INFO | Opened 92 baz files so far | 20260506 08:41:26.252 | INFO | Opened 93 baz files so far | 20260506 08:41:26.254 | INFO | Opened 94 baz files so far | 20260506 08:41:26.268 | INFO | Opened 95 baz files so far | 20260506 08:41:26.301 | INFO | Opened 96 baz files so far | 20260506 08:41:26.311 | INFO | Finished opening a total of 96 baz files | 20260506 08:41:26.311 | INFO | OOM score went from 688 to 692 at BazWriterBody checkpoint 3 | 20260506 08:41:26.311 | INFO | memory usage change: vm:81604165 RSS:10054979 shared:6770339 text:4978 data:6143290 at BazWriterBody checkpoint 3 | 20260506 08:41:26.974 | INFO | AddNode CreateBazSaver done | 20260506 08:41:26.974 | INFO | OOM score went from 692 to 691 at SmrtBasecaller::RunAnalyzer checkpoint 7 | 20260506 08:41:26.974 | INFO | memory usage change: vm:81359103 RSS:9828349 shared:6770339 text:4978 data:5898228 at SmrtBasecaller::RunAnalyzer checkpoint 7 | 20260506 08:41:26.975 | INFO | Added 192 lanes to the ROI | 20260506 08:41:26.975 | INFO | Added 192 lanes to the ROI | 20260506 08:41:26.976 | INFO | Added 20 lanes to the ROI | 20260506 08:41:26.976 | INFO | Added 20 lanes to the ROI | 20260506 08:41:30.548 | INFO | memory usage change: vm:81359103 RSS:9828397 shared:6770355 text:4978 data:5898228 at AddNode CreateSampledBazSaver done | 20260506 08:41:30.587 | INFO | OOM score went from 691 to 690 at SmrtBasecaller::RunAnalyzer checkpoint 9 | 20260506 08:41:30.587 | INFO | memory usage change: vm:81260798 RSS:9730092 shared:6770355 text:4978 data:5799923 at SmrtBasecaller::RunAnalyzer checkpoint 9 | 20260506 08:41:30.587 | INFO | Entered sourceRunning loop, awaiting data from datasource ... | 20260506 08:41:36.587 | WARN | No incoming data for 6.00007 seconds, receive wait 6.00007 seconds, last deliver duration 0 seconds | 20260506 08:41:37.192 | INFO | Base Frame index set to 8917945 | 20260506 08:41:37.192 | INFO | Movie started at frame:0, expectedEnd Frame set to:8640000 | 20260506 08:41:37.192 | INFO | Analyzing chunk frames = [0,512) | 20260506 08:41:37.192 | INFO | MemInfo: { | 20260506 08:41:37.192 | INFO | "gpu" : " 11 GB", | 20260506 08:41:37.192 | INFO | "huge" : "", | 20260506 08:41:37.192 | INFO | "huge_shared" : " 25 GB", | 20260506 08:41:37.192 | INFO | "malloc" : " 15 GB", | 20260506 08:41:37.192 | INFO | "pinned" : " 27 GB" | 20260506 08:41:37.192 | INFO | } | 20260506 08:41:37.192 | INFO | Perf report for the last 63.8916 seconds: | 20260506 08:41:37.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:41:37.192 | INFO | REPACKER : 0.0%, 0.0, 0.0ms, 0 | 20260506 08:41:37.192 | INFO | PREP_TRACE : 0.0%, 0.0, 0.0ms, 0 | 20260506 08:41:37.192 | INFO | SAVE_TRACE : 0.0%, 0.0, 0.0ms, 0 | 20260506 08:41:37.192 | INFO | ANALYSIS : 0.0%, 0.0, 0.0ms, 0 | 20260506 08:41:37.192 | INFO | PREP_RT_METRICS : 0.0%, 0.0, 0.0ms, 0 | 20260506 08:41:37.192 | INFO | RT_METRICS : 0.0%, 0.0, 0.0ms, 0 | 20260506 08:41:37.192 | INFO | PRE_HQ : 0.0%, 0.0, 0.0ms, 0 | 20260506 08:41:37.192 | INFO | BAZWRITER : 0.0%, 0.0, 0.0ms, 0 | 20260506 08:41:37.192 | INFO | BAZ_SAMPLING : 0.0%, 0.0, 0.0ms, 0 | 20260506 08:41:42.331 | INFO | Analyzing chunk frames = [512,1024) | 20260506 08:41:42.331 | INFO | MemInfo: { | 20260506 08:41:42.331 | INFO | "gpu" : " 13 GB", | 20260506 08:41:42.331 | INFO | "huge" : "", | 20260506 08:41:42.331 | INFO | "huge_shared" : " 27 GB", | 20260506 08:41:42.331 | INFO | "malloc" : " 15 GB", | 20260506 08:41:42.331 | INFO | "pinned" : " 28 GB" | 20260506 08:41:42.331 | INFO | } | 20260506 08:41:42.331 | INFO | Perf report for the last 5.13915 seconds: | 20260506 08:41:42.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:41:42.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:42.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:42.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:42.331 | INFO | ANALYSIS : 90.3%, 3.0, 140.7ms, 0 | 20260506 08:41:42.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:42.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:42.331 | INFO | PRE_HQ : 5.7%, 1.0, 2.9ms, 0 | 20260506 08:41:42.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:42.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:42.331 | INFO | Uploaded 36GiB to GPU in 1.70854s (21.0706GiB/s) | 20260506 08:41:42.332 | INFO | Downloaded 24GiB to GPU in 1.07475s (22.3309GiB/s) | 20260506 08:41:47.470 | INFO | Analyzing chunk frames = [1024,1536) | 20260506 08:41:47.470 | INFO | MemInfo: { | 20260506 08:41:47.470 | INFO | "gpu" : " 13 GB", | 20260506 08:41:47.470 | INFO | "huge" : "", | 20260506 08:41:47.470 | INFO | "huge_shared" : " 27 GB", | 20260506 08:41:47.470 | INFO | "malloc" : " 15 GB", | 20260506 08:41:47.470 | INFO | "pinned" : " 28 GB" | 20260506 08:41:47.470 | INFO | } | 20260506 08:41:47.470 | INFO | Perf report for the last 5.13938 seconds: | 20260506 08:41:47.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:41:47.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:47.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:47.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:47.471 | INFO | ANALYSIS : 42.4%, 3.0, 66.0ms, 0 | 20260506 08:41:47.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:47.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:47.471 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:41:47.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:47.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:47.471 | INFO | Uploaded 36GiB to GPU in 1.59064s (22.6324GiB/s) | 20260506 08:41:47.471 | INFO | Downloaded 24GiB to GPU in 1.11003s (21.621GiB/s) | 20260506 08:41:52.610 | INFO | Analyzing chunk frames = [1536,2048) | 20260506 08:41:52.610 | INFO | MemInfo: { | 20260506 08:41:52.610 | INFO | "gpu" : " 13 GB", | 20260506 08:41:52.610 | INFO | "huge" : "", | 20260506 08:41:52.610 | INFO | "huge_shared" : " 27 GB", | 20260506 08:41:52.610 | INFO | "malloc" : " 15 GB", | 20260506 08:41:52.610 | INFO | "pinned" : " 28 GB" | 20260506 08:41:52.610 | INFO | } | 20260506 08:41:52.610 | INFO | Perf report for the last 5.13936 seconds: | 20260506 08:41:52.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:41:52.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:52.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:52.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:52.610 | INFO | ANALYSIS : 42.4%, 3.0, 66.0ms, 0 | 20260506 08:41:52.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:52.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:52.610 | INFO | PRE_HQ : 5.7%, 1.0, 2.9ms, 0 | 20260506 08:41:52.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:52.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:52.610 | INFO | Uploaded 36GiB to GPU in 1.59048s (22.6347GiB/s) | 20260506 08:41:52.610 | INFO | Downloaded 24GiB to GPU in 1.11162s (21.59GiB/s) | 20260506 08:41:57.749 | INFO | Analyzing chunk frames = [2048,2560) | 20260506 08:41:57.749 | INFO | MemInfo: { | 20260506 08:41:57.749 | INFO | "gpu" : " 13 GB", | 20260506 08:41:57.749 | INFO | "huge" : "", | 20260506 08:41:57.749 | INFO | "huge_shared" : " 27 GB", | 20260506 08:41:57.749 | INFO | "malloc" : " 15 GB", | 20260506 08:41:57.749 | INFO | "pinned" : " 28 GB" | 20260506 08:41:57.749 | INFO | } | 20260506 08:41:57.749 | INFO | Perf report for the last 5.13952 seconds: | 20260506 08:41:57.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:41:57.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:57.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:57.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:57.749 | INFO | ANALYSIS : 42.5%, 3.0, 66.1ms, 0 | 20260506 08:41:57.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:57.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:57.749 | INFO | PRE_HQ : 5.7%, 1.0, 2.9ms, 0 | 20260506 08:41:57.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:57.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:41:57.750 | INFO | Uploaded 36GiB to GPU in 1.59083s (22.6297GiB/s) | 20260506 08:41:57.750 | INFO | Downloaded 24GiB to GPU in 1.11958s (21.4365GiB/s) | 20260506 08:42:02.888 | INFO | Analyzing chunk frames = [2560,3072) | 20260506 08:42:02.888 | INFO | MemInfo: { | 20260506 08:42:02.888 | INFO | "gpu" : " 13 GB", | 20260506 08:42:02.888 | INFO | "huge" : "", | 20260506 08:42:02.888 | INFO | "huge_shared" : " 27 GB", | 20260506 08:42:02.888 | INFO | "malloc" : " 15 GB", | 20260506 08:42:02.888 | INFO | "pinned" : " 28 GB" | 20260506 08:42:02.889 | INFO | } | 20260506 08:42:02.889 | INFO | Perf report for the last 5.13926 seconds: | 20260506 08:42:02.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:42:02.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:02.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:02.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:02.889 | INFO | ANALYSIS : 42.6%, 3.0, 66.3ms, 0 | 20260506 08:42:02.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:02.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:02.889 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:42:02.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:02.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:02.889 | INFO | Uploaded 36GiB to GPU in 1.59062s (22.6326GiB/s) | 20260506 08:42:02.889 | INFO | Downloaded 24GiB to GPU in 1.1191s (21.4459GiB/s) | 20260506 08:42:08.028 | INFO | Analyzing chunk frames = [3072,3584) | 20260506 08:42:08.028 | INFO | MemInfo: { | 20260506 08:42:08.028 | INFO | "gpu" : " 13 GB", | 20260506 08:42:08.028 | INFO | "huge" : "", | 20260506 08:42:08.028 | INFO | "huge_shared" : " 27 GB", | 20260506 08:42:08.028 | INFO | "malloc" : " 15 GB", | 20260506 08:42:08.028 | INFO | "pinned" : " 28 GB" | 20260506 08:42:08.028 | INFO | } | 20260506 08:42:08.028 | INFO | Perf report for the last 5.13937 seconds: | 20260506 08:42:08.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:42:08.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:08.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:08.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:08.028 | INFO | ANALYSIS : 42.4%, 3.0, 65.9ms, 0 | 20260506 08:42:08.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:08.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:08.028 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:42:08.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:08.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:08.028 | INFO | Uploaded 36GiB to GPU in 1.59191s (22.6143GiB/s) | 20260506 08:42:08.028 | INFO | Downloaded 24GiB to GPU in 1.11912s (21.4454GiB/s) | 20260506 08:42:13.167 | INFO | Analyzing chunk frames = [3584,4096) | 20260506 08:42:13.167 | INFO | MemInfo: { | 20260506 08:42:13.167 | INFO | "gpu" : " 13 GB", | 20260506 08:42:13.167 | INFO | "huge" : "", | 20260506 08:42:13.167 | INFO | "huge_shared" : " 27 GB", | 20260506 08:42:13.167 | INFO | "malloc" : " 15 GB", | 20260506 08:42:13.167 | INFO | "pinned" : " 28 GB" | 20260506 08:42:13.167 | INFO | } | 20260506 08:42:13.167 | INFO | Perf report for the last 5.13922 seconds: | 20260506 08:42:13.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:42:13.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:13.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:13.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:13.167 | INFO | ANALYSIS : 42.5%, 3.0, 66.2ms, 0 | 20260506 08:42:13.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:13.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:13.167 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:42:13.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:13.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:13.168 | INFO | Uploaded 36GiB to GPU in 1.59107s (22.6262GiB/s) | 20260506 08:42:13.168 | INFO | Downloaded 24GiB to GPU in 1.11981s (21.4323GiB/s) | 20260506 08:42:13.237 | INFO | Starting a round of Baz IO | 20260506 08:42:15.386 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.149 seconds | 20260506 08:42:15.386 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260506 08:42:15.386 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0% / 0% / 0% | 20260506 08:42:15.386 | INFO | Baz IO overall: | 20260506 08:42:15.386 | INFO | BytesWritten : 2571416390 | 20260506 08:42:15.386 | INFO | NumEvents : 0 | 20260506 08:42:18.306 | INFO | Analyzing chunk frames = [4096,4608) | 20260506 08:42:18.306 | INFO | MemInfo: { | 20260506 08:42:18.306 | INFO | "gpu" : " 13 GB", | 20260506 08:42:18.306 | INFO | "huge" : "", | 20260506 08:42:18.306 | INFO | "huge_shared" : " 27 GB", | 20260506 08:42:18.306 | INFO | "malloc" : " 15 GB", | 20260506 08:42:18.306 | INFO | "pinned" : " 28 GB" | 20260506 08:42:18.306 | INFO | } | 20260506 08:42:18.307 | INFO | Perf report for the last 5.13932 seconds: | 20260506 08:42:18.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:42:18.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:18.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:18.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:18.307 | INFO | ANALYSIS : 42.5%, 3.0, 66.2ms, 0 | 20260506 08:42:18.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:18.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:18.307 | INFO | PRE_HQ : 40.1%, 1.2, 24.0ms, 0 | 20260506 08:42:18.307 | INFO | BAZWRITER : 11.5%, 1.0, 5.9ms, 0 | 20260506 08:42:18.307 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.1ms, 0 | 20260506 08:42:18.307 | INFO | Uploaded 36GiB to GPU in 1.59724s (22.5389GiB/s) | 20260506 08:42:18.307 | INFO | Downloaded 24GiB to GPU in 1.12589s (21.3164GiB/s) | 20260506 08:42:23.446 | INFO | Analyzing chunk frames = [4608,5120) | 20260506 08:42:23.446 | INFO | MemInfo: { | 20260506 08:42:23.446 | INFO | "gpu" : " 13 GB", | 20260506 08:42:23.446 | INFO | "huge" : "", | 20260506 08:42:23.446 | INFO | "huge_shared" : " 27 GB", | 20260506 08:42:23.446 | INFO | "malloc" : " 15 GB", | 20260506 08:42:23.446 | INFO | "pinned" : " 28 GB" | 20260506 08:42:23.446 | INFO | } | 20260506 08:42:23.446 | INFO | Perf report for the last 5.13933 seconds: | 20260506 08:42:23.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:42:23.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:23.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:23.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:23.446 | INFO | ANALYSIS : 42.4%, 3.0, 66.0ms, 0 | 20260506 08:42:23.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:23.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:23.446 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:42:23.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:23.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:23.446 | INFO | Uploaded 36GiB to GPU in 1.59162s (22.6184GiB/s) | 20260506 08:42:23.446 | INFO | Downloaded 24GiB to GPU in 1.11682s (21.4897GiB/s) | 20260506 08:42:28.585 | INFO | Analyzing chunk frames = [5120,5632) | 20260506 08:42:28.585 | INFO | MemInfo: { | 20260506 08:42:28.585 | INFO | "gpu" : " 13 GB", | 20260506 08:42:28.585 | INFO | "huge" : "", | 20260506 08:42:28.585 | INFO | "huge_shared" : " 27 GB", | 20260506 08:42:28.585 | INFO | "malloc" : " 15 GB", | 20260506 08:42:28.585 | INFO | "pinned" : " 28 GB" | 20260506 08:42:28.585 | INFO | } | 20260506 08:42:28.585 | INFO | Perf report for the last 5.13925 seconds: | 20260506 08:42:28.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:42:28.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:28.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:28.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:28.585 | INFO | ANALYSIS : 42.4%, 3.0, 66.0ms, 0 | 20260506 08:42:28.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:28.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:28.585 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:42:28.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:28.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:28.585 | INFO | Uploaded 36GiB to GPU in 1.59145s (22.6209GiB/s) | 20260506 08:42:28.585 | INFO | Downloaded 24GiB to GPU in 1.1174s (21.4785GiB/s) | 20260506 08:42:33.724 | INFO | Analyzing chunk frames = [5632,6144) | 20260506 08:42:33.724 | INFO | MemInfo: { | 20260506 08:42:33.724 | INFO | "gpu" : " 13 GB", | 20260506 08:42:33.724 | INFO | "huge" : "", | 20260506 08:42:33.725 | INFO | "huge_shared" : " 27 GB", | 20260506 08:42:33.725 | INFO | "malloc" : " 15 GB", | 20260506 08:42:33.725 | INFO | "pinned" : " 28 GB" | 20260506 08:42:33.725 | INFO | } | 20260506 08:42:33.725 | INFO | Perf report for the last 5.13945 seconds: | 20260506 08:42:33.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:42:33.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:33.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:33.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:33.725 | INFO | ANALYSIS : 42.6%, 3.0, 66.3ms, 0 | 20260506 08:42:33.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:33.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:33.725 | INFO | PRE_HQ : 5.7%, 1.0, 3.0ms, 0 | 20260506 08:42:33.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:33.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:33.725 | INFO | Uploaded 36GiB to GPU in 1.59043s (22.6354GiB/s) | 20260506 08:42:33.725 | INFO | Downloaded 24GiB to GPU in 1.11598s (21.5057GiB/s) | 20260506 08:42:38.864 | INFO | Analyzing chunk frames = [6144,6656) | 20260506 08:42:38.864 | INFO | MemInfo: { | 20260506 08:42:38.864 | INFO | "gpu" : " 13 GB", | 20260506 08:42:38.864 | INFO | "huge" : "", | 20260506 08:42:38.864 | INFO | "huge_shared" : " 27 GB", | 20260506 08:42:38.864 | INFO | "malloc" : " 15 GB", | 20260506 08:42:38.864 | INFO | "pinned" : " 28 GB" | 20260506 08:42:38.864 | INFO | } | 20260506 08:42:38.864 | INFO | Perf report for the last 5.1392 seconds: | 20260506 08:42:38.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:42:38.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:38.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:38.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:38.864 | INFO | ANALYSIS : 42.5%, 3.0, 66.1ms, 0 | 20260506 08:42:38.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:38.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:38.864 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:42:38.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:38.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:38.864 | INFO | Uploaded 36GiB to GPU in 1.59038s (22.636GiB/s) | 20260506 08:42:38.864 | INFO | Downloaded 24GiB to GPU in 1.11546s (21.5157GiB/s) | 20260506 08:42:44.003 | INFO | Analyzing chunk frames = [6656,7168) | 20260506 08:42:44.003 | INFO | MemInfo: { | 20260506 08:42:44.003 | INFO | "gpu" : " 13 GB", | 20260506 08:42:44.003 | INFO | "huge" : "", | 20260506 08:42:44.003 | INFO | "huge_shared" : " 27 GB", | 20260506 08:42:44.003 | INFO | "malloc" : " 15 GB", | 20260506 08:42:44.003 | INFO | "pinned" : " 28 GB" | 20260506 08:42:44.003 | INFO | } | 20260506 08:42:44.003 | INFO | Perf report for the last 5.13941 seconds: | 20260506 08:42:44.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:42:44.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:44.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:44.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:44.003 | INFO | ANALYSIS : 42.3%, 3.0, 65.9ms, 0 | 20260506 08:42:44.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:44.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:44.003 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:42:44.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:44.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:44.003 | INFO | Uploaded 36GiB to GPU in 1.59152s (22.6199GiB/s) | 20260506 08:42:44.004 | INFO | Downloaded 24GiB to GPU in 1.11724s (21.4816GiB/s) | 20260506 08:42:49.142 | INFO | Analyzing chunk frames = [7168,7680) | 20260506 08:42:49.142 | INFO | MemInfo: { | 20260506 08:42:49.142 | INFO | "gpu" : " 13 GB", | 20260506 08:42:49.142 | INFO | "huge" : "", | 20260506 08:42:49.143 | INFO | "huge_shared" : " 27 GB", | 20260506 08:42:49.143 | INFO | "malloc" : " 15 GB", | 20260506 08:42:49.143 | INFO | "pinned" : " 28 GB" | 20260506 08:42:49.143 | INFO | } | 20260506 08:42:49.143 | INFO | Perf report for the last 5.13939 seconds: | 20260506 08:42:49.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:42:49.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:49.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:49.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:49.143 | INFO | ANALYSIS : 42.5%, 3.0, 66.1ms, 0 | 20260506 08:42:49.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:49.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:49.143 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:42:49.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:49.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:49.143 | INFO | Uploaded 36GiB to GPU in 1.59085s (22.6294GiB/s) | 20260506 08:42:49.143 | INFO | Downloaded 24GiB to GPU in 1.11762s (21.4742GiB/s) | 20260506 08:42:54.282 | INFO | Analyzing chunk frames = [7680,8192) | 20260506 08:42:54.282 | INFO | MemInfo: { | 20260506 08:42:54.282 | INFO | "gpu" : " 13 GB", | 20260506 08:42:54.282 | INFO | "huge" : "", | 20260506 08:42:54.282 | INFO | "huge_shared" : " 27 GB", | 20260506 08:42:54.282 | INFO | "malloc" : " 15 GB", | 20260506 08:42:54.282 | INFO | "pinned" : " 28 GB" | 20260506 08:42:54.282 | INFO | } | 20260506 08:42:54.282 | INFO | Perf report for the last 5.13909 seconds: | 20260506 08:42:54.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:42:54.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:54.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:54.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:54.282 | INFO | ANALYSIS : 42.4%, 3.0, 66.0ms, 0 | 20260506 08:42:54.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:54.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:54.282 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:42:54.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:54.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:54.282 | INFO | Uploaded 36GiB to GPU in 1.59118s (22.6247GiB/s) | 20260506 08:42:54.282 | INFO | Downloaded 24GiB to GPU in 1.11767s (21.4733GiB/s) | 20260506 08:42:54.282 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 08:42:54.282 | INFO | Total mem usage: 14.6 GiB | 20260506 08:42:54.282 | INFO | All Time High Water Marks sum: 14.6 GiB (Savings: 0.0527%) | 20260506 08:42:54.282 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: 0.0527%) | 20260506 08:42:54.282 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:42:54.282 | INFO | High Water Breakdown: | 20260506 08:42:54.282 | INFO | 54.6% : 7.96 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 08:42:54.282 | INFO | 45.3% : 6.6 GiB ( 100% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 08:42:54.282 | INFO | 0.128% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260506 08:42:54.282 | INFO | ------------------------------------------ | 20260506 08:42:54.282 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:42:54.282 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 08:42:54.282 | INFO | Total mem usage: 26.5 GiB | 20260506 08:42:54.282 | INFO | All Time High Water Marks sum: 26.5 GiB (Savings: 0%) | 20260506 08:42:54.282 | INFO | Recent High Water Marks Sum: 26.5 GiB (Savings: 0%) | 20260506 08:42:54.282 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:42:54.282 | INFO | High Water Breakdown: | 20260506 08:42:54.282 | INFO | 54.1% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:42:54.282 | INFO | 35% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:42:54.282 | INFO | 3.54% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:42:54.282 | INFO | 3.01% : 816 MiB ( 100% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 08:42:54.282 | INFO | 1.9% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 08:42:54.282 | INFO | 1.41% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:42:54.282 | INFO | 0.987% : 268 MiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 08:42:54.282 | INFO | 0.0147% : 4 MiB ( 100% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 08:42:54.282 | INFO | 0.0147% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 08:42:54.282 | INFO | 0.00147% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 08:42:54.282 | INFO | 9.03e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 08:42:54.282 | INFO | ------------------------------------------ | 20260506 08:42:54.282 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:42:54.282 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 08:42:54.282 | INFO | Total mem usage: 25.6 GiB | 20260506 08:42:54.282 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 08:42:54.282 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: 0%) | 20260506 08:42:54.282 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:42:54.282 | INFO | High Water Breakdown: | 20260506 08:42:54.282 | INFO | 100% : 25.6 GiB ( 100% Of Peak) : WXIPCDataSourceConfig | 20260506 08:42:54.282 | INFO | ------------------------------------------ | 20260506 08:42:54.282 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:42:54.282 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 08:42:54.282 | INFO | Total mem usage: 12.5 GiB | 20260506 08:42:54.283 | INFO | All Time High Water Marks sum: 12.5 GiB (Savings: 0%) | 20260506 08:42:54.283 | INFO | Recent High Water Marks Sum: 12.5 GiB (Savings: 0%) | 20260506 08:42:54.283 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:42:54.283 | INFO | High Water Breakdown: | 20260506 08:42:54.283 | INFO | 43.3% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 08:42:54.283 | INFO | 17.7% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 08:42:54.283 | INFO | 11.3% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 08:42:54.283 | INFO | 9.03% : 1.12 GiB ( 100% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 08:42:54.283 | INFO | 4.51% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 08:42:54.283 | INFO | 3.6% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:42:54.283 | INFO | 3.01% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 08:42:54.283 | INFO | 2.33% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:42:54.283 | INFO | 2.19% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 08:42:54.283 | INFO | 1% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 08:42:54.283 | INFO | 1% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 08:42:54.283 | INFO | 0.752% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 08:42:54.283 | INFO | 0.235% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:42:54.283 | INFO | 0.0941% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:42:54.283 | INFO | ------------------------------------------ | 20260506 08:42:54.283 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:42:54.283 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 08:42:54.283 | INFO | PerfReport: Gathered 1440 samples. | 20260506 08:42:54.283 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:42:54.283 | INFO | PerfReport: Full Execution -- 7.25e+01 / -nan, 2.89e+01 / 0.00e+00 | 20260506 08:42:54.283 | INFO | PerfReport: 22.97%, Upload -- 1.67e+01 / 0.00e+00, 4.36e+00 / 0.00e+00 | 20260506 08:42:54.283 | INFO | PerfReport: 16.01%, Download -- 1.16e+01 / 0.00e+00, 1.89e-01 / 0.00e+00 | 20260506 08:42:54.283 | INFO | PerfReport: 33.28%, Compute -- 2.41e+01 / 0.00e+00, 1.46e+01 / 0.00e+00 | 20260506 08:42:54.283 | INFO | PerfReport: 0.71%, LockUpload -- 5.19e-01 / 0.00e+00, 3.76e+00 / 0.00e+00 | 20260506 08:42:54.283 | INFO | PerfReport: 0.00%, LockDownload -- 5.37e-04 / 0.00e+00, 2.38e-04 / 0.00e+00 | 20260506 08:42:54.283 | INFO | PerfReport: 27.02%, LockCompute -- 1.96e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260506 08:42:54.283 | INFO | PerfReport: 0.01%, Misc -- 9.09e-03 / 0.00e+00, 9.23e-03 / 0.00e+00 | 20260506 08:42:54.351 | INFO | Starting a round of Baz IO | 20260506 08:42:56.478 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.127 seconds | 20260506 08:42:56.478 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260506 08:42:56.478 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0% / 0% / 0% | 20260506 08:42:56.478 | INFO | Baz IO overall: | 20260506 08:42:56.478 | INFO | BytesWritten : 2571419942 | 20260506 08:42:56.478 | INFO | NumEvents : 0 | 20260506 08:42:59.421 | INFO | Analyzing chunk frames = [8192,8704) | 20260506 08:42:59.421 | INFO | MemInfo: { | 20260506 08:42:59.421 | INFO | "gpu" : " 13 GB", | 20260506 08:42:59.421 | INFO | "huge" : "", | 20260506 08:42:59.421 | INFO | "huge_shared" : " 27 GB", | 20260506 08:42:59.421 | INFO | "malloc" : " 15 GB", | 20260506 08:42:59.421 | INFO | "pinned" : " 28 GB" | 20260506 08:42:59.421 | INFO | } | 20260506 08:42:59.421 | INFO | Perf report for the last 5.13968 seconds: | 20260506 08:42:59.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:42:59.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:59.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:59.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:59.422 | INFO | ANALYSIS : 42.3%, 3.0, 65.8ms, 0 | 20260506 08:42:59.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:59.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:42:59.422 | INFO | PRE_HQ : 38.6%, 1.1, 21.3ms, 0 | 20260506 08:42:59.422 | INFO | BAZWRITER : 10.1%, 1.0, 5.2ms, 0 | 20260506 08:42:59.422 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260506 08:42:59.422 | INFO | Uploaded 36GiB to GPU in 1.59841s (22.5224GiB/s) | 20260506 08:42:59.422 | INFO | Downloaded 24GiB to GPU in 1.12472s (21.3386GiB/s) | 20260506 08:43:04.560 | INFO | Analyzing chunk frames = [8704,9216) | 20260506 08:43:04.560 | INFO | MemInfo: { | 20260506 08:43:04.560 | INFO | "gpu" : " 13 GB", | 20260506 08:43:04.560 | INFO | "huge" : "", | 20260506 08:43:04.560 | INFO | "huge_shared" : " 27 GB", | 20260506 08:43:04.561 | INFO | "malloc" : " 15 GB", | 20260506 08:43:04.561 | INFO | "pinned" : " 28 GB" | 20260506 08:43:04.561 | INFO | } | 20260506 08:43:04.561 | INFO | Perf report for the last 5.13915 seconds: | 20260506 08:43:04.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:43:04.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:04.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:04.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:04.561 | INFO | ANALYSIS : 42.6%, 3.0, 66.3ms, 0 | 20260506 08:43:04.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:04.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:04.561 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:43:04.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:04.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:04.561 | INFO | Uploaded 36GiB to GPU in 1.59075s (22.6308GiB/s) | 20260506 08:43:04.561 | INFO | Downloaded 24GiB to GPU in 1.11817s (21.4637GiB/s) | 20260506 08:43:09.700 | INFO | Analyzing chunk frames = [9216,9728) | 20260506 08:43:09.700 | INFO | MemInfo: { | 20260506 08:43:09.700 | INFO | "gpu" : " 13 GB", | 20260506 08:43:09.700 | INFO | "huge" : "", | 20260506 08:43:09.700 | INFO | "huge_shared" : " 27 GB", | 20260506 08:43:09.700 | INFO | "malloc" : " 15 GB", | 20260506 08:43:09.700 | INFO | "pinned" : " 28 GB" | 20260506 08:43:09.700 | INFO | } | 20260506 08:43:09.700 | INFO | Perf report for the last 5.13936 seconds: | 20260506 08:43:09.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:43:09.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:09.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:09.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:09.700 | INFO | ANALYSIS : 42.2%, 3.0, 65.7ms, 0 | 20260506 08:43:09.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:09.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:09.700 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:43:09.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:09.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:09.701 | INFO | Uploaded 36GiB to GPU in 1.59239s (22.6076GiB/s) | 20260506 08:43:09.701 | INFO | Downloaded 24GiB to GPU in 1.1173s (21.4803GiB/s) | 20260506 08:43:14.839 | INFO | Analyzing chunk frames = [9728,10240) | 20260506 08:43:14.839 | INFO | MemInfo: { | 20260506 08:43:14.839 | INFO | "gpu" : " 13 GB", | 20260506 08:43:14.839 | INFO | "huge" : "", | 20260506 08:43:14.839 | INFO | "huge_shared" : " 27 GB", | 20260506 08:43:14.839 | INFO | "malloc" : " 15 GB", | 20260506 08:43:14.839 | INFO | "pinned" : " 28 GB" | 20260506 08:43:14.839 | INFO | } | 20260506 08:43:14.839 | INFO | Perf report for the last 5.13915 seconds: | 20260506 08:43:14.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:43:14.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:14.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:14.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:14.839 | INFO | ANALYSIS : 42.6%, 3.0, 66.4ms, 0 | 20260506 08:43:14.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:14.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:14.839 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:43:14.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:14.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:14.839 | INFO | Uploaded 36GiB to GPU in 1.59021s (22.6386GiB/s) | 20260506 08:43:14.839 | INFO | Downloaded 24GiB to GPU in 1.11596s (21.5062GiB/s) | 20260506 08:43:19.978 | INFO | Analyzing chunk frames = [10240,10752) | 20260506 08:43:19.978 | INFO | MemInfo: { | 20260506 08:43:19.978 | INFO | "gpu" : " 13 GB", | 20260506 08:43:19.978 | INFO | "huge" : "", | 20260506 08:43:19.978 | INFO | "huge_shared" : " 27 GB", | 20260506 08:43:19.978 | INFO | "malloc" : " 15 GB", | 20260506 08:43:19.978 | INFO | "pinned" : " 28 GB" | 20260506 08:43:19.978 | INFO | } | 20260506 08:43:19.979 | INFO | Perf report for the last 5.13937 seconds: | 20260506 08:43:19.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:43:19.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:19.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:19.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:19.979 | INFO | ANALYSIS : 42.8%, 3.0, 66.7ms, 0 | 20260506 08:43:19.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:19.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:19.979 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:43:19.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:19.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:19.979 | INFO | Uploaded 36GiB to GPU in 1.5881s (22.6686GiB/s) | 20260506 08:43:19.979 | INFO | Downloaded 24GiB to GPU in 1.11477s (21.529GiB/s) | 20260506 08:43:25.118 | INFO | Analyzing chunk frames = [10752,11264) | 20260506 08:43:25.118 | INFO | MemInfo: { | 20260506 08:43:25.118 | INFO | "gpu" : " 13 GB", | 20260506 08:43:25.118 | INFO | "huge" : "", | 20260506 08:43:25.118 | INFO | "huge_shared" : " 27 GB", | 20260506 08:43:25.118 | INFO | "malloc" : " 15 GB", | 20260506 08:43:25.118 | INFO | "pinned" : " 28 GB" | 20260506 08:43:25.118 | INFO | } | 20260506 08:43:25.118 | INFO | Perf report for the last 5.13931 seconds: | 20260506 08:43:25.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:43:25.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:25.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:25.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:25.118 | INFO | ANALYSIS : 43.2%, 3.0, 67.3ms, 0 | 20260506 08:43:25.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:25.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:25.118 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:43:25.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:25.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:25.118 | INFO | Uploaded 36GiB to GPU in 1.58585s (22.7008GiB/s) | 20260506 08:43:25.118 | INFO | Downloaded 24GiB to GPU in 1.11363s (21.5511GiB/s) | 20260506 08:43:30.257 | INFO | Analyzing chunk frames = [11264,11776) | 20260506 08:43:30.257 | INFO | MemInfo: { | 20260506 08:43:30.257 | INFO | "gpu" : " 13 GB", | 20260506 08:43:30.257 | INFO | "huge" : "", | 20260506 08:43:30.257 | INFO | "huge_shared" : " 27 GB", | 20260506 08:43:30.257 | INFO | "malloc" : " 15 GB", | 20260506 08:43:30.257 | INFO | "pinned" : " 28 GB" | 20260506 08:43:30.257 | INFO | } | 20260506 08:43:30.257 | INFO | Perf report for the last 5.13947 seconds: | 20260506 08:43:30.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:43:30.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:30.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:30.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:30.257 | INFO | ANALYSIS : 43.3%, 3.0, 67.4ms, 0 | 20260506 08:43:30.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:30.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:30.258 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:43:30.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:30.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:30.258 | INFO | Uploaded 36GiB to GPU in 1.58406s (22.7264GiB/s) | 20260506 08:43:30.258 | INFO | Downloaded 24GiB to GPU in 1.10654s (21.6893GiB/s) | 20260506 08:43:35.396 | INFO | Analyzing chunk frames = [11776,12288) | 20260506 08:43:35.396 | INFO | MemInfo: { | 20260506 08:43:35.396 | INFO | "gpu" : " 13 GB", | 20260506 08:43:35.397 | INFO | "huge" : "", | 20260506 08:43:35.397 | INFO | "huge_shared" : " 27 GB", | 20260506 08:43:35.397 | INFO | "malloc" : " 15 GB", | 20260506 08:43:35.397 | INFO | "pinned" : " 28 GB" | 20260506 08:43:35.397 | INFO | } | 20260506 08:43:35.397 | INFO | Perf report for the last 5.13935 seconds: | 20260506 08:43:35.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:43:35.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:35.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:35.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:35.397 | INFO | ANALYSIS : 43.9%, 3.0, 68.3ms, 0 | 20260506 08:43:35.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:35.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:35.397 | INFO | PRE_HQ : 5.7%, 1.0, 2.9ms, 0 | 20260506 08:43:35.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:35.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:35.397 | INFO | Uploaded 36GiB to GPU in 1.57938s (22.7937GiB/s) | 20260506 08:43:35.397 | INFO | Downloaded 24GiB to GPU in 1.10369s (21.7452GiB/s) | 20260506 08:43:35.472 | INFO | Starting a round of Baz IO | 20260506 08:43:37.672 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.2 seconds | 20260506 08:43:37.672 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260506 08:43:37.672 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0% / 0% / 0% | 20260506 08:43:37.672 | INFO | Baz IO overall: | 20260506 08:43:37.672 | INFO | BytesWritten : 2571423494 | 20260506 08:43:37.672 | INFO | NumEvents : 0 | 20260506 08:43:40.536 | INFO | Analyzing chunk frames = [12288,12800) | 20260506 08:43:40.536 | INFO | MemInfo: { | 20260506 08:43:40.536 | INFO | "gpu" : " 14 GB", | 20260506 08:43:40.536 | INFO | "huge" : "", | 20260506 08:43:40.536 | INFO | "huge_shared" : " 27 GB", | 20260506 08:43:40.536 | INFO | "malloc" : " 15 GB", | 20260506 08:43:40.536 | INFO | "pinned" : " 28 GB" | 20260506 08:43:40.536 | INFO | } | 20260506 08:43:40.536 | INFO | Perf report for the last 5.13936 seconds: | 20260506 08:43:40.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:43:40.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:40.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:40.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:40.536 | INFO | ANALYSIS : 43.8%, 3.0, 68.2ms, 0 | 20260506 08:43:40.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:40.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:40.536 | INFO | PRE_HQ : 39.1%, 1.0, 21.0ms, 0 | 20260506 08:43:40.536 | INFO | BAZWRITER : 10.3%, 1.0, 5.3ms, 0 | 20260506 08:43:40.536 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260506 08:43:40.536 | INFO | Uploaded 36GiB to GPU in 1.58609s (22.6974GiB/s) | 20260506 08:43:40.536 | INFO | Downloaded 24GiB to GPU in 1.1105s (21.6118GiB/s) | 20260506 08:43:45.675 | INFO | Analyzing chunk frames = [12800,13312) | 20260506 08:43:45.675 | INFO | MemInfo: { | 20260506 08:43:45.675 | INFO | "gpu" : " 14 GB", | 20260506 08:43:45.675 | INFO | "huge" : "", | 20260506 08:43:45.675 | INFO | "huge_shared" : " 27 GB", | 20260506 08:43:45.675 | INFO | "malloc" : " 15 GB", | 20260506 08:43:45.675 | INFO | "pinned" : " 28 GB" | 20260506 08:43:45.675 | INFO | } | 20260506 08:43:45.675 | INFO | Perf report for the last 5.13913 seconds: | 20260506 08:43:45.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:43:45.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:45.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:45.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:45.675 | INFO | ANALYSIS : 44.4%, 3.0, 69.1ms, 0 | 20260506 08:43:45.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:45.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:45.675 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:43:45.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:45.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:45.675 | INFO | Uploaded 36GiB to GPU in 1.57559s (22.8486GiB/s) | 20260506 08:43:45.676 | INFO | Downloaded 24GiB to GPU in 1.10255s (21.7678GiB/s) | 20260506 08:43:50.814 | INFO | Analyzing chunk frames = [13312,13824) | 20260506 08:43:50.814 | INFO | MemInfo: { | 20260506 08:43:50.814 | INFO | "gpu" : " 14 GB", | 20260506 08:43:50.814 | INFO | "huge" : "", | 20260506 08:43:50.814 | INFO | "huge_shared" : " 27 GB", | 20260506 08:43:50.814 | INFO | "malloc" : " 15 GB", | 20260506 08:43:50.814 | INFO | "pinned" : " 28 GB" | 20260506 08:43:50.815 | INFO | } | 20260506 08:43:50.815 | INFO | Perf report for the last 5.13934 seconds: | 20260506 08:43:50.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:43:50.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:50.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:50.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:50.815 | INFO | ANALYSIS : 44.4%, 3.0, 69.2ms, 0 | 20260506 08:43:50.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:50.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:50.815 | INFO | PRE_HQ : 5.6%, 1.0, 2.9ms, 0 | 20260506 08:43:50.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:50.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:50.815 | INFO | Uploaded 36GiB to GPU in 1.57542s (22.8511GiB/s) | 20260506 08:43:50.815 | INFO | Downloaded 24GiB to GPU in 1.0994s (21.8301GiB/s) | 20260506 08:43:55.954 | INFO | Analyzing chunk frames = [13824,14336) | 20260506 08:43:55.954 | INFO | MemInfo: { | 20260506 08:43:55.954 | INFO | "gpu" : " 14 GB", | 20260506 08:43:55.954 | INFO | "huge" : "", | 20260506 08:43:55.954 | INFO | "huge_shared" : " 27 GB", | 20260506 08:43:55.954 | INFO | "malloc" : " 15 GB", | 20260506 08:43:55.954 | INFO | "pinned" : " 28 GB" | 20260506 08:43:55.954 | INFO | } | 20260506 08:43:55.954 | INFO | Perf report for the last 5.1394 seconds: | 20260506 08:43:55.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:43:55.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:55.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:55.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:55.954 | INFO | ANALYSIS : 44.6%, 3.0, 69.5ms, 0 | 20260506 08:43:55.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:55.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:55.954 | INFO | PRE_HQ : 5.7%, 1.0, 3.0ms, 0 | 20260506 08:43:55.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:55.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:43:55.954 | INFO | Uploaded 36GiB to GPU in 1.57354s (22.8783GiB/s) | 20260506 08:43:55.954 | INFO | Downloaded 24GiB to GPU in 1.09682s (21.8814GiB/s) | 20260506 08:44:01.093 | INFO | Analyzing chunk frames = [14336,14848) | 20260506 08:44:01.093 | INFO | MemInfo: { | 20260506 08:44:01.093 | INFO | "gpu" : " 16 GB", | 20260506 08:44:01.093 | INFO | "huge" : "", | 20260506 08:44:01.093 | INFO | "huge_shared" : " 27 GB", | 20260506 08:44:01.093 | INFO | "malloc" : " 15 GB", | 20260506 08:44:01.093 | INFO | "pinned" : " 28 GB" | 20260506 08:44:01.093 | INFO | } | 20260506 08:44:01.093 | INFO | Perf report for the last 5.13926 seconds: | 20260506 08:44:01.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:44:01.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:01.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:01.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:01.093 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 08:44:01.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:01.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:01.093 | INFO | PRE_HQ : 10.6%, 1.0, 5.7ms, 0 | 20260506 08:44:01.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:01.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:01.093 | INFO | Uploaded 36GiB to GPU in 1.57506s (22.8562GiB/s) | 20260506 08:44:01.093 | INFO | Downloaded 26GiB to GPU in 1.21784s (21.3493GiB/s) | 20260506 08:44:06.233 | INFO | Analyzing chunk frames = [14848,15360) | 20260506 08:44:06.233 | INFO | MemInfo: { | 20260506 08:44:06.233 | INFO | "gpu" : " 16 GB", | 20260506 08:44:06.233 | INFO | "huge" : "", | 20260506 08:44:06.233 | INFO | "huge_shared" : " 27 GB", | 20260506 08:44:06.233 | INFO | "malloc" : " 15 GB", | 20260506 08:44:06.233 | INFO | "pinned" : " 28 GB" | 20260506 08:44:06.233 | INFO | } | 20260506 08:44:06.233 | INFO | Perf report for the last 5.13938 seconds: | 20260506 08:44:06.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:44:06.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:06.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:06.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:06.233 | INFO | ANALYSIS : 60.9%, 3.0, 95.1ms, 0 | 20260506 08:44:06.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:06.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:06.233 | INFO | PRE_HQ : 14.6%, 1.0, 8.0ms, 0 | 20260506 08:44:06.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:06.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:06.233 | INFO | Uploaded 36GiB to GPU in 1.57566s (22.8475GiB/s) | 20260506 08:44:06.233 | INFO | Downloaded 27GiB to GPU in 1.20338s (22.4369GiB/s) | 20260506 08:44:11.372 | INFO | Analyzing chunk frames = [15360,15872) | 20260506 08:44:11.372 | INFO | MemInfo: { | 20260506 08:44:11.372 | INFO | "gpu" : " 16 GB", | 20260506 08:44:11.372 | INFO | "huge" : "", | 20260506 08:44:11.372 | INFO | "huge_shared" : " 27 GB", | 20260506 08:44:11.372 | INFO | "malloc" : " 15 GB", | 20260506 08:44:11.372 | INFO | "pinned" : " 28 GB" | 20260506 08:44:11.372 | INFO | } | 20260506 08:44:11.372 | INFO | Perf report for the last 5.13942 seconds: | 20260506 08:44:11.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:44:11.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:11.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:11.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:11.372 | INFO | ANALYSIS : 63.0%, 3.0, 98.4ms, 0 | 20260506 08:44:11.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:11.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:11.372 | INFO | PRE_HQ : 18.1%, 1.0, 9.7ms, 0 | 20260506 08:44:11.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:11.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:11.372 | INFO | Uploaded 36GiB to GPU in 1.64502s (21.8842GiB/s) | 20260506 08:44:11.372 | INFO | Downloaded 28GiB to GPU in 1.32122s (21.1925GiB/s) | 20260506 08:44:16.511 | INFO | Analyzing chunk frames = [15872,16384) | 20260506 08:44:16.511 | INFO | MemInfo: { | 20260506 08:44:16.511 | INFO | "gpu" : " 16 GB", | 20260506 08:44:16.511 | INFO | "huge" : "", | 20260506 08:44:16.511 | INFO | "huge_shared" : " 27 GB", | 20260506 08:44:16.511 | INFO | "malloc" : " 15 GB", | 20260506 08:44:16.511 | INFO | "pinned" : " 28 GB" | 20260506 08:44:16.511 | INFO | } | 20260506 08:44:16.511 | INFO | Perf report for the last 5.13925 seconds: | 20260506 08:44:16.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:44:16.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:16.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:16.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:16.511 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 08:44:16.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:16.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:16.511 | INFO | PRE_HQ : 20.4%, 1.0, 10.9ms, 0 | 20260506 08:44:16.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:16.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:16.512 | INFO | Uploaded 36GiB to GPU in 1.58466s (22.7178GiB/s) | 20260506 08:44:16.512 | INFO | Downloaded 29GiB to GPU in 1.3079s (22.173GiB/s) | 20260506 08:44:16.512 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 08:44:16.512 | INFO | Total mem usage: 14.6 GiB | 20260506 08:44:16.512 | INFO | All Time High Water Marks sum: 14.6 GiB (Savings: 0.0527%) | 20260506 08:44:16.512 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: 0.0461%) | 20260506 08:44:16.512 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:44:16.512 | INFO | High Water Breakdown: | 20260506 08:44:16.512 | INFO | 54.6% : 7.96 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 08:44:16.512 | INFO | 45.3% : 6.6 GiB ( 100% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 08:44:16.512 | INFO | 0.128% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260506 08:44:16.512 | INFO | ------------------------------------------ | 20260506 08:44:16.512 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:44:16.512 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 08:44:16.512 | INFO | Total mem usage: 26.5 GiB | 20260506 08:44:16.512 | INFO | All Time High Water Marks sum: 26.7 GiB (Savings: 0.768%) | 20260506 08:44:16.512 | INFO | Recent High Water Marks Sum: 26.7 GiB (Savings: 0.768%) | 20260506 08:44:16.512 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:44:16.512 | INFO | High Water Breakdown: | 20260506 08:44:16.512 | INFO | 53.7% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:44:16.512 | INFO | 34.7% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:44:16.512 | INFO | 3.51% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:44:16.512 | INFO | 2.98% : 816 MiB ( 100% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 08:44:16.512 | INFO | 1.89% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 08:44:16.512 | INFO | 1.4% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:44:16.512 | INFO | 0.98% : 268 MiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 08:44:16.512 | INFO | 0.768% : 210 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:44:16.512 | INFO | 0.0146% : 4 MiB ( 100% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 08:44:16.512 | INFO | 0.0146% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 08:44:16.512 | INFO | 0.00146% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 08:44:16.512 | INFO | 8.96e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 08:44:16.512 | INFO | ------------------------------------------ | 20260506 08:44:16.512 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:44:16.512 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 08:44:16.512 | INFO | Total mem usage: 25.6 GiB | 20260506 08:44:16.512 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 08:44:16.512 | INFO | Recent High Water Marks Sum: 24.5 GiB (Savings: -4.59%) | 20260506 08:44:16.512 | INFO | Unused Cache: 1.12 GiB (Waste: 4.39%) | 20260506 08:44:16.512 | INFO | High Water Breakdown: | 20260506 08:44:16.512 | INFO | 100% : 24.5 GiB ( 95.6% Of Peak) : WXIPCDataSourceConfig | 20260506 08:44:16.512 | INFO | ------------------------------------------ | 20260506 08:44:16.512 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:44:16.512 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 08:44:16.512 | INFO | Total mem usage: 14.7 GiB | 20260506 08:44:16.512 | INFO | All Time High Water Marks sum: 16 GiB (Savings: 8.25%) | 20260506 08:44:16.512 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: 8.25%) | 20260506 08:44:16.512 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:44:16.512 | INFO | High Water Breakdown: | 20260506 08:44:16.512 | INFO | 33.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 08:44:16.512 | INFO | 13.8% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 08:44:16.512 | INFO | 8.8% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 08:44:16.512 | INFO | 7.04% : 1.12 GiB ( 100% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 08:44:16.512 | INFO | 6.45% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 08:44:16.512 | INFO | 5.08% : 832 MiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 08:44:16.512 | INFO | 5.04% : 824 MiB ( 100% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 08:44:16.512 | INFO | 3.52% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 08:44:16.512 | INFO | 3.16% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 08:44:16.512 | INFO | 2.8% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:44:16.512 | INFO | 2.35% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 08:44:16.512 | INFO | 1.81% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:44:16.512 | INFO | 1.71% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 08:44:16.512 | INFO | 1.63% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 08:44:16.512 | INFO | 0.782% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 08:44:16.512 | INFO | 0.782% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 08:44:16.512 | INFO | 0.642% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:44:16.512 | INFO | 0.587% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 08:44:16.512 | INFO | 0.183% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:44:16.512 | INFO | 0.0733% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:44:16.512 | INFO | 0.0244% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 08:44:16.512 | INFO | 0.00611% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 08:44:16.512 | INFO | ------------------------------------------ | 20260506 08:44:16.512 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:44:16.512 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 08:44:16.512 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:44:16.512 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:44:16.512 | INFO | PerfReport: Full Execution -- 7.64e+01 / -nan, 2.21e+01 / 0.00e+00 | 20260506 08:44:16.512 | INFO | PerfReport: 21.65%, Upload -- 1.65e+01 / 0.00e+00, 1.67e+00 / 0.00e+00 | 20260506 08:44:16.512 | INFO | PerfReport: 15.67%, Download -- 1.20e+01 / 0.00e+00, 2.68e+00 / 0.00e+00 | 20260506 08:44:16.512 | INFO | PerfReport: 33.27%, Compute -- 2.54e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 08:44:16.512 | INFO | PerfReport: 0.66%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 08:44:16.512 | INFO | PerfReport: 0.00%, LockDownload -- 5.20e-04 / 0.00e+00, 1.54e-04 / 0.00e+00 | 20260506 08:44:16.512 | INFO | PerfReport: 28.74%, LockCompute -- 2.20e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260506 08:44:16.512 | INFO | PerfReport: 0.01%, Misc -- 9.75e-03 / 0.00e+00, 9.69e-03 / 0.00e+00 | 20260506 08:44:16.640 | INFO | Starting a round of Baz IO | 20260506 08:44:19.969 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 3.328 seconds | 20260506 08:44:19.969 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260506 08:44:19.969 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0197808% / 0% / 0% | 20260506 08:44:19.969 | INFO | Baz IO overall: | 20260506 08:44:19.969 | INFO | BytesWritten : 2571427046 | 20260506 08:44:19.969 | INFO | NumEvents : 0 | 20260506 08:44:21.651 | INFO | Analyzing chunk frames = [16384,16896) | 20260506 08:44:21.651 | INFO | MemInfo: { | 20260506 08:44:21.651 | INFO | "gpu" : " 17 GB", | 20260506 08:44:21.651 | INFO | "huge" : "", | 20260506 08:44:21.651 | INFO | "huge_shared" : " 27 GB", | 20260506 08:44:21.651 | INFO | "malloc" : " 15 GB", | 20260506 08:44:21.651 | INFO | "pinned" : " 29 GB" | 20260506 08:44:21.651 | INFO | } | 20260506 08:44:21.651 | INFO | Perf report for the last 5.13935 seconds: | 20260506 08:44:21.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:44:21.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:21.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:21.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:21.651 | INFO | ANALYSIS : 66.7%, 3.0, 104.2ms, 0 | 20260506 08:44:21.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:21.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:21.651 | INFO | PRE_HQ : 52.9%, 1.6, 43.8ms, 0 | 20260506 08:44:21.651 | INFO | BAZWRITER : 10.1%, 1.0, 5.4ms, 0 | 20260506 08:44:21.651 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.1ms, 0 | 20260506 08:44:21.651 | INFO | Uploaded 36GiB to GPU in 1.79255s (20.0831GiB/s) | 20260506 08:44:21.651 | INFO | Downloaded 33GiB to GPU in 1.6717s (19.7404GiB/s) | 20260506 08:44:26.790 | INFO | Analyzing chunk frames = [16896,17408) | 20260506 08:44:26.790 | INFO | MemInfo: { | 20260506 08:44:26.790 | INFO | "gpu" : " 17 GB", | 20260506 08:44:26.790 | INFO | "huge" : "", | 20260506 08:44:26.790 | INFO | "huge_shared" : " 27 GB", | 20260506 08:44:26.790 | INFO | "malloc" : " 15 GB", | 20260506 08:44:26.790 | INFO | "pinned" : " 29 GB" | 20260506 08:44:26.790 | INFO | } | 20260506 08:44:26.790 | INFO | Perf report for the last 5.13938 seconds: | 20260506 08:44:26.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:44:26.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:26.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:26.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:26.790 | INFO | ANALYSIS : 63.2%, 3.0, 98.7ms, 0 | 20260506 08:44:26.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:26.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:26.790 | INFO | PRE_HQ : 34.3%, 1.1, 19.7ms, 0 | 20260506 08:44:26.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:26.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:26.790 | INFO | Uploaded 36GiB to GPU in 1.59377s (22.588GiB/s) | 20260506 08:44:26.790 | INFO | Downloaded 32GiB to GPU in 1.4134s (22.6405GiB/s) | 20260506 08:44:31.929 | INFO | Analyzing chunk frames = [17408,17920) | 20260506 08:44:31.929 | INFO | MemInfo: { | 20260506 08:44:31.929 | INFO | "gpu" : " 17 GB", | 20260506 08:44:31.929 | INFO | "huge" : "", | 20260506 08:44:31.929 | INFO | "huge_shared" : " 27 GB", | 20260506 08:44:31.929 | INFO | "malloc" : " 15 GB", | 20260506 08:44:31.929 | INFO | "pinned" : " 29 GB" | 20260506 08:44:31.929 | INFO | } | 20260506 08:44:31.929 | INFO | Perf report for the last 5.13933 seconds: | 20260506 08:44:31.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:44:31.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:31.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:31.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:31.929 | INFO | ANALYSIS : 62.9%, 3.0, 97.7ms, 0 | 20260506 08:44:31.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:31.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:31.930 | INFO | PRE_HQ : 29.9%, 1.0, 15.7ms, 0 | 20260506 08:44:31.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:31.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:31.930 | INFO | Uploaded 36GiB to GPU in 1.60257s (22.4639GiB/s) | 20260506 08:44:31.930 | INFO | Downloaded 33GiB to GPU in 1.47196s (22.4191GiB/s) | 20260506 08:44:37.069 | INFO | Analyzing chunk frames = [17920,18432) | 20260506 08:44:37.069 | INFO | MemInfo: { | 20260506 08:44:37.069 | INFO | "gpu" : " 17 GB", | 20260506 08:44:37.069 | INFO | "huge" : "", | 20260506 08:44:37.069 | INFO | "huge_shared" : " 27 GB", | 20260506 08:44:37.069 | INFO | "malloc" : " 15 GB", | 20260506 08:44:37.069 | INFO | "pinned" : " 29 GB" | 20260506 08:44:37.069 | INFO | } | 20260506 08:44:37.069 | INFO | Perf report for the last 5.13934 seconds: | 20260506 08:44:37.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:44:37.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:37.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:37.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:37.069 | INFO | ANALYSIS : 63.1%, 3.0, 97.7ms, 0 | 20260506 08:44:37.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:37.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:37.069 | INFO | PRE_HQ : 32.3%, 1.0, 17.2ms, 0 | 20260506 08:44:37.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:37.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:37.069 | INFO | Uploaded 36GiB to GPU in 1.6058s (22.4187GiB/s) | 20260506 08:44:37.069 | INFO | Downloaded 34GiB to GPU in 1.5241s (22.3083GiB/s) | 20260506 08:44:42.208 | INFO | Analyzing chunk frames = [18432,18944) | 20260506 08:44:42.208 | INFO | MemInfo: { | 20260506 08:44:42.208 | INFO | "gpu" : " 17 GB", | 20260506 08:44:42.208 | INFO | "huge" : "", | 20260506 08:44:42.208 | INFO | "huge_shared" : " 27 GB", | 20260506 08:44:42.208 | INFO | "malloc" : " 15 GB", | 20260506 08:44:42.208 | INFO | "pinned" : " 29 GB" | 20260506 08:44:42.208 | INFO | } | 20260506 08:44:42.208 | INFO | Perf report for the last 5.13925 seconds: | 20260506 08:44:42.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:44:42.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:42.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:42.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:42.208 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260506 08:44:42.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:42.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:42.208 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 08:44:42.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:42.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:42.208 | INFO | Uploaded 36GiB to GPU in 1.60396s (22.4445GiB/s) | 20260506 08:44:42.208 | INFO | Downloaded 34GiB to GPU in 1.5215s (22.3464GiB/s) | 20260506 08:44:47.347 | INFO | Analyzing chunk frames = [18944,19456) | 20260506 08:44:47.347 | INFO | MemInfo: { | 20260506 08:44:47.347 | INFO | "gpu" : " 17 GB", | 20260506 08:44:47.347 | INFO | "huge" : "", | 20260506 08:44:47.347 | INFO | "huge_shared" : " 27 GB", | 20260506 08:44:47.347 | INFO | "malloc" : " 15 GB", | 20260506 08:44:47.347 | INFO | "pinned" : " 29 GB" | 20260506 08:44:47.347 | INFO | } | 20260506 08:44:47.347 | INFO | Perf report for the last 5.1393 seconds: | 20260506 08:44:47.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:44:47.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:47.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:47.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:47.347 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 08:44:47.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:47.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:47.347 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 08:44:47.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:47.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:47.348 | INFO | Uploaded 36GiB to GPU in 1.60367s (22.4486GiB/s) | 20260506 08:44:47.348 | INFO | Downloaded 34GiB to GPU in 1.52299s (22.3245GiB/s) | 20260506 08:44:52.486 | INFO | Analyzing chunk frames = [19456,19968) | 20260506 08:44:52.486 | INFO | MemInfo: { | 20260506 08:44:52.486 | INFO | "gpu" : " 17 GB", | 20260506 08:44:52.487 | INFO | "huge" : "", | 20260506 08:44:52.487 | INFO | "huge_shared" : " 27 GB", | 20260506 08:44:52.487 | INFO | "malloc" : " 15 GB", | 20260506 08:44:52.487 | INFO | "pinned" : " 29 GB" | 20260506 08:44:52.487 | INFO | } | 20260506 08:44:52.487 | INFO | Perf report for the last 5.13927 seconds: | 20260506 08:44:52.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:44:52.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:52.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:52.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:52.487 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 08:44:52.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:52.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:52.487 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 08:44:52.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:52.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:52.487 | INFO | Uploaded 36GiB to GPU in 1.60221s (22.4689GiB/s) | 20260506 08:44:52.487 | INFO | Downloaded 34GiB to GPU in 1.52287s (22.3263GiB/s) | 20260506 08:44:57.626 | INFO | Analyzing chunk frames = [19968,20480) | 20260506 08:44:57.626 | INFO | MemInfo: { | 20260506 08:44:57.626 | INFO | "gpu" : " 17 GB", | 20260506 08:44:57.626 | INFO | "huge" : "", | 20260506 08:44:57.626 | INFO | "huge_shared" : " 27 GB", | 20260506 08:44:57.626 | INFO | "malloc" : " 15 GB", | 20260506 08:44:57.626 | INFO | "pinned" : " 29 GB" | 20260506 08:44:57.626 | INFO | } | 20260506 08:44:57.626 | INFO | Perf report for the last 5.13929 seconds: | 20260506 08:44:57.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:44:57.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:57.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:57.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:57.626 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 08:44:57.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:57.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:57.626 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 08:44:57.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:57.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:44:57.626 | INFO | Uploaded 36GiB to GPU in 1.60428s (22.44GiB/s) | 20260506 08:44:57.626 | INFO | Downloaded 34GiB to GPU in 1.52193s (22.3401GiB/s) | 20260506 08:44:57.720 | INFO | Starting a round of Baz IO | 20260506 08:45:00.887 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 3.166 seconds | 20260506 08:45:00.887 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260506 08:45:00.887 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.88887% / 0% / 0% | 20260506 08:45:00.887 | INFO | Baz IO overall: | 20260506 08:45:00.887 | INFO | BytesWritten : 2571430598 | 20260506 08:45:00.887 | INFO | NumEvents : 0 | 20260506 08:45:02.765 | INFO | Analyzing chunk frames = [20480,20992) | 20260506 08:45:02.765 | INFO | MemInfo: { | 20260506 08:45:02.765 | INFO | "gpu" : " 17 GB", | 20260506 08:45:02.765 | INFO | "huge" : "", | 20260506 08:45:02.766 | INFO | "huge_shared" : " 27 GB", | 20260506 08:45:02.766 | INFO | "malloc" : " 15 GB", | 20260506 08:45:02.766 | INFO | "pinned" : " 29 GB" | 20260506 08:45:02.766 | INFO | } | 20260506 08:45:02.766 | INFO | Perf report for the last 5.13965 seconds: | 20260506 08:45:02.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:45:02.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:02.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:02.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:02.766 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 08:45:02.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:02.766 | INFO | RT_METRICS : 32.3%, 1.0, 22.4ms, 0 | 20260506 08:45:02.766 | INFO | PRE_HQ : 55.7%, 1.9, 54.7ms, 0 | 20260506 08:45:02.766 | INFO | BAZWRITER : 10.6%, 1.0, 5.4ms, 0 | 20260506 08:45:02.766 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260506 08:45:02.766 | INFO | Uploaded 36GiB to GPU in 1.6391s (21.9633GiB/s) | 20260506 08:45:02.766 | INFO | Downloaded 38GiB to GPU in 1.70118s (22.3374GiB/s) | 20260506 08:45:07.905 | INFO | Analyzing chunk frames = [20992,21504) | 20260506 08:45:07.905 | INFO | MemInfo: { | 20260506 08:45:07.905 | INFO | "gpu" : " 17 GB", | 20260506 08:45:07.905 | INFO | "huge" : "", | 20260506 08:45:07.905 | INFO | "huge_shared" : " 27 GB", | 20260506 08:45:07.905 | INFO | "malloc" : " 15 GB", | 20260506 08:45:07.905 | INFO | "pinned" : " 29 GB" | 20260506 08:45:07.905 | INFO | } | 20260506 08:45:07.905 | INFO | Perf report for the last 5.13908 seconds: | 20260506 08:45:07.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:45:07.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:07.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:07.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:07.905 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 08:45:07.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:07.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:07.905 | INFO | PRE_HQ : 36.6%, 1.0, 19.9ms, 0 | 20260506 08:45:07.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:07.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:07.905 | INFO | Uploaded 36GiB to GPU in 1.60066s (22.4908GiB/s) | 20260506 08:45:07.905 | INFO | Downloaded 34GiB to GPU in 1.51832s (22.3931GiB/s) | 20260506 08:45:13.044 | INFO | Analyzing chunk frames = [21504,22016) | 20260506 08:45:13.044 | INFO | MemInfo: { | 20260506 08:45:13.044 | INFO | "gpu" : " 17 GB", | 20260506 08:45:13.044 | INFO | "huge" : "", | 20260506 08:45:13.044 | INFO | "huge_shared" : " 27 GB", | 20260506 08:45:13.044 | INFO | "malloc" : " 15 GB", | 20260506 08:45:13.044 | INFO | "pinned" : " 29 GB" | 20260506 08:45:13.044 | INFO | } | 20260506 08:45:13.044 | INFO | Perf report for the last 5.13946 seconds: | 20260506 08:45:13.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:45:13.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:13.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:13.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:13.044 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 08:45:13.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:13.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:13.044 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 08:45:13.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:13.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:13.044 | INFO | Uploaded 36GiB to GPU in 1.60476s (22.4333GiB/s) | 20260506 08:45:13.044 | INFO | Downloaded 34GiB to GPU in 1.52283s (22.3269GiB/s) | 20260506 08:45:18.183 | INFO | Analyzing chunk frames = [22016,22528) | 20260506 08:45:18.183 | INFO | MemInfo: { | 20260506 08:45:18.183 | INFO | "gpu" : " 17 GB", | 20260506 08:45:18.183 | INFO | "huge" : "", | 20260506 08:45:18.183 | INFO | "huge_shared" : " 27 GB", | 20260506 08:45:18.183 | INFO | "malloc" : " 15 GB", | 20260506 08:45:18.184 | INFO | "pinned" : " 29 GB" | 20260506 08:45:18.184 | INFO | } | 20260506 08:45:18.184 | INFO | Perf report for the last 5.13935 seconds: | 20260506 08:45:18.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:45:18.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:18.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:18.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:18.184 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 08:45:18.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:18.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:18.184 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 08:45:18.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:18.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:18.184 | INFO | Uploaded 36GiB to GPU in 1.6055s (22.4229GiB/s) | 20260506 08:45:18.184 | INFO | Downloaded 34GiB to GPU in 1.52318s (22.3217GiB/s) | 20260506 08:45:23.323 | INFO | Analyzing chunk frames = [22528,23040) | 20260506 08:45:23.323 | INFO | MemInfo: { | 20260506 08:45:23.323 | INFO | "gpu" : " 17 GB", | 20260506 08:45:23.323 | INFO | "huge" : "", | 20260506 08:45:23.323 | INFO | "huge_shared" : " 27 GB", | 20260506 08:45:23.323 | INFO | "malloc" : " 15 GB", | 20260506 08:45:23.323 | INFO | "pinned" : " 29 GB" | 20260506 08:45:23.323 | INFO | } | 20260506 08:45:23.323 | INFO | Perf report for the last 5.13919 seconds: | 20260506 08:45:23.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:45:23.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:23.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:23.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:23.323 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 08:45:23.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:23.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:23.323 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 08:45:23.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:23.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:23.323 | INFO | Uploaded 36GiB to GPU in 1.60277s (22.4611GiB/s) | 20260506 08:45:23.323 | INFO | Downloaded 34GiB to GPU in 1.52001s (22.3683GiB/s) | 20260506 08:45:28.462 | INFO | Analyzing chunk frames = [23040,23552) | 20260506 08:45:28.462 | INFO | MemInfo: { | 20260506 08:45:28.462 | INFO | "gpu" : " 17 GB", | 20260506 08:45:28.462 | INFO | "huge" : "", | 20260506 08:45:28.462 | INFO | "huge_shared" : " 27 GB", | 20260506 08:45:28.462 | INFO | "malloc" : " 15 GB", | 20260506 08:45:28.462 | INFO | "pinned" : " 29 GB" | 20260506 08:45:28.462 | INFO | } | 20260506 08:45:28.462 | INFO | Perf report for the last 5.13926 seconds: | 20260506 08:45:28.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:45:28.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:28.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:28.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:28.462 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260506 08:45:28.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:28.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:28.462 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 08:45:28.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:28.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:28.462 | INFO | Uploaded 36GiB to GPU in 1.60342s (22.452GiB/s) | 20260506 08:45:28.462 | INFO | Downloaded 34GiB to GPU in 1.52089s (22.3553GiB/s) | 20260506 08:45:33.601 | INFO | Analyzing chunk frames = [23552,24064) | 20260506 08:45:33.601 | INFO | MemInfo: { | 20260506 08:45:33.601 | INFO | "gpu" : " 17 GB", | 20260506 08:45:33.601 | INFO | "huge" : "", | 20260506 08:45:33.601 | INFO | "huge_shared" : " 27 GB", | 20260506 08:45:33.601 | INFO | "malloc" : " 15 GB", | 20260506 08:45:33.601 | INFO | "pinned" : " 29 GB" | 20260506 08:45:33.601 | INFO | } | 20260506 08:45:33.602 | INFO | Perf report for the last 5.13945 seconds: | 20260506 08:45:33.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:45:33.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:33.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:33.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:33.602 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 08:45:33.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:33.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:33.602 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 08:45:33.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:33.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:33.602 | INFO | Uploaded 36GiB to GPU in 1.60388s (22.4456GiB/s) | 20260506 08:45:33.602 | INFO | Downloaded 34GiB to GPU in 1.52146s (22.347GiB/s) | 20260506 08:45:38.741 | INFO | Analyzing chunk frames = [24064,24576) | 20260506 08:45:38.741 | INFO | MemInfo: { | 20260506 08:45:38.741 | INFO | "gpu" : " 17 GB", | 20260506 08:45:38.741 | INFO | "huge" : "", | 20260506 08:45:38.741 | INFO | "huge_shared" : " 27 GB", | 20260506 08:45:38.741 | INFO | "malloc" : " 15 GB", | 20260506 08:45:38.741 | INFO | "pinned" : " 29 GB" | 20260506 08:45:38.741 | INFO | } | 20260506 08:45:38.741 | INFO | Perf report for the last 5.13933 seconds: | 20260506 08:45:38.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:45:38.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:38.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:38.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:38.741 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 08:45:38.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:38.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:38.741 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 08:45:38.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:38.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:38.741 | INFO | Uploaded 36GiB to GPU in 1.60345s (22.4516GiB/s) | 20260506 08:45:38.741 | INFO | Downloaded 34GiB to GPU in 1.52096s (22.3543GiB/s) | 20260506 08:45:38.741 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 08:45:38.741 | INFO | Total mem usage: 14.7 GiB | 20260506 08:45:38.741 | INFO | All Time High Water Marks sum: 14.7 GiB (Savings: 0.0524%) | 20260506 08:45:38.741 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: 0.0524%) | 20260506 08:45:38.741 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:45:38.741 | INFO | High Water Breakdown: | 20260506 08:45:38.741 | INFO | 54.8% : 8.04 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 08:45:38.741 | INFO | 45.1% : 6.61 GiB ( 100% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 08:45:38.741 | INFO | 0.127% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260506 08:45:38.742 | INFO | ------------------------------------------ | 20260506 08:45:38.742 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:45:38.742 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 08:45:38.742 | INFO | Total mem usage: 26.5 GiB | 20260506 08:45:38.742 | INFO | All Time High Water Marks sum: 27 GiB (Savings: 1.91%) | 20260506 08:45:38.742 | INFO | Recent High Water Marks Sum: 26.2 GiB (Savings: -1.07%) | 20260506 08:45:38.742 | INFO | Unused Cache: 1.03 GiB (Waste: 3.9%) | 20260506 08:45:38.742 | INFO | High Water Breakdown: | 20260506 08:45:38.742 | INFO | 54.7% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:45:38.742 | INFO | 35.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:45:38.742 | INFO | 3.57% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:45:38.742 | INFO | 1.93% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 08:45:38.742 | INFO | 1.56% : 418 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:45:38.742 | INFO | 1.43% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:45:38.742 | INFO | 0.998% : 268 MiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 08:45:38.742 | INFO | 0.408% : 110 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:45:38.742 | INFO | 0.0149% : 4 MiB ( 100% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 08:45:38.742 | INFO | 0.0149% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 08:45:38.742 | INFO | 0.00148% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 08:45:38.742 | INFO | 9.13e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 08:45:38.742 | INFO | 8.52e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:45:38.742 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 08:45:38.742 | INFO | ------------------------------------------ | 20260506 08:45:38.742 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:45:38.742 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 08:45:38.742 | INFO | Total mem usage: 25.6 GiB | 20260506 08:45:38.742 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 08:45:38.742 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260506 08:45:38.742 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260506 08:45:38.742 | INFO | High Water Breakdown: | 20260506 08:45:38.742 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260506 08:45:38.742 | INFO | ------------------------------------------ | 20260506 08:45:38.742 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:45:38.742 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 08:45:38.742 | INFO | Total mem usage: 15.7 GiB | 20260506 08:45:38.742 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 08:45:38.742 | INFO | Recent High Water Marks Sum: 17 GiB (Savings: 7.99%) | 20260506 08:45:38.742 | INFO | Unused Cache: 786 MiB (Waste: 4.89%) | 20260506 08:45:38.742 | INFO | High Water Breakdown: | 20260506 08:45:38.742 | INFO | 31.6% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 08:45:38.742 | INFO | 12.9% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 08:45:38.742 | INFO | 8.25% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 08:45:38.742 | INFO | 8.14% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 08:45:38.742 | INFO | 6.05% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 08:45:38.742 | INFO | 5.92% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 08:45:38.742 | INFO | 4.73% : 824 MiB ( 100% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 08:45:38.742 | INFO | 3.3% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 08:45:38.742 | INFO | 3.3% : 576 MiB ( 50% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 08:45:38.742 | INFO | 2.96% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 08:45:38.742 | INFO | 2.63% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:45:38.742 | INFO | 2.2% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 08:45:38.742 | INFO | 1.7% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:45:38.742 | INFO | 1.6% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 08:45:38.742 | INFO | 1.53% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 08:45:38.742 | INFO | 0.734% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 08:45:38.742 | INFO | 0.734% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 08:45:38.742 | INFO | 0.602% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:45:38.742 | INFO | 0.55% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 08:45:38.742 | INFO | 0.209% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:45:38.742 | INFO | 0.172% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:45:38.742 | INFO | 0.0688% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:45:38.742 | INFO | 0.0229% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 08:45:38.742 | INFO | 0.00573% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 08:45:38.742 | INFO | 1.31e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:45:38.742 | INFO | ------------------------------------------ | 20260506 08:45:38.742 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:45:38.742 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 08:45:38.742 | INFO | PerfReport: Gathered 288 samples. | 20260506 08:45:38.742 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:45:38.742 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.11e+01 / 0.00e+00 | 20260506 08:45:38.742 | INFO | PerfReport: 0.28%, AggregateStats -- 8.57e-02 / 0.00e+00, 3.35e-03 / 0.00e+00 | 20260506 08:45:38.742 | INFO | PerfReport: 10.89%, Baseline -- 3.29e+00 / 0.00e+00, 1.40e-02 / 0.00e+00 | 20260506 08:45:38.742 | INFO | PerfReport: 3.01%, Histogram -- 9.09e-01 / 0.00e+00, 4.41e-02 / 0.00e+00 | 20260506 08:45:38.742 | INFO | PerfReport: 16.26%, DME -- 3.93e+01 / 0.00e+00, 1.42e+00 / 0.00e+00 | 20260506 08:45:38.742 | INFO | PerfReport: 56.97%, FrameLabeling -- 1.72e+01 / 0.00e+00, 3.94e-02 / 0.00e+00 | 20260506 08:45:38.742 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.27e+00 / 0.00e+00, 1.35e-01 / 0.00e+00 | 20260506 08:45:38.742 | INFO | PerfReport: 3.57%, Metrics -- 1.08e+00 / 0.00e+00, 1.44e-01 / 0.00e+00 | 20260506 08:45:38.742 | INFO | PerfReport: 1.50%, PulseSerializing -- 4.55e-01 / 0.00e+00, 6.93e-02 / 0.00e+00 | 20260506 08:45:38.742 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 08:45:38.742 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:45:38.742 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:45:38.743 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.76e+01 / 0.00e+00 | 20260506 08:45:38.743 | INFO | PerfReport: 17.38%, Upload -- 1.68e+01 / 0.00e+00, 2.53e+00 / 0.00e+00 | 20260506 08:45:38.743 | INFO | PerfReport: 16.47%, Download -- 1.60e+01 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260506 08:45:38.743 | INFO | PerfReport: 33.31%, Compute -- 3.23e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260506 08:45:38.743 | INFO | PerfReport: 0.52%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 08:45:38.743 | INFO | PerfReport: 0.02%, LockDownload -- 1.83e-02 / 0.00e+00, 4.01e-01 / 0.00e+00 | 20260506 08:45:38.743 | INFO | PerfReport: 32.29%, LockCompute -- 3.13e+01 / 0.00e+00, 2.45e+01 / 0.00e+00 | 20260506 08:45:38.743 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260506 08:45:38.833 | INFO | Starting a round of Baz IO | 20260506 08:45:41.767 | INFO | Populating RTMetrics report | 20260506 08:45:41.768 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.934 seconds | 20260506 08:45:41.768 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260506 08:45:41.768 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.83478% / 0% / 0% | 20260506 08:45:41.768 | INFO | Baz IO overall: | 20260506 08:45:41.768 | INFO | BytesWritten : 2571434150 | 20260506 08:45:41.768 | INFO | NumEvents : 0 | 20260506 08:45:41.768 | INFO | RTMetrics report populated | 20260506 08:45:41.768 | INFO | Opening tmp RTMetrics json file | 20260506 08:45:41.770 | INFO | Serializing RTMetrics to json | 20260506 08:45:41.771 | INFO | Serialization done, swapping tmp file now... | 20260506 08:45:41.771 | INFO | RTMetrics done | 20260506 08:45:43.880 | INFO | Analyzing chunk frames = [24576,25088) | 20260506 08:45:43.880 | INFO | MemInfo: { | 20260506 08:45:43.880 | INFO | "gpu" : " 17 GB", | 20260506 08:45:43.880 | INFO | "huge" : "", | 20260506 08:45:43.880 | INFO | "huge_shared" : " 27 GB", | 20260506 08:45:43.880 | INFO | "malloc" : " 15 GB", | 20260506 08:45:43.880 | INFO | "pinned" : " 29 GB" | 20260506 08:45:43.880 | INFO | } | 20260506 08:45:43.880 | INFO | Perf report for the last 5.13915 seconds: | 20260506 08:45:43.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:45:43.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:43.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:43.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:43.880 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260506 08:45:43.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:43.880 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260506 08:45:43.880 | INFO | PRE_HQ : 55.3%, 1.9, 54.4ms, 0 | 20260506 08:45:43.880 | INFO | BAZWRITER : 10.3%, 1.0, 5.3ms, 0 | 20260506 08:45:43.880 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260506 08:45:43.880 | INFO | Uploaded 36GiB to GPU in 1.6743s (21.5016GiB/s) | 20260506 08:45:43.880 | INFO | Downloaded 38GiB to GPU in 1.73851s (21.8578GiB/s) | 20260506 08:45:49.019 | INFO | Analyzing chunk frames = [25088,25600) | 20260506 08:45:49.019 | INFO | MemInfo: { | 20260506 08:45:49.019 | INFO | "gpu" : " 17 GB", | 20260506 08:45:49.019 | INFO | "huge" : "", | 20260506 08:45:49.019 | INFO | "huge_shared" : " 27 GB", | 20260506 08:45:49.019 | INFO | "malloc" : " 15 GB", | 20260506 08:45:49.019 | INFO | "pinned" : " 29 GB" | 20260506 08:45:49.019 | INFO | } | 20260506 08:45:49.019 | INFO | Perf report for the last 5.13945 seconds: | 20260506 08:45:49.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:45:49.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:49.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:49.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:49.020 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260506 08:45:49.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:49.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:49.020 | INFO | PRE_HQ : 35.4%, 1.1, 19.4ms, 0 | 20260506 08:45:49.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:49.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:49.020 | INFO | Uploaded 36GiB to GPU in 1.60129s (22.4819GiB/s) | 20260506 08:45:49.020 | INFO | Downloaded 34GiB to GPU in 1.51776s (22.4014GiB/s) | 20260506 08:45:54.159 | INFO | Analyzing chunk frames = [25600,26112) | 20260506 08:45:54.159 | INFO | MemInfo: { | 20260506 08:45:54.159 | INFO | "gpu" : " 17 GB", | 20260506 08:45:54.159 | INFO | "huge" : "", | 20260506 08:45:54.159 | INFO | "huge_shared" : " 27 GB", | 20260506 08:45:54.159 | INFO | "malloc" : " 15 GB", | 20260506 08:45:54.159 | INFO | "pinned" : " 29 GB" | 20260506 08:45:54.159 | INFO | } | 20260506 08:45:54.159 | INFO | Perf report for the last 5.13935 seconds: | 20260506 08:45:54.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:45:54.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:54.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:54.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:54.159 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260506 08:45:54.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:54.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:54.159 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 08:45:54.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:54.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:54.159 | INFO | Uploaded 36GiB to GPU in 1.60481s (22.4325GiB/s) | 20260506 08:45:54.159 | INFO | Downloaded 34GiB to GPU in 1.52239s (22.3334GiB/s) | 20260506 08:45:59.298 | INFO | Analyzing chunk frames = [26112,26624) | 20260506 08:45:59.298 | INFO | MemInfo: { | 20260506 08:45:59.298 | INFO | "gpu" : " 17 GB", | 20260506 08:45:59.298 | INFO | "huge" : "", | 20260506 08:45:59.298 | INFO | "huge_shared" : " 27 GB", | 20260506 08:45:59.298 | INFO | "malloc" : " 15 GB", | 20260506 08:45:59.298 | INFO | "pinned" : " 29 GB" | 20260506 08:45:59.298 | INFO | } | 20260506 08:45:59.298 | INFO | Perf report for the last 5.13934 seconds: | 20260506 08:45:59.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:45:59.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:59.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:59.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:59.298 | INFO | ANALYSIS : 56.8%, 3.0, 87.9ms, 0 | 20260506 08:45:59.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:59.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:59.298 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 08:45:59.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:59.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:45:59.298 | INFO | Uploaded 36GiB to GPU in 1.60496s (22.4305GiB/s) | 20260506 08:45:59.299 | INFO | Downloaded 34GiB to GPU in 1.52251s (22.3315GiB/s) | 20260506 08:46:04.437 | INFO | Analyzing chunk frames = [26624,27136) | 20260506 08:46:04.437 | INFO | MemInfo: { | 20260506 08:46:04.437 | INFO | "gpu" : " 17 GB", | 20260506 08:46:04.437 | INFO | "huge" : "", | 20260506 08:46:04.437 | INFO | "huge_shared" : " 27 GB", | 20260506 08:46:04.437 | INFO | "malloc" : " 15 GB", | 20260506 08:46:04.438 | INFO | "pinned" : " 29 GB" | 20260506 08:46:04.438 | INFO | } | 20260506 08:46:04.438 | INFO | Perf report for the last 5.13935 seconds: | 20260506 08:46:04.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:46:04.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:04.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:04.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:04.438 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 08:46:04.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:04.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:04.438 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 08:46:04.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:04.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:04.438 | INFO | Uploaded 36GiB to GPU in 1.60247s (22.4653GiB/s) | 20260506 08:46:04.438 | INFO | Downloaded 34GiB to GPU in 1.5191s (22.3817GiB/s) | 20260506 08:46:09.577 | INFO | Analyzing chunk frames = [27136,27648) | 20260506 08:46:09.577 | INFO | MemInfo: { | 20260506 08:46:09.577 | INFO | "gpu" : " 17 GB", | 20260506 08:46:09.577 | INFO | "huge" : "", | 20260506 08:46:09.577 | INFO | "huge_shared" : " 27 GB", | 20260506 08:46:09.577 | INFO | "malloc" : " 15 GB", | 20260506 08:46:09.577 | INFO | "pinned" : " 29 GB" | 20260506 08:46:09.577 | INFO | } | 20260506 08:46:09.577 | INFO | Perf report for the last 5.13934 seconds: | 20260506 08:46:09.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:46:09.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:09.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:09.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:09.577 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 08:46:09.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:09.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:09.577 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 08:46:09.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:09.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:09.577 | INFO | Uploaded 36GiB to GPU in 1.60269s (22.4622GiB/s) | 20260506 08:46:09.577 | INFO | Downloaded 34GiB to GPU in 1.51997s (22.3689GiB/s) | 20260506 08:46:14.716 | INFO | Analyzing chunk frames = [27648,28160) | 20260506 08:46:14.716 | INFO | MemInfo: { | 20260506 08:46:14.716 | INFO | "gpu" : " 17 GB", | 20260506 08:46:14.716 | INFO | "huge" : "", | 20260506 08:46:14.716 | INFO | "huge_shared" : " 27 GB", | 20260506 08:46:14.716 | INFO | "malloc" : " 15 GB", | 20260506 08:46:14.716 | INFO | "pinned" : " 29 GB" | 20260506 08:46:14.716 | INFO | } | 20260506 08:46:14.716 | INFO | Perf report for the last 5.13915 seconds: | 20260506 08:46:14.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:46:14.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:14.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:14.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:14.716 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 08:46:14.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:14.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:14.716 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 08:46:14.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:14.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:14.717 | INFO | Uploaded 36GiB to GPU in 1.60245s (22.4656GiB/s) | 20260506 08:46:14.717 | INFO | Downloaded 34GiB to GPU in 1.52052s (22.3607GiB/s) | 20260506 08:46:19.855 | INFO | Analyzing chunk frames = [28160,28672) | 20260506 08:46:19.855 | INFO | MemInfo: { | 20260506 08:46:19.855 | INFO | "gpu" : " 17 GB", | 20260506 08:46:19.855 | INFO | "huge" : "", | 20260506 08:46:19.855 | INFO | "huge_shared" : " 27 GB", | 20260506 08:46:19.855 | INFO | "malloc" : " 15 GB", | 20260506 08:46:19.855 | INFO | "pinned" : " 29 GB" | 20260506 08:46:19.855 | INFO | } | 20260506 08:46:19.855 | INFO | Perf report for the last 5.13931 seconds: | 20260506 08:46:19.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:46:19.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:19.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:19.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:19.856 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 08:46:19.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:19.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:19.856 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 08:46:19.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:19.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:19.856 | INFO | Uploaded 36GiB to GPU in 1.60365s (22.4488GiB/s) | 20260506 08:46:19.856 | INFO | Downloaded 34GiB to GPU in 1.52108s (22.3526GiB/s) | 20260506 08:46:19.950 | INFO | Starting a round of Baz IO | 20260506 08:46:22.873 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.922 seconds | 20260506 08:46:22.873 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260506 08:46:22.873 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.3089% / 0% / 0% | 20260506 08:46:22.873 | INFO | Baz IO overall: | 20260506 08:46:22.873 | INFO | BytesWritten : 2571437702 | 20260506 08:46:22.873 | INFO | NumEvents : 0 | 20260506 08:46:22.873 | INFO | Populating RTMetrics report | 20260506 08:46:22.874 | INFO | RTMetrics report populated | 20260506 08:46:22.874 | INFO | Opening tmp RTMetrics json file | 20260506 08:46:22.874 | INFO | Serializing RTMetrics to json | 20260506 08:46:22.876 | INFO | Serialization done, swapping tmp file now... | 20260506 08:46:22.876 | INFO | RTMetrics done | 20260506 08:46:24.995 | INFO | Analyzing chunk frames = [28672,29184) | 20260506 08:46:24.995 | INFO | MemInfo: { | 20260506 08:46:24.995 | INFO | "gpu" : " 17 GB", | 20260506 08:46:24.995 | INFO | "huge" : "", | 20260506 08:46:24.995 | INFO | "huge_shared" : " 27 GB", | 20260506 08:46:24.995 | INFO | "malloc" : " 15 GB", | 20260506 08:46:24.995 | INFO | "pinned" : " 29 GB" | 20260506 08:46:24.995 | INFO | } | 20260506 08:46:24.995 | INFO | Perf report for the last 5.13932 seconds: | 20260506 08:46:24.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:46:24.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:24.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:24.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:24.995 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260506 08:46:24.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:24.995 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260506 08:46:24.995 | INFO | PRE_HQ : 55.1%, 1.9, 53.7ms, 0 | 20260506 08:46:24.995 | INFO | BAZWRITER : 10.3%, 1.0, 5.3ms, 0 | 20260506 08:46:24.995 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260506 08:46:24.995 | INFO | Uploaded 36GiB to GPU in 1.64946s (21.8254GiB/s) | 20260506 08:46:24.995 | INFO | Downloaded 38GiB to GPU in 1.71169s (22.2003GiB/s) | 20260506 08:46:30.134 | INFO | Analyzing chunk frames = [29184,29696) | 20260506 08:46:30.134 | INFO | MemInfo: { | 20260506 08:46:30.134 | INFO | "gpu" : " 17 GB", | 20260506 08:46:30.134 | INFO | "huge" : "", | 20260506 08:46:30.134 | INFO | "huge_shared" : " 27 GB", | 20260506 08:46:30.134 | INFO | "malloc" : " 15 GB", | 20260506 08:46:30.134 | INFO | "pinned" : " 29 GB" | 20260506 08:46:30.134 | INFO | } | 20260506 08:46:30.134 | INFO | Perf report for the last 5.13951 seconds: | 20260506 08:46:30.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:46:30.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:30.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:30.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:30.134 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260506 08:46:30.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:30.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:30.134 | INFO | PRE_HQ : 34.3%, 1.0, 18.1ms, 0 | 20260506 08:46:30.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:30.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:30.134 | INFO | Uploaded 36GiB to GPU in 1.60129s (22.4819GiB/s) | 20260506 08:46:30.134 | INFO | Downloaded 34GiB to GPU in 1.51808s (22.3968GiB/s) | 20260506 08:46:35.274 | INFO | Analyzing chunk frames = [29696,30208) | 20260506 08:46:35.274 | INFO | MemInfo: { | 20260506 08:46:35.274 | INFO | "gpu" : " 17 GB", | 20260506 08:46:35.274 | INFO | "huge" : "", | 20260506 08:46:35.274 | INFO | "huge_shared" : " 27 GB", | 20260506 08:46:35.274 | INFO | "malloc" : " 15 GB", | 20260506 08:46:35.274 | INFO | "pinned" : " 29 GB" | 20260506 08:46:35.274 | INFO | } | 20260506 08:46:35.274 | INFO | Perf report for the last 5.13952 seconds: | 20260506 08:46:35.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:46:35.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:35.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:35.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:35.274 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260506 08:46:35.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:35.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:35.274 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260506 08:46:35.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:35.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:35.274 | INFO | Uploaded 36GiB to GPU in 1.60432s (22.4394GiB/s) | 20260506 08:46:35.274 | INFO | Downloaded 34GiB to GPU in 1.52149s (22.3466GiB/s) | 20260506 08:46:40.413 | INFO | Analyzing chunk frames = [30208,30720) | 20260506 08:46:40.413 | INFO | MemInfo: { | 20260506 08:46:40.413 | INFO | "gpu" : " 17 GB", | 20260506 08:46:40.413 | INFO | "huge" : "", | 20260506 08:46:40.413 | INFO | "huge_shared" : " 27 GB", | 20260506 08:46:40.413 | INFO | "malloc" : " 15 GB", | 20260506 08:46:40.413 | INFO | "pinned" : " 29 GB" | 20260506 08:46:40.413 | INFO | } | 20260506 08:46:40.413 | INFO | Perf report for the last 5.13916 seconds: | 20260506 08:46:40.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:46:40.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:40.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:40.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:40.413 | INFO | ANALYSIS : 56.8%, 3.0, 88.0ms, 0 | 20260506 08:46:40.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:40.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:40.413 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 08:46:40.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:40.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:40.413 | INFO | Uploaded 36GiB to GPU in 1.60442s (22.438GiB/s) | 20260506 08:46:40.413 | INFO | Downloaded 34GiB to GPU in 1.52238s (22.3334GiB/s) | 20260506 08:46:45.552 | INFO | Analyzing chunk frames = [30720,31232) | 20260506 08:46:45.552 | INFO | MemInfo: { | 20260506 08:46:45.552 | INFO | "gpu" : " 17 GB", | 20260506 08:46:45.553 | INFO | "huge" : "", | 20260506 08:46:45.553 | INFO | "huge_shared" : " 27 GB", | 20260506 08:46:45.553 | INFO | "malloc" : " 15 GB", | 20260506 08:46:45.553 | INFO | "pinned" : " 29 GB" | 20260506 08:46:45.553 | INFO | } | 20260506 08:46:45.553 | INFO | Perf report for the last 5.13962 seconds: | 20260506 08:46:45.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:46:45.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:45.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:45.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:45.553 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 08:46:45.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:45.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:45.553 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 08:46:45.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:45.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:45.553 | INFO | Uploaded 36GiB to GPU in 1.60205s (22.4712GiB/s) | 20260506 08:46:45.553 | INFO | Downloaded 34GiB to GPU in 1.51882s (22.3858GiB/s) | 20260506 08:46:50.692 | INFO | Analyzing chunk frames = [31232,31744) | 20260506 08:46:50.692 | INFO | MemInfo: { | 20260506 08:46:50.692 | INFO | "gpu" : " 17 GB", | 20260506 08:46:50.692 | INFO | "huge" : "", | 20260506 08:46:50.692 | INFO | "huge_shared" : " 27 GB", | 20260506 08:46:50.692 | INFO | "malloc" : " 15 GB", | 20260506 08:46:50.692 | INFO | "pinned" : " 29 GB" | 20260506 08:46:50.692 | INFO | } | 20260506 08:46:50.692 | INFO | Perf report for the last 5.13904 seconds: | 20260506 08:46:50.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:46:50.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:50.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:50.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:50.692 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 08:46:50.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:50.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:50.692 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 08:46:50.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:50.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:50.692 | INFO | Uploaded 36GiB to GPU in 1.60278s (22.4609GiB/s) | 20260506 08:46:50.692 | INFO | Downloaded 34GiB to GPU in 1.52009s (22.3671GiB/s) | 20260506 08:46:55.831 | INFO | Analyzing chunk frames = [31744,32256) | 20260506 08:46:55.831 | INFO | MemInfo: { | 20260506 08:46:55.831 | INFO | "gpu" : " 17 GB", | 20260506 08:46:55.831 | INFO | "huge" : "", | 20260506 08:46:55.831 | INFO | "huge_shared" : " 27 GB", | 20260506 08:46:55.831 | INFO | "malloc" : " 15 GB", | 20260506 08:46:55.831 | INFO | "pinned" : " 29 GB" | 20260506 08:46:55.831 | INFO | } | 20260506 08:46:55.831 | INFO | Perf report for the last 5.1393 seconds: | 20260506 08:46:55.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:46:55.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:55.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:55.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:55.831 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 08:46:55.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:55.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:55.831 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 08:46:55.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:55.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:46:55.831 | INFO | Uploaded 36GiB to GPU in 1.60242s (22.466GiB/s) | 20260506 08:46:55.831 | INFO | Downloaded 34GiB to GPU in 1.52001s (22.3682GiB/s) | 20260506 08:47:00.970 | INFO | Analyzing chunk frames = [32256,32768) | 20260506 08:47:00.970 | INFO | MemInfo: { | 20260506 08:47:00.970 | INFO | "gpu" : " 17 GB", | 20260506 08:47:00.970 | INFO | "huge" : "", | 20260506 08:47:00.970 | INFO | "huge_shared" : " 27 GB", | 20260506 08:47:00.970 | INFO | "malloc" : " 15 GB", | 20260506 08:47:00.970 | INFO | "pinned" : " 29 GB" | 20260506 08:47:00.970 | INFO | } | 20260506 08:47:00.970 | INFO | Perf report for the last 5.13943 seconds: | 20260506 08:47:00.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:47:00.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:00.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:00.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:00.970 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 08:47:00.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:00.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:00.970 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 08:47:00.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:00.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:00.971 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 08:47:00.971 | INFO | Total mem usage: 14.7 GiB | 20260506 08:47:00.971 | INFO | All Time High Water Marks sum: 14.7 GiB (Savings: 0.0524%) | 20260506 08:47:00.971 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: 0.0524%) | 20260506 08:47:00.971 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:47:00.971 | INFO | High Water Breakdown: | 20260506 08:47:00.971 | INFO | 54.8% : 8.04 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 08:47:00.971 | INFO | 45.1% : 6.61 GiB ( 100% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 08:47:00.971 | INFO | 0.127% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260506 08:47:00.971 | INFO | ------------------------------------------ | 20260506 08:47:00.971 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:47:00.971 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 08:47:00.971 | INFO | Total mem usage: 26.5 GiB | 20260506 08:47:00.971 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 08:47:00.971 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 08:47:00.971 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 08:47:00.971 | INFO | High Water Breakdown: | 20260506 08:47:00.971 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:47:00.971 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:47:00.971 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:47:00.971 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:47:00.971 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:47:00.971 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:47:00.971 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 08:47:00.971 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 08:47:00.971 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:47:00.971 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 08:47:00.971 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 08:47:00.971 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 08:47:00.971 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 08:47:00.971 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 08:47:00.971 | INFO | ------------------------------------------ | 20260506 08:47:00.971 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:47:00.971 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 08:47:00.971 | INFO | Total mem usage: 25.6 GiB | 20260506 08:47:00.971 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 08:47:00.971 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 08:47:00.971 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 08:47:00.971 | INFO | High Water Breakdown: | 20260506 08:47:00.971 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 08:47:00.971 | INFO | ------------------------------------------ | 20260506 08:47:00.971 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:47:00.971 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 08:47:00.971 | INFO | Total mem usage: 15.7 GiB | 20260506 08:47:00.971 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 08:47:00.971 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 08:47:00.971 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 08:47:00.971 | INFO | High Water Breakdown: | 20260506 08:47:00.971 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 08:47:00.971 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 08:47:00.971 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 08:47:00.971 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 08:47:00.971 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 08:47:00.971 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 08:47:00.971 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 08:47:00.971 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 08:47:00.971 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 08:47:00.971 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:47:00.971 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 08:47:00.971 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:47:00.971 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 08:47:00.971 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 08:47:00.971 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 08:47:00.971 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 08:47:00.971 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:47:00.971 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 08:47:00.971 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:47:00.971 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:47:00.971 | INFO | Uploaded 36GiB to GPU in 1.6029s (22.4593GiB/s) | 20260506 08:47:00.972 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:47:00.972 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 08:47:00.972 | INFO | Downloaded 34GiB to GPU in 1.52122s (22.3505GiB/s) | 20260506 08:47:00.972 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 08:47:00.972 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:47:00.972 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 08:47:00.972 | INFO | ------------------------------------------ | 20260506 08:47:00.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:47:00.972 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 08:47:00.972 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:47:00.972 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:47:00.972 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 08:47:00.972 | INFO | PerfReport: 0.28%, AggregateStats -- 8.60e-02 / 0.00e+00, 4.36e-03 / 0.00e+00 | 20260506 08:47:00.972 | INFO | PerfReport: 10.88%, Baseline -- 3.29e+00 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260506 08:47:00.972 | INFO | PerfReport: 3.00%, Histogram -- 9.09e-01 / 0.00e+00, 5.09e-02 / 0.00e+00 | 20260506 08:47:00.972 | INFO | PerfReport: 16.07%, DME -- 3.89e+01 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 08:47:00.972 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.29e-02 / 0.00e+00 | 20260506 08:47:00.972 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.29e+00 / 0.00e+00, 2.69e-01 / 0.00e+00 | 20260506 08:47:00.972 | INFO | PerfReport: 3.84%, Metrics -- 1.16e+00 / 0.00e+00, 1.83e-01 / 0.00e+00 | 20260506 08:47:00.972 | INFO | PerfReport: 1.52%, PulseSerializing -- 4.61e-01 / 0.00e+00, 6.94e-02 / 0.00e+00 | 20260506 08:47:00.972 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 08:47:00.972 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:47:00.972 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:47:00.972 | INFO | PerfReport: Full Execution -- 9.09e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260506 08:47:00.972 | INFO | PerfReport: 18.45%, Upload -- 1.68e+01 / 0.00e+00, 7.31e-01 / 0.00e+00 | 20260506 08:47:00.972 | INFO | PerfReport: 17.71%, Download -- 1.61e+01 / 0.00e+00, 1.01e+00 / 0.00e+00 | 20260506 08:47:00.972 | INFO | PerfReport: 33.29%, Compute -- 3.03e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 08:47:00.972 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 08:47:00.972 | INFO | PerfReport: 0.01%, LockDownload -- 7.31e-03 / 0.00e+00, 2.19e-01 / 0.00e+00 | 20260506 08:47:00.972 | INFO | PerfReport: 29.97%, LockCompute -- 2.73e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 08:47:00.972 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.04e-02 / 0.00e+00 | 20260506 08:47:01.064 | INFO | Starting a round of Baz IO | 20260506 08:47:03.987 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.923 seconds | 20260506 08:47:03.987 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260506 08:47:03.987 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.8055% / 0% / 0% | 20260506 08:47:03.987 | INFO | Baz IO overall: | 20260506 08:47:03.987 | INFO | BytesWritten : 2571441254 | 20260506 08:47:03.987 | INFO | NumEvents : 0 | 20260506 08:47:03.989 | INFO | Populating RTMetrics report | 20260506 08:47:03.989 | INFO | RTMetrics report populated | 20260506 08:47:03.989 | INFO | Opening tmp RTMetrics json file | 20260506 08:47:03.990 | INFO | Serializing RTMetrics to json | 20260506 08:47:03.991 | INFO | Serialization done, swapping tmp file now... | 20260506 08:47:03.992 | INFO | RTMetrics done | 20260506 08:47:06.110 | INFO | Analyzing chunk frames = [32768,33280) | 20260506 08:47:06.110 | INFO | MemInfo: { | 20260506 08:47:06.110 | INFO | "gpu" : " 17 GB", | 20260506 08:47:06.110 | INFO | "huge" : "", | 20260506 08:47:06.110 | INFO | "huge_shared" : " 27 GB", | 20260506 08:47:06.110 | INFO | "malloc" : " 15 GB", | 20260506 08:47:06.110 | INFO | "pinned" : " 29 GB" | 20260506 08:47:06.110 | INFO | } | 20260506 08:47:06.110 | INFO | Perf report for the last 5.13916 seconds: | 20260506 08:47:06.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:47:06.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:06.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:06.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:06.110 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260506 08:47:06.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:06.110 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260506 08:47:06.110 | INFO | PRE_HQ : 55.0%, 1.9, 53.3ms, 0 | 20260506 08:47:06.110 | INFO | BAZWRITER : 10.7%, 1.0, 5.5ms, 0 | 20260506 08:47:06.110 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260506 08:47:06.110 | INFO | Uploaded 36GiB to GPU in 1.6476s (21.85GiB/s) | 20260506 08:47:06.110 | INFO | Downloaded 38GiB to GPU in 1.70923s (22.2322GiB/s) | 20260506 08:47:11.249 | INFO | Analyzing chunk frames = [33280,33792) | 20260506 08:47:11.249 | INFO | MemInfo: { | 20260506 08:47:11.249 | INFO | "gpu" : " 17 GB", | 20260506 08:47:11.249 | INFO | "huge" : "", | 20260506 08:47:11.249 | INFO | "huge_shared" : " 27 GB", | 20260506 08:47:11.249 | INFO | "malloc" : " 15 GB", | 20260506 08:47:11.249 | INFO | "pinned" : " 29 GB" | 20260506 08:47:11.249 | INFO | } | 20260506 08:47:11.249 | INFO | Perf report for the last 5.13961 seconds: | 20260506 08:47:11.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:47:11.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:11.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:11.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:11.250 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260506 08:47:11.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:11.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:11.250 | INFO | PRE_HQ : 33.0%, 1.0, 17.2ms, 0 | 20260506 08:47:11.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:11.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:11.250 | INFO | Uploaded 36GiB to GPU in 1.60053s (22.4926GiB/s) | 20260506 08:47:11.250 | INFO | Downloaded 34GiB to GPU in 1.51793s (22.3989GiB/s) | 20260506 08:47:16.388 | INFO | Analyzing chunk frames = [33792,34304) | 20260506 08:47:16.388 | INFO | MemInfo: { | 20260506 08:47:16.388 | INFO | "gpu" : " 17 GB", | 20260506 08:47:16.388 | INFO | "huge" : "", | 20260506 08:47:16.388 | INFO | "huge_shared" : " 27 GB", | 20260506 08:47:16.388 | INFO | "malloc" : " 15 GB", | 20260506 08:47:16.388 | INFO | "pinned" : " 29 GB" | 20260506 08:47:16.388 | INFO | } | 20260506 08:47:16.388 | INFO | Perf report for the last 5.13917 seconds: | 20260506 08:47:16.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:47:16.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:16.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:16.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:16.388 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260506 08:47:16.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:16.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:16.389 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 08:47:16.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:16.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:16.389 | INFO | Uploaded 36GiB to GPU in 1.60479s (22.4328GiB/s) | 20260506 08:47:16.389 | INFO | Downloaded 34GiB to GPU in 1.52377s (22.3131GiB/s) | 20260506 08:47:21.528 | INFO | Analyzing chunk frames = [34304,34816) | 20260506 08:47:21.528 | INFO | MemInfo: { | 20260506 08:47:21.528 | INFO | "gpu" : " 17 GB", | 20260506 08:47:21.528 | INFO | "huge" : "", | 20260506 08:47:21.528 | INFO | "huge_shared" : " 27 GB", | 20260506 08:47:21.528 | INFO | "malloc" : " 15 GB", | 20260506 08:47:21.528 | INFO | "pinned" : " 29 GB" | 20260506 08:47:21.528 | INFO | } | 20260506 08:47:21.528 | INFO | Perf report for the last 5.13931 seconds: | 20260506 08:47:21.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:47:21.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:21.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:21.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:21.528 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260506 08:47:21.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:21.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:21.528 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260506 08:47:21.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:21.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:21.528 | INFO | Uploaded 36GiB to GPU in 1.60199s (22.4721GiB/s) | 20260506 08:47:21.528 | INFO | Downloaded 34GiB to GPU in 1.52108s (22.3526GiB/s) | 20260506 08:47:26.667 | INFO | Analyzing chunk frames = [34816,35328) | 20260506 08:47:26.667 | INFO | MemInfo: { | 20260506 08:47:26.667 | INFO | "gpu" : " 17 GB", | 20260506 08:47:26.667 | INFO | "huge" : "", | 20260506 08:47:26.667 | INFO | "huge_shared" : " 27 GB", | 20260506 08:47:26.667 | INFO | "malloc" : " 15 GB", | 20260506 08:47:26.667 | INFO | "pinned" : " 29 GB" | 20260506 08:47:26.667 | INFO | } | 20260506 08:47:26.667 | INFO | Perf report for the last 5.13938 seconds: | 20260506 08:47:26.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:47:26.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:26.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:26.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:26.667 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 08:47:26.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:26.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:26.667 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 08:47:26.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:26.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:26.667 | INFO | Uploaded 36GiB to GPU in 1.60067s (22.4906GiB/s) | 20260506 08:47:26.668 | INFO | Downloaded 34GiB to GPU in 1.51918s (22.3804GiB/s) | 20260506 08:47:31.806 | INFO | Analyzing chunk frames = [35328,35840) | 20260506 08:47:31.806 | INFO | MemInfo: { | 20260506 08:47:31.806 | INFO | "gpu" : " 17 GB", | 20260506 08:47:31.806 | INFO | "huge" : "", | 20260506 08:47:31.806 | INFO | "huge_shared" : " 27 GB", | 20260506 08:47:31.806 | INFO | "malloc" : " 15 GB", | 20260506 08:47:31.806 | INFO | "pinned" : " 29 GB" | 20260506 08:47:31.806 | INFO | } | 20260506 08:47:31.807 | INFO | Perf report for the last 5.13938 seconds: | 20260506 08:47:31.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:47:31.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:31.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:31.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:31.807 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 08:47:31.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:31.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:31.807 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 08:47:31.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:31.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:31.807 | INFO | Uploaded 36GiB to GPU in 1.60266s (22.4627GiB/s) | 20260506 08:47:31.807 | INFO | Downloaded 34GiB to GPU in 1.51994s (22.3693GiB/s) | 20260506 08:47:36.946 | INFO | Analyzing chunk frames = [35840,36352) | 20260506 08:47:36.946 | INFO | MemInfo: { | 20260506 08:47:36.946 | INFO | "gpu" : " 17 GB", | 20260506 08:47:36.946 | INFO | "huge" : "", | 20260506 08:47:36.946 | INFO | "huge_shared" : " 27 GB", | 20260506 08:47:36.946 | INFO | "malloc" : " 15 GB", | 20260506 08:47:36.946 | INFO | "pinned" : " 29 GB" | 20260506 08:47:36.946 | INFO | } | 20260506 08:47:36.946 | INFO | Perf report for the last 5.13933 seconds: | 20260506 08:47:36.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:47:36.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:36.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:36.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:36.946 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 08:47:36.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:36.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:36.946 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 08:47:36.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:36.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:36.946 | INFO | Uploaded 36GiB to GPU in 1.60139s (22.4805GiB/s) | 20260506 08:47:36.946 | INFO | Downloaded 34GiB to GPU in 1.52067s (22.3585GiB/s) | 20260506 08:47:42.085 | INFO | Analyzing chunk frames = [36352,36864) | 20260506 08:47:42.085 | INFO | MemInfo: { | 20260506 08:47:42.085 | INFO | "gpu" : " 17 GB", | 20260506 08:47:42.085 | INFO | "huge" : "", | 20260506 08:47:42.085 | INFO | "huge_shared" : " 27 GB", | 20260506 08:47:42.085 | INFO | "malloc" : " 15 GB", | 20260506 08:47:42.085 | INFO | "pinned" : " 29 GB" | 20260506 08:47:42.085 | INFO | } | 20260506 08:47:42.085 | INFO | Perf report for the last 5.13933 seconds: | 20260506 08:47:42.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:47:42.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:42.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:42.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:42.085 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 08:47:42.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:42.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:42.085 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 08:47:42.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:42.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:42.085 | INFO | Uploaded 36GiB to GPU in 1.60249s (22.4651GiB/s) | 20260506 08:47:42.086 | INFO | Downloaded 34GiB to GPU in 1.52066s (22.3588GiB/s) | 20260506 08:47:42.178 | INFO | Starting a round of Baz IO | 20260506 08:47:45.107 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.929 seconds | 20260506 08:47:45.107 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260506 08:47:45.107 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.1699% / 0% / 0% | 20260506 08:47:45.107 | INFO | Baz IO overall: | 20260506 08:47:45.107 | INFO | BytesWritten : 2571444806 | 20260506 08:47:45.107 | INFO | NumEvents : 0 | 20260506 08:47:45.108 | INFO | Populating RTMetrics report | 20260506 08:47:45.109 | INFO | RTMetrics report populated | 20260506 08:47:45.109 | INFO | Opening tmp RTMetrics json file | 20260506 08:47:45.109 | INFO | Serializing RTMetrics to json | 20260506 08:47:45.111 | INFO | Serialization done, swapping tmp file now... | 20260506 08:47:45.111 | INFO | RTMetrics done | 20260506 08:47:47.224 | INFO | Analyzing chunk frames = [36864,37376) | 20260506 08:47:47.224 | INFO | MemInfo: { | 20260506 08:47:47.224 | INFO | "gpu" : " 17 GB", | 20260506 08:47:47.224 | INFO | "huge" : "", | 20260506 08:47:47.224 | INFO | "huge_shared" : " 27 GB", | 20260506 08:47:47.224 | INFO | "malloc" : " 15 GB", | 20260506 08:47:47.224 | INFO | "pinned" : " 29 GB" | 20260506 08:47:47.224 | INFO | } | 20260506 08:47:47.225 | INFO | Perf report for the last 5.13929 seconds: | 20260506 08:47:47.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:47:47.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:47.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:47.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:47.225 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260506 08:47:47.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:47.225 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 08:47:47.225 | INFO | PRE_HQ : 55.2%, 1.9, 53.5ms, 0 | 20260506 08:47:47.225 | INFO | BAZWRITER : 11.1%, 1.0, 5.7ms, 0 | 20260506 08:47:47.225 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260506 08:47:47.225 | INFO | Uploaded 36GiB to GPU in 1.64065s (21.9425GiB/s) | 20260506 08:47:47.225 | INFO | Downloaded 38GiB to GPU in 1.70455s (22.2933GiB/s) | 20260506 08:47:52.364 | INFO | Analyzing chunk frames = [37376,37888) | 20260506 08:47:52.364 | INFO | MemInfo: { | 20260506 08:47:52.364 | INFO | "gpu" : " 17 GB", | 20260506 08:47:52.364 | INFO | "huge" : "", | 20260506 08:47:52.364 | INFO | "huge_shared" : " 27 GB", | 20260506 08:47:52.364 | INFO | "malloc" : " 15 GB", | 20260506 08:47:52.364 | INFO | "pinned" : " 29 GB" | 20260506 08:47:52.364 | INFO | } | 20260506 08:47:52.364 | INFO | Perf report for the last 5.13938 seconds: | 20260506 08:47:52.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:47:52.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:52.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:52.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:52.364 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260506 08:47:52.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:52.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:52.364 | INFO | PRE_HQ : 32.4%, 1.0, 16.9ms, 0 | 20260506 08:47:52.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:52.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:52.364 | INFO | Uploaded 36GiB to GPU in 1.59956s (22.5061GiB/s) | 20260506 08:47:52.364 | INFO | Downloaded 34GiB to GPU in 1.5172s (22.4096GiB/s) | 20260506 08:47:57.503 | INFO | Analyzing chunk frames = [37888,38400) | 20260506 08:47:57.503 | INFO | MemInfo: { | 20260506 08:47:57.503 | INFO | "gpu" : " 17 GB", | 20260506 08:47:57.503 | INFO | "huge" : "", | 20260506 08:47:57.503 | INFO | "huge_shared" : " 27 GB", | 20260506 08:47:57.503 | INFO | "malloc" : " 15 GB", | 20260506 08:47:57.503 | INFO | "pinned" : " 29 GB" | 20260506 08:47:57.503 | INFO | } | 20260506 08:47:57.503 | INFO | Perf report for the last 5.13926 seconds: | 20260506 08:47:57.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:47:57.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:57.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:57.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:57.503 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260506 08:47:57.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:57.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:57.503 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260506 08:47:57.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:57.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:47:57.503 | INFO | Uploaded 36GiB to GPU in 1.60368s (22.4483GiB/s) | 20260506 08:47:57.503 | INFO | Downloaded 34GiB to GPU in 1.52229s (22.3348GiB/s) | 20260506 08:48:02.642 | INFO | Analyzing chunk frames = [38400,38912) | 20260506 08:48:02.642 | INFO | MemInfo: { | 20260506 08:48:02.642 | INFO | "gpu" : " 17 GB", | 20260506 08:48:02.643 | INFO | "huge" : "", | 20260506 08:48:02.643 | INFO | "huge_shared" : " 27 GB", | 20260506 08:48:02.643 | INFO | "malloc" : " 15 GB", | 20260506 08:48:02.643 | INFO | "pinned" : " 29 GB" | 20260506 08:48:02.643 | INFO | } | 20260506 08:48:02.643 | INFO | Perf report for the last 5.13949 seconds: | 20260506 08:48:02.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:48:02.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:02.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:02.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:02.643 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260506 08:48:02.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:02.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:02.643 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 08:48:02.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:02.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:02.643 | INFO | Uploaded 36GiB to GPU in 1.60365s (22.4489GiB/s) | 20260506 08:48:02.643 | INFO | Downloaded 34GiB to GPU in 1.52161s (22.3447GiB/s) | 20260506 08:48:07.782 | INFO | Analyzing chunk frames = [38912,39424) | 20260506 08:48:07.782 | INFO | MemInfo: { | 20260506 08:48:07.782 | INFO | "gpu" : " 17 GB", | 20260506 08:48:07.782 | INFO | "huge" : "", | 20260506 08:48:07.782 | INFO | "huge_shared" : " 27 GB", | 20260506 08:48:07.782 | INFO | "malloc" : " 15 GB", | 20260506 08:48:07.782 | INFO | "pinned" : " 29 GB" | 20260506 08:48:07.782 | INFO | } | 20260506 08:48:07.782 | INFO | Perf report for the last 5.13931 seconds: | 20260506 08:48:07.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:48:07.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:07.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:07.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:07.782 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 08:48:07.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:07.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:07.782 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 08:48:07.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:07.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:07.782 | INFO | Uploaded 36GiB to GPU in 1.60183s (22.4743GiB/s) | 20260506 08:48:07.782 | INFO | Downloaded 34GiB to GPU in 1.51885s (22.3853GiB/s) | 20260506 08:48:12.921 | INFO | Analyzing chunk frames = [39424,39936) | 20260506 08:48:12.921 | INFO | MemInfo: { | 20260506 08:48:12.921 | INFO | "gpu" : " 17 GB", | 20260506 08:48:12.921 | INFO | "huge" : "", | 20260506 08:48:12.921 | INFO | "huge_shared" : " 27 GB", | 20260506 08:48:12.921 | INFO | "malloc" : " 15 GB", | 20260506 08:48:12.921 | INFO | "pinned" : " 29 GB" | 20260506 08:48:12.921 | INFO | } | 20260506 08:48:12.921 | INFO | Perf report for the last 5.13923 seconds: | 20260506 08:48:12.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:48:12.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:12.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:12.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:12.921 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 08:48:12.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:12.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:12.921 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 08:48:12.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:12.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:12.922 | INFO | Uploaded 36GiB to GPU in 1.60194s (22.4727GiB/s) | 20260506 08:48:12.922 | INFO | Downloaded 34GiB to GPU in 1.51927s (22.3791GiB/s) | 20260506 08:48:18.060 | INFO | Analyzing chunk frames = [39936,40448) | 20260506 08:48:18.060 | INFO | MemInfo: { | 20260506 08:48:18.060 | INFO | "gpu" : " 17 GB", | 20260506 08:48:18.060 | INFO | "huge" : "", | 20260506 08:48:18.060 | INFO | "huge_shared" : " 27 GB", | 20260506 08:48:18.060 | INFO | "malloc" : " 15 GB", | 20260506 08:48:18.060 | INFO | "pinned" : " 29 GB" | 20260506 08:48:18.060 | INFO | } | 20260506 08:48:18.061 | INFO | Perf report for the last 5.13922 seconds: | 20260506 08:48:18.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:48:18.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:18.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:18.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:18.061 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 08:48:18.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:18.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:18.061 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 08:48:18.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:18.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:18.061 | INFO | Uploaded 36GiB to GPU in 1.60275s (22.4613GiB/s) | 20260506 08:48:18.061 | INFO | Downloaded 34GiB to GPU in 1.52164s (22.3443GiB/s) | 20260506 08:48:23.200 | INFO | Analyzing chunk frames = [40448,40960) | 20260506 08:48:23.200 | INFO | MemInfo: { | 20260506 08:48:23.200 | INFO | "gpu" : " 17 GB", | 20260506 08:48:23.200 | INFO | "huge" : "", | 20260506 08:48:23.200 | INFO | "huge_shared" : " 27 GB", | 20260506 08:48:23.200 | INFO | "malloc" : " 15 GB", | 20260506 08:48:23.200 | INFO | "pinned" : " 29 GB" | 20260506 08:48:23.200 | INFO | } | 20260506 08:48:23.200 | INFO | Perf report for the last 5.1395 seconds: | 20260506 08:48:23.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:48:23.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:23.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:23.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:23.200 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 08:48:23.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:23.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:23.200 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 08:48:23.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:23.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:23.200 | INFO | Uploaded 36GiB to GPU in 1.60227s (22.4681GiB/s) | 20260506 08:48:23.200 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 08:48:23.200 | INFO | Total mem usage: 14.7 GiB | 20260506 08:48:23.200 | INFO | Downloaded 34GiB to GPU in 1.51989s (22.37GiB/s) | 20260506 08:48:23.200 | INFO | All Time High Water Marks sum: 14.7 GiB (Savings: 0.0524%) | 20260506 08:48:23.200 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -0.685%) | 20260506 08:48:23.200 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:48:23.200 | INFO | High Water Breakdown: | 20260506 08:48:23.200 | INFO | 54.6% : 7.96 GiB ( 99% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 08:48:23.201 | INFO | 45.2% : 6.59 GiB ( 99.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 08:48:23.201 | INFO | 0.128% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260506 08:48:23.201 | INFO | ------------------------------------------ | 20260506 08:48:23.201 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:48:23.201 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 08:48:23.201 | INFO | Total mem usage: 26.5 GiB | 20260506 08:48:23.201 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 08:48:23.201 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 08:48:23.201 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260506 08:48:23.201 | INFO | High Water Breakdown: | 20260506 08:48:23.201 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:48:23.201 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:48:23.201 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:48:23.201 | INFO | 1.6% : 418 MiB ( 99.5% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:48:23.201 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:48:23.201 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:48:23.201 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 08:48:23.201 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 08:48:23.201 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:48:23.201 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 08:48:23.201 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 08:48:23.201 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 08:48:23.201 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 08:48:23.201 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 08:48:23.201 | INFO | ------------------------------------------ | 20260506 08:48:23.201 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:48:23.201 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 08:48:23.201 | INFO | Total mem usage: 25.6 GiB | 20260506 08:48:23.201 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 08:48:23.201 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 08:48:23.201 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 08:48:23.201 | INFO | High Water Breakdown: | 20260506 08:48:23.201 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 08:48:23.201 | INFO | ------------------------------------------ | 20260506 08:48:23.201 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:48:23.201 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 08:48:23.201 | INFO | Total mem usage: 15.7 GiB | 20260506 08:48:23.201 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 08:48:23.201 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 08:48:23.201 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 08:48:23.201 | INFO | High Water Breakdown: | 20260506 08:48:23.201 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 08:48:23.201 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 08:48:23.201 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 08:48:23.201 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 08:48:23.201 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 08:48:23.201 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 08:48:23.201 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 08:48:23.201 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 08:48:23.201 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 08:48:23.201 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:48:23.201 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 08:48:23.201 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:48:23.201 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 08:48:23.201 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 08:48:23.201 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 08:48:23.201 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 08:48:23.201 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:48:23.201 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 08:48:23.201 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:48:23.201 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:48:23.201 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:48:23.201 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 08:48:23.201 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 08:48:23.201 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:48:23.201 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 08:48:23.201 | INFO | ------------------------------------------ | 20260506 08:48:23.201 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:48:23.201 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 08:48:23.201 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:48:23.201 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:48:23.201 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 08:48:23.201 | INFO | PerfReport: 0.28%, AggregateStats -- 8.61e-02 / 0.00e+00, 6.34e-03 / 0.00e+00 | 20260506 08:48:23.201 | INFO | PerfReport: 10.85%, Baseline -- 3.29e+00 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260506 08:48:23.202 | INFO | PerfReport: 3.00%, Histogram -- 9.11e-01 / 0.00e+00, 5.13e-02 / 0.00e+00 | 20260506 08:48:23.202 | INFO | PerfReport: 16.10%, DME -- 3.91e+01 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 08:48:23.202 | INFO | PerfReport: 56.73%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.18e-02 / 0.00e+00 | 20260506 08:48:23.202 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.29e+00 / 0.00e+00, 1.38e-01 / 0.00e+00 | 20260506 08:48:23.202 | INFO | PerfReport: 3.90%, Metrics -- 1.18e+00 / 0.00e+00, 1.89e-01 / 0.00e+00 | 20260506 08:48:23.202 | INFO | PerfReport: 1.55%, PulseSerializing -- 4.68e-01 / 0.00e+00, 7.10e-02 / 0.00e+00 | 20260506 08:48:23.202 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 08:48:23.202 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:48:23.202 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:48:23.202 | INFO | PerfReport: Full Execution -- 9.11e+01 / -nan, 1.91e+01 / 0.00e+00 | 20260506 08:48:23.202 | INFO | PerfReport: 18.38%, Upload -- 1.67e+01 / 0.00e+00, 3.90e-01 / 0.00e+00 | 20260506 08:48:23.202 | INFO | PerfReport: 17.65%, Download -- 1.61e+01 / 0.00e+00, 7.10e-01 / 0.00e+00 | 20260506 08:48:23.202 | INFO | PerfReport: 33.29%, Compute -- 3.03e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 08:48:23.202 | INFO | PerfReport: 0.55%, LockUpload -- 5.03e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 08:48:23.202 | INFO | PerfReport: 0.00%, LockDownload -- 4.92e-04 / 0.00e+00, 1.71e-04 / 0.00e+00 | 20260506 08:48:23.202 | INFO | PerfReport: 30.10%, LockCompute -- 2.74e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260506 08:48:23.202 | INFO | PerfReport: 0.02%, Misc -- 1.42e-02 / 0.00e+00, 3.79e-02 / 0.00e+00 | 20260506 08:48:23.294 | INFO | Starting a round of Baz IO | 20260506 08:48:26.224 | INFO | Populating RTMetrics report | 20260506 08:48:26.224 | INFO | RTMetrics report populated | 20260506 08:48:26.224 | INFO | Opening tmp RTMetrics json file | 20260506 08:48:26.226 | INFO | Serializing RTMetrics to json | 20260506 08:48:26.228 | INFO | Serialization done, swapping tmp file now... | 20260506 08:48:26.228 | INFO | RTMetrics done | 20260506 08:48:26.234 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.939 seconds | 20260506 08:48:26.234 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260506 08:48:26.234 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.3307% / 0% / 0% | 20260506 08:48:26.234 | INFO | Baz IO overall: | 20260506 08:48:26.234 | INFO | BytesWritten : 2571448358 | 20260506 08:48:26.234 | INFO | NumEvents : 0 | 20260506 08:48:28.339 | INFO | Analyzing chunk frames = [40960,41472) | 20260506 08:48:28.339 | INFO | MemInfo: { | 20260506 08:48:28.339 | INFO | "gpu" : " 17 GB", | 20260506 08:48:28.339 | INFO | "huge" : "", | 20260506 08:48:28.339 | INFO | "huge_shared" : " 27 GB", | 20260506 08:48:28.339 | INFO | "malloc" : " 15 GB", | 20260506 08:48:28.339 | INFO | "pinned" : " 29 GB" | 20260506 08:48:28.339 | INFO | } | 20260506 08:48:28.339 | INFO | Perf report for the last 5.1391 seconds: | 20260506 08:48:28.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:48:28.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:28.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:28.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:28.339 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260506 08:48:28.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:28.339 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 08:48:28.339 | INFO | PRE_HQ : 55.9%, 1.9, 55.0ms, 0 | 20260506 08:48:28.339 | INFO | BAZWRITER : 11.0%, 1.0, 5.6ms, 0 | 20260506 08:48:28.339 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260506 08:48:28.339 | INFO | Uploaded 36GiB to GPU in 1.64361s (21.903GiB/s) | 20260506 08:48:28.339 | INFO | Downloaded 38GiB to GPU in 1.70472s (22.2911GiB/s) | 20260506 08:48:33.478 | INFO | Analyzing chunk frames = [41472,41984) | 20260506 08:48:33.479 | INFO | MemInfo: { | 20260506 08:48:33.479 | INFO | "gpu" : " 17 GB", | 20260506 08:48:33.479 | INFO | "huge" : "", | 20260506 08:48:33.479 | INFO | "huge_shared" : " 27 GB", | 20260506 08:48:33.479 | INFO | "malloc" : " 15 GB", | 20260506 08:48:33.479 | INFO | "pinned" : " 29 GB" | 20260506 08:48:33.479 | INFO | } | 20260506 08:48:33.479 | INFO | Perf report for the last 5.13952 seconds: | 20260506 08:48:33.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:48:33.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:33.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:33.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:33.479 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260506 08:48:33.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:33.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:33.479 | INFO | PRE_HQ : 34.0%, 1.0, 18.0ms, 0 | 20260506 08:48:33.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:33.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:33.479 | INFO | Uploaded 36GiB to GPU in 1.59971s (22.5041GiB/s) | 20260506 08:48:33.479 | INFO | Downloaded 34GiB to GPU in 1.51833s (22.393GiB/s) | 20260506 08:48:38.618 | INFO | Analyzing chunk frames = [41984,42496) | 20260506 08:48:38.618 | INFO | MemInfo: { | 20260506 08:48:38.618 | INFO | "gpu" : " 17 GB", | 20260506 08:48:38.618 | INFO | "huge" : "", | 20260506 08:48:38.618 | INFO | "huge_shared" : " 27 GB", | 20260506 08:48:38.618 | INFO | "malloc" : " 15 GB", | 20260506 08:48:38.618 | INFO | "pinned" : " 29 GB" | 20260506 08:48:38.618 | INFO | } | 20260506 08:48:38.618 | INFO | Perf report for the last 5.13939 seconds: | 20260506 08:48:38.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:48:38.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:38.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:38.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:38.618 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260506 08:48:38.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:38.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:38.618 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 08:48:38.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:38.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:38.618 | INFO | Uploaded 36GiB to GPU in 1.60312s (22.4562GiB/s) | 20260506 08:48:38.618 | INFO | Downloaded 34GiB to GPU in 1.52179s (22.3421GiB/s) | 20260506 08:48:43.757 | INFO | Analyzing chunk frames = [42496,43008) | 20260506 08:48:43.757 | INFO | MemInfo: { | 20260506 08:48:43.757 | INFO | "gpu" : " 17 GB", | 20260506 08:48:43.757 | INFO | "huge" : "", | 20260506 08:48:43.757 | INFO | "huge_shared" : " 27 GB", | 20260506 08:48:43.757 | INFO | "malloc" : " 15 GB", | 20260506 08:48:43.757 | INFO | "pinned" : " 29 GB" | 20260506 08:48:43.757 | INFO | } | 20260506 08:48:43.757 | INFO | Perf report for the last 5.13916 seconds: | 20260506 08:48:43.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:48:43.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:43.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:43.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:43.757 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260506 08:48:43.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:43.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:43.757 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 08:48:43.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:43.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:43.758 | INFO | Uploaded 36GiB to GPU in 1.60341s (22.4522GiB/s) | 20260506 08:48:43.758 | INFO | Downloaded 34GiB to GPU in 1.52096s (22.3544GiB/s) | 20260506 08:48:48.897 | INFO | Analyzing chunk frames = [43008,43520) | 20260506 08:48:48.897 | INFO | MemInfo: { | 20260506 08:48:48.897 | INFO | "gpu" : " 17 GB", | 20260506 08:48:48.897 | INFO | "huge" : "", | 20260506 08:48:48.897 | INFO | "huge_shared" : " 27 GB", | 20260506 08:48:48.897 | INFO | "malloc" : " 15 GB", | 20260506 08:48:48.897 | INFO | "pinned" : " 29 GB" | 20260506 08:48:48.897 | INFO | } | 20260506 08:48:48.897 | INFO | Perf report for the last 5.13949 seconds: | 20260506 08:48:48.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:48:48.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:48.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:48.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:48.897 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 08:48:48.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:48.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:48.897 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 08:48:48.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:48.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:48.897 | INFO | Uploaded 36GiB to GPU in 1.60059s (22.4918GiB/s) | 20260506 08:48:48.897 | INFO | Downloaded 34GiB to GPU in 1.51912s (22.3814GiB/s) | 20260506 08:48:54.036 | INFO | Analyzing chunk frames = [43520,44032) | 20260506 08:48:54.036 | INFO | MemInfo: { | 20260506 08:48:54.036 | INFO | "gpu" : " 17 GB", | 20260506 08:48:54.036 | INFO | "huge" : "", | 20260506 08:48:54.036 | INFO | "huge_shared" : " 27 GB", | 20260506 08:48:54.036 | INFO | "malloc" : " 15 GB", | 20260506 08:48:54.036 | INFO | "pinned" : " 29 GB" | 20260506 08:48:54.036 | INFO | } | 20260506 08:48:54.036 | INFO | Perf report for the last 5.13925 seconds: | 20260506 08:48:54.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:48:54.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:54.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:54.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:54.036 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 08:48:54.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:54.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:54.036 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 08:48:54.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:54.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:54.036 | INFO | Uploaded 36GiB to GPU in 1.60146s (22.4795GiB/s) | 20260506 08:48:54.036 | INFO | Downloaded 34GiB to GPU in 1.51881s (22.3859GiB/s) | 20260506 08:48:59.175 | INFO | Analyzing chunk frames = [44032,44544) | 20260506 08:48:59.175 | INFO | MemInfo: { | 20260506 08:48:59.175 | INFO | "gpu" : " 17 GB", | 20260506 08:48:59.175 | INFO | "huge" : "", | 20260506 08:48:59.175 | INFO | "huge_shared" : " 27 GB", | 20260506 08:48:59.175 | INFO | "malloc" : " 15 GB", | 20260506 08:48:59.175 | INFO | "pinned" : " 29 GB" | 20260506 08:48:59.175 | INFO | } | 20260506 08:48:59.175 | INFO | Perf report for the last 5.13943 seconds: | 20260506 08:48:59.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:48:59.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:59.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:59.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:59.175 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 08:48:59.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:59.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:59.176 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 08:48:59.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:59.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:48:59.176 | INFO | Uploaded 36GiB to GPU in 1.60116s (22.4837GiB/s) | 20260506 08:48:59.176 | INFO | Downloaded 34GiB to GPU in 1.51995s (22.3692GiB/s) | 20260506 08:49:04.315 | INFO | Analyzing chunk frames = [44544,45056) | 20260506 08:49:04.315 | INFO | MemInfo: { | 20260506 08:49:04.315 | INFO | "gpu" : " 17 GB", | 20260506 08:49:04.315 | INFO | "huge" : "", | 20260506 08:49:04.315 | INFO | "huge_shared" : " 27 GB", | 20260506 08:49:04.315 | INFO | "malloc" : " 15 GB", | 20260506 08:49:04.315 | INFO | "pinned" : " 29 GB" | 20260506 08:49:04.315 | INFO | } | 20260506 08:49:04.315 | INFO | Perf report for the last 5.13936 seconds: | 20260506 08:49:04.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:49:04.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:04.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:04.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:04.315 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 08:49:04.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:04.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:04.315 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 08:49:04.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:04.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:04.315 | INFO | Uploaded 36GiB to GPU in 1.60156s (22.478GiB/s) | 20260506 08:49:04.315 | INFO | Downloaded 34GiB to GPU in 1.52087s (22.3556GiB/s) | 20260506 08:49:04.409 | INFO | Starting a round of Baz IO | 20260506 08:49:07.344 | INFO | Populating RTMetrics report | 20260506 08:49:07.344 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.935 seconds | 20260506 08:49:07.344 | INFO | RTMetrics report populated | 20260506 08:49:07.344 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260506 08:49:07.344 | INFO | Opening tmp RTMetrics json file | 20260506 08:49:07.344 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.4871% / 0% / 0% | 20260506 08:49:07.344 | INFO | Baz IO overall: | 20260506 08:49:07.344 | INFO | BytesWritten : 2571451910 | 20260506 08:49:07.344 | INFO | NumEvents : 0 | 20260506 08:49:07.345 | INFO | Serializing RTMetrics to json | 20260506 08:49:07.346 | INFO | Serialization done, swapping tmp file now... | 20260506 08:49:07.347 | INFO | RTMetrics done | 20260506 08:49:09.454 | INFO | Analyzing chunk frames = [45056,45568) | 20260506 08:49:09.454 | INFO | MemInfo: { | 20260506 08:49:09.454 | INFO | "gpu" : " 17 GB", | 20260506 08:49:09.454 | INFO | "huge" : "", | 20260506 08:49:09.454 | INFO | "huge_shared" : " 27 GB", | 20260506 08:49:09.454 | INFO | "malloc" : " 15 GB", | 20260506 08:49:09.454 | INFO | "pinned" : " 29 GB" | 20260506 08:49:09.454 | INFO | } | 20260506 08:49:09.454 | INFO | Perf report for the last 5.13917 seconds: | 20260506 08:49:09.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:49:09.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:09.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:09.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:09.454 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260506 08:49:09.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:09.454 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 08:49:09.454 | INFO | PRE_HQ : 55.8%, 1.9, 54.9ms, 0 | 20260506 08:49:09.454 | INFO | BAZWRITER : 11.2%, 1.0, 5.7ms, 0 | 20260506 08:49:09.454 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260506 08:49:09.455 | INFO | Uploaded 36GiB to GPU in 1.63996s (21.9518GiB/s) | 20260506 08:49:09.455 | INFO | Downloaded 38GiB to GPU in 1.70183s (22.3289GiB/s) | 20260506 08:49:14.593 | INFO | Analyzing chunk frames = [45568,46080) | 20260506 08:49:14.593 | INFO | MemInfo: { | 20260506 08:49:14.593 | INFO | "gpu" : " 17 GB", | 20260506 08:49:14.593 | INFO | "huge" : "", | 20260506 08:49:14.593 | INFO | "huge_shared" : " 27 GB", | 20260506 08:49:14.593 | INFO | "malloc" : " 15 GB", | 20260506 08:49:14.593 | INFO | "pinned" : " 29 GB" | 20260506 08:49:14.593 | INFO | } | 20260506 08:49:14.593 | INFO | Perf report for the last 5.13935 seconds: | 20260506 08:49:14.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:49:14.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:14.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:14.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:14.593 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260506 08:49:14.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:14.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:14.593 | INFO | PRE_HQ : 31.9%, 1.0, 16.9ms, 0 | 20260506 08:49:14.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:14.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:14.594 | INFO | Uploaded 36GiB to GPU in 1.59928s (22.5102GiB/s) | 20260506 08:49:14.594 | INFO | Downloaded 34GiB to GPU in 1.517s (22.4126GiB/s) | 20260506 08:49:19.733 | INFO | Analyzing chunk frames = [46080,46592) | 20260506 08:49:19.733 | INFO | MemInfo: { | 20260506 08:49:19.733 | INFO | "gpu" : " 17 GB", | 20260506 08:49:19.733 | INFO | "huge" : "", | 20260506 08:49:19.733 | INFO | "huge_shared" : " 27 GB", | 20260506 08:49:19.733 | INFO | "malloc" : " 15 GB", | 20260506 08:49:19.733 | INFO | "pinned" : " 29 GB" | 20260506 08:49:19.733 | INFO | } | 20260506 08:49:19.733 | INFO | Perf report for the last 5.13936 seconds: | 20260506 08:49:19.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:49:19.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:19.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:19.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:19.733 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260506 08:49:19.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:19.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:19.733 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 08:49:19.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:19.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:19.733 | INFO | Uploaded 36GiB to GPU in 1.60317s (22.4555GiB/s) | 20260506 08:49:19.733 | INFO | Downloaded 34GiB to GPU in 1.52191s (22.3403GiB/s) | 20260506 08:49:24.872 | INFO | Analyzing chunk frames = [46592,47104) | 20260506 08:49:24.872 | INFO | MemInfo: { | 20260506 08:49:24.872 | INFO | "gpu" : " 17 GB", | 20260506 08:49:24.872 | INFO | "huge" : "", | 20260506 08:49:24.872 | INFO | "huge_shared" : " 27 GB", | 20260506 08:49:24.872 | INFO | "malloc" : " 15 GB", | 20260506 08:49:24.872 | INFO | "pinned" : " 29 GB" | 20260506 08:49:24.872 | INFO | } | 20260506 08:49:24.872 | INFO | Perf report for the last 5.13929 seconds: | 20260506 08:49:24.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:49:24.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:24.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:24.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:24.872 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260506 08:49:24.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:24.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:24.872 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 08:49:24.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:24.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:24.872 | INFO | Uploaded 36GiB to GPU in 1.60359s (22.4497GiB/s) | 20260506 08:49:24.872 | INFO | Downloaded 34GiB to GPU in 1.52096s (22.3543GiB/s) | 20260506 08:49:30.011 | INFO | Analyzing chunk frames = [47104,47616) | 20260506 08:49:30.011 | INFO | MemInfo: { | 20260506 08:49:30.011 | INFO | "gpu" : " 17 GB", | 20260506 08:49:30.011 | INFO | "huge" : "", | 20260506 08:49:30.011 | INFO | "huge_shared" : " 27 GB", | 20260506 08:49:30.011 | INFO | "malloc" : " 15 GB", | 20260506 08:49:30.011 | INFO | "pinned" : " 29 GB" | 20260506 08:49:30.011 | INFO | } | 20260506 08:49:30.012 | INFO | Perf report for the last 5.13944 seconds: | 20260506 08:49:30.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:49:30.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:30.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:30.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:30.012 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 08:49:30.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:30.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:30.012 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 08:49:30.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:30.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:30.012 | INFO | Uploaded 36GiB to GPU in 1.60118s (22.4834GiB/s) | 20260506 08:49:30.012 | INFO | Downloaded 34GiB to GPU in 1.518s (22.3979GiB/s) | 20260506 08:49:35.150 | INFO | Analyzing chunk frames = [47616,48128) | 20260506 08:49:35.150 | INFO | MemInfo: { | 20260506 08:49:35.150 | INFO | "gpu" : " 17 GB", | 20260506 08:49:35.150 | INFO | "huge" : "", | 20260506 08:49:35.150 | INFO | "huge_shared" : " 27 GB", | 20260506 08:49:35.150 | INFO | "malloc" : " 15 GB", | 20260506 08:49:35.151 | INFO | "pinned" : " 29 GB" | 20260506 08:49:35.151 | INFO | } | 20260506 08:49:35.151 | INFO | Perf report for the last 5.13911 seconds: | 20260506 08:49:35.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:49:35.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:35.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:35.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:35.151 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 08:49:35.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:35.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:35.151 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 08:49:35.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:35.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:35.151 | INFO | Uploaded 36GiB to GPU in 1.60183s (22.4743GiB/s) | 20260506 08:49:35.151 | INFO | Downloaded 34GiB to GPU in 1.51876s (22.3867GiB/s) | 20260506 08:49:40.290 | INFO | Analyzing chunk frames = [48128,48640) | 20260506 08:49:40.290 | INFO | MemInfo: { | 20260506 08:49:40.290 | INFO | "gpu" : " 17 GB", | 20260506 08:49:40.290 | INFO | "huge" : "", | 20260506 08:49:40.290 | INFO | "huge_shared" : " 27 GB", | 20260506 08:49:40.290 | INFO | "malloc" : " 15 GB", | 20260506 08:49:40.290 | INFO | "pinned" : " 29 GB" | 20260506 08:49:40.290 | INFO | } | 20260506 08:49:40.290 | INFO | Perf report for the last 5.13948 seconds: | 20260506 08:49:40.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:49:40.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:40.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:40.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:40.290 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 08:49:40.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:40.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:40.290 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 08:49:40.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:40.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:40.291 | INFO | Uploaded 36GiB to GPU in 1.60143s (22.4799GiB/s) | 20260506 08:49:40.291 | INFO | Downloaded 34GiB to GPU in 1.51976s (22.3719GiB/s) | 20260506 08:49:45.429 | INFO | Analyzing chunk frames = [48640,49152) | 20260506 08:49:45.429 | INFO | MemInfo: { | 20260506 08:49:45.429 | INFO | "gpu" : " 17 GB", | 20260506 08:49:45.429 | INFO | "huge" : "", | 20260506 08:49:45.429 | INFO | "huge_shared" : " 27 GB", | 20260506 08:49:45.429 | INFO | "malloc" : " 15 GB", | 20260506 08:49:45.429 | INFO | "pinned" : " 29 GB" | 20260506 08:49:45.429 | INFO | } | 20260506 08:49:45.429 | INFO | Perf report for the last 5.13921 seconds: | 20260506 08:49:45.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:49:45.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:45.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:45.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:45.429 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 08:49:45.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:45.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:45.429 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 08:49:45.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:45.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:45.430 | INFO | Uploaded 36GiB to GPU in 1.60155s (22.4782GiB/s) | 20260506 08:49:45.430 | INFO | Downloaded 34GiB to GPU in 1.5201s (22.367GiB/s) | 20260506 08:49:45.430 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 08:49:45.430 | INFO | Total mem usage: 14.7 GiB | 20260506 08:49:45.430 | INFO | All Time High Water Marks sum: 14.7 GiB (Savings: 0.0524%) | 20260506 08:49:45.430 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -0.685%) | 20260506 08:49:45.430 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:49:45.430 | INFO | High Water Breakdown: | 20260506 08:49:45.430 | INFO | 54.6% : 7.96 GiB ( 99% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 08:49:45.430 | INFO | 45.2% : 6.59 GiB ( 99.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 08:49:45.430 | INFO | 0.128% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260506 08:49:45.430 | INFO | ------------------------------------------ | 20260506 08:49:45.430 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:49:45.430 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 08:49:45.430 | INFO | Total mem usage: 26.5 GiB | 20260506 08:49:45.430 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 08:49:45.430 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 08:49:45.430 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260506 08:49:45.430 | INFO | High Water Breakdown: | 20260506 08:49:45.430 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:49:45.430 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:49:45.430 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:49:45.430 | INFO | 1.6% : 418 MiB ( 99.5% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:49:45.430 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:49:45.430 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:49:45.430 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 08:49:45.430 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 08:49:45.430 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:49:45.430 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 08:49:45.430 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 08:49:45.430 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 08:49:45.430 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 08:49:45.430 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 08:49:45.430 | INFO | ------------------------------------------ | 20260506 08:49:45.430 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:49:45.430 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 08:49:45.430 | INFO | Total mem usage: 25.6 GiB | 20260506 08:49:45.430 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 08:49:45.430 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 08:49:45.430 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 08:49:45.430 | INFO | High Water Breakdown: | 20260506 08:49:45.430 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 08:49:45.430 | INFO | ------------------------------------------ | 20260506 08:49:45.430 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:49:45.430 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 08:49:45.430 | INFO | Total mem usage: 15.7 GiB | 20260506 08:49:45.430 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 08:49:45.430 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 08:49:45.430 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 08:49:45.430 | INFO | High Water Breakdown: | 20260506 08:49:45.430 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 08:49:45.430 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 08:49:45.430 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 08:49:45.430 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 08:49:45.430 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 08:49:45.431 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 08:49:45.431 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 08:49:45.431 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 08:49:45.431 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 08:49:45.431 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:49:45.431 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 08:49:45.431 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:49:45.431 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 08:49:45.431 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 08:49:45.431 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 08:49:45.431 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 08:49:45.431 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:49:45.431 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 08:49:45.431 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:49:45.431 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:49:45.431 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:49:45.431 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 08:49:45.431 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 08:49:45.431 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:49:45.431 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 08:49:45.431 | INFO | ------------------------------------------ | 20260506 08:49:45.431 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:49:45.431 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 08:49:45.431 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:49:45.431 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:49:45.431 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 08:49:45.431 | INFO | PerfReport: 0.28%, AggregateStats -- 8.61e-02 / 0.00e+00, 4.83e-03 / 0.00e+00 | 20260506 08:49:45.431 | INFO | PerfReport: 10.84%, Baseline -- 3.29e+00 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260506 08:49:45.431 | INFO | PerfReport: 3.01%, Histogram -- 9.13e-01 / 0.00e+00, 5.14e-02 / 0.00e+00 | 20260506 08:49:45.431 | INFO | PerfReport: 16.09%, DME -- 3.91e+01 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260506 08:49:45.431 | INFO | PerfReport: 56.68%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.20e-02 / 0.00e+00 | 20260506 08:49:45.431 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.31e+00 / 0.00e+00, 1.40e-01 / 0.00e+00 | 20260506 08:49:45.431 | INFO | PerfReport: 3.94%, Metrics -- 1.20e+00 / 0.00e+00, 1.89e-01 / 0.00e+00 | 20260506 08:49:45.431 | INFO | PerfReport: 1.56%, PulseSerializing -- 4.72e-01 / 0.00e+00, 7.17e-02 / 0.00e+00 | 20260506 08:49:45.431 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 08:49:45.431 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:49:45.431 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:49:45.431 | INFO | PerfReport: Full Execution -- 9.12e+01 / -nan, 1.92e+01 / 0.00e+00 | 20260506 08:49:45.431 | INFO | PerfReport: 18.35%, Upload -- 1.67e+01 / 0.00e+00, 3.84e-01 / 0.00e+00 | 20260506 08:49:45.431 | INFO | PerfReport: 17.62%, Download -- 1.61e+01 / 0.00e+00, 6.94e-01 / 0.00e+00 | 20260506 08:49:45.431 | INFO | PerfReport: 33.29%, Compute -- 3.04e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 08:49:45.431 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 08:49:45.431 | INFO | PerfReport: 0.00%, LockDownload -- 4.75e-04 / 0.00e+00, 1.24e-04 / 0.00e+00 | 20260506 08:49:45.431 | INFO | PerfReport: 30.16%, LockCompute -- 2.75e+01 / 0.00e+00, 1.73e+01 / 0.00e+00 | 20260506 08:49:45.431 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 2.58e-02 / 0.00e+00 | 20260506 08:49:45.524 | INFO | Starting a round of Baz IO | 20260506 08:49:48.458 | INFO | Finished a round of Baz IO: Wrote 3.469 KiB in 2.934 seconds | 20260506 08:49:48.458 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 100.00% / 0.00% / 0.00% | 20260506 08:49:48.458 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.6494% / 0% / 0% | 20260506 08:49:48.458 | INFO | Baz IO overall: | 20260506 08:49:48.458 | INFO | BytesWritten : 2571455462 | 20260506 08:49:48.458 | INFO | NumEvents : 0 | 20260506 08:49:48.458 | INFO | Populating RTMetrics report | 20260506 08:49:48.459 | INFO | RTMetrics report populated | 20260506 08:49:48.459 | INFO | Opening tmp RTMetrics json file | 20260506 08:49:48.460 | INFO | Serializing RTMetrics to json | 20260506 08:49:48.461 | INFO | Serialization done, swapping tmp file now... | 20260506 08:49:48.461 | INFO | RTMetrics done | 20260506 08:49:50.569 | INFO | Analyzing chunk frames = [49152,49664) | 20260506 08:49:50.569 | INFO | MemInfo: { | 20260506 08:49:50.569 | INFO | "gpu" : " 17 GB", | 20260506 08:49:50.569 | INFO | "huge" : "", | 20260506 08:49:50.569 | INFO | "huge_shared" : " 27 GB", | 20260506 08:49:50.569 | INFO | "malloc" : " 15 GB", | 20260506 08:49:50.569 | INFO | "pinned" : " 29 GB" | 20260506 08:49:50.569 | INFO | } | 20260506 08:49:50.569 | INFO | Perf report for the last 5.1397 seconds: | 20260506 08:49:50.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:49:50.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:50.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:50.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:50.569 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260506 08:49:50.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:50.569 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 08:49:50.569 | INFO | PRE_HQ : 55.5%, 1.9, 54.9ms, 0 | 20260506 08:49:50.569 | INFO | BAZWRITER : 10.7%, 1.0, 5.5ms, 0 | 20260506 08:49:50.569 | INFO | BAZ_SAMPLING : 0.1%, 1.0, 0.0ms, 0 | 20260506 08:49:50.569 | INFO | Uploaded 36GiB to GPU in 1.64748s (21.8516GiB/s) | 20260506 08:49:50.569 | INFO | Downloaded 38GiB to GPU in 1.70816s (22.2462GiB/s) | 20260506 08:49:55.708 | INFO | Analyzing chunk frames = [49664,50176) | 20260506 08:49:55.708 | INFO | MemInfo: { | 20260506 08:49:55.708 | INFO | "gpu" : " 17 GB", | 20260506 08:49:55.708 | INFO | "huge" : "", | 20260506 08:49:55.708 | INFO | "huge_shared" : " 27 GB", | 20260506 08:49:55.708 | INFO | "malloc" : " 15 GB", | 20260506 08:49:55.708 | INFO | "pinned" : " 29 GB" | 20260506 08:49:55.708 | INFO | } | 20260506 08:49:55.709 | INFO | Perf report for the last 5.13948 seconds: | 20260506 08:49:55.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:49:55.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:55.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:55.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:55.709 | INFO | ANALYSIS : 57.9%, 3.0, 90.2ms, 0 | 20260506 08:49:55.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:55.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:55.709 | INFO | PRE_HQ : 33.1%, 1.0, 17.5ms, 0 | 20260506 08:49:55.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:55.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:49:55.709 | INFO | Uploaded 36GiB to GPU in 1.60008s (22.4988GiB/s) | 20260506 08:49:55.709 | INFO | Downloaded 34GiB to GPU in 1.51679s (22.4157GiB/s) | 20260506 08:50:00.847 | INFO | Analyzing chunk frames = [50176,50688) | 20260506 08:50:00.847 | INFO | MemInfo: { | 20260506 08:50:00.847 | INFO | "gpu" : " 17 GB", | 20260506 08:50:00.847 | INFO | "huge" : "", | 20260506 08:50:00.847 | INFO | "huge_shared" : " 27 GB", | 20260506 08:50:00.847 | INFO | "malloc" : " 15 GB", | 20260506 08:50:00.847 | INFO | "pinned" : " 29 GB" | 20260506 08:50:00.847 | INFO | } | 20260506 08:50:00.847 | INFO | Perf report for the last 5.13876 seconds: | 20260506 08:50:00.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:50:00.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:00.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:00.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:00.847 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260506 08:50:00.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:00.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:00.847 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260506 08:50:00.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:00.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:00.848 | INFO | Uploaded 36GiB to GPU in 1.60328s (22.454GiB/s) | 20260506 08:50:00.848 | INFO | Downloaded 34GiB to GPU in 1.52101s (22.3536GiB/s) | 20260506 08:50:05.987 | INFO | Analyzing chunk frames = [50688,51200) | 20260506 08:50:05.987 | INFO | MemInfo: { | 20260506 08:50:05.987 | INFO | "gpu" : " 17 GB", | 20260506 08:50:05.987 | INFO | "huge" : "", | 20260506 08:50:05.987 | INFO | "huge_shared" : " 27 GB", | 20260506 08:50:05.987 | INFO | "malloc" : " 15 GB", | 20260506 08:50:05.987 | INFO | "pinned" : " 29 GB" | 20260506 08:50:05.987 | INFO | } | 20260506 08:50:05.987 | INFO | Perf report for the last 5.13971 seconds: | 20260506 08:50:05.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:50:05.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:05.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:05.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:05.987 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260506 08:50:05.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:05.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:05.987 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 08:50:05.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:05.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:05.987 | INFO | Uploaded 36GiB to GPU in 1.60312s (22.4561GiB/s) | 20260506 08:50:05.987 | INFO | Downloaded 34GiB to GPU in 1.52026s (22.3646GiB/s) | 20260506 08:50:11.126 | INFO | Analyzing chunk frames = [51200,51712) | 20260506 08:50:11.126 | INFO | MemInfo: { | 20260506 08:50:11.126 | INFO | "gpu" : " 17 GB", | 20260506 08:50:11.126 | INFO | "huge" : "", | 20260506 08:50:11.126 | INFO | "huge_shared" : " 27 GB", | 20260506 08:50:11.126 | INFO | "malloc" : " 15 GB", | 20260506 08:50:11.126 | INFO | "pinned" : " 29 GB" | 20260506 08:50:11.126 | INFO | } | 20260506 08:50:11.126 | INFO | Perf report for the last 5.13918 seconds: | 20260506 08:50:11.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:50:11.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:11.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:11.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:11.126 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 08:50:11.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:11.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:11.126 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 08:50:11.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:11.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:11.126 | INFO | Uploaded 36GiB to GPU in 1.60088s (22.4876GiB/s) | 20260506 08:50:11.127 | INFO | Downloaded 34GiB to GPU in 1.5177s (22.4023GiB/s) | 20260506 08:50:16.265 | INFO | Analyzing chunk frames = [51712,52224) | 20260506 08:50:16.265 | INFO | MemInfo: { | 20260506 08:50:16.265 | INFO | "gpu" : " 17 GB", | 20260506 08:50:16.265 | INFO | "huge" : "", | 20260506 08:50:16.265 | INFO | "huge_shared" : " 27 GB", | 20260506 08:50:16.265 | INFO | "malloc" : " 15 GB", | 20260506 08:50:16.265 | INFO | "pinned" : " 29 GB" | 20260506 08:50:16.265 | INFO | } | 20260506 08:50:16.265 | INFO | Perf report for the last 5.13915 seconds: | 20260506 08:50:16.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:50:16.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:16.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:16.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:16.265 | INFO | ANALYSIS : 57.2%, 3.0, 89.1ms, 0 | 20260506 08:50:16.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:16.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:16.266 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260506 08:50:16.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:16.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:16.266 | INFO | Uploaded 36GiB to GPU in 1.60197s (22.4723GiB/s) | 20260506 08:50:16.266 | INFO | Downloaded 34GiB to GPU in 1.51881s (22.3859GiB/s) | 20260506 08:50:21.405 | INFO | Analyzing chunk frames = [52224,52736) | 20260506 08:50:21.405 | INFO | MemInfo: { | 20260506 08:50:21.405 | INFO | "gpu" : " 17 GB", | 20260506 08:50:21.405 | INFO | "huge" : "", | 20260506 08:50:21.405 | INFO | "huge_shared" : " 27 GB", | 20260506 08:50:21.405 | INFO | "malloc" : " 15 GB", | 20260506 08:50:21.405 | INFO | "pinned" : " 29 GB" | 20260506 08:50:21.405 | INFO | } | 20260506 08:50:21.405 | INFO | Perf report for the last 5.1394 seconds: | 20260506 08:50:21.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:50:21.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:21.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:21.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:21.405 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 08:50:21.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:21.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:21.405 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 08:50:21.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:21.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:21.405 | INFO | Uploaded 36GiB to GPU in 1.60121s (22.4831GiB/s) | 20260506 08:50:21.405 | INFO | Downloaded 34GiB to GPU in 1.51928s (22.379GiB/s) | 20260506 08:50:26.544 | INFO | Analyzing chunk frames = [52736,53248) | 20260506 08:50:26.544 | INFO | MemInfo: { | 20260506 08:50:26.544 | INFO | "gpu" : " 17 GB", | 20260506 08:50:26.544 | INFO | "huge" : "", | 20260506 08:50:26.544 | INFO | "huge_shared" : " 27 GB", | 20260506 08:50:26.544 | INFO | "malloc" : " 15 GB", | 20260506 08:50:26.544 | INFO | "pinned" : " 29 GB" | 20260506 08:50:26.544 | INFO | } | 20260506 08:50:26.544 | INFO | Perf report for the last 5.13942 seconds: | 20260506 08:50:26.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:50:26.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:26.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:26.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:26.544 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260506 08:50:26.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:26.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:26.544 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 08:50:26.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:26.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:26.545 | INFO | Uploaded 36GiB to GPU in 1.60203s (22.4714GiB/s) | 20260506 08:50:26.545 | INFO | Downloaded 34GiB to GPU in 1.51954s (22.3753GiB/s) | 20260506 08:50:26.643 | INFO | Starting a round of Baz IO | 20260506 08:50:29.575 | INFO | Populating RTMetrics report | 20260506 08:50:29.575 | INFO | RTMetrics report populated | 20260506 08:50:29.575 | INFO | Opening tmp RTMetrics json file | 20260506 08:50:29.576 | INFO | Serializing RTMetrics to json | 20260506 08:50:29.577 | INFO | Serialization done, swapping tmp file now... | 20260506 08:50:29.578 | INFO | RTMetrics done | 20260506 08:50:29.840 | INFO | Finished a round of Baz IO: Wrote 1.182 GiB in 3.197 seconds | 20260506 08:50:29.840 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 79.78% / 20.22% / 0.00% | 20260506 08:50:29.840 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.7081% / 0.0946442% / 0% | 20260506 08:50:29.840 | INFO | Baz IO overall: | 20260506 08:50:29.840 | INFO | BytesWritten : 3840862423 | 20260506 08:50:29.840 | INFO | NumEvents : 476545777 | 20260506 08:50:31.683 | INFO | Analyzing chunk frames = [53248,53760) | 20260506 08:50:31.683 | INFO | MemInfo: { | 20260506 08:50:31.683 | INFO | "gpu" : " 17 GB", | 20260506 08:50:31.683 | INFO | "huge" : "", | 20260506 08:50:31.683 | INFO | "huge_shared" : " 27 GB", | 20260506 08:50:31.683 | INFO | "malloc" : " 15 GB", | 20260506 08:50:31.683 | INFO | "pinned" : " 29 GB" | 20260506 08:50:31.683 | INFO | } | 20260506 08:50:31.683 | INFO | Perf report for the last 5.13922 seconds: | 20260506 08:50:31.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:50:31.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:31.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:31.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:31.684 | INFO | ANALYSIS : 58.1%, 2.9, 88.9ms, 0 | 20260506 08:50:31.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:31.684 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 08:50:31.684 | INFO | PRE_HQ : 57.7%, 2.1, 62.1ms, 0 | 20260506 08:50:31.684 | INFO | BAZWRITER : 61.8%, 4.2, 133.9ms, 0 | 20260506 08:50:31.684 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 08:50:31.684 | INFO | Uploaded 36GiB to GPU in 1.65062s (21.81GiB/s) | 20260506 08:50:31.684 | INFO | Downloaded 38GiB to GPU in 1.71324s (22.1802GiB/s) | 20260506 08:50:36.823 | INFO | Analyzing chunk frames = [53760,54272) | 20260506 08:50:36.823 | INFO | MemInfo: { | 20260506 08:50:36.823 | INFO | "gpu" : " 17 GB", | 20260506 08:50:36.823 | INFO | "huge" : "", | 20260506 08:50:36.823 | INFO | "huge_shared" : " 27 GB", | 20260506 08:50:36.823 | INFO | "malloc" : " 15 GB", | 20260506 08:50:36.823 | INFO | "pinned" : " 29 GB" | 20260506 08:50:36.823 | INFO | } | 20260506 08:50:36.823 | INFO | Perf report for the last 5.1394 seconds: | 20260506 08:50:36.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:50:36.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:36.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:36.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:36.823 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260506 08:50:36.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:36.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:36.823 | INFO | PRE_HQ : 32.1%, 1.0, 17.2ms, 0 | 20260506 08:50:36.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:36.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:36.823 | INFO | Uploaded 36GiB to GPU in 1.59994s (22.5009GiB/s) | 20260506 08:50:36.823 | INFO | Downloaded 34GiB to GPU in 1.51644s (22.4209GiB/s) | 20260506 08:50:41.962 | INFO | Analyzing chunk frames = [54272,54784) | 20260506 08:50:41.962 | INFO | MemInfo: { | 20260506 08:50:41.962 | INFO | "gpu" : " 17 GB", | 20260506 08:50:41.962 | INFO | "huge" : "", | 20260506 08:50:41.962 | INFO | "huge_shared" : " 27 GB", | 20260506 08:50:41.962 | INFO | "malloc" : " 15 GB", | 20260506 08:50:41.962 | INFO | "pinned" : " 29 GB" | 20260506 08:50:41.962 | INFO | } | 20260506 08:50:41.962 | INFO | Perf report for the last 5.13936 seconds: | 20260506 08:50:41.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:50:41.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:41.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:41.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:41.962 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260506 08:50:41.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:41.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:41.962 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 08:50:41.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:41.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:41.963 | INFO | Uploaded 36GiB to GPU in 1.60367s (22.4485GiB/s) | 20260506 08:50:41.963 | INFO | Downloaded 34GiB to GPU in 1.52176s (22.3425GiB/s) | 20260506 08:50:47.101 | INFO | Analyzing chunk frames = [54784,55296) | 20260506 08:50:47.101 | INFO | MemInfo: { | 20260506 08:50:47.101 | INFO | "gpu" : " 17 GB", | 20260506 08:50:47.101 | INFO | "huge" : "", | 20260506 08:50:47.101 | INFO | "huge_shared" : " 27 GB", | 20260506 08:50:47.101 | INFO | "malloc" : " 15 GB", | 20260506 08:50:47.101 | INFO | "pinned" : " 29 GB" | 20260506 08:50:47.101 | INFO | } | 20260506 08:50:47.101 | INFO | Perf report for the last 5.13922 seconds: | 20260506 08:50:47.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:50:47.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:47.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:47.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:47.102 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260506 08:50:47.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:47.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:47.102 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260506 08:50:47.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:47.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:47.102 | INFO | Uploaded 36GiB to GPU in 1.60293s (22.4589GiB/s) | 20260506 08:50:47.102 | INFO | Downloaded 34GiB to GPU in 1.52002s (22.3682GiB/s) | 20260506 08:50:52.241 | INFO | Analyzing chunk frames = [55296,55808) | 20260506 08:50:52.241 | INFO | MemInfo: { | 20260506 08:50:52.241 | INFO | "gpu" : " 17 GB", | 20260506 08:50:52.241 | INFO | "huge" : "", | 20260506 08:50:52.241 | INFO | "huge_shared" : " 27 GB", | 20260506 08:50:52.241 | INFO | "malloc" : " 15 GB", | 20260506 08:50:52.241 | INFO | "pinned" : " 29 GB" | 20260506 08:50:52.241 | INFO | } | 20260506 08:50:52.241 | INFO | Perf report for the last 5.13949 seconds: | 20260506 08:50:52.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:50:52.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:52.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:52.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:52.241 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 08:50:52.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:52.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:52.241 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 08:50:52.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:52.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:52.241 | INFO | Uploaded 36GiB to GPU in 1.60144s (22.4798GiB/s) | 20260506 08:50:52.241 | INFO | Downloaded 34GiB to GPU in 1.51775s (22.4015GiB/s) | 20260506 08:50:57.380 | INFO | Analyzing chunk frames = [55808,56320) | 20260506 08:50:57.380 | INFO | MemInfo: { | 20260506 08:50:57.380 | INFO | "gpu" : " 17 GB", | 20260506 08:50:57.380 | INFO | "huge" : "", | 20260506 08:50:57.380 | INFO | "huge_shared" : " 27 GB", | 20260506 08:50:57.380 | INFO | "malloc" : " 15 GB", | 20260506 08:50:57.380 | INFO | "pinned" : " 29 GB" | 20260506 08:50:57.380 | INFO | } | 20260506 08:50:57.380 | INFO | Perf report for the last 5.13923 seconds: | 20260506 08:50:57.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:50:57.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:57.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:57.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:57.380 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 08:50:57.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:57.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:57.380 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 08:50:57.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:57.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:50:57.381 | INFO | Uploaded 36GiB to GPU in 1.60198s (22.4722GiB/s) | 20260506 08:50:57.381 | INFO | Downloaded 34GiB to GPU in 1.51849s (22.3906GiB/s) | 20260506 08:51:02.519 | INFO | Analyzing chunk frames = [56320,56832) | 20260506 08:51:02.519 | INFO | MemInfo: { | 20260506 08:51:02.519 | INFO | "gpu" : " 17 GB", | 20260506 08:51:02.519 | INFO | "huge" : "", | 20260506 08:51:02.519 | INFO | "huge_shared" : " 27 GB", | 20260506 08:51:02.519 | INFO | "malloc" : " 15 GB", | 20260506 08:51:02.519 | INFO | "pinned" : " 29 GB" | 20260506 08:51:02.519 | INFO | } | 20260506 08:51:02.519 | INFO | Perf report for the last 5.13923 seconds: | 20260506 08:51:02.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:51:02.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:02.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:02.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:02.520 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 08:51:02.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:02.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:02.520 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 08:51:02.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:02.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:02.520 | INFO | Uploaded 36GiB to GPU in 1.60169s (22.4762GiB/s) | 20260506 08:51:02.520 | INFO | Downloaded 34GiB to GPU in 1.51994s (22.3693GiB/s) | 20260506 08:51:07.659 | INFO | Analyzing chunk frames = [56832,57344) | 20260506 08:51:07.659 | INFO | MemInfo: { | 20260506 08:51:07.659 | INFO | "gpu" : " 17 GB", | 20260506 08:51:07.659 | INFO | "huge" : "", | 20260506 08:51:07.659 | INFO | "huge_shared" : " 27 GB", | 20260506 08:51:07.659 | INFO | "malloc" : " 15 GB", | 20260506 08:51:07.659 | INFO | "pinned" : " 29 GB" | 20260506 08:51:07.659 | INFO | } | 20260506 08:51:07.659 | INFO | Perf report for the last 5.1393 seconds: | 20260506 08:51:07.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:51:07.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:07.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:07.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:07.659 | INFO | ANALYSIS : 57.5%, 3.0, 89.8ms, 0 | 20260506 08:51:07.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:07.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:07.659 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 08:51:07.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:07.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:07.659 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 08:51:07.659 | INFO | Total mem usage: 15.5 GiB | 20260506 08:51:07.659 | INFO | All Time High Water Marks sum: 15.6 GiB (Savings: 0.772%) | 20260506 08:51:07.659 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: 0.772%) | 20260506 08:51:07.659 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:51:07.659 | INFO | High Water Breakdown: | 20260506 08:51:07.659 | INFO | 57.1% : 8.93 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 08:51:07.659 | INFO | Uploaded 36GiB to GPU in 1.60286s (22.4598GiB/s) | 20260506 08:51:07.659 | INFO | 42.8% : 6.69 GiB ( 100% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 08:51:07.659 | INFO | 0.119% : 19.1 MiB ( 100% Of Peak) : SampledBazData | 20260506 08:51:07.659 | INFO | ------------------------------------------ | 20260506 08:51:07.659 | INFO | Downloaded 34GiB to GPU in 1.52051s (22.3609GiB/s) | 20260506 08:51:07.659 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:51:07.659 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 08:51:07.659 | INFO | Total mem usage: 26.5 GiB | 20260506 08:51:07.659 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 08:51:07.659 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 08:51:07.659 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 08:51:07.659 | INFO | High Water Breakdown: | 20260506 08:51:07.659 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:51:07.659 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:51:07.659 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:51:07.659 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:51:07.659 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:51:07.659 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:51:07.659 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 08:51:07.659 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 08:51:07.659 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:51:07.659 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 08:51:07.659 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 08:51:07.659 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 08:51:07.659 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 08:51:07.659 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 08:51:07.660 | INFO | ------------------------------------------ | 20260506 08:51:07.660 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:51:07.660 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 08:51:07.660 | INFO | Total mem usage: 25.6 GiB | 20260506 08:51:07.660 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 08:51:07.660 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 08:51:07.660 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 08:51:07.660 | INFO | High Water Breakdown: | 20260506 08:51:07.660 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 08:51:07.660 | INFO | ------------------------------------------ | 20260506 08:51:07.660 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:51:07.660 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 08:51:07.660 | INFO | Total mem usage: 15.7 GiB | 20260506 08:51:07.660 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 08:51:07.660 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 08:51:07.660 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 08:51:07.660 | INFO | High Water Breakdown: | 20260506 08:51:07.660 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 08:51:07.660 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 08:51:07.660 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 08:51:07.660 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 08:51:07.660 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 08:51:07.660 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 08:51:07.660 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 08:51:07.660 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 08:51:07.660 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 08:51:07.660 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:51:07.660 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 08:51:07.660 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:51:07.660 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 08:51:07.660 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 08:51:07.660 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 08:51:07.660 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 08:51:07.660 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:51:07.660 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 08:51:07.660 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:51:07.660 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:51:07.660 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:51:07.660 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 08:51:07.660 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 08:51:07.660 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:51:07.660 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 08:51:07.660 | INFO | ------------------------------------------ | 20260506 08:51:07.660 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:51:07.660 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 08:51:07.660 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:51:07.660 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:51:07.660 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 08:51:07.660 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / 0.00e+00, 4.87e-03 / 0.00e+00 | 20260506 08:51:07.660 | INFO | PerfReport: 10.84%, Baseline -- 3.29e+00 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260506 08:51:07.660 | INFO | PerfReport: 3.01%, Histogram -- 9.16e-01 / 0.00e+00, 5.13e-02 / 0.00e+00 | 20260506 08:51:07.660 | INFO | PerfReport: 16.06%, DME -- 3.90e+01 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260506 08:51:07.660 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.18e-02 / 0.00e+00 | 20260506 08:51:07.660 | INFO | PerfReport: 7.62%, PulseAccumulating -- 2.32e+00 / 0.00e+00, 1.41e-01 / 0.00e+00 | 20260506 08:51:07.660 | INFO | PerfReport: 3.97%, Metrics -- 1.21e+00 / 0.00e+00, 1.89e-01 / 0.00e+00 | 20260506 08:51:07.660 | INFO | PerfReport: 1.57%, PulseSerializing -- 4.77e-01 / 0.00e+00, 7.13e-02 / 0.00e+00 | 20260506 08:51:07.660 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 08:51:07.660 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:51:07.660 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:51:07.660 | INFO | PerfReport: Full Execution -- 9.12e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260506 08:51:07.660 | INFO | PerfReport: 18.37%, Upload -- 1.67e+01 / 0.00e+00, 4.09e-01 / 0.00e+00 | 20260506 08:51:07.660 | INFO | PerfReport: 17.63%, Download -- 1.61e+01 / 0.00e+00, 7.33e-01 / 0.00e+00 | 20260506 08:51:07.660 | INFO | PerfReport: 33.33%, Compute -- 3.04e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 08:51:07.660 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 08:51:07.660 | INFO | PerfReport: 0.00%, LockDownload -- 4.87e-04 / 0.00e+00, 1.26e-04 / 0.00e+00 | 20260506 08:51:07.660 | INFO | PerfReport: 30.10%, LockCompute -- 2.74e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 08:51:07.660 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260506 08:51:07.758 | INFO | Starting a round of Baz IO | 20260506 08:51:10.701 | INFO | Populating RTMetrics report | 20260506 08:51:10.702 | INFO | RTMetrics report populated | 20260506 08:51:10.702 | INFO | Opening tmp RTMetrics json file | 20260506 08:51:10.851 | INFO | Serializing RTMetrics to json | 20260506 08:51:10.874 | INFO | Serialization done, swapping tmp file now... | 20260506 08:51:10.879 | INFO | RTMetrics done | 20260506 08:51:10.975 | INFO | Finished a round of Baz IO: Wrote 1.628 GiB in 3.216 seconds | 20260506 08:51:10.975 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 67.29% / 12.49% / 20.22% | 20260506 08:51:10.975 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.7897% / 0.0591874% / 0% | 20260506 08:51:10.975 | INFO | Baz IO overall: | 20260506 08:51:10.975 | INFO | BytesWritten : 5589210462 | 20260506 08:51:10.975 | INFO | NumEvents : 1246955461 | 20260506 08:51:12.798 | INFO | Analyzing chunk frames = [57344,57856) | 20260506 08:51:12.798 | INFO | MemInfo: { | 20260506 08:51:12.798 | INFO | "gpu" : " 17 GB", | 20260506 08:51:12.798 | INFO | "huge" : "", | 20260506 08:51:12.798 | INFO | "huge_shared" : " 27 GB", | 20260506 08:51:12.798 | INFO | "malloc" : " 15 GB", | 20260506 08:51:12.798 | INFO | "pinned" : " 29 GB" | 20260506 08:51:12.798 | INFO | } | 20260506 08:51:12.798 | INFO | Perf report for the last 5.13921 seconds: | 20260506 08:51:12.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:51:12.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:12.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:12.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:12.798 | INFO | ANALYSIS : 58.3%, 2.8, 86.0ms, 0 | 20260506 08:51:12.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:12.798 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260506 08:51:12.798 | INFO | PRE_HQ : 57.8%, 2.1, 64.0ms, 0 | 20260506 08:51:12.798 | INFO | BAZWRITER : 62.1%, 4.6, 145.7ms, 0 | 20260506 08:51:12.798 | INFO | BAZ_SAMPLING : 3.0%, 1.0, 1.5ms, 0 | 20260506 08:51:12.798 | INFO | Uploaded 36GiB to GPU in 1.65828s (21.7092GiB/s) | 20260506 08:51:12.798 | INFO | Downloaded 38GiB to GPU in 1.71908s (22.1049GiB/s) | 20260506 08:51:17.937 | INFO | Analyzing chunk frames = [57856,58368) | 20260506 08:51:17.937 | INFO | MemInfo: { | 20260506 08:51:17.937 | INFO | "gpu" : " 17 GB", | 20260506 08:51:17.937 | INFO | "huge" : "", | 20260506 08:51:17.937 | INFO | "huge_shared" : " 27 GB", | 20260506 08:51:17.937 | INFO | "malloc" : " 15 GB", | 20260506 08:51:17.937 | INFO | "pinned" : " 29 GB" | 20260506 08:51:17.937 | INFO | } | 20260506 08:51:17.937 | INFO | Perf report for the last 5.13941 seconds: | 20260506 08:51:17.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:51:17.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:17.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:17.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:17.937 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260506 08:51:17.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:17.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:17.938 | INFO | PRE_HQ : 32.5%, 1.0, 17.2ms, 0 | 20260506 08:51:17.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:17.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:17.938 | INFO | Uploaded 36GiB to GPU in 1.59981s (22.5027GiB/s) | 20260506 08:51:17.938 | INFO | Downloaded 34GiB to GPU in 1.51697s (22.4131GiB/s) | 20260506 08:51:23.077 | INFO | Analyzing chunk frames = [58368,58880) | 20260506 08:51:23.077 | INFO | MemInfo: { | 20260506 08:51:23.077 | INFO | "gpu" : " 17 GB", | 20260506 08:51:23.077 | INFO | "huge" : "", | 20260506 08:51:23.077 | INFO | "huge_shared" : " 27 GB", | 20260506 08:51:23.077 | INFO | "malloc" : " 15 GB", | 20260506 08:51:23.077 | INFO | "pinned" : " 29 GB" | 20260506 08:51:23.077 | INFO | } | 20260506 08:51:23.077 | INFO | Perf report for the last 5.13968 seconds: | 20260506 08:51:23.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:51:23.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:23.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:23.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:23.077 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260506 08:51:23.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:23.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:23.077 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 08:51:23.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:23.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:23.077 | INFO | Uploaded 36GiB to GPU in 1.60317s (22.4556GiB/s) | 20260506 08:51:23.077 | INFO | Downloaded 34GiB to GPU in 1.52071s (22.3579GiB/s) | 20260506 08:51:28.216 | INFO | Analyzing chunk frames = [58880,59392) | 20260506 08:51:28.216 | INFO | MemInfo: { | 20260506 08:51:28.216 | INFO | "gpu" : " 17 GB", | 20260506 08:51:28.216 | INFO | "huge" : "", | 20260506 08:51:28.216 | INFO | "huge_shared" : " 27 GB", | 20260506 08:51:28.216 | INFO | "malloc" : " 15 GB", | 20260506 08:51:28.216 | INFO | "pinned" : " 29 GB" | 20260506 08:51:28.216 | INFO | } | 20260506 08:51:28.216 | INFO | Perf report for the last 5.13914 seconds: | 20260506 08:51:28.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:51:28.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:28.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:28.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:28.216 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260506 08:51:28.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:28.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:28.216 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 08:51:28.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:28.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:28.217 | INFO | Uploaded 36GiB to GPU in 1.60116s (22.4837GiB/s) | 20260506 08:51:28.217 | INFO | Downloaded 34GiB to GPU in 1.5207s (22.3582GiB/s) | 20260506 08:51:33.355 | INFO | Analyzing chunk frames = [59392,59904) | 20260506 08:51:33.355 | INFO | MemInfo: { | 20260506 08:51:33.355 | INFO | "gpu" : " 17 GB", | 20260506 08:51:33.356 | INFO | "huge" : "", | 20260506 08:51:33.356 | INFO | "huge_shared" : " 27 GB", | 20260506 08:51:33.356 | INFO | "malloc" : " 15 GB", | 20260506 08:51:33.356 | INFO | "pinned" : " 29 GB" | 20260506 08:51:33.356 | INFO | } | 20260506 08:51:33.356 | INFO | Perf report for the last 5.13934 seconds: | 20260506 08:51:33.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:51:33.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:33.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:33.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:33.356 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 08:51:33.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:33.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:33.356 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 08:51:33.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:33.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:33.356 | INFO | Uploaded 36GiB to GPU in 1.60093s (22.4869GiB/s) | 20260506 08:51:33.356 | INFO | Downloaded 34GiB to GPU in 1.51827s (22.394GiB/s) | 20260506 08:51:38.495 | INFO | Analyzing chunk frames = [59904,60416) | 20260506 08:51:38.495 | INFO | MemInfo: { | 20260506 08:51:38.495 | INFO | "gpu" : " 17 GB", | 20260506 08:51:38.495 | INFO | "huge" : "", | 20260506 08:51:38.495 | INFO | "huge_shared" : " 27 GB", | 20260506 08:51:38.495 | INFO | "malloc" : " 15 GB", | 20260506 08:51:38.495 | INFO | "pinned" : " 29 GB" | 20260506 08:51:38.495 | INFO | } | 20260506 08:51:38.495 | INFO | Perf report for the last 5.13922 seconds: | 20260506 08:51:38.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:51:38.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:38.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:38.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:38.495 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 08:51:38.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:38.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:38.495 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 08:51:38.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:38.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:38.495 | INFO | Uploaded 36GiB to GPU in 1.60121s (22.483GiB/s) | 20260506 08:51:38.495 | INFO | Downloaded 34GiB to GPU in 1.51862s (22.3887GiB/s) | 20260506 08:51:43.634 | INFO | Analyzing chunk frames = [60416,60928) | 20260506 08:51:43.634 | INFO | MemInfo: { | 20260506 08:51:43.634 | INFO | "gpu" : " 17 GB", | 20260506 08:51:43.634 | INFO | "huge" : "", | 20260506 08:51:43.634 | INFO | "huge_shared" : " 27 GB", | 20260506 08:51:43.634 | INFO | "malloc" : " 15 GB", | 20260506 08:51:43.634 | INFO | "pinned" : " 29 GB" | 20260506 08:51:43.634 | INFO | } | 20260506 08:51:43.634 | INFO | Perf report for the last 5.13948 seconds: | 20260506 08:51:43.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:51:43.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:43.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:43.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:43.634 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 08:51:43.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:43.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:43.635 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 08:51:43.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:43.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:43.635 | INFO | Uploaded 36GiB to GPU in 1.6017s (22.4761GiB/s) | 20260506 08:51:43.635 | INFO | Downloaded 34GiB to GPU in 1.51942s (22.3769GiB/s) | 20260506 08:51:48.773 | INFO | Analyzing chunk frames = [60928,61440) | 20260506 08:51:48.773 | INFO | MemInfo: { | 20260506 08:51:48.773 | INFO | "gpu" : " 17 GB", | 20260506 08:51:48.773 | INFO | "huge" : "", | 20260506 08:51:48.774 | INFO | "huge_shared" : " 27 GB", | 20260506 08:51:48.774 | INFO | "malloc" : " 15 GB", | 20260506 08:51:48.774 | INFO | "pinned" : " 29 GB" | 20260506 08:51:48.774 | INFO | } | 20260506 08:51:48.774 | INFO | Perf report for the last 5.13925 seconds: | 20260506 08:51:48.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:51:48.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:48.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:48.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:48.774 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 08:51:48.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:48.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:48.774 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 08:51:48.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:48.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:48.774 | INFO | Uploaded 36GiB to GPU in 1.60036s (22.4949GiB/s) | 20260506 08:51:48.774 | INFO | Downloaded 34GiB to GPU in 1.52085s (22.3559GiB/s) | 20260506 08:51:48.873 | INFO | Starting a round of Baz IO | 20260506 08:51:51.807 | INFO | Populating RTMetrics report | 20260506 08:51:51.807 | INFO | RTMetrics report populated | 20260506 08:51:51.807 | INFO | Opening tmp RTMetrics json file | 20260506 08:51:51.810 | INFO | Serializing RTMetrics to json | 20260506 08:51:51.832 | INFO | Serialization done, swapping tmp file now... | 20260506 08:51:51.859 | INFO | RTMetrics done | 20260506 08:51:51.884 | INFO | Finished a round of Baz IO: Wrote 1.497 GiB in 3.01 seconds | 20260506 08:51:51.884 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 66.59% / 0.70% / 32.71% | 20260506 08:51:51.884 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.8948% / 0.00323852% / 0% | 20260506 08:51:51.884 | INFO | Baz IO overall: | 20260506 08:51:51.884 | INFO | BytesWritten : 7197115811 | 20260506 08:51:51.884 | INFO | NumEvents : 2032670617 | 20260506 08:51:53.913 | INFO | Analyzing chunk frames = [61440,61952) | 20260506 08:51:53.913 | INFO | MemInfo: { | 20260506 08:51:53.913 | INFO | "gpu" : " 17 GB", | 20260506 08:51:53.913 | INFO | "huge" : "", | 20260506 08:51:53.913 | INFO | "huge_shared" : " 27 GB", | 20260506 08:51:53.913 | INFO | "malloc" : " 15 GB", | 20260506 08:51:53.913 | INFO | "pinned" : " 29 GB" | 20260506 08:51:53.913 | INFO | } | 20260506 08:51:53.913 | INFO | Perf report for the last 5.1394 seconds: | 20260506 08:51:53.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:51:53.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:53.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:53.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:53.913 | INFO | ANALYSIS : 58.1%, 3.0, 90.6ms, 0 | 20260506 08:51:53.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:53.913 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 08:51:53.913 | INFO | PRE_HQ : 57.6%, 2.1, 63.8ms, 0 | 20260506 08:51:53.913 | INFO | BAZWRITER : 58.1%, 3.6, 107.6ms, 0 | 20260506 08:51:53.913 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 08:51:53.913 | INFO | Uploaded 36GiB to GPU in 1.63403s (22.0314GiB/s) | 20260506 08:51:53.913 | INFO | Downloaded 38GiB to GPU in 1.69554s (22.4117GiB/s) | 20260506 08:51:59.052 | INFO | Analyzing chunk frames = [61952,62464) | 20260506 08:51:59.052 | INFO | MemInfo: { | 20260506 08:51:59.052 | INFO | "gpu" : " 17 GB", | 20260506 08:51:59.052 | INFO | "huge" : "", | 20260506 08:51:59.052 | INFO | "huge_shared" : " 27 GB", | 20260506 08:51:59.052 | INFO | "malloc" : " 15 GB", | 20260506 08:51:59.052 | INFO | "pinned" : " 29 GB" | 20260506 08:51:59.052 | INFO | } | 20260506 08:51:59.052 | INFO | Perf report for the last 5.13925 seconds: | 20260506 08:51:59.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:51:59.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:59.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:59.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:59.052 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260506 08:51:59.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:59.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:59.052 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 08:51:59.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:59.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:51:59.052 | INFO | Uploaded 36GiB to GPU in 1.59947s (22.5075GiB/s) | 20260506 08:51:59.053 | INFO | Downloaded 34GiB to GPU in 1.51612s (22.4257GiB/s) | 20260506 08:52:04.191 | INFO | Analyzing chunk frames = [62464,62976) | 20260506 08:52:04.191 | INFO | MemInfo: { | 20260506 08:52:04.191 | INFO | "gpu" : " 17 GB", | 20260506 08:52:04.191 | INFO | "huge" : "", | 20260506 08:52:04.191 | INFO | "huge_shared" : " 27 GB", | 20260506 08:52:04.191 | INFO | "malloc" : " 15 GB", | 20260506 08:52:04.192 | INFO | "pinned" : " 29 GB" | 20260506 08:52:04.192 | INFO | } | 20260506 08:52:04.192 | INFO | Perf report for the last 5.13925 seconds: | 20260506 08:52:04.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:52:04.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:04.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:04.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:04.192 | INFO | ANALYSIS : 57.4%, 3.0, 89.1ms, 0 | 20260506 08:52:04.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:04.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:04.192 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 08:52:04.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:04.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:04.192 | INFO | Uploaded 36GiB to GPU in 1.60314s (22.456GiB/s) | 20260506 08:52:04.192 | INFO | Downloaded 34GiB to GPU in 1.52111s (22.3521GiB/s) | 20260506 08:52:09.331 | INFO | Analyzing chunk frames = [62976,63488) | 20260506 08:52:09.331 | INFO | MemInfo: { | 20260506 08:52:09.331 | INFO | "gpu" : " 17 GB", | 20260506 08:52:09.331 | INFO | "huge" : "", | 20260506 08:52:09.331 | INFO | "huge_shared" : " 27 GB", | 20260506 08:52:09.331 | INFO | "malloc" : " 15 GB", | 20260506 08:52:09.331 | INFO | "pinned" : " 29 GB" | 20260506 08:52:09.331 | INFO | } | 20260506 08:52:09.331 | INFO | Perf report for the last 5.13943 seconds: | 20260506 08:52:09.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:52:09.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:09.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:09.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:09.331 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260506 08:52:09.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:09.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:09.331 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 08:52:09.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:09.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:09.331 | INFO | Uploaded 36GiB to GPU in 1.60338s (22.4525GiB/s) | 20260506 08:52:09.331 | INFO | Downloaded 34GiB to GPU in 1.52081s (22.3565GiB/s) | 20260506 08:52:14.470 | INFO | Analyzing chunk frames = [63488,64000) | 20260506 08:52:14.470 | INFO | MemInfo: { | 20260506 08:52:14.470 | INFO | "gpu" : " 17 GB", | 20260506 08:52:14.470 | INFO | "huge" : "", | 20260506 08:52:14.470 | INFO | "huge_shared" : " 27 GB", | 20260506 08:52:14.470 | INFO | "malloc" : " 15 GB", | 20260506 08:52:14.470 | INFO | "pinned" : " 29 GB" | 20260506 08:52:14.470 | INFO | } | 20260506 08:52:14.470 | INFO | Perf report for the last 5.13942 seconds: | 20260506 08:52:14.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:52:14.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:14.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:14.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:14.471 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 08:52:14.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:14.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:14.471 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 08:52:14.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:14.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:14.471 | INFO | Uploaded 36GiB to GPU in 1.60047s (22.4935GiB/s) | 20260506 08:52:14.471 | INFO | Downloaded 34GiB to GPU in 1.51754s (22.4046GiB/s) | 20260506 08:52:19.609 | INFO | Analyzing chunk frames = [64000,64512) | 20260506 08:52:19.609 | INFO | MemInfo: { | 20260506 08:52:19.609 | INFO | "gpu" : " 17 GB", | 20260506 08:52:19.609 | INFO | "huge" : "", | 20260506 08:52:19.610 | INFO | "huge_shared" : " 27 GB", | 20260506 08:52:19.610 | INFO | "malloc" : " 15 GB", | 20260506 08:52:19.610 | INFO | "pinned" : " 29 GB" | 20260506 08:52:19.610 | INFO | } | 20260506 08:52:19.610 | INFO | Perf report for the last 5.13928 seconds: | 20260506 08:52:19.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:52:19.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:19.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:19.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:19.610 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 08:52:19.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:19.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:19.610 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 08:52:19.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:19.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:19.610 | INFO | Uploaded 36GiB to GPU in 1.60092s (22.4871GiB/s) | 20260506 08:52:19.610 | INFO | Downloaded 34GiB to GPU in 1.51774s (22.4018GiB/s) | 20260506 08:52:24.749 | INFO | Analyzing chunk frames = [64512,65024) | 20260506 08:52:24.749 | INFO | MemInfo: { | 20260506 08:52:24.749 | INFO | "gpu" : " 17 GB", | 20260506 08:52:24.749 | INFO | "huge" : "", | 20260506 08:52:24.749 | INFO | "huge_shared" : " 27 GB", | 20260506 08:52:24.749 | INFO | "malloc" : " 15 GB", | 20260506 08:52:24.749 | INFO | "pinned" : " 29 GB" | 20260506 08:52:24.749 | INFO | } | 20260506 08:52:24.749 | INFO | Perf report for the last 5.13929 seconds: | 20260506 08:52:24.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:52:24.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:24.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:24.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:24.749 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260506 08:52:24.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:24.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:24.749 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 08:52:24.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:24.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:24.749 | INFO | Uploaded 36GiB to GPU in 1.60103s (22.4855GiB/s) | 20260506 08:52:24.749 | INFO | Downloaded 34GiB to GPU in 1.51881s (22.386GiB/s) | 20260506 08:52:29.888 | INFO | Analyzing chunk frames = [65024,65536) | 20260506 08:52:29.888 | INFO | MemInfo: { | 20260506 08:52:29.888 | INFO | "gpu" : " 17 GB", | 20260506 08:52:29.888 | INFO | "huge" : "", | 20260506 08:52:29.888 | INFO | "huge_shared" : " 27 GB", | 20260506 08:52:29.888 | INFO | "malloc" : " 15 GB", | 20260506 08:52:29.888 | INFO | "pinned" : " 29 GB" | 20260506 08:52:29.888 | INFO | } | 20260506 08:52:29.888 | INFO | Perf report for the last 5.13925 seconds: | 20260506 08:52:29.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:52:29.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:29.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:29.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:29.888 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 08:52:29.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:29.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:29.888 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 08:52:29.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:29.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:29.889 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 08:52:29.889 | INFO | Total mem usage: 15.5 GiB | 20260506 08:52:29.889 | INFO | All Time High Water Marks sum: 15.6 GiB (Savings: 0.781%) | 20260506 08:52:29.889 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -1.17%) | 20260506 08:52:29.889 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:52:29.889 | INFO | High Water Breakdown: | 20260506 08:52:29.889 | INFO | 58% : 8.91 GiB ( 99.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 08:52:29.889 | INFO | 41.8% : 6.42 GiB ( 95.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 08:52:29.889 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 08:52:29.889 | INFO | ------------------------------------------ | 20260506 08:52:29.889 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:52:29.889 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 08:52:29.889 | INFO | Total mem usage: 26.5 GiB | 20260506 08:52:29.889 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 08:52:29.889 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 08:52:29.889 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 08:52:29.889 | INFO | High Water Breakdown: | 20260506 08:52:29.889 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:52:29.889 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:52:29.889 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:52:29.889 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:52:29.889 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:52:29.889 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:52:29.889 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 08:52:29.889 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 08:52:29.889 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:52:29.889 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 08:52:29.889 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 08:52:29.889 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 08:52:29.889 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 08:52:29.889 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 08:52:29.889 | INFO | ------------------------------------------ | 20260506 08:52:29.889 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:52:29.889 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 08:52:29.889 | INFO | Total mem usage: 25.6 GiB | 20260506 08:52:29.889 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 08:52:29.889 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 08:52:29.889 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 08:52:29.889 | INFO | High Water Breakdown: | 20260506 08:52:29.889 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 08:52:29.889 | INFO | ------------------------------------------ | 20260506 08:52:29.889 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:52:29.889 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 08:52:29.889 | INFO | Total mem usage: 15.7 GiB | 20260506 08:52:29.889 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 08:52:29.889 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 08:52:29.889 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 08:52:29.889 | INFO | High Water Breakdown: | 20260506 08:52:29.889 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 08:52:29.889 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 08:52:29.889 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 08:52:29.889 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 08:52:29.889 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 08:52:29.889 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 08:52:29.889 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 08:52:29.889 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 08:52:29.889 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 08:52:29.889 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:52:29.889 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 08:52:29.889 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:52:29.889 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 08:52:29.889 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 08:52:29.889 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 08:52:29.889 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 08:52:29.889 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:52:29.889 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 08:52:29.889 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:52:29.889 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:52:29.889 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:52:29.889 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 08:52:29.889 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 08:52:29.889 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:52:29.889 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 08:52:29.889 | INFO | ------------------------------------------ | 20260506 08:52:29.889 | INFO | Uploaded 36GiB to GPU in 1.60252s (22.4646GiB/s) | 20260506 08:52:29.889 | INFO | Downloaded 34GiB to GPU in 1.51968s (22.3732GiB/s) | 20260506 08:52:29.889 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:52:29.889 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 08:52:29.889 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:52:29.889 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:52:29.889 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260506 08:52:29.889 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / 0.00e+00, 1.32e-02 / 0.00e+00 | 20260506 08:52:29.889 | INFO | PerfReport: 10.82%, Baseline -- 3.29e+00 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260506 08:52:29.889 | INFO | PerfReport: 3.02%, Histogram -- 9.18e-01 / 0.00e+00, 5.08e-02 / 0.00e+00 | 20260506 08:52:29.889 | INFO | PerfReport: 16.03%, DME -- 3.90e+01 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260506 08:52:29.889 | INFO | PerfReport: 56.59%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.25e-02 / 0.00e+00 | 20260506 08:52:29.889 | INFO | PerfReport: 7.65%, PulseAccumulating -- 2.33e+00 / 0.00e+00, 1.42e-01 / 0.00e+00 | 20260506 08:52:29.889 | INFO | PerfReport: 4.01%, Metrics -- 1.22e+00 / 0.00e+00, 1.90e-01 / 0.00e+00 | 20260506 08:52:29.889 | INFO | PerfReport: 1.60%, PulseSerializing -- 4.85e-01 / 0.00e+00, 1.65e-01 / 0.00e+00 | 20260506 08:52:29.889 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 08:52:29.889 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:52:29.889 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:52:29.889 | INFO | PerfReport: Full Execution -- 9.10e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260506 08:52:29.889 | INFO | PerfReport: 18.39%, Upload -- 1.67e+01 / 0.00e+00, 4.08e-01 / 0.00e+00 | 20260506 08:52:29.889 | INFO | PerfReport: 17.65%, Download -- 1.61e+01 / 0.00e+00, 7.29e-01 / 0.00e+00 | 20260506 08:52:29.889 | INFO | PerfReport: 33.41%, Compute -- 3.04e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260506 08:52:29.890 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 08:52:29.890 | INFO | PerfReport: 0.00%, LockDownload -- 4.77e-04 / 0.00e+00, 1.28e-04 / 0.00e+00 | 20260506 08:52:29.890 | INFO | PerfReport: 29.98%, LockCompute -- 2.73e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 08:52:29.890 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260506 08:52:29.986 | INFO | Starting a round of Baz IO | 20260506 08:52:32.926 | INFO | Populating RTMetrics report | 20260506 08:52:32.926 | INFO | RTMetrics report populated | 20260506 08:52:32.926 | INFO | Opening tmp RTMetrics json file | 20260506 08:52:32.945 | INFO | Serializing RTMetrics to json | 20260506 08:52:32.956 | INFO | Serialization done, swapping tmp file now... | 20260506 08:52:32.972 | INFO | RTMetrics done | 20260506 08:52:32.996 | INFO | Finished a round of Baz IO: Wrote 1.526 GiB in 3.009 seconds | 20260506 08:52:32.996 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 65.96% / 0.63% / 33.41% | 20260506 08:52:32.996 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.9846% / 0.00267029% / 0% | 20260506 08:52:32.996 | INFO | Baz IO overall: | 20260506 08:52:32.996 | INFO | BytesWritten : 8835998060 | 20260506 08:52:32.996 | INFO | NumEvents : 2831009675 | 20260506 08:52:35.028 | INFO | Analyzing chunk frames = [65536,66048) | 20260506 08:52:35.028 | INFO | MemInfo: { | 20260506 08:52:35.028 | INFO | "gpu" : " 17 GB", | 20260506 08:52:35.028 | INFO | "huge" : "", | 20260506 08:52:35.028 | INFO | "huge_shared" : " 27 GB", | 20260506 08:52:35.028 | INFO | "malloc" : " 15 GB", | 20260506 08:52:35.028 | INFO | "pinned" : " 29 GB" | 20260506 08:52:35.028 | INFO | } | 20260506 08:52:35.028 | INFO | Perf report for the last 5.13932 seconds: | 20260506 08:52:35.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:52:35.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:35.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:35.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:35.028 | INFO | ANALYSIS : 58.2%, 3.0, 90.7ms, 0 | 20260506 08:52:35.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:35.028 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 08:52:35.028 | INFO | PRE_HQ : 57.5%, 2.1, 63.7ms, 0 | 20260506 08:52:35.028 | INFO | BAZWRITER : 58.0%, 3.7, 109.7ms, 0 | 20260506 08:52:35.028 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 08:52:35.028 | INFO | Uploaded 36GiB to GPU in 1.6347s (22.0223GiB/s) | 20260506 08:52:35.028 | INFO | Downloaded 38GiB to GPU in 1.69652s (22.3988GiB/s) | 20260506 08:52:40.167 | INFO | Analyzing chunk frames = [66048,66560) | 20260506 08:52:40.167 | INFO | MemInfo: { | 20260506 08:52:40.167 | INFO | "gpu" : " 17 GB", | 20260506 08:52:40.167 | INFO | "huge" : "", | 20260506 08:52:40.167 | INFO | "huge_shared" : " 27 GB", | 20260506 08:52:40.167 | INFO | "malloc" : " 15 GB", | 20260506 08:52:40.167 | INFO | "pinned" : " 29 GB" | 20260506 08:52:40.167 | INFO | } | 20260506 08:52:40.167 | INFO | Perf report for the last 5.13946 seconds: | 20260506 08:52:40.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:52:40.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:40.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:40.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:40.167 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260506 08:52:40.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:40.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:40.167 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 08:52:40.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:40.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:40.167 | INFO | Uploaded 36GiB to GPU in 1.59968s (22.5045GiB/s) | 20260506 08:52:40.167 | INFO | Downloaded 34GiB to GPU in 1.5162s (22.4246GiB/s) | 20260506 08:52:45.306 | INFO | Analyzing chunk frames = [66560,67072) | 20260506 08:52:45.306 | INFO | MemInfo: { | 20260506 08:52:45.306 | INFO | "gpu" : " 17 GB", | 20260506 08:52:45.306 | INFO | "huge" : "", | 20260506 08:52:45.306 | INFO | "huge_shared" : " 27 GB", | 20260506 08:52:45.306 | INFO | "malloc" : " 15 GB", | 20260506 08:52:45.306 | INFO | "pinned" : " 29 GB" | 20260506 08:52:45.306 | INFO | } | 20260506 08:52:45.306 | INFO | Perf report for the last 5.13922 seconds: | 20260506 08:52:45.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:52:45.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:45.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:45.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:45.306 | INFO | ANALYSIS : 57.4%, 3.0, 89.1ms, 0 | 20260506 08:52:45.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:45.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:45.306 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 08:52:45.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:45.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:45.307 | INFO | Uploaded 36GiB to GPU in 1.60342s (22.452GiB/s) | 20260506 08:52:45.307 | INFO | Downloaded 34GiB to GPU in 1.52087s (22.3556GiB/s) | 20260506 08:52:50.446 | INFO | Analyzing chunk frames = [67072,67584) | 20260506 08:52:50.446 | INFO | MemInfo: { | 20260506 08:52:50.446 | INFO | "gpu" : " 17 GB", | 20260506 08:52:50.446 | INFO | "huge" : "", | 20260506 08:52:50.446 | INFO | "huge_shared" : " 27 GB", | 20260506 08:52:50.446 | INFO | "malloc" : " 15 GB", | 20260506 08:52:50.446 | INFO | "pinned" : " 29 GB" | 20260506 08:52:50.446 | INFO | } | 20260506 08:52:50.446 | INFO | Perf report for the last 5.13931 seconds: | 20260506 08:52:50.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:52:50.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:50.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:50.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:50.446 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260506 08:52:50.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:50.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:50.446 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 08:52:50.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:50.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:50.446 | INFO | Uploaded 36GiB to GPU in 1.60301s (22.4577GiB/s) | 20260506 08:52:50.446 | INFO | Downloaded 34GiB to GPU in 1.52056s (22.3602GiB/s) | 20260506 08:52:55.585 | INFO | Analyzing chunk frames = [67584,68096) | 20260506 08:52:55.585 | INFO | MemInfo: { | 20260506 08:52:55.585 | INFO | "gpu" : " 17 GB", | 20260506 08:52:55.585 | INFO | "huge" : "", | 20260506 08:52:55.585 | INFO | "huge_shared" : " 27 GB", | 20260506 08:52:55.585 | INFO | "malloc" : " 15 GB", | 20260506 08:52:55.585 | INFO | "pinned" : " 29 GB" | 20260506 08:52:55.585 | INFO | } | 20260506 08:52:55.585 | INFO | Perf report for the last 5.13933 seconds: | 20260506 08:52:55.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:52:55.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:55.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:55.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:55.585 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 08:52:55.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:55.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:55.585 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 08:52:55.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:55.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:52:55.585 | INFO | Uploaded 36GiB to GPU in 1.60082s (22.4885GiB/s) | 20260506 08:52:55.585 | INFO | Downloaded 34GiB to GPU in 1.51775s (22.4016GiB/s) | 20260506 08:53:00.724 | INFO | Analyzing chunk frames = [68096,68608) | 20260506 08:53:00.724 | INFO | MemInfo: { | 20260506 08:53:00.724 | INFO | "gpu" : " 17 GB", | 20260506 08:53:00.724 | INFO | "huge" : "", | 20260506 08:53:00.724 | INFO | "huge_shared" : " 27 GB", | 20260506 08:53:00.724 | INFO | "malloc" : " 15 GB", | 20260506 08:53:00.724 | INFO | "pinned" : " 29 GB" | 20260506 08:53:00.724 | INFO | } | 20260506 08:53:00.725 | INFO | Perf report for the last 5.13949 seconds: | 20260506 08:53:00.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:53:00.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:00.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:00.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:00.725 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 08:53:00.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:00.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:00.725 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 08:53:00.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:00.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:00.725 | INFO | Uploaded 36GiB to GPU in 1.60171s (22.476GiB/s) | 20260506 08:53:00.725 | INFO | Downloaded 34GiB to GPU in 1.51757s (22.4042GiB/s) | 20260506 08:53:05.864 | INFO | Analyzing chunk frames = [68608,69120) | 20260506 08:53:05.864 | INFO | MemInfo: { | 20260506 08:53:05.864 | INFO | "gpu" : " 17 GB", | 20260506 08:53:05.864 | INFO | "huge" : "", | 20260506 08:53:05.864 | INFO | "huge_shared" : " 27 GB", | 20260506 08:53:05.864 | INFO | "malloc" : " 15 GB", | 20260506 08:53:05.864 | INFO | "pinned" : " 29 GB" | 20260506 08:53:05.864 | INFO | } | 20260506 08:53:05.864 | INFO | Perf report for the last 5.1393 seconds: | 20260506 08:53:05.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:53:05.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:05.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:05.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:05.864 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 08:53:05.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:05.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:05.864 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 08:53:05.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:05.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:05.864 | INFO | Uploaded 36GiB to GPU in 1.60117s (22.4835GiB/s) | 20260506 08:53:05.864 | INFO | Downloaded 34GiB to GPU in 1.51904s (22.3825GiB/s) | 20260506 08:53:11.003 | INFO | Analyzing chunk frames = [69120,69632) | 20260506 08:53:11.003 | INFO | MemInfo: { | 20260506 08:53:11.003 | INFO | "gpu" : " 17 GB", | 20260506 08:53:11.003 | INFO | "huge" : "", | 20260506 08:53:11.003 | INFO | "huge_shared" : " 27 GB", | 20260506 08:53:11.003 | INFO | "malloc" : " 15 GB", | 20260506 08:53:11.003 | INFO | "pinned" : " 29 GB" | 20260506 08:53:11.003 | INFO | } | 20260506 08:53:11.003 | INFO | Perf report for the last 5.13937 seconds: | 20260506 08:53:11.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:53:11.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:11.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:11.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:11.003 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 08:53:11.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:11.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:11.003 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 08:53:11.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:11.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:11.004 | INFO | Uploaded 36GiB to GPU in 1.60265s (22.4627GiB/s) | 20260506 08:53:11.004 | INFO | Downloaded 34GiB to GPU in 1.51952s (22.3755GiB/s) | 20260506 08:53:11.100 | INFO | Starting a round of Baz IO | 20260506 08:53:14.041 | INFO | Populating RTMetrics report | 20260506 08:53:14.042 | INFO | RTMetrics report populated | 20260506 08:53:14.042 | INFO | Opening tmp RTMetrics json file | 20260506 08:53:14.043 | INFO | Serializing RTMetrics to json | 20260506 08:53:14.062 | INFO | Serialization done, swapping tmp file now... | 20260506 08:53:14.067 | INFO | RTMetrics done | 20260506 08:53:14.118 | INFO | Finished a round of Baz IO: Wrote 1.548 GiB in 3.017 seconds | 20260506 08:53:14.118 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 65.39% / 0.57% / 34.04% | 20260506 08:53:14.118 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.9909% / 0.00266631% / 0% | 20260506 08:53:14.118 | INFO | Baz IO overall: | 20260506 08:53:14.118 | INFO | BytesWritten : 10498573767 | 20260506 08:53:14.118 | INFO | NumEvents : 3639323598 | 20260506 08:53:16.142 | INFO | Analyzing chunk frames = [69632,70144) | 20260506 08:53:16.142 | INFO | MemInfo: { | 20260506 08:53:16.142 | INFO | "gpu" : " 17 GB", | 20260506 08:53:16.142 | INFO | "huge" : "", | 20260506 08:53:16.142 | INFO | "huge_shared" : " 27 GB", | 20260506 08:53:16.142 | INFO | "malloc" : " 15 GB", | 20260506 08:53:16.142 | INFO | "pinned" : " 29 GB" | 20260506 08:53:16.142 | INFO | } | 20260506 08:53:16.142 | INFO | Perf report for the last 5.13924 seconds: | 20260506 08:53:16.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:53:16.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:16.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:16.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:16.143 | INFO | ANALYSIS : 58.2%, 3.0, 90.5ms, 0 | 20260506 08:53:16.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:16.143 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260506 08:53:16.143 | INFO | PRE_HQ : 57.4%, 2.1, 63.4ms, 0 | 20260506 08:53:16.143 | INFO | BAZWRITER : 58.3%, 3.9, 115.5ms, 0 | 20260506 08:53:16.143 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 08:53:16.143 | INFO | Uploaded 36GiB to GPU in 1.63683s (21.9938GiB/s) | 20260506 08:53:16.143 | INFO | Downloaded 38GiB to GPU in 1.69743s (22.3867GiB/s) | 20260506 08:53:21.282 | INFO | Analyzing chunk frames = [70144,70656) | 20260506 08:53:21.282 | INFO | MemInfo: { | 20260506 08:53:21.282 | INFO | "gpu" : " 17 GB", | 20260506 08:53:21.282 | INFO | "huge" : "", | 20260506 08:53:21.282 | INFO | "huge_shared" : " 27 GB", | 20260506 08:53:21.282 | INFO | "malloc" : " 15 GB", | 20260506 08:53:21.282 | INFO | "pinned" : " 29 GB" | 20260506 08:53:21.282 | INFO | } | 20260506 08:53:21.282 | INFO | Perf report for the last 5.1394 seconds: | 20260506 08:53:21.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:53:21.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:21.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:21.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:21.282 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260506 08:53:21.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:21.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:21.282 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 08:53:21.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:21.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:21.282 | INFO | Uploaded 36GiB to GPU in 1.59961s (22.5055GiB/s) | 20260506 08:53:21.282 | INFO | Downloaded 34GiB to GPU in 1.51627s (22.4234GiB/s) | 20260506 08:53:26.421 | INFO | Analyzing chunk frames = [70656,71168) | 20260506 08:53:26.421 | INFO | MemInfo: { | 20260506 08:53:26.421 | INFO | "gpu" : " 17 GB", | 20260506 08:53:26.421 | INFO | "huge" : "", | 20260506 08:53:26.421 | INFO | "huge_shared" : " 27 GB", | 20260506 08:53:26.421 | INFO | "malloc" : " 15 GB", | 20260506 08:53:26.421 | INFO | "pinned" : " 29 GB" | 20260506 08:53:26.421 | INFO | } | 20260506 08:53:26.421 | INFO | Perf report for the last 5.13929 seconds: | 20260506 08:53:26.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:53:26.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:26.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:26.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:26.421 | INFO | ANALYSIS : 57.4%, 3.0, 89.2ms, 0 | 20260506 08:53:26.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:26.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:26.421 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 08:53:26.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:26.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:26.421 | INFO | Uploaded 36GiB to GPU in 1.60305s (22.4573GiB/s) | 20260506 08:53:26.422 | INFO | Downloaded 34GiB to GPU in 1.52091s (22.355GiB/s) | 20260506 08:53:31.560 | INFO | Analyzing chunk frames = [71168,71680) | 20260506 08:53:31.560 | INFO | MemInfo: { | 20260506 08:53:31.560 | INFO | "gpu" : " 17 GB", | 20260506 08:53:31.560 | INFO | "huge" : "", | 20260506 08:53:31.560 | INFO | "huge_shared" : " 27 GB", | 20260506 08:53:31.560 | INFO | "malloc" : " 15 GB", | 20260506 08:53:31.560 | INFO | "pinned" : " 29 GB" | 20260506 08:53:31.560 | INFO | } | 20260506 08:53:31.561 | INFO | Perf report for the last 5.13933 seconds: | 20260506 08:53:31.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:53:31.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:31.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:31.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:31.561 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260506 08:53:31.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:31.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:31.561 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260506 08:53:31.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:31.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:31.561 | INFO | Uploaded 36GiB to GPU in 1.60288s (22.4596GiB/s) | 20260506 08:53:31.561 | INFO | Downloaded 34GiB to GPU in 1.51982s (22.3711GiB/s) | 20260506 08:53:36.700 | INFO | Analyzing chunk frames = [71680,72192) | 20260506 08:53:36.700 | INFO | MemInfo: { | 20260506 08:53:36.700 | INFO | "gpu" : " 17 GB", | 20260506 08:53:36.700 | INFO | "huge" : "", | 20260506 08:53:36.700 | INFO | "huge_shared" : " 27 GB", | 20260506 08:53:36.700 | INFO | "malloc" : " 15 GB", | 20260506 08:53:36.700 | INFO | "pinned" : " 29 GB" | 20260506 08:53:36.700 | INFO | } | 20260506 08:53:36.700 | INFO | Perf report for the last 5.13925 seconds: | 20260506 08:53:36.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:53:36.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:36.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:36.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:36.700 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260506 08:53:36.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:36.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:36.700 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 08:53:36.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:36.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:36.700 | INFO | Uploaded 36GiB to GPU in 1.60066s (22.4908GiB/s) | 20260506 08:53:36.700 | INFO | Downloaded 34GiB to GPU in 1.51737s (22.4072GiB/s) | 20260506 08:53:41.839 | INFO | Analyzing chunk frames = [72192,72704) | 20260506 08:53:41.839 | INFO | MemInfo: { | 20260506 08:53:41.839 | INFO | "gpu" : " 17 GB", | 20260506 08:53:41.839 | INFO | "huge" : "", | 20260506 08:53:41.839 | INFO | "huge_shared" : " 27 GB", | 20260506 08:53:41.839 | INFO | "malloc" : " 15 GB", | 20260506 08:53:41.839 | INFO | "pinned" : " 29 GB" | 20260506 08:53:41.839 | INFO | } | 20260506 08:53:41.839 | INFO | Perf report for the last 5.13933 seconds: | 20260506 08:53:41.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:53:41.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:41.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:41.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:41.839 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 08:53:41.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:41.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:41.839 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 08:53:41.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:41.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:41.839 | INFO | Uploaded 36GiB to GPU in 1.60129s (22.4819GiB/s) | 20260506 08:53:41.840 | INFO | Downloaded 34GiB to GPU in 1.5184s (22.392GiB/s) | 20260506 08:53:46.978 | INFO | Analyzing chunk frames = [72704,73216) | 20260506 08:53:46.978 | INFO | MemInfo: { | 20260506 08:53:46.978 | INFO | "gpu" : " 17 GB", | 20260506 08:53:46.978 | INFO | "huge" : "", | 20260506 08:53:46.978 | INFO | "huge_shared" : " 27 GB", | 20260506 08:53:46.978 | INFO | "malloc" : " 15 GB", | 20260506 08:53:46.978 | INFO | "pinned" : " 29 GB" | 20260506 08:53:46.978 | INFO | } | 20260506 08:53:46.979 | INFO | Perf report for the last 5.13938 seconds: | 20260506 08:53:46.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:53:46.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:46.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:46.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:46.979 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 08:53:46.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:46.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:46.979 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 08:53:46.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:46.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:46.979 | INFO | Uploaded 36GiB to GPU in 1.60123s (22.4827GiB/s) | 20260506 08:53:46.979 | INFO | Downloaded 34GiB to GPU in 1.51927s (22.3792GiB/s) | 20260506 08:53:52.118 | INFO | Analyzing chunk frames = [73216,73728) | 20260506 08:53:52.118 | INFO | MemInfo: { | 20260506 08:53:52.118 | INFO | "gpu" : " 17 GB", | 20260506 08:53:52.118 | INFO | "huge" : "", | 20260506 08:53:52.118 | INFO | "huge_shared" : " 27 GB", | 20260506 08:53:52.118 | INFO | "malloc" : " 15 GB", | 20260506 08:53:52.118 | INFO | "pinned" : " 29 GB" | 20260506 08:53:52.118 | INFO | } | 20260506 08:53:52.118 | INFO | Perf report for the last 5.13939 seconds: | 20260506 08:53:52.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:53:52.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:52.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:52.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:52.118 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260506 08:53:52.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:52.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:52.118 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 08:53:52.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:52.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:52.118 | INFO | Uploaded 36GiB to GPU in 1.60159s (22.4777GiB/s) | 20260506 08:53:52.118 | INFO | Downloaded 34GiB to GPU in 1.51903s (22.3827GiB/s) | 20260506 08:53:52.118 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 08:53:52.118 | INFO | Total mem usage: 15.5 GiB | 20260506 08:53:52.118 | INFO | All Time High Water Marks sum: 15.6 GiB (Savings: 0.781%) | 20260506 08:53:52.118 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -5.45%) | 20260506 08:53:52.118 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:53:52.118 | INFO | High Water Breakdown: | 20260506 08:53:52.118 | INFO | 57.8% : 8.51 GiB ( 95.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 08:53:52.118 | INFO | 42.1% : 6.19 GiB ( 92.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 08:53:52.118 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 08:53:52.118 | INFO | ------------------------------------------ | 20260506 08:53:52.118 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:53:52.118 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 08:53:52.118 | INFO | Total mem usage: 26.5 GiB | 20260506 08:53:52.118 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 08:53:52.118 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 08:53:52.118 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 08:53:52.118 | INFO | High Water Breakdown: | 20260506 08:53:52.118 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:53:52.118 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:53:52.118 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:53:52.118 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:53:52.118 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:53:52.118 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:53:52.118 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 08:53:52.118 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 08:53:52.119 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:53:52.119 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 08:53:52.119 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 08:53:52.119 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 08:53:52.119 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 08:53:52.119 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 08:53:52.119 | INFO | ------------------------------------------ | 20260506 08:53:52.119 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:53:52.119 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 08:53:52.119 | INFO | Total mem usage: 25.6 GiB | 20260506 08:53:52.119 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 08:53:52.119 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 08:53:52.119 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 08:53:52.119 | INFO | High Water Breakdown: | 20260506 08:53:52.119 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 08:53:52.119 | INFO | ------------------------------------------ | 20260506 08:53:52.119 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:53:52.119 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 08:53:52.119 | INFO | Total mem usage: 15.7 GiB | 20260506 08:53:52.119 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 08:53:52.119 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 08:53:52.119 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 08:53:52.119 | INFO | High Water Breakdown: | 20260506 08:53:52.119 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 08:53:52.119 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 08:53:52.119 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 08:53:52.119 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 08:53:52.119 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 08:53:52.119 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 08:53:52.119 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 08:53:52.119 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 08:53:52.119 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 08:53:52.119 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:53:52.119 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 08:53:52.119 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:53:52.119 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 08:53:52.119 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 08:53:52.119 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 08:53:52.119 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 08:53:52.119 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:53:52.119 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 08:53:52.119 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:53:52.119 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:53:52.119 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:53:52.119 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 08:53:52.119 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 08:53:52.119 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:53:52.119 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 08:53:52.119 | INFO | ------------------------------------------ | 20260506 08:53:52.119 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:53:52.119 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 08:53:52.119 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:53:52.119 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:53:52.119 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 08:53:52.119 | INFO | PerfReport: 0.28%, AggregateStats -- 8.61e-02 / 0.00e+00, 4.67e-03 / 0.00e+00 | 20260506 08:53:52.119 | INFO | PerfReport: 10.83%, Baseline -- 3.30e+00 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260506 08:53:52.119 | INFO | PerfReport: 3.03%, Histogram -- 9.21e-01 / 0.00e+00, 5.15e-02 / 0.00e+00 | 20260506 08:53:52.119 | INFO | PerfReport: 16.01%, DME -- 3.90e+01 / 0.00e+00, 3.18e+00 / 0.00e+00 | 20260506 08:53:52.119 | INFO | PerfReport: 56.56%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.22e-02 / 0.00e+00 | 20260506 08:53:52.119 | INFO | PerfReport: 7.67%, PulseAccumulating -- 2.33e+00 / 0.00e+00, 1.42e-01 / 0.00e+00 | 20260506 08:53:52.119 | INFO | PerfReport: 4.03%, Metrics -- 1.23e+00 / 0.00e+00, 1.91e-01 / 0.00e+00 | 20260506 08:53:52.119 | INFO | PerfReport: 1.59%, PulseSerializing -- 4.84e-01 / 0.00e+00, 7.01e-02 / 0.00e+00 | 20260506 08:53:52.119 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 08:53:52.119 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:53:52.119 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:53:52.119 | INFO | PerfReport: Full Execution -- 9.14e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260506 08:53:52.119 | INFO | PerfReport: 18.31%, Upload -- 1.67e+01 / 0.00e+00, 3.67e-01 / 0.00e+00 | 20260506 08:53:52.119 | INFO | PerfReport: 17.57%, Download -- 1.61e+01 / 0.00e+00, 6.78e-01 / 0.00e+00 | 20260506 08:53:52.119 | INFO | PerfReport: 33.30%, Compute -- 3.04e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 08:53:52.119 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260506 08:53:52.119 | INFO | PerfReport: 0.00%, LockDownload -- 5.10e-04 / 0.00e+00, 2.39e-04 / 0.00e+00 | 20260506 08:53:52.119 | INFO | PerfReport: 30.25%, LockCompute -- 2.76e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260506 08:53:52.119 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260506 08:53:52.216 | INFO | Starting a round of Baz IO | 20260506 08:53:55.155 | INFO | Populating RTMetrics report | 20260506 08:53:55.156 | INFO | RTMetrics report populated | 20260506 08:53:55.156 | INFO | Opening tmp RTMetrics json file | 20260506 08:53:55.191 | INFO | Serializing RTMetrics to json | 20260506 08:53:55.195 | INFO | Serialization done, swapping tmp file now... | 20260506 08:53:55.198 | INFO | RTMetrics done | 20260506 08:53:55.326 | INFO | Finished a round of Baz IO: Wrote 1.575 GiB in 3.11 seconds | 20260506 08:53:55.326 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 64.85% / 0.53% / 34.61% | 20260506 08:53:55.326 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.0687% / 0.002714% / 3.97364e-06% | 20260506 08:53:55.326 | INFO | Baz IO overall: | 20260506 08:53:55.326 | INFO | BytesWritten : 12190048847 | 20260506 08:53:55.326 | INFO | NumEvents : 4460473638 | 20260506 08:53:57.257 | INFO | Analyzing chunk frames = [73728,74240) | 20260506 08:53:57.257 | INFO | MemInfo: { | 20260506 08:53:57.257 | INFO | "gpu" : " 17 GB", | 20260506 08:53:57.257 | INFO | "huge" : "", | 20260506 08:53:57.257 | INFO | "huge_shared" : " 27 GB", | 20260506 08:53:57.257 | INFO | "malloc" : " 15 GB", | 20260506 08:53:57.257 | INFO | "pinned" : " 29 GB" | 20260506 08:53:57.257 | INFO | } | 20260506 08:53:57.257 | INFO | Perf report for the last 5.13926 seconds: | 20260506 08:53:57.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:53:57.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:57.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:57.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:57.257 | INFO | ANALYSIS : 58.2%, 2.9, 88.9ms, 0 | 20260506 08:53:57.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:53:57.257 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 08:53:57.257 | INFO | PRE_HQ : 57.5%, 2.1, 63.2ms, 0 | 20260506 08:53:57.257 | INFO | BAZWRITER : 60.0%, 4.2, 128.7ms, 0 | 20260506 08:53:57.257 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 08:53:57.258 | INFO | Uploaded 36GiB to GPU in 1.64261s (21.9164GiB/s) | 20260506 08:53:57.258 | INFO | Downloaded 38GiB to GPU in 1.70515s (22.2854GiB/s) | 20260506 08:54:02.396 | INFO | Analyzing chunk frames = [74240,74752) | 20260506 08:54:02.396 | INFO | MemInfo: { | 20260506 08:54:02.396 | INFO | "gpu" : " 17 GB", | 20260506 08:54:02.396 | INFO | "huge" : "", | 20260506 08:54:02.396 | INFO | "huge_shared" : " 27 GB", | 20260506 08:54:02.397 | INFO | "malloc" : " 15 GB", | 20260506 08:54:02.397 | INFO | "pinned" : " 29 GB" | 20260506 08:54:02.397 | INFO | } | 20260506 08:54:02.397 | INFO | Perf report for the last 5.13943 seconds: | 20260506 08:54:02.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:54:02.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:02.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:02.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:02.397 | INFO | ANALYSIS : 58.0%, 3.0, 90.5ms, 0 | 20260506 08:54:02.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:02.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:02.397 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 08:54:02.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:02.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:02.397 | INFO | Uploaded 36GiB to GPU in 1.59944s (22.5078GiB/s) | 20260506 08:54:02.397 | INFO | Downloaded 34GiB to GPU in 1.51643s (22.4211GiB/s) | 20260506 08:54:07.536 | INFO | Analyzing chunk frames = [74752,75264) | 20260506 08:54:07.536 | INFO | MemInfo: { | 20260506 08:54:07.536 | INFO | "gpu" : " 17 GB", | 20260506 08:54:07.536 | INFO | "huge" : "", | 20260506 08:54:07.536 | INFO | "huge_shared" : " 27 GB", | 20260506 08:54:07.536 | INFO | "malloc" : " 15 GB", | 20260506 08:54:07.536 | INFO | "pinned" : " 29 GB" | 20260506 08:54:07.536 | INFO | } | 20260506 08:54:07.536 | INFO | Perf report for the last 5.13928 seconds: | 20260506 08:54:07.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:54:07.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:07.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:07.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:07.536 | INFO | ANALYSIS : 57.4%, 3.0, 89.1ms, 0 | 20260506 08:54:07.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:07.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:07.536 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 08:54:07.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:07.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:07.537 | INFO | Uploaded 36GiB to GPU in 1.6028s (22.4607GiB/s) | 20260506 08:54:07.537 | INFO | Downloaded 34GiB to GPU in 1.52043s (22.3621GiB/s) | 20260506 08:54:12.675 | INFO | Analyzing chunk frames = [75264,75776) | 20260506 08:54:12.675 | INFO | MemInfo: { | 20260506 08:54:12.675 | INFO | "gpu" : " 17 GB", | 20260506 08:54:12.675 | INFO | "huge" : "", | 20260506 08:54:12.675 | INFO | "huge_shared" : " 27 GB", | 20260506 08:54:12.675 | INFO | "malloc" : " 15 GB", | 20260506 08:54:12.675 | INFO | "pinned" : " 29 GB" | 20260506 08:54:12.675 | INFO | } | 20260506 08:54:12.675 | INFO | Perf report for the last 5.13929 seconds: | 20260506 08:54:12.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:54:12.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:12.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:12.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:12.675 | INFO | ANALYSIS : 57.1%, 3.0, 88.5ms, 0 | 20260506 08:54:12.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:12.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:12.675 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 08:54:12.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:12.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:12.676 | INFO | Uploaded 36GiB to GPU in 1.60209s (22.4706GiB/s) | 20260506 08:54:12.676 | INFO | Downloaded 34GiB to GPU in 1.5187s (22.3876GiB/s) | 20260506 08:54:17.815 | INFO | Analyzing chunk frames = [75776,76288) | 20260506 08:54:17.815 | INFO | MemInfo: { | 20260506 08:54:17.815 | INFO | "gpu" : " 17 GB", | 20260506 08:54:17.815 | INFO | "huge" : "", | 20260506 08:54:17.815 | INFO | "huge_shared" : " 27 GB", | 20260506 08:54:17.815 | INFO | "malloc" : " 15 GB", | 20260506 08:54:17.815 | INFO | "pinned" : " 29 GB" | 20260506 08:54:17.815 | INFO | } | 20260506 08:54:17.815 | INFO | Perf report for the last 5.13948 seconds: | 20260506 08:54:17.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:54:17.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:17.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:17.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:17.815 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 08:54:17.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:17.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:17.815 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 08:54:17.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:17.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:17.815 | INFO | Uploaded 36GiB to GPU in 1.60143s (22.4798GiB/s) | 20260506 08:54:17.815 | INFO | Downloaded 34GiB to GPU in 1.5183s (22.3935GiB/s) | 20260506 08:54:22.954 | INFO | Analyzing chunk frames = [76288,76800) | 20260506 08:54:22.954 | INFO | MemInfo: { | 20260506 08:54:22.954 | INFO | "gpu" : " 17 GB", | 20260506 08:54:22.954 | INFO | "huge" : "", | 20260506 08:54:22.954 | INFO | "huge_shared" : " 27 GB", | 20260506 08:54:22.954 | INFO | "malloc" : " 15 GB", | 20260506 08:54:22.954 | INFO | "pinned" : " 29 GB" | 20260506 08:54:22.954 | INFO | } | 20260506 08:54:22.954 | INFO | Perf report for the last 5.13919 seconds: | 20260506 08:54:22.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:54:22.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:22.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:22.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:22.954 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 08:54:22.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:22.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:22.954 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 08:54:22.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:22.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:22.954 | INFO | Uploaded 36GiB to GPU in 1.59947s (22.5074GiB/s) | 20260506 08:54:22.954 | INFO | Downloaded 34GiB to GPU in 1.51722s (22.4094GiB/s) | 20260506 08:54:28.093 | INFO | Analyzing chunk frames = [76800,77312) | 20260506 08:54:28.093 | INFO | MemInfo: { | 20260506 08:54:28.093 | INFO | "gpu" : " 17 GB", | 20260506 08:54:28.093 | INFO | "huge" : "", | 20260506 08:54:28.093 | INFO | "huge_shared" : " 27 GB", | 20260506 08:54:28.093 | INFO | "malloc" : " 15 GB", | 20260506 08:54:28.093 | INFO | "pinned" : " 29 GB" | 20260506 08:54:28.093 | INFO | } | 20260506 08:54:28.093 | INFO | Perf report for the last 5.13941 seconds: | 20260506 08:54:28.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:54:28.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:28.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:28.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:28.093 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260506 08:54:28.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:28.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:28.093 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 08:54:28.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:28.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:28.093 | INFO | Uploaded 36GiB to GPU in 1.60078s (22.4891GiB/s) | 20260506 08:54:28.094 | INFO | Downloaded 34GiB to GPU in 1.51955s (22.3751GiB/s) | 20260506 08:54:33.233 | INFO | Analyzing chunk frames = [77312,77824) | 20260506 08:54:33.233 | INFO | MemInfo: { | 20260506 08:54:33.233 | INFO | "gpu" : " 17 GB", | 20260506 08:54:33.233 | INFO | "huge" : "", | 20260506 08:54:33.233 | INFO | "huge_shared" : " 27 GB", | 20260506 08:54:33.233 | INFO | "malloc" : " 15 GB", | 20260506 08:54:33.233 | INFO | "pinned" : " 29 GB" | 20260506 08:54:33.233 | INFO | } | 20260506 08:54:33.233 | INFO | Perf report for the last 5.13951 seconds: | 20260506 08:54:33.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:54:33.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:33.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:33.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:33.233 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260506 08:54:33.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:33.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:33.233 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260506 08:54:33.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:33.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:33.233 | INFO | Uploaded 36GiB to GPU in 1.60182s (22.4745GiB/s) | 20260506 08:54:33.233 | INFO | Downloaded 34GiB to GPU in 1.51957s (22.3747GiB/s) | 20260506 08:54:33.330 | INFO | Starting a round of Baz IO | 20260506 08:54:36.270 | INFO | Populating RTMetrics report | 20260506 08:54:36.271 | INFO | RTMetrics report populated | 20260506 08:54:36.271 | INFO | Opening tmp RTMetrics json file | 20260506 08:54:36.276 | INFO | Serializing RTMetrics to json | 20260506 08:54:36.304 | INFO | Serialization done, swapping tmp file now... | 20260506 08:54:36.343 | INFO | RTMetrics done | 20260506 08:54:36.365 | INFO | Finished a round of Baz IO: Wrote 1.597 GiB in 3.035 seconds | 20260506 08:54:36.365 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 64.34% / 0.51% / 35.15% | 20260506 08:54:36.365 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.1044% / 0.0023524% / 7.94729e-06% | 20260506 08:54:36.365 | INFO | Baz IO overall: | 20260506 08:54:36.365 | INFO | BytesWritten : 13904616392 | 20260506 08:54:36.365 | INFO | NumEvents : 5291629252 | 20260506 08:54:38.372 | INFO | Analyzing chunk frames = [77824,78336) | 20260506 08:54:38.372 | INFO | MemInfo: { | 20260506 08:54:38.372 | INFO | "gpu" : " 17 GB", | 20260506 08:54:38.372 | INFO | "huge" : "", | 20260506 08:54:38.372 | INFO | "huge_shared" : " 27 GB", | 20260506 08:54:38.372 | INFO | "malloc" : " 15 GB", | 20260506 08:54:38.372 | INFO | "pinned" : " 29 GB" | 20260506 08:54:38.372 | INFO | } | 20260506 08:54:38.372 | INFO | Perf report for the last 5.13913 seconds: | 20260506 08:54:38.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:54:38.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:38.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:38.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:38.372 | INFO | ANALYSIS : 58.2%, 2.9, 90.0ms, 0 | 20260506 08:54:38.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:38.372 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 08:54:38.372 | INFO | PRE_HQ : 57.5%, 2.1, 63.5ms, 0 | 20260506 08:54:38.372 | INFO | BAZWRITER : 58.5%, 3.9, 117.9ms, 0 | 20260506 08:54:38.372 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 08:54:38.372 | INFO | Uploaded 36GiB to GPU in 1.63197s (22.0593GiB/s) | 20260506 08:54:38.372 | INFO | Downloaded 38GiB to GPU in 1.69468s (22.4231GiB/s) | 20260506 08:54:43.511 | INFO | Analyzing chunk frames = [78336,78848) | 20260506 08:54:43.511 | INFO | MemInfo: { | 20260506 08:54:43.511 | INFO | "gpu" : " 17 GB", | 20260506 08:54:43.511 | INFO | "huge" : "", | 20260506 08:54:43.511 | INFO | "huge_shared" : " 27 GB", | 20260506 08:54:43.511 | INFO | "malloc" : " 15 GB", | 20260506 08:54:43.511 | INFO | "pinned" : " 29 GB" | 20260506 08:54:43.511 | INFO | } | 20260506 08:54:43.511 | INFO | Perf report for the last 5.13935 seconds: | 20260506 08:54:43.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:54:43.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:43.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:43.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:43.511 | INFO | ANALYSIS : 58.2%, 3.0, 90.8ms, 0 | 20260506 08:54:43.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:43.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:43.511 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 08:54:43.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:43.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:43.512 | INFO | Uploaded 36GiB to GPU in 1.60655s (22.4083GiB/s) | 20260506 08:54:43.512 | INFO | Downloaded 34GiB to GPU in 1.51594s (22.4283GiB/s) | 20260506 08:54:48.651 | INFO | Analyzing chunk frames = [78848,79360) | 20260506 08:54:48.651 | INFO | MemInfo: { | 20260506 08:54:48.651 | INFO | "gpu" : " 17 GB", | 20260506 08:54:48.651 | INFO | "huge" : "", | 20260506 08:54:48.651 | INFO | "huge_shared" : " 27 GB", | 20260506 08:54:48.651 | INFO | "malloc" : " 15 GB", | 20260506 08:54:48.651 | INFO | "pinned" : " 29 GB" | 20260506 08:54:48.651 | INFO | } | 20260506 08:54:48.651 | INFO | Perf report for the last 5.1395 seconds: | 20260506 08:54:48.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:54:48.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:48.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:48.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:48.651 | INFO | ANALYSIS : 57.4%, 3.0, 89.1ms, 0 | 20260506 08:54:48.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:48.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:48.651 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 08:54:48.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:48.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:48.651 | INFO | Uploaded 36GiB to GPU in 1.60498s (22.4302GiB/s) | 20260506 08:54:48.651 | INFO | Downloaded 34GiB to GPU in 1.52246s (22.3323GiB/s) | 20260506 08:54:53.790 | INFO | Analyzing chunk frames = [79360,79872) | 20260506 08:54:53.790 | INFO | MemInfo: { | 20260506 08:54:53.790 | INFO | "gpu" : " 17 GB", | 20260506 08:54:53.790 | INFO | "huge" : "", | 20260506 08:54:53.790 | INFO | "huge_shared" : " 27 GB", | 20260506 08:54:53.790 | INFO | "malloc" : " 15 GB", | 20260506 08:54:53.790 | INFO | "pinned" : " 29 GB" | 20260506 08:54:53.790 | INFO | } | 20260506 08:54:53.790 | INFO | Perf report for the last 5.13922 seconds: | 20260506 08:54:53.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:54:53.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:53.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:53.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:53.790 | INFO | ANALYSIS : 57.0%, 3.0, 88.5ms, 0 | 20260506 08:54:53.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:53.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:53.790 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 08:54:53.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:53.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:53.790 | INFO | Uploaded 36GiB to GPU in 1.60268s (22.4624GiB/s) | 20260506 08:54:53.790 | INFO | Downloaded 34GiB to GPU in 1.51964s (22.3738GiB/s) | 20260506 08:54:58.929 | INFO | Analyzing chunk frames = [79872,80384) | 20260506 08:54:58.929 | INFO | MemInfo: { | 20260506 08:54:58.929 | INFO | "gpu" : " 17 GB", | 20260506 08:54:58.929 | INFO | "huge" : "", | 20260506 08:54:58.929 | INFO | "huge_shared" : " 27 GB", | 20260506 08:54:58.929 | INFO | "malloc" : " 15 GB", | 20260506 08:54:58.929 | INFO | "pinned" : " 29 GB" | 20260506 08:54:58.929 | INFO | } | 20260506 08:54:58.929 | INFO | Perf report for the last 5.13913 seconds: | 20260506 08:54:58.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:54:58.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:58.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:58.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:58.929 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 08:54:58.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:58.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:58.929 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 08:54:58.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:58.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:54:58.929 | INFO | Uploaded 36GiB to GPU in 1.60083s (22.4883GiB/s) | 20260506 08:54:58.929 | INFO | Downloaded 34GiB to GPU in 1.51731s (22.408GiB/s) | 20260506 08:55:04.069 | INFO | Analyzing chunk frames = [80384,80896) | 20260506 08:55:04.069 | INFO | MemInfo: { | 20260506 08:55:04.069 | INFO | "gpu" : " 17 GB", | 20260506 08:55:04.069 | INFO | "huge" : "", | 20260506 08:55:04.069 | INFO | "huge_shared" : " 27 GB", | 20260506 08:55:04.069 | INFO | "malloc" : " 15 GB", | 20260506 08:55:04.069 | INFO | "pinned" : " 29 GB" | 20260506 08:55:04.069 | INFO | } | 20260506 08:55:04.069 | INFO | Perf report for the last 5.13958 seconds: | 20260506 08:55:04.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:55:04.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:04.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:04.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:04.069 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 08:55:04.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:04.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:04.069 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 08:55:04.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:04.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:04.069 | INFO | Uploaded 36GiB to GPU in 1.60117s (22.4836GiB/s) | 20260506 08:55:04.069 | INFO | Downloaded 34GiB to GPU in 1.5179s (22.3994GiB/s) | 20260506 08:55:09.208 | INFO | Analyzing chunk frames = [80896,81408) | 20260506 08:55:09.208 | INFO | MemInfo: { | 20260506 08:55:09.208 | INFO | "gpu" : " 17 GB", | 20260506 08:55:09.208 | INFO | "huge" : "", | 20260506 08:55:09.208 | INFO | "huge_shared" : " 27 GB", | 20260506 08:55:09.208 | INFO | "malloc" : " 15 GB", | 20260506 08:55:09.208 | INFO | "pinned" : " 29 GB" | 20260506 08:55:09.208 | INFO | } | 20260506 08:55:09.208 | INFO | Perf report for the last 5.1392 seconds: | 20260506 08:55:09.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:55:09.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:09.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:09.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:09.208 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260506 08:55:09.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:09.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:09.208 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 08:55:09.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:09.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:09.208 | INFO | Uploaded 36GiB to GPU in 1.6016s (22.4775GiB/s) | 20260506 08:55:09.208 | INFO | Downloaded 34GiB to GPU in 1.51887s (22.385GiB/s) | 20260506 08:55:14.348 | INFO | Analyzing chunk frames = [81408,81920) | 20260506 08:55:14.348 | INFO | MemInfo: { | 20260506 08:55:14.348 | INFO | "gpu" : " 17 GB", | 20260506 08:55:14.348 | INFO | "huge" : "", | 20260506 08:55:14.348 | INFO | "huge_shared" : " 27 GB", | 20260506 08:55:14.348 | INFO | "malloc" : " 15 GB", | 20260506 08:55:14.348 | INFO | "pinned" : " 29 GB" | 20260506 08:55:14.348 | INFO | } | 20260506 08:55:14.348 | INFO | Perf report for the last 5.13965 seconds: | 20260506 08:55:14.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:55:14.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:14.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:14.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:14.348 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260506 08:55:14.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:14.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:14.348 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 08:55:14.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:14.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:14.348 | INFO | Uploaded 36GiB to GPU in 1.60214s (22.47GiB/s) | 20260506 08:55:14.348 | INFO | Downloaded 34GiB to GPU in 1.5194s (22.3772GiB/s) | 20260506 08:55:14.348 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 08:55:14.348 | INFO | Total mem usage: 15.5 GiB | 20260506 08:55:14.348 | INFO | All Time High Water Marks sum: 15.6 GiB (Savings: 0.781%) | 20260506 08:55:14.348 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -2.55%) | 20260506 08:55:14.348 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:55:14.348 | INFO | High Water Breakdown: | 20260506 08:55:14.348 | INFO | 58.4% : 8.84 GiB ( 98.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 08:55:14.349 | INFO | 41.5% : 6.28 GiB ( 93.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 08:55:14.349 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 08:55:14.349 | INFO | ------------------------------------------ | 20260506 08:55:14.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:55:14.349 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 08:55:14.349 | INFO | Total mem usage: 26.5 GiB | 20260506 08:55:14.349 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 08:55:14.349 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 08:55:14.349 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 08:55:14.349 | INFO | High Water Breakdown: | 20260506 08:55:14.349 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:55:14.349 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:55:14.349 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:55:14.349 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:55:14.349 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:55:14.349 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:55:14.349 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 08:55:14.349 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 08:55:14.349 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:55:14.349 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 08:55:14.349 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 08:55:14.349 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 08:55:14.349 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 08:55:14.349 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 08:55:14.349 | INFO | ------------------------------------------ | 20260506 08:55:14.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:55:14.349 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 08:55:14.349 | INFO | Total mem usage: 25.6 GiB | 20260506 08:55:14.349 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 08:55:14.349 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 08:55:14.349 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 08:55:14.349 | INFO | High Water Breakdown: | 20260506 08:55:14.349 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 08:55:14.349 | INFO | ------------------------------------------ | 20260506 08:55:14.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:55:14.349 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 08:55:14.349 | INFO | Total mem usage: 15.7 GiB | 20260506 08:55:14.349 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 08:55:14.349 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 08:55:14.349 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 08:55:14.349 | INFO | High Water Breakdown: | 20260506 08:55:14.349 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 08:55:14.349 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 08:55:14.349 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 08:55:14.349 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 08:55:14.349 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 08:55:14.349 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 08:55:14.349 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 08:55:14.349 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 08:55:14.349 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 08:55:14.349 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:55:14.349 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 08:55:14.349 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:55:14.349 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 08:55:14.349 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 08:55:14.349 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 08:55:14.349 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 08:55:14.349 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:55:14.349 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 08:55:14.349 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:55:14.349 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:55:14.349 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:55:14.349 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 08:55:14.349 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 08:55:14.349 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:55:14.349 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 08:55:14.349 | INFO | ------------------------------------------ | 20260506 08:55:14.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:55:14.349 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 08:55:14.349 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:55:14.349 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:55:14.349 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260506 08:55:14.349 | INFO | PerfReport: 0.28%, AggregateStats -- 8.61e-02 / 0.00e+00, 4.33e-03 / 0.00e+00 | 20260506 08:55:14.349 | INFO | PerfReport: 10.81%, Baseline -- 3.29e+00 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260506 08:55:14.349 | INFO | PerfReport: 3.03%, Histogram -- 9.23e-01 / 0.00e+00, 5.29e-02 / 0.00e+00 | 20260506 08:55:14.349 | INFO | PerfReport: 15.99%, DME -- 3.90e+01 / 0.00e+00, 3.33e+00 / 0.00e+00 | 20260506 08:55:14.349 | INFO | PerfReport: 56.52%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.24e-02 / 0.00e+00 | 20260506 08:55:14.349 | INFO | PerfReport: 7.68%, PulseAccumulating -- 2.34e+00 / 0.00e+00, 1.43e-01 / 0.00e+00 | 20260506 08:55:14.349 | INFO | PerfReport: 4.07%, Metrics -- 1.24e+00 / 0.00e+00, 3.33e-01 / 0.00e+00 | 20260506 08:55:14.349 | INFO | PerfReport: 1.60%, PulseSerializing -- 4.86e-01 / 0.00e+00, 7.02e-02 / 0.00e+00 | 20260506 08:55:14.350 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 08:55:14.350 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:55:14.350 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:55:14.350 | INFO | PerfReport: Full Execution -- 9.13e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260506 08:55:14.350 | INFO | PerfReport: 18.33%, Upload -- 1.67e+01 / 0.00e+00, 4.27e-01 / 0.00e+00 | 20260506 08:55:14.350 | INFO | PerfReport: 17.59%, Download -- 1.61e+01 / 0.00e+00, 6.99e-01 / 0.00e+00 | 20260506 08:55:14.350 | INFO | PerfReport: 33.35%, Compute -- 3.04e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260506 08:55:14.350 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 08:55:14.350 | INFO | PerfReport: 0.00%, LockDownload -- 5.03e-04 / 0.00e+00, 1.95e-04 / 0.00e+00 | 20260506 08:55:14.350 | INFO | PerfReport: 30.16%, LockCompute -- 2.75e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 08:55:14.350 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.27e-02 / 0.00e+00 | 20260506 08:55:14.445 | INFO | Starting a round of Baz IO | 20260506 08:55:17.383 | INFO | Populating RTMetrics report | 20260506 08:55:17.384 | INFO | RTMetrics report populated | 20260506 08:55:17.384 | INFO | Opening tmp RTMetrics json file | 20260506 08:55:17.405 | INFO | Serializing RTMetrics to json | 20260506 08:55:17.410 | INFO | Serialization done, swapping tmp file now... | 20260506 08:55:17.412 | INFO | RTMetrics done | 20260506 08:55:17.533 | INFO | Finished a round of Baz IO: Wrote 1.604 GiB in 3.087 seconds | 20260506 08:55:17.533 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 63.86% / 0.48% / 35.66% | 20260506 08:55:17.533 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 12.0685% / 0.00231663% / 3.97364e-06% | 20260506 08:55:17.533 | INFO | Baz IO overall: | 20260506 08:55:17.533 | INFO | BytesWritten : 15627206749 | 20260506 08:55:17.533 | INFO | NumEvents : 6128249341 | 20260506 08:55:19.487 | INFO | Analyzing chunk frames = [81920,82432) | 20260506 08:55:19.487 | INFO | MemInfo: { | 20260506 08:55:19.487 | INFO | "gpu" : " 17 GB", | 20260506 08:55:19.487 | INFO | "huge" : "", | 20260506 08:55:19.487 | INFO | "huge_shared" : " 27 GB", | 20260506 08:55:19.487 | INFO | "malloc" : " 15 GB", | 20260506 08:55:19.487 | INFO | "pinned" : " 29 GB" | 20260506 08:55:19.487 | INFO | } | 20260506 08:55:19.487 | INFO | Perf report for the last 5.13902 seconds: | 20260506 08:55:19.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:55:19.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:19.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:19.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:19.487 | INFO | ANALYSIS : 58.2%, 3.0, 90.1ms, 0 | 20260506 08:55:19.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:19.487 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 08:55:19.487 | INFO | PRE_HQ : 57.4%, 2.1, 63.4ms, 0 | 20260506 08:55:19.487 | INFO | BAZWRITER : 59.3%, 3.9, 117.5ms, 0 | 20260506 08:55:19.487 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 08:55:19.487 | INFO | Uploaded 36GiB to GPU in 1.63926s (21.9611GiB/s) | 20260506 08:55:19.487 | INFO | Downloaded 38GiB to GPU in 1.70118s (22.3375GiB/s) | 20260506 08:55:24.626 | INFO | Analyzing chunk frames = [82432,82944) | 20260506 08:55:24.626 | INFO | MemInfo: { | 20260506 08:55:24.626 | INFO | "gpu" : " 17 GB", | 20260506 08:55:24.626 | INFO | "huge" : "", | 20260506 08:55:24.626 | INFO | "huge_shared" : " 27 GB", | 20260506 08:55:24.626 | INFO | "malloc" : " 15 GB", | 20260506 08:55:24.626 | INFO | "pinned" : " 29 GB" | 20260506 08:55:24.626 | INFO | } | 20260506 08:55:24.626 | INFO | Perf report for the last 5.13948 seconds: | 20260506 08:55:24.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:55:24.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:24.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:24.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:24.626 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260506 08:55:24.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:24.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:24.626 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260506 08:55:24.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:24.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:24.627 | INFO | Uploaded 36GiB to GPU in 1.59954s (22.5065GiB/s) | 20260506 08:55:24.627 | INFO | Downloaded 34GiB to GPU in 1.51592s (22.4286GiB/s) | 20260506 08:55:29.765 | INFO | Analyzing chunk frames = [82944,83456) | 20260506 08:55:29.765 | INFO | MemInfo: { | 20260506 08:55:29.765 | INFO | "gpu" : " 17 GB", | 20260506 08:55:29.765 | INFO | "huge" : "", | 20260506 08:55:29.765 | INFO | "huge_shared" : " 27 GB", | 20260506 08:55:29.765 | INFO | "malloc" : " 15 GB", | 20260506 08:55:29.765 | INFO | "pinned" : " 29 GB" | 20260506 08:55:29.765 | INFO | } | 20260506 08:55:29.766 | INFO | Perf report for the last 5.13922 seconds: | 20260506 08:55:29.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:55:29.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:29.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:29.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:29.766 | INFO | ANALYSIS : 57.4%, 3.0, 89.1ms, 0 | 20260506 08:55:29.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:29.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:29.766 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 08:55:29.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:29.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:29.766 | INFO | Uploaded 36GiB to GPU in 1.60353s (22.4504GiB/s) | 20260506 08:55:29.766 | INFO | Downloaded 34GiB to GPU in 1.52078s (22.357GiB/s) | 20260506 08:55:34.905 | INFO | Analyzing chunk frames = [83456,83968) | 20260506 08:55:34.905 | INFO | MemInfo: { | 20260506 08:55:34.905 | INFO | "gpu" : " 17 GB", | 20260506 08:55:34.905 | INFO | "huge" : "", | 20260506 08:55:34.905 | INFO | "huge_shared" : " 27 GB", | 20260506 08:55:34.905 | INFO | "malloc" : " 15 GB", | 20260506 08:55:34.905 | INFO | "pinned" : " 29 GB" | 20260506 08:55:34.905 | INFO | } | 20260506 08:55:34.905 | INFO | Perf report for the last 5.13953 seconds: | 20260506 08:55:34.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:55:34.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:34.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:34.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:34.905 | INFO | ANALYSIS : 57.0%, 3.0, 88.5ms, 0 | 20260506 08:55:34.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:34.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:34.905 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 08:55:34.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:34.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:34.905 | INFO | Uploaded 36GiB to GPU in 1.60283s (22.4603GiB/s) | 20260506 08:55:34.905 | INFO | Downloaded 34GiB to GPU in 1.52036s (22.3631GiB/s) | 20260506 08:55:40.044 | INFO | Analyzing chunk frames = [83968,84480) | 20260506 08:55:40.044 | INFO | MemInfo: { | 20260506 08:55:40.044 | INFO | "gpu" : " 17 GB", | 20260506 08:55:40.044 | INFO | "huge" : "", | 20260506 08:55:40.044 | INFO | "huge_shared" : " 27 GB", | 20260506 08:55:40.044 | INFO | "malloc" : " 15 GB", | 20260506 08:55:40.044 | INFO | "pinned" : " 29 GB" | 20260506 08:55:40.044 | INFO | } | 20260506 08:55:40.044 | INFO | Perf report for the last 5.13917 seconds: | 20260506 08:55:40.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:55:40.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:40.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:40.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:40.044 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 08:55:40.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:40.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:40.044 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 08:55:40.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:40.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:40.044 | INFO | Uploaded 36GiB to GPU in 1.60109s (22.4847GiB/s) | 20260506 08:55:40.045 | INFO | Downloaded 34GiB to GPU in 1.51769s (22.4024GiB/s) | 20260506 08:55:45.183 | INFO | Analyzing chunk frames = [84480,84992) | 20260506 08:55:45.183 | INFO | MemInfo: { | 20260506 08:55:45.183 | INFO | "gpu" : " 17 GB", | 20260506 08:55:45.183 | INFO | "huge" : "", | 20260506 08:55:45.183 | INFO | "huge_shared" : " 27 GB", | 20260506 08:55:45.183 | INFO | "malloc" : " 15 GB", | 20260506 08:55:45.183 | INFO | "pinned" : " 29 GB" | 20260506 08:55:45.183 | INFO | } | 20260506 08:55:45.183 | INFO | Perf report for the last 5.13921 seconds: | 20260506 08:55:45.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:55:45.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:45.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:45.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:45.183 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 08:55:45.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:45.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:45.184 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 08:55:45.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:45.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:45.184 | INFO | Uploaded 36GiB to GPU in 1.6013s (22.4817GiB/s) | 20260506 08:55:45.184 | INFO | Downloaded 34GiB to GPU in 1.51797s (22.3984GiB/s) | 20260506 08:55:50.323 | INFO | Analyzing chunk frames = [84992,85504) | 20260506 08:55:50.323 | INFO | MemInfo: { | 20260506 08:55:50.323 | INFO | "gpu" : " 17 GB", | 20260506 08:55:50.323 | INFO | "huge" : "", | 20260506 08:55:50.323 | INFO | "huge_shared" : " 27 GB", | 20260506 08:55:50.323 | INFO | "malloc" : " 15 GB", | 20260506 08:55:50.323 | INFO | "pinned" : " 29 GB" | 20260506 08:55:50.323 | INFO | } | 20260506 08:55:50.323 | INFO | Perf report for the last 5.13934 seconds: | 20260506 08:55:50.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:55:50.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:50.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:50.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:50.323 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 08:55:50.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:50.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:50.323 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 08:55:50.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:50.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:50.323 | INFO | Uploaded 36GiB to GPU in 1.60208s (22.4708GiB/s) | 20260506 08:55:50.323 | INFO | Downloaded 34GiB to GPU in 1.51915s (22.3809GiB/s) | 20260506 08:55:55.462 | INFO | Analyzing chunk frames = [85504,86016) | 20260506 08:55:55.462 | INFO | MemInfo: { | 20260506 08:55:55.462 | INFO | "gpu" : " 17 GB", | 20260506 08:55:55.462 | INFO | "huge" : "", | 20260506 08:55:55.462 | INFO | "huge_shared" : " 27 GB", | 20260506 08:55:55.462 | INFO | "malloc" : " 15 GB", | 20260506 08:55:55.462 | INFO | "pinned" : " 29 GB" | 20260506 08:55:55.462 | INFO | } | 20260506 08:55:55.462 | INFO | Perf report for the last 5.13929 seconds: | 20260506 08:55:55.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:55:55.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:55.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:55.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:55.462 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 08:55:55.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:55.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:55.462 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 08:55:55.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:55.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:55:55.462 | INFO | Uploaded 36GiB to GPU in 1.60225s (22.4684GiB/s) | 20260506 08:55:55.462 | INFO | Downloaded 34GiB to GPU in 1.51964s (22.3737GiB/s) | 20260506 08:55:55.559 | INFO | Starting a round of Baz IO | 20260506 08:55:58.499 | INFO | Populating RTMetrics report | 20260506 08:55:58.499 | INFO | RTMetrics report populated | 20260506 08:55:58.499 | INFO | Opening tmp RTMetrics json file | 20260506 08:55:58.501 | INFO | Serializing RTMetrics to json | 20260506 08:55:58.517 | INFO | Serialization done, swapping tmp file now... | 20260506 08:55:58.522 | INFO | RTMetrics done | 20260506 08:55:58.564 | INFO | Finished a round of Baz IO: Wrote 1.608 GiB in 3.004 seconds | 20260506 08:55:58.564 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 63.39% / 0.47% / 36.14% | 20260506 08:55:58.564 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.9684% / 0.00194709% / 7.94729e-06% | 20260506 08:55:58.564 | INFO | Baz IO overall: | 20260506 08:55:58.564 | INFO | BytesWritten : 17353247403 | 20260506 08:55:58.564 | INFO | NumEvents : 6968896385 | 20260506 08:56:00.601 | INFO | Analyzing chunk frames = [86016,86528) | 20260506 08:56:00.601 | INFO | MemInfo: { | 20260506 08:56:00.601 | INFO | "gpu" : " 17 GB", | 20260506 08:56:00.601 | INFO | "huge" : "", | 20260506 08:56:00.601 | INFO | "huge_shared" : " 27 GB", | 20260506 08:56:00.601 | INFO | "malloc" : " 15 GB", | 20260506 08:56:00.601 | INFO | "pinned" : " 29 GB" | 20260506 08:56:00.601 | INFO | } | 20260506 08:56:00.602 | INFO | Perf report for the last 5.13941 seconds: | 20260506 08:56:00.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:56:00.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:00.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:00.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:00.602 | INFO | ANALYSIS : 58.2%, 3.0, 90.6ms, 0 | 20260506 08:56:00.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:00.602 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 08:56:00.602 | INFO | PRE_HQ : 57.5%, 2.1, 62.8ms, 0 | 20260506 08:56:00.602 | INFO | BAZWRITER : 58.0%, 3.7, 109.6ms, 0 | 20260506 08:56:00.602 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260506 08:56:00.602 | INFO | Uploaded 36GiB to GPU in 1.64142s (21.9322GiB/s) | 20260506 08:56:00.602 | INFO | Downloaded 38GiB to GPU in 1.70419s (22.298GiB/s) | 20260506 08:56:05.741 | INFO | Analyzing chunk frames = [86528,87040) | 20260506 08:56:05.741 | INFO | MemInfo: { | 20260506 08:56:05.741 | INFO | "gpu" : " 17 GB", | 20260506 08:56:05.741 | INFO | "huge" : "", | 20260506 08:56:05.741 | INFO | "huge_shared" : " 27 GB", | 20260506 08:56:05.741 | INFO | "malloc" : " 15 GB", | 20260506 08:56:05.741 | INFO | "pinned" : " 29 GB" | 20260506 08:56:05.741 | INFO | } | 20260506 08:56:05.741 | INFO | Perf report for the last 5.13944 seconds: | 20260506 08:56:05.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:56:05.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:05.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:05.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:05.741 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260506 08:56:05.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:05.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:05.741 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 08:56:05.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:05.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:05.741 | INFO | Uploaded 36GiB to GPU in 1.60015s (22.4979GiB/s) | 20260506 08:56:05.741 | INFO | Downloaded 34GiB to GPU in 1.517s (22.4126GiB/s) | 20260506 08:56:10.880 | INFO | Analyzing chunk frames = [87040,87552) | 20260506 08:56:10.880 | INFO | MemInfo: { | 20260506 08:56:10.880 | INFO | "gpu" : " 17 GB", | 20260506 08:56:10.880 | INFO | "huge" : "", | 20260506 08:56:10.880 | INFO | "huge_shared" : " 27 GB", | 20260506 08:56:10.880 | INFO | "malloc" : " 15 GB", | 20260506 08:56:10.880 | INFO | "pinned" : " 29 GB" | 20260506 08:56:10.880 | INFO | } | 20260506 08:56:10.880 | INFO | Perf report for the last 5.13925 seconds: | 20260506 08:56:10.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:56:10.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:10.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:10.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:10.880 | INFO | ANALYSIS : 57.3%, 3.0, 89.1ms, 0 | 20260506 08:56:10.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:10.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:10.880 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 08:56:10.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:10.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:10.880 | INFO | Uploaded 36GiB to GPU in 1.60319s (22.4553GiB/s) | 20260506 08:56:10.881 | INFO | Downloaded 34GiB to GPU in 1.52051s (22.361GiB/s) | 20260506 08:56:16.019 | INFO | Analyzing chunk frames = [87552,88064) | 20260506 08:56:16.019 | INFO | MemInfo: { | 20260506 08:56:16.020 | INFO | "gpu" : " 17 GB", | 20260506 08:56:16.020 | INFO | "huge" : "", | 20260506 08:56:16.020 | INFO | "huge_shared" : " 27 GB", | 20260506 08:56:16.020 | INFO | "malloc" : " 15 GB", | 20260506 08:56:16.020 | INFO | "pinned" : " 29 GB" | 20260506 08:56:16.020 | INFO | } | 20260506 08:56:16.020 | INFO | Perf report for the last 5.13939 seconds: | 20260506 08:56:16.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:56:16.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:16.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:16.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:16.020 | INFO | ANALYSIS : 57.0%, 3.0, 88.3ms, 0 | 20260506 08:56:16.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:16.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:16.020 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 08:56:16.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:16.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:16.020 | INFO | Uploaded 36GiB to GPU in 1.60328s (22.4539GiB/s) | 20260506 08:56:16.020 | INFO | Downloaded 34GiB to GPU in 1.52055s (22.3603GiB/s) | 20260506 08:56:21.159 | INFO | Analyzing chunk frames = [88064,88576) | 20260506 08:56:21.159 | INFO | MemInfo: { | 20260506 08:56:21.159 | INFO | "gpu" : " 17 GB", | 20260506 08:56:21.159 | INFO | "huge" : "", | 20260506 08:56:21.159 | INFO | "huge_shared" : " 27 GB", | 20260506 08:56:21.159 | INFO | "malloc" : " 15 GB", | 20260506 08:56:21.159 | INFO | "pinned" : " 29 GB" | 20260506 08:56:21.159 | INFO | } | 20260506 08:56:21.159 | INFO | Perf report for the last 5.13931 seconds: | 20260506 08:56:21.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:56:21.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:21.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:21.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:21.159 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 08:56:21.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:21.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:21.159 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 08:56:21.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:21.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:21.159 | INFO | Uploaded 36GiB to GPU in 1.60086s (22.4878GiB/s) | 20260506 08:56:21.159 | INFO | Downloaded 34GiB to GPU in 1.51741s (22.4066GiB/s) | 20260506 08:56:26.298 | INFO | Analyzing chunk frames = [88576,89088) | 20260506 08:56:26.298 | INFO | MemInfo: { | 20260506 08:56:26.298 | INFO | "gpu" : " 17 GB", | 20260506 08:56:26.298 | INFO | "huge" : "", | 20260506 08:56:26.298 | INFO | "huge_shared" : " 27 GB", | 20260506 08:56:26.298 | INFO | "malloc" : " 15 GB", | 20260506 08:56:26.298 | INFO | "pinned" : " 29 GB" | 20260506 08:56:26.298 | INFO | } | 20260506 08:56:26.298 | INFO | Perf report for the last 5.13931 seconds: | 20260506 08:56:26.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:56:26.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:26.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:26.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:26.298 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 08:56:26.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:26.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:26.298 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260506 08:56:26.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:26.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:26.298 | INFO | Uploaded 36GiB to GPU in 1.60134s (22.4812GiB/s) | 20260506 08:56:26.299 | INFO | Downloaded 34GiB to GPU in 1.51774s (22.4017GiB/s) | 20260506 08:56:31.438 | INFO | Analyzing chunk frames = [89088,89600) | 20260506 08:56:31.438 | INFO | MemInfo: { | 20260506 08:56:31.438 | INFO | "gpu" : " 17 GB", | 20260506 08:56:31.438 | INFO | "huge" : "", | 20260506 08:56:31.438 | INFO | "huge_shared" : " 27 GB", | 20260506 08:56:31.438 | INFO | "malloc" : " 15 GB", | 20260506 08:56:31.438 | INFO | "pinned" : " 29 GB" | 20260506 08:56:31.438 | INFO | } | 20260506 08:56:31.438 | INFO | Perf report for the last 5.13941 seconds: | 20260506 08:56:31.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:56:31.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:31.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:31.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:31.438 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 08:56:31.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:31.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:31.438 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 08:56:31.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:31.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:31.438 | INFO | Uploaded 36GiB to GPU in 1.60224s (22.4685GiB/s) | 20260506 08:56:31.438 | INFO | Downloaded 34GiB to GPU in 1.51963s (22.3739GiB/s) | 20260506 08:56:36.577 | INFO | Analyzing chunk frames = [89600,90112) | 20260506 08:56:36.577 | INFO | MemInfo: { | 20260506 08:56:36.577 | INFO | "gpu" : " 17 GB", | 20260506 08:56:36.577 | INFO | "huge" : "", | 20260506 08:56:36.577 | INFO | "huge_shared" : " 27 GB", | 20260506 08:56:36.577 | INFO | "malloc" : " 15 GB", | 20260506 08:56:36.577 | INFO | "pinned" : " 29 GB" | 20260506 08:56:36.577 | INFO | } | 20260506 08:56:36.577 | INFO | Perf report for the last 5.13921 seconds: | 20260506 08:56:36.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:56:36.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:36.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:36.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:36.577 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260506 08:56:36.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:36.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:36.577 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 08:56:36.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:36.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:36.577 | INFO | Uploaded 36GiB to GPU in 1.60283s (22.4602GiB/s) | 20260506 08:56:36.577 | INFO | Downloaded 34GiB to GPU in 1.51956s (22.3749GiB/s) | 20260506 08:56:36.577 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 08:56:36.577 | INFO | Total mem usage: 15.5 GiB | 20260506 08:56:36.577 | INFO | All Time High Water Marks sum: 15.6 GiB (Savings: 0.781%) | 20260506 08:56:36.577 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -4.28%) | 20260506 08:56:36.577 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:56:36.577 | INFO | High Water Breakdown: | 20260506 08:56:36.577 | INFO | 58.2% : 8.67 GiB ( 97% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 08:56:36.577 | INFO | 41.7% : 6.2 GiB ( 92.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 08:56:36.577 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 08:56:36.577 | INFO | ------------------------------------------ | 20260506 08:56:36.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:56:36.577 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 08:56:36.577 | INFO | Total mem usage: 26.5 GiB | 20260506 08:56:36.577 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 08:56:36.577 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 08:56:36.577 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 08:56:36.577 | INFO | High Water Breakdown: | 20260506 08:56:36.578 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:56:36.578 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:56:36.578 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:56:36.578 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:56:36.578 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:56:36.578 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:56:36.578 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 08:56:36.578 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 08:56:36.578 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:56:36.578 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 08:56:36.578 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 08:56:36.578 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 08:56:36.578 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 08:56:36.578 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 08:56:36.578 | INFO | ------------------------------------------ | 20260506 08:56:36.578 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:56:36.578 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 08:56:36.578 | INFO | Total mem usage: 25.6 GiB | 20260506 08:56:36.578 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 08:56:36.578 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 08:56:36.578 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 08:56:36.578 | INFO | High Water Breakdown: | 20260506 08:56:36.578 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 08:56:36.578 | INFO | ------------------------------------------ | 20260506 08:56:36.578 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:56:36.578 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 08:56:36.578 | INFO | Total mem usage: 15.7 GiB | 20260506 08:56:36.578 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 08:56:36.578 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 08:56:36.578 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 08:56:36.578 | INFO | High Water Breakdown: | 20260506 08:56:36.578 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 08:56:36.578 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 08:56:36.578 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 08:56:36.578 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 08:56:36.578 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 08:56:36.578 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 08:56:36.578 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 08:56:36.578 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 08:56:36.578 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 08:56:36.578 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:56:36.578 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 08:56:36.578 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:56:36.578 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 08:56:36.578 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 08:56:36.578 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 08:56:36.578 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 08:56:36.578 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:56:36.578 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 08:56:36.578 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:56:36.578 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:56:36.578 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:56:36.578 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 08:56:36.578 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 08:56:36.578 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:56:36.578 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 08:56:36.578 | INFO | ------------------------------------------ | 20260506 08:56:36.578 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:56:36.578 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 08:56:36.578 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:56:36.578 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:56:36.578 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260506 08:56:36.578 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / 0.00e+00, 6.08e-03 / 0.00e+00 | 20260506 08:56:36.578 | INFO | PerfReport: 10.83%, Baseline -- 3.29e+00 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260506 08:56:36.578 | INFO | PerfReport: 3.04%, Histogram -- 9.23e-01 / 0.00e+00, 5.06e-02 / 0.00e+00 | 20260506 08:56:36.578 | INFO | PerfReport: 15.98%, DME -- 3.89e+01 / 0.00e+00, 3.22e+00 / 0.00e+00 | 20260506 08:56:36.578 | INFO | PerfReport: 56.59%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.12e-02 / 0.00e+00 | 20260506 08:56:36.578 | INFO | PerfReport: 7.67%, PulseAccumulating -- 2.33e+00 / 0.00e+00, 1.43e-01 / 0.00e+00 | 20260506 08:56:36.578 | INFO | PerfReport: 4.03%, Metrics -- 1.23e+00 / 0.00e+00, 1.92e-01 / 0.00e+00 | 20260506 08:56:36.578 | INFO | PerfReport: 1.58%, PulseSerializing -- 4.82e-01 / 0.00e+00, 7.06e-02 / 0.00e+00 | 20260506 08:56:36.578 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 08:56:36.578 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:56:36.578 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:56:36.578 | INFO | PerfReport: Full Execution -- 9.13e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260506 08:56:36.578 | INFO | PerfReport: 18.33%, Upload -- 1.67e+01 / 0.00e+00, 3.80e-01 / 0.00e+00 | 20260506 08:56:36.578 | INFO | PerfReport: 17.59%, Download -- 1.61e+01 / 0.00e+00, 6.98e-01 / 0.00e+00 | 20260506 08:56:36.578 | INFO | PerfReport: 33.31%, Compute -- 3.04e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260506 08:56:36.578 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260506 08:56:36.578 | INFO | PerfReport: 0.00%, LockDownload -- 5.09e-04 / 0.00e+00, 2.31e-04 / 0.00e+00 | 20260506 08:56:36.578 | INFO | PerfReport: 30.20%, LockCompute -- 2.76e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 08:56:36.578 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260506 08:56:36.674 | INFO | Starting a round of Baz IO | 20260506 08:56:39.620 | INFO | Populating RTMetrics report | 20260506 08:56:39.620 | INFO | RTMetrics report populated | 20260506 08:56:39.620 | INFO | Opening tmp RTMetrics json file | 20260506 08:56:39.643 | INFO | Serializing RTMetrics to json | 20260506 08:56:39.669 | INFO | Serialization done, swapping tmp file now... | 20260506 08:56:39.672 | INFO | RTMetrics done | 20260506 08:56:39.693 | INFO | Finished a round of Baz IO: Wrote 1.614 GiB in 3.019 seconds | 20260506 08:56:39.693 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 62.93% / 0.46% / 36.61% | 20260506 08:56:39.693 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.8448% / 0.00188748% / 1.19209e-05% | 20260506 08:56:39.693 | INFO | Baz IO overall: | 20260506 08:56:39.693 | INFO | BytesWritten : 19085874379 | 20260506 08:56:39.693 | INFO | NumEvents : 7813850214 | 20260506 08:56:41.716 | INFO | Analyzing chunk frames = [90112,90624) | 20260506 08:56:41.716 | INFO | MemInfo: { | 20260506 08:56:41.716 | INFO | "gpu" : " 17 GB", | 20260506 08:56:41.716 | INFO | "huge" : "", | 20260506 08:56:41.716 | INFO | "huge_shared" : " 27 GB", | 20260506 08:56:41.716 | INFO | "malloc" : " 15 GB", | 20260506 08:56:41.716 | INFO | "pinned" : " 29 GB" | 20260506 08:56:41.716 | INFO | } | 20260506 08:56:41.716 | INFO | Perf report for the last 5.13941 seconds: | 20260506 08:56:41.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:56:41.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:41.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:41.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:41.716 | INFO | ANALYSIS : 58.3%, 3.0, 90.6ms, 0 | 20260506 08:56:41.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:41.716 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 08:56:41.716 | INFO | PRE_HQ : 57.4%, 2.1, 62.3ms, 0 | 20260506 08:56:41.716 | INFO | BAZWRITER : 58.4%, 3.8, 113.6ms, 0 | 20260506 08:56:41.716 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 08:56:41.717 | INFO | Uploaded 36GiB to GPU in 1.64311s (21.9097GiB/s) | 20260506 08:56:41.717 | INFO | Downloaded 38GiB to GPU in 1.70449s (22.2941GiB/s) | 20260506 08:56:46.856 | INFO | Analyzing chunk frames = [90624,91136) | 20260506 08:56:46.856 | INFO | MemInfo: { | 20260506 08:56:46.856 | INFO | "gpu" : " 17 GB", | 20260506 08:56:46.856 | INFO | "huge" : "", | 20260506 08:56:46.856 | INFO | "huge_shared" : " 27 GB", | 20260506 08:56:46.856 | INFO | "malloc" : " 15 GB", | 20260506 08:56:46.856 | INFO | "pinned" : " 29 GB" | 20260506 08:56:46.856 | INFO | } | 20260506 08:56:46.856 | INFO | Perf report for the last 5.13947 seconds: | 20260506 08:56:46.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:56:46.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:46.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:46.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:46.856 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260506 08:56:46.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:46.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:46.856 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 08:56:46.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:46.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:46.856 | INFO | Uploaded 36GiB to GPU in 1.60033s (22.4954GiB/s) | 20260506 08:56:46.856 | INFO | Downloaded 34GiB to GPU in 1.51705s (22.4119GiB/s) | 20260506 08:56:51.995 | INFO | Analyzing chunk frames = [91136,91648) | 20260506 08:56:51.995 | INFO | MemInfo: { | 20260506 08:56:51.995 | INFO | "gpu" : " 17 GB", | 20260506 08:56:51.995 | INFO | "huge" : "", | 20260506 08:56:51.995 | INFO | "huge_shared" : " 27 GB", | 20260506 08:56:51.995 | INFO | "malloc" : " 15 GB", | 20260506 08:56:51.995 | INFO | "pinned" : " 29 GB" | 20260506 08:56:51.995 | INFO | } | 20260506 08:56:51.995 | INFO | Perf report for the last 5.13931 seconds: | 20260506 08:56:51.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:56:51.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:51.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:51.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:51.995 | INFO | ANALYSIS : 57.3%, 3.0, 89.1ms, 0 | 20260506 08:56:51.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:51.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:51.995 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 08:56:51.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:51.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:51.996 | INFO | Uploaded 36GiB to GPU in 1.60387s (22.4458GiB/s) | 20260506 08:56:51.996 | INFO | Downloaded 34GiB to GPU in 1.52125s (22.3501GiB/s) | 20260506 08:56:57.134 | INFO | Analyzing chunk frames = [91648,92160) | 20260506 08:56:57.134 | INFO | MemInfo: { | 20260506 08:56:57.134 | INFO | "gpu" : " 17 GB", | 20260506 08:56:57.134 | INFO | "huge" : "", | 20260506 08:56:57.134 | INFO | "huge_shared" : " 27 GB", | 20260506 08:56:57.134 | INFO | "malloc" : " 15 GB", | 20260506 08:56:57.134 | INFO | "pinned" : " 29 GB" | 20260506 08:56:57.134 | INFO | } | 20260506 08:56:57.134 | INFO | Perf report for the last 5.13921 seconds: | 20260506 08:56:57.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:56:57.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:57.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:57.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:57.135 | INFO | ANALYSIS : 57.0%, 3.0, 88.3ms, 0 | 20260506 08:56:57.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:57.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:57.135 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 08:56:57.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:57.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:56:57.135 | INFO | Uploaded 36GiB to GPU in 1.60348s (22.4512GiB/s) | 20260506 08:56:57.135 | INFO | Downloaded 34GiB to GPU in 1.52043s (22.3621GiB/s) | 20260506 08:57:02.274 | INFO | Analyzing chunk frames = [92160,92672) | 20260506 08:57:02.274 | INFO | MemInfo: { | 20260506 08:57:02.274 | INFO | "gpu" : " 17 GB", | 20260506 08:57:02.274 | INFO | "huge" : "", | 20260506 08:57:02.274 | INFO | "huge_shared" : " 27 GB", | 20260506 08:57:02.274 | INFO | "malloc" : " 15 GB", | 20260506 08:57:02.274 | INFO | "pinned" : " 29 GB" | 20260506 08:57:02.274 | INFO | } | 20260506 08:57:02.274 | INFO | Perf report for the last 5.13954 seconds: | 20260506 08:57:02.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:57:02.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:02.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:02.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:02.274 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 08:57:02.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:02.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:02.274 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 08:57:02.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:02.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:02.275 | INFO | Uploaded 36GiB to GPU in 1.60109s (22.4846GiB/s) | 20260506 08:57:02.275 | INFO | Downloaded 34GiB to GPU in 1.51781s (22.4007GiB/s) | 20260506 08:57:07.413 | INFO | Analyzing chunk frames = [92672,93184) | 20260506 08:57:07.413 | INFO | MemInfo: { | 20260506 08:57:07.413 | INFO | "gpu" : " 17 GB", | 20260506 08:57:07.413 | INFO | "huge" : "", | 20260506 08:57:07.413 | INFO | "huge_shared" : " 27 GB", | 20260506 08:57:07.413 | INFO | "malloc" : " 15 GB", | 20260506 08:57:07.413 | INFO | "pinned" : " 29 GB" | 20260506 08:57:07.413 | INFO | } | 20260506 08:57:07.413 | INFO | Perf report for the last 5.1391 seconds: | 20260506 08:57:07.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:57:07.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:07.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:07.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:07.413 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 08:57:07.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:07.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:07.413 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 08:57:07.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:07.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:07.413 | INFO | Uploaded 36GiB to GPU in 1.60081s (22.4886GiB/s) | 20260506 08:57:07.413 | INFO | Downloaded 34GiB to GPU in 1.51853s (22.3901GiB/s) | 20260506 08:57:12.552 | INFO | Analyzing chunk frames = [93184,93696) | 20260506 08:57:12.552 | INFO | MemInfo: { | 20260506 08:57:12.552 | INFO | "gpu" : " 17 GB", | 20260506 08:57:12.552 | INFO | "huge" : "", | 20260506 08:57:12.552 | INFO | "huge_shared" : " 27 GB", | 20260506 08:57:12.552 | INFO | "malloc" : " 15 GB", | 20260506 08:57:12.552 | INFO | "pinned" : " 29 GB" | 20260506 08:57:12.552 | INFO | } | 20260506 08:57:12.552 | INFO | Perf report for the last 5.13933 seconds: | 20260506 08:57:12.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:57:12.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:12.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:12.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:12.552 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 08:57:12.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:12.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:12.553 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 08:57:12.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:12.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:12.553 | INFO | Uploaded 36GiB to GPU in 1.60186s (22.4738GiB/s) | 20260506 08:57:12.553 | INFO | Downloaded 34GiB to GPU in 1.51946s (22.3764GiB/s) | 20260506 08:57:17.692 | INFO | Analyzing chunk frames = [93696,94208) | 20260506 08:57:17.692 | INFO | MemInfo: { | 20260506 08:57:17.692 | INFO | "gpu" : " 17 GB", | 20260506 08:57:17.692 | INFO | "huge" : "", | 20260506 08:57:17.692 | INFO | "huge_shared" : " 27 GB", | 20260506 08:57:17.692 | INFO | "malloc" : " 15 GB", | 20260506 08:57:17.692 | INFO | "pinned" : " 29 GB" | 20260506 08:57:17.692 | INFO | } | 20260506 08:57:17.692 | INFO | Perf report for the last 5.13939 seconds: | 20260506 08:57:17.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:57:17.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:17.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:17.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:17.692 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 08:57:17.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:17.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:17.692 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 08:57:17.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:17.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:17.692 | INFO | Uploaded 36GiB to GPU in 1.60288s (22.4595GiB/s) | 20260506 08:57:17.692 | INFO | Downloaded 34GiB to GPU in 1.52074s (22.3575GiB/s) | 20260506 08:57:17.789 | INFO | Starting a round of Baz IO | 20260506 08:57:20.729 | INFO | Populating RTMetrics report | 20260506 08:57:20.729 | INFO | RTMetrics report populated | 20260506 08:57:20.729 | INFO | Opening tmp RTMetrics json file | 20260506 08:57:20.730 | INFO | Serializing RTMetrics to json | 20260506 08:57:20.754 | INFO | Serialization done, swapping tmp file now... | 20260506 08:57:20.782 | INFO | RTMetrics done | 20260506 08:57:20.809 | INFO | Finished a round of Baz IO: Wrote 1.623 GiB in 3.02 seconds | 20260506 08:57:20.810 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 62.50% / 0.44% / 37.07% | 20260506 08:57:20.810 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.7219% / 0.00183185% / 3.97364e-06% | 20260506 08:57:20.810 | INFO | Baz IO overall: | 20260506 08:57:20.810 | INFO | BytesWritten : 20828022709 | 20260506 08:57:20.810 | INFO | NumEvents : 8663152175 | 20260506 08:57:22.831 | INFO | Analyzing chunk frames = [94208,94720) | 20260506 08:57:22.831 | INFO | MemInfo: { | 20260506 08:57:22.831 | INFO | "gpu" : " 17 GB", | 20260506 08:57:22.831 | INFO | "huge" : "", | 20260506 08:57:22.831 | INFO | "huge_shared" : " 27 GB", | 20260506 08:57:22.831 | INFO | "malloc" : " 15 GB", | 20260506 08:57:22.831 | INFO | "pinned" : " 29 GB" | 20260506 08:57:22.831 | INFO | } | 20260506 08:57:22.831 | INFO | Perf report for the last 5.13926 seconds: | 20260506 08:57:22.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:57:22.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:22.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:22.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:22.831 | INFO | ANALYSIS : 58.1%, 2.9, 89.3ms, 0 | 20260506 08:57:22.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:22.831 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 08:57:22.831 | INFO | PRE_HQ : 57.5%, 2.1, 62.6ms, 0 | 20260506 08:57:22.831 | INFO | BAZWRITER : 58.4%, 4.2, 125.4ms, 0 | 20260506 08:57:22.831 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 08:57:22.831 | INFO | Uploaded 36GiB to GPU in 1.64746s (21.8519GiB/s) | 20260506 08:57:22.831 | INFO | Downloaded 38GiB to GPU in 1.70973s (22.2258GiB/s) | 20260506 08:57:27.970 | INFO | Analyzing chunk frames = [94720,95232) | 20260506 08:57:27.970 | INFO | MemInfo: { | 20260506 08:57:27.970 | INFO | "gpu" : " 17 GB", | 20260506 08:57:27.970 | INFO | "huge" : "", | 20260506 08:57:27.970 | INFO | "huge_shared" : " 27 GB", | 20260506 08:57:27.970 | INFO | "malloc" : " 15 GB", | 20260506 08:57:27.970 | INFO | "pinned" : " 29 GB" | 20260506 08:57:27.970 | INFO | } | 20260506 08:57:27.971 | INFO | Perf report for the last 5.13939 seconds: | 20260506 08:57:27.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:57:27.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:27.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:27.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:27.971 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260506 08:57:27.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:27.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:27.971 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 08:57:27.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:27.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:27.971 | INFO | Uploaded 36GiB to GPU in 1.59983s (22.5024GiB/s) | 20260506 08:57:27.971 | INFO | Downloaded 34GiB to GPU in 1.51572s (22.4316GiB/s) | 20260506 08:57:33.110 | INFO | Analyzing chunk frames = [95232,95744) | 20260506 08:57:33.110 | INFO | MemInfo: { | 20260506 08:57:33.110 | INFO | "gpu" : " 17 GB", | 20260506 08:57:33.110 | INFO | "huge" : "", | 20260506 08:57:33.110 | INFO | "huge_shared" : " 27 GB", | 20260506 08:57:33.110 | INFO | "malloc" : " 15 GB", | 20260506 08:57:33.110 | INFO | "pinned" : " 29 GB" | 20260506 08:57:33.110 | INFO | } | 20260506 08:57:33.110 | INFO | Perf report for the last 5.13931 seconds: | 20260506 08:57:33.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:57:33.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:33.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:33.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:33.110 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260506 08:57:33.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:33.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:33.110 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 08:57:33.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:33.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:33.110 | INFO | Uploaded 36GiB to GPU in 1.6043s (22.4397GiB/s) | 20260506 08:57:33.110 | INFO | Downloaded 34GiB to GPU in 1.52185s (22.3413GiB/s) | 20260506 08:57:38.249 | INFO | Analyzing chunk frames = [95744,96256) | 20260506 08:57:38.249 | INFO | MemInfo: { | 20260506 08:57:38.249 | INFO | "gpu" : " 17 GB", | 20260506 08:57:38.249 | INFO | "huge" : "", | 20260506 08:57:38.249 | INFO | "huge_shared" : " 27 GB", | 20260506 08:57:38.249 | INFO | "malloc" : " 15 GB", | 20260506 08:57:38.249 | INFO | "pinned" : " 29 GB" | 20260506 08:57:38.249 | INFO | } | 20260506 08:57:38.249 | INFO | Perf report for the last 5.13938 seconds: | 20260506 08:57:38.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:57:38.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:38.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:38.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:38.249 | INFO | ANALYSIS : 57.0%, 3.0, 88.4ms, 0 | 20260506 08:57:38.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:38.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:38.249 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 08:57:38.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:38.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:38.249 | INFO | Uploaded 36GiB to GPU in 1.60341s (22.4522GiB/s) | 20260506 08:57:38.249 | INFO | Downloaded 34GiB to GPU in 1.51996s (22.3691GiB/s) | 20260506 08:57:43.388 | INFO | Analyzing chunk frames = [96256,96768) | 20260506 08:57:43.388 | INFO | MemInfo: { | 20260506 08:57:43.388 | INFO | "gpu" : " 17 GB", | 20260506 08:57:43.388 | INFO | "huge" : "", | 20260506 08:57:43.388 | INFO | "huge_shared" : " 27 GB", | 20260506 08:57:43.388 | INFO | "malloc" : " 15 GB", | 20260506 08:57:43.388 | INFO | "pinned" : " 29 GB" | 20260506 08:57:43.388 | INFO | } | 20260506 08:57:43.389 | INFO | Perf report for the last 5.13927 seconds: | 20260506 08:57:43.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:57:43.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:43.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:43.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:43.389 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 08:57:43.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:43.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:43.389 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260506 08:57:43.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:43.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:43.389 | INFO | Uploaded 36GiB to GPU in 1.60181s (22.4746GiB/s) | 20260506 08:57:43.389 | INFO | Downloaded 34GiB to GPU in 1.51902s (22.3829GiB/s) | 20260506 08:57:48.528 | INFO | Analyzing chunk frames = [96768,97280) | 20260506 08:57:48.528 | INFO | MemInfo: { | 20260506 08:57:48.528 | INFO | "gpu" : " 17 GB", | 20260506 08:57:48.528 | INFO | "huge" : "", | 20260506 08:57:48.528 | INFO | "huge_shared" : " 27 GB", | 20260506 08:57:48.528 | INFO | "malloc" : " 15 GB", | 20260506 08:57:48.528 | INFO | "pinned" : " 29 GB" | 20260506 08:57:48.528 | INFO | } | 20260506 08:57:48.528 | INFO | Perf report for the last 5.13947 seconds: | 20260506 08:57:48.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:57:48.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:48.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:48.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:48.528 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 08:57:48.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:48.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:48.528 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 08:57:48.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:48.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:48.528 | INFO | Uploaded 36GiB to GPU in 1.60194s (22.4727GiB/s) | 20260506 08:57:48.528 | INFO | Downloaded 34GiB to GPU in 1.51849s (22.3907GiB/s) | 20260506 08:57:53.667 | INFO | Analyzing chunk frames = [97280,97792) | 20260506 08:57:53.667 | INFO | MemInfo: { | 20260506 08:57:53.667 | INFO | "gpu" : " 17 GB", | 20260506 08:57:53.667 | INFO | "huge" : "", | 20260506 08:57:53.667 | INFO | "huge_shared" : " 27 GB", | 20260506 08:57:53.667 | INFO | "malloc" : " 15 GB", | 20260506 08:57:53.667 | INFO | "pinned" : " 29 GB" | 20260506 08:57:53.667 | INFO | } | 20260506 08:57:53.667 | INFO | Perf report for the last 5.13921 seconds: | 20260506 08:57:53.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:57:53.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:53.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:53.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:53.667 | INFO | ANALYSIS : 57.3%, 3.0, 89.5ms, 0 | 20260506 08:57:53.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:53.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:53.667 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 08:57:53.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:53.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:53.668 | INFO | Uploaded 36GiB to GPU in 1.60067s (22.4906GiB/s) | 20260506 08:57:53.668 | INFO | Downloaded 34GiB to GPU in 1.52013s (22.3665GiB/s) | 20260506 08:57:58.806 | INFO | Analyzing chunk frames = [97792,98304) | 20260506 08:57:58.806 | INFO | MemInfo: { | 20260506 08:57:58.806 | INFO | "gpu" : " 17 GB", | 20260506 08:57:58.806 | INFO | "huge" : "", | 20260506 08:57:58.806 | INFO | "huge_shared" : " 27 GB", | 20260506 08:57:58.806 | INFO | "malloc" : " 15 GB", | 20260506 08:57:58.806 | INFO | "pinned" : " 29 GB" | 20260506 08:57:58.806 | INFO | } | 20260506 08:57:58.806 | INFO | Perf report for the last 5.13925 seconds: | 20260506 08:57:58.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:57:58.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:58.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:58.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:58.807 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 08:57:58.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:58.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:58.807 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 08:57:58.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:58.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:57:58.807 | INFO | Uploaded 36GiB to GPU in 1.6021s (22.4705GiB/s) | 20260506 08:57:58.807 | INFO | Downloaded 34GiB to GPU in 1.51967s (22.3732GiB/s) | 20260506 08:57:58.807 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 08:57:58.807 | INFO | Total mem usage: 15.5 GiB | 20260506 08:57:58.807 | INFO | All Time High Water Marks sum: 15.6 GiB (Savings: 0.781%) | 20260506 08:57:58.807 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -5.08%) | 20260506 08:57:58.807 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:57:58.807 | INFO | High Water Breakdown: | 20260506 08:57:58.807 | INFO | 58.2% : 8.59 GiB ( 96.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 08:57:58.807 | INFO | 41.7% : 6.16 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 08:57:58.807 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 08:57:58.807 | INFO | ------------------------------------------ | 20260506 08:57:58.807 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:57:58.807 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 08:57:58.807 | INFO | Total mem usage: 26.5 GiB | 20260506 08:57:58.807 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 08:57:58.807 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 08:57:58.807 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 08:57:58.807 | INFO | High Water Breakdown: | 20260506 08:57:58.807 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:57:58.807 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:57:58.807 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:57:58.807 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:57:58.807 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:57:58.807 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:57:58.807 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 08:57:58.807 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 08:57:58.807 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:57:58.807 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 08:57:58.807 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 08:57:58.807 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 08:57:58.807 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 08:57:58.807 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 08:57:58.807 | INFO | ------------------------------------------ | 20260506 08:57:58.807 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:57:58.807 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 08:57:58.807 | INFO | Total mem usage: 25.6 GiB | 20260506 08:57:58.807 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 08:57:58.807 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 08:57:58.807 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 08:57:58.807 | INFO | High Water Breakdown: | 20260506 08:57:58.807 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 08:57:58.807 | INFO | ------------------------------------------ | 20260506 08:57:58.807 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:57:58.807 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 08:57:58.807 | INFO | Total mem usage: 15.7 GiB | 20260506 08:57:58.807 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 08:57:58.808 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 08:57:58.808 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 08:57:58.808 | INFO | High Water Breakdown: | 20260506 08:57:58.808 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 08:57:58.808 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 08:57:58.808 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 08:57:58.808 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 08:57:58.808 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 08:57:58.808 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 08:57:58.808 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 08:57:58.808 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 08:57:58.808 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 08:57:58.808 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:57:58.808 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 08:57:58.808 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:57:58.808 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 08:57:58.808 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 08:57:58.808 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 08:57:58.808 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 08:57:58.808 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:57:58.808 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 08:57:58.808 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:57:58.808 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:57:58.808 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:57:58.808 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 08:57:58.808 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 08:57:58.808 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:57:58.808 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 08:57:58.808 | INFO | ------------------------------------------ | 20260506 08:57:58.808 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:57:58.808 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 08:57:58.808 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:57:58.808 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:57:58.808 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260506 08:57:58.808 | INFO | PerfReport: 0.30%, AggregateStats -- 8.99e-02 / 0.00e+00, 1.48e-01 / 0.00e+00 | 20260506 08:57:58.808 | INFO | PerfReport: 10.84%, Baseline -- 3.29e+00 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260506 08:57:58.808 | INFO | PerfReport: 3.04%, Histogram -- 9.23e-01 / 0.00e+00, 5.22e-02 / 0.00e+00 | 20260506 08:57:58.808 | INFO | PerfReport: 15.99%, DME -- 3.89e+01 / 0.00e+00, 3.24e+00 / 0.00e+00 | 20260506 08:57:58.808 | INFO | PerfReport: 56.61%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.33e-02 / 0.00e+00 | 20260506 08:57:58.808 | INFO | PerfReport: 7.65%, PulseAccumulating -- 2.33e+00 / 0.00e+00, 1.42e-01 / 0.00e+00 | 20260506 08:57:58.808 | INFO | PerfReport: 4.01%, Metrics -- 1.22e+00 / 0.00e+00, 1.90e-01 / 0.00e+00 | 20260506 08:57:58.808 | INFO | PerfReport: 1.57%, PulseSerializing -- 4.78e-01 / 0.00e+00, 6.95e-02 / 0.00e+00 | 20260506 08:57:58.808 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 08:57:58.808 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:57:58.808 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:57:58.808 | INFO | PerfReport: Full Execution -- 9.12e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260506 08:57:58.808 | INFO | PerfReport: 18.36%, Upload -- 1.67e+01 / 0.00e+00, 4.18e-01 / 0.00e+00 | 20260506 08:57:58.808 | INFO | PerfReport: 17.62%, Download -- 1.61e+01 / 0.00e+00, 7.40e-01 / 0.00e+00 | 20260506 08:57:58.808 | INFO | PerfReport: 33.33%, Compute -- 3.04e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260506 08:57:58.808 | INFO | PerfReport: 0.56%, LockUpload -- 5.09e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 08:57:58.808 | INFO | PerfReport: 0.00%, LockDownload -- 5.00e-04 / 0.00e+00, 1.25e-04 / 0.00e+00 | 20260506 08:57:58.808 | INFO | PerfReport: 30.11%, LockCompute -- 2.75e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 08:57:58.808 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.17e-02 / 0.00e+00 | 20260506 08:57:58.906 | INFO | Starting a round of Baz IO | 20260506 08:58:01.840 | INFO | Populating RTMetrics report | 20260506 08:58:01.840 | INFO | RTMetrics report populated | 20260506 08:58:01.841 | INFO | Opening tmp RTMetrics json file | 20260506 08:58:01.896 | INFO | Serializing RTMetrics to json | 20260506 08:58:01.934 | INFO | Serialization done, swapping tmp file now... | 20260506 08:58:01.938 | INFO | RTMetrics done | 20260506 08:58:01.959 | INFO | Finished a round of Baz IO: Wrote 1.633 GiB in 3.053 seconds | 20260506 08:58:01.959 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 62.11% / 0.39% / 37.50% | 20260506 08:58:01.959 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.669% / 0.00156959% / 1.58946e-05% | 20260506 08:58:01.959 | INFO | Baz IO overall: | 20260506 08:58:01.959 | INFO | BytesWritten : 22581343448 | 20260506 08:58:01.959 | INFO | NumEvents : 9517944419 | 20260506 08:58:03.946 | INFO | Analyzing chunk frames = [98304,98816) | 20260506 08:58:03.946 | INFO | MemInfo: { | 20260506 08:58:03.946 | INFO | "gpu" : " 17 GB", | 20260506 08:58:03.946 | INFO | "huge" : "", | 20260506 08:58:03.946 | INFO | "huge_shared" : " 27 GB", | 20260506 08:58:03.946 | INFO | "malloc" : " 15 GB", | 20260506 08:58:03.946 | INFO | "pinned" : " 29 GB" | 20260506 08:58:03.946 | INFO | } | 20260506 08:58:03.946 | INFO | Perf report for the last 5.13931 seconds: | 20260506 08:58:03.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:58:03.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:03.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:03.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:03.946 | INFO | ANALYSIS : 58.1%, 2.9, 87.0ms, 0 | 20260506 08:58:03.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:03.946 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 08:58:03.946 | INFO | PRE_HQ : 57.4%, 2.1, 62.2ms, 0 | 20260506 08:58:03.946 | INFO | BAZWRITER : 59.0%, 4.6, 140.9ms, 0 | 20260506 08:58:03.946 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 08:58:03.946 | INFO | Uploaded 36GiB to GPU in 1.6587s (21.7038GiB/s) | 20260506 08:58:03.946 | INFO | Downloaded 38GiB to GPU in 1.72386s (22.0436GiB/s) | 20260506 08:58:09.085 | INFO | Analyzing chunk frames = [98816,99328) | 20260506 08:58:09.085 | INFO | MemInfo: { | 20260506 08:58:09.085 | INFO | "gpu" : " 17 GB", | 20260506 08:58:09.085 | INFO | "huge" : "", | 20260506 08:58:09.085 | INFO | "huge_shared" : " 27 GB", | 20260506 08:58:09.085 | INFO | "malloc" : " 15 GB", | 20260506 08:58:09.085 | INFO | "pinned" : " 29 GB" | 20260506 08:58:09.085 | INFO | } | 20260506 08:58:09.085 | INFO | Perf report for the last 5.13917 seconds: | 20260506 08:58:09.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:58:09.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:09.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:09.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:09.085 | INFO | ANALYSIS : 57.9%, 3.0, 90.4ms, 0 | 20260506 08:58:09.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:09.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:09.085 | INFO | PRE_HQ : 31.0%, 1.0, 16.3ms, 0 | 20260506 08:58:09.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:09.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:09.086 | INFO | Uploaded 36GiB to GPU in 1.59984s (22.5023GiB/s) | 20260506 08:58:09.086 | INFO | Downloaded 34GiB to GPU in 1.51625s (22.4237GiB/s) | 20260506 08:58:14.224 | INFO | Analyzing chunk frames = [99328,99840) | 20260506 08:58:14.224 | INFO | MemInfo: { | 20260506 08:58:14.224 | INFO | "gpu" : " 17 GB", | 20260506 08:58:14.224 | INFO | "huge" : "", | 20260506 08:58:14.224 | INFO | "huge_shared" : " 27 GB", | 20260506 08:58:14.224 | INFO | "malloc" : " 15 GB", | 20260506 08:58:14.224 | INFO | "pinned" : " 29 GB" | 20260506 08:58:14.224 | INFO | } | 20260506 08:58:14.225 | INFO | Perf report for the last 5.13954 seconds: | 20260506 08:58:14.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:58:14.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:14.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:14.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:14.225 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260506 08:58:14.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:14.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:14.225 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 08:58:14.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:14.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:14.225 | INFO | Uploaded 36GiB to GPU in 1.60439s (22.4384GiB/s) | 20260506 08:58:14.225 | INFO | Downloaded 34GiB to GPU in 1.52147s (22.3468GiB/s) | 20260506 08:58:19.364 | INFO | Analyzing chunk frames = [99840,100352) | 20260506 08:58:19.364 | INFO | MemInfo: { | 20260506 08:58:19.364 | INFO | "gpu" : " 17 GB", | 20260506 08:58:19.364 | INFO | "huge" : "", | 20260506 08:58:19.364 | INFO | "huge_shared" : " 27 GB", | 20260506 08:58:19.364 | INFO | "malloc" : " 15 GB", | 20260506 08:58:19.364 | INFO | "pinned" : " 29 GB" | 20260506 08:58:19.364 | INFO | } | 20260506 08:58:19.364 | INFO | Perf report for the last 5.1391 seconds: | 20260506 08:58:19.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:58:19.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:19.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:19.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:19.364 | INFO | ANALYSIS : 57.0%, 3.0, 88.3ms, 0 | 20260506 08:58:19.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:19.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:19.364 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 08:58:19.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:19.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:19.364 | INFO | Uploaded 36GiB to GPU in 1.60305s (22.4572GiB/s) | 20260506 08:58:19.364 | INFO | Downloaded 34GiB to GPU in 1.52062s (22.3593GiB/s) | 20260506 08:58:24.503 | INFO | Analyzing chunk frames = [100352,100864) | 20260506 08:58:24.503 | INFO | MemInfo: { | 20260506 08:58:24.503 | INFO | "gpu" : " 17 GB", | 20260506 08:58:24.503 | INFO | "huge" : "", | 20260506 08:58:24.503 | INFO | "huge_shared" : " 27 GB", | 20260506 08:58:24.503 | INFO | "malloc" : " 15 GB", | 20260506 08:58:24.503 | INFO | "pinned" : " 29 GB" | 20260506 08:58:24.503 | INFO | } | 20260506 08:58:24.503 | INFO | Perf report for the last 5.13946 seconds: | 20260506 08:58:24.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:58:24.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:24.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:24.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:24.503 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 08:58:24.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:24.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:24.503 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 08:58:24.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:24.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:24.503 | INFO | Uploaded 36GiB to GPU in 1.60164s (22.4769GiB/s) | 20260506 08:58:24.503 | INFO | Downloaded 34GiB to GPU in 1.51888s (22.3849GiB/s) | 20260506 08:58:29.642 | INFO | Analyzing chunk frames = [100864,101376) | 20260506 08:58:29.642 | INFO | MemInfo: { | 20260506 08:58:29.642 | INFO | "gpu" : " 17 GB", | 20260506 08:58:29.642 | INFO | "huge" : "", | 20260506 08:58:29.642 | INFO | "huge_shared" : " 27 GB", | 20260506 08:58:29.642 | INFO | "malloc" : " 15 GB", | 20260506 08:58:29.642 | INFO | "pinned" : " 29 GB" | 20260506 08:58:29.642 | INFO | } | 20260506 08:58:29.642 | INFO | Perf report for the last 5.13935 seconds: | 20260506 08:58:29.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:58:29.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:29.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:29.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:29.643 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 08:58:29.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:29.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:29.643 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 08:58:29.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:29.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:29.643 | INFO | Uploaded 36GiB to GPU in 1.60184s (22.4742GiB/s) | 20260506 08:58:29.643 | INFO | Downloaded 34GiB to GPU in 1.51905s (22.3824GiB/s) | 20260506 08:58:34.782 | INFO | Analyzing chunk frames = [101376,101888) | 20260506 08:58:34.782 | INFO | MemInfo: { | 20260506 08:58:34.782 | INFO | "gpu" : " 17 GB", | 20260506 08:58:34.782 | INFO | "huge" : "", | 20260506 08:58:34.782 | INFO | "huge_shared" : " 27 GB", | 20260506 08:58:34.782 | INFO | "malloc" : " 15 GB", | 20260506 08:58:34.782 | INFO | "pinned" : " 29 GB" | 20260506 08:58:34.782 | INFO | } | 20260506 08:58:34.782 | INFO | Perf report for the last 5.13929 seconds: | 20260506 08:58:34.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:58:34.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:34.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:34.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:34.782 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 08:58:34.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:34.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:34.782 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 08:58:34.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:34.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:34.782 | INFO | Uploaded 36GiB to GPU in 1.60207s (22.471GiB/s) | 20260506 08:58:34.782 | INFO | Downloaded 34GiB to GPU in 1.52039s (22.3627GiB/s) | 20260506 08:58:39.921 | INFO | Analyzing chunk frames = [101888,102400) | 20260506 08:58:39.921 | INFO | MemInfo: { | 20260506 08:58:39.921 | INFO | "gpu" : " 17 GB", | 20260506 08:58:39.921 | INFO | "huge" : "", | 20260506 08:58:39.921 | INFO | "huge_shared" : " 27 GB", | 20260506 08:58:39.921 | INFO | "malloc" : " 15 GB", | 20260506 08:58:39.921 | INFO | "pinned" : " 29 GB" | 20260506 08:58:39.921 | INFO | } | 20260506 08:58:39.921 | INFO | Perf report for the last 5.13935 seconds: | 20260506 08:58:39.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:58:39.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:39.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:39.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:39.921 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 08:58:39.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:39.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:39.921 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 08:58:39.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:39.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:39.921 | INFO | Uploaded 36GiB to GPU in 1.60276s (22.4612GiB/s) | 20260506 08:58:39.921 | INFO | Downloaded 34GiB to GPU in 1.52007s (22.3674GiB/s) | 20260506 08:58:40.018 | INFO | Starting a round of Baz IO | 20260506 08:58:42.955 | INFO | Populating RTMetrics report | 20260506 08:58:42.955 | INFO | RTMetrics report populated | 20260506 08:58:42.955 | INFO | Opening tmp RTMetrics json file | 20260506 08:58:43.140 | INFO | Serializing RTMetrics to json | 20260506 08:58:43.221 | INFO | Serialization done, swapping tmp file now... | 20260506 08:58:43.225 | INFO | RTMetrics done | 20260506 08:58:43.246 | INFO | Finished a round of Baz IO: Wrote 1.641 GiB in 3.227 seconds | 20260506 08:58:43.246 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 61.75% / 0.36% / 37.89% | 20260506 08:58:43.246 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.5737% / 0.00156562% / 1.58946e-05% | 20260506 08:58:43.246 | INFO | Baz IO overall: | 20260506 08:58:43.246 | INFO | BytesWritten : 24343424569 | 20260506 08:58:43.246 | INFO | NumEvents : 10376791667 | 20260506 08:58:45.060 | INFO | Analyzing chunk frames = [102400,102912) | 20260506 08:58:45.060 | INFO | MemInfo: { | 20260506 08:58:45.060 | INFO | "gpu" : " 17 GB", | 20260506 08:58:45.060 | INFO | "huge" : "", | 20260506 08:58:45.060 | INFO | "huge_shared" : " 27 GB", | 20260506 08:58:45.060 | INFO | "malloc" : " 15 GB", | 20260506 08:58:45.060 | INFO | "pinned" : " 29 GB" | 20260506 08:58:45.060 | INFO | } | 20260506 08:58:45.061 | INFO | Perf report for the last 5.13935 seconds: | 20260506 08:58:45.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:58:45.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:45.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:45.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:45.061 | INFO | ANALYSIS : 58.1%, 2.9, 89.0ms, 0 | 20260506 08:58:45.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:45.061 | INFO | RT_METRICS : 48.6%, 1.0, 25.5ms, 0 | 20260506 08:58:45.061 | INFO | PRE_HQ : 57.4%, 2.1, 62.6ms, 0 | 20260506 08:58:45.061 | INFO | BAZWRITER : 62.2%, 4.4, 139.3ms, 0 | 20260506 08:58:45.061 | INFO | BAZ_SAMPLING : 5.0%, 1.0, 2.6ms, 0 | 20260506 08:58:45.061 | INFO | Uploaded 36GiB to GPU in 1.64952s (21.8245GiB/s) | 20260506 08:58:45.061 | INFO | Downloaded 38GiB to GPU in 1.71273s (22.1868GiB/s) | 20260506 08:58:50.200 | INFO | Analyzing chunk frames = [102912,103424) | 20260506 08:58:50.200 | INFO | MemInfo: { | 20260506 08:58:50.200 | INFO | "gpu" : " 17 GB", | 20260506 08:58:50.200 | INFO | "huge" : "", | 20260506 08:58:50.200 | INFO | "huge_shared" : " 27 GB", | 20260506 08:58:50.200 | INFO | "malloc" : " 15 GB", | 20260506 08:58:50.200 | INFO | "pinned" : " 29 GB" | 20260506 08:58:50.200 | INFO | } | 20260506 08:58:50.200 | INFO | Perf report for the last 5.13949 seconds: | 20260506 08:58:50.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:58:50.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:50.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:50.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:50.200 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260506 08:58:50.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:50.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:50.200 | INFO | PRE_HQ : 30.8%, 1.0, 16.2ms, 0 | 20260506 08:58:50.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:50.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:50.200 | INFO | Uploaded 36GiB to GPU in 1.60044s (22.4938GiB/s) | 20260506 08:58:50.201 | INFO | Downloaded 34GiB to GPU in 1.51648s (22.4204GiB/s) | 20260506 08:58:55.339 | INFO | Analyzing chunk frames = [103424,103936) | 20260506 08:58:55.339 | INFO | MemInfo: { | 20260506 08:58:55.339 | INFO | "gpu" : " 17 GB", | 20260506 08:58:55.339 | INFO | "huge" : "", | 20260506 08:58:55.339 | INFO | "huge_shared" : " 27 GB", | 20260506 08:58:55.339 | INFO | "malloc" : " 15 GB", | 20260506 08:58:55.339 | INFO | "pinned" : " 29 GB" | 20260506 08:58:55.339 | INFO | } | 20260506 08:58:55.339 | INFO | Perf report for the last 5.13932 seconds: | 20260506 08:58:55.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:58:55.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:55.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:55.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:55.339 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260506 08:58:55.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:55.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:55.340 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 08:58:55.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:55.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:58:55.340 | INFO | Uploaded 36GiB to GPU in 1.60431s (22.4396GiB/s) | 20260506 08:58:55.340 | INFO | Downloaded 34GiB to GPU in 1.52242s (22.3329GiB/s) | 20260506 08:59:00.478 | INFO | Analyzing chunk frames = [103936,104448) | 20260506 08:59:00.478 | INFO | MemInfo: { | 20260506 08:59:00.478 | INFO | "gpu" : " 17 GB", | 20260506 08:59:00.478 | INFO | "huge" : "", | 20260506 08:59:00.478 | INFO | "huge_shared" : " 27 GB", | 20260506 08:59:00.478 | INFO | "malloc" : " 15 GB", | 20260506 08:59:00.479 | INFO | "pinned" : " 29 GB" | 20260506 08:59:00.479 | INFO | } | 20260506 08:59:00.479 | INFO | Perf report for the last 5.13926 seconds: | 20260506 08:59:00.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:59:00.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:00.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:00.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:00.479 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260506 08:59:00.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:00.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:00.479 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 08:59:00.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:00.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:00.479 | INFO | Uploaded 36GiB to GPU in 1.60325s (22.4544GiB/s) | 20260506 08:59:00.479 | INFO | Downloaded 34GiB to GPU in 1.52012s (22.3667GiB/s) | 20260506 08:59:05.618 | INFO | Analyzing chunk frames = [104448,104960) | 20260506 08:59:05.618 | INFO | MemInfo: { | 20260506 08:59:05.618 | INFO | "gpu" : " 17 GB", | 20260506 08:59:05.618 | INFO | "huge" : "", | 20260506 08:59:05.618 | INFO | "huge_shared" : " 27 GB", | 20260506 08:59:05.618 | INFO | "malloc" : " 15 GB", | 20260506 08:59:05.618 | INFO | "pinned" : " 29 GB" | 20260506 08:59:05.618 | INFO | } | 20260506 08:59:05.618 | INFO | Perf report for the last 5.1394 seconds: | 20260506 08:59:05.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:59:05.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:05.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:05.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:05.618 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 08:59:05.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:05.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:05.618 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 08:59:05.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:05.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:05.618 | INFO | Uploaded 36GiB to GPU in 1.60151s (22.4788GiB/s) | 20260506 08:59:05.618 | INFO | Downloaded 34GiB to GPU in 1.51855s (22.3898GiB/s) | 20260506 08:59:10.757 | INFO | Analyzing chunk frames = [104960,105472) | 20260506 08:59:10.757 | INFO | MemInfo: { | 20260506 08:59:10.757 | INFO | "gpu" : " 17 GB", | 20260506 08:59:10.757 | INFO | "huge" : "", | 20260506 08:59:10.757 | INFO | "huge_shared" : " 27 GB", | 20260506 08:59:10.757 | INFO | "malloc" : " 15 GB", | 20260506 08:59:10.757 | INFO | "pinned" : " 29 GB" | 20260506 08:59:10.757 | INFO | } | 20260506 08:59:10.757 | INFO | Perf report for the last 5.13896 seconds: | 20260506 08:59:10.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:59:10.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:10.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:10.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:10.757 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 08:59:10.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:10.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:10.757 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260506 08:59:10.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:10.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:10.757 | INFO | Uploaded 36GiB to GPU in 1.60226s (22.4683GiB/s) | 20260506 08:59:10.757 | INFO | Downloaded 34GiB to GPU in 1.51949s (22.376GiB/s) | 20260506 08:59:15.896 | INFO | Analyzing chunk frames = [105472,105984) | 20260506 08:59:15.896 | INFO | MemInfo: { | 20260506 08:59:15.896 | INFO | "gpu" : " 17 GB", | 20260506 08:59:15.896 | INFO | "huge" : "", | 20260506 08:59:15.896 | INFO | "huge_shared" : " 27 GB", | 20260506 08:59:15.897 | INFO | "malloc" : " 15 GB", | 20260506 08:59:15.897 | INFO | "pinned" : " 29 GB" | 20260506 08:59:15.897 | INFO | } | 20260506 08:59:15.897 | INFO | Perf report for the last 5.13959 seconds: | 20260506 08:59:15.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:59:15.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:15.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:15.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:15.897 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 08:59:15.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:15.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:15.897 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 08:59:15.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:15.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:15.897 | INFO | Uploaded 36GiB to GPU in 1.60172s (22.4759GiB/s) | 20260506 08:59:15.897 | INFO | Downloaded 34GiB to GPU in 1.5193s (22.3788GiB/s) | 20260506 08:59:21.036 | INFO | Analyzing chunk frames = [105984,106496) | 20260506 08:59:21.036 | INFO | MemInfo: { | 20260506 08:59:21.036 | INFO | "gpu" : " 17 GB", | 20260506 08:59:21.036 | INFO | "huge" : "", | 20260506 08:59:21.036 | INFO | "huge_shared" : " 27 GB", | 20260506 08:59:21.036 | INFO | "malloc" : " 15 GB", | 20260506 08:59:21.036 | INFO | "pinned" : " 29 GB" | 20260506 08:59:21.036 | INFO | } | 20260506 08:59:21.036 | INFO | Perf report for the last 5.13922 seconds: | 20260506 08:59:21.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:59:21.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:21.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:21.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:21.036 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 08:59:21.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:21.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:21.036 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 08:59:21.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:21.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:21.036 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 08:59:21.036 | INFO | Total mem usage: 15.5 GiB | 20260506 08:59:21.036 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 1.29%) | 20260506 08:59:21.036 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -1.26%) | 20260506 08:59:21.036 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 08:59:21.036 | INFO | Uploaded 36GiB to GPU in 1.60325s (22.4543GiB/s) | 20260506 08:59:21.036 | INFO | High Water Breakdown: | 20260506 08:59:21.036 | INFO | 58.8% : 9.02 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 08:59:21.036 | INFO | Downloaded 34GiB to GPU in 1.52027s (22.3645GiB/s) | 20260506 08:59:21.036 | INFO | 41.1% : 6.3 GiB ( 94.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 08:59:21.036 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 08:59:21.036 | INFO | ------------------------------------------ | 20260506 08:59:21.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:59:21.036 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 08:59:21.036 | INFO | Total mem usage: 26.5 GiB | 20260506 08:59:21.036 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 08:59:21.036 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 08:59:21.036 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 08:59:21.036 | INFO | High Water Breakdown: | 20260506 08:59:21.036 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:59:21.036 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:59:21.036 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:59:21.036 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:59:21.036 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:59:21.036 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:59:21.036 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 08:59:21.036 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 08:59:21.036 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:59:21.036 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 08:59:21.036 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 08:59:21.036 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 08:59:21.036 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 08:59:21.036 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 08:59:21.036 | INFO | ------------------------------------------ | 20260506 08:59:21.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:59:21.036 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 08:59:21.036 | INFO | Total mem usage: 25.6 GiB | 20260506 08:59:21.036 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 08:59:21.036 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 08:59:21.036 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 08:59:21.036 | INFO | High Water Breakdown: | 20260506 08:59:21.036 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 08:59:21.036 | INFO | ------------------------------------------ | 20260506 08:59:21.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:59:21.036 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 08:59:21.036 | INFO | Total mem usage: 15.7 GiB | 20260506 08:59:21.036 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 08:59:21.036 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 08:59:21.036 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 08:59:21.036 | INFO | High Water Breakdown: | 20260506 08:59:21.036 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 08:59:21.036 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 08:59:21.036 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 08:59:21.036 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 08:59:21.036 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 08:59:21.036 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 08:59:21.036 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 08:59:21.036 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 08:59:21.036 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 08:59:21.036 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 08:59:21.036 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 08:59:21.036 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 08:59:21.036 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 08:59:21.036 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 08:59:21.036 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 08:59:21.036 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 08:59:21.036 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 08:59:21.036 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 08:59:21.036 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 08:59:21.036 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 08:59:21.036 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 08:59:21.036 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 08:59:21.036 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 08:59:21.036 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 08:59:21.036 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 08:59:21.036 | INFO | ------------------------------------------ | 20260506 08:59:21.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 08:59:21.036 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 08:59:21.036 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:59:21.036 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:59:21.036 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 08:59:21.036 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / 0.00e+00, 6.57e-03 / 0.00e+00 | 20260506 08:59:21.036 | INFO | PerfReport: 10.84%, Baseline -- 3.29e+00 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260506 08:59:21.036 | INFO | PerfReport: 3.04%, Histogram -- 9.22e-01 / 0.00e+00, 5.19e-02 / 0.00e+00 | 20260506 08:59:21.036 | INFO | PerfReport: 16.00%, DME -- 3.89e+01 / 0.00e+00, 3.29e+00 / 0.00e+00 | 20260506 08:59:21.036 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.31e-02 / 0.00e+00 | 20260506 08:59:21.036 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.32e+00 / 0.00e+00, 1.41e-01 / 0.00e+00 | 20260506 08:59:21.036 | INFO | PerfReport: 3.99%, Metrics -- 1.21e+00 / 0.00e+00, 1.90e-01 / 0.00e+00 | 20260506 08:59:21.036 | INFO | PerfReport: 1.57%, PulseSerializing -- 4.75e-01 / 0.00e+00, 6.90e-02 / 0.00e+00 | 20260506 08:59:21.036 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 08:59:21.037 | INFO | PerfReport: Gathered 1536 samples. | 20260506 08:59:21.037 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 08:59:21.037 | INFO | PerfReport: Full Execution -- 9.09e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260506 08:59:21.037 | INFO | PerfReport: 18.43%, Upload -- 1.68e+01 / 0.00e+00, 4.35e-01 / 0.00e+00 | 20260506 08:59:21.037 | INFO | PerfReport: 17.70%, Download -- 1.61e+01 / 0.00e+00, 7.73e-01 / 0.00e+00 | 20260506 08:59:21.037 | INFO | PerfReport: 33.41%, Compute -- 3.04e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 08:59:21.037 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 08:59:21.037 | INFO | PerfReport: 0.00%, LockDownload -- 4.87e-04 / 0.00e+00, 1.23e-04 / 0.00e+00 | 20260506 08:59:21.037 | INFO | PerfReport: 29.89%, LockCompute -- 2.72e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 08:59:21.037 | INFO | PerfReport: 0.02%, Misc -- 1.42e-02 / 0.00e+00, 2.81e-02 / 0.00e+00 | 20260506 08:59:21.133 | INFO | Starting a round of Baz IO | 20260506 08:59:24.070 | INFO | Populating RTMetrics report | 20260506 08:59:24.071 | INFO | RTMetrics report populated | 20260506 08:59:24.071 | INFO | Opening tmp RTMetrics json file | 20260506 08:59:24.072 | INFO | Serializing RTMetrics to json | 20260506 08:59:24.074 | INFO | Serialization done, swapping tmp file now... | 20260506 08:59:24.075 | INFO | RTMetrics done | 20260506 08:59:24.165 | INFO | Finished a round of Baz IO: Wrote 1.65 GiB in 3.031 seconds | 20260506 08:59:24.165 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 61.40% / 0.34% / 38.25% | 20260506 08:59:24.165 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.5128% / 0.00133117% / 7.94729e-06% | 20260506 08:59:24.165 | INFO | Baz IO overall: | 20260506 08:59:24.165 | INFO | BytesWritten : 26115531587 | 20260506 08:59:24.165 | INFO | NumEvents : 11240489255 | 20260506 08:59:26.175 | INFO | Analyzing chunk frames = [106496,107008) | 20260506 08:59:26.175 | INFO | MemInfo: { | 20260506 08:59:26.175 | INFO | "gpu" : " 17 GB", | 20260506 08:59:26.175 | INFO | "huge" : "", | 20260506 08:59:26.175 | INFO | "huge_shared" : " 27 GB", | 20260506 08:59:26.175 | INFO | "malloc" : " 14 GB", | 20260506 08:59:26.175 | INFO | "pinned" : " 29 GB" | 20260506 08:59:26.175 | INFO | } | 20260506 08:59:26.175 | INFO | Perf report for the last 5.13922 seconds: | 20260506 08:59:26.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:59:26.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:26.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:26.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:26.175 | INFO | ANALYSIS : 58.1%, 3.0, 90.1ms, 0 | 20260506 08:59:26.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:26.175 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260506 08:59:26.175 | INFO | PRE_HQ : 57.3%, 2.1, 62.8ms, 0 | 20260506 08:59:26.175 | INFO | BAZWRITER : 58.5%, 4.0, 119.2ms, 0 | 20260506 08:59:26.175 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 08:59:26.175 | INFO | Uploaded 36GiB to GPU in 1.64072s (21.9416GiB/s) | 20260506 08:59:26.175 | INFO | Downloaded 38GiB to GPU in 1.7017s (22.3306GiB/s) | 20260506 08:59:31.314 | INFO | Analyzing chunk frames = [107008,107520) | 20260506 08:59:31.314 | INFO | MemInfo: { | 20260506 08:59:31.314 | INFO | "gpu" : " 17 GB", | 20260506 08:59:31.314 | INFO | "huge" : "", | 20260506 08:59:31.314 | INFO | "huge_shared" : " 27 GB", | 20260506 08:59:31.314 | INFO | "malloc" : " 14 GB", | 20260506 08:59:31.314 | INFO | "pinned" : " 29 GB" | 20260506 08:59:31.314 | INFO | } | 20260506 08:59:31.315 | INFO | Perf report for the last 5.13945 seconds: | 20260506 08:59:31.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:59:31.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:31.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:31.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:31.315 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260506 08:59:31.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:31.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:31.315 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260506 08:59:31.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:31.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:31.315 | INFO | Uploaded 36GiB to GPU in 1.60103s (22.4855GiB/s) | 20260506 08:59:31.315 | INFO | Downloaded 34GiB to GPU in 1.5181s (22.3965GiB/s) | 20260506 08:59:36.454 | INFO | Analyzing chunk frames = [107520,108032) | 20260506 08:59:36.454 | INFO | MemInfo: { | 20260506 08:59:36.454 | INFO | "gpu" : " 17 GB", | 20260506 08:59:36.454 | INFO | "huge" : "", | 20260506 08:59:36.454 | INFO | "huge_shared" : " 27 GB", | 20260506 08:59:36.454 | INFO | "malloc" : " 14 GB", | 20260506 08:59:36.454 | INFO | "pinned" : " 29 GB" | 20260506 08:59:36.454 | INFO | } | 20260506 08:59:36.454 | INFO | Perf report for the last 5.13919 seconds: | 20260506 08:59:36.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:59:36.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:36.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:36.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:36.454 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260506 08:59:36.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:36.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:36.454 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 08:59:36.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:36.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:36.454 | INFO | Uploaded 36GiB to GPU in 1.60374s (22.4475GiB/s) | 20260506 08:59:36.454 | INFO | Downloaded 34GiB to GPU in 1.52164s (22.3443GiB/s) | 20260506 08:59:41.593 | INFO | Analyzing chunk frames = [108032,108544) | 20260506 08:59:41.593 | INFO | MemInfo: { | 20260506 08:59:41.593 | INFO | "gpu" : " 17 GB", | 20260506 08:59:41.593 | INFO | "huge" : "", | 20260506 08:59:41.593 | INFO | "huge_shared" : " 27 GB", | 20260506 08:59:41.593 | INFO | "malloc" : " 14 GB", | 20260506 08:59:41.593 | INFO | "pinned" : " 29 GB" | 20260506 08:59:41.593 | INFO | } | 20260506 08:59:41.593 | INFO | Perf report for the last 5.13956 seconds: | 20260506 08:59:41.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:59:41.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:41.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:41.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:41.593 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260506 08:59:41.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:41.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:41.593 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 08:59:41.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:41.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:41.594 | INFO | Uploaded 36GiB to GPU in 1.6039s (22.4453GiB/s) | 20260506 08:59:41.594 | INFO | Downloaded 34GiB to GPU in 1.52107s (22.3527GiB/s) | 20260506 08:59:46.732 | INFO | Analyzing chunk frames = [108544,109056) | 20260506 08:59:46.732 | INFO | MemInfo: { | 20260506 08:59:46.732 | INFO | "gpu" : " 17 GB", | 20260506 08:59:46.732 | INFO | "huge" : "", | 20260506 08:59:46.732 | INFO | "huge_shared" : " 27 GB", | 20260506 08:59:46.732 | INFO | "malloc" : " 14 GB", | 20260506 08:59:46.732 | INFO | "pinned" : " 29 GB" | 20260506 08:59:46.733 | INFO | } | 20260506 08:59:46.733 | INFO | Perf report for the last 5.13924 seconds: | 20260506 08:59:46.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:59:46.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:46.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:46.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:46.733 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 08:59:46.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:46.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:46.733 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 08:59:46.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:46.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:46.733 | INFO | Uploaded 36GiB to GPU in 1.6018s (22.4747GiB/s) | 20260506 08:59:46.733 | INFO | Downloaded 34GiB to GPU in 1.51787s (22.3997GiB/s) | 20260506 08:59:51.872 | INFO | Analyzing chunk frames = [109056,109568) | 20260506 08:59:51.872 | INFO | MemInfo: { | 20260506 08:59:51.872 | INFO | "gpu" : " 17 GB", | 20260506 08:59:51.872 | INFO | "huge" : "", | 20260506 08:59:51.872 | INFO | "huge_shared" : " 27 GB", | 20260506 08:59:51.872 | INFO | "malloc" : " 14 GB", | 20260506 08:59:51.872 | INFO | "pinned" : " 29 GB" | 20260506 08:59:51.872 | INFO | } | 20260506 08:59:51.872 | INFO | Perf report for the last 5.1395 seconds: | 20260506 08:59:51.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:59:51.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:51.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:51.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:51.872 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 08:59:51.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:51.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:51.872 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 08:59:51.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:51.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:51.872 | INFO | Uploaded 36GiB to GPU in 1.60078s (22.489GiB/s) | 20260506 08:59:51.872 | INFO | Downloaded 34GiB to GPU in 1.51836s (22.3926GiB/s) | 20260506 08:59:57.011 | INFO | Analyzing chunk frames = [109568,110080) | 20260506 08:59:57.011 | INFO | MemInfo: { | 20260506 08:59:57.011 | INFO | "gpu" : " 17 GB", | 20260506 08:59:57.011 | INFO | "huge" : "", | 20260506 08:59:57.011 | INFO | "huge_shared" : " 27 GB", | 20260506 08:59:57.011 | INFO | "malloc" : " 14 GB", | 20260506 08:59:57.011 | INFO | "pinned" : " 29 GB" | 20260506 08:59:57.011 | INFO | } | 20260506 08:59:57.011 | INFO | Perf report for the last 5.13912 seconds: | 20260506 08:59:57.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 08:59:57.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:57.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:57.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:57.011 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 08:59:57.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:57.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:57.011 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 08:59:57.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:57.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 08:59:57.011 | INFO | Uploaded 36GiB to GPU in 1.60219s (22.4693GiB/s) | 20260506 08:59:57.011 | INFO | Downloaded 34GiB to GPU in 1.51911s (22.3815GiB/s) | 20260506 09:00:02.150 | INFO | Analyzing chunk frames = [110080,110592) | 20260506 09:00:02.150 | INFO | MemInfo: { | 20260506 09:00:02.150 | INFO | "gpu" : " 17 GB", | 20260506 09:00:02.150 | INFO | "huge" : "", | 20260506 09:00:02.150 | INFO | "huge_shared" : " 27 GB", | 20260506 09:00:02.150 | INFO | "malloc" : " 15 GB", | 20260506 09:00:02.150 | INFO | "pinned" : " 29 GB" | 20260506 09:00:02.150 | INFO | } | 20260506 09:00:02.151 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:00:02.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:00:02.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:02.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:02.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:02.151 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260506 09:00:02.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:02.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:02.151 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 09:00:02.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:02.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:02.151 | INFO | Uploaded 36GiB to GPU in 1.6034s (22.4523GiB/s) | 20260506 09:00:02.151 | INFO | Downloaded 34GiB to GPU in 1.52094s (22.3546GiB/s) | 20260506 09:00:02.248 | INFO | Starting a round of Baz IO | 20260506 09:00:05.185 | INFO | Populating RTMetrics report | 20260506 09:00:05.185 | INFO | RTMetrics report populated | 20260506 09:00:05.185 | INFO | Opening tmp RTMetrics json file | 20260506 09:00:05.188 | INFO | Serializing RTMetrics to json | 20260506 09:00:05.216 | INFO | Serialization done, swapping tmp file now... | 20260506 09:00:05.262 | INFO | RTMetrics done | 20260506 09:00:05.295 | INFO | Finished a round of Baz IO: Wrote 1.659 GiB in 3.047 seconds | 20260506 09:00:05.295 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 61.07% / 0.33% / 38.60% | 20260506 09:00:05.295 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.4363% / 0.0013113% / 1.19209e-05% | 20260506 09:00:05.295 | INFO | Baz IO overall: | 20260506 09:00:05.295 | INFO | BytesWritten : 27897336187 | 20260506 09:00:05.295 | INFO | NumEvents : 12108339561 | 20260506 09:00:07.290 | INFO | Analyzing chunk frames = [110592,111104) | 20260506 09:00:07.290 | INFO | MemInfo: { | 20260506 09:00:07.290 | INFO | "gpu" : " 17 GB", | 20260506 09:00:07.290 | INFO | "huge" : "", | 20260506 09:00:07.290 | INFO | "huge_shared" : " 27 GB", | 20260506 09:00:07.290 | INFO | "malloc" : " 14 GB", | 20260506 09:00:07.290 | INFO | "pinned" : " 29 GB" | 20260506 09:00:07.290 | INFO | } | 20260506 09:00:07.290 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:00:07.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:00:07.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:07.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:07.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:07.290 | INFO | ANALYSIS : 58.1%, 2.9, 88.8ms, 0 | 20260506 09:00:07.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:07.290 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260506 09:00:07.290 | INFO | PRE_HQ : 57.3%, 2.1, 62.3ms, 0 | 20260506 09:00:07.290 | INFO | BAZWRITER : 58.6%, 4.1, 124.6ms, 0 | 20260506 09:00:07.290 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 09:00:07.290 | INFO | Uploaded 36GiB to GPU in 1.65042s (21.8126GiB/s) | 20260506 09:00:07.290 | INFO | Downloaded 38GiB to GPU in 1.71329s (22.1796GiB/s) | 20260506 09:00:12.429 | INFO | Analyzing chunk frames = [111104,111616) | 20260506 09:00:12.429 | INFO | MemInfo: { | 20260506 09:00:12.429 | INFO | "gpu" : " 17 GB", | 20260506 09:00:12.429 | INFO | "huge" : "", | 20260506 09:00:12.429 | INFO | "huge_shared" : " 27 GB", | 20260506 09:00:12.429 | INFO | "malloc" : " 14 GB", | 20260506 09:00:12.429 | INFO | "pinned" : " 29 GB" | 20260506 09:00:12.429 | INFO | } | 20260506 09:00:12.429 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:00:12.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:00:12.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:12.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:12.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:12.429 | INFO | ANALYSIS : 57.9%, 3.0, 90.2ms, 0 | 20260506 09:00:12.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:12.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:12.429 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260506 09:00:12.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:12.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:12.429 | INFO | Uploaded 36GiB to GPU in 1.60007s (22.4989GiB/s) | 20260506 09:00:12.429 | INFO | Downloaded 34GiB to GPU in 1.51684s (22.415GiB/s) | 20260506 09:00:17.568 | INFO | Analyzing chunk frames = [111616,112128) | 20260506 09:00:17.568 | INFO | MemInfo: { | 20260506 09:00:17.568 | INFO | "gpu" : " 17 GB", | 20260506 09:00:17.568 | INFO | "huge" : "", | 20260506 09:00:17.568 | INFO | "huge_shared" : " 27 GB", | 20260506 09:00:17.569 | INFO | "malloc" : " 14 GB", | 20260506 09:00:17.569 | INFO | "pinned" : " 29 GB" | 20260506 09:00:17.569 | INFO | } | 20260506 09:00:17.569 | INFO | Perf report for the last 5.13956 seconds: | 20260506 09:00:17.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:00:17.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:17.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:17.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:17.569 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260506 09:00:17.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:17.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:17.569 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:00:17.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:17.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:17.569 | INFO | Uploaded 36GiB to GPU in 1.60412s (22.4422GiB/s) | 20260506 09:00:17.569 | INFO | Downloaded 34GiB to GPU in 1.52148s (22.3466GiB/s) | 20260506 09:00:22.708 | INFO | Analyzing chunk frames = [112128,112640) | 20260506 09:00:22.708 | INFO | MemInfo: { | 20260506 09:00:22.708 | INFO | "gpu" : " 17 GB", | 20260506 09:00:22.708 | INFO | "huge" : "", | 20260506 09:00:22.708 | INFO | "huge_shared" : " 27 GB", | 20260506 09:00:22.708 | INFO | "malloc" : " 14 GB", | 20260506 09:00:22.708 | INFO | "pinned" : " 29 GB" | 20260506 09:00:22.708 | INFO | } | 20260506 09:00:22.708 | INFO | Perf report for the last 5.13911 seconds: | 20260506 09:00:22.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:00:22.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:22.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:22.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:22.708 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260506 09:00:22.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:22.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:22.708 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:00:22.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:22.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:22.708 | INFO | Uploaded 36GiB to GPU in 1.60358s (22.4498GiB/s) | 20260506 09:00:22.708 | INFO | Downloaded 34GiB to GPU in 1.5213s (22.3493GiB/s) | 20260506 09:00:27.847 | INFO | Analyzing chunk frames = [112640,113152) | 20260506 09:00:27.847 | INFO | MemInfo: { | 20260506 09:00:27.847 | INFO | "gpu" : " 17 GB", | 20260506 09:00:27.847 | INFO | "huge" : "", | 20260506 09:00:27.847 | INFO | "huge_shared" : " 27 GB", | 20260506 09:00:27.847 | INFO | "malloc" : " 14 GB", | 20260506 09:00:27.847 | INFO | "pinned" : " 29 GB" | 20260506 09:00:27.847 | INFO | } | 20260506 09:00:27.847 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:00:27.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:00:27.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:27.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:27.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:27.847 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 09:00:27.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:27.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:27.847 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:00:27.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:27.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:27.847 | INFO | Uploaded 36GiB to GPU in 1.6004s (22.4944GiB/s) | 20260506 09:00:27.847 | INFO | Downloaded 34GiB to GPU in 1.51827s (22.394GiB/s) | 20260506 09:00:32.986 | INFO | Analyzing chunk frames = [113152,113664) | 20260506 09:00:32.986 | INFO | MemInfo: { | 20260506 09:00:32.986 | INFO | "gpu" : " 17 GB", | 20260506 09:00:32.986 | INFO | "huge" : "", | 20260506 09:00:32.986 | INFO | "huge_shared" : " 27 GB", | 20260506 09:00:32.986 | INFO | "malloc" : " 14 GB", | 20260506 09:00:32.986 | INFO | "pinned" : " 29 GB" | 20260506 09:00:32.986 | INFO | } | 20260506 09:00:32.987 | INFO | Perf report for the last 5.13943 seconds: | 20260506 09:00:32.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:00:32.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:32.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:32.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:32.987 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:00:32.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:32.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:32.987 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 09:00:32.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:32.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:32.987 | INFO | Uploaded 36GiB to GPU in 1.60226s (22.4683GiB/s) | 20260506 09:00:32.987 | INFO | Downloaded 34GiB to GPU in 1.51963s (22.3739GiB/s) | 20260506 09:00:38.126 | INFO | Analyzing chunk frames = [113664,114176) | 20260506 09:00:38.126 | INFO | MemInfo: { | 20260506 09:00:38.126 | INFO | "gpu" : " 17 GB", | 20260506 09:00:38.126 | INFO | "huge" : "", | 20260506 09:00:38.126 | INFO | "huge_shared" : " 27 GB", | 20260506 09:00:38.126 | INFO | "malloc" : " 14 GB", | 20260506 09:00:38.126 | INFO | "pinned" : " 29 GB" | 20260506 09:00:38.126 | INFO | } | 20260506 09:00:38.126 | INFO | Perf report for the last 5.13919 seconds: | 20260506 09:00:38.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:00:38.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:38.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:38.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:38.126 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 09:00:38.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:38.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:38.126 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:00:38.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:38.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:38.126 | INFO | Uploaded 36GiB to GPU in 1.6021s (22.4705GiB/s) | 20260506 09:00:38.126 | INFO | Downloaded 34GiB to GPU in 1.51949s (22.3759GiB/s) | 20260506 09:00:43.265 | INFO | Analyzing chunk frames = [114176,114688) | 20260506 09:00:43.265 | INFO | MemInfo: { | 20260506 09:00:43.265 | INFO | "gpu" : " 17 GB", | 20260506 09:00:43.265 | INFO | "huge" : "", | 20260506 09:00:43.265 | INFO | "huge_shared" : " 27 GB", | 20260506 09:00:43.265 | INFO | "malloc" : " 15 GB", | 20260506 09:00:43.265 | INFO | "pinned" : " 29 GB" | 20260506 09:00:43.265 | INFO | } | 20260506 09:00:43.265 | INFO | Perf report for the last 5.1395 seconds: | 20260506 09:00:43.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:00:43.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:43.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:43.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:43.265 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 09:00:43.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:43.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:43.265 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 09:00:43.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:43.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:43.266 | INFO | Uploaded 36GiB to GPU in 1.60182s (22.4745GiB/s) | 20260506 09:00:43.266 | INFO | Downloaded 34GiB to GPU in 1.52075s (22.3574GiB/s) | 20260506 09:00:43.266 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:00:43.266 | INFO | Total mem usage: 15.5 GiB | 20260506 09:00:43.266 | INFO | All Time High Water Marks sum: 15.7 GiB (Savings: 1.29%) | 20260506 09:00:43.266 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -4.56%) | 20260506 09:00:43.266 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:00:43.266 | INFO | High Water Breakdown: | 20260506 09:00:43.266 | INFO | 58.4% : 8.67 GiB ( 96.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:00:43.266 | INFO | 41.5% : 6.16 GiB ( 92% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:00:43.266 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:00:43.266 | INFO | ------------------------------------------ | 20260506 09:00:43.266 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:00:43.266 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:00:43.266 | INFO | Total mem usage: 26.5 GiB | 20260506 09:00:43.266 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:00:43.266 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:00:43.266 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:00:43.266 | INFO | High Water Breakdown: | 20260506 09:00:43.266 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:00:43.266 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:00:43.266 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:00:43.266 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:00:43.266 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:00:43.266 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:00:43.266 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:00:43.266 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:00:43.266 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:00:43.266 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:00:43.266 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:00:43.266 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:00:43.266 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:00:43.266 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:00:43.266 | INFO | ------------------------------------------ | 20260506 09:00:43.266 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:00:43.266 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:00:43.266 | INFO | Total mem usage: 25.6 GiB | 20260506 09:00:43.266 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:00:43.266 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:00:43.266 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:00:43.266 | INFO | High Water Breakdown: | 20260506 09:00:43.266 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:00:43.266 | INFO | ------------------------------------------ | 20260506 09:00:43.266 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:00:43.266 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:00:43.266 | INFO | Total mem usage: 15.7 GiB | 20260506 09:00:43.266 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:00:43.266 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:00:43.266 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:00:43.266 | INFO | High Water Breakdown: | 20260506 09:00:43.266 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:00:43.266 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:00:43.266 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:00:43.266 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:00:43.266 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:00:43.266 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:00:43.266 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:00:43.266 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:00:43.266 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:00:43.266 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:00:43.266 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:00:43.266 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:00:43.266 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:00:43.266 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:00:43.266 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:00:43.266 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:00:43.266 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:00:43.266 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:00:43.266 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:00:43.266 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:00:43.266 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:00:43.266 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:00:43.266 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:00:43.266 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:00:43.266 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:00:43.266 | INFO | ------------------------------------------ | 20260506 09:00:43.266 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:00:43.266 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:00:43.266 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:00:43.266 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:00:43.267 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 09:00:43.267 | INFO | PerfReport: 0.28%, AggregateStats -- 8.65e-02 / 0.00e+00, 7.26e-03 / 0.00e+00 | 20260506 09:00:43.267 | INFO | PerfReport: 10.84%, Baseline -- 3.29e+00 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260506 09:00:43.267 | INFO | PerfReport: 3.04%, Histogram -- 9.22e-01 / 0.00e+00, 5.20e-02 / 0.00e+00 | 20260506 09:00:43.267 | INFO | PerfReport: 16.01%, DME -- 3.89e+01 / 0.00e+00, 3.22e+00 / 0.00e+00 | 20260506 09:00:43.267 | INFO | PerfReport: 56.65%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.13e-02 / 0.00e+00 | 20260506 09:00:43.267 | INFO | PerfReport: 7.63%, PulseAccumulating -- 2.32e+00 / 0.00e+00, 1.41e-01 / 0.00e+00 | 20260506 09:00:43.267 | INFO | PerfReport: 3.98%, Metrics -- 1.21e+00 / 0.00e+00, 1.88e-01 / 0.00e+00 | 20260506 09:00:43.267 | INFO | PerfReport: 1.56%, PulseSerializing -- 4.74e-01 / 0.00e+00, 6.87e-02 / 0.00e+00 | 20260506 09:00:43.267 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:00:43.267 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:00:43.267 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:00:43.267 | INFO | PerfReport: Full Execution -- 9.11e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260506 09:00:43.267 | INFO | PerfReport: 18.39%, Upload -- 1.67e+01 / 0.00e+00, 4.02e-01 / 0.00e+00 | 20260506 09:00:43.267 | INFO | PerfReport: 17.65%, Download -- 1.61e+01 / 0.00e+00, 7.21e-01 / 0.00e+00 | 20260506 09:00:43.267 | INFO | PerfReport: 33.35%, Compute -- 3.04e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 09:00:43.267 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:00:43.267 | INFO | PerfReport: 0.00%, LockDownload -- 4.90e-04 / 0.00e+00, 1.18e-04 / 0.00e+00 | 20260506 09:00:43.267 | INFO | PerfReport: 30.05%, LockCompute -- 2.74e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 09:00:43.267 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260506 09:00:43.370 | INFO | Starting a round of Baz IO | 20260506 09:00:46.301 | INFO | Populating RTMetrics report | 20260506 09:00:46.301 | INFO | RTMetrics report populated | 20260506 09:00:46.301 | INFO | Opening tmp RTMetrics json file | 20260506 09:00:46.409 | INFO | Serializing RTMetrics to json | 20260506 09:00:46.429 | INFO | Serialization done, swapping tmp file now... | 20260506 09:00:46.430 | INFO | RTMetrics done | 20260506 09:00:46.604 | INFO | Finished a round of Baz IO: Wrote 1.669 GiB in 3.233 seconds | 20260506 09:00:46.604 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 60.75% / 0.32% / 38.93% | 20260506 09:00:46.604 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.3783% / 0.00121196% / 3.57628e-05% | 20260506 09:00:46.604 | INFO | Baz IO overall: | 20260506 09:00:46.604 | INFO | BytesWritten : 29689304485 | 20260506 09:00:46.604 | INFO | NumEvents : 12981112907 | 20260506 09:00:48.404 | INFO | Analyzing chunk frames = [114688,115200) | 20260506 09:00:48.404 | INFO | MemInfo: { | 20260506 09:00:48.404 | INFO | "gpu" : " 17 GB", | 20260506 09:00:48.404 | INFO | "huge" : "", | 20260506 09:00:48.404 | INFO | "huge_shared" : " 27 GB", | 20260506 09:00:48.404 | INFO | "malloc" : " 14 GB", | 20260506 09:00:48.404 | INFO | "pinned" : " 29 GB" | 20260506 09:00:48.404 | INFO | } | 20260506 09:00:48.404 | INFO | Perf report for the last 5.13912 seconds: | 20260506 09:00:48.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:00:48.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:48.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:48.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:48.405 | INFO | ANALYSIS : 58.1%, 2.9, 88.4ms, 0 | 20260506 09:00:48.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:48.405 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 09:00:48.405 | INFO | PRE_HQ : 57.3%, 2.1, 62.7ms, 0 | 20260506 09:00:48.405 | INFO | BAZWRITER : 62.6%, 4.3, 139.3ms, 0 | 20260506 09:00:48.405 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 09:00:48.405 | INFO | Uploaded 36GiB to GPU in 1.64818s (21.8423GiB/s) | 20260506 09:00:48.405 | INFO | Downloaded 38GiB to GPU in 1.71206s (22.1954GiB/s) | 20260506 09:00:53.544 | INFO | Analyzing chunk frames = [115200,115712) | 20260506 09:00:53.544 | INFO | MemInfo: { | 20260506 09:00:53.544 | INFO | "gpu" : " 17 GB", | 20260506 09:00:53.544 | INFO | "huge" : "", | 20260506 09:00:53.544 | INFO | "huge_shared" : " 27 GB", | 20260506 09:00:53.544 | INFO | "malloc" : " 14 GB", | 20260506 09:00:53.544 | INFO | "pinned" : " 29 GB" | 20260506 09:00:53.544 | INFO | } | 20260506 09:00:53.544 | INFO | Perf report for the last 5.13967 seconds: | 20260506 09:00:53.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:00:53.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:53.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:53.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:53.544 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260506 09:00:53.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:53.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:53.544 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 09:00:53.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:53.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:53.544 | INFO | Uploaded 36GiB to GPU in 1.60045s (22.4936GiB/s) | 20260506 09:00:53.544 | INFO | Downloaded 34GiB to GPU in 1.51673s (22.4166GiB/s) | 20260506 09:00:58.683 | INFO | Analyzing chunk frames = [115712,116224) | 20260506 09:00:58.683 | INFO | MemInfo: { | 20260506 09:00:58.683 | INFO | "gpu" : " 17 GB", | 20260506 09:00:58.683 | INFO | "huge" : "", | 20260506 09:00:58.683 | INFO | "huge_shared" : " 27 GB", | 20260506 09:00:58.683 | INFO | "malloc" : " 14 GB", | 20260506 09:00:58.683 | INFO | "pinned" : " 29 GB" | 20260506 09:00:58.683 | INFO | } | 20260506 09:00:58.683 | INFO | Perf report for the last 5.13901 seconds: | 20260506 09:00:58.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:00:58.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:58.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:58.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:58.683 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260506 09:00:58.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:58.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:58.683 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:00:58.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:58.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:00:58.683 | INFO | Uploaded 36GiB to GPU in 1.60415s (22.4418GiB/s) | 20260506 09:00:58.683 | INFO | Downloaded 34GiB to GPU in 1.52172s (22.3432GiB/s) | 20260506 09:01:03.822 | INFO | Analyzing chunk frames = [116224,116736) | 20260506 09:01:03.822 | INFO | MemInfo: { | 20260506 09:01:03.822 | INFO | "gpu" : " 17 GB", | 20260506 09:01:03.823 | INFO | "huge" : "", | 20260506 09:01:03.823 | INFO | "huge_shared" : " 27 GB", | 20260506 09:01:03.823 | INFO | "malloc" : " 14 GB", | 20260506 09:01:03.823 | INFO | "pinned" : " 29 GB" | 20260506 09:01:03.823 | INFO | } | 20260506 09:01:03.823 | INFO | Perf report for the last 5.13952 seconds: | 20260506 09:01:03.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:01:03.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:03.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:03.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:03.823 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260506 09:01:03.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:03.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:03.823 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 09:01:03.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:03.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:03.823 | INFO | Uploaded 36GiB to GPU in 1.60313s (22.456GiB/s) | 20260506 09:01:03.823 | INFO | Downloaded 34GiB to GPU in 1.52163s (22.3445GiB/s) | 20260506 09:01:08.962 | INFO | Analyzing chunk frames = [116736,117248) | 20260506 09:01:08.962 | INFO | MemInfo: { | 20260506 09:01:08.962 | INFO | "gpu" : " 17 GB", | 20260506 09:01:08.962 | INFO | "huge" : "", | 20260506 09:01:08.962 | INFO | "huge_shared" : " 27 GB", | 20260506 09:01:08.962 | INFO | "malloc" : " 14 GB", | 20260506 09:01:08.962 | INFO | "pinned" : " 29 GB" | 20260506 09:01:08.962 | INFO | } | 20260506 09:01:08.962 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:01:08.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:01:08.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:08.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:08.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:08.962 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:01:08.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:08.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:08.962 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260506 09:01:08.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:08.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:08.962 | INFO | Uploaded 36GiB to GPU in 1.60208s (22.4708GiB/s) | 20260506 09:01:08.962 | INFO | Downloaded 34GiB to GPU in 1.5186s (22.389GiB/s) | 20260506 09:01:14.101 | INFO | Analyzing chunk frames = [117248,117760) | 20260506 09:01:14.101 | INFO | MemInfo: { | 20260506 09:01:14.101 | INFO | "gpu" : " 17 GB", | 20260506 09:01:14.101 | INFO | "huge" : "", | 20260506 09:01:14.101 | INFO | "huge_shared" : " 27 GB", | 20260506 09:01:14.101 | INFO | "malloc" : " 14 GB", | 20260506 09:01:14.101 | INFO | "pinned" : " 29 GB" | 20260506 09:01:14.101 | INFO | } | 20260506 09:01:14.101 | INFO | Perf report for the last 5.13956 seconds: | 20260506 09:01:14.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:01:14.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:14.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:14.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:14.102 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:01:14.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:14.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:14.102 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 09:01:14.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:14.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:14.102 | INFO | Uploaded 36GiB to GPU in 1.60016s (22.4978GiB/s) | 20260506 09:01:14.102 | INFO | Downloaded 34GiB to GPU in 1.51864s (22.3885GiB/s) | 20260506 09:01:19.240 | INFO | Analyzing chunk frames = [117760,118272) | 20260506 09:01:19.240 | INFO | MemInfo: { | 20260506 09:01:19.240 | INFO | "gpu" : " 17 GB", | 20260506 09:01:19.241 | INFO | "huge" : "", | 20260506 09:01:19.241 | INFO | "huge_shared" : " 27 GB", | 20260506 09:01:19.241 | INFO | "malloc" : " 14 GB", | 20260506 09:01:19.241 | INFO | "pinned" : " 29 GB" | 20260506 09:01:19.241 | INFO | } | 20260506 09:01:19.241 | INFO | Perf report for the last 5.13923 seconds: | 20260506 09:01:19.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:01:19.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:19.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:19.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:19.241 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:01:19.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:19.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:19.241 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 09:01:19.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:19.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:19.241 | INFO | Uploaded 36GiB to GPU in 1.6025s (22.4649GiB/s) | 20260506 09:01:19.241 | INFO | Downloaded 34GiB to GPU in 1.51986s (22.3705GiB/s) | 20260506 09:01:24.380 | INFO | Analyzing chunk frames = [118272,118784) | 20260506 09:01:24.380 | INFO | MemInfo: { | 20260506 09:01:24.380 | INFO | "gpu" : " 17 GB", | 20260506 09:01:24.380 | INFO | "huge" : "", | 20260506 09:01:24.380 | INFO | "huge_shared" : " 27 GB", | 20260506 09:01:24.380 | INFO | "malloc" : " 15 GB", | 20260506 09:01:24.380 | INFO | "pinned" : " 29 GB" | 20260506 09:01:24.380 | INFO | } | 20260506 09:01:24.380 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:01:24.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:01:24.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:24.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:24.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:24.380 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260506 09:01:24.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:24.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:24.380 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:01:24.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:24.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:24.380 | INFO | Uploaded 36GiB to GPU in 1.60186s (22.4738GiB/s) | 20260506 09:01:24.380 | INFO | Downloaded 34GiB to GPU in 1.52s (22.3685GiB/s) | 20260506 09:01:24.477 | INFO | Starting a round of Baz IO | 20260506 09:01:27.411 | INFO | Populating RTMetrics report | 20260506 09:01:27.412 | INFO | RTMetrics report populated | 20260506 09:01:27.412 | INFO | Opening tmp RTMetrics json file | 20260506 09:01:27.412 | INFO | Serializing RTMetrics to json | 20260506 09:01:27.413 | INFO | Serialization done, swapping tmp file now... | 20260506 09:01:27.414 | INFO | RTMetrics done | 20260506 09:01:27.497 | INFO | Finished a round of Baz IO: Wrote 1.677 GiB in 3.019 seconds | 20260506 09:01:27.497 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 60.44% / 0.31% / 39.25% | 20260506 09:01:27.497 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.3036% / 0.00116428% / 1.19209e-05% | 20260506 09:01:27.497 | INFO | Baz IO overall: | 20260506 09:01:27.497 | INFO | BytesWritten : 31490153806 | 20260506 09:01:27.497 | INFO | NumEvents : 13858060503 | 20260506 09:01:29.519 | INFO | Analyzing chunk frames = [118784,119296) | 20260506 09:01:29.519 | INFO | MemInfo: { | 20260506 09:01:29.519 | INFO | "gpu" : " 17 GB", | 20260506 09:01:29.519 | INFO | "huge" : "", | 20260506 09:01:29.519 | INFO | "huge_shared" : " 27 GB", | 20260506 09:01:29.519 | INFO | "malloc" : " 14 GB", | 20260506 09:01:29.519 | INFO | "pinned" : " 29 GB" | 20260506 09:01:29.519 | INFO | } | 20260506 09:01:29.520 | INFO | Perf report for the last 5.13963 seconds: | 20260506 09:01:29.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:01:29.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:29.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:29.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:29.520 | INFO | ANALYSIS : 58.0%, 3.0, 90.4ms, 0 | 20260506 09:01:29.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:29.520 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260506 09:01:29.520 | INFO | PRE_HQ : 57.2%, 2.1, 62.6ms, 0 | 20260506 09:01:29.520 | INFO | BAZWRITER : 58.3%, 4.0, 121.2ms, 0 | 20260506 09:01:29.520 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 09:01:29.520 | INFO | Uploaded 36GiB to GPU in 1.63711s (21.99GiB/s) | 20260506 09:01:29.520 | INFO | Downloaded 38GiB to GPU in 1.69859s (22.3716GiB/s) | 20260506 09:01:34.658 | INFO | Analyzing chunk frames = [119296,119808) | 20260506 09:01:34.658 | INFO | MemInfo: { | 20260506 09:01:34.659 | INFO | "gpu" : " 17 GB", | 20260506 09:01:34.659 | INFO | "huge" : "", | 20260506 09:01:34.659 | INFO | "huge_shared" : " 27 GB", | 20260506 09:01:34.659 | INFO | "malloc" : " 14 GB", | 20260506 09:01:34.659 | INFO | "pinned" : " 29 GB" | 20260506 09:01:34.659 | INFO | } | 20260506 09:01:34.659 | INFO | Perf report for the last 5.13896 seconds: | 20260506 09:01:34.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:01:34.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:34.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:34.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:34.659 | INFO | ANALYSIS : 57.9%, 3.0, 90.2ms, 0 | 20260506 09:01:34.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:34.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:34.659 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:01:34.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:34.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:34.659 | INFO | Uploaded 36GiB to GPU in 1.60068s (22.4905GiB/s) | 20260506 09:01:34.659 | INFO | Downloaded 34GiB to GPU in 1.51734s (22.4077GiB/s) | 20260506 09:01:39.798 | INFO | Analyzing chunk frames = [119808,120320) | 20260506 09:01:39.798 | INFO | MemInfo: { | 20260506 09:01:39.798 | INFO | "gpu" : " 17 GB", | 20260506 09:01:39.798 | INFO | "huge" : "", | 20260506 09:01:39.798 | INFO | "huge_shared" : " 27 GB", | 20260506 09:01:39.798 | INFO | "malloc" : " 14 GB", | 20260506 09:01:39.798 | INFO | "pinned" : " 29 GB" | 20260506 09:01:39.798 | INFO | } | 20260506 09:01:39.798 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:01:39.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:01:39.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:39.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:39.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:39.798 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260506 09:01:39.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:39.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:39.798 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:01:39.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:39.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:39.799 | INFO | Uploaded 36GiB to GPU in 1.60453s (22.4364GiB/s) | 20260506 09:01:39.799 | INFO | Downloaded 34GiB to GPU in 1.52223s (22.3357GiB/s) | 20260506 09:01:44.937 | INFO | Analyzing chunk frames = [120320,120832) | 20260506 09:01:44.937 | INFO | MemInfo: { | 20260506 09:01:44.937 | INFO | "gpu" : " 17 GB", | 20260506 09:01:44.937 | INFO | "huge" : "", | 20260506 09:01:44.937 | INFO | "huge_shared" : " 27 GB", | 20260506 09:01:44.937 | INFO | "malloc" : " 14 GB", | 20260506 09:01:44.937 | INFO | "pinned" : " 29 GB" | 20260506 09:01:44.937 | INFO | } | 20260506 09:01:44.938 | INFO | Perf report for the last 5.13947 seconds: | 20260506 09:01:44.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:01:44.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:44.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:44.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:44.938 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260506 09:01:44.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:44.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:44.938 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260506 09:01:44.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:44.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:44.938 | INFO | Uploaded 36GiB to GPU in 1.6033s (22.4537GiB/s) | 20260506 09:01:44.938 | INFO | Downloaded 34GiB to GPU in 1.52044s (22.3619GiB/s) | 20260506 09:01:50.077 | INFO | Analyzing chunk frames = [120832,121344) | 20260506 09:01:50.077 | INFO | MemInfo: { | 20260506 09:01:50.077 | INFO | "gpu" : " 17 GB", | 20260506 09:01:50.077 | INFO | "huge" : "", | 20260506 09:01:50.077 | INFO | "huge_shared" : " 27 GB", | 20260506 09:01:50.077 | INFO | "malloc" : " 14 GB", | 20260506 09:01:50.077 | INFO | "pinned" : " 29 GB" | 20260506 09:01:50.077 | INFO | } | 20260506 09:01:50.077 | INFO | Perf report for the last 5.13914 seconds: | 20260506 09:01:50.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:01:50.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:50.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:50.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:50.077 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:01:50.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:50.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:50.077 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260506 09:01:50.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:50.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:50.077 | INFO | Uploaded 36GiB to GPU in 1.60219s (22.4692GiB/s) | 20260506 09:01:50.077 | INFO | Downloaded 34GiB to GPU in 1.51871s (22.3874GiB/s) | 20260506 09:01:55.216 | INFO | Analyzing chunk frames = [121344,121856) | 20260506 09:01:55.216 | INFO | MemInfo: { | 20260506 09:01:55.216 | INFO | "gpu" : " 17 GB", | 20260506 09:01:55.216 | INFO | "huge" : "", | 20260506 09:01:55.216 | INFO | "huge_shared" : " 27 GB", | 20260506 09:01:55.216 | INFO | "malloc" : " 14 GB", | 20260506 09:01:55.216 | INFO | "pinned" : " 29 GB" | 20260506 09:01:55.216 | INFO | } | 20260506 09:01:55.216 | INFO | Perf report for the last 5.13919 seconds: | 20260506 09:01:55.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:01:55.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:55.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:55.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:55.216 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:01:55.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:55.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:55.216 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:01:55.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:55.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:01:55.216 | INFO | Uploaded 36GiB to GPU in 1.60224s (22.4685GiB/s) | 20260506 09:01:55.216 | INFO | Downloaded 34GiB to GPU in 1.51918s (22.3805GiB/s) | 20260506 09:02:00.355 | INFO | Analyzing chunk frames = [121856,122368) | 20260506 09:02:00.355 | INFO | MemInfo: { | 20260506 09:02:00.355 | INFO | "gpu" : " 17 GB", | 20260506 09:02:00.355 | INFO | "huge" : "", | 20260506 09:02:00.355 | INFO | "huge_shared" : " 27 GB", | 20260506 09:02:00.355 | INFO | "malloc" : " 14 GB", | 20260506 09:02:00.355 | INFO | "pinned" : " 29 GB" | 20260506 09:02:00.355 | INFO | } | 20260506 09:02:00.355 | INFO | Perf report for the last 5.13952 seconds: | 20260506 09:02:00.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:02:00.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:00.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:00.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:00.356 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:02:00.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:00.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:00.356 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:02:00.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:00.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:00.356 | INFO | Uploaded 36GiB to GPU in 1.60292s (22.459GiB/s) | 20260506 09:02:00.356 | INFO | Downloaded 34GiB to GPU in 1.5208s (22.3566GiB/s) | 20260506 09:02:05.495 | INFO | Analyzing chunk frames = [122368,122880) | 20260506 09:02:05.495 | INFO | MemInfo: { | 20260506 09:02:05.495 | INFO | "gpu" : " 17 GB", | 20260506 09:02:05.495 | INFO | "huge" : "", | 20260506 09:02:05.495 | INFO | "huge_shared" : " 27 GB", | 20260506 09:02:05.495 | INFO | "malloc" : " 15 GB", | 20260506 09:02:05.495 | INFO | "pinned" : " 29 GB" | 20260506 09:02:05.495 | INFO | } | 20260506 09:02:05.495 | INFO | Perf report for the last 5.1392 seconds: | 20260506 09:02:05.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:02:05.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:05.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:05.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:05.495 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 09:02:05.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:05.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:05.495 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:02:05.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:05.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:05.495 | INFO | Uploaded 36GiB to GPU in 1.60353s (22.4504GiB/s) | 20260506 09:02:05.495 | INFO | Downloaded 34GiB to GPU in 1.52056s (22.3603GiB/s) | 20260506 09:02:05.495 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:02:05.495 | INFO | Total mem usage: 15.5 GiB | 20260506 09:02:05.495 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.23%) | 20260506 09:02:05.495 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -0.207%) | 20260506 09:02:05.495 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:02:05.495 | INFO | High Water Breakdown: | 20260506 09:02:05.495 | INFO | 59.2% : 9.17 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:02:05.495 | INFO | 40.7% : 6.31 GiB ( 94.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:02:05.495 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:02:05.495 | INFO | ------------------------------------------ | 20260506 09:02:05.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:02:05.495 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:02:05.495 | INFO | Total mem usage: 26.5 GiB | 20260506 09:02:05.495 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:02:05.495 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:02:05.495 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:02:05.495 | INFO | High Water Breakdown: | 20260506 09:02:05.495 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:02:05.495 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:02:05.495 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:02:05.495 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:02:05.495 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:02:05.495 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:02:05.495 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:02:05.495 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:02:05.495 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:02:05.495 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:02:05.495 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:02:05.495 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:02:05.495 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:02:05.495 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:02:05.495 | INFO | ------------------------------------------ | 20260506 09:02:05.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:02:05.495 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:02:05.495 | INFO | Total mem usage: 25.6 GiB | 20260506 09:02:05.495 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:02:05.495 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:02:05.495 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:02:05.495 | INFO | High Water Breakdown: | 20260506 09:02:05.495 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:02:05.495 | INFO | ------------------------------------------ | 20260506 09:02:05.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:02:05.495 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:02:05.495 | INFO | Total mem usage: 15.7 GiB | 20260506 09:02:05.495 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:02:05.495 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:02:05.495 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:02:05.496 | INFO | High Water Breakdown: | 20260506 09:02:05.496 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:02:05.496 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:02:05.496 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:02:05.496 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:02:05.496 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:02:05.496 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:02:05.496 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:02:05.496 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:02:05.496 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:02:05.496 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:02:05.496 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:02:05.496 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:02:05.496 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:02:05.496 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:02:05.496 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:02:05.496 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:02:05.496 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:02:05.496 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:02:05.496 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:02:05.496 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:02:05.496 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:02:05.496 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:02:05.496 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:02:05.496 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:02:05.496 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:02:05.496 | INFO | ------------------------------------------ | 20260506 09:02:05.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:02:05.496 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:02:05.496 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:02:05.496 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:02:05.496 | INFO | PerfReport: Full Execution -- 3.04e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 09:02:05.496 | INFO | PerfReport: 0.28%, AggregateStats -- 8.64e-02 / 0.00e+00, 6.36e-03 / 0.00e+00 | 20260506 09:02:05.496 | INFO | PerfReport: 10.85%, Baseline -- 3.29e+00 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260506 09:02:05.496 | INFO | PerfReport: 3.04%, Histogram -- 9.22e-01 / 0.00e+00, 5.18e-02 / 0.00e+00 | 20260506 09:02:05.496 | INFO | PerfReport: 16.01%, DME -- 3.89e+01 / 0.00e+00, 3.23e+00 / 0.00e+00 | 20260506 09:02:05.496 | INFO | PerfReport: 56.67%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.24e-02 / 0.00e+00 | 20260506 09:02:05.496 | INFO | PerfReport: 7.62%, PulseAccumulating -- 2.31e+00 / 0.00e+00, 1.41e-01 / 0.00e+00 | 20260506 09:02:05.496 | INFO | PerfReport: 3.97%, Metrics -- 1.20e+00 / 0.00e+00, 1.88e-01 / 0.00e+00 | 20260506 09:02:05.496 | INFO | PerfReport: 1.55%, PulseSerializing -- 4.72e-01 / 0.00e+00, 6.89e-02 / 0.00e+00 | 20260506 09:02:05.496 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:02:05.496 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:02:05.496 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:02:05.496 | INFO | PerfReport: Full Execution -- 9.10e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260506 09:02:05.496 | INFO | PerfReport: 18.39%, Upload -- 1.67e+01 / 0.00e+00, 3.99e-01 / 0.00e+00 | 20260506 09:02:05.496 | INFO | PerfReport: 17.66%, Download -- 1.61e+01 / 0.00e+00, 7.20e-01 / 0.00e+00 | 20260506 09:02:05.496 | INFO | PerfReport: 33.35%, Compute -- 3.04e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 09:02:05.496 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:02:05.496 | INFO | PerfReport: 0.00%, LockDownload -- 4.82e-04 / 0.00e+00, 1.18e-04 / 0.00e+00 | 20260506 09:02:05.496 | INFO | PerfReport: 30.03%, LockCompute -- 2.73e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 09:02:05.496 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260506 09:02:05.592 | INFO | Starting a round of Baz IO | 20260506 09:02:08.523 | INFO | Populating RTMetrics report | 20260506 09:02:08.524 | INFO | RTMetrics report populated | 20260506 09:02:08.524 | INFO | Opening tmp RTMetrics json file | 20260506 09:02:08.525 | INFO | Serializing RTMetrics to json | 20260506 09:02:08.526 | INFO | Serialization done, swapping tmp file now... | 20260506 09:02:08.526 | INFO | RTMetrics done | 20260506 09:02:08.615 | INFO | Finished a round of Baz IO: Wrote 1.684 GiB in 3.022 seconds | 20260506 09:02:08.615 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 60.15% / 0.29% / 39.56% | 20260506 09:02:08.615 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.237% / 0.000913938% / 1.98682e-05% | 20260506 09:02:08.615 | INFO | Baz IO overall: | 20260506 09:02:08.615 | INFO | BytesWritten : 33298486114 | 20260506 09:02:08.615 | INFO | NumEvents : 14738504134 | 20260506 09:02:10.634 | INFO | Analyzing chunk frames = [122880,123392) | 20260506 09:02:10.634 | INFO | MemInfo: { | 20260506 09:02:10.634 | INFO | "gpu" : " 17 GB", | 20260506 09:02:10.634 | INFO | "huge" : "", | 20260506 09:02:10.634 | INFO | "huge_shared" : " 27 GB", | 20260506 09:02:10.634 | INFO | "malloc" : " 14 GB", | 20260506 09:02:10.634 | INFO | "pinned" : " 29 GB" | 20260506 09:02:10.634 | INFO | } | 20260506 09:02:10.634 | INFO | Perf report for the last 5.13946 seconds: | 20260506 09:02:10.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:02:10.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:10.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:10.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:10.634 | INFO | ANALYSIS : 58.0%, 3.0, 90.1ms, 0 | 20260506 09:02:10.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:10.634 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260506 09:02:10.634 | INFO | PRE_HQ : 57.1%, 2.1, 62.4ms, 0 | 20260506 09:02:10.634 | INFO | BAZWRITER : 58.4%, 4.0, 120.1ms, 0 | 20260506 09:02:10.634 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 09:02:10.634 | INFO | Uploaded 36GiB to GPU in 1.63838s (21.9729GiB/s) | 20260506 09:02:10.634 | INFO | Downloaded 38GiB to GPU in 1.6998s (22.3555GiB/s) | 20260506 09:02:15.774 | INFO | Analyzing chunk frames = [123392,123904) | 20260506 09:02:15.774 | INFO | MemInfo: { | 20260506 09:02:15.774 | INFO | "gpu" : " 17 GB", | 20260506 09:02:15.774 | INFO | "huge" : "", | 20260506 09:02:15.774 | INFO | "huge_shared" : " 27 GB", | 20260506 09:02:15.774 | INFO | "malloc" : " 14 GB", | 20260506 09:02:15.774 | INFO | "pinned" : " 29 GB" | 20260506 09:02:15.774 | INFO | } | 20260506 09:02:15.774 | INFO | Perf report for the last 5.13965 seconds: | 20260506 09:02:15.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:02:15.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:15.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:15.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:15.774 | INFO | ANALYSIS : 57.9%, 3.0, 90.3ms, 0 | 20260506 09:02:15.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:15.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:15.774 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:02:15.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:15.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:15.774 | INFO | Uploaded 36GiB to GPU in 1.60018s (22.4975GiB/s) | 20260506 09:02:15.774 | INFO | Downloaded 34GiB to GPU in 1.51637s (22.4219GiB/s) | 20260506 09:02:20.913 | INFO | Analyzing chunk frames = [123904,124416) | 20260506 09:02:20.913 | INFO | MemInfo: { | 20260506 09:02:20.913 | INFO | "gpu" : " 17 GB", | 20260506 09:02:20.913 | INFO | "huge" : "", | 20260506 09:02:20.913 | INFO | "huge_shared" : " 27 GB", | 20260506 09:02:20.913 | INFO | "malloc" : " 14 GB", | 20260506 09:02:20.913 | INFO | "pinned" : " 29 GB" | 20260506 09:02:20.913 | INFO | } | 20260506 09:02:20.913 | INFO | Perf report for the last 5.13916 seconds: | 20260506 09:02:20.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:02:20.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:20.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:20.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:20.913 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260506 09:02:20.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:20.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:20.913 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:02:20.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:20.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:20.914 | INFO | Uploaded 36GiB to GPU in 1.60425s (22.4403GiB/s) | 20260506 09:02:20.914 | INFO | Downloaded 34GiB to GPU in 1.5221s (22.3375GiB/s) | 20260506 09:02:26.052 | INFO | Analyzing chunk frames = [124416,124928) | 20260506 09:02:26.052 | INFO | MemInfo: { | 20260506 09:02:26.052 | INFO | "gpu" : " 17 GB", | 20260506 09:02:26.052 | INFO | "huge" : "", | 20260506 09:02:26.052 | INFO | "huge_shared" : " 27 GB", | 20260506 09:02:26.052 | INFO | "malloc" : " 14 GB", | 20260506 09:02:26.052 | INFO | "pinned" : " 29 GB" | 20260506 09:02:26.052 | INFO | } | 20260506 09:02:26.052 | INFO | Perf report for the last 5.13923 seconds: | 20260506 09:02:26.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:02:26.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:26.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:26.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:26.052 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260506 09:02:26.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:26.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:26.052 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:02:26.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:26.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:26.053 | INFO | Uploaded 36GiB to GPU in 1.60319s (22.4553GiB/s) | 20260506 09:02:26.053 | INFO | Downloaded 34GiB to GPU in 1.52108s (22.3525GiB/s) | 20260506 09:02:31.191 | INFO | Analyzing chunk frames = [124928,125440) | 20260506 09:02:31.191 | INFO | MemInfo: { | 20260506 09:02:31.191 | INFO | "gpu" : " 17 GB", | 20260506 09:02:31.191 | INFO | "huge" : "", | 20260506 09:02:31.191 | INFO | "huge_shared" : " 27 GB", | 20260506 09:02:31.191 | INFO | "malloc" : " 14 GB", | 20260506 09:02:31.191 | INFO | "pinned" : " 29 GB" | 20260506 09:02:31.191 | INFO | } | 20260506 09:02:31.192 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:02:31.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:02:31.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:31.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:31.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:31.192 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:02:31.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:31.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:31.192 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 09:02:31.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:31.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:31.192 | INFO | Uploaded 36GiB to GPU in 1.60257s (22.4639GiB/s) | 20260506 09:02:31.192 | INFO | Downloaded 34GiB to GPU in 1.51921s (22.3801GiB/s) | 20260506 09:02:36.331 | INFO | Analyzing chunk frames = [125440,125952) | 20260506 09:02:36.331 | INFO | MemInfo: { | 20260506 09:02:36.331 | INFO | "gpu" : " 17 GB", | 20260506 09:02:36.331 | INFO | "huge" : "", | 20260506 09:02:36.331 | INFO | "huge_shared" : " 27 GB", | 20260506 09:02:36.331 | INFO | "malloc" : " 14 GB", | 20260506 09:02:36.331 | INFO | "pinned" : " 29 GB" | 20260506 09:02:36.331 | INFO | } | 20260506 09:02:36.331 | INFO | Perf report for the last 5.13947 seconds: | 20260506 09:02:36.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:02:36.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:36.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:36.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:36.331 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:02:36.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:36.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:36.331 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 09:02:36.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:36.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:36.331 | INFO | Uploaded 36GiB to GPU in 1.6006s (22.4915GiB/s) | 20260506 09:02:36.331 | INFO | Downloaded 34GiB to GPU in 1.51944s (22.3766GiB/s) | 20260506 09:02:41.470 | INFO | Analyzing chunk frames = [125952,126464) | 20260506 09:02:41.470 | INFO | MemInfo: { | 20260506 09:02:41.470 | INFO | "gpu" : " 17 GB", | 20260506 09:02:41.470 | INFO | "huge" : "", | 20260506 09:02:41.470 | INFO | "huge_shared" : " 27 GB", | 20260506 09:02:41.470 | INFO | "malloc" : " 14 GB", | 20260506 09:02:41.470 | INFO | "pinned" : " 29 GB" | 20260506 09:02:41.470 | INFO | } | 20260506 09:02:41.470 | INFO | Perf report for the last 5.1391 seconds: | 20260506 09:02:41.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:02:41.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:41.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:41.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:41.470 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 09:02:41.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:41.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:41.470 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 09:02:41.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:41.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:41.470 | INFO | Uploaded 36GiB to GPU in 1.60263s (22.4631GiB/s) | 20260506 09:02:41.470 | INFO | Downloaded 34GiB to GPU in 1.51967s (22.3732GiB/s) | 20260506 09:02:46.609 | INFO | Analyzing chunk frames = [126464,126976) | 20260506 09:02:46.609 | INFO | MemInfo: { | 20260506 09:02:46.609 | INFO | "gpu" : " 17 GB", | 20260506 09:02:46.609 | INFO | "huge" : "", | 20260506 09:02:46.610 | INFO | "huge_shared" : " 27 GB", | 20260506 09:02:46.610 | INFO | "malloc" : " 14 GB", | 20260506 09:02:46.610 | INFO | "pinned" : " 29 GB" | 20260506 09:02:46.610 | INFO | } | 20260506 09:02:46.610 | INFO | Perf report for the last 5.13946 seconds: | 20260506 09:02:46.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:02:46.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:46.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:46.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:46.610 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 09:02:46.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:46.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:46.610 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:02:46.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:46.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:46.610 | INFO | Uploaded 36GiB to GPU in 1.60319s (22.4553GiB/s) | 20260506 09:02:46.610 | INFO | Downloaded 34GiB to GPU in 1.52148s (22.3467GiB/s) | 20260506 09:02:46.707 | INFO | Starting a round of Baz IO | 20260506 09:02:49.639 | INFO | Populating RTMetrics report | 20260506 09:02:49.639 | INFO | RTMetrics report populated | 20260506 09:02:49.639 | INFO | Opening tmp RTMetrics json file | 20260506 09:02:49.641 | INFO | Serializing RTMetrics to json | 20260506 09:02:49.645 | INFO | Serialization done, swapping tmp file now... | 20260506 09:02:49.645 | INFO | RTMetrics done | 20260506 09:02:49.837 | INFO | Finished a round of Baz IO: Wrote 1.691 GiB in 3.13 seconds | 20260506 09:02:49.837 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 59.88% / 0.27% / 39.85% | 20260506 09:02:49.837 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.1758% / 0.00109673% / 1.98682e-05% | 20260506 09:02:49.837 | INFO | Baz IO overall: | 20260506 09:02:49.838 | INFO | BytesWritten : 35114303456 | 20260506 09:02:49.838 | INFO | NumEvents : 15622438586 | 20260506 09:02:51.749 | INFO | Analyzing chunk frames = [126976,127488) | 20260506 09:02:51.749 | INFO | MemInfo: { | 20260506 09:02:51.749 | INFO | "gpu" : " 17 GB", | 20260506 09:02:51.749 | INFO | "huge" : "", | 20260506 09:02:51.749 | INFO | "huge_shared" : " 27 GB", | 20260506 09:02:51.749 | INFO | "malloc" : " 14 GB", | 20260506 09:02:51.749 | INFO | "pinned" : " 29 GB" | 20260506 09:02:51.749 | INFO | } | 20260506 09:02:51.749 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:02:51.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:02:51.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:51.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:51.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:51.749 | INFO | ANALYSIS : 58.0%, 2.9, 88.3ms, 0 | 20260506 09:02:51.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:51.749 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 09:02:51.749 | INFO | PRE_HQ : 57.4%, 2.1, 62.1ms, 0 | 20260506 09:02:51.749 | INFO | BAZWRITER : 60.3%, 4.3, 134.8ms, 0 | 20260506 09:02:51.749 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 09:02:51.749 | INFO | Uploaded 36GiB to GPU in 1.64915s (21.8295GiB/s) | 20260506 09:02:51.749 | INFO | Downloaded 38GiB to GPU in 1.71345s (22.1775GiB/s) | 20260506 09:02:56.888 | INFO | Analyzing chunk frames = [127488,128000) | 20260506 09:02:56.888 | INFO | MemInfo: { | 20260506 09:02:56.888 | INFO | "gpu" : " 17 GB", | 20260506 09:02:56.888 | INFO | "huge" : "", | 20260506 09:02:56.888 | INFO | "huge_shared" : " 27 GB", | 20260506 09:02:56.888 | INFO | "malloc" : " 14 GB", | 20260506 09:02:56.888 | INFO | "pinned" : " 29 GB" | 20260506 09:02:56.888 | INFO | } | 20260506 09:02:56.888 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:02:56.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:02:56.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:56.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:56.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:56.888 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260506 09:02:56.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:56.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:56.888 | INFO | PRE_HQ : 30.7%, 1.0, 16.1ms, 0 | 20260506 09:02:56.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:56.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:02:56.888 | INFO | Uploaded 36GiB to GPU in 1.60078s (22.489GiB/s) | 20260506 09:02:56.888 | INFO | Downloaded 34GiB to GPU in 1.51801s (22.3978GiB/s) | 20260506 09:03:02.027 | INFO | Analyzing chunk frames = [128000,128512) | 20260506 09:03:02.027 | INFO | MemInfo: { | 20260506 09:03:02.027 | INFO | "gpu" : " 17 GB", | 20260506 09:03:02.028 | INFO | "huge" : "", | 20260506 09:03:02.028 | INFO | "huge_shared" : " 27 GB", | 20260506 09:03:02.028 | INFO | "malloc" : " 14 GB", | 20260506 09:03:02.028 | INFO | "pinned" : " 29 GB" | 20260506 09:03:02.028 | INFO | } | 20260506 09:03:02.028 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:03:02.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:03:02.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:02.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:02.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:02.028 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260506 09:03:02.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:02.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:02.028 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:03:02.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:02.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:02.028 | INFO | Uploaded 36GiB to GPU in 1.60365s (22.4488GiB/s) | 20260506 09:03:02.028 | INFO | Downloaded 34GiB to GPU in 1.52143s (22.3475GiB/s) | 20260506 09:03:07.167 | INFO | Analyzing chunk frames = [128512,129024) | 20260506 09:03:07.167 | INFO | MemInfo: { | 20260506 09:03:07.167 | INFO | "gpu" : " 17 GB", | 20260506 09:03:07.167 | INFO | "huge" : "", | 20260506 09:03:07.167 | INFO | "huge_shared" : " 27 GB", | 20260506 09:03:07.167 | INFO | "malloc" : " 14 GB", | 20260506 09:03:07.167 | INFO | "pinned" : " 29 GB" | 20260506 09:03:07.167 | INFO | } | 20260506 09:03:07.167 | INFO | Perf report for the last 5.13937 seconds: | 20260506 09:03:07.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:03:07.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:07.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:07.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:07.167 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260506 09:03:07.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:07.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:07.167 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 09:03:07.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:07.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:07.167 | INFO | Uploaded 36GiB to GPU in 1.60305s (22.4572GiB/s) | 20260506 09:03:07.167 | INFO | Downloaded 34GiB to GPU in 1.52162s (22.3446GiB/s) | 20260506 09:03:12.306 | INFO | Analyzing chunk frames = [129024,129536) | 20260506 09:03:12.306 | INFO | MemInfo: { | 20260506 09:03:12.306 | INFO | "gpu" : " 17 GB", | 20260506 09:03:12.306 | INFO | "huge" : "", | 20260506 09:03:12.306 | INFO | "huge_shared" : " 27 GB", | 20260506 09:03:12.306 | INFO | "malloc" : " 14 GB", | 20260506 09:03:12.306 | INFO | "pinned" : " 29 GB" | 20260506 09:03:12.306 | INFO | } | 20260506 09:03:12.307 | INFO | Perf report for the last 5.13951 seconds: | 20260506 09:03:12.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:03:12.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:12.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:12.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:12.307 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:03:12.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:12.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:12.307 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 09:03:12.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:12.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:12.307 | INFO | Uploaded 36GiB to GPU in 1.60168s (22.4765GiB/s) | 20260506 09:03:12.307 | INFO | Downloaded 34GiB to GPU in 1.51908s (22.382GiB/s) | 20260506 09:03:17.446 | INFO | Analyzing chunk frames = [129536,130048) | 20260506 09:03:17.446 | INFO | MemInfo: { | 20260506 09:03:17.446 | INFO | "gpu" : " 17 GB", | 20260506 09:03:17.446 | INFO | "huge" : "", | 20260506 09:03:17.446 | INFO | "huge_shared" : " 27 GB", | 20260506 09:03:17.446 | INFO | "malloc" : " 14 GB", | 20260506 09:03:17.446 | INFO | "pinned" : " 29 GB" | 20260506 09:03:17.446 | INFO | } | 20260506 09:03:17.446 | INFO | Perf report for the last 5.13947 seconds: | 20260506 09:03:17.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:03:17.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:17.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:17.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:17.446 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:03:17.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:17.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:17.446 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 09:03:17.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:17.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:17.447 | INFO | Uploaded 36GiB to GPU in 1.60308s (22.4567GiB/s) | 20260506 09:03:17.447 | INFO | Downloaded 34GiB to GPU in 1.52056s (22.3602GiB/s) | 20260506 09:03:22.585 | INFO | Analyzing chunk frames = [130048,130560) | 20260506 09:03:22.585 | INFO | MemInfo: { | 20260506 09:03:22.585 | INFO | "gpu" : " 17 GB", | 20260506 09:03:22.585 | INFO | "huge" : "", | 20260506 09:03:22.585 | INFO | "huge_shared" : " 27 GB", | 20260506 09:03:22.585 | INFO | "malloc" : " 14 GB", | 20260506 09:03:22.585 | INFO | "pinned" : " 29 GB" | 20260506 09:03:22.585 | INFO | } | 20260506 09:03:22.585 | INFO | Perf report for the last 5.13895 seconds: | 20260506 09:03:22.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:03:22.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:22.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:22.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:22.585 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:03:22.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:22.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:22.585 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:03:22.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:22.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:22.585 | INFO | Uploaded 36GiB to GPU in 1.6026s (22.4636GiB/s) | 20260506 09:03:22.585 | INFO | Downloaded 34GiB to GPU in 1.52004s (22.3679GiB/s) | 20260506 09:03:27.724 | INFO | Analyzing chunk frames = [130560,131072) | 20260506 09:03:27.724 | INFO | MemInfo: { | 20260506 09:03:27.724 | INFO | "gpu" : " 17 GB", | 20260506 09:03:27.724 | INFO | "huge" : "", | 20260506 09:03:27.724 | INFO | "huge_shared" : " 27 GB", | 20260506 09:03:27.724 | INFO | "malloc" : " 14 GB", | 20260506 09:03:27.725 | INFO | "pinned" : " 29 GB" | 20260506 09:03:27.725 | INFO | } | 20260506 09:03:27.725 | INFO | Perf report for the last 5.13969 seconds: | 20260506 09:03:27.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:03:27.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:27.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:27.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:27.725 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 09:03:27.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:27.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:27.725 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:03:27.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:27.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:27.725 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:03:27.725 | INFO | Total mem usage: 15.5 GiB | 20260506 09:03:27.725 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.23%) | 20260506 09:03:27.725 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -2.48%) | 20260506 09:03:27.725 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:03:27.725 | INFO | High Water Breakdown: | 20260506 09:03:27.725 | INFO | 58.8% : 8.92 GiB ( 97.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:03:27.725 | INFO | 41% : 6.21 GiB ( 92.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:03:27.725 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:03:27.725 | INFO | ------------------------------------------ | 20260506 09:03:27.725 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:03:27.725 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:03:27.725 | INFO | Total mem usage: 26.5 GiB | 20260506 09:03:27.725 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:03:27.725 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:03:27.725 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:03:27.725 | INFO | High Water Breakdown: | 20260506 09:03:27.725 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:03:27.725 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:03:27.725 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:03:27.725 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:03:27.725 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:03:27.725 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:03:27.725 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:03:27.725 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:03:27.725 | INFO | Uploaded 36GiB to GPU in 1.60338s (22.4525GiB/s) | 20260506 09:03:27.725 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:03:27.725 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:03:27.725 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:03:27.725 | INFO | Downloaded 34GiB to GPU in 1.5205s (22.361GiB/s) | 20260506 09:03:27.725 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:03:27.725 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:03:27.725 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:03:27.725 | INFO | ------------------------------------------ | 20260506 09:03:27.725 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:03:27.725 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:03:27.725 | INFO | Total mem usage: 25.6 GiB | 20260506 09:03:27.725 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:03:27.725 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:03:27.725 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:03:27.725 | INFO | High Water Breakdown: | 20260506 09:03:27.725 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:03:27.725 | INFO | ------------------------------------------ | 20260506 09:03:27.725 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:03:27.725 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:03:27.725 | INFO | Total mem usage: 15.7 GiB | 20260506 09:03:27.725 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:03:27.725 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:03:27.725 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:03:27.725 | INFO | High Water Breakdown: | 20260506 09:03:27.725 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:03:27.725 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:03:27.725 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:03:27.725 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:03:27.725 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:03:27.725 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:03:27.725 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:03:27.725 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:03:27.725 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:03:27.725 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:03:27.725 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:03:27.725 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:03:27.725 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:03:27.725 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:03:27.725 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:03:27.725 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:03:27.725 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:03:27.725 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:03:27.725 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:03:27.725 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:03:27.726 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:03:27.726 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:03:27.726 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:03:27.726 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:03:27.726 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:03:27.726 | INFO | ------------------------------------------ | 20260506 09:03:27.726 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:03:27.726 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:03:27.726 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:03:27.726 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:03:27.726 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 09:03:27.726 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / 0.00e+00, 4.75e-03 / 0.00e+00 | 20260506 09:03:27.726 | INFO | PerfReport: 10.85%, Baseline -- 3.29e+00 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260506 09:03:27.726 | INFO | PerfReport: 3.04%, Histogram -- 9.22e-01 / 0.00e+00, 5.22e-02 / 0.00e+00 | 20260506 09:03:27.726 | INFO | PerfReport: 16.02%, DME -- 3.89e+01 / 0.00e+00, 3.20e+00 / 0.00e+00 | 20260506 09:03:27.726 | INFO | PerfReport: 56.69%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.13e-02 / 0.00e+00 | 20260506 09:03:27.726 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.31e+00 / 0.00e+00, 1.42e-01 / 0.00e+00 | 20260506 09:03:27.726 | INFO | PerfReport: 3.96%, Metrics -- 1.20e+00 / 0.00e+00, 1.88e-01 / 0.00e+00 | 20260506 09:03:27.726 | INFO | PerfReport: 1.55%, PulseSerializing -- 4.71e-01 / 0.00e+00, 6.82e-02 / 0.00e+00 | 20260506 09:03:27.726 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:03:27.726 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:03:27.726 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:03:27.726 | INFO | PerfReport: Full Execution -- 9.10e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260506 09:03:27.726 | INFO | PerfReport: 18.40%, Upload -- 1.67e+01 / 0.00e+00, 3.98e-01 / 0.00e+00 | 20260506 09:03:27.726 | INFO | PerfReport: 17.67%, Download -- 1.61e+01 / 0.00e+00, 7.16e-01 / 0.00e+00 | 20260506 09:03:27.726 | INFO | PerfReport: 33.35%, Compute -- 3.03e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 09:03:27.726 | INFO | PerfReport: 0.55%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:03:27.726 | INFO | PerfReport: 0.00%, LockDownload -- 4.87e-04 / 0.00e+00, 1.85e-04 / 0.00e+00 | 20260506 09:03:27.726 | INFO | PerfReport: 30.01%, LockCompute -- 2.73e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 09:03:27.726 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.52e-02 / 0.00e+00 | 20260506 09:03:27.824 | INFO | Starting a round of Baz IO | 20260506 09:03:30.755 | INFO | Populating RTMetrics report | 20260506 09:03:30.756 | INFO | RTMetrics report populated | 20260506 09:03:30.756 | INFO | Opening tmp RTMetrics json file | 20260506 09:03:30.781 | INFO | Serializing RTMetrics to json | 20260506 09:03:30.784 | INFO | Serialization done, swapping tmp file now... | 20260506 09:03:30.785 | INFO | RTMetrics done | 20260506 09:03:30.917 | INFO | Finished a round of Baz IO: Wrote 1.699 GiB in 3.093 seconds | 20260506 09:03:30.917 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 59.64% / 0.25% / 40.12% | 20260506 09:03:30.917 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.1396% / 0.000969569% / 3.17891e-05% | 20260506 09:03:30.917 | INFO | Baz IO overall: | 20260506 09:03:30.917 | INFO | BytesWritten : 36938269955 | 20260506 09:03:30.917 | INFO | NumEvents : 16510248150 | 20260506 09:03:32.864 | INFO | Analyzing chunk frames = [131072,131584) | 20260506 09:03:32.864 | INFO | MemInfo: { | 20260506 09:03:32.864 | INFO | "gpu" : " 17 GB", | 20260506 09:03:32.864 | INFO | "huge" : "", | 20260506 09:03:32.864 | INFO | "huge_shared" : " 27 GB", | 20260506 09:03:32.864 | INFO | "malloc" : " 14 GB", | 20260506 09:03:32.864 | INFO | "pinned" : " 29 GB" | 20260506 09:03:32.864 | INFO | } | 20260506 09:03:32.864 | INFO | Perf report for the last 5.13909 seconds: | 20260506 09:03:32.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:03:32.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:32.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:32.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:32.864 | INFO | ANALYSIS : 58.0%, 2.9, 89.0ms, 0 | 20260506 09:03:32.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:32.864 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 09:03:32.864 | INFO | PRE_HQ : 57.2%, 2.1, 62.7ms, 0 | 20260506 09:03:32.864 | INFO | BAZWRITER : 59.8%, 4.2, 128.4ms, 0 | 20260506 09:03:32.864 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 09:03:32.864 | INFO | Uploaded 36GiB to GPU in 1.64377s (21.9009GiB/s) | 20260506 09:03:32.864 | INFO | Downloaded 38GiB to GPU in 1.70639s (22.2692GiB/s) | 20260506 09:03:38.003 | INFO | Analyzing chunk frames = [131584,132096) | 20260506 09:03:38.003 | INFO | MemInfo: { | 20260506 09:03:38.003 | INFO | "gpu" : " 17 GB", | 20260506 09:03:38.003 | INFO | "huge" : "", | 20260506 09:03:38.003 | INFO | "huge_shared" : " 27 GB", | 20260506 09:03:38.003 | INFO | "malloc" : " 14 GB", | 20260506 09:03:38.003 | INFO | "pinned" : " 29 GB" | 20260506 09:03:38.003 | INFO | } | 20260506 09:03:38.003 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:03:38.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:03:38.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:38.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:38.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:38.003 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260506 09:03:38.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:38.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:38.003 | INFO | PRE_HQ : 30.3%, 1.0, 15.9ms, 0 | 20260506 09:03:38.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:38.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:38.003 | INFO | Uploaded 36GiB to GPU in 1.59925s (22.5105GiB/s) | 20260506 09:03:38.004 | INFO | Downloaded 34GiB to GPU in 1.51746s (22.4059GiB/s) | 20260506 09:03:43.142 | INFO | Analyzing chunk frames = [132096,132608) | 20260506 09:03:43.142 | INFO | MemInfo: { | 20260506 09:03:43.142 | INFO | "gpu" : " 17 GB", | 20260506 09:03:43.143 | INFO | "huge" : "", | 20260506 09:03:43.143 | INFO | "huge_shared" : " 27 GB", | 20260506 09:03:43.143 | INFO | "malloc" : " 14 GB", | 20260506 09:03:43.143 | INFO | "pinned" : " 29 GB" | 20260506 09:03:43.143 | INFO | } | 20260506 09:03:43.143 | INFO | Perf report for the last 5.13946 seconds: | 20260506 09:03:43.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:03:43.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:43.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:43.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:43.143 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260506 09:03:43.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:43.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:43.143 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:03:43.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:43.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:43.143 | INFO | Uploaded 36GiB to GPU in 1.60279s (22.4608GiB/s) | 20260506 09:03:43.143 | INFO | Downloaded 34GiB to GPU in 1.52193s (22.34GiB/s) | 20260506 09:03:48.282 | INFO | Analyzing chunk frames = [132608,133120) | 20260506 09:03:48.282 | INFO | MemInfo: { | 20260506 09:03:48.282 | INFO | "gpu" : " 17 GB", | 20260506 09:03:48.282 | INFO | "huge" : "", | 20260506 09:03:48.282 | INFO | "huge_shared" : " 27 GB", | 20260506 09:03:48.282 | INFO | "malloc" : " 14 GB", | 20260506 09:03:48.282 | INFO | "pinned" : " 29 GB" | 20260506 09:03:48.282 | INFO | } | 20260506 09:03:48.282 | INFO | Perf report for the last 5.13919 seconds: | 20260506 09:03:48.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:03:48.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:48.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:48.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:48.282 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260506 09:03:48.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:48.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:48.282 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:03:48.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:48.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:48.282 | INFO | Uploaded 36GiB to GPU in 1.60379s (22.4468GiB/s) | 20260506 09:03:48.282 | INFO | Downloaded 34GiB to GPU in 1.52142s (22.3476GiB/s) | 20260506 09:03:53.421 | INFO | Analyzing chunk frames = [133120,133632) | 20260506 09:03:53.421 | INFO | MemInfo: { | 20260506 09:03:53.421 | INFO | "gpu" : " 17 GB", | 20260506 09:03:53.421 | INFO | "huge" : "", | 20260506 09:03:53.421 | INFO | "huge_shared" : " 27 GB", | 20260506 09:03:53.421 | INFO | "malloc" : " 14 GB", | 20260506 09:03:53.421 | INFO | "pinned" : " 29 GB" | 20260506 09:03:53.421 | INFO | } | 20260506 09:03:53.421 | INFO | Perf report for the last 5.13935 seconds: | 20260506 09:03:53.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:03:53.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:53.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:53.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:53.421 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:03:53.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:53.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:53.421 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:03:53.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:53.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:53.421 | INFO | Uploaded 36GiB to GPU in 1.60121s (22.4829GiB/s) | 20260506 09:03:53.421 | INFO | Downloaded 34GiB to GPU in 1.51874s (22.387GiB/s) | 20260506 09:03:58.561 | INFO | Analyzing chunk frames = [133632,134144) | 20260506 09:03:58.561 | INFO | MemInfo: { | 20260506 09:03:58.561 | INFO | "gpu" : " 17 GB", | 20260506 09:03:58.561 | INFO | "huge" : "", | 20260506 09:03:58.561 | INFO | "huge_shared" : " 27 GB", | 20260506 09:03:58.561 | INFO | "malloc" : " 14 GB", | 20260506 09:03:58.561 | INFO | "pinned" : " 29 GB" | 20260506 09:03:58.561 | INFO | } | 20260506 09:03:58.561 | INFO | Perf report for the last 5.13952 seconds: | 20260506 09:03:58.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:03:58.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:58.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:58.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:58.561 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:03:58.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:58.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:58.561 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:03:58.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:58.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:03:58.561 | INFO | Uploaded 36GiB to GPU in 1.60185s (22.474GiB/s) | 20260506 09:03:58.561 | INFO | Downloaded 34GiB to GPU in 1.51989s (22.3701GiB/s) | 20260506 09:04:03.700 | INFO | Analyzing chunk frames = [134144,134656) | 20260506 09:04:03.700 | INFO | MemInfo: { | 20260506 09:04:03.700 | INFO | "gpu" : " 17 GB", | 20260506 09:04:03.700 | INFO | "huge" : "", | 20260506 09:04:03.700 | INFO | "huge_shared" : " 27 GB", | 20260506 09:04:03.700 | INFO | "malloc" : " 14 GB", | 20260506 09:04:03.700 | INFO | "pinned" : " 29 GB" | 20260506 09:04:03.700 | INFO | } | 20260506 09:04:03.700 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:04:03.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:04:03.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:03.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:03.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:03.700 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:04:03.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:03.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:03.700 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:04:03.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:03.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:03.700 | INFO | Uploaded 36GiB to GPU in 1.60203s (22.4715GiB/s) | 20260506 09:04:03.700 | INFO | Downloaded 34GiB to GPU in 1.52073s (22.3577GiB/s) | 20260506 09:04:08.839 | INFO | Analyzing chunk frames = [134656,135168) | 20260506 09:04:08.839 | INFO | MemInfo: { | 20260506 09:04:08.839 | INFO | "gpu" : " 17 GB", | 20260506 09:04:08.839 | INFO | "huge" : "", | 20260506 09:04:08.839 | INFO | "huge_shared" : " 27 GB", | 20260506 09:04:08.839 | INFO | "malloc" : " 14 GB", | 20260506 09:04:08.839 | INFO | "pinned" : " 29 GB" | 20260506 09:04:08.839 | INFO | } | 20260506 09:04:08.839 | INFO | Perf report for the last 5.13924 seconds: | 20260506 09:04:08.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:04:08.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:08.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:08.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:08.839 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 09:04:08.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:08.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:08.839 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 09:04:08.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:08.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:08.840 | INFO | Uploaded 36GiB to GPU in 1.60221s (22.469GiB/s) | 20260506 09:04:08.840 | INFO | Downloaded 34GiB to GPU in 1.52103s (22.3533GiB/s) | 20260506 09:04:08.937 | INFO | Starting a round of Baz IO | 20260506 09:04:11.872 | INFO | Populating RTMetrics report | 20260506 09:04:11.872 | INFO | RTMetrics report populated | 20260506 09:04:11.873 | INFO | Opening tmp RTMetrics json file | 20260506 09:04:11.873 | INFO | Serializing RTMetrics to json | 20260506 09:04:11.875 | INFO | Serialization done, swapping tmp file now... | 20260506 09:04:11.875 | INFO | RTMetrics done | 20260506 09:04:11.959 | INFO | Finished a round of Baz IO: Wrote 1.703 GiB in 3.022 seconds | 20260506 09:04:11.959 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 59.40% / 0.24% / 40.36% | 20260506 09:04:11.959 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.0277% / 0.000902017% / 1.19209e-05% | 20260506 09:04:11.959 | INFO | Baz IO overall: | 20260506 09:04:11.959 | INFO | BytesWritten : 38766892829 | 20260506 09:04:11.959 | INFO | NumEvents : 17399382019 | 20260506 09:04:13.978 | INFO | Analyzing chunk frames = [135168,135680) | 20260506 09:04:13.978 | INFO | MemInfo: { | 20260506 09:04:13.979 | INFO | "gpu" : " 17 GB", | 20260506 09:04:13.979 | INFO | "huge" : "", | 20260506 09:04:13.979 | INFO | "huge_shared" : " 27 GB", | 20260506 09:04:13.979 | INFO | "malloc" : " 14 GB", | 20260506 09:04:13.979 | INFO | "pinned" : " 29 GB" | 20260506 09:04:13.979 | INFO | } | 20260506 09:04:13.979 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:04:13.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:04:13.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:13.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:13.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:13.979 | INFO | ANALYSIS : 58.1%, 3.0, 90.4ms, 0 | 20260506 09:04:13.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:13.979 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260506 09:04:13.979 | INFO | PRE_HQ : 57.1%, 2.1, 62.3ms, 0 | 20260506 09:04:13.979 | INFO | BAZWRITER : 58.4%, 3.8, 113.6ms, 0 | 20260506 09:04:13.979 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 09:04:13.979 | INFO | Uploaded 36GiB to GPU in 1.63809s (21.9768GiB/s) | 20260506 09:04:13.979 | INFO | Downloaded 38GiB to GPU in 1.69861s (22.3713GiB/s) | 20260506 09:04:19.118 | INFO | Analyzing chunk frames = [135680,136192) | 20260506 09:04:19.118 | INFO | MemInfo: { | 20260506 09:04:19.118 | INFO | "gpu" : " 17 GB", | 20260506 09:04:19.118 | INFO | "huge" : "", | 20260506 09:04:19.118 | INFO | "huge_shared" : " 27 GB", | 20260506 09:04:19.118 | INFO | "malloc" : " 14 GB", | 20260506 09:04:19.118 | INFO | "pinned" : " 29 GB" | 20260506 09:04:19.118 | INFO | } | 20260506 09:04:19.118 | INFO | Perf report for the last 5.13958 seconds: | 20260506 09:04:19.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:04:19.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:19.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:19.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:19.118 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260506 09:04:19.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:19.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:19.118 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:04:19.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:19.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:19.118 | INFO | Uploaded 36GiB to GPU in 1.60074s (22.4897GiB/s) | 20260506 09:04:19.119 | INFO | Downloaded 34GiB to GPU in 1.51735s (22.4074GiB/s) | 20260506 09:04:24.257 | INFO | Analyzing chunk frames = [136192,136704) | 20260506 09:04:24.257 | INFO | MemInfo: { | 20260506 09:04:24.257 | INFO | "gpu" : " 17 GB", | 20260506 09:04:24.257 | INFO | "huge" : "", | 20260506 09:04:24.257 | INFO | "huge_shared" : " 27 GB", | 20260506 09:04:24.257 | INFO | "malloc" : " 14 GB", | 20260506 09:04:24.257 | INFO | "pinned" : " 29 GB" | 20260506 09:04:24.257 | INFO | } | 20260506 09:04:24.258 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:04:24.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:04:24.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:24.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:24.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:24.258 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260506 09:04:24.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:24.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:24.258 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:04:24.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:24.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:24.258 | INFO | Uploaded 36GiB to GPU in 1.60479s (22.4329GiB/s) | 20260506 09:04:24.258 | INFO | Downloaded 34GiB to GPU in 1.52239s (22.3333GiB/s) | 20260506 09:04:29.397 | INFO | Analyzing chunk frames = [136704,137216) | 20260506 09:04:29.397 | INFO | MemInfo: { | 20260506 09:04:29.397 | INFO | "gpu" : " 17 GB", | 20260506 09:04:29.397 | INFO | "huge" : "", | 20260506 09:04:29.397 | INFO | "huge_shared" : " 27 GB", | 20260506 09:04:29.397 | INFO | "malloc" : " 14 GB", | 20260506 09:04:29.397 | INFO | "pinned" : " 29 GB" | 20260506 09:04:29.397 | INFO | } | 20260506 09:04:29.397 | INFO | Perf report for the last 5.13953 seconds: | 20260506 09:04:29.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:04:29.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:29.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:29.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:29.397 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260506 09:04:29.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:29.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:29.397 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:04:29.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:29.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:29.397 | INFO | Uploaded 36GiB to GPU in 1.60415s (22.4419GiB/s) | 20260506 09:04:29.397 | INFO | Downloaded 34GiB to GPU in 1.52123s (22.3503GiB/s) | 20260506 09:04:34.536 | INFO | Analyzing chunk frames = [137216,137728) | 20260506 09:04:34.536 | INFO | MemInfo: { | 20260506 09:04:34.536 | INFO | "gpu" : " 17 GB", | 20260506 09:04:34.537 | INFO | "huge" : "", | 20260506 09:04:34.537 | INFO | "huge_shared" : " 27 GB", | 20260506 09:04:34.537 | INFO | "malloc" : " 14 GB", | 20260506 09:04:34.537 | INFO | "pinned" : " 29 GB" | 20260506 09:04:34.537 | INFO | } | 20260506 09:04:34.537 | INFO | Perf report for the last 5.13956 seconds: | 20260506 09:04:34.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:04:34.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:34.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:34.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:34.537 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 09:04:34.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:34.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:34.537 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:04:34.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:34.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:34.537 | INFO | Uploaded 36GiB to GPU in 1.60294s (22.4587GiB/s) | 20260506 09:04:34.537 | INFO | Downloaded 34GiB to GPU in 1.5202s (22.3655GiB/s) | 20260506 09:04:39.676 | INFO | Analyzing chunk frames = [137728,138240) | 20260506 09:04:39.676 | INFO | MemInfo: { | 20260506 09:04:39.676 | INFO | "gpu" : " 17 GB", | 20260506 09:04:39.676 | INFO | "huge" : "", | 20260506 09:04:39.676 | INFO | "huge_shared" : " 27 GB", | 20260506 09:04:39.676 | INFO | "malloc" : " 14 GB", | 20260506 09:04:39.676 | INFO | "pinned" : " 29 GB" | 20260506 09:04:39.676 | INFO | } | 20260506 09:04:39.676 | INFO | Perf report for the last 5.13914 seconds: | 20260506 09:04:39.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:04:39.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:39.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:39.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:39.676 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:04:39.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:39.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:39.676 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:04:39.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:39.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:39.676 | INFO | Uploaded 36GiB to GPU in 1.60248s (22.4652GiB/s) | 20260506 09:04:39.676 | INFO | Downloaded 34GiB to GPU in 1.51921s (22.38GiB/s) | 20260506 09:04:44.815 | INFO | Analyzing chunk frames = [138240,138752) | 20260506 09:04:44.815 | INFO | MemInfo: { | 20260506 09:04:44.815 | INFO | "gpu" : " 17 GB", | 20260506 09:04:44.815 | INFO | "huge" : "", | 20260506 09:04:44.815 | INFO | "huge_shared" : " 27 GB", | 20260506 09:04:44.815 | INFO | "malloc" : " 14 GB", | 20260506 09:04:44.815 | INFO | "pinned" : " 29 GB" | 20260506 09:04:44.815 | INFO | } | 20260506 09:04:44.815 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:04:44.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:04:44.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:44.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:44.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:44.815 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:04:44.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:44.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:44.815 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:04:44.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:44.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:44.815 | INFO | Uploaded 36GiB to GPU in 1.60341s (22.4521GiB/s) | 20260506 09:04:44.815 | INFO | Downloaded 34GiB to GPU in 1.52049s (22.3612GiB/s) | 20260506 09:04:49.954 | INFO | Analyzing chunk frames = [138752,139264) | 20260506 09:04:49.954 | INFO | MemInfo: { | 20260506 09:04:49.954 | INFO | "gpu" : " 17 GB", | 20260506 09:04:49.954 | INFO | "huge" : "", | 20260506 09:04:49.954 | INFO | "huge_shared" : " 27 GB", | 20260506 09:04:49.954 | INFO | "malloc" : " 14 GB", | 20260506 09:04:49.954 | INFO | "pinned" : " 29 GB" | 20260506 09:04:49.954 | INFO | } | 20260506 09:04:49.954 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:04:49.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:04:49.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:49.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:49.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:49.954 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 09:04:49.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:49.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:49.954 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 09:04:49.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:49.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:49.955 | INFO | Uploaded 36GiB to GPU in 1.60366s (22.4486GiB/s) | 20260506 09:04:49.955 | INFO | Downloaded 34GiB to GPU in 1.52083s (22.3563GiB/s) | 20260506 09:04:49.955 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:04:49.955 | INFO | Total mem usage: 15.5 GiB | 20260506 09:04:49.955 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.23%) | 20260506 09:04:49.955 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -4.64%) | 20260506 09:04:49.955 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:04:49.955 | INFO | High Water Breakdown: | 20260506 09:04:49.955 | INFO | 58.4% : 8.67 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:04:49.955 | INFO | 41.4% : 6.15 GiB ( 91.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:04:49.955 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:04:49.955 | INFO | ------------------------------------------ | 20260506 09:04:49.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:04:49.955 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:04:49.955 | INFO | Total mem usage: 26.5 GiB | 20260506 09:04:49.955 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:04:49.955 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:04:49.955 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:04:49.955 | INFO | High Water Breakdown: | 20260506 09:04:49.955 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:04:49.955 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:04:49.955 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:04:49.955 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:04:49.955 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:04:49.955 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:04:49.955 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:04:49.955 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:04:49.955 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:04:49.955 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:04:49.955 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:04:49.955 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:04:49.955 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:04:49.955 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:04:49.955 | INFO | ------------------------------------------ | 20260506 09:04:49.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:04:49.955 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:04:49.955 | INFO | Total mem usage: 25.6 GiB | 20260506 09:04:49.955 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:04:49.955 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:04:49.955 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:04:49.955 | INFO | High Water Breakdown: | 20260506 09:04:49.955 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:04:49.955 | INFO | ------------------------------------------ | 20260506 09:04:49.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:04:49.955 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:04:49.955 | INFO | Total mem usage: 15.7 GiB | 20260506 09:04:49.955 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:04:49.955 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:04:49.955 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:04:49.955 | INFO | High Water Breakdown: | 20260506 09:04:49.955 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:04:49.955 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:04:49.955 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:04:49.955 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:04:49.955 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:04:49.955 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:04:49.956 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:04:49.956 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:04:49.956 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:04:49.956 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:04:49.956 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:04:49.956 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:04:49.956 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:04:49.956 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:04:49.956 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:04:49.956 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:04:49.956 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:04:49.956 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:04:49.956 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:04:49.956 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:04:49.956 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:04:49.956 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:04:49.956 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:04:49.956 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:04:49.956 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:04:49.956 | INFO | ------------------------------------------ | 20260506 09:04:49.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:04:49.956 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:04:49.956 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:04:49.956 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:04:49.956 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 09:04:49.956 | INFO | PerfReport: 0.28%, AggregateStats -- 8.59e-02 / 0.00e+00, 4.52e-03 / 0.00e+00 | 20260506 09:04:49.956 | INFO | PerfReport: 10.86%, Baseline -- 3.29e+00 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260506 09:04:49.956 | INFO | PerfReport: 3.04%, Histogram -- 9.22e-01 / 0.00e+00, 5.16e-02 / 0.00e+00 | 20260506 09:04:49.956 | INFO | PerfReport: 16.02%, DME -- 3.89e+01 / 0.00e+00, 3.19e+00 / 0.00e+00 | 20260506 09:04:49.956 | INFO | PerfReport: 56.70%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.17e-02 / 0.00e+00 | 20260506 09:04:49.956 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.31e+00 / 0.00e+00, 1.40e-01 / 0.00e+00 | 20260506 09:04:49.956 | INFO | PerfReport: 3.95%, Metrics -- 1.20e+00 / 0.00e+00, 1.87e-01 / 0.00e+00 | 20260506 09:04:49.956 | INFO | PerfReport: 1.55%, PulseSerializing -- 4.69e-01 / 0.00e+00, 6.83e-02 / 0.00e+00 | 20260506 09:04:49.956 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:04:49.956 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:04:49.956 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:04:49.956 | INFO | PerfReport: Full Execution -- 9.10e+01 / -nan, 1.90e+01 / 0.00e+00 | 20260506 09:04:49.956 | INFO | PerfReport: 18.39%, Upload -- 1.67e+01 / 0.00e+00, 3.88e-01 / 0.00e+00 | 20260506 09:04:49.956 | INFO | PerfReport: 17.66%, Download -- 1.61e+01 / 0.00e+00, 6.95e-01 / 0.00e+00 | 20260506 09:04:49.956 | INFO | PerfReport: 33.33%, Compute -- 3.03e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 09:04:49.956 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:04:49.956 | INFO | PerfReport: 0.00%, LockDownload -- 4.73e-04 / 0.00e+00, 1.22e-04 / 0.00e+00 | 20260506 09:04:49.956 | INFO | PerfReport: 30.05%, LockCompute -- 2.74e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 09:04:49.956 | INFO | PerfReport: 0.02%, Misc -- 1.40e-02 / 0.00e+00, 2.30e-02 / 0.00e+00 | 20260506 09:04:50.051 | INFO | Starting a round of Baz IO | 20260506 09:04:52.984 | INFO | Populating RTMetrics report | 20260506 09:04:52.984 | INFO | RTMetrics report populated | 20260506 09:04:52.984 | INFO | Opening tmp RTMetrics json file | 20260506 09:04:52.985 | INFO | Serializing RTMetrics to json | 20260506 09:04:52.986 | INFO | Serialization done, swapping tmp file now... | 20260506 09:04:52.987 | INFO | RTMetrics done | 20260506 09:04:53.067 | INFO | Finished a round of Baz IO: Wrote 1.711 GiB in 3.016 seconds | 20260506 09:04:53.068 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 59.17% / 0.23% / 40.60% | 20260506 09:04:53.068 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 11.006% / 0.000826518% / 1.58946e-05% | 20260506 09:04:53.068 | INFO | Baz IO overall: | 20260506 09:04:53.068 | INFO | BytesWritten : 40603660653 | 20260506 09:04:53.068 | INFO | NumEvents : 18292754766 | 20260506 09:04:55.094 | INFO | Analyzing chunk frames = [139264,139776) | 20260506 09:04:55.094 | INFO | MemInfo: { | 20260506 09:04:55.094 | INFO | "gpu" : " 17 GB", | 20260506 09:04:55.094 | INFO | "huge" : "", | 20260506 09:04:55.094 | INFO | "huge_shared" : " 27 GB", | 20260506 09:04:55.094 | INFO | "malloc" : " 14 GB", | 20260506 09:04:55.094 | INFO | "pinned" : " 29 GB" | 20260506 09:04:55.094 | INFO | } | 20260506 09:04:55.094 | INFO | Perf report for the last 5.13937 seconds: | 20260506 09:04:55.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:04:55.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:55.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:55.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:55.094 | INFO | ANALYSIS : 58.0%, 2.9, 89.3ms, 0 | 20260506 09:04:55.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:04:55.094 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260506 09:04:55.094 | INFO | PRE_HQ : 57.1%, 2.1, 62.1ms, 0 | 20260506 09:04:55.094 | INFO | BAZWRITER : 58.3%, 4.3, 128.4ms, 0 | 20260506 09:04:55.094 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 09:04:55.094 | INFO | Uploaded 36GiB to GPU in 1.64528s (21.8807GiB/s) | 20260506 09:04:55.094 | INFO | Downloaded 38GiB to GPU in 1.70665s (22.2658GiB/s) | 20260506 09:05:00.233 | INFO | Analyzing chunk frames = [139776,140288) | 20260506 09:05:00.233 | INFO | MemInfo: { | 20260506 09:05:00.233 | INFO | "gpu" : " 17 GB", | 20260506 09:05:00.233 | INFO | "huge" : "", | 20260506 09:05:00.233 | INFO | "huge_shared" : " 27 GB", | 20260506 09:05:00.233 | INFO | "malloc" : " 14 GB", | 20260506 09:05:00.233 | INFO | "pinned" : " 29 GB" | 20260506 09:05:00.233 | INFO | } | 20260506 09:05:00.233 | INFO | Perf report for the last 5.13958 seconds: | 20260506 09:05:00.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:05:00.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:00.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:00.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:00.233 | INFO | ANALYSIS : 57.9%, 3.0, 90.2ms, 0 | 20260506 09:05:00.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:00.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:00.233 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:05:00.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:00.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:00.234 | INFO | Uploaded 36GiB to GPU in 1.60011s (22.4985GiB/s) | 20260506 09:05:00.234 | INFO | Downloaded 34GiB to GPU in 1.51742s (22.4065GiB/s) | 20260506 09:05:05.372 | INFO | Analyzing chunk frames = [140288,140800) | 20260506 09:05:05.372 | INFO | MemInfo: { | 20260506 09:05:05.372 | INFO | "gpu" : " 17 GB", | 20260506 09:05:05.372 | INFO | "huge" : "", | 20260506 09:05:05.373 | INFO | "huge_shared" : " 27 GB", | 20260506 09:05:05.373 | INFO | "malloc" : " 14 GB", | 20260506 09:05:05.373 | INFO | "pinned" : " 29 GB" | 20260506 09:05:05.373 | INFO | } | 20260506 09:05:05.373 | INFO | Perf report for the last 5.13926 seconds: | 20260506 09:05:05.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:05:05.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:05.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:05.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:05.373 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260506 09:05:05.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:05.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:05.373 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:05:05.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:05.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:05.373 | INFO | Uploaded 36GiB to GPU in 1.60458s (22.4358GiB/s) | 20260506 09:05:05.373 | INFO | Downloaded 34GiB to GPU in 1.52238s (22.3335GiB/s) | 20260506 09:05:10.512 | INFO | Analyzing chunk frames = [140800,141312) | 20260506 09:05:10.512 | INFO | MemInfo: { | 20260506 09:05:10.512 | INFO | "gpu" : " 17 GB", | 20260506 09:05:10.512 | INFO | "huge" : "", | 20260506 09:05:10.512 | INFO | "huge_shared" : " 27 GB", | 20260506 09:05:10.512 | INFO | "malloc" : " 14 GB", | 20260506 09:05:10.512 | INFO | "pinned" : " 29 GB" | 20260506 09:05:10.512 | INFO | } | 20260506 09:05:10.512 | INFO | Perf report for the last 5.13923 seconds: | 20260506 09:05:10.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:05:10.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:10.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:10.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:10.512 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260506 09:05:10.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:10.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:10.512 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:05:10.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:10.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:10.512 | INFO | Uploaded 36GiB to GPU in 1.60371s (22.4479GiB/s) | 20260506 09:05:10.512 | INFO | Downloaded 34GiB to GPU in 1.52111s (22.352GiB/s) | 20260506 09:05:15.651 | INFO | Analyzing chunk frames = [141312,141824) | 20260506 09:05:15.651 | INFO | MemInfo: { | 20260506 09:05:15.651 | INFO | "gpu" : " 17 GB", | 20260506 09:05:15.651 | INFO | "huge" : "", | 20260506 09:05:15.651 | INFO | "huge_shared" : " 27 GB", | 20260506 09:05:15.651 | INFO | "malloc" : " 14 GB", | 20260506 09:05:15.651 | INFO | "pinned" : " 29 GB" | 20260506 09:05:15.651 | INFO | } | 20260506 09:05:15.651 | INFO | Perf report for the last 5.13937 seconds: | 20260506 09:05:15.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:05:15.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:15.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:15.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:15.651 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 09:05:15.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:15.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:15.651 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:05:15.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:15.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:15.652 | INFO | Uploaded 36GiB to GPU in 1.60228s (22.468GiB/s) | 20260506 09:05:15.652 | INFO | Downloaded 34GiB to GPU in 1.51891s (22.3844GiB/s) | 20260506 09:05:20.790 | INFO | Analyzing chunk frames = [141824,142336) | 20260506 09:05:20.790 | INFO | MemInfo: { | 20260506 09:05:20.791 | INFO | "gpu" : " 17 GB", | 20260506 09:05:20.791 | INFO | "huge" : "", | 20260506 09:05:20.791 | INFO | "huge_shared" : " 27 GB", | 20260506 09:05:20.791 | INFO | "malloc" : " 14 GB", | 20260506 09:05:20.791 | INFO | "pinned" : " 29 GB" | 20260506 09:05:20.791 | INFO | } | 20260506 09:05:20.791 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:05:20.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:05:20.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:20.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:20.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:20.791 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:05:20.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:20.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:20.791 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:05:20.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:20.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:20.791 | INFO | Uploaded 36GiB to GPU in 1.60234s (22.4671GiB/s) | 20260506 09:05:20.791 | INFO | Downloaded 34GiB to GPU in 1.52009s (22.3671GiB/s) | 20260506 09:05:25.930 | INFO | Analyzing chunk frames = [142336,142848) | 20260506 09:05:25.930 | INFO | MemInfo: { | 20260506 09:05:25.930 | INFO | "gpu" : " 17 GB", | 20260506 09:05:25.930 | INFO | "huge" : "", | 20260506 09:05:25.930 | INFO | "huge_shared" : " 27 GB", | 20260506 09:05:25.930 | INFO | "malloc" : " 14 GB", | 20260506 09:05:25.930 | INFO | "pinned" : " 29 GB" | 20260506 09:05:25.930 | INFO | } | 20260506 09:05:25.930 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:05:25.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:05:25.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:25.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:25.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:25.930 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 09:05:25.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:25.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:25.930 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:05:25.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:25.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:25.930 | INFO | Uploaded 36GiB to GPU in 1.60276s (22.4612GiB/s) | 20260506 09:05:25.930 | INFO | Downloaded 34GiB to GPU in 1.52058s (22.3599GiB/s) | 20260506 09:05:31.069 | INFO | Analyzing chunk frames = [142848,143360) | 20260506 09:05:31.069 | INFO | MemInfo: { | 20260506 09:05:31.069 | INFO | "gpu" : " 17 GB", | 20260506 09:05:31.069 | INFO | "huge" : "", | 20260506 09:05:31.069 | INFO | "huge_shared" : " 27 GB", | 20260506 09:05:31.069 | INFO | "malloc" : " 14 GB", | 20260506 09:05:31.069 | INFO | "pinned" : " 29 GB" | 20260506 09:05:31.069 | INFO | } | 20260506 09:05:31.069 | INFO | Perf report for the last 5.13938 seconds: | 20260506 09:05:31.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:05:31.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:31.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:31.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:31.069 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 09:05:31.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:31.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:31.069 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 09:05:31.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:31.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:31.070 | INFO | Uploaded 36GiB to GPU in 1.60319s (22.4552GiB/s) | 20260506 09:05:31.070 | INFO | Downloaded 34GiB to GPU in 1.52079s (22.3568GiB/s) | 20260506 09:05:31.166 | INFO | Starting a round of Baz IO | 20260506 09:05:34.097 | INFO | Populating RTMetrics report | 20260506 09:05:34.097 | INFO | RTMetrics report populated | 20260506 09:05:34.097 | INFO | Opening tmp RTMetrics json file | 20260506 09:05:34.107 | INFO | Serializing RTMetrics to json | 20260506 09:05:34.133 | INFO | Serialization done, swapping tmp file now... | 20260506 09:05:34.134 | INFO | RTMetrics done | 20260506 09:05:34.378 | INFO | Finished a round of Baz IO: Wrote 1.719 GiB in 3.211 seconds | 20260506 09:05:34.378 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.95% / 0.22% / 40.83% | 20260506 09:05:34.378 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.9887% / 0.000810623% / 2.78155e-05% | 20260506 09:05:34.378 | INFO | Baz IO overall: | 20260506 09:05:34.378 | INFO | BytesWritten : 42449784885 | 20260506 09:05:34.378 | INFO | NumEvents : 19190809389 | 20260506 09:05:36.209 | INFO | Analyzing chunk frames = [143360,143872) | 20260506 09:05:36.209 | INFO | MemInfo: { | 20260506 09:05:36.209 | INFO | "gpu" : " 17 GB", | 20260506 09:05:36.209 | INFO | "huge" : "", | 20260506 09:05:36.209 | INFO | "huge_shared" : " 27 GB", | 20260506 09:05:36.209 | INFO | "malloc" : " 14 GB", | 20260506 09:05:36.209 | INFO | "pinned" : " 29 GB" | 20260506 09:05:36.209 | INFO | } | 20260506 09:05:36.209 | INFO | Perf report for the last 5.13935 seconds: | 20260506 09:05:36.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:05:36.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:36.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:36.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:36.209 | INFO | ANALYSIS : 58.0%, 2.9, 86.7ms, 0 | 20260506 09:05:36.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:36.209 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 09:05:36.209 | INFO | PRE_HQ : 57.2%, 2.1, 61.7ms, 0 | 20260506 09:05:36.209 | INFO | BAZWRITER : 62.0%, 4.6, 147.7ms, 0 | 20260506 09:05:36.209 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 09:05:36.209 | INFO | Uploaded 36GiB to GPU in 1.66298s (21.6478GiB/s) | 20260506 09:05:36.209 | INFO | Downloaded 38GiB to GPU in 1.7267s (22.0073GiB/s) | 20260506 09:05:41.348 | INFO | Analyzing chunk frames = [143872,144384) | 20260506 09:05:41.348 | INFO | MemInfo: { | 20260506 09:05:41.348 | INFO | "gpu" : " 17 GB", | 20260506 09:05:41.348 | INFO | "huge" : "", | 20260506 09:05:41.348 | INFO | "huge_shared" : " 27 GB", | 20260506 09:05:41.348 | INFO | "malloc" : " 14 GB", | 20260506 09:05:41.348 | INFO | "pinned" : " 29 GB" | 20260506 09:05:41.348 | INFO | } | 20260506 09:05:41.348 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:05:41.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:05:41.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:41.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:41.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:41.348 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260506 09:05:41.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:41.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:41.348 | INFO | PRE_HQ : 30.6%, 1.0, 16.0ms, 0 | 20260506 09:05:41.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:41.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:41.348 | INFO | Uploaded 36GiB to GPU in 1.60088s (22.4876GiB/s) | 20260506 09:05:41.348 | INFO | Downloaded 34GiB to GPU in 1.518s (22.3978GiB/s) | 20260506 09:05:46.487 | INFO | Analyzing chunk frames = [144384,144896) | 20260506 09:05:46.487 | INFO | MemInfo: { | 20260506 09:05:46.487 | INFO | "gpu" : " 17 GB", | 20260506 09:05:46.487 | INFO | "huge" : "", | 20260506 09:05:46.487 | INFO | "huge_shared" : " 27 GB", | 20260506 09:05:46.487 | INFO | "malloc" : " 14 GB", | 20260506 09:05:46.487 | INFO | "pinned" : " 29 GB" | 20260506 09:05:46.487 | INFO | } | 20260506 09:05:46.488 | INFO | Perf report for the last 5.13952 seconds: | 20260506 09:05:46.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:05:46.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:46.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:46.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:46.488 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260506 09:05:46.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:46.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:46.488 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:05:46.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:46.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:46.488 | INFO | Uploaded 36GiB to GPU in 1.60562s (22.4213GiB/s) | 20260506 09:05:46.488 | INFO | Downloaded 34GiB to GPU in 1.5226s (22.3302GiB/s) | 20260506 09:05:51.627 | INFO | Analyzing chunk frames = [144896,145408) | 20260506 09:05:51.627 | INFO | MemInfo: { | 20260506 09:05:51.627 | INFO | "gpu" : " 17 GB", | 20260506 09:05:51.627 | INFO | "huge" : "", | 20260506 09:05:51.627 | INFO | "huge_shared" : " 27 GB", | 20260506 09:05:51.627 | INFO | "malloc" : " 14 GB", | 20260506 09:05:51.627 | INFO | "pinned" : " 29 GB" | 20260506 09:05:51.627 | INFO | } | 20260506 09:05:51.627 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:05:51.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:05:51.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:51.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:51.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:51.627 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260506 09:05:51.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:51.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:51.627 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:05:51.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:51.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:51.627 | INFO | Uploaded 36GiB to GPU in 1.60393s (22.4449GiB/s) | 20260506 09:05:51.627 | INFO | Downloaded 34GiB to GPU in 1.52118s (22.3511GiB/s) | 20260506 09:05:56.766 | INFO | Analyzing chunk frames = [145408,145920) | 20260506 09:05:56.766 | INFO | MemInfo: { | 20260506 09:05:56.766 | INFO | "gpu" : " 17 GB", | 20260506 09:05:56.766 | INFO | "huge" : "", | 20260506 09:05:56.766 | INFO | "huge_shared" : " 27 GB", | 20260506 09:05:56.766 | INFO | "malloc" : " 14 GB", | 20260506 09:05:56.766 | INFO | "pinned" : " 29 GB" | 20260506 09:05:56.766 | INFO | } | 20260506 09:05:56.766 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:05:56.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:05:56.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:56.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:56.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:56.766 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 09:05:56.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:56.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:56.766 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:05:56.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:56.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:05:56.766 | INFO | Uploaded 36GiB to GPU in 1.60027s (22.4962GiB/s) | 20260506 09:05:56.767 | INFO | Downloaded 34GiB to GPU in 1.51995s (22.3691GiB/s) | 20260506 09:06:01.905 | INFO | Analyzing chunk frames = [145920,146432) | 20260506 09:06:01.905 | INFO | MemInfo: { | 20260506 09:06:01.905 | INFO | "gpu" : " 17 GB", | 20260506 09:06:01.905 | INFO | "huge" : "", | 20260506 09:06:01.905 | INFO | "huge_shared" : " 27 GB", | 20260506 09:06:01.905 | INFO | "malloc" : " 14 GB", | 20260506 09:06:01.905 | INFO | "pinned" : " 29 GB" | 20260506 09:06:01.905 | INFO | } | 20260506 09:06:01.905 | INFO | Perf report for the last 5.13936 seconds: | 20260506 09:06:01.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:06:01.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:01.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:01.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:01.905 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:06:01.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:01.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:01.905 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:06:01.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:01.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:01.906 | INFO | Uploaded 36GiB to GPU in 1.6032s (22.4551GiB/s) | 20260506 09:06:01.906 | INFO | Downloaded 34GiB to GPU in 1.51977s (22.3717GiB/s) | 20260506 09:06:07.045 | INFO | Analyzing chunk frames = [146432,146944) | 20260506 09:06:07.045 | INFO | MemInfo: { | 20260506 09:06:07.045 | INFO | "gpu" : " 17 GB", | 20260506 09:06:07.045 | INFO | "huge" : "", | 20260506 09:06:07.045 | INFO | "huge_shared" : " 27 GB", | 20260506 09:06:07.045 | INFO | "malloc" : " 14 GB", | 20260506 09:06:07.045 | INFO | "pinned" : " 29 GB" | 20260506 09:06:07.045 | INFO | } | 20260506 09:06:07.045 | INFO | Perf report for the last 5.13938 seconds: | 20260506 09:06:07.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:06:07.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:07.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:07.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:07.045 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:06:07.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:07.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:07.045 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:06:07.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:07.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:07.045 | INFO | Uploaded 36GiB to GPU in 1.60203s (22.4715GiB/s) | 20260506 09:06:07.045 | INFO | Downloaded 34GiB to GPU in 1.5206s (22.3596GiB/s) | 20260506 09:06:12.184 | INFO | Analyzing chunk frames = [146944,147456) | 20260506 09:06:12.184 | INFO | MemInfo: { | 20260506 09:06:12.184 | INFO | "gpu" : " 17 GB", | 20260506 09:06:12.184 | INFO | "huge" : "", | 20260506 09:06:12.184 | INFO | "huge_shared" : " 27 GB", | 20260506 09:06:12.184 | INFO | "malloc" : " 14 GB", | 20260506 09:06:12.184 | INFO | "pinned" : " 29 GB" | 20260506 09:06:12.184 | INFO | } | 20260506 09:06:12.184 | INFO | Perf report for the last 5.13922 seconds: | 20260506 09:06:12.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:06:12.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:12.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:12.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:12.184 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 09:06:12.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:12.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:12.184 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 09:06:12.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:12.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:12.184 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:06:12.184 | INFO | Total mem usage: 15.5 GiB | 20260506 09:06:12.184 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.23%) | 20260506 09:06:12.184 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -1.07%) | 20260506 09:06:12.184 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:06:12.184 | INFO | Uploaded 36GiB to GPU in 1.6038s (22.4467GiB/s) | 20260506 09:06:12.184 | INFO | Downloaded 34GiB to GPU in 1.52065s (22.3589GiB/s) | 20260506 09:06:12.184 | INFO | High Water Breakdown: | 20260506 09:06:12.184 | INFO | 59.2% : 9.09 GiB ( 99.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:06:12.184 | INFO | 40.7% : 6.25 GiB ( 93.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:06:12.184 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:06:12.184 | INFO | ------------------------------------------ | 20260506 09:06:12.184 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:06:12.184 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:06:12.185 | INFO | Total mem usage: 26.5 GiB | 20260506 09:06:12.185 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:06:12.185 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:06:12.185 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:06:12.185 | INFO | High Water Breakdown: | 20260506 09:06:12.185 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:06:12.185 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:06:12.185 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:06:12.185 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:06:12.185 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:06:12.185 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:06:12.185 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:06:12.185 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:06:12.185 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:06:12.185 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:06:12.185 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:06:12.185 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:06:12.185 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:06:12.185 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:06:12.185 | INFO | ------------------------------------------ | 20260506 09:06:12.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:06:12.185 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:06:12.185 | INFO | Total mem usage: 25.6 GiB | 20260506 09:06:12.185 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:06:12.185 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:06:12.185 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:06:12.185 | INFO | High Water Breakdown: | 20260506 09:06:12.185 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:06:12.185 | INFO | ------------------------------------------ | 20260506 09:06:12.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:06:12.185 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:06:12.185 | INFO | Total mem usage: 15.7 GiB | 20260506 09:06:12.185 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:06:12.185 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:06:12.185 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:06:12.185 | INFO | High Water Breakdown: | 20260506 09:06:12.185 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:06:12.185 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:06:12.185 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:06:12.185 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:06:12.185 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:06:12.185 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:06:12.185 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:06:12.185 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:06:12.185 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:06:12.185 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:06:12.185 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:06:12.185 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:06:12.185 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:06:12.185 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:06:12.185 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:06:12.185 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:06:12.185 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:06:12.185 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:06:12.185 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:06:12.185 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:06:12.185 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:06:12.185 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:06:12.185 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:06:12.185 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:06:12.185 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:06:12.185 | INFO | ------------------------------------------ | 20260506 09:06:12.185 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:06:12.185 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:06:12.185 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:06:12.185 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:06:12.185 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 09:06:12.185 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / 0.00e+00, 4.46e-03 / 0.00e+00 | 20260506 09:06:12.185 | INFO | PerfReport: 10.85%, Baseline -- 3.29e+00 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260506 09:06:12.185 | INFO | PerfReport: 3.04%, Histogram -- 9.22e-01 / 0.00e+00, 5.23e-02 / 0.00e+00 | 20260506 09:06:12.185 | INFO | PerfReport: 16.02%, DME -- 3.89e+01 / 0.00e+00, 3.21e+00 / 0.00e+00 | 20260506 09:06:12.185 | INFO | PerfReport: 56.71%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.17e-02 / 0.00e+00 | 20260506 09:06:12.185 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.31e+00 / 0.00e+00, 1.39e-01 / 0.00e+00 | 20260506 09:06:12.185 | INFO | PerfReport: 3.94%, Metrics -- 1.20e+00 / 0.00e+00, 1.85e-01 / 0.00e+00 | 20260506 09:06:12.185 | INFO | PerfReport: 1.54%, PulseSerializing -- 4.68e-01 / 0.00e+00, 6.79e-02 / 0.00e+00 | 20260506 09:06:12.185 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:06:12.185 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:06:12.185 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:06:12.185 | INFO | PerfReport: Full Execution -- 9.08e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 09:06:12.185 | INFO | PerfReport: 18.46%, Upload -- 1.68e+01 / 0.00e+00, 4.37e-01 / 0.00e+00 | 20260506 09:06:12.185 | INFO | PerfReport: 17.72%, Download -- 1.61e+01 / 0.00e+00, 7.68e-01 / 0.00e+00 | 20260506 09:06:12.185 | INFO | PerfReport: 33.41%, Compute -- 3.03e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 09:06:12.185 | INFO | PerfReport: 0.56%, LockUpload -- 5.08e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:06:12.185 | INFO | PerfReport: 0.00%, LockDownload -- 4.90e-04 / 0.00e+00, 2.09e-04 / 0.00e+00 | 20260506 09:06:12.185 | INFO | PerfReport: 29.84%, LockCompute -- 2.71e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 09:06:12.185 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260506 09:06:12.281 | INFO | Starting a round of Baz IO | 20260506 09:06:15.214 | INFO | Populating RTMetrics report | 20260506 09:06:15.214 | INFO | RTMetrics report populated | 20260506 09:06:15.214 | INFO | Opening tmp RTMetrics json file | 20260506 09:06:15.313 | INFO | Serializing RTMetrics to json | 20260506 09:06:15.318 | INFO | Serialization done, swapping tmp file now... | 20260506 09:06:15.331 | INFO | RTMetrics done | 20260506 09:06:15.359 | INFO | Finished a round of Baz IO: Wrote 1.725 GiB in 3.078 seconds | 20260506 09:06:15.359 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.74% / 0.21% / 41.05% | 20260506 09:06:15.359 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.9344% / 0.000735124% / 3.17891e-05% | 20260506 09:06:15.359 | INFO | Baz IO overall: | 20260506 09:06:15.359 | INFO | BytesWritten : 44301522494 | 20260506 09:06:15.359 | INFO | NumEvents : 20091164944 | 20260506 09:06:17.323 | INFO | Analyzing chunk frames = [147456,147968) | 20260506 09:06:17.323 | INFO | MemInfo: { | 20260506 09:06:17.323 | INFO | "gpu" : " 17 GB", | 20260506 09:06:17.323 | INFO | "huge" : "", | 20260506 09:06:17.323 | INFO | "huge_shared" : " 27 GB", | 20260506 09:06:17.323 | INFO | "malloc" : " 14 GB", | 20260506 09:06:17.323 | INFO | "pinned" : " 29 GB" | 20260506 09:06:17.323 | INFO | } | 20260506 09:06:17.324 | INFO | Perf report for the last 5.13945 seconds: | 20260506 09:06:17.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:06:17.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:17.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:17.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:17.324 | INFO | ANALYSIS : 58.0%, 2.9, 89.4ms, 0 | 20260506 09:06:17.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:17.324 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 09:06:17.324 | INFO | PRE_HQ : 57.2%, 2.1, 62.4ms, 0 | 20260506 09:06:17.324 | INFO | BAZWRITER : 59.5%, 4.1, 124.6ms, 0 | 20260506 09:06:17.324 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 09:06:17.324 | INFO | Uploaded 36GiB to GPU in 1.63934s (21.96GiB/s) | 20260506 09:06:17.324 | INFO | Downloaded 38GiB to GPU in 1.70113s (22.338GiB/s) | 20260506 09:06:22.463 | INFO | Analyzing chunk frames = [147968,148480) | 20260506 09:06:22.463 | INFO | MemInfo: { | 20260506 09:06:22.463 | INFO | "gpu" : " 17 GB", | 20260506 09:06:22.463 | INFO | "huge" : "", | 20260506 09:06:22.463 | INFO | "huge_shared" : " 27 GB", | 20260506 09:06:22.463 | INFO | "malloc" : " 14 GB", | 20260506 09:06:22.463 | INFO | "pinned" : " 29 GB" | 20260506 09:06:22.463 | INFO | } | 20260506 09:06:22.463 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:06:22.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:06:22.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:22.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:22.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:22.463 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260506 09:06:22.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:22.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:22.463 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:06:22.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:22.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:22.463 | INFO | Uploaded 36GiB to GPU in 1.60094s (22.4868GiB/s) | 20260506 09:06:22.463 | INFO | Downloaded 34GiB to GPU in 1.51714s (22.4105GiB/s) | 20260506 09:06:27.602 | INFO | Analyzing chunk frames = [148480,148992) | 20260506 09:06:27.602 | INFO | MemInfo: { | 20260506 09:06:27.602 | INFO | "gpu" : " 17 GB", | 20260506 09:06:27.602 | INFO | "huge" : "", | 20260506 09:06:27.602 | INFO | "huge_shared" : " 27 GB", | 20260506 09:06:27.602 | INFO | "malloc" : " 14 GB", | 20260506 09:06:27.602 | INFO | "pinned" : " 29 GB" | 20260506 09:06:27.602 | INFO | } | 20260506 09:06:27.602 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:06:27.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:06:27.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:27.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:27.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:27.602 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260506 09:06:27.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:27.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:27.602 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:06:27.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:27.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:27.602 | INFO | Uploaded 36GiB to GPU in 1.60429s (22.4399GiB/s) | 20260506 09:06:27.602 | INFO | Downloaded 34GiB to GPU in 1.52199s (22.3392GiB/s) | 20260506 09:06:32.741 | INFO | Analyzing chunk frames = [148992,149504) | 20260506 09:06:32.741 | INFO | MemInfo: { | 20260506 09:06:32.741 | INFO | "gpu" : " 17 GB", | 20260506 09:06:32.741 | INFO | "huge" : "", | 20260506 09:06:32.741 | INFO | "huge_shared" : " 27 GB", | 20260506 09:06:32.741 | INFO | "malloc" : " 14 GB", | 20260506 09:06:32.742 | INFO | "pinned" : " 29 GB" | 20260506 09:06:32.742 | INFO | } | 20260506 09:06:32.742 | INFO | Perf report for the last 5.13955 seconds: | 20260506 09:06:32.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:06:32.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:32.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:32.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:32.742 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 09:06:32.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:32.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:32.742 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:06:32.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:32.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:32.742 | INFO | Uploaded 36GiB to GPU in 1.60394s (22.4447GiB/s) | 20260506 09:06:32.742 | INFO | Downloaded 34GiB to GPU in 1.5222s (22.3361GiB/s) | 20260506 09:06:37.881 | INFO | Analyzing chunk frames = [149504,150016) | 20260506 09:06:37.881 | INFO | MemInfo: { | 20260506 09:06:37.881 | INFO | "gpu" : " 17 GB", | 20260506 09:06:37.881 | INFO | "huge" : "", | 20260506 09:06:37.881 | INFO | "huge_shared" : " 27 GB", | 20260506 09:06:37.881 | INFO | "malloc" : " 14 GB", | 20260506 09:06:37.881 | INFO | "pinned" : " 29 GB" | 20260506 09:06:37.881 | INFO | } | 20260506 09:06:37.881 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:06:37.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:06:37.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:37.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:37.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:37.881 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:06:37.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:37.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:37.881 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:06:37.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:37.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:37.882 | INFO | Uploaded 36GiB to GPU in 1.60181s (22.4746GiB/s) | 20260506 09:06:37.882 | INFO | Downloaded 34GiB to GPU in 1.51921s (22.3801GiB/s) | 20260506 09:06:43.020 | INFO | Analyzing chunk frames = [150016,150528) | 20260506 09:06:43.020 | INFO | MemInfo: { | 20260506 09:06:43.020 | INFO | "gpu" : " 17 GB", | 20260506 09:06:43.020 | INFO | "huge" : "", | 20260506 09:06:43.020 | INFO | "huge_shared" : " 27 GB", | 20260506 09:06:43.020 | INFO | "malloc" : " 14 GB", | 20260506 09:06:43.020 | INFO | "pinned" : " 29 GB" | 20260506 09:06:43.020 | INFO | } | 20260506 09:06:43.020 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:06:43.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:06:43.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:43.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:43.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:43.020 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:06:43.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:43.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:43.020 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:06:43.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:43.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:43.021 | INFO | Uploaded 36GiB to GPU in 1.60217s (22.4695GiB/s) | 20260506 09:06:43.021 | INFO | Downloaded 34GiB to GPU in 1.51959s (22.3744GiB/s) | 20260506 09:06:48.160 | INFO | Analyzing chunk frames = [150528,151040) | 20260506 09:06:48.160 | INFO | MemInfo: { | 20260506 09:06:48.160 | INFO | "gpu" : " 17 GB", | 20260506 09:06:48.160 | INFO | "huge" : "", | 20260506 09:06:48.160 | INFO | "huge_shared" : " 27 GB", | 20260506 09:06:48.160 | INFO | "malloc" : " 14 GB", | 20260506 09:06:48.160 | INFO | "pinned" : " 29 GB" | 20260506 09:06:48.160 | INFO | } | 20260506 09:06:48.160 | INFO | Perf report for the last 5.13946 seconds: | 20260506 09:06:48.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:06:48.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:48.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:48.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:48.160 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:06:48.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:48.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:48.160 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:06:48.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:48.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:48.160 | INFO | Uploaded 36GiB to GPU in 1.60218s (22.4694GiB/s) | 20260506 09:06:48.160 | INFO | Downloaded 34GiB to GPU in 1.52064s (22.359GiB/s) | 20260506 09:06:53.299 | INFO | Analyzing chunk frames = [151040,151552) | 20260506 09:06:53.299 | INFO | MemInfo: { | 20260506 09:06:53.299 | INFO | "gpu" : " 17 GB", | 20260506 09:06:53.299 | INFO | "huge" : "", | 20260506 09:06:53.299 | INFO | "huge_shared" : " 27 GB", | 20260506 09:06:53.299 | INFO | "malloc" : " 14 GB", | 20260506 09:06:53.299 | INFO | "pinned" : " 29 GB" | 20260506 09:06:53.299 | INFO | } | 20260506 09:06:53.299 | INFO | Perf report for the last 5.13913 seconds: | 20260506 09:06:53.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:06:53.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:53.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:53.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:53.299 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 09:06:53.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:53.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:53.299 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 09:06:53.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:53.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:53.299 | INFO | Uploaded 36GiB to GPU in 1.60355s (22.4502GiB/s) | 20260506 09:06:53.299 | INFO | Downloaded 34GiB to GPU in 1.52024s (22.3648GiB/s) | 20260506 09:06:53.396 | INFO | Starting a round of Baz IO | 20260506 09:06:56.329 | INFO | Populating RTMetrics report | 20260506 09:06:56.330 | INFO | RTMetrics report populated | 20260506 09:06:56.330 | INFO | Opening tmp RTMetrics json file | 20260506 09:06:56.330 | INFO | Serializing RTMetrics to json | 20260506 09:06:56.332 | INFO | Serialization done, swapping tmp file now... | 20260506 09:06:56.335 | INFO | RTMetrics done | 20260506 09:06:56.440 | INFO | Finished a round of Baz IO: Wrote 1.731 GiB in 3.044 seconds | 20260506 09:06:56.440 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.53% / 0.21% / 41.26% | 20260506 09:06:56.440 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.9005% / 0.000766913% / 1.58946e-05% | 20260506 09:06:56.440 | INFO | Baz IO overall: | 20260506 09:06:56.440 | INFO | BytesWritten : 46159901172 | 20260506 09:06:56.440 | INFO | NumEvents : 20994965509 | 20260506 09:06:58.438 | INFO | Analyzing chunk frames = [151552,152064) | 20260506 09:06:58.438 | INFO | MemInfo: { | 20260506 09:06:58.438 | INFO | "gpu" : " 17 GB", | 20260506 09:06:58.438 | INFO | "huge" : "", | 20260506 09:06:58.438 | INFO | "huge_shared" : " 27 GB", | 20260506 09:06:58.438 | INFO | "malloc" : " 14 GB", | 20260506 09:06:58.438 | INFO | "pinned" : " 29 GB" | 20260506 09:06:58.438 | INFO | } | 20260506 09:06:58.439 | INFO | Perf report for the last 5.13958 seconds: | 20260506 09:06:58.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:06:58.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:58.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:58.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:58.439 | INFO | ANALYSIS : 58.0%, 3.0, 89.9ms, 0 | 20260506 09:06:58.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:06:58.439 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 09:06:58.439 | INFO | PRE_HQ : 57.1%, 2.1, 62.3ms, 0 | 20260506 09:06:58.439 | INFO | BAZWRITER : 58.8%, 4.0, 121.3ms, 0 | 20260506 09:06:58.439 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 09:06:58.439 | INFO | Uploaded 36GiB to GPU in 1.62979s (22.0888GiB/s) | 20260506 09:06:58.439 | INFO | Downloaded 38GiB to GPU in 1.69269s (22.4495GiB/s) | 20260506 09:07:03.578 | INFO | Analyzing chunk frames = [152064,152576) | 20260506 09:07:03.578 | INFO | MemInfo: { | 20260506 09:07:03.578 | INFO | "gpu" : " 17 GB", | 20260506 09:07:03.578 | INFO | "huge" : "", | 20260506 09:07:03.578 | INFO | "huge_shared" : " 27 GB", | 20260506 09:07:03.578 | INFO | "malloc" : " 14 GB", | 20260506 09:07:03.578 | INFO | "pinned" : " 29 GB" | 20260506 09:07:03.578 | INFO | } | 20260506 09:07:03.578 | INFO | Perf report for the last 5.13918 seconds: | 20260506 09:07:03.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:07:03.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:03.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:03.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:03.578 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260506 09:07:03.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:03.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:03.578 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 09:07:03.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:03.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:03.578 | INFO | Uploaded 36GiB to GPU in 1.60075s (22.4894GiB/s) | 20260506 09:07:03.578 | INFO | Downloaded 34GiB to GPU in 1.51797s (22.3984GiB/s) | 20260506 09:07:08.717 | INFO | Analyzing chunk frames = [152576,153088) | 20260506 09:07:08.717 | INFO | MemInfo: { | 20260506 09:07:08.717 | INFO | "gpu" : " 17 GB", | 20260506 09:07:08.717 | INFO | "huge" : "", | 20260506 09:07:08.717 | INFO | "huge_shared" : " 27 GB", | 20260506 09:07:08.717 | INFO | "malloc" : " 14 GB", | 20260506 09:07:08.717 | INFO | "pinned" : " 29 GB" | 20260506 09:07:08.717 | INFO | } | 20260506 09:07:08.717 | INFO | Perf report for the last 5.13935 seconds: | 20260506 09:07:08.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:07:08.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:08.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:08.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:08.717 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260506 09:07:08.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:08.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:08.717 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:07:08.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:08.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:08.717 | INFO | Uploaded 36GiB to GPU in 1.60447s (22.4374GiB/s) | 20260506 09:07:08.717 | INFO | Downloaded 34GiB to GPU in 1.52161s (22.3447GiB/s) | 20260506 09:07:13.856 | INFO | Analyzing chunk frames = [153088,153600) | 20260506 09:07:13.856 | INFO | MemInfo: { | 20260506 09:07:13.856 | INFO | "gpu" : " 17 GB", | 20260506 09:07:13.856 | INFO | "huge" : "", | 20260506 09:07:13.856 | INFO | "huge_shared" : " 27 GB", | 20260506 09:07:13.856 | INFO | "malloc" : " 14 GB", | 20260506 09:07:13.856 | INFO | "pinned" : " 29 GB" | 20260506 09:07:13.856 | INFO | } | 20260506 09:07:13.856 | INFO | Perf report for the last 5.13916 seconds: | 20260506 09:07:13.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:07:13.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:13.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:13.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:13.856 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260506 09:07:13.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:13.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:13.856 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:07:13.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:13.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:13.856 | INFO | Uploaded 36GiB to GPU in 1.60349s (22.4511GiB/s) | 20260506 09:07:13.856 | INFO | Downloaded 34GiB to GPU in 1.52064s (22.359GiB/s) | 20260506 09:07:18.996 | INFO | Analyzing chunk frames = [153600,154112) | 20260506 09:07:18.996 | INFO | MemInfo: { | 20260506 09:07:18.996 | INFO | "gpu" : " 17 GB", | 20260506 09:07:18.996 | INFO | "huge" : "", | 20260506 09:07:18.996 | INFO | "huge_shared" : " 27 GB", | 20260506 09:07:18.996 | INFO | "malloc" : " 14 GB", | 20260506 09:07:18.996 | INFO | "pinned" : " 29 GB" | 20260506 09:07:18.996 | INFO | } | 20260506 09:07:18.996 | INFO | Perf report for the last 5.13968 seconds: | 20260506 09:07:18.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:07:18.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:18.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:18.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:18.996 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:07:18.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:18.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:18.996 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:07:18.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:18.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:18.997 | INFO | Uploaded 36GiB to GPU in 1.60186s (22.4739GiB/s) | 20260506 09:07:18.997 | INFO | Downloaded 34GiB to GPU in 1.51872s (22.3872GiB/s) | 20260506 09:07:24.135 | INFO | Analyzing chunk frames = [154112,154624) | 20260506 09:07:24.135 | INFO | MemInfo: { | 20260506 09:07:24.135 | INFO | "gpu" : " 17 GB", | 20260506 09:07:24.135 | INFO | "huge" : "", | 20260506 09:07:24.135 | INFO | "huge_shared" : " 27 GB", | 20260506 09:07:24.135 | INFO | "malloc" : " 14 GB", | 20260506 09:07:24.135 | INFO | "pinned" : " 29 GB" | 20260506 09:07:24.135 | INFO | } | 20260506 09:07:24.135 | INFO | Perf report for the last 5.13923 seconds: | 20260506 09:07:24.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:07:24.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:24.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:24.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:24.135 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:07:24.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:24.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:24.135 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:07:24.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:24.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:24.136 | INFO | Uploaded 36GiB to GPU in 1.60215s (22.4697GiB/s) | 20260506 09:07:24.136 | INFO | Downloaded 34GiB to GPU in 1.52015s (22.3662GiB/s) | 20260506 09:07:29.274 | INFO | Analyzing chunk frames = [154624,155136) | 20260506 09:07:29.274 | INFO | MemInfo: { | 20260506 09:07:29.274 | INFO | "gpu" : " 17 GB", | 20260506 09:07:29.275 | INFO | "huge" : "", | 20260506 09:07:29.275 | INFO | "huge_shared" : " 27 GB", | 20260506 09:07:29.275 | INFO | "malloc" : " 14 GB", | 20260506 09:07:29.275 | INFO | "pinned" : " 29 GB" | 20260506 09:07:29.275 | INFO | } | 20260506 09:07:29.275 | INFO | Perf report for the last 5.13947 seconds: | 20260506 09:07:29.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:07:29.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:29.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:29.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:29.275 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:07:29.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:29.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:29.275 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:07:29.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:29.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:29.275 | INFO | Uploaded 36GiB to GPU in 1.60296s (22.4585GiB/s) | 20260506 09:07:29.275 | INFO | Downloaded 34GiB to GPU in 1.52062s (22.3593GiB/s) | 20260506 09:07:34.414 | INFO | Analyzing chunk frames = [155136,155648) | 20260506 09:07:34.414 | INFO | MemInfo: { | 20260506 09:07:34.414 | INFO | "gpu" : " 17 GB", | 20260506 09:07:34.414 | INFO | "huge" : "", | 20260506 09:07:34.414 | INFO | "huge_shared" : " 27 GB", | 20260506 09:07:34.414 | INFO | "malloc" : " 14 GB", | 20260506 09:07:34.414 | INFO | "pinned" : " 29 GB" | 20260506 09:07:34.414 | INFO | } | 20260506 09:07:34.414 | INFO | Perf report for the last 5.13918 seconds: | 20260506 09:07:34.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:07:34.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:34.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:34.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:34.414 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:07:34.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:34.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:34.414 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:07:34.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:34.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:34.414 | INFO | Uploaded 36GiB to GPU in 1.60379s (22.4469GiB/s) | 20260506 09:07:34.414 | INFO | Downloaded 34GiB to GPU in 1.52141s (22.3477GiB/s) | 20260506 09:07:34.414 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:07:34.414 | INFO | Total mem usage: 15.5 GiB | 20260506 09:07:34.414 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.23%) | 20260506 09:07:34.414 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -4.79%) | 20260506 09:07:34.414 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:07:34.414 | INFO | High Water Breakdown: | 20260506 09:07:34.414 | INFO | 58.6% : 8.68 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:07:34.414 | INFO | 41.3% : 6.12 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:07:34.414 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:07:34.414 | INFO | ------------------------------------------ | 20260506 09:07:34.414 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:07:34.414 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:07:34.414 | INFO | Total mem usage: 26.5 GiB | 20260506 09:07:34.414 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:07:34.414 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:07:34.414 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:07:34.414 | INFO | High Water Breakdown: | 20260506 09:07:34.414 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:07:34.414 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:07:34.414 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:07:34.414 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:07:34.415 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:07:34.415 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:07:34.415 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:07:34.415 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:07:34.415 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:07:34.415 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:07:34.415 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:07:34.415 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:07:34.415 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:07:34.415 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:07:34.415 | INFO | ------------------------------------------ | 20260506 09:07:34.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:07:34.415 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:07:34.415 | INFO | Total mem usage: 25.6 GiB | 20260506 09:07:34.415 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:07:34.415 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:07:34.415 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:07:34.415 | INFO | High Water Breakdown: | 20260506 09:07:34.415 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:07:34.415 | INFO | ------------------------------------------ | 20260506 09:07:34.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:07:34.415 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:07:34.415 | INFO | Total mem usage: 15.7 GiB | 20260506 09:07:34.415 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:07:34.415 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:07:34.415 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:07:34.415 | INFO | High Water Breakdown: | 20260506 09:07:34.415 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:07:34.415 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:07:34.415 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:07:34.415 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:07:34.415 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:07:34.415 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:07:34.415 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:07:34.415 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:07:34.415 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:07:34.415 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:07:34.415 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:07:34.415 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:07:34.415 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:07:34.415 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:07:34.415 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:07:34.415 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:07:34.415 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:07:34.415 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:07:34.415 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:07:34.415 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:07:34.415 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:07:34.415 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:07:34.415 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:07:34.415 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:07:34.415 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:07:34.415 | INFO | ------------------------------------------ | 20260506 09:07:34.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:07:34.415 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:07:34.415 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:07:34.415 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:07:34.415 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 09:07:34.415 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / 0.00e+00, 4.72e-03 / 0.00e+00 | 20260506 09:07:34.415 | INFO | PerfReport: 10.86%, Baseline -- 3.29e+00 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260506 09:07:34.415 | INFO | PerfReport: 3.04%, Histogram -- 9.22e-01 / 0.00e+00, 5.11e-02 / 0.00e+00 | 20260506 09:07:34.415 | INFO | PerfReport: 16.02%, DME -- 3.89e+01 / 0.00e+00, 3.14e+00 / 0.00e+00 | 20260506 09:07:34.415 | INFO | PerfReport: 56.72%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.15e-02 / 0.00e+00 | 20260506 09:07:34.415 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.31e+00 / 0.00e+00, 1.41e-01 / 0.00e+00 | 20260506 09:07:34.415 | INFO | PerfReport: 3.94%, Metrics -- 1.19e+00 / 0.00e+00, 1.85e-01 / 0.00e+00 | 20260506 09:07:34.415 | INFO | PerfReport: 1.54%, PulseSerializing -- 4.68e-01 / 0.00e+00, 6.78e-02 / 0.00e+00 | 20260506 09:07:34.415 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:07:34.415 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:07:34.415 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:07:34.415 | INFO | PerfReport: Full Execution -- 9.10e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260506 09:07:34.415 | INFO | PerfReport: 18.39%, Upload -- 1.67e+01 / 0.00e+00, 3.72e-01 / 0.00e+00 | 20260506 09:07:34.415 | INFO | PerfReport: 17.65%, Download -- 1.61e+01 / 0.00e+00, 6.79e-01 / 0.00e+00 | 20260506 09:07:34.415 | INFO | PerfReport: 33.33%, Compute -- 3.03e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260506 09:07:34.415 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260506 09:07:34.415 | INFO | PerfReport: 0.00%, LockDownload -- 4.83e-04 / 0.00e+00, 1.24e-04 / 0.00e+00 | 20260506 09:07:34.415 | INFO | PerfReport: 30.05%, LockCompute -- 2.73e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 09:07:34.415 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260506 09:07:34.511 | INFO | Starting a round of Baz IO | 20260506 09:07:37.446 | INFO | Populating RTMetrics report | 20260506 09:07:37.446 | INFO | RTMetrics report populated | 20260506 09:07:37.446 | INFO | Opening tmp RTMetrics json file | 20260506 09:07:37.447 | INFO | Serializing RTMetrics to json | 20260506 09:07:37.448 | INFO | Serialization done, swapping tmp file now... | 20260506 09:07:37.449 | INFO | RTMetrics done | 20260506 09:07:37.537 | INFO | Finished a round of Baz IO: Wrote 1.735 GiB in 3.025 seconds | 20260506 09:07:37.537 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.33% / 0.20% / 41.47% | 20260506 09:07:37.537 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.8405% / 0.000659625% / 1.98682e-05% | 20260506 09:07:37.537 | INFO | Baz IO overall: | 20260506 09:07:37.537 | INFO | BytesWritten : 48023253107 | 20260506 09:07:37.537 | INFO | NumEvents : 21901072492 | 20260506 09:07:39.553 | INFO | Analyzing chunk frames = [155648,156160) | 20260506 09:07:39.553 | INFO | MemInfo: { | 20260506 09:07:39.553 | INFO | "gpu" : " 17 GB", | 20260506 09:07:39.553 | INFO | "huge" : "", | 20260506 09:07:39.553 | INFO | "huge_shared" : " 27 GB", | 20260506 09:07:39.553 | INFO | "malloc" : " 14 GB", | 20260506 09:07:39.553 | INFO | "pinned" : " 29 GB" | 20260506 09:07:39.553 | INFO | } | 20260506 09:07:39.553 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:07:39.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:07:39.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:39.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:39.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:39.553 | INFO | ANALYSIS : 58.0%, 2.9, 87.2ms, 0 | 20260506 09:07:39.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:39.553 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 09:07:39.553 | INFO | PRE_HQ : 57.2%, 2.1, 61.9ms, 0 | 20260506 09:07:39.553 | INFO | BAZWRITER : 58.4%, 4.6, 137.9ms, 0 | 20260506 09:07:39.553 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 09:07:39.553 | INFO | Uploaded 36GiB to GPU in 1.65858s (21.7054GiB/s) | 20260506 09:07:39.553 | INFO | Downloaded 38GiB to GPU in 1.72236s (22.0628GiB/s) | 20260506 09:07:44.692 | INFO | Analyzing chunk frames = [156160,156672) | 20260506 09:07:44.692 | INFO | MemInfo: { | 20260506 09:07:44.692 | INFO | "gpu" : " 17 GB", | 20260506 09:07:44.692 | INFO | "huge" : "", | 20260506 09:07:44.692 | INFO | "huge_shared" : " 27 GB", | 20260506 09:07:44.692 | INFO | "malloc" : " 14 GB", | 20260506 09:07:44.692 | INFO | "pinned" : " 29 GB" | 20260506 09:07:44.692 | INFO | } | 20260506 09:07:44.692 | INFO | Perf report for the last 5.1392 seconds: | 20260506 09:07:44.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:07:44.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:44.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:44.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:44.692 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260506 09:07:44.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:44.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:44.692 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:07:44.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:44.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:44.693 | INFO | Uploaded 36GiB to GPU in 1.60095s (22.4866GiB/s) | 20260506 09:07:44.693 | INFO | Downloaded 34GiB to GPU in 1.51749s (22.4054GiB/s) | 20260506 09:07:49.831 | INFO | Analyzing chunk frames = [156672,157184) | 20260506 09:07:49.831 | INFO | MemInfo: { | 20260506 09:07:49.831 | INFO | "gpu" : " 17 GB", | 20260506 09:07:49.832 | INFO | "huge" : "", | 20260506 09:07:49.832 | INFO | "huge_shared" : " 27 GB", | 20260506 09:07:49.832 | INFO | "malloc" : " 14 GB", | 20260506 09:07:49.832 | INFO | "pinned" : " 29 GB" | 20260506 09:07:49.832 | INFO | } | 20260506 09:07:49.832 | INFO | Perf report for the last 5.13923 seconds: | 20260506 09:07:49.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:07:49.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:49.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:49.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:49.832 | INFO | ANALYSIS : 57.2%, 3.0, 88.9ms, 0 | 20260506 09:07:49.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:49.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:49.832 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:07:49.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:49.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:49.832 | INFO | Uploaded 36GiB to GPU in 1.6043s (22.4397GiB/s) | 20260506 09:07:49.832 | INFO | Downloaded 34GiB to GPU in 1.52204s (22.3384GiB/s) | 20260506 09:07:54.971 | INFO | Analyzing chunk frames = [157184,157696) | 20260506 09:07:54.971 | INFO | MemInfo: { | 20260506 09:07:54.971 | INFO | "gpu" : " 17 GB", | 20260506 09:07:54.971 | INFO | "huge" : "", | 20260506 09:07:54.971 | INFO | "huge_shared" : " 27 GB", | 20260506 09:07:54.971 | INFO | "malloc" : " 14 GB", | 20260506 09:07:54.971 | INFO | "pinned" : " 29 GB" | 20260506 09:07:54.971 | INFO | } | 20260506 09:07:54.971 | INFO | Perf report for the last 5.13953 seconds: | 20260506 09:07:54.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:07:54.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:54.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:54.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:54.971 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260506 09:07:54.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:54.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:54.971 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:07:54.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:54.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:07:54.971 | INFO | Uploaded 36GiB to GPU in 1.60359s (22.4497GiB/s) | 20260506 09:07:54.971 | INFO | Downloaded 34GiB to GPU in 1.52142s (22.3475GiB/s) | 20260506 09:08:00.110 | INFO | Analyzing chunk frames = [157696,158208) | 20260506 09:08:00.110 | INFO | MemInfo: { | 20260506 09:08:00.110 | INFO | "gpu" : " 17 GB", | 20260506 09:08:00.110 | INFO | "huge" : "", | 20260506 09:08:00.110 | INFO | "huge_shared" : " 27 GB", | 20260506 09:08:00.110 | INFO | "malloc" : " 14 GB", | 20260506 09:08:00.110 | INFO | "pinned" : " 29 GB" | 20260506 09:08:00.110 | INFO | } | 20260506 09:08:00.110 | INFO | Perf report for the last 5.13921 seconds: | 20260506 09:08:00.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:08:00.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:00.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:00.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:00.110 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 09:08:00.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:00.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:00.110 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:08:00.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:00.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:00.111 | INFO | Uploaded 36GiB to GPU in 1.60234s (22.4671GiB/s) | 20260506 09:08:00.111 | INFO | Downloaded 34GiB to GPU in 1.51986s (22.3705GiB/s) | 20260506 09:08:05.249 | INFO | Analyzing chunk frames = [158208,158720) | 20260506 09:08:05.249 | INFO | MemInfo: { | 20260506 09:08:05.249 | INFO | "gpu" : " 17 GB", | 20260506 09:08:05.249 | INFO | "huge" : "", | 20260506 09:08:05.250 | INFO | "huge_shared" : " 27 GB", | 20260506 09:08:05.250 | INFO | "malloc" : " 14 GB", | 20260506 09:08:05.250 | INFO | "pinned" : " 29 GB" | 20260506 09:08:05.250 | INFO | } | 20260506 09:08:05.250 | INFO | Perf report for the last 5.13923 seconds: | 20260506 09:08:05.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:08:05.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:05.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:05.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:05.250 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:08:05.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:05.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:05.250 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 09:08:05.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:05.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:05.250 | INFO | Uploaded 36GiB to GPU in 1.6015s (22.4789GiB/s) | 20260506 09:08:05.250 | INFO | Downloaded 34GiB to GPU in 1.51945s (22.3765GiB/s) | 20260506 09:08:10.389 | INFO | Analyzing chunk frames = [158720,159232) | 20260506 09:08:10.389 | INFO | MemInfo: { | 20260506 09:08:10.389 | INFO | "gpu" : " 17 GB", | 20260506 09:08:10.389 | INFO | "huge" : "", | 20260506 09:08:10.389 | INFO | "huge_shared" : " 27 GB", | 20260506 09:08:10.389 | INFO | "malloc" : " 14 GB", | 20260506 09:08:10.389 | INFO | "pinned" : " 29 GB" | 20260506 09:08:10.389 | INFO | } | 20260506 09:08:10.389 | INFO | Perf report for the last 5.1394 seconds: | 20260506 09:08:10.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:08:10.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:10.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:10.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:10.389 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 09:08:10.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:10.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:10.389 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:08:10.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:10.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:10.389 | INFO | Uploaded 36GiB to GPU in 1.60338s (22.4525GiB/s) | 20260506 09:08:10.389 | INFO | Downloaded 34GiB to GPU in 1.52096s (22.3544GiB/s) | 20260506 09:08:15.528 | INFO | Analyzing chunk frames = [159232,159744) | 20260506 09:08:15.528 | INFO | MemInfo: { | 20260506 09:08:15.528 | INFO | "gpu" : " 17 GB", | 20260506 09:08:15.528 | INFO | "huge" : "", | 20260506 09:08:15.528 | INFO | "huge_shared" : " 27 GB", | 20260506 09:08:15.528 | INFO | "malloc" : " 14 GB", | 20260506 09:08:15.528 | INFO | "pinned" : " 29 GB" | 20260506 09:08:15.528 | INFO | } | 20260506 09:08:15.528 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:08:15.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:08:15.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:15.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:15.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:15.528 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 09:08:15.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:15.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:15.528 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 09:08:15.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:15.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:15.528 | INFO | Uploaded 36GiB to GPU in 1.60292s (22.459GiB/s) | 20260506 09:08:15.529 | INFO | Downloaded 34GiB to GPU in 1.52081s (22.3566GiB/s) | 20260506 09:08:15.648 | INFO | Starting a round of Baz IO | 20260506 09:08:18.555 | INFO | Populating RTMetrics report | 20260506 09:08:18.556 | INFO | RTMetrics report populated | 20260506 09:08:18.556 | INFO | Opening tmp RTMetrics json file | 20260506 09:08:18.556 | INFO | Serializing RTMetrics to json | 20260506 09:08:18.558 | INFO | Serialization done, swapping tmp file now... | 20260506 09:08:18.559 | INFO | RTMetrics done | 20260506 09:08:18.648 | INFO | Finished a round of Baz IO: Wrote 1.74 GiB in 3 seconds | 20260506 09:08:18.648 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 58.14% / 0.19% / 41.67% | 20260506 09:08:18.648 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.7903% / 0.000639757% / 3.17891e-05% | 20260506 09:08:18.648 | INFO | Baz IO overall: | 20260506 09:08:18.648 | INFO | BytesWritten : 49891567518 | 20260506 09:08:18.648 | INFO | NumEvents : 22809524909 | 20260506 09:08:20.668 | INFO | Analyzing chunk frames = [159744,160256) | 20260506 09:08:20.668 | INFO | MemInfo: { | 20260506 09:08:20.668 | INFO | "gpu" : " 17 GB", | 20260506 09:08:20.668 | INFO | "huge" : "", | 20260506 09:08:20.668 | INFO | "huge_shared" : " 27 GB", | 20260506 09:08:20.668 | INFO | "malloc" : " 14 GB", | 20260506 09:08:20.668 | INFO | "pinned" : " 29 GB" | 20260506 09:08:20.668 | INFO | } | 20260506 09:08:20.668 | INFO | Perf report for the last 5.13966 seconds: | 20260506 09:08:20.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:08:20.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:20.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:20.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:20.668 | INFO | ANALYSIS : 57.9%, 3.0, 90.0ms, 0 | 20260506 09:08:20.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:20.668 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 09:08:20.668 | INFO | PRE_HQ : 56.9%, 2.1, 62.1ms, 0 | 20260506 09:08:20.668 | INFO | BAZWRITER : 58.4%, 4.1, 123.2ms, 0 | 20260506 09:08:20.668 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 09:08:20.668 | INFO | Uploaded 36GiB to GPU in 1.63945s (21.9586GiB/s) | 20260506 09:08:20.668 | INFO | Downloaded 38GiB to GPU in 1.7004s (22.3477GiB/s) | 20260506 09:08:25.807 | INFO | Analyzing chunk frames = [160256,160768) | 20260506 09:08:25.807 | INFO | MemInfo: { | 20260506 09:08:25.807 | INFO | "gpu" : " 17 GB", | 20260506 09:08:25.807 | INFO | "huge" : "", | 20260506 09:08:25.807 | INFO | "huge_shared" : " 27 GB", | 20260506 09:08:25.807 | INFO | "malloc" : " 14 GB", | 20260506 09:08:25.807 | INFO | "pinned" : " 29 GB" | 20260506 09:08:25.807 | INFO | } | 20260506 09:08:25.807 | INFO | Perf report for the last 5.13905 seconds: | 20260506 09:08:25.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:08:25.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:25.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:25.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:25.807 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260506 09:08:25.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:25.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:25.807 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:08:25.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:25.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:25.807 | INFO | Uploaded 36GiB to GPU in 1.6007s (22.4901GiB/s) | 20260506 09:08:25.807 | INFO | Downloaded 34GiB to GPU in 1.51707s (22.4117GiB/s) | 20260506 09:08:30.946 | INFO | Analyzing chunk frames = [160768,161280) | 20260506 09:08:30.946 | INFO | MemInfo: { | 20260506 09:08:30.946 | INFO | "gpu" : " 17 GB", | 20260506 09:08:30.946 | INFO | "huge" : "", | 20260506 09:08:30.946 | INFO | "huge_shared" : " 27 GB", | 20260506 09:08:30.946 | INFO | "malloc" : " 14 GB", | 20260506 09:08:30.946 | INFO | "pinned" : " 29 GB" | 20260506 09:08:30.946 | INFO | } | 20260506 09:08:30.946 | INFO | Perf report for the last 5.13935 seconds: | 20260506 09:08:30.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:08:30.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:30.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:30.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:30.946 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260506 09:08:30.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:30.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:30.946 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:08:30.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:30.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:30.947 | INFO | Uploaded 36GiB to GPU in 1.60523s (22.4267GiB/s) | 20260506 09:08:30.947 | INFO | Downloaded 34GiB to GPU in 1.52281s (22.3272GiB/s) | 20260506 09:08:36.086 | INFO | Analyzing chunk frames = [161280,161792) | 20260506 09:08:36.086 | INFO | MemInfo: { | 20260506 09:08:36.086 | INFO | "gpu" : " 17 GB", | 20260506 09:08:36.086 | INFO | "huge" : "", | 20260506 09:08:36.086 | INFO | "huge_shared" : " 27 GB", | 20260506 09:08:36.086 | INFO | "malloc" : " 14 GB", | 20260506 09:08:36.086 | INFO | "pinned" : " 29 GB" | 20260506 09:08:36.086 | INFO | } | 20260506 09:08:36.086 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:08:36.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:08:36.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:36.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:36.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:36.086 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260506 09:08:36.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:36.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:36.086 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:08:36.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:36.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:36.086 | INFO | Uploaded 36GiB to GPU in 1.60405s (22.4431GiB/s) | 20260506 09:08:36.087 | INFO | Downloaded 34GiB to GPU in 1.52125s (22.35GiB/s) | 20260506 09:08:41.225 | INFO | Analyzing chunk frames = [161792,162304) | 20260506 09:08:41.225 | INFO | MemInfo: { | 20260506 09:08:41.225 | INFO | "gpu" : " 17 GB", | 20260506 09:08:41.225 | INFO | "huge" : "", | 20260506 09:08:41.225 | INFO | "huge_shared" : " 27 GB", | 20260506 09:08:41.225 | INFO | "malloc" : " 14 GB", | 20260506 09:08:41.225 | INFO | "pinned" : " 29 GB" | 20260506 09:08:41.225 | INFO | } | 20260506 09:08:41.225 | INFO | Perf report for the last 5.13907 seconds: | 20260506 09:08:41.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:08:41.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:41.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:41.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:41.225 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 09:08:41.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:41.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:41.225 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:08:41.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:41.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:41.225 | INFO | Uploaded 36GiB to GPU in 1.60277s (22.461GiB/s) | 20260506 09:08:41.225 | INFO | Downloaded 34GiB to GPU in 1.51939s (22.3774GiB/s) | 20260506 09:08:46.364 | INFO | Analyzing chunk frames = [162304,162816) | 20260506 09:08:46.364 | INFO | MemInfo: { | 20260506 09:08:46.364 | INFO | "gpu" : " 17 GB", | 20260506 09:08:46.364 | INFO | "huge" : "", | 20260506 09:08:46.365 | INFO | "huge_shared" : " 27 GB", | 20260506 09:08:46.365 | INFO | "malloc" : " 14 GB", | 20260506 09:08:46.365 | INFO | "pinned" : " 29 GB" | 20260506 09:08:46.365 | INFO | } | 20260506 09:08:46.365 | INFO | Perf report for the last 5.13967 seconds: | 20260506 09:08:46.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:08:46.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:46.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:46.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:46.365 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:08:46.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:46.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:46.365 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:08:46.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:46.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:46.365 | INFO | Uploaded 36GiB to GPU in 1.60346s (22.4515GiB/s) | 20260506 09:08:46.365 | INFO | Downloaded 34GiB to GPU in 1.51999s (22.3686GiB/s) | 20260506 09:08:51.504 | INFO | Analyzing chunk frames = [162816,163328) | 20260506 09:08:51.504 | INFO | MemInfo: { | 20260506 09:08:51.504 | INFO | "gpu" : " 17 GB", | 20260506 09:08:51.504 | INFO | "huge" : "", | 20260506 09:08:51.504 | INFO | "huge_shared" : " 27 GB", | 20260506 09:08:51.504 | INFO | "malloc" : " 14 GB", | 20260506 09:08:51.504 | INFO | "pinned" : " 29 GB" | 20260506 09:08:51.504 | INFO | } | 20260506 09:08:51.504 | INFO | Perf report for the last 5.13918 seconds: | 20260506 09:08:51.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:08:51.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:51.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:51.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:51.504 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 09:08:51.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:51.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:51.504 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:08:51.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:51.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:51.504 | INFO | Uploaded 36GiB to GPU in 1.60343s (22.4518GiB/s) | 20260506 09:08:51.504 | INFO | Downloaded 34GiB to GPU in 1.52074s (22.3575GiB/s) | 20260506 09:08:56.643 | INFO | Analyzing chunk frames = [163328,163840) | 20260506 09:08:56.643 | INFO | MemInfo: { | 20260506 09:08:56.643 | INFO | "gpu" : " 17 GB", | 20260506 09:08:56.643 | INFO | "huge" : "", | 20260506 09:08:56.643 | INFO | "huge_shared" : " 27 GB", | 20260506 09:08:56.643 | INFO | "malloc" : " 14 GB", | 20260506 09:08:56.643 | INFO | "pinned" : " 29 GB" | 20260506 09:08:56.643 | INFO | } | 20260506 09:08:56.643 | INFO | Perf report for the last 5.13936 seconds: | 20260506 09:08:56.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:08:56.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:56.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:56.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:56.643 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 09:08:56.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:56.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:56.643 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 09:08:56.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:56.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:08:56.643 | INFO | Uploaded 36GiB to GPU in 1.60327s (22.4541GiB/s) | 20260506 09:08:56.643 | INFO | Downloaded 34GiB to GPU in 1.52024s (22.3649GiB/s) | 20260506 09:08:56.644 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:08:56.644 | INFO | Total mem usage: 15.5 GiB | 20260506 09:08:56.644 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.23%) | 20260506 09:08:56.644 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -4.83%) | 20260506 09:08:56.644 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:08:56.644 | INFO | High Water Breakdown: | 20260506 09:08:56.644 | INFO | 58.6% : 8.67 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:08:56.644 | INFO | 41.3% : 6.12 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:08:56.644 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:08:56.644 | INFO | ------------------------------------------ | 20260506 09:08:56.644 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:08:56.644 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:08:56.644 | INFO | Total mem usage: 26.5 GiB | 20260506 09:08:56.644 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:08:56.644 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:08:56.644 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:08:56.644 | INFO | High Water Breakdown: | 20260506 09:08:56.644 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:08:56.644 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:08:56.644 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:08:56.644 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:08:56.644 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:08:56.644 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:08:56.644 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:08:56.644 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:08:56.644 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:08:56.644 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:08:56.644 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:08:56.644 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:08:56.644 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:08:56.644 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:08:56.644 | INFO | ------------------------------------------ | 20260506 09:08:56.644 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:08:56.644 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:08:56.644 | INFO | Total mem usage: 25.6 GiB | 20260506 09:08:56.644 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:08:56.644 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:08:56.644 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:08:56.644 | INFO | High Water Breakdown: | 20260506 09:08:56.644 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:08:56.644 | INFO | ------------------------------------------ | 20260506 09:08:56.644 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:08:56.644 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:08:56.644 | INFO | Total mem usage: 15.7 GiB | 20260506 09:08:56.644 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:08:56.644 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:08:56.644 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:08:56.644 | INFO | High Water Breakdown: | 20260506 09:08:56.644 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:08:56.644 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:08:56.644 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:08:56.644 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:08:56.644 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:08:56.644 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:08:56.644 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:08:56.644 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:08:56.644 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:08:56.644 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:08:56.644 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:08:56.644 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:08:56.644 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:08:56.644 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:08:56.644 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:08:56.644 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:08:56.644 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:08:56.644 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:08:56.644 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:08:56.644 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:08:56.644 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:08:56.644 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:08:56.644 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:08:56.644 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:08:56.644 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:08:56.644 | INFO | ------------------------------------------ | 20260506 09:08:56.644 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:08:56.644 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:08:56.644 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:08:56.644 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:08:56.644 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 09:08:56.644 | INFO | PerfReport: 0.29%, AggregateStats -- 8.64e-02 / 0.00e+00, 5.21e-03 / 0.00e+00 | 20260506 09:08:56.644 | INFO | PerfReport: 10.86%, Baseline -- 3.29e+00 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260506 09:08:56.644 | INFO | PerfReport: 3.04%, Histogram -- 9.22e-01 / 0.00e+00, 5.22e-02 / 0.00e+00 | 20260506 09:08:56.644 | INFO | PerfReport: 16.02%, DME -- 3.89e+01 / 0.00e+00, 3.23e+00 / 0.00e+00 | 20260506 09:08:56.644 | INFO | PerfReport: 56.73%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.22e-02 / 0.00e+00 | 20260506 09:08:56.644 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.30e+00 / 0.00e+00, 1.40e-01 / 0.00e+00 | 20260506 09:08:56.644 | INFO | PerfReport: 3.93%, Metrics -- 1.19e+00 / 0.00e+00, 1.86e-01 / 0.00e+00 | 20260506 09:08:56.644 | INFO | PerfReport: 1.54%, PulseSerializing -- 4.67e-01 / 0.00e+00, 7.43e-02 / 0.00e+00 | 20260506 09:08:56.644 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:08:56.644 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:08:56.644 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:08:56.644 | INFO | PerfReport: Full Execution -- 9.08e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 09:08:56.644 | INFO | PerfReport: 18.45%, Upload -- 1.68e+01 / 0.00e+00, 4.19e-01 / 0.00e+00 | 20260506 09:08:56.644 | INFO | PerfReport: 17.71%, Download -- 1.61e+01 / 0.00e+00, 7.39e-01 / 0.00e+00 | 20260506 09:08:56.644 | INFO | PerfReport: 33.38%, Compute -- 3.03e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 09:08:56.644 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:08:56.644 | INFO | PerfReport: 0.00%, LockDownload -- 4.87e-04 / 0.00e+00, 1.66e-04 / 0.00e+00 | 20260506 09:08:56.644 | INFO | PerfReport: 29.90%, LockCompute -- 2.72e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 09:08:56.644 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260506 09:08:56.740 | INFO | Starting a round of Baz IO | 20260506 09:08:59.670 | INFO | Populating RTMetrics report | 20260506 09:08:59.671 | INFO | RTMetrics report populated | 20260506 09:08:59.671 | INFO | Opening tmp RTMetrics json file | 20260506 09:08:59.675 | INFO | Serializing RTMetrics to json | 20260506 09:08:59.699 | INFO | Serialization done, swapping tmp file now... | 20260506 09:08:59.712 | INFO | RTMetrics done | 20260506 09:08:59.764 | INFO | Finished a round of Baz IO: Wrote 1.744 GiB in 3.023 seconds | 20260506 09:08:59.764 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.96% / 0.18% / 41.86% | 20260506 09:08:59.764 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.7222% / 0.000683467% / 7.94729e-06% | 20260506 09:08:59.764 | INFO | Baz IO overall: | 20260506 09:08:59.764 | INFO | BytesWritten : 51764665688 | 20260506 09:08:59.764 | INFO | NumEvents : 23719920817 | 20260506 09:09:01.782 | INFO | Analyzing chunk frames = [163840,164352) | 20260506 09:09:01.782 | INFO | MemInfo: { | 20260506 09:09:01.782 | INFO | "gpu" : " 17 GB", | 20260506 09:09:01.782 | INFO | "huge" : "", | 20260506 09:09:01.782 | INFO | "huge_shared" : " 27 GB", | 20260506 09:09:01.782 | INFO | "malloc" : " 14 GB", | 20260506 09:09:01.782 | INFO | "pinned" : " 29 GB" | 20260506 09:09:01.782 | INFO | } | 20260506 09:09:01.782 | INFO | Perf report for the last 5.1392 seconds: | 20260506 09:09:01.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:09:01.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:01.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:01.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:01.782 | INFO | ANALYSIS : 58.0%, 2.9, 89.2ms, 0 | 20260506 09:09:01.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:01.782 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 09:09:01.782 | INFO | PRE_HQ : 57.1%, 2.1, 62.2ms, 0 | 20260506 09:09:01.782 | INFO | BAZWRITER : 58.4%, 4.2, 127.2ms, 0 | 20260506 09:09:01.782 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 09:09:01.783 | INFO | Uploaded 36GiB to GPU in 1.64615s (21.8692GiB/s) | 20260506 09:09:01.783 | INFO | Downloaded 38GiB to GPU in 1.70763s (22.253GiB/s) | 20260506 09:09:06.922 | INFO | Analyzing chunk frames = [164352,164864) | 20260506 09:09:06.922 | INFO | MemInfo: { | 20260506 09:09:06.922 | INFO | "gpu" : " 17 GB", | 20260506 09:09:06.922 | INFO | "huge" : "", | 20260506 09:09:06.922 | INFO | "huge_shared" : " 27 GB", | 20260506 09:09:06.922 | INFO | "malloc" : " 14 GB", | 20260506 09:09:06.922 | INFO | "pinned" : " 29 GB" | 20260506 09:09:06.922 | INFO | } | 20260506 09:09:06.922 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:09:06.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:09:06.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:06.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:06.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:06.922 | INFO | ANALYSIS : 57.8%, 3.0, 90.2ms, 0 | 20260506 09:09:06.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:06.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:06.922 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:09:06.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:06.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:06.922 | INFO | Uploaded 36GiB to GPU in 1.601s (22.486GiB/s) | 20260506 09:09:06.922 | INFO | Downloaded 34GiB to GPU in 1.51778s (22.4012GiB/s) | 20260506 09:09:12.061 | INFO | Analyzing chunk frames = [164864,165376) | 20260506 09:09:12.061 | INFO | MemInfo: { | 20260506 09:09:12.061 | INFO | "gpu" : " 17 GB", | 20260506 09:09:12.061 | INFO | "huge" : "", | 20260506 09:09:12.061 | INFO | "huge_shared" : " 27 GB", | 20260506 09:09:12.061 | INFO | "malloc" : " 14 GB", | 20260506 09:09:12.061 | INFO | "pinned" : " 29 GB" | 20260506 09:09:12.061 | INFO | } | 20260506 09:09:12.061 | INFO | Perf report for the last 5.13945 seconds: | 20260506 09:09:12.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:09:12.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:12.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:12.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:12.061 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260506 09:09:12.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:12.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:12.061 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:09:12.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:12.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:12.061 | INFO | Uploaded 36GiB to GPU in 1.60531s (22.4256GiB/s) | 20260506 09:09:12.062 | INFO | Downloaded 34GiB to GPU in 1.52326s (22.3205GiB/s) | 20260506 09:09:17.200 | INFO | Analyzing chunk frames = [165376,165888) | 20260506 09:09:17.200 | INFO | MemInfo: { | 20260506 09:09:17.200 | INFO | "gpu" : " 17 GB", | 20260506 09:09:17.200 | INFO | "huge" : "", | 20260506 09:09:17.200 | INFO | "huge_shared" : " 27 GB", | 20260506 09:09:17.200 | INFO | "malloc" : " 14 GB", | 20260506 09:09:17.200 | INFO | "pinned" : " 29 GB" | 20260506 09:09:17.201 | INFO | } | 20260506 09:09:17.201 | INFO | Perf report for the last 5.1394 seconds: | 20260506 09:09:17.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:09:17.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:17.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:17.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:17.201 | INFO | ANALYSIS : 56.9%, 3.0, 88.2ms, 0 | 20260506 09:09:17.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:17.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:17.201 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:09:17.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:17.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:17.201 | INFO | Uploaded 36GiB to GPU in 1.60466s (22.4347GiB/s) | 20260506 09:09:17.201 | INFO | Downloaded 34GiB to GPU in 1.52154s (22.3458GiB/s) | 20260506 09:09:22.340 | INFO | Analyzing chunk frames = [165888,166400) | 20260506 09:09:22.340 | INFO | MemInfo: { | 20260506 09:09:22.340 | INFO | "gpu" : " 17 GB", | 20260506 09:09:22.340 | INFO | "huge" : "", | 20260506 09:09:22.340 | INFO | "huge_shared" : " 27 GB", | 20260506 09:09:22.340 | INFO | "malloc" : " 14 GB", | 20260506 09:09:22.340 | INFO | "pinned" : " 29 GB" | 20260506 09:09:22.340 | INFO | } | 20260506 09:09:22.340 | INFO | Perf report for the last 5.13915 seconds: | 20260506 09:09:22.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:09:22.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:22.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:22.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:22.340 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 09:09:22.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:22.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:22.340 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:09:22.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:22.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:22.340 | INFO | Uploaded 36GiB to GPU in 1.60291s (22.4591GiB/s) | 20260506 09:09:22.340 | INFO | Downloaded 34GiB to GPU in 1.51988s (22.3701GiB/s) | 20260506 09:09:27.479 | INFO | Analyzing chunk frames = [166400,166912) | 20260506 09:09:27.479 | INFO | MemInfo: { | 20260506 09:09:27.479 | INFO | "gpu" : " 17 GB", | 20260506 09:09:27.479 | INFO | "huge" : "", | 20260506 09:09:27.479 | INFO | "huge_shared" : " 27 GB", | 20260506 09:09:27.479 | INFO | "malloc" : " 14 GB", | 20260506 09:09:27.479 | INFO | "pinned" : " 29 GB" | 20260506 09:09:27.479 | INFO | } | 20260506 09:09:27.479 | INFO | Perf report for the last 5.13938 seconds: | 20260506 09:09:27.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:09:27.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:27.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:27.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:27.479 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:09:27.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:27.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:27.479 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:09:27.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:27.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:27.479 | INFO | Uploaded 36GiB to GPU in 1.60254s (22.4643GiB/s) | 20260506 09:09:27.479 | INFO | Downloaded 34GiB to GPU in 1.51948s (22.376GiB/s) | 20260506 09:09:32.618 | INFO | Analyzing chunk frames = [166912,167424) | 20260506 09:09:32.618 | INFO | MemInfo: { | 20260506 09:09:32.618 | INFO | "gpu" : " 17 GB", | 20260506 09:09:32.618 | INFO | "huge" : "", | 20260506 09:09:32.618 | INFO | "huge_shared" : " 27 GB", | 20260506 09:09:32.618 | INFO | "malloc" : " 14 GB", | 20260506 09:09:32.618 | INFO | "pinned" : " 29 GB" | 20260506 09:09:32.618 | INFO | } | 20260506 09:09:32.618 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:09:32.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:09:32.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:32.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:32.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:32.618 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 09:09:32.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:32.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:32.619 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:09:32.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:32.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:32.619 | INFO | Uploaded 36GiB to GPU in 1.60296s (22.4584GiB/s) | 20260506 09:09:32.619 | INFO | Downloaded 34GiB to GPU in 1.5219s (22.3406GiB/s) | 20260506 09:09:37.758 | INFO | Analyzing chunk frames = [167424,167936) | 20260506 09:09:37.758 | INFO | MemInfo: { | 20260506 09:09:37.758 | INFO | "gpu" : " 17 GB", | 20260506 09:09:37.758 | INFO | "huge" : "", | 20260506 09:09:37.758 | INFO | "huge_shared" : " 27 GB", | 20260506 09:09:37.758 | INFO | "malloc" : " 14 GB", | 20260506 09:09:37.758 | INFO | "pinned" : " 29 GB" | 20260506 09:09:37.758 | INFO | } | 20260506 09:09:37.758 | INFO | Perf report for the last 5.13922 seconds: | 20260506 09:09:37.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:09:37.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:37.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:37.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:37.758 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 09:09:37.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:37.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:37.758 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 09:09:37.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:37.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:37.758 | INFO | Uploaded 36GiB to GPU in 1.60265s (22.4628GiB/s) | 20260506 09:09:37.758 | INFO | Downloaded 34GiB to GPU in 1.52097s (22.3541GiB/s) | 20260506 09:09:37.855 | INFO | Starting a round of Baz IO | 20260506 09:09:40.790 | INFO | Populating RTMetrics report | 20260506 09:09:40.790 | INFO | RTMetrics report populated | 20260506 09:09:40.790 | INFO | Opening tmp RTMetrics json file | 20260506 09:09:40.791 | INFO | Serializing RTMetrics to json | 20260506 09:09:40.793 | INFO | Serialization done, swapping tmp file now... | 20260506 09:09:40.793 | INFO | RTMetrics done | 20260506 09:09:40.869 | INFO | Finished a round of Baz IO: Wrote 1.749 GiB in 3.014 seconds | 20260506 09:09:40.869 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.78% / 0.18% / 42.04% | 20260506 09:09:40.869 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.6679% / 0.000615915% / 5.16574e-05% | 20260506 09:09:40.869 | INFO | Baz IO overall: | 20260506 09:09:40.869 | INFO | BytesWritten : 53643137080 | 20260506 09:09:40.869 | INFO | NumEvents : 24632859859 | 20260506 09:09:42.897 | INFO | Analyzing chunk frames = [167936,168448) | 20260506 09:09:42.897 | INFO | MemInfo: { | 20260506 09:09:42.897 | INFO | "gpu" : " 17 GB", | 20260506 09:09:42.897 | INFO | "huge" : "", | 20260506 09:09:42.897 | INFO | "huge_shared" : " 27 GB", | 20260506 09:09:42.897 | INFO | "malloc" : " 14 GB", | 20260506 09:09:42.897 | INFO | "pinned" : " 29 GB" | 20260506 09:09:42.897 | INFO | } | 20260506 09:09:42.897 | INFO | Perf report for the last 5.13948 seconds: | 20260506 09:09:42.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:09:42.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:42.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:42.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:42.897 | INFO | ANALYSIS : 58.0%, 2.9, 88.1ms, 0 | 20260506 09:09:42.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:42.897 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 09:09:42.897 | INFO | PRE_HQ : 57.3%, 2.1, 62.3ms, 0 | 20260506 09:09:42.897 | INFO | BAZWRITER : 58.2%, 4.4, 132.1ms, 0 | 20260506 09:09:42.897 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 09:09:42.897 | INFO | Uploaded 36GiB to GPU in 1.65597s (21.7395GiB/s) | 20260506 09:09:42.897 | INFO | Downloaded 38GiB to GPU in 1.71889s (22.1073GiB/s) | 20260506 09:09:48.036 | INFO | Analyzing chunk frames = [168448,168960) | 20260506 09:09:48.036 | INFO | MemInfo: { | 20260506 09:09:48.036 | INFO | "gpu" : " 17 GB", | 20260506 09:09:48.036 | INFO | "huge" : "", | 20260506 09:09:48.036 | INFO | "huge_shared" : " 27 GB", | 20260506 09:09:48.036 | INFO | "malloc" : " 14 GB", | 20260506 09:09:48.036 | INFO | "pinned" : " 29 GB" | 20260506 09:09:48.036 | INFO | } | 20260506 09:09:48.036 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:09:48.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:09:48.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:48.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:48.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:48.037 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260506 09:09:48.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:48.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:48.037 | INFO | PRE_HQ : 30.5%, 1.0, 16.1ms, 0 | 20260506 09:09:48.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:48.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:48.037 | INFO | Uploaded 36GiB to GPU in 1.60108s (22.4849GiB/s) | 20260506 09:09:48.037 | INFO | Downloaded 34GiB to GPU in 1.51822s (22.3947GiB/s) | 20260506 09:09:53.176 | INFO | Analyzing chunk frames = [168960,169472) | 20260506 09:09:53.176 | INFO | MemInfo: { | 20260506 09:09:53.176 | INFO | "gpu" : " 17 GB", | 20260506 09:09:53.176 | INFO | "huge" : "", | 20260506 09:09:53.176 | INFO | "huge_shared" : " 27 GB", | 20260506 09:09:53.176 | INFO | "malloc" : " 14 GB", | 20260506 09:09:53.176 | INFO | "pinned" : " 29 GB" | 20260506 09:09:53.176 | INFO | } | 20260506 09:09:53.176 | INFO | Perf report for the last 5.13953 seconds: | 20260506 09:09:53.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:09:53.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:53.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:53.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:53.176 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260506 09:09:53.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:53.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:53.176 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:09:53.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:53.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:53.176 | INFO | Uploaded 36GiB to GPU in 1.60454s (22.4363GiB/s) | 20260506 09:09:53.176 | INFO | Downloaded 34GiB to GPU in 1.52208s (22.3378GiB/s) | 20260506 09:09:58.315 | INFO | Analyzing chunk frames = [169472,169984) | 20260506 09:09:58.315 | INFO | MemInfo: { | 20260506 09:09:58.315 | INFO | "gpu" : " 17 GB", | 20260506 09:09:58.315 | INFO | "huge" : "", | 20260506 09:09:58.315 | INFO | "huge_shared" : " 27 GB", | 20260506 09:09:58.315 | INFO | "malloc" : " 14 GB", | 20260506 09:09:58.315 | INFO | "pinned" : " 29 GB" | 20260506 09:09:58.315 | INFO | } | 20260506 09:09:58.315 | INFO | Perf report for the last 5.13912 seconds: | 20260506 09:09:58.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:09:58.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:58.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:58.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:58.315 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260506 09:09:58.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:58.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:58.315 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:09:58.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:58.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:09:58.315 | INFO | Uploaded 36GiB to GPU in 1.60387s (22.4457GiB/s) | 20260506 09:09:58.315 | INFO | Downloaded 34GiB to GPU in 1.52169s (22.3436GiB/s) | 20260506 09:10:03.454 | INFO | Analyzing chunk frames = [169984,170496) | 20260506 09:10:03.454 | INFO | MemInfo: { | 20260506 09:10:03.454 | INFO | "gpu" : " 17 GB", | 20260506 09:10:03.454 | INFO | "huge" : "", | 20260506 09:10:03.454 | INFO | "huge_shared" : " 27 GB", | 20260506 09:10:03.454 | INFO | "malloc" : " 14 GB", | 20260506 09:10:03.454 | INFO | "pinned" : " 29 GB" | 20260506 09:10:03.454 | INFO | } | 20260506 09:10:03.454 | INFO | Perf report for the last 5.13922 seconds: | 20260506 09:10:03.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:10:03.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:03.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:03.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:03.454 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 09:10:03.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:03.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:03.454 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:10:03.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:03.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:03.455 | INFO | Uploaded 36GiB to GPU in 1.6021s (22.4705GiB/s) | 20260506 09:10:03.455 | INFO | Downloaded 34GiB to GPU in 1.51891s (22.3845GiB/s) | 20260506 09:10:08.594 | INFO | Analyzing chunk frames = [170496,171008) | 20260506 09:10:08.594 | INFO | MemInfo: { | 20260506 09:10:08.594 | INFO | "gpu" : " 17 GB", | 20260506 09:10:08.594 | INFO | "huge" : "", | 20260506 09:10:08.594 | INFO | "huge_shared" : " 27 GB", | 20260506 09:10:08.594 | INFO | "malloc" : " 14 GB", | 20260506 09:10:08.594 | INFO | "pinned" : " 29 GB" | 20260506 09:10:08.594 | INFO | } | 20260506 09:10:08.594 | INFO | Perf report for the last 5.1395 seconds: | 20260506 09:10:08.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:10:08.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:08.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:08.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:08.594 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:10:08.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:08.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:08.594 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:10:08.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:08.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:08.594 | INFO | Uploaded 36GiB to GPU in 1.60243s (22.4659GiB/s) | 20260506 09:10:08.594 | INFO | Downloaded 34GiB to GPU in 1.52007s (22.3674GiB/s) | 20260506 09:10:13.733 | INFO | Analyzing chunk frames = [171008,171520) | 20260506 09:10:13.733 | INFO | MemInfo: { | 20260506 09:10:13.733 | INFO | "gpu" : " 17 GB", | 20260506 09:10:13.733 | INFO | "huge" : "", | 20260506 09:10:13.733 | INFO | "huge_shared" : " 27 GB", | 20260506 09:10:13.733 | INFO | "malloc" : " 14 GB", | 20260506 09:10:13.733 | INFO | "pinned" : " 29 GB" | 20260506 09:10:13.733 | INFO | } | 20260506 09:10:13.733 | INFO | Perf report for the last 5.13926 seconds: | 20260506 09:10:13.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:10:13.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:13.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:13.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:13.733 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:10:13.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:13.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:13.733 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:10:13.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:13.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:13.733 | INFO | Uploaded 36GiB to GPU in 1.60296s (22.4585GiB/s) | 20260506 09:10:13.733 | INFO | Downloaded 34GiB to GPU in 1.52049s (22.3612GiB/s) | 20260506 09:10:18.872 | INFO | Analyzing chunk frames = [171520,172032) | 20260506 09:10:18.872 | INFO | MemInfo: { | 20260506 09:10:18.872 | INFO | "gpu" : " 17 GB", | 20260506 09:10:18.872 | INFO | "huge" : "", | 20260506 09:10:18.872 | INFO | "huge_shared" : " 27 GB", | 20260506 09:10:18.872 | INFO | "malloc" : " 14 GB", | 20260506 09:10:18.872 | INFO | "pinned" : " 29 GB" | 20260506 09:10:18.872 | INFO | } | 20260506 09:10:18.872 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:10:18.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:10:18.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:18.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:18.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:18.872 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:10:18.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:18.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:18.873 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:10:18.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:18.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:18.873 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:10:18.873 | INFO | Total mem usage: 15.5 GiB | 20260506 09:10:18.873 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.23%) | 20260506 09:10:18.873 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -4.07%) | 20260506 09:10:18.873 | INFO | Uploaded 36GiB to GPU in 1.60382s (22.4464GiB/s) | 20260506 09:10:18.873 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:10:18.873 | INFO | High Water Breakdown: | 20260506 09:10:18.873 | INFO | Downloaded 34GiB to GPU in 1.52129s (22.3494GiB/s) | 20260506 09:10:18.873 | INFO | 58.7% : 8.76 GiB ( 95.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:10:18.873 | INFO | 41.1% : 6.13 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:10:18.873 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:10:18.873 | INFO | ------------------------------------------ | 20260506 09:10:18.873 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:10:18.873 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:10:18.873 | INFO | Total mem usage: 26.5 GiB | 20260506 09:10:18.873 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:10:18.873 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:10:18.873 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:10:18.873 | INFO | High Water Breakdown: | 20260506 09:10:18.873 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:10:18.873 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:10:18.873 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:10:18.873 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:10:18.873 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:10:18.873 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:10:18.873 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:10:18.873 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:10:18.873 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:10:18.873 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:10:18.873 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:10:18.873 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:10:18.873 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:10:18.873 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:10:18.873 | INFO | ------------------------------------------ | 20260506 09:10:18.873 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:10:18.873 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:10:18.873 | INFO | Total mem usage: 25.6 GiB | 20260506 09:10:18.873 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:10:18.873 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:10:18.873 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:10:18.873 | INFO | High Water Breakdown: | 20260506 09:10:18.873 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:10:18.873 | INFO | ------------------------------------------ | 20260506 09:10:18.873 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:10:18.873 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:10:18.873 | INFO | Total mem usage: 15.7 GiB | 20260506 09:10:18.873 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:10:18.873 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:10:18.873 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:10:18.873 | INFO | High Water Breakdown: | 20260506 09:10:18.873 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:10:18.873 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:10:18.873 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:10:18.873 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:10:18.873 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:10:18.873 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:10:18.873 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:10:18.873 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:10:18.873 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:10:18.873 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:10:18.873 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:10:18.873 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:10:18.873 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:10:18.873 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:10:18.873 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:10:18.873 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:10:18.873 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:10:18.873 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:10:18.873 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:10:18.873 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:10:18.873 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:10:18.873 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:10:18.873 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:10:18.873 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:10:18.874 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:10:18.874 | INFO | ------------------------------------------ | 20260506 09:10:18.874 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:10:18.874 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:10:18.874 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:10:18.874 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:10:18.874 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 09:10:18.874 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / 0.00e+00, 8.31e-03 / 0.00e+00 | 20260506 09:10:18.874 | INFO | PerfReport: 10.86%, Baseline -- 3.29e+00 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260506 09:10:18.874 | INFO | PerfReport: 3.04%, Histogram -- 9.22e-01 / 0.00e+00, 5.22e-02 / 0.00e+00 | 20260506 09:10:18.874 | INFO | PerfReport: 16.03%, DME -- 3.89e+01 / 0.00e+00, 3.18e+00 / 0.00e+00 | 20260506 09:10:18.874 | INFO | PerfReport: 56.74%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.16e-02 / 0.00e+00 | 20260506 09:10:18.874 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.30e+00 / 0.00e+00, 1.39e-01 / 0.00e+00 | 20260506 09:10:18.874 | INFO | PerfReport: 3.92%, Metrics -- 1.19e+00 / 0.00e+00, 1.85e-01 / 0.00e+00 | 20260506 09:10:18.874 | INFO | PerfReport: 1.53%, PulseSerializing -- 4.65e-01 / 0.00e+00, 6.74e-02 / 0.00e+00 | 20260506 09:10:18.874 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:10:18.874 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:10:18.874 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:10:18.874 | INFO | PerfReport: Full Execution -- 9.08e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 09:10:18.874 | INFO | PerfReport: 18.45%, Upload -- 1.68e+01 / 0.00e+00, 4.21e-01 / 0.00e+00 | 20260506 09:10:18.874 | INFO | PerfReport: 17.71%, Download -- 1.61e+01 / 0.00e+00, 7.49e-01 / 0.00e+00 | 20260506 09:10:18.874 | INFO | PerfReport: 33.37%, Compute -- 3.03e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 09:10:18.874 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:10:18.874 | INFO | PerfReport: 0.00%, LockDownload -- 4.92e-04 / 0.00e+00, 1.75e-04 / 0.00e+00 | 20260506 09:10:18.874 | INFO | PerfReport: 29.89%, LockCompute -- 2.72e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 09:10:18.874 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.39e-02 / 0.00e+00 | 20260506 09:10:18.970 | INFO | Starting a round of Baz IO | 20260506 09:10:21.906 | INFO | Populating RTMetrics report | 20260506 09:10:21.907 | INFO | RTMetrics report populated | 20260506 09:10:21.907 | INFO | Opening tmp RTMetrics json file | 20260506 09:10:21.910 | INFO | Serializing RTMetrics to json | 20260506 09:10:21.913 | INFO | Serialization done, swapping tmp file now... | 20260506 09:10:21.918 | INFO | RTMetrics done | 20260506 09:10:21.999 | INFO | Finished a round of Baz IO: Wrote 1.753 GiB in 3.029 seconds | 20260506 09:10:22.000 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.61% / 0.17% / 42.22% | 20260506 09:10:22.000 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.6074% / 0.000659625% / 1.58946e-05% | 20260506 09:10:22.000 | INFO | Baz IO overall: | 20260506 09:10:22.000 | INFO | BytesWritten : 55525431123 | 20260506 09:10:22.000 | INFO | NumEvents : 25547250862 | 20260506 09:10:24.012 | INFO | Analyzing chunk frames = [172032,172544) | 20260506 09:10:24.012 | INFO | MemInfo: { | 20260506 09:10:24.012 | INFO | "gpu" : " 17 GB", | 20260506 09:10:24.012 | INFO | "huge" : "", | 20260506 09:10:24.012 | INFO | "huge_shared" : " 27 GB", | 20260506 09:10:24.012 | INFO | "malloc" : " 14 GB", | 20260506 09:10:24.012 | INFO | "pinned" : " 29 GB" | 20260506 09:10:24.012 | INFO | } | 20260506 09:10:24.012 | INFO | Perf report for the last 5.13932 seconds: | 20260506 09:10:24.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:10:24.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:24.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:24.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:24.012 | INFO | ANALYSIS : 58.1%, 2.9, 88.6ms, 0 | 20260506 09:10:24.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:24.012 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 09:10:24.012 | INFO | PRE_HQ : 57.2%, 2.1, 61.9ms, 0 | 20260506 09:10:24.012 | INFO | BAZWRITER : 58.6%, 4.4, 133.5ms, 0 | 20260506 09:10:24.012 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 09:10:24.012 | INFO | Uploaded 36GiB to GPU in 1.64722s (21.855GiB/s) | 20260506 09:10:24.012 | INFO | Downloaded 38GiB to GPU in 1.70842s (22.2427GiB/s) | 20260506 09:10:29.151 | INFO | Analyzing chunk frames = [172544,173056) | 20260506 09:10:29.151 | INFO | MemInfo: { | 20260506 09:10:29.151 | INFO | "gpu" : " 17 GB", | 20260506 09:10:29.151 | INFO | "huge" : "", | 20260506 09:10:29.151 | INFO | "huge_shared" : " 27 GB", | 20260506 09:10:29.151 | INFO | "malloc" : " 14 GB", | 20260506 09:10:29.151 | INFO | "pinned" : " 29 GB" | 20260506 09:10:29.151 | INFO | } | 20260506 09:10:29.151 | INFO | Perf report for the last 5.13946 seconds: | 20260506 09:10:29.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:10:29.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:29.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:29.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:29.151 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260506 09:10:29.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:29.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:29.151 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:10:29.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:29.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:29.152 | INFO | Uploaded 36GiB to GPU in 1.60123s (22.4827GiB/s) | 20260506 09:10:29.152 | INFO | Downloaded 34GiB to GPU in 1.51792s (22.3991GiB/s) | 20260506 09:10:34.290 | INFO | Analyzing chunk frames = [173056,173568) | 20260506 09:10:34.290 | INFO | MemInfo: { | 20260506 09:10:34.290 | INFO | "gpu" : " 17 GB", | 20260506 09:10:34.290 | INFO | "huge" : "", | 20260506 09:10:34.290 | INFO | "huge_shared" : " 27 GB", | 20260506 09:10:34.290 | INFO | "malloc" : " 14 GB", | 20260506 09:10:34.290 | INFO | "pinned" : " 29 GB" | 20260506 09:10:34.290 | INFO | } | 20260506 09:10:34.290 | INFO | Perf report for the last 5.13919 seconds: | 20260506 09:10:34.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:10:34.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:34.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:34.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:34.291 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260506 09:10:34.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:34.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:34.291 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:10:34.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:34.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:34.291 | INFO | Uploaded 36GiB to GPU in 1.60448s (22.4371GiB/s) | 20260506 09:10:34.291 | INFO | Downloaded 34GiB to GPU in 1.52155s (22.3456GiB/s) | 20260506 09:10:39.430 | INFO | Analyzing chunk frames = [173568,174080) | 20260506 09:10:39.430 | INFO | MemInfo: { | 20260506 09:10:39.430 | INFO | "gpu" : " 17 GB", | 20260506 09:10:39.430 | INFO | "huge" : "", | 20260506 09:10:39.430 | INFO | "huge_shared" : " 27 GB", | 20260506 09:10:39.430 | INFO | "malloc" : " 14 GB", | 20260506 09:10:39.430 | INFO | "pinned" : " 29 GB" | 20260506 09:10:39.430 | INFO | } | 20260506 09:10:39.430 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:10:39.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:10:39.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:39.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:39.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:39.430 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260506 09:10:39.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:39.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:39.430 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:10:39.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:39.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:39.430 | INFO | Uploaded 36GiB to GPU in 1.60466s (22.4347GiB/s) | 20260506 09:10:39.430 | INFO | Downloaded 34GiB to GPU in 1.52224s (22.3355GiB/s) | 20260506 09:10:44.569 | INFO | Analyzing chunk frames = [174080,174592) | 20260506 09:10:44.569 | INFO | MemInfo: { | 20260506 09:10:44.569 | INFO | "gpu" : " 17 GB", | 20260506 09:10:44.569 | INFO | "huge" : "", | 20260506 09:10:44.569 | INFO | "huge_shared" : " 27 GB", | 20260506 09:10:44.569 | INFO | "malloc" : " 14 GB", | 20260506 09:10:44.569 | INFO | "pinned" : " 29 GB" | 20260506 09:10:44.569 | INFO | } | 20260506 09:10:44.569 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:10:44.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:10:44.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:44.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:44.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:44.569 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 09:10:44.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:44.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:44.569 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:10:44.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:44.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:44.569 | INFO | Uploaded 36GiB to GPU in 1.60223s (22.4687GiB/s) | 20260506 09:10:44.569 | INFO | Downloaded 34GiB to GPU in 1.51898s (22.3834GiB/s) | 20260506 09:10:49.709 | INFO | Analyzing chunk frames = [174592,175104) | 20260506 09:10:49.709 | INFO | MemInfo: { | 20260506 09:10:49.709 | INFO | "gpu" : " 17 GB", | 20260506 09:10:49.709 | INFO | "huge" : "", | 20260506 09:10:49.709 | INFO | "huge_shared" : " 27 GB", | 20260506 09:10:49.709 | INFO | "malloc" : " 14 GB", | 20260506 09:10:49.709 | INFO | "pinned" : " 29 GB" | 20260506 09:10:49.709 | INFO | } | 20260506 09:10:49.709 | INFO | Perf report for the last 5.13954 seconds: | 20260506 09:10:49.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:10:49.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:49.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:49.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:49.709 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260506 09:10:49.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:49.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:49.709 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:10:49.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:49.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:49.709 | INFO | Uploaded 36GiB to GPU in 1.60282s (22.4604GiB/s) | 20260506 09:10:49.709 | INFO | Downloaded 34GiB to GPU in 1.51985s (22.3706GiB/s) | 20260506 09:10:54.848 | INFO | Analyzing chunk frames = [175104,175616) | 20260506 09:10:54.848 | INFO | MemInfo: { | 20260506 09:10:54.848 | INFO | "gpu" : " 17 GB", | 20260506 09:10:54.848 | INFO | "huge" : "", | 20260506 09:10:54.848 | INFO | "huge_shared" : " 27 GB", | 20260506 09:10:54.848 | INFO | "malloc" : " 14 GB", | 20260506 09:10:54.848 | INFO | "pinned" : " 29 GB" | 20260506 09:10:54.848 | INFO | } | 20260506 09:10:54.848 | INFO | Perf report for the last 5.13902 seconds: | 20260506 09:10:54.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:10:54.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:54.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:54.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:54.848 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 09:10:54.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:54.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:54.848 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:10:54.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:54.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:54.848 | INFO | Uploaded 36GiB to GPU in 1.60284s (22.4601GiB/s) | 20260506 09:10:54.848 | INFO | Downloaded 34GiB to GPU in 1.52012s (22.3666GiB/s) | 20260506 09:10:59.987 | INFO | Analyzing chunk frames = [175616,176128) | 20260506 09:10:59.987 | INFO | MemInfo: { | 20260506 09:10:59.987 | INFO | "gpu" : " 17 GB", | 20260506 09:10:59.987 | INFO | "huge" : "", | 20260506 09:10:59.987 | INFO | "huge_shared" : " 27 GB", | 20260506 09:10:59.987 | INFO | "malloc" : " 14 GB", | 20260506 09:10:59.987 | INFO | "pinned" : " 29 GB" | 20260506 09:10:59.987 | INFO | } | 20260506 09:10:59.987 | INFO | Perf report for the last 5.1395 seconds: | 20260506 09:10:59.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:10:59.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:59.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:59.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:59.987 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:10:59.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:59.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:59.987 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:10:59.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:59.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:10:59.988 | INFO | Uploaded 36GiB to GPU in 1.60326s (22.4542GiB/s) | 20260506 09:10:59.988 | INFO | Downloaded 34GiB to GPU in 1.52119s (22.351GiB/s) | 20260506 09:11:00.084 | INFO | Starting a round of Baz IO | 20260506 09:11:03.020 | INFO | Populating RTMetrics report | 20260506 09:11:03.020 | INFO | RTMetrics report populated | 20260506 09:11:03.020 | INFO | Opening tmp RTMetrics json file | 20260506 09:11:03.032 | INFO | Serializing RTMetrics to json | 20260506 09:11:03.037 | INFO | Serialization done, swapping tmp file now... | 20260506 09:11:03.042 | INFO | RTMetrics done | 20260506 09:11:03.225 | INFO | Finished a round of Baz IO: Wrote 1.761 GiB in 3.14 seconds | 20260506 09:11:03.225 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.44% / 0.17% / 42.39% | 20260506 09:11:03.225 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.6127% / 0.000619888% / 3.17891e-05% | 20260506 09:11:03.225 | INFO | Baz IO overall: | 20260506 09:11:03.225 | INFO | BytesWritten : 57416219158 | 20260506 09:11:03.225 | INFO | NumEvents : 26466870419 | 20260506 09:11:05.127 | INFO | Analyzing chunk frames = [176128,176640) | 20260506 09:11:05.127 | INFO | MemInfo: { | 20260506 09:11:05.127 | INFO | "gpu" : " 17 GB", | 20260506 09:11:05.127 | INFO | "huge" : "", | 20260506 09:11:05.127 | INFO | "huge_shared" : " 27 GB", | 20260506 09:11:05.127 | INFO | "malloc" : " 14 GB", | 20260506 09:11:05.127 | INFO | "pinned" : " 29 GB" | 20260506 09:11:05.127 | INFO | } | 20260506 09:11:05.127 | INFO | Perf report for the last 5.13946 seconds: | 20260506 09:11:05.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:11:05.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:05.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:05.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:05.127 | INFO | ANALYSIS : 58.1%, 2.9, 88.8ms, 0 | 20260506 09:11:05.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:05.127 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 09:11:05.127 | INFO | PRE_HQ : 57.1%, 2.1, 62.0ms, 0 | 20260506 09:11:05.127 | INFO | BAZWRITER : 60.7%, 4.4, 137.4ms, 0 | 20260506 09:11:05.127 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 09:11:05.127 | INFO | Uploaded 36GiB to GPU in 1.64448s (21.8915GiB/s) | 20260506 09:11:05.127 | INFO | Downloaded 38GiB to GPU in 1.70744s (22.2555GiB/s) | 20260506 09:11:10.266 | INFO | Analyzing chunk frames = [176640,177152) | 20260506 09:11:10.266 | INFO | MemInfo: { | 20260506 09:11:10.266 | INFO | "gpu" : " 17 GB", | 20260506 09:11:10.266 | INFO | "huge" : "", | 20260506 09:11:10.266 | INFO | "huge_shared" : " 27 GB", | 20260506 09:11:10.266 | INFO | "malloc" : " 14 GB", | 20260506 09:11:10.266 | INFO | "pinned" : " 29 GB" | 20260506 09:11:10.266 | INFO | } | 20260506 09:11:10.266 | INFO | Perf report for the last 5.139 seconds: | 20260506 09:11:10.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:11:10.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:10.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:10.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:10.266 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260506 09:11:10.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:10.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:10.266 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:11:10.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:10.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:10.266 | INFO | Uploaded 36GiB to GPU in 1.60108s (22.4848GiB/s) | 20260506 09:11:10.266 | INFO | Downloaded 34GiB to GPU in 1.51698s (22.4129GiB/s) | 20260506 09:11:15.405 | INFO | Analyzing chunk frames = [177152,177664) | 20260506 09:11:15.405 | INFO | MemInfo: { | 20260506 09:11:15.405 | INFO | "gpu" : " 17 GB", | 20260506 09:11:15.405 | INFO | "huge" : "", | 20260506 09:11:15.405 | INFO | "huge_shared" : " 27 GB", | 20260506 09:11:15.405 | INFO | "malloc" : " 14 GB", | 20260506 09:11:15.405 | INFO | "pinned" : " 29 GB" | 20260506 09:11:15.405 | INFO | } | 20260506 09:11:15.405 | INFO | Perf report for the last 5.13943 seconds: | 20260506 09:11:15.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:11:15.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:15.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:15.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:15.405 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260506 09:11:15.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:15.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:15.405 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:11:15.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:15.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:15.406 | INFO | Uploaded 36GiB to GPU in 1.60479s (22.4328GiB/s) | 20260506 09:11:15.406 | INFO | Downloaded 34GiB to GPU in 1.52195s (22.3397GiB/s) | 20260506 09:11:20.545 | INFO | Analyzing chunk frames = [177664,178176) | 20260506 09:11:20.545 | INFO | MemInfo: { | 20260506 09:11:20.545 | INFO | "gpu" : " 17 GB", | 20260506 09:11:20.545 | INFO | "huge" : "", | 20260506 09:11:20.545 | INFO | "huge_shared" : " 27 GB", | 20260506 09:11:20.545 | INFO | "malloc" : " 14 GB", | 20260506 09:11:20.545 | INFO | "pinned" : " 29 GB" | 20260506 09:11:20.545 | INFO | } | 20260506 09:11:20.545 | INFO | Perf report for the last 5.13936 seconds: | 20260506 09:11:20.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:11:20.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:20.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:20.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:20.545 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 09:11:20.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:20.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:20.545 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:11:20.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:20.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:20.545 | INFO | Uploaded 36GiB to GPU in 1.60387s (22.4458GiB/s) | 20260506 09:11:20.545 | INFO | Downloaded 34GiB to GPU in 1.52142s (22.3476GiB/s) | 20260506 09:11:25.684 | INFO | Analyzing chunk frames = [178176,178688) | 20260506 09:11:25.684 | INFO | MemInfo: { | 20260506 09:11:25.684 | INFO | "gpu" : " 17 GB", | 20260506 09:11:25.684 | INFO | "huge" : "", | 20260506 09:11:25.684 | INFO | "huge_shared" : " 27 GB", | 20260506 09:11:25.684 | INFO | "malloc" : " 14 GB", | 20260506 09:11:25.684 | INFO | "pinned" : " 29 GB" | 20260506 09:11:25.684 | INFO | } | 20260506 09:11:25.684 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:11:25.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:11:25.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:25.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:25.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:25.684 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 09:11:25.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:25.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:25.684 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:11:25.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:25.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:25.684 | INFO | Uploaded 36GiB to GPU in 1.60239s (22.4664GiB/s) | 20260506 09:11:25.684 | INFO | Downloaded 34GiB to GPU in 1.5197s (22.3728GiB/s) | 20260506 09:11:30.823 | INFO | Analyzing chunk frames = [178688,179200) | 20260506 09:11:30.823 | INFO | MemInfo: { | 20260506 09:11:30.823 | INFO | "gpu" : " 17 GB", | 20260506 09:11:30.823 | INFO | "huge" : "", | 20260506 09:11:30.823 | INFO | "huge_shared" : " 27 GB", | 20260506 09:11:30.823 | INFO | "malloc" : " 14 GB", | 20260506 09:11:30.823 | INFO | "pinned" : " 29 GB" | 20260506 09:11:30.823 | INFO | } | 20260506 09:11:30.823 | INFO | Perf report for the last 5.13932 seconds: | 20260506 09:11:30.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:11:30.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:30.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:30.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:30.823 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:11:30.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:30.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:30.823 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:11:30.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:30.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:30.824 | INFO | Uploaded 36GiB to GPU in 1.60241s (22.4662GiB/s) | 20260506 09:11:30.824 | INFO | Downloaded 34GiB to GPU in 1.51872s (22.3872GiB/s) | 20260506 09:11:35.963 | INFO | Analyzing chunk frames = [179200,179712) | 20260506 09:11:35.963 | INFO | MemInfo: { | 20260506 09:11:35.963 | INFO | "gpu" : " 17 GB", | 20260506 09:11:35.963 | INFO | "huge" : "", | 20260506 09:11:35.963 | INFO | "huge_shared" : " 27 GB", | 20260506 09:11:35.963 | INFO | "malloc" : " 14 GB", | 20260506 09:11:35.963 | INFO | "pinned" : " 29 GB" | 20260506 09:11:35.963 | INFO | } | 20260506 09:11:35.963 | INFO | Perf report for the last 5.13937 seconds: | 20260506 09:11:35.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:11:35.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:35.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:35.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:35.963 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 09:11:35.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:35.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:35.963 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:11:35.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:35.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:35.963 | INFO | Uploaded 36GiB to GPU in 1.60359s (22.4497GiB/s) | 20260506 09:11:35.963 | INFO | Downloaded 34GiB to GPU in 1.52134s (22.3488GiB/s) | 20260506 09:11:41.102 | INFO | Analyzing chunk frames = [179712,180224) | 20260506 09:11:41.102 | INFO | MemInfo: { | 20260506 09:11:41.102 | INFO | "gpu" : " 17 GB", | 20260506 09:11:41.102 | INFO | "huge" : "", | 20260506 09:11:41.102 | INFO | "huge_shared" : " 27 GB", | 20260506 09:11:41.102 | INFO | "malloc" : " 14 GB", | 20260506 09:11:41.102 | INFO | "pinned" : " 29 GB" | 20260506 09:11:41.102 | INFO | } | 20260506 09:11:41.102 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:11:41.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:11:41.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:41.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:41.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:41.102 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 09:11:41.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:41.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:41.102 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 09:11:41.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:41.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:41.102 | INFO | Uploaded 36GiB to GPU in 1.60346s (22.4514GiB/s) | 20260506 09:11:41.102 | INFO | Downloaded 34GiB to GPU in 1.52076s (22.3572GiB/s) | 20260506 09:11:41.102 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:11:41.102 | INFO | Total mem usage: 15.5 GiB | 20260506 09:11:41.102 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.23%) | 20260506 09:11:41.102 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -3.45%) | 20260506 09:11:41.102 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:11:41.102 | INFO | High Water Breakdown: | 20260506 09:11:41.102 | INFO | 58.9% : 8.84 GiB ( 96.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:11:41.102 | INFO | 41% : 6.15 GiB ( 91.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:11:41.102 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:11:41.102 | INFO | ------------------------------------------ | 20260506 09:11:41.102 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:11:41.102 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:11:41.102 | INFO | Total mem usage: 26.5 GiB | 20260506 09:11:41.102 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:11:41.102 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:11:41.102 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:11:41.102 | INFO | High Water Breakdown: | 20260506 09:11:41.102 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:11:41.102 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:11:41.102 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:11:41.102 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:11:41.102 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:11:41.102 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:11:41.102 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:11:41.102 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:11:41.102 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:11:41.102 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:11:41.102 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:11:41.102 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:11:41.102 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:11:41.102 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:11:41.102 | INFO | ------------------------------------------ | 20260506 09:11:41.102 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:11:41.102 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:11:41.102 | INFO | Total mem usage: 25.6 GiB | 20260506 09:11:41.102 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:11:41.102 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:11:41.102 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:11:41.102 | INFO | High Water Breakdown: | 20260506 09:11:41.102 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:11:41.102 | INFO | ------------------------------------------ | 20260506 09:11:41.102 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:11:41.102 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:11:41.102 | INFO | Total mem usage: 15.7 GiB | 20260506 09:11:41.102 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:11:41.102 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:11:41.102 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:11:41.102 | INFO | High Water Breakdown: | 20260506 09:11:41.102 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:11:41.102 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:11:41.102 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:11:41.102 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:11:41.102 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:11:41.102 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:11:41.102 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:11:41.102 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:11:41.103 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:11:41.103 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:11:41.103 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:11:41.103 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:11:41.103 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:11:41.103 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:11:41.103 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:11:41.103 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:11:41.103 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:11:41.103 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:11:41.103 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:11:41.103 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:11:41.103 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:11:41.103 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:11:41.103 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:11:41.103 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:11:41.103 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:11:41.103 | INFO | ------------------------------------------ | 20260506 09:11:41.103 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:11:41.103 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:11:41.103 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:11:41.103 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:11:41.103 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 09:11:41.103 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / 0.00e+00, 4.33e-03 / 0.00e+00 | 20260506 09:11:41.103 | INFO | PerfReport: 10.87%, Baseline -- 3.29e+00 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260506 09:11:41.103 | INFO | PerfReport: 3.04%, Histogram -- 9.21e-01 / 0.00e+00, 5.21e-02 / 0.00e+00 | 20260506 09:11:41.103 | INFO | PerfReport: 16.03%, DME -- 3.89e+01 / 0.00e+00, 3.10e+00 / 0.00e+00 | 20260506 09:11:41.103 | INFO | PerfReport: 56.74%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.18e-02 / 0.00e+00 | 20260506 09:11:41.103 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.30e+00 / 0.00e+00, 1.39e-01 / 0.00e+00 | 20260506 09:11:41.103 | INFO | PerfReport: 3.92%, Metrics -- 1.19e+00 / 0.00e+00, 1.84e-01 / 0.00e+00 | 20260506 09:11:41.103 | INFO | PerfReport: 1.53%, PulseSerializing -- 4.65e-01 / 0.00e+00, 6.76e-02 / 0.00e+00 | 20260506 09:11:41.103 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:11:41.103 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:11:41.103 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:11:41.103 | INFO | PerfReport: Full Execution -- 9.08e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260506 09:11:41.103 | INFO | PerfReport: 18.44%, Upload -- 1.68e+01 / 0.00e+00, 4.09e-01 / 0.00e+00 | 20260506 09:11:41.103 | INFO | PerfReport: 17.70%, Download -- 1.61e+01 / 0.00e+00, 7.32e-01 / 0.00e+00 | 20260506 09:11:41.103 | INFO | PerfReport: 33.37%, Compute -- 3.03e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 09:11:41.103 | INFO | PerfReport: 0.56%, LockUpload -- 5.08e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260506 09:11:41.103 | INFO | PerfReport: 0.00%, LockDownload -- 4.81e-04 / 0.00e+00, 1.80e-04 / 0.00e+00 | 20260506 09:11:41.103 | INFO | PerfReport: 29.90%, LockCompute -- 2.72e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 09:11:41.103 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.46e-02 / 0.00e+00 | 20260506 09:11:41.200 | INFO | Starting a round of Baz IO | 20260506 09:11:44.128 | INFO | Populating RTMetrics report | 20260506 09:11:44.128 | INFO | RTMetrics report populated | 20260506 09:11:44.128 | INFO | Opening tmp RTMetrics json file | 20260506 09:11:44.253 | INFO | Serializing RTMetrics to json | 20260506 09:11:44.351 | INFO | Serialization done, swapping tmp file now... | 20260506 09:11:44.386 | INFO | RTMetrics done | 20260506 09:11:44.410 | INFO | Finished a round of Baz IO: Wrote 1.764 GiB in 3.21 seconds | 20260506 09:11:44.410 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.27% / 0.17% / 42.56% | 20260506 09:11:44.410 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.5503% / 0.000544389% / 2.78155e-05% | 20260506 09:11:44.410 | INFO | Baz IO overall: | 20260506 09:11:44.410 | INFO | BytesWritten : 59310075020 | 20260506 09:11:44.410 | INFO | NumEvents : 27387731786 | 20260506 09:11:46.241 | INFO | Analyzing chunk frames = [180224,180736) | 20260506 09:11:46.241 | INFO | MemInfo: { | 20260506 09:11:46.241 | INFO | "gpu" : " 17 GB", | 20260506 09:11:46.241 | INFO | "huge" : "", | 20260506 09:11:46.241 | INFO | "huge_shared" : " 27 GB", | 20260506 09:11:46.241 | INFO | "malloc" : " 14 GB", | 20260506 09:11:46.241 | INFO | "pinned" : " 29 GB" | 20260506 09:11:46.241 | INFO | } | 20260506 09:11:46.241 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:11:46.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:11:46.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:46.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:46.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:46.241 | INFO | ANALYSIS : 57.9%, 2.9, 88.3ms, 0 | 20260506 09:11:46.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:46.241 | INFO | RT_METRICS : 48.8%, 1.0, 25.6ms, 0 | 20260506 09:11:46.241 | INFO | PRE_HQ : 57.0%, 2.1, 62.1ms, 0 | 20260506 09:11:46.241 | INFO | BAZWRITER : 62.0%, 4.2, 135.2ms, 0 | 20260506 09:11:46.241 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 09:11:46.242 | INFO | Uploaded 36GiB to GPU in 1.64899s (21.8315GiB/s) | 20260506 09:11:46.242 | INFO | Downloaded 38GiB to GPU in 1.71215s (22.1943GiB/s) | 20260506 09:11:51.381 | INFO | Analyzing chunk frames = [180736,181248) | 20260506 09:11:51.381 | INFO | MemInfo: { | 20260506 09:11:51.381 | INFO | "gpu" : " 17 GB", | 20260506 09:11:51.381 | INFO | "huge" : "", | 20260506 09:11:51.381 | INFO | "huge_shared" : " 27 GB", | 20260506 09:11:51.381 | INFO | "malloc" : " 14 GB", | 20260506 09:11:51.381 | INFO | "pinned" : " 29 GB" | 20260506 09:11:51.381 | INFO | } | 20260506 09:11:51.381 | INFO | Perf report for the last 5.13936 seconds: | 20260506 09:11:51.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:11:51.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:51.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:51.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:51.381 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260506 09:11:51.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:51.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:51.381 | INFO | PRE_HQ : 30.2%, 1.0, 15.9ms, 0 | 20260506 09:11:51.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:51.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:51.381 | INFO | Uploaded 36GiB to GPU in 1.60057s (22.492GiB/s) | 20260506 09:11:51.381 | INFO | Downloaded 34GiB to GPU in 1.51732s (22.4079GiB/s) | 20260506 09:11:56.520 | INFO | Analyzing chunk frames = [181248,181760) | 20260506 09:11:56.520 | INFO | MemInfo: { | 20260506 09:11:56.520 | INFO | "gpu" : " 17 GB", | 20260506 09:11:56.520 | INFO | "huge" : "", | 20260506 09:11:56.520 | INFO | "huge_shared" : " 27 GB", | 20260506 09:11:56.520 | INFO | "malloc" : " 14 GB", | 20260506 09:11:56.520 | INFO | "pinned" : " 29 GB" | 20260506 09:11:56.520 | INFO | } | 20260506 09:11:56.520 | INFO | Perf report for the last 5.13936 seconds: | 20260506 09:11:56.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:11:56.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:56.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:56.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:56.520 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260506 09:11:56.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:56.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:56.520 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:11:56.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:56.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:11:56.520 | INFO | Uploaded 36GiB to GPU in 1.60539s (22.4245GiB/s) | 20260506 09:11:56.520 | INFO | Downloaded 34GiB to GPU in 1.52267s (22.3292GiB/s) | 20260506 09:12:01.659 | INFO | Analyzing chunk frames = [181760,182272) | 20260506 09:12:01.659 | INFO | MemInfo: { | 20260506 09:12:01.659 | INFO | "gpu" : " 17 GB", | 20260506 09:12:01.659 | INFO | "huge" : "", | 20260506 09:12:01.659 | INFO | "huge_shared" : " 27 GB", | 20260506 09:12:01.659 | INFO | "malloc" : " 14 GB", | 20260506 09:12:01.659 | INFO | "pinned" : " 29 GB" | 20260506 09:12:01.659 | INFO | } | 20260506 09:12:01.660 | INFO | Perf report for the last 5.13942 seconds: | 20260506 09:12:01.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:12:01.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:01.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:01.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:01.660 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260506 09:12:01.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:01.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:01.660 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:12:01.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:01.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:01.660 | INFO | Uploaded 36GiB to GPU in 1.6047s (22.4342GiB/s) | 20260506 09:12:01.660 | INFO | Downloaded 34GiB to GPU in 1.52154s (22.3458GiB/s) | 20260506 09:12:06.799 | INFO | Analyzing chunk frames = [182272,182784) | 20260506 09:12:06.799 | INFO | MemInfo: { | 20260506 09:12:06.799 | INFO | "gpu" : " 17 GB", | 20260506 09:12:06.799 | INFO | "huge" : "", | 20260506 09:12:06.799 | INFO | "huge_shared" : " 27 GB", | 20260506 09:12:06.799 | INFO | "malloc" : " 14 GB", | 20260506 09:12:06.799 | INFO | "pinned" : " 29 GB" | 20260506 09:12:06.799 | INFO | } | 20260506 09:12:06.799 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:12:06.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:12:06.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:06.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:06.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:06.799 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 09:12:06.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:06.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:06.799 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:12:06.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:06.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:06.799 | INFO | Uploaded 36GiB to GPU in 1.60284s (22.4601GiB/s) | 20260506 09:12:06.799 | INFO | Downloaded 34GiB to GPU in 1.51993s (22.3695GiB/s) | 20260506 09:12:11.938 | INFO | Analyzing chunk frames = [182784,183296) | 20260506 09:12:11.938 | INFO | MemInfo: { | 20260506 09:12:11.938 | INFO | "gpu" : " 17 GB", | 20260506 09:12:11.938 | INFO | "huge" : "", | 20260506 09:12:11.938 | INFO | "huge_shared" : " 27 GB", | 20260506 09:12:11.938 | INFO | "malloc" : " 14 GB", | 20260506 09:12:11.938 | INFO | "pinned" : " 29 GB" | 20260506 09:12:11.938 | INFO | } | 20260506 09:12:11.938 | INFO | Perf report for the last 5.13926 seconds: | 20260506 09:12:11.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:12:11.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:11.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:11.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:11.938 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260506 09:12:11.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:11.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:11.938 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:12:11.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:11.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:11.939 | INFO | Uploaded 36GiB to GPU in 1.60269s (22.4623GiB/s) | 20260506 09:12:11.939 | INFO | Downloaded 34GiB to GPU in 1.51971s (22.3726GiB/s) | 20260506 09:12:17.078 | INFO | Analyzing chunk frames = [183296,183808) | 20260506 09:12:17.078 | INFO | MemInfo: { | 20260506 09:12:17.078 | INFO | "gpu" : " 17 GB", | 20260506 09:12:17.078 | INFO | "huge" : "", | 20260506 09:12:17.078 | INFO | "huge_shared" : " 27 GB", | 20260506 09:12:17.078 | INFO | "malloc" : " 14 GB", | 20260506 09:12:17.078 | INFO | "pinned" : " 29 GB" | 20260506 09:12:17.078 | INFO | } | 20260506 09:12:17.078 | INFO | Perf report for the last 5.13959 seconds: | 20260506 09:12:17.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:12:17.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:17.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:17.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:17.078 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 09:12:17.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:17.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:17.078 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:12:17.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:17.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:17.078 | INFO | Uploaded 36GiB to GPU in 1.60265s (22.4628GiB/s) | 20260506 09:12:17.078 | INFO | Downloaded 34GiB to GPU in 1.52176s (22.3425GiB/s) | 20260506 09:12:22.217 | INFO | Analyzing chunk frames = [183808,184320) | 20260506 09:12:22.217 | INFO | MemInfo: { | 20260506 09:12:22.217 | INFO | "gpu" : " 17 GB", | 20260506 09:12:22.217 | INFO | "huge" : "", | 20260506 09:12:22.217 | INFO | "huge_shared" : " 27 GB", | 20260506 09:12:22.217 | INFO | "malloc" : " 14 GB", | 20260506 09:12:22.217 | INFO | "pinned" : " 29 GB" | 20260506 09:12:22.217 | INFO | } | 20260506 09:12:22.217 | INFO | Perf report for the last 5.13912 seconds: | 20260506 09:12:22.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:12:22.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:22.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:22.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:22.217 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 09:12:22.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:22.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:22.217 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 09:12:22.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:22.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:22.217 | INFO | Uploaded 36GiB to GPU in 1.60348s (22.4511GiB/s) | 20260506 09:12:22.217 | INFO | Downloaded 34GiB to GPU in 1.52006s (22.3675GiB/s) | 20260506 09:12:22.314 | INFO | Starting a round of Baz IO | 20260506 09:12:25.243 | INFO | Populating RTMetrics report | 20260506 09:12:25.243 | INFO | RTMetrics report populated | 20260506 09:12:25.243 | INFO | Opening tmp RTMetrics json file | 20260506 09:12:25.258 | INFO | Serializing RTMetrics to json | 20260506 09:12:25.277 | INFO | Serialization done, swapping tmp file now... | 20260506 09:12:25.292 | INFO | RTMetrics done | 20260506 09:12:25.353 | INFO | Finished a round of Baz IO: Wrote 1.768 GiB in 3.039 seconds | 20260506 09:12:25.353 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 57.11% / 0.16% / 42.73% | 20260506 09:12:25.353 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.4891% / 0.000639757% / 2.38419e-05% | 20260506 09:12:25.353 | INFO | Baz IO overall: | 20260506 09:12:25.353 | INFO | BytesWritten : 61208383290 | 20260506 09:12:25.353 | INFO | NumEvents : 28309919148 | 20260506 09:12:27.356 | INFO | Analyzing chunk frames = [184320,184832) | 20260506 09:12:27.356 | INFO | MemInfo: { | 20260506 09:12:27.356 | INFO | "gpu" : " 17 GB", | 20260506 09:12:27.356 | INFO | "huge" : "", | 20260506 09:12:27.356 | INFO | "huge_shared" : " 27 GB", | 20260506 09:12:27.356 | INFO | "malloc" : " 14 GB", | 20260506 09:12:27.356 | INFO | "pinned" : " 29 GB" | 20260506 09:12:27.356 | INFO | } | 20260506 09:12:27.356 | INFO | Perf report for the last 5.13945 seconds: | 20260506 09:12:27.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:12:27.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:27.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:27.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:27.356 | INFO | ANALYSIS : 57.9%, 3.0, 89.7ms, 0 | 20260506 09:12:27.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:27.356 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 09:12:27.356 | INFO | PRE_HQ : 57.0%, 2.1, 62.1ms, 0 | 20260506 09:12:27.356 | INFO | BAZWRITER : 58.6%, 4.1, 124.0ms, 0 | 20260506 09:12:27.356 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 09:12:27.357 | INFO | Uploaded 36GiB to GPU in 1.64309s (21.91GiB/s) | 20260506 09:12:27.357 | INFO | Downloaded 38GiB to GPU in 1.70383s (22.3027GiB/s) | 20260506 09:12:32.495 | INFO | Analyzing chunk frames = [184832,185344) | 20260506 09:12:32.495 | INFO | MemInfo: { | 20260506 09:12:32.495 | INFO | "gpu" : " 17 GB", | 20260506 09:12:32.495 | INFO | "huge" : "", | 20260506 09:12:32.495 | INFO | "huge_shared" : " 27 GB", | 20260506 09:12:32.495 | INFO | "malloc" : " 14 GB", | 20260506 09:12:32.495 | INFO | "pinned" : " 29 GB" | 20260506 09:12:32.495 | INFO | } | 20260506 09:12:32.495 | INFO | Perf report for the last 5.13913 seconds: | 20260506 09:12:32.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:12:32.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:32.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:32.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:32.495 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260506 09:12:32.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:32.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:32.495 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:12:32.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:32.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:32.496 | INFO | Uploaded 36GiB to GPU in 1.60116s (22.4836GiB/s) | 20260506 09:12:32.496 | INFO | Downloaded 34GiB to GPU in 1.51872s (22.3873GiB/s) | 20260506 09:12:37.635 | INFO | Analyzing chunk frames = [185344,185856) | 20260506 09:12:37.635 | INFO | MemInfo: { | 20260506 09:12:37.635 | INFO | "gpu" : " 17 GB", | 20260506 09:12:37.635 | INFO | "huge" : "", | 20260506 09:12:37.635 | INFO | "huge_shared" : " 27 GB", | 20260506 09:12:37.635 | INFO | "malloc" : " 14 GB", | 20260506 09:12:37.635 | INFO | "pinned" : " 29 GB" | 20260506 09:12:37.635 | INFO | } | 20260506 09:12:37.635 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:12:37.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:12:37.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:37.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:37.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:37.635 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260506 09:12:37.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:37.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:37.635 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:12:37.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:37.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:37.635 | INFO | Uploaded 36GiB to GPU in 1.6054s (22.4244GiB/s) | 20260506 09:12:37.635 | INFO | Downloaded 34GiB to GPU in 1.5231s (22.3229GiB/s) | 20260506 09:12:42.774 | INFO | Analyzing chunk frames = [185856,186368) | 20260506 09:12:42.774 | INFO | MemInfo: { | 20260506 09:12:42.774 | INFO | "gpu" : " 17 GB", | 20260506 09:12:42.774 | INFO | "huge" : "", | 20260506 09:12:42.774 | INFO | "huge_shared" : " 27 GB", | 20260506 09:12:42.774 | INFO | "malloc" : " 14 GB", | 20260506 09:12:42.774 | INFO | "pinned" : " 29 GB" | 20260506 09:12:42.774 | INFO | } | 20260506 09:12:42.774 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:12:42.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:12:42.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:42.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:42.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:42.774 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260506 09:12:42.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:42.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:42.774 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:12:42.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:42.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:42.774 | INFO | Uploaded 36GiB to GPU in 1.60397s (22.4443GiB/s) | 20260506 09:12:42.774 | INFO | Downloaded 34GiB to GPU in 1.5219s (22.3406GiB/s) | 20260506 09:12:47.913 | INFO | Analyzing chunk frames = [186368,186880) | 20260506 09:12:47.913 | INFO | MemInfo: { | 20260506 09:12:47.913 | INFO | "gpu" : " 17 GB", | 20260506 09:12:47.913 | INFO | "huge" : "", | 20260506 09:12:47.913 | INFO | "huge_shared" : " 27 GB", | 20260506 09:12:47.913 | INFO | "malloc" : " 14 GB", | 20260506 09:12:47.913 | INFO | "pinned" : " 29 GB" | 20260506 09:12:47.913 | INFO | } | 20260506 09:12:47.913 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:12:47.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:12:47.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:47.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:47.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:47.913 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 09:12:47.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:47.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:47.913 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:12:47.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:47.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:47.914 | INFO | Uploaded 36GiB to GPU in 1.60299s (22.4581GiB/s) | 20260506 09:12:47.914 | INFO | Downloaded 34GiB to GPU in 1.52032s (22.3637GiB/s) | 20260506 09:12:53.053 | INFO | Analyzing chunk frames = [186880,187392) | 20260506 09:12:53.053 | INFO | MemInfo: { | 20260506 09:12:53.053 | INFO | "gpu" : " 17 GB", | 20260506 09:12:53.053 | INFO | "huge" : "", | 20260506 09:12:53.053 | INFO | "huge_shared" : " 27 GB", | 20260506 09:12:53.053 | INFO | "malloc" : " 14 GB", | 20260506 09:12:53.053 | INFO | "pinned" : " 29 GB" | 20260506 09:12:53.053 | INFO | } | 20260506 09:12:53.053 | INFO | Perf report for the last 5.13948 seconds: | 20260506 09:12:53.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:12:53.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:53.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:53.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:53.053 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260506 09:12:53.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:53.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:53.053 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:12:53.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:53.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:53.053 | INFO | Uploaded 36GiB to GPU in 1.60287s (22.4597GiB/s) | 20260506 09:12:53.053 | INFO | Downloaded 34GiB to GPU in 1.52024s (22.3649GiB/s) | 20260506 09:12:58.192 | INFO | Analyzing chunk frames = [187392,187904) | 20260506 09:12:58.192 | INFO | MemInfo: { | 20260506 09:12:58.192 | INFO | "gpu" : " 17 GB", | 20260506 09:12:58.192 | INFO | "huge" : "", | 20260506 09:12:58.192 | INFO | "huge_shared" : " 27 GB", | 20260506 09:12:58.192 | INFO | "malloc" : " 14 GB", | 20260506 09:12:58.192 | INFO | "pinned" : " 29 GB" | 20260506 09:12:58.192 | INFO | } | 20260506 09:12:58.192 | INFO | Perf report for the last 5.13942 seconds: | 20260506 09:12:58.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:12:58.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:58.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:58.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:58.192 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:12:58.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:58.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:58.192 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:12:58.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:58.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:12:58.193 | INFO | Uploaded 36GiB to GPU in 1.6034s (22.4523GiB/s) | 20260506 09:12:58.193 | INFO | Downloaded 34GiB to GPU in 1.52037s (22.3629GiB/s) | 20260506 09:13:03.331 | INFO | Analyzing chunk frames = [187904,188416) | 20260506 09:13:03.331 | INFO | MemInfo: { | 20260506 09:13:03.331 | INFO | "gpu" : " 17 GB", | 20260506 09:13:03.331 | INFO | "huge" : "", | 20260506 09:13:03.331 | INFO | "huge_shared" : " 27 GB", | 20260506 09:13:03.331 | INFO | "malloc" : " 14 GB", | 20260506 09:13:03.332 | INFO | "pinned" : " 29 GB" | 20260506 09:13:03.332 | INFO | } | 20260506 09:13:03.332 | INFO | Perf report for the last 5.13923 seconds: | 20260506 09:13:03.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:13:03.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:03.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:03.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:03.332 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:13:03.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:03.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:03.332 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 09:13:03.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:03.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:03.332 | INFO | Uploaded 36GiB to GPU in 1.60398s (22.4442GiB/s) | 20260506 09:13:03.332 | INFO | Downloaded 34GiB to GPU in 1.52061s (22.3594GiB/s) | 20260506 09:13:03.332 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:13:03.332 | INFO | Total mem usage: 15.5 GiB | 20260506 09:13:03.332 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.23%) | 20260506 09:13:03.332 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -3.07%) | 20260506 09:13:03.332 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:13:03.332 | INFO | High Water Breakdown: | 20260506 09:13:03.332 | INFO | 59% : 8.89 GiB ( 97% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:13:03.332 | INFO | 40.9% : 6.16 GiB ( 92% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:13:03.332 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:13:03.332 | INFO | ------------------------------------------ | 20260506 09:13:03.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:13:03.332 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:13:03.332 | INFO | Total mem usage: 26.5 GiB | 20260506 09:13:03.332 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:13:03.332 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:13:03.332 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:13:03.332 | INFO | High Water Breakdown: | 20260506 09:13:03.332 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:13:03.332 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:13:03.332 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:13:03.332 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:13:03.332 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:13:03.332 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:13:03.332 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:13:03.332 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:13:03.332 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:13:03.332 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:13:03.332 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:13:03.332 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:13:03.332 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:13:03.332 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:13:03.332 | INFO | ------------------------------------------ | 20260506 09:13:03.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:13:03.333 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:13:03.333 | INFO | Total mem usage: 25.6 GiB | 20260506 09:13:03.333 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:13:03.333 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:13:03.333 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:13:03.333 | INFO | High Water Breakdown: | 20260506 09:13:03.333 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:13:03.333 | INFO | ------------------------------------------ | 20260506 09:13:03.333 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:13:03.333 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:13:03.333 | INFO | Total mem usage: 15.7 GiB | 20260506 09:13:03.333 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:13:03.333 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:13:03.333 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:13:03.333 | INFO | High Water Breakdown: | 20260506 09:13:03.333 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:13:03.333 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:13:03.333 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:13:03.333 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:13:03.333 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:13:03.333 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:13:03.333 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:13:03.333 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:13:03.333 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:13:03.333 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:13:03.333 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:13:03.333 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:13:03.333 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:13:03.333 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:13:03.333 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:13:03.333 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:13:03.333 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:13:03.333 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:13:03.333 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:13:03.333 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:13:03.333 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:13:03.333 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:13:03.333 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:13:03.333 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:13:03.333 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:13:03.333 | INFO | ------------------------------------------ | 20260506 09:13:03.333 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:13:03.333 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:13:03.333 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:13:03.333 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:13:03.333 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 09:13:03.333 | INFO | PerfReport: 0.28%, AggregateStats -- 8.61e-02 / 0.00e+00, 4.88e-03 / 0.00e+00 | 20260506 09:13:03.333 | INFO | PerfReport: 10.87%, Baseline -- 3.29e+00 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260506 09:13:03.333 | INFO | PerfReport: 3.04%, Histogram -- 9.22e-01 / 0.00e+00, 5.21e-02 / 0.00e+00 | 20260506 09:13:03.333 | INFO | PerfReport: 16.01%, DME -- 3.88e+01 / 0.00e+00, 3.18e+00 / 0.00e+00 | 20260506 09:13:03.333 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.10e-02 / 0.00e+00 | 20260506 09:13:03.333 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.30e+00 / 0.00e+00, 1.39e-01 / 0.00e+00 | 20260506 09:13:03.333 | INFO | PerfReport: 3.91%, Metrics -- 1.18e+00 / 0.00e+00, 1.85e-01 / 0.00e+00 | 20260506 09:13:03.333 | INFO | PerfReport: 1.53%, PulseSerializing -- 4.63e-01 / 0.00e+00, 6.76e-02 / 0.00e+00 | 20260506 09:13:03.333 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:13:03.333 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:13:03.333 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:13:03.333 | INFO | PerfReport: Full Execution -- 9.08e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260506 09:13:03.333 | INFO | PerfReport: 18.45%, Upload -- 1.68e+01 / 0.00e+00, 4.01e-01 / 0.00e+00 | 20260506 09:13:03.333 | INFO | PerfReport: 17.71%, Download -- 1.61e+01 / 0.00e+00, 7.25e-01 / 0.00e+00 | 20260506 09:13:03.333 | INFO | PerfReport: 33.35%, Compute -- 3.03e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 09:13:03.333 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 09:13:03.333 | INFO | PerfReport: 0.00%, LockDownload -- 4.85e-04 / 0.00e+00, 1.52e-04 / 0.00e+00 | 20260506 09:13:03.333 | INFO | PerfReport: 29.92%, LockCompute -- 2.72e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 09:13:03.333 | INFO | PerfReport: 0.02%, Misc -- 1.38e-02 / 0.00e+00, 2.52e-02 / 0.00e+00 | 20260506 09:13:03.429 | INFO | Starting a round of Baz IO | 20260506 09:13:06.359 | INFO | Populating RTMetrics report | 20260506 09:13:06.360 | INFO | RTMetrics report populated | 20260506 09:13:06.360 | INFO | Opening tmp RTMetrics json file | 20260506 09:13:06.363 | INFO | Serializing RTMetrics to json | 20260506 09:13:06.367 | INFO | Serialization done, swapping tmp file now... | 20260506 09:13:06.374 | INFO | RTMetrics done | 20260506 09:13:06.445 | INFO | Finished a round of Baz IO: Wrote 1.772 GiB in 3.016 seconds | 20260506 09:13:06.445 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.95% / 0.16% / 42.89% | 20260506 09:13:06.445 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.4524% / 0.000456969% / 2.38419e-05% | 20260506 09:13:06.446 | INFO | Baz IO overall: | 20260506 09:13:06.446 | INFO | BytesWritten : 63111435859 | 20260506 09:13:06.446 | INFO | NumEvents : 29234254135 | 20260506 09:13:08.471 | INFO | Analyzing chunk frames = [188416,188928) | 20260506 09:13:08.471 | INFO | MemInfo: { | 20260506 09:13:08.471 | INFO | "gpu" : " 17 GB", | 20260506 09:13:08.471 | INFO | "huge" : "", | 20260506 09:13:08.471 | INFO | "huge_shared" : " 27 GB", | 20260506 09:13:08.471 | INFO | "malloc" : " 14 GB", | 20260506 09:13:08.471 | INFO | "pinned" : " 29 GB" | 20260506 09:13:08.471 | INFO | } | 20260506 09:13:08.471 | INFO | Perf report for the last 5.13935 seconds: | 20260506 09:13:08.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:13:08.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:08.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:08.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:08.471 | INFO | ANALYSIS : 58.0%, 3.0, 89.7ms, 0 | 20260506 09:13:08.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:08.471 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 09:13:08.471 | INFO | PRE_HQ : 57.0%, 2.1, 62.0ms, 0 | 20260506 09:13:08.471 | INFO | BAZWRITER : 58.3%, 4.1, 121.6ms, 0 | 20260506 09:13:08.471 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260506 09:13:08.472 | INFO | Uploaded 36GiB to GPU in 1.64365s (21.9025GiB/s) | 20260506 09:13:08.472 | INFO | Downloaded 38GiB to GPU in 1.70515s (22.2854GiB/s) | 20260506 09:13:13.610 | INFO | Analyzing chunk frames = [188928,189440) | 20260506 09:13:13.610 | INFO | MemInfo: { | 20260506 09:13:13.610 | INFO | "gpu" : " 17 GB", | 20260506 09:13:13.610 | INFO | "huge" : "", | 20260506 09:13:13.610 | INFO | "huge_shared" : " 27 GB", | 20260506 09:13:13.610 | INFO | "malloc" : " 14 GB", | 20260506 09:13:13.610 | INFO | "pinned" : " 29 GB" | 20260506 09:13:13.610 | INFO | } | 20260506 09:13:13.610 | INFO | Perf report for the last 5.13916 seconds: | 20260506 09:13:13.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:13:13.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:13.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:13.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:13.610 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260506 09:13:13.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:13.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:13.610 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:13:13.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:13.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:13.610 | INFO | Uploaded 36GiB to GPU in 1.60137s (22.4808GiB/s) | 20260506 09:13:13.610 | INFO | Downloaded 34GiB to GPU in 1.51851s (22.3904GiB/s) | 20260506 09:13:18.749 | INFO | Analyzing chunk frames = [189440,189952) | 20260506 09:13:18.749 | INFO | MemInfo: { | 20260506 09:13:18.749 | INFO | "gpu" : " 17 GB", | 20260506 09:13:18.749 | INFO | "huge" : "", | 20260506 09:13:18.749 | INFO | "huge_shared" : " 27 GB", | 20260506 09:13:18.749 | INFO | "malloc" : " 14 GB", | 20260506 09:13:18.750 | INFO | "pinned" : " 29 GB" | 20260506 09:13:18.750 | INFO | } | 20260506 09:13:18.750 | INFO | Perf report for the last 5.13942 seconds: | 20260506 09:13:18.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:13:18.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:18.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:18.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:18.750 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260506 09:13:18.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:18.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:18.750 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:13:18.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:18.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:18.750 | INFO | Uploaded 36GiB to GPU in 1.60518s (22.4274GiB/s) | 20260506 09:13:18.750 | INFO | Downloaded 34GiB to GPU in 1.52218s (22.3364GiB/s) | 20260506 09:13:23.889 | INFO | Analyzing chunk frames = [189952,190464) | 20260506 09:13:23.889 | INFO | MemInfo: { | 20260506 09:13:23.889 | INFO | "gpu" : " 17 GB", | 20260506 09:13:23.889 | INFO | "huge" : "", | 20260506 09:13:23.889 | INFO | "huge_shared" : " 27 GB", | 20260506 09:13:23.889 | INFO | "malloc" : " 14 GB", | 20260506 09:13:23.889 | INFO | "pinned" : " 29 GB" | 20260506 09:13:23.889 | INFO | } | 20260506 09:13:23.889 | INFO | Perf report for the last 5.13936 seconds: | 20260506 09:13:23.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:13:23.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:23.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:23.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:23.889 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260506 09:13:23.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:23.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:23.889 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:13:23.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:23.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:23.889 | INFO | Uploaded 36GiB to GPU in 1.60475s (22.4334GiB/s) | 20260506 09:13:23.890 | INFO | Downloaded 34GiB to GPU in 1.5213s (22.3493GiB/s) | 20260506 09:13:29.028 | INFO | Analyzing chunk frames = [190464,190976) | 20260506 09:13:29.028 | INFO | MemInfo: { | 20260506 09:13:29.028 | INFO | "gpu" : " 17 GB", | 20260506 09:13:29.028 | INFO | "huge" : "", | 20260506 09:13:29.028 | INFO | "huge_shared" : " 27 GB", | 20260506 09:13:29.028 | INFO | "malloc" : " 14 GB", | 20260506 09:13:29.028 | INFO | "pinned" : " 29 GB" | 20260506 09:13:29.028 | INFO | } | 20260506 09:13:29.028 | INFO | Perf report for the last 5.13946 seconds: | 20260506 09:13:29.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:13:29.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:29.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:29.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:29.028 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 09:13:29.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:29.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:29.028 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:13:29.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:29.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:29.029 | INFO | Uploaded 36GiB to GPU in 1.60252s (22.4646GiB/s) | 20260506 09:13:29.029 | INFO | Downloaded 34GiB to GPU in 1.51929s (22.3789GiB/s) | 20260506 09:13:34.167 | INFO | Analyzing chunk frames = [190976,191488) | 20260506 09:13:34.167 | INFO | MemInfo: { | 20260506 09:13:34.167 | INFO | "gpu" : " 17 GB", | 20260506 09:13:34.167 | INFO | "huge" : "", | 20260506 09:13:34.167 | INFO | "huge_shared" : " 27 GB", | 20260506 09:13:34.167 | INFO | "malloc" : " 14 GB", | 20260506 09:13:34.167 | INFO | "pinned" : " 29 GB" | 20260506 09:13:34.167 | INFO | } | 20260506 09:13:34.167 | INFO | Perf report for the last 5.1391 seconds: | 20260506 09:13:34.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:13:34.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:34.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:34.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:34.168 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:13:34.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:34.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:34.168 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:13:34.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:34.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:34.168 | INFO | Uploaded 36GiB to GPU in 1.60311s (22.4563GiB/s) | 20260506 09:13:34.168 | INFO | Downloaded 34GiB to GPU in 1.52053s (22.3607GiB/s) | 20260506 09:13:39.307 | INFO | Analyzing chunk frames = [191488,192000) | 20260506 09:13:39.307 | INFO | MemInfo: { | 20260506 09:13:39.307 | INFO | "gpu" : " 17 GB", | 20260506 09:13:39.307 | INFO | "huge" : "", | 20260506 09:13:39.307 | INFO | "huge_shared" : " 27 GB", | 20260506 09:13:39.307 | INFO | "malloc" : " 14 GB", | 20260506 09:13:39.307 | INFO | "pinned" : " 29 GB" | 20260506 09:13:39.307 | INFO | } | 20260506 09:13:39.307 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:13:39.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:13:39.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:39.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:39.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:39.307 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 09:13:39.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:39.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:39.307 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:13:39.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:39.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:39.307 | INFO | Uploaded 36GiB to GPU in 1.60315s (22.4558GiB/s) | 20260506 09:13:39.307 | INFO | Downloaded 34GiB to GPU in 1.52018s (22.3657GiB/s) | 20260506 09:13:44.446 | INFO | Analyzing chunk frames = [192000,192512) | 20260506 09:13:44.446 | INFO | MemInfo: { | 20260506 09:13:44.446 | INFO | "gpu" : " 17 GB", | 20260506 09:13:44.446 | INFO | "huge" : "", | 20260506 09:13:44.446 | INFO | "huge_shared" : " 27 GB", | 20260506 09:13:44.446 | INFO | "malloc" : " 14 GB", | 20260506 09:13:44.446 | INFO | "pinned" : " 29 GB" | 20260506 09:13:44.446 | INFO | } | 20260506 09:13:44.446 | INFO | Perf report for the last 5.13947 seconds: | 20260506 09:13:44.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:13:44.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:44.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:44.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:44.446 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:13:44.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:44.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:44.446 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 09:13:44.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:44.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:44.446 | INFO | Uploaded 36GiB to GPU in 1.60396s (22.4444GiB/s) | 20260506 09:13:44.447 | INFO | Downloaded 34GiB to GPU in 1.52057s (22.3601GiB/s) | 20260506 09:13:44.544 | INFO | Starting a round of Baz IO | 20260506 09:13:47.479 | INFO | Populating RTMetrics report | 20260506 09:13:47.479 | INFO | RTMetrics report populated | 20260506 09:13:47.479 | INFO | Opening tmp RTMetrics json file | 20260506 09:13:47.480 | INFO | Serializing RTMetrics to json | 20260506 09:13:47.485 | INFO | Serialization done, swapping tmp file now... | 20260506 09:13:47.485 | INFO | RTMetrics done | 20260506 09:13:47.572 | INFO | Finished a round of Baz IO: Wrote 1.776 GiB in 3.027 seconds | 20260506 09:13:47.572 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.80% / 0.15% / 43.05% | 20260506 09:13:47.572 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.4015% / 0.00060002% / 4.76837e-05% | 20260506 09:13:47.572 | INFO | Baz IO overall: | 20260506 09:13:47.572 | INFO | BytesWritten : 65018924798 | 20260506 09:13:47.572 | INFO | NumEvents : 30161105490 | 20260506 09:13:49.586 | INFO | Analyzing chunk frames = [192512,193024) | 20260506 09:13:49.586 | INFO | MemInfo: { | 20260506 09:13:49.586 | INFO | "gpu" : " 17 GB", | 20260506 09:13:49.586 | INFO | "huge" : "", | 20260506 09:13:49.586 | INFO | "huge_shared" : " 27 GB", | 20260506 09:13:49.586 | INFO | "malloc" : " 14 GB", | 20260506 09:13:49.586 | INFO | "pinned" : " 29 GB" | 20260506 09:13:49.586 | INFO | } | 20260506 09:13:49.586 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:13:49.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:13:49.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:49.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:49.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:49.586 | INFO | ANALYSIS : 58.0%, 2.9, 88.0ms, 0 | 20260506 09:13:49.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:49.586 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 09:13:49.586 | INFO | PRE_HQ : 57.2%, 2.1, 61.7ms, 0 | 20260506 09:13:49.586 | INFO | BAZWRITER : 58.5%, 4.5, 134.7ms, 0 | 20260506 09:13:49.586 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 09:13:49.586 | INFO | Uploaded 36GiB to GPU in 1.65291s (21.7798GiB/s) | 20260506 09:13:49.586 | INFO | Downloaded 38GiB to GPU in 1.71586s (22.1464GiB/s) | 20260506 09:13:54.725 | INFO | Analyzing chunk frames = [193024,193536) | 20260506 09:13:54.725 | INFO | MemInfo: { | 20260506 09:13:54.725 | INFO | "gpu" : " 17 GB", | 20260506 09:13:54.725 | INFO | "huge" : "", | 20260506 09:13:54.725 | INFO | "huge_shared" : " 27 GB", | 20260506 09:13:54.725 | INFO | "malloc" : " 14 GB", | 20260506 09:13:54.725 | INFO | "pinned" : " 29 GB" | 20260506 09:13:54.725 | INFO | } | 20260506 09:13:54.725 | INFO | Perf report for the last 5.13926 seconds: | 20260506 09:13:54.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:13:54.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:54.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:54.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:54.725 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260506 09:13:54.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:54.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:54.725 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:13:54.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:54.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:54.725 | INFO | Uploaded 36GiB to GPU in 1.60108s (22.4848GiB/s) | 20260506 09:13:54.725 | INFO | Downloaded 34GiB to GPU in 1.51786s (22.3999GiB/s) | 20260506 09:13:59.864 | INFO | Analyzing chunk frames = [193536,194048) | 20260506 09:13:59.864 | INFO | MemInfo: { | 20260506 09:13:59.864 | INFO | "gpu" : " 17 GB", | 20260506 09:13:59.864 | INFO | "huge" : "", | 20260506 09:13:59.864 | INFO | "huge_shared" : " 27 GB", | 20260506 09:13:59.864 | INFO | "malloc" : " 14 GB", | 20260506 09:13:59.864 | INFO | "pinned" : " 29 GB" | 20260506 09:13:59.864 | INFO | } | 20260506 09:13:59.864 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:13:59.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:13:59.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:59.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:59.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:59.864 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260506 09:13:59.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:59.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:59.864 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:13:59.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:59.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:13:59.865 | INFO | Uploaded 36GiB to GPU in 1.60497s (22.4303GiB/s) | 20260506 09:13:59.865 | INFO | Downloaded 34GiB to GPU in 1.52224s (22.3355GiB/s) | 20260506 09:14:05.004 | INFO | Analyzing chunk frames = [194048,194560) | 20260506 09:14:05.004 | INFO | MemInfo: { | 20260506 09:14:05.004 | INFO | "gpu" : " 17 GB", | 20260506 09:14:05.004 | INFO | "huge" : "", | 20260506 09:14:05.004 | INFO | "huge_shared" : " 27 GB", | 20260506 09:14:05.004 | INFO | "malloc" : " 14 GB", | 20260506 09:14:05.004 | INFO | "pinned" : " 29 GB" | 20260506 09:14:05.004 | INFO | } | 20260506 09:14:05.004 | INFO | Perf report for the last 5.13945 seconds: | 20260506 09:14:05.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:14:05.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:05.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:05.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:05.004 | INFO | ANALYSIS : 56.8%, 3.0, 88.0ms, 0 | 20260506 09:14:05.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:05.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:05.004 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:14:05.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:05.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:05.004 | INFO | Uploaded 36GiB to GPU in 1.60432s (22.4394GiB/s) | 20260506 09:14:05.004 | INFO | Downloaded 34GiB to GPU in 1.52123s (22.3504GiB/s) | 20260506 09:14:10.143 | INFO | Analyzing chunk frames = [194560,195072) | 20260506 09:14:10.143 | INFO | MemInfo: { | 20260506 09:14:10.143 | INFO | "gpu" : " 17 GB", | 20260506 09:14:10.143 | INFO | "huge" : "", | 20260506 09:14:10.143 | INFO | "huge_shared" : " 27 GB", | 20260506 09:14:10.143 | INFO | "malloc" : " 14 GB", | 20260506 09:14:10.143 | INFO | "pinned" : " 29 GB" | 20260506 09:14:10.143 | INFO | } | 20260506 09:14:10.143 | INFO | Perf report for the last 5.13954 seconds: | 20260506 09:14:10.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:14:10.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:10.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:10.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:10.143 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 09:14:10.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:10.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:10.143 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:14:10.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:10.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:10.144 | INFO | Uploaded 36GiB to GPU in 1.60241s (22.4662GiB/s) | 20260506 09:14:10.144 | INFO | Downloaded 34GiB to GPU in 1.51934s (22.3782GiB/s) | 20260506 09:14:15.282 | INFO | Analyzing chunk frames = [195072,195584) | 20260506 09:14:15.282 | INFO | MemInfo: { | 20260506 09:14:15.282 | INFO | "gpu" : " 17 GB", | 20260506 09:14:15.282 | INFO | "huge" : "", | 20260506 09:14:15.282 | INFO | "huge_shared" : " 27 GB", | 20260506 09:14:15.282 | INFO | "malloc" : " 14 GB", | 20260506 09:14:15.282 | INFO | "pinned" : " 29 GB" | 20260506 09:14:15.282 | INFO | } | 20260506 09:14:15.282 | INFO | Perf report for the last 5.13916 seconds: | 20260506 09:14:15.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:14:15.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:15.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:15.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:15.283 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:14:15.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:15.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:15.283 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:14:15.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:15.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:15.283 | INFO | Uploaded 36GiB to GPU in 1.60321s (22.4549GiB/s) | 20260506 09:14:15.283 | INFO | Downloaded 34GiB to GPU in 1.52039s (22.3627GiB/s) | 20260506 09:14:20.422 | INFO | Analyzing chunk frames = [195584,196096) | 20260506 09:14:20.422 | INFO | MemInfo: { | 20260506 09:14:20.422 | INFO | "gpu" : " 17 GB", | 20260506 09:14:20.422 | INFO | "huge" : "", | 20260506 09:14:20.422 | INFO | "huge_shared" : " 27 GB", | 20260506 09:14:20.422 | INFO | "malloc" : " 14 GB", | 20260506 09:14:20.422 | INFO | "pinned" : " 29 GB" | 20260506 09:14:20.422 | INFO | } | 20260506 09:14:20.422 | INFO | Perf report for the last 5.13942 seconds: | 20260506 09:14:20.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:14:20.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:20.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:20.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:20.422 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:14:20.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:20.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:20.422 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:14:20.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:20.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:20.422 | INFO | Uploaded 36GiB to GPU in 1.60327s (22.4541GiB/s) | 20260506 09:14:20.422 | INFO | Downloaded 34GiB to GPU in 1.52068s (22.3584GiB/s) | 20260506 09:14:25.561 | INFO | Analyzing chunk frames = [196096,196608) | 20260506 09:14:25.561 | INFO | MemInfo: { | 20260506 09:14:25.561 | INFO | "gpu" : " 17 GB", | 20260506 09:14:25.561 | INFO | "huge" : "", | 20260506 09:14:25.561 | INFO | "huge_shared" : " 27 GB", | 20260506 09:14:25.561 | INFO | "malloc" : " 14 GB", | 20260506 09:14:25.561 | INFO | "pinned" : " 29 GB" | 20260506 09:14:25.561 | INFO | } | 20260506 09:14:25.561 | INFO | Perf report for the last 5.13903 seconds: | 20260506 09:14:25.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:14:25.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:25.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:25.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:25.561 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 09:14:25.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:25.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:25.561 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 09:14:25.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:25.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:25.561 | INFO | Uploaded 36GiB to GPU in 1.60395s (22.4445GiB/s) | 20260506 09:14:25.561 | INFO | Downloaded 34GiB to GPU in 1.52122s (22.3505GiB/s) | 20260506 09:14:25.561 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:14:25.561 | INFO | Total mem usage: 15.5 GiB | 20260506 09:14:25.561 | INFO | All Time High Water Marks sum: 15.9 GiB (Savings: 2.23%) | 20260506 09:14:25.561 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -3.51%) | 20260506 09:14:25.561 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:14:25.561 | INFO | High Water Breakdown: | 20260506 09:14:25.561 | INFO | 58.9% : 8.84 GiB ( 96.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:14:25.561 | INFO | 40.9% : 6.14 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:14:25.561 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:14:25.561 | INFO | ------------------------------------------ | 20260506 09:14:25.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:14:25.561 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:14:25.561 | INFO | Total mem usage: 26.5 GiB | 20260506 09:14:25.561 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:14:25.561 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:14:25.561 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:14:25.561 | INFO | High Water Breakdown: | 20260506 09:14:25.561 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:14:25.561 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:14:25.561 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:14:25.561 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:14:25.561 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:14:25.561 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:14:25.561 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:14:25.561 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:14:25.561 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:14:25.561 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:14:25.562 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:14:25.562 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:14:25.562 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:14:25.562 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:14:25.562 | INFO | ------------------------------------------ | 20260506 09:14:25.562 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:14:25.562 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:14:25.562 | INFO | Total mem usage: 25.6 GiB | 20260506 09:14:25.562 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:14:25.562 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:14:25.562 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:14:25.562 | INFO | High Water Breakdown: | 20260506 09:14:25.562 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:14:25.562 | INFO | ------------------------------------------ | 20260506 09:14:25.562 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:14:25.562 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:14:25.562 | INFO | Total mem usage: 15.7 GiB | 20260506 09:14:25.562 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:14:25.562 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:14:25.562 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:14:25.562 | INFO | High Water Breakdown: | 20260506 09:14:25.562 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:14:25.562 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:14:25.562 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:14:25.562 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:14:25.562 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:14:25.562 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:14:25.562 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:14:25.562 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:14:25.562 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:14:25.562 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:14:25.562 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:14:25.562 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:14:25.562 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:14:25.562 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:14:25.562 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:14:25.562 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:14:25.562 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:14:25.562 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:14:25.562 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:14:25.562 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:14:25.562 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:14:25.562 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:14:25.562 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:14:25.562 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:14:25.562 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:14:25.562 | INFO | ------------------------------------------ | 20260506 09:14:25.562 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:14:25.562 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:14:25.562 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:14:25.562 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:14:25.562 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 09:14:25.562 | INFO | PerfReport: 0.28%, AggregateStats -- 8.63e-02 / 0.00e+00, 4.39e-03 / 0.00e+00 | 20260506 09:14:25.562 | INFO | PerfReport: 10.87%, Baseline -- 3.29e+00 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260506 09:14:25.562 | INFO | PerfReport: 3.04%, Histogram -- 9.22e-01 / 0.00e+00, 5.20e-02 / 0.00e+00 | 20260506 09:14:25.562 | INFO | PerfReport: 16.03%, DME -- 3.89e+01 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260506 09:14:25.562 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.14e-02 / 0.00e+00 | 20260506 09:14:25.562 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.29e+00 / 0.00e+00, 1.39e-01 / 0.00e+00 | 20260506 09:14:25.562 | INFO | PerfReport: 3.90%, Metrics -- 1.18e+00 / 0.00e+00, 1.85e-01 / 0.00e+00 | 20260506 09:14:25.562 | INFO | PerfReport: 1.53%, PulseSerializing -- 4.62e-01 / 0.00e+00, 6.70e-02 / 0.00e+00 | 20260506 09:14:25.562 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:14:25.562 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:14:25.562 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:14:25.562 | INFO | PerfReport: Full Execution -- 9.08e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260506 09:14:25.562 | INFO | PerfReport: 18.45%, Upload -- 1.68e+01 / 0.00e+00, 4.13e-01 / 0.00e+00 | 20260506 09:14:25.562 | INFO | PerfReport: 17.71%, Download -- 1.61e+01 / 0.00e+00, 7.38e-01 / 0.00e+00 | 20260506 09:14:25.562 | INFO | PerfReport: 33.36%, Compute -- 3.03e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 09:14:25.562 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:14:25.562 | INFO | PerfReport: 0.00%, LockDownload -- 4.91e-04 / 0.00e+00, 1.49e-04 / 0.00e+00 | 20260506 09:14:25.562 | INFO | PerfReport: 29.90%, LockCompute -- 2.72e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 09:14:25.562 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260506 09:14:25.658 | INFO | Starting a round of Baz IO | 20260506 09:14:28.589 | INFO | Populating RTMetrics report | 20260506 09:14:28.589 | INFO | RTMetrics report populated | 20260506 09:14:28.589 | INFO | Opening tmp RTMetrics json file | 20260506 09:14:28.591 | INFO | Serializing RTMetrics to json | 20260506 09:14:28.600 | INFO | Serialization done, swapping tmp file now... | 20260506 09:14:28.639 | INFO | RTMetrics done | 20260506 09:14:29.041 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.382 seconds | 20260506 09:14:29.041 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.65% / 0.15% / 43.20% | 20260506 09:14:29.041 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.3797% / 0.000560284% / 5.16574e-05% | 20260506 09:14:29.041 | INFO | Baz IO overall: | 20260506 09:14:29.041 | INFO | BytesWritten : 66931728888 | 20260506 09:14:29.041 | INFO | NumEvents : 31091023963 | 20260506 09:14:30.700 | INFO | Analyzing chunk frames = [196608,197120) | 20260506 09:14:30.700 | INFO | MemInfo: { | 20260506 09:14:30.700 | INFO | "gpu" : " 17 GB", | 20260506 09:14:30.700 | INFO | "huge" : "", | 20260506 09:14:30.700 | INFO | "huge_shared" : " 27 GB", | 20260506 09:14:30.700 | INFO | "malloc" : " 14 GB", | 20260506 09:14:30.700 | INFO | "pinned" : " 29 GB" | 20260506 09:14:30.700 | INFO | } | 20260506 09:14:30.700 | INFO | Perf report for the last 5.13944 seconds: | 20260506 09:14:30.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:14:30.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:30.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:30.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:30.700 | INFO | ANALYSIS : 58.0%, 2.8, 86.4ms, 0 | 20260506 09:14:30.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:30.700 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 09:14:30.700 | INFO | PRE_HQ : 57.1%, 2.1, 61.3ms, 0 | 20260506 09:14:30.701 | INFO | BAZWRITER : 65.3%, 4.7, 159.0ms, 0 | 20260506 09:14:30.701 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 09:14:30.701 | INFO | Uploaded 36GiB to GPU in 1.66921s (21.5671GiB/s) | 20260506 09:14:30.701 | INFO | Downloaded 38GiB to GPU in 1.73393s (21.9155GiB/s) | 20260506 09:14:35.840 | INFO | Analyzing chunk frames = [197120,197632) | 20260506 09:14:35.840 | INFO | MemInfo: { | 20260506 09:14:35.840 | INFO | "gpu" : " 17 GB", | 20260506 09:14:35.840 | INFO | "huge" : "", | 20260506 09:14:35.840 | INFO | "huge_shared" : " 27 GB", | 20260506 09:14:35.840 | INFO | "malloc" : " 14 GB", | 20260506 09:14:35.840 | INFO | "pinned" : " 29 GB" | 20260506 09:14:35.840 | INFO | } | 20260506 09:14:35.840 | INFO | Perf report for the last 5.13932 seconds: | 20260506 09:14:35.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:14:35.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:35.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:35.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:35.840 | INFO | ANALYSIS : 57.7%, 3.0, 90.1ms, 0 | 20260506 09:14:35.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:35.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:35.840 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 09:14:35.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:35.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:35.840 | INFO | Uploaded 36GiB to GPU in 1.60125s (22.4824GiB/s) | 20260506 09:14:35.840 | INFO | Downloaded 34GiB to GPU in 1.51785s (22.4002GiB/s) | 20260506 09:14:40.979 | INFO | Analyzing chunk frames = [197632,198144) | 20260506 09:14:40.979 | INFO | MemInfo: { | 20260506 09:14:40.979 | INFO | "gpu" : " 17 GB", | 20260506 09:14:40.979 | INFO | "huge" : "", | 20260506 09:14:40.979 | INFO | "huge_shared" : " 27 GB", | 20260506 09:14:40.979 | INFO | "malloc" : " 14 GB", | 20260506 09:14:40.979 | INFO | "pinned" : " 29 GB" | 20260506 09:14:40.979 | INFO | } | 20260506 09:14:40.979 | INFO | Perf report for the last 5.13937 seconds: | 20260506 09:14:40.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:14:40.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:40.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:40.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:40.979 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260506 09:14:40.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:40.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:40.979 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:14:40.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:40.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:40.979 | INFO | Uploaded 36GiB to GPU in 1.60493s (22.431GiB/s) | 20260506 09:14:40.979 | INFO | Downloaded 34GiB to GPU in 1.52185s (22.3413GiB/s) | 20260506 09:14:46.118 | INFO | Analyzing chunk frames = [198144,198656) | 20260506 09:14:46.118 | INFO | MemInfo: { | 20260506 09:14:46.118 | INFO | "gpu" : " 17 GB", | 20260506 09:14:46.118 | INFO | "huge" : "", | 20260506 09:14:46.118 | INFO | "huge_shared" : " 27 GB", | 20260506 09:14:46.118 | INFO | "malloc" : " 14 GB", | 20260506 09:14:46.118 | INFO | "pinned" : " 29 GB" | 20260506 09:14:46.118 | INFO | } | 20260506 09:14:46.118 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:14:46.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:14:46.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:46.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:46.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:46.118 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260506 09:14:46.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:46.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:46.119 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:14:46.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:46.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:46.119 | INFO | Uploaded 36GiB to GPU in 1.60434s (22.4391GiB/s) | 20260506 09:14:46.119 | INFO | Downloaded 34GiB to GPU in 1.52238s (22.3335GiB/s) | 20260506 09:14:51.258 | INFO | Analyzing chunk frames = [198656,199168) | 20260506 09:14:51.258 | INFO | MemInfo: { | 20260506 09:14:51.258 | INFO | "gpu" : " 17 GB", | 20260506 09:14:51.258 | INFO | "huge" : "", | 20260506 09:14:51.258 | INFO | "huge_shared" : " 27 GB", | 20260506 09:14:51.258 | INFO | "malloc" : " 14 GB", | 20260506 09:14:51.258 | INFO | "pinned" : " 29 GB" | 20260506 09:14:51.258 | INFO | } | 20260506 09:14:51.258 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:14:51.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:14:51.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:51.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:51.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:51.258 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 09:14:51.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:51.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:51.258 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:14:51.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:51.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:51.258 | INFO | Uploaded 36GiB to GPU in 1.602s (22.4719GiB/s) | 20260506 09:14:51.258 | INFO | Downloaded 34GiB to GPU in 1.52004s (22.3679GiB/s) | 20260506 09:14:56.397 | INFO | Analyzing chunk frames = [199168,199680) | 20260506 09:14:56.397 | INFO | MemInfo: { | 20260506 09:14:56.397 | INFO | "gpu" : " 17 GB", | 20260506 09:14:56.397 | INFO | "huge" : "", | 20260506 09:14:56.397 | INFO | "huge_shared" : " 27 GB", | 20260506 09:14:56.397 | INFO | "malloc" : " 14 GB", | 20260506 09:14:56.397 | INFO | "pinned" : " 29 GB" | 20260506 09:14:56.397 | INFO | } | 20260506 09:14:56.397 | INFO | Perf report for the last 5.1394 seconds: | 20260506 09:14:56.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:14:56.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:56.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:56.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:56.397 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:14:56.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:56.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:56.397 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:14:56.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:56.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:14:56.397 | INFO | Uploaded 36GiB to GPU in 1.60206s (22.471GiB/s) | 20260506 09:14:56.398 | INFO | Downloaded 34GiB to GPU in 1.51983s (22.371GiB/s) | 20260506 09:15:01.536 | INFO | Analyzing chunk frames = [199680,200192) | 20260506 09:15:01.536 | INFO | MemInfo: { | 20260506 09:15:01.536 | INFO | "gpu" : " 17 GB", | 20260506 09:15:01.536 | INFO | "huge" : "", | 20260506 09:15:01.536 | INFO | "huge_shared" : " 27 GB", | 20260506 09:15:01.536 | INFO | "malloc" : " 14 GB", | 20260506 09:15:01.536 | INFO | "pinned" : " 29 GB" | 20260506 09:15:01.536 | INFO | } | 20260506 09:15:01.537 | INFO | Perf report for the last 5.13936 seconds: | 20260506 09:15:01.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:15:01.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:01.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:01.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:01.537 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 09:15:01.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:01.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:01.537 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:15:01.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:01.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:01.537 | INFO | Uploaded 36GiB to GPU in 1.60356s (22.4501GiB/s) | 20260506 09:15:01.537 | INFO | Downloaded 34GiB to GPU in 1.52137s (22.3483GiB/s) | 20260506 09:15:06.676 | INFO | Analyzing chunk frames = [200192,200704) | 20260506 09:15:06.676 | INFO | MemInfo: { | 20260506 09:15:06.676 | INFO | "gpu" : " 17 GB", | 20260506 09:15:06.676 | INFO | "huge" : "", | 20260506 09:15:06.676 | INFO | "huge_shared" : " 27 GB", | 20260506 09:15:06.676 | INFO | "malloc" : " 14 GB", | 20260506 09:15:06.676 | INFO | "pinned" : " 29 GB" | 20260506 09:15:06.676 | INFO | } | 20260506 09:15:06.676 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:15:06.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:15:06.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:06.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:06.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:06.676 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:15:06.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:06.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:06.676 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 09:15:06.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:06.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:06.677 | INFO | Uploaded 36GiB to GPU in 1.60242s (22.466GiB/s) | 20260506 09:15:06.677 | INFO | Downloaded 34GiB to GPU in 1.52045s (22.3618GiB/s) | 20260506 09:15:06.773 | INFO | Starting a round of Baz IO | 20260506 09:15:09.702 | INFO | Populating RTMetrics report | 20260506 09:15:09.703 | INFO | RTMetrics report populated | 20260506 09:15:09.703 | INFO | Opening tmp RTMetrics json file | 20260506 09:15:09.718 | INFO | Serializing RTMetrics to json | 20260506 09:15:09.742 | INFO | Serialization done, swapping tmp file now... | 20260506 09:15:09.746 | INFO | RTMetrics done | 20260506 09:15:10.052 | INFO | Finished a round of Baz IO: Wrote 1.786 GiB in 3.279 seconds | 20260506 09:15:10.053 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.50% / 0.14% / 43.35% | 20260506 09:15:10.053 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.3222% / 0.000488758% / 4.37101e-05% | 20260506 09:15:10.053 | INFO | Baz IO overall: | 20260506 09:15:10.053 | INFO | BytesWritten : 68849345700 | 20260506 09:15:10.053 | INFO | NumEvents : 32022698941 | 20260506 09:15:11.815 | INFO | Analyzing chunk frames = [200704,201216) | 20260506 09:15:11.815 | INFO | MemInfo: { | 20260506 09:15:11.815 | INFO | "gpu" : " 17 GB", | 20260506 09:15:11.815 | INFO | "huge" : "", | 20260506 09:15:11.815 | INFO | "huge_shared" : " 27 GB", | 20260506 09:15:11.815 | INFO | "malloc" : " 14 GB", | 20260506 09:15:11.815 | INFO | "pinned" : " 29 GB" | 20260506 09:15:11.815 | INFO | } | 20260506 09:15:11.815 | INFO | Perf report for the last 5.13932 seconds: | 20260506 09:15:11.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:15:11.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:11.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:11.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:11.815 | INFO | ANALYSIS : 57.9%, 2.9, 87.5ms, 0 | 20260506 09:15:11.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:11.815 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260506 09:15:11.815 | INFO | PRE_HQ : 57.1%, 2.1, 61.6ms, 0 | 20260506 09:15:11.815 | INFO | BAZWRITER : 63.4%, 4.5, 147.1ms, 0 | 20260506 09:15:11.815 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 09:15:11.815 | INFO | Uploaded 36GiB to GPU in 1.65413s (21.7637GiB/s) | 20260506 09:15:11.816 | INFO | Downloaded 38GiB to GPU in 1.71736s (22.127GiB/s) | 20260506 09:15:16.954 | INFO | Analyzing chunk frames = [201216,201728) | 20260506 09:15:16.954 | INFO | MemInfo: { | 20260506 09:15:16.954 | INFO | "gpu" : " 17 GB", | 20260506 09:15:16.954 | INFO | "huge" : "", | 20260506 09:15:16.954 | INFO | "huge_shared" : " 27 GB", | 20260506 09:15:16.954 | INFO | "malloc" : " 14 GB", | 20260506 09:15:16.955 | INFO | "pinned" : " 29 GB" | 20260506 09:15:16.955 | INFO | } | 20260506 09:15:16.955 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:15:16.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:15:16.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:16.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:16.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:16.955 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260506 09:15:16.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:16.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:16.955 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:15:16.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:16.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:16.955 | INFO | Uploaded 36GiB to GPU in 1.60158s (22.4778GiB/s) | 20260506 09:15:16.955 | INFO | Downloaded 34GiB to GPU in 1.51814s (22.3958GiB/s) | 20260506 09:15:22.094 | INFO | Analyzing chunk frames = [201728,202240) | 20260506 09:15:22.094 | INFO | MemInfo: { | 20260506 09:15:22.094 | INFO | "gpu" : " 17 GB", | 20260506 09:15:22.094 | INFO | "huge" : "", | 20260506 09:15:22.094 | INFO | "huge_shared" : " 27 GB", | 20260506 09:15:22.094 | INFO | "malloc" : " 14 GB", | 20260506 09:15:22.094 | INFO | "pinned" : " 29 GB" | 20260506 09:15:22.094 | INFO | } | 20260506 09:15:22.094 | INFO | Perf report for the last 5.13917 seconds: | 20260506 09:15:22.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:15:22.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:22.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:22.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:22.094 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260506 09:15:22.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:22.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:22.094 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:15:22.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:22.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:22.094 | INFO | Uploaded 36GiB to GPU in 1.60539s (22.4245GiB/s) | 20260506 09:15:22.094 | INFO | Downloaded 34GiB to GPU in 1.52306s (22.3235GiB/s) | 20260506 09:15:27.233 | INFO | Analyzing chunk frames = [202240,202752) | 20260506 09:15:27.233 | INFO | MemInfo: { | 20260506 09:15:27.233 | INFO | "gpu" : " 17 GB", | 20260506 09:15:27.233 | INFO | "huge" : "", | 20260506 09:15:27.233 | INFO | "huge_shared" : " 27 GB", | 20260506 09:15:27.233 | INFO | "malloc" : " 14 GB", | 20260506 09:15:27.233 | INFO | "pinned" : " 29 GB" | 20260506 09:15:27.233 | INFO | } | 20260506 09:15:27.233 | INFO | Perf report for the last 5.13943 seconds: | 20260506 09:15:27.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:15:27.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:27.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:27.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:27.233 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260506 09:15:27.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:27.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:27.233 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:15:27.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:27.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:27.233 | INFO | Uploaded 36GiB to GPU in 1.60448s (22.4371GiB/s) | 20260506 09:15:27.234 | INFO | Downloaded 34GiB to GPU in 1.52091s (22.3551GiB/s) | 20260506 09:15:32.373 | INFO | Analyzing chunk frames = [202752,203264) | 20260506 09:15:32.373 | INFO | MemInfo: { | 20260506 09:15:32.373 | INFO | "gpu" : " 17 GB", | 20260506 09:15:32.373 | INFO | "huge" : "", | 20260506 09:15:32.373 | INFO | "huge_shared" : " 27 GB", | 20260506 09:15:32.373 | INFO | "malloc" : " 14 GB", | 20260506 09:15:32.373 | INFO | "pinned" : " 29 GB" | 20260506 09:15:32.373 | INFO | } | 20260506 09:15:32.373 | INFO | Perf report for the last 5.13942 seconds: | 20260506 09:15:32.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:15:32.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:32.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:32.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:32.373 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 09:15:32.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:32.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:32.373 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:15:32.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:32.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:32.373 | INFO | Uploaded 36GiB to GPU in 1.60359s (22.4497GiB/s) | 20260506 09:15:32.373 | INFO | Downloaded 34GiB to GPU in 1.52071s (22.358GiB/s) | 20260506 09:15:37.512 | INFO | Analyzing chunk frames = [203264,203776) | 20260506 09:15:37.512 | INFO | MemInfo: { | 20260506 09:15:37.512 | INFO | "gpu" : " 17 GB", | 20260506 09:15:37.512 | INFO | "huge" : "", | 20260506 09:15:37.512 | INFO | "huge_shared" : " 27 GB", | 20260506 09:15:37.512 | INFO | "malloc" : " 14 GB", | 20260506 09:15:37.512 | INFO | "pinned" : " 29 GB" | 20260506 09:15:37.512 | INFO | } | 20260506 09:15:37.512 | INFO | Perf report for the last 5.13915 seconds: | 20260506 09:15:37.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:15:37.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:37.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:37.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:37.512 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:15:37.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:37.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:37.512 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:15:37.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:37.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:37.512 | INFO | Uploaded 36GiB to GPU in 1.60238s (22.4666GiB/s) | 20260506 09:15:37.512 | INFO | Downloaded 34GiB to GPU in 1.51994s (22.3693GiB/s) | 20260506 09:15:42.651 | INFO | Analyzing chunk frames = [203776,204288) | 20260506 09:15:42.651 | INFO | MemInfo: { | 20260506 09:15:42.651 | INFO | "gpu" : " 17 GB", | 20260506 09:15:42.651 | INFO | "huge" : "", | 20260506 09:15:42.651 | INFO | "huge_shared" : " 27 GB", | 20260506 09:15:42.651 | INFO | "malloc" : " 14 GB", | 20260506 09:15:42.651 | INFO | "pinned" : " 29 GB" | 20260506 09:15:42.651 | INFO | } | 20260506 09:15:42.651 | INFO | Perf report for the last 5.13951 seconds: | 20260506 09:15:42.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:15:42.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:42.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:42.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:42.651 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:15:42.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:42.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:42.652 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:15:42.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:42.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:42.652 | INFO | Uploaded 36GiB to GPU in 1.60378s (22.447GiB/s) | 20260506 09:15:42.652 | INFO | Downloaded 34GiB to GPU in 1.52167s (22.3439GiB/s) | 20260506 09:15:47.791 | INFO | Analyzing chunk frames = [204288,204800) | 20260506 09:15:47.791 | INFO | MemInfo: { | 20260506 09:15:47.791 | INFO | "gpu" : " 17 GB", | 20260506 09:15:47.791 | INFO | "huge" : "", | 20260506 09:15:47.791 | INFO | "huge_shared" : " 27 GB", | 20260506 09:15:47.791 | INFO | "malloc" : " 14 GB", | 20260506 09:15:47.791 | INFO | "pinned" : " 29 GB" | 20260506 09:15:47.791 | INFO | } | 20260506 09:15:47.791 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:15:47.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:15:47.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:47.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:47.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:47.791 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:15:47.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:47.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:47.791 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 09:15:47.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:47.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:47.791 | INFO | Uploaded 36GiB to GPU in 1.60448s (22.4372GiB/s) | 20260506 09:15:47.791 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:15:47.791 | INFO | Total mem usage: 15.7 GiB | 20260506 09:15:47.791 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.38%) | 20260506 09:15:47.791 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: 0.0488%) | 20260506 09:15:47.791 | INFO | Downloaded 34GiB to GPU in 1.52156s (22.3454GiB/s) | 20260506 09:15:47.791 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:15:47.791 | INFO | High Water Breakdown: | 20260506 09:15:47.791 | INFO | 59.8% : 9.42 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:15:47.791 | INFO | 40.1% : 6.32 GiB ( 94.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:15:47.791 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:15:47.791 | INFO | ------------------------------------------ | 20260506 09:15:47.791 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:15:47.791 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:15:47.791 | INFO | Total mem usage: 26.5 GiB | 20260506 09:15:47.791 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:15:47.791 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:15:47.791 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:15:47.791 | INFO | High Water Breakdown: | 20260506 09:15:47.791 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:15:47.791 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:15:47.791 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:15:47.791 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:15:47.791 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:15:47.791 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:15:47.791 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:15:47.791 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:15:47.791 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:15:47.791 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:15:47.791 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:15:47.791 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:15:47.791 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:15:47.791 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:15:47.791 | INFO | ------------------------------------------ | 20260506 09:15:47.791 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:15:47.791 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:15:47.791 | INFO | Total mem usage: 25.6 GiB | 20260506 09:15:47.791 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:15:47.791 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:15:47.791 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:15:47.791 | INFO | High Water Breakdown: | 20260506 09:15:47.791 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:15:47.791 | INFO | ------------------------------------------ | 20260506 09:15:47.791 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:15:47.792 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:15:47.792 | INFO | Total mem usage: 15.7 GiB | 20260506 09:15:47.792 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:15:47.792 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:15:47.792 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:15:47.792 | INFO | High Water Breakdown: | 20260506 09:15:47.792 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:15:47.792 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:15:47.792 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:15:47.792 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:15:47.792 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:15:47.792 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:15:47.792 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:15:47.792 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:15:47.792 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:15:47.792 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:15:47.792 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:15:47.792 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:15:47.792 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:15:47.792 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:15:47.792 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:15:47.792 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:15:47.792 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:15:47.792 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:15:47.792 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:15:47.792 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:15:47.792 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:15:47.792 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:15:47.792 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:15:47.792 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:15:47.792 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:15:47.792 | INFO | ------------------------------------------ | 20260506 09:15:47.792 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:15:47.792 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:15:47.792 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:15:47.792 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:15:47.792 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 09:15:47.792 | INFO | PerfReport: 0.28%, AggregateStats -- 8.61e-02 / 0.00e+00, 4.38e-03 / 0.00e+00 | 20260506 09:15:47.792 | INFO | PerfReport: 10.87%, Baseline -- 3.29e+00 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260506 09:15:47.792 | INFO | PerfReport: 3.04%, Histogram -- 9.22e-01 / 0.00e+00, 5.20e-02 / 0.00e+00 | 20260506 09:15:47.792 | INFO | PerfReport: 16.03%, DME -- 3.88e+01 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260506 09:15:47.792 | INFO | PerfReport: 56.78%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.17e-02 / 0.00e+00 | 20260506 09:15:47.792 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.29e+00 / 0.00e+00, 1.38e-01 / 0.00e+00 | 20260506 09:15:47.792 | INFO | PerfReport: 3.90%, Metrics -- 1.18e+00 / 0.00e+00, 1.84e-01 / 0.00e+00 | 20260506 09:15:47.792 | INFO | PerfReport: 1.52%, PulseSerializing -- 4.61e-01 / 0.00e+00, 6.71e-02 / 0.00e+00 | 20260506 09:15:47.792 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:15:47.792 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:15:47.792 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:15:47.792 | INFO | PerfReport: Full Execution -- 9.06e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 09:15:47.792 | INFO | PerfReport: 18.52%, Upload -- 1.68e+01 / 0.00e+00, 4.56e-01 / 0.00e+00 | 20260506 09:15:47.792 | INFO | PerfReport: 17.78%, Download -- 1.61e+01 / 0.00e+00, 8.02e-01 / 0.00e+00 | 20260506 09:15:47.792 | INFO | PerfReport: 33.45%, Compute -- 3.03e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 09:15:47.792 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 09:15:47.792 | INFO | PerfReport: 0.00%, LockDownload -- 4.94e-04 / 0.00e+00, 1.56e-04 / 0.00e+00 | 20260506 09:15:47.792 | INFO | PerfReport: 29.67%, LockCompute -- 2.69e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 09:15:47.792 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.32e-02 / 0.00e+00 | 20260506 09:15:47.887 | INFO | Starting a round of Baz IO | 20260506 09:15:50.817 | INFO | Populating RTMetrics report | 20260506 09:15:50.817 | INFO | RTMetrics report populated | 20260506 09:15:50.817 | INFO | Opening tmp RTMetrics json file | 20260506 09:15:50.951 | INFO | Serializing RTMetrics to json | 20260506 09:15:50.954 | INFO | Serialization done, swapping tmp file now... | 20260506 09:15:50.955 | INFO | RTMetrics done | 20260506 09:15:50.980 | INFO | Finished a round of Baz IO: Wrote 1.788 GiB in 3.092 seconds | 20260506 09:15:50.980 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.36% / 0.14% / 43.50% | 20260506 09:15:50.980 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.2461% / 0.000492732% / 3.57628e-05% | 20260506 09:15:50.980 | INFO | Baz IO overall: | 20260506 09:15:50.980 | INFO | BytesWritten : 70769478308 | 20260506 09:15:50.980 | INFO | NumEvents : 32954930622 | 20260506 09:15:52.930 | INFO | Analyzing chunk frames = [204800,205312) | 20260506 09:15:52.930 | INFO | MemInfo: { | 20260506 09:15:52.930 | INFO | "gpu" : " 17 GB", | 20260506 09:15:52.930 | INFO | "huge" : "", | 20260506 09:15:52.930 | INFO | "huge_shared" : " 27 GB", | 20260506 09:15:52.930 | INFO | "malloc" : " 14 GB", | 20260506 09:15:52.930 | INFO | "pinned" : " 29 GB" | 20260506 09:15:52.930 | INFO | } | 20260506 09:15:52.930 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:15:52.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:15:52.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:52.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:52.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:52.930 | INFO | ANALYSIS : 57.9%, 3.0, 89.9ms, 0 | 20260506 09:15:52.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:52.930 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260506 09:15:52.930 | INFO | PRE_HQ : 57.0%, 2.1, 62.0ms, 0 | 20260506 09:15:52.930 | INFO | BAZWRITER : 59.8%, 4.1, 127.2ms, 0 | 20260506 09:15:52.930 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 09:15:52.931 | INFO | Uploaded 36GiB to GPU in 1.64016s (21.9491GiB/s) | 20260506 09:15:52.931 | INFO | Downloaded 38GiB to GPU in 1.70231s (22.3225GiB/s) | 20260506 09:15:58.069 | INFO | Analyzing chunk frames = [205312,205824) | 20260506 09:15:58.069 | INFO | MemInfo: { | 20260506 09:15:58.069 | INFO | "gpu" : " 17 GB", | 20260506 09:15:58.069 | INFO | "huge" : "", | 20260506 09:15:58.069 | INFO | "huge_shared" : " 27 GB", | 20260506 09:15:58.069 | INFO | "malloc" : " 14 GB", | 20260506 09:15:58.069 | INFO | "pinned" : " 29 GB" | 20260506 09:15:58.069 | INFO | } | 20260506 09:15:58.069 | INFO | Perf report for the last 5.13923 seconds: | 20260506 09:15:58.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:15:58.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:58.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:58.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:58.069 | INFO | ANALYSIS : 57.8%, 3.0, 90.1ms, 0 | 20260506 09:15:58.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:58.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:58.069 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:15:58.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:58.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:15:58.070 | INFO | Uploaded 36GiB to GPU in 1.60079s (22.4888GiB/s) | 20260506 09:15:58.070 | INFO | Downloaded 34GiB to GPU in 1.51796s (22.3985GiB/s) | 20260506 09:16:03.209 | INFO | Analyzing chunk frames = [205824,206336) | 20260506 09:16:03.209 | INFO | MemInfo: { | 20260506 09:16:03.209 | INFO | "gpu" : " 17 GB", | 20260506 09:16:03.209 | INFO | "huge" : "", | 20260506 09:16:03.209 | INFO | "huge_shared" : " 27 GB", | 20260506 09:16:03.209 | INFO | "malloc" : " 14 GB", | 20260506 09:16:03.209 | INFO | "pinned" : " 29 GB" | 20260506 09:16:03.209 | INFO | } | 20260506 09:16:03.209 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:16:03.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:16:03.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:03.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:03.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:03.209 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260506 09:16:03.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:03.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:03.209 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:16:03.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:03.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:03.209 | INFO | Uploaded 36GiB to GPU in 1.6058s (22.4187GiB/s) | 20260506 09:16:03.209 | INFO | Downloaded 34GiB to GPU in 1.52303s (22.3239GiB/s) | 20260506 09:16:08.348 | INFO | Analyzing chunk frames = [206336,206848) | 20260506 09:16:08.348 | INFO | MemInfo: { | 20260506 09:16:08.348 | INFO | "gpu" : " 17 GB", | 20260506 09:16:08.348 | INFO | "huge" : "", | 20260506 09:16:08.348 | INFO | "huge_shared" : " 27 GB", | 20260506 09:16:08.348 | INFO | "malloc" : " 14 GB", | 20260506 09:16:08.348 | INFO | "pinned" : " 29 GB" | 20260506 09:16:08.348 | INFO | } | 20260506 09:16:08.348 | INFO | Perf report for the last 5.13942 seconds: | 20260506 09:16:08.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:16:08.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:08.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:08.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:08.348 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 09:16:08.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:08.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:08.348 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:16:08.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:08.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:08.348 | INFO | Uploaded 36GiB to GPU in 1.6043s (22.4397GiB/s) | 20260506 09:16:08.349 | INFO | Downloaded 34GiB to GPU in 1.52197s (22.3395GiB/s) | 20260506 09:16:13.487 | INFO | Analyzing chunk frames = [206848,207360) | 20260506 09:16:13.487 | INFO | MemInfo: { | 20260506 09:16:13.487 | INFO | "gpu" : " 17 GB", | 20260506 09:16:13.487 | INFO | "huge" : "", | 20260506 09:16:13.487 | INFO | "huge_shared" : " 27 GB", | 20260506 09:16:13.487 | INFO | "malloc" : " 14 GB", | 20260506 09:16:13.487 | INFO | "pinned" : " 29 GB" | 20260506 09:16:13.487 | INFO | } | 20260506 09:16:13.487 | INFO | Perf report for the last 5.13916 seconds: | 20260506 09:16:13.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:16:13.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:13.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:13.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:13.487 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 09:16:13.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:13.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:13.487 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:16:13.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:13.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:13.488 | INFO | Uploaded 36GiB to GPU in 1.60291s (22.4591GiB/s) | 20260506 09:16:13.488 | INFO | Downloaded 34GiB to GPU in 1.51974s (22.3723GiB/s) | 20260506 09:16:18.627 | INFO | Analyzing chunk frames = [207360,207872) | 20260506 09:16:18.627 | INFO | MemInfo: { | 20260506 09:16:18.627 | INFO | "gpu" : " 17 GB", | 20260506 09:16:18.627 | INFO | "huge" : "", | 20260506 09:16:18.627 | INFO | "huge_shared" : " 27 GB", | 20260506 09:16:18.627 | INFO | "malloc" : " 14 GB", | 20260506 09:16:18.627 | INFO | "pinned" : " 29 GB" | 20260506 09:16:18.627 | INFO | } | 20260506 09:16:18.627 | INFO | Perf report for the last 5.13985 seconds: | 20260506 09:16:18.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:16:18.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:18.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:18.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:18.627 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260506 09:16:18.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:18.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:18.627 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260506 09:16:18.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:18.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:18.627 | INFO | Uploaded 36GiB to GPU in 1.60361s (22.4494GiB/s) | 20260506 09:16:18.627 | INFO | Downloaded 34GiB to GPU in 1.52129s (22.3495GiB/s) | 20260506 09:16:23.766 | INFO | Analyzing chunk frames = [207872,208384) | 20260506 09:16:23.766 | INFO | MemInfo: { | 20260506 09:16:23.766 | INFO | "gpu" : " 17 GB", | 20260506 09:16:23.766 | INFO | "huge" : "", | 20260506 09:16:23.766 | INFO | "huge_shared" : " 27 GB", | 20260506 09:16:23.766 | INFO | "malloc" : " 14 GB", | 20260506 09:16:23.766 | INFO | "pinned" : " 29 GB" | 20260506 09:16:23.766 | INFO | } | 20260506 09:16:23.766 | INFO | Perf report for the last 5.13892 seconds: | 20260506 09:16:23.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:16:23.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:23.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:23.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:23.766 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:16:23.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:23.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:23.766 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:16:23.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:23.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:23.766 | INFO | Uploaded 36GiB to GPU in 1.60377s (22.447GiB/s) | 20260506 09:16:23.766 | INFO | Downloaded 34GiB to GPU in 1.52077s (22.3571GiB/s) | 20260506 09:16:28.905 | INFO | Analyzing chunk frames = [208384,208896) | 20260506 09:16:28.905 | INFO | MemInfo: { | 20260506 09:16:28.905 | INFO | "gpu" : " 17 GB", | 20260506 09:16:28.905 | INFO | "huge" : "", | 20260506 09:16:28.905 | INFO | "huge_shared" : " 27 GB", | 20260506 09:16:28.905 | INFO | "malloc" : " 14 GB", | 20260506 09:16:28.905 | INFO | "pinned" : " 29 GB" | 20260506 09:16:28.905 | INFO | } | 20260506 09:16:28.905 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:16:28.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:16:28.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:28.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:28.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:28.905 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:16:28.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:28.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:28.905 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 09:16:28.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:28.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:28.906 | INFO | Uploaded 36GiB to GPU in 1.60393s (22.4448GiB/s) | 20260506 09:16:28.906 | INFO | Downloaded 34GiB to GPU in 1.521s (22.3537GiB/s) | 20260506 09:16:29.002 | INFO | Starting a round of Baz IO | 20260506 09:16:31.936 | INFO | Populating RTMetrics report | 20260506 09:16:31.936 | INFO | RTMetrics report populated | 20260506 09:16:31.936 | INFO | Opening tmp RTMetrics json file | 20260506 09:16:31.937 | INFO | Serializing RTMetrics to json | 20260506 09:16:31.938 | INFO | Serialization done, swapping tmp file now... | 20260506 09:16:31.939 | INFO | RTMetrics done | 20260506 09:16:32.029 | INFO | Finished a round of Baz IO: Wrote 1.79 GiB in 3.026 seconds | 20260506 09:16:32.029 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.22% / 0.14% / 43.64% | 20260506 09:16:32.029 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.207% / 0.000401338% / 3.17891e-05% | 20260506 09:16:32.029 | INFO | Baz IO overall: | 20260506 09:16:32.029 | INFO | BytesWritten : 72691560563 | 20260506 09:16:32.029 | INFO | NumEvents : 33888109249 | 20260506 09:16:34.045 | INFO | Analyzing chunk frames = [208896,209408) | 20260506 09:16:34.045 | INFO | MemInfo: { | 20260506 09:16:34.045 | INFO | "gpu" : " 17 GB", | 20260506 09:16:34.045 | INFO | "huge" : "", | 20260506 09:16:34.045 | INFO | "huge_shared" : " 27 GB", | 20260506 09:16:34.045 | INFO | "malloc" : " 14 GB", | 20260506 09:16:34.045 | INFO | "pinned" : " 29 GB" | 20260506 09:16:34.045 | INFO | } | 20260506 09:16:34.045 | INFO | Perf report for the last 5.13963 seconds: | 20260506 09:16:34.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:16:34.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:34.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:34.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:34.045 | INFO | ANALYSIS : 58.0%, 2.9, 88.0ms, 0 | 20260506 09:16:34.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:34.045 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 09:16:34.045 | INFO | PRE_HQ : 57.1%, 2.1, 61.8ms, 0 | 20260506 09:16:34.045 | INFO | BAZWRITER : 58.5%, 4.5, 136.5ms, 0 | 20260506 09:16:34.045 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.5ms, 0 | 20260506 09:16:34.045 | INFO | Uploaded 36GiB to GPU in 1.65421s (21.7626GiB/s) | 20260506 09:16:34.046 | INFO | Downloaded 38GiB to GPU in 1.72007s (22.0921GiB/s) | 20260506 09:16:39.184 | INFO | Analyzing chunk frames = [209408,209920) | 20260506 09:16:39.184 | INFO | MemInfo: { | 20260506 09:16:39.184 | INFO | "gpu" : " 17 GB", | 20260506 09:16:39.184 | INFO | "huge" : "", | 20260506 09:16:39.184 | INFO | "huge_shared" : " 27 GB", | 20260506 09:16:39.184 | INFO | "malloc" : " 14 GB", | 20260506 09:16:39.184 | INFO | "pinned" : " 29 GB" | 20260506 09:16:39.184 | INFO | } | 20260506 09:16:39.184 | INFO | Perf report for the last 5.13908 seconds: | 20260506 09:16:39.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:16:39.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:39.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:39.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:39.184 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260506 09:16:39.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:39.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:39.184 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:16:39.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:39.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:39.184 | INFO | Uploaded 36GiB to GPU in 1.60093s (22.4869GiB/s) | 20260506 09:16:39.184 | INFO | Downloaded 34GiB to GPU in 1.51829s (22.3936GiB/s) | 20260506 09:16:44.323 | INFO | Analyzing chunk frames = [209920,210432) | 20260506 09:16:44.323 | INFO | MemInfo: { | 20260506 09:16:44.323 | INFO | "gpu" : " 17 GB", | 20260506 09:16:44.323 | INFO | "huge" : "", | 20260506 09:16:44.323 | INFO | "huge_shared" : " 27 GB", | 20260506 09:16:44.323 | INFO | "malloc" : " 14 GB", | 20260506 09:16:44.323 | INFO | "pinned" : " 29 GB" | 20260506 09:16:44.324 | INFO | } | 20260506 09:16:44.324 | INFO | Perf report for the last 5.1394 seconds: | 20260506 09:16:44.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:16:44.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:44.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:44.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:44.324 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260506 09:16:44.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:44.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:44.324 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:16:44.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:44.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:44.324 | INFO | Uploaded 36GiB to GPU in 1.60479s (22.4329GiB/s) | 20260506 09:16:44.324 | INFO | Downloaded 34GiB to GPU in 1.52224s (22.3356GiB/s) | 20260506 09:16:49.463 | INFO | Analyzing chunk frames = [210432,210944) | 20260506 09:16:49.463 | INFO | MemInfo: { | 20260506 09:16:49.463 | INFO | "gpu" : " 17 GB", | 20260506 09:16:49.466 | INFO | "huge" : "", | 20260506 09:16:49.466 | INFO | "huge_shared" : " 27 GB", | 20260506 09:16:49.466 | INFO | "malloc" : " 14 GB", | 20260506 09:16:49.466 | INFO | "pinned" : " 29 GB" | 20260506 09:16:49.466 | INFO | } | 20260506 09:16:49.466 | INFO | Perf report for the last 5.14293 seconds: | 20260506 09:16:49.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:16:49.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:49.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:49.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:49.467 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 09:16:49.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:49.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:49.467 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:16:49.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:49.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:49.467 | INFO | Uploaded 36GiB to GPU in 1.60452s (22.4367GiB/s) | 20260506 09:16:49.467 | INFO | Downloaded 34GiB to GPU in 1.52204s (22.3384GiB/s) | 20260506 09:16:54.602 | INFO | Analyzing chunk frames = [210944,211456) | 20260506 09:16:54.602 | INFO | MemInfo: { | 20260506 09:16:54.602 | INFO | "gpu" : " 17 GB", | 20260506 09:16:54.606 | INFO | "huge" : "", | 20260506 09:16:54.606 | INFO | "huge_shared" : " 27 GB", | 20260506 09:16:54.606 | INFO | "malloc" : " 14 GB", | 20260506 09:16:54.606 | INFO | "pinned" : " 29 GB" | 20260506 09:16:54.606 | INFO | } | 20260506 09:16:54.606 | INFO | Perf report for the last 5.13918 seconds: | 20260506 09:16:54.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:16:54.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:54.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:54.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:54.606 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 09:16:54.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:54.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:54.606 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:16:54.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:54.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:54.606 | INFO | Uploaded 36GiB to GPU in 1.60258s (22.4638GiB/s) | 20260506 09:16:54.606 | INFO | Downloaded 34GiB to GPU in 1.5202s (22.3655GiB/s) | 20260506 09:16:59.742 | INFO | Analyzing chunk frames = [211456,211968) | 20260506 09:16:59.742 | INFO | MemInfo: { | 20260506 09:16:59.742 | INFO | "gpu" : " 17 GB", | 20260506 09:16:59.742 | INFO | "huge" : "", | 20260506 09:16:59.742 | INFO | "huge_shared" : " 27 GB", | 20260506 09:16:59.742 | INFO | "malloc" : " 14 GB", | 20260506 09:16:59.742 | INFO | "pinned" : " 29 GB" | 20260506 09:16:59.742 | INFO | } | 20260506 09:16:59.742 | INFO | Perf report for the last 5.13599 seconds: | 20260506 09:16:59.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:16:59.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:59.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:59.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:59.742 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:16:59.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:59.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:59.742 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:16:59.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:59.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:16:59.742 | INFO | Uploaded 36GiB to GPU in 1.60324s (22.4545GiB/s) | 20260506 09:16:59.742 | INFO | Downloaded 34GiB to GPU in 1.51969s (22.3729GiB/s) | 20260506 09:17:04.881 | INFO | Analyzing chunk frames = [211968,212480) | 20260506 09:17:04.881 | INFO | MemInfo: { | 20260506 09:17:04.881 | INFO | "gpu" : " 17 GB", | 20260506 09:17:04.881 | INFO | "huge" : "", | 20260506 09:17:04.881 | INFO | "huge_shared" : " 27 GB", | 20260506 09:17:04.881 | INFO | "malloc" : " 14 GB", | 20260506 09:17:04.881 | INFO | "pinned" : " 29 GB" | 20260506 09:17:04.881 | INFO | } | 20260506 09:17:04.881 | INFO | Perf report for the last 5.1392 seconds: | 20260506 09:17:04.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:17:04.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:04.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:04.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:04.881 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 09:17:04.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:04.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:04.881 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:17:04.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:04.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:04.881 | INFO | Uploaded 36GiB to GPU in 1.60319s (22.4552GiB/s) | 20260506 09:17:04.881 | INFO | Downloaded 34GiB to GPU in 1.5207s (22.3582GiB/s) | 20260506 09:17:10.021 | INFO | Analyzing chunk frames = [212480,212992) | 20260506 09:17:10.021 | INFO | MemInfo: { | 20260506 09:17:10.021 | INFO | "gpu" : " 17 GB", | 20260506 09:17:10.022 | INFO | "huge" : "", | 20260506 09:17:10.022 | INFO | "huge_shared" : " 27 GB", | 20260506 09:17:10.022 | INFO | "malloc" : " 14 GB", | 20260506 09:17:10.022 | INFO | "pinned" : " 29 GB" | 20260506 09:17:10.022 | INFO | } | 20260506 09:17:10.022 | INFO | Perf report for the last 5.1407 seconds: | 20260506 09:17:10.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:17:10.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:10.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:10.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:10.022 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:17:10.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:10.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:10.022 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 09:17:10.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:10.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:10.022 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:17:10.022 | INFO | Total mem usage: 15.7 GiB | 20260506 09:17:10.022 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.38%) | 20260506 09:17:10.022 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -6.5%) | 20260506 09:17:10.022 | INFO | Uploaded 36GiB to GPU in 1.60399s (22.444GiB/s) | 20260506 09:17:10.022 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:17:10.022 | INFO | Downloaded 34GiB to GPU in 1.52137s (22.3482GiB/s) | 20260506 09:17:10.022 | INFO | High Water Breakdown: | 20260506 09:17:10.022 | INFO | 58.7% : 8.68 GiB ( 92.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:17:10.022 | INFO | 41.1% : 6.08 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:17:10.022 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:17:10.022 | INFO | ------------------------------------------ | 20260506 09:17:10.022 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:17:10.022 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:17:10.022 | INFO | Total mem usage: 26.5 GiB | 20260506 09:17:10.022 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:17:10.022 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:17:10.022 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:17:10.022 | INFO | High Water Breakdown: | 20260506 09:17:10.022 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:17:10.022 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:17:10.022 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:17:10.022 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:17:10.022 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:17:10.022 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:17:10.022 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:17:10.022 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:17:10.022 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:17:10.022 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:17:10.022 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:17:10.022 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:17:10.022 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:17:10.022 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:17:10.022 | INFO | ------------------------------------------ | 20260506 09:17:10.022 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:17:10.022 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:17:10.022 | INFO | Total mem usage: 25.6 GiB | 20260506 09:17:10.022 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:17:10.022 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:17:10.022 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:17:10.022 | INFO | High Water Breakdown: | 20260506 09:17:10.022 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:17:10.022 | INFO | ------------------------------------------ | 20260506 09:17:10.022 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:17:10.022 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:17:10.022 | INFO | Total mem usage: 15.7 GiB | 20260506 09:17:10.022 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:17:10.022 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:17:10.022 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:17:10.022 | INFO | High Water Breakdown: | 20260506 09:17:10.022 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:17:10.022 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:17:10.022 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:17:10.022 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:17:10.022 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:17:10.022 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:17:10.022 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:17:10.022 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:17:10.022 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:17:10.022 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:17:10.022 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:17:10.022 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:17:10.022 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:17:10.023 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:17:10.023 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:17:10.023 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:17:10.023 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:17:10.023 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:17:10.023 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:17:10.023 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:17:10.023 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:17:10.023 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:17:10.023 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:17:10.023 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:17:10.023 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:17:10.023 | INFO | ------------------------------------------ | 20260506 09:17:10.023 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:17:10.023 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:17:10.023 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:17:10.023 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:17:10.023 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 09:17:10.023 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / 0.00e+00, 4.61e-03 / 0.00e+00 | 20260506 09:17:10.023 | INFO | PerfReport: 10.87%, Baseline -- 3.29e+00 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260506 09:17:10.023 | INFO | PerfReport: 3.04%, Histogram -- 9.22e-01 / 0.00e+00, 5.20e-02 / 0.00e+00 | 20260506 09:17:10.023 | INFO | PerfReport: 16.03%, DME -- 3.89e+01 / 0.00e+00, 3.15e+00 / 0.00e+00 | 20260506 09:17:10.023 | INFO | PerfReport: 56.78%, FrameLabeling -- 1.72e+01 / 0.00e+00, 8.54e-02 / 0.00e+00 | 20260506 09:17:10.023 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.29e+00 / 0.00e+00, 1.39e-01 / 0.00e+00 | 20260506 09:17:10.023 | INFO | PerfReport: 3.89%, Metrics -- 1.18e+00 / 0.00e+00, 1.84e-01 / 0.00e+00 | 20260506 09:17:10.023 | INFO | PerfReport: 1.52%, PulseSerializing -- 4.61e-01 / 0.00e+00, 6.73e-02 / 0.00e+00 | 20260506 09:17:10.023 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:17:10.023 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:17:10.023 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:17:10.023 | INFO | PerfReport: Full Execution -- 9.08e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260506 09:17:10.023 | INFO | PerfReport: 18.45%, Upload -- 1.68e+01 / 0.00e+00, 4.13e-01 / 0.00e+00 | 20260506 09:17:10.023 | INFO | PerfReport: 17.71%, Download -- 1.61e+01 / 0.00e+00, 7.35e-01 / 0.00e+00 | 20260506 09:17:10.023 | INFO | PerfReport: 33.36%, Compute -- 3.03e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 09:17:10.023 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:17:10.023 | INFO | PerfReport: 0.00%, LockDownload -- 4.92e-04 / 0.00e+00, 2.08e-04 / 0.00e+00 | 20260506 09:17:10.023 | INFO | PerfReport: 29.90%, LockCompute -- 2.72e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 09:17:10.023 | INFO | PerfReport: 0.02%, Misc -- 1.40e-02 / 0.00e+00, 2.32e-02 / 0.00e+00 | 20260506 09:17:10.119 | INFO | Starting a round of Baz IO | 20260506 09:17:13.048 | INFO | Populating RTMetrics report | 20260506 09:17:13.048 | INFO | RTMetrics report populated | 20260506 09:17:13.048 | INFO | Opening tmp RTMetrics json file | 20260506 09:17:13.244 | INFO | Serializing RTMetrics to json | 20260506 09:17:13.375 | INFO | Serialization done, swapping tmp file now... | 20260506 09:17:13.376 | INFO | RTMetrics done | 20260506 09:17:13.403 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.284 seconds | 20260506 09:17:13.403 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 56.09% / 0.14% / 43.78% | 20260506 09:17:13.403 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.2047% / 0.000433127% / 3.57628e-05% | 20260506 09:17:13.403 | INFO | Baz IO overall: | 20260506 09:17:13.403 | INFO | BytesWritten : 74621649345 | 20260506 09:17:13.403 | INFO | NumEvents : 34826394185 | 20260506 09:17:15.160 | INFO | Analyzing chunk frames = [212992,213504) | 20260506 09:17:15.160 | INFO | MemInfo: { | 20260506 09:17:15.160 | INFO | "gpu" : " 17 GB", | 20260506 09:17:15.160 | INFO | "huge" : "", | 20260506 09:17:15.160 | INFO | "huge_shared" : " 27 GB", | 20260506 09:17:15.160 | INFO | "malloc" : " 14 GB", | 20260506 09:17:15.160 | INFO | "pinned" : " 29 GB" | 20260506 09:17:15.160 | INFO | } | 20260506 09:17:15.160 | INFO | Perf report for the last 5.13817 seconds: | 20260506 09:17:15.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:17:15.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:15.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:15.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:15.160 | INFO | ANALYSIS : 57.9%, 2.9, 88.5ms, 0 | 20260506 09:17:15.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:15.160 | INFO | RT_METRICS : 50.1%, 1.0, 26.3ms, 0 | 20260506 09:17:15.160 | INFO | PRE_HQ : 57.0%, 2.1, 61.7ms, 0 | 20260506 09:17:15.160 | INFO | BAZWRITER : 63.3%, 4.4, 144.0ms, 0 | 20260506 09:17:15.160 | INFO | BAZ_SAMPLING : 3.4%, 1.0, 1.8ms, 0 | 20260506 09:17:15.160 | INFO | Uploaded 36GiB to GPU in 1.65102s (21.8047GiB/s) | 20260506 09:17:15.160 | INFO | Downloaded 38GiB to GPU in 1.71429s (22.1666GiB/s) | 20260506 09:17:20.299 | INFO | Analyzing chunk frames = [213504,214016) | 20260506 09:17:20.299 | INFO | MemInfo: { | 20260506 09:17:20.299 | INFO | "gpu" : " 17 GB", | 20260506 09:17:20.299 | INFO | "huge" : "", | 20260506 09:17:20.299 | INFO | "huge_shared" : " 27 GB", | 20260506 09:17:20.299 | INFO | "malloc" : " 14 GB", | 20260506 09:17:20.299 | INFO | "pinned" : " 29 GB" | 20260506 09:17:20.299 | INFO | } | 20260506 09:17:20.299 | INFO | Perf report for the last 5.1391 seconds: | 20260506 09:17:20.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:17:20.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:20.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:20.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:20.299 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260506 09:17:20.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:20.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:20.299 | INFO | PRE_HQ : 30.0%, 1.0, 15.7ms, 0 | 20260506 09:17:20.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:20.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:20.299 | INFO | Uploaded 36GiB to GPU in 1.60123s (22.4828GiB/s) | 20260506 09:17:20.299 | INFO | Downloaded 34GiB to GPU in 1.51891s (22.3844GiB/s) | 20260506 09:17:25.438 | INFO | Analyzing chunk frames = [214016,214528) | 20260506 09:17:25.438 | INFO | MemInfo: { | 20260506 09:17:25.438 | INFO | "gpu" : " 17 GB", | 20260506 09:17:25.438 | INFO | "huge" : "", | 20260506 09:17:25.438 | INFO | "huge_shared" : " 27 GB", | 20260506 09:17:25.438 | INFO | "malloc" : " 14 GB", | 20260506 09:17:25.438 | INFO | "pinned" : " 29 GB" | 20260506 09:17:25.438 | INFO | } | 20260506 09:17:25.438 | INFO | Perf report for the last 5.1394 seconds: | 20260506 09:17:25.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:17:25.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:25.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:25.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:25.438 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260506 09:17:25.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:25.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:25.438 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:17:25.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:25.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:25.439 | INFO | Uploaded 36GiB to GPU in 1.60489s (22.4314GiB/s) | 20260506 09:17:25.439 | INFO | Downloaded 34GiB to GPU in 1.52187s (22.341GiB/s) | 20260506 09:17:30.578 | INFO | Analyzing chunk frames = [214528,215040) | 20260506 09:17:30.578 | INFO | MemInfo: { | 20260506 09:17:30.578 | INFO | "gpu" : " 17 GB", | 20260506 09:17:30.578 | INFO | "huge" : "", | 20260506 09:17:30.578 | INFO | "huge_shared" : " 27 GB", | 20260506 09:17:30.578 | INFO | "malloc" : " 14 GB", | 20260506 09:17:30.578 | INFO | "pinned" : " 29 GB" | 20260506 09:17:30.579 | INFO | } | 20260506 09:17:30.579 | INFO | Perf report for the last 5.14071 seconds: | 20260506 09:17:30.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:17:30.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:30.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:30.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:30.579 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260506 09:17:30.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:30.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:30.579 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:17:30.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:30.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:30.579 | INFO | Uploaded 36GiB to GPU in 1.60437s (22.4387GiB/s) | 20260506 09:17:30.579 | INFO | Downloaded 34GiB to GPU in 1.5222s (22.3361GiB/s) | 20260506 09:17:35.717 | INFO | Analyzing chunk frames = [215040,215552) | 20260506 09:17:35.717 | INFO | MemInfo: { | 20260506 09:17:35.717 | INFO | "gpu" : " 17 GB", | 20260506 09:17:35.717 | INFO | "huge" : "", | 20260506 09:17:35.717 | INFO | "huge_shared" : " 27 GB", | 20260506 09:17:35.717 | INFO | "malloc" : " 14 GB", | 20260506 09:17:35.717 | INFO | "pinned" : " 29 GB" | 20260506 09:17:35.717 | INFO | } | 20260506 09:17:35.717 | INFO | Perf report for the last 5.13814 seconds: | 20260506 09:17:35.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:17:35.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:35.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:35.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:35.717 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 09:17:35.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:35.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:35.717 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:17:35.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:35.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:35.718 | INFO | Uploaded 36GiB to GPU in 1.60234s (22.4671GiB/s) | 20260506 09:17:35.718 | INFO | Downloaded 34GiB to GPU in 1.51944s (22.3767GiB/s) | 20260506 09:17:40.856 | INFO | Analyzing chunk frames = [215552,216064) | 20260506 09:17:40.856 | INFO | MemInfo: { | 20260506 09:17:40.856 | INFO | "gpu" : " 17 GB", | 20260506 09:17:40.856 | INFO | "huge" : "", | 20260506 09:17:40.856 | INFO | "huge_shared" : " 27 GB", | 20260506 09:17:40.856 | INFO | "malloc" : " 14 GB", | 20260506 09:17:40.856 | INFO | "pinned" : " 29 GB" | 20260506 09:17:40.856 | INFO | } | 20260506 09:17:40.856 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:17:40.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:17:40.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:40.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:40.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:40.857 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:17:40.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:40.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:40.857 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:17:40.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:40.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:40.857 | INFO | Uploaded 36GiB to GPU in 1.60318s (22.4554GiB/s) | 20260506 09:17:40.857 | INFO | Downloaded 34GiB to GPU in 1.52001s (22.3682GiB/s) | 20260506 09:17:45.996 | INFO | Analyzing chunk frames = [216064,216576) | 20260506 09:17:45.996 | INFO | MemInfo: { | 20260506 09:17:45.996 | INFO | "gpu" : " 17 GB", | 20260506 09:17:45.996 | INFO | "huge" : "", | 20260506 09:17:45.996 | INFO | "huge_shared" : " 27 GB", | 20260506 09:17:45.996 | INFO | "malloc" : " 14 GB", | 20260506 09:17:45.996 | INFO | "pinned" : " 29 GB" | 20260506 09:17:45.996 | INFO | } | 20260506 09:17:45.996 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:17:45.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:17:45.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:45.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:45.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:45.996 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 09:17:45.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:45.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:45.996 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:17:45.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:45.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:45.996 | INFO | Uploaded 36GiB to GPU in 1.60356s (22.4501GiB/s) | 20260506 09:17:45.996 | INFO | Downloaded 34GiB to GPU in 1.52057s (22.36GiB/s) | 20260506 09:17:51.135 | INFO | Analyzing chunk frames = [216576,217088) | 20260506 09:17:51.135 | INFO | MemInfo: { | 20260506 09:17:51.135 | INFO | "gpu" : " 17 GB", | 20260506 09:17:51.135 | INFO | "huge" : "", | 20260506 09:17:51.135 | INFO | "huge_shared" : " 27 GB", | 20260506 09:17:51.135 | INFO | "malloc" : " 14 GB", | 20260506 09:17:51.135 | INFO | "pinned" : " 29 GB" | 20260506 09:17:51.135 | INFO | } | 20260506 09:17:51.135 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:17:51.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:17:51.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:51.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:51.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:51.136 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 09:17:51.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:51.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:51.136 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:17:51.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:51.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:51.136 | INFO | Uploaded 36GiB to GPU in 1.60408s (22.4427GiB/s) | 20260506 09:17:51.136 | INFO | Downloaded 34GiB to GPU in 1.52109s (22.3524GiB/s) | 20260506 09:17:51.232 | INFO | Starting a round of Baz IO | 20260506 09:17:54.163 | INFO | Populating RTMetrics report | 20260506 09:17:54.163 | INFO | RTMetrics report populated | 20260506 09:17:54.163 | INFO | Opening tmp RTMetrics json file | 20260506 09:17:54.166 | INFO | Serializing RTMetrics to json | 20260506 09:17:54.172 | INFO | Serialization done, swapping tmp file now... | 20260506 09:17:54.183 | INFO | RTMetrics done | 20260506 09:17:54.261 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.028 seconds | 20260506 09:17:54.261 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.95% / 0.13% / 43.91% | 20260506 09:17:54.261 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.1375% / 0.000445048% / 3.97364e-05% | 20260506 09:17:54.261 | INFO | Baz IO overall: | 20260506 09:17:54.261 | INFO | BytesWritten : 76553501828 | 20260506 09:17:54.261 | INFO | NumEvents : 35765117778 | 20260506 09:17:56.274 | INFO | Analyzing chunk frames = [217088,217600) | 20260506 09:17:56.274 | INFO | MemInfo: { | 20260506 09:17:56.274 | INFO | "gpu" : " 17 GB", | 20260506 09:17:56.274 | INFO | "huge" : "", | 20260506 09:17:56.274 | INFO | "huge_shared" : " 27 GB", | 20260506 09:17:56.274 | INFO | "malloc" : " 14 GB", | 20260506 09:17:56.274 | INFO | "pinned" : " 29 GB" | 20260506 09:17:56.275 | INFO | } | 20260506 09:17:56.275 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:17:56.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:17:56.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:56.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:56.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:56.275 | INFO | ANALYSIS : 57.9%, 2.9, 88.0ms, 0 | 20260506 09:17:56.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:17:56.275 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 09:17:56.275 | INFO | PRE_HQ : 57.0%, 2.1, 61.6ms, 0 | 20260506 09:17:56.275 | INFO | BAZWRITER : 58.6%, 4.5, 136.0ms, 0 | 20260506 09:17:56.275 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 09:17:56.275 | INFO | Uploaded 36GiB to GPU in 1.6519s (21.7931GiB/s) | 20260506 09:17:56.275 | INFO | Downloaded 38GiB to GPU in 1.71512s (22.1558GiB/s) | 20260506 09:18:01.414 | INFO | Analyzing chunk frames = [217600,218112) | 20260506 09:18:01.414 | INFO | MemInfo: { | 20260506 09:18:01.414 | INFO | "gpu" : " 17 GB", | 20260506 09:18:01.414 | INFO | "huge" : "", | 20260506 09:18:01.414 | INFO | "huge_shared" : " 27 GB", | 20260506 09:18:01.414 | INFO | "malloc" : " 14 GB", | 20260506 09:18:01.414 | INFO | "pinned" : " 29 GB" | 20260506 09:18:01.414 | INFO | } | 20260506 09:18:01.414 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:18:01.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:18:01.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:01.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:01.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:01.414 | INFO | ANALYSIS : 57.7%, 3.0, 89.9ms, 0 | 20260506 09:18:01.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:01.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:01.414 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:18:01.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:01.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:01.414 | INFO | Uploaded 36GiB to GPU in 1.60147s (22.4794GiB/s) | 20260506 09:18:01.414 | INFO | Downloaded 34GiB to GPU in 1.51952s (22.3754GiB/s) | 20260506 09:18:06.553 | INFO | Analyzing chunk frames = [218112,218624) | 20260506 09:18:06.553 | INFO | MemInfo: { | 20260506 09:18:06.553 | INFO | "gpu" : " 17 GB", | 20260506 09:18:06.553 | INFO | "huge" : "", | 20260506 09:18:06.553 | INFO | "huge_shared" : " 27 GB", | 20260506 09:18:06.553 | INFO | "malloc" : " 14 GB", | 20260506 09:18:06.553 | INFO | "pinned" : " 29 GB" | 20260506 09:18:06.553 | INFO | } | 20260506 09:18:06.553 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:18:06.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:18:06.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:06.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:06.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:06.553 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260506 09:18:06.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:06.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:06.553 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:18:06.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:06.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:06.554 | INFO | Uploaded 36GiB to GPU in 1.60439s (22.4385GiB/s) | 20260506 09:18:06.554 | INFO | Downloaded 34GiB to GPU in 1.5223s (22.3346GiB/s) | 20260506 09:18:11.692 | INFO | Analyzing chunk frames = [218624,219136) | 20260506 09:18:11.692 | INFO | MemInfo: { | 20260506 09:18:11.692 | INFO | "gpu" : " 17 GB", | 20260506 09:18:11.692 | INFO | "huge" : "", | 20260506 09:18:11.692 | INFO | "huge_shared" : " 27 GB", | 20260506 09:18:11.692 | INFO | "malloc" : " 14 GB", | 20260506 09:18:11.692 | INFO | "pinned" : " 29 GB" | 20260506 09:18:11.692 | INFO | } | 20260506 09:18:11.693 | INFO | Perf report for the last 5.13923 seconds: | 20260506 09:18:11.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:18:11.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:11.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:11.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:11.693 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260506 09:18:11.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:11.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:11.693 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:18:11.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:11.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:11.693 | INFO | Uploaded 36GiB to GPU in 1.60374s (22.4475GiB/s) | 20260506 09:18:11.693 | INFO | Downloaded 34GiB to GPU in 1.52222s (22.3358GiB/s) | 20260506 09:18:16.832 | INFO | Analyzing chunk frames = [219136,219648) | 20260506 09:18:16.832 | INFO | MemInfo: { | 20260506 09:18:16.832 | INFO | "gpu" : " 17 GB", | 20260506 09:18:16.832 | INFO | "huge" : "", | 20260506 09:18:16.832 | INFO | "huge_shared" : " 27 GB", | 20260506 09:18:16.832 | INFO | "malloc" : " 14 GB", | 20260506 09:18:16.832 | INFO | "pinned" : " 29 GB" | 20260506 09:18:16.832 | INFO | } | 20260506 09:18:16.832 | INFO | Perf report for the last 5.13932 seconds: | 20260506 09:18:16.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:18:16.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:16.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:16.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:16.832 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 09:18:16.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:16.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:16.832 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:18:16.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:16.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:16.832 | INFO | Uploaded 36GiB to GPU in 1.60235s (22.467GiB/s) | 20260506 09:18:16.832 | INFO | Downloaded 34GiB to GPU in 1.52031s (22.3639GiB/s) | 20260506 09:18:21.971 | INFO | Analyzing chunk frames = [219648,220160) | 20260506 09:18:21.971 | INFO | MemInfo: { | 20260506 09:18:21.971 | INFO | "gpu" : " 17 GB", | 20260506 09:18:21.971 | INFO | "huge" : "", | 20260506 09:18:21.971 | INFO | "huge_shared" : " 27 GB", | 20260506 09:18:21.971 | INFO | "malloc" : " 14 GB", | 20260506 09:18:21.971 | INFO | "pinned" : " 29 GB" | 20260506 09:18:21.971 | INFO | } | 20260506 09:18:21.971 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:18:21.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:18:21.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:21.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:21.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:21.971 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:18:21.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:21.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:21.971 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:18:21.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:21.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:21.971 | INFO | Uploaded 36GiB to GPU in 1.60315s (22.4557GiB/s) | 20260506 09:18:21.971 | INFO | Downloaded 34GiB to GPU in 1.51938s (22.3776GiB/s) | 20260506 09:18:27.110 | INFO | Analyzing chunk frames = [220160,220672) | 20260506 09:18:27.110 | INFO | MemInfo: { | 20260506 09:18:27.110 | INFO | "gpu" : " 17 GB", | 20260506 09:18:27.110 | INFO | "huge" : "", | 20260506 09:18:27.110 | INFO | "huge_shared" : " 27 GB", | 20260506 09:18:27.110 | INFO | "malloc" : " 14 GB", | 20260506 09:18:27.110 | INFO | "pinned" : " 29 GB" | 20260506 09:18:27.110 | INFO | } | 20260506 09:18:27.110 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:18:27.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:18:27.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:27.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:27.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:27.110 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:18:27.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:27.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:27.111 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:18:27.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:27.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:27.111 | INFO | Uploaded 36GiB to GPU in 1.60379s (22.4469GiB/s) | 20260506 09:18:27.111 | INFO | Downloaded 34GiB to GPU in 1.52157s (22.3453GiB/s) | 20260506 09:18:32.250 | INFO | Analyzing chunk frames = [220672,221184) | 20260506 09:18:32.250 | INFO | MemInfo: { | 20260506 09:18:32.250 | INFO | "gpu" : " 17 GB", | 20260506 09:18:32.250 | INFO | "huge" : "", | 20260506 09:18:32.250 | INFO | "huge_shared" : " 27 GB", | 20260506 09:18:32.250 | INFO | "malloc" : " 14 GB", | 20260506 09:18:32.250 | INFO | "pinned" : " 29 GB" | 20260506 09:18:32.250 | INFO | } | 20260506 09:18:32.250 | INFO | Perf report for the last 5.13938 seconds: | 20260506 09:18:32.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:18:32.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:32.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:32.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:32.250 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:18:32.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:32.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:32.250 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:18:32.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:32.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:32.250 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:18:32.250 | INFO | Total mem usage: 15.7 GiB | 20260506 09:18:32.250 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.38%) | 20260506 09:18:32.250 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -4.28%) | 20260506 09:18:32.250 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:18:32.250 | INFO | High Water Breakdown: | 20260506 09:18:32.250 | INFO | 59.1% : 8.93 GiB ( 94.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:18:32.250 | INFO | 40.8% : 6.16 GiB ( 92% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:18:32.250 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:18:32.250 | INFO | ------------------------------------------ | 20260506 09:18:32.250 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:18:32.250 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:18:32.250 | INFO | Total mem usage: 26.5 GiB | 20260506 09:18:32.250 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:18:32.250 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:18:32.250 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:18:32.250 | INFO | High Water Breakdown: | 20260506 09:18:32.250 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:18:32.250 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:18:32.250 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:18:32.250 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:18:32.250 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:18:32.250 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:18:32.250 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:18:32.250 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:18:32.250 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:18:32.250 | INFO | Uploaded 36GiB to GPU in 1.60339s (22.4525GiB/s) | 20260506 09:18:32.250 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:18:32.250 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:18:32.250 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:18:32.250 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:18:32.250 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:18:32.250 | INFO | Downloaded 34GiB to GPU in 1.52155s (22.3456GiB/s) | 20260506 09:18:32.250 | INFO | ------------------------------------------ | 20260506 09:18:32.250 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:18:32.250 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:18:32.250 | INFO | Total mem usage: 25.6 GiB | 20260506 09:18:32.251 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:18:32.251 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:18:32.251 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:18:32.251 | INFO | High Water Breakdown: | 20260506 09:18:32.251 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:18:32.251 | INFO | ------------------------------------------ | 20260506 09:18:32.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:18:32.251 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:18:32.251 | INFO | Total mem usage: 15.7 GiB | 20260506 09:18:32.251 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:18:32.251 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:18:32.251 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:18:32.251 | INFO | High Water Breakdown: | 20260506 09:18:32.251 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:18:32.251 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:18:32.251 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:18:32.251 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:18:32.251 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:18:32.251 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:18:32.251 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:18:32.251 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:18:32.251 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:18:32.251 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:18:32.251 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:18:32.251 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:18:32.251 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:18:32.251 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:18:32.251 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:18:32.251 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:18:32.251 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:18:32.251 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:18:32.251 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:18:32.251 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:18:32.251 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:18:32.251 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:18:32.251 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:18:32.251 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:18:32.251 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:18:32.251 | INFO | ------------------------------------------ | 20260506 09:18:32.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:18:32.251 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:18:32.251 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:18:32.251 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:18:32.251 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 09:18:32.251 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / 0.00e+00, 4.68e-03 / 0.00e+00 | 20260506 09:18:32.251 | INFO | PerfReport: 10.88%, Baseline -- 3.29e+00 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260506 09:18:32.251 | INFO | PerfReport: 3.04%, Histogram -- 9.22e-01 / 0.00e+00, 5.20e-02 / 0.00e+00 | 20260506 09:18:32.251 | INFO | PerfReport: 16.03%, DME -- 3.88e+01 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260506 09:18:32.251 | INFO | PerfReport: 56.80%, FrameLabeling -- 1.72e+01 / 0.00e+00, 4.27e-02 / 0.00e+00 | 20260506 09:18:32.251 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.29e+00 / 0.00e+00, 1.38e-01 / 0.00e+00 | 20260506 09:18:32.251 | INFO | PerfReport: 3.88%, Metrics -- 1.18e+00 / 0.00e+00, 1.83e-01 / 0.00e+00 | 20260506 09:18:32.251 | INFO | PerfReport: 1.52%, PulseSerializing -- 4.59e-01 / 0.00e+00, 6.70e-02 / 0.00e+00 | 20260506 09:18:32.251 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:18:32.251 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:18:32.251 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:18:32.251 | INFO | PerfReport: Full Execution -- 9.07e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260506 09:18:32.251 | INFO | PerfReport: 18.48%, Upload -- 1.68e+01 / 0.00e+00, 4.22e-01 / 0.00e+00 | 20260506 09:18:32.251 | INFO | PerfReport: 17.74%, Download -- 1.61e+01 / 0.00e+00, 7.50e-01 / 0.00e+00 | 20260506 09:18:32.251 | INFO | PerfReport: 33.39%, Compute -- 3.03e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 09:18:32.251 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260506 09:18:32.251 | INFO | PerfReport: 0.00%, LockDownload -- 4.88e-04 / 0.00e+00, 1.88e-04 / 0.00e+00 | 20260506 09:18:32.251 | INFO | PerfReport: 29.81%, LockCompute -- 2.70e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 09:18:32.251 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.41e-02 / 0.00e+00 | 20260506 09:18:32.347 | INFO | Starting a round of Baz IO | 20260506 09:18:35.278 | INFO | Populating RTMetrics report | 20260506 09:18:35.279 | INFO | RTMetrics report populated | 20260506 09:18:35.279 | INFO | Opening tmp RTMetrics json file | 20260506 09:18:35.294 | INFO | Serializing RTMetrics to json | 20260506 09:18:35.316 | INFO | Serialization done, swapping tmp file now... | 20260506 09:18:35.324 | INFO | RTMetrics done | 20260506 09:18:35.428 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.08 seconds | 20260506 09:18:35.428 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.82% / 0.13% / 44.05% | 20260506 09:18:35.428 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.0594% / 0.000421206% / 4.76837e-05% | 20260506 09:18:35.428 | INFO | Baz IO overall: | 20260506 09:18:35.428 | INFO | BytesWritten : 78485918524 | 20260506 09:18:35.428 | INFO | NumEvents : 36703501294 | 20260506 09:18:37.389 | INFO | Analyzing chunk frames = [221184,221696) | 20260506 09:18:37.389 | INFO | MemInfo: { | 20260506 09:18:37.389 | INFO | "gpu" : " 17 GB", | 20260506 09:18:37.389 | INFO | "huge" : "", | 20260506 09:18:37.389 | INFO | "huge_shared" : " 27 GB", | 20260506 09:18:37.389 | INFO | "malloc" : " 14 GB", | 20260506 09:18:37.389 | INFO | "pinned" : " 29 GB" | 20260506 09:18:37.389 | INFO | } | 20260506 09:18:37.389 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:18:37.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:18:37.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:37.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:37.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:37.389 | INFO | ANALYSIS : 58.0%, 2.9, 88.9ms, 0 | 20260506 09:18:37.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:37.389 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 09:18:37.389 | INFO | PRE_HQ : 56.9%, 2.1, 61.8ms, 0 | 20260506 09:18:37.389 | INFO | BAZWRITER : 59.5%, 4.3, 132.9ms, 0 | 20260506 09:18:37.389 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260506 09:18:37.390 | INFO | Uploaded 36GiB to GPU in 1.64627s (21.8676GiB/s) | 20260506 09:18:37.390 | INFO | Downloaded 38GiB to GPU in 1.71464s (22.1621GiB/s) | 20260506 09:18:42.528 | INFO | Analyzing chunk frames = [221696,222208) | 20260506 09:18:42.528 | INFO | MemInfo: { | 20260506 09:18:42.528 | INFO | "gpu" : " 17 GB", | 20260506 09:18:42.528 | INFO | "huge" : "", | 20260506 09:18:42.528 | INFO | "huge_shared" : " 27 GB", | 20260506 09:18:42.528 | INFO | "malloc" : " 14 GB", | 20260506 09:18:42.528 | INFO | "pinned" : " 29 GB" | 20260506 09:18:42.528 | INFO | } | 20260506 09:18:42.529 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:18:42.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:18:42.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:42.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:42.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:42.529 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260506 09:18:42.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:42.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:42.529 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:18:42.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:42.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:42.529 | INFO | Uploaded 36GiB to GPU in 1.60072s (22.4899GiB/s) | 20260506 09:18:42.529 | INFO | Downloaded 34GiB to GPU in 1.51862s (22.3887GiB/s) | 20260506 09:18:47.668 | INFO | Analyzing chunk frames = [222208,222720) | 20260506 09:18:47.668 | INFO | MemInfo: { | 20260506 09:18:47.668 | INFO | "gpu" : " 17 GB", | 20260506 09:18:47.668 | INFO | "huge" : "", | 20260506 09:18:47.668 | INFO | "huge_shared" : " 27 GB", | 20260506 09:18:47.668 | INFO | "malloc" : " 14 GB", | 20260506 09:18:47.668 | INFO | "pinned" : " 29 GB" | 20260506 09:18:47.668 | INFO | } | 20260506 09:18:47.668 | INFO | Perf report for the last 5.13938 seconds: | 20260506 09:18:47.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:18:47.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:47.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:47.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:47.668 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260506 09:18:47.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:47.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:47.668 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:18:47.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:47.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:47.668 | INFO | Uploaded 36GiB to GPU in 1.60576s (22.4193GiB/s) | 20260506 09:18:47.668 | INFO | Downloaded 34GiB to GPU in 1.52463s (22.3004GiB/s) | 20260506 09:18:52.807 | INFO | Analyzing chunk frames = [222720,223232) | 20260506 09:18:52.807 | INFO | MemInfo: { | 20260506 09:18:52.807 | INFO | "gpu" : " 17 GB", | 20260506 09:18:52.807 | INFO | "huge" : "", | 20260506 09:18:52.807 | INFO | "huge_shared" : " 27 GB", | 20260506 09:18:52.807 | INFO | "malloc" : " 14 GB", | 20260506 09:18:52.807 | INFO | "pinned" : " 29 GB" | 20260506 09:18:52.807 | INFO | } | 20260506 09:18:52.807 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:18:52.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:18:52.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:52.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:52.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:52.807 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260506 09:18:52.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:52.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:52.807 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:18:52.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:52.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:52.808 | INFO | Uploaded 36GiB to GPU in 1.60335s (22.4529GiB/s) | 20260506 09:18:52.808 | INFO | Downloaded 34GiB to GPU in 1.52156s (22.3455GiB/s) | 20260506 09:18:57.946 | INFO | Analyzing chunk frames = [223232,223744) | 20260506 09:18:57.946 | INFO | MemInfo: { | 20260506 09:18:57.946 | INFO | "gpu" : " 17 GB", | 20260506 09:18:57.946 | INFO | "huge" : "", | 20260506 09:18:57.946 | INFO | "huge_shared" : " 27 GB", | 20260506 09:18:57.946 | INFO | "malloc" : " 14 GB", | 20260506 09:18:57.946 | INFO | "pinned" : " 29 GB" | 20260506 09:18:57.946 | INFO | } | 20260506 09:18:57.947 | INFO | Perf report for the last 5.13932 seconds: | 20260506 09:18:57.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:18:57.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:57.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:57.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:57.947 | INFO | ANALYSIS : 56.6%, 3.0, 88.4ms, 0 | 20260506 09:18:57.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:57.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:57.947 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:18:57.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:57.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:18:57.947 | INFO | Uploaded 36GiB to GPU in 1.60273s (22.4617GiB/s) | 20260506 09:18:57.947 | INFO | Downloaded 34GiB to GPU in 1.52032s (22.3637GiB/s) | 20260506 09:19:03.086 | INFO | Analyzing chunk frames = [223744,224256) | 20260506 09:19:03.086 | INFO | MemInfo: { | 20260506 09:19:03.086 | INFO | "gpu" : " 17 GB", | 20260506 09:19:03.086 | INFO | "huge" : "", | 20260506 09:19:03.086 | INFO | "huge_shared" : " 27 GB", | 20260506 09:19:03.086 | INFO | "malloc" : " 14 GB", | 20260506 09:19:03.086 | INFO | "pinned" : " 29 GB" | 20260506 09:19:03.086 | INFO | } | 20260506 09:19:03.086 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:19:03.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:19:03.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:03.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:03.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:03.086 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:19:03.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:03.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:03.086 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:19:03.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:03.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:03.086 | INFO | Uploaded 36GiB to GPU in 1.60385s (22.446GiB/s) | 20260506 09:19:03.086 | INFO | Downloaded 34GiB to GPU in 1.52052s (22.3607GiB/s) | 20260506 09:19:08.225 | INFO | Analyzing chunk frames = [224256,224768) | 20260506 09:19:08.225 | INFO | MemInfo: { | 20260506 09:19:08.225 | INFO | "gpu" : " 17 GB", | 20260506 09:19:08.225 | INFO | "huge" : "", | 20260506 09:19:08.225 | INFO | "huge_shared" : " 27 GB", | 20260506 09:19:08.225 | INFO | "malloc" : " 14 GB", | 20260506 09:19:08.225 | INFO | "pinned" : " 29 GB" | 20260506 09:19:08.225 | INFO | } | 20260506 09:19:08.226 | INFO | Perf report for the last 5.13966 seconds: | 20260506 09:19:08.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:19:08.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:08.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:08.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:08.226 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:19:08.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:08.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:08.226 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:19:08.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:08.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:08.226 | INFO | Uploaded 36GiB to GPU in 1.60271s (22.4619GiB/s) | 20260506 09:19:08.226 | INFO | Downloaded 34GiB to GPU in 1.52119s (22.351GiB/s) | 20260506 09:19:13.364 | INFO | Analyzing chunk frames = [224768,225280) | 20260506 09:19:13.364 | INFO | MemInfo: { | 20260506 09:19:13.364 | INFO | "gpu" : " 17 GB", | 20260506 09:19:13.364 | INFO | "huge" : "", | 20260506 09:19:13.364 | INFO | "huge_shared" : " 27 GB", | 20260506 09:19:13.364 | INFO | "malloc" : " 14 GB", | 20260506 09:19:13.365 | INFO | "pinned" : " 29 GB" | 20260506 09:19:13.365 | INFO | } | 20260506 09:19:13.365 | INFO | Perf report for the last 5.13902 seconds: | 20260506 09:19:13.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:19:13.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:13.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:13.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:13.365 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:19:13.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:13.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:13.365 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 09:19:13.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:13.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:13.365 | INFO | Uploaded 36GiB to GPU in 1.60323s (22.4546GiB/s) | 20260506 09:19:13.365 | INFO | Downloaded 34GiB to GPU in 1.52122s (22.3504GiB/s) | 20260506 09:19:13.465 | INFO | Starting a round of Baz IO | 20260506 09:19:16.390 | INFO | Populating RTMetrics report | 20260506 09:19:16.390 | INFO | RTMetrics report populated | 20260506 09:19:16.390 | INFO | Opening tmp RTMetrics json file | 20260506 09:19:16.391 | INFO | Serializing RTMetrics to json | 20260506 09:19:16.392 | INFO | Serialization done, swapping tmp file now... | 20260506 09:19:16.393 | INFO | RTMetrics done | 20260506 09:19:16.474 | INFO | Finished a round of Baz IO: Wrote 1.805 GiB in 3.008 seconds | 20260506 09:19:16.474 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.69% / 0.13% / 44.18% | 20260506 09:19:16.474 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 10.0286% / 0.000437101% / 3.57628e-05% | 20260506 09:19:16.474 | INFO | Baz IO overall: | 20260506 09:19:16.474 | INFO | BytesWritten : 80423755009 | 20260506 09:19:16.474 | INFO | NumEvents : 37645020541 | 20260506 09:19:18.504 | INFO | Analyzing chunk frames = [225280,225792) | 20260506 09:19:18.504 | INFO | MemInfo: { | 20260506 09:19:18.504 | INFO | "gpu" : " 17 GB", | 20260506 09:19:18.504 | INFO | "huge" : "", | 20260506 09:19:18.504 | INFO | "huge_shared" : " 27 GB", | 20260506 09:19:18.504 | INFO | "malloc" : " 14 GB", | 20260506 09:19:18.504 | INFO | "pinned" : " 29 GB" | 20260506 09:19:18.504 | INFO | } | 20260506 09:19:18.504 | INFO | Perf report for the last 5.13954 seconds: | 20260506 09:19:18.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:19:18.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:18.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:18.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:18.504 | INFO | ANALYSIS : 57.9%, 2.9, 88.9ms, 0 | 20260506 09:19:18.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:18.504 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 09:19:18.504 | INFO | PRE_HQ : 56.9%, 2.1, 62.1ms, 0 | 20260506 09:19:18.504 | INFO | BAZWRITER : 58.2%, 4.3, 128.4ms, 0 | 20260506 09:19:18.504 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 09:19:18.504 | INFO | Uploaded 36GiB to GPU in 1.6454s (21.8792GiB/s) | 20260506 09:19:18.504 | INFO | Downloaded 38GiB to GPU in 1.70759s (22.2536GiB/s) | 20260506 09:19:23.643 | INFO | Analyzing chunk frames = [225792,226304) | 20260506 09:19:23.643 | INFO | MemInfo: { | 20260506 09:19:23.643 | INFO | "gpu" : " 17 GB", | 20260506 09:19:23.643 | INFO | "huge" : "", | 20260506 09:19:23.643 | INFO | "huge_shared" : " 27 GB", | 20260506 09:19:23.643 | INFO | "malloc" : " 14 GB", | 20260506 09:19:23.643 | INFO | "pinned" : " 29 GB" | 20260506 09:19:23.643 | INFO | } | 20260506 09:19:23.643 | INFO | Perf report for the last 5.13913 seconds: | 20260506 09:19:23.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:19:23.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:23.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:23.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:23.643 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260506 09:19:23.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:23.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:23.643 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:19:23.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:23.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:23.644 | INFO | Uploaded 36GiB to GPU in 1.60099s (22.486GiB/s) | 20260506 09:19:23.644 | INFO | Downloaded 34GiB to GPU in 1.51845s (22.3913GiB/s) | 20260506 09:19:28.782 | INFO | Analyzing chunk frames = [226304,226816) | 20260506 09:19:28.783 | INFO | MemInfo: { | 20260506 09:19:28.783 | INFO | "gpu" : " 17 GB", | 20260506 09:19:28.783 | INFO | "huge" : "", | 20260506 09:19:28.783 | INFO | "huge_shared" : " 27 GB", | 20260506 09:19:28.783 | INFO | "malloc" : " 14 GB", | 20260506 09:19:28.783 | INFO | "pinned" : " 29 GB" | 20260506 09:19:28.783 | INFO | } | 20260506 09:19:28.783 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:19:28.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:19:28.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:28.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:28.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:28.783 | INFO | ANALYSIS : 57.2%, 3.0, 88.8ms, 0 | 20260506 09:19:28.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:28.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:28.783 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:19:28.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:28.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:28.783 | INFO | Uploaded 36GiB to GPU in 1.60526s (22.4262GiB/s) | 20260506 09:19:28.783 | INFO | Downloaded 34GiB to GPU in 1.52242s (22.3328GiB/s) | 20260506 09:19:33.922 | INFO | Analyzing chunk frames = [226816,227328) | 20260506 09:19:33.922 | INFO | MemInfo: { | 20260506 09:19:33.922 | INFO | "gpu" : " 17 GB", | 20260506 09:19:33.922 | INFO | "huge" : "", | 20260506 09:19:33.922 | INFO | "huge_shared" : " 27 GB", | 20260506 09:19:33.922 | INFO | "malloc" : " 14 GB", | 20260506 09:19:33.922 | INFO | "pinned" : " 29 GB" | 20260506 09:19:33.922 | INFO | } | 20260506 09:19:33.922 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:19:33.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:19:33.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:33.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:33.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:33.922 | INFO | ANALYSIS : 56.8%, 3.0, 88.1ms, 0 | 20260506 09:19:33.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:33.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:33.922 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:19:33.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:33.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:33.922 | INFO | Uploaded 36GiB to GPU in 1.60482s (22.4324GiB/s) | 20260506 09:19:33.922 | INFO | Downloaded 34GiB to GPU in 1.5227s (22.3288GiB/s) | 20260506 09:19:39.061 | INFO | Analyzing chunk frames = [227328,227840) | 20260506 09:19:39.061 | INFO | MemInfo: { | 20260506 09:19:39.061 | INFO | "gpu" : " 17 GB", | 20260506 09:19:39.061 | INFO | "huge" : "", | 20260506 09:19:39.061 | INFO | "huge_shared" : " 27 GB", | 20260506 09:19:39.061 | INFO | "malloc" : " 14 GB", | 20260506 09:19:39.061 | INFO | "pinned" : " 29 GB" | 20260506 09:19:39.061 | INFO | } | 20260506 09:19:39.061 | INFO | Perf report for the last 5.13919 seconds: | 20260506 09:19:39.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:19:39.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:39.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:39.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:39.061 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:19:39.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:39.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:39.061 | INFO | PRE_HQ : 30.0%, 1.0, 15.5ms, 0 | 20260506 09:19:39.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:39.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:39.062 | INFO | Uploaded 36GiB to GPU in 1.60229s (22.4678GiB/s) | 20260506 09:19:39.062 | INFO | Downloaded 34GiB to GPU in 1.51918s (22.3805GiB/s) | 20260506 09:19:44.201 | INFO | Analyzing chunk frames = [227840,228352) | 20260506 09:19:44.201 | INFO | MemInfo: { | 20260506 09:19:44.201 | INFO | "gpu" : " 17 GB", | 20260506 09:19:44.201 | INFO | "huge" : "", | 20260506 09:19:44.201 | INFO | "huge_shared" : " 27 GB", | 20260506 09:19:44.201 | INFO | "malloc" : " 14 GB", | 20260506 09:19:44.201 | INFO | "pinned" : " 29 GB" | 20260506 09:19:44.201 | INFO | } | 20260506 09:19:44.201 | INFO | Perf report for the last 5.13959 seconds: | 20260506 09:19:44.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:19:44.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:44.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:44.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:44.201 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:19:44.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:44.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:44.201 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:19:44.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:44.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:44.201 | INFO | Uploaded 36GiB to GPU in 1.6068s (22.4047GiB/s) | 20260506 09:19:44.202 | INFO | Downloaded 34GiB to GPU in 1.52032s (22.3638GiB/s) | 20260506 09:19:49.340 | INFO | Analyzing chunk frames = [228352,228864) | 20260506 09:19:49.340 | INFO | MemInfo: { | 20260506 09:19:49.340 | INFO | "gpu" : " 17 GB", | 20260506 09:19:49.340 | INFO | "huge" : "", | 20260506 09:19:49.340 | INFO | "huge_shared" : " 27 GB", | 20260506 09:19:49.340 | INFO | "malloc" : " 14 GB", | 20260506 09:19:49.340 | INFO | "pinned" : " 29 GB" | 20260506 09:19:49.340 | INFO | } | 20260506 09:19:49.340 | INFO | Perf report for the last 5.1392 seconds: | 20260506 09:19:49.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:19:49.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:49.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:49.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:49.340 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:19:49.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:49.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:49.340 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:19:49.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:49.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:49.341 | INFO | Uploaded 36GiB to GPU in 1.60355s (22.4502GiB/s) | 20260506 09:19:49.341 | INFO | Downloaded 34GiB to GPU in 1.52051s (22.3609GiB/s) | 20260506 09:19:54.479 | INFO | Analyzing chunk frames = [228864,229376) | 20260506 09:19:54.479 | INFO | MemInfo: { | 20260506 09:19:54.479 | INFO | "gpu" : " 17 GB", | 20260506 09:19:54.479 | INFO | "huge" : "", | 20260506 09:19:54.479 | INFO | "huge_shared" : " 27 GB", | 20260506 09:19:54.479 | INFO | "malloc" : " 14 GB", | 20260506 09:19:54.479 | INFO | "pinned" : " 29 GB" | 20260506 09:19:54.479 | INFO | } | 20260506 09:19:54.479 | INFO | Perf report for the last 5.13935 seconds: | 20260506 09:19:54.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:19:54.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:54.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:54.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:54.480 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 09:19:54.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:54.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:54.480 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 09:19:54.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:54.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:54.480 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:19:54.480 | INFO | Total mem usage: 15.7 GiB | 20260506 09:19:54.480 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.38%) | 20260506 09:19:54.480 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -6.47%) | 20260506 09:19:54.480 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:19:54.480 | INFO | High Water Breakdown: | 20260506 09:19:54.480 | INFO | 58.8% : 8.7 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:19:54.480 | INFO | 41.1% : 6.07 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:19:54.480 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:19:54.480 | INFO | ------------------------------------------ | 20260506 09:19:54.480 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:19:54.480 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:19:54.480 | INFO | Total mem usage: 26.5 GiB | 20260506 09:19:54.480 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:19:54.480 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:19:54.480 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:19:54.480 | INFO | High Water Breakdown: | 20260506 09:19:54.480 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:19:54.480 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:19:54.480 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:19:54.480 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:19:54.480 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:19:54.480 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:19:54.480 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:19:54.480 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:19:54.480 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:19:54.480 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:19:54.480 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:19:54.480 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:19:54.480 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:19:54.480 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:19:54.480 | INFO | ------------------------------------------ | 20260506 09:19:54.480 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:19:54.480 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:19:54.480 | INFO | Total mem usage: 25.6 GiB | 20260506 09:19:54.480 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:19:54.480 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:19:54.480 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:19:54.480 | INFO | High Water Breakdown: | 20260506 09:19:54.480 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:19:54.480 | INFO | ------------------------------------------ | 20260506 09:19:54.480 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:19:54.480 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:19:54.480 | INFO | Total mem usage: 15.7 GiB | 20260506 09:19:54.480 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:19:54.480 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:19:54.480 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:19:54.480 | INFO | High Water Breakdown: | 20260506 09:19:54.480 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:19:54.480 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:19:54.480 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:19:54.480 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:19:54.480 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:19:54.480 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:19:54.480 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:19:54.480 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:19:54.480 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:19:54.480 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:19:54.480 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:19:54.480 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:19:54.480 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:19:54.480 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:19:54.480 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:19:54.480 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:19:54.480 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:19:54.480 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:19:54.480 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:19:54.480 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:19:54.480 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:19:54.480 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:19:54.480 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:19:54.480 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:19:54.480 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:19:54.480 | INFO | ------------------------------------------ | 20260506 09:19:54.480 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:19:54.480 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:19:54.480 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:19:54.480 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:19:54.480 | INFO | PerfReport: Full Execution -- 3.03e+01 / 6.96e+01, 1.27e+01 / 0.00e+00 | 20260506 09:19:54.480 | INFO | PerfReport: 0.30%, AggregateStats -- 9.02e-02 / 8.63e-02, 1.23e-01 / 0.00e+00 | 20260506 09:19:54.480 | INFO | PerfReport: 10.88%, Baseline -- 3.30e+00 / 3.28e+00, 9.78e-02 / 0.00e+00 | 20260506 09:19:54.480 | INFO | PerfReport: 3.05%, Histogram -- 9.24e-01 / 1.06e+00, 1.02e-01 / 0.00e+00 | 20260506 09:19:54.480 | INFO | PerfReport: 16.03%, DME -- 3.88e+01 / 4.02e+01, 3.14e+00 / 0.00e+00 | 20260506 09:19:54.480 | INFO | PerfReport: 56.78%, FrameLabeling -- 1.72e+01 / 1.72e+01, 4.00e-02 / 0.00e+00 | 20260506 09:19:54.480 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.29e+00 / 2.35e+00, 1.55e-01 / 0.00e+00 | 20260506 09:19:54.480 | INFO | PerfReport: 3.88%, Metrics -- 1.17e+00 / 1.18e+00, 1.82e-01 / 0.00e+00 | 20260506 09:19:54.480 | INFO | PerfReport: 1.52%, PulseSerializing -- 4.61e-01 / 4.28e+00, 1.18e-01 / 0.00e+00 | 20260506 09:19:54.480 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:19:54.480 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:19:54.480 | INFO | Uploaded 36GiB to GPU in 1.60378s (22.447GiB/s) | 20260506 09:19:54.481 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:19:54.481 | INFO | PerfReport: Full Execution -- 9.08e+01 / -nan, 1.89e+01 / 0.00e+00 | 20260506 09:19:54.481 | INFO | Downloaded 34GiB to GPU in 1.52084s (22.356GiB/s) | 20260506 09:19:54.481 | INFO | PerfReport: 18.46%, Upload -- 1.68e+01 / 0.00e+00, 4.21e-01 / 0.00e+00 | 20260506 09:19:54.481 | INFO | PerfReport: 17.72%, Download -- 1.61e+01 / 0.00e+00, 7.38e-01 / 0.00e+00 | 20260506 09:19:54.481 | INFO | PerfReport: 33.36%, Compute -- 3.03e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 09:19:54.481 | INFO | PerfReport: 0.55%, LockUpload -- 5.04e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 09:19:54.481 | INFO | PerfReport: 0.00%, LockDownload -- 4.87e-04 / 0.00e+00, 1.92e-04 / 0.00e+00 | 20260506 09:19:54.481 | INFO | PerfReport: 29.90%, LockCompute -- 2.71e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 09:19:54.481 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260506 09:19:54.578 | INFO | Starting a round of Baz IO | 20260506 09:19:57.508 | INFO | Populating RTMetrics report | 20260506 09:19:57.508 | INFO | RTMetrics report populated | 20260506 09:19:57.508 | INFO | Opening tmp RTMetrics json file | 20260506 09:19:57.510 | INFO | Serializing RTMetrics to json | 20260506 09:19:57.513 | INFO | Serialization done, swapping tmp file now... | 20260506 09:19:57.514 | INFO | RTMetrics done | 20260506 09:19:57.604 | INFO | Finished a round of Baz IO: Wrote 1.808 GiB in 3.026 seconds | 20260506 09:19:57.604 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.56% / 0.13% / 44.31% | 20260506 09:19:57.604 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.99125% / 0.000464916% / 3.97364e-05% | 20260506 09:19:57.604 | INFO | Baz IO overall: | 20260506 09:19:57.604 | INFO | BytesWritten : 82364961549 | 20260506 09:19:57.604 | INFO | NumEvents : 38587999554 | 20260506 09:19:59.619 | INFO | Analyzing chunk frames = [229376,229888) | 20260506 09:19:59.619 | INFO | MemInfo: { | 20260506 09:19:59.619 | INFO | "gpu" : " 17 GB", | 20260506 09:19:59.619 | INFO | "huge" : "", | 20260506 09:19:59.619 | INFO | "huge_shared" : " 27 GB", | 20260506 09:19:59.619 | INFO | "malloc" : " 14 GB", | 20260506 09:19:59.619 | INFO | "pinned" : " 29 GB" | 20260506 09:19:59.619 | INFO | } | 20260506 09:19:59.619 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:19:59.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:19:59.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:59.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:59.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:59.619 | INFO | ANALYSIS : 58.0%, 2.9, 88.1ms, 0 | 20260506 09:19:59.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:19:59.619 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 09:19:59.619 | INFO | PRE_HQ : 57.1%, 2.1, 61.9ms, 0 | 20260506 09:19:59.619 | INFO | BAZWRITER : 58.5%, 4.4, 133.1ms, 0 | 20260506 09:19:59.619 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 09:19:59.619 | INFO | Uploaded 36GiB to GPU in 1.64826s (21.8413GiB/s) | 20260506 09:19:59.619 | INFO | Downloaded 38GiB to GPU in 1.71182s (22.1985GiB/s) | 20260506 09:20:04.758 | INFO | Analyzing chunk frames = [229888,230400) | 20260506 09:20:04.758 | INFO | MemInfo: { | 20260506 09:20:04.758 | INFO | "gpu" : " 17 GB", | 20260506 09:20:04.758 | INFO | "huge" : "", | 20260506 09:20:04.758 | INFO | "huge_shared" : " 27 GB", | 20260506 09:20:04.758 | INFO | "malloc" : " 14 GB", | 20260506 09:20:04.758 | INFO | "pinned" : " 29 GB" | 20260506 09:20:04.758 | INFO | } | 20260506 09:20:04.758 | INFO | Perf report for the last 5.13917 seconds: | 20260506 09:20:04.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:20:04.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:04.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:04.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:04.758 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260506 09:20:04.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:04.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:04.758 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:20:04.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:04.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:04.759 | INFO | Uploaded 36GiB to GPU in 1.60104s (22.4855GiB/s) | 20260506 09:20:04.759 | INFO | Downloaded 34GiB to GPU in 1.51883s (22.3857GiB/s) | 20260506 09:20:09.897 | INFO | Analyzing chunk frames = [230400,230912) | 20260506 09:20:09.897 | INFO | MemInfo: { | 20260506 09:20:09.897 | INFO | "gpu" : " 17 GB", | 20260506 09:20:09.897 | INFO | "huge" : "", | 20260506 09:20:09.897 | INFO | "huge_shared" : " 27 GB", | 20260506 09:20:09.897 | INFO | "malloc" : " 14 GB", | 20260506 09:20:09.897 | INFO | "pinned" : " 29 GB" | 20260506 09:20:09.897 | INFO | } | 20260506 09:20:09.897 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:20:09.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:20:09.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:09.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:09.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:09.897 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260506 09:20:09.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:09.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:09.897 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:20:09.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:09.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:09.898 | INFO | Uploaded 36GiB to GPU in 1.60474s (22.4335GiB/s) | 20260506 09:20:09.898 | INFO | Downloaded 34GiB to GPU in 1.52216s (22.3366GiB/s) | 20260506 09:20:15.037 | INFO | Analyzing chunk frames = [230912,231424) | 20260506 09:20:15.037 | INFO | MemInfo: { | 20260506 09:20:15.037 | INFO | "gpu" : " 17 GB", | 20260506 09:20:15.037 | INFO | "huge" : "", | 20260506 09:20:15.037 | INFO | "huge_shared" : " 27 GB", | 20260506 09:20:15.037 | INFO | "malloc" : " 14 GB", | 20260506 09:20:15.037 | INFO | "pinned" : " 29 GB" | 20260506 09:20:15.037 | INFO | } | 20260506 09:20:15.037 | INFO | Perf report for the last 5.13967 seconds: | 20260506 09:20:15.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:20:15.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:15.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:15.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:15.037 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260506 09:20:15.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:15.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:15.037 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:20:15.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:15.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:15.037 | INFO | Uploaded 36GiB to GPU in 1.60473s (22.4336GiB/s) | 20260506 09:20:15.037 | INFO | Downloaded 34GiB to GPU in 1.52224s (22.3355GiB/s) | 20260506 09:20:20.176 | INFO | Analyzing chunk frames = [231424,231936) | 20260506 09:20:20.176 | INFO | MemInfo: { | 20260506 09:20:20.176 | INFO | "gpu" : " 17 GB", | 20260506 09:20:20.176 | INFO | "huge" : "", | 20260506 09:20:20.176 | INFO | "huge_shared" : " 27 GB", | 20260506 09:20:20.176 | INFO | "malloc" : " 14 GB", | 20260506 09:20:20.176 | INFO | "pinned" : " 29 GB" | 20260506 09:20:20.176 | INFO | } | 20260506 09:20:20.176 | INFO | Perf report for the last 5.13903 seconds: | 20260506 09:20:20.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:20:20.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:20.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:20.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:20.176 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 09:20:20.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:20.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:20.176 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:20:20.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:20.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:20.176 | INFO | Uploaded 36GiB to GPU in 1.60276s (22.4613GiB/s) | 20260506 09:20:20.176 | INFO | Downloaded 34GiB to GPU in 1.5202s (22.3654GiB/s) | 20260506 09:20:25.315 | INFO | Analyzing chunk frames = [231936,232448) | 20260506 09:20:25.315 | INFO | MemInfo: { | 20260506 09:20:25.315 | INFO | "gpu" : " 17 GB", | 20260506 09:20:25.315 | INFO | "huge" : "", | 20260506 09:20:25.315 | INFO | "huge_shared" : " 27 GB", | 20260506 09:20:25.315 | INFO | "malloc" : " 14 GB", | 20260506 09:20:25.315 | INFO | "pinned" : " 29 GB" | 20260506 09:20:25.315 | INFO | } | 20260506 09:20:25.315 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:20:25.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:20:25.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:25.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:25.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:25.315 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 09:20:25.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:25.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:25.315 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:20:25.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:25.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:25.316 | INFO | Uploaded 36GiB to GPU in 1.60355s (22.4502GiB/s) | 20260506 09:20:25.316 | INFO | Downloaded 34GiB to GPU in 1.52075s (22.3574GiB/s) | 20260506 09:20:30.455 | INFO | Analyzing chunk frames = [232448,232960) | 20260506 09:20:30.455 | INFO | MemInfo: { | 20260506 09:20:30.455 | INFO | "gpu" : " 17 GB", | 20260506 09:20:30.455 | INFO | "huge" : "", | 20260506 09:20:30.455 | INFO | "huge_shared" : " 27 GB", | 20260506 09:20:30.455 | INFO | "malloc" : " 14 GB", | 20260506 09:20:30.455 | INFO | "pinned" : " 29 GB" | 20260506 09:20:30.455 | INFO | } | 20260506 09:20:30.455 | INFO | Perf report for the last 5.13956 seconds: | 20260506 09:20:30.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:20:30.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:30.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:30.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:30.455 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:20:30.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:30.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:30.455 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:20:30.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:30.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:30.455 | INFO | Uploaded 36GiB to GPU in 1.6037s (22.4481GiB/s) | 20260506 09:20:30.455 | INFO | Downloaded 34GiB to GPU in 1.52062s (22.3593GiB/s) | 20260506 09:20:35.594 | INFO | Analyzing chunk frames = [232960,233472) | 20260506 09:20:35.594 | INFO | MemInfo: { | 20260506 09:20:35.594 | INFO | "gpu" : " 17 GB", | 20260506 09:20:35.594 | INFO | "huge" : "", | 20260506 09:20:35.594 | INFO | "huge_shared" : " 27 GB", | 20260506 09:20:35.594 | INFO | "malloc" : " 14 GB", | 20260506 09:20:35.594 | INFO | "pinned" : " 29 GB" | 20260506 09:20:35.594 | INFO | } | 20260506 09:20:35.594 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:20:35.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:20:35.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:35.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:35.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:35.594 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 09:20:35.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:35.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:35.594 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 09:20:35.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:35.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:35.594 | INFO | Uploaded 36GiB to GPU in 1.60393s (22.4449GiB/s) | 20260506 09:20:35.594 | INFO | Downloaded 34GiB to GPU in 1.52103s (22.3533GiB/s) | 20260506 09:20:35.691 | INFO | Starting a round of Baz IO | 20260506 09:20:38.620 | INFO | Populating RTMetrics report | 20260506 09:20:38.620 | INFO | RTMetrics report populated | 20260506 09:20:38.620 | INFO | Opening tmp RTMetrics json file | 20260506 09:20:38.622 | INFO | Serializing RTMetrics to json | 20260506 09:20:38.625 | INFO | Serialization done, swapping tmp file now... | 20260506 09:20:38.625 | INFO | RTMetrics done | 20260506 09:20:38.701 | INFO | Finished a round of Baz IO: Wrote 1.81 GiB in 3.009 seconds | 20260506 09:20:38.701 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.44% / 0.13% / 44.44% | 20260506 09:20:38.701 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.9254% / 0.000377496% / 4.76837e-05% | 20260506 09:20:38.701 | INFO | Baz IO overall: | 20260506 09:20:38.701 | INFO | BytesWritten : 84308484176 | 20260506 09:20:38.701 | INFO | NumEvents : 39531645039 | 20260506 09:20:40.733 | INFO | Analyzing chunk frames = [233472,233984) | 20260506 09:20:40.733 | INFO | MemInfo: { | 20260506 09:20:40.733 | INFO | "gpu" : " 17 GB", | 20260506 09:20:40.733 | INFO | "huge" : "", | 20260506 09:20:40.733 | INFO | "huge_shared" : " 27 GB", | 20260506 09:20:40.733 | INFO | "malloc" : " 14 GB", | 20260506 09:20:40.733 | INFO | "pinned" : " 29 GB" | 20260506 09:20:40.733 | INFO | } | 20260506 09:20:40.733 | INFO | Perf report for the last 5.13917 seconds: | 20260506 09:20:40.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:20:40.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:40.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:40.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:40.733 | INFO | ANALYSIS : 57.9%, 3.0, 89.8ms, 0 | 20260506 09:20:40.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:40.734 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 09:20:40.734 | INFO | PRE_HQ : 56.9%, 2.1, 61.9ms, 0 | 20260506 09:20:40.734 | INFO | BAZWRITER : 58.1%, 4.1, 123.1ms, 0 | 20260506 09:20:40.734 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 09:20:40.734 | INFO | Uploaded 36GiB to GPU in 1.64203s (21.9241GiB/s) | 20260506 09:20:40.734 | INFO | Downloaded 38GiB to GPU in 1.7043s (22.2965GiB/s) | 20260506 09:20:45.872 | INFO | Analyzing chunk frames = [233984,234496) | 20260506 09:20:45.872 | INFO | MemInfo: { | 20260506 09:20:45.872 | INFO | "gpu" : " 17 GB", | 20260506 09:20:45.872 | INFO | "huge" : "", | 20260506 09:20:45.872 | INFO | "huge_shared" : " 27 GB", | 20260506 09:20:45.872 | INFO | "malloc" : " 14 GB", | 20260506 09:20:45.873 | INFO | "pinned" : " 29 GB" | 20260506 09:20:45.873 | INFO | } | 20260506 09:20:45.873 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:20:45.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:20:45.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:45.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:45.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:45.873 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260506 09:20:45.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:45.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:45.873 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:20:45.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:45.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:45.873 | INFO | Uploaded 36GiB to GPU in 1.60137s (22.4808GiB/s) | 20260506 09:20:45.873 | INFO | Downloaded 34GiB to GPU in 1.51844s (22.3914GiB/s) | 20260506 09:20:51.012 | INFO | Analyzing chunk frames = [234496,235008) | 20260506 09:20:51.012 | INFO | MemInfo: { | 20260506 09:20:51.012 | INFO | "gpu" : " 17 GB", | 20260506 09:20:51.012 | INFO | "huge" : "", | 20260506 09:20:51.012 | INFO | "huge_shared" : " 27 GB", | 20260506 09:20:51.012 | INFO | "malloc" : " 14 GB", | 20260506 09:20:51.012 | INFO | "pinned" : " 29 GB" | 20260506 09:20:51.012 | INFO | } | 20260506 09:20:51.012 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:20:51.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:20:51.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:51.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:51.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:51.012 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260506 09:20:51.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:51.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:51.012 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:20:51.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:51.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:51.012 | INFO | Uploaded 36GiB to GPU in 1.60554s (22.4224GiB/s) | 20260506 09:20:51.012 | INFO | Downloaded 34GiB to GPU in 1.52312s (22.3226GiB/s) | 20260506 09:20:56.151 | INFO | Analyzing chunk frames = [235008,235520) | 20260506 09:20:56.151 | INFO | MemInfo: { | 20260506 09:20:56.151 | INFO | "gpu" : " 17 GB", | 20260506 09:20:56.151 | INFO | "huge" : "", | 20260506 09:20:56.151 | INFO | "huge_shared" : " 27 GB", | 20260506 09:20:56.151 | INFO | "malloc" : " 14 GB", | 20260506 09:20:56.151 | INFO | "pinned" : " 29 GB" | 20260506 09:20:56.151 | INFO | } | 20260506 09:20:56.151 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:20:56.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:20:56.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:56.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:56.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:56.151 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260506 09:20:56.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:56.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:56.151 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:20:56.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:56.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:20:56.152 | INFO | Uploaded 36GiB to GPU in 1.605s (22.4299GiB/s) | 20260506 09:20:56.152 | INFO | Downloaded 34GiB to GPU in 1.52283s (22.3269GiB/s) | 20260506 09:21:01.290 | INFO | Analyzing chunk frames = [235520,236032) | 20260506 09:21:01.290 | INFO | MemInfo: { | 20260506 09:21:01.290 | INFO | "gpu" : " 17 GB", | 20260506 09:21:01.290 | INFO | "huge" : "", | 20260506 09:21:01.290 | INFO | "huge_shared" : " 27 GB", | 20260506 09:21:01.291 | INFO | "malloc" : " 14 GB", | 20260506 09:21:01.291 | INFO | "pinned" : " 29 GB" | 20260506 09:21:01.291 | INFO | } | 20260506 09:21:01.291 | INFO | Perf report for the last 5.13926 seconds: | 20260506 09:21:01.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:21:01.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:01.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:01.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:01.291 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 09:21:01.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:01.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:01.291 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:21:01.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:01.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:01.291 | INFO | Uploaded 36GiB to GPU in 1.60319s (22.4552GiB/s) | 20260506 09:21:01.291 | INFO | Downloaded 34GiB to GPU in 1.52015s (22.3662GiB/s) | 20260506 09:21:06.430 | INFO | Analyzing chunk frames = [236032,236544) | 20260506 09:21:06.430 | INFO | MemInfo: { | 20260506 09:21:06.430 | INFO | "gpu" : " 17 GB", | 20260506 09:21:06.430 | INFO | "huge" : "", | 20260506 09:21:06.430 | INFO | "huge_shared" : " 27 GB", | 20260506 09:21:06.430 | INFO | "malloc" : " 14 GB", | 20260506 09:21:06.430 | INFO | "pinned" : " 29 GB" | 20260506 09:21:06.430 | INFO | } | 20260506 09:21:06.430 | INFO | Perf report for the last 5.13942 seconds: | 20260506 09:21:06.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:21:06.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:06.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:06.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:06.430 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 09:21:06.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:06.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:06.430 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:21:06.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:06.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:06.430 | INFO | Uploaded 36GiB to GPU in 1.60356s (22.45GiB/s) | 20260506 09:21:06.430 | INFO | Downloaded 34GiB to GPU in 1.52086s (22.3557GiB/s) | 20260506 09:21:11.569 | INFO | Analyzing chunk frames = [236544,237056) | 20260506 09:21:11.569 | INFO | MemInfo: { | 20260506 09:21:11.569 | INFO | "gpu" : " 17 GB", | 20260506 09:21:11.569 | INFO | "huge" : "", | 20260506 09:21:11.569 | INFO | "huge_shared" : " 27 GB", | 20260506 09:21:11.569 | INFO | "malloc" : " 14 GB", | 20260506 09:21:11.569 | INFO | "pinned" : " 29 GB" | 20260506 09:21:11.569 | INFO | } | 20260506 09:21:11.569 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:21:11.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:21:11.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:11.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:11.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:11.569 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:21:11.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:11.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:11.569 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 09:21:11.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:11.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:11.570 | INFO | Uploaded 36GiB to GPU in 1.6036s (22.4494GiB/s) | 20260506 09:21:11.570 | INFO | Downloaded 34GiB to GPU in 1.52048s (22.3613GiB/s) | 20260506 09:21:16.709 | INFO | Analyzing chunk frames = [237056,237568) | 20260506 09:21:16.709 | INFO | MemInfo: { | 20260506 09:21:16.709 | INFO | "gpu" : " 17 GB", | 20260506 09:21:16.709 | INFO | "huge" : "", | 20260506 09:21:16.709 | INFO | "huge_shared" : " 27 GB", | 20260506 09:21:16.709 | INFO | "malloc" : " 14 GB", | 20260506 09:21:16.709 | INFO | "pinned" : " 29 GB" | 20260506 09:21:16.709 | INFO | } | 20260506 09:21:16.709 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:21:16.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:21:16.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:16.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:16.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:16.709 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 09:21:16.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:16.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:16.709 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 09:21:16.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:16.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:16.709 | INFO | Uploaded 36GiB to GPU in 1.60405s (22.4433GiB/s) | 20260506 09:21:16.709 | INFO | Downloaded 34GiB to GPU in 1.52153s (22.3459GiB/s) | 20260506 09:21:16.709 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:21:16.709 | INFO | Total mem usage: 15.7 GiB | 20260506 09:21:16.709 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.38%) | 20260506 09:21:16.709 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -6.48%) | 20260506 09:21:16.709 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:21:16.709 | INFO | High Water Breakdown: | 20260506 09:21:16.709 | INFO | 58.8% : 8.7 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:21:16.709 | INFO | 41.1% : 6.07 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:21:16.709 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:21:16.709 | INFO | ------------------------------------------ | 20260506 09:21:16.709 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:21:16.709 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:21:16.710 | INFO | Total mem usage: 26.5 GiB | 20260506 09:21:16.710 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:21:16.710 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:21:16.710 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:21:16.710 | INFO | High Water Breakdown: | 20260506 09:21:16.710 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:21:16.710 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:21:16.710 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:21:16.710 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:21:16.710 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:21:16.710 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:21:16.710 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:21:16.710 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:21:16.710 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:21:16.710 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:21:16.710 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:21:16.710 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:21:16.710 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:21:16.710 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:21:16.710 | INFO | ------------------------------------------ | 20260506 09:21:16.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:21:16.710 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:21:16.710 | INFO | Total mem usage: 25.6 GiB | 20260506 09:21:16.710 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:21:16.710 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:21:16.710 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:21:16.710 | INFO | High Water Breakdown: | 20260506 09:21:16.710 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:21:16.710 | INFO | ------------------------------------------ | 20260506 09:21:16.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:21:16.710 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:21:16.710 | INFO | Total mem usage: 15.7 GiB | 20260506 09:21:16.710 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:21:16.710 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:21:16.710 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:21:16.710 | INFO | High Water Breakdown: | 20260506 09:21:16.710 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:21:16.710 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:21:16.710 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:21:16.710 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:21:16.710 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:21:16.710 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:21:16.710 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:21:16.710 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:21:16.710 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:21:16.710 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:21:16.710 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:21:16.710 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:21:16.710 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:21:16.710 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:21:16.710 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:21:16.710 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:21:16.710 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:21:16.710 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:21:16.710 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:21:16.710 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:21:16.710 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:21:16.710 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:21:16.710 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:21:16.710 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:21:16.710 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:21:16.710 | INFO | ------------------------------------------ | 20260506 09:21:16.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:21:16.710 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:21:16.710 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:21:16.710 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:21:16.710 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 09:21:16.710 | INFO | PerfReport: 0.29%, AggregateStats -- 8.64e-02 / -nan, 5.45e-03 / 0.00e+00 | 20260506 09:21:16.710 | INFO | PerfReport: 10.88%, Baseline -- 3.29e+00 / -nan, 1.89e-02 / 0.00e+00 | 20260506 09:21:16.710 | INFO | PerfReport: 3.04%, Histogram -- 9.22e-01 / -nan, 5.19e-02 / 0.00e+00 | 20260506 09:21:16.710 | INFO | PerfReport: 16.03%, DME -- 3.88e+01 / -nan, 3.10e+00 / 0.00e+00 | 20260506 09:21:16.710 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.72e+01 / -nan, 4.17e-02 / 0.00e+00 | 20260506 09:21:16.710 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.28e+00 / -nan, 1.37e-01 / 0.00e+00 | 20260506 09:21:16.710 | INFO | PerfReport: 3.87%, Metrics -- 1.17e+00 / -nan, 1.84e-01 / 0.00e+00 | 20260506 09:21:16.710 | INFO | PerfReport: 1.51%, PulseSerializing -- 4.58e-01 / -nan, 6.68e-02 / 0.00e+00 | 20260506 09:21:16.711 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:21:16.711 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:21:16.711 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:21:16.711 | INFO | PerfReport: Full Execution -- 9.08e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 09:21:16.711 | INFO | PerfReport: 18.47%, Upload -- 1.68e+01 / 0.00e+00, 4.04e-01 / 0.00e+00 | 20260506 09:21:16.711 | INFO | PerfReport: 17.73%, Download -- 1.61e+01 / 0.00e+00, 7.28e-01 / 0.00e+00 | 20260506 09:21:16.711 | INFO | PerfReport: 33.35%, Compute -- 3.03e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 09:21:16.711 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260506 09:21:16.711 | INFO | PerfReport: 0.00%, LockDownload -- 4.88e-04 / 0.00e+00, 1.14e-04 / 0.00e+00 | 20260506 09:21:16.711 | INFO | PerfReport: 29.88%, LockCompute -- 2.71e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 09:21:16.711 | INFO | PerfReport: 0.02%, Misc -- 1.52e-02 / 0.00e+00, 4.42e-02 / 0.00e+00 | 20260506 09:21:16.806 | INFO | Starting a round of Baz IO | 20260506 09:21:19.732 | INFO | Populating RTMetrics report | 20260506 09:21:19.732 | INFO | RTMetrics report populated | 20260506 09:21:19.732 | INFO | Opening tmp RTMetrics json file | 20260506 09:21:19.828 | INFO | Serializing RTMetrics to json | 20260506 09:21:19.837 | INFO | Serialization done, swapping tmp file now... | 20260506 09:21:19.846 | INFO | RTMetrics done | 20260506 09:21:20.262 | INFO | Finished a round of Baz IO: Wrote 1.814 GiB in 3.455 seconds | 20260506 09:21:20.262 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.32% / 0.12% / 44.56% | 20260506 09:21:20.262 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.8972% / 0.000377496% / 4.76837e-05% | 20260506 09:21:20.262 | INFO | Baz IO overall: | 20260506 09:21:20.262 | INFO | BytesWritten : 86256143612 | 20260506 09:21:20.262 | INFO | NumEvents : 40477403865 | 20260506 09:21:21.848 | INFO | Analyzing chunk frames = [237568,238080) | 20260506 09:21:21.848 | INFO | MemInfo: { | 20260506 09:21:21.848 | INFO | "gpu" : " 17 GB", | 20260506 09:21:21.848 | INFO | "huge" : "", | 20260506 09:21:21.848 | INFO | "huge_shared" : " 27 GB", | 20260506 09:21:21.848 | INFO | "malloc" : " 14 GB", | 20260506 09:21:21.848 | INFO | "pinned" : " 29 GB" | 20260506 09:21:21.848 | INFO | } | 20260506 09:21:21.848 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:21:21.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:21:21.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:21.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:21.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:21.848 | INFO | ANALYSIS : 57.9%, 2.9, 88.3ms, 0 | 20260506 09:21:21.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:21.848 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 09:21:21.848 | INFO | PRE_HQ : 57.0%, 2.1, 61.7ms, 0 | 20260506 09:21:21.848 | INFO | BAZWRITER : 66.7%, 4.4, 152.2ms, 0 | 20260506 09:21:21.848 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 09:21:21.848 | INFO | Uploaded 36GiB to GPU in 1.65s (21.8182GiB/s) | 20260506 09:21:21.848 | INFO | Downloaded 38GiB to GPU in 1.71358s (22.1758GiB/s) | 20260506 09:21:26.987 | INFO | Analyzing chunk frames = [238080,238592) | 20260506 09:21:26.987 | INFO | MemInfo: { | 20260506 09:21:26.987 | INFO | "gpu" : " 17 GB", | 20260506 09:21:26.987 | INFO | "huge" : "", | 20260506 09:21:26.987 | INFO | "huge_shared" : " 27 GB", | 20260506 09:21:26.987 | INFO | "malloc" : " 14 GB", | 20260506 09:21:26.987 | INFO | "pinned" : " 29 GB" | 20260506 09:21:26.987 | INFO | } | 20260506 09:21:26.987 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:21:26.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:21:26.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:26.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:26.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:26.987 | INFO | ANALYSIS : 57.7%, 3.0, 89.9ms, 0 | 20260506 09:21:26.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:26.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:26.987 | INFO | PRE_HQ : 30.4%, 1.0, 16.2ms, 0 | 20260506 09:21:26.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:26.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:26.988 | INFO | Uploaded 36GiB to GPU in 1.60204s (22.4713GiB/s) | 20260506 09:21:26.988 | INFO | Downloaded 34GiB to GPU in 1.5192s (22.3802GiB/s) | 20260506 09:21:32.127 | INFO | Analyzing chunk frames = [238592,239104) | 20260506 09:21:32.127 | INFO | MemInfo: { | 20260506 09:21:32.127 | INFO | "gpu" : " 17 GB", | 20260506 09:21:32.127 | INFO | "huge" : "", | 20260506 09:21:32.127 | INFO | "huge_shared" : " 27 GB", | 20260506 09:21:32.127 | INFO | "malloc" : " 14 GB", | 20260506 09:21:32.127 | INFO | "pinned" : " 29 GB" | 20260506 09:21:32.127 | INFO | } | 20260506 09:21:32.127 | INFO | Perf report for the last 5.13935 seconds: | 20260506 09:21:32.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:21:32.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:32.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:32.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:32.127 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260506 09:21:32.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:32.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:32.127 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:21:32.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:32.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:32.127 | INFO | Uploaded 36GiB to GPU in 1.60566s (22.4208GiB/s) | 20260506 09:21:32.127 | INFO | Downloaded 34GiB to GPU in 1.52301s (22.3242GiB/s) | 20260506 09:21:37.266 | INFO | Analyzing chunk frames = [239104,239616) | 20260506 09:21:37.266 | INFO | MemInfo: { | 20260506 09:21:37.266 | INFO | "gpu" : " 17 GB", | 20260506 09:21:37.266 | INFO | "huge" : "", | 20260506 09:21:37.266 | INFO | "huge_shared" : " 27 GB", | 20260506 09:21:37.266 | INFO | "malloc" : " 14 GB", | 20260506 09:21:37.266 | INFO | "pinned" : " 29 GB" | 20260506 09:21:37.266 | INFO | } | 20260506 09:21:37.266 | INFO | Perf report for the last 5.1395 seconds: | 20260506 09:21:37.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:21:37.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:37.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:37.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:37.266 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260506 09:21:37.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:37.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:37.266 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:21:37.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:37.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:37.266 | INFO | Uploaded 36GiB to GPU in 1.6038s (22.4467GiB/s) | 20260506 09:21:37.266 | INFO | Downloaded 34GiB to GPU in 1.52174s (22.3428GiB/s) | 20260506 09:21:42.405 | INFO | Analyzing chunk frames = [239616,240128) | 20260506 09:21:42.405 | INFO | MemInfo: { | 20260506 09:21:42.405 | INFO | "gpu" : " 17 GB", | 20260506 09:21:42.405 | INFO | "huge" : "", | 20260506 09:21:42.405 | INFO | "huge_shared" : " 27 GB", | 20260506 09:21:42.405 | INFO | "malloc" : " 14 GB", | 20260506 09:21:42.405 | INFO | "pinned" : " 29 GB" | 20260506 09:21:42.405 | INFO | } | 20260506 09:21:42.406 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:21:42.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:21:42.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:42.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:42.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:42.406 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 09:21:42.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:42.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:42.406 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:21:42.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:42.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:42.406 | INFO | Uploaded 36GiB to GPU in 1.60228s (22.468GiB/s) | 20260506 09:21:42.406 | INFO | Downloaded 34GiB to GPU in 1.52047s (22.3615GiB/s) | 20260506 09:21:47.545 | INFO | Analyzing chunk frames = [240128,240640) | 20260506 09:21:47.545 | INFO | MemInfo: { | 20260506 09:21:47.545 | INFO | "gpu" : " 17 GB", | 20260506 09:21:47.545 | INFO | "huge" : "", | 20260506 09:21:47.545 | INFO | "huge_shared" : " 27 GB", | 20260506 09:21:47.545 | INFO | "malloc" : " 14 GB", | 20260506 09:21:47.545 | INFO | "pinned" : " 29 GB" | 20260506 09:21:47.545 | INFO | } | 20260506 09:21:47.545 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:21:47.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:21:47.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:47.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:47.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:47.545 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:21:47.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:47.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:47.545 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260506 09:21:47.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:47.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:47.545 | INFO | Uploaded 36GiB to GPU in 1.60357s (22.4499GiB/s) | 20260506 09:21:47.545 | INFO | Downloaded 34GiB to GPU in 1.52037s (22.363GiB/s) | 20260506 09:21:52.684 | INFO | Analyzing chunk frames = [240640,241152) | 20260506 09:21:52.684 | INFO | MemInfo: { | 20260506 09:21:52.684 | INFO | "gpu" : " 17 GB", | 20260506 09:21:52.684 | INFO | "huge" : "", | 20260506 09:21:52.684 | INFO | "huge_shared" : " 27 GB", | 20260506 09:21:52.684 | INFO | "malloc" : " 14 GB", | 20260506 09:21:52.684 | INFO | "pinned" : " 29 GB" | 20260506 09:21:52.684 | INFO | } | 20260506 09:21:52.684 | INFO | Perf report for the last 5.1394 seconds: | 20260506 09:21:52.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:21:52.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:52.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:52.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:52.684 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:21:52.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:52.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:52.684 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 09:21:52.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:52.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:52.685 | INFO | Uploaded 36GiB to GPU in 1.60434s (22.4391GiB/s) | 20260506 09:21:52.685 | INFO | Downloaded 34GiB to GPU in 1.5212s (22.3508GiB/s) | 20260506 09:21:57.823 | INFO | Analyzing chunk frames = [241152,241664) | 20260506 09:21:57.823 | INFO | MemInfo: { | 20260506 09:21:57.823 | INFO | "gpu" : " 17 GB", | 20260506 09:21:57.823 | INFO | "huge" : "", | 20260506 09:21:57.823 | INFO | "huge_shared" : " 27 GB", | 20260506 09:21:57.823 | INFO | "malloc" : " 14 GB", | 20260506 09:21:57.823 | INFO | "pinned" : " 29 GB" | 20260506 09:21:57.823 | INFO | } | 20260506 09:21:57.824 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:21:57.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:21:57.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:57.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:57.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:57.824 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 09:21:57.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:57.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:57.824 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 09:21:57.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:57.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:21:57.824 | INFO | Uploaded 36GiB to GPU in 1.60286s (22.4599GiB/s) | 20260506 09:21:57.824 | INFO | Downloaded 34GiB to GPU in 1.52137s (22.3483GiB/s) | 20260506 09:21:57.921 | INFO | Starting a round of Baz IO | 20260506 09:22:00.847 | INFO | Populating RTMetrics report | 20260506 09:22:00.847 | INFO | RTMetrics report populated | 20260506 09:22:00.847 | INFO | Opening tmp RTMetrics json file | 20260506 09:22:00.848 | INFO | Serializing RTMetrics to json | 20260506 09:22:00.852 | INFO | Serialization done, swapping tmp file now... | 20260506 09:22:00.856 | INFO | RTMetrics done | 20260506 09:22:00.931 | INFO | Finished a round of Baz IO: Wrote 1.816 GiB in 3.01 seconds | 20260506 09:22:00.931 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.19% / 0.12% / 44.68% | 20260506 09:22:00.931 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.84383% / 0.000397364% / 5.16574e-05% | 20260506 09:22:00.931 | INFO | Baz IO overall: | 20260506 09:22:00.931 | INFO | BytesWritten : 88205925273 | 20260506 09:22:00.931 | INFO | NumEvents : 41423978234 | 20260506 09:22:02.963 | INFO | Analyzing chunk frames = [241664,242176) | 20260506 09:22:02.963 | INFO | MemInfo: { | 20260506 09:22:02.963 | INFO | "gpu" : " 17 GB", | 20260506 09:22:02.963 | INFO | "huge" : "", | 20260506 09:22:02.963 | INFO | "huge_shared" : " 27 GB", | 20260506 09:22:02.963 | INFO | "malloc" : " 14 GB", | 20260506 09:22:02.963 | INFO | "pinned" : " 29 GB" | 20260506 09:22:02.963 | INFO | } | 20260506 09:22:02.963 | INFO | Perf report for the last 5.13919 seconds: | 20260506 09:22:02.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:22:02.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:02.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:02.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:02.963 | INFO | ANALYSIS : 57.9%, 2.9, 89.2ms, 0 | 20260506 09:22:02.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:02.963 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 09:22:02.963 | INFO | PRE_HQ : 57.0%, 2.1, 62.7ms, 0 | 20260506 09:22:02.963 | INFO | BAZWRITER : 58.2%, 4.2, 126.7ms, 0 | 20260506 09:22:02.963 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 09:22:02.963 | INFO | Uploaded 36GiB to GPU in 1.64271s (21.915GiB/s) | 20260506 09:22:02.963 | INFO | Downloaded 38GiB to GPU in 1.70611s (22.2729GiB/s) | 20260506 09:22:08.102 | INFO | Analyzing chunk frames = [242176,242688) | 20260506 09:22:08.102 | INFO | MemInfo: { | 20260506 09:22:08.102 | INFO | "gpu" : " 17 GB", | 20260506 09:22:08.102 | INFO | "huge" : "", | 20260506 09:22:08.102 | INFO | "huge_shared" : " 27 GB", | 20260506 09:22:08.102 | INFO | "malloc" : " 14 GB", | 20260506 09:22:08.102 | INFO | "pinned" : " 29 GB" | 20260506 09:22:08.102 | INFO | } | 20260506 09:22:08.102 | INFO | Perf report for the last 5.13952 seconds: | 20260506 09:22:08.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:22:08.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:08.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:08.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:08.102 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260506 09:22:08.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:08.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:08.102 | INFO | PRE_HQ : 30.0%, 1.0, 15.7ms, 0 | 20260506 09:22:08.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:08.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:08.102 | INFO | Uploaded 36GiB to GPU in 1.60126s (22.4823GiB/s) | 20260506 09:22:08.102 | INFO | Downloaded 34GiB to GPU in 1.51871s (22.3875GiB/s) | 20260506 09:22:13.241 | INFO | Analyzing chunk frames = [242688,243200) | 20260506 09:22:13.241 | INFO | MemInfo: { | 20260506 09:22:13.241 | INFO | "gpu" : " 17 GB", | 20260506 09:22:13.241 | INFO | "huge" : "", | 20260506 09:22:13.241 | INFO | "huge_shared" : " 27 GB", | 20260506 09:22:13.241 | INFO | "malloc" : " 14 GB", | 20260506 09:22:13.241 | INFO | "pinned" : " 29 GB" | 20260506 09:22:13.241 | INFO | } | 20260506 09:22:13.242 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:22:13.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:22:13.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:13.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:13.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:13.242 | INFO | ANALYSIS : 57.0%, 3.0, 88.6ms, 0 | 20260506 09:22:13.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:13.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:13.242 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:22:13.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:13.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:13.242 | INFO | Uploaded 36GiB to GPU in 1.60535s (22.4251GiB/s) | 20260506 09:22:13.242 | INFO | Downloaded 34GiB to GPU in 1.52342s (22.3183GiB/s) | 20260506 09:22:18.381 | INFO | Analyzing chunk frames = [243200,243712) | 20260506 09:22:18.381 | INFO | MemInfo: { | 20260506 09:22:18.381 | INFO | "gpu" : " 17 GB", | 20260506 09:22:18.381 | INFO | "huge" : "", | 20260506 09:22:18.381 | INFO | "huge_shared" : " 27 GB", | 20260506 09:22:18.381 | INFO | "malloc" : " 14 GB", | 20260506 09:22:18.381 | INFO | "pinned" : " 29 GB" | 20260506 09:22:18.381 | INFO | } | 20260506 09:22:18.381 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:22:18.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:22:18.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:18.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:18.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:18.381 | INFO | ANALYSIS : 56.8%, 3.0, 88.0ms, 0 | 20260506 09:22:18.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:18.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:18.381 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:22:18.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:18.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:18.381 | INFO | Uploaded 36GiB to GPU in 1.6054s (22.4243GiB/s) | 20260506 09:22:18.381 | INFO | Downloaded 34GiB to GPU in 1.523s (22.3244GiB/s) | 20260506 09:22:23.520 | INFO | Analyzing chunk frames = [243712,244224) | 20260506 09:22:23.520 | INFO | MemInfo: { | 20260506 09:22:23.520 | INFO | "gpu" : " 17 GB", | 20260506 09:22:23.520 | INFO | "huge" : "", | 20260506 09:22:23.520 | INFO | "huge_shared" : " 27 GB", | 20260506 09:22:23.520 | INFO | "malloc" : " 14 GB", | 20260506 09:22:23.520 | INFO | "pinned" : " 29 GB" | 20260506 09:22:23.520 | INFO | } | 20260506 09:22:23.520 | INFO | Perf report for the last 5.13945 seconds: | 20260506 09:22:23.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:22:23.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:23.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:23.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:23.520 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:22:23.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:23.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:23.520 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:22:23.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:23.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:23.521 | INFO | Uploaded 36GiB to GPU in 1.60255s (22.4642GiB/s) | 20260506 09:22:23.521 | INFO | Downloaded 34GiB to GPU in 1.52013s (22.3665GiB/s) | 20260506 09:22:28.659 | INFO | Analyzing chunk frames = [244224,244736) | 20260506 09:22:28.659 | INFO | MemInfo: { | 20260506 09:22:28.659 | INFO | "gpu" : " 17 GB", | 20260506 09:22:28.660 | INFO | "huge" : "", | 20260506 09:22:28.660 | INFO | "huge_shared" : " 27 GB", | 20260506 09:22:28.660 | INFO | "malloc" : " 14 GB", | 20260506 09:22:28.660 | INFO | "pinned" : " 29 GB" | 20260506 09:22:28.660 | INFO | } | 20260506 09:22:28.660 | INFO | Perf report for the last 5.13926 seconds: | 20260506 09:22:28.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:22:28.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:28.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:28.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:28.660 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:22:28.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:28.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:28.660 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:22:28.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:28.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:28.660 | INFO | Uploaded 36GiB to GPU in 1.60316s (22.4556GiB/s) | 20260506 09:22:28.660 | INFO | Downloaded 34GiB to GPU in 1.51936s (22.3779GiB/s) | 20260506 09:22:33.799 | INFO | Analyzing chunk frames = [244736,245248) | 20260506 09:22:33.799 | INFO | MemInfo: { | 20260506 09:22:33.799 | INFO | "gpu" : " 17 GB", | 20260506 09:22:33.799 | INFO | "huge" : "", | 20260506 09:22:33.799 | INFO | "huge_shared" : " 27 GB", | 20260506 09:22:33.799 | INFO | "malloc" : " 14 GB", | 20260506 09:22:33.799 | INFO | "pinned" : " 29 GB" | 20260506 09:22:33.799 | INFO | } | 20260506 09:22:33.799 | INFO | Perf report for the last 5.13923 seconds: | 20260506 09:22:33.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:22:33.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:33.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:33.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:33.799 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:22:33.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:33.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:33.799 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:22:33.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:33.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:33.799 | INFO | Uploaded 36GiB to GPU in 1.60403s (22.4434GiB/s) | 20260506 09:22:33.799 | INFO | Downloaded 34GiB to GPU in 1.52239s (22.3333GiB/s) | 20260506 09:22:38.938 | INFO | Analyzing chunk frames = [245248,245760) | 20260506 09:22:38.938 | INFO | MemInfo: { | 20260506 09:22:38.938 | INFO | "gpu" : " 17 GB", | 20260506 09:22:38.938 | INFO | "huge" : "", | 20260506 09:22:38.938 | INFO | "huge_shared" : " 27 GB", | 20260506 09:22:38.938 | INFO | "malloc" : " 14 GB", | 20260506 09:22:38.938 | INFO | "pinned" : " 29 GB" | 20260506 09:22:38.938 | INFO | } | 20260506 09:22:38.938 | INFO | Perf report for the last 5.13916 seconds: | 20260506 09:22:38.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:22:38.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:38.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:38.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:38.938 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 09:22:38.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:38.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:38.938 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 09:22:38.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:38.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:38.938 | INFO | Uploaded 36GiB to GPU in 1.60396s (22.4444GiB/s) | 20260506 09:22:38.938 | INFO | Downloaded 34GiB to GPU in 1.5216s (22.3449GiB/s) | 20260506 09:22:38.938 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:22:38.938 | INFO | Total mem usage: 15.7 GiB | 20260506 09:22:38.938 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.38%) | 20260506 09:22:38.938 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -0.111%) | 20260506 09:22:38.938 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:22:38.938 | INFO | High Water Breakdown: | 20260506 09:22:38.938 | INFO | 59.9% : 9.42 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:22:38.938 | INFO | 40% : 6.29 GiB ( 94% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:22:38.938 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:22:38.938 | INFO | ------------------------------------------ | 20260506 09:22:38.938 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:22:38.938 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:22:38.939 | INFO | Total mem usage: 26.5 GiB | 20260506 09:22:38.939 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:22:38.939 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:22:38.939 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:22:38.939 | INFO | High Water Breakdown: | 20260506 09:22:38.939 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:22:38.939 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:22:38.939 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:22:38.939 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:22:38.939 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:22:38.939 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:22:38.939 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:22:38.939 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:22:38.939 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:22:38.939 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:22:38.939 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:22:38.939 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:22:38.939 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:22:38.939 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:22:38.939 | INFO | ------------------------------------------ | 20260506 09:22:38.939 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:22:38.939 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:22:38.939 | INFO | Total mem usage: 25.6 GiB | 20260506 09:22:38.939 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:22:38.939 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:22:38.939 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:22:38.939 | INFO | High Water Breakdown: | 20260506 09:22:38.939 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:22:38.939 | INFO | ------------------------------------------ | 20260506 09:22:38.939 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:22:38.939 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:22:38.939 | INFO | Total mem usage: 15.7 GiB | 20260506 09:22:38.939 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:22:38.939 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:22:38.939 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:22:38.939 | INFO | High Water Breakdown: | 20260506 09:22:38.939 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:22:38.939 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:22:38.939 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:22:38.939 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:22:38.939 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:22:38.939 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:22:38.939 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:22:38.939 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:22:38.939 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:22:38.939 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:22:38.939 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:22:38.939 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:22:38.939 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:22:38.939 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:22:38.939 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:22:38.939 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:22:38.939 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:22:38.939 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:22:38.939 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:22:38.939 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:22:38.939 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:22:38.939 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:22:38.939 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:22:38.939 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:22:38.939 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:22:38.939 | INFO | ------------------------------------------ | 20260506 09:22:38.939 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:22:38.939 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:22:38.939 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:22:38.939 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:22:38.939 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 09:22:38.939 | INFO | PerfReport: 0.28%, AggregateStats -- 8.61e-02 / -nan, 4.76e-03 / 0.00e+00 | 20260506 09:22:38.939 | INFO | PerfReport: 10.88%, Baseline -- 3.29e+00 / -nan, 1.78e-02 / 0.00e+00 | 20260506 09:22:38.939 | INFO | PerfReport: 3.05%, Histogram -- 9.22e-01 / -nan, 5.17e-02 / 0.00e+00 | 20260506 09:22:38.939 | INFO | PerfReport: 16.03%, DME -- 3.88e+01 / -nan, 3.13e+00 / 0.00e+00 | 20260506 09:22:38.939 | INFO | PerfReport: 56.83%, FrameLabeling -- 1.72e+01 / -nan, 4.26e-02 / 0.00e+00 | 20260506 09:22:38.939 | INFO | PerfReport: 7.55%, PulseAccumulating -- 2.29e+00 / -nan, 1.36e-01 / 0.00e+00 | 20260506 09:22:38.939 | INFO | PerfReport: 3.87%, Metrics -- 1.17e+00 / -nan, 1.81e-01 / 0.00e+00 | 20260506 09:22:38.939 | INFO | PerfReport: 1.51%, PulseSerializing -- 4.57e-01 / -nan, 6.67e-02 / 0.00e+00 | 20260506 09:22:38.939 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:22:38.939 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:22:38.939 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:22:38.939 | INFO | PerfReport: Full Execution -- 9.07e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 09:22:38.939 | INFO | PerfReport: 18.47%, Upload -- 1.68e+01 / 0.00e+00, 4.09e-01 / 0.00e+00 | 20260506 09:22:38.940 | INFO | PerfReport: 17.74%, Download -- 1.61e+01 / 0.00e+00, 7.33e-01 / 0.00e+00 | 20260506 09:22:38.940 | INFO | PerfReport: 33.36%, Compute -- 3.03e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 09:22:38.940 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:22:38.940 | INFO | PerfReport: 0.00%, LockDownload -- 4.85e-04 / 0.00e+00, 1.90e-04 / 0.00e+00 | 20260506 09:22:38.940 | INFO | PerfReport: 29.86%, LockCompute -- 2.71e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260506 09:22:38.940 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.43e-02 / 0.00e+00 | 20260506 09:22:39.035 | INFO | Starting a round of Baz IO | 20260506 09:22:41.960 | INFO | Populating RTMetrics report | 20260506 09:22:41.960 | INFO | RTMetrics report populated | 20260506 09:22:41.960 | INFO | Opening tmp RTMetrics json file | 20260506 09:22:41.963 | INFO | Serializing RTMetrics to json | 20260506 09:22:41.968 | INFO | Serialization done, swapping tmp file now... | 20260506 09:22:41.968 | INFO | RTMetrics done | 20260506 09:22:42.059 | INFO | Finished a round of Baz IO: Wrote 1.82 GiB in 3.023 seconds | 20260506 09:22:42.059 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 55.08% / 0.12% / 44.81% | 20260506 09:22:42.059 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.81119% / 0.000373522% / 3.57628e-05% | 20260506 09:22:42.059 | INFO | Baz IO overall: | 20260506 09:22:42.059 | INFO | BytesWritten : 90160288416 | 20260506 09:22:42.059 | INFO | NumEvents : 42373082681 | 20260506 09:22:44.077 | INFO | Analyzing chunk frames = [245760,246272) | 20260506 09:22:44.077 | INFO | MemInfo: { | 20260506 09:22:44.077 | INFO | "gpu" : " 17 GB", | 20260506 09:22:44.077 | INFO | "huge" : "", | 20260506 09:22:44.077 | INFO | "huge_shared" : " 27 GB", | 20260506 09:22:44.077 | INFO | "malloc" : " 14 GB", | 20260506 09:22:44.077 | INFO | "pinned" : " 29 GB" | 20260506 09:22:44.078 | INFO | } | 20260506 09:22:44.078 | INFO | Perf report for the last 5.13952 seconds: | 20260506 09:22:44.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:22:44.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:44.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:44.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:44.078 | INFO | ANALYSIS : 57.8%, 2.9, 87.3ms, 0 | 20260506 09:22:44.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:44.078 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 09:22:44.078 | INFO | PRE_HQ : 56.9%, 2.1, 61.6ms, 0 | 20260506 09:22:44.078 | INFO | BAZWRITER : 58.3%, 4.5, 135.1ms, 0 | 20260506 09:22:44.078 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 09:22:44.078 | INFO | Uploaded 36GiB to GPU in 1.65905s (21.6992GiB/s) | 20260506 09:22:44.078 | INFO | Downloaded 38GiB to GPU in 1.72347s (22.0486GiB/s) | 20260506 09:22:49.217 | INFO | Analyzing chunk frames = [246272,246784) | 20260506 09:22:49.217 | INFO | MemInfo: { | 20260506 09:22:49.217 | INFO | "gpu" : " 17 GB", | 20260506 09:22:49.217 | INFO | "huge" : "", | 20260506 09:22:49.217 | INFO | "huge_shared" : " 27 GB", | 20260506 09:22:49.217 | INFO | "malloc" : " 14 GB", | 20260506 09:22:49.217 | INFO | "pinned" : " 29 GB" | 20260506 09:22:49.217 | INFO | } | 20260506 09:22:49.217 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:22:49.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:22:49.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:49.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:49.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:49.217 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260506 09:22:49.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:49.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:49.217 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:22:49.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:49.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:49.217 | INFO | Uploaded 36GiB to GPU in 1.60112s (22.4843GiB/s) | 20260506 09:22:49.217 | INFO | Downloaded 34GiB to GPU in 1.51804s (22.3972GiB/s) | 20260506 09:22:54.356 | INFO | Analyzing chunk frames = [246784,247296) | 20260506 09:22:54.356 | INFO | MemInfo: { | 20260506 09:22:54.356 | INFO | "gpu" : " 17 GB", | 20260506 09:22:54.356 | INFO | "huge" : "", | 20260506 09:22:54.356 | INFO | "huge_shared" : " 27 GB", | 20260506 09:22:54.356 | INFO | "malloc" : " 14 GB", | 20260506 09:22:54.356 | INFO | "pinned" : " 29 GB" | 20260506 09:22:54.356 | INFO | } | 20260506 09:22:54.356 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:22:54.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:22:54.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:54.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:54.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:54.356 | INFO | ANALYSIS : 57.0%, 3.0, 88.6ms, 0 | 20260506 09:22:54.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:54.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:54.356 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:22:54.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:54.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:54.356 | INFO | Uploaded 36GiB to GPU in 1.60543s (22.424GiB/s) | 20260506 09:22:54.356 | INFO | Downloaded 34GiB to GPU in 1.52319s (22.3216GiB/s) | 20260506 09:22:59.495 | INFO | Analyzing chunk frames = [247296,247808) | 20260506 09:22:59.495 | INFO | MemInfo: { | 20260506 09:22:59.495 | INFO | "gpu" : " 17 GB", | 20260506 09:22:59.495 | INFO | "huge" : "", | 20260506 09:22:59.495 | INFO | "huge_shared" : " 27 GB", | 20260506 09:22:59.495 | INFO | "malloc" : " 14 GB", | 20260506 09:22:59.495 | INFO | "pinned" : " 29 GB" | 20260506 09:22:59.495 | INFO | } | 20260506 09:22:59.496 | INFO | Perf report for the last 5.13935 seconds: | 20260506 09:22:59.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:22:59.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:59.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:59.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:59.496 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260506 09:22:59.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:59.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:59.496 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:22:59.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:59.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:22:59.496 | INFO | Uploaded 36GiB to GPU in 1.60478s (22.433GiB/s) | 20260506 09:22:59.496 | INFO | Downloaded 34GiB to GPU in 1.52237s (22.3337GiB/s) | 20260506 09:23:04.635 | INFO | Analyzing chunk frames = [247808,248320) | 20260506 09:23:04.635 | INFO | MemInfo: { | 20260506 09:23:04.635 | INFO | "gpu" : " 17 GB", | 20260506 09:23:04.635 | INFO | "huge" : "", | 20260506 09:23:04.635 | INFO | "huge_shared" : " 27 GB", | 20260506 09:23:04.635 | INFO | "malloc" : " 14 GB", | 20260506 09:23:04.635 | INFO | "pinned" : " 29 GB" | 20260506 09:23:04.635 | INFO | } | 20260506 09:23:04.635 | INFO | Perf report for the last 5.13945 seconds: | 20260506 09:23:04.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:23:04.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:04.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:04.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:04.635 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:23:04.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:04.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:04.635 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:23:04.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:04.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:04.636 | INFO | Uploaded 36GiB to GPU in 1.6026s (22.4634GiB/s) | 20260506 09:23:04.636 | INFO | Downloaded 34GiB to GPU in 1.52009s (22.3671GiB/s) | 20260506 09:23:09.774 | INFO | Analyzing chunk frames = [248320,248832) | 20260506 09:23:09.774 | INFO | MemInfo: { | 20260506 09:23:09.774 | INFO | "gpu" : " 17 GB", | 20260506 09:23:09.774 | INFO | "huge" : "", | 20260506 09:23:09.774 | INFO | "huge_shared" : " 27 GB", | 20260506 09:23:09.774 | INFO | "malloc" : " 14 GB", | 20260506 09:23:09.774 | INFO | "pinned" : " 29 GB" | 20260506 09:23:09.774 | INFO | } | 20260506 09:23:09.775 | INFO | Perf report for the last 5.13955 seconds: | 20260506 09:23:09.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:23:09.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:09.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:09.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:09.775 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260506 09:23:09.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:09.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:09.775 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:23:09.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:09.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:09.775 | INFO | Uploaded 36GiB to GPU in 1.60344s (22.4517GiB/s) | 20260506 09:23:09.775 | INFO | Downloaded 34GiB to GPU in 1.51988s (22.3702GiB/s) | 20260506 09:23:14.913 | INFO | Analyzing chunk frames = [248832,249344) | 20260506 09:23:14.913 | INFO | MemInfo: { | 20260506 09:23:14.913 | INFO | "gpu" : " 17 GB", | 20260506 09:23:14.914 | INFO | "huge" : "", | 20260506 09:23:14.914 | INFO | "huge_shared" : " 27 GB", | 20260506 09:23:14.914 | INFO | "malloc" : " 14 GB", | 20260506 09:23:14.914 | INFO | "pinned" : " 29 GB" | 20260506 09:23:14.914 | INFO | } | 20260506 09:23:14.914 | INFO | Perf report for the last 5.139 seconds: | 20260506 09:23:14.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:23:14.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:14.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:14.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:14.914 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:23:14.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:14.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:14.914 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:23:14.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:14.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:14.914 | INFO | Uploaded 36GiB to GPU in 1.60406s (22.443GiB/s) | 20260506 09:23:14.914 | INFO | Downloaded 34GiB to GPU in 1.52118s (22.351GiB/s) | 20260506 09:23:20.053 | INFO | Analyzing chunk frames = [249344,249856) | 20260506 09:23:20.053 | INFO | MemInfo: { | 20260506 09:23:20.053 | INFO | "gpu" : " 17 GB", | 20260506 09:23:20.053 | INFO | "huge" : "", | 20260506 09:23:20.053 | INFO | "huge_shared" : " 27 GB", | 20260506 09:23:20.053 | INFO | "malloc" : " 14 GB", | 20260506 09:23:20.053 | INFO | "pinned" : " 29 GB" | 20260506 09:23:20.053 | INFO | } | 20260506 09:23:20.053 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:23:20.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:23:20.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:20.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:20.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:20.053 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 09:23:20.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:20.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:20.053 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 09:23:20.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:20.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:20.053 | INFO | Uploaded 36GiB to GPU in 1.60394s (22.4447GiB/s) | 20260506 09:23:20.053 | INFO | Downloaded 34GiB to GPU in 1.52121s (22.3506GiB/s) | 20260506 09:23:20.150 | INFO | Starting a round of Baz IO | 20260506 09:23:23.075 | INFO | Populating RTMetrics report | 20260506 09:23:23.076 | INFO | RTMetrics report populated | 20260506 09:23:23.076 | INFO | Opening tmp RTMetrics json file | 20260506 09:23:23.078 | INFO | Serializing RTMetrics to json | 20260506 09:23:23.081 | INFO | Serialization done, swapping tmp file now... | 20260506 09:23:23.083 | INFO | RTMetrics done | 20260506 09:23:23.172 | INFO | Finished a round of Baz IO: Wrote 1.822 GiB in 3.021 seconds | 20260506 09:23:23.172 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.96% / 0.12% / 44.92% | 20260506 09:23:23.172 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.75859% / 0.000333786% / 5.5631e-05% | 20260506 09:23:23.172 | INFO | Baz IO overall: | 20260506 09:23:23.172 | INFO | BytesWritten : 92116206801 | 20260506 09:23:23.172 | INFO | NumEvents : 43322782228 | 20260506 09:23:25.192 | INFO | Analyzing chunk frames = [249856,250368) | 20260506 09:23:25.192 | INFO | MemInfo: { | 20260506 09:23:25.192 | INFO | "gpu" : " 17 GB", | 20260506 09:23:25.192 | INFO | "huge" : "", | 20260506 09:23:25.192 | INFO | "huge_shared" : " 27 GB", | 20260506 09:23:25.192 | INFO | "malloc" : " 14 GB", | 20260506 09:23:25.192 | INFO | "pinned" : " 29 GB" | 20260506 09:23:25.192 | INFO | } | 20260506 09:23:25.192 | INFO | Perf report for the last 5.13937 seconds: | 20260506 09:23:25.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:23:25.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:25.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:25.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:25.192 | INFO | ANALYSIS : 57.8%, 2.9, 88.2ms, 0 | 20260506 09:23:25.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:25.192 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 09:23:25.192 | INFO | PRE_HQ : 56.9%, 2.1, 61.8ms, 0 | 20260506 09:23:25.192 | INFO | BAZWRITER : 58.4%, 4.4, 131.9ms, 0 | 20260506 09:23:25.192 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 09:23:25.193 | INFO | Uploaded 36GiB to GPU in 1.64994s (21.819GiB/s) | 20260506 09:23:25.193 | INFO | Downloaded 38GiB to GPU in 1.71288s (22.1848GiB/s) | 20260506 09:23:30.332 | INFO | Analyzing chunk frames = [250368,250880) | 20260506 09:23:30.332 | INFO | MemInfo: { | 20260506 09:23:30.332 | INFO | "gpu" : " 17 GB", | 20260506 09:23:30.332 | INFO | "huge" : "", | 20260506 09:23:30.332 | INFO | "huge_shared" : " 27 GB", | 20260506 09:23:30.332 | INFO | "malloc" : " 14 GB", | 20260506 09:23:30.332 | INFO | "pinned" : " 29 GB" | 20260506 09:23:30.332 | INFO | } | 20260506 09:23:30.332 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:23:30.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:23:30.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:30.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:30.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:30.332 | INFO | ANALYSIS : 57.7%, 3.0, 89.9ms, 0 | 20260506 09:23:30.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:30.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:30.332 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:23:30.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:30.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:30.332 | INFO | Uploaded 36GiB to GPU in 1.60137s (22.4808GiB/s) | 20260506 09:23:30.332 | INFO | Downloaded 34GiB to GPU in 1.5189s (22.3847GiB/s) | 20260506 09:23:35.471 | INFO | Analyzing chunk frames = [250880,251392) | 20260506 09:23:35.471 | INFO | MemInfo: { | 20260506 09:23:35.471 | INFO | "gpu" : " 17 GB", | 20260506 09:23:35.471 | INFO | "huge" : "", | 20260506 09:23:35.471 | INFO | "huge_shared" : " 27 GB", | 20260506 09:23:35.471 | INFO | "malloc" : " 14 GB", | 20260506 09:23:35.471 | INFO | "pinned" : " 29 GB" | 20260506 09:23:35.471 | INFO | } | 20260506 09:23:35.471 | INFO | Perf report for the last 5.13948 seconds: | 20260506 09:23:35.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:23:35.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:35.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:35.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:35.471 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260506 09:23:35.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:35.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:35.471 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:23:35.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:35.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:35.472 | INFO | Uploaded 36GiB to GPU in 1.60533s (22.4254GiB/s) | 20260506 09:23:35.472 | INFO | Downloaded 34GiB to GPU in 1.52267s (22.3293GiB/s) | 20260506 09:23:40.610 | INFO | Analyzing chunk frames = [251392,251904) | 20260506 09:23:40.610 | INFO | MemInfo: { | 20260506 09:23:40.610 | INFO | "gpu" : " 17 GB", | 20260506 09:23:40.610 | INFO | "huge" : "", | 20260506 09:23:40.610 | INFO | "huge_shared" : " 27 GB", | 20260506 09:23:40.610 | INFO | "malloc" : " 14 GB", | 20260506 09:23:40.610 | INFO | "pinned" : " 29 GB" | 20260506 09:23:40.610 | INFO | } | 20260506 09:23:40.610 | INFO | Perf report for the last 5.13914 seconds: | 20260506 09:23:40.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:23:40.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:40.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:40.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:40.610 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:23:40.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:40.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:40.610 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:23:40.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:40.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:40.611 | INFO | Uploaded 36GiB to GPU in 1.60549s (22.423GiB/s) | 20260506 09:23:40.611 | INFO | Downloaded 34GiB to GPU in 1.52316s (22.322GiB/s) | 20260506 09:23:45.750 | INFO | Analyzing chunk frames = [251904,252416) | 20260506 09:23:45.750 | INFO | MemInfo: { | 20260506 09:23:45.750 | INFO | "gpu" : " 17 GB", | 20260506 09:23:45.750 | INFO | "huge" : "", | 20260506 09:23:45.750 | INFO | "huge_shared" : " 27 GB", | 20260506 09:23:45.750 | INFO | "malloc" : " 14 GB", | 20260506 09:23:45.750 | INFO | "pinned" : " 29 GB" | 20260506 09:23:45.750 | INFO | } | 20260506 09:23:45.750 | INFO | Perf report for the last 5.13947 seconds: | 20260506 09:23:45.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:23:45.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:45.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:45.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:45.750 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 09:23:45.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:45.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:45.750 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:23:45.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:45.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:45.750 | INFO | Uploaded 36GiB to GPU in 1.60259s (22.4636GiB/s) | 20260506 09:23:45.750 | INFO | Downloaded 34GiB to GPU in 1.51972s (22.3726GiB/s) | 20260506 09:23:50.889 | INFO | Analyzing chunk frames = [252416,252928) | 20260506 09:23:50.889 | INFO | MemInfo: { | 20260506 09:23:50.889 | INFO | "gpu" : " 17 GB", | 20260506 09:23:50.889 | INFO | "huge" : "", | 20260506 09:23:50.889 | INFO | "huge_shared" : " 27 GB", | 20260506 09:23:50.889 | INFO | "malloc" : " 14 GB", | 20260506 09:23:50.889 | INFO | "pinned" : " 29 GB" | 20260506 09:23:50.889 | INFO | } | 20260506 09:23:50.889 | INFO | Perf report for the last 5.1391 seconds: | 20260506 09:23:50.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:23:50.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:50.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:50.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:50.889 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 09:23:50.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:50.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:50.889 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:23:50.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:50.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:50.889 | INFO | Uploaded 36GiB to GPU in 1.60346s (22.4514GiB/s) | 20260506 09:23:50.889 | INFO | Downloaded 34GiB to GPU in 1.52059s (22.3597GiB/s) | 20260506 09:23:56.028 | INFO | Analyzing chunk frames = [252928,253440) | 20260506 09:23:56.028 | INFO | MemInfo: { | 20260506 09:23:56.028 | INFO | "gpu" : " 17 GB", | 20260506 09:23:56.028 | INFO | "huge" : "", | 20260506 09:23:56.028 | INFO | "huge_shared" : " 27 GB", | 20260506 09:23:56.028 | INFO | "malloc" : " 14 GB", | 20260506 09:23:56.028 | INFO | "pinned" : " 29 GB" | 20260506 09:23:56.028 | INFO | } | 20260506 09:23:56.028 | INFO | Perf report for the last 5.13937 seconds: | 20260506 09:23:56.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:23:56.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:56.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:56.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:56.028 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:23:56.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:56.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:56.028 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:23:56.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:56.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:23:56.029 | INFO | Uploaded 36GiB to GPU in 1.60394s (22.4448GiB/s) | 20260506 09:23:56.029 | INFO | Downloaded 34GiB to GPU in 1.52121s (22.3506GiB/s) | 20260506 09:24:01.168 | INFO | Analyzing chunk frames = [253440,253952) | 20260506 09:24:01.168 | INFO | MemInfo: { | 20260506 09:24:01.168 | INFO | "gpu" : " 17 GB", | 20260506 09:24:01.168 | INFO | "huge" : "", | 20260506 09:24:01.168 | INFO | "huge_shared" : " 27 GB", | 20260506 09:24:01.168 | INFO | "malloc" : " 14 GB", | 20260506 09:24:01.168 | INFO | "pinned" : " 29 GB" | 20260506 09:24:01.168 | INFO | } | 20260506 09:24:01.168 | INFO | Perf report for the last 5.13936 seconds: | 20260506 09:24:01.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:24:01.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:01.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:01.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:01.168 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:24:01.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:01.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:01.168 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 09:24:01.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:01.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:01.168 | INFO | Uploaded 36GiB to GPU in 1.60459s (22.4356GiB/s) | 20260506 09:24:01.168 | INFO | Downloaded 34GiB to GPU in 1.52232s (22.3344GiB/s) | 20260506 09:24:01.168 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:24:01.168 | INFO | Total mem usage: 15.7 GiB | 20260506 09:24:01.168 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.38%) | 20260506 09:24:01.168 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -6.66%) | 20260506 09:24:01.168 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:24:01.168 | INFO | High Water Breakdown: | 20260506 09:24:01.168 | INFO | 58.8% : 8.68 GiB ( 92.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:24:01.168 | INFO | 41.1% : 6.06 GiB ( 90.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:24:01.168 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:24:01.168 | INFO | ------------------------------------------ | 20260506 09:24:01.168 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:24:01.168 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:24:01.168 | INFO | Total mem usage: 26.5 GiB | 20260506 09:24:01.168 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:24:01.168 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:24:01.168 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:24:01.168 | INFO | High Water Breakdown: | 20260506 09:24:01.168 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:24:01.168 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:24:01.168 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:24:01.168 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:24:01.168 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:24:01.168 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:24:01.168 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:24:01.168 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:24:01.168 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:24:01.168 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:24:01.168 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:24:01.168 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:24:01.168 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:24:01.168 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:24:01.168 | INFO | ------------------------------------------ | 20260506 09:24:01.168 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:24:01.168 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:24:01.168 | INFO | Total mem usage: 25.6 GiB | 20260506 09:24:01.168 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:24:01.168 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:24:01.168 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:24:01.168 | INFO | High Water Breakdown: | 20260506 09:24:01.168 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:24:01.168 | INFO | ------------------------------------------ | 20260506 09:24:01.169 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:24:01.169 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:24:01.169 | INFO | Total mem usage: 15.7 GiB | 20260506 09:24:01.169 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:24:01.169 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:24:01.169 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:24:01.169 | INFO | High Water Breakdown: | 20260506 09:24:01.169 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:24:01.169 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:24:01.169 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:24:01.169 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:24:01.169 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:24:01.169 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:24:01.169 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:24:01.169 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:24:01.169 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:24:01.169 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:24:01.169 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:24:01.169 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:24:01.169 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:24:01.169 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:24:01.169 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:24:01.169 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:24:01.169 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:24:01.169 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:24:01.169 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:24:01.169 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:24:01.169 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:24:01.169 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:24:01.169 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:24:01.169 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:24:01.169 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:24:01.169 | INFO | ------------------------------------------ | 20260506 09:24:01.169 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:24:01.169 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:24:01.169 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:24:01.169 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:24:01.169 | INFO | PerfReport: Full Execution -- 3.03e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 09:24:01.169 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / -nan, 4.53e-03 / 0.00e+00 | 20260506 09:24:01.169 | INFO | PerfReport: 10.88%, Baseline -- 3.29e+00 / -nan, 1.74e-02 / 0.00e+00 | 20260506 09:24:01.169 | INFO | PerfReport: 3.05%, Histogram -- 9.21e-01 / -nan, 5.15e-02 / 0.00e+00 | 20260506 09:24:01.169 | INFO | PerfReport: 16.03%, DME -- 3.88e+01 / -nan, 3.13e+00 / 0.00e+00 | 20260506 09:24:01.169 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.72e+01 / -nan, 4.18e-02 / 0.00e+00 | 20260506 09:24:01.169 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.28e+00 / -nan, 1.38e-01 / 0.00e+00 | 20260506 09:24:01.169 | INFO | PerfReport: 3.86%, Metrics -- 1.17e+00 / -nan, 1.84e-01 / 0.00e+00 | 20260506 09:24:01.169 | INFO | PerfReport: 1.51%, PulseSerializing -- 4.56e-01 / -nan, 6.60e-02 / 0.00e+00 | 20260506 09:24:01.169 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:24:01.169 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:24:01.169 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:24:01.169 | INFO | PerfReport: Full Execution -- 9.06e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 09:24:01.169 | INFO | PerfReport: 18.52%, Upload -- 1.68e+01 / 0.00e+00, 4.38e-01 / 0.00e+00 | 20260506 09:24:01.169 | INFO | PerfReport: 17.78%, Download -- 1.61e+01 / 0.00e+00, 7.73e-01 / 0.00e+00 | 20260506 09:24:01.169 | INFO | PerfReport: 33.41%, Compute -- 3.03e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 09:24:01.169 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260506 09:24:01.169 | INFO | PerfReport: 0.00%, LockDownload -- 5.00e-04 / 0.00e+00, 1.19e-04 / 0.00e+00 | 20260506 09:24:01.169 | INFO | PerfReport: 29.72%, LockCompute -- 2.69e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 09:24:01.169 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260506 09:24:01.265 | INFO | Starting a round of Baz IO | 20260506 09:24:04.191 | INFO | Populating RTMetrics report | 20260506 09:24:04.192 | INFO | RTMetrics report populated | 20260506 09:24:04.192 | INFO | Opening tmp RTMetrics json file | 20260506 09:24:04.197 | INFO | Serializing RTMetrics to json | 20260506 09:24:04.212 | INFO | Serialization done, swapping tmp file now... | 20260506 09:24:04.228 | INFO | RTMetrics done | 20260506 09:24:04.292 | INFO | Finished a round of Baz IO: Wrote 1.824 GiB in 3.026 seconds | 20260506 09:24:04.292 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.84% / 0.12% / 45.04% | 20260506 09:24:04.292 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.71745% / 0.000353654% / 4.76837e-05% | 20260506 09:24:04.292 | INFO | Baz IO overall: | 20260506 09:24:04.292 | INFO | BytesWritten : 94074573060 | 20260506 09:24:04.292 | INFO | NumEvents : 44273592759 | 20260506 09:24:06.307 | INFO | Analyzing chunk frames = [253952,254464) | 20260506 09:24:06.307 | INFO | MemInfo: { | 20260506 09:24:06.307 | INFO | "gpu" : " 17 GB", | 20260506 09:24:06.307 | INFO | "huge" : "", | 20260506 09:24:06.307 | INFO | "huge_shared" : " 27 GB", | 20260506 09:24:06.307 | INFO | "malloc" : " 14 GB", | 20260506 09:24:06.307 | INFO | "pinned" : " 29 GB" | 20260506 09:24:06.307 | INFO | } | 20260506 09:24:06.307 | INFO | Perf report for the last 5.13952 seconds: | 20260506 09:24:06.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:24:06.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:06.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:06.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:06.307 | INFO | ANALYSIS : 57.9%, 2.9, 88.8ms, 0 | 20260506 09:24:06.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:06.307 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 09:24:06.307 | INFO | PRE_HQ : 57.0%, 2.1, 61.9ms, 0 | 20260506 09:24:06.307 | INFO | BAZWRITER : 58.5%, 4.3, 129.4ms, 0 | 20260506 09:24:06.307 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 09:24:06.308 | INFO | Uploaded 36GiB to GPU in 1.64983s (21.8205GiB/s) | 20260506 09:24:06.308 | INFO | Downloaded 38GiB to GPU in 1.71357s (22.1759GiB/s) | 20260506 09:24:11.446 | INFO | Analyzing chunk frames = [254464,254976) | 20260506 09:24:11.446 | INFO | MemInfo: { | 20260506 09:24:11.446 | INFO | "gpu" : " 17 GB", | 20260506 09:24:11.446 | INFO | "huge" : "", | 20260506 09:24:11.446 | INFO | "huge_shared" : " 27 GB", | 20260506 09:24:11.446 | INFO | "malloc" : " 14 GB", | 20260506 09:24:11.446 | INFO | "pinned" : " 29 GB" | 20260506 09:24:11.446 | INFO | } | 20260506 09:24:11.446 | INFO | Perf report for the last 5.13905 seconds: | 20260506 09:24:11.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:24:11.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:11.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:11.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:11.446 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260506 09:24:11.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:11.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:11.446 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:24:11.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:11.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:11.447 | INFO | Uploaded 36GiB to GPU in 1.60188s (22.4736GiB/s) | 20260506 09:24:11.447 | INFO | Downloaded 34GiB to GPU in 1.51875s (22.3869GiB/s) | 20260506 09:24:16.586 | INFO | Analyzing chunk frames = [254976,255488) | 20260506 09:24:16.586 | INFO | MemInfo: { | 20260506 09:24:16.586 | INFO | "gpu" : " 17 GB", | 20260506 09:24:16.586 | INFO | "huge" : "", | 20260506 09:24:16.586 | INFO | "huge_shared" : " 27 GB", | 20260506 09:24:16.586 | INFO | "malloc" : " 14 GB", | 20260506 09:24:16.586 | INFO | "pinned" : " 29 GB" | 20260506 09:24:16.586 | INFO | } | 20260506 09:24:16.586 | INFO | Perf report for the last 5.13957 seconds: | 20260506 09:24:16.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:24:16.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:16.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:16.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:16.586 | INFO | ANALYSIS : 57.1%, 3.0, 88.7ms, 0 | 20260506 09:24:16.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:16.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:16.586 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:24:16.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:16.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:16.586 | INFO | Uploaded 36GiB to GPU in 1.60561s (22.4214GiB/s) | 20260506 09:24:16.586 | INFO | Downloaded 34GiB to GPU in 1.52286s (22.3264GiB/s) | 20260506 09:24:21.725 | INFO | Analyzing chunk frames = [255488,256000) | 20260506 09:24:21.725 | INFO | MemInfo: { | 20260506 09:24:21.725 | INFO | "gpu" : " 17 GB", | 20260506 09:24:21.725 | INFO | "huge" : "", | 20260506 09:24:21.725 | INFO | "huge_shared" : " 27 GB", | 20260506 09:24:21.725 | INFO | "malloc" : " 14 GB", | 20260506 09:24:21.725 | INFO | "pinned" : " 29 GB" | 20260506 09:24:21.725 | INFO | } | 20260506 09:24:21.725 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:24:21.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:24:21.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:21.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:21.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:21.725 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:24:21.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:21.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:21.725 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:24:21.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:21.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:21.725 | INFO | Uploaded 36GiB to GPU in 1.60506s (22.429GiB/s) | 20260506 09:24:21.726 | INFO | Downloaded 34GiB to GPU in 1.5226s (22.3302GiB/s) | 20260506 09:24:26.864 | INFO | Analyzing chunk frames = [256000,256512) | 20260506 09:24:26.864 | INFO | MemInfo: { | 20260506 09:24:26.864 | INFO | "gpu" : " 17 GB", | 20260506 09:24:26.864 | INFO | "huge" : "", | 20260506 09:24:26.864 | INFO | "huge_shared" : " 27 GB", | 20260506 09:24:26.864 | INFO | "malloc" : " 14 GB", | 20260506 09:24:26.864 | INFO | "pinned" : " 29 GB" | 20260506 09:24:26.864 | INFO | } | 20260506 09:24:26.864 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:24:26.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:24:26.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:26.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:26.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:26.865 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:24:26.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:26.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:26.865 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:24:26.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:26.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:26.865 | INFO | Uploaded 36GiB to GPU in 1.60304s (22.4574GiB/s) | 20260506 09:24:26.865 | INFO | Downloaded 34GiB to GPU in 1.52034s (22.3635GiB/s) | 20260506 09:24:32.004 | INFO | Analyzing chunk frames = [256512,257024) | 20260506 09:24:32.004 | INFO | MemInfo: { | 20260506 09:24:32.004 | INFO | "gpu" : " 17 GB", | 20260506 09:24:32.004 | INFO | "huge" : "", | 20260506 09:24:32.004 | INFO | "huge_shared" : " 27 GB", | 20260506 09:24:32.004 | INFO | "malloc" : " 14 GB", | 20260506 09:24:32.004 | INFO | "pinned" : " 29 GB" | 20260506 09:24:32.004 | INFO | } | 20260506 09:24:32.004 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:24:32.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:24:32.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:32.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:32.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:32.004 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 09:24:32.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:32.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:32.004 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:24:32.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:32.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:32.004 | INFO | Uploaded 36GiB to GPU in 1.60423s (22.4407GiB/s) | 20260506 09:24:32.004 | INFO | Downloaded 34GiB to GPU in 1.52106s (22.3529GiB/s) | 20260506 09:24:37.143 | INFO | Analyzing chunk frames = [257024,257536) | 20260506 09:24:37.143 | INFO | MemInfo: { | 20260506 09:24:37.143 | INFO | "gpu" : " 17 GB", | 20260506 09:24:37.143 | INFO | "huge" : "", | 20260506 09:24:37.143 | INFO | "huge_shared" : " 27 GB", | 20260506 09:24:37.143 | INFO | "malloc" : " 14 GB", | 20260506 09:24:37.143 | INFO | "pinned" : " 29 GB" | 20260506 09:24:37.143 | INFO | } | 20260506 09:24:37.143 | INFO | Perf report for the last 5.13946 seconds: | 20260506 09:24:37.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:24:37.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:37.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:37.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:37.143 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:24:37.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:37.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:37.143 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:24:37.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:37.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:37.144 | INFO | Uploaded 36GiB to GPU in 1.60403s (22.4435GiB/s) | 20260506 09:24:37.144 | INFO | Downloaded 34GiB to GPU in 1.52159s (22.3451GiB/s) | 20260506 09:24:42.282 | INFO | Analyzing chunk frames = [257536,258048) | 20260506 09:24:42.282 | INFO | MemInfo: { | 20260506 09:24:42.282 | INFO | "gpu" : " 17 GB", | 20260506 09:24:42.282 | INFO | "huge" : "", | 20260506 09:24:42.282 | INFO | "huge_shared" : " 27 GB", | 20260506 09:24:42.282 | INFO | "malloc" : " 14 GB", | 20260506 09:24:42.282 | INFO | "pinned" : " 29 GB" | 20260506 09:24:42.282 | INFO | } | 20260506 09:24:42.283 | INFO | Perf report for the last 5.13926 seconds: | 20260506 09:24:42.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:24:42.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:42.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:42.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:42.283 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:24:42.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:42.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:42.283 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 09:24:42.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:42.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:42.283 | INFO | Uploaded 36GiB to GPU in 1.60445s (22.4376GiB/s) | 20260506 09:24:42.283 | INFO | Downloaded 34GiB to GPU in 1.52115s (22.3515GiB/s) | 20260506 09:24:42.379 | INFO | Starting a round of Baz IO | 20260506 09:24:45.308 | INFO | Populating RTMetrics report | 20260506 09:24:45.308 | INFO | RTMetrics report populated | 20260506 09:24:45.308 | INFO | Opening tmp RTMetrics json file | 20260506 09:24:45.345 | INFO | Serializing RTMetrics to json | 20260506 09:24:45.370 | INFO | Serialization done, swapping tmp file now... | 20260506 09:24:45.372 | INFO | RTMetrics done | 20260506 09:24:45.661 | INFO | Finished a round of Baz IO: Wrote 1.827 GiB in 3.281 seconds | 20260506 09:24:45.661 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.73% / 0.11% / 45.16% | 20260506 09:24:45.661 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.66529% / 0.000321865% / 4.37101e-05% | 20260506 09:24:45.661 | INFO | Baz IO overall: | 20260506 09:24:45.661 | INFO | BytesWritten : 96036418828 | 20260506 09:24:45.661 | INFO | NumEvents : 45225885843 | 20260506 09:24:47.422 | INFO | Analyzing chunk frames = [258048,258560) | 20260506 09:24:47.422 | INFO | MemInfo: { | 20260506 09:24:47.422 | INFO | "gpu" : " 17 GB", | 20260506 09:24:47.422 | INFO | "huge" : "", | 20260506 09:24:47.422 | INFO | "huge_shared" : " 27 GB", | 20260506 09:24:47.422 | INFO | "malloc" : " 14 GB", | 20260506 09:24:47.422 | INFO | "pinned" : " 29 GB" | 20260506 09:24:47.422 | INFO | } | 20260506 09:24:47.422 | INFO | Perf report for the last 5.13924 seconds: | 20260506 09:24:47.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:24:47.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:47.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:47.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:47.422 | INFO | ANALYSIS : 57.9%, 2.8, 85.6ms, 0 | 20260506 09:24:47.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:47.422 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 09:24:47.422 | INFO | PRE_HQ : 57.0%, 2.1, 61.7ms, 0 | 20260506 09:24:47.422 | INFO | BAZWRITER : 63.4%, 4.8, 155.6ms, 0 | 20260506 09:24:47.422 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 09:24:47.422 | INFO | Uploaded 36GiB to GPU in 1.67444s (21.4997GiB/s) | 20260506 09:24:47.422 | INFO | Downloaded 38GiB to GPU in 1.73954s (21.8448GiB/s) | 20260506 09:24:52.561 | INFO | Analyzing chunk frames = [258560,259072) | 20260506 09:24:52.561 | INFO | MemInfo: { | 20260506 09:24:52.561 | INFO | "gpu" : " 17 GB", | 20260506 09:24:52.561 | INFO | "huge" : "", | 20260506 09:24:52.561 | INFO | "huge_shared" : " 27 GB", | 20260506 09:24:52.561 | INFO | "malloc" : " 14 GB", | 20260506 09:24:52.561 | INFO | "pinned" : " 29 GB" | 20260506 09:24:52.561 | INFO | } | 20260506 09:24:52.561 | INFO | Perf report for the last 5.13938 seconds: | 20260506 09:24:52.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:24:52.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:52.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:52.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:52.561 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260506 09:24:52.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:52.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:52.561 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:24:52.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:52.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:52.561 | INFO | Uploaded 36GiB to GPU in 1.60165s (22.4769GiB/s) | 20260506 09:24:52.561 | INFO | Downloaded 34GiB to GPU in 1.51802s (22.3976GiB/s) | 20260506 09:24:57.700 | INFO | Analyzing chunk frames = [259072,259584) | 20260506 09:24:57.700 | INFO | MemInfo: { | 20260506 09:24:57.700 | INFO | "gpu" : " 17 GB", | 20260506 09:24:57.700 | INFO | "huge" : "", | 20260506 09:24:57.700 | INFO | "huge_shared" : " 27 GB", | 20260506 09:24:57.700 | INFO | "malloc" : " 14 GB", | 20260506 09:24:57.700 | INFO | "pinned" : " 29 GB" | 20260506 09:24:57.701 | INFO | } | 20260506 09:24:57.701 | INFO | Perf report for the last 5.13937 seconds: | 20260506 09:24:57.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:24:57.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:57.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:57.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:57.701 | INFO | ANALYSIS : 57.0%, 3.0, 88.6ms, 0 | 20260506 09:24:57.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:57.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:57.701 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:24:57.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:57.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:24:57.701 | INFO | Uploaded 36GiB to GPU in 1.60542s (22.424GiB/s) | 20260506 09:24:57.701 | INFO | Downloaded 34GiB to GPU in 1.52312s (22.3226GiB/s) | 20260506 09:25:02.840 | INFO | Analyzing chunk frames = [259584,260096) | 20260506 09:25:02.840 | INFO | MemInfo: { | 20260506 09:25:02.840 | INFO | "gpu" : " 17 GB", | 20260506 09:25:02.840 | INFO | "huge" : "", | 20260506 09:25:02.840 | INFO | "huge_shared" : " 27 GB", | 20260506 09:25:02.840 | INFO | "malloc" : " 14 GB", | 20260506 09:25:02.840 | INFO | "pinned" : " 29 GB" | 20260506 09:25:02.840 | INFO | } | 20260506 09:25:02.840 | INFO | Perf report for the last 5.13944 seconds: | 20260506 09:25:02.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:25:02.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:02.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:02.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:02.840 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:25:02.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:02.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:02.840 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:25:02.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:02.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:02.840 | INFO | Uploaded 36GiB to GPU in 1.60485s (22.4319GiB/s) | 20260506 09:25:02.840 | INFO | Downloaded 34GiB to GPU in 1.52234s (22.334GiB/s) | 20260506 09:25:07.979 | INFO | Analyzing chunk frames = [260096,260608) | 20260506 09:25:07.979 | INFO | MemInfo: { | 20260506 09:25:07.979 | INFO | "gpu" : " 17 GB", | 20260506 09:25:07.979 | INFO | "huge" : "", | 20260506 09:25:07.979 | INFO | "huge_shared" : " 27 GB", | 20260506 09:25:07.979 | INFO | "malloc" : " 14 GB", | 20260506 09:25:07.979 | INFO | "pinned" : " 29 GB" | 20260506 09:25:07.979 | INFO | } | 20260506 09:25:07.979 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:25:07.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:25:07.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:07.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:07.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:07.979 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 09:25:07.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:07.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:07.979 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:25:07.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:07.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:07.980 | INFO | Uploaded 36GiB to GPU in 1.60283s (22.4603GiB/s) | 20260506 09:25:07.980 | INFO | Downloaded 34GiB to GPU in 1.52064s (22.359GiB/s) | 20260506 09:25:13.119 | INFO | Analyzing chunk frames = [260608,261120) | 20260506 09:25:13.119 | INFO | MemInfo: { | 20260506 09:25:13.119 | INFO | "gpu" : " 17 GB", | 20260506 09:25:13.119 | INFO | "huge" : "", | 20260506 09:25:13.119 | INFO | "huge_shared" : " 27 GB", | 20260506 09:25:13.119 | INFO | "malloc" : " 14 GB", | 20260506 09:25:13.119 | INFO | "pinned" : " 29 GB" | 20260506 09:25:13.119 | INFO | } | 20260506 09:25:13.119 | INFO | Perf report for the last 5.13942 seconds: | 20260506 09:25:13.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:25:13.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:13.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:13.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:13.119 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260506 09:25:13.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:13.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:13.119 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:25:13.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:13.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:13.119 | INFO | Uploaded 36GiB to GPU in 1.60264s (22.4629GiB/s) | 20260506 09:25:13.119 | INFO | Downloaded 34GiB to GPU in 1.51983s (22.371GiB/s) | 20260506 09:25:18.258 | INFO | Analyzing chunk frames = [261120,261632) | 20260506 09:25:18.258 | INFO | MemInfo: { | 20260506 09:25:18.258 | INFO | "gpu" : " 17 GB", | 20260506 09:25:18.258 | INFO | "huge" : "", | 20260506 09:25:18.258 | INFO | "huge_shared" : " 27 GB", | 20260506 09:25:18.258 | INFO | "malloc" : " 14 GB", | 20260506 09:25:18.258 | INFO | "pinned" : " 29 GB" | 20260506 09:25:18.258 | INFO | } | 20260506 09:25:18.258 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:25:18.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:25:18.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:18.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:18.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:18.258 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:25:18.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:18.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:18.258 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:25:18.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:18.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:18.258 | INFO | Uploaded 36GiB to GPU in 1.60416s (22.4417GiB/s) | 20260506 09:25:18.258 | INFO | Downloaded 34GiB to GPU in 1.52126s (22.3499GiB/s) | 20260506 09:25:23.397 | INFO | Analyzing chunk frames = [261632,262144) | 20260506 09:25:23.397 | INFO | MemInfo: { | 20260506 09:25:23.397 | INFO | "gpu" : " 17 GB", | 20260506 09:25:23.397 | INFO | "huge" : "", | 20260506 09:25:23.397 | INFO | "huge_shared" : " 27 GB", | 20260506 09:25:23.397 | INFO | "malloc" : " 14 GB", | 20260506 09:25:23.397 | INFO | "pinned" : " 29 GB" | 20260506 09:25:23.397 | INFO | } | 20260506 09:25:23.397 | INFO | Perf report for the last 5.13924 seconds: | 20260506 09:25:23.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:25:23.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:23.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:23.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:23.397 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 09:25:23.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:23.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:23.397 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 09:25:23.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:23.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:23.398 | INFO | Uploaded 36GiB to GPU in 1.60284s (22.4602GiB/s) | 20260506 09:25:23.398 | INFO | Downloaded 34GiB to GPU in 1.52223s (22.3357GiB/s) | 20260506 09:25:23.398 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:25:23.398 | INFO | Total mem usage: 15.7 GiB | 20260506 09:25:23.398 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.38%) | 20260506 09:25:23.398 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -0.783%) | 20260506 09:25:23.398 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:25:23.398 | INFO | High Water Breakdown: | 20260506 09:25:23.398 | INFO | 59.8% : 9.34 GiB ( 99.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:25:23.398 | INFO | 40.1% : 6.26 GiB ( 93.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:25:23.398 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:25:23.398 | INFO | ------------------------------------------ | 20260506 09:25:23.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:25:23.398 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:25:23.398 | INFO | Total mem usage: 26.5 GiB | 20260506 09:25:23.398 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:25:23.398 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:25:23.398 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:25:23.398 | INFO | High Water Breakdown: | 20260506 09:25:23.398 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:25:23.398 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:25:23.398 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:25:23.398 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:25:23.398 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:25:23.398 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:25:23.398 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:25:23.398 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:25:23.398 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:25:23.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:25:23.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:25:23.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:25:23.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:25:23.398 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:25:23.398 | INFO | ------------------------------------------ | 20260506 09:25:23.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:25:23.398 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:25:23.398 | INFO | Total mem usage: 25.6 GiB | 20260506 09:25:23.398 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:25:23.398 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:25:23.398 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:25:23.398 | INFO | High Water Breakdown: | 20260506 09:25:23.398 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:25:23.398 | INFO | ------------------------------------------ | 20260506 09:25:23.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:25:23.398 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:25:23.398 | INFO | Total mem usage: 15.7 GiB | 20260506 09:25:23.398 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:25:23.398 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:25:23.398 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:25:23.398 | INFO | High Water Breakdown: | 20260506 09:25:23.399 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:25:23.399 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:25:23.399 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:25:23.399 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:25:23.399 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:25:23.399 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:25:23.399 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:25:23.399 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:25:23.399 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:25:23.399 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:25:23.399 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:25:23.399 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:25:23.399 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:25:23.399 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:25:23.399 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:25:23.399 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:25:23.399 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:25:23.399 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:25:23.399 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:25:23.399 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:25:23.399 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:25:23.399 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:25:23.399 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:25:23.399 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:25:23.399 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:25:23.399 | INFO | ------------------------------------------ | 20260506 09:25:23.399 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:25:23.399 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:25:23.399 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:25:23.399 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:25:23.399 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 09:25:23.399 | INFO | PerfReport: 0.28%, AggregateStats -- 8.62e-02 / -nan, 6.70e-03 / 0.00e+00 | 20260506 09:25:23.399 | INFO | PerfReport: 10.89%, Baseline -- 3.29e+00 / -nan, 1.75e-02 / 0.00e+00 | 20260506 09:25:23.399 | INFO | PerfReport: 3.05%, Histogram -- 9.22e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260506 09:25:23.399 | INFO | PerfReport: 16.03%, DME -- 3.88e+01 / -nan, 3.12e+00 / 0.00e+00 | 20260506 09:25:23.399 | INFO | PerfReport: 56.85%, FrameLabeling -- 1.72e+01 / -nan, 4.22e-02 / 0.00e+00 | 20260506 09:25:23.399 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.28e+00 / -nan, 1.35e-01 / 0.00e+00 | 20260506 09:25:23.399 | INFO | PerfReport: 3.85%, Metrics -- 1.17e+00 / -nan, 1.80e-01 / 0.00e+00 | 20260506 09:25:23.399 | INFO | PerfReport: 1.50%, PulseSerializing -- 4.55e-01 / -nan, 6.58e-02 / 0.00e+00 | 20260506 09:25:23.399 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:25:23.399 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:25:23.399 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:25:23.399 | INFO | PerfReport: Full Execution -- 9.05e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 09:25:23.399 | INFO | PerfReport: 18.55%, Upload -- 1.68e+01 / 0.00e+00, 4.63e-01 / 0.00e+00 | 20260506 09:25:23.399 | INFO | PerfReport: 17.81%, Download -- 1.61e+01 / 0.00e+00, 8.08e-01 / 0.00e+00 | 20260506 09:25:23.399 | INFO | PerfReport: 33.43%, Compute -- 3.03e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 09:25:23.399 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:25:23.399 | INFO | PerfReport: 0.00%, LockDownload -- 4.95e-04 / 0.00e+00, 1.79e-04 / 0.00e+00 | 20260506 09:25:23.399 | INFO | PerfReport: 29.64%, LockCompute -- 2.68e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 09:25:23.399 | INFO | PerfReport: 0.01%, Misc -- 1.22e-02 / 0.00e+00, 9.86e-03 / 0.00e+00 | 20260506 09:25:23.494 | INFO | Starting a round of Baz IO | 20260506 09:25:26.420 | INFO | Populating RTMetrics report | 20260506 09:25:26.420 | INFO | RTMetrics report populated | 20260506 09:25:26.420 | INFO | Opening tmp RTMetrics json file | 20260506 09:25:26.470 | INFO | Serializing RTMetrics to json | 20260506 09:25:26.479 | INFO | Serialization done, swapping tmp file now... | 20260506 09:25:26.482 | INFO | RTMetrics done | 20260506 09:25:26.702 | INFO | Finished a round of Baz IO: Wrote 1.832 GiB in 3.207 seconds | 20260506 09:25:26.702 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.61% / 0.11% / 45.27% | 20260506 09:25:26.702 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.63107% / 0.00038147% / 2.38419e-05% | 20260506 09:25:26.702 | INFO | Baz IO overall: | 20260506 09:25:26.702 | INFO | BytesWritten : 98003190917 | 20260506 09:25:26.702 | INFO | NumEvents : 46180330849 | 20260506 09:25:28.536 | INFO | Analyzing chunk frames = [262144,262656) | 20260506 09:25:28.536 | INFO | MemInfo: { | 20260506 09:25:28.536 | INFO | "gpu" : " 17 GB", | 20260506 09:25:28.536 | INFO | "huge" : "", | 20260506 09:25:28.536 | INFO | "huge_shared" : " 27 GB", | 20260506 09:25:28.536 | INFO | "malloc" : " 14 GB", | 20260506 09:25:28.537 | INFO | "pinned" : " 29 GB" | 20260506 09:25:28.537 | INFO | } | 20260506 09:25:28.537 | INFO | Perf report for the last 5.13921 seconds: | 20260506 09:25:28.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:25:28.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:28.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:28.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:28.537 | INFO | ANALYSIS : 57.8%, 2.9, 86.6ms, 0 | 20260506 09:25:28.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:28.537 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260506 09:25:28.537 | INFO | PRE_HQ : 57.0%, 2.1, 61.7ms, 0 | 20260506 09:25:28.537 | INFO | BAZWRITER : 62.0%, 4.6, 147.8ms, 0 | 20260506 09:25:28.537 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 09:25:28.537 | INFO | Uploaded 36GiB to GPU in 1.66356s (21.6404GiB/s) | 20260506 09:25:28.537 | INFO | Downloaded 38GiB to GPU in 1.7285s (21.9844GiB/s) | 20260506 09:25:33.676 | INFO | Analyzing chunk frames = [262656,263168) | 20260506 09:25:33.676 | INFO | MemInfo: { | 20260506 09:25:33.676 | INFO | "gpu" : " 17 GB", | 20260506 09:25:33.676 | INFO | "huge" : "", | 20260506 09:25:33.676 | INFO | "huge_shared" : " 27 GB", | 20260506 09:25:33.676 | INFO | "malloc" : " 14 GB", | 20260506 09:25:33.676 | INFO | "pinned" : " 29 GB" | 20260506 09:25:33.676 | INFO | } | 20260506 09:25:33.676 | INFO | Perf report for the last 5.13947 seconds: | 20260506 09:25:33.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:25:33.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:33.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:33.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:33.676 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260506 09:25:33.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:33.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:33.676 | INFO | PRE_HQ : 30.0%, 1.0, 15.7ms, 0 | 20260506 09:25:33.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:33.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:33.677 | INFO | Uploaded 36GiB to GPU in 1.60205s (22.4712GiB/s) | 20260506 09:25:33.677 | INFO | Downloaded 34GiB to GPU in 1.51874s (22.3869GiB/s) | 20260506 09:25:38.816 | INFO | Analyzing chunk frames = [263168,263680) | 20260506 09:25:38.816 | INFO | MemInfo: { | 20260506 09:25:38.816 | INFO | "gpu" : " 17 GB", | 20260506 09:25:38.816 | INFO | "huge" : "", | 20260506 09:25:38.816 | INFO | "huge_shared" : " 27 GB", | 20260506 09:25:38.816 | INFO | "malloc" : " 14 GB", | 20260506 09:25:38.816 | INFO | "pinned" : " 29 GB" | 20260506 09:25:38.816 | INFO | } | 20260506 09:25:38.816 | INFO | Perf report for the last 5.13971 seconds: | 20260506 09:25:38.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:25:38.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:38.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:38.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:38.816 | INFO | ANALYSIS : 57.0%, 3.0, 88.5ms, 0 | 20260506 09:25:38.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:38.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:38.816 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:25:38.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:38.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:38.816 | INFO | Uploaded 36GiB to GPU in 1.60588s (22.4176GiB/s) | 20260506 09:25:38.816 | INFO | Downloaded 34GiB to GPU in 1.52318s (22.3218GiB/s) | 20260506 09:25:43.955 | INFO | Analyzing chunk frames = [263680,264192) | 20260506 09:25:43.955 | INFO | MemInfo: { | 20260506 09:25:43.955 | INFO | "gpu" : " 17 GB", | 20260506 09:25:43.955 | INFO | "huge" : "", | 20260506 09:25:43.955 | INFO | "huge_shared" : " 27 GB", | 20260506 09:25:43.955 | INFO | "malloc" : " 14 GB", | 20260506 09:25:43.955 | INFO | "pinned" : " 29 GB" | 20260506 09:25:43.955 | INFO | } | 20260506 09:25:43.955 | INFO | Perf report for the last 5.13897 seconds: | 20260506 09:25:43.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:25:43.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:43.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:43.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:43.955 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260506 09:25:43.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:43.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:43.955 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:25:43.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:43.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:43.955 | INFO | Uploaded 36GiB to GPU in 1.60525s (22.4264GiB/s) | 20260506 09:25:43.955 | INFO | Downloaded 34GiB to GPU in 1.52167s (22.3439GiB/s) | 20260506 09:25:49.094 | INFO | Analyzing chunk frames = [264192,264704) | 20260506 09:25:49.094 | INFO | MemInfo: { | 20260506 09:25:49.094 | INFO | "gpu" : " 17 GB", | 20260506 09:25:49.094 | INFO | "huge" : "", | 20260506 09:25:49.094 | INFO | "huge_shared" : " 27 GB", | 20260506 09:25:49.094 | INFO | "malloc" : " 14 GB", | 20260506 09:25:49.094 | INFO | "pinned" : " 29 GB" | 20260506 09:25:49.094 | INFO | } | 20260506 09:25:49.094 | INFO | Perf report for the last 5.13915 seconds: | 20260506 09:25:49.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:25:49.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:49.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:49.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:49.094 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:25:49.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:49.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:49.094 | INFO | PRE_HQ : 30.0%, 1.0, 15.5ms, 0 | 20260506 09:25:49.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:49.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:49.094 | INFO | Uploaded 36GiB to GPU in 1.60324s (22.4546GiB/s) | 20260506 09:25:49.094 | INFO | Downloaded 34GiB to GPU in 1.52057s (22.36GiB/s) | 20260506 09:25:54.233 | INFO | Analyzing chunk frames = [264704,265216) | 20260506 09:25:54.233 | INFO | MemInfo: { | 20260506 09:25:54.233 | INFO | "gpu" : " 17 GB", | 20260506 09:25:54.233 | INFO | "huge" : "", | 20260506 09:25:54.233 | INFO | "huge_shared" : " 27 GB", | 20260506 09:25:54.233 | INFO | "malloc" : " 14 GB", | 20260506 09:25:54.233 | INFO | "pinned" : " 29 GB" | 20260506 09:25:54.233 | INFO | } | 20260506 09:25:54.233 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:25:54.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:25:54.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:54.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:54.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:54.233 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260506 09:25:54.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:54.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:54.233 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:25:54.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:54.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:54.234 | INFO | Uploaded 36GiB to GPU in 1.60395s (22.4445GiB/s) | 20260506 09:25:54.234 | INFO | Downloaded 34GiB to GPU in 1.52022s (22.3652GiB/s) | 20260506 09:25:59.373 | INFO | Analyzing chunk frames = [265216,265728) | 20260506 09:25:59.373 | INFO | MemInfo: { | 20260506 09:25:59.373 | INFO | "gpu" : " 17 GB", | 20260506 09:25:59.373 | INFO | "huge" : "", | 20260506 09:25:59.373 | INFO | "huge_shared" : " 27 GB", | 20260506 09:25:59.373 | INFO | "malloc" : " 14 GB", | 20260506 09:25:59.373 | INFO | "pinned" : " 29 GB" | 20260506 09:25:59.373 | INFO | } | 20260506 09:25:59.373 | INFO | Perf report for the last 5.13948 seconds: | 20260506 09:25:59.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:25:59.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:59.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:59.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:59.373 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:25:59.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:59.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:59.373 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:25:59.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:59.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:25:59.373 | INFO | Uploaded 36GiB to GPU in 1.60444s (22.4377GiB/s) | 20260506 09:25:59.373 | INFO | Downloaded 34GiB to GPU in 1.52158s (22.3452GiB/s) | 20260506 09:26:04.512 | INFO | Analyzing chunk frames = [265728,266240) | 20260506 09:26:04.512 | INFO | MemInfo: { | 20260506 09:26:04.512 | INFO | "gpu" : " 17 GB", | 20260506 09:26:04.512 | INFO | "huge" : "", | 20260506 09:26:04.512 | INFO | "huge_shared" : " 27 GB", | 20260506 09:26:04.512 | INFO | "malloc" : " 14 GB", | 20260506 09:26:04.512 | INFO | "pinned" : " 29 GB" | 20260506 09:26:04.512 | INFO | } | 20260506 09:26:04.512 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:26:04.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:26:04.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:04.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:04.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:04.512 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 09:26:04.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:04.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:04.512 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 09:26:04.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:04.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:04.512 | INFO | Uploaded 36GiB to GPU in 1.60416s (22.4416GiB/s) | 20260506 09:26:04.512 | INFO | Downloaded 34GiB to GPU in 1.52147s (22.3468GiB/s) | 20260506 09:26:04.609 | INFO | Starting a round of Baz IO | 20260506 09:26:07.534 | INFO | Populating RTMetrics report | 20260506 09:26:07.534 | INFO | RTMetrics report populated | 20260506 09:26:07.534 | INFO | Opening tmp RTMetrics json file | 20260506 09:26:07.539 | INFO | Serializing RTMetrics to json | 20260506 09:26:07.542 | INFO | Serialization done, swapping tmp file now... | 20260506 09:26:07.543 | INFO | RTMetrics done | 20260506 09:26:07.848 | INFO | Finished a round of Baz IO: Wrote 1.835 GiB in 3.238 seconds | 20260506 09:26:07.848 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.50% / 0.11% / 45.39% | 20260506 09:26:07.848 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.59556% / 0.000286102% / 3.97364e-05% | 20260506 09:26:07.848 | INFO | Baz IO overall: | 20260506 09:26:07.848 | INFO | BytesWritten : 99973161434 | 20260506 09:26:07.848 | INFO | NumEvents : 47135932118 | 20260506 09:26:09.651 | INFO | Analyzing chunk frames = [266240,266752) | 20260506 09:26:09.651 | INFO | MemInfo: { | 20260506 09:26:09.651 | INFO | "gpu" : " 17 GB", | 20260506 09:26:09.651 | INFO | "huge" : "", | 20260506 09:26:09.651 | INFO | "huge_shared" : " 27 GB", | 20260506 09:26:09.651 | INFO | "malloc" : " 14 GB", | 20260506 09:26:09.651 | INFO | "pinned" : " 29 GB" | 20260506 09:26:09.651 | INFO | } | 20260506 09:26:09.652 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:26:09.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:26:09.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:09.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:09.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:09.652 | INFO | ANALYSIS : 57.8%, 2.9, 88.0ms, 0 | 20260506 09:26:09.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:09.652 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 09:26:09.652 | INFO | PRE_HQ : 56.9%, 2.1, 61.7ms, 0 | 20260506 09:26:09.652 | INFO | BAZWRITER : 62.6%, 4.4, 141.9ms, 0 | 20260506 09:26:09.652 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 09:26:09.652 | INFO | Uploaded 36GiB to GPU in 1.65807s (21.712GiB/s) | 20260506 09:26:09.652 | INFO | Downloaded 38GiB to GPU in 1.72239s (22.0624GiB/s) | 20260506 09:26:14.791 | INFO | Analyzing chunk frames = [266752,267264) | 20260506 09:26:14.791 | INFO | MemInfo: { | 20260506 09:26:14.791 | INFO | "gpu" : " 17 GB", | 20260506 09:26:14.791 | INFO | "huge" : "", | 20260506 09:26:14.791 | INFO | "huge_shared" : " 27 GB", | 20260506 09:26:14.791 | INFO | "malloc" : " 14 GB", | 20260506 09:26:14.791 | INFO | "pinned" : " 29 GB" | 20260506 09:26:14.791 | INFO | } | 20260506 09:26:14.791 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:26:14.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:26:14.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:14.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:14.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:14.791 | INFO | ANALYSIS : 57.7%, 3.0, 90.0ms, 0 | 20260506 09:26:14.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:14.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:14.791 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:26:14.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:14.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:14.791 | INFO | Uploaded 36GiB to GPU in 1.60108s (22.4848GiB/s) | 20260506 09:26:14.791 | INFO | Downloaded 34GiB to GPU in 1.51892s (22.3844GiB/s) | 20260506 09:26:19.930 | INFO | Analyzing chunk frames = [267264,267776) | 20260506 09:26:19.930 | INFO | MemInfo: { | 20260506 09:26:19.930 | INFO | "gpu" : " 17 GB", | 20260506 09:26:19.930 | INFO | "huge" : "", | 20260506 09:26:19.930 | INFO | "huge_shared" : " 27 GB", | 20260506 09:26:19.930 | INFO | "malloc" : " 14 GB", | 20260506 09:26:19.930 | INFO | "pinned" : " 29 GB" | 20260506 09:26:19.930 | INFO | } | 20260506 09:26:19.930 | INFO | Perf report for the last 5.13959 seconds: | 20260506 09:26:19.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:26:19.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:19.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:19.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:19.931 | INFO | ANALYSIS : 57.0%, 3.0, 88.6ms, 0 | 20260506 09:26:19.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:19.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:19.931 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:26:19.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:19.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:19.931 | INFO | Uploaded 36GiB to GPU in 1.60559s (22.4217GiB/s) | 20260506 09:26:19.931 | INFO | Downloaded 34GiB to GPU in 1.52318s (22.3218GiB/s) | 20260506 09:26:25.069 | INFO | Analyzing chunk frames = [267776,268288) | 20260506 09:26:25.069 | INFO | MemInfo: { | 20260506 09:26:25.069 | INFO | "gpu" : " 17 GB", | 20260506 09:26:25.069 | INFO | "huge" : "", | 20260506 09:26:25.070 | INFO | "huge_shared" : " 27 GB", | 20260506 09:26:25.070 | INFO | "malloc" : " 14 GB", | 20260506 09:26:25.070 | INFO | "pinned" : " 29 GB" | 20260506 09:26:25.070 | INFO | } | 20260506 09:26:25.070 | INFO | Perf report for the last 5.13916 seconds: | 20260506 09:26:25.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:26:25.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:25.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:25.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:25.070 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260506 09:26:25.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:25.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:25.070 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:26:25.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:25.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:25.070 | INFO | Uploaded 36GiB to GPU in 1.60518s (22.4273GiB/s) | 20260506 09:26:25.070 | INFO | Downloaded 34GiB to GPU in 1.522s (22.339GiB/s) | 20260506 09:26:30.209 | INFO | Analyzing chunk frames = [268288,268800) | 20260506 09:26:30.209 | INFO | MemInfo: { | 20260506 09:26:30.209 | INFO | "gpu" : " 17 GB", | 20260506 09:26:30.209 | INFO | "huge" : "", | 20260506 09:26:30.209 | INFO | "huge_shared" : " 27 GB", | 20260506 09:26:30.209 | INFO | "malloc" : " 14 GB", | 20260506 09:26:30.209 | INFO | "pinned" : " 29 GB" | 20260506 09:26:30.209 | INFO | } | 20260506 09:26:30.209 | INFO | Perf report for the last 5.13936 seconds: | 20260506 09:26:30.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:26:30.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:30.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:30.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:30.209 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:26:30.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:30.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:30.209 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:26:30.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:30.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:30.209 | INFO | Uploaded 36GiB to GPU in 1.60317s (22.4556GiB/s) | 20260506 09:26:30.209 | INFO | Downloaded 34GiB to GPU in 1.52058s (22.3599GiB/s) | 20260506 09:26:35.348 | INFO | Analyzing chunk frames = [268800,269312) | 20260506 09:26:35.348 | INFO | MemInfo: { | 20260506 09:26:35.348 | INFO | "gpu" : " 17 GB", | 20260506 09:26:35.348 | INFO | "huge" : "", | 20260506 09:26:35.348 | INFO | "huge_shared" : " 27 GB", | 20260506 09:26:35.348 | INFO | "malloc" : " 14 GB", | 20260506 09:26:35.348 | INFO | "pinned" : " 29 GB" | 20260506 09:26:35.348 | INFO | } | 20260506 09:26:35.348 | INFO | Perf report for the last 5.13915 seconds: | 20260506 09:26:35.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:26:35.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:35.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:35.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:35.348 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:26:35.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:35.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:35.348 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:26:35.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:35.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:35.348 | INFO | Uploaded 36GiB to GPU in 1.60356s (22.4501GiB/s) | 20260506 09:26:35.348 | INFO | Downloaded 34GiB to GPU in 1.52142s (22.3476GiB/s) | 20260506 09:26:40.488 | INFO | Analyzing chunk frames = [269312,269824) | 20260506 09:26:40.488 | INFO | MemInfo: { | 20260506 09:26:40.488 | INFO | "gpu" : " 17 GB", | 20260506 09:26:40.488 | INFO | "huge" : "", | 20260506 09:26:40.488 | INFO | "huge_shared" : " 27 GB", | 20260506 09:26:40.488 | INFO | "malloc" : " 14 GB", | 20260506 09:26:40.488 | INFO | "pinned" : " 29 GB" | 20260506 09:26:40.488 | INFO | } | 20260506 09:26:40.488 | INFO | Perf report for the last 5.13966 seconds: | 20260506 09:26:40.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:26:40.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:40.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:40.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:40.488 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:26:40.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:40.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:40.488 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:26:40.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:40.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:40.489 | INFO | Uploaded 36GiB to GPU in 1.6038s (22.4467GiB/s) | 20260506 09:26:40.489 | INFO | Downloaded 34GiB to GPU in 1.52094s (22.3547GiB/s) | 20260506 09:26:45.627 | INFO | Analyzing chunk frames = [269824,270336) | 20260506 09:26:45.627 | INFO | MemInfo: { | 20260506 09:26:45.627 | INFO | "gpu" : " 17 GB", | 20260506 09:26:45.627 | INFO | "huge" : "", | 20260506 09:26:45.627 | INFO | "huge_shared" : " 27 GB", | 20260506 09:26:45.627 | INFO | "malloc" : " 14 GB", | 20260506 09:26:45.627 | INFO | "pinned" : " 29 GB" | 20260506 09:26:45.627 | INFO | } | 20260506 09:26:45.627 | INFO | Perf report for the last 5.13916 seconds: | 20260506 09:26:45.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:26:45.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:45.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:45.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:45.627 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 09:26:45.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:45.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:45.627 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 09:26:45.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:45.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:45.627 | INFO | Uploaded 36GiB to GPU in 1.6039s (22.4453GiB/s) | 20260506 09:26:45.627 | INFO | Downloaded 34GiB to GPU in 1.52115s (22.3516GiB/s) | 20260506 09:26:45.627 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:26:45.627 | INFO | Total mem usage: 15.7 GiB | 20260506 09:26:45.627 | INFO | All Time High Water Marks sum: 16.1 GiB (Savings: 2.38%) | 20260506 09:26:45.627 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -2.95%) | 20260506 09:26:45.627 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:26:45.627 | INFO | High Water Breakdown: | 20260506 09:26:45.627 | INFO | 59.4% : 9.09 GiB ( 96.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:26:45.627 | INFO | 40.4% : 6.18 GiB ( 92.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:26:45.627 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:26:45.627 | INFO | ------------------------------------------ | 20260506 09:26:45.627 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:26:45.627 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:26:45.627 | INFO | Total mem usage: 26.5 GiB | 20260506 09:26:45.627 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:26:45.627 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:26:45.627 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:26:45.627 | INFO | High Water Breakdown: | 20260506 09:26:45.628 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:26:45.628 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:26:45.628 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:26:45.628 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:26:45.628 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:26:45.628 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:26:45.628 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:26:45.628 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:26:45.628 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:26:45.628 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:26:45.628 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:26:45.628 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:26:45.628 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:26:45.628 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:26:45.628 | INFO | ------------------------------------------ | 20260506 09:26:45.628 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:26:45.628 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:26:45.628 | INFO | Total mem usage: 25.6 GiB | 20260506 09:26:45.628 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:26:45.628 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:26:45.628 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:26:45.628 | INFO | High Water Breakdown: | 20260506 09:26:45.628 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:26:45.628 | INFO | ------------------------------------------ | 20260506 09:26:45.628 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:26:45.628 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:26:45.628 | INFO | Total mem usage: 15.7 GiB | 20260506 09:26:45.628 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:26:45.628 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:26:45.628 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:26:45.628 | INFO | High Water Breakdown: | 20260506 09:26:45.628 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:26:45.628 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:26:45.628 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:26:45.628 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:26:45.628 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:26:45.628 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:26:45.628 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:26:45.628 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:26:45.628 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:26:45.628 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:26:45.628 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:26:45.628 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:26:45.628 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:26:45.628 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:26:45.628 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:26:45.628 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:26:45.628 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:26:45.628 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:26:45.628 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:26:45.628 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:26:45.628 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:26:45.628 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:26:45.628 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:26:45.628 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:26:45.628 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:26:45.628 | INFO | ------------------------------------------ | 20260506 09:26:45.628 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:26:45.628 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:26:45.628 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:26:45.628 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:26:45.628 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 09:26:45.628 | INFO | PerfReport: 0.29%, AggregateStats -- 8.63e-02 / -nan, 5.44e-03 / 0.00e+00 | 20260506 09:26:45.628 | INFO | PerfReport: 10.88%, Baseline -- 3.29e+00 / -nan, 1.74e-02 / 0.00e+00 | 20260506 09:26:45.628 | INFO | PerfReport: 3.05%, Histogram -- 9.22e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260506 09:26:45.628 | INFO | PerfReport: 16.04%, DME -- 3.88e+01 / -nan, 3.09e+00 / 0.00e+00 | 20260506 09:26:45.628 | INFO | PerfReport: 56.85%, FrameLabeling -- 1.72e+01 / -nan, 4.17e-02 / 0.00e+00 | 20260506 09:26:45.628 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.28e+00 / -nan, 1.36e-01 / 0.00e+00 | 20260506 09:26:45.628 | INFO | PerfReport: 3.85%, Metrics -- 1.17e+00 / -nan, 1.82e-01 / 0.00e+00 | 20260506 09:26:45.628 | INFO | PerfReport: 1.50%, PulseSerializing -- 4.55e-01 / -nan, 6.60e-02 / 0.00e+00 | 20260506 09:26:45.628 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:26:45.628 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:26:45.628 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:26:45.628 | INFO | PerfReport: Full Execution -- 9.05e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 09:26:45.628 | INFO | PerfReport: 18.54%, Upload -- 1.68e+01 / 0.00e+00, 4.56e-01 / 0.00e+00 | 20260506 09:26:45.628 | INFO | PerfReport: 17.80%, Download -- 1.61e+01 / 0.00e+00, 7.97e-01 / 0.00e+00 | 20260506 09:26:45.628 | INFO | PerfReport: 33.42%, Compute -- 3.03e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 09:26:45.628 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:26:45.628 | INFO | PerfReport: 0.00%, LockDownload -- 4.94e-04 / 0.00e+00, 1.92e-04 / 0.00e+00 | 20260506 09:26:45.628 | INFO | PerfReport: 29.66%, LockCompute -- 2.68e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 09:26:45.628 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260506 09:26:45.724 | INFO | Starting a round of Baz IO | 20260506 09:26:48.650 | INFO | Populating RTMetrics report | 20260506 09:26:48.650 | INFO | RTMetrics report populated | 20260506 09:26:48.650 | INFO | Opening tmp RTMetrics json file | 20260506 09:26:48.657 | INFO | Serializing RTMetrics to json | 20260506 09:26:48.669 | INFO | Serialization done, swapping tmp file now... | 20260506 09:26:48.689 | INFO | RTMetrics done | 20260506 09:26:48.752 | INFO | Finished a round of Baz IO: Wrote 1.837 GiB in 3.027 seconds | 20260506 09:26:48.752 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.39% / 0.11% / 45.50% | 20260506 09:26:48.752 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.56006% / 0.00029405% / 4.37101e-05% | 20260506 09:26:48.752 | INFO | Baz IO overall: | 20260506 09:26:48.752 | INFO | BytesWritten : 101945368093 | 20260506 09:26:48.752 | INFO | NumEvents : 48092956539 | 20260506 09:26:50.766 | INFO | Analyzing chunk frames = [270336,270848) | 20260506 09:26:50.766 | INFO | MemInfo: { | 20260506 09:26:50.766 | INFO | "gpu" : " 17 GB", | 20260506 09:26:50.766 | INFO | "huge" : "", | 20260506 09:26:50.766 | INFO | "huge_shared" : " 27 GB", | 20260506 09:26:50.766 | INFO | "malloc" : " 14 GB", | 20260506 09:26:50.766 | INFO | "pinned" : " 29 GB" | 20260506 09:26:50.766 | INFO | } | 20260506 09:26:50.766 | INFO | Perf report for the last 5.13915 seconds: | 20260506 09:26:50.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:26:50.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:50.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:50.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:50.766 | INFO | ANALYSIS : 57.8%, 2.9, 87.6ms, 0 | 20260506 09:26:50.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:50.766 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 09:26:50.766 | INFO | PRE_HQ : 57.0%, 2.1, 62.1ms, 0 | 20260506 09:26:50.766 | INFO | BAZWRITER : 58.5%, 4.5, 136.0ms, 0 | 20260506 09:26:50.766 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 09:26:50.766 | INFO | Uploaded 36GiB to GPU in 1.65636s (21.7345GiB/s) | 20260506 09:26:50.767 | INFO | Downloaded 38GiB to GPU in 1.72003s (22.0926GiB/s) | 20260506 09:26:55.905 | INFO | Analyzing chunk frames = [270848,271360) | 20260506 09:26:55.905 | INFO | MemInfo: { | 20260506 09:26:55.905 | INFO | "gpu" : " 17 GB", | 20260506 09:26:55.905 | INFO | "huge" : "", | 20260506 09:26:55.905 | INFO | "huge_shared" : " 27 GB", | 20260506 09:26:55.905 | INFO | "malloc" : " 14 GB", | 20260506 09:26:55.905 | INFO | "pinned" : " 29 GB" | 20260506 09:26:55.905 | INFO | } | 20260506 09:26:55.906 | INFO | Perf report for the last 5.13938 seconds: | 20260506 09:26:55.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:26:55.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:55.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:55.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:55.906 | INFO | ANALYSIS : 57.7%, 3.0, 89.9ms, 0 | 20260506 09:26:55.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:55.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:55.906 | INFO | PRE_HQ : 30.2%, 1.0, 15.9ms, 0 | 20260506 09:26:55.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:55.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:26:55.906 | INFO | Uploaded 36GiB to GPU in 1.60143s (22.4799GiB/s) | 20260506 09:26:55.906 | INFO | Downloaded 34GiB to GPU in 1.5185s (22.3905GiB/s) | 20260506 09:27:01.045 | INFO | Analyzing chunk frames = [271360,271872) | 20260506 09:27:01.045 | INFO | MemInfo: { | 20260506 09:27:01.045 | INFO | "gpu" : " 17 GB", | 20260506 09:27:01.045 | INFO | "huge" : "", | 20260506 09:27:01.045 | INFO | "huge_shared" : " 27 GB", | 20260506 09:27:01.045 | INFO | "malloc" : " 14 GB", | 20260506 09:27:01.045 | INFO | "pinned" : " 29 GB" | 20260506 09:27:01.045 | INFO | } | 20260506 09:27:01.045 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:27:01.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:27:01.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:01.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:01.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:01.045 | INFO | ANALYSIS : 57.1%, 3.0, 88.6ms, 0 | 20260506 09:27:01.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:01.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:01.045 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:27:01.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:01.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:01.045 | INFO | Uploaded 36GiB to GPU in 1.60591s (22.4172GiB/s) | 20260506 09:27:01.045 | INFO | Downloaded 34GiB to GPU in 1.52311s (22.3228GiB/s) | 20260506 09:27:06.184 | INFO | Analyzing chunk frames = [271872,272384) | 20260506 09:27:06.184 | INFO | MemInfo: { | 20260506 09:27:06.184 | INFO | "gpu" : " 17 GB", | 20260506 09:27:06.184 | INFO | "huge" : "", | 20260506 09:27:06.184 | INFO | "huge_shared" : " 27 GB", | 20260506 09:27:06.184 | INFO | "malloc" : " 14 GB", | 20260506 09:27:06.184 | INFO | "pinned" : " 29 GB" | 20260506 09:27:06.184 | INFO | } | 20260506 09:27:06.184 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:27:06.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:27:06.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:06.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:06.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:06.184 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:27:06.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:06.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:06.184 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:27:06.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:06.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:06.185 | INFO | Uploaded 36GiB to GPU in 1.60541s (22.4241GiB/s) | 20260506 09:27:06.185 | INFO | Downloaded 34GiB to GPU in 1.52225s (22.3353GiB/s) | 20260506 09:27:11.323 | INFO | Analyzing chunk frames = [272384,272896) | 20260506 09:27:11.323 | INFO | MemInfo: { | 20260506 09:27:11.323 | INFO | "gpu" : " 17 GB", | 20260506 09:27:11.323 | INFO | "huge" : "", | 20260506 09:27:11.324 | INFO | "huge_shared" : " 27 GB", | 20260506 09:27:11.324 | INFO | "malloc" : " 14 GB", | 20260506 09:27:11.324 | INFO | "pinned" : " 29 GB" | 20260506 09:27:11.324 | INFO | } | 20260506 09:27:11.324 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:27:11.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:27:11.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:11.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:11.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:11.324 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:27:11.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:11.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:11.324 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:27:11.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:11.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:11.324 | INFO | Uploaded 36GiB to GPU in 1.60351s (22.4507GiB/s) | 20260506 09:27:11.324 | INFO | Downloaded 34GiB to GPU in 1.51966s (22.3734GiB/s) | 20260506 09:27:16.463 | INFO | Analyzing chunk frames = [272896,273408) | 20260506 09:27:16.463 | INFO | MemInfo: { | 20260506 09:27:16.463 | INFO | "gpu" : " 17 GB", | 20260506 09:27:16.463 | INFO | "huge" : "", | 20260506 09:27:16.463 | INFO | "huge_shared" : " 27 GB", | 20260506 09:27:16.463 | INFO | "malloc" : " 14 GB", | 20260506 09:27:16.463 | INFO | "pinned" : " 29 GB" | 20260506 09:27:16.463 | INFO | } | 20260506 09:27:16.463 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:27:16.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:27:16.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:16.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:16.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:16.463 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 09:27:16.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:16.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:16.463 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:27:16.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:16.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:16.464 | INFO | Uploaded 36GiB to GPU in 1.60459s (22.4356GiB/s) | 20260506 09:27:16.464 | INFO | Downloaded 34GiB to GPU in 1.52111s (22.3521GiB/s) | 20260506 09:27:21.602 | INFO | Analyzing chunk frames = [273408,273920) | 20260506 09:27:21.602 | INFO | MemInfo: { | 20260506 09:27:21.602 | INFO | "gpu" : " 17 GB", | 20260506 09:27:21.602 | INFO | "huge" : "", | 20260506 09:27:21.602 | INFO | "huge_shared" : " 27 GB", | 20260506 09:27:21.602 | INFO | "malloc" : " 14 GB", | 20260506 09:27:21.602 | INFO | "pinned" : " 29 GB" | 20260506 09:27:21.603 | INFO | } | 20260506 09:27:21.603 | INFO | Perf report for the last 5.13964 seconds: | 20260506 09:27:21.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:27:21.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:21.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:21.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:21.603 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:27:21.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:21.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:21.603 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:27:21.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:21.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:21.603 | INFO | Uploaded 36GiB to GPU in 1.60413s (22.4421GiB/s) | 20260506 09:27:21.603 | INFO | Downloaded 34GiB to GPU in 1.52096s (22.3542GiB/s) | 20260506 09:27:26.742 | INFO | Analyzing chunk frames = [273920,274432) | 20260506 09:27:26.742 | INFO | MemInfo: { | 20260506 09:27:26.742 | INFO | "gpu" : " 17 GB", | 20260506 09:27:26.742 | INFO | "huge" : "", | 20260506 09:27:26.742 | INFO | "huge_shared" : " 27 GB", | 20260506 09:27:26.742 | INFO | "malloc" : " 14 GB", | 20260506 09:27:26.742 | INFO | "pinned" : " 29 GB" | 20260506 09:27:26.742 | INFO | } | 20260506 09:27:26.742 | INFO | Perf report for the last 5.1391 seconds: | 20260506 09:27:26.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:27:26.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:26.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:26.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:26.742 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:27:26.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:26.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:26.742 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 09:27:26.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:26.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:26.742 | INFO | Uploaded 36GiB to GPU in 1.60446s (22.4375GiB/s) | 20260506 09:27:26.742 | INFO | Downloaded 34GiB to GPU in 1.5221s (22.3375GiB/s) | 20260506 09:27:26.839 | INFO | Starting a round of Baz IO | 20260506 09:27:29.766 | INFO | Populating RTMetrics report | 20260506 09:27:29.766 | INFO | RTMetrics report populated | 20260506 09:27:29.766 | INFO | Opening tmp RTMetrics json file | 20260506 09:27:29.784 | INFO | Serializing RTMetrics to json | 20260506 09:27:29.786 | INFO | Serialization done, swapping tmp file now... | 20260506 09:27:29.787 | INFO | RTMetrics done | 20260506 09:27:30.114 | INFO | Finished a round of Baz IO: Wrote 1.837 GiB in 3.275 seconds | 20260506 09:27:30.114 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.28% / 0.11% / 45.61% | 20260506 09:27:30.114 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.50088% / 0.000353654% / 4.76837e-05% | 20260506 09:27:30.114 | INFO | Baz IO overall: | 20260506 09:27:30.114 | INFO | BytesWritten : 103918330989 | 20260506 09:27:30.114 | INFO | NumEvents : 49050211798 | 20260506 09:27:31.881 | INFO | Analyzing chunk frames = [274432,274944) | 20260506 09:27:31.881 | INFO | MemInfo: { | 20260506 09:27:31.881 | INFO | "gpu" : " 17 GB", | 20260506 09:27:31.881 | INFO | "huge" : "", | 20260506 09:27:31.881 | INFO | "huge_shared" : " 27 GB", | 20260506 09:27:31.881 | INFO | "malloc" : " 14 GB", | 20260506 09:27:31.881 | INFO | "pinned" : " 29 GB" | 20260506 09:27:31.881 | INFO | } | 20260506 09:27:31.881 | INFO | Perf report for the last 5.13935 seconds: | 20260506 09:27:31.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:27:31.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:31.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:31.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:31.881 | INFO | ANALYSIS : 57.9%, 2.8, 85.6ms, 0 | 20260506 09:27:31.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:31.881 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 09:27:31.881 | INFO | PRE_HQ : 57.2%, 2.1, 61.6ms, 0 | 20260506 09:27:31.881 | INFO | BAZWRITER : 63.4%, 4.8, 155.5ms, 0 | 20260506 09:27:31.881 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 09:27:31.881 | INFO | Uploaded 36GiB to GPU in 1.66965s (21.5613GiB/s) | 20260506 09:27:31.881 | INFO | Downloaded 38GiB to GPU in 1.7355s (21.8957GiB/s) | 20260506 09:27:37.020 | INFO | Analyzing chunk frames = [274944,275456) | 20260506 09:27:37.020 | INFO | MemInfo: { | 20260506 09:27:37.020 | INFO | "gpu" : " 17 GB", | 20260506 09:27:37.020 | INFO | "huge" : "", | 20260506 09:27:37.020 | INFO | "huge_shared" : " 27 GB", | 20260506 09:27:37.020 | INFO | "malloc" : " 14 GB", | 20260506 09:27:37.020 | INFO | "pinned" : " 29 GB" | 20260506 09:27:37.020 | INFO | } | 20260506 09:27:37.020 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:27:37.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:27:37.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:37.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:37.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:37.020 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260506 09:27:37.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:37.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:37.021 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:27:37.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:37.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:37.021 | INFO | Uploaded 36GiB to GPU in 1.60187s (22.4737GiB/s) | 20260506 09:27:37.021 | INFO | Downloaded 34GiB to GPU in 1.51907s (22.3821GiB/s) | 20260506 09:27:42.160 | INFO | Analyzing chunk frames = [275456,275968) | 20260506 09:27:42.160 | INFO | MemInfo: { | 20260506 09:27:42.160 | INFO | "gpu" : " 17 GB", | 20260506 09:27:42.160 | INFO | "huge" : "", | 20260506 09:27:42.160 | INFO | "huge_shared" : " 27 GB", | 20260506 09:27:42.160 | INFO | "malloc" : " 14 GB", | 20260506 09:27:42.160 | INFO | "pinned" : " 29 GB" | 20260506 09:27:42.160 | INFO | } | 20260506 09:27:42.160 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:27:42.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:27:42.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:42.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:42.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:42.160 | INFO | ANALYSIS : 57.0%, 3.0, 88.6ms, 0 | 20260506 09:27:42.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:42.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:42.160 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:27:42.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:42.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:42.160 | INFO | Uploaded 36GiB to GPU in 1.60553s (22.4224GiB/s) | 20260506 09:27:42.160 | INFO | Downloaded 34GiB to GPU in 1.5226s (22.3302GiB/s) | 20260506 09:27:47.299 | INFO | Analyzing chunk frames = [275968,276480) | 20260506 09:27:47.299 | INFO | MemInfo: { | 20260506 09:27:47.299 | INFO | "gpu" : " 17 GB", | 20260506 09:27:47.299 | INFO | "huge" : "", | 20260506 09:27:47.299 | INFO | "huge_shared" : " 27 GB", | 20260506 09:27:47.299 | INFO | "malloc" : " 14 GB", | 20260506 09:27:47.299 | INFO | "pinned" : " 29 GB" | 20260506 09:27:47.299 | INFO | } | 20260506 09:27:47.299 | INFO | Perf report for the last 5.13935 seconds: | 20260506 09:27:47.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:27:47.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:47.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:47.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:47.299 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:27:47.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:47.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:47.299 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:27:47.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:47.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:47.299 | INFO | Uploaded 36GiB to GPU in 1.60575s (22.4194GiB/s) | 20260506 09:27:47.299 | INFO | Downloaded 34GiB to GPU in 1.52337s (22.319GiB/s) | 20260506 09:27:52.438 | INFO | Analyzing chunk frames = [276480,276992) | 20260506 09:27:52.438 | INFO | MemInfo: { | 20260506 09:27:52.438 | INFO | "gpu" : " 17 GB", | 20260506 09:27:52.438 | INFO | "huge" : "", | 20260506 09:27:52.438 | INFO | "huge_shared" : " 27 GB", | 20260506 09:27:52.438 | INFO | "malloc" : " 14 GB", | 20260506 09:27:52.438 | INFO | "pinned" : " 29 GB" | 20260506 09:27:52.438 | INFO | } | 20260506 09:27:52.438 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:27:52.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:27:52.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:52.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:52.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:52.438 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 09:27:52.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:52.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:52.438 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 09:27:52.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:52.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:52.439 | INFO | Uploaded 36GiB to GPU in 1.60316s (22.4556GiB/s) | 20260506 09:27:52.439 | INFO | Downloaded 34GiB to GPU in 1.52014s (22.3664GiB/s) | 20260506 09:27:57.578 | INFO | Analyzing chunk frames = [276992,277504) | 20260506 09:27:57.578 | INFO | MemInfo: { | 20260506 09:27:57.578 | INFO | "gpu" : " 17 GB", | 20260506 09:27:57.578 | INFO | "huge" : "", | 20260506 09:27:57.578 | INFO | "huge_shared" : " 27 GB", | 20260506 09:27:57.578 | INFO | "malloc" : " 14 GB", | 20260506 09:27:57.578 | INFO | "pinned" : " 29 GB" | 20260506 09:27:57.578 | INFO | } | 20260506 09:27:57.578 | INFO | Perf report for the last 5.13935 seconds: | 20260506 09:27:57.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:27:57.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:57.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:57.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:57.578 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 09:27:57.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:57.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:57.578 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:27:57.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:57.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:27:57.578 | INFO | Uploaded 36GiB to GPU in 1.60444s (22.4378GiB/s) | 20260506 09:27:57.578 | INFO | Downloaded 34GiB to GPU in 1.52118s (22.3511GiB/s) | 20260506 09:28:02.717 | INFO | Analyzing chunk frames = [277504,278016) | 20260506 09:28:02.717 | INFO | MemInfo: { | 20260506 09:28:02.717 | INFO | "gpu" : " 17 GB", | 20260506 09:28:02.717 | INFO | "huge" : "", | 20260506 09:28:02.717 | INFO | "huge_shared" : " 27 GB", | 20260506 09:28:02.717 | INFO | "malloc" : " 14 GB", | 20260506 09:28:02.717 | INFO | "pinned" : " 29 GB" | 20260506 09:28:02.717 | INFO | } | 20260506 09:28:02.717 | INFO | Perf report for the last 5.13967 seconds: | 20260506 09:28:02.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:28:02.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:02.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:02.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:02.717 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:28:02.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:02.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:02.718 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 09:28:02.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:02.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:02.718 | INFO | Uploaded 36GiB to GPU in 1.60424s (22.4406GiB/s) | 20260506 09:28:02.718 | INFO | Downloaded 34GiB to GPU in 1.52219s (22.3362GiB/s) | 20260506 09:28:07.856 | INFO | Analyzing chunk frames = [278016,278528) | 20260506 09:28:07.856 | INFO | MemInfo: { | 20260506 09:28:07.856 | INFO | "gpu" : " 17 GB", | 20260506 09:28:07.856 | INFO | "huge" : "", | 20260506 09:28:07.856 | INFO | "huge_shared" : " 27 GB", | 20260506 09:28:07.856 | INFO | "malloc" : " 14 GB", | 20260506 09:28:07.856 | INFO | "pinned" : " 29 GB" | 20260506 09:28:07.856 | INFO | } | 20260506 09:28:07.857 | INFO | Perf report for the last 5.13908 seconds: | 20260506 09:28:07.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:28:07.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:07.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:07.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:07.857 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:28:07.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:07.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:07.857 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 09:28:07.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:07.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:07.857 | INFO | Uploaded 36GiB to GPU in 1.6048s (22.4327GiB/s) | 20260506 09:28:07.857 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:28:07.857 | INFO | Total mem usage: 15.7 GiB | 20260506 09:28:07.857 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 2.56%) | 20260506 09:28:07.857 | INFO | Downloaded 34GiB to GPU in 1.52214s (22.337GiB/s) | 20260506 09:28:07.857 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: 0.00708%) | 20260506 09:28:07.857 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:28:07.857 | INFO | High Water Breakdown: | 20260506 09:28:07.857 | INFO | 60% : 9.45 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:28:07.857 | INFO | 39.9% : 6.28 GiB ( 93.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:28:07.857 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:28:07.857 | INFO | ------------------------------------------ | 20260506 09:28:07.857 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:28:07.857 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:28:07.857 | INFO | Total mem usage: 26.5 GiB | 20260506 09:28:07.857 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:28:07.857 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:28:07.857 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:28:07.857 | INFO | High Water Breakdown: | 20260506 09:28:07.857 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:28:07.857 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:28:07.857 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:28:07.857 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:28:07.857 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:28:07.857 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:28:07.857 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:28:07.857 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:28:07.857 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:28:07.857 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:28:07.857 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:28:07.857 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:28:07.857 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:28:07.857 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:28:07.857 | INFO | ------------------------------------------ | 20260506 09:28:07.857 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:28:07.857 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:28:07.857 | INFO | Total mem usage: 25.6 GiB | 20260506 09:28:07.857 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:28:07.857 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:28:07.857 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:28:07.857 | INFO | High Water Breakdown: | 20260506 09:28:07.857 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:28:07.857 | INFO | ------------------------------------------ | 20260506 09:28:07.857 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:28:07.857 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:28:07.857 | INFO | Total mem usage: 15.7 GiB | 20260506 09:28:07.857 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:28:07.857 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:28:07.857 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:28:07.857 | INFO | High Water Breakdown: | 20260506 09:28:07.857 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:28:07.857 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:28:07.857 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:28:07.857 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:28:07.857 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:28:07.857 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:28:07.857 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:28:07.858 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:28:07.858 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:28:07.858 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:28:07.858 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:28:07.858 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:28:07.858 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:28:07.858 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:28:07.858 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:28:07.858 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:28:07.858 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:28:07.858 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:28:07.858 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:28:07.858 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:28:07.858 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:28:07.858 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:28:07.858 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:28:07.858 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:28:07.858 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:28:07.858 | INFO | ------------------------------------------ | 20260506 09:28:07.858 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:28:07.858 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:28:07.858 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:28:07.858 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:28:07.858 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 09:28:07.858 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 5.78e-03 / 0.00e+00 | 20260506 09:28:07.858 | INFO | PerfReport: 10.89%, Baseline -- 3.29e+00 / -nan, 1.74e-02 / 0.00e+00 | 20260506 09:28:07.858 | INFO | PerfReport: 3.05%, Histogram -- 9.22e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260506 09:28:07.858 | INFO | PerfReport: 16.04%, DME -- 3.88e+01 / -nan, 3.01e+00 / 0.00e+00 | 20260506 09:28:07.858 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.72e+01 / -nan, 4.18e-02 / 0.00e+00 | 20260506 09:28:07.858 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.28e+00 / -nan, 1.35e-01 / 0.00e+00 | 20260506 09:28:07.858 | INFO | PerfReport: 3.84%, Metrics -- 1.16e+00 / -nan, 1.80e-01 / 0.00e+00 | 20260506 09:28:07.858 | INFO | PerfReport: 1.50%, PulseSerializing -- 4.54e-01 / -nan, 6.57e-02 / 0.00e+00 | 20260506 09:28:07.858 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:28:07.858 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:28:07.858 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:28:07.858 | INFO | PerfReport: Full Execution -- 9.04e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 09:28:07.858 | INFO | PerfReport: 18.57%, Upload -- 1.68e+01 / 0.00e+00, 4.58e-01 / 0.00e+00 | 20260506 09:28:07.858 | INFO | PerfReport: 17.83%, Download -- 1.61e+01 / 0.00e+00, 8.00e-01 / 0.00e+00 | 20260506 09:28:07.858 | INFO | PerfReport: 33.46%, Compute -- 3.02e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 09:28:07.858 | INFO | PerfReport: 0.56%, LockUpload -- 5.09e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:28:07.858 | INFO | PerfReport: 0.00%, LockDownload -- 4.85e-04 / 0.00e+00, 1.26e-04 / 0.00e+00 | 20260506 09:28:07.858 | INFO | PerfReport: 29.56%, LockCompute -- 2.67e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 09:28:07.858 | INFO | PerfReport: 0.02%, Misc -- 1.38e-02 / 0.00e+00, 2.49e-02 / 0.00e+00 | 20260506 09:28:07.954 | INFO | Starting a round of Baz IO | 20260506 09:28:10.878 | INFO | Populating RTMetrics report | 20260506 09:28:10.879 | INFO | RTMetrics report populated | 20260506 09:28:10.879 | INFO | Opening tmp RTMetrics json file | 20260506 09:28:11.020 | INFO | Serializing RTMetrics to json | 20260506 09:28:11.035 | INFO | Serialization done, swapping tmp file now... | 20260506 09:28:11.039 | INFO | RTMetrics done | 20260506 09:28:11.062 | INFO | Finished a round of Baz IO: Wrote 1.84 GiB in 3.108 seconds | 20260506 09:28:11.062 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.18% / 0.11% / 45.72% | 20260506 09:28:11.062 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.4614% / 0.000333786% / 4.76837e-05% | 20260506 09:28:11.062 | INFO | Baz IO overall: | 20260506 09:28:11.062 | INFO | BytesWritten : 105894374884 | 20260506 09:28:11.062 | INFO | NumEvents : 50009617161 | 20260506 09:28:12.996 | INFO | Analyzing chunk frames = [278528,279040) | 20260506 09:28:12.996 | INFO | MemInfo: { | 20260506 09:28:12.996 | INFO | "gpu" : " 17 GB", | 20260506 09:28:12.996 | INFO | "huge" : "", | 20260506 09:28:12.996 | INFO | "huge_shared" : " 27 GB", | 20260506 09:28:12.996 | INFO | "malloc" : " 14 GB", | 20260506 09:28:12.996 | INFO | "pinned" : " 29 GB" | 20260506 09:28:12.996 | INFO | } | 20260506 09:28:12.996 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:28:12.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:28:12.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:12.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:12.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:12.996 | INFO | ANALYSIS : 57.8%, 2.9, 88.2ms, 0 | 20260506 09:28:12.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:12.996 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260506 09:28:12.996 | INFO | PRE_HQ : 57.0%, 2.1, 62.1ms, 0 | 20260506 09:28:12.996 | INFO | BAZWRITER : 60.1%, 4.3, 133.0ms, 0 | 20260506 09:28:12.996 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.5ms, 0 | 20260506 09:28:12.996 | INFO | Uploaded 36GiB to GPU in 1.64352s (21.9041GiB/s) | 20260506 09:28:12.996 | INFO | Downloaded 38GiB to GPU in 1.70685s (22.2632GiB/s) | 20260506 09:28:18.135 | INFO | Analyzing chunk frames = [279040,279552) | 20260506 09:28:18.135 | INFO | MemInfo: { | 20260506 09:28:18.135 | INFO | "gpu" : " 17 GB", | 20260506 09:28:18.135 | INFO | "huge" : "", | 20260506 09:28:18.135 | INFO | "huge_shared" : " 27 GB", | 20260506 09:28:18.135 | INFO | "malloc" : " 14 GB", | 20260506 09:28:18.135 | INFO | "pinned" : " 29 GB" | 20260506 09:28:18.135 | INFO | } | 20260506 09:28:18.135 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:28:18.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:28:18.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:18.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:18.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:18.135 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260506 09:28:18.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:18.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:18.135 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:28:18.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:18.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:18.135 | INFO | Uploaded 36GiB to GPU in 1.60132s (22.4814GiB/s) | 20260506 09:28:18.135 | INFO | Downloaded 34GiB to GPU in 1.51922s (22.3799GiB/s) | 20260506 09:28:23.274 | INFO | Analyzing chunk frames = [279552,280064) | 20260506 09:28:23.274 | INFO | MemInfo: { | 20260506 09:28:23.274 | INFO | "gpu" : " 17 GB", | 20260506 09:28:23.274 | INFO | "huge" : "", | 20260506 09:28:23.274 | INFO | "huge_shared" : " 27 GB", | 20260506 09:28:23.274 | INFO | "malloc" : " 14 GB", | 20260506 09:28:23.274 | INFO | "pinned" : " 29 GB" | 20260506 09:28:23.274 | INFO | } | 20260506 09:28:23.275 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:28:23.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:28:23.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:23.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:23.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:23.275 | INFO | ANALYSIS : 57.0%, 3.0, 88.6ms, 0 | 20260506 09:28:23.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:23.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:23.275 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 09:28:23.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:23.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:23.275 | INFO | Uploaded 36GiB to GPU in 1.60608s (22.4148GiB/s) | 20260506 09:28:23.275 | INFO | Downloaded 34GiB to GPU in 1.52314s (22.3223GiB/s) | 20260506 09:28:28.414 | INFO | Analyzing chunk frames = [280064,280576) | 20260506 09:28:28.414 | INFO | MemInfo: { | 20260506 09:28:28.414 | INFO | "gpu" : " 17 GB", | 20260506 09:28:28.414 | INFO | "huge" : "", | 20260506 09:28:28.414 | INFO | "huge_shared" : " 27 GB", | 20260506 09:28:28.414 | INFO | "malloc" : " 14 GB", | 20260506 09:28:28.414 | INFO | "pinned" : " 29 GB" | 20260506 09:28:28.414 | INFO | } | 20260506 09:28:28.414 | INFO | Perf report for the last 5.13936 seconds: | 20260506 09:28:28.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:28:28.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:28.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:28.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:28.414 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:28:28.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:28.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:28.414 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260506 09:28:28.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:28.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:28.414 | INFO | Uploaded 36GiB to GPU in 1.60576s (22.4193GiB/s) | 20260506 09:28:28.414 | INFO | Downloaded 34GiB to GPU in 1.52346s (22.3176GiB/s) | 20260506 09:28:33.553 | INFO | Analyzing chunk frames = [280576,281088) | 20260506 09:28:33.553 | INFO | MemInfo: { | 20260506 09:28:33.553 | INFO | "gpu" : " 17 GB", | 20260506 09:28:33.553 | INFO | "huge" : "", | 20260506 09:28:33.553 | INFO | "huge_shared" : " 27 GB", | 20260506 09:28:33.553 | INFO | "malloc" : " 14 GB", | 20260506 09:28:33.553 | INFO | "pinned" : " 29 GB" | 20260506 09:28:33.553 | INFO | } | 20260506 09:28:33.553 | INFO | Perf report for the last 5.13921 seconds: | 20260506 09:28:33.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:28:33.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:33.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:33.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:33.553 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:28:33.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:33.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:33.553 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 09:28:33.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:33.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:33.554 | INFO | Uploaded 36GiB to GPU in 1.60257s (22.464GiB/s) | 20260506 09:28:33.554 | INFO | Downloaded 34GiB to GPU in 1.52082s (22.3563GiB/s) | 20260506 09:28:38.692 | INFO | Analyzing chunk frames = [281088,281600) | 20260506 09:28:38.692 | INFO | MemInfo: { | 20260506 09:28:38.692 | INFO | "gpu" : " 17 GB", | 20260506 09:28:38.692 | INFO | "huge" : "", | 20260506 09:28:38.692 | INFO | "huge_shared" : " 27 GB", | 20260506 09:28:38.692 | INFO | "malloc" : " 14 GB", | 20260506 09:28:38.692 | INFO | "pinned" : " 29 GB" | 20260506 09:28:38.693 | INFO | } | 20260506 09:28:38.693 | INFO | Perf report for the last 5.1394 seconds: | 20260506 09:28:38.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:28:38.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:38.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:38.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:38.693 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 09:28:38.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:38.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:38.693 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:28:38.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:38.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:38.693 | INFO | Uploaded 36GiB to GPU in 1.60413s (22.4421GiB/s) | 20260506 09:28:38.693 | INFO | Downloaded 34GiB to GPU in 1.52085s (22.3559GiB/s) | 20260506 09:28:43.832 | INFO | Analyzing chunk frames = [281600,282112) | 20260506 09:28:43.832 | INFO | MemInfo: { | 20260506 09:28:43.832 | INFO | "gpu" : " 17 GB", | 20260506 09:28:43.832 | INFO | "huge" : "", | 20260506 09:28:43.832 | INFO | "huge_shared" : " 27 GB", | 20260506 09:28:43.832 | INFO | "malloc" : " 14 GB", | 20260506 09:28:43.832 | INFO | "pinned" : " 29 GB" | 20260506 09:28:43.832 | INFO | } | 20260506 09:28:43.832 | INFO | Perf report for the last 5.13952 seconds: | 20260506 09:28:43.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:28:43.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:43.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:43.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:43.832 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:28:43.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:43.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:43.832 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:28:43.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:43.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:43.832 | INFO | Uploaded 36GiB to GPU in 1.60311s (22.4563GiB/s) | 20260506 09:28:43.832 | INFO | Downloaded 34GiB to GPU in 1.52099s (22.3539GiB/s) | 20260506 09:28:48.971 | INFO | Analyzing chunk frames = [282112,282624) | 20260506 09:28:48.971 | INFO | MemInfo: { | 20260506 09:28:48.971 | INFO | "gpu" : " 17 GB", | 20260506 09:28:48.971 | INFO | "huge" : "", | 20260506 09:28:48.971 | INFO | "huge_shared" : " 27 GB", | 20260506 09:28:48.971 | INFO | "malloc" : " 14 GB", | 20260506 09:28:48.971 | INFO | "pinned" : " 29 GB" | 20260506 09:28:48.971 | INFO | } | 20260506 09:28:48.971 | INFO | Perf report for the last 5.13912 seconds: | 20260506 09:28:48.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:28:48.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:48.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:48.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:48.971 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 09:28:48.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:48.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:48.971 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 09:28:48.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:48.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:48.971 | INFO | Uploaded 36GiB to GPU in 1.60478s (22.433GiB/s) | 20260506 09:28:48.971 | INFO | Downloaded 34GiB to GPU in 1.52155s (22.3456GiB/s) | 20260506 09:28:49.080 | INFO | Starting a round of Baz IO | 20260506 09:28:51.998 | INFO | Populating RTMetrics report | 20260506 09:28:51.999 | INFO | RTMetrics report populated | 20260506 09:28:51.999 | INFO | Opening tmp RTMetrics json file | 20260506 09:28:52.016 | INFO | Serializing RTMetrics to json | 20260506 09:28:52.040 | INFO | Serialization done, swapping tmp file now... | 20260506 09:28:52.042 | INFO | RTMetrics done | 20260506 09:28:52.107 | INFO | Finished a round of Baz IO: Wrote 1.842 GiB in 3.027 seconds | 20260506 09:28:52.107 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 54.07% / 0.11% / 45.82% | 20260506 09:28:52.107 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.40886% / 0.000393391% / 5.16574e-05% | 20260506 09:28:52.107 | INFO | Baz IO overall: | 20260506 09:28:52.107 | INFO | BytesWritten : 107872102305 | 20260506 09:28:52.107 | INFO | NumEvents : 50969916780 | 20260506 09:28:54.111 | INFO | Analyzing chunk frames = [282624,283136) | 20260506 09:28:54.111 | INFO | MemInfo: { | 20260506 09:28:54.111 | INFO | "gpu" : " 17 GB", | 20260506 09:28:54.111 | INFO | "huge" : "", | 20260506 09:28:54.111 | INFO | "huge_shared" : " 27 GB", | 20260506 09:28:54.111 | INFO | "malloc" : " 14 GB", | 20260506 09:28:54.111 | INFO | "pinned" : " 29 GB" | 20260506 09:28:54.111 | INFO | } | 20260506 09:28:54.111 | INFO | Perf report for the last 5.13959 seconds: | 20260506 09:28:54.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:28:54.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:54.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:54.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:54.111 | INFO | ANALYSIS : 57.9%, 2.8, 85.3ms, 0 | 20260506 09:28:54.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:54.111 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260506 09:28:54.111 | INFO | PRE_HQ : 57.2%, 2.1, 62.3ms, 0 | 20260506 09:28:54.111 | INFO | BAZWRITER : 58.7%, 4.8, 143.4ms, 0 | 20260506 09:28:54.111 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 09:28:54.111 | INFO | Uploaded 36GiB to GPU in 1.66941s (21.5645GiB/s) | 20260506 09:28:54.111 | INFO | Downloaded 38GiB to GPU in 1.73502s (21.9018GiB/s) | 20260506 09:28:59.250 | INFO | Analyzing chunk frames = [283136,283648) | 20260506 09:28:59.250 | INFO | MemInfo: { | 20260506 09:28:59.250 | INFO | "gpu" : " 17 GB", | 20260506 09:28:59.250 | INFO | "huge" : "", | 20260506 09:28:59.250 | INFO | "huge_shared" : " 27 GB", | 20260506 09:28:59.250 | INFO | "malloc" : " 14 GB", | 20260506 09:28:59.250 | INFO | "pinned" : " 29 GB" | 20260506 09:28:59.250 | INFO | } | 20260506 09:28:59.250 | INFO | Perf report for the last 5.13916 seconds: | 20260506 09:28:59.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:28:59.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:59.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:59.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:59.250 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260506 09:28:59.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:59.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:59.250 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:28:59.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:59.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:28:59.251 | INFO | Uploaded 36GiB to GPU in 1.60126s (22.4823GiB/s) | 20260506 09:28:59.251 | INFO | Downloaded 34GiB to GPU in 1.51828s (22.3937GiB/s) | 20260506 09:29:04.389 | INFO | Analyzing chunk frames = [283648,284160) | 20260506 09:29:04.389 | INFO | MemInfo: { | 20260506 09:29:04.389 | INFO | "gpu" : " 17 GB", | 20260506 09:29:04.389 | INFO | "huge" : "", | 20260506 09:29:04.389 | INFO | "huge_shared" : " 27 GB", | 20260506 09:29:04.389 | INFO | "malloc" : " 14 GB", | 20260506 09:29:04.389 | INFO | "pinned" : " 29 GB" | 20260506 09:29:04.389 | INFO | } | 20260506 09:29:04.389 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:29:04.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:29:04.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:04.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:04.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:04.389 | INFO | ANALYSIS : 57.0%, 3.0, 88.5ms, 0 | 20260506 09:29:04.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:04.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:04.389 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:29:04.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:04.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:04.390 | INFO | Uploaded 36GiB to GPU in 1.60579s (22.4188GiB/s) | 20260506 09:29:04.390 | INFO | Downloaded 34GiB to GPU in 1.52354s (22.3165GiB/s) | 20260506 09:29:09.528 | INFO | Analyzing chunk frames = [284160,284672) | 20260506 09:29:09.528 | INFO | MemInfo: { | 20260506 09:29:09.529 | INFO | "gpu" : " 17 GB", | 20260506 09:29:09.529 | INFO | "huge" : "", | 20260506 09:29:09.529 | INFO | "huge_shared" : " 27 GB", | 20260506 09:29:09.529 | INFO | "malloc" : " 14 GB", | 20260506 09:29:09.529 | INFO | "pinned" : " 29 GB" | 20260506 09:29:09.529 | INFO | } | 20260506 09:29:09.529 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:29:09.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:29:09.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:09.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:09.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:09.529 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:29:09.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:09.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:09.529 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:29:09.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:09.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:09.529 | INFO | Uploaded 36GiB to GPU in 1.60514s (22.4279GiB/s) | 20260506 09:29:09.529 | INFO | Downloaded 34GiB to GPU in 1.52173s (22.343GiB/s) | 20260506 09:29:14.668 | INFO | Analyzing chunk frames = [284672,285184) | 20260506 09:29:14.668 | INFO | MemInfo: { | 20260506 09:29:14.668 | INFO | "gpu" : " 17 GB", | 20260506 09:29:14.668 | INFO | "huge" : "", | 20260506 09:29:14.668 | INFO | "huge_shared" : " 27 GB", | 20260506 09:29:14.668 | INFO | "malloc" : " 14 GB", | 20260506 09:29:14.668 | INFO | "pinned" : " 29 GB" | 20260506 09:29:14.668 | INFO | } | 20260506 09:29:14.668 | INFO | Perf report for the last 5.13952 seconds: | 20260506 09:29:14.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:29:14.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:14.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:14.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:14.668 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:29:14.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:14.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:14.668 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:29:14.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:14.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:14.668 | INFO | Uploaded 36GiB to GPU in 1.60324s (22.4545GiB/s) | 20260506 09:29:14.669 | INFO | Downloaded 34GiB to GPU in 1.52076s (22.3573GiB/s) | 20260506 09:29:19.807 | INFO | Analyzing chunk frames = [285184,285696) | 20260506 09:29:19.807 | INFO | MemInfo: { | 20260506 09:29:19.807 | INFO | "gpu" : " 17 GB", | 20260506 09:29:19.807 | INFO | "huge" : "", | 20260506 09:29:19.807 | INFO | "huge_shared" : " 27 GB", | 20260506 09:29:19.807 | INFO | "malloc" : " 14 GB", | 20260506 09:29:19.807 | INFO | "pinned" : " 29 GB" | 20260506 09:29:19.807 | INFO | } | 20260506 09:29:19.807 | INFO | Perf report for the last 5.13914 seconds: | 20260506 09:29:19.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:29:19.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:19.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:19.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:19.807 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 09:29:19.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:19.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:19.807 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:29:19.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:19.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:19.808 | INFO | Uploaded 36GiB to GPU in 1.60322s (22.4548GiB/s) | 20260506 09:29:19.808 | INFO | Downloaded 34GiB to GPU in 1.52055s (22.3604GiB/s) | 20260506 09:29:24.946 | INFO | Analyzing chunk frames = [285696,286208) | 20260506 09:29:24.947 | INFO | MemInfo: { | 20260506 09:29:24.947 | INFO | "gpu" : " 17 GB", | 20260506 09:29:24.947 | INFO | "huge" : "", | 20260506 09:29:24.947 | INFO | "huge_shared" : " 27 GB", | 20260506 09:29:24.947 | INFO | "malloc" : " 14 GB", | 20260506 09:29:24.947 | INFO | "pinned" : " 29 GB" | 20260506 09:29:24.947 | INFO | } | 20260506 09:29:24.947 | INFO | Perf report for the last 5.13926 seconds: | 20260506 09:29:24.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:29:24.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:24.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:24.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:24.947 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:29:24.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:24.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:24.947 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:29:24.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:24.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:24.947 | INFO | Uploaded 36GiB to GPU in 1.60412s (22.4423GiB/s) | 20260506 09:29:24.947 | INFO | Downloaded 34GiB to GPU in 1.52134s (22.3487GiB/s) | 20260506 09:29:30.086 | INFO | Analyzing chunk frames = [286208,286720) | 20260506 09:29:30.086 | INFO | MemInfo: { | 20260506 09:29:30.086 | INFO | "gpu" : " 17 GB", | 20260506 09:29:30.086 | INFO | "huge" : "", | 20260506 09:29:30.086 | INFO | "huge_shared" : " 27 GB", | 20260506 09:29:30.086 | INFO | "malloc" : " 14 GB", | 20260506 09:29:30.086 | INFO | "pinned" : " 29 GB" | 20260506 09:29:30.086 | INFO | } | 20260506 09:29:30.086 | INFO | Perf report for the last 5.13946 seconds: | 20260506 09:29:30.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:29:30.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:30.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:30.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:30.086 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 09:29:30.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:30.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:30.086 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 09:29:30.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:30.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:30.087 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:29:30.087 | INFO | Total mem usage: 15.7 GiB | 20260506 09:29:30.087 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 2.56%) | 20260506 09:29:30.087 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -5.2%) | 20260506 09:29:30.087 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:29:30.087 | INFO | High Water Breakdown: | 20260506 09:29:30.087 | INFO | 59.1% : 8.85 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:29:30.087 | INFO | 40.7% : 6.1 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:29:30.087 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:29:30.087 | INFO | ------------------------------------------ | 20260506 09:29:30.087 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:29:30.087 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:29:30.087 | INFO | Total mem usage: 26.5 GiB | 20260506 09:29:30.087 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:29:30.087 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:29:30.087 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:29:30.087 | INFO | High Water Breakdown: | 20260506 09:29:30.087 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:29:30.087 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:29:30.087 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:29:30.087 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:29:30.087 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:29:30.087 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:29:30.087 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:29:30.087 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:29:30.087 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:29:30.087 | INFO | Uploaded 36GiB to GPU in 1.60405s (22.4432GiB/s) | 20260506 09:29:30.087 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:29:30.087 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:29:30.087 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:29:30.087 | INFO | Downloaded 34GiB to GPU in 1.52167s (22.3439GiB/s) | 20260506 09:29:30.087 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:29:30.087 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:29:30.087 | INFO | ------------------------------------------ | 20260506 09:29:30.087 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:29:30.087 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:29:30.087 | INFO | Total mem usage: 25.6 GiB | 20260506 09:29:30.087 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:29:30.087 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:29:30.087 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:29:30.087 | INFO | High Water Breakdown: | 20260506 09:29:30.087 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:29:30.087 | INFO | ------------------------------------------ | 20260506 09:29:30.087 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:29:30.087 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:29:30.087 | INFO | Total mem usage: 15.7 GiB | 20260506 09:29:30.087 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:29:30.087 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:29:30.087 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:29:30.087 | INFO | High Water Breakdown: | 20260506 09:29:30.087 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:29:30.087 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:29:30.087 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:29:30.087 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:29:30.087 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:29:30.087 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:29:30.087 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:29:30.087 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:29:30.087 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:29:30.087 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:29:30.087 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:29:30.087 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:29:30.087 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:29:30.087 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:29:30.087 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:29:30.087 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:29:30.087 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:29:30.087 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:29:30.087 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:29:30.087 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:29:30.087 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:29:30.087 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:29:30.087 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:29:30.087 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:29:30.087 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:29:30.088 | INFO | ------------------------------------------ | 20260506 09:29:30.088 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:29:30.088 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:29:30.088 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:29:30.088 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:29:30.088 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 09:29:30.088 | INFO | PerfReport: 0.29%, AggregateStats -- 8.63e-02 / -nan, 4.34e-03 / 0.00e+00 | 20260506 09:29:30.088 | INFO | PerfReport: 10.89%, Baseline -- 3.29e+00 / -nan, 4.19e-02 / 0.00e+00 | 20260506 09:29:30.088 | INFO | PerfReport: 3.05%, Histogram -- 9.22e-01 / -nan, 5.29e-02 / 0.00e+00 | 20260506 09:29:30.088 | INFO | PerfReport: 16.04%, DME -- 3.88e+01 / -nan, 3.16e+00 / 0.00e+00 | 20260506 09:29:30.088 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.72e+01 / -nan, 4.19e-02 / 0.00e+00 | 20260506 09:29:30.088 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.28e+00 / -nan, 1.36e-01 / 0.00e+00 | 20260506 09:29:30.088 | INFO | PerfReport: 3.84%, Metrics -- 1.16e+00 / -nan, 1.80e-01 / 0.00e+00 | 20260506 09:29:30.088 | INFO | PerfReport: 1.50%, PulseSerializing -- 4.53e-01 / -nan, 6.62e-02 / 0.00e+00 | 20260506 09:29:30.088 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:29:30.088 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:29:30.088 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:29:30.088 | INFO | PerfReport: Full Execution -- 9.04e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 09:29:30.088 | INFO | PerfReport: 18.56%, Upload -- 1.68e+01 / 0.00e+00, 4.45e-01 / 0.00e+00 | 20260506 09:29:30.088 | INFO | PerfReport: 17.82%, Download -- 1.61e+01 / 0.00e+00, 7.77e-01 / 0.00e+00 | 20260506 09:29:30.088 | INFO | PerfReport: 33.45%, Compute -- 3.02e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 09:29:30.088 | INFO | PerfReport: 0.56%, LockUpload -- 5.08e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:29:30.088 | INFO | PerfReport: 0.00%, LockDownload -- 4.95e-04 / 0.00e+00, 2.89e-04 / 0.00e+00 | 20260506 09:29:30.088 | INFO | PerfReport: 29.60%, LockCompute -- 2.68e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 09:29:30.088 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260506 09:29:30.184 | INFO | Starting a round of Baz IO | 20260506 09:29:33.108 | INFO | Populating RTMetrics report | 20260506 09:29:33.108 | INFO | RTMetrics report populated | 20260506 09:29:33.108 | INFO | Opening tmp RTMetrics json file | 20260506 09:29:33.226 | INFO | Serializing RTMetrics to json | 20260506 09:29:33.307 | INFO | Serialization done, swapping tmp file now... | 20260506 09:29:33.313 | INFO | RTMetrics done | 20260506 09:29:33.456 | INFO | Finished a round of Baz IO: Wrote 1.847 GiB in 3.271 seconds | 20260506 09:29:33.456 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.97% / 0.10% / 45.93% | 20260506 09:29:33.456 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.40729% / 0.000329812% / 4.37101e-05% | 20260506 09:29:33.456 | INFO | Baz IO overall: | 20260506 09:29:33.456 | INFO | BytesWritten : 109855394982 | 20260506 09:29:33.456 | INFO | NumEvents : 51933608041 | 20260506 09:29:35.225 | INFO | Analyzing chunk frames = [286720,287232) | 20260506 09:29:35.225 | INFO | MemInfo: { | 20260506 09:29:35.225 | INFO | "gpu" : " 17 GB", | 20260506 09:29:35.225 | INFO | "huge" : "", | 20260506 09:29:35.225 | INFO | "huge_shared" : " 27 GB", | 20260506 09:29:35.225 | INFO | "malloc" : " 14 GB", | 20260506 09:29:35.225 | INFO | "pinned" : " 29 GB" | 20260506 09:29:35.225 | INFO | } | 20260506 09:29:35.225 | INFO | Perf report for the last 5.1392 seconds: | 20260506 09:29:35.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:29:35.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:35.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:35.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:35.225 | INFO | ANALYSIS : 57.8%, 2.8, 86.3ms, 0 | 20260506 09:29:35.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:35.225 | INFO | RT_METRICS : 47.9%, 1.0, 25.1ms, 0 | 20260506 09:29:35.225 | INFO | PRE_HQ : 57.0%, 2.1, 61.8ms, 0 | 20260506 09:29:35.225 | INFO | BAZWRITER : 63.2%, 4.7, 151.8ms, 0 | 20260506 09:29:35.225 | INFO | BAZ_SAMPLING : 3.2%, 1.0, 1.7ms, 0 | 20260506 09:29:35.226 | INFO | Uploaded 36GiB to GPU in 1.66504s (21.6211GiB/s) | 20260506 09:29:35.226 | INFO | Downloaded 38GiB to GPU in 1.72915s (21.9761GiB/s) | 20260506 09:29:40.364 | INFO | Analyzing chunk frames = [287232,287744) | 20260506 09:29:40.364 | INFO | MemInfo: { | 20260506 09:29:40.364 | INFO | "gpu" : " 17 GB", | 20260506 09:29:40.364 | INFO | "huge" : "", | 20260506 09:29:40.364 | INFO | "huge_shared" : " 27 GB", | 20260506 09:29:40.364 | INFO | "malloc" : " 14 GB", | 20260506 09:29:40.365 | INFO | "pinned" : " 29 GB" | 20260506 09:29:40.365 | INFO | } | 20260506 09:29:40.365 | INFO | Perf report for the last 5.13921 seconds: | 20260506 09:29:40.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:29:40.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:40.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:40.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:40.365 | INFO | ANALYSIS : 57.7%, 3.0, 89.9ms, 0 | 20260506 09:29:40.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:40.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:40.365 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:29:40.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:40.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:40.365 | INFO | Uploaded 36GiB to GPU in 1.60165s (22.4768GiB/s) | 20260506 09:29:40.365 | INFO | Downloaded 34GiB to GPU in 1.51887s (22.3851GiB/s) | 20260506 09:29:45.504 | INFO | Analyzing chunk frames = [287744,288256) | 20260506 09:29:45.504 | INFO | MemInfo: { | 20260506 09:29:45.504 | INFO | "gpu" : " 17 GB", | 20260506 09:29:45.504 | INFO | "huge" : "", | 20260506 09:29:45.504 | INFO | "huge_shared" : " 27 GB", | 20260506 09:29:45.504 | INFO | "malloc" : " 14 GB", | 20260506 09:29:45.504 | INFO | "pinned" : " 29 GB" | 20260506 09:29:45.504 | INFO | } | 20260506 09:29:45.504 | INFO | Perf report for the last 5.13946 seconds: | 20260506 09:29:45.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:29:45.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:45.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:45.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:45.504 | INFO | ANALYSIS : 57.0%, 3.0, 88.5ms, 0 | 20260506 09:29:45.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:45.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:45.504 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260506 09:29:45.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:45.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:45.504 | INFO | Uploaded 36GiB to GPU in 1.60549s (22.423GiB/s) | 20260506 09:29:45.504 | INFO | Downloaded 34GiB to GPU in 1.5232s (22.3214GiB/s) | 20260506 09:29:50.643 | INFO | Analyzing chunk frames = [288256,288768) | 20260506 09:29:50.643 | INFO | MemInfo: { | 20260506 09:29:50.643 | INFO | "gpu" : " 17 GB", | 20260506 09:29:50.643 | INFO | "huge" : "", | 20260506 09:29:50.643 | INFO | "huge_shared" : " 27 GB", | 20260506 09:29:50.643 | INFO | "malloc" : " 14 GB", | 20260506 09:29:50.643 | INFO | "pinned" : " 29 GB" | 20260506 09:29:50.643 | INFO | } | 20260506 09:29:50.643 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:29:50.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:29:50.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:50.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:50.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:50.643 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260506 09:29:50.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:50.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:50.643 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:29:50.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:50.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:50.643 | INFO | Uploaded 36GiB to GPU in 1.60539s (22.4245GiB/s) | 20260506 09:29:50.643 | INFO | Downloaded 34GiB to GPU in 1.52275s (22.328GiB/s) | 20260506 09:29:55.783 | INFO | Analyzing chunk frames = [288768,289280) | 20260506 09:29:55.783 | INFO | MemInfo: { | 20260506 09:29:55.783 | INFO | "gpu" : " 17 GB", | 20260506 09:29:55.783 | INFO | "huge" : "", | 20260506 09:29:55.783 | INFO | "huge_shared" : " 27 GB", | 20260506 09:29:55.783 | INFO | "malloc" : " 14 GB", | 20260506 09:29:55.783 | INFO | "pinned" : " 29 GB" | 20260506 09:29:55.783 | INFO | } | 20260506 09:29:55.783 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:29:55.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:29:55.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:55.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:55.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:55.783 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:29:55.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:55.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:55.783 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:29:55.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:55.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:29:55.783 | INFO | Uploaded 36GiB to GPU in 1.60315s (22.4558GiB/s) | 20260506 09:29:55.783 | INFO | Downloaded 34GiB to GPU in 1.52088s (22.3555GiB/s) | 20260506 09:30:00.922 | INFO | Analyzing chunk frames = [289280,289792) | 20260506 09:30:00.922 | INFO | MemInfo: { | 20260506 09:30:00.922 | INFO | "gpu" : " 17 GB", | 20260506 09:30:00.922 | INFO | "huge" : "", | 20260506 09:30:00.922 | INFO | "huge_shared" : " 27 GB", | 20260506 09:30:00.922 | INFO | "malloc" : " 14 GB", | 20260506 09:30:00.922 | INFO | "pinned" : " 29 GB" | 20260506 09:30:00.922 | INFO | } | 20260506 09:30:00.922 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:30:00.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:30:00.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:00.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:00.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:00.922 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 09:30:00.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:00.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:00.922 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:30:00.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:00.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:00.922 | INFO | Uploaded 36GiB to GPU in 1.6037s (22.4481GiB/s) | 20260506 09:30:00.922 | INFO | Downloaded 34GiB to GPU in 1.52082s (22.3564GiB/s) | 20260506 09:30:06.061 | INFO | Analyzing chunk frames = [289792,290304) | 20260506 09:30:06.061 | INFO | MemInfo: { | 20260506 09:30:06.061 | INFO | "gpu" : " 17 GB", | 20260506 09:30:06.061 | INFO | "huge" : "", | 20260506 09:30:06.061 | INFO | "huge_shared" : " 27 GB", | 20260506 09:30:06.061 | INFO | "malloc" : " 14 GB", | 20260506 09:30:06.061 | INFO | "pinned" : " 29 GB" | 20260506 09:30:06.061 | INFO | } | 20260506 09:30:06.061 | INFO | Perf report for the last 5.13918 seconds: | 20260506 09:30:06.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:30:06.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:06.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:06.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:06.061 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:30:06.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:06.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:06.061 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:30:06.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:06.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:06.061 | INFO | Uploaded 36GiB to GPU in 1.6035s (22.4508GiB/s) | 20260506 09:30:06.062 | INFO | Downloaded 34GiB to GPU in 1.52127s (22.3497GiB/s) | 20260506 09:30:11.200 | INFO | Analyzing chunk frames = [290304,290816) | 20260506 09:30:11.200 | INFO | MemInfo: { | 20260506 09:30:11.200 | INFO | "gpu" : " 17 GB", | 20260506 09:30:11.200 | INFO | "huge" : "", | 20260506 09:30:11.201 | INFO | "huge_shared" : " 27 GB", | 20260506 09:30:11.201 | INFO | "malloc" : " 14 GB", | 20260506 09:30:11.201 | INFO | "pinned" : " 29 GB" | 20260506 09:30:11.201 | INFO | } | 20260506 09:30:11.201 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:30:11.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:30:11.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:11.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:11.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:11.201 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 09:30:11.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:11.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:11.201 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 09:30:11.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:11.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:11.201 | INFO | Uploaded 36GiB to GPU in 1.60395s (22.4446GiB/s) | 20260506 09:30:11.201 | INFO | Downloaded 34GiB to GPU in 1.52215s (22.3368GiB/s) | 20260506 09:30:11.297 | INFO | Starting a round of Baz IO | 20260506 09:30:14.228 | INFO | Populating RTMetrics report | 20260506 09:30:14.228 | INFO | RTMetrics report populated | 20260506 09:30:14.228 | INFO | Opening tmp RTMetrics json file | 20260506 09:30:14.261 | INFO | Serializing RTMetrics to json | 20260506 09:30:14.268 | INFO | Serialization done, swapping tmp file now... | 20260506 09:30:14.269 | INFO | RTMetrics done | 20260506 09:30:14.484 | INFO | Finished a round of Baz IO: Wrote 1.85 GiB in 3.186 seconds | 20260506 09:30:14.484 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.87% / 0.10% / 46.03% | 20260506 09:30:14.484 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.34703% / 0.000305971% / 5.5631e-05% | 20260506 09:30:14.484 | INFO | Baz IO overall: | 20260506 09:30:14.484 | INFO | BytesWritten : 111841562149 | 20260506 09:30:14.484 | INFO | NumEvents : 52898023213 | 20260506 09:30:16.340 | INFO | Analyzing chunk frames = [290816,291328) | 20260506 09:30:16.340 | INFO | MemInfo: { | 20260506 09:30:16.340 | INFO | "gpu" : " 17 GB", | 20260506 09:30:16.340 | INFO | "huge" : "", | 20260506 09:30:16.340 | INFO | "huge_shared" : " 27 GB", | 20260506 09:30:16.340 | INFO | "malloc" : " 14 GB", | 20260506 09:30:16.340 | INFO | "pinned" : " 29 GB" | 20260506 09:30:16.340 | INFO | } | 20260506 09:30:16.340 | INFO | Perf report for the last 5.13944 seconds: | 20260506 09:30:16.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:30:16.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:16.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:16.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:16.340 | INFO | ANALYSIS : 57.9%, 2.8, 85.8ms, 0 | 20260506 09:30:16.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:16.340 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260506 09:30:16.340 | INFO | PRE_HQ : 57.1%, 2.1, 61.5ms, 0 | 20260506 09:30:16.340 | INFO | BAZWRITER : 61.6%, 4.8, 151.1ms, 0 | 20260506 09:30:16.340 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 09:30:16.340 | INFO | Uploaded 36GiB to GPU in 1.67165s (21.5356GiB/s) | 20260506 09:30:16.340 | INFO | Downloaded 38GiB to GPU in 1.73837s (21.8596GiB/s) | 20260506 09:30:21.479 | INFO | Analyzing chunk frames = [291328,291840) | 20260506 09:30:21.480 | INFO | MemInfo: { | 20260506 09:30:21.480 | INFO | "gpu" : " 17 GB", | 20260506 09:30:21.480 | INFO | "huge" : "", | 20260506 09:30:21.480 | INFO | "huge_shared" : " 27 GB", | 20260506 09:30:21.480 | INFO | "malloc" : " 14 GB", | 20260506 09:30:21.480 | INFO | "pinned" : " 29 GB" | 20260506 09:30:21.480 | INFO | } | 20260506 09:30:21.480 | INFO | Perf report for the last 5.13956 seconds: | 20260506 09:30:21.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:30:21.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:21.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:21.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:21.480 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260506 09:30:21.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:21.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:21.480 | INFO | PRE_HQ : 30.1%, 1.0, 15.7ms, 0 | 20260506 09:30:21.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:21.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:21.480 | INFO | Uploaded 36GiB to GPU in 1.60156s (22.4781GiB/s) | 20260506 09:30:21.480 | INFO | Downloaded 34GiB to GPU in 1.51932s (22.3785GiB/s) | 20260506 09:30:26.619 | INFO | Analyzing chunk frames = [291840,292352) | 20260506 09:30:26.619 | INFO | MemInfo: { | 20260506 09:30:26.619 | INFO | "gpu" : " 17 GB", | 20260506 09:30:26.619 | INFO | "huge" : "", | 20260506 09:30:26.619 | INFO | "huge_shared" : " 27 GB", | 20260506 09:30:26.619 | INFO | "malloc" : " 14 GB", | 20260506 09:30:26.619 | INFO | "pinned" : " 29 GB" | 20260506 09:30:26.619 | INFO | } | 20260506 09:30:26.619 | INFO | Perf report for the last 5.13912 seconds: | 20260506 09:30:26.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:30:26.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:26.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:26.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:26.619 | INFO | ANALYSIS : 57.0%, 3.0, 88.6ms, 0 | 20260506 09:30:26.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:26.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:26.619 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260506 09:30:26.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:26.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:26.619 | INFO | Uploaded 36GiB to GPU in 1.60449s (22.437GiB/s) | 20260506 09:30:26.619 | INFO | Downloaded 34GiB to GPU in 1.52293s (22.3254GiB/s) | 20260506 09:30:31.758 | INFO | Analyzing chunk frames = [292352,292864) | 20260506 09:30:31.758 | INFO | MemInfo: { | 20260506 09:30:31.758 | INFO | "gpu" : " 17 GB", | 20260506 09:30:31.758 | INFO | "huge" : "", | 20260506 09:30:31.758 | INFO | "huge_shared" : " 27 GB", | 20260506 09:30:31.758 | INFO | "malloc" : " 14 GB", | 20260506 09:30:31.758 | INFO | "pinned" : " 29 GB" | 20260506 09:30:31.758 | INFO | } | 20260506 09:30:31.758 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:30:31.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:30:31.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:31.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:31.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:31.758 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:30:31.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:31.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:31.758 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:30:31.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:31.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:31.758 | INFO | Uploaded 36GiB to GPU in 1.6055s (22.423GiB/s) | 20260506 09:30:31.758 | INFO | Downloaded 34GiB to GPU in 1.52378s (22.3129GiB/s) | 20260506 09:30:36.897 | INFO | Analyzing chunk frames = [292864,293376) | 20260506 09:30:36.897 | INFO | MemInfo: { | 20260506 09:30:36.897 | INFO | "gpu" : " 17 GB", | 20260506 09:30:36.897 | INFO | "huge" : "", | 20260506 09:30:36.897 | INFO | "huge_shared" : " 27 GB", | 20260506 09:30:36.897 | INFO | "malloc" : " 14 GB", | 20260506 09:30:36.897 | INFO | "pinned" : " 29 GB" | 20260506 09:30:36.897 | INFO | } | 20260506 09:30:36.897 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:30:36.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:30:36.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:36.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:36.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:36.897 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:30:36.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:36.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:36.897 | INFO | PRE_HQ : 30.0%, 1.0, 15.5ms, 0 | 20260506 09:30:36.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:36.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:36.898 | INFO | Uploaded 36GiB to GPU in 1.60243s (22.4659GiB/s) | 20260506 09:30:36.898 | INFO | Downloaded 34GiB to GPU in 1.52047s (22.3615GiB/s) | 20260506 09:30:42.037 | INFO | Analyzing chunk frames = [293376,293888) | 20260506 09:30:42.037 | INFO | MemInfo: { | 20260506 09:30:42.037 | INFO | "gpu" : " 17 GB", | 20260506 09:30:42.037 | INFO | "huge" : "", | 20260506 09:30:42.037 | INFO | "huge_shared" : " 27 GB", | 20260506 09:30:42.037 | INFO | "malloc" : " 14 GB", | 20260506 09:30:42.037 | INFO | "pinned" : " 29 GB" | 20260506 09:30:42.037 | INFO | } | 20260506 09:30:42.037 | INFO | Perf report for the last 5.13943 seconds: | 20260506 09:30:42.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:30:42.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:42.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:42.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:42.037 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 09:30:42.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:42.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:42.037 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:30:42.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:42.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:42.037 | INFO | Uploaded 36GiB to GPU in 1.60421s (22.4409GiB/s) | 20260506 09:30:42.037 | INFO | Downloaded 34GiB to GPU in 1.52109s (22.3524GiB/s) | 20260506 09:30:47.176 | INFO | Analyzing chunk frames = [293888,294400) | 20260506 09:30:47.176 | INFO | MemInfo: { | 20260506 09:30:47.176 | INFO | "gpu" : " 17 GB", | 20260506 09:30:47.176 | INFO | "huge" : "", | 20260506 09:30:47.176 | INFO | "huge_shared" : " 27 GB", | 20260506 09:30:47.176 | INFO | "malloc" : " 14 GB", | 20260506 09:30:47.176 | INFO | "pinned" : " 29 GB" | 20260506 09:30:47.176 | INFO | } | 20260506 09:30:47.176 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:30:47.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:30:47.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:47.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:47.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:47.176 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:30:47.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:47.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:47.176 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:30:47.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:47.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:47.177 | INFO | Uploaded 36GiB to GPU in 1.6041s (22.4425GiB/s) | 20260506 09:30:47.177 | INFO | Downloaded 34GiB to GPU in 1.52061s (22.3594GiB/s) | 20260506 09:30:52.315 | INFO | Analyzing chunk frames = [294400,294912) | 20260506 09:30:52.315 | INFO | MemInfo: { | 20260506 09:30:52.315 | INFO | "gpu" : " 17 GB", | 20260506 09:30:52.315 | INFO | "huge" : "", | 20260506 09:30:52.315 | INFO | "huge_shared" : " 27 GB", | 20260506 09:30:52.315 | INFO | "malloc" : " 14 GB", | 20260506 09:30:52.315 | INFO | "pinned" : " 29 GB" | 20260506 09:30:52.315 | INFO | } | 20260506 09:30:52.315 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:30:52.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:30:52.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:52.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:52.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:52.315 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:30:52.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:52.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:52.315 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 09:30:52.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:52.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:52.316 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:30:52.316 | INFO | Total mem usage: 15.7 GiB | 20260506 09:30:52.316 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 2.56%) | 20260506 09:30:52.316 | INFO | Uploaded 36GiB to GPU in 1.60488s (22.4317GiB/s) | 20260506 09:30:52.316 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -2.96%) | 20260506 09:30:52.316 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:30:52.316 | INFO | High Water Breakdown: | 20260506 09:30:52.316 | INFO | 59.5% : 9.1 GiB ( 96.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:30:52.316 | INFO | 40.3% : 6.17 GiB ( 92.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:30:52.316 | INFO | Downloaded 34GiB to GPU in 1.52206s (22.3381GiB/s) | 20260506 09:30:52.316 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:30:52.316 | INFO | ------------------------------------------ | 20260506 09:30:52.316 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:30:52.316 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:30:52.316 | INFO | Total mem usage: 26.5 GiB | 20260506 09:30:52.316 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:30:52.316 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:30:52.316 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:30:52.316 | INFO | High Water Breakdown: | 20260506 09:30:52.316 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:30:52.316 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:30:52.316 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:30:52.316 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:30:52.316 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:30:52.316 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:30:52.316 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:30:52.316 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:30:52.316 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:30:52.316 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:30:52.316 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:30:52.316 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:30:52.316 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:30:52.316 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:30:52.316 | INFO | ------------------------------------------ | 20260506 09:30:52.316 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:30:52.316 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:30:52.316 | INFO | Total mem usage: 25.6 GiB | 20260506 09:30:52.316 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:30:52.316 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:30:52.316 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:30:52.316 | INFO | High Water Breakdown: | 20260506 09:30:52.316 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:30:52.316 | INFO | ------------------------------------------ | 20260506 09:30:52.316 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:30:52.316 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:30:52.316 | INFO | Total mem usage: 15.7 GiB | 20260506 09:30:52.316 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:30:52.316 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:30:52.316 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:30:52.316 | INFO | High Water Breakdown: | 20260506 09:30:52.316 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:30:52.316 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:30:52.316 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:30:52.316 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:30:52.316 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:30:52.316 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:30:52.316 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:30:52.316 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:30:52.316 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:30:52.316 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:30:52.316 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:30:52.316 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:30:52.316 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:30:52.316 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:30:52.316 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:30:52.316 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:30:52.316 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:30:52.316 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:30:52.316 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:30:52.316 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:30:52.316 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:30:52.316 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:30:52.316 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:30:52.316 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:30:52.316 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:30:52.316 | INFO | ------------------------------------------ | 20260506 09:30:52.316 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:30:52.316 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:30:52.316 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:30:52.316 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:30:52.316 | INFO | PerfReport: Full Execution -- 3.02e+01 / 6.96e+01, 1.26e+01 / 0.00e+00 | 20260506 09:30:52.316 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / 8.61e-02, 4.56e-03 / 0.00e+00 | 20260506 09:30:52.317 | INFO | PerfReport: 10.89%, Baseline -- 3.29e+00 / 3.29e+00, 1.70e-02 / 0.00e+00 | 20260506 09:30:52.317 | INFO | PerfReport: 3.05%, Histogram -- 9.21e-01 / 1.06e+00, 5.19e-02 / 0.00e+00 | 20260506 09:30:52.317 | INFO | PerfReport: 16.05%, DME -- 3.88e+01 / 4.37e+01, 3.14e+00 / 0.00e+00 | 20260506 09:30:52.317 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.72e+01 / 1.71e+01, 4.16e-02 / 0.00e+00 | 20260506 09:30:52.317 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.27e+00 / 2.37e+00, 1.36e-01 / 0.00e+00 | 20260506 09:30:52.317 | INFO | PerfReport: 3.84%, Metrics -- 1.16e+00 / 1.44e+00, 1.79e-01 / 0.00e+00 | 20260506 09:30:52.317 | INFO | PerfReport: 1.50%, PulseSerializing -- 4.52e-01 / 5.07e-01, 6.55e-02 / 0.00e+00 | 20260506 09:30:52.317 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:30:52.317 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:30:52.317 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:30:52.317 | INFO | PerfReport: Full Execution -- 9.03e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 09:30:52.317 | INFO | PerfReport: 18.59%, Upload -- 1.68e+01 / 0.00e+00, 4.85e-01 / 0.00e+00 | 20260506 09:30:52.317 | INFO | PerfReport: 17.86%, Download -- 1.61e+01 / 0.00e+00, 8.36e-01 / 0.00e+00 | 20260506 09:30:52.317 | INFO | PerfReport: 33.49%, Compute -- 3.02e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 09:30:52.317 | INFO | PerfReport: 0.56%, LockUpload -- 5.09e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260506 09:30:52.317 | INFO | PerfReport: 0.00%, LockDownload -- 4.95e-04 / 0.00e+00, 1.69e-04 / 0.00e+00 | 20260506 09:30:52.317 | INFO | PerfReport: 29.49%, LockCompute -- 2.66e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 09:30:52.317 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260506 09:30:52.412 | INFO | Starting a round of Baz IO | 20260506 09:30:55.338 | INFO | Populating RTMetrics report | 20260506 09:30:55.338 | INFO | RTMetrics report populated | 20260506 09:30:55.338 | INFO | Opening tmp RTMetrics json file | 20260506 09:30:55.399 | INFO | Serializing RTMetrics to json | 20260506 09:30:55.409 | INFO | Serialization done, swapping tmp file now... | 20260506 09:30:55.409 | INFO | RTMetrics done | 20260506 09:30:55.499 | INFO | Finished a round of Baz IO: Wrote 1.853 GiB in 3.086 seconds | 20260506 09:30:55.499 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.77% / 0.10% / 46.13% | 20260506 09:30:55.499 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.32749% / 0.000333786% / 3.17891e-05% | 20260506 09:30:55.499 | INFO | Baz IO overall: | 20260506 09:30:55.499 | INFO | BytesWritten : 113831254900 | 20260506 09:30:55.499 | INFO | NumEvents : 53864232692 | 20260506 09:30:57.455 | INFO | Analyzing chunk frames = [294912,295424) | 20260506 09:30:57.455 | INFO | MemInfo: { | 20260506 09:30:57.455 | INFO | "gpu" : " 17 GB", | 20260506 09:30:57.455 | INFO | "huge" : "", | 20260506 09:30:57.455 | INFO | "huge_shared" : " 27 GB", | 20260506 09:30:57.455 | INFO | "malloc" : " 14 GB", | 20260506 09:30:57.455 | INFO | "pinned" : " 29 GB" | 20260506 09:30:57.455 | INFO | } | 20260506 09:30:57.455 | INFO | Perf report for the last 5.13938 seconds: | 20260506 09:30:57.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:30:57.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:57.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:57.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:57.455 | INFO | ANALYSIS : 57.8%, 2.9, 86.7ms, 0 | 20260506 09:30:57.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:30:57.455 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 09:30:57.455 | INFO | PRE_HQ : 57.0%, 2.1, 61.6ms, 0 | 20260506 09:30:57.455 | INFO | BAZWRITER : 59.7%, 4.6, 141.8ms, 0 | 20260506 09:30:57.455 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 09:30:57.455 | INFO | Uploaded 36GiB to GPU in 1.66115s (21.6717GiB/s) | 20260506 09:30:57.455 | INFO | Downloaded 38GiB to GPU in 1.72505s (22.0284GiB/s) | 20260506 09:31:02.594 | INFO | Analyzing chunk frames = [295424,295936) | 20260506 09:31:02.594 | INFO | MemInfo: { | 20260506 09:31:02.594 | INFO | "gpu" : " 17 GB", | 20260506 09:31:02.594 | INFO | "huge" : "", | 20260506 09:31:02.594 | INFO | "huge_shared" : " 27 GB", | 20260506 09:31:02.594 | INFO | "malloc" : " 14 GB", | 20260506 09:31:02.594 | INFO | "pinned" : " 29 GB" | 20260506 09:31:02.594 | INFO | } | 20260506 09:31:02.594 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:31:02.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:31:02.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:02.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:02.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:02.594 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260506 09:31:02.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:02.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:02.594 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:31:02.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:02.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:02.594 | INFO | Uploaded 36GiB to GPU in 1.60278s (22.461GiB/s) | 20260506 09:31:02.594 | INFO | Downloaded 34GiB to GPU in 1.52002s (22.3682GiB/s) | 20260506 09:31:07.733 | INFO | Analyzing chunk frames = [295936,296448) | 20260506 09:31:07.733 | INFO | MemInfo: { | 20260506 09:31:07.733 | INFO | "gpu" : " 17 GB", | 20260506 09:31:07.733 | INFO | "huge" : "", | 20260506 09:31:07.733 | INFO | "huge_shared" : " 27 GB", | 20260506 09:31:07.733 | INFO | "malloc" : " 14 GB", | 20260506 09:31:07.733 | INFO | "pinned" : " 29 GB" | 20260506 09:31:07.733 | INFO | } | 20260506 09:31:07.734 | INFO | Perf report for the last 5.13932 seconds: | 20260506 09:31:07.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:31:07.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:07.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:07.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:07.734 | INFO | ANALYSIS : 57.0%, 3.0, 88.6ms, 0 | 20260506 09:31:07.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:07.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:07.734 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:31:07.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:07.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:07.734 | INFO | Uploaded 36GiB to GPU in 1.60525s (22.4264GiB/s) | 20260506 09:31:07.734 | INFO | Downloaded 34GiB to GPU in 1.52307s (22.3234GiB/s) | 20260506 09:31:12.873 | INFO | Analyzing chunk frames = [296448,296960) | 20260506 09:31:12.873 | INFO | MemInfo: { | 20260506 09:31:12.873 | INFO | "gpu" : " 17 GB", | 20260506 09:31:12.873 | INFO | "huge" : "", | 20260506 09:31:12.873 | INFO | "huge_shared" : " 27 GB", | 20260506 09:31:12.873 | INFO | "malloc" : " 14 GB", | 20260506 09:31:12.873 | INFO | "pinned" : " 29 GB" | 20260506 09:31:12.873 | INFO | } | 20260506 09:31:12.873 | INFO | Perf report for the last 5.13936 seconds: | 20260506 09:31:12.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:31:12.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:12.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:12.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:12.873 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:31:12.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:12.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:12.873 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:31:12.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:12.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:12.873 | INFO | Uploaded 36GiB to GPU in 1.60546s (22.4235GiB/s) | 20260506 09:31:12.873 | INFO | Downloaded 34GiB to GPU in 1.52207s (22.338GiB/s) | 20260506 09:31:18.012 | INFO | Analyzing chunk frames = [296960,297472) | 20260506 09:31:18.012 | INFO | MemInfo: { | 20260506 09:31:18.012 | INFO | "gpu" : " 17 GB", | 20260506 09:31:18.012 | INFO | "huge" : "", | 20260506 09:31:18.012 | INFO | "huge_shared" : " 27 GB", | 20260506 09:31:18.012 | INFO | "malloc" : " 14 GB", | 20260506 09:31:18.012 | INFO | "pinned" : " 29 GB" | 20260506 09:31:18.012 | INFO | } | 20260506 09:31:18.012 | INFO | Perf report for the last 5.13924 seconds: | 20260506 09:31:18.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:31:18.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:18.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:18.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:18.012 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:31:18.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:18.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:18.012 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:31:18.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:18.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:18.012 | INFO | Uploaded 36GiB to GPU in 1.60392s (22.445GiB/s) | 20260506 09:31:18.012 | INFO | Downloaded 34GiB to GPU in 1.52097s (22.3542GiB/s) | 20260506 09:31:23.151 | INFO | Analyzing chunk frames = [297472,297984) | 20260506 09:31:23.151 | INFO | MemInfo: { | 20260506 09:31:23.151 | INFO | "gpu" : " 17 GB", | 20260506 09:31:23.151 | INFO | "huge" : "", | 20260506 09:31:23.151 | INFO | "huge_shared" : " 27 GB", | 20260506 09:31:23.151 | INFO | "malloc" : " 14 GB", | 20260506 09:31:23.151 | INFO | "pinned" : " 29 GB" | 20260506 09:31:23.151 | INFO | } | 20260506 09:31:23.151 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:31:23.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:31:23.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:23.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:23.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:23.152 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 09:31:23.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:23.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:23.152 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:31:23.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:23.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:23.152 | INFO | Uploaded 36GiB to GPU in 1.6045s (22.4369GiB/s) | 20260506 09:31:23.152 | INFO | Downloaded 34GiB to GPU in 1.52119s (22.3509GiB/s) | 20260506 09:31:28.291 | INFO | Analyzing chunk frames = [297984,298496) | 20260506 09:31:28.291 | INFO | MemInfo: { | 20260506 09:31:28.291 | INFO | "gpu" : " 17 GB", | 20260506 09:31:28.291 | INFO | "huge" : "", | 20260506 09:31:28.291 | INFO | "huge_shared" : " 27 GB", | 20260506 09:31:28.291 | INFO | "malloc" : " 14 GB", | 20260506 09:31:28.291 | INFO | "pinned" : " 29 GB" | 20260506 09:31:28.291 | INFO | } | 20260506 09:31:28.291 | INFO | Perf report for the last 5.13942 seconds: | 20260506 09:31:28.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:31:28.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:28.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:28.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:28.291 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:31:28.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:28.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:28.291 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:31:28.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:28.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:28.291 | INFO | Uploaded 36GiB to GPU in 1.6036s (22.4495GiB/s) | 20260506 09:31:28.291 | INFO | Downloaded 34GiB to GPU in 1.52116s (22.3513GiB/s) | 20260506 09:31:33.430 | INFO | Analyzing chunk frames = [298496,299008) | 20260506 09:31:33.430 | INFO | MemInfo: { | 20260506 09:31:33.430 | INFO | "gpu" : " 17 GB", | 20260506 09:31:33.430 | INFO | "huge" : "", | 20260506 09:31:33.430 | INFO | "huge_shared" : " 27 GB", | 20260506 09:31:33.430 | INFO | "malloc" : " 14 GB", | 20260506 09:31:33.430 | INFO | "pinned" : " 29 GB" | 20260506 09:31:33.430 | INFO | } | 20260506 09:31:33.430 | INFO | Perf report for the last 5.1395 seconds: | 20260506 09:31:33.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:31:33.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:33.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:33.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:33.430 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:31:33.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:33.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:33.430 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 09:31:33.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:33.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:33.431 | INFO | Uploaded 36GiB to GPU in 1.60437s (22.4387GiB/s) | 20260506 09:31:33.431 | INFO | Downloaded 34GiB to GPU in 1.52207s (22.338GiB/s) | 20260506 09:31:33.528 | INFO | Starting a round of Baz IO | 20260506 09:31:36.451 | INFO | Populating RTMetrics report | 20260506 09:31:36.452 | INFO | RTMetrics report populated | 20260506 09:31:36.452 | INFO | Opening tmp RTMetrics json file | 20260506 09:31:36.494 | INFO | Serializing RTMetrics to json | 20260506 09:31:36.548 | INFO | Serialization done, swapping tmp file now... | 20260506 09:31:36.578 | INFO | RTMetrics done | 20260506 09:31:36.628 | INFO | Finished a round of Baz IO: Wrote 1.858 GiB in 3.1 seconds | 20260506 09:31:36.628 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.67% / 0.10% / 46.23% | 20260506 09:31:36.628 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.32072% / 0.000278155% / 5.5631e-05% | 20260506 09:31:36.628 | INFO | Baz IO overall: | 20260506 09:31:36.628 | INFO | BytesWritten : 115825872659 | 20260506 09:31:36.628 | INFO | NumEvents : 54833675302 | 20260506 09:31:38.569 | INFO | Analyzing chunk frames = [299008,299520) | 20260506 09:31:38.569 | INFO | MemInfo: { | 20260506 09:31:38.569 | INFO | "gpu" : " 17 GB", | 20260506 09:31:38.569 | INFO | "huge" : "", | 20260506 09:31:38.569 | INFO | "huge_shared" : " 27 GB", | 20260506 09:31:38.569 | INFO | "malloc" : " 14 GB", | 20260506 09:31:38.569 | INFO | "pinned" : " 29 GB" | 20260506 09:31:38.569 | INFO | } | 20260506 09:31:38.569 | INFO | Perf report for the last 5.13913 seconds: | 20260506 09:31:38.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:31:38.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:38.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:38.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:38.570 | INFO | ANALYSIS : 57.8%, 2.9, 87.5ms, 0 | 20260506 09:31:38.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:38.570 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260506 09:31:38.570 | INFO | PRE_HQ : 56.9%, 2.1, 62.1ms, 0 | 20260506 09:31:38.570 | INFO | BAZWRITER : 59.9%, 4.5, 139.6ms, 0 | 20260506 09:31:38.570 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 09:31:38.570 | INFO | Uploaded 36GiB to GPU in 1.64359s (21.9033GiB/s) | 20260506 09:31:38.570 | INFO | Downloaded 38GiB to GPU in 1.70804s (22.2477GiB/s) | 20260506 09:31:43.709 | INFO | Analyzing chunk frames = [299520,300032) | 20260506 09:31:43.709 | INFO | MemInfo: { | 20260506 09:31:43.709 | INFO | "gpu" : " 17 GB", | 20260506 09:31:43.709 | INFO | "huge" : "", | 20260506 09:31:43.709 | INFO | "huge_shared" : " 27 GB", | 20260506 09:31:43.709 | INFO | "malloc" : " 14 GB", | 20260506 09:31:43.709 | INFO | "pinned" : " 29 GB" | 20260506 09:31:43.709 | INFO | } | 20260506 09:31:43.709 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:31:43.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:31:43.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:43.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:43.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:43.709 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260506 09:31:43.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:43.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:43.709 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:31:43.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:43.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:43.709 | INFO | Uploaded 36GiB to GPU in 1.60157s (22.478GiB/s) | 20260506 09:31:43.709 | INFO | Downloaded 34GiB to GPU in 1.51913s (22.3812GiB/s) | 20260506 09:31:48.848 | INFO | Analyzing chunk frames = [300032,300544) | 20260506 09:31:48.848 | INFO | MemInfo: { | 20260506 09:31:48.848 | INFO | "gpu" : " 17 GB", | 20260506 09:31:48.848 | INFO | "huge" : "", | 20260506 09:31:48.848 | INFO | "huge_shared" : " 27 GB", | 20260506 09:31:48.848 | INFO | "malloc" : " 14 GB", | 20260506 09:31:48.848 | INFO | "pinned" : " 29 GB" | 20260506 09:31:48.848 | INFO | } | 20260506 09:31:48.848 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:31:48.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:31:48.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:48.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:48.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:48.848 | INFO | ANALYSIS : 57.0%, 3.0, 88.6ms, 0 | 20260506 09:31:48.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:48.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:48.848 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:31:48.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:48.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:48.849 | INFO | Uploaded 36GiB to GPU in 1.60599s (22.416GiB/s) | 20260506 09:31:48.849 | INFO | Downloaded 34GiB to GPU in 1.52421s (22.3067GiB/s) | 20260506 09:31:53.987 | INFO | Analyzing chunk frames = [300544,301056) | 20260506 09:31:53.987 | INFO | MemInfo: { | 20260506 09:31:53.987 | INFO | "gpu" : " 17 GB", | 20260506 09:31:53.987 | INFO | "huge" : "", | 20260506 09:31:53.987 | INFO | "huge_shared" : " 27 GB", | 20260506 09:31:53.987 | INFO | "malloc" : " 14 GB", | 20260506 09:31:53.987 | INFO | "pinned" : " 29 GB" | 20260506 09:31:53.987 | INFO | } | 20260506 09:31:53.987 | INFO | Perf report for the last 5.13924 seconds: | 20260506 09:31:53.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:31:53.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:53.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:53.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:53.988 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:31:53.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:53.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:53.988 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:31:53.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:53.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:53.988 | INFO | Uploaded 36GiB to GPU in 1.60481s (22.4326GiB/s) | 20260506 09:31:53.988 | INFO | Downloaded 34GiB to GPU in 1.52284s (22.3268GiB/s) | 20260506 09:31:59.127 | INFO | Analyzing chunk frames = [301056,301568) | 20260506 09:31:59.127 | INFO | MemInfo: { | 20260506 09:31:59.127 | INFO | "gpu" : " 17 GB", | 20260506 09:31:59.127 | INFO | "huge" : "", | 20260506 09:31:59.127 | INFO | "huge_shared" : " 27 GB", | 20260506 09:31:59.127 | INFO | "malloc" : " 14 GB", | 20260506 09:31:59.127 | INFO | "pinned" : " 29 GB" | 20260506 09:31:59.127 | INFO | } | 20260506 09:31:59.127 | INFO | Perf report for the last 5.13964 seconds: | 20260506 09:31:59.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:31:59.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:59.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:59.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:59.127 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:31:59.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:59.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:59.127 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:31:59.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:59.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:31:59.128 | INFO | Uploaded 36GiB to GPU in 1.6031s (22.4565GiB/s) | 20260506 09:31:59.128 | INFO | Downloaded 34GiB to GPU in 1.52166s (22.344GiB/s) | 20260506 09:32:04.266 | INFO | Analyzing chunk frames = [301568,302080) | 20260506 09:32:04.266 | INFO | MemInfo: { | 20260506 09:32:04.266 | INFO | "gpu" : " 17 GB", | 20260506 09:32:04.266 | INFO | "huge" : "", | 20260506 09:32:04.266 | INFO | "huge_shared" : " 27 GB", | 20260506 09:32:04.266 | INFO | "malloc" : " 14 GB", | 20260506 09:32:04.266 | INFO | "pinned" : " 29 GB" | 20260506 09:32:04.266 | INFO | } | 20260506 09:32:04.266 | INFO | Perf report for the last 5.13905 seconds: | 20260506 09:32:04.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:32:04.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:04.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:04.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:04.266 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 09:32:04.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:04.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:04.266 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:32:04.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:04.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:04.266 | INFO | Uploaded 36GiB to GPU in 1.60414s (22.4419GiB/s) | 20260506 09:32:04.266 | INFO | Downloaded 34GiB to GPU in 1.52089s (22.3554GiB/s) | 20260506 09:32:09.405 | INFO | Analyzing chunk frames = [302080,302592) | 20260506 09:32:09.405 | INFO | MemInfo: { | 20260506 09:32:09.405 | INFO | "gpu" : " 17 GB", | 20260506 09:32:09.406 | INFO | "huge" : "", | 20260506 09:32:09.406 | INFO | "huge_shared" : " 27 GB", | 20260506 09:32:09.406 | INFO | "malloc" : " 14 GB", | 20260506 09:32:09.406 | INFO | "pinned" : " 29 GB" | 20260506 09:32:09.406 | INFO | } | 20260506 09:32:09.406 | INFO | Perf report for the last 5.13945 seconds: | 20260506 09:32:09.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:32:09.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:09.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:09.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:09.406 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:32:09.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:09.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:09.406 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:32:09.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:09.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:09.406 | INFO | Uploaded 36GiB to GPU in 1.60435s (22.4391GiB/s) | 20260506 09:32:09.406 | INFO | Downloaded 34GiB to GPU in 1.52174s (22.3428GiB/s) | 20260506 09:32:14.545 | INFO | Analyzing chunk frames = [302592,303104) | 20260506 09:32:14.545 | INFO | MemInfo: { | 20260506 09:32:14.545 | INFO | "gpu" : " 17 GB", | 20260506 09:32:14.545 | INFO | "huge" : "", | 20260506 09:32:14.545 | INFO | "huge_shared" : " 27 GB", | 20260506 09:32:14.545 | INFO | "malloc" : " 14 GB", | 20260506 09:32:14.545 | INFO | "pinned" : " 29 GB" | 20260506 09:32:14.545 | INFO | } | 20260506 09:32:14.545 | INFO | Perf report for the last 5.13937 seconds: | 20260506 09:32:14.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:32:14.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:14.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:14.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:14.545 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:32:14.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:14.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:14.545 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:32:14.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:14.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:14.546 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:32:14.546 | INFO | Total mem usage: 15.7 GiB | 20260506 09:32:14.546 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 2.56%) | 20260506 09:32:14.546 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -6.71%) | 20260506 09:32:14.546 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:32:14.546 | INFO | High Water Breakdown: | 20260506 09:32:14.546 | INFO | 58.9% : 8.7 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:32:14.546 | INFO | 40.9% : 6.04 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:32:14.546 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:32:14.546 | INFO | ------------------------------------------ | 20260506 09:32:14.546 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:32:14.546 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:32:14.546 | INFO | Total mem usage: 26.5 GiB | 20260506 09:32:14.546 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:32:14.546 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:32:14.546 | INFO | Uploaded 36GiB to GPU in 1.6048s (22.4327GiB/s) | 20260506 09:32:14.546 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:32:14.546 | INFO | Downloaded 34GiB to GPU in 1.52214s (22.337GiB/s) | 20260506 09:32:14.546 | INFO | High Water Breakdown: | 20260506 09:32:14.546 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:32:14.546 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:32:14.546 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:32:14.546 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:32:14.546 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:32:14.546 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:32:14.546 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:32:14.546 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:32:14.546 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:32:14.546 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:32:14.546 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:32:14.546 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:32:14.546 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:32:14.546 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:32:14.546 | INFO | ------------------------------------------ | 20260506 09:32:14.546 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:32:14.546 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:32:14.546 | INFO | Total mem usage: 25.6 GiB | 20260506 09:32:14.546 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:32:14.546 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:32:14.546 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:32:14.546 | INFO | High Water Breakdown: | 20260506 09:32:14.546 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:32:14.546 | INFO | ------------------------------------------ | 20260506 09:32:14.546 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:32:14.546 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:32:14.546 | INFO | Total mem usage: 15.7 GiB | 20260506 09:32:14.546 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:32:14.546 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:32:14.546 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:32:14.546 | INFO | High Water Breakdown: | 20260506 09:32:14.546 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:32:14.546 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:32:14.546 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:32:14.546 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:32:14.546 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:32:14.546 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:32:14.546 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:32:14.546 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:32:14.546 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:32:14.546 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:32:14.546 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:32:14.546 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:32:14.546 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:32:14.546 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:32:14.546 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:32:14.546 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:32:14.546 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:32:14.546 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:32:14.546 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:32:14.546 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:32:14.546 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:32:14.546 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:32:14.546 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:32:14.546 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:32:14.546 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:32:14.546 | INFO | ------------------------------------------ | 20260506 09:32:14.546 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:32:14.546 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:32:14.546 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:32:14.546 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:32:14.546 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 09:32:14.546 | INFO | PerfReport: 0.28%, AggregateStats -- 8.61e-02 / -nan, 4.48e-03 / 0.00e+00 | 20260506 09:32:14.546 | INFO | PerfReport: 10.90%, Baseline -- 3.29e+00 / -nan, 1.98e-02 / 0.00e+00 | 20260506 09:32:14.546 | INFO | PerfReport: 3.05%, Histogram -- 9.22e-01 / -nan, 5.27e-02 / 0.00e+00 | 20260506 09:32:14.546 | INFO | PerfReport: 16.04%, DME -- 3.88e+01 / -nan, 3.09e+00 / 0.00e+00 | 20260506 09:32:14.546 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.72e+01 / -nan, 4.07e-02 / 0.00e+00 | 20260506 09:32:14.546 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.27e+00 / -nan, 1.36e-01 / 0.00e+00 | 20260506 09:32:14.546 | INFO | PerfReport: 3.83%, Metrics -- 1.16e+00 / -nan, 1.81e-01 / 0.00e+00 | 20260506 09:32:14.546 | INFO | PerfReport: 1.49%, PulseSerializing -- 4.52e-01 / -nan, 6.53e-02 / 0.00e+00 | 20260506 09:32:14.546 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:32:14.546 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:32:14.546 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:32:14.546 | INFO | PerfReport: Full Execution -- 9.04e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 09:32:14.546 | INFO | PerfReport: 18.55%, Upload -- 1.68e+01 / 0.00e+00, 4.31e-01 / 0.00e+00 | 20260506 09:32:14.546 | INFO | PerfReport: 17.81%, Download -- 1.61e+01 / 0.00e+00, 7.63e-01 / 0.00e+00 | 20260506 09:32:14.546 | INFO | PerfReport: 33.44%, Compute -- 3.02e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 09:32:14.546 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 09:32:14.546 | INFO | PerfReport: 0.00%, LockDownload -- 4.88e-04 / 0.00e+00, 2.40e-04 / 0.00e+00 | 20260506 09:32:14.546 | INFO | PerfReport: 29.63%, LockCompute -- 2.68e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 09:32:14.547 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.17e-02 / 0.00e+00 | 20260506 09:32:14.642 | INFO | Starting a round of Baz IO | 20260506 09:32:17.563 | INFO | Populating RTMetrics report | 20260506 09:32:17.563 | INFO | RTMetrics report populated | 20260506 09:32:17.563 | INFO | Opening tmp RTMetrics json file | 20260506 09:32:17.616 | INFO | Serializing RTMetrics to json | 20260506 09:32:17.621 | INFO | Serialization done, swapping tmp file now... | 20260506 09:32:17.628 | INFO | RTMetrics done | 20260506 09:32:17.779 | INFO | Finished a round of Baz IO: Wrote 1.856 GiB in 3.136 seconds | 20260506 09:32:17.779 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.57% / 0.10% / 46.33% | 20260506 09:32:17.779 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.23427% / 0.000309944% / 6.75519e-05% | 20260506 09:32:17.779 | INFO | Baz IO overall: | 20260506 09:32:17.779 | INFO | BytesWritten : 117819090290 | 20260506 09:32:17.779 | INFO | NumEvents : 55801501017 | 20260506 09:32:19.684 | INFO | Analyzing chunk frames = [303104,303616) | 20260506 09:32:19.684 | INFO | MemInfo: { | 20260506 09:32:19.684 | INFO | "gpu" : " 17 GB", | 20260506 09:32:19.684 | INFO | "huge" : "", | 20260506 09:32:19.684 | INFO | "huge_shared" : " 27 GB", | 20260506 09:32:19.684 | INFO | "malloc" : " 14 GB", | 20260506 09:32:19.684 | INFO | "pinned" : " 29 GB" | 20260506 09:32:19.684 | INFO | } | 20260506 09:32:19.684 | INFO | Perf report for the last 5.13918 seconds: | 20260506 09:32:19.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:32:19.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:19.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:19.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:19.684 | INFO | ANALYSIS : 57.8%, 2.9, 88.6ms, 0 | 20260506 09:32:19.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:19.684 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 09:32:19.684 | INFO | PRE_HQ : 56.9%, 2.1, 62.2ms, 0 | 20260506 09:32:19.684 | INFO | BAZWRITER : 60.6%, 4.3, 135.4ms, 0 | 20260506 09:32:19.684 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 09:32:19.685 | INFO | Uploaded 36GiB to GPU in 1.64488s (21.8861GiB/s) | 20260506 09:32:19.685 | INFO | Downloaded 38GiB to GPU in 1.70765s (22.2528GiB/s) | 20260506 09:32:24.824 | INFO | Analyzing chunk frames = [303616,304128) | 20260506 09:32:24.824 | INFO | MemInfo: { | 20260506 09:32:24.824 | INFO | "gpu" : " 17 GB", | 20260506 09:32:24.824 | INFO | "huge" : "", | 20260506 09:32:24.824 | INFO | "huge_shared" : " 27 GB", | 20260506 09:32:24.824 | INFO | "malloc" : " 14 GB", | 20260506 09:32:24.824 | INFO | "pinned" : " 29 GB" | 20260506 09:32:24.824 | INFO | } | 20260506 09:32:24.824 | INFO | Perf report for the last 5.13943 seconds: | 20260506 09:32:24.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:32:24.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:24.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:24.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:24.824 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260506 09:32:24.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:24.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:24.824 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:32:24.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:24.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:24.824 | INFO | Uploaded 36GiB to GPU in 1.6018s (22.4747GiB/s) | 20260506 09:32:24.824 | INFO | Downloaded 34GiB to GPU in 1.51836s (22.3926GiB/s) | 20260506 09:32:29.963 | INFO | Analyzing chunk frames = [304128,304640) | 20260506 09:32:29.963 | INFO | MemInfo: { | 20260506 09:32:29.963 | INFO | "gpu" : " 17 GB", | 20260506 09:32:29.963 | INFO | "huge" : "", | 20260506 09:32:29.963 | INFO | "huge_shared" : " 27 GB", | 20260506 09:32:29.963 | INFO | "malloc" : " 14 GB", | 20260506 09:32:29.963 | INFO | "pinned" : " 29 GB" | 20260506 09:32:29.963 | INFO | } | 20260506 09:32:29.963 | INFO | Perf report for the last 5.13938 seconds: | 20260506 09:32:29.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:32:29.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:29.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:29.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:29.963 | INFO | ANALYSIS : 57.0%, 3.0, 88.5ms, 0 | 20260506 09:32:29.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:29.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:29.963 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:32:29.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:29.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:29.963 | INFO | Uploaded 36GiB to GPU in 1.60588s (22.4176GiB/s) | 20260506 09:32:29.963 | INFO | Downloaded 34GiB to GPU in 1.52289s (22.326GiB/s) | 20260506 09:32:35.102 | INFO | Analyzing chunk frames = [304640,305152) | 20260506 09:32:35.102 | INFO | MemInfo: { | 20260506 09:32:35.102 | INFO | "gpu" : " 17 GB", | 20260506 09:32:35.102 | INFO | "huge" : "", | 20260506 09:32:35.102 | INFO | "huge_shared" : " 27 GB", | 20260506 09:32:35.102 | INFO | "malloc" : " 14 GB", | 20260506 09:32:35.102 | INFO | "pinned" : " 29 GB" | 20260506 09:32:35.102 | INFO | } | 20260506 09:32:35.102 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:32:35.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:32:35.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:35.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:35.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:35.102 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:32:35.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:35.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:35.103 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260506 09:32:35.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:35.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:35.103 | INFO | Uploaded 36GiB to GPU in 1.60599s (22.4161GiB/s) | 20260506 09:32:35.103 | INFO | Downloaded 34GiB to GPU in 1.52305s (22.3236GiB/s) | 20260506 09:32:40.241 | INFO | Analyzing chunk frames = [305152,305664) | 20260506 09:32:40.241 | INFO | MemInfo: { | 20260506 09:32:40.241 | INFO | "gpu" : " 17 GB", | 20260506 09:32:40.242 | INFO | "huge" : "", | 20260506 09:32:40.242 | INFO | "huge_shared" : " 27 GB", | 20260506 09:32:40.242 | INFO | "malloc" : " 14 GB", | 20260506 09:32:40.242 | INFO | "pinned" : " 29 GB" | 20260506 09:32:40.242 | INFO | } | 20260506 09:32:40.242 | INFO | Perf report for the last 5.13921 seconds: | 20260506 09:32:40.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:32:40.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:40.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:40.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:40.242 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 09:32:40.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:40.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:40.242 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:32:40.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:40.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:40.242 | INFO | Uploaded 36GiB to GPU in 1.60324s (22.4546GiB/s) | 20260506 09:32:40.242 | INFO | Downloaded 34GiB to GPU in 1.52074s (22.3575GiB/s) | 20260506 09:32:45.381 | INFO | Analyzing chunk frames = [305664,306176) | 20260506 09:32:45.381 | INFO | MemInfo: { | 20260506 09:32:45.381 | INFO | "gpu" : " 17 GB", | 20260506 09:32:45.381 | INFO | "huge" : "", | 20260506 09:32:45.381 | INFO | "huge_shared" : " 27 GB", | 20260506 09:32:45.381 | INFO | "malloc" : " 14 GB", | 20260506 09:32:45.381 | INFO | "pinned" : " 29 GB" | 20260506 09:32:45.381 | INFO | } | 20260506 09:32:45.381 | INFO | Perf report for the last 5.13967 seconds: | 20260506 09:32:45.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:32:45.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:45.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:45.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:45.381 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 09:32:45.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:45.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:45.381 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:32:45.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:45.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:45.382 | INFO | Uploaded 36GiB to GPU in 1.60402s (22.4437GiB/s) | 20260506 09:32:45.382 | INFO | Downloaded 34GiB to GPU in 1.52095s (22.3544GiB/s) | 20260506 09:32:50.520 | INFO | Analyzing chunk frames = [306176,306688) | 20260506 09:32:50.520 | INFO | MemInfo: { | 20260506 09:32:50.520 | INFO | "gpu" : " 17 GB", | 20260506 09:32:50.520 | INFO | "huge" : "", | 20260506 09:32:50.520 | INFO | "huge_shared" : " 27 GB", | 20260506 09:32:50.520 | INFO | "malloc" : " 14 GB", | 20260506 09:32:50.520 | INFO | "pinned" : " 29 GB" | 20260506 09:32:50.520 | INFO | } | 20260506 09:32:50.520 | INFO | Perf report for the last 5.13899 seconds: | 20260506 09:32:50.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:32:50.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:50.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:50.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:50.520 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:32:50.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:50.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:50.520 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:32:50.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:50.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:50.521 | INFO | Uploaded 36GiB to GPU in 1.60429s (22.4399GiB/s) | 20260506 09:32:50.521 | INFO | Downloaded 34GiB to GPU in 1.52247s (22.3322GiB/s) | 20260506 09:32:55.659 | INFO | Analyzing chunk frames = [306688,307200) | 20260506 09:32:55.659 | INFO | MemInfo: { | 20260506 09:32:55.659 | INFO | "gpu" : " 17 GB", | 20260506 09:32:55.659 | INFO | "huge" : "", | 20260506 09:32:55.659 | INFO | "huge_shared" : " 27 GB", | 20260506 09:32:55.659 | INFO | "malloc" : " 14 GB", | 20260506 09:32:55.659 | INFO | "pinned" : " 29 GB" | 20260506 09:32:55.660 | INFO | } | 20260506 09:32:55.660 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:32:55.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:32:55.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:55.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:55.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:55.660 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:32:55.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:55.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:55.660 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 09:32:55.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:55.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:32:55.660 | INFO | Uploaded 36GiB to GPU in 1.60457s (22.436GiB/s) | 20260506 09:32:55.660 | INFO | Downloaded 34GiB to GPU in 1.5224s (22.3332GiB/s) | 20260506 09:32:55.757 | INFO | Starting a round of Baz IO | 20260506 09:32:58.678 | INFO | Populating RTMetrics report | 20260506 09:32:58.678 | INFO | RTMetrics report populated | 20260506 09:32:58.678 | INFO | Opening tmp RTMetrics json file | 20260506 09:32:58.679 | INFO | Serializing RTMetrics to json | 20260506 09:32:58.683 | INFO | Serialization done, swapping tmp file now... | 20260506 09:32:58.684 | INFO | RTMetrics done | 20260506 09:32:58.798 | INFO | Finished a round of Baz IO: Wrote 1.856 GiB in 3.041 seconds | 20260506 09:32:58.798 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.47% / 0.10% / 46.43% | 20260506 09:32:58.798 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.16639% / 0.000290076% / 5.16574e-05% | 20260506 09:32:58.799 | INFO | Baz IO overall: | 20260506 09:32:58.799 | INFO | BytesWritten : 119811837875 | 20260506 09:32:58.799 | INFO | NumEvents : 56768779102 | 20260506 09:33:00.799 | INFO | Analyzing chunk frames = [307200,307712) | 20260506 09:33:00.799 | INFO | MemInfo: { | 20260506 09:33:00.799 | INFO | "gpu" : " 17 GB", | 20260506 09:33:00.799 | INFO | "huge" : "", | 20260506 09:33:00.799 | INFO | "huge_shared" : " 27 GB", | 20260506 09:33:00.799 | INFO | "malloc" : " 14 GB", | 20260506 09:33:00.799 | INFO | "pinned" : " 29 GB" | 20260506 09:33:00.799 | INFO | } | 20260506 09:33:00.799 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:33:00.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:33:00.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:00.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:00.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:00.799 | INFO | ANALYSIS : 57.7%, 2.9, 87.6ms, 0 | 20260506 09:33:00.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:00.799 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 09:33:00.799 | INFO | PRE_HQ : 56.9%, 2.1, 62.2ms, 0 | 20260506 09:33:00.799 | INFO | BAZWRITER : 58.7%, 4.5, 134.8ms, 0 | 20260506 09:33:00.799 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 09:33:00.800 | INFO | Uploaded 36GiB to GPU in 1.65367s (21.7698GiB/s) | 20260506 09:33:00.800 | INFO | Downloaded 38GiB to GPU in 1.71731s (22.1276GiB/s) | 20260506 09:33:05.938 | INFO | Analyzing chunk frames = [307712,308224) | 20260506 09:33:05.938 | INFO | MemInfo: { | 20260506 09:33:05.938 | INFO | "gpu" : " 17 GB", | 20260506 09:33:05.938 | INFO | "huge" : "", | 20260506 09:33:05.938 | INFO | "huge_shared" : " 27 GB", | 20260506 09:33:05.938 | INFO | "malloc" : " 14 GB", | 20260506 09:33:05.938 | INFO | "pinned" : " 29 GB" | 20260506 09:33:05.938 | INFO | } | 20260506 09:33:05.938 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:33:05.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:33:05.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:05.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:05.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:05.938 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260506 09:33:05.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:05.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:05.938 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:33:05.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:05.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:05.939 | INFO | Uploaded 36GiB to GPU in 1.60161s (22.4774GiB/s) | 20260506 09:33:05.939 | INFO | Downloaded 34GiB to GPU in 1.51837s (22.3925GiB/s) | 20260506 09:33:11.078 | INFO | Analyzing chunk frames = [308224,308736) | 20260506 09:33:11.078 | INFO | MemInfo: { | 20260506 09:33:11.078 | INFO | "gpu" : " 17 GB", | 20260506 09:33:11.078 | INFO | "huge" : "", | 20260506 09:33:11.078 | INFO | "huge_shared" : " 27 GB", | 20260506 09:33:11.078 | INFO | "malloc" : " 14 GB", | 20260506 09:33:11.078 | INFO | "pinned" : " 29 GB" | 20260506 09:33:11.078 | INFO | } | 20260506 09:33:11.078 | INFO | Perf report for the last 5.1394 seconds: | 20260506 09:33:11.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:33:11.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:11.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:11.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:11.078 | INFO | ANALYSIS : 56.9%, 3.0, 88.5ms, 0 | 20260506 09:33:11.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:11.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:11.078 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:33:11.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:11.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:11.078 | INFO | Uploaded 36GiB to GPU in 1.6059s (22.4173GiB/s) | 20260506 09:33:11.078 | INFO | Downloaded 34GiB to GPU in 1.52324s (22.3209GiB/s) | 20260506 09:33:16.217 | INFO | Analyzing chunk frames = [308736,309248) | 20260506 09:33:16.217 | INFO | MemInfo: { | 20260506 09:33:16.217 | INFO | "gpu" : " 17 GB", | 20260506 09:33:16.217 | INFO | "huge" : "", | 20260506 09:33:16.217 | INFO | "huge_shared" : " 27 GB", | 20260506 09:33:16.217 | INFO | "malloc" : " 14 GB", | 20260506 09:33:16.217 | INFO | "pinned" : " 29 GB" | 20260506 09:33:16.217 | INFO | } | 20260506 09:33:16.217 | INFO | Perf report for the last 5.13924 seconds: | 20260506 09:33:16.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:33:16.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:16.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:16.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:16.217 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:33:16.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:16.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:16.217 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:33:16.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:16.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:16.217 | INFO | Uploaded 36GiB to GPU in 1.6061s (22.4146GiB/s) | 20260506 09:33:16.217 | INFO | Downloaded 34GiB to GPU in 1.52335s (22.3193GiB/s) | 20260506 09:33:21.356 | INFO | Analyzing chunk frames = [309248,309760) | 20260506 09:33:21.356 | INFO | MemInfo: { | 20260506 09:33:21.356 | INFO | "gpu" : " 17 GB", | 20260506 09:33:21.356 | INFO | "huge" : "", | 20260506 09:33:21.356 | INFO | "huge_shared" : " 27 GB", | 20260506 09:33:21.356 | INFO | "malloc" : " 14 GB", | 20260506 09:33:21.356 | INFO | "pinned" : " 29 GB" | 20260506 09:33:21.356 | INFO | } | 20260506 09:33:21.356 | INFO | Perf report for the last 5.13943 seconds: | 20260506 09:33:21.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:33:21.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:21.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:21.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:21.357 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 09:33:21.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:21.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:21.357 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:33:21.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:21.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:21.357 | INFO | Uploaded 36GiB to GPU in 1.60326s (22.4542GiB/s) | 20260506 09:33:21.357 | INFO | Downloaded 34GiB to GPU in 1.52086s (22.3557GiB/s) | 20260506 09:33:26.495 | INFO | Analyzing chunk frames = [309760,310272) | 20260506 09:33:26.496 | INFO | MemInfo: { | 20260506 09:33:26.496 | INFO | "gpu" : " 17 GB", | 20260506 09:33:26.496 | INFO | "huge" : "", | 20260506 09:33:26.496 | INFO | "huge_shared" : " 27 GB", | 20260506 09:33:26.496 | INFO | "malloc" : " 14 GB", | 20260506 09:33:26.496 | INFO | "pinned" : " 29 GB" | 20260506 09:33:26.496 | INFO | } | 20260506 09:33:26.496 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:33:26.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:33:26.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:26.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:26.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:26.496 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 09:33:26.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:26.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:26.496 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:33:26.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:26.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:26.496 | INFO | Uploaded 36GiB to GPU in 1.60413s (22.4421GiB/s) | 20260506 09:33:26.496 | INFO | Downloaded 34GiB to GPU in 1.52088s (22.3555GiB/s) | 20260506 09:33:31.635 | INFO | Analyzing chunk frames = [310272,310784) | 20260506 09:33:31.635 | INFO | MemInfo: { | 20260506 09:33:31.635 | INFO | "gpu" : " 17 GB", | 20260506 09:33:31.635 | INFO | "huge" : "", | 20260506 09:33:31.635 | INFO | "huge_shared" : " 27 GB", | 20260506 09:33:31.635 | INFO | "malloc" : " 14 GB", | 20260506 09:33:31.635 | INFO | "pinned" : " 29 GB" | 20260506 09:33:31.635 | INFO | } | 20260506 09:33:31.635 | INFO | Perf report for the last 5.13936 seconds: | 20260506 09:33:31.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:33:31.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:31.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:31.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:31.635 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:33:31.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:31.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:31.635 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:33:31.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:31.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:31.635 | INFO | Uploaded 36GiB to GPU in 1.60404s (22.4433GiB/s) | 20260506 09:33:31.635 | INFO | Downloaded 34GiB to GPU in 1.52279s (22.3274GiB/s) | 20260506 09:33:36.774 | INFO | Analyzing chunk frames = [310784,311296) | 20260506 09:33:36.774 | INFO | MemInfo: { | 20260506 09:33:36.774 | INFO | "gpu" : " 17 GB", | 20260506 09:33:36.774 | INFO | "huge" : "", | 20260506 09:33:36.774 | INFO | "huge_shared" : " 27 GB", | 20260506 09:33:36.774 | INFO | "malloc" : " 14 GB", | 20260506 09:33:36.774 | INFO | "pinned" : " 29 GB" | 20260506 09:33:36.774 | INFO | } | 20260506 09:33:36.774 | INFO | Perf report for the last 5.13936 seconds: | 20260506 09:33:36.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:33:36.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:36.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:36.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:36.775 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:33:36.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:36.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:36.775 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 09:33:36.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:36.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:36.775 | INFO | Uploaded 36GiB to GPU in 1.60283s (22.4603GiB/s) | 20260506 09:33:36.775 | INFO | Downloaded 34GiB to GPU in 1.52168s (22.3437GiB/s) | 20260506 09:33:36.775 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:33:36.775 | INFO | Total mem usage: 15.7 GiB | 20260506 09:33:36.775 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 2.56%) | 20260506 09:33:36.775 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -6.04%) | 20260506 09:33:36.775 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:33:36.775 | INFO | High Water Breakdown: | 20260506 09:33:36.775 | INFO | 58.9% : 8.75 GiB ( 92.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:33:36.775 | INFO | 40.9% : 6.08 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:33:36.775 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:33:36.775 | INFO | ------------------------------------------ | 20260506 09:33:36.775 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:33:36.775 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:33:36.775 | INFO | Total mem usage: 26.5 GiB | 20260506 09:33:36.775 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:33:36.775 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:33:36.775 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:33:36.775 | INFO | High Water Breakdown: | 20260506 09:33:36.775 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:33:36.775 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:33:36.775 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:33:36.775 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:33:36.775 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:33:36.775 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:33:36.775 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:33:36.775 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:33:36.775 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:33:36.775 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:33:36.775 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:33:36.775 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:33:36.775 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:33:36.775 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:33:36.775 | INFO | ------------------------------------------ | 20260506 09:33:36.775 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:33:36.775 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:33:36.775 | INFO | Total mem usage: 25.6 GiB | 20260506 09:33:36.775 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:33:36.775 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:33:36.775 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:33:36.775 | INFO | High Water Breakdown: | 20260506 09:33:36.775 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:33:36.775 | INFO | ------------------------------------------ | 20260506 09:33:36.775 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:33:36.775 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:33:36.776 | INFO | Total mem usage: 15.7 GiB | 20260506 09:33:36.776 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:33:36.776 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:33:36.776 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:33:36.776 | INFO | High Water Breakdown: | 20260506 09:33:36.776 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:33:36.776 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:33:36.776 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:33:36.776 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:33:36.776 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:33:36.776 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:33:36.776 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:33:36.776 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:33:36.776 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:33:36.776 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:33:36.776 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:33:36.776 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:33:36.776 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:33:36.776 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:33:36.776 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:33:36.776 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:33:36.776 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:33:36.776 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:33:36.776 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:33:36.776 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:33:36.776 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:33:36.776 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:33:36.776 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:33:36.776 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:33:36.776 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:33:36.776 | INFO | ------------------------------------------ | 20260506 09:33:36.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:33:36.776 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:33:36.776 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:33:36.776 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:33:36.776 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 09:33:36.776 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / -nan, 4.56e-03 / 0.00e+00 | 20260506 09:33:36.776 | INFO | PerfReport: 10.90%, Baseline -- 3.29e+00 / -nan, 1.74e-02 / 0.00e+00 | 20260506 09:33:36.776 | INFO | PerfReport: 3.05%, Histogram -- 9.21e-01 / -nan, 5.13e-02 / 0.00e+00 | 20260506 09:33:36.776 | INFO | PerfReport: 16.04%, DME -- 3.88e+01 / -nan, 3.13e+00 / 0.00e+00 | 20260506 09:33:36.776 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.72e+01 / -nan, 4.18e-02 / 0.00e+00 | 20260506 09:33:36.776 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.27e+00 / -nan, 1.35e-01 / 0.00e+00 | 20260506 09:33:36.776 | INFO | PerfReport: 3.82%, Metrics -- 1.16e+00 / -nan, 1.81e-01 / 0.00e+00 | 20260506 09:33:36.776 | INFO | PerfReport: 1.49%, PulseSerializing -- 4.50e-01 / -nan, 6.54e-02 / 0.00e+00 | 20260506 09:33:36.776 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:33:36.776 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:33:36.776 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:33:36.776 | INFO | PerfReport: Full Execution -- 9.05e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 09:33:36.776 | INFO | PerfReport: 18.52%, Upload -- 1.68e+01 / 0.00e+00, 4.16e-01 / 0.00e+00 | 20260506 09:33:36.776 | INFO | PerfReport: 17.78%, Download -- 1.61e+01 / 0.00e+00, 7.42e-01 / 0.00e+00 | 20260506 09:33:36.776 | INFO | PerfReport: 33.38%, Compute -- 3.02e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 09:33:36.776 | INFO | PerfReport: 0.56%, LockUpload -- 5.09e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:33:36.776 | INFO | PerfReport: 0.00%, LockDownload -- 4.92e-04 / 0.00e+00, 1.29e-04 / 0.00e+00 | 20260506 09:33:36.776 | INFO | PerfReport: 29.73%, LockCompute -- 2.69e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260506 09:33:36.776 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260506 09:33:36.873 | INFO | Starting a round of Baz IO | 20260506 09:33:39.797 | INFO | Populating RTMetrics report | 20260506 09:33:39.797 | INFO | RTMetrics report populated | 20260506 09:33:39.797 | INFO | Opening tmp RTMetrics json file | 20260506 09:33:39.942 | INFO | Serializing RTMetrics to json | 20260506 09:33:39.969 | INFO | Serialization done, swapping tmp file now... | 20260506 09:33:39.970 | INFO | RTMetrics done | 20260506 09:33:40.097 | INFO | Finished a round of Baz IO: Wrote 1.858 GiB in 3.224 seconds | 20260506 09:33:40.097 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.37% / 0.10% / 46.53% | 20260506 09:33:40.097 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.1212% / 0.000317891% / 5.5631e-05% | 20260506 09:33:40.098 | INFO | Baz IO overall: | 20260506 09:33:40.098 | INFO | BytesWritten : 121807249515 | 20260506 09:33:40.098 | INFO | NumEvents : 57737222617 | 20260506 09:33:41.914 | INFO | Analyzing chunk frames = [311296,311808) | 20260506 09:33:41.914 | INFO | MemInfo: { | 20260506 09:33:41.914 | INFO | "gpu" : " 17 GB", | 20260506 09:33:41.914 | INFO | "huge" : "", | 20260506 09:33:41.914 | INFO | "huge_shared" : " 27 GB", | 20260506 09:33:41.914 | INFO | "malloc" : " 14 GB", | 20260506 09:33:41.914 | INFO | "pinned" : " 29 GB" | 20260506 09:33:41.914 | INFO | } | 20260506 09:33:41.914 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:33:41.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:33:41.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:41.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:41.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:41.914 | INFO | ANALYSIS : 57.8%, 2.8, 85.7ms, 0 | 20260506 09:33:41.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:41.914 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260506 09:33:41.914 | INFO | PRE_HQ : 57.0%, 2.1, 61.5ms, 0 | 20260506 09:33:41.914 | INFO | BAZWRITER : 62.3%, 4.7, 151.2ms, 0 | 20260506 09:33:41.914 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.5ms, 0 | 20260506 09:33:41.914 | INFO | Uploaded 36GiB to GPU in 1.67214s (21.5293GiB/s) | 20260506 09:33:41.914 | INFO | Downloaded 38GiB to GPU in 1.73556s (21.895GiB/s) | 20260506 09:33:47.053 | INFO | Analyzing chunk frames = [311808,312320) | 20260506 09:33:47.053 | INFO | MemInfo: { | 20260506 09:33:47.053 | INFO | "gpu" : " 17 GB", | 20260506 09:33:47.053 | INFO | "huge" : "", | 20260506 09:33:47.053 | INFO | "huge_shared" : " 27 GB", | 20260506 09:33:47.053 | INFO | "malloc" : " 14 GB", | 20260506 09:33:47.053 | INFO | "pinned" : " 29 GB" | 20260506 09:33:47.053 | INFO | } | 20260506 09:33:47.053 | INFO | Perf report for the last 5.13937 seconds: | 20260506 09:33:47.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:33:47.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:47.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:47.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:47.053 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260506 09:33:47.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:47.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:47.053 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:33:47.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:47.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:47.054 | INFO | Uploaded 36GiB to GPU in 1.6117s (22.3367GiB/s) | 20260506 09:33:47.054 | INFO | Downloaded 34GiB to GPU in 1.51948s (22.3761GiB/s) | 20260506 09:33:52.192 | INFO | Analyzing chunk frames = [312320,312832) | 20260506 09:33:52.192 | INFO | MemInfo: { | 20260506 09:33:52.192 | INFO | "gpu" : " 17 GB", | 20260506 09:33:52.192 | INFO | "huge" : "", | 20260506 09:33:52.192 | INFO | "huge_shared" : " 27 GB", | 20260506 09:33:52.192 | INFO | "malloc" : " 14 GB", | 20260506 09:33:52.192 | INFO | "pinned" : " 29 GB" | 20260506 09:33:52.192 | INFO | } | 20260506 09:33:52.192 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:33:52.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:33:52.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:52.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:52.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:52.192 | INFO | ANALYSIS : 56.9%, 3.0, 88.5ms, 0 | 20260506 09:33:52.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:52.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:52.193 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:33:52.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:52.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:52.193 | INFO | Uploaded 36GiB to GPU in 1.60625s (22.4124GiB/s) | 20260506 09:33:52.193 | INFO | Downloaded 34GiB to GPU in 1.52372s (22.3138GiB/s) | 20260506 09:33:57.332 | INFO | Analyzing chunk frames = [312832,313344) | 20260506 09:33:57.332 | INFO | MemInfo: { | 20260506 09:33:57.332 | INFO | "gpu" : " 17 GB", | 20260506 09:33:57.332 | INFO | "huge" : "", | 20260506 09:33:57.332 | INFO | "huge_shared" : " 27 GB", | 20260506 09:33:57.332 | INFO | "malloc" : " 14 GB", | 20260506 09:33:57.332 | INFO | "pinned" : " 29 GB" | 20260506 09:33:57.332 | INFO | } | 20260506 09:33:57.332 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:33:57.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:33:57.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:57.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:57.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:57.332 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:33:57.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:57.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:57.332 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260506 09:33:57.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:57.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:33:57.332 | INFO | Uploaded 36GiB to GPU in 1.60583s (22.4184GiB/s) | 20260506 09:33:57.332 | INFO | Downloaded 34GiB to GPU in 1.52255s (22.331GiB/s) | 20260506 09:34:02.471 | INFO | Analyzing chunk frames = [313344,313856) | 20260506 09:34:02.471 | INFO | MemInfo: { | 20260506 09:34:02.471 | INFO | "gpu" : " 17 GB", | 20260506 09:34:02.471 | INFO | "huge" : "", | 20260506 09:34:02.471 | INFO | "huge_shared" : " 27 GB", | 20260506 09:34:02.471 | INFO | "malloc" : " 14 GB", | 20260506 09:34:02.471 | INFO | "pinned" : " 29 GB" | 20260506 09:34:02.471 | INFO | } | 20260506 09:34:02.471 | INFO | Perf report for the last 5.13952 seconds: | 20260506 09:34:02.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:34:02.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:02.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:02.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:02.471 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:34:02.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:02.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:02.471 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:34:02.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:02.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:02.472 | INFO | Uploaded 36GiB to GPU in 1.60404s (22.4434GiB/s) | 20260506 09:34:02.472 | INFO | Downloaded 34GiB to GPU in 1.5219s (22.3406GiB/s) | 20260506 09:34:07.610 | INFO | Analyzing chunk frames = [313856,314368) | 20260506 09:34:07.610 | INFO | MemInfo: { | 20260506 09:34:07.610 | INFO | "gpu" : " 17 GB", | 20260506 09:34:07.610 | INFO | "huge" : "", | 20260506 09:34:07.610 | INFO | "huge_shared" : " 27 GB", | 20260506 09:34:07.610 | INFO | "malloc" : " 14 GB", | 20260506 09:34:07.610 | INFO | "pinned" : " 29 GB" | 20260506 09:34:07.610 | INFO | } | 20260506 09:34:07.610 | INFO | Perf report for the last 5.13915 seconds: | 20260506 09:34:07.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:34:07.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:07.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:07.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:07.610 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 09:34:07.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:07.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:07.610 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:34:07.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:07.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:07.611 | INFO | Uploaded 36GiB to GPU in 1.60422s (22.4408GiB/s) | 20260506 09:34:07.611 | INFO | Downloaded 34GiB to GPU in 1.52092s (22.355GiB/s) | 20260506 09:34:12.750 | INFO | Analyzing chunk frames = [314368,314880) | 20260506 09:34:12.750 | INFO | MemInfo: { | 20260506 09:34:12.750 | INFO | "gpu" : " 17 GB", | 20260506 09:34:12.750 | INFO | "huge" : "", | 20260506 09:34:12.750 | INFO | "huge_shared" : " 27 GB", | 20260506 09:34:12.750 | INFO | "malloc" : " 14 GB", | 20260506 09:34:12.750 | INFO | "pinned" : " 29 GB" | 20260506 09:34:12.750 | INFO | } | 20260506 09:34:12.750 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:34:12.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:34:12.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:12.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:12.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:12.750 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:34:12.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:12.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:12.750 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:34:12.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:12.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:12.750 | INFO | Uploaded 36GiB to GPU in 1.60366s (22.4487GiB/s) | 20260506 09:34:12.750 | INFO | Downloaded 34GiB to GPU in 1.52153s (22.3459GiB/s) | 20260506 09:34:17.889 | INFO | Analyzing chunk frames = [314880,315392) | 20260506 09:34:17.889 | INFO | MemInfo: { | 20260506 09:34:17.889 | INFO | "gpu" : " 17 GB", | 20260506 09:34:17.889 | INFO | "huge" : "", | 20260506 09:34:17.889 | INFO | "huge_shared" : " 27 GB", | 20260506 09:34:17.889 | INFO | "malloc" : " 14 GB", | 20260506 09:34:17.889 | INFO | "pinned" : " 29 GB" | 20260506 09:34:17.889 | INFO | } | 20260506 09:34:17.889 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:34:17.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:34:17.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:17.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:17.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:17.889 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:34:17.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:17.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:17.889 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 09:34:17.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:17.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:17.889 | INFO | Uploaded 36GiB to GPU in 1.60439s (22.4384GiB/s) | 20260506 09:34:17.889 | INFO | Downloaded 34GiB to GPU in 1.52183s (22.3415GiB/s) | 20260506 09:34:17.986 | INFO | Starting a round of Baz IO | 20260506 09:34:20.909 | INFO | Populating RTMetrics report | 20260506 09:34:20.909 | INFO | RTMetrics report populated | 20260506 09:34:20.909 | INFO | Opening tmp RTMetrics json file | 20260506 09:34:20.984 | INFO | Serializing RTMetrics to json | 20260506 09:34:20.989 | INFO | Serialization done, swapping tmp file now... | 20260506 09:34:20.990 | INFO | RTMetrics done | 20260506 09:34:21.286 | INFO | Finished a round of Baz IO: Wrote 1.863 GiB in 3.299 seconds | 20260506 09:34:21.286 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.27% / 0.10% / 46.63% | 20260506 09:34:21.286 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.09994% / 0.000329812% / 4.76837e-05% | 20260506 09:34:21.286 | INFO | Baz IO overall: | 20260506 09:34:21.286 | INFO | BytesWritten : 123807944178 | 20260506 09:34:21.286 | INFO | NumEvents : 58708187365 | 20260506 09:34:23.028 | INFO | Analyzing chunk frames = [315392,315904) | 20260506 09:34:23.028 | INFO | MemInfo: { | 20260506 09:34:23.028 | INFO | "gpu" : " 17 GB", | 20260506 09:34:23.028 | INFO | "huge" : "", | 20260506 09:34:23.028 | INFO | "huge_shared" : " 27 GB", | 20260506 09:34:23.028 | INFO | "malloc" : " 14 GB", | 20260506 09:34:23.028 | INFO | "pinned" : " 29 GB" | 20260506 09:34:23.028 | INFO | } | 20260506 09:34:23.028 | INFO | Perf report for the last 5.13937 seconds: | 20260506 09:34:23.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:34:23.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:23.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:23.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:23.029 | INFO | ANALYSIS : 57.8%, 2.9, 88.3ms, 0 | 20260506 09:34:23.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:23.029 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260506 09:34:23.029 | INFO | PRE_HQ : 56.9%, 2.1, 61.6ms, 0 | 20260506 09:34:23.029 | INFO | BAZWRITER : 63.8%, 4.5, 146.4ms, 0 | 20260506 09:34:23.029 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 09:34:23.029 | INFO | Uploaded 36GiB to GPU in 1.64503s (21.8841GiB/s) | 20260506 09:34:23.029 | INFO | Downloaded 38GiB to GPU in 1.70897s (22.2356GiB/s) | 20260506 09:34:28.168 | INFO | Analyzing chunk frames = [315904,316416) | 20260506 09:34:28.168 | INFO | MemInfo: { | 20260506 09:34:28.168 | INFO | "gpu" : " 17 GB", | 20260506 09:34:28.168 | INFO | "huge" : "", | 20260506 09:34:28.168 | INFO | "huge_shared" : " 27 GB", | 20260506 09:34:28.168 | INFO | "malloc" : " 14 GB", | 20260506 09:34:28.168 | INFO | "pinned" : " 29 GB" | 20260506 09:34:28.168 | INFO | } | 20260506 09:34:28.168 | INFO | Perf report for the last 5.13914 seconds: | 20260506 09:34:28.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:34:28.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:28.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:28.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:28.168 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260506 09:34:28.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:28.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:28.168 | INFO | PRE_HQ : 29.8%, 1.0, 15.6ms, 0 | 20260506 09:34:28.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:28.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:28.168 | INFO | Uploaded 36GiB to GPU in 1.60158s (22.4778GiB/s) | 20260506 09:34:28.168 | INFO | Downloaded 34GiB to GPU in 1.51962s (22.374GiB/s) | 20260506 09:34:33.307 | INFO | Analyzing chunk frames = [316416,316928) | 20260506 09:34:33.307 | INFO | MemInfo: { | 20260506 09:34:33.307 | INFO | "gpu" : " 17 GB", | 20260506 09:34:33.307 | INFO | "huge" : "", | 20260506 09:34:33.307 | INFO | "huge_shared" : " 27 GB", | 20260506 09:34:33.307 | INFO | "malloc" : " 14 GB", | 20260506 09:34:33.307 | INFO | "pinned" : " 29 GB" | 20260506 09:34:33.307 | INFO | } | 20260506 09:34:33.307 | INFO | Perf report for the last 5.13947 seconds: | 20260506 09:34:33.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:34:33.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:33.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:33.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:33.307 | INFO | ANALYSIS : 57.0%, 3.0, 88.5ms, 0 | 20260506 09:34:33.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:33.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:33.307 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:34:33.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:33.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:33.307 | INFO | Uploaded 36GiB to GPU in 1.60621s (22.413GiB/s) | 20260506 09:34:33.308 | INFO | Downloaded 34GiB to GPU in 1.52328s (22.3203GiB/s) | 20260506 09:34:38.447 | INFO | Analyzing chunk frames = [316928,317440) | 20260506 09:34:38.447 | INFO | MemInfo: { | 20260506 09:34:38.447 | INFO | "gpu" : " 17 GB", | 20260506 09:34:38.447 | INFO | "huge" : "", | 20260506 09:34:38.447 | INFO | "huge_shared" : " 27 GB", | 20260506 09:34:38.447 | INFO | "malloc" : " 14 GB", | 20260506 09:34:38.447 | INFO | "pinned" : " 29 GB" | 20260506 09:34:38.447 | INFO | } | 20260506 09:34:38.447 | INFO | Perf report for the last 5.13961 seconds: | 20260506 09:34:38.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:34:38.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:38.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:38.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:38.447 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:34:38.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:38.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:38.447 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260506 09:34:38.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:38.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:38.447 | INFO | Uploaded 36GiB to GPU in 1.6054s (22.4243GiB/s) | 20260506 09:34:38.447 | INFO | Downloaded 34GiB to GPU in 1.5226s (22.3302GiB/s) | 20260506 09:34:43.586 | INFO | Analyzing chunk frames = [317440,317952) | 20260506 09:34:43.586 | INFO | MemInfo: { | 20260506 09:34:43.586 | INFO | "gpu" : " 17 GB", | 20260506 09:34:43.586 | INFO | "huge" : "", | 20260506 09:34:43.586 | INFO | "huge_shared" : " 27 GB", | 20260506 09:34:43.586 | INFO | "malloc" : " 14 GB", | 20260506 09:34:43.586 | INFO | "pinned" : " 29 GB" | 20260506 09:34:43.586 | INFO | } | 20260506 09:34:43.586 | INFO | Perf report for the last 5.13905 seconds: | 20260506 09:34:43.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:34:43.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:43.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:43.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:43.586 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:34:43.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:43.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:43.586 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:34:43.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:43.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:43.586 | INFO | Uploaded 36GiB to GPU in 1.60135s (22.481GiB/s) | 20260506 09:34:43.586 | INFO | Downloaded 34GiB to GPU in 1.51989s (22.37GiB/s) | 20260506 09:34:48.725 | INFO | Analyzing chunk frames = [317952,318464) | 20260506 09:34:48.725 | INFO | MemInfo: { | 20260506 09:34:48.725 | INFO | "gpu" : " 17 GB", | 20260506 09:34:48.725 | INFO | "huge" : "", | 20260506 09:34:48.725 | INFO | "huge_shared" : " 27 GB", | 20260506 09:34:48.725 | INFO | "malloc" : " 14 GB", | 20260506 09:34:48.725 | INFO | "pinned" : " 29 GB" | 20260506 09:34:48.725 | INFO | } | 20260506 09:34:48.725 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:34:48.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:34:48.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:48.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:48.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:48.725 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:34:48.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:48.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:48.725 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:34:48.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:48.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:48.726 | INFO | Uploaded 36GiB to GPU in 1.6047s (22.434GiB/s) | 20260506 09:34:48.726 | INFO | Downloaded 34GiB to GPU in 1.52191s (22.3404GiB/s) | 20260506 09:34:53.864 | INFO | Analyzing chunk frames = [318464,318976) | 20260506 09:34:53.864 | INFO | MemInfo: { | 20260506 09:34:53.864 | INFO | "gpu" : " 17 GB", | 20260506 09:34:53.864 | INFO | "huge" : "", | 20260506 09:34:53.864 | INFO | "huge_shared" : " 27 GB", | 20260506 09:34:53.865 | INFO | "malloc" : " 14 GB", | 20260506 09:34:53.865 | INFO | "pinned" : " 29 GB" | 20260506 09:34:53.865 | INFO | } | 20260506 09:34:53.865 | INFO | Perf report for the last 5.13942 seconds: | 20260506 09:34:53.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:34:53.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:53.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:53.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:53.865 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:34:53.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:53.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:53.865 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:34:53.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:53.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:53.865 | INFO | Uploaded 36GiB to GPU in 1.60245s (22.4656GiB/s) | 20260506 09:34:53.865 | INFO | Downloaded 34GiB to GPU in 1.52187s (22.3409GiB/s) | 20260506 09:34:59.004 | INFO | Analyzing chunk frames = [318976,319488) | 20260506 09:34:59.004 | INFO | MemInfo: { | 20260506 09:34:59.004 | INFO | "gpu" : " 17 GB", | 20260506 09:34:59.004 | INFO | "huge" : "", | 20260506 09:34:59.004 | INFO | "huge_shared" : " 27 GB", | 20260506 09:34:59.004 | INFO | "malloc" : " 14 GB", | 20260506 09:34:59.004 | INFO | "pinned" : " 29 GB" | 20260506 09:34:59.004 | INFO | } | 20260506 09:34:59.004 | INFO | Perf report for the last 5.1392 seconds: | 20260506 09:34:59.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:34:59.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:59.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:59.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:59.004 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:34:59.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:59.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:59.004 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 09:34:59.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:59.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:34:59.004 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:34:59.004 | INFO | Total mem usage: 15.7 GiB | 20260506 09:34:59.004 | INFO | Uploaded 36GiB to GPU in 1.60409s (22.4426GiB/s) | 20260506 09:34:59.004 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 2.56%) | 20260506 09:34:59.004 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -3.09%) | 20260506 09:34:59.004 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:34:59.004 | INFO | High Water Breakdown: | 20260506 09:34:59.004 | INFO | Downloaded 34GiB to GPU in 1.52322s (22.3211GiB/s) | 20260506 09:34:59.004 | INFO | 59.5% : 9.09 GiB ( 96.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:34:59.004 | INFO | 40.3% : 6.16 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:34:59.004 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:34:59.004 | INFO | ------------------------------------------ | 20260506 09:34:59.004 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:34:59.004 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:34:59.004 | INFO | Total mem usage: 26.5 GiB | 20260506 09:34:59.004 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:34:59.004 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:34:59.004 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:34:59.004 | INFO | High Water Breakdown: | 20260506 09:34:59.004 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:34:59.004 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:34:59.004 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:34:59.004 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:34:59.004 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:34:59.004 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:34:59.004 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:34:59.004 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:34:59.004 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:34:59.004 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:34:59.004 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:34:59.004 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:34:59.004 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:34:59.004 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:34:59.004 | INFO | ------------------------------------------ | 20260506 09:34:59.004 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:34:59.004 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:34:59.005 | INFO | Total mem usage: 25.6 GiB | 20260506 09:34:59.005 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:34:59.005 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:34:59.005 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:34:59.005 | INFO | High Water Breakdown: | 20260506 09:34:59.005 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:34:59.005 | INFO | ------------------------------------------ | 20260506 09:34:59.005 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:34:59.005 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:34:59.005 | INFO | Total mem usage: 15.7 GiB | 20260506 09:34:59.005 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:34:59.005 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:34:59.005 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:34:59.005 | INFO | High Water Breakdown: | 20260506 09:34:59.005 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:34:59.005 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:34:59.005 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:34:59.005 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:34:59.005 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:34:59.005 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:34:59.005 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:34:59.005 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:34:59.005 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:34:59.005 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:34:59.005 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:34:59.005 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:34:59.005 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:34:59.005 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:34:59.005 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:34:59.005 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:34:59.005 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:34:59.005 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:34:59.005 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:34:59.005 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:34:59.005 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:34:59.005 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:34:59.005 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:34:59.005 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:34:59.005 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:34:59.005 | INFO | ------------------------------------------ | 20260506 09:34:59.005 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:34:59.005 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:34:59.005 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:34:59.005 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:34:59.005 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 09:34:59.005 | INFO | PerfReport: 0.29%, AggregateStats -- 8.61e-02 / -nan, 4.24e-03 / 0.00e+00 | 20260506 09:34:59.005 | INFO | PerfReport: 10.89%, Baseline -- 3.29e+00 / -nan, 1.93e-02 / 0.00e+00 | 20260506 09:34:59.005 | INFO | PerfReport: 3.05%, Histogram -- 9.21e-01 / -nan, 5.15e-02 / 0.00e+00 | 20260506 09:34:59.005 | INFO | PerfReport: 16.04%, DME -- 3.88e+01 / -nan, 3.08e+00 / 0.00e+00 | 20260506 09:34:59.005 | INFO | PerfReport: 56.91%, FrameLabeling -- 1.72e+01 / -nan, 4.14e-02 / 0.00e+00 | 20260506 09:34:59.005 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.27e+00 / -nan, 1.35e-01 / 0.00e+00 | 20260506 09:34:59.005 | INFO | PerfReport: 3.82%, Metrics -- 1.15e+00 / -nan, 1.80e-01 / 0.00e+00 | 20260506 09:34:59.005 | INFO | PerfReport: 1.49%, PulseSerializing -- 4.49e-01 / -nan, 6.52e-02 / 0.00e+00 | 20260506 09:34:59.005 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:34:59.005 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:34:59.005 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:34:59.005 | INFO | PerfReport: Full Execution -- 9.04e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 09:34:59.005 | INFO | PerfReport: 18.57%, Upload -- 1.68e+01 / 0.00e+00, 4.60e-01 / 0.00e+00 | 20260506 09:34:59.005 | INFO | PerfReport: 17.83%, Download -- 1.61e+01 / 0.00e+00, 7.86e-01 / 0.00e+00 | 20260506 09:34:59.005 | INFO | PerfReport: 33.43%, Compute -- 3.02e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 09:34:59.005 | INFO | PerfReport: 0.56%, LockUpload -- 5.08e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260506 09:34:59.005 | INFO | PerfReport: 0.00%, LockDownload -- 4.86e-04 / 0.00e+00, 1.62e-04 / 0.00e+00 | 20260506 09:34:59.006 | INFO | PerfReport: 29.58%, LockCompute -- 2.67e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 09:34:59.006 | INFO | PerfReport: 0.01%, Misc -- 1.22e-02 / 0.00e+00, 1.24e-02 / 0.00e+00 | 20260506 09:34:59.101 | INFO | Starting a round of Baz IO | 20260506 09:35:02.025 | INFO | Populating RTMetrics report | 20260506 09:35:02.026 | INFO | RTMetrics report populated | 20260506 09:35:02.026 | INFO | Opening tmp RTMetrics json file | 20260506 09:35:02.064 | INFO | Serializing RTMetrics to json | 20260506 09:35:02.067 | INFO | Serialization done, swapping tmp file now... | 20260506 09:35:02.072 | INFO | RTMetrics done | 20260506 09:35:02.221 | INFO | Finished a round of Baz IO: Wrote 1.864 GiB in 3.12 seconds | 20260506 09:35:02.221 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.18% / 0.09% / 46.73% | 20260506 09:35:02.221 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.05837% / 0.000258287% / 4.37101e-05% | 20260506 09:35:02.221 | INFO | Baz IO overall: | 20260506 09:35:02.221 | INFO | BytesWritten : 125809872774 | 20260506 09:35:02.221 | INFO | NumEvents : 59679512175 | 20260506 09:35:04.143 | INFO | Analyzing chunk frames = [319488,320000) | 20260506 09:35:04.143 | INFO | MemInfo: { | 20260506 09:35:04.143 | INFO | "gpu" : " 17 GB", | 20260506 09:35:04.143 | INFO | "huge" : "", | 20260506 09:35:04.143 | INFO | "huge_shared" : " 27 GB", | 20260506 09:35:04.143 | INFO | "malloc" : " 14 GB", | 20260506 09:35:04.143 | INFO | "pinned" : " 29 GB" | 20260506 09:35:04.143 | INFO | } | 20260506 09:35:04.143 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:35:04.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:35:04.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:04.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:04.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:04.143 | INFO | ANALYSIS : 57.8%, 2.9, 88.0ms, 0 | 20260506 09:35:04.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:04.143 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 09:35:04.143 | INFO | PRE_HQ : 56.9%, 2.1, 62.0ms, 0 | 20260506 09:35:04.143 | INFO | BAZWRITER : 60.3%, 4.5, 139.2ms, 0 | 20260506 09:35:04.143 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 09:35:04.143 | INFO | Uploaded 36GiB to GPU in 1.6465s (21.8646GiB/s) | 20260506 09:35:04.144 | INFO | Downloaded 38GiB to GPU in 1.70881s (22.2377GiB/s) | 20260506 09:35:09.282 | INFO | Analyzing chunk frames = [320000,320512) | 20260506 09:35:09.282 | INFO | MemInfo: { | 20260506 09:35:09.282 | INFO | "gpu" : " 17 GB", | 20260506 09:35:09.282 | INFO | "huge" : "", | 20260506 09:35:09.282 | INFO | "huge_shared" : " 27 GB", | 20260506 09:35:09.282 | INFO | "malloc" : " 14 GB", | 20260506 09:35:09.282 | INFO | "pinned" : " 29 GB" | 20260506 09:35:09.282 | INFO | } | 20260506 09:35:09.283 | INFO | Perf report for the last 5.13938 seconds: | 20260506 09:35:09.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:35:09.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:09.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:09.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:09.283 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 09:35:09.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:09.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:09.283 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:35:09.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:09.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:09.283 | INFO | Uploaded 36GiB to GPU in 1.60226s (22.4683GiB/s) | 20260506 09:35:09.283 | INFO | Downloaded 34GiB to GPU in 1.51936s (22.3778GiB/s) | 20260506 09:35:14.422 | INFO | Analyzing chunk frames = [320512,321024) | 20260506 09:35:14.422 | INFO | MemInfo: { | 20260506 09:35:14.422 | INFO | "gpu" : " 17 GB", | 20260506 09:35:14.422 | INFO | "huge" : "", | 20260506 09:35:14.422 | INFO | "huge_shared" : " 27 GB", | 20260506 09:35:14.422 | INFO | "malloc" : " 14 GB", | 20260506 09:35:14.422 | INFO | "pinned" : " 29 GB" | 20260506 09:35:14.422 | INFO | } | 20260506 09:35:14.422 | INFO | Perf report for the last 5.13956 seconds: | 20260506 09:35:14.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:35:14.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:14.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:14.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:14.422 | INFO | ANALYSIS : 57.0%, 3.0, 88.5ms, 0 | 20260506 09:35:14.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:14.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:14.422 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260506 09:35:14.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:14.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:14.423 | INFO | Uploaded 36GiB to GPU in 1.60579s (22.4189GiB/s) | 20260506 09:35:14.423 | INFO | Downloaded 34GiB to GPU in 1.5227s (22.3288GiB/s) | 20260506 09:35:19.561 | INFO | Analyzing chunk frames = [321024,321536) | 20260506 09:35:19.561 | INFO | MemInfo: { | 20260506 09:35:19.561 | INFO | "gpu" : " 17 GB", | 20260506 09:35:19.561 | INFO | "huge" : "", | 20260506 09:35:19.561 | INFO | "huge_shared" : " 27 GB", | 20260506 09:35:19.561 | INFO | "malloc" : " 14 GB", | 20260506 09:35:19.561 | INFO | "pinned" : " 29 GB" | 20260506 09:35:19.561 | INFO | } | 20260506 09:35:19.561 | INFO | Perf report for the last 5.13922 seconds: | 20260506 09:35:19.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:35:19.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:19.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:19.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:19.561 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:35:19.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:19.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:19.561 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:35:19.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:19.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:19.562 | INFO | Uploaded 36GiB to GPU in 1.60607s (22.415GiB/s) | 20260506 09:35:19.562 | INFO | Downloaded 34GiB to GPU in 1.52234s (22.334GiB/s) | 20260506 09:35:24.701 | INFO | Analyzing chunk frames = [321536,322048) | 20260506 09:35:24.701 | INFO | MemInfo: { | 20260506 09:35:24.701 | INFO | "gpu" : " 17 GB", | 20260506 09:35:24.701 | INFO | "huge" : "", | 20260506 09:35:24.701 | INFO | "huge_shared" : " 27 GB", | 20260506 09:35:24.701 | INFO | "malloc" : " 14 GB", | 20260506 09:35:24.701 | INFO | "pinned" : " 29 GB" | 20260506 09:35:24.701 | INFO | } | 20260506 09:35:24.701 | INFO | Perf report for the last 5.13961 seconds: | 20260506 09:35:24.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:35:24.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:24.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:24.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:24.701 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 09:35:24.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:24.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:24.701 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:35:24.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:24.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:24.702 | INFO | Uploaded 36GiB to GPU in 1.60377s (22.4471GiB/s) | 20260506 09:35:24.702 | INFO | Downloaded 34GiB to GPU in 1.52059s (22.3598GiB/s) | 20260506 09:35:29.840 | INFO | Analyzing chunk frames = [322048,322560) | 20260506 09:35:29.840 | INFO | MemInfo: { | 20260506 09:35:29.840 | INFO | "gpu" : " 17 GB", | 20260506 09:35:29.840 | INFO | "huge" : "", | 20260506 09:35:29.840 | INFO | "huge_shared" : " 27 GB", | 20260506 09:35:29.840 | INFO | "malloc" : " 14 GB", | 20260506 09:35:29.840 | INFO | "pinned" : " 29 GB" | 20260506 09:35:29.840 | INFO | } | 20260506 09:35:29.840 | INFO | Perf report for the last 5.13902 seconds: | 20260506 09:35:29.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:35:29.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:29.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:29.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:29.840 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 09:35:29.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:29.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:29.840 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:35:29.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:29.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:29.841 | INFO | Uploaded 36GiB to GPU in 1.60438s (22.4385GiB/s) | 20260506 09:35:29.841 | INFO | Downloaded 34GiB to GPU in 1.52076s (22.3572GiB/s) | 20260506 09:35:34.979 | INFO | Analyzing chunk frames = [322560,323072) | 20260506 09:35:34.979 | INFO | MemInfo: { | 20260506 09:35:34.979 | INFO | "gpu" : " 17 GB", | 20260506 09:35:34.979 | INFO | "huge" : "", | 20260506 09:35:34.979 | INFO | "huge_shared" : " 27 GB", | 20260506 09:35:34.979 | INFO | "malloc" : " 14 GB", | 20260506 09:35:34.979 | INFO | "pinned" : " 29 GB" | 20260506 09:35:34.979 | INFO | } | 20260506 09:35:34.979 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:35:34.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:35:34.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:34.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:34.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:34.979 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:35:34.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:34.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:34.979 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:35:34.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:34.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:34.980 | INFO | Uploaded 36GiB to GPU in 1.60414s (22.4419GiB/s) | 20260506 09:35:34.980 | INFO | Downloaded 34GiB to GPU in 1.52132s (22.3489GiB/s) | 20260506 09:35:40.119 | INFO | Analyzing chunk frames = [323072,323584) | 20260506 09:35:40.119 | INFO | MemInfo: { | 20260506 09:35:40.119 | INFO | "gpu" : " 17 GB", | 20260506 09:35:40.119 | INFO | "huge" : "", | 20260506 09:35:40.119 | INFO | "huge_shared" : " 27 GB", | 20260506 09:35:40.119 | INFO | "malloc" : " 14 GB", | 20260506 09:35:40.119 | INFO | "pinned" : " 29 GB" | 20260506 09:35:40.119 | INFO | } | 20260506 09:35:40.119 | INFO | Perf report for the last 5.13947 seconds: | 20260506 09:35:40.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:35:40.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:40.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:40.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:40.119 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:35:40.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:40.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:40.119 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 09:35:40.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:40.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:40.119 | INFO | Uploaded 36GiB to GPU in 1.60416s (22.4416GiB/s) | 20260506 09:35:40.119 | INFO | Downloaded 34GiB to GPU in 1.52199s (22.3392GiB/s) | 20260506 09:35:40.219 | INFO | Starting a round of Baz IO | 20260506 09:35:43.138 | INFO | Populating RTMetrics report | 20260506 09:35:43.138 | INFO | RTMetrics report populated | 20260506 09:35:43.138 | INFO | Opening tmp RTMetrics json file | 20260506 09:35:43.139 | INFO | Serializing RTMetrics to json | 20260506 09:35:43.144 | INFO | Serialization done, swapping tmp file now... | 20260506 09:35:43.148 | INFO | RTMetrics done | 20260506 09:35:43.231 | INFO | Finished a round of Baz IO: Wrote 1.866 GiB in 3.011 seconds | 20260506 09:35:43.231 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 53.09% / 0.09% / 46.82% | 20260506 09:35:43.231 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.02771% / 0.000186761% / 4.37101e-05% | 20260506 09:35:43.231 | INFO | Baz IO overall: | 20260506 09:35:43.231 | INFO | BytesWritten : 127813351327 | 20260506 09:35:43.231 | INFO | NumEvents : 60651718114 | 20260506 09:35:45.258 | INFO | Analyzing chunk frames = [323584,324096) | 20260506 09:35:45.258 | INFO | MemInfo: { | 20260506 09:35:45.258 | INFO | "gpu" : " 17 GB", | 20260506 09:35:45.258 | INFO | "huge" : "", | 20260506 09:35:45.258 | INFO | "huge_shared" : " 27 GB", | 20260506 09:35:45.258 | INFO | "malloc" : " 14 GB", | 20260506 09:35:45.258 | INFO | "pinned" : " 29 GB" | 20260506 09:35:45.258 | INFO | } | 20260506 09:35:45.258 | INFO | Perf report for the last 5.13926 seconds: | 20260506 09:35:45.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:35:45.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:45.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:45.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:45.258 | INFO | ANALYSIS : 57.8%, 2.8, 85.8ms, 0 | 20260506 09:35:45.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:45.258 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 09:35:45.258 | INFO | PRE_HQ : 57.0%, 2.1, 61.5ms, 0 | 20260506 09:35:45.258 | INFO | BAZWRITER : 58.3%, 4.7, 140.7ms, 0 | 20260506 09:35:45.258 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 09:35:45.259 | INFO | Uploaded 36GiB to GPU in 1.66712s (21.5942GiB/s) | 20260506 09:35:45.259 | INFO | Downloaded 38GiB to GPU in 1.73103s (21.9523GiB/s) | 20260506 09:35:50.397 | INFO | Analyzing chunk frames = [324096,324608) | 20260506 09:35:50.397 | INFO | MemInfo: { | 20260506 09:35:50.397 | INFO | "gpu" : " 17 GB", | 20260506 09:35:50.397 | INFO | "huge" : "", | 20260506 09:35:50.397 | INFO | "huge_shared" : " 27 GB", | 20260506 09:35:50.397 | INFO | "malloc" : " 14 GB", | 20260506 09:35:50.397 | INFO | "pinned" : " 29 GB" | 20260506 09:35:50.397 | INFO | } | 20260506 09:35:50.397 | INFO | Perf report for the last 5.13926 seconds: | 20260506 09:35:50.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:35:50.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:50.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:50.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:50.397 | INFO | ANALYSIS : 57.6%, 3.0, 89.9ms, 0 | 20260506 09:35:50.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:50.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:50.397 | INFO | PRE_HQ : 30.2%, 1.0, 15.9ms, 0 | 20260506 09:35:50.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:50.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:50.398 | INFO | Uploaded 36GiB to GPU in 1.60169s (22.4762GiB/s) | 20260506 09:35:50.398 | INFO | Downloaded 34GiB to GPU in 1.51841s (22.3918GiB/s) | 20260506 09:35:55.537 | INFO | Analyzing chunk frames = [324608,325120) | 20260506 09:35:55.537 | INFO | MemInfo: { | 20260506 09:35:55.537 | INFO | "gpu" : " 17 GB", | 20260506 09:35:55.537 | INFO | "huge" : "", | 20260506 09:35:55.537 | INFO | "huge_shared" : " 27 GB", | 20260506 09:35:55.537 | INFO | "malloc" : " 14 GB", | 20260506 09:35:55.537 | INFO | "pinned" : " 29 GB" | 20260506 09:35:55.537 | INFO | } | 20260506 09:35:55.537 | INFO | Perf report for the last 5.13947 seconds: | 20260506 09:35:55.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:35:55.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:55.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:55.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:55.537 | INFO | ANALYSIS : 57.0%, 3.0, 88.5ms, 0 | 20260506 09:35:55.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:55.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:55.537 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:35:55.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:55.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:35:55.537 | INFO | Uploaded 36GiB to GPU in 1.60581s (22.4186GiB/s) | 20260506 09:35:55.537 | INFO | Downloaded 34GiB to GPU in 1.52347s (22.3174GiB/s) | 20260506 09:36:00.676 | INFO | Analyzing chunk frames = [325120,325632) | 20260506 09:36:00.676 | INFO | MemInfo: { | 20260506 09:36:00.676 | INFO | "gpu" : " 17 GB", | 20260506 09:36:00.676 | INFO | "huge" : "", | 20260506 09:36:00.676 | INFO | "huge_shared" : " 27 GB", | 20260506 09:36:00.676 | INFO | "malloc" : " 14 GB", | 20260506 09:36:00.676 | INFO | "pinned" : " 29 GB" | 20260506 09:36:00.676 | INFO | } | 20260506 09:36:00.676 | INFO | Perf report for the last 5.13914 seconds: | 20260506 09:36:00.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:36:00.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:00.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:00.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:00.676 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:36:00.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:00.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:00.676 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:36:00.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:00.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:00.676 | INFO | Uploaded 36GiB to GPU in 1.60554s (22.4224GiB/s) | 20260506 09:36:00.676 | INFO | Downloaded 34GiB to GPU in 1.52259s (22.3304GiB/s) | 20260506 09:36:05.815 | INFO | Analyzing chunk frames = [325632,326144) | 20260506 09:36:05.815 | INFO | MemInfo: { | 20260506 09:36:05.815 | INFO | "gpu" : " 17 GB", | 20260506 09:36:05.815 | INFO | "huge" : "", | 20260506 09:36:05.815 | INFO | "huge_shared" : " 27 GB", | 20260506 09:36:05.815 | INFO | "malloc" : " 14 GB", | 20260506 09:36:05.815 | INFO | "pinned" : " 29 GB" | 20260506 09:36:05.815 | INFO | } | 20260506 09:36:05.815 | INFO | Perf report for the last 5.13936 seconds: | 20260506 09:36:05.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:36:05.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:05.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:05.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:05.815 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 09:36:05.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:05.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:05.816 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:36:05.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:05.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:05.816 | INFO | Uploaded 36GiB to GPU in 1.60399s (22.444GiB/s) | 20260506 09:36:05.816 | INFO | Downloaded 34GiB to GPU in 1.52126s (22.3498GiB/s) | 20260506 09:36:10.955 | INFO | Analyzing chunk frames = [326144,326656) | 20260506 09:36:10.955 | INFO | MemInfo: { | 20260506 09:36:10.955 | INFO | "gpu" : " 17 GB", | 20260506 09:36:10.955 | INFO | "huge" : "", | 20260506 09:36:10.955 | INFO | "huge_shared" : " 27 GB", | 20260506 09:36:10.955 | INFO | "malloc" : " 14 GB", | 20260506 09:36:10.955 | INFO | "pinned" : " 29 GB" | 20260506 09:36:10.955 | INFO | } | 20260506 09:36:10.955 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:36:10.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:36:10.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:10.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:10.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:10.955 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 09:36:10.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:10.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:10.955 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:36:10.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:10.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:10.955 | INFO | Uploaded 36GiB to GPU in 1.6039s (22.4452GiB/s) | 20260506 09:36:10.955 | INFO | Downloaded 34GiB to GPU in 1.51991s (22.3698GiB/s) | 20260506 09:36:16.094 | INFO | Analyzing chunk frames = [326656,327168) | 20260506 09:36:16.094 | INFO | MemInfo: { | 20260506 09:36:16.094 | INFO | "gpu" : " 17 GB", | 20260506 09:36:16.094 | INFO | "huge" : "", | 20260506 09:36:16.094 | INFO | "huge_shared" : " 27 GB", | 20260506 09:36:16.094 | INFO | "malloc" : " 14 GB", | 20260506 09:36:16.094 | INFO | "pinned" : " 29 GB" | 20260506 09:36:16.094 | INFO | } | 20260506 09:36:16.094 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:36:16.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:36:16.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:16.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:16.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:16.094 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260506 09:36:16.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:16.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:16.094 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:36:16.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:16.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:16.094 | INFO | Uploaded 36GiB to GPU in 1.60457s (22.4359GiB/s) | 20260506 09:36:16.094 | INFO | Downloaded 34GiB to GPU in 1.52272s (22.3285GiB/s) | 20260506 09:36:21.233 | INFO | Analyzing chunk frames = [327168,327680) | 20260506 09:36:21.233 | INFO | MemInfo: { | 20260506 09:36:21.233 | INFO | "gpu" : " 17 GB", | 20260506 09:36:21.233 | INFO | "huge" : "", | 20260506 09:36:21.233 | INFO | "huge_shared" : " 27 GB", | 20260506 09:36:21.233 | INFO | "malloc" : " 14 GB", | 20260506 09:36:21.233 | INFO | "pinned" : " 29 GB" | 20260506 09:36:21.233 | INFO | } | 20260506 09:36:21.233 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:36:21.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:36:21.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:21.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:21.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:21.233 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:36:21.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:21.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:21.233 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:36:21.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:21.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:21.234 | INFO | Uploaded 36GiB to GPU in 1.60356s (22.45GiB/s) | 20260506 09:36:21.234 | INFO | Downloaded 34GiB to GPU in 1.52282s (22.327GiB/s) | 20260506 09:36:21.234 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:36:21.234 | INFO | Total mem usage: 15.7 GiB | 20260506 09:36:21.234 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 2.56%) | 20260506 09:36:21.234 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -5.22%) | 20260506 09:36:21.234 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:36:21.234 | INFO | High Water Breakdown: | 20260506 09:36:21.234 | INFO | 59.2% : 8.86 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:36:21.234 | INFO | 40.7% : 6.08 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:36:21.234 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:36:21.234 | INFO | ------------------------------------------ | 20260506 09:36:21.234 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:36:21.234 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:36:21.234 | INFO | Total mem usage: 26.5 GiB | 20260506 09:36:21.234 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:36:21.234 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:36:21.234 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:36:21.234 | INFO | High Water Breakdown: | 20260506 09:36:21.234 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:36:21.234 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:36:21.234 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:36:21.234 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:36:21.234 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:36:21.234 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:36:21.234 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:36:21.234 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:36:21.234 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:36:21.234 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:36:21.234 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:36:21.234 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:36:21.234 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:36:21.234 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:36:21.234 | INFO | ------------------------------------------ | 20260506 09:36:21.234 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:36:21.234 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:36:21.234 | INFO | Total mem usage: 25.6 GiB | 20260506 09:36:21.234 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:36:21.234 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:36:21.234 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:36:21.234 | INFO | High Water Breakdown: | 20260506 09:36:21.234 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:36:21.234 | INFO | ------------------------------------------ | 20260506 09:36:21.234 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:36:21.234 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:36:21.234 | INFO | Total mem usage: 15.7 GiB | 20260506 09:36:21.234 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:36:21.234 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:36:21.234 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:36:21.234 | INFO | High Water Breakdown: | 20260506 09:36:21.234 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:36:21.234 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:36:21.234 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:36:21.234 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:36:21.234 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:36:21.234 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:36:21.234 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:36:21.234 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:36:21.234 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:36:21.234 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:36:21.234 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:36:21.234 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:36:21.234 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:36:21.234 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:36:21.234 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:36:21.234 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:36:21.234 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:36:21.234 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:36:21.234 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:36:21.235 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:36:21.235 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:36:21.235 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:36:21.235 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:36:21.235 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:36:21.235 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:36:21.235 | INFO | ------------------------------------------ | 20260506 09:36:21.235 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:36:21.235 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:36:21.235 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:36:21.235 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:36:21.235 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 09:36:21.235 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / -nan, 4.14e-03 / 0.00e+00 | 20260506 09:36:21.235 | INFO | PerfReport: 10.90%, Baseline -- 3.29e+00 / -nan, 1.70e-02 / 0.00e+00 | 20260506 09:36:21.235 | INFO | PerfReport: 3.05%, Histogram -- 9.21e-01 / -nan, 5.17e-02 / 0.00e+00 | 20260506 09:36:21.235 | INFO | PerfReport: 16.04%, DME -- 3.88e+01 / -nan, 2.87e+00 / 0.00e+00 | 20260506 09:36:21.235 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.72e+01 / -nan, 4.18e-02 / 0.00e+00 | 20260506 09:36:21.235 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.27e+00 / -nan, 1.35e-01 / 0.00e+00 | 20260506 09:36:21.235 | INFO | PerfReport: 3.81%, Metrics -- 1.15e+00 / -nan, 1.80e-01 / 0.00e+00 | 20260506 09:36:21.235 | INFO | PerfReport: 1.49%, PulseSerializing -- 4.49e-01 / -nan, 6.47e-02 / 0.00e+00 | 20260506 09:36:21.235 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:36:21.235 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:36:21.235 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:36:21.235 | INFO | PerfReport: Full Execution -- 9.04e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 09:36:21.235 | INFO | PerfReport: 18.57%, Upload -- 1.68e+01 / 0.00e+00, 4.41e-01 / 0.00e+00 | 20260506 09:36:21.235 | INFO | PerfReport: 17.82%, Download -- 1.61e+01 / 0.00e+00, 7.77e-01 / 0.00e+00 | 20260506 09:36:21.235 | INFO | PerfReport: 33.44%, Compute -- 3.02e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 09:36:21.235 | INFO | PerfReport: 0.56%, LockUpload -- 5.10e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 09:36:21.235 | INFO | PerfReport: 0.00%, LockDownload -- 4.80e-04 / 0.00e+00, 1.19e-04 / 0.00e+00 | 20260506 09:36:21.235 | INFO | PerfReport: 29.59%, LockCompute -- 2.67e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 09:36:21.235 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260506 09:36:21.331 | INFO | Starting a round of Baz IO | 20260506 09:36:24.252 | INFO | Populating RTMetrics report | 20260506 09:36:24.253 | INFO | RTMetrics report populated | 20260506 09:36:24.253 | INFO | Opening tmp RTMetrics json file | 20260506 09:36:24.282 | INFO | Serializing RTMetrics to json | 20260506 09:36:24.284 | INFO | Serialization done, swapping tmp file now... | 20260506 09:36:24.284 | INFO | RTMetrics done | 20260506 09:36:24.495 | INFO | Finished a round of Baz IO: Wrote 1.869 GiB in 3.164 seconds | 20260506 09:36:24.495 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.99% / 0.09% / 46.91% | 20260506 09:36:24.495 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.00391% / 0.000246366% / 6.35783e-05% | 20260506 09:36:24.495 | INFO | Baz IO overall: | 20260506 09:36:24.496 | INFO | BytesWritten : 129819954672 | 20260506 09:36:24.496 | INFO | NumEvents : 61625783429 | 20260506 09:36:26.373 | INFO | Analyzing chunk frames = [327680,328192) | 20260506 09:36:26.373 | INFO | MemInfo: { | 20260506 09:36:26.373 | INFO | "gpu" : " 17 GB", | 20260506 09:36:26.373 | INFO | "huge" : "", | 20260506 09:36:26.373 | INFO | "huge_shared" : " 27 GB", | 20260506 09:36:26.373 | INFO | "malloc" : " 14 GB", | 20260506 09:36:26.373 | INFO | "pinned" : " 29 GB" | 20260506 09:36:26.373 | INFO | } | 20260506 09:36:26.373 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:36:26.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:36:26.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:26.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:26.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:26.373 | INFO | ANALYSIS : 57.8%, 2.9, 88.6ms, 0 | 20260506 09:36:26.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:26.373 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 09:36:26.373 | INFO | PRE_HQ : 56.9%, 2.1, 61.8ms, 0 | 20260506 09:36:26.373 | INFO | BAZWRITER : 61.1%, 4.4, 139.8ms, 0 | 20260506 09:36:26.373 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 09:36:26.373 | INFO | Uploaded 36GiB to GPU in 1.64731s (21.8537GiB/s) | 20260506 09:36:26.373 | INFO | Downloaded 38GiB to GPU in 1.70937s (22.2304GiB/s) | 20260506 09:36:31.512 | INFO | Analyzing chunk frames = [328192,328704) | 20260506 09:36:31.512 | INFO | MemInfo: { | 20260506 09:36:31.512 | INFO | "gpu" : " 17 GB", | 20260506 09:36:31.512 | INFO | "huge" : "", | 20260506 09:36:31.512 | INFO | "huge_shared" : " 27 GB", | 20260506 09:36:31.512 | INFO | "malloc" : " 14 GB", | 20260506 09:36:31.512 | INFO | "pinned" : " 29 GB" | 20260506 09:36:31.512 | INFO | } | 20260506 09:36:31.512 | INFO | Perf report for the last 5.13962 seconds: | 20260506 09:36:31.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:36:31.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:31.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:31.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:31.512 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260506 09:36:31.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:31.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:31.512 | INFO | PRE_HQ : 29.7%, 1.0, 15.6ms, 0 | 20260506 09:36:31.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:31.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:31.512 | INFO | Uploaded 36GiB to GPU in 1.60254s (22.4643GiB/s) | 20260506 09:36:31.513 | INFO | Downloaded 34GiB to GPU in 1.51998s (22.3687GiB/s) | 20260506 09:36:36.652 | INFO | Analyzing chunk frames = [328704,329216) | 20260506 09:36:36.652 | INFO | MemInfo: { | 20260506 09:36:36.652 | INFO | "gpu" : " 17 GB", | 20260506 09:36:36.652 | INFO | "huge" : "", | 20260506 09:36:36.652 | INFO | "huge_shared" : " 27 GB", | 20260506 09:36:36.652 | INFO | "malloc" : " 14 GB", | 20260506 09:36:36.652 | INFO | "pinned" : " 29 GB" | 20260506 09:36:36.652 | INFO | } | 20260506 09:36:36.652 | INFO | Perf report for the last 5.1394 seconds: | 20260506 09:36:36.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:36:36.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:36.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:36.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:36.652 | INFO | ANALYSIS : 56.9%, 3.0, 88.5ms, 0 | 20260506 09:36:36.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:36.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:36.652 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:36:36.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:36.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:36.652 | INFO | Uploaded 36GiB to GPU in 1.60552s (22.4227GiB/s) | 20260506 09:36:36.652 | INFO | Downloaded 34GiB to GPU in 1.52396s (22.3103GiB/s) | 20260506 09:36:41.791 | INFO | Analyzing chunk frames = [329216,329728) | 20260506 09:36:41.791 | INFO | MemInfo: { | 20260506 09:36:41.791 | INFO | "gpu" : " 17 GB", | 20260506 09:36:41.791 | INFO | "huge" : "", | 20260506 09:36:41.791 | INFO | "huge_shared" : " 27 GB", | 20260506 09:36:41.791 | INFO | "malloc" : " 14 GB", | 20260506 09:36:41.791 | INFO | "pinned" : " 29 GB" | 20260506 09:36:41.791 | INFO | } | 20260506 09:36:41.791 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:36:41.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:36:41.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:41.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:41.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:41.791 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:36:41.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:41.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:41.791 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:36:41.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:41.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:41.791 | INFO | Uploaded 36GiB to GPU in 1.60455s (22.4363GiB/s) | 20260506 09:36:41.791 | INFO | Downloaded 34GiB to GPU in 1.52326s (22.3205GiB/s) | 20260506 09:36:46.950 | INFO | Analyzing chunk frames = [329728,330240) | 20260506 09:36:46.950 | INFO | MemInfo: { | 20260506 09:36:46.950 | INFO | "gpu" : " 17 GB", | 20260506 09:36:46.950 | INFO | "huge" : "", | 20260506 09:36:46.950 | INFO | "huge_shared" : " 27 GB", | 20260506 09:36:46.950 | INFO | "malloc" : " 14 GB", | 20260506 09:36:46.950 | INFO | "pinned" : " 29 GB" | 20260506 09:36:46.950 | INFO | } | 20260506 09:36:46.950 | INFO | Perf report for the last 5.15909 seconds: | 20260506 09:36:46.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:36:46.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:46.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:46.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:46.950 | INFO | ANALYSIS : 56.4%, 3.0, 88.2ms, 0 | 20260506 09:36:46.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:46.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:46.950 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260506 09:36:46.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:46.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:46.950 | INFO | Uploaded 36GiB to GPU in 1.60417s (22.4416GiB/s) | 20260506 09:36:46.950 | INFO | Downloaded 34GiB to GPU in 1.52114s (22.3516GiB/s) | 20260506 09:36:52.069 | INFO | Analyzing chunk frames = [330240,330752) | 20260506 09:36:52.069 | INFO | MemInfo: { | 20260506 09:36:52.069 | INFO | "gpu" : " 17 GB", | 20260506 09:36:52.069 | INFO | "huge" : "", | 20260506 09:36:52.070 | INFO | "huge_shared" : " 27 GB", | 20260506 09:36:52.070 | INFO | "malloc" : " 14 GB", | 20260506 09:36:52.070 | INFO | "pinned" : " 29 GB" | 20260506 09:36:52.070 | INFO | } | 20260506 09:36:52.070 | INFO | Perf report for the last 5.11985 seconds: | 20260506 09:36:52.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:36:52.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:52.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:52.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:52.070 | INFO | ANALYSIS : 57.1%, 3.0, 88.8ms, 0 | 20260506 09:36:52.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:52.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:52.070 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:36:52.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:52.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:52.070 | INFO | Uploaded 36GiB to GPU in 1.60469s (22.4343GiB/s) | 20260506 09:36:52.070 | INFO | Downloaded 34GiB to GPU in 1.52213s (22.3371GiB/s) | 20260506 09:36:57.209 | INFO | Analyzing chunk frames = [330752,331264) | 20260506 09:36:57.209 | INFO | MemInfo: { | 20260506 09:36:57.209 | INFO | "gpu" : " 17 GB", | 20260506 09:36:57.209 | INFO | "huge" : "", | 20260506 09:36:57.209 | INFO | "huge_shared" : " 27 GB", | 20260506 09:36:57.209 | INFO | "malloc" : " 14 GB", | 20260506 09:36:57.209 | INFO | "pinned" : " 29 GB" | 20260506 09:36:57.209 | INFO | } | 20260506 09:36:57.209 | INFO | Perf report for the last 5.13895 seconds: | 20260506 09:36:57.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:36:57.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:57.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:57.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:57.209 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:36:57.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:57.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:57.209 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:36:57.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:57.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:36:57.209 | INFO | Uploaded 36GiB to GPU in 1.60512s (22.4282GiB/s) | 20260506 09:36:57.209 | INFO | Downloaded 34GiB to GPU in 1.52277s (22.3277GiB/s) | 20260506 09:37:02.348 | INFO | Analyzing chunk frames = [331264,331776) | 20260506 09:37:02.348 | INFO | MemInfo: { | 20260506 09:37:02.348 | INFO | "gpu" : " 17 GB", | 20260506 09:37:02.348 | INFO | "huge" : "", | 20260506 09:37:02.348 | INFO | "huge_shared" : " 27 GB", | 20260506 09:37:02.348 | INFO | "malloc" : " 14 GB", | 20260506 09:37:02.348 | INFO | "pinned" : " 29 GB" | 20260506 09:37:02.348 | INFO | } | 20260506 09:37:02.348 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:37:02.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:37:02.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:02.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:02.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:02.348 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:37:02.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:02.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:02.348 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 09:37:02.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:02.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:02.349 | INFO | Uploaded 36GiB to GPU in 1.60418s (22.4414GiB/s) | 20260506 09:37:02.349 | INFO | Downloaded 34GiB to GPU in 1.52308s (22.3232GiB/s) | 20260506 09:37:02.445 | INFO | Starting a round of Baz IO | 20260506 09:37:05.369 | INFO | Populating RTMetrics report | 20260506 09:37:05.369 | INFO | RTMetrics report populated | 20260506 09:37:05.369 | INFO | Opening tmp RTMetrics json file | 20260506 09:37:05.370 | INFO | Serializing RTMetrics to json | 20260506 09:37:05.374 | INFO | Serialization done, swapping tmp file now... | 20260506 09:37:05.379 | INFO | RTMetrics done | 20260506 09:37:05.603 | INFO | Finished a round of Baz IO: Wrote 1.875 GiB in 3.158 seconds | 20260506 09:37:05.603 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.90% / 0.09% / 47.01% | 20260506 09:37:05.603 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 9.02052% / 0.000222524% / 5.16574e-05% | 20260506 09:37:05.603 | INFO | Baz IO overall: | 20260506 09:37:05.603 | INFO | BytesWritten : 131833058026 | 20260506 09:37:05.603 | INFO | NumEvents : 62604096611 | 20260506 09:37:07.487 | INFO | Analyzing chunk frames = [331776,332288) | 20260506 09:37:07.487 | INFO | MemInfo: { | 20260506 09:37:07.487 | INFO | "gpu" : " 17 GB", | 20260506 09:37:07.488 | INFO | "huge" : "", | 20260506 09:37:07.488 | INFO | "huge_shared" : " 27 GB", | 20260506 09:37:07.488 | INFO | "malloc" : " 14 GB", | 20260506 09:37:07.488 | INFO | "pinned" : " 29 GB" | 20260506 09:37:07.488 | INFO | } | 20260506 09:37:07.488 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:37:07.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:37:07.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:07.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:07.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:07.488 | INFO | ANALYSIS : 57.8%, 2.8, 85.1ms, 0 | 20260506 09:37:07.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:07.488 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 09:37:07.488 | INFO | PRE_HQ : 57.0%, 2.1, 61.5ms, 0 | 20260506 09:37:07.488 | INFO | BAZWRITER : 61.0%, 4.8, 149.9ms, 0 | 20260506 09:37:07.488 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 09:37:07.488 | INFO | Uploaded 36GiB to GPU in 1.66741s (21.5904GiB/s) | 20260506 09:37:07.488 | INFO | Downloaded 38GiB to GPU in 1.73282s (21.9296GiB/s) | 20260506 09:37:12.645 | INFO | Analyzing chunk frames = [332288,332800) | 20260506 09:37:12.645 | INFO | MemInfo: { | 20260506 09:37:12.645 | INFO | "gpu" : " 17 GB", | 20260506 09:37:12.645 | INFO | "huge" : "", | 20260506 09:37:12.645 | INFO | "huge_shared" : " 27 GB", | 20260506 09:37:12.645 | INFO | "malloc" : " 14 GB", | 20260506 09:37:12.645 | INFO | "pinned" : " 29 GB" | 20260506 09:37:12.645 | INFO | } | 20260506 09:37:12.645 | INFO | Perf report for the last 5.15781 seconds: | 20260506 09:37:12.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:37:12.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:12.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:12.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:12.645 | INFO | ANALYSIS : 57.4%, 3.0, 89.8ms, 0 | 20260506 09:37:12.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:12.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:12.646 | INFO | PRE_HQ : 30.4%, 1.0, 16.3ms, 0 | 20260506 09:37:12.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:12.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:12.646 | INFO | Uploaded 36GiB to GPU in 1.60203s (22.4715GiB/s) | 20260506 09:37:12.646 | INFO | Downloaded 34GiB to GPU in 1.51848s (22.3908GiB/s) | 20260506 09:37:17.766 | INFO | Analyzing chunk frames = [332800,333312) | 20260506 09:37:17.766 | INFO | MemInfo: { | 20260506 09:37:17.766 | INFO | "gpu" : " 17 GB", | 20260506 09:37:17.766 | INFO | "huge" : "", | 20260506 09:37:17.766 | INFO | "huge_shared" : " 27 GB", | 20260506 09:37:17.766 | INFO | "malloc" : " 14 GB", | 20260506 09:37:17.766 | INFO | "pinned" : " 29 GB" | 20260506 09:37:17.766 | INFO | } | 20260506 09:37:17.766 | INFO | Perf report for the last 5.12085 seconds: | 20260506 09:37:17.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:37:17.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:17.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:17.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:17.766 | INFO | ANALYSIS : 57.2%, 3.0, 88.5ms, 0 | 20260506 09:37:17.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:17.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:17.766 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:37:17.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:17.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:17.767 | INFO | Uploaded 36GiB to GPU in 1.60688s (22.4037GiB/s) | 20260506 09:37:17.767 | INFO | Downloaded 34GiB to GPU in 1.52514s (22.293GiB/s) | 20260506 09:37:22.906 | INFO | Analyzing chunk frames = [333312,333824) | 20260506 09:37:22.906 | INFO | MemInfo: { | 20260506 09:37:22.906 | INFO | "gpu" : " 17 GB", | 20260506 09:37:22.906 | INFO | "huge" : "", | 20260506 09:37:22.906 | INFO | "huge_shared" : " 27 GB", | 20260506 09:37:22.906 | INFO | "malloc" : " 14 GB", | 20260506 09:37:22.906 | INFO | "pinned" : " 29 GB" | 20260506 09:37:22.906 | INFO | } | 20260506 09:37:22.906 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:37:22.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:37:22.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:22.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:22.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:22.906 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:37:22.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:22.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:22.906 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:37:22.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:22.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:22.906 | INFO | Uploaded 36GiB to GPU in 1.6061s (22.4145GiB/s) | 20260506 09:37:22.906 | INFO | Downloaded 34GiB to GPU in 1.52331s (22.3197GiB/s) | 20260506 09:37:28.045 | INFO | Analyzing chunk frames = [333824,334336) | 20260506 09:37:28.045 | INFO | MemInfo: { | 20260506 09:37:28.045 | INFO | "gpu" : " 17 GB", | 20260506 09:37:28.045 | INFO | "huge" : "", | 20260506 09:37:28.045 | INFO | "huge_shared" : " 27 GB", | 20260506 09:37:28.045 | INFO | "malloc" : " 14 GB", | 20260506 09:37:28.045 | INFO | "pinned" : " 29 GB" | 20260506 09:37:28.045 | INFO | } | 20260506 09:37:28.045 | INFO | Perf report for the last 5.13918 seconds: | 20260506 09:37:28.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:37:28.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:28.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:28.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:28.045 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:37:28.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:28.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:28.045 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:37:28.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:28.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:28.045 | INFO | Uploaded 36GiB to GPU in 1.6036s (22.4495GiB/s) | 20260506 09:37:28.045 | INFO | Downloaded 34GiB to GPU in 1.52123s (22.3504GiB/s) | 20260506 09:37:33.184 | INFO | Analyzing chunk frames = [334336,334848) | 20260506 09:37:33.184 | INFO | MemInfo: { | 20260506 09:37:33.184 | INFO | "gpu" : " 17 GB", | 20260506 09:37:33.184 | INFO | "huge" : "", | 20260506 09:37:33.184 | INFO | "huge_shared" : " 27 GB", | 20260506 09:37:33.184 | INFO | "malloc" : " 14 GB", | 20260506 09:37:33.184 | INFO | "pinned" : " 29 GB" | 20260506 09:37:33.184 | INFO | } | 20260506 09:37:33.184 | INFO | Perf report for the last 5.13951 seconds: | 20260506 09:37:33.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:37:33.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:33.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:33.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:33.184 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:37:33.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:33.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:33.185 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:37:33.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:33.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:33.185 | INFO | Uploaded 36GiB to GPU in 1.60468s (22.4343GiB/s) | 20260506 09:37:33.185 | INFO | Downloaded 34GiB to GPU in 1.52131s (22.3492GiB/s) | 20260506 09:37:38.324 | INFO | Analyzing chunk frames = [334848,335360) | 20260506 09:37:38.324 | INFO | MemInfo: { | 20260506 09:37:38.324 | INFO | "gpu" : " 17 GB", | 20260506 09:37:38.324 | INFO | "huge" : "", | 20260506 09:37:38.324 | INFO | "huge_shared" : " 27 GB", | 20260506 09:37:38.324 | INFO | "malloc" : " 14 GB", | 20260506 09:37:38.324 | INFO | "pinned" : " 29 GB" | 20260506 09:37:38.324 | INFO | } | 20260506 09:37:38.324 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:37:38.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:37:38.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:38.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:38.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:38.324 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:37:38.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:38.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:38.324 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:37:38.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:38.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:38.324 | INFO | Uploaded 36GiB to GPU in 1.60413s (22.442GiB/s) | 20260506 09:37:38.324 | INFO | Downloaded 34GiB to GPU in 1.52232s (22.3343GiB/s) | 20260506 09:37:43.463 | INFO | Analyzing chunk frames = [335360,335872) | 20260506 09:37:43.463 | INFO | MemInfo: { | 20260506 09:37:43.463 | INFO | "gpu" : " 17 GB", | 20260506 09:37:43.463 | INFO | "huge" : "", | 20260506 09:37:43.463 | INFO | "huge_shared" : " 27 GB", | 20260506 09:37:43.463 | INFO | "malloc" : " 14 GB", | 20260506 09:37:43.463 | INFO | "pinned" : " 29 GB" | 20260506 09:37:43.463 | INFO | } | 20260506 09:37:43.463 | INFO | Perf report for the last 5.1392 seconds: | 20260506 09:37:43.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:37:43.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:43.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:43.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:43.463 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:37:43.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:43.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:43.463 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 09:37:43.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:43.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:43.463 | INFO | Uploaded 36GiB to GPU in 1.60484s (22.4322GiB/s) | 20260506 09:37:43.463 | INFO | Downloaded 34GiB to GPU in 1.52236s (22.3338GiB/s) | 20260506 09:37:43.463 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:37:43.463 | INFO | Total mem usage: 15.7 GiB | 20260506 09:37:43.463 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 2.56%) | 20260506 09:37:43.463 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -3.74%) | 20260506 09:37:43.463 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:37:43.463 | INFO | High Water Breakdown: | 20260506 09:37:43.463 | INFO | 59.5% : 9.03 GiB ( 95.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:37:43.463 | INFO | 40.4% : 6.13 GiB ( 91.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:37:43.463 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:37:43.463 | INFO | ------------------------------------------ | 20260506 09:37:43.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:37:43.463 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:37:43.463 | INFO | Total mem usage: 26.5 GiB | 20260506 09:37:43.463 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:37:43.463 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:37:43.463 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:37:43.463 | INFO | High Water Breakdown: | 20260506 09:37:43.463 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:37:43.463 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:37:43.463 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:37:43.463 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:37:43.463 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:37:43.463 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:37:43.463 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:37:43.463 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:37:43.463 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:37:43.463 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:37:43.463 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:37:43.463 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:37:43.463 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:37:43.463 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:37:43.463 | INFO | ------------------------------------------ | 20260506 09:37:43.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:37:43.463 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:37:43.463 | INFO | Total mem usage: 25.6 GiB | 20260506 09:37:43.463 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:37:43.463 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:37:43.463 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:37:43.463 | INFO | High Water Breakdown: | 20260506 09:37:43.463 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:37:43.463 | INFO | ------------------------------------------ | 20260506 09:37:43.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:37:43.463 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:37:43.463 | INFO | Total mem usage: 15.7 GiB | 20260506 09:37:43.463 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:37:43.463 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:37:43.463 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:37:43.463 | INFO | High Water Breakdown: | 20260506 09:37:43.463 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:37:43.463 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:37:43.463 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:37:43.463 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:37:43.463 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:37:43.463 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:37:43.463 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:37:43.463 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:37:43.463 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:37:43.463 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:37:43.463 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:37:43.463 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:37:43.463 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:37:43.463 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:37:43.463 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:37:43.463 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:37:43.463 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:37:43.463 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:37:43.463 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:37:43.463 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:37:43.463 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:37:43.463 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:37:43.463 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:37:43.463 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:37:43.463 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:37:43.463 | INFO | ------------------------------------------ | 20260506 09:37:43.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:37:43.463 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:37:43.463 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:37:43.463 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:37:43.463 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 09:37:43.463 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / -nan, 4.48e-03 / 0.00e+00 | 20260506 09:37:43.463 | INFO | PerfReport: 10.90%, Baseline -- 3.29e+00 / -nan, 1.59e-02 / 0.00e+00 | 20260506 09:37:43.463 | INFO | PerfReport: 3.05%, Histogram -- 9.21e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260506 09:37:43.463 | INFO | PerfReport: 16.05%, DME -- 3.88e+01 / -nan, 3.03e+00 / 0.00e+00 | 20260506 09:37:43.463 | INFO | PerfReport: 56.91%, FrameLabeling -- 1.72e+01 / -nan, 4.14e-02 / 0.00e+00 | 20260506 09:37:43.463 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.27e+00 / -nan, 1.35e-01 / 0.00e+00 | 20260506 09:37:43.463 | INFO | PerfReport: 3.82%, Metrics -- 1.15e+00 / -nan, 1.80e-01 / 0.00e+00 | 20260506 09:37:43.463 | INFO | PerfReport: 1.48%, PulseSerializing -- 4.48e-01 / -nan, 6.46e-02 / 0.00e+00 | 20260506 09:37:43.463 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:37:43.463 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:37:43.463 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:37:43.463 | INFO | PerfReport: Full Execution -- 9.04e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 09:37:43.463 | INFO | PerfReport: 18.57%, Upload -- 1.68e+01 / 0.00e+00, 4.46e-01 / 0.00e+00 | 20260506 09:37:43.464 | INFO | PerfReport: 17.83%, Download -- 1.61e+01 / 0.00e+00, 7.74e-01 / 0.00e+00 | 20260506 09:37:43.464 | INFO | PerfReport: 33.44%, Compute -- 3.02e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 09:37:43.464 | INFO | PerfReport: 0.56%, LockUpload -- 5.10e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260506 09:37:43.464 | INFO | PerfReport: 0.00%, LockDownload -- 4.91e-04 / 0.00e+00, 2.15e-04 / 0.00e+00 | 20260506 09:37:43.464 | INFO | PerfReport: 29.57%, LockCompute -- 2.67e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 09:37:43.464 | INFO | PerfReport: 0.02%, Misc -- 1.37e-02 / 0.00e+00, 2.28e-02 / 0.00e+00 | 20260506 09:37:43.560 | INFO | Starting a round of Baz IO | 20260506 09:37:46.484 | INFO | Populating RTMetrics report | 20260506 09:37:46.485 | INFO | RTMetrics report populated | 20260506 09:37:46.485 | INFO | Opening tmp RTMetrics json file | 20260506 09:37:46.596 | INFO | Serializing RTMetrics to json | 20260506 09:37:46.645 | INFO | Serialization done, swapping tmp file now... | 20260506 09:37:46.646 | INFO | RTMetrics done | 20260506 09:37:46.848 | INFO | Finished a round of Baz IO: Wrote 1.875 GiB in 3.287 seconds | 20260506 09:37:46.848 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.81% / 0.09% / 47.10% | 20260506 09:37:46.848 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.95416% / 0.000309944% / 5.96046e-05% | 20260506 09:37:46.848 | INFO | Baz IO overall: | 20260506 09:37:46.848 | INFO | BytesWritten : 133846559717 | 20260506 09:37:46.848 | INFO | NumEvents : 63581894478 | 20260506 09:37:48.602 | INFO | Analyzing chunk frames = [335872,336384) | 20260506 09:37:48.602 | INFO | MemInfo: { | 20260506 09:37:48.602 | INFO | "gpu" : " 17 GB", | 20260506 09:37:48.602 | INFO | "huge" : "", | 20260506 09:37:48.602 | INFO | "huge_shared" : " 27 GB", | 20260506 09:37:48.602 | INFO | "malloc" : " 14 GB", | 20260506 09:37:48.602 | INFO | "pinned" : " 29 GB" | 20260506 09:37:48.602 | INFO | } | 20260506 09:37:48.602 | INFO | Perf report for the last 5.1394 seconds: | 20260506 09:37:48.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:37:48.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:48.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:48.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:48.602 | INFO | ANALYSIS : 57.8%, 2.9, 86.5ms, 0 | 20260506 09:37:48.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:48.602 | INFO | RT_METRICS : 47.2%, 1.0, 24.7ms, 0 | 20260506 09:37:48.602 | INFO | PRE_HQ : 57.0%, 2.1, 61.6ms, 0 | 20260506 09:37:48.602 | INFO | BAZWRITER : 63.4%, 4.7, 154.0ms, 0 | 20260506 09:37:48.602 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.3ms, 0 | 20260506 09:37:48.603 | INFO | Uploaded 36GiB to GPU in 1.67121s (21.5413GiB/s) | 20260506 09:37:48.603 | INFO | Downloaded 38GiB to GPU in 1.73533s (21.8978GiB/s) | 20260506 09:37:53.742 | INFO | Analyzing chunk frames = [336384,336896) | 20260506 09:37:53.742 | INFO | MemInfo: { | 20260506 09:37:53.742 | INFO | "gpu" : " 17 GB", | 20260506 09:37:53.742 | INFO | "huge" : "", | 20260506 09:37:53.742 | INFO | "huge_shared" : " 27 GB", | 20260506 09:37:53.742 | INFO | "malloc" : " 14 GB", | 20260506 09:37:53.742 | INFO | "pinned" : " 29 GB" | 20260506 09:37:53.742 | INFO | } | 20260506 09:37:53.742 | INFO | Perf report for the last 5.1396 seconds: | 20260506 09:37:53.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:37:53.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:53.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:53.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:53.742 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260506 09:37:53.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:53.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:53.742 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260506 09:37:53.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:53.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:53.742 | INFO | Uploaded 36GiB to GPU in 1.60267s (22.4626GiB/s) | 20260506 09:37:53.742 | INFO | Downloaded 34GiB to GPU in 1.52025s (22.3647GiB/s) | 20260506 09:37:58.881 | INFO | Analyzing chunk frames = [336896,337408) | 20260506 09:37:58.881 | INFO | MemInfo: { | 20260506 09:37:58.881 | INFO | "gpu" : " 17 GB", | 20260506 09:37:58.881 | INFO | "huge" : "", | 20260506 09:37:58.881 | INFO | "huge_shared" : " 27 GB", | 20260506 09:37:58.881 | INFO | "malloc" : " 14 GB", | 20260506 09:37:58.881 | INFO | "pinned" : " 29 GB" | 20260506 09:37:58.881 | INFO | } | 20260506 09:37:58.881 | INFO | Perf report for the last 5.1392 seconds: | 20260506 09:37:58.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:37:58.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:58.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:58.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:58.881 | INFO | ANALYSIS : 57.0%, 3.0, 88.5ms, 0 | 20260506 09:37:58.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:58.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:58.881 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:37:58.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:58.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:37:58.881 | INFO | Uploaded 36GiB to GPU in 1.60593s (22.4169GiB/s) | 20260506 09:37:58.881 | INFO | Downloaded 34GiB to GPU in 1.52377s (22.3131GiB/s) | 20260506 09:38:04.020 | INFO | Analyzing chunk frames = [337408,337920) | 20260506 09:38:04.020 | INFO | MemInfo: { | 20260506 09:38:04.020 | INFO | "gpu" : " 17 GB", | 20260506 09:38:04.020 | INFO | "huge" : "", | 20260506 09:38:04.020 | INFO | "huge_shared" : " 27 GB", | 20260506 09:38:04.020 | INFO | "malloc" : " 14 GB", | 20260506 09:38:04.020 | INFO | "pinned" : " 29 GB" | 20260506 09:38:04.020 | INFO | } | 20260506 09:38:04.020 | INFO | Perf report for the last 5.13921 seconds: | 20260506 09:38:04.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:38:04.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:04.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:04.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:04.020 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:38:04.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:04.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:04.021 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:38:04.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:04.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:04.021 | INFO | Uploaded 36GiB to GPU in 1.60588s (22.4176GiB/s) | 20260506 09:38:04.021 | INFO | Downloaded 34GiB to GPU in 1.52304s (22.3238GiB/s) | 20260506 09:38:09.160 | INFO | Analyzing chunk frames = [337920,338432) | 20260506 09:38:09.160 | INFO | MemInfo: { | 20260506 09:38:09.160 | INFO | "gpu" : " 17 GB", | 20260506 09:38:09.160 | INFO | "huge" : "", | 20260506 09:38:09.160 | INFO | "huge_shared" : " 27 GB", | 20260506 09:38:09.160 | INFO | "malloc" : " 14 GB", | 20260506 09:38:09.160 | INFO | "pinned" : " 29 GB" | 20260506 09:38:09.160 | INFO | } | 20260506 09:38:09.160 | INFO | Perf report for the last 5.13965 seconds: | 20260506 09:38:09.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:38:09.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:09.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:09.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:09.160 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 09:38:09.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:09.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:09.160 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:38:09.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:09.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:09.160 | INFO | Uploaded 36GiB to GPU in 1.60378s (22.447GiB/s) | 20260506 09:38:09.160 | INFO | Downloaded 34GiB to GPU in 1.52039s (22.3627GiB/s) | 20260506 09:38:14.299 | INFO | Analyzing chunk frames = [338432,338944) | 20260506 09:38:14.299 | INFO | MemInfo: { | 20260506 09:38:14.299 | INFO | "gpu" : " 17 GB", | 20260506 09:38:14.299 | INFO | "huge" : "", | 20260506 09:38:14.299 | INFO | "huge_shared" : " 27 GB", | 20260506 09:38:14.299 | INFO | "malloc" : " 14 GB", | 20260506 09:38:14.299 | INFO | "pinned" : " 29 GB" | 20260506 09:38:14.299 | INFO | } | 20260506 09:38:14.299 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:38:14.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:38:14.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:14.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:14.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:14.299 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:38:14.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:14.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:14.299 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:38:14.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:14.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:14.300 | INFO | Uploaded 36GiB to GPU in 1.6043s (22.4397GiB/s) | 20260506 09:38:14.300 | INFO | Downloaded 34GiB to GPU in 1.5209s (22.3552GiB/s) | 20260506 09:38:19.439 | INFO | Analyzing chunk frames = [338944,339456) | 20260506 09:38:19.439 | INFO | MemInfo: { | 20260506 09:38:19.439 | INFO | "gpu" : " 17 GB", | 20260506 09:38:19.439 | INFO | "huge" : "", | 20260506 09:38:19.439 | INFO | "huge_shared" : " 27 GB", | 20260506 09:38:19.439 | INFO | "malloc" : " 14 GB", | 20260506 09:38:19.439 | INFO | "pinned" : " 29 GB" | 20260506 09:38:19.439 | INFO | } | 20260506 09:38:19.439 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:38:19.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:38:19.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:19.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:19.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:19.439 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:38:19.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:19.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:19.439 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:38:19.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:19.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:19.439 | INFO | Uploaded 36GiB to GPU in 1.6043s (22.4396GiB/s) | 20260506 09:38:19.439 | INFO | Downloaded 34GiB to GPU in 1.52199s (22.3391GiB/s) | 20260506 09:38:24.578 | INFO | Analyzing chunk frames = [339456,339968) | 20260506 09:38:24.578 | INFO | MemInfo: { | 20260506 09:38:24.578 | INFO | "gpu" : " 17 GB", | 20260506 09:38:24.578 | INFO | "huge" : "", | 20260506 09:38:24.578 | INFO | "huge_shared" : " 27 GB", | 20260506 09:38:24.578 | INFO | "malloc" : " 14 GB", | 20260506 09:38:24.578 | INFO | "pinned" : " 29 GB" | 20260506 09:38:24.578 | INFO | } | 20260506 09:38:24.578 | INFO | Perf report for the last 5.13916 seconds: | 20260506 09:38:24.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:38:24.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:24.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:24.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:24.578 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:38:24.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:24.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:24.578 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 09:38:24.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:24.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:24.578 | INFO | Uploaded 36GiB to GPU in 1.60483s (22.4323GiB/s) | 20260506 09:38:24.578 | INFO | Downloaded 34GiB to GPU in 1.52283s (22.3269GiB/s) | 20260506 09:38:24.675 | INFO | Starting a round of Baz IO | 20260506 09:38:27.596 | INFO | Populating RTMetrics report | 20260506 09:38:27.597 | INFO | RTMetrics report populated | 20260506 09:38:27.597 | INFO | Opening tmp RTMetrics json file | 20260506 09:38:27.602 | INFO | Serializing RTMetrics to json | 20260506 09:38:27.627 | INFO | Serialization done, swapping tmp file now... | 20260506 09:38:27.628 | INFO | RTMetrics done | 20260506 09:38:27.705 | INFO | Finished a round of Baz IO: Wrote 1.877 GiB in 3.03 seconds | 20260506 09:38:27.705 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.72% / 0.09% / 47.19% | 20260506 09:38:27.705 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.91739% / 0.00025034% / 4.76837e-05% | 20260506 09:38:27.705 | INFO | Baz IO overall: | 20260506 09:38:27.705 | INFO | BytesWritten : 135862228765 | 20260506 09:38:27.705 | INFO | NumEvents : 64560345237 | 20260506 09:38:29.717 | INFO | Analyzing chunk frames = [339968,340480) | 20260506 09:38:29.717 | INFO | MemInfo: { | 20260506 09:38:29.717 | INFO | "gpu" : " 17 GB", | 20260506 09:38:29.717 | INFO | "huge" : "", | 20260506 09:38:29.717 | INFO | "huge_shared" : " 27 GB", | 20260506 09:38:29.717 | INFO | "malloc" : " 14 GB", | 20260506 09:38:29.717 | INFO | "pinned" : " 29 GB" | 20260506 09:38:29.717 | INFO | } | 20260506 09:38:29.717 | INFO | Perf report for the last 5.13947 seconds: | 20260506 09:38:29.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:38:29.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:29.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:29.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:29.717 | INFO | ANALYSIS : 57.8%, 2.9, 86.8ms, 0 | 20260506 09:38:29.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:29.717 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 09:38:29.717 | INFO | PRE_HQ : 57.1%, 2.1, 62.4ms, 0 | 20260506 09:38:29.717 | INFO | BAZWRITER : 58.5%, 4.6, 138.5ms, 0 | 20260506 09:38:29.718 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 09:38:29.718 | INFO | Uploaded 36GiB to GPU in 1.65187s (21.7935GiB/s) | 20260506 09:38:29.718 | INFO | Downloaded 38GiB to GPU in 1.71628s (22.1409GiB/s) | 20260506 09:38:34.857 | INFO | Analyzing chunk frames = [340480,340992) | 20260506 09:38:34.857 | INFO | MemInfo: { | 20260506 09:38:34.857 | INFO | "gpu" : " 17 GB", | 20260506 09:38:34.857 | INFO | "huge" : "", | 20260506 09:38:34.857 | INFO | "huge_shared" : " 27 GB", | 20260506 09:38:34.857 | INFO | "malloc" : " 14 GB", | 20260506 09:38:34.857 | INFO | "pinned" : " 29 GB" | 20260506 09:38:34.857 | INFO | } | 20260506 09:38:34.857 | INFO | Perf report for the last 5.1395 seconds: | 20260506 09:38:34.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:38:34.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:34.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:34.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:34.857 | INFO | ANALYSIS : 57.5%, 3.0, 89.8ms, 0 | 20260506 09:38:34.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:34.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:34.857 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260506 09:38:34.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:34.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:34.857 | INFO | Uploaded 36GiB to GPU in 1.6022s (22.4691GiB/s) | 20260506 09:38:34.857 | INFO | Downloaded 34GiB to GPU in 1.51939s (22.3774GiB/s) | 20260506 09:38:39.996 | INFO | Analyzing chunk frames = [340992,341504) | 20260506 09:38:39.996 | INFO | MemInfo: { | 20260506 09:38:39.996 | INFO | "gpu" : " 17 GB", | 20260506 09:38:39.996 | INFO | "huge" : "", | 20260506 09:38:39.996 | INFO | "huge_shared" : " 27 GB", | 20260506 09:38:39.996 | INFO | "malloc" : " 14 GB", | 20260506 09:38:39.996 | INFO | "pinned" : " 29 GB" | 20260506 09:38:39.996 | INFO | } | 20260506 09:38:39.996 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:38:39.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:38:39.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:39.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:39.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:39.996 | INFO | ANALYSIS : 57.0%, 3.0, 88.5ms, 0 | 20260506 09:38:39.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:39.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:39.996 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:38:39.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:39.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:39.996 | INFO | Uploaded 36GiB to GPU in 1.60602s (22.4156GiB/s) | 20260506 09:38:39.997 | INFO | Downloaded 34GiB to GPU in 1.52387s (22.3116GiB/s) | 20260506 09:38:45.135 | INFO | Analyzing chunk frames = [341504,342016) | 20260506 09:38:45.135 | INFO | MemInfo: { | 20260506 09:38:45.135 | INFO | "gpu" : " 17 GB", | 20260506 09:38:45.135 | INFO | "huge" : "", | 20260506 09:38:45.135 | INFO | "huge_shared" : " 27 GB", | 20260506 09:38:45.135 | INFO | "malloc" : " 14 GB", | 20260506 09:38:45.135 | INFO | "pinned" : " 29 GB" | 20260506 09:38:45.135 | INFO | } | 20260506 09:38:45.135 | INFO | Perf report for the last 5.13924 seconds: | 20260506 09:38:45.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:38:45.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:45.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:45.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:45.135 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:38:45.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:45.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:45.136 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 09:38:45.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:45.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:45.136 | INFO | Uploaded 36GiB to GPU in 1.60621s (22.4131GiB/s) | 20260506 09:38:45.136 | INFO | Downloaded 34GiB to GPU in 1.52372s (22.3138GiB/s) | 20260506 09:38:50.275 | INFO | Analyzing chunk frames = [342016,342528) | 20260506 09:38:50.275 | INFO | MemInfo: { | 20260506 09:38:50.275 | INFO | "gpu" : " 17 GB", | 20260506 09:38:50.275 | INFO | "huge" : "", | 20260506 09:38:50.275 | INFO | "huge_shared" : " 27 GB", | 20260506 09:38:50.275 | INFO | "malloc" : " 14 GB", | 20260506 09:38:50.275 | INFO | "pinned" : " 29 GB" | 20260506 09:38:50.275 | INFO | } | 20260506 09:38:50.275 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:38:50.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:38:50.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:50.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:50.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:50.275 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:38:50.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:50.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:50.275 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:38:50.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:50.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:50.275 | INFO | Uploaded 36GiB to GPU in 1.60427s (22.4401GiB/s) | 20260506 09:38:50.275 | INFO | Downloaded 34GiB to GPU in 1.52078s (22.3569GiB/s) | 20260506 09:38:55.414 | INFO | Analyzing chunk frames = [342528,343040) | 20260506 09:38:55.414 | INFO | MemInfo: { | 20260506 09:38:55.414 | INFO | "gpu" : " 17 GB", | 20260506 09:38:55.414 | INFO | "huge" : "", | 20260506 09:38:55.414 | INFO | "huge_shared" : " 27 GB", | 20260506 09:38:55.414 | INFO | "malloc" : " 14 GB", | 20260506 09:38:55.414 | INFO | "pinned" : " 29 GB" | 20260506 09:38:55.414 | INFO | } | 20260506 09:38:55.414 | INFO | Perf report for the last 5.13948 seconds: | 20260506 09:38:55.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:38:55.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:55.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:55.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:55.414 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:38:55.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:55.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:55.414 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:38:55.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:55.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:38:55.414 | INFO | Uploaded 36GiB to GPU in 1.60473s (22.4337GiB/s) | 20260506 09:38:55.415 | INFO | Downloaded 34GiB to GPU in 1.52149s (22.3466GiB/s) | 20260506 09:39:00.553 | INFO | Analyzing chunk frames = [343040,343552) | 20260506 09:39:00.553 | INFO | MemInfo: { | 20260506 09:39:00.553 | INFO | "gpu" : " 17 GB", | 20260506 09:39:00.553 | INFO | "huge" : "", | 20260506 09:39:00.553 | INFO | "huge_shared" : " 27 GB", | 20260506 09:39:00.553 | INFO | "malloc" : " 14 GB", | 20260506 09:39:00.553 | INFO | "pinned" : " 29 GB" | 20260506 09:39:00.553 | INFO | } | 20260506 09:39:00.553 | INFO | Perf report for the last 5.13915 seconds: | 20260506 09:39:00.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:39:00.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:00.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:00.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:00.553 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:39:00.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:00.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:00.553 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:39:00.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:00.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:00.554 | INFO | Uploaded 36GiB to GPU in 1.60462s (22.4352GiB/s) | 20260506 09:39:00.554 | INFO | Downloaded 34GiB to GPU in 1.52208s (22.3378GiB/s) | 20260506 09:39:05.693 | INFO | Analyzing chunk frames = [343552,344064) | 20260506 09:39:05.693 | INFO | MemInfo: { | 20260506 09:39:05.693 | INFO | "gpu" : " 17 GB", | 20260506 09:39:05.693 | INFO | "huge" : "", | 20260506 09:39:05.693 | INFO | "huge_shared" : " 27 GB", | 20260506 09:39:05.693 | INFO | "malloc" : " 14 GB", | 20260506 09:39:05.693 | INFO | "pinned" : " 29 GB" | 20260506 09:39:05.693 | INFO | } | 20260506 09:39:05.693 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:39:05.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:39:05.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:05.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:05.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:05.693 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:39:05.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:05.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:05.693 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:39:05.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:05.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:05.693 | INFO | Uploaded 36GiB to GPU in 1.60524s (22.4266GiB/s) | 20260506 09:39:05.693 | INFO | Downloaded 34GiB to GPU in 1.52323s (22.321GiB/s) | 20260506 09:39:05.693 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:39:05.693 | INFO | Total mem usage: 15.7 GiB | 20260506 09:39:05.693 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 2.56%) | 20260506 09:39:05.693 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -2.43%) | 20260506 09:39:05.693 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:39:05.693 | INFO | High Water Breakdown: | 20260506 09:39:05.693 | INFO | 59.7% : 9.18 GiB ( 97.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:39:05.693 | INFO | 40.2% : 6.18 GiB ( 92.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:39:05.693 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:39:05.693 | INFO | ------------------------------------------ | 20260506 09:39:05.693 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:39:05.693 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:39:05.693 | INFO | Total mem usage: 26.5 GiB | 20260506 09:39:05.693 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:39:05.693 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:39:05.693 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:39:05.693 | INFO | High Water Breakdown: | 20260506 09:39:05.693 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:39:05.693 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:39:05.693 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:39:05.693 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:39:05.693 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:39:05.693 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:39:05.693 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:39:05.693 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:39:05.693 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:39:05.693 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:39:05.693 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:39:05.693 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:39:05.693 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:39:05.693 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:39:05.693 | INFO | ------------------------------------------ | 20260506 09:39:05.693 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:39:05.693 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:39:05.693 | INFO | Total mem usage: 25.6 GiB | 20260506 09:39:05.693 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:39:05.693 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:39:05.693 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:39:05.693 | INFO | High Water Breakdown: | 20260506 09:39:05.693 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:39:05.693 | INFO | ------------------------------------------ | 20260506 09:39:05.693 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:39:05.693 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:39:05.693 | INFO | Total mem usage: 15.7 GiB | 20260506 09:39:05.693 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:39:05.694 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:39:05.694 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:39:05.694 | INFO | High Water Breakdown: | 20260506 09:39:05.694 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:39:05.694 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:39:05.694 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:39:05.694 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:39:05.694 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:39:05.694 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:39:05.694 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:39:05.694 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:39:05.694 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:39:05.694 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:39:05.694 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:39:05.694 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:39:05.694 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:39:05.694 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:39:05.694 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:39:05.694 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:39:05.694 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:39:05.694 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:39:05.694 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:39:05.694 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:39:05.694 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:39:05.694 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:39:05.694 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:39:05.694 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:39:05.694 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:39:05.694 | INFO | ------------------------------------------ | 20260506 09:39:05.694 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:39:05.694 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:39:05.694 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:39:05.694 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:39:05.694 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 09:39:05.694 | INFO | PerfReport: 0.29%, AggregateStats -- 8.61e-02 / -nan, 4.26e-03 / 0.00e+00 | 20260506 09:39:05.694 | INFO | PerfReport: 10.90%, Baseline -- 3.29e+00 / -nan, 1.75e-02 / 0.00e+00 | 20260506 09:39:05.694 | INFO | PerfReport: 3.05%, Histogram -- 9.21e-01 / -nan, 5.24e-02 / 0.00e+00 | 20260506 09:39:05.694 | INFO | PerfReport: 16.04%, DME -- 3.88e+01 / -nan, 3.14e+00 / 0.00e+00 | 20260506 09:39:05.694 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.72e+01 / -nan, 4.23e-02 / 0.00e+00 | 20260506 09:39:05.694 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.26e+00 / -nan, 1.35e-01 / 0.00e+00 | 20260506 09:39:05.694 | INFO | PerfReport: 3.81%, Metrics -- 1.15e+00 / -nan, 1.79e-01 / 0.00e+00 | 20260506 09:39:05.694 | INFO | PerfReport: 1.48%, PulseSerializing -- 4.48e-01 / -nan, 6.48e-02 / 0.00e+00 | 20260506 09:39:05.694 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:39:05.694 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:39:05.694 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:39:05.694 | INFO | PerfReport: Full Execution -- 9.03e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 09:39:05.694 | INFO | PerfReport: 18.59%, Upload -- 1.68e+01 / 0.00e+00, 4.56e-01 / 0.00e+00 | 20260506 09:39:05.694 | INFO | PerfReport: 17.85%, Download -- 1.61e+01 / 0.00e+00, 7.97e-01 / 0.00e+00 | 20260506 09:39:05.694 | INFO | PerfReport: 33.46%, Compute -- 3.02e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 09:39:05.694 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 09:39:05.694 | INFO | PerfReport: 0.00%, LockDownload -- 5.00e-04 / 0.00e+00, 1.83e-04 / 0.00e+00 | 20260506 09:39:05.694 | INFO | PerfReport: 29.53%, LockCompute -- 2.67e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 09:39:05.694 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 2.38e-02 / 0.00e+00 | 20260506 09:39:05.792 | INFO | Starting a round of Baz IO | 20260506 09:39:08.712 | INFO | Populating RTMetrics report | 20260506 09:39:08.712 | INFO | RTMetrics report populated | 20260506 09:39:08.712 | INFO | Opening tmp RTMetrics json file | 20260506 09:39:08.730 | INFO | Serializing RTMetrics to json | 20260506 09:39:08.735 | INFO | Serialization done, swapping tmp file now... | 20260506 09:39:08.740 | INFO | RTMetrics done | 20260506 09:39:08.984 | INFO | Finished a round of Baz IO: Wrote 1.879 GiB in 3.191 seconds | 20260506 09:39:08.984 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.63% / 0.09% / 47.28% | 20260506 09:39:08.984 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.86982% / 0.000282129% / 5.16574e-05% | 20260506 09:39:08.984 | INFO | Baz IO overall: | 20260506 09:39:08.984 | INFO | BytesWritten : 137879719344 | 20260506 09:39:08.984 | INFO | NumEvents : 65539592837 | 20260506 09:39:10.832 | INFO | Analyzing chunk frames = [344064,344576) | 20260506 09:39:10.832 | INFO | MemInfo: { | 20260506 09:39:10.832 | INFO | "gpu" : " 17 GB", | 20260506 09:39:10.832 | INFO | "huge" : "", | 20260506 09:39:10.832 | INFO | "huge_shared" : " 27 GB", | 20260506 09:39:10.832 | INFO | "malloc" : " 14 GB", | 20260506 09:39:10.832 | INFO | "pinned" : " 29 GB" | 20260506 09:39:10.832 | INFO | } | 20260506 09:39:10.832 | INFO | Perf report for the last 5.1392 seconds: | 20260506 09:39:10.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:39:10.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:10.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:10.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:10.832 | INFO | ANALYSIS : 57.8%, 2.8, 85.4ms, 0 | 20260506 09:39:10.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:10.832 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 09:39:10.832 | INFO | PRE_HQ : 57.0%, 2.1, 61.4ms, 0 | 20260506 09:39:10.832 | INFO | BAZWRITER : 61.8%, 4.8, 151.4ms, 0 | 20260506 09:39:10.832 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 09:39:10.832 | INFO | Uploaded 36GiB to GPU in 1.67298s (21.5185GiB/s) | 20260506 09:39:10.833 | INFO | Downloaded 38GiB to GPU in 1.7386s (21.8567GiB/s) | 20260506 09:39:15.971 | INFO | Analyzing chunk frames = [344576,345088) | 20260506 09:39:15.971 | INFO | MemInfo: { | 20260506 09:39:15.971 | INFO | "gpu" : " 17 GB", | 20260506 09:39:15.971 | INFO | "huge" : "", | 20260506 09:39:15.971 | INFO | "huge_shared" : " 27 GB", | 20260506 09:39:15.971 | INFO | "malloc" : " 14 GB", | 20260506 09:39:15.972 | INFO | "pinned" : " 29 GB" | 20260506 09:39:15.972 | INFO | } | 20260506 09:39:15.972 | INFO | Perf report for the last 5.13952 seconds: | 20260506 09:39:15.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:39:15.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:15.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:15.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:15.972 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260506 09:39:15.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:15.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:15.972 | INFO | PRE_HQ : 30.1%, 1.0, 15.9ms, 0 | 20260506 09:39:15.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:15.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:15.972 | INFO | Uploaded 36GiB to GPU in 1.60223s (22.4687GiB/s) | 20260506 09:39:15.972 | INFO | Downloaded 34GiB to GPU in 1.51944s (22.3767GiB/s) | 20260506 09:39:21.111 | INFO | Analyzing chunk frames = [345088,345600) | 20260506 09:39:21.111 | INFO | MemInfo: { | 20260506 09:39:21.111 | INFO | "gpu" : " 17 GB", | 20260506 09:39:21.111 | INFO | "huge" : "", | 20260506 09:39:21.111 | INFO | "huge_shared" : " 27 GB", | 20260506 09:39:21.111 | INFO | "malloc" : " 14 GB", | 20260506 09:39:21.111 | INFO | "pinned" : " 29 GB" | 20260506 09:39:21.111 | INFO | } | 20260506 09:39:21.111 | INFO | Perf report for the last 5.13938 seconds: | 20260506 09:39:21.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:39:21.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:21.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:21.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:21.111 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:39:21.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:21.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:21.111 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:39:21.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:21.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:21.111 | INFO | Uploaded 36GiB to GPU in 1.60599s (22.416GiB/s) | 20260506 09:39:21.111 | INFO | Downloaded 34GiB to GPU in 1.52385s (22.3119GiB/s) | 20260506 09:39:26.250 | INFO | Analyzing chunk frames = [345600,346112) | 20260506 09:39:26.250 | INFO | MemInfo: { | 20260506 09:39:26.250 | INFO | "gpu" : " 17 GB", | 20260506 09:39:26.250 | INFO | "huge" : "", | 20260506 09:39:26.250 | INFO | "huge_shared" : " 27 GB", | 20260506 09:39:26.250 | INFO | "malloc" : " 14 GB", | 20260506 09:39:26.250 | INFO | "pinned" : " 29 GB" | 20260506 09:39:26.250 | INFO | } | 20260506 09:39:26.250 | INFO | Perf report for the last 5.13926 seconds: | 20260506 09:39:26.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:39:26.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:26.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:26.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:26.250 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:39:26.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:26.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:26.250 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:39:26.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:26.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:26.250 | INFO | Uploaded 36GiB to GPU in 1.60576s (22.4193GiB/s) | 20260506 09:39:26.251 | INFO | Downloaded 34GiB to GPU in 1.5233s (22.32GiB/s) | 20260506 09:39:31.389 | INFO | Analyzing chunk frames = [346112,346624) | 20260506 09:39:31.389 | INFO | MemInfo: { | 20260506 09:39:31.389 | INFO | "gpu" : " 17 GB", | 20260506 09:39:31.389 | INFO | "huge" : "", | 20260506 09:39:31.390 | INFO | "huge_shared" : " 27 GB", | 20260506 09:39:31.390 | INFO | "malloc" : " 14 GB", | 20260506 09:39:31.390 | INFO | "pinned" : " 29 GB" | 20260506 09:39:31.390 | INFO | } | 20260506 09:39:31.390 | INFO | Perf report for the last 5.13935 seconds: | 20260506 09:39:31.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:39:31.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:31.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:31.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:31.390 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 09:39:31.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:31.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:31.390 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:39:31.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:31.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:31.390 | INFO | Uploaded 36GiB to GPU in 1.60496s (22.4304GiB/s) | 20260506 09:39:31.390 | INFO | Downloaded 34GiB to GPU in 1.52256s (22.3309GiB/s) | 20260506 09:39:36.529 | INFO | Analyzing chunk frames = [346624,347136) | 20260506 09:39:36.529 | INFO | MemInfo: { | 20260506 09:39:36.529 | INFO | "gpu" : " 17 GB", | 20260506 09:39:36.529 | INFO | "huge" : "", | 20260506 09:39:36.529 | INFO | "huge_shared" : " 27 GB", | 20260506 09:39:36.529 | INFO | "malloc" : " 14 GB", | 20260506 09:39:36.529 | INFO | "pinned" : " 29 GB" | 20260506 09:39:36.529 | INFO | } | 20260506 09:39:36.529 | INFO | Perf report for the last 5.13946 seconds: | 20260506 09:39:36.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:39:36.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:36.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:36.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:36.529 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 09:39:36.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:36.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:36.529 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:39:36.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:36.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:36.529 | INFO | Uploaded 36GiB to GPU in 1.60432s (22.4394GiB/s) | 20260506 09:39:36.529 | INFO | Downloaded 34GiB to GPU in 1.52102s (22.3534GiB/s) | 20260506 09:39:41.668 | INFO | Analyzing chunk frames = [347136,347648) | 20260506 09:39:41.668 | INFO | MemInfo: { | 20260506 09:39:41.668 | INFO | "gpu" : " 17 GB", | 20260506 09:39:41.668 | INFO | "huge" : "", | 20260506 09:39:41.668 | INFO | "huge_shared" : " 27 GB", | 20260506 09:39:41.668 | INFO | "malloc" : " 14 GB", | 20260506 09:39:41.668 | INFO | "pinned" : " 29 GB" | 20260506 09:39:41.668 | INFO | } | 20260506 09:39:41.668 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:39:41.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:39:41.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:41.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:41.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:41.669 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:39:41.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:41.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:41.669 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:39:41.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:41.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:41.669 | INFO | Uploaded 36GiB to GPU in 1.60398s (22.4442GiB/s) | 20260506 09:39:41.669 | INFO | Downloaded 34GiB to GPU in 1.52291s (22.3257GiB/s) | 20260506 09:39:46.808 | INFO | Analyzing chunk frames = [347648,348160) | 20260506 09:39:46.808 | INFO | MemInfo: { | 20260506 09:39:46.808 | INFO | "gpu" : " 17 GB", | 20260506 09:39:46.808 | INFO | "huge" : "", | 20260506 09:39:46.808 | INFO | "huge_shared" : " 27 GB", | 20260506 09:39:46.808 | INFO | "malloc" : " 14 GB", | 20260506 09:39:46.808 | INFO | "pinned" : " 29 GB" | 20260506 09:39:46.808 | INFO | } | 20260506 09:39:46.808 | INFO | Perf report for the last 5.13956 seconds: | 20260506 09:39:46.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:39:46.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:46.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:46.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:46.808 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:39:46.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:46.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:46.808 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:39:46.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:46.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:46.809 | INFO | Uploaded 36GiB to GPU in 1.60525s (22.4264GiB/s) | 20260506 09:39:46.809 | INFO | Downloaded 34GiB to GPU in 1.52312s (22.3225GiB/s) | 20260506 09:39:46.906 | INFO | Starting a round of Baz IO | 20260506 09:39:49.824 | INFO | Populating RTMetrics report | 20260506 09:39:49.825 | INFO | RTMetrics report populated | 20260506 09:39:49.825 | INFO | Opening tmp RTMetrics json file | 20260506 09:39:49.868 | INFO | Serializing RTMetrics to json | 20260506 09:39:49.897 | INFO | Serialization done, swapping tmp file now... | 20260506 09:39:49.908 | INFO | RTMetrics done | 20260506 09:39:50.134 | INFO | Finished a round of Baz IO: Wrote 1.882 GiB in 3.228 seconds | 20260506 09:39:50.134 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.54% / 0.09% / 47.37% | 20260506 09:39:50.134 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.85296% / 0.000270208% / 3.97364e-05% | 20260506 09:39:50.134 | INFO | Baz IO overall: | 20260506 09:39:50.134 | INFO | BytesWritten : 139900386877 | 20260506 09:39:50.134 | INFO | NumEvents : 66520832957 | 20260506 09:39:51.947 | INFO | Analyzing chunk frames = [348160,348672) | 20260506 09:39:51.947 | INFO | MemInfo: { | 20260506 09:39:51.947 | INFO | "gpu" : " 17 GB", | 20260506 09:39:51.947 | INFO | "huge" : "", | 20260506 09:39:51.947 | INFO | "huge_shared" : " 27 GB", | 20260506 09:39:51.947 | INFO | "malloc" : " 14 GB", | 20260506 09:39:51.947 | INFO | "pinned" : " 29 GB" | 20260506 09:39:51.947 | INFO | } | 20260506 09:39:51.947 | INFO | Perf report for the last 5.1391 seconds: | 20260506 09:39:51.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:39:51.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:51.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:51.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:51.947 | INFO | ANALYSIS : 57.7%, 2.9, 87.5ms, 0 | 20260506 09:39:51.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:51.947 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 09:39:51.947 | INFO | PRE_HQ : 56.7%, 2.1, 61.6ms, 0 | 20260506 09:39:51.947 | INFO | BAZWRITER : 62.4%, 4.5, 144.8ms, 0 | 20260506 09:39:51.947 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 09:39:51.947 | INFO | Uploaded 36GiB to GPU in 1.65233s (21.7874GiB/s) | 20260506 09:39:51.947 | INFO | Downloaded 38GiB to GPU in 1.71572s (22.1481GiB/s) | 20260506 09:39:57.086 | INFO | Analyzing chunk frames = [348672,349184) | 20260506 09:39:57.086 | INFO | MemInfo: { | 20260506 09:39:57.086 | INFO | "gpu" : " 17 GB", | 20260506 09:39:57.086 | INFO | "huge" : "", | 20260506 09:39:57.086 | INFO | "huge_shared" : " 27 GB", | 20260506 09:39:57.086 | INFO | "malloc" : " 14 GB", | 20260506 09:39:57.086 | INFO | "pinned" : " 29 GB" | 20260506 09:39:57.086 | INFO | } | 20260506 09:39:57.087 | INFO | Perf report for the last 5.13937 seconds: | 20260506 09:39:57.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:39:57.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:57.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:57.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:57.087 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260506 09:39:57.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:57.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:57.087 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:39:57.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:57.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:39:57.087 | INFO | Uploaded 36GiB to GPU in 1.60193s (22.4728GiB/s) | 20260506 09:39:57.087 | INFO | Downloaded 34GiB to GPU in 1.519s (22.3832GiB/s) | 20260506 09:40:02.226 | INFO | Analyzing chunk frames = [349184,349696) | 20260506 09:40:02.226 | INFO | MemInfo: { | 20260506 09:40:02.226 | INFO | "gpu" : " 17 GB", | 20260506 09:40:02.226 | INFO | "huge" : "", | 20260506 09:40:02.226 | INFO | "huge_shared" : " 27 GB", | 20260506 09:40:02.226 | INFO | "malloc" : " 14 GB", | 20260506 09:40:02.226 | INFO | "pinned" : " 29 GB" | 20260506 09:40:02.226 | INFO | } | 20260506 09:40:02.226 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:40:02.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:40:02.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:02.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:02.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:02.226 | INFO | ANALYSIS : 56.9%, 3.0, 88.5ms, 0 | 20260506 09:40:02.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:02.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:02.226 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:40:02.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:02.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:02.226 | INFO | Uploaded 36GiB to GPU in 1.60648s (22.4092GiB/s) | 20260506 09:40:02.226 | INFO | Downloaded 34GiB to GPU in 1.52432s (22.305GiB/s) | 20260506 09:40:07.365 | INFO | Analyzing chunk frames = [349696,350208) | 20260506 09:40:07.365 | INFO | MemInfo: { | 20260506 09:40:07.365 | INFO | "gpu" : " 17 GB", | 20260506 09:40:07.365 | INFO | "huge" : "", | 20260506 09:40:07.365 | INFO | "huge_shared" : " 27 GB", | 20260506 09:40:07.365 | INFO | "malloc" : " 14 GB", | 20260506 09:40:07.365 | INFO | "pinned" : " 29 GB" | 20260506 09:40:07.365 | INFO | } | 20260506 09:40:07.365 | INFO | Perf report for the last 5.13957 seconds: | 20260506 09:40:07.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:40:07.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:07.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:07.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:07.365 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:40:07.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:07.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:07.366 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:40:07.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:07.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:07.366 | INFO | Uploaded 36GiB to GPU in 1.60506s (22.4291GiB/s) | 20260506 09:40:07.366 | INFO | Downloaded 34GiB to GPU in 1.52291s (22.3257GiB/s) | 20260506 09:40:12.520 | INFO | Analyzing chunk frames = [350208,350720) | 20260506 09:40:12.520 | INFO | MemInfo: { | 20260506 09:40:12.520 | INFO | "gpu" : " 17 GB", | 20260506 09:40:12.520 | INFO | "huge" : "", | 20260506 09:40:12.520 | INFO | "huge_shared" : " 27 GB", | 20260506 09:40:12.520 | INFO | "malloc" : " 14 GB", | 20260506 09:40:12.520 | INFO | "pinned" : " 29 GB" | 20260506 09:40:12.520 | INFO | } | 20260506 09:40:12.520 | INFO | Perf report for the last 5.1545 seconds: | 20260506 09:40:12.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:40:12.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:12.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:12.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:12.520 | INFO | ANALYSIS : 56.4%, 3.0, 88.2ms, 0 | 20260506 09:40:12.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:12.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:12.520 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:40:12.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:12.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:12.520 | INFO | Uploaded 36GiB to GPU in 1.60393s (22.4449GiB/s) | 20260506 09:40:12.520 | INFO | Downloaded 34GiB to GPU in 1.52089s (22.3553GiB/s) | 20260506 09:40:17.644 | INFO | Analyzing chunk frames = [350720,351232) | 20260506 09:40:17.644 | INFO | MemInfo: { | 20260506 09:40:17.644 | INFO | "gpu" : " 17 GB", | 20260506 09:40:17.644 | INFO | "huge" : "", | 20260506 09:40:17.644 | INFO | "huge_shared" : " 27 GB", | 20260506 09:40:17.644 | INFO | "malloc" : " 14 GB", | 20260506 09:40:17.644 | INFO | "pinned" : " 29 GB" | 20260506 09:40:17.644 | INFO | } | 20260506 09:40:17.644 | INFO | Perf report for the last 5.12381 seconds: | 20260506 09:40:17.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:40:17.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:17.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:17.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:17.644 | INFO | ANALYSIS : 57.0%, 3.0, 88.7ms, 0 | 20260506 09:40:17.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:17.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:17.644 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:40:17.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:17.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:17.644 | INFO | Uploaded 36GiB to GPU in 1.60341s (22.4521GiB/s) | 20260506 09:40:17.644 | INFO | Downloaded 34GiB to GPU in 1.52166s (22.344GiB/s) | 20260506 09:40:22.783 | INFO | Analyzing chunk frames = [351232,351744) | 20260506 09:40:22.783 | INFO | MemInfo: { | 20260506 09:40:22.783 | INFO | "gpu" : " 17 GB", | 20260506 09:40:22.783 | INFO | "huge" : "", | 20260506 09:40:22.783 | INFO | "huge_shared" : " 27 GB", | 20260506 09:40:22.783 | INFO | "malloc" : " 14 GB", | 20260506 09:40:22.783 | INFO | "pinned" : " 29 GB" | 20260506 09:40:22.783 | INFO | } | 20260506 09:40:22.783 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:40:22.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:40:22.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:22.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:22.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:22.783 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:40:22.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:22.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:22.783 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:40:22.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:22.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:22.783 | INFO | Uploaded 36GiB to GPU in 1.60586s (22.4179GiB/s) | 20260506 09:40:22.783 | INFO | Downloaded 34GiB to GPU in 1.52371s (22.3139GiB/s) | 20260506 09:40:27.922 | INFO | Analyzing chunk frames = [351744,352256) | 20260506 09:40:27.922 | INFO | MemInfo: { | 20260506 09:40:27.922 | INFO | "gpu" : " 17 GB", | 20260506 09:40:27.922 | INFO | "huge" : "", | 20260506 09:40:27.922 | INFO | "huge_shared" : " 27 GB", | 20260506 09:40:27.922 | INFO | "malloc" : " 14 GB", | 20260506 09:40:27.922 | INFO | "pinned" : " 29 GB" | 20260506 09:40:27.922 | INFO | } | 20260506 09:40:27.923 | INFO | Perf report for the last 5.13954 seconds: | 20260506 09:40:27.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:40:27.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:27.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:27.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:27.923 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:40:27.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:27.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:27.923 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 09:40:27.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:27.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:27.923 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:40:27.923 | INFO | Total mem usage: 15.7 GiB | 20260506 09:40:27.923 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 2.56%) | 20260506 09:40:27.923 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -3.12%) | 20260506 09:40:27.923 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:40:27.923 | INFO | High Water Breakdown: | 20260506 09:40:27.923 | INFO | 59.6% : 9.1 GiB ( 96.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:40:27.923 | INFO | 40.3% : 6.15 GiB ( 91.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:40:27.923 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:40:27.923 | INFO | ------------------------------------------ | 20260506 09:40:27.923 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:40:27.923 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:40:27.923 | INFO | Total mem usage: 26.5 GiB | 20260506 09:40:27.923 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:40:27.923 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:40:27.923 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:40:27.923 | INFO | High Water Breakdown: | 20260506 09:40:27.923 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:40:27.923 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:40:27.923 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:40:27.923 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:40:27.923 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:40:27.923 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:40:27.923 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:40:27.923 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:40:27.923 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:40:27.923 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:40:27.923 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:40:27.923 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:40:27.923 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:40:27.923 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:40:27.923 | INFO | ------------------------------------------ | 20260506 09:40:27.923 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:40:27.923 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:40:27.923 | INFO | Total mem usage: 25.6 GiB | 20260506 09:40:27.923 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:40:27.923 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:40:27.923 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:40:27.923 | INFO | High Water Breakdown: | 20260506 09:40:27.923 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:40:27.923 | INFO | ------------------------------------------ | 20260506 09:40:27.923 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:40:27.923 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:40:27.923 | INFO | Total mem usage: 15.7 GiB | 20260506 09:40:27.923 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:40:27.923 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:40:27.923 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:40:27.923 | INFO | High Water Breakdown: | 20260506 09:40:27.923 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:40:27.923 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:40:27.923 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:40:27.923 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:40:27.923 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:40:27.923 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:40:27.923 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:40:27.923 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:40:27.923 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:40:27.923 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:40:27.923 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:40:27.923 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:40:27.923 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:40:27.923 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:40:27.923 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:40:27.923 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:40:27.923 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:40:27.923 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:40:27.923 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:40:27.923 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:40:27.923 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:40:27.923 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:40:27.923 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:40:27.923 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:40:27.923 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:40:27.923 | INFO | ------------------------------------------ | 20260506 09:40:27.923 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:40:27.923 | INFO | Uploaded 36GiB to GPU in 1.60411s (22.4424GiB/s) | 20260506 09:40:27.923 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:40:27.923 | INFO | Downloaded 34GiB to GPU in 1.52215s (22.3368GiB/s) | 20260506 09:40:27.923 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:40:27.923 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:40:27.923 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 09:40:27.923 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / -nan, 4.42e-03 / 0.00e+00 | 20260506 09:40:27.923 | INFO | PerfReport: 10.90%, Baseline -- 3.29e+00 / -nan, 1.81e-02 / 0.00e+00 | 20260506 09:40:27.923 | INFO | PerfReport: 3.05%, Histogram -- 9.21e-01 / -nan, 5.19e-02 / 0.00e+00 | 20260506 09:40:27.923 | INFO | PerfReport: 16.04%, DME -- 3.88e+01 / -nan, 3.09e+00 / 0.00e+00 | 20260506 09:40:27.923 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.72e+01 / -nan, 4.23e-02 / 0.00e+00 | 20260506 09:40:27.923 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.26e+00 / -nan, 1.35e-01 / 0.00e+00 | 20260506 09:40:27.923 | INFO | PerfReport: 3.80%, Metrics -- 1.15e+00 / -nan, 1.79e-01 / 0.00e+00 | 20260506 09:40:27.923 | INFO | PerfReport: 1.48%, PulseSerializing -- 4.47e-01 / -nan, 6.46e-02 / 0.00e+00 | 20260506 09:40:27.923 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:40:27.923 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:40:27.923 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:40:27.923 | INFO | PerfReport: Full Execution -- 9.03e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 09:40:27.923 | INFO | PerfReport: 18.60%, Upload -- 1.68e+01 / 0.00e+00, 4.60e-01 / 0.00e+00 | 20260506 09:40:27.923 | INFO | PerfReport: 17.86%, Download -- 1.61e+01 / 0.00e+00, 8.02e-01 / 0.00e+00 | 20260506 09:40:27.923 | INFO | PerfReport: 33.46%, Compute -- 3.02e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 09:40:27.923 | INFO | PerfReport: 0.56%, LockUpload -- 5.09e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260506 09:40:27.923 | INFO | PerfReport: 0.00%, LockDownload -- 4.80e-04 / 0.00e+00, 1.22e-04 / 0.00e+00 | 20260506 09:40:27.923 | INFO | PerfReport: 29.50%, LockCompute -- 2.66e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260506 09:40:27.923 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260506 09:40:28.020 | INFO | Starting a round of Baz IO | 20260506 09:40:30.944 | INFO | Populating RTMetrics report | 20260506 09:40:30.944 | INFO | RTMetrics report populated | 20260506 09:40:30.944 | INFO | Opening tmp RTMetrics json file | 20260506 09:40:30.997 | INFO | Serializing RTMetrics to json | 20260506 09:40:31.009 | INFO | Serialization done, swapping tmp file now... | 20260506 09:40:31.013 | INFO | RTMetrics done | 20260506 09:40:31.157 | INFO | Finished a round of Baz IO: Wrote 1.882 GiB in 3.136 seconds | 20260506 09:40:31.157 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.46% / 0.09% / 47.46% | 20260506 09:40:31.157 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.79516% / 0.000258287% / 5.16574e-05% | 20260506 09:40:31.157 | INFO | Baz IO overall: | 20260506 09:40:31.157 | INFO | BytesWritten : 141921337632 | 20260506 09:40:31.157 | INFO | NumEvents : 67501850636 | 20260506 09:40:33.062 | INFO | Analyzing chunk frames = [352256,352768) | 20260506 09:40:33.062 | INFO | MemInfo: { | 20260506 09:40:33.062 | INFO | "gpu" : " 17 GB", | 20260506 09:40:33.062 | INFO | "huge" : "", | 20260506 09:40:33.062 | INFO | "huge_shared" : " 27 GB", | 20260506 09:40:33.062 | INFO | "malloc" : " 14 GB", | 20260506 09:40:33.062 | INFO | "pinned" : " 29 GB" | 20260506 09:40:33.062 | INFO | } | 20260506 09:40:33.062 | INFO | Perf report for the last 5.1391 seconds: | 20260506 09:40:33.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:40:33.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:33.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:33.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:33.062 | INFO | ANALYSIS : 57.8%, 2.8, 85.2ms, 0 | 20260506 09:40:33.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:33.062 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260506 09:40:33.062 | INFO | PRE_HQ : 57.0%, 2.1, 61.6ms, 0 | 20260506 09:40:33.062 | INFO | BAZWRITER : 60.4%, 4.8, 148.7ms, 0 | 20260506 09:40:33.062 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 09:40:33.062 | INFO | Uploaded 36GiB to GPU in 1.66783s (21.5849GiB/s) | 20260506 09:40:33.062 | INFO | Downloaded 38GiB to GPU in 1.73428s (21.9112GiB/s) | 20260506 09:40:38.201 | INFO | Analyzing chunk frames = [352768,353280) | 20260506 09:40:38.201 | INFO | MemInfo: { | 20260506 09:40:38.201 | INFO | "gpu" : " 17 GB", | 20260506 09:40:38.201 | INFO | "huge" : "", | 20260506 09:40:38.201 | INFO | "huge_shared" : " 27 GB", | 20260506 09:40:38.201 | INFO | "malloc" : " 14 GB", | 20260506 09:40:38.201 | INFO | "pinned" : " 29 GB" | 20260506 09:40:38.201 | INFO | } | 20260506 09:40:38.204 | INFO | Perf report for the last 5.14254 seconds: | 20260506 09:40:38.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:40:38.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:38.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:38.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:38.204 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260506 09:40:38.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:38.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:38.204 | INFO | PRE_HQ : 30.3%, 1.0, 15.9ms, 0 | 20260506 09:40:38.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:38.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:38.205 | INFO | Uploaded 36GiB to GPU in 1.60207s (22.4709GiB/s) | 20260506 09:40:38.205 | INFO | Downloaded 34GiB to GPU in 1.51935s (22.378GiB/s) | 20260506 09:40:43.340 | INFO | Analyzing chunk frames = [353280,353792) | 20260506 09:40:43.340 | INFO | MemInfo: { | 20260506 09:40:43.340 | INFO | "gpu" : " 17 GB", | 20260506 09:40:43.340 | INFO | "huge" : "", | 20260506 09:40:43.340 | INFO | "huge_shared" : " 27 GB", | 20260506 09:40:43.340 | INFO | "malloc" : " 14 GB", | 20260506 09:40:43.340 | INFO | "pinned" : " 29 GB" | 20260506 09:40:43.340 | INFO | } | 20260506 09:40:43.340 | INFO | Perf report for the last 5.13613 seconds: | 20260506 09:40:43.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:40:43.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:43.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:43.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:43.341 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:40:43.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:43.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:43.341 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:40:43.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:43.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:43.341 | INFO | Uploaded 36GiB to GPU in 1.60604s (22.4153GiB/s) | 20260506 09:40:43.341 | INFO | Downloaded 34GiB to GPU in 1.52434s (22.3047GiB/s) | 20260506 09:40:48.480 | INFO | Analyzing chunk frames = [353792,354304) | 20260506 09:40:48.480 | INFO | MemInfo: { | 20260506 09:40:48.480 | INFO | "gpu" : " 17 GB", | 20260506 09:40:48.480 | INFO | "huge" : "", | 20260506 09:40:48.480 | INFO | "huge_shared" : " 27 GB", | 20260506 09:40:48.480 | INFO | "malloc" : " 14 GB", | 20260506 09:40:48.480 | INFO | "pinned" : " 29 GB" | 20260506 09:40:48.480 | INFO | } | 20260506 09:40:48.480 | INFO | Perf report for the last 5.13949 seconds: | 20260506 09:40:48.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:40:48.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:48.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:48.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:48.480 | INFO | ANALYSIS : 56.7%, 3.0, 87.9ms, 0 | 20260506 09:40:48.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:48.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:48.480 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:40:48.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:48.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:48.480 | INFO | Uploaded 36GiB to GPU in 1.60586s (22.418GiB/s) | 20260506 09:40:48.480 | INFO | Downloaded 34GiB to GPU in 1.52335s (22.3192GiB/s) | 20260506 09:40:53.619 | INFO | Analyzing chunk frames = [354304,354816) | 20260506 09:40:53.619 | INFO | MemInfo: { | 20260506 09:40:53.619 | INFO | "gpu" : " 17 GB", | 20260506 09:40:53.619 | INFO | "huge" : "", | 20260506 09:40:53.619 | INFO | "huge_shared" : " 27 GB", | 20260506 09:40:53.619 | INFO | "malloc" : " 14 GB", | 20260506 09:40:53.619 | INFO | "pinned" : " 29 GB" | 20260506 09:40:53.619 | INFO | } | 20260506 09:40:53.619 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:40:53.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:40:53.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:53.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:53.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:53.619 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:40:53.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:53.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:53.620 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:40:53.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:53.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:53.620 | INFO | Uploaded 36GiB to GPU in 1.60375s (22.4474GiB/s) | 20260506 09:40:53.620 | INFO | Downloaded 34GiB to GPU in 1.52024s (22.3649GiB/s) | 20260506 09:40:58.758 | INFO | Analyzing chunk frames = [354816,355328) | 20260506 09:40:58.758 | INFO | MemInfo: { | 20260506 09:40:58.758 | INFO | "gpu" : " 17 GB", | 20260506 09:40:58.758 | INFO | "huge" : "", | 20260506 09:40:58.758 | INFO | "huge_shared" : " 27 GB", | 20260506 09:40:58.758 | INFO | "malloc" : " 14 GB", | 20260506 09:40:58.758 | INFO | "pinned" : " 29 GB" | 20260506 09:40:58.758 | INFO | } | 20260506 09:40:58.758 | INFO | Perf report for the last 5.13901 seconds: | 20260506 09:40:58.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:40:58.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:58.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:58.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:58.759 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:40:58.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:58.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:58.759 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:40:58.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:58.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:40:58.759 | INFO | Uploaded 36GiB to GPU in 1.60388s (22.4456GiB/s) | 20260506 09:40:58.759 | INFO | Downloaded 34GiB to GPU in 1.52056s (22.3602GiB/s) | 20260506 09:41:03.898 | INFO | Analyzing chunk frames = [355328,355840) | 20260506 09:41:03.898 | INFO | MemInfo: { | 20260506 09:41:03.898 | INFO | "gpu" : " 17 GB", | 20260506 09:41:03.898 | INFO | "huge" : "", | 20260506 09:41:03.898 | INFO | "huge_shared" : " 27 GB", | 20260506 09:41:03.898 | INFO | "malloc" : " 14 GB", | 20260506 09:41:03.898 | INFO | "pinned" : " 29 GB" | 20260506 09:41:03.898 | INFO | } | 20260506 09:41:03.898 | INFO | Perf report for the last 5.13956 seconds: | 20260506 09:41:03.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:41:03.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:03.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:03.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:03.898 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:41:03.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:03.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:03.898 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:41:03.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:03.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:03.899 | INFO | Uploaded 36GiB to GPU in 1.60435s (22.439GiB/s) | 20260506 09:41:03.899 | INFO | Downloaded 34GiB to GPU in 1.52213s (22.3371GiB/s) | 20260506 09:41:09.037 | INFO | Analyzing chunk frames = [355840,356352) | 20260506 09:41:09.037 | INFO | MemInfo: { | 20260506 09:41:09.037 | INFO | "gpu" : " 17 GB", | 20260506 09:41:09.037 | INFO | "huge" : "", | 20260506 09:41:09.037 | INFO | "huge_shared" : " 27 GB", | 20260506 09:41:09.037 | INFO | "malloc" : " 14 GB", | 20260506 09:41:09.037 | INFO | "pinned" : " 29 GB" | 20260506 09:41:09.037 | INFO | } | 20260506 09:41:09.037 | INFO | Perf report for the last 5.13917 seconds: | 20260506 09:41:09.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:41:09.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:09.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:09.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:09.037 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:41:09.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:09.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:09.037 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 09:41:09.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:09.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:09.038 | INFO | Uploaded 36GiB to GPU in 1.60482s (22.4324GiB/s) | 20260506 09:41:09.038 | INFO | Downloaded 34GiB to GPU in 1.5223s (22.3347GiB/s) | 20260506 09:41:09.135 | INFO | Starting a round of Baz IO | 20260506 09:41:12.054 | INFO | Populating RTMetrics report | 20260506 09:41:12.055 | INFO | RTMetrics report populated | 20260506 09:41:12.055 | INFO | Opening tmp RTMetrics json file | 20260506 09:41:12.056 | INFO | Serializing RTMetrics to json | 20260506 09:41:12.060 | INFO | Serialization done, swapping tmp file now... | 20260506 09:41:12.060 | INFO | RTMetrics done | 20260506 09:41:12.239 | INFO | Finished a round of Baz IO: Wrote 1.888 GiB in 3.104 seconds | 20260506 09:41:12.239 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.37% / 0.09% / 47.54% | 20260506 09:41:12.239 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.79569% / 0.00021855% / 5.5631e-05% | 20260506 09:41:12.239 | INFO | Baz IO overall: | 20260506 09:41:12.239 | INFO | BytesWritten : 143948243008 | 20260506 09:41:12.239 | INFO | NumEvents : 68486329081 | 20260506 09:41:14.176 | INFO | Analyzing chunk frames = [356352,356864) | 20260506 09:41:14.176 | INFO | MemInfo: { | 20260506 09:41:14.176 | INFO | "gpu" : " 17 GB", | 20260506 09:41:14.176 | INFO | "huge" : "", | 20260506 09:41:14.176 | INFO | "huge_shared" : " 27 GB", | 20260506 09:41:14.177 | INFO | "malloc" : " 14 GB", | 20260506 09:41:14.177 | INFO | "pinned" : " 29 GB" | 20260506 09:41:14.177 | INFO | } | 20260506 09:41:14.177 | INFO | Perf report for the last 5.13947 seconds: | 20260506 09:41:14.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:41:14.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:14.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:14.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:14.177 | INFO | ANALYSIS : 57.7%, 2.8, 85.6ms, 0 | 20260506 09:41:14.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:14.177 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 09:41:14.177 | INFO | PRE_HQ : 56.9%, 2.1, 61.1ms, 0 | 20260506 09:41:14.177 | INFO | BAZWRITER : 60.0%, 4.8, 147.2ms, 0 | 20260506 09:41:14.177 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 09:41:14.177 | INFO | Uploaded 36GiB to GPU in 1.67657s (21.4724GiB/s) | 20260506 09:41:14.177 | INFO | Downloaded 38GiB to GPU in 1.74178s (21.8167GiB/s) | 20260506 09:41:19.316 | INFO | Analyzing chunk frames = [356864,357376) | 20260506 09:41:19.316 | INFO | MemInfo: { | 20260506 09:41:19.316 | INFO | "gpu" : " 17 GB", | 20260506 09:41:19.316 | INFO | "huge" : "", | 20260506 09:41:19.316 | INFO | "huge_shared" : " 27 GB", | 20260506 09:41:19.316 | INFO | "malloc" : " 14 GB", | 20260506 09:41:19.316 | INFO | "pinned" : " 29 GB" | 20260506 09:41:19.316 | INFO | } | 20260506 09:41:19.316 | INFO | Perf report for the last 5.13942 seconds: | 20260506 09:41:19.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:41:19.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:19.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:19.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:19.316 | INFO | ANALYSIS : 57.5%, 3.0, 89.8ms, 0 | 20260506 09:41:19.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:19.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:19.316 | INFO | PRE_HQ : 29.8%, 1.0, 15.7ms, 0 | 20260506 09:41:19.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:19.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:19.316 | INFO | Uploaded 36GiB to GPU in 1.60205s (22.4712GiB/s) | 20260506 09:41:19.317 | INFO | Downloaded 34GiB to GPU in 1.51934s (22.3781GiB/s) | 20260506 09:41:24.455 | INFO | Analyzing chunk frames = [357376,357888) | 20260506 09:41:24.455 | INFO | MemInfo: { | 20260506 09:41:24.455 | INFO | "gpu" : " 17 GB", | 20260506 09:41:24.455 | INFO | "huge" : "", | 20260506 09:41:24.455 | INFO | "huge_shared" : " 27 GB", | 20260506 09:41:24.455 | INFO | "malloc" : " 14 GB", | 20260506 09:41:24.455 | INFO | "pinned" : " 29 GB" | 20260506 09:41:24.455 | INFO | } | 20260506 09:41:24.455 | INFO | Perf report for the last 5.13924 seconds: | 20260506 09:41:24.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:41:24.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:24.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:24.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:24.455 | INFO | ANALYSIS : 57.0%, 3.0, 88.5ms, 0 | 20260506 09:41:24.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:24.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:24.455 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:41:24.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:24.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:24.456 | INFO | Uploaded 36GiB to GPU in 1.60531s (22.4256GiB/s) | 20260506 09:41:24.456 | INFO | Downloaded 34GiB to GPU in 1.52265s (22.3295GiB/s) | 20260506 09:41:29.594 | INFO | Analyzing chunk frames = [357888,358400) | 20260506 09:41:29.594 | INFO | MemInfo: { | 20260506 09:41:29.594 | INFO | "gpu" : " 17 GB", | 20260506 09:41:29.594 | INFO | "huge" : "", | 20260506 09:41:29.594 | INFO | "huge_shared" : " 27 GB", | 20260506 09:41:29.594 | INFO | "malloc" : " 14 GB", | 20260506 09:41:29.594 | INFO | "pinned" : " 29 GB" | 20260506 09:41:29.595 | INFO | } | 20260506 09:41:29.595 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:41:29.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:41:29.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:29.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:29.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:29.595 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:41:29.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:29.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:29.595 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 09:41:29.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:29.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:29.595 | INFO | Uploaded 36GiB to GPU in 1.60532s (22.4254GiB/s) | 20260506 09:41:29.595 | INFO | Downloaded 34GiB to GPU in 1.5232s (22.3215GiB/s) | 20260506 09:41:34.734 | INFO | Analyzing chunk frames = [358400,358912) | 20260506 09:41:34.734 | INFO | MemInfo: { | 20260506 09:41:34.734 | INFO | "gpu" : " 17 GB", | 20260506 09:41:34.734 | INFO | "huge" : "", | 20260506 09:41:34.734 | INFO | "huge_shared" : " 27 GB", | 20260506 09:41:34.734 | INFO | "malloc" : " 14 GB", | 20260506 09:41:34.734 | INFO | "pinned" : " 29 GB" | 20260506 09:41:34.734 | INFO | } | 20260506 09:41:34.734 | INFO | Perf report for the last 5.13932 seconds: | 20260506 09:41:34.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:41:34.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:34.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:34.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:34.734 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 09:41:34.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:34.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:34.734 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:41:34.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:34.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:34.734 | INFO | Uploaded 36GiB to GPU in 1.60327s (22.4541GiB/s) | 20260506 09:41:34.735 | INFO | Downloaded 34GiB to GPU in 1.52079s (22.3568GiB/s) | 20260506 09:41:39.873 | INFO | Analyzing chunk frames = [358912,359424) | 20260506 09:41:39.873 | INFO | MemInfo: { | 20260506 09:41:39.873 | INFO | "gpu" : " 17 GB", | 20260506 09:41:39.873 | INFO | "huge" : "", | 20260506 09:41:39.873 | INFO | "huge_shared" : " 27 GB", | 20260506 09:41:39.873 | INFO | "malloc" : " 14 GB", | 20260506 09:41:39.873 | INFO | "pinned" : " 29 GB" | 20260506 09:41:39.873 | INFO | } | 20260506 09:41:39.873 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:41:39.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:41:39.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:39.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:39.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:39.873 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:41:39.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:39.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:39.873 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:41:39.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:39.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:39.874 | INFO | Uploaded 36GiB to GPU in 1.60459s (22.4357GiB/s) | 20260506 09:41:39.874 | INFO | Downloaded 34GiB to GPU in 1.52151s (22.3462GiB/s) | 20260506 09:41:45.012 | INFO | Analyzing chunk frames = [359424,359936) | 20260506 09:41:45.012 | INFO | MemInfo: { | 20260506 09:41:45.012 | INFO | "gpu" : " 17 GB", | 20260506 09:41:45.012 | INFO | "huge" : "", | 20260506 09:41:45.012 | INFO | "huge_shared" : " 27 GB", | 20260506 09:41:45.012 | INFO | "malloc" : " 14 GB", | 20260506 09:41:45.012 | INFO | "pinned" : " 29 GB" | 20260506 09:41:45.012 | INFO | } | 20260506 09:41:45.012 | INFO | Perf report for the last 5.13916 seconds: | 20260506 09:41:45.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:41:45.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:45.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:45.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:45.013 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:41:45.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:45.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:45.013 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:41:45.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:45.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:45.013 | INFO | Uploaded 36GiB to GPU in 1.60333s (22.4533GiB/s) | 20260506 09:41:45.013 | INFO | Downloaded 34GiB to GPU in 1.5218s (22.342GiB/s) | 20260506 09:41:50.152 | INFO | Analyzing chunk frames = [359936,360448) | 20260506 09:41:50.152 | INFO | MemInfo: { | 20260506 09:41:50.152 | INFO | "gpu" : " 17 GB", | 20260506 09:41:50.152 | INFO | "huge" : "", | 20260506 09:41:50.152 | INFO | "huge_shared" : " 27 GB", | 20260506 09:41:50.152 | INFO | "malloc" : " 14 GB", | 20260506 09:41:50.152 | INFO | "pinned" : " 29 GB" | 20260506 09:41:50.152 | INFO | } | 20260506 09:41:50.152 | INFO | Perf report for the last 5.13956 seconds: | 20260506 09:41:50.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:41:50.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:50.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:50.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:50.152 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:41:50.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:50.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:50.152 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 09:41:50.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:50.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:50.153 | INFO | Uploaded 36GiB to GPU in 1.60464s (22.435GiB/s) | 20260506 09:41:50.153 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:41:50.153 | INFO | Total mem usage: 15.7 GiB | 20260506 09:41:50.153 | INFO | All Time High Water Marks sum: 16.2 GiB (Savings: 2.56%) | 20260506 09:41:50.153 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -3.81%) | 20260506 09:41:50.153 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:41:50.153 | INFO | High Water Breakdown: | 20260506 09:41:50.153 | INFO | 59.5% : 9.02 GiB ( 95.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:41:50.153 | INFO | 40.4% : 6.13 GiB ( 91.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:41:50.153 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:41:50.153 | INFO | ------------------------------------------ | 20260506 09:41:50.153 | INFO | Downloaded 34GiB to GPU in 1.52298s (22.3247GiB/s) | 20260506 09:41:50.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:41:50.153 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:41:50.153 | INFO | Total mem usage: 26.5 GiB | 20260506 09:41:50.153 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:41:50.153 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:41:50.153 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:41:50.153 | INFO | High Water Breakdown: | 20260506 09:41:50.153 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:41:50.153 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:41:50.153 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:41:50.153 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:41:50.153 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:41:50.153 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:41:50.153 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:41:50.153 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:41:50.153 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:41:50.153 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:41:50.153 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:41:50.153 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:41:50.153 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:41:50.153 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:41:50.153 | INFO | ------------------------------------------ | 20260506 09:41:50.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:41:50.153 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:41:50.153 | INFO | Total mem usage: 25.6 GiB | 20260506 09:41:50.153 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:41:50.153 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:41:50.153 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:41:50.153 | INFO | High Water Breakdown: | 20260506 09:41:50.153 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:41:50.153 | INFO | ------------------------------------------ | 20260506 09:41:50.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:41:50.153 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:41:50.153 | INFO | Total mem usage: 15.7 GiB | 20260506 09:41:50.153 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:41:50.153 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:41:50.153 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:41:50.153 | INFO | High Water Breakdown: | 20260506 09:41:50.153 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:41:50.153 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:41:50.153 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:41:50.153 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:41:50.153 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:41:50.153 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:41:50.153 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:41:50.153 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:41:50.153 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:41:50.153 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:41:50.153 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:41:50.153 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:41:50.153 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:41:50.153 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:41:50.153 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:41:50.153 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:41:50.153 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:41:50.153 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:41:50.153 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:41:50.153 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:41:50.153 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:41:50.153 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:41:50.153 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:41:50.153 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:41:50.153 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:41:50.153 | INFO | ------------------------------------------ | 20260506 09:41:50.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:41:50.153 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:41:50.153 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:41:50.153 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:41:50.153 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 09:41:50.153 | INFO | PerfReport: 0.29%, AggregateStats -- 8.61e-02 / -nan, 4.37e-03 / 0.00e+00 | 20260506 09:41:50.153 | INFO | PerfReport: 10.90%, Baseline -- 3.29e+00 / -nan, 1.72e-02 / 0.00e+00 | 20260506 09:41:50.153 | INFO | PerfReport: 3.05%, Histogram -- 9.21e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260506 09:41:50.153 | INFO | PerfReport: 16.04%, DME -- 3.88e+01 / -nan, 3.12e+00 / 0.00e+00 | 20260506 09:41:50.153 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.72e+01 / -nan, 4.22e-02 / 0.00e+00 | 20260506 09:41:50.153 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.26e+00 / -nan, 1.33e-01 / 0.00e+00 | 20260506 09:41:50.153 | INFO | PerfReport: 3.80%, Metrics -- 1.15e+00 / -nan, 1.78e-01 / 0.00e+00 | 20260506 09:41:50.153 | INFO | PerfReport: 1.48%, PulseSerializing -- 4.46e-01 / -nan, 6.44e-02 / 0.00e+00 | 20260506 09:41:50.153 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:41:50.153 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:41:50.154 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:41:50.154 | INFO | PerfReport: Full Execution -- 9.01e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 09:41:50.154 | INFO | PerfReport: 18.64%, Upload -- 1.68e+01 / 0.00e+00, 4.89e-01 / 0.00e+00 | 20260506 09:41:50.154 | INFO | PerfReport: 17.90%, Download -- 1.61e+01 / 0.00e+00, 8.45e-01 / 0.00e+00 | 20260506 09:41:50.154 | INFO | PerfReport: 33.51%, Compute -- 3.02e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 09:41:50.154 | INFO | PerfReport: 0.56%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:41:50.154 | INFO | PerfReport: 0.00%, LockDownload -- 4.92e-04 / 0.00e+00, 1.61e-04 / 0.00e+00 | 20260506 09:41:50.154 | INFO | PerfReport: 29.38%, LockCompute -- 2.65e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 09:41:50.154 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260506 09:41:50.249 | INFO | Starting a round of Baz IO | 20260506 09:41:53.174 | INFO | Populating RTMetrics report | 20260506 09:41:53.175 | INFO | RTMetrics report populated | 20260506 09:41:53.175 | INFO | Opening tmp RTMetrics json file | 20260506 09:41:53.196 | INFO | Serializing RTMetrics to json | 20260506 09:41:53.200 | INFO | Serialization done, swapping tmp file now... | 20260506 09:41:53.205 | INFO | RTMetrics done | 20260506 09:41:53.650 | INFO | Finished a round of Baz IO: Wrote 1.889 GiB in 3.4 seconds | 20260506 09:41:53.650 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.28% / 0.09% / 47.63% | 20260506 09:41:53.650 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.74519% / 0.000202656% / 3.97364e-05% | 20260506 09:41:53.650 | INFO | Baz IO overall: | 20260506 09:41:53.650 | INFO | BytesWritten : 145976443895 | 20260506 09:41:53.650 | INFO | NumEvents : 69470780829 | 20260506 09:41:55.291 | INFO | Analyzing chunk frames = [360448,360960) | 20260506 09:41:55.291 | INFO | MemInfo: { | 20260506 09:41:55.291 | INFO | "gpu" : " 17 GB", | 20260506 09:41:55.291 | INFO | "huge" : "", | 20260506 09:41:55.291 | INFO | "huge_shared" : " 27 GB", | 20260506 09:41:55.291 | INFO | "malloc" : " 14 GB", | 20260506 09:41:55.291 | INFO | "pinned" : " 29 GB" | 20260506 09:41:55.291 | INFO | } | 20260506 09:41:55.291 | INFO | Perf report for the last 5.13904 seconds: | 20260506 09:41:55.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:41:55.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:55.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:55.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:55.291 | INFO | ANALYSIS : 57.8%, 2.8, 85.7ms, 0 | 20260506 09:41:55.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:41:55.291 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 09:41:55.291 | INFO | PRE_HQ : 56.9%, 2.1, 60.9ms, 0 | 20260506 09:41:55.291 | INFO | BAZWRITER : 65.8%, 4.8, 161.6ms, 0 | 20260506 09:41:55.291 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 09:41:55.292 | INFO | Uploaded 36GiB to GPU in 1.6762s (21.4771GiB/s) | 20260506 09:41:55.292 | INFO | Downloaded 38GiB to GPU in 1.74157s (21.8194GiB/s) | 20260506 09:42:00.430 | INFO | Analyzing chunk frames = [360960,361472) | 20260506 09:42:00.430 | INFO | MemInfo: { | 20260506 09:42:00.430 | INFO | "gpu" : " 17 GB", | 20260506 09:42:00.430 | INFO | "huge" : "", | 20260506 09:42:00.430 | INFO | "huge_shared" : " 27 GB", | 20260506 09:42:00.430 | INFO | "malloc" : " 14 GB", | 20260506 09:42:00.430 | INFO | "pinned" : " 29 GB" | 20260506 09:42:00.430 | INFO | } | 20260506 09:42:00.430 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:42:00.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:42:00.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:00.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:00.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:00.431 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 09:42:00.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:00.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:00.431 | INFO | PRE_HQ : 30.4%, 1.0, 16.2ms, 0 | 20260506 09:42:00.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:00.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:00.431 | INFO | Uploaded 36GiB to GPU in 1.60246s (22.4654GiB/s) | 20260506 09:42:00.431 | INFO | Downloaded 34GiB to GPU in 1.51932s (22.3785GiB/s) | 20260506 09:42:05.570 | INFO | Analyzing chunk frames = [361472,361984) | 20260506 09:42:05.570 | INFO | MemInfo: { | 20260506 09:42:05.570 | INFO | "gpu" : " 17 GB", | 20260506 09:42:05.570 | INFO | "huge" : "", | 20260506 09:42:05.570 | INFO | "huge_shared" : " 27 GB", | 20260506 09:42:05.570 | INFO | "malloc" : " 14 GB", | 20260506 09:42:05.570 | INFO | "pinned" : " 29 GB" | 20260506 09:42:05.570 | INFO | } | 20260506 09:42:05.570 | INFO | Perf report for the last 5.13969 seconds: | 20260506 09:42:05.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:42:05.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:05.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:05.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:05.570 | INFO | ANALYSIS : 56.9%, 3.0, 88.5ms, 0 | 20260506 09:42:05.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:05.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:05.570 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:42:05.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:05.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:05.570 | INFO | Uploaded 36GiB to GPU in 1.60595s (22.4167GiB/s) | 20260506 09:42:05.571 | INFO | Downloaded 34GiB to GPU in 1.52422s (22.3065GiB/s) | 20260506 09:42:10.709 | INFO | Analyzing chunk frames = [361984,362496) | 20260506 09:42:10.709 | INFO | MemInfo: { | 20260506 09:42:10.709 | INFO | "gpu" : " 17 GB", | 20260506 09:42:10.709 | INFO | "huge" : "", | 20260506 09:42:10.709 | INFO | "huge_shared" : " 27 GB", | 20260506 09:42:10.709 | INFO | "malloc" : " 14 GB", | 20260506 09:42:10.709 | INFO | "pinned" : " 29 GB" | 20260506 09:42:10.709 | INFO | } | 20260506 09:42:10.709 | INFO | Perf report for the last 5.13894 seconds: | 20260506 09:42:10.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:42:10.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:10.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:10.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:10.709 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:42:10.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:10.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:10.709 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:42:10.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:10.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:10.710 | INFO | Uploaded 36GiB to GPU in 1.60635s (22.411GiB/s) | 20260506 09:42:10.710 | INFO | Downloaded 34GiB to GPU in 1.52347s (22.3174GiB/s) | 20260506 09:42:15.849 | INFO | Analyzing chunk frames = [362496,363008) | 20260506 09:42:15.849 | INFO | MemInfo: { | 20260506 09:42:15.849 | INFO | "gpu" : " 17 GB", | 20260506 09:42:15.849 | INFO | "huge" : "", | 20260506 09:42:15.849 | INFO | "huge_shared" : " 27 GB", | 20260506 09:42:15.849 | INFO | "malloc" : " 14 GB", | 20260506 09:42:15.849 | INFO | "pinned" : " 29 GB" | 20260506 09:42:15.849 | INFO | } | 20260506 09:42:15.849 | INFO | Perf report for the last 5.13959 seconds: | 20260506 09:42:15.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:42:15.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:15.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:15.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:15.849 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 09:42:15.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:15.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:15.849 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260506 09:42:15.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:15.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:15.849 | INFO | Uploaded 36GiB to GPU in 1.60442s (22.438GiB/s) | 20260506 09:42:15.849 | INFO | Downloaded 34GiB to GPU in 1.52077s (22.3571GiB/s) | 20260506 09:42:20.988 | INFO | Analyzing chunk frames = [363008,363520) | 20260506 09:42:20.988 | INFO | MemInfo: { | 20260506 09:42:20.988 | INFO | "gpu" : " 17 GB", | 20260506 09:42:20.988 | INFO | "huge" : "", | 20260506 09:42:20.988 | INFO | "huge_shared" : " 27 GB", | 20260506 09:42:20.988 | INFO | "malloc" : " 14 GB", | 20260506 09:42:20.988 | INFO | "pinned" : " 29 GB" | 20260506 09:42:20.988 | INFO | } | 20260506 09:42:20.988 | INFO | Perf report for the last 5.13946 seconds: | 20260506 09:42:20.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:42:20.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:20.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:20.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:20.988 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:42:20.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:20.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:20.988 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:42:20.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:20.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:20.989 | INFO | Uploaded 36GiB to GPU in 1.60483s (22.4323GiB/s) | 20260506 09:42:20.989 | INFO | Downloaded 34GiB to GPU in 1.52138s (22.3482GiB/s) | 20260506 09:42:26.127 | INFO | Analyzing chunk frames = [363520,364032) | 20260506 09:42:26.127 | INFO | MemInfo: { | 20260506 09:42:26.127 | INFO | "gpu" : " 17 GB", | 20260506 09:42:26.127 | INFO | "huge" : "", | 20260506 09:42:26.127 | INFO | "huge_shared" : " 27 GB", | 20260506 09:42:26.127 | INFO | "malloc" : " 14 GB", | 20260506 09:42:26.127 | INFO | "pinned" : " 29 GB" | 20260506 09:42:26.127 | INFO | } | 20260506 09:42:26.127 | INFO | Perf report for the last 5.13913 seconds: | 20260506 09:42:26.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:42:26.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:26.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:26.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:26.127 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:42:26.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:26.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:26.127 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 09:42:26.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:26.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:26.128 | INFO | Uploaded 36GiB to GPU in 1.60417s (22.4415GiB/s) | 20260506 09:42:26.128 | INFO | Downloaded 34GiB to GPU in 1.52215s (22.3368GiB/s) | 20260506 09:42:31.267 | INFO | Analyzing chunk frames = [364032,364544) | 20260506 09:42:31.267 | INFO | MemInfo: { | 20260506 09:42:31.267 | INFO | "gpu" : " 17 GB", | 20260506 09:42:31.267 | INFO | "huge" : "", | 20260506 09:42:31.267 | INFO | "huge_shared" : " 27 GB", | 20260506 09:42:31.267 | INFO | "malloc" : " 14 GB", | 20260506 09:42:31.267 | INFO | "pinned" : " 29 GB" | 20260506 09:42:31.267 | INFO | } | 20260506 09:42:31.267 | INFO | Perf report for the last 5.13974 seconds: | 20260506 09:42:31.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:42:31.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:31.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:31.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:31.267 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:42:31.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:31.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:31.267 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 09:42:31.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:31.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:31.268 | INFO | Uploaded 36GiB to GPU in 1.60559s (22.4217GiB/s) | 20260506 09:42:31.268 | INFO | Downloaded 34GiB to GPU in 1.52364s (22.315GiB/s) | 20260506 09:42:31.364 | INFO | Starting a round of Baz IO | 20260506 09:42:34.287 | INFO | Populating RTMetrics report | 20260506 09:42:34.288 | INFO | RTMetrics report populated | 20260506 09:42:34.288 | INFO | Opening tmp RTMetrics json file | 20260506 09:42:34.342 | INFO | Serializing RTMetrics to json | 20260506 09:42:34.381 | INFO | Serialization done, swapping tmp file now... | 20260506 09:42:34.390 | INFO | RTMetrics done | 20260506 09:42:34.608 | INFO | Finished a round of Baz IO: Wrote 1.891 GiB in 3.243 seconds | 20260506 09:42:34.608 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.20% / 0.09% / 47.72% | 20260506 09:42:34.608 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.71194% / 0.000270208% / 5.96046e-05% | 20260506 09:42:34.608 | INFO | Baz IO overall: | 20260506 09:42:34.608 | INFO | BytesWritten : 148007064517 | 20260506 09:42:34.608 | INFO | NumEvents : 70456455619 | 20260506 09:42:36.406 | INFO | Analyzing chunk frames = [364544,365056) | 20260506 09:42:36.406 | INFO | MemInfo: { | 20260506 09:42:36.406 | INFO | "gpu" : " 17 GB", | 20260506 09:42:36.406 | INFO | "huge" : "", | 20260506 09:42:36.406 | INFO | "huge_shared" : " 27 GB", | 20260506 09:42:36.406 | INFO | "malloc" : " 14 GB", | 20260506 09:42:36.406 | INFO | "pinned" : " 29 GB" | 20260506 09:42:36.406 | INFO | } | 20260506 09:42:36.406 | INFO | Perf report for the last 5.1391 seconds: | 20260506 09:42:36.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:42:36.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:36.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:36.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:36.406 | INFO | ANALYSIS : 57.8%, 2.9, 86.8ms, 0 | 20260506 09:42:36.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:36.406 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260506 09:42:36.406 | INFO | PRE_HQ : 56.9%, 2.1, 62.0ms, 0 | 20260506 09:42:36.406 | INFO | BAZWRITER : 62.7%, 4.7, 150.2ms, 0 | 20260506 09:42:36.406 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 09:42:36.407 | INFO | Uploaded 36GiB to GPU in 1.65406s (21.7646GiB/s) | 20260506 09:42:36.407 | INFO | Downloaded 38GiB to GPU in 1.71914s (22.104GiB/s) | 20260506 09:42:41.545 | INFO | Analyzing chunk frames = [365056,365568) | 20260506 09:42:41.545 | INFO | MemInfo: { | 20260506 09:42:41.545 | INFO | "gpu" : " 17 GB", | 20260506 09:42:41.545 | INFO | "huge" : "", | 20260506 09:42:41.545 | INFO | "huge_shared" : " 27 GB", | 20260506 09:42:41.545 | INFO | "malloc" : " 14 GB", | 20260506 09:42:41.545 | INFO | "pinned" : " 29 GB" | 20260506 09:42:41.545 | INFO | } | 20260506 09:42:41.545 | INFO | Perf report for the last 5.13924 seconds: | 20260506 09:42:41.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:42:41.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:41.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:41.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:41.546 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 09:42:41.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:41.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:41.546 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:42:41.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:41.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:41.546 | INFO | Uploaded 36GiB to GPU in 1.60201s (22.4717GiB/s) | 20260506 09:42:41.546 | INFO | Downloaded 34GiB to GPU in 1.51887s (22.385GiB/s) | 20260506 09:42:46.685 | INFO | Analyzing chunk frames = [365568,366080) | 20260506 09:42:46.685 | INFO | MemInfo: { | 20260506 09:42:46.685 | INFO | "gpu" : " 17 GB", | 20260506 09:42:46.685 | INFO | "huge" : "", | 20260506 09:42:46.685 | INFO | "huge_shared" : " 27 GB", | 20260506 09:42:46.685 | INFO | "malloc" : " 14 GB", | 20260506 09:42:46.685 | INFO | "pinned" : " 29 GB" | 20260506 09:42:46.685 | INFO | } | 20260506 09:42:46.685 | INFO | Perf report for the last 5.13948 seconds: | 20260506 09:42:46.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:42:46.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:46.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:46.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:46.685 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:42:46.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:46.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:46.685 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:42:46.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:46.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:46.685 | INFO | Uploaded 36GiB to GPU in 1.60619s (22.4133GiB/s) | 20260506 09:42:46.685 | INFO | Downloaded 34GiB to GPU in 1.52393s (22.3108GiB/s) | 20260506 09:42:51.824 | INFO | Analyzing chunk frames = [366080,366592) | 20260506 09:42:51.824 | INFO | MemInfo: { | 20260506 09:42:51.824 | INFO | "gpu" : " 17 GB", | 20260506 09:42:51.824 | INFO | "huge" : "", | 20260506 09:42:51.824 | INFO | "huge_shared" : " 27 GB", | 20260506 09:42:51.824 | INFO | "malloc" : " 14 GB", | 20260506 09:42:51.824 | INFO | "pinned" : " 29 GB" | 20260506 09:42:51.824 | INFO | } | 20260506 09:42:51.824 | INFO | Perf report for the last 5.13922 seconds: | 20260506 09:42:51.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:42:51.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:51.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:51.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:51.824 | INFO | ANALYSIS : 56.7%, 3.0, 87.8ms, 0 | 20260506 09:42:51.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:51.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:51.824 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:42:51.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:51.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:51.824 | INFO | Uploaded 36GiB to GPU in 1.6056s (22.4216GiB/s) | 20260506 09:42:51.824 | INFO | Downloaded 34GiB to GPU in 1.52313s (22.3224GiB/s) | 20260506 09:42:56.963 | INFO | Analyzing chunk frames = [366592,367104) | 20260506 09:42:56.963 | INFO | MemInfo: { | 20260506 09:42:56.963 | INFO | "gpu" : " 17 GB", | 20260506 09:42:56.963 | INFO | "huge" : "", | 20260506 09:42:56.963 | INFO | "huge_shared" : " 27 GB", | 20260506 09:42:56.963 | INFO | "malloc" : " 14 GB", | 20260506 09:42:56.963 | INFO | "pinned" : " 29 GB" | 20260506 09:42:56.963 | INFO | } | 20260506 09:42:56.964 | INFO | Perf report for the last 5.13935 seconds: | 20260506 09:42:56.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:42:56.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:56.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:56.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:56.964 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 09:42:56.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:56.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:56.964 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:42:56.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:56.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:42:56.964 | INFO | Uploaded 36GiB to GPU in 1.60394s (22.4447GiB/s) | 20260506 09:42:56.964 | INFO | Downloaded 34GiB to GPU in 1.5207s (22.3581GiB/s) | 20260506 09:43:02.103 | INFO | Analyzing chunk frames = [367104,367616) | 20260506 09:43:02.103 | INFO | MemInfo: { | 20260506 09:43:02.103 | INFO | "gpu" : " 17 GB", | 20260506 09:43:02.103 | INFO | "huge" : "", | 20260506 09:43:02.103 | INFO | "huge_shared" : " 27 GB", | 20260506 09:43:02.103 | INFO | "malloc" : " 14 GB", | 20260506 09:43:02.103 | INFO | "pinned" : " 29 GB" | 20260506 09:43:02.103 | INFO | } | 20260506 09:43:02.103 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:43:02.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:43:02.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:02.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:02.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:02.103 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:43:02.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:02.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:02.103 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:43:02.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:02.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:02.103 | INFO | Uploaded 36GiB to GPU in 1.60491s (22.4311GiB/s) | 20260506 09:43:02.103 | INFO | Downloaded 34GiB to GPU in 1.52195s (22.3398GiB/s) | 20260506 09:43:07.242 | INFO | Analyzing chunk frames = [367616,368128) | 20260506 09:43:07.242 | INFO | MemInfo: { | 20260506 09:43:07.242 | INFO | "gpu" : " 17 GB", | 20260506 09:43:07.242 | INFO | "huge" : "", | 20260506 09:43:07.242 | INFO | "huge_shared" : " 27 GB", | 20260506 09:43:07.242 | INFO | "malloc" : " 14 GB", | 20260506 09:43:07.242 | INFO | "pinned" : " 29 GB" | 20260506 09:43:07.242 | INFO | } | 20260506 09:43:07.242 | INFO | Perf report for the last 5.13932 seconds: | 20260506 09:43:07.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:43:07.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:07.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:07.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:07.242 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260506 09:43:07.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:07.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:07.242 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:43:07.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:07.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:07.243 | INFO | Uploaded 36GiB to GPU in 1.60453s (22.4364GiB/s) | 20260506 09:43:07.243 | INFO | Downloaded 34GiB to GPU in 1.52278s (22.3276GiB/s) | 20260506 09:43:12.385 | INFO | Analyzing chunk frames = [368128,368640) | 20260506 09:43:12.385 | INFO | MemInfo: { | 20260506 09:43:12.385 | INFO | "gpu" : " 17 GB", | 20260506 09:43:12.385 | INFO | "huge" : "", | 20260506 09:43:12.385 | INFO | "huge_shared" : " 27 GB", | 20260506 09:43:12.385 | INFO | "malloc" : " 14 GB", | 20260506 09:43:12.385 | INFO | "pinned" : " 29 GB" | 20260506 09:43:12.385 | INFO | } | 20260506 09:43:12.385 | INFO | Perf report for the last 5.14308 seconds: | 20260506 09:43:12.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:43:12.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:12.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:12.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:12.385 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:43:12.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:12.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:12.386 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 09:43:12.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:12.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:12.386 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:43:12.386 | INFO | Total mem usage: 15.9 GiB | 20260506 09:43:12.386 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 2.48%) | 20260506 09:43:12.386 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: 0.0483%) | 20260506 09:43:12.386 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:43:12.386 | INFO | High Water Breakdown: | 20260506 09:43:12.386 | INFO | 60.3% : 9.6 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:43:12.386 | INFO | 39.6% : 6.3 GiB ( 94.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:43:12.386 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:43:12.386 | INFO | ------------------------------------------ | 20260506 09:43:12.386 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:43:12.386 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:43:12.386 | INFO | Total mem usage: 26.5 GiB | 20260506 09:43:12.386 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:43:12.386 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:43:12.386 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:43:12.386 | INFO | High Water Breakdown: | 20260506 09:43:12.386 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:43:12.386 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:43:12.386 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:43:12.386 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:43:12.386 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:43:12.386 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:43:12.386 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:43:12.386 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:43:12.386 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:43:12.386 | INFO | Uploaded 36GiB to GPU in 1.60396s (22.4445GiB/s) | 20260506 09:43:12.386 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:43:12.386 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:43:12.386 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:43:12.386 | INFO | Downloaded 34GiB to GPU in 1.52266s (22.3293GiB/s) | 20260506 09:43:12.386 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:43:12.386 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:43:12.386 | INFO | ------------------------------------------ | 20260506 09:43:12.386 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:43:12.386 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:43:12.386 | INFO | Total mem usage: 25.6 GiB | 20260506 09:43:12.386 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:43:12.386 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:43:12.386 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:43:12.386 | INFO | High Water Breakdown: | 20260506 09:43:12.386 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:43:12.386 | INFO | ------------------------------------------ | 20260506 09:43:12.386 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:43:12.386 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:43:12.386 | INFO | Total mem usage: 15.7 GiB | 20260506 09:43:12.386 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:43:12.386 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:43:12.386 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:43:12.386 | INFO | High Water Breakdown: | 20260506 09:43:12.386 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:43:12.386 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:43:12.386 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:43:12.386 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:43:12.386 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:43:12.386 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:43:12.386 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:43:12.386 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:43:12.386 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:43:12.386 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:43:12.386 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:43:12.386 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:43:12.386 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:43:12.386 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:43:12.386 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:43:12.386 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:43:12.386 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:43:12.386 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:43:12.386 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:43:12.386 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:43:12.387 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:43:12.387 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:43:12.387 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:43:12.387 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:43:12.387 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:43:12.387 | INFO | ------------------------------------------ | 20260506 09:43:12.387 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:43:12.387 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:43:12.387 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:43:12.387 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:43:12.387 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 09:43:12.387 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / -nan, 4.65e-03 / 0.00e+00 | 20260506 09:43:12.387 | INFO | PerfReport: 10.91%, Baseline -- 3.29e+00 / -nan, 1.90e-02 / 0.00e+00 | 20260506 09:43:12.387 | INFO | PerfReport: 3.05%, Histogram -- 9.21e-01 / -nan, 5.25e-02 / 0.00e+00 | 20260506 09:43:12.387 | INFO | PerfReport: 16.04%, DME -- 3.87e+01 / -nan, 3.07e+00 / 0.00e+00 | 20260506 09:43:12.387 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.72e+01 / -nan, 4.12e-02 / 0.00e+00 | 20260506 09:43:12.387 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.26e+00 / -nan, 1.34e-01 / 0.00e+00 | 20260506 09:43:12.387 | INFO | PerfReport: 3.80%, Metrics -- 1.15e+00 / -nan, 1.78e-01 / 0.00e+00 | 20260506 09:43:12.387 | INFO | PerfReport: 1.48%, PulseSerializing -- 4.46e-01 / -nan, 6.46e-02 / 0.00e+00 | 20260506 09:43:12.387 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:43:12.387 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:43:12.387 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:43:12.387 | INFO | PerfReport: Full Execution -- 9.02e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 09:43:12.387 | INFO | PerfReport: 18.61%, Upload -- 1.68e+01 / 0.00e+00, 4.71e-01 / 0.00e+00 | 20260506 09:43:12.387 | INFO | PerfReport: 17.88%, Download -- 1.61e+01 / 0.00e+00, 8.19e-01 / 0.00e+00 | 20260506 09:43:12.387 | INFO | PerfReport: 33.47%, Compute -- 3.02e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 09:43:12.387 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 09:43:12.387 | INFO | PerfReport: 0.00%, LockDownload -- 4.82e-04 / 0.00e+00, 1.19e-04 / 0.00e+00 | 20260506 09:43:12.387 | INFO | PerfReport: 29.46%, LockCompute -- 2.66e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 09:43:12.387 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260506 09:43:12.483 | INFO | Starting a round of Baz IO | 20260506 09:43:15.407 | INFO | Populating RTMetrics report | 20260506 09:43:15.407 | INFO | RTMetrics report populated | 20260506 09:43:15.407 | INFO | Opening tmp RTMetrics json file | 20260506 09:43:15.422 | INFO | Serializing RTMetrics to json | 20260506 09:43:15.427 | INFO | Serialization done, swapping tmp file now... | 20260506 09:43:15.431 | INFO | RTMetrics done | 20260506 09:43:15.662 | INFO | Finished a round of Baz IO: Wrote 1.892 GiB in 3.179 seconds | 20260506 09:43:15.662 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.12% / 0.08% / 47.80% | 20260506 09:43:15.662 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.67558% / 0.000230471% / 4.76837e-05% | 20260506 09:43:15.662 | INFO | Baz IO overall: | 20260506 09:43:15.663 | INFO | BytesWritten : 150038930922 | 20260506 09:43:15.663 | INFO | NumEvents : 71442757866 | 20260506 09:43:17.538 | INFO | Analyzing chunk frames = [368640,369152) | 20260506 09:43:17.538 | INFO | MemInfo: { | 20260506 09:43:17.538 | INFO | "gpu" : " 17 GB", | 20260506 09:43:17.538 | INFO | "huge" : "", | 20260506 09:43:17.538 | INFO | "huge_shared" : " 27 GB", | 20260506 09:43:17.538 | INFO | "malloc" : " 14 GB", | 20260506 09:43:17.538 | INFO | "pinned" : " 29 GB" | 20260506 09:43:17.538 | INFO | } | 20260506 09:43:17.538 | INFO | Perf report for the last 5.15283 seconds: | 20260506 09:43:17.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:43:17.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:17.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:17.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:17.538 | INFO | ANALYSIS : 57.7%, 2.8, 85.3ms, 0 | 20260506 09:43:17.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:17.538 | INFO | RT_METRICS : 44.4%, 1.0, 23.4ms, 0 | 20260506 09:43:17.538 | INFO | PRE_HQ : 56.9%, 2.1, 61.7ms, 0 | 20260506 09:43:17.538 | INFO | BAZWRITER : 61.3%, 4.8, 150.6ms, 0 | 20260506 09:43:17.538 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 09:43:17.539 | INFO | Uploaded 36GiB to GPU in 1.6725s (21.5246GiB/s) | 20260506 09:43:17.539 | INFO | Downloaded 38GiB to GPU in 1.73856s (21.8572GiB/s) | 20260506 09:43:22.660 | INFO | Analyzing chunk frames = [369152,369664) | 20260506 09:43:22.660 | INFO | MemInfo: { | 20260506 09:43:22.660 | INFO | "gpu" : " 17 GB", | 20260506 09:43:22.660 | INFO | "huge" : "", | 20260506 09:43:22.660 | INFO | "huge_shared" : " 27 GB", | 20260506 09:43:22.660 | INFO | "malloc" : " 14 GB", | 20260506 09:43:22.660 | INFO | "pinned" : " 29 GB" | 20260506 09:43:22.660 | INFO | } | 20260506 09:43:22.660 | INFO | Perf report for the last 5.12191 seconds: | 20260506 09:43:22.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:43:22.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:22.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:22.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:22.660 | INFO | ANALYSIS : 57.8%, 3.0, 89.8ms, 0 | 20260506 09:43:22.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:22.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:22.660 | INFO | PRE_HQ : 30.8%, 1.0, 16.1ms, 0 | 20260506 09:43:22.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:22.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:22.661 | INFO | Uploaded 36GiB to GPU in 1.60282s (22.4604GiB/s) | 20260506 09:43:22.661 | INFO | Downloaded 34GiB to GPU in 1.52051s (22.3609GiB/s) | 20260506 09:43:27.799 | INFO | Analyzing chunk frames = [369664,370176) | 20260506 09:43:27.799 | INFO | MemInfo: { | 20260506 09:43:27.799 | INFO | "gpu" : " 17 GB", | 20260506 09:43:27.799 | INFO | "huge" : "", | 20260506 09:43:27.799 | INFO | "huge_shared" : " 27 GB", | 20260506 09:43:27.799 | INFO | "malloc" : " 14 GB", | 20260506 09:43:27.799 | INFO | "pinned" : " 29 GB" | 20260506 09:43:27.799 | INFO | } | 20260506 09:43:27.799 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:43:27.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:43:27.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:27.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:27.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:27.800 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:43:27.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:27.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:27.800 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:43:27.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:27.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:27.800 | INFO | Uploaded 36GiB to GPU in 1.60473s (22.4336GiB/s) | 20260506 09:43:27.800 | INFO | Downloaded 34GiB to GPU in 1.52412s (22.308GiB/s) | 20260506 09:43:32.939 | INFO | Analyzing chunk frames = [370176,370688) | 20260506 09:43:32.939 | INFO | MemInfo: { | 20260506 09:43:32.939 | INFO | "gpu" : " 17 GB", | 20260506 09:43:32.939 | INFO | "huge" : "", | 20260506 09:43:32.939 | INFO | "huge_shared" : " 27 GB", | 20260506 09:43:32.939 | INFO | "malloc" : " 14 GB", | 20260506 09:43:32.939 | INFO | "pinned" : " 29 GB" | 20260506 09:43:32.939 | INFO | } | 20260506 09:43:32.939 | INFO | Perf report for the last 5.13945 seconds: | 20260506 09:43:32.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:43:32.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:32.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:32.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:32.939 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:43:32.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:32.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:32.939 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:43:32.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:32.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:32.939 | INFO | Uploaded 36GiB to GPU in 1.60397s (22.4444GiB/s) | 20260506 09:43:32.939 | INFO | Downloaded 34GiB to GPU in 1.52373s (22.3137GiB/s) | 20260506 09:43:38.078 | INFO | Analyzing chunk frames = [370688,371200) | 20260506 09:43:38.078 | INFO | MemInfo: { | 20260506 09:43:38.078 | INFO | "gpu" : " 17 GB", | 20260506 09:43:38.078 | INFO | "huge" : "", | 20260506 09:43:38.078 | INFO | "huge_shared" : " 27 GB", | 20260506 09:43:38.078 | INFO | "malloc" : " 14 GB", | 20260506 09:43:38.078 | INFO | "pinned" : " 29 GB" | 20260506 09:43:38.078 | INFO | } | 20260506 09:43:38.078 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:43:38.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:43:38.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:38.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:38.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:38.078 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 09:43:38.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:38.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:38.078 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:43:38.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:38.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:38.079 | INFO | Uploaded 36GiB to GPU in 1.60341s (22.4522GiB/s) | 20260506 09:43:38.079 | INFO | Downloaded 34GiB to GPU in 1.52087s (22.3556GiB/s) | 20260506 09:43:43.217 | INFO | Analyzing chunk frames = [371200,371712) | 20260506 09:43:43.217 | INFO | MemInfo: { | 20260506 09:43:43.217 | INFO | "gpu" : " 17 GB", | 20260506 09:43:43.217 | INFO | "huge" : "", | 20260506 09:43:43.217 | INFO | "huge_shared" : " 27 GB", | 20260506 09:43:43.217 | INFO | "malloc" : " 14 GB", | 20260506 09:43:43.217 | INFO | "pinned" : " 29 GB" | 20260506 09:43:43.217 | INFO | } | 20260506 09:43:43.217 | INFO | Perf report for the last 5.13914 seconds: | 20260506 09:43:43.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:43:43.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:43.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:43.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:43.217 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 09:43:43.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:43.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:43.218 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:43:43.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:43.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:43.218 | INFO | Uploaded 36GiB to GPU in 1.6036s (22.4495GiB/s) | 20260506 09:43:43.218 | INFO | Downloaded 34GiB to GPU in 1.5209s (22.3552GiB/s) | 20260506 09:43:48.357 | INFO | Analyzing chunk frames = [371712,372224) | 20260506 09:43:48.357 | INFO | MemInfo: { | 20260506 09:43:48.357 | INFO | "gpu" : " 17 GB", | 20260506 09:43:48.357 | INFO | "huge" : "", | 20260506 09:43:48.357 | INFO | "huge_shared" : " 27 GB", | 20260506 09:43:48.357 | INFO | "malloc" : " 14 GB", | 20260506 09:43:48.357 | INFO | "pinned" : " 29 GB" | 20260506 09:43:48.357 | INFO | } | 20260506 09:43:48.357 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:43:48.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:43:48.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:48.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:48.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:48.357 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:43:48.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:48.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:48.357 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:43:48.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:48.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:48.357 | INFO | Uploaded 36GiB to GPU in 1.60479s (22.4328GiB/s) | 20260506 09:43:48.357 | INFO | Downloaded 34GiB to GPU in 1.52255s (22.3309GiB/s) | 20260506 09:43:53.496 | INFO | Analyzing chunk frames = [372224,372736) | 20260506 09:43:53.496 | INFO | MemInfo: { | 20260506 09:43:53.496 | INFO | "gpu" : " 17 GB", | 20260506 09:43:53.496 | INFO | "huge" : "", | 20260506 09:43:53.496 | INFO | "huge_shared" : " 27 GB", | 20260506 09:43:53.496 | INFO | "malloc" : " 14 GB", | 20260506 09:43:53.496 | INFO | "pinned" : " 29 GB" | 20260506 09:43:53.496 | INFO | } | 20260506 09:43:53.496 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:43:53.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:43:53.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:53.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:53.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:53.496 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:43:53.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:53.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:53.496 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 09:43:53.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:53.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:53.497 | INFO | Uploaded 36GiB to GPU in 1.60352s (22.4507GiB/s) | 20260506 09:43:53.497 | INFO | Downloaded 34GiB to GPU in 1.52234s (22.334GiB/s) | 20260506 09:43:53.594 | INFO | Starting a round of Baz IO | 20260506 09:43:56.512 | INFO | Populating RTMetrics report | 20260506 09:43:56.512 | INFO | RTMetrics report populated | 20260506 09:43:56.512 | INFO | Opening tmp RTMetrics json file | 20260506 09:43:56.518 | INFO | Serializing RTMetrics to json | 20260506 09:43:56.523 | INFO | Serialization done, swapping tmp file now... | 20260506 09:43:56.528 | INFO | RTMetrics done | 20260506 09:43:56.935 | INFO | Finished a round of Baz IO: Wrote 1.896 GiB in 3.341 seconds | 20260506 09:43:56.935 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 52.03% / 0.08% / 47.88% | 20260506 09:43:56.935 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.65699% / 0.00021855% / 5.16574e-05% | 20260506 09:43:56.935 | INFO | Baz IO overall: | 20260506 09:43:56.935 | INFO | BytesWritten : 152074513046 | 20260506 09:43:56.935 | INFO | NumEvents : 72431153252 | 20260506 09:43:58.635 | INFO | Analyzing chunk frames = [372736,373248) | 20260506 09:43:58.635 | INFO | MemInfo: { | 20260506 09:43:58.635 | INFO | "gpu" : " 17 GB", | 20260506 09:43:58.635 | INFO | "huge" : "", | 20260506 09:43:58.635 | INFO | "huge_shared" : " 27 GB", | 20260506 09:43:58.635 | INFO | "malloc" : " 14 GB", | 20260506 09:43:58.635 | INFO | "pinned" : " 29 GB" | 20260506 09:43:58.635 | INFO | } | 20260506 09:43:58.635 | INFO | Perf report for the last 5.13932 seconds: | 20260506 09:43:58.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:43:58.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:58.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:58.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:58.636 | INFO | ANALYSIS : 57.7%, 2.8, 85.6ms, 0 | 20260506 09:43:58.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:43:58.636 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 09:43:58.636 | INFO | PRE_HQ : 56.9%, 2.1, 61.2ms, 0 | 20260506 09:43:58.636 | INFO | BAZWRITER : 64.6%, 4.8, 158.1ms, 0 | 20260506 09:43:58.636 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 09:43:58.636 | INFO | Uploaded 36GiB to GPU in 1.67033s (21.5527GiB/s) | 20260506 09:43:58.636 | INFO | Downloaded 38GiB to GPU in 1.7357s (21.8932GiB/s) | 20260506 09:44:03.775 | INFO | Analyzing chunk frames = [373248,373760) | 20260506 09:44:03.775 | INFO | MemInfo: { | 20260506 09:44:03.775 | INFO | "gpu" : " 17 GB", | 20260506 09:44:03.775 | INFO | "huge" : "", | 20260506 09:44:03.775 | INFO | "huge_shared" : " 27 GB", | 20260506 09:44:03.775 | INFO | "malloc" : " 14 GB", | 20260506 09:44:03.775 | INFO | "pinned" : " 29 GB" | 20260506 09:44:03.775 | INFO | } | 20260506 09:44:03.775 | INFO | Perf report for the last 5.13948 seconds: | 20260506 09:44:03.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:44:03.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:03.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:03.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:03.775 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260506 09:44:03.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:03.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:03.775 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:44:03.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:03.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:03.776 | INFO | Uploaded 36GiB to GPU in 1.60231s (22.4675GiB/s) | 20260506 09:44:03.776 | INFO | Downloaded 34GiB to GPU in 1.5196s (22.3743GiB/s) | 20260506 09:44:08.914 | INFO | Analyzing chunk frames = [373760,374272) | 20260506 09:44:08.914 | INFO | MemInfo: { | 20260506 09:44:08.914 | INFO | "gpu" : " 17 GB", | 20260506 09:44:08.914 | INFO | "huge" : "", | 20260506 09:44:08.914 | INFO | "huge_shared" : " 27 GB", | 20260506 09:44:08.914 | INFO | "malloc" : " 14 GB", | 20260506 09:44:08.914 | INFO | "pinned" : " 29 GB" | 20260506 09:44:08.914 | INFO | } | 20260506 09:44:08.914 | INFO | Perf report for the last 5.13919 seconds: | 20260506 09:44:08.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:44:08.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:08.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:08.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:08.914 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:44:08.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:08.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:08.914 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 09:44:08.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:08.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:08.915 | INFO | Uploaded 36GiB to GPU in 1.6064s (22.4103GiB/s) | 20260506 09:44:08.915 | INFO | Downloaded 34GiB to GPU in 1.52453s (22.3019GiB/s) | 20260506 09:44:14.053 | INFO | Analyzing chunk frames = [374272,374784) | 20260506 09:44:14.053 | INFO | MemInfo: { | 20260506 09:44:14.053 | INFO | "gpu" : " 17 GB", | 20260506 09:44:14.053 | INFO | "huge" : "", | 20260506 09:44:14.053 | INFO | "huge_shared" : " 27 GB", | 20260506 09:44:14.054 | INFO | "malloc" : " 14 GB", | 20260506 09:44:14.054 | INFO | "pinned" : " 29 GB" | 20260506 09:44:14.054 | INFO | } | 20260506 09:44:14.054 | INFO | Perf report for the last 5.13942 seconds: | 20260506 09:44:14.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:44:14.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:14.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:14.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:14.054 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:44:14.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:14.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:14.054 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 09:44:14.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:14.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:14.054 | INFO | Uploaded 36GiB to GPU in 1.60559s (22.4217GiB/s) | 20260506 09:44:14.054 | INFO | Downloaded 34GiB to GPU in 1.52282s (22.327GiB/s) | 20260506 09:44:19.193 | INFO | Analyzing chunk frames = [374784,375296) | 20260506 09:44:19.193 | INFO | MemInfo: { | 20260506 09:44:19.193 | INFO | "gpu" : " 17 GB", | 20260506 09:44:19.193 | INFO | "huge" : "", | 20260506 09:44:19.193 | INFO | "huge_shared" : " 27 GB", | 20260506 09:44:19.193 | INFO | "malloc" : " 14 GB", | 20260506 09:44:19.193 | INFO | "pinned" : " 29 GB" | 20260506 09:44:19.193 | INFO | } | 20260506 09:44:19.193 | INFO | Perf report for the last 5.13922 seconds: | 20260506 09:44:19.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:44:19.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:19.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:19.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:19.193 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 09:44:19.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:19.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:19.193 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:44:19.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:19.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:19.193 | INFO | Uploaded 36GiB to GPU in 1.60491s (22.4312GiB/s) | 20260506 09:44:19.193 | INFO | Downloaded 34GiB to GPU in 1.52193s (22.3401GiB/s) | 20260506 09:44:24.332 | INFO | Analyzing chunk frames = [375296,375808) | 20260506 09:44:24.332 | INFO | MemInfo: { | 20260506 09:44:24.332 | INFO | "gpu" : " 17 GB", | 20260506 09:44:24.332 | INFO | "huge" : "", | 20260506 09:44:24.332 | INFO | "huge_shared" : " 27 GB", | 20260506 09:44:24.332 | INFO | "malloc" : " 14 GB", | 20260506 09:44:24.332 | INFO | "pinned" : " 29 GB" | 20260506 09:44:24.332 | INFO | } | 20260506 09:44:24.332 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:44:24.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:44:24.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:24.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:24.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:24.332 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:44:24.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:24.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:24.332 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:44:24.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:24.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:24.332 | INFO | Uploaded 36GiB to GPU in 1.60435s (22.439GiB/s) | 20260506 09:44:24.332 | INFO | Downloaded 34GiB to GPU in 1.52112s (22.3519GiB/s) | 20260506 09:44:29.471 | INFO | Analyzing chunk frames = [375808,376320) | 20260506 09:44:29.471 | INFO | MemInfo: { | 20260506 09:44:29.471 | INFO | "gpu" : " 17 GB", | 20260506 09:44:29.471 | INFO | "huge" : "", | 20260506 09:44:29.471 | INFO | "huge_shared" : " 27 GB", | 20260506 09:44:29.471 | INFO | "malloc" : " 14 GB", | 20260506 09:44:29.471 | INFO | "pinned" : " 29 GB" | 20260506 09:44:29.471 | INFO | } | 20260506 09:44:29.471 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:44:29.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:44:29.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:29.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:29.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:29.472 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:44:29.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:29.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:29.472 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:44:29.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:29.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:29.472 | INFO | Uploaded 36GiB to GPU in 1.60367s (22.4484GiB/s) | 20260506 09:44:29.472 | INFO | Downloaded 34GiB to GPU in 1.52228s (22.3349GiB/s) | 20260506 09:44:34.611 | INFO | Analyzing chunk frames = [376320,376832) | 20260506 09:44:34.611 | INFO | MemInfo: { | 20260506 09:44:34.611 | INFO | "gpu" : " 17 GB", | 20260506 09:44:34.611 | INFO | "huge" : "", | 20260506 09:44:34.611 | INFO | "huge_shared" : " 27 GB", | 20260506 09:44:34.611 | INFO | "malloc" : " 14 GB", | 20260506 09:44:34.611 | INFO | "pinned" : " 29 GB" | 20260506 09:44:34.611 | INFO | } | 20260506 09:44:34.611 | INFO | Perf report for the last 5.13932 seconds: | 20260506 09:44:34.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:44:34.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:34.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:34.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:34.611 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:44:34.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:34.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:34.611 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 09:44:34.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:34.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:34.611 | INFO | Uploaded 36GiB to GPU in 1.60495s (22.4306GiB/s) | 20260506 09:44:34.611 | INFO | Downloaded 34GiB to GPU in 1.52358s (22.3159GiB/s) | 20260506 09:44:34.611 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:44:34.611 | INFO | Total mem usage: 15.9 GiB | 20260506 09:44:34.611 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 2.48%) | 20260506 09:44:34.611 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -2.08%) | 20260506 09:44:34.611 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:44:34.611 | INFO | High Water Breakdown: | 20260506 09:44:34.611 | INFO | 60% : 9.34 GiB ( 97.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:44:34.611 | INFO | 39.9% : 6.22 GiB ( 92.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:44:34.611 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:44:34.611 | INFO | ------------------------------------------ | 20260506 09:44:34.611 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:44:34.611 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:44:34.611 | INFO | Total mem usage: 26.5 GiB | 20260506 09:44:34.611 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:44:34.611 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:44:34.611 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:44:34.611 | INFO | High Water Breakdown: | 20260506 09:44:34.611 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:44:34.611 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:44:34.611 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:44:34.611 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:44:34.612 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:44:34.612 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:44:34.612 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:44:34.612 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:44:34.612 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:44:34.612 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:44:34.612 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:44:34.612 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:44:34.612 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:44:34.612 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:44:34.612 | INFO | ------------------------------------------ | 20260506 09:44:34.612 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:44:34.612 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:44:34.612 | INFO | Total mem usage: 25.6 GiB | 20260506 09:44:34.612 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:44:34.612 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:44:34.612 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:44:34.612 | INFO | High Water Breakdown: | 20260506 09:44:34.612 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:44:34.612 | INFO | ------------------------------------------ | 20260506 09:44:34.612 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:44:34.612 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:44:34.612 | INFO | Total mem usage: 15.7 GiB | 20260506 09:44:34.612 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:44:34.612 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:44:34.612 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:44:34.612 | INFO | High Water Breakdown: | 20260506 09:44:34.612 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:44:34.612 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:44:34.612 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:44:34.612 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:44:34.612 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:44:34.612 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:44:34.612 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:44:34.612 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:44:34.612 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:44:34.612 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:44:34.612 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:44:34.612 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:44:34.612 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:44:34.612 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:44:34.612 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:44:34.612 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:44:34.612 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:44:34.612 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:44:34.612 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:44:34.612 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:44:34.612 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:44:34.612 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:44:34.612 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:44:34.612 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:44:34.612 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:44:34.612 | INFO | ------------------------------------------ | 20260506 09:44:34.612 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:44:34.612 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:44:34.612 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:44:34.612 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:44:34.612 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260506 09:44:34.612 | INFO | PerfReport: 0.28%, AggregateStats -- 8.60e-02 / -nan, 4.59e-03 / 0.00e+00 | 20260506 09:44:34.612 | INFO | PerfReport: 10.90%, Baseline -- 3.29e+00 / -nan, 1.64e-02 / 0.00e+00 | 20260506 09:44:34.612 | INFO | PerfReport: 3.05%, Histogram -- 9.22e-01 / -nan, 5.35e-02 / 0.00e+00 | 20260506 09:44:34.612 | INFO | PerfReport: 16.05%, DME -- 3.88e+01 / -nan, 3.09e+00 / 0.00e+00 | 20260506 09:44:34.612 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.72e+01 / -nan, 4.19e-02 / 0.00e+00 | 20260506 09:44:34.612 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.26e+00 / -nan, 1.34e-01 / 0.00e+00 | 20260506 09:44:34.612 | INFO | PerfReport: 3.79%, Metrics -- 1.14e+00 / -nan, 1.78e-01 / 0.00e+00 | 20260506 09:44:34.612 | INFO | PerfReport: 1.47%, PulseSerializing -- 4.45e-01 / -nan, 6.47e-02 / 0.00e+00 | 20260506 09:44:34.612 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:44:34.612 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:44:34.612 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:44:34.612 | INFO | PerfReport: Full Execution -- 9.01e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 09:44:34.612 | INFO | PerfReport: 18.64%, Upload -- 1.68e+01 / 0.00e+00, 4.86e-01 / 0.00e+00 | 20260506 09:44:34.612 | INFO | PerfReport: 17.91%, Download -- 1.61e+01 / 0.00e+00, 8.41e-01 / 0.00e+00 | 20260506 09:44:34.612 | INFO | PerfReport: 33.50%, Compute -- 3.02e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260506 09:44:34.612 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260506 09:44:34.612 | INFO | PerfReport: 0.00%, LockDownload -- 4.89e-04 / 0.00e+00, 1.32e-04 / 0.00e+00 | 20260506 09:44:34.612 | INFO | PerfReport: 29.37%, LockCompute -- 2.65e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 09:44:34.612 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.23e-02 / 0.00e+00 | 20260506 09:44:34.708 | INFO | Starting a round of Baz IO | 20260506 09:44:37.629 | INFO | Populating RTMetrics report | 20260506 09:44:37.629 | INFO | RTMetrics report populated | 20260506 09:44:37.629 | INFO | Opening tmp RTMetrics json file | 20260506 09:44:37.814 | INFO | Serializing RTMetrics to json | 20260506 09:44:37.819 | INFO | Serialization done, swapping tmp file now... | 20260506 09:44:37.824 | INFO | RTMetrics done | 20260506 09:44:37.936 | INFO | Finished a round of Baz IO: Wrote 1.895 GiB in 3.228 seconds | 20260506 09:44:37.936 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.95% / 0.08% / 47.97% | 20260506 09:44:37.936 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.5875% / 0.00021855% / 4.76837e-05% | 20260506 09:44:37.936 | INFO | Baz IO overall: | 20260506 09:44:37.937 | INFO | BytesWritten : 154108894830 | 20260506 09:44:37.937 | INFO | NumEvents : 73418134006 | 20260506 09:44:39.750 | INFO | Analyzing chunk frames = [376832,377344) | 20260506 09:44:39.750 | INFO | MemInfo: { | 20260506 09:44:39.750 | INFO | "gpu" : " 17 GB", | 20260506 09:44:39.750 | INFO | "huge" : "", | 20260506 09:44:39.750 | INFO | "huge_shared" : " 27 GB", | 20260506 09:44:39.750 | INFO | "malloc" : " 14 GB", | 20260506 09:44:39.750 | INFO | "pinned" : " 29 GB" | 20260506 09:44:39.750 | INFO | } | 20260506 09:44:39.750 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:44:39.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:44:39.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:39.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:39.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:39.750 | INFO | ANALYSIS : 57.7%, 2.9, 87.5ms, 0 | 20260506 09:44:39.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:39.750 | INFO | RT_METRICS : 47.8%, 1.0, 25.1ms, 0 | 20260506 09:44:39.750 | INFO | PRE_HQ : 56.8%, 2.1, 61.3ms, 0 | 20260506 09:44:39.750 | INFO | BAZWRITER : 62.4%, 4.5, 145.9ms, 0 | 20260506 09:44:39.750 | INFO | BAZ_SAMPLING : 3.1%, 1.0, 1.6ms, 0 | 20260506 09:44:39.751 | INFO | Uploaded 36GiB to GPU in 1.65469s (21.7563GiB/s) | 20260506 09:44:39.751 | INFO | Downloaded 38GiB to GPU in 1.71805s (22.1181GiB/s) | 20260506 09:44:44.889 | INFO | Analyzing chunk frames = [377344,377856) | 20260506 09:44:44.889 | INFO | MemInfo: { | 20260506 09:44:44.889 | INFO | "gpu" : " 17 GB", | 20260506 09:44:44.889 | INFO | "huge" : "", | 20260506 09:44:44.889 | INFO | "huge_shared" : " 27 GB", | 20260506 09:44:44.889 | INFO | "malloc" : " 14 GB", | 20260506 09:44:44.889 | INFO | "pinned" : " 29 GB" | 20260506 09:44:44.889 | INFO | } | 20260506 09:44:44.890 | INFO | Perf report for the last 5.13945 seconds: | 20260506 09:44:44.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:44:44.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:44.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:44.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:44.890 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 09:44:44.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:44.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:44.890 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:44:44.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:44.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:44.890 | INFO | Uploaded 36GiB to GPU in 1.60268s (22.4623GiB/s) | 20260506 09:44:44.890 | INFO | Downloaded 34GiB to GPU in 1.51962s (22.3739GiB/s) | 20260506 09:44:50.029 | INFO | Analyzing chunk frames = [377856,378368) | 20260506 09:44:50.029 | INFO | MemInfo: { | 20260506 09:44:50.029 | INFO | "gpu" : " 17 GB", | 20260506 09:44:50.029 | INFO | "huge" : "", | 20260506 09:44:50.029 | INFO | "huge_shared" : " 27 GB", | 20260506 09:44:50.029 | INFO | "malloc" : " 14 GB", | 20260506 09:44:50.029 | INFO | "pinned" : " 29 GB" | 20260506 09:44:50.029 | INFO | } | 20260506 09:44:50.029 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:44:50.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:44:50.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:50.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:50.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:50.029 | INFO | ANALYSIS : 56.9%, 3.0, 88.5ms, 0 | 20260506 09:44:50.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:50.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:50.029 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:44:50.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:50.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:50.030 | INFO | Uploaded 36GiB to GPU in 1.60593s (22.4169GiB/s) | 20260506 09:44:50.030 | INFO | Downloaded 34GiB to GPU in 1.52408s (22.3086GiB/s) | 20260506 09:44:55.168 | INFO | Analyzing chunk frames = [378368,378880) | 20260506 09:44:55.168 | INFO | MemInfo: { | 20260506 09:44:55.168 | INFO | "gpu" : " 17 GB", | 20260506 09:44:55.168 | INFO | "huge" : "", | 20260506 09:44:55.168 | INFO | "huge_shared" : " 27 GB", | 20260506 09:44:55.168 | INFO | "malloc" : " 14 GB", | 20260506 09:44:55.168 | INFO | "pinned" : " 29 GB" | 20260506 09:44:55.168 | INFO | } | 20260506 09:44:55.168 | INFO | Perf report for the last 5.13918 seconds: | 20260506 09:44:55.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:44:55.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:55.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:55.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:55.168 | INFO | ANALYSIS : 56.6%, 3.0, 87.7ms, 0 | 20260506 09:44:55.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:55.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:55.168 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:44:55.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:55.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:44:55.168 | INFO | Uploaded 36GiB to GPU in 1.6058s (22.4188GiB/s) | 20260506 09:44:55.169 | INFO | Downloaded 34GiB to GPU in 1.52338s (22.3188GiB/s) | 20260506 09:45:00.307 | INFO | Analyzing chunk frames = [378880,379392) | 20260506 09:45:00.307 | INFO | MemInfo: { | 20260506 09:45:00.307 | INFO | "gpu" : " 17 GB", | 20260506 09:45:00.307 | INFO | "huge" : "", | 20260506 09:45:00.307 | INFO | "huge_shared" : " 27 GB", | 20260506 09:45:00.307 | INFO | "malloc" : " 14 GB", | 20260506 09:45:00.307 | INFO | "pinned" : " 29 GB" | 20260506 09:45:00.308 | INFO | } | 20260506 09:45:00.308 | INFO | Perf report for the last 5.13953 seconds: | 20260506 09:45:00.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:45:00.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:00.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:00.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:00.308 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 09:45:00.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:00.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:00.308 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:45:00.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:00.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:00.308 | INFO | Uploaded 36GiB to GPU in 1.60406s (22.4431GiB/s) | 20260506 09:45:00.308 | INFO | Downloaded 34GiB to GPU in 1.52036s (22.3631GiB/s) | 20260506 09:45:05.447 | INFO | Analyzing chunk frames = [379392,379904) | 20260506 09:45:05.447 | INFO | MemInfo: { | 20260506 09:45:05.447 | INFO | "gpu" : " 17 GB", | 20260506 09:45:05.447 | INFO | "huge" : "", | 20260506 09:45:05.447 | INFO | "huge_shared" : " 27 GB", | 20260506 09:45:05.447 | INFO | "malloc" : " 14 GB", | 20260506 09:45:05.447 | INFO | "pinned" : " 29 GB" | 20260506 09:45:05.447 | INFO | } | 20260506 09:45:05.447 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:45:05.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:45:05.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:05.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:05.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:05.447 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:45:05.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:05.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:05.447 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:45:05.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:05.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:05.447 | INFO | Uploaded 36GiB to GPU in 1.605s (22.4299GiB/s) | 20260506 09:45:05.447 | INFO | Downloaded 34GiB to GPU in 1.52157s (22.3454GiB/s) | 20260506 09:45:10.586 | INFO | Analyzing chunk frames = [379904,380416) | 20260506 09:45:10.586 | INFO | MemInfo: { | 20260506 09:45:10.586 | INFO | "gpu" : " 17 GB", | 20260506 09:45:10.586 | INFO | "huge" : "", | 20260506 09:45:10.586 | INFO | "huge_shared" : " 27 GB", | 20260506 09:45:10.586 | INFO | "malloc" : " 14 GB", | 20260506 09:45:10.586 | INFO | "pinned" : " 29 GB" | 20260506 09:45:10.586 | INFO | } | 20260506 09:45:10.586 | INFO | Perf report for the last 5.13913 seconds: | 20260506 09:45:10.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:45:10.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:10.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:10.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:10.586 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260506 09:45:10.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:10.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:10.586 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:45:10.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:10.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:10.586 | INFO | Uploaded 36GiB to GPU in 1.60375s (22.4474GiB/s) | 20260506 09:45:10.586 | INFO | Downloaded 34GiB to GPU in 1.52187s (22.3409GiB/s) | 20260506 09:45:15.725 | INFO | Analyzing chunk frames = [380416,380928) | 20260506 09:45:15.725 | INFO | MemInfo: { | 20260506 09:45:15.725 | INFO | "gpu" : " 17 GB", | 20260506 09:45:15.725 | INFO | "huge" : "", | 20260506 09:45:15.725 | INFO | "huge_shared" : " 27 GB", | 20260506 09:45:15.725 | INFO | "malloc" : " 14 GB", | 20260506 09:45:15.725 | INFO | "pinned" : " 29 GB" | 20260506 09:45:15.725 | INFO | } | 20260506 09:45:15.726 | INFO | Perf report for the last 5.13949 seconds: | 20260506 09:45:15.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:45:15.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:15.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:15.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:15.726 | INFO | ANALYSIS : 57.1%, 3.0, 88.1ms, 0 | 20260506 09:45:15.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:15.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:15.726 | INFO | PRE_HQ : 29.0%, 1.0, 14.9ms, 0 | 20260506 09:45:15.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:15.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:15.726 | INFO | Uploaded 36GiB to GPU in 1.60441s (22.4382GiB/s) | 20260506 09:45:15.726 | INFO | Downloaded 34GiB to GPU in 1.52318s (22.3217GiB/s) | 20260506 09:45:15.823 | INFO | Starting a round of Baz IO | 20260506 09:45:18.745 | INFO | Populating RTMetrics report | 20260506 09:45:18.745 | INFO | RTMetrics report populated | 20260506 09:45:18.745 | INFO | Opening tmp RTMetrics json file | 20260506 09:45:18.759 | INFO | Serializing RTMetrics to json | 20260506 09:45:18.782 | INFO | Serialization done, swapping tmp file now... | 20260506 09:45:18.786 | INFO | RTMetrics done | 20260506 09:45:18.850 | INFO | Finished a round of Baz IO: Wrote 1.898 GiB in 3.026 seconds | 20260506 09:45:18.850 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.87% / 0.08% / 48.05% | 20260506 09:45:18.850 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.56749% / 0.000242392% / 4.37101e-05% | 20260506 09:45:18.850 | INFO | Baz IO overall: | 20260506 09:45:18.850 | INFO | BytesWritten : 156146673273 | 20260506 09:45:18.850 | INFO | NumEvents : 74407147443 | 20260506 09:45:20.865 | INFO | Analyzing chunk frames = [380928,381440) | 20260506 09:45:20.865 | INFO | MemInfo: { | 20260506 09:45:20.865 | INFO | "gpu" : " 17 GB", | 20260506 09:45:20.865 | INFO | "huge" : "", | 20260506 09:45:20.865 | INFO | "huge_shared" : " 27 GB", | 20260506 09:45:20.865 | INFO | "malloc" : " 14 GB", | 20260506 09:45:20.865 | INFO | "pinned" : " 29 GB" | 20260506 09:45:20.865 | INFO | } | 20260506 09:45:20.865 | INFO | Perf report for the last 5.13948 seconds: | 20260506 09:45:20.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:45:20.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:20.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:20.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:20.865 | INFO | ANALYSIS : 57.7%, 2.9, 86.6ms, 0 | 20260506 09:45:20.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:20.865 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 09:45:20.865 | INFO | PRE_HQ : 56.9%, 2.1, 61.2ms, 0 | 20260506 09:45:20.865 | INFO | BAZWRITER : 58.4%, 4.7, 139.8ms, 0 | 20260506 09:45:20.865 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 09:45:20.865 | INFO | Uploaded 36GiB to GPU in 1.66478s (21.6245GiB/s) | 20260506 09:45:20.865 | INFO | Downloaded 38GiB to GPU in 1.72922s (21.9752GiB/s) | 20260506 09:45:26.004 | INFO | Analyzing chunk frames = [381440,381952) | 20260506 09:45:26.004 | INFO | MemInfo: { | 20260506 09:45:26.004 | INFO | "gpu" : " 17 GB", | 20260506 09:45:26.004 | INFO | "huge" : "", | 20260506 09:45:26.004 | INFO | "huge_shared" : " 27 GB", | 20260506 09:45:26.004 | INFO | "malloc" : " 14 GB", | 20260506 09:45:26.004 | INFO | "pinned" : " 29 GB" | 20260506 09:45:26.004 | INFO | } | 20260506 09:45:26.004 | INFO | Perf report for the last 5.13917 seconds: | 20260506 09:45:26.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:45:26.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:26.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:26.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:26.004 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 09:45:26.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:26.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:26.004 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 09:45:26.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:26.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:26.004 | INFO | Uploaded 36GiB to GPU in 1.60278s (22.461GiB/s) | 20260506 09:45:26.005 | INFO | Downloaded 34GiB to GPU in 1.51942s (22.3769GiB/s) | 20260506 09:45:31.143 | INFO | Analyzing chunk frames = [381952,382464) | 20260506 09:45:31.143 | INFO | MemInfo: { | 20260506 09:45:31.143 | INFO | "gpu" : " 17 GB", | 20260506 09:45:31.143 | INFO | "huge" : "", | 20260506 09:45:31.143 | INFO | "huge_shared" : " 27 GB", | 20260506 09:45:31.143 | INFO | "malloc" : " 14 GB", | 20260506 09:45:31.144 | INFO | "pinned" : " 29 GB" | 20260506 09:45:31.144 | INFO | } | 20260506 09:45:31.144 | INFO | Perf report for the last 5.13932 seconds: | 20260506 09:45:31.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:45:31.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:31.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:31.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:31.144 | INFO | ANALYSIS : 56.9%, 3.0, 88.5ms, 0 | 20260506 09:45:31.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:31.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:31.144 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260506 09:45:31.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:31.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:31.144 | INFO | Uploaded 36GiB to GPU in 1.60627s (22.4122GiB/s) | 20260506 09:45:31.144 | INFO | Downloaded 34GiB to GPU in 1.52452s (22.3021GiB/s) | 20260506 09:45:36.283 | INFO | Analyzing chunk frames = [382464,382976) | 20260506 09:45:36.283 | INFO | MemInfo: { | 20260506 09:45:36.283 | INFO | "gpu" : " 17 GB", | 20260506 09:45:36.283 | INFO | "huge" : "", | 20260506 09:45:36.283 | INFO | "huge_shared" : " 27 GB", | 20260506 09:45:36.283 | INFO | "malloc" : " 14 GB", | 20260506 09:45:36.283 | INFO | "pinned" : " 29 GB" | 20260506 09:45:36.283 | INFO | } | 20260506 09:45:36.283 | INFO | Perf report for the last 5.13938 seconds: | 20260506 09:45:36.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:45:36.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:36.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:36.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:36.283 | INFO | ANALYSIS : 56.6%, 3.0, 87.7ms, 0 | 20260506 09:45:36.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:36.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:36.283 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:45:36.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:36.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:36.283 | INFO | Uploaded 36GiB to GPU in 1.606s (22.4159GiB/s) | 20260506 09:45:36.283 | INFO | Downloaded 34GiB to GPU in 1.52301s (22.3241GiB/s) | 20260506 09:45:41.422 | INFO | Analyzing chunk frames = [382976,383488) | 20260506 09:45:41.422 | INFO | MemInfo: { | 20260506 09:45:41.422 | INFO | "gpu" : " 17 GB", | 20260506 09:45:41.422 | INFO | "huge" : "", | 20260506 09:45:41.422 | INFO | "huge_shared" : " 27 GB", | 20260506 09:45:41.422 | INFO | "malloc" : " 14 GB", | 20260506 09:45:41.422 | INFO | "pinned" : " 29 GB" | 20260506 09:45:41.422 | INFO | } | 20260506 09:45:41.422 | INFO | Perf report for the last 5.13946 seconds: | 20260506 09:45:41.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:45:41.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:41.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:41.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:41.423 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 09:45:41.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:41.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:41.423 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:45:41.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:41.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:41.423 | INFO | Uploaded 36GiB to GPU in 1.60374s (22.4476GiB/s) | 20260506 09:45:41.423 | INFO | Downloaded 34GiB to GPU in 1.52129s (22.3494GiB/s) | 20260506 09:45:46.561 | INFO | Analyzing chunk frames = [383488,384000) | 20260506 09:45:46.561 | INFO | MemInfo: { | 20260506 09:45:46.561 | INFO | "gpu" : " 17 GB", | 20260506 09:45:46.561 | INFO | "huge" : "", | 20260506 09:45:46.561 | INFO | "huge_shared" : " 27 GB", | 20260506 09:45:46.561 | INFO | "malloc" : " 14 GB", | 20260506 09:45:46.562 | INFO | "pinned" : " 29 GB" | 20260506 09:45:46.562 | INFO | } | 20260506 09:45:46.562 | INFO | Perf report for the last 5.13914 seconds: | 20260506 09:45:46.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:45:46.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:46.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:46.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:46.562 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:45:46.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:46.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:46.562 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:45:46.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:46.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:46.562 | INFO | Uploaded 36GiB to GPU in 1.60538s (22.4246GiB/s) | 20260506 09:45:46.562 | INFO | Downloaded 34GiB to GPU in 1.52227s (22.3351GiB/s) | 20260506 09:45:51.701 | INFO | Analyzing chunk frames = [384000,384512) | 20260506 09:45:51.701 | INFO | MemInfo: { | 20260506 09:45:51.701 | INFO | "gpu" : " 17 GB", | 20260506 09:45:51.701 | INFO | "huge" : "", | 20260506 09:45:51.701 | INFO | "huge_shared" : " 27 GB", | 20260506 09:45:51.701 | INFO | "malloc" : " 14 GB", | 20260506 09:45:51.701 | INFO | "pinned" : " 29 GB" | 20260506 09:45:51.701 | INFO | } | 20260506 09:45:51.701 | INFO | Perf report for the last 5.1394 seconds: | 20260506 09:45:51.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:45:51.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:51.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:51.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:51.701 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:45:51.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:51.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:51.701 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:45:51.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:51.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:51.701 | INFO | Uploaded 36GiB to GPU in 1.60391s (22.4451GiB/s) | 20260506 09:45:51.701 | INFO | Downloaded 34GiB to GPU in 1.52209s (22.3377GiB/s) | 20260506 09:45:56.840 | INFO | Analyzing chunk frames = [384512,385024) | 20260506 09:45:56.840 | INFO | MemInfo: { | 20260506 09:45:56.840 | INFO | "gpu" : " 17 GB", | 20260506 09:45:56.840 | INFO | "huge" : "", | 20260506 09:45:56.840 | INFO | "huge_shared" : " 27 GB", | 20260506 09:45:56.840 | INFO | "malloc" : " 14 GB", | 20260506 09:45:56.840 | INFO | "pinned" : " 29 GB" | 20260506 09:45:56.840 | INFO | } | 20260506 09:45:56.840 | INFO | Perf report for the last 5.13926 seconds: | 20260506 09:45:56.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:45:56.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:56.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:56.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:56.840 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:45:56.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:56.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:56.840 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 09:45:56.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:56.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:45:56.841 | INFO | Uploaded 36GiB to GPU in 1.60518s (22.4274GiB/s) | 20260506 09:45:56.841 | INFO | Downloaded 34GiB to GPU in 1.52388s (22.3114GiB/s) | 20260506 09:45:56.841 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:45:56.841 | INFO | Total mem usage: 15.9 GiB | 20260506 09:45:56.841 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 2.48%) | 20260506 09:45:56.841 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -6.51%) | 20260506 09:45:56.841 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:45:56.841 | INFO | High Water Breakdown: | 20260506 09:45:56.841 | INFO | 59.2% : 8.84 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:45:56.841 | INFO | 40.7% : 6.07 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:45:56.841 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:45:56.841 | INFO | ------------------------------------------ | 20260506 09:45:56.841 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:45:56.841 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:45:56.841 | INFO | Total mem usage: 26.5 GiB | 20260506 09:45:56.841 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:45:56.841 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:45:56.841 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:45:56.841 | INFO | High Water Breakdown: | 20260506 09:45:56.841 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:45:56.841 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:45:56.841 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:45:56.841 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:45:56.841 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:45:56.841 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:45:56.841 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:45:56.841 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:45:56.841 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:45:56.841 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:45:56.841 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:45:56.841 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:45:56.841 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:45:56.841 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:45:56.841 | INFO | ------------------------------------------ | 20260506 09:45:56.841 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:45:56.841 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:45:56.841 | INFO | Total mem usage: 25.6 GiB | 20260506 09:45:56.841 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:45:56.841 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:45:56.841 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:45:56.841 | INFO | High Water Breakdown: | 20260506 09:45:56.841 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:45:56.841 | INFO | ------------------------------------------ | 20260506 09:45:56.842 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:45:56.842 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:45:56.842 | INFO | Total mem usage: 15.7 GiB | 20260506 09:45:56.842 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:45:56.842 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:45:56.842 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:45:56.842 | INFO | High Water Breakdown: | 20260506 09:45:56.842 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:45:56.842 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:45:56.842 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:45:56.842 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:45:56.842 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:45:56.842 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:45:56.842 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:45:56.842 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:45:56.842 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:45:56.842 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:45:56.842 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:45:56.842 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:45:56.842 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:45:56.842 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:45:56.842 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:45:56.842 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:45:56.842 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:45:56.842 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:45:56.842 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:45:56.842 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:45:56.842 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:45:56.842 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:45:56.842 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:45:56.842 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:45:56.842 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:45:56.842 | INFO | ------------------------------------------ | 20260506 09:45:56.842 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:45:56.842 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:45:56.842 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:45:56.842 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:45:56.842 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 09:45:56.842 | INFO | PerfReport: 0.29%, AggregateStats -- 8.60e-02 / -nan, 4.59e-03 / 0.00e+00 | 20260506 09:45:56.842 | INFO | PerfReport: 10.91%, Baseline -- 3.29e+00 / -nan, 1.80e-02 / 0.00e+00 | 20260506 09:45:56.842 | INFO | PerfReport: 3.05%, Histogram -- 9.21e-01 / -nan, 5.17e-02 / 0.00e+00 | 20260506 09:45:56.842 | INFO | PerfReport: 16.04%, DME -- 3.87e+01 / -nan, 3.04e+00 / 0.00e+00 | 20260506 09:45:56.842 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.72e+01 / -nan, 4.17e-02 / 0.00e+00 | 20260506 09:45:56.842 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.26e+00 / -nan, 1.34e-01 / 0.00e+00 | 20260506 09:45:56.842 | INFO | PerfReport: 3.79%, Metrics -- 1.14e+00 / -nan, 1.78e-01 / 0.00e+00 | 20260506 09:45:56.842 | INFO | PerfReport: 1.47%, PulseSerializing -- 4.44e-01 / -nan, 6.40e-02 / 0.00e+00 | 20260506 09:45:56.842 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:45:56.842 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:45:56.842 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:45:56.842 | INFO | PerfReport: Full Execution -- 9.03e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 09:45:56.842 | INFO | PerfReport: 18.59%, Upload -- 1.68e+01 / 0.00e+00, 4.52e-01 / 0.00e+00 | 20260506 09:45:56.842 | INFO | PerfReport: 17.85%, Download -- 1.61e+01 / 0.00e+00, 7.91e-01 / 0.00e+00 | 20260506 09:45:56.842 | INFO | PerfReport: 33.43%, Compute -- 3.02e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 09:45:56.842 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:45:56.842 | INFO | PerfReport: 0.00%, LockDownload -- 5.03e-04 / 0.00e+00, 1.15e-04 / 0.00e+00 | 20260506 09:45:56.842 | INFO | PerfReport: 29.55%, LockCompute -- 2.67e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 09:45:56.842 | INFO | PerfReport: 0.02%, Misc -- 1.38e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260506 09:45:56.937 | INFO | Starting a round of Baz IO | 20260506 09:45:59.859 | INFO | Populating RTMetrics report | 20260506 09:45:59.860 | INFO | RTMetrics report populated | 20260506 09:45:59.860 | INFO | Opening tmp RTMetrics json file | 20260506 09:45:59.882 | INFO | Serializing RTMetrics to json | 20260506 09:45:59.884 | INFO | Serialization done, swapping tmp file now... | 20260506 09:45:59.888 | INFO | RTMetrics done | 20260506 09:46:00.090 | INFO | Finished a round of Baz IO: Wrote 1.899 GiB in 3.152 seconds | 20260506 09:46:00.090 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.79% / 0.08% / 48.13% | 20260506 09:46:00.090 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.52304% / 0.000234445% / 5.16574e-05% | 20260506 09:46:00.090 | INFO | Baz IO overall: | 20260506 09:46:00.090 | INFO | BytesWritten : 158185864064 | 20260506 09:46:00.090 | INFO | NumEvents : 75396310784 | 20260506 09:46:01.980 | INFO | Analyzing chunk frames = [385024,385536) | 20260506 09:46:01.980 | INFO | MemInfo: { | 20260506 09:46:01.980 | INFO | "gpu" : " 17 GB", | 20260506 09:46:01.980 | INFO | "huge" : "", | 20260506 09:46:01.980 | INFO | "huge_shared" : " 27 GB", | 20260506 09:46:01.980 | INFO | "malloc" : " 14 GB", | 20260506 09:46:01.980 | INFO | "pinned" : " 29 GB" | 20260506 09:46:01.980 | INFO | } | 20260506 09:46:01.980 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:46:01.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:46:01.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:01.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:01.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:01.980 | INFO | ANALYSIS : 57.8%, 2.8, 85.2ms, 0 | 20260506 09:46:01.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:01.980 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 09:46:01.980 | INFO | PRE_HQ : 56.9%, 2.1, 61.5ms, 0 | 20260506 09:46:01.980 | INFO | BAZWRITER : 61.0%, 4.8, 149.5ms, 0 | 20260506 09:46:01.980 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 09:46:01.980 | INFO | Uploaded 36GiB to GPU in 1.66931s (21.5659GiB/s) | 20260506 09:46:01.980 | INFO | Downloaded 38GiB to GPU in 1.73532s (21.898GiB/s) | 20260506 09:46:07.119 | INFO | Analyzing chunk frames = [385536,386048) | 20260506 09:46:07.119 | INFO | MemInfo: { | 20260506 09:46:07.119 | INFO | "gpu" : " 17 GB", | 20260506 09:46:07.119 | INFO | "huge" : "", | 20260506 09:46:07.119 | INFO | "huge_shared" : " 27 GB", | 20260506 09:46:07.119 | INFO | "malloc" : " 14 GB", | 20260506 09:46:07.119 | INFO | "pinned" : " 29 GB" | 20260506 09:46:07.119 | INFO | } | 20260506 09:46:07.119 | INFO | Perf report for the last 5.13962 seconds: | 20260506 09:46:07.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:46:07.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:07.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:07.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:07.119 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 09:46:07.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:07.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:07.119 | INFO | PRE_HQ : 30.2%, 1.0, 16.1ms, 0 | 20260506 09:46:07.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:07.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:07.120 | INFO | Uploaded 36GiB to GPU in 1.60235s (22.467GiB/s) | 20260506 09:46:07.120 | INFO | Downloaded 34GiB to GPU in 1.52008s (22.3672GiB/s) | 20260506 09:46:12.262 | INFO | Analyzing chunk frames = [386048,386560) | 20260506 09:46:12.262 | INFO | MemInfo: { | 20260506 09:46:12.262 | INFO | "gpu" : " 17 GB", | 20260506 09:46:12.262 | INFO | "huge" : "", | 20260506 09:46:12.262 | INFO | "huge_shared" : " 27 GB", | 20260506 09:46:12.262 | INFO | "malloc" : " 14 GB", | 20260506 09:46:12.262 | INFO | "pinned" : " 29 GB" | 20260506 09:46:12.262 | INFO | } | 20260506 09:46:12.262 | INFO | Perf report for the last 5.14277 seconds: | 20260506 09:46:12.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:46:12.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:12.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:12.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:12.262 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:46:12.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:12.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:12.262 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:46:12.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:12.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:12.262 | INFO | Uploaded 36GiB to GPU in 1.60646s (22.4096GiB/s) | 20260506 09:46:12.262 | INFO | Downloaded 34GiB to GPU in 1.5243s (22.3053GiB/s) | 20260506 09:46:17.418 | INFO | Analyzing chunk frames = [386560,387072) | 20260506 09:46:17.418 | INFO | MemInfo: { | 20260506 09:46:17.418 | INFO | "gpu" : " 17 GB", | 20260506 09:46:17.418 | INFO | "huge" : "", | 20260506 09:46:17.418 | INFO | "huge_shared" : " 27 GB", | 20260506 09:46:17.418 | INFO | "malloc" : " 14 GB", | 20260506 09:46:17.418 | INFO | "pinned" : " 29 GB" | 20260506 09:46:17.418 | INFO | } | 20260506 09:46:17.418 | INFO | Perf report for the last 5.15641 seconds: | 20260506 09:46:17.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:46:17.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:17.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:17.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:17.419 | INFO | ANALYSIS : 56.4%, 3.0, 87.8ms, 0 | 20260506 09:46:17.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:17.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:17.419 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260506 09:46:17.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:17.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:17.419 | INFO | Uploaded 36GiB to GPU in 1.60643s (22.4099GiB/s) | 20260506 09:46:17.419 | INFO | Downloaded 34GiB to GPU in 1.52406s (22.3089GiB/s) | 20260506 09:46:22.537 | INFO | Analyzing chunk frames = [387072,387584) | 20260506 09:46:22.537 | INFO | MemInfo: { | 20260506 09:46:22.537 | INFO | "gpu" : " 17 GB", | 20260506 09:46:22.537 | INFO | "huge" : "", | 20260506 09:46:22.537 | INFO | "huge_shared" : " 27 GB", | 20260506 09:46:22.537 | INFO | "malloc" : " 14 GB", | 20260506 09:46:22.537 | INFO | "pinned" : " 29 GB" | 20260506 09:46:22.537 | INFO | } | 20260506 09:46:22.537 | INFO | Perf report for the last 5.11871 seconds: | 20260506 09:46:22.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:46:22.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:22.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:22.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:22.537 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 09:46:22.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:22.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:22.537 | INFO | PRE_HQ : 30.7%, 1.0, 15.8ms, 0 | 20260506 09:46:22.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:22.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:22.538 | INFO | Uploaded 36GiB to GPU in 1.60415s (22.4418GiB/s) | 20260506 09:46:22.538 | INFO | Downloaded 34GiB to GPU in 1.52295s (22.325GiB/s) | 20260506 09:46:27.676 | INFO | Analyzing chunk frames = [387584,388096) | 20260506 09:46:27.676 | INFO | MemInfo: { | 20260506 09:46:27.676 | INFO | "gpu" : " 17 GB", | 20260506 09:46:27.676 | INFO | "huge" : "", | 20260506 09:46:27.676 | INFO | "huge_shared" : " 27 GB", | 20260506 09:46:27.676 | INFO | "malloc" : " 14 GB", | 20260506 09:46:27.676 | INFO | "pinned" : " 29 GB" | 20260506 09:46:27.676 | INFO | } | 20260506 09:46:27.676 | INFO | Perf report for the last 5.13922 seconds: | 20260506 09:46:27.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:46:27.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:27.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:27.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:27.676 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:46:27.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:27.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:27.677 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:46:27.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:27.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:27.677 | INFO | Uploaded 36GiB to GPU in 1.60446s (22.4374GiB/s) | 20260506 09:46:27.677 | INFO | Downloaded 34GiB to GPU in 1.52124s (22.3502GiB/s) | 20260506 09:46:32.815 | INFO | Analyzing chunk frames = [388096,388608) | 20260506 09:46:32.815 | INFO | MemInfo: { | 20260506 09:46:32.815 | INFO | "gpu" : " 17 GB", | 20260506 09:46:32.815 | INFO | "huge" : "", | 20260506 09:46:32.816 | INFO | "huge_shared" : " 27 GB", | 20260506 09:46:32.816 | INFO | "malloc" : " 14 GB", | 20260506 09:46:32.816 | INFO | "pinned" : " 29 GB" | 20260506 09:46:32.816 | INFO | } | 20260506 09:46:32.816 | INFO | Perf report for the last 5.13924 seconds: | 20260506 09:46:32.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:46:32.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:32.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:32.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:32.816 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:46:32.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:32.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:32.816 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 09:46:32.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:32.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:32.816 | INFO | Uploaded 36GiB to GPU in 1.60535s (22.425GiB/s) | 20260506 09:46:32.816 | INFO | Downloaded 34GiB to GPU in 1.52328s (22.3203GiB/s) | 20260506 09:46:37.955 | INFO | Analyzing chunk frames = [388608,389120) | 20260506 09:46:37.955 | INFO | MemInfo: { | 20260506 09:46:37.955 | INFO | "gpu" : " 17 GB", | 20260506 09:46:37.955 | INFO | "huge" : "", | 20260506 09:46:37.955 | INFO | "huge_shared" : " 27 GB", | 20260506 09:46:37.955 | INFO | "malloc" : " 14 GB", | 20260506 09:46:37.955 | INFO | "pinned" : " 29 GB" | 20260506 09:46:37.955 | INFO | } | 20260506 09:46:37.955 | INFO | Perf report for the last 5.13938 seconds: | 20260506 09:46:37.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:46:37.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:37.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:37.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:37.955 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:46:37.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:37.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:37.955 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:46:37.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:37.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:37.955 | INFO | Uploaded 36GiB to GPU in 1.6044s (22.4382GiB/s) | 20260506 09:46:37.955 | INFO | Downloaded 34GiB to GPU in 1.5228s (22.3273GiB/s) | 20260506 09:46:38.053 | INFO | Starting a round of Baz IO | 20260506 09:46:40.969 | INFO | Populating RTMetrics report | 20260506 09:46:40.969 | INFO | RTMetrics report populated | 20260506 09:46:40.969 | INFO | Opening tmp RTMetrics json file | 20260506 09:46:41.019 | INFO | Serializing RTMetrics to json | 20260506 09:46:41.091 | INFO | Serialization done, swapping tmp file now... | 20260506 09:46:41.096 | INFO | RTMetrics done | 20260506 09:46:41.493 | INFO | Finished a round of Baz IO: Wrote 1.9 GiB in 3.439 seconds | 20260506 09:46:41.493 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.71% / 0.08% / 48.21% | 20260506 09:46:41.493 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.49458% / 0.000214577% / 4.76837e-05% | 20260506 09:46:41.493 | INFO | Baz IO overall: | 20260506 09:46:41.493 | INFO | BytesWritten : 160226470541 | 20260506 09:46:41.493 | INFO | NumEvents : 76386198081 | 20260506 09:46:43.094 | INFO | Analyzing chunk frames = [389120,389632) | 20260506 09:46:43.094 | INFO | MemInfo: { | 20260506 09:46:43.094 | INFO | "gpu" : " 17 GB", | 20260506 09:46:43.094 | INFO | "huge" : "", | 20260506 09:46:43.094 | INFO | "huge_shared" : " 27 GB", | 20260506 09:46:43.095 | INFO | "malloc" : " 14 GB", | 20260506 09:46:43.095 | INFO | "pinned" : " 29 GB" | 20260506 09:46:43.095 | INFO | } | 20260506 09:46:43.095 | INFO | Perf report for the last 5.13957 seconds: | 20260506 09:46:43.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:46:43.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:43.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:43.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:43.095 | INFO | ANALYSIS : 57.7%, 2.9, 86.3ms, 0 | 20260506 09:46:43.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:43.095 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260506 09:46:43.095 | INFO | PRE_HQ : 56.8%, 2.1, 61.3ms, 0 | 20260506 09:46:43.095 | INFO | BAZWRITER : 66.5%, 4.7, 160.3ms, 0 | 20260506 09:46:43.095 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.4ms, 0 | 20260506 09:46:43.095 | INFO | Uploaded 36GiB to GPU in 1.66746s (21.5897GiB/s) | 20260506 09:46:43.095 | INFO | Downloaded 38GiB to GPU in 1.73172s (21.9434GiB/s) | 20260506 09:46:48.234 | INFO | Analyzing chunk frames = [389632,390144) | 20260506 09:46:48.234 | INFO | MemInfo: { | 20260506 09:46:48.234 | INFO | "gpu" : " 17 GB", | 20260506 09:46:48.234 | INFO | "huge" : "", | 20260506 09:46:48.234 | INFO | "huge_shared" : " 27 GB", | 20260506 09:46:48.234 | INFO | "malloc" : " 14 GB", | 20260506 09:46:48.234 | INFO | "pinned" : " 29 GB" | 20260506 09:46:48.234 | INFO | } | 20260506 09:46:48.234 | INFO | Perf report for the last 5.13915 seconds: | 20260506 09:46:48.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:46:48.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:48.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:48.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:48.234 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 09:46:48.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:48.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:48.234 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:46:48.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:48.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:48.234 | INFO | Uploaded 36GiB to GPU in 1.60345s (22.4515GiB/s) | 20260506 09:46:48.235 | INFO | Downloaded 34GiB to GPU in 1.52013s (22.3665GiB/s) | 20260506 09:46:53.373 | INFO | Analyzing chunk frames = [390144,390656) | 20260506 09:46:53.373 | INFO | MemInfo: { | 20260506 09:46:53.373 | INFO | "gpu" : " 17 GB", | 20260506 09:46:53.373 | INFO | "huge" : "", | 20260506 09:46:53.373 | INFO | "huge_shared" : " 27 GB", | 20260506 09:46:53.373 | INFO | "malloc" : " 14 GB", | 20260506 09:46:53.373 | INFO | "pinned" : " 29 GB" | 20260506 09:46:53.373 | INFO | } | 20260506 09:46:53.373 | INFO | Perf report for the last 5.13952 seconds: | 20260506 09:46:53.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:46:53.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:53.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:53.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:53.373 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:46:53.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:53.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:53.373 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:46:53.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:53.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:53.374 | INFO | Uploaded 36GiB to GPU in 1.60683s (22.4044GiB/s) | 20260506 09:46:53.374 | INFO | Downloaded 34GiB to GPU in 1.52459s (22.3011GiB/s) | 20260506 09:46:58.512 | INFO | Analyzing chunk frames = [390656,391168) | 20260506 09:46:58.512 | INFO | MemInfo: { | 20260506 09:46:58.512 | INFO | "gpu" : " 17 GB", | 20260506 09:46:58.512 | INFO | "huge" : "", | 20260506 09:46:58.512 | INFO | "huge_shared" : " 27 GB", | 20260506 09:46:58.512 | INFO | "malloc" : " 14 GB", | 20260506 09:46:58.512 | INFO | "pinned" : " 29 GB" | 20260506 09:46:58.513 | INFO | } | 20260506 09:46:58.513 | INFO | Perf report for the last 5.13924 seconds: | 20260506 09:46:58.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:46:58.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:58.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:58.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:58.513 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:46:58.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:58.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:58.513 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:46:58.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:58.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:46:58.513 | INFO | Uploaded 36GiB to GPU in 1.60549s (22.423GiB/s) | 20260506 09:46:58.513 | INFO | Downloaded 34GiB to GPU in 1.52283s (22.3268GiB/s) | 20260506 09:47:03.652 | INFO | Analyzing chunk frames = [391168,391680) | 20260506 09:47:03.652 | INFO | MemInfo: { | 20260506 09:47:03.652 | INFO | "gpu" : " 17 GB", | 20260506 09:47:03.652 | INFO | "huge" : "", | 20260506 09:47:03.652 | INFO | "huge_shared" : " 27 GB", | 20260506 09:47:03.652 | INFO | "malloc" : " 14 GB", | 20260506 09:47:03.652 | INFO | "pinned" : " 29 GB" | 20260506 09:47:03.652 | INFO | } | 20260506 09:47:03.652 | INFO | Perf report for the last 5.13919 seconds: | 20260506 09:47:03.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:47:03.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:03.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:03.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:03.652 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 09:47:03.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:03.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:03.652 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:47:03.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:03.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:03.652 | INFO | Uploaded 36GiB to GPU in 1.60503s (22.4295GiB/s) | 20260506 09:47:03.652 | INFO | Downloaded 34GiB to GPU in 1.52232s (22.3343GiB/s) | 20260506 09:47:08.791 | INFO | Analyzing chunk frames = [391680,392192) | 20260506 09:47:08.791 | INFO | MemInfo: { | 20260506 09:47:08.791 | INFO | "gpu" : " 17 GB", | 20260506 09:47:08.791 | INFO | "huge" : "", | 20260506 09:47:08.791 | INFO | "huge_shared" : " 27 GB", | 20260506 09:47:08.791 | INFO | "malloc" : " 14 GB", | 20260506 09:47:08.791 | INFO | "pinned" : " 29 GB" | 20260506 09:47:08.791 | INFO | } | 20260506 09:47:08.791 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:47:08.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:47:08.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:08.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:08.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:08.791 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:47:08.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:08.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:08.791 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:47:08.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:08.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:08.792 | INFO | Uploaded 36GiB to GPU in 1.60415s (22.4418GiB/s) | 20260506 09:47:08.792 | INFO | Downloaded 34GiB to GPU in 1.52289s (22.326GiB/s) | 20260506 09:47:13.930 | INFO | Analyzing chunk frames = [392192,392704) | 20260506 09:47:13.930 | INFO | MemInfo: { | 20260506 09:47:13.930 | INFO | "gpu" : " 17 GB", | 20260506 09:47:13.930 | INFO | "huge" : "", | 20260506 09:47:13.930 | INFO | "huge_shared" : " 27 GB", | 20260506 09:47:13.930 | INFO | "malloc" : " 14 GB", | 20260506 09:47:13.930 | INFO | "pinned" : " 29 GB" | 20260506 09:47:13.930 | INFO | } | 20260506 09:47:13.930 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:47:13.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:47:13.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:13.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:13.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:13.930 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:47:13.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:13.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:13.931 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:47:13.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:13.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:13.931 | INFO | Uploaded 36GiB to GPU in 1.60495s (22.4306GiB/s) | 20260506 09:47:13.931 | INFO | Downloaded 34GiB to GPU in 1.52336s (22.319GiB/s) | 20260506 09:47:19.070 | INFO | Analyzing chunk frames = [392704,393216) | 20260506 09:47:19.070 | INFO | MemInfo: { | 20260506 09:47:19.070 | INFO | "gpu" : " 17 GB", | 20260506 09:47:19.070 | INFO | "huge" : "", | 20260506 09:47:19.070 | INFO | "huge_shared" : " 27 GB", | 20260506 09:47:19.070 | INFO | "malloc" : " 14 GB", | 20260506 09:47:19.070 | INFO | "pinned" : " 29 GB" | 20260506 09:47:19.070 | INFO | } | 20260506 09:47:19.070 | INFO | Perf report for the last 5.13937 seconds: | 20260506 09:47:19.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:47:19.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:19.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:19.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:19.070 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:47:19.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:19.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:19.070 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 09:47:19.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:19.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:19.070 | INFO | Uploaded 36GiB to GPU in 1.60396s (22.4444GiB/s) | 20260506 09:47:19.070 | INFO | Downloaded 34GiB to GPU in 1.52244s (22.3325GiB/s) | 20260506 09:47:19.070 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:47:19.070 | INFO | Total mem usage: 15.9 GiB | 20260506 09:47:19.070 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 2.5%) | 20260506 09:47:19.070 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: 0.0322%) | 20260506 09:47:19.070 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:47:19.070 | INFO | High Water Breakdown: | 20260506 09:47:19.070 | INFO | 60.4% : 9.61 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:47:19.070 | INFO | 39.5% : 6.29 GiB ( 94% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:47:19.070 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:47:19.070 | INFO | ------------------------------------------ | 20260506 09:47:19.070 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:47:19.070 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:47:19.070 | INFO | Total mem usage: 26.5 GiB | 20260506 09:47:19.070 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:47:19.070 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:47:19.070 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:47:19.070 | INFO | High Water Breakdown: | 20260506 09:47:19.070 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:47:19.070 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:47:19.070 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:47:19.070 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:47:19.070 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:47:19.070 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:47:19.070 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:47:19.070 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:47:19.070 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:47:19.070 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:47:19.070 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:47:19.070 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:47:19.070 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:47:19.070 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:47:19.070 | INFO | ------------------------------------------ | 20260506 09:47:19.070 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:47:19.070 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:47:19.070 | INFO | Total mem usage: 25.6 GiB | 20260506 09:47:19.070 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:47:19.070 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 09:47:19.070 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 09:47:19.070 | INFO | High Water Breakdown: | 20260506 09:47:19.070 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 09:47:19.070 | INFO | ------------------------------------------ | 20260506 09:47:19.070 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:47:19.070 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:47:19.070 | INFO | Total mem usage: 15.7 GiB | 20260506 09:47:19.070 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:47:19.070 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:47:19.070 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:47:19.070 | INFO | High Water Breakdown: | 20260506 09:47:19.070 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:47:19.070 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:47:19.070 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:47:19.070 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:47:19.070 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:47:19.070 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:47:19.070 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:47:19.070 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:47:19.070 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:47:19.070 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:47:19.070 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:47:19.070 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:47:19.070 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:47:19.070 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:47:19.070 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:47:19.070 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:47:19.070 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:47:19.070 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:47:19.070 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:47:19.070 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:47:19.070 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:47:19.070 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:47:19.070 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:47:19.070 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:47:19.070 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:47:19.070 | INFO | ------------------------------------------ | 20260506 09:47:19.070 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:47:19.070 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:47:19.070 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:47:19.070 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:47:19.070 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 09:47:19.070 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / -nan, 4.17e-03 / 0.00e+00 | 20260506 09:47:19.070 | INFO | PerfReport: 10.91%, Baseline -- 3.29e+00 / -nan, 1.83e-02 / 0.00e+00 | 20260506 09:47:19.070 | INFO | PerfReport: 3.05%, Histogram -- 9.21e-01 / -nan, 5.25e-02 / 0.00e+00 | 20260506 09:47:19.070 | INFO | PerfReport: 16.05%, DME -- 3.88e+01 / -nan, 3.14e+00 / 0.00e+00 | 20260506 09:47:19.070 | INFO | PerfReport: 56.97%, FrameLabeling -- 1.72e+01 / -nan, 4.16e-02 / 0.00e+00 | 20260506 09:47:19.070 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.26e+00 / -nan, 1.33e-01 / 0.00e+00 | 20260506 09:47:19.070 | INFO | PerfReport: 3.78%, Metrics -- 1.14e+00 / -nan, 1.76e-01 / 0.00e+00 | 20260506 09:47:19.070 | INFO | PerfReport: 1.47%, PulseSerializing -- 4.43e-01 / -nan, 6.37e-02 / 0.00e+00 | 20260506 09:47:19.070 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:47:19.070 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:47:19.070 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:47:19.070 | INFO | PerfReport: Full Execution -- 9.01e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 09:47:19.070 | INFO | PerfReport: 18.64%, Upload -- 1.68e+01 / 0.00e+00, 4.87e-01 / 0.00e+00 | 20260506 09:47:19.070 | INFO | PerfReport: 17.90%, Download -- 1.61e+01 / 0.00e+00, 8.24e-01 / 0.00e+00 | 20260506 09:47:19.070 | INFO | PerfReport: 33.49%, Compute -- 3.02e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 09:47:19.070 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 09:47:19.070 | INFO | PerfReport: 0.00%, LockDownload -- 5.03e-04 / 0.00e+00, 2.07e-04 / 0.00e+00 | 20260506 09:47:19.070 | INFO | PerfReport: 29.39%, LockCompute -- 2.65e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260506 09:47:19.070 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260506 09:47:19.167 | INFO | Starting a round of Baz IO | 20260506 09:47:22.085 | INFO | Populating RTMetrics report | 20260506 09:47:22.085 | INFO | RTMetrics report populated | 20260506 09:47:22.085 | INFO | Opening tmp RTMetrics json file | 20260506 09:47:22.202 | INFO | Serializing RTMetrics to json | 20260506 09:47:22.206 | INFO | Serialization done, swapping tmp file now... | 20260506 09:47:22.211 | INFO | RTMetrics done | 20260506 09:47:22.623 | INFO | Finished a round of Baz IO: Wrote 1.903 GiB in 3.456 seconds | 20260506 09:47:22.623 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.63% / 0.08% / 48.29% | 20260506 09:47:22.623 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.46558% / 0.000238419% / 3.97364e-05% | 20260506 09:47:22.623 | INFO | Baz IO overall: | 20260506 09:47:22.623 | INFO | BytesWritten : 162269501971 | 20260506 09:47:22.623 | INFO | NumEvents : 77377309464 | 20260506 09:47:24.209 | INFO | Analyzing chunk frames = [393216,393728) | 20260506 09:47:24.209 | INFO | MemInfo: { | 20260506 09:47:24.209 | INFO | "gpu" : " 17 GB", | 20260506 09:47:24.209 | INFO | "huge" : "", | 20260506 09:47:24.209 | INFO | "huge_shared" : " 27 GB", | 20260506 09:47:24.209 | INFO | "malloc" : " 14 GB", | 20260506 09:47:24.209 | INFO | "pinned" : " 29 GB" | 20260506 09:47:24.209 | INFO | } | 20260506 09:47:24.209 | INFO | Perf report for the last 5.13913 seconds: | 20260506 09:47:24.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:47:24.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:24.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:24.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:24.209 | INFO | ANALYSIS : 57.7%, 2.8, 85.3ms, 0 | 20260506 09:47:24.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:24.209 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260506 09:47:24.209 | INFO | PRE_HQ : 56.8%, 2.1, 61.3ms, 0 | 20260506 09:47:24.209 | INFO | BAZWRITER : 66.6%, 4.8, 164.6ms, 0 | 20260506 09:47:24.209 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 09:47:24.210 | INFO | Uploaded 36GiB to GPU in 1.67322s (21.5154GiB/s) | 20260506 09:47:24.210 | INFO | Downloaded 38GiB to GPU in 1.73722s (21.874GiB/s) | 20260506 09:47:29.348 | INFO | Analyzing chunk frames = [393728,394240) | 20260506 09:47:29.348 | INFO | MemInfo: { | 20260506 09:47:29.348 | INFO | "gpu" : " 17 GB", | 20260506 09:47:29.348 | INFO | "huge" : "", | 20260506 09:47:29.348 | INFO | "huge_shared" : " 27 GB", | 20260506 09:47:29.348 | INFO | "malloc" : " 14 GB", | 20260506 09:47:29.348 | INFO | "pinned" : " 29 GB" | 20260506 09:47:29.348 | INFO | } | 20260506 09:47:29.349 | INFO | Perf report for the last 5.13952 seconds: | 20260506 09:47:29.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:47:29.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:29.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:29.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:29.349 | INFO | ANALYSIS : 57.5%, 3.0, 89.8ms, 0 | 20260506 09:47:29.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:29.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:29.349 | INFO | PRE_HQ : 31.2%, 1.0, 16.4ms, 0 | 20260506 09:47:29.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:29.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:29.349 | INFO | Uploaded 36GiB to GPU in 1.60217s (22.4696GiB/s) | 20260506 09:47:29.349 | INFO | Downloaded 34GiB to GPU in 1.51899s (22.3833GiB/s) | 20260506 09:47:34.488 | INFO | Analyzing chunk frames = [394240,394752) | 20260506 09:47:34.488 | INFO | MemInfo: { | 20260506 09:47:34.488 | INFO | "gpu" : " 17 GB", | 20260506 09:47:34.488 | INFO | "huge" : "", | 20260506 09:47:34.488 | INFO | "huge_shared" : " 27 GB", | 20260506 09:47:34.488 | INFO | "malloc" : " 14 GB", | 20260506 09:47:34.488 | INFO | "pinned" : " 29 GB" | 20260506 09:47:34.488 | INFO | } | 20260506 09:47:34.488 | INFO | Perf report for the last 5.13955 seconds: | 20260506 09:47:34.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:47:34.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:34.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:34.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:34.488 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:47:34.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:34.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:34.488 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:47:34.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:34.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:34.488 | INFO | Uploaded 36GiB to GPU in 1.60619s (22.4133GiB/s) | 20260506 09:47:34.489 | INFO | Downloaded 34GiB to GPU in 1.52407s (22.3086GiB/s) | 20260506 09:47:39.627 | INFO | Analyzing chunk frames = [394752,395264) | 20260506 09:47:39.627 | INFO | MemInfo: { | 20260506 09:47:39.627 | INFO | "gpu" : " 17 GB", | 20260506 09:47:39.627 | INFO | "huge" : "", | 20260506 09:47:39.627 | INFO | "huge_shared" : " 27 GB", | 20260506 09:47:39.627 | INFO | "malloc" : " 14 GB", | 20260506 09:47:39.627 | INFO | "pinned" : " 29 GB" | 20260506 09:47:39.627 | INFO | } | 20260506 09:47:39.627 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:47:39.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:47:39.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:39.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:39.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:39.627 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:47:39.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:39.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:39.627 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:47:39.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:39.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:39.628 | INFO | Uploaded 36GiB to GPU in 1.60565s (22.4209GiB/s) | 20260506 09:47:39.628 | INFO | Downloaded 34GiB to GPU in 1.52277s (22.3277GiB/s) | 20260506 09:47:44.766 | INFO | Analyzing chunk frames = [395264,395776) | 20260506 09:47:44.766 | INFO | MemInfo: { | 20260506 09:47:44.766 | INFO | "gpu" : " 17 GB", | 20260506 09:47:44.766 | INFO | "huge" : "", | 20260506 09:47:44.766 | INFO | "huge_shared" : " 27 GB", | 20260506 09:47:44.766 | INFO | "malloc" : " 14 GB", | 20260506 09:47:44.766 | INFO | "pinned" : " 29 GB" | 20260506 09:47:44.767 | INFO | } | 20260506 09:47:44.767 | INFO | Perf report for the last 5.13914 seconds: | 20260506 09:47:44.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:47:44.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:44.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:44.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:44.767 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 09:47:44.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:44.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:44.767 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:47:44.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:44.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:44.767 | INFO | Uploaded 36GiB to GPU in 1.60428s (22.4399GiB/s) | 20260506 09:47:44.767 | INFO | Downloaded 34GiB to GPU in 1.52086s (22.3557GiB/s) | 20260506 09:47:49.906 | INFO | Analyzing chunk frames = [395776,396288) | 20260506 09:47:49.906 | INFO | MemInfo: { | 20260506 09:47:49.906 | INFO | "gpu" : " 17 GB", | 20260506 09:47:49.906 | INFO | "huge" : "", | 20260506 09:47:49.906 | INFO | "huge_shared" : " 27 GB", | 20260506 09:47:49.906 | INFO | "malloc" : " 14 GB", | 20260506 09:47:49.906 | INFO | "pinned" : " 29 GB" | 20260506 09:47:49.906 | INFO | } | 20260506 09:47:49.906 | INFO | Perf report for the last 5.13943 seconds: | 20260506 09:47:49.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:47:49.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:49.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:49.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:49.906 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:47:49.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:49.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:49.906 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:47:49.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:49.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:49.907 | INFO | Uploaded 36GiB to GPU in 1.60464s (22.4349GiB/s) | 20260506 09:47:49.907 | INFO | Downloaded 34GiB to GPU in 1.52216s (22.3367GiB/s) | 20260506 09:47:55.045 | INFO | Analyzing chunk frames = [396288,396800) | 20260506 09:47:55.045 | INFO | MemInfo: { | 20260506 09:47:55.045 | INFO | "gpu" : " 17 GB", | 20260506 09:47:55.045 | INFO | "huge" : "", | 20260506 09:47:55.045 | INFO | "huge_shared" : " 27 GB", | 20260506 09:47:55.045 | INFO | "malloc" : " 14 GB", | 20260506 09:47:55.045 | INFO | "pinned" : " 29 GB" | 20260506 09:47:55.045 | INFO | } | 20260506 09:47:55.045 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:47:55.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:47:55.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:55.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:55.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:55.045 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 09:47:55.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:55.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:55.045 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 09:47:55.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:55.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:47:55.046 | INFO | Uploaded 36GiB to GPU in 1.61077s (22.3495GiB/s) | 20260506 09:47:55.046 | INFO | Downloaded 34GiB to GPU in 1.52858s (22.2428GiB/s) | 20260506 09:48:00.184 | INFO | Analyzing chunk frames = [396800,397312) | 20260506 09:48:00.184 | INFO | MemInfo: { | 20260506 09:48:00.184 | INFO | "gpu" : " 17 GB", | 20260506 09:48:00.184 | INFO | "huge" : "", | 20260506 09:48:00.184 | INFO | "huge_shared" : " 27 GB", | 20260506 09:48:00.184 | INFO | "malloc" : " 14 GB", | 20260506 09:48:00.184 | INFO | "pinned" : " 29 GB" | 20260506 09:48:00.185 | INFO | } | 20260506 09:48:00.185 | INFO | Perf report for the last 5.13923 seconds: | 20260506 09:48:00.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:48:00.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:00.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:00.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:00.185 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:48:00.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:00.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:00.185 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 09:48:00.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:00.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:00.185 | INFO | Uploaded 36GiB to GPU in 1.60485s (22.4321GiB/s) | 20260506 09:48:00.185 | INFO | Downloaded 34GiB to GPU in 1.52299s (22.3245GiB/s) | 20260506 09:48:00.282 | INFO | Starting a round of Baz IO | 20260506 09:48:03.202 | INFO | Populating RTMetrics report | 20260506 09:48:03.202 | INFO | RTMetrics report populated | 20260506 09:48:03.202 | INFO | Opening tmp RTMetrics json file | 20260506 09:48:03.235 | INFO | Serializing RTMetrics to json | 20260506 09:48:03.344 | INFO | Serialization done, swapping tmp file now... | 20260506 09:48:03.349 | INFO | RTMetrics done | 20260506 09:48:03.374 | INFO | Finished a round of Baz IO: Wrote 1.906 GiB in 3.092 seconds | 20260506 09:48:03.374 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.55% / 0.08% / 48.37% | 20260506 09:48:03.374 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.4516% / 0.00021855% / 2.38419e-05% | 20260506 09:48:03.374 | INFO | Baz IO overall: | 20260506 09:48:03.374 | INFO | BytesWritten : 164316029559 | 20260506 09:48:03.374 | INFO | NumEvents : 78370614419 | 20260506 09:48:05.324 | INFO | Analyzing chunk frames = [397312,397824) | 20260506 09:48:05.324 | INFO | MemInfo: { | 20260506 09:48:05.324 | INFO | "gpu" : " 17 GB", | 20260506 09:48:05.324 | INFO | "huge" : "", | 20260506 09:48:05.324 | INFO | "huge_shared" : " 27 GB", | 20260506 09:48:05.324 | INFO | "malloc" : " 14 GB", | 20260506 09:48:05.324 | INFO | "pinned" : " 29 GB" | 20260506 09:48:05.324 | INFO | } | 20260506 09:48:05.324 | INFO | Perf report for the last 5.13937 seconds: | 20260506 09:48:05.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:48:05.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:05.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:05.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:05.324 | INFO | ANALYSIS : 57.7%, 2.9, 88.6ms, 0 | 20260506 09:48:05.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:05.324 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260506 09:48:05.324 | INFO | PRE_HQ : 56.8%, 2.1, 61.7ms, 0 | 20260506 09:48:05.324 | INFO | BAZWRITER : 59.7%, 4.4, 136.1ms, 0 | 20260506 09:48:05.324 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 09:48:05.324 | INFO | Uploaded 36GiB to GPU in 1.64581s (21.8737GiB/s) | 20260506 09:48:05.324 | INFO | Downloaded 38GiB to GPU in 1.70805s (22.2476GiB/s) | 20260506 09:48:10.463 | INFO | Analyzing chunk frames = [397824,398336) | 20260506 09:48:10.463 | INFO | MemInfo: { | 20260506 09:48:10.463 | INFO | "gpu" : " 17 GB", | 20260506 09:48:10.463 | INFO | "huge" : "", | 20260506 09:48:10.463 | INFO | "huge_shared" : " 27 GB", | 20260506 09:48:10.463 | INFO | "malloc" : " 14 GB", | 20260506 09:48:10.463 | INFO | "pinned" : " 29 GB" | 20260506 09:48:10.463 | INFO | } | 20260506 09:48:10.463 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:48:10.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:48:10.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:10.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:10.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:10.463 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 09:48:10.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:10.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:10.463 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 09:48:10.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:10.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:10.464 | INFO | Uploaded 36GiB to GPU in 1.60246s (22.4655GiB/s) | 20260506 09:48:10.464 | INFO | Downloaded 34GiB to GPU in 1.52013s (22.3665GiB/s) | 20260506 09:48:15.602 | INFO | Analyzing chunk frames = [398336,398848) | 20260506 09:48:15.602 | INFO | MemInfo: { | 20260506 09:48:15.602 | INFO | "gpu" : " 17 GB", | 20260506 09:48:15.602 | INFO | "huge" : "", | 20260506 09:48:15.602 | INFO | "huge_shared" : " 27 GB", | 20260506 09:48:15.602 | INFO | "malloc" : " 14 GB", | 20260506 09:48:15.603 | INFO | "pinned" : " 29 GB" | 20260506 09:48:15.603 | INFO | } | 20260506 09:48:15.603 | INFO | Perf report for the last 5.13932 seconds: | 20260506 09:48:15.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:48:15.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:15.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:15.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:15.603 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:48:15.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:15.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:15.603 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:48:15.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:15.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:15.603 | INFO | Uploaded 36GiB to GPU in 1.60601s (22.4158GiB/s) | 20260506 09:48:15.603 | INFO | Downloaded 34GiB to GPU in 1.52417s (22.3072GiB/s) | 20260506 09:48:20.742 | INFO | Analyzing chunk frames = [398848,399360) | 20260506 09:48:20.742 | INFO | MemInfo: { | 20260506 09:48:20.742 | INFO | "gpu" : " 17 GB", | 20260506 09:48:20.742 | INFO | "huge" : "", | 20260506 09:48:20.742 | INFO | "huge_shared" : " 27 GB", | 20260506 09:48:20.742 | INFO | "malloc" : " 14 GB", | 20260506 09:48:20.742 | INFO | "pinned" : " 29 GB" | 20260506 09:48:20.742 | INFO | } | 20260506 09:48:20.742 | INFO | Perf report for the last 5.13953 seconds: | 20260506 09:48:20.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:48:20.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:20.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:20.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:20.742 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:48:20.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:20.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:20.742 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:48:20.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:20.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:20.742 | INFO | Uploaded 36GiB to GPU in 1.6067s (22.4062GiB/s) | 20260506 09:48:20.742 | INFO | Downloaded 34GiB to GPU in 1.52328s (22.3203GiB/s) | 20260506 09:48:25.881 | INFO | Analyzing chunk frames = [399360,399872) | 20260506 09:48:25.881 | INFO | MemInfo: { | 20260506 09:48:25.881 | INFO | "gpu" : " 17 GB", | 20260506 09:48:25.881 | INFO | "huge" : "", | 20260506 09:48:25.881 | INFO | "huge_shared" : " 27 GB", | 20260506 09:48:25.881 | INFO | "malloc" : " 14 GB", | 20260506 09:48:25.881 | INFO | "pinned" : " 29 GB" | 20260506 09:48:25.881 | INFO | } | 20260506 09:48:25.881 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:48:25.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:48:25.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:25.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:25.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:25.881 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 09:48:25.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:25.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:25.882 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:48:25.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:25.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:25.882 | INFO | Uploaded 36GiB to GPU in 1.6048s (22.4327GiB/s) | 20260506 09:48:25.882 | INFO | Downloaded 34GiB to GPU in 1.52122s (22.3505GiB/s) | 20260506 09:48:31.020 | INFO | Analyzing chunk frames = [399872,400384) | 20260506 09:48:31.020 | INFO | MemInfo: { | 20260506 09:48:31.020 | INFO | "gpu" : " 17 GB", | 20260506 09:48:31.020 | INFO | "huge" : "", | 20260506 09:48:31.020 | INFO | "huge_shared" : " 27 GB", | 20260506 09:48:31.021 | INFO | "malloc" : " 14 GB", | 20260506 09:48:31.021 | INFO | "pinned" : " 29 GB" | 20260506 09:48:31.021 | INFO | } | 20260506 09:48:31.021 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:48:31.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:48:31.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:31.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:31.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:31.021 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:48:31.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:31.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:31.021 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:48:31.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:31.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:31.021 | INFO | Uploaded 36GiB to GPU in 1.60486s (22.4318GiB/s) | 20260506 09:48:31.021 | INFO | Downloaded 34GiB to GPU in 1.5216s (22.3449GiB/s) | 20260506 09:48:36.160 | INFO | Analyzing chunk frames = [400384,400896) | 20260506 09:48:36.160 | INFO | MemInfo: { | 20260506 09:48:36.160 | INFO | "gpu" : " 17 GB", | 20260506 09:48:36.160 | INFO | "huge" : "", | 20260506 09:48:36.160 | INFO | "huge_shared" : " 27 GB", | 20260506 09:48:36.160 | INFO | "malloc" : " 14 GB", | 20260506 09:48:36.160 | INFO | "pinned" : " 29 GB" | 20260506 09:48:36.160 | INFO | } | 20260506 09:48:36.160 | INFO | Perf report for the last 5.1395 seconds: | 20260506 09:48:36.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:48:36.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:36.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:36.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:36.160 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260506 09:48:36.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:36.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:36.160 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 09:48:36.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:36.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:36.161 | INFO | Uploaded 36GiB to GPU in 1.6042s (22.441GiB/s) | 20260506 09:48:36.161 | INFO | Downloaded 34GiB to GPU in 1.52201s (22.3388GiB/s) | 20260506 09:48:41.299 | INFO | Analyzing chunk frames = [400896,401408) | 20260506 09:48:41.299 | INFO | MemInfo: { | 20260506 09:48:41.299 | INFO | "gpu" : " 17 GB", | 20260506 09:48:41.299 | INFO | "huge" : "", | 20260506 09:48:41.299 | INFO | "huge_shared" : " 27 GB", | 20260506 09:48:41.299 | INFO | "malloc" : " 14 GB", | 20260506 09:48:41.299 | INFO | "pinned" : " 29 GB" | 20260506 09:48:41.299 | INFO | } | 20260506 09:48:41.299 | INFO | Perf report for the last 5.13902 seconds: | 20260506 09:48:41.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:48:41.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:41.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:41.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:41.299 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:48:41.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:41.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:41.299 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 09:48:41.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:41.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:41.300 | INFO | Uploaded 36GiB to GPU in 1.60478s (22.433GiB/s) | 20260506 09:48:41.300 | INFO | Downloaded 34GiB to GPU in 1.52288s (22.3262GiB/s) | 20260506 09:48:41.300 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:48:41.300 | INFO | Total mem usage: 15.9 GiB | 20260506 09:48:41.300 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 2.5%) | 20260506 09:48:41.300 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -0.703%) | 20260506 09:48:41.300 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:48:41.300 | INFO | High Water Breakdown: | 20260506 09:48:41.300 | INFO | 60.2% : 9.52 GiB ( 99.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:48:41.300 | INFO | 39.6% : 6.26 GiB ( 93.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:48:41.300 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:48:41.300 | INFO | ------------------------------------------ | 20260506 09:48:41.300 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:48:41.300 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:48:41.300 | INFO | Total mem usage: 26.5 GiB | 20260506 09:48:41.300 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:48:41.300 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:48:41.300 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:48:41.300 | INFO | High Water Breakdown: | 20260506 09:48:41.300 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:48:41.300 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:48:41.300 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:48:41.300 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:48:41.300 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:48:41.300 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:48:41.300 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:48:41.300 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:48:41.300 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:48:41.300 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:48:41.300 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:48:41.300 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:48:41.300 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:48:41.300 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:48:41.300 | INFO | ------------------------------------------ | 20260506 09:48:41.300 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:48:41.300 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:48:41.300 | INFO | Total mem usage: 25.6 GiB | 20260506 09:48:41.300 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:48:41.300 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:48:41.300 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:48:41.300 | INFO | High Water Breakdown: | 20260506 09:48:41.300 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:48:41.300 | INFO | ------------------------------------------ | 20260506 09:48:41.300 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:48:41.300 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:48:41.300 | INFO | Total mem usage: 15.7 GiB | 20260506 09:48:41.300 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:48:41.300 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:48:41.300 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:48:41.301 | INFO | High Water Breakdown: | 20260506 09:48:41.301 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:48:41.301 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:48:41.301 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:48:41.301 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:48:41.301 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:48:41.301 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:48:41.301 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:48:41.301 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:48:41.301 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:48:41.301 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:48:41.301 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:48:41.301 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:48:41.301 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:48:41.301 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:48:41.301 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:48:41.301 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:48:41.301 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:48:41.301 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:48:41.301 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:48:41.301 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:48:41.301 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:48:41.301 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:48:41.301 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:48:41.301 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:48:41.301 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:48:41.301 | INFO | ------------------------------------------ | 20260506 09:48:41.301 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:48:41.301 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:48:41.301 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:48:41.301 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:48:41.301 | INFO | PerfReport: Full Execution -- 3.02e+01 / 7.24e+01, 1.25e+01 / 0.00e+00 | 20260506 09:48:41.301 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / 8.62e-02, 6.99e-03 / 0.00e+00 | 20260506 09:48:41.301 | INFO | PerfReport: 10.91%, Baseline -- 3.29e+00 / 3.30e+00, 1.77e-02 / 0.00e+00 | 20260506 09:48:41.301 | INFO | PerfReport: 3.05%, Histogram -- 9.21e-01 / 1.06e+00, 5.16e-02 / 0.00e+00 | 20260506 09:48:41.301 | INFO | PerfReport: 16.06%, DME -- 3.88e+01 / 4.68e+01, 3.19e+00 / 0.00e+00 | 20260506 09:48:41.301 | INFO | PerfReport: 56.97%, FrameLabeling -- 1.72e+01 / 1.72e+01, 4.20e-02 / 0.00e+00 | 20260506 09:48:41.301 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.26e+00 / 2.36e+00, 1.33e-01 / 0.00e+00 | 20260506 09:48:41.301 | INFO | PerfReport: 3.78%, Metrics -- 1.14e+00 / 1.18e+00, 1.78e-01 / 0.00e+00 | 20260506 09:48:41.301 | INFO | PerfReport: 1.47%, PulseSerializing -- 4.43e-01 / 4.97e-01, 6.38e-02 / 0.00e+00 | 20260506 09:48:41.301 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:48:41.301 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:48:41.301 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:48:41.301 | INFO | PerfReport: Full Execution -- 9.03e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 09:48:41.301 | INFO | PerfReport: 18.60%, Upload -- 1.68e+01 / 0.00e+00, 4.82e-01 / 0.00e+00 | 20260506 09:48:41.301 | INFO | PerfReport: 17.85%, Download -- 1.61e+01 / 0.00e+00, 7.99e-01 / 0.00e+00 | 20260506 09:48:41.301 | INFO | PerfReport: 33.43%, Compute -- 3.02e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 09:48:41.301 | INFO | PerfReport: 0.56%, LockUpload -- 5.04e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:48:41.301 | INFO | PerfReport: 0.00%, LockDownload -- 4.93e-04 / 0.00e+00, 1.52e-04 / 0.00e+00 | 20260506 09:48:41.301 | INFO | PerfReport: 29.54%, LockCompute -- 2.67e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 09:48:41.301 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260506 09:48:41.396 | INFO | Starting a round of Baz IO | 20260506 09:48:44.314 | INFO | Populating RTMetrics report | 20260506 09:48:44.315 | INFO | RTMetrics report populated | 20260506 09:48:44.315 | INFO | Opening tmp RTMetrics json file | 20260506 09:48:44.318 | INFO | Serializing RTMetrics to json | 20260506 09:48:44.321 | INFO | Serialization done, swapping tmp file now... | 20260506 09:48:44.322 | INFO | RTMetrics done | 20260506 09:48:44.638 | INFO | Finished a round of Baz IO: Wrote 1.906 GiB in 3.241 seconds | 20260506 09:48:44.638 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.47% / 0.08% / 48.45% | 20260506 09:48:44.639 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.4187% / 0.000202656% / 5.5631e-05% | 20260506 09:48:44.639 | INFO | Baz IO overall: | 20260506 09:48:44.639 | INFO | BytesWritten : 166363109335 | 20260506 09:48:44.639 | INFO | NumEvents : 79364095900 | 20260506 09:48:46.439 | INFO | Analyzing chunk frames = [401408,401920) | 20260506 09:48:46.439 | INFO | MemInfo: { | 20260506 09:48:46.439 | INFO | "gpu" : " 17 GB", | 20260506 09:48:46.439 | INFO | "huge" : "", | 20260506 09:48:46.439 | INFO | "huge_shared" : " 27 GB", | 20260506 09:48:46.439 | INFO | "malloc" : " 14 GB", | 20260506 09:48:46.439 | INFO | "pinned" : " 29 GB" | 20260506 09:48:46.439 | INFO | } | 20260506 09:48:46.439 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:48:46.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:48:46.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:46.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:46.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:46.439 | INFO | ANALYSIS : 57.7%, 2.8, 85.3ms, 0 | 20260506 09:48:46.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:46.439 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 09:48:46.439 | INFO | PRE_HQ : 56.8%, 2.1, 61.2ms, 0 | 20260506 09:48:46.439 | INFO | BAZWRITER : 62.7%, 4.8, 153.7ms, 0 | 20260506 09:48:46.439 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 09:48:46.439 | INFO | Uploaded 36GiB to GPU in 1.67367s (21.5096GiB/s) | 20260506 09:48:46.439 | INFO | Downloaded 38GiB to GPU in 1.73954s (21.8449GiB/s) | 20260506 09:48:51.578 | INFO | Analyzing chunk frames = [401920,402432) | 20260506 09:48:51.578 | INFO | MemInfo: { | 20260506 09:48:51.578 | INFO | "gpu" : " 17 GB", | 20260506 09:48:51.578 | INFO | "huge" : "", | 20260506 09:48:51.578 | INFO | "huge_shared" : " 27 GB", | 20260506 09:48:51.578 | INFO | "malloc" : " 14 GB", | 20260506 09:48:51.578 | INFO | "pinned" : " 29 GB" | 20260506 09:48:51.578 | INFO | } | 20260506 09:48:51.578 | INFO | Perf report for the last 5.13957 seconds: | 20260506 09:48:51.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:48:51.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:51.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:51.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:51.578 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 09:48:51.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:51.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:51.578 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:48:51.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:51.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:51.579 | INFO | Uploaded 36GiB to GPU in 1.60262s (22.4632GiB/s) | 20260506 09:48:51.579 | INFO | Downloaded 34GiB to GPU in 1.5201s (22.367GiB/s) | 20260506 09:48:56.717 | INFO | Analyzing chunk frames = [402432,402944) | 20260506 09:48:56.717 | INFO | MemInfo: { | 20260506 09:48:56.717 | INFO | "gpu" : " 17 GB", | 20260506 09:48:56.717 | INFO | "huge" : "", | 20260506 09:48:56.717 | INFO | "huge_shared" : " 27 GB", | 20260506 09:48:56.717 | INFO | "malloc" : " 14 GB", | 20260506 09:48:56.717 | INFO | "pinned" : " 29 GB" | 20260506 09:48:56.717 | INFO | } | 20260506 09:48:56.717 | INFO | Perf report for the last 5.13909 seconds: | 20260506 09:48:56.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:48:56.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:56.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:56.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:56.717 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:48:56.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:56.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:56.717 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:48:56.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:56.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:48:56.718 | INFO | Uploaded 36GiB to GPU in 1.6058s (22.4187GiB/s) | 20260506 09:48:56.718 | INFO | Downloaded 34GiB to GPU in 1.52348s (22.3173GiB/s) | 20260506 09:49:01.857 | INFO | Analyzing chunk frames = [402944,403456) | 20260506 09:49:01.857 | INFO | MemInfo: { | 20260506 09:49:01.857 | INFO | "gpu" : " 17 GB", | 20260506 09:49:01.857 | INFO | "huge" : "", | 20260506 09:49:01.857 | INFO | "huge_shared" : " 27 GB", | 20260506 09:49:01.857 | INFO | "malloc" : " 14 GB", | 20260506 09:49:01.857 | INFO | "pinned" : " 29 GB" | 20260506 09:49:01.857 | INFO | } | 20260506 09:49:01.857 | INFO | Perf report for the last 5.13968 seconds: | 20260506 09:49:01.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:49:01.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:01.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:01.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:01.857 | INFO | ANALYSIS : 56.6%, 3.0, 87.7ms, 0 | 20260506 09:49:01.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:01.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:01.857 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:49:01.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:01.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:01.857 | INFO | Uploaded 36GiB to GPU in 1.60644s (22.4098GiB/s) | 20260506 09:49:01.857 | INFO | Downloaded 34GiB to GPU in 1.52419s (22.3069GiB/s) | 20260506 09:49:06.996 | INFO | Analyzing chunk frames = [403456,403968) | 20260506 09:49:06.996 | INFO | MemInfo: { | 20260506 09:49:06.996 | INFO | "gpu" : " 17 GB", | 20260506 09:49:06.996 | INFO | "huge" : "", | 20260506 09:49:06.996 | INFO | "huge_shared" : " 27 GB", | 20260506 09:49:06.996 | INFO | "malloc" : " 14 GB", | 20260506 09:49:06.996 | INFO | "pinned" : " 29 GB" | 20260506 09:49:06.996 | INFO | } | 20260506 09:49:06.996 | INFO | Perf report for the last 5.13895 seconds: | 20260506 09:49:06.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:49:06.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:06.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:06.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:06.996 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 09:49:06.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:06.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:06.996 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:49:06.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:06.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:06.997 | INFO | Uploaded 36GiB to GPU in 1.60416s (22.4417GiB/s) | 20260506 09:49:06.997 | INFO | Downloaded 34GiB to GPU in 1.52083s (22.3562GiB/s) | 20260506 09:49:12.138 | INFO | Analyzing chunk frames = [403968,404480) | 20260506 09:49:12.138 | INFO | MemInfo: { | 20260506 09:49:12.138 | INFO | "gpu" : " 17 GB", | 20260506 09:49:12.138 | INFO | "huge" : "", | 20260506 09:49:12.138 | INFO | "huge_shared" : " 27 GB", | 20260506 09:49:12.138 | INFO | "malloc" : " 14 GB", | 20260506 09:49:12.138 | INFO | "pinned" : " 29 GB" | 20260506 09:49:12.138 | INFO | } | 20260506 09:49:12.138 | INFO | Perf report for the last 5.14219 seconds: | 20260506 09:49:12.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:49:12.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:12.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:12.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:12.138 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 09:49:12.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:12.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:12.138 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:49:12.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:12.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:12.138 | INFO | Uploaded 36GiB to GPU in 1.60501s (22.4298GiB/s) | 20260506 09:49:12.138 | INFO | Downloaded 34GiB to GPU in 1.52167s (22.3438GiB/s) | 20260506 09:49:17.298 | INFO | Analyzing chunk frames = [404480,404992) | 20260506 09:49:17.298 | INFO | MemInfo: { | 20260506 09:49:17.298 | INFO | "gpu" : " 17 GB", | 20260506 09:49:17.298 | INFO | "huge" : "", | 20260506 09:49:17.298 | INFO | "huge_shared" : " 27 GB", | 20260506 09:49:17.298 | INFO | "malloc" : " 14 GB", | 20260506 09:49:17.298 | INFO | "pinned" : " 29 GB" | 20260506 09:49:17.298 | INFO | } | 20260506 09:49:17.298 | INFO | Perf report for the last 5.15974 seconds: | 20260506 09:49:17.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:49:17.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:17.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:17.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:17.298 | INFO | ANALYSIS : 56.8%, 3.0, 88.9ms, 0 | 20260506 09:49:17.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:17.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:17.298 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260506 09:49:17.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:17.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:17.298 | INFO | Uploaded 36GiB to GPU in 1.6054s (22.4243GiB/s) | 20260506 09:49:17.299 | INFO | Downloaded 34GiB to GPU in 1.52358s (22.3158GiB/s) | 20260506 09:49:22.414 | INFO | Analyzing chunk frames = [404992,405504) | 20260506 09:49:22.414 | INFO | MemInfo: { | 20260506 09:49:22.414 | INFO | "gpu" : " 17 GB", | 20260506 09:49:22.414 | INFO | "huge" : "", | 20260506 09:49:22.414 | INFO | "huge_shared" : " 27 GB", | 20260506 09:49:22.414 | INFO | "malloc" : " 14 GB", | 20260506 09:49:22.414 | INFO | "pinned" : " 29 GB" | 20260506 09:49:22.414 | INFO | } | 20260506 09:49:22.414 | INFO | Perf report for the last 5.11624 seconds: | 20260506 09:49:22.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:49:22.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:22.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:22.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:22.414 | INFO | ANALYSIS : 57.3%, 3.0, 89.0ms, 0 | 20260506 09:49:22.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:22.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:22.414 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:49:22.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:22.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:22.414 | INFO | Uploaded 36GiB to GPU in 1.60571s (22.42GiB/s) | 20260506 09:49:22.415 | INFO | Downloaded 34GiB to GPU in 1.52363s (22.3151GiB/s) | 20260506 09:49:22.512 | INFO | Starting a round of Baz IO | 20260506 09:49:25.432 | INFO | Populating RTMetrics report | 20260506 09:49:25.432 | INFO | RTMetrics report populated | 20260506 09:49:25.432 | INFO | Opening tmp RTMetrics json file | 20260506 09:49:25.447 | INFO | Serializing RTMetrics to json | 20260506 09:49:25.450 | INFO | Serialization done, swapping tmp file now... | 20260506 09:49:25.451 | INFO | RTMetrics done | 20260506 09:49:25.957 | INFO | Finished a round of Baz IO: Wrote 1.908 GiB in 3.445 seconds | 20260506 09:49:25.958 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.40% / 0.08% / 48.53% | 20260506 09:49:25.958 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.37981% / 0.000238419% / 5.16574e-05% | 20260506 09:49:25.958 | INFO | Baz IO overall: | 20260506 09:49:25.958 | INFO | BytesWritten : 168411826567 | 20260506 09:49:25.958 | INFO | NumEvents : 80358731400 | 20260506 09:49:27.553 | INFO | Analyzing chunk frames = [405504,406016) | 20260506 09:49:27.553 | INFO | MemInfo: { | 20260506 09:49:27.553 | INFO | "gpu" : " 17 GB", | 20260506 09:49:27.553 | INFO | "huge" : "", | 20260506 09:49:27.553 | INFO | "huge_shared" : " 27 GB", | 20260506 09:49:27.556 | INFO | "malloc" : " 14 GB", | 20260506 09:49:27.556 | INFO | "pinned" : " 29 GB" | 20260506 09:49:27.556 | INFO | } | 20260506 09:49:27.556 | INFO | Perf report for the last 5.14138 seconds: | 20260506 09:49:27.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:49:27.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:27.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:27.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:27.556 | INFO | ANALYSIS : 57.7%, 2.8, 85.7ms, 0 | 20260506 09:49:27.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:27.556 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 09:49:27.556 | INFO | PRE_HQ : 56.9%, 2.1, 60.9ms, 0 | 20260506 09:49:27.556 | INFO | BAZWRITER : 66.7%, 4.8, 164.6ms, 0 | 20260506 09:49:27.556 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 09:49:27.556 | INFO | Uploaded 36GiB to GPU in 1.67404s (21.5048GiB/s) | 20260506 09:49:27.556 | INFO | Downloaded 38GiB to GPU in 1.73842s (21.859GiB/s) | 20260506 09:49:32.693 | INFO | Analyzing chunk frames = [406016,406528) | 20260506 09:49:32.693 | INFO | MemInfo: { | 20260506 09:49:32.693 | INFO | "gpu" : " 17 GB", | 20260506 09:49:32.693 | INFO | "huge" : "", | 20260506 09:49:32.693 | INFO | "huge_shared" : " 27 GB", | 20260506 09:49:32.693 | INFO | "malloc" : " 14 GB", | 20260506 09:49:32.693 | INFO | "pinned" : " 29 GB" | 20260506 09:49:32.693 | INFO | } | 20260506 09:49:32.693 | INFO | Perf report for the last 5.13729 seconds: | 20260506 09:49:32.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:49:32.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:32.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:32.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:32.693 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 09:49:32.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:32.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:32.693 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:49:32.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:32.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:32.693 | INFO | Uploaded 36GiB to GPU in 1.60302s (22.4576GiB/s) | 20260506 09:49:32.694 | INFO | Downloaded 34GiB to GPU in 1.51991s (22.3698GiB/s) | 20260506 09:49:37.832 | INFO | Analyzing chunk frames = [406528,407040) | 20260506 09:49:37.832 | INFO | MemInfo: { | 20260506 09:49:37.832 | INFO | "gpu" : " 17 GB", | 20260506 09:49:37.832 | INFO | "huge" : "", | 20260506 09:49:37.832 | INFO | "huge_shared" : " 27 GB", | 20260506 09:49:37.832 | INFO | "malloc" : " 14 GB", | 20260506 09:49:37.832 | INFO | "pinned" : " 29 GB" | 20260506 09:49:37.832 | INFO | } | 20260506 09:49:37.832 | INFO | Perf report for the last 5.13912 seconds: | 20260506 09:49:37.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:49:37.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:37.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:37.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:37.833 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:49:37.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:37.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:37.833 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:49:37.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:37.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:37.833 | INFO | Uploaded 36GiB to GPU in 1.60622s (22.4129GiB/s) | 20260506 09:49:37.833 | INFO | Downloaded 34GiB to GPU in 1.52365s (22.3149GiB/s) | 20260506 09:49:42.971 | INFO | Analyzing chunk frames = [407040,407552) | 20260506 09:49:42.971 | INFO | MemInfo: { | 20260506 09:49:42.971 | INFO | "gpu" : " 17 GB", | 20260506 09:49:42.971 | INFO | "huge" : "", | 20260506 09:49:42.971 | INFO | "huge_shared" : " 27 GB", | 20260506 09:49:42.971 | INFO | "malloc" : " 14 GB", | 20260506 09:49:42.971 | INFO | "pinned" : " 29 GB" | 20260506 09:49:42.971 | INFO | } | 20260506 09:49:42.971 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:49:42.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:49:42.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:42.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:42.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:42.971 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:49:42.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:42.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:42.972 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:49:42.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:42.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:42.972 | INFO | Uploaded 36GiB to GPU in 1.60605s (22.4152GiB/s) | 20260506 09:49:42.972 | INFO | Downloaded 34GiB to GPU in 1.52303s (22.3239GiB/s) | 20260506 09:49:48.111 | INFO | Analyzing chunk frames = [407552,408064) | 20260506 09:49:48.111 | INFO | MemInfo: { | 20260506 09:49:48.111 | INFO | "gpu" : " 17 GB", | 20260506 09:49:48.111 | INFO | "huge" : "", | 20260506 09:49:48.111 | INFO | "huge_shared" : " 27 GB", | 20260506 09:49:48.111 | INFO | "malloc" : " 14 GB", | 20260506 09:49:48.111 | INFO | "pinned" : " 29 GB" | 20260506 09:49:48.111 | INFO | } | 20260506 09:49:48.111 | INFO | Perf report for the last 5.13957 seconds: | 20260506 09:49:48.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:49:48.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:48.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:48.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:48.111 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 09:49:48.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:48.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:48.111 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:49:48.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:48.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:48.112 | INFO | Uploaded 36GiB to GPU in 1.60493s (22.4308GiB/s) | 20260506 09:49:48.112 | INFO | Downloaded 34GiB to GPU in 1.52197s (22.3394GiB/s) | 20260506 09:49:53.250 | INFO | Analyzing chunk frames = [408064,408576) | 20260506 09:49:53.250 | INFO | MemInfo: { | 20260506 09:49:53.250 | INFO | "gpu" : " 17 GB", | 20260506 09:49:53.250 | INFO | "huge" : "", | 20260506 09:49:53.250 | INFO | "huge_shared" : " 27 GB", | 20260506 09:49:53.250 | INFO | "malloc" : " 14 GB", | 20260506 09:49:53.250 | INFO | "pinned" : " 29 GB" | 20260506 09:49:53.250 | INFO | } | 20260506 09:49:53.250 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:49:53.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:49:53.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:53.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:53.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:53.250 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:49:53.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:53.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:53.250 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260506 09:49:53.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:53.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:53.251 | INFO | Uploaded 36GiB to GPU in 1.60373s (22.4477GiB/s) | 20260506 09:49:53.251 | INFO | Downloaded 34GiB to GPU in 1.52052s (22.3608GiB/s) | 20260506 09:49:58.389 | INFO | Analyzing chunk frames = [408576,409088) | 20260506 09:49:58.389 | INFO | MemInfo: { | 20260506 09:49:58.389 | INFO | "gpu" : " 17 GB", | 20260506 09:49:58.389 | INFO | "huge" : "", | 20260506 09:49:58.389 | INFO | "huge_shared" : " 27 GB", | 20260506 09:49:58.389 | INFO | "malloc" : " 14 GB", | 20260506 09:49:58.390 | INFO | "pinned" : " 29 GB" | 20260506 09:49:58.390 | INFO | } | 20260506 09:49:58.390 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:49:58.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:49:58.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:58.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:58.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:58.390 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:49:58.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:58.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:58.390 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:49:58.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:58.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:49:58.390 | INFO | Uploaded 36GiB to GPU in 1.60448s (22.4372GiB/s) | 20260506 09:49:58.390 | INFO | Downloaded 34GiB to GPU in 1.52282s (22.3271GiB/s) | 20260506 09:50:03.529 | INFO | Analyzing chunk frames = [409088,409600) | 20260506 09:50:03.529 | INFO | MemInfo: { | 20260506 09:50:03.529 | INFO | "gpu" : " 17 GB", | 20260506 09:50:03.529 | INFO | "huge" : "", | 20260506 09:50:03.529 | INFO | "huge_shared" : " 27 GB", | 20260506 09:50:03.529 | INFO | "malloc" : " 14 GB", | 20260506 09:50:03.529 | INFO | "pinned" : " 29 GB" | 20260506 09:50:03.529 | INFO | } | 20260506 09:50:03.529 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:50:03.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:50:03.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:03.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:03.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:03.529 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:50:03.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:03.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:03.529 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 09:50:03.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:03.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:03.529 | INFO | Uploaded 36GiB to GPU in 1.60547s (22.4233GiB/s) | 20260506 09:50:03.529 | INFO | Downloaded 34GiB to GPU in 1.52322s (22.3211GiB/s) | 20260506 09:50:03.529 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:50:03.529 | INFO | Total mem usage: 15.9 GiB | 20260506 09:50:03.529 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 2.53%) | 20260506 09:50:03.529 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: 0.0483%) | 20260506 09:50:03.529 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:50:03.529 | INFO | High Water Breakdown: | 20260506 09:50:03.530 | INFO | 60.4% : 9.63 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:50:03.530 | INFO | 39.5% : 6.29 GiB ( 93.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:50:03.530 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:50:03.530 | INFO | ------------------------------------------ | 20260506 09:50:03.530 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:50:03.530 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:50:03.530 | INFO | Total mem usage: 26.5 GiB | 20260506 09:50:03.530 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:50:03.530 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:50:03.530 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:50:03.530 | INFO | High Water Breakdown: | 20260506 09:50:03.530 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:50:03.530 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:50:03.530 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:50:03.530 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:50:03.530 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:50:03.530 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:50:03.530 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:50:03.530 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:50:03.530 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:50:03.530 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:50:03.530 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:50:03.530 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:50:03.530 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:50:03.530 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:50:03.530 | INFO | ------------------------------------------ | 20260506 09:50:03.530 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:50:03.530 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:50:03.530 | INFO | Total mem usage: 25.6 GiB | 20260506 09:50:03.530 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:50:03.530 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:50:03.530 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:50:03.530 | INFO | High Water Breakdown: | 20260506 09:50:03.530 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:50:03.530 | INFO | ------------------------------------------ | 20260506 09:50:03.530 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:50:03.530 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:50:03.530 | INFO | Total mem usage: 15.7 GiB | 20260506 09:50:03.530 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:50:03.530 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:50:03.530 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:50:03.530 | INFO | High Water Breakdown: | 20260506 09:50:03.530 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:50:03.530 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:50:03.530 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:50:03.530 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:50:03.530 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:50:03.530 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:50:03.530 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:50:03.530 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:50:03.530 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:50:03.530 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:50:03.530 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:50:03.530 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:50:03.530 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:50:03.530 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:50:03.530 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:50:03.530 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:50:03.530 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:50:03.531 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:50:03.531 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:50:03.531 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:50:03.531 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:50:03.531 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:50:03.531 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:50:03.531 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:50:03.531 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:50:03.531 | INFO | ------------------------------------------ | 20260506 09:50:03.531 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:50:03.531 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:50:03.531 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:50:03.531 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:50:03.531 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 09:50:03.531 | INFO | PerfReport: 0.29%, AggregateStats -- 8.63e-02 / -nan, 5.46e-03 / 0.00e+00 | 20260506 09:50:03.531 | INFO | PerfReport: 10.91%, Baseline -- 3.29e+00 / -nan, 1.74e-02 / 0.00e+00 | 20260506 09:50:03.531 | INFO | PerfReport: 3.05%, Histogram -- 9.22e-01 / -nan, 5.26e-02 / 0.00e+00 | 20260506 09:50:03.531 | INFO | PerfReport: 16.05%, DME -- 3.87e+01 / -nan, 3.11e+00 / 0.00e+00 | 20260506 09:50:03.531 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.72e+01 / -nan, 4.19e-02 / 0.00e+00 | 20260506 09:50:03.531 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.25e+00 / -nan, 1.33e-01 / 0.00e+00 | 20260506 09:50:03.531 | INFO | PerfReport: 3.78%, Metrics -- 1.14e+00 / -nan, 1.75e-01 / 0.00e+00 | 20260506 09:50:03.531 | INFO | PerfReport: 1.47%, PulseSerializing -- 4.42e-01 / -nan, 6.35e-02 / 0.00e+00 | 20260506 09:50:03.531 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:50:03.531 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:50:03.531 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:50:03.531 | INFO | PerfReport: Full Execution -- 9.01e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 09:50:03.531 | INFO | PerfReport: 18.66%, Upload -- 1.68e+01 / 0.00e+00, 4.93e-01 / 0.00e+00 | 20260506 09:50:03.531 | INFO | PerfReport: 17.92%, Download -- 1.61e+01 / 0.00e+00, 8.47e-01 / 0.00e+00 | 20260506 09:50:03.531 | INFO | PerfReport: 33.50%, Compute -- 3.02e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 09:50:03.531 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 09:50:03.531 | INFO | PerfReport: 0.00%, LockDownload -- 4.79e-04 / 0.00e+00, 1.21e-04 / 0.00e+00 | 20260506 09:50:03.531 | INFO | PerfReport: 29.34%, LockCompute -- 2.64e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 09:50:03.531 | INFO | PerfReport: 0.02%, Misc -- 1.42e-02 / 0.00e+00, 2.55e-02 / 0.00e+00 | 20260506 09:50:03.626 | INFO | Starting a round of Baz IO | 20260506 09:50:06.546 | INFO | Populating RTMetrics report | 20260506 09:50:06.546 | INFO | RTMetrics report populated | 20260506 09:50:06.546 | INFO | Opening tmp RTMetrics json file | 20260506 09:50:06.549 | INFO | Serializing RTMetrics to json | 20260506 09:50:06.553 | INFO | Serialization done, swapping tmp file now... | 20260506 09:50:06.556 | INFO | RTMetrics done | 20260506 09:50:06.963 | INFO | Finished a round of Baz IO: Wrote 1.91 GiB in 3.336 seconds | 20260506 09:50:06.963 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.32% / 0.08% / 48.60% | 20260506 09:50:06.963 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.36219% / 0.000162919% / 3.57628e-05% | 20260506 09:50:06.963 | INFO | Baz IO overall: | 20260506 09:50:06.963 | INFO | BytesWritten : 170463121821 | 20260506 09:50:06.963 | INFO | NumEvents : 81355127608 | 20260506 09:50:08.668 | INFO | Analyzing chunk frames = [409600,410112) | 20260506 09:50:08.668 | INFO | MemInfo: { | 20260506 09:50:08.668 | INFO | "gpu" : " 17 GB", | 20260506 09:50:08.668 | INFO | "huge" : "", | 20260506 09:50:08.668 | INFO | "huge_shared" : " 27 GB", | 20260506 09:50:08.668 | INFO | "malloc" : " 14 GB", | 20260506 09:50:08.668 | INFO | "pinned" : " 29 GB" | 20260506 09:50:08.668 | INFO | } | 20260506 09:50:08.668 | INFO | Perf report for the last 5.13915 seconds: | 20260506 09:50:08.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:50:08.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:08.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:08.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:08.668 | INFO | ANALYSIS : 57.7%, 2.9, 87.0ms, 0 | 20260506 09:50:08.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:08.668 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 09:50:08.668 | INFO | PRE_HQ : 56.8%, 2.1, 61.2ms, 0 | 20260506 09:50:08.668 | INFO | BAZWRITER : 64.5%, 4.7, 154.4ms, 0 | 20260506 09:50:08.668 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 09:50:08.669 | INFO | Uploaded 36GiB to GPU in 1.66256s (21.6534GiB/s) | 20260506 09:50:08.669 | INFO | Downloaded 38GiB to GPU in 1.72668s (22.0075GiB/s) | 20260506 09:50:13.808 | INFO | Analyzing chunk frames = [410112,410624) | 20260506 09:50:13.808 | INFO | MemInfo: { | 20260506 09:50:13.808 | INFO | "gpu" : " 17 GB", | 20260506 09:50:13.808 | INFO | "huge" : "", | 20260506 09:50:13.808 | INFO | "huge_shared" : " 27 GB", | 20260506 09:50:13.808 | INFO | "malloc" : " 14 GB", | 20260506 09:50:13.808 | INFO | "pinned" : " 29 GB" | 20260506 09:50:13.808 | INFO | } | 20260506 09:50:13.808 | INFO | Perf report for the last 5.13967 seconds: | 20260506 09:50:13.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:50:13.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:13.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:13.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:13.808 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 09:50:13.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:13.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:13.808 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:50:13.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:13.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:13.808 | INFO | Uploaded 36GiB to GPU in 1.60187s (22.4738GiB/s) | 20260506 09:50:13.808 | INFO | Downloaded 34GiB to GPU in 1.51885s (22.3854GiB/s) | 20260506 09:50:18.947 | INFO | Analyzing chunk frames = [410624,411136) | 20260506 09:50:18.947 | INFO | MemInfo: { | 20260506 09:50:18.947 | INFO | "gpu" : " 17 GB", | 20260506 09:50:18.947 | INFO | "huge" : "", | 20260506 09:50:18.947 | INFO | "huge_shared" : " 27 GB", | 20260506 09:50:18.947 | INFO | "malloc" : " 14 GB", | 20260506 09:50:18.947 | INFO | "pinned" : " 29 GB" | 20260506 09:50:18.947 | INFO | } | 20260506 09:50:18.947 | INFO | Perf report for the last 5.13909 seconds: | 20260506 09:50:18.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:50:18.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:18.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:18.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:18.947 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:50:18.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:18.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:18.947 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260506 09:50:18.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:18.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:18.947 | INFO | Uploaded 36GiB to GPU in 1.60728s (22.3981GiB/s) | 20260506 09:50:18.948 | INFO | Downloaded 34GiB to GPU in 1.5255s (22.2877GiB/s) | 20260506 09:50:24.086 | INFO | Analyzing chunk frames = [411136,411648) | 20260506 09:50:24.086 | INFO | MemInfo: { | 20260506 09:50:24.086 | INFO | "gpu" : " 17 GB", | 20260506 09:50:24.086 | INFO | "huge" : "", | 20260506 09:50:24.086 | INFO | "huge_shared" : " 27 GB", | 20260506 09:50:24.086 | INFO | "malloc" : " 14 GB", | 20260506 09:50:24.086 | INFO | "pinned" : " 29 GB" | 20260506 09:50:24.087 | INFO | } | 20260506 09:50:24.087 | INFO | Perf report for the last 5.13964 seconds: | 20260506 09:50:24.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:50:24.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:24.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:24.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:24.087 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:50:24.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:24.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:24.087 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:50:24.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:24.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:24.087 | INFO | Uploaded 36GiB to GPU in 1.60662s (22.4073GiB/s) | 20260506 09:50:24.087 | INFO | Downloaded 34GiB to GPU in 1.52304s (22.3237GiB/s) | 20260506 09:50:29.225 | INFO | Analyzing chunk frames = [411648,412160) | 20260506 09:50:29.225 | INFO | MemInfo: { | 20260506 09:50:29.225 | INFO | "gpu" : " 17 GB", | 20260506 09:50:29.225 | INFO | "huge" : "", | 20260506 09:50:29.225 | INFO | "huge_shared" : " 27 GB", | 20260506 09:50:29.225 | INFO | "malloc" : " 14 GB", | 20260506 09:50:29.225 | INFO | "pinned" : " 29 GB" | 20260506 09:50:29.226 | INFO | } | 20260506 09:50:29.226 | INFO | Perf report for the last 5.13901 seconds: | 20260506 09:50:29.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:50:29.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:29.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:29.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:29.226 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 09:50:29.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:29.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:29.226 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:50:29.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:29.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:29.226 | INFO | Uploaded 36GiB to GPU in 1.60462s (22.4353GiB/s) | 20260506 09:50:29.226 | INFO | Downloaded 34GiB to GPU in 1.52092s (22.3549GiB/s) | 20260506 09:50:34.365 | INFO | Analyzing chunk frames = [412160,412672) | 20260506 09:50:34.365 | INFO | MemInfo: { | 20260506 09:50:34.365 | INFO | "gpu" : " 17 GB", | 20260506 09:50:34.365 | INFO | "huge" : "", | 20260506 09:50:34.365 | INFO | "huge_shared" : " 27 GB", | 20260506 09:50:34.365 | INFO | "malloc" : " 14 GB", | 20260506 09:50:34.365 | INFO | "pinned" : " 29 GB" | 20260506 09:50:34.365 | INFO | } | 20260506 09:50:34.365 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:50:34.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:50:34.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:34.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:34.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:34.365 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 09:50:34.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:34.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:34.365 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:50:34.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:34.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:34.365 | INFO | Uploaded 36GiB to GPU in 1.60544s (22.4238GiB/s) | 20260506 09:50:34.365 | INFO | Downloaded 34GiB to GPU in 1.52372s (22.3139GiB/s) | 20260506 09:50:39.504 | INFO | Analyzing chunk frames = [412672,413184) | 20260506 09:50:39.504 | INFO | MemInfo: { | 20260506 09:50:39.504 | INFO | "gpu" : " 17 GB", | 20260506 09:50:39.504 | INFO | "huge" : "", | 20260506 09:50:39.504 | INFO | "huge_shared" : " 27 GB", | 20260506 09:50:39.504 | INFO | "malloc" : " 14 GB", | 20260506 09:50:39.504 | INFO | "pinned" : " 29 GB" | 20260506 09:50:39.504 | INFO | } | 20260506 09:50:39.504 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:50:39.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:50:39.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:39.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:39.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:39.504 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:50:39.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:39.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:39.504 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:50:39.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:39.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:39.505 | INFO | Uploaded 36GiB to GPU in 1.60433s (22.4393GiB/s) | 20260506 09:50:39.505 | INFO | Downloaded 34GiB to GPU in 1.52323s (22.321GiB/s) | 20260506 09:50:44.644 | INFO | Analyzing chunk frames = [413184,413696) | 20260506 09:50:44.644 | INFO | MemInfo: { | 20260506 09:50:44.644 | INFO | "gpu" : " 17 GB", | 20260506 09:50:44.644 | INFO | "huge" : "", | 20260506 09:50:44.644 | INFO | "huge_shared" : " 27 GB", | 20260506 09:50:44.644 | INFO | "malloc" : " 14 GB", | 20260506 09:50:44.644 | INFO | "pinned" : " 29 GB" | 20260506 09:50:44.644 | INFO | } | 20260506 09:50:44.644 | INFO | Perf report for the last 5.1394 seconds: | 20260506 09:50:44.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:50:44.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:44.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:44.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:44.644 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:50:44.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:44.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:44.644 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 09:50:44.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:44.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:44.644 | INFO | Uploaded 36GiB to GPU in 1.60453s (22.4365GiB/s) | 20260506 09:50:44.644 | INFO | Downloaded 34GiB to GPU in 1.52293s (22.3253GiB/s) | 20260506 09:50:44.741 | INFO | Starting a round of Baz IO | 20260506 09:50:47.659 | INFO | Populating RTMetrics report | 20260506 09:50:47.659 | INFO | RTMetrics report populated | 20260506 09:50:47.659 | INFO | Opening tmp RTMetrics json file | 20260506 09:50:47.696 | INFO | Serializing RTMetrics to json | 20260506 09:50:47.733 | INFO | Serialization done, swapping tmp file now... | 20260506 09:50:47.739 | INFO | RTMetrics done | 20260506 09:50:47.981 | INFO | Finished a round of Baz IO: Wrote 1.912 GiB in 3.239 seconds | 20260506 09:50:47.981 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.24% / 0.08% / 48.68% | 20260506 09:50:47.981 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.34314% / 0.00021855% / 5.5631e-05% | 20260506 09:50:47.981 | INFO | Baz IO overall: | 20260506 09:50:47.981 | INFO | BytesWritten : 172516338215 | 20260506 09:50:47.981 | INFO | NumEvents : 82352895093 | 20260506 09:50:49.783 | INFO | Analyzing chunk frames = [413696,414208) | 20260506 09:50:49.783 | INFO | MemInfo: { | 20260506 09:50:49.783 | INFO | "gpu" : " 17 GB", | 20260506 09:50:49.783 | INFO | "huge" : "", | 20260506 09:50:49.783 | INFO | "huge_shared" : " 27 GB", | 20260506 09:50:49.783 | INFO | "malloc" : " 14 GB", | 20260506 09:50:49.783 | INFO | "pinned" : " 29 GB" | 20260506 09:50:49.783 | INFO | } | 20260506 09:50:49.783 | INFO | Perf report for the last 5.13923 seconds: | 20260506 09:50:49.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:50:49.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:49.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:49.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:49.783 | INFO | ANALYSIS : 57.7%, 2.9, 87.5ms, 0 | 20260506 09:50:49.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:49.783 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260506 09:50:49.783 | INFO | PRE_HQ : 56.8%, 2.1, 61.9ms, 0 | 20260506 09:50:49.783 | INFO | BAZWRITER : 62.6%, 4.5, 145.5ms, 0 | 20260506 09:50:49.784 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 09:50:49.784 | INFO | Uploaded 36GiB to GPU in 1.65164s (21.7965GiB/s) | 20260506 09:50:49.784 | INFO | Downloaded 38GiB to GPU in 1.71581s (22.147GiB/s) | 20260506 09:50:54.922 | INFO | Analyzing chunk frames = [414208,414720) | 20260506 09:50:54.922 | INFO | MemInfo: { | 20260506 09:50:54.922 | INFO | "gpu" : " 17 GB", | 20260506 09:50:54.922 | INFO | "huge" : "", | 20260506 09:50:54.922 | INFO | "huge_shared" : " 27 GB", | 20260506 09:50:54.922 | INFO | "malloc" : " 14 GB", | 20260506 09:50:54.922 | INFO | "pinned" : " 29 GB" | 20260506 09:50:54.922 | INFO | } | 20260506 09:50:54.923 | INFO | Perf report for the last 5.1395 seconds: | 20260506 09:50:54.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:50:54.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:54.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:54.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:54.923 | INFO | ANALYSIS : 57.5%, 3.0, 89.8ms, 0 | 20260506 09:50:54.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:54.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:54.923 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:50:54.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:54.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:50:54.923 | INFO | Uploaded 36GiB to GPU in 1.60191s (22.4732GiB/s) | 20260506 09:50:54.923 | INFO | Downloaded 34GiB to GPU in 1.51896s (22.3838GiB/s) | 20260506 09:51:00.062 | INFO | Analyzing chunk frames = [414720,415232) | 20260506 09:51:00.062 | INFO | MemInfo: { | 20260506 09:51:00.062 | INFO | "gpu" : " 17 GB", | 20260506 09:51:00.062 | INFO | "huge" : "", | 20260506 09:51:00.062 | INFO | "huge_shared" : " 27 GB", | 20260506 09:51:00.062 | INFO | "malloc" : " 14 GB", | 20260506 09:51:00.062 | INFO | "pinned" : " 29 GB" | 20260506 09:51:00.062 | INFO | } | 20260506 09:51:00.062 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:51:00.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:51:00.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:00.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:00.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:00.062 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:51:00.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:00.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:00.062 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:51:00.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:00.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:00.062 | INFO | Uploaded 36GiB to GPU in 1.60653s (22.4086GiB/s) | 20260506 09:51:00.062 | INFO | Downloaded 34GiB to GPU in 1.52501s (22.295GiB/s) | 20260506 09:51:05.201 | INFO | Analyzing chunk frames = [415232,415744) | 20260506 09:51:05.201 | INFO | MemInfo: { | 20260506 09:51:05.201 | INFO | "gpu" : " 17 GB", | 20260506 09:51:05.201 | INFO | "huge" : "", | 20260506 09:51:05.201 | INFO | "huge_shared" : " 27 GB", | 20260506 09:51:05.201 | INFO | "malloc" : " 14 GB", | 20260506 09:51:05.201 | INFO | "pinned" : " 29 GB" | 20260506 09:51:05.201 | INFO | } | 20260506 09:51:05.201 | INFO | Perf report for the last 5.13953 seconds: | 20260506 09:51:05.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:51:05.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:05.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:05.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:05.201 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:51:05.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:05.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:05.201 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:51:05.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:05.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:05.202 | INFO | Uploaded 36GiB to GPU in 1.60593s (22.4169GiB/s) | 20260506 09:51:05.202 | INFO | Downloaded 34GiB to GPU in 1.52348s (22.3174GiB/s) | 20260506 09:51:10.340 | INFO | Analyzing chunk frames = [415744,416256) | 20260506 09:51:10.340 | INFO | MemInfo: { | 20260506 09:51:10.340 | INFO | "gpu" : " 17 GB", | 20260506 09:51:10.340 | INFO | "huge" : "", | 20260506 09:51:10.340 | INFO | "huge_shared" : " 27 GB", | 20260506 09:51:10.340 | INFO | "malloc" : " 14 GB", | 20260506 09:51:10.340 | INFO | "pinned" : " 29 GB" | 20260506 09:51:10.340 | INFO | } | 20260506 09:51:10.340 | INFO | Perf report for the last 5.13913 seconds: | 20260506 09:51:10.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:51:10.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:10.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:10.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:10.341 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 09:51:10.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:10.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:10.341 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:51:10.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:10.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:10.341 | INFO | Uploaded 36GiB to GPU in 1.60512s (22.4283GiB/s) | 20260506 09:51:10.341 | INFO | Downloaded 34GiB to GPU in 1.5213s (22.3493GiB/s) | 20260506 09:51:15.480 | INFO | Analyzing chunk frames = [416256,416768) | 20260506 09:51:15.480 | INFO | MemInfo: { | 20260506 09:51:15.480 | INFO | "gpu" : " 17 GB", | 20260506 09:51:15.480 | INFO | "huge" : "", | 20260506 09:51:15.480 | INFO | "huge_shared" : " 27 GB", | 20260506 09:51:15.480 | INFO | "malloc" : " 14 GB", | 20260506 09:51:15.480 | INFO | "pinned" : " 29 GB" | 20260506 09:51:15.480 | INFO | } | 20260506 09:51:15.480 | INFO | Perf report for the last 5.13932 seconds: | 20260506 09:51:15.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:51:15.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:15.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:15.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:15.480 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:51:15.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:15.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:15.484 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:51:15.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:15.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:15.484 | INFO | Uploaded 36GiB to GPU in 1.60423s (22.4406GiB/s) | 20260506 09:51:15.484 | INFO | Downloaded 34GiB to GPU in 1.52215s (22.3369GiB/s) | 20260506 09:51:20.619 | INFO | Analyzing chunk frames = [416768,417280) | 20260506 09:51:20.619 | INFO | MemInfo: { | 20260506 09:51:20.619 | INFO | "gpu" : " 17 GB", | 20260506 09:51:20.619 | INFO | "huge" : "", | 20260506 09:51:20.619 | INFO | "huge_shared" : " 27 GB", | 20260506 09:51:20.619 | INFO | "malloc" : " 14 GB", | 20260506 09:51:20.619 | INFO | "pinned" : " 29 GB" | 20260506 09:51:20.619 | INFO | } | 20260506 09:51:20.619 | INFO | Perf report for the last 5.13943 seconds: | 20260506 09:51:20.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:51:20.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:20.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:20.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:20.619 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:51:20.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:20.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:20.619 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 09:51:20.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:20.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:20.620 | INFO | Uploaded 36GiB to GPU in 1.60471s (22.434GiB/s) | 20260506 09:51:20.620 | INFO | Downloaded 34GiB to GPU in 1.52228s (22.335GiB/s) | 20260506 09:51:25.759 | INFO | Analyzing chunk frames = [417280,417792) | 20260506 09:51:25.759 | INFO | MemInfo: { | 20260506 09:51:25.759 | INFO | "gpu" : " 17 GB", | 20260506 09:51:25.759 | INFO | "huge" : "", | 20260506 09:51:25.759 | INFO | "huge_shared" : " 27 GB", | 20260506 09:51:25.759 | INFO | "malloc" : " 14 GB", | 20260506 09:51:25.759 | INFO | "pinned" : " 29 GB" | 20260506 09:51:25.759 | INFO | } | 20260506 09:51:25.759 | INFO | Perf report for the last 5.14017 seconds: | 20260506 09:51:25.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:51:25.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:25.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:25.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:25.760 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:51:25.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:25.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:25.760 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:51:25.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:25.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:25.760 | INFO | Uploaded 36GiB to GPU in 1.60502s (22.4297GiB/s) | 20260506 09:51:25.760 | INFO | Downloaded 34GiB to GPU in 1.52331s (22.3199GiB/s) | 20260506 09:51:25.760 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:51:25.760 | INFO | Total mem usage: 15.9 GiB | 20260506 09:51:25.760 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 2.53%) | 20260506 09:51:25.760 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -2.27%) | 20260506 09:51:25.760 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:51:25.760 | INFO | High Water Breakdown: | 20260506 09:51:25.760 | INFO | 60% : 9.35 GiB ( 97.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:51:25.760 | INFO | 39.9% : 6.21 GiB ( 92.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:51:25.760 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:51:25.760 | INFO | ------------------------------------------ | 20260506 09:51:25.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:51:25.760 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:51:25.760 | INFO | Total mem usage: 26.5 GiB | 20260506 09:51:25.760 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:51:25.760 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:51:25.760 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:51:25.760 | INFO | High Water Breakdown: | 20260506 09:51:25.760 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:51:25.760 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:51:25.760 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:51:25.760 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:51:25.760 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:51:25.760 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:51:25.760 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:51:25.760 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:51:25.760 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:51:25.760 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:51:25.760 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:51:25.760 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:51:25.760 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:51:25.760 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:51:25.760 | INFO | ------------------------------------------ | 20260506 09:51:25.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:51:25.760 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:51:25.760 | INFO | Total mem usage: 25.6 GiB | 20260506 09:51:25.760 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:51:25.760 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:51:25.760 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:51:25.760 | INFO | High Water Breakdown: | 20260506 09:51:25.760 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:51:25.760 | INFO | ------------------------------------------ | 20260506 09:51:25.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:51:25.760 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:51:25.761 | INFO | Total mem usage: 15.7 GiB | 20260506 09:51:25.761 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:51:25.761 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:51:25.761 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:51:25.761 | INFO | High Water Breakdown: | 20260506 09:51:25.761 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:51:25.761 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:51:25.761 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:51:25.761 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:51:25.761 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:51:25.761 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:51:25.761 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:51:25.761 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:51:25.761 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:51:25.761 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:51:25.761 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:51:25.761 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:51:25.761 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:51:25.761 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:51:25.761 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:51:25.761 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:51:25.761 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:51:25.761 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:51:25.761 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:51:25.761 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:51:25.761 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:51:25.761 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:51:25.761 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:51:25.761 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:51:25.761 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:51:25.761 | INFO | ------------------------------------------ | 20260506 09:51:25.761 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:51:25.761 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:51:25.761 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:51:25.761 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:51:25.761 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 09:51:25.761 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 6.46e-03 / 0.00e+00 | 20260506 09:51:25.761 | INFO | PerfReport: 10.91%, Baseline -- 3.29e+00 / -nan, 1.66e-02 / 0.00e+00 | 20260506 09:51:25.761 | INFO | PerfReport: 3.05%, Histogram -- 9.22e-01 / -nan, 5.15e-02 / 0.00e+00 | 20260506 09:51:25.761 | INFO | PerfReport: 16.04%, DME -- 3.87e+01 / -nan, 3.13e+00 / 0.00e+00 | 20260506 09:51:25.761 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.72e+01 / -nan, 4.65e-02 / 0.00e+00 | 20260506 09:51:25.761 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.25e+00 / -nan, 1.33e-01 / 0.00e+00 | 20260506 09:51:25.761 | INFO | PerfReport: 3.78%, Metrics -- 1.14e+00 / -nan, 1.77e-01 / 0.00e+00 | 20260506 09:51:25.761 | INFO | PerfReport: 1.47%, PulseSerializing -- 4.42e-01 / -nan, 6.39e-02 / 0.00e+00 | 20260506 09:51:25.761 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:51:25.761 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:51:25.761 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:51:25.761 | INFO | PerfReport: Full Execution -- 9.03e+01 / -nan, 1.85e+01 / 0.00e+00 | 20260506 09:51:25.761 | INFO | PerfReport: 18.59%, Upload -- 1.68e+01 / 0.00e+00, 4.39e-01 / 0.00e+00 | 20260506 09:51:25.761 | INFO | PerfReport: 17.85%, Download -- 1.61e+01 / 0.00e+00, 7.74e-01 / 0.00e+00 | 20260506 09:51:25.761 | INFO | PerfReport: 33.42%, Compute -- 3.02e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 09:51:25.761 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:51:25.761 | INFO | PerfReport: 0.00%, LockDownload -- 4.88e-04 / 0.00e+00, 1.91e-04 / 0.00e+00 | 20260506 09:51:25.761 | INFO | PerfReport: 29.57%, LockCompute -- 2.67e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 09:51:25.761 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.37e-02 / 0.00e+00 | 20260506 09:51:25.857 | INFO | Starting a round of Baz IO | 20260506 09:51:28.777 | INFO | Populating RTMetrics report | 20260506 09:51:28.777 | INFO | RTMetrics report populated | 20260506 09:51:28.777 | INFO | Opening tmp RTMetrics json file | 20260506 09:51:28.861 | INFO | Serializing RTMetrics to json | 20260506 09:51:28.900 | INFO | Serialization done, swapping tmp file now... | 20260506 09:51:28.905 | INFO | RTMetrics done | 20260506 09:51:28.922 | INFO | Finished a round of Baz IO: Wrote 1.913 GiB in 3.065 seconds | 20260506 09:51:28.922 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.17% / 0.08% / 48.76% | 20260506 09:51:28.922 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.29464% / 0.000190735% / 3.17891e-05% | 20260506 09:51:28.922 | INFO | Baz IO overall: | 20260506 09:51:28.922 | INFO | BytesWritten : 174570356425 | 20260506 09:51:28.922 | INFO | NumEvents : 83350255905 | 20260506 09:51:30.898 | INFO | Analyzing chunk frames = [417792,418304) | 20260506 09:51:30.898 | INFO | MemInfo: { | 20260506 09:51:30.898 | INFO | "gpu" : " 17 GB", | 20260506 09:51:30.898 | INFO | "huge" : "", | 20260506 09:51:30.898 | INFO | "huge_shared" : " 27 GB", | 20260506 09:51:30.898 | INFO | "malloc" : " 14 GB", | 20260506 09:51:30.898 | INFO | "pinned" : " 29 GB" | 20260506 09:51:30.898 | INFO | } | 20260506 09:51:30.898 | INFO | Perf report for the last 5.13841 seconds: | 20260506 09:51:30.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:51:30.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:30.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:30.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:30.898 | INFO | ANALYSIS : 57.7%, 2.9, 87.5ms, 0 | 20260506 09:51:30.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:30.898 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260506 09:51:30.898 | INFO | PRE_HQ : 56.8%, 2.1, 61.7ms, 0 | 20260506 09:51:30.898 | INFO | BAZWRITER : 59.2%, 4.5, 137.5ms, 0 | 20260506 09:51:30.898 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 09:51:30.899 | INFO | Uploaded 36GiB to GPU in 1.65389s (21.7669GiB/s) | 20260506 09:51:30.899 | INFO | Downloaded 38GiB to GPU in 1.71663s (22.1364GiB/s) | 20260506 09:51:36.037 | INFO | Analyzing chunk frames = [418304,418816) | 20260506 09:51:36.037 | INFO | MemInfo: { | 20260506 09:51:36.037 | INFO | "gpu" : " 17 GB", | 20260506 09:51:36.037 | INFO | "huge" : "", | 20260506 09:51:36.037 | INFO | "huge_shared" : " 27 GB", | 20260506 09:51:36.037 | INFO | "malloc" : " 14 GB", | 20260506 09:51:36.037 | INFO | "pinned" : " 29 GB" | 20260506 09:51:36.037 | INFO | } | 20260506 09:51:36.037 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:51:36.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:51:36.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:36.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:36.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:36.037 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 09:51:36.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:36.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:36.037 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:51:36.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:36.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:36.038 | INFO | Uploaded 36GiB to GPU in 1.60253s (22.4645GiB/s) | 20260506 09:51:36.038 | INFO | Downloaded 34GiB to GPU in 1.51924s (22.3796GiB/s) | 20260506 09:51:41.177 | INFO | Analyzing chunk frames = [418816,419328) | 20260506 09:51:41.177 | INFO | MemInfo: { | 20260506 09:51:41.177 | INFO | "gpu" : " 17 GB", | 20260506 09:51:41.177 | INFO | "huge" : "", | 20260506 09:51:41.177 | INFO | "huge_shared" : " 27 GB", | 20260506 09:51:41.177 | INFO | "malloc" : " 14 GB", | 20260506 09:51:41.177 | INFO | "pinned" : " 29 GB" | 20260506 09:51:41.177 | INFO | } | 20260506 09:51:41.177 | INFO | Perf report for the last 5.1394 seconds: | 20260506 09:51:41.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:51:41.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:41.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:41.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:41.177 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:51:41.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:41.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:41.177 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:51:41.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:41.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:41.177 | INFO | Uploaded 36GiB to GPU in 1.60619s (22.4133GiB/s) | 20260506 09:51:41.177 | INFO | Downloaded 34GiB to GPU in 1.52385s (22.3118GiB/s) | 20260506 09:51:46.316 | INFO | Analyzing chunk frames = [419328,419840) | 20260506 09:51:46.316 | INFO | MemInfo: { | 20260506 09:51:46.316 | INFO | "gpu" : " 17 GB", | 20260506 09:51:46.316 | INFO | "huge" : "", | 20260506 09:51:46.316 | INFO | "huge_shared" : " 27 GB", | 20260506 09:51:46.316 | INFO | "malloc" : " 14 GB", | 20260506 09:51:46.316 | INFO | "pinned" : " 29 GB" | 20260506 09:51:46.316 | INFO | } | 20260506 09:51:46.316 | INFO | Perf report for the last 5.13945 seconds: | 20260506 09:51:46.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:51:46.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:46.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:46.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:46.316 | INFO | ANALYSIS : 56.6%, 3.0, 87.7ms, 0 | 20260506 09:51:46.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:46.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:46.316 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:51:46.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:46.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:46.317 | INFO | Uploaded 36GiB to GPU in 1.60658s (22.4078GiB/s) | 20260506 09:51:46.317 | INFO | Downloaded 34GiB to GPU in 1.52368s (22.3145GiB/s) | 20260506 09:51:51.455 | INFO | Analyzing chunk frames = [419840,420352) | 20260506 09:51:51.455 | INFO | MemInfo: { | 20260506 09:51:51.455 | INFO | "gpu" : " 17 GB", | 20260506 09:51:51.455 | INFO | "huge" : "", | 20260506 09:51:51.455 | INFO | "huge_shared" : " 27 GB", | 20260506 09:51:51.455 | INFO | "malloc" : " 14 GB", | 20260506 09:51:51.455 | INFO | "pinned" : " 29 GB" | 20260506 09:51:51.455 | INFO | } | 20260506 09:51:51.456 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:51:51.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:51:51.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:51.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:51.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:51.456 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 09:51:51.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:51.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:51.456 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:51:51.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:51.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:51.456 | INFO | Uploaded 36GiB to GPU in 1.6044s (22.4384GiB/s) | 20260506 09:51:51.456 | INFO | Downloaded 34GiB to GPU in 1.52108s (22.3526GiB/s) | 20260506 09:51:56.595 | INFO | Analyzing chunk frames = [420352,420864) | 20260506 09:51:56.595 | INFO | MemInfo: { | 20260506 09:51:56.595 | INFO | "gpu" : " 17 GB", | 20260506 09:51:56.595 | INFO | "huge" : "", | 20260506 09:51:56.595 | INFO | "huge_shared" : " 27 GB", | 20260506 09:51:56.595 | INFO | "malloc" : " 14 GB", | 20260506 09:51:56.595 | INFO | "pinned" : " 29 GB" | 20260506 09:51:56.595 | INFO | } | 20260506 09:51:56.595 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:51:56.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:51:56.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:56.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:56.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:56.595 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 09:51:56.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:56.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:56.595 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:51:56.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:56.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:51:56.595 | INFO | Uploaded 36GiB to GPU in 1.60453s (22.4365GiB/s) | 20260506 09:51:56.595 | INFO | Downloaded 34GiB to GPU in 1.52112s (22.3519GiB/s) | 20260506 09:52:01.734 | INFO | Analyzing chunk frames = [420864,421376) | 20260506 09:52:01.734 | INFO | MemInfo: { | 20260506 09:52:01.734 | INFO | "gpu" : " 17 GB", | 20260506 09:52:01.734 | INFO | "huge" : "", | 20260506 09:52:01.734 | INFO | "huge_shared" : " 27 GB", | 20260506 09:52:01.734 | INFO | "malloc" : " 14 GB", | 20260506 09:52:01.734 | INFO | "pinned" : " 29 GB" | 20260506 09:52:01.734 | INFO | } | 20260506 09:52:01.734 | INFO | Perf report for the last 5.13963 seconds: | 20260506 09:52:01.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:52:01.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:01.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:01.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:01.735 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260506 09:52:01.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:01.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:01.735 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 09:52:01.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:01.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:01.735 | INFO | Uploaded 36GiB to GPU in 1.60545s (22.4236GiB/s) | 20260506 09:52:01.735 | INFO | Downloaded 34GiB to GPU in 1.52334s (22.3193GiB/s) | 20260506 09:52:06.873 | INFO | Analyzing chunk frames = [421376,421888) | 20260506 09:52:06.873 | INFO | MemInfo: { | 20260506 09:52:06.873 | INFO | "gpu" : " 17 GB", | 20260506 09:52:06.873 | INFO | "huge" : "", | 20260506 09:52:06.873 | INFO | "huge_shared" : " 27 GB", | 20260506 09:52:06.874 | INFO | "malloc" : " 14 GB", | 20260506 09:52:06.874 | INFO | "pinned" : " 29 GB" | 20260506 09:52:06.874 | INFO | } | 20260506 09:52:06.874 | INFO | Perf report for the last 5.13914 seconds: | 20260506 09:52:06.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:52:06.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:06.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:06.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:06.874 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:52:06.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:06.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:06.874 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 09:52:06.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:06.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:06.874 | INFO | Uploaded 36GiB to GPU in 1.60554s (22.4224GiB/s) | 20260506 09:52:06.874 | INFO | Downloaded 34GiB to GPU in 1.52348s (22.3173GiB/s) | 20260506 09:52:06.971 | INFO | Starting a round of Baz IO | 20260506 09:52:09.891 | INFO | Populating RTMetrics report | 20260506 09:52:09.891 | INFO | RTMetrics report populated | 20260506 09:52:09.891 | INFO | Opening tmp RTMetrics json file | 20260506 09:52:09.893 | INFO | Serializing RTMetrics to json | 20260506 09:52:09.895 | INFO | Serialization done, swapping tmp file now... | 20260506 09:52:09.896 | INFO | RTMetrics done | 20260506 09:52:09.994 | INFO | Finished a round of Baz IO: Wrote 1.916 GiB in 3.023 seconds | 20260506 09:52:09.994 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.09% / 0.07% / 48.83% | 20260506 09:52:09.994 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.27279% / 0.000147025% / 3.17891e-05% | 20260506 09:52:09.994 | INFO | Baz IO overall: | 20260506 09:52:09.994 | INFO | BytesWritten : 176627717376 | 20260506 09:52:09.994 | INFO | NumEvents : 84349138478 | 20260506 09:52:12.015 | INFO | Analyzing chunk frames = [421888,422400) | 20260506 09:52:12.015 | INFO | MemInfo: { | 20260506 09:52:12.015 | INFO | "gpu" : " 17 GB", | 20260506 09:52:12.015 | INFO | "huge" : "", | 20260506 09:52:12.015 | INFO | "huge_shared" : " 27 GB", | 20260506 09:52:12.015 | INFO | "malloc" : " 14 GB", | 20260506 09:52:12.015 | INFO | "pinned" : " 29 GB" | 20260506 09:52:12.015 | INFO | } | 20260506 09:52:12.015 | INFO | Perf report for the last 5.14167 seconds: | 20260506 09:52:12.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:52:12.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:12.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:12.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:12.015 | INFO | ANALYSIS : 57.7%, 2.9, 87.1ms, 0 | 20260506 09:52:12.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:12.015 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 09:52:12.015 | INFO | PRE_HQ : 56.8%, 2.1, 61.5ms, 0 | 20260506 09:52:12.015 | INFO | BAZWRITER : 58.4%, 4.5, 136.0ms, 0 | 20260506 09:52:12.015 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 09:52:12.016 | INFO | Uploaded 36GiB to GPU in 1.6575s (21.7194GiB/s) | 20260506 09:52:12.016 | INFO | Downloaded 38GiB to GPU in 1.72207s (22.0665GiB/s) | 20260506 09:52:17.171 | INFO | Analyzing chunk frames = [422400,422912) | 20260506 09:52:17.171 | INFO | MemInfo: { | 20260506 09:52:17.171 | INFO | "gpu" : " 17 GB", | 20260506 09:52:17.171 | INFO | "huge" : "", | 20260506 09:52:17.171 | INFO | "huge_shared" : " 27 GB", | 20260506 09:52:17.171 | INFO | "malloc" : " 14 GB", | 20260506 09:52:17.171 | INFO | "pinned" : " 29 GB" | 20260506 09:52:17.171 | INFO | } | 20260506 09:52:17.171 | INFO | Perf report for the last 5.15616 seconds: | 20260506 09:52:17.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:52:17.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:17.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:17.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:17.172 | INFO | ANALYSIS : 57.3%, 3.0, 89.7ms, 0 | 20260506 09:52:17.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:17.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:17.172 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260506 09:52:17.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:17.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:17.172 | INFO | Uploaded 36GiB to GPU in 1.60269s (22.4622GiB/s) | 20260506 09:52:17.172 | INFO | Downloaded 34GiB to GPU in 1.51989s (22.37GiB/s) | 20260506 09:52:22.291 | INFO | Analyzing chunk frames = [422912,423424) | 20260506 09:52:22.291 | INFO | MemInfo: { | 20260506 09:52:22.291 | INFO | "gpu" : " 17 GB", | 20260506 09:52:22.291 | INFO | "huge" : "", | 20260506 09:52:22.291 | INFO | "huge_shared" : " 27 GB", | 20260506 09:52:22.291 | INFO | "malloc" : " 14 GB", | 20260506 09:52:22.291 | INFO | "pinned" : " 29 GB" | 20260506 09:52:22.292 | INFO | } | 20260506 09:52:22.292 | INFO | Perf report for the last 5.12006 seconds: | 20260506 09:52:22.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:52:22.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:22.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:22.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:22.292 | INFO | ANALYSIS : 57.1%, 3.0, 88.4ms, 0 | 20260506 09:52:22.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:22.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:22.292 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:52:22.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:22.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:22.292 | INFO | Uploaded 36GiB to GPU in 1.60684s (22.4042GiB/s) | 20260506 09:52:22.292 | INFO | Downloaded 34GiB to GPU in 1.52542s (22.2889GiB/s) | 20260506 09:52:27.431 | INFO | Analyzing chunk frames = [423424,423936) | 20260506 09:52:27.431 | INFO | MemInfo: { | 20260506 09:52:27.431 | INFO | "gpu" : " 17 GB", | 20260506 09:52:27.431 | INFO | "huge" : "", | 20260506 09:52:27.431 | INFO | "huge_shared" : " 27 GB", | 20260506 09:52:27.431 | INFO | "malloc" : " 14 GB", | 20260506 09:52:27.435 | INFO | "pinned" : " 29 GB" | 20260506 09:52:27.435 | INFO | } | 20260506 09:52:27.435 | INFO | Perf report for the last 5.1433 seconds: | 20260506 09:52:27.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:52:27.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:27.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:27.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:27.435 | INFO | ANALYSIS : 56.5%, 3.0, 87.7ms, 0 | 20260506 09:52:27.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:27.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:27.435 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:52:27.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:27.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:27.435 | INFO | Uploaded 36GiB to GPU in 1.60591s (22.4172GiB/s) | 20260506 09:52:27.435 | INFO | Downloaded 34GiB to GPU in 1.52312s (22.3226GiB/s) | 20260506 09:52:32.570 | INFO | Analyzing chunk frames = [423936,424448) | 20260506 09:52:32.570 | INFO | MemInfo: { | 20260506 09:52:32.570 | INFO | "gpu" : " 17 GB", | 20260506 09:52:32.570 | INFO | "huge" : "", | 20260506 09:52:32.570 | INFO | "huge_shared" : " 27 GB", | 20260506 09:52:32.570 | INFO | "malloc" : " 14 GB", | 20260506 09:52:32.570 | INFO | "pinned" : " 29 GB" | 20260506 09:52:32.570 | INFO | } | 20260506 09:52:32.570 | INFO | Perf report for the last 5.13524 seconds: | 20260506 09:52:32.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:52:32.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:32.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:32.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:32.574 | INFO | ANALYSIS : 56.6%, 3.0, 88.1ms, 0 | 20260506 09:52:32.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:32.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:32.574 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:52:32.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:32.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:32.574 | INFO | Uploaded 36GiB to GPU in 1.60455s (22.4362GiB/s) | 20260506 09:52:32.574 | INFO | Downloaded 34GiB to GPU in 1.52129s (22.3495GiB/s) | 20260506 09:52:37.709 | INFO | Analyzing chunk frames = [424448,424960) | 20260506 09:52:37.709 | INFO | MemInfo: { | 20260506 09:52:37.709 | INFO | "gpu" : " 17 GB", | 20260506 09:52:37.709 | INFO | "huge" : "", | 20260506 09:52:37.709 | INFO | "huge_shared" : " 27 GB", | 20260506 09:52:37.709 | INFO | "malloc" : " 14 GB", | 20260506 09:52:37.710 | INFO | "pinned" : " 29 GB" | 20260506 09:52:37.710 | INFO | } | 20260506 09:52:37.710 | INFO | Perf report for the last 5.13953 seconds: | 20260506 09:52:37.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:52:37.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:37.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:37.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:37.710 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 09:52:37.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:37.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:37.710 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:52:37.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:37.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:37.710 | INFO | Uploaded 36GiB to GPU in 1.60402s (22.4436GiB/s) | 20260506 09:52:37.710 | INFO | Downloaded 34GiB to GPU in 1.52236s (22.3338GiB/s) | 20260506 09:52:42.849 | INFO | Analyzing chunk frames = [424960,425472) | 20260506 09:52:42.849 | INFO | MemInfo: { | 20260506 09:52:42.849 | INFO | "gpu" : " 17 GB", | 20260506 09:52:42.849 | INFO | "huge" : "", | 20260506 09:52:42.849 | INFO | "huge_shared" : " 27 GB", | 20260506 09:52:42.849 | INFO | "malloc" : " 14 GB", | 20260506 09:52:42.849 | INFO | "pinned" : " 29 GB" | 20260506 09:52:42.849 | INFO | } | 20260506 09:52:42.849 | INFO | Perf report for the last 5.13915 seconds: | 20260506 09:52:42.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:52:42.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:42.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:42.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:42.850 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:52:42.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:42.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:42.850 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:52:42.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:42.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:42.850 | INFO | Uploaded 36GiB to GPU in 1.60463s (22.4351GiB/s) | 20260506 09:52:42.850 | INFO | Downloaded 34GiB to GPU in 1.52225s (22.3353GiB/s) | 20260506 09:52:47.988 | INFO | Analyzing chunk frames = [425472,425984) | 20260506 09:52:47.988 | INFO | MemInfo: { | 20260506 09:52:47.988 | INFO | "gpu" : " 17 GB", | 20260506 09:52:47.988 | INFO | "huge" : "", | 20260506 09:52:47.988 | INFO | "huge_shared" : " 27 GB", | 20260506 09:52:47.988 | INFO | "malloc" : " 14 GB", | 20260506 09:52:47.988 | INFO | "pinned" : " 29 GB" | 20260506 09:52:47.988 | INFO | } | 20260506 09:52:47.988 | INFO | Perf report for the last 5.13952 seconds: | 20260506 09:52:47.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:52:47.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:47.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:47.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:47.988 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:52:47.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:47.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:47.988 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 09:52:47.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:47.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:47.989 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:52:47.989 | INFO | Total mem usage: 15.9 GiB | 20260506 09:52:47.989 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 2.53%) | 20260506 09:52:47.989 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -7.86%) | 20260506 09:52:47.989 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:52:47.989 | INFO | High Water Breakdown: | 20260506 09:52:47.989 | INFO | 59.2% : 8.74 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:52:47.989 | INFO | 40.7% : 6.01 GiB ( 89.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:52:47.989 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:52:47.989 | INFO | ------------------------------------------ | 20260506 09:52:47.989 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:52:47.989 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:52:47.989 | INFO | Total mem usage: 26.5 GiB | 20260506 09:52:47.989 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:52:47.989 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:52:47.989 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:52:47.989 | INFO | High Water Breakdown: | 20260506 09:52:47.989 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:52:47.989 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:52:47.989 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:52:47.989 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:52:47.989 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:52:47.989 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:52:47.989 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:52:47.989 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:52:47.989 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:52:47.989 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:52:47.989 | INFO | Uploaded 36GiB to GPU in 1.60585s (22.418GiB/s) | 20260506 09:52:47.989 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:52:47.989 | INFO | Downloaded 34GiB to GPU in 1.5232s (22.3214GiB/s) | 20260506 09:52:47.989 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:52:47.989 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:52:47.989 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:52:47.989 | INFO | ------------------------------------------ | 20260506 09:52:47.989 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:52:47.989 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:52:47.989 | INFO | Total mem usage: 25.6 GiB | 20260506 09:52:47.989 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:52:47.989 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:52:47.989 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:52:47.989 | INFO | High Water Breakdown: | 20260506 09:52:47.989 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:52:47.989 | INFO | ------------------------------------------ | 20260506 09:52:47.989 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:52:47.989 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:52:47.989 | INFO | Total mem usage: 15.7 GiB | 20260506 09:52:47.989 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:52:47.990 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:52:47.990 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 09:52:47.990 | INFO | High Water Breakdown: | 20260506 09:52:47.990 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:52:47.990 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:52:47.990 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:52:47.990 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:52:47.990 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:52:47.990 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:52:47.990 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:52:47.990 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:52:47.990 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:52:47.990 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:52:47.990 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:52:47.990 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:52:47.990 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:52:47.990 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:52:47.990 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:52:47.990 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:52:47.990 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:52:47.990 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:52:47.990 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:52:47.990 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:52:47.990 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:52:47.990 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:52:47.990 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:52:47.990 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:52:47.990 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:52:47.990 | INFO | ------------------------------------------ | 20260506 09:52:47.990 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:52:47.990 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:52:47.990 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:52:47.990 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:52:47.990 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 09:52:47.990 | INFO | PerfReport: 0.29%, AggregateStats -- 8.64e-02 / -nan, 4.87e-03 / 0.00e+00 | 20260506 09:52:47.990 | INFO | PerfReport: 10.91%, Baseline -- 3.29e+00 / -nan, 1.73e-02 / 0.00e+00 | 20260506 09:52:47.990 | INFO | PerfReport: 3.05%, Histogram -- 9.22e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260506 09:52:47.990 | INFO | PerfReport: 16.05%, DME -- 3.87e+01 / -nan, 3.04e+00 / 0.00e+00 | 20260506 09:52:47.990 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.72e+01 / -nan, 4.24e-02 / 0.00e+00 | 20260506 09:52:47.990 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.25e+00 / -nan, 1.32e-01 / 0.00e+00 | 20260506 09:52:47.990 | INFO | PerfReport: 3.77%, Metrics -- 1.14e+00 / -nan, 1.78e-01 / 0.00e+00 | 20260506 09:52:47.990 | INFO | PerfReport: 1.46%, PulseSerializing -- 4.41e-01 / -nan, 6.35e-02 / 0.00e+00 | 20260506 09:52:47.990 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:52:47.990 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:52:47.990 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:52:47.990 | INFO | PerfReport: Full Execution -- 9.03e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 09:52:47.990 | INFO | PerfReport: 18.59%, Upload -- 1.68e+01 / 0.00e+00, 4.37e-01 / 0.00e+00 | 20260506 09:52:47.990 | INFO | PerfReport: 17.84%, Download -- 1.61e+01 / 0.00e+00, 7.71e-01 / 0.00e+00 | 20260506 09:52:47.990 | INFO | PerfReport: 33.42%, Compute -- 3.02e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 09:52:47.990 | INFO | PerfReport: 0.56%, LockUpload -- 5.08e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:52:47.990 | INFO | PerfReport: 0.00%, LockDownload -- 4.86e-04 / 0.00e+00, 1.22e-04 / 0.00e+00 | 20260506 09:52:47.990 | INFO | PerfReport: 29.57%, LockCompute -- 2.67e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 09:52:47.990 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.26e-02 / 0.00e+00 | 20260506 09:52:48.086 | INFO | Starting a round of Baz IO | 20260506 09:52:51.007 | INFO | Populating RTMetrics report | 20260506 09:52:51.007 | INFO | RTMetrics report populated | 20260506 09:52:51.007 | INFO | Opening tmp RTMetrics json file | 20260506 09:52:51.086 | INFO | Serializing RTMetrics to json | 20260506 09:52:51.107 | INFO | Serialization done, swapping tmp file now... | 20260506 09:52:51.108 | INFO | RTMetrics done | 20260506 09:52:51.303 | INFO | Finished a round of Baz IO: Wrote 1.916 GiB in 3.216 seconds | 20260506 09:52:51.303 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 51.02% / 0.07% / 48.91% | 20260506 09:52:51.303 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.21824% / 0.000230471% / 1.98682e-05% | 20260506 09:52:51.303 | INFO | Baz IO overall: | 20260506 09:52:51.303 | INFO | BytesWritten : 178685121050 | 20260506 09:52:51.303 | INFO | NumEvents : 85347895157 | 20260506 09:52:53.127 | INFO | Analyzing chunk frames = [425984,426496) | 20260506 09:52:53.127 | INFO | MemInfo: { | 20260506 09:52:53.127 | INFO | "gpu" : " 17 GB", | 20260506 09:52:53.127 | INFO | "huge" : "", | 20260506 09:52:53.127 | INFO | "huge_shared" : " 27 GB", | 20260506 09:52:53.127 | INFO | "malloc" : " 14 GB", | 20260506 09:52:53.127 | INFO | "pinned" : " 29 GB" | 20260506 09:52:53.127 | INFO | } | 20260506 09:52:53.128 | INFO | Perf report for the last 5.13914 seconds: | 20260506 09:52:53.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:52:53.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:53.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:53.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:53.128 | INFO | ANALYSIS : 57.7%, 2.8, 85.6ms, 0 | 20260506 09:52:53.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:53.128 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 09:52:53.128 | INFO | PRE_HQ : 56.9%, 2.1, 61.0ms, 0 | 20260506 09:52:53.128 | INFO | BAZWRITER : 62.2%, 4.8, 153.4ms, 0 | 20260506 09:52:53.128 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 09:52:53.128 | INFO | Uploaded 36GiB to GPU in 1.67498s (21.4928GiB/s) | 20260506 09:52:53.128 | INFO | Downloaded 38GiB to GPU in 1.74001s (21.839GiB/s) | 20260506 09:52:58.267 | INFO | Analyzing chunk frames = [426496,427008) | 20260506 09:52:58.267 | INFO | MemInfo: { | 20260506 09:52:58.267 | INFO | "gpu" : " 17 GB", | 20260506 09:52:58.267 | INFO | "huge" : "", | 20260506 09:52:58.267 | INFO | "huge_shared" : " 27 GB", | 20260506 09:52:58.267 | INFO | "malloc" : " 14 GB", | 20260506 09:52:58.267 | INFO | "pinned" : " 29 GB" | 20260506 09:52:58.267 | INFO | } | 20260506 09:52:58.267 | INFO | Perf report for the last 5.13955 seconds: | 20260506 09:52:58.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:52:58.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:58.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:58.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:58.267 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260506 09:52:58.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:58.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:58.267 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 09:52:58.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:58.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:52:58.267 | INFO | Uploaded 36GiB to GPU in 1.60321s (22.4549GiB/s) | 20260506 09:52:58.267 | INFO | Downloaded 34GiB to GPU in 1.52078s (22.3569GiB/s) | 20260506 09:53:03.407 | INFO | Analyzing chunk frames = [427008,427520) | 20260506 09:53:03.407 | INFO | MemInfo: { | 20260506 09:53:03.407 | INFO | "gpu" : " 17 GB", | 20260506 09:53:03.407 | INFO | "huge" : "", | 20260506 09:53:03.407 | INFO | "huge_shared" : " 27 GB", | 20260506 09:53:03.407 | INFO | "malloc" : " 14 GB", | 20260506 09:53:03.407 | INFO | "pinned" : " 29 GB" | 20260506 09:53:03.407 | INFO | } | 20260506 09:53:03.407 | INFO | Perf report for the last 5.13954 seconds: | 20260506 09:53:03.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:53:03.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:03.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:03.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:03.407 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:53:03.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:03.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:03.407 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:53:03.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:03.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:03.407 | INFO | Uploaded 36GiB to GPU in 1.6069s (22.4034GiB/s) | 20260506 09:53:03.408 | INFO | Downloaded 34GiB to GPU in 1.52419s (22.3069GiB/s) | 20260506 09:53:08.546 | INFO | Analyzing chunk frames = [427520,428032) | 20260506 09:53:08.546 | INFO | MemInfo: { | 20260506 09:53:08.546 | INFO | "gpu" : " 17 GB", | 20260506 09:53:08.546 | INFO | "huge" : "", | 20260506 09:53:08.546 | INFO | "huge_shared" : " 27 GB", | 20260506 09:53:08.546 | INFO | "malloc" : " 14 GB", | 20260506 09:53:08.546 | INFO | "pinned" : " 29 GB" | 20260506 09:53:08.546 | INFO | } | 20260506 09:53:08.546 | INFO | Perf report for the last 5.13918 seconds: | 20260506 09:53:08.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:53:08.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:08.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:08.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:08.546 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:53:08.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:08.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:08.546 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:53:08.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:08.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:08.546 | INFO | Uploaded 36GiB to GPU in 1.60548s (22.4232GiB/s) | 20260506 09:53:08.546 | INFO | Downloaded 34GiB to GPU in 1.5231s (22.323GiB/s) | 20260506 09:53:13.685 | INFO | Analyzing chunk frames = [428032,428544) | 20260506 09:53:13.685 | INFO | MemInfo: { | 20260506 09:53:13.685 | INFO | "gpu" : " 17 GB", | 20260506 09:53:13.685 | INFO | "huge" : "", | 20260506 09:53:13.685 | INFO | "huge_shared" : " 27 GB", | 20260506 09:53:13.685 | INFO | "malloc" : " 14 GB", | 20260506 09:53:13.685 | INFO | "pinned" : " 29 GB" | 20260506 09:53:13.685 | INFO | } | 20260506 09:53:13.685 | INFO | Perf report for the last 5.13919 seconds: | 20260506 09:53:13.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:53:13.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:13.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:13.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:13.685 | INFO | ANALYSIS : 56.5%, 3.0, 88.0ms, 0 | 20260506 09:53:13.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:13.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:13.685 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:53:13.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:13.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:13.686 | INFO | Uploaded 36GiB to GPU in 1.60414s (22.4419GiB/s) | 20260506 09:53:13.686 | INFO | Downloaded 34GiB to GPU in 1.52167s (22.3439GiB/s) | 20260506 09:53:18.825 | INFO | Analyzing chunk frames = [428544,429056) | 20260506 09:53:18.825 | INFO | MemInfo: { | 20260506 09:53:18.825 | INFO | "gpu" : " 17 GB", | 20260506 09:53:18.825 | INFO | "huge" : "", | 20260506 09:53:18.825 | INFO | "huge_shared" : " 27 GB", | 20260506 09:53:18.825 | INFO | "malloc" : " 14 GB", | 20260506 09:53:18.825 | INFO | "pinned" : " 29 GB" | 20260506 09:53:18.825 | INFO | } | 20260506 09:53:18.825 | INFO | Perf report for the last 5.14044 seconds: | 20260506 09:53:18.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:53:18.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:18.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:18.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:18.826 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:53:18.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:18.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:18.826 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:53:18.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:18.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:18.826 | INFO | Uploaded 36GiB to GPU in 1.60542s (22.4241GiB/s) | 20260506 09:53:18.826 | INFO | Downloaded 34GiB to GPU in 1.52201s (22.3388GiB/s) | 20260506 09:53:23.964 | INFO | Analyzing chunk frames = [429056,429568) | 20260506 09:53:23.964 | INFO | MemInfo: { | 20260506 09:53:23.964 | INFO | "gpu" : " 17 GB", | 20260506 09:53:23.964 | INFO | "huge" : "", | 20260506 09:53:23.964 | INFO | "huge_shared" : " 27 GB", | 20260506 09:53:23.964 | INFO | "malloc" : " 14 GB", | 20260506 09:53:23.964 | INFO | "pinned" : " 29 GB" | 20260506 09:53:23.964 | INFO | } | 20260506 09:53:23.964 | INFO | Perf report for the last 5.13831 seconds: | 20260506 09:53:23.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:53:23.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:23.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:23.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:23.964 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:53:23.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:23.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:23.964 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:53:23.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:23.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:23.964 | INFO | Uploaded 36GiB to GPU in 1.60528s (22.426GiB/s) | 20260506 09:53:23.964 | INFO | Downloaded 34GiB to GPU in 1.5235s (22.317GiB/s) | 20260506 09:53:29.103 | INFO | Analyzing chunk frames = [429568,430080) | 20260506 09:53:29.103 | INFO | MemInfo: { | 20260506 09:53:29.103 | INFO | "gpu" : " 17 GB", | 20260506 09:53:29.103 | INFO | "huge" : "", | 20260506 09:53:29.103 | INFO | "huge_shared" : " 27 GB", | 20260506 09:53:29.103 | INFO | "malloc" : " 14 GB", | 20260506 09:53:29.103 | INFO | "pinned" : " 29 GB" | 20260506 09:53:29.103 | INFO | } | 20260506 09:53:29.103 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:53:29.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:53:29.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:29.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:29.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:29.103 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:53:29.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:29.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:29.103 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 09:53:29.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:29.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:29.104 | INFO | Uploaded 36GiB to GPU in 1.60514s (22.428GiB/s) | 20260506 09:53:29.104 | INFO | Downloaded 34GiB to GPU in 1.52249s (22.3319GiB/s) | 20260506 09:53:29.202 | INFO | Starting a round of Baz IO | 20260506 09:53:32.118 | INFO | Populating RTMetrics report | 20260506 09:53:32.119 | INFO | RTMetrics report populated | 20260506 09:53:32.119 | INFO | Opening tmp RTMetrics json file | 20260506 09:53:32.137 | INFO | Serializing RTMetrics to json | 20260506 09:53:32.181 | INFO | Serialization done, swapping tmp file now... | 20260506 09:53:32.197 | INFO | RTMetrics done | 20260506 09:53:32.377 | INFO | Finished a round of Baz IO: Wrote 1.921 GiB in 3.175 seconds | 20260506 09:53:32.377 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.94% / 0.07% / 48.98% | 20260506 09:53:32.377 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.21583% / 0.000202656% / 3.57628e-05% | 20260506 09:53:32.377 | INFO | Baz IO overall: | 20260506 09:53:32.377 | INFO | BytesWritten : 180747751722 | 20260506 09:53:32.377 | INFO | NumEvents : 86349770533 | 20260506 09:53:34.242 | INFO | Analyzing chunk frames = [430080,430592) | 20260506 09:53:34.242 | INFO | MemInfo: { | 20260506 09:53:34.242 | INFO | "gpu" : " 17 GB", | 20260506 09:53:34.242 | INFO | "huge" : "", | 20260506 09:53:34.242 | INFO | "huge_shared" : " 27 GB", | 20260506 09:53:34.242 | INFO | "malloc" : " 14 GB", | 20260506 09:53:34.242 | INFO | "pinned" : " 29 GB" | 20260506 09:53:34.242 | INFO | } | 20260506 09:53:34.243 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:53:34.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:53:34.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:34.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:34.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:34.243 | INFO | ANALYSIS : 57.7%, 2.8, 85.5ms, 0 | 20260506 09:53:34.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:34.243 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260506 09:53:34.243 | INFO | PRE_HQ : 56.8%, 2.1, 60.9ms, 0 | 20260506 09:53:34.243 | INFO | BAZWRITER : 61.4%, 4.8, 151.0ms, 0 | 20260506 09:53:34.243 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 09:53:34.243 | INFO | Uploaded 36GiB to GPU in 1.67476s (21.4956GiB/s) | 20260506 09:53:34.243 | INFO | Downloaded 38GiB to GPU in 1.73975s (21.8422GiB/s) | 20260506 09:53:39.382 | INFO | Analyzing chunk frames = [430592,431104) | 20260506 09:53:39.382 | INFO | MemInfo: { | 20260506 09:53:39.382 | INFO | "gpu" : " 17 GB", | 20260506 09:53:39.382 | INFO | "huge" : "", | 20260506 09:53:39.382 | INFO | "huge_shared" : " 27 GB", | 20260506 09:53:39.382 | INFO | "malloc" : " 14 GB", | 20260506 09:53:39.382 | INFO | "pinned" : " 29 GB" | 20260506 09:53:39.382 | INFO | } | 20260506 09:53:39.382 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:53:39.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:53:39.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:39.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:39.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:39.382 | INFO | ANALYSIS : 57.5%, 3.0, 89.8ms, 0 | 20260506 09:53:39.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:39.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:39.382 | INFO | PRE_HQ : 29.7%, 1.0, 15.6ms, 0 | 20260506 09:53:39.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:39.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:39.382 | INFO | Uploaded 36GiB to GPU in 1.60224s (22.4685GiB/s) | 20260506 09:53:39.382 | INFO | Downloaded 34GiB to GPU in 1.51952s (22.3755GiB/s) | 20260506 09:53:44.521 | INFO | Analyzing chunk frames = [431104,431616) | 20260506 09:53:44.521 | INFO | MemInfo: { | 20260506 09:53:44.521 | INFO | "gpu" : " 17 GB", | 20260506 09:53:44.521 | INFO | "huge" : "", | 20260506 09:53:44.521 | INFO | "huge_shared" : " 27 GB", | 20260506 09:53:44.521 | INFO | "malloc" : " 14 GB", | 20260506 09:53:44.521 | INFO | "pinned" : " 29 GB" | 20260506 09:53:44.521 | INFO | } | 20260506 09:53:44.521 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:53:44.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:53:44.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:44.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:44.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:44.521 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:53:44.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:44.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:44.521 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 09:53:44.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:44.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:44.522 | INFO | Uploaded 36GiB to GPU in 1.60643s (22.4099GiB/s) | 20260506 09:53:44.522 | INFO | Downloaded 34GiB to GPU in 1.52466s (22.3GiB/s) | 20260506 09:53:49.660 | INFO | Analyzing chunk frames = [431616,432128) | 20260506 09:53:49.660 | INFO | MemInfo: { | 20260506 09:53:49.660 | INFO | "gpu" : " 17 GB", | 20260506 09:53:49.660 | INFO | "huge" : "", | 20260506 09:53:49.660 | INFO | "huge_shared" : " 27 GB", | 20260506 09:53:49.660 | INFO | "malloc" : " 14 GB", | 20260506 09:53:49.660 | INFO | "pinned" : " 29 GB" | 20260506 09:53:49.660 | INFO | } | 20260506 09:53:49.661 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:53:49.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:53:49.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:49.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:49.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:49.661 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:53:49.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:49.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:49.661 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:53:49.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:49.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:49.661 | INFO | Uploaded 36GiB to GPU in 1.60557s (22.422GiB/s) | 20260506 09:53:49.661 | INFO | Downloaded 34GiB to GPU in 1.52304s (22.3237GiB/s) | 20260506 09:53:54.800 | INFO | Analyzing chunk frames = [432128,432640) | 20260506 09:53:54.800 | INFO | MemInfo: { | 20260506 09:53:54.800 | INFO | "gpu" : " 17 GB", | 20260506 09:53:54.800 | INFO | "huge" : "", | 20260506 09:53:54.800 | INFO | "huge_shared" : " 27 GB", | 20260506 09:53:54.800 | INFO | "malloc" : " 14 GB", | 20260506 09:53:54.800 | INFO | "pinned" : " 29 GB" | 20260506 09:53:54.800 | INFO | } | 20260506 09:53:54.800 | INFO | Perf report for the last 5.13915 seconds: | 20260506 09:53:54.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:53:54.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:54.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:54.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:54.800 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 09:53:54.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:54.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:54.800 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:53:54.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:54.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:54.800 | INFO | Uploaded 36GiB to GPU in 1.60455s (22.4362GiB/s) | 20260506 09:53:54.800 | INFO | Downloaded 34GiB to GPU in 1.52147s (22.3468GiB/s) | 20260506 09:53:59.940 | INFO | Analyzing chunk frames = [432640,433152) | 20260506 09:53:59.940 | INFO | MemInfo: { | 20260506 09:53:59.940 | INFO | "gpu" : " 17 GB", | 20260506 09:53:59.940 | INFO | "huge" : "", | 20260506 09:53:59.941 | INFO | "huge_shared" : " 27 GB", | 20260506 09:53:59.941 | INFO | "malloc" : " 14 GB", | 20260506 09:53:59.941 | INFO | "pinned" : " 29 GB" | 20260506 09:53:59.941 | INFO | } | 20260506 09:53:59.941 | INFO | Perf report for the last 5.14089 seconds: | 20260506 09:53:59.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:53:59.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:59.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:59.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:59.941 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:53:59.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:59.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:59.941 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 09:53:59.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:59.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:53:59.941 | INFO | Uploaded 36GiB to GPU in 1.60447s (22.4373GiB/s) | 20260506 09:53:59.941 | INFO | Downloaded 34GiB to GPU in 1.52208s (22.3379GiB/s) | 20260506 09:54:05.078 | INFO | Analyzing chunk frames = [433152,433664) | 20260506 09:54:05.078 | INFO | MemInfo: { | 20260506 09:54:05.078 | INFO | "gpu" : " 17 GB", | 20260506 09:54:05.078 | INFO | "huge" : "", | 20260506 09:54:05.078 | INFO | "huge_shared" : " 27 GB", | 20260506 09:54:05.078 | INFO | "malloc" : " 14 GB", | 20260506 09:54:05.078 | INFO | "pinned" : " 29 GB" | 20260506 09:54:05.078 | INFO | } | 20260506 09:54:05.078 | INFO | Perf report for the last 5.13762 seconds: | 20260506 09:54:05.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:54:05.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:05.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:05.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:05.078 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:54:05.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:05.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:05.078 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:54:05.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:05.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:05.078 | INFO | Uploaded 36GiB to GPU in 1.60492s (22.431GiB/s) | 20260506 09:54:05.078 | INFO | Downloaded 34GiB to GPU in 1.52289s (22.326GiB/s) | 20260506 09:54:10.217 | INFO | Analyzing chunk frames = [433664,434176) | 20260506 09:54:10.217 | INFO | MemInfo: { | 20260506 09:54:10.217 | INFO | "gpu" : " 17 GB", | 20260506 09:54:10.217 | INFO | "huge" : "", | 20260506 09:54:10.218 | INFO | "huge_shared" : " 27 GB", | 20260506 09:54:10.218 | INFO | "malloc" : " 14 GB", | 20260506 09:54:10.218 | INFO | "pinned" : " 29 GB" | 20260506 09:54:10.218 | INFO | } | 20260506 09:54:10.218 | INFO | Perf report for the last 5.13945 seconds: | 20260506 09:54:10.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:54:10.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:10.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:10.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:10.218 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:54:10.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:10.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:10.218 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 09:54:10.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:10.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:10.218 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:54:10.218 | INFO | Total mem usage: 15.9 GiB | 20260506 09:54:10.218 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 2.53%) | 20260506 09:54:10.218 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -3.68%) | 20260506 09:54:10.218 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:54:10.218 | INFO | High Water Breakdown: | 20260506 09:54:10.218 | INFO | 59.8% : 9.19 GiB ( 95.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:54:10.218 | INFO | 40% : 6.15 GiB ( 91.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:54:10.218 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:54:10.218 | INFO | Uploaded 36GiB to GPU in 1.60384s (22.4462GiB/s) | 20260506 09:54:10.218 | INFO | ------------------------------------------ | 20260506 09:54:10.218 | INFO | Downloaded 34GiB to GPU in 1.52275s (22.3281GiB/s) | 20260506 09:54:10.218 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:54:10.218 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:54:10.218 | INFO | Total mem usage: 26.5 GiB | 20260506 09:54:10.218 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:54:10.219 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:54:10.219 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:54:10.219 | INFO | High Water Breakdown: | 20260506 09:54:10.219 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:54:10.219 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:54:10.219 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:54:10.219 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:54:10.219 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:54:10.219 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:54:10.219 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:54:10.219 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:54:10.219 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:54:10.219 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:54:10.219 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:54:10.219 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:54:10.219 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:54:10.219 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:54:10.219 | INFO | ------------------------------------------ | 20260506 09:54:10.219 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:54:10.219 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:54:10.219 | INFO | Total mem usage: 25.6 GiB | 20260506 09:54:10.219 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:54:10.219 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:54:10.219 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:54:10.219 | INFO | High Water Breakdown: | 20260506 09:54:10.219 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:54:10.219 | INFO | ------------------------------------------ | 20260506 09:54:10.219 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:54:10.219 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:54:10.219 | INFO | Total mem usage: 15.7 GiB | 20260506 09:54:10.219 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:54:10.219 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:54:10.219 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:54:10.219 | INFO | High Water Breakdown: | 20260506 09:54:10.219 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:54:10.219 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:54:10.219 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:54:10.219 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:54:10.219 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:54:10.219 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:54:10.219 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:54:10.219 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:54:10.219 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:54:10.219 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:54:10.219 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:54:10.219 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:54:10.219 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:54:10.219 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:54:10.219 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:54:10.219 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:54:10.219 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:54:10.219 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:54:10.219 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:54:10.219 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:54:10.219 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:54:10.219 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:54:10.219 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:54:10.219 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:54:10.219 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:54:10.219 | INFO | ------------------------------------------ | 20260506 09:54:10.219 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:54:10.219 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:54:10.219 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:54:10.219 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:54:10.219 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 09:54:10.219 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / -nan, 4.68e-03 / 0.00e+00 | 20260506 09:54:10.219 | INFO | PerfReport: 10.92%, Baseline -- 3.29e+00 / -nan, 1.85e-02 / 0.00e+00 | 20260506 09:54:10.219 | INFO | PerfReport: 3.05%, Histogram -- 9.21e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260506 09:54:10.219 | INFO | PerfReport: 16.06%, DME -- 3.88e+01 / -nan, 3.06e+00 / 0.00e+00 | 20260506 09:54:10.219 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.72e+01 / -nan, 4.21e-02 / 0.00e+00 | 20260506 09:54:10.219 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.25e+00 / -nan, 1.32e-01 / 0.00e+00 | 20260506 09:54:10.219 | INFO | PerfReport: 3.77%, Metrics -- 1.14e+00 / -nan, 1.77e-01 / 0.00e+00 | 20260506 09:54:10.219 | INFO | PerfReport: 1.46%, PulseSerializing -- 4.41e-01 / -nan, 6.34e-02 / 0.00e+00 | 20260506 09:54:10.219 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:54:10.219 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:54:10.219 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:54:10.219 | INFO | PerfReport: Full Execution -- 9.01e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 09:54:10.219 | INFO | PerfReport: 18.66%, Upload -- 1.68e+01 / 0.00e+00, 4.93e-01 / 0.00e+00 | 20260506 09:54:10.219 | INFO | PerfReport: 17.92%, Download -- 1.61e+01 / 0.00e+00, 8.47e-01 / 0.00e+00 | 20260506 09:54:10.219 | INFO | PerfReport: 33.50%, Compute -- 3.02e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 09:54:10.219 | INFO | PerfReport: 0.56%, LockUpload -- 5.08e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260506 09:54:10.219 | INFO | PerfReport: 0.00%, LockDownload -- 4.93e-04 / 0.00e+00, 1.52e-04 / 0.00e+00 | 20260506 09:54:10.219 | INFO | PerfReport: 29.34%, LockCompute -- 2.64e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 09:54:10.219 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 3.61e-02 / 0.00e+00 | 20260506 09:54:10.315 | INFO | Starting a round of Baz IO | 20260506 09:54:13.232 | INFO | Populating RTMetrics report | 20260506 09:54:13.233 | INFO | RTMetrics report populated | 20260506 09:54:13.233 | INFO | Opening tmp RTMetrics json file | 20260506 09:54:13.327 | INFO | Serializing RTMetrics to json | 20260506 09:54:13.364 | INFO | Serialization done, swapping tmp file now... | 20260506 09:54:13.365 | INFO | RTMetrics done | 20260506 09:54:13.387 | INFO | Finished a round of Baz IO: Wrote 1.922 GiB in 3.072 seconds | 20260506 09:54:13.387 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.87% / 0.07% / 49.06% | 20260506 09:54:13.387 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.17592% / 0.000182788% / 2.38419e-05% | 20260506 09:54:13.387 | INFO | Baz IO overall: | 20260506 09:54:13.387 | INFO | BytesWritten : 182811511579 | 20260506 09:54:13.387 | INFO | NumEvents : 87351758493 | 20260506 09:54:15.357 | INFO | Analyzing chunk frames = [434176,434688) | 20260506 09:54:15.357 | INFO | MemInfo: { | 20260506 09:54:15.357 | INFO | "gpu" : " 17 GB", | 20260506 09:54:15.357 | INFO | "huge" : "", | 20260506 09:54:15.357 | INFO | "huge_shared" : " 27 GB", | 20260506 09:54:15.357 | INFO | "malloc" : " 14 GB", | 20260506 09:54:15.357 | INFO | "pinned" : " 29 GB" | 20260506 09:54:15.357 | INFO | } | 20260506 09:54:15.357 | INFO | Perf report for the last 5.13923 seconds: | 20260506 09:54:15.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:54:15.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:15.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:15.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:15.357 | INFO | ANALYSIS : 57.7%, 2.9, 88.1ms, 0 | 20260506 09:54:15.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:15.357 | INFO | RT_METRICS : 46.6%, 1.0, 24.5ms, 0 | 20260506 09:54:15.357 | INFO | PRE_HQ : 56.7%, 2.1, 61.6ms, 0 | 20260506 09:54:15.357 | INFO | BAZWRITER : 59.4%, 4.5, 136.0ms, 0 | 20260506 09:54:15.357 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 09:54:15.357 | INFO | Uploaded 36GiB to GPU in 1.64809s (21.8434GiB/s) | 20260506 09:54:15.357 | INFO | Downloaded 38GiB to GPU in 1.71231s (22.1923GiB/s) | 20260506 09:54:20.496 | INFO | Analyzing chunk frames = [434688,435200) | 20260506 09:54:20.496 | INFO | MemInfo: { | 20260506 09:54:20.496 | INFO | "gpu" : " 17 GB", | 20260506 09:54:20.496 | INFO | "huge" : "", | 20260506 09:54:20.496 | INFO | "huge_shared" : " 27 GB", | 20260506 09:54:20.496 | INFO | "malloc" : " 14 GB", | 20260506 09:54:20.496 | INFO | "pinned" : " 29 GB" | 20260506 09:54:20.496 | INFO | } | 20260506 09:54:20.496 | INFO | Perf report for the last 5.13946 seconds: | 20260506 09:54:20.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:54:20.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:20.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:20.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:20.496 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 09:54:20.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:20.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:20.497 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 09:54:20.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:20.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:20.497 | INFO | Uploaded 36GiB to GPU in 1.60222s (22.4688GiB/s) | 20260506 09:54:20.497 | INFO | Downloaded 34GiB to GPU in 1.51977s (22.3718GiB/s) | 20260506 09:54:25.636 | INFO | Analyzing chunk frames = [435200,435712) | 20260506 09:54:25.636 | INFO | MemInfo: { | 20260506 09:54:25.636 | INFO | "gpu" : " 17 GB", | 20260506 09:54:25.636 | INFO | "huge" : "", | 20260506 09:54:25.636 | INFO | "huge_shared" : " 27 GB", | 20260506 09:54:25.636 | INFO | "malloc" : " 14 GB", | 20260506 09:54:25.636 | INFO | "pinned" : " 29 GB" | 20260506 09:54:25.636 | INFO | } | 20260506 09:54:25.636 | INFO | Perf report for the last 5.13938 seconds: | 20260506 09:54:25.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:54:25.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:25.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:25.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:25.636 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260506 09:54:25.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:25.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:25.636 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 09:54:25.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:25.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:25.636 | INFO | Uploaded 36GiB to GPU in 1.6064s (22.4103GiB/s) | 20260506 09:54:25.636 | INFO | Downloaded 34GiB to GPU in 1.52406s (22.3088GiB/s) | 20260506 09:54:30.775 | INFO | Analyzing chunk frames = [435712,436224) | 20260506 09:54:30.775 | INFO | MemInfo: { | 20260506 09:54:30.775 | INFO | "gpu" : " 17 GB", | 20260506 09:54:30.775 | INFO | "huge" : "", | 20260506 09:54:30.775 | INFO | "huge_shared" : " 27 GB", | 20260506 09:54:30.775 | INFO | "malloc" : " 14 GB", | 20260506 09:54:30.775 | INFO | "pinned" : " 29 GB" | 20260506 09:54:30.775 | INFO | } | 20260506 09:54:30.775 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:54:30.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:54:30.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:30.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:30.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:30.775 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:54:30.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:30.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:30.775 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:54:30.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:30.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:30.776 | INFO | Uploaded 36GiB to GPU in 1.60596s (22.4165GiB/s) | 20260506 09:54:30.776 | INFO | Downloaded 34GiB to GPU in 1.52378s (22.3129GiB/s) | 20260506 09:54:35.914 | INFO | Analyzing chunk frames = [436224,436736) | 20260506 09:54:35.914 | INFO | MemInfo: { | 20260506 09:54:35.914 | INFO | "gpu" : " 17 GB", | 20260506 09:54:35.914 | INFO | "huge" : "", | 20260506 09:54:35.914 | INFO | "huge_shared" : " 27 GB", | 20260506 09:54:35.914 | INFO | "malloc" : " 14 GB", | 20260506 09:54:35.914 | INFO | "pinned" : " 29 GB" | 20260506 09:54:35.914 | INFO | } | 20260506 09:54:35.914 | INFO | Perf report for the last 5.13925 seconds: | 20260506 09:54:35.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:54:35.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:35.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:35.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:35.914 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 09:54:35.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:35.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:35.915 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:54:35.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:35.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:35.915 | INFO | Uploaded 36GiB to GPU in 1.60506s (22.4291GiB/s) | 20260506 09:54:35.915 | INFO | Downloaded 34GiB to GPU in 1.52194s (22.3399GiB/s) | 20260506 09:54:41.053 | INFO | Analyzing chunk frames = [436736,437248) | 20260506 09:54:41.054 | INFO | MemInfo: { | 20260506 09:54:41.054 | INFO | "gpu" : " 17 GB", | 20260506 09:54:41.054 | INFO | "huge" : "", | 20260506 09:54:41.054 | INFO | "huge_shared" : " 27 GB", | 20260506 09:54:41.054 | INFO | "malloc" : " 14 GB", | 20260506 09:54:41.054 | INFO | "pinned" : " 29 GB" | 20260506 09:54:41.054 | INFO | } | 20260506 09:54:41.054 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:54:41.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:54:41.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:41.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:41.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:41.054 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:54:41.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:41.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:41.054 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 09:54:41.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:41.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:41.054 | INFO | Uploaded 36GiB to GPU in 1.60435s (22.4389GiB/s) | 20260506 09:54:41.054 | INFO | Downloaded 34GiB to GPU in 1.52101s (22.3536GiB/s) | 20260506 09:54:46.193 | INFO | Analyzing chunk frames = [437248,437760) | 20260506 09:54:46.193 | INFO | MemInfo: { | 20260506 09:54:46.193 | INFO | "gpu" : " 17 GB", | 20260506 09:54:46.193 | INFO | "huge" : "", | 20260506 09:54:46.193 | INFO | "huge_shared" : " 27 GB", | 20260506 09:54:46.193 | INFO | "malloc" : " 14 GB", | 20260506 09:54:46.193 | INFO | "pinned" : " 29 GB" | 20260506 09:54:46.193 | INFO | } | 20260506 09:54:46.193 | INFO | Perf report for the last 5.13921 seconds: | 20260506 09:54:46.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:54:46.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:46.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:46.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:46.193 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:54:46.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:46.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:46.193 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:54:46.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:46.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:46.193 | INFO | Uploaded 36GiB to GPU in 1.60499s (22.4301GiB/s) | 20260506 09:54:46.193 | INFO | Downloaded 34GiB to GPU in 1.52317s (22.3218GiB/s) | 20260506 09:54:51.332 | INFO | Analyzing chunk frames = [437760,438272) | 20260506 09:54:51.332 | INFO | MemInfo: { | 20260506 09:54:51.332 | INFO | "gpu" : " 17 GB", | 20260506 09:54:51.332 | INFO | "huge" : "", | 20260506 09:54:51.332 | INFO | "huge_shared" : " 27 GB", | 20260506 09:54:51.332 | INFO | "malloc" : " 14 GB", | 20260506 09:54:51.332 | INFO | "pinned" : " 29 GB" | 20260506 09:54:51.332 | INFO | } | 20260506 09:54:51.332 | INFO | Perf report for the last 5.13953 seconds: | 20260506 09:54:51.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:54:51.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:51.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:51.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:51.333 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:54:51.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:51.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:51.333 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 09:54:51.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:51.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:51.333 | INFO | Uploaded 36GiB to GPU in 1.60489s (22.4314GiB/s) | 20260506 09:54:51.333 | INFO | Downloaded 34GiB to GPU in 1.52351s (22.3169GiB/s) | 20260506 09:54:51.430 | INFO | Starting a round of Baz IO | 20260506 09:54:54.347 | INFO | Populating RTMetrics report | 20260506 09:54:54.348 | INFO | RTMetrics report populated | 20260506 09:54:54.348 | INFO | Opening tmp RTMetrics json file | 20260506 09:54:54.349 | INFO | Serializing RTMetrics to json | 20260506 09:54:54.352 | INFO | Serialization done, swapping tmp file now... | 20260506 09:54:54.352 | INFO | RTMetrics done | 20260506 09:54:54.442 | INFO | Finished a round of Baz IO: Wrote 1.919 GiB in 3.012 seconds | 20260506 09:54:54.442 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.80% / 0.07% / 49.13% | 20260506 09:54:54.442 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.09671% / 0.000182788% / 1.98682e-05% | 20260506 09:54:54.442 | INFO | Baz IO overall: | 20260506 09:54:54.442 | INFO | BytesWritten : 184872113918 | 20260506 09:54:54.442 | INFO | NumEvents : 88351331372 | 20260506 09:54:56.472 | INFO | Analyzing chunk frames = [438272,438784) | 20260506 09:54:56.472 | INFO | MemInfo: { | 20260506 09:54:56.472 | INFO | "gpu" : " 17 GB", | 20260506 09:54:56.472 | INFO | "huge" : "", | 20260506 09:54:56.472 | INFO | "huge_shared" : " 27 GB", | 20260506 09:54:56.472 | INFO | "malloc" : " 14 GB", | 20260506 09:54:56.472 | INFO | "pinned" : " 29 GB" | 20260506 09:54:56.472 | INFO | } | 20260506 09:54:56.472 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:54:56.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:54:56.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:56.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:56.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:56.472 | INFO | ANALYSIS : 57.7%, 2.9, 86.3ms, 0 | 20260506 09:54:56.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:54:56.472 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 09:54:56.472 | INFO | PRE_HQ : 56.8%, 2.1, 61.0ms, 0 | 20260506 09:54:56.472 | INFO | BAZWRITER : 58.2%, 4.7, 140.5ms, 0 | 20260506 09:54:56.472 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 09:54:56.472 | INFO | Uploaded 36GiB to GPU in 1.66373s (21.6381GiB/s) | 20260506 09:54:56.472 | INFO | Downloaded 38GiB to GPU in 1.72891s (21.9792GiB/s) | 20260506 09:55:01.611 | INFO | Analyzing chunk frames = [438784,439296) | 20260506 09:55:01.611 | INFO | MemInfo: { | 20260506 09:55:01.611 | INFO | "gpu" : " 17 GB", | 20260506 09:55:01.611 | INFO | "huge" : "", | 20260506 09:55:01.611 | INFO | "huge_shared" : " 27 GB", | 20260506 09:55:01.611 | INFO | "malloc" : " 14 GB", | 20260506 09:55:01.611 | INFO | "pinned" : " 29 GB" | 20260506 09:55:01.611 | INFO | } | 20260506 09:55:01.611 | INFO | Perf report for the last 5.13953 seconds: | 20260506 09:55:01.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:55:01.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:01.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:01.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:01.611 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 09:55:01.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:01.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:01.611 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 09:55:01.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:01.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:01.612 | INFO | Uploaded 36GiB to GPU in 1.60308s (22.4568GiB/s) | 20260506 09:55:01.612 | INFO | Downloaded 34GiB to GPU in 1.51986s (22.3705GiB/s) | 20260506 09:55:06.750 | INFO | Analyzing chunk frames = [439296,439808) | 20260506 09:55:06.750 | INFO | MemInfo: { | 20260506 09:55:06.750 | INFO | "gpu" : " 17 GB", | 20260506 09:55:06.750 | INFO | "huge" : "", | 20260506 09:55:06.750 | INFO | "huge_shared" : " 27 GB", | 20260506 09:55:06.750 | INFO | "malloc" : " 14 GB", | 20260506 09:55:06.750 | INFO | "pinned" : " 29 GB" | 20260506 09:55:06.751 | INFO | } | 20260506 09:55:06.751 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:55:06.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:55:06.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:06.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:06.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:06.751 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260506 09:55:06.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:06.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:06.751 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 09:55:06.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:06.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:06.751 | INFO | Uploaded 36GiB to GPU in 1.60697s (22.4024GiB/s) | 20260506 09:55:06.751 | INFO | Downloaded 34GiB to GPU in 1.52454s (22.3018GiB/s) | 20260506 09:55:11.890 | INFO | Analyzing chunk frames = [439808,440320) | 20260506 09:55:11.890 | INFO | MemInfo: { | 20260506 09:55:11.890 | INFO | "gpu" : " 17 GB", | 20260506 09:55:11.890 | INFO | "huge" : "", | 20260506 09:55:11.890 | INFO | "huge_shared" : " 27 GB", | 20260506 09:55:11.890 | INFO | "malloc" : " 14 GB", | 20260506 09:55:11.890 | INFO | "pinned" : " 29 GB" | 20260506 09:55:11.890 | INFO | } | 20260506 09:55:11.890 | INFO | Perf report for the last 5.13906 seconds: | 20260506 09:55:11.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:55:11.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:11.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:11.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:11.890 | INFO | ANALYSIS : 56.6%, 3.0, 87.7ms, 0 | 20260506 09:55:11.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:11.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:11.890 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:55:11.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:11.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:11.890 | INFO | Uploaded 36GiB to GPU in 1.60634s (22.4111GiB/s) | 20260506 09:55:11.890 | INFO | Downloaded 34GiB to GPU in 1.52359s (22.3157GiB/s) | 20260506 09:55:17.041 | INFO | Analyzing chunk frames = [440320,440832) | 20260506 09:55:17.041 | INFO | MemInfo: { | 20260506 09:55:17.041 | INFO | "gpu" : " 17 GB", | 20260506 09:55:17.041 | INFO | "huge" : "", | 20260506 09:55:17.041 | INFO | "huge_shared" : " 27 GB", | 20260506 09:55:17.041 | INFO | "malloc" : " 14 GB", | 20260506 09:55:17.041 | INFO | "pinned" : " 29 GB" | 20260506 09:55:17.041 | INFO | } | 20260506 09:55:17.041 | INFO | Perf report for the last 5.15162 seconds: | 20260506 09:55:17.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:55:17.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:17.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:17.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:17.041 | INFO | ANALYSIS : 56.3%, 3.0, 88.1ms, 0 | 20260506 09:55:17.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:17.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:17.041 | INFO | PRE_HQ : 29.5%, 1.0, 15.4ms, 0 | 20260506 09:55:17.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:17.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:17.042 | INFO | Uploaded 36GiB to GPU in 1.60471s (22.434GiB/s) | 20260506 09:55:17.042 | INFO | Downloaded 34GiB to GPU in 1.52109s (22.3524GiB/s) | 20260506 09:55:22.168 | INFO | Analyzing chunk frames = [440832,441344) | 20260506 09:55:22.168 | INFO | MemInfo: { | 20260506 09:55:22.168 | INFO | "gpu" : " 17 GB", | 20260506 09:55:22.169 | INFO | "huge" : "", | 20260506 09:55:22.169 | INFO | "huge_shared" : " 27 GB", | 20260506 09:55:22.169 | INFO | "malloc" : " 14 GB", | 20260506 09:55:22.169 | INFO | "pinned" : " 29 GB" | 20260506 09:55:22.169 | INFO | } | 20260506 09:55:22.169 | INFO | Perf report for the last 5.12745 seconds: | 20260506 09:55:22.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:55:22.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:22.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:22.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:22.169 | INFO | ANALYSIS : 56.9%, 3.0, 88.5ms, 0 | 20260506 09:55:22.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:22.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:22.169 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260506 09:55:22.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:22.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:22.169 | INFO | Uploaded 36GiB to GPU in 1.60622s (22.4129GiB/s) | 20260506 09:55:22.169 | INFO | Downloaded 34GiB to GPU in 1.52428s (22.3056GiB/s) | 20260506 09:55:27.308 | INFO | Analyzing chunk frames = [441344,441856) | 20260506 09:55:27.308 | INFO | MemInfo: { | 20260506 09:55:27.308 | INFO | "gpu" : " 17 GB", | 20260506 09:55:27.308 | INFO | "huge" : "", | 20260506 09:55:27.308 | INFO | "huge_shared" : " 27 GB", | 20260506 09:55:27.308 | INFO | "malloc" : " 14 GB", | 20260506 09:55:27.308 | INFO | "pinned" : " 29 GB" | 20260506 09:55:27.308 | INFO | } | 20260506 09:55:27.308 | INFO | Perf report for the last 5.13916 seconds: | 20260506 09:55:27.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:55:27.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:27.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:27.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:27.308 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:55:27.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:27.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:27.308 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 09:55:27.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:27.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:27.309 | INFO | Uploaded 36GiB to GPU in 1.60478s (22.433GiB/s) | 20260506 09:55:27.309 | INFO | Downloaded 34GiB to GPU in 1.52292s (22.3255GiB/s) | 20260506 09:55:32.447 | INFO | Analyzing chunk frames = [441856,442368) | 20260506 09:55:32.447 | INFO | MemInfo: { | 20260506 09:55:32.447 | INFO | "gpu" : " 17 GB", | 20260506 09:55:32.447 | INFO | "huge" : "", | 20260506 09:55:32.447 | INFO | "huge_shared" : " 27 GB", | 20260506 09:55:32.447 | INFO | "malloc" : " 14 GB", | 20260506 09:55:32.447 | INFO | "pinned" : " 29 GB" | 20260506 09:55:32.447 | INFO | } | 20260506 09:55:32.447 | INFO | Perf report for the last 5.13929 seconds: | 20260506 09:55:32.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:55:32.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:32.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:32.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:32.447 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260506 09:55:32.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:32.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:32.447 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 09:55:32.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:32.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:32.448 | INFO | Uploaded 36GiB to GPU in 1.60613s (22.4141GiB/s) | 20260506 09:55:32.448 | INFO | Downloaded 34GiB to GPU in 1.52406s (22.3088GiB/s) | 20260506 09:55:32.448 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:55:32.448 | INFO | Total mem usage: 15.9 GiB | 20260506 09:55:32.448 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 2.53%) | 20260506 09:55:32.448 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -7.5%) | 20260506 09:55:32.448 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:55:32.448 | INFO | High Water Breakdown: | 20260506 09:55:32.448 | INFO | 59.2% : 8.77 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:55:32.448 | INFO | 40.6% : 6.02 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:55:32.448 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:55:32.448 | INFO | ------------------------------------------ | 20260506 09:55:32.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:55:32.448 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:55:32.448 | INFO | Total mem usage: 26.5 GiB | 20260506 09:55:32.448 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:55:32.448 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:55:32.448 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:55:32.448 | INFO | High Water Breakdown: | 20260506 09:55:32.448 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:55:32.448 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:55:32.448 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:55:32.448 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:55:32.448 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:55:32.448 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:55:32.448 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:55:32.448 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:55:32.448 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:55:32.448 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:55:32.448 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:55:32.448 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:55:32.448 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:55:32.448 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:55:32.448 | INFO | ------------------------------------------ | 20260506 09:55:32.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:55:32.448 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:55:32.448 | INFO | Total mem usage: 25.6 GiB | 20260506 09:55:32.448 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:55:32.448 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:55:32.448 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:55:32.448 | INFO | High Water Breakdown: | 20260506 09:55:32.448 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:55:32.448 | INFO | ------------------------------------------ | 20260506 09:55:32.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:55:32.448 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:55:32.448 | INFO | Total mem usage: 15.7 GiB | 20260506 09:55:32.448 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:55:32.448 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:55:32.448 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:55:32.448 | INFO | High Water Breakdown: | 20260506 09:55:32.448 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:55:32.448 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:55:32.448 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:55:32.448 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:55:32.448 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:55:32.448 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:55:32.448 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:55:32.448 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:55:32.448 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:55:32.448 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:55:32.448 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:55:32.448 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:55:32.448 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:55:32.448 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:55:32.448 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:55:32.448 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:55:32.448 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:55:32.448 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:55:32.448 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:55:32.448 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:55:32.448 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:55:32.448 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:55:32.448 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:55:32.448 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:55:32.448 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:55:32.448 | INFO | ------------------------------------------ | 20260506 09:55:32.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:55:32.448 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:55:32.448 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:55:32.448 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:55:32.448 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 09:55:32.448 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / -nan, 4.44e-03 / 0.00e+00 | 20260506 09:55:32.448 | INFO | PerfReport: 10.92%, Baseline -- 3.29e+00 / -nan, 1.76e-02 / 0.00e+00 | 20260506 09:55:32.448 | INFO | PerfReport: 3.05%, Histogram -- 9.21e-01 / -nan, 5.12e-02 / 0.00e+00 | 20260506 09:55:32.448 | INFO | PerfReport: 16.05%, DME -- 3.87e+01 / -nan, 3.05e+00 / 0.00e+00 | 20260506 09:55:32.448 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.72e+01 / -nan, 4.18e-02 / 0.00e+00 | 20260506 09:55:32.448 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.25e+00 / -nan, 1.32e-01 / 0.00e+00 | 20260506 09:55:32.448 | INFO | PerfReport: 3.76%, Metrics -- 1.13e+00 / -nan, 1.77e-01 / 0.00e+00 | 20260506 09:55:32.448 | INFO | PerfReport: 1.46%, PulseSerializing -- 4.39e-01 / -nan, 6.29e-02 / 0.00e+00 | 20260506 09:55:32.448 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:55:32.448 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:55:32.448 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:55:32.448 | INFO | PerfReport: Full Execution -- 9.03e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 09:55:32.448 | INFO | PerfReport: 18.60%, Upload -- 1.68e+01 / 0.00e+00, 4.36e-01 / 0.00e+00 | 20260506 09:55:32.448 | INFO | PerfReport: 17.86%, Download -- 1.61e+01 / 0.00e+00, 7.72e-01 / 0.00e+00 | 20260506 09:55:32.448 | INFO | PerfReport: 33.42%, Compute -- 3.02e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 09:55:32.449 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 09:55:32.449 | INFO | PerfReport: 0.00%, LockDownload -- 4.89e-04 / 0.00e+00, 1.16e-04 / 0.00e+00 | 20260506 09:55:32.449 | INFO | PerfReport: 29.55%, LockCompute -- 2.67e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260506 09:55:32.449 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260506 09:55:32.544 | INFO | Starting a round of Baz IO | 20260506 09:55:35.462 | INFO | Populating RTMetrics report | 20260506 09:55:35.463 | INFO | RTMetrics report populated | 20260506 09:55:35.463 | INFO | Opening tmp RTMetrics json file | 20260506 09:55:35.467 | INFO | Serializing RTMetrics to json | 20260506 09:55:35.470 | INFO | Serialization done, swapping tmp file now... | 20260506 09:55:35.472 | INFO | RTMetrics done | 20260506 09:55:35.684 | INFO | Finished a round of Baz IO: Wrote 1.923 GiB in 3.14 seconds | 20260506 09:55:35.684 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.72% / 0.07% / 49.20% | 20260506 09:55:35.684 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.0936% / 0.000166893% / 3.17891e-05% | 20260506 09:55:35.684 | INFO | Baz IO overall: | 20260506 09:55:35.684 | INFO | BytesWritten : 186937005233 | 20260506 09:55:35.684 | INFO | NumEvents : 89353982004 | 20260506 09:55:37.586 | INFO | Analyzing chunk frames = [442368,442880) | 20260506 09:55:37.586 | INFO | MemInfo: { | 20260506 09:55:37.586 | INFO | "gpu" : " 17 GB", | 20260506 09:55:37.586 | INFO | "huge" : "", | 20260506 09:55:37.586 | INFO | "huge_shared" : " 27 GB", | 20260506 09:55:37.586 | INFO | "malloc" : " 14 GB", | 20260506 09:55:37.586 | INFO | "pinned" : " 29 GB" | 20260506 09:55:37.586 | INFO | } | 20260506 09:55:37.586 | INFO | Perf report for the last 5.13917 seconds: | 20260506 09:55:37.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:55:37.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:37.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:37.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:37.587 | INFO | ANALYSIS : 57.7%, 2.8, 86.0ms, 0 | 20260506 09:55:37.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:37.587 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 09:55:37.587 | INFO | PRE_HQ : 56.9%, 2.1, 61.3ms, 0 | 20260506 09:55:37.587 | INFO | BAZWRITER : 60.6%, 4.7, 147.4ms, 0 | 20260506 09:55:37.587 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 09:55:37.587 | INFO | Uploaded 36GiB to GPU in 1.66665s (21.6003GiB/s) | 20260506 09:55:37.587 | INFO | Downloaded 38GiB to GPU in 1.73184s (21.9419GiB/s) | 20260506 09:55:42.726 | INFO | Analyzing chunk frames = [442880,443392) | 20260506 09:55:42.726 | INFO | MemInfo: { | 20260506 09:55:42.726 | INFO | "gpu" : " 17 GB", | 20260506 09:55:42.726 | INFO | "huge" : "", | 20260506 09:55:42.726 | INFO | "huge_shared" : " 27 GB", | 20260506 09:55:42.726 | INFO | "malloc" : " 14 GB", | 20260506 09:55:42.726 | INFO | "pinned" : " 29 GB" | 20260506 09:55:42.726 | INFO | } | 20260506 09:55:42.726 | INFO | Perf report for the last 5.13945 seconds: | 20260506 09:55:42.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:55:42.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:42.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:42.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:42.726 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260506 09:55:42.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:42.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:42.726 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 09:55:42.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:42.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:42.726 | INFO | Uploaded 36GiB to GPU in 1.60259s (22.4636GiB/s) | 20260506 09:55:42.726 | INFO | Downloaded 34GiB to GPU in 1.5201s (22.367GiB/s) | 20260506 09:55:47.865 | INFO | Analyzing chunk frames = [443392,443904) | 20260506 09:55:47.865 | INFO | MemInfo: { | 20260506 09:55:47.865 | INFO | "gpu" : " 17 GB", | 20260506 09:55:47.865 | INFO | "huge" : "", | 20260506 09:55:47.865 | INFO | "huge_shared" : " 27 GB", | 20260506 09:55:47.865 | INFO | "malloc" : " 14 GB", | 20260506 09:55:47.865 | INFO | "pinned" : " 29 GB" | 20260506 09:55:47.865 | INFO | } | 20260506 09:55:47.865 | INFO | Perf report for the last 5.13938 seconds: | 20260506 09:55:47.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:55:47.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:47.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:47.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:47.865 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:55:47.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:47.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:47.865 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 09:55:47.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:47.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:47.866 | INFO | Uploaded 36GiB to GPU in 1.6055s (22.4229GiB/s) | 20260506 09:55:47.866 | INFO | Downloaded 34GiB to GPU in 1.52446s (22.303GiB/s) | 20260506 09:55:53.004 | INFO | Analyzing chunk frames = [443904,444416) | 20260506 09:55:53.004 | INFO | MemInfo: { | 20260506 09:55:53.004 | INFO | "gpu" : " 17 GB", | 20260506 09:55:53.004 | INFO | "huge" : "", | 20260506 09:55:53.004 | INFO | "huge_shared" : " 27 GB", | 20260506 09:55:53.005 | INFO | "malloc" : " 14 GB", | 20260506 09:55:53.005 | INFO | "pinned" : " 29 GB" | 20260506 09:55:53.005 | INFO | } | 20260506 09:55:53.005 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:55:53.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:55:53.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:53.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:53.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:53.005 | INFO | ANALYSIS : 56.6%, 3.0, 87.7ms, 0 | 20260506 09:55:53.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:53.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:53.005 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:55:53.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:53.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:53.005 | INFO | Uploaded 36GiB to GPU in 1.60595s (22.4166GiB/s) | 20260506 09:55:53.005 | INFO | Downloaded 34GiB to GPU in 1.52425s (22.3061GiB/s) | 20260506 09:55:58.144 | INFO | Analyzing chunk frames = [444416,444928) | 20260506 09:55:58.144 | INFO | MemInfo: { | 20260506 09:55:58.144 | INFO | "gpu" : " 17 GB", | 20260506 09:55:58.144 | INFO | "huge" : "", | 20260506 09:55:58.144 | INFO | "huge_shared" : " 27 GB", | 20260506 09:55:58.144 | INFO | "malloc" : " 14 GB", | 20260506 09:55:58.144 | INFO | "pinned" : " 29 GB" | 20260506 09:55:58.144 | INFO | } | 20260506 09:55:58.144 | INFO | Perf report for the last 5.13926 seconds: | 20260506 09:55:58.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:55:58.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:58.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:58.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:58.144 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 09:55:58.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:58.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:58.144 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:55:58.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:58.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:55:58.144 | INFO | Uploaded 36GiB to GPU in 1.60315s (22.4558GiB/s) | 20260506 09:55:58.145 | INFO | Downloaded 34GiB to GPU in 1.52098s (22.3541GiB/s) | 20260506 09:56:03.283 | INFO | Analyzing chunk frames = [444928,445440) | 20260506 09:56:03.283 | INFO | MemInfo: { | 20260506 09:56:03.283 | INFO | "gpu" : " 17 GB", | 20260506 09:56:03.283 | INFO | "huge" : "", | 20260506 09:56:03.283 | INFO | "huge_shared" : " 27 GB", | 20260506 09:56:03.283 | INFO | "malloc" : " 14 GB", | 20260506 09:56:03.283 | INFO | "pinned" : " 29 GB" | 20260506 09:56:03.283 | INFO | } | 20260506 09:56:03.283 | INFO | Perf report for the last 5.1393 seconds: | 20260506 09:56:03.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:56:03.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:03.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:03.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:03.283 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 09:56:03.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:03.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:03.283 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:56:03.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:03.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:03.284 | INFO | Uploaded 36GiB to GPU in 1.60457s (22.436GiB/s) | 20260506 09:56:03.284 | INFO | Downloaded 34GiB to GPU in 1.52294s (22.3253GiB/s) | 20260506 09:56:08.422 | INFO | Analyzing chunk frames = [445440,445952) | 20260506 09:56:08.422 | INFO | MemInfo: { | 20260506 09:56:08.422 | INFO | "gpu" : " 17 GB", | 20260506 09:56:08.422 | INFO | "huge" : "", | 20260506 09:56:08.423 | INFO | "huge_shared" : " 27 GB", | 20260506 09:56:08.423 | INFO | "malloc" : " 14 GB", | 20260506 09:56:08.423 | INFO | "pinned" : " 29 GB" | 20260506 09:56:08.423 | INFO | } | 20260506 09:56:08.423 | INFO | Perf report for the last 5.13944 seconds: | 20260506 09:56:08.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:56:08.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:08.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:08.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:08.423 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:56:08.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:08.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:08.423 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:56:08.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:08.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:08.423 | INFO | Uploaded 36GiB to GPU in 1.60501s (22.4297GiB/s) | 20260506 09:56:08.423 | INFO | Downloaded 34GiB to GPU in 1.52287s (22.3263GiB/s) | 20260506 09:56:13.562 | INFO | Analyzing chunk frames = [445952,446464) | 20260506 09:56:13.562 | INFO | MemInfo: { | 20260506 09:56:13.562 | INFO | "gpu" : " 17 GB", | 20260506 09:56:13.562 | INFO | "huge" : "", | 20260506 09:56:13.562 | INFO | "huge_shared" : " 27 GB", | 20260506 09:56:13.562 | INFO | "malloc" : " 14 GB", | 20260506 09:56:13.562 | INFO | "pinned" : " 29 GB" | 20260506 09:56:13.562 | INFO | } | 20260506 09:56:13.562 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:56:13.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:56:13.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:13.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:13.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:13.562 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260506 09:56:13.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:13.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:13.562 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 09:56:13.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:13.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:13.562 | INFO | Uploaded 36GiB to GPU in 1.6049s (22.4313GiB/s) | 20260506 09:56:13.563 | INFO | Downloaded 34GiB to GPU in 1.52311s (22.3228GiB/s) | 20260506 09:56:13.659 | INFO | Starting a round of Baz IO | 20260506 09:56:16.578 | INFO | Populating RTMetrics report | 20260506 09:56:16.579 | INFO | RTMetrics report populated | 20260506 09:56:16.579 | INFO | Opening tmp RTMetrics json file | 20260506 09:56:16.595 | INFO | Serializing RTMetrics to json | 20260506 09:56:16.706 | INFO | Serialization done, swapping tmp file now... | 20260506 09:56:16.729 | INFO | RTMetrics done | 20260506 09:56:16.759 | INFO | Finished a round of Baz IO: Wrote 1.925 GiB in 3.099 seconds | 20260506 09:56:16.759 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.65% / 0.07% / 49.28% | 20260506 09:56:16.759 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.06818% / 0.000222524% / 2.38419e-05% | 20260506 09:56:16.759 | INFO | Baz IO overall: | 20260506 09:56:16.759 | INFO | BytesWritten : 189003937008 | 20260506 09:56:16.759 | INFO | NumEvents : 90357034362 | 20260506 09:56:18.701 | INFO | Analyzing chunk frames = [446464,446976) | 20260506 09:56:18.701 | INFO | MemInfo: { | 20260506 09:56:18.701 | INFO | "gpu" : " 17 GB", | 20260506 09:56:18.701 | INFO | "huge" : "", | 20260506 09:56:18.701 | INFO | "huge_shared" : " 27 GB", | 20260506 09:56:18.701 | INFO | "malloc" : " 14 GB", | 20260506 09:56:18.701 | INFO | "pinned" : " 29 GB" | 20260506 09:56:18.701 | INFO | } | 20260506 09:56:18.701 | INFO | Perf report for the last 5.13922 seconds: | 20260506 09:56:18.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:56:18.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:18.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:18.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:18.701 | INFO | ANALYSIS : 57.7%, 2.9, 87.9ms, 0 | 20260506 09:56:18.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:18.701 | INFO | RT_METRICS : 47.0%, 1.0, 24.7ms, 0 | 20260506 09:56:18.701 | INFO | PRE_HQ : 56.9%, 2.1, 62.1ms, 0 | 20260506 09:56:18.701 | INFO | BAZWRITER : 59.9%, 4.5, 139.1ms, 0 | 20260506 09:56:18.701 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 09:56:18.702 | INFO | Uploaded 36GiB to GPU in 1.6563s (21.7352GiB/s) | 20260506 09:56:18.702 | INFO | Downloaded 38GiB to GPU in 1.72005s (22.0923GiB/s) | 20260506 09:56:23.840 | INFO | Analyzing chunk frames = [446976,447488) | 20260506 09:56:23.840 | INFO | MemInfo: { | 20260506 09:56:23.840 | INFO | "gpu" : " 17 GB", | 20260506 09:56:23.840 | INFO | "huge" : "", | 20260506 09:56:23.841 | INFO | "huge_shared" : " 27 GB", | 20260506 09:56:23.841 | INFO | "malloc" : " 14 GB", | 20260506 09:56:23.841 | INFO | "pinned" : " 29 GB" | 20260506 09:56:23.841 | INFO | } | 20260506 09:56:23.841 | INFO | Perf report for the last 5.1394 seconds: | 20260506 09:56:23.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:56:23.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:23.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:23.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:23.841 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260506 09:56:23.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:23.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:23.841 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:56:23.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:23.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:23.841 | INFO | Uploaded 36GiB to GPU in 1.60257s (22.4639GiB/s) | 20260506 09:56:23.841 | INFO | Downloaded 34GiB to GPU in 1.51936s (22.3778GiB/s) | 20260506 09:56:28.980 | INFO | Analyzing chunk frames = [447488,448000) | 20260506 09:56:28.980 | INFO | MemInfo: { | 20260506 09:56:28.980 | INFO | "gpu" : " 17 GB", | 20260506 09:56:28.980 | INFO | "huge" : "", | 20260506 09:56:28.980 | INFO | "huge_shared" : " 27 GB", | 20260506 09:56:28.980 | INFO | "malloc" : " 14 GB", | 20260506 09:56:28.980 | INFO | "pinned" : " 29 GB" | 20260506 09:56:28.980 | INFO | } | 20260506 09:56:28.980 | INFO | Perf report for the last 5.13906 seconds: | 20260506 09:56:28.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:56:28.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:28.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:28.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:28.980 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:56:28.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:28.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:28.980 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:56:28.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:28.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:28.980 | INFO | Uploaded 36GiB to GPU in 1.60622s (22.4128GiB/s) | 20260506 09:56:28.980 | INFO | Downloaded 34GiB to GPU in 1.52345s (22.3177GiB/s) | 20260506 09:56:34.119 | INFO | Analyzing chunk frames = [448000,448512) | 20260506 09:56:34.119 | INFO | MemInfo: { | 20260506 09:56:34.119 | INFO | "gpu" : " 17 GB", | 20260506 09:56:34.119 | INFO | "huge" : "", | 20260506 09:56:34.119 | INFO | "huge_shared" : " 27 GB", | 20260506 09:56:34.119 | INFO | "malloc" : " 14 GB", | 20260506 09:56:34.119 | INFO | "pinned" : " 29 GB" | 20260506 09:56:34.119 | INFO | } | 20260506 09:56:34.119 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:56:34.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:56:34.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:34.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:34.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:34.119 | INFO | ANALYSIS : 56.6%, 3.0, 87.8ms, 0 | 20260506 09:56:34.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:34.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:34.119 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:56:34.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:34.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:34.119 | INFO | Uploaded 36GiB to GPU in 1.60578s (22.419GiB/s) | 20260506 09:56:34.119 | INFO | Downloaded 34GiB to GPU in 1.52343s (22.3181GiB/s) | 20260506 09:56:39.258 | INFO | Analyzing chunk frames = [448512,449024) | 20260506 09:56:39.258 | INFO | MemInfo: { | 20260506 09:56:39.258 | INFO | "gpu" : " 17 GB", | 20260506 09:56:39.258 | INFO | "huge" : "", | 20260506 09:56:39.258 | INFO | "huge_shared" : " 27 GB", | 20260506 09:56:39.258 | INFO | "malloc" : " 14 GB", | 20260506 09:56:39.258 | INFO | "pinned" : " 29 GB" | 20260506 09:56:39.258 | INFO | } | 20260506 09:56:39.259 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:56:39.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:56:39.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:39.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:39.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:39.259 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 09:56:39.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:39.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:39.259 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:56:39.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:39.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:39.259 | INFO | Uploaded 36GiB to GPU in 1.60476s (22.4333GiB/s) | 20260506 09:56:39.259 | INFO | Downloaded 34GiB to GPU in 1.52156s (22.3455GiB/s) | 20260506 09:56:44.398 | INFO | Analyzing chunk frames = [449024,449536) | 20260506 09:56:44.398 | INFO | MemInfo: { | 20260506 09:56:44.398 | INFO | "gpu" : " 17 GB", | 20260506 09:56:44.398 | INFO | "huge" : "", | 20260506 09:56:44.398 | INFO | "huge_shared" : " 27 GB", | 20260506 09:56:44.398 | INFO | "malloc" : " 14 GB", | 20260506 09:56:44.398 | INFO | "pinned" : " 29 GB" | 20260506 09:56:44.398 | INFO | } | 20260506 09:56:44.398 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:56:44.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:56:44.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:44.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:44.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:44.398 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:56:44.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:44.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:44.398 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 09:56:44.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:44.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:44.398 | INFO | Uploaded 36GiB to GPU in 1.60447s (22.4373GiB/s) | 20260506 09:56:44.398 | INFO | Downloaded 34GiB to GPU in 1.52196s (22.3396GiB/s) | 20260506 09:56:49.537 | INFO | Analyzing chunk frames = [449536,450048) | 20260506 09:56:49.537 | INFO | MemInfo: { | 20260506 09:56:49.537 | INFO | "gpu" : " 17 GB", | 20260506 09:56:49.537 | INFO | "huge" : "", | 20260506 09:56:49.537 | INFO | "huge_shared" : " 27 GB", | 20260506 09:56:49.537 | INFO | "malloc" : " 14 GB", | 20260506 09:56:49.537 | INFO | "pinned" : " 29 GB" | 20260506 09:56:49.537 | INFO | } | 20260506 09:56:49.537 | INFO | Perf report for the last 5.13938 seconds: | 20260506 09:56:49.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:56:49.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:49.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:49.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:49.537 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 09:56:49.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:49.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:49.537 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 09:56:49.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:49.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:49.537 | INFO | Uploaded 36GiB to GPU in 1.60523s (22.4267GiB/s) | 20260506 09:56:49.537 | INFO | Downloaded 34GiB to GPU in 1.52354s (22.3164GiB/s) | 20260506 09:56:54.676 | INFO | Analyzing chunk frames = [450048,450560) | 20260506 09:56:54.676 | INFO | MemInfo: { | 20260506 09:56:54.676 | INFO | "gpu" : " 17 GB", | 20260506 09:56:54.676 | INFO | "huge" : "", | 20260506 09:56:54.676 | INFO | "huge_shared" : " 27 GB", | 20260506 09:56:54.676 | INFO | "malloc" : " 14 GB", | 20260506 09:56:54.676 | INFO | "pinned" : " 29 GB" | 20260506 09:56:54.676 | INFO | } | 20260506 09:56:54.676 | INFO | Perf report for the last 5.13931 seconds: | 20260506 09:56:54.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:56:54.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:54.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:54.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:54.677 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 09:56:54.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:54.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:54.677 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 09:56:54.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:54.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:54.677 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:56:54.677 | INFO | Total mem usage: 15.9 GiB | 20260506 09:56:54.677 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 2.53%) | 20260506 09:56:54.677 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -6.06%) | 20260506 09:56:54.677 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:56:54.677 | INFO | High Water Breakdown: | 20260506 09:56:54.677 | INFO | 59.5% : 8.93 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:56:54.677 | INFO | 40.4% : 6.07 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:56:54.677 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:56:54.677 | INFO | ------------------------------------------ | 20260506 09:56:54.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:56:54.677 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:56:54.677 | INFO | Total mem usage: 26.5 GiB | 20260506 09:56:54.677 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:56:54.677 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:56:54.677 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:56:54.677 | INFO | High Water Breakdown: | 20260506 09:56:54.677 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:56:54.677 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:56:54.677 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:56:54.677 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:56:54.677 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:56:54.677 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:56:54.677 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:56:54.677 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:56:54.677 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:56:54.677 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:56:54.677 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:56:54.677 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:56:54.677 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:56:54.677 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:56:54.677 | INFO | ------------------------------------------ | 20260506 09:56:54.677 | INFO | Uploaded 36GiB to GPU in 1.60464s (22.4349GiB/s) | 20260506 09:56:54.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:56:54.677 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:56:54.677 | INFO | Downloaded 34GiB to GPU in 1.52306s (22.3234GiB/s) | 20260506 09:56:54.677 | INFO | Total mem usage: 25.6 GiB | 20260506 09:56:54.677 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:56:54.677 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:56:54.677 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:56:54.677 | INFO | High Water Breakdown: | 20260506 09:56:54.677 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:56:54.677 | INFO | ------------------------------------------ | 20260506 09:56:54.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:56:54.677 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:56:54.678 | INFO | Total mem usage: 15.7 GiB | 20260506 09:56:54.678 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:56:54.678 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:56:54.678 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:56:54.678 | INFO | High Water Breakdown: | 20260506 09:56:54.678 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:56:54.678 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:56:54.678 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:56:54.678 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:56:54.678 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:56:54.678 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:56:54.678 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:56:54.678 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:56:54.678 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:56:54.678 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:56:54.678 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:56:54.678 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:56:54.678 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:56:54.678 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:56:54.678 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:56:54.678 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:56:54.678 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:56:54.678 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:56:54.678 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:56:54.678 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:56:54.678 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:56:54.678 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:56:54.678 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:56:54.678 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:56:54.678 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:56:54.678 | INFO | ------------------------------------------ | 20260506 09:56:54.678 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:56:54.678 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:56:54.678 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:56:54.678 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:56:54.678 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 09:56:54.678 | INFO | PerfReport: 0.29%, AggregateStats -- 8.61e-02 / -nan, 4.71e-03 / 0.00e+00 | 20260506 09:56:54.678 | INFO | PerfReport: 10.92%, Baseline -- 3.29e+00 / -nan, 1.89e-02 / 0.00e+00 | 20260506 09:56:54.678 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.19e-02 / 0.00e+00 | 20260506 09:56:54.678 | INFO | PerfReport: 16.05%, DME -- 3.87e+01 / -nan, 3.09e+00 / 0.00e+00 | 20260506 09:56:54.678 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.72e+01 / -nan, 4.24e-02 / 0.00e+00 | 20260506 09:56:54.678 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.25e+00 / -nan, 1.33e-01 / 0.00e+00 | 20260506 09:56:54.678 | INFO | PerfReport: 3.76%, Metrics -- 1.13e+00 / -nan, 1.77e-01 / 0.00e+00 | 20260506 09:56:54.678 | INFO | PerfReport: 1.46%, PulseSerializing -- 4.39e-01 / -nan, 6.32e-02 / 0.00e+00 | 20260506 09:56:54.678 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:56:54.678 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:56:54.678 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:56:54.678 | INFO | PerfReport: Full Execution -- 9.02e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 09:56:54.678 | INFO | PerfReport: 18.61%, Upload -- 1.68e+01 / 0.00e+00, 4.51e-01 / 0.00e+00 | 20260506 09:56:54.678 | INFO | PerfReport: 17.87%, Download -- 1.61e+01 / 0.00e+00, 7.92e-01 / 0.00e+00 | 20260506 09:56:54.678 | INFO | PerfReport: 33.43%, Compute -- 3.02e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 09:56:54.678 | INFO | PerfReport: 0.57%, LockUpload -- 5.10e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260506 09:56:54.678 | INFO | PerfReport: 0.00%, LockDownload -- 4.91e-04 / 0.00e+00, 1.68e-04 / 0.00e+00 | 20260506 09:56:54.678 | INFO | PerfReport: 29.51%, LockCompute -- 2.66e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 09:56:54.678 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260506 09:56:54.774 | INFO | Starting a round of Baz IO | 20260506 09:56:57.694 | INFO | Populating RTMetrics report | 20260506 09:56:57.695 | INFO | RTMetrics report populated | 20260506 09:56:57.695 | INFO | Opening tmp RTMetrics json file | 20260506 09:56:57.716 | INFO | Serializing RTMetrics to json | 20260506 09:56:57.721 | INFO | Serialization done, swapping tmp file now... | 20260506 09:56:57.725 | INFO | RTMetrics done | 20260506 09:56:57.924 | INFO | Finished a round of Baz IO: Wrote 1.929 GiB in 3.15 seconds | 20260506 09:56:57.924 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.58% / 0.07% / 49.35% | 20260506 09:56:57.924 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.06069% / 0.000150998% / 3.17891e-05% | 20260506 09:56:57.924 | INFO | Baz IO overall: | 20260506 09:56:57.924 | INFO | BytesWritten : 191075508846 | 20260506 09:56:57.924 | INFO | NumEvents : 91362982113 | 20260506 09:56:59.816 | INFO | Analyzing chunk frames = [450560,451072) | 20260506 09:56:59.816 | INFO | MemInfo: { | 20260506 09:56:59.816 | INFO | "gpu" : " 17 GB", | 20260506 09:56:59.816 | INFO | "huge" : "", | 20260506 09:56:59.816 | INFO | "huge_shared" : " 27 GB", | 20260506 09:56:59.816 | INFO | "malloc" : " 14 GB", | 20260506 09:56:59.816 | INFO | "pinned" : " 29 GB" | 20260506 09:56:59.816 | INFO | } | 20260506 09:56:59.816 | INFO | Perf report for the last 5.13924 seconds: | 20260506 09:56:59.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:56:59.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:59.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:59.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:59.816 | INFO | ANALYSIS : 57.7%, 2.8, 85.5ms, 0 | 20260506 09:56:59.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:56:59.816 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 09:56:59.816 | INFO | PRE_HQ : 56.9%, 2.1, 61.1ms, 0 | 20260506 09:56:59.816 | INFO | BAZWRITER : 60.9%, 4.8, 149.2ms, 0 | 20260506 09:56:59.816 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 09:56:59.816 | INFO | Uploaded 36GiB to GPU in 1.67272s (21.5218GiB/s) | 20260506 09:56:59.816 | INFO | Downloaded 38GiB to GPU in 1.73797s (21.8646GiB/s) | 20260506 09:57:04.955 | INFO | Analyzing chunk frames = [451072,451584) | 20260506 09:57:04.955 | INFO | MemInfo: { | 20260506 09:57:04.955 | INFO | "gpu" : " 17 GB", | 20260506 09:57:04.955 | INFO | "huge" : "", | 20260506 09:57:04.955 | INFO | "huge_shared" : " 27 GB", | 20260506 09:57:04.955 | INFO | "malloc" : " 14 GB", | 20260506 09:57:04.955 | INFO | "pinned" : " 29 GB" | 20260506 09:57:04.955 | INFO | } | 20260506 09:57:04.955 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:57:04.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:57:04.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:04.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:04.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:04.955 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 09:57:04.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:04.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:04.955 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:57:04.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:04.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:04.955 | INFO | Uploaded 36GiB to GPU in 1.60218s (22.4694GiB/s) | 20260506 09:57:04.956 | INFO | Downloaded 34GiB to GPU in 1.51953s (22.3754GiB/s) | 20260506 09:57:10.094 | INFO | Analyzing chunk frames = [451584,452096) | 20260506 09:57:10.094 | INFO | MemInfo: { | 20260506 09:57:10.094 | INFO | "gpu" : " 17 GB", | 20260506 09:57:10.094 | INFO | "huge" : "", | 20260506 09:57:10.094 | INFO | "huge_shared" : " 27 GB", | 20260506 09:57:10.094 | INFO | "malloc" : " 14 GB", | 20260506 09:57:10.095 | INFO | "pinned" : " 29 GB" | 20260506 09:57:10.095 | INFO | } | 20260506 09:57:10.095 | INFO | Perf report for the last 5.13945 seconds: | 20260506 09:57:10.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:57:10.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:10.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:10.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:10.095 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260506 09:57:10.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:10.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:10.095 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 09:57:10.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:10.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:10.095 | INFO | Uploaded 36GiB to GPU in 1.60714s (22.4GiB/s) | 20260506 09:57:10.095 | INFO | Downloaded 34GiB to GPU in 1.52467s (22.2999GiB/s) | 20260506 09:57:15.234 | INFO | Analyzing chunk frames = [452096,452608) | 20260506 09:57:15.234 | INFO | MemInfo: { | 20260506 09:57:15.234 | INFO | "gpu" : " 17 GB", | 20260506 09:57:15.234 | INFO | "huge" : "", | 20260506 09:57:15.234 | INFO | "huge_shared" : " 27 GB", | 20260506 09:57:15.234 | INFO | "malloc" : " 14 GB", | 20260506 09:57:15.234 | INFO | "pinned" : " 29 GB" | 20260506 09:57:15.234 | INFO | } | 20260506 09:57:15.234 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:57:15.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:57:15.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:15.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:15.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:15.234 | INFO | ANALYSIS : 56.6%, 3.0, 87.7ms, 0 | 20260506 09:57:15.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:15.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:15.234 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:57:15.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:15.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:15.235 | INFO | Uploaded 36GiB to GPU in 1.60594s (22.4168GiB/s) | 20260506 09:57:15.235 | INFO | Downloaded 34GiB to GPU in 1.52317s (22.3218GiB/s) | 20260506 09:57:20.373 | INFO | Analyzing chunk frames = [452608,453120) | 20260506 09:57:20.373 | INFO | MemInfo: { | 20260506 09:57:20.373 | INFO | "gpu" : " 17 GB", | 20260506 09:57:20.373 | INFO | "huge" : "", | 20260506 09:57:20.373 | INFO | "huge_shared" : " 27 GB", | 20260506 09:57:20.373 | INFO | "malloc" : " 14 GB", | 20260506 09:57:20.373 | INFO | "pinned" : " 29 GB" | 20260506 09:57:20.373 | INFO | } | 20260506 09:57:20.373 | INFO | Perf report for the last 5.13934 seconds: | 20260506 09:57:20.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:57:20.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:20.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:20.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:20.373 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 09:57:20.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:20.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:20.373 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:57:20.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:20.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:20.374 | INFO | Uploaded 36GiB to GPU in 1.60506s (22.4291GiB/s) | 20260506 09:57:20.374 | INFO | Downloaded 34GiB to GPU in 1.52178s (22.3423GiB/s) | 20260506 09:57:25.512 | INFO | Analyzing chunk frames = [453120,453632) | 20260506 09:57:25.512 | INFO | MemInfo: { | 20260506 09:57:25.512 | INFO | "gpu" : " 17 GB", | 20260506 09:57:25.512 | INFO | "huge" : "", | 20260506 09:57:25.513 | INFO | "huge_shared" : " 27 GB", | 20260506 09:57:25.513 | INFO | "malloc" : " 14 GB", | 20260506 09:57:25.513 | INFO | "pinned" : " 29 GB" | 20260506 09:57:25.513 | INFO | } | 20260506 09:57:25.513 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:57:25.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:57:25.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:25.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:25.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:25.513 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 09:57:25.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:25.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:25.513 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:57:25.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:25.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:25.513 | INFO | Uploaded 36GiB to GPU in 1.60386s (22.4458GiB/s) | 20260506 09:57:25.513 | INFO | Downloaded 34GiB to GPU in 1.52152s (22.346GiB/s) | 20260506 09:57:30.652 | INFO | Analyzing chunk frames = [453632,454144) | 20260506 09:57:30.652 | INFO | MemInfo: { | 20260506 09:57:30.652 | INFO | "gpu" : " 17 GB", | 20260506 09:57:30.652 | INFO | "huge" : "", | 20260506 09:57:30.652 | INFO | "huge_shared" : " 27 GB", | 20260506 09:57:30.652 | INFO | "malloc" : " 14 GB", | 20260506 09:57:30.652 | INFO | "pinned" : " 29 GB" | 20260506 09:57:30.652 | INFO | } | 20260506 09:57:30.652 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:57:30.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:57:30.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:30.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:30.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:30.652 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 09:57:30.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:30.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:30.652 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 09:57:30.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:30.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:30.652 | INFO | Uploaded 36GiB to GPU in 1.60509s (22.4286GiB/s) | 20260506 09:57:30.652 | INFO | Downloaded 34GiB to GPU in 1.5233s (22.3199GiB/s) | 20260506 09:57:35.791 | INFO | Analyzing chunk frames = [454144,454656) | 20260506 09:57:35.791 | INFO | MemInfo: { | 20260506 09:57:35.791 | INFO | "gpu" : " 17 GB", | 20260506 09:57:35.791 | INFO | "huge" : "", | 20260506 09:57:35.791 | INFO | "huge_shared" : " 27 GB", | 20260506 09:57:35.791 | INFO | "malloc" : " 14 GB", | 20260506 09:57:35.791 | INFO | "pinned" : " 29 GB" | 20260506 09:57:35.791 | INFO | } | 20260506 09:57:35.791 | INFO | Perf report for the last 5.13918 seconds: | 20260506 09:57:35.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:57:35.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:35.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:35.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:35.791 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:57:35.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:35.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:35.791 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 09:57:35.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:35.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:35.792 | INFO | Uploaded 36GiB to GPU in 1.60519s (22.4273GiB/s) | 20260506 09:57:35.792 | INFO | Downloaded 34GiB to GPU in 1.52337s (22.3189GiB/s) | 20260506 09:57:35.889 | INFO | Starting a round of Baz IO | 20260506 09:57:38.809 | INFO | Populating RTMetrics report | 20260506 09:57:38.810 | INFO | RTMetrics report populated | 20260506 09:57:38.810 | INFO | Opening tmp RTMetrics json file | 20260506 09:57:38.814 | INFO | Serializing RTMetrics to json | 20260506 09:57:38.849 | INFO | Serialization done, swapping tmp file now... | 20260506 09:57:38.850 | INFO | RTMetrics done | 20260506 09:57:39.210 | INFO | Finished a round of Baz IO: Wrote 1.93 GiB in 3.321 seconds | 20260506 09:57:39.210 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.51% / 0.07% / 49.42% | 20260506 09:57:39.210 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 8.02816% / 0.000166893% / 4.76837e-05% | 20260506 09:57:39.210 | INFO | Baz IO overall: | 20260506 09:57:39.210 | INFO | BytesWritten : 193148098907 | 20260506 09:57:39.210 | INFO | NumEvents : 92369244667 | 20260506 09:57:40.930 | INFO | Analyzing chunk frames = [454656,455168) | 20260506 09:57:40.930 | INFO | MemInfo: { | 20260506 09:57:40.930 | INFO | "gpu" : " 17 GB", | 20260506 09:57:40.930 | INFO | "huge" : "", | 20260506 09:57:40.930 | INFO | "huge_shared" : " 27 GB", | 20260506 09:57:40.930 | INFO | "malloc" : " 14 GB", | 20260506 09:57:40.931 | INFO | "pinned" : " 29 GB" | 20260506 09:57:40.931 | INFO | } | 20260506 09:57:40.931 | INFO | Perf report for the last 5.1395 seconds: | 20260506 09:57:40.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:57:40.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:40.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:40.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:40.931 | INFO | ANALYSIS : 57.7%, 2.9, 86.5ms, 0 | 20260506 09:57:40.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:40.931 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 09:57:40.931 | INFO | PRE_HQ : 56.8%, 2.1, 61.4ms, 0 | 20260506 09:57:40.931 | INFO | BAZWRITER : 64.2%, 4.7, 153.7ms, 0 | 20260506 09:57:40.931 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 09:57:40.931 | INFO | Uploaded 36GiB to GPU in 1.6594s (21.6946GiB/s) | 20260506 09:57:40.931 | INFO | Downloaded 38GiB to GPU in 1.72294s (22.0553GiB/s) | 20260506 09:57:46.070 | INFO | Analyzing chunk frames = [455168,455680) | 20260506 09:57:46.070 | INFO | MemInfo: { | 20260506 09:57:46.070 | INFO | "gpu" : " 17 GB", | 20260506 09:57:46.070 | INFO | "huge" : "", | 20260506 09:57:46.070 | INFO | "huge_shared" : " 27 GB", | 20260506 09:57:46.070 | INFO | "malloc" : " 14 GB", | 20260506 09:57:46.070 | INFO | "pinned" : " 29 GB" | 20260506 09:57:46.070 | INFO | } | 20260506 09:57:46.070 | INFO | Perf report for the last 5.13952 seconds: | 20260506 09:57:46.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:57:46.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:46.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:46.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:46.070 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 09:57:46.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:46.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:46.070 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:57:46.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:46.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:46.070 | INFO | Uploaded 36GiB to GPU in 1.60272s (22.4619GiB/s) | 20260506 09:57:46.070 | INFO | Downloaded 34GiB to GPU in 1.52069s (22.3583GiB/s) | 20260506 09:57:51.209 | INFO | Analyzing chunk frames = [455680,456192) | 20260506 09:57:51.209 | INFO | MemInfo: { | 20260506 09:57:51.209 | INFO | "gpu" : " 17 GB", | 20260506 09:57:51.209 | INFO | "huge" : "", | 20260506 09:57:51.209 | INFO | "huge_shared" : " 27 GB", | 20260506 09:57:51.209 | INFO | "malloc" : " 14 GB", | 20260506 09:57:51.209 | INFO | "pinned" : " 29 GB" | 20260506 09:57:51.209 | INFO | } | 20260506 09:57:51.209 | INFO | Perf report for the last 5.1391 seconds: | 20260506 09:57:51.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:57:51.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:51.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:51.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:51.209 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260506 09:57:51.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:51.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:51.209 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:57:51.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:51.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:51.210 | INFO | Uploaded 36GiB to GPU in 1.60617s (22.4136GiB/s) | 20260506 09:57:51.210 | INFO | Downloaded 34GiB to GPU in 1.52542s (22.2889GiB/s) | 20260506 09:57:56.348 | INFO | Analyzing chunk frames = [456192,456704) | 20260506 09:57:56.348 | INFO | MemInfo: { | 20260506 09:57:56.348 | INFO | "gpu" : " 17 GB", | 20260506 09:57:56.348 | INFO | "huge" : "", | 20260506 09:57:56.348 | INFO | "huge_shared" : " 27 GB", | 20260506 09:57:56.349 | INFO | "malloc" : " 14 GB", | 20260506 09:57:56.349 | INFO | "pinned" : " 29 GB" | 20260506 09:57:56.349 | INFO | } | 20260506 09:57:56.349 | INFO | Perf report for the last 5.13939 seconds: | 20260506 09:57:56.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:57:56.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:56.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:56.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:56.349 | INFO | ANALYSIS : 56.6%, 3.0, 87.7ms, 0 | 20260506 09:57:56.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:56.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:56.349 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 09:57:56.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:56.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:57:56.349 | INFO | Uploaded 36GiB to GPU in 1.60615s (22.4139GiB/s) | 20260506 09:57:56.349 | INFO | Downloaded 34GiB to GPU in 1.52347s (22.3174GiB/s) | 20260506 09:58:01.488 | INFO | Analyzing chunk frames = [456704,457216) | 20260506 09:58:01.488 | INFO | MemInfo: { | 20260506 09:58:01.488 | INFO | "gpu" : " 17 GB", | 20260506 09:58:01.488 | INFO | "huge" : "", | 20260506 09:58:01.488 | INFO | "huge_shared" : " 27 GB", | 20260506 09:58:01.488 | INFO | "malloc" : " 14 GB", | 20260506 09:58:01.488 | INFO | "pinned" : " 29 GB" | 20260506 09:58:01.488 | INFO | } | 20260506 09:58:01.488 | INFO | Perf report for the last 5.13932 seconds: | 20260506 09:58:01.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:58:01.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:01.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:01.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:01.488 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 09:58:01.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:01.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:01.488 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 09:58:01.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:01.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:01.489 | INFO | Uploaded 36GiB to GPU in 1.60518s (22.4274GiB/s) | 20260506 09:58:01.489 | INFO | Downloaded 34GiB to GPU in 1.52201s (22.3388GiB/s) | 20260506 09:58:06.627 | INFO | Analyzing chunk frames = [457216,457728) | 20260506 09:58:06.627 | INFO | MemInfo: { | 20260506 09:58:06.627 | INFO | "gpu" : " 17 GB", | 20260506 09:58:06.627 | INFO | "huge" : "", | 20260506 09:58:06.627 | INFO | "huge_shared" : " 27 GB", | 20260506 09:58:06.627 | INFO | "malloc" : " 14 GB", | 20260506 09:58:06.627 | INFO | "pinned" : " 29 GB" | 20260506 09:58:06.627 | INFO | } | 20260506 09:58:06.628 | INFO | Perf report for the last 5.13955 seconds: | 20260506 09:58:06.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:58:06.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:06.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:06.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:06.628 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 09:58:06.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:06.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:06.628 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 09:58:06.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:06.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:06.628 | INFO | Uploaded 36GiB to GPU in 1.60387s (22.4457GiB/s) | 20260506 09:58:06.628 | INFO | Downloaded 34GiB to GPU in 1.52205s (22.3383GiB/s) | 20260506 09:58:11.767 | INFO | Analyzing chunk frames = [457728,458240) | 20260506 09:58:11.767 | INFO | MemInfo: { | 20260506 09:58:11.767 | INFO | "gpu" : " 17 GB", | 20260506 09:58:11.767 | INFO | "huge" : "", | 20260506 09:58:11.767 | INFO | "huge_shared" : " 27 GB", | 20260506 09:58:11.767 | INFO | "malloc" : " 14 GB", | 20260506 09:58:11.767 | INFO | "pinned" : " 29 GB" | 20260506 09:58:11.767 | INFO | } | 20260506 09:58:11.768 | INFO | Perf report for the last 5.13997 seconds: | 20260506 09:58:11.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:58:11.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:11.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:11.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:11.768 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 09:58:11.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:11.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:11.768 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 09:58:11.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:11.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:11.768 | INFO | Uploaded 36GiB to GPU in 1.60832s (22.3837GiB/s) | 20260506 09:58:11.768 | INFO | Downloaded 34GiB to GPU in 1.52237s (22.3336GiB/s) | 20260506 09:58:16.917 | INFO | Analyzing chunk frames = [458240,458752) | 20260506 09:58:16.917 | INFO | MemInfo: { | 20260506 09:58:16.917 | INFO | "gpu" : " 17 GB", | 20260506 09:58:16.917 | INFO | "huge" : "", | 20260506 09:58:16.917 | INFO | "huge_shared" : " 27 GB", | 20260506 09:58:16.917 | INFO | "malloc" : " 14 GB", | 20260506 09:58:16.917 | INFO | "pinned" : " 29 GB" | 20260506 09:58:16.917 | INFO | } | 20260506 09:58:16.917 | INFO | Perf report for the last 5.14975 seconds: | 20260506 09:58:16.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:58:16.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:16.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:16.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:16.917 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260506 09:58:16.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:16.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:16.917 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 09:58:16.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:16.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:16.918 | INFO | Uploaded 36GiB to GPU in 1.60405s (22.4432GiB/s) | 20260506 09:58:16.918 | INFO | Downloaded 34GiB to GPU in 1.52297s (22.3248GiB/s) | 20260506 09:58:16.918 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:58:16.918 | INFO | Total mem usage: 15.9 GiB | 20260506 09:58:16.918 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 2.53%) | 20260506 09:58:16.918 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -3.06%) | 20260506 09:58:16.918 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:58:16.918 | INFO | High Water Breakdown: | 20260506 09:58:16.918 | INFO | 59.9% : 9.26 GiB ( 96.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:58:16.918 | INFO | 39.9% : 6.17 GiB ( 92.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:58:16.918 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:58:16.918 | INFO | ------------------------------------------ | 20260506 09:58:16.918 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:58:16.918 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:58:16.918 | INFO | Total mem usage: 26.5 GiB | 20260506 09:58:16.918 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:58:16.918 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:58:16.918 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:58:16.918 | INFO | High Water Breakdown: | 20260506 09:58:16.918 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:58:16.918 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:58:16.918 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:58:16.918 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:58:16.918 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:58:16.918 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:58:16.918 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:58:16.918 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:58:16.918 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:58:16.918 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:58:16.918 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:58:16.918 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:58:16.918 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:58:16.918 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:58:16.918 | INFO | ------------------------------------------ | 20260506 09:58:16.918 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:58:16.918 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:58:16.918 | INFO | Total mem usage: 25.6 GiB | 20260506 09:58:16.918 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:58:16.918 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:58:16.918 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:58:16.919 | INFO | High Water Breakdown: | 20260506 09:58:16.919 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:58:16.919 | INFO | ------------------------------------------ | 20260506 09:58:16.919 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:58:16.919 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:58:16.919 | INFO | Total mem usage: 15.7 GiB | 20260506 09:58:16.919 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:58:16.919 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:58:16.919 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:58:16.919 | INFO | High Water Breakdown: | 20260506 09:58:16.919 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:58:16.919 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:58:16.919 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:58:16.919 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:58:16.919 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:58:16.919 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:58:16.919 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:58:16.919 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:58:16.919 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:58:16.919 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:58:16.919 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:58:16.919 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:58:16.919 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:58:16.919 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:58:16.919 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:58:16.919 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:58:16.919 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:58:16.919 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:58:16.919 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:58:16.919 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:58:16.919 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:58:16.919 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:58:16.919 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:58:16.919 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:58:16.919 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:58:16.919 | INFO | ------------------------------------------ | 20260506 09:58:16.919 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:58:16.919 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:58:16.919 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:58:16.919 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:58:16.919 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260506 09:58:16.919 | INFO | PerfReport: 0.29%, AggregateStats -- 8.64e-02 / -nan, 5.08e-03 / 0.00e+00 | 20260506 09:58:16.919 | INFO | PerfReport: 10.93%, Baseline -- 3.29e+00 / -nan, 1.14e-01 / 0.00e+00 | 20260506 09:58:16.919 | INFO | PerfReport: 3.06%, Histogram -- 9.22e-01 / -nan, 5.13e-02 / 0.00e+00 | 20260506 09:58:16.919 | INFO | PerfReport: 16.04%, DME -- 3.87e+01 / -nan, 3.13e+00 / 0.00e+00 | 20260506 09:58:16.919 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.72e+01 / -nan, 4.18e-02 / 0.00e+00 | 20260506 09:58:16.919 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.25e+00 / -nan, 1.31e-01 / 0.00e+00 | 20260506 09:58:16.919 | INFO | PerfReport: 3.75%, Metrics -- 1.13e+00 / -nan, 1.75e-01 / 0.00e+00 | 20260506 09:58:16.919 | INFO | PerfReport: 1.46%, PulseSerializing -- 4.39e-01 / -nan, 6.29e-02 / 0.00e+00 | 20260506 09:58:16.919 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:58:16.919 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:58:16.919 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:58:16.919 | INFO | PerfReport: Full Execution -- 9.01e+01 / -nan, 1.85e+01 / 0.00e+00 | 20260506 09:58:16.919 | INFO | PerfReport: 18.65%, Upload -- 1.68e+01 / 0.00e+00, 4.82e-01 / 0.00e+00 | 20260506 09:58:16.919 | INFO | PerfReport: 17.90%, Download -- 1.61e+01 / 0.00e+00, 8.13e-01 / 0.00e+00 | 20260506 09:58:16.919 | INFO | PerfReport: 33.48%, Compute -- 3.02e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260506 09:58:16.919 | INFO | PerfReport: 0.57%, LockUpload -- 5.09e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 09:58:16.919 | INFO | PerfReport: 0.00%, LockDownload -- 4.89e-04 / 0.00e+00, 1.30e-04 / 0.00e+00 | 20260506 09:58:16.919 | INFO | PerfReport: 29.39%, LockCompute -- 2.65e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260506 09:58:16.919 | INFO | PerfReport: 0.01%, Misc -- 1.22e-02 / 0.00e+00, 1.03e-02 / 0.00e+00 | 20260506 09:58:17.015 | INFO | Starting a round of Baz IO | 20260506 09:58:19.933 | INFO | Populating RTMetrics report | 20260506 09:58:19.934 | INFO | RTMetrics report populated | 20260506 09:58:19.934 | INFO | Opening tmp RTMetrics json file | 20260506 09:58:19.938 | INFO | Serializing RTMetrics to json | 20260506 09:58:19.941 | INFO | Serialization done, swapping tmp file now... | 20260506 09:58:19.943 | INFO | RTMetrics done | 20260506 09:58:20.083 | INFO | Finished a round of Baz IO: Wrote 1.929 GiB in 3.068 seconds | 20260506 09:58:20.083 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.44% / 0.07% / 49.49% | 20260506 09:58:20.083 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.97017% / 0.000123183% / 3.17891e-05% | 20260506 09:58:20.083 | INFO | Baz IO overall: | 20260506 09:58:20.083 | INFO | BytesWritten : 195219128508 | 20260506 09:58:20.083 | INFO | NumEvents : 93374101296 | 20260506 09:58:22.045 | INFO | Analyzing chunk frames = [458752,459264) | 20260506 09:58:22.045 | INFO | MemInfo: { | 20260506 09:58:22.045 | INFO | "gpu" : " 17 GB", | 20260506 09:58:22.045 | INFO | "huge" : "", | 20260506 09:58:22.045 | INFO | "huge_shared" : " 27 GB", | 20260506 09:58:22.045 | INFO | "malloc" : " 14 GB", | 20260506 09:58:22.045 | INFO | "pinned" : " 29 GB" | 20260506 09:58:22.045 | INFO | } | 20260506 09:58:22.045 | INFO | Perf report for the last 5.12807 seconds: | 20260506 09:58:22.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:58:22.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:22.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:22.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:22.045 | INFO | ANALYSIS : 57.8%, 2.8, 84.3ms, 0 | 20260506 09:58:22.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:22.045 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260506 09:58:22.045 | INFO | PRE_HQ : 57.3%, 2.1, 62.5ms, 0 | 20260506 09:58:22.045 | INFO | BAZWRITER : 59.4%, 4.8, 145.2ms, 0 | 20260506 09:58:22.045 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 09:58:22.046 | INFO | Uploaded 36GiB to GPU in 1.66336s (21.6429GiB/s) | 20260506 09:58:22.046 | INFO | Downloaded 38GiB to GPU in 1.72798s (21.991GiB/s) | 20260506 09:58:27.185 | INFO | Analyzing chunk frames = [459264,459776) | 20260506 09:58:27.185 | INFO | MemInfo: { | 20260506 09:58:27.185 | INFO | "gpu" : " 17 GB", | 20260506 09:58:27.185 | INFO | "huge" : "", | 20260506 09:58:27.185 | INFO | "huge_shared" : " 27 GB", | 20260506 09:58:27.185 | INFO | "malloc" : " 14 GB", | 20260506 09:58:27.185 | INFO | "pinned" : " 29 GB" | 20260506 09:58:27.185 | INFO | } | 20260506 09:58:27.185 | INFO | Perf report for the last 5.13943 seconds: | 20260506 09:58:27.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:58:27.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:27.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:27.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:27.185 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 09:58:27.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:27.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:27.185 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 09:58:27.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:27.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:27.186 | INFO | Uploaded 36GiB to GPU in 1.60189s (22.4734GiB/s) | 20260506 09:58:27.186 | INFO | Downloaded 34GiB to GPU in 1.52018s (22.3658GiB/s) | 20260506 09:58:32.324 | INFO | Analyzing chunk frames = [459776,460288) | 20260506 09:58:32.324 | INFO | MemInfo: { | 20260506 09:58:32.324 | INFO | "gpu" : " 17 GB", | 20260506 09:58:32.324 | INFO | "huge" : "", | 20260506 09:58:32.324 | INFO | "huge_shared" : " 27 GB", | 20260506 09:58:32.324 | INFO | "malloc" : " 14 GB", | 20260506 09:58:32.324 | INFO | "pinned" : " 29 GB" | 20260506 09:58:32.324 | INFO | } | 20260506 09:58:32.324 | INFO | Perf report for the last 5.13944 seconds: | 20260506 09:58:32.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:58:32.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:32.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:32.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:32.324 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260506 09:58:32.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:32.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:32.324 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 09:58:32.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:32.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:32.325 | INFO | Uploaded 36GiB to GPU in 1.60593s (22.4169GiB/s) | 20260506 09:58:32.325 | INFO | Downloaded 34GiB to GPU in 1.52537s (22.2897GiB/s) | 20260506 09:58:37.463 | INFO | Analyzing chunk frames = [460288,460800) | 20260506 09:58:37.463 | INFO | MemInfo: { | 20260506 09:58:37.463 | INFO | "gpu" : " 17 GB", | 20260506 09:58:37.463 | INFO | "huge" : "", | 20260506 09:58:37.464 | INFO | "huge_shared" : " 27 GB", | 20260506 09:58:37.464 | INFO | "malloc" : " 14 GB", | 20260506 09:58:37.464 | INFO | "pinned" : " 29 GB" | 20260506 09:58:37.464 | INFO | } | 20260506 09:58:37.464 | INFO | Perf report for the last 5.13941 seconds: | 20260506 09:58:37.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:58:37.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:37.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:37.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:37.464 | INFO | ANALYSIS : 56.5%, 3.0, 87.7ms, 0 | 20260506 09:58:37.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:37.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:37.464 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:58:37.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:37.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:37.464 | INFO | Uploaded 36GiB to GPU in 1.60577s (22.4191GiB/s) | 20260506 09:58:37.464 | INFO | Downloaded 34GiB to GPU in 1.52353s (22.3166GiB/s) | 20260506 09:58:42.603 | INFO | Analyzing chunk frames = [460800,461312) | 20260506 09:58:42.603 | INFO | MemInfo: { | 20260506 09:58:42.603 | INFO | "gpu" : " 17 GB", | 20260506 09:58:42.603 | INFO | "huge" : "", | 20260506 09:58:42.603 | INFO | "huge_shared" : " 27 GB", | 20260506 09:58:42.603 | INFO | "malloc" : " 14 GB", | 20260506 09:58:42.603 | INFO | "pinned" : " 29 GB" | 20260506 09:58:42.603 | INFO | } | 20260506 09:58:42.603 | INFO | Perf report for the last 5.13919 seconds: | 20260506 09:58:42.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:58:42.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:42.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:42.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:42.603 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 09:58:42.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:42.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:42.603 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 09:58:42.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:42.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:42.604 | INFO | Uploaded 36GiB to GPU in 1.60362s (22.4492GiB/s) | 20260506 09:58:42.604 | INFO | Downloaded 34GiB to GPU in 1.5204s (22.3626GiB/s) | 20260506 09:58:47.742 | INFO | Analyzing chunk frames = [461312,461824) | 20260506 09:58:47.742 | INFO | MemInfo: { | 20260506 09:58:47.742 | INFO | "gpu" : " 17 GB", | 20260506 09:58:47.742 | INFO | "huge" : "", | 20260506 09:58:47.742 | INFO | "huge_shared" : " 27 GB", | 20260506 09:58:47.742 | INFO | "malloc" : " 14 GB", | 20260506 09:58:47.742 | INFO | "pinned" : " 29 GB" | 20260506 09:58:47.742 | INFO | } | 20260506 09:58:47.742 | INFO | Perf report for the last 5.1394 seconds: | 20260506 09:58:47.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:58:47.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:47.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:47.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:47.742 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 09:58:47.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:47.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:47.742 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 09:58:47.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:47.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:47.743 | INFO | Uploaded 36GiB to GPU in 1.60436s (22.4389GiB/s) | 20260506 09:58:47.743 | INFO | Downloaded 34GiB to GPU in 1.52251s (22.3316GiB/s) | 20260506 09:58:52.881 | INFO | Analyzing chunk frames = [461824,462336) | 20260506 09:58:52.881 | INFO | MemInfo: { | 20260506 09:58:52.881 | INFO | "gpu" : " 17 GB", | 20260506 09:58:52.881 | INFO | "huge" : "", | 20260506 09:58:52.881 | INFO | "huge_shared" : " 27 GB", | 20260506 09:58:52.881 | INFO | "malloc" : " 14 GB", | 20260506 09:58:52.882 | INFO | "pinned" : " 29 GB" | 20260506 09:58:52.882 | INFO | } | 20260506 09:58:52.882 | INFO | Perf report for the last 5.13927 seconds: | 20260506 09:58:52.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:58:52.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:52.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:52.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:52.882 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:58:52.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:52.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:52.882 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 09:58:52.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:52.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:52.882 | INFO | Uploaded 36GiB to GPU in 1.60381s (22.4465GiB/s) | 20260506 09:58:52.882 | INFO | Downloaded 34GiB to GPU in 1.52272s (22.3285GiB/s) | 20260506 09:58:58.021 | INFO | Analyzing chunk frames = [462336,462848) | 20260506 09:58:58.021 | INFO | MemInfo: { | 20260506 09:58:58.021 | INFO | "gpu" : " 17 GB", | 20260506 09:58:58.021 | INFO | "huge" : "", | 20260506 09:58:58.021 | INFO | "huge_shared" : " 27 GB", | 20260506 09:58:58.021 | INFO | "malloc" : " 14 GB", | 20260506 09:58:58.021 | INFO | "pinned" : " 29 GB" | 20260506 09:58:58.021 | INFO | } | 20260506 09:58:58.021 | INFO | Perf report for the last 5.13933 seconds: | 20260506 09:58:58.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:58:58.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:58.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:58.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:58.021 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:58:58.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:58.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:58.021 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 09:58:58.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:58.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:58:58.021 | INFO | Uploaded 36GiB to GPU in 1.60517s (22.4275GiB/s) | 20260506 09:58:58.021 | INFO | Downloaded 34GiB to GPU in 1.52343s (22.3181GiB/s) | 20260506 09:58:58.118 | INFO | Starting a round of Baz IO | 20260506 09:59:01.037 | INFO | Populating RTMetrics report | 20260506 09:59:01.037 | INFO | RTMetrics report populated | 20260506 09:59:01.037 | INFO | Opening tmp RTMetrics json file | 20260506 09:59:01.083 | INFO | Serializing RTMetrics to json | 20260506 09:59:01.117 | INFO | Serialization done, swapping tmp file now... | 20260506 09:59:01.119 | INFO | RTMetrics done | 20260506 09:59:01.472 | INFO | Finished a round of Baz IO: Wrote 1.933 GiB in 3.354 seconds | 20260506 09:59:01.473 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.37% / 0.07% / 49.56% | 20260506 09:59:01.473 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.96829% / 0.000139078% / 1.98682e-05% | 20260506 09:59:01.473 | INFO | Baz IO overall: | 20260506 09:59:01.473 | INFO | BytesWritten : 197294825065 | 20260506 09:59:01.473 | INFO | NumEvents : 94381885156 | 20260506 09:59:03.160 | INFO | Analyzing chunk frames = [462848,463360) | 20260506 09:59:03.160 | INFO | MemInfo: { | 20260506 09:59:03.160 | INFO | "gpu" : " 17 GB", | 20260506 09:59:03.160 | INFO | "huge" : "", | 20260506 09:59:03.160 | INFO | "huge_shared" : " 27 GB", | 20260506 09:59:03.160 | INFO | "malloc" : " 14 GB", | 20260506 09:59:03.160 | INFO | "pinned" : " 29 GB" | 20260506 09:59:03.160 | INFO | } | 20260506 09:59:03.160 | INFO | Perf report for the last 5.13928 seconds: | 20260506 09:59:03.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:59:03.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:03.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:03.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:03.160 | INFO | ANALYSIS : 57.7%, 2.9, 87.9ms, 0 | 20260506 09:59:03.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:03.160 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260506 09:59:03.160 | INFO | PRE_HQ : 56.7%, 2.1, 61.4ms, 0 | 20260506 09:59:03.160 | INFO | BAZWRITER : 64.9%, 4.5, 150.6ms, 0 | 20260506 09:59:03.160 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 09:59:03.161 | INFO | Uploaded 36GiB to GPU in 1.65447s (21.7592GiB/s) | 20260506 09:59:03.161 | INFO | Downloaded 38GiB to GPU in 1.71837s (22.114GiB/s) | 20260506 09:59:08.299 | INFO | Analyzing chunk frames = [463360,463872) | 20260506 09:59:08.300 | INFO | MemInfo: { | 20260506 09:59:08.300 | INFO | "gpu" : " 17 GB", | 20260506 09:59:08.300 | INFO | "huge" : "", | 20260506 09:59:08.300 | INFO | "huge_shared" : " 27 GB", | 20260506 09:59:08.300 | INFO | "malloc" : " 14 GB", | 20260506 09:59:08.300 | INFO | "pinned" : " 29 GB" | 20260506 09:59:08.300 | INFO | } | 20260506 09:59:08.300 | INFO | Perf report for the last 5.13952 seconds: | 20260506 09:59:08.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:59:08.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:08.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:08.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:08.300 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 09:59:08.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:08.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:08.300 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 09:59:08.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:08.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:08.300 | INFO | Uploaded 36GiB to GPU in 1.60303s (22.4574GiB/s) | 20260506 09:59:08.300 | INFO | Downloaded 34GiB to GPU in 1.51956s (22.375GiB/s) | 20260506 09:59:13.439 | INFO | Analyzing chunk frames = [463872,464384) | 20260506 09:59:13.439 | INFO | MemInfo: { | 20260506 09:59:13.439 | INFO | "gpu" : " 17 GB", | 20260506 09:59:13.439 | INFO | "huge" : "", | 20260506 09:59:13.439 | INFO | "huge_shared" : " 27 GB", | 20260506 09:59:13.439 | INFO | "malloc" : " 14 GB", | 20260506 09:59:13.439 | INFO | "pinned" : " 29 GB" | 20260506 09:59:13.439 | INFO | } | 20260506 09:59:13.439 | INFO | Perf report for the last 5.13909 seconds: | 20260506 09:59:13.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:59:13.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:13.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:13.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:13.439 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:59:13.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:13.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:13.439 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 09:59:13.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:13.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:13.439 | INFO | Uploaded 36GiB to GPU in 1.60655s (22.4083GiB/s) | 20260506 09:59:13.440 | INFO | Downloaded 34GiB to GPU in 1.52399s (22.3098GiB/s) | 20260506 09:59:18.578 | INFO | Analyzing chunk frames = [464384,464896) | 20260506 09:59:18.578 | INFO | MemInfo: { | 20260506 09:59:18.578 | INFO | "gpu" : " 17 GB", | 20260506 09:59:18.578 | INFO | "huge" : "", | 20260506 09:59:18.578 | INFO | "huge_shared" : " 27 GB", | 20260506 09:59:18.578 | INFO | "malloc" : " 14 GB", | 20260506 09:59:18.578 | INFO | "pinned" : " 29 GB" | 20260506 09:59:18.578 | INFO | } | 20260506 09:59:18.578 | INFO | Perf report for the last 5.1394 seconds: | 20260506 09:59:18.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:59:18.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:18.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:18.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:18.578 | INFO | ANALYSIS : 56.5%, 3.0, 87.7ms, 0 | 20260506 09:59:18.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:18.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:18.578 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 09:59:18.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:18.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:18.579 | INFO | Uploaded 36GiB to GPU in 1.60739s (22.3966GiB/s) | 20260506 09:59:18.579 | INFO | Downloaded 34GiB to GPU in 1.52479s (22.2981GiB/s) | 20260506 09:59:23.717 | INFO | Analyzing chunk frames = [464896,465408) | 20260506 09:59:23.717 | INFO | MemInfo: { | 20260506 09:59:23.717 | INFO | "gpu" : " 17 GB", | 20260506 09:59:23.717 | INFO | "huge" : "", | 20260506 09:59:23.718 | INFO | "huge_shared" : " 27 GB", | 20260506 09:59:23.718 | INFO | "malloc" : " 14 GB", | 20260506 09:59:23.718 | INFO | "pinned" : " 29 GB" | 20260506 09:59:23.718 | INFO | } | 20260506 09:59:23.718 | INFO | Perf report for the last 5.1394 seconds: | 20260506 09:59:23.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:59:23.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:23.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:23.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:23.718 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 09:59:23.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:23.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:23.718 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:59:23.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:23.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:23.718 | INFO | Uploaded 36GiB to GPU in 1.60374s (22.4475GiB/s) | 20260506 09:59:23.718 | INFO | Downloaded 34GiB to GPU in 1.52074s (22.3575GiB/s) | 20260506 09:59:28.857 | INFO | Analyzing chunk frames = [465408,465920) | 20260506 09:59:28.857 | INFO | MemInfo: { | 20260506 09:59:28.857 | INFO | "gpu" : " 17 GB", | 20260506 09:59:28.857 | INFO | "huge" : "", | 20260506 09:59:28.857 | INFO | "huge_shared" : " 27 GB", | 20260506 09:59:28.857 | INFO | "malloc" : " 14 GB", | 20260506 09:59:28.857 | INFO | "pinned" : " 29 GB" | 20260506 09:59:28.857 | INFO | } | 20260506 09:59:28.857 | INFO | Perf report for the last 5.13922 seconds: | 20260506 09:59:28.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:59:28.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:28.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:28.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:28.857 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 09:59:28.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:28.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:28.857 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 09:59:28.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:28.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:28.857 | INFO | Uploaded 36GiB to GPU in 1.60459s (22.4357GiB/s) | 20260506 09:59:28.857 | INFO | Downloaded 34GiB to GPU in 1.52249s (22.3318GiB/s) | 20260506 09:59:33.996 | INFO | Analyzing chunk frames = [465920,466432) | 20260506 09:59:33.996 | INFO | MemInfo: { | 20260506 09:59:33.996 | INFO | "gpu" : " 17 GB", | 20260506 09:59:33.996 | INFO | "huge" : "", | 20260506 09:59:33.996 | INFO | "huge_shared" : " 27 GB", | 20260506 09:59:33.996 | INFO | "malloc" : " 14 GB", | 20260506 09:59:33.996 | INFO | "pinned" : " 29 GB" | 20260506 09:59:33.996 | INFO | } | 20260506 09:59:33.996 | INFO | Perf report for the last 5.13935 seconds: | 20260506 09:59:33.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:59:33.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:33.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:33.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:33.996 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 09:59:33.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:33.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:33.996 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 09:59:33.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:33.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:33.997 | INFO | Uploaded 36GiB to GPU in 1.60542s (22.4241GiB/s) | 20260506 09:59:33.997 | INFO | Downloaded 34GiB to GPU in 1.52249s (22.3319GiB/s) | 20260506 09:59:39.136 | INFO | Analyzing chunk frames = [466432,466944) | 20260506 09:59:39.136 | INFO | MemInfo: { | 20260506 09:59:39.136 | INFO | "gpu" : " 17 GB", | 20260506 09:59:39.136 | INFO | "huge" : "", | 20260506 09:59:39.136 | INFO | "huge_shared" : " 27 GB", | 20260506 09:59:39.136 | INFO | "malloc" : " 14 GB", | 20260506 09:59:39.136 | INFO | "pinned" : " 29 GB" | 20260506 09:59:39.136 | INFO | } | 20260506 09:59:39.136 | INFO | Perf report for the last 5.13947 seconds: | 20260506 09:59:39.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:59:39.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:39.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:39.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:39.136 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 09:59:39.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:39.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:39.136 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 09:59:39.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:39.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:39.136 | INFO | Uploaded 36GiB to GPU in 1.60526s (22.4262GiB/s) | 20260506 09:59:39.136 | INFO | Downloaded 34GiB to GPU in 1.52386s (22.3118GiB/s) | 20260506 09:59:39.136 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 09:59:39.136 | INFO | Total mem usage: 15.9 GiB | 20260506 09:59:39.136 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 2.53%) | 20260506 09:59:39.136 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -2.35%) | 20260506 09:59:39.136 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 09:59:39.136 | INFO | High Water Breakdown: | 20260506 09:59:39.136 | INFO | 60.1% : 9.35 GiB ( 97.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 09:59:39.136 | INFO | 39.8% : 6.19 GiB ( 92.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 09:59:39.136 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 09:59:39.136 | INFO | ------------------------------------------ | 20260506 09:59:39.136 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:59:39.136 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 09:59:39.136 | INFO | Total mem usage: 26.5 GiB | 20260506 09:59:39.136 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 09:59:39.136 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 09:59:39.136 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 09:59:39.136 | INFO | High Water Breakdown: | 20260506 09:59:39.136 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:59:39.136 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:59:39.136 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:59:39.136 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:59:39.136 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:59:39.136 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:59:39.136 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 09:59:39.136 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 09:59:39.136 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:59:39.136 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 09:59:39.136 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 09:59:39.136 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 09:59:39.136 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 09:59:39.136 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 09:59:39.136 | INFO | ------------------------------------------ | 20260506 09:59:39.136 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:59:39.136 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 09:59:39.136 | INFO | Total mem usage: 25.6 GiB | 20260506 09:59:39.136 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 09:59:39.136 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 09:59:39.136 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 09:59:39.136 | INFO | High Water Breakdown: | 20260506 09:59:39.136 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 09:59:39.136 | INFO | ------------------------------------------ | 20260506 09:59:39.136 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:59:39.136 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 09:59:39.136 | INFO | Total mem usage: 15.7 GiB | 20260506 09:59:39.136 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 09:59:39.136 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 09:59:39.136 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 09:59:39.136 | INFO | High Water Breakdown: | 20260506 09:59:39.136 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 09:59:39.136 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 09:59:39.136 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 09:59:39.136 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 09:59:39.136 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 09:59:39.136 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 09:59:39.136 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 09:59:39.136 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 09:59:39.136 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 09:59:39.136 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 09:59:39.136 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 09:59:39.136 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 09:59:39.136 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 09:59:39.136 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 09:59:39.136 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 09:59:39.136 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 09:59:39.136 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 09:59:39.136 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 09:59:39.136 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 09:59:39.136 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 09:59:39.136 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 09:59:39.136 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 09:59:39.136 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 09:59:39.136 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 09:59:39.136 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 09:59:39.136 | INFO | ------------------------------------------ | 20260506 09:59:39.136 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 09:59:39.136 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 09:59:39.136 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:59:39.136 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:59:39.136 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 09:59:39.136 | INFO | PerfReport: 0.29%, AggregateStats -- 8.61e-02 / -nan, 4.23e-03 / 0.00e+00 | 20260506 09:59:39.136 | INFO | PerfReport: 10.92%, Baseline -- 3.29e+00 / -nan, 1.79e-02 / 0.00e+00 | 20260506 09:59:39.136 | INFO | PerfReport: 3.06%, Histogram -- 9.22e-01 / -nan, 5.18e-02 / 0.00e+00 | 20260506 09:59:39.136 | INFO | PerfReport: 16.05%, DME -- 3.87e+01 / -nan, 3.16e+00 / 0.00e+00 | 20260506 09:59:39.136 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.72e+01 / -nan, 4.24e-02 / 0.00e+00 | 20260506 09:59:39.136 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.25e+00 / -nan, 1.33e-01 / 0.00e+00 | 20260506 09:59:39.136 | INFO | PerfReport: 3.75%, Metrics -- 1.13e+00 / -nan, 1.76e-01 / 0.00e+00 | 20260506 09:59:39.136 | INFO | PerfReport: 1.45%, PulseSerializing -- 4.38e-01 / -nan, 6.32e-02 / 0.00e+00 | 20260506 09:59:39.136 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 09:59:39.136 | INFO | PerfReport: Gathered 1536 samples. | 20260506 09:59:39.136 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 09:59:39.136 | INFO | PerfReport: Full Execution -- 9.01e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 09:59:39.136 | INFO | PerfReport: 18.63%, Upload -- 1.68e+01 / 0.00e+00, 4.50e-01 / 0.00e+00 | 20260506 09:59:39.136 | INFO | PerfReport: 17.89%, Download -- 1.61e+01 / 0.00e+00, 7.83e-01 / 0.00e+00 | 20260506 09:59:39.136 | INFO | PerfReport: 33.47%, Compute -- 3.02e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 09:59:39.136 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 09:59:39.136 | INFO | PerfReport: 0.00%, LockDownload -- 4.93e-04 / 0.00e+00, 2.57e-04 / 0.00e+00 | 20260506 09:59:39.136 | INFO | PerfReport: 29.42%, LockCompute -- 2.65e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260506 09:59:39.136 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 3.41e-02 / 0.00e+00 | 20260506 09:59:39.233 | INFO | Starting a round of Baz IO | 20260506 09:59:42.151 | INFO | Populating RTMetrics report | 20260506 09:59:42.152 | INFO | RTMetrics report populated | 20260506 09:59:42.152 | INFO | Opening tmp RTMetrics json file | 20260506 09:59:42.180 | INFO | Serializing RTMetrics to json | 20260506 09:59:42.181 | INFO | Serialization done, swapping tmp file now... | 20260506 09:59:42.182 | INFO | RTMetrics done | 20260506 09:59:42.439 | INFO | Finished a round of Baz IO: Wrote 1.937 GiB in 3.206 seconds | 20260506 09:59:42.439 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.30% / 0.07% / 49.63% | 20260506 09:59:42.439 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.94942% / 0.000154972% / 1.58946e-05% | 20260506 09:59:42.439 | INFO | Baz IO overall: | 20260506 09:59:42.439 | INFO | BytesWritten : 199375066008 | 20260506 09:59:42.439 | INFO | NumEvents : 95391887984 | 20260506 09:59:44.275 | INFO | Analyzing chunk frames = [466944,467456) | 20260506 09:59:44.275 | INFO | MemInfo: { | 20260506 09:59:44.275 | INFO | "gpu" : " 17 GB", | 20260506 09:59:44.275 | INFO | "huge" : "", | 20260506 09:59:44.275 | INFO | "huge_shared" : " 27 GB", | 20260506 09:59:44.275 | INFO | "malloc" : " 14 GB", | 20260506 09:59:44.275 | INFO | "pinned" : " 29 GB" | 20260506 09:59:44.275 | INFO | } | 20260506 09:59:44.275 | INFO | Perf report for the last 5.1392 seconds: | 20260506 09:59:44.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:59:44.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:44.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:44.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:44.275 | INFO | ANALYSIS : 57.7%, 2.8, 85.7ms, 0 | 20260506 09:59:44.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:44.275 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 09:59:44.275 | INFO | PRE_HQ : 56.8%, 2.1, 60.7ms, 0 | 20260506 09:59:44.275 | INFO | BAZWRITER : 62.0%, 4.8, 151.9ms, 0 | 20260506 09:59:44.275 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 09:59:44.275 | INFO | Uploaded 36GiB to GPU in 1.67459s (21.4978GiB/s) | 20260506 09:59:44.276 | INFO | Downloaded 38GiB to GPU in 1.73912s (21.8501GiB/s) | 20260506 09:59:49.416 | INFO | Analyzing chunk frames = [467456,467968) | 20260506 09:59:49.416 | INFO | MemInfo: { | 20260506 09:59:49.416 | INFO | "gpu" : " 17 GB", | 20260506 09:59:49.416 | INFO | "huge" : "", | 20260506 09:59:49.416 | INFO | "huge_shared" : " 27 GB", | 20260506 09:59:49.416 | INFO | "malloc" : " 14 GB", | 20260506 09:59:49.416 | INFO | "pinned" : " 29 GB" | 20260506 09:59:49.416 | INFO | } | 20260506 09:59:49.416 | INFO | Perf report for the last 5.14076 seconds: | 20260506 09:59:49.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:59:49.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:49.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:49.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:49.416 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 09:59:49.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:49.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:49.416 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 09:59:49.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:49.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:49.416 | INFO | Uploaded 36GiB to GPU in 1.60293s (22.4588GiB/s) | 20260506 09:59:49.416 | INFO | Downloaded 34GiB to GPU in 1.51937s (22.3777GiB/s) | 20260506 09:59:54.557 | INFO | Analyzing chunk frames = [467968,468480) | 20260506 09:59:54.557 | INFO | MemInfo: { | 20260506 09:59:54.557 | INFO | "gpu" : " 17 GB", | 20260506 09:59:54.557 | INFO | "huge" : "", | 20260506 09:59:54.557 | INFO | "huge_shared" : " 27 GB", | 20260506 09:59:54.557 | INFO | "malloc" : " 14 GB", | 20260506 09:59:54.557 | INFO | "pinned" : " 29 GB" | 20260506 09:59:54.557 | INFO | } | 20260506 09:59:54.557 | INFO | Perf report for the last 5.14148 seconds: | 20260506 09:59:54.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:59:54.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:54.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:54.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:54.557 | INFO | ANALYSIS : 56.9%, 3.0, 88.4ms, 0 | 20260506 09:59:54.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:54.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:54.557 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 09:59:54.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:54.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:54.557 | INFO | Uploaded 36GiB to GPU in 1.60617s (22.4136GiB/s) | 20260506 09:59:54.557 | INFO | Downloaded 34GiB to GPU in 1.52398s (22.3099GiB/s) | 20260506 09:59:59.693 | INFO | Analyzing chunk frames = [468480,468992) | 20260506 09:59:59.693 | INFO | MemInfo: { | 20260506 09:59:59.693 | INFO | "gpu" : " 17 GB", | 20260506 09:59:59.693 | INFO | "huge" : "", | 20260506 09:59:59.693 | INFO | "huge_shared" : " 27 GB", | 20260506 09:59:59.693 | INFO | "malloc" : " 14 GB", | 20260506 09:59:59.693 | INFO | "pinned" : " 29 GB" | 20260506 09:59:59.693 | INFO | } | 20260506 09:59:59.693 | INFO | Perf report for the last 5.13586 seconds: | 20260506 09:59:59.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 09:59:59.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:59.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:59.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:59.693 | INFO | ANALYSIS : 56.6%, 3.0, 87.7ms, 0 | 20260506 09:59:59.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:59.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:59.693 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 09:59:59.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:59.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 09:59:59.694 | INFO | Uploaded 36GiB to GPU in 1.60577s (22.4191GiB/s) | 20260506 09:59:59.694 | INFO | Downloaded 34GiB to GPU in 1.52357s (22.316GiB/s) | 20260506 10:00:04.836 | INFO | Analyzing chunk frames = [468992,469504) | 20260506 10:00:04.836 | INFO | MemInfo: { | 20260506 10:00:04.836 | INFO | "gpu" : " 17 GB", | 20260506 10:00:04.836 | INFO | "huge" : "", | 20260506 10:00:04.836 | INFO | "huge_shared" : " 27 GB", | 20260506 10:00:04.836 | INFO | "malloc" : " 14 GB", | 20260506 10:00:04.836 | INFO | "pinned" : " 29 GB" | 20260506 10:00:04.836 | INFO | } | 20260506 10:00:04.836 | INFO | Perf report for the last 5.14285 seconds: | 20260506 10:00:04.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:00:04.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:04.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:04.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:04.836 | INFO | ANALYSIS : 56.4%, 3.0, 88.1ms, 0 | 20260506 10:00:04.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:04.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:04.836 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:00:04.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:04.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:04.836 | INFO | Uploaded 36GiB to GPU in 1.60454s (22.4363GiB/s) | 20260506 10:00:04.836 | INFO | Downloaded 34GiB to GPU in 1.5216s (22.3448GiB/s) | 20260506 10:00:09.972 | INFO | Analyzing chunk frames = [469504,470016) | 20260506 10:00:09.972 | INFO | MemInfo: { | 20260506 10:00:09.972 | INFO | "gpu" : " 17 GB", | 20260506 10:00:09.972 | INFO | "huge" : "", | 20260506 10:00:09.972 | INFO | "huge_shared" : " 27 GB", | 20260506 10:00:09.972 | INFO | "malloc" : " 14 GB", | 20260506 10:00:09.975 | INFO | "pinned" : " 29 GB" | 20260506 10:00:09.975 | INFO | } | 20260506 10:00:09.975 | INFO | Perf report for the last 5.13913 seconds: | 20260506 10:00:09.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:00:09.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:09.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:09.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:09.975 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 10:00:09.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:09.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:09.975 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:00:09.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:09.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:09.975 | INFO | Uploaded 36GiB to GPU in 1.60433s (22.4393GiB/s) | 20260506 10:00:09.976 | INFO | Downloaded 34GiB to GPU in 1.5217s (22.3434GiB/s) | 20260506 10:00:15.111 | INFO | Analyzing chunk frames = [470016,470528) | 20260506 10:00:15.111 | INFO | MemInfo: { | 20260506 10:00:15.111 | INFO | "gpu" : " 17 GB", | 20260506 10:00:15.111 | INFO | "huge" : "", | 20260506 10:00:15.111 | INFO | "huge_shared" : " 27 GB", | 20260506 10:00:15.111 | INFO | "malloc" : " 14 GB", | 20260506 10:00:15.111 | INFO | "pinned" : " 29 GB" | 20260506 10:00:15.111 | INFO | } | 20260506 10:00:15.111 | INFO | Perf report for the last 5.13602 seconds: | 20260506 10:00:15.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:00:15.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:15.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:15.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:15.111 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:00:15.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:15.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:15.111 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:00:15.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:15.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:15.111 | INFO | Uploaded 36GiB to GPU in 1.60529s (22.4259GiB/s) | 20260506 10:00:15.112 | INFO | Downloaded 34GiB to GPU in 1.5226s (22.3302GiB/s) | 20260506 10:00:20.250 | INFO | Analyzing chunk frames = [470528,471040) | 20260506 10:00:20.250 | INFO | MemInfo: { | 20260506 10:00:20.250 | INFO | "gpu" : " 17 GB", | 20260506 10:00:20.250 | INFO | "huge" : "", | 20260506 10:00:20.250 | INFO | "huge_shared" : " 27 GB", | 20260506 10:00:20.250 | INFO | "malloc" : " 14 GB", | 20260506 10:00:20.250 | INFO | "pinned" : " 29 GB" | 20260506 10:00:20.250 | INFO | } | 20260506 10:00:20.251 | INFO | Perf report for the last 5.13939 seconds: | 20260506 10:00:20.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:00:20.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:20.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:20.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:20.251 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 10:00:20.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:20.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:20.251 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:00:20.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:20.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:20.251 | INFO | Uploaded 36GiB to GPU in 1.60522s (22.4268GiB/s) | 20260506 10:00:20.251 | INFO | Downloaded 34GiB to GPU in 1.5229s (22.3258GiB/s) | 20260506 10:00:20.349 | INFO | Starting a round of Baz IO | 20260506 10:00:23.265 | INFO | Populating RTMetrics report | 20260506 10:00:23.265 | INFO | RTMetrics report populated | 20260506 10:00:23.265 | INFO | Opening tmp RTMetrics json file | 20260506 10:00:23.345 | INFO | Serializing RTMetrics to json | 20260506 10:00:23.459 | INFO | Serialization done, swapping tmp file now... | 20260506 10:00:23.486 | INFO | RTMetrics done | 20260506 10:00:23.761 | INFO | Finished a round of Baz IO: Wrote 1.939 GiB in 3.412 seconds | 20260506 10:00:23.761 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.23% / 0.07% / 49.70% | 20260506 10:00:23.761 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.929% / 0.000182788% / 3.17891e-05% | 20260506 10:00:23.761 | INFO | Baz IO overall: | 20260506 10:00:23.761 | INFO | BytesWritten : 201456916059 | 20260506 10:00:23.761 | INFO | NumEvents : 96402830062 | 20260506 10:00:25.390 | INFO | Analyzing chunk frames = [471040,471552) | 20260506 10:00:25.390 | INFO | MemInfo: { | 20260506 10:00:25.390 | INFO | "gpu" : " 17 GB", | 20260506 10:00:25.390 | INFO | "huge" : "", | 20260506 10:00:25.390 | INFO | "huge_shared" : " 27 GB", | 20260506 10:00:25.390 | INFO | "malloc" : " 14 GB", | 20260506 10:00:25.390 | INFO | "pinned" : " 29 GB" | 20260506 10:00:25.390 | INFO | } | 20260506 10:00:25.390 | INFO | Perf report for the last 5.13917 seconds: | 20260506 10:00:25.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:00:25.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:25.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:25.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:25.390 | INFO | ANALYSIS : 57.6%, 2.8, 85.4ms, 0 | 20260506 10:00:25.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:25.390 | INFO | RT_METRICS : 48.3%, 1.0, 25.4ms, 0 | 20260506 10:00:25.390 | INFO | PRE_HQ : 56.8%, 2.1, 61.0ms, 0 | 20260506 10:00:25.390 | INFO | BAZWRITER : 66.0%, 4.8, 162.0ms, 0 | 20260506 10:00:25.390 | INFO | BAZ_SAMPLING : 4.0%, 1.0, 2.1ms, 0 | 20260506 10:00:25.390 | INFO | Uploaded 36GiB to GPU in 1.6751s (21.4913GiB/s) | 20260506 10:00:25.390 | INFO | Downloaded 38GiB to GPU in 1.74042s (21.8339GiB/s) | 20260506 10:00:30.531 | INFO | Analyzing chunk frames = [471552,472064) | 20260506 10:00:30.531 | INFO | MemInfo: { | 20260506 10:00:30.531 | INFO | "gpu" : " 17 GB", | 20260506 10:00:30.531 | INFO | "huge" : "", | 20260506 10:00:30.531 | INFO | "huge_shared" : " 27 GB", | 20260506 10:00:30.531 | INFO | "malloc" : " 14 GB", | 20260506 10:00:30.531 | INFO | "pinned" : " 29 GB" | 20260506 10:00:30.531 | INFO | } | 20260506 10:00:30.531 | INFO | Perf report for the last 5.1412 seconds: | 20260506 10:00:30.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:00:30.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:30.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:30.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:30.531 | INFO | ANALYSIS : 57.6%, 3.0, 89.8ms, 0 | 20260506 10:00:30.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:30.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:30.531 | INFO | PRE_HQ : 30.0%, 1.0, 15.8ms, 0 | 20260506 10:00:30.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:30.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:30.531 | INFO | Uploaded 36GiB to GPU in 1.60295s (22.4586GiB/s) | 20260506 10:00:30.531 | INFO | Downloaded 34GiB to GPU in 1.51868s (22.3878GiB/s) | 20260506 10:00:35.668 | INFO | Analyzing chunk frames = [472064,472576) | 20260506 10:00:35.668 | INFO | MemInfo: { | 20260506 10:00:35.668 | INFO | "gpu" : " 17 GB", | 20260506 10:00:35.668 | INFO | "huge" : "", | 20260506 10:00:35.668 | INFO | "huge_shared" : " 27 GB", | 20260506 10:00:35.668 | INFO | "malloc" : " 14 GB", | 20260506 10:00:35.668 | INFO | "pinned" : " 29 GB" | 20260506 10:00:35.669 | INFO | } | 20260506 10:00:35.669 | INFO | Perf report for the last 5.1376 seconds: | 20260506 10:00:35.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:00:35.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:35.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:35.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:35.669 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260506 10:00:35.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:35.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:35.669 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 10:00:35.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:35.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:35.669 | INFO | Uploaded 36GiB to GPU in 1.60733s (22.3973GiB/s) | 20260506 10:00:35.669 | INFO | Downloaded 34GiB to GPU in 1.52551s (22.2876GiB/s) | 20260506 10:00:40.808 | INFO | Analyzing chunk frames = [472576,473088) | 20260506 10:00:40.808 | INFO | MemInfo: { | 20260506 10:00:40.808 | INFO | "gpu" : " 17 GB", | 20260506 10:00:40.808 | INFO | "huge" : "", | 20260506 10:00:40.808 | INFO | "huge_shared" : " 27 GB", | 20260506 10:00:40.808 | INFO | "malloc" : " 14 GB", | 20260506 10:00:40.808 | INFO | "pinned" : " 29 GB" | 20260506 10:00:40.808 | INFO | } | 20260506 10:00:40.808 | INFO | Perf report for the last 5.13931 seconds: | 20260506 10:00:40.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:00:40.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:40.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:40.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:40.808 | INFO | ANALYSIS : 56.5%, 3.0, 87.7ms, 0 | 20260506 10:00:40.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:40.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:40.808 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:00:40.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:40.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:40.808 | INFO | Uploaded 36GiB to GPU in 1.60618s (22.4134GiB/s) | 20260506 10:00:40.808 | INFO | Downloaded 34GiB to GPU in 1.52329s (22.3202GiB/s) | 20260506 10:00:45.947 | INFO | Analyzing chunk frames = [473088,473600) | 20260506 10:00:45.947 | INFO | MemInfo: { | 20260506 10:00:45.947 | INFO | "gpu" : " 17 GB", | 20260506 10:00:45.947 | INFO | "huge" : "", | 20260506 10:00:45.947 | INFO | "huge_shared" : " 27 GB", | 20260506 10:00:45.947 | INFO | "malloc" : " 14 GB", | 20260506 10:00:45.947 | INFO | "pinned" : " 29 GB" | 20260506 10:00:45.947 | INFO | } | 20260506 10:00:45.947 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:00:45.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:00:45.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:45.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:45.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:45.947 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:00:45.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:45.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:45.947 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:00:45.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:45.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:45.948 | INFO | Uploaded 36GiB to GPU in 1.60454s (22.4364GiB/s) | 20260506 10:00:45.948 | INFO | Downloaded 34GiB to GPU in 1.52183s (22.3416GiB/s) | 20260506 10:00:51.086 | INFO | Analyzing chunk frames = [473600,474112) | 20260506 10:00:51.086 | INFO | MemInfo: { | 20260506 10:00:51.086 | INFO | "gpu" : " 17 GB", | 20260506 10:00:51.086 | INFO | "huge" : "", | 20260506 10:00:51.086 | INFO | "huge_shared" : " 27 GB", | 20260506 10:00:51.086 | INFO | "malloc" : " 14 GB", | 20260506 10:00:51.086 | INFO | "pinned" : " 29 GB" | 20260506 10:00:51.086 | INFO | } | 20260506 10:00:51.086 | INFO | Perf report for the last 5.13922 seconds: | 20260506 10:00:51.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:00:51.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:51.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:51.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:51.087 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 10:00:51.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:51.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:51.087 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 10:00:51.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:51.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:51.087 | INFO | Uploaded 36GiB to GPU in 1.60437s (22.4387GiB/s) | 20260506 10:00:51.087 | INFO | Downloaded 34GiB to GPU in 1.52199s (22.3392GiB/s) | 20260506 10:00:56.226 | INFO | Analyzing chunk frames = [474112,474624) | 20260506 10:00:56.226 | INFO | MemInfo: { | 20260506 10:00:56.226 | INFO | "gpu" : " 17 GB", | 20260506 10:00:56.226 | INFO | "huge" : "", | 20260506 10:00:56.226 | INFO | "huge_shared" : " 27 GB", | 20260506 10:00:56.226 | INFO | "malloc" : " 14 GB", | 20260506 10:00:56.226 | INFO | "pinned" : " 29 GB" | 20260506 10:00:56.226 | INFO | } | 20260506 10:00:56.226 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:00:56.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:00:56.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:56.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:56.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:56.226 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:00:56.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:56.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:56.226 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 10:00:56.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:56.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:00:56.226 | INFO | Uploaded 36GiB to GPU in 1.60538s (22.4247GiB/s) | 20260506 10:00:56.226 | INFO | Downloaded 34GiB to GPU in 1.52301s (22.3242GiB/s) | 20260506 10:01:01.365 | INFO | Analyzing chunk frames = [474624,475136) | 20260506 10:01:01.365 | INFO | MemInfo: { | 20260506 10:01:01.365 | INFO | "gpu" : " 17 GB", | 20260506 10:01:01.365 | INFO | "huge" : "", | 20260506 10:01:01.365 | INFO | "huge_shared" : " 27 GB", | 20260506 10:01:01.365 | INFO | "malloc" : " 14 GB", | 20260506 10:01:01.365 | INFO | "pinned" : " 29 GB" | 20260506 10:01:01.365 | INFO | } | 20260506 10:01:01.365 | INFO | Perf report for the last 5.13959 seconds: | 20260506 10:01:01.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:01:01.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:01.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:01.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:01.366 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 10:01:01.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:01.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:01.366 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:01:01.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:01.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:01.366 | INFO | Uploaded 36GiB to GPU in 1.60601s (22.4157GiB/s) | 20260506 10:01:01.366 | INFO | Downloaded 34GiB to GPU in 1.52304s (22.3238GiB/s) | 20260506 10:01:01.366 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:01:01.366 | INFO | Total mem usage: 15.9 GiB | 20260506 10:01:01.366 | INFO | All Time High Water Marks sum: 16.3 GiB (Savings: 2.53%) | 20260506 10:01:01.366 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -2.94%) | 20260506 10:01:01.366 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:01:01.366 | INFO | High Water Breakdown: | 20260506 10:01:01.366 | INFO | 60% : 9.28 GiB ( 96.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:01:01.366 | INFO | 39.9% : 6.17 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:01:01.366 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:01:01.366 | INFO | ------------------------------------------ | 20260506 10:01:01.366 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:01:01.366 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:01:01.366 | INFO | Total mem usage: 26.5 GiB | 20260506 10:01:01.366 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:01:01.366 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:01:01.366 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:01:01.366 | INFO | High Water Breakdown: | 20260506 10:01:01.366 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:01:01.366 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:01:01.366 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:01:01.366 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:01:01.366 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:01:01.366 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:01:01.366 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:01:01.366 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:01:01.366 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:01:01.366 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:01:01.367 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:01:01.367 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:01:01.367 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:01:01.367 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:01:01.367 | INFO | ------------------------------------------ | 20260506 10:01:01.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:01:01.367 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:01:01.367 | INFO | Total mem usage: 25.6 GiB | 20260506 10:01:01.367 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:01:01.367 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:01:01.367 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:01:01.367 | INFO | High Water Breakdown: | 20260506 10:01:01.367 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:01:01.367 | INFO | ------------------------------------------ | 20260506 10:01:01.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:01:01.367 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:01:01.367 | INFO | Total mem usage: 15.7 GiB | 20260506 10:01:01.367 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:01:01.367 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:01:01.367 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:01:01.367 | INFO | High Water Breakdown: | 20260506 10:01:01.367 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:01:01.367 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:01:01.367 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:01:01.367 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:01:01.367 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:01:01.367 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:01:01.367 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:01:01.367 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:01:01.367 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:01:01.367 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:01:01.367 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:01:01.367 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:01:01.367 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:01:01.367 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:01:01.367 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:01:01.367 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:01:01.367 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:01:01.367 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:01:01.367 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:01:01.367 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:01:01.367 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:01:01.367 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:01:01.367 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:01:01.367 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:01:01.367 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:01:01.367 | INFO | ------------------------------------------ | 20260506 10:01:01.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:01:01.367 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:01:01.367 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:01:01.367 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:01:01.367 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260506 10:01:01.367 | INFO | PerfReport: 0.28%, AggregateStats -- 8.59e-02 / -nan, 4.07e-03 / 0.00e+00 | 20260506 10:01:01.367 | INFO | PerfReport: 10.92%, Baseline -- 3.29e+00 / -nan, 1.71e-02 / 0.00e+00 | 20260506 10:01:01.367 | INFO | PerfReport: 3.06%, Histogram -- 9.22e-01 / -nan, 5.15e-02 / 0.00e+00 | 20260506 10:01:01.367 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.05e+00 / 0.00e+00 | 20260506 10:01:01.367 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.72e+01 / -nan, 4.26e-02 / 0.00e+00 | 20260506 10:01:01.367 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.25e+00 / -nan, 1.32e-01 / 0.00e+00 | 20260506 10:01:01.367 | INFO | PerfReport: 3.76%, Metrics -- 1.13e+00 / -nan, 2.14e-01 / 0.00e+00 | 20260506 10:01:01.367 | INFO | PerfReport: 1.45%, PulseSerializing -- 4.38e-01 / -nan, 6.29e-02 / 0.00e+00 | 20260506 10:01:01.367 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:01:01.367 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:01:01.367 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:01:01.367 | INFO | PerfReport: Full Execution -- 9.01e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 10:01:01.367 | INFO | PerfReport: 18.67%, Upload -- 1.68e+01 / 0.00e+00, 4.98e-01 / 0.00e+00 | 20260506 10:01:01.367 | INFO | PerfReport: 17.93%, Download -- 1.61e+01 / 0.00e+00, 8.53e-01 / 0.00e+00 | 20260506 10:01:01.367 | INFO | PerfReport: 33.49%, Compute -- 3.02e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260506 10:01:01.368 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260506 10:01:01.368 | INFO | PerfReport: 0.00%, LockDownload -- 4.93e-04 / 0.00e+00, 1.15e-04 / 0.00e+00 | 20260506 10:01:01.368 | INFO | PerfReport: 29.33%, LockCompute -- 2.64e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 10:01:01.368 | INFO | PerfReport: 0.02%, Misc -- 1.52e-02 / 0.00e+00, 4.65e-02 / 0.00e+00 | 20260506 10:01:01.463 | INFO | Starting a round of Baz IO | 20260506 10:01:04.379 | INFO | Populating RTMetrics report | 20260506 10:01:04.379 | INFO | RTMetrics report populated | 20260506 10:01:04.379 | INFO | Opening tmp RTMetrics json file | 20260506 10:01:04.414 | INFO | Serializing RTMetrics to json | 20260506 10:01:04.420 | INFO | Serialization done, swapping tmp file now... | 20260506 10:01:04.425 | INFO | RTMetrics done | 20260506 10:01:04.578 | INFO | Finished a round of Baz IO: Wrote 1.937 GiB in 3.115 seconds | 20260506 10:01:04.578 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.16% / 0.07% / 49.77% | 20260506 10:01:04.579 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.87513% / 0.000178814% / 3.57628e-05% | 20260506 10:01:04.579 | INFO | Baz IO overall: | 20260506 10:01:04.579 | INFO | BytesWritten : 203537273211 | 20260506 10:01:04.579 | INFO | NumEvents : 97412664609 | 20260506 10:01:06.504 | INFO | Analyzing chunk frames = [475136,475648) | 20260506 10:01:06.504 | INFO | MemInfo: { | 20260506 10:01:06.504 | INFO | "gpu" : " 17 GB", | 20260506 10:01:06.504 | INFO | "huge" : "", | 20260506 10:01:06.504 | INFO | "huge_shared" : " 27 GB", | 20260506 10:01:06.504 | INFO | "malloc" : " 14 GB", | 20260506 10:01:06.504 | INFO | "pinned" : " 29 GB" | 20260506 10:01:06.504 | INFO | } | 20260506 10:01:06.504 | INFO | Perf report for the last 5.13895 seconds: | 20260506 10:01:06.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:01:06.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:06.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:06.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:06.505 | INFO | ANALYSIS : 57.6%, 2.9, 88.0ms, 0 | 20260506 10:01:06.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:06.505 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 10:01:06.505 | INFO | PRE_HQ : 56.7%, 2.1, 61.7ms, 0 | 20260506 10:01:06.505 | INFO | BAZWRITER : 60.0%, 4.5, 139.9ms, 0 | 20260506 10:01:06.505 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.2ms, 0 | 20260506 10:01:06.505 | INFO | Uploaded 36GiB to GPU in 1.65189s (21.7932GiB/s) | 20260506 10:01:06.505 | INFO | Downloaded 38GiB to GPU in 1.71456s (22.1631GiB/s) | 20260506 10:01:11.644 | INFO | Analyzing chunk frames = [475648,476160) | 20260506 10:01:11.644 | INFO | MemInfo: { | 20260506 10:01:11.644 | INFO | "gpu" : " 17 GB", | 20260506 10:01:11.644 | INFO | "huge" : "", | 20260506 10:01:11.644 | INFO | "huge_shared" : " 27 GB", | 20260506 10:01:11.644 | INFO | "malloc" : " 14 GB", | 20260506 10:01:11.644 | INFO | "pinned" : " 29 GB" | 20260506 10:01:11.644 | INFO | } | 20260506 10:01:11.644 | INFO | Perf report for the last 5.13961 seconds: | 20260506 10:01:11.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:01:11.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:11.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:11.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:11.644 | INFO | ANALYSIS : 57.5%, 3.0, 89.7ms, 0 | 20260506 10:01:11.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:11.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:11.644 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:01:11.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:11.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:11.644 | INFO | Uploaded 36GiB to GPU in 1.60266s (22.4626GiB/s) | 20260506 10:01:11.645 | INFO | Downloaded 34GiB to GPU in 1.52001s (22.3683GiB/s) | 20260506 10:01:16.795 | INFO | Analyzing chunk frames = [476160,476672) | 20260506 10:01:16.795 | INFO | MemInfo: { | 20260506 10:01:16.795 | INFO | "gpu" : " 17 GB", | 20260506 10:01:16.795 | INFO | "huge" : "", | 20260506 10:01:16.795 | INFO | "huge_shared" : " 27 GB", | 20260506 10:01:16.795 | INFO | "malloc" : " 14 GB", | 20260506 10:01:16.795 | INFO | "pinned" : " 29 GB" | 20260506 10:01:16.795 | INFO | } | 20260506 10:01:16.795 | INFO | Perf report for the last 5.15102 seconds: | 20260506 10:01:16.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:01:16.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:16.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:16.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:16.795 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 10:01:16.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:16.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:16.795 | INFO | PRE_HQ : 29.5%, 1.0, 15.4ms, 0 | 20260506 10:01:16.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:16.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:16.796 | INFO | Uploaded 36GiB to GPU in 1.60701s (22.4019GiB/s) | 20260506 10:01:16.796 | INFO | Downloaded 34GiB to GPU in 1.52546s (22.2884GiB/s) | 20260506 10:01:21.922 | INFO | Analyzing chunk frames = [476672,477184) | 20260506 10:01:21.922 | INFO | MemInfo: { | 20260506 10:01:21.922 | INFO | "gpu" : " 17 GB", | 20260506 10:01:21.922 | INFO | "huge" : "", | 20260506 10:01:21.923 | INFO | "huge_shared" : " 27 GB", | 20260506 10:01:21.923 | INFO | "malloc" : " 14 GB", | 20260506 10:01:21.923 | INFO | "pinned" : " 29 GB" | 20260506 10:01:21.923 | INFO | } | 20260506 10:01:21.923 | INFO | Perf report for the last 5.12765 seconds: | 20260506 10:01:21.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:01:21.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:21.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:21.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:21.923 | INFO | ANALYSIS : 56.6%, 3.0, 87.6ms, 0 | 20260506 10:01:21.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:21.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:21.923 | INFO | PRE_HQ : 30.0%, 1.0, 15.5ms, 0 | 20260506 10:01:21.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:21.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:21.923 | INFO | Uploaded 36GiB to GPU in 1.60662s (22.4073GiB/s) | 20260506 10:01:21.923 | INFO | Downloaded 34GiB to GPU in 1.52595s (22.2812GiB/s) | 20260506 10:01:27.062 | INFO | Analyzing chunk frames = [477184,477696) | 20260506 10:01:27.062 | INFO | MemInfo: { | 20260506 10:01:27.062 | INFO | "gpu" : " 17 GB", | 20260506 10:01:27.062 | INFO | "huge" : "", | 20260506 10:01:27.062 | INFO | "huge_shared" : " 27 GB", | 20260506 10:01:27.062 | INFO | "malloc" : " 14 GB", | 20260506 10:01:27.062 | INFO | "pinned" : " 29 GB" | 20260506 10:01:27.062 | INFO | } | 20260506 10:01:27.062 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:01:27.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:01:27.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:27.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:27.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:27.062 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:01:27.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:27.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:27.062 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:01:27.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:27.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:27.063 | INFO | Uploaded 36GiB to GPU in 1.60449s (22.4371GiB/s) | 20260506 10:01:27.063 | INFO | Downloaded 34GiB to GPU in 1.52226s (22.3352GiB/s) | 20260506 10:01:32.201 | INFO | Analyzing chunk frames = [477696,478208) | 20260506 10:01:32.201 | INFO | MemInfo: { | 20260506 10:01:32.201 | INFO | "gpu" : " 17 GB", | 20260506 10:01:32.201 | INFO | "huge" : "", | 20260506 10:01:32.201 | INFO | "huge_shared" : " 27 GB", | 20260506 10:01:32.201 | INFO | "malloc" : " 14 GB", | 20260506 10:01:32.201 | INFO | "pinned" : " 29 GB" | 20260506 10:01:32.201 | INFO | } | 20260506 10:01:32.201 | INFO | Perf report for the last 5.13922 seconds: | 20260506 10:01:32.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:01:32.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:32.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:32.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:32.201 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 10:01:32.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:32.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:32.201 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:01:32.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:32.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:32.202 | INFO | Uploaded 36GiB to GPU in 1.60499s (22.43GiB/s) | 20260506 10:01:32.202 | INFO | Downloaded 34GiB to GPU in 1.52227s (22.335GiB/s) | 20260506 10:01:37.340 | INFO | Analyzing chunk frames = [478208,478720) | 20260506 10:01:37.340 | INFO | MemInfo: { | 20260506 10:01:37.340 | INFO | "gpu" : " 17 GB", | 20260506 10:01:37.340 | INFO | "huge" : "", | 20260506 10:01:37.340 | INFO | "huge_shared" : " 27 GB", | 20260506 10:01:37.341 | INFO | "malloc" : " 14 GB", | 20260506 10:01:37.341 | INFO | "pinned" : " 29 GB" | 20260506 10:01:37.341 | INFO | } | 20260506 10:01:37.341 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:01:37.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:01:37.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:37.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:37.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:37.341 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:01:37.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:37.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:37.341 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:01:37.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:37.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:37.342 | INFO | Uploaded 36GiB to GPU in 1.60573s (22.4197GiB/s) | 20260506 10:01:37.342 | INFO | Downloaded 34GiB to GPU in 1.52396s (22.3103GiB/s) | 20260506 10:01:42.480 | INFO | Analyzing chunk frames = [478720,479232) | 20260506 10:01:42.480 | INFO | MemInfo: { | 20260506 10:01:42.480 | INFO | "gpu" : " 17 GB", | 20260506 10:01:42.480 | INFO | "huge" : "", | 20260506 10:01:42.480 | INFO | "huge_shared" : " 27 GB", | 20260506 10:01:42.480 | INFO | "malloc" : " 14 GB", | 20260506 10:01:42.480 | INFO | "pinned" : " 29 GB" | 20260506 10:01:42.480 | INFO | } | 20260506 10:01:42.480 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:01:42.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:01:42.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:42.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:42.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:42.480 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260506 10:01:42.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:42.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:42.480 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 10:01:42.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:42.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:42.480 | INFO | Uploaded 36GiB to GPU in 1.60538s (22.4246GiB/s) | 20260506 10:01:42.481 | INFO | Downloaded 34GiB to GPU in 1.52327s (22.3205GiB/s) | 20260506 10:01:42.578 | INFO | Starting a round of Baz IO | 20260506 10:01:45.493 | INFO | Populating RTMetrics report | 20260506 10:01:45.493 | INFO | RTMetrics report populated | 20260506 10:01:45.493 | INFO | Opening tmp RTMetrics json file | 20260506 10:01:45.534 | INFO | Serializing RTMetrics to json | 20260506 10:01:45.719 | INFO | Serialization done, swapping tmp file now... | 20260506 10:01:45.795 | INFO | RTMetrics done | 20260506 10:01:46.190 | INFO | Finished a round of Baz IO: Wrote 1.938 GiB in 3.612 seconds | 20260506 10:01:46.190 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.10% / 0.07% / 49.84% | 20260506 10:01:46.190 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.83131% / 0.000147025% / 3.97364e-05% | 20260506 10:01:46.190 | INFO | Baz IO overall: | 20260506 10:01:46.190 | INFO | BytesWritten : 205618086706 | 20260506 10:01:46.190 | INFO | NumEvents : 98422677882 | 20260506 10:01:47.619 | INFO | Analyzing chunk frames = [479232,479744) | 20260506 10:01:47.619 | INFO | MemInfo: { | 20260506 10:01:47.619 | INFO | "gpu" : " 17 GB", | 20260506 10:01:47.619 | INFO | "huge" : "", | 20260506 10:01:47.619 | INFO | "huge_shared" : " 27 GB", | 20260506 10:01:47.619 | INFO | "malloc" : " 14 GB", | 20260506 10:01:47.619 | INFO | "pinned" : " 29 GB" | 20260506 10:01:47.619 | INFO | } | 20260506 10:01:47.619 | INFO | Perf report for the last 5.13918 seconds: | 20260506 10:01:47.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:01:47.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:47.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:47.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:47.619 | INFO | ANALYSIS : 57.6%, 2.8, 85.9ms, 0 | 20260506 10:01:47.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:47.619 | INFO | RT_METRICS : 50.0%, 1.0, 26.2ms, 0 | 20260506 10:01:47.619 | INFO | PRE_HQ : 56.7%, 2.1, 61.0ms, 0 | 20260506 10:01:47.619 | INFO | BAZWRITER : 69.9%, 4.7, 170.5ms, 0 | 20260506 10:01:47.619 | INFO | BAZ_SAMPLING : 4.7%, 1.0, 2.4ms, 0 | 20260506 10:01:47.620 | INFO | Uploaded 36GiB to GPU in 1.67178s (21.534GiB/s) | 20260506 10:01:47.620 | INFO | Downloaded 38GiB to GPU in 1.73609s (21.8883GiB/s) | 20260506 10:01:52.758 | INFO | Analyzing chunk frames = [479744,480256) | 20260506 10:01:52.758 | INFO | MemInfo: { | 20260506 10:01:52.758 | INFO | "gpu" : " 17 GB", | 20260506 10:01:52.758 | INFO | "huge" : "", | 20260506 10:01:52.758 | INFO | "huge_shared" : " 27 GB", | 20260506 10:01:52.758 | INFO | "malloc" : " 14 GB", | 20260506 10:01:52.759 | INFO | "pinned" : " 29 GB" | 20260506 10:01:52.759 | INFO | } | 20260506 10:01:52.759 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:01:52.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:01:52.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:52.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:52.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:52.759 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 10:01:52.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:52.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:52.759 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:01:52.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:52.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:52.759 | INFO | Uploaded 36GiB to GPU in 1.60262s (22.4632GiB/s) | 20260506 10:01:52.759 | INFO | Downloaded 34GiB to GPU in 1.51959s (22.3744GiB/s) | 20260506 10:01:57.898 | INFO | Analyzing chunk frames = [480256,480768) | 20260506 10:01:57.898 | INFO | MemInfo: { | 20260506 10:01:57.898 | INFO | "gpu" : " 17 GB", | 20260506 10:01:57.898 | INFO | "huge" : "", | 20260506 10:01:57.898 | INFO | "huge_shared" : " 27 GB", | 20260506 10:01:57.898 | INFO | "malloc" : " 14 GB", | 20260506 10:01:57.898 | INFO | "pinned" : " 29 GB" | 20260506 10:01:57.898 | INFO | } | 20260506 10:01:57.898 | INFO | Perf report for the last 5.13948 seconds: | 20260506 10:01:57.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:01:57.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:57.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:57.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:57.898 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260506 10:01:57.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:57.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:57.898 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:01:57.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:57.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:01:57.899 | INFO | Uploaded 36GiB to GPU in 1.60679s (22.4049GiB/s) | 20260506 10:01:57.899 | INFO | Downloaded 34GiB to GPU in 1.52488s (22.2969GiB/s) | 20260506 10:02:03.037 | INFO | Analyzing chunk frames = [480768,481280) | 20260506 10:02:03.037 | INFO | MemInfo: { | 20260506 10:02:03.037 | INFO | "gpu" : " 17 GB", | 20260506 10:02:03.037 | INFO | "huge" : "", | 20260506 10:02:03.037 | INFO | "huge_shared" : " 27 GB", | 20260506 10:02:03.037 | INFO | "malloc" : " 14 GB", | 20260506 10:02:03.037 | INFO | "pinned" : " 29 GB" | 20260506 10:02:03.037 | INFO | } | 20260506 10:02:03.037 | INFO | Perf report for the last 5.13929 seconds: | 20260506 10:02:03.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:02:03.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:03.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:03.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:03.038 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:02:03.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:03.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:03.038 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 10:02:03.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:03.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:03.038 | INFO | Uploaded 36GiB to GPU in 1.60673s (22.4058GiB/s) | 20260506 10:02:03.038 | INFO | Downloaded 34GiB to GPU in 1.52442s (22.3036GiB/s) | 20260506 10:02:08.176 | INFO | Analyzing chunk frames = [481280,481792) | 20260506 10:02:08.176 | INFO | MemInfo: { | 20260506 10:02:08.176 | INFO | "gpu" : " 17 GB", | 20260506 10:02:08.176 | INFO | "huge" : "", | 20260506 10:02:08.177 | INFO | "huge_shared" : " 27 GB", | 20260506 10:02:08.177 | INFO | "malloc" : " 14 GB", | 20260506 10:02:08.177 | INFO | "pinned" : " 29 GB" | 20260506 10:02:08.177 | INFO | } | 20260506 10:02:08.177 | INFO | Perf report for the last 5.13929 seconds: | 20260506 10:02:08.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:02:08.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:08.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:08.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:08.177 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 10:02:08.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:08.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:08.177 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:02:08.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:08.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:08.177 | INFO | Uploaded 36GiB to GPU in 1.60455s (22.4362GiB/s) | 20260506 10:02:08.177 | INFO | Downloaded 34GiB to GPU in 1.52204s (22.3384GiB/s) | 20260506 10:02:13.316 | INFO | Analyzing chunk frames = [481792,482304) | 20260506 10:02:13.316 | INFO | MemInfo: { | 20260506 10:02:13.316 | INFO | "gpu" : " 17 GB", | 20260506 10:02:13.316 | INFO | "huge" : "", | 20260506 10:02:13.316 | INFO | "huge_shared" : " 27 GB", | 20260506 10:02:13.316 | INFO | "malloc" : " 14 GB", | 20260506 10:02:13.316 | INFO | "pinned" : " 29 GB" | 20260506 10:02:13.316 | INFO | } | 20260506 10:02:13.316 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:02:13.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:02:13.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:13.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:13.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:13.316 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:02:13.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:13.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:13.316 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:02:13.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:13.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:13.317 | INFO | Uploaded 36GiB to GPU in 1.60485s (22.4321GiB/s) | 20260506 10:02:13.317 | INFO | Downloaded 34GiB to GPU in 1.52261s (22.3301GiB/s) | 20260506 10:02:18.455 | INFO | Analyzing chunk frames = [482304,482816) | 20260506 10:02:18.455 | INFO | MemInfo: { | 20260506 10:02:18.455 | INFO | "gpu" : " 17 GB", | 20260506 10:02:18.455 | INFO | "huge" : "", | 20260506 10:02:18.455 | INFO | "huge_shared" : " 27 GB", | 20260506 10:02:18.455 | INFO | "malloc" : " 14 GB", | 20260506 10:02:18.455 | INFO | "pinned" : " 29 GB" | 20260506 10:02:18.456 | INFO | } | 20260506 10:02:18.456 | INFO | Perf report for the last 5.13944 seconds: | 20260506 10:02:18.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:02:18.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:18.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:18.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:18.456 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:02:18.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:18.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:18.456 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 10:02:18.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:18.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:18.456 | INFO | Uploaded 36GiB to GPU in 1.60523s (22.4267GiB/s) | 20260506 10:02:18.456 | INFO | Downloaded 34GiB to GPU in 1.52306s (22.3235GiB/s) | 20260506 10:02:23.595 | INFO | Analyzing chunk frames = [482816,483328) | 20260506 10:02:23.595 | INFO | MemInfo: { | 20260506 10:02:23.595 | INFO | "gpu" : " 17 GB", | 20260506 10:02:23.595 | INFO | "huge" : "", | 20260506 10:02:23.595 | INFO | "huge_shared" : " 27 GB", | 20260506 10:02:23.595 | INFO | "malloc" : " 14 GB", | 20260506 10:02:23.595 | INFO | "pinned" : " 29 GB" | 20260506 10:02:23.595 | INFO | } | 20260506 10:02:23.595 | INFO | Perf report for the last 5.1393 seconds: | 20260506 10:02:23.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:02:23.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:23.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:23.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:23.595 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:02:23.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:23.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:23.595 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:02:23.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:23.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:23.595 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:02:23.595 | INFO | Total mem usage: 16 GiB | 20260506 10:02:23.595 | INFO | All Time High Water Marks sum: 16.4 GiB (Savings: 2.51%) | 20260506 10:02:23.595 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: 0.0481%) | 20260506 10:02:23.595 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:02:23.595 | INFO | High Water Breakdown: | 20260506 10:02:23.595 | INFO | 60.6% : 9.69 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:02:23.595 | INFO | Uploaded 36GiB to GPU in 1.60569s (22.4202GiB/s) | 20260506 10:02:23.595 | INFO | Downloaded 34GiB to GPU in 1.52369s (22.3143GiB/s) | 20260506 10:02:23.595 | INFO | 39.3% : 6.29 GiB ( 94% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:02:23.596 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:02:23.596 | INFO | ------------------------------------------ | 20260506 10:02:23.596 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:02:23.596 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:02:23.596 | INFO | Total mem usage: 26.5 GiB | 20260506 10:02:23.596 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:02:23.596 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:02:23.596 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:02:23.596 | INFO | High Water Breakdown: | 20260506 10:02:23.596 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:02:23.596 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:02:23.596 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:02:23.596 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:02:23.596 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:02:23.596 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:02:23.596 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:02:23.596 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:02:23.596 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:02:23.596 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:02:23.596 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:02:23.596 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:02:23.596 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:02:23.596 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:02:23.596 | INFO | ------------------------------------------ | 20260506 10:02:23.596 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:02:23.596 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:02:23.596 | INFO | Total mem usage: 25.6 GiB | 20260506 10:02:23.596 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:02:23.596 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:02:23.596 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:02:23.596 | INFO | High Water Breakdown: | 20260506 10:02:23.596 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:02:23.596 | INFO | ------------------------------------------ | 20260506 10:02:23.596 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:02:23.596 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:02:23.596 | INFO | Total mem usage: 15.7 GiB | 20260506 10:02:23.596 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:02:23.596 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:02:23.596 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:02:23.596 | INFO | High Water Breakdown: | 20260506 10:02:23.596 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:02:23.596 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:02:23.596 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:02:23.596 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:02:23.596 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:02:23.596 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:02:23.596 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:02:23.596 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:02:23.596 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:02:23.596 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:02:23.596 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:02:23.596 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:02:23.596 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:02:23.596 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:02:23.596 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:02:23.596 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:02:23.596 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:02:23.596 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:02:23.596 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:02:23.596 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:02:23.596 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:02:23.596 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:02:23.596 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:02:23.596 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:02:23.596 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:02:23.596 | INFO | ------------------------------------------ | 20260506 10:02:23.596 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:02:23.596 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:02:23.596 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:02:23.596 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:02:23.596 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 10:02:23.596 | INFO | PerfReport: 0.29%, AggregateStats -- 8.64e-02 / -nan, 5.16e-03 / 0.00e+00 | 20260506 10:02:23.596 | INFO | PerfReport: 10.92%, Baseline -- 3.29e+00 / -nan, 1.61e-02 / 0.00e+00 | 20260506 10:02:23.596 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.26e-02 / 0.00e+00 | 20260506 10:02:23.596 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.04e+00 / 0.00e+00 | 20260506 10:02:23.596 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.72e+01 / -nan, 4.20e-02 / 0.00e+00 | 20260506 10:02:23.596 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.24e+00 / -nan, 1.32e-01 / 0.00e+00 | 20260506 10:02:23.596 | INFO | PerfReport: 3.74%, Metrics -- 1.13e+00 / -nan, 1.77e-01 / 0.00e+00 | 20260506 10:02:23.596 | INFO | PerfReport: 1.45%, PulseSerializing -- 4.37e-01 / -nan, 6.25e-02 / 0.00e+00 | 20260506 10:02:23.596 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:02:23.596 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:02:23.596 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:02:23.596 | INFO | PerfReport: Full Execution -- 9.02e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:02:23.596 | INFO | PerfReport: 18.62%, Upload -- 1.68e+01 / 0.00e+00, 4.56e-01 / 0.00e+00 | 20260506 10:02:23.596 | INFO | PerfReport: 17.89%, Download -- 1.61e+01 / 0.00e+00, 7.94e-01 / 0.00e+00 | 20260506 10:02:23.596 | INFO | PerfReport: 33.43%, Compute -- 3.01e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 10:02:23.597 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:02:23.597 | INFO | PerfReport: 0.00%, LockDownload -- 4.89e-04 / 0.00e+00, 1.18e-04 / 0.00e+00 | 20260506 10:02:23.597 | INFO | PerfReport: 29.48%, LockCompute -- 2.66e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 10:02:23.597 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.32e-02 / 0.00e+00 | 20260506 10:02:23.693 | INFO | Starting a round of Baz IO | 20260506 10:02:26.609 | INFO | Populating RTMetrics report | 20260506 10:02:26.609 | INFO | RTMetrics report populated | 20260506 10:02:26.609 | INFO | Opening tmp RTMetrics json file | 20260506 10:02:26.731 | INFO | Serializing RTMetrics to json | 20260506 10:02:26.800 | INFO | Serialization done, swapping tmp file now... | 20260506 10:02:26.829 | INFO | RTMetrics done | 20260506 10:02:27.055 | INFO | Finished a round of Baz IO: Wrote 1.939 GiB in 3.362 seconds | 20260506 10:02:27.055 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 50.03% / 0.07% / 49.90% | 20260506 10:02:27.055 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.80039% / 0.000115236% / 2.78155e-05% | 20260506 10:02:27.055 | INFO | Baz IO overall: | 20260506 10:02:27.055 | INFO | BytesWritten : 207699927358 | 20260506 10:02:27.055 | INFO | NumEvents : 99433037099 | 20260506 10:02:28.734 | INFO | Analyzing chunk frames = [483328,483840) | 20260506 10:02:28.734 | INFO | MemInfo: { | 20260506 10:02:28.734 | INFO | "gpu" : " 17 GB", | 20260506 10:02:28.734 | INFO | "huge" : "", | 20260506 10:02:28.734 | INFO | "huge_shared" : " 27 GB", | 20260506 10:02:28.734 | INFO | "malloc" : " 14 GB", | 20260506 10:02:28.734 | INFO | "pinned" : " 29 GB" | 20260506 10:02:28.734 | INFO | } | 20260506 10:02:28.734 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:02:28.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:02:28.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:28.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:28.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:28.734 | INFO | ANALYSIS : 57.6%, 2.8, 86.1ms, 0 | 20260506 10:02:28.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:28.734 | INFO | RT_METRICS : 48.3%, 1.0, 25.3ms, 0 | 20260506 10:02:28.734 | INFO | PRE_HQ : 56.7%, 2.1, 60.7ms, 0 | 20260506 10:02:28.734 | INFO | BAZWRITER : 65.0%, 4.8, 158.9ms, 0 | 20260506 10:02:28.734 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 10:02:28.735 | INFO | Uploaded 36GiB to GPU in 1.6723s (21.5273GiB/s) | 20260506 10:02:28.735 | INFO | Downloaded 38GiB to GPU in 1.73641s (21.8842GiB/s) | 20260506 10:02:33.874 | INFO | Analyzing chunk frames = [483840,484352) | 20260506 10:02:33.874 | INFO | MemInfo: { | 20260506 10:02:33.874 | INFO | "gpu" : " 17 GB", | 20260506 10:02:33.874 | INFO | "huge" : "", | 20260506 10:02:33.874 | INFO | "huge_shared" : " 27 GB", | 20260506 10:02:33.874 | INFO | "malloc" : " 14 GB", | 20260506 10:02:33.874 | INFO | "pinned" : " 29 GB" | 20260506 10:02:33.874 | INFO | } | 20260506 10:02:33.874 | INFO | Perf report for the last 5.13961 seconds: | 20260506 10:02:33.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:02:33.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:33.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:33.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:33.874 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:02:33.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:33.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:33.874 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:02:33.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:33.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:33.874 | INFO | Uploaded 36GiB to GPU in 1.6036s (22.4495GiB/s) | 20260506 10:02:33.874 | INFO | Downloaded 34GiB to GPU in 1.52045s (22.3618GiB/s) | 20260506 10:02:39.013 | INFO | Analyzing chunk frames = [484352,484864) | 20260506 10:02:39.013 | INFO | MemInfo: { | 20260506 10:02:39.013 | INFO | "gpu" : " 17 GB", | 20260506 10:02:39.013 | INFO | "huge" : "", | 20260506 10:02:39.013 | INFO | "huge_shared" : " 27 GB", | 20260506 10:02:39.013 | INFO | "malloc" : " 14 GB", | 20260506 10:02:39.013 | INFO | "pinned" : " 29 GB" | 20260506 10:02:39.013 | INFO | } | 20260506 10:02:39.013 | INFO | Perf report for the last 5.13911 seconds: | 20260506 10:02:39.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:02:39.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:39.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:39.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:39.013 | INFO | ANALYSIS : 56.9%, 3.0, 88.3ms, 0 | 20260506 10:02:39.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:39.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:39.013 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:02:39.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:39.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:39.014 | INFO | Uploaded 36GiB to GPU in 1.60675s (22.4054GiB/s) | 20260506 10:02:39.014 | INFO | Downloaded 34GiB to GPU in 1.52445s (22.3031GiB/s) | 20260506 10:02:44.152 | INFO | Analyzing chunk frames = [484864,485376) | 20260506 10:02:44.152 | INFO | MemInfo: { | 20260506 10:02:44.152 | INFO | "gpu" : " 17 GB", | 20260506 10:02:44.152 | INFO | "huge" : "", | 20260506 10:02:44.152 | INFO | "huge_shared" : " 27 GB", | 20260506 10:02:44.152 | INFO | "malloc" : " 14 GB", | 20260506 10:02:44.152 | INFO | "pinned" : " 29 GB" | 20260506 10:02:44.152 | INFO | } | 20260506 10:02:44.152 | INFO | Perf report for the last 5.13925 seconds: | 20260506 10:02:44.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:02:44.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:44.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:44.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:44.152 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:02:44.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:44.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:44.152 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:02:44.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:44.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:44.152 | INFO | Uploaded 36GiB to GPU in 1.60671s (22.406GiB/s) | 20260506 10:02:44.152 | INFO | Downloaded 34GiB to GPU in 1.52465s (22.3002GiB/s) | 20260506 10:02:49.291 | INFO | Analyzing chunk frames = [485376,485888) | 20260506 10:02:49.291 | INFO | MemInfo: { | 20260506 10:02:49.291 | INFO | "gpu" : " 17 GB", | 20260506 10:02:49.291 | INFO | "huge" : "", | 20260506 10:02:49.291 | INFO | "huge_shared" : " 27 GB", | 20260506 10:02:49.291 | INFO | "malloc" : " 14 GB", | 20260506 10:02:49.291 | INFO | "pinned" : " 29 GB" | 20260506 10:02:49.291 | INFO | } | 20260506 10:02:49.291 | INFO | Perf report for the last 5.13913 seconds: | 20260506 10:02:49.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:02:49.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:49.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:49.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:49.291 | INFO | ANALYSIS : 56.5%, 3.0, 88.0ms, 0 | 20260506 10:02:49.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:49.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:49.291 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:02:49.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:49.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:49.292 | INFO | Uploaded 36GiB to GPU in 1.60438s (22.4386GiB/s) | 20260506 10:02:49.292 | INFO | Downloaded 34GiB to GPU in 1.52142s (22.3475GiB/s) | 20260506 10:02:54.431 | INFO | Analyzing chunk frames = [485888,486400) | 20260506 10:02:54.431 | INFO | MemInfo: { | 20260506 10:02:54.431 | INFO | "gpu" : " 17 GB", | 20260506 10:02:54.431 | INFO | "huge" : "", | 20260506 10:02:54.431 | INFO | "huge_shared" : " 27 GB", | 20260506 10:02:54.431 | INFO | "malloc" : " 14 GB", | 20260506 10:02:54.431 | INFO | "pinned" : " 29 GB" | 20260506 10:02:54.431 | INFO | } | 20260506 10:02:54.431 | INFO | Perf report for the last 5.1395 seconds: | 20260506 10:02:54.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:02:54.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:54.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:54.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:54.431 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:02:54.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:54.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:54.431 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:02:54.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:54.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:54.431 | INFO | Uploaded 36GiB to GPU in 1.60468s (22.4343GiB/s) | 20260506 10:02:54.431 | INFO | Downloaded 34GiB to GPU in 1.52273s (22.3283GiB/s) | 20260506 10:02:59.571 | INFO | Analyzing chunk frames = [486400,486912) | 20260506 10:02:59.571 | INFO | MemInfo: { | 20260506 10:02:59.571 | INFO | "gpu" : " 17 GB", | 20260506 10:02:59.571 | INFO | "huge" : "", | 20260506 10:02:59.571 | INFO | "huge_shared" : " 27 GB", | 20260506 10:02:59.571 | INFO | "malloc" : " 14 GB", | 20260506 10:02:59.571 | INFO | "pinned" : " 29 GB" | 20260506 10:02:59.571 | INFO | } | 20260506 10:02:59.572 | INFO | Perf report for the last 5.1407 seconds: | 20260506 10:02:59.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:02:59.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:59.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:59.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:59.572 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:02:59.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:59.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:59.572 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 10:02:59.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:59.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:02:59.572 | INFO | Uploaded 36GiB to GPU in 1.60493s (22.4309GiB/s) | 20260506 10:02:59.572 | INFO | Downloaded 34GiB to GPU in 1.52286s (22.3264GiB/s) | 20260506 10:03:04.709 | INFO | Analyzing chunk frames = [486912,487424) | 20260506 10:03:04.709 | INFO | MemInfo: { | 20260506 10:03:04.709 | INFO | "gpu" : " 17 GB", | 20260506 10:03:04.709 | INFO | "huge" : "", | 20260506 10:03:04.709 | INFO | "huge_shared" : " 27 GB", | 20260506 10:03:04.709 | INFO | "malloc" : " 14 GB", | 20260506 10:03:04.710 | INFO | "pinned" : " 29 GB" | 20260506 10:03:04.710 | INFO | } | 20260506 10:03:04.710 | INFO | Perf report for the last 5.13807 seconds: | 20260506 10:03:04.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:03:04.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:04.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:04.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:04.710 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:03:04.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:04.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:04.710 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:03:04.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:04.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:04.710 | INFO | Uploaded 36GiB to GPU in 1.60552s (22.4226GiB/s) | 20260506 10:03:04.710 | INFO | Downloaded 34GiB to GPU in 1.52367s (22.3145GiB/s) | 20260506 10:03:04.806 | INFO | Starting a round of Baz IO | 20260506 10:03:07.726 | INFO | Populating RTMetrics report | 20260506 10:03:07.726 | INFO | RTMetrics report populated | 20260506 10:03:07.726 | INFO | Opening tmp RTMetrics json file | 20260506 10:03:07.828 | INFO | Serializing RTMetrics to json | 20260506 10:03:07.831 | INFO | Serialization done, swapping tmp file now... | 20260506 10:03:07.834 | INFO | RTMetrics done | 20260506 10:03:08.135 | INFO | Finished a round of Baz IO: Wrote 1.942 GiB in 3.328 seconds | 20260506 10:03:08.135 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.96% / 0.07% / 49.97% | 20260506 10:03:08.135 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.78597% / 0.000186761% / 3.57628e-05% | 20260506 10:03:08.135 | INFO | Baz IO overall: | 20260506 10:03:08.135 | INFO | BytesWritten : 209785346276 | 20260506 10:03:08.135 | INFO | NumEvents : 100445328494 | 20260506 10:03:09.849 | INFO | Analyzing chunk frames = [487424,487936) | 20260506 10:03:09.849 | INFO | MemInfo: { | 20260506 10:03:09.849 | INFO | "gpu" : " 17 GB", | 20260506 10:03:09.849 | INFO | "huge" : "", | 20260506 10:03:09.849 | INFO | "huge_shared" : " 27 GB", | 20260506 10:03:09.849 | INFO | "malloc" : " 14 GB", | 20260506 10:03:09.849 | INFO | "pinned" : " 29 GB" | 20260506 10:03:09.849 | INFO | } | 20260506 10:03:09.849 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:03:09.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:03:09.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:09.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:09.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:09.849 | INFO | ANALYSIS : 57.7%, 2.8, 85.5ms, 0 | 20260506 10:03:09.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:09.849 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260506 10:03:09.849 | INFO | PRE_HQ : 56.8%, 2.1, 60.8ms, 0 | 20260506 10:03:09.849 | INFO | BAZWRITER : 64.3%, 4.8, 157.8ms, 0 | 20260506 10:03:09.849 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.5ms, 0 | 20260506 10:03:09.849 | INFO | Uploaded 36GiB to GPU in 1.67615s (21.4778GiB/s) | 20260506 10:03:09.849 | INFO | Downloaded 38GiB to GPU in 1.74098s (21.8267GiB/s) | 20260506 10:03:14.988 | INFO | Analyzing chunk frames = [487936,488448) | 20260506 10:03:14.988 | INFO | MemInfo: { | 20260506 10:03:14.988 | INFO | "gpu" : " 17 GB", | 20260506 10:03:14.988 | INFO | "huge" : "", | 20260506 10:03:14.988 | INFO | "huge_shared" : " 27 GB", | 20260506 10:03:14.988 | INFO | "malloc" : " 14 GB", | 20260506 10:03:14.988 | INFO | "pinned" : " 29 GB" | 20260506 10:03:14.988 | INFO | } | 20260506 10:03:14.988 | INFO | Perf report for the last 5.13949 seconds: | 20260506 10:03:14.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:03:14.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:14.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:14.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:14.989 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 10:03:14.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:14.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:14.989 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:03:14.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:14.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:14.989 | INFO | Uploaded 36GiB to GPU in 1.60233s (22.4673GiB/s) | 20260506 10:03:14.989 | INFO | Downloaded 34GiB to GPU in 1.51972s (22.3725GiB/s) | 20260506 10:03:20.127 | INFO | Analyzing chunk frames = [488448,488960) | 20260506 10:03:20.127 | INFO | MemInfo: { | 20260506 10:03:20.127 | INFO | "gpu" : " 17 GB", | 20260506 10:03:20.127 | INFO | "huge" : "", | 20260506 10:03:20.127 | INFO | "huge_shared" : " 27 GB", | 20260506 10:03:20.128 | INFO | "malloc" : " 14 GB", | 20260506 10:03:20.128 | INFO | "pinned" : " 29 GB" | 20260506 10:03:20.128 | INFO | } | 20260506 10:03:20.128 | INFO | Perf report for the last 5.1392 seconds: | 20260506 10:03:20.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:03:20.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:20.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:20.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:20.128 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260506 10:03:20.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:20.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:20.128 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:03:20.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:20.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:20.128 | INFO | Uploaded 36GiB to GPU in 1.60635s (22.411GiB/s) | 20260506 10:03:20.128 | INFO | Downloaded 34GiB to GPU in 1.52435s (22.3046GiB/s) | 20260506 10:03:25.267 | INFO | Analyzing chunk frames = [488960,489472) | 20260506 10:03:25.267 | INFO | MemInfo: { | 20260506 10:03:25.267 | INFO | "gpu" : " 17 GB", | 20260506 10:03:25.267 | INFO | "huge" : "", | 20260506 10:03:25.267 | INFO | "huge_shared" : " 27 GB", | 20260506 10:03:25.267 | INFO | "malloc" : " 14 GB", | 20260506 10:03:25.267 | INFO | "pinned" : " 29 GB" | 20260506 10:03:25.267 | INFO | } | 20260506 10:03:25.267 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:03:25.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:03:25.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:25.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:25.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:25.267 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:03:25.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:25.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:25.267 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:03:25.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:25.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:25.267 | INFO | Uploaded 36GiB to GPU in 1.6063s (22.4117GiB/s) | 20260506 10:03:25.270 | INFO | Downloaded 34GiB to GPU in 1.52385s (22.3118GiB/s) | 20260506 10:03:30.406 | INFO | Analyzing chunk frames = [489472,489984) | 20260506 10:03:30.406 | INFO | MemInfo: { | 20260506 10:03:30.406 | INFO | "gpu" : " 17 GB", | 20260506 10:03:30.406 | INFO | "huge" : "", | 20260506 10:03:30.406 | INFO | "huge_shared" : " 27 GB", | 20260506 10:03:30.406 | INFO | "malloc" : " 14 GB", | 20260506 10:03:30.406 | INFO | "pinned" : " 29 GB" | 20260506 10:03:30.406 | INFO | } | 20260506 10:03:30.406 | INFO | Perf report for the last 5.13944 seconds: | 20260506 10:03:30.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:03:30.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:30.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:30.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:30.406 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 10:03:30.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:30.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:30.407 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:03:30.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:30.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:30.407 | INFO | Uploaded 36GiB to GPU in 1.60415s (22.4418GiB/s) | 20260506 10:03:30.407 | INFO | Downloaded 34GiB to GPU in 1.52155s (22.3456GiB/s) | 20260506 10:03:35.545 | INFO | Analyzing chunk frames = [489984,490496) | 20260506 10:03:35.545 | INFO | MemInfo: { | 20260506 10:03:35.545 | INFO | "gpu" : " 17 GB", | 20260506 10:03:35.545 | INFO | "huge" : "", | 20260506 10:03:35.545 | INFO | "huge_shared" : " 27 GB", | 20260506 10:03:35.546 | INFO | "malloc" : " 14 GB", | 20260506 10:03:35.546 | INFO | "pinned" : " 29 GB" | 20260506 10:03:35.546 | INFO | } | 20260506 10:03:35.546 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:03:35.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:03:35.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:35.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:35.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:35.546 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 10:03:35.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:35.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:35.546 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:03:35.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:35.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:35.546 | INFO | Uploaded 36GiB to GPU in 1.60445s (22.4376GiB/s) | 20260506 10:03:35.546 | INFO | Downloaded 34GiB to GPU in 1.52245s (22.3324GiB/s) | 20260506 10:03:40.685 | INFO | Analyzing chunk frames = [490496,491008) | 20260506 10:03:40.685 | INFO | MemInfo: { | 20260506 10:03:40.685 | INFO | "gpu" : " 17 GB", | 20260506 10:03:40.685 | INFO | "huge" : "", | 20260506 10:03:40.685 | INFO | "huge_shared" : " 27 GB", | 20260506 10:03:40.685 | INFO | "malloc" : " 14 GB", | 20260506 10:03:40.685 | INFO | "pinned" : " 29 GB" | 20260506 10:03:40.685 | INFO | } | 20260506 10:03:40.685 | INFO | Perf report for the last 5.13945 seconds: | 20260506 10:03:40.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:03:40.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:40.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:40.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:40.685 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:03:40.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:40.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:40.685 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:03:40.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:40.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:40.685 | INFO | Uploaded 36GiB to GPU in 1.60516s (22.4277GiB/s) | 20260506 10:03:40.686 | INFO | Downloaded 34GiB to GPU in 1.52316s (22.322GiB/s) | 20260506 10:03:45.824 | INFO | Analyzing chunk frames = [491008,491520) | 20260506 10:03:45.824 | INFO | MemInfo: { | 20260506 10:03:45.824 | INFO | "gpu" : " 17 GB", | 20260506 10:03:45.824 | INFO | "huge" : "", | 20260506 10:03:45.824 | INFO | "huge_shared" : " 27 GB", | 20260506 10:03:45.824 | INFO | "malloc" : " 14 GB", | 20260506 10:03:45.824 | INFO | "pinned" : " 29 GB" | 20260506 10:03:45.824 | INFO | } | 20260506 10:03:45.824 | INFO | Perf report for the last 5.13916 seconds: | 20260506 10:03:45.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:03:45.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:45.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:45.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:45.824 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:03:45.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:45.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:45.824 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:03:45.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:45.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:45.825 | INFO | Uploaded 36GiB to GPU in 1.60526s (22.4262GiB/s) | 20260506 10:03:45.825 | INFO | Downloaded 34GiB to GPU in 1.52338s (22.3188GiB/s) | 20260506 10:03:45.825 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:03:45.825 | INFO | Total mem usage: 16 GiB | 20260506 10:03:45.825 | INFO | All Time High Water Marks sum: 16.4 GiB (Savings: 2.51%) | 20260506 10:03:45.825 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -3.53%) | 20260506 10:03:45.825 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:03:45.825 | INFO | High Water Breakdown: | 20260506 10:03:45.825 | INFO | 60% : 9.27 GiB ( 95.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:03:45.825 | INFO | 39.9% : 6.16 GiB ( 92% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:03:45.825 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:03:45.825 | INFO | ------------------------------------------ | 20260506 10:03:45.825 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:03:45.825 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:03:45.825 | INFO | Total mem usage: 26.5 GiB | 20260506 10:03:45.825 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:03:45.825 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:03:45.825 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:03:45.825 | INFO | High Water Breakdown: | 20260506 10:03:45.825 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:03:45.825 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:03:45.825 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:03:45.825 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:03:45.825 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:03:45.825 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:03:45.825 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:03:45.825 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:03:45.825 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:03:45.825 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:03:45.825 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:03:45.825 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:03:45.825 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:03:45.825 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:03:45.825 | INFO | ------------------------------------------ | 20260506 10:03:45.825 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:03:45.825 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:03:45.826 | INFO | Total mem usage: 25.6 GiB | 20260506 10:03:45.826 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:03:45.826 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:03:45.826 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:03:45.826 | INFO | High Water Breakdown: | 20260506 10:03:45.826 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:03:45.826 | INFO | ------------------------------------------ | 20260506 10:03:45.826 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:03:45.826 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:03:45.826 | INFO | Total mem usage: 15.7 GiB | 20260506 10:03:45.826 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:03:45.826 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:03:45.826 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:03:45.826 | INFO | High Water Breakdown: | 20260506 10:03:45.826 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:03:45.826 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:03:45.826 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:03:45.826 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:03:45.826 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:03:45.826 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:03:45.826 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:03:45.826 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:03:45.826 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:03:45.826 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:03:45.826 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:03:45.826 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:03:45.826 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:03:45.826 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:03:45.826 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:03:45.826 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:03:45.826 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:03:45.826 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:03:45.826 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:03:45.826 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:03:45.826 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:03:45.826 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:03:45.826 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:03:45.826 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:03:45.826 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:03:45.826 | INFO | ------------------------------------------ | 20260506 10:03:45.826 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:03:45.826 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:03:45.826 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:03:45.826 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:03:45.826 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 10:03:45.826 | INFO | PerfReport: 0.29%, AggregateStats -- 8.63e-02 / -nan, 4.86e-03 / 0.00e+00 | 20260506 10:03:45.826 | INFO | PerfReport: 10.93%, Baseline -- 3.29e+00 / -nan, 1.82e-02 / 0.00e+00 | 20260506 10:03:45.826 | INFO | PerfReport: 3.06%, Histogram -- 9.22e-01 / -nan, 5.27e-02 / 0.00e+00 | 20260506 10:03:45.826 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.09e+00 / 0.00e+00 | 20260506 10:03:45.826 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.72e+01 / -nan, 4.13e-02 / 0.00e+00 | 20260506 10:03:45.826 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.24e+00 / -nan, 1.32e-01 / 0.00e+00 | 20260506 10:03:45.826 | INFO | PerfReport: 3.74%, Metrics -- 1.13e+00 / -nan, 1.73e-01 / 0.00e+00 | 20260506 10:03:45.826 | INFO | PerfReport: 1.45%, PulseSerializing -- 4.36e-01 / -nan, 6.16e-02 / 0.00e+00 | 20260506 10:03:45.826 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:03:45.826 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:03:45.826 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:03:45.826 | INFO | PerfReport: Full Execution -- 9.00e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 10:03:45.826 | INFO | PerfReport: 18.67%, Upload -- 1.68e+01 / 0.00e+00, 4.91e-01 / 0.00e+00 | 20260506 10:03:45.826 | INFO | PerfReport: 17.93%, Download -- 1.61e+01 / 0.00e+00, 8.41e-01 / 0.00e+00 | 20260506 10:03:45.826 | INFO | PerfReport: 33.48%, Compute -- 3.01e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 10:03:45.826 | INFO | PerfReport: 0.56%, LockUpload -- 5.08e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:03:45.826 | INFO | PerfReport: 0.00%, LockDownload -- 4.92e-04 / 0.00e+00, 1.14e-04 / 0.00e+00 | 20260506 10:03:45.826 | INFO | PerfReport: 29.34%, LockCompute -- 2.64e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 10:03:45.826 | INFO | PerfReport: 0.02%, Misc -- 1.97e-02 / 0.00e+00, 1.44e-01 / 0.00e+00 | 20260506 10:03:45.922 | INFO | Starting a round of Baz IO | 20260506 10:03:48.833 | INFO | Populating RTMetrics report | 20260506 10:03:48.833 | INFO | RTMetrics report populated | 20260506 10:03:48.833 | INFO | Opening tmp RTMetrics json file | 20260506 10:03:48.936 | INFO | Serializing RTMetrics to json | 20260506 10:03:48.944 | INFO | Serialization done, swapping tmp file now... | 20260506 10:03:48.948 | INFO | RTMetrics done | 20260506 10:03:49.262 | INFO | Finished a round of Baz IO: Wrote 1.942 GiB in 3.34 seconds | 20260506 10:03:49.262 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.90% / 0.07% / 50.04% | 20260506 10:03:49.262 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.73958% / 0.000139078% / 4.37101e-05% | 20260506 10:03:49.262 | INFO | Baz IO overall: | 20260506 10:03:49.262 | INFO | BytesWritten : 211870954514 | 20260506 10:03:49.262 | INFO | NumEvents : 101457227980 | 20260506 10:03:50.964 | INFO | Analyzing chunk frames = [491520,492032) | 20260506 10:03:50.964 | INFO | MemInfo: { | 20260506 10:03:50.964 | INFO | "gpu" : " 17 GB", | 20260506 10:03:50.964 | INFO | "huge" : "", | 20260506 10:03:50.964 | INFO | "huge_shared" : " 27 GB", | 20260506 10:03:50.964 | INFO | "malloc" : " 14 GB", | 20260506 10:03:50.964 | INFO | "pinned" : " 29 GB" | 20260506 10:03:50.964 | INFO | } | 20260506 10:03:50.964 | INFO | Perf report for the last 5.13939 seconds: | 20260506 10:03:50.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:03:50.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:50.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:50.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:50.964 | INFO | ANALYSIS : 57.5%, 2.8, 85.3ms, 0 | 20260506 10:03:50.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:50.964 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260506 10:03:50.964 | INFO | PRE_HQ : 56.6%, 2.1, 60.6ms, 0 | 20260506 10:03:50.964 | INFO | BAZWRITER : 64.6%, 4.7, 157.5ms, 0 | 20260506 10:03:50.964 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 10:03:50.964 | INFO | Uploaded 36GiB to GPU in 1.6777s (21.4579GiB/s) | 20260506 10:03:50.964 | INFO | Downloaded 38GiB to GPU in 1.74117s (21.8244GiB/s) | 20260506 10:03:56.103 | INFO | Analyzing chunk frames = [492032,492544) | 20260506 10:03:56.103 | INFO | MemInfo: { | 20260506 10:03:56.103 | INFO | "gpu" : " 17 GB", | 20260506 10:03:56.103 | INFO | "huge" : "", | 20260506 10:03:56.103 | INFO | "huge_shared" : " 27 GB", | 20260506 10:03:56.103 | INFO | "malloc" : " 14 GB", | 20260506 10:03:56.103 | INFO | "pinned" : " 29 GB" | 20260506 10:03:56.103 | INFO | } | 20260506 10:03:56.103 | INFO | Perf report for the last 5.13942 seconds: | 20260506 10:03:56.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:03:56.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:56.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:56.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:56.103 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 10:03:56.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:56.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:56.103 | INFO | PRE_HQ : 29.9%, 1.0, 15.7ms, 0 | 20260506 10:03:56.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:56.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:03:56.103 | INFO | Uploaded 36GiB to GPU in 1.60269s (22.4623GiB/s) | 20260506 10:03:56.104 | INFO | Downloaded 34GiB to GPU in 1.51956s (22.3749GiB/s) | 20260506 10:04:01.242 | INFO | Analyzing chunk frames = [492544,493056) | 20260506 10:04:01.242 | INFO | MemInfo: { | 20260506 10:04:01.242 | INFO | "gpu" : " 17 GB", | 20260506 10:04:01.242 | INFO | "huge" : "", | 20260506 10:04:01.242 | INFO | "huge_shared" : " 27 GB", | 20260506 10:04:01.242 | INFO | "malloc" : " 14 GB", | 20260506 10:04:01.242 | INFO | "pinned" : " 29 GB" | 20260506 10:04:01.242 | INFO | } | 20260506 10:04:01.242 | INFO | Perf report for the last 5.13914 seconds: | 20260506 10:04:01.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:04:01.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:01.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:01.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:01.242 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 10:04:01.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:01.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:01.242 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:04:01.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:01.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:01.243 | INFO | Uploaded 36GiB to GPU in 1.60834s (22.3834GiB/s) | 20260506 10:04:01.243 | INFO | Downloaded 34GiB to GPU in 1.52636s (22.2752GiB/s) | 20260506 10:04:06.382 | INFO | Analyzing chunk frames = [493056,493568) | 20260506 10:04:06.382 | INFO | MemInfo: { | 20260506 10:04:06.382 | INFO | "gpu" : " 17 GB", | 20260506 10:04:06.382 | INFO | "huge" : "", | 20260506 10:04:06.382 | INFO | "huge_shared" : " 27 GB", | 20260506 10:04:06.382 | INFO | "malloc" : " 14 GB", | 20260506 10:04:06.382 | INFO | "pinned" : " 29 GB" | 20260506 10:04:06.382 | INFO | } | 20260506 10:04:06.382 | INFO | Perf report for the last 5.13947 seconds: | 20260506 10:04:06.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:04:06.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:06.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:06.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:06.382 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:04:06.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:06.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:06.382 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 10:04:06.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:06.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:06.382 | INFO | Uploaded 36GiB to GPU in 1.60515s (22.4279GiB/s) | 20260506 10:04:06.382 | INFO | Downloaded 34GiB to GPU in 1.52424s (22.3062GiB/s) | 20260506 10:04:11.521 | INFO | Analyzing chunk frames = [493568,494080) | 20260506 10:04:11.521 | INFO | MemInfo: { | 20260506 10:04:11.521 | INFO | "gpu" : " 17 GB", | 20260506 10:04:11.521 | INFO | "huge" : "", | 20260506 10:04:11.521 | INFO | "huge_shared" : " 27 GB", | 20260506 10:04:11.521 | INFO | "malloc" : " 14 GB", | 20260506 10:04:11.521 | INFO | "pinned" : " 29 GB" | 20260506 10:04:11.521 | INFO | } | 20260506 10:04:11.521 | INFO | Perf report for the last 5.13952 seconds: | 20260506 10:04:11.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:04:11.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:11.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:11.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:11.521 | INFO | ANALYSIS : 56.4%, 3.0, 87.1ms, 0 | 20260506 10:04:11.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:11.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:11.521 | INFO | PRE_HQ : 29.3%, 1.0, 15.1ms, 0 | 20260506 10:04:11.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:11.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:11.522 | INFO | Uploaded 36GiB to GPU in 1.60426s (22.4402GiB/s) | 20260506 10:04:11.522 | INFO | Downloaded 34GiB to GPU in 1.52217s (22.3365GiB/s) | 20260506 10:04:16.672 | INFO | Analyzing chunk frames = [494080,494592) | 20260506 10:04:16.672 | INFO | MemInfo: { | 20260506 10:04:16.672 | INFO | "gpu" : " 17 GB", | 20260506 10:04:16.672 | INFO | "huge" : "", | 20260506 10:04:16.672 | INFO | "huge_shared" : " 27 GB", | 20260506 10:04:16.672 | INFO | "malloc" : " 14 GB", | 20260506 10:04:16.672 | INFO | "pinned" : " 29 GB" | 20260506 10:04:16.672 | INFO | } | 20260506 10:04:16.672 | INFO | Perf report for the last 5.15101 seconds: | 20260506 10:04:16.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:04:16.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:16.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:16.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:16.672 | INFO | ANALYSIS : 56.6%, 3.0, 88.5ms, 0 | 20260506 10:04:16.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:16.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:16.673 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260506 10:04:16.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:16.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:16.673 | INFO | Uploaded 36GiB to GPU in 1.60534s (22.4251GiB/s) | 20260506 10:04:16.673 | INFO | Downloaded 34GiB to GPU in 1.52288s (22.3261GiB/s) | 20260506 10:04:21.800 | INFO | Analyzing chunk frames = [494592,495104) | 20260506 10:04:21.800 | INFO | MemInfo: { | 20260506 10:04:21.800 | INFO | "gpu" : " 17 GB", | 20260506 10:04:21.800 | INFO | "huge" : "", | 20260506 10:04:21.800 | INFO | "huge_shared" : " 27 GB", | 20260506 10:04:21.800 | INFO | "malloc" : " 14 GB", | 20260506 10:04:21.800 | INFO | "pinned" : " 29 GB" | 20260506 10:04:21.800 | INFO | } | 20260506 10:04:21.800 | INFO | Perf report for the last 5.12734 seconds: | 20260506 10:04:21.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:04:21.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:21.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:21.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:21.800 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260506 10:04:21.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:21.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:21.800 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 10:04:21.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:21.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:21.800 | INFO | Uploaded 36GiB to GPU in 1.60611s (22.4144GiB/s) | 20260506 10:04:21.800 | INFO | Downloaded 34GiB to GPU in 1.52412s (22.308GiB/s) | 20260506 10:04:26.939 | INFO | Analyzing chunk frames = [495104,495616) | 20260506 10:04:26.939 | INFO | MemInfo: { | 20260506 10:04:26.939 | INFO | "gpu" : " 17 GB", | 20260506 10:04:26.939 | INFO | "huge" : "", | 20260506 10:04:26.939 | INFO | "huge_shared" : " 27 GB", | 20260506 10:04:26.939 | INFO | "malloc" : " 14 GB", | 20260506 10:04:26.939 | INFO | "pinned" : " 29 GB" | 20260506 10:04:26.939 | INFO | } | 20260506 10:04:26.939 | INFO | Perf report for the last 5.13915 seconds: | 20260506 10:04:26.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:04:26.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:26.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:26.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:26.939 | INFO | ANALYSIS : 57.1%, 3.0, 89.0ms, 0 | 20260506 10:04:26.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:26.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:26.939 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:04:26.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:26.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:26.939 | INFO | Uploaded 36GiB to GPU in 1.60452s (22.4366GiB/s) | 20260506 10:04:26.939 | INFO | Downloaded 34GiB to GPU in 1.52329s (22.3201GiB/s) | 20260506 10:04:27.059 | INFO | Starting a round of Baz IO | 20260506 10:04:29.952 | INFO | Populating RTMetrics report | 20260506 10:04:29.952 | INFO | RTMetrics report populated | 20260506 10:04:29.952 | INFO | Opening tmp RTMetrics json file | 20260506 10:04:29.975 | INFO | Serializing RTMetrics to json | 20260506 10:04:30.086 | INFO | Serialization done, swapping tmp file now... | 20260506 10:04:30.089 | INFO | RTMetrics done | 20260506 10:04:30.407 | INFO | Finished a round of Baz IO: Wrote 1.943 GiB in 3.348 seconds | 20260506 10:04:30.407 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.83% / 0.07% / 50.10% | 20260506 10:04:30.407 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.69686% / 0.00017484% / 3.17891e-05% | 20260506 10:04:30.407 | INFO | Baz IO overall: | 20260506 10:04:30.407 | INFO | BytesWritten : 213957485032 | 20260506 10:04:30.407 | INFO | NumEvents : 102469735635 | 20260506 10:04:32.078 | INFO | Analyzing chunk frames = [495616,496128) | 20260506 10:04:32.078 | INFO | MemInfo: { | 20260506 10:04:32.078 | INFO | "gpu" : " 17 GB", | 20260506 10:04:32.078 | INFO | "huge" : "", | 20260506 10:04:32.078 | INFO | "huge_shared" : " 27 GB", | 20260506 10:04:32.078 | INFO | "malloc" : " 14 GB", | 20260506 10:04:32.078 | INFO | "pinned" : " 29 GB" | 20260506 10:04:32.078 | INFO | } | 20260506 10:04:32.079 | INFO | Perf report for the last 5.13957 seconds: | 20260506 10:04:32.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:04:32.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:32.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:32.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:32.079 | INFO | ANALYSIS : 57.6%, 2.8, 85.2ms, 0 | 20260506 10:04:32.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:32.079 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260506 10:04:32.079 | INFO | PRE_HQ : 56.7%, 2.1, 61.3ms, 0 | 20260506 10:04:32.079 | INFO | BAZWRITER : 65.2%, 4.7, 158.5ms, 0 | 20260506 10:04:32.079 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260506 10:04:32.079 | INFO | Uploaded 36GiB to GPU in 1.66687s (21.5974GiB/s) | 20260506 10:04:32.079 | INFO | Downloaded 38GiB to GPU in 1.73093s (21.9535GiB/s) | 20260506 10:04:37.218 | INFO | Analyzing chunk frames = [496128,496640) | 20260506 10:04:37.218 | INFO | MemInfo: { | 20260506 10:04:37.218 | INFO | "gpu" : " 17 GB", | 20260506 10:04:37.218 | INFO | "huge" : "", | 20260506 10:04:37.218 | INFO | "huge_shared" : " 27 GB", | 20260506 10:04:37.218 | INFO | "malloc" : " 14 GB", | 20260506 10:04:37.218 | INFO | "pinned" : " 29 GB" | 20260506 10:04:37.218 | INFO | } | 20260506 10:04:37.218 | INFO | Perf report for the last 5.13946 seconds: | 20260506 10:04:37.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:04:37.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:37.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:37.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:37.218 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260506 10:04:37.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:37.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:37.218 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:04:37.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:37.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:37.219 | INFO | Uploaded 36GiB to GPU in 1.60269s (22.4623GiB/s) | 20260506 10:04:37.219 | INFO | Downloaded 34GiB to GPU in 1.51971s (22.3727GiB/s) | 20260506 10:04:42.357 | INFO | Analyzing chunk frames = [496640,497152) | 20260506 10:04:42.357 | INFO | MemInfo: { | 20260506 10:04:42.357 | INFO | "gpu" : " 17 GB", | 20260506 10:04:42.357 | INFO | "huge" : "", | 20260506 10:04:42.357 | INFO | "huge_shared" : " 27 GB", | 20260506 10:04:42.357 | INFO | "malloc" : " 14 GB", | 20260506 10:04:42.357 | INFO | "pinned" : " 29 GB" | 20260506 10:04:42.357 | INFO | } | 20260506 10:04:42.357 | INFO | Perf report for the last 5.13909 seconds: | 20260506 10:04:42.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:04:42.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:42.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:42.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:42.357 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 10:04:42.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:42.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:42.357 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:04:42.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:42.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:42.358 | INFO | Uploaded 36GiB to GPU in 1.60693s (22.403GiB/s) | 20260506 10:04:42.358 | INFO | Downloaded 34GiB to GPU in 1.52434s (22.3047GiB/s) | 20260506 10:04:47.496 | INFO | Analyzing chunk frames = [497152,497664) | 20260506 10:04:47.496 | INFO | MemInfo: { | 20260506 10:04:47.496 | INFO | "gpu" : " 17 GB", | 20260506 10:04:47.496 | INFO | "huge" : "", | 20260506 10:04:47.496 | INFO | "huge_shared" : " 27 GB", | 20260506 10:04:47.497 | INFO | "malloc" : " 14 GB", | 20260506 10:04:47.497 | INFO | "pinned" : " 29 GB" | 20260506 10:04:47.497 | INFO | } | 20260506 10:04:47.497 | INFO | Perf report for the last 5.1396 seconds: | 20260506 10:04:47.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:04:47.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:47.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:47.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:47.497 | INFO | ANALYSIS : 56.5%, 3.0, 87.7ms, 0 | 20260506 10:04:47.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:47.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:47.497 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 10:04:47.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:47.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:47.497 | INFO | Uploaded 36GiB to GPU in 1.60649s (22.409GiB/s) | 20260506 10:04:47.497 | INFO | Downloaded 34GiB to GPU in 1.525s (22.2951GiB/s) | 20260506 10:04:52.636 | INFO | Analyzing chunk frames = [497664,498176) | 20260506 10:04:52.636 | INFO | MemInfo: { | 20260506 10:04:52.636 | INFO | "gpu" : " 17 GB", | 20260506 10:04:52.636 | INFO | "huge" : "", | 20260506 10:04:52.636 | INFO | "huge_shared" : " 27 GB", | 20260506 10:04:52.636 | INFO | "malloc" : " 14 GB", | 20260506 10:04:52.636 | INFO | "pinned" : " 29 GB" | 20260506 10:04:52.636 | INFO | } | 20260506 10:04:52.636 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:04:52.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:04:52.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:52.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:52.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:52.636 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:04:52.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:52.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:52.636 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:04:52.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:52.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:52.636 | INFO | Uploaded 36GiB to GPU in 1.60528s (22.426GiB/s) | 20260506 10:04:52.636 | INFO | Downloaded 34GiB to GPU in 1.52276s (22.3279GiB/s) | 20260506 10:04:57.775 | INFO | Analyzing chunk frames = [498176,498688) | 20260506 10:04:57.775 | INFO | MemInfo: { | 20260506 10:04:57.775 | INFO | "gpu" : " 17 GB", | 20260506 10:04:57.775 | INFO | "huge" : "", | 20260506 10:04:57.775 | INFO | "huge_shared" : " 27 GB", | 20260506 10:04:57.775 | INFO | "malloc" : " 14 GB", | 20260506 10:04:57.775 | INFO | "pinned" : " 29 GB" | 20260506 10:04:57.775 | INFO | } | 20260506 10:04:57.775 | INFO | Perf report for the last 5.13921 seconds: | 20260506 10:04:57.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:04:57.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:57.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:57.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:57.775 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 10:04:57.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:57.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:57.775 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:04:57.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:57.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:04:57.776 | INFO | Uploaded 36GiB to GPU in 1.60477s (22.4331GiB/s) | 20260506 10:04:57.776 | INFO | Downloaded 34GiB to GPU in 1.52209s (22.3378GiB/s) | 20260506 10:05:02.914 | INFO | Analyzing chunk frames = [498688,499200) | 20260506 10:05:02.914 | INFO | MemInfo: { | 20260506 10:05:02.914 | INFO | "gpu" : " 17 GB", | 20260506 10:05:02.914 | INFO | "huge" : "", | 20260506 10:05:02.914 | INFO | "huge_shared" : " 27 GB", | 20260506 10:05:02.915 | INFO | "malloc" : " 14 GB", | 20260506 10:05:02.915 | INFO | "pinned" : " 29 GB" | 20260506 10:05:02.915 | INFO | } | 20260506 10:05:02.915 | INFO | Perf report for the last 5.13947 seconds: | 20260506 10:05:02.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:05:02.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:02.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:02.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:02.915 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:05:02.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:02.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:02.915 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:05:02.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:02.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:02.915 | INFO | Uploaded 36GiB to GPU in 1.60584s (22.4182GiB/s) | 20260506 10:05:02.915 | INFO | Downloaded 34GiB to GPU in 1.52339s (22.3187GiB/s) | 20260506 10:05:08.054 | INFO | Analyzing chunk frames = [499200,499712) | 20260506 10:05:08.054 | INFO | MemInfo: { | 20260506 10:05:08.054 | INFO | "gpu" : " 17 GB", | 20260506 10:05:08.054 | INFO | "huge" : "", | 20260506 10:05:08.054 | INFO | "huge_shared" : " 27 GB", | 20260506 10:05:08.054 | INFO | "malloc" : " 14 GB", | 20260506 10:05:08.054 | INFO | "pinned" : " 29 GB" | 20260506 10:05:08.054 | INFO | } | 20260506 10:05:08.054 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:05:08.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:05:08.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:08.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:08.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:08.054 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:05:08.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:08.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:08.054 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 10:05:08.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:08.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:08.054 | INFO | Uploaded 36GiB to GPU in 1.60473s (22.4337GiB/s) | 20260506 10:05:08.054 | INFO | Downloaded 34GiB to GPU in 1.52379s (22.3128GiB/s) | 20260506 10:05:08.055 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:05:08.055 | INFO | Total mem usage: 16 GiB | 20260506 10:05:08.055 | INFO | All Time High Water Marks sum: 16.4 GiB (Savings: 2.51%) | 20260506 10:05:08.055 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -2.83%) | 20260506 10:05:08.055 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:05:08.055 | INFO | High Water Breakdown: | 20260506 10:05:08.055 | INFO | 60.1% : 9.35 GiB ( 96.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:05:08.055 | INFO | 39.8% : 6.18 GiB ( 92.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:05:08.055 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:05:08.055 | INFO | ------------------------------------------ | 20260506 10:05:08.055 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:05:08.055 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:05:08.055 | INFO | Total mem usage: 26.5 GiB | 20260506 10:05:08.055 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:05:08.055 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:05:08.055 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:05:08.055 | INFO | High Water Breakdown: | 20260506 10:05:08.055 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:05:08.055 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:05:08.055 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:05:08.055 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:05:08.055 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:05:08.055 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:05:08.055 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:05:08.055 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:05:08.055 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:05:08.055 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:05:08.055 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:05:08.055 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:05:08.055 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:05:08.055 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:05:08.055 | INFO | ------------------------------------------ | 20260506 10:05:08.055 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:05:08.055 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:05:08.055 | INFO | Total mem usage: 25.6 GiB | 20260506 10:05:08.055 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:05:08.055 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:05:08.055 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:05:08.055 | INFO | High Water Breakdown: | 20260506 10:05:08.055 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:05:08.055 | INFO | ------------------------------------------ | 20260506 10:05:08.055 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:05:08.055 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:05:08.055 | INFO | Total mem usage: 15.7 GiB | 20260506 10:05:08.055 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:05:08.055 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:05:08.055 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:05:08.055 | INFO | High Water Breakdown: | 20260506 10:05:08.055 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:05:08.055 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:05:08.055 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:05:08.055 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:05:08.055 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:05:08.055 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:05:08.055 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:05:08.055 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:05:08.055 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:05:08.055 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:05:08.055 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:05:08.055 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:05:08.055 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:05:08.055 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:05:08.055 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:05:08.055 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:05:08.055 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:05:08.056 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:05:08.056 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:05:08.056 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:05:08.056 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:05:08.056 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:05:08.056 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:05:08.056 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:05:08.056 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:05:08.056 | INFO | ------------------------------------------ | 20260506 10:05:08.056 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:05:08.056 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:05:08.056 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:05:08.056 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:05:08.056 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 10:05:08.056 | INFO | PerfReport: 0.29%, AggregateStats -- 8.64e-02 / -nan, 7.71e-03 / 0.00e+00 | 20260506 10:05:08.056 | INFO | PerfReport: 10.92%, Baseline -- 3.29e+00 / -nan, 1.70e-02 / 0.00e+00 | 20260506 10:05:08.056 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.18e-02 / 0.00e+00 | 20260506 10:05:08.056 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 2.97e+00 / 0.00e+00 | 20260506 10:05:08.056 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.72e+01 / -nan, 4.15e-02 / 0.00e+00 | 20260506 10:05:08.056 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.24e+00 / -nan, 1.32e-01 / 0.00e+00 | 20260506 10:05:08.056 | INFO | PerfReport: 3.73%, Metrics -- 1.13e+00 / -nan, 1.75e-01 / 0.00e+00 | 20260506 10:05:08.056 | INFO | PerfReport: 1.45%, PulseSerializing -- 4.36e-01 / -nan, 6.24e-02 / 0.00e+00 | 20260506 10:05:08.056 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:05:08.056 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:05:08.056 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:05:08.056 | INFO | PerfReport: Full Execution -- 8.99e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:05:08.056 | INFO | PerfReport: 18.69%, Upload -- 1.68e+01 / 0.00e+00, 4.85e-01 / 0.00e+00 | 20260506 10:05:08.056 | INFO | PerfReport: 17.95%, Download -- 1.61e+01 / 0.00e+00, 8.33e-01 / 0.00e+00 | 20260506 10:05:08.056 | INFO | PerfReport: 33.50%, Compute -- 3.01e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 10:05:08.056 | INFO | PerfReport: 0.57%, LockUpload -- 5.09e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:05:08.056 | INFO | PerfReport: 0.00%, LockDownload -- 4.86e-04 / 0.00e+00, 2.22e-04 / 0.00e+00 | 20260506 10:05:08.056 | INFO | PerfReport: 29.28%, LockCompute -- 2.63e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260506 10:05:08.056 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260506 10:05:08.151 | INFO | Starting a round of Baz IO | 20260506 10:05:11.065 | INFO | Populating RTMetrics report | 20260506 10:05:11.065 | INFO | RTMetrics report populated | 20260506 10:05:11.065 | INFO | Opening tmp RTMetrics json file | 20260506 10:05:11.159 | INFO | Serializing RTMetrics to json | 20260506 10:05:11.164 | INFO | Serialization done, swapping tmp file now... | 20260506 10:05:11.168 | INFO | RTMetrics done | 20260506 10:05:11.363 | INFO | Finished a round of Baz IO: Wrote 1.945 GiB in 3.211 seconds | 20260506 10:05:11.363 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.77% / 0.07% / 50.17% | 20260506 10:05:11.363 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.67494% / 0.000147025% / 5.96046e-05% | 20260506 10:05:11.363 | INFO | Baz IO overall: | 20260506 10:05:11.363 | INFO | BytesWritten : 216045780047 | 20260506 10:05:11.363 | INFO | NumEvents : 103483232095 | 20260506 10:05:13.193 | INFO | Analyzing chunk frames = [499712,500224) | 20260506 10:05:13.193 | INFO | MemInfo: { | 20260506 10:05:13.193 | INFO | "gpu" : " 17 GB", | 20260506 10:05:13.193 | INFO | "huge" : "", | 20260506 10:05:13.193 | INFO | "huge_shared" : " 27 GB", | 20260506 10:05:13.193 | INFO | "malloc" : " 14 GB", | 20260506 10:05:13.193 | INFO | "pinned" : " 29 GB" | 20260506 10:05:13.193 | INFO | } | 20260506 10:05:13.193 | INFO | Perf report for the last 5.13927 seconds: | 20260506 10:05:13.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:05:13.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:13.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:13.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:13.193 | INFO | ANALYSIS : 57.6%, 2.9, 86.4ms, 0 | 20260506 10:05:13.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:13.193 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260506 10:05:13.193 | INFO | PRE_HQ : 56.7%, 2.1, 61.3ms, 0 | 20260506 10:05:13.193 | INFO | BAZWRITER : 62.1%, 4.7, 151.0ms, 0 | 20260506 10:05:13.193 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.3ms, 0 | 20260506 10:05:13.194 | INFO | Uploaded 36GiB to GPU in 1.66652s (21.6019GiB/s) | 20260506 10:05:13.194 | INFO | Downloaded 38GiB to GPU in 1.73036s (21.9608GiB/s) | 20260506 10:05:18.332 | INFO | Analyzing chunk frames = [500224,500736) | 20260506 10:05:18.332 | INFO | MemInfo: { | 20260506 10:05:18.332 | INFO | "gpu" : " 17 GB", | 20260506 10:05:18.332 | INFO | "huge" : "", | 20260506 10:05:18.332 | INFO | "huge_shared" : " 27 GB", | 20260506 10:05:18.332 | INFO | "malloc" : " 14 GB", | 20260506 10:05:18.333 | INFO | "pinned" : " 29 GB" | 20260506 10:05:18.333 | INFO | } | 20260506 10:05:18.333 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:05:18.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:05:18.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:18.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:18.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:18.333 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260506 10:05:18.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:18.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:18.333 | INFO | PRE_HQ : 29.8%, 1.0, 15.7ms, 0 | 20260506 10:05:18.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:18.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:18.333 | INFO | Uploaded 36GiB to GPU in 1.60291s (22.4592GiB/s) | 20260506 10:05:18.333 | INFO | Downloaded 34GiB to GPU in 1.52013s (22.3665GiB/s) | 20260506 10:05:23.472 | INFO | Analyzing chunk frames = [500736,501248) | 20260506 10:05:23.472 | INFO | MemInfo: { | 20260506 10:05:23.472 | INFO | "gpu" : " 17 GB", | 20260506 10:05:23.472 | INFO | "huge" : "", | 20260506 10:05:23.472 | INFO | "huge_shared" : " 27 GB", | 20260506 10:05:23.472 | INFO | "malloc" : " 14 GB", | 20260506 10:05:23.472 | INFO | "pinned" : " 29 GB" | 20260506 10:05:23.472 | INFO | } | 20260506 10:05:23.472 | INFO | Perf report for the last 5.13939 seconds: | 20260506 10:05:23.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:05:23.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:23.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:23.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:23.472 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260506 10:05:23.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:23.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:23.472 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:05:23.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:23.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:23.473 | INFO | Uploaded 36GiB to GPU in 1.60645s (22.4096GiB/s) | 20260506 10:05:23.473 | INFO | Downloaded 34GiB to GPU in 1.5243s (22.3053GiB/s) | 20260506 10:05:28.611 | INFO | Analyzing chunk frames = [501248,501760) | 20260506 10:05:28.611 | INFO | MemInfo: { | 20260506 10:05:28.611 | INFO | "gpu" : " 17 GB", | 20260506 10:05:28.611 | INFO | "huge" : "", | 20260506 10:05:28.611 | INFO | "huge_shared" : " 27 GB", | 20260506 10:05:28.611 | INFO | "malloc" : " 14 GB", | 20260506 10:05:28.611 | INFO | "pinned" : " 29 GB" | 20260506 10:05:28.611 | INFO | } | 20260506 10:05:28.611 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:05:28.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:05:28.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:28.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:28.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:28.611 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:05:28.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:28.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:28.612 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:05:28.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:28.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:28.612 | INFO | Uploaded 36GiB to GPU in 1.60655s (22.4082GiB/s) | 20260506 10:05:28.612 | INFO | Downloaded 34GiB to GPU in 1.52429s (22.3055GiB/s) | 20260506 10:05:33.750 | INFO | Analyzing chunk frames = [501760,502272) | 20260506 10:05:33.751 | INFO | MemInfo: { | 20260506 10:05:33.751 | INFO | "gpu" : " 17 GB", | 20260506 10:05:33.751 | INFO | "huge" : "", | 20260506 10:05:33.751 | INFO | "huge_shared" : " 27 GB", | 20260506 10:05:33.751 | INFO | "malloc" : " 14 GB", | 20260506 10:05:33.751 | INFO | "pinned" : " 29 GB" | 20260506 10:05:33.751 | INFO | } | 20260506 10:05:33.751 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:05:33.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:05:33.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:33.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:33.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:33.751 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:05:33.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:33.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:33.751 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 10:05:33.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:33.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:33.751 | INFO | Uploaded 36GiB to GPU in 1.60427s (22.4401GiB/s) | 20260506 10:05:33.751 | INFO | Downloaded 34GiB to GPU in 1.52183s (22.3415GiB/s) | 20260506 10:05:38.890 | INFO | Analyzing chunk frames = [502272,502784) | 20260506 10:05:38.890 | INFO | MemInfo: { | 20260506 10:05:38.890 | INFO | "gpu" : " 17 GB", | 20260506 10:05:38.890 | INFO | "huge" : "", | 20260506 10:05:38.890 | INFO | "huge_shared" : " 27 GB", | 20260506 10:05:38.890 | INFO | "malloc" : " 14 GB", | 20260506 10:05:38.890 | INFO | "pinned" : " 29 GB" | 20260506 10:05:38.890 | INFO | } | 20260506 10:05:38.890 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:05:38.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:05:38.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:38.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:38.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:38.890 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:05:38.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:38.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:38.890 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:05:38.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:38.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:38.891 | INFO | Uploaded 36GiB to GPU in 1.60489s (22.4314GiB/s) | 20260506 10:05:38.891 | INFO | Downloaded 34GiB to GPU in 1.52274s (22.3282GiB/s) | 20260506 10:05:44.029 | INFO | Analyzing chunk frames = [502784,503296) | 20260506 10:05:44.029 | INFO | MemInfo: { | 20260506 10:05:44.029 | INFO | "gpu" : " 17 GB", | 20260506 10:05:44.029 | INFO | "huge" : "", | 20260506 10:05:44.029 | INFO | "huge_shared" : " 27 GB", | 20260506 10:05:44.029 | INFO | "malloc" : " 14 GB", | 20260506 10:05:44.030 | INFO | "pinned" : " 29 GB" | 20260506 10:05:44.030 | INFO | } | 20260506 10:05:44.030 | INFO | Perf report for the last 5.13943 seconds: | 20260506 10:05:44.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:05:44.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:44.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:44.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:44.030 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:05:44.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:44.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:44.030 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:05:44.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:44.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:44.030 | INFO | Uploaded 36GiB to GPU in 1.60476s (22.4332GiB/s) | 20260506 10:05:44.030 | INFO | Downloaded 34GiB to GPU in 1.5229s (22.3259GiB/s) | 20260506 10:05:49.169 | INFO | Analyzing chunk frames = [503296,503808) | 20260506 10:05:49.169 | INFO | MemInfo: { | 20260506 10:05:49.169 | INFO | "gpu" : " 17 GB", | 20260506 10:05:49.169 | INFO | "huge" : "", | 20260506 10:05:49.169 | INFO | "huge_shared" : " 27 GB", | 20260506 10:05:49.169 | INFO | "malloc" : " 14 GB", | 20260506 10:05:49.169 | INFO | "pinned" : " 29 GB" | 20260506 10:05:49.169 | INFO | } | 20260506 10:05:49.169 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:05:49.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:05:49.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:49.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:49.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:49.169 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:05:49.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:49.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:49.169 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:05:49.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:49.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:49.169 | INFO | Uploaded 36GiB to GPU in 1.60566s (22.4206GiB/s) | 20260506 10:05:49.169 | INFO | Downloaded 34GiB to GPU in 1.52302s (22.3241GiB/s) | 20260506 10:05:49.266 | INFO | Starting a round of Baz IO | 20260506 10:05:52.182 | INFO | Populating RTMetrics report | 20260506 10:05:52.182 | INFO | RTMetrics report populated | 20260506 10:05:52.182 | INFO | Opening tmp RTMetrics json file | 20260506 10:05:52.208 | INFO | Serializing RTMetrics to json | 20260506 10:05:52.277 | INFO | Serialization done, swapping tmp file now... | 20260506 10:05:52.317 | INFO | RTMetrics done | 20260506 10:05:52.553 | INFO | Finished a round of Baz IO: Wrote 1.947 GiB in 3.286 seconds | 20260506 10:05:52.553 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.70% / 0.06% / 50.23% | 20260506 10:05:52.553 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.67835% / 0.000162919% / 3.97364e-05% | 20260506 10:05:52.553 | INFO | Baz IO overall: | 20260506 10:05:52.553 | INFO | BytesWritten : 218136653052 | 20260506 10:05:52.553 | INFO | NumEvents : 104498180929 | 20260506 10:05:54.308 | INFO | Analyzing chunk frames = [503808,504320) | 20260506 10:05:54.308 | INFO | MemInfo: { | 20260506 10:05:54.308 | INFO | "gpu" : " 17 GB", | 20260506 10:05:54.308 | INFO | "huge" : "", | 20260506 10:05:54.308 | INFO | "huge_shared" : " 27 GB", | 20260506 10:05:54.308 | INFO | "malloc" : " 14 GB", | 20260506 10:05:54.308 | INFO | "pinned" : " 29 GB" | 20260506 10:05:54.308 | INFO | } | 20260506 10:05:54.308 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:05:54.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:05:54.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:54.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:54.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:54.308 | INFO | ANALYSIS : 57.6%, 2.9, 87.3ms, 0 | 20260506 10:05:54.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:54.308 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260506 10:05:54.308 | INFO | PRE_HQ : 56.8%, 2.1, 61.2ms, 0 | 20260506 10:05:54.308 | INFO | BAZWRITER : 63.6%, 4.6, 149.8ms, 0 | 20260506 10:05:54.308 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 10:05:54.308 | INFO | Uploaded 36GiB to GPU in 1.65753s (21.719GiB/s) | 20260506 10:05:54.309 | INFO | Downloaded 38GiB to GPU in 1.72208s (22.0664GiB/s) | 20260506 10:05:59.447 | INFO | Analyzing chunk frames = [504320,504832) | 20260506 10:05:59.447 | INFO | MemInfo: { | 20260506 10:05:59.447 | INFO | "gpu" : " 17 GB", | 20260506 10:05:59.447 | INFO | "huge" : "", | 20260506 10:05:59.447 | INFO | "huge_shared" : " 27 GB", | 20260506 10:05:59.448 | INFO | "malloc" : " 14 GB", | 20260506 10:05:59.448 | INFO | "pinned" : " 29 GB" | 20260506 10:05:59.448 | INFO | } | 20260506 10:05:59.448 | INFO | Perf report for the last 5.13948 seconds: | 20260506 10:05:59.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:05:59.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:59.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:59.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:59.448 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:05:59.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:59.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:59.448 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:05:59.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:59.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:05:59.448 | INFO | Uploaded 36GiB to GPU in 1.60301s (22.4577GiB/s) | 20260506 10:05:59.448 | INFO | Downloaded 34GiB to GPU in 1.51997s (22.3688GiB/s) | 20260506 10:06:04.587 | INFO | Analyzing chunk frames = [504832,505344) | 20260506 10:06:04.587 | INFO | MemInfo: { | 20260506 10:06:04.587 | INFO | "gpu" : " 17 GB", | 20260506 10:06:04.587 | INFO | "huge" : "", | 20260506 10:06:04.587 | INFO | "huge_shared" : " 27 GB", | 20260506 10:06:04.587 | INFO | "malloc" : " 14 GB", | 20260506 10:06:04.587 | INFO | "pinned" : " 29 GB" | 20260506 10:06:04.587 | INFO | } | 20260506 10:06:04.587 | INFO | Perf report for the last 5.13923 seconds: | 20260506 10:06:04.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:06:04.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:04.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:04.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:04.587 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260506 10:06:04.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:04.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:04.587 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 10:06:04.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:04.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:04.587 | INFO | Uploaded 36GiB to GPU in 1.60709s (22.4007GiB/s) | 20260506 10:06:04.587 | INFO | Downloaded 34GiB to GPU in 1.52465s (22.3002GiB/s) | 20260506 10:06:09.726 | INFO | Analyzing chunk frames = [505344,505856) | 20260506 10:06:09.726 | INFO | MemInfo: { | 20260506 10:06:09.726 | INFO | "gpu" : " 17 GB", | 20260506 10:06:09.726 | INFO | "huge" : "", | 20260506 10:06:09.726 | INFO | "huge_shared" : " 27 GB", | 20260506 10:06:09.726 | INFO | "malloc" : " 14 GB", | 20260506 10:06:09.726 | INFO | "pinned" : " 29 GB" | 20260506 10:06:09.726 | INFO | } | 20260506 10:06:09.726 | INFO | Perf report for the last 5.13944 seconds: | 20260506 10:06:09.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:06:09.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:09.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:09.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:09.726 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:06:09.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:09.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:09.726 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 10:06:09.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:09.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:09.727 | INFO | Uploaded 36GiB to GPU in 1.60659s (22.4077GiB/s) | 20260506 10:06:09.727 | INFO | Downloaded 34GiB to GPU in 1.52393s (22.3108GiB/s) | 20260506 10:06:14.865 | INFO | Analyzing chunk frames = [505856,506368) | 20260506 10:06:14.865 | INFO | MemInfo: { | 20260506 10:06:14.865 | INFO | "gpu" : " 17 GB", | 20260506 10:06:14.865 | INFO | "huge" : "", | 20260506 10:06:14.865 | INFO | "huge_shared" : " 27 GB", | 20260506 10:06:14.865 | INFO | "malloc" : " 14 GB", | 20260506 10:06:14.865 | INFO | "pinned" : " 29 GB" | 20260506 10:06:14.865 | INFO | } | 20260506 10:06:14.865 | INFO | Perf report for the last 5.13917 seconds: | 20260506 10:06:14.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:06:14.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:14.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:14.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:14.866 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:06:14.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:14.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:14.866 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:06:14.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:14.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:14.866 | INFO | Uploaded 36GiB to GPU in 1.60446s (22.4374GiB/s) | 20260506 10:06:14.866 | INFO | Downloaded 34GiB to GPU in 1.52134s (22.3488GiB/s) | 20260506 10:06:20.005 | INFO | Analyzing chunk frames = [506368,506880) | 20260506 10:06:20.005 | INFO | MemInfo: { | 20260506 10:06:20.005 | INFO | "gpu" : " 17 GB", | 20260506 10:06:20.005 | INFO | "huge" : "", | 20260506 10:06:20.005 | INFO | "huge_shared" : " 27 GB", | 20260506 10:06:20.005 | INFO | "malloc" : " 14 GB", | 20260506 10:06:20.005 | INFO | "pinned" : " 29 GB" | 20260506 10:06:20.005 | INFO | } | 20260506 10:06:20.005 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:06:20.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:06:20.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:20.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:20.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:20.005 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:06:20.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:20.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:20.005 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:06:20.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:20.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:20.005 | INFO | Uploaded 36GiB to GPU in 1.6051s (22.4285GiB/s) | 20260506 10:06:20.005 | INFO | Downloaded 34GiB to GPU in 1.52243s (22.3328GiB/s) | 20260506 10:06:25.144 | INFO | Analyzing chunk frames = [506880,507392) | 20260506 10:06:25.144 | INFO | MemInfo: { | 20260506 10:06:25.144 | INFO | "gpu" : " 17 GB", | 20260506 10:06:25.144 | INFO | "huge" : "", | 20260506 10:06:25.144 | INFO | "huge_shared" : " 27 GB", | 20260506 10:06:25.144 | INFO | "malloc" : " 14 GB", | 20260506 10:06:25.144 | INFO | "pinned" : " 29 GB" | 20260506 10:06:25.144 | INFO | } | 20260506 10:06:25.144 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:06:25.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:06:25.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:25.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:25.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:25.144 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260506 10:06:25.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:25.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:25.144 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 10:06:25.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:25.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:25.145 | INFO | Uploaded 36GiB to GPU in 1.60543s (22.4239GiB/s) | 20260506 10:06:25.145 | INFO | Downloaded 34GiB to GPU in 1.52284s (22.3267GiB/s) | 20260506 10:06:30.283 | INFO | Analyzing chunk frames = [507392,507904) | 20260506 10:06:30.283 | INFO | MemInfo: { | 20260506 10:06:30.283 | INFO | "gpu" : " 17 GB", | 20260506 10:06:30.283 | INFO | "huge" : "", | 20260506 10:06:30.284 | INFO | "huge_shared" : " 27 GB", | 20260506 10:06:30.284 | INFO | "malloc" : " 14 GB", | 20260506 10:06:30.284 | INFO | "pinned" : " 29 GB" | 20260506 10:06:30.284 | INFO | } | 20260506 10:06:30.284 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:06:30.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:06:30.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:30.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:30.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:30.284 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:06:30.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:30.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:30.284 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:06:30.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:30.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:30.284 | INFO | Uploaded 36GiB to GPU in 1.606s (22.4159GiB/s) | 20260506 10:06:30.284 | INFO | Downloaded 34GiB to GPU in 1.52312s (22.3226GiB/s) | 20260506 10:06:30.284 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:06:30.284 | INFO | Total mem usage: 16 GiB | 20260506 10:06:30.284 | INFO | All Time High Water Marks sum: 16.4 GiB (Savings: 2.51%) | 20260506 10:06:30.284 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -5.11%) | 20260506 10:06:30.284 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:06:30.284 | INFO | High Water Breakdown: | 20260506 10:06:30.284 | INFO | 59.8% : 9.09 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:06:30.284 | INFO | 40.1% : 6.1 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:06:30.284 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:06:30.284 | INFO | ------------------------------------------ | 20260506 10:06:30.284 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:06:30.284 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:06:30.284 | INFO | Total mem usage: 26.5 GiB | 20260506 10:06:30.285 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:06:30.285 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:06:30.285 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:06:30.285 | INFO | High Water Breakdown: | 20260506 10:06:30.285 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:06:30.285 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:06:30.285 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:06:30.285 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:06:30.285 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:06:30.285 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:06:30.285 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:06:30.285 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:06:30.285 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:06:30.285 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:06:30.285 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:06:30.285 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:06:30.285 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:06:30.285 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:06:30.285 | INFO | ------------------------------------------ | 20260506 10:06:30.285 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:06:30.285 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:06:30.285 | INFO | Total mem usage: 25.6 GiB | 20260506 10:06:30.285 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:06:30.285 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:06:30.285 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:06:30.285 | INFO | High Water Breakdown: | 20260506 10:06:30.285 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:06:30.285 | INFO | ------------------------------------------ | 20260506 10:06:30.285 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:06:30.285 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:06:30.285 | INFO | Total mem usage: 15.7 GiB | 20260506 10:06:30.285 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:06:30.285 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:06:30.285 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 10:06:30.285 | INFO | High Water Breakdown: | 20260506 10:06:30.285 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:06:30.285 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:06:30.285 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:06:30.285 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:06:30.285 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:06:30.285 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:06:30.285 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:06:30.285 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:06:30.285 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:06:30.285 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:06:30.285 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:06:30.285 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:06:30.285 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:06:30.285 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:06:30.285 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:06:30.285 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:06:30.285 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:06:30.285 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:06:30.285 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:06:30.285 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:06:30.285 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:06:30.285 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:06:30.285 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:06:30.285 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:06:30.285 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:06:30.285 | INFO | ------------------------------------------ | 20260506 10:06:30.285 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:06:30.285 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:06:30.285 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:06:30.285 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:06:30.285 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 10:06:30.285 | INFO | PerfReport: 0.29%, AggregateStats -- 8.63e-02 / -nan, 4.52e-03 / 0.00e+00 | 20260506 10:06:30.285 | INFO | PerfReport: 10.93%, Baseline -- 3.29e+00 / -nan, 1.92e-02 / 0.00e+00 | 20260506 10:06:30.285 | INFO | PerfReport: 3.06%, Histogram -- 9.22e-01 / -nan, 5.15e-02 / 0.00e+00 | 20260506 10:06:30.285 | INFO | PerfReport: 16.05%, DME -- 3.87e+01 / -nan, 3.11e+00 / 0.00e+00 | 20260506 10:06:30.285 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.72e+01 / -nan, 4.14e-02 / 0.00e+00 | 20260506 10:06:30.285 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.24e+00 / -nan, 1.32e-01 / 0.00e+00 | 20260506 10:06:30.285 | INFO | PerfReport: 3.73%, Metrics -- 1.13e+00 / -nan, 1.75e-01 / 0.00e+00 | 20260506 10:06:30.285 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.35e-01 / -nan, 6.22e-02 / 0.00e+00 | 20260506 10:06:30.285 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:06:30.285 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:06:30.285 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:06:30.285 | INFO | PerfReport: Full Execution -- 9.01e+01 / -nan, 1.85e+01 / 0.00e+00 | 20260506 10:06:30.285 | INFO | PerfReport: 18.63%, Upload -- 1.68e+01 / 0.00e+00, 4.55e-01 / 0.00e+00 | 20260506 10:06:30.285 | INFO | PerfReport: 17.89%, Download -- 1.61e+01 / 0.00e+00, 7.97e-01 / 0.00e+00 | 20260506 10:06:30.285 | INFO | PerfReport: 33.43%, Compute -- 3.01e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 10:06:30.285 | INFO | PerfReport: 0.56%, LockUpload -- 5.08e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 10:06:30.285 | INFO | PerfReport: 0.00%, LockDownload -- 4.89e-04 / 0.00e+00, 1.52e-04 / 0.00e+00 | 20260506 10:06:30.285 | INFO | PerfReport: 29.46%, LockCompute -- 2.66e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 10:06:30.285 | INFO | PerfReport: 0.01%, Misc -- 1.24e-02 / 0.00e+00, 1.39e-02 / 0.00e+00 | 20260506 10:06:30.381 | INFO | Starting a round of Baz IO | 20260506 10:06:33.298 | INFO | Populating RTMetrics report | 20260506 10:06:33.298 | INFO | RTMetrics report populated | 20260506 10:06:33.298 | INFO | Opening tmp RTMetrics json file | 20260506 10:06:33.304 | INFO | Serializing RTMetrics to json | 20260506 10:06:33.328 | INFO | Serialization done, swapping tmp file now... | 20260506 10:06:33.329 | INFO | RTMetrics done | 20260506 10:06:33.793 | INFO | Finished a round of Baz IO: Wrote 1.949 GiB in 3.412 seconds | 20260506 10:06:33.793 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.64% / 0.07% / 50.30% | 20260506 10:06:33.793 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.64349% / 0.000139078% / 2.78155e-05% | 20260506 10:06:33.793 | INFO | Baz IO overall: | 20260506 10:06:33.793 | INFO | BytesWritten : 220228932404 | 20260506 10:06:33.793 | INFO | NumEvents : 105513686558 | 20260506 10:06:35.423 | INFO | Analyzing chunk frames = [507904,508416) | 20260506 10:06:35.423 | INFO | MemInfo: { | 20260506 10:06:35.423 | INFO | "gpu" : " 17 GB", | 20260506 10:06:35.423 | INFO | "huge" : "", | 20260506 10:06:35.423 | INFO | "huge_shared" : " 27 GB", | 20260506 10:06:35.423 | INFO | "malloc" : " 14 GB", | 20260506 10:06:35.423 | INFO | "pinned" : " 29 GB" | 20260506 10:06:35.423 | INFO | } | 20260506 10:06:35.423 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:06:35.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:06:35.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:35.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:35.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:35.423 | INFO | ANALYSIS : 57.6%, 2.8, 85.2ms, 0 | 20260506 10:06:35.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:35.423 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 10:06:35.423 | INFO | PRE_HQ : 56.7%, 2.1, 60.9ms, 0 | 20260506 10:06:35.423 | INFO | BAZWRITER : 66.0%, 4.8, 162.2ms, 0 | 20260506 10:06:35.423 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 10:06:35.423 | INFO | Uploaded 36GiB to GPU in 1.67293s (21.5191GiB/s) | 20260506 10:06:35.423 | INFO | Downloaded 38GiB to GPU in 1.73696s (21.8773GiB/s) | 20260506 10:06:40.562 | INFO | Analyzing chunk frames = [508416,508928) | 20260506 10:06:40.562 | INFO | MemInfo: { | 20260506 10:06:40.562 | INFO | "gpu" : " 17 GB", | 20260506 10:06:40.562 | INFO | "huge" : "", | 20260506 10:06:40.562 | INFO | "huge_shared" : " 27 GB", | 20260506 10:06:40.562 | INFO | "malloc" : " 14 GB", | 20260506 10:06:40.562 | INFO | "pinned" : " 29 GB" | 20260506 10:06:40.562 | INFO | } | 20260506 10:06:40.562 | INFO | Perf report for the last 5.13953 seconds: | 20260506 10:06:40.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:06:40.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:40.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:40.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:40.563 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 10:06:40.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:40.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:40.563 | INFO | PRE_HQ : 30.2%, 1.0, 16.3ms, 0 | 20260506 10:06:40.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:40.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:40.563 | INFO | Uploaded 36GiB to GPU in 1.60263s (22.4631GiB/s) | 20260506 10:06:40.563 | INFO | Downloaded 34GiB to GPU in 1.51993s (22.3694GiB/s) | 20260506 10:06:45.701 | INFO | Analyzing chunk frames = [508928,509440) | 20260506 10:06:45.701 | INFO | MemInfo: { | 20260506 10:06:45.701 | INFO | "gpu" : " 17 GB", | 20260506 10:06:45.701 | INFO | "huge" : "", | 20260506 10:06:45.701 | INFO | "huge_shared" : " 27 GB", | 20260506 10:06:45.702 | INFO | "malloc" : " 14 GB", | 20260506 10:06:45.702 | INFO | "pinned" : " 29 GB" | 20260506 10:06:45.702 | INFO | } | 20260506 10:06:45.702 | INFO | Perf report for the last 5.13918 seconds: | 20260506 10:06:45.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:06:45.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:45.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:45.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:45.702 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260506 10:06:45.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:45.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:45.702 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:06:45.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:45.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:45.702 | INFO | Uploaded 36GiB to GPU in 1.60644s (22.4098GiB/s) | 20260506 10:06:45.702 | INFO | Downloaded 34GiB to GPU in 1.52451s (22.3023GiB/s) | 20260506 10:06:50.841 | INFO | Analyzing chunk frames = [509440,509952) | 20260506 10:06:50.841 | INFO | MemInfo: { | 20260506 10:06:50.841 | INFO | "gpu" : " 17 GB", | 20260506 10:06:50.841 | INFO | "huge" : "", | 20260506 10:06:50.841 | INFO | "huge_shared" : " 27 GB", | 20260506 10:06:50.841 | INFO | "malloc" : " 14 GB", | 20260506 10:06:50.841 | INFO | "pinned" : " 29 GB" | 20260506 10:06:50.841 | INFO | } | 20260506 10:06:50.841 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:06:50.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:06:50.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:50.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:50.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:50.841 | INFO | ANALYSIS : 56.5%, 3.0, 87.5ms, 0 | 20260506 10:06:50.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:50.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:50.841 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:06:50.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:50.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:50.841 | INFO | Uploaded 36GiB to GPU in 1.60639s (22.4106GiB/s) | 20260506 10:06:50.841 | INFO | Downloaded 34GiB to GPU in 1.52457s (22.3014GiB/s) | 20260506 10:06:55.980 | INFO | Analyzing chunk frames = [509952,510464) | 20260506 10:06:55.980 | INFO | MemInfo: { | 20260506 10:06:55.980 | INFO | "gpu" : " 17 GB", | 20260506 10:06:55.980 | INFO | "huge" : "", | 20260506 10:06:55.980 | INFO | "huge_shared" : " 27 GB", | 20260506 10:06:55.980 | INFO | "malloc" : " 14 GB", | 20260506 10:06:55.980 | INFO | "pinned" : " 29 GB" | 20260506 10:06:55.980 | INFO | } | 20260506 10:06:55.980 | INFO | Perf report for the last 5.13925 seconds: | 20260506 10:06:55.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:06:55.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:55.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:55.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:55.980 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:06:55.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:55.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:55.980 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 10:06:55.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:55.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:06:55.981 | INFO | Uploaded 36GiB to GPU in 1.60427s (22.4402GiB/s) | 20260506 10:06:55.981 | INFO | Downloaded 34GiB to GPU in 1.52149s (22.3465GiB/s) | 20260506 10:07:01.119 | INFO | Analyzing chunk frames = [510464,510976) | 20260506 10:07:01.119 | INFO | MemInfo: { | 20260506 10:07:01.119 | INFO | "gpu" : " 17 GB", | 20260506 10:07:01.119 | INFO | "huge" : "", | 20260506 10:07:01.119 | INFO | "huge_shared" : " 27 GB", | 20260506 10:07:01.120 | INFO | "malloc" : " 14 GB", | 20260506 10:07:01.120 | INFO | "pinned" : " 29 GB" | 20260506 10:07:01.120 | INFO | } | 20260506 10:07:01.120 | INFO | Perf report for the last 5.13951 seconds: | 20260506 10:07:01.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:07:01.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:01.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:01.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:01.120 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 10:07:01.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:01.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:01.120 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:07:01.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:01.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:01.120 | INFO | Uploaded 36GiB to GPU in 1.60477s (22.4331GiB/s) | 20260506 10:07:01.120 | INFO | Downloaded 34GiB to GPU in 1.52317s (22.3219GiB/s) | 20260506 10:07:06.259 | INFO | Analyzing chunk frames = [510976,511488) | 20260506 10:07:06.259 | INFO | MemInfo: { | 20260506 10:07:06.259 | INFO | "gpu" : " 17 GB", | 20260506 10:07:06.259 | INFO | "huge" : "", | 20260506 10:07:06.259 | INFO | "huge_shared" : " 27 GB", | 20260506 10:07:06.259 | INFO | "malloc" : " 14 GB", | 20260506 10:07:06.259 | INFO | "pinned" : " 29 GB" | 20260506 10:07:06.259 | INFO | } | 20260506 10:07:06.259 | INFO | Perf report for the last 5.13956 seconds: | 20260506 10:07:06.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:07:06.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:06.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:06.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:06.259 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:07:06.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:06.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:06.259 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:07:06.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:06.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:06.260 | INFO | Uploaded 36GiB to GPU in 1.60485s (22.4319GiB/s) | 20260506 10:07:06.260 | INFO | Downloaded 34GiB to GPU in 1.52373s (22.3137GiB/s) | 20260506 10:07:11.398 | INFO | Analyzing chunk frames = [511488,512000) | 20260506 10:07:11.398 | INFO | MemInfo: { | 20260506 10:07:11.398 | INFO | "gpu" : " 17 GB", | 20260506 10:07:11.398 | INFO | "huge" : "", | 20260506 10:07:11.398 | INFO | "huge_shared" : " 27 GB", | 20260506 10:07:11.398 | INFO | "malloc" : " 14 GB", | 20260506 10:07:11.398 | INFO | "pinned" : " 29 GB" | 20260506 10:07:11.398 | INFO | } | 20260506 10:07:11.398 | INFO | Perf report for the last 5.13899 seconds: | 20260506 10:07:11.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:07:11.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:11.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:11.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:11.398 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:07:11.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:11.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:11.398 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:07:11.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:11.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:11.399 | INFO | Uploaded 36GiB to GPU in 1.60509s (22.4286GiB/s) | 20260506 10:07:11.399 | INFO | Downloaded 34GiB to GPU in 1.52337s (22.3189GiB/s) | 20260506 10:07:11.495 | INFO | Starting a round of Baz IO | 20260506 10:07:14.413 | INFO | Populating RTMetrics report | 20260506 10:07:14.413 | INFO | RTMetrics report populated | 20260506 10:07:14.413 | INFO | Opening tmp RTMetrics json file | 20260506 10:07:14.471 | INFO | Serializing RTMetrics to json | 20260506 10:07:14.614 | INFO | Serialization done, swapping tmp file now... | 20260506 10:07:14.615 | INFO | RTMetrics done | 20260506 10:07:14.881 | INFO | Finished a round of Baz IO: Wrote 1.95 GiB in 3.386 seconds | 20260506 10:07:14.881 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.57% / 0.06% / 50.36% | 20260506 10:07:14.881 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.61404% / 0.00017484% / 4.76837e-05% | 20260506 10:07:14.882 | INFO | Baz IO overall: | 20260506 10:07:14.882 | INFO | BytesWritten : 222323183670 | 20260506 10:07:14.882 | INFO | NumEvents : 106530523555 | 20260506 10:07:16.537 | INFO | Analyzing chunk frames = [512000,512512) | 20260506 10:07:16.537 | INFO | MemInfo: { | 20260506 10:07:16.537 | INFO | "gpu" : " 17 GB", | 20260506 10:07:16.537 | INFO | "huge" : "", | 20260506 10:07:16.538 | INFO | "huge_shared" : " 27 GB", | 20260506 10:07:16.538 | INFO | "malloc" : " 14 GB", | 20260506 10:07:16.538 | INFO | "pinned" : " 29 GB" | 20260506 10:07:16.538 | INFO | } | 20260506 10:07:16.538 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:07:16.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:07:16.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:16.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:16.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:16.538 | INFO | ANALYSIS : 57.7%, 2.8, 85.6ms, 0 | 20260506 10:07:16.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:16.538 | INFO | RT_METRICS : 48.1%, 1.0, 25.2ms, 0 | 20260506 10:07:16.538 | INFO | PRE_HQ : 56.8%, 2.1, 61.4ms, 0 | 20260506 10:07:16.538 | INFO | BAZWRITER : 65.5%, 4.7, 159.6ms, 0 | 20260506 10:07:16.538 | INFO | BAZ_SAMPLING : 4.2%, 1.0, 2.2ms, 0 | 20260506 10:07:16.538 | INFO | Uploaded 36GiB to GPU in 1.65788s (21.7145GiB/s) | 20260506 10:07:16.538 | INFO | Downloaded 38GiB to GPU in 1.72345s (22.0487GiB/s) | 20260506 10:07:21.677 | INFO | Analyzing chunk frames = [512512,513024) | 20260506 10:07:21.677 | INFO | MemInfo: { | 20260506 10:07:21.677 | INFO | "gpu" : " 17 GB", | 20260506 10:07:21.677 | INFO | "huge" : "", | 20260506 10:07:21.677 | INFO | "huge_shared" : " 27 GB", | 20260506 10:07:21.677 | INFO | "malloc" : " 14 GB", | 20260506 10:07:21.677 | INFO | "pinned" : " 29 GB" | 20260506 10:07:21.677 | INFO | } | 20260506 10:07:21.677 | INFO | Perf report for the last 5.13946 seconds: | 20260506 10:07:21.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:07:21.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:21.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:21.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:21.677 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260506 10:07:21.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:21.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:21.677 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:07:21.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:21.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:21.678 | INFO | Uploaded 36GiB to GPU in 1.60211s (22.4703GiB/s) | 20260506 10:07:21.678 | INFO | Downloaded 34GiB to GPU in 1.51972s (22.3726GiB/s) | 20260506 10:07:26.816 | INFO | Analyzing chunk frames = [513024,513536) | 20260506 10:07:26.816 | INFO | MemInfo: { | 20260506 10:07:26.816 | INFO | "gpu" : " 17 GB", | 20260506 10:07:26.816 | INFO | "huge" : "", | 20260506 10:07:26.816 | INFO | "huge_shared" : " 27 GB", | 20260506 10:07:26.816 | INFO | "malloc" : " 14 GB", | 20260506 10:07:26.816 | INFO | "pinned" : " 29 GB" | 20260506 10:07:26.816 | INFO | } | 20260506 10:07:26.816 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:07:26.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:07:26.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:26.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:26.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:26.816 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 10:07:26.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:26.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:26.816 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 10:07:26.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:26.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:26.817 | INFO | Uploaded 36GiB to GPU in 1.60647s (22.4093GiB/s) | 20260506 10:07:26.817 | INFO | Downloaded 34GiB to GPU in 1.52437s (22.3043GiB/s) | 20260506 10:07:31.955 | INFO | Analyzing chunk frames = [513536,514048) | 20260506 10:07:31.955 | INFO | MemInfo: { | 20260506 10:07:31.955 | INFO | "gpu" : " 17 GB", | 20260506 10:07:31.955 | INFO | "huge" : "", | 20260506 10:07:31.955 | INFO | "huge_shared" : " 27 GB", | 20260506 10:07:31.956 | INFO | "malloc" : " 14 GB", | 20260506 10:07:31.956 | INFO | "pinned" : " 29 GB" | 20260506 10:07:31.956 | INFO | } | 20260506 10:07:31.956 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:07:31.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:07:31.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:31.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:31.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:31.956 | INFO | ANALYSIS : 56.5%, 3.0, 87.7ms, 0 | 20260506 10:07:31.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:31.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:31.956 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:07:31.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:31.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:31.956 | INFO | Uploaded 36GiB to GPU in 1.60616s (22.4138GiB/s) | 20260506 10:07:31.956 | INFO | Downloaded 34GiB to GPU in 1.52567s (22.2853GiB/s) | 20260506 10:07:37.095 | INFO | Analyzing chunk frames = [514048,514560) | 20260506 10:07:37.095 | INFO | MemInfo: { | 20260506 10:07:37.095 | INFO | "gpu" : " 17 GB", | 20260506 10:07:37.095 | INFO | "huge" : "", | 20260506 10:07:37.095 | INFO | "huge_shared" : " 27 GB", | 20260506 10:07:37.095 | INFO | "malloc" : " 14 GB", | 20260506 10:07:37.095 | INFO | "pinned" : " 29 GB" | 20260506 10:07:37.095 | INFO | } | 20260506 10:07:37.095 | INFO | Perf report for the last 5.13945 seconds: | 20260506 10:07:37.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:07:37.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:37.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:37.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:37.095 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:07:37.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:37.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:37.095 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:07:37.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:37.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:37.095 | INFO | Uploaded 36GiB to GPU in 1.60479s (22.4328GiB/s) | 20260506 10:07:37.095 | INFO | Downloaded 34GiB to GPU in 1.52203s (22.3386GiB/s) | 20260506 10:07:42.234 | INFO | Analyzing chunk frames = [514560,515072) | 20260506 10:07:42.234 | INFO | MemInfo: { | 20260506 10:07:42.234 | INFO | "gpu" : " 17 GB", | 20260506 10:07:42.234 | INFO | "huge" : "", | 20260506 10:07:42.234 | INFO | "huge_shared" : " 27 GB", | 20260506 10:07:42.234 | INFO | "malloc" : " 14 GB", | 20260506 10:07:42.234 | INFO | "pinned" : " 29 GB" | 20260506 10:07:42.234 | INFO | } | 20260506 10:07:42.234 | INFO | Perf report for the last 5.1392 seconds: | 20260506 10:07:42.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:07:42.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:42.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:42.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:42.234 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 10:07:42.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:42.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:42.235 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:07:42.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:42.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:42.235 | INFO | Uploaded 36GiB to GPU in 1.60398s (22.4442GiB/s) | 20260506 10:07:42.235 | INFO | Downloaded 34GiB to GPU in 1.52163s (22.3445GiB/s) | 20260506 10:07:47.374 | INFO | Analyzing chunk frames = [515072,515584) | 20260506 10:07:47.374 | INFO | MemInfo: { | 20260506 10:07:47.374 | INFO | "gpu" : " 17 GB", | 20260506 10:07:47.374 | INFO | "huge" : "", | 20260506 10:07:47.374 | INFO | "huge_shared" : " 27 GB", | 20260506 10:07:47.374 | INFO | "malloc" : " 14 GB", | 20260506 10:07:47.374 | INFO | "pinned" : " 29 GB" | 20260506 10:07:47.374 | INFO | } | 20260506 10:07:47.374 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:07:47.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:07:47.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:47.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:47.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:47.374 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:07:47.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:47.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:47.374 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 10:07:47.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:47.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:47.374 | INFO | Uploaded 36GiB to GPU in 1.60564s (22.421GiB/s) | 20260506 10:07:47.374 | INFO | Downloaded 34GiB to GPU in 1.52465s (22.3001GiB/s) | 20260506 10:07:52.513 | INFO | Analyzing chunk frames = [515584,516096) | 20260506 10:07:52.513 | INFO | MemInfo: { | 20260506 10:07:52.513 | INFO | "gpu" : " 17 GB", | 20260506 10:07:52.513 | INFO | "huge" : "", | 20260506 10:07:52.513 | INFO | "huge_shared" : " 27 GB", | 20260506 10:07:52.513 | INFO | "malloc" : " 14 GB", | 20260506 10:07:52.513 | INFO | "pinned" : " 29 GB" | 20260506 10:07:52.513 | INFO | } | 20260506 10:07:52.513 | INFO | Perf report for the last 5.13931 seconds: | 20260506 10:07:52.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:07:52.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:52.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:52.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:52.513 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260506 10:07:52.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:52.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:52.513 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:07:52.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:52.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:52.513 | INFO | Uploaded 36GiB to GPU in 1.60489s (22.4314GiB/s) | 20260506 10:07:52.513 | INFO | Downloaded 34GiB to GPU in 1.52339s (22.3186GiB/s) | 20260506 10:07:52.514 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:07:52.514 | INFO | Total mem usage: 16 GiB | 20260506 10:07:52.514 | INFO | All Time High Water Marks sum: 16.4 GiB (Savings: 2.51%) | 20260506 10:07:52.514 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -0.603%) | 20260506 10:07:52.514 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:07:52.514 | INFO | High Water Breakdown: | 20260506 10:07:52.514 | INFO | 60.4% : 9.61 GiB ( 99.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:07:52.514 | INFO | 39.4% : 6.27 GiB ( 93.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:07:52.514 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:07:52.514 | INFO | ------------------------------------------ | 20260506 10:07:52.514 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:07:52.514 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:07:52.514 | INFO | Total mem usage: 26.5 GiB | 20260506 10:07:52.514 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:07:52.514 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:07:52.514 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:07:52.514 | INFO | High Water Breakdown: | 20260506 10:07:52.514 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:07:52.514 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:07:52.514 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:07:52.514 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:07:52.514 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:07:52.514 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:07:52.514 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:07:52.514 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:07:52.514 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:07:52.514 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:07:52.514 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:07:52.514 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:07:52.514 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:07:52.514 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:07:52.514 | INFO | ------------------------------------------ | 20260506 10:07:52.514 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:07:52.514 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:07:52.514 | INFO | Total mem usage: 25.6 GiB | 20260506 10:07:52.514 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:07:52.514 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:07:52.514 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:07:52.514 | INFO | High Water Breakdown: | 20260506 10:07:52.514 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:07:52.514 | INFO | ------------------------------------------ | 20260506 10:07:52.514 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:07:52.514 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:07:52.514 | INFO | Total mem usage: 15.7 GiB | 20260506 10:07:52.514 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:07:52.514 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:07:52.514 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:07:52.514 | INFO | High Water Breakdown: | 20260506 10:07:52.514 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:07:52.514 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:07:52.514 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:07:52.514 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:07:52.514 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:07:52.514 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:07:52.514 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:07:52.514 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:07:52.514 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:07:52.514 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:07:52.514 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:07:52.514 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:07:52.514 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:07:52.514 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:07:52.514 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:07:52.514 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:07:52.514 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:07:52.514 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:07:52.514 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:07:52.514 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:07:52.514 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:07:52.514 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:07:52.514 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:07:52.514 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:07:52.514 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:07:52.514 | INFO | ------------------------------------------ | 20260506 10:07:52.514 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:07:52.514 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:07:52.514 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:07:52.514 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:07:52.514 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 10:07:52.514 | INFO | PerfReport: 0.29%, AggregateStats -- 8.64e-02 / -nan, 6.98e-03 / 0.00e+00 | 20260506 10:07:52.514 | INFO | PerfReport: 10.93%, Baseline -- 3.29e+00 / -nan, 1.88e-02 / 0.00e+00 | 20260506 10:07:52.514 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.26e-02 / 0.00e+00 | 20260506 10:07:52.514 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.11e+00 / 0.00e+00 | 20260506 10:07:52.514 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.72e+01 / -nan, 4.09e-02 / 0.00e+00 | 20260506 10:07:52.514 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.24e+00 / -nan, 1.31e-01 / 0.00e+00 | 20260506 10:07:52.514 | INFO | PerfReport: 3.73%, Metrics -- 1.12e+00 / -nan, 1.76e-01 / 0.00e+00 | 20260506 10:07:52.514 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.35e-01 / -nan, 6.26e-02 / 0.00e+00 | 20260506 10:07:52.514 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:07:52.514 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:07:52.514 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:07:52.514 | INFO | PerfReport: Full Execution -- 9.00e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 10:07:52.514 | INFO | PerfReport: 18.67%, Upload -- 1.68e+01 / 0.00e+00, 4.69e-01 / 0.00e+00 | 20260506 10:07:52.514 | INFO | PerfReport: 17.93%, Download -- 1.61e+01 / 0.00e+00, 8.15e-01 / 0.00e+00 | 20260506 10:07:52.514 | INFO | PerfReport: 33.50%, Compute -- 3.01e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 10:07:52.514 | INFO | PerfReport: 0.56%, LockUpload -- 5.08e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260506 10:07:52.514 | INFO | PerfReport: 0.00%, LockDownload -- 4.95e-04 / 0.00e+00, 2.15e-04 / 0.00e+00 | 20260506 10:07:52.514 | INFO | PerfReport: 29.32%, LockCompute -- 2.64e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 10:07:52.514 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260506 10:07:52.610 | INFO | Starting a round of Baz IO | 20260506 10:07:55.523 | INFO | Populating RTMetrics report | 20260506 10:07:55.523 | INFO | RTMetrics report populated | 20260506 10:07:55.523 | INFO | Opening tmp RTMetrics json file | 20260506 10:07:55.742 | INFO | Serializing RTMetrics to json | 20260506 10:07:55.800 | INFO | Serialization done, swapping tmp file now... | 20260506 10:07:55.834 | INFO | RTMetrics done | 20260506 10:07:56.373 | INFO | Finished a round of Baz IO: Wrote 1.952 GiB in 3.762 seconds | 20260506 10:07:56.373 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.51% / 0.06% / 50.43% | 20260506 10:07:56.373 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.59675% / 0.00013113% / 3.57628e-05% | 20260506 10:07:56.373 | INFO | Baz IO overall: | 20260506 10:07:56.373 | INFO | BytesWritten : 224419611993 | 20260506 10:07:56.373 | INFO | NumEvents : 107548888396 | 20260506 10:07:57.652 | INFO | Analyzing chunk frames = [516096,516608) | 20260506 10:07:57.652 | INFO | MemInfo: { | 20260506 10:07:57.652 | INFO | "gpu" : " 17 GB", | 20260506 10:07:57.652 | INFO | "huge" : "", | 20260506 10:07:57.652 | INFO | "huge_shared" : " 27 GB", | 20260506 10:07:57.652 | INFO | "malloc" : " 14 GB", | 20260506 10:07:57.652 | INFO | "pinned" : " 29 GB" | 20260506 10:07:57.652 | INFO | } | 20260506 10:07:57.652 | INFO | Perf report for the last 5.13925 seconds: | 20260506 10:07:57.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:07:57.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:57.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:57.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:57.653 | INFO | ANALYSIS : 57.6%, 2.8, 85.5ms, 0 | 20260506 10:07:57.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:07:57.653 | INFO | RT_METRICS : 50.1%, 1.0, 26.3ms, 0 | 20260506 10:07:57.653 | INFO | PRE_HQ : 56.6%, 2.1, 60.7ms, 0 | 20260506 10:07:57.653 | INFO | BAZWRITER : 72.8%, 4.8, 178.7ms, 0 | 20260506 10:07:57.653 | INFO | BAZ_SAMPLING : 5.1%, 1.0, 2.6ms, 0 | 20260506 10:07:57.653 | INFO | Uploaded 36GiB to GPU in 1.67079s (21.5467GiB/s) | 20260506 10:07:57.653 | INFO | Downloaded 38GiB to GPU in 1.73459s (21.9072GiB/s) | 20260506 10:08:02.792 | INFO | Analyzing chunk frames = [516608,517120) | 20260506 10:08:02.792 | INFO | MemInfo: { | 20260506 10:08:02.792 | INFO | "gpu" : " 17 GB", | 20260506 10:08:02.792 | INFO | "huge" : "", | 20260506 10:08:02.792 | INFO | "huge_shared" : " 27 GB", | 20260506 10:08:02.792 | INFO | "malloc" : " 14 GB", | 20260506 10:08:02.792 | INFO | "pinned" : " 29 GB" | 20260506 10:08:02.792 | INFO | } | 20260506 10:08:02.792 | INFO | Perf report for the last 5.13947 seconds: | 20260506 10:08:02.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:08:02.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:02.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:02.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:02.792 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260506 10:08:02.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:02.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:02.792 | INFO | PRE_HQ : 30.0%, 1.0, 15.8ms, 0 | 20260506 10:08:02.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:02.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:02.793 | INFO | Uploaded 36GiB to GPU in 1.60271s (22.462GiB/s) | 20260506 10:08:02.793 | INFO | Downloaded 34GiB to GPU in 1.52071s (22.3579GiB/s) | 20260506 10:08:07.931 | INFO | Analyzing chunk frames = [517120,517632) | 20260506 10:08:07.931 | INFO | MemInfo: { | 20260506 10:08:07.931 | INFO | "gpu" : " 17 GB", | 20260506 10:08:07.931 | INFO | "huge" : "", | 20260506 10:08:07.931 | INFO | "huge_shared" : " 27 GB", | 20260506 10:08:07.931 | INFO | "malloc" : " 14 GB", | 20260506 10:08:07.931 | INFO | "pinned" : " 29 GB" | 20260506 10:08:07.931 | INFO | } | 20260506 10:08:07.931 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:08:07.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:08:07.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:07.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:07.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:07.931 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 10:08:07.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:07.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:07.931 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 10:08:07.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:07.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:07.931 | INFO | Uploaded 36GiB to GPU in 1.6065s (22.4089GiB/s) | 20260506 10:08:07.932 | INFO | Downloaded 34GiB to GPU in 1.5256s (22.2863GiB/s) | 20260506 10:08:13.070 | INFO | Analyzing chunk frames = [517632,518144) | 20260506 10:08:13.070 | INFO | MemInfo: { | 20260506 10:08:13.070 | INFO | "gpu" : " 17 GB", | 20260506 10:08:13.070 | INFO | "huge" : "", | 20260506 10:08:13.070 | INFO | "huge_shared" : " 27 GB", | 20260506 10:08:13.070 | INFO | "malloc" : " 14 GB", | 20260506 10:08:13.070 | INFO | "pinned" : " 29 GB" | 20260506 10:08:13.070 | INFO | } | 20260506 10:08:13.070 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:08:13.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:08:13.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:13.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:13.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:13.071 | INFO | ANALYSIS : 56.5%, 3.0, 87.7ms, 0 | 20260506 10:08:13.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:13.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:13.071 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:08:13.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:13.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:13.071 | INFO | Uploaded 36GiB to GPU in 1.6064s (22.4104GiB/s) | 20260506 10:08:13.071 | INFO | Downloaded 34GiB to GPU in 1.52416s (22.3073GiB/s) | 20260506 10:08:18.210 | INFO | Analyzing chunk frames = [518144,518656) | 20260506 10:08:18.210 | INFO | MemInfo: { | 20260506 10:08:18.210 | INFO | "gpu" : " 17 GB", | 20260506 10:08:18.210 | INFO | "huge" : "", | 20260506 10:08:18.210 | INFO | "huge_shared" : " 27 GB", | 20260506 10:08:18.210 | INFO | "malloc" : " 14 GB", | 20260506 10:08:18.210 | INFO | "pinned" : " 29 GB" | 20260506 10:08:18.210 | INFO | } | 20260506 10:08:18.210 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:08:18.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:08:18.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:18.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:18.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:18.210 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:08:18.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:18.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:18.210 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:08:18.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:18.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:18.210 | INFO | Uploaded 36GiB to GPU in 1.60474s (22.4335GiB/s) | 20260506 10:08:18.210 | INFO | Downloaded 34GiB to GPU in 1.52214s (22.337GiB/s) | 20260506 10:08:23.350 | INFO | Analyzing chunk frames = [518656,519168) | 20260506 10:08:23.350 | INFO | MemInfo: { | 20260506 10:08:23.350 | INFO | "gpu" : " 17 GB", | 20260506 10:08:23.350 | INFO | "huge" : "", | 20260506 10:08:23.350 | INFO | "huge_shared" : " 27 GB", | 20260506 10:08:23.350 | INFO | "malloc" : " 14 GB", | 20260506 10:08:23.350 | INFO | "pinned" : " 29 GB" | 20260506 10:08:23.350 | INFO | } | 20260506 10:08:23.350 | INFO | Perf report for the last 5.14021 seconds: | 20260506 10:08:23.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:08:23.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:23.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:23.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:23.350 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 10:08:23.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:23.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:23.350 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 10:08:23.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:23.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:23.350 | INFO | Uploaded 36GiB to GPU in 1.60464s (22.4349GiB/s) | 20260506 10:08:23.350 | INFO | Downloaded 34GiB to GPU in 1.52248s (22.332GiB/s) | 20260506 10:08:28.489 | INFO | Analyzing chunk frames = [519168,519680) | 20260506 10:08:28.489 | INFO | MemInfo: { | 20260506 10:08:28.489 | INFO | "gpu" : " 17 GB", | 20260506 10:08:28.489 | INFO | "huge" : "", | 20260506 10:08:28.489 | INFO | "huge_shared" : " 27 GB", | 20260506 10:08:28.489 | INFO | "malloc" : " 14 GB", | 20260506 10:08:28.489 | INFO | "pinned" : " 29 GB" | 20260506 10:08:28.489 | INFO | } | 20260506 10:08:28.489 | INFO | Perf report for the last 5.13922 seconds: | 20260506 10:08:28.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:08:28.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:28.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:28.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:28.489 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:08:28.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:28.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:28.489 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:08:28.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:28.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:28.490 | INFO | Uploaded 36GiB to GPU in 1.60516s (22.4276GiB/s) | 20260506 10:08:28.490 | INFO | Downloaded 34GiB to GPU in 1.52395s (22.3105GiB/s) | 20260506 10:08:33.628 | INFO | Analyzing chunk frames = [519680,520192) | 20260506 10:08:33.628 | INFO | MemInfo: { | 20260506 10:08:33.628 | INFO | "gpu" : " 17 GB", | 20260506 10:08:33.628 | INFO | "huge" : "", | 20260506 10:08:33.628 | INFO | "huge_shared" : " 27 GB", | 20260506 10:08:33.628 | INFO | "malloc" : " 14 GB", | 20260506 10:08:33.628 | INFO | "pinned" : " 29 GB" | 20260506 10:08:33.628 | INFO | } | 20260506 10:08:33.628 | INFO | Perf report for the last 5.1386 seconds: | 20260506 10:08:33.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:08:33.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:33.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:33.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:33.628 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:08:33.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:33.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:33.628 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:08:33.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:33.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:33.629 | INFO | Uploaded 36GiB to GPU in 1.60626s (22.4123GiB/s) | 20260506 10:08:33.629 | INFO | Downloaded 34GiB to GPU in 1.5239s (22.3111GiB/s) | 20260506 10:08:33.725 | INFO | Starting a round of Baz IO | 20260506 10:08:36.649 | INFO | Populating RTMetrics report | 20260506 10:08:36.650 | INFO | RTMetrics report populated | 20260506 10:08:36.650 | INFO | Opening tmp RTMetrics json file | 20260506 10:08:36.651 | INFO | Serializing RTMetrics to json | 20260506 10:08:36.656 | INFO | Serialization done, swapping tmp file now... | 20260506 10:08:36.657 | INFO | RTMetrics done | 20260506 10:08:36.936 | INFO | Finished a round of Baz IO: Wrote 1.953 GiB in 3.21 seconds | 20260506 10:08:36.936 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.44% / 0.06% / 50.49% | 20260506 10:08:36.936 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.56193% / 0.000170867% / 5.16574e-05% | 20260506 10:08:36.936 | INFO | Baz IO overall: | 20260506 10:08:36.936 | INFO | BytesWritten : 226516964958 | 20260506 10:08:36.936 | INFO | NumEvents : 108567079719 | 20260506 10:08:38.767 | INFO | Analyzing chunk frames = [520192,520704) | 20260506 10:08:38.767 | INFO | MemInfo: { | 20260506 10:08:38.767 | INFO | "gpu" : " 17 GB", | 20260506 10:08:38.767 | INFO | "huge" : "", | 20260506 10:08:38.767 | INFO | "huge_shared" : " 27 GB", | 20260506 10:08:38.767 | INFO | "malloc" : " 14 GB", | 20260506 10:08:38.767 | INFO | "pinned" : " 29 GB" | 20260506 10:08:38.767 | INFO | } | 20260506 10:08:38.767 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:08:38.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:08:38.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:38.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:38.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:38.767 | INFO | ANALYSIS : 57.8%, 2.8, 85.9ms, 0 | 20260506 10:08:38.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:38.767 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 10:08:38.767 | INFO | PRE_HQ : 56.9%, 2.1, 61.1ms, 0 | 20260506 10:08:38.767 | INFO | BAZWRITER : 62.1%, 4.7, 151.2ms, 0 | 20260506 10:08:38.767 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 10:08:38.768 | INFO | Uploaded 36GiB to GPU in 1.67437s (21.5006GiB/s) | 20260506 10:08:38.768 | INFO | Downloaded 38GiB to GPU in 1.7314s (21.9475GiB/s) | 20260506 10:08:43.906 | INFO | Analyzing chunk frames = [520704,521216) | 20260506 10:08:43.906 | INFO | MemInfo: { | 20260506 10:08:43.906 | INFO | "gpu" : " 17 GB", | 20260506 10:08:43.906 | INFO | "huge" : "", | 20260506 10:08:43.907 | INFO | "huge_shared" : " 27 GB", | 20260506 10:08:43.907 | INFO | "malloc" : " 14 GB", | 20260506 10:08:43.907 | INFO | "pinned" : " 29 GB" | 20260506 10:08:43.907 | INFO | } | 20260506 10:08:43.907 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:08:43.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:08:43.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:43.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:43.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:43.907 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260506 10:08:43.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:43.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:43.907 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:08:43.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:43.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:43.907 | INFO | Uploaded 36GiB to GPU in 1.60253s (22.4645GiB/s) | 20260506 10:08:43.907 | INFO | Downloaded 34GiB to GPU in 1.51981s (22.3713GiB/s) | 20260506 10:08:49.046 | INFO | Analyzing chunk frames = [521216,521728) | 20260506 10:08:49.046 | INFO | MemInfo: { | 20260506 10:08:49.046 | INFO | "gpu" : " 17 GB", | 20260506 10:08:49.046 | INFO | "huge" : "", | 20260506 10:08:49.046 | INFO | "huge_shared" : " 27 GB", | 20260506 10:08:49.046 | INFO | "malloc" : " 14 GB", | 20260506 10:08:49.046 | INFO | "pinned" : " 29 GB" | 20260506 10:08:49.046 | INFO | } | 20260506 10:08:49.046 | INFO | Perf report for the last 5.13939 seconds: | 20260506 10:08:49.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:08:49.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:49.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:49.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:49.046 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260506 10:08:49.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:49.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:49.046 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:08:49.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:49.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:49.046 | INFO | Uploaded 36GiB to GPU in 1.60667s (22.4066GiB/s) | 20260506 10:08:49.046 | INFO | Downloaded 34GiB to GPU in 1.52487s (22.2969GiB/s) | 20260506 10:08:54.185 | INFO | Analyzing chunk frames = [521728,522240) | 20260506 10:08:54.185 | INFO | MemInfo: { | 20260506 10:08:54.185 | INFO | "gpu" : " 17 GB", | 20260506 10:08:54.185 | INFO | "huge" : "", | 20260506 10:08:54.185 | INFO | "huge_shared" : " 27 GB", | 20260506 10:08:54.185 | INFO | "malloc" : " 14 GB", | 20260506 10:08:54.185 | INFO | "pinned" : " 29 GB" | 20260506 10:08:54.185 | INFO | } | 20260506 10:08:54.185 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:08:54.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:08:54.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:54.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:54.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:54.185 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:08:54.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:54.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:54.185 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 10:08:54.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:54.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:54.186 | INFO | Uploaded 36GiB to GPU in 1.60658s (22.4078GiB/s) | 20260506 10:08:54.186 | INFO | Downloaded 34GiB to GPU in 1.52452s (22.3021GiB/s) | 20260506 10:08:59.324 | INFO | Analyzing chunk frames = [522240,522752) | 20260506 10:08:59.324 | INFO | MemInfo: { | 20260506 10:08:59.324 | INFO | "gpu" : " 17 GB", | 20260506 10:08:59.324 | INFO | "huge" : "", | 20260506 10:08:59.325 | INFO | "huge_shared" : " 27 GB", | 20260506 10:08:59.325 | INFO | "malloc" : " 14 GB", | 20260506 10:08:59.325 | INFO | "pinned" : " 29 GB" | 20260506 10:08:59.325 | INFO | } | 20260506 10:08:59.325 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:08:59.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:08:59.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:59.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:59.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:59.325 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:08:59.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:59.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:59.325 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:08:59.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:59.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:08:59.325 | INFO | Uploaded 36GiB to GPU in 1.60482s (22.4325GiB/s) | 20260506 10:08:59.325 | INFO | Downloaded 34GiB to GPU in 1.52231s (22.3345GiB/s) | 20260506 10:09:04.465 | INFO | Analyzing chunk frames = [522752,523264) | 20260506 10:09:04.465 | INFO | MemInfo: { | 20260506 10:09:04.465 | INFO | "gpu" : " 17 GB", | 20260506 10:09:04.465 | INFO | "huge" : "", | 20260506 10:09:04.465 | INFO | "huge_shared" : " 27 GB", | 20260506 10:09:04.465 | INFO | "malloc" : " 14 GB", | 20260506 10:09:04.465 | INFO | "pinned" : " 29 GB" | 20260506 10:09:04.465 | INFO | } | 20260506 10:09:04.465 | INFO | Perf report for the last 5.14044 seconds: | 20260506 10:09:04.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:09:04.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:04.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:04.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:04.465 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:09:04.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:04.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:04.465 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:09:04.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:04.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:04.465 | INFO | Uploaded 36GiB to GPU in 1.60433s (22.4393GiB/s) | 20260506 10:09:04.465 | INFO | Downloaded 34GiB to GPU in 1.5225s (22.3317GiB/s) | 20260506 10:09:09.603 | INFO | Analyzing chunk frames = [523264,523776) | 20260506 10:09:09.603 | INFO | MemInfo: { | 20260506 10:09:09.603 | INFO | "gpu" : " 17 GB", | 20260506 10:09:09.603 | INFO | "huge" : "", | 20260506 10:09:09.603 | INFO | "huge_shared" : " 27 GB", | 20260506 10:09:09.603 | INFO | "malloc" : " 14 GB", | 20260506 10:09:09.603 | INFO | "pinned" : " 29 GB" | 20260506 10:09:09.603 | INFO | } | 20260506 10:09:09.603 | INFO | Perf report for the last 5.13837 seconds: | 20260506 10:09:09.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:09:09.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:09.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:09.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:09.603 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:09:09.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:09.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:09.603 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:09:09.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:09.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:09.604 | INFO | Uploaded 36GiB to GPU in 1.60518s (22.4273GiB/s) | 20260506 10:09:09.604 | INFO | Downloaded 34GiB to GPU in 1.52293s (22.3254GiB/s) | 20260506 10:09:14.742 | INFO | Analyzing chunk frames = [523776,524288) | 20260506 10:09:14.742 | INFO | MemInfo: { | 20260506 10:09:14.742 | INFO | "gpu" : " 17 GB", | 20260506 10:09:14.742 | INFO | "huge" : "", | 20260506 10:09:14.743 | INFO | "huge_shared" : " 27 GB", | 20260506 10:09:14.743 | INFO | "malloc" : " 14 GB", | 20260506 10:09:14.743 | INFO | "pinned" : " 29 GB" | 20260506 10:09:14.743 | INFO | } | 20260506 10:09:14.743 | INFO | Perf report for the last 5.1395 seconds: | 20260506 10:09:14.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:09:14.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:14.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:14.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:14.743 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:09:14.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:14.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:14.743 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:09:14.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:14.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:14.743 | INFO | Uploaded 36GiB to GPU in 1.60593s (22.4169GiB/s) | 20260506 10:09:14.743 | INFO | Downloaded 34GiB to GPU in 1.52361s (22.3155GiB/s) | 20260506 10:09:14.744 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:09:14.744 | INFO | Total mem usage: 16.5 GiB | 20260506 10:09:14.744 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:09:14.744 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 0.0465%) | 20260506 10:09:14.744 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:09:14.744 | INFO | High Water Breakdown: | 20260506 10:09:14.744 | INFO | 61.1% : 10.1 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:09:14.744 | INFO | 38.8% : 6.4 GiB ( 95.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:09:14.744 | INFO | 0.12% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:09:14.744 | INFO | ------------------------------------------ | 20260506 10:09:14.744 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:09:14.744 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:09:14.744 | INFO | Total mem usage: 26.5 GiB | 20260506 10:09:14.744 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:09:14.744 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:09:14.744 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:09:14.744 | INFO | High Water Breakdown: | 20260506 10:09:14.744 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:09:14.744 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:09:14.744 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:09:14.744 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:09:14.744 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:09:14.744 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:09:14.744 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:09:14.744 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:09:14.744 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:09:14.744 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:09:14.744 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:09:14.744 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:09:14.744 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:09:14.744 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:09:14.744 | INFO | ------------------------------------------ | 20260506 10:09:14.744 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:09:14.744 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:09:14.744 | INFO | Total mem usage: 25.6 GiB | 20260506 10:09:14.744 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:09:14.744 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:09:14.744 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:09:14.744 | INFO | High Water Breakdown: | 20260506 10:09:14.744 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:09:14.744 | INFO | ------------------------------------------ | 20260506 10:09:14.744 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:09:14.744 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:09:14.744 | INFO | Total mem usage: 15.7 GiB | 20260506 10:09:14.744 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:09:14.744 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:09:14.744 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:09:14.744 | INFO | High Water Breakdown: | 20260506 10:09:14.744 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:09:14.744 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:09:14.744 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:09:14.744 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:09:14.744 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:09:14.744 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:09:14.744 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:09:14.744 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:09:14.744 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:09:14.744 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:09:14.744 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:09:14.744 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:09:14.744 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:09:14.744 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:09:14.744 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:09:14.744 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:09:14.744 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:09:14.744 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:09:14.744 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:09:14.744 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:09:14.744 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:09:14.744 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:09:14.744 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:09:14.744 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:09:14.744 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:09:14.744 | INFO | ------------------------------------------ | 20260506 10:09:14.744 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:09:14.744 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:09:14.744 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:09:14.744 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:09:14.744 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 10:09:14.745 | INFO | PerfReport: 0.29%, AggregateStats -- 8.61e-02 / -nan, 4.85e-03 / 0.00e+00 | 20260506 10:09:14.745 | INFO | PerfReport: 10.92%, Baseline -- 3.29e+00 / -nan, 1.74e-02 / 0.00e+00 | 20260506 10:09:14.745 | INFO | PerfReport: 3.06%, Histogram -- 9.22e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260506 10:09:14.745 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.00e+00 / 0.00e+00 | 20260506 10:09:14.745 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.72e+01 / -nan, 4.17e-02 / 0.00e+00 | 20260506 10:09:14.745 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.24e+00 / -nan, 1.30e-01 / 0.00e+00 | 20260506 10:09:14.745 | INFO | PerfReport: 3.74%, Metrics -- 1.13e+00 / -nan, 2.94e-01 / 0.00e+00 | 20260506 10:09:14.745 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.34e-01 / -nan, 6.21e-02 / 0.00e+00 | 20260506 10:09:14.745 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:09:14.745 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:09:14.745 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:09:14.745 | INFO | PerfReport: Full Execution -- 9.00e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 10:09:14.745 | INFO | PerfReport: 18.68%, Upload -- 1.68e+01 / 0.00e+00, 5.04e-01 / 0.00e+00 | 20260506 10:09:14.745 | INFO | PerfReport: 17.93%, Download -- 1.61e+01 / 0.00e+00, 8.23e-01 / 0.00e+00 | 20260506 10:09:14.745 | INFO | PerfReport: 33.49%, Compute -- 3.01e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 10:09:14.745 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:09:14.745 | INFO | PerfReport: 0.00%, LockDownload -- 4.79e-04 / 0.00e+00, 1.26e-04 / 0.00e+00 | 20260506 10:09:14.745 | INFO | PerfReport: 29.32%, LockCompute -- 2.64e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 10:09:14.745 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260506 10:09:14.841 | INFO | Starting a round of Baz IO | 20260506 10:09:17.756 | INFO | Populating RTMetrics report | 20260506 10:09:17.757 | INFO | RTMetrics report populated | 20260506 10:09:17.757 | INFO | Opening tmp RTMetrics json file | 20260506 10:09:17.795 | INFO | Serializing RTMetrics to json | 20260506 10:09:17.797 | INFO | Serialization done, swapping tmp file now... | 20260506 10:09:17.797 | INFO | RTMetrics done | 20260506 10:09:18.429 | INFO | Finished a round of Baz IO: Wrote 1.956 GiB in 3.588 seconds | 20260506 10:09:18.429 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.38% / 0.06% / 50.56% | 20260506 10:09:18.429 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.54096% / 0.000135104% / 5.5631e-05% | 20260506 10:09:18.429 | INFO | Baz IO overall: | 20260506 10:09:18.429 | INFO | BytesWritten : 228616687286 | 20260506 10:09:18.429 | INFO | NumEvents : 109586957609 | 20260506 10:09:19.882 | INFO | Analyzing chunk frames = [524288,524800) | 20260506 10:09:19.882 | INFO | MemInfo: { | 20260506 10:09:19.882 | INFO | "gpu" : " 17 GB", | 20260506 10:09:19.882 | INFO | "huge" : "", | 20260506 10:09:19.882 | INFO | "huge_shared" : " 27 GB", | 20260506 10:09:19.882 | INFO | "malloc" : " 14 GB", | 20260506 10:09:19.882 | INFO | "pinned" : " 29 GB" | 20260506 10:09:19.882 | INFO | } | 20260506 10:09:19.882 | INFO | Perf report for the last 5.13885 seconds: | 20260506 10:09:19.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:09:19.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:19.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:19.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:19.883 | INFO | ANALYSIS : 57.6%, 2.8, 85.5ms, 0 | 20260506 10:09:19.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:19.883 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 10:09:19.883 | INFO | PRE_HQ : 56.7%, 2.1, 60.6ms, 0 | 20260506 10:09:19.883 | INFO | BAZWRITER : 69.4%, 4.8, 170.8ms, 0 | 20260506 10:09:19.883 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 10:09:19.883 | INFO | Uploaded 36GiB to GPU in 1.67275s (21.5214GiB/s) | 20260506 10:09:19.883 | INFO | Downloaded 38GiB to GPU in 1.73641s (21.8842GiB/s) | 20260506 10:09:25.021 | INFO | Analyzing chunk frames = [524800,525312) | 20260506 10:09:25.021 | INFO | MemInfo: { | 20260506 10:09:25.021 | INFO | "gpu" : " 17 GB", | 20260506 10:09:25.021 | INFO | "huge" : "", | 20260506 10:09:25.021 | INFO | "huge_shared" : " 27 GB", | 20260506 10:09:25.021 | INFO | "malloc" : " 14 GB", | 20260506 10:09:25.021 | INFO | "pinned" : " 29 GB" | 20260506 10:09:25.021 | INFO | } | 20260506 10:09:25.021 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:09:25.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:09:25.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:25.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:25.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:25.021 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 10:09:25.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:25.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:25.021 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:09:25.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:25.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:25.022 | INFO | Uploaded 36GiB to GPU in 1.60306s (22.457GiB/s) | 20260506 10:09:25.022 | INFO | Downloaded 34GiB to GPU in 1.5202s (22.3655GiB/s) | 20260506 10:09:30.160 | INFO | Analyzing chunk frames = [525312,525824) | 20260506 10:09:30.160 | INFO | MemInfo: { | 20260506 10:09:30.160 | INFO | "gpu" : " 17 GB", | 20260506 10:09:30.160 | INFO | "huge" : "", | 20260506 10:09:30.161 | INFO | "huge_shared" : " 27 GB", | 20260506 10:09:30.161 | INFO | "malloc" : " 14 GB", | 20260506 10:09:30.161 | INFO | "pinned" : " 29 GB" | 20260506 10:09:30.161 | INFO | } | 20260506 10:09:30.161 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:09:30.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:09:30.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:30.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:30.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:30.161 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260506 10:09:30.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:30.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:30.161 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:09:30.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:30.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:30.161 | INFO | Uploaded 36GiB to GPU in 1.6068s (22.4047GiB/s) | 20260506 10:09:30.161 | INFO | Downloaded 34GiB to GPU in 1.52503s (22.2946GiB/s) | 20260506 10:09:35.300 | INFO | Analyzing chunk frames = [525824,526336) | 20260506 10:09:35.300 | INFO | MemInfo: { | 20260506 10:09:35.300 | INFO | "gpu" : " 17 GB", | 20260506 10:09:35.300 | INFO | "huge" : "", | 20260506 10:09:35.300 | INFO | "huge_shared" : " 27 GB", | 20260506 10:09:35.300 | INFO | "malloc" : " 14 GB", | 20260506 10:09:35.300 | INFO | "pinned" : " 29 GB" | 20260506 10:09:35.300 | INFO | } | 20260506 10:09:35.300 | INFO | Perf report for the last 5.13923 seconds: | 20260506 10:09:35.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:09:35.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:35.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:35.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:35.300 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:09:35.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:35.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:35.300 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260506 10:09:35.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:35.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:35.300 | INFO | Uploaded 36GiB to GPU in 1.60638s (22.4106GiB/s) | 20260506 10:09:35.300 | INFO | Downloaded 34GiB to GPU in 1.52389s (22.3113GiB/s) | 20260506 10:09:40.439 | INFO | Analyzing chunk frames = [526336,526848) | 20260506 10:09:40.439 | INFO | MemInfo: { | 20260506 10:09:40.439 | INFO | "gpu" : " 17 GB", | 20260506 10:09:40.439 | INFO | "huge" : "", | 20260506 10:09:40.439 | INFO | "huge_shared" : " 27 GB", | 20260506 10:09:40.439 | INFO | "malloc" : " 14 GB", | 20260506 10:09:40.440 | INFO | "pinned" : " 29 GB" | 20260506 10:09:40.440 | INFO | } | 20260506 10:09:40.440 | INFO | Perf report for the last 5.14044 seconds: | 20260506 10:09:40.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:09:40.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:40.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:40.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:40.440 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:09:40.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:40.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:40.440 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:09:40.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:40.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:40.441 | INFO | Uploaded 36GiB to GPU in 1.6047s (22.4341GiB/s) | 20260506 10:09:40.441 | INFO | Downloaded 34GiB to GPU in 1.52133s (22.3488GiB/s) | 20260506 10:09:45.582 | INFO | Analyzing chunk frames = [526848,527360) | 20260506 10:09:45.582 | INFO | MemInfo: { | 20260506 10:09:45.582 | INFO | "gpu" : " 17 GB", | 20260506 10:09:45.582 | INFO | "huge" : "", | 20260506 10:09:45.582 | INFO | "huge_shared" : " 27 GB", | 20260506 10:09:45.582 | INFO | "malloc" : " 14 GB", | 20260506 10:09:45.582 | INFO | "pinned" : " 29 GB" | 20260506 10:09:45.582 | INFO | } | 20260506 10:09:45.582 | INFO | Perf report for the last 5.14179 seconds: | 20260506 10:09:45.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:09:45.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:45.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:45.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:45.582 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:09:45.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:45.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:45.582 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:09:45.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:45.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:45.582 | INFO | Uploaded 36GiB to GPU in 1.60524s (22.4265GiB/s) | 20260506 10:09:45.582 | INFO | Downloaded 34GiB to GPU in 1.52293s (22.3254GiB/s) | 20260506 10:09:50.718 | INFO | Analyzing chunk frames = [527360,527872) | 20260506 10:09:50.718 | INFO | MemInfo: { | 20260506 10:09:50.718 | INFO | "gpu" : " 17 GB", | 20260506 10:09:50.718 | INFO | "huge" : "", | 20260506 10:09:50.718 | INFO | "huge_shared" : " 27 GB", | 20260506 10:09:50.718 | INFO | "malloc" : " 14 GB", | 20260506 10:09:50.718 | INFO | "pinned" : " 29 GB" | 20260506 10:09:50.718 | INFO | } | 20260506 10:09:50.718 | INFO | Perf report for the last 5.1358 seconds: | 20260506 10:09:50.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:09:50.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:50.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:50.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:50.718 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:09:50.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:50.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:50.718 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 10:09:50.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:50.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:50.718 | INFO | Uploaded 36GiB to GPU in 1.60591s (22.4172GiB/s) | 20260506 10:09:50.718 | INFO | Downloaded 34GiB to GPU in 1.52288s (22.3262GiB/s) | 20260506 10:09:55.857 | INFO | Analyzing chunk frames = [527872,528384) | 20260506 10:09:55.857 | INFO | MemInfo: { | 20260506 10:09:55.857 | INFO | "gpu" : " 17 GB", | 20260506 10:09:55.857 | INFO | "huge" : "", | 20260506 10:09:55.857 | INFO | "huge_shared" : " 27 GB", | 20260506 10:09:55.857 | INFO | "malloc" : " 14 GB", | 20260506 10:09:55.857 | INFO | "pinned" : " 29 GB" | 20260506 10:09:55.857 | INFO | } | 20260506 10:09:55.857 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:09:55.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:09:55.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:55.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:55.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:55.857 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:09:55.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:55.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:55.857 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:09:55.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:55.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:09:55.858 | INFO | Uploaded 36GiB to GPU in 1.60595s (22.4166GiB/s) | 20260506 10:09:55.858 | INFO | Downloaded 34GiB to GPU in 1.52389s (22.3114GiB/s) | 20260506 10:09:55.954 | INFO | Starting a round of Baz IO | 20260506 10:09:58.877 | INFO | Populating RTMetrics report | 20260506 10:09:58.877 | INFO | RTMetrics report populated | 20260506 10:09:58.877 | INFO | Opening tmp RTMetrics json file | 20260506 10:09:58.900 | INFO | Serializing RTMetrics to json | 20260506 10:09:58.961 | INFO | Serialization done, swapping tmp file now... | 20260506 10:09:58.983 | INFO | RTMetrics done | 20260506 10:09:59.195 | INFO | Finished a round of Baz IO: Wrote 1.956 GiB in 3.24 seconds | 20260506 10:09:59.195 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.32% / 0.06% / 50.62% | 20260506 10:09:59.195 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.5074% / 0.000166893% / 4.37101e-05% | 20260506 10:09:59.195 | INFO | Baz IO overall: | 20260506 10:09:59.195 | INFO | BytesWritten : 230717261353 | 20260506 10:09:59.195 | INFO | NumEvents : 110606498112 | 20260506 10:10:00.997 | INFO | Analyzing chunk frames = [528384,528896) | 20260506 10:10:00.997 | INFO | MemInfo: { | 20260506 10:10:00.997 | INFO | "gpu" : " 17 GB", | 20260506 10:10:00.997 | INFO | "huge" : "", | 20260506 10:10:00.997 | INFO | "huge_shared" : " 27 GB", | 20260506 10:10:00.997 | INFO | "malloc" : " 14 GB", | 20260506 10:10:00.997 | INFO | "pinned" : " 29 GB" | 20260506 10:10:00.997 | INFO | } | 20260506 10:10:00.997 | INFO | Perf report for the last 5.13952 seconds: | 20260506 10:10:00.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:10:00.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:00.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:00.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:00.997 | INFO | ANALYSIS : 57.6%, 2.9, 86.7ms, 0 | 20260506 10:10:00.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:00.997 | INFO | RT_METRICS : 46.2%, 1.0, 24.3ms, 0 | 20260506 10:10:00.997 | INFO | PRE_HQ : 57.1%, 2.1, 61.3ms, 0 | 20260506 10:10:00.997 | INFO | BAZWRITER : 62.6%, 4.6, 149.5ms, 0 | 20260506 10:10:00.997 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 10:10:00.997 | INFO | Uploaded 36GiB to GPU in 1.66398s (21.6349GiB/s) | 20260506 10:10:00.997 | INFO | Downloaded 38GiB to GPU in 1.72625s (22.0131GiB/s) | 20260506 10:10:06.136 | INFO | Analyzing chunk frames = [528896,529408) | 20260506 10:10:06.136 | INFO | MemInfo: { | 20260506 10:10:06.136 | INFO | "gpu" : " 17 GB", | 20260506 10:10:06.136 | INFO | "huge" : "", | 20260506 10:10:06.136 | INFO | "huge_shared" : " 27 GB", | 20260506 10:10:06.136 | INFO | "malloc" : " 14 GB", | 20260506 10:10:06.136 | INFO | "pinned" : " 29 GB" | 20260506 10:10:06.136 | INFO | } | 20260506 10:10:06.136 | INFO | Perf report for the last 5.13922 seconds: | 20260506 10:10:06.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:10:06.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:06.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:06.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:06.136 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:10:06.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:06.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:06.136 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 10:10:06.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:06.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:06.136 | INFO | Uploaded 36GiB to GPU in 1.60302s (22.4576GiB/s) | 20260506 10:10:06.136 | INFO | Downloaded 34GiB to GPU in 1.52008s (22.3673GiB/s) | 20260506 10:10:11.275 | INFO | Analyzing chunk frames = [529408,529920) | 20260506 10:10:11.275 | INFO | MemInfo: { | 20260506 10:10:11.275 | INFO | "gpu" : " 17 GB", | 20260506 10:10:11.275 | INFO | "huge" : "", | 20260506 10:10:11.275 | INFO | "huge_shared" : " 27 GB", | 20260506 10:10:11.275 | INFO | "malloc" : " 14 GB", | 20260506 10:10:11.275 | INFO | "pinned" : " 29 GB" | 20260506 10:10:11.275 | INFO | } | 20260506 10:10:11.275 | INFO | Perf report for the last 5.13927 seconds: | 20260506 10:10:11.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:10:11.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:11.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:11.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:11.275 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260506 10:10:11.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:11.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:11.275 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:10:11.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:11.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:11.276 | INFO | Uploaded 36GiB to GPU in 1.60662s (22.4072GiB/s) | 20260506 10:10:11.276 | INFO | Downloaded 34GiB to GPU in 1.52439s (22.304GiB/s) | 20260506 10:10:16.414 | INFO | Analyzing chunk frames = [529920,530432) | 20260506 10:10:16.414 | INFO | MemInfo: { | 20260506 10:10:16.414 | INFO | "gpu" : " 17 GB", | 20260506 10:10:16.414 | INFO | "huge" : "", | 20260506 10:10:16.414 | INFO | "huge_shared" : " 27 GB", | 20260506 10:10:16.415 | INFO | "malloc" : " 14 GB", | 20260506 10:10:16.415 | INFO | "pinned" : " 29 GB" | 20260506 10:10:16.415 | INFO | } | 20260506 10:10:16.415 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:10:16.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:10:16.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:16.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:16.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:16.415 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:10:16.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:16.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:16.415 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 10:10:16.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:16.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:16.415 | INFO | Uploaded 36GiB to GPU in 1.60711s (22.4005GiB/s) | 20260506 10:10:16.415 | INFO | Downloaded 34GiB to GPU in 1.52543s (22.2888GiB/s) | 20260506 10:10:21.554 | INFO | Analyzing chunk frames = [530432,530944) | 20260506 10:10:21.554 | INFO | MemInfo: { | 20260506 10:10:21.554 | INFO | "gpu" : " 17 GB", | 20260506 10:10:21.554 | INFO | "huge" : "", | 20260506 10:10:21.554 | INFO | "huge_shared" : " 27 GB", | 20260506 10:10:21.554 | INFO | "malloc" : " 14 GB", | 20260506 10:10:21.554 | INFO | "pinned" : " 29 GB" | 20260506 10:10:21.554 | INFO | } | 20260506 10:10:21.554 | INFO | Perf report for the last 5.1393 seconds: | 20260506 10:10:21.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:10:21.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:21.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:21.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:21.554 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:10:21.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:21.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:21.554 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 10:10:21.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:21.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:21.554 | INFO | Uploaded 36GiB to GPU in 1.60409s (22.4427GiB/s) | 20260506 10:10:21.554 | INFO | Downloaded 34GiB to GPU in 1.52212s (22.3372GiB/s) | 20260506 10:10:26.693 | INFO | Analyzing chunk frames = [530944,531456) | 20260506 10:10:26.693 | INFO | MemInfo: { | 20260506 10:10:26.693 | INFO | "gpu" : " 17 GB", | 20260506 10:10:26.693 | INFO | "huge" : "", | 20260506 10:10:26.693 | INFO | "huge_shared" : " 27 GB", | 20260506 10:10:26.693 | INFO | "malloc" : " 14 GB", | 20260506 10:10:26.693 | INFO | "pinned" : " 29 GB" | 20260506 10:10:26.693 | INFO | } | 20260506 10:10:26.693 | INFO | Perf report for the last 5.13929 seconds: | 20260506 10:10:26.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:10:26.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:26.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:26.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:26.693 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:10:26.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:26.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:26.693 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:10:26.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:26.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:26.694 | INFO | Uploaded 36GiB to GPU in 1.60468s (22.4344GiB/s) | 20260506 10:10:26.694 | INFO | Downloaded 34GiB to GPU in 1.52282s (22.327GiB/s) | 20260506 10:10:31.832 | INFO | Analyzing chunk frames = [531456,531968) | 20260506 10:10:31.832 | INFO | MemInfo: { | 20260506 10:10:31.832 | INFO | "gpu" : " 17 GB", | 20260506 10:10:31.832 | INFO | "huge" : "", | 20260506 10:10:31.832 | INFO | "huge_shared" : " 27 GB", | 20260506 10:10:31.833 | INFO | "malloc" : " 14 GB", | 20260506 10:10:31.833 | INFO | "pinned" : " 29 GB" | 20260506 10:10:31.833 | INFO | } | 20260506 10:10:31.833 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:10:31.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:10:31.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:31.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:31.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:31.833 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:10:31.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:31.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:31.833 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:10:31.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:31.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:31.833 | INFO | Uploaded 36GiB to GPU in 1.60525s (22.4264GiB/s) | 20260506 10:10:31.833 | INFO | Downloaded 34GiB to GPU in 1.52389s (22.3113GiB/s) | 20260506 10:10:36.972 | INFO | Analyzing chunk frames = [531968,532480) | 20260506 10:10:36.972 | INFO | MemInfo: { | 20260506 10:10:36.972 | INFO | "gpu" : " 17 GB", | 20260506 10:10:36.972 | INFO | "huge" : "", | 20260506 10:10:36.972 | INFO | "huge_shared" : " 27 GB", | 20260506 10:10:36.972 | INFO | "malloc" : " 14 GB", | 20260506 10:10:36.972 | INFO | "pinned" : " 29 GB" | 20260506 10:10:36.972 | INFO | } | 20260506 10:10:36.972 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:10:36.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:10:36.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:36.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:36.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:36.972 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:10:36.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:36.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:36.972 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:10:36.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:36.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:36.973 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:10:36.973 | INFO | Total mem usage: 16.5 GiB | 20260506 10:10:36.973 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:10:36.973 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -0.626%) | 20260506 10:10:36.973 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:10:36.973 | INFO | High Water Breakdown: | 20260506 10:10:36.973 | INFO | 61% : 10 GiB ( 99.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:10:36.973 | INFO | 38.9% : 6.38 GiB ( 95.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:10:36.973 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:10:36.973 | INFO | ------------------------------------------ | 20260506 10:10:36.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:10:36.973 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:10:36.973 | INFO | Total mem usage: 26.5 GiB | 20260506 10:10:36.973 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:10:36.973 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:10:36.973 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:10:36.973 | INFO | High Water Breakdown: | 20260506 10:10:36.973 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:10:36.973 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:10:36.973 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:10:36.973 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:10:36.973 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:10:36.973 | INFO | Uploaded 36GiB to GPU in 1.60568s (22.4204GiB/s) | 20260506 10:10:36.973 | INFO | Downloaded 34GiB to GPU in 1.52369s (22.3143GiB/s) | 20260506 10:10:36.973 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:10:36.973 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:10:36.973 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:10:36.973 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:10:36.973 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:10:36.973 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:10:36.973 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:10:36.973 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:10:36.973 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:10:36.973 | INFO | ------------------------------------------ | 20260506 10:10:36.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:10:36.973 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:10:36.973 | INFO | Total mem usage: 25.6 GiB | 20260506 10:10:36.973 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:10:36.973 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:10:36.973 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:10:36.973 | INFO | High Water Breakdown: | 20260506 10:10:36.973 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:10:36.973 | INFO | ------------------------------------------ | 20260506 10:10:36.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:10:36.973 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:10:36.973 | INFO | Total mem usage: 15.7 GiB | 20260506 10:10:36.973 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:10:36.973 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:10:36.973 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:10:36.973 | INFO | High Water Breakdown: | 20260506 10:10:36.973 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:10:36.973 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:10:36.973 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:10:36.973 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:10:36.973 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:10:36.973 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:10:36.973 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:10:36.973 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:10:36.973 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:10:36.973 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:10:36.973 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:10:36.973 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:10:36.973 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:10:36.973 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:10:36.973 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:10:36.973 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:10:36.973 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:10:36.973 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:10:36.973 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:10:36.973 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:10:36.973 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:10:36.973 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:10:36.973 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:10:36.973 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:10:36.973 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:10:36.973 | INFO | ------------------------------------------ | 20260506 10:10:36.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:10:36.973 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:10:36.973 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:10:36.973 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:10:36.973 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260506 10:10:36.973 | INFO | PerfReport: 0.29%, AggregateStats -- 8.63e-02 / -nan, 4.47e-03 / 0.00e+00 | 20260506 10:10:36.973 | INFO | PerfReport: 10.93%, Baseline -- 3.29e+00 / -nan, 1.86e-02 / 0.00e+00 | 20260506 10:10:36.973 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.26e-02 / 0.00e+00 | 20260506 10:10:36.974 | INFO | PerfReport: 16.05%, DME -- 3.87e+01 / -nan, 3.15e+00 / 0.00e+00 | 20260506 10:10:36.974 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.72e+01 / -nan, 4.34e-02 / 0.00e+00 | 20260506 10:10:36.974 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.24e+00 / -nan, 1.31e-01 / 0.00e+00 | 20260506 10:10:36.974 | INFO | PerfReport: 3.72%, Metrics -- 1.12e+00 / -nan, 1.75e-01 / 0.00e+00 | 20260506 10:10:36.974 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.34e-01 / -nan, 6.19e-02 / 0.00e+00 | 20260506 10:10:36.974 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:10:36.974 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:10:36.974 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:10:36.974 | INFO | PerfReport: Full Execution -- 9.00e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:10:36.974 | INFO | PerfReport: 18.66%, Upload -- 1.68e+01 / 0.00e+00, 4.77e-01 / 0.00e+00 | 20260506 10:10:36.974 | INFO | PerfReport: 17.92%, Download -- 1.61e+01 / 0.00e+00, 8.15e-01 / 0.00e+00 | 20260506 10:10:36.974 | INFO | PerfReport: 33.47%, Compute -- 3.01e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260506 10:10:36.974 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260506 10:10:36.974 | INFO | PerfReport: 0.00%, LockDownload -- 4.85e-04 / 0.00e+00, 1.42e-04 / 0.00e+00 | 20260506 10:10:36.974 | INFO | PerfReport: 29.37%, LockCompute -- 2.64e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 10:10:36.974 | INFO | PerfReport: 0.01%, Misc -- 1.22e-02 / 0.00e+00, 9.56e-03 / 0.00e+00 | 20260506 10:10:37.070 | INFO | Starting a round of Baz IO | 20260506 10:10:39.984 | INFO | Populating RTMetrics report | 20260506 10:10:39.984 | INFO | RTMetrics report populated | 20260506 10:10:39.984 | INFO | Opening tmp RTMetrics json file | 20260506 10:10:40.024 | INFO | Serializing RTMetrics to json | 20260506 10:10:40.025 | INFO | Serialization done, swapping tmp file now... | 20260506 10:10:40.025 | INFO | RTMetrics done | 20260506 10:10:40.479 | INFO | Finished a round of Baz IO: Wrote 1.959 GiB in 3.408 seconds | 20260506 10:10:40.479 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.26% / 0.06% / 50.68% | 20260506 10:10:40.479 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.48359% / 0.000139078% / 5.96046e-05% | 20260506 10:10:40.479 | INFO | Baz IO overall: | 20260506 10:10:40.479 | INFO | BytesWritten : 232820730561 | 20260506 10:10:40.479 | INFO | NumEvents : 111627234501 | 20260506 10:10:42.111 | INFO | Analyzing chunk frames = [532480,532992) | 20260506 10:10:42.111 | INFO | MemInfo: { | 20260506 10:10:42.111 | INFO | "gpu" : " 17 GB", | 20260506 10:10:42.111 | INFO | "huge" : "", | 20260506 10:10:42.111 | INFO | "huge_shared" : " 27 GB", | 20260506 10:10:42.111 | INFO | "malloc" : " 14 GB", | 20260506 10:10:42.111 | INFO | "pinned" : " 29 GB" | 20260506 10:10:42.111 | INFO | } | 20260506 10:10:42.111 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:10:42.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:10:42.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:42.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:42.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:42.111 | INFO | ANALYSIS : 57.6%, 2.8, 85.3ms, 0 | 20260506 10:10:42.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:42.111 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260506 10:10:42.111 | INFO | PRE_HQ : 56.7%, 2.1, 61.6ms, 0 | 20260506 10:10:42.112 | INFO | BAZWRITER : 65.8%, 4.8, 161.8ms, 0 | 20260506 10:10:42.112 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 10:10:42.112 | INFO | Uploaded 36GiB to GPU in 1.67389s (21.5068GiB/s) | 20260506 10:10:42.112 | INFO | Downloaded 38GiB to GPU in 1.73975s (21.8422GiB/s) | 20260506 10:10:47.250 | INFO | Analyzing chunk frames = [532992,533504) | 20260506 10:10:47.250 | INFO | MemInfo: { | 20260506 10:10:47.250 | INFO | "gpu" : " 17 GB", | 20260506 10:10:47.250 | INFO | "huge" : "", | 20260506 10:10:47.251 | INFO | "huge_shared" : " 27 GB", | 20260506 10:10:47.251 | INFO | "malloc" : " 14 GB", | 20260506 10:10:47.251 | INFO | "pinned" : " 29 GB" | 20260506 10:10:47.251 | INFO | } | 20260506 10:10:47.251 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:10:47.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:10:47.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:47.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:47.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:47.251 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 10:10:47.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:47.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:47.251 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:10:47.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:47.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:47.251 | INFO | Uploaded 36GiB to GPU in 1.60346s (22.4515GiB/s) | 20260506 10:10:47.251 | INFO | Downloaded 34GiB to GPU in 1.52102s (22.3534GiB/s) | 20260506 10:10:52.390 | INFO | Analyzing chunk frames = [533504,534016) | 20260506 10:10:52.390 | INFO | MemInfo: { | 20260506 10:10:52.390 | INFO | "gpu" : " 17 GB", | 20260506 10:10:52.390 | INFO | "huge" : "", | 20260506 10:10:52.390 | INFO | "huge_shared" : " 27 GB", | 20260506 10:10:52.390 | INFO | "malloc" : " 14 GB", | 20260506 10:10:52.390 | INFO | "pinned" : " 29 GB" | 20260506 10:10:52.390 | INFO | } | 20260506 10:10:52.390 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:10:52.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:10:52.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:52.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:52.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:52.390 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 10:10:52.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:52.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:52.390 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 10:10:52.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:52.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:52.390 | INFO | Uploaded 36GiB to GPU in 1.60598s (22.4162GiB/s) | 20260506 10:10:52.390 | INFO | Downloaded 34GiB to GPU in 1.52481s (22.2978GiB/s) | 20260506 10:10:57.529 | INFO | Analyzing chunk frames = [534016,534528) | 20260506 10:10:57.529 | INFO | MemInfo: { | 20260506 10:10:57.529 | INFO | "gpu" : " 17 GB", | 20260506 10:10:57.529 | INFO | "huge" : "", | 20260506 10:10:57.529 | INFO | "huge_shared" : " 27 GB", | 20260506 10:10:57.529 | INFO | "malloc" : " 14 GB", | 20260506 10:10:57.529 | INFO | "pinned" : " 29 GB" | 20260506 10:10:57.529 | INFO | } | 20260506 10:10:57.529 | INFO | Perf report for the last 5.13921 seconds: | 20260506 10:10:57.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:10:57.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:57.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:57.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:57.529 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:10:57.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:57.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:57.529 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260506 10:10:57.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:57.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:10:57.530 | INFO | Uploaded 36GiB to GPU in 1.60576s (22.4193GiB/s) | 20260506 10:10:57.530 | INFO | Downloaded 34GiB to GPU in 1.52331s (22.3198GiB/s) | 20260506 10:11:02.668 | INFO | Analyzing chunk frames = [534528,535040) | 20260506 10:11:02.669 | INFO | MemInfo: { | 20260506 10:11:02.669 | INFO | "gpu" : " 17 GB", | 20260506 10:11:02.669 | INFO | "huge" : "", | 20260506 10:11:02.669 | INFO | "huge_shared" : " 27 GB", | 20260506 10:11:02.669 | INFO | "malloc" : " 14 GB", | 20260506 10:11:02.669 | INFO | "pinned" : " 29 GB" | 20260506 10:11:02.669 | INFO | } | 20260506 10:11:02.669 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:11:02.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:11:02.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:02.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:02.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:02.669 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:11:02.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:02.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:02.669 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:11:02.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:02.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:02.669 | INFO | Uploaded 36GiB to GPU in 1.6063s (22.4117GiB/s) | 20260506 10:11:02.669 | INFO | Downloaded 34GiB to GPU in 1.52272s (22.3285GiB/s) | 20260506 10:11:07.808 | INFO | Analyzing chunk frames = [535040,535552) | 20260506 10:11:07.808 | INFO | MemInfo: { | 20260506 10:11:07.808 | INFO | "gpu" : " 17 GB", | 20260506 10:11:07.808 | INFO | "huge" : "", | 20260506 10:11:07.808 | INFO | "huge_shared" : " 27 GB", | 20260506 10:11:07.808 | INFO | "malloc" : " 14 GB", | 20260506 10:11:07.808 | INFO | "pinned" : " 29 GB" | 20260506 10:11:07.808 | INFO | } | 20260506 10:11:07.808 | INFO | Perf report for the last 5.13939 seconds: | 20260506 10:11:07.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:11:07.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:07.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:07.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:07.808 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 10:11:07.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:07.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:07.808 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:11:07.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:07.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:07.809 | INFO | Uploaded 36GiB to GPU in 1.60451s (22.4367GiB/s) | 20260506 10:11:07.809 | INFO | Downloaded 34GiB to GPU in 1.52206s (22.3381GiB/s) | 20260506 10:11:12.947 | INFO | Analyzing chunk frames = [535552,536064) | 20260506 10:11:12.947 | INFO | MemInfo: { | 20260506 10:11:12.947 | INFO | "gpu" : " 17 GB", | 20260506 10:11:12.947 | INFO | "huge" : "", | 20260506 10:11:12.947 | INFO | "huge_shared" : " 27 GB", | 20260506 10:11:12.947 | INFO | "malloc" : " 14 GB", | 20260506 10:11:12.947 | INFO | "pinned" : " 29 GB" | 20260506 10:11:12.947 | INFO | } | 20260506 10:11:12.947 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:11:12.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:11:12.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:12.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:12.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:12.948 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:11:12.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:12.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:12.948 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 10:11:12.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:12.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:12.948 | INFO | Uploaded 36GiB to GPU in 1.60573s (22.4198GiB/s) | 20260506 10:11:12.948 | INFO | Downloaded 34GiB to GPU in 1.52336s (22.3191GiB/s) | 20260506 10:11:18.087 | INFO | Analyzing chunk frames = [536064,536576) | 20260506 10:11:18.087 | INFO | MemInfo: { | 20260506 10:11:18.087 | INFO | "gpu" : " 17 GB", | 20260506 10:11:18.087 | INFO | "huge" : "", | 20260506 10:11:18.087 | INFO | "huge_shared" : " 27 GB", | 20260506 10:11:18.087 | INFO | "malloc" : " 14 GB", | 20260506 10:11:18.087 | INFO | "pinned" : " 29 GB" | 20260506 10:11:18.087 | INFO | } | 20260506 10:11:18.087 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:11:18.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:11:18.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:18.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:18.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:18.087 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:11:18.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:18.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:18.087 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:11:18.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:18.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:18.087 | INFO | Uploaded 36GiB to GPU in 1.605s (22.4299GiB/s) | 20260506 10:11:18.087 | INFO | Downloaded 34GiB to GPU in 1.52382s (22.3124GiB/s) | 20260506 10:11:18.184 | INFO | Starting a round of Baz IO | 20260506 10:11:21.099 | INFO | Populating RTMetrics report | 20260506 10:11:21.099 | INFO | RTMetrics report populated | 20260506 10:11:21.099 | INFO | Opening tmp RTMetrics json file | 20260506 10:11:21.100 | INFO | Serializing RTMetrics to json | 20260506 10:11:21.191 | INFO | Serialization done, swapping tmp file now... | 20260506 10:11:21.276 | INFO | RTMetrics done | 20260506 10:11:21.475 | INFO | Finished a round of Baz IO: Wrote 1.958 GiB in 3.291 seconds | 20260506 10:11:21.475 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.19% / 0.06% / 50.74% | 20260506 10:11:21.475 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.44433% / 0.000115236% / 3.57628e-05% | 20260506 10:11:21.475 | INFO | Baz IO overall: | 20260506 10:11:21.475 | INFO | BytesWritten : 234923379118 | 20260506 10:11:21.475 | INFO | NumEvents : 112647731013 | 20260506 10:11:23.226 | INFO | Analyzing chunk frames = [536576,537088) | 20260506 10:11:23.226 | INFO | MemInfo: { | 20260506 10:11:23.226 | INFO | "gpu" : " 17 GB", | 20260506 10:11:23.226 | INFO | "huge" : "", | 20260506 10:11:23.226 | INFO | "huge_shared" : " 27 GB", | 20260506 10:11:23.226 | INFO | "malloc" : " 14 GB", | 20260506 10:11:23.226 | INFO | "pinned" : " 29 GB" | 20260506 10:11:23.226 | INFO | } | 20260506 10:11:23.226 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:11:23.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:11:23.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:23.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:23.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:23.226 | INFO | ANALYSIS : 57.6%, 2.9, 86.3ms, 0 | 20260506 10:11:23.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:23.226 | INFO | RT_METRICS : 47.6%, 1.0, 25.0ms, 0 | 20260506 10:11:23.226 | INFO | PRE_HQ : 56.6%, 2.1, 61.2ms, 0 | 20260506 10:11:23.226 | INFO | BAZWRITER : 63.6%, 4.6, 151.7ms, 0 | 20260506 10:11:23.226 | INFO | BAZ_SAMPLING : 2.9%, 1.0, 1.5ms, 0 | 20260506 10:11:23.226 | INFO | Uploaded 36GiB to GPU in 1.65693s (21.727GiB/s) | 20260506 10:11:23.227 | INFO | Downloaded 38GiB to GPU in 1.72224s (22.0643GiB/s) | 20260506 10:11:28.365 | INFO | Analyzing chunk frames = [537088,537600) | 20260506 10:11:28.365 | INFO | MemInfo: { | 20260506 10:11:28.365 | INFO | "gpu" : " 17 GB", | 20260506 10:11:28.365 | INFO | "huge" : "", | 20260506 10:11:28.365 | INFO | "huge_shared" : " 27 GB", | 20260506 10:11:28.365 | INFO | "malloc" : " 14 GB", | 20260506 10:11:28.366 | INFO | "pinned" : " 29 GB" | 20260506 10:11:28.366 | INFO | } | 20260506 10:11:28.366 | INFO | Perf report for the last 5.13939 seconds: | 20260506 10:11:28.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:11:28.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:28.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:28.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:28.366 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 10:11:28.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:28.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:28.366 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 10:11:28.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:28.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:28.366 | INFO | Uploaded 36GiB to GPU in 1.60303s (22.4574GiB/s) | 20260506 10:11:28.366 | INFO | Downloaded 34GiB to GPU in 1.5202s (22.3655GiB/s) | 20260506 10:11:33.505 | INFO | Analyzing chunk frames = [537600,538112) | 20260506 10:11:33.505 | INFO | MemInfo: { | 20260506 10:11:33.505 | INFO | "gpu" : " 17 GB", | 20260506 10:11:33.505 | INFO | "huge" : "", | 20260506 10:11:33.505 | INFO | "huge_shared" : " 27 GB", | 20260506 10:11:33.505 | INFO | "malloc" : " 14 GB", | 20260506 10:11:33.505 | INFO | "pinned" : " 29 GB" | 20260506 10:11:33.505 | INFO | } | 20260506 10:11:33.505 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:11:33.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:11:33.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:33.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:33.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:33.505 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 10:11:33.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:33.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:33.505 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 10:11:33.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:33.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:33.505 | INFO | Uploaded 36GiB to GPU in 1.60805s (22.3874GiB/s) | 20260506 10:11:33.505 | INFO | Downloaded 34GiB to GPU in 1.52602s (22.2801GiB/s) | 20260506 10:11:38.645 | INFO | Analyzing chunk frames = [538112,538624) | 20260506 10:11:38.646 | INFO | MemInfo: { | 20260506 10:11:38.646 | INFO | "gpu" : " 17 GB", | 20260506 10:11:38.646 | INFO | "huge" : "", | 20260506 10:11:38.646 | INFO | "huge_shared" : " 27 GB", | 20260506 10:11:38.646 | INFO | "malloc" : " 14 GB", | 20260506 10:11:38.646 | INFO | "pinned" : " 29 GB" | 20260506 10:11:38.646 | INFO | } | 20260506 10:11:38.646 | INFO | Perf report for the last 5.14076 seconds: | 20260506 10:11:38.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:11:38.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:38.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:38.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:38.646 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:11:38.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:38.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:38.646 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 10:11:38.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:38.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:38.646 | INFO | Uploaded 36GiB to GPU in 1.60673s (22.4057GiB/s) | 20260506 10:11:38.646 | INFO | Downloaded 34GiB to GPU in 1.52367s (22.3146GiB/s) | 20260506 10:11:43.783 | INFO | Analyzing chunk frames = [538624,539136) | 20260506 10:11:43.783 | INFO | MemInfo: { | 20260506 10:11:43.783 | INFO | "gpu" : " 17 GB", | 20260506 10:11:43.784 | INFO | "huge" : "", | 20260506 10:11:43.784 | INFO | "huge_shared" : " 27 GB", | 20260506 10:11:43.784 | INFO | "malloc" : " 14 GB", | 20260506 10:11:43.784 | INFO | "pinned" : " 29 GB" | 20260506 10:11:43.784 | INFO | } | 20260506 10:11:43.784 | INFO | Perf report for the last 5.13798 seconds: | 20260506 10:11:43.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:11:43.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:43.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:43.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:43.784 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260506 10:11:43.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:43.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:43.784 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 10:11:43.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:43.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:43.784 | INFO | Uploaded 36GiB to GPU in 1.60513s (22.4281GiB/s) | 20260506 10:11:43.784 | INFO | Downloaded 34GiB to GPU in 1.52208s (22.3379GiB/s) | 20260506 10:11:48.923 | INFO | Analyzing chunk frames = [539136,539648) | 20260506 10:11:48.923 | INFO | MemInfo: { | 20260506 10:11:48.923 | INFO | "gpu" : " 17 GB", | 20260506 10:11:48.923 | INFO | "huge" : "", | 20260506 10:11:48.923 | INFO | "huge_shared" : " 27 GB", | 20260506 10:11:48.923 | INFO | "malloc" : " 14 GB", | 20260506 10:11:48.923 | INFO | "pinned" : " 29 GB" | 20260506 10:11:48.923 | INFO | } | 20260506 10:11:48.923 | INFO | Perf report for the last 5.13903 seconds: | 20260506 10:11:48.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:11:48.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:48.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:48.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:48.923 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:11:48.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:48.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:48.923 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 10:11:48.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:48.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:48.923 | INFO | Uploaded 36GiB to GPU in 1.60551s (22.4228GiB/s) | 20260506 10:11:48.924 | INFO | Downloaded 34GiB to GPU in 1.52297s (22.3248GiB/s) | 20260506 10:11:54.062 | INFO | Analyzing chunk frames = [539648,540160) | 20260506 10:11:54.062 | INFO | MemInfo: { | 20260506 10:11:54.062 | INFO | "gpu" : " 17 GB", | 20260506 10:11:54.062 | INFO | "huge" : "", | 20260506 10:11:54.062 | INFO | "huge_shared" : " 27 GB", | 20260506 10:11:54.062 | INFO | "malloc" : " 14 GB", | 20260506 10:11:54.062 | INFO | "pinned" : " 29 GB" | 20260506 10:11:54.062 | INFO | } | 20260506 10:11:54.062 | INFO | Perf report for the last 5.13946 seconds: | 20260506 10:11:54.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:11:54.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:54.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:54.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:54.062 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:11:54.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:54.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:54.062 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 10:11:54.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:54.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:54.063 | INFO | Uploaded 36GiB to GPU in 1.60609s (22.4147GiB/s) | 20260506 10:11:54.063 | INFO | Downloaded 34GiB to GPU in 1.52302s (22.3241GiB/s) | 20260506 10:11:59.201 | INFO | Analyzing chunk frames = [540160,540672) | 20260506 10:11:59.201 | INFO | MemInfo: { | 20260506 10:11:59.201 | INFO | "gpu" : " 17 GB", | 20260506 10:11:59.201 | INFO | "huge" : "", | 20260506 10:11:59.201 | INFO | "huge_shared" : " 27 GB", | 20260506 10:11:59.201 | INFO | "malloc" : " 14 GB", | 20260506 10:11:59.201 | INFO | "pinned" : " 29 GB" | 20260506 10:11:59.202 | INFO | } | 20260506 10:11:59.202 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:11:59.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:11:59.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:59.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:59.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:59.202 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:11:59.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:59.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:59.202 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:11:59.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:59.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:11:59.202 | INFO | Uploaded 36GiB to GPU in 1.6053s (22.4257GiB/s) | 20260506 10:11:59.202 | INFO | Downloaded 34GiB to GPU in 1.52323s (22.321GiB/s) | 20260506 10:11:59.202 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:11:59.202 | INFO | Total mem usage: 16.5 GiB | 20260506 10:11:59.202 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:11:59.202 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -4.03%) | 20260506 10:11:59.202 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:11:59.202 | INFO | High Water Breakdown: | 20260506 10:11:59.202 | INFO | 60.5% : 9.61 GiB ( 95.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:11:59.202 | INFO | 39.4% : 6.25 GiB ( 93.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:11:59.202 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:11:59.202 | INFO | ------------------------------------------ | 20260506 10:11:59.202 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:11:59.202 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:11:59.202 | INFO | Total mem usage: 26.5 GiB | 20260506 10:11:59.202 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:11:59.202 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:11:59.202 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:11:59.202 | INFO | High Water Breakdown: | 20260506 10:11:59.202 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:11:59.202 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:11:59.203 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:11:59.203 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:11:59.203 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:11:59.203 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:11:59.203 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:11:59.203 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:11:59.203 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:11:59.203 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:11:59.203 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:11:59.203 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:11:59.203 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:11:59.203 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:11:59.203 | INFO | ------------------------------------------ | 20260506 10:11:59.203 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:11:59.203 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:11:59.203 | INFO | Total mem usage: 25.6 GiB | 20260506 10:11:59.203 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:11:59.203 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:11:59.203 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:11:59.203 | INFO | High Water Breakdown: | 20260506 10:11:59.203 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:11:59.203 | INFO | ------------------------------------------ | 20260506 10:11:59.203 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:11:59.203 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:11:59.203 | INFO | Total mem usage: 15.7 GiB | 20260506 10:11:59.203 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:11:59.203 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:11:59.203 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:11:59.203 | INFO | High Water Breakdown: | 20260506 10:11:59.203 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:11:59.203 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:11:59.203 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:11:59.203 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:11:59.203 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:11:59.203 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:11:59.203 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:11:59.203 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:11:59.203 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:11:59.203 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:11:59.203 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:11:59.203 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:11:59.203 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:11:59.203 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:11:59.203 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:11:59.203 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:11:59.203 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:11:59.203 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:11:59.203 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:11:59.203 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:11:59.203 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:11:59.203 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:11:59.203 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:11:59.203 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:11:59.203 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:11:59.203 | INFO | ------------------------------------------ | 20260506 10:11:59.203 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:11:59.203 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:11:59.203 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:11:59.203 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:11:59.203 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 10:11:59.203 | INFO | PerfReport: 0.29%, AggregateStats -- 8.62e-02 / -nan, 5.33e-03 / 0.00e+00 | 20260506 10:11:59.203 | INFO | PerfReport: 10.93%, Baseline -- 3.29e+00 / -nan, 1.86e-02 / 0.00e+00 | 20260506 10:11:59.203 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.29e-02 / 0.00e+00 | 20260506 10:11:59.203 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.12e+00 / 0.00e+00 | 20260506 10:11:59.203 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.72e+01 / -nan, 4.22e-02 / 0.00e+00 | 20260506 10:11:59.203 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.24e+00 / -nan, 1.30e-01 / 0.00e+00 | 20260506 10:11:59.203 | INFO | PerfReport: 3.72%, Metrics -- 1.12e+00 / -nan, 1.74e-01 / 0.00e+00 | 20260506 10:11:59.203 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.33e-01 / -nan, 6.19e-02 / 0.00e+00 | 20260506 10:11:59.203 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:11:59.203 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:11:59.203 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:11:59.203 | INFO | PerfReport: Full Execution -- 9.00e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 10:11:59.203 | INFO | PerfReport: 18.67%, Upload -- 1.68e+01 / 0.00e+00, 4.71e-01 / 0.00e+00 | 20260506 10:11:59.203 | INFO | PerfReport: 17.93%, Download -- 1.61e+01 / 0.00e+00, 8.19e-01 / 0.00e+00 | 20260506 10:11:59.203 | INFO | PerfReport: 33.48%, Compute -- 3.01e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 10:11:59.203 | INFO | PerfReport: 0.57%, LockUpload -- 5.09e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:11:59.203 | INFO | PerfReport: 0.00%, LockDownload -- 4.97e-04 / 0.00e+00, 1.71e-04 / 0.00e+00 | 20260506 10:11:59.203 | INFO | PerfReport: 29.33%, LockCompute -- 2.64e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 10:11:59.204 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 5.15e-02 / 0.00e+00 | 20260506 10:11:59.299 | INFO | Starting a round of Baz IO | 20260506 10:12:02.215 | INFO | Populating RTMetrics report | 20260506 10:12:02.216 | INFO | RTMetrics report populated | 20260506 10:12:02.216 | INFO | Opening tmp RTMetrics json file | 20260506 10:12:02.220 | INFO | Serializing RTMetrics to json | 20260506 10:12:02.225 | INFO | Serialization done, swapping tmp file now... | 20260506 10:12:02.227 | INFO | RTMetrics done | 20260506 10:12:02.516 | INFO | Finished a round of Baz IO: Wrote 1.957 GiB in 3.217 seconds | 20260506 10:12:02.516 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.13% / 0.06% / 50.81% | 20260506 10:12:02.516 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.39052% / 0.000123183% / 3.57628e-05% | 20260506 10:12:02.516 | INFO | Baz IO overall: | 20260506 10:12:02.516 | INFO | BytesWritten : 237024382661 | 20260506 10:12:02.516 | INFO | NumEvents : 113667223905 | 20260506 10:12:04.341 | INFO | Analyzing chunk frames = [540672,541184) | 20260506 10:12:04.341 | INFO | MemInfo: { | 20260506 10:12:04.341 | INFO | "gpu" : " 17 GB", | 20260506 10:12:04.341 | INFO | "huge" : "", | 20260506 10:12:04.341 | INFO | "huge_shared" : " 27 GB", | 20260506 10:12:04.341 | INFO | "malloc" : " 14 GB", | 20260506 10:12:04.341 | INFO | "pinned" : " 29 GB" | 20260506 10:12:04.341 | INFO | } | 20260506 10:12:04.341 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:12:04.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:12:04.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:04.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:04.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:04.341 | INFO | ANALYSIS : 57.6%, 2.8, 85.4ms, 0 | 20260506 10:12:04.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:04.341 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 10:12:04.341 | INFO | PRE_HQ : 56.7%, 2.1, 60.7ms, 0 | 20260506 10:12:04.341 | INFO | BAZWRITER : 62.2%, 4.8, 152.5ms, 0 | 20260506 10:12:04.341 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 10:12:04.342 | INFO | Uploaded 36GiB to GPU in 1.67673s (21.4703GiB/s) | 20260506 10:12:04.342 | INFO | Downloaded 38GiB to GPU in 1.74149s (21.8203GiB/s) | 20260506 10:12:09.481 | INFO | Analyzing chunk frames = [541184,541696) | 20260506 10:12:09.481 | INFO | MemInfo: { | 20260506 10:12:09.481 | INFO | "gpu" : " 17 GB", | 20260506 10:12:09.481 | INFO | "huge" : "", | 20260506 10:12:09.481 | INFO | "huge_shared" : " 27 GB", | 20260506 10:12:09.481 | INFO | "malloc" : " 14 GB", | 20260506 10:12:09.481 | INFO | "pinned" : " 29 GB" | 20260506 10:12:09.481 | INFO | } | 20260506 10:12:09.481 | INFO | Perf report for the last 5.14014 seconds: | 20260506 10:12:09.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:12:09.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:09.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:09.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:09.481 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 10:12:09.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:09.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:09.481 | INFO | PRE_HQ : 29.5%, 1.0, 15.6ms, 0 | 20260506 10:12:09.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:09.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:09.481 | INFO | Uploaded 36GiB to GPU in 1.60283s (22.4603GiB/s) | 20260506 10:12:09.481 | INFO | Downloaded 34GiB to GPU in 1.52038s (22.3629GiB/s) | 20260506 10:12:14.620 | INFO | Analyzing chunk frames = [541696,542208) | 20260506 10:12:14.620 | INFO | MemInfo: { | 20260506 10:12:14.620 | INFO | "gpu" : " 17 GB", | 20260506 10:12:14.620 | INFO | "huge" : "", | 20260506 10:12:14.620 | INFO | "huge_shared" : " 27 GB", | 20260506 10:12:14.620 | INFO | "malloc" : " 14 GB", | 20260506 10:12:14.620 | INFO | "pinned" : " 29 GB" | 20260506 10:12:14.620 | INFO | } | 20260506 10:12:14.620 | INFO | Perf report for the last 5.13919 seconds: | 20260506 10:12:14.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:12:14.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:14.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:14.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:14.620 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260506 10:12:14.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:14.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:14.620 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:12:14.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:14.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:14.621 | INFO | Uploaded 36GiB to GPU in 1.60684s (22.4043GiB/s) | 20260506 10:12:14.621 | INFO | Downloaded 34GiB to GPU in 1.52445s (22.3031GiB/s) | 20260506 10:12:19.759 | INFO | Analyzing chunk frames = [542208,542720) | 20260506 10:12:19.759 | INFO | MemInfo: { | 20260506 10:12:19.759 | INFO | "gpu" : " 17 GB", | 20260506 10:12:19.759 | INFO | "huge" : "", | 20260506 10:12:19.759 | INFO | "huge_shared" : " 27 GB", | 20260506 10:12:19.759 | INFO | "malloc" : " 14 GB", | 20260506 10:12:19.759 | INFO | "pinned" : " 29 GB" | 20260506 10:12:19.759 | INFO | } | 20260506 10:12:19.759 | INFO | Perf report for the last 5.13864 seconds: | 20260506 10:12:19.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:12:19.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:19.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:19.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:19.760 | INFO | ANALYSIS : 56.5%, 3.0, 87.5ms, 0 | 20260506 10:12:19.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:19.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:19.760 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:12:19.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:19.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:19.760 | INFO | Uploaded 36GiB to GPU in 1.60714s (22.4001GiB/s) | 20260506 10:12:19.760 | INFO | Downloaded 34GiB to GPU in 1.52456s (22.3015GiB/s) | 20260506 10:12:24.898 | INFO | Analyzing chunk frames = [542720,543232) | 20260506 10:12:24.898 | INFO | MemInfo: { | 20260506 10:12:24.898 | INFO | "gpu" : " 17 GB", | 20260506 10:12:24.898 | INFO | "huge" : "", | 20260506 10:12:24.898 | INFO | "huge_shared" : " 27 GB", | 20260506 10:12:24.898 | INFO | "malloc" : " 14 GB", | 20260506 10:12:24.898 | INFO | "pinned" : " 29 GB" | 20260506 10:12:24.898 | INFO | } | 20260506 10:12:24.899 | INFO | Perf report for the last 5.13949 seconds: | 20260506 10:12:24.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:12:24.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:24.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:24.899 | INFO | SAVE_TRACE : 0.2%, 1.0, 0.1ms, 0 | 20260506 10:12:24.899 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:12:24.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:24.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:24.899 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 10:12:24.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:24.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:24.899 | INFO | Uploaded 36GiB to GPU in 1.60489s (22.4315GiB/s) | 20260506 10:12:24.899 | INFO | Downloaded 34GiB to GPU in 1.52164s (22.3443GiB/s) | 20260506 10:12:30.037 | INFO | Analyzing chunk frames = [543232,543744) | 20260506 10:12:30.037 | INFO | MemInfo: { | 20260506 10:12:30.037 | INFO | "gpu" : " 17 GB", | 20260506 10:12:30.037 | INFO | "huge" : "", | 20260506 10:12:30.037 | INFO | "huge_shared" : " 27 GB", | 20260506 10:12:30.037 | INFO | "malloc" : " 14 GB", | 20260506 10:12:30.037 | INFO | "pinned" : " 29 GB" | 20260506 10:12:30.037 | INFO | } | 20260506 10:12:30.038 | INFO | Perf report for the last 5.13901 seconds: | 20260506 10:12:30.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:12:30.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:30.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:30.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:30.038 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:12:30.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:30.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:30.038 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:12:30.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:30.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:30.038 | INFO | Uploaded 36GiB to GPU in 1.60489s (22.4314GiB/s) | 20260506 10:12:30.038 | INFO | Downloaded 34GiB to GPU in 1.52194s (22.34GiB/s) | 20260506 10:12:35.177 | INFO | Analyzing chunk frames = [543744,544256) | 20260506 10:12:35.177 | INFO | MemInfo: { | 20260506 10:12:35.177 | INFO | "gpu" : " 17 GB", | 20260506 10:12:35.177 | INFO | "huge" : "", | 20260506 10:12:35.177 | INFO | "huge_shared" : " 27 GB", | 20260506 10:12:35.177 | INFO | "malloc" : " 14 GB", | 20260506 10:12:35.177 | INFO | "pinned" : " 29 GB" | 20260506 10:12:35.177 | INFO | } | 20260506 10:12:35.177 | INFO | Perf report for the last 5.13945 seconds: | 20260506 10:12:35.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:12:35.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:35.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:35.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:35.177 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:12:35.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:35.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:35.177 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:12:35.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:35.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:35.177 | INFO | Uploaded 36GiB to GPU in 1.60499s (22.43GiB/s) | 20260506 10:12:35.177 | INFO | Downloaded 34GiB to GPU in 1.52301s (22.3242GiB/s) | 20260506 10:12:40.316 | INFO | Analyzing chunk frames = [544256,544768) | 20260506 10:12:40.316 | INFO | MemInfo: { | 20260506 10:12:40.316 | INFO | "gpu" : " 17 GB", | 20260506 10:12:40.316 | INFO | "huge" : "", | 20260506 10:12:40.316 | INFO | "huge_shared" : " 27 GB", | 20260506 10:12:40.316 | INFO | "malloc" : " 14 GB", | 20260506 10:12:40.316 | INFO | "pinned" : " 29 GB" | 20260506 10:12:40.316 | INFO | } | 20260506 10:12:40.316 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:12:40.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:12:40.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:40.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:40.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:40.316 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:12:40.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:40.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:40.317 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:12:40.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:40.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:40.317 | INFO | Uploaded 36GiB to GPU in 1.60582s (22.4185GiB/s) | 20260506 10:12:40.317 | INFO | Downloaded 34GiB to GPU in 1.52319s (22.3215GiB/s) | 20260506 10:12:40.416 | INFO | Starting a round of Baz IO | 20260506 10:12:43.330 | INFO | Populating RTMetrics report | 20260506 10:12:43.330 | INFO | RTMetrics report populated | 20260506 10:12:43.330 | INFO | Opening tmp RTMetrics json file | 20260506 10:12:43.333 | INFO | Serializing RTMetrics to json | 20260506 10:12:43.338 | INFO | Serialization done, swapping tmp file now... | 20260506 10:12:43.340 | INFO | RTMetrics done | 20260506 10:12:43.926 | INFO | Finished a round of Baz IO: Wrote 1.958 GiB in 3.509 seconds | 20260506 10:12:43.926 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.07% / 0.06% / 50.87% | 20260506 10:12:43.926 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.36699% / 0.000198682% / 5.96046e-05% | 20260506 10:12:43.926 | INFO | Baz IO overall: | 20260506 10:12:43.926 | INFO | BytesWritten : 239127263550 | 20260506 10:12:43.926 | INFO | NumEvents : 114687723371 | 20260506 10:12:45.455 | INFO | Analyzing chunk frames = [544768,545280) | 20260506 10:12:45.455 | INFO | MemInfo: { | 20260506 10:12:45.455 | INFO | "gpu" : " 17 GB", | 20260506 10:12:45.455 | INFO | "huge" : "", | 20260506 10:12:45.455 | INFO | "huge_shared" : " 27 GB", | 20260506 10:12:45.456 | INFO | "malloc" : " 14 GB", | 20260506 10:12:45.456 | INFO | "pinned" : " 29 GB" | 20260506 10:12:45.456 | INFO | } | 20260506 10:12:45.456 | INFO | Perf report for the last 5.1392 seconds: | 20260506 10:12:45.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:12:45.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:45.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:45.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:45.456 | INFO | ANALYSIS : 57.6%, 2.8, 85.7ms, 0 | 20260506 10:12:45.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:45.456 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 10:12:45.456 | INFO | PRE_HQ : 56.8%, 2.1, 61.0ms, 0 | 20260506 10:12:45.456 | INFO | BAZWRITER : 67.9%, 4.8, 166.5ms, 0 | 20260506 10:12:45.456 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 10:12:45.456 | INFO | Uploaded 36GiB to GPU in 1.66661s (21.6007GiB/s) | 20260506 10:12:45.456 | INFO | Downloaded 38GiB to GPU in 1.73101s (21.9525GiB/s) | 20260506 10:12:50.595 | INFO | Analyzing chunk frames = [545280,545792) | 20260506 10:12:50.595 | INFO | MemInfo: { | 20260506 10:12:50.595 | INFO | "gpu" : " 17 GB", | 20260506 10:12:50.595 | INFO | "huge" : "", | 20260506 10:12:50.595 | INFO | "huge_shared" : " 27 GB", | 20260506 10:12:50.595 | INFO | "malloc" : " 14 GB", | 20260506 10:12:50.595 | INFO | "pinned" : " 29 GB" | 20260506 10:12:50.595 | INFO | } | 20260506 10:12:50.595 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:12:50.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:12:50.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:50.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:50.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:50.595 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:12:50.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:50.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:50.595 | INFO | PRE_HQ : 29.9%, 1.0, 16.0ms, 0 | 20260506 10:12:50.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:50.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:50.595 | INFO | Uploaded 36GiB to GPU in 1.60266s (22.4627GiB/s) | 20260506 10:12:50.595 | INFO | Downloaded 34GiB to GPU in 1.51948s (22.3761GiB/s) | 20260506 10:12:55.734 | INFO | Analyzing chunk frames = [545792,546304) | 20260506 10:12:55.734 | INFO | MemInfo: { | 20260506 10:12:55.734 | INFO | "gpu" : " 17 GB", | 20260506 10:12:55.734 | INFO | "huge" : "", | 20260506 10:12:55.734 | INFO | "huge_shared" : " 27 GB", | 20260506 10:12:55.734 | INFO | "malloc" : " 14 GB", | 20260506 10:12:55.734 | INFO | "pinned" : " 29 GB" | 20260506 10:12:55.734 | INFO | } | 20260506 10:12:55.734 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:12:55.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:12:55.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:55.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:55.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:55.734 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260506 10:12:55.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:55.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:55.734 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:12:55.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:55.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:12:55.735 | INFO | Uploaded 36GiB to GPU in 1.60698s (22.4023GiB/s) | 20260506 10:12:55.735 | INFO | Downloaded 34GiB to GPU in 1.52533s (22.2903GiB/s) | 20260506 10:13:00.873 | INFO | Analyzing chunk frames = [546304,546816) | 20260506 10:13:00.874 | INFO | MemInfo: { | 20260506 10:13:00.874 | INFO | "gpu" : " 17 GB", | 20260506 10:13:00.874 | INFO | "huge" : "", | 20260506 10:13:00.874 | INFO | "huge_shared" : " 27 GB", | 20260506 10:13:00.874 | INFO | "malloc" : " 14 GB", | 20260506 10:13:00.874 | INFO | "pinned" : " 29 GB" | 20260506 10:13:00.874 | INFO | } | 20260506 10:13:00.874 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:13:00.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:13:00.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:00.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:00.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:00.874 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:13:00.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:00.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:00.874 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:13:00.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:00.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:00.874 | INFO | Uploaded 36GiB to GPU in 1.60625s (22.4125GiB/s) | 20260506 10:13:00.874 | INFO | Downloaded 34GiB to GPU in 1.52365s (22.3148GiB/s) | 20260506 10:13:06.013 | INFO | Analyzing chunk frames = [546816,547328) | 20260506 10:13:06.013 | INFO | MemInfo: { | 20260506 10:13:06.013 | INFO | "gpu" : " 17 GB", | 20260506 10:13:06.013 | INFO | "huge" : "", | 20260506 10:13:06.013 | INFO | "huge_shared" : " 27 GB", | 20260506 10:13:06.013 | INFO | "malloc" : " 14 GB", | 20260506 10:13:06.013 | INFO | "pinned" : " 29 GB" | 20260506 10:13:06.013 | INFO | } | 20260506 10:13:06.013 | INFO | Perf report for the last 5.13949 seconds: | 20260506 10:13:06.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:13:06.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:06.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:06.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:06.013 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:13:06.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:06.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:06.013 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:13:06.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:06.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:06.014 | INFO | Uploaded 36GiB to GPU in 1.60445s (22.4376GiB/s) | 20260506 10:13:06.014 | INFO | Downloaded 34GiB to GPU in 1.52105s (22.353GiB/s) | 20260506 10:13:11.152 | INFO | Analyzing chunk frames = [547328,547840) | 20260506 10:13:11.152 | INFO | MemInfo: { | 20260506 10:13:11.152 | INFO | "gpu" : " 17 GB", | 20260506 10:13:11.152 | INFO | "huge" : "", | 20260506 10:13:11.152 | INFO | "huge_shared" : " 27 GB", | 20260506 10:13:11.152 | INFO | "malloc" : " 14 GB", | 20260506 10:13:11.152 | INFO | "pinned" : " 29 GB" | 20260506 10:13:11.152 | INFO | } | 20260506 10:13:11.152 | INFO | Perf report for the last 5.13923 seconds: | 20260506 10:13:11.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:13:11.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:11.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:11.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:11.153 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:13:11.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:11.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:11.153 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:13:11.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:11.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:11.153 | INFO | Uploaded 36GiB to GPU in 1.60503s (22.4294GiB/s) | 20260506 10:13:11.153 | INFO | Downloaded 34GiB to GPU in 1.52232s (22.3343GiB/s) | 20260506 10:13:16.291 | INFO | Analyzing chunk frames = [547840,548352) | 20260506 10:13:16.291 | INFO | MemInfo: { | 20260506 10:13:16.291 | INFO | "gpu" : " 17 GB", | 20260506 10:13:16.291 | INFO | "huge" : "", | 20260506 10:13:16.291 | INFO | "huge_shared" : " 27 GB", | 20260506 10:13:16.291 | INFO | "malloc" : " 14 GB", | 20260506 10:13:16.291 | INFO | "pinned" : " 29 GB" | 20260506 10:13:16.291 | INFO | } | 20260506 10:13:16.292 | INFO | Perf report for the last 5.13925 seconds: | 20260506 10:13:16.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:13:16.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:16.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:16.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:16.292 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:13:16.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:16.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:16.292 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:13:16.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:16.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:16.292 | INFO | Uploaded 36GiB to GPU in 1.6057s (22.4201GiB/s) | 20260506 10:13:16.292 | INFO | Downloaded 34GiB to GPU in 1.52326s (22.3206GiB/s) | 20260506 10:13:21.431 | INFO | Analyzing chunk frames = [548352,548864) | 20260506 10:13:21.431 | INFO | MemInfo: { | 20260506 10:13:21.431 | INFO | "gpu" : " 17 GB", | 20260506 10:13:21.431 | INFO | "huge" : "", | 20260506 10:13:21.431 | INFO | "huge_shared" : " 27 GB", | 20260506 10:13:21.431 | INFO | "malloc" : " 14 GB", | 20260506 10:13:21.431 | INFO | "pinned" : " 29 GB" | 20260506 10:13:21.431 | INFO | } | 20260506 10:13:21.431 | INFO | Perf report for the last 5.13946 seconds: | 20260506 10:13:21.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:13:21.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:21.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:21.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:21.431 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260506 10:13:21.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:21.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:21.431 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:13:21.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:21.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:21.432 | INFO | Uploaded 36GiB to GPU in 1.60588s (22.4177GiB/s) | 20260506 10:13:21.432 | INFO | Downloaded 34GiB to GPU in 1.52333s (22.3195GiB/s) | 20260506 10:13:21.432 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:13:21.432 | INFO | Total mem usage: 16.5 GiB | 20260506 10:13:21.432 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:13:21.432 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -2.73%) | 20260506 10:13:21.432 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:13:21.432 | INFO | High Water Breakdown: | 20260506 10:13:21.432 | INFO | 60.8% : 9.77 GiB ( 96.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:13:21.432 | INFO | 39.1% : 6.29 GiB ( 93.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:13:21.432 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:13:21.432 | INFO | ------------------------------------------ | 20260506 10:13:21.432 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:13:21.432 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:13:21.432 | INFO | Total mem usage: 26.5 GiB | 20260506 10:13:21.432 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:13:21.432 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:13:21.432 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:13:21.432 | INFO | High Water Breakdown: | 20260506 10:13:21.432 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:13:21.432 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:13:21.432 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:13:21.432 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:13:21.432 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:13:21.432 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:13:21.432 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:13:21.432 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:13:21.432 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:13:21.432 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:13:21.432 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:13:21.432 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:13:21.432 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:13:21.432 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:13:21.432 | INFO | ------------------------------------------ | 20260506 10:13:21.432 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:13:21.432 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:13:21.432 | INFO | Total mem usage: 25.6 GiB | 20260506 10:13:21.432 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:13:21.432 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:13:21.432 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:13:21.432 | INFO | High Water Breakdown: | 20260506 10:13:21.432 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:13:21.432 | INFO | ------------------------------------------ | 20260506 10:13:21.432 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:13:21.432 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:13:21.432 | INFO | Total mem usage: 15.7 GiB | 20260506 10:13:21.433 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:13:21.433 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:13:21.433 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:13:21.433 | INFO | High Water Breakdown: | 20260506 10:13:21.433 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:13:21.433 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:13:21.433 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:13:21.433 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:13:21.433 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:13:21.433 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:13:21.433 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:13:21.433 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:13:21.433 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:13:21.433 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:13:21.433 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:13:21.433 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:13:21.433 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:13:21.433 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:13:21.433 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:13:21.433 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:13:21.433 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:13:21.433 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:13:21.433 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:13:21.433 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:13:21.433 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:13:21.433 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:13:21.433 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:13:21.433 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:13:21.433 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:13:21.433 | INFO | ------------------------------------------ | 20260506 10:13:21.433 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:13:21.433 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:13:21.433 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:13:21.433 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:13:21.433 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 10:13:21.433 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.22e-03 / 0.00e+00 | 20260506 10:13:21.433 | INFO | PerfReport: 10.94%, Baseline -- 3.29e+00 / -nan, 1.84e-02 / 0.00e+00 | 20260506 10:13:21.433 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.27e-02 / 0.00e+00 | 20260506 10:13:21.433 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.00e+00 / 0.00e+00 | 20260506 10:13:21.433 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.72e+01 / -nan, 4.23e-02 / 0.00e+00 | 20260506 10:13:21.433 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.24e+00 / -nan, 1.30e-01 / 0.00e+00 | 20260506 10:13:21.433 | INFO | PerfReport: 3.71%, Metrics -- 1.12e+00 / -nan, 1.73e-01 / 0.00e+00 | 20260506 10:13:21.433 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.33e-01 / -nan, 6.19e-02 / 0.00e+00 | 20260506 10:13:21.433 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:13:21.433 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:13:21.433 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:13:21.433 | INFO | PerfReport: Full Execution -- 9.00e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 10:13:21.433 | INFO | PerfReport: 18.69%, Upload -- 1.68e+01 / 0.00e+00, 4.79e-01 / 0.00e+00 | 20260506 10:13:21.433 | INFO | PerfReport: 17.94%, Download -- 1.61e+01 / 0.00e+00, 8.29e-01 / 0.00e+00 | 20260506 10:13:21.433 | INFO | PerfReport: 33.49%, Compute -- 3.01e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 10:13:21.433 | INFO | PerfReport: 0.57%, LockUpload -- 5.09e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 10:13:21.433 | INFO | PerfReport: 0.00%, LockDownload -- 4.78e-04 / 0.00e+00, 1.21e-04 / 0.00e+00 | 20260506 10:13:21.433 | INFO | PerfReport: 29.30%, LockCompute -- 2.64e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 10:13:21.433 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.19e-02 / 0.00e+00 | 20260506 10:13:21.528 | INFO | Starting a round of Baz IO | 20260506 10:13:24.444 | INFO | Populating RTMetrics report | 20260506 10:13:24.445 | INFO | RTMetrics report populated | 20260506 10:13:24.445 | INFO | Opening tmp RTMetrics json file | 20260506 10:13:24.535 | INFO | Serializing RTMetrics to json | 20260506 10:13:24.539 | INFO | Serialization done, swapping tmp file now... | 20260506 10:13:24.542 | INFO | RTMetrics done | 20260506 10:13:24.781 | INFO | Finished a round of Baz IO: Wrote 1.963 GiB in 3.252 seconds | 20260506 10:13:24.781 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 49.01% / 0.06% / 50.93% | 20260506 10:13:24.781 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.36171% / 0.000123183% / 3.97364e-05% | 20260506 10:13:24.781 | INFO | Baz IO overall: | 20260506 10:13:24.781 | INFO | BytesWritten : 241234602289 | 20260506 10:13:24.781 | INFO | NumEvents : 115710746435 | 20260506 10:13:26.570 | INFO | Analyzing chunk frames = [548864,549376) | 20260506 10:13:26.570 | INFO | MemInfo: { | 20260506 10:13:26.570 | INFO | "gpu" : " 17 GB", | 20260506 10:13:26.570 | INFO | "huge" : "", | 20260506 10:13:26.570 | INFO | "huge_shared" : " 27 GB", | 20260506 10:13:26.570 | INFO | "malloc" : " 14 GB", | 20260506 10:13:26.570 | INFO | "pinned" : " 29 GB" | 20260506 10:13:26.570 | INFO | } | 20260506 10:13:26.570 | INFO | Perf report for the last 5.1392 seconds: | 20260506 10:13:26.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:13:26.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:26.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:26.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:26.571 | INFO | ANALYSIS : 57.6%, 2.9, 88.1ms, 0 | 20260506 10:13:26.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:26.571 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 10:13:26.571 | INFO | PRE_HQ : 56.7%, 2.1, 61.1ms, 0 | 20260506 10:13:26.571 | INFO | BAZWRITER : 62.9%, 4.4, 141.8ms, 0 | 20260506 10:13:26.571 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 10:13:26.571 | INFO | Uploaded 36GiB to GPU in 1.65975s (21.69GiB/s) | 20260506 10:13:26.571 | INFO | Downloaded 38GiB to GPU in 1.72345s (22.0488GiB/s) | 20260506 10:13:31.710 | INFO | Analyzing chunk frames = [549376,549888) | 20260506 10:13:31.710 | INFO | MemInfo: { | 20260506 10:13:31.710 | INFO | "gpu" : " 17 GB", | 20260506 10:13:31.710 | INFO | "huge" : "", | 20260506 10:13:31.710 | INFO | "huge_shared" : " 27 GB", | 20260506 10:13:31.710 | INFO | "malloc" : " 14 GB", | 20260506 10:13:31.710 | INFO | "pinned" : " 29 GB" | 20260506 10:13:31.710 | INFO | } | 20260506 10:13:31.710 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:13:31.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:13:31.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:31.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:31.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:31.710 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:13:31.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:31.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:31.710 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:13:31.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:31.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:31.710 | INFO | Uploaded 36GiB to GPU in 1.60286s (22.4598GiB/s) | 20260506 10:13:31.710 | INFO | Downloaded 34GiB to GPU in 1.52042s (22.3622GiB/s) | 20260506 10:13:36.849 | INFO | Analyzing chunk frames = [549888,550400) | 20260506 10:13:36.849 | INFO | MemInfo: { | 20260506 10:13:36.849 | INFO | "gpu" : " 17 GB", | 20260506 10:13:36.849 | INFO | "huge" : "", | 20260506 10:13:36.849 | INFO | "huge_shared" : " 27 GB", | 20260506 10:13:36.849 | INFO | "malloc" : " 14 GB", | 20260506 10:13:36.849 | INFO | "pinned" : " 29 GB" | 20260506 10:13:36.849 | INFO | } | 20260506 10:13:36.849 | INFO | Perf report for the last 5.13939 seconds: | 20260506 10:13:36.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:13:36.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:36.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:36.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:36.849 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260506 10:13:36.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:36.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:36.849 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:13:36.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:36.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:36.850 | INFO | Uploaded 36GiB to GPU in 1.60616s (22.4137GiB/s) | 20260506 10:13:36.850 | INFO | Downloaded 34GiB to GPU in 1.52431s (22.3051GiB/s) | 20260506 10:13:41.988 | INFO | Analyzing chunk frames = [550400,550912) | 20260506 10:13:41.988 | INFO | MemInfo: { | 20260506 10:13:41.988 | INFO | "gpu" : " 17 GB", | 20260506 10:13:41.988 | INFO | "huge" : "", | 20260506 10:13:41.988 | INFO | "huge_shared" : " 27 GB", | 20260506 10:13:41.988 | INFO | "malloc" : " 14 GB", | 20260506 10:13:41.988 | INFO | "pinned" : " 29 GB" | 20260506 10:13:41.988 | INFO | } | 20260506 10:13:41.988 | INFO | Perf report for the last 5.13925 seconds: | 20260506 10:13:41.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:13:41.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:41.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:41.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:41.988 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:13:41.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:41.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:41.989 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:13:41.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:41.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:41.989 | INFO | Uploaded 36GiB to GPU in 1.60662s (22.4073GiB/s) | 20260506 10:13:41.989 | INFO | Downloaded 34GiB to GPU in 1.52447s (22.3029GiB/s) | 20260506 10:13:47.128 | INFO | Analyzing chunk frames = [550912,551424) | 20260506 10:13:47.128 | INFO | MemInfo: { | 20260506 10:13:47.128 | INFO | "gpu" : " 17 GB", | 20260506 10:13:47.128 | INFO | "huge" : "", | 20260506 10:13:47.128 | INFO | "huge_shared" : " 27 GB", | 20260506 10:13:47.128 | INFO | "malloc" : " 14 GB", | 20260506 10:13:47.128 | INFO | "pinned" : " 29 GB" | 20260506 10:13:47.128 | INFO | } | 20260506 10:13:47.128 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:13:47.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:13:47.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:47.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:47.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:47.128 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:13:47.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:47.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:47.128 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:13:47.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:47.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:47.128 | INFO | Uploaded 36GiB to GPU in 1.60525s (22.4264GiB/s) | 20260506 10:13:47.129 | INFO | Downloaded 34GiB to GPU in 1.52177s (22.3424GiB/s) | 20260506 10:13:52.267 | INFO | Analyzing chunk frames = [551424,551936) | 20260506 10:13:52.267 | INFO | MemInfo: { | 20260506 10:13:52.267 | INFO | "gpu" : " 17 GB", | 20260506 10:13:52.267 | INFO | "huge" : "", | 20260506 10:13:52.267 | INFO | "huge_shared" : " 27 GB", | 20260506 10:13:52.267 | INFO | "malloc" : " 14 GB", | 20260506 10:13:52.267 | INFO | "pinned" : " 29 GB" | 20260506 10:13:52.267 | INFO | } | 20260506 10:13:52.267 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:13:52.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:13:52.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:52.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:52.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:52.267 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:13:52.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:52.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:52.267 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:13:52.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:52.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:52.268 | INFO | Uploaded 36GiB to GPU in 1.60516s (22.4277GiB/s) | 20260506 10:13:52.268 | INFO | Downloaded 34GiB to GPU in 1.52213s (22.3371GiB/s) | 20260506 10:13:57.406 | INFO | Analyzing chunk frames = [551936,552448) | 20260506 10:13:57.406 | INFO | MemInfo: { | 20260506 10:13:57.406 | INFO | "gpu" : " 17 GB", | 20260506 10:13:57.406 | INFO | "huge" : "", | 20260506 10:13:57.406 | INFO | "huge_shared" : " 27 GB", | 20260506 10:13:57.406 | INFO | "malloc" : " 14 GB", | 20260506 10:13:57.406 | INFO | "pinned" : " 29 GB" | 20260506 10:13:57.406 | INFO | } | 20260506 10:13:57.406 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:13:57.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:13:57.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:57.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:57.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:57.407 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:13:57.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:57.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:57.407 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:13:57.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:57.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:13:57.407 | INFO | Uploaded 36GiB to GPU in 1.60483s (22.4323GiB/s) | 20260506 10:13:57.407 | INFO | Downloaded 34GiB to GPU in 1.5229s (22.3259GiB/s) | 20260506 10:14:02.546 | INFO | Analyzing chunk frames = [552448,552960) | 20260506 10:14:02.546 | INFO | MemInfo: { | 20260506 10:14:02.546 | INFO | "gpu" : " 17 GB", | 20260506 10:14:02.546 | INFO | "huge" : "", | 20260506 10:14:02.546 | INFO | "huge_shared" : " 27 GB", | 20260506 10:14:02.546 | INFO | "malloc" : " 14 GB", | 20260506 10:14:02.546 | INFO | "pinned" : " 29 GB" | 20260506 10:14:02.546 | INFO | } | 20260506 10:14:02.546 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:14:02.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:14:02.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:02.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:02.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:02.546 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:14:02.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:02.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:02.546 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 10:14:02.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:02.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:02.547 | INFO | Uploaded 36GiB to GPU in 1.60476s (22.4332GiB/s) | 20260506 10:14:02.547 | INFO | Downloaded 34GiB to GPU in 1.52312s (22.3226GiB/s) | 20260506 10:14:02.643 | INFO | Starting a round of Baz IO | 20260506 10:14:05.559 | INFO | Populating RTMetrics report | 20260506 10:14:05.559 | INFO | RTMetrics report populated | 20260506 10:14:05.559 | INFO | Opening tmp RTMetrics json file | 20260506 10:14:05.572 | INFO | Serializing RTMetrics to json | 20260506 10:14:05.594 | INFO | Serialization done, swapping tmp file now... | 20260506 10:14:05.631 | INFO | RTMetrics done | 20260506 10:14:05.713 | INFO | Finished a round of Baz IO: Wrote 1.961 GiB in 3.069 seconds | 20260506 10:14:05.713 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.95% / 0.06% / 50.99% | 20260506 10:14:05.713 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.31238% / 0.000123183% / 6.35783e-05% | 20260506 10:14:05.713 | INFO | Baz IO overall: | 20260506 10:14:05.713 | INFO | BytesWritten : 243340471869 | 20260506 10:14:05.713 | INFO | NumEvents : 116732928449 | 20260506 10:14:07.685 | INFO | Analyzing chunk frames = [552960,553472) | 20260506 10:14:07.685 | INFO | MemInfo: { | 20260506 10:14:07.685 | INFO | "gpu" : " 17 GB", | 20260506 10:14:07.685 | INFO | "huge" : "", | 20260506 10:14:07.685 | INFO | "huge_shared" : " 27 GB", | 20260506 10:14:07.685 | INFO | "malloc" : " 14 GB", | 20260506 10:14:07.685 | INFO | "pinned" : " 29 GB" | 20260506 10:14:07.685 | INFO | } | 20260506 10:14:07.685 | INFO | Perf report for the last 5.13939 seconds: | 20260506 10:14:07.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:14:07.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:07.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:07.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:07.685 | INFO | ANALYSIS : 57.6%, 2.9, 87.0ms, 0 | 20260506 10:14:07.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:07.685 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 10:14:07.685 | INFO | PRE_HQ : 56.7%, 2.1, 61.5ms, 0 | 20260506 10:14:07.685 | INFO | BAZWRITER : 59.3%, 4.6, 141.4ms, 0 | 20260506 10:14:07.685 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 10:14:07.685 | INFO | Uploaded 36GiB to GPU in 1.65592s (21.7402GiB/s) | 20260506 10:14:07.685 | INFO | Downloaded 38GiB to GPU in 1.72093s (22.0811GiB/s) | 20260506 10:14:12.824 | INFO | Analyzing chunk frames = [553472,553984) | 20260506 10:14:12.824 | INFO | MemInfo: { | 20260506 10:14:12.824 | INFO | "gpu" : " 17 GB", | 20260506 10:14:12.824 | INFO | "huge" : "", | 20260506 10:14:12.824 | INFO | "huge_shared" : " 27 GB", | 20260506 10:14:12.824 | INFO | "malloc" : " 14 GB", | 20260506 10:14:12.824 | INFO | "pinned" : " 29 GB" | 20260506 10:14:12.824 | INFO | } | 20260506 10:14:12.824 | INFO | Perf report for the last 5.13912 seconds: | 20260506 10:14:12.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:14:12.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:12.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:12.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:12.825 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 10:14:12.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:12.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:12.825 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:14:12.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:12.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:12.825 | INFO | Uploaded 36GiB to GPU in 1.60234s (22.4671GiB/s) | 20260506 10:14:12.825 | INFO | Downloaded 34GiB to GPU in 1.51997s (22.3689GiB/s) | 20260506 10:14:17.964 | INFO | Analyzing chunk frames = [553984,554496) | 20260506 10:14:17.964 | INFO | MemInfo: { | 20260506 10:14:17.964 | INFO | "gpu" : " 17 GB", | 20260506 10:14:17.964 | INFO | "huge" : "", | 20260506 10:14:17.964 | INFO | "huge_shared" : " 27 GB", | 20260506 10:14:17.964 | INFO | "malloc" : " 14 GB", | 20260506 10:14:17.964 | INFO | "pinned" : " 29 GB" | 20260506 10:14:17.964 | INFO | } | 20260506 10:14:17.964 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:14:17.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:14:17.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:17.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:17.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:17.964 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 10:14:17.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:17.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:17.964 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:14:17.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:17.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:17.964 | INFO | Uploaded 36GiB to GPU in 1.60837s (22.3829GiB/s) | 20260506 10:14:17.965 | INFO | Downloaded 34GiB to GPU in 1.52619s (22.2777GiB/s) | 20260506 10:14:23.103 | INFO | Analyzing chunk frames = [554496,555008) | 20260506 10:14:23.103 | INFO | MemInfo: { | 20260506 10:14:23.103 | INFO | "gpu" : " 17 GB", | 20260506 10:14:23.103 | INFO | "huge" : "", | 20260506 10:14:23.103 | INFO | "huge_shared" : " 27 GB", | 20260506 10:14:23.103 | INFO | "malloc" : " 14 GB", | 20260506 10:14:23.103 | INFO | "pinned" : " 29 GB" | 20260506 10:14:23.103 | INFO | } | 20260506 10:14:23.103 | INFO | Perf report for the last 5.1396 seconds: | 20260506 10:14:23.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:14:23.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:23.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:23.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:23.103 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:14:23.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:23.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:23.103 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:14:23.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:23.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:23.104 | INFO | Uploaded 36GiB to GPU in 1.60555s (22.4222GiB/s) | 20260506 10:14:23.104 | INFO | Downloaded 34GiB to GPU in 1.52409s (22.3084GiB/s) | 20260506 10:14:28.242 | INFO | Analyzing chunk frames = [555008,555520) | 20260506 10:14:28.242 | INFO | MemInfo: { | 20260506 10:14:28.242 | INFO | "gpu" : " 17 GB", | 20260506 10:14:28.242 | INFO | "huge" : "", | 20260506 10:14:28.242 | INFO | "huge_shared" : " 27 GB", | 20260506 10:14:28.242 | INFO | "malloc" : " 14 GB", | 20260506 10:14:28.243 | INFO | "pinned" : " 29 GB" | 20260506 10:14:28.243 | INFO | } | 20260506 10:14:28.243 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:14:28.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:14:28.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:28.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:28.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:28.243 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260506 10:14:28.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:28.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:28.243 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:14:28.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:28.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:28.243 | INFO | Uploaded 36GiB to GPU in 1.60468s (22.4344GiB/s) | 20260506 10:14:28.243 | INFO | Downloaded 34GiB to GPU in 1.52189s (22.3406GiB/s) | 20260506 10:14:33.382 | INFO | Analyzing chunk frames = [555520,556032) | 20260506 10:14:33.382 | INFO | MemInfo: { | 20260506 10:14:33.382 | INFO | "gpu" : " 17 GB", | 20260506 10:14:33.382 | INFO | "huge" : "", | 20260506 10:14:33.382 | INFO | "huge_shared" : " 27 GB", | 20260506 10:14:33.382 | INFO | "malloc" : " 14 GB", | 20260506 10:14:33.382 | INFO | "pinned" : " 29 GB" | 20260506 10:14:33.382 | INFO | } | 20260506 10:14:33.382 | INFO | Perf report for the last 5.13949 seconds: | 20260506 10:14:33.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:14:33.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:33.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:33.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:33.382 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 10:14:33.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:33.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:33.382 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 10:14:33.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:33.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:33.383 | INFO | Uploaded 36GiB to GPU in 1.60564s (22.421GiB/s) | 20260506 10:14:33.383 | INFO | Downloaded 34GiB to GPU in 1.52368s (22.3144GiB/s) | 20260506 10:14:38.521 | INFO | Analyzing chunk frames = [556032,556544) | 20260506 10:14:38.521 | INFO | MemInfo: { | 20260506 10:14:38.521 | INFO | "gpu" : " 17 GB", | 20260506 10:14:38.521 | INFO | "huge" : "", | 20260506 10:14:38.521 | INFO | "huge_shared" : " 27 GB", | 20260506 10:14:38.521 | INFO | "malloc" : " 14 GB", | 20260506 10:14:38.521 | INFO | "pinned" : " 29 GB" | 20260506 10:14:38.521 | INFO | } | 20260506 10:14:38.521 | INFO | Perf report for the last 5.13906 seconds: | 20260506 10:14:38.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:14:38.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:38.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:38.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:38.521 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:14:38.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:38.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:38.521 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:14:38.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:38.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:38.522 | INFO | Uploaded 36GiB to GPU in 1.60595s (22.4166GiB/s) | 20260506 10:14:38.522 | INFO | Downloaded 34GiB to GPU in 1.52345s (22.3178GiB/s) | 20260506 10:14:43.660 | INFO | Analyzing chunk frames = [556544,557056) | 20260506 10:14:43.660 | INFO | MemInfo: { | 20260506 10:14:43.660 | INFO | "gpu" : " 17 GB", | 20260506 10:14:43.660 | INFO | "huge" : "", | 20260506 10:14:43.660 | INFO | "huge_shared" : " 27 GB", | 20260506 10:14:43.660 | INFO | "malloc" : " 14 GB", | 20260506 10:14:43.660 | INFO | "pinned" : " 29 GB" | 20260506 10:14:43.660 | INFO | } | 20260506 10:14:43.660 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:14:43.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:14:43.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:43.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:43.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:43.661 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:14:43.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:43.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:43.661 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:14:43.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:43.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:43.661 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:14:43.661 | INFO | Total mem usage: 16.5 GiB | 20260506 10:14:43.661 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:14:43.661 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -11%) | 20260506 10:14:43.661 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:14:43.661 | INFO | High Water Breakdown: | 20260506 10:14:43.661 | INFO | 59.4% : 8.84 GiB ( 87.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:14:43.661 | INFO | 40.4% : 6.02 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:14:43.661 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:14:43.661 | INFO | ------------------------------------------ | 20260506 10:14:43.661 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:14:43.661 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:14:43.661 | INFO | Total mem usage: 26.5 GiB | 20260506 10:14:43.661 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:14:43.661 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:14:43.661 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:14:43.661 | INFO | High Water Breakdown: | 20260506 10:14:43.661 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:14:43.661 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:14:43.661 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:14:43.661 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:14:43.661 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:14:43.661 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:14:43.661 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:14:43.661 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:14:43.661 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:14:43.661 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:14:43.661 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:14:43.661 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:14:43.661 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:14:43.661 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:14:43.661 | INFO | ------------------------------------------ | 20260506 10:14:43.661 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:14:43.661 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:14:43.661 | INFO | Total mem usage: 25.6 GiB | 20260506 10:14:43.661 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:14:43.661 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:14:43.661 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:14:43.661 | INFO | High Water Breakdown: | 20260506 10:14:43.661 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:14:43.661 | INFO | ------------------------------------------ | 20260506 10:14:43.661 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:14:43.661 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:14:43.661 | INFO | Total mem usage: 15.7 GiB | 20260506 10:14:43.661 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:14:43.661 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:14:43.661 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 10:14:43.661 | INFO | High Water Breakdown: | 20260506 10:14:43.661 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:14:43.661 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:14:43.661 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:14:43.661 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:14:43.661 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:14:43.661 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:14:43.661 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:14:43.661 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:14:43.661 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:14:43.661 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:14:43.661 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:14:43.661 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:14:43.661 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:14:43.661 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:14:43.661 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:14:43.661 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:14:43.661 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:14:43.661 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:14:43.661 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:14:43.661 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:14:43.661 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:14:43.661 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:14:43.661 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:14:43.661 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:14:43.661 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:14:43.661 | INFO | ------------------------------------------ | 20260506 10:14:43.661 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:14:43.661 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:14:43.661 | INFO | Uploaded 36GiB to GPU in 1.60541s (22.4242GiB/s) | 20260506 10:14:43.661 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:14:43.661 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:14:43.661 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.28e+01 / 0.00e+00 | 20260506 10:14:43.661 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 5.02e-03 / 0.00e+00 | 20260506 10:14:43.661 | INFO | Downloaded 34GiB to GPU in 1.52303s (22.3239GiB/s) | 20260506 10:14:43.661 | INFO | PerfReport: 10.94%, Baseline -- 3.30e+00 / -nan, 1.78e-02 / 0.00e+00 | 20260506 10:14:43.661 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.34e-02 / 0.00e+00 | 20260506 10:14:43.661 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.11e+00 / 0.00e+00 | 20260506 10:14:43.661 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.72e+01 / -nan, 4.21e-02 / 0.00e+00 | 20260506 10:14:43.661 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.24e+00 / -nan, 1.29e-01 / 0.00e+00 | 20260506 10:14:43.661 | INFO | PerfReport: 3.71%, Metrics -- 1.12e+00 / -nan, 1.74e-01 / 0.00e+00 | 20260506 10:14:43.661 | INFO | PerfReport: 1.44%, PulseSerializing -- 4.33e-01 / -nan, 6.15e-02 / 0.00e+00 | 20260506 10:14:43.661 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:14:43.661 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:14:43.661 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:14:43.661 | INFO | PerfReport: Full Execution -- 9.02e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 10:14:43.661 | INFO | PerfReport: 18.61%, Upload -- 1.68e+01 / 0.00e+00, 4.44e-01 / 0.00e+00 | 20260506 10:14:43.661 | INFO | PerfReport: 17.88%, Download -- 1.61e+01 / 0.00e+00, 7.77e-01 / 0.00e+00 | 20260506 10:14:43.661 | INFO | PerfReport: 33.40%, Compute -- 3.01e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260506 10:14:43.661 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260506 10:14:43.661 | INFO | PerfReport: 0.00%, LockDownload -- 4.85e-04 / 0.00e+00, 1.51e-04 / 0.00e+00 | 20260506 10:14:43.661 | INFO | PerfReport: 29.54%, LockCompute -- 2.66e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 10:14:43.661 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.24e-02 / 0.00e+00 | 20260506 10:14:43.758 | INFO | Starting a round of Baz IO | 20260506 10:14:46.672 | INFO | Populating RTMetrics report | 20260506 10:14:46.672 | INFO | RTMetrics report populated | 20260506 10:14:46.672 | INFO | Opening tmp RTMetrics json file | 20260506 10:14:46.677 | INFO | Serializing RTMetrics to json | 20260506 10:14:46.680 | INFO | Serialization done, swapping tmp file now... | 20260506 10:14:46.684 | INFO | RTMetrics done | 20260506 10:14:46.960 | INFO | Finished a round of Baz IO: Wrote 1.963 GiB in 3.202 seconds | 20260506 10:14:46.960 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.89% / 0.06% / 51.05% | 20260506 10:14:46.960 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.28999% / 9.13938e-05% / 4.76837e-05% | 20260506 10:14:46.960 | INFO | Baz IO overall: | 20260506 10:14:46.960 | INFO | BytesWritten : 245447748471 | 20260506 10:14:46.960 | INFO | NumEvents : 117756234087 | 20260506 10:14:48.800 | INFO | Analyzing chunk frames = [557056,557568) | 20260506 10:14:48.800 | INFO | MemInfo: { | 20260506 10:14:48.800 | INFO | "gpu" : " 17 GB", | 20260506 10:14:48.800 | INFO | "huge" : "", | 20260506 10:14:48.800 | INFO | "huge_shared" : " 27 GB", | 20260506 10:14:48.800 | INFO | "malloc" : " 14 GB", | 20260506 10:14:48.800 | INFO | "pinned" : " 29 GB" | 20260506 10:14:48.800 | INFO | } | 20260506 10:14:48.800 | INFO | Perf report for the last 5.13942 seconds: | 20260506 10:14:48.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:14:48.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:48.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:48.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:48.800 | INFO | ANALYSIS : 57.6%, 2.8, 85.4ms, 0 | 20260506 10:14:48.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:48.800 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 10:14:48.800 | INFO | PRE_HQ : 56.7%, 2.1, 60.6ms, 0 | 20260506 10:14:48.800 | INFO | BAZWRITER : 61.9%, 4.8, 151.7ms, 0 | 20260506 10:14:48.800 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 10:14:48.800 | INFO | Uploaded 36GiB to GPU in 1.67476s (21.4957GiB/s) | 20260506 10:14:48.800 | INFO | Downloaded 38GiB to GPU in 1.74026s (21.8358GiB/s) | 20260506 10:14:53.939 | INFO | Analyzing chunk frames = [557568,558080) | 20260506 10:14:53.939 | INFO | MemInfo: { | 20260506 10:14:53.939 | INFO | "gpu" : " 17 GB", | 20260506 10:14:53.939 | INFO | "huge" : "", | 20260506 10:14:53.939 | INFO | "huge_shared" : " 27 GB", | 20260506 10:14:53.939 | INFO | "malloc" : " 14 GB", | 20260506 10:14:53.939 | INFO | "pinned" : " 29 GB" | 20260506 10:14:53.939 | INFO | } | 20260506 10:14:53.939 | INFO | Perf report for the last 5.13956 seconds: | 20260506 10:14:53.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:14:53.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:53.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:53.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:53.940 | INFO | ANALYSIS : 57.5%, 3.0, 89.6ms, 0 | 20260506 10:14:53.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:53.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:53.940 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:14:53.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:53.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:53.940 | INFO | Uploaded 36GiB to GPU in 1.60407s (22.4429GiB/s) | 20260506 10:14:53.940 | INFO | Downloaded 34GiB to GPU in 1.52114s (22.3517GiB/s) | 20260506 10:14:59.079 | INFO | Analyzing chunk frames = [558080,558592) | 20260506 10:14:59.079 | INFO | MemInfo: { | 20260506 10:14:59.079 | INFO | "gpu" : " 17 GB", | 20260506 10:14:59.079 | INFO | "huge" : "", | 20260506 10:14:59.079 | INFO | "huge_shared" : " 27 GB", | 20260506 10:14:59.079 | INFO | "malloc" : " 14 GB", | 20260506 10:14:59.079 | INFO | "pinned" : " 29 GB" | 20260506 10:14:59.079 | INFO | } | 20260506 10:14:59.079 | INFO | Perf report for the last 5.13953 seconds: | 20260506 10:14:59.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:14:59.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:59.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:59.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:59.079 | INFO | ANALYSIS : 56.7%, 3.0, 88.2ms, 0 | 20260506 10:14:59.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:59.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:59.079 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:14:59.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:59.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:14:59.079 | INFO | Uploaded 36GiB to GPU in 1.60729s (22.3979GiB/s) | 20260506 10:14:59.079 | INFO | Downloaded 34GiB to GPU in 1.52469s (22.2996GiB/s) | 20260506 10:15:04.218 | INFO | Analyzing chunk frames = [558592,559104) | 20260506 10:15:04.218 | INFO | MemInfo: { | 20260506 10:15:04.218 | INFO | "gpu" : " 17 GB", | 20260506 10:15:04.218 | INFO | "huge" : "", | 20260506 10:15:04.218 | INFO | "huge_shared" : " 27 GB", | 20260506 10:15:04.218 | INFO | "malloc" : " 14 GB", | 20260506 10:15:04.218 | INFO | "pinned" : " 29 GB" | 20260506 10:15:04.218 | INFO | } | 20260506 10:15:04.218 | INFO | Perf report for the last 5.13914 seconds: | 20260506 10:15:04.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:15:04.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:04.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:04.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:04.218 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:15:04.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:04.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:04.218 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:15:04.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:04.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:04.219 | INFO | Uploaded 36GiB to GPU in 1.60653s (22.4086GiB/s) | 20260506 10:15:04.219 | INFO | Downloaded 34GiB to GPU in 1.52426s (22.306GiB/s) | 20260506 10:15:09.357 | INFO | Analyzing chunk frames = [559104,559616) | 20260506 10:15:09.357 | INFO | MemInfo: { | 20260506 10:15:09.357 | INFO | "gpu" : " 17 GB", | 20260506 10:15:09.357 | INFO | "huge" : "", | 20260506 10:15:09.357 | INFO | "huge_shared" : " 27 GB", | 20260506 10:15:09.357 | INFO | "malloc" : " 14 GB", | 20260506 10:15:09.357 | INFO | "pinned" : " 29 GB" | 20260506 10:15:09.358 | INFO | } | 20260506 10:15:09.358 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:15:09.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:15:09.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:09.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:09.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:09.358 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260506 10:15:09.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:09.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:09.358 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:15:09.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:09.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:09.358 | INFO | Uploaded 36GiB to GPU in 1.60561s (22.4214GiB/s) | 20260506 10:15:09.358 | INFO | Downloaded 34GiB to GPU in 1.52188s (22.3408GiB/s) | 20260506 10:15:14.497 | INFO | Analyzing chunk frames = [559616,560128) | 20260506 10:15:14.497 | INFO | MemInfo: { | 20260506 10:15:14.497 | INFO | "gpu" : " 17 GB", | 20260506 10:15:14.497 | INFO | "huge" : "", | 20260506 10:15:14.497 | INFO | "huge_shared" : " 27 GB", | 20260506 10:15:14.497 | INFO | "malloc" : " 14 GB", | 20260506 10:15:14.497 | INFO | "pinned" : " 29 GB" | 20260506 10:15:14.497 | INFO | } | 20260506 10:15:14.497 | INFO | Perf report for the last 5.13954 seconds: | 20260506 10:15:14.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:15:14.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:14.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:14.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:14.497 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:15:14.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:14.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:14.497 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:15:14.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:14.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:14.498 | INFO | Uploaded 36GiB to GPU in 1.60468s (22.4344GiB/s) | 20260506 10:15:14.498 | INFO | Downloaded 34GiB to GPU in 1.52216s (22.3367GiB/s) | 20260506 10:15:19.636 | INFO | Analyzing chunk frames = [560128,560640) | 20260506 10:15:19.636 | INFO | MemInfo: { | 20260506 10:15:19.636 | INFO | "gpu" : " 17 GB", | 20260506 10:15:19.636 | INFO | "huge" : "", | 20260506 10:15:19.636 | INFO | "huge_shared" : " 27 GB", | 20260506 10:15:19.636 | INFO | "malloc" : " 14 GB", | 20260506 10:15:19.636 | INFO | "pinned" : " 29 GB" | 20260506 10:15:19.636 | INFO | } | 20260506 10:15:19.636 | INFO | Perf report for the last 5.13914 seconds: | 20260506 10:15:19.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:15:19.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:19.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:19.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:19.636 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:15:19.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:19.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:19.636 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:15:19.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:19.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:19.637 | INFO | Uploaded 36GiB to GPU in 1.60569s (22.4203GiB/s) | 20260506 10:15:19.637 | INFO | Downloaded 34GiB to GPU in 1.52371s (22.3139GiB/s) | 20260506 10:15:24.775 | INFO | Analyzing chunk frames = [560640,561152) | 20260506 10:15:24.775 | INFO | MemInfo: { | 20260506 10:15:24.775 | INFO | "gpu" : " 17 GB", | 20260506 10:15:24.775 | INFO | "huge" : "", | 20260506 10:15:24.775 | INFO | "huge_shared" : " 27 GB", | 20260506 10:15:24.775 | INFO | "malloc" : " 14 GB", | 20260506 10:15:24.776 | INFO | "pinned" : " 29 GB" | 20260506 10:15:24.776 | INFO | } | 20260506 10:15:24.776 | INFO | Perf report for the last 5.1393 seconds: | 20260506 10:15:24.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:15:24.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:24.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:24.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:24.776 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:15:24.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:24.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:24.776 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 10:15:24.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:24.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:24.776 | INFO | Uploaded 36GiB to GPU in 1.60516s (22.4277GiB/s) | 20260506 10:15:24.776 | INFO | Downloaded 34GiB to GPU in 1.52229s (22.3347GiB/s) | 20260506 10:15:24.873 | INFO | Starting a round of Baz IO | 20260506 10:15:27.783 | INFO | Populating RTMetrics report | 20260506 10:15:27.783 | INFO | RTMetrics report populated | 20260506 10:15:27.783 | INFO | Opening tmp RTMetrics json file | 20260506 10:15:27.807 | INFO | Serializing RTMetrics to json | 20260506 10:15:27.875 | INFO | Serialization done, swapping tmp file now... | 20260506 10:15:27.889 | INFO | RTMetrics done | 20260506 10:15:28.080 | INFO | Finished a round of Baz IO: Wrote 1.961 GiB in 3.206 seconds | 20260506 10:15:28.080 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.83% / 0.06% / 51.11% | 20260506 10:15:28.080 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.23961% / 0.000150998% / 4.76837e-05% | 20260506 10:15:28.080 | INFO | Baz IO overall: | 20260506 10:15:28.080 | INFO | BytesWritten : 247553759019 | 20260506 10:15:28.080 | INFO | NumEvents : 118777847389 | 20260506 10:15:29.915 | INFO | Analyzing chunk frames = [561152,561664) | 20260506 10:15:29.915 | INFO | MemInfo: { | 20260506 10:15:29.915 | INFO | "gpu" : " 17 GB", | 20260506 10:15:29.915 | INFO | "huge" : "", | 20260506 10:15:29.915 | INFO | "huge_shared" : " 27 GB", | 20260506 10:15:29.915 | INFO | "malloc" : " 14 GB", | 20260506 10:15:29.915 | INFO | "pinned" : " 29 GB" | 20260506 10:15:29.915 | INFO | } | 20260506 10:15:29.915 | INFO | Perf report for the last 5.13927 seconds: | 20260506 10:15:29.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:15:29.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:29.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:29.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:29.915 | INFO | ANALYSIS : 57.5%, 2.8, 85.4ms, 0 | 20260506 10:15:29.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:29.915 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 10:15:29.915 | INFO | PRE_HQ : 56.7%, 2.1, 60.5ms, 0 | 20260506 10:15:29.915 | INFO | BAZWRITER : 61.8%, 4.8, 151.4ms, 0 | 20260506 10:15:29.915 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 10:15:29.915 | INFO | Uploaded 36GiB to GPU in 1.67459s (21.4978GiB/s) | 20260506 10:15:29.915 | INFO | Downloaded 38GiB to GPU in 1.74003s (21.8388GiB/s) | 20260506 10:15:35.054 | INFO | Analyzing chunk frames = [561664,562176) | 20260506 10:15:35.054 | INFO | MemInfo: { | 20260506 10:15:35.054 | INFO | "gpu" : " 17 GB", | 20260506 10:15:35.054 | INFO | "huge" : "", | 20260506 10:15:35.054 | INFO | "huge_shared" : " 27 GB", | 20260506 10:15:35.054 | INFO | "malloc" : " 14 GB", | 20260506 10:15:35.054 | INFO | "pinned" : " 29 GB" | 20260506 10:15:35.054 | INFO | } | 20260506 10:15:35.054 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:15:35.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:15:35.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:35.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:35.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:35.054 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 10:15:35.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:35.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:35.054 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:15:35.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:35.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:35.054 | INFO | Uploaded 36GiB to GPU in 1.60275s (22.4615GiB/s) | 20260506 10:15:35.054 | INFO | Downloaded 34GiB to GPU in 1.52s (22.3685GiB/s) | 20260506 10:15:40.193 | INFO | Analyzing chunk frames = [562176,562688) | 20260506 10:15:40.193 | INFO | MemInfo: { | 20260506 10:15:40.193 | INFO | "gpu" : " 17 GB", | 20260506 10:15:40.193 | INFO | "huge" : "", | 20260506 10:15:40.193 | INFO | "huge_shared" : " 27 GB", | 20260506 10:15:40.193 | INFO | "malloc" : " 14 GB", | 20260506 10:15:40.193 | INFO | "pinned" : " 29 GB" | 20260506 10:15:40.193 | INFO | } | 20260506 10:15:40.193 | INFO | Perf report for the last 5.13921 seconds: | 20260506 10:15:40.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:15:40.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:40.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:40.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:40.194 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 10:15:40.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:40.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:40.194 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:15:40.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:40.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:40.194 | INFO | Uploaded 36GiB to GPU in 1.60673s (22.4058GiB/s) | 20260506 10:15:40.194 | INFO | Downloaded 34GiB to GPU in 1.52514s (22.293GiB/s) | 20260506 10:15:45.333 | INFO | Analyzing chunk frames = [562688,563200) | 20260506 10:15:45.333 | INFO | MemInfo: { | 20260506 10:15:45.333 | INFO | "gpu" : " 17 GB", | 20260506 10:15:45.333 | INFO | "huge" : "", | 20260506 10:15:45.333 | INFO | "huge_shared" : " 27 GB", | 20260506 10:15:45.333 | INFO | "malloc" : " 14 GB", | 20260506 10:15:45.333 | INFO | "pinned" : " 29 GB" | 20260506 10:15:45.333 | INFO | } | 20260506 10:15:45.333 | INFO | Perf report for the last 5.13954 seconds: | 20260506 10:15:45.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:15:45.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:45.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:45.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:45.333 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:15:45.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:45.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:45.333 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:15:45.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:45.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:45.334 | INFO | Uploaded 36GiB to GPU in 1.6061s (22.4145GiB/s) | 20260506 10:15:45.334 | INFO | Downloaded 34GiB to GPU in 1.52414s (22.3076GiB/s) | 20260506 10:15:50.472 | INFO | Analyzing chunk frames = [563200,563712) | 20260506 10:15:50.472 | INFO | MemInfo: { | 20260506 10:15:50.472 | INFO | "gpu" : " 17 GB", | 20260506 10:15:50.472 | INFO | "huge" : "", | 20260506 10:15:50.472 | INFO | "huge_shared" : " 27 GB", | 20260506 10:15:50.472 | INFO | "malloc" : " 14 GB", | 20260506 10:15:50.472 | INFO | "pinned" : " 29 GB" | 20260506 10:15:50.472 | INFO | } | 20260506 10:15:50.472 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:15:50.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:15:50.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:50.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:50.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:50.472 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:15:50.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:50.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:50.472 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:15:50.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:50.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:50.473 | INFO | Uploaded 36GiB to GPU in 1.60449s (22.4371GiB/s) | 20260506 10:15:50.473 | INFO | Downloaded 34GiB to GPU in 1.52185s (22.3412GiB/s) | 20260506 10:15:55.611 | INFO | Analyzing chunk frames = [563712,564224) | 20260506 10:15:55.611 | INFO | MemInfo: { | 20260506 10:15:55.611 | INFO | "gpu" : " 17 GB", | 20260506 10:15:55.611 | INFO | "huge" : "", | 20260506 10:15:55.611 | INFO | "huge_shared" : " 27 GB", | 20260506 10:15:55.611 | INFO | "malloc" : " 14 GB", | 20260506 10:15:55.611 | INFO | "pinned" : " 29 GB" | 20260506 10:15:55.611 | INFO | } | 20260506 10:15:55.611 | INFO | Perf report for the last 5.13905 seconds: | 20260506 10:15:55.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:15:55.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:55.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:55.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:55.612 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:15:55.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:55.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:55.612 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:15:55.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:55.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:15:55.612 | INFO | Uploaded 36GiB to GPU in 1.60469s (22.4342GiB/s) | 20260506 10:15:55.612 | INFO | Downloaded 34GiB to GPU in 1.52271s (22.3287GiB/s) | 20260506 10:16:00.751 | INFO | Analyzing chunk frames = [564224,564736) | 20260506 10:16:00.751 | INFO | MemInfo: { | 20260506 10:16:00.751 | INFO | "gpu" : " 17 GB", | 20260506 10:16:00.751 | INFO | "huge" : "", | 20260506 10:16:00.751 | INFO | "huge_shared" : " 27 GB", | 20260506 10:16:00.751 | INFO | "malloc" : " 14 GB", | 20260506 10:16:00.751 | INFO | "pinned" : " 29 GB" | 20260506 10:16:00.751 | INFO | } | 20260506 10:16:00.751 | INFO | Perf report for the last 5.13974 seconds: | 20260506 10:16:00.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:16:00.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:00.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:00.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:00.751 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:16:00.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:00.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:00.751 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:16:00.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:00.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:00.751 | INFO | Uploaded 36GiB to GPU in 1.60507s (22.4289GiB/s) | 20260506 10:16:00.751 | INFO | Downloaded 34GiB to GPU in 1.52243s (22.3327GiB/s) | 20260506 10:16:05.890 | INFO | Analyzing chunk frames = [564736,565248) | 20260506 10:16:05.890 | INFO | MemInfo: { | 20260506 10:16:05.890 | INFO | "gpu" : " 17 GB", | 20260506 10:16:05.890 | INFO | "huge" : "", | 20260506 10:16:05.890 | INFO | "huge_shared" : " 27 GB", | 20260506 10:16:05.890 | INFO | "malloc" : " 14 GB", | 20260506 10:16:05.890 | INFO | "pinned" : " 29 GB" | 20260506 10:16:05.890 | INFO | } | 20260506 10:16:05.890 | INFO | Perf report for the last 5.13913 seconds: | 20260506 10:16:05.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:16:05.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:05.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:05.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:05.890 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260506 10:16:05.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:05.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:05.890 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 10:16:05.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:05.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:05.890 | INFO | Uploaded 36GiB to GPU in 1.60567s (22.4206GiB/s) | 20260506 10:16:05.891 | INFO | Downloaded 34GiB to GPU in 1.52413s (22.3078GiB/s) | 20260506 10:16:05.891 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:16:05.891 | INFO | Total mem usage: 16.5 GiB | 20260506 10:16:05.891 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:16:05.891 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -7.62%) | 20260506 10:16:05.891 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:16:05.891 | INFO | High Water Breakdown: | 20260506 10:16:05.891 | INFO | 60% : 9.21 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:16:05.891 | INFO | 39.9% : 6.12 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:16:05.891 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:16:05.891 | INFO | ------------------------------------------ | 20260506 10:16:05.891 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:16:05.891 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:16:05.891 | INFO | Total mem usage: 26.5 GiB | 20260506 10:16:05.891 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:16:05.891 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:16:05.891 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:16:05.891 | INFO | High Water Breakdown: | 20260506 10:16:05.891 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:16:05.891 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:16:05.891 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:16:05.891 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:16:05.891 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:16:05.891 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:16:05.891 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:16:05.891 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:16:05.891 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:16:05.891 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:16:05.891 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:16:05.891 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:16:05.891 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:16:05.891 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:16:05.892 | INFO | ------------------------------------------ | 20260506 10:16:05.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:16:05.892 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:16:05.892 | INFO | Total mem usage: 25.6 GiB | 20260506 10:16:05.892 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:16:05.892 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:16:05.892 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:16:05.892 | INFO | High Water Breakdown: | 20260506 10:16:05.892 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:16:05.892 | INFO | ------------------------------------------ | 20260506 10:16:05.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:16:05.892 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:16:05.892 | INFO | Total mem usage: 15.7 GiB | 20260506 10:16:05.892 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:16:05.892 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:16:05.892 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:16:05.892 | INFO | High Water Breakdown: | 20260506 10:16:05.892 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:16:05.892 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:16:05.892 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:16:05.892 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:16:05.892 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:16:05.892 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:16:05.892 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:16:05.892 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:16:05.892 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:16:05.892 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:16:05.892 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:16:05.892 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:16:05.892 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:16:05.892 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:16:05.892 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:16:05.892 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:16:05.892 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:16:05.892 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:16:05.892 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:16:05.892 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:16:05.892 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:16:05.892 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:16:05.892 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:16:05.892 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:16:05.892 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:16:05.892 | INFO | ------------------------------------------ | 20260506 10:16:05.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:16:05.892 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:16:05.892 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:16:05.892 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:16:05.892 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 10:16:05.892 | INFO | PerfReport: 0.29%, AggregateStats -- 8.84e-02 / -nan, 7.58e-02 / 0.00e+00 | 20260506 10:16:05.892 | INFO | PerfReport: 10.94%, Baseline -- 3.29e+00 / -nan, 1.73e-02 / 0.00e+00 | 20260506 10:16:05.892 | INFO | PerfReport: 3.06%, Histogram -- 9.22e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260506 10:16:05.892 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.10e+00 / 0.00e+00 | 20260506 10:16:05.892 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.72e+01 / -nan, 4.24e-02 / 0.00e+00 | 20260506 10:16:05.892 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.23e+00 / -nan, 1.29e-01 / 0.00e+00 | 20260506 10:16:05.892 | INFO | PerfReport: 3.70%, Metrics -- 1.11e+00 / -nan, 1.73e-01 / 0.00e+00 | 20260506 10:16:05.892 | INFO | PerfReport: 1.43%, PulseSerializing -- 4.31e-01 / -nan, 6.14e-02 / 0.00e+00 | 20260506 10:16:05.892 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:16:05.892 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:16:05.892 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:16:05.892 | INFO | PerfReport: Full Execution -- 8.99e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:16:05.892 | INFO | PerfReport: 18.70%, Upload -- 1.68e+01 / 0.00e+00, 4.98e-01 / 0.00e+00 | 20260506 10:16:05.892 | INFO | PerfReport: 17.96%, Download -- 1.61e+01 / 0.00e+00, 8.48e-01 / 0.00e+00 | 20260506 10:16:05.892 | INFO | PerfReport: 33.49%, Compute -- 3.01e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 10:16:05.893 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260506 10:16:05.893 | INFO | PerfReport: 0.00%, LockDownload -- 4.93e-04 / 0.00e+00, 2.44e-04 / 0.00e+00 | 20260506 10:16:05.893 | INFO | PerfReport: 29.28%, LockCompute -- 2.63e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 10:16:05.893 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.15e-02 / 0.00e+00 | 20260506 10:16:05.987 | INFO | Starting a round of Baz IO | 20260506 10:16:08.902 | INFO | Populating RTMetrics report | 20260506 10:16:08.902 | INFO | RTMetrics report populated | 20260506 10:16:08.903 | INFO | Opening tmp RTMetrics json file | 20260506 10:16:08.974 | INFO | Serializing RTMetrics to json | 20260506 10:16:08.979 | INFO | Serialization done, swapping tmp file now... | 20260506 10:16:08.984 | INFO | RTMetrics done | 20260506 10:16:09.362 | INFO | Finished a round of Baz IO: Wrote 1.965 GiB in 3.374 seconds | 20260506 10:16:09.362 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.78% / 0.06% / 51.17% | 20260506 10:16:09.362 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.22498% / 0.000143051% / 5.96046e-05% | 20260506 10:16:09.362 | INFO | Baz IO overall: | 20260506 10:16:09.362 | INFO | BytesWritten : 249663899178 | 20260506 10:16:09.362 | INFO | NumEvents : 119801644831 | 20260506 10:16:11.029 | INFO | Analyzing chunk frames = [565248,565760) | 20260506 10:16:11.029 | INFO | MemInfo: { | 20260506 10:16:11.029 | INFO | "gpu" : " 17 GB", | 20260506 10:16:11.029 | INFO | "huge" : "", | 20260506 10:16:11.029 | INFO | "huge_shared" : " 27 GB", | 20260506 10:16:11.029 | INFO | "malloc" : " 14 GB", | 20260506 10:16:11.029 | INFO | "pinned" : " 29 GB" | 20260506 10:16:11.029 | INFO | } | 20260506 10:16:11.029 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:16:11.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:16:11.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:11.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:11.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:11.030 | INFO | ANALYSIS : 57.6%, 2.8, 86.1ms, 0 | 20260506 10:16:11.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:11.030 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260506 10:16:11.030 | INFO | PRE_HQ : 56.8%, 2.1, 61.0ms, 0 | 20260506 10:16:11.030 | INFO | BAZWRITER : 65.2%, 4.6, 154.9ms, 0 | 20260506 10:16:11.030 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 10:16:11.030 | INFO | Uploaded 36GiB to GPU in 1.67055s (21.5498GiB/s) | 20260506 10:16:11.030 | INFO | Downloaded 38GiB to GPU in 1.73521s (21.8994GiB/s) | 20260506 10:16:16.169 | INFO | Analyzing chunk frames = [565760,566272) | 20260506 10:16:16.169 | INFO | MemInfo: { | 20260506 10:16:16.169 | INFO | "gpu" : " 17 GB", | 20260506 10:16:16.169 | INFO | "huge" : "", | 20260506 10:16:16.169 | INFO | "huge_shared" : " 27 GB", | 20260506 10:16:16.169 | INFO | "malloc" : " 14 GB", | 20260506 10:16:16.169 | INFO | "pinned" : " 29 GB" | 20260506 10:16:16.169 | INFO | } | 20260506 10:16:16.169 | INFO | Perf report for the last 5.13952 seconds: | 20260506 10:16:16.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:16:16.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:16.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:16.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:16.169 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:16:16.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:16.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:16.169 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:16:16.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:16.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:16.169 | INFO | Uploaded 36GiB to GPU in 1.6034s (22.4523GiB/s) | 20260506 10:16:16.169 | INFO | Downloaded 34GiB to GPU in 1.52075s (22.3574GiB/s) | 20260506 10:16:21.308 | INFO | Analyzing chunk frames = [566272,566784) | 20260506 10:16:21.308 | INFO | MemInfo: { | 20260506 10:16:21.308 | INFO | "gpu" : " 17 GB", | 20260506 10:16:21.308 | INFO | "huge" : "", | 20260506 10:16:21.308 | INFO | "huge_shared" : " 27 GB", | 20260506 10:16:21.308 | INFO | "malloc" : " 14 GB", | 20260506 10:16:21.308 | INFO | "pinned" : " 29 GB" | 20260506 10:16:21.308 | INFO | } | 20260506 10:16:21.308 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:16:21.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:16:21.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:21.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:21.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:21.308 | INFO | ANALYSIS : 56.8%, 3.0, 88.3ms, 0 | 20260506 10:16:21.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:21.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:21.309 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:16:21.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:21.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:21.309 | INFO | Uploaded 36GiB to GPU in 1.60666s (22.4067GiB/s) | 20260506 10:16:21.309 | INFO | Downloaded 34GiB to GPU in 1.52451s (22.3022GiB/s) | 20260506 10:16:26.447 | INFO | Analyzing chunk frames = [566784,567296) | 20260506 10:16:26.448 | INFO | MemInfo: { | 20260506 10:16:26.448 | INFO | "gpu" : " 17 GB", | 20260506 10:16:26.448 | INFO | "huge" : "", | 20260506 10:16:26.448 | INFO | "huge_shared" : " 27 GB", | 20260506 10:16:26.448 | INFO | "malloc" : " 14 GB", | 20260506 10:16:26.448 | INFO | "pinned" : " 29 GB" | 20260506 10:16:26.448 | INFO | } | 20260506 10:16:26.448 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:16:26.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:16:26.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:26.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:26.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:26.448 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:16:26.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:26.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:26.448 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:16:26.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:26.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:26.448 | INFO | Uploaded 36GiB to GPU in 1.6059s (22.4173GiB/s) | 20260506 10:16:26.448 | INFO | Downloaded 34GiB to GPU in 1.52374s (22.3135GiB/s) | 20260506 10:16:31.587 | INFO | Analyzing chunk frames = [567296,567808) | 20260506 10:16:31.587 | INFO | MemInfo: { | 20260506 10:16:31.587 | INFO | "gpu" : " 17 GB", | 20260506 10:16:31.587 | INFO | "huge" : "", | 20260506 10:16:31.587 | INFO | "huge_shared" : " 27 GB", | 20260506 10:16:31.587 | INFO | "malloc" : " 14 GB", | 20260506 10:16:31.587 | INFO | "pinned" : " 29 GB" | 20260506 10:16:31.587 | INFO | } | 20260506 10:16:31.587 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:16:31.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:16:31.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:31.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:31.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:31.587 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:16:31.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:31.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:31.587 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:16:31.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:31.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:31.588 | INFO | Uploaded 36GiB to GPU in 1.60496s (22.4304GiB/s) | 20260506 10:16:31.588 | INFO | Downloaded 34GiB to GPU in 1.52206s (22.3381GiB/s) | 20260506 10:16:36.726 | INFO | Analyzing chunk frames = [567808,568320) | 20260506 10:16:36.726 | INFO | MemInfo: { | 20260506 10:16:36.726 | INFO | "gpu" : " 17 GB", | 20260506 10:16:36.726 | INFO | "huge" : "", | 20260506 10:16:36.726 | INFO | "huge_shared" : " 27 GB", | 20260506 10:16:36.726 | INFO | "malloc" : " 14 GB", | 20260506 10:16:36.726 | INFO | "pinned" : " 29 GB" | 20260506 10:16:36.726 | INFO | } | 20260506 10:16:36.726 | INFO | Perf report for the last 5.13911 seconds: | 20260506 10:16:36.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:16:36.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:36.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:36.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:36.726 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:16:36.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:36.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:36.726 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:16:36.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:36.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:36.727 | INFO | Uploaded 36GiB to GPU in 1.60404s (22.4433GiB/s) | 20260506 10:16:36.727 | INFO | Downloaded 34GiB to GPU in 1.5227s (22.3288GiB/s) | 20260506 10:16:41.866 | INFO | Analyzing chunk frames = [568320,568832) | 20260506 10:16:41.866 | INFO | MemInfo: { | 20260506 10:16:41.866 | INFO | "gpu" : " 17 GB", | 20260506 10:16:41.866 | INFO | "huge" : "", | 20260506 10:16:41.866 | INFO | "huge_shared" : " 27 GB", | 20260506 10:16:41.866 | INFO | "malloc" : " 14 GB", | 20260506 10:16:41.866 | INFO | "pinned" : " 29 GB" | 20260506 10:16:41.866 | INFO | } | 20260506 10:16:41.866 | INFO | Perf report for the last 5.13947 seconds: | 20260506 10:16:41.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:16:41.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:41.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:41.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:41.866 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:16:41.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:41.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:41.866 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:16:41.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:41.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:41.866 | INFO | Uploaded 36GiB to GPU in 1.60409s (22.4427GiB/s) | 20260506 10:16:41.866 | INFO | Downloaded 34GiB to GPU in 1.52339s (22.3186GiB/s) | 20260506 10:16:47.005 | INFO | Analyzing chunk frames = [568832,569344) | 20260506 10:16:47.005 | INFO | MemInfo: { | 20260506 10:16:47.005 | INFO | "gpu" : " 17 GB", | 20260506 10:16:47.005 | INFO | "huge" : "", | 20260506 10:16:47.005 | INFO | "huge_shared" : " 27 GB", | 20260506 10:16:47.005 | INFO | "malloc" : " 14 GB", | 20260506 10:16:47.005 | INFO | "pinned" : " 29 GB" | 20260506 10:16:47.005 | INFO | } | 20260506 10:16:47.005 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:16:47.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:16:47.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:47.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:47.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:47.005 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:16:47.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:47.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:47.005 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 10:16:47.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:47.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:47.005 | INFO | Uploaded 36GiB to GPU in 1.60457s (22.436GiB/s) | 20260506 10:16:47.006 | INFO | Downloaded 34GiB to GPU in 1.52404s (22.3091GiB/s) | 20260506 10:16:47.103 | INFO | Starting a round of Baz IO | 20260506 10:16:50.016 | INFO | Populating RTMetrics report | 20260506 10:16:50.017 | INFO | RTMetrics report populated | 20260506 10:16:50.017 | INFO | Opening tmp RTMetrics json file | 20260506 10:16:50.018 | INFO | Serializing RTMetrics to json | 20260506 10:16:50.022 | INFO | Serialization done, swapping tmp file now... | 20260506 10:16:50.026 | INFO | RTMetrics done | 20260506 10:16:50.207 | INFO | Finished a round of Baz IO: Wrote 1.966 GiB in 3.103 seconds | 20260506 10:16:50.207 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.72% / 0.06% / 51.22% | 20260506 10:16:50.207 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.19128% / 0.000150998% / 3.97364e-05% | 20260506 10:16:50.207 | INFO | Baz IO overall: | 20260506 10:16:50.207 | INFO | BytesWritten : 251774822241 | 20260506 10:16:50.207 | INFO | NumEvents : 120825660311 | 20260506 10:16:52.144 | INFO | Analyzing chunk frames = [569344,569856) | 20260506 10:16:52.144 | INFO | MemInfo: { | 20260506 10:16:52.144 | INFO | "gpu" : " 17 GB", | 20260506 10:16:52.144 | INFO | "huge" : "", | 20260506 10:16:52.144 | INFO | "huge_shared" : " 27 GB", | 20260506 10:16:52.144 | INFO | "malloc" : " 14 GB", | 20260506 10:16:52.144 | INFO | "pinned" : " 29 GB" | 20260506 10:16:52.144 | INFO | } | 20260506 10:16:52.144 | INFO | Perf report for the last 5.13918 seconds: | 20260506 10:16:52.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:16:52.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:52.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:52.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:52.144 | INFO | ANALYSIS : 57.6%, 2.8, 85.4ms, 0 | 20260506 10:16:52.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:52.144 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 10:16:52.144 | INFO | PRE_HQ : 56.6%, 2.1, 60.8ms, 0 | 20260506 10:16:52.144 | INFO | BAZWRITER : 59.9%, 4.8, 146.3ms, 0 | 20260506 10:16:52.144 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 10:16:52.145 | INFO | Uploaded 36GiB to GPU in 1.67122s (21.5411GiB/s) | 20260506 10:16:52.145 | INFO | Downloaded 38GiB to GPU in 1.73763s (21.8689GiB/s) | 20260506 10:16:57.283 | INFO | Analyzing chunk frames = [569856,570368) | 20260506 10:16:57.284 | INFO | MemInfo: { | 20260506 10:16:57.284 | INFO | "gpu" : " 17 GB", | 20260506 10:16:57.284 | INFO | "huge" : "", | 20260506 10:16:57.284 | INFO | "huge_shared" : " 27 GB", | 20260506 10:16:57.284 | INFO | "malloc" : " 14 GB", | 20260506 10:16:57.284 | INFO | "pinned" : " 29 GB" | 20260506 10:16:57.284 | INFO | } | 20260506 10:16:57.284 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:16:57.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:16:57.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:57.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:57.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:57.284 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:16:57.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:57.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:57.284 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:16:57.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:57.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:16:57.284 | INFO | Uploaded 36GiB to GPU in 1.60297s (22.4583GiB/s) | 20260506 10:16:57.284 | INFO | Downloaded 34GiB to GPU in 1.5198s (22.3714GiB/s) | 20260506 10:17:02.423 | INFO | Analyzing chunk frames = [570368,570880) | 20260506 10:17:02.423 | INFO | MemInfo: { | 20260506 10:17:02.423 | INFO | "gpu" : " 17 GB", | 20260506 10:17:02.423 | INFO | "huge" : "", | 20260506 10:17:02.423 | INFO | "huge_shared" : " 27 GB", | 20260506 10:17:02.423 | INFO | "malloc" : " 14 GB", | 20260506 10:17:02.423 | INFO | "pinned" : " 29 GB" | 20260506 10:17:02.423 | INFO | } | 20260506 10:17:02.423 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:17:02.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:17:02.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:02.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:02.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:02.423 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 10:17:02.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:02.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:02.423 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:17:02.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:02.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:02.424 | INFO | Uploaded 36GiB to GPU in 1.60668s (22.4065GiB/s) | 20260506 10:17:02.424 | INFO | Downloaded 34GiB to GPU in 1.52596s (22.2811GiB/s) | 20260506 10:17:07.562 | INFO | Analyzing chunk frames = [570880,571392) | 20260506 10:17:07.562 | INFO | MemInfo: { | 20260506 10:17:07.562 | INFO | "gpu" : " 17 GB", | 20260506 10:17:07.562 | INFO | "huge" : "", | 20260506 10:17:07.562 | INFO | "huge_shared" : " 27 GB", | 20260506 10:17:07.562 | INFO | "malloc" : " 14 GB", | 20260506 10:17:07.562 | INFO | "pinned" : " 29 GB" | 20260506 10:17:07.562 | INFO | } | 20260506 10:17:07.563 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:17:07.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:17:07.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:07.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:07.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:07.563 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:17:07.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:07.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:07.563 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 10:17:07.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:07.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:07.563 | INFO | Uploaded 36GiB to GPU in 1.60436s (22.4389GiB/s) | 20260506 10:17:07.563 | INFO | Downloaded 34GiB to GPU in 1.52391s (22.311GiB/s) | 20260506 10:17:12.702 | INFO | Analyzing chunk frames = [571392,571904) | 20260506 10:17:12.702 | INFO | MemInfo: { | 20260506 10:17:12.702 | INFO | "gpu" : " 17 GB", | 20260506 10:17:12.702 | INFO | "huge" : "", | 20260506 10:17:12.702 | INFO | "huge_shared" : " 27 GB", | 20260506 10:17:12.702 | INFO | "malloc" : " 14 GB", | 20260506 10:17:12.702 | INFO | "pinned" : " 29 GB" | 20260506 10:17:12.702 | INFO | } | 20260506 10:17:12.702 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:17:12.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:17:12.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:12.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:12.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:12.702 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:17:12.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:12.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:12.702 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:17:12.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:12.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:12.702 | INFO | Uploaded 36GiB to GPU in 1.60445s (22.4376GiB/s) | 20260506 10:17:12.702 | INFO | Downloaded 34GiB to GPU in 1.52206s (22.3381GiB/s) | 20260506 10:17:17.841 | INFO | Analyzing chunk frames = [571904,572416) | 20260506 10:17:17.841 | INFO | MemInfo: { | 20260506 10:17:17.841 | INFO | "gpu" : " 17 GB", | 20260506 10:17:17.841 | INFO | "huge" : "", | 20260506 10:17:17.841 | INFO | "huge_shared" : " 27 GB", | 20260506 10:17:17.841 | INFO | "malloc" : " 14 GB", | 20260506 10:17:17.841 | INFO | "pinned" : " 29 GB" | 20260506 10:17:17.841 | INFO | } | 20260506 10:17:17.841 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:17:17.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:17:17.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:17.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:17.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:17.841 | INFO | ANALYSIS : 56.6%, 3.0, 88.4ms, 0 | 20260506 10:17:17.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:17.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:17.841 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:17:17.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:17.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:17.841 | INFO | Uploaded 36GiB to GPU in 1.60577s (22.4191GiB/s) | 20260506 10:17:17.842 | INFO | Downloaded 34GiB to GPU in 1.52358s (22.3159GiB/s) | 20260506 10:17:22.980 | INFO | Analyzing chunk frames = [572416,572928) | 20260506 10:17:22.980 | INFO | MemInfo: { | 20260506 10:17:22.980 | INFO | "gpu" : " 17 GB", | 20260506 10:17:22.980 | INFO | "huge" : "", | 20260506 10:17:22.980 | INFO | "huge_shared" : " 27 GB", | 20260506 10:17:22.980 | INFO | "malloc" : " 14 GB", | 20260506 10:17:22.980 | INFO | "pinned" : " 29 GB" | 20260506 10:17:22.980 | INFO | } | 20260506 10:17:22.981 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:17:22.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:17:22.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:22.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:22.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:22.981 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:17:22.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:22.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:22.981 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:17:22.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:22.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:22.981 | INFO | Uploaded 36GiB to GPU in 1.60433s (22.4393GiB/s) | 20260506 10:17:22.981 | INFO | Downloaded 34GiB to GPU in 1.52269s (22.3289GiB/s) | 20260506 10:17:28.120 | INFO | Analyzing chunk frames = [572928,573440) | 20260506 10:17:28.120 | INFO | MemInfo: { | 20260506 10:17:28.120 | INFO | "gpu" : " 17 GB", | 20260506 10:17:28.120 | INFO | "huge" : "", | 20260506 10:17:28.120 | INFO | "huge_shared" : " 27 GB", | 20260506 10:17:28.120 | INFO | "malloc" : " 14 GB", | 20260506 10:17:28.120 | INFO | "pinned" : " 29 GB" | 20260506 10:17:28.120 | INFO | } | 20260506 10:17:28.120 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:17:28.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:17:28.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:28.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:28.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:28.120 | INFO | ANALYSIS : 57.0%, 3.0, 89.0ms, 0 | 20260506 10:17:28.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:28.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:28.120 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 10:17:28.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:28.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:28.120 | INFO | Uploaded 36GiB to GPU in 1.60453s (22.4365GiB/s) | 20260506 10:17:28.120 | INFO | Downloaded 34GiB to GPU in 1.52264s (22.3297GiB/s) | 20260506 10:17:28.120 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:17:28.120 | INFO | Total mem usage: 16.5 GiB | 20260506 10:17:28.120 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:17:28.120 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -7.84%) | 20260506 10:17:28.120 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:17:28.120 | INFO | High Water Breakdown: | 20260506 10:17:28.120 | INFO | 59.9% : 9.18 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:17:28.120 | INFO | 39.9% : 6.12 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:17:28.120 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:17:28.120 | INFO | ------------------------------------------ | 20260506 10:17:28.120 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:17:28.120 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:17:28.120 | INFO | Total mem usage: 26.5 GiB | 20260506 10:17:28.120 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:17:28.120 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:17:28.120 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:17:28.120 | INFO | High Water Breakdown: | 20260506 10:17:28.120 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:17:28.120 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:17:28.120 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:17:28.120 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:17:28.120 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:17:28.120 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:17:28.120 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:17:28.120 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:17:28.120 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:17:28.120 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:17:28.120 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:17:28.120 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:17:28.120 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:17:28.120 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:17:28.120 | INFO | ------------------------------------------ | 20260506 10:17:28.120 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:17:28.120 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:17:28.121 | INFO | Total mem usage: 25.6 GiB | 20260506 10:17:28.121 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:17:28.121 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:17:28.121 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:17:28.121 | INFO | High Water Breakdown: | 20260506 10:17:28.121 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:17:28.121 | INFO | ------------------------------------------ | 20260506 10:17:28.121 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:17:28.121 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:17:28.121 | INFO | Total mem usage: 15.7 GiB | 20260506 10:17:28.121 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:17:28.121 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:17:28.121 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:17:28.121 | INFO | High Water Breakdown: | 20260506 10:17:28.121 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:17:28.121 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:17:28.121 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:17:28.121 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:17:28.121 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:17:28.121 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:17:28.121 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:17:28.121 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:17:28.121 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:17:28.121 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:17:28.121 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:17:28.121 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:17:28.121 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:17:28.121 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:17:28.121 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:17:28.121 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:17:28.121 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:17:28.121 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:17:28.121 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:17:28.121 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:17:28.121 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:17:28.121 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:17:28.121 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:17:28.121 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:17:28.121 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:17:28.121 | INFO | ------------------------------------------ | 20260506 10:17:28.121 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:17:28.121 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:17:28.121 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:17:28.121 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:17:28.121 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 10:17:28.121 | INFO | PerfReport: 0.29%, AggregateStats -- 8.63e-02 / -nan, 5.16e-03 / 0.00e+00 | 20260506 10:17:28.121 | INFO | PerfReport: 10.94%, Baseline -- 3.29e+00 / -nan, 1.97e-02 / 0.00e+00 | 20260506 10:17:28.121 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260506 10:17:28.121 | INFO | PerfReport: 16.07%, DME -- 3.87e+01 / -nan, 3.05e+00 / 0.00e+00 | 20260506 10:17:28.121 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.72e+01 / -nan, 4.32e-02 / 0.00e+00 | 20260506 10:17:28.121 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.23e+00 / -nan, 1.30e-01 / 0.00e+00 | 20260506 10:17:28.121 | INFO | PerfReport: 3.70%, Metrics -- 1.11e+00 / -nan, 1.73e-01 / 0.00e+00 | 20260506 10:17:28.121 | INFO | PerfReport: 1.43%, PulseSerializing -- 4.31e-01 / -nan, 6.11e-02 / 0.00e+00 | 20260506 10:17:28.121 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:17:28.121 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:17:28.121 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:17:28.121 | INFO | PerfReport: Full Execution -- 8.99e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 10:17:28.121 | INFO | PerfReport: 18.68%, Upload -- 1.68e+01 / 0.00e+00, 4.77e-01 / 0.00e+00 | 20260506 10:17:28.121 | INFO | PerfReport: 17.95%, Download -- 1.61e+01 / 0.00e+00, 8.35e-01 / 0.00e+00 | 20260506 10:17:28.121 | INFO | PerfReport: 33.48%, Compute -- 3.01e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 10:17:28.121 | INFO | PerfReport: 0.57%, LockUpload -- 5.10e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:17:28.121 | INFO | PerfReport: 0.00%, LockDownload -- 4.84e-04 / 0.00e+00, 1.28e-04 / 0.00e+00 | 20260506 10:17:28.122 | INFO | PerfReport: 29.31%, LockCompute -- 2.64e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 10:17:28.122 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260506 10:17:28.218 | INFO | Starting a round of Baz IO | 20260506 10:17:31.131 | INFO | Populating RTMetrics report | 20260506 10:17:31.131 | INFO | RTMetrics report populated | 20260506 10:17:31.132 | INFO | Opening tmp RTMetrics json file | 20260506 10:17:31.201 | INFO | Serializing RTMetrics to json | 20260506 10:17:31.225 | INFO | Serialization done, swapping tmp file now... | 20260506 10:17:31.229 | INFO | RTMetrics done | 20260506 10:17:31.615 | INFO | Finished a round of Baz IO: Wrote 1.968 GiB in 3.397 seconds | 20260506 10:17:31.615 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.66% / 0.06% / 51.28% | 20260506 10:17:31.615 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.16649% / 9.93411e-05% / 1.98682e-05% | 20260506 10:17:31.615 | INFO | Baz IO overall: | 20260506 10:17:31.615 | INFO | BytesWritten : 253887777178 | 20260506 10:17:31.615 | INFO | NumEvents : 121850863473 | 20260506 10:17:33.259 | INFO | Analyzing chunk frames = [573440,573952) | 20260506 10:17:33.259 | INFO | MemInfo: { | 20260506 10:17:33.259 | INFO | "gpu" : " 17 GB", | 20260506 10:17:33.259 | INFO | "huge" : "", | 20260506 10:17:33.259 | INFO | "huge_shared" : " 27 GB", | 20260506 10:17:33.259 | INFO | "malloc" : " 14 GB", | 20260506 10:17:33.259 | INFO | "pinned" : " 29 GB" | 20260506 10:17:33.259 | INFO | } | 20260506 10:17:33.259 | INFO | Perf report for the last 5.13913 seconds: | 20260506 10:17:33.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:17:33.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:33.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:33.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:33.259 | INFO | ANALYSIS : 57.6%, 2.8, 85.1ms, 0 | 20260506 10:17:33.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:33.259 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260506 10:17:33.259 | INFO | PRE_HQ : 56.7%, 2.1, 60.9ms, 0 | 20260506 10:17:33.259 | INFO | BAZWRITER : 65.6%, 4.8, 161.6ms, 0 | 20260506 10:17:33.259 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 10:17:33.260 | INFO | Uploaded 36GiB to GPU in 1.67171s (21.5348GiB/s) | 20260506 10:17:33.260 | INFO | Downloaded 38GiB to GPU in 1.73908s (21.8507GiB/s) | 20260506 10:17:38.398 | INFO | Analyzing chunk frames = [573952,574464) | 20260506 10:17:38.398 | INFO | MemInfo: { | 20260506 10:17:38.398 | INFO | "gpu" : " 17 GB", | 20260506 10:17:38.398 | INFO | "huge" : "", | 20260506 10:17:38.398 | INFO | "huge_shared" : " 27 GB", | 20260506 10:17:38.399 | INFO | "malloc" : " 14 GB", | 20260506 10:17:38.399 | INFO | "pinned" : " 29 GB" | 20260506 10:17:38.399 | INFO | } | 20260506 10:17:38.399 | INFO | Perf report for the last 5.13963 seconds: | 20260506 10:17:38.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:17:38.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:38.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:38.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:38.399 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 10:17:38.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:38.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:38.399 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:17:38.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:38.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:38.399 | INFO | Uploaded 36GiB to GPU in 1.60225s (22.4684GiB/s) | 20260506 10:17:38.399 | INFO | Downloaded 34GiB to GPU in 1.51945s (22.3765GiB/s) | 20260506 10:17:43.538 | INFO | Analyzing chunk frames = [574464,574976) | 20260506 10:17:43.538 | INFO | MemInfo: { | 20260506 10:17:43.538 | INFO | "gpu" : " 17 GB", | 20260506 10:17:43.538 | INFO | "huge" : "", | 20260506 10:17:43.538 | INFO | "huge_shared" : " 27 GB", | 20260506 10:17:43.538 | INFO | "malloc" : " 14 GB", | 20260506 10:17:43.538 | INFO | "pinned" : " 29 GB" | 20260506 10:17:43.538 | INFO | } | 20260506 10:17:43.538 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:17:43.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:17:43.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:43.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:43.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:43.538 | INFO | ANALYSIS : 56.7%, 3.0, 88.2ms, 0 | 20260506 10:17:43.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:43.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:43.538 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:17:43.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:43.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:43.538 | INFO | Uploaded 36GiB to GPU in 1.60678s (22.4051GiB/s) | 20260506 10:17:43.538 | INFO | Downloaded 34GiB to GPU in 1.52492s (22.2963GiB/s) | 20260506 10:17:48.677 | INFO | Analyzing chunk frames = [574976,575488) | 20260506 10:17:48.677 | INFO | MemInfo: { | 20260506 10:17:48.677 | INFO | "gpu" : " 17 GB", | 20260506 10:17:48.677 | INFO | "huge" : "", | 20260506 10:17:48.677 | INFO | "huge_shared" : " 27 GB", | 20260506 10:17:48.677 | INFO | "malloc" : " 14 GB", | 20260506 10:17:48.677 | INFO | "pinned" : " 29 GB" | 20260506 10:17:48.677 | INFO | } | 20260506 10:17:48.677 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:17:48.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:17:48.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:48.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:48.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:48.677 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:17:48.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:48.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:48.678 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:17:48.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:48.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:48.678 | INFO | Uploaded 36GiB to GPU in 1.60741s (22.3962GiB/s) | 20260506 10:17:48.678 | INFO | Downloaded 34GiB to GPU in 1.5251s (22.2937GiB/s) | 20260506 10:17:53.816 | INFO | Analyzing chunk frames = [575488,576000) | 20260506 10:17:53.816 | INFO | MemInfo: { | 20260506 10:17:53.816 | INFO | "gpu" : " 17 GB", | 20260506 10:17:53.816 | INFO | "huge" : "", | 20260506 10:17:53.817 | INFO | "huge_shared" : " 27 GB", | 20260506 10:17:53.817 | INFO | "malloc" : " 14 GB", | 20260506 10:17:53.817 | INFO | "pinned" : " 29 GB" | 20260506 10:17:53.817 | INFO | } | 20260506 10:17:53.817 | INFO | Perf report for the last 5.1393 seconds: | 20260506 10:17:53.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:17:53.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:53.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:53.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:53.817 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260506 10:17:53.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:53.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:53.817 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:17:53.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:53.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:53.817 | INFO | Uploaded 36GiB to GPU in 1.60383s (22.4463GiB/s) | 20260506 10:17:53.817 | INFO | Downloaded 34GiB to GPU in 1.52156s (22.3455GiB/s) | 20260506 10:17:58.956 | INFO | Analyzing chunk frames = [576000,576512) | 20260506 10:17:58.956 | INFO | MemInfo: { | 20260506 10:17:58.956 | INFO | "gpu" : " 17 GB", | 20260506 10:17:58.956 | INFO | "huge" : "", | 20260506 10:17:58.956 | INFO | "huge_shared" : " 27 GB", | 20260506 10:17:58.956 | INFO | "malloc" : " 14 GB", | 20260506 10:17:58.956 | INFO | "pinned" : " 29 GB" | 20260506 10:17:58.956 | INFO | } | 20260506 10:17:58.956 | INFO | Perf report for the last 5.13948 seconds: | 20260506 10:17:58.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:17:58.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:58.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:58.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:58.956 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:17:58.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:58.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:58.956 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:17:58.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:58.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:17:58.956 | INFO | Uploaded 36GiB to GPU in 1.60439s (22.4385GiB/s) | 20260506 10:17:58.957 | INFO | Downloaded 34GiB to GPU in 1.5221s (22.3376GiB/s) | 20260506 10:18:04.095 | INFO | Analyzing chunk frames = [576512,577024) | 20260506 10:18:04.095 | INFO | MemInfo: { | 20260506 10:18:04.095 | INFO | "gpu" : " 17 GB", | 20260506 10:18:04.095 | INFO | "huge" : "", | 20260506 10:18:04.095 | INFO | "huge_shared" : " 27 GB", | 20260506 10:18:04.095 | INFO | "malloc" : " 14 GB", | 20260506 10:18:04.095 | INFO | "pinned" : " 29 GB" | 20260506 10:18:04.095 | INFO | } | 20260506 10:18:04.095 | INFO | Perf report for the last 5.13912 seconds: | 20260506 10:18:04.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:18:04.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:04.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:04.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:04.095 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:18:04.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:04.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:04.095 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:18:04.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:04.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:04.096 | INFO | Uploaded 36GiB to GPU in 1.60565s (22.4208GiB/s) | 20260506 10:18:04.096 | INFO | Downloaded 34GiB to GPU in 1.52399s (22.3099GiB/s) | 20260506 10:18:09.234 | INFO | Analyzing chunk frames = [577024,577536) | 20260506 10:18:09.234 | INFO | MemInfo: { | 20260506 10:18:09.234 | INFO | "gpu" : " 17 GB", | 20260506 10:18:09.234 | INFO | "huge" : "", | 20260506 10:18:09.235 | INFO | "huge_shared" : " 27 GB", | 20260506 10:18:09.235 | INFO | "malloc" : " 14 GB", | 20260506 10:18:09.235 | INFO | "pinned" : " 29 GB" | 20260506 10:18:09.235 | INFO | } | 20260506 10:18:09.235 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:18:09.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:18:09.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:09.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:09.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:09.235 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:18:09.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:09.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:09.235 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 10:18:09.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:09.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:09.235 | INFO | Uploaded 36GiB to GPU in 1.6054s (22.4243GiB/s) | 20260506 10:18:09.235 | INFO | Downloaded 34GiB to GPU in 1.52341s (22.3183GiB/s) | 20260506 10:18:09.334 | INFO | Starting a round of Baz IO | 20260506 10:18:12.243 | INFO | Populating RTMetrics report | 20260506 10:18:12.243 | INFO | RTMetrics report populated | 20260506 10:18:12.243 | INFO | Opening tmp RTMetrics json file | 20260506 10:18:12.269 | INFO | Serializing RTMetrics to json | 20260506 10:18:12.271 | INFO | Serialization done, swapping tmp file now... | 20260506 10:18:12.272 | INFO | RTMetrics done | 20260506 10:18:12.769 | INFO | Finished a round of Baz IO: Wrote 1.969 GiB in 3.434 seconds | 20260506 10:18:12.769 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.60% / 0.06% / 51.34% | 20260506 10:18:12.769 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.15118% / 0.000182788% / 4.76837e-05% | 20260506 10:18:12.769 | INFO | Baz IO overall: | 20260506 10:18:12.769 | INFO | BytesWritten : 256002046751 | 20260506 10:18:12.769 | INFO | NumEvents : 122876846073 | 20260506 10:18:14.374 | INFO | Analyzing chunk frames = [577536,578048) | 20260506 10:18:14.374 | INFO | MemInfo: { | 20260506 10:18:14.374 | INFO | "gpu" : " 17 GB", | 20260506 10:18:14.374 | INFO | "huge" : "", | 20260506 10:18:14.374 | INFO | "huge_shared" : " 27 GB", | 20260506 10:18:14.374 | INFO | "malloc" : " 14 GB", | 20260506 10:18:14.374 | INFO | "pinned" : " 29 GB" | 20260506 10:18:14.374 | INFO | } | 20260506 10:18:14.374 | INFO | Perf report for the last 5.13963 seconds: | 20260506 10:18:14.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:18:14.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:14.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:14.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:14.374 | INFO | ANALYSIS : 57.5%, 2.8, 85.6ms, 0 | 20260506 10:18:14.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:14.374 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 10:18:14.375 | INFO | PRE_HQ : 56.5%, 2.1, 60.5ms, 0 | 20260506 10:18:14.375 | INFO | BAZWRITER : 66.4%, 4.7, 161.7ms, 0 | 20260506 10:18:14.375 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 10:18:14.375 | INFO | Uploaded 36GiB to GPU in 1.67823s (21.4512GiB/s) | 20260506 10:18:14.375 | INFO | Downloaded 38GiB to GPU in 1.74292s (21.8025GiB/s) | 20260506 10:18:19.513 | INFO | Analyzing chunk frames = [578048,578560) | 20260506 10:18:19.513 | INFO | MemInfo: { | 20260506 10:18:19.513 | INFO | "gpu" : " 17 GB", | 20260506 10:18:19.513 | INFO | "huge" : "", | 20260506 10:18:19.513 | INFO | "huge_shared" : " 27 GB", | 20260506 10:18:19.513 | INFO | "malloc" : " 14 GB", | 20260506 10:18:19.513 | INFO | "pinned" : " 29 GB" | 20260506 10:18:19.513 | INFO | } | 20260506 10:18:19.513 | INFO | Perf report for the last 5.13881 seconds: | 20260506 10:18:19.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:18:19.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:19.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:19.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:19.513 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:18:19.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:19.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:19.513 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:18:19.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:19.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:19.514 | INFO | Uploaded 36GiB to GPU in 1.60291s (22.4592GiB/s) | 20260506 10:18:19.514 | INFO | Downloaded 34GiB to GPU in 1.52064s (22.3589GiB/s) | 20260506 10:18:24.652 | INFO | Analyzing chunk frames = [578560,579072) | 20260506 10:18:24.652 | INFO | MemInfo: { | 20260506 10:18:24.652 | INFO | "gpu" : " 17 GB", | 20260506 10:18:24.652 | INFO | "huge" : "", | 20260506 10:18:24.652 | INFO | "huge_shared" : " 27 GB", | 20260506 10:18:24.653 | INFO | "malloc" : " 14 GB", | 20260506 10:18:24.653 | INFO | "pinned" : " 29 GB" | 20260506 10:18:24.653 | INFO | } | 20260506 10:18:24.653 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:18:24.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:18:24.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:24.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:24.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:24.653 | INFO | ANALYSIS : 56.7%, 3.0, 88.2ms, 0 | 20260506 10:18:24.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:24.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:24.653 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:18:24.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:24.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:24.653 | INFO | Uploaded 36GiB to GPU in 1.60702s (22.4017GiB/s) | 20260506 10:18:24.653 | INFO | Downloaded 34GiB to GPU in 1.52524s (22.2916GiB/s) | 20260506 10:18:29.792 | INFO | Analyzing chunk frames = [579072,579584) | 20260506 10:18:29.792 | INFO | MemInfo: { | 20260506 10:18:29.792 | INFO | "gpu" : " 17 GB", | 20260506 10:18:29.792 | INFO | "huge" : "", | 20260506 10:18:29.792 | INFO | "huge_shared" : " 27 GB", | 20260506 10:18:29.792 | INFO | "malloc" : " 14 GB", | 20260506 10:18:29.792 | INFO | "pinned" : " 29 GB" | 20260506 10:18:29.792 | INFO | } | 20260506 10:18:29.792 | INFO | Perf report for the last 5.13942 seconds: | 20260506 10:18:29.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:18:29.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:29.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:29.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:29.792 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:18:29.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:29.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:29.792 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:18:29.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:29.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:29.792 | INFO | Uploaded 36GiB to GPU in 1.60639s (22.4105GiB/s) | 20260506 10:18:29.792 | INFO | Downloaded 34GiB to GPU in 1.52403s (22.3093GiB/s) | 20260506 10:18:34.931 | INFO | Analyzing chunk frames = [579584,580096) | 20260506 10:18:34.931 | INFO | MemInfo: { | 20260506 10:18:34.931 | INFO | "gpu" : " 17 GB", | 20260506 10:18:34.931 | INFO | "huge" : "", | 20260506 10:18:34.931 | INFO | "huge_shared" : " 27 GB", | 20260506 10:18:34.931 | INFO | "malloc" : " 14 GB", | 20260506 10:18:34.931 | INFO | "pinned" : " 29 GB" | 20260506 10:18:34.931 | INFO | } | 20260506 10:18:34.932 | INFO | Perf report for the last 5.13944 seconds: | 20260506 10:18:34.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:18:34.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:34.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:34.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:34.932 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260506 10:18:34.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:34.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:34.932 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:18:34.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:34.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:34.932 | INFO | Uploaded 36GiB to GPU in 1.6048s (22.4327GiB/s) | 20260506 10:18:34.932 | INFO | Downloaded 34GiB to GPU in 1.52164s (22.3442GiB/s) | 20260506 10:18:40.070 | INFO | Analyzing chunk frames = [580096,580608) | 20260506 10:18:40.070 | INFO | MemInfo: { | 20260506 10:18:40.070 | INFO | "gpu" : " 17 GB", | 20260506 10:18:40.070 | INFO | "huge" : "", | 20260506 10:18:40.070 | INFO | "huge_shared" : " 27 GB", | 20260506 10:18:40.070 | INFO | "malloc" : " 14 GB", | 20260506 10:18:40.071 | INFO | "pinned" : " 29 GB" | 20260506 10:18:40.071 | INFO | } | 20260506 10:18:40.071 | INFO | Perf report for the last 5.13917 seconds: | 20260506 10:18:40.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:18:40.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:40.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:40.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:40.071 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:18:40.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:40.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:40.071 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:18:40.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:40.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:40.071 | INFO | Uploaded 36GiB to GPU in 1.60494s (22.4308GiB/s) | 20260506 10:18:40.071 | INFO | Downloaded 34GiB to GPU in 1.52199s (22.3392GiB/s) | 20260506 10:18:45.210 | INFO | Analyzing chunk frames = [580608,581120) | 20260506 10:18:45.210 | INFO | MemInfo: { | 20260506 10:18:45.210 | INFO | "gpu" : " 17 GB", | 20260506 10:18:45.210 | INFO | "huge" : "", | 20260506 10:18:45.210 | INFO | "huge_shared" : " 27 GB", | 20260506 10:18:45.210 | INFO | "malloc" : " 14 GB", | 20260506 10:18:45.210 | INFO | "pinned" : " 29 GB" | 20260506 10:18:45.210 | INFO | } | 20260506 10:18:45.210 | INFO | Perf report for the last 5.13927 seconds: | 20260506 10:18:45.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:18:45.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:45.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:45.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:45.210 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:18:45.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:45.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:45.210 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:18:45.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:45.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:45.210 | INFO | Uploaded 36GiB to GPU in 1.60539s (22.4245GiB/s) | 20260506 10:18:45.210 | INFO | Downloaded 34GiB to GPU in 1.52334s (22.3194GiB/s) | 20260506 10:18:50.349 | INFO | Analyzing chunk frames = [581120,581632) | 20260506 10:18:50.349 | INFO | MemInfo: { | 20260506 10:18:50.349 | INFO | "gpu" : " 17 GB", | 20260506 10:18:50.349 | INFO | "huge" : "", | 20260506 10:18:50.349 | INFO | "huge_shared" : " 27 GB", | 20260506 10:18:50.349 | INFO | "malloc" : " 14 GB", | 20260506 10:18:50.349 | INFO | "pinned" : " 29 GB" | 20260506 10:18:50.349 | INFO | } | 20260506 10:18:50.349 | INFO | Perf report for the last 5.13951 seconds: | 20260506 10:18:50.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:18:50.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:50.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:50.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:50.350 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:18:50.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:50.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:50.350 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 10:18:50.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:50.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:50.350 | INFO | Uploaded 36GiB to GPU in 1.605s (22.4299GiB/s) | 20260506 10:18:50.350 | INFO | Downloaded 34GiB to GPU in 1.52319s (22.3215GiB/s) | 20260506 10:18:50.350 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:18:50.350 | INFO | Total mem usage: 16.5 GiB | 20260506 10:18:50.350 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:18:50.350 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -4.11%) | 20260506 10:18:50.350 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:18:50.350 | INFO | High Water Breakdown: | 20260506 10:18:50.350 | INFO | 60.5% : 9.6 GiB ( 95.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:18:50.350 | INFO | 39.3% : 6.24 GiB ( 93.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:18:50.350 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:18:50.350 | INFO | ------------------------------------------ | 20260506 10:18:50.350 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:18:50.350 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:18:50.350 | INFO | Total mem usage: 26.5 GiB | 20260506 10:18:50.350 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:18:50.350 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:18:50.350 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:18:50.350 | INFO | High Water Breakdown: | 20260506 10:18:50.350 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:18:50.350 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:18:50.350 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:18:50.350 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:18:50.350 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:18:50.350 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:18:50.350 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:18:50.350 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:18:50.350 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:18:50.350 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:18:50.350 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:18:50.350 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:18:50.350 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:18:50.350 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:18:50.350 | INFO | ------------------------------------------ | 20260506 10:18:50.350 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:18:50.350 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:18:50.350 | INFO | Total mem usage: 25.6 GiB | 20260506 10:18:50.350 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:18:50.350 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:18:50.350 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:18:50.350 | INFO | High Water Breakdown: | 20260506 10:18:50.350 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:18:50.351 | INFO | ------------------------------------------ | 20260506 10:18:50.351 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:18:50.351 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:18:50.351 | INFO | Total mem usage: 15.7 GiB | 20260506 10:18:50.351 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:18:50.351 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:18:50.351 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:18:50.351 | INFO | High Water Breakdown: | 20260506 10:18:50.351 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:18:50.351 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:18:50.351 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:18:50.351 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:18:50.351 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:18:50.351 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:18:50.351 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:18:50.351 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:18:50.351 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:18:50.351 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:18:50.351 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:18:50.351 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:18:50.351 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:18:50.351 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:18:50.351 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:18:50.351 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:18:50.351 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:18:50.351 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:18:50.351 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:18:50.351 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:18:50.351 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:18:50.351 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:18:50.351 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:18:50.351 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:18:50.351 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:18:50.351 | INFO | ------------------------------------------ | 20260506 10:18:50.351 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:18:50.351 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:18:50.351 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:18:50.351 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:18:50.351 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 10:18:50.351 | INFO | PerfReport: 0.29%, AggregateStats -- 8.68e-02 / -nan, 6.96e-03 / 0.00e+00 | 20260506 10:18:50.351 | INFO | PerfReport: 10.94%, Baseline -- 3.29e+00 / -nan, 1.78e-02 / 0.00e+00 | 20260506 10:18:50.351 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.26e-02 / 0.00e+00 | 20260506 10:18:50.351 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.03e+00 / 0.00e+00 | 20260506 10:18:50.351 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.72e+01 / -nan, 4.31e-02 / 0.00e+00 | 20260506 10:18:50.351 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.23e+00 / -nan, 1.30e-01 / 0.00e+00 | 20260506 10:18:50.351 | INFO | PerfReport: 3.69%, Metrics -- 1.11e+00 / -nan, 1.73e-01 / 0.00e+00 | 20260506 10:18:50.351 | INFO | PerfReport: 1.43%, PulseSerializing -- 4.30e-01 / -nan, 6.07e-02 / 0.00e+00 | 20260506 10:18:50.351 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:18:50.351 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:18:50.351 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:18:50.351 | INFO | PerfReport: Full Execution -- 8.99e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 10:18:50.351 | INFO | PerfReport: 18.70%, Upload -- 1.68e+01 / 0.00e+00, 4.98e-01 / 0.00e+00 | 20260506 10:18:50.351 | INFO | PerfReport: 17.96%, Download -- 1.61e+01 / 0.00e+00, 8.54e-01 / 0.00e+00 | 20260506 10:18:50.351 | INFO | PerfReport: 33.50%, Compute -- 3.01e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 10:18:50.351 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 10:18:50.351 | INFO | PerfReport: 0.00%, LockDownload -- 4.95e-04 / 0.00e+00, 1.96e-04 / 0.00e+00 | 20260506 10:18:50.351 | INFO | PerfReport: 29.27%, LockCompute -- 2.63e+01 / 0.00e+00, 1.72e+01 / 0.00e+00 | 20260506 10:18:50.351 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260506 10:18:50.447 | INFO | Starting a round of Baz IO | 20260506 10:18:53.360 | INFO | Populating RTMetrics report | 20260506 10:18:53.360 | INFO | RTMetrics report populated | 20260506 10:18:53.360 | INFO | Opening tmp RTMetrics json file | 20260506 10:18:53.403 | INFO | Serializing RTMetrics to json | 20260506 10:18:53.407 | INFO | Serialization done, swapping tmp file now... | 20260506 10:18:53.411 | INFO | RTMetrics done | 20260506 10:18:53.546 | INFO | Finished a round of Baz IO: Wrote 1.97 GiB in 3.098 seconds | 20260506 10:18:53.546 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.55% / 0.06% / 51.40% | 20260506 10:18:53.546 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.12328% / 0.000143051% / 5.5631e-05% | 20260506 10:18:53.546 | INFO | Baz IO overall: | 20260506 10:18:53.546 | INFO | BytesWritten : 258117398417 | 20260506 10:18:53.546 | INFO | NumEvents : 123903327791 | 20260506 10:18:55.489 | INFO | Analyzing chunk frames = [581632,582144) | 20260506 10:18:55.489 | INFO | MemInfo: { | 20260506 10:18:55.489 | INFO | "gpu" : " 17 GB", | 20260506 10:18:55.489 | INFO | "huge" : "", | 20260506 10:18:55.489 | INFO | "huge_shared" : " 27 GB", | 20260506 10:18:55.489 | INFO | "malloc" : " 14 GB", | 20260506 10:18:55.489 | INFO | "pinned" : " 29 GB" | 20260506 10:18:55.489 | INFO | } | 20260506 10:18:55.489 | INFO | Perf report for the last 5.1391 seconds: | 20260506 10:18:55.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:18:55.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:55.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:55.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:55.489 | INFO | ANALYSIS : 57.6%, 2.8, 85.8ms, 0 | 20260506 10:18:55.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:18:55.489 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260506 10:18:55.489 | INFO | PRE_HQ : 56.6%, 2.1, 60.6ms, 0 | 20260506 10:18:55.489 | INFO | BAZWRITER : 59.9%, 4.7, 145.2ms, 0 | 20260506 10:18:55.489 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 10:18:55.489 | INFO | Uploaded 36GiB to GPU in 1.67202s (21.5309GiB/s) | 20260506 10:18:55.489 | INFO | Downloaded 38GiB to GPU in 1.73505s (21.9014GiB/s) | 20260506 10:19:00.628 | INFO | Analyzing chunk frames = [582144,582656) | 20260506 10:19:00.628 | INFO | MemInfo: { | 20260506 10:19:00.628 | INFO | "gpu" : " 17 GB", | 20260506 10:19:00.628 | INFO | "huge" : "", | 20260506 10:19:00.628 | INFO | "huge_shared" : " 27 GB", | 20260506 10:19:00.628 | INFO | "malloc" : " 14 GB", | 20260506 10:19:00.628 | INFO | "pinned" : " 29 GB" | 20260506 10:19:00.628 | INFO | } | 20260506 10:19:00.628 | INFO | Perf report for the last 5.1396 seconds: | 20260506 10:19:00.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:19:00.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:00.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:00.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:00.628 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:19:00.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:00.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:00.628 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260506 10:19:00.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:00.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:00.629 | INFO | Uploaded 36GiB to GPU in 1.60303s (22.4575GiB/s) | 20260506 10:19:00.629 | INFO | Downloaded 34GiB to GPU in 1.52005s (22.3678GiB/s) | 20260506 10:19:05.767 | INFO | Analyzing chunk frames = [582656,583168) | 20260506 10:19:05.767 | INFO | MemInfo: { | 20260506 10:19:05.767 | INFO | "gpu" : " 17 GB", | 20260506 10:19:05.767 | INFO | "huge" : "", | 20260506 10:19:05.767 | INFO | "huge_shared" : " 27 GB", | 20260506 10:19:05.767 | INFO | "malloc" : " 14 GB", | 20260506 10:19:05.767 | INFO | "pinned" : " 29 GB" | 20260506 10:19:05.767 | INFO | } | 20260506 10:19:05.767 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:19:05.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:19:05.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:05.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:05.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:05.768 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 10:19:05.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:05.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:05.768 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:19:05.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:05.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:05.768 | INFO | Uploaded 36GiB to GPU in 1.60714s (22.4001GiB/s) | 20260506 10:19:05.768 | INFO | Downloaded 34GiB to GPU in 1.52498s (22.2954GiB/s) | 20260506 10:19:10.907 | INFO | Analyzing chunk frames = [583168,583680) | 20260506 10:19:10.907 | INFO | MemInfo: { | 20260506 10:19:10.907 | INFO | "gpu" : " 17 GB", | 20260506 10:19:10.907 | INFO | "huge" : "", | 20260506 10:19:10.907 | INFO | "huge_shared" : " 27 GB", | 20260506 10:19:10.907 | INFO | "malloc" : " 14 GB", | 20260506 10:19:10.907 | INFO | "pinned" : " 29 GB" | 20260506 10:19:10.907 | INFO | } | 20260506 10:19:10.907 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:19:10.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:19:10.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:10.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:10.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:10.907 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:19:10.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:10.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:10.907 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:19:10.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:10.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:10.907 | INFO | Uploaded 36GiB to GPU in 1.60634s (22.4111GiB/s) | 20260506 10:19:10.907 | INFO | Downloaded 34GiB to GPU in 1.52398s (22.31GiB/s) | 20260506 10:19:16.046 | INFO | Analyzing chunk frames = [583680,584192) | 20260506 10:19:16.046 | INFO | MemInfo: { | 20260506 10:19:16.046 | INFO | "gpu" : " 17 GB", | 20260506 10:19:16.046 | INFO | "huge" : "", | 20260506 10:19:16.046 | INFO | "huge_shared" : " 27 GB", | 20260506 10:19:16.046 | INFO | "malloc" : " 14 GB", | 20260506 10:19:16.046 | INFO | "pinned" : " 29 GB" | 20260506 10:19:16.046 | INFO | } | 20260506 10:19:16.046 | INFO | Perf report for the last 5.1392 seconds: | 20260506 10:19:16.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:19:16.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:16.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:16.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:16.046 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260506 10:19:16.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:16.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:16.046 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:19:16.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:16.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:16.046 | INFO | Uploaded 36GiB to GPU in 1.60468s (22.4344GiB/s) | 20260506 10:19:16.046 | INFO | Downloaded 34GiB to GPU in 1.52141s (22.3477GiB/s) | 20260506 10:19:21.185 | INFO | Analyzing chunk frames = [584192,584704) | 20260506 10:19:21.185 | INFO | MemInfo: { | 20260506 10:19:21.185 | INFO | "gpu" : " 17 GB", | 20260506 10:19:21.185 | INFO | "huge" : "", | 20260506 10:19:21.185 | INFO | "huge_shared" : " 27 GB", | 20260506 10:19:21.185 | INFO | "malloc" : " 14 GB", | 20260506 10:19:21.185 | INFO | "pinned" : " 29 GB" | 20260506 10:19:21.185 | INFO | } | 20260506 10:19:21.185 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:19:21.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:19:21.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:21.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:21.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:21.185 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:19:21.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:21.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:21.185 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:19:21.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:21.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:21.186 | INFO | Uploaded 36GiB to GPU in 1.60547s (22.4234GiB/s) | 20260506 10:19:21.186 | INFO | Downloaded 34GiB to GPU in 1.52294s (22.3252GiB/s) | 20260506 10:19:26.325 | INFO | Analyzing chunk frames = [584704,585216) | 20260506 10:19:26.325 | INFO | MemInfo: { | 20260506 10:19:26.325 | INFO | "gpu" : " 17 GB", | 20260506 10:19:26.325 | INFO | "huge" : "", | 20260506 10:19:26.325 | INFO | "huge_shared" : " 27 GB", | 20260506 10:19:26.325 | INFO | "malloc" : " 14 GB", | 20260506 10:19:26.325 | INFO | "pinned" : " 29 GB" | 20260506 10:19:26.325 | INFO | } | 20260506 10:19:26.325 | INFO | Perf report for the last 5.13939 seconds: | 20260506 10:19:26.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:19:26.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:26.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:26.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:26.325 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:19:26.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:26.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:26.325 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:19:26.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:26.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:26.325 | INFO | Uploaded 36GiB to GPU in 1.60525s (22.4265GiB/s) | 20260506 10:19:26.326 | INFO | Downloaded 34GiB to GPU in 1.52253s (22.3312GiB/s) | 20260506 10:19:31.464 | INFO | Analyzing chunk frames = [585216,585728) | 20260506 10:19:31.464 | INFO | MemInfo: { | 20260506 10:19:31.464 | INFO | "gpu" : " 17 GB", | 20260506 10:19:31.464 | INFO | "huge" : "", | 20260506 10:19:31.464 | INFO | "huge_shared" : " 27 GB", | 20260506 10:19:31.464 | INFO | "malloc" : " 14 GB", | 20260506 10:19:31.464 | INFO | "pinned" : " 29 GB" | 20260506 10:19:31.464 | INFO | } | 20260506 10:19:31.464 | INFO | Perf report for the last 5.13947 seconds: | 20260506 10:19:31.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:19:31.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:31.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:31.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:31.464 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:19:31.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:31.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:31.464 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 10:19:31.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:31.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:31.465 | INFO | Uploaded 36GiB to GPU in 1.60611s (22.4144GiB/s) | 20260506 10:19:31.465 | INFO | Downloaded 34GiB to GPU in 1.5238s (22.3127GiB/s) | 20260506 10:19:31.561 | INFO | Starting a round of Baz IO | 20260506 10:19:34.474 | INFO | Populating RTMetrics report | 20260506 10:19:34.475 | INFO | RTMetrics report populated | 20260506 10:19:34.475 | INFO | Opening tmp RTMetrics json file | 20260506 10:19:34.488 | INFO | Serializing RTMetrics to json | 20260506 10:19:34.517 | INFO | Serialization done, swapping tmp file now... | 20260506 10:19:34.529 | INFO | RTMetrics done | 20260506 10:19:34.867 | INFO | Finished a round of Baz IO: Wrote 1.972 GiB in 3.306 seconds | 20260506 10:19:34.868 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.49% / 0.06% / 51.45% | 20260506 10:19:34.868 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.10611% / 0.000147025% / 4.37101e-05% | 20260506 10:19:34.868 | INFO | Baz IO overall: | 20260506 10:19:34.868 | INFO | BytesWritten : 260234966669 | 20260506 10:19:34.868 | INFO | NumEvents : 124930816669 | 20260506 10:19:36.603 | INFO | Analyzing chunk frames = [585728,586240) | 20260506 10:19:36.603 | INFO | MemInfo: { | 20260506 10:19:36.603 | INFO | "gpu" : " 17 GB", | 20260506 10:19:36.603 | INFO | "huge" : "", | 20260506 10:19:36.603 | INFO | "huge_shared" : " 27 GB", | 20260506 10:19:36.603 | INFO | "malloc" : " 14 GB", | 20260506 10:19:36.603 | INFO | "pinned" : " 29 GB" | 20260506 10:19:36.603 | INFO | } | 20260506 10:19:36.604 | INFO | Perf report for the last 5.13927 seconds: | 20260506 10:19:36.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:19:36.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:36.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:36.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:36.604 | INFO | ANALYSIS : 57.6%, 2.8, 85.4ms, 0 | 20260506 10:19:36.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:36.604 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 10:19:36.604 | INFO | PRE_HQ : 56.6%, 2.1, 60.6ms, 0 | 20260506 10:19:36.604 | INFO | BAZWRITER : 63.9%, 4.8, 157.3ms, 0 | 20260506 10:19:36.604 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 10:19:36.604 | INFO | Uploaded 36GiB to GPU in 1.67946s (21.4354GiB/s) | 20260506 10:19:36.604 | INFO | Downloaded 38GiB to GPU in 1.74467s (21.7806GiB/s) | 20260506 10:19:41.743 | INFO | Analyzing chunk frames = [586240,586752) | 20260506 10:19:41.743 | INFO | MemInfo: { | 20260506 10:19:41.743 | INFO | "gpu" : " 17 GB", | 20260506 10:19:41.743 | INFO | "huge" : "", | 20260506 10:19:41.743 | INFO | "huge_shared" : " 27 GB", | 20260506 10:19:41.743 | INFO | "malloc" : " 14 GB", | 20260506 10:19:41.743 | INFO | "pinned" : " 29 GB" | 20260506 10:19:41.743 | INFO | } | 20260506 10:19:41.743 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:19:41.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:19:41.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:41.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:41.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:41.743 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:19:41.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:41.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:41.743 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:19:41.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:41.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:41.744 | INFO | Uploaded 36GiB to GPU in 1.60334s (22.4531GiB/s) | 20260506 10:19:41.744 | INFO | Downloaded 34GiB to GPU in 1.5212s (22.3507GiB/s) | 20260506 10:19:46.882 | INFO | Analyzing chunk frames = [586752,587264) | 20260506 10:19:46.882 | INFO | MemInfo: { | 20260506 10:19:46.882 | INFO | "gpu" : " 17 GB", | 20260506 10:19:46.882 | INFO | "huge" : "", | 20260506 10:19:46.882 | INFO | "huge_shared" : " 27 GB", | 20260506 10:19:46.882 | INFO | "malloc" : " 14 GB", | 20260506 10:19:46.882 | INFO | "pinned" : " 29 GB" | 20260506 10:19:46.882 | INFO | } | 20260506 10:19:46.882 | INFO | Perf report for the last 5.13919 seconds: | 20260506 10:19:46.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:19:46.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:46.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:46.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:46.882 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 10:19:46.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:46.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:46.882 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:19:46.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:46.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:46.882 | INFO | Uploaded 36GiB to GPU in 1.60718s (22.3995GiB/s) | 20260506 10:19:46.882 | INFO | Downloaded 34GiB to GPU in 1.5252s (22.2921GiB/s) | 20260506 10:19:52.021 | INFO | Analyzing chunk frames = [587264,587776) | 20260506 10:19:52.021 | INFO | MemInfo: { | 20260506 10:19:52.021 | INFO | "gpu" : " 17 GB", | 20260506 10:19:52.021 | INFO | "huge" : "", | 20260506 10:19:52.021 | INFO | "huge_shared" : " 27 GB", | 20260506 10:19:52.021 | INFO | "malloc" : " 14 GB", | 20260506 10:19:52.021 | INFO | "pinned" : " 29 GB" | 20260506 10:19:52.021 | INFO | } | 20260506 10:19:52.021 | INFO | Perf report for the last 5.13912 seconds: | 20260506 10:19:52.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:19:52.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:52.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:52.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:52.021 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:19:52.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:52.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:52.021 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:19:52.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:52.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:52.022 | INFO | Uploaded 36GiB to GPU in 1.60678s (22.405GiB/s) | 20260506 10:19:52.022 | INFO | Downloaded 34GiB to GPU in 1.52436s (22.3044GiB/s) | 20260506 10:19:57.160 | INFO | Analyzing chunk frames = [587776,588288) | 20260506 10:19:57.160 | INFO | MemInfo: { | 20260506 10:19:57.160 | INFO | "gpu" : " 17 GB", | 20260506 10:19:57.160 | INFO | "huge" : "", | 20260506 10:19:57.160 | INFO | "huge_shared" : " 27 GB", | 20260506 10:19:57.160 | INFO | "malloc" : " 14 GB", | 20260506 10:19:57.160 | INFO | "pinned" : " 29 GB" | 20260506 10:19:57.160 | INFO | } | 20260506 10:19:57.161 | INFO | Perf report for the last 5.13922 seconds: | 20260506 10:19:57.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:19:57.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:57.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:57.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:57.161 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 10:19:57.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:57.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:57.161 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 10:19:57.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:57.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:19:57.161 | INFO | Uploaded 36GiB to GPU in 1.60349s (22.451GiB/s) | 20260506 10:19:57.161 | INFO | Downloaded 34GiB to GPU in 1.52128s (22.3495GiB/s) | 20260506 10:20:02.300 | INFO | Analyzing chunk frames = [588288,588800) | 20260506 10:20:02.300 | INFO | MemInfo: { | 20260506 10:20:02.300 | INFO | "gpu" : " 17 GB", | 20260506 10:20:02.300 | INFO | "huge" : "", | 20260506 10:20:02.300 | INFO | "huge_shared" : " 27 GB", | 20260506 10:20:02.300 | INFO | "malloc" : " 14 GB", | 20260506 10:20:02.300 | INFO | "pinned" : " 29 GB" | 20260506 10:20:02.300 | INFO | } | 20260506 10:20:02.300 | INFO | Perf report for the last 5.13942 seconds: | 20260506 10:20:02.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:20:02.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:02.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:02.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:02.300 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:20:02.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:02.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:02.300 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:20:02.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:02.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:02.300 | INFO | Uploaded 36GiB to GPU in 1.60615s (22.4139GiB/s) | 20260506 10:20:02.300 | INFO | Downloaded 34GiB to GPU in 1.52375s (22.3134GiB/s) | 20260506 10:20:07.439 | INFO | Analyzing chunk frames = [588800,589312) | 20260506 10:20:07.439 | INFO | MemInfo: { | 20260506 10:20:07.439 | INFO | "gpu" : " 17 GB", | 20260506 10:20:07.439 | INFO | "huge" : "", | 20260506 10:20:07.439 | INFO | "huge_shared" : " 27 GB", | 20260506 10:20:07.439 | INFO | "malloc" : " 14 GB", | 20260506 10:20:07.439 | INFO | "pinned" : " 29 GB" | 20260506 10:20:07.439 | INFO | } | 20260506 10:20:07.439 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:20:07.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:20:07.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:07.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:07.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:07.439 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:20:07.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:07.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:07.439 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 10:20:07.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:07.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:07.440 | INFO | Uploaded 36GiB to GPU in 1.60436s (22.4389GiB/s) | 20260506 10:20:07.440 | INFO | Downloaded 34GiB to GPU in 1.52258s (22.3305GiB/s) | 20260506 10:20:12.578 | INFO | Analyzing chunk frames = [589312,589824) | 20260506 10:20:12.578 | INFO | MemInfo: { | 20260506 10:20:12.578 | INFO | "gpu" : " 17 GB", | 20260506 10:20:12.578 | INFO | "huge" : "", | 20260506 10:20:12.578 | INFO | "huge_shared" : " 27 GB", | 20260506 10:20:12.578 | INFO | "malloc" : " 14 GB", | 20260506 10:20:12.579 | INFO | "pinned" : " 29 GB" | 20260506 10:20:12.579 | INFO | } | 20260506 10:20:12.579 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:20:12.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:20:12.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:12.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:12.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:12.579 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:20:12.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:12.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:12.579 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:20:12.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:12.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:12.579 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:20:12.579 | INFO | Total mem usage: 16.5 GiB | 20260506 10:20:12.579 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:20:12.579 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -6.22%) | 20260506 10:20:12.579 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:20:12.579 | INFO | High Water Breakdown: | 20260506 10:20:12.579 | INFO | 60.2% : 9.36 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:20:12.579 | INFO | 39.7% : 6.17 GiB ( 92.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:20:12.579 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:20:12.579 | INFO | ------------------------------------------ | 20260506 10:20:12.579 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:20:12.579 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:20:12.579 | INFO | Total mem usage: 26.5 GiB | 20260506 10:20:12.579 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:20:12.579 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:20:12.579 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:20:12.579 | INFO | High Water Breakdown: | 20260506 10:20:12.579 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:20:12.579 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:20:12.579 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:20:12.579 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:20:12.579 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:20:12.579 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:20:12.579 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:20:12.579 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:20:12.579 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:20:12.579 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:20:12.579 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:20:12.579 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:20:12.579 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:20:12.579 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:20:12.579 | INFO | ------------------------------------------ | 20260506 10:20:12.579 | INFO | Uploaded 36GiB to GPU in 1.60596s (22.4166GiB/s) | 20260506 10:20:12.579 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:20:12.579 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:20:12.579 | INFO | Total mem usage: 25.6 GiB | 20260506 10:20:12.579 | INFO | Downloaded 34GiB to GPU in 1.52349s (22.3172GiB/s) | 20260506 10:20:12.579 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:20:12.579 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:20:12.579 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:20:12.579 | INFO | High Water Breakdown: | 20260506 10:20:12.579 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:20:12.579 | INFO | ------------------------------------------ | 20260506 10:20:12.579 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:20:12.580 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:20:12.580 | INFO | Total mem usage: 15.7 GiB | 20260506 10:20:12.580 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:20:12.580 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:20:12.580 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:20:12.580 | INFO | High Water Breakdown: | 20260506 10:20:12.580 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:20:12.580 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:20:12.580 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:20:12.580 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:20:12.580 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:20:12.580 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:20:12.580 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:20:12.580 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:20:12.580 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:20:12.580 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:20:12.580 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:20:12.580 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:20:12.580 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:20:12.580 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:20:12.580 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:20:12.580 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:20:12.580 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:20:12.580 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:20:12.580 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:20:12.580 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:20:12.580 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:20:12.580 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:20:12.580 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:20:12.580 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:20:12.580 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:20:12.580 | INFO | ------------------------------------------ | 20260506 10:20:12.580 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:20:12.580 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:20:12.580 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:20:12.580 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:20:12.580 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 10:20:12.580 | INFO | PerfReport: 0.29%, AggregateStats -- 8.64e-02 / -nan, 6.06e-03 / 0.00e+00 | 20260506 10:20:12.580 | INFO | PerfReport: 10.94%, Baseline -- 3.29e+00 / -nan, 1.71e-02 / 0.00e+00 | 20260506 10:20:12.580 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260506 10:20:12.580 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.08e+00 / 0.00e+00 | 20260506 10:20:12.580 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.72e+01 / -nan, 4.17e-02 / 0.00e+00 | 20260506 10:20:12.580 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.23e+00 / -nan, 1.29e-01 / 0.00e+00 | 20260506 10:20:12.580 | INFO | PerfReport: 3.69%, Metrics -- 1.11e+00 / -nan, 1.72e-01 / 0.00e+00 | 20260506 10:20:12.580 | INFO | PerfReport: 1.43%, PulseSerializing -- 4.30e-01 / -nan, 6.13e-02 / 0.00e+00 | 20260506 10:20:12.580 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:20:12.580 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:20:12.580 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:20:12.580 | INFO | PerfReport: Full Execution -- 8.99e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:20:12.580 | INFO | PerfReport: 18.70%, Upload -- 1.68e+01 / 0.00e+00, 5.03e-01 / 0.00e+00 | 20260506 10:20:12.580 | INFO | PerfReport: 17.96%, Download -- 1.61e+01 / 0.00e+00, 8.51e-01 / 0.00e+00 | 20260506 10:20:12.580 | INFO | PerfReport: 33.49%, Compute -- 3.01e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 10:20:12.580 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 10:20:12.580 | INFO | PerfReport: 0.00%, LockDownload -- 4.93e-04 / 0.00e+00, 1.27e-04 / 0.00e+00 | 20260506 10:20:12.580 | INFO | PerfReport: 29.27%, LockCompute -- 2.63e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260506 10:20:12.580 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.05e-02 / 0.00e+00 | 20260506 10:20:12.676 | INFO | Starting a round of Baz IO | 20260506 10:20:15.590 | INFO | Populating RTMetrics report | 20260506 10:20:15.590 | INFO | RTMetrics report populated | 20260506 10:20:15.590 | INFO | Opening tmp RTMetrics json file | 20260506 10:20:15.704 | INFO | Serializing RTMetrics to json | 20260506 10:20:15.765 | INFO | Serialization done, swapping tmp file now... | 20260506 10:20:15.796 | INFO | RTMetrics done | 20260506 10:20:16.028 | INFO | Finished a round of Baz IO: Wrote 1.972 GiB in 3.351 seconds | 20260506 10:20:16.028 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.43% / 0.06% / 51.51% | 20260506 10:20:16.028 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.07595% / 0.000119209% / 3.57628e-05% | 20260506 10:20:16.028 | INFO | Baz IO overall: | 20260506 10:20:16.028 | INFO | BytesWritten : 262352751647 | 20260506 10:20:16.028 | INFO | NumEvents : 125958676461 | 20260506 10:20:17.718 | INFO | Analyzing chunk frames = [589824,590336) | 20260506 10:20:17.718 | INFO | MemInfo: { | 20260506 10:20:17.718 | INFO | "gpu" : " 17 GB", | 20260506 10:20:17.718 | INFO | "huge" : "", | 20260506 10:20:17.718 | INFO | "huge_shared" : " 27 GB", | 20260506 10:20:17.718 | INFO | "malloc" : " 14 GB", | 20260506 10:20:17.718 | INFO | "pinned" : " 29 GB" | 20260506 10:20:17.718 | INFO | } | 20260506 10:20:17.718 | INFO | Perf report for the last 5.13942 seconds: | 20260506 10:20:17.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:20:17.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:17.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:17.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:17.718 | INFO | ANALYSIS : 57.6%, 2.9, 86.7ms, 0 | 20260506 10:20:17.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:17.718 | INFO | RT_METRICS : 48.2%, 1.0, 25.3ms, 0 | 20260506 10:20:17.718 | INFO | PRE_HQ : 56.7%, 2.1, 61.2ms, 0 | 20260506 10:20:17.718 | INFO | BAZWRITER : 64.6%, 4.7, 155.4ms, 0 | 20260506 10:20:17.718 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 10:20:17.718 | INFO | Uploaded 36GiB to GPU in 1.66429s (21.6309GiB/s) | 20260506 10:20:17.718 | INFO | Downloaded 38GiB to GPU in 1.72806s (21.99GiB/s) | 20260506 10:20:22.857 | INFO | Analyzing chunk frames = [590336,590848) | 20260506 10:20:22.857 | INFO | MemInfo: { | 20260506 10:20:22.857 | INFO | "gpu" : " 17 GB", | 20260506 10:20:22.857 | INFO | "huge" : "", | 20260506 10:20:22.857 | INFO | "huge_shared" : " 27 GB", | 20260506 10:20:22.857 | INFO | "malloc" : " 14 GB", | 20260506 10:20:22.857 | INFO | "pinned" : " 29 GB" | 20260506 10:20:22.857 | INFO | } | 20260506 10:20:22.857 | INFO | Perf report for the last 5.13909 seconds: | 20260506 10:20:22.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:20:22.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:22.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:22.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:22.857 | INFO | ANALYSIS : 57.4%, 3.0, 89.4ms, 0 | 20260506 10:20:22.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:22.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:22.857 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:20:22.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:22.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:22.858 | INFO | Uploaded 36GiB to GPU in 1.60326s (22.4542GiB/s) | 20260506 10:20:22.858 | INFO | Downloaded 34GiB to GPU in 1.52066s (22.3588GiB/s) | 20260506 10:20:27.996 | INFO | Analyzing chunk frames = [590848,591360) | 20260506 10:20:27.996 | INFO | MemInfo: { | 20260506 10:20:27.996 | INFO | "gpu" : " 17 GB", | 20260506 10:20:27.996 | INFO | "huge" : "", | 20260506 10:20:27.996 | INFO | "huge_shared" : " 27 GB", | 20260506 10:20:27.996 | INFO | "malloc" : " 14 GB", | 20260506 10:20:27.997 | INFO | "pinned" : " 29 GB" | 20260506 10:20:27.997 | INFO | } | 20260506 10:20:27.997 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:20:27.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:20:27.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:27.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:27.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:27.997 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 10:20:27.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:27.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:27.997 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:20:27.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:27.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:27.997 | INFO | Uploaded 36GiB to GPU in 1.60702s (22.4017GiB/s) | 20260506 10:20:27.997 | INFO | Downloaded 34GiB to GPU in 1.52407s (22.3087GiB/s) | 20260506 10:20:33.136 | INFO | Analyzing chunk frames = [591360,591872) | 20260506 10:20:33.136 | INFO | MemInfo: { | 20260506 10:20:33.136 | INFO | "gpu" : " 17 GB", | 20260506 10:20:33.136 | INFO | "huge" : "", | 20260506 10:20:33.136 | INFO | "huge_shared" : " 27 GB", | 20260506 10:20:33.136 | INFO | "malloc" : " 14 GB", | 20260506 10:20:33.136 | INFO | "pinned" : " 29 GB" | 20260506 10:20:33.136 | INFO | } | 20260506 10:20:33.136 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:20:33.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:20:33.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:33.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:33.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:33.136 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:20:33.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:33.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:33.136 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:20:33.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:33.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:33.136 | INFO | Uploaded 36GiB to GPU in 1.60807s (22.3871GiB/s) | 20260506 10:20:33.136 | INFO | Downloaded 34GiB to GPU in 1.52558s (22.2866GiB/s) | 20260506 10:20:38.275 | INFO | Analyzing chunk frames = [591872,592384) | 20260506 10:20:38.275 | INFO | MemInfo: { | 20260506 10:20:38.275 | INFO | "gpu" : " 17 GB", | 20260506 10:20:38.275 | INFO | "huge" : "", | 20260506 10:20:38.275 | INFO | "huge_shared" : " 27 GB", | 20260506 10:20:38.275 | INFO | "malloc" : " 14 GB", | 20260506 10:20:38.275 | INFO | "pinned" : " 29 GB" | 20260506 10:20:38.275 | INFO | } | 20260506 10:20:38.275 | INFO | Perf report for the last 5.13948 seconds: | 20260506 10:20:38.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:20:38.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:38.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:38.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:38.275 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260506 10:20:38.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:38.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:38.275 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:20:38.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:38.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:38.276 | INFO | Uploaded 36GiB to GPU in 1.60484s (22.4321GiB/s) | 20260506 10:20:38.276 | INFO | Downloaded 34GiB to GPU in 1.52111s (22.352GiB/s) | 20260506 10:20:43.414 | INFO | Analyzing chunk frames = [592384,592896) | 20260506 10:20:43.414 | INFO | MemInfo: { | 20260506 10:20:43.414 | INFO | "gpu" : " 17 GB", | 20260506 10:20:43.414 | INFO | "huge" : "", | 20260506 10:20:43.414 | INFO | "huge_shared" : " 27 GB", | 20260506 10:20:43.414 | INFO | "malloc" : " 14 GB", | 20260506 10:20:43.415 | INFO | "pinned" : " 29 GB" | 20260506 10:20:43.415 | INFO | } | 20260506 10:20:43.415 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:20:43.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:20:43.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:43.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:43.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:43.415 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:20:43.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:43.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:43.415 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:20:43.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:43.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:43.415 | INFO | Uploaded 36GiB to GPU in 1.60549s (22.423GiB/s) | 20260506 10:20:43.415 | INFO | Downloaded 34GiB to GPU in 1.52225s (22.3354GiB/s) | 20260506 10:20:48.554 | INFO | Analyzing chunk frames = [592896,593408) | 20260506 10:20:48.554 | INFO | MemInfo: { | 20260506 10:20:48.554 | INFO | "gpu" : " 17 GB", | 20260506 10:20:48.554 | INFO | "huge" : "", | 20260506 10:20:48.554 | INFO | "huge_shared" : " 27 GB", | 20260506 10:20:48.554 | INFO | "malloc" : " 14 GB", | 20260506 10:20:48.554 | INFO | "pinned" : " 29 GB" | 20260506 10:20:48.554 | INFO | } | 20260506 10:20:48.554 | INFO | Perf report for the last 5.13946 seconds: | 20260506 10:20:48.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:20:48.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:48.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:48.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:48.554 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:20:48.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:48.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:48.554 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:20:48.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:48.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:48.554 | INFO | Uploaded 36GiB to GPU in 1.6052s (22.4271GiB/s) | 20260506 10:20:48.554 | INFO | Downloaded 34GiB to GPU in 1.52299s (22.3245GiB/s) | 20260506 10:20:53.693 | INFO | Analyzing chunk frames = [593408,593920) | 20260506 10:20:53.693 | INFO | MemInfo: { | 20260506 10:20:53.693 | INFO | "gpu" : " 17 GB", | 20260506 10:20:53.693 | INFO | "huge" : "", | 20260506 10:20:53.693 | INFO | "huge_shared" : " 27 GB", | 20260506 10:20:53.693 | INFO | "malloc" : " 14 GB", | 20260506 10:20:53.693 | INFO | "pinned" : " 29 GB" | 20260506 10:20:53.693 | INFO | } | 20260506 10:20:53.693 | INFO | Perf report for the last 5.1393 seconds: | 20260506 10:20:53.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:20:53.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:53.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:53.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:53.694 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:20:53.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:53.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:53.694 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:20:53.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:53.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:53.694 | INFO | Uploaded 36GiB to GPU in 1.60582s (22.4184GiB/s) | 20260506 10:20:53.694 | INFO | Downloaded 34GiB to GPU in 1.52354s (22.3165GiB/s) | 20260506 10:20:53.791 | INFO | Starting a round of Baz IO | 20260506 10:20:56.704 | INFO | Populating RTMetrics report | 20260506 10:20:56.705 | INFO | RTMetrics report populated | 20260506 10:20:56.705 | INFO | Opening tmp RTMetrics json file | 20260506 10:20:56.738 | INFO | Serializing RTMetrics to json | 20260506 10:20:56.763 | INFO | Serialization done, swapping tmp file now... | 20260506 10:20:56.766 | INFO | RTMetrics done | 20260506 10:20:57.270 | INFO | Finished a round of Baz IO: Wrote 1.973 GiB in 3.478 seconds | 20260506 10:20:57.270 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.38% / 0.06% / 51.57% | 20260506 10:20:57.270 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.05814% / 9.13938e-05% / 5.5631e-05% | 20260506 10:20:57.270 | INFO | Baz IO overall: | 20260506 10:20:57.270 | INFO | BytesWritten : 264471344615 | 20260506 10:20:57.270 | INFO | NumEvents : 126987096317 | 20260506 10:20:58.833 | INFO | Analyzing chunk frames = [593920,594432) | 20260506 10:20:58.833 | INFO | MemInfo: { | 20260506 10:20:58.833 | INFO | "gpu" : " 17 GB", | 20260506 10:20:58.833 | INFO | "huge" : "", | 20260506 10:20:58.833 | INFO | "huge_shared" : " 27 GB", | 20260506 10:20:58.833 | INFO | "malloc" : " 14 GB", | 20260506 10:20:58.833 | INFO | "pinned" : " 29 GB" | 20260506 10:20:58.833 | INFO | } | 20260506 10:20:58.833 | INFO | Perf report for the last 5.13953 seconds: | 20260506 10:20:58.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:20:58.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:58.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:58.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:58.833 | INFO | ANALYSIS : 57.6%, 2.8, 85.6ms, 0 | 20260506 10:20:58.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:20:58.833 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 10:20:58.833 | INFO | PRE_HQ : 56.6%, 2.1, 60.4ms, 0 | 20260506 10:20:58.833 | INFO | BAZWRITER : 67.1%, 4.8, 164.7ms, 0 | 20260506 10:20:58.833 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 10:20:58.833 | INFO | Uploaded 36GiB to GPU in 1.67675s (21.4701GiB/s) | 20260506 10:20:58.833 | INFO | Downloaded 38GiB to GPU in 1.74243s (21.8087GiB/s) | 20260506 10:21:03.972 | INFO | Analyzing chunk frames = [594432,594944) | 20260506 10:21:03.972 | INFO | MemInfo: { | 20260506 10:21:03.972 | INFO | "gpu" : " 17 GB", | 20260506 10:21:03.972 | INFO | "huge" : "", | 20260506 10:21:03.972 | INFO | "huge_shared" : " 27 GB", | 20260506 10:21:03.972 | INFO | "malloc" : " 14 GB", | 20260506 10:21:03.972 | INFO | "pinned" : " 29 GB" | 20260506 10:21:03.972 | INFO | } | 20260506 10:21:03.972 | INFO | Perf report for the last 5.13918 seconds: | 20260506 10:21:03.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:21:03.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:03.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:03.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:03.972 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:21:03.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:03.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:03.972 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:21:03.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:03.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:03.973 | INFO | Uploaded 36GiB to GPU in 1.6034s (22.4523GiB/s) | 20260506 10:21:03.973 | INFO | Downloaded 34GiB to GPU in 1.52032s (22.3637GiB/s) | 20260506 10:21:09.111 | INFO | Analyzing chunk frames = [594944,595456) | 20260506 10:21:09.111 | INFO | MemInfo: { | 20260506 10:21:09.111 | INFO | "gpu" : " 17 GB", | 20260506 10:21:09.111 | INFO | "huge" : "", | 20260506 10:21:09.111 | INFO | "huge_shared" : " 27 GB", | 20260506 10:21:09.111 | INFO | "malloc" : " 14 GB", | 20260506 10:21:09.111 | INFO | "pinned" : " 29 GB" | 20260506 10:21:09.111 | INFO | } | 20260506 10:21:09.112 | INFO | Perf report for the last 5.13939 seconds: | 20260506 10:21:09.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:21:09.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:09.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:09.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:09.112 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:21:09.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:09.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:09.112 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:21:09.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:09.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:09.112 | INFO | Uploaded 36GiB to GPU in 1.60805s (22.3874GiB/s) | 20260506 10:21:09.112 | INFO | Downloaded 34GiB to GPU in 1.52589s (22.2821GiB/s) | 20260506 10:21:14.251 | INFO | Analyzing chunk frames = [595456,595968) | 20260506 10:21:14.251 | INFO | MemInfo: { | 20260506 10:21:14.251 | INFO | "gpu" : " 17 GB", | 20260506 10:21:14.251 | INFO | "huge" : "", | 20260506 10:21:14.251 | INFO | "huge_shared" : " 27 GB", | 20260506 10:21:14.251 | INFO | "malloc" : " 14 GB", | 20260506 10:21:14.251 | INFO | "pinned" : " 29 GB" | 20260506 10:21:14.251 | INFO | } | 20260506 10:21:14.251 | INFO | Perf report for the last 5.13953 seconds: | 20260506 10:21:14.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:21:14.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:14.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:14.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:14.251 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:21:14.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:14.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:14.251 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:21:14.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:14.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:14.251 | INFO | Uploaded 36GiB to GPU in 1.60667s (22.4066GiB/s) | 20260506 10:21:14.251 | INFO | Downloaded 34GiB to GPU in 1.52382s (22.3123GiB/s) | 20260506 10:21:19.390 | INFO | Analyzing chunk frames = [595968,596480) | 20260506 10:21:19.390 | INFO | MemInfo: { | 20260506 10:21:19.390 | INFO | "gpu" : " 17 GB", | 20260506 10:21:19.390 | INFO | "huge" : "", | 20260506 10:21:19.390 | INFO | "huge_shared" : " 27 GB", | 20260506 10:21:19.390 | INFO | "malloc" : " 14 GB", | 20260506 10:21:19.390 | INFO | "pinned" : " 29 GB" | 20260506 10:21:19.390 | INFO | } | 20260506 10:21:19.390 | INFO | Perf report for the last 5.13907 seconds: | 20260506 10:21:19.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:21:19.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:19.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:19.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:19.390 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260506 10:21:19.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:19.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:19.390 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:21:19.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:19.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:19.391 | INFO | Uploaded 36GiB to GPU in 1.60558s (22.4218GiB/s) | 20260506 10:21:19.391 | INFO | Downloaded 34GiB to GPU in 1.52249s (22.3319GiB/s) | 20260506 10:21:24.529 | INFO | Analyzing chunk frames = [596480,596992) | 20260506 10:21:24.529 | INFO | MemInfo: { | 20260506 10:21:24.529 | INFO | "gpu" : " 17 GB", | 20260506 10:21:24.529 | INFO | "huge" : "", | 20260506 10:21:24.529 | INFO | "huge_shared" : " 27 GB", | 20260506 10:21:24.530 | INFO | "malloc" : " 14 GB", | 20260506 10:21:24.530 | INFO | "pinned" : " 29 GB" | 20260506 10:21:24.530 | INFO | } | 20260506 10:21:24.530 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:21:24.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:21:24.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:24.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:24.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:24.530 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:21:24.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:24.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:24.530 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:21:24.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:24.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:24.530 | INFO | Uploaded 36GiB to GPU in 1.60524s (22.4265GiB/s) | 20260506 10:21:24.530 | INFO | Downloaded 34GiB to GPU in 1.52248s (22.332GiB/s) | 20260506 10:21:29.669 | INFO | Analyzing chunk frames = [596992,597504) | 20260506 10:21:29.669 | INFO | MemInfo: { | 20260506 10:21:29.669 | INFO | "gpu" : " 17 GB", | 20260506 10:21:29.669 | INFO | "huge" : "", | 20260506 10:21:29.669 | INFO | "huge_shared" : " 27 GB", | 20260506 10:21:29.669 | INFO | "malloc" : " 14 GB", | 20260506 10:21:29.669 | INFO | "pinned" : " 29 GB" | 20260506 10:21:29.669 | INFO | } | 20260506 10:21:29.669 | INFO | Perf report for the last 5.13925 seconds: | 20260506 10:21:29.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:21:29.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:29.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:29.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:29.669 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:21:29.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:29.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:29.669 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:21:29.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:29.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:29.670 | INFO | Uploaded 36GiB to GPU in 1.60561s (22.4213GiB/s) | 20260506 10:21:29.670 | INFO | Downloaded 34GiB to GPU in 1.52364s (22.315GiB/s) | 20260506 10:21:34.808 | INFO | Analyzing chunk frames = [597504,598016) | 20260506 10:21:34.808 | INFO | MemInfo: { | 20260506 10:21:34.808 | INFO | "gpu" : " 17 GB", | 20260506 10:21:34.808 | INFO | "huge" : "", | 20260506 10:21:34.808 | INFO | "huge_shared" : " 27 GB", | 20260506 10:21:34.808 | INFO | "malloc" : " 14 GB", | 20260506 10:21:34.808 | INFO | "pinned" : " 29 GB" | 20260506 10:21:34.808 | INFO | } | 20260506 10:21:34.808 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:21:34.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:21:34.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:34.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:34.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:34.808 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:21:34.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:34.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:34.808 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:21:34.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:34.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:34.808 | INFO | Uploaded 36GiB to GPU in 1.60558s (22.4218GiB/s) | 20260506 10:21:34.808 | INFO | Downloaded 34GiB to GPU in 1.52295s (22.3251GiB/s) | 20260506 10:21:34.809 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:21:34.809 | INFO | Total mem usage: 16.5 GiB | 20260506 10:21:34.809 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:21:34.809 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -3.47%) | 20260506 10:21:34.809 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:21:34.809 | INFO | High Water Breakdown: | 20260506 10:21:34.809 | INFO | 60.7% : 9.68 GiB ( 95.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:21:34.809 | INFO | 39.2% : 6.26 GiB ( 93.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:21:34.809 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:21:34.809 | INFO | ------------------------------------------ | 20260506 10:21:34.809 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:21:34.809 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:21:34.809 | INFO | Total mem usage: 26.5 GiB | 20260506 10:21:34.809 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:21:34.809 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:21:34.809 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:21:34.809 | INFO | High Water Breakdown: | 20260506 10:21:34.809 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:21:34.809 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:21:34.809 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:21:34.809 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:21:34.809 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:21:34.809 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:21:34.809 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:21:34.809 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:21:34.809 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:21:34.809 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:21:34.809 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:21:34.809 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:21:34.809 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:21:34.809 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:21:34.809 | INFO | ------------------------------------------ | 20260506 10:21:34.809 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:21:34.809 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:21:34.809 | INFO | Total mem usage: 25.6 GiB | 20260506 10:21:34.809 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:21:34.809 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:21:34.809 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:21:34.809 | INFO | High Water Breakdown: | 20260506 10:21:34.809 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:21:34.809 | INFO | ------------------------------------------ | 20260506 10:21:34.809 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:21:34.809 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:21:34.809 | INFO | Total mem usage: 15.7 GiB | 20260506 10:21:34.809 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:21:34.809 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:21:34.809 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:21:34.809 | INFO | High Water Breakdown: | 20260506 10:21:34.809 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:21:34.809 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:21:34.809 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:21:34.809 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:21:34.809 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:21:34.809 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:21:34.809 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:21:34.809 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:21:34.809 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:21:34.809 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:21:34.809 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:21:34.809 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:21:34.809 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:21:34.809 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:21:34.809 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:21:34.809 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:21:34.809 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:21:34.809 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:21:34.809 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:21:34.809 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:21:34.809 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:21:34.809 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:21:34.809 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:21:34.809 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:21:34.810 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:21:34.810 | INFO | ------------------------------------------ | 20260506 10:21:34.810 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:21:34.810 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:21:34.810 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:21:34.810 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:21:34.810 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260506 10:21:34.810 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 4.75e-03 / 0.00e+00 | 20260506 10:21:34.810 | INFO | PerfReport: 10.95%, Baseline -- 3.30e+00 / -nan, 1.78e-02 / 0.00e+00 | 20260506 10:21:34.810 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.13e-02 / 0.00e+00 | 20260506 10:21:34.810 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.06e+00 / 0.00e+00 | 20260506 10:21:34.810 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.72e+01 / -nan, 4.36e-02 / 0.00e+00 | 20260506 10:21:34.810 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.23e+00 / -nan, 1.29e-01 / 0.00e+00 | 20260506 10:21:34.810 | INFO | PerfReport: 3.69%, Metrics -- 1.11e+00 / -nan, 1.73e-01 / 0.00e+00 | 20260506 10:21:34.810 | INFO | PerfReport: 1.43%, PulseSerializing -- 4.30e-01 / -nan, 6.11e-02 / 0.00e+00 | 20260506 10:21:34.810 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:21:34.810 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:21:34.810 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:21:34.810 | INFO | PerfReport: Full Execution -- 9.00e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 10:21:34.810 | INFO | PerfReport: 18.68%, Upload -- 1.68e+01 / 0.00e+00, 4.82e-01 / 0.00e+00 | 20260506 10:21:34.810 | INFO | PerfReport: 17.94%, Download -- 1.61e+01 / 0.00e+00, 8.29e-01 / 0.00e+00 | 20260506 10:21:34.810 | INFO | PerfReport: 33.46%, Compute -- 3.01e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260506 10:21:34.810 | INFO | PerfReport: 0.56%, LockUpload -- 5.04e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260506 10:21:34.810 | INFO | PerfReport: 0.00%, LockDownload -- 5.03e-04 / 0.00e+00, 1.26e-04 / 0.00e+00 | 20260506 10:21:34.810 | INFO | PerfReport: 29.34%, LockCompute -- 2.64e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260506 10:21:34.810 | INFO | PerfReport: 0.02%, Misc -- 1.39e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260506 10:21:34.906 | INFO | Starting a round of Baz IO | 20260506 10:21:37.815 | INFO | Populating RTMetrics report | 20260506 10:21:37.815 | INFO | RTMetrics report populated | 20260506 10:21:37.815 | INFO | Opening tmp RTMetrics json file | 20260506 10:21:37.838 | INFO | Serializing RTMetrics to json | 20260506 10:21:37.842 | INFO | Serialization done, swapping tmp file now... | 20260506 10:21:37.847 | INFO | RTMetrics done | 20260506 10:21:38.311 | INFO | Finished a round of Baz IO: Wrote 1.976 GiB in 3.404 seconds | 20260506 10:21:38.311 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.32% / 0.05% / 51.62% | 20260506 10:21:38.311 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.03512% / 8.74201e-05% / 4.76837e-05% | 20260506 10:21:38.311 | INFO | Baz IO overall: | 20260506 10:21:38.311 | INFO | BytesWritten : 266592602961 | 20260506 10:21:38.311 | INFO | NumEvents : 128016630179 | 20260506 10:21:39.947 | INFO | Analyzing chunk frames = [598016,598528) | 20260506 10:21:39.947 | INFO | MemInfo: { | 20260506 10:21:39.947 | INFO | "gpu" : " 17 GB", | 20260506 10:21:39.947 | INFO | "huge" : "", | 20260506 10:21:39.947 | INFO | "huge_shared" : " 27 GB", | 20260506 10:21:39.947 | INFO | "malloc" : " 14 GB", | 20260506 10:21:39.947 | INFO | "pinned" : " 29 GB" | 20260506 10:21:39.947 | INFO | } | 20260506 10:21:39.947 | INFO | Perf report for the last 5.13927 seconds: | 20260506 10:21:39.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:21:39.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:39.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:39.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:39.948 | INFO | ANALYSIS : 57.5%, 2.8, 85.6ms, 0 | 20260506 10:21:39.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:39.948 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 10:21:39.948 | INFO | PRE_HQ : 56.6%, 2.1, 60.4ms, 0 | 20260506 10:21:39.948 | INFO | BAZWRITER : 65.9%, 4.8, 161.9ms, 0 | 20260506 10:21:39.948 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 10:21:39.948 | INFO | Uploaded 36GiB to GPU in 1.67986s (21.4303GiB/s) | 20260506 10:21:39.948 | INFO | Downloaded 38GiB to GPU in 1.7455s (21.7703GiB/s) | 20260506 10:21:45.087 | INFO | Analyzing chunk frames = [598528,599040) | 20260506 10:21:45.087 | INFO | MemInfo: { | 20260506 10:21:45.087 | INFO | "gpu" : " 17 GB", | 20260506 10:21:45.087 | INFO | "huge" : "", | 20260506 10:21:45.087 | INFO | "huge_shared" : " 27 GB", | 20260506 10:21:45.087 | INFO | "malloc" : " 14 GB", | 20260506 10:21:45.087 | INFO | "pinned" : " 29 GB" | 20260506 10:21:45.087 | INFO | } | 20260506 10:21:45.087 | INFO | Perf report for the last 5.13953 seconds: | 20260506 10:21:45.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:21:45.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:45.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:45.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:45.087 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:21:45.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:45.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:45.087 | INFO | PRE_HQ : 29.4%, 1.0, 15.4ms, 0 | 20260506 10:21:45.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:45.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:45.088 | INFO | Uploaded 36GiB to GPU in 1.60267s (22.4625GiB/s) | 20260506 10:21:45.088 | INFO | Downloaded 34GiB to GPU in 1.52003s (22.368GiB/s) | 20260506 10:21:50.226 | INFO | Analyzing chunk frames = [599040,599552) | 20260506 10:21:50.226 | INFO | MemInfo: { | 20260506 10:21:50.226 | INFO | "gpu" : " 17 GB", | 20260506 10:21:50.226 | INFO | "huge" : "", | 20260506 10:21:50.226 | INFO | "huge_shared" : " 27 GB", | 20260506 10:21:50.226 | INFO | "malloc" : " 14 GB", | 20260506 10:21:50.226 | INFO | "pinned" : " 29 GB" | 20260506 10:21:50.226 | INFO | } | 20260506 10:21:50.226 | INFO | Perf report for the last 5.13921 seconds: | 20260506 10:21:50.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:21:50.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:50.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:50.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:50.226 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:21:50.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:50.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:50.226 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:21:50.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:50.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:50.226 | INFO | Uploaded 36GiB to GPU in 1.60732s (22.3976GiB/s) | 20260506 10:21:50.226 | INFO | Downloaded 34GiB to GPU in 1.52442s (22.3036GiB/s) | 20260506 10:21:55.365 | INFO | Analyzing chunk frames = [599552,600064) | 20260506 10:21:55.365 | INFO | MemInfo: { | 20260506 10:21:55.365 | INFO | "gpu" : " 17 GB", | 20260506 10:21:55.365 | INFO | "huge" : "", | 20260506 10:21:55.365 | INFO | "huge_shared" : " 27 GB", | 20260506 10:21:55.365 | INFO | "malloc" : " 14 GB", | 20260506 10:21:55.365 | INFO | "pinned" : " 29 GB" | 20260506 10:21:55.365 | INFO | } | 20260506 10:21:55.366 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:21:55.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:21:55.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:55.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:55.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:55.366 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:21:55.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:55.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:55.366 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:21:55.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:55.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:21:55.366 | INFO | Uploaded 36GiB to GPU in 1.60623s (22.4128GiB/s) | 20260506 10:21:55.366 | INFO | Downloaded 34GiB to GPU in 1.52369s (22.3143GiB/s) | 20260506 10:22:00.505 | INFO | Analyzing chunk frames = [600064,600576) | 20260506 10:22:00.505 | INFO | MemInfo: { | 20260506 10:22:00.505 | INFO | "gpu" : " 17 GB", | 20260506 10:22:00.505 | INFO | "huge" : "", | 20260506 10:22:00.505 | INFO | "huge_shared" : " 27 GB", | 20260506 10:22:00.505 | INFO | "malloc" : " 14 GB", | 20260506 10:22:00.505 | INFO | "pinned" : " 29 GB" | 20260506 10:22:00.505 | INFO | } | 20260506 10:22:00.505 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:22:00.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:22:00.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:00.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:00.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:00.505 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260506 10:22:00.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:00.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:00.505 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:22:00.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:00.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:00.505 | INFO | Uploaded 36GiB to GPU in 1.6051s (22.4285GiB/s) | 20260506 10:22:00.505 | INFO | Downloaded 34GiB to GPU in 1.52217s (22.3365GiB/s) | 20260506 10:22:05.644 | INFO | Analyzing chunk frames = [600576,601088) | 20260506 10:22:05.644 | INFO | MemInfo: { | 20260506 10:22:05.644 | INFO | "gpu" : " 17 GB", | 20260506 10:22:05.644 | INFO | "huge" : "", | 20260506 10:22:05.644 | INFO | "huge_shared" : " 27 GB", | 20260506 10:22:05.644 | INFO | "malloc" : " 14 GB", | 20260506 10:22:05.644 | INFO | "pinned" : " 29 GB" | 20260506 10:22:05.644 | INFO | } | 20260506 10:22:05.644 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:22:05.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:22:05.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:05.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:05.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:05.644 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:22:05.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:05.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:05.644 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:22:05.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:05.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:05.645 | INFO | Uploaded 36GiB to GPU in 1.60493s (22.4309GiB/s) | 20260506 10:22:05.645 | INFO | Downloaded 34GiB to GPU in 1.52181s (22.3418GiB/s) | 20260506 10:22:10.783 | INFO | Analyzing chunk frames = [601088,601600) | 20260506 10:22:10.783 | INFO | MemInfo: { | 20260506 10:22:10.783 | INFO | "gpu" : " 17 GB", | 20260506 10:22:10.783 | INFO | "huge" : "", | 20260506 10:22:10.784 | INFO | "huge_shared" : " 27 GB", | 20260506 10:22:10.784 | INFO | "malloc" : " 14 GB", | 20260506 10:22:10.784 | INFO | "pinned" : " 29 GB" | 20260506 10:22:10.784 | INFO | } | 20260506 10:22:10.784 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:22:10.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:22:10.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:10.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:10.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:10.784 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:22:10.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:10.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:10.784 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:22:10.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:10.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:10.784 | INFO | Uploaded 36GiB to GPU in 1.60552s (22.4227GiB/s) | 20260506 10:22:10.784 | INFO | Downloaded 34GiB to GPU in 1.52318s (22.3218GiB/s) | 20260506 10:22:15.923 | INFO | Analyzing chunk frames = [601600,602112) | 20260506 10:22:15.923 | INFO | MemInfo: { | 20260506 10:22:15.923 | INFO | "gpu" : " 17 GB", | 20260506 10:22:15.923 | INFO | "huge" : "", | 20260506 10:22:15.923 | INFO | "huge_shared" : " 27 GB", | 20260506 10:22:15.923 | INFO | "malloc" : " 14 GB", | 20260506 10:22:15.923 | INFO | "pinned" : " 29 GB" | 20260506 10:22:15.923 | INFO | } | 20260506 10:22:15.923 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:22:15.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:22:15.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:15.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:15.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:15.923 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:22:15.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:15.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:15.923 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 10:22:15.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:15.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:15.923 | INFO | Uploaded 36GiB to GPU in 1.60524s (22.4266GiB/s) | 20260506 10:22:15.923 | INFO | Downloaded 34GiB to GPU in 1.52287s (22.3262GiB/s) | 20260506 10:22:16.020 | INFO | Starting a round of Baz IO | 20260506 10:22:18.932 | INFO | Populating RTMetrics report | 20260506 10:22:18.933 | INFO | RTMetrics report populated | 20260506 10:22:18.933 | INFO | Opening tmp RTMetrics json file | 20260506 10:22:18.934 | INFO | Serializing RTMetrics to json | 20260506 10:22:18.938 | INFO | Serialization done, swapping tmp file now... | 20260506 10:22:18.938 | INFO | RTMetrics done | 20260506 10:22:19.581 | INFO | Finished a round of Baz IO: Wrote 1.979 GiB in 3.561 seconds | 20260506 10:22:19.581 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.27% / 0.05% / 51.68% | 20260506 10:22:19.581 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 7.02456% / 0.000127157% / 3.97364e-05% | 20260506 10:22:19.581 | INFO | Baz IO overall: | 20260506 10:22:19.581 | INFO | BytesWritten : 268717150755 | 20260506 10:22:19.582 | INFO | NumEvents : 129048644145 | 20260506 10:22:21.062 | INFO | Analyzing chunk frames = [602112,602624) | 20260506 10:22:21.062 | INFO | MemInfo: { | 20260506 10:22:21.062 | INFO | "gpu" : " 17 GB", | 20260506 10:22:21.062 | INFO | "huge" : "", | 20260506 10:22:21.062 | INFO | "huge_shared" : " 27 GB", | 20260506 10:22:21.062 | INFO | "malloc" : " 14 GB", | 20260506 10:22:21.062 | INFO | "pinned" : " 29 GB" | 20260506 10:22:21.062 | INFO | } | 20260506 10:22:21.062 | INFO | Perf report for the last 5.13922 seconds: | 20260506 10:22:21.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:22:21.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:21.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:21.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:21.062 | INFO | ANALYSIS : 57.6%, 2.8, 85.7ms, 0 | 20260506 10:22:21.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:21.062 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 10:22:21.062 | INFO | PRE_HQ : 56.6%, 2.1, 60.5ms, 0 | 20260506 10:22:21.062 | INFO | BAZWRITER : 68.9%, 4.8, 169.2ms, 0 | 20260506 10:22:21.062 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 10:22:21.063 | INFO | Uploaded 36GiB to GPU in 1.67414s (21.5036GiB/s) | 20260506 10:22:21.063 | INFO | Downloaded 38GiB to GPU in 1.7388s (21.8541GiB/s) | 20260506 10:22:26.201 | INFO | Analyzing chunk frames = [602624,603136) | 20260506 10:22:26.201 | INFO | MemInfo: { | 20260506 10:22:26.201 | INFO | "gpu" : " 17 GB", | 20260506 10:22:26.201 | INFO | "huge" : "", | 20260506 10:22:26.201 | INFO | "huge_shared" : " 27 GB", | 20260506 10:22:26.202 | INFO | "malloc" : " 14 GB", | 20260506 10:22:26.202 | INFO | "pinned" : " 29 GB" | 20260506 10:22:26.202 | INFO | } | 20260506 10:22:26.202 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:22:26.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:22:26.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:26.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:26.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:26.202 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 10:22:26.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:26.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:26.202 | INFO | PRE_HQ : 29.5%, 1.0, 15.6ms, 0 | 20260506 10:22:26.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:26.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:26.202 | INFO | Uploaded 36GiB to GPU in 1.60281s (22.4605GiB/s) | 20260506 10:22:26.202 | INFO | Downloaded 34GiB to GPU in 1.51953s (22.3753GiB/s) | 20260506 10:22:31.341 | INFO | Analyzing chunk frames = [603136,603648) | 20260506 10:22:31.341 | INFO | MemInfo: { | 20260506 10:22:31.341 | INFO | "gpu" : " 17 GB", | 20260506 10:22:31.341 | INFO | "huge" : "", | 20260506 10:22:31.341 | INFO | "huge_shared" : " 27 GB", | 20260506 10:22:31.341 | INFO | "malloc" : " 14 GB", | 20260506 10:22:31.341 | INFO | "pinned" : " 29 GB" | 20260506 10:22:31.341 | INFO | } | 20260506 10:22:31.341 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:22:31.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:22:31.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:31.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:31.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:31.341 | INFO | ANALYSIS : 56.7%, 3.0, 88.2ms, 0 | 20260506 10:22:31.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:31.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:31.341 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:22:31.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:31.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:31.341 | INFO | Uploaded 36GiB to GPU in 1.60774s (22.3917GiB/s) | 20260506 10:22:31.341 | INFO | Downloaded 34GiB to GPU in 1.52539s (22.2893GiB/s) | 20260506 10:22:36.480 | INFO | Analyzing chunk frames = [603648,604160) | 20260506 10:22:36.480 | INFO | MemInfo: { | 20260506 10:22:36.480 | INFO | "gpu" : " 17 GB", | 20260506 10:22:36.480 | INFO | "huge" : "", | 20260506 10:22:36.480 | INFO | "huge_shared" : " 27 GB", | 20260506 10:22:36.480 | INFO | "malloc" : " 14 GB", | 20260506 10:22:36.480 | INFO | "pinned" : " 29 GB" | 20260506 10:22:36.480 | INFO | } | 20260506 10:22:36.480 | INFO | Perf report for the last 5.13914 seconds: | 20260506 10:22:36.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:22:36.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:36.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:36.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:36.480 | INFO | ANALYSIS : 56.5%, 3.0, 87.6ms, 0 | 20260506 10:22:36.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:36.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:36.480 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:22:36.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:36.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:36.481 | INFO | Uploaded 36GiB to GPU in 1.60679s (22.4049GiB/s) | 20260506 10:22:36.481 | INFO | Downloaded 34GiB to GPU in 1.52368s (22.3145GiB/s) | 20260506 10:22:41.619 | INFO | Analyzing chunk frames = [604160,604672) | 20260506 10:22:41.619 | INFO | MemInfo: { | 20260506 10:22:41.619 | INFO | "gpu" : " 17 GB", | 20260506 10:22:41.620 | INFO | "huge" : "", | 20260506 10:22:41.620 | INFO | "huge_shared" : " 27 GB", | 20260506 10:22:41.620 | INFO | "malloc" : " 14 GB", | 20260506 10:22:41.620 | INFO | "pinned" : " 29 GB" | 20260506 10:22:41.620 | INFO | } | 20260506 10:22:41.620 | INFO | Perf report for the last 5.13942 seconds: | 20260506 10:22:41.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:22:41.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:41.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:41.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:41.620 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:22:41.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:41.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:41.620 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:22:41.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:41.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:41.620 | INFO | Uploaded 36GiB to GPU in 1.61468s (22.2955GiB/s) | 20260506 10:22:41.620 | INFO | Downloaded 34GiB to GPU in 1.53239s (22.1876GiB/s) | 20260506 10:22:46.759 | INFO | Analyzing chunk frames = [604672,605184) | 20260506 10:22:46.759 | INFO | MemInfo: { | 20260506 10:22:46.759 | INFO | "gpu" : " 17 GB", | 20260506 10:22:46.759 | INFO | "huge" : "", | 20260506 10:22:46.759 | INFO | "huge_shared" : " 27 GB", | 20260506 10:22:46.759 | INFO | "malloc" : " 14 GB", | 20260506 10:22:46.759 | INFO | "pinned" : " 29 GB" | 20260506 10:22:46.759 | INFO | } | 20260506 10:22:46.759 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:22:46.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:22:46.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:46.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:46.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:46.759 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:22:46.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:46.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:46.759 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:22:46.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:46.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:46.759 | INFO | Uploaded 36GiB to GPU in 1.60511s (22.4283GiB/s) | 20260506 10:22:46.759 | INFO | Downloaded 34GiB to GPU in 1.52348s (22.3173GiB/s) | 20260506 10:22:51.898 | INFO | Analyzing chunk frames = [605184,605696) | 20260506 10:22:51.898 | INFO | MemInfo: { | 20260506 10:22:51.898 | INFO | "gpu" : " 17 GB", | 20260506 10:22:51.898 | INFO | "huge" : "", | 20260506 10:22:51.898 | INFO | "huge_shared" : " 27 GB", | 20260506 10:22:51.898 | INFO | "malloc" : " 14 GB", | 20260506 10:22:51.898 | INFO | "pinned" : " 29 GB" | 20260506 10:22:51.898 | INFO | } | 20260506 10:22:51.898 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:22:51.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:22:51.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:51.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:51.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:51.898 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:22:51.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:51.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:51.898 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:22:51.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:51.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:51.899 | INFO | Uploaded 36GiB to GPU in 1.60436s (22.4388GiB/s) | 20260506 10:22:51.899 | INFO | Downloaded 34GiB to GPU in 1.52358s (22.3158GiB/s) | 20260506 10:22:57.037 | INFO | Analyzing chunk frames = [605696,606208) | 20260506 10:22:57.037 | INFO | MemInfo: { | 20260506 10:22:57.037 | INFO | "gpu" : " 17 GB", | 20260506 10:22:57.037 | INFO | "huge" : "", | 20260506 10:22:57.038 | INFO | "huge_shared" : " 27 GB", | 20260506 10:22:57.038 | INFO | "malloc" : " 14 GB", | 20260506 10:22:57.038 | INFO | "pinned" : " 29 GB" | 20260506 10:22:57.038 | INFO | } | 20260506 10:22:57.038 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:22:57.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:22:57.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:57.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:57.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:57.038 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:22:57.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:57.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:57.038 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:22:57.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:57.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:22:57.038 | INFO | Uploaded 36GiB to GPU in 1.60599s (22.416GiB/s) | 20260506 10:22:57.038 | INFO | Downloaded 34GiB to GPU in 1.52383s (22.3122GiB/s) | 20260506 10:22:57.038 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:22:57.038 | INFO | Total mem usage: 16.5 GiB | 20260506 10:22:57.038 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:22:57.038 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -4.16%) | 20260506 10:22:57.038 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:22:57.038 | INFO | High Water Breakdown: | 20260506 10:22:57.038 | INFO | 60.6% : 9.61 GiB ( 95.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:22:57.038 | INFO | 39.3% : 6.23 GiB ( 93.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:22:57.038 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:22:57.038 | INFO | ------------------------------------------ | 20260506 10:22:57.038 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:22:57.038 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:22:57.038 | INFO | Total mem usage: 26.5 GiB | 20260506 10:22:57.038 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:22:57.038 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:22:57.038 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:22:57.038 | INFO | High Water Breakdown: | 20260506 10:22:57.038 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:22:57.038 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:22:57.038 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:22:57.038 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:22:57.038 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:22:57.038 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:22:57.038 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:22:57.038 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:22:57.038 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:22:57.038 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:22:57.038 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:22:57.038 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:22:57.038 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:22:57.038 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:22:57.038 | INFO | ------------------------------------------ | 20260506 10:22:57.038 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:22:57.038 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:22:57.038 | INFO | Total mem usage: 25.6 GiB | 20260506 10:22:57.038 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:22:57.038 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:22:57.038 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:22:57.038 | INFO | High Water Breakdown: | 20260506 10:22:57.038 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:22:57.038 | INFO | ------------------------------------------ | 20260506 10:22:57.038 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:22:57.038 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:22:57.038 | INFO | Total mem usage: 15.7 GiB | 20260506 10:22:57.038 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:22:57.038 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:22:57.038 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:22:57.038 | INFO | High Water Breakdown: | 20260506 10:22:57.038 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:22:57.038 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:22:57.038 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:22:57.038 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:22:57.038 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:22:57.038 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:22:57.038 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:22:57.038 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:22:57.038 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:22:57.038 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:22:57.038 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:22:57.038 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:22:57.038 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:22:57.038 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:22:57.038 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:22:57.038 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:22:57.038 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:22:57.038 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:22:57.038 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:22:57.038 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:22:57.038 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:22:57.038 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:22:57.038 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:22:57.038 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:22:57.038 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:22:57.038 | INFO | ------------------------------------------ | 20260506 10:22:57.038 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:22:57.038 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:22:57.038 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:22:57.038 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:22:57.038 | INFO | PerfReport: Full Execution -- 3.01e+01 / 7.66e+01, 1.27e+01 / 0.00e+00 | 20260506 10:22:57.038 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / 8.60e-02, 4.91e-03 / 0.00e+00 | 20260506 10:22:57.038 | INFO | PerfReport: 10.94%, Baseline -- 3.30e+00 / 3.31e+00, 1.79e-02 / 0.00e+00 | 20260506 10:22:57.038 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / 1.06e+00, 5.21e-02 / 0.00e+00 | 20260506 10:22:57.038 | INFO | PerfReport: 16.09%, DME -- 3.88e+01 / 5.12e+01, 3.14e+00 / 0.00e+00 | 20260506 10:22:57.038 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.72e+01 / 1.71e+01, 4.28e-02 / 0.00e+00 | 20260506 10:22:57.038 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.23e+00 / 2.25e+00, 1.29e-01 / 0.00e+00 | 20260506 10:22:57.038 | INFO | PerfReport: 3.69%, Metrics -- 1.11e+00 / 1.07e+00, 1.73e-01 / 0.00e+00 | 20260506 10:22:57.038 | INFO | PerfReport: 1.43%, PulseSerializing -- 4.29e-01 / 4.33e-01, 6.08e-02 / 0.00e+00 | 20260506 10:22:57.038 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:22:57.038 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:22:57.038 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:22:57.038 | INFO | PerfReport: Full Execution -- 8.99e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 10:22:57.038 | INFO | PerfReport: 18.70%, Upload -- 1.68e+01 / 0.00e+00, 5.67e-01 / 0.00e+00 | 20260506 10:22:57.038 | INFO | PerfReport: 17.96%, Download -- 1.62e+01 / 0.00e+00, 9.05e-01 / 0.00e+00 | 20260506 10:22:57.038 | INFO | PerfReport: 33.48%, Compute -- 3.01e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 10:22:57.038 | INFO | PerfReport: 0.56%, LockUpload -- 5.08e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:22:57.038 | INFO | PerfReport: 0.00%, LockDownload -- 4.95e-04 / 0.00e+00, 1.24e-04 / 0.00e+00 | 20260506 10:22:57.038 | INFO | PerfReport: 29.27%, LockCompute -- 2.63e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 10:22:57.038 | INFO | PerfReport: 0.02%, Misc -- 1.36e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260506 10:22:57.135 | INFO | Starting a round of Baz IO | 20260506 10:23:00.049 | INFO | Populating RTMetrics report | 20260506 10:23:00.050 | INFO | RTMetrics report populated | 20260506 10:23:00.050 | INFO | Opening tmp RTMetrics json file | 20260506 10:23:00.052 | INFO | Serializing RTMetrics to json | 20260506 10:23:00.053 | INFO | Serialization done, swapping tmp file now... | 20260506 10:23:00.060 | INFO | RTMetrics done | 20260506 10:23:00.295 | INFO | Finished a round of Baz IO: Wrote 1.978 GiB in 3.16 seconds | 20260506 10:23:00.295 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.21% / 0.06% / 51.73% | 20260506 10:23:00.295 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.99188% / 0.000111262% / 4.76837e-05% | 20260506 10:23:00.295 | INFO | Baz IO overall: | 20260506 10:23:00.295 | INFO | BytesWritten : 270841357325 | 20260506 10:23:00.295 | INFO | NumEvents : 130080044749 | 20260506 10:23:02.177 | INFO | Analyzing chunk frames = [606208,606720) | 20260506 10:23:02.177 | INFO | MemInfo: { | 20260506 10:23:02.177 | INFO | "gpu" : " 17 GB", | 20260506 10:23:02.177 | INFO | "huge" : "", | 20260506 10:23:02.177 | INFO | "huge_shared" : " 27 GB", | 20260506 10:23:02.177 | INFO | "malloc" : " 14 GB", | 20260506 10:23:02.177 | INFO | "pinned" : " 29 GB" | 20260506 10:23:02.177 | INFO | } | 20260506 10:23:02.177 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:23:02.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:23:02.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:02.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:02.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:02.177 | INFO | ANALYSIS : 57.6%, 2.8, 85.1ms, 0 | 20260506 10:23:02.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:02.177 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 10:23:02.177 | INFO | PRE_HQ : 56.7%, 2.1, 61.1ms, 0 | 20260506 10:23:02.177 | INFO | BAZWRITER : 61.1%, 4.7, 149.1ms, 0 | 20260506 10:23:02.177 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 10:23:02.177 | INFO | Uploaded 36GiB to GPU in 1.67252s (21.5244GiB/s) | 20260506 10:23:02.177 | INFO | Downloaded 38GiB to GPU in 1.73959s (21.8442GiB/s) | 20260506 10:23:07.316 | INFO | Analyzing chunk frames = [606720,607232) | 20260506 10:23:07.316 | INFO | MemInfo: { | 20260506 10:23:07.316 | INFO | "gpu" : " 17 GB", | 20260506 10:23:07.316 | INFO | "huge" : "", | 20260506 10:23:07.316 | INFO | "huge_shared" : " 27 GB", | 20260506 10:23:07.316 | INFO | "malloc" : " 14 GB", | 20260506 10:23:07.316 | INFO | "pinned" : " 29 GB" | 20260506 10:23:07.316 | INFO | } | 20260506 10:23:07.316 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:23:07.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:23:07.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:07.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:07.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:07.317 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:23:07.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:07.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:07.317 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260506 10:23:07.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:07.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:07.317 | INFO | Uploaded 36GiB to GPU in 1.60343s (22.4519GiB/s) | 20260506 10:23:07.317 | INFO | Downloaded 34GiB to GPU in 1.51999s (22.3686GiB/s) | 20260506 10:23:12.455 | INFO | Analyzing chunk frames = [607232,607744) | 20260506 10:23:12.455 | INFO | MemInfo: { | 20260506 10:23:12.455 | INFO | "gpu" : " 17 GB", | 20260506 10:23:12.455 | INFO | "huge" : "", | 20260506 10:23:12.455 | INFO | "huge_shared" : " 27 GB", | 20260506 10:23:12.456 | INFO | "malloc" : " 14 GB", | 20260506 10:23:12.456 | INFO | "pinned" : " 29 GB" | 20260506 10:23:12.456 | INFO | } | 20260506 10:23:12.456 | INFO | Perf report for the last 5.13919 seconds: | 20260506 10:23:12.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:23:12.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:12.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:12.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:12.456 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 10:23:12.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:12.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:12.456 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:23:12.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:12.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:12.456 | INFO | Uploaded 36GiB to GPU in 1.60687s (22.4038GiB/s) | 20260506 10:23:12.456 | INFO | Downloaded 34GiB to GPU in 1.52477s (22.2984GiB/s) | 20260506 10:23:17.595 | INFO | Analyzing chunk frames = [607744,608256) | 20260506 10:23:17.595 | INFO | MemInfo: { | 20260506 10:23:17.595 | INFO | "gpu" : " 17 GB", | 20260506 10:23:17.595 | INFO | "huge" : "", | 20260506 10:23:17.595 | INFO | "huge_shared" : " 27 GB", | 20260506 10:23:17.595 | INFO | "malloc" : " 14 GB", | 20260506 10:23:17.595 | INFO | "pinned" : " 29 GB" | 20260506 10:23:17.595 | INFO | } | 20260506 10:23:17.595 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:23:17.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:23:17.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:17.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:17.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:17.595 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:23:17.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:17.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:17.595 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:23:17.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:17.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:17.595 | INFO | Uploaded 36GiB to GPU in 1.60713s (22.4001GiB/s) | 20260506 10:23:17.595 | INFO | Downloaded 34GiB to GPU in 1.52547s (22.2882GiB/s) | 20260506 10:23:22.734 | INFO | Analyzing chunk frames = [608256,608768) | 20260506 10:23:22.734 | INFO | MemInfo: { | 20260506 10:23:22.734 | INFO | "gpu" : " 17 GB", | 20260506 10:23:22.734 | INFO | "huge" : "", | 20260506 10:23:22.734 | INFO | "huge_shared" : " 27 GB", | 20260506 10:23:22.734 | INFO | "malloc" : " 14 GB", | 20260506 10:23:22.734 | INFO | "pinned" : " 29 GB" | 20260506 10:23:22.734 | INFO | } | 20260506 10:23:22.734 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:23:22.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:23:22.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:22.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:22.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:22.735 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260506 10:23:22.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:22.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:22.735 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:23:22.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:22.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:22.735 | INFO | Uploaded 36GiB to GPU in 1.60492s (22.431GiB/s) | 20260506 10:23:22.735 | INFO | Downloaded 34GiB to GPU in 1.52152s (22.3461GiB/s) | 20260506 10:23:27.873 | INFO | Analyzing chunk frames = [608768,609280) | 20260506 10:23:27.873 | INFO | MemInfo: { | 20260506 10:23:27.873 | INFO | "gpu" : " 17 GB", | 20260506 10:23:27.873 | INFO | "huge" : "", | 20260506 10:23:27.873 | INFO | "huge_shared" : " 27 GB", | 20260506 10:23:27.874 | INFO | "malloc" : " 14 GB", | 20260506 10:23:27.874 | INFO | "pinned" : " 29 GB" | 20260506 10:23:27.874 | INFO | } | 20260506 10:23:27.874 | INFO | Perf report for the last 5.1391 seconds: | 20260506 10:23:27.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:23:27.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:27.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:27.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:27.874 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:23:27.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:27.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:27.874 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:23:27.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:27.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:27.874 | INFO | Uploaded 36GiB to GPU in 1.60572s (22.4199GiB/s) | 20260506 10:23:27.874 | INFO | Downloaded 34GiB to GPU in 1.52243s (22.3327GiB/s) | 20260506 10:23:33.013 | INFO | Analyzing chunk frames = [609280,609792) | 20260506 10:23:33.013 | INFO | MemInfo: { | 20260506 10:23:33.013 | INFO | "gpu" : " 17 GB", | 20260506 10:23:33.013 | INFO | "huge" : "", | 20260506 10:23:33.013 | INFO | "huge_shared" : " 27 GB", | 20260506 10:23:33.013 | INFO | "malloc" : " 14 GB", | 20260506 10:23:33.013 | INFO | "pinned" : " 29 GB" | 20260506 10:23:33.013 | INFO | } | 20260506 10:23:33.013 | INFO | Perf report for the last 5.13952 seconds: | 20260506 10:23:33.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:23:33.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:33.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:33.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:33.013 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:23:33.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:33.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:33.013 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:23:33.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:33.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:33.013 | INFO | Uploaded 36GiB to GPU in 1.60599s (22.4161GiB/s) | 20260506 10:23:33.013 | INFO | Downloaded 34GiB to GPU in 1.52501s (22.2949GiB/s) | 20260506 10:23:38.152 | INFO | Analyzing chunk frames = [609792,610304) | 20260506 10:23:38.152 | INFO | MemInfo: { | 20260506 10:23:38.152 | INFO | "gpu" : " 17 GB", | 20260506 10:23:38.152 | INFO | "huge" : "", | 20260506 10:23:38.152 | INFO | "huge_shared" : " 27 GB", | 20260506 10:23:38.152 | INFO | "malloc" : " 14 GB", | 20260506 10:23:38.152 | INFO | "pinned" : " 29 GB" | 20260506 10:23:38.152 | INFO | } | 20260506 10:23:38.152 | INFO | Perf report for the last 5.13915 seconds: | 20260506 10:23:38.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:23:38.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:38.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:38.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:38.152 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:23:38.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:38.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:38.152 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:23:38.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:38.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:38.153 | INFO | Uploaded 36GiB to GPU in 1.60578s (22.419GiB/s) | 20260506 10:23:38.153 | INFO | Downloaded 34GiB to GPU in 1.52364s (22.315GiB/s) | 20260506 10:23:38.250 | INFO | Starting a round of Baz IO | 20260506 10:23:41.163 | INFO | Populating RTMetrics report | 20260506 10:23:41.163 | INFO | RTMetrics report populated | 20260506 10:23:41.163 | INFO | Opening tmp RTMetrics json file | 20260506 10:23:41.167 | INFO | Serializing RTMetrics to json | 20260506 10:23:41.173 | INFO | Serialization done, swapping tmp file now... | 20260506 10:23:41.175 | INFO | RTMetrics done | 20260506 10:23:41.470 | INFO | Finished a round of Baz IO: Wrote 1.979 GiB in 3.219 seconds | 20260506 10:23:41.470 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.16% / 0.05% / 51.79% | 20260506 10:23:41.470 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.95051% / 0.000103315% / 3.97364e-05% | 20260506 10:23:41.470 | INFO | Baz IO overall: | 20260506 10:23:41.470 | INFO | BytesWritten : 272966342422 | 20260506 10:23:41.470 | INFO | NumEvents : 131111150097 | 20260506 10:23:43.291 | INFO | Analyzing chunk frames = [610304,610816) | 20260506 10:23:43.292 | INFO | MemInfo: { | 20260506 10:23:43.292 | INFO | "gpu" : " 17 GB", | 20260506 10:23:43.292 | INFO | "huge" : "", | 20260506 10:23:43.292 | INFO | "huge_shared" : " 27 GB", | 20260506 10:23:43.292 | INFO | "malloc" : " 14 GB", | 20260506 10:23:43.292 | INFO | "pinned" : " 29 GB" | 20260506 10:23:43.292 | INFO | } | 20260506 10:23:43.292 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:23:43.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:23:43.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:43.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:43.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:43.292 | INFO | ANALYSIS : 57.6%, 2.8, 85.4ms, 0 | 20260506 10:23:43.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:43.292 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 10:23:43.292 | INFO | PRE_HQ : 56.8%, 2.1, 60.6ms, 0 | 20260506 10:23:43.292 | INFO | BAZWRITER : 62.2%, 4.8, 152.6ms, 0 | 20260506 10:23:43.292 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 10:23:43.292 | INFO | Uploaded 36GiB to GPU in 1.67808s (21.4531GiB/s) | 20260506 10:23:43.292 | INFO | Downloaded 38GiB to GPU in 1.7434s (21.7965GiB/s) | 20260506 10:23:48.431 | INFO | Analyzing chunk frames = [610816,611328) | 20260506 10:23:48.431 | INFO | MemInfo: { | 20260506 10:23:48.431 | INFO | "gpu" : " 17 GB", | 20260506 10:23:48.431 | INFO | "huge" : "", | 20260506 10:23:48.431 | INFO | "huge_shared" : " 27 GB", | 20260506 10:23:48.431 | INFO | "malloc" : " 14 GB", | 20260506 10:23:48.431 | INFO | "pinned" : " 29 GB" | 20260506 10:23:48.431 | INFO | } | 20260506 10:23:48.431 | INFO | Perf report for the last 5.13929 seconds: | 20260506 10:23:48.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:23:48.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:48.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:48.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:48.431 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:23:48.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:48.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:48.431 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:23:48.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:48.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:48.432 | INFO | Uploaded 36GiB to GPU in 1.6031s (22.4565GiB/s) | 20260506 10:23:48.432 | INFO | Downloaded 34GiB to GPU in 1.52074s (22.3576GiB/s) | 20260506 10:23:53.570 | INFO | Analyzing chunk frames = [611328,611840) | 20260506 10:23:53.570 | INFO | MemInfo: { | 20260506 10:23:53.570 | INFO | "gpu" : " 17 GB", | 20260506 10:23:53.570 | INFO | "huge" : "", | 20260506 10:23:53.570 | INFO | "huge_shared" : " 27 GB", | 20260506 10:23:53.570 | INFO | "malloc" : " 14 GB", | 20260506 10:23:53.570 | INFO | "pinned" : " 29 GB" | 20260506 10:23:53.570 | INFO | } | 20260506 10:23:53.570 | INFO | Perf report for the last 5.1393 seconds: | 20260506 10:23:53.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:23:53.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:53.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:53.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:53.570 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 10:23:53.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:53.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:53.570 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:23:53.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:53.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:53.571 | INFO | Uploaded 36GiB to GPU in 1.60646s (22.4096GiB/s) | 20260506 10:23:53.571 | INFO | Downloaded 34GiB to GPU in 1.52492s (22.2963GiB/s) | 20260506 10:23:58.709 | INFO | Analyzing chunk frames = [611840,612352) | 20260506 10:23:58.709 | INFO | MemInfo: { | 20260506 10:23:58.709 | INFO | "gpu" : " 17 GB", | 20260506 10:23:58.709 | INFO | "huge" : "", | 20260506 10:23:58.709 | INFO | "huge_shared" : " 27 GB", | 20260506 10:23:58.709 | INFO | "malloc" : " 14 GB", | 20260506 10:23:58.709 | INFO | "pinned" : " 29 GB" | 20260506 10:23:58.710 | INFO | } | 20260506 10:23:58.710 | INFO | Perf report for the last 5.13923 seconds: | 20260506 10:23:58.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:23:58.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:58.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:58.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:58.710 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:23:58.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:58.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:58.710 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:23:58.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:58.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:23:58.710 | INFO | Uploaded 36GiB to GPU in 1.60686s (22.4039GiB/s) | 20260506 10:23:58.710 | INFO | Downloaded 34GiB to GPU in 1.52453s (22.302GiB/s) | 20260506 10:24:03.849 | INFO | Analyzing chunk frames = [612352,612864) | 20260506 10:24:03.849 | INFO | MemInfo: { | 20260506 10:24:03.849 | INFO | "gpu" : " 17 GB", | 20260506 10:24:03.849 | INFO | "huge" : "", | 20260506 10:24:03.849 | INFO | "huge_shared" : " 27 GB", | 20260506 10:24:03.849 | INFO | "malloc" : " 14 GB", | 20260506 10:24:03.849 | INFO | "pinned" : " 29 GB" | 20260506 10:24:03.849 | INFO | } | 20260506 10:24:03.849 | INFO | Perf report for the last 5.13948 seconds: | 20260506 10:24:03.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:24:03.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:03.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:03.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:03.849 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260506 10:24:03.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:03.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:03.849 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:24:03.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:03.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:03.849 | INFO | Uploaded 36GiB to GPU in 1.60564s (22.421GiB/s) | 20260506 10:24:03.849 | INFO | Downloaded 34GiB to GPU in 1.52218s (22.3363GiB/s) | 20260506 10:24:08.988 | INFO | Analyzing chunk frames = [612864,613376) | 20260506 10:24:08.988 | INFO | MemInfo: { | 20260506 10:24:08.988 | INFO | "gpu" : " 17 GB", | 20260506 10:24:08.988 | INFO | "huge" : "", | 20260506 10:24:08.988 | INFO | "huge_shared" : " 27 GB", | 20260506 10:24:08.988 | INFO | "malloc" : " 14 GB", | 20260506 10:24:08.988 | INFO | "pinned" : " 29 GB" | 20260506 10:24:08.988 | INFO | } | 20260506 10:24:08.989 | INFO | Perf report for the last 5.13942 seconds: | 20260506 10:24:08.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:24:08.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:08.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:08.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:08.989 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:24:08.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:08.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:08.989 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:24:08.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:08.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:08.989 | INFO | Uploaded 36GiB to GPU in 1.60561s (22.4214GiB/s) | 20260506 10:24:08.989 | INFO | Downloaded 34GiB to GPU in 1.52292s (22.3255GiB/s) | 20260506 10:24:14.128 | INFO | Analyzing chunk frames = [613376,613888) | 20260506 10:24:14.128 | INFO | MemInfo: { | 20260506 10:24:14.128 | INFO | "gpu" : " 17 GB", | 20260506 10:24:14.128 | INFO | "huge" : "", | 20260506 10:24:14.128 | INFO | "huge_shared" : " 27 GB", | 20260506 10:24:14.128 | INFO | "malloc" : " 14 GB", | 20260506 10:24:14.128 | INFO | "pinned" : " 29 GB" | 20260506 10:24:14.128 | INFO | } | 20260506 10:24:14.128 | INFO | Perf report for the last 5.13959 seconds: | 20260506 10:24:14.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:24:14.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:14.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:14.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:14.128 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:24:14.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:14.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:14.128 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:24:14.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:14.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:14.129 | INFO | Uploaded 36GiB to GPU in 1.60506s (22.4291GiB/s) | 20260506 10:24:14.129 | INFO | Downloaded 34GiB to GPU in 1.5228s (22.3272GiB/s) | 20260506 10:24:19.267 | INFO | Analyzing chunk frames = [613888,614400) | 20260506 10:24:19.267 | INFO | MemInfo: { | 20260506 10:24:19.267 | INFO | "gpu" : " 17 GB", | 20260506 10:24:19.267 | INFO | "huge" : "", | 20260506 10:24:19.267 | INFO | "huge_shared" : " 27 GB", | 20260506 10:24:19.267 | INFO | "malloc" : " 14 GB", | 20260506 10:24:19.267 | INFO | "pinned" : " 29 GB" | 20260506 10:24:19.267 | INFO | } | 20260506 10:24:19.267 | INFO | Perf report for the last 5.13889 seconds: | 20260506 10:24:19.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:24:19.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:19.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:19.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:19.267 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:24:19.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:19.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:19.267 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:24:19.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:19.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:19.268 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:24:19.268 | INFO | Total mem usage: 16.5 GiB | 20260506 10:24:19.268 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:24:19.268 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -10.2%) | 20260506 10:24:19.268 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:24:19.268 | INFO | High Water Breakdown: | 20260506 10:24:19.268 | INFO | 59.6% : 8.93 GiB ( 88.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:24:19.268 | INFO | 40.3% : 6.04 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:24:19.268 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:24:19.268 | INFO | ------------------------------------------ | 20260506 10:24:19.268 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:24:19.268 | INFO | Uploaded 36GiB to GPU in 1.60622s (22.4129GiB/s) | 20260506 10:24:19.268 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:24:19.268 | INFO | Downloaded 34GiB to GPU in 1.52389s (22.3114GiB/s) | 20260506 10:24:19.268 | INFO | Total mem usage: 26.5 GiB | 20260506 10:24:19.268 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:24:19.268 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:24:19.268 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:24:19.268 | INFO | High Water Breakdown: | 20260506 10:24:19.268 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:24:19.268 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:24:19.268 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:24:19.268 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:24:19.268 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:24:19.268 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:24:19.268 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:24:19.268 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:24:19.268 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:24:19.268 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:24:19.268 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:24:19.268 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:24:19.268 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:24:19.268 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:24:19.268 | INFO | ------------------------------------------ | 20260506 10:24:19.268 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:24:19.268 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:24:19.268 | INFO | Total mem usage: 25.6 GiB | 20260506 10:24:19.268 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:24:19.268 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:24:19.268 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:24:19.268 | INFO | High Water Breakdown: | 20260506 10:24:19.268 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:24:19.268 | INFO | ------------------------------------------ | 20260506 10:24:19.268 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:24:19.268 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:24:19.268 | INFO | Total mem usage: 15.7 GiB | 20260506 10:24:19.268 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:24:19.268 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:24:19.268 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:24:19.268 | INFO | High Water Breakdown: | 20260506 10:24:19.268 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:24:19.268 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:24:19.268 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:24:19.268 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:24:19.268 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:24:19.268 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:24:19.268 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:24:19.268 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:24:19.268 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:24:19.268 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:24:19.268 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:24:19.268 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:24:19.269 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:24:19.269 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:24:19.269 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:24:19.269 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:24:19.269 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:24:19.269 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:24:19.269 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:24:19.269 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:24:19.269 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:24:19.269 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:24:19.269 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:24:19.269 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:24:19.269 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:24:19.269 | INFO | ------------------------------------------ | 20260506 10:24:19.269 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:24:19.269 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:24:19.269 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:24:19.269 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:24:19.269 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 10:24:19.269 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.46e-03 / 0.00e+00 | 20260506 10:24:19.269 | INFO | PerfReport: 10.95%, Baseline -- 3.29e+00 / -nan, 1.75e-02 / 0.00e+00 | 20260506 10:24:19.269 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.19e-02 / 0.00e+00 | 20260506 10:24:19.269 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.15e+00 / 0.00e+00 | 20260506 10:24:19.269 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.72e+01 / -nan, 4.24e-02 / 0.00e+00 | 20260506 10:24:19.269 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.23e+00 / -nan, 1.29e-01 / 0.00e+00 | 20260506 10:24:19.269 | INFO | PerfReport: 3.68%, Metrics -- 1.11e+00 / -nan, 1.73e-01 / 0.00e+00 | 20260506 10:24:19.269 | INFO | PerfReport: 1.42%, PulseSerializing -- 4.29e-01 / -nan, 6.09e-02 / 0.00e+00 | 20260506 10:24:19.269 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:24:19.269 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:24:19.269 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:24:19.269 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 10:24:19.269 | INFO | PerfReport: 18.72%, Upload -- 1.68e+01 / 0.00e+00, 4.94e-01 / 0.00e+00 | 20260506 10:24:19.269 | INFO | PerfReport: 17.98%, Download -- 1.62e+01 / 0.00e+00, 8.54e-01 / 0.00e+00 | 20260506 10:24:19.269 | INFO | PerfReport: 33.50%, Compute -- 3.01e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 10:24:19.269 | INFO | PerfReport: 0.57%, LockUpload -- 5.09e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 10:24:19.269 | INFO | PerfReport: 0.00%, LockDownload -- 4.98e-04 / 0.00e+00, 1.32e-04 / 0.00e+00 | 20260506 10:24:19.269 | INFO | PerfReport: 29.22%, LockCompute -- 2.62e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 10:24:19.269 | INFO | PerfReport: 0.02%, Misc -- 1.35e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260506 10:24:19.365 | INFO | Starting a round of Baz IO | 20260506 10:24:22.276 | INFO | Populating RTMetrics report | 20260506 10:24:22.277 | INFO | RTMetrics report populated | 20260506 10:24:22.277 | INFO | Opening tmp RTMetrics json file | 20260506 10:24:22.312 | INFO | Serializing RTMetrics to json | 20260506 10:24:22.323 | INFO | Serialization done, swapping tmp file now... | 20260506 10:24:22.323 | INFO | RTMetrics done | 20260506 10:24:22.488 | INFO | Finished a round of Baz IO: Wrote 1.98 GiB in 3.123 seconds | 20260506 10:24:22.488 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.10% / 0.05% / 51.84% | 20260506 10:24:22.488 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.92796% / 0.000127157% / 4.37101e-05% | 20260506 10:24:22.488 | INFO | Baz IO overall: | 20260506 10:24:22.488 | INFO | BytesWritten : 275092429760 | 20260506 10:24:22.488 | INFO | NumEvents : 132142799872 | 20260506 10:24:24.406 | INFO | Analyzing chunk frames = [614400,614912) | 20260506 10:24:24.406 | INFO | MemInfo: { | 20260506 10:24:24.406 | INFO | "gpu" : " 17 GB", | 20260506 10:24:24.406 | INFO | "huge" : "", | 20260506 10:24:24.406 | INFO | "huge_shared" : " 27 GB", | 20260506 10:24:24.406 | INFO | "malloc" : " 14 GB", | 20260506 10:24:24.406 | INFO | "pinned" : " 29 GB" | 20260506 10:24:24.406 | INFO | } | 20260506 10:24:24.406 | INFO | Perf report for the last 5.13939 seconds: | 20260506 10:24:24.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:24:24.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:24.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:24.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:24.406 | INFO | ANALYSIS : 57.5%, 2.8, 85.1ms, 0 | 20260506 10:24:24.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:24.406 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 10:24:24.407 | INFO | PRE_HQ : 56.6%, 2.1, 60.8ms, 0 | 20260506 10:24:24.407 | INFO | BAZWRITER : 60.4%, 4.8, 148.9ms, 0 | 20260506 10:24:24.407 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 10:24:24.407 | INFO | Uploaded 36GiB to GPU in 1.67428s (21.5018GiB/s) | 20260506 10:24:24.407 | INFO | Downloaded 38GiB to GPU in 1.73894s (21.8524GiB/s) | 20260506 10:24:29.546 | INFO | Analyzing chunk frames = [614912,615424) | 20260506 10:24:29.546 | INFO | MemInfo: { | 20260506 10:24:29.546 | INFO | "gpu" : " 17 GB", | 20260506 10:24:29.546 | INFO | "huge" : "", | 20260506 10:24:29.546 | INFO | "huge_shared" : " 27 GB", | 20260506 10:24:29.546 | INFO | "malloc" : " 14 GB", | 20260506 10:24:29.546 | INFO | "pinned" : " 29 GB" | 20260506 10:24:29.546 | INFO | } | 20260506 10:24:29.546 | INFO | Perf report for the last 5.13968 seconds: | 20260506 10:24:29.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:24:29.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:29.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:29.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:29.546 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:24:29.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:29.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:29.546 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260506 10:24:29.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:29.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:29.546 | INFO | Uploaded 36GiB to GPU in 1.603s (22.4579GiB/s) | 20260506 10:24:29.546 | INFO | Downloaded 34GiB to GPU in 1.52035s (22.3632GiB/s) | 20260506 10:24:34.685 | INFO | Analyzing chunk frames = [615424,615936) | 20260506 10:24:34.685 | INFO | MemInfo: { | 20260506 10:24:34.685 | INFO | "gpu" : " 17 GB", | 20260506 10:24:34.685 | INFO | "huge" : "", | 20260506 10:24:34.685 | INFO | "huge_shared" : " 27 GB", | 20260506 10:24:34.685 | INFO | "malloc" : " 14 GB", | 20260506 10:24:34.685 | INFO | "pinned" : " 29 GB" | 20260506 10:24:34.685 | INFO | } | 20260506 10:24:34.685 | INFO | Perf report for the last 5.13899 seconds: | 20260506 10:24:34.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:24:34.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:34.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:34.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:34.685 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:24:34.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:34.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:34.685 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:24:34.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:34.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:34.685 | INFO | Uploaded 36GiB to GPU in 1.60747s (22.3954GiB/s) | 20260506 10:24:34.685 | INFO | Downloaded 34GiB to GPU in 1.52533s (22.2902GiB/s) | 20260506 10:24:39.824 | INFO | Analyzing chunk frames = [615936,616448) | 20260506 10:24:39.824 | INFO | MemInfo: { | 20260506 10:24:39.824 | INFO | "gpu" : " 17 GB", | 20260506 10:24:39.824 | INFO | "huge" : "", | 20260506 10:24:39.824 | INFO | "huge_shared" : " 27 GB", | 20260506 10:24:39.824 | INFO | "malloc" : " 14 GB", | 20260506 10:24:39.824 | INFO | "pinned" : " 29 GB" | 20260506 10:24:39.824 | INFO | } | 20260506 10:24:39.824 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:24:39.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:24:39.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:39.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:39.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:39.825 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:24:39.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:39.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:39.825 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:24:39.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:39.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:39.825 | INFO | Uploaded 36GiB to GPU in 1.60677s (22.4051GiB/s) | 20260506 10:24:39.825 | INFO | Downloaded 34GiB to GPU in 1.52403s (22.3092GiB/s) | 20260506 10:24:44.964 | INFO | Analyzing chunk frames = [616448,616960) | 20260506 10:24:44.964 | INFO | MemInfo: { | 20260506 10:24:44.964 | INFO | "gpu" : " 17 GB", | 20260506 10:24:44.964 | INFO | "huge" : "", | 20260506 10:24:44.964 | INFO | "huge_shared" : " 27 GB", | 20260506 10:24:44.964 | INFO | "malloc" : " 14 GB", | 20260506 10:24:44.964 | INFO | "pinned" : " 29 GB" | 20260506 10:24:44.964 | INFO | } | 20260506 10:24:44.964 | INFO | Perf report for the last 5.13944 seconds: | 20260506 10:24:44.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:24:44.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:44.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:44.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:44.964 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:24:44.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:44.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:44.964 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:24:44.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:44.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:44.964 | INFO | Uploaded 36GiB to GPU in 1.60531s (22.4255GiB/s) | 20260506 10:24:44.964 | INFO | Downloaded 34GiB to GPU in 1.52236s (22.3338GiB/s) | 20260506 10:24:50.103 | INFO | Analyzing chunk frames = [616960,617472) | 20260506 10:24:50.103 | INFO | MemInfo: { | 20260506 10:24:50.103 | INFO | "gpu" : " 17 GB", | 20260506 10:24:50.103 | INFO | "huge" : "", | 20260506 10:24:50.103 | INFO | "huge_shared" : " 27 GB", | 20260506 10:24:50.103 | INFO | "malloc" : " 14 GB", | 20260506 10:24:50.103 | INFO | "pinned" : " 29 GB" | 20260506 10:24:50.103 | INFO | } | 20260506 10:24:50.103 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:24:50.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:24:50.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:50.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:50.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:50.103 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:24:50.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:50.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:50.103 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:24:50.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:50.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:50.104 | INFO | Uploaded 36GiB to GPU in 1.60532s (22.4255GiB/s) | 20260506 10:24:50.104 | INFO | Downloaded 34GiB to GPU in 1.5226s (22.3302GiB/s) | 20260506 10:24:55.242 | INFO | Analyzing chunk frames = [617472,617984) | 20260506 10:24:55.242 | INFO | MemInfo: { | 20260506 10:24:55.242 | INFO | "gpu" : " 17 GB", | 20260506 10:24:55.242 | INFO | "huge" : "", | 20260506 10:24:55.242 | INFO | "huge_shared" : " 27 GB", | 20260506 10:24:55.242 | INFO | "malloc" : " 14 GB", | 20260506 10:24:55.242 | INFO | "pinned" : " 29 GB" | 20260506 10:24:55.243 | INFO | } | 20260506 10:24:55.243 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:24:55.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:24:55.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:55.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:55.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:55.243 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:24:55.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:55.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:55.243 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:24:55.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:55.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:24:55.243 | INFO | Uploaded 36GiB to GPU in 1.60526s (22.4263GiB/s) | 20260506 10:24:55.243 | INFO | Downloaded 34GiB to GPU in 1.52347s (22.3174GiB/s) | 20260506 10:25:00.382 | INFO | Analyzing chunk frames = [617984,618496) | 20260506 10:25:00.382 | INFO | MemInfo: { | 20260506 10:25:00.382 | INFO | "gpu" : " 17 GB", | 20260506 10:25:00.382 | INFO | "huge" : "", | 20260506 10:25:00.382 | INFO | "huge_shared" : " 27 GB", | 20260506 10:25:00.382 | INFO | "malloc" : " 14 GB", | 20260506 10:25:00.382 | INFO | "pinned" : " 29 GB" | 20260506 10:25:00.382 | INFO | } | 20260506 10:25:00.382 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:25:00.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:25:00.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:00.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:00.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:00.382 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:25:00.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:00.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:00.382 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 10:25:00.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:00.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:00.382 | INFO | Uploaded 36GiB to GPU in 1.60569s (22.4202GiB/s) | 20260506 10:25:00.382 | INFO | Downloaded 34GiB to GPU in 1.52378s (22.3129GiB/s) | 20260506 10:25:00.479 | INFO | Starting a round of Baz IO | 20260506 10:25:03.392 | INFO | Populating RTMetrics report | 20260506 10:25:03.392 | INFO | RTMetrics report populated | 20260506 10:25:03.392 | INFO | Opening tmp RTMetrics json file | 20260506 10:25:03.400 | INFO | Serializing RTMetrics to json | 20260506 10:25:03.425 | INFO | Serialization done, swapping tmp file now... | 20260506 10:25:03.427 | INFO | RTMetrics done | 20260506 10:25:03.882 | INFO | Finished a round of Baz IO: Wrote 1.98 GiB in 3.402 seconds | 20260506 10:25:03.882 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.05% / 0.05% / 51.90% | 20260506 10:25:03.882 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.89476% / 9.13938e-05% / 4.76837e-05% | 20260506 10:25:03.882 | INFO | Baz IO overall: | 20260506 10:25:03.882 | INFO | BytesWritten : 277218763257 | 20260506 10:25:03.882 | INFO | NumEvents : 133174330584 | 20260506 10:25:05.521 | INFO | Analyzing chunk frames = [618496,619008) | 20260506 10:25:05.521 | INFO | MemInfo: { | 20260506 10:25:05.521 | INFO | "gpu" : " 17 GB", | 20260506 10:25:05.521 | INFO | "huge" : "", | 20260506 10:25:05.521 | INFO | "huge_shared" : " 27 GB", | 20260506 10:25:05.521 | INFO | "malloc" : " 14 GB", | 20260506 10:25:05.521 | INFO | "pinned" : " 29 GB" | 20260506 10:25:05.521 | INFO | } | 20260506 10:25:05.521 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:25:05.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:25:05.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:05.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:05.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:05.521 | INFO | ANALYSIS : 57.6%, 2.8, 85.6ms, 0 | 20260506 10:25:05.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:05.521 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 10:25:05.521 | INFO | PRE_HQ : 56.6%, 2.1, 60.5ms, 0 | 20260506 10:25:05.521 | INFO | BAZWRITER : 65.7%, 4.8, 160.6ms, 0 | 20260506 10:25:05.521 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 10:25:05.522 | INFO | Uploaded 36GiB to GPU in 1.68033s (21.4243GiB/s) | 20260506 10:25:05.522 | INFO | Downloaded 38GiB to GPU in 1.74453s (21.7823GiB/s) | 20260506 10:25:10.660 | INFO | Analyzing chunk frames = [619008,619520) | 20260506 10:25:10.660 | INFO | MemInfo: { | 20260506 10:25:10.660 | INFO | "gpu" : " 17 GB", | 20260506 10:25:10.660 | INFO | "huge" : "", | 20260506 10:25:10.660 | INFO | "huge_shared" : " 27 GB", | 20260506 10:25:10.660 | INFO | "malloc" : " 14 GB", | 20260506 10:25:10.660 | INFO | "pinned" : " 29 GB" | 20260506 10:25:10.660 | INFO | } | 20260506 10:25:10.661 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:25:10.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:25:10.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:10.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:10.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:10.661 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:25:10.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:10.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:10.661 | INFO | PRE_HQ : 29.4%, 1.0, 15.4ms, 0 | 20260506 10:25:10.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:10.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:10.661 | INFO | Uploaded 36GiB to GPU in 1.60298s (22.4582GiB/s) | 20260506 10:25:10.661 | INFO | Downloaded 34GiB to GPU in 1.52019s (22.3657GiB/s) | 20260506 10:25:15.800 | INFO | Analyzing chunk frames = [619520,620032) | 20260506 10:25:15.800 | INFO | MemInfo: { | 20260506 10:25:15.800 | INFO | "gpu" : " 17 GB", | 20260506 10:25:15.800 | INFO | "huge" : "", | 20260506 10:25:15.800 | INFO | "huge_shared" : " 27 GB", | 20260506 10:25:15.800 | INFO | "malloc" : " 14 GB", | 20260506 10:25:15.800 | INFO | "pinned" : " 29 GB" | 20260506 10:25:15.800 | INFO | } | 20260506 10:25:15.800 | INFO | Perf report for the last 5.13942 seconds: | 20260506 10:25:15.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:25:15.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:15.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:15.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:15.800 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:25:15.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:15.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:15.800 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:25:15.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:15.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:15.801 | INFO | Uploaded 36GiB to GPU in 1.60749s (22.3951GiB/s) | 20260506 10:25:15.801 | INFO | Downloaded 34GiB to GPU in 1.52499s (22.2952GiB/s) | 20260506 10:25:20.939 | INFO | Analyzing chunk frames = [620032,620544) | 20260506 10:25:20.939 | INFO | MemInfo: { | 20260506 10:25:20.939 | INFO | "gpu" : " 17 GB", | 20260506 10:25:20.939 | INFO | "huge" : "", | 20260506 10:25:20.939 | INFO | "huge_shared" : " 27 GB", | 20260506 10:25:20.939 | INFO | "malloc" : " 14 GB", | 20260506 10:25:20.939 | INFO | "pinned" : " 29 GB" | 20260506 10:25:20.939 | INFO | } | 20260506 10:25:20.939 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:25:20.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:25:20.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:20.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:20.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:20.939 | INFO | ANALYSIS : 56.5%, 3.0, 87.5ms, 0 | 20260506 10:25:20.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:20.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:20.939 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 10:25:20.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:20.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:20.940 | INFO | Uploaded 36GiB to GPU in 1.60655s (22.4082GiB/s) | 20260506 10:25:20.940 | INFO | Downloaded 34GiB to GPU in 1.52389s (22.3114GiB/s) | 20260506 10:25:26.079 | INFO | Analyzing chunk frames = [620544,621056) | 20260506 10:25:26.079 | INFO | MemInfo: { | 20260506 10:25:26.079 | INFO | "gpu" : " 17 GB", | 20260506 10:25:26.079 | INFO | "huge" : "", | 20260506 10:25:26.079 | INFO | "huge_shared" : " 27 GB", | 20260506 10:25:26.079 | INFO | "malloc" : " 14 GB", | 20260506 10:25:26.079 | INFO | "pinned" : " 29 GB" | 20260506 10:25:26.079 | INFO | } | 20260506 10:25:26.079 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:25:26.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:25:26.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:26.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:26.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:26.079 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:25:26.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:26.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:26.079 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:25:26.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:26.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:26.079 | INFO | Uploaded 36GiB to GPU in 1.6054s (22.4243GiB/s) | 20260506 10:25:26.079 | INFO | Downloaded 34GiB to GPU in 1.52206s (22.3381GiB/s) | 20260506 10:25:31.218 | INFO | Analyzing chunk frames = [621056,621568) | 20260506 10:25:31.218 | INFO | MemInfo: { | 20260506 10:25:31.218 | INFO | "gpu" : " 17 GB", | 20260506 10:25:31.218 | INFO | "huge" : "", | 20260506 10:25:31.218 | INFO | "huge_shared" : " 27 GB", | 20260506 10:25:31.218 | INFO | "malloc" : " 14 GB", | 20260506 10:25:31.218 | INFO | "pinned" : " 29 GB" | 20260506 10:25:31.218 | INFO | } | 20260506 10:25:31.218 | INFO | Perf report for the last 5.1395 seconds: | 20260506 10:25:31.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:25:31.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:31.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:31.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:31.218 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:25:31.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:31.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:31.218 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:25:31.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:31.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:31.218 | INFO | Uploaded 36GiB to GPU in 1.60588s (22.4177GiB/s) | 20260506 10:25:31.218 | INFO | Downloaded 34GiB to GPU in 1.52316s (22.322GiB/s) | 20260506 10:25:36.357 | INFO | Analyzing chunk frames = [621568,622080) | 20260506 10:25:36.357 | INFO | MemInfo: { | 20260506 10:25:36.357 | INFO | "gpu" : " 17 GB", | 20260506 10:25:36.357 | INFO | "huge" : "", | 20260506 10:25:36.357 | INFO | "huge_shared" : " 27 GB", | 20260506 10:25:36.357 | INFO | "malloc" : " 14 GB", | 20260506 10:25:36.357 | INFO | "pinned" : " 29 GB" | 20260506 10:25:36.357 | INFO | } | 20260506 10:25:36.357 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:25:36.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:25:36.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:36.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:36.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:36.357 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:25:36.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:36.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:36.358 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:25:36.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:36.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:36.358 | INFO | Uploaded 36GiB to GPU in 1.60436s (22.4389GiB/s) | 20260506 10:25:36.358 | INFO | Downloaded 34GiB to GPU in 1.52382s (22.3124GiB/s) | 20260506 10:25:41.497 | INFO | Analyzing chunk frames = [622080,622592) | 20260506 10:25:41.497 | INFO | MemInfo: { | 20260506 10:25:41.497 | INFO | "gpu" : " 17 GB", | 20260506 10:25:41.497 | INFO | "huge" : "", | 20260506 10:25:41.497 | INFO | "huge_shared" : " 27 GB", | 20260506 10:25:41.497 | INFO | "malloc" : " 14 GB", | 20260506 10:25:41.497 | INFO | "pinned" : " 29 GB" | 20260506 10:25:41.497 | INFO | } | 20260506 10:25:41.497 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:25:41.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:25:41.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:41.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:41.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:41.497 | INFO | ANALYSIS : 57.0%, 3.0, 88.9ms, 0 | 20260506 10:25:41.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:41.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:41.497 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:25:41.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:41.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:41.497 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:25:41.497 | INFO | Total mem usage: 16.5 GiB | 20260506 10:25:41.497 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:25:41.497 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -4.2%) | 20260506 10:25:41.497 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:25:41.497 | INFO | High Water Breakdown: | 20260506 10:25:41.497 | INFO | 60.6% : 9.6 GiB ( 95.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:25:41.497 | INFO | 39.3% : 6.23 GiB ( 93.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:25:41.497 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:25:41.497 | INFO | ------------------------------------------ | 20260506 10:25:41.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:25:41.497 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:25:41.497 | INFO | Uploaded 36GiB to GPU in 1.60564s (22.4209GiB/s) | 20260506 10:25:41.497 | INFO | Total mem usage: 26.5 GiB | 20260506 10:25:41.497 | INFO | Downloaded 34GiB to GPU in 1.52384s (22.3121GiB/s) | 20260506 10:25:41.498 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:25:41.498 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:25:41.498 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:25:41.498 | INFO | High Water Breakdown: | 20260506 10:25:41.498 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:25:41.498 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:25:41.498 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:25:41.498 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:25:41.498 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:25:41.498 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:25:41.498 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:25:41.498 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:25:41.498 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:25:41.498 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:25:41.498 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:25:41.498 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:25:41.498 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:25:41.498 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:25:41.498 | INFO | ------------------------------------------ | 20260506 10:25:41.498 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:25:41.498 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:25:41.498 | INFO | Total mem usage: 25.6 GiB | 20260506 10:25:41.498 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:25:41.498 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:25:41.498 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:25:41.498 | INFO | High Water Breakdown: | 20260506 10:25:41.498 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:25:41.498 | INFO | ------------------------------------------ | 20260506 10:25:41.498 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:25:41.498 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:25:41.498 | INFO | Total mem usage: 15.7 GiB | 20260506 10:25:41.498 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:25:41.498 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:25:41.498 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:25:41.498 | INFO | High Water Breakdown: | 20260506 10:25:41.498 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:25:41.498 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:25:41.498 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:25:41.498 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:25:41.498 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:25:41.498 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:25:41.498 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:25:41.498 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:25:41.498 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:25:41.498 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:25:41.498 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:25:41.498 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:25:41.498 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:25:41.498 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:25:41.498 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:25:41.498 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:25:41.498 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:25:41.498 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:25:41.498 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:25:41.498 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:25:41.498 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:25:41.498 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:25:41.498 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:25:41.498 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:25:41.498 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:25:41.498 | INFO | ------------------------------------------ | 20260506 10:25:41.498 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:25:41.498 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:25:41.498 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:25:41.498 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:25:41.498 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 10:25:41.498 | INFO | PerfReport: 0.29%, AggregateStats -- 8.67e-02 / -nan, 5.66e-03 / 0.00e+00 | 20260506 10:25:41.498 | INFO | PerfReport: 10.95%, Baseline -- 3.30e+00 / -nan, 1.79e-02 / 0.00e+00 | 20260506 10:25:41.498 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260506 10:25:41.498 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.10e+00 / 0.00e+00 | 20260506 10:25:41.498 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.72e+01 / -nan, 4.18e-02 / 0.00e+00 | 20260506 10:25:41.498 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.22e+00 / -nan, 1.28e-01 / 0.00e+00 | 20260506 10:25:41.498 | INFO | PerfReport: 3.68%, Metrics -- 1.11e+00 / -nan, 1.72e-01 / 0.00e+00 | 20260506 10:25:41.498 | INFO | PerfReport: 1.42%, PulseSerializing -- 4.28e-01 / -nan, 6.06e-02 / 0.00e+00 | 20260506 10:25:41.498 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:25:41.498 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:25:41.498 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:25:41.498 | INFO | PerfReport: Full Execution -- 8.99e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 10:25:41.498 | INFO | PerfReport: 18.72%, Upload -- 1.68e+01 / 0.00e+00, 5.04e-01 / 0.00e+00 | 20260506 10:25:41.498 | INFO | PerfReport: 17.98%, Download -- 1.62e+01 / 0.00e+00, 8.61e-01 / 0.00e+00 | 20260506 10:25:41.498 | INFO | PerfReport: 33.50%, Compute -- 3.01e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 10:25:41.498 | INFO | PerfReport: 0.57%, LockUpload -- 5.10e-01 / 0.00e+00, 3.64e+00 / 0.00e+00 | 20260506 10:25:41.498 | INFO | PerfReport: 0.00%, LockDownload -- 4.84e-04 / 0.00e+00, 1.28e-04 / 0.00e+00 | 20260506 10:25:41.498 | INFO | PerfReport: 29.23%, LockCompute -- 2.63e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 10:25:41.498 | INFO | PerfReport: 0.02%, Misc -- 1.35e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260506 10:25:41.594 | INFO | Starting a round of Baz IO | 20260506 10:25:44.502 | INFO | Populating RTMetrics report | 20260506 10:25:44.503 | INFO | RTMetrics report populated | 20260506 10:25:44.503 | INFO | Opening tmp RTMetrics json file | 20260506 10:25:44.521 | INFO | Serializing RTMetrics to json | 20260506 10:25:44.587 | INFO | Serialization done, swapping tmp file now... | 20260506 10:25:44.592 | INFO | RTMetrics done | 20260506 10:25:45.087 | INFO | Finished a round of Baz IO: Wrote 1.981 GiB in 3.492 seconds | 20260506 10:25:45.087 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 48.00% / 0.05% / 51.95% | 20260506 10:25:45.087 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.87551% / 9.13938e-05% / 4.37101e-05% | 20260506 10:25:45.087 | INFO | Baz IO overall: | 20260506 10:25:45.087 | INFO | BytesWritten : 279345912430 | 20260506 10:25:45.087 | INFO | NumEvents : 134206656326 | 20260506 10:25:46.636 | INFO | Analyzing chunk frames = [622592,623104) | 20260506 10:25:46.636 | INFO | MemInfo: { | 20260506 10:25:46.636 | INFO | "gpu" : " 17 GB", | 20260506 10:25:46.636 | INFO | "huge" : "", | 20260506 10:25:46.636 | INFO | "huge_shared" : " 27 GB", | 20260506 10:25:46.636 | INFO | "malloc" : " 14 GB", | 20260506 10:25:46.636 | INFO | "pinned" : " 29 GB" | 20260506 10:25:46.636 | INFO | } | 20260506 10:25:46.636 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:25:46.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:25:46.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:46.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:46.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:46.636 | INFO | ANALYSIS : 57.5%, 2.8, 85.3ms, 0 | 20260506 10:25:46.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:46.636 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260506 10:25:46.636 | INFO | PRE_HQ : 56.6%, 2.1, 60.8ms, 0 | 20260506 10:25:46.636 | INFO | BAZWRITER : 67.5%, 4.7, 164.6ms, 0 | 20260506 10:25:46.636 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 10:25:46.637 | INFO | Uploaded 36GiB to GPU in 1.6731s (21.5169GiB/s) | 20260506 10:25:46.637 | INFO | Downloaded 38GiB to GPU in 1.73927s (21.8482GiB/s) | 20260506 10:25:51.775 | INFO | Analyzing chunk frames = [623104,623616) | 20260506 10:25:51.775 | INFO | MemInfo: { | 20260506 10:25:51.775 | INFO | "gpu" : " 17 GB", | 20260506 10:25:51.775 | INFO | "huge" : "", | 20260506 10:25:51.775 | INFO | "huge_shared" : " 27 GB", | 20260506 10:25:51.775 | INFO | "malloc" : " 14 GB", | 20260506 10:25:51.775 | INFO | "pinned" : " 29 GB" | 20260506 10:25:51.775 | INFO | } | 20260506 10:25:51.775 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:25:51.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:25:51.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:51.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:51.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:51.775 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:25:51.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:51.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:51.776 | INFO | PRE_HQ : 29.6%, 1.0, 15.6ms, 0 | 20260506 10:25:51.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:51.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:51.776 | INFO | Uploaded 36GiB to GPU in 1.60269s (22.4622GiB/s) | 20260506 10:25:51.776 | INFO | Downloaded 34GiB to GPU in 1.51953s (22.3754GiB/s) | 20260506 10:25:56.915 | INFO | Analyzing chunk frames = [623616,624128) | 20260506 10:25:56.915 | INFO | MemInfo: { | 20260506 10:25:56.915 | INFO | "gpu" : " 17 GB", | 20260506 10:25:56.915 | INFO | "huge" : "", | 20260506 10:25:56.915 | INFO | "huge_shared" : " 27 GB", | 20260506 10:25:56.915 | INFO | "malloc" : " 14 GB", | 20260506 10:25:56.915 | INFO | "pinned" : " 29 GB" | 20260506 10:25:56.915 | INFO | } | 20260506 10:25:56.915 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:25:56.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:25:56.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:56.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:56.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:56.915 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:25:56.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:56.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:56.915 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:25:56.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:56.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:25:56.915 | INFO | Uploaded 36GiB to GPU in 1.6098s (22.363GiB/s) | 20260506 10:25:56.915 | INFO | Downloaded 34GiB to GPU in 1.52917s (22.2343GiB/s) | 20260506 10:26:02.054 | INFO | Analyzing chunk frames = [624128,624640) | 20260506 10:26:02.054 | INFO | MemInfo: { | 20260506 10:26:02.054 | INFO | "gpu" : " 17 GB", | 20260506 10:26:02.054 | INFO | "huge" : "", | 20260506 10:26:02.054 | INFO | "huge_shared" : " 27 GB", | 20260506 10:26:02.054 | INFO | "malloc" : " 14 GB", | 20260506 10:26:02.054 | INFO | "pinned" : " 29 GB" | 20260506 10:26:02.054 | INFO | } | 20260506 10:26:02.054 | INFO | Perf report for the last 5.13925 seconds: | 20260506 10:26:02.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:26:02.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:02.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:02.054 | INFO | SAVE_TRACE : 0.1%, 1.0, 0.0ms, 0 | 20260506 10:26:02.054 | INFO | ANALYSIS : 56.5%, 3.0, 87.5ms, 0 | 20260506 10:26:02.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:02.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:02.054 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:26:02.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:02.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:02.054 | INFO | Uploaded 36GiB to GPU in 1.60694s (22.4029GiB/s) | 20260506 10:26:02.055 | INFO | Downloaded 34GiB to GPU in 1.5241s (22.3083GiB/s) | 20260506 10:26:07.193 | INFO | Analyzing chunk frames = [624640,625152) | 20260506 10:26:07.193 | INFO | MemInfo: { | 20260506 10:26:07.193 | INFO | "gpu" : " 17 GB", | 20260506 10:26:07.193 | INFO | "huge" : "", | 20260506 10:26:07.193 | INFO | "huge_shared" : " 27 GB", | 20260506 10:26:07.193 | INFO | "malloc" : " 14 GB", | 20260506 10:26:07.193 | INFO | "pinned" : " 29 GB" | 20260506 10:26:07.194 | INFO | } | 20260506 10:26:07.194 | INFO | Perf report for the last 5.13945 seconds: | 20260506 10:26:07.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:26:07.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:07.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:07.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:07.194 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:26:07.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:07.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:07.194 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:26:07.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:07.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:07.194 | INFO | Uploaded 36GiB to GPU in 1.60554s (22.4224GiB/s) | 20260506 10:26:07.194 | INFO | Downloaded 34GiB to GPU in 1.52259s (22.3304GiB/s) | 20260506 10:26:12.333 | INFO | Analyzing chunk frames = [625152,625664) | 20260506 10:26:12.333 | INFO | MemInfo: { | 20260506 10:26:12.333 | INFO | "gpu" : " 17 GB", | 20260506 10:26:12.333 | INFO | "huge" : "", | 20260506 10:26:12.333 | INFO | "huge_shared" : " 27 GB", | 20260506 10:26:12.333 | INFO | "malloc" : " 14 GB", | 20260506 10:26:12.333 | INFO | "pinned" : " 29 GB" | 20260506 10:26:12.333 | INFO | } | 20260506 10:26:12.333 | INFO | Perf report for the last 5.13921 seconds: | 20260506 10:26:12.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:26:12.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:12.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:12.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:12.333 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:26:12.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:12.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:12.333 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:26:12.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:12.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:12.333 | INFO | Uploaded 36GiB to GPU in 1.60529s (22.4259GiB/s) | 20260506 10:26:12.333 | INFO | Downloaded 34GiB to GPU in 1.52227s (22.3351GiB/s) | 20260506 10:26:17.472 | INFO | Analyzing chunk frames = [625664,626176) | 20260506 10:26:17.472 | INFO | MemInfo: { | 20260506 10:26:17.472 | INFO | "gpu" : " 17 GB", | 20260506 10:26:17.472 | INFO | "huge" : "", | 20260506 10:26:17.472 | INFO | "huge_shared" : " 27 GB", | 20260506 10:26:17.472 | INFO | "malloc" : " 14 GB", | 20260506 10:26:17.472 | INFO | "pinned" : " 29 GB" | 20260506 10:26:17.472 | INFO | } | 20260506 10:26:17.472 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:26:17.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:26:17.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:17.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:17.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:17.472 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:26:17.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:17.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:17.472 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:26:17.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:17.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:17.472 | INFO | Uploaded 36GiB to GPU in 1.60653s (22.4085GiB/s) | 20260506 10:26:17.472 | INFO | Downloaded 34GiB to GPU in 1.52543s (22.2887GiB/s) | 20260506 10:26:22.611 | INFO | Analyzing chunk frames = [626176,626688) | 20260506 10:26:22.611 | INFO | MemInfo: { | 20260506 10:26:22.611 | INFO | "gpu" : " 17 GB", | 20260506 10:26:22.611 | INFO | "huge" : "", | 20260506 10:26:22.611 | INFO | "huge_shared" : " 27 GB", | 20260506 10:26:22.611 | INFO | "malloc" : " 14 GB", | 20260506 10:26:22.611 | INFO | "pinned" : " 29 GB" | 20260506 10:26:22.611 | INFO | } | 20260506 10:26:22.612 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:26:22.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:26:22.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:22.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:22.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:22.612 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:26:22.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:22.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:22.612 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 10:26:22.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:22.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:22.612 | INFO | Uploaded 36GiB to GPU in 1.60597s (22.4163GiB/s) | 20260506 10:26:22.612 | INFO | Downloaded 34GiB to GPU in 1.52357s (22.316GiB/s) | 20260506 10:26:22.709 | INFO | Starting a round of Baz IO | 20260506 10:26:25.623 | INFO | Populating RTMetrics report | 20260506 10:26:25.623 | INFO | RTMetrics report populated | 20260506 10:26:25.623 | INFO | Opening tmp RTMetrics json file | 20260506 10:26:25.668 | INFO | Serializing RTMetrics to json | 20260506 10:26:25.735 | INFO | Serialization done, swapping tmp file now... | 20260506 10:26:25.736 | INFO | RTMetrics done | 20260506 10:26:25.989 | INFO | Finished a round of Baz IO: Wrote 1.981 GiB in 3.279 seconds | 20260506 10:26:25.989 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.94% / 0.05% / 52.00% | 20260506 10:26:25.989 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.84456% / 0.000135104% / 2.78155e-05% | 20260506 10:26:25.989 | INFO | Baz IO overall: | 20260506 10:26:25.989 | INFO | BytesWritten : 281473456141 | 20260506 10:26:25.989 | INFO | NumEvents : 135238843372 | 20260506 10:26:27.751 | INFO | Analyzing chunk frames = [626688,627200) | 20260506 10:26:27.751 | INFO | MemInfo: { | 20260506 10:26:27.751 | INFO | "gpu" : " 17 GB", | 20260506 10:26:27.751 | INFO | "huge" : "", | 20260506 10:26:27.751 | INFO | "huge_shared" : " 27 GB", | 20260506 10:26:27.751 | INFO | "malloc" : " 14 GB", | 20260506 10:26:27.751 | INFO | "pinned" : " 29 GB" | 20260506 10:26:27.751 | INFO | } | 20260506 10:26:27.751 | INFO | Perf report for the last 5.13942 seconds: | 20260506 10:26:27.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:26:27.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:27.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:27.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:27.751 | INFO | ANALYSIS : 57.6%, 2.9, 86.6ms, 0 | 20260506 10:26:27.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:27.751 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260506 10:26:27.751 | INFO | PRE_HQ : 56.7%, 2.1, 61.3ms, 0 | 20260506 10:26:27.751 | INFO | BAZWRITER : 63.4%, 4.7, 151.6ms, 0 | 20260506 10:26:27.751 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260506 10:26:27.752 | INFO | Uploaded 36GiB to GPU in 1.65723s (21.7231GiB/s) | 20260506 10:26:27.752 | INFO | Downloaded 38GiB to GPU in 1.72231s (22.0634GiB/s) | 20260506 10:26:32.890 | INFO | Analyzing chunk frames = [627200,627712) | 20260506 10:26:32.890 | INFO | MemInfo: { | 20260506 10:26:32.890 | INFO | "gpu" : " 17 GB", | 20260506 10:26:32.890 | INFO | "huge" : "", | 20260506 10:26:32.890 | INFO | "huge_shared" : " 27 GB", | 20260506 10:26:32.890 | INFO | "malloc" : " 14 GB", | 20260506 10:26:32.890 | INFO | "pinned" : " 29 GB" | 20260506 10:26:32.890 | INFO | } | 20260506 10:26:32.890 | INFO | Perf report for the last 5.13939 seconds: | 20260506 10:26:32.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:26:32.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:32.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:32.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:32.890 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:26:32.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:32.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:32.890 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:26:32.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:32.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:32.891 | INFO | Uploaded 36GiB to GPU in 1.60343s (22.4519GiB/s) | 20260506 10:26:32.891 | INFO | Downloaded 34GiB to GPU in 1.52096s (22.3542GiB/s) | 20260506 10:26:38.029 | INFO | Analyzing chunk frames = [627712,628224) | 20260506 10:26:38.029 | INFO | MemInfo: { | 20260506 10:26:38.029 | INFO | "gpu" : " 17 GB", | 20260506 10:26:38.029 | INFO | "huge" : "", | 20260506 10:26:38.029 | INFO | "huge_shared" : " 27 GB", | 20260506 10:26:38.029 | INFO | "malloc" : " 14 GB", | 20260506 10:26:38.029 | INFO | "pinned" : " 29 GB" | 20260506 10:26:38.029 | INFO | } | 20260506 10:26:38.030 | INFO | Perf report for the last 5.13917 seconds: | 20260506 10:26:38.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:26:38.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:38.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:38.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:38.030 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 10:26:38.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:38.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:38.030 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:26:38.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:38.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:38.030 | INFO | Uploaded 36GiB to GPU in 1.60699s (22.4022GiB/s) | 20260506 10:26:38.030 | INFO | Downloaded 34GiB to GPU in 1.52432s (22.305GiB/s) | 20260506 10:26:43.169 | INFO | Analyzing chunk frames = [628224,628736) | 20260506 10:26:43.169 | INFO | MemInfo: { | 20260506 10:26:43.169 | INFO | "gpu" : " 17 GB", | 20260506 10:26:43.169 | INFO | "huge" : "", | 20260506 10:26:43.169 | INFO | "huge_shared" : " 27 GB", | 20260506 10:26:43.169 | INFO | "malloc" : " 14 GB", | 20260506 10:26:43.169 | INFO | "pinned" : " 29 GB" | 20260506 10:26:43.169 | INFO | } | 20260506 10:26:43.169 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:26:43.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:26:43.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:43.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:43.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:43.169 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:26:43.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:43.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:43.169 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:26:43.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:43.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:43.170 | INFO | Uploaded 36GiB to GPU in 1.60696s (22.4026GiB/s) | 20260506 10:26:43.170 | INFO | Downloaded 34GiB to GPU in 1.52467s (22.2999GiB/s) | 20260506 10:26:48.308 | INFO | Analyzing chunk frames = [628736,629248) | 20260506 10:26:48.308 | INFO | MemInfo: { | 20260506 10:26:48.308 | INFO | "gpu" : " 17 GB", | 20260506 10:26:48.308 | INFO | "huge" : "", | 20260506 10:26:48.308 | INFO | "huge_shared" : " 27 GB", | 20260506 10:26:48.308 | INFO | "malloc" : " 14 GB", | 20260506 10:26:48.308 | INFO | "pinned" : " 29 GB" | 20260506 10:26:48.308 | INFO | } | 20260506 10:26:48.308 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:26:48.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:26:48.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:48.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:48.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:48.308 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260506 10:26:48.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:48.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:48.308 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:26:48.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:48.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:48.309 | INFO | Uploaded 36GiB to GPU in 1.60532s (22.4254GiB/s) | 20260506 10:26:48.309 | INFO | Downloaded 34GiB to GPU in 1.5228s (22.3273GiB/s) | 20260506 10:26:53.447 | INFO | Analyzing chunk frames = [629248,629760) | 20260506 10:26:53.447 | INFO | MemInfo: { | 20260506 10:26:53.447 | INFO | "gpu" : " 17 GB", | 20260506 10:26:53.447 | INFO | "huge" : "", | 20260506 10:26:53.447 | INFO | "huge_shared" : " 27 GB", | 20260506 10:26:53.447 | INFO | "malloc" : " 14 GB", | 20260506 10:26:53.448 | INFO | "pinned" : " 29 GB" | 20260506 10:26:53.448 | INFO | } | 20260506 10:26:53.448 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:26:53.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:26:53.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:53.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:53.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:53.448 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:26:53.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:53.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:53.448 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:26:53.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:53.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:53.448 | INFO | Uploaded 36GiB to GPU in 1.60546s (22.4235GiB/s) | 20260506 10:26:53.448 | INFO | Downloaded 34GiB to GPU in 1.52313s (22.3225GiB/s) | 20260506 10:26:58.587 | INFO | Analyzing chunk frames = [629760,630272) | 20260506 10:26:58.587 | INFO | MemInfo: { | 20260506 10:26:58.587 | INFO | "gpu" : " 17 GB", | 20260506 10:26:58.587 | INFO | "huge" : "", | 20260506 10:26:58.587 | INFO | "huge_shared" : " 27 GB", | 20260506 10:26:58.587 | INFO | "malloc" : " 14 GB", | 20260506 10:26:58.587 | INFO | "pinned" : " 29 GB" | 20260506 10:26:58.587 | INFO | } | 20260506 10:26:58.587 | INFO | Perf report for the last 5.13946 seconds: | 20260506 10:26:58.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:26:58.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:58.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:58.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:58.587 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:26:58.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:58.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:58.587 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:26:58.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:58.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:26:58.587 | INFO | Uploaded 36GiB to GPU in 1.60479s (22.4328GiB/s) | 20260506 10:26:58.587 | INFO | Downloaded 34GiB to GPU in 1.52311s (22.3227GiB/s) | 20260506 10:27:03.726 | INFO | Analyzing chunk frames = [630272,630784) | 20260506 10:27:03.726 | INFO | MemInfo: { | 20260506 10:27:03.726 | INFO | "gpu" : " 17 GB", | 20260506 10:27:03.726 | INFO | "huge" : "", | 20260506 10:27:03.726 | INFO | "huge_shared" : " 27 GB", | 20260506 10:27:03.727 | INFO | "malloc" : " 14 GB", | 20260506 10:27:03.727 | INFO | "pinned" : " 29 GB" | 20260506 10:27:03.727 | INFO | } | 20260506 10:27:03.727 | INFO | Perf report for the last 5.13954 seconds: | 20260506 10:27:03.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:27:03.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:03.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:03.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:03.727 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:27:03.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:03.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:03.727 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:27:03.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:03.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:03.727 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:27:03.727 | INFO | Total mem usage: 16.5 GiB | 20260506 10:27:03.727 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:27:03.727 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -4.22%) | 20260506 10:27:03.727 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:27:03.727 | INFO | High Water Breakdown: | 20260506 10:27:03.727 | INFO | 60.6% : 9.6 GiB ( 95.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:27:03.727 | INFO | 39.3% : 6.23 GiB ( 93% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:27:03.727 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:27:03.727 | INFO | ------------------------------------------ | 20260506 10:27:03.727 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:27:03.727 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:27:03.727 | INFO | Total mem usage: 26.5 GiB | 20260506 10:27:03.727 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:27:03.727 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:27:03.727 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:27:03.727 | INFO | High Water Breakdown: | 20260506 10:27:03.727 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:27:03.727 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:27:03.727 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:27:03.727 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:27:03.727 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:27:03.727 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:27:03.727 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:27:03.727 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:27:03.727 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:27:03.727 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:27:03.727 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:27:03.727 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:27:03.727 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:27:03.727 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:27:03.727 | INFO | ------------------------------------------ | 20260506 10:27:03.727 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:27:03.727 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:27:03.727 | INFO | Total mem usage: 25.6 GiB | 20260506 10:27:03.727 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:27:03.727 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:27:03.727 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:27:03.727 | INFO | High Water Breakdown: | 20260506 10:27:03.727 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:27:03.727 | INFO | ------------------------------------------ | 20260506 10:27:03.727 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:27:03.727 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:27:03.727 | INFO | Total mem usage: 15.7 GiB | 20260506 10:27:03.727 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:27:03.727 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:27:03.727 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:27:03.727 | INFO | High Water Breakdown: | 20260506 10:27:03.727 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:27:03.727 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:27:03.727 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:27:03.727 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:27:03.727 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:27:03.727 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:27:03.727 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:27:03.727 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:27:03.727 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:27:03.727 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:27:03.727 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:27:03.727 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:27:03.727 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:27:03.727 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:27:03.727 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:27:03.727 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:27:03.727 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:27:03.727 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:27:03.727 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:27:03.727 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:27:03.727 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:27:03.727 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:27:03.727 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:27:03.727 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:27:03.727 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:27:03.727 | INFO | ------------------------------------------ | 20260506 10:27:03.727 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:27:03.727 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:27:03.727 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:27:03.727 | INFO | Uploaded 36GiB to GPU in 1.60669s (22.4063GiB/s) | 20260506 10:27:03.728 | INFO | Downloaded 34GiB to GPU in 1.52456s (22.3016GiB/s) | 20260506 10:27:03.727 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:27:03.728 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 10:27:03.728 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 5.21e-03 / 0.00e+00 | 20260506 10:27:03.728 | INFO | PerfReport: 10.95%, Baseline -- 3.29e+00 / -nan, 1.68e-02 / 0.00e+00 | 20260506 10:27:03.728 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.18e-02 / 0.00e+00 | 20260506 10:27:03.728 | INFO | PerfReport: 16.07%, DME -- 3.87e+01 / -nan, 2.98e+00 / 0.00e+00 | 20260506 10:27:03.728 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.72e+01 / -nan, 4.28e-02 / 0.00e+00 | 20260506 10:27:03.728 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.23e+00 / -nan, 1.35e-01 / 0.00e+00 | 20260506 10:27:03.728 | INFO | PerfReport: 3.68%, Metrics -- 1.11e+00 / -nan, 1.72e-01 / 0.00e+00 | 20260506 10:27:03.728 | INFO | PerfReport: 1.42%, PulseSerializing -- 4.27e-01 / -nan, 6.04e-02 / 0.00e+00 | 20260506 10:27:03.728 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:27:03.728 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:27:03.728 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:27:03.728 | INFO | PerfReport: Full Execution -- 8.99e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 10:27:03.728 | INFO | PerfReport: 18.69%, Upload -- 1.68e+01 / 0.00e+00, 4.68e-01 / 0.00e+00 | 20260506 10:27:03.728 | INFO | PerfReport: 17.95%, Download -- 1.61e+01 / 0.00e+00, 8.17e-01 / 0.00e+00 | 20260506 10:27:03.728 | INFO | PerfReport: 33.47%, Compute -- 3.01e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 10:27:03.728 | INFO | PerfReport: 0.57%, LockUpload -- 5.12e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 10:27:03.728 | INFO | PerfReport: 0.00%, LockDownload -- 4.87e-04 / 0.00e+00, 1.26e-04 / 0.00e+00 | 20260506 10:27:03.728 | INFO | PerfReport: 29.31%, LockCompute -- 2.64e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 10:27:03.728 | INFO | PerfReport: 0.02%, Misc -- 1.40e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260506 10:27:03.825 | INFO | Starting a round of Baz IO | 20260506 10:27:06.737 | INFO | Populating RTMetrics report | 20260506 10:27:06.737 | INFO | RTMetrics report populated | 20260506 10:27:06.737 | INFO | Opening tmp RTMetrics json file | 20260506 10:27:06.739 | INFO | Serializing RTMetrics to json | 20260506 10:27:06.741 | INFO | Serialization done, swapping tmp file now... | 20260506 10:27:06.741 | INFO | RTMetrics done | 20260506 10:27:06.926 | INFO | Finished a round of Baz IO: Wrote 1.984 GiB in 3.1 seconds | 20260506 10:27:06.926 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.89% / 0.05% / 52.06% | 20260506 10:27:06.926 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.84164% / 9.53674e-05% / 4.76837e-05% | 20260506 10:27:06.926 | INFO | Baz IO overall: | 20260506 10:27:06.926 | INFO | BytesWritten : 283603965089 | 20260506 10:27:06.926 | INFO | NumEvents : 136273087945 | 20260506 10:27:08.866 | INFO | Analyzing chunk frames = [630784,631296) | 20260506 10:27:08.866 | INFO | MemInfo: { | 20260506 10:27:08.866 | INFO | "gpu" : " 17 GB", | 20260506 10:27:08.866 | INFO | "huge" : "", | 20260506 10:27:08.866 | INFO | "huge_shared" : " 27 GB", | 20260506 10:27:08.866 | INFO | "malloc" : " 14 GB", | 20260506 10:27:08.866 | INFO | "pinned" : " 29 GB" | 20260506 10:27:08.866 | INFO | } | 20260506 10:27:08.866 | INFO | Perf report for the last 5.13921 seconds: | 20260506 10:27:08.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:27:08.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:08.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:08.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:08.866 | INFO | ANALYSIS : 57.5%, 2.8, 85.4ms, 0 | 20260506 10:27:08.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:08.866 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 10:27:08.866 | INFO | PRE_HQ : 56.6%, 2.1, 60.5ms, 0 | 20260506 10:27:08.866 | INFO | BAZWRITER : 60.0%, 4.7, 146.3ms, 0 | 20260506 10:27:08.866 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 10:27:08.866 | INFO | Uploaded 36GiB to GPU in 1.67868s (21.4454GiB/s) | 20260506 10:27:08.866 | INFO | Downloaded 38GiB to GPU in 1.74445s (21.7833GiB/s) | 20260506 10:27:14.005 | INFO | Analyzing chunk frames = [631296,631808) | 20260506 10:27:14.005 | INFO | MemInfo: { | 20260506 10:27:14.005 | INFO | "gpu" : " 17 GB", | 20260506 10:27:14.005 | INFO | "huge" : "", | 20260506 10:27:14.005 | INFO | "huge_shared" : " 27 GB", | 20260506 10:27:14.005 | INFO | "malloc" : " 14 GB", | 20260506 10:27:14.005 | INFO | "pinned" : " 29 GB" | 20260506 10:27:14.005 | INFO | } | 20260506 10:27:14.005 | INFO | Perf report for the last 5.13949 seconds: | 20260506 10:27:14.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:27:14.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:14.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:14.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:14.005 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:27:14.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:14.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:14.005 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:27:14.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:14.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:14.006 | INFO | Uploaded 36GiB to GPU in 1.60327s (22.4541GiB/s) | 20260506 10:27:14.006 | INFO | Downloaded 34GiB to GPU in 1.52079s (22.3568GiB/s) | 20260506 10:27:19.144 | INFO | Analyzing chunk frames = [631808,632320) | 20260506 10:27:19.144 | INFO | MemInfo: { | 20260506 10:27:19.144 | INFO | "gpu" : " 17 GB", | 20260506 10:27:19.144 | INFO | "huge" : "", | 20260506 10:27:19.144 | INFO | "huge_shared" : " 27 GB", | 20260506 10:27:19.144 | INFO | "malloc" : " 14 GB", | 20260506 10:27:19.144 | INFO | "pinned" : " 29 GB" | 20260506 10:27:19.144 | INFO | } | 20260506 10:27:19.144 | INFO | Perf report for the last 5.13906 seconds: | 20260506 10:27:19.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:27:19.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:19.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:19.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:19.144 | INFO | ANALYSIS : 56.7%, 3.0, 88.2ms, 0 | 20260506 10:27:19.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:19.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:19.145 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:27:19.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:19.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:19.145 | INFO | Uploaded 36GiB to GPU in 1.60718s (22.3995GiB/s) | 20260506 10:27:19.145 | INFO | Downloaded 34GiB to GPU in 1.52535s (22.2899GiB/s) | 20260506 10:27:24.284 | INFO | Analyzing chunk frames = [632320,632832) | 20260506 10:27:24.284 | INFO | MemInfo: { | 20260506 10:27:24.284 | INFO | "gpu" : " 17 GB", | 20260506 10:27:24.284 | INFO | "huge" : "", | 20260506 10:27:24.284 | INFO | "huge_shared" : " 27 GB", | 20260506 10:27:24.284 | INFO | "malloc" : " 14 GB", | 20260506 10:27:24.284 | INFO | "pinned" : " 29 GB" | 20260506 10:27:24.284 | INFO | } | 20260506 10:27:24.284 | INFO | Perf report for the last 5.13944 seconds: | 20260506 10:27:24.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:27:24.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:24.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:24.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:24.284 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:27:24.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:24.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:24.284 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:27:24.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:24.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:24.284 | INFO | Uploaded 36GiB to GPU in 1.60704s (22.4015GiB/s) | 20260506 10:27:24.284 | INFO | Downloaded 34GiB to GPU in 1.52441s (22.3038GiB/s) | 20260506 10:27:29.423 | INFO | Analyzing chunk frames = [632832,633344) | 20260506 10:27:29.423 | INFO | MemInfo: { | 20260506 10:27:29.423 | INFO | "gpu" : " 17 GB", | 20260506 10:27:29.423 | INFO | "huge" : "", | 20260506 10:27:29.423 | INFO | "huge_shared" : " 27 GB", | 20260506 10:27:29.423 | INFO | "malloc" : " 14 GB", | 20260506 10:27:29.423 | INFO | "pinned" : " 29 GB" | 20260506 10:27:29.423 | INFO | } | 20260506 10:27:29.423 | INFO | Perf report for the last 5.13943 seconds: | 20260506 10:27:29.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:27:29.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:29.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:29.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:29.423 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260506 10:27:29.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:29.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:29.423 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:27:29.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:29.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:29.424 | INFO | Uploaded 36GiB to GPU in 1.60514s (22.4279GiB/s) | 20260506 10:27:29.424 | INFO | Downloaded 34GiB to GPU in 1.5223s (22.3346GiB/s) | 20260506 10:27:34.562 | INFO | Analyzing chunk frames = [633344,633856) | 20260506 10:27:34.562 | INFO | MemInfo: { | 20260506 10:27:34.562 | INFO | "gpu" : " 17 GB", | 20260506 10:27:34.562 | INFO | "huge" : "", | 20260506 10:27:34.562 | INFO | "huge_shared" : " 27 GB", | 20260506 10:27:34.562 | INFO | "malloc" : " 14 GB", | 20260506 10:27:34.562 | INFO | "pinned" : " 29 GB" | 20260506 10:27:34.562 | INFO | } | 20260506 10:27:34.562 | INFO | Perf report for the last 5.13908 seconds: | 20260506 10:27:34.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:27:34.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:34.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:34.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:34.562 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:27:34.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:34.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:34.562 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:27:34.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:34.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:34.563 | INFO | Uploaded 36GiB to GPU in 1.60618s (22.4135GiB/s) | 20260506 10:27:34.563 | INFO | Downloaded 34GiB to GPU in 1.52348s (22.3173GiB/s) | 20260506 10:27:39.702 | INFO | Analyzing chunk frames = [633856,634368) | 20260506 10:27:39.702 | INFO | MemInfo: { | 20260506 10:27:39.702 | INFO | "gpu" : " 17 GB", | 20260506 10:27:39.702 | INFO | "huge" : "", | 20260506 10:27:39.702 | INFO | "huge_shared" : " 27 GB", | 20260506 10:27:39.702 | INFO | "malloc" : " 14 GB", | 20260506 10:27:39.702 | INFO | "pinned" : " 29 GB" | 20260506 10:27:39.702 | INFO | } | 20260506 10:27:39.702 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:27:39.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:27:39.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:39.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:39.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:39.702 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 10:27:39.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:39.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:39.702 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:27:39.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:39.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:39.702 | INFO | Uploaded 36GiB to GPU in 1.60593s (22.4169GiB/s) | 20260506 10:27:39.702 | INFO | Downloaded 34GiB to GPU in 1.52267s (22.3292GiB/s) | 20260506 10:27:44.841 | INFO | Analyzing chunk frames = [634368,634880) | 20260506 10:27:44.841 | INFO | MemInfo: { | 20260506 10:27:44.841 | INFO | "gpu" : " 17 GB", | 20260506 10:27:44.841 | INFO | "huge" : "", | 20260506 10:27:44.841 | INFO | "huge_shared" : " 27 GB", | 20260506 10:27:44.841 | INFO | "malloc" : " 14 GB", | 20260506 10:27:44.841 | INFO | "pinned" : " 29 GB" | 20260506 10:27:44.841 | INFO | } | 20260506 10:27:44.841 | INFO | Perf report for the last 5.13929 seconds: | 20260506 10:27:44.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:27:44.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:44.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:44.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:44.841 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:27:44.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:44.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:44.841 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 10:27:44.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:44.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:44.841 | INFO | Uploaded 36GiB to GPU in 1.60659s (22.4077GiB/s) | 20260506 10:27:44.841 | INFO | Downloaded 34GiB to GPU in 1.52444s (22.3033GiB/s) | 20260506 10:27:44.938 | INFO | Starting a round of Baz IO | 20260506 10:27:47.850 | INFO | Populating RTMetrics report | 20260506 10:27:47.851 | INFO | RTMetrics report populated | 20260506 10:27:47.851 | INFO | Opening tmp RTMetrics json file | 20260506 10:27:47.858 | INFO | Serializing RTMetrics to json | 20260506 10:27:47.862 | INFO | Serialization done, swapping tmp file now... | 20260506 10:27:47.866 | INFO | RTMetrics done | 20260506 10:27:48.315 | INFO | Finished a round of Baz IO: Wrote 1.986 GiB in 3.376 seconds | 20260506 10:27:48.315 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.84% / 0.05% / 52.11% | 20260506 10:27:48.315 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.8281% / 9.13938e-05% / 4.76837e-05% | 20260506 10:27:48.315 | INFO | Baz IO overall: | 20260506 10:27:48.315 | INFO | BytesWritten : 285736634620 | 20260506 10:27:48.315 | INFO | NumEvents : 137308616180 | 20260506 10:27:49.980 | INFO | Analyzing chunk frames = [634880,635392) | 20260506 10:27:49.980 | INFO | MemInfo: { | 20260506 10:27:49.980 | INFO | "gpu" : " 17 GB", | 20260506 10:27:49.980 | INFO | "huge" : "", | 20260506 10:27:49.980 | INFO | "huge_shared" : " 27 GB", | 20260506 10:27:49.980 | INFO | "malloc" : " 14 GB", | 20260506 10:27:49.980 | INFO | "pinned" : " 29 GB" | 20260506 10:27:49.981 | INFO | } | 20260506 10:27:49.981 | INFO | Perf report for the last 5.13949 seconds: | 20260506 10:27:49.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:27:49.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:49.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:49.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:49.981 | INFO | ANALYSIS : 57.5%, 2.8, 85.4ms, 0 | 20260506 10:27:49.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:49.981 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 10:27:49.981 | INFO | PRE_HQ : 56.6%, 2.1, 60.4ms, 0 | 20260506 10:27:49.981 | INFO | BAZWRITER : 65.3%, 4.8, 161.6ms, 0 | 20260506 10:27:49.981 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 10:27:49.981 | INFO | Uploaded 36GiB to GPU in 1.6769s (21.4682GiB/s) | 20260506 10:27:49.981 | INFO | Downloaded 38GiB to GPU in 1.74153s (21.8199GiB/s) | 20260506 10:27:55.120 | INFO | Analyzing chunk frames = [635392,635904) | 20260506 10:27:55.120 | INFO | MemInfo: { | 20260506 10:27:55.120 | INFO | "gpu" : " 17 GB", | 20260506 10:27:55.120 | INFO | "huge" : "", | 20260506 10:27:55.120 | INFO | "huge_shared" : " 27 GB", | 20260506 10:27:55.120 | INFO | "malloc" : " 14 GB", | 20260506 10:27:55.120 | INFO | "pinned" : " 29 GB" | 20260506 10:27:55.120 | INFO | } | 20260506 10:27:55.120 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:27:55.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:27:55.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:55.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:55.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:55.120 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:27:55.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:55.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:55.120 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:27:55.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:55.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:27:55.121 | INFO | Uploaded 36GiB to GPU in 1.60307s (22.4569GiB/s) | 20260506 10:27:55.121 | INFO | Downloaded 34GiB to GPU in 1.5198s (22.3714GiB/s) | 20260506 10:28:00.259 | INFO | Analyzing chunk frames = [635904,636416) | 20260506 10:28:00.259 | INFO | MemInfo: { | 20260506 10:28:00.259 | INFO | "gpu" : " 17 GB", | 20260506 10:28:00.259 | INFO | "huge" : "", | 20260506 10:28:00.259 | INFO | "huge_shared" : " 27 GB", | 20260506 10:28:00.259 | INFO | "malloc" : " 14 GB", | 20260506 10:28:00.259 | INFO | "pinned" : " 29 GB" | 20260506 10:28:00.259 | INFO | } | 20260506 10:28:00.259 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:28:00.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:28:00.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:00.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:00.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:00.259 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:28:00.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:00.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:00.259 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:28:00.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:00.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:00.260 | INFO | Uploaded 36GiB to GPU in 1.60764s (22.393GiB/s) | 20260506 10:28:00.260 | INFO | Downloaded 34GiB to GPU in 1.525s (22.2951GiB/s) | 20260506 10:28:05.398 | INFO | Analyzing chunk frames = [636416,636928) | 20260506 10:28:05.398 | INFO | MemInfo: { | 20260506 10:28:05.398 | INFO | "gpu" : " 17 GB", | 20260506 10:28:05.398 | INFO | "huge" : "", | 20260506 10:28:05.398 | INFO | "huge_shared" : " 27 GB", | 20260506 10:28:05.398 | INFO | "malloc" : " 14 GB", | 20260506 10:28:05.398 | INFO | "pinned" : " 29 GB" | 20260506 10:28:05.398 | INFO | } | 20260506 10:28:05.399 | INFO | Perf report for the last 5.13931 seconds: | 20260506 10:28:05.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:28:05.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:05.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:05.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:05.399 | INFO | ANALYSIS : 56.5%, 3.0, 87.5ms, 0 | 20260506 10:28:05.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:05.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:05.399 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:28:05.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:05.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:05.399 | INFO | Uploaded 36GiB to GPU in 1.60655s (22.4083GiB/s) | 20260506 10:28:05.399 | INFO | Downloaded 34GiB to GPU in 1.52412s (22.308GiB/s) | 20260506 10:28:10.538 | INFO | Analyzing chunk frames = [636928,637440) | 20260506 10:28:10.538 | INFO | MemInfo: { | 20260506 10:28:10.538 | INFO | "gpu" : " 17 GB", | 20260506 10:28:10.538 | INFO | "huge" : "", | 20260506 10:28:10.538 | INFO | "huge_shared" : " 27 GB", | 20260506 10:28:10.538 | INFO | "malloc" : " 14 GB", | 20260506 10:28:10.538 | INFO | "pinned" : " 29 GB" | 20260506 10:28:10.538 | INFO | } | 20260506 10:28:10.538 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:28:10.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:28:10.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:10.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:10.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:10.538 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:28:10.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:10.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:10.538 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:28:10.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:10.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:10.538 | INFO | Uploaded 36GiB to GPU in 1.60549s (22.423GiB/s) | 20260506 10:28:10.539 | INFO | Downloaded 34GiB to GPU in 1.52231s (22.3345GiB/s) | 20260506 10:28:15.677 | INFO | Analyzing chunk frames = [637440,637952) | 20260506 10:28:15.677 | INFO | MemInfo: { | 20260506 10:28:15.677 | INFO | "gpu" : " 17 GB", | 20260506 10:28:15.677 | INFO | "huge" : "", | 20260506 10:28:15.677 | INFO | "huge_shared" : " 27 GB", | 20260506 10:28:15.677 | INFO | "malloc" : " 14 GB", | 20260506 10:28:15.677 | INFO | "pinned" : " 29 GB" | 20260506 10:28:15.677 | INFO | } | 20260506 10:28:15.677 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:28:15.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:28:15.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:15.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:15.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:15.677 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:28:15.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:15.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:15.677 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:28:15.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:15.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:15.678 | INFO | Uploaded 36GiB to GPU in 1.60542s (22.4241GiB/s) | 20260506 10:28:15.678 | INFO | Downloaded 34GiB to GPU in 1.52214s (22.3369GiB/s) | 20260506 10:28:20.816 | INFO | Analyzing chunk frames = [637952,638464) | 20260506 10:28:20.816 | INFO | MemInfo: { | 20260506 10:28:20.816 | INFO | "gpu" : " 17 GB", | 20260506 10:28:20.816 | INFO | "huge" : "", | 20260506 10:28:20.816 | INFO | "huge_shared" : " 27 GB", | 20260506 10:28:20.816 | INFO | "malloc" : " 14 GB", | 20260506 10:28:20.816 | INFO | "pinned" : " 29 GB" | 20260506 10:28:20.816 | INFO | } | 20260506 10:28:20.817 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:28:20.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:28:20.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:20.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:20.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:20.817 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:28:20.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:20.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:20.817 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:28:20.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:20.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:20.817 | INFO | Uploaded 36GiB to GPU in 1.60588s (22.4177GiB/s) | 20260506 10:28:20.817 | INFO | Downloaded 34GiB to GPU in 1.52366s (22.3147GiB/s) | 20260506 10:28:25.956 | INFO | Analyzing chunk frames = [638464,638976) | 20260506 10:28:25.956 | INFO | MemInfo: { | 20260506 10:28:25.956 | INFO | "gpu" : " 17 GB", | 20260506 10:28:25.956 | INFO | "huge" : "", | 20260506 10:28:25.956 | INFO | "huge_shared" : " 27 GB", | 20260506 10:28:25.956 | INFO | "malloc" : " 14 GB", | 20260506 10:28:25.956 | INFO | "pinned" : " 29 GB" | 20260506 10:28:25.956 | INFO | } | 20260506 10:28:25.956 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:28:25.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:28:25.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:25.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:25.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:25.956 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:28:25.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:25.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:25.956 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 10:28:25.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:25.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:25.956 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:28:25.956 | INFO | Total mem usage: 16.5 GiB | 20260506 10:28:25.956 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:28:25.956 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -5.68%) | 20260506 10:28:25.956 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:28:25.956 | INFO | High Water Breakdown: | 20260506 10:28:25.956 | INFO | 60.4% : 9.43 GiB ( 93.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:28:25.956 | INFO | 39.5% : 6.18 GiB ( 92.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:28:25.956 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:28:25.956 | INFO | ------------------------------------------ | 20260506 10:28:25.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:28:25.956 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:28:25.956 | INFO | Uploaded 36GiB to GPU in 1.60549s (22.423GiB/s) | 20260506 10:28:25.957 | INFO | Downloaded 34GiB to GPU in 1.52285s (22.3266GiB/s) | 20260506 10:28:25.956 | INFO | Total mem usage: 26.5 GiB | 20260506 10:28:25.957 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:28:25.957 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:28:25.957 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:28:25.957 | INFO | High Water Breakdown: | 20260506 10:28:25.957 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:28:25.957 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:28:25.957 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:28:25.957 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:28:25.957 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:28:25.957 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:28:25.957 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:28:25.957 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:28:25.957 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:28:25.957 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:28:25.957 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:28:25.957 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:28:25.957 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:28:25.957 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:28:25.957 | INFO | ------------------------------------------ | 20260506 10:28:25.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:28:25.957 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:28:25.957 | INFO | Total mem usage: 25.6 GiB | 20260506 10:28:25.957 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:28:25.957 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:28:25.957 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:28:25.957 | INFO | High Water Breakdown: | 20260506 10:28:25.957 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:28:25.957 | INFO | ------------------------------------------ | 20260506 10:28:25.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:28:25.957 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:28:25.957 | INFO | Total mem usage: 15.7 GiB | 20260506 10:28:25.957 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:28:25.957 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:28:25.957 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:28:25.957 | INFO | High Water Breakdown: | 20260506 10:28:25.957 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:28:25.957 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:28:25.957 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:28:25.957 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:28:25.957 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:28:25.957 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:28:25.957 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:28:25.957 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:28:25.957 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:28:25.957 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:28:25.957 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:28:25.957 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:28:25.957 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:28:25.957 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:28:25.957 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:28:25.957 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:28:25.957 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:28:25.957 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:28:25.958 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:28:25.958 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:28:25.958 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:28:25.958 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:28:25.958 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:28:25.958 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:28:25.958 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:28:25.958 | INFO | ------------------------------------------ | 20260506 10:28:25.958 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:28:25.958 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:28:25.958 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:28:25.958 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:28:25.958 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 10:28:25.958 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 4.28e-03 / 0.00e+00 | 20260506 10:28:25.958 | INFO | PerfReport: 10.96%, Baseline -- 3.30e+00 / -nan, 1.74e-02 / 0.00e+00 | 20260506 10:28:25.958 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.25e-02 / 0.00e+00 | 20260506 10:28:25.958 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.03e+00 / 0.00e+00 | 20260506 10:28:25.958 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.72e+01 / -nan, 4.31e-02 / 0.00e+00 | 20260506 10:28:25.958 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.22e+00 / -nan, 1.29e-01 / 0.00e+00 | 20260506 10:28:25.958 | INFO | PerfReport: 3.67%, Metrics -- 1.11e+00 / -nan, 1.71e-01 / 0.00e+00 | 20260506 10:28:25.958 | INFO | PerfReport: 1.42%, PulseSerializing -- 4.27e-01 / -nan, 6.04e-02 / 0.00e+00 | 20260506 10:28:25.958 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:28:25.958 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:28:25.958 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:28:25.958 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:28:25.958 | INFO | PerfReport: 18.72%, Upload -- 1.68e+01 / 0.00e+00, 5.00e-01 / 0.00e+00 | 20260506 10:28:25.958 | INFO | PerfReport: 17.98%, Download -- 1.62e+01 / 0.00e+00, 8.58e-01 / 0.00e+00 | 20260506 10:28:25.958 | INFO | PerfReport: 33.49%, Compute -- 3.01e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 10:28:25.958 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:28:25.958 | INFO | PerfReport: 0.00%, LockDownload -- 4.84e-04 / 0.00e+00, 1.62e-04 / 0.00e+00 | 20260506 10:28:25.958 | INFO | PerfReport: 29.22%, LockCompute -- 2.63e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 10:28:25.958 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 3.34e-02 / 0.00e+00 | 20260506 10:28:26.053 | INFO | Starting a round of Baz IO | 20260506 10:28:28.966 | INFO | Populating RTMetrics report | 20260506 10:28:28.967 | INFO | RTMetrics report populated | 20260506 10:28:28.967 | INFO | Opening tmp RTMetrics json file | 20260506 10:28:29.018 | INFO | Serializing RTMetrics to json | 20260506 10:28:29.046 | INFO | Serialization done, swapping tmp file now... | 20260506 10:28:29.074 | INFO | RTMetrics done | 20260506 10:28:29.537 | INFO | Finished a round of Baz IO: Wrote 1.988 GiB in 3.484 seconds | 20260506 10:28:29.537 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.79% / 0.05% / 52.16% | 20260506 10:28:29.538 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.80153% / 9.93411e-05% / 4.76837e-05% | 20260506 10:28:29.538 | INFO | Baz IO overall: | 20260506 10:28:29.538 | INFO | BytesWritten : 287871057368 | 20260506 10:28:29.538 | INFO | NumEvents : 138344851879 | 20260506 10:28:31.095 | INFO | Analyzing chunk frames = [638976,639488) | 20260506 10:28:31.095 | INFO | MemInfo: { | 20260506 10:28:31.095 | INFO | "gpu" : " 17 GB", | 20260506 10:28:31.095 | INFO | "huge" : "", | 20260506 10:28:31.095 | INFO | "huge_shared" : " 27 GB", | 20260506 10:28:31.095 | INFO | "malloc" : " 14 GB", | 20260506 10:28:31.095 | INFO | "pinned" : " 29 GB" | 20260506 10:28:31.095 | INFO | } | 20260506 10:28:31.095 | INFO | Perf report for the last 5.13944 seconds: | 20260506 10:28:31.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:28:31.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:31.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:31.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:31.095 | INFO | ANALYSIS : 57.5%, 2.8, 85.5ms, 0 | 20260506 10:28:31.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:31.095 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260506 10:28:31.095 | INFO | PRE_HQ : 56.5%, 2.1, 60.3ms, 0 | 20260506 10:28:31.096 | INFO | BAZWRITER : 67.4%, 4.8, 165.7ms, 0 | 20260506 10:28:31.096 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 10:28:31.096 | INFO | Uploaded 36GiB to GPU in 1.67885s (21.4432GiB/s) | 20260506 10:28:31.096 | INFO | Downloaded 38GiB to GPU in 1.74482s (21.7788GiB/s) | 20260506 10:28:36.234 | INFO | Analyzing chunk frames = [639488,640000) | 20260506 10:28:36.235 | INFO | MemInfo: { | 20260506 10:28:36.235 | INFO | "gpu" : " 17 GB", | 20260506 10:28:36.235 | INFO | "huge" : "", | 20260506 10:28:36.235 | INFO | "huge_shared" : " 27 GB", | 20260506 10:28:36.235 | INFO | "malloc" : " 14 GB", | 20260506 10:28:36.235 | INFO | "pinned" : " 29 GB" | 20260506 10:28:36.235 | INFO | } | 20260506 10:28:36.235 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:28:36.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:28:36.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:36.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:36.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:36.235 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:28:36.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:36.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:36.235 | INFO | PRE_HQ : 29.0%, 1.0, 15.2ms, 0 | 20260506 10:28:36.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:36.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:36.235 | INFO | Uploaded 36GiB to GPU in 1.60314s (22.4559GiB/s) | 20260506 10:28:36.235 | INFO | Downloaded 34GiB to GPU in 1.52034s (22.3635GiB/s) | 20260506 10:28:41.374 | INFO | Analyzing chunk frames = [640000,640512) | 20260506 10:28:41.374 | INFO | MemInfo: { | 20260506 10:28:41.374 | INFO | "gpu" : " 17 GB", | 20260506 10:28:41.374 | INFO | "huge" : "", | 20260506 10:28:41.374 | INFO | "huge_shared" : " 27 GB", | 20260506 10:28:41.374 | INFO | "malloc" : " 14 GB", | 20260506 10:28:41.374 | INFO | "pinned" : " 29 GB" | 20260506 10:28:41.374 | INFO | } | 20260506 10:28:41.374 | INFO | Perf report for the last 5.13927 seconds: | 20260506 10:28:41.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:28:41.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:41.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:41.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:41.374 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:28:41.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:41.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:41.374 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:28:41.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:41.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:41.374 | INFO | Uploaded 36GiB to GPU in 1.60775s (22.3916GiB/s) | 20260506 10:28:41.374 | INFO | Downloaded 34GiB to GPU in 1.52543s (22.2889GiB/s) | 20260506 10:28:46.513 | INFO | Analyzing chunk frames = [640512,641024) | 20260506 10:28:46.513 | INFO | MemInfo: { | 20260506 10:28:46.513 | INFO | "gpu" : " 17 GB", | 20260506 10:28:46.513 | INFO | "huge" : "", | 20260506 10:28:46.513 | INFO | "huge_shared" : " 27 GB", | 20260506 10:28:46.513 | INFO | "malloc" : " 14 GB", | 20260506 10:28:46.513 | INFO | "pinned" : " 29 GB" | 20260506 10:28:46.513 | INFO | } | 20260506 10:28:46.513 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:28:46.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:28:46.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:46.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:46.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:46.513 | INFO | ANALYSIS : 56.5%, 3.0, 87.5ms, 0 | 20260506 10:28:46.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:46.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:46.513 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:28:46.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:46.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:46.514 | INFO | Uploaded 36GiB to GPU in 1.6076s (22.3936GiB/s) | 20260506 10:28:46.514 | INFO | Downloaded 34GiB to GPU in 1.52476s (22.2985GiB/s) | 20260506 10:28:51.653 | INFO | Analyzing chunk frames = [641024,641536) | 20260506 10:28:51.653 | INFO | MemInfo: { | 20260506 10:28:51.653 | INFO | "gpu" : " 17 GB", | 20260506 10:28:51.653 | INFO | "huge" : "", | 20260506 10:28:51.653 | INFO | "huge_shared" : " 27 GB", | 20260506 10:28:51.653 | INFO | "malloc" : " 14 GB", | 20260506 10:28:51.653 | INFO | "pinned" : " 29 GB" | 20260506 10:28:51.653 | INFO | } | 20260506 10:28:51.653 | INFO | Perf report for the last 5.13955 seconds: | 20260506 10:28:51.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:28:51.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:51.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:51.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:51.653 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:28:51.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:51.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:51.653 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:28:51.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:51.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:51.653 | INFO | Uploaded 36GiB to GPU in 1.60526s (22.4263GiB/s) | 20260506 10:28:51.653 | INFO | Downloaded 34GiB to GPU in 1.52316s (22.3221GiB/s) | 20260506 10:28:56.792 | INFO | Analyzing chunk frames = [641536,642048) | 20260506 10:28:56.792 | INFO | MemInfo: { | 20260506 10:28:56.792 | INFO | "gpu" : " 17 GB", | 20260506 10:28:56.792 | INFO | "huge" : "", | 20260506 10:28:56.792 | INFO | "huge_shared" : " 27 GB", | 20260506 10:28:56.792 | INFO | "malloc" : " 14 GB", | 20260506 10:28:56.792 | INFO | "pinned" : " 29 GB" | 20260506 10:28:56.792 | INFO | } | 20260506 10:28:56.792 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:28:56.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:28:56.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:56.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:56.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:56.792 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:28:56.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:56.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:56.792 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:28:56.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:56.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:28:56.792 | INFO | Uploaded 36GiB to GPU in 1.6055s (22.4229GiB/s) | 20260506 10:28:56.793 | INFO | Downloaded 34GiB to GPU in 1.52233s (22.3342GiB/s) | 20260506 10:29:01.931 | INFO | Analyzing chunk frames = [642048,642560) | 20260506 10:29:01.931 | INFO | MemInfo: { | 20260506 10:29:01.931 | INFO | "gpu" : " 17 GB", | 20260506 10:29:01.931 | INFO | "huge" : "", | 20260506 10:29:01.931 | INFO | "huge_shared" : " 27 GB", | 20260506 10:29:01.931 | INFO | "malloc" : " 14 GB", | 20260506 10:29:01.931 | INFO | "pinned" : " 29 GB" | 20260506 10:29:01.931 | INFO | } | 20260506 10:29:01.931 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:29:01.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:29:01.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:01.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:01.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:01.932 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:29:01.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:01.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:01.932 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:29:01.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:01.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:01.932 | INFO | Uploaded 36GiB to GPU in 1.60586s (22.4179GiB/s) | 20260506 10:29:01.932 | INFO | Downloaded 34GiB to GPU in 1.52476s (22.2986GiB/s) | 20260506 10:29:07.070 | INFO | Analyzing chunk frames = [642560,643072) | 20260506 10:29:07.071 | INFO | MemInfo: { | 20260506 10:29:07.071 | INFO | "gpu" : " 17 GB", | 20260506 10:29:07.071 | INFO | "huge" : "", | 20260506 10:29:07.071 | INFO | "huge_shared" : " 27 GB", | 20260506 10:29:07.071 | INFO | "malloc" : " 14 GB", | 20260506 10:29:07.071 | INFO | "pinned" : " 29 GB" | 20260506 10:29:07.071 | INFO | } | 20260506 10:29:07.071 | INFO | Perf report for the last 5.13923 seconds: | 20260506 10:29:07.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:29:07.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:07.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:07.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:07.071 | INFO | ANALYSIS : 57.0%, 3.0, 88.8ms, 0 | 20260506 10:29:07.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:07.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:07.071 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 10:29:07.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:07.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:07.071 | INFO | Uploaded 36GiB to GPU in 1.60573s (22.4197GiB/s) | 20260506 10:29:07.071 | INFO | Downloaded 34GiB to GPU in 1.52325s (22.3207GiB/s) | 20260506 10:29:07.168 | INFO | Starting a round of Baz IO | 20260506 10:29:10.076 | INFO | Populating RTMetrics report | 20260506 10:29:10.076 | INFO | RTMetrics report populated | 20260506 10:29:10.076 | INFO | Opening tmp RTMetrics json file | 20260506 10:29:10.160 | INFO | Serializing RTMetrics to json | 20260506 10:29:10.214 | INFO | Serialization done, swapping tmp file now... | 20260506 10:29:10.219 | INFO | RTMetrics done | 20260506 10:29:10.492 | INFO | Finished a round of Baz IO: Wrote 1.988 GiB in 3.324 seconds | 20260506 10:29:10.492 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.73% / 0.05% / 52.21% | 20260506 10:29:10.492 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.77111% / 9.93411e-05% / 5.96046e-05% | 20260506 10:29:10.492 | INFO | Baz IO overall: | 20260506 10:29:10.492 | INFO | BytesWritten : 290006138809 | 20260506 10:29:10.492 | INFO | NumEvents : 139381030602 | 20260506 10:29:12.210 | INFO | Analyzing chunk frames = [643072,643584) | 20260506 10:29:12.210 | INFO | MemInfo: { | 20260506 10:29:12.210 | INFO | "gpu" : " 17 GB", | 20260506 10:29:12.210 | INFO | "huge" : "", | 20260506 10:29:12.210 | INFO | "huge_shared" : " 27 GB", | 20260506 10:29:12.210 | INFO | "malloc" : " 14 GB", | 20260506 10:29:12.210 | INFO | "pinned" : " 29 GB" | 20260506 10:29:12.210 | INFO | } | 20260506 10:29:12.210 | INFO | Perf report for the last 5.1395 seconds: | 20260506 10:29:12.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:29:12.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:12.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:12.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:12.210 | INFO | ANALYSIS : 57.5%, 2.8, 85.8ms, 0 | 20260506 10:29:12.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:12.210 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260506 10:29:12.210 | INFO | PRE_HQ : 56.4%, 2.1, 61.1ms, 0 | 20260506 10:29:12.210 | INFO | BAZWRITER : 64.3%, 4.7, 154.7ms, 0 | 20260506 10:29:12.210 | INFO | BAZ_SAMPLING : 3.1%, 1.0, 1.6ms, 0 | 20260506 10:29:12.211 | INFO | Uploaded 36GiB to GPU in 1.6651s (21.6203GiB/s) | 20260506 10:29:12.211 | INFO | Downloaded 38GiB to GPU in 1.73005s (21.9647GiB/s) | 20260506 10:29:17.349 | INFO | Analyzing chunk frames = [643584,644096) | 20260506 10:29:17.349 | INFO | MemInfo: { | 20260506 10:29:17.349 | INFO | "gpu" : " 17 GB", | 20260506 10:29:17.349 | INFO | "huge" : "", | 20260506 10:29:17.349 | INFO | "huge_shared" : " 27 GB", | 20260506 10:29:17.349 | INFO | "malloc" : " 14 GB", | 20260506 10:29:17.349 | INFO | "pinned" : " 29 GB" | 20260506 10:29:17.349 | INFO | } | 20260506 10:29:17.350 | INFO | Perf report for the last 5.13929 seconds: | 20260506 10:29:17.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:29:17.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:17.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:17.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:17.350 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:29:17.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:17.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:17.350 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:29:17.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:17.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:17.350 | INFO | Uploaded 36GiB to GPU in 1.60358s (22.4498GiB/s) | 20260506 10:29:17.350 | INFO | Downloaded 34GiB to GPU in 1.52074s (22.3575GiB/s) | 20260506 10:29:22.489 | INFO | Analyzing chunk frames = [644096,644608) | 20260506 10:29:22.489 | INFO | MemInfo: { | 20260506 10:29:22.489 | INFO | "gpu" : " 17 GB", | 20260506 10:29:22.489 | INFO | "huge" : "", | 20260506 10:29:22.489 | INFO | "huge_shared" : " 27 GB", | 20260506 10:29:22.489 | INFO | "malloc" : " 14 GB", | 20260506 10:29:22.489 | INFO | "pinned" : " 29 GB" | 20260506 10:29:22.489 | INFO | } | 20260506 10:29:22.489 | INFO | Perf report for the last 5.13943 seconds: | 20260506 10:29:22.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:29:22.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:22.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:22.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:22.489 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:29:22.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:22.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:22.489 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:29:22.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:22.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:22.489 | INFO | Uploaded 36GiB to GPU in 1.60702s (22.4018GiB/s) | 20260506 10:29:22.489 | INFO | Downloaded 34GiB to GPU in 1.52526s (22.2913GiB/s) | 20260506 10:29:27.628 | INFO | Analyzing chunk frames = [644608,645120) | 20260506 10:29:27.628 | INFO | MemInfo: { | 20260506 10:29:27.628 | INFO | "gpu" : " 17 GB", | 20260506 10:29:27.628 | INFO | "huge" : "", | 20260506 10:29:27.628 | INFO | "huge_shared" : " 27 GB", | 20260506 10:29:27.628 | INFO | "malloc" : " 14 GB", | 20260506 10:29:27.628 | INFO | "pinned" : " 29 GB" | 20260506 10:29:27.628 | INFO | } | 20260506 10:29:27.628 | INFO | Perf report for the last 5.1392 seconds: | 20260506 10:29:27.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:29:27.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:27.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:27.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:27.628 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:29:27.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:27.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:27.628 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:29:27.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:27.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:27.628 | INFO | Uploaded 36GiB to GPU in 1.60681s (22.4047GiB/s) | 20260506 10:29:27.628 | INFO | Downloaded 34GiB to GPU in 1.52386s (22.3117GiB/s) | 20260506 10:29:32.767 | INFO | Analyzing chunk frames = [645120,645632) | 20260506 10:29:32.767 | INFO | MemInfo: { | 20260506 10:29:32.767 | INFO | "gpu" : " 17 GB", | 20260506 10:29:32.767 | INFO | "huge" : "", | 20260506 10:29:32.767 | INFO | "huge_shared" : " 27 GB", | 20260506 10:29:32.767 | INFO | "malloc" : " 14 GB", | 20260506 10:29:32.768 | INFO | "pinned" : " 29 GB" | 20260506 10:29:32.768 | INFO | } | 20260506 10:29:32.768 | INFO | Perf report for the last 5.13944 seconds: | 20260506 10:29:32.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:29:32.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:32.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:32.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:32.768 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:29:32.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:32.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:32.768 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:29:32.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:32.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:32.768 | INFO | Uploaded 36GiB to GPU in 1.60599s (22.4161GiB/s) | 20260506 10:29:32.768 | INFO | Downloaded 34GiB to GPU in 1.52268s (22.329GiB/s) | 20260506 10:29:37.907 | INFO | Analyzing chunk frames = [645632,646144) | 20260506 10:29:37.907 | INFO | MemInfo: { | 20260506 10:29:37.907 | INFO | "gpu" : " 17 GB", | 20260506 10:29:37.907 | INFO | "huge" : "", | 20260506 10:29:37.907 | INFO | "huge_shared" : " 27 GB", | 20260506 10:29:37.907 | INFO | "malloc" : " 14 GB", | 20260506 10:29:37.907 | INFO | "pinned" : " 29 GB" | 20260506 10:29:37.907 | INFO | } | 20260506 10:29:37.907 | INFO | Perf report for the last 5.13918 seconds: | 20260506 10:29:37.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:29:37.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:37.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:37.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:37.907 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:29:37.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:37.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:37.907 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:29:37.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:37.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:37.907 | INFO | Uploaded 36GiB to GPU in 1.60562s (22.4212GiB/s) | 20260506 10:29:37.907 | INFO | Downloaded 34GiB to GPU in 1.52248s (22.3319GiB/s) | 20260506 10:29:43.046 | INFO | Analyzing chunk frames = [646144,646656) | 20260506 10:29:43.046 | INFO | MemInfo: { | 20260506 10:29:43.046 | INFO | "gpu" : " 17 GB", | 20260506 10:29:43.046 | INFO | "huge" : "", | 20260506 10:29:43.046 | INFO | "huge_shared" : " 27 GB", | 20260506 10:29:43.046 | INFO | "malloc" : " 14 GB", | 20260506 10:29:43.046 | INFO | "pinned" : " 29 GB" | 20260506 10:29:43.046 | INFO | } | 20260506 10:29:43.046 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:29:43.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:29:43.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:43.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:43.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:43.046 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 10:29:43.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:43.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:43.046 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:29:43.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:43.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:43.047 | INFO | Uploaded 36GiB to GPU in 1.60557s (22.422GiB/s) | 20260506 10:29:43.047 | INFO | Downloaded 34GiB to GPU in 1.52307s (22.3234GiB/s) | 20260506 10:29:48.185 | INFO | Analyzing chunk frames = [646656,647168) | 20260506 10:29:48.185 | INFO | MemInfo: { | 20260506 10:29:48.185 | INFO | "gpu" : " 17 GB", | 20260506 10:29:48.185 | INFO | "huge" : "", | 20260506 10:29:48.185 | INFO | "huge_shared" : " 27 GB", | 20260506 10:29:48.185 | INFO | "malloc" : " 14 GB", | 20260506 10:29:48.185 | INFO | "pinned" : " 29 GB" | 20260506 10:29:48.186 | INFO | } | 20260506 10:29:48.186 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:29:48.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:29:48.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:48.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:48.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:48.186 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:29:48.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:48.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:48.186 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 10:29:48.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:48.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:48.186 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:29:48.186 | INFO | Total mem usage: 16.5 GiB | 20260506 10:29:48.186 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:29:48.186 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -2.73%) | 20260506 10:29:48.186 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:29:48.186 | INFO | High Water Breakdown: | 20260506 10:29:48.186 | INFO | 60.8% : 9.78 GiB ( 96.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:29:48.186 | INFO | 39% : 6.28 GiB ( 93.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:29:48.186 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:29:48.186 | INFO | ------------------------------------------ | 20260506 10:29:48.186 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:29:48.186 | INFO | Uploaded 36GiB to GPU in 1.6066s (22.4075GiB/s) | 20260506 10:29:48.186 | INFO | Downloaded 34GiB to GPU in 1.52409s (22.3084GiB/s) | 20260506 10:29:48.186 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:29:48.186 | INFO | Total mem usage: 26.5 GiB | 20260506 10:29:48.186 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:29:48.186 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:29:48.186 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:29:48.186 | INFO | High Water Breakdown: | 20260506 10:29:48.186 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:29:48.186 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:29:48.186 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:29:48.186 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:29:48.186 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:29:48.186 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:29:48.186 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:29:48.186 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:29:48.187 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:29:48.187 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:29:48.187 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:29:48.187 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:29:48.187 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:29:48.187 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:29:48.187 | INFO | ------------------------------------------ | 20260506 10:29:48.187 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:29:48.187 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:29:48.187 | INFO | Total mem usage: 25.6 GiB | 20260506 10:29:48.187 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:29:48.187 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:29:48.187 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:29:48.187 | INFO | High Water Breakdown: | 20260506 10:29:48.187 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:29:48.187 | INFO | ------------------------------------------ | 20260506 10:29:48.187 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:29:48.187 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:29:48.187 | INFO | Total mem usage: 15.7 GiB | 20260506 10:29:48.187 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:29:48.187 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:29:48.187 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:29:48.187 | INFO | High Water Breakdown: | 20260506 10:29:48.187 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:29:48.187 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:29:48.187 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:29:48.187 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:29:48.187 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:29:48.187 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:29:48.187 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:29:48.187 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:29:48.187 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:29:48.187 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:29:48.187 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:29:48.187 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:29:48.187 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:29:48.187 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:29:48.187 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:29:48.187 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:29:48.187 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:29:48.187 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:29:48.187 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:29:48.187 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:29:48.187 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:29:48.187 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:29:48.187 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:29:48.187 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:29:48.187 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:29:48.187 | INFO | ------------------------------------------ | 20260506 10:29:48.187 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:29:48.187 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:29:48.187 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:29:48.187 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:29:48.187 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 10:29:48.187 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 5.47e-03 / 0.00e+00 | 20260506 10:29:48.187 | INFO | PerfReport: 10.95%, Baseline -- 3.29e+00 / -nan, 1.85e-02 / 0.00e+00 | 20260506 10:29:48.187 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.25e-02 / 0.00e+00 | 20260506 10:29:48.187 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.05e+00 / 0.00e+00 | 20260506 10:29:48.187 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.72e+01 / -nan, 4.36e-02 / 0.00e+00 | 20260506 10:29:48.187 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.22e+00 / -nan, 1.29e-01 / 0.00e+00 | 20260506 10:29:48.187 | INFO | PerfReport: 3.67%, Metrics -- 1.10e+00 / -nan, 1.72e-01 / 0.00e+00 | 20260506 10:29:48.187 | INFO | PerfReport: 1.42%, PulseSerializing -- 4.27e-01 / -nan, 6.07e-02 / 0.00e+00 | 20260506 10:29:48.187 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:29:48.187 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:29:48.187 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:29:48.188 | INFO | PerfReport: Full Execution -- 8.99e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:29:48.188 | INFO | PerfReport: 18.71%, Upload -- 1.68e+01 / 0.00e+00, 4.87e-01 / 0.00e+00 | 20260506 10:29:48.188 | INFO | PerfReport: 17.97%, Download -- 1.61e+01 / 0.00e+00, 8.40e-01 / 0.00e+00 | 20260506 10:29:48.188 | INFO | PerfReport: 33.48%, Compute -- 3.01e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 10:29:48.188 | INFO | PerfReport: 0.57%, LockUpload -- 5.08e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:29:48.188 | INFO | PerfReport: 0.00%, LockDownload -- 4.95e-04 / 0.00e+00, 1.54e-04 / 0.00e+00 | 20260506 10:29:48.188 | INFO | PerfReport: 29.26%, LockCompute -- 2.63e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 10:29:48.188 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.36e-02 / 0.00e+00 | 20260506 10:29:48.283 | INFO | Starting a round of Baz IO | 20260506 10:29:51.195 | INFO | Populating RTMetrics report | 20260506 10:29:51.196 | INFO | RTMetrics report populated | 20260506 10:29:51.196 | INFO | Opening tmp RTMetrics json file | 20260506 10:29:51.238 | INFO | Serializing RTMetrics to json | 20260506 10:29:51.261 | INFO | Serialization done, swapping tmp file now... | 20260506 10:29:51.276 | INFO | RTMetrics done | 20260506 10:29:51.458 | INFO | Finished a round of Baz IO: Wrote 1.988 GiB in 3.174 seconds | 20260506 10:29:51.458 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.68% / 0.05% / 52.27% | 20260506 10:29:51.458 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.74216% / 9.93411e-05% / 3.57628e-05% | 20260506 10:29:51.458 | INFO | Baz IO overall: | 20260506 10:29:51.458 | INFO | BytesWritten : 292141029826 | 20260506 10:29:51.458 | INFO | NumEvents : 140416979069 | 20260506 10:29:53.325 | INFO | Analyzing chunk frames = [647168,647680) | 20260506 10:29:53.325 | INFO | MemInfo: { | 20260506 10:29:53.325 | INFO | "gpu" : " 17 GB", | 20260506 10:29:53.325 | INFO | "huge" : "", | 20260506 10:29:53.325 | INFO | "huge_shared" : " 27 GB", | 20260506 10:29:53.325 | INFO | "malloc" : " 14 GB", | 20260506 10:29:53.325 | INFO | "pinned" : " 29 GB" | 20260506 10:29:53.325 | INFO | } | 20260506 10:29:53.325 | INFO | Perf report for the last 5.13927 seconds: | 20260506 10:29:53.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:29:53.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:53.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:53.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:53.325 | INFO | ANALYSIS : 57.5%, 2.8, 85.4ms, 0 | 20260506 10:29:53.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:53.325 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 10:29:53.325 | INFO | PRE_HQ : 56.5%, 2.1, 60.5ms, 0 | 20260506 10:29:53.325 | INFO | BAZWRITER : 61.4%, 4.8, 150.3ms, 0 | 20260506 10:29:53.325 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 10:29:53.325 | INFO | Uploaded 36GiB to GPU in 1.67785s (21.456GiB/s) | 20260506 10:29:53.325 | INFO | Downloaded 38GiB to GPU in 1.74379s (21.7916GiB/s) | 20260506 10:29:58.464 | INFO | Analyzing chunk frames = [647680,648192) | 20260506 10:29:58.464 | INFO | MemInfo: { | 20260506 10:29:58.464 | INFO | "gpu" : " 17 GB", | 20260506 10:29:58.464 | INFO | "huge" : "", | 20260506 10:29:58.464 | INFO | "huge_shared" : " 27 GB", | 20260506 10:29:58.464 | INFO | "malloc" : " 14 GB", | 20260506 10:29:58.464 | INFO | "pinned" : " 29 GB" | 20260506 10:29:58.464 | INFO | } | 20260506 10:29:58.464 | INFO | Perf report for the last 5.13952 seconds: | 20260506 10:29:58.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:29:58.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:58.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:58.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:58.464 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:29:58.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:58.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:58.464 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:29:58.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:58.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:29:58.465 | INFO | Uploaded 36GiB to GPU in 1.60329s (22.4539GiB/s) | 20260506 10:29:58.465 | INFO | Downloaded 34GiB to GPU in 1.52051s (22.3609GiB/s) | 20260506 10:30:03.603 | INFO | Analyzing chunk frames = [648192,648704) | 20260506 10:30:03.603 | INFO | MemInfo: { | 20260506 10:30:03.603 | INFO | "gpu" : " 17 GB", | 20260506 10:30:03.603 | INFO | "huge" : "", | 20260506 10:30:03.604 | INFO | "huge_shared" : " 27 GB", | 20260506 10:30:03.604 | INFO | "malloc" : " 14 GB", | 20260506 10:30:03.604 | INFO | "pinned" : " 29 GB" | 20260506 10:30:03.604 | INFO | } | 20260506 10:30:03.604 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:30:03.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:30:03.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:03.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:03.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:03.604 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:30:03.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:03.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:03.604 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:30:03.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:03.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:03.604 | INFO | Uploaded 36GiB to GPU in 1.60752s (22.3948GiB/s) | 20260506 10:30:03.605 | INFO | Downloaded 34GiB to GPU in 1.52557s (22.2868GiB/s) | 20260506 10:30:08.743 | INFO | Analyzing chunk frames = [648704,649216) | 20260506 10:30:08.743 | INFO | MemInfo: { | 20260506 10:30:08.743 | INFO | "gpu" : " 17 GB", | 20260506 10:30:08.743 | INFO | "huge" : "", | 20260506 10:30:08.743 | INFO | "huge_shared" : " 27 GB", | 20260506 10:30:08.743 | INFO | "malloc" : " 14 GB", | 20260506 10:30:08.743 | INFO | "pinned" : " 29 GB" | 20260506 10:30:08.743 | INFO | } | 20260506 10:30:08.743 | INFO | Perf report for the last 5.13925 seconds: | 20260506 10:30:08.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:30:08.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:08.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:08.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:08.743 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:30:08.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:08.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:08.743 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:30:08.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:08.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:08.744 | INFO | Uploaded 36GiB to GPU in 1.60653s (22.4086GiB/s) | 20260506 10:30:08.744 | INFO | Downloaded 34GiB to GPU in 1.52461s (22.3008GiB/s) | 20260506 10:30:13.882 | INFO | Analyzing chunk frames = [649216,649728) | 20260506 10:30:13.882 | INFO | MemInfo: { | 20260506 10:30:13.882 | INFO | "gpu" : " 17 GB", | 20260506 10:30:13.882 | INFO | "huge" : "", | 20260506 10:30:13.882 | INFO | "huge_shared" : " 27 GB", | 20260506 10:30:13.882 | INFO | "malloc" : " 14 GB", | 20260506 10:30:13.882 | INFO | "pinned" : " 29 GB" | 20260506 10:30:13.882 | INFO | } | 20260506 10:30:13.882 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:30:13.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:30:13.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:13.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:13.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:13.882 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260506 10:30:13.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:13.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:13.882 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:30:13.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:13.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:13.883 | INFO | Uploaded 36GiB to GPU in 1.60497s (22.4303GiB/s) | 20260506 10:30:13.883 | INFO | Downloaded 34GiB to GPU in 1.52165s (22.3442GiB/s) | 20260506 10:30:19.022 | INFO | Analyzing chunk frames = [649728,650240) | 20260506 10:30:19.022 | INFO | MemInfo: { | 20260506 10:30:19.022 | INFO | "gpu" : " 17 GB", | 20260506 10:30:19.022 | INFO | "huge" : "", | 20260506 10:30:19.022 | INFO | "huge_shared" : " 27 GB", | 20260506 10:30:19.022 | INFO | "malloc" : " 14 GB", | 20260506 10:30:19.022 | INFO | "pinned" : " 29 GB" | 20260506 10:30:19.022 | INFO | } | 20260506 10:30:19.022 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:30:19.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:30:19.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:19.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:19.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:19.022 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:30:19.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:19.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:19.022 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:30:19.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:19.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:19.022 | INFO | Uploaded 36GiB to GPU in 1.60567s (22.4206GiB/s) | 20260506 10:30:19.022 | INFO | Downloaded 34GiB to GPU in 1.52284s (22.3268GiB/s) | 20260506 10:30:24.161 | INFO | Analyzing chunk frames = [650240,650752) | 20260506 10:30:24.161 | INFO | MemInfo: { | 20260506 10:30:24.161 | INFO | "gpu" : " 17 GB", | 20260506 10:30:24.161 | INFO | "huge" : "", | 20260506 10:30:24.161 | INFO | "huge_shared" : " 27 GB", | 20260506 10:30:24.161 | INFO | "malloc" : " 14 GB", | 20260506 10:30:24.161 | INFO | "pinned" : " 29 GB" | 20260506 10:30:24.161 | INFO | } | 20260506 10:30:24.161 | INFO | Perf report for the last 5.13912 seconds: | 20260506 10:30:24.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:30:24.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:24.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:24.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:24.161 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:30:24.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:24.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:24.161 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:30:24.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:24.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:24.161 | INFO | Uploaded 36GiB to GPU in 1.60572s (22.4198GiB/s) | 20260506 10:30:24.161 | INFO | Downloaded 34GiB to GPU in 1.524s (22.3098GiB/s) | 20260506 10:30:29.300 | INFO | Analyzing chunk frames = [650752,651264) | 20260506 10:30:29.300 | INFO | MemInfo: { | 20260506 10:30:29.300 | INFO | "gpu" : " 17 GB", | 20260506 10:30:29.300 | INFO | "huge" : "", | 20260506 10:30:29.301 | INFO | "huge_shared" : " 27 GB", | 20260506 10:30:29.301 | INFO | "malloc" : " 14 GB", | 20260506 10:30:29.301 | INFO | "pinned" : " 29 GB" | 20260506 10:30:29.301 | INFO | } | 20260506 10:30:29.301 | INFO | Perf report for the last 5.13975 seconds: | 20260506 10:30:29.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:30:29.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:29.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:29.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:29.301 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:30:29.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:29.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:29.301 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 10:30:29.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:29.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:29.301 | INFO | Uploaded 36GiB to GPU in 1.60616s (22.4138GiB/s) | 20260506 10:30:29.301 | INFO | Downloaded 34GiB to GPU in 1.52364s (22.3149GiB/s) | 20260506 10:30:29.398 | INFO | Starting a round of Baz IO | 20260506 10:30:32.309 | INFO | Populating RTMetrics report | 20260506 10:30:32.310 | INFO | RTMetrics report populated | 20260506 10:30:32.310 | INFO | Opening tmp RTMetrics json file | 20260506 10:30:32.363 | INFO | Serializing RTMetrics to json | 20260506 10:30:32.455 | INFO | Serialization done, swapping tmp file now... | 20260506 10:30:32.460 | INFO | RTMetrics done | 20260506 10:30:32.906 | INFO | Finished a round of Baz IO: Wrote 1.989 GiB in 3.508 seconds | 20260506 10:30:32.906 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.63% / 0.05% / 52.32% | 20260506 10:30:32.906 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.70995% / 0.000127157% / 5.16574e-05% | 20260506 10:30:32.906 | INFO | Baz IO overall: | 20260506 10:30:32.906 | INFO | BytesWritten : 294276831804 | 20260506 10:30:32.906 | INFO | NumEvents : 141452737194 | 20260506 10:30:34.440 | INFO | Analyzing chunk frames = [651264,651776) | 20260506 10:30:34.440 | INFO | MemInfo: { | 20260506 10:30:34.440 | INFO | "gpu" : " 17 GB", | 20260506 10:30:34.440 | INFO | "huge" : "", | 20260506 10:30:34.440 | INFO | "huge_shared" : " 27 GB", | 20260506 10:30:34.440 | INFO | "malloc" : " 14 GB", | 20260506 10:30:34.440 | INFO | "pinned" : " 29 GB" | 20260506 10:30:34.440 | INFO | } | 20260506 10:30:34.440 | INFO | Perf report for the last 5.13901 seconds: | 20260506 10:30:34.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:30:34.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:34.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:34.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:34.440 | INFO | ANALYSIS : 57.5%, 2.8, 85.9ms, 0 | 20260506 10:30:34.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:34.440 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260506 10:30:34.440 | INFO | PRE_HQ : 56.5%, 2.1, 60.2ms, 0 | 20260506 10:30:34.440 | INFO | BAZWRITER : 67.8%, 4.8, 166.0ms, 0 | 20260506 10:30:34.440 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.3ms, 0 | 20260506 10:30:34.440 | INFO | Uploaded 36GiB to GPU in 1.67887s (21.4429GiB/s) | 20260506 10:30:34.440 | INFO | Downloaded 38GiB to GPU in 1.74291s (21.8026GiB/s) | 20260506 10:30:39.579 | INFO | Analyzing chunk frames = [651776,652288) | 20260506 10:30:39.579 | INFO | MemInfo: { | 20260506 10:30:39.579 | INFO | "gpu" : " 17 GB", | 20260506 10:30:39.579 | INFO | "huge" : "", | 20260506 10:30:39.579 | INFO | "huge_shared" : " 27 GB", | 20260506 10:30:39.579 | INFO | "malloc" : " 14 GB", | 20260506 10:30:39.579 | INFO | "pinned" : " 29 GB" | 20260506 10:30:39.579 | INFO | } | 20260506 10:30:39.579 | INFO | Perf report for the last 5.13945 seconds: | 20260506 10:30:39.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:30:39.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:39.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:39.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:39.579 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:30:39.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:39.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:39.579 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:30:39.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:39.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:39.580 | INFO | Uploaded 36GiB to GPU in 1.60304s (22.4573GiB/s) | 20260506 10:30:39.580 | INFO | Downloaded 34GiB to GPU in 1.52063s (22.3592GiB/s) | 20260506 10:30:44.718 | INFO | Analyzing chunk frames = [652288,652800) | 20260506 10:30:44.718 | INFO | MemInfo: { | 20260506 10:30:44.718 | INFO | "gpu" : " 17 GB", | 20260506 10:30:44.718 | INFO | "huge" : "", | 20260506 10:30:44.718 | INFO | "huge_shared" : " 27 GB", | 20260506 10:30:44.718 | INFO | "malloc" : " 14 GB", | 20260506 10:30:44.718 | INFO | "pinned" : " 29 GB" | 20260506 10:30:44.718 | INFO | } | 20260506 10:30:44.718 | INFO | Perf report for the last 5.13921 seconds: | 20260506 10:30:44.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:30:44.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:44.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:44.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:44.718 | INFO | ANALYSIS : 56.8%, 3.0, 88.2ms, 0 | 20260506 10:30:44.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:44.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:44.719 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:30:44.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:44.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:44.719 | INFO | Uploaded 36GiB to GPU in 1.60669s (22.4064GiB/s) | 20260506 10:30:44.719 | INFO | Downloaded 34GiB to GPU in 1.5251s (22.2936GiB/s) | 20260506 10:30:49.858 | INFO | Analyzing chunk frames = [652800,653312) | 20260506 10:30:49.858 | INFO | MemInfo: { | 20260506 10:30:49.858 | INFO | "gpu" : " 17 GB", | 20260506 10:30:49.858 | INFO | "huge" : "", | 20260506 10:30:49.858 | INFO | "huge_shared" : " 27 GB", | 20260506 10:30:49.858 | INFO | "malloc" : " 14 GB", | 20260506 10:30:49.858 | INFO | "pinned" : " 29 GB" | 20260506 10:30:49.858 | INFO | } | 20260506 10:30:49.858 | INFO | Perf report for the last 5.13948 seconds: | 20260506 10:30:49.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:30:49.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:49.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:49.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:49.858 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:30:49.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:49.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:49.858 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:30:49.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:49.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:49.858 | INFO | Uploaded 36GiB to GPU in 1.60683s (22.4044GiB/s) | 20260506 10:30:49.858 | INFO | Downloaded 34GiB to GPU in 1.52415s (22.3075GiB/s) | 20260506 10:30:54.997 | INFO | Analyzing chunk frames = [653312,653824) | 20260506 10:30:54.997 | INFO | MemInfo: { | 20260506 10:30:54.997 | INFO | "gpu" : " 17 GB", | 20260506 10:30:54.997 | INFO | "huge" : "", | 20260506 10:30:54.997 | INFO | "huge_shared" : " 27 GB", | 20260506 10:30:54.997 | INFO | "malloc" : " 14 GB", | 20260506 10:30:54.997 | INFO | "pinned" : " 29 GB" | 20260506 10:30:54.997 | INFO | } | 20260506 10:30:54.997 | INFO | Perf report for the last 5.13919 seconds: | 20260506 10:30:54.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:30:54.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:54.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:54.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:54.997 | INFO | ANALYSIS : 56.4%, 3.0, 87.9ms, 0 | 20260506 10:30:54.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:54.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:54.997 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:30:54.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:54.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:30:54.998 | INFO | Uploaded 36GiB to GPU in 1.60499s (22.4301GiB/s) | 20260506 10:30:54.998 | INFO | Downloaded 34GiB to GPU in 1.522s (22.3391GiB/s) | 20260506 10:31:00.136 | INFO | Analyzing chunk frames = [653824,654336) | 20260506 10:31:00.136 | INFO | MemInfo: { | 20260506 10:31:00.136 | INFO | "gpu" : " 17 GB", | 20260506 10:31:00.137 | INFO | "huge" : "", | 20260506 10:31:00.137 | INFO | "huge_shared" : " 27 GB", | 20260506 10:31:00.137 | INFO | "malloc" : " 14 GB", | 20260506 10:31:00.137 | INFO | "pinned" : " 29 GB" | 20260506 10:31:00.137 | INFO | } | 20260506 10:31:00.137 | INFO | Perf report for the last 5.13955 seconds: | 20260506 10:31:00.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:31:00.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:00.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:00.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:00.137 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:31:00.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:00.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:00.137 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:31:00.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:00.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:00.137 | INFO | Uploaded 36GiB to GPU in 1.6056s (22.4215GiB/s) | 20260506 10:31:00.137 | INFO | Downloaded 34GiB to GPU in 1.52277s (22.3277GiB/s) | 20260506 10:31:05.276 | INFO | Analyzing chunk frames = [654336,654848) | 20260506 10:31:05.276 | INFO | MemInfo: { | 20260506 10:31:05.276 | INFO | "gpu" : " 17 GB", | 20260506 10:31:05.276 | INFO | "huge" : "", | 20260506 10:31:05.276 | INFO | "huge_shared" : " 27 GB", | 20260506 10:31:05.276 | INFO | "malloc" : " 14 GB", | 20260506 10:31:05.276 | INFO | "pinned" : " 29 GB" | 20260506 10:31:05.276 | INFO | } | 20260506 10:31:05.276 | INFO | Perf report for the last 5.1392 seconds: | 20260506 10:31:05.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:31:05.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:05.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:05.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:05.276 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 10:31:05.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:05.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:05.276 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:31:05.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:05.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:05.276 | INFO | Uploaded 36GiB to GPU in 1.60539s (22.4245GiB/s) | 20260506 10:31:05.276 | INFO | Downloaded 34GiB to GPU in 1.52288s (22.3261GiB/s) | 20260506 10:31:10.415 | INFO | Analyzing chunk frames = [654848,655360) | 20260506 10:31:10.415 | INFO | MemInfo: { | 20260506 10:31:10.415 | INFO | "gpu" : " 17 GB", | 20260506 10:31:10.415 | INFO | "huge" : "", | 20260506 10:31:10.415 | INFO | "huge_shared" : " 27 GB", | 20260506 10:31:10.415 | INFO | "malloc" : " 14 GB", | 20260506 10:31:10.415 | INFO | "pinned" : " 29 GB" | 20260506 10:31:10.415 | INFO | } | 20260506 10:31:10.415 | INFO | Perf report for the last 5.13946 seconds: | 20260506 10:31:10.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:31:10.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:10.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:10.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:10.415 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:31:10.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:10.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:10.415 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 10:31:10.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:10.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:10.416 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:31:10.416 | INFO | Total mem usage: 16.5 GiB | 20260506 10:31:10.416 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:31:10.416 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -2.79%) | 20260506 10:31:10.416 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:31:10.416 | INFO | High Water Breakdown: | 20260506 10:31:10.416 | INFO | 60.8% : 9.77 GiB ( 96.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:31:10.416 | INFO | 39.1% : 6.28 GiB ( 93.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:31:10.416 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:31:10.416 | INFO | ------------------------------------------ | 20260506 10:31:10.416 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:31:10.416 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:31:10.416 | INFO | Total mem usage: 26.5 GiB | 20260506 10:31:10.416 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:31:10.416 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:31:10.416 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:31:10.416 | INFO | High Water Breakdown: | 20260506 10:31:10.416 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:31:10.416 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:31:10.416 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:31:10.416 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:31:10.416 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:31:10.416 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:31:10.416 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:31:10.416 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:31:10.416 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:31:10.416 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:31:10.416 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:31:10.416 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:31:10.416 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:31:10.416 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:31:10.416 | INFO | ------------------------------------------ | 20260506 10:31:10.416 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:31:10.416 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:31:10.416 | INFO | Total mem usage: 25.6 GiB | 20260506 10:31:10.416 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:31:10.416 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:31:10.416 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:31:10.416 | INFO | High Water Breakdown: | 20260506 10:31:10.416 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:31:10.416 | INFO | ------------------------------------------ | 20260506 10:31:10.416 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:31:10.416 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:31:10.416 | INFO | Total mem usage: 15.7 GiB | 20260506 10:31:10.416 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:31:10.416 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:31:10.416 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:31:10.416 | INFO | High Water Breakdown: | 20260506 10:31:10.416 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:31:10.416 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:31:10.416 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:31:10.416 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:31:10.416 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:31:10.416 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:31:10.416 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:31:10.416 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:31:10.416 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:31:10.416 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:31:10.416 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:31:10.416 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:31:10.416 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:31:10.416 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:31:10.416 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:31:10.416 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:31:10.416 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:31:10.416 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:31:10.416 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:31:10.416 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:31:10.416 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:31:10.416 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:31:10.416 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:31:10.416 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:31:10.416 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:31:10.416 | INFO | ------------------------------------------ | 20260506 10:31:10.416 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:31:10.416 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:31:10.416 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:31:10.416 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:31:10.416 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 10:31:10.416 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.99e-03 / 0.00e+00 | 20260506 10:31:10.416 | INFO | PerfReport: 10.95%, Baseline -- 3.29e+00 / -nan, 1.75e-02 / 0.00e+00 | 20260506 10:31:10.416 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.21e-02 / 0.00e+00 | 20260506 10:31:10.416 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.04e+00 / 0.00e+00 | 20260506 10:31:10.416 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.72e+01 / -nan, 4.22e-02 / 0.00e+00 | 20260506 10:31:10.416 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.22e+00 / -nan, 1.28e-01 / 0.00e+00 | 20260506 10:31:10.416 | INFO | PerfReport: 3.67%, Metrics -- 1.10e+00 / -nan, 1.71e-01 / 0.00e+00 | 20260506 10:31:10.416 | INFO | PerfReport: 1.42%, PulseSerializing -- 4.26e-01 / -nan, 6.02e-02 / 0.00e+00 | 20260506 10:31:10.416 | INFO | Uploaded 36GiB to GPU in 1.60574s (22.4196GiB/s) | 20260506 10:31:10.416 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:31:10.416 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:31:10.416 | INFO | Downloaded 34GiB to GPU in 1.52355s (22.3162GiB/s) | 20260506 10:31:10.416 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:31:10.416 | INFO | PerfReport: Full Execution -- 8.99e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:31:10.416 | INFO | PerfReport: 18.72%, Upload -- 1.68e+01 / 0.00e+00, 5.08e-01 / 0.00e+00 | 20260506 10:31:10.416 | INFO | PerfReport: 17.98%, Download -- 1.62e+01 / 0.00e+00, 8.58e-01 / 0.00e+00 | 20260506 10:31:10.416 | INFO | PerfReport: 33.48%, Compute -- 3.01e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 10:31:10.416 | INFO | PerfReport: 0.56%, LockUpload -- 5.08e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:31:10.417 | INFO | PerfReport: 0.00%, LockDownload -- 4.98e-04 / 0.00e+00, 2.08e-04 / 0.00e+00 | 20260506 10:31:10.417 | INFO | PerfReport: 29.24%, LockCompute -- 2.63e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 10:31:10.417 | INFO | PerfReport: 0.02%, Misc -- 1.39e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260506 10:31:10.513 | INFO | Starting a round of Baz IO | 20260506 10:31:13.423 | INFO | Populating RTMetrics report | 20260506 10:31:13.423 | INFO | RTMetrics report populated | 20260506 10:31:13.423 | INFO | Opening tmp RTMetrics json file | 20260506 10:31:13.551 | INFO | Serializing RTMetrics to json | 20260506 10:31:13.590 | INFO | Serialization done, swapping tmp file now... | 20260506 10:31:13.591 | INFO | RTMetrics done | 20260506 10:31:13.976 | INFO | Finished a round of Baz IO: Wrote 1.993 GiB in 3.462 seconds | 20260506 10:31:13.976 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.58% / 0.05% / 52.37% | 20260506 10:31:13.976 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.70988% / 7.15256e-05% / 7.94729e-05% | 20260506 10:31:13.976 | INFO | Baz IO overall: | 20260506 10:31:13.976 | INFO | BytesWritten : 296417065743 | 20260506 10:31:13.976 | INFO | NumEvents : 142491179156 | 20260506 10:31:15.554 | INFO | Analyzing chunk frames = [655360,655872) | 20260506 10:31:15.554 | INFO | MemInfo: { | 20260506 10:31:15.554 | INFO | "gpu" : " 17 GB", | 20260506 10:31:15.554 | INFO | "huge" : "", | 20260506 10:31:15.554 | INFO | "huge_shared" : " 27 GB", | 20260506 10:31:15.554 | INFO | "malloc" : " 14 GB", | 20260506 10:31:15.554 | INFO | "pinned" : " 29 GB" | 20260506 10:31:15.554 | INFO | } | 20260506 10:31:15.554 | INFO | Perf report for the last 5.13906 seconds: | 20260506 10:31:15.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:31:15.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:15.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:15.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:15.555 | INFO | ANALYSIS : 57.5%, 2.9, 87.0ms, 0 | 20260506 10:31:15.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:15.555 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260506 10:31:15.555 | INFO | PRE_HQ : 56.5%, 2.1, 60.6ms, 0 | 20260506 10:31:15.555 | INFO | BAZWRITER : 66.9%, 4.7, 160.0ms, 0 | 20260506 10:31:15.555 | INFO | BAZ_SAMPLING : 3.3%, 1.0, 1.7ms, 0 | 20260506 10:31:15.555 | INFO | Uploaded 36GiB to GPU in 1.66195s (21.6613GiB/s) | 20260506 10:31:15.555 | INFO | Downloaded 38GiB to GPU in 1.72605s (22.0156GiB/s) | 20260506 10:31:20.694 | INFO | Analyzing chunk frames = [655872,656384) | 20260506 10:31:20.694 | INFO | MemInfo: { | 20260506 10:31:20.694 | INFO | "gpu" : " 17 GB", | 20260506 10:31:20.694 | INFO | "huge" : "", | 20260506 10:31:20.694 | INFO | "huge_shared" : " 27 GB", | 20260506 10:31:20.694 | INFO | "malloc" : " 14 GB", | 20260506 10:31:20.694 | INFO | "pinned" : " 29 GB" | 20260506 10:31:20.694 | INFO | } | 20260506 10:31:20.694 | INFO | Perf report for the last 5.13964 seconds: | 20260506 10:31:20.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:31:20.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:20.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:20.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:20.694 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:31:20.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:20.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:20.694 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:31:20.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:20.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:20.694 | INFO | Uploaded 36GiB to GPU in 1.60333s (22.4532GiB/s) | 20260506 10:31:20.694 | INFO | Downloaded 34GiB to GPU in 1.52072s (22.3578GiB/s) | 20260506 10:31:25.833 | INFO | Analyzing chunk frames = [656384,656896) | 20260506 10:31:25.833 | INFO | MemInfo: { | 20260506 10:31:25.833 | INFO | "gpu" : " 17 GB", | 20260506 10:31:25.833 | INFO | "huge" : "", | 20260506 10:31:25.833 | INFO | "huge_shared" : " 27 GB", | 20260506 10:31:25.833 | INFO | "malloc" : " 14 GB", | 20260506 10:31:25.833 | INFO | "pinned" : " 29 GB" | 20260506 10:31:25.833 | INFO | } | 20260506 10:31:25.833 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:31:25.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:31:25.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:25.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:25.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:25.833 | INFO | ANALYSIS : 56.7%, 3.0, 88.2ms, 0 | 20260506 10:31:25.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:25.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:25.834 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:31:25.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:25.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:25.834 | INFO | Uploaded 36GiB to GPU in 1.60681s (22.4046GiB/s) | 20260506 10:31:25.834 | INFO | Downloaded 34GiB to GPU in 1.52466s (22.3GiB/s) | 20260506 10:31:30.972 | INFO | Analyzing chunk frames = [656896,657408) | 20260506 10:31:30.972 | INFO | MemInfo: { | 20260506 10:31:30.972 | INFO | "gpu" : " 17 GB", | 20260506 10:31:30.972 | INFO | "huge" : "", | 20260506 10:31:30.973 | INFO | "huge_shared" : " 27 GB", | 20260506 10:31:30.973 | INFO | "malloc" : " 14 GB", | 20260506 10:31:30.973 | INFO | "pinned" : " 29 GB" | 20260506 10:31:30.973 | INFO | } | 20260506 10:31:30.973 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:31:30.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:31:30.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:30.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:30.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:30.973 | INFO | ANALYSIS : 56.5%, 3.0, 87.5ms, 0 | 20260506 10:31:30.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:30.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:30.973 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:31:30.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:30.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:30.973 | INFO | Uploaded 36GiB to GPU in 1.60659s (22.4077GiB/s) | 20260506 10:31:30.973 | INFO | Downloaded 34GiB to GPU in 1.52417s (22.3072GiB/s) | 20260506 10:31:36.112 | INFO | Analyzing chunk frames = [657408,657920) | 20260506 10:31:36.112 | INFO | MemInfo: { | 20260506 10:31:36.112 | INFO | "gpu" : " 17 GB", | 20260506 10:31:36.112 | INFO | "huge" : "", | 20260506 10:31:36.112 | INFO | "huge_shared" : " 27 GB", | 20260506 10:31:36.112 | INFO | "malloc" : " 14 GB", | 20260506 10:31:36.112 | INFO | "pinned" : " 29 GB" | 20260506 10:31:36.112 | INFO | } | 20260506 10:31:36.112 | INFO | Perf report for the last 5.13942 seconds: | 20260506 10:31:36.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:31:36.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:36.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:36.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:36.112 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:31:36.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:36.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:36.112 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:31:36.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:36.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:36.113 | INFO | Uploaded 36GiB to GPU in 1.60537s (22.4247GiB/s) | 20260506 10:31:36.113 | INFO | Downloaded 34GiB to GPU in 1.52231s (22.3345GiB/s) | 20260506 10:31:41.251 | INFO | Analyzing chunk frames = [657920,658432) | 20260506 10:31:41.251 | INFO | MemInfo: { | 20260506 10:31:41.251 | INFO | "gpu" : " 17 GB", | 20260506 10:31:41.251 | INFO | "huge" : "", | 20260506 10:31:41.251 | INFO | "huge_shared" : " 27 GB", | 20260506 10:31:41.251 | INFO | "malloc" : " 14 GB", | 20260506 10:31:41.251 | INFO | "pinned" : " 29 GB" | 20260506 10:31:41.251 | INFO | } | 20260506 10:31:41.251 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:31:41.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:31:41.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:41.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:41.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:41.251 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:31:41.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:41.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:41.251 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:31:41.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:41.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:41.252 | INFO | Uploaded 36GiB to GPU in 1.60535s (22.4251GiB/s) | 20260506 10:31:41.252 | INFO | Downloaded 34GiB to GPU in 1.52229s (22.3348GiB/s) | 20260506 10:31:46.390 | INFO | Analyzing chunk frames = [658432,658944) | 20260506 10:31:46.390 | INFO | MemInfo: { | 20260506 10:31:46.390 | INFO | "gpu" : " 17 GB", | 20260506 10:31:46.390 | INFO | "huge" : "", | 20260506 10:31:46.390 | INFO | "huge_shared" : " 27 GB", | 20260506 10:31:46.390 | INFO | "malloc" : " 14 GB", | 20260506 10:31:46.391 | INFO | "pinned" : " 29 GB" | 20260506 10:31:46.391 | INFO | } | 20260506 10:31:46.391 | INFO | Perf report for the last 5.13919 seconds: | 20260506 10:31:46.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:31:46.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:46.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:46.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:46.391 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:31:46.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:46.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:46.391 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:31:46.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:46.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:46.391 | INFO | Uploaded 36GiB to GPU in 1.60639s (22.4105GiB/s) | 20260506 10:31:46.391 | INFO | Downloaded 34GiB to GPU in 1.5246s (22.3009GiB/s) | 20260506 10:31:51.530 | INFO | Analyzing chunk frames = [658944,659456) | 20260506 10:31:51.530 | INFO | MemInfo: { | 20260506 10:31:51.530 | INFO | "gpu" : " 17 GB", | 20260506 10:31:51.530 | INFO | "huge" : "", | 20260506 10:31:51.530 | INFO | "huge_shared" : " 27 GB", | 20260506 10:31:51.530 | INFO | "malloc" : " 14 GB", | 20260506 10:31:51.530 | INFO | "pinned" : " 29 GB" | 20260506 10:31:51.530 | INFO | } | 20260506 10:31:51.530 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:31:51.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:31:51.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:51.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:51.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:51.530 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:31:51.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:51.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:51.530 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 10:31:51.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:51.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:51.531 | INFO | Uploaded 36GiB to GPU in 1.606s (22.416GiB/s) | 20260506 10:31:51.531 | INFO | Downloaded 34GiB to GPU in 1.5232s (22.3215GiB/s) | 20260506 10:31:51.627 | INFO | Starting a round of Baz IO | 20260506 10:31:54.536 | INFO | Populating RTMetrics report | 20260506 10:31:54.536 | INFO | RTMetrics report populated | 20260506 10:31:54.536 | INFO | Opening tmp RTMetrics json file | 20260506 10:31:54.554 | INFO | Serializing RTMetrics to json | 20260506 10:31:54.575 | INFO | Serialization done, swapping tmp file now... | 20260506 10:31:54.579 | INFO | RTMetrics done | 20260506 10:31:54.812 | INFO | Finished a round of Baz IO: Wrote 1.994 GiB in 3.185 seconds | 20260506 10:31:54.812 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.53% / 0.05% / 52.42% | 20260506 10:31:54.812 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.6861% / 0.000115236% / 5.5631e-05% | 20260506 10:31:54.812 | INFO | Baz IO overall: | 20260506 10:31:54.812 | INFO | BytesWritten : 298557920493 | 20260506 10:31:54.812 | INFO | NumEvents : 143529982066 | 20260506 10:31:56.669 | INFO | Analyzing chunk frames = [659456,659968) | 20260506 10:31:56.669 | INFO | MemInfo: { | 20260506 10:31:56.669 | INFO | "gpu" : " 17 GB", | 20260506 10:31:56.669 | INFO | "huge" : "", | 20260506 10:31:56.669 | INFO | "huge_shared" : " 27 GB", | 20260506 10:31:56.669 | INFO | "malloc" : " 14 GB", | 20260506 10:31:56.669 | INFO | "pinned" : " 29 GB" | 20260506 10:31:56.669 | INFO | } | 20260506 10:31:56.669 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:31:56.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:31:56.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:56.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:56.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:56.669 | INFO | ANALYSIS : 57.5%, 2.8, 85.5ms, 0 | 20260506 10:31:56.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:31:56.669 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 10:31:56.670 | INFO | PRE_HQ : 56.5%, 2.1, 60.4ms, 0 | 20260506 10:31:56.670 | INFO | BAZWRITER : 61.4%, 4.8, 150.3ms, 0 | 20260506 10:31:56.670 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 10:31:56.670 | INFO | Uploaded 36GiB to GPU in 1.67633s (21.4754GiB/s) | 20260506 10:31:56.670 | INFO | Downloaded 38GiB to GPU in 1.74249s (21.8078GiB/s) | 20260506 10:32:01.808 | INFO | Analyzing chunk frames = [659968,660480) | 20260506 10:32:01.808 | INFO | MemInfo: { | 20260506 10:32:01.808 | INFO | "gpu" : " 17 GB", | 20260506 10:32:01.808 | INFO | "huge" : "", | 20260506 10:32:01.809 | INFO | "huge_shared" : " 27 GB", | 20260506 10:32:01.809 | INFO | "malloc" : " 14 GB", | 20260506 10:32:01.809 | INFO | "pinned" : " 29 GB" | 20260506 10:32:01.809 | INFO | } | 20260506 10:32:01.809 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:32:01.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:32:01.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:01.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:01.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:01.809 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:32:01.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:01.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:01.809 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:32:01.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:01.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:01.809 | INFO | Uploaded 36GiB to GPU in 1.60341s (22.4522GiB/s) | 20260506 10:32:01.810 | INFO | Downloaded 34GiB to GPU in 1.52115s (22.3516GiB/s) | 20260506 10:32:06.948 | INFO | Analyzing chunk frames = [660480,660992) | 20260506 10:32:06.948 | INFO | MemInfo: { | 20260506 10:32:06.948 | INFO | "gpu" : " 17 GB", | 20260506 10:32:06.948 | INFO | "huge" : "", | 20260506 10:32:06.948 | INFO | "huge_shared" : " 27 GB", | 20260506 10:32:06.948 | INFO | "malloc" : " 14 GB", | 20260506 10:32:06.948 | INFO | "pinned" : " 29 GB" | 20260506 10:32:06.948 | INFO | } | 20260506 10:32:06.948 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:32:06.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:32:06.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:06.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:06.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:06.948 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:32:06.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:06.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:06.948 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:32:06.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:06.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:06.949 | INFO | Uploaded 36GiB to GPU in 1.60741s (22.3963GiB/s) | 20260506 10:32:06.949 | INFO | Downloaded 34GiB to GPU in 1.52454s (22.3018GiB/s) | 20260506 10:32:12.087 | INFO | Analyzing chunk frames = [660992,661504) | 20260506 10:32:12.087 | INFO | MemInfo: { | 20260506 10:32:12.087 | INFO | "gpu" : " 17 GB", | 20260506 10:32:12.087 | INFO | "huge" : "", | 20260506 10:32:12.087 | INFO | "huge_shared" : " 27 GB", | 20260506 10:32:12.087 | INFO | "malloc" : " 14 GB", | 20260506 10:32:12.087 | INFO | "pinned" : " 29 GB" | 20260506 10:32:12.087 | INFO | } | 20260506 10:32:12.087 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:32:12.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:32:12.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:12.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:12.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:12.088 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:32:12.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:12.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:12.088 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:32:12.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:12.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:12.088 | INFO | Uploaded 36GiB to GPU in 1.60671s (22.406GiB/s) | 20260506 10:32:12.088 | INFO | Downloaded 34GiB to GPU in 1.52338s (22.3188GiB/s) | 20260506 10:32:17.227 | INFO | Analyzing chunk frames = [661504,662016) | 20260506 10:32:17.227 | INFO | MemInfo: { | 20260506 10:32:17.227 | INFO | "gpu" : " 17 GB", | 20260506 10:32:17.227 | INFO | "huge" : "", | 20260506 10:32:17.227 | INFO | "huge_shared" : " 27 GB", | 20260506 10:32:17.227 | INFO | "malloc" : " 14 GB", | 20260506 10:32:17.227 | INFO | "pinned" : " 29 GB" | 20260506 10:32:17.227 | INFO | } | 20260506 10:32:17.227 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:32:17.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:32:17.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:17.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:17.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:17.227 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:32:17.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:17.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:17.227 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:32:17.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:17.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:17.227 | INFO | Uploaded 36GiB to GPU in 1.60558s (22.4218GiB/s) | 20260506 10:32:17.227 | INFO | Downloaded 34GiB to GPU in 1.52367s (22.3145GiB/s) | 20260506 10:32:22.366 | INFO | Analyzing chunk frames = [662016,662528) | 20260506 10:32:22.366 | INFO | MemInfo: { | 20260506 10:32:22.366 | INFO | "gpu" : " 17 GB", | 20260506 10:32:22.366 | INFO | "huge" : "", | 20260506 10:32:22.366 | INFO | "huge_shared" : " 27 GB", | 20260506 10:32:22.366 | INFO | "malloc" : " 14 GB", | 20260506 10:32:22.366 | INFO | "pinned" : " 29 GB" | 20260506 10:32:22.366 | INFO | } | 20260506 10:32:22.366 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:32:22.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:32:22.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:22.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:22.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:22.366 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:32:22.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:22.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:22.366 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:32:22.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:22.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:22.367 | INFO | Uploaded 36GiB to GPU in 1.60521s (22.4269GiB/s) | 20260506 10:32:22.367 | INFO | Downloaded 34GiB to GPU in 1.52239s (22.3332GiB/s) | 20260506 10:32:27.505 | INFO | Analyzing chunk frames = [662528,663040) | 20260506 10:32:27.505 | INFO | MemInfo: { | 20260506 10:32:27.505 | INFO | "gpu" : " 17 GB", | 20260506 10:32:27.505 | INFO | "huge" : "", | 20260506 10:32:27.505 | INFO | "huge_shared" : " 27 GB", | 20260506 10:32:27.505 | INFO | "malloc" : " 14 GB", | 20260506 10:32:27.505 | INFO | "pinned" : " 29 GB" | 20260506 10:32:27.505 | INFO | } | 20260506 10:32:27.505 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:32:27.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:32:27.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:27.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:27.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:27.506 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:32:27.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:27.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:27.506 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:32:27.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:27.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:27.506 | INFO | Uploaded 36GiB to GPU in 1.6056s (22.4216GiB/s) | 20260506 10:32:27.506 | INFO | Downloaded 34GiB to GPU in 1.52362s (22.3153GiB/s) | 20260506 10:32:32.645 | INFO | Analyzing chunk frames = [663040,663552) | 20260506 10:32:32.645 | INFO | MemInfo: { | 20260506 10:32:32.645 | INFO | "gpu" : " 17 GB", | 20260506 10:32:32.645 | INFO | "huge" : "", | 20260506 10:32:32.645 | INFO | "huge_shared" : " 27 GB", | 20260506 10:32:32.645 | INFO | "malloc" : " 14 GB", | 20260506 10:32:32.645 | INFO | "pinned" : " 29 GB" | 20260506 10:32:32.645 | INFO | } | 20260506 10:32:32.645 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:32:32.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:32:32.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:32.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:32.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:32.645 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:32:32.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:32.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:32.645 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:32:32.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:32.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:32.645 | INFO | Uploaded 36GiB to GPU in 1.60609s (22.4146GiB/s) | 20260506 10:32:32.645 | INFO | Downloaded 34GiB to GPU in 1.52356s (22.3161GiB/s) | 20260506 10:32:32.646 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:32:32.646 | INFO | Total mem usage: 16.5 GiB | 20260506 10:32:32.646 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:32:32.646 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -6.43%) | 20260506 10:32:32.646 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:32:32.646 | INFO | High Water Breakdown: | 20260506 10:32:32.646 | INFO | 60.3% : 9.35 GiB ( 92.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:32:32.646 | INFO | 39.6% : 6.15 GiB ( 91.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:32:32.646 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:32:32.646 | INFO | ------------------------------------------ | 20260506 10:32:32.646 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:32:32.646 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:32:32.646 | INFO | Total mem usage: 26.5 GiB | 20260506 10:32:32.646 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:32:32.646 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:32:32.646 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:32:32.646 | INFO | High Water Breakdown: | 20260506 10:32:32.646 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:32:32.646 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:32:32.646 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:32:32.646 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:32:32.646 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:32:32.646 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:32:32.646 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:32:32.646 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:32:32.646 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:32:32.646 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:32:32.646 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:32:32.646 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:32:32.646 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:32:32.646 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:32:32.646 | INFO | ------------------------------------------ | 20260506 10:32:32.646 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:32:32.646 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:32:32.646 | INFO | Total mem usage: 25.6 GiB | 20260506 10:32:32.646 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:32:32.646 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:32:32.646 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:32:32.646 | INFO | High Water Breakdown: | 20260506 10:32:32.646 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:32:32.646 | INFO | ------------------------------------------ | 20260506 10:32:32.646 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:32:32.646 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:32:32.646 | INFO | Total mem usage: 15.7 GiB | 20260506 10:32:32.646 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:32:32.646 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:32:32.646 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:32:32.646 | INFO | High Water Breakdown: | 20260506 10:32:32.646 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:32:32.646 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:32:32.646 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:32:32.646 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:32:32.646 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:32:32.646 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:32:32.646 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:32:32.646 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:32:32.646 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:32:32.646 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:32:32.646 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:32:32.646 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:32:32.647 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:32:32.647 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:32:32.647 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:32:32.647 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:32:32.647 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:32:32.647 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:32:32.647 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:32:32.647 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:32:32.647 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:32:32.647 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:32:32.647 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:32:32.647 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:32:32.647 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:32:32.647 | INFO | ------------------------------------------ | 20260506 10:32:32.647 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:32:32.647 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:32:32.647 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:32:32.647 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:32:32.647 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 10:32:32.647 | INFO | PerfReport: 0.29%, AggregateStats -- 8.63e-02 / -nan, 4.52e-03 / 0.00e+00 | 20260506 10:32:32.647 | INFO | PerfReport: 10.96%, Baseline -- 3.30e+00 / -nan, 1.80e-02 / 0.00e+00 | 20260506 10:32:32.647 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.21e-02 / 0.00e+00 | 20260506 10:32:32.647 | INFO | PerfReport: 16.06%, DME -- 3.86e+01 / -nan, 3.10e+00 / 0.00e+00 | 20260506 10:32:32.647 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.72e+01 / -nan, 4.21e-02 / 0.00e+00 | 20260506 10:32:32.647 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.22e+00 / -nan, 1.27e-01 / 0.00e+00 | 20260506 10:32:32.647 | INFO | PerfReport: 3.66%, Metrics -- 1.10e+00 / -nan, 1.72e-01 / 0.00e+00 | 20260506 10:32:32.647 | INFO | PerfReport: 1.42%, PulseSerializing -- 4.26e-01 / -nan, 6.03e-02 / 0.00e+00 | 20260506 10:32:32.647 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:32:32.647 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:32:32.647 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:32:32.647 | INFO | PerfReport: Full Execution -- 8.99e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 10:32:32.647 | INFO | PerfReport: 18.69%, Upload -- 1.68e+01 / 0.00e+00, 4.77e-01 / 0.00e+00 | 20260506 10:32:32.647 | INFO | PerfReport: 17.95%, Download -- 1.61e+01 / 0.00e+00, 8.28e-01 / 0.00e+00 | 20260506 10:32:32.647 | INFO | PerfReport: 33.45%, Compute -- 3.01e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 10:32:32.647 | INFO | PerfReport: 0.56%, LockUpload -- 5.08e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:32:32.647 | INFO | PerfReport: 0.00%, LockDownload -- 4.88e-04 / 0.00e+00, 1.25e-04 / 0.00e+00 | 20260506 10:32:32.647 | INFO | PerfReport: 29.33%, LockCompute -- 2.64e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 10:32:32.647 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.17e-02 / 0.00e+00 | 20260506 10:32:32.742 | INFO | Starting a round of Baz IO | 20260506 10:32:35.655 | INFO | Populating RTMetrics report | 20260506 10:32:35.655 | INFO | RTMetrics report populated | 20260506 10:32:35.655 | INFO | Opening tmp RTMetrics json file | 20260506 10:32:35.693 | INFO | Serializing RTMetrics to json | 20260506 10:32:35.740 | INFO | Serialization done, swapping tmp file now... | 20260506 10:32:35.741 | INFO | RTMetrics done | 20260506 10:32:36.183 | INFO | Finished a round of Baz IO: Wrote 1.992 GiB in 3.441 seconds | 20260506 10:32:36.184 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.48% / 0.05% / 52.47% | 20260506 10:32:36.184 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.64353% / 0.00013113% / 3.17891e-05% | 20260506 10:32:36.184 | INFO | Baz IO overall: | 20260506 10:32:36.184 | INFO | BytesWritten : 300697147874 | 20260506 10:32:36.184 | INFO | NumEvents : 144567908502 | 20260506 10:32:37.784 | INFO | Analyzing chunk frames = [663552,664064) | 20260506 10:32:37.784 | INFO | MemInfo: { | 20260506 10:32:37.784 | INFO | "gpu" : " 17 GB", | 20260506 10:32:37.784 | INFO | "huge" : "", | 20260506 10:32:37.784 | INFO | "huge_shared" : " 27 GB", | 20260506 10:32:37.784 | INFO | "malloc" : " 14 GB", | 20260506 10:32:37.784 | INFO | "pinned" : " 29 GB" | 20260506 10:32:37.784 | INFO | } | 20260506 10:32:37.784 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:32:37.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:32:37.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:37.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:37.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:37.784 | INFO | ANALYSIS : 57.5%, 2.8, 85.1ms, 0 | 20260506 10:32:37.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:37.784 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260506 10:32:37.784 | INFO | PRE_HQ : 56.6%, 2.1, 60.7ms, 0 | 20260506 10:32:37.784 | INFO | BAZWRITER : 66.5%, 4.8, 163.2ms, 0 | 20260506 10:32:37.784 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 10:32:37.784 | INFO | Uploaded 36GiB to GPU in 1.67081s (21.5464GiB/s) | 20260506 10:32:37.784 | INFO | Downloaded 38GiB to GPU in 1.7364s (21.8843GiB/s) | 20260506 10:32:42.923 | INFO | Analyzing chunk frames = [664064,664576) | 20260506 10:32:42.923 | INFO | MemInfo: { | 20260506 10:32:42.923 | INFO | "gpu" : " 17 GB", | 20260506 10:32:42.923 | INFO | "huge" : "", | 20260506 10:32:42.924 | INFO | "huge_shared" : " 27 GB", | 20260506 10:32:42.924 | INFO | "malloc" : " 14 GB", | 20260506 10:32:42.924 | INFO | "pinned" : " 29 GB" | 20260506 10:32:42.924 | INFO | } | 20260506 10:32:42.924 | INFO | Perf report for the last 5.13944 seconds: | 20260506 10:32:42.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:32:42.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:42.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:42.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:42.924 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:32:42.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:42.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:42.924 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:32:42.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:42.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:42.924 | INFO | Uploaded 36GiB to GPU in 1.60299s (22.4581GiB/s) | 20260506 10:32:42.924 | INFO | Downloaded 34GiB to GPU in 1.52021s (22.3653GiB/s) | 20260506 10:32:48.063 | INFO | Analyzing chunk frames = [664576,665088) | 20260506 10:32:48.063 | INFO | MemInfo: { | 20260506 10:32:48.063 | INFO | "gpu" : " 17 GB", | 20260506 10:32:48.063 | INFO | "huge" : "", | 20260506 10:32:48.063 | INFO | "huge_shared" : " 27 GB", | 20260506 10:32:48.063 | INFO | "malloc" : " 14 GB", | 20260506 10:32:48.063 | INFO | "pinned" : " 29 GB" | 20260506 10:32:48.063 | INFO | } | 20260506 10:32:48.063 | INFO | Perf report for the last 5.13918 seconds: | 20260506 10:32:48.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:32:48.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:48.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:48.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:48.063 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:32:48.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:48.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:48.063 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:32:48.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:48.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:48.063 | INFO | Uploaded 36GiB to GPU in 1.60803s (22.3876GiB/s) | 20260506 10:32:48.063 | INFO | Downloaded 34GiB to GPU in 1.52588s (22.2823GiB/s) | 20260506 10:32:53.202 | INFO | Analyzing chunk frames = [665088,665600) | 20260506 10:32:53.202 | INFO | MemInfo: { | 20260506 10:32:53.202 | INFO | "gpu" : " 17 GB", | 20260506 10:32:53.202 | INFO | "huge" : "", | 20260506 10:32:53.202 | INFO | "huge_shared" : " 27 GB", | 20260506 10:32:53.202 | INFO | "malloc" : " 14 GB", | 20260506 10:32:53.202 | INFO | "pinned" : " 29 GB" | 20260506 10:32:53.202 | INFO | } | 20260506 10:32:53.202 | INFO | Perf report for the last 5.13907 seconds: | 20260506 10:32:53.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:32:53.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:53.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:53.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:53.202 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:32:53.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:53.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:53.202 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:32:53.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:53.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:53.202 | INFO | Uploaded 36GiB to GPU in 1.60715s (22.3999GiB/s) | 20260506 10:32:53.202 | INFO | Downloaded 34GiB to GPU in 1.52477s (22.2984GiB/s) | 20260506 10:32:58.341 | INFO | Analyzing chunk frames = [665600,666112) | 20260506 10:32:58.341 | INFO | MemInfo: { | 20260506 10:32:58.341 | INFO | "gpu" : " 17 GB", | 20260506 10:32:58.341 | INFO | "huge" : "", | 20260506 10:32:58.341 | INFO | "huge_shared" : " 27 GB", | 20260506 10:32:58.341 | INFO | "malloc" : " 14 GB", | 20260506 10:32:58.341 | INFO | "pinned" : " 29 GB" | 20260506 10:32:58.341 | INFO | } | 20260506 10:32:58.341 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:32:58.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:32:58.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:58.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:58.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:58.341 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:32:58.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:58.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:58.341 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:32:58.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:58.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:32:58.342 | INFO | Uploaded 36GiB to GPU in 1.60535s (22.425GiB/s) | 20260506 10:32:58.342 | INFO | Downloaded 34GiB to GPU in 1.52216s (22.3367GiB/s) | 20260506 10:33:03.480 | INFO | Analyzing chunk frames = [666112,666624) | 20260506 10:33:03.480 | INFO | MemInfo: { | 20260506 10:33:03.480 | INFO | "gpu" : " 17 GB", | 20260506 10:33:03.480 | INFO | "huge" : "", | 20260506 10:33:03.481 | INFO | "huge_shared" : " 27 GB", | 20260506 10:33:03.481 | INFO | "malloc" : " 14 GB", | 20260506 10:33:03.481 | INFO | "pinned" : " 29 GB" | 20260506 10:33:03.481 | INFO | } | 20260506 10:33:03.481 | INFO | Perf report for the last 5.13947 seconds: | 20260506 10:33:03.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:33:03.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:03.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:03.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:03.481 | INFO | ANALYSIS : 56.6%, 3.0, 88.4ms, 0 | 20260506 10:33:03.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:03.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:03.481 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:33:03.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:03.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:03.481 | INFO | Uploaded 36GiB to GPU in 1.6056s (22.4215GiB/s) | 20260506 10:33:03.481 | INFO | Downloaded 34GiB to GPU in 1.52395s (22.3104GiB/s) | 20260506 10:33:08.620 | INFO | Analyzing chunk frames = [666624,667136) | 20260506 10:33:08.620 | INFO | MemInfo: { | 20260506 10:33:08.620 | INFO | "gpu" : " 17 GB", | 20260506 10:33:08.620 | INFO | "huge" : "", | 20260506 10:33:08.620 | INFO | "huge_shared" : " 27 GB", | 20260506 10:33:08.620 | INFO | "malloc" : " 14 GB", | 20260506 10:33:08.620 | INFO | "pinned" : " 29 GB" | 20260506 10:33:08.620 | INFO | } | 20260506 10:33:08.620 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:33:08.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:33:08.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:08.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:08.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:08.620 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:33:08.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:08.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:08.620 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:33:08.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:08.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:08.621 | INFO | Uploaded 36GiB to GPU in 1.60541s (22.4241GiB/s) | 20260506 10:33:08.621 | INFO | Downloaded 34GiB to GPU in 1.5236s (22.3156GiB/s) | 20260506 10:33:13.759 | INFO | Analyzing chunk frames = [667136,667648) | 20260506 10:33:13.759 | INFO | MemInfo: { | 20260506 10:33:13.759 | INFO | "gpu" : " 17 GB", | 20260506 10:33:13.759 | INFO | "huge" : "", | 20260506 10:33:13.759 | INFO | "huge_shared" : " 27 GB", | 20260506 10:33:13.759 | INFO | "malloc" : " 14 GB", | 20260506 10:33:13.759 | INFO | "pinned" : " 29 GB" | 20260506 10:33:13.759 | INFO | } | 20260506 10:33:13.759 | INFO | Perf report for the last 5.1393 seconds: | 20260506 10:33:13.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:33:13.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:13.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:13.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:13.760 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:33:13.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:13.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:13.760 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:33:13.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:13.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:13.760 | INFO | Uploaded 36GiB to GPU in 1.60535s (22.425GiB/s) | 20260506 10:33:13.760 | INFO | Downloaded 34GiB to GPU in 1.5242s (22.3067GiB/s) | 20260506 10:33:13.862 | INFO | Starting a round of Baz IO | 20260506 10:33:16.768 | INFO | Populating RTMetrics report | 20260506 10:33:16.769 | INFO | RTMetrics report populated | 20260506 10:33:16.769 | INFO | Opening tmp RTMetrics json file | 20260506 10:33:16.781 | INFO | Serializing RTMetrics to json | 20260506 10:33:16.831 | INFO | Serialization done, swapping tmp file now... | 20260506 10:33:16.893 | INFO | RTMetrics done | 20260506 10:33:17.307 | INFO | Finished a round of Baz IO: Wrote 1.994 GiB in 3.445 seconds | 20260506 10:33:17.307 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.43% / 0.05% / 52.52% | 20260506 10:33:17.307 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.63849% / 9.13938e-05% / 3.57628e-05% | 20260506 10:33:17.307 | INFO | Baz IO overall: | 20260506 10:33:17.308 | INFO | BytesWritten : 302838222744 | 20260506 10:33:17.308 | INFO | NumEvents : 145607354203 | 20260506 10:33:18.899 | INFO | Analyzing chunk frames = [667648,668160) | 20260506 10:33:18.899 | INFO | MemInfo: { | 20260506 10:33:18.899 | INFO | "gpu" : " 17 GB", | 20260506 10:33:18.899 | INFO | "huge" : "", | 20260506 10:33:18.899 | INFO | "huge_shared" : " 27 GB", | 20260506 10:33:18.899 | INFO | "malloc" : " 14 GB", | 20260506 10:33:18.899 | INFO | "pinned" : " 29 GB" | 20260506 10:33:18.899 | INFO | } | 20260506 10:33:18.899 | INFO | Perf report for the last 5.1392 seconds: | 20260506 10:33:18.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:33:18.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:18.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:18.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:18.899 | INFO | ANALYSIS : 57.6%, 2.8, 85.6ms, 0 | 20260506 10:33:18.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:18.899 | INFO | RT_METRICS : 46.6%, 1.0, 24.5ms, 0 | 20260506 10:33:18.899 | INFO | PRE_HQ : 56.7%, 2.1, 61.3ms, 0 | 20260506 10:33:18.899 | INFO | BAZWRITER : 66.5%, 4.8, 162.9ms, 0 | 20260506 10:33:18.899 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 10:33:18.899 | INFO | Uploaded 36GiB to GPU in 1.66461s (21.6267GiB/s) | 20260506 10:33:18.899 | INFO | Downloaded 38GiB to GPU in 1.72955s (21.971GiB/s) | 20260506 10:33:24.038 | INFO | Analyzing chunk frames = [668160,668672) | 20260506 10:33:24.038 | INFO | MemInfo: { | 20260506 10:33:24.038 | INFO | "gpu" : " 17 GB", | 20260506 10:33:24.038 | INFO | "huge" : "", | 20260506 10:33:24.038 | INFO | "huge_shared" : " 27 GB", | 20260506 10:33:24.038 | INFO | "malloc" : " 14 GB", | 20260506 10:33:24.038 | INFO | "pinned" : " 29 GB" | 20260506 10:33:24.038 | INFO | } | 20260506 10:33:24.038 | INFO | Perf report for the last 5.13956 seconds: | 20260506 10:33:24.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:33:24.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:24.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:24.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:24.038 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:33:24.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:24.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:24.038 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:33:24.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:24.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:24.039 | INFO | Uploaded 36GiB to GPU in 1.6033s (22.4537GiB/s) | 20260506 10:33:24.039 | INFO | Downloaded 34GiB to GPU in 1.52022s (22.3652GiB/s) | 20260506 10:33:29.178 | INFO | Analyzing chunk frames = [668672,669184) | 20260506 10:33:29.178 | INFO | MemInfo: { | 20260506 10:33:29.178 | INFO | "gpu" : " 17 GB", | 20260506 10:33:29.178 | INFO | "huge" : "", | 20260506 10:33:29.178 | INFO | "huge_shared" : " 27 GB", | 20260506 10:33:29.178 | INFO | "malloc" : " 14 GB", | 20260506 10:33:29.178 | INFO | "pinned" : " 29 GB" | 20260506 10:33:29.178 | INFO | } | 20260506 10:33:29.178 | INFO | Perf report for the last 5.13965 seconds: | 20260506 10:33:29.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:33:29.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:29.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:29.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:29.178 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:33:29.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:29.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:29.178 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:33:29.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:29.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:29.179 | INFO | Uploaded 36GiB to GPU in 1.60706s (22.4011GiB/s) | 20260506 10:33:29.179 | INFO | Downloaded 34GiB to GPU in 1.52493s (22.296GiB/s) | 20260506 10:33:34.317 | INFO | Analyzing chunk frames = [669184,669696) | 20260506 10:33:34.317 | INFO | MemInfo: { | 20260506 10:33:34.317 | INFO | "gpu" : " 17 GB", | 20260506 10:33:34.317 | INFO | "huge" : "", | 20260506 10:33:34.317 | INFO | "huge_shared" : " 27 GB", | 20260506 10:33:34.317 | INFO | "malloc" : " 14 GB", | 20260506 10:33:34.317 | INFO | "pinned" : " 29 GB" | 20260506 10:33:34.317 | INFO | } | 20260506 10:33:34.317 | INFO | Perf report for the last 5.13909 seconds: | 20260506 10:33:34.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:33:34.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:34.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:34.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:34.317 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 10:33:34.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:34.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:34.317 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:33:34.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:34.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:34.318 | INFO | Uploaded 36GiB to GPU in 1.60704s (22.4015GiB/s) | 20260506 10:33:34.318 | INFO | Downloaded 34GiB to GPU in 1.52458s (22.3013GiB/s) | 20260506 10:33:39.456 | INFO | Analyzing chunk frames = [669696,670208) | 20260506 10:33:39.456 | INFO | MemInfo: { | 20260506 10:33:39.456 | INFO | "gpu" : " 17 GB", | 20260506 10:33:39.456 | INFO | "huge" : "", | 20260506 10:33:39.456 | INFO | "huge_shared" : " 27 GB", | 20260506 10:33:39.456 | INFO | "malloc" : " 14 GB", | 20260506 10:33:39.456 | INFO | "pinned" : " 29 GB" | 20260506 10:33:39.456 | INFO | } | 20260506 10:33:39.456 | INFO | Perf report for the last 5.1392 seconds: | 20260506 10:33:39.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:33:39.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:39.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:39.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:39.456 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260506 10:33:39.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:39.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:39.456 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:33:39.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:39.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:39.457 | INFO | Uploaded 36GiB to GPU in 1.60526s (22.4263GiB/s) | 20260506 10:33:39.457 | INFO | Downloaded 34GiB to GPU in 1.5222s (22.3361GiB/s) | 20260506 10:33:44.596 | INFO | Analyzing chunk frames = [670208,670720) | 20260506 10:33:44.596 | INFO | MemInfo: { | 20260506 10:33:44.596 | INFO | "gpu" : " 17 GB", | 20260506 10:33:44.596 | INFO | "huge" : "", | 20260506 10:33:44.596 | INFO | "huge_shared" : " 27 GB", | 20260506 10:33:44.596 | INFO | "malloc" : " 14 GB", | 20260506 10:33:44.596 | INFO | "pinned" : " 29 GB" | 20260506 10:33:44.596 | INFO | } | 20260506 10:33:44.596 | INFO | Perf report for the last 5.13965 seconds: | 20260506 10:33:44.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:33:44.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:44.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:44.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:44.596 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 10:33:44.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:44.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:44.596 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:33:44.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:44.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:44.596 | INFO | Uploaded 36GiB to GPU in 1.6053s (22.4257GiB/s) | 20260506 10:33:44.596 | INFO | Downloaded 34GiB to GPU in 1.52274s (22.3281GiB/s) | 20260506 10:33:49.735 | INFO | Analyzing chunk frames = [670720,671232) | 20260506 10:33:49.735 | INFO | MemInfo: { | 20260506 10:33:49.735 | INFO | "gpu" : " 17 GB", | 20260506 10:33:49.735 | INFO | "huge" : "", | 20260506 10:33:49.735 | INFO | "huge_shared" : " 27 GB", | 20260506 10:33:49.735 | INFO | "malloc" : " 14 GB", | 20260506 10:33:49.735 | INFO | "pinned" : " 29 GB" | 20260506 10:33:49.735 | INFO | } | 20260506 10:33:49.735 | INFO | Perf report for the last 5.13914 seconds: | 20260506 10:33:49.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:33:49.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:49.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:49.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:49.735 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:33:49.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:49.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:49.735 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:33:49.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:49.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:49.735 | INFO | Uploaded 36GiB to GPU in 1.60548s (22.4232GiB/s) | 20260506 10:33:49.735 | INFO | Downloaded 34GiB to GPU in 1.52357s (22.316GiB/s) | 20260506 10:33:54.874 | INFO | Analyzing chunk frames = [671232,671744) | 20260506 10:33:54.874 | INFO | MemInfo: { | 20260506 10:33:54.874 | INFO | "gpu" : " 17 GB", | 20260506 10:33:54.874 | INFO | "huge" : "", | 20260506 10:33:54.874 | INFO | "huge_shared" : " 27 GB", | 20260506 10:33:54.874 | INFO | "malloc" : " 14 GB", | 20260506 10:33:54.874 | INFO | "pinned" : " 29 GB" | 20260506 10:33:54.874 | INFO | } | 20260506 10:33:54.874 | INFO | Perf report for the last 5.13939 seconds: | 20260506 10:33:54.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:33:54.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:54.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:54.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:54.875 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:33:54.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:54.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:54.875 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:33:54.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:54.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:33:54.875 | INFO | Uploaded 36GiB to GPU in 1.60608s (22.4149GiB/s) | 20260506 10:33:54.875 | INFO | Downloaded 34GiB to GPU in 1.52409s (22.3085GiB/s) | 20260506 10:33:54.875 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:33:54.875 | INFO | Total mem usage: 16.5 GiB | 20260506 10:33:54.875 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:33:54.875 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -2.79%) | 20260506 10:33:54.875 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:33:54.875 | INFO | High Water Breakdown: | 20260506 10:33:54.875 | INFO | 60.8% : 9.77 GiB ( 96.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:33:54.875 | INFO | 39% : 6.27 GiB ( 93.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:33:54.875 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:33:54.875 | INFO | ------------------------------------------ | 20260506 10:33:54.875 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:33:54.875 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:33:54.875 | INFO | Total mem usage: 26.5 GiB | 20260506 10:33:54.875 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:33:54.875 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:33:54.875 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:33:54.875 | INFO | High Water Breakdown: | 20260506 10:33:54.875 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:33:54.875 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:33:54.875 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:33:54.875 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:33:54.875 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:33:54.875 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:33:54.875 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:33:54.876 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:33:54.876 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:33:54.876 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:33:54.876 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:33:54.876 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:33:54.876 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:33:54.876 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:33:54.876 | INFO | ------------------------------------------ | 20260506 10:33:54.876 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:33:54.876 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:33:54.876 | INFO | Total mem usage: 25.6 GiB | 20260506 10:33:54.876 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:33:54.876 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:33:54.876 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:33:54.876 | INFO | High Water Breakdown: | 20260506 10:33:54.876 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:33:54.876 | INFO | ------------------------------------------ | 20260506 10:33:54.876 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:33:54.876 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:33:54.876 | INFO | Total mem usage: 15.7 GiB | 20260506 10:33:54.876 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:33:54.876 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:33:54.876 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:33:54.876 | INFO | High Water Breakdown: | 20260506 10:33:54.876 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:33:54.876 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:33:54.876 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:33:54.876 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:33:54.876 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:33:54.876 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:33:54.876 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:33:54.876 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:33:54.876 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:33:54.876 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:33:54.876 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:33:54.876 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:33:54.876 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:33:54.876 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:33:54.876 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:33:54.876 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:33:54.876 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:33:54.876 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:33:54.876 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:33:54.876 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:33:54.876 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:33:54.876 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:33:54.876 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:33:54.876 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:33:54.876 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:33:54.876 | INFO | ------------------------------------------ | 20260506 10:33:54.876 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:33:54.876 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:33:54.876 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:33:54.876 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:33:54.876 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 10:33:54.876 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.58e-03 / 0.00e+00 | 20260506 10:33:54.876 | INFO | PerfReport: 10.95%, Baseline -- 3.29e+00 / -nan, 1.81e-02 / 0.00e+00 | 20260506 10:33:54.876 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.27e-02 / 0.00e+00 | 20260506 10:33:54.876 | INFO | PerfReport: 16.06%, DME -- 3.86e+01 / -nan, 3.04e+00 / 0.00e+00 | 20260506 10:33:54.876 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.72e+01 / -nan, 4.33e-02 / 0.00e+00 | 20260506 10:33:54.876 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.22e+00 / -nan, 1.27e-01 / 0.00e+00 | 20260506 10:33:54.876 | INFO | PerfReport: 3.66%, Metrics -- 1.10e+00 / -nan, 1.72e-01 / 0.00e+00 | 20260506 10:33:54.876 | INFO | PerfReport: 1.42%, PulseSerializing -- 4.26e-01 / -nan, 6.03e-02 / 0.00e+00 | 20260506 10:33:54.876 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:33:54.876 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:33:54.876 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:33:54.876 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.85e+01 / 0.00e+00 | 20260506 10:33:54.876 | INFO | PerfReport: 18.71%, Upload -- 1.68e+01 / 0.00e+00, 4.72e-01 / 0.00e+00 | 20260506 10:33:54.876 | INFO | PerfReport: 17.97%, Download -- 1.61e+01 / 0.00e+00, 8.21e-01 / 0.00e+00 | 20260506 10:33:54.876 | INFO | PerfReport: 33.50%, Compute -- 3.01e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 10:33:54.876 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:33:54.876 | INFO | PerfReport: 0.00%, LockDownload -- 4.89e-04 / 0.00e+00, 1.66e-04 / 0.00e+00 | 20260506 10:33:54.876 | INFO | PerfReport: 29.24%, LockCompute -- 2.63e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260506 10:33:54.876 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.26e-02 / 0.00e+00 | 20260506 10:33:54.971 | INFO | Starting a round of Baz IO | 20260506 10:33:57.887 | INFO | Populating RTMetrics report | 20260506 10:33:57.888 | INFO | RTMetrics report populated | 20260506 10:33:57.888 | INFO | Opening tmp RTMetrics json file | 20260506 10:33:57.936 | INFO | Serializing RTMetrics to json | 20260506 10:33:57.938 | INFO | Serialization done, swapping tmp file now... | 20260506 10:33:57.938 | INFO | RTMetrics done | 20260506 10:33:58.227 | INFO | Finished a round of Baz IO: Wrote 1.996 GiB in 3.255 seconds | 20260506 10:33:58.227 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.38% / 0.05% / 52.57% | 20260506 10:33:58.227 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.62173% / 0.000127157% / 3.17891e-05% | 20260506 10:33:58.227 | INFO | Baz IO overall: | 20260506 10:33:58.227 | INFO | BytesWritten : 304981697072 | 20260506 10:33:58.227 | INFO | NumEvents : 146648215251 | 20260506 10:34:00.013 | INFO | Analyzing chunk frames = [671744,672256) | 20260506 10:34:00.013 | INFO | MemInfo: { | 20260506 10:34:00.013 | INFO | "gpu" : " 17 GB", | 20260506 10:34:00.013 | INFO | "huge" : "", | 20260506 10:34:00.013 | INFO | "huge_shared" : " 27 GB", | 20260506 10:34:00.013 | INFO | "malloc" : " 14 GB", | 20260506 10:34:00.013 | INFO | "pinned" : " 29 GB" | 20260506 10:34:00.013 | INFO | } | 20260506 10:34:00.013 | INFO | Perf report for the last 5.13897 seconds: | 20260506 10:34:00.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:34:00.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:00.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:00.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:00.014 | INFO | ANALYSIS : 57.6%, 2.9, 86.1ms, 0 | 20260506 10:34:00.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:00.014 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 10:34:00.014 | INFO | PRE_HQ : 56.6%, 2.1, 60.6ms, 0 | 20260506 10:34:00.014 | INFO | BAZWRITER : 62.9%, 4.7, 151.6ms, 0 | 20260506 10:34:00.014 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 10:34:00.014 | INFO | Uploaded 36GiB to GPU in 1.67701s (21.4667GiB/s) | 20260506 10:34:00.014 | INFO | Downloaded 38GiB to GPU in 1.74086s (21.8282GiB/s) | 20260506 10:34:05.153 | INFO | Analyzing chunk frames = [672256,672768) | 20260506 10:34:05.153 | INFO | MemInfo: { | 20260506 10:34:05.153 | INFO | "gpu" : " 17 GB", | 20260506 10:34:05.153 | INFO | "huge" : "", | 20260506 10:34:05.153 | INFO | "huge_shared" : " 27 GB", | 20260506 10:34:05.153 | INFO | "malloc" : " 14 GB", | 20260506 10:34:05.153 | INFO | "pinned" : " 29 GB" | 20260506 10:34:05.153 | INFO | } | 20260506 10:34:05.153 | INFO | Perf report for the last 5.13959 seconds: | 20260506 10:34:05.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:34:05.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:05.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:05.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:05.153 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:34:05.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:05.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:05.153 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:34:05.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:05.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:05.154 | INFO | Uploaded 36GiB to GPU in 1.6032s (22.4551GiB/s) | 20260506 10:34:05.154 | INFO | Downloaded 34GiB to GPU in 1.52021s (22.3653GiB/s) | 20260506 10:34:10.292 | INFO | Analyzing chunk frames = [672768,673280) | 20260506 10:34:10.292 | INFO | MemInfo: { | 20260506 10:34:10.292 | INFO | "gpu" : " 17 GB", | 20260506 10:34:10.292 | INFO | "huge" : "", | 20260506 10:34:10.292 | INFO | "huge_shared" : " 27 GB", | 20260506 10:34:10.292 | INFO | "malloc" : " 14 GB", | 20260506 10:34:10.292 | INFO | "pinned" : " 29 GB" | 20260506 10:34:10.292 | INFO | } | 20260506 10:34:10.292 | INFO | Perf report for the last 5.13929 seconds: | 20260506 10:34:10.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:34:10.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:10.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:10.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:10.292 | INFO | ANALYSIS : 56.7%, 3.0, 88.2ms, 0 | 20260506 10:34:10.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:10.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:10.292 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:34:10.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:10.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:10.293 | INFO | Uploaded 36GiB to GPU in 1.60688s (22.4037GiB/s) | 20260506 10:34:10.293 | INFO | Downloaded 34GiB to GPU in 1.52438s (22.3041GiB/s) | 20260506 10:34:15.431 | INFO | Analyzing chunk frames = [673280,673792) | 20260506 10:34:15.431 | INFO | MemInfo: { | 20260506 10:34:15.431 | INFO | "gpu" : " 17 GB", | 20260506 10:34:15.431 | INFO | "huge" : "", | 20260506 10:34:15.431 | INFO | "huge_shared" : " 27 GB", | 20260506 10:34:15.431 | INFO | "malloc" : " 14 GB", | 20260506 10:34:15.431 | INFO | "pinned" : " 29 GB" | 20260506 10:34:15.432 | INFO | } | 20260506 10:34:15.432 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:34:15.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:34:15.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:15.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:15.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:15.432 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:34:15.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:15.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:15.432 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:34:15.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:15.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:15.432 | INFO | Uploaded 36GiB to GPU in 1.60717s (22.3996GiB/s) | 20260506 10:34:15.432 | INFO | Downloaded 34GiB to GPU in 1.52485s (22.2972GiB/s) | 20260506 10:34:20.571 | INFO | Analyzing chunk frames = [673792,674304) | 20260506 10:34:20.571 | INFO | MemInfo: { | 20260506 10:34:20.571 | INFO | "gpu" : " 17 GB", | 20260506 10:34:20.571 | INFO | "huge" : "", | 20260506 10:34:20.571 | INFO | "huge_shared" : " 27 GB", | 20260506 10:34:20.571 | INFO | "malloc" : " 14 GB", | 20260506 10:34:20.571 | INFO | "pinned" : " 29 GB" | 20260506 10:34:20.571 | INFO | } | 20260506 10:34:20.571 | INFO | Perf report for the last 5.1393 seconds: | 20260506 10:34:20.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:34:20.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:20.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:20.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:20.571 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260506 10:34:20.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:20.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:20.571 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:34:20.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:20.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:20.571 | INFO | Uploaded 36GiB to GPU in 1.6054s (22.4243GiB/s) | 20260506 10:34:20.571 | INFO | Downloaded 34GiB to GPU in 1.52238s (22.3334GiB/s) | 20260506 10:34:25.710 | INFO | Analyzing chunk frames = [674304,674816) | 20260506 10:34:25.710 | INFO | MemInfo: { | 20260506 10:34:25.710 | INFO | "gpu" : " 17 GB", | 20260506 10:34:25.710 | INFO | "huge" : "", | 20260506 10:34:25.710 | INFO | "huge_shared" : " 27 GB", | 20260506 10:34:25.710 | INFO | "malloc" : " 14 GB", | 20260506 10:34:25.710 | INFO | "pinned" : " 29 GB" | 20260506 10:34:25.710 | INFO | } | 20260506 10:34:25.710 | INFO | Perf report for the last 5.13925 seconds: | 20260506 10:34:25.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:34:25.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:25.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:25.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:25.710 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:34:25.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:25.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:25.710 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:34:25.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:25.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:25.711 | INFO | Uploaded 36GiB to GPU in 1.60525s (22.4263GiB/s) | 20260506 10:34:25.711 | INFO | Downloaded 34GiB to GPU in 1.52315s (22.3222GiB/s) | 20260506 10:34:30.849 | INFO | Analyzing chunk frames = [674816,675328) | 20260506 10:34:30.849 | INFO | MemInfo: { | 20260506 10:34:30.849 | INFO | "gpu" : " 17 GB", | 20260506 10:34:30.849 | INFO | "huge" : "", | 20260506 10:34:30.849 | INFO | "huge_shared" : " 27 GB", | 20260506 10:34:30.849 | INFO | "malloc" : " 14 GB", | 20260506 10:34:30.850 | INFO | "pinned" : " 29 GB" | 20260506 10:34:30.850 | INFO | } | 20260506 10:34:30.850 | INFO | Perf report for the last 5.13943 seconds: | 20260506 10:34:30.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:34:30.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:30.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:30.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:30.850 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 10:34:30.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:30.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:30.850 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:34:30.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:30.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:30.850 | INFO | Uploaded 36GiB to GPU in 1.60562s (22.4213GiB/s) | 20260506 10:34:30.850 | INFO | Downloaded 34GiB to GPU in 1.52349s (22.3171GiB/s) | 20260506 10:34:35.989 | INFO | Analyzing chunk frames = [675328,675840) | 20260506 10:34:35.989 | INFO | MemInfo: { | 20260506 10:34:35.989 | INFO | "gpu" : " 17 GB", | 20260506 10:34:35.989 | INFO | "huge" : "", | 20260506 10:34:35.989 | INFO | "huge_shared" : " 27 GB", | 20260506 10:34:35.989 | INFO | "malloc" : " 14 GB", | 20260506 10:34:35.989 | INFO | "pinned" : " 29 GB" | 20260506 10:34:35.989 | INFO | } | 20260506 10:34:35.989 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:34:35.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:34:35.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:35.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:35.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:35.989 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:34:35.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:35.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:35.989 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:34:35.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:35.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:35.989 | INFO | Uploaded 36GiB to GPU in 1.60596s (22.4166GiB/s) | 20260506 10:34:35.989 | INFO | Downloaded 34GiB to GPU in 1.52408s (22.3086GiB/s) | 20260506 10:34:36.086 | INFO | Starting a round of Baz IO | 20260506 10:34:39.002 | INFO | Populating RTMetrics report | 20260506 10:34:39.002 | INFO | RTMetrics report populated | 20260506 10:34:39.002 | INFO | Opening tmp RTMetrics json file | 20260506 10:34:39.017 | INFO | Serializing RTMetrics to json | 20260506 10:34:39.028 | INFO | Serialization done, swapping tmp file now... | 20260506 10:34:39.029 | INFO | RTMetrics done | 20260506 10:34:39.314 | INFO | Finished a round of Baz IO: Wrote 1.998 GiB in 3.227 seconds | 20260506 10:34:39.314 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.33% / 0.05% / 52.62% | 20260506 10:34:39.314 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.60696% / 9.93411e-05% / 4.37101e-05% | 20260506 10:34:39.314 | INFO | Baz IO overall: | 20260506 10:34:39.314 | INFO | BytesWritten : 307127233197 | 20260506 10:34:39.314 | INFO | NumEvents : 147690152946 | 20260506 10:34:41.128 | INFO | Analyzing chunk frames = [675840,676352) | 20260506 10:34:41.128 | INFO | MemInfo: { | 20260506 10:34:41.128 | INFO | "gpu" : " 17 GB", | 20260506 10:34:41.128 | INFO | "huge" : "", | 20260506 10:34:41.128 | INFO | "huge_shared" : " 27 GB", | 20260506 10:34:41.128 | INFO | "malloc" : " 14 GB", | 20260506 10:34:41.128 | INFO | "pinned" : " 29 GB" | 20260506 10:34:41.128 | INFO | } | 20260506 10:34:41.128 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:34:41.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:34:41.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:41.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:41.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:41.128 | INFO | ANALYSIS : 57.6%, 2.8, 86.0ms, 0 | 20260506 10:34:41.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:41.128 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 10:34:41.128 | INFO | PRE_HQ : 56.6%, 2.1, 60.6ms, 0 | 20260506 10:34:41.128 | INFO | BAZWRITER : 62.4%, 4.8, 152.3ms, 0 | 20260506 10:34:41.129 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.5ms, 0 | 20260506 10:34:41.129 | INFO | Uploaded 36GiB to GPU in 1.67362s (21.5103GiB/s) | 20260506 10:34:41.129 | INFO | Downloaded 38GiB to GPU in 1.73831s (21.8603GiB/s) | 20260506 10:34:46.267 | INFO | Analyzing chunk frames = [676352,676864) | 20260506 10:34:46.267 | INFO | MemInfo: { | 20260506 10:34:46.267 | INFO | "gpu" : " 17 GB", | 20260506 10:34:46.267 | INFO | "huge" : "", | 20260506 10:34:46.267 | INFO | "huge_shared" : " 27 GB", | 20260506 10:34:46.267 | INFO | "malloc" : " 14 GB", | 20260506 10:34:46.268 | INFO | "pinned" : " 29 GB" | 20260506 10:34:46.268 | INFO | } | 20260506 10:34:46.268 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:34:46.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:34:46.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:46.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:46.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:46.268 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:34:46.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:46.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:46.268 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:34:46.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:46.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:46.268 | INFO | Uploaded 36GiB to GPU in 1.6042s (22.441GiB/s) | 20260506 10:34:46.268 | INFO | Downloaded 34GiB to GPU in 1.52146s (22.3469GiB/s) | 20260506 10:34:51.407 | INFO | Analyzing chunk frames = [676864,677376) | 20260506 10:34:51.407 | INFO | MemInfo: { | 20260506 10:34:51.407 | INFO | "gpu" : " 17 GB", | 20260506 10:34:51.407 | INFO | "huge" : "", | 20260506 10:34:51.407 | INFO | "huge_shared" : " 27 GB", | 20260506 10:34:51.407 | INFO | "malloc" : " 14 GB", | 20260506 10:34:51.407 | INFO | "pinned" : " 29 GB" | 20260506 10:34:51.407 | INFO | } | 20260506 10:34:51.407 | INFO | Perf report for the last 5.13951 seconds: | 20260506 10:34:51.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:34:51.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:51.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:51.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:51.407 | INFO | ANALYSIS : 56.7%, 3.0, 88.2ms, 0 | 20260506 10:34:51.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:51.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:51.407 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:34:51.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:51.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:51.408 | INFO | Uploaded 36GiB to GPU in 1.60714s (22.4001GiB/s) | 20260506 10:34:51.408 | INFO | Downloaded 34GiB to GPU in 1.52467s (22.2999GiB/s) | 20260506 10:34:56.546 | INFO | Analyzing chunk frames = [677376,677888) | 20260506 10:34:56.546 | INFO | MemInfo: { | 20260506 10:34:56.546 | INFO | "gpu" : " 17 GB", | 20260506 10:34:56.546 | INFO | "huge" : "", | 20260506 10:34:56.546 | INFO | "huge_shared" : " 27 GB", | 20260506 10:34:56.546 | INFO | "malloc" : " 14 GB", | 20260506 10:34:56.546 | INFO | "pinned" : " 29 GB" | 20260506 10:34:56.546 | INFO | } | 20260506 10:34:56.546 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:34:56.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:34:56.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:56.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:56.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:56.547 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:34:56.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:56.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:56.547 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:34:56.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:56.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:34:56.547 | INFO | Uploaded 36GiB to GPU in 1.60659s (22.4077GiB/s) | 20260506 10:34:56.547 | INFO | Downloaded 34GiB to GPU in 1.52425s (22.306GiB/s) | 20260506 10:35:01.685 | INFO | Analyzing chunk frames = [677888,678400) | 20260506 10:35:01.685 | INFO | MemInfo: { | 20260506 10:35:01.686 | INFO | "gpu" : " 17 GB", | 20260506 10:35:01.686 | INFO | "huge" : "", | 20260506 10:35:01.686 | INFO | "huge_shared" : " 27 GB", | 20260506 10:35:01.686 | INFO | "malloc" : " 14 GB", | 20260506 10:35:01.686 | INFO | "pinned" : " 29 GB" | 20260506 10:35:01.686 | INFO | } | 20260506 10:35:01.686 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:35:01.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:35:01.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:01.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:01.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:01.686 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:35:01.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:01.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:01.686 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:35:01.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:01.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:01.686 | INFO | Uploaded 36GiB to GPU in 1.60588s (22.4176GiB/s) | 20260506 10:35:01.686 | INFO | Downloaded 34GiB to GPU in 1.52344s (22.3179GiB/s) | 20260506 10:35:06.825 | INFO | Analyzing chunk frames = [678400,678912) | 20260506 10:35:06.825 | INFO | MemInfo: { | 20260506 10:35:06.825 | INFO | "gpu" : " 17 GB", | 20260506 10:35:06.825 | INFO | "huge" : "", | 20260506 10:35:06.825 | INFO | "huge_shared" : " 27 GB", | 20260506 10:35:06.825 | INFO | "malloc" : " 14 GB", | 20260506 10:35:06.825 | INFO | "pinned" : " 29 GB" | 20260506 10:35:06.825 | INFO | } | 20260506 10:35:06.825 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:35:06.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:35:06.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:06.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:06.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:06.825 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:35:06.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:06.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:06.825 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:35:06.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:06.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:06.826 | INFO | Uploaded 36GiB to GPU in 1.60521s (22.427GiB/s) | 20260506 10:35:06.826 | INFO | Downloaded 34GiB to GPU in 1.52282s (22.327GiB/s) | 20260506 10:35:11.964 | INFO | Analyzing chunk frames = [678912,679424) | 20260506 10:35:11.964 | INFO | MemInfo: { | 20260506 10:35:11.964 | INFO | "gpu" : " 17 GB", | 20260506 10:35:11.964 | INFO | "huge" : "", | 20260506 10:35:11.964 | INFO | "huge_shared" : " 27 GB", | 20260506 10:35:11.964 | INFO | "malloc" : " 14 GB", | 20260506 10:35:11.964 | INFO | "pinned" : " 29 GB" | 20260506 10:35:11.964 | INFO | } | 20260506 10:35:11.964 | INFO | Perf report for the last 5.13922 seconds: | 20260506 10:35:11.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:35:11.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:11.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:11.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:11.964 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:35:11.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:11.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:11.965 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:35:11.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:11.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:11.965 | INFO | Uploaded 36GiB to GPU in 1.60582s (22.4184GiB/s) | 20260506 10:35:11.965 | INFO | Downloaded 34GiB to GPU in 1.5239s (22.3111GiB/s) | 20260506 10:35:17.104 | INFO | Analyzing chunk frames = [679424,679936) | 20260506 10:35:17.104 | INFO | MemInfo: { | 20260506 10:35:17.104 | INFO | "gpu" : " 17 GB", | 20260506 10:35:17.104 | INFO | "huge" : "", | 20260506 10:35:17.104 | INFO | "huge_shared" : " 27 GB", | 20260506 10:35:17.104 | INFO | "malloc" : " 14 GB", | 20260506 10:35:17.104 | INFO | "pinned" : " 29 GB" | 20260506 10:35:17.104 | INFO | } | 20260506 10:35:17.104 | INFO | Perf report for the last 5.13954 seconds: | 20260506 10:35:17.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:35:17.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:17.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:17.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:17.104 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:35:17.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:17.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:17.104 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:35:17.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:17.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:17.104 | INFO | Uploaded 36GiB to GPU in 1.60682s (22.4045GiB/s) | 20260506 10:35:17.104 | INFO | Downloaded 34GiB to GPU in 1.52539s (22.2894GiB/s) | 20260506 10:35:17.104 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:35:17.104 | INFO | Total mem usage: 16.5 GiB | 20260506 10:35:17.104 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:35:17.104 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -9.47%) | 20260506 10:35:17.104 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:35:17.104 | INFO | High Water Breakdown: | 20260506 10:35:17.104 | INFO | 59.8% : 9.02 GiB ( 89.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:35:17.104 | INFO | 40.1% : 6.05 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:35:17.104 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:35:17.104 | INFO | ------------------------------------------ | 20260506 10:35:17.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:35:17.104 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:35:17.104 | INFO | Total mem usage: 26.5 GiB | 20260506 10:35:17.104 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:35:17.104 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:35:17.104 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:35:17.104 | INFO | High Water Breakdown: | 20260506 10:35:17.104 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:35:17.104 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:35:17.104 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:35:17.104 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:35:17.104 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:35:17.104 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:35:17.104 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:35:17.104 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:35:17.104 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:35:17.104 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:35:17.104 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:35:17.104 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:35:17.104 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:35:17.104 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:35:17.104 | INFO | ------------------------------------------ | 20260506 10:35:17.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:35:17.104 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:35:17.104 | INFO | Total mem usage: 25.6 GiB | 20260506 10:35:17.104 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:35:17.104 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:35:17.104 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:35:17.104 | INFO | High Water Breakdown: | 20260506 10:35:17.104 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:35:17.104 | INFO | ------------------------------------------ | 20260506 10:35:17.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:35:17.104 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:35:17.104 | INFO | Total mem usage: 15.7 GiB | 20260506 10:35:17.104 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:35:17.104 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:35:17.104 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:35:17.105 | INFO | High Water Breakdown: | 20260506 10:35:17.105 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:35:17.105 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:35:17.105 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:35:17.105 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:35:17.105 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:35:17.105 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:35:17.105 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:35:17.105 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:35:17.105 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:35:17.105 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:35:17.105 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:35:17.105 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:35:17.105 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:35:17.105 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:35:17.105 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:35:17.105 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:35:17.105 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:35:17.105 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:35:17.105 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:35:17.105 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:35:17.105 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:35:17.105 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:35:17.105 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:35:17.105 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:35:17.105 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:35:17.105 | INFO | ------------------------------------------ | 20260506 10:35:17.105 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:35:17.105 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:35:17.105 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:35:17.105 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:35:17.105 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 10:35:17.105 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 4.91e-03 / 0.00e+00 | 20260506 10:35:17.105 | INFO | PerfReport: 10.96%, Baseline -- 3.30e+00 / -nan, 6.84e-02 / 0.00e+00 | 20260506 10:35:17.105 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260506 10:35:17.106 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.10e+00 / 0.00e+00 | 20260506 10:35:17.106 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.72e+01 / -nan, 1.08e-01 / 0.00e+00 | 20260506 10:35:17.106 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.22e+00 / -nan, 1.28e-01 / 0.00e+00 | 20260506 10:35:17.106 | INFO | PerfReport: 3.66%, Metrics -- 1.10e+00 / -nan, 1.71e-01 / 0.00e+00 | 20260506 10:35:17.106 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.25e-01 / -nan, 5.96e-02 / 0.00e+00 | 20260506 10:35:17.106 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:35:17.106 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:35:17.106 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:35:17.106 | INFO | PerfReport: Full Execution -- 8.99e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:35:17.106 | INFO | PerfReport: 18.71%, Upload -- 1.68e+01 / 0.00e+00, 5.03e-01 / 0.00e+00 | 20260506 10:35:17.106 | INFO | PerfReport: 17.96%, Download -- 1.62e+01 / 0.00e+00, 8.51e-01 / 0.00e+00 | 20260506 10:35:17.106 | INFO | PerfReport: 33.47%, Compute -- 3.01e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 10:35:17.106 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:35:17.106 | INFO | PerfReport: 0.00%, LockDownload -- 4.86e-04 / 0.00e+00, 1.99e-04 / 0.00e+00 | 20260506 10:35:17.106 | INFO | PerfReport: 29.28%, LockCompute -- 2.63e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 10:35:17.106 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.32e-02 / 0.00e+00 | 20260506 10:35:17.203 | INFO | Starting a round of Baz IO | 20260506 10:35:20.112 | INFO | Populating RTMetrics report | 20260506 10:35:20.112 | INFO | RTMetrics report populated | 20260506 10:35:20.112 | INFO | Opening tmp RTMetrics json file | 20260506 10:35:20.183 | INFO | Serializing RTMetrics to json | 20260506 10:35:20.189 | INFO | Serialization done, swapping tmp file now... | 20260506 10:35:20.189 | INFO | RTMetrics done | 20260506 10:35:20.386 | INFO | Finished a round of Baz IO: Wrote 2.001 GiB in 3.182 seconds | 20260506 10:35:20.386 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.28% / 0.05% / 52.67% | 20260506 10:35:20.386 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.58505% / 0.000107288% / 2.38419e-05% | 20260506 10:35:20.386 | INFO | Baz IO overall: | 20260506 10:35:20.386 | INFO | BytesWritten : 309275381446 | 20260506 10:35:20.386 | INFO | NumEvents : 148733407695 | 20260506 10:35:22.243 | INFO | Analyzing chunk frames = [679936,680448) | 20260506 10:35:22.243 | INFO | MemInfo: { | 20260506 10:35:22.243 | INFO | "gpu" : " 17 GB", | 20260506 10:35:22.243 | INFO | "huge" : "", | 20260506 10:35:22.243 | INFO | "huge_shared" : " 27 GB", | 20260506 10:35:22.243 | INFO | "malloc" : " 14 GB", | 20260506 10:35:22.243 | INFO | "pinned" : " 29 GB" | 20260506 10:35:22.243 | INFO | } | 20260506 10:35:22.243 | INFO | Perf report for the last 5.13905 seconds: | 20260506 10:35:22.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:35:22.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:22.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:22.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:22.243 | INFO | ANALYSIS : 57.5%, 2.8, 85.3ms, 0 | 20260506 10:35:22.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:22.243 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 10:35:22.243 | INFO | PRE_HQ : 56.6%, 2.1, 60.6ms, 0 | 20260506 10:35:22.243 | INFO | BAZWRITER : 61.5%, 4.8, 151.1ms, 0 | 20260506 10:35:22.243 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 10:35:22.243 | INFO | Uploaded 36GiB to GPU in 1.67871s (21.4451GiB/s) | 20260506 10:35:22.244 | INFO | Downloaded 38GiB to GPU in 1.74386s (21.7908GiB/s) | 20260506 10:35:27.382 | INFO | Analyzing chunk frames = [680448,680960) | 20260506 10:35:27.382 | INFO | MemInfo: { | 20260506 10:35:27.382 | INFO | "gpu" : " 17 GB", | 20260506 10:35:27.382 | INFO | "huge" : "", | 20260506 10:35:27.382 | INFO | "huge_shared" : " 27 GB", | 20260506 10:35:27.382 | INFO | "malloc" : " 14 GB", | 20260506 10:35:27.382 | INFO | "pinned" : " 29 GB" | 20260506 10:35:27.382 | INFO | } | 20260506 10:35:27.382 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:35:27.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:35:27.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:27.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:27.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:27.382 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:35:27.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:27.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:27.382 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:35:27.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:27.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:27.383 | INFO | Uploaded 36GiB to GPU in 1.60304s (22.4573GiB/s) | 20260506 10:35:27.383 | INFO | Downloaded 34GiB to GPU in 1.52006s (22.3676GiB/s) | 20260506 10:35:32.521 | INFO | Analyzing chunk frames = [680960,681472) | 20260506 10:35:32.521 | INFO | MemInfo: { | 20260506 10:35:32.521 | INFO | "gpu" : " 17 GB", | 20260506 10:35:32.521 | INFO | "huge" : "", | 20260506 10:35:32.521 | INFO | "huge_shared" : " 27 GB", | 20260506 10:35:32.522 | INFO | "malloc" : " 14 GB", | 20260506 10:35:32.522 | INFO | "pinned" : " 29 GB" | 20260506 10:35:32.522 | INFO | } | 20260506 10:35:32.522 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:35:32.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:35:32.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:32.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:32.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:32.522 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:35:32.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:32.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:32.522 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:35:32.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:32.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:32.522 | INFO | Uploaded 36GiB to GPU in 1.60819s (22.3854GiB/s) | 20260506 10:35:32.522 | INFO | Downloaded 34GiB to GPU in 1.52646s (22.2737GiB/s) | 20260506 10:35:37.661 | INFO | Analyzing chunk frames = [681472,681984) | 20260506 10:35:37.661 | INFO | MemInfo: { | 20260506 10:35:37.661 | INFO | "gpu" : " 17 GB", | 20260506 10:35:37.661 | INFO | "huge" : "", | 20260506 10:35:37.661 | INFO | "huge_shared" : " 27 GB", | 20260506 10:35:37.661 | INFO | "malloc" : " 14 GB", | 20260506 10:35:37.661 | INFO | "pinned" : " 29 GB" | 20260506 10:35:37.661 | INFO | } | 20260506 10:35:37.661 | INFO | Perf report for the last 5.13944 seconds: | 20260506 10:35:37.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:35:37.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:37.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:37.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:37.661 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:35:37.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:37.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:37.661 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:35:37.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:37.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:37.662 | INFO | Uploaded 36GiB to GPU in 1.60723s (22.3988GiB/s) | 20260506 10:35:37.662 | INFO | Downloaded 34GiB to GPU in 1.52498s (22.2953GiB/s) | 20260506 10:35:42.800 | INFO | Analyzing chunk frames = [681984,682496) | 20260506 10:35:42.800 | INFO | MemInfo: { | 20260506 10:35:42.800 | INFO | "gpu" : " 17 GB", | 20260506 10:35:42.800 | INFO | "huge" : "", | 20260506 10:35:42.800 | INFO | "huge_shared" : " 27 GB", | 20260506 10:35:42.800 | INFO | "malloc" : " 14 GB", | 20260506 10:35:42.800 | INFO | "pinned" : " 29 GB" | 20260506 10:35:42.800 | INFO | } | 20260506 10:35:42.800 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:35:42.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:35:42.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:42.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:42.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:42.800 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:35:42.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:42.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:42.801 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 10:35:42.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:42.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:42.801 | INFO | Uploaded 36GiB to GPU in 1.60593s (22.4169GiB/s) | 20260506 10:35:42.801 | INFO | Downloaded 34GiB to GPU in 1.5224s (22.3331GiB/s) | 20260506 10:35:47.939 | INFO | Analyzing chunk frames = [682496,683008) | 20260506 10:35:47.939 | INFO | MemInfo: { | 20260506 10:35:47.939 | INFO | "gpu" : " 17 GB", | 20260506 10:35:47.939 | INFO | "huge" : "", | 20260506 10:35:47.939 | INFO | "huge_shared" : " 27 GB", | 20260506 10:35:47.939 | INFO | "malloc" : " 14 GB", | 20260506 10:35:47.939 | INFO | "pinned" : " 29 GB" | 20260506 10:35:47.940 | INFO | } | 20260506 10:35:47.940 | INFO | Perf report for the last 5.13929 seconds: | 20260506 10:35:47.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:35:47.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:47.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:47.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:47.940 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:35:47.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:47.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:47.940 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:35:47.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:47.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:47.940 | INFO | Uploaded 36GiB to GPU in 1.60532s (22.4254GiB/s) | 20260506 10:35:47.940 | INFO | Downloaded 34GiB to GPU in 1.52274s (22.3282GiB/s) | 20260506 10:35:53.079 | INFO | Analyzing chunk frames = [683008,683520) | 20260506 10:35:53.079 | INFO | MemInfo: { | 20260506 10:35:53.079 | INFO | "gpu" : " 17 GB", | 20260506 10:35:53.079 | INFO | "huge" : "", | 20260506 10:35:53.079 | INFO | "huge_shared" : " 27 GB", | 20260506 10:35:53.079 | INFO | "malloc" : " 14 GB", | 20260506 10:35:53.079 | INFO | "pinned" : " 29 GB" | 20260506 10:35:53.079 | INFO | } | 20260506 10:35:53.079 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:35:53.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:35:53.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:53.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:53.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:53.079 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:35:53.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:53.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:53.079 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:35:53.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:53.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:53.079 | INFO | Uploaded 36GiB to GPU in 1.60547s (22.4233GiB/s) | 20260506 10:35:53.079 | INFO | Downloaded 34GiB to GPU in 1.52345s (22.3177GiB/s) | 20260506 10:35:58.218 | INFO | Analyzing chunk frames = [683520,684032) | 20260506 10:35:58.218 | INFO | MemInfo: { | 20260506 10:35:58.218 | INFO | "gpu" : " 17 GB", | 20260506 10:35:58.218 | INFO | "huge" : "", | 20260506 10:35:58.218 | INFO | "huge_shared" : " 27 GB", | 20260506 10:35:58.218 | INFO | "malloc" : " 14 GB", | 20260506 10:35:58.218 | INFO | "pinned" : " 29 GB" | 20260506 10:35:58.218 | INFO | } | 20260506 10:35:58.218 | INFO | Perf report for the last 5.13919 seconds: | 20260506 10:35:58.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:35:58.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:58.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:58.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:58.218 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:35:58.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:58.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:58.218 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 10:35:58.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:58.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:35:58.219 | INFO | Uploaded 36GiB to GPU in 1.60528s (22.426GiB/s) | 20260506 10:35:58.219 | INFO | Downloaded 34GiB to GPU in 1.52337s (22.3189GiB/s) | 20260506 10:35:58.316 | INFO | Starting a round of Baz IO | 20260506 10:36:01.227 | INFO | Populating RTMetrics report | 20260506 10:36:01.228 | INFO | RTMetrics report populated | 20260506 10:36:01.228 | INFO | Opening tmp RTMetrics json file | 20260506 10:36:01.260 | INFO | Serializing RTMetrics to json | 20260506 10:36:01.323 | INFO | Serialization done, swapping tmp file now... | 20260506 10:36:01.362 | INFO | RTMetrics done | 20260506 10:36:01.804 | INFO | Finished a round of Baz IO: Wrote 2 GiB in 3.488 seconds | 20260506 10:36:01.804 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.23% / 0.05% / 52.72% | 20260506 10:36:01.804 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.55679% / 7.15256e-05% / 5.16574e-05% | 20260506 10:36:01.804 | INFO | Baz IO overall: | 20260506 10:36:01.804 | INFO | BytesWritten : 311423280584 | 20260506 10:36:01.804 | INFO | NumEvents : 149775820859 | 20260506 10:36:03.357 | INFO | Analyzing chunk frames = [684032,684544) | 20260506 10:36:03.357 | INFO | MemInfo: { | 20260506 10:36:03.357 | INFO | "gpu" : " 17 GB", | 20260506 10:36:03.357 | INFO | "huge" : "", | 20260506 10:36:03.357 | INFO | "huge_shared" : " 27 GB", | 20260506 10:36:03.357 | INFO | "malloc" : " 14 GB", | 20260506 10:36:03.357 | INFO | "pinned" : " 29 GB" | 20260506 10:36:03.358 | INFO | } | 20260506 10:36:03.358 | INFO | Perf report for the last 5.1393 seconds: | 20260506 10:36:03.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:36:03.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:03.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:03.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:03.358 | INFO | ANALYSIS : 57.5%, 2.8, 85.5ms, 0 | 20260506 10:36:03.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:03.358 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260506 10:36:03.358 | INFO | PRE_HQ : 56.6%, 2.1, 61.0ms, 0 | 20260506 10:36:03.358 | INFO | BAZWRITER : 67.5%, 4.8, 164.8ms, 0 | 20260506 10:36:03.358 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 10:36:03.358 | INFO | Uploaded 36GiB to GPU in 1.66984s (21.5589GiB/s) | 20260506 10:36:03.358 | INFO | Downloaded 38GiB to GPU in 1.73431s (21.9108GiB/s) | 20260506 10:36:08.497 | INFO | Analyzing chunk frames = [684544,685056) | 20260506 10:36:08.497 | INFO | MemInfo: { | 20260506 10:36:08.497 | INFO | "gpu" : " 17 GB", | 20260506 10:36:08.497 | INFO | "huge" : "", | 20260506 10:36:08.497 | INFO | "huge_shared" : " 27 GB", | 20260506 10:36:08.497 | INFO | "malloc" : " 14 GB", | 20260506 10:36:08.497 | INFO | "pinned" : " 29 GB" | 20260506 10:36:08.497 | INFO | } | 20260506 10:36:08.497 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:36:08.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:36:08.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:08.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:08.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:08.497 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:36:08.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:08.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:08.497 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:36:08.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:08.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:08.498 | INFO | Uploaded 36GiB to GPU in 1.6032s (22.4551GiB/s) | 20260506 10:36:08.498 | INFO | Downloaded 34GiB to GPU in 1.52064s (22.359GiB/s) | 20260506 10:36:13.636 | INFO | Analyzing chunk frames = [685056,685568) | 20260506 10:36:13.636 | INFO | MemInfo: { | 20260506 10:36:13.636 | INFO | "gpu" : " 17 GB", | 20260506 10:36:13.636 | INFO | "huge" : "", | 20260506 10:36:13.636 | INFO | "huge_shared" : " 27 GB", | 20260506 10:36:13.636 | INFO | "malloc" : " 14 GB", | 20260506 10:36:13.636 | INFO | "pinned" : " 29 GB" | 20260506 10:36:13.636 | INFO | } | 20260506 10:36:13.636 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:36:13.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:36:13.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:13.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:13.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:13.636 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:36:13.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:13.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:13.636 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:36:13.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:13.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:13.637 | INFO | Uploaded 36GiB to GPU in 1.60728s (22.3981GiB/s) | 20260506 10:36:13.637 | INFO | Downloaded 34GiB to GPU in 1.52575s (22.2842GiB/s) | 20260506 10:36:18.775 | INFO | Analyzing chunk frames = [685568,686080) | 20260506 10:36:18.775 | INFO | MemInfo: { | 20260506 10:36:18.775 | INFO | "gpu" : " 17 GB", | 20260506 10:36:18.775 | INFO | "huge" : "", | 20260506 10:36:18.775 | INFO | "huge_shared" : " 27 GB", | 20260506 10:36:18.775 | INFO | "malloc" : " 14 GB", | 20260506 10:36:18.775 | INFO | "pinned" : " 29 GB" | 20260506 10:36:18.776 | INFO | } | 20260506 10:36:18.776 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:36:18.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:36:18.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:18.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:18.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:18.776 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:36:18.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:18.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:18.776 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:36:18.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:18.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:18.776 | INFO | Uploaded 36GiB to GPU in 1.60719s (22.3994GiB/s) | 20260506 10:36:18.776 | INFO | Downloaded 34GiB to GPU in 1.52449s (22.3025GiB/s) | 20260506 10:36:23.915 | INFO | Analyzing chunk frames = [686080,686592) | 20260506 10:36:23.915 | INFO | MemInfo: { | 20260506 10:36:23.915 | INFO | "gpu" : " 17 GB", | 20260506 10:36:23.915 | INFO | "huge" : "", | 20260506 10:36:23.915 | INFO | "huge_shared" : " 27 GB", | 20260506 10:36:23.915 | INFO | "malloc" : " 14 GB", | 20260506 10:36:23.915 | INFO | "pinned" : " 29 GB" | 20260506 10:36:23.915 | INFO | } | 20260506 10:36:23.915 | INFO | Perf report for the last 5.13917 seconds: | 20260506 10:36:23.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:36:23.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:23.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:23.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:23.915 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:36:23.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:23.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:23.915 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:36:23.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:23.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:23.915 | INFO | Uploaded 36GiB to GPU in 1.60543s (22.424GiB/s) | 20260506 10:36:23.915 | INFO | Downloaded 34GiB to GPU in 1.5221s (22.3376GiB/s) | 20260506 10:36:29.054 | INFO | Analyzing chunk frames = [686592,687104) | 20260506 10:36:29.054 | INFO | MemInfo: { | 20260506 10:36:29.054 | INFO | "gpu" : " 17 GB", | 20260506 10:36:29.054 | INFO | "huge" : "", | 20260506 10:36:29.054 | INFO | "huge_shared" : " 27 GB", | 20260506 10:36:29.054 | INFO | "malloc" : " 14 GB", | 20260506 10:36:29.054 | INFO | "pinned" : " 29 GB" | 20260506 10:36:29.055 | INFO | } | 20260506 10:36:29.055 | INFO | Perf report for the last 5.13971 seconds: | 20260506 10:36:29.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:36:29.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:29.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:29.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:29.055 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:36:29.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:29.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:29.055 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:36:29.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:29.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:29.055 | INFO | Uploaded 36GiB to GPU in 1.60546s (22.4235GiB/s) | 20260506 10:36:29.055 | INFO | Downloaded 34GiB to GPU in 1.52272s (22.3285GiB/s) | 20260506 10:36:34.193 | INFO | Analyzing chunk frames = [687104,687616) | 20260506 10:36:34.193 | INFO | MemInfo: { | 20260506 10:36:34.193 | INFO | "gpu" : " 17 GB", | 20260506 10:36:34.193 | INFO | "huge" : "", | 20260506 10:36:34.193 | INFO | "huge_shared" : " 27 GB", | 20260506 10:36:34.193 | INFO | "malloc" : " 14 GB", | 20260506 10:36:34.193 | INFO | "pinned" : " 29 GB" | 20260506 10:36:34.193 | INFO | } | 20260506 10:36:34.194 | INFO | Perf report for the last 5.13899 seconds: | 20260506 10:36:34.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:36:34.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:34.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:34.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:34.194 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:36:34.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:34.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:34.194 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:36:34.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:34.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:34.194 | INFO | Uploaded 36GiB to GPU in 1.60527s (22.4261GiB/s) | 20260506 10:36:34.194 | INFO | Downloaded 34GiB to GPU in 1.52368s (22.3143GiB/s) | 20260506 10:36:39.333 | INFO | Analyzing chunk frames = [687616,688128) | 20260506 10:36:39.333 | INFO | MemInfo: { | 20260506 10:36:39.333 | INFO | "gpu" : " 17 GB", | 20260506 10:36:39.333 | INFO | "huge" : "", | 20260506 10:36:39.333 | INFO | "huge_shared" : " 27 GB", | 20260506 10:36:39.333 | INFO | "malloc" : " 14 GB", | 20260506 10:36:39.333 | INFO | "pinned" : " 29 GB" | 20260506 10:36:39.333 | INFO | } | 20260506 10:36:39.333 | INFO | Perf report for the last 5.13918 seconds: | 20260506 10:36:39.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:36:39.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:39.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:39.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:39.333 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:36:39.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:39.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:39.333 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:36:39.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:39.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:39.334 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:36:39.334 | INFO | Total mem usage: 16.5 GiB | 20260506 10:36:39.334 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:36:39.334 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -4.99%) | 20260506 10:36:39.334 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:36:39.334 | INFO | High Water Breakdown: | 20260506 10:36:39.334 | INFO | 60.5% : 9.52 GiB ( 94.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:36:39.334 | INFO | 39.4% : 6.19 GiB ( 92.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:36:39.334 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:36:39.334 | INFO | ------------------------------------------ | 20260506 10:36:39.334 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:36:39.334 | INFO | Uploaded 36GiB to GPU in 1.60587s (22.4178GiB/s) | 20260506 10:36:39.334 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:36:39.334 | INFO | Total mem usage: 26.5 GiB | 20260506 10:36:39.334 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:36:39.334 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:36:39.334 | INFO | Downloaded 34GiB to GPU in 1.52469s (22.2996GiB/s) | 20260506 10:36:39.334 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:36:39.334 | INFO | High Water Breakdown: | 20260506 10:36:39.334 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:36:39.334 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:36:39.334 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:36:39.334 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:36:39.334 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:36:39.334 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:36:39.334 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:36:39.334 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:36:39.334 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:36:39.334 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:36:39.334 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:36:39.334 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:36:39.334 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:36:39.334 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:36:39.334 | INFO | ------------------------------------------ | 20260506 10:36:39.334 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:36:39.335 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:36:39.335 | INFO | Total mem usage: 25.6 GiB | 20260506 10:36:39.335 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:36:39.335 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:36:39.335 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:36:39.335 | INFO | High Water Breakdown: | 20260506 10:36:39.335 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:36:39.335 | INFO | ------------------------------------------ | 20260506 10:36:39.335 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:36:39.335 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:36:39.335 | INFO | Total mem usage: 15.7 GiB | 20260506 10:36:39.335 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:36:39.335 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:36:39.335 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:36:39.335 | INFO | High Water Breakdown: | 20260506 10:36:39.335 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:36:39.335 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:36:39.335 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:36:39.335 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:36:39.335 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:36:39.335 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:36:39.335 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:36:39.335 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:36:39.335 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:36:39.335 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:36:39.335 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:36:39.335 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:36:39.335 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:36:39.335 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:36:39.335 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:36:39.335 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:36:39.335 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:36:39.335 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:36:39.335 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:36:39.335 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:36:39.335 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:36:39.335 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:36:39.335 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:36:39.335 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:36:39.335 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:36:39.335 | INFO | ------------------------------------------ | 20260506 10:36:39.335 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:36:39.335 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:36:39.335 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:36:39.335 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:36:39.335 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 10:36:39.335 | INFO | PerfReport: 0.29%, AggregateStats -- 8.68e-02 / -nan, 5.15e-03 / 0.00e+00 | 20260506 10:36:39.335 | INFO | PerfReport: 10.96%, Baseline -- 3.30e+00 / -nan, 1.84e-02 / 0.00e+00 | 20260506 10:36:39.335 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.28e-02 / 0.00e+00 | 20260506 10:36:39.335 | INFO | PerfReport: 16.06%, DME -- 3.87e+01 / -nan, 3.03e+00 / 0.00e+00 | 20260506 10:36:39.335 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.72e+01 / -nan, 4.26e-02 / 0.00e+00 | 20260506 10:36:39.335 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.22e+00 / -nan, 1.28e-01 / 0.00e+00 | 20260506 10:36:39.335 | INFO | PerfReport: 3.66%, Metrics -- 1.10e+00 / -nan, 1.73e-01 / 0.00e+00 | 20260506 10:36:39.335 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.25e-01 / -nan, 5.95e-02 / 0.00e+00 | 20260506 10:36:39.335 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:36:39.335 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:36:39.335 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:36:39.335 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 10:36:39.335 | INFO | PerfReport: 18.72%, Upload -- 1.68e+01 / 0.00e+00, 4.91e-01 / 0.00e+00 | 20260506 10:36:39.335 | INFO | PerfReport: 17.98%, Download -- 1.62e+01 / 0.00e+00, 8.44e-01 / 0.00e+00 | 20260506 10:36:39.335 | INFO | PerfReport: 33.49%, Compute -- 3.01e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 10:36:39.335 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:36:39.335 | INFO | PerfReport: 0.00%, LockDownload -- 4.85e-04 / 0.00e+00, 1.63e-04 / 0.00e+00 | 20260506 10:36:39.335 | INFO | PerfReport: 29.23%, LockCompute -- 2.63e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 10:36:39.335 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.37e-02 / 0.00e+00 | 20260506 10:36:39.432 | INFO | Starting a round of Baz IO | 20260506 10:36:42.342 | INFO | Populating RTMetrics report | 20260506 10:36:42.343 | INFO | RTMetrics report populated | 20260506 10:36:42.343 | INFO | Opening tmp RTMetrics json file | 20260506 10:36:42.346 | INFO | Serializing RTMetrics to json | 20260506 10:36:42.354 | INFO | Serialization done, swapping tmp file now... | 20260506 10:36:42.369 | INFO | RTMetrics done | 20260506 10:36:42.867 | INFO | Finished a round of Baz IO: Wrote 2.003 GiB in 3.434 seconds | 20260506 10:36:42.867 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.18% / 0.05% / 52.77% | 20260506 10:36:42.867 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.55382% / 7.54992e-05% / 3.57628e-05% | 20260506 10:36:42.867 | INFO | Baz IO overall: | 20260506 10:36:42.867 | INFO | BytesWritten : 313574151812 | 20260506 10:36:42.867 | INFO | NumEvents : 150820863437 | 20260506 10:36:44.472 | INFO | Analyzing chunk frames = [688128,688640) | 20260506 10:36:44.472 | INFO | MemInfo: { | 20260506 10:36:44.472 | INFO | "gpu" : " 17 GB", | 20260506 10:36:44.472 | INFO | "huge" : "", | 20260506 10:36:44.472 | INFO | "huge_shared" : " 27 GB", | 20260506 10:36:44.472 | INFO | "malloc" : " 14 GB", | 20260506 10:36:44.472 | INFO | "pinned" : " 29 GB" | 20260506 10:36:44.472 | INFO | } | 20260506 10:36:44.472 | INFO | Perf report for the last 5.13959 seconds: | 20260506 10:36:44.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:36:44.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:44.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:44.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:44.473 | INFO | ANALYSIS : 57.5%, 2.8, 85.6ms, 0 | 20260506 10:36:44.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:44.473 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 10:36:44.473 | INFO | PRE_HQ : 56.6%, 2.0, 60.2ms, 0 | 20260506 10:36:44.473 | INFO | BAZWRITER : 66.5%, 4.8, 163.9ms, 0 | 20260506 10:36:44.473 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 10:36:44.473 | INFO | Uploaded 36GiB to GPU in 1.68219s (21.4006GiB/s) | 20260506 10:36:44.473 | INFO | Downloaded 38GiB to GPU in 1.7471s (21.7503GiB/s) | 20260506 10:36:49.611 | INFO | Analyzing chunk frames = [688640,689152) | 20260506 10:36:49.611 | INFO | MemInfo: { | 20260506 10:36:49.611 | INFO | "gpu" : " 17 GB", | 20260506 10:36:49.611 | INFO | "huge" : "", | 20260506 10:36:49.611 | INFO | "huge_shared" : " 27 GB", | 20260506 10:36:49.611 | INFO | "malloc" : " 14 GB", | 20260506 10:36:49.612 | INFO | "pinned" : " 29 GB" | 20260506 10:36:49.612 | INFO | } | 20260506 10:36:49.612 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:36:49.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:36:49.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:49.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:49.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:49.612 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:36:49.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:49.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:49.612 | INFO | PRE_HQ : 29.5%, 1.0, 15.4ms, 0 | 20260506 10:36:49.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:49.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:49.612 | INFO | Uploaded 36GiB to GPU in 1.60358s (22.4497GiB/s) | 20260506 10:36:49.612 | INFO | Downloaded 34GiB to GPU in 1.52041s (22.3624GiB/s) | 20260506 10:36:54.751 | INFO | Analyzing chunk frames = [689152,689664) | 20260506 10:36:54.751 | INFO | MemInfo: { | 20260506 10:36:54.751 | INFO | "gpu" : " 17 GB", | 20260506 10:36:54.751 | INFO | "huge" : "", | 20260506 10:36:54.751 | INFO | "huge_shared" : " 27 GB", | 20260506 10:36:54.751 | INFO | "malloc" : " 14 GB", | 20260506 10:36:54.751 | INFO | "pinned" : " 29 GB" | 20260506 10:36:54.751 | INFO | } | 20260506 10:36:54.751 | INFO | Perf report for the last 5.13919 seconds: | 20260506 10:36:54.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:36:54.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:54.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:54.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:54.751 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:36:54.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:54.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:54.751 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:36:54.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:54.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:54.751 | INFO | Uploaded 36GiB to GPU in 1.60714s (22.4GiB/s) | 20260506 10:36:54.752 | INFO | Downloaded 34GiB to GPU in 1.52524s (22.2916GiB/s) | 20260506 10:36:59.890 | INFO | Analyzing chunk frames = [689664,690176) | 20260506 10:36:59.890 | INFO | MemInfo: { | 20260506 10:36:59.890 | INFO | "gpu" : " 17 GB", | 20260506 10:36:59.890 | INFO | "huge" : "", | 20260506 10:36:59.890 | INFO | "huge_shared" : " 27 GB", | 20260506 10:36:59.890 | INFO | "malloc" : " 14 GB", | 20260506 10:36:59.890 | INFO | "pinned" : " 29 GB" | 20260506 10:36:59.890 | INFO | } | 20260506 10:36:59.890 | INFO | Perf report for the last 5.13942 seconds: | 20260506 10:36:59.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:36:59.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:59.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:59.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:59.890 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 10:36:59.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:59.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:59.890 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:36:59.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:59.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:36:59.891 | INFO | Uploaded 36GiB to GPU in 1.60702s (22.4018GiB/s) | 20260506 10:36:59.891 | INFO | Downloaded 34GiB to GPU in 1.52467s (22.2998GiB/s) | 20260506 10:37:05.029 | INFO | Analyzing chunk frames = [690176,690688) | 20260506 10:37:05.029 | INFO | MemInfo: { | 20260506 10:37:05.029 | INFO | "gpu" : " 17 GB", | 20260506 10:37:05.029 | INFO | "huge" : "", | 20260506 10:37:05.029 | INFO | "huge_shared" : " 27 GB", | 20260506 10:37:05.029 | INFO | "malloc" : " 14 GB", | 20260506 10:37:05.029 | INFO | "pinned" : " 29 GB" | 20260506 10:37:05.029 | INFO | } | 20260506 10:37:05.030 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:37:05.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:37:05.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:05.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:05.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:05.030 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260506 10:37:05.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:05.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:05.030 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:37:05.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:05.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:05.030 | INFO | Uploaded 36GiB to GPU in 1.60533s (22.4253GiB/s) | 20260506 10:37:05.030 | INFO | Downloaded 34GiB to GPU in 1.52218s (22.3364GiB/s) | 20260506 10:37:10.169 | INFO | Analyzing chunk frames = [690688,691200) | 20260506 10:37:10.169 | INFO | MemInfo: { | 20260506 10:37:10.169 | INFO | "gpu" : " 17 GB", | 20260506 10:37:10.169 | INFO | "huge" : "", | 20260506 10:37:10.169 | INFO | "huge_shared" : " 27 GB", | 20260506 10:37:10.169 | INFO | "malloc" : " 14 GB", | 20260506 10:37:10.169 | INFO | "pinned" : " 29 GB" | 20260506 10:37:10.169 | INFO | } | 20260506 10:37:10.169 | INFO | Perf report for the last 5.13942 seconds: | 20260506 10:37:10.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:37:10.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:10.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:10.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:10.169 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 10:37:10.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:10.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:10.169 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:37:10.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:10.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:10.169 | INFO | Uploaded 36GiB to GPU in 1.60561s (22.4214GiB/s) | 20260506 10:37:10.169 | INFO | Downloaded 34GiB to GPU in 1.5222s (22.3361GiB/s) | 20260506 10:37:15.308 | INFO | Analyzing chunk frames = [691200,691712) | 20260506 10:37:15.308 | INFO | MemInfo: { | 20260506 10:37:15.308 | INFO | "gpu" : " 17 GB", | 20260506 10:37:15.308 | INFO | "huge" : "", | 20260506 10:37:15.308 | INFO | "huge_shared" : " 27 GB", | 20260506 10:37:15.308 | INFO | "malloc" : " 14 GB", | 20260506 10:37:15.308 | INFO | "pinned" : " 29 GB" | 20260506 10:37:15.308 | INFO | } | 20260506 10:37:15.308 | INFO | Perf report for the last 5.13925 seconds: | 20260506 10:37:15.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:37:15.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:15.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:15.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:15.308 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:37:15.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:15.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:15.308 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:37:15.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:15.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:15.309 | INFO | Uploaded 36GiB to GPU in 1.60579s (22.4188GiB/s) | 20260506 10:37:15.309 | INFO | Downloaded 34GiB to GPU in 1.524s (22.3096GiB/s) | 20260506 10:37:20.448 | INFO | Analyzing chunk frames = [691712,692224) | 20260506 10:37:20.448 | INFO | MemInfo: { | 20260506 10:37:20.448 | INFO | "gpu" : " 17 GB", | 20260506 10:37:20.448 | INFO | "huge" : "", | 20260506 10:37:20.448 | INFO | "huge_shared" : " 27 GB", | 20260506 10:37:20.448 | INFO | "malloc" : " 14 GB", | 20260506 10:37:20.448 | INFO | "pinned" : " 29 GB" | 20260506 10:37:20.448 | INFO | } | 20260506 10:37:20.448 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:37:20.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:37:20.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:20.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:20.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:20.448 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:37:20.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:20.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:20.448 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 10:37:20.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:20.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:20.448 | INFO | Uploaded 36GiB to GPU in 1.60711s (22.4005GiB/s) | 20260506 10:37:20.448 | INFO | Downloaded 34GiB to GPU in 1.52571s (22.2847GiB/s) | 20260506 10:37:20.545 | INFO | Starting a round of Baz IO | 20260506 10:37:23.465 | INFO | Populating RTMetrics report | 20260506 10:37:23.465 | INFO | RTMetrics report populated | 20260506 10:37:23.465 | INFO | Opening tmp RTMetrics json file | 20260506 10:37:23.466 | INFO | Serializing RTMetrics to json | 20260506 10:37:23.502 | INFO | Serialization done, swapping tmp file now... | 20260506 10:37:23.510 | INFO | RTMetrics done | 20260506 10:37:24.046 | INFO | Finished a round of Baz IO: Wrote 2.002 GiB in 3.5 seconds | 20260506 10:37:24.046 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.13% / 0.05% / 52.82% | 20260506 10:37:24.046 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.52289% / 7.15256e-05% / 4.37101e-05% | 20260506 10:37:24.046 | INFO | Baz IO overall: | 20260506 10:37:24.046 | INFO | BytesWritten : 315723826882 | 20260506 10:37:24.046 | INFO | NumEvents : 151864787502 | 20260506 10:37:25.587 | INFO | Analyzing chunk frames = [692224,692736) | 20260506 10:37:25.587 | INFO | MemInfo: { | 20260506 10:37:25.587 | INFO | "gpu" : " 17 GB", | 20260506 10:37:25.587 | INFO | "huge" : "", | 20260506 10:37:25.587 | INFO | "huge_shared" : " 27 GB", | 20260506 10:37:25.587 | INFO | "malloc" : " 14 GB", | 20260506 10:37:25.587 | INFO | "pinned" : " 29 GB" | 20260506 10:37:25.587 | INFO | } | 20260506 10:37:25.587 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:37:25.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:37:25.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:25.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:25.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:25.587 | INFO | ANALYSIS : 57.7%, 2.8, 85.8ms, 0 | 20260506 10:37:25.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:25.587 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260506 10:37:25.587 | INFO | PRE_HQ : 56.7%, 2.1, 61.1ms, 0 | 20260506 10:37:25.587 | INFO | BAZWRITER : 67.7%, 4.7, 164.8ms, 0 | 20260506 10:37:25.587 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 10:37:25.587 | INFO | Uploaded 36GiB to GPU in 1.66506s (21.6209GiB/s) | 20260506 10:37:25.587 | INFO | Downloaded 38GiB to GPU in 1.73008s (21.9642GiB/s) | 20260506 10:37:30.726 | INFO | Analyzing chunk frames = [692736,693248) | 20260506 10:37:30.726 | INFO | MemInfo: { | 20260506 10:37:30.726 | INFO | "gpu" : " 17 GB", | 20260506 10:37:30.726 | INFO | "huge" : "", | 20260506 10:37:30.726 | INFO | "huge_shared" : " 27 GB", | 20260506 10:37:30.726 | INFO | "malloc" : " 14 GB", | 20260506 10:37:30.726 | INFO | "pinned" : " 29 GB" | 20260506 10:37:30.726 | INFO | } | 20260506 10:37:30.726 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:37:30.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:37:30.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:30.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:30.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:30.727 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 10:37:30.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:30.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:30.727 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:37:30.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:30.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:30.727 | INFO | Uploaded 36GiB to GPU in 1.60348s (22.4512GiB/s) | 20260506 10:37:30.727 | INFO | Downloaded 34GiB to GPU in 1.52064s (22.3589GiB/s) | 20260506 10:37:35.866 | INFO | Analyzing chunk frames = [693248,693760) | 20260506 10:37:35.866 | INFO | MemInfo: { | 20260506 10:37:35.866 | INFO | "gpu" : " 17 GB", | 20260506 10:37:35.866 | INFO | "huge" : "", | 20260506 10:37:35.866 | INFO | "huge_shared" : " 27 GB", | 20260506 10:37:35.866 | INFO | "malloc" : " 14 GB", | 20260506 10:37:35.866 | INFO | "pinned" : " 29 GB" | 20260506 10:37:35.866 | INFO | } | 20260506 10:37:35.866 | INFO | Perf report for the last 5.13953 seconds: | 20260506 10:37:35.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:37:35.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:35.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:35.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:35.866 | INFO | ANALYSIS : 56.7%, 3.0, 88.2ms, 0 | 20260506 10:37:35.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:35.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:35.866 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:37:35.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:35.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:35.866 | INFO | Uploaded 36GiB to GPU in 1.6071s (22.4006GiB/s) | 20260506 10:37:35.866 | INFO | Downloaded 34GiB to GPU in 1.52477s (22.2985GiB/s) | 20260506 10:37:41.005 | INFO | Analyzing chunk frames = [693760,694272) | 20260506 10:37:41.005 | INFO | MemInfo: { | 20260506 10:37:41.005 | INFO | "gpu" : " 17 GB", | 20260506 10:37:41.005 | INFO | "huge" : "", | 20260506 10:37:41.005 | INFO | "huge_shared" : " 27 GB", | 20260506 10:37:41.005 | INFO | "malloc" : " 14 GB", | 20260506 10:37:41.005 | INFO | "pinned" : " 29 GB" | 20260506 10:37:41.005 | INFO | } | 20260506 10:37:41.005 | INFO | Perf report for the last 5.1392 seconds: | 20260506 10:37:41.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:37:41.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:41.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:41.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:41.005 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 10:37:41.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:41.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:41.005 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:37:41.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:41.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:41.006 | INFO | Uploaded 36GiB to GPU in 1.60727s (22.3982GiB/s) | 20260506 10:37:41.006 | INFO | Downloaded 34GiB to GPU in 1.52514s (22.293GiB/s) | 20260506 10:37:46.144 | INFO | Analyzing chunk frames = [694272,694784) | 20260506 10:37:46.144 | INFO | MemInfo: { | 20260506 10:37:46.144 | INFO | "gpu" : " 17 GB", | 20260506 10:37:46.144 | INFO | "huge" : "", | 20260506 10:37:46.144 | INFO | "huge_shared" : " 27 GB", | 20260506 10:37:46.144 | INFO | "malloc" : " 14 GB", | 20260506 10:37:46.144 | INFO | "pinned" : " 29 GB" | 20260506 10:37:46.144 | INFO | } | 20260506 10:37:46.144 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:37:46.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:37:46.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:46.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:46.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:46.145 | INFO | ANALYSIS : 56.3%, 3.0, 87.9ms, 0 | 20260506 10:37:46.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:46.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:46.145 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:37:46.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:46.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:46.145 | INFO | Uploaded 36GiB to GPU in 1.60532s (22.4254GiB/s) | 20260506 10:37:46.145 | INFO | Downloaded 34GiB to GPU in 1.52244s (22.3326GiB/s) | 20260506 10:37:51.283 | INFO | Analyzing chunk frames = [694784,695296) | 20260506 10:37:51.284 | INFO | MemInfo: { | 20260506 10:37:51.284 | INFO | "gpu" : " 17 GB", | 20260506 10:37:51.284 | INFO | "huge" : "", | 20260506 10:37:51.284 | INFO | "huge_shared" : " 27 GB", | 20260506 10:37:51.284 | INFO | "malloc" : " 14 GB", | 20260506 10:37:51.284 | INFO | "pinned" : " 29 GB" | 20260506 10:37:51.284 | INFO | } | 20260506 10:37:51.284 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:37:51.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:37:51.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:51.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:51.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:51.284 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 10:37:51.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:51.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:51.284 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 10:37:51.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:51.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:51.284 | INFO | Uploaded 36GiB to GPU in 1.60541s (22.4242GiB/s) | 20260506 10:37:51.284 | INFO | Downloaded 34GiB to GPU in 1.52332s (22.3197GiB/s) | 20260506 10:37:56.423 | INFO | Analyzing chunk frames = [695296,695808) | 20260506 10:37:56.423 | INFO | MemInfo: { | 20260506 10:37:56.423 | INFO | "gpu" : " 17 GB", | 20260506 10:37:56.423 | INFO | "huge" : "", | 20260506 10:37:56.423 | INFO | "huge_shared" : " 27 GB", | 20260506 10:37:56.423 | INFO | "malloc" : " 14 GB", | 20260506 10:37:56.423 | INFO | "pinned" : " 29 GB" | 20260506 10:37:56.423 | INFO | } | 20260506 10:37:56.423 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:37:56.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:37:56.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:56.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:56.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:56.423 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:37:56.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:56.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:56.423 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:37:56.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:56.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:37:56.423 | INFO | Uploaded 36GiB to GPU in 1.60568s (22.4204GiB/s) | 20260506 10:37:56.423 | INFO | Downloaded 34GiB to GPU in 1.52374s (22.3136GiB/s) | 20260506 10:38:01.562 | INFO | Analyzing chunk frames = [695808,696320) | 20260506 10:38:01.562 | INFO | MemInfo: { | 20260506 10:38:01.562 | INFO | "gpu" : " 17 GB", | 20260506 10:38:01.562 | INFO | "huge" : "", | 20260506 10:38:01.562 | INFO | "huge_shared" : " 27 GB", | 20260506 10:38:01.562 | INFO | "malloc" : " 14 GB", | 20260506 10:38:01.562 | INFO | "pinned" : " 29 GB" | 20260506 10:38:01.562 | INFO | } | 20260506 10:38:01.562 | INFO | Perf report for the last 5.13945 seconds: | 20260506 10:38:01.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:38:01.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:01.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:01.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:01.563 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:38:01.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:01.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:01.563 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:38:01.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:01.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:01.563 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:38:01.563 | INFO | Total mem usage: 16.5 GiB | 20260506 10:38:01.563 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:38:01.563 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -2.84%) | 20260506 10:38:01.563 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:38:01.563 | INFO | High Water Breakdown: | 20260506 10:38:01.563 | INFO | 60.9% : 9.77 GiB ( 96.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:38:01.563 | INFO | 39% : 6.27 GiB ( 93.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:38:01.563 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:38:01.563 | INFO | Uploaded 36GiB to GPU in 1.60714s (22.4001GiB/s) | 20260506 10:38:01.563 | INFO | Downloaded 34GiB to GPU in 1.52549s (22.2879GiB/s) | 20260506 10:38:01.563 | INFO | ------------------------------------------ | 20260506 10:38:01.563 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:38:01.563 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:38:01.563 | INFO | Total mem usage: 26.5 GiB | 20260506 10:38:01.563 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:38:01.563 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:38:01.563 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:38:01.563 | INFO | High Water Breakdown: | 20260506 10:38:01.563 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:38:01.563 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:38:01.563 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:38:01.563 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:38:01.563 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:38:01.563 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:38:01.563 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:38:01.564 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:38:01.564 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:38:01.564 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:38:01.564 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:38:01.564 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:38:01.564 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:38:01.564 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:38:01.564 | INFO | ------------------------------------------ | 20260506 10:38:01.564 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:38:01.564 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:38:01.564 | INFO | Total mem usage: 25.6 GiB | 20260506 10:38:01.564 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:38:01.564 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:38:01.564 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:38:01.564 | INFO | High Water Breakdown: | 20260506 10:38:01.564 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:38:01.564 | INFO | ------------------------------------------ | 20260506 10:38:01.564 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:38:01.564 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:38:01.564 | INFO | Total mem usage: 15.7 GiB | 20260506 10:38:01.564 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:38:01.564 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:38:01.564 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:38:01.564 | INFO | High Water Breakdown: | 20260506 10:38:01.564 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:38:01.564 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:38:01.564 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:38:01.564 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:38:01.564 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:38:01.564 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:38:01.564 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:38:01.564 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:38:01.564 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:38:01.564 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:38:01.564 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:38:01.564 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:38:01.564 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:38:01.564 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:38:01.564 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:38:01.564 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:38:01.564 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:38:01.564 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:38:01.564 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:38:01.564 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:38:01.564 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:38:01.564 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:38:01.564 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:38:01.564 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:38:01.564 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:38:01.564 | INFO | ------------------------------------------ | 20260506 10:38:01.564 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:38:01.564 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:38:01.564 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:38:01.564 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:38:01.564 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 10:38:01.564 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 3.98e-03 / 0.00e+00 | 20260506 10:38:01.564 | INFO | PerfReport: 10.96%, Baseline -- 3.30e+00 / -nan, 1.81e-02 / 0.00e+00 | 20260506 10:38:01.564 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260506 10:38:01.564 | INFO | PerfReport: 16.06%, DME -- 3.86e+01 / -nan, 3.09e+00 / 0.00e+00 | 20260506 10:38:01.564 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.72e+01 / -nan, 4.26e-02 / 0.00e+00 | 20260506 10:38:01.564 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.22e+00 / -nan, 1.28e-01 / 0.00e+00 | 20260506 10:38:01.564 | INFO | PerfReport: 3.66%, Metrics -- 1.10e+00 / -nan, 1.70e-01 / 0.00e+00 | 20260506 10:38:01.564 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.25e-01 / -nan, 5.94e-02 / 0.00e+00 | 20260506 10:38:01.564 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:38:01.564 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:38:01.564 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:38:01.564 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.85e+01 / 0.00e+00 | 20260506 10:38:01.564 | INFO | PerfReport: 18.72%, Upload -- 1.68e+01 / 0.00e+00, 4.93e-01 / 0.00e+00 | 20260506 10:38:01.564 | INFO | PerfReport: 17.98%, Download -- 1.62e+01 / 0.00e+00, 8.38e-01 / 0.00e+00 | 20260506 10:38:01.564 | INFO | PerfReport: 33.48%, Compute -- 3.01e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 10:38:01.564 | INFO | PerfReport: 0.57%, LockUpload -- 5.10e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 10:38:01.564 | INFO | PerfReport: 0.00%, LockDownload -- 4.92e-04 / 0.00e+00, 1.66e-04 / 0.00e+00 | 20260506 10:38:01.564 | INFO | PerfReport: 29.24%, LockCompute -- 2.63e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 10:38:01.565 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 3.44e-02 / 0.00e+00 | 20260506 10:38:01.660 | INFO | Starting a round of Baz IO | 20260506 10:38:04.570 | INFO | Populating RTMetrics report | 20260506 10:38:04.570 | INFO | RTMetrics report populated | 20260506 10:38:04.570 | INFO | Opening tmp RTMetrics json file | 20260506 10:38:04.572 | INFO | Serializing RTMetrics to json | 20260506 10:38:04.574 | INFO | Serialization done, swapping tmp file now... | 20260506 10:38:04.574 | INFO | RTMetrics done | 20260506 10:38:04.883 | INFO | Finished a round of Baz IO: Wrote 2.004 GiB in 3.223 seconds | 20260506 10:38:04.883 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.08% / 0.05% / 52.87% | 20260506 10:38:04.883 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.50305% / 0.000127157% / 4.37101e-05% | 20260506 10:38:04.883 | INFO | Baz IO overall: | 20260506 10:38:04.883 | INFO | BytesWritten : 317875295386 | 20260506 10:38:04.883 | INFO | NumEvents : 152909537303 | 20260506 10:38:06.702 | INFO | Analyzing chunk frames = [696320,696832) | 20260506 10:38:06.702 | INFO | MemInfo: { | 20260506 10:38:06.702 | INFO | "gpu" : " 17 GB", | 20260506 10:38:06.702 | INFO | "huge" : "", | 20260506 10:38:06.702 | INFO | "huge_shared" : " 27 GB", | 20260506 10:38:06.702 | INFO | "malloc" : " 14 GB", | 20260506 10:38:06.702 | INFO | "pinned" : " 29 GB" | 20260506 10:38:06.702 | INFO | } | 20260506 10:38:06.702 | INFO | Perf report for the last 5.13913 seconds: | 20260506 10:38:06.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:38:06.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:06.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:06.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:06.702 | INFO | ANALYSIS : 57.5%, 2.8, 85.1ms, 0 | 20260506 10:38:06.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:06.702 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 10:38:06.702 | INFO | PRE_HQ : 56.6%, 2.1, 60.6ms, 0 | 20260506 10:38:06.702 | INFO | BAZWRITER : 62.3%, 4.7, 150.9ms, 0 | 20260506 10:38:06.702 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 10:38:06.702 | INFO | Uploaded 36GiB to GPU in 1.67683s (21.4691GiB/s) | 20260506 10:38:06.702 | INFO | Downloaded 38GiB to GPU in 1.74123s (21.8236GiB/s) | 20260506 10:38:11.841 | INFO | Analyzing chunk frames = [696832,697344) | 20260506 10:38:11.841 | INFO | MemInfo: { | 20260506 10:38:11.841 | INFO | "gpu" : " 17 GB", | 20260506 10:38:11.841 | INFO | "huge" : "", | 20260506 10:38:11.841 | INFO | "huge_shared" : " 27 GB", | 20260506 10:38:11.841 | INFO | "malloc" : " 14 GB", | 20260506 10:38:11.841 | INFO | "pinned" : " 29 GB" | 20260506 10:38:11.841 | INFO | } | 20260506 10:38:11.841 | INFO | Perf report for the last 5.13946 seconds: | 20260506 10:38:11.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:38:11.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:11.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:11.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:11.841 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 10:38:11.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:11.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:11.841 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:38:11.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:11.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:11.842 | INFO | Uploaded 36GiB to GPU in 1.60303s (22.4574GiB/s) | 20260506 10:38:11.842 | INFO | Downloaded 34GiB to GPU in 1.52056s (22.3602GiB/s) | 20260506 10:38:16.980 | INFO | Analyzing chunk frames = [697344,697856) | 20260506 10:38:16.980 | INFO | MemInfo: { | 20260506 10:38:16.980 | INFO | "gpu" : " 17 GB", | 20260506 10:38:16.980 | INFO | "huge" : "", | 20260506 10:38:16.980 | INFO | "huge_shared" : " 27 GB", | 20260506 10:38:16.980 | INFO | "malloc" : " 14 GB", | 20260506 10:38:16.980 | INFO | "pinned" : " 29 GB" | 20260506 10:38:16.980 | INFO | } | 20260506 10:38:16.980 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:38:16.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:38:16.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:16.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:16.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:16.980 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:38:16.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:16.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:16.981 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:38:16.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:16.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:16.981 | INFO | Uploaded 36GiB to GPU in 1.60746s (22.3956GiB/s) | 20260506 10:38:16.981 | INFO | Downloaded 34GiB to GPU in 1.52514s (22.293GiB/s) | 20260506 10:38:22.120 | INFO | Analyzing chunk frames = [697856,698368) | 20260506 10:38:22.120 | INFO | MemInfo: { | 20260506 10:38:22.120 | INFO | "gpu" : " 17 GB", | 20260506 10:38:22.120 | INFO | "huge" : "", | 20260506 10:38:22.120 | INFO | "huge_shared" : " 27 GB", | 20260506 10:38:22.120 | INFO | "malloc" : " 14 GB", | 20260506 10:38:22.120 | INFO | "pinned" : " 29 GB" | 20260506 10:38:22.120 | INFO | } | 20260506 10:38:22.120 | INFO | Perf report for the last 5.13957 seconds: | 20260506 10:38:22.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:38:22.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:22.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:22.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:22.120 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 10:38:22.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:22.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:22.120 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:38:22.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:22.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:22.120 | INFO | Uploaded 36GiB to GPU in 1.60698s (22.4023GiB/s) | 20260506 10:38:22.120 | INFO | Downloaded 34GiB to GPU in 1.52531s (22.2905GiB/s) | 20260506 10:38:27.259 | INFO | Analyzing chunk frames = [698368,698880) | 20260506 10:38:27.259 | INFO | MemInfo: { | 20260506 10:38:27.259 | INFO | "gpu" : " 17 GB", | 20260506 10:38:27.259 | INFO | "huge" : "", | 20260506 10:38:27.259 | INFO | "huge_shared" : " 27 GB", | 20260506 10:38:27.259 | INFO | "malloc" : " 14 GB", | 20260506 10:38:27.259 | INFO | "pinned" : " 29 GB" | 20260506 10:38:27.259 | INFO | } | 20260506 10:38:27.259 | INFO | Perf report for the last 5.13919 seconds: | 20260506 10:38:27.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:38:27.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:27.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:27.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:27.259 | INFO | ANALYSIS : 56.3%, 3.0, 87.7ms, 0 | 20260506 10:38:27.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:27.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:27.259 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:38:27.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:27.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:27.259 | INFO | Uploaded 36GiB to GPU in 1.60556s (22.4221GiB/s) | 20260506 10:38:27.260 | INFO | Downloaded 34GiB to GPU in 1.52319s (22.3216GiB/s) | 20260506 10:38:32.398 | INFO | Analyzing chunk frames = [698880,699392) | 20260506 10:38:32.398 | INFO | MemInfo: { | 20260506 10:38:32.398 | INFO | "gpu" : " 17 GB", | 20260506 10:38:32.398 | INFO | "huge" : "", | 20260506 10:38:32.398 | INFO | "huge_shared" : " 27 GB", | 20260506 10:38:32.398 | INFO | "malloc" : " 14 GB", | 20260506 10:38:32.399 | INFO | "pinned" : " 29 GB" | 20260506 10:38:32.399 | INFO | } | 20260506 10:38:32.399 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:38:32.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:38:32.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:32.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:32.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:32.399 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:38:32.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:32.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:32.399 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:38:32.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:32.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:32.399 | INFO | Uploaded 36GiB to GPU in 1.60582s (22.4185GiB/s) | 20260506 10:38:32.399 | INFO | Downloaded 34GiB to GPU in 1.52369s (22.3143GiB/s) | 20260506 10:38:37.538 | INFO | Analyzing chunk frames = [699392,699904) | 20260506 10:38:37.538 | INFO | MemInfo: { | 20260506 10:38:37.538 | INFO | "gpu" : " 17 GB", | 20260506 10:38:37.538 | INFO | "huge" : "", | 20260506 10:38:37.538 | INFO | "huge_shared" : " 27 GB", | 20260506 10:38:37.538 | INFO | "malloc" : " 14 GB", | 20260506 10:38:37.538 | INFO | "pinned" : " 29 GB" | 20260506 10:38:37.538 | INFO | } | 20260506 10:38:37.538 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:38:37.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:38:37.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:37.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:37.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:37.538 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:38:37.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:37.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:37.538 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:38:37.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:37.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:37.538 | INFO | Uploaded 36GiB to GPU in 1.60586s (22.4179GiB/s) | 20260506 10:38:37.538 | INFO | Downloaded 34GiB to GPU in 1.52403s (22.3093GiB/s) | 20260506 10:38:42.677 | INFO | Analyzing chunk frames = [699904,700416) | 20260506 10:38:42.677 | INFO | MemInfo: { | 20260506 10:38:42.677 | INFO | "gpu" : " 17 GB", | 20260506 10:38:42.677 | INFO | "huge" : "", | 20260506 10:38:42.677 | INFO | "huge_shared" : " 27 GB", | 20260506 10:38:42.677 | INFO | "malloc" : " 14 GB", | 20260506 10:38:42.677 | INFO | "pinned" : " 29 GB" | 20260506 10:38:42.677 | INFO | } | 20260506 10:38:42.677 | INFO | Perf report for the last 5.13929 seconds: | 20260506 10:38:42.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:38:42.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:42.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:42.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:42.677 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:38:42.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:42.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:42.677 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:38:42.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:42.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:42.678 | INFO | Uploaded 36GiB to GPU in 1.6052s (22.4271GiB/s) | 20260506 10:38:42.678 | INFO | Downloaded 34GiB to GPU in 1.5237s (22.3141GiB/s) | 20260506 10:38:42.774 | INFO | Starting a round of Baz IO | 20260506 10:38:45.683 | INFO | Populating RTMetrics report | 20260506 10:38:45.683 | INFO | RTMetrics report populated | 20260506 10:38:45.683 | INFO | Opening tmp RTMetrics json file | 20260506 10:38:45.684 | INFO | Serializing RTMetrics to json | 20260506 10:38:45.686 | INFO | Serialization done, swapping tmp file now... | 20260506 10:38:45.686 | INFO | RTMetrics done | 20260506 10:38:45.819 | INFO | Finished a round of Baz IO: Wrote 2.001 GiB in 3.044 seconds | 20260506 10:38:45.819 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 47.04% / 0.05% / 52.92% | 20260506 10:38:45.819 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.44177% / 8.74201e-05% / 3.17891e-05% | 20260506 10:38:45.819 | INFO | Baz IO overall: | 20260506 10:38:45.819 | INFO | BytesWritten : 320023700434 | 20260506 10:38:45.819 | INFO | NumEvents : 153951177126 | 20260506 10:38:47.816 | INFO | Analyzing chunk frames = [700416,700928) | 20260506 10:38:47.816 | INFO | MemInfo: { | 20260506 10:38:47.816 | INFO | "gpu" : " 17 GB", | 20260506 10:38:47.816 | INFO | "huge" : "", | 20260506 10:38:47.816 | INFO | "huge_shared" : " 27 GB", | 20260506 10:38:47.816 | INFO | "malloc" : " 14 GB", | 20260506 10:38:47.817 | INFO | "pinned" : " 29 GB" | 20260506 10:38:47.817 | INFO | } | 20260506 10:38:47.817 | INFO | Perf report for the last 5.13927 seconds: | 20260506 10:38:47.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:38:47.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:47.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:47.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:47.817 | INFO | ANALYSIS : 57.5%, 2.8, 85.2ms, 0 | 20260506 10:38:47.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:47.817 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 10:38:47.817 | INFO | PRE_HQ : 56.5%, 2.1, 60.7ms, 0 | 20260506 10:38:47.817 | INFO | BAZWRITER : 58.9%, 4.8, 144.7ms, 0 | 20260506 10:38:47.817 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 10:38:47.817 | INFO | Uploaded 36GiB to GPU in 1.67419s (21.5029GiB/s) | 20260506 10:38:47.817 | INFO | Downloaded 38GiB to GPU in 1.73929s (21.848GiB/s) | 20260506 10:38:52.956 | INFO | Analyzing chunk frames = [700928,701440) | 20260506 10:38:52.956 | INFO | MemInfo: { | 20260506 10:38:52.956 | INFO | "gpu" : " 17 GB", | 20260506 10:38:52.956 | INFO | "huge" : "", | 20260506 10:38:52.956 | INFO | "huge_shared" : " 27 GB", | 20260506 10:38:52.956 | INFO | "malloc" : " 14 GB", | 20260506 10:38:52.956 | INFO | "pinned" : " 29 GB" | 20260506 10:38:52.956 | INFO | } | 20260506 10:38:52.956 | INFO | Perf report for the last 5.13944 seconds: | 20260506 10:38:52.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:38:52.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:52.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:52.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:52.956 | INFO | ANALYSIS : 57.3%, 3.0, 89.5ms, 0 | 20260506 10:38:52.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:52.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:52.956 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:38:52.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:52.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:52.956 | INFO | Uploaded 36GiB to GPU in 1.60312s (22.4563GiB/s) | 20260506 10:38:52.956 | INFO | Downloaded 34GiB to GPU in 1.52009s (22.3671GiB/s) | 20260506 10:38:58.095 | INFO | Analyzing chunk frames = [701440,701952) | 20260506 10:38:58.095 | INFO | MemInfo: { | 20260506 10:38:58.095 | INFO | "gpu" : " 17 GB", | 20260506 10:38:58.095 | INFO | "huge" : "", | 20260506 10:38:58.095 | INFO | "huge_shared" : " 27 GB", | 20260506 10:38:58.095 | INFO | "malloc" : " 14 GB", | 20260506 10:38:58.095 | INFO | "pinned" : " 29 GB" | 20260506 10:38:58.095 | INFO | } | 20260506 10:38:58.095 | INFO | Perf report for the last 5.13922 seconds: | 20260506 10:38:58.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:38:58.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:58.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:58.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:58.095 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:38:58.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:58.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:58.095 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:38:58.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:58.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:38:58.095 | INFO | Uploaded 36GiB to GPU in 1.60764s (22.393GiB/s) | 20260506 10:38:58.096 | INFO | Downloaded 34GiB to GPU in 1.52521s (22.292GiB/s) | 20260506 10:39:03.234 | INFO | Analyzing chunk frames = [701952,702464) | 20260506 10:39:03.234 | INFO | MemInfo: { | 20260506 10:39:03.234 | INFO | "gpu" : " 17 GB", | 20260506 10:39:03.234 | INFO | "huge" : "", | 20260506 10:39:03.235 | INFO | "huge_shared" : " 27 GB", | 20260506 10:39:03.235 | INFO | "malloc" : " 14 GB", | 20260506 10:39:03.235 | INFO | "pinned" : " 29 GB" | 20260506 10:39:03.235 | INFO | } | 20260506 10:39:03.235 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:39:03.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:39:03.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:03.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:03.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:03.235 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:39:03.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:03.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:03.235 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:39:03.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:03.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:03.235 | INFO | Uploaded 36GiB to GPU in 1.60759s (22.3938GiB/s) | 20260506 10:39:03.235 | INFO | Downloaded 34GiB to GPU in 1.52485s (22.2973GiB/s) | 20260506 10:39:08.374 | INFO | Analyzing chunk frames = [702464,702976) | 20260506 10:39:08.374 | INFO | MemInfo: { | 20260506 10:39:08.374 | INFO | "gpu" : " 17 GB", | 20260506 10:39:08.374 | INFO | "huge" : "", | 20260506 10:39:08.374 | INFO | "huge_shared" : " 27 GB", | 20260506 10:39:08.374 | INFO | "malloc" : " 14 GB", | 20260506 10:39:08.374 | INFO | "pinned" : " 29 GB" | 20260506 10:39:08.374 | INFO | } | 20260506 10:39:08.374 | INFO | Perf report for the last 5.13929 seconds: | 20260506 10:39:08.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:39:08.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:08.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:08.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:08.374 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:39:08.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:08.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:08.374 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:39:08.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:08.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:08.374 | INFO | Uploaded 36GiB to GPU in 1.60558s (22.4218GiB/s) | 20260506 10:39:08.374 | INFO | Downloaded 34GiB to GPU in 1.52279s (22.3274GiB/s) | 20260506 10:39:13.513 | INFO | Analyzing chunk frames = [702976,703488) | 20260506 10:39:13.513 | INFO | MemInfo: { | 20260506 10:39:13.513 | INFO | "gpu" : " 17 GB", | 20260506 10:39:13.513 | INFO | "huge" : "", | 20260506 10:39:13.513 | INFO | "huge_shared" : " 27 GB", | 20260506 10:39:13.513 | INFO | "malloc" : " 14 GB", | 20260506 10:39:13.513 | INFO | "pinned" : " 29 GB" | 20260506 10:39:13.513 | INFO | } | 20260506 10:39:13.513 | INFO | Perf report for the last 5.13939 seconds: | 20260506 10:39:13.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:39:13.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:13.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:13.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:13.513 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:39:13.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:13.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:13.513 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:39:13.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:13.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:13.514 | INFO | Uploaded 36GiB to GPU in 1.60545s (22.4236GiB/s) | 20260506 10:39:13.514 | INFO | Downloaded 34GiB to GPU in 1.52209s (22.3378GiB/s) | 20260506 10:39:18.653 | INFO | Analyzing chunk frames = [703488,704000) | 20260506 10:39:18.653 | INFO | MemInfo: { | 20260506 10:39:18.653 | INFO | "gpu" : " 17 GB", | 20260506 10:39:18.653 | INFO | "huge" : "", | 20260506 10:39:18.653 | INFO | "huge_shared" : " 27 GB", | 20260506 10:39:18.653 | INFO | "malloc" : " 14 GB", | 20260506 10:39:18.653 | INFO | "pinned" : " 29 GB" | 20260506 10:39:18.653 | INFO | } | 20260506 10:39:18.653 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:39:18.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:39:18.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:18.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:18.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:18.653 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:39:18.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:18.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:18.653 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:39:18.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:18.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:18.653 | INFO | Uploaded 36GiB to GPU in 1.60682s (22.4045GiB/s) | 20260506 10:39:18.653 | INFO | Downloaded 34GiB to GPU in 1.52427s (22.3057GiB/s) | 20260506 10:39:23.792 | INFO | Analyzing chunk frames = [704000,704512) | 20260506 10:39:23.792 | INFO | MemInfo: { | 20260506 10:39:23.792 | INFO | "gpu" : " 17 GB", | 20260506 10:39:23.792 | INFO | "huge" : "", | 20260506 10:39:23.792 | INFO | "huge_shared" : " 27 GB", | 20260506 10:39:23.792 | INFO | "malloc" : " 14 GB", | 20260506 10:39:23.792 | INFO | "pinned" : " 29 GB" | 20260506 10:39:23.792 | INFO | } | 20260506 10:39:23.792 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:39:23.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:39:23.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:23.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:23.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:23.792 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:39:23.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:23.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:23.792 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 10:39:23.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:23.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:23.792 | INFO | Uploaded 36GiB to GPU in 1.60626s (22.4123GiB/s) | 20260506 10:39:23.792 | INFO | Downloaded 34GiB to GPU in 1.52394s (22.3105GiB/s) | 20260506 10:39:23.793 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:39:23.793 | INFO | Total mem usage: 16.5 GiB | 20260506 10:39:23.793 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:39:23.793 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -9.81%) | 20260506 10:39:23.793 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:39:23.793 | INFO | High Water Breakdown: | 20260506 10:39:23.793 | INFO | 59.8% : 9 GiB ( 89.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:39:23.793 | INFO | 40% : 6.02 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:39:23.793 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:39:23.793 | INFO | ------------------------------------------ | 20260506 10:39:23.793 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:39:23.793 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:39:23.793 | INFO | Total mem usage: 26.5 GiB | 20260506 10:39:23.793 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:39:23.793 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:39:23.793 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:39:23.793 | INFO | High Water Breakdown: | 20260506 10:39:23.793 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:39:23.793 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:39:23.793 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:39:23.793 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:39:23.793 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:39:23.793 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:39:23.793 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:39:23.793 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:39:23.793 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:39:23.793 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:39:23.793 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:39:23.793 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:39:23.793 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:39:23.793 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:39:23.793 | INFO | ------------------------------------------ | 20260506 10:39:23.793 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:39:23.793 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:39:23.793 | INFO | Total mem usage: 25.6 GiB | 20260506 10:39:23.793 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:39:23.793 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:39:23.793 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:39:23.793 | INFO | High Water Breakdown: | 20260506 10:39:23.793 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:39:23.793 | INFO | ------------------------------------------ | 20260506 10:39:23.793 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:39:23.793 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:39:23.793 | INFO | Total mem usage: 15.7 GiB | 20260506 10:39:23.793 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:39:23.793 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:39:23.793 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:39:23.793 | INFO | High Water Breakdown: | 20260506 10:39:23.793 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:39:23.793 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:39:23.793 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:39:23.793 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:39:23.793 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:39:23.793 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:39:23.793 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:39:23.793 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:39:23.793 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:39:23.793 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:39:23.793 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:39:23.793 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:39:23.793 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:39:23.793 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:39:23.793 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:39:23.793 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:39:23.793 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:39:23.793 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:39:23.793 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:39:23.793 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:39:23.793 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:39:23.793 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:39:23.793 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:39:23.793 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:39:23.793 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:39:23.793 | INFO | ------------------------------------------ | 20260506 10:39:23.793 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:39:23.793 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:39:23.793 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:39:23.793 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:39:23.793 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 10:39:23.793 | INFO | PerfReport: 0.29%, AggregateStats -- 8.68e-02 / -nan, 6.81e-03 / 0.00e+00 | 20260506 10:39:23.793 | INFO | PerfReport: 10.96%, Baseline -- 3.30e+00 / -nan, 1.53e-02 / 0.00e+00 | 20260506 10:39:23.793 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.24e-02 / 0.00e+00 | 20260506 10:39:23.793 | INFO | PerfReport: 16.06%, DME -- 3.86e+01 / -nan, 3.01e+00 / 0.00e+00 | 20260506 10:39:23.793 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.72e+01 / -nan, 4.26e-02 / 0.00e+00 | 20260506 10:39:23.793 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.22e+00 / -nan, 1.28e-01 / 0.00e+00 | 20260506 10:39:23.793 | INFO | PerfReport: 3.65%, Metrics -- 1.10e+00 / -nan, 1.71e-01 / 0.00e+00 | 20260506 10:39:23.793 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.24e-01 / -nan, 5.94e-02 / 0.00e+00 | 20260506 10:39:23.793 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:39:23.793 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:39:23.793 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:39:23.793 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:39:23.793 | INFO | PerfReport: 18.74%, Upload -- 1.68e+01 / 0.00e+00, 4.96e-01 / 0.00e+00 | 20260506 10:39:23.793 | INFO | PerfReport: 17.99%, Download -- 1.62e+01 / 0.00e+00, 8.51e-01 / 0.00e+00 | 20260506 10:39:23.793 | INFO | PerfReport: 33.50%, Compute -- 3.01e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 10:39:23.793 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:39:23.793 | INFO | PerfReport: 0.00%, LockDownload -- 4.92e-04 / 0.00e+00, 2.28e-04 / 0.00e+00 | 20260506 10:39:23.793 | INFO | PerfReport: 29.19%, LockCompute -- 2.62e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 10:39:23.793 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260506 10:39:23.889 | INFO | Starting a round of Baz IO | 20260506 10:39:26.797 | INFO | Populating RTMetrics report | 20260506 10:39:26.797 | INFO | RTMetrics report populated | 20260506 10:39:26.797 | INFO | Opening tmp RTMetrics json file | 20260506 10:39:26.813 | INFO | Serializing RTMetrics to json | 20260506 10:39:26.818 | INFO | Serialization done, swapping tmp file now... | 20260506 10:39:26.823 | INFO | RTMetrics done | 20260506 10:39:27.105 | INFO | Finished a round of Baz IO: Wrote 2.005 GiB in 3.215 seconds | 20260506 10:39:27.105 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.99% / 0.05% / 52.96% | 20260506 10:39:27.105 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.43935% / 6.35783e-05% / 4.76837e-05% | 20260506 10:39:27.105 | INFO | Baz IO overall: | 20260506 10:39:27.105 | INFO | BytesWritten : 322176932271 | 20260506 10:39:27.105 | INFO | NumEvents : 154995670630 | 20260506 10:39:28.932 | INFO | Analyzing chunk frames = [704512,705024) | 20260506 10:39:28.932 | INFO | MemInfo: { | 20260506 10:39:28.932 | INFO | "gpu" : " 17 GB", | 20260506 10:39:28.932 | INFO | "huge" : "", | 20260506 10:39:28.932 | INFO | "huge_shared" : " 27 GB", | 20260506 10:39:28.932 | INFO | "malloc" : " 14 GB", | 20260506 10:39:28.932 | INFO | "pinned" : " 29 GB" | 20260506 10:39:28.932 | INFO | } | 20260506 10:39:28.932 | INFO | Perf report for the last 5.13952 seconds: | 20260506 10:39:28.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:39:28.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:28.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:28.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:28.932 | INFO | ANALYSIS : 57.5%, 2.8, 85.1ms, 0 | 20260506 10:39:28.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:28.932 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 10:39:28.932 | INFO | PRE_HQ : 56.6%, 2.1, 60.6ms, 0 | 20260506 10:39:28.932 | INFO | BAZWRITER : 62.2%, 4.8, 152.1ms, 0 | 20260506 10:39:28.932 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 10:39:28.932 | INFO | Uploaded 36GiB to GPU in 1.67498s (21.4928GiB/s) | 20260506 10:39:28.932 | INFO | Downloaded 38GiB to GPU in 1.74158s (21.8193GiB/s) | 20260506 10:39:34.071 | INFO | Analyzing chunk frames = [705024,705536) | 20260506 10:39:34.071 | INFO | MemInfo: { | 20260506 10:39:34.071 | INFO | "gpu" : " 17 GB", | 20260506 10:39:34.071 | INFO | "huge" : "", | 20260506 10:39:34.071 | INFO | "huge_shared" : " 27 GB", | 20260506 10:39:34.071 | INFO | "malloc" : " 14 GB", | 20260506 10:39:34.071 | INFO | "pinned" : " 29 GB" | 20260506 10:39:34.071 | INFO | } | 20260506 10:39:34.071 | INFO | Perf report for the last 5.13902 seconds: | 20260506 10:39:34.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:39:34.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:34.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:34.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:34.071 | INFO | ANALYSIS : 57.4%, 3.0, 89.5ms, 0 | 20260506 10:39:34.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:34.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:34.071 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:39:34.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:34.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:34.071 | INFO | Uploaded 36GiB to GPU in 1.60287s (22.4597GiB/s) | 20260506 10:39:34.072 | INFO | Downloaded 34GiB to GPU in 1.52017s (22.3659GiB/s) | 20260506 10:39:39.210 | INFO | Analyzing chunk frames = [705536,706048) | 20260506 10:39:39.210 | INFO | MemInfo: { | 20260506 10:39:39.210 | INFO | "gpu" : " 17 GB", | 20260506 10:39:39.210 | INFO | "huge" : "", | 20260506 10:39:39.210 | INFO | "huge_shared" : " 27 GB", | 20260506 10:39:39.210 | INFO | "malloc" : " 14 GB", | 20260506 10:39:39.210 | INFO | "pinned" : " 29 GB" | 20260506 10:39:39.210 | INFO | } | 20260506 10:39:39.210 | INFO | Perf report for the last 5.13951 seconds: | 20260506 10:39:39.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:39:39.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:39.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:39.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:39.210 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260506 10:39:39.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:39.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:39.210 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:39:39.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:39.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:39.211 | INFO | Uploaded 36GiB to GPU in 1.60761s (22.3935GiB/s) | 20260506 10:39:39.211 | INFO | Downloaded 34GiB to GPU in 1.52562s (22.286GiB/s) | 20260506 10:39:44.350 | INFO | Analyzing chunk frames = [706048,706560) | 20260506 10:39:44.350 | INFO | MemInfo: { | 20260506 10:39:44.350 | INFO | "gpu" : " 17 GB", | 20260506 10:39:44.350 | INFO | "huge" : "", | 20260506 10:39:44.350 | INFO | "huge_shared" : " 27 GB", | 20260506 10:39:44.350 | INFO | "malloc" : " 14 GB", | 20260506 10:39:44.350 | INFO | "pinned" : " 29 GB" | 20260506 10:39:44.350 | INFO | } | 20260506 10:39:44.350 | INFO | Perf report for the last 5.13957 seconds: | 20260506 10:39:44.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:39:44.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:44.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:44.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:44.350 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:39:44.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:44.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:44.350 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:39:44.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:44.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:44.350 | INFO | Uploaded 36GiB to GPU in 1.60704s (22.4015GiB/s) | 20260506 10:39:44.350 | INFO | Downloaded 34GiB to GPU in 1.52451s (22.3023GiB/s) | 20260506 10:39:49.489 | INFO | Analyzing chunk frames = [706560,707072) | 20260506 10:39:49.489 | INFO | MemInfo: { | 20260506 10:39:49.489 | INFO | "gpu" : " 17 GB", | 20260506 10:39:49.489 | INFO | "huge" : "", | 20260506 10:39:49.489 | INFO | "huge_shared" : " 27 GB", | 20260506 10:39:49.489 | INFO | "malloc" : " 14 GB", | 20260506 10:39:49.489 | INFO | "pinned" : " 29 GB" | 20260506 10:39:49.489 | INFO | } | 20260506 10:39:49.489 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:39:49.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:39:49.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:49.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:49.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:49.489 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:39:49.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:49.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:49.489 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:39:49.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:49.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:49.489 | INFO | Uploaded 36GiB to GPU in 1.60584s (22.4182GiB/s) | 20260506 10:39:49.489 | INFO | Downloaded 34GiB to GPU in 1.52302s (22.324GiB/s) | 20260506 10:39:54.628 | INFO | Analyzing chunk frames = [707072,707584) | 20260506 10:39:54.628 | INFO | MemInfo: { | 20260506 10:39:54.628 | INFO | "gpu" : " 17 GB", | 20260506 10:39:54.628 | INFO | "huge" : "", | 20260506 10:39:54.628 | INFO | "huge_shared" : " 27 GB", | 20260506 10:39:54.628 | INFO | "malloc" : " 14 GB", | 20260506 10:39:54.628 | INFO | "pinned" : " 29 GB" | 20260506 10:39:54.628 | INFO | } | 20260506 10:39:54.628 | INFO | Perf report for the last 5.13905 seconds: | 20260506 10:39:54.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:39:54.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:54.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:54.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:54.628 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:39:54.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:54.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:54.628 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:39:54.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:54.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:54.629 | INFO | Uploaded 36GiB to GPU in 1.60531s (22.4255GiB/s) | 20260506 10:39:54.629 | INFO | Downloaded 34GiB to GPU in 1.52239s (22.3333GiB/s) | 20260506 10:39:59.767 | INFO | Analyzing chunk frames = [707584,708096) | 20260506 10:39:59.767 | INFO | MemInfo: { | 20260506 10:39:59.767 | INFO | "gpu" : " 17 GB", | 20260506 10:39:59.767 | INFO | "huge" : "", | 20260506 10:39:59.767 | INFO | "huge_shared" : " 27 GB", | 20260506 10:39:59.767 | INFO | "malloc" : " 14 GB", | 20260506 10:39:59.767 | INFO | "pinned" : " 29 GB" | 20260506 10:39:59.768 | INFO | } | 20260506 10:39:59.768 | INFO | Perf report for the last 5.13944 seconds: | 20260506 10:39:59.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:39:59.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:59.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:59.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:59.768 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:39:59.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:59.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:59.768 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:39:59.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:59.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:39:59.768 | INFO | Uploaded 36GiB to GPU in 1.60585s (22.418GiB/s) | 20260506 10:39:59.768 | INFO | Downloaded 34GiB to GPU in 1.52361s (22.3155GiB/s) | 20260506 10:40:04.907 | INFO | Analyzing chunk frames = [708096,708608) | 20260506 10:40:04.907 | INFO | MemInfo: { | 20260506 10:40:04.907 | INFO | "gpu" : " 17 GB", | 20260506 10:40:04.907 | INFO | "huge" : "", | 20260506 10:40:04.907 | INFO | "huge_shared" : " 27 GB", | 20260506 10:40:04.907 | INFO | "malloc" : " 14 GB", | 20260506 10:40:04.907 | INFO | "pinned" : " 29 GB" | 20260506 10:40:04.907 | INFO | } | 20260506 10:40:04.907 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:40:04.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:40:04.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:04.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:04.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:04.907 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:40:04.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:04.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:04.907 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:40:04.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:04.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:04.908 | INFO | Uploaded 36GiB to GPU in 1.60572s (22.4198GiB/s) | 20260506 10:40:04.908 | INFO | Downloaded 34GiB to GPU in 1.5233s (22.32GiB/s) | 20260506 10:40:05.004 | INFO | Starting a round of Baz IO | 20260506 10:40:07.919 | INFO | Populating RTMetrics report | 20260506 10:40:07.920 | INFO | RTMetrics report populated | 20260506 10:40:07.920 | INFO | Opening tmp RTMetrics json file | 20260506 10:40:07.927 | INFO | Serializing RTMetrics to json | 20260506 10:40:07.929 | INFO | Serialization done, swapping tmp file now... | 20260506 10:40:07.933 | INFO | RTMetrics done | 20260506 10:40:08.365 | INFO | Finished a round of Baz IO: Wrote 2.005 GiB in 3.36 seconds | 20260506 10:40:08.365 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.94% / 0.05% / 53.01% | 20260506 10:40:08.365 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.42043% / 8.74201e-05% / 5.16574e-05% | 20260506 10:40:08.365 | INFO | Baz IO overall: | 20260506 10:40:08.365 | INFO | BytesWritten : 324330254098 | 20260506 10:40:08.365 | INFO | NumEvents : 156040314677 | 20260506 10:40:10.046 | INFO | Analyzing chunk frames = [708608,709120) | 20260506 10:40:10.046 | INFO | MemInfo: { | 20260506 10:40:10.046 | INFO | "gpu" : " 17 GB", | 20260506 10:40:10.046 | INFO | "huge" : "", | 20260506 10:40:10.046 | INFO | "huge_shared" : " 27 GB", | 20260506 10:40:10.046 | INFO | "malloc" : " 14 GB", | 20260506 10:40:10.046 | INFO | "pinned" : " 29 GB" | 20260506 10:40:10.046 | INFO | } | 20260506 10:40:10.046 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:40:10.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:40:10.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:10.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:10.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:10.046 | INFO | ANALYSIS : 57.6%, 2.9, 86.3ms, 0 | 20260506 10:40:10.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:10.046 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 10:40:10.046 | INFO | PRE_HQ : 56.8%, 2.1, 61.0ms, 0 | 20260506 10:40:10.046 | INFO | BAZWRITER : 65.0%, 4.7, 156.3ms, 0 | 20260506 10:40:10.046 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 10:40:10.047 | INFO | Uploaded 36GiB to GPU in 1.66599s (21.6088GiB/s) | 20260506 10:40:10.047 | INFO | Downloaded 38GiB to GPU in 1.72945s (21.9723GiB/s) | 20260506 10:40:15.185 | INFO | Analyzing chunk frames = [709120,709632) | 20260506 10:40:15.185 | INFO | MemInfo: { | 20260506 10:40:15.185 | INFO | "gpu" : " 17 GB", | 20260506 10:40:15.185 | INFO | "huge" : "", | 20260506 10:40:15.185 | INFO | "huge_shared" : " 27 GB", | 20260506 10:40:15.185 | INFO | "malloc" : " 14 GB", | 20260506 10:40:15.186 | INFO | "pinned" : " 29 GB" | 20260506 10:40:15.186 | INFO | } | 20260506 10:40:15.186 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:40:15.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:40:15.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:15.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:15.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:15.186 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 10:40:15.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:15.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:15.186 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:40:15.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:15.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:15.186 | INFO | Uploaded 36GiB to GPU in 1.60359s (22.4497GiB/s) | 20260506 10:40:15.186 | INFO | Downloaded 34GiB to GPU in 1.52117s (22.3512GiB/s) | 20260506 10:40:20.325 | INFO | Analyzing chunk frames = [709632,710144) | 20260506 10:40:20.325 | INFO | MemInfo: { | 20260506 10:40:20.325 | INFO | "gpu" : " 17 GB", | 20260506 10:40:20.325 | INFO | "huge" : "", | 20260506 10:40:20.325 | INFO | "huge_shared" : " 27 GB", | 20260506 10:40:20.325 | INFO | "malloc" : " 14 GB", | 20260506 10:40:20.325 | INFO | "pinned" : " 29 GB" | 20260506 10:40:20.325 | INFO | } | 20260506 10:40:20.325 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:40:20.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:40:20.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:20.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:20.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:20.325 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:40:20.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:20.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:20.325 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:40:20.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:20.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:20.325 | INFO | Uploaded 36GiB to GPU in 1.60737s (22.3969GiB/s) | 20260506 10:40:20.325 | INFO | Downloaded 34GiB to GPU in 1.5252s (22.2921GiB/s) | 20260506 10:40:25.464 | INFO | Analyzing chunk frames = [710144,710656) | 20260506 10:40:25.464 | INFO | MemInfo: { | 20260506 10:40:25.464 | INFO | "gpu" : " 17 GB", | 20260506 10:40:25.464 | INFO | "huge" : "", | 20260506 10:40:25.464 | INFO | "huge_shared" : " 27 GB", | 20260506 10:40:25.464 | INFO | "malloc" : " 14 GB", | 20260506 10:40:25.464 | INFO | "pinned" : " 29 GB" | 20260506 10:40:25.464 | INFO | } | 20260506 10:40:25.464 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:40:25.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:40:25.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:25.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:25.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:25.464 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 10:40:25.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:25.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:25.464 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:40:25.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:25.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:25.465 | INFO | Uploaded 36GiB to GPU in 1.60725s (22.3985GiB/s) | 20260506 10:40:25.465 | INFO | Downloaded 34GiB to GPU in 1.52473s (22.299GiB/s) | 20260506 10:40:30.603 | INFO | Analyzing chunk frames = [710656,711168) | 20260506 10:40:30.603 | INFO | MemInfo: { | 20260506 10:40:30.603 | INFO | "gpu" : " 17 GB", | 20260506 10:40:30.603 | INFO | "huge" : "", | 20260506 10:40:30.603 | INFO | "huge_shared" : " 27 GB", | 20260506 10:40:30.603 | INFO | "malloc" : " 14 GB", | 20260506 10:40:30.603 | INFO | "pinned" : " 29 GB" | 20260506 10:40:30.603 | INFO | } | 20260506 10:40:30.604 | INFO | Perf report for the last 5.13923 seconds: | 20260506 10:40:30.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:40:30.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:30.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:30.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:30.604 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:40:30.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:30.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:30.604 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:40:30.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:30.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:30.604 | INFO | Uploaded 36GiB to GPU in 1.60514s (22.428GiB/s) | 20260506 10:40:30.604 | INFO | Downloaded 34GiB to GPU in 1.52238s (22.3335GiB/s) | 20260506 10:40:35.743 | INFO | Analyzing chunk frames = [711168,711680) | 20260506 10:40:35.743 | INFO | MemInfo: { | 20260506 10:40:35.743 | INFO | "gpu" : " 17 GB", | 20260506 10:40:35.743 | INFO | "huge" : "", | 20260506 10:40:35.743 | INFO | "huge_shared" : " 27 GB", | 20260506 10:40:35.743 | INFO | "malloc" : " 14 GB", | 20260506 10:40:35.743 | INFO | "pinned" : " 29 GB" | 20260506 10:40:35.743 | INFO | } | 20260506 10:40:35.743 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:40:35.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:40:35.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:35.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:35.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:35.743 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 10:40:35.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:35.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:35.743 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:40:35.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:35.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:35.743 | INFO | Uploaded 36GiB to GPU in 1.60591s (22.4172GiB/s) | 20260506 10:40:35.743 | INFO | Downloaded 34GiB to GPU in 1.52265s (22.3294GiB/s) | 20260506 10:40:40.882 | INFO | Analyzing chunk frames = [711680,712192) | 20260506 10:40:40.882 | INFO | MemInfo: { | 20260506 10:40:40.882 | INFO | "gpu" : " 17 GB", | 20260506 10:40:40.882 | INFO | "huge" : "", | 20260506 10:40:40.882 | INFO | "huge_shared" : " 27 GB", | 20260506 10:40:40.882 | INFO | "malloc" : " 14 GB", | 20260506 10:40:40.882 | INFO | "pinned" : " 29 GB" | 20260506 10:40:40.882 | INFO | } | 20260506 10:40:40.882 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:40:40.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:40:40.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:40.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:40.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:40.882 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:40:40.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:40.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:40.882 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:40:40.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:40.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:40.883 | INFO | Uploaded 36GiB to GPU in 1.60606s (22.4151GiB/s) | 20260506 10:40:40.883 | INFO | Downloaded 34GiB to GPU in 1.52343s (22.3181GiB/s) | 20260506 10:40:46.021 | INFO | Analyzing chunk frames = [712192,712704) | 20260506 10:40:46.021 | INFO | MemInfo: { | 20260506 10:40:46.021 | INFO | "gpu" : " 17 GB", | 20260506 10:40:46.021 | INFO | "huge" : "", | 20260506 10:40:46.021 | INFO | "huge_shared" : " 27 GB", | 20260506 10:40:46.022 | INFO | "malloc" : " 14 GB", | 20260506 10:40:46.022 | INFO | "pinned" : " 29 GB" | 20260506 10:40:46.022 | INFO | } | 20260506 10:40:46.022 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:40:46.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:40:46.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:46.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:46.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:46.022 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:40:46.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:46.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:46.022 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 10:40:46.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:46.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:46.022 | INFO | Uploaded 36GiB to GPU in 1.60621s (22.413GiB/s) | 20260506 10:40:46.022 | INFO | Downloaded 34GiB to GPU in 1.5239s (22.3112GiB/s) | 20260506 10:40:46.022 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:40:46.022 | INFO | Total mem usage: 16.5 GiB | 20260506 10:40:46.022 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:40:46.022 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -8.04%) | 20260506 10:40:46.022 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:40:46.022 | INFO | High Water Breakdown: | 20260506 10:40:46.022 | INFO | 60.1% : 9.18 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:40:46.022 | INFO | 39.8% : 6.08 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:40:46.022 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:40:46.022 | INFO | ------------------------------------------ | 20260506 10:40:46.022 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:40:46.022 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:40:46.022 | INFO | Total mem usage: 26.5 GiB | 20260506 10:40:46.022 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:40:46.022 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:40:46.022 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:40:46.022 | INFO | High Water Breakdown: | 20260506 10:40:46.022 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:40:46.022 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:40:46.022 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:40:46.022 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:40:46.022 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:40:46.022 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:40:46.023 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:40:46.023 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:40:46.023 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:40:46.023 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:40:46.023 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:40:46.023 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:40:46.023 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:40:46.023 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:40:46.023 | INFO | ------------------------------------------ | 20260506 10:40:46.023 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:40:46.023 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:40:46.023 | INFO | Total mem usage: 25.6 GiB | 20260506 10:40:46.023 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:40:46.023 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:40:46.023 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:40:46.023 | INFO | High Water Breakdown: | 20260506 10:40:46.023 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:40:46.023 | INFO | ------------------------------------------ | 20260506 10:40:46.023 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:40:46.023 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:40:46.023 | INFO | Total mem usage: 15.7 GiB | 20260506 10:40:46.023 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:40:46.023 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:40:46.023 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:40:46.023 | INFO | High Water Breakdown: | 20260506 10:40:46.023 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:40:46.023 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:40:46.023 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:40:46.023 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:40:46.023 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:40:46.023 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:40:46.023 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:40:46.023 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:40:46.023 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:40:46.023 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:40:46.023 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:40:46.023 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:40:46.023 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:40:46.023 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:40:46.023 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:40:46.023 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:40:46.023 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:40:46.023 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:40:46.023 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:40:46.023 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:40:46.023 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:40:46.023 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:40:46.023 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:40:46.023 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:40:46.023 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:40:46.023 | INFO | ------------------------------------------ | 20260506 10:40:46.023 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:40:46.023 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:40:46.023 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:40:46.023 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:40:46.023 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 10:40:46.023 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.96e-03 / 0.00e+00 | 20260506 10:40:46.023 | INFO | PerfReport: 10.96%, Baseline -- 3.29e+00 / -nan, 1.65e-02 / 0.00e+00 | 20260506 10:40:46.023 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.22e-02 / 0.00e+00 | 20260506 10:40:46.023 | INFO | PerfReport: 16.07%, DME -- 3.87e+01 / -nan, 3.07e+00 / 0.00e+00 | 20260506 10:40:46.023 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.72e+01 / -nan, 4.24e-02 / 0.00e+00 | 20260506 10:40:46.023 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.22e+00 / -nan, 1.28e-01 / 0.00e+00 | 20260506 10:40:46.023 | INFO | PerfReport: 3.65%, Metrics -- 1.10e+00 / -nan, 1.70e-01 / 0.00e+00 | 20260506 10:40:46.023 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.24e-01 / -nan, 5.96e-02 / 0.00e+00 | 20260506 10:40:46.023 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:40:46.023 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:40:46.023 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:40:46.023 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.85e+01 / 0.00e+00 | 20260506 10:40:46.023 | INFO | PerfReport: 18.71%, Upload -- 1.68e+01 / 0.00e+00, 4.78e-01 / 0.00e+00 | 20260506 10:40:46.023 | INFO | PerfReport: 17.97%, Download -- 1.61e+01 / 0.00e+00, 8.30e-01 / 0.00e+00 | 20260506 10:40:46.023 | INFO | PerfReport: 33.48%, Compute -- 3.01e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 10:40:46.023 | INFO | PerfReport: 0.57%, LockUpload -- 5.11e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:40:46.023 | INFO | PerfReport: 0.00%, LockDownload -- 4.82e-04 / 0.00e+00, 1.18e-04 / 0.00e+00 | 20260506 10:40:46.023 | INFO | PerfReport: 29.25%, LockCompute -- 2.63e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 10:40:46.023 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260506 10:40:46.119 | INFO | Starting a round of Baz IO | 20260506 10:40:49.027 | INFO | Populating RTMetrics report | 20260506 10:40:49.027 | INFO | RTMetrics report populated | 20260506 10:40:49.027 | INFO | Opening tmp RTMetrics json file | 20260506 10:40:49.037 | INFO | Serializing RTMetrics to json | 20260506 10:40:49.040 | INFO | Serialization done, swapping tmp file now... | 20260506 10:40:49.043 | INFO | RTMetrics done | 20260506 10:40:49.560 | INFO | Finished a round of Baz IO: Wrote 2.006 GiB in 3.441 seconds | 20260506 10:40:49.560 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.89% / 0.05% / 53.06% | 20260506 10:40:49.560 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.39307% / 9.53674e-05% / 4.37101e-05% | 20260506 10:40:49.560 | INFO | Baz IO overall: | 20260506 10:40:49.561 | INFO | BytesWritten : 326484219494 | 20260506 10:40:49.561 | INFO | NumEvents : 157085254542 | 20260506 10:40:51.161 | INFO | Analyzing chunk frames = [712704,713216) | 20260506 10:40:51.161 | INFO | MemInfo: { | 20260506 10:40:51.161 | INFO | "gpu" : " 17 GB", | 20260506 10:40:51.161 | INFO | "huge" : "", | 20260506 10:40:51.161 | INFO | "huge_shared" : " 27 GB", | 20260506 10:40:51.161 | INFO | "malloc" : " 14 GB", | 20260506 10:40:51.161 | INFO | "pinned" : " 29 GB" | 20260506 10:40:51.161 | INFO | } | 20260506 10:40:51.161 | INFO | Perf report for the last 5.13946 seconds: | 20260506 10:40:51.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:40:51.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:51.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:51.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:51.161 | INFO | ANALYSIS : 57.5%, 2.8, 85.6ms, 0 | 20260506 10:40:51.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:51.161 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 10:40:51.161 | INFO | PRE_HQ : 56.5%, 2.1, 60.1ms, 0 | 20260506 10:40:51.161 | INFO | BAZWRITER : 66.5%, 4.8, 163.4ms, 0 | 20260506 10:40:51.161 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260506 10:40:51.162 | INFO | Uploaded 36GiB to GPU in 1.67932s (21.4373GiB/s) | 20260506 10:40:51.162 | INFO | Downloaded 38GiB to GPU in 1.74455s (21.7821GiB/s) | 20260506 10:40:56.300 | INFO | Analyzing chunk frames = [713216,713728) | 20260506 10:40:56.300 | INFO | MemInfo: { | 20260506 10:40:56.300 | INFO | "gpu" : " 17 GB", | 20260506 10:40:56.300 | INFO | "huge" : "", | 20260506 10:40:56.300 | INFO | "huge_shared" : " 27 GB", | 20260506 10:40:56.300 | INFO | "malloc" : " 14 GB", | 20260506 10:40:56.300 | INFO | "pinned" : " 29 GB" | 20260506 10:40:56.300 | INFO | } | 20260506 10:40:56.300 | INFO | Perf report for the last 5.13921 seconds: | 20260506 10:40:56.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:40:56.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:56.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:56.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:56.300 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 10:40:56.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:56.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:56.300 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:40:56.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:56.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:40:56.301 | INFO | Uploaded 36GiB to GPU in 1.60325s (22.4543GiB/s) | 20260506 10:40:56.301 | INFO | Downloaded 34GiB to GPU in 1.5205s (22.361GiB/s) | 20260506 10:41:01.439 | INFO | Analyzing chunk frames = [713728,714240) | 20260506 10:41:01.439 | INFO | MemInfo: { | 20260506 10:41:01.439 | INFO | "gpu" : " 17 GB", | 20260506 10:41:01.439 | INFO | "huge" : "", | 20260506 10:41:01.439 | INFO | "huge_shared" : " 27 GB", | 20260506 10:41:01.439 | INFO | "malloc" : " 14 GB", | 20260506 10:41:01.440 | INFO | "pinned" : " 29 GB" | 20260506 10:41:01.440 | INFO | } | 20260506 10:41:01.440 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:41:01.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:41:01.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:01.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:01.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:01.440 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:41:01.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:01.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:01.440 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:41:01.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:01.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:01.440 | INFO | Uploaded 36GiB to GPU in 1.6075s (22.3951GiB/s) | 20260506 10:41:01.440 | INFO | Downloaded 34GiB to GPU in 1.5253s (22.2907GiB/s) | 20260506 10:41:06.579 | INFO | Analyzing chunk frames = [714240,714752) | 20260506 10:41:06.579 | INFO | MemInfo: { | 20260506 10:41:06.579 | INFO | "gpu" : " 17 GB", | 20260506 10:41:06.579 | INFO | "huge" : "", | 20260506 10:41:06.579 | INFO | "huge_shared" : " 27 GB", | 20260506 10:41:06.579 | INFO | "malloc" : " 14 GB", | 20260506 10:41:06.579 | INFO | "pinned" : " 29 GB" | 20260506 10:41:06.579 | INFO | } | 20260506 10:41:06.579 | INFO | Perf report for the last 5.13947 seconds: | 20260506 10:41:06.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:41:06.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:06.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:06.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:06.579 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 10:41:06.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:06.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:06.579 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:41:06.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:06.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:06.579 | INFO | Uploaded 36GiB to GPU in 1.60674s (22.4057GiB/s) | 20260506 10:41:06.579 | INFO | Downloaded 34GiB to GPU in 1.5248s (22.298GiB/s) | 20260506 10:41:11.718 | INFO | Analyzing chunk frames = [714752,715264) | 20260506 10:41:11.718 | INFO | MemInfo: { | 20260506 10:41:11.718 | INFO | "gpu" : " 17 GB", | 20260506 10:41:11.718 | INFO | "huge" : "", | 20260506 10:41:11.718 | INFO | "huge_shared" : " 27 GB", | 20260506 10:41:11.718 | INFO | "malloc" : " 14 GB", | 20260506 10:41:11.718 | INFO | "pinned" : " 29 GB" | 20260506 10:41:11.718 | INFO | } | 20260506 10:41:11.718 | INFO | Perf report for the last 5.13921 seconds: | 20260506 10:41:11.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:41:11.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:11.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:11.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:11.718 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:41:11.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:11.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:11.718 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:41:11.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:11.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:11.719 | INFO | Uploaded 36GiB to GPU in 1.60517s (22.4275GiB/s) | 20260506 10:41:11.719 | INFO | Downloaded 34GiB to GPU in 1.5224s (22.3331GiB/s) | 20260506 10:41:16.858 | INFO | Analyzing chunk frames = [715264,715776) | 20260506 10:41:16.858 | INFO | MemInfo: { | 20260506 10:41:16.858 | INFO | "gpu" : " 17 GB", | 20260506 10:41:16.858 | INFO | "huge" : "", | 20260506 10:41:16.858 | INFO | "huge_shared" : " 27 GB", | 20260506 10:41:16.858 | INFO | "malloc" : " 14 GB", | 20260506 10:41:16.858 | INFO | "pinned" : " 29 GB" | 20260506 10:41:16.858 | INFO | } | 20260506 10:41:16.858 | INFO | Perf report for the last 5.13944 seconds: | 20260506 10:41:16.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:41:16.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:16.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:16.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:16.858 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 10:41:16.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:16.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:16.858 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:41:16.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:16.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:16.858 | INFO | Uploaded 36GiB to GPU in 1.60609s (22.4147GiB/s) | 20260506 10:41:16.858 | INFO | Downloaded 34GiB to GPU in 1.52459s (22.3011GiB/s) | 20260506 10:41:21.997 | INFO | Analyzing chunk frames = [715776,716288) | 20260506 10:41:21.997 | INFO | MemInfo: { | 20260506 10:41:21.997 | INFO | "gpu" : " 17 GB", | 20260506 10:41:21.997 | INFO | "huge" : "", | 20260506 10:41:21.997 | INFO | "huge_shared" : " 27 GB", | 20260506 10:41:21.997 | INFO | "malloc" : " 14 GB", | 20260506 10:41:21.997 | INFO | "pinned" : " 29 GB" | 20260506 10:41:21.997 | INFO | } | 20260506 10:41:21.997 | INFO | Perf report for the last 5.13931 seconds: | 20260506 10:41:21.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:41:21.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:21.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:21.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:21.997 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:41:21.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:21.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:21.997 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:41:21.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:21.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:21.998 | INFO | Uploaded 36GiB to GPU in 1.60536s (22.4249GiB/s) | 20260506 10:41:21.998 | INFO | Downloaded 34GiB to GPU in 1.52407s (22.3088GiB/s) | 20260506 10:41:27.136 | INFO | Analyzing chunk frames = [716288,716800) | 20260506 10:41:27.136 | INFO | MemInfo: { | 20260506 10:41:27.136 | INFO | "gpu" : " 17 GB", | 20260506 10:41:27.136 | INFO | "huge" : "", | 20260506 10:41:27.136 | INFO | "huge_shared" : " 27 GB", | 20260506 10:41:27.136 | INFO | "malloc" : " 14 GB", | 20260506 10:41:27.136 | INFO | "pinned" : " 29 GB" | 20260506 10:41:27.136 | INFO | } | 20260506 10:41:27.137 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:41:27.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:41:27.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:27.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:27.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:27.137 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:41:27.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:27.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:27.137 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 10:41:27.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:27.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:27.137 | INFO | Uploaded 36GiB to GPU in 1.60613s (22.4141GiB/s) | 20260506 10:41:27.137 | INFO | Downloaded 34GiB to GPU in 1.52439s (22.304GiB/s) | 20260506 10:41:27.234 | INFO | Starting a round of Baz IO | 20260506 10:41:30.147 | INFO | Populating RTMetrics report | 20260506 10:41:30.147 | INFO | RTMetrics report populated | 20260506 10:41:30.147 | INFO | Opening tmp RTMetrics json file | 20260506 10:41:30.162 | INFO | Serializing RTMetrics to json | 20260506 10:41:30.226 | INFO | Serialization done, swapping tmp file now... | 20260506 10:41:30.244 | INFO | RTMetrics done | 20260506 10:41:30.515 | INFO | Finished a round of Baz IO: Wrote 2.009 GiB in 3.28 seconds | 20260506 10:41:30.515 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.85% / 0.05% / 53.11% | 20260506 10:41:30.515 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.39832% / 9.13938e-05% / 4.37101e-05% | 20260506 10:41:30.515 | INFO | Baz IO overall: | 20260506 10:41:30.515 | INFO | BytesWritten : 328641594555 | 20260506 10:41:30.515 | INFO | NumEvents : 158132768099 | 20260506 10:41:32.276 | INFO | Analyzing chunk frames = [716800,717312) | 20260506 10:41:32.276 | INFO | MemInfo: { | 20260506 10:41:32.276 | INFO | "gpu" : " 17 GB", | 20260506 10:41:32.276 | INFO | "huge" : "", | 20260506 10:41:32.276 | INFO | "huge_shared" : " 27 GB", | 20260506 10:41:32.276 | INFO | "malloc" : " 14 GB", | 20260506 10:41:32.276 | INFO | "pinned" : " 29 GB" | 20260506 10:41:32.276 | INFO | } | 20260506 10:41:32.276 | INFO | Perf report for the last 5.13945 seconds: | 20260506 10:41:32.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:41:32.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:32.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:32.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:32.276 | INFO | ANALYSIS : 57.6%, 2.8, 85.3ms, 0 | 20260506 10:41:32.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:32.276 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260506 10:41:32.276 | INFO | PRE_HQ : 56.6%, 2.1, 61.1ms, 0 | 20260506 10:41:32.276 | INFO | BAZWRITER : 63.4%, 4.7, 153.8ms, 0 | 20260506 10:41:32.276 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 10:41:32.276 | INFO | Uploaded 36GiB to GPU in 1.67152s (21.5373GiB/s) | 20260506 10:41:32.276 | INFO | Downloaded 38GiB to GPU in 1.73725s (21.8736GiB/s) | 20260506 10:41:37.415 | INFO | Analyzing chunk frames = [717312,717824) | 20260506 10:41:37.415 | INFO | MemInfo: { | 20260506 10:41:37.415 | INFO | "gpu" : " 17 GB", | 20260506 10:41:37.415 | INFO | "huge" : "", | 20260506 10:41:37.415 | INFO | "huge_shared" : " 27 GB", | 20260506 10:41:37.415 | INFO | "malloc" : " 14 GB", | 20260506 10:41:37.415 | INFO | "pinned" : " 29 GB" | 20260506 10:41:37.415 | INFO | } | 20260506 10:41:37.415 | INFO | Perf report for the last 5.13944 seconds: | 20260506 10:41:37.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:41:37.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:37.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:37.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:37.416 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 10:41:37.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:37.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:37.416 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:41:37.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:37.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:37.416 | INFO | Uploaded 36GiB to GPU in 1.60358s (22.4498GiB/s) | 20260506 10:41:37.416 | INFO | Downloaded 34GiB to GPU in 1.52116s (22.3514GiB/s) | 20260506 10:41:42.554 | INFO | Analyzing chunk frames = [717824,718336) | 20260506 10:41:42.554 | INFO | MemInfo: { | 20260506 10:41:42.554 | INFO | "gpu" : " 17 GB", | 20260506 10:41:42.554 | INFO | "huge" : "", | 20260506 10:41:42.554 | INFO | "huge_shared" : " 27 GB", | 20260506 10:41:42.554 | INFO | "malloc" : " 14 GB", | 20260506 10:41:42.554 | INFO | "pinned" : " 29 GB" | 20260506 10:41:42.555 | INFO | } | 20260506 10:41:42.555 | INFO | Perf report for the last 5.1391 seconds: | 20260506 10:41:42.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:41:42.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:42.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:42.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:42.555 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:41:42.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:42.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:42.555 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:41:42.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:42.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:42.555 | INFO | Uploaded 36GiB to GPU in 1.60729s (22.398GiB/s) | 20260506 10:41:42.555 | INFO | Downloaded 34GiB to GPU in 1.52488s (22.2969GiB/s) | 20260506 10:41:47.694 | INFO | Analyzing chunk frames = [718336,718848) | 20260506 10:41:47.694 | INFO | MemInfo: { | 20260506 10:41:47.694 | INFO | "gpu" : " 17 GB", | 20260506 10:41:47.694 | INFO | "huge" : "", | 20260506 10:41:47.694 | INFO | "huge_shared" : " 27 GB", | 20260506 10:41:47.694 | INFO | "malloc" : " 14 GB", | 20260506 10:41:47.694 | INFO | "pinned" : " 29 GB" | 20260506 10:41:47.694 | INFO | } | 20260506 10:41:47.694 | INFO | Perf report for the last 5.13946 seconds: | 20260506 10:41:47.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:41:47.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:47.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:47.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:47.694 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 10:41:47.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:47.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:47.694 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:41:47.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:47.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:47.694 | INFO | Uploaded 36GiB to GPU in 1.60814s (22.3861GiB/s) | 20260506 10:41:47.694 | INFO | Downloaded 34GiB to GPU in 1.52526s (22.2913GiB/s) | 20260506 10:41:52.833 | INFO | Analyzing chunk frames = [718848,719360) | 20260506 10:41:52.833 | INFO | MemInfo: { | 20260506 10:41:52.833 | INFO | "gpu" : " 17 GB", | 20260506 10:41:52.833 | INFO | "huge" : "", | 20260506 10:41:52.833 | INFO | "huge_shared" : " 27 GB", | 20260506 10:41:52.833 | INFO | "malloc" : " 14 GB", | 20260506 10:41:52.833 | INFO | "pinned" : " 29 GB" | 20260506 10:41:52.833 | INFO | } | 20260506 10:41:52.833 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:41:52.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:41:52.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:52.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:52.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:52.833 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:41:52.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:52.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:52.833 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:41:52.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:52.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:52.834 | INFO | Uploaded 36GiB to GPU in 1.60549s (22.423GiB/s) | 20260506 10:41:52.834 | INFO | Downloaded 34GiB to GPU in 1.52217s (22.3365GiB/s) | 20260506 10:41:57.973 | INFO | Analyzing chunk frames = [719360,719872) | 20260506 10:41:57.973 | INFO | MemInfo: { | 20260506 10:41:57.973 | INFO | "gpu" : " 17 GB", | 20260506 10:41:57.973 | INFO | "huge" : "", | 20260506 10:41:57.973 | INFO | "huge_shared" : " 27 GB", | 20260506 10:41:57.973 | INFO | "malloc" : " 14 GB", | 20260506 10:41:57.973 | INFO | "pinned" : " 29 GB" | 20260506 10:41:57.973 | INFO | } | 20260506 10:41:57.973 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:41:57.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:41:57.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:57.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:57.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:57.973 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:41:57.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:57.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:57.973 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:41:57.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:57.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:41:57.973 | INFO | Uploaded 36GiB to GPU in 1.60505s (22.4293GiB/s) | 20260506 10:41:57.973 | INFO | Downloaded 34GiB to GPU in 1.52211s (22.3375GiB/s) | 20260506 10:42:03.112 | INFO | Analyzing chunk frames = [719872,720384) | 20260506 10:42:03.112 | INFO | MemInfo: { | 20260506 10:42:03.112 | INFO | "gpu" : " 17 GB", | 20260506 10:42:03.112 | INFO | "huge" : "", | 20260506 10:42:03.112 | INFO | "huge_shared" : " 27 GB", | 20260506 10:42:03.112 | INFO | "malloc" : " 14 GB", | 20260506 10:42:03.112 | INFO | "pinned" : " 29 GB" | 20260506 10:42:03.112 | INFO | } | 20260506 10:42:03.112 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:42:03.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:42:03.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:03.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:03.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:03.112 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:42:03.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:03.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:03.112 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:42:03.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:03.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:03.113 | INFO | Uploaded 36GiB to GPU in 1.60658s (22.4078GiB/s) | 20260506 10:42:03.113 | INFO | Downloaded 34GiB to GPU in 1.52483s (22.2975GiB/s) | 20260506 10:42:08.251 | INFO | Analyzing chunk frames = [720384,720896) | 20260506 10:42:08.251 | INFO | MemInfo: { | 20260506 10:42:08.251 | INFO | "gpu" : " 17 GB", | 20260506 10:42:08.251 | INFO | "huge" : "", | 20260506 10:42:08.251 | INFO | "huge_shared" : " 27 GB", | 20260506 10:42:08.251 | INFO | "malloc" : " 14 GB", | 20260506 10:42:08.251 | INFO | "pinned" : " 29 GB" | 20260506 10:42:08.251 | INFO | } | 20260506 10:42:08.251 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:42:08.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:42:08.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:08.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:08.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:08.251 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:42:08.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:08.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:08.251 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 10:42:08.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:08.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:08.252 | INFO | Uploaded 36GiB to GPU in 1.6058s (22.4188GiB/s) | 20260506 10:42:08.252 | INFO | Downloaded 34GiB to GPU in 1.5242s (22.3068GiB/s) | 20260506 10:42:08.252 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:42:08.252 | INFO | Total mem usage: 16.5 GiB | 20260506 10:42:08.252 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:42:08.252 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -2.86%) | 20260506 10:42:08.252 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:42:08.252 | INFO | High Water Breakdown: | 20260506 10:42:08.252 | INFO | 60.9% : 9.78 GiB ( 96.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:42:08.252 | INFO | 39% : 6.26 GiB ( 93.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:42:08.252 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:42:08.252 | INFO | ------------------------------------------ | 20260506 10:42:08.252 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:42:08.252 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:42:08.252 | INFO | Total mem usage: 26.5 GiB | 20260506 10:42:08.252 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:42:08.252 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:42:08.252 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:42:08.252 | INFO | High Water Breakdown: | 20260506 10:42:08.252 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:42:08.252 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:42:08.252 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:42:08.252 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:42:08.252 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:42:08.252 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:42:08.252 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:42:08.252 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:42:08.252 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:42:08.252 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:42:08.252 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:42:08.252 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:42:08.252 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:42:08.252 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:42:08.252 | INFO | ------------------------------------------ | 20260506 10:42:08.252 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:42:08.252 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:42:08.252 | INFO | Total mem usage: 25.6 GiB | 20260506 10:42:08.252 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:42:08.252 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:42:08.252 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:42:08.252 | INFO | High Water Breakdown: | 20260506 10:42:08.252 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:42:08.252 | INFO | ------------------------------------------ | 20260506 10:42:08.252 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:42:08.252 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:42:08.253 | INFO | Total mem usage: 15.7 GiB | 20260506 10:42:08.253 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:42:08.253 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:42:08.253 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:42:08.253 | INFO | High Water Breakdown: | 20260506 10:42:08.253 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:42:08.253 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:42:08.253 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:42:08.253 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:42:08.253 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:42:08.253 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:42:08.253 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:42:08.253 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:42:08.253 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:42:08.253 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:42:08.253 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:42:08.253 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:42:08.253 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:42:08.253 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:42:08.253 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:42:08.253 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:42:08.253 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:42:08.253 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:42:08.253 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:42:08.253 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:42:08.253 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:42:08.253 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:42:08.253 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:42:08.253 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:42:08.253 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:42:08.253 | INFO | ------------------------------------------ | 20260506 10:42:08.253 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:42:08.253 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:42:08.253 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:42:08.253 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:42:08.253 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 10:42:08.253 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 4.62e-03 / 0.00e+00 | 20260506 10:42:08.253 | INFO | PerfReport: 10.96%, Baseline -- 3.30e+00 / -nan, 1.60e-02 / 0.00e+00 | 20260506 10:42:08.253 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260506 10:42:08.253 | INFO | PerfReport: 16.06%, DME -- 3.86e+01 / -nan, 3.02e+00 / 0.00e+00 | 20260506 10:42:08.253 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.72e+01 / -nan, 4.30e-02 / 0.00e+00 | 20260506 10:42:08.253 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.22e+00 / -nan, 1.26e-01 / 0.00e+00 | 20260506 10:42:08.253 | INFO | PerfReport: 3.65%, Metrics -- 1.10e+00 / -nan, 1.70e-01 / 0.00e+00 | 20260506 10:42:08.253 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.23e-01 / -nan, 5.94e-02 / 0.00e+00 | 20260506 10:42:08.253 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:42:08.253 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:42:08.253 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:42:08.253 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 10:42:08.253 | INFO | PerfReport: 18.73%, Upload -- 1.68e+01 / 0.00e+00, 4.93e-01 / 0.00e+00 | 20260506 10:42:08.253 | INFO | PerfReport: 17.99%, Download -- 1.62e+01 / 0.00e+00, 8.53e-01 / 0.00e+00 | 20260506 10:42:08.253 | INFO | PerfReport: 33.49%, Compute -- 3.01e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 10:42:08.253 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:42:08.253 | INFO | PerfReport: 0.00%, LockDownload -- 4.83e-04 / 0.00e+00, 1.23e-04 / 0.00e+00 | 20260506 10:42:08.253 | INFO | PerfReport: 29.21%, LockCompute -- 2.62e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 10:42:08.253 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 9.74e-03 / 0.00e+00 | 20260506 10:42:08.349 | INFO | Starting a round of Baz IO | 20260506 10:42:11.256 | INFO | Populating RTMetrics report | 20260506 10:42:11.257 | INFO | RTMetrics report populated | 20260506 10:42:11.257 | INFO | Opening tmp RTMetrics json file | 20260506 10:42:11.262 | INFO | Serializing RTMetrics to json | 20260506 10:42:11.285 | INFO | Serialization done, swapping tmp file now... | 20260506 10:42:11.290 | INFO | RTMetrics done | 20260506 10:42:11.701 | INFO | Finished a round of Baz IO: Wrote 2.009 GiB in 3.352 seconds | 20260506 10:42:11.701 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.80% / 0.05% / 53.15% | 20260506 10:42:11.701 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.37822% / 8.74201e-05% / 3.97364e-05% | 20260506 10:42:11.701 | INFO | Baz IO overall: | 20260506 10:42:11.701 | INFO | BytesWritten : 330798865454 | 20260506 10:42:11.702 | INFO | NumEvents : 159180545261 | 20260506 10:42:13.391 | INFO | Analyzing chunk frames = [720896,721408) | 20260506 10:42:13.391 | INFO | MemInfo: { | 20260506 10:42:13.391 | INFO | "gpu" : " 17 GB", | 20260506 10:42:13.391 | INFO | "huge" : "", | 20260506 10:42:13.391 | INFO | "huge_shared" : " 27 GB", | 20260506 10:42:13.391 | INFO | "malloc" : " 14 GB", | 20260506 10:42:13.391 | INFO | "pinned" : " 29 GB" | 20260506 10:42:13.391 | INFO | } | 20260506 10:42:13.391 | INFO | Perf report for the last 5.13931 seconds: | 20260506 10:42:13.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:42:13.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:13.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:13.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:13.391 | INFO | ANALYSIS : 57.5%, 2.8, 85.3ms, 0 | 20260506 10:42:13.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:13.391 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 10:42:13.391 | INFO | PRE_HQ : 56.4%, 2.1, 60.2ms, 0 | 20260506 10:42:13.391 | INFO | BAZWRITER : 64.8%, 4.8, 159.4ms, 0 | 20260506 10:42:13.391 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 10:42:13.391 | INFO | Uploaded 36GiB to GPU in 1.67732s (21.4628GiB/s) | 20260506 10:42:13.391 | INFO | Downloaded 38GiB to GPU in 1.7429s (21.8027GiB/s) | 20260506 10:42:18.530 | INFO | Analyzing chunk frames = [721408,721920) | 20260506 10:42:18.530 | INFO | MemInfo: { | 20260506 10:42:18.530 | INFO | "gpu" : " 17 GB", | 20260506 10:42:18.530 | INFO | "huge" : "", | 20260506 10:42:18.530 | INFO | "huge_shared" : " 27 GB", | 20260506 10:42:18.530 | INFO | "malloc" : " 14 GB", | 20260506 10:42:18.530 | INFO | "pinned" : " 29 GB" | 20260506 10:42:18.530 | INFO | } | 20260506 10:42:18.530 | INFO | Perf report for the last 5.13948 seconds: | 20260506 10:42:18.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:42:18.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:18.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:18.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:18.530 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 10:42:18.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:18.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:18.530 | INFO | PRE_HQ : 29.2%, 1.0, 15.3ms, 0 | 20260506 10:42:18.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:18.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:18.530 | INFO | Uploaded 36GiB to GPU in 1.60825s (22.3846GiB/s) | 20260506 10:42:18.531 | INFO | Downloaded 34GiB to GPU in 1.52525s (22.2914GiB/s) | 20260506 10:42:23.669 | INFO | Analyzing chunk frames = [721920,722432) | 20260506 10:42:23.669 | INFO | MemInfo: { | 20260506 10:42:23.669 | INFO | "gpu" : " 17 GB", | 20260506 10:42:23.669 | INFO | "huge" : "", | 20260506 10:42:23.669 | INFO | "huge_shared" : " 27 GB", | 20260506 10:42:23.669 | INFO | "malloc" : " 14 GB", | 20260506 10:42:23.669 | INFO | "pinned" : " 29 GB" | 20260506 10:42:23.669 | INFO | } | 20260506 10:42:23.669 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:42:23.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:42:23.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:23.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:23.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:23.669 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260506 10:42:23.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:23.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:23.670 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:42:23.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:23.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:23.670 | INFO | Uploaded 36GiB to GPU in 1.60778s (22.3911GiB/s) | 20260506 10:42:23.670 | INFO | Downloaded 34GiB to GPU in 1.52621s (22.2774GiB/s) | 20260506 10:42:28.809 | INFO | Analyzing chunk frames = [722432,722944) | 20260506 10:42:28.809 | INFO | MemInfo: { | 20260506 10:42:28.809 | INFO | "gpu" : " 17 GB", | 20260506 10:42:28.809 | INFO | "huge" : "", | 20260506 10:42:28.809 | INFO | "huge_shared" : " 27 GB", | 20260506 10:42:28.809 | INFO | "malloc" : " 14 GB", | 20260506 10:42:28.809 | INFO | "pinned" : " 29 GB" | 20260506 10:42:28.809 | INFO | } | 20260506 10:42:28.809 | INFO | Perf report for the last 5.13929 seconds: | 20260506 10:42:28.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:42:28.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:28.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:28.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:28.809 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 10:42:28.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:28.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:28.809 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:42:28.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:28.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:28.809 | INFO | Uploaded 36GiB to GPU in 1.60724s (22.3986GiB/s) | 20260506 10:42:28.809 | INFO | Downloaded 34GiB to GPU in 1.52423s (22.3063GiB/s) | 20260506 10:42:33.948 | INFO | Analyzing chunk frames = [722944,723456) | 20260506 10:42:33.948 | INFO | MemInfo: { | 20260506 10:42:33.948 | INFO | "gpu" : " 17 GB", | 20260506 10:42:33.948 | INFO | "huge" : "", | 20260506 10:42:33.948 | INFO | "huge_shared" : " 27 GB", | 20260506 10:42:33.948 | INFO | "malloc" : " 14 GB", | 20260506 10:42:33.948 | INFO | "pinned" : " 29 GB" | 20260506 10:42:33.948 | INFO | } | 20260506 10:42:33.948 | INFO | Perf report for the last 5.13946 seconds: | 20260506 10:42:33.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:42:33.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:33.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:33.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:33.948 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:42:33.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:33.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:33.948 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:42:33.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:33.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:33.949 | INFO | Uploaded 36GiB to GPU in 1.60624s (22.4126GiB/s) | 20260506 10:42:33.949 | INFO | Downloaded 34GiB to GPU in 1.52317s (22.3219GiB/s) | 20260506 10:42:39.087 | INFO | Analyzing chunk frames = [723456,723968) | 20260506 10:42:39.087 | INFO | MemInfo: { | 20260506 10:42:39.087 | INFO | "gpu" : " 17 GB", | 20260506 10:42:39.087 | INFO | "huge" : "", | 20260506 10:42:39.087 | INFO | "huge_shared" : " 27 GB", | 20260506 10:42:39.087 | INFO | "malloc" : " 14 GB", | 20260506 10:42:39.088 | INFO | "pinned" : " 29 GB" | 20260506 10:42:39.088 | INFO | } | 20260506 10:42:39.088 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:42:39.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:42:39.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:39.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:39.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:39.088 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:42:39.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:39.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:39.088 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:42:39.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:39.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:39.088 | INFO | Uploaded 36GiB to GPU in 1.60575s (22.4194GiB/s) | 20260506 10:42:39.088 | INFO | Downloaded 34GiB to GPU in 1.52266s (22.3293GiB/s) | 20260506 10:42:44.227 | INFO | Analyzing chunk frames = [723968,724480) | 20260506 10:42:44.227 | INFO | MemInfo: { | 20260506 10:42:44.227 | INFO | "gpu" : " 17 GB", | 20260506 10:42:44.227 | INFO | "huge" : "", | 20260506 10:42:44.227 | INFO | "huge_shared" : " 27 GB", | 20260506 10:42:44.227 | INFO | "malloc" : " 14 GB", | 20260506 10:42:44.227 | INFO | "pinned" : " 29 GB" | 20260506 10:42:44.227 | INFO | } | 20260506 10:42:44.227 | INFO | Perf report for the last 5.13946 seconds: | 20260506 10:42:44.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:42:44.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:44.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:44.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:44.227 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:42:44.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:44.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:44.227 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:42:44.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:44.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:44.228 | INFO | Uploaded 36GiB to GPU in 1.60613s (22.4141GiB/s) | 20260506 10:42:44.228 | INFO | Downloaded 34GiB to GPU in 1.52391s (22.311GiB/s) | 20260506 10:42:49.366 | INFO | Analyzing chunk frames = [724480,724992) | 20260506 10:42:49.366 | INFO | MemInfo: { | 20260506 10:42:49.366 | INFO | "gpu" : " 17 GB", | 20260506 10:42:49.366 | INFO | "huge" : "", | 20260506 10:42:49.366 | INFO | "huge_shared" : " 27 GB", | 20260506 10:42:49.366 | INFO | "malloc" : " 14 GB", | 20260506 10:42:49.366 | INFO | "pinned" : " 29 GB" | 20260506 10:42:49.366 | INFO | } | 20260506 10:42:49.366 | INFO | Perf report for the last 5.13919 seconds: | 20260506 10:42:49.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:42:49.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:49.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:49.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:49.366 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:42:49.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:49.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:49.366 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 10:42:49.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:49.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:49.367 | INFO | Uploaded 36GiB to GPU in 1.60595s (22.4167GiB/s) | 20260506 10:42:49.367 | INFO | Downloaded 34GiB to GPU in 1.52321s (22.3212GiB/s) | 20260506 10:42:49.486 | INFO | Starting a round of Baz IO | 20260506 10:42:52.371 | INFO | Populating RTMetrics report | 20260506 10:42:52.371 | INFO | RTMetrics report populated | 20260506 10:42:52.371 | INFO | Opening tmp RTMetrics json file | 20260506 10:42:52.372 | INFO | Serializing RTMetrics to json | 20260506 10:42:52.378 | INFO | Serialization done, swapping tmp file now... | 20260506 10:42:52.383 | INFO | RTMetrics done | 20260506 10:42:52.923 | INFO | Finished a round of Baz IO: Wrote 2.008 GiB in 3.436 seconds | 20260506 10:42:52.923 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.75% / 0.05% / 53.20% | 20260506 10:42:52.923 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.3434% / 0.000123183% / 4.37101e-05% | 20260506 10:42:52.923 | INFO | Baz IO overall: | 20260506 10:42:52.923 | INFO | BytesWritten : 332955095557 | 20260506 10:42:52.923 | INFO | NumEvents : 160227127849 | 20260506 10:42:54.505 | INFO | Analyzing chunk frames = [724992,725504) | 20260506 10:42:54.505 | INFO | MemInfo: { | 20260506 10:42:54.505 | INFO | "gpu" : " 17 GB", | 20260506 10:42:54.505 | INFO | "huge" : "", | 20260506 10:42:54.505 | INFO | "huge_shared" : " 27 GB", | 20260506 10:42:54.505 | INFO | "malloc" : " 14 GB", | 20260506 10:42:54.505 | INFO | "pinned" : " 29 GB" | 20260506 10:42:54.505 | INFO | } | 20260506 10:42:54.506 | INFO | Perf report for the last 5.13927 seconds: | 20260506 10:42:54.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:42:54.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:54.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:54.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:54.506 | INFO | ANALYSIS : 57.4%, 2.8, 85.3ms, 0 | 20260506 10:42:54.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:54.506 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 10:42:54.506 | INFO | PRE_HQ : 56.6%, 2.1, 60.5ms, 0 | 20260506 10:42:54.506 | INFO | BAZWRITER : 66.9%, 4.8, 164.3ms, 0 | 20260506 10:42:54.506 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 10:42:54.506 | INFO | Uploaded 36GiB to GPU in 1.67516s (21.4905GiB/s) | 20260506 10:42:54.506 | INFO | Downloaded 38GiB to GPU in 1.74037s (21.8344GiB/s) | 20260506 10:42:59.645 | INFO | Analyzing chunk frames = [725504,726016) | 20260506 10:42:59.645 | INFO | MemInfo: { | 20260506 10:42:59.645 | INFO | "gpu" : " 17 GB", | 20260506 10:42:59.645 | INFO | "huge" : "", | 20260506 10:42:59.645 | INFO | "huge_shared" : " 27 GB", | 20260506 10:42:59.645 | INFO | "malloc" : " 14 GB", | 20260506 10:42:59.645 | INFO | "pinned" : " 29 GB" | 20260506 10:42:59.645 | INFO | } | 20260506 10:42:59.645 | INFO | Perf report for the last 5.13958 seconds: | 20260506 10:42:59.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:42:59.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:59.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:59.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:59.645 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:42:59.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:59.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:59.645 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:42:59.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:59.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:42:59.645 | INFO | Uploaded 36GiB to GPU in 1.60298s (22.4581GiB/s) | 20260506 10:42:59.645 | INFO | Downloaded 34GiB to GPU in 1.52049s (22.3612GiB/s) | 20260506 10:43:04.784 | INFO | Analyzing chunk frames = [726016,726528) | 20260506 10:43:04.784 | INFO | MemInfo: { | 20260506 10:43:04.784 | INFO | "gpu" : " 17 GB", | 20260506 10:43:04.784 | INFO | "huge" : "", | 20260506 10:43:04.784 | INFO | "huge_shared" : " 27 GB", | 20260506 10:43:04.784 | INFO | "malloc" : " 14 GB", | 20260506 10:43:04.784 | INFO | "pinned" : " 29 GB" | 20260506 10:43:04.784 | INFO | } | 20260506 10:43:04.784 | INFO | Perf report for the last 5.13914 seconds: | 20260506 10:43:04.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:43:04.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:04.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:04.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:04.784 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260506 10:43:04.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:04.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:04.784 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:43:04.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:04.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:04.785 | INFO | Uploaded 36GiB to GPU in 1.6076s (22.3936GiB/s) | 20260506 10:43:04.785 | INFO | Downloaded 34GiB to GPU in 1.52608s (22.2793GiB/s) | 20260506 10:43:09.923 | INFO | Analyzing chunk frames = [726528,727040) | 20260506 10:43:09.924 | INFO | MemInfo: { | 20260506 10:43:09.924 | INFO | "gpu" : " 17 GB", | 20260506 10:43:09.924 | INFO | "huge" : "", | 20260506 10:43:09.924 | INFO | "huge_shared" : " 27 GB", | 20260506 10:43:09.924 | INFO | "malloc" : " 14 GB", | 20260506 10:43:09.924 | INFO | "pinned" : " 29 GB" | 20260506 10:43:09.924 | INFO | } | 20260506 10:43:09.924 | INFO | Perf report for the last 5.13948 seconds: | 20260506 10:43:09.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:43:09.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:09.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:09.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:09.924 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 10:43:09.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:09.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:09.924 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:43:09.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:09.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:09.924 | INFO | Uploaded 36GiB to GPU in 1.60671s (22.406GiB/s) | 20260506 10:43:09.924 | INFO | Downloaded 34GiB to GPU in 1.52433s (22.3049GiB/s) | 20260506 10:43:15.063 | INFO | Analyzing chunk frames = [727040,727552) | 20260506 10:43:15.063 | INFO | MemInfo: { | 20260506 10:43:15.063 | INFO | "gpu" : " 17 GB", | 20260506 10:43:15.063 | INFO | "huge" : "", | 20260506 10:43:15.063 | INFO | "huge_shared" : " 27 GB", | 20260506 10:43:15.063 | INFO | "malloc" : " 14 GB", | 20260506 10:43:15.063 | INFO | "pinned" : " 29 GB" | 20260506 10:43:15.063 | INFO | } | 20260506 10:43:15.063 | INFO | Perf report for the last 5.13917 seconds: | 20260506 10:43:15.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:43:15.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:15.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:15.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:15.063 | INFO | ANALYSIS : 56.3%, 3.0, 87.7ms, 0 | 20260506 10:43:15.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:15.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:15.063 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:43:15.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:15.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:15.064 | INFO | Uploaded 36GiB to GPU in 1.60537s (22.4248GiB/s) | 20260506 10:43:15.064 | INFO | Downloaded 34GiB to GPU in 1.52247s (22.3322GiB/s) | 20260506 10:43:20.202 | INFO | Analyzing chunk frames = [727552,728064) | 20260506 10:43:20.202 | INFO | MemInfo: { | 20260506 10:43:20.202 | INFO | "gpu" : " 17 GB", | 20260506 10:43:20.202 | INFO | "huge" : "", | 20260506 10:43:20.202 | INFO | "huge_shared" : " 27 GB", | 20260506 10:43:20.202 | INFO | "malloc" : " 14 GB", | 20260506 10:43:20.202 | INFO | "pinned" : " 29 GB" | 20260506 10:43:20.202 | INFO | } | 20260506 10:43:20.202 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:43:20.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:43:20.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:20.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:20.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:20.202 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:43:20.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:20.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:20.202 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:43:20.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:20.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:20.203 | INFO | Uploaded 36GiB to GPU in 1.60505s (22.4293GiB/s) | 20260506 10:43:20.203 | INFO | Downloaded 34GiB to GPU in 1.52336s (22.319GiB/s) | 20260506 10:43:25.341 | INFO | Analyzing chunk frames = [728064,728576) | 20260506 10:43:25.341 | INFO | MemInfo: { | 20260506 10:43:25.341 | INFO | "gpu" : " 17 GB", | 20260506 10:43:25.341 | INFO | "huge" : "", | 20260506 10:43:25.341 | INFO | "huge_shared" : " 27 GB", | 20260506 10:43:25.341 | INFO | "malloc" : " 14 GB", | 20260506 10:43:25.342 | INFO | "pinned" : " 29 GB" | 20260506 10:43:25.342 | INFO | } | 20260506 10:43:25.342 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:43:25.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:43:25.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:25.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:25.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:25.342 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:43:25.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:25.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:25.342 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:43:25.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:25.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:25.342 | INFO | Uploaded 36GiB to GPU in 1.60574s (22.4196GiB/s) | 20260506 10:43:25.342 | INFO | Downloaded 34GiB to GPU in 1.5241s (22.3083GiB/s) | 20260506 10:43:30.481 | INFO | Analyzing chunk frames = [728576,729088) | 20260506 10:43:30.481 | INFO | MemInfo: { | 20260506 10:43:30.481 | INFO | "gpu" : " 17 GB", | 20260506 10:43:30.481 | INFO | "huge" : "", | 20260506 10:43:30.481 | INFO | "huge_shared" : " 27 GB", | 20260506 10:43:30.481 | INFO | "malloc" : " 14 GB", | 20260506 10:43:30.481 | INFO | "pinned" : " 29 GB" | 20260506 10:43:30.481 | INFO | } | 20260506 10:43:30.481 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:43:30.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:43:30.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:30.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:30.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:30.481 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:43:30.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:30.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:30.481 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 10:43:30.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:30.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:30.481 | INFO | Uploaded 36GiB to GPU in 1.6057s (22.4201GiB/s) | 20260506 10:43:30.481 | INFO | Downloaded 34GiB to GPU in 1.52387s (22.3117GiB/s) | 20260506 10:43:30.481 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:43:30.481 | INFO | Total mem usage: 16.5 GiB | 20260506 10:43:30.481 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:43:30.481 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -2.88%) | 20260506 10:43:30.481 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:43:30.482 | INFO | High Water Breakdown: | 20260506 10:43:30.482 | INFO | 60.9% : 9.77 GiB ( 96.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:43:30.482 | INFO | 39% : 6.26 GiB ( 93.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:43:30.482 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:43:30.482 | INFO | ------------------------------------------ | 20260506 10:43:30.482 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:43:30.482 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:43:30.482 | INFO | Total mem usage: 26.5 GiB | 20260506 10:43:30.482 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:43:30.482 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:43:30.482 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:43:30.482 | INFO | High Water Breakdown: | 20260506 10:43:30.482 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:43:30.482 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:43:30.482 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:43:30.482 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:43:30.482 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:43:30.482 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:43:30.482 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:43:30.482 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:43:30.482 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:43:30.482 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:43:30.482 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:43:30.482 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:43:30.482 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:43:30.482 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:43:30.482 | INFO | ------------------------------------------ | 20260506 10:43:30.482 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:43:30.482 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:43:30.482 | INFO | Total mem usage: 25.6 GiB | 20260506 10:43:30.482 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:43:30.482 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:43:30.482 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:43:30.482 | INFO | High Water Breakdown: | 20260506 10:43:30.482 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:43:30.482 | INFO | ------------------------------------------ | 20260506 10:43:30.482 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:43:30.482 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:43:30.482 | INFO | Total mem usage: 15.7 GiB | 20260506 10:43:30.482 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:43:30.482 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:43:30.482 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:43:30.482 | INFO | High Water Breakdown: | 20260506 10:43:30.482 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:43:30.482 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:43:30.482 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:43:30.482 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:43:30.482 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:43:30.482 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:43:30.482 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:43:30.482 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:43:30.482 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:43:30.482 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:43:30.482 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:43:30.482 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:43:30.482 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:43:30.482 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:43:30.482 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:43:30.482 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:43:30.482 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:43:30.482 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:43:30.482 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:43:30.482 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:43:30.482 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:43:30.482 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:43:30.482 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:43:30.482 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:43:30.482 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:43:30.482 | INFO | ------------------------------------------ | 20260506 10:43:30.482 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:43:30.482 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:43:30.482 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:43:30.482 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:43:30.482 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 10:43:30.482 | INFO | PerfReport: 0.30%, AggregateStats -- 9.04e-02 / -nan, 1.52e-01 / 0.00e+00 | 20260506 10:43:30.482 | INFO | PerfReport: 10.96%, Baseline -- 3.30e+00 / -nan, 1.70e-02 / 0.00e+00 | 20260506 10:43:30.482 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.21e-02 / 0.00e+00 | 20260506 10:43:30.482 | INFO | PerfReport: 16.06%, DME -- 3.86e+01 / -nan, 3.06e+00 / 0.00e+00 | 20260506 10:43:30.482 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.72e+01 / -nan, 4.20e-02 / 0.00e+00 | 20260506 10:43:30.482 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.21e+00 / -nan, 1.27e-01 / 0.00e+00 | 20260506 10:43:30.482 | INFO | PerfReport: 3.64%, Metrics -- 1.10e+00 / -nan, 1.70e-01 / 0.00e+00 | 20260506 10:43:30.482 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.22e-01 / -nan, 5.91e-02 / 0.00e+00 | 20260506 10:43:30.482 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:43:30.482 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:43:30.482 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:43:30.482 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 10:43:30.482 | INFO | PerfReport: 18.74%, Upload -- 1.68e+01 / 0.00e+00, 5.09e-01 / 0.00e+00 | 20260506 10:43:30.482 | INFO | PerfReport: 18.00%, Download -- 1.62e+01 / 0.00e+00, 8.57e-01 / 0.00e+00 | 20260506 10:43:30.482 | INFO | PerfReport: 33.49%, Compute -- 3.01e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 10:43:30.482 | INFO | PerfReport: 0.57%, LockUpload -- 5.08e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260506 10:43:30.482 | INFO | PerfReport: 0.00%, LockDownload -- 5.01e-04 / 0.00e+00, 2.01e-04 / 0.00e+00 | 20260506 10:43:30.482 | INFO | PerfReport: 29.19%, LockCompute -- 2.62e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 10:43:30.482 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.40e-02 / 0.00e+00 | 20260506 10:43:30.578 | INFO | Starting a round of Baz IO | 20260506 10:43:33.488 | INFO | Populating RTMetrics report | 20260506 10:43:33.489 | INFO | RTMetrics report populated | 20260506 10:43:33.489 | INFO | Opening tmp RTMetrics json file | 20260506 10:43:33.534 | INFO | Serializing RTMetrics to json | 20260506 10:43:33.544 | INFO | Serialization done, swapping tmp file now... | 20260506 10:43:33.554 | INFO | RTMetrics done | 20260506 10:43:33.685 | INFO | Finished a round of Baz IO: Wrote 2.01 GiB in 3.106 seconds | 20260506 10:43:33.685 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.71% / 0.05% / 53.25% | 20260506 10:43:33.685 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.32313% / 0.000107288% / 3.97364e-05% | 20260506 10:43:33.685 | INFO | Baz IO overall: | 20260506 10:43:33.685 | INFO | BytesWritten : 335113249895 | 20260506 10:43:33.685 | INFO | NumEvents : 161274466265 | 20260506 10:43:35.620 | INFO | Analyzing chunk frames = [729088,729600) | 20260506 10:43:35.620 | INFO | MemInfo: { | 20260506 10:43:35.620 | INFO | "gpu" : " 17 GB", | 20260506 10:43:35.620 | INFO | "huge" : "", | 20260506 10:43:35.620 | INFO | "huge_shared" : " 27 GB", | 20260506 10:43:35.620 | INFO | "malloc" : " 14 GB", | 20260506 10:43:35.620 | INFO | "pinned" : " 29 GB" | 20260506 10:43:35.620 | INFO | } | 20260506 10:43:35.620 | INFO | Perf report for the last 5.13911 seconds: | 20260506 10:43:35.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:43:35.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:35.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:35.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:35.620 | INFO | ANALYSIS : 57.5%, 2.8, 85.2ms, 0 | 20260506 10:43:35.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:35.620 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260506 10:43:35.620 | INFO | PRE_HQ : 56.6%, 2.1, 60.6ms, 0 | 20260506 10:43:35.620 | INFO | BAZWRITER : 59.9%, 4.8, 147.6ms, 0 | 20260506 10:43:35.620 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 10:43:35.620 | INFO | Uploaded 36GiB to GPU in 1.67825s (21.451GiB/s) | 20260506 10:43:35.621 | INFO | Downloaded 38GiB to GPU in 1.74364s (21.7935GiB/s) | 20260506 10:43:40.759 | INFO | Analyzing chunk frames = [729600,730112) | 20260506 10:43:40.759 | INFO | MemInfo: { | 20260506 10:43:40.759 | INFO | "gpu" : " 17 GB", | 20260506 10:43:40.759 | INFO | "huge" : "", | 20260506 10:43:40.759 | INFO | "huge_shared" : " 27 GB", | 20260506 10:43:40.759 | INFO | "malloc" : " 14 GB", | 20260506 10:43:40.760 | INFO | "pinned" : " 29 GB" | 20260506 10:43:40.760 | INFO | } | 20260506 10:43:40.760 | INFO | Perf report for the last 5.13945 seconds: | 20260506 10:43:40.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:43:40.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:40.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:40.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:40.760 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:43:40.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:40.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:40.760 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 10:43:40.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:40.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:40.760 | INFO | Uploaded 36GiB to GPU in 1.60376s (22.4472GiB/s) | 20260506 10:43:40.760 | INFO | Downloaded 34GiB to GPU in 1.52127s (22.3498GiB/s) | 20260506 10:43:45.899 | INFO | Analyzing chunk frames = [730112,730624) | 20260506 10:43:45.899 | INFO | MemInfo: { | 20260506 10:43:45.899 | INFO | "gpu" : " 17 GB", | 20260506 10:43:45.899 | INFO | "huge" : "", | 20260506 10:43:45.899 | INFO | "huge_shared" : " 27 GB", | 20260506 10:43:45.899 | INFO | "malloc" : " 14 GB", | 20260506 10:43:45.899 | INFO | "pinned" : " 29 GB" | 20260506 10:43:45.899 | INFO | } | 20260506 10:43:45.899 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:43:45.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:43:45.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:45.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:45.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:45.899 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:43:45.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:45.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:45.899 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:43:45.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:45.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:45.899 | INFO | Uploaded 36GiB to GPU in 1.60722s (22.3989GiB/s) | 20260506 10:43:45.899 | INFO | Downloaded 34GiB to GPU in 1.52502s (22.2949GiB/s) | 20260506 10:43:51.038 | INFO | Analyzing chunk frames = [730624,731136) | 20260506 10:43:51.038 | INFO | MemInfo: { | 20260506 10:43:51.038 | INFO | "gpu" : " 17 GB", | 20260506 10:43:51.038 | INFO | "huge" : "", | 20260506 10:43:51.038 | INFO | "huge_shared" : " 27 GB", | 20260506 10:43:51.038 | INFO | "malloc" : " 14 GB", | 20260506 10:43:51.038 | INFO | "pinned" : " 29 GB" | 20260506 10:43:51.038 | INFO | } | 20260506 10:43:51.038 | INFO | Perf report for the last 5.13922 seconds: | 20260506 10:43:51.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:43:51.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:51.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:51.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:51.038 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260506 10:43:51.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:51.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:51.038 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:43:51.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:51.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:51.039 | INFO | Uploaded 36GiB to GPU in 1.60725s (22.3986GiB/s) | 20260506 10:43:51.039 | INFO | Downloaded 34GiB to GPU in 1.52481s (22.2978GiB/s) | 20260506 10:43:56.178 | INFO | Analyzing chunk frames = [731136,731648) | 20260506 10:43:56.178 | INFO | MemInfo: { | 20260506 10:43:56.178 | INFO | "gpu" : " 17 GB", | 20260506 10:43:56.178 | INFO | "huge" : "", | 20260506 10:43:56.178 | INFO | "huge_shared" : " 27 GB", | 20260506 10:43:56.178 | INFO | "malloc" : " 14 GB", | 20260506 10:43:56.178 | INFO | "pinned" : " 29 GB" | 20260506 10:43:56.178 | INFO | } | 20260506 10:43:56.178 | INFO | Perf report for the last 5.13955 seconds: | 20260506 10:43:56.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:43:56.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:56.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:56.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:56.178 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:43:56.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:56.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:56.178 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:43:56.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:56.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:43:56.178 | INFO | Uploaded 36GiB to GPU in 1.6055s (22.4229GiB/s) | 20260506 10:43:56.178 | INFO | Downloaded 34GiB to GPU in 1.52256s (22.3309GiB/s) | 20260506 10:44:01.317 | INFO | Analyzing chunk frames = [731648,732160) | 20260506 10:44:01.317 | INFO | MemInfo: { | 20260506 10:44:01.317 | INFO | "gpu" : " 17 GB", | 20260506 10:44:01.317 | INFO | "huge" : "", | 20260506 10:44:01.317 | INFO | "huge_shared" : " 27 GB", | 20260506 10:44:01.317 | INFO | "malloc" : " 14 GB", | 20260506 10:44:01.317 | INFO | "pinned" : " 29 GB" | 20260506 10:44:01.317 | INFO | } | 20260506 10:44:01.317 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:44:01.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:44:01.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:01.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:01.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:01.317 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 10:44:01.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:01.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:01.317 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:44:01.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:01.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:01.318 | INFO | Uploaded 36GiB to GPU in 1.60645s (22.4097GiB/s) | 20260506 10:44:01.318 | INFO | Downloaded 34GiB to GPU in 1.52316s (22.322GiB/s) | 20260506 10:44:06.456 | INFO | Analyzing chunk frames = [732160,732672) | 20260506 10:44:06.456 | INFO | MemInfo: { | 20260506 10:44:06.456 | INFO | "gpu" : " 17 GB", | 20260506 10:44:06.456 | INFO | "huge" : "", | 20260506 10:44:06.456 | INFO | "huge_shared" : " 27 GB", | 20260506 10:44:06.456 | INFO | "malloc" : " 14 GB", | 20260506 10:44:06.456 | INFO | "pinned" : " 29 GB" | 20260506 10:44:06.456 | INFO | } | 20260506 10:44:06.456 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:44:06.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:44:06.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:06.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:06.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:06.457 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:44:06.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:06.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:06.457 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:44:06.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:06.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:06.457 | INFO | Uploaded 36GiB to GPU in 1.60597s (22.4164GiB/s) | 20260506 10:44:06.457 | INFO | Downloaded 34GiB to GPU in 1.52377s (22.3131GiB/s) | 20260506 10:44:11.596 | INFO | Analyzing chunk frames = [732672,733184) | 20260506 10:44:11.596 | INFO | MemInfo: { | 20260506 10:44:11.596 | INFO | "gpu" : " 17 GB", | 20260506 10:44:11.596 | INFO | "huge" : "", | 20260506 10:44:11.596 | INFO | "huge_shared" : " 27 GB", | 20260506 10:44:11.596 | INFO | "malloc" : " 14 GB", | 20260506 10:44:11.596 | INFO | "pinned" : " 29 GB" | 20260506 10:44:11.596 | INFO | } | 20260506 10:44:11.596 | INFO | Perf report for the last 5.13929 seconds: | 20260506 10:44:11.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:44:11.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:11.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:11.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:11.596 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:44:11.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:11.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:11.596 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:44:11.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:11.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:11.596 | INFO | Uploaded 36GiB to GPU in 1.60658s (22.4078GiB/s) | 20260506 10:44:11.596 | INFO | Downloaded 34GiB to GPU in 1.52474s (22.2989GiB/s) | 20260506 10:44:11.693 | INFO | Starting a round of Baz IO | 20260506 10:44:14.614 | INFO | Populating RTMetrics report | 20260506 10:44:14.614 | INFO | RTMetrics report populated | 20260506 10:44:14.614 | INFO | Opening tmp RTMetrics json file | 20260506 10:44:14.619 | INFO | Serializing RTMetrics to json | 20260506 10:44:14.621 | INFO | Serialization done, swapping tmp file now... | 20260506 10:44:14.622 | INFO | RTMetrics done | 20260506 10:44:14.980 | INFO | Finished a round of Baz IO: Wrote 2.013 GiB in 3.286 seconds | 20260506 10:44:14.980 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.66% / 0.05% / 53.29% | 20260506 10:44:14.980 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.31945% / 8.34465e-05% / 3.57628e-05% | 20260506 10:44:14.980 | INFO | Baz IO overall: | 20260506 10:44:14.980 | INFO | BytesWritten : 337274366173 | 20260506 10:44:14.980 | INFO | NumEvents : 162323843274 | 20260506 10:44:16.735 | INFO | Analyzing chunk frames = [733184,733696) | 20260506 10:44:16.735 | INFO | MemInfo: { | 20260506 10:44:16.735 | INFO | "gpu" : " 17 GB", | 20260506 10:44:16.735 | INFO | "huge" : "", | 20260506 10:44:16.735 | INFO | "huge_shared" : " 27 GB", | 20260506 10:44:16.735 | INFO | "malloc" : " 14 GB", | 20260506 10:44:16.735 | INFO | "pinned" : " 29 GB" | 20260506 10:44:16.735 | INFO | } | 20260506 10:44:16.735 | INFO | Perf report for the last 5.13919 seconds: | 20260506 10:44:16.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:44:16.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:16.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:16.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:16.735 | INFO | ANALYSIS : 57.7%, 2.8, 85.4ms, 0 | 20260506 10:44:16.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:16.735 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 10:44:16.735 | INFO | PRE_HQ : 56.9%, 2.1, 61.1ms, 0 | 20260506 10:44:16.735 | INFO | BAZWRITER : 63.6%, 4.8, 156.3ms, 0 | 20260506 10:44:16.735 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 10:44:16.735 | INFO | Uploaded 36GiB to GPU in 1.67254s (21.5242GiB/s) | 20260506 10:44:16.735 | INFO | Downloaded 38GiB to GPU in 1.73868s (21.8556GiB/s) | 20260506 10:44:21.874 | INFO | Analyzing chunk frames = [733696,734208) | 20260506 10:44:21.874 | INFO | MemInfo: { | 20260506 10:44:21.874 | INFO | "gpu" : " 17 GB", | 20260506 10:44:21.874 | INFO | "huge" : "", | 20260506 10:44:21.874 | INFO | "huge_shared" : " 27 GB", | 20260506 10:44:21.874 | INFO | "malloc" : " 14 GB", | 20260506 10:44:21.874 | INFO | "pinned" : " 29 GB" | 20260506 10:44:21.874 | INFO | } | 20260506 10:44:21.874 | INFO | Perf report for the last 5.13945 seconds: | 20260506 10:44:21.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:44:21.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:21.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:21.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:21.874 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:44:21.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:21.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:21.875 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:44:21.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:21.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:21.875 | INFO | Uploaded 36GiB to GPU in 1.60372s (22.4478GiB/s) | 20260506 10:44:21.875 | INFO | Downloaded 34GiB to GPU in 1.52058s (22.3599GiB/s) | 20260506 10:44:27.014 | INFO | Analyzing chunk frames = [734208,734720) | 20260506 10:44:27.014 | INFO | MemInfo: { | 20260506 10:44:27.014 | INFO | "gpu" : " 17 GB", | 20260506 10:44:27.014 | INFO | "huge" : "", | 20260506 10:44:27.014 | INFO | "huge_shared" : " 27 GB", | 20260506 10:44:27.014 | INFO | "malloc" : " 14 GB", | 20260506 10:44:27.014 | INFO | "pinned" : " 29 GB" | 20260506 10:44:27.014 | INFO | } | 20260506 10:44:27.014 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:44:27.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:44:27.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:27.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:27.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:27.014 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:44:27.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:27.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:27.014 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:44:27.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:27.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:27.014 | INFO | Uploaded 36GiB to GPU in 1.60715s (22.3999GiB/s) | 20260506 10:44:27.014 | INFO | Downloaded 34GiB to GPU in 1.52509s (22.2938GiB/s) | 20260506 10:44:32.153 | INFO | Analyzing chunk frames = [734720,735232) | 20260506 10:44:32.153 | INFO | MemInfo: { | 20260506 10:44:32.153 | INFO | "gpu" : " 17 GB", | 20260506 10:44:32.153 | INFO | "huge" : "", | 20260506 10:44:32.153 | INFO | "huge_shared" : " 27 GB", | 20260506 10:44:32.153 | INFO | "malloc" : " 14 GB", | 20260506 10:44:32.153 | INFO | "pinned" : " 29 GB" | 20260506 10:44:32.153 | INFO | } | 20260506 10:44:32.153 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:44:32.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:44:32.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:32.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:32.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:32.153 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260506 10:44:32.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:32.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:32.153 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:44:32.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:32.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:32.153 | INFO | Uploaded 36GiB to GPU in 1.60776s (22.3915GiB/s) | 20260506 10:44:32.153 | INFO | Downloaded 34GiB to GPU in 1.52596s (22.281GiB/s) | 20260506 10:44:37.292 | INFO | Analyzing chunk frames = [735232,735744) | 20260506 10:44:37.292 | INFO | MemInfo: { | 20260506 10:44:37.292 | INFO | "gpu" : " 17 GB", | 20260506 10:44:37.292 | INFO | "huge" : "", | 20260506 10:44:37.292 | INFO | "huge_shared" : " 27 GB", | 20260506 10:44:37.293 | INFO | "malloc" : " 14 GB", | 20260506 10:44:37.293 | INFO | "pinned" : " 29 GB" | 20260506 10:44:37.293 | INFO | } | 20260506 10:44:37.293 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:44:37.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:44:37.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:37.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:37.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:37.293 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:44:37.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:37.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:37.293 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:44:37.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:37.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:37.293 | INFO | Uploaded 36GiB to GPU in 1.60492s (22.431GiB/s) | 20260506 10:44:37.293 | INFO | Downloaded 34GiB to GPU in 1.52239s (22.3333GiB/s) | 20260506 10:44:42.432 | INFO | Analyzing chunk frames = [735744,736256) | 20260506 10:44:42.432 | INFO | MemInfo: { | 20260506 10:44:42.432 | INFO | "gpu" : " 17 GB", | 20260506 10:44:42.432 | INFO | "huge" : "", | 20260506 10:44:42.432 | INFO | "huge_shared" : " 27 GB", | 20260506 10:44:42.432 | INFO | "malloc" : " 14 GB", | 20260506 10:44:42.432 | INFO | "pinned" : " 29 GB" | 20260506 10:44:42.432 | INFO | } | 20260506 10:44:42.432 | INFO | Perf report for the last 5.13915 seconds: | 20260506 10:44:42.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:44:42.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:42.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:42.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:42.432 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 10:44:42.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:42.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:42.432 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:44:42.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:42.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:42.432 | INFO | Uploaded 36GiB to GPU in 1.6057s (22.4202GiB/s) | 20260506 10:44:42.432 | INFO | Downloaded 34GiB to GPU in 1.52297s (22.3248GiB/s) | 20260506 10:44:47.571 | INFO | Analyzing chunk frames = [736256,736768) | 20260506 10:44:47.571 | INFO | MemInfo: { | 20260506 10:44:47.571 | INFO | "gpu" : " 17 GB", | 20260506 10:44:47.571 | INFO | "huge" : "", | 20260506 10:44:47.571 | INFO | "huge_shared" : " 27 GB", | 20260506 10:44:47.571 | INFO | "malloc" : " 14 GB", | 20260506 10:44:47.571 | INFO | "pinned" : " 29 GB" | 20260506 10:44:47.571 | INFO | } | 20260506 10:44:47.571 | INFO | Perf report for the last 5.13953 seconds: | 20260506 10:44:47.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:44:47.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:47.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:47.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:47.571 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:44:47.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:47.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:47.571 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:44:47.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:47.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:47.572 | INFO | Uploaded 36GiB to GPU in 1.60533s (22.4253GiB/s) | 20260506 10:44:47.572 | INFO | Downloaded 34GiB to GPU in 1.52481s (22.2979GiB/s) | 20260506 10:44:52.710 | INFO | Analyzing chunk frames = [736768,737280) | 20260506 10:44:52.710 | INFO | MemInfo: { | 20260506 10:44:52.710 | INFO | "gpu" : " 17 GB", | 20260506 10:44:52.710 | INFO | "huge" : "", | 20260506 10:44:52.711 | INFO | "huge_shared" : " 27 GB", | 20260506 10:44:52.711 | INFO | "malloc" : " 14 GB", | 20260506 10:44:52.711 | INFO | "pinned" : " 29 GB" | 20260506 10:44:52.711 | INFO | } | 20260506 10:44:52.711 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:44:52.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:44:52.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:52.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:52.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:52.711 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 10:44:52.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:52.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:52.711 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:44:52.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:52.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:52.711 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:44:52.711 | INFO | Total mem usage: 16.5 GiB | 20260506 10:44:52.711 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:44:52.711 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -5.67%) | 20260506 10:44:52.711 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:44:52.711 | INFO | High Water Breakdown: | 20260506 10:44:52.711 | INFO | 60.5% : 9.45 GiB ( 93.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:44:52.711 | INFO | 39.4% : 6.16 GiB ( 92% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:44:52.711 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:44:52.711 | INFO | ------------------------------------------ | 20260506 10:44:52.711 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:44:52.711 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:44:52.711 | INFO | Total mem usage: 26.5 GiB | 20260506 10:44:52.711 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:44:52.711 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:44:52.711 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:44:52.711 | INFO | High Water Breakdown: | 20260506 10:44:52.711 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:44:52.711 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:44:52.711 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:44:52.711 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:44:52.711 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:44:52.711 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:44:52.711 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:44:52.711 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:44:52.711 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:44:52.711 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:44:52.711 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:44:52.711 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:44:52.711 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:44:52.711 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:44:52.711 | INFO | ------------------------------------------ | 20260506 10:44:52.711 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:44:52.711 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:44:52.711 | INFO | Total mem usage: 25.6 GiB | 20260506 10:44:52.711 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:44:52.711 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:44:52.711 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:44:52.711 | INFO | High Water Breakdown: | 20260506 10:44:52.711 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:44:52.711 | INFO | ------------------------------------------ | 20260506 10:44:52.711 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:44:52.711 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:44:52.711 | INFO | Total mem usage: 15.7 GiB | 20260506 10:44:52.711 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:44:52.711 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:44:52.711 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:44:52.711 | INFO | High Water Breakdown: | 20260506 10:44:52.711 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:44:52.711 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:44:52.711 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:44:52.711 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:44:52.711 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:44:52.711 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:44:52.711 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:44:52.711 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:44:52.711 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:44:52.711 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:44:52.711 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:44:52.711 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:44:52.712 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:44:52.712 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:44:52.712 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:44:52.712 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:44:52.712 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:44:52.712 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:44:52.711 | INFO | Uploaded 36GiB to GPU in 1.6059s (22.4174GiB/s) | 20260506 10:44:52.712 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:44:52.712 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:44:52.712 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:44:52.712 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:44:52.712 | INFO | Downloaded 34GiB to GPU in 1.52423s (22.3064GiB/s) | 20260506 10:44:52.712 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:44:52.712 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:44:52.712 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:44:52.712 | INFO | ------------------------------------------ | 20260506 10:44:52.712 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:44:52.712 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:44:52.712 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:44:52.712 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:44:52.712 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 10:44:52.712 | INFO | PerfReport: 0.29%, AggregateStats -- 8.64e-02 / -nan, 4.40e-03 / 0.00e+00 | 20260506 10:44:52.712 | INFO | PerfReport: 10.96%, Baseline -- 3.29e+00 / -nan, 1.65e-02 / 0.00e+00 | 20260506 10:44:52.712 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.21e-02 / 0.00e+00 | 20260506 10:44:52.712 | INFO | PerfReport: 16.06%, DME -- 3.86e+01 / -nan, 3.08e+00 / 0.00e+00 | 20260506 10:44:52.712 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.72e+01 / -nan, 4.34e-02 / 0.00e+00 | 20260506 10:44:52.712 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.21e+00 / -nan, 1.27e-01 / 0.00e+00 | 20260506 10:44:52.712 | INFO | PerfReport: 3.64%, Metrics -- 1.10e+00 / -nan, 1.71e-01 / 0.00e+00 | 20260506 10:44:52.712 | INFO | PerfReport: 1.41%, PulseSerializing -- 4.23e-01 / -nan, 5.92e-02 / 0.00e+00 | 20260506 10:44:52.712 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:44:52.712 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:44:52.712 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:44:52.712 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:44:52.712 | INFO | PerfReport: 18.74%, Upload -- 1.68e+01 / 0.00e+00, 4.96e-01 / 0.00e+00 | 20260506 10:44:52.712 | INFO | PerfReport: 18.00%, Download -- 1.62e+01 / 0.00e+00, 8.47e-01 / 0.00e+00 | 20260506 10:44:52.712 | INFO | PerfReport: 33.50%, Compute -- 3.01e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 10:44:52.712 | INFO | PerfReport: 0.57%, LockUpload -- 5.09e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:44:52.712 | INFO | PerfReport: 0.00%, LockDownload -- 4.96e-04 / 0.00e+00, 1.67e-04 / 0.00e+00 | 20260506 10:44:52.712 | INFO | PerfReport: 29.18%, LockCompute -- 2.62e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260506 10:44:52.712 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260506 10:44:52.809 | INFO | Starting a round of Baz IO | 20260506 10:44:55.718 | INFO | Populating RTMetrics report | 20260506 10:44:55.719 | INFO | RTMetrics report populated | 20260506 10:44:55.719 | INFO | Opening tmp RTMetrics json file | 20260506 10:44:55.776 | INFO | Serializing RTMetrics to json | 20260506 10:44:55.801 | INFO | Serialization done, swapping tmp file now... | 20260506 10:44:55.802 | INFO | RTMetrics done | 20260506 10:44:56.320 | INFO | Finished a round of Baz IO: Wrote 2.011 GiB in 3.511 seconds | 20260506 10:44:56.320 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.62% / 0.05% / 53.34% | 20260506 10:44:56.320 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.28141% / 0.000123183% / 3.17891e-05% | 20260506 10:44:56.320 | INFO | Baz IO overall: | 20260506 10:44:56.320 | INFO | BytesWritten : 339434102663 | 20260506 10:44:56.320 | INFO | NumEvents : 163372099195 | 20260506 10:44:57.850 | INFO | Analyzing chunk frames = [737280,737792) | 20260506 10:44:57.850 | INFO | MemInfo: { | 20260506 10:44:57.850 | INFO | "gpu" : " 17 GB", | 20260506 10:44:57.850 | INFO | "huge" : "", | 20260506 10:44:57.850 | INFO | "huge_shared" : " 27 GB", | 20260506 10:44:57.850 | INFO | "malloc" : " 14 GB", | 20260506 10:44:57.850 | INFO | "pinned" : " 29 GB" | 20260506 10:44:57.850 | INFO | } | 20260506 10:44:57.850 | INFO | Perf report for the last 5.1392 seconds: | 20260506 10:44:57.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:44:57.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:57.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:57.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:57.850 | INFO | ANALYSIS : 57.5%, 2.8, 85.4ms, 0 | 20260506 10:44:57.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:44:57.850 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 10:44:57.850 | INFO | PRE_HQ : 56.5%, 2.1, 60.3ms, 0 | 20260506 10:44:57.850 | INFO | BAZWRITER : 68.0%, 4.8, 168.0ms, 0 | 20260506 10:44:57.850 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 10:44:57.850 | INFO | Uploaded 36GiB to GPU in 1.6748s (21.4952GiB/s) | 20260506 10:44:57.850 | INFO | Downloaded 38GiB to GPU in 1.74052s (21.8325GiB/s) | 20260506 10:45:02.989 | INFO | Analyzing chunk frames = [737792,738304) | 20260506 10:45:02.989 | INFO | MemInfo: { | 20260506 10:45:02.989 | INFO | "gpu" : " 17 GB", | 20260506 10:45:02.989 | INFO | "huge" : "", | 20260506 10:45:02.989 | INFO | "huge_shared" : " 27 GB", | 20260506 10:45:02.989 | INFO | "malloc" : " 14 GB", | 20260506 10:45:02.989 | INFO | "pinned" : " 29 GB" | 20260506 10:45:02.989 | INFO | } | 20260506 10:45:02.989 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:45:02.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:45:02.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:02.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:02.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:02.989 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 10:45:02.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:02.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:02.989 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:45:02.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:02.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:02.990 | INFO | Uploaded 36GiB to GPU in 1.60326s (22.4543GiB/s) | 20260506 10:45:02.990 | INFO | Downloaded 34GiB to GPU in 1.52209s (22.3377GiB/s) | 20260506 10:45:08.128 | INFO | Analyzing chunk frames = [738304,738816) | 20260506 10:45:08.128 | INFO | MemInfo: { | 20260506 10:45:08.128 | INFO | "gpu" : " 17 GB", | 20260506 10:45:08.128 | INFO | "huge" : "", | 20260506 10:45:08.128 | INFO | "huge_shared" : " 27 GB", | 20260506 10:45:08.128 | INFO | "malloc" : " 14 GB", | 20260506 10:45:08.128 | INFO | "pinned" : " 29 GB" | 20260506 10:45:08.129 | INFO | } | 20260506 10:45:08.129 | INFO | Perf report for the last 5.13927 seconds: | 20260506 10:45:08.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:45:08.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:08.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:08.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:08.129 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:45:08.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:08.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:08.129 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:45:08.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:08.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:08.129 | INFO | Uploaded 36GiB to GPU in 1.60691s (22.4032GiB/s) | 20260506 10:45:08.129 | INFO | Downloaded 34GiB to GPU in 1.5253s (22.2907GiB/s) | 20260506 10:45:13.268 | INFO | Analyzing chunk frames = [738816,739328) | 20260506 10:45:13.268 | INFO | MemInfo: { | 20260506 10:45:13.268 | INFO | "gpu" : " 17 GB", | 20260506 10:45:13.268 | INFO | "huge" : "", | 20260506 10:45:13.268 | INFO | "huge_shared" : " 27 GB", | 20260506 10:45:13.268 | INFO | "malloc" : " 14 GB", | 20260506 10:45:13.268 | INFO | "pinned" : " 29 GB" | 20260506 10:45:13.268 | INFO | } | 20260506 10:45:13.268 | INFO | Perf report for the last 5.13951 seconds: | 20260506 10:45:13.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:45:13.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:13.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:13.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:13.268 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 10:45:13.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:13.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:13.268 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:45:13.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:13.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:13.268 | INFO | Uploaded 36GiB to GPU in 1.60694s (22.4028GiB/s) | 20260506 10:45:13.268 | INFO | Downloaded 34GiB to GPU in 1.52531s (22.2905GiB/s) | 20260506 10:45:18.407 | INFO | Analyzing chunk frames = [739328,739840) | 20260506 10:45:18.407 | INFO | MemInfo: { | 20260506 10:45:18.407 | INFO | "gpu" : " 17 GB", | 20260506 10:45:18.407 | INFO | "huge" : "", | 20260506 10:45:18.407 | INFO | "huge_shared" : " 27 GB", | 20260506 10:45:18.407 | INFO | "malloc" : " 14 GB", | 20260506 10:45:18.407 | INFO | "pinned" : " 29 GB" | 20260506 10:45:18.407 | INFO | } | 20260506 10:45:18.407 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:45:18.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:45:18.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:18.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:18.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:18.407 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:45:18.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:18.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:18.407 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:45:18.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:18.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:18.408 | INFO | Uploaded 36GiB to GPU in 1.60583s (22.4183GiB/s) | 20260506 10:45:18.408 | INFO | Downloaded 34GiB to GPU in 1.52378s (22.313GiB/s) | 20260506 10:45:23.546 | INFO | Analyzing chunk frames = [739840,740352) | 20260506 10:45:23.546 | INFO | MemInfo: { | 20260506 10:45:23.546 | INFO | "gpu" : " 17 GB", | 20260506 10:45:23.546 | INFO | "huge" : "", | 20260506 10:45:23.547 | INFO | "huge_shared" : " 27 GB", | 20260506 10:45:23.547 | INFO | "malloc" : " 14 GB", | 20260506 10:45:23.547 | INFO | "pinned" : " 29 GB" | 20260506 10:45:23.547 | INFO | } | 20260506 10:45:23.547 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:45:23.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:45:23.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:23.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:23.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:23.547 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:45:23.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:23.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:23.547 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:45:23.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:23.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:23.547 | INFO | Uploaded 36GiB to GPU in 1.60524s (22.4266GiB/s) | 20260506 10:45:23.547 | INFO | Downloaded 34GiB to GPU in 1.52269s (22.3289GiB/s) | 20260506 10:45:28.686 | INFO | Analyzing chunk frames = [740352,740864) | 20260506 10:45:28.686 | INFO | MemInfo: { | 20260506 10:45:28.686 | INFO | "gpu" : " 17 GB", | 20260506 10:45:28.686 | INFO | "huge" : "", | 20260506 10:45:28.686 | INFO | "huge_shared" : " 27 GB", | 20260506 10:45:28.686 | INFO | "malloc" : " 14 GB", | 20260506 10:45:28.686 | INFO | "pinned" : " 29 GB" | 20260506 10:45:28.686 | INFO | } | 20260506 10:45:28.686 | INFO | Perf report for the last 5.13919 seconds: | 20260506 10:45:28.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:45:28.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:28.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:28.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:28.686 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:45:28.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:28.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:28.686 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:45:28.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:28.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:28.686 | INFO | Uploaded 36GiB to GPU in 1.60653s (22.4086GiB/s) | 20260506 10:45:28.686 | INFO | Downloaded 34GiB to GPU in 1.52397s (22.3101GiB/s) | 20260506 10:45:33.825 | INFO | Analyzing chunk frames = [740864,741376) | 20260506 10:45:33.825 | INFO | MemInfo: { | 20260506 10:45:33.825 | INFO | "gpu" : " 17 GB", | 20260506 10:45:33.825 | INFO | "huge" : "", | 20260506 10:45:33.825 | INFO | "huge_shared" : " 27 GB", | 20260506 10:45:33.825 | INFO | "malloc" : " 14 GB", | 20260506 10:45:33.825 | INFO | "pinned" : " 29 GB" | 20260506 10:45:33.825 | INFO | } | 20260506 10:45:33.825 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:45:33.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:45:33.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:33.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:33.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:33.825 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:45:33.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:33.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:33.825 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 10:45:33.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:33.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:33.826 | INFO | Uploaded 36GiB to GPU in 1.60569s (22.4203GiB/s) | 20260506 10:45:33.826 | INFO | Downloaded 34GiB to GPU in 1.52334s (22.3194GiB/s) | 20260506 10:45:33.923 | INFO | Starting a round of Baz IO | 20260506 10:45:36.832 | INFO | Populating RTMetrics report | 20260506 10:45:36.833 | INFO | RTMetrics report populated | 20260506 10:45:36.833 | INFO | Opening tmp RTMetrics json file | 20260506 10:45:36.868 | INFO | Serializing RTMetrics to json | 20260506 10:45:36.871 | INFO | Serialization done, swapping tmp file now... | 20260506 10:45:36.875 | INFO | RTMetrics done | 20260506 10:45:37.260 | INFO | Finished a round of Baz IO: Wrote 2.013 GiB in 3.337 seconds | 20260506 10:45:37.260 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.57% / 0.05% / 53.38% | 20260506 10:45:37.260 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.26279% / 8.74201e-05% / 3.17891e-05% | 20260506 10:45:37.260 | INFO | Baz IO overall: | 20260506 10:45:37.260 | INFO | BytesWritten : 341595375854 | 20260506 10:45:37.260 | INFO | NumEvents : 164421078357 | 20260506 10:45:38.964 | INFO | Analyzing chunk frames = [741376,741888) | 20260506 10:45:38.964 | INFO | MemInfo: { | 20260506 10:45:38.964 | INFO | "gpu" : " 17 GB", | 20260506 10:45:38.964 | INFO | "huge" : "", | 20260506 10:45:38.964 | INFO | "huge_shared" : " 27 GB", | 20260506 10:45:38.964 | INFO | "malloc" : " 14 GB", | 20260506 10:45:38.964 | INFO | "pinned" : " 29 GB" | 20260506 10:45:38.964 | INFO | } | 20260506 10:45:38.964 | INFO | Perf report for the last 5.13914 seconds: | 20260506 10:45:38.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:45:38.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:38.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:38.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:38.965 | INFO | ANALYSIS : 57.5%, 2.8, 85.7ms, 0 | 20260506 10:45:38.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:38.965 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 10:45:38.965 | INFO | PRE_HQ : 56.6%, 2.1, 60.7ms, 0 | 20260506 10:45:38.965 | INFO | BAZWRITER : 64.5%, 4.7, 157.1ms, 0 | 20260506 10:45:38.965 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 10:45:38.965 | INFO | Uploaded 36GiB to GPU in 1.66838s (21.5778GiB/s) | 20260506 10:45:38.965 | INFO | Downloaded 38GiB to GPU in 1.73294s (21.9281GiB/s) | 20260506 10:45:44.104 | INFO | Analyzing chunk frames = [741888,742400) | 20260506 10:45:44.104 | INFO | MemInfo: { | 20260506 10:45:44.104 | INFO | "gpu" : " 17 GB", | 20260506 10:45:44.104 | INFO | "huge" : "", | 20260506 10:45:44.104 | INFO | "huge_shared" : " 27 GB", | 20260506 10:45:44.104 | INFO | "malloc" : " 14 GB", | 20260506 10:45:44.104 | INFO | "pinned" : " 29 GB" | 20260506 10:45:44.104 | INFO | } | 20260506 10:45:44.104 | INFO | Perf report for the last 5.13978 seconds: | 20260506 10:45:44.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:45:44.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:44.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:44.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:44.104 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:45:44.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:44.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:44.104 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:45:44.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:44.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:44.105 | INFO | Uploaded 36GiB to GPU in 1.60358s (22.4498GiB/s) | 20260506 10:45:44.105 | INFO | Downloaded 34GiB to GPU in 1.52024s (22.3649GiB/s) | 20260506 10:45:49.243 | INFO | Analyzing chunk frames = [742400,742912) | 20260506 10:45:49.243 | INFO | MemInfo: { | 20260506 10:45:49.243 | INFO | "gpu" : " 17 GB", | 20260506 10:45:49.243 | INFO | "huge" : "", | 20260506 10:45:49.243 | INFO | "huge_shared" : " 27 GB", | 20260506 10:45:49.243 | INFO | "malloc" : " 14 GB", | 20260506 10:45:49.243 | INFO | "pinned" : " 29 GB" | 20260506 10:45:49.243 | INFO | } | 20260506 10:45:49.243 | INFO | Perf report for the last 5.13911 seconds: | 20260506 10:45:49.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:45:49.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:49.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:49.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:49.243 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260506 10:45:49.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:49.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:49.244 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:45:49.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:49.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:49.244 | INFO | Uploaded 36GiB to GPU in 1.60788s (22.3897GiB/s) | 20260506 10:45:49.244 | INFO | Downloaded 34GiB to GPU in 1.52614s (22.2785GiB/s) | 20260506 10:45:54.382 | INFO | Analyzing chunk frames = [742912,743424) | 20260506 10:45:54.382 | INFO | MemInfo: { | 20260506 10:45:54.382 | INFO | "gpu" : " 17 GB", | 20260506 10:45:54.382 | INFO | "huge" : "", | 20260506 10:45:54.383 | INFO | "huge_shared" : " 27 GB", | 20260506 10:45:54.383 | INFO | "malloc" : " 14 GB", | 20260506 10:45:54.383 | INFO | "pinned" : " 29 GB" | 20260506 10:45:54.383 | INFO | } | 20260506 10:45:54.383 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:45:54.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:45:54.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:54.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:54.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:54.383 | INFO | ANALYSIS : 56.4%, 3.0, 87.5ms, 0 | 20260506 10:45:54.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:54.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:54.383 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:45:54.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:54.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:54.383 | INFO | Uploaded 36GiB to GPU in 1.60732s (22.3975GiB/s) | 20260506 10:45:54.383 | INFO | Downloaded 34GiB to GPU in 1.52492s (22.2963GiB/s) | 20260506 10:45:59.522 | INFO | Analyzing chunk frames = [743424,743936) | 20260506 10:45:59.522 | INFO | MemInfo: { | 20260506 10:45:59.522 | INFO | "gpu" : " 17 GB", | 20260506 10:45:59.522 | INFO | "huge" : "", | 20260506 10:45:59.522 | INFO | "huge_shared" : " 27 GB", | 20260506 10:45:59.522 | INFO | "malloc" : " 14 GB", | 20260506 10:45:59.522 | INFO | "pinned" : " 29 GB" | 20260506 10:45:59.522 | INFO | } | 20260506 10:45:59.522 | INFO | Perf report for the last 5.13963 seconds: | 20260506 10:45:59.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:45:59.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:59.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:59.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:59.522 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:45:59.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:59.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:59.522 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:45:59.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:59.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:45:59.523 | INFO | Uploaded 36GiB to GPU in 1.6061s (22.4145GiB/s) | 20260506 10:45:59.523 | INFO | Downloaded 34GiB to GPU in 1.52272s (22.3284GiB/s) | 20260506 10:46:04.661 | INFO | Analyzing chunk frames = [743936,744448) | 20260506 10:46:04.661 | INFO | MemInfo: { | 20260506 10:46:04.661 | INFO | "gpu" : " 17 GB", | 20260506 10:46:04.661 | INFO | "huge" : "", | 20260506 10:46:04.661 | INFO | "huge_shared" : " 27 GB", | 20260506 10:46:04.661 | INFO | "malloc" : " 14 GB", | 20260506 10:46:04.661 | INFO | "pinned" : " 29 GB" | 20260506 10:46:04.661 | INFO | } | 20260506 10:46:04.661 | INFO | Perf report for the last 5.13904 seconds: | 20260506 10:46:04.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:46:04.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:04.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:04.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:04.661 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:46:04.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:04.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:04.662 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:46:04.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:04.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:04.662 | INFO | Uploaded 36GiB to GPU in 1.60575s (22.4194GiB/s) | 20260506 10:46:04.662 | INFO | Downloaded 34GiB to GPU in 1.52302s (22.3241GiB/s) | 20260506 10:46:09.800 | INFO | Analyzing chunk frames = [744448,744960) | 20260506 10:46:09.800 | INFO | MemInfo: { | 20260506 10:46:09.800 | INFO | "gpu" : " 17 GB", | 20260506 10:46:09.801 | INFO | "huge" : "", | 20260506 10:46:09.801 | INFO | "huge_shared" : " 27 GB", | 20260506 10:46:09.801 | INFO | "malloc" : " 14 GB", | 20260506 10:46:09.801 | INFO | "pinned" : " 29 GB" | 20260506 10:46:09.801 | INFO | } | 20260506 10:46:09.801 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:46:09.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:46:09.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:09.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:09.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:09.801 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 10:46:09.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:09.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:09.801 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:46:09.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:09.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:09.801 | INFO | Uploaded 36GiB to GPU in 1.60628s (22.4121GiB/s) | 20260506 10:46:09.801 | INFO | Downloaded 34GiB to GPU in 1.52343s (22.318GiB/s) | 20260506 10:46:14.940 | INFO | Analyzing chunk frames = [744960,745472) | 20260506 10:46:14.940 | INFO | MemInfo: { | 20260506 10:46:14.940 | INFO | "gpu" : " 17 GB", | 20260506 10:46:14.940 | INFO | "huge" : "", | 20260506 10:46:14.940 | INFO | "huge_shared" : " 27 GB", | 20260506 10:46:14.940 | INFO | "malloc" : " 14 GB", | 20260506 10:46:14.940 | INFO | "pinned" : " 29 GB" | 20260506 10:46:14.940 | INFO | } | 20260506 10:46:14.940 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:46:14.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:46:14.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:14.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:14.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:14.940 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:46:14.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:14.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:14.940 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 10:46:14.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:14.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:14.941 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:46:14.941 | INFO | Total mem usage: 16.5 GiB | 20260506 10:46:14.941 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:46:14.941 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -2.17%) | 20260506 10:46:14.941 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:46:14.941 | INFO | High Water Breakdown: | 20260506 10:46:14.941 | INFO | 61% : 9.86 GiB ( 97.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:46:14.941 | INFO | 38.9% : 6.28 GiB ( 93.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:46:14.941 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:46:14.941 | INFO | ------------------------------------------ | 20260506 10:46:14.941 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:46:14.941 | INFO | Uploaded 36GiB to GPU in 1.60578s (22.4191GiB/s) | 20260506 10:46:14.941 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:46:14.941 | INFO | Total mem usage: 26.5 GiB | 20260506 10:46:14.941 | INFO | Downloaded 34GiB to GPU in 1.52293s (22.3254GiB/s) | 20260506 10:46:14.941 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:46:14.941 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:46:14.941 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:46:14.941 | INFO | High Water Breakdown: | 20260506 10:46:14.941 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:46:14.941 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:46:14.941 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:46:14.941 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:46:14.941 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:46:14.941 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:46:14.941 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:46:14.941 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:46:14.941 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:46:14.941 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:46:14.941 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:46:14.941 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:46:14.941 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:46:14.941 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:46:14.941 | INFO | ------------------------------------------ | 20260506 10:46:14.941 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:46:14.941 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:46:14.941 | INFO | Total mem usage: 25.6 GiB | 20260506 10:46:14.941 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:46:14.941 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:46:14.941 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:46:14.941 | INFO | High Water Breakdown: | 20260506 10:46:14.941 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:46:14.941 | INFO | ------------------------------------------ | 20260506 10:46:14.941 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:46:14.941 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:46:14.941 | INFO | Total mem usage: 15.7 GiB | 20260506 10:46:14.941 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:46:14.941 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:46:14.941 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:46:14.941 | INFO | High Water Breakdown: | 20260506 10:46:14.941 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:46:14.941 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:46:14.941 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:46:14.941 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:46:14.941 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:46:14.941 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:46:14.941 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:46:14.941 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:46:14.941 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:46:14.941 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:46:14.941 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:46:14.941 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:46:14.941 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:46:14.941 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:46:14.941 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:46:14.941 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:46:14.941 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:46:14.941 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:46:14.941 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:46:14.941 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:46:14.941 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:46:14.942 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:46:14.942 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:46:14.942 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:46:14.942 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:46:14.942 | INFO | ------------------------------------------ | 20260506 10:46:14.942 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:46:14.942 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:46:14.942 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:46:14.942 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:46:14.942 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260506 10:46:14.942 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.86e-03 / 0.00e+00 | 20260506 10:46:14.942 | INFO | PerfReport: 10.97%, Baseline -- 3.30e+00 / -nan, 1.71e-02 / 0.00e+00 | 20260506 10:46:14.942 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.15e-02 / 0.00e+00 | 20260506 10:46:14.942 | INFO | PerfReport: 16.06%, DME -- 3.86e+01 / -nan, 3.08e+00 / 0.00e+00 | 20260506 10:46:14.942 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.72e+01 / -nan, 4.31e-02 / 0.00e+00 | 20260506 10:46:14.942 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.21e+00 / -nan, 1.27e-01 / 0.00e+00 | 20260506 10:46:14.942 | INFO | PerfReport: 3.64%, Metrics -- 1.09e+00 / -nan, 1.71e-01 / 0.00e+00 | 20260506 10:46:14.942 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.22e-01 / -nan, 5.91e-02 / 0.00e+00 | 20260506 10:46:14.942 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:46:14.942 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:46:14.942 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:46:14.942 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:46:14.942 | INFO | PerfReport: 18.73%, Upload -- 1.68e+01 / 0.00e+00, 4.87e-01 / 0.00e+00 | 20260506 10:46:14.942 | INFO | PerfReport: 17.99%, Download -- 1.61e+01 / 0.00e+00, 8.38e-01 / 0.00e+00 | 20260506 10:46:14.942 | INFO | PerfReport: 33.48%, Compute -- 3.01e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260506 10:46:14.942 | INFO | PerfReport: 0.57%, LockUpload -- 5.09e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:46:14.942 | INFO | PerfReport: 0.00%, LockDownload -- 4.92e-04 / 0.00e+00, 2.28e-04 / 0.00e+00 | 20260506 10:46:14.942 | INFO | PerfReport: 29.22%, LockCompute -- 2.62e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 10:46:14.942 | INFO | PerfReport: 0.02%, Misc -- 1.38e-02 / 0.00e+00, 2.66e-02 / 0.00e+00 | 20260506 10:46:15.038 | INFO | Starting a round of Baz IO | 20260506 10:46:17.943 | INFO | Populating RTMetrics report | 20260506 10:46:17.943 | INFO | RTMetrics report populated | 20260506 10:46:17.943 | INFO | Opening tmp RTMetrics json file | 20260506 10:46:17.994 | INFO | Serializing RTMetrics to json | 20260506 10:46:18.021 | INFO | Serialization done, swapping tmp file now... | 20260506 10:46:18.022 | INFO | RTMetrics done | 20260506 10:46:18.615 | INFO | Finished a round of Baz IO: Wrote 2.014 GiB in 3.576 seconds | 20260506 10:46:18.615 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.52% / 0.05% / 53.43% | 20260506 10:46:18.615 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.24391% / 0.000103315% / 5.5631e-05% | 20260506 10:46:18.615 | INFO | Baz IO overall: | 20260506 10:46:18.615 | INFO | BytesWritten : 343758085940 | 20260506 10:46:18.615 | INFO | NumEvents : 165471200377 | 20260506 10:46:20.079 | INFO | Analyzing chunk frames = [745472,745984) | 20260506 10:46:20.079 | INFO | MemInfo: { | 20260506 10:46:20.079 | INFO | "gpu" : " 17 GB", | 20260506 10:46:20.079 | INFO | "huge" : "", | 20260506 10:46:20.079 | INFO | "huge_shared" : " 27 GB", | 20260506 10:46:20.079 | INFO | "malloc" : " 14 GB", | 20260506 10:46:20.079 | INFO | "pinned" : " 29 GB" | 20260506 10:46:20.079 | INFO | } | 20260506 10:46:20.079 | INFO | Perf report for the last 5.1392 seconds: | 20260506 10:46:20.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:46:20.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:20.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:20.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:20.079 | INFO | ANALYSIS : 57.4%, 2.8, 85.8ms, 0 | 20260506 10:46:20.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:20.079 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260506 10:46:20.079 | INFO | PRE_HQ : 56.5%, 2.0, 59.8ms, 0 | 20260506 10:46:20.079 | INFO | BAZWRITER : 69.2%, 4.8, 170.2ms, 0 | 20260506 10:46:20.079 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 10:46:20.079 | INFO | Uploaded 36GiB to GPU in 1.6811s (21.4145GiB/s) | 20260506 10:46:20.080 | INFO | Downloaded 38GiB to GPU in 1.74509s (21.7754GiB/s) | 20260506 10:46:25.218 | INFO | Analyzing chunk frames = [745984,746496) | 20260506 10:46:25.218 | INFO | MemInfo: { | 20260506 10:46:25.218 | INFO | "gpu" : " 17 GB", | 20260506 10:46:25.218 | INFO | "huge" : "", | 20260506 10:46:25.219 | INFO | "huge_shared" : " 27 GB", | 20260506 10:46:25.219 | INFO | "malloc" : " 14 GB", | 20260506 10:46:25.219 | INFO | "pinned" : " 29 GB" | 20260506 10:46:25.219 | INFO | } | 20260506 10:46:25.219 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:46:25.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:46:25.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:25.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:25.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:25.219 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:46:25.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:25.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:25.219 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:46:25.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:25.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:25.219 | INFO | Uploaded 36GiB to GPU in 1.60284s (22.4602GiB/s) | 20260506 10:46:25.219 | INFO | Downloaded 34GiB to GPU in 1.52025s (22.3647GiB/s) | 20260506 10:46:30.358 | INFO | Analyzing chunk frames = [746496,747008) | 20260506 10:46:30.358 | INFO | MemInfo: { | 20260506 10:46:30.358 | INFO | "gpu" : " 17 GB", | 20260506 10:46:30.358 | INFO | "huge" : "", | 20260506 10:46:30.358 | INFO | "huge_shared" : " 27 GB", | 20260506 10:46:30.358 | INFO | "malloc" : " 14 GB", | 20260506 10:46:30.358 | INFO | "pinned" : " 29 GB" | 20260506 10:46:30.358 | INFO | } | 20260506 10:46:30.358 | INFO | Perf report for the last 5.13948 seconds: | 20260506 10:46:30.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:46:30.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:30.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:30.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:30.358 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260506 10:46:30.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:30.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:30.358 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:46:30.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:30.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:30.358 | INFO | Uploaded 36GiB to GPU in 1.607s (22.402GiB/s) | 20260506 10:46:30.359 | INFO | Downloaded 34GiB to GPU in 1.52534s (22.2901GiB/s) | 20260506 10:46:35.497 | INFO | Analyzing chunk frames = [747008,747520) | 20260506 10:46:35.497 | INFO | MemInfo: { | 20260506 10:46:35.497 | INFO | "gpu" : " 17 GB", | 20260506 10:46:35.497 | INFO | "huge" : "", | 20260506 10:46:35.497 | INFO | "huge_shared" : " 27 GB", | 20260506 10:46:35.497 | INFO | "malloc" : " 14 GB", | 20260506 10:46:35.497 | INFO | "pinned" : " 29 GB" | 20260506 10:46:35.497 | INFO | } | 20260506 10:46:35.497 | INFO | Perf report for the last 5.13918 seconds: | 20260506 10:46:35.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:46:35.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:35.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:35.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:35.497 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 10:46:35.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:35.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:35.497 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:46:35.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:35.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:35.498 | INFO | Uploaded 36GiB to GPU in 1.60696s (22.4026GiB/s) | 20260506 10:46:35.498 | INFO | Downloaded 34GiB to GPU in 1.52506s (22.2942GiB/s) | 20260506 10:46:40.637 | INFO | Analyzing chunk frames = [747520,748032) | 20260506 10:46:40.637 | INFO | MemInfo: { | 20260506 10:46:40.637 | INFO | "gpu" : " 17 GB", | 20260506 10:46:40.637 | INFO | "huge" : "", | 20260506 10:46:40.637 | INFO | "huge_shared" : " 27 GB", | 20260506 10:46:40.637 | INFO | "malloc" : " 14 GB", | 20260506 10:46:40.637 | INFO | "pinned" : " 29 GB" | 20260506 10:46:40.637 | INFO | } | 20260506 10:46:40.637 | INFO | Perf report for the last 5.13946 seconds: | 20260506 10:46:40.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:46:40.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:40.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:40.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:40.637 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:46:40.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:40.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:40.637 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:46:40.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:40.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:40.637 | INFO | Uploaded 36GiB to GPU in 1.60529s (22.4258GiB/s) | 20260506 10:46:40.637 | INFO | Downloaded 34GiB to GPU in 1.52258s (22.3305GiB/s) | 20260506 10:46:45.776 | INFO | Analyzing chunk frames = [748032,748544) | 20260506 10:46:45.776 | INFO | MemInfo: { | 20260506 10:46:45.776 | INFO | "gpu" : " 17 GB", | 20260506 10:46:45.776 | INFO | "huge" : "", | 20260506 10:46:45.776 | INFO | "huge_shared" : " 27 GB", | 20260506 10:46:45.776 | INFO | "malloc" : " 14 GB", | 20260506 10:46:45.776 | INFO | "pinned" : " 29 GB" | 20260506 10:46:45.776 | INFO | } | 20260506 10:46:45.776 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:46:45.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:46:45.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:45.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:45.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:45.776 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:46:45.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:45.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:45.776 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:46:45.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:45.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:45.776 | INFO | Uploaded 36GiB to GPU in 1.60537s (22.4248GiB/s) | 20260506 10:46:45.776 | INFO | Downloaded 34GiB to GPU in 1.52256s (22.3308GiB/s) | 20260506 10:46:50.915 | INFO | Analyzing chunk frames = [748544,749056) | 20260506 10:46:50.915 | INFO | MemInfo: { | 20260506 10:46:50.915 | INFO | "gpu" : " 17 GB", | 20260506 10:46:50.915 | INFO | "huge" : "", | 20260506 10:46:50.915 | INFO | "huge_shared" : " 27 GB", | 20260506 10:46:50.915 | INFO | "malloc" : " 14 GB", | 20260506 10:46:50.915 | INFO | "pinned" : " 29 GB" | 20260506 10:46:50.915 | INFO | } | 20260506 10:46:50.915 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:46:50.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:46:50.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:50.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:50.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:50.916 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:46:50.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:50.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:50.916 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:46:50.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:50.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:50.916 | INFO | Uploaded 36GiB to GPU in 1.60578s (22.419GiB/s) | 20260506 10:46:50.916 | INFO | Downloaded 34GiB to GPU in 1.52385s (22.3119GiB/s) | 20260506 10:46:56.055 | INFO | Analyzing chunk frames = [749056,749568) | 20260506 10:46:56.055 | INFO | MemInfo: { | 20260506 10:46:56.055 | INFO | "gpu" : " 17 GB", | 20260506 10:46:56.055 | INFO | "huge" : "", | 20260506 10:46:56.055 | INFO | "huge_shared" : " 27 GB", | 20260506 10:46:56.055 | INFO | "malloc" : " 14 GB", | 20260506 10:46:56.055 | INFO | "pinned" : " 29 GB" | 20260506 10:46:56.055 | INFO | } | 20260506 10:46:56.055 | INFO | Perf report for the last 5.13923 seconds: | 20260506 10:46:56.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:46:56.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:56.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:56.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:56.055 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:46:56.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:56.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:56.055 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 10:46:56.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:56.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:46:56.055 | INFO | Uploaded 36GiB to GPU in 1.6059s (22.4173GiB/s) | 20260506 10:46:56.055 | INFO | Downloaded 34GiB to GPU in 1.52372s (22.3138GiB/s) | 20260506 10:46:56.152 | INFO | Starting a round of Baz IO | 20260506 10:46:59.057 | INFO | Populating RTMetrics report | 20260506 10:46:59.057 | INFO | RTMetrics report populated | 20260506 10:46:59.057 | INFO | Opening tmp RTMetrics json file | 20260506 10:46:59.062 | INFO | Serializing RTMetrics to json | 20260506 10:46:59.063 | INFO | Serialization done, swapping tmp file now... | 20260506 10:46:59.064 | INFO | RTMetrics done | 20260506 10:46:59.581 | INFO | Finished a round of Baz IO: Wrote 2.018 GiB in 3.429 seconds | 20260506 10:46:59.581 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.48% / 0.05% / 53.48% | 20260506 10:46:59.581 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.2387% / 0.000107288% / 3.57628e-05% | 20260506 10:46:59.581 | INFO | Baz IO overall: | 20260506 10:46:59.581 | INFO | BytesWritten : 345924785216 | 20260506 10:46:59.581 | INFO | NumEvents : 166523450134 | 20260506 10:47:01.194 | INFO | Analyzing chunk frames = [749568,750080) | 20260506 10:47:01.194 | INFO | MemInfo: { | 20260506 10:47:01.194 | INFO | "gpu" : " 17 GB", | 20260506 10:47:01.194 | INFO | "huge" : "", | 20260506 10:47:01.194 | INFO | "huge_shared" : " 27 GB", | 20260506 10:47:01.194 | INFO | "malloc" : " 14 GB", | 20260506 10:47:01.194 | INFO | "pinned" : " 29 GB" | 20260506 10:47:01.194 | INFO | } | 20260506 10:47:01.194 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:47:01.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:47:01.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:01.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:01.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:01.194 | INFO | ANALYSIS : 57.4%, 2.8, 85.1ms, 0 | 20260506 10:47:01.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:01.194 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 10:47:01.194 | INFO | PRE_HQ : 56.4%, 2.1, 60.6ms, 0 | 20260506 10:47:01.194 | INFO | BAZWRITER : 66.4%, 4.8, 164.2ms, 0 | 20260506 10:47:01.194 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 10:47:01.194 | INFO | Uploaded 36GiB to GPU in 1.67858s (21.4467GiB/s) | 20260506 10:47:01.194 | INFO | Downloaded 38GiB to GPU in 1.74373s (21.7924GiB/s) | 20260506 10:47:06.333 | INFO | Analyzing chunk frames = [750080,750592) | 20260506 10:47:06.333 | INFO | MemInfo: { | 20260506 10:47:06.333 | INFO | "gpu" : " 17 GB", | 20260506 10:47:06.333 | INFO | "huge" : "", | 20260506 10:47:06.333 | INFO | "huge_shared" : " 27 GB", | 20260506 10:47:06.333 | INFO | "malloc" : " 14 GB", | 20260506 10:47:06.333 | INFO | "pinned" : " 29 GB" | 20260506 10:47:06.333 | INFO | } | 20260506 10:47:06.333 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:47:06.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:47:06.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:06.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:06.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:06.333 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:47:06.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:06.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:06.333 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:47:06.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:06.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:06.334 | INFO | Uploaded 36GiB to GPU in 1.60323s (22.4547GiB/s) | 20260506 10:47:06.334 | INFO | Downloaded 34GiB to GPU in 1.52033s (22.3635GiB/s) | 20260506 10:47:11.472 | INFO | Analyzing chunk frames = [750592,751104) | 20260506 10:47:11.472 | INFO | MemInfo: { | 20260506 10:47:11.472 | INFO | "gpu" : " 17 GB", | 20260506 10:47:11.472 | INFO | "huge" : "", | 20260506 10:47:11.473 | INFO | "huge_shared" : " 27 GB", | 20260506 10:47:11.473 | INFO | "malloc" : " 14 GB", | 20260506 10:47:11.473 | INFO | "pinned" : " 29 GB" | 20260506 10:47:11.473 | INFO | } | 20260506 10:47:11.473 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:47:11.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:47:11.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:11.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:11.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:11.473 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:47:11.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:11.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:11.473 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:47:11.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:11.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:11.473 | INFO | Uploaded 36GiB to GPU in 1.60754s (22.3944GiB/s) | 20260506 10:47:11.473 | INFO | Downloaded 34GiB to GPU in 1.52476s (22.2986GiB/s) | 20260506 10:47:16.612 | INFO | Analyzing chunk frames = [751104,751616) | 20260506 10:47:16.612 | INFO | MemInfo: { | 20260506 10:47:16.612 | INFO | "gpu" : " 17 GB", | 20260506 10:47:16.612 | INFO | "huge" : "", | 20260506 10:47:16.612 | INFO | "huge_shared" : " 27 GB", | 20260506 10:47:16.612 | INFO | "malloc" : " 14 GB", | 20260506 10:47:16.612 | INFO | "pinned" : " 29 GB" | 20260506 10:47:16.612 | INFO | } | 20260506 10:47:16.612 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:47:16.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:47:16.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:16.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:16.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:16.612 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260506 10:47:16.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:16.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:16.612 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:47:16.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:16.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:16.612 | INFO | Uploaded 36GiB to GPU in 1.60808s (22.387GiB/s) | 20260506 10:47:16.612 | INFO | Downloaded 34GiB to GPU in 1.5257s (22.2848GiB/s) | 20260506 10:47:21.751 | INFO | Analyzing chunk frames = [751616,752128) | 20260506 10:47:21.751 | INFO | MemInfo: { | 20260506 10:47:21.751 | INFO | "gpu" : " 17 GB", | 20260506 10:47:21.751 | INFO | "huge" : "", | 20260506 10:47:21.751 | INFO | "huge_shared" : " 27 GB", | 20260506 10:47:21.751 | INFO | "malloc" : " 14 GB", | 20260506 10:47:21.751 | INFO | "pinned" : " 29 GB" | 20260506 10:47:21.751 | INFO | } | 20260506 10:47:21.751 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:47:21.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:47:21.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:21.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:21.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:21.751 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:47:21.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:21.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:21.751 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:47:21.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:21.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:21.752 | INFO | Uploaded 36GiB to GPU in 1.60545s (22.4236GiB/s) | 20260506 10:47:21.752 | INFO | Downloaded 34GiB to GPU in 1.522s (22.339GiB/s) | 20260506 10:47:26.890 | INFO | Analyzing chunk frames = [752128,752640) | 20260506 10:47:26.890 | INFO | MemInfo: { | 20260506 10:47:26.890 | INFO | "gpu" : " 17 GB", | 20260506 10:47:26.891 | INFO | "huge" : "", | 20260506 10:47:26.891 | INFO | "huge_shared" : " 27 GB", | 20260506 10:47:26.891 | INFO | "malloc" : " 14 GB", | 20260506 10:47:26.891 | INFO | "pinned" : " 29 GB" | 20260506 10:47:26.891 | INFO | } | 20260506 10:47:26.891 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:47:26.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:47:26.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:26.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:26.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:26.891 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 10:47:26.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:26.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:26.891 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:47:26.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:26.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:26.891 | INFO | Uploaded 36GiB to GPU in 1.60607s (22.415GiB/s) | 20260506 10:47:26.891 | INFO | Downloaded 34GiB to GPU in 1.52274s (22.3281GiB/s) | 20260506 10:47:32.030 | INFO | Analyzing chunk frames = [752640,753152) | 20260506 10:47:32.030 | INFO | MemInfo: { | 20260506 10:47:32.030 | INFO | "gpu" : " 17 GB", | 20260506 10:47:32.030 | INFO | "huge" : "", | 20260506 10:47:32.030 | INFO | "huge_shared" : " 27 GB", | 20260506 10:47:32.030 | INFO | "malloc" : " 14 GB", | 20260506 10:47:32.030 | INFO | "pinned" : " 29 GB" | 20260506 10:47:32.030 | INFO | } | 20260506 10:47:32.030 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:47:32.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:47:32.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:32.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:32.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:32.030 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:47:32.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:32.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:32.030 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:47:32.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:32.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:32.030 | INFO | Uploaded 36GiB to GPU in 1.60621s (22.4129GiB/s) | 20260506 10:47:32.030 | INFO | Downloaded 34GiB to GPU in 1.52385s (22.3119GiB/s) | 20260506 10:47:37.169 | INFO | Analyzing chunk frames = [753152,753664) | 20260506 10:47:37.169 | INFO | MemInfo: { | 20260506 10:47:37.169 | INFO | "gpu" : " 17 GB", | 20260506 10:47:37.169 | INFO | "huge" : "", | 20260506 10:47:37.169 | INFO | "huge_shared" : " 27 GB", | 20260506 10:47:37.169 | INFO | "malloc" : " 14 GB", | 20260506 10:47:37.169 | INFO | "pinned" : " 29 GB" | 20260506 10:47:37.169 | INFO | } | 20260506 10:47:37.169 | INFO | Perf report for the last 5.1393 seconds: | 20260506 10:47:37.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:47:37.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:37.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:37.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:37.170 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:47:37.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:37.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:37.170 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:47:37.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:37.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:37.170 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:47:37.170 | INFO | Total mem usage: 16.5 GiB | 20260506 10:47:37.170 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:47:37.170 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -0.842%) | 20260506 10:47:37.170 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:47:37.170 | INFO | High Water Breakdown: | 20260506 10:47:37.170 | INFO | 61.2% : 10 GiB ( 99.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:47:37.170 | INFO | 38.7% : 6.33 GiB ( 94.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:47:37.170 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:47:37.170 | INFO | ------------------------------------------ | 20260506 10:47:37.170 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:47:37.170 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:47:37.170 | INFO | Total mem usage: 26.5 GiB | 20260506 10:47:37.170 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:47:37.170 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:47:37.170 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:47:37.170 | INFO | High Water Breakdown: | 20260506 10:47:37.170 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:47:37.170 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:47:37.170 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:47:37.170 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:47:37.170 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:47:37.170 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:47:37.170 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:47:37.170 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:47:37.170 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:47:37.170 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:47:37.170 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:47:37.170 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:47:37.170 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:47:37.170 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:47:37.170 | INFO | ------------------------------------------ | 20260506 10:47:37.170 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:47:37.170 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:47:37.170 | INFO | Total mem usage: 25.6 GiB | 20260506 10:47:37.170 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:47:37.170 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:47:37.170 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:47:37.170 | INFO | High Water Breakdown: | 20260506 10:47:37.170 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:47:37.170 | INFO | ------------------------------------------ | 20260506 10:47:37.170 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:47:37.170 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:47:37.170 | INFO | Total mem usage: 15.7 GiB | 20260506 10:47:37.170 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:47:37.170 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:47:37.170 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:47:37.170 | INFO | High Water Breakdown: | 20260506 10:47:37.170 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:47:37.170 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:47:37.170 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:47:37.170 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:47:37.170 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:47:37.170 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:47:37.170 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:47:37.170 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:47:37.170 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:47:37.170 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:47:37.170 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:47:37.170 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:47:37.170 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:47:37.170 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:47:37.170 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:47:37.170 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:47:37.170 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:47:37.170 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:47:37.170 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:47:37.170 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:47:37.170 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:47:37.170 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:47:37.170 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:47:37.170 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:47:37.170 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:47:37.170 | INFO | ------------------------------------------ | 20260506 10:47:37.170 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:47:37.170 | INFO | Uploaded 36GiB to GPU in 1.60653s (22.4086GiB/s) | 20260506 10:47:37.170 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:47:37.170 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:47:37.170 | INFO | Downloaded 34GiB to GPU in 1.52404s (22.3092GiB/s) | 20260506 10:47:37.170 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:47:37.170 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 10:47:37.170 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 4.19e-03 / 0.00e+00 | 20260506 10:47:37.170 | INFO | PerfReport: 10.96%, Baseline -- 3.29e+00 / -nan, 1.63e-02 / 0.00e+00 | 20260506 10:47:37.170 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.28e-02 / 0.00e+00 | 20260506 10:47:37.170 | INFO | PerfReport: 16.06%, DME -- 3.86e+01 / -nan, 3.06e+00 / 0.00e+00 | 20260506 10:47:37.170 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.72e+01 / -nan, 4.18e-02 / 0.00e+00 | 20260506 10:47:37.170 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.21e+00 / -nan, 1.26e-01 / 0.00e+00 | 20260506 10:47:37.170 | INFO | PerfReport: 3.64%, Metrics -- 1.09e+00 / -nan, 1.71e-01 / 0.00e+00 | 20260506 10:47:37.170 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.22e-01 / -nan, 5.89e-02 / 0.00e+00 | 20260506 10:47:37.171 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:47:37.171 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:47:37.171 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:47:37.171 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:47:37.171 | INFO | PerfReport: 18.74%, Upload -- 1.68e+01 / 0.00e+00, 5.11e-01 / 0.00e+00 | 20260506 10:47:37.171 | INFO | PerfReport: 18.00%, Download -- 1.62e+01 / 0.00e+00, 8.63e-01 / 0.00e+00 | 20260506 10:47:37.171 | INFO | PerfReport: 33.48%, Compute -- 3.01e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 10:47:37.171 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260506 10:47:37.171 | INFO | PerfReport: 0.00%, LockDownload -- 4.88e-04 / 0.00e+00, 1.56e-04 / 0.00e+00 | 20260506 10:47:37.171 | INFO | PerfReport: 29.20%, LockCompute -- 2.62e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 10:47:37.171 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.36e-02 / 0.00e+00 | 20260506 10:47:37.267 | INFO | Starting a round of Baz IO | 20260506 10:47:40.176 | INFO | Populating RTMetrics report | 20260506 10:47:40.177 | INFO | RTMetrics report populated | 20260506 10:47:40.177 | INFO | Opening tmp RTMetrics json file | 20260506 10:47:40.182 | INFO | Serializing RTMetrics to json | 20260506 10:47:40.210 | INFO | Serialization done, swapping tmp file now... | 20260506 10:47:40.238 | INFO | RTMetrics done | 20260506 10:47:40.712 | INFO | Finished a round of Baz IO: Wrote 2.018 GiB in 3.445 seconds | 20260506 10:47:40.712 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.43% / 0.05% / 53.52% | 20260506 10:47:40.713 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.20212% / 0.000139078% / 7.15256e-05% | 20260506 10:47:40.713 | INFO | Baz IO overall: | 20260506 10:47:40.713 | INFO | BytesWritten : 348091784416 | 20260506 10:47:40.713 | INFO | NumEvents : 167574783791 | 20260506 10:47:42.309 | INFO | Analyzing chunk frames = [753664,754176) | 20260506 10:47:42.309 | INFO | MemInfo: { | 20260506 10:47:42.309 | INFO | "gpu" : " 17 GB", | 20260506 10:47:42.309 | INFO | "huge" : "", | 20260506 10:47:42.309 | INFO | "huge_shared" : " 27 GB", | 20260506 10:47:42.309 | INFO | "malloc" : " 14 GB", | 20260506 10:47:42.309 | INFO | "pinned" : " 29 GB" | 20260506 10:47:42.309 | INFO | } | 20260506 10:47:42.309 | INFO | Perf report for the last 5.13922 seconds: | 20260506 10:47:42.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:47:42.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:42.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:42.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:42.309 | INFO | ANALYSIS : 57.5%, 2.8, 85.2ms, 0 | 20260506 10:47:42.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:42.309 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260506 10:47:42.309 | INFO | PRE_HQ : 56.6%, 2.1, 60.6ms, 0 | 20260506 10:47:42.309 | INFO | BAZWRITER : 66.6%, 4.8, 163.4ms, 0 | 20260506 10:47:42.309 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 10:47:42.309 | INFO | Uploaded 36GiB to GPU in 1.67546s (21.4867GiB/s) | 20260506 10:47:42.309 | INFO | Downloaded 38GiB to GPU in 1.74021s (21.8364GiB/s) | 20260506 10:47:47.448 | INFO | Analyzing chunk frames = [754176,754688) | 20260506 10:47:47.448 | INFO | MemInfo: { | 20260506 10:47:47.448 | INFO | "gpu" : " 17 GB", | 20260506 10:47:47.448 | INFO | "huge" : "", | 20260506 10:47:47.448 | INFO | "huge_shared" : " 27 GB", | 20260506 10:47:47.448 | INFO | "malloc" : " 14 GB", | 20260506 10:47:47.448 | INFO | "pinned" : " 29 GB" | 20260506 10:47:47.448 | INFO | } | 20260506 10:47:47.448 | INFO | Perf report for the last 5.13944 seconds: | 20260506 10:47:47.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:47:47.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:47.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:47.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:47.448 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 10:47:47.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:47.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:47.448 | INFO | PRE_HQ : 29.1%, 1.0, 15.3ms, 0 | 20260506 10:47:47.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:47.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:47.449 | INFO | Uploaded 36GiB to GPU in 1.60414s (22.4419GiB/s) | 20260506 10:47:47.449 | INFO | Downloaded 34GiB to GPU in 1.52136s (22.3485GiB/s) | 20260506 10:47:52.587 | INFO | Analyzing chunk frames = [754688,755200) | 20260506 10:47:52.587 | INFO | MemInfo: { | 20260506 10:47:52.587 | INFO | "gpu" : " 17 GB", | 20260506 10:47:52.587 | INFO | "huge" : "", | 20260506 10:47:52.587 | INFO | "huge_shared" : " 27 GB", | 20260506 10:47:52.587 | INFO | "malloc" : " 14 GB", | 20260506 10:47:52.588 | INFO | "pinned" : " 29 GB" | 20260506 10:47:52.588 | INFO | } | 20260506 10:47:52.588 | INFO | Perf report for the last 5.13949 seconds: | 20260506 10:47:52.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:47:52.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:52.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:52.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:52.588 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260506 10:47:52.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:52.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:52.588 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:47:52.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:52.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:52.588 | INFO | Uploaded 36GiB to GPU in 1.60559s (22.4217GiB/s) | 20260506 10:47:52.588 | INFO | Downloaded 34GiB to GPU in 1.52534s (22.2902GiB/s) | 20260506 10:47:57.727 | INFO | Analyzing chunk frames = [755200,755712) | 20260506 10:47:57.727 | INFO | MemInfo: { | 20260506 10:47:57.727 | INFO | "gpu" : " 17 GB", | 20260506 10:47:57.727 | INFO | "huge" : "", | 20260506 10:47:57.727 | INFO | "huge_shared" : " 27 GB", | 20260506 10:47:57.727 | INFO | "malloc" : " 14 GB", | 20260506 10:47:57.727 | INFO | "pinned" : " 29 GB" | 20260506 10:47:57.727 | INFO | } | 20260506 10:47:57.727 | INFO | Perf report for the last 5.13914 seconds: | 20260506 10:47:57.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:47:57.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:57.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:57.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:57.727 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260506 10:47:57.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:57.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:57.727 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:47:57.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:57.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:47:57.727 | INFO | Uploaded 36GiB to GPU in 1.60732s (22.3975GiB/s) | 20260506 10:47:57.727 | INFO | Downloaded 34GiB to GPU in 1.52509s (22.2938GiB/s) | 20260506 10:48:02.866 | INFO | Analyzing chunk frames = [755712,756224) | 20260506 10:48:02.866 | INFO | MemInfo: { | 20260506 10:48:02.866 | INFO | "gpu" : " 17 GB", | 20260506 10:48:02.866 | INFO | "huge" : "", | 20260506 10:48:02.866 | INFO | "huge_shared" : " 27 GB", | 20260506 10:48:02.866 | INFO | "malloc" : " 14 GB", | 20260506 10:48:02.866 | INFO | "pinned" : " 29 GB" | 20260506 10:48:02.866 | INFO | } | 20260506 10:48:02.866 | INFO | Perf report for the last 5.13939 seconds: | 20260506 10:48:02.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:48:02.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:02.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:02.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:02.866 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:48:02.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:02.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:02.866 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:48:02.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:02.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:02.867 | INFO | Uploaded 36GiB to GPU in 1.60498s (22.4303GiB/s) | 20260506 10:48:02.867 | INFO | Downloaded 34GiB to GPU in 1.52265s (22.3295GiB/s) | 20260506 10:48:08.005 | INFO | Analyzing chunk frames = [756224,756736) | 20260506 10:48:08.005 | INFO | MemInfo: { | 20260506 10:48:08.005 | INFO | "gpu" : " 17 GB", | 20260506 10:48:08.005 | INFO | "huge" : "", | 20260506 10:48:08.005 | INFO | "huge_shared" : " 27 GB", | 20260506 10:48:08.005 | INFO | "malloc" : " 14 GB", | 20260506 10:48:08.006 | INFO | "pinned" : " 29 GB" | 20260506 10:48:08.006 | INFO | } | 20260506 10:48:08.006 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:48:08.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:48:08.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:08.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:08.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:08.006 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 10:48:08.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:08.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:08.006 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:48:08.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:08.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:08.006 | INFO | Uploaded 36GiB to GPU in 1.60568s (22.4204GiB/s) | 20260506 10:48:08.006 | INFO | Downloaded 34GiB to GPU in 1.5226s (22.3302GiB/s) | 20260506 10:48:13.145 | INFO | Analyzing chunk frames = [756736,757248) | 20260506 10:48:13.145 | INFO | MemInfo: { | 20260506 10:48:13.145 | INFO | "gpu" : " 17 GB", | 20260506 10:48:13.145 | INFO | "huge" : "", | 20260506 10:48:13.145 | INFO | "huge_shared" : " 27 GB", | 20260506 10:48:13.145 | INFO | "malloc" : " 14 GB", | 20260506 10:48:13.145 | INFO | "pinned" : " 29 GB" | 20260506 10:48:13.145 | INFO | } | 20260506 10:48:13.145 | INFO | Perf report for the last 5.13925 seconds: | 20260506 10:48:13.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:48:13.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:13.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:13.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:13.145 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 10:48:13.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:13.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:13.145 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:48:13.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:13.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:13.145 | INFO | Uploaded 36GiB to GPU in 1.60577s (22.4191GiB/s) | 20260506 10:48:13.145 | INFO | Downloaded 34GiB to GPU in 1.52369s (22.3143GiB/s) | 20260506 10:48:18.284 | INFO | Analyzing chunk frames = [757248,757760) | 20260506 10:48:18.284 | INFO | MemInfo: { | 20260506 10:48:18.284 | INFO | "gpu" : " 17 GB", | 20260506 10:48:18.284 | INFO | "huge" : "", | 20260506 10:48:18.284 | INFO | "huge_shared" : " 27 GB", | 20260506 10:48:18.284 | INFO | "malloc" : " 14 GB", | 20260506 10:48:18.284 | INFO | "pinned" : " 29 GB" | 20260506 10:48:18.284 | INFO | } | 20260506 10:48:18.284 | INFO | Perf report for the last 5.13939 seconds: | 20260506 10:48:18.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:48:18.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:18.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:18.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:18.284 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:48:18.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:18.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:18.284 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:48:18.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:18.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:18.285 | INFO | Uploaded 36GiB to GPU in 1.60747s (22.3955GiB/s) | 20260506 10:48:18.285 | INFO | Downloaded 34GiB to GPU in 1.52553s (22.2873GiB/s) | 20260506 10:48:18.382 | INFO | Starting a round of Baz IO | 20260506 10:48:21.289 | INFO | Populating RTMetrics report | 20260506 10:48:21.290 | INFO | RTMetrics report populated | 20260506 10:48:21.290 | INFO | Opening tmp RTMetrics json file | 20260506 10:48:21.335 | INFO | Serializing RTMetrics to json | 20260506 10:48:21.370 | INFO | Serialization done, swapping tmp file now... | 20260506 10:48:21.371 | INFO | RTMetrics done | 20260506 10:48:21.666 | INFO | Finished a round of Baz IO: Wrote 2.018 GiB in 3.284 seconds | 20260506 10:48:21.666 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.39% / 0.04% / 53.57% | 20260506 10:48:21.666 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.19195% / 8.74201e-05% / 6.35783e-05% | 20260506 10:48:21.666 | INFO | Baz IO overall: | 20260506 10:48:21.666 | INFO | BytesWritten : 350258988639 | 20260506 10:48:21.666 | INFO | NumEvents : 168626670515 | 20260506 10:48:23.423 | INFO | Analyzing chunk frames = [757760,758272) | 20260506 10:48:23.423 | INFO | MemInfo: { | 20260506 10:48:23.423 | INFO | "gpu" : " 17 GB", | 20260506 10:48:23.423 | INFO | "huge" : "", | 20260506 10:48:23.424 | INFO | "huge_shared" : " 27 GB", | 20260506 10:48:23.424 | INFO | "malloc" : " 14 GB", | 20260506 10:48:23.424 | INFO | "pinned" : " 29 GB" | 20260506 10:48:23.424 | INFO | } | 20260506 10:48:23.424 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:48:23.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:48:23.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:23.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:23.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:23.424 | INFO | ANALYSIS : 57.5%, 2.8, 85.1ms, 0 | 20260506 10:48:23.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:23.424 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260506 10:48:23.424 | INFO | PRE_HQ : 56.6%, 2.1, 60.8ms, 0 | 20260506 10:48:23.424 | INFO | BAZWRITER : 63.4%, 4.8, 156.1ms, 0 | 20260506 10:48:23.424 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 10:48:23.425 | INFO | Uploaded 36GiB to GPU in 1.6737s (21.5093GiB/s) | 20260506 10:48:23.425 | INFO | Downloaded 38GiB to GPU in 1.73984s (21.8411GiB/s) | 20260506 10:48:28.563 | INFO | Analyzing chunk frames = [758272,758784) | 20260506 10:48:28.563 | INFO | MemInfo: { | 20260506 10:48:28.563 | INFO | "gpu" : " 17 GB", | 20260506 10:48:28.563 | INFO | "huge" : "", | 20260506 10:48:28.563 | INFO | "huge_shared" : " 27 GB", | 20260506 10:48:28.563 | INFO | "malloc" : " 14 GB", | 20260506 10:48:28.563 | INFO | "pinned" : " 29 GB" | 20260506 10:48:28.563 | INFO | } | 20260506 10:48:28.563 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:48:28.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:48:28.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:28.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:28.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:28.563 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 10:48:28.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:28.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:28.563 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:48:28.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:28.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:28.563 | INFO | Uploaded 36GiB to GPU in 1.60334s (22.4531GiB/s) | 20260506 10:48:28.563 | INFO | Downloaded 34GiB to GPU in 1.52119s (22.3509GiB/s) | 20260506 10:48:33.702 | INFO | Analyzing chunk frames = [758784,759296) | 20260506 10:48:33.702 | INFO | MemInfo: { | 20260506 10:48:33.702 | INFO | "gpu" : " 17 GB", | 20260506 10:48:33.702 | INFO | "huge" : "", | 20260506 10:48:33.702 | INFO | "huge_shared" : " 27 GB", | 20260506 10:48:33.702 | INFO | "malloc" : " 14 GB", | 20260506 10:48:33.702 | INFO | "pinned" : " 29 GB" | 20260506 10:48:33.702 | INFO | } | 20260506 10:48:33.702 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:48:33.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:48:33.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:33.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:33.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:33.702 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260506 10:48:33.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:33.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:33.703 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:48:33.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:33.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:33.703 | INFO | Uploaded 36GiB to GPU in 1.60755s (22.3943GiB/s) | 20260506 10:48:33.703 | INFO | Downloaded 34GiB to GPU in 1.52616s (22.2782GiB/s) | 20260506 10:48:38.841 | INFO | Analyzing chunk frames = [759296,759808) | 20260506 10:48:38.841 | INFO | MemInfo: { | 20260506 10:48:38.841 | INFO | "gpu" : " 17 GB", | 20260506 10:48:38.841 | INFO | "huge" : "", | 20260506 10:48:38.841 | INFO | "huge_shared" : " 27 GB", | 20260506 10:48:38.842 | INFO | "malloc" : " 14 GB", | 20260506 10:48:38.842 | INFO | "pinned" : " 29 GB" | 20260506 10:48:38.842 | INFO | } | 20260506 10:48:38.842 | INFO | Perf report for the last 5.13922 seconds: | 20260506 10:48:38.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:48:38.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:38.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:38.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:38.842 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 10:48:38.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:38.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:38.842 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:48:38.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:38.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:38.842 | INFO | Uploaded 36GiB to GPU in 1.60757s (22.3941GiB/s) | 20260506 10:48:38.842 | INFO | Downloaded 34GiB to GPU in 1.52486s (22.2971GiB/s) | 20260506 10:48:43.981 | INFO | Analyzing chunk frames = [759808,760320) | 20260506 10:48:43.981 | INFO | MemInfo: { | 20260506 10:48:43.981 | INFO | "gpu" : " 17 GB", | 20260506 10:48:43.981 | INFO | "huge" : "", | 20260506 10:48:43.981 | INFO | "huge_shared" : " 27 GB", | 20260506 10:48:43.981 | INFO | "malloc" : " 14 GB", | 20260506 10:48:43.981 | INFO | "pinned" : " 29 GB" | 20260506 10:48:43.981 | INFO | } | 20260506 10:48:43.981 | INFO | Perf report for the last 5.13963 seconds: | 20260506 10:48:43.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:48:43.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:43.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:43.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:43.981 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:48:43.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:43.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:43.981 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:48:43.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:43.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:43.982 | INFO | Uploaded 36GiB to GPU in 1.60554s (22.4224GiB/s) | 20260506 10:48:43.982 | INFO | Downloaded 34GiB to GPU in 1.52231s (22.3345GiB/s) | 20260506 10:48:49.120 | INFO | Analyzing chunk frames = [760320,760832) | 20260506 10:48:49.120 | INFO | MemInfo: { | 20260506 10:48:49.120 | INFO | "gpu" : " 17 GB", | 20260506 10:48:49.120 | INFO | "huge" : "", | 20260506 10:48:49.120 | INFO | "huge_shared" : " 27 GB", | 20260506 10:48:49.120 | INFO | "malloc" : " 14 GB", | 20260506 10:48:49.120 | INFO | "pinned" : " 29 GB" | 20260506 10:48:49.120 | INFO | } | 20260506 10:48:49.120 | INFO | Perf report for the last 5.13897 seconds: | 20260506 10:48:49.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:48:49.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:49.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:49.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:49.120 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:48:49.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:49.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:49.120 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:48:49.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:49.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:49.121 | INFO | Uploaded 36GiB to GPU in 1.60542s (22.424GiB/s) | 20260506 10:48:49.121 | INFO | Downloaded 34GiB to GPU in 1.52223s (22.3357GiB/s) | 20260506 10:48:54.260 | INFO | Analyzing chunk frames = [760832,761344) | 20260506 10:48:54.260 | INFO | MemInfo: { | 20260506 10:48:54.260 | INFO | "gpu" : " 17 GB", | 20260506 10:48:54.260 | INFO | "huge" : "", | 20260506 10:48:54.260 | INFO | "huge_shared" : " 27 GB", | 20260506 10:48:54.260 | INFO | "malloc" : " 14 GB", | 20260506 10:48:54.260 | INFO | "pinned" : " 29 GB" | 20260506 10:48:54.260 | INFO | } | 20260506 10:48:54.260 | INFO | Perf report for the last 5.13958 seconds: | 20260506 10:48:54.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:48:54.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:54.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:54.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:54.260 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 10:48:54.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:54.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:54.260 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:48:54.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:54.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:54.260 | INFO | Uploaded 36GiB to GPU in 1.60604s (22.4154GiB/s) | 20260506 10:48:54.260 | INFO | Downloaded 34GiB to GPU in 1.52406s (22.3089GiB/s) | 20260506 10:48:59.399 | INFO | Analyzing chunk frames = [761344,761856) | 20260506 10:48:59.399 | INFO | MemInfo: { | 20260506 10:48:59.399 | INFO | "gpu" : " 17 GB", | 20260506 10:48:59.399 | INFO | "huge" : "", | 20260506 10:48:59.399 | INFO | "huge_shared" : " 27 GB", | 20260506 10:48:59.399 | INFO | "malloc" : " 14 GB", | 20260506 10:48:59.399 | INFO | "pinned" : " 29 GB" | 20260506 10:48:59.399 | INFO | } | 20260506 10:48:59.399 | INFO | Perf report for the last 5.13951 seconds: | 20260506 10:48:59.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:48:59.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:59.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:59.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:59.399 | INFO | ANALYSIS : 56.9%, 3.0, 88.8ms, 0 | 20260506 10:48:59.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:59.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:59.399 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 10:48:59.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:59.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:48:59.400 | INFO | Uploaded 36GiB to GPU in 1.60607s (22.415GiB/s) | 20260506 10:48:59.400 | INFO | Downloaded 34GiB to GPU in 1.52354s (22.3164GiB/s) | 20260506 10:48:59.400 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:48:59.400 | INFO | Total mem usage: 16.5 GiB | 20260506 10:48:59.400 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:48:59.400 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -5.02%) | 20260506 10:48:59.400 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:48:59.400 | INFO | High Water Breakdown: | 20260506 10:48:59.400 | INFO | 60.6% : 9.53 GiB ( 94.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:48:59.400 | INFO | 39.3% : 6.18 GiB ( 92.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:48:59.400 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:48:59.400 | INFO | ------------------------------------------ | 20260506 10:48:59.400 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:48:59.400 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:48:59.400 | INFO | Total mem usage: 26.5 GiB | 20260506 10:48:59.400 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:48:59.400 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:48:59.400 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:48:59.400 | INFO | High Water Breakdown: | 20260506 10:48:59.400 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:48:59.400 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:48:59.400 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:48:59.400 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:48:59.400 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:48:59.400 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:48:59.400 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:48:59.400 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:48:59.400 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:48:59.400 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:48:59.400 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:48:59.400 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:48:59.400 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:48:59.400 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:48:59.400 | INFO | ------------------------------------------ | 20260506 10:48:59.400 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:48:59.400 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:48:59.400 | INFO | Total mem usage: 25.6 GiB | 20260506 10:48:59.400 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:48:59.400 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:48:59.400 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:48:59.400 | INFO | High Water Breakdown: | 20260506 10:48:59.400 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:48:59.400 | INFO | ------------------------------------------ | 20260506 10:48:59.400 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:48:59.401 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:48:59.401 | INFO | Total mem usage: 15.7 GiB | 20260506 10:48:59.401 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:48:59.401 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:48:59.401 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:48:59.401 | INFO | High Water Breakdown: | 20260506 10:48:59.401 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:48:59.401 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:48:59.401 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:48:59.401 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:48:59.401 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:48:59.401 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:48:59.401 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:48:59.401 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:48:59.401 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:48:59.401 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:48:59.401 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:48:59.401 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:48:59.401 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:48:59.401 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:48:59.401 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:48:59.401 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:48:59.401 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:48:59.401 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:48:59.401 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:48:59.401 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:48:59.401 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:48:59.401 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:48:59.401 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:48:59.401 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:48:59.401 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:48:59.401 | INFO | ------------------------------------------ | 20260506 10:48:59.401 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:48:59.401 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:48:59.401 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:48:59.401 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:48:59.401 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 10:48:59.401 | INFO | PerfReport: 0.29%, AggregateStats -- 8.64e-02 / -nan, 4.32e-03 / 0.00e+00 | 20260506 10:48:59.401 | INFO | PerfReport: 10.97%, Baseline -- 3.30e+00 / -nan, 1.83e-02 / 0.00e+00 | 20260506 10:48:59.401 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.18e-02 / 0.00e+00 | 20260506 10:48:59.401 | INFO | PerfReport: 16.07%, DME -- 3.86e+01 / -nan, 2.94e+00 / 0.00e+00 | 20260506 10:48:59.401 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.72e+01 / -nan, 4.24e-02 / 0.00e+00 | 20260506 10:48:59.401 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.21e+00 / -nan, 1.26e-01 / 0.00e+00 | 20260506 10:48:59.401 | INFO | PerfReport: 3.63%, Metrics -- 1.09e+00 / -nan, 1.70e-01 / 0.00e+00 | 20260506 10:48:59.401 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.21e-01 / -nan, 5.90e-02 / 0.00e+00 | 20260506 10:48:59.401 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:48:59.401 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:48:59.401 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:48:59.401 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.85e+01 / 0.00e+00 | 20260506 10:48:59.401 | INFO | PerfReport: 18.74%, Upload -- 1.68e+01 / 0.00e+00, 4.93e-01 / 0.00e+00 | 20260506 10:48:59.401 | INFO | PerfReport: 18.00%, Download -- 1.62e+01 / 0.00e+00, 8.49e-01 / 0.00e+00 | 20260506 10:48:59.401 | INFO | PerfReport: 33.50%, Compute -- 3.01e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 10:48:59.401 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260506 10:48:59.401 | INFO | PerfReport: 0.00%, LockDownload -- 4.72e-04 / 0.00e+00, 1.32e-04 / 0.00e+00 | 20260506 10:48:59.401 | INFO | PerfReport: 29.18%, LockCompute -- 2.62e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 10:48:59.401 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.08e-02 / 0.00e+00 | 20260506 10:48:59.497 | INFO | Starting a round of Baz IO | 20260506 10:49:02.401 | INFO | Populating RTMetrics report | 20260506 10:49:02.401 | INFO | RTMetrics report populated | 20260506 10:49:02.401 | INFO | Opening tmp RTMetrics json file | 20260506 10:49:02.455 | INFO | Serializing RTMetrics to json | 20260506 10:49:02.473 | INFO | Serialization done, swapping tmp file now... | 20260506 10:49:02.474 | INFO | RTMetrics done | 20260506 10:49:02.931 | INFO | Finished a round of Baz IO: Wrote 2.017 GiB in 3.433 seconds | 20260506 10:49:02.931 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.34% / 0.04% / 53.61% | 20260506 10:49:02.931 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.15429% / 5.96046e-05% / 3.97364e-05% | 20260506 10:49:02.931 | INFO | Baz IO overall: | 20260506 10:49:02.931 | INFO | BytesWritten : 352424589584 | 20260506 10:49:02.931 | INFO | NumEvents : 169677261253 | 20260506 10:49:04.538 | INFO | Analyzing chunk frames = [761856,762368) | 20260506 10:49:04.538 | INFO | MemInfo: { | 20260506 10:49:04.538 | INFO | "gpu" : " 17 GB", | 20260506 10:49:04.538 | INFO | "huge" : "", | 20260506 10:49:04.538 | INFO | "huge_shared" : " 27 GB", | 20260506 10:49:04.538 | INFO | "malloc" : " 14 GB", | 20260506 10:49:04.538 | INFO | "pinned" : " 29 GB" | 20260506 10:49:04.538 | INFO | } | 20260506 10:49:04.538 | INFO | Perf report for the last 5.13895 seconds: | 20260506 10:49:04.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:49:04.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:04.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:04.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:04.538 | INFO | ANALYSIS : 57.4%, 2.8, 85.1ms, 0 | 20260506 10:49:04.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:04.539 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260506 10:49:04.539 | INFO | PRE_HQ : 56.4%, 2.1, 60.5ms, 0 | 20260506 10:49:04.539 | INFO | BAZWRITER : 66.2%, 4.8, 162.9ms, 0 | 20260506 10:49:04.539 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 10:49:04.539 | INFO | Uploaded 36GiB to GPU in 1.67373s (21.5089GiB/s) | 20260506 10:49:04.539 | INFO | Downloaded 38GiB to GPU in 1.74056s (21.8321GiB/s) | 20260506 10:49:09.678 | INFO | Analyzing chunk frames = [762368,762880) | 20260506 10:49:09.678 | INFO | MemInfo: { | 20260506 10:49:09.678 | INFO | "gpu" : " 17 GB", | 20260506 10:49:09.678 | INFO | "huge" : "", | 20260506 10:49:09.678 | INFO | "huge_shared" : " 27 GB", | 20260506 10:49:09.678 | INFO | "malloc" : " 14 GB", | 20260506 10:49:09.678 | INFO | "pinned" : " 29 GB" | 20260506 10:49:09.678 | INFO | } | 20260506 10:49:09.678 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:49:09.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:49:09.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:09.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:09.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:09.678 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:49:09.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:09.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:09.678 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:49:09.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:09.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:09.678 | INFO | Uploaded 36GiB to GPU in 1.60272s (22.4619GiB/s) | 20260506 10:49:09.678 | INFO | Downloaded 34GiB to GPU in 1.52055s (22.3603GiB/s) | 20260506 10:49:14.817 | INFO | Analyzing chunk frames = [762880,763392) | 20260506 10:49:14.817 | INFO | MemInfo: { | 20260506 10:49:14.817 | INFO | "gpu" : " 17 GB", | 20260506 10:49:14.817 | INFO | "huge" : "", | 20260506 10:49:14.817 | INFO | "huge_shared" : " 27 GB", | 20260506 10:49:14.817 | INFO | "malloc" : " 14 GB", | 20260506 10:49:14.817 | INFO | "pinned" : " 29 GB" | 20260506 10:49:14.817 | INFO | } | 20260506 10:49:14.817 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:49:14.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:49:14.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:14.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:14.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:14.817 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260506 10:49:14.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:14.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:14.817 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 10:49:14.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:14.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:14.817 | INFO | Uploaded 36GiB to GPU in 1.60813s (22.3863GiB/s) | 20260506 10:49:14.817 | INFO | Downloaded 34GiB to GPU in 1.52671s (22.2701GiB/s) | 20260506 10:49:19.956 | INFO | Analyzing chunk frames = [763392,763904) | 20260506 10:49:19.956 | INFO | MemInfo: { | 20260506 10:49:19.956 | INFO | "gpu" : " 17 GB", | 20260506 10:49:19.956 | INFO | "huge" : "", | 20260506 10:49:19.956 | INFO | "huge_shared" : " 27 GB", | 20260506 10:49:19.956 | INFO | "malloc" : " 14 GB", | 20260506 10:49:19.956 | INFO | "pinned" : " 29 GB" | 20260506 10:49:19.956 | INFO | } | 20260506 10:49:19.956 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:49:19.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:49:19.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:19.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:19.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:19.957 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 10:49:19.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:19.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:19.957 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:49:19.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:19.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:19.957 | INFO | Uploaded 36GiB to GPU in 1.60718s (22.3995GiB/s) | 20260506 10:49:19.957 | INFO | Downloaded 34GiB to GPU in 1.52496s (22.2956GiB/s) | 20260506 10:49:25.096 | INFO | Analyzing chunk frames = [763904,764416) | 20260506 10:49:25.096 | INFO | MemInfo: { | 20260506 10:49:25.096 | INFO | "gpu" : " 17 GB", | 20260506 10:49:25.096 | INFO | "huge" : "", | 20260506 10:49:25.096 | INFO | "huge_shared" : " 27 GB", | 20260506 10:49:25.096 | INFO | "malloc" : " 14 GB", | 20260506 10:49:25.096 | INFO | "pinned" : " 29 GB" | 20260506 10:49:25.096 | INFO | } | 20260506 10:49:25.096 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:49:25.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:49:25.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:25.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:25.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:25.096 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260506 10:49:25.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:25.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:25.096 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 10:49:25.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:25.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:25.096 | INFO | Uploaded 36GiB to GPU in 1.6055s (22.4229GiB/s) | 20260506 10:49:25.096 | INFO | Downloaded 34GiB to GPU in 1.52294s (22.3253GiB/s) | 20260506 10:49:30.235 | INFO | Analyzing chunk frames = [764416,764928) | 20260506 10:49:30.235 | INFO | MemInfo: { | 20260506 10:49:30.235 | INFO | "gpu" : " 17 GB", | 20260506 10:49:30.235 | INFO | "huge" : "", | 20260506 10:49:30.235 | INFO | "huge_shared" : " 27 GB", | 20260506 10:49:30.235 | INFO | "malloc" : " 14 GB", | 20260506 10:49:30.235 | INFO | "pinned" : " 29 GB" | 20260506 10:49:30.235 | INFO | } | 20260506 10:49:30.235 | INFO | Perf report for the last 5.1395 seconds: | 20260506 10:49:30.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:49:30.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:30.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:30.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:30.235 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 10:49:30.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:30.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:30.235 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 10:49:30.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:30.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:30.236 | INFO | Uploaded 36GiB to GPU in 1.60513s (22.4281GiB/s) | 20260506 10:49:30.236 | INFO | Downloaded 34GiB to GPU in 1.52287s (22.3263GiB/s) | 20260506 10:49:35.374 | INFO | Analyzing chunk frames = [764928,765440) | 20260506 10:49:35.374 | INFO | MemInfo: { | 20260506 10:49:35.375 | INFO | "gpu" : " 17 GB", | 20260506 10:49:35.375 | INFO | "huge" : "", | 20260506 10:49:35.375 | INFO | "huge_shared" : " 27 GB", | 20260506 10:49:35.375 | INFO | "malloc" : " 14 GB", | 20260506 10:49:35.375 | INFO | "pinned" : " 29 GB" | 20260506 10:49:35.375 | INFO | } | 20260506 10:49:35.375 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:49:35.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:49:35.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:35.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:35.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:35.375 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 10:49:35.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:35.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:35.375 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:49:35.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:35.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:35.375 | INFO | Uploaded 36GiB to GPU in 1.60641s (22.4103GiB/s) | 20260506 10:49:35.375 | INFO | Downloaded 34GiB to GPU in 1.52417s (22.3073GiB/s) | 20260506 10:49:40.514 | INFO | Analyzing chunk frames = [765440,765952) | 20260506 10:49:40.514 | INFO | MemInfo: { | 20260506 10:49:40.514 | INFO | "gpu" : " 17 GB", | 20260506 10:49:40.514 | INFO | "huge" : "", | 20260506 10:49:40.514 | INFO | "huge_shared" : " 27 GB", | 20260506 10:49:40.514 | INFO | "malloc" : " 14 GB", | 20260506 10:49:40.514 | INFO | "pinned" : " 29 GB" | 20260506 10:49:40.514 | INFO | } | 20260506 10:49:40.514 | INFO | Perf report for the last 5.13906 seconds: | 20260506 10:49:40.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:49:40.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:40.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:40.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:40.514 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:49:40.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:40.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:40.514 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:49:40.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:40.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:40.514 | INFO | Uploaded 36GiB to GPU in 1.60623s (22.4127GiB/s) | 20260506 10:49:40.514 | INFO | Downloaded 34GiB to GPU in 1.52505s (22.2943GiB/s) | 20260506 10:49:40.611 | INFO | Starting a round of Baz IO | 20260506 10:49:43.516 | INFO | Populating RTMetrics report | 20260506 10:49:43.516 | INFO | RTMetrics report populated | 20260506 10:49:43.517 | INFO | Opening tmp RTMetrics json file | 20260506 10:49:43.534 | INFO | Serializing RTMetrics to json | 20260506 10:49:43.536 | INFO | Serialization done, swapping tmp file now... | 20260506 10:49:43.536 | INFO | RTMetrics done | 20260506 10:49:44.142 | INFO | Finished a round of Baz IO: Wrote 2.017 GiB in 3.53 seconds | 20260506 10:49:44.142 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.30% / 0.04% / 53.66% | 20260506 10:49:44.142 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.13023% / 0.000115236% / 3.17891e-05% | 20260506 10:49:44.142 | INFO | Baz IO overall: | 20260506 10:49:44.142 | INFO | BytesWritten : 354590682855 | 20260506 10:49:44.142 | INFO | NumEvents : 170727993820 | 20260506 10:49:45.653 | INFO | Analyzing chunk frames = [765952,766464) | 20260506 10:49:45.653 | INFO | MemInfo: { | 20260506 10:49:45.653 | INFO | "gpu" : " 17 GB", | 20260506 10:49:45.653 | INFO | "huge" : "", | 20260506 10:49:45.653 | INFO | "huge_shared" : " 27 GB", | 20260506 10:49:45.653 | INFO | "malloc" : " 14 GB", | 20260506 10:49:45.653 | INFO | "pinned" : " 29 GB" | 20260506 10:49:45.653 | INFO | } | 20260506 10:49:45.653 | INFO | Perf report for the last 5.13949 seconds: | 20260506 10:49:45.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:49:45.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:45.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:45.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:45.653 | INFO | ANALYSIS : 57.4%, 2.8, 85.3ms, 0 | 20260506 10:49:45.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:45.653 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 10:49:45.653 | INFO | PRE_HQ : 56.4%, 2.1, 60.2ms, 0 | 20260506 10:49:45.653 | INFO | BAZWRITER : 68.1%, 4.8, 166.4ms, 0 | 20260506 10:49:45.653 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 10:49:45.654 | INFO | Uploaded 36GiB to GPU in 1.676s (21.4797GiB/s) | 20260506 10:49:45.654 | INFO | Downloaded 38GiB to GPU in 1.74082s (21.8288GiB/s) | 20260506 10:49:50.792 | INFO | Analyzing chunk frames = [766464,766976) | 20260506 10:49:50.792 | INFO | MemInfo: { | 20260506 10:49:50.792 | INFO | "gpu" : " 17 GB", | 20260506 10:49:50.792 | INFO | "huge" : "", | 20260506 10:49:50.792 | INFO | "huge_shared" : " 27 GB", | 20260506 10:49:50.793 | INFO | "malloc" : " 14 GB", | 20260506 10:49:50.793 | INFO | "pinned" : " 29 GB" | 20260506 10:49:50.793 | INFO | } | 20260506 10:49:50.793 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:49:50.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:49:50.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:50.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:50.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:50.793 | INFO | ANALYSIS : 57.4%, 3.0, 89.6ms, 0 | 20260506 10:49:50.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:50.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:50.793 | INFO | PRE_HQ : 29.4%, 1.0, 15.7ms, 0 | 20260506 10:49:50.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:50.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:50.793 | INFO | Uploaded 36GiB to GPU in 1.602s (22.4719GiB/s) | 20260506 10:49:50.793 | INFO | Downloaded 34GiB to GPU in 1.51938s (22.3775GiB/s) | 20260506 10:49:55.932 | INFO | Analyzing chunk frames = [766976,767488) | 20260506 10:49:55.932 | INFO | MemInfo: { | 20260506 10:49:55.932 | INFO | "gpu" : " 17 GB", | 20260506 10:49:55.932 | INFO | "huge" : "", | 20260506 10:49:55.932 | INFO | "huge_shared" : " 27 GB", | 20260506 10:49:55.932 | INFO | "malloc" : " 14 GB", | 20260506 10:49:55.932 | INFO | "pinned" : " 29 GB" | 20260506 10:49:55.932 | INFO | } | 20260506 10:49:55.932 | INFO | Perf report for the last 5.1395 seconds: | 20260506 10:49:55.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:49:55.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:55.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:55.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:55.932 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260506 10:49:55.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:55.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:55.932 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:49:55.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:55.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:49:55.933 | INFO | Uploaded 36GiB to GPU in 1.61066s (22.3511GiB/s) | 20260506 10:49:55.933 | INFO | Downloaded 34GiB to GPU in 1.52625s (22.2768GiB/s) | 20260506 10:50:01.071 | INFO | Analyzing chunk frames = [767488,768000) | 20260506 10:50:01.071 | INFO | MemInfo: { | 20260506 10:50:01.071 | INFO | "gpu" : " 17 GB", | 20260506 10:50:01.071 | INFO | "huge" : "", | 20260506 10:50:01.071 | INFO | "huge_shared" : " 27 GB", | 20260506 10:50:01.071 | INFO | "malloc" : " 14 GB", | 20260506 10:50:01.071 | INFO | "pinned" : " 29 GB" | 20260506 10:50:01.071 | INFO | } | 20260506 10:50:01.071 | INFO | Perf report for the last 5.13917 seconds: | 20260506 10:50:01.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:50:01.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:01.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:01.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:01.071 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 10:50:01.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:01.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:01.071 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:50:01.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:01.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:01.072 | INFO | Uploaded 36GiB to GPU in 1.6068s (22.4047GiB/s) | 20260506 10:50:01.072 | INFO | Downloaded 34GiB to GPU in 1.52482s (22.2977GiB/s) | 20260506 10:50:06.210 | INFO | Analyzing chunk frames = [768000,768512) | 20260506 10:50:06.210 | INFO | MemInfo: { | 20260506 10:50:06.210 | INFO | "gpu" : " 17 GB", | 20260506 10:50:06.210 | INFO | "huge" : "", | 20260506 10:50:06.210 | INFO | "huge_shared" : " 27 GB", | 20260506 10:50:06.210 | INFO | "malloc" : " 14 GB", | 20260506 10:50:06.211 | INFO | "pinned" : " 29 GB" | 20260506 10:50:06.211 | INFO | } | 20260506 10:50:06.211 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:50:06.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:50:06.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:06.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:06.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:06.211 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260506 10:50:06.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:06.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:06.211 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:50:06.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:06.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:06.211 | INFO | Uploaded 36GiB to GPU in 1.60564s (22.4209GiB/s) | 20260506 10:50:06.211 | INFO | Downloaded 34GiB to GPU in 1.52384s (22.312GiB/s) | 20260506 10:50:11.350 | INFO | Analyzing chunk frames = [768512,769024) | 20260506 10:50:11.350 | INFO | MemInfo: { | 20260506 10:50:11.350 | INFO | "gpu" : " 17 GB", | 20260506 10:50:11.350 | INFO | "huge" : "", | 20260506 10:50:11.350 | INFO | "huge_shared" : " 27 GB", | 20260506 10:50:11.350 | INFO | "malloc" : " 14 GB", | 20260506 10:50:11.350 | INFO | "pinned" : " 29 GB" | 20260506 10:50:11.350 | INFO | } | 20260506 10:50:11.350 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:50:11.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:50:11.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:11.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:11.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:11.350 | INFO | ANALYSIS : 56.6%, 3.0, 88.3ms, 0 | 20260506 10:50:11.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:11.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:11.350 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:50:11.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:11.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:11.350 | INFO | Uploaded 36GiB to GPU in 1.60545s (22.4236GiB/s) | 20260506 10:50:11.350 | INFO | Downloaded 34GiB to GPU in 1.5227s (22.3287GiB/s) | 20260506 10:50:16.489 | INFO | Analyzing chunk frames = [769024,769536) | 20260506 10:50:16.489 | INFO | MemInfo: { | 20260506 10:50:16.489 | INFO | "gpu" : " 17 GB", | 20260506 10:50:16.489 | INFO | "huge" : "", | 20260506 10:50:16.489 | INFO | "huge_shared" : " 27 GB", | 20260506 10:50:16.489 | INFO | "malloc" : " 14 GB", | 20260506 10:50:16.489 | INFO | "pinned" : " 29 GB" | 20260506 10:50:16.489 | INFO | } | 20260506 10:50:16.489 | INFO | Perf report for the last 5.13948 seconds: | 20260506 10:50:16.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:50:16.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:16.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:16.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:16.489 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 10:50:16.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:16.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:16.490 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:50:16.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:16.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:16.490 | INFO | Uploaded 36GiB to GPU in 1.60609s (22.4147GiB/s) | 20260506 10:50:16.490 | INFO | Downloaded 34GiB to GPU in 1.52429s (22.3055GiB/s) | 20260506 10:50:21.628 | INFO | Analyzing chunk frames = [769536,770048) | 20260506 10:50:21.628 | INFO | MemInfo: { | 20260506 10:50:21.628 | INFO | "gpu" : " 17 GB", | 20260506 10:50:21.628 | INFO | "huge" : "", | 20260506 10:50:21.628 | INFO | "huge_shared" : " 27 GB", | 20260506 10:50:21.628 | INFO | "malloc" : " 14 GB", | 20260506 10:50:21.628 | INFO | "pinned" : " 29 GB" | 20260506 10:50:21.629 | INFO | } | 20260506 10:50:21.629 | INFO | Perf report for the last 5.13916 seconds: | 20260506 10:50:21.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:50:21.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:21.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:21.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:21.629 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:50:21.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:21.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:21.629 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 10:50:21.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:21.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:21.629 | INFO | Uploaded 36GiB to GPU in 1.60607s (22.415GiB/s) | 20260506 10:50:21.629 | INFO | Downloaded 34GiB to GPU in 1.52379s (22.3127GiB/s) | 20260506 10:50:21.629 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:50:21.629 | INFO | Total mem usage: 16.5 GiB | 20260506 10:50:21.629 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:50:21.629 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -3.49%) | 20260506 10:50:21.629 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:50:21.629 | INFO | High Water Breakdown: | 20260506 10:50:21.629 | INFO | 60.8% : 9.7 GiB ( 96% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:50:21.629 | INFO | 39.1% : 6.24 GiB ( 93.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:50:21.629 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:50:21.629 | INFO | ------------------------------------------ | 20260506 10:50:21.629 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:50:21.629 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:50:21.629 | INFO | Total mem usage: 26.5 GiB | 20260506 10:50:21.629 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:50:21.629 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:50:21.629 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:50:21.629 | INFO | High Water Breakdown: | 20260506 10:50:21.629 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:50:21.629 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:50:21.629 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:50:21.629 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:50:21.629 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:50:21.629 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:50:21.629 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:50:21.629 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:50:21.629 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:50:21.629 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:50:21.629 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:50:21.629 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:50:21.629 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:50:21.629 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:50:21.629 | INFO | ------------------------------------------ | 20260506 10:50:21.629 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:50:21.629 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:50:21.629 | INFO | Total mem usage: 25.6 GiB | 20260506 10:50:21.629 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:50:21.629 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:50:21.629 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:50:21.629 | INFO | High Water Breakdown: | 20260506 10:50:21.629 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:50:21.629 | INFO | ------------------------------------------ | 20260506 10:50:21.629 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:50:21.629 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:50:21.629 | INFO | Total mem usage: 15.7 GiB | 20260506 10:50:21.629 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:50:21.629 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:50:21.629 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:50:21.629 | INFO | High Water Breakdown: | 20260506 10:50:21.629 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:50:21.629 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:50:21.629 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:50:21.629 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:50:21.629 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:50:21.629 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:50:21.629 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:50:21.629 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:50:21.629 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:50:21.629 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:50:21.629 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:50:21.629 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:50:21.629 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:50:21.629 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:50:21.629 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:50:21.629 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:50:21.629 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:50:21.629 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:50:21.629 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:50:21.629 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:50:21.629 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:50:21.629 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:50:21.629 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:50:21.629 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:50:21.629 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:50:21.629 | INFO | ------------------------------------------ | 20260506 10:50:21.629 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:50:21.629 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:50:21.629 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:50:21.629 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:50:21.629 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 10:50:21.629 | INFO | PerfReport: 0.29%, AggregateStats -- 8.63e-02 / -nan, 4.35e-03 / 0.00e+00 | 20260506 10:50:21.629 | INFO | PerfReport: 10.96%, Baseline -- 3.29e+00 / -nan, 1.80e-02 / 0.00e+00 | 20260506 10:50:21.629 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.24e-02 / 0.00e+00 | 20260506 10:50:21.629 | INFO | PerfReport: 16.07%, DME -- 3.86e+01 / -nan, 3.06e+00 / 0.00e+00 | 20260506 10:50:21.629 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.72e+01 / -nan, 7.91e-02 / 0.00e+00 | 20260506 10:50:21.629 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.21e+00 / -nan, 1.27e-01 / 0.00e+00 | 20260506 10:50:21.629 | INFO | PerfReport: 3.63%, Metrics -- 1.09e+00 / -nan, 1.93e-01 / 0.00e+00 | 20260506 10:50:21.629 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.20e-01 / -nan, 5.85e-02 / 0.00e+00 | 20260506 10:50:21.629 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:50:21.629 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:50:21.629 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:50:21.629 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:50:21.629 | INFO | PerfReport: 18.74%, Upload -- 1.68e+01 / 0.00e+00, 5.03e-01 / 0.00e+00 | 20260506 10:50:21.629 | INFO | PerfReport: 18.00%, Download -- 1.62e+01 / 0.00e+00, 8.47e-01 / 0.00e+00 | 20260506 10:50:21.629 | INFO | PerfReport: 33.50%, Compute -- 3.01e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 10:50:21.629 | INFO | PerfReport: 0.57%, LockUpload -- 5.08e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:50:21.629 | INFO | PerfReport: 0.00%, LockDownload -- 4.95e-04 / 0.00e+00, 2.64e-04 / 0.00e+00 | 20260506 10:50:21.629 | INFO | PerfReport: 29.17%, LockCompute -- 2.62e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 10:50:21.629 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260506 10:50:21.726 | INFO | Starting a round of Baz IO | 20260506 10:50:24.632 | INFO | Populating RTMetrics report | 20260506 10:50:24.633 | INFO | RTMetrics report populated | 20260506 10:50:24.633 | INFO | Opening tmp RTMetrics json file | 20260506 10:50:24.668 | INFO | Serializing RTMetrics to json | 20260506 10:50:24.801 | INFO | Serialization done, swapping tmp file now... | 20260506 10:50:24.807 | INFO | RTMetrics done | 20260506 10:50:25.213 | INFO | Finished a round of Baz IO: Wrote 2.019 GiB in 3.486 seconds | 20260506 10:50:25.213 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.26% / 0.04% / 53.70% | 20260506 10:50:25.213 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.10936% / 7.54992e-05% / 4.37101e-05% | 20260506 10:50:25.213 | INFO | Baz IO overall: | 20260506 10:50:25.213 | INFO | BytesWritten : 356758385030 | 20260506 10:50:25.213 | INFO | NumEvents : 171779845528 | 20260506 10:50:26.768 | INFO | Analyzing chunk frames = [770048,770560) | 20260506 10:50:26.768 | INFO | MemInfo: { | 20260506 10:50:26.768 | INFO | "gpu" : " 17 GB", | 20260506 10:50:26.768 | INFO | "huge" : "", | 20260506 10:50:26.768 | INFO | "huge_shared" : " 27 GB", | 20260506 10:50:26.768 | INFO | "malloc" : " 14 GB", | 20260506 10:50:26.768 | INFO | "pinned" : " 29 GB" | 20260506 10:50:26.768 | INFO | } | 20260506 10:50:26.768 | INFO | Perf report for the last 5.13942 seconds: | 20260506 10:50:26.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:50:26.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:26.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:26.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:26.768 | INFO | ANALYSIS : 57.4%, 2.8, 85.3ms, 0 | 20260506 10:50:26.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:26.768 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260506 10:50:26.768 | INFO | PRE_HQ : 56.4%, 2.1, 60.2ms, 0 | 20260506 10:50:26.768 | INFO | BAZWRITER : 67.4%, 4.8, 165.2ms, 0 | 20260506 10:50:26.768 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 10:50:26.769 | INFO | Uploaded 36GiB to GPU in 1.6789s (21.4426GiB/s) | 20260506 10:50:26.769 | INFO | Downloaded 38GiB to GPU in 1.74431s (21.7852GiB/s) | 20260506 10:50:31.907 | INFO | Analyzing chunk frames = [770560,771072) | 20260506 10:50:31.907 | INFO | MemInfo: { | 20260506 10:50:31.907 | INFO | "gpu" : " 17 GB", | 20260506 10:50:31.907 | INFO | "huge" : "", | 20260506 10:50:31.907 | INFO | "huge_shared" : " 27 GB", | 20260506 10:50:31.907 | INFO | "malloc" : " 14 GB", | 20260506 10:50:31.907 | INFO | "pinned" : " 29 GB" | 20260506 10:50:31.907 | INFO | } | 20260506 10:50:31.907 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:50:31.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:50:31.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:31.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:31.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:31.907 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:50:31.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:31.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:31.907 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:50:31.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:31.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:31.908 | INFO | Uploaded 36GiB to GPU in 1.60406s (22.443GiB/s) | 20260506 10:50:31.908 | INFO | Downloaded 34GiB to GPU in 1.52166s (22.344GiB/s) | 20260506 10:50:37.046 | INFO | Analyzing chunk frames = [771072,771584) | 20260506 10:50:37.046 | INFO | MemInfo: { | 20260506 10:50:37.046 | INFO | "gpu" : " 17 GB", | 20260506 10:50:37.046 | INFO | "huge" : "", | 20260506 10:50:37.047 | INFO | "huge_shared" : " 27 GB", | 20260506 10:50:37.047 | INFO | "malloc" : " 14 GB", | 20260506 10:50:37.047 | INFO | "pinned" : " 29 GB" | 20260506 10:50:37.047 | INFO | } | 20260506 10:50:37.047 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:50:37.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:50:37.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:37.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:37.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:37.047 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:50:37.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:37.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:37.047 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:50:37.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:37.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:37.047 | INFO | Uploaded 36GiB to GPU in 1.60717s (22.3996GiB/s) | 20260506 10:50:37.047 | INFO | Downloaded 34GiB to GPU in 1.52526s (22.2913GiB/s) | 20260506 10:50:42.186 | INFO | Analyzing chunk frames = [771584,772096) | 20260506 10:50:42.186 | INFO | MemInfo: { | 20260506 10:50:42.186 | INFO | "gpu" : " 17 GB", | 20260506 10:50:42.186 | INFO | "huge" : "", | 20260506 10:50:42.186 | INFO | "huge_shared" : " 27 GB", | 20260506 10:50:42.186 | INFO | "malloc" : " 14 GB", | 20260506 10:50:42.186 | INFO | "pinned" : " 29 GB" | 20260506 10:50:42.186 | INFO | } | 20260506 10:50:42.186 | INFO | Perf report for the last 5.13919 seconds: | 20260506 10:50:42.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:50:42.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:42.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:42.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:42.186 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260506 10:50:42.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:42.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:42.186 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:50:42.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:42.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:42.186 | INFO | Uploaded 36GiB to GPU in 1.60744s (22.3959GiB/s) | 20260506 10:50:42.186 | INFO | Downloaded 34GiB to GPU in 1.52522s (22.2919GiB/s) | 20260506 10:50:47.325 | INFO | Analyzing chunk frames = [772096,772608) | 20260506 10:50:47.325 | INFO | MemInfo: { | 20260506 10:50:47.325 | INFO | "gpu" : " 17 GB", | 20260506 10:50:47.325 | INFO | "huge" : "", | 20260506 10:50:47.325 | INFO | "huge_shared" : " 27 GB", | 20260506 10:50:47.325 | INFO | "malloc" : " 14 GB", | 20260506 10:50:47.325 | INFO | "pinned" : " 29 GB" | 20260506 10:50:47.325 | INFO | } | 20260506 10:50:47.326 | INFO | Perf report for the last 5.13961 seconds: | 20260506 10:50:47.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:50:47.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:47.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:47.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:47.326 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:50:47.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:47.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:47.326 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:50:47.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:47.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:47.326 | INFO | Uploaded 36GiB to GPU in 1.60649s (22.4092GiB/s) | 20260506 10:50:47.326 | INFO | Downloaded 34GiB to GPU in 1.5231s (22.3229GiB/s) | 20260506 10:50:52.464 | INFO | Analyzing chunk frames = [772608,773120) | 20260506 10:50:52.464 | INFO | MemInfo: { | 20260506 10:50:52.464 | INFO | "gpu" : " 17 GB", | 20260506 10:50:52.464 | INFO | "huge" : "", | 20260506 10:50:52.465 | INFO | "huge_shared" : " 27 GB", | 20260506 10:50:52.465 | INFO | "malloc" : " 14 GB", | 20260506 10:50:52.465 | INFO | "pinned" : " 29 GB" | 20260506 10:50:52.465 | INFO | } | 20260506 10:50:52.465 | INFO | Perf report for the last 5.13918 seconds: | 20260506 10:50:52.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:50:52.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:52.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:52.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:52.465 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 10:50:52.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:52.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:52.465 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:50:52.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:52.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:52.465 | INFO | Uploaded 36GiB to GPU in 1.60583s (22.4184GiB/s) | 20260506 10:50:52.465 | INFO | Downloaded 34GiB to GPU in 1.52405s (22.3089GiB/s) | 20260506 10:50:57.604 | INFO | Analyzing chunk frames = [773120,773632) | 20260506 10:50:57.604 | INFO | MemInfo: { | 20260506 10:50:57.604 | INFO | "gpu" : " 17 GB", | 20260506 10:50:57.604 | INFO | "huge" : "", | 20260506 10:50:57.604 | INFO | "huge_shared" : " 27 GB", | 20260506 10:50:57.604 | INFO | "malloc" : " 14 GB", | 20260506 10:50:57.604 | INFO | "pinned" : " 29 GB" | 20260506 10:50:57.604 | INFO | } | 20260506 10:50:57.604 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:50:57.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:50:57.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:57.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:57.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:57.604 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:50:57.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:57.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:57.604 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:50:57.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:57.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:50:57.604 | INFO | Uploaded 36GiB to GPU in 1.60574s (22.4196GiB/s) | 20260506 10:50:57.604 | INFO | Downloaded 34GiB to GPU in 1.52383s (22.3123GiB/s) | 20260506 10:51:02.743 | INFO | Analyzing chunk frames = [773632,774144) | 20260506 10:51:02.743 | INFO | MemInfo: { | 20260506 10:51:02.743 | INFO | "gpu" : " 17 GB", | 20260506 10:51:02.743 | INFO | "huge" : "", | 20260506 10:51:02.743 | INFO | "huge_shared" : " 27 GB", | 20260506 10:51:02.743 | INFO | "malloc" : " 14 GB", | 20260506 10:51:02.743 | INFO | "pinned" : " 29 GB" | 20260506 10:51:02.743 | INFO | } | 20260506 10:51:02.743 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:51:02.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:51:02.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:02.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:02.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:02.743 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:51:02.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:02.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:02.743 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:51:02.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:02.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:02.744 | INFO | Uploaded 36GiB to GPU in 1.60712s (22.4003GiB/s) | 20260506 10:51:02.744 | INFO | Downloaded 34GiB to GPU in 1.52471s (22.2994GiB/s) | 20260506 10:51:02.840 | INFO | Starting a round of Baz IO | 20260506 10:51:05.748 | INFO | Populating RTMetrics report | 20260506 10:51:05.748 | INFO | RTMetrics report populated | 20260506 10:51:05.748 | INFO | Opening tmp RTMetrics json file | 20260506 10:51:05.754 | INFO | Serializing RTMetrics to json | 20260506 10:51:05.773 | INFO | Serialization done, swapping tmp file now... | 20260506 10:51:05.777 | INFO | RTMetrics done | 20260506 10:51:05.909 | INFO | Finished a round of Baz IO: Wrote 2.022 GiB in 3.068 seconds | 20260506 10:51:05.909 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.21% / 0.04% / 53.74% | 20260506 10:51:05.909 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.10342% / 9.13938e-05% / 3.97364e-05% | 20260506 10:51:05.909 | INFO | Baz IO overall: | 20260506 10:51:05.909 | INFO | BytesWritten : 358929455127 | 20260506 10:51:05.909 | INFO | NumEvents : 172833644512 | 20260506 10:51:07.882 | INFO | Analyzing chunk frames = [774144,774656) | 20260506 10:51:07.883 | INFO | MemInfo: { | 20260506 10:51:07.883 | INFO | "gpu" : " 17 GB", | 20260506 10:51:07.883 | INFO | "huge" : "", | 20260506 10:51:07.883 | INFO | "huge_shared" : " 27 GB", | 20260506 10:51:07.883 | INFO | "malloc" : " 14 GB", | 20260506 10:51:07.883 | INFO | "pinned" : " 29 GB" | 20260506 10:51:07.883 | INFO | } | 20260506 10:51:07.883 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:51:07.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:51:07.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:07.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:07.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:07.883 | INFO | ANALYSIS : 57.4%, 2.8, 85.3ms, 0 | 20260506 10:51:07.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:07.883 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 10:51:07.883 | INFO | PRE_HQ : 56.6%, 2.1, 60.4ms, 0 | 20260506 10:51:07.883 | INFO | BAZWRITER : 59.3%, 4.8, 145.5ms, 0 | 20260506 10:51:07.883 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 10:51:07.883 | INFO | Uploaded 36GiB to GPU in 1.68032s (21.4245GiB/s) | 20260506 10:51:07.883 | INFO | Downloaded 38GiB to GPU in 1.74632s (21.76GiB/s) | 20260506 10:51:13.022 | INFO | Analyzing chunk frames = [774656,775168) | 20260506 10:51:13.022 | INFO | MemInfo: { | 20260506 10:51:13.022 | INFO | "gpu" : " 17 GB", | 20260506 10:51:13.022 | INFO | "huge" : "", | 20260506 10:51:13.022 | INFO | "huge_shared" : " 27 GB", | 20260506 10:51:13.022 | INFO | "malloc" : " 14 GB", | 20260506 10:51:13.022 | INFO | "pinned" : " 29 GB" | 20260506 10:51:13.022 | INFO | } | 20260506 10:51:13.022 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:51:13.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:51:13.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:13.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:13.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:13.022 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 10:51:13.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:13.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:13.022 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:51:13.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:13.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:13.022 | INFO | Uploaded 36GiB to GPU in 1.60291s (22.4591GiB/s) | 20260506 10:51:13.022 | INFO | Downloaded 34GiB to GPU in 1.52013s (22.3664GiB/s) | 20260506 10:51:18.161 | INFO | Analyzing chunk frames = [775168,775680) | 20260506 10:51:18.161 | INFO | MemInfo: { | 20260506 10:51:18.161 | INFO | "gpu" : " 17 GB", | 20260506 10:51:18.161 | INFO | "huge" : "", | 20260506 10:51:18.161 | INFO | "huge_shared" : " 27 GB", | 20260506 10:51:18.161 | INFO | "malloc" : " 14 GB", | 20260506 10:51:18.161 | INFO | "pinned" : " 29 GB" | 20260506 10:51:18.161 | INFO | } | 20260506 10:51:18.161 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:51:18.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:51:18.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:18.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:18.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:18.162 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:51:18.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:18.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:18.162 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:51:18.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:18.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:18.162 | INFO | Uploaded 36GiB to GPU in 1.60803s (22.3876GiB/s) | 20260506 10:51:18.162 | INFO | Downloaded 34GiB to GPU in 1.52556s (22.2869GiB/s) | 20260506 10:51:23.301 | INFO | Analyzing chunk frames = [775680,776192) | 20260506 10:51:23.301 | INFO | MemInfo: { | 20260506 10:51:23.301 | INFO | "gpu" : " 17 GB", | 20260506 10:51:23.301 | INFO | "huge" : "", | 20260506 10:51:23.301 | INFO | "huge_shared" : " 27 GB", | 20260506 10:51:23.301 | INFO | "malloc" : " 14 GB", | 20260506 10:51:23.301 | INFO | "pinned" : " 29 GB" | 20260506 10:51:23.301 | INFO | } | 20260506 10:51:23.301 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:51:23.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:51:23.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:23.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:23.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:23.301 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260506 10:51:23.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:23.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:23.301 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:51:23.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:23.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:23.301 | INFO | Uploaded 36GiB to GPU in 1.60749s (22.3952GiB/s) | 20260506 10:51:23.302 | INFO | Downloaded 34GiB to GPU in 1.5248s (22.2981GiB/s) | 20260506 10:51:28.440 | INFO | Analyzing chunk frames = [776192,776704) | 20260506 10:51:28.440 | INFO | MemInfo: { | 20260506 10:51:28.440 | INFO | "gpu" : " 17 GB", | 20260506 10:51:28.440 | INFO | "huge" : "", | 20260506 10:51:28.440 | INFO | "huge_shared" : " 27 GB", | 20260506 10:51:28.440 | INFO | "malloc" : " 14 GB", | 20260506 10:51:28.440 | INFO | "pinned" : " 29 GB" | 20260506 10:51:28.440 | INFO | } | 20260506 10:51:28.440 | INFO | Perf report for the last 5.13925 seconds: | 20260506 10:51:28.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:51:28.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:28.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:28.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:28.440 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:51:28.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:28.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:28.440 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:51:28.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:28.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:28.440 | INFO | Uploaded 36GiB to GPU in 1.6053s (22.4258GiB/s) | 20260506 10:51:28.440 | INFO | Downloaded 34GiB to GPU in 1.52272s (22.3285GiB/s) | 20260506 10:51:33.579 | INFO | Analyzing chunk frames = [776704,777216) | 20260506 10:51:33.579 | INFO | MemInfo: { | 20260506 10:51:33.579 | INFO | "gpu" : " 17 GB", | 20260506 10:51:33.579 | INFO | "huge" : "", | 20260506 10:51:33.579 | INFO | "huge_shared" : " 27 GB", | 20260506 10:51:33.579 | INFO | "malloc" : " 14 GB", | 20260506 10:51:33.579 | INFO | "pinned" : " 29 GB" | 20260506 10:51:33.579 | INFO | } | 20260506 10:51:33.579 | INFO | Perf report for the last 5.13929 seconds: | 20260506 10:51:33.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:51:33.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:33.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:33.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:33.579 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 10:51:33.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:33.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:33.580 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:51:33.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:33.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:33.580 | INFO | Uploaded 36GiB to GPU in 1.60662s (22.4073GiB/s) | 20260506 10:51:33.580 | INFO | Downloaded 34GiB to GPU in 1.52399s (22.3098GiB/s) | 20260506 10:51:38.718 | INFO | Analyzing chunk frames = [777216,777728) | 20260506 10:51:38.718 | INFO | MemInfo: { | 20260506 10:51:38.718 | INFO | "gpu" : " 17 GB", | 20260506 10:51:38.718 | INFO | "huge" : "", | 20260506 10:51:38.718 | INFO | "huge_shared" : " 27 GB", | 20260506 10:51:38.718 | INFO | "malloc" : " 14 GB", | 20260506 10:51:38.718 | INFO | "pinned" : " 29 GB" | 20260506 10:51:38.719 | INFO | } | 20260506 10:51:38.719 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:51:38.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:51:38.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:38.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:38.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:38.719 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:51:38.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:38.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:38.719 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:51:38.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:38.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:38.719 | INFO | Uploaded 36GiB to GPU in 1.60583s (22.4183GiB/s) | 20260506 10:51:38.719 | INFO | Downloaded 34GiB to GPU in 1.52443s (22.3034GiB/s) | 20260506 10:51:43.858 | INFO | Analyzing chunk frames = [777728,778240) | 20260506 10:51:43.858 | INFO | MemInfo: { | 20260506 10:51:43.858 | INFO | "gpu" : " 17 GB", | 20260506 10:51:43.858 | INFO | "huge" : "", | 20260506 10:51:43.858 | INFO | "huge_shared" : " 27 GB", | 20260506 10:51:43.858 | INFO | "malloc" : " 14 GB", | 20260506 10:51:43.858 | INFO | "pinned" : " 29 GB" | 20260506 10:51:43.858 | INFO | } | 20260506 10:51:43.858 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:51:43.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:51:43.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:43.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:43.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:43.858 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:51:43.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:43.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:43.858 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 10:51:43.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:43.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:43.858 | INFO | Uploaded 36GiB to GPU in 1.60587s (22.4178GiB/s) | 20260506 10:51:43.858 | INFO | Downloaded 34GiB to GPU in 1.52368s (22.3144GiB/s) | 20260506 10:51:43.859 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:51:43.859 | INFO | Total mem usage: 16.5 GiB | 20260506 10:51:43.859 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:51:43.859 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -5.82%) | 20260506 10:51:43.859 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:51:43.859 | INFO | High Water Breakdown: | 20260506 10:51:43.859 | INFO | 60.5% : 9.44 GiB ( 93.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:51:43.859 | INFO | 39.4% : 6.15 GiB ( 91.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:51:43.859 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:51:43.859 | INFO | ------------------------------------------ | 20260506 10:51:43.859 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:51:43.859 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:51:43.859 | INFO | Total mem usage: 26.5 GiB | 20260506 10:51:43.859 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:51:43.859 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:51:43.859 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:51:43.859 | INFO | High Water Breakdown: | 20260506 10:51:43.859 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:51:43.859 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:51:43.859 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:51:43.859 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:51:43.859 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:51:43.859 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:51:43.859 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:51:43.859 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:51:43.859 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:51:43.859 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:51:43.859 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:51:43.859 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:51:43.859 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:51:43.859 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:51:43.859 | INFO | ------------------------------------------ | 20260506 10:51:43.859 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:51:43.859 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:51:43.859 | INFO | Total mem usage: 25.6 GiB | 20260506 10:51:43.859 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:51:43.859 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:51:43.859 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:51:43.859 | INFO | High Water Breakdown: | 20260506 10:51:43.859 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:51:43.859 | INFO | ------------------------------------------ | 20260506 10:51:43.859 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:51:43.859 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:51:43.859 | INFO | Total mem usage: 15.7 GiB | 20260506 10:51:43.859 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:51:43.859 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:51:43.859 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 10:51:43.859 | INFO | High Water Breakdown: | 20260506 10:51:43.859 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:51:43.859 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:51:43.859 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:51:43.859 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:51:43.859 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:51:43.859 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:51:43.859 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:51:43.859 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:51:43.859 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:51:43.859 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:51:43.859 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:51:43.859 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:51:43.859 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:51:43.859 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:51:43.859 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:51:43.859 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:51:43.859 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:51:43.859 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:51:43.859 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:51:43.859 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:51:43.859 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:51:43.859 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:51:43.859 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:51:43.859 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:51:43.859 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:51:43.859 | INFO | ------------------------------------------ | 20260506 10:51:43.859 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:51:43.859 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:51:43.859 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:51:43.859 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:51:43.859 | INFO | PerfReport: Full Execution -- 3.01e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 10:51:43.859 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 5.05e-03 / 0.00e+00 | 20260506 10:51:43.859 | INFO | PerfReport: 10.97%, Baseline -- 3.30e+00 / -nan, 1.81e-02 / 0.00e+00 | 20260506 10:51:43.860 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.26e-02 / 0.00e+00 | 20260506 10:51:43.860 | INFO | PerfReport: 16.07%, DME -- 3.86e+01 / -nan, 3.04e+00 / 0.00e+00 | 20260506 10:51:43.860 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.72e+01 / -nan, 4.30e-02 / 0.00e+00 | 20260506 10:51:43.860 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.21e+00 / -nan, 1.25e-01 / 0.00e+00 | 20260506 10:51:43.860 | INFO | PerfReport: 3.63%, Metrics -- 1.09e+00 / -nan, 1.70e-01 / 0.00e+00 | 20260506 10:51:43.860 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.20e-01 / -nan, 5.85e-02 / 0.00e+00 | 20260506 10:51:43.860 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:51:43.860 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:51:43.860 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:51:43.860 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 10:51:43.860 | INFO | PerfReport: 18.75%, Upload -- 1.68e+01 / 0.00e+00, 5.11e-01 / 0.00e+00 | 20260506 10:51:43.860 | INFO | PerfReport: 18.01%, Download -- 1.62e+01 / 0.00e+00, 8.69e-01 / 0.00e+00 | 20260506 10:51:43.860 | INFO | PerfReport: 33.49%, Compute -- 3.01e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 10:51:43.860 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 10:51:43.860 | INFO | PerfReport: 0.00%, LockDownload -- 5.03e-04 / 0.00e+00, 1.62e-04 / 0.00e+00 | 20260506 10:51:43.860 | INFO | PerfReport: 29.17%, LockCompute -- 2.62e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 10:51:43.860 | INFO | PerfReport: 0.02%, Misc -- 1.42e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260506 10:51:43.956 | INFO | Starting a round of Baz IO | 20260506 10:51:46.863 | INFO | Populating RTMetrics report | 20260506 10:51:46.864 | INFO | RTMetrics report populated | 20260506 10:51:46.864 | INFO | Opening tmp RTMetrics json file | 20260506 10:51:46.949 | INFO | Serializing RTMetrics to json | 20260506 10:51:46.980 | INFO | Serialization done, swapping tmp file now... | 20260506 10:51:46.983 | INFO | RTMetrics done | 20260506 10:51:47.529 | INFO | Finished a round of Baz IO: Wrote 2.023 GiB in 3.572 seconds | 20260506 10:51:47.529 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.17% / 0.04% / 53.79% | 20260506 10:51:47.529 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.0892% / 9.53674e-05% / 3.97364e-05% | 20260506 10:51:47.529 | INFO | Baz IO overall: | 20260506 10:51:47.529 | INFO | BytesWritten : 361101223267 | 20260506 10:51:47.529 | INFO | NumEvents : 173887543676 | 20260506 10:51:48.997 | INFO | Analyzing chunk frames = [778240,778752) | 20260506 10:51:48.997 | INFO | MemInfo: { | 20260506 10:51:48.997 | INFO | "gpu" : " 17 GB", | 20260506 10:51:48.997 | INFO | "huge" : "", | 20260506 10:51:48.997 | INFO | "huge_shared" : " 27 GB", | 20260506 10:51:48.997 | INFO | "malloc" : " 14 GB", | 20260506 10:51:48.997 | INFO | "pinned" : " 29 GB" | 20260506 10:51:48.997 | INFO | } | 20260506 10:51:48.997 | INFO | Perf report for the last 5.13913 seconds: | 20260506 10:51:48.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:51:48.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:48.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:48.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:48.997 | INFO | ANALYSIS : 57.5%, 2.9, 85.9ms, 0 | 20260506 10:51:48.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:48.997 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260506 10:51:48.997 | INFO | PRE_HQ : 56.4%, 2.1, 60.1ms, 0 | 20260506 10:51:48.997 | INFO | BAZWRITER : 69.1%, 4.8, 169.7ms, 0 | 20260506 10:51:48.997 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 10:51:48.998 | INFO | Uploaded 36GiB to GPU in 1.67497s (21.4929GiB/s) | 20260506 10:51:48.998 | INFO | Downloaded 38GiB to GPU in 1.73954s (21.8448GiB/s) | 20260506 10:51:54.137 | INFO | Analyzing chunk frames = [778752,779264) | 20260506 10:51:54.137 | INFO | MemInfo: { | 20260506 10:51:54.137 | INFO | "gpu" : " 17 GB", | 20260506 10:51:54.137 | INFO | "huge" : "", | 20260506 10:51:54.137 | INFO | "huge_shared" : " 27 GB", | 20260506 10:51:54.137 | INFO | "malloc" : " 14 GB", | 20260506 10:51:54.137 | INFO | "pinned" : " 29 GB" | 20260506 10:51:54.137 | INFO | } | 20260506 10:51:54.137 | INFO | Perf report for the last 5.13956 seconds: | 20260506 10:51:54.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:51:54.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:54.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:54.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:54.137 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 10:51:54.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:54.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:54.137 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:51:54.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:54.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:54.137 | INFO | Uploaded 36GiB to GPU in 1.60402s (22.4437GiB/s) | 20260506 10:51:54.137 | INFO | Downloaded 34GiB to GPU in 1.52086s (22.3558GiB/s) | 20260506 10:51:59.276 | INFO | Analyzing chunk frames = [779264,779776) | 20260506 10:51:59.276 | INFO | MemInfo: { | 20260506 10:51:59.276 | INFO | "gpu" : " 17 GB", | 20260506 10:51:59.276 | INFO | "huge" : "", | 20260506 10:51:59.276 | INFO | "huge_shared" : " 27 GB", | 20260506 10:51:59.276 | INFO | "malloc" : " 14 GB", | 20260506 10:51:59.277 | INFO | "pinned" : " 29 GB" | 20260506 10:51:59.277 | INFO | } | 20260506 10:51:59.277 | INFO | Perf report for the last 5.13976 seconds: | 20260506 10:51:59.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:51:59.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:59.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:59.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:59.277 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:51:59.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:59.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:59.277 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:51:59.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:59.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:51:59.277 | INFO | Uploaded 36GiB to GPU in 1.60745s (22.3958GiB/s) | 20260506 10:51:59.277 | INFO | Downloaded 34GiB to GPU in 1.52481s (22.2978GiB/s) | 20260506 10:52:04.415 | INFO | Analyzing chunk frames = [779776,780288) | 20260506 10:52:04.415 | INFO | MemInfo: { | 20260506 10:52:04.415 | INFO | "gpu" : " 17 GB", | 20260506 10:52:04.415 | INFO | "huge" : "", | 20260506 10:52:04.416 | INFO | "huge_shared" : " 27 GB", | 20260506 10:52:04.416 | INFO | "malloc" : " 14 GB", | 20260506 10:52:04.416 | INFO | "pinned" : " 29 GB" | 20260506 10:52:04.416 | INFO | } | 20260506 10:52:04.416 | INFO | Perf report for the last 5.13905 seconds: | 20260506 10:52:04.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:52:04.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:04.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:04.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:04.416 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 10:52:04.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:04.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:04.416 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:52:04.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:04.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:04.416 | INFO | Uploaded 36GiB to GPU in 1.60782s (22.3905GiB/s) | 20260506 10:52:04.416 | INFO | Downloaded 34GiB to GPU in 1.52541s (22.2891GiB/s) | 20260506 10:52:09.555 | INFO | Analyzing chunk frames = [780288,780800) | 20260506 10:52:09.555 | INFO | MemInfo: { | 20260506 10:52:09.555 | INFO | "gpu" : " 17 GB", | 20260506 10:52:09.555 | INFO | "huge" : "", | 20260506 10:52:09.555 | INFO | "huge_shared" : " 27 GB", | 20260506 10:52:09.555 | INFO | "malloc" : " 14 GB", | 20260506 10:52:09.555 | INFO | "pinned" : " 29 GB" | 20260506 10:52:09.555 | INFO | } | 20260506 10:52:09.555 | INFO | Perf report for the last 5.1393 seconds: | 20260506 10:52:09.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:52:09.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:09.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:09.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:09.555 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:52:09.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:09.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:09.555 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:52:09.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:09.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:09.555 | INFO | Uploaded 36GiB to GPU in 1.6058s (22.4187GiB/s) | 20260506 10:52:09.555 | INFO | Downloaded 34GiB to GPU in 1.52212s (22.3373GiB/s) | 20260506 10:52:14.694 | INFO | Analyzing chunk frames = [780800,781312) | 20260506 10:52:14.694 | INFO | MemInfo: { | 20260506 10:52:14.694 | INFO | "gpu" : " 17 GB", | 20260506 10:52:14.694 | INFO | "huge" : "", | 20260506 10:52:14.694 | INFO | "huge_shared" : " 27 GB", | 20260506 10:52:14.694 | INFO | "malloc" : " 14 GB", | 20260506 10:52:14.694 | INFO | "pinned" : " 29 GB" | 20260506 10:52:14.694 | INFO | } | 20260506 10:52:14.694 | INFO | Perf report for the last 5.13939 seconds: | 20260506 10:52:14.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:52:14.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:14.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:14.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:14.694 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 10:52:14.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:14.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:14.695 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:52:14.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:14.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:14.695 | INFO | Uploaded 36GiB to GPU in 1.6065s (22.4089GiB/s) | 20260506 10:52:14.695 | INFO | Downloaded 34GiB to GPU in 1.52347s (22.3174GiB/s) | 20260506 10:52:19.833 | INFO | Analyzing chunk frames = [781312,781824) | 20260506 10:52:19.833 | INFO | MemInfo: { | 20260506 10:52:19.833 | INFO | "gpu" : " 17 GB", | 20260506 10:52:19.833 | INFO | "huge" : "", | 20260506 10:52:19.833 | INFO | "huge_shared" : " 27 GB", | 20260506 10:52:19.833 | INFO | "malloc" : " 14 GB", | 20260506 10:52:19.833 | INFO | "pinned" : " 29 GB" | 20260506 10:52:19.833 | INFO | } | 20260506 10:52:19.834 | INFO | Perf report for the last 5.13911 seconds: | 20260506 10:52:19.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:52:19.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:19.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:19.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:19.834 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:52:19.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:19.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:19.834 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 10:52:19.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:19.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:19.834 | INFO | Uploaded 36GiB to GPU in 1.60618s (22.4134GiB/s) | 20260506 10:52:19.834 | INFO | Downloaded 34GiB to GPU in 1.52378s (22.3129GiB/s) | 20260506 10:52:24.973 | INFO | Analyzing chunk frames = [781824,782336) | 20260506 10:52:24.973 | INFO | MemInfo: { | 20260506 10:52:24.973 | INFO | "gpu" : " 17 GB", | 20260506 10:52:24.973 | INFO | "huge" : "", | 20260506 10:52:24.973 | INFO | "huge_shared" : " 27 GB", | 20260506 10:52:24.973 | INFO | "malloc" : " 14 GB", | 20260506 10:52:24.973 | INFO | "pinned" : " 29 GB" | 20260506 10:52:24.973 | INFO | } | 20260506 10:52:24.973 | INFO | Perf report for the last 5.13946 seconds: | 20260506 10:52:24.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:52:24.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:24.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:24.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:24.973 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:52:24.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:24.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:24.973 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 10:52:24.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:24.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:24.973 | INFO | Uploaded 36GiB to GPU in 1.60639s (22.4104GiB/s) | 20260506 10:52:24.973 | INFO | Downloaded 34GiB to GPU in 1.52352s (22.3168GiB/s) | 20260506 10:52:25.071 | INFO | Starting a round of Baz IO | 20260506 10:52:27.975 | INFO | Populating RTMetrics report | 20260506 10:52:27.975 | INFO | RTMetrics report populated | 20260506 10:52:27.975 | INFO | Opening tmp RTMetrics json file | 20260506 10:52:27.990 | INFO | Serializing RTMetrics to json | 20260506 10:52:28.065 | INFO | Serialization done, swapping tmp file now... | 20260506 10:52:28.106 | INFO | RTMetrics done | 20260506 10:52:28.297 | INFO | Finished a round of Baz IO: Wrote 2.022 GiB in 3.226 seconds | 20260506 10:52:28.297 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.13% / 0.04% / 53.83% | 20260506 10:52:28.297 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.05622% / 7.15256e-05% / 2.78155e-05% | 20260506 10:52:28.297 | INFO | Baz IO overall: | 20260506 10:52:28.297 | INFO | BytesWritten : 363271994913 | 20260506 10:52:28.297 | INFO | NumEvents : 174940493864 | 20260506 10:52:30.112 | INFO | Analyzing chunk frames = [782336,782848) | 20260506 10:52:30.112 | INFO | MemInfo: { | 20260506 10:52:30.112 | INFO | "gpu" : " 17 GB", | 20260506 10:52:30.112 | INFO | "huge" : "", | 20260506 10:52:30.112 | INFO | "huge_shared" : " 27 GB", | 20260506 10:52:30.112 | INFO | "malloc" : " 14 GB", | 20260506 10:52:30.112 | INFO | "pinned" : " 29 GB" | 20260506 10:52:30.112 | INFO | } | 20260506 10:52:30.112 | INFO | Perf report for the last 5.13919 seconds: | 20260506 10:52:30.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:52:30.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:30.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:30.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:30.112 | INFO | ANALYSIS : 57.4%, 2.8, 85.3ms, 0 | 20260506 10:52:30.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:30.112 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260506 10:52:30.112 | INFO | PRE_HQ : 56.4%, 2.1, 60.0ms, 0 | 20260506 10:52:30.112 | INFO | BAZWRITER : 62.4%, 4.8, 152.6ms, 0 | 20260506 10:52:30.112 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.1ms, 0 | 20260506 10:52:30.113 | INFO | Uploaded 36GiB to GPU in 1.67854s (21.4473GiB/s) | 20260506 10:52:30.113 | INFO | Downloaded 38GiB to GPU in 1.74295s (21.8022GiB/s) | 20260506 10:52:35.251 | INFO | Analyzing chunk frames = [782848,783360) | 20260506 10:52:35.251 | INFO | MemInfo: { | 20260506 10:52:35.251 | INFO | "gpu" : " 17 GB", | 20260506 10:52:35.251 | INFO | "huge" : "", | 20260506 10:52:35.251 | INFO | "huge_shared" : " 27 GB", | 20260506 10:52:35.251 | INFO | "malloc" : " 14 GB", | 20260506 10:52:35.251 | INFO | "pinned" : " 29 GB" | 20260506 10:52:35.251 | INFO | } | 20260506 10:52:35.251 | INFO | Perf report for the last 5.13913 seconds: | 20260506 10:52:35.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:52:35.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:35.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:35.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:35.251 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 10:52:35.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:35.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:35.252 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:52:35.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:35.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:35.252 | INFO | Uploaded 36GiB to GPU in 1.60367s (22.4485GiB/s) | 20260506 10:52:35.252 | INFO | Downloaded 34GiB to GPU in 1.52143s (22.3474GiB/s) | 20260506 10:52:40.391 | INFO | Analyzing chunk frames = [783360,783872) | 20260506 10:52:40.391 | INFO | MemInfo: { | 20260506 10:52:40.391 | INFO | "gpu" : " 17 GB", | 20260506 10:52:40.391 | INFO | "huge" : "", | 20260506 10:52:40.391 | INFO | "huge_shared" : " 27 GB", | 20260506 10:52:40.391 | INFO | "malloc" : " 14 GB", | 20260506 10:52:40.391 | INFO | "pinned" : " 29 GB" | 20260506 10:52:40.391 | INFO | } | 20260506 10:52:40.391 | INFO | Perf report for the last 5.13971 seconds: | 20260506 10:52:40.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:52:40.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:40.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:40.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:40.391 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260506 10:52:40.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:40.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:40.391 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:52:40.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:40.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:40.391 | INFO | Uploaded 36GiB to GPU in 1.6082s (22.3853GiB/s) | 20260506 10:52:40.391 | INFO | Downloaded 34GiB to GPU in 1.52574s (22.2843GiB/s) | 20260506 10:52:45.530 | INFO | Analyzing chunk frames = [783872,784384) | 20260506 10:52:45.530 | INFO | MemInfo: { | 20260506 10:52:45.530 | INFO | "gpu" : " 17 GB", | 20260506 10:52:45.530 | INFO | "huge" : "", | 20260506 10:52:45.530 | INFO | "huge_shared" : " 27 GB", | 20260506 10:52:45.530 | INFO | "malloc" : " 14 GB", | 20260506 10:52:45.530 | INFO | "pinned" : " 29 GB" | 20260506 10:52:45.530 | INFO | } | 20260506 10:52:45.530 | INFO | Perf report for the last 5.13931 seconds: | 20260506 10:52:45.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:52:45.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:45.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:45.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:45.530 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 10:52:45.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:45.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:45.530 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:52:45.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:45.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:45.531 | INFO | Uploaded 36GiB to GPU in 1.60753s (22.3945GiB/s) | 20260506 10:52:45.531 | INFO | Downloaded 34GiB to GPU in 1.52475s (22.2988GiB/s) | 20260506 10:52:50.669 | INFO | Analyzing chunk frames = [784384,784896) | 20260506 10:52:50.669 | INFO | MemInfo: { | 20260506 10:52:50.669 | INFO | "gpu" : " 17 GB", | 20260506 10:52:50.669 | INFO | "huge" : "", | 20260506 10:52:50.670 | INFO | "huge_shared" : " 27 GB", | 20260506 10:52:50.670 | INFO | "malloc" : " 14 GB", | 20260506 10:52:50.670 | INFO | "pinned" : " 29 GB" | 20260506 10:52:50.670 | INFO | } | 20260506 10:52:50.670 | INFO | Perf report for the last 5.13929 seconds: | 20260506 10:52:50.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:52:50.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:50.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:50.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:50.670 | INFO | ANALYSIS : 56.3%, 3.0, 87.7ms, 0 | 20260506 10:52:50.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:50.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:50.670 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:52:50.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:50.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:50.670 | INFO | Uploaded 36GiB to GPU in 1.60638s (22.4106GiB/s) | 20260506 10:52:50.670 | INFO | Downloaded 34GiB to GPU in 1.52365s (22.3148GiB/s) | 20260506 10:52:55.809 | INFO | Analyzing chunk frames = [784896,785408) | 20260506 10:52:55.809 | INFO | MemInfo: { | 20260506 10:52:55.809 | INFO | "gpu" : " 17 GB", | 20260506 10:52:55.809 | INFO | "huge" : "", | 20260506 10:52:55.809 | INFO | "huge_shared" : " 27 GB", | 20260506 10:52:55.809 | INFO | "malloc" : " 14 GB", | 20260506 10:52:55.809 | INFO | "pinned" : " 29 GB" | 20260506 10:52:55.809 | INFO | } | 20260506 10:52:55.809 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:52:55.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:52:55.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:55.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:55.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:55.809 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 10:52:55.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:55.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:55.809 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 10:52:55.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:55.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:52:55.809 | INFO | Uploaded 36GiB to GPU in 1.60617s (22.4136GiB/s) | 20260506 10:52:55.809 | INFO | Downloaded 34GiB to GPU in 1.52278s (22.3276GiB/s) | 20260506 10:53:00.948 | INFO | Analyzing chunk frames = [785408,785920) | 20260506 10:53:00.948 | INFO | MemInfo: { | 20260506 10:53:00.948 | INFO | "gpu" : " 17 GB", | 20260506 10:53:00.948 | INFO | "huge" : "", | 20260506 10:53:00.948 | INFO | "huge_shared" : " 27 GB", | 20260506 10:53:00.948 | INFO | "malloc" : " 14 GB", | 20260506 10:53:00.948 | INFO | "pinned" : " 29 GB" | 20260506 10:53:00.948 | INFO | } | 20260506 10:53:00.948 | INFO | Perf report for the last 5.13893 seconds: | 20260506 10:53:00.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:53:00.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:00.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:00.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:00.948 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 10:53:00.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:00.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:00.948 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:53:00.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:00.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:00.948 | INFO | Uploaded 36GiB to GPU in 1.60691s (22.4032GiB/s) | 20260506 10:53:00.949 | INFO | Downloaded 34GiB to GPU in 1.52487s (22.297GiB/s) | 20260506 10:53:06.087 | INFO | Analyzing chunk frames = [785920,786432) | 20260506 10:53:06.087 | INFO | MemInfo: { | 20260506 10:53:06.087 | INFO | "gpu" : " 17 GB", | 20260506 10:53:06.087 | INFO | "huge" : "", | 20260506 10:53:06.087 | INFO | "huge_shared" : " 27 GB", | 20260506 10:53:06.087 | INFO | "malloc" : " 14 GB", | 20260506 10:53:06.088 | INFO | "pinned" : " 29 GB" | 20260506 10:53:06.088 | INFO | } | 20260506 10:53:06.088 | INFO | Perf report for the last 5.13965 seconds: | 20260506 10:53:06.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:53:06.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:06.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:06.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:06.088 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 10:53:06.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:06.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:06.088 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 10:53:06.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:06.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:06.088 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:53:06.088 | INFO | Total mem usage: 16.5 GiB | 20260506 10:53:06.088 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:53:06.088 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -0.843%) | 20260506 10:53:06.088 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:53:06.088 | INFO | High Water Breakdown: | 20260506 10:53:06.088 | INFO | 61.2% : 10 GiB ( 99.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:53:06.088 | INFO | 38.6% : 6.33 GiB ( 94.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:53:06.088 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:53:06.088 | INFO | ------------------------------------------ | 20260506 10:53:06.088 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:53:06.088 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:53:06.088 | INFO | Total mem usage: 26.5 GiB | 20260506 10:53:06.088 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:53:06.088 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:53:06.088 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:53:06.088 | INFO | High Water Breakdown: | 20260506 10:53:06.088 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:53:06.088 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:53:06.088 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:53:06.088 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:53:06.088 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:53:06.088 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:53:06.088 | INFO | Uploaded 36GiB to GPU in 1.60659s (22.4076GiB/s) | 20260506 10:53:06.089 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:53:06.089 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:53:06.089 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:53:06.089 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:53:06.089 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:53:06.089 | INFO | Downloaded 34GiB to GPU in 1.52441s (22.3038GiB/s) | 20260506 10:53:06.089 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:53:06.089 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:53:06.089 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:53:06.089 | INFO | ------------------------------------------ | 20260506 10:53:06.089 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:53:06.089 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:53:06.089 | INFO | Total mem usage: 25.6 GiB | 20260506 10:53:06.089 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:53:06.089 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:53:06.089 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:53:06.089 | INFO | High Water Breakdown: | 20260506 10:53:06.089 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:53:06.089 | INFO | ------------------------------------------ | 20260506 10:53:06.089 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:53:06.089 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:53:06.089 | INFO | Total mem usage: 15.7 GiB | 20260506 10:53:06.089 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:53:06.089 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:53:06.089 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:53:06.089 | INFO | High Water Breakdown: | 20260506 10:53:06.089 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:53:06.089 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:53:06.089 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:53:06.089 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:53:06.089 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:53:06.089 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:53:06.089 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:53:06.089 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:53:06.089 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:53:06.089 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:53:06.089 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:53:06.089 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:53:06.089 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:53:06.089 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:53:06.089 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:53:06.089 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:53:06.089 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:53:06.089 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:53:06.089 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:53:06.089 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:53:06.089 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:53:06.089 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:53:06.089 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:53:06.089 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:53:06.089 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:53:06.089 | INFO | ------------------------------------------ | 20260506 10:53:06.089 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:53:06.089 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:53:06.089 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:53:06.089 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:53:06.089 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 10:53:06.089 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.31e-03 / 0.00e+00 | 20260506 10:53:06.089 | INFO | PerfReport: 10.97%, Baseline -- 3.30e+00 / -nan, 1.68e-02 / 0.00e+00 | 20260506 10:53:06.089 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260506 10:53:06.089 | INFO | PerfReport: 16.07%, DME -- 3.86e+01 / -nan, 3.09e+00 / 0.00e+00 | 20260506 10:53:06.089 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.72e+01 / -nan, 4.21e-02 / 0.00e+00 | 20260506 10:53:06.089 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.21e+00 / -nan, 1.26e-01 / 0.00e+00 | 20260506 10:53:06.089 | INFO | PerfReport: 3.62%, Metrics -- 1.09e+00 / -nan, 1.69e-01 / 0.00e+00 | 20260506 10:53:06.089 | INFO | PerfReport: 1.40%, PulseSerializing -- 4.20e-01 / -nan, 5.83e-02 / 0.00e+00 | 20260506 10:53:06.089 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:53:06.089 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:53:06.089 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:53:06.089 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 10:53:06.089 | INFO | PerfReport: 18.74%, Upload -- 1.68e+01 / 0.00e+00, 4.99e-01 / 0.00e+00 | 20260506 10:53:06.089 | INFO | PerfReport: 18.00%, Download -- 1.62e+01 / 0.00e+00, 8.57e-01 / 0.00e+00 | 20260506 10:53:06.089 | INFO | PerfReport: 33.47%, Compute -- 3.00e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 10:53:06.089 | INFO | PerfReport: 0.57%, LockUpload -- 5.08e-01 / 0.00e+00, 3.63e+00 / 0.00e+00 | 20260506 10:53:06.089 | INFO | PerfReport: 0.00%, LockDownload -- 4.88e-04 / 0.00e+00, 1.17e-04 / 0.00e+00 | 20260506 10:53:06.089 | INFO | PerfReport: 29.20%, LockCompute -- 2.62e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 10:53:06.089 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260506 10:53:06.185 | INFO | Starting a round of Baz IO | 20260506 10:53:09.089 | INFO | Populating RTMetrics report | 20260506 10:53:09.089 | INFO | RTMetrics report populated | 20260506 10:53:09.089 | INFO | Opening tmp RTMetrics json file | 20260506 10:53:09.161 | INFO | Serializing RTMetrics to json | 20260506 10:53:09.184 | INFO | Serialization done, swapping tmp file now... | 20260506 10:53:09.187 | INFO | RTMetrics done | 20260506 10:53:09.751 | INFO | Finished a round of Baz IO: Wrote 2.022 GiB in 3.565 seconds | 20260506 10:53:09.751 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.08% / 0.04% / 53.87% | 20260506 10:53:09.751 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.03344% / 0.000103315% / 3.57628e-05% | 20260506 10:53:09.751 | INFO | Baz IO overall: | 20260506 10:53:09.751 | INFO | BytesWritten : 365443265630 | 20260506 10:53:09.751 | INFO | NumEvents : 175993787874 | 20260506 10:53:11.227 | INFO | Analyzing chunk frames = [786432,786944) | 20260506 10:53:11.227 | INFO | MemInfo: { | 20260506 10:53:11.227 | INFO | "gpu" : " 17 GB", | 20260506 10:53:11.227 | INFO | "huge" : "", | 20260506 10:53:11.227 | INFO | "huge_shared" : " 27 GB", | 20260506 10:53:11.227 | INFO | "malloc" : " 14 GB", | 20260506 10:53:11.227 | INFO | "pinned" : " 29 GB" | 20260506 10:53:11.227 | INFO | } | 20260506 10:53:11.227 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:53:11.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:53:11.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:11.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:11.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:11.227 | INFO | ANALYSIS : 57.4%, 2.8, 85.4ms, 0 | 20260506 10:53:11.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:11.227 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260506 10:53:11.227 | INFO | PRE_HQ : 56.3%, 2.0, 59.7ms, 0 | 20260506 10:53:11.227 | INFO | BAZWRITER : 68.9%, 4.8, 170.0ms, 0 | 20260506 10:53:11.227 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 10:53:11.228 | INFO | Uploaded 36GiB to GPU in 1.67966s (21.4329GiB/s) | 20260506 10:53:11.228 | INFO | Downloaded 38GiB to GPU in 1.74388s (21.7904GiB/s) | 20260506 10:53:16.366 | INFO | Analyzing chunk frames = [786944,787456) | 20260506 10:53:16.366 | INFO | MemInfo: { | 20260506 10:53:16.366 | INFO | "gpu" : " 17 GB", | 20260506 10:53:16.366 | INFO | "huge" : "", | 20260506 10:53:16.366 | INFO | "huge_shared" : " 27 GB", | 20260506 10:53:16.366 | INFO | "malloc" : " 14 GB", | 20260506 10:53:16.366 | INFO | "pinned" : " 29 GB" | 20260506 10:53:16.366 | INFO | } | 20260506 10:53:16.366 | INFO | Perf report for the last 5.13935 seconds: | 20260506 10:53:16.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:53:16.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:16.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:16.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:16.366 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 10:53:16.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:16.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:16.367 | INFO | PRE_HQ : 29.1%, 1.0, 15.2ms, 0 | 20260506 10:53:16.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:16.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:16.367 | INFO | Uploaded 36GiB to GPU in 1.60333s (22.4533GiB/s) | 20260506 10:53:16.367 | INFO | Downloaded 34GiB to GPU in 1.51989s (22.37GiB/s) | 20260506 10:53:21.506 | INFO | Analyzing chunk frames = [787456,787968) | 20260506 10:53:21.506 | INFO | MemInfo: { | 20260506 10:53:21.506 | INFO | "gpu" : " 17 GB", | 20260506 10:53:21.506 | INFO | "huge" : "", | 20260506 10:53:21.506 | INFO | "huge_shared" : " 27 GB", | 20260506 10:53:21.506 | INFO | "malloc" : " 14 GB", | 20260506 10:53:21.506 | INFO | "pinned" : " 29 GB" | 20260506 10:53:21.506 | INFO | } | 20260506 10:53:21.506 | INFO | Perf report for the last 5.13948 seconds: | 20260506 10:53:21.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:53:21.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:21.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:21.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:21.506 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260506 10:53:21.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:21.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:21.506 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:53:21.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:21.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:21.506 | INFO | Uploaded 36GiB to GPU in 1.60807s (22.387GiB/s) | 20260506 10:53:21.506 | INFO | Downloaded 34GiB to GPU in 1.52592s (22.2817GiB/s) | 20260506 10:53:26.645 | INFO | Analyzing chunk frames = [787968,788480) | 20260506 10:53:26.645 | INFO | MemInfo: { | 20260506 10:53:26.645 | INFO | "gpu" : " 17 GB", | 20260506 10:53:26.645 | INFO | "huge" : "", | 20260506 10:53:26.645 | INFO | "huge_shared" : " 27 GB", | 20260506 10:53:26.645 | INFO | "malloc" : " 14 GB", | 20260506 10:53:26.645 | INFO | "pinned" : " 29 GB" | 20260506 10:53:26.645 | INFO | } | 20260506 10:53:26.645 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:53:26.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:53:26.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:26.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:26.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:26.645 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 10:53:26.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:26.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:26.645 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 10:53:26.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:26.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:26.646 | INFO | Uploaded 36GiB to GPU in 1.6071s (22.4006GiB/s) | 20260506 10:53:26.646 | INFO | Downloaded 34GiB to GPU in 1.52492s (22.2962GiB/s) | 20260506 10:53:31.785 | INFO | Analyzing chunk frames = [788480,788992) | 20260506 10:53:31.785 | INFO | MemInfo: { | 20260506 10:53:31.785 | INFO | "gpu" : " 17 GB", | 20260506 10:53:31.785 | INFO | "huge" : "", | 20260506 10:53:31.785 | INFO | "huge_shared" : " 27 GB", | 20260506 10:53:31.785 | INFO | "malloc" : " 14 GB", | 20260506 10:53:31.785 | INFO | "pinned" : " 29 GB" | 20260506 10:53:31.785 | INFO | } | 20260506 10:53:31.785 | INFO | Perf report for the last 5.13969 seconds: | 20260506 10:53:31.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:53:31.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:31.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:31.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:31.785 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:53:31.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:31.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:31.785 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:53:31.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:31.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:31.785 | INFO | Uploaded 36GiB to GPU in 1.6061s (22.4145GiB/s) | 20260506 10:53:31.785 | INFO | Downloaded 34GiB to GPU in 1.52295s (22.325GiB/s) | 20260506 10:53:36.924 | INFO | Analyzing chunk frames = [788992,789504) | 20260506 10:53:36.924 | INFO | MemInfo: { | 20260506 10:53:36.924 | INFO | "gpu" : " 17 GB", | 20260506 10:53:36.924 | INFO | "huge" : "", | 20260506 10:53:36.924 | INFO | "huge_shared" : " 27 GB", | 20260506 10:53:36.924 | INFO | "malloc" : " 14 GB", | 20260506 10:53:36.924 | INFO | "pinned" : " 29 GB" | 20260506 10:53:36.924 | INFO | } | 20260506 10:53:36.924 | INFO | Perf report for the last 5.13899 seconds: | 20260506 10:53:36.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:53:36.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:36.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:36.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:36.924 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 10:53:36.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:36.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:36.924 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 10:53:36.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:36.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:36.924 | INFO | Uploaded 36GiB to GPU in 1.60573s (22.4197GiB/s) | 20260506 10:53:36.924 | INFO | Downloaded 34GiB to GPU in 1.52275s (22.3281GiB/s) | 20260506 10:53:42.063 | INFO | Analyzing chunk frames = [789504,790016) | 20260506 10:53:42.063 | INFO | MemInfo: { | 20260506 10:53:42.063 | INFO | "gpu" : " 17 GB", | 20260506 10:53:42.063 | INFO | "huge" : "", | 20260506 10:53:42.063 | INFO | "huge_shared" : " 27 GB", | 20260506 10:53:42.063 | INFO | "malloc" : " 14 GB", | 20260506 10:53:42.063 | INFO | "pinned" : " 29 GB" | 20260506 10:53:42.063 | INFO | } | 20260506 10:53:42.063 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:53:42.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:53:42.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:42.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:42.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:42.063 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 10:53:42.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:42.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:42.063 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 10:53:42.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:42.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:42.064 | INFO | Uploaded 36GiB to GPU in 1.60594s (22.4167GiB/s) | 20260506 10:53:42.064 | INFO | Downloaded 34GiB to GPU in 1.52359s (22.3157GiB/s) | 20260506 10:53:47.202 | INFO | Analyzing chunk frames = [790016,790528) | 20260506 10:53:47.202 | INFO | MemInfo: { | 20260506 10:53:47.202 | INFO | "gpu" : " 17 GB", | 20260506 10:53:47.202 | INFO | "huge" : "", | 20260506 10:53:47.202 | INFO | "huge_shared" : " 27 GB", | 20260506 10:53:47.202 | INFO | "malloc" : " 14 GB", | 20260506 10:53:47.202 | INFO | "pinned" : " 29 GB" | 20260506 10:53:47.202 | INFO | } | 20260506 10:53:47.202 | INFO | Perf report for the last 5.13919 seconds: | 20260506 10:53:47.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:53:47.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:47.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:47.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:47.202 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:53:47.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:47.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:47.203 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 10:53:47.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:47.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:47.203 | INFO | Uploaded 36GiB to GPU in 1.60664s (22.407GiB/s) | 20260506 10:53:47.203 | INFO | Downloaded 34GiB to GPU in 1.52512s (22.2933GiB/s) | 20260506 10:53:47.300 | INFO | Starting a round of Baz IO | 20260506 10:53:50.209 | INFO | Populating RTMetrics report | 20260506 10:53:50.209 | INFO | RTMetrics report populated | 20260506 10:53:50.209 | INFO | Opening tmp RTMetrics json file | 20260506 10:53:50.216 | INFO | Serializing RTMetrics to json | 20260506 10:53:50.220 | INFO | Serialization done, swapping tmp file now... | 20260506 10:53:50.220 | INFO | RTMetrics done | 20260506 10:53:50.836 | INFO | Finished a round of Baz IO: Wrote 2.023 GiB in 3.536 seconds | 20260506 10:53:50.836 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.04% / 0.04% / 53.92% | 20260506 10:53:50.836 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.00201% / 5.96046e-05% / 3.57628e-05% | 20260506 10:53:50.836 | INFO | Baz IO overall: | 20260506 10:53:50.836 | INFO | BytesWritten : 367615150853 | 20260506 10:53:50.836 | INFO | NumEvents : 177047067857 | 20260506 10:53:52.342 | INFO | Analyzing chunk frames = [790528,791040) | 20260506 10:53:52.342 | INFO | MemInfo: { | 20260506 10:53:52.342 | INFO | "gpu" : " 17 GB", | 20260506 10:53:52.342 | INFO | "huge" : "", | 20260506 10:53:52.342 | INFO | "huge_shared" : " 27 GB", | 20260506 10:53:52.342 | INFO | "malloc" : " 14 GB", | 20260506 10:53:52.342 | INFO | "pinned" : " 29 GB" | 20260506 10:53:52.342 | INFO | } | 20260506 10:53:52.342 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:53:52.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:53:52.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:52.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:52.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:52.342 | INFO | ANALYSIS : 57.5%, 2.9, 86.1ms, 0 | 20260506 10:53:52.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:52.342 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 10:53:52.342 | INFO | PRE_HQ : 56.4%, 2.1, 60.3ms, 0 | 20260506 10:53:52.342 | INFO | BAZWRITER : 68.4%, 4.7, 165.7ms, 0 | 20260506 10:53:52.342 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 10:53:52.342 | INFO | Uploaded 36GiB to GPU in 1.6709s (21.5452GiB/s) | 20260506 10:53:52.342 | INFO | Downloaded 38GiB to GPU in 1.7362s (21.8869GiB/s) | 20260506 10:53:57.481 | INFO | Analyzing chunk frames = [791040,791552) | 20260506 10:53:57.481 | INFO | MemInfo: { | 20260506 10:53:57.481 | INFO | "gpu" : " 17 GB", | 20260506 10:53:57.481 | INFO | "huge" : "", | 20260506 10:53:57.481 | INFO | "huge_shared" : " 27 GB", | 20260506 10:53:57.481 | INFO | "malloc" : " 14 GB", | 20260506 10:53:57.481 | INFO | "pinned" : " 29 GB" | 20260506 10:53:57.481 | INFO | } | 20260506 10:53:57.481 | INFO | Perf report for the last 5.13923 seconds: | 20260506 10:53:57.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:53:57.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:57.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:57.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:57.481 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 10:53:57.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:57.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:57.481 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 10:53:57.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:57.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:53:57.481 | INFO | Uploaded 36GiB to GPU in 1.60341s (22.4521GiB/s) | 20260506 10:53:57.481 | INFO | Downloaded 34GiB to GPU in 1.52034s (22.3634GiB/s) | 20260506 10:54:02.620 | INFO | Analyzing chunk frames = [791552,792064) | 20260506 10:54:02.620 | INFO | MemInfo: { | 20260506 10:54:02.620 | INFO | "gpu" : " 17 GB", | 20260506 10:54:02.620 | INFO | "huge" : "", | 20260506 10:54:02.620 | INFO | "huge_shared" : " 27 GB", | 20260506 10:54:02.621 | INFO | "malloc" : " 14 GB", | 20260506 10:54:02.621 | INFO | "pinned" : " 29 GB" | 20260506 10:54:02.621 | INFO | } | 20260506 10:54:02.621 | INFO | Perf report for the last 5.13953 seconds: | 20260506 10:54:02.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:54:02.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:02.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:02.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:02.621 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260506 10:54:02.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:02.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:02.621 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:54:02.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:02.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:02.621 | INFO | Uploaded 36GiB to GPU in 1.608s (22.3881GiB/s) | 20260506 10:54:02.621 | INFO | Downloaded 34GiB to GPU in 1.5256s (22.2864GiB/s) | 20260506 10:54:07.760 | INFO | Analyzing chunk frames = [792064,792576) | 20260506 10:54:07.760 | INFO | MemInfo: { | 20260506 10:54:07.760 | INFO | "gpu" : " 17 GB", | 20260506 10:54:07.760 | INFO | "huge" : "", | 20260506 10:54:07.760 | INFO | "huge_shared" : " 27 GB", | 20260506 10:54:07.760 | INFO | "malloc" : " 14 GB", | 20260506 10:54:07.760 | INFO | "pinned" : " 29 GB" | 20260506 10:54:07.760 | INFO | } | 20260506 10:54:07.760 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:54:07.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:54:07.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:07.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:07.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:07.760 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 10:54:07.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:07.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:07.760 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:54:07.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:07.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:07.760 | INFO | Uploaded 36GiB to GPU in 1.60741s (22.3962GiB/s) | 20260506 10:54:07.760 | INFO | Downloaded 34GiB to GPU in 1.52515s (22.2929GiB/s) | 20260506 10:54:12.899 | INFO | Analyzing chunk frames = [792576,793088) | 20260506 10:54:12.899 | INFO | MemInfo: { | 20260506 10:54:12.899 | INFO | "gpu" : " 17 GB", | 20260506 10:54:12.899 | INFO | "huge" : "", | 20260506 10:54:12.899 | INFO | "huge_shared" : " 27 GB", | 20260506 10:54:12.899 | INFO | "malloc" : " 14 GB", | 20260506 10:54:12.899 | INFO | "pinned" : " 29 GB" | 20260506 10:54:12.899 | INFO | } | 20260506 10:54:12.899 | INFO | Perf report for the last 5.13921 seconds: | 20260506 10:54:12.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:54:12.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:12.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:12.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:12.899 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:54:12.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:12.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:12.899 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:54:12.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:12.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:12.899 | INFO | Uploaded 36GiB to GPU in 1.60545s (22.4236GiB/s) | 20260506 10:54:12.900 | INFO | Downloaded 34GiB to GPU in 1.52254s (22.3311GiB/s) | 20260506 10:54:18.038 | INFO | Analyzing chunk frames = [793088,793600) | 20260506 10:54:18.038 | INFO | MemInfo: { | 20260506 10:54:18.038 | INFO | "gpu" : " 17 GB", | 20260506 10:54:18.038 | INFO | "huge" : "", | 20260506 10:54:18.038 | INFO | "huge_shared" : " 27 GB", | 20260506 10:54:18.038 | INFO | "malloc" : " 14 GB", | 20260506 10:54:18.038 | INFO | "pinned" : " 29 GB" | 20260506 10:54:18.038 | INFO | } | 20260506 10:54:18.039 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:54:18.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:54:18.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:18.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:18.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:18.039 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 10:54:18.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:18.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:18.039 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:54:18.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:18.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:18.039 | INFO | Uploaded 36GiB to GPU in 1.60726s (22.3984GiB/s) | 20260506 10:54:18.039 | INFO | Downloaded 34GiB to GPU in 1.52366s (22.3146GiB/s) | 20260506 10:54:23.178 | INFO | Analyzing chunk frames = [793600,794112) | 20260506 10:54:23.178 | INFO | MemInfo: { | 20260506 10:54:23.178 | INFO | "gpu" : " 17 GB", | 20260506 10:54:23.178 | INFO | "huge" : "", | 20260506 10:54:23.178 | INFO | "huge_shared" : " 27 GB", | 20260506 10:54:23.178 | INFO | "malloc" : " 14 GB", | 20260506 10:54:23.178 | INFO | "pinned" : " 29 GB" | 20260506 10:54:23.178 | INFO | } | 20260506 10:54:23.178 | INFO | Perf report for the last 5.13945 seconds: | 20260506 10:54:23.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:54:23.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:23.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:23.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:23.178 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:54:23.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:23.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:23.178 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:54:23.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:23.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:23.179 | INFO | Uploaded 36GiB to GPU in 1.60619s (22.4133GiB/s) | 20260506 10:54:23.179 | INFO | Downloaded 34GiB to GPU in 1.52365s (22.3148GiB/s) | 20260506 10:54:28.317 | INFO | Analyzing chunk frames = [794112,794624) | 20260506 10:54:28.317 | INFO | MemInfo: { | 20260506 10:54:28.317 | INFO | "gpu" : " 17 GB", | 20260506 10:54:28.317 | INFO | "huge" : "", | 20260506 10:54:28.317 | INFO | "huge_shared" : " 27 GB", | 20260506 10:54:28.317 | INFO | "malloc" : " 14 GB", | 20260506 10:54:28.317 | INFO | "pinned" : " 29 GB" | 20260506 10:54:28.317 | INFO | } | 20260506 10:54:28.317 | INFO | Perf report for the last 5.13894 seconds: | 20260506 10:54:28.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:54:28.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:28.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:28.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:28.317 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:54:28.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:28.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:28.317 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:54:28.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:28.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:28.317 | INFO | Uploaded 36GiB to GPU in 1.60637s (22.4107GiB/s) | 20260506 10:54:28.317 | INFO | Downloaded 34GiB to GPU in 1.52415s (22.3075GiB/s) | 20260506 10:54:28.317 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:54:28.317 | INFO | Total mem usage: 16.5 GiB | 20260506 10:54:28.317 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:54:28.317 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -0.775%) | 20260506 10:54:28.317 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:54:28.317 | INFO | High Water Breakdown: | 20260506 10:54:28.317 | INFO | 61.3% : 10 GiB ( 99.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:54:28.317 | INFO | 38.6% : 6.33 GiB ( 94.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:54:28.317 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:54:28.317 | INFO | ------------------------------------------ | 20260506 10:54:28.317 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:54:28.317 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:54:28.317 | INFO | Total mem usage: 26.5 GiB | 20260506 10:54:28.317 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:54:28.317 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:54:28.317 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:54:28.317 | INFO | High Water Breakdown: | 20260506 10:54:28.317 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:54:28.317 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:54:28.317 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:54:28.317 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:54:28.317 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:54:28.317 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:54:28.317 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:54:28.317 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:54:28.317 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:54:28.317 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:54:28.317 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:54:28.317 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:54:28.317 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:54:28.317 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:54:28.317 | INFO | ------------------------------------------ | 20260506 10:54:28.317 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:54:28.317 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:54:28.317 | INFO | Total mem usage: 25.6 GiB | 20260506 10:54:28.317 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:54:28.317 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:54:28.317 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:54:28.318 | INFO | High Water Breakdown: | 20260506 10:54:28.318 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:54:28.318 | INFO | ------------------------------------------ | 20260506 10:54:28.318 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:54:28.318 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:54:28.318 | INFO | Total mem usage: 15.7 GiB | 20260506 10:54:28.318 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:54:28.318 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:54:28.318 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:54:28.318 | INFO | High Water Breakdown: | 20260506 10:54:28.318 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:54:28.318 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:54:28.318 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:54:28.318 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:54:28.318 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:54:28.318 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:54:28.318 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:54:28.318 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:54:28.318 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:54:28.318 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:54:28.318 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:54:28.318 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:54:28.318 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:54:28.318 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:54:28.318 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:54:28.318 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:54:28.318 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:54:28.318 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:54:28.318 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:54:28.318 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:54:28.318 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:54:28.318 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:54:28.318 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:54:28.318 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:54:28.318 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:54:28.318 | INFO | ------------------------------------------ | 20260506 10:54:28.318 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:54:28.318 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:54:28.319 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:54:28.319 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:54:28.319 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 10:54:28.319 | INFO | PerfReport: 0.29%, AggregateStats -- 8.67e-02 / -nan, 5.49e-03 / 0.00e+00 | 20260506 10:54:28.319 | INFO | PerfReport: 10.97%, Baseline -- 3.29e+00 / -nan, 1.65e-02 / 0.00e+00 | 20260506 10:54:28.319 | INFO | PerfReport: 3.07%, Histogram -- 9.21e-01 / -nan, 5.27e-02 / 0.00e+00 | 20260506 10:54:28.319 | INFO | PerfReport: 16.07%, DME -- 3.86e+01 / -nan, 2.97e+00 / 0.00e+00 | 20260506 10:54:28.319 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.72e+01 / -nan, 4.23e-02 / 0.00e+00 | 20260506 10:54:28.319 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.21e+00 / -nan, 1.26e-01 / 0.00e+00 | 20260506 10:54:28.319 | INFO | PerfReport: 3.62%, Metrics -- 1.09e+00 / -nan, 1.69e-01 / 0.00e+00 | 20260506 10:54:28.319 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.19e-01 / -nan, 5.83e-02 / 0.00e+00 | 20260506 10:54:28.319 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:54:28.319 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:54:28.319 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:54:28.319 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:54:28.319 | INFO | PerfReport: 18.74%, Upload -- 1.68e+01 / 0.00e+00, 4.96e-01 / 0.00e+00 | 20260506 10:54:28.319 | INFO | PerfReport: 17.99%, Download -- 1.62e+01 / 0.00e+00, 8.52e-01 / 0.00e+00 | 20260506 10:54:28.319 | INFO | PerfReport: 33.47%, Compute -- 3.00e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 10:54:28.319 | INFO | PerfReport: 0.57%, LockUpload -- 5.08e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 10:54:28.319 | INFO | PerfReport: 0.00%, LockDownload -- 4.76e-04 / 0.00e+00, 1.24e-04 / 0.00e+00 | 20260506 10:54:28.319 | INFO | PerfReport: 29.22%, LockCompute -- 2.62e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 10:54:28.319 | INFO | PerfReport: 0.02%, Misc -- 1.35e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260506 10:54:28.414 | INFO | Starting a round of Baz IO | 20260506 10:54:31.320 | INFO | Populating RTMetrics report | 20260506 10:54:31.320 | INFO | RTMetrics report populated | 20260506 10:54:31.320 | INFO | Opening tmp RTMetrics json file | 20260506 10:54:31.356 | INFO | Serializing RTMetrics to json | 20260506 10:54:31.367 | INFO | Serialization done, swapping tmp file now... | 20260506 10:54:31.384 | INFO | RTMetrics done | 20260506 10:54:31.707 | INFO | Finished a round of Baz IO: Wrote 2.025 GiB in 3.292 seconds | 20260506 10:54:31.707 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 46.00% / 0.04% / 53.96% | 20260506 10:54:31.707 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.99443% / 8.74201e-05% / 7.15256e-05% | 20260506 10:54:31.707 | INFO | Baz IO overall: | 20260506 10:54:31.707 | INFO | BytesWritten : 369789884344 | 20260506 10:54:31.707 | INFO | NumEvents : 178102303008 | 20260506 10:54:33.456 | INFO | Analyzing chunk frames = [794624,795136) | 20260506 10:54:33.456 | INFO | MemInfo: { | 20260506 10:54:33.456 | INFO | "gpu" : " 17 GB", | 20260506 10:54:33.456 | INFO | "huge" : "", | 20260506 10:54:33.456 | INFO | "huge_shared" : " 27 GB", | 20260506 10:54:33.456 | INFO | "malloc" : " 14 GB", | 20260506 10:54:33.456 | INFO | "pinned" : " 29 GB" | 20260506 10:54:33.456 | INFO | } | 20260506 10:54:33.456 | INFO | Perf report for the last 5.1393 seconds: | 20260506 10:54:33.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:54:33.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:33.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:33.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:33.456 | INFO | ANALYSIS : 57.4%, 2.8, 84.9ms, 0 | 20260506 10:54:33.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:33.456 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 10:54:33.456 | INFO | PRE_HQ : 56.4%, 2.1, 60.7ms, 0 | 20260506 10:54:33.456 | INFO | BAZWRITER : 63.6%, 4.8, 156.0ms, 0 | 20260506 10:54:33.456 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 10:54:33.457 | INFO | Uploaded 36GiB to GPU in 1.67483s (21.4948GiB/s) | 20260506 10:54:33.457 | INFO | Downloaded 38GiB to GPU in 1.74016s (21.8371GiB/s) | 20260506 10:54:38.596 | INFO | Analyzing chunk frames = [795136,795648) | 20260506 10:54:38.596 | INFO | MemInfo: { | 20260506 10:54:38.596 | INFO | "gpu" : " 17 GB", | 20260506 10:54:38.596 | INFO | "huge" : "", | 20260506 10:54:38.596 | INFO | "huge_shared" : " 27 GB", | 20260506 10:54:38.596 | INFO | "malloc" : " 14 GB", | 20260506 10:54:38.596 | INFO | "pinned" : " 29 GB" | 20260506 10:54:38.596 | INFO | } | 20260506 10:54:38.596 | INFO | Perf report for the last 5.13973 seconds: | 20260506 10:54:38.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:54:38.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:38.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:38.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:38.596 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 10:54:38.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:38.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:38.596 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:54:38.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:38.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:38.596 | INFO | Uploaded 36GiB to GPU in 1.60356s (22.4501GiB/s) | 20260506 10:54:38.596 | INFO | Downloaded 34GiB to GPU in 1.52087s (22.3557GiB/s) | 20260506 10:54:43.735 | INFO | Analyzing chunk frames = [795648,796160) | 20260506 10:54:43.735 | INFO | MemInfo: { | 20260506 10:54:43.735 | INFO | "gpu" : " 17 GB", | 20260506 10:54:43.735 | INFO | "huge" : "", | 20260506 10:54:43.735 | INFO | "huge_shared" : " 27 GB", | 20260506 10:54:43.735 | INFO | "malloc" : " 14 GB", | 20260506 10:54:43.735 | INFO | "pinned" : " 29 GB" | 20260506 10:54:43.735 | INFO | } | 20260506 10:54:43.735 | INFO | Perf report for the last 5.13923 seconds: | 20260506 10:54:43.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:54:43.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:43.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:43.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:43.735 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260506 10:54:43.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:43.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:43.735 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:54:43.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:43.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:43.736 | INFO | Uploaded 36GiB to GPU in 1.60749s (22.3952GiB/s) | 20260506 10:54:43.736 | INFO | Downloaded 34GiB to GPU in 1.52565s (22.2856GiB/s) | 20260506 10:54:48.874 | INFO | Analyzing chunk frames = [796160,796672) | 20260506 10:54:48.874 | INFO | MemInfo: { | 20260506 10:54:48.874 | INFO | "gpu" : " 17 GB", | 20260506 10:54:48.874 | INFO | "huge" : "", | 20260506 10:54:48.874 | INFO | "huge_shared" : " 27 GB", | 20260506 10:54:48.874 | INFO | "malloc" : " 14 GB", | 20260506 10:54:48.874 | INFO | "pinned" : " 29 GB" | 20260506 10:54:48.874 | INFO | } | 20260506 10:54:48.874 | INFO | Perf report for the last 5.13921 seconds: | 20260506 10:54:48.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:54:48.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:48.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:48.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:48.875 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260506 10:54:48.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:48.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:48.875 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:54:48.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:48.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:48.875 | INFO | Uploaded 36GiB to GPU in 1.6066s (22.4076GiB/s) | 20260506 10:54:48.875 | INFO | Downloaded 34GiB to GPU in 1.52585s (22.2826GiB/s) | 20260506 10:54:54.014 | INFO | Analyzing chunk frames = [796672,797184) | 20260506 10:54:54.014 | INFO | MemInfo: { | 20260506 10:54:54.014 | INFO | "gpu" : " 17 GB", | 20260506 10:54:54.014 | INFO | "huge" : "", | 20260506 10:54:54.014 | INFO | "huge_shared" : " 27 GB", | 20260506 10:54:54.014 | INFO | "malloc" : " 14 GB", | 20260506 10:54:54.014 | INFO | "pinned" : " 29 GB" | 20260506 10:54:54.014 | INFO | } | 20260506 10:54:54.014 | INFO | Perf report for the last 5.13949 seconds: | 20260506 10:54:54.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:54:54.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:54.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:54.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:54.014 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:54:54.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:54.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:54.014 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:54:54.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:54.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:54.014 | INFO | Uploaded 36GiB to GPU in 1.60516s (22.4277GiB/s) | 20260506 10:54:54.014 | INFO | Downloaded 34GiB to GPU in 1.52227s (22.3351GiB/s) | 20260506 10:54:59.153 | INFO | Analyzing chunk frames = [797184,797696) | 20260506 10:54:59.153 | INFO | MemInfo: { | 20260506 10:54:59.153 | INFO | "gpu" : " 17 GB", | 20260506 10:54:59.153 | INFO | "huge" : "", | 20260506 10:54:59.153 | INFO | "huge_shared" : " 27 GB", | 20260506 10:54:59.153 | INFO | "malloc" : " 14 GB", | 20260506 10:54:59.153 | INFO | "pinned" : " 29 GB" | 20260506 10:54:59.153 | INFO | } | 20260506 10:54:59.153 | INFO | Perf report for the last 5.13936 seconds: | 20260506 10:54:59.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:54:59.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:59.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:59.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:59.153 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 10:54:59.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:59.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:59.153 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:54:59.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:59.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:54:59.154 | INFO | Uploaded 36GiB to GPU in 1.60585s (22.418GiB/s) | 20260506 10:54:59.154 | INFO | Downloaded 34GiB to GPU in 1.52308s (22.3231GiB/s) | 20260506 10:55:04.292 | INFO | Analyzing chunk frames = [797696,798208) | 20260506 10:55:04.292 | INFO | MemInfo: { | 20260506 10:55:04.292 | INFO | "gpu" : " 17 GB", | 20260506 10:55:04.292 | INFO | "huge" : "", | 20260506 10:55:04.293 | INFO | "huge_shared" : " 27 GB", | 20260506 10:55:04.293 | INFO | "malloc" : " 14 GB", | 20260506 10:55:04.293 | INFO | "pinned" : " 29 GB" | 20260506 10:55:04.293 | INFO | } | 20260506 10:55:04.293 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:55:04.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:55:04.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:04.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:04.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:04.293 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:55:04.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:04.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:04.293 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:55:04.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:04.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:04.293 | INFO | Uploaded 36GiB to GPU in 1.60616s (22.4138GiB/s) | 20260506 10:55:04.293 | INFO | Downloaded 34GiB to GPU in 1.52423s (22.3063GiB/s) | 20260506 10:55:09.432 | INFO | Analyzing chunk frames = [798208,798720) | 20260506 10:55:09.432 | INFO | MemInfo: { | 20260506 10:55:09.432 | INFO | "gpu" : " 17 GB", | 20260506 10:55:09.432 | INFO | "huge" : "", | 20260506 10:55:09.432 | INFO | "huge_shared" : " 27 GB", | 20260506 10:55:09.432 | INFO | "malloc" : " 14 GB", | 20260506 10:55:09.432 | INFO | "pinned" : " 29 GB" | 20260506 10:55:09.432 | INFO | } | 20260506 10:55:09.432 | INFO | Perf report for the last 5.1393 seconds: | 20260506 10:55:09.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:55:09.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:09.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:09.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:09.432 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:55:09.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:09.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:09.432 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:55:09.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:09.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:09.432 | INFO | Uploaded 36GiB to GPU in 1.60662s (22.4073GiB/s) | 20260506 10:55:09.432 | INFO | Downloaded 34GiB to GPU in 1.52498s (22.2954GiB/s) | 20260506 10:55:09.529 | INFO | Starting a round of Baz IO | 20260506 10:55:12.435 | INFO | Populating RTMetrics report | 20260506 10:55:12.436 | INFO | RTMetrics report populated | 20260506 10:55:12.436 | INFO | Opening tmp RTMetrics json file | 20260506 10:55:12.441 | INFO | Serializing RTMetrics to json | 20260506 10:55:12.443 | INFO | Serialization done, swapping tmp file now... | 20260506 10:55:12.444 | INFO | RTMetrics done | 20260506 10:55:13.088 | INFO | Finished a round of Baz IO: Wrote 2.027 GiB in 3.558 seconds | 20260506 10:55:13.088 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.95% / 0.04% / 54.00% | 20260506 10:55:13.088 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 6.00039% / 6.75519e-05% / 5.16574e-05% | 20260506 10:55:13.088 | INFO | Baz IO overall: | 20260506 10:55:13.088 | INFO | BytesWritten : 371966018228 | 20260506 10:55:13.088 | INFO | NumEvents : 179158882911 | 20260506 10:55:14.571 | INFO | Analyzing chunk frames = [798720,799232) | 20260506 10:55:14.571 | INFO | MemInfo: { | 20260506 10:55:14.571 | INFO | "gpu" : " 17 GB", | 20260506 10:55:14.571 | INFO | "huge" : "", | 20260506 10:55:14.571 | INFO | "huge_shared" : " 27 GB", | 20260506 10:55:14.571 | INFO | "malloc" : " 14 GB", | 20260506 10:55:14.571 | INFO | "pinned" : " 29 GB" | 20260506 10:55:14.571 | INFO | } | 20260506 10:55:14.572 | INFO | Perf report for the last 5.13953 seconds: | 20260506 10:55:14.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:55:14.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:14.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:14.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:14.572 | INFO | ANALYSIS : 57.4%, 2.8, 85.6ms, 0 | 20260506 10:55:14.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:14.572 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 10:55:14.572 | INFO | PRE_HQ : 56.4%, 2.0, 59.9ms, 0 | 20260506 10:55:14.572 | INFO | BAZWRITER : 68.9%, 4.8, 169.4ms, 0 | 20260506 10:55:14.572 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 10:55:14.572 | INFO | Uploaded 36GiB to GPU in 1.67849s (21.4478GiB/s) | 20260506 10:55:14.572 | INFO | Downloaded 38GiB to GPU in 1.74317s (21.7993GiB/s) | 20260506 10:55:19.710 | INFO | Analyzing chunk frames = [799232,799744) | 20260506 10:55:19.710 | INFO | MemInfo: { | 20260506 10:55:19.710 | INFO | "gpu" : " 17 GB", | 20260506 10:55:19.710 | INFO | "huge" : "", | 20260506 10:55:19.711 | INFO | "huge_shared" : " 27 GB", | 20260506 10:55:19.711 | INFO | "malloc" : " 14 GB", | 20260506 10:55:19.711 | INFO | "pinned" : " 29 GB" | 20260506 10:55:19.711 | INFO | } | 20260506 10:55:19.711 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:55:19.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:55:19.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:19.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:19.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:19.711 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 10:55:19.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:19.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:19.711 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:55:19.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:19.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:19.711 | INFO | Uploaded 36GiB to GPU in 1.60383s (22.4463GiB/s) | 20260506 10:55:19.711 | INFO | Downloaded 34GiB to GPU in 1.52106s (22.3528GiB/s) | 20260506 10:55:24.850 | INFO | Analyzing chunk frames = [799744,800256) | 20260506 10:55:24.850 | INFO | MemInfo: { | 20260506 10:55:24.850 | INFO | "gpu" : " 17 GB", | 20260506 10:55:24.850 | INFO | "huge" : "", | 20260506 10:55:24.850 | INFO | "huge_shared" : " 27 GB", | 20260506 10:55:24.850 | INFO | "malloc" : " 14 GB", | 20260506 10:55:24.850 | INFO | "pinned" : " 29 GB" | 20260506 10:55:24.850 | INFO | } | 20260506 10:55:24.850 | INFO | Perf report for the last 5.1392 seconds: | 20260506 10:55:24.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:55:24.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:24.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:24.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:24.850 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:55:24.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:24.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:24.850 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:55:24.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:24.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:24.850 | INFO | Uploaded 36GiB to GPU in 1.60749s (22.3951GiB/s) | 20260506 10:55:24.850 | INFO | Downloaded 34GiB to GPU in 1.52528s (22.291GiB/s) | 20260506 10:55:29.989 | INFO | Analyzing chunk frames = [800256,800768) | 20260506 10:55:29.989 | INFO | MemInfo: { | 20260506 10:55:29.989 | INFO | "gpu" : " 17 GB", | 20260506 10:55:29.989 | INFO | "huge" : "", | 20260506 10:55:29.989 | INFO | "huge_shared" : " 27 GB", | 20260506 10:55:29.989 | INFO | "malloc" : " 14 GB", | 20260506 10:55:29.989 | INFO | "pinned" : " 29 GB" | 20260506 10:55:29.989 | INFO | } | 20260506 10:55:29.989 | INFO | Perf report for the last 5.13947 seconds: | 20260506 10:55:29.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:55:29.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:29.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:29.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:29.990 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 10:55:29.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:29.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:29.990 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:55:29.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:29.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:29.990 | INFO | Uploaded 36GiB to GPU in 1.60757s (22.3941GiB/s) | 20260506 10:55:29.990 | INFO | Downloaded 34GiB to GPU in 1.52474s (22.2988GiB/s) | 20260506 10:55:35.129 | INFO | Analyzing chunk frames = [800768,801280) | 20260506 10:55:35.129 | INFO | MemInfo: { | 20260506 10:55:35.129 | INFO | "gpu" : " 17 GB", | 20260506 10:55:35.129 | INFO | "huge" : "", | 20260506 10:55:35.129 | INFO | "huge_shared" : " 27 GB", | 20260506 10:55:35.129 | INFO | "malloc" : " 14 GB", | 20260506 10:55:35.129 | INFO | "pinned" : " 29 GB" | 20260506 10:55:35.129 | INFO | } | 20260506 10:55:35.129 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:55:35.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:55:35.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:35.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:35.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:35.129 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:55:35.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:35.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:35.129 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:55:35.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:35.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:35.129 | INFO | Uploaded 36GiB to GPU in 1.60484s (22.4322GiB/s) | 20260506 10:55:35.129 | INFO | Downloaded 34GiB to GPU in 1.52271s (22.3285GiB/s) | 20260506 10:55:40.268 | INFO | Analyzing chunk frames = [801280,801792) | 20260506 10:55:40.268 | INFO | MemInfo: { | 20260506 10:55:40.268 | INFO | "gpu" : " 17 GB", | 20260506 10:55:40.268 | INFO | "huge" : "", | 20260506 10:55:40.268 | INFO | "huge_shared" : " 27 GB", | 20260506 10:55:40.268 | INFO | "malloc" : " 14 GB", | 20260506 10:55:40.268 | INFO | "pinned" : " 29 GB" | 20260506 10:55:40.268 | INFO | } | 20260506 10:55:40.268 | INFO | Perf report for the last 5.13921 seconds: | 20260506 10:55:40.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:55:40.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:40.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:40.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:40.268 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 10:55:40.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:40.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:40.268 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 10:55:40.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:40.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:40.268 | INFO | Uploaded 36GiB to GPU in 1.60484s (22.4322GiB/s) | 20260506 10:55:40.268 | INFO | Downloaded 34GiB to GPU in 1.52247s (22.3321GiB/s) | 20260506 10:55:45.407 | INFO | Analyzing chunk frames = [801792,802304) | 20260506 10:55:45.407 | INFO | MemInfo: { | 20260506 10:55:45.407 | INFO | "gpu" : " 17 GB", | 20260506 10:55:45.407 | INFO | "huge" : "", | 20260506 10:55:45.407 | INFO | "huge_shared" : " 27 GB", | 20260506 10:55:45.407 | INFO | "malloc" : " 14 GB", | 20260506 10:55:45.407 | INFO | "pinned" : " 29 GB" | 20260506 10:55:45.407 | INFO | } | 20260506 10:55:45.407 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:55:45.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:55:45.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:45.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:45.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:45.407 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:55:45.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:45.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:45.407 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 10:55:45.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:45.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:45.408 | INFO | Uploaded 36GiB to GPU in 1.60533s (22.4253GiB/s) | 20260506 10:55:45.408 | INFO | Downloaded 34GiB to GPU in 1.52329s (22.3201GiB/s) | 20260506 10:55:50.546 | INFO | Analyzing chunk frames = [802304,802816) | 20260506 10:55:50.546 | INFO | MemInfo: { | 20260506 10:55:50.546 | INFO | "gpu" : " 17 GB", | 20260506 10:55:50.546 | INFO | "huge" : "", | 20260506 10:55:50.546 | INFO | "huge_shared" : " 27 GB", | 20260506 10:55:50.547 | INFO | "malloc" : " 14 GB", | 20260506 10:55:50.547 | INFO | "pinned" : " 29 GB" | 20260506 10:55:50.547 | INFO | } | 20260506 10:55:50.547 | INFO | Perf report for the last 5.1393 seconds: | 20260506 10:55:50.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:55:50.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:50.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:50.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:50.547 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:55:50.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:50.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:50.547 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 10:55:50.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:50.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:50.547 | INFO | Uploaded 36GiB to GPU in 1.60567s (22.4205GiB/s) | 20260506 10:55:50.547 | INFO | Downloaded 34GiB to GPU in 1.52471s (22.2993GiB/s) | 20260506 10:55:50.547 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:55:50.547 | INFO | Total mem usage: 16.5 GiB | 20260506 10:55:50.547 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:55:50.547 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -0.848%) | 20260506 10:55:50.547 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:55:50.547 | INFO | High Water Breakdown: | 20260506 10:55:50.547 | INFO | 61.3% : 10 GiB ( 99.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:55:50.547 | INFO | 38.6% : 6.32 GiB ( 94.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:55:50.547 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:55:50.547 | INFO | ------------------------------------------ | 20260506 10:55:50.547 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:55:50.547 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:55:50.547 | INFO | Total mem usage: 26.5 GiB | 20260506 10:55:50.547 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:55:50.547 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:55:50.547 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:55:50.547 | INFO | High Water Breakdown: | 20260506 10:55:50.547 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:55:50.547 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:55:50.547 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:55:50.547 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:55:50.548 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:55:50.548 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:55:50.548 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:55:50.548 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:55:50.548 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:55:50.548 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:55:50.548 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:55:50.548 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:55:50.548 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:55:50.548 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:55:50.548 | INFO | ------------------------------------------ | 20260506 10:55:50.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:55:50.548 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:55:50.548 | INFO | Total mem usage: 25.6 GiB | 20260506 10:55:50.548 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:55:50.548 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:55:50.548 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:55:50.548 | INFO | High Water Breakdown: | 20260506 10:55:50.548 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:55:50.548 | INFO | ------------------------------------------ | 20260506 10:55:50.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:55:50.548 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:55:50.548 | INFO | Total mem usage: 15.7 GiB | 20260506 10:55:50.548 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:55:50.548 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:55:50.548 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:55:50.548 | INFO | High Water Breakdown: | 20260506 10:55:50.548 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:55:50.548 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:55:50.548 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:55:50.548 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:55:50.548 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:55:50.548 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:55:50.548 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:55:50.548 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:55:50.548 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:55:50.548 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:55:50.548 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:55:50.548 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:55:50.548 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:55:50.548 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:55:50.548 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:55:50.548 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:55:50.548 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:55:50.548 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:55:50.548 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:55:50.548 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:55:50.548 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:55:50.548 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:55:50.548 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:55:50.548 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:55:50.548 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:55:50.548 | INFO | ------------------------------------------ | 20260506 10:55:50.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:55:50.548 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:55:50.548 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:55:50.548 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:55:50.548 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260506 10:55:50.548 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.42e-03 / 0.00e+00 | 20260506 10:55:50.548 | INFO | PerfReport: 10.97%, Baseline -- 3.30e+00 / -nan, 1.73e-02 / 0.00e+00 | 20260506 10:55:50.548 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.12e-02 / 0.00e+00 | 20260506 10:55:50.548 | INFO | PerfReport: 16.06%, DME -- 3.86e+01 / -nan, 3.08e+00 / 0.00e+00 | 20260506 10:55:50.548 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.72e+01 / -nan, 4.25e-02 / 0.00e+00 | 20260506 10:55:50.548 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.21e+00 / -nan, 1.25e-01 / 0.00e+00 | 20260506 10:55:50.548 | INFO | PerfReport: 3.62%, Metrics -- 1.09e+00 / -nan, 1.70e-01 / 0.00e+00 | 20260506 10:55:50.548 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.19e-01 / -nan, 5.85e-02 / 0.00e+00 | 20260506 10:55:50.548 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:55:50.548 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:55:50.548 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:55:50.548 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:55:50.548 | INFO | PerfReport: 18.75%, Upload -- 1.68e+01 / 0.00e+00, 5.02e-01 / 0.00e+00 | 20260506 10:55:50.548 | INFO | PerfReport: 18.01%, Download -- 1.62e+01 / 0.00e+00, 8.50e-01 / 0.00e+00 | 20260506 10:55:50.548 | INFO | PerfReport: 33.50%, Compute -- 3.01e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260506 10:55:50.549 | INFO | PerfReport: 0.56%, LockUpload -- 5.07e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 10:55:50.549 | INFO | PerfReport: 0.00%, LockDownload -- 4.87e-04 / 0.00e+00, 1.28e-04 / 0.00e+00 | 20260506 10:55:50.549 | INFO | PerfReport: 29.17%, LockCompute -- 2.62e+01 / 0.00e+00, 1.67e+01 / 0.00e+00 | 20260506 10:55:50.549 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260506 10:55:50.644 | INFO | Starting a round of Baz IO | 20260506 10:55:53.550 | INFO | Populating RTMetrics report | 20260506 10:55:53.551 | INFO | RTMetrics report populated | 20260506 10:55:53.551 | INFO | Opening tmp RTMetrics json file | 20260506 10:55:53.597 | INFO | Serializing RTMetrics to json | 20260506 10:55:53.649 | INFO | Serialization done, swapping tmp file now... | 20260506 10:55:53.653 | INFO | RTMetrics done | 20260506 10:55:54.133 | INFO | Finished a round of Baz IO: Wrote 2.027 GiB in 3.489 seconds | 20260506 10:55:54.133 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.91% / 0.04% / 54.05% | 20260506 10:55:54.133 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.9726% / 7.94729e-05% / 4.37101e-05% | 20260506 10:55:54.133 | INFO | Baz IO overall: | 20260506 10:55:54.133 | INFO | BytesWritten : 374142038511 | 20260506 10:55:54.133 | INFO | NumEvents : 180215136121 | 20260506 10:55:55.686 | INFO | Analyzing chunk frames = [802816,803328) | 20260506 10:55:55.686 | INFO | MemInfo: { | 20260506 10:55:55.686 | INFO | "gpu" : " 17 GB", | 20260506 10:55:55.686 | INFO | "huge" : "", | 20260506 10:55:55.686 | INFO | "huge_shared" : " 27 GB", | 20260506 10:55:55.686 | INFO | "malloc" : " 14 GB", | 20260506 10:55:55.686 | INFO | "pinned" : " 29 GB" | 20260506 10:55:55.686 | INFO | } | 20260506 10:55:55.686 | INFO | Perf report for the last 5.13927 seconds: | 20260506 10:55:55.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:55:55.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:55.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:55.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:55.686 | INFO | ANALYSIS : 57.4%, 2.8, 85.7ms, 0 | 20260506 10:55:55.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:55:55.686 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 10:55:55.686 | INFO | PRE_HQ : 56.5%, 2.0, 60.1ms, 0 | 20260506 10:55:55.686 | INFO | BAZWRITER : 67.5%, 4.8, 164.8ms, 0 | 20260506 10:55:55.686 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 10:55:55.687 | INFO | Uploaded 36GiB to GPU in 1.67439s (21.5004GiB/s) | 20260506 10:55:55.687 | INFO | Downloaded 38GiB to GPU in 1.74016s (21.8371GiB/s) | 20260506 10:56:00.825 | INFO | Analyzing chunk frames = [803328,803840) | 20260506 10:56:00.825 | INFO | MemInfo: { | 20260506 10:56:00.825 | INFO | "gpu" : " 17 GB", | 20260506 10:56:00.825 | INFO | "huge" : "", | 20260506 10:56:00.825 | INFO | "huge_shared" : " 27 GB", | 20260506 10:56:00.825 | INFO | "malloc" : " 14 GB", | 20260506 10:56:00.825 | INFO | "pinned" : " 29 GB" | 20260506 10:56:00.825 | INFO | } | 20260506 10:56:00.825 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:56:00.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:56:00.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:00.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:00.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:00.825 | INFO | ANALYSIS : 57.3%, 3.0, 89.4ms, 0 | 20260506 10:56:00.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:00.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:00.825 | INFO | PRE_HQ : 29.4%, 1.0, 15.4ms, 0 | 20260506 10:56:00.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:00.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:00.826 | INFO | Uploaded 36GiB to GPU in 1.60321s (22.455GiB/s) | 20260506 10:56:00.826 | INFO | Downloaded 34GiB to GPU in 1.51974s (22.3723GiB/s) | 20260506 10:56:05.965 | INFO | Analyzing chunk frames = [803840,804352) | 20260506 10:56:05.965 | INFO | MemInfo: { | 20260506 10:56:05.965 | INFO | "gpu" : " 17 GB", | 20260506 10:56:05.965 | INFO | "huge" : "", | 20260506 10:56:05.965 | INFO | "huge_shared" : " 27 GB", | 20260506 10:56:05.965 | INFO | "malloc" : " 14 GB", | 20260506 10:56:05.965 | INFO | "pinned" : " 29 GB" | 20260506 10:56:05.965 | INFO | } | 20260506 10:56:05.965 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:56:05.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:56:05.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:05.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:05.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:05.965 | INFO | ANALYSIS : 56.7%, 3.0, 88.1ms, 0 | 20260506 10:56:05.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:05.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:05.965 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:56:05.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:05.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:05.965 | INFO | Uploaded 36GiB to GPU in 1.60916s (22.3719GiB/s) | 20260506 10:56:05.965 | INFO | Downloaded 34GiB to GPU in 1.52531s (22.2906GiB/s) | 20260506 10:56:11.104 | INFO | Analyzing chunk frames = [804352,804864) | 20260506 10:56:11.104 | INFO | MemInfo: { | 20260506 10:56:11.104 | INFO | "gpu" : " 17 GB", | 20260506 10:56:11.104 | INFO | "huge" : "", | 20260506 10:56:11.104 | INFO | "huge_shared" : " 27 GB", | 20260506 10:56:11.104 | INFO | "malloc" : " 14 GB", | 20260506 10:56:11.104 | INFO | "pinned" : " 29 GB" | 20260506 10:56:11.104 | INFO | } | 20260506 10:56:11.104 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:56:11.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:56:11.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:11.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:11.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:11.104 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 10:56:11.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:11.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:11.104 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 10:56:11.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:11.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:11.104 | INFO | Uploaded 36GiB to GPU in 1.60759s (22.3938GiB/s) | 20260506 10:56:11.104 | INFO | Downloaded 34GiB to GPU in 1.52528s (22.291GiB/s) | 20260506 10:56:16.243 | INFO | Analyzing chunk frames = [804864,805376) | 20260506 10:56:16.243 | INFO | MemInfo: { | 20260506 10:56:16.243 | INFO | "gpu" : " 17 GB", | 20260506 10:56:16.243 | INFO | "huge" : "", | 20260506 10:56:16.243 | INFO | "huge_shared" : " 27 GB", | 20260506 10:56:16.243 | INFO | "malloc" : " 14 GB", | 20260506 10:56:16.243 | INFO | "pinned" : " 29 GB" | 20260506 10:56:16.243 | INFO | } | 20260506 10:56:16.243 | INFO | Perf report for the last 5.1393 seconds: | 20260506 10:56:16.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:56:16.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:16.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:16.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:16.243 | INFO | ANALYSIS : 56.3%, 3.0, 87.7ms, 0 | 20260506 10:56:16.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:16.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:16.244 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 10:56:16.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:16.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:16.244 | INFO | Uploaded 36GiB to GPU in 1.60699s (22.4022GiB/s) | 20260506 10:56:16.244 | INFO | Downloaded 34GiB to GPU in 1.52432s (22.305GiB/s) | 20260506 10:56:21.383 | INFO | Analyzing chunk frames = [805376,805888) | 20260506 10:56:21.383 | INFO | MemInfo: { | 20260506 10:56:21.383 | INFO | "gpu" : " 17 GB", | 20260506 10:56:21.383 | INFO | "huge" : "", | 20260506 10:56:21.383 | INFO | "huge_shared" : " 27 GB", | 20260506 10:56:21.383 | INFO | "malloc" : " 14 GB", | 20260506 10:56:21.383 | INFO | "pinned" : " 29 GB" | 20260506 10:56:21.383 | INFO | } | 20260506 10:56:21.383 | INFO | Perf report for the last 5.13929 seconds: | 20260506 10:56:21.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:56:21.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:21.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:21.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:21.383 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 10:56:21.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:21.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:21.383 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:56:21.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:21.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:21.383 | INFO | Uploaded 36GiB to GPU in 1.60551s (22.4228GiB/s) | 20260506 10:56:21.383 | INFO | Downloaded 34GiB to GPU in 1.5233s (22.32GiB/s) | 20260506 10:56:26.522 | INFO | Analyzing chunk frames = [805888,806400) | 20260506 10:56:26.522 | INFO | MemInfo: { | 20260506 10:56:26.522 | INFO | "gpu" : " 17 GB", | 20260506 10:56:26.522 | INFO | "huge" : "", | 20260506 10:56:26.522 | INFO | "huge_shared" : " 27 GB", | 20260506 10:56:26.522 | INFO | "malloc" : " 14 GB", | 20260506 10:56:26.522 | INFO | "pinned" : " 29 GB" | 20260506 10:56:26.522 | INFO | } | 20260506 10:56:26.522 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:56:26.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:56:26.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:26.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:26.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:26.522 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:56:26.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:26.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:26.522 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:56:26.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:26.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:26.522 | INFO | Uploaded 36GiB to GPU in 1.60648s (22.4092GiB/s) | 20260506 10:56:26.522 | INFO | Downloaded 34GiB to GPU in 1.52402s (22.3094GiB/s) | 20260506 10:56:31.661 | INFO | Analyzing chunk frames = [806400,806912) | 20260506 10:56:31.661 | INFO | MemInfo: { | 20260506 10:56:31.661 | INFO | "gpu" : " 17 GB", | 20260506 10:56:31.661 | INFO | "huge" : "", | 20260506 10:56:31.661 | INFO | "huge_shared" : " 27 GB", | 20260506 10:56:31.661 | INFO | "malloc" : " 14 GB", | 20260506 10:56:31.661 | INFO | "pinned" : " 29 GB" | 20260506 10:56:31.661 | INFO | } | 20260506 10:56:31.661 | INFO | Perf report for the last 5.13948 seconds: | 20260506 10:56:31.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:56:31.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:31.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:31.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:31.661 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:56:31.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:31.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:31.662 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 10:56:31.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:31.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:31.662 | INFO | Uploaded 36GiB to GPU in 1.60629s (22.4119GiB/s) | 20260506 10:56:31.662 | INFO | Downloaded 34GiB to GPU in 1.52465s (22.3002GiB/s) | 20260506 10:56:31.761 | INFO | Starting a round of Baz IO | 20260506 10:56:34.666 | INFO | Populating RTMetrics report | 20260506 10:56:34.666 | INFO | RTMetrics report populated | 20260506 10:56:34.666 | INFO | Opening tmp RTMetrics json file | 20260506 10:56:34.691 | INFO | Serializing RTMetrics to json | 20260506 10:56:34.817 | INFO | Serialization done, swapping tmp file now... | 20260506 10:56:34.853 | INFO | RTMetrics done | 20260506 10:56:35.328 | INFO | Finished a round of Baz IO: Wrote 2.028 GiB in 3.567 seconds | 20260506 10:56:35.328 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.87% / 0.04% / 54.09% | 20260506 10:56:35.328 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.95848% / 9.53674e-05% / 5.5631e-05% | 20260506 10:56:35.328 | INFO | Baz IO overall: | 20260506 10:56:35.328 | INFO | BytesWritten : 376319247696 | 20260506 10:56:35.328 | INFO | NumEvents : 181272198097 | 20260506 10:56:36.801 | INFO | Analyzing chunk frames = [806912,807424) | 20260506 10:56:36.801 | INFO | MemInfo: { | 20260506 10:56:36.801 | INFO | "gpu" : " 17 GB", | 20260506 10:56:36.801 | INFO | "huge" : "", | 20260506 10:56:36.801 | INFO | "huge_shared" : " 27 GB", | 20260506 10:56:36.801 | INFO | "malloc" : " 14 GB", | 20260506 10:56:36.801 | INFO | "pinned" : " 29 GB" | 20260506 10:56:36.801 | INFO | } | 20260506 10:56:36.801 | INFO | Perf report for the last 5.13924 seconds: | 20260506 10:56:36.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:56:36.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:36.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:36.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:36.801 | INFO | ANALYSIS : 57.5%, 2.8, 85.8ms, 0 | 20260506 10:56:36.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:36.801 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260506 10:56:36.801 | INFO | PRE_HQ : 56.6%, 2.1, 60.5ms, 0 | 20260506 10:56:36.801 | INFO | BAZWRITER : 69.0%, 4.7, 168.2ms, 0 | 20260506 10:56:36.801 | INFO | BAZ_SAMPLING : 3.2%, 1.0, 1.7ms, 0 | 20260506 10:56:36.801 | INFO | Uploaded 36GiB to GPU in 1.66793s (21.5836GiB/s) | 20260506 10:56:36.801 | INFO | Downloaded 38GiB to GPU in 1.73246s (21.9341GiB/s) | 20260506 10:56:41.940 | INFO | Analyzing chunk frames = [807424,807936) | 20260506 10:56:41.940 | INFO | MemInfo: { | 20260506 10:56:41.940 | INFO | "gpu" : " 17 GB", | 20260506 10:56:41.940 | INFO | "huge" : "", | 20260506 10:56:41.940 | INFO | "huge_shared" : " 27 GB", | 20260506 10:56:41.940 | INFO | "malloc" : " 14 GB", | 20260506 10:56:41.940 | INFO | "pinned" : " 29 GB" | 20260506 10:56:41.940 | INFO | } | 20260506 10:56:41.940 | INFO | Perf report for the last 5.13942 seconds: | 20260506 10:56:41.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:56:41.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:41.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:41.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:41.940 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 10:56:41.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:41.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:41.940 | INFO | PRE_HQ : 28.8%, 1.0, 15.1ms, 0 | 20260506 10:56:41.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:41.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:41.941 | INFO | Uploaded 36GiB to GPU in 1.6031s (22.4565GiB/s) | 20260506 10:56:41.941 | INFO | Downloaded 34GiB to GPU in 1.52072s (22.3579GiB/s) | 20260506 10:56:47.079 | INFO | Analyzing chunk frames = [807936,808448) | 20260506 10:56:47.079 | INFO | MemInfo: { | 20260506 10:56:47.079 | INFO | "gpu" : " 17 GB", | 20260506 10:56:47.079 | INFO | "huge" : "", | 20260506 10:56:47.079 | INFO | "huge_shared" : " 27 GB", | 20260506 10:56:47.079 | INFO | "malloc" : " 14 GB", | 20260506 10:56:47.080 | INFO | "pinned" : " 29 GB" | 20260506 10:56:47.080 | INFO | } | 20260506 10:56:47.080 | INFO | Perf report for the last 5.1394 seconds: | 20260506 10:56:47.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:56:47.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:47.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:47.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:47.080 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260506 10:56:47.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:47.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:47.080 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:56:47.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:47.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:47.080 | INFO | Uploaded 36GiB to GPU in 1.60758s (22.3939GiB/s) | 20260506 10:56:47.080 | INFO | Downloaded 34GiB to GPU in 1.52601s (22.2804GiB/s) | 20260506 10:56:52.219 | INFO | Analyzing chunk frames = [808448,808960) | 20260506 10:56:52.219 | INFO | MemInfo: { | 20260506 10:56:52.219 | INFO | "gpu" : " 17 GB", | 20260506 10:56:52.219 | INFO | "huge" : "", | 20260506 10:56:52.219 | INFO | "huge_shared" : " 27 GB", | 20260506 10:56:52.219 | INFO | "malloc" : " 14 GB", | 20260506 10:56:52.219 | INFO | "pinned" : " 29 GB" | 20260506 10:56:52.219 | INFO | } | 20260506 10:56:52.219 | INFO | Perf report for the last 5.1393 seconds: | 20260506 10:56:52.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:56:52.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:52.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:52.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:52.219 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 10:56:52.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:52.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:52.219 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:56:52.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:52.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:52.219 | INFO | Uploaded 36GiB to GPU in 1.60719s (22.3993GiB/s) | 20260506 10:56:52.219 | INFO | Downloaded 34GiB to GPU in 1.52505s (22.2943GiB/s) | 20260506 10:56:57.358 | INFO | Analyzing chunk frames = [808960,809472) | 20260506 10:56:57.358 | INFO | MemInfo: { | 20260506 10:56:57.358 | INFO | "gpu" : " 17 GB", | 20260506 10:56:57.358 | INFO | "huge" : "", | 20260506 10:56:57.358 | INFO | "huge_shared" : " 27 GB", | 20260506 10:56:57.358 | INFO | "malloc" : " 14 GB", | 20260506 10:56:57.358 | INFO | "pinned" : " 29 GB" | 20260506 10:56:57.358 | INFO | } | 20260506 10:56:57.358 | INFO | Perf report for the last 5.13931 seconds: | 20260506 10:56:57.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:56:57.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:57.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:57.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:57.358 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260506 10:56:57.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:57.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:57.358 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 10:56:57.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:57.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:56:57.358 | INFO | Uploaded 36GiB to GPU in 1.60587s (22.4177GiB/s) | 20260506 10:56:57.358 | INFO | Downloaded 34GiB to GPU in 1.52327s (22.3205GiB/s) | 20260506 10:57:02.497 | INFO | Analyzing chunk frames = [809472,809984) | 20260506 10:57:02.497 | INFO | MemInfo: { | 20260506 10:57:02.497 | INFO | "gpu" : " 17 GB", | 20260506 10:57:02.497 | INFO | "huge" : "", | 20260506 10:57:02.497 | INFO | "huge_shared" : " 27 GB", | 20260506 10:57:02.498 | INFO | "malloc" : " 14 GB", | 20260506 10:57:02.498 | INFO | "pinned" : " 29 GB" | 20260506 10:57:02.498 | INFO | } | 20260506 10:57:02.498 | INFO | Perf report for the last 5.13937 seconds: | 20260506 10:57:02.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:57:02.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:02.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:02.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:02.498 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 10:57:02.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:02.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:02.498 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:57:02.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:02.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:02.498 | INFO | Uploaded 36GiB to GPU in 1.60643s (22.4099GiB/s) | 20260506 10:57:02.498 | INFO | Downloaded 34GiB to GPU in 1.52358s (22.3159GiB/s) | 20260506 10:57:07.637 | INFO | Analyzing chunk frames = [809984,810496) | 20260506 10:57:07.637 | INFO | MemInfo: { | 20260506 10:57:07.637 | INFO | "gpu" : " 17 GB", | 20260506 10:57:07.637 | INFO | "huge" : "", | 20260506 10:57:07.637 | INFO | "huge_shared" : " 27 GB", | 20260506 10:57:07.637 | INFO | "malloc" : " 14 GB", | 20260506 10:57:07.637 | INFO | "pinned" : " 29 GB" | 20260506 10:57:07.637 | INFO | } | 20260506 10:57:07.637 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:57:07.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:57:07.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:07.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:07.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:07.637 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:57:07.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:07.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:07.637 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 10:57:07.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:07.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:07.637 | INFO | Uploaded 36GiB to GPU in 1.60637s (22.4107GiB/s) | 20260506 10:57:07.637 | INFO | Downloaded 34GiB to GPU in 1.52434s (22.3048GiB/s) | 20260506 10:57:12.776 | INFO | Analyzing chunk frames = [810496,811008) | 20260506 10:57:12.776 | INFO | MemInfo: { | 20260506 10:57:12.776 | INFO | "gpu" : " 17 GB", | 20260506 10:57:12.776 | INFO | "huge" : "", | 20260506 10:57:12.776 | INFO | "huge_shared" : " 27 GB", | 20260506 10:57:12.776 | INFO | "malloc" : " 14 GB", | 20260506 10:57:12.776 | INFO | "pinned" : " 29 GB" | 20260506 10:57:12.776 | INFO | } | 20260506 10:57:12.776 | INFO | Perf report for the last 5.13938 seconds: | 20260506 10:57:12.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:57:12.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:12.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:12.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:12.776 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 10:57:12.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:12.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:12.776 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:57:12.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:12.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:12.777 | INFO | Uploaded 36GiB to GPU in 1.60544s (22.4237GiB/s) | 20260506 10:57:12.777 | INFO | Downloaded 34GiB to GPU in 1.52397s (22.3101GiB/s) | 20260506 10:57:12.777 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:57:12.777 | INFO | Total mem usage: 16.5 GiB | 20260506 10:57:12.777 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:57:12.777 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -2.25%) | 20260506 10:57:12.777 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:57:12.777 | INFO | High Water Breakdown: | 20260506 10:57:12.777 | INFO | 61.1% : 9.86 GiB ( 97.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:57:12.777 | INFO | 38.8% : 6.27 GiB ( 93.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:57:12.777 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:57:12.777 | INFO | ------------------------------------------ | 20260506 10:57:12.777 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:57:12.777 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:57:12.777 | INFO | Total mem usage: 26.5 GiB | 20260506 10:57:12.777 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:57:12.777 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:57:12.777 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:57:12.777 | INFO | High Water Breakdown: | 20260506 10:57:12.777 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:57:12.777 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:57:12.777 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:57:12.777 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:57:12.777 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:57:12.777 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:57:12.777 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:57:12.777 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:57:12.777 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:57:12.777 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:57:12.777 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:57:12.777 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:57:12.777 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:57:12.777 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:57:12.777 | INFO | ------------------------------------------ | 20260506 10:57:12.777 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:57:12.777 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:57:12.777 | INFO | Total mem usage: 25.6 GiB | 20260506 10:57:12.777 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:57:12.777 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:57:12.777 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:57:12.777 | INFO | High Water Breakdown: | 20260506 10:57:12.777 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:57:12.777 | INFO | ------------------------------------------ | 20260506 10:57:12.777 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:57:12.777 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:57:12.777 | INFO | Total mem usage: 15.7 GiB | 20260506 10:57:12.777 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:57:12.777 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:57:12.777 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:57:12.777 | INFO | High Water Breakdown: | 20260506 10:57:12.777 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:57:12.777 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:57:12.777 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:57:12.777 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:57:12.777 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:57:12.777 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:57:12.777 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:57:12.777 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:57:12.777 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:57:12.777 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:57:12.777 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:57:12.777 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:57:12.777 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:57:12.777 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:57:12.777 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:57:12.777 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:57:12.777 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:57:12.777 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:57:12.778 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:57:12.778 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:57:12.778 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:57:12.778 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:57:12.778 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:57:12.778 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:57:12.778 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:57:12.778 | INFO | ------------------------------------------ | 20260506 10:57:12.778 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:57:12.778 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:57:12.778 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:57:12.778 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:57:12.778 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 10:57:12.778 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 5.85e-03 / 0.00e+00 | 20260506 10:57:12.778 | INFO | PerfReport: 10.97%, Baseline -- 3.30e+00 / -nan, 1.82e-02 / 0.00e+00 | 20260506 10:57:12.778 | INFO | PerfReport: 3.06%, Histogram -- 9.21e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260506 10:57:12.778 | INFO | PerfReport: 16.06%, DME -- 3.86e+01 / -nan, 3.05e+00 / 0.00e+00 | 20260506 10:57:12.778 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.72e+01 / -nan, 4.33e-02 / 0.00e+00 | 20260506 10:57:12.778 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.21e+00 / -nan, 1.91e-01 / 0.00e+00 | 20260506 10:57:12.778 | INFO | PerfReport: 3.62%, Metrics -- 1.09e+00 / -nan, 1.71e-01 / 0.00e+00 | 20260506 10:57:12.778 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.19e-01 / -nan, 5.83e-02 / 0.00e+00 | 20260506 10:57:12.778 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:57:12.778 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:57:12.778 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:57:12.778 | INFO | PerfReport: Full Execution -- 8.98e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:57:12.778 | INFO | PerfReport: 18.73%, Upload -- 1.68e+01 / 0.00e+00, 4.83e-01 / 0.00e+00 | 20260506 10:57:12.778 | INFO | PerfReport: 17.99%, Download -- 1.62e+01 / 0.00e+00, 8.34e-01 / 0.00e+00 | 20260506 10:57:12.778 | INFO | PerfReport: 33.47%, Compute -- 3.00e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 10:57:12.778 | INFO | PerfReport: 0.57%, LockUpload -- 5.08e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:57:12.778 | INFO | PerfReport: 0.00%, LockDownload -- 4.94e-04 / 0.00e+00, 1.33e-04 / 0.00e+00 | 20260506 10:57:12.778 | INFO | PerfReport: 29.23%, LockCompute -- 2.62e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 10:57:12.778 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260506 10:57:12.873 | INFO | Starting a round of Baz IO | 20260506 10:57:15.780 | INFO | Populating RTMetrics report | 20260506 10:57:15.780 | INFO | RTMetrics report populated | 20260506 10:57:15.780 | INFO | Opening tmp RTMetrics json file | 20260506 10:57:15.784 | INFO | Serializing RTMetrics to json | 20260506 10:57:15.785 | INFO | Serialization done, swapping tmp file now... | 20260506 10:57:15.785 | INFO | RTMetrics done | 20260506 10:57:15.998 | INFO | Finished a round of Baz IO: Wrote 2.024 GiB in 3.124 seconds | 20260506 10:57:15.998 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.83% / 0.04% / 54.13% | 20260506 10:57:15.998 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.9103% / 8.34465e-05% / 3.57628e-05% | 20260506 10:57:15.998 | INFO | Baz IO overall: | 20260506 10:57:15.998 | INFO | BytesWritten : 378492440778 | 20260506 10:57:15.998 | INFO | NumEvents : 182326691582 | 20260506 10:57:17.915 | INFO | Analyzing chunk frames = [811008,811520) | 20260506 10:57:17.915 | INFO | MemInfo: { | 20260506 10:57:17.915 | INFO | "gpu" : " 17 GB", | 20260506 10:57:17.915 | INFO | "huge" : "", | 20260506 10:57:17.915 | INFO | "huge_shared" : " 27 GB", | 20260506 10:57:17.915 | INFO | "malloc" : " 14 GB", | 20260506 10:57:17.915 | INFO | "pinned" : " 29 GB" | 20260506 10:57:17.915 | INFO | } | 20260506 10:57:17.916 | INFO | Perf report for the last 5.13925 seconds: | 20260506 10:57:17.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:57:17.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:17.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:17.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:17.916 | INFO | ANALYSIS : 57.4%, 2.8, 85.2ms, 0 | 20260506 10:57:17.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:17.916 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 10:57:17.916 | INFO | PRE_HQ : 56.4%, 2.1, 60.6ms, 0 | 20260506 10:57:17.916 | INFO | BAZWRITER : 60.3%, 4.8, 148.8ms, 0 | 20260506 10:57:17.916 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 10:57:17.916 | INFO | Uploaded 36GiB to GPU in 1.67362s (21.5102GiB/s) | 20260506 10:57:17.916 | INFO | Downloaded 38GiB to GPU in 1.73981s (21.8415GiB/s) | 20260506 10:57:23.055 | INFO | Analyzing chunk frames = [811520,812032) | 20260506 10:57:23.055 | INFO | MemInfo: { | 20260506 10:57:23.055 | INFO | "gpu" : " 17 GB", | 20260506 10:57:23.055 | INFO | "huge" : "", | 20260506 10:57:23.055 | INFO | "huge_shared" : " 27 GB", | 20260506 10:57:23.055 | INFO | "malloc" : " 14 GB", | 20260506 10:57:23.055 | INFO | "pinned" : " 29 GB" | 20260506 10:57:23.055 | INFO | } | 20260506 10:57:23.055 | INFO | Perf report for the last 5.13927 seconds: | 20260506 10:57:23.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:57:23.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:23.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:23.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:23.055 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 10:57:23.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:23.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:23.055 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:57:23.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:23.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:23.055 | INFO | Uploaded 36GiB to GPU in 1.60366s (22.4487GiB/s) | 20260506 10:57:23.055 | INFO | Downloaded 34GiB to GPU in 1.52128s (22.3497GiB/s) | 20260506 10:57:28.194 | INFO | Analyzing chunk frames = [812032,812544) | 20260506 10:57:28.194 | INFO | MemInfo: { | 20260506 10:57:28.194 | INFO | "gpu" : " 17 GB", | 20260506 10:57:28.194 | INFO | "huge" : "", | 20260506 10:57:28.194 | INFO | "huge_shared" : " 27 GB", | 20260506 10:57:28.194 | INFO | "malloc" : " 14 GB", | 20260506 10:57:28.194 | INFO | "pinned" : " 29 GB" | 20260506 10:57:28.194 | INFO | } | 20260506 10:57:28.194 | INFO | Perf report for the last 5.13948 seconds: | 20260506 10:57:28.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:57:28.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:28.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:28.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:28.194 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260506 10:57:28.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:28.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:28.195 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:57:28.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:28.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:28.195 | INFO | Uploaded 36GiB to GPU in 1.60801s (22.3879GiB/s) | 20260506 10:57:28.195 | INFO | Downloaded 34GiB to GPU in 1.52607s (22.2795GiB/s) | 20260506 10:57:33.333 | INFO | Analyzing chunk frames = [812544,813056) | 20260506 10:57:33.333 | INFO | MemInfo: { | 20260506 10:57:33.333 | INFO | "gpu" : " 17 GB", | 20260506 10:57:33.333 | INFO | "huge" : "", | 20260506 10:57:33.333 | INFO | "huge_shared" : " 27 GB", | 20260506 10:57:33.333 | INFO | "malloc" : " 14 GB", | 20260506 10:57:33.334 | INFO | "pinned" : " 29 GB" | 20260506 10:57:33.334 | INFO | } | 20260506 10:57:33.334 | INFO | Perf report for the last 5.13931 seconds: | 20260506 10:57:33.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:57:33.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:33.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:33.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:33.334 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260506 10:57:33.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:33.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:33.334 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 10:57:33.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:33.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:33.334 | INFO | Uploaded 36GiB to GPU in 1.60818s (22.3856GiB/s) | 20260506 10:57:33.334 | INFO | Downloaded 34GiB to GPU in 1.52532s (22.2904GiB/s) | 20260506 10:57:38.473 | INFO | Analyzing chunk frames = [813056,813568) | 20260506 10:57:38.473 | INFO | MemInfo: { | 20260506 10:57:38.473 | INFO | "gpu" : " 17 GB", | 20260506 10:57:38.473 | INFO | "huge" : "", | 20260506 10:57:38.473 | INFO | "huge_shared" : " 27 GB", | 20260506 10:57:38.473 | INFO | "malloc" : " 14 GB", | 20260506 10:57:38.473 | INFO | "pinned" : " 29 GB" | 20260506 10:57:38.473 | INFO | } | 20260506 10:57:38.473 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:57:38.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:57:38.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:38.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:38.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:38.473 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:57:38.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:38.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:38.473 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 10:57:38.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:38.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:38.473 | INFO | Uploaded 36GiB to GPU in 1.60456s (22.4361GiB/s) | 20260506 10:57:38.473 | INFO | Downloaded 34GiB to GPU in 1.52222s (22.3358GiB/s) | 20260506 10:57:43.612 | INFO | Analyzing chunk frames = [813568,814080) | 20260506 10:57:43.612 | INFO | MemInfo: { | 20260506 10:57:43.612 | INFO | "gpu" : " 17 GB", | 20260506 10:57:43.612 | INFO | "huge" : "", | 20260506 10:57:43.612 | INFO | "huge_shared" : " 27 GB", | 20260506 10:57:43.612 | INFO | "malloc" : " 14 GB", | 20260506 10:57:43.612 | INFO | "pinned" : " 29 GB" | 20260506 10:57:43.612 | INFO | } | 20260506 10:57:43.613 | INFO | Perf report for the last 5.13952 seconds: | 20260506 10:57:43.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:57:43.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:43.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:43.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:43.613 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 10:57:43.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:43.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:43.613 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:57:43.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:43.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:43.613 | INFO | Uploaded 36GiB to GPU in 1.60583s (22.4183GiB/s) | 20260506 10:57:43.613 | INFO | Downloaded 34GiB to GPU in 1.52402s (22.3095GiB/s) | 20260506 10:57:48.751 | INFO | Analyzing chunk frames = [814080,814592) | 20260506 10:57:48.752 | INFO | MemInfo: { | 20260506 10:57:48.752 | INFO | "gpu" : " 17 GB", | 20260506 10:57:48.752 | INFO | "huge" : "", | 20260506 10:57:48.752 | INFO | "huge_shared" : " 27 GB", | 20260506 10:57:48.752 | INFO | "malloc" : " 14 GB", | 20260506 10:57:48.752 | INFO | "pinned" : " 29 GB" | 20260506 10:57:48.752 | INFO | } | 20260506 10:57:48.752 | INFO | Perf report for the last 5.13918 seconds: | 20260506 10:57:48.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:57:48.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:48.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:48.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:48.752 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:57:48.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:48.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:48.752 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:57:48.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:48.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:48.752 | INFO | Uploaded 36GiB to GPU in 1.60593s (22.4169GiB/s) | 20260506 10:57:48.752 | INFO | Downloaded 34GiB to GPU in 1.52532s (22.2904GiB/s) | 20260506 10:57:53.891 | INFO | Analyzing chunk frames = [814592,815104) | 20260506 10:57:53.891 | INFO | MemInfo: { | 20260506 10:57:53.891 | INFO | "gpu" : " 17 GB", | 20260506 10:57:53.891 | INFO | "huge" : "", | 20260506 10:57:53.891 | INFO | "huge_shared" : " 27 GB", | 20260506 10:57:53.891 | INFO | "malloc" : " 14 GB", | 20260506 10:57:53.891 | INFO | "pinned" : " 29 GB" | 20260506 10:57:53.891 | INFO | } | 20260506 10:57:53.891 | INFO | Perf report for the last 5.13914 seconds: | 20260506 10:57:53.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:57:53.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:53.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:53.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:53.891 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:57:53.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:53.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:53.891 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:57:53.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:53.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:53.891 | INFO | Uploaded 36GiB to GPU in 1.60568s (22.4203GiB/s) | 20260506 10:57:53.891 | INFO | Downloaded 34GiB to GPU in 1.5248s (22.298GiB/s) | 20260506 10:57:53.988 | INFO | Starting a round of Baz IO | 20260506 10:57:56.895 | INFO | Populating RTMetrics report | 20260506 10:57:56.895 | INFO | RTMetrics report populated | 20260506 10:57:56.895 | INFO | Opening tmp RTMetrics json file | 20260506 10:57:56.938 | INFO | Serializing RTMetrics to json | 20260506 10:57:56.999 | INFO | Serialization done, swapping tmp file now... | 20260506 10:57:57.001 | INFO | RTMetrics done | 20260506 10:57:57.423 | INFO | Finished a round of Baz IO: Wrote 2.027 GiB in 3.434 seconds | 20260506 10:57:57.423 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.78% / 0.04% / 54.17% | 20260506 10:57:57.423 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.89617% / 5.96046e-05% / 4.37101e-05% | 20260506 10:57:57.423 | INFO | Baz IO overall: | 20260506 10:57:57.423 | INFO | BytesWritten : 380668710528 | 20260506 10:57:57.423 | INFO | NumEvents : 183383058140 | 20260506 10:57:59.030 | INFO | Analyzing chunk frames = [815104,815616) | 20260506 10:57:59.030 | INFO | MemInfo: { | 20260506 10:57:59.030 | INFO | "gpu" : " 17 GB", | 20260506 10:57:59.030 | INFO | "huge" : "", | 20260506 10:57:59.030 | INFO | "huge_shared" : " 27 GB", | 20260506 10:57:59.030 | INFO | "malloc" : " 14 GB", | 20260506 10:57:59.030 | INFO | "pinned" : " 29 GB" | 20260506 10:57:59.030 | INFO | } | 20260506 10:57:59.030 | INFO | Perf report for the last 5.13952 seconds: | 20260506 10:57:59.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:57:59.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:59.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:59.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:59.031 | INFO | ANALYSIS : 57.4%, 2.9, 86.0ms, 0 | 20260506 10:57:59.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:57:59.031 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260506 10:57:59.031 | INFO | PRE_HQ : 56.5%, 2.1, 60.4ms, 0 | 20260506 10:57:59.031 | INFO | BAZWRITER : 66.5%, 4.7, 161.4ms, 0 | 20260506 10:57:59.031 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 10:57:59.031 | INFO | Uploaded 36GiB to GPU in 1.6681s (21.5814GiB/s) | 20260506 10:57:59.031 | INFO | Downloaded 38GiB to GPU in 1.7335s (21.921GiB/s) | 20260506 10:58:04.170 | INFO | Analyzing chunk frames = [815616,816128) | 20260506 10:58:04.170 | INFO | MemInfo: { | 20260506 10:58:04.170 | INFO | "gpu" : " 17 GB", | 20260506 10:58:04.170 | INFO | "huge" : "", | 20260506 10:58:04.170 | INFO | "huge_shared" : " 27 GB", | 20260506 10:58:04.170 | INFO | "malloc" : " 14 GB", | 20260506 10:58:04.170 | INFO | "pinned" : " 29 GB" | 20260506 10:58:04.170 | INFO | } | 20260506 10:58:04.170 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:58:04.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:58:04.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:04.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:04.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:04.170 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 10:58:04.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:04.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:04.170 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:58:04.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:04.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:04.170 | INFO | Uploaded 36GiB to GPU in 1.60384s (22.4461GiB/s) | 20260506 10:58:04.170 | INFO | Downloaded 34GiB to GPU in 1.52169s (22.3436GiB/s) | 20260506 10:58:09.309 | INFO | Analyzing chunk frames = [816128,816640) | 20260506 10:58:09.309 | INFO | MemInfo: { | 20260506 10:58:09.309 | INFO | "gpu" : " 17 GB", | 20260506 10:58:09.309 | INFO | "huge" : "", | 20260506 10:58:09.309 | INFO | "huge_shared" : " 27 GB", | 20260506 10:58:09.309 | INFO | "malloc" : " 14 GB", | 20260506 10:58:09.309 | INFO | "pinned" : " 29 GB" | 20260506 10:58:09.309 | INFO | } | 20260506 10:58:09.309 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:58:09.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:58:09.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:09.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:09.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:09.309 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260506 10:58:09.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:09.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:09.309 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:58:09.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:09.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:09.310 | INFO | Uploaded 36GiB to GPU in 1.60769s (22.3923GiB/s) | 20260506 10:58:09.310 | INFO | Downloaded 34GiB to GPU in 1.52563s (22.2859GiB/s) | 20260506 10:58:14.448 | INFO | Analyzing chunk frames = [816640,817152) | 20260506 10:58:14.449 | INFO | MemInfo: { | 20260506 10:58:14.449 | INFO | "gpu" : " 17 GB", | 20260506 10:58:14.449 | INFO | "huge" : "", | 20260506 10:58:14.449 | INFO | "huge_shared" : " 27 GB", | 20260506 10:58:14.449 | INFO | "malloc" : " 14 GB", | 20260506 10:58:14.449 | INFO | "pinned" : " 29 GB" | 20260506 10:58:14.449 | INFO | } | 20260506 10:58:14.449 | INFO | Perf report for the last 5.1396 seconds: | 20260506 10:58:14.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:58:14.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:14.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:14.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:14.449 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 10:58:14.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:14.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:14.449 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:58:14.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:14.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:14.449 | INFO | Uploaded 36GiB to GPU in 1.60706s (22.4012GiB/s) | 20260506 10:58:14.449 | INFO | Downloaded 34GiB to GPU in 1.52528s (22.291GiB/s) | 20260506 10:58:19.588 | INFO | Analyzing chunk frames = [817152,817664) | 20260506 10:58:19.588 | INFO | MemInfo: { | 20260506 10:58:19.588 | INFO | "gpu" : " 17 GB", | 20260506 10:58:19.588 | INFO | "huge" : "", | 20260506 10:58:19.588 | INFO | "huge_shared" : " 27 GB", | 20260506 10:58:19.588 | INFO | "malloc" : " 14 GB", | 20260506 10:58:19.588 | INFO | "pinned" : " 29 GB" | 20260506 10:58:19.588 | INFO | } | 20260506 10:58:19.588 | INFO | Perf report for the last 5.13908 seconds: | 20260506 10:58:19.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:58:19.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:19.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:19.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:19.588 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 10:58:19.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:19.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:19.588 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:58:19.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:19.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:19.588 | INFO | Uploaded 36GiB to GPU in 1.60543s (22.4239GiB/s) | 20260506 10:58:19.588 | INFO | Downloaded 34GiB to GPU in 1.52293s (22.3254GiB/s) | 20260506 10:58:24.727 | INFO | Analyzing chunk frames = [817664,818176) | 20260506 10:58:24.727 | INFO | MemInfo: { | 20260506 10:58:24.727 | INFO | "gpu" : " 17 GB", | 20260506 10:58:24.727 | INFO | "huge" : "", | 20260506 10:58:24.727 | INFO | "huge_shared" : " 27 GB", | 20260506 10:58:24.727 | INFO | "malloc" : " 14 GB", | 20260506 10:58:24.727 | INFO | "pinned" : " 29 GB" | 20260506 10:58:24.727 | INFO | } | 20260506 10:58:24.727 | INFO | Perf report for the last 5.13932 seconds: | 20260506 10:58:24.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:58:24.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:24.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:24.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:24.727 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 10:58:24.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:24.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:24.727 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:58:24.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:24.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:24.728 | INFO | Uploaded 36GiB to GPU in 1.60611s (22.4144GiB/s) | 20260506 10:58:24.728 | INFO | Downloaded 34GiB to GPU in 1.52313s (22.3225GiB/s) | 20260506 10:58:29.866 | INFO | Analyzing chunk frames = [818176,818688) | 20260506 10:58:29.866 | INFO | MemInfo: { | 20260506 10:58:29.866 | INFO | "gpu" : " 17 GB", | 20260506 10:58:29.866 | INFO | "huge" : "", | 20260506 10:58:29.866 | INFO | "huge_shared" : " 27 GB", | 20260506 10:58:29.866 | INFO | "malloc" : " 14 GB", | 20260506 10:58:29.866 | INFO | "pinned" : " 29 GB" | 20260506 10:58:29.866 | INFO | } | 20260506 10:58:29.866 | INFO | Perf report for the last 5.13929 seconds: | 20260506 10:58:29.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:58:29.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:29.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:29.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:29.867 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 10:58:29.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:29.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:29.867 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:58:29.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:29.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:29.867 | INFO | Uploaded 36GiB to GPU in 1.60623s (22.4128GiB/s) | 20260506 10:58:29.867 | INFO | Downloaded 34GiB to GPU in 1.52409s (22.3084GiB/s) | 20260506 10:58:35.006 | INFO | Analyzing chunk frames = [818688,819200) | 20260506 10:58:35.006 | INFO | MemInfo: { | 20260506 10:58:35.006 | INFO | "gpu" : " 17 GB", | 20260506 10:58:35.006 | INFO | "huge" : "", | 20260506 10:58:35.006 | INFO | "huge_shared" : " 27 GB", | 20260506 10:58:35.006 | INFO | "malloc" : " 14 GB", | 20260506 10:58:35.006 | INFO | "pinned" : " 29 GB" | 20260506 10:58:35.006 | INFO | } | 20260506 10:58:35.006 | INFO | Perf report for the last 5.1395 seconds: | 20260506 10:58:35.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:58:35.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:35.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:35.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:35.006 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 10:58:35.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:35.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:35.006 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 10:58:35.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:35.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:35.007 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:58:35.007 | INFO | Total mem usage: 16.5 GiB | 20260506 10:58:35.007 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:58:35.007 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -5.88%) | 20260506 10:58:35.007 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:58:35.007 | INFO | High Water Breakdown: | 20260506 10:58:35.007 | INFO | 60.5% : 9.44 GiB ( 93.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:58:35.007 | INFO | 39.4% : 6.14 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:58:35.007 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:58:35.007 | INFO | ------------------------------------------ | 20260506 10:58:35.007 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:58:35.007 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:58:35.007 | INFO | Uploaded 36GiB to GPU in 1.60656s (22.4082GiB/s) | 20260506 10:58:35.007 | INFO | Total mem usage: 26.5 GiB | 20260506 10:58:35.007 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:58:35.007 | INFO | Downloaded 34GiB to GPU in 1.52452s (22.3022GiB/s) | 20260506 10:58:35.007 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:58:35.007 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:58:35.007 | INFO | High Water Breakdown: | 20260506 10:58:35.007 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:58:35.007 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:58:35.007 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:58:35.007 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:58:35.007 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:58:35.007 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:58:35.007 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:58:35.007 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:58:35.007 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:58:35.007 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:58:35.007 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:58:35.007 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:58:35.007 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:58:35.007 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:58:35.007 | INFO | ------------------------------------------ | 20260506 10:58:35.007 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:58:35.007 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:58:35.007 | INFO | Total mem usage: 25.6 GiB | 20260506 10:58:35.007 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:58:35.007 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:58:35.007 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:58:35.007 | INFO | High Water Breakdown: | 20260506 10:58:35.007 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:58:35.007 | INFO | ------------------------------------------ | 20260506 10:58:35.007 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:58:35.007 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:58:35.007 | INFO | Total mem usage: 15.7 GiB | 20260506 10:58:35.007 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:58:35.007 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:58:35.007 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:58:35.007 | INFO | High Water Breakdown: | 20260506 10:58:35.007 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:58:35.007 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:58:35.007 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:58:35.007 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:58:35.007 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:58:35.007 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:58:35.007 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:58:35.007 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:58:35.007 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:58:35.007 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:58:35.007 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:58:35.007 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:58:35.008 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:58:35.008 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:58:35.008 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:58:35.008 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:58:35.008 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:58:35.008 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:58:35.008 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:58:35.008 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:58:35.008 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:58:35.008 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:58:35.008 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:58:35.008 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:58:35.008 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:58:35.008 | INFO | ------------------------------------------ | 20260506 10:58:35.008 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:58:35.008 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:58:35.008 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:58:35.008 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:58:35.008 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 10:58:35.008 | INFO | PerfReport: 0.29%, AggregateStats -- 8.64e-02 / -nan, 4.29e-03 / 0.00e+00 | 20260506 10:58:35.008 | INFO | PerfReport: 10.97%, Baseline -- 3.29e+00 / -nan, 1.74e-02 / 0.00e+00 | 20260506 10:58:35.008 | INFO | PerfReport: 3.07%, Histogram -- 9.21e-01 / -nan, 5.26e-02 / 0.00e+00 | 20260506 10:58:35.008 | INFO | PerfReport: 16.07%, DME -- 3.86e+01 / -nan, 2.96e+00 / 0.00e+00 | 20260506 10:58:35.008 | INFO | PerfReport: 57.26%, FrameLabeling -- 1.72e+01 / -nan, 4.27e-02 / 0.00e+00 | 20260506 10:58:35.008 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.20e+00 / -nan, 1.25e-01 / 0.00e+00 | 20260506 10:58:35.008 | INFO | PerfReport: 3.61%, Metrics -- 1.09e+00 / -nan, 1.67e-01 / 0.00e+00 | 20260506 10:58:35.008 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.18e-01 / -nan, 5.83e-02 / 0.00e+00 | 20260506 10:58:35.008 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:58:35.008 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:58:35.008 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:58:35.008 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 10:58:35.008 | INFO | PerfReport: 18.74%, Upload -- 1.68e+01 / 0.00e+00, 4.86e-01 / 0.00e+00 | 20260506 10:58:35.008 | INFO | PerfReport: 18.00%, Download -- 1.62e+01 / 0.00e+00, 8.34e-01 / 0.00e+00 | 20260506 10:58:35.008 | INFO | PerfReport: 33.48%, Compute -- 3.00e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 10:58:35.008 | INFO | PerfReport: 0.57%, LockUpload -- 5.10e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:58:35.008 | INFO | PerfReport: 0.00%, LockDownload -- 4.93e-04 / 0.00e+00, 1.77e-04 / 0.00e+00 | 20260506 10:58:35.008 | INFO | PerfReport: 29.20%, LockCompute -- 2.62e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 10:58:35.008 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.28e-02 / 0.00e+00 | 20260506 10:58:35.104 | INFO | Starting a round of Baz IO | 20260506 10:58:38.011 | INFO | Populating RTMetrics report | 20260506 10:58:38.011 | INFO | RTMetrics report populated | 20260506 10:58:38.011 | INFO | Opening tmp RTMetrics json file | 20260506 10:58:38.057 | INFO | Serializing RTMetrics to json | 20260506 10:58:38.083 | INFO | Serialization done, swapping tmp file now... | 20260506 10:58:38.084 | INFO | RTMetrics done | 20260506 10:58:38.284 | INFO | Finished a round of Baz IO: Wrote 2.028 GiB in 3.179 seconds | 20260506 10:58:38.284 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.74% / 0.04% / 54.22% | 20260506 10:58:38.284 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.88048% / 6.75519e-05% / 3.97364e-05% | 20260506 10:58:38.284 | INFO | Baz IO overall: | 20260506 10:58:38.284 | INFO | BytesWritten : 382845989304 | 20260506 10:58:38.284 | INFO | NumEvents : 184439741223 | 20260506 10:58:40.145 | INFO | Analyzing chunk frames = [819200,819712) | 20260506 10:58:40.145 | INFO | MemInfo: { | 20260506 10:58:40.145 | INFO | "gpu" : " 17 GB", | 20260506 10:58:40.145 | INFO | "huge" : "", | 20260506 10:58:40.145 | INFO | "huge_shared" : " 27 GB", | 20260506 10:58:40.145 | INFO | "malloc" : " 14 GB", | 20260506 10:58:40.145 | INFO | "pinned" : " 29 GB" | 20260506 10:58:40.145 | INFO | } | 20260506 10:58:40.145 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:58:40.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:58:40.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:40.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:40.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:40.145 | INFO | ANALYSIS : 57.4%, 2.8, 85.4ms, 0 | 20260506 10:58:40.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:40.145 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260506 10:58:40.145 | INFO | PRE_HQ : 56.5%, 2.1, 60.1ms, 0 | 20260506 10:58:40.145 | INFO | BAZWRITER : 61.5%, 4.8, 151.2ms, 0 | 20260506 10:58:40.145 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 10:58:40.146 | INFO | Uploaded 36GiB to GPU in 1.67867s (21.4456GiB/s) | 20260506 10:58:40.146 | INFO | Downloaded 38GiB to GPU in 1.74272s (21.8051GiB/s) | 20260506 10:58:45.284 | INFO | Analyzing chunk frames = [819712,820224) | 20260506 10:58:45.284 | INFO | MemInfo: { | 20260506 10:58:45.284 | INFO | "gpu" : " 17 GB", | 20260506 10:58:45.284 | INFO | "huge" : "", | 20260506 10:58:45.284 | INFO | "huge_shared" : " 27 GB", | 20260506 10:58:45.285 | INFO | "malloc" : " 14 GB", | 20260506 10:58:45.285 | INFO | "pinned" : " 29 GB" | 20260506 10:58:45.285 | INFO | } | 20260506 10:58:45.285 | INFO | Perf report for the last 5.13933 seconds: | 20260506 10:58:45.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:58:45.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:45.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:45.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:45.285 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 10:58:45.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:45.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:45.285 | INFO | PRE_HQ : 28.7%, 1.0, 15.0ms, 0 | 20260506 10:58:45.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:45.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:45.285 | INFO | Uploaded 36GiB to GPU in 1.60349s (22.4511GiB/s) | 20260506 10:58:45.285 | INFO | Downloaded 34GiB to GPU in 1.52128s (22.3496GiB/s) | 20260506 10:58:50.424 | INFO | Analyzing chunk frames = [820224,820736) | 20260506 10:58:50.424 | INFO | MemInfo: { | 20260506 10:58:50.424 | INFO | "gpu" : " 17 GB", | 20260506 10:58:50.424 | INFO | "huge" : "", | 20260506 10:58:50.424 | INFO | "huge_shared" : " 27 GB", | 20260506 10:58:50.424 | INFO | "malloc" : " 14 GB", | 20260506 10:58:50.424 | INFO | "pinned" : " 29 GB" | 20260506 10:58:50.424 | INFO | } | 20260506 10:58:50.424 | INFO | Perf report for the last 5.13947 seconds: | 20260506 10:58:50.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:58:50.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:50.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:50.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:50.424 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260506 10:58:50.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:50.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:50.424 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 10:58:50.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:50.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:50.424 | INFO | Uploaded 36GiB to GPU in 1.6077s (22.3922GiB/s) | 20260506 10:58:50.425 | INFO | Downloaded 34GiB to GPU in 1.526s (22.2804GiB/s) | 20260506 10:58:55.563 | INFO | Analyzing chunk frames = [820736,821248) | 20260506 10:58:55.563 | INFO | MemInfo: { | 20260506 10:58:55.563 | INFO | "gpu" : " 17 GB", | 20260506 10:58:55.563 | INFO | "huge" : "", | 20260506 10:58:55.563 | INFO | "huge_shared" : " 27 GB", | 20260506 10:58:55.563 | INFO | "malloc" : " 14 GB", | 20260506 10:58:55.563 | INFO | "pinned" : " 29 GB" | 20260506 10:58:55.563 | INFO | } | 20260506 10:58:55.563 | INFO | Perf report for the last 5.13922 seconds: | 20260506 10:58:55.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:58:55.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:55.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:55.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:55.563 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 10:58:55.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:55.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:55.563 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 10:58:55.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:55.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:58:55.564 | INFO | Uploaded 36GiB to GPU in 1.60753s (22.3946GiB/s) | 20260506 10:58:55.564 | INFO | Downloaded 34GiB to GPU in 1.52529s (22.2908GiB/s) | 20260506 10:59:00.703 | INFO | Analyzing chunk frames = [821248,821760) | 20260506 10:59:00.703 | INFO | MemInfo: { | 20260506 10:59:00.703 | INFO | "gpu" : " 17 GB", | 20260506 10:59:00.703 | INFO | "huge" : "", | 20260506 10:59:00.703 | INFO | "huge_shared" : " 27 GB", | 20260506 10:59:00.703 | INFO | "malloc" : " 14 GB", | 20260506 10:59:00.703 | INFO | "pinned" : " 29 GB" | 20260506 10:59:00.703 | INFO | } | 20260506 10:59:00.703 | INFO | Perf report for the last 5.13941 seconds: | 20260506 10:59:00.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:59:00.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:00.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:00.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:00.703 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260506 10:59:00.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:00.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:00.703 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:59:00.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:00.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:00.703 | INFO | Uploaded 36GiB to GPU in 1.60577s (22.4191GiB/s) | 20260506 10:59:00.704 | INFO | Downloaded 34GiB to GPU in 1.52274s (22.3282GiB/s) | 20260506 10:59:05.842 | INFO | Analyzing chunk frames = [821760,822272) | 20260506 10:59:05.842 | INFO | MemInfo: { | 20260506 10:59:05.842 | INFO | "gpu" : " 17 GB", | 20260506 10:59:05.842 | INFO | "huge" : "", | 20260506 10:59:05.842 | INFO | "huge_shared" : " 27 GB", | 20260506 10:59:05.842 | INFO | "malloc" : " 14 GB", | 20260506 10:59:05.842 | INFO | "pinned" : " 29 GB" | 20260506 10:59:05.842 | INFO | } | 20260506 10:59:05.842 | INFO | Perf report for the last 5.13926 seconds: | 20260506 10:59:05.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:59:05.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:05.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:05.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:05.842 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 10:59:05.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:05.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:05.842 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 10:59:05.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:05.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:05.842 | INFO | Uploaded 36GiB to GPU in 1.60594s (22.4168GiB/s) | 20260506 10:59:05.842 | INFO | Downloaded 34GiB to GPU in 1.523s (22.3244GiB/s) | 20260506 10:59:10.981 | INFO | Analyzing chunk frames = [822272,822784) | 20260506 10:59:10.981 | INFO | MemInfo: { | 20260506 10:59:10.981 | INFO | "gpu" : " 17 GB", | 20260506 10:59:10.981 | INFO | "huge" : "", | 20260506 10:59:10.981 | INFO | "huge_shared" : " 27 GB", | 20260506 10:59:10.981 | INFO | "malloc" : " 14 GB", | 20260506 10:59:10.981 | INFO | "pinned" : " 29 GB" | 20260506 10:59:10.981 | INFO | } | 20260506 10:59:10.982 | INFO | Perf report for the last 5.13944 seconds: | 20260506 10:59:10.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:59:10.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:10.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:10.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:10.982 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 10:59:10.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:10.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:10.982 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:59:10.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:10.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:10.982 | INFO | Uploaded 36GiB to GPU in 1.60606s (22.4151GiB/s) | 20260506 10:59:10.982 | INFO | Downloaded 34GiB to GPU in 1.52399s (22.3098GiB/s) | 20260506 10:59:16.120 | INFO | Analyzing chunk frames = [822784,823296) | 20260506 10:59:16.121 | INFO | MemInfo: { | 20260506 10:59:16.121 | INFO | "gpu" : " 17 GB", | 20260506 10:59:16.121 | INFO | "huge" : "", | 20260506 10:59:16.121 | INFO | "huge_shared" : " 27 GB", | 20260506 10:59:16.121 | INFO | "malloc" : " 14 GB", | 20260506 10:59:16.121 | INFO | "pinned" : " 29 GB" | 20260506 10:59:16.121 | INFO | } | 20260506 10:59:16.121 | INFO | Perf report for the last 5.13923 seconds: | 20260506 10:59:16.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:59:16.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:16.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:16.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:16.121 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 10:59:16.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:16.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:16.121 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 10:59:16.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:16.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:16.121 | INFO | Uploaded 36GiB to GPU in 1.60682s (22.4045GiB/s) | 20260506 10:59:16.121 | INFO | Downloaded 34GiB to GPU in 1.52422s (22.3065GiB/s) | 20260506 10:59:16.218 | INFO | Starting a round of Baz IO | 20260506 10:59:19.121 | INFO | Populating RTMetrics report | 20260506 10:59:19.122 | INFO | RTMetrics report populated | 20260506 10:59:19.122 | INFO | Opening tmp RTMetrics json file | 20260506 10:59:19.139 | INFO | Serializing RTMetrics to json | 20260506 10:59:19.179 | INFO | Serialization done, swapping tmp file now... | 20260506 10:59:19.208 | INFO | RTMetrics done | 20260506 10:59:19.613 | INFO | Finished a round of Baz IO: Wrote 2.03 GiB in 3.394 seconds | 20260506 10:59:19.613 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.70% / 0.04% / 54.26% | 20260506 10:59:19.613 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.87822% / 8.74201e-05% / 2.38419e-05% | 20260506 10:59:19.613 | INFO | Baz IO overall: | 20260506 10:59:19.613 | INFO | BytesWritten : 385025984855 | 20260506 10:59:19.613 | INFO | NumEvents : 185498378242 | 20260506 10:59:21.260 | INFO | Analyzing chunk frames = [823296,823808) | 20260506 10:59:21.260 | INFO | MemInfo: { | 20260506 10:59:21.260 | INFO | "gpu" : " 17 GB", | 20260506 10:59:21.260 | INFO | "huge" : "", | 20260506 10:59:21.260 | INFO | "huge_shared" : " 27 GB", | 20260506 10:59:21.260 | INFO | "malloc" : " 14 GB", | 20260506 10:59:21.260 | INFO | "pinned" : " 29 GB" | 20260506 10:59:21.260 | INFO | } | 20260506 10:59:21.260 | INFO | Perf report for the last 5.13942 seconds: | 20260506 10:59:21.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:59:21.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:21.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:21.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:21.260 | INFO | ANALYSIS : 57.4%, 2.8, 85.3ms, 0 | 20260506 10:59:21.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:21.260 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260506 10:59:21.260 | INFO | PRE_HQ : 56.5%, 2.0, 60.1ms, 0 | 20260506 10:59:21.260 | INFO | BAZWRITER : 65.6%, 4.8, 161.3ms, 0 | 20260506 10:59:21.260 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 10:59:21.261 | INFO | Uploaded 36GiB to GPU in 1.67604s (21.4793GiB/s) | 20260506 10:59:21.261 | INFO | Downloaded 38GiB to GPU in 1.74151s (21.8201GiB/s) | 20260506 10:59:26.399 | INFO | Analyzing chunk frames = [823808,824320) | 20260506 10:59:26.399 | INFO | MemInfo: { | 20260506 10:59:26.399 | INFO | "gpu" : " 17 GB", | 20260506 10:59:26.399 | INFO | "huge" : "", | 20260506 10:59:26.399 | INFO | "huge_shared" : " 27 GB", | 20260506 10:59:26.399 | INFO | "malloc" : " 14 GB", | 20260506 10:59:26.399 | INFO | "pinned" : " 29 GB" | 20260506 10:59:26.399 | INFO | } | 20260506 10:59:26.399 | INFO | Perf report for the last 5.1391 seconds: | 20260506 10:59:26.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:59:26.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:26.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:26.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:26.399 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 10:59:26.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:26.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:26.399 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 10:59:26.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:26.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:26.400 | INFO | Uploaded 36GiB to GPU in 1.60345s (22.4516GiB/s) | 20260506 10:59:26.400 | INFO | Downloaded 34GiB to GPU in 1.52084s (22.356GiB/s) | 20260506 10:59:31.539 | INFO | Analyzing chunk frames = [824320,824832) | 20260506 10:59:31.539 | INFO | MemInfo: { | 20260506 10:59:31.539 | INFO | "gpu" : " 17 GB", | 20260506 10:59:31.539 | INFO | "huge" : "", | 20260506 10:59:31.539 | INFO | "huge_shared" : " 27 GB", | 20260506 10:59:31.539 | INFO | "malloc" : " 14 GB", | 20260506 10:59:31.539 | INFO | "pinned" : " 29 GB" | 20260506 10:59:31.539 | INFO | } | 20260506 10:59:31.539 | INFO | Perf report for the last 5.13949 seconds: | 20260506 10:59:31.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:59:31.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:31.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:31.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:31.539 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260506 10:59:31.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:31.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:31.539 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 10:59:31.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:31.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:31.539 | INFO | Uploaded 36GiB to GPU in 1.60886s (22.3761GiB/s) | 20260506 10:59:31.539 | INFO | Downloaded 34GiB to GPU in 1.52693s (22.2669GiB/s) | 20260506 10:59:36.678 | INFO | Analyzing chunk frames = [824832,825344) | 20260506 10:59:36.678 | INFO | MemInfo: { | 20260506 10:59:36.678 | INFO | "gpu" : " 17 GB", | 20260506 10:59:36.678 | INFO | "huge" : "", | 20260506 10:59:36.678 | INFO | "huge_shared" : " 27 GB", | 20260506 10:59:36.678 | INFO | "malloc" : " 14 GB", | 20260506 10:59:36.678 | INFO | "pinned" : " 29 GB" | 20260506 10:59:36.678 | INFO | } | 20260506 10:59:36.678 | INFO | Perf report for the last 5.13906 seconds: | 20260506 10:59:36.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:59:36.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:36.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:36.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:36.678 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260506 10:59:36.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:36.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:36.678 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 10:59:36.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:36.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:36.678 | INFO | Uploaded 36GiB to GPU in 1.60679s (22.4049GiB/s) | 20260506 10:59:36.678 | INFO | Downloaded 34GiB to GPU in 1.5251s (22.2936GiB/s) | 20260506 10:59:41.817 | INFO | Analyzing chunk frames = [825344,825856) | 20260506 10:59:41.817 | INFO | MemInfo: { | 20260506 10:59:41.817 | INFO | "gpu" : " 17 GB", | 20260506 10:59:41.817 | INFO | "huge" : "", | 20260506 10:59:41.817 | INFO | "huge_shared" : " 27 GB", | 20260506 10:59:41.817 | INFO | "malloc" : " 14 GB", | 20260506 10:59:41.817 | INFO | "pinned" : " 29 GB" | 20260506 10:59:41.817 | INFO | } | 20260506 10:59:41.818 | INFO | Perf report for the last 5.13963 seconds: | 20260506 10:59:41.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:59:41.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:41.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:41.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:41.818 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260506 10:59:41.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:41.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:41.818 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 10:59:41.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:41.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:41.818 | INFO | Uploaded 36GiB to GPU in 1.60612s (22.4143GiB/s) | 20260506 10:59:41.818 | INFO | Downloaded 34GiB to GPU in 1.52303s (22.3239GiB/s) | 20260506 10:59:46.957 | INFO | Analyzing chunk frames = [825856,826368) | 20260506 10:59:46.957 | INFO | MemInfo: { | 20260506 10:59:46.957 | INFO | "gpu" : " 17 GB", | 20260506 10:59:46.957 | INFO | "huge" : "", | 20260506 10:59:46.957 | INFO | "huge_shared" : " 27 GB", | 20260506 10:59:46.957 | INFO | "malloc" : " 14 GB", | 20260506 10:59:46.957 | INFO | "pinned" : " 29 GB" | 20260506 10:59:46.957 | INFO | } | 20260506 10:59:46.957 | INFO | Perf report for the last 5.13928 seconds: | 20260506 10:59:46.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:59:46.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:46.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:46.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:46.957 | INFO | ANALYSIS : 56.6%, 3.0, 88.2ms, 0 | 20260506 10:59:46.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:46.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:46.957 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 10:59:46.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:46.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:46.957 | INFO | Uploaded 36GiB to GPU in 1.60646s (22.4095GiB/s) | 20260506 10:59:46.957 | INFO | Downloaded 34GiB to GPU in 1.52393s (22.3108GiB/s) | 20260506 10:59:52.096 | INFO | Analyzing chunk frames = [826368,826880) | 20260506 10:59:52.096 | INFO | MemInfo: { | 20260506 10:59:52.096 | INFO | "gpu" : " 17 GB", | 20260506 10:59:52.096 | INFO | "huge" : "", | 20260506 10:59:52.096 | INFO | "huge_shared" : " 27 GB", | 20260506 10:59:52.096 | INFO | "malloc" : " 14 GB", | 20260506 10:59:52.096 | INFO | "pinned" : " 29 GB" | 20260506 10:59:52.096 | INFO | } | 20260506 10:59:52.096 | INFO | Perf report for the last 5.13944 seconds: | 20260506 10:59:52.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:59:52.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:52.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:52.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:52.096 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 10:59:52.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:52.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:52.096 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 10:59:52.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:52.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:52.097 | INFO | Uploaded 36GiB to GPU in 1.60614s (22.4139GiB/s) | 20260506 10:59:52.097 | INFO | Downloaded 34GiB to GPU in 1.52388s (22.3115GiB/s) | 20260506 10:59:57.235 | INFO | Analyzing chunk frames = [826880,827392) | 20260506 10:59:57.235 | INFO | MemInfo: { | 20260506 10:59:57.235 | INFO | "gpu" : " 17 GB", | 20260506 10:59:57.235 | INFO | "huge" : "", | 20260506 10:59:57.235 | INFO | "huge_shared" : " 27 GB", | 20260506 10:59:57.235 | INFO | "malloc" : " 14 GB", | 20260506 10:59:57.235 | INFO | "pinned" : " 29 GB" | 20260506 10:59:57.235 | INFO | } | 20260506 10:59:57.236 | INFO | Perf report for the last 5.13934 seconds: | 20260506 10:59:57.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 10:59:57.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:57.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:57.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:57.236 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 10:59:57.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:57.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:57.236 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 10:59:57.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:57.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 10:59:57.236 | INFO | Uploaded 36GiB to GPU in 1.60623s (22.4127GiB/s) | 20260506 10:59:57.236 | INFO | Downloaded 34GiB to GPU in 1.52404s (22.3092GiB/s) | 20260506 10:59:57.236 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 10:59:57.236 | INFO | Total mem usage: 16.5 GiB | 20260506 10:59:57.236 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.76%) | 20260506 10:59:57.236 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -5.14%) | 20260506 10:59:57.236 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 10:59:57.236 | INFO | High Water Breakdown: | 20260506 10:59:57.236 | INFO | 60.6% : 9.52 GiB ( 94.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 10:59:57.236 | INFO | 39.3% : 6.17 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 10:59:57.236 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 10:59:57.236 | INFO | ------------------------------------------ | 20260506 10:59:57.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:59:57.236 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 10:59:57.236 | INFO | Total mem usage: 26.5 GiB | 20260506 10:59:57.236 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 10:59:57.237 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 10:59:57.237 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 10:59:57.237 | INFO | High Water Breakdown: | 20260506 10:59:57.237 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:59:57.237 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:59:57.237 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:59:57.237 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:59:57.237 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:59:57.237 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:59:57.237 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 10:59:57.237 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 10:59:57.237 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:59:57.237 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 10:59:57.237 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 10:59:57.237 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 10:59:57.237 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 10:59:57.237 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 10:59:57.237 | INFO | ------------------------------------------ | 20260506 10:59:57.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:59:57.237 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 10:59:57.237 | INFO | Total mem usage: 25.6 GiB | 20260506 10:59:57.237 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 10:59:57.237 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 10:59:57.237 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 10:59:57.237 | INFO | High Water Breakdown: | 20260506 10:59:57.237 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 10:59:57.237 | INFO | ------------------------------------------ | 20260506 10:59:57.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:59:57.237 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 10:59:57.237 | INFO | Total mem usage: 15.7 GiB | 20260506 10:59:57.237 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 10:59:57.237 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 10:59:57.237 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 10:59:57.237 | INFO | High Water Breakdown: | 20260506 10:59:57.237 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 10:59:57.237 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 10:59:57.237 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 10:59:57.237 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 10:59:57.237 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 10:59:57.237 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 10:59:57.237 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 10:59:57.237 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 10:59:57.237 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 10:59:57.237 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 10:59:57.237 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 10:59:57.237 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 10:59:57.237 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 10:59:57.237 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 10:59:57.237 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 10:59:57.237 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 10:59:57.237 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 10:59:57.237 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 10:59:57.237 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 10:59:57.237 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 10:59:57.237 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 10:59:57.237 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 10:59:57.237 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 10:59:57.237 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 10:59:57.237 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 10:59:57.237 | INFO | ------------------------------------------ | 20260506 10:59:57.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 10:59:57.237 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 10:59:57.237 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:59:57.237 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:59:57.237 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 10:59:57.237 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 5.42e-03 / 0.00e+00 | 20260506 10:59:57.237 | INFO | PerfReport: 10.98%, Baseline -- 3.30e+00 / -nan, 1.59e-02 / 0.00e+00 | 20260506 10:59:57.237 | INFO | PerfReport: 3.07%, Histogram -- 9.21e-01 / -nan, 5.18e-02 / 0.00e+00 | 20260506 10:59:57.237 | INFO | PerfReport: 16.06%, DME -- 3.86e+01 / -nan, 3.04e+00 / 0.00e+00 | 20260506 10:59:57.237 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.72e+01 / -nan, 4.29e-02 / 0.00e+00 | 20260506 10:59:57.237 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.20e+00 / -nan, 1.26e-01 / 0.00e+00 | 20260506 10:59:57.237 | INFO | PerfReport: 3.62%, Metrics -- 1.09e+00 / -nan, 1.69e-01 / 0.00e+00 | 20260506 10:59:57.237 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.18e-01 / -nan, 5.82e-02 / 0.00e+00 | 20260506 10:59:57.237 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 10:59:57.237 | INFO | PerfReport: Gathered 1536 samples. | 20260506 10:59:57.237 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 10:59:57.238 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 10:59:57.238 | INFO | PerfReport: 18.76%, Upload -- 1.68e+01 / 0.00e+00, 4.98e-01 / 0.00e+00 | 20260506 10:59:57.238 | INFO | PerfReport: 18.02%, Download -- 1.62e+01 / 0.00e+00, 8.50e-01 / 0.00e+00 | 20260506 10:59:57.238 | INFO | PerfReport: 33.49%, Compute -- 3.00e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 10:59:57.238 | INFO | PerfReport: 0.57%, LockUpload -- 5.07e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 10:59:57.238 | INFO | PerfReport: 0.00%, LockDownload -- 4.88e-04 / 0.00e+00, 1.18e-04 / 0.00e+00 | 20260506 10:59:57.238 | INFO | PerfReport: 29.16%, LockCompute -- 2.62e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 10:59:57.238 | INFO | PerfReport: 0.02%, Misc -- 1.40e-02 / 0.00e+00, 2.37e-02 / 0.00e+00 | 20260506 10:59:57.333 | INFO | Starting a round of Baz IO | 20260506 11:00:00.237 | INFO | Populating RTMetrics report | 20260506 11:00:00.238 | INFO | RTMetrics report populated | 20260506 11:00:00.238 | INFO | Opening tmp RTMetrics json file | 20260506 11:00:00.336 | INFO | Serializing RTMetrics to json | 20260506 11:00:00.358 | INFO | Serialization done, swapping tmp file now... | 20260506 11:00:00.363 | INFO | RTMetrics done | 20260506 11:00:00.571 | INFO | Finished a round of Baz IO: Wrote 2.031 GiB in 3.238 seconds | 20260506 11:00:00.571 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.66% / 0.04% / 54.30% | 20260506 11:00:00.571 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.86405% / 6.75519e-05% / 2.38419e-05% | 20260506 11:00:00.571 | INFO | Baz IO overall: | 20260506 11:00:00.571 | INFO | BytesWritten : 387206995025 | 20260506 11:00:00.571 | INFO | NumEvents : 186557916333 | 20260506 11:00:02.375 | INFO | Analyzing chunk frames = [827392,827904) | 20260506 11:00:02.375 | INFO | MemInfo: { | 20260506 11:00:02.375 | INFO | "gpu" : " 17 GB", | 20260506 11:00:02.375 | INFO | "huge" : "", | 20260506 11:00:02.375 | INFO | "huge_shared" : " 27 GB", | 20260506 11:00:02.375 | INFO | "malloc" : " 14 GB", | 20260506 11:00:02.375 | INFO | "pinned" : " 29 GB" | 20260506 11:00:02.375 | INFO | } | 20260506 11:00:02.375 | INFO | Perf report for the last 5.1391 seconds: | 20260506 11:00:02.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:00:02.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:02.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:02.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:02.375 | INFO | ANALYSIS : 57.4%, 2.8, 85.4ms, 0 | 20260506 11:00:02.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:02.375 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260506 11:00:02.375 | INFO | PRE_HQ : 56.5%, 2.1, 60.2ms, 0 | 20260506 11:00:02.375 | INFO | BAZWRITER : 62.4%, 4.8, 153.8ms, 0 | 20260506 11:00:02.375 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 11:00:02.375 | INFO | Uploaded 36GiB to GPU in 1.67937s (21.4366GiB/s) | 20260506 11:00:02.375 | INFO | Downloaded 38GiB to GPU in 1.74478s (21.7792GiB/s) | 20260506 11:00:07.514 | INFO | Analyzing chunk frames = [827904,828416) | 20260506 11:00:07.514 | INFO | MemInfo: { | 20260506 11:00:07.514 | INFO | "gpu" : " 17 GB", | 20260506 11:00:07.514 | INFO | "huge" : "", | 20260506 11:00:07.514 | INFO | "huge_shared" : " 27 GB", | 20260506 11:00:07.514 | INFO | "malloc" : " 14 GB", | 20260506 11:00:07.514 | INFO | "pinned" : " 29 GB" | 20260506 11:00:07.514 | INFO | } | 20260506 11:00:07.514 | INFO | Perf report for the last 5.13939 seconds: | 20260506 11:00:07.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:00:07.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:07.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:07.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:07.514 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 11:00:07.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:07.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:07.514 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 11:00:07.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:07.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:07.515 | INFO | Uploaded 36GiB to GPU in 1.6037s (22.4481GiB/s) | 20260506 11:00:07.515 | INFO | Downloaded 34GiB to GPU in 1.52046s (22.3616GiB/s) | 20260506 11:00:12.653 | INFO | Analyzing chunk frames = [828416,828928) | 20260506 11:00:12.653 | INFO | MemInfo: { | 20260506 11:00:12.653 | INFO | "gpu" : " 17 GB", | 20260506 11:00:12.653 | INFO | "huge" : "", | 20260506 11:00:12.653 | INFO | "huge_shared" : " 27 GB", | 20260506 11:00:12.653 | INFO | "malloc" : " 14 GB", | 20260506 11:00:12.653 | INFO | "pinned" : " 29 GB" | 20260506 11:00:12.654 | INFO | } | 20260506 11:00:12.654 | INFO | Perf report for the last 5.13942 seconds: | 20260506 11:00:12.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:00:12.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:12.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:12.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:12.654 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260506 11:00:12.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:12.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:12.654 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:00:12.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:12.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:12.654 | INFO | Uploaded 36GiB to GPU in 1.60829s (22.384GiB/s) | 20260506 11:00:12.654 | INFO | Downloaded 34GiB to GPU in 1.52588s (22.2822GiB/s) | 20260506 11:00:17.793 | INFO | Analyzing chunk frames = [828928,829440) | 20260506 11:00:17.793 | INFO | MemInfo: { | 20260506 11:00:17.793 | INFO | "gpu" : " 17 GB", | 20260506 11:00:17.793 | INFO | "huge" : "", | 20260506 11:00:17.793 | INFO | "huge_shared" : " 27 GB", | 20260506 11:00:17.793 | INFO | "malloc" : " 14 GB", | 20260506 11:00:17.793 | INFO | "pinned" : " 29 GB" | 20260506 11:00:17.793 | INFO | } | 20260506 11:00:17.793 | INFO | Perf report for the last 5.13932 seconds: | 20260506 11:00:17.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:00:17.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:17.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:17.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:17.793 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260506 11:00:17.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:17.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:17.793 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:00:17.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:17.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:17.793 | INFO | Uploaded 36GiB to GPU in 1.60761s (22.3935GiB/s) | 20260506 11:00:17.793 | INFO | Downloaded 34GiB to GPU in 1.52477s (22.2985GiB/s) | 20260506 11:00:22.932 | INFO | Analyzing chunk frames = [829440,829952) | 20260506 11:00:22.932 | INFO | MemInfo: { | 20260506 11:00:22.932 | INFO | "gpu" : " 17 GB", | 20260506 11:00:22.932 | INFO | "huge" : "", | 20260506 11:00:22.932 | INFO | "huge_shared" : " 27 GB", | 20260506 11:00:22.932 | INFO | "malloc" : " 14 GB", | 20260506 11:00:22.932 | INFO | "pinned" : " 29 GB" | 20260506 11:00:22.932 | INFO | } | 20260506 11:00:22.932 | INFO | Perf report for the last 5.13926 seconds: | 20260506 11:00:22.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:00:22.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:22.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:22.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:22.932 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260506 11:00:22.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:22.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:22.932 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:00:22.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:22.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:22.932 | INFO | Uploaded 36GiB to GPU in 1.60574s (22.4196GiB/s) | 20260506 11:00:22.932 | INFO | Downloaded 34GiB to GPU in 1.523s (22.3243GiB/s) | 20260506 11:00:28.072 | INFO | Analyzing chunk frames = [829952,830464) | 20260506 11:00:28.072 | INFO | MemInfo: { | 20260506 11:00:28.072 | INFO | "gpu" : " 17 GB", | 20260506 11:00:28.072 | INFO | "huge" : "", | 20260506 11:00:28.072 | INFO | "huge_shared" : " 27 GB", | 20260506 11:00:28.072 | INFO | "malloc" : " 14 GB", | 20260506 11:00:28.072 | INFO | "pinned" : " 29 GB" | 20260506 11:00:28.072 | INFO | } | 20260506 11:00:28.072 | INFO | Perf report for the last 5.13953 seconds: | 20260506 11:00:28.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:00:28.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:28.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:28.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:28.072 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 11:00:28.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:28.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:28.072 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 11:00:28.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:28.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:28.072 | INFO | Uploaded 36GiB to GPU in 1.60534s (22.4252GiB/s) | 20260506 11:00:28.072 | INFO | Downloaded 34GiB to GPU in 1.52297s (22.3249GiB/s) | 20260506 11:00:33.211 | INFO | Analyzing chunk frames = [830464,830976) | 20260506 11:00:33.211 | INFO | MemInfo: { | 20260506 11:00:33.211 | INFO | "gpu" : " 17 GB", | 20260506 11:00:33.211 | INFO | "huge" : "", | 20260506 11:00:33.211 | INFO | "huge_shared" : " 27 GB", | 20260506 11:00:33.211 | INFO | "malloc" : " 14 GB", | 20260506 11:00:33.211 | INFO | "pinned" : " 29 GB" | 20260506 11:00:33.211 | INFO | } | 20260506 11:00:33.211 | INFO | Perf report for the last 5.13929 seconds: | 20260506 11:00:33.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:00:33.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:33.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:33.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:33.211 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 11:00:33.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:33.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:33.211 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:00:33.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:33.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:33.211 | INFO | Uploaded 36GiB to GPU in 1.60714s (22.4001GiB/s) | 20260506 11:00:33.211 | INFO | Downloaded 34GiB to GPU in 1.52546s (22.2884GiB/s) | 20260506 11:00:38.350 | INFO | Analyzing chunk frames = [830976,831488) | 20260506 11:00:38.350 | INFO | MemInfo: { | 20260506 11:00:38.350 | INFO | "gpu" : " 17 GB", | 20260506 11:00:38.350 | INFO | "huge" : "", | 20260506 11:00:38.350 | INFO | "huge_shared" : " 27 GB", | 20260506 11:00:38.350 | INFO | "malloc" : " 14 GB", | 20260506 11:00:38.350 | INFO | "pinned" : " 29 GB" | 20260506 11:00:38.350 | INFO | } | 20260506 11:00:38.350 | INFO | Perf report for the last 5.13934 seconds: | 20260506 11:00:38.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:00:38.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:38.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:38.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:38.350 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 11:00:38.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:38.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:38.351 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:00:38.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:38.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:38.351 | INFO | Uploaded 36GiB to GPU in 1.60582s (22.4185GiB/s) | 20260506 11:00:38.351 | INFO | Downloaded 34GiB to GPU in 1.52502s (22.2948GiB/s) | 20260506 11:00:38.447 | INFO | Starting a round of Baz IO | 20260506 11:00:41.348 | INFO | Populating RTMetrics report | 20260506 11:00:41.348 | INFO | RTMetrics report populated | 20260506 11:00:41.348 | INFO | Opening tmp RTMetrics json file | 20260506 11:00:41.368 | INFO | Serializing RTMetrics to json | 20260506 11:00:41.382 | INFO | Serialization done, swapping tmp file now... | 20260506 11:00:41.452 | INFO | RTMetrics done | 20260506 11:00:42.298 | INFO | Finished a round of Baz IO: Wrote 2.031 GiB in 3.85 seconds | 20260506 11:00:42.298 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.62% / 0.04% / 54.34% | 20260506 11:00:42.298 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.82575% / 7.54992e-05% / 6.75519e-05% | 20260506 11:00:42.298 | INFO | Baz IO overall: | 20260506 11:00:42.298 | INFO | BytesWritten : 389387541436 | 20260506 11:00:42.298 | INFO | NumEvents : 187615897471 | 20260506 11:00:43.490 | INFO | Analyzing chunk frames = [831488,832000) | 20260506 11:00:43.490 | INFO | MemInfo: { | 20260506 11:00:43.490 | INFO | "gpu" : " 17 GB", | 20260506 11:00:43.490 | INFO | "huge" : "", | 20260506 11:00:43.490 | INFO | "huge_shared" : " 27 GB", | 20260506 11:00:43.490 | INFO | "malloc" : " 14 GB", | 20260506 11:00:43.490 | INFO | "pinned" : " 29 GB" | 20260506 11:00:43.490 | INFO | } | 20260506 11:00:43.490 | INFO | Perf report for the last 5.13923 seconds: | 20260506 11:00:43.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:00:43.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:43.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:43.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:43.490 | INFO | ANALYSIS : 57.3%, 2.8, 85.5ms, 0 | 20260506 11:00:43.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:43.490 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 11:00:43.490 | INFO | PRE_HQ : 56.4%, 2.1, 60.2ms, 0 | 20260506 11:00:43.490 | INFO | BAZWRITER : 74.5%, 4.8, 183.6ms, 0 | 20260506 11:00:43.490 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 11:00:43.490 | INFO | Uploaded 36GiB to GPU in 1.67381s (21.5078GiB/s) | 20260506 11:00:43.490 | INFO | Downloaded 38GiB to GPU in 1.73875s (21.8548GiB/s) | 20260506 11:00:48.629 | INFO | Analyzing chunk frames = [832000,832512) | 20260506 11:00:48.629 | INFO | MemInfo: { | 20260506 11:00:48.629 | INFO | "gpu" : " 17 GB", | 20260506 11:00:48.629 | INFO | "huge" : "", | 20260506 11:00:48.629 | INFO | "huge_shared" : " 27 GB", | 20260506 11:00:48.629 | INFO | "malloc" : " 14 GB", | 20260506 11:00:48.629 | INFO | "pinned" : " 29 GB" | 20260506 11:00:48.629 | INFO | } | 20260506 11:00:48.629 | INFO | Perf report for the last 5.13952 seconds: | 20260506 11:00:48.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:00:48.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:48.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:48.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:48.629 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 11:00:48.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:48.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:48.629 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:00:48.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:48.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:48.629 | INFO | Uploaded 36GiB to GPU in 1.60334s (22.4532GiB/s) | 20260506 11:00:48.630 | INFO | Downloaded 34GiB to GPU in 1.52138s (22.3481GiB/s) | 20260506 11:00:53.768 | INFO | Analyzing chunk frames = [832512,833024) | 20260506 11:00:53.768 | INFO | MemInfo: { | 20260506 11:00:53.768 | INFO | "gpu" : " 17 GB", | 20260506 11:00:53.768 | INFO | "huge" : "", | 20260506 11:00:53.768 | INFO | "huge_shared" : " 27 GB", | 20260506 11:00:53.768 | INFO | "malloc" : " 14 GB", | 20260506 11:00:53.768 | INFO | "pinned" : " 29 GB" | 20260506 11:00:53.768 | INFO | } | 20260506 11:00:53.768 | INFO | Perf report for the last 5.1392 seconds: | 20260506 11:00:53.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:00:53.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:53.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:53.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:53.768 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260506 11:00:53.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:53.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:53.769 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:00:53.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:53.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:53.769 | INFO | Uploaded 36GiB to GPU in 1.60875s (22.3776GiB/s) | 20260506 11:00:53.769 | INFO | Downloaded 34GiB to GPU in 1.52615s (22.2783GiB/s) | 20260506 11:00:58.907 | INFO | Analyzing chunk frames = [833024,833536) | 20260506 11:00:58.907 | INFO | MemInfo: { | 20260506 11:00:58.908 | INFO | "gpu" : " 17 GB", | 20260506 11:00:58.908 | INFO | "huge" : "", | 20260506 11:00:58.908 | INFO | "huge_shared" : " 27 GB", | 20260506 11:00:58.908 | INFO | "malloc" : " 14 GB", | 20260506 11:00:58.908 | INFO | "pinned" : " 29 GB" | 20260506 11:00:58.908 | INFO | } | 20260506 11:00:58.908 | INFO | Perf report for the last 5.13928 seconds: | 20260506 11:00:58.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:00:58.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:58.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:58.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:58.908 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 11:00:58.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:58.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:58.908 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:00:58.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:58.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:00:58.908 | INFO | Uploaded 36GiB to GPU in 1.60716s (22.3997GiB/s) | 20260506 11:00:58.908 | INFO | Downloaded 34GiB to GPU in 1.52509s (22.2938GiB/s) | 20260506 11:01:04.047 | INFO | Analyzing chunk frames = [833536,834048) | 20260506 11:01:04.047 | INFO | MemInfo: { | 20260506 11:01:04.047 | INFO | "gpu" : " 17 GB", | 20260506 11:01:04.047 | INFO | "huge" : "", | 20260506 11:01:04.047 | INFO | "huge_shared" : " 27 GB", | 20260506 11:01:04.047 | INFO | "malloc" : " 14 GB", | 20260506 11:01:04.047 | INFO | "pinned" : " 29 GB" | 20260506 11:01:04.047 | INFO | } | 20260506 11:01:04.047 | INFO | Perf report for the last 5.13932 seconds: | 20260506 11:01:04.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:01:04.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:04.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:04.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:04.047 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260506 11:01:04.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:04.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:04.047 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:01:04.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:04.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:04.047 | INFO | Uploaded 36GiB to GPU in 1.60635s (22.411GiB/s) | 20260506 11:01:04.047 | INFO | Downloaded 34GiB to GPU in 1.52391s (22.311GiB/s) | 20260506 11:01:09.186 | INFO | Analyzing chunk frames = [834048,834560) | 20260506 11:01:09.186 | INFO | MemInfo: { | 20260506 11:01:09.186 | INFO | "gpu" : " 17 GB", | 20260506 11:01:09.186 | INFO | "huge" : "", | 20260506 11:01:09.186 | INFO | "huge_shared" : " 27 GB", | 20260506 11:01:09.186 | INFO | "malloc" : " 14 GB", | 20260506 11:01:09.186 | INFO | "pinned" : " 29 GB" | 20260506 11:01:09.186 | INFO | } | 20260506 11:01:09.186 | INFO | Perf report for the last 5.13929 seconds: | 20260506 11:01:09.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:01:09.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:09.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:09.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:09.186 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 11:01:09.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:09.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:09.186 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:01:09.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:09.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:09.187 | INFO | Uploaded 36GiB to GPU in 1.60653s (22.4085GiB/s) | 20260506 11:01:09.187 | INFO | Downloaded 34GiB to GPU in 1.52333s (22.3196GiB/s) | 20260506 11:01:14.326 | INFO | Analyzing chunk frames = [834560,835072) | 20260506 11:01:14.326 | INFO | MemInfo: { | 20260506 11:01:14.326 | INFO | "gpu" : " 17 GB", | 20260506 11:01:14.326 | INFO | "huge" : "", | 20260506 11:01:14.326 | INFO | "huge_shared" : " 27 GB", | 20260506 11:01:14.326 | INFO | "malloc" : " 14 GB", | 20260506 11:01:14.326 | INFO | "pinned" : " 29 GB" | 20260506 11:01:14.326 | INFO | } | 20260506 11:01:14.326 | INFO | Perf report for the last 5.13969 seconds: | 20260506 11:01:14.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:01:14.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:14.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:14.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:14.326 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 11:01:14.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:14.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:14.326 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:01:14.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:14.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:14.326 | INFO | Uploaded 36GiB to GPU in 1.60595s (22.4166GiB/s) | 20260506 11:01:14.326 | INFO | Downloaded 34GiB to GPU in 1.52324s (22.3208GiB/s) | 20260506 11:01:19.465 | INFO | Analyzing chunk frames = [835072,835584) | 20260506 11:01:19.465 | INFO | MemInfo: { | 20260506 11:01:19.465 | INFO | "gpu" : " 17 GB", | 20260506 11:01:19.465 | INFO | "huge" : "", | 20260506 11:01:19.465 | INFO | "huge_shared" : " 27 GB", | 20260506 11:01:19.465 | INFO | "malloc" : " 14 GB", | 20260506 11:01:19.465 | INFO | "pinned" : " 29 GB" | 20260506 11:01:19.465 | INFO | } | 20260506 11:01:19.465 | INFO | Perf report for the last 5.13916 seconds: | 20260506 11:01:19.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:01:19.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:19.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:19.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:19.465 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 11:01:19.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:19.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:19.465 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:01:19.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:19.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:19.466 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:01:19.466 | INFO | Total mem usage: 16.5 GiB | 20260506 11:01:19.466 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.85%) | 20260506 11:01:19.466 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: -0.239%) | 20260506 11:01:19.466 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:01:19.466 | INFO | High Water Breakdown: | 20260506 11:01:19.466 | INFO | 61.4% : 10.1 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:01:19.466 | INFO | 38.5% : 6.34 GiB ( 94.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:01:19.466 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:01:19.466 | INFO | ------------------------------------------ | 20260506 11:01:19.466 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:01:19.466 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:01:19.466 | INFO | Total mem usage: 26.5 GiB | 20260506 11:01:19.466 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 11:01:19.466 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:01:19.466 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 11:01:19.466 | INFO | High Water Breakdown: | 20260506 11:01:19.466 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:01:19.466 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:01:19.466 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:01:19.466 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:01:19.466 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:01:19.466 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:01:19.466 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:01:19.466 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:01:19.466 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:01:19.466 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:01:19.466 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:01:19.466 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:01:19.466 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:01:19.466 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:01:19.466 | INFO | ------------------------------------------ | 20260506 11:01:19.466 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:01:19.466 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:01:19.466 | INFO | Total mem usage: 25.6 GiB | 20260506 11:01:19.466 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:01:19.466 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 11:01:19.466 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 11:01:19.466 | INFO | High Water Breakdown: | 20260506 11:01:19.466 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 11:01:19.466 | INFO | ------------------------------------------ | 20260506 11:01:19.466 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:01:19.466 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:01:19.466 | INFO | Total mem usage: 15.7 GiB | 20260506 11:01:19.466 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:01:19.466 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:01:19.466 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:01:19.466 | INFO | High Water Breakdown: | 20260506 11:01:19.466 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:01:19.466 | INFO | Uploaded 36GiB to GPU in 1.60721s (22.3991GiB/s) | 20260506 11:01:19.466 | INFO | Downloaded 34GiB to GPU in 1.52447s (22.3028GiB/s) | 20260506 11:01:19.466 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:01:19.466 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:01:19.466 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:01:19.466 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:01:19.466 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:01:19.466 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:01:19.466 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:01:19.466 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:01:19.466 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:01:19.466 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:01:19.466 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:01:19.466 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:01:19.466 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:01:19.466 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:01:19.466 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:01:19.466 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:01:19.466 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:01:19.466 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:01:19.466 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:01:19.466 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:01:19.466 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:01:19.466 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:01:19.467 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:01:19.467 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:01:19.467 | INFO | ------------------------------------------ | 20260506 11:01:19.467 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:01:19.467 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:01:19.467 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:01:19.467 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:01:19.467 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 11:01:19.467 | INFO | PerfReport: 0.29%, AggregateStats -- 8.66e-02 / -nan, 4.60e-03 / 0.00e+00 | 20260506 11:01:19.467 | INFO | PerfReport: 10.98%, Baseline -- 3.30e+00 / -nan, 1.75e-02 / 0.00e+00 | 20260506 11:01:19.467 | INFO | PerfReport: 3.07%, Histogram -- 9.21e-01 / -nan, 5.27e-02 / 0.00e+00 | 20260506 11:01:19.467 | INFO | PerfReport: 16.06%, DME -- 3.86e+01 / -nan, 3.04e+00 / 0.00e+00 | 20260506 11:01:19.467 | INFO | PerfReport: 57.26%, FrameLabeling -- 1.72e+01 / -nan, 4.24e-02 / 0.00e+00 | 20260506 11:01:19.467 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.20e+00 / -nan, 1.24e-01 / 0.00e+00 | 20260506 11:01:19.467 | INFO | PerfReport: 3.61%, Metrics -- 1.08e+00 / -nan, 1.68e-01 / 0.00e+00 | 20260506 11:01:19.467 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.17e-01 / -nan, 5.84e-02 / 0.00e+00 | 20260506 11:01:19.467 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:01:19.467 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:01:19.467 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:01:19.467 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 11:01:19.467 | INFO | PerfReport: 18.76%, Upload -- 1.68e+01 / 0.00e+00, 4.99e-01 / 0.00e+00 | 20260506 11:01:19.467 | INFO | PerfReport: 18.02%, Download -- 1.62e+01 / 0.00e+00, 8.55e-01 / 0.00e+00 | 20260506 11:01:19.467 | INFO | PerfReport: 33.48%, Compute -- 3.00e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 11:01:19.467 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 11:01:19.467 | INFO | PerfReport: 0.00%, LockDownload -- 4.88e-04 / 0.00e+00, 1.35e-04 / 0.00e+00 | 20260506 11:01:19.467 | INFO | PerfReport: 29.16%, LockCompute -- 2.62e+01 / 0.00e+00, 1.71e+01 / 0.00e+00 | 20260506 11:01:19.467 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.08e-02 / 0.00e+00 | 20260506 11:01:19.563 | INFO | Starting a round of Baz IO | 20260506 11:01:22.470 | INFO | Populating RTMetrics report | 20260506 11:01:22.471 | INFO | RTMetrics report populated | 20260506 11:01:22.471 | INFO | Opening tmp RTMetrics json file | 20260506 11:01:22.520 | INFO | Serializing RTMetrics to json | 20260506 11:01:22.533 | INFO | Serialization done, swapping tmp file now... | 20260506 11:01:22.540 | INFO | RTMetrics done | 20260506 11:01:22.767 | INFO | Finished a round of Baz IO: Wrote 2.031 GiB in 3.203 seconds | 20260506 11:01:22.767 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.58% / 0.04% / 54.38% | 20260506 11:01:22.767 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.80823% / 5.5631e-05% / 6.35783e-05% | 20260506 11:01:22.767 | INFO | Baz IO overall: | 20260506 11:01:22.767 | INFO | BytesWritten : 391568809265 | 20260506 11:01:22.767 | INFO | NumEvents : 188674395854 | 20260506 11:01:24.604 | INFO | Analyzing chunk frames = [835584,836096) | 20260506 11:01:24.604 | INFO | MemInfo: { | 20260506 11:01:24.604 | INFO | "gpu" : " 17 GB", | 20260506 11:01:24.604 | INFO | "huge" : "", | 20260506 11:01:24.604 | INFO | "huge_shared" : " 27 GB", | 20260506 11:01:24.604 | INFO | "malloc" : " 14 GB", | 20260506 11:01:24.604 | INFO | "pinned" : " 29 GB" | 20260506 11:01:24.604 | INFO | } | 20260506 11:01:24.604 | INFO | Perf report for the last 5.13919 seconds: | 20260506 11:01:24.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:01:24.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:24.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:24.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:24.604 | INFO | ANALYSIS : 57.5%, 2.8, 85.2ms, 0 | 20260506 11:01:24.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:24.604 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260506 11:01:24.604 | INFO | PRE_HQ : 56.5%, 2.1, 60.3ms, 0 | 20260506 11:01:24.605 | INFO | BAZWRITER : 62.0%, 4.8, 151.8ms, 0 | 20260506 11:01:24.605 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 11:01:24.605 | INFO | Uploaded 36GiB to GPU in 1.67549s (21.4863GiB/s) | 20260506 11:01:24.605 | INFO | Downloaded 38GiB to GPU in 1.73955s (21.8447GiB/s) | 20260506 11:01:29.744 | INFO | Analyzing chunk frames = [836096,836608) | 20260506 11:01:29.744 | INFO | MemInfo: { | 20260506 11:01:29.744 | INFO | "gpu" : " 17 GB", | 20260506 11:01:29.744 | INFO | "huge" : "", | 20260506 11:01:29.744 | INFO | "huge_shared" : " 27 GB", | 20260506 11:01:29.744 | INFO | "malloc" : " 14 GB", | 20260506 11:01:29.744 | INFO | "pinned" : " 29 GB" | 20260506 11:01:29.744 | INFO | } | 20260506 11:01:29.744 | INFO | Perf report for the last 5.1394 seconds: | 20260506 11:01:29.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:01:29.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:29.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:29.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:29.744 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 11:01:29.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:29.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:29.744 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:01:29.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:29.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:29.744 | INFO | Uploaded 36GiB to GPU in 1.60376s (22.4473GiB/s) | 20260506 11:01:29.744 | INFO | Downloaded 34GiB to GPU in 1.52144s (22.3473GiB/s) | 20260506 11:01:34.883 | INFO | Analyzing chunk frames = [836608,837120) | 20260506 11:01:34.883 | INFO | MemInfo: { | 20260506 11:01:34.883 | INFO | "gpu" : " 17 GB", | 20260506 11:01:34.883 | INFO | "huge" : "", | 20260506 11:01:34.883 | INFO | "huge_shared" : " 27 GB", | 20260506 11:01:34.883 | INFO | "malloc" : " 14 GB", | 20260506 11:01:34.883 | INFO | "pinned" : " 29 GB" | 20260506 11:01:34.883 | INFO | } | 20260506 11:01:34.883 | INFO | Perf report for the last 5.13941 seconds: | 20260506 11:01:34.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:01:34.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:34.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:34.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:34.883 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260506 11:01:34.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:34.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:34.883 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:01:34.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:34.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:34.884 | INFO | Uploaded 36GiB to GPU in 1.6077s (22.3923GiB/s) | 20260506 11:01:34.884 | INFO | Downloaded 34GiB to GPU in 1.5256s (22.2862GiB/s) | 20260506 11:01:40.022 | INFO | Analyzing chunk frames = [837120,837632) | 20260506 11:01:40.022 | INFO | MemInfo: { | 20260506 11:01:40.022 | INFO | "gpu" : " 17 GB", | 20260506 11:01:40.022 | INFO | "huge" : "", | 20260506 11:01:40.022 | INFO | "huge_shared" : " 27 GB", | 20260506 11:01:40.022 | INFO | "malloc" : " 14 GB", | 20260506 11:01:40.022 | INFO | "pinned" : " 29 GB" | 20260506 11:01:40.022 | INFO | } | 20260506 11:01:40.022 | INFO | Perf report for the last 5.13905 seconds: | 20260506 11:01:40.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:01:40.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:40.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:40.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:40.022 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 11:01:40.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:40.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:40.022 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:01:40.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:40.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:40.023 | INFO | Uploaded 36GiB to GPU in 1.60795s (22.3888GiB/s) | 20260506 11:01:40.023 | INFO | Downloaded 34GiB to GPU in 1.52531s (22.2905GiB/s) | 20260506 11:01:45.162 | INFO | Analyzing chunk frames = [837632,838144) | 20260506 11:01:45.162 | INFO | MemInfo: { | 20260506 11:01:45.162 | INFO | "gpu" : " 17 GB", | 20260506 11:01:45.162 | INFO | "huge" : "", | 20260506 11:01:45.162 | INFO | "huge_shared" : " 27 GB", | 20260506 11:01:45.162 | INFO | "malloc" : " 14 GB", | 20260506 11:01:45.162 | INFO | "pinned" : " 29 GB" | 20260506 11:01:45.162 | INFO | } | 20260506 11:01:45.162 | INFO | Perf report for the last 5.13969 seconds: | 20260506 11:01:45.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:01:45.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:45.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:45.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:45.162 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260506 11:01:45.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:45.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:45.162 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:01:45.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:45.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:45.163 | INFO | Uploaded 36GiB to GPU in 1.60575s (22.4194GiB/s) | 20260506 11:01:45.163 | INFO | Downloaded 34GiB to GPU in 1.52337s (22.3189GiB/s) | 20260506 11:01:50.301 | INFO | Analyzing chunk frames = [838144,838656) | 20260506 11:01:50.301 | INFO | MemInfo: { | 20260506 11:01:50.301 | INFO | "gpu" : " 17 GB", | 20260506 11:01:50.301 | INFO | "huge" : "", | 20260506 11:01:50.301 | INFO | "huge_shared" : " 27 GB", | 20260506 11:01:50.301 | INFO | "malloc" : " 14 GB", | 20260506 11:01:50.301 | INFO | "pinned" : " 29 GB" | 20260506 11:01:50.301 | INFO | } | 20260506 11:01:50.301 | INFO | Perf report for the last 5.13917 seconds: | 20260506 11:01:50.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:01:50.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:50.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:50.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:50.301 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 11:01:50.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:50.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:50.301 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:01:50.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:50.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:50.301 | INFO | Uploaded 36GiB to GPU in 1.60686s (22.404GiB/s) | 20260506 11:01:50.301 | INFO | Downloaded 34GiB to GPU in 1.52382s (22.3124GiB/s) | 20260506 11:01:55.440 | INFO | Analyzing chunk frames = [838656,839168) | 20260506 11:01:55.440 | INFO | MemInfo: { | 20260506 11:01:55.440 | INFO | "gpu" : " 17 GB", | 20260506 11:01:55.440 | INFO | "huge" : "", | 20260506 11:01:55.440 | INFO | "huge_shared" : " 27 GB", | 20260506 11:01:55.440 | INFO | "malloc" : " 14 GB", | 20260506 11:01:55.441 | INFO | "pinned" : " 29 GB" | 20260506 11:01:55.441 | INFO | } | 20260506 11:01:55.441 | INFO | Perf report for the last 5.13941 seconds: | 20260506 11:01:55.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:01:55.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:55.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:55.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:55.441 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 11:01:55.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:55.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:55.441 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:01:55.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:55.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:01:55.441 | INFO | Uploaded 36GiB to GPU in 1.60639s (22.4105GiB/s) | 20260506 11:01:55.441 | INFO | Downloaded 34GiB to GPU in 1.52471s (22.2993GiB/s) | 20260506 11:02:00.580 | INFO | Analyzing chunk frames = [839168,839680) | 20260506 11:02:00.580 | INFO | MemInfo: { | 20260506 11:02:00.580 | INFO | "gpu" : " 17 GB", | 20260506 11:02:00.580 | INFO | "huge" : "", | 20260506 11:02:00.580 | INFO | "huge_shared" : " 27 GB", | 20260506 11:02:00.580 | INFO | "malloc" : " 14 GB", | 20260506 11:02:00.580 | INFO | "pinned" : " 29 GB" | 20260506 11:02:00.580 | INFO | } | 20260506 11:02:00.580 | INFO | Perf report for the last 5.13945 seconds: | 20260506 11:02:00.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:02:00.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:00.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:00.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:00.580 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 11:02:00.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:00.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:00.580 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 11:02:00.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:00.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:00.580 | INFO | Uploaded 36GiB to GPU in 1.60701s (22.4018GiB/s) | 20260506 11:02:00.580 | INFO | Downloaded 34GiB to GPU in 1.52486s (22.2971GiB/s) | 20260506 11:02:00.678 | INFO | Starting a round of Baz IO | 20260506 11:02:03.585 | INFO | Populating RTMetrics report | 20260506 11:02:03.585 | INFO | RTMetrics report populated | 20260506 11:02:03.585 | INFO | Opening tmp RTMetrics json file | 20260506 11:02:03.591 | INFO | Serializing RTMetrics to json | 20260506 11:02:03.597 | INFO | Serialization done, swapping tmp file now... | 20260506 11:02:03.601 | INFO | RTMetrics done | 20260506 11:02:04.068 | INFO | Finished a round of Baz IO: Wrote 2.035 GiB in 3.39 seconds | 20260506 11:02:04.068 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.54% / 0.04% / 54.42% | 20260506 11:02:04.068 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.806% / 6.75519e-05% / 1.58946e-05% | 20260506 11:02:04.068 | INFO | Baz IO overall: | 20260506 11:02:04.068 | INFO | BytesWritten : 393753661527 | 20260506 11:02:04.068 | INFO | NumEvents : 189735316289 | 20260506 11:02:05.719 | INFO | Analyzing chunk frames = [839680,840192) | 20260506 11:02:05.719 | INFO | MemInfo: { | 20260506 11:02:05.719 | INFO | "gpu" : " 17 GB", | 20260506 11:02:05.719 | INFO | "huge" : "", | 20260506 11:02:05.719 | INFO | "huge_shared" : " 27 GB", | 20260506 11:02:05.719 | INFO | "malloc" : " 14 GB", | 20260506 11:02:05.719 | INFO | "pinned" : " 29 GB" | 20260506 11:02:05.719 | INFO | } | 20260506 11:02:05.719 | INFO | Perf report for the last 5.13906 seconds: | 20260506 11:02:05.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:02:05.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:05.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:05.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:05.719 | INFO | ANALYSIS : 57.5%, 2.8, 85.5ms, 0 | 20260506 11:02:05.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:05.719 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 11:02:05.719 | INFO | PRE_HQ : 56.5%, 2.0, 60.0ms, 0 | 20260506 11:02:05.719 | INFO | BAZWRITER : 65.6%, 4.8, 162.3ms, 0 | 20260506 11:02:05.719 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.5ms, 0 | 20260506 11:02:05.720 | INFO | Uploaded 36GiB to GPU in 1.68209s (21.4019GiB/s) | 20260506 11:02:05.720 | INFO | Downloaded 38GiB to GPU in 1.74629s (21.7604GiB/s) | 20260506 11:02:10.858 | INFO | Analyzing chunk frames = [840192,840704) | 20260506 11:02:10.858 | INFO | MemInfo: { | 20260506 11:02:10.858 | INFO | "gpu" : " 17 GB", | 20260506 11:02:10.858 | INFO | "huge" : "", | 20260506 11:02:10.858 | INFO | "huge_shared" : " 27 GB", | 20260506 11:02:10.858 | INFO | "malloc" : " 14 GB", | 20260506 11:02:10.858 | INFO | "pinned" : " 29 GB" | 20260506 11:02:10.858 | INFO | } | 20260506 11:02:10.858 | INFO | Perf report for the last 5.13934 seconds: | 20260506 11:02:10.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:02:10.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:10.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:10.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:10.859 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 11:02:10.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:10.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:10.859 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:02:10.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:10.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:10.859 | INFO | Uploaded 36GiB to GPU in 1.60388s (22.4456GiB/s) | 20260506 11:02:10.859 | INFO | Downloaded 34GiB to GPU in 1.52092s (22.355GiB/s) | 20260506 11:02:15.998 | INFO | Analyzing chunk frames = [840704,841216) | 20260506 11:02:15.998 | INFO | MemInfo: { | 20260506 11:02:15.998 | INFO | "gpu" : " 17 GB", | 20260506 11:02:15.998 | INFO | "huge" : "", | 20260506 11:02:15.998 | INFO | "huge_shared" : " 27 GB", | 20260506 11:02:15.998 | INFO | "malloc" : " 14 GB", | 20260506 11:02:15.998 | INFO | "pinned" : " 29 GB" | 20260506 11:02:15.998 | INFO | } | 20260506 11:02:15.998 | INFO | Perf report for the last 5.13945 seconds: | 20260506 11:02:15.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:02:15.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:15.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:15.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:15.998 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260506 11:02:15.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:15.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:15.998 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:02:15.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:15.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:15.998 | INFO | Uploaded 36GiB to GPU in 1.60784s (22.3902GiB/s) | 20260506 11:02:15.998 | INFO | Downloaded 34GiB to GPU in 1.52573s (22.2845GiB/s) | 20260506 11:02:21.137 | INFO | Analyzing chunk frames = [841216,841728) | 20260506 11:02:21.137 | INFO | MemInfo: { | 20260506 11:02:21.137 | INFO | "gpu" : " 17 GB", | 20260506 11:02:21.137 | INFO | "huge" : "", | 20260506 11:02:21.137 | INFO | "huge_shared" : " 27 GB", | 20260506 11:02:21.137 | INFO | "malloc" : " 14 GB", | 20260506 11:02:21.137 | INFO | "pinned" : " 29 GB" | 20260506 11:02:21.137 | INFO | } | 20260506 11:02:21.137 | INFO | Perf report for the last 5.13921 seconds: | 20260506 11:02:21.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:02:21.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:21.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:21.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:21.137 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 11:02:21.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:21.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:21.137 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:02:21.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:21.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:21.137 | INFO | Uploaded 36GiB to GPU in 1.60767s (22.3926GiB/s) | 20260506 11:02:21.137 | INFO | Downloaded 34GiB to GPU in 1.52492s (22.2963GiB/s) | 20260506 11:02:26.276 | INFO | Analyzing chunk frames = [841728,842240) | 20260506 11:02:26.276 | INFO | MemInfo: { | 20260506 11:02:26.276 | INFO | "gpu" : " 17 GB", | 20260506 11:02:26.276 | INFO | "huge" : "", | 20260506 11:02:26.276 | INFO | "huge_shared" : " 27 GB", | 20260506 11:02:26.276 | INFO | "malloc" : " 14 GB", | 20260506 11:02:26.276 | INFO | "pinned" : " 29 GB" | 20260506 11:02:26.276 | INFO | } | 20260506 11:02:26.277 | INFO | Perf report for the last 5.13936 seconds: | 20260506 11:02:26.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:02:26.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:26.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:26.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:26.277 | INFO | ANALYSIS : 56.3%, 3.0, 87.7ms, 0 | 20260506 11:02:26.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:26.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:26.277 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:02:26.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:26.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:26.277 | INFO | Uploaded 36GiB to GPU in 1.60563s (22.4211GiB/s) | 20260506 11:02:26.277 | INFO | Downloaded 34GiB to GPU in 1.52217s (22.3366GiB/s) | 20260506 11:02:31.416 | INFO | Analyzing chunk frames = [842240,842752) | 20260506 11:02:31.416 | INFO | MemInfo: { | 20260506 11:02:31.416 | INFO | "gpu" : " 17 GB", | 20260506 11:02:31.416 | INFO | "huge" : "", | 20260506 11:02:31.416 | INFO | "huge_shared" : " 27 GB", | 20260506 11:02:31.416 | INFO | "malloc" : " 14 GB", | 20260506 11:02:31.416 | INFO | "pinned" : " 29 GB" | 20260506 11:02:31.416 | INFO | } | 20260506 11:02:31.416 | INFO | Perf report for the last 5.13918 seconds: | 20260506 11:02:31.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:02:31.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:31.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:31.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:31.416 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 11:02:31.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:31.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:31.416 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:02:31.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:31.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:31.416 | INFO | Uploaded 36GiB to GPU in 1.60663s (22.4072GiB/s) | 20260506 11:02:31.416 | INFO | Downloaded 34GiB to GPU in 1.52367s (22.3145GiB/s) | 20260506 11:02:36.555 | INFO | Analyzing chunk frames = [842752,843264) | 20260506 11:02:36.555 | INFO | MemInfo: { | 20260506 11:02:36.555 | INFO | "gpu" : " 17 GB", | 20260506 11:02:36.555 | INFO | "huge" : "", | 20260506 11:02:36.555 | INFO | "huge_shared" : " 27 GB", | 20260506 11:02:36.555 | INFO | "malloc" : " 14 GB", | 20260506 11:02:36.555 | INFO | "pinned" : " 29 GB" | 20260506 11:02:36.555 | INFO | } | 20260506 11:02:36.555 | INFO | Perf report for the last 5.13949 seconds: | 20260506 11:02:36.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:02:36.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:36.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:36.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:36.555 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 11:02:36.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:36.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:36.555 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:02:36.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:36.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:36.556 | INFO | Uploaded 36GiB to GPU in 1.60597s (22.4163GiB/s) | 20260506 11:02:36.556 | INFO | Downloaded 34GiB to GPU in 1.52384s (22.312GiB/s) | 20260506 11:02:41.694 | INFO | Analyzing chunk frames = [843264,843776) | 20260506 11:02:41.694 | INFO | MemInfo: { | 20260506 11:02:41.694 | INFO | "gpu" : " 17 GB", | 20260506 11:02:41.694 | INFO | "huge" : "", | 20260506 11:02:41.695 | INFO | "huge_shared" : " 27 GB", | 20260506 11:02:41.695 | INFO | "malloc" : " 14 GB", | 20260506 11:02:41.695 | INFO | "pinned" : " 29 GB" | 20260506 11:02:41.695 | INFO | } | 20260506 11:02:41.695 | INFO | Perf report for the last 5.13942 seconds: | 20260506 11:02:41.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:02:41.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:41.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:41.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:41.695 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 11:02:41.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:41.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:41.695 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:02:41.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:41.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:41.695 | INFO | Uploaded 36GiB to GPU in 1.60673s (22.4057GiB/s) | 20260506 11:02:41.695 | INFO | Downloaded 34GiB to GPU in 1.52511s (22.2935GiB/s) | 20260506 11:02:41.695 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:02:41.695 | INFO | Total mem usage: 16.5 GiB | 20260506 11:02:41.695 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.85%) | 20260506 11:02:41.695 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -4.97%) | 20260506 11:02:41.695 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:02:41.695 | INFO | High Water Breakdown: | 20260506 11:02:41.695 | INFO | 60.7% : 9.55 GiB ( 94.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:02:41.695 | INFO | 39.2% : 6.16 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:02:41.695 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:02:41.695 | INFO | ------------------------------------------ | 20260506 11:02:41.695 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:02:41.695 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:02:41.695 | INFO | Total mem usage: 26.5 GiB | 20260506 11:02:41.696 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 11:02:41.696 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:02:41.696 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 11:02:41.696 | INFO | High Water Breakdown: | 20260506 11:02:41.696 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:02:41.696 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:02:41.696 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:02:41.696 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:02:41.696 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:02:41.696 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:02:41.696 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:02:41.696 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:02:41.696 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:02:41.696 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:02:41.696 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:02:41.696 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:02:41.696 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:02:41.696 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:02:41.696 | INFO | ------------------------------------------ | 20260506 11:02:41.696 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:02:41.696 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:02:41.696 | INFO | Total mem usage: 25.6 GiB | 20260506 11:02:41.696 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:02:41.696 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 11:02:41.696 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 11:02:41.696 | INFO | High Water Breakdown: | 20260506 11:02:41.696 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 11:02:41.696 | INFO | ------------------------------------------ | 20260506 11:02:41.696 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:02:41.696 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:02:41.696 | INFO | Total mem usage: 15.7 GiB | 20260506 11:02:41.696 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:02:41.696 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:02:41.696 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:02:41.696 | INFO | High Water Breakdown: | 20260506 11:02:41.696 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:02:41.696 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:02:41.696 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:02:41.696 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:02:41.696 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:02:41.696 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:02:41.696 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:02:41.696 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:02:41.696 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:02:41.696 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:02:41.696 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:02:41.696 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:02:41.696 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:02:41.696 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:02:41.696 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:02:41.696 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:02:41.696 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:02:41.696 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:02:41.696 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:02:41.696 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:02:41.696 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:02:41.696 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:02:41.696 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:02:41.696 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:02:41.696 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:02:41.696 | INFO | ------------------------------------------ | 20260506 11:02:41.696 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:02:41.696 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:02:41.696 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:02:41.696 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:02:41.696 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 11:02:41.696 | INFO | PerfReport: 0.29%, AggregateStats -- 8.64e-02 / -nan, 4.35e-03 / 0.00e+00 | 20260506 11:02:41.696 | INFO | PerfReport: 10.97%, Baseline -- 3.30e+00 / -nan, 1.66e-02 / 0.00e+00 | 20260506 11:02:41.696 | INFO | PerfReport: 3.07%, Histogram -- 9.21e-01 / -nan, 5.19e-02 / 0.00e+00 | 20260506 11:02:41.696 | INFO | PerfReport: 16.06%, DME -- 3.86e+01 / -nan, 3.00e+00 / 0.00e+00 | 20260506 11:02:41.696 | INFO | PerfReport: 57.26%, FrameLabeling -- 1.72e+01 / -nan, 4.29e-02 / 0.00e+00 | 20260506 11:02:41.696 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.20e+00 / -nan, 1.25e-01 / 0.00e+00 | 20260506 11:02:41.696 | INFO | PerfReport: 3.61%, Metrics -- 1.08e+00 / -nan, 1.69e-01 / 0.00e+00 | 20260506 11:02:41.696 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.18e-01 / -nan, 5.84e-02 / 0.00e+00 | 20260506 11:02:41.696 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:02:41.696 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:02:41.696 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:02:41.696 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.85e+01 / 0.00e+00 | 20260506 11:02:41.696 | INFO | PerfReport: 18.77%, Upload -- 1.68e+01 / 0.00e+00, 5.07e-01 / 0.00e+00 | 20260506 11:02:41.696 | INFO | PerfReport: 18.02%, Download -- 1.62e+01 / 0.00e+00, 8.59e-01 / 0.00e+00 | 20260506 11:02:41.696 | INFO | PerfReport: 33.49%, Compute -- 3.00e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 11:02:41.696 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260506 11:02:41.696 | INFO | PerfReport: 0.00%, LockDownload -- 4.99e-04 / 0.00e+00, 1.74e-04 / 0.00e+00 | 20260506 11:02:41.696 | INFO | PerfReport: 29.14%, LockCompute -- 2.61e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 11:02:41.696 | INFO | PerfReport: 0.02%, Misc -- 1.37e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260506 11:02:41.792 | INFO | Starting a round of Baz IO | 20260506 11:02:44.700 | INFO | Populating RTMetrics report | 20260506 11:02:44.700 | INFO | RTMetrics report populated | 20260506 11:02:44.700 | INFO | Opening tmp RTMetrics json file | 20260506 11:02:44.807 | INFO | Serializing RTMetrics to json | 20260506 11:02:44.812 | INFO | Serialization done, swapping tmp file now... | 20260506 11:02:44.817 | INFO | RTMetrics done | 20260506 11:02:45.139 | INFO | Finished a round of Baz IO: Wrote 2.038 GiB in 3.347 seconds | 20260506 11:02:45.139 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.50% / 0.04% / 54.46% | 20260506 11:02:45.139 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.80255% / 6.75519e-05% / 3.57628e-05% | 20260506 11:02:45.139 | INFO | Baz IO overall: | 20260506 11:02:45.139 | INFO | BytesWritten : 395941953297 | 20260506 11:02:45.139 | INFO | NumEvents : 190797591849 | 20260506 11:02:46.834 | INFO | Analyzing chunk frames = [843776,844288) | 20260506 11:02:46.834 | INFO | MemInfo: { | 20260506 11:02:46.834 | INFO | "gpu" : " 17 GB", | 20260506 11:02:46.834 | INFO | "huge" : "", | 20260506 11:02:46.834 | INFO | "huge_shared" : " 27 GB", | 20260506 11:02:46.834 | INFO | "malloc" : " 14 GB", | 20260506 11:02:46.834 | INFO | "pinned" : " 29 GB" | 20260506 11:02:46.834 | INFO | } | 20260506 11:02:46.834 | INFO | Perf report for the last 5.13908 seconds: | 20260506 11:02:46.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:02:46.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:46.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:46.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:46.834 | INFO | ANALYSIS : 57.5%, 2.9, 86.2ms, 0 | 20260506 11:02:46.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:46.834 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260506 11:02:46.834 | INFO | PRE_HQ : 56.4%, 2.1, 60.3ms, 0 | 20260506 11:02:46.834 | INFO | BAZWRITER : 64.7%, 4.7, 156.7ms, 0 | 20260506 11:02:46.834 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.3ms, 0 | 20260506 11:02:46.834 | INFO | Uploaded 36GiB to GPU in 1.67198s (21.5314GiB/s) | 20260506 11:02:46.834 | INFO | Downloaded 38GiB to GPU in 1.73679s (21.8795GiB/s) | 20260506 11:02:51.973 | INFO | Analyzing chunk frames = [844288,844800) | 20260506 11:02:51.973 | INFO | MemInfo: { | 20260506 11:02:51.973 | INFO | "gpu" : " 17 GB", | 20260506 11:02:51.973 | INFO | "huge" : "", | 20260506 11:02:51.973 | INFO | "huge_shared" : " 27 GB", | 20260506 11:02:51.973 | INFO | "malloc" : " 14 GB", | 20260506 11:02:51.973 | INFO | "pinned" : " 29 GB" | 20260506 11:02:51.973 | INFO | } | 20260506 11:02:51.973 | INFO | Perf report for the last 5.13956 seconds: | 20260506 11:02:51.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:02:51.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:51.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:51.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:51.973 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 11:02:51.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:51.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:51.973 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:02:51.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:51.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:51.974 | INFO | Uploaded 36GiB to GPU in 1.60358s (22.4498GiB/s) | 20260506 11:02:51.974 | INFO | Downloaded 34GiB to GPU in 1.52056s (22.3602GiB/s) | 20260506 11:02:57.112 | INFO | Analyzing chunk frames = [844800,845312) | 20260506 11:02:57.112 | INFO | MemInfo: { | 20260506 11:02:57.112 | INFO | "gpu" : " 17 GB", | 20260506 11:02:57.112 | INFO | "huge" : "", | 20260506 11:02:57.113 | INFO | "huge_shared" : " 27 GB", | 20260506 11:02:57.113 | INFO | "malloc" : " 14 GB", | 20260506 11:02:57.113 | INFO | "pinned" : " 29 GB" | 20260506 11:02:57.113 | INFO | } | 20260506 11:02:57.113 | INFO | Perf report for the last 5.1393 seconds: | 20260506 11:02:57.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:02:57.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:57.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:57.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:57.113 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260506 11:02:57.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:57.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:57.113 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:02:57.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:57.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:02:57.113 | INFO | Uploaded 36GiB to GPU in 1.60815s (22.3859GiB/s) | 20260506 11:02:57.113 | INFO | Downloaded 34GiB to GPU in 1.5261s (22.279GiB/s) | 20260506 11:03:02.252 | INFO | Analyzing chunk frames = [845312,845824) | 20260506 11:03:02.252 | INFO | MemInfo: { | 20260506 11:03:02.252 | INFO | "gpu" : " 17 GB", | 20260506 11:03:02.252 | INFO | "huge" : "", | 20260506 11:03:02.252 | INFO | "huge_shared" : " 27 GB", | 20260506 11:03:02.252 | INFO | "malloc" : " 14 GB", | 20260506 11:03:02.252 | INFO | "pinned" : " 29 GB" | 20260506 11:03:02.252 | INFO | } | 20260506 11:03:02.252 | INFO | Perf report for the last 5.13916 seconds: | 20260506 11:03:02.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:03:02.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:02.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:02.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:02.252 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 11:03:02.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:02.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:02.252 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:03:02.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:02.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:02.252 | INFO | Uploaded 36GiB to GPU in 1.60779s (22.391GiB/s) | 20260506 11:03:02.252 | INFO | Downloaded 34GiB to GPU in 1.52583s (22.283GiB/s) | 20260506 11:03:07.391 | INFO | Analyzing chunk frames = [845824,846336) | 20260506 11:03:07.391 | INFO | MemInfo: { | 20260506 11:03:07.391 | INFO | "gpu" : " 17 GB", | 20260506 11:03:07.391 | INFO | "huge" : "", | 20260506 11:03:07.391 | INFO | "huge_shared" : " 27 GB", | 20260506 11:03:07.391 | INFO | "malloc" : " 14 GB", | 20260506 11:03:07.391 | INFO | "pinned" : " 29 GB" | 20260506 11:03:07.391 | INFO | } | 20260506 11:03:07.391 | INFO | Perf report for the last 5.13933 seconds: | 20260506 11:03:07.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:03:07.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:07.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:07.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:07.391 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260506 11:03:07.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:07.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:07.391 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:03:07.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:07.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:07.392 | INFO | Uploaded 36GiB to GPU in 1.6061s (22.4146GiB/s) | 20260506 11:03:07.392 | INFO | Downloaded 34GiB to GPU in 1.52319s (22.3215GiB/s) | 20260506 11:03:12.530 | INFO | Analyzing chunk frames = [846336,846848) | 20260506 11:03:12.530 | INFO | MemInfo: { | 20260506 11:03:12.530 | INFO | "gpu" : " 17 GB", | 20260506 11:03:12.530 | INFO | "huge" : "", | 20260506 11:03:12.531 | INFO | "huge_shared" : " 27 GB", | 20260506 11:03:12.531 | INFO | "malloc" : " 14 GB", | 20260506 11:03:12.531 | INFO | "pinned" : " 29 GB" | 20260506 11:03:12.531 | INFO | } | 20260506 11:03:12.531 | INFO | Perf report for the last 5.13945 seconds: | 20260506 11:03:12.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:03:12.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:12.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:12.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:12.531 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 11:03:12.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:12.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:12.531 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:03:12.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:12.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:12.531 | INFO | Uploaded 36GiB to GPU in 1.60581s (22.4186GiB/s) | 20260506 11:03:12.531 | INFO | Downloaded 34GiB to GPU in 1.52401s (22.3096GiB/s) | 20260506 11:03:17.670 | INFO | Analyzing chunk frames = [846848,847360) | 20260506 11:03:17.670 | INFO | MemInfo: { | 20260506 11:03:17.670 | INFO | "gpu" : " 17 GB", | 20260506 11:03:17.670 | INFO | "huge" : "", | 20260506 11:03:17.670 | INFO | "huge_shared" : " 27 GB", | 20260506 11:03:17.670 | INFO | "malloc" : " 14 GB", | 20260506 11:03:17.670 | INFO | "pinned" : " 29 GB" | 20260506 11:03:17.670 | INFO | } | 20260506 11:03:17.670 | INFO | Perf report for the last 5.13936 seconds: | 20260506 11:03:17.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:03:17.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:17.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:17.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:17.670 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 11:03:17.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:17.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:17.670 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:03:17.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:17.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:17.670 | INFO | Uploaded 36GiB to GPU in 1.60687s (22.4039GiB/s) | 20260506 11:03:17.670 | INFO | Downloaded 34GiB to GPU in 1.52443s (22.3035GiB/s) | 20260506 11:03:22.809 | INFO | Analyzing chunk frames = [847360,847872) | 20260506 11:03:22.809 | INFO | MemInfo: { | 20260506 11:03:22.809 | INFO | "gpu" : " 17 GB", | 20260506 11:03:22.809 | INFO | "huge" : "", | 20260506 11:03:22.809 | INFO | "huge_shared" : " 27 GB", | 20260506 11:03:22.809 | INFO | "malloc" : " 14 GB", | 20260506 11:03:22.809 | INFO | "pinned" : " 29 GB" | 20260506 11:03:22.809 | INFO | } | 20260506 11:03:22.809 | INFO | Perf report for the last 5.13927 seconds: | 20260506 11:03:22.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:03:22.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:22.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:22.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:22.809 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 11:03:22.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:22.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:22.809 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 11:03:22.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:22.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:22.810 | INFO | Uploaded 36GiB to GPU in 1.60609s (22.4146GiB/s) | 20260506 11:03:22.810 | INFO | Downloaded 34GiB to GPU in 1.52463s (22.3005GiB/s) | 20260506 11:03:22.907 | INFO | Starting a round of Baz IO | 20260506 11:03:25.811 | INFO | Populating RTMetrics report | 20260506 11:03:25.811 | INFO | RTMetrics report populated | 20260506 11:03:25.811 | INFO | Opening tmp RTMetrics json file | 20260506 11:03:25.829 | INFO | Serializing RTMetrics to json | 20260506 11:03:25.854 | INFO | Serialization done, swapping tmp file now... | 20260506 11:03:25.854 | INFO | RTMetrics done | 20260506 11:03:26.167 | INFO | Finished a round of Baz IO: Wrote 2.036 GiB in 3.259 seconds | 20260506 11:03:26.167 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.45% / 0.04% / 54.50% | 20260506 11:03:26.167 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.77224% / 7.54992e-05% / 5.16574e-05% | 20260506 11:03:26.167 | INFO | Baz IO overall: | 20260506 11:03:26.167 | INFO | BytesWritten : 398127886123 | 20260506 11:03:26.167 | INFO | NumEvents : 191858482498 | 20260506 11:03:27.948 | INFO | Analyzing chunk frames = [847872,848384) | 20260506 11:03:27.948 | INFO | MemInfo: { | 20260506 11:03:27.948 | INFO | "gpu" : " 17 GB", | 20260506 11:03:27.948 | INFO | "huge" : "", | 20260506 11:03:27.948 | INFO | "huge_shared" : " 27 GB", | 20260506 11:03:27.949 | INFO | "malloc" : " 14 GB", | 20260506 11:03:27.949 | INFO | "pinned" : " 29 GB" | 20260506 11:03:27.949 | INFO | } | 20260506 11:03:27.949 | INFO | Perf report for the last 5.13931 seconds: | 20260506 11:03:27.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:03:27.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:27.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:27.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:27.949 | INFO | ANALYSIS : 57.4%, 2.8, 85.5ms, 0 | 20260506 11:03:27.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:27.949 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260506 11:03:27.949 | INFO | PRE_HQ : 56.3%, 2.0, 59.7ms, 0 | 20260506 11:03:27.949 | INFO | BAZWRITER : 63.0%, 4.7, 153.6ms, 0 | 20260506 11:03:27.949 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.5ms, 0 | 20260506 11:03:27.949 | INFO | Uploaded 36GiB to GPU in 1.68273s (21.3938GiB/s) | 20260506 11:03:27.949 | INFO | Downloaded 38GiB to GPU in 1.74772s (21.7426GiB/s) | 20260506 11:03:33.088 | INFO | Analyzing chunk frames = [848384,848896) | 20260506 11:03:33.088 | INFO | MemInfo: { | 20260506 11:03:33.088 | INFO | "gpu" : " 17 GB", | 20260506 11:03:33.088 | INFO | "huge" : "", | 20260506 11:03:33.088 | INFO | "huge_shared" : " 27 GB", | 20260506 11:03:33.088 | INFO | "malloc" : " 14 GB", | 20260506 11:03:33.088 | INFO | "pinned" : " 29 GB" | 20260506 11:03:33.088 | INFO | } | 20260506 11:03:33.088 | INFO | Perf report for the last 5.13943 seconds: | 20260506 11:03:33.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:03:33.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:33.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:33.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:33.088 | INFO | ANALYSIS : 57.3%, 3.0, 89.3ms, 0 | 20260506 11:03:33.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:33.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:33.088 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:03:33.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:33.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:33.089 | INFO | Uploaded 36GiB to GPU in 1.60449s (22.437GiB/s) | 20260506 11:03:33.089 | INFO | Downloaded 34GiB to GPU in 1.52152s (22.3461GiB/s) | 20260506 11:03:38.227 | INFO | Analyzing chunk frames = [848896,849408) | 20260506 11:03:38.227 | INFO | MemInfo: { | 20260506 11:03:38.227 | INFO | "gpu" : " 17 GB", | 20260506 11:03:38.227 | INFO | "huge" : "", | 20260506 11:03:38.227 | INFO | "huge_shared" : " 27 GB", | 20260506 11:03:38.227 | INFO | "malloc" : " 14 GB", | 20260506 11:03:38.227 | INFO | "pinned" : " 29 GB" | 20260506 11:03:38.227 | INFO | } | 20260506 11:03:38.227 | INFO | Perf report for the last 5.13933 seconds: | 20260506 11:03:38.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:03:38.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:38.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:38.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:38.227 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260506 11:03:38.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:38.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:38.228 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:03:38.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:38.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:38.228 | INFO | Uploaded 36GiB to GPU in 1.60834s (22.3834GiB/s) | 20260506 11:03:38.228 | INFO | Downloaded 34GiB to GPU in 1.52625s (22.2768GiB/s) | 20260506 11:03:43.366 | INFO | Analyzing chunk frames = [849408,849920) | 20260506 11:03:43.366 | INFO | MemInfo: { | 20260506 11:03:43.366 | INFO | "gpu" : " 17 GB", | 20260506 11:03:43.366 | INFO | "huge" : "", | 20260506 11:03:43.367 | INFO | "huge_shared" : " 27 GB", | 20260506 11:03:43.367 | INFO | "malloc" : " 14 GB", | 20260506 11:03:43.367 | INFO | "pinned" : " 29 GB" | 20260506 11:03:43.367 | INFO | } | 20260506 11:03:43.367 | INFO | Perf report for the last 5.13931 seconds: | 20260506 11:03:43.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:03:43.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:43.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:43.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:43.367 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260506 11:03:43.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:43.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:43.367 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:03:43.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:43.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:43.367 | INFO | Uploaded 36GiB to GPU in 1.60757s (22.394GiB/s) | 20260506 11:03:43.367 | INFO | Downloaded 34GiB to GPU in 1.52499s (22.2952GiB/s) | 20260506 11:03:48.506 | INFO | Analyzing chunk frames = [849920,850432) | 20260506 11:03:48.506 | INFO | MemInfo: { | 20260506 11:03:48.506 | INFO | "gpu" : " 17 GB", | 20260506 11:03:48.506 | INFO | "huge" : "", | 20260506 11:03:48.506 | INFO | "huge_shared" : " 27 GB", | 20260506 11:03:48.506 | INFO | "malloc" : " 14 GB", | 20260506 11:03:48.506 | INFO | "pinned" : " 29 GB" | 20260506 11:03:48.506 | INFO | } | 20260506 11:03:48.506 | INFO | Perf report for the last 5.13935 seconds: | 20260506 11:03:48.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:03:48.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:48.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:48.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:48.506 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260506 11:03:48.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:48.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:48.506 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:03:48.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:48.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:48.506 | INFO | Uploaded 36GiB to GPU in 1.60647s (22.4093GiB/s) | 20260506 11:03:48.506 | INFO | Downloaded 34GiB to GPU in 1.5238s (22.3126GiB/s) | 20260506 11:03:53.645 | INFO | Analyzing chunk frames = [850432,850944) | 20260506 11:03:53.645 | INFO | MemInfo: { | 20260506 11:03:53.645 | INFO | "gpu" : " 17 GB", | 20260506 11:03:53.645 | INFO | "huge" : "", | 20260506 11:03:53.645 | INFO | "huge_shared" : " 27 GB", | 20260506 11:03:53.645 | INFO | "malloc" : " 14 GB", | 20260506 11:03:53.645 | INFO | "pinned" : " 29 GB" | 20260506 11:03:53.645 | INFO | } | 20260506 11:03:53.645 | INFO | Perf report for the last 5.13918 seconds: | 20260506 11:03:53.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:03:53.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:53.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:53.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:53.645 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 11:03:53.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:53.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:53.645 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:03:53.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:53.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:53.646 | INFO | Uploaded 36GiB to GPU in 1.60611s (22.4144GiB/s) | 20260506 11:03:53.646 | INFO | Downloaded 34GiB to GPU in 1.52374s (22.3135GiB/s) | 20260506 11:03:58.785 | INFO | Analyzing chunk frames = [850944,851456) | 20260506 11:03:58.785 | INFO | MemInfo: { | 20260506 11:03:58.785 | INFO | "gpu" : " 17 GB", | 20260506 11:03:58.785 | INFO | "huge" : "", | 20260506 11:03:58.785 | INFO | "huge_shared" : " 27 GB", | 20260506 11:03:58.785 | INFO | "malloc" : " 14 GB", | 20260506 11:03:58.785 | INFO | "pinned" : " 29 GB" | 20260506 11:03:58.785 | INFO | } | 20260506 11:03:58.785 | INFO | Perf report for the last 5.13954 seconds: | 20260506 11:03:58.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:03:58.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:58.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:58.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:58.785 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 11:03:58.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:58.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:58.785 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:03:58.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:58.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:03:58.785 | INFO | Uploaded 36GiB to GPU in 1.60658s (22.4079GiB/s) | 20260506 11:03:58.785 | INFO | Downloaded 34GiB to GPU in 1.52382s (22.3123GiB/s) | 20260506 11:04:03.924 | INFO | Analyzing chunk frames = [851456,851968) | 20260506 11:04:03.924 | INFO | MemInfo: { | 20260506 11:04:03.924 | INFO | "gpu" : " 17 GB", | 20260506 11:04:03.924 | INFO | "huge" : "", | 20260506 11:04:03.924 | INFO | "huge_shared" : " 27 GB", | 20260506 11:04:03.924 | INFO | "malloc" : " 14 GB", | 20260506 11:04:03.924 | INFO | "pinned" : " 29 GB" | 20260506 11:04:03.924 | INFO | } | 20260506 11:04:03.924 | INFO | Perf report for the last 5.13932 seconds: | 20260506 11:04:03.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:04:03.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:03.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:03.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:03.924 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 11:04:03.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:03.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:03.924 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:04:03.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:03.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:03.925 | INFO | Uploaded 36GiB to GPU in 1.6056s (22.4216GiB/s) | 20260506 11:04:03.925 | INFO | Downloaded 34GiB to GPU in 1.52382s (22.3123GiB/s) | 20260506 11:04:03.925 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:04:03.925 | INFO | Total mem usage: 16.5 GiB | 20260506 11:04:03.925 | INFO | All Time High Water Marks sum: 16.8 GiB (Savings: 1.85%) | 20260506 11:04:03.925 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -8.18%) | 20260506 11:04:03.925 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:04:03.925 | INFO | High Water Breakdown: | 20260506 11:04:03.925 | INFO | 60.2% : 9.19 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:04:03.925 | INFO | 39.7% : 6.06 GiB ( 90.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:04:03.925 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:04:03.925 | INFO | ------------------------------------------ | 20260506 11:04:03.925 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:04:03.925 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:04:03.925 | INFO | Total mem usage: 26.5 GiB | 20260506 11:04:03.925 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 11:04:03.925 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:04:03.925 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 11:04:03.925 | INFO | High Water Breakdown: | 20260506 11:04:03.925 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:04:03.925 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:04:03.925 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:04:03.925 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:04:03.925 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:04:03.925 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:04:03.925 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:04:03.925 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:04:03.925 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:04:03.925 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:04:03.925 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:04:03.925 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:04:03.925 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:04:03.925 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:04:03.925 | INFO | ------------------------------------------ | 20260506 11:04:03.925 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:04:03.925 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:04:03.925 | INFO | Total mem usage: 25.6 GiB | 20260506 11:04:03.925 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:04:03.925 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 11:04:03.925 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 11:04:03.925 | INFO | High Water Breakdown: | 20260506 11:04:03.925 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 11:04:03.925 | INFO | ------------------------------------------ | 20260506 11:04:03.925 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:04:03.925 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:04:03.925 | INFO | Total mem usage: 15.7 GiB | 20260506 11:04:03.925 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:04:03.925 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:04:03.925 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:04:03.925 | INFO | High Water Breakdown: | 20260506 11:04:03.925 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:04:03.925 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:04:03.925 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:04:03.925 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:04:03.925 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:04:03.925 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:04:03.925 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:04:03.925 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:04:03.925 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:04:03.925 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:04:03.925 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:04:03.925 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:04:03.925 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:04:03.925 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:04:03.925 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:04:03.925 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:04:03.925 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:04:03.925 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:04:03.925 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:04:03.925 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:04:03.925 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:04:03.925 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:04:03.925 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:04:03.925 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:04:03.925 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:04:03.925 | INFO | ------------------------------------------ | 20260506 11:04:03.925 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:04:03.925 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:04:03.925 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:04:03.925 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:04:03.925 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.27e+01 / 0.00e+00 | 20260506 11:04:03.926 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 5.19e-03 / 0.00e+00 | 20260506 11:04:03.926 | INFO | PerfReport: 10.98%, Baseline -- 3.30e+00 / -nan, 1.74e-02 / 0.00e+00 | 20260506 11:04:03.926 | INFO | PerfReport: 3.07%, Histogram -- 9.21e-01 / -nan, 5.25e-02 / 0.00e+00 | 20260506 11:04:03.926 | INFO | PerfReport: 16.06%, DME -- 3.86e+01 / -nan, 3.08e+00 / 0.00e+00 | 20260506 11:04:03.926 | INFO | PerfReport: 57.26%, FrameLabeling -- 1.72e+01 / -nan, 4.28e-02 / 0.00e+00 | 20260506 11:04:03.926 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.20e+00 / -nan, 1.25e-01 / 0.00e+00 | 20260506 11:04:03.926 | INFO | PerfReport: 3.61%, Metrics -- 1.08e+00 / -nan, 1.69e-01 / 0.00e+00 | 20260506 11:04:03.926 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.17e-01 / -nan, 5.84e-02 / 0.00e+00 | 20260506 11:04:03.926 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:04:03.926 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:04:03.926 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:04:03.926 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 11:04:03.926 | INFO | PerfReport: 18.75%, Upload -- 1.68e+01 / 0.00e+00, 5.00e-01 / 0.00e+00 | 20260506 11:04:03.926 | INFO | PerfReport: 18.01%, Download -- 1.62e+01 / 0.00e+00, 8.51e-01 / 0.00e+00 | 20260506 11:04:03.926 | INFO | PerfReport: 33.46%, Compute -- 3.00e+01 / 0.00e+00, 1.27e+01 / 0.00e+00 | 20260506 11:04:03.926 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 11:04:03.926 | INFO | PerfReport: 0.00%, LockDownload -- 4.93e-04 / 0.00e+00, 1.25e-04 / 0.00e+00 | 20260506 11:04:03.926 | INFO | PerfReport: 29.20%, LockCompute -- 2.62e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 11:04:03.926 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.19e-02 / 0.00e+00 | 20260506 11:04:04.021 | INFO | Starting a round of Baz IO | 20260506 11:04:06.927 | INFO | Populating RTMetrics report | 20260506 11:04:06.927 | INFO | RTMetrics report populated | 20260506 11:04:06.927 | INFO | Opening tmp RTMetrics json file | 20260506 11:04:06.943 | INFO | Serializing RTMetrics to json | 20260506 11:04:06.961 | INFO | Serialization done, swapping tmp file now... | 20260506 11:04:06.970 | INFO | RTMetrics done | 20260506 11:04:07.724 | INFO | Finished a round of Baz IO: Wrote 2.034 GiB in 3.702 seconds | 20260506 11:04:07.724 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.41% / 0.04% / 54.55% | 20260506 11:04:07.724 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.74336% / 7.54992e-05% / 1.98682e-05% | 20260506 11:04:07.724 | INFO | Baz IO overall: | 20260506 11:04:07.724 | INFO | BytesWritten : 400311842317 | 20260506 11:04:07.724 | INFO | NumEvents : 192918339955 | 20260506 11:04:09.063 | INFO | Analyzing chunk frames = [851968,852480) | 20260506 11:04:09.063 | INFO | MemInfo: { | 20260506 11:04:09.063 | INFO | "gpu" : " 17 GB", | 20260506 11:04:09.063 | INFO | "huge" : "", | 20260506 11:04:09.063 | INFO | "huge_shared" : " 27 GB", | 20260506 11:04:09.063 | INFO | "malloc" : " 14 GB", | 20260506 11:04:09.063 | INFO | "pinned" : " 29 GB" | 20260506 11:04:09.063 | INFO | } | 20260506 11:04:09.063 | INFO | Perf report for the last 5.1392 seconds: | 20260506 11:04:09.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:04:09.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:09.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:09.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:09.063 | INFO | ANALYSIS : 57.4%, 2.8, 85.5ms, 0 | 20260506 11:04:09.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:09.064 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260506 11:04:09.064 | INFO | PRE_HQ : 56.3%, 2.0, 59.8ms, 0 | 20260506 11:04:09.064 | INFO | BAZWRITER : 71.6%, 4.8, 175.1ms, 0 | 20260506 11:04:09.064 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 11:04:09.064 | INFO | Uploaded 36GiB to GPU in 1.67893s (21.4422GiB/s) | 20260506 11:04:09.064 | INFO | Downloaded 38GiB to GPU in 1.74472s (21.78GiB/s) | 20260506 11:04:14.203 | INFO | Analyzing chunk frames = [852480,852992) | 20260506 11:04:14.203 | INFO | MemInfo: { | 20260506 11:04:14.203 | INFO | "gpu" : " 17 GB", | 20260506 11:04:14.203 | INFO | "huge" : "", | 20260506 11:04:14.203 | INFO | "huge_shared" : " 27 GB", | 20260506 11:04:14.203 | INFO | "malloc" : " 14 GB", | 20260506 11:04:14.203 | INFO | "pinned" : " 29 GB" | 20260506 11:04:14.203 | INFO | } | 20260506 11:04:14.203 | INFO | Perf report for the last 5.13957 seconds: | 20260506 11:04:14.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:04:14.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:14.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:14.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:14.203 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 11:04:14.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:14.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:14.203 | INFO | PRE_HQ : 29.0%, 1.0, 15.2ms, 0 | 20260506 11:04:14.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:14.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:14.203 | INFO | Uploaded 36GiB to GPU in 1.60368s (22.4483GiB/s) | 20260506 11:04:14.203 | INFO | Downloaded 34GiB to GPU in 1.52088s (22.3555GiB/s) | 20260506 11:04:19.342 | INFO | Analyzing chunk frames = [852992,853504) | 20260506 11:04:19.342 | INFO | MemInfo: { | 20260506 11:04:19.342 | INFO | "gpu" : " 17 GB", | 20260506 11:04:19.342 | INFO | "huge" : "", | 20260506 11:04:19.342 | INFO | "huge_shared" : " 27 GB", | 20260506 11:04:19.342 | INFO | "malloc" : " 14 GB", | 20260506 11:04:19.342 | INFO | "pinned" : " 29 GB" | 20260506 11:04:19.342 | INFO | } | 20260506 11:04:19.342 | INFO | Perf report for the last 5.13899 seconds: | 20260506 11:04:19.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:04:19.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:19.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:19.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:19.342 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260506 11:04:19.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:19.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:19.342 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:04:19.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:19.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:19.342 | INFO | Uploaded 36GiB to GPU in 1.60834s (22.3834GiB/s) | 20260506 11:04:19.342 | INFO | Downloaded 34GiB to GPU in 1.52613s (22.2786GiB/s) | 20260506 11:04:24.481 | INFO | Analyzing chunk frames = [853504,854016) | 20260506 11:04:24.481 | INFO | MemInfo: { | 20260506 11:04:24.481 | INFO | "gpu" : " 17 GB", | 20260506 11:04:24.481 | INFO | "huge" : "", | 20260506 11:04:24.481 | INFO | "huge_shared" : " 27 GB", | 20260506 11:04:24.481 | INFO | "malloc" : " 14 GB", | 20260506 11:04:24.481 | INFO | "pinned" : " 29 GB" | 20260506 11:04:24.481 | INFO | } | 20260506 11:04:24.481 | INFO | Perf report for the last 5.13927 seconds: | 20260506 11:04:24.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:04:24.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:24.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:24.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:24.481 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260506 11:04:24.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:24.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:24.481 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:04:24.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:24.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:24.482 | INFO | Uploaded 36GiB to GPU in 1.6076s (22.3936GiB/s) | 20260506 11:04:24.482 | INFO | Downloaded 34GiB to GPU in 1.52513s (22.2932GiB/s) | 20260506 11:04:29.621 | INFO | Analyzing chunk frames = [854016,854528) | 20260506 11:04:29.621 | INFO | MemInfo: { | 20260506 11:04:29.621 | INFO | "gpu" : " 17 GB", | 20260506 11:04:29.621 | INFO | "huge" : "", | 20260506 11:04:29.621 | INFO | "huge_shared" : " 27 GB", | 20260506 11:04:29.621 | INFO | "malloc" : " 14 GB", | 20260506 11:04:29.621 | INFO | "pinned" : " 29 GB" | 20260506 11:04:29.621 | INFO | } | 20260506 11:04:29.621 | INFO | Perf report for the last 5.13966 seconds: | 20260506 11:04:29.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:04:29.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:29.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:29.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:29.621 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260506 11:04:29.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:29.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:29.621 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:04:29.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:29.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:29.621 | INFO | Uploaded 36GiB to GPU in 1.60559s (22.4217GiB/s) | 20260506 11:04:29.621 | INFO | Downloaded 34GiB to GPU in 1.52286s (22.3264GiB/s) | 20260506 11:04:34.760 | INFO | Analyzing chunk frames = [854528,855040) | 20260506 11:04:34.760 | INFO | MemInfo: { | 20260506 11:04:34.760 | INFO | "gpu" : " 17 GB", | 20260506 11:04:34.760 | INFO | "huge" : "", | 20260506 11:04:34.760 | INFO | "huge_shared" : " 27 GB", | 20260506 11:04:34.760 | INFO | "malloc" : " 14 GB", | 20260506 11:04:34.760 | INFO | "pinned" : " 29 GB" | 20260506 11:04:34.760 | INFO | } | 20260506 11:04:34.760 | INFO | Perf report for the last 5.13908 seconds: | 20260506 11:04:34.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:04:34.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:34.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:34.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:34.760 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 11:04:34.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:34.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:34.760 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:04:34.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:34.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:34.761 | INFO | Uploaded 36GiB to GPU in 1.60636s (22.4109GiB/s) | 20260506 11:04:34.761 | INFO | Downloaded 34GiB to GPU in 1.52323s (22.3209GiB/s) | 20260506 11:04:39.899 | INFO | Analyzing chunk frames = [855040,855552) | 20260506 11:04:39.899 | INFO | MemInfo: { | 20260506 11:04:39.899 | INFO | "gpu" : " 17 GB", | 20260506 11:04:39.899 | INFO | "huge" : "", | 20260506 11:04:39.899 | INFO | "huge_shared" : " 27 GB", | 20260506 11:04:39.899 | INFO | "malloc" : " 14 GB", | 20260506 11:04:39.899 | INFO | "pinned" : " 29 GB" | 20260506 11:04:39.899 | INFO | } | 20260506 11:04:39.899 | INFO | Perf report for the last 5.13943 seconds: | 20260506 11:04:39.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:04:39.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:39.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:39.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:39.900 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 11:04:39.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:39.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:39.900 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:04:39.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:39.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:39.900 | INFO | Uploaded 36GiB to GPU in 1.60625s (22.4124GiB/s) | 20260506 11:04:39.900 | INFO | Downloaded 34GiB to GPU in 1.52374s (22.3135GiB/s) | 20260506 11:04:45.038 | INFO | Analyzing chunk frames = [855552,856064) | 20260506 11:04:45.039 | INFO | MemInfo: { | 20260506 11:04:45.039 | INFO | "gpu" : " 17 GB", | 20260506 11:04:45.039 | INFO | "huge" : "", | 20260506 11:04:45.039 | INFO | "huge_shared" : " 27 GB", | 20260506 11:04:45.039 | INFO | "malloc" : " 14 GB", | 20260506 11:04:45.039 | INFO | "pinned" : " 29 GB" | 20260506 11:04:45.039 | INFO | } | 20260506 11:04:45.039 | INFO | Perf report for the last 5.13926 seconds: | 20260506 11:04:45.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:04:45.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:45.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:45.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:45.039 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 11:04:45.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:45.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:45.039 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 11:04:45.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:45.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:45.039 | INFO | Uploaded 36GiB to GPU in 1.60669s (22.4063GiB/s) | 20260506 11:04:45.039 | INFO | Downloaded 34GiB to GPU in 1.52448s (22.3027GiB/s) | 20260506 11:04:45.136 | INFO | Starting a round of Baz IO | 20260506 11:04:48.043 | INFO | Populating RTMetrics report | 20260506 11:04:48.043 | INFO | RTMetrics report populated | 20260506 11:04:48.043 | INFO | Opening tmp RTMetrics json file | 20260506 11:04:48.094 | INFO | Serializing RTMetrics to json | 20260506 11:04:48.198 | INFO | Serialization done, swapping tmp file now... | 20260506 11:04:48.229 | INFO | RTMetrics done | 20260506 11:04:48.548 | INFO | Finished a round of Baz IO: Wrote 2.036 GiB in 3.411 seconds | 20260506 11:04:48.548 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.37% / 0.04% / 54.59% | 20260506 11:04:48.548 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.72199% / 4.76837e-05% / 3.57628e-05% | 20260506 11:04:48.548 | INFO | Baz IO overall: | 20260506 11:04:48.548 | INFO | BytesWritten : 402498254264 | 20260506 11:04:48.548 | INFO | NumEvents : 193979063860 | 20260506 11:04:50.178 | INFO | Analyzing chunk frames = [856064,856576) | 20260506 11:04:50.178 | INFO | MemInfo: { | 20260506 11:04:50.178 | INFO | "gpu" : " 17 GB", | 20260506 11:04:50.178 | INFO | "huge" : "", | 20260506 11:04:50.178 | INFO | "huge_shared" : " 27 GB", | 20260506 11:04:50.178 | INFO | "malloc" : " 14 GB", | 20260506 11:04:50.178 | INFO | "pinned" : " 29 GB" | 20260506 11:04:50.178 | INFO | } | 20260506 11:04:50.178 | INFO | Perf report for the last 5.13949 seconds: | 20260506 11:04:50.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:04:50.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:50.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:50.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:50.178 | INFO | ANALYSIS : 57.4%, 2.8, 85.3ms, 0 | 20260506 11:04:50.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:50.178 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260506 11:04:50.178 | INFO | PRE_HQ : 56.4%, 2.0, 60.0ms, 0 | 20260506 11:04:50.178 | INFO | BAZWRITER : 66.0%, 4.8, 162.1ms, 0 | 20260506 11:04:50.178 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 11:04:50.179 | INFO | Uploaded 36GiB to GPU in 1.67622s (21.4769GiB/s) | 20260506 11:04:50.179 | INFO | Downloaded 38GiB to GPU in 1.74239s (21.8092GiB/s) | 20260506 11:04:55.317 | INFO | Analyzing chunk frames = [856576,857088) | 20260506 11:04:55.317 | INFO | MemInfo: { | 20260506 11:04:55.317 | INFO | "gpu" : " 17 GB", | 20260506 11:04:55.317 | INFO | "huge" : "", | 20260506 11:04:55.317 | INFO | "huge_shared" : " 27 GB", | 20260506 11:04:55.317 | INFO | "malloc" : " 14 GB", | 20260506 11:04:55.317 | INFO | "pinned" : " 29 GB" | 20260506 11:04:55.317 | INFO | } | 20260506 11:04:55.317 | INFO | Perf report for the last 5.13924 seconds: | 20260506 11:04:55.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:04:55.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:55.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:55.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:55.318 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 11:04:55.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:55.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:55.318 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:04:55.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:55.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:04:55.318 | INFO | Uploaded 36GiB to GPU in 1.60379s (22.4468GiB/s) | 20260506 11:04:55.318 | INFO | Downloaded 34GiB to GPU in 1.52138s (22.3481GiB/s) | 20260506 11:05:00.457 | INFO | Analyzing chunk frames = [857088,857600) | 20260506 11:05:00.457 | INFO | MemInfo: { | 20260506 11:05:00.457 | INFO | "gpu" : " 17 GB", | 20260506 11:05:00.457 | INFO | "huge" : "", | 20260506 11:05:00.457 | INFO | "huge_shared" : " 27 GB", | 20260506 11:05:00.457 | INFO | "malloc" : " 14 GB", | 20260506 11:05:00.457 | INFO | "pinned" : " 29 GB" | 20260506 11:05:00.457 | INFO | } | 20260506 11:05:00.457 | INFO | Perf report for the last 5.13926 seconds: | 20260506 11:05:00.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:05:00.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:00.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:00.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:00.457 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260506 11:05:00.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:00.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:00.457 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:05:00.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:00.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:00.457 | INFO | Uploaded 36GiB to GPU in 1.60796s (22.3887GiB/s) | 20260506 11:05:00.457 | INFO | Downloaded 34GiB to GPU in 1.52604s (22.2799GiB/s) | 20260506 11:05:05.596 | INFO | Analyzing chunk frames = [857600,858112) | 20260506 11:05:05.596 | INFO | MemInfo: { | 20260506 11:05:05.596 | INFO | "gpu" : " 17 GB", | 20260506 11:05:05.596 | INFO | "huge" : "", | 20260506 11:05:05.596 | INFO | "huge_shared" : " 27 GB", | 20260506 11:05:05.596 | INFO | "malloc" : " 14 GB", | 20260506 11:05:05.596 | INFO | "pinned" : " 29 GB" | 20260506 11:05:05.596 | INFO | } | 20260506 11:05:05.596 | INFO | Perf report for the last 5.13953 seconds: | 20260506 11:05:05.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:05:05.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:05.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:05.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:05.596 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 11:05:05.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:05.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:05.596 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:05:05.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:05.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:05.597 | INFO | Uploaded 36GiB to GPU in 1.60806s (22.3873GiB/s) | 20260506 11:05:05.597 | INFO | Downloaded 34GiB to GPU in 1.5253s (22.2907GiB/s) | 20260506 11:05:10.735 | INFO | Analyzing chunk frames = [858112,858624) | 20260506 11:05:10.735 | INFO | MemInfo: { | 20260506 11:05:10.735 | INFO | "gpu" : " 17 GB", | 20260506 11:05:10.735 | INFO | "huge" : "", | 20260506 11:05:10.735 | INFO | "huge_shared" : " 27 GB", | 20260506 11:05:10.735 | INFO | "malloc" : " 14 GB", | 20260506 11:05:10.735 | INFO | "pinned" : " 29 GB" | 20260506 11:05:10.735 | INFO | } | 20260506 11:05:10.735 | INFO | Perf report for the last 5.13911 seconds: | 20260506 11:05:10.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:05:10.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:10.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:10.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:10.735 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260506 11:05:10.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:10.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:10.736 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:05:10.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:10.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:10.736 | INFO | Uploaded 36GiB to GPU in 1.60603s (22.4156GiB/s) | 20260506 11:05:10.736 | INFO | Downloaded 34GiB to GPU in 1.52338s (22.3187GiB/s) | 20260506 11:05:15.875 | INFO | Analyzing chunk frames = [858624,859136) | 20260506 11:05:15.875 | INFO | MemInfo: { | 20260506 11:05:15.875 | INFO | "gpu" : " 17 GB", | 20260506 11:05:15.875 | INFO | "huge" : "", | 20260506 11:05:15.875 | INFO | "huge_shared" : " 27 GB", | 20260506 11:05:15.875 | INFO | "malloc" : " 14 GB", | 20260506 11:05:15.875 | INFO | "pinned" : " 29 GB" | 20260506 11:05:15.875 | INFO | } | 20260506 11:05:15.875 | INFO | Perf report for the last 5.13934 seconds: | 20260506 11:05:15.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:05:15.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:15.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:15.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:15.875 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 11:05:15.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:15.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:15.875 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:05:15.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:15.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:15.875 | INFO | Uploaded 36GiB to GPU in 1.6066s (22.4075GiB/s) | 20260506 11:05:15.875 | INFO | Downloaded 34GiB to GPU in 1.52396s (22.3104GiB/s) | 20260506 11:05:21.014 | INFO | Analyzing chunk frames = [859136,859648) | 20260506 11:05:21.014 | INFO | MemInfo: { | 20260506 11:05:21.014 | INFO | "gpu" : " 17 GB", | 20260506 11:05:21.014 | INFO | "huge" : "", | 20260506 11:05:21.014 | INFO | "huge_shared" : " 27 GB", | 20260506 11:05:21.014 | INFO | "malloc" : " 14 GB", | 20260506 11:05:21.014 | INFO | "pinned" : " 29 GB" | 20260506 11:05:21.014 | INFO | } | 20260506 11:05:21.014 | INFO | Perf report for the last 5.13958 seconds: | 20260506 11:05:21.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:05:21.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:21.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:21.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:21.014 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 11:05:21.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:21.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:21.014 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:05:21.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:21.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:21.015 | INFO | Uploaded 36GiB to GPU in 1.6065s (22.4089GiB/s) | 20260506 11:05:21.015 | INFO | Downloaded 34GiB to GPU in 1.52405s (22.309GiB/s) | 20260506 11:05:26.153 | INFO | Analyzing chunk frames = [859648,860160) | 20260506 11:05:26.153 | INFO | MemInfo: { | 20260506 11:05:26.153 | INFO | "gpu" : " 17 GB", | 20260506 11:05:26.153 | INFO | "huge" : "", | 20260506 11:05:26.154 | INFO | "huge_shared" : " 27 GB", | 20260506 11:05:26.154 | INFO | "malloc" : " 14 GB", | 20260506 11:05:26.154 | INFO | "pinned" : " 29 GB" | 20260506 11:05:26.154 | INFO | } | 20260506 11:05:26.154 | INFO | Perf report for the last 5.13937 seconds: | 20260506 11:05:26.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:05:26.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:26.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:26.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:26.154 | INFO | ANALYSIS : 56.9%, 3.0, 88.7ms, 0 | 20260506 11:05:26.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:26.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:26.154 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 11:05:26.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:26.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:26.154 | INFO | Uploaded 36GiB to GPU in 1.61057s (22.3524GiB/s) | 20260506 11:05:26.154 | INFO | Downloaded 34GiB to GPU in 1.52811s (22.2497GiB/s) | 20260506 11:05:26.154 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:05:26.154 | INFO | Total mem usage: 16.8 GiB | 20260506 11:05:26.154 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:05:26.154 | INFO | Recent High Water Marks Sum: 16.8 GiB (Savings: 0.0457%) | 20260506 11:05:26.154 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:05:26.154 | INFO | High Water Breakdown: | 20260506 11:05:26.154 | INFO | 61.7% : 10.4 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:05:26.154 | INFO | 38.2% : 6.42 GiB ( 95.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:05:26.154 | INFO | 0.118% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:05:26.154 | INFO | ------------------------------------------ | 20260506 11:05:26.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:05:26.154 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:05:26.154 | INFO | Total mem usage: 26.5 GiB | 20260506 11:05:26.154 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 11:05:26.154 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:05:26.154 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 11:05:26.154 | INFO | High Water Breakdown: | 20260506 11:05:26.154 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:05:26.154 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:05:26.154 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:05:26.154 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:05:26.154 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:05:26.154 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:05:26.154 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:05:26.154 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:05:26.154 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:05:26.154 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:05:26.154 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:05:26.154 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:05:26.154 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:05:26.154 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:05:26.154 | INFO | ------------------------------------------ | 20260506 11:05:26.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:05:26.154 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:05:26.154 | INFO | Total mem usage: 25.6 GiB | 20260506 11:05:26.154 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:05:26.154 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 11:05:26.154 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 11:05:26.154 | INFO | High Water Breakdown: | 20260506 11:05:26.154 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 11:05:26.154 | INFO | ------------------------------------------ | 20260506 11:05:26.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:05:26.154 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:05:26.154 | INFO | Total mem usage: 15.7 GiB | 20260506 11:05:26.154 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:05:26.154 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:05:26.154 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:05:26.154 | INFO | High Water Breakdown: | 20260506 11:05:26.154 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:05:26.154 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:05:26.154 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:05:26.154 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:05:26.154 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:05:26.154 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:05:26.154 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:05:26.154 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:05:26.154 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:05:26.154 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:05:26.154 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:05:26.154 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:05:26.154 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:05:26.154 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:05:26.154 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:05:26.154 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:05:26.154 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:05:26.154 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:05:26.154 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:05:26.154 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:05:26.154 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:05:26.154 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:05:26.154 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:05:26.154 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:05:26.154 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:05:26.154 | INFO | ------------------------------------------ | 20260506 11:05:26.154 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:05:26.155 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:05:26.155 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:05:26.155 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:05:26.155 | INFO | PerfReport: Full Execution -- 3.00e+01 / 6.94e+01, 1.26e+01 / 0.00e+00 | 20260506 11:05:26.155 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / 8.60e-02, 4.63e-03 / 0.00e+00 | 20260506 11:05:26.155 | INFO | PerfReport: 10.98%, Baseline -- 3.30e+00 / 3.28e+00, 1.76e-02 / 0.00e+00 | 20260506 11:05:26.155 | INFO | PerfReport: 3.07%, Histogram -- 9.21e-01 / 1.06e+00, 5.21e-02 / 0.00e+00 | 20260506 11:05:26.155 | INFO | PerfReport: 16.08%, DME -- 3.86e+01 / 4.39e+01, 3.06e+00 / 0.00e+00 | 20260506 11:05:26.155 | INFO | PerfReport: 57.26%, FrameLabeling -- 1.72e+01 / 1.72e+01, 4.32e-02 / 0.00e+00 | 20260506 11:05:26.155 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.20e+00 / 2.29e+00, 1.24e-01 / 0.00e+00 | 20260506 11:05:26.155 | INFO | PerfReport: 3.60%, Metrics -- 1.08e+00 / 1.10e+00, 1.69e-01 / 0.00e+00 | 20260506 11:05:26.155 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.16e-01 / 4.67e-01, 5.84e-02 / 0.00e+00 | 20260506 11:05:26.155 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:05:26.155 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:05:26.155 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:05:26.155 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 11:05:26.155 | INFO | PerfReport: 18.76%, Upload -- 1.68e+01 / 0.00e+00, 5.13e-01 / 0.00e+00 | 20260506 11:05:26.155 | INFO | PerfReport: 18.02%, Download -- 1.62e+01 / 0.00e+00, 8.61e-01 / 0.00e+00 | 20260506 11:05:26.155 | INFO | PerfReport: 33.48%, Compute -- 3.00e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 11:05:26.155 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 11:05:26.155 | INFO | PerfReport: 0.00%, LockDownload -- 5.01e-04 / 0.00e+00, 2.83e-04 / 0.00e+00 | 20260506 11:05:26.155 | INFO | PerfReport: 29.15%, LockCompute -- 2.61e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 11:05:26.155 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260506 11:05:26.251 | INFO | Starting a round of Baz IO | 20260506 11:05:29.160 | INFO | Populating RTMetrics report | 20260506 11:05:29.161 | INFO | RTMetrics report populated | 20260506 11:05:29.161 | INFO | Opening tmp RTMetrics json file | 20260506 11:05:29.166 | INFO | Serializing RTMetrics to json | 20260506 11:05:29.193 | INFO | Serialization done, swapping tmp file now... | 20260506 11:05:29.231 | INFO | RTMetrics done | 20260506 11:05:29.688 | INFO | Finished a round of Baz IO: Wrote 2.033 GiB in 3.436 seconds | 20260506 11:05:29.688 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.33% / 0.04% / 54.63% | 20260506 11:05:29.688 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.67679% / 8.34465e-05% / 4.76837e-05% | 20260506 11:05:29.688 | INFO | Baz IO overall: | 20260506 11:05:29.688 | INFO | BytesWritten : 404680881202 | 20260506 11:05:29.688 | INFO | NumEvents : 195037102890 | 20260506 11:05:31.293 | INFO | Analyzing chunk frames = [860160,860672) | 20260506 11:05:31.293 | INFO | MemInfo: { | 20260506 11:05:31.293 | INFO | "gpu" : " 17 GB", | 20260506 11:05:31.293 | INFO | "huge" : "", | 20260506 11:05:31.293 | INFO | "huge_shared" : " 27 GB", | 20260506 11:05:31.293 | INFO | "malloc" : " 14 GB", | 20260506 11:05:31.293 | INFO | "pinned" : " 29 GB" | 20260506 11:05:31.293 | INFO | } | 20260506 11:05:31.293 | INFO | Perf report for the last 5.13913 seconds: | 20260506 11:05:31.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:05:31.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:31.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:31.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:31.293 | INFO | ANALYSIS : 57.5%, 2.8, 85.2ms, 0 | 20260506 11:05:31.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:31.293 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 11:05:31.293 | INFO | PRE_HQ : 56.4%, 2.1, 60.3ms, 0 | 20260506 11:05:31.293 | INFO | BAZWRITER : 66.5%, 4.8, 162.8ms, 0 | 20260506 11:05:31.293 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 11:05:31.293 | INFO | Uploaded 36GiB to GPU in 1.67823s (21.4512GiB/s) | 20260506 11:05:31.293 | INFO | Downloaded 38GiB to GPU in 1.74307s (21.8006GiB/s) | 20260506 11:05:36.432 | INFO | Analyzing chunk frames = [860672,861184) | 20260506 11:05:36.432 | INFO | MemInfo: { | 20260506 11:05:36.432 | INFO | "gpu" : " 17 GB", | 20260506 11:05:36.432 | INFO | "huge" : "", | 20260506 11:05:36.432 | INFO | "huge_shared" : " 27 GB", | 20260506 11:05:36.432 | INFO | "malloc" : " 14 GB", | 20260506 11:05:36.432 | INFO | "pinned" : " 29 GB" | 20260506 11:05:36.432 | INFO | } | 20260506 11:05:36.432 | INFO | Perf report for the last 5.13947 seconds: | 20260506 11:05:36.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:05:36.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:36.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:36.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:36.432 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 11:05:36.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:36.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:36.433 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:05:36.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:36.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:36.433 | INFO | Uploaded 36GiB to GPU in 1.60376s (22.4473GiB/s) | 20260506 11:05:36.433 | INFO | Downloaded 34GiB to GPU in 1.52148s (22.3467GiB/s) | 20260506 11:05:41.571 | INFO | Analyzing chunk frames = [861184,861696) | 20260506 11:05:41.571 | INFO | MemInfo: { | 20260506 11:05:41.571 | INFO | "gpu" : " 17 GB", | 20260506 11:05:41.571 | INFO | "huge" : "", | 20260506 11:05:41.572 | INFO | "huge_shared" : " 27 GB", | 20260506 11:05:41.572 | INFO | "malloc" : " 14 GB", | 20260506 11:05:41.572 | INFO | "pinned" : " 29 GB" | 20260506 11:05:41.572 | INFO | } | 20260506 11:05:41.572 | INFO | Perf report for the last 5.13927 seconds: | 20260506 11:05:41.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:05:41.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:41.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:41.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:41.572 | INFO | ANALYSIS : 56.7%, 3.0, 88.0ms, 0 | 20260506 11:05:41.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:41.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:41.572 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:05:41.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:41.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:41.572 | INFO | Uploaded 36GiB to GPU in 1.60772s (22.3919GiB/s) | 20260506 11:05:41.572 | INFO | Downloaded 34GiB to GPU in 1.52554s (22.2873GiB/s) | 20260506 11:05:46.711 | INFO | Analyzing chunk frames = [861696,862208) | 20260506 11:05:46.711 | INFO | MemInfo: { | 20260506 11:05:46.711 | INFO | "gpu" : " 17 GB", | 20260506 11:05:46.711 | INFO | "huge" : "", | 20260506 11:05:46.711 | INFO | "huge_shared" : " 27 GB", | 20260506 11:05:46.711 | INFO | "malloc" : " 14 GB", | 20260506 11:05:46.711 | INFO | "pinned" : " 29 GB" | 20260506 11:05:46.711 | INFO | } | 20260506 11:05:46.711 | INFO | Perf report for the last 5.13924 seconds: | 20260506 11:05:46.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:05:46.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:46.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:46.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:46.711 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 11:05:46.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:46.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:46.711 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:05:46.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:46.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:46.711 | INFO | Uploaded 36GiB to GPU in 1.60761s (22.3934GiB/s) | 20260506 11:05:46.711 | INFO | Downloaded 34GiB to GPU in 1.52552s (22.2874GiB/s) | 20260506 11:05:51.850 | INFO | Analyzing chunk frames = [862208,862720) | 20260506 11:05:51.850 | INFO | MemInfo: { | 20260506 11:05:51.850 | INFO | "gpu" : " 17 GB", | 20260506 11:05:51.850 | INFO | "huge" : "", | 20260506 11:05:51.850 | INFO | "huge_shared" : " 27 GB", | 20260506 11:05:51.850 | INFO | "malloc" : " 14 GB", | 20260506 11:05:51.850 | INFO | "pinned" : " 29 GB" | 20260506 11:05:51.850 | INFO | } | 20260506 11:05:51.850 | INFO | Perf report for the last 5.13942 seconds: | 20260506 11:05:51.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:05:51.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:51.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:51.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:51.850 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260506 11:05:51.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:51.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:51.850 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:05:51.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:51.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:51.851 | INFO | Uploaded 36GiB to GPU in 1.60497s (22.4303GiB/s) | 20260506 11:05:51.851 | INFO | Downloaded 34GiB to GPU in 1.52285s (22.3265GiB/s) | 20260506 11:05:56.989 | INFO | Analyzing chunk frames = [862720,863232) | 20260506 11:05:56.989 | INFO | MemInfo: { | 20260506 11:05:56.989 | INFO | "gpu" : " 17 GB", | 20260506 11:05:56.990 | INFO | "huge" : "", | 20260506 11:05:56.990 | INFO | "huge_shared" : " 27 GB", | 20260506 11:05:56.990 | INFO | "malloc" : " 14 GB", | 20260506 11:05:56.990 | INFO | "pinned" : " 29 GB" | 20260506 11:05:56.990 | INFO | } | 20260506 11:05:56.990 | INFO | Perf report for the last 5.13934 seconds: | 20260506 11:05:56.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:05:56.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:56.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:56.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:56.990 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 11:05:56.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:56.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:56.990 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:05:56.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:56.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:05:56.990 | INFO | Uploaded 36GiB to GPU in 1.60511s (22.4283GiB/s) | 20260506 11:05:56.990 | INFO | Downloaded 34GiB to GPU in 1.52343s (22.3181GiB/s) | 20260506 11:06:02.129 | INFO | Analyzing chunk frames = [863232,863744) | 20260506 11:06:02.129 | INFO | MemInfo: { | 20260506 11:06:02.129 | INFO | "gpu" : " 17 GB", | 20260506 11:06:02.129 | INFO | "huge" : "", | 20260506 11:06:02.129 | INFO | "huge_shared" : " 27 GB", | 20260506 11:06:02.129 | INFO | "malloc" : " 14 GB", | 20260506 11:06:02.129 | INFO | "pinned" : " 29 GB" | 20260506 11:06:02.129 | INFO | } | 20260506 11:06:02.129 | INFO | Perf report for the last 5.13918 seconds: | 20260506 11:06:02.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:06:02.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:02.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:02.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:02.129 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 11:06:02.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:02.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:02.129 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:06:02.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:02.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:02.129 | INFO | Uploaded 36GiB to GPU in 1.60673s (22.4057GiB/s) | 20260506 11:06:02.129 | INFO | Downloaded 34GiB to GPU in 1.52497s (22.2955GiB/s) | 20260506 11:06:07.268 | INFO | Analyzing chunk frames = [863744,864256) | 20260506 11:06:07.268 | INFO | MemInfo: { | 20260506 11:06:07.268 | INFO | "gpu" : " 17 GB", | 20260506 11:06:07.268 | INFO | "huge" : "", | 20260506 11:06:07.268 | INFO | "huge_shared" : " 27 GB", | 20260506 11:06:07.268 | INFO | "malloc" : " 14 GB", | 20260506 11:06:07.268 | INFO | "pinned" : " 29 GB" | 20260506 11:06:07.268 | INFO | } | 20260506 11:06:07.268 | INFO | Perf report for the last 5.13946 seconds: | 20260506 11:06:07.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:06:07.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:07.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:07.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:07.268 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 11:06:07.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:07.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:07.269 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 11:06:07.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:07.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:07.269 | INFO | Uploaded 36GiB to GPU in 1.60604s (22.4154GiB/s) | 20260506 11:06:07.269 | INFO | Downloaded 34GiB to GPU in 1.52433s (22.3048GiB/s) | 20260506 11:06:07.366 | INFO | Starting a round of Baz IO | 20260506 11:06:10.271 | INFO | Populating RTMetrics report | 20260506 11:06:10.272 | INFO | RTMetrics report populated | 20260506 11:06:10.272 | INFO | Opening tmp RTMetrics json file | 20260506 11:06:10.284 | INFO | Serializing RTMetrics to json | 20260506 11:06:10.306 | INFO | Serialization done, swapping tmp file now... | 20260506 11:06:10.306 | INFO | RTMetrics done | 20260506 11:06:10.748 | INFO | Finished a round of Baz IO: Wrote 2.038 GiB in 3.382 seconds | 20260506 11:06:10.748 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.30% / 0.04% / 54.67% | 20260506 11:06:10.748 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.6901% / 5.96046e-05% / 3.57628e-05% | 20260506 11:06:10.748 | INFO | Baz IO overall: | 20260506 11:06:10.748 | INFO | BytesWritten : 406868711313 | 20260506 11:06:10.748 | INFO | NumEvents : 196099305930 | 20260506 11:06:12.408 | INFO | Analyzing chunk frames = [864256,864768) | 20260506 11:06:12.408 | INFO | MemInfo: { | 20260506 11:06:12.408 | INFO | "gpu" : " 17 GB", | 20260506 11:06:12.408 | INFO | "huge" : "", | 20260506 11:06:12.408 | INFO | "huge_shared" : " 27 GB", | 20260506 11:06:12.408 | INFO | "malloc" : " 14 GB", | 20260506 11:06:12.408 | INFO | "pinned" : " 29 GB" | 20260506 11:06:12.408 | INFO | } | 20260506 11:06:12.408 | INFO | Perf report for the last 5.13923 seconds: | 20260506 11:06:12.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:06:12.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:12.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:12.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:12.408 | INFO | ANALYSIS : 57.4%, 2.8, 85.5ms, 0 | 20260506 11:06:12.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:12.408 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 11:06:12.408 | INFO | PRE_HQ : 56.4%, 2.0, 59.9ms, 0 | 20260506 11:06:12.408 | INFO | BAZWRITER : 65.4%, 4.8, 160.8ms, 0 | 20260506 11:06:12.408 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 11:06:12.408 | INFO | Uploaded 36GiB to GPU in 1.6765s (21.4733GiB/s) | 20260506 11:06:12.408 | INFO | Downloaded 38GiB to GPU in 1.74283s (21.8037GiB/s) | 20260506 11:06:17.547 | INFO | Analyzing chunk frames = [864768,865280) | 20260506 11:06:17.547 | INFO | MemInfo: { | 20260506 11:06:17.547 | INFO | "gpu" : " 17 GB", | 20260506 11:06:17.547 | INFO | "huge" : "", | 20260506 11:06:17.547 | INFO | "huge_shared" : " 27 GB", | 20260506 11:06:17.547 | INFO | "malloc" : " 14 GB", | 20260506 11:06:17.547 | INFO | "pinned" : " 29 GB" | 20260506 11:06:17.547 | INFO | } | 20260506 11:06:17.547 | INFO | Perf report for the last 5.1395 seconds: | 20260506 11:06:17.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:06:17.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:17.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:17.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:17.547 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 11:06:17.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:17.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:17.547 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:06:17.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:17.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:17.547 | INFO | Uploaded 36GiB to GPU in 1.6035s (22.4509GiB/s) | 20260506 11:06:17.548 | INFO | Downloaded 34GiB to GPU in 1.52091s (22.355GiB/s) | 20260506 11:06:22.686 | INFO | Analyzing chunk frames = [865280,865792) | 20260506 11:06:22.686 | INFO | MemInfo: { | 20260506 11:06:22.686 | INFO | "gpu" : " 17 GB", | 20260506 11:06:22.686 | INFO | "huge" : "", | 20260506 11:06:22.686 | INFO | "huge_shared" : " 27 GB", | 20260506 11:06:22.686 | INFO | "malloc" : " 14 GB", | 20260506 11:06:22.686 | INFO | "pinned" : " 29 GB" | 20260506 11:06:22.686 | INFO | } | 20260506 11:06:22.687 | INFO | Perf report for the last 5.13934 seconds: | 20260506 11:06:22.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:06:22.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:22.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:22.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:22.687 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260506 11:06:22.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:22.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:22.687 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:06:22.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:22.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:22.687 | INFO | Uploaded 36GiB to GPU in 1.60808s (22.3869GiB/s) | 20260506 11:06:22.687 | INFO | Downloaded 34GiB to GPU in 1.52686s (22.268GiB/s) | 20260506 11:06:27.826 | INFO | Analyzing chunk frames = [865792,866304) | 20260506 11:06:27.826 | INFO | MemInfo: { | 20260506 11:06:27.826 | INFO | "gpu" : " 17 GB", | 20260506 11:06:27.826 | INFO | "huge" : "", | 20260506 11:06:27.826 | INFO | "huge_shared" : " 27 GB", | 20260506 11:06:27.826 | INFO | "malloc" : " 14 GB", | 20260506 11:06:27.826 | INFO | "pinned" : " 29 GB" | 20260506 11:06:27.826 | INFO | } | 20260506 11:06:27.826 | INFO | Perf report for the last 5.13929 seconds: | 20260506 11:06:27.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:06:27.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:27.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:27.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:27.826 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260506 11:06:27.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:27.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:27.826 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:06:27.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:27.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:27.826 | INFO | Uploaded 36GiB to GPU in 1.60729s (22.398GiB/s) | 20260506 11:06:27.826 | INFO | Downloaded 34GiB to GPU in 1.52518s (22.2924GiB/s) | 20260506 11:06:32.965 | INFO | Analyzing chunk frames = [866304,866816) | 20260506 11:06:32.965 | INFO | MemInfo: { | 20260506 11:06:32.965 | INFO | "gpu" : " 17 GB", | 20260506 11:06:32.965 | INFO | "huge" : "", | 20260506 11:06:32.965 | INFO | "huge_shared" : " 27 GB", | 20260506 11:06:32.965 | INFO | "malloc" : " 14 GB", | 20260506 11:06:32.965 | INFO | "pinned" : " 29 GB" | 20260506 11:06:32.965 | INFO | } | 20260506 11:06:32.965 | INFO | Perf report for the last 5.13926 seconds: | 20260506 11:06:32.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:06:32.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:32.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:32.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:32.965 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260506 11:06:32.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:32.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:32.965 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:06:32.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:32.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:32.965 | INFO | Uploaded 36GiB to GPU in 1.60634s (22.4112GiB/s) | 20260506 11:06:32.965 | INFO | Downloaded 34GiB to GPU in 1.52345s (22.3177GiB/s) | 20260506 11:06:38.104 | INFO | Analyzing chunk frames = [866816,867328) | 20260506 11:06:38.104 | INFO | MemInfo: { | 20260506 11:06:38.104 | INFO | "gpu" : " 17 GB", | 20260506 11:06:38.104 | INFO | "huge" : "", | 20260506 11:06:38.104 | INFO | "huge_shared" : " 27 GB", | 20260506 11:06:38.104 | INFO | "malloc" : " 14 GB", | 20260506 11:06:38.104 | INFO | "pinned" : " 29 GB" | 20260506 11:06:38.104 | INFO | } | 20260506 11:06:38.105 | INFO | Perf report for the last 5.13946 seconds: | 20260506 11:06:38.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:06:38.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:38.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:38.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:38.105 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 11:06:38.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:38.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:38.105 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:06:38.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:38.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:38.105 | INFO | Uploaded 36GiB to GPU in 1.60575s (22.4194GiB/s) | 20260506 11:06:38.105 | INFO | Downloaded 34GiB to GPU in 1.52301s (22.3242GiB/s) | 20260506 11:06:43.244 | INFO | Analyzing chunk frames = [867328,867840) | 20260506 11:06:43.244 | INFO | MemInfo: { | 20260506 11:06:43.244 | INFO | "gpu" : " 17 GB", | 20260506 11:06:43.244 | INFO | "huge" : "", | 20260506 11:06:43.244 | INFO | "huge_shared" : " 27 GB", | 20260506 11:06:43.244 | INFO | "malloc" : " 14 GB", | 20260506 11:06:43.244 | INFO | "pinned" : " 29 GB" | 20260506 11:06:43.244 | INFO | } | 20260506 11:06:43.244 | INFO | Perf report for the last 5.13928 seconds: | 20260506 11:06:43.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:06:43.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:43.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:43.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:43.244 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 11:06:43.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:43.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:43.244 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:06:43.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:43.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:43.244 | INFO | Uploaded 36GiB to GPU in 1.60665s (22.4069GiB/s) | 20260506 11:06:43.244 | INFO | Downloaded 34GiB to GPU in 1.52473s (22.2991GiB/s) | 20260506 11:06:48.383 | INFO | Analyzing chunk frames = [867840,868352) | 20260506 11:06:48.383 | INFO | MemInfo: { | 20260506 11:06:48.383 | INFO | "gpu" : " 17 GB", | 20260506 11:06:48.383 | INFO | "huge" : "", | 20260506 11:06:48.383 | INFO | "huge_shared" : " 27 GB", | 20260506 11:06:48.383 | INFO | "malloc" : " 14 GB", | 20260506 11:06:48.383 | INFO | "pinned" : " 29 GB" | 20260506 11:06:48.383 | INFO | } | 20260506 11:06:48.383 | INFO | Perf report for the last 5.13953 seconds: | 20260506 11:06:48.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:06:48.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:48.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:48.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:48.383 | INFO | ANALYSIS : 56.8%, 3.0, 88.7ms, 0 | 20260506 11:06:48.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:48.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:48.383 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 11:06:48.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:48.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:48.384 | INFO | Uploaded 36GiB to GPU in 1.60637s (22.4108GiB/s) | 20260506 11:06:48.384 | INFO | Downloaded 34GiB to GPU in 1.52397s (22.3102GiB/s) | 20260506 11:06:48.384 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:06:48.384 | INFO | Total mem usage: 16.8 GiB | 20260506 11:06:48.384 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:06:48.384 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -6.97%) | 20260506 11:06:48.384 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:06:48.384 | INFO | High Water Breakdown: | 20260506 11:06:48.384 | INFO | 60.7% : 9.53 GiB ( 91.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:06:48.384 | INFO | 39.2% : 6.16 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:06:48.384 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:06:48.384 | INFO | ------------------------------------------ | 20260506 11:06:48.384 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:06:48.384 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:06:48.384 | INFO | Total mem usage: 26.5 GiB | 20260506 11:06:48.384 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 11:06:48.384 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:06:48.384 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 11:06:48.384 | INFO | High Water Breakdown: | 20260506 11:06:48.384 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:06:48.384 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:06:48.384 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:06:48.384 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:06:48.384 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:06:48.384 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:06:48.384 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:06:48.384 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:06:48.384 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:06:48.384 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:06:48.384 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:06:48.384 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:06:48.384 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:06:48.384 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:06:48.384 | INFO | ------------------------------------------ | 20260506 11:06:48.384 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:06:48.384 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:06:48.384 | INFO | Total mem usage: 25.6 GiB | 20260506 11:06:48.384 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:06:48.384 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 11:06:48.384 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 11:06:48.384 | INFO | High Water Breakdown: | 20260506 11:06:48.384 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 11:06:48.384 | INFO | ------------------------------------------ | 20260506 11:06:48.385 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:06:48.385 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:06:48.385 | INFO | Total mem usage: 15.7 GiB | 20260506 11:06:48.385 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:06:48.385 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:06:48.385 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:06:48.385 | INFO | High Water Breakdown: | 20260506 11:06:48.385 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:06:48.385 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:06:48.385 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:06:48.385 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:06:48.385 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:06:48.385 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:06:48.385 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:06:48.385 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:06:48.385 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:06:48.385 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:06:48.385 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:06:48.385 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:06:48.385 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:06:48.385 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:06:48.385 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:06:48.385 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:06:48.385 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:06:48.385 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:06:48.385 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:06:48.385 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:06:48.385 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:06:48.385 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:06:48.385 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:06:48.385 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:06:48.385 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:06:48.385 | INFO | ------------------------------------------ | 20260506 11:06:48.385 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:06:48.385 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:06:48.385 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:06:48.385 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:06:48.385 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 11:06:48.385 | INFO | PerfReport: 0.29%, AggregateStats -- 8.63e-02 / -nan, 4.74e-03 / 0.00e+00 | 20260506 11:06:48.385 | INFO | PerfReport: 10.98%, Baseline -- 3.30e+00 / -nan, 1.80e-02 / 0.00e+00 | 20260506 11:06:48.385 | INFO | PerfReport: 3.07%, Histogram -- 9.21e-01 / -nan, 5.23e-02 / 0.00e+00 | 20260506 11:06:48.385 | INFO | PerfReport: 16.08%, DME -- 3.86e+01 / -nan, 2.98e+00 / 0.00e+00 | 20260506 11:06:48.385 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.72e+01 / -nan, 4.36e-02 / 0.00e+00 | 20260506 11:06:48.385 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.20e+00 / -nan, 1.24e-01 / 0.00e+00 | 20260506 11:06:48.385 | INFO | PerfReport: 3.60%, Metrics -- 1.08e+00 / -nan, 1.69e-01 / 0.00e+00 | 20260506 11:06:48.385 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.16e-01 / -nan, 5.81e-02 / 0.00e+00 | 20260506 11:06:48.385 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:06:48.385 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:06:48.385 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:06:48.385 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 11:06:48.385 | INFO | PerfReport: 18.76%, Upload -- 1.68e+01 / 0.00e+00, 5.06e-01 / 0.00e+00 | 20260506 11:06:48.385 | INFO | PerfReport: 18.02%, Download -- 1.62e+01 / 0.00e+00, 8.59e-01 / 0.00e+00 | 20260506 11:06:48.385 | INFO | PerfReport: 33.49%, Compute -- 3.00e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 11:06:48.385 | INFO | PerfReport: 0.56%, LockUpload -- 5.04e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260506 11:06:48.385 | INFO | PerfReport: 0.00%, LockDownload -- 4.93e-04 / 0.00e+00, 1.58e-04 / 0.00e+00 | 20260506 11:06:48.385 | INFO | PerfReport: 29.15%, LockCompute -- 2.61e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 11:06:48.385 | INFO | PerfReport: 0.02%, Misc -- 1.35e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260506 11:06:48.481 | INFO | Starting a round of Baz IO | 20260506 11:06:51.384 | INFO | Populating RTMetrics report | 20260506 11:06:51.384 | INFO | RTMetrics report populated | 20260506 11:06:51.384 | INFO | Opening tmp RTMetrics json file | 20260506 11:06:51.401 | INFO | Serializing RTMetrics to json | 20260506 11:06:51.418 | INFO | Serialization done, swapping tmp file now... | 20260506 11:06:51.431 | INFO | RTMetrics done | 20260506 11:06:52.014 | INFO | Finished a round of Baz IO: Wrote 2.038 GiB in 3.532 seconds | 20260506 11:06:52.014 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.26% / 0.04% / 54.70% | 20260506 11:06:52.014 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.64882% / 0.000103315% / 3.97364e-05% | 20260506 11:06:52.014 | INFO | Baz IO overall: | 20260506 11:06:52.014 | INFO | BytesWritten : 409056538977 | 20260506 11:06:52.014 | INFO | NumEvents : 197160033023 | 20260506 11:06:53.522 | INFO | Analyzing chunk frames = [868352,868864) | 20260506 11:06:53.522 | INFO | MemInfo: { | 20260506 11:06:53.522 | INFO | "gpu" : " 17 GB", | 20260506 11:06:53.522 | INFO | "huge" : "", | 20260506 11:06:53.522 | INFO | "huge_shared" : " 27 GB", | 20260506 11:06:53.522 | INFO | "malloc" : " 14 GB", | 20260506 11:06:53.522 | INFO | "pinned" : " 29 GB" | 20260506 11:06:53.522 | INFO | } | 20260506 11:06:53.522 | INFO | Perf report for the last 5.13909 seconds: | 20260506 11:06:53.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:06:53.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:53.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:53.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:53.523 | INFO | ANALYSIS : 57.4%, 2.8, 85.4ms, 0 | 20260506 11:06:53.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:53.523 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 11:06:53.523 | INFO | PRE_HQ : 56.3%, 2.0, 59.9ms, 0 | 20260506 11:06:53.523 | INFO | BAZWRITER : 68.3%, 4.8, 167.7ms, 0 | 20260506 11:06:53.523 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 11:06:53.523 | INFO | Uploaded 36GiB to GPU in 1.67206s (21.5304GiB/s) | 20260506 11:06:53.523 | INFO | Downloaded 38GiB to GPU in 1.73779s (21.8669GiB/s) | 20260506 11:06:58.662 | INFO | Analyzing chunk frames = [868864,869376) | 20260506 11:06:58.662 | INFO | MemInfo: { | 20260506 11:06:58.662 | INFO | "gpu" : " 17 GB", | 20260506 11:06:58.662 | INFO | "huge" : "", | 20260506 11:06:58.662 | INFO | "huge_shared" : " 27 GB", | 20260506 11:06:58.662 | INFO | "malloc" : " 14 GB", | 20260506 11:06:58.662 | INFO | "pinned" : " 29 GB" | 20260506 11:06:58.662 | INFO | } | 20260506 11:06:58.662 | INFO | Perf report for the last 5.13959 seconds: | 20260506 11:06:58.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:06:58.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:58.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:58.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:58.662 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 11:06:58.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:58.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:58.662 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:06:58.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:58.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:06:58.662 | INFO | Uploaded 36GiB to GPU in 1.60358s (22.4498GiB/s) | 20260506 11:06:58.662 | INFO | Downloaded 34GiB to GPU in 1.5208s (22.3567GiB/s) | 20260506 11:07:03.801 | INFO | Analyzing chunk frames = [869376,869888) | 20260506 11:07:03.801 | INFO | MemInfo: { | 20260506 11:07:03.801 | INFO | "gpu" : " 17 GB", | 20260506 11:07:03.801 | INFO | "huge" : "", | 20260506 11:07:03.801 | INFO | "huge_shared" : " 27 GB", | 20260506 11:07:03.801 | INFO | "malloc" : " 14 GB", | 20260506 11:07:03.801 | INFO | "pinned" : " 29 GB" | 20260506 11:07:03.801 | INFO | } | 20260506 11:07:03.801 | INFO | Perf report for the last 5.13925 seconds: | 20260506 11:07:03.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:07:03.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:03.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:03.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:03.801 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260506 11:07:03.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:03.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:03.801 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:07:03.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:03.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:03.802 | INFO | Uploaded 36GiB to GPU in 1.60846s (22.3817GiB/s) | 20260506 11:07:03.802 | INFO | Downloaded 34GiB to GPU in 1.5261s (22.2791GiB/s) | 20260506 11:07:08.940 | INFO | Analyzing chunk frames = [869888,870400) | 20260506 11:07:08.940 | INFO | MemInfo: { | 20260506 11:07:08.940 | INFO | "gpu" : " 17 GB", | 20260506 11:07:08.940 | INFO | "huge" : "", | 20260506 11:07:08.940 | INFO | "huge_shared" : " 27 GB", | 20260506 11:07:08.940 | INFO | "malloc" : " 14 GB", | 20260506 11:07:08.941 | INFO | "pinned" : " 29 GB" | 20260506 11:07:08.941 | INFO | } | 20260506 11:07:08.941 | INFO | Perf report for the last 5.13922 seconds: | 20260506 11:07:08.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:07:08.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:08.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:08.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:08.941 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 11:07:08.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:08.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:08.941 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:07:08.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:08.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:08.941 | INFO | Uploaded 36GiB to GPU in 1.6074s (22.3964GiB/s) | 20260506 11:07:08.941 | INFO | Downloaded 34GiB to GPU in 1.52498s (22.2953GiB/s) | 20260506 11:07:14.080 | INFO | Analyzing chunk frames = [870400,870912) | 20260506 11:07:14.080 | INFO | MemInfo: { | 20260506 11:07:14.080 | INFO | "gpu" : " 17 GB", | 20260506 11:07:14.080 | INFO | "huge" : "", | 20260506 11:07:14.080 | INFO | "huge_shared" : " 27 GB", | 20260506 11:07:14.080 | INFO | "malloc" : " 14 GB", | 20260506 11:07:14.080 | INFO | "pinned" : " 29 GB" | 20260506 11:07:14.080 | INFO | } | 20260506 11:07:14.080 | INFO | Perf report for the last 5.13979 seconds: | 20260506 11:07:14.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:07:14.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:14.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:14.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:14.080 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260506 11:07:14.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:14.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:14.080 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:07:14.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:14.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:14.081 | INFO | Uploaded 36GiB to GPU in 1.60597s (22.4164GiB/s) | 20260506 11:07:14.081 | INFO | Downloaded 34GiB to GPU in 1.52289s (22.326GiB/s) | 20260506 11:07:19.219 | INFO | Analyzing chunk frames = [870912,871424) | 20260506 11:07:19.219 | INFO | MemInfo: { | 20260506 11:07:19.219 | INFO | "gpu" : " 17 GB", | 20260506 11:07:19.219 | INFO | "huge" : "", | 20260506 11:07:19.219 | INFO | "huge_shared" : " 27 GB", | 20260506 11:07:19.219 | INFO | "malloc" : " 14 GB", | 20260506 11:07:19.219 | INFO | "pinned" : " 29 GB" | 20260506 11:07:19.219 | INFO | } | 20260506 11:07:19.219 | INFO | Perf report for the last 5.13905 seconds: | 20260506 11:07:19.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:07:19.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:19.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:19.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:19.219 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 11:07:19.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:19.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:19.220 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 11:07:19.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:19.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:19.220 | INFO | Uploaded 36GiB to GPU in 1.60645s (22.4096GiB/s) | 20260506 11:07:19.220 | INFO | Downloaded 34GiB to GPU in 1.52329s (22.3201GiB/s) | 20260506 11:07:24.358 | INFO | Analyzing chunk frames = [871424,871936) | 20260506 11:07:24.358 | INFO | MemInfo: { | 20260506 11:07:24.358 | INFO | "gpu" : " 17 GB", | 20260506 11:07:24.358 | INFO | "huge" : "", | 20260506 11:07:24.358 | INFO | "huge_shared" : " 27 GB", | 20260506 11:07:24.359 | INFO | "malloc" : " 14 GB", | 20260506 11:07:24.359 | INFO | "pinned" : " 29 GB" | 20260506 11:07:24.359 | INFO | } | 20260506 11:07:24.359 | INFO | Perf report for the last 5.13913 seconds: | 20260506 11:07:24.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:07:24.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:24.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:24.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:24.359 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 11:07:24.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:24.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:24.359 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:07:24.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:24.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:24.359 | INFO | Uploaded 36GiB to GPU in 1.60684s (22.4042GiB/s) | 20260506 11:07:24.359 | INFO | Downloaded 34GiB to GPU in 1.52471s (22.2993GiB/s) | 20260506 11:07:29.498 | INFO | Analyzing chunk frames = [871936,872448) | 20260506 11:07:29.498 | INFO | MemInfo: { | 20260506 11:07:29.498 | INFO | "gpu" : " 17 GB", | 20260506 11:07:29.498 | INFO | "huge" : "", | 20260506 11:07:29.498 | INFO | "huge_shared" : " 27 GB", | 20260506 11:07:29.498 | INFO | "malloc" : " 14 GB", | 20260506 11:07:29.498 | INFO | "pinned" : " 29 GB" | 20260506 11:07:29.498 | INFO | } | 20260506 11:07:29.498 | INFO | Perf report for the last 5.13958 seconds: | 20260506 11:07:29.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:07:29.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:29.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:29.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:29.498 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 11:07:29.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:29.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:29.498 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:07:29.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:29.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:29.498 | INFO | Uploaded 36GiB to GPU in 1.60654s (22.4084GiB/s) | 20260506 11:07:29.498 | INFO | Downloaded 34GiB to GPU in 1.52442s (22.3036GiB/s) | 20260506 11:07:29.595 | INFO | Starting a round of Baz IO | 20260506 11:07:32.500 | INFO | Populating RTMetrics report | 20260506 11:07:32.501 | INFO | RTMetrics report populated | 20260506 11:07:32.501 | INFO | Opening tmp RTMetrics json file | 20260506 11:07:32.529 | INFO | Serializing RTMetrics to json | 20260506 11:07:32.588 | INFO | Serialization done, swapping tmp file now... | 20260506 11:07:32.590 | INFO | RTMetrics done | 20260506 11:07:33.083 | INFO | Finished a round of Baz IO: Wrote 2.039 GiB in 3.488 seconds | 20260506 11:07:33.083 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.22% / 0.04% / 54.74% | 20260506 11:07:33.083 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.62615% / 5.5631e-05% / 3.17891e-05% | 20260506 11:07:33.084 | INFO | Baz IO overall: | 20260506 11:07:33.084 | INFO | BytesWritten : 411245880303 | 20260506 11:07:33.084 | INFO | NumEvents : 198221996616 | 20260506 11:07:34.637 | INFO | Analyzing chunk frames = [872448,872960) | 20260506 11:07:34.637 | INFO | MemInfo: { | 20260506 11:07:34.637 | INFO | "gpu" : " 17 GB", | 20260506 11:07:34.637 | INFO | "huge" : "", | 20260506 11:07:34.637 | INFO | "huge_shared" : " 27 GB", | 20260506 11:07:34.637 | INFO | "malloc" : " 14 GB", | 20260506 11:07:34.637 | INFO | "pinned" : " 29 GB" | 20260506 11:07:34.637 | INFO | } | 20260506 11:07:34.637 | INFO | Perf report for the last 5.13906 seconds: | 20260506 11:07:34.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:07:34.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:34.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:34.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:34.637 | INFO | ANALYSIS : 57.4%, 2.8, 85.3ms, 0 | 20260506 11:07:34.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:34.637 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260506 11:07:34.637 | INFO | PRE_HQ : 56.4%, 2.1, 60.4ms, 0 | 20260506 11:07:34.637 | INFO | BAZWRITER : 67.4%, 4.8, 164.8ms, 0 | 20260506 11:07:34.637 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 11:07:34.638 | INFO | Uploaded 36GiB to GPU in 1.67109s (21.5429GiB/s) | 20260506 11:07:34.638 | INFO | Downloaded 38GiB to GPU in 1.73587s (21.891GiB/s) | 20260506 11:07:39.777 | INFO | Analyzing chunk frames = [872960,873472) | 20260506 11:07:39.777 | INFO | MemInfo: { | 20260506 11:07:39.777 | INFO | "gpu" : " 17 GB", | 20260506 11:07:39.777 | INFO | "huge" : "", | 20260506 11:07:39.777 | INFO | "huge_shared" : " 27 GB", | 20260506 11:07:39.777 | INFO | "malloc" : " 14 GB", | 20260506 11:07:39.777 | INFO | "pinned" : " 29 GB" | 20260506 11:07:39.777 | INFO | } | 20260506 11:07:39.777 | INFO | Perf report for the last 5.13956 seconds: | 20260506 11:07:39.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:07:39.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:39.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:39.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:39.777 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 11:07:39.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:39.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:39.777 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:07:39.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:39.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:39.777 | INFO | Uploaded 36GiB to GPU in 1.60373s (22.4477GiB/s) | 20260506 11:07:39.777 | INFO | Downloaded 34GiB to GPU in 1.52127s (22.3497GiB/s) | 20260506 11:07:44.916 | INFO | Analyzing chunk frames = [873472,873984) | 20260506 11:07:44.916 | INFO | MemInfo: { | 20260506 11:07:44.916 | INFO | "gpu" : " 17 GB", | 20260506 11:07:44.916 | INFO | "huge" : "", | 20260506 11:07:44.916 | INFO | "huge_shared" : " 27 GB", | 20260506 11:07:44.916 | INFO | "malloc" : " 14 GB", | 20260506 11:07:44.916 | INFO | "pinned" : " 29 GB" | 20260506 11:07:44.916 | INFO | } | 20260506 11:07:44.916 | INFO | Perf report for the last 5.139 seconds: | 20260506 11:07:44.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:07:44.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:44.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:44.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:44.916 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260506 11:07:44.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:44.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:44.916 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:07:44.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:44.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:44.916 | INFO | Uploaded 36GiB to GPU in 1.60848s (22.3814GiB/s) | 20260506 11:07:44.916 | INFO | Downloaded 34GiB to GPU in 1.52712s (22.2641GiB/s) | 20260506 11:07:50.055 | INFO | Analyzing chunk frames = [873984,874496) | 20260506 11:07:50.055 | INFO | MemInfo: { | 20260506 11:07:50.055 | INFO | "gpu" : " 17 GB", | 20260506 11:07:50.055 | INFO | "huge" : "", | 20260506 11:07:50.055 | INFO | "huge_shared" : " 27 GB", | 20260506 11:07:50.055 | INFO | "malloc" : " 14 GB", | 20260506 11:07:50.055 | INFO | "pinned" : " 29 GB" | 20260506 11:07:50.055 | INFO | } | 20260506 11:07:50.055 | INFO | Perf report for the last 5.13947 seconds: | 20260506 11:07:50.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:07:50.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:50.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:50.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:50.055 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 11:07:50.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:50.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:50.055 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:07:50.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:50.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:50.056 | INFO | Uploaded 36GiB to GPU in 1.60756s (22.3942GiB/s) | 20260506 11:07:50.056 | INFO | Downloaded 34GiB to GPU in 1.52473s (22.2991GiB/s) | 20260506 11:07:55.195 | INFO | Analyzing chunk frames = [874496,875008) | 20260506 11:07:55.195 | INFO | MemInfo: { | 20260506 11:07:55.195 | INFO | "gpu" : " 17 GB", | 20260506 11:07:55.195 | INFO | "huge" : "", | 20260506 11:07:55.195 | INFO | "huge_shared" : " 27 GB", | 20260506 11:07:55.195 | INFO | "malloc" : " 14 GB", | 20260506 11:07:55.195 | INFO | "pinned" : " 29 GB" | 20260506 11:07:55.195 | INFO | } | 20260506 11:07:55.195 | INFO | Perf report for the last 5.13957 seconds: | 20260506 11:07:55.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:07:55.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:55.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:55.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:55.195 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260506 11:07:55.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:55.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:55.195 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 11:07:55.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:55.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:07:55.196 | INFO | Uploaded 36GiB to GPU in 1.60626s (22.4123GiB/s) | 20260506 11:07:55.196 | INFO | Downloaded 34GiB to GPU in 1.52381s (22.3124GiB/s) | 20260506 11:08:00.334 | INFO | Analyzing chunk frames = [875008,875520) | 20260506 11:08:00.334 | INFO | MemInfo: { | 20260506 11:08:00.334 | INFO | "gpu" : " 17 GB", | 20260506 11:08:00.334 | INFO | "huge" : "", | 20260506 11:08:00.334 | INFO | "huge_shared" : " 27 GB", | 20260506 11:08:00.334 | INFO | "malloc" : " 14 GB", | 20260506 11:08:00.334 | INFO | "pinned" : " 29 GB" | 20260506 11:08:00.334 | INFO | } | 20260506 11:08:00.334 | INFO | Perf report for the last 5.13925 seconds: | 20260506 11:08:00.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:08:00.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:00.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:00.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:00.334 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 11:08:00.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:00.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:00.334 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:08:00.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:00.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:00.335 | INFO | Uploaded 36GiB to GPU in 1.60629s (22.4119GiB/s) | 20260506 11:08:00.335 | INFO | Downloaded 34GiB to GPU in 1.52315s (22.3222GiB/s) | 20260506 11:08:05.473 | INFO | Analyzing chunk frames = [875520,876032) | 20260506 11:08:05.474 | INFO | MemInfo: { | 20260506 11:08:05.474 | INFO | "gpu" : " 17 GB", | 20260506 11:08:05.474 | INFO | "huge" : "", | 20260506 11:08:05.474 | INFO | "huge_shared" : " 27 GB", | 20260506 11:08:05.474 | INFO | "malloc" : " 14 GB", | 20260506 11:08:05.474 | INFO | "pinned" : " 29 GB" | 20260506 11:08:05.474 | INFO | } | 20260506 11:08:05.474 | INFO | Perf report for the last 5.13957 seconds: | 20260506 11:08:05.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:08:05.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:05.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:05.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:05.474 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 11:08:05.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:05.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:05.474 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 11:08:05.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:05.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:05.474 | INFO | Uploaded 36GiB to GPU in 1.60697s (22.4024GiB/s) | 20260506 11:08:05.474 | INFO | Downloaded 34GiB to GPU in 1.52462s (22.3006GiB/s) | 20260506 11:08:10.613 | INFO | Analyzing chunk frames = [876032,876544) | 20260506 11:08:10.613 | INFO | MemInfo: { | 20260506 11:08:10.613 | INFO | "gpu" : " 17 GB", | 20260506 11:08:10.613 | INFO | "huge" : "", | 20260506 11:08:10.613 | INFO | "huge_shared" : " 27 GB", | 20260506 11:08:10.613 | INFO | "malloc" : " 14 GB", | 20260506 11:08:10.613 | INFO | "pinned" : " 29 GB" | 20260506 11:08:10.613 | INFO | } | 20260506 11:08:10.613 | INFO | Perf report for the last 5.1389 seconds: | 20260506 11:08:10.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:08:10.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:10.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:10.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:10.613 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 11:08:10.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:10.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:10.613 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 11:08:10.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:10.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:10.613 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:08:10.613 | INFO | Total mem usage: 16.8 GiB | 20260506 11:08:10.613 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:08:10.613 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -4.13%) | 20260506 11:08:10.613 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:08:10.613 | INFO | High Water Breakdown: | 20260506 11:08:10.613 | INFO | 61.1% : 9.86 GiB ( 95% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:08:10.613 | INFO | 38.8% : 6.26 GiB ( 93.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:08:10.613 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:08:10.613 | INFO | ------------------------------------------ | 20260506 11:08:10.613 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:08:10.613 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:08:10.613 | INFO | Total mem usage: 26.5 GiB | 20260506 11:08:10.613 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 11:08:10.613 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:08:10.613 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 11:08:10.613 | INFO | High Water Breakdown: | 20260506 11:08:10.613 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:08:10.613 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:08:10.613 | INFO | Uploaded 36GiB to GPU in 1.60694s (22.4029GiB/s) | 20260506 11:08:10.613 | INFO | Downloaded 34GiB to GPU in 1.52539s (22.2894GiB/s) | 20260506 11:08:10.613 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:08:10.614 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:08:10.614 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:08:10.614 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:08:10.614 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:08:10.614 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:08:10.614 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:08:10.614 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:08:10.614 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:08:10.614 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:08:10.614 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:08:10.614 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:08:10.614 | INFO | ------------------------------------------ | 20260506 11:08:10.614 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:08:10.614 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:08:10.614 | INFO | Total mem usage: 25.6 GiB | 20260506 11:08:10.614 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:08:10.614 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 11:08:10.614 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 11:08:10.614 | INFO | High Water Breakdown: | 20260506 11:08:10.614 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 11:08:10.614 | INFO | ------------------------------------------ | 20260506 11:08:10.614 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:08:10.614 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:08:10.614 | INFO | Total mem usage: 15.7 GiB | 20260506 11:08:10.614 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:08:10.614 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:08:10.614 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:08:10.614 | INFO | High Water Breakdown: | 20260506 11:08:10.614 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:08:10.614 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:08:10.614 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:08:10.614 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:08:10.614 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:08:10.614 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:08:10.614 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:08:10.614 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:08:10.614 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:08:10.614 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:08:10.614 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:08:10.614 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:08:10.614 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:08:10.614 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:08:10.614 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:08:10.614 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:08:10.614 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:08:10.614 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:08:10.614 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:08:10.614 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:08:10.614 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:08:10.614 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:08:10.614 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:08:10.614 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:08:10.614 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:08:10.614 | INFO | ------------------------------------------ | 20260506 11:08:10.614 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:08:10.614 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:08:10.614 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:08:10.614 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:08:10.614 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260506 11:08:10.614 | INFO | PerfReport: 0.29%, AggregateStats -- 8.67e-02 / -nan, 5.85e-03 / 0.00e+00 | 20260506 11:08:10.614 | INFO | PerfReport: 10.98%, Baseline -- 3.30e+00 / -nan, 1.80e-02 / 0.00e+00 | 20260506 11:08:10.614 | INFO | PerfReport: 3.07%, Histogram -- 9.21e-01 / -nan, 5.15e-02 / 0.00e+00 | 20260506 11:08:10.614 | INFO | PerfReport: 16.06%, DME -- 3.86e+01 / -nan, 3.06e+00 / 0.00e+00 | 20260506 11:08:10.614 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.72e+01 / -nan, 4.33e-02 / 0.00e+00 | 20260506 11:08:10.614 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.20e+00 / -nan, 1.24e-01 / 0.00e+00 | 20260506 11:08:10.614 | INFO | PerfReport: 3.60%, Metrics -- 1.08e+00 / -nan, 1.68e-01 / 0.00e+00 | 20260506 11:08:10.614 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.15e-01 / -nan, 5.74e-02 / 0.00e+00 | 20260506 11:08:10.614 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:08:10.614 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:08:10.614 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:08:10.614 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 11:08:10.614 | INFO | PerfReport: 18.76%, Upload -- 1.68e+01 / 0.00e+00, 4.88e-01 / 0.00e+00 | 20260506 11:08:10.614 | INFO | PerfReport: 18.02%, Download -- 1.62e+01 / 0.00e+00, 8.36e-01 / 0.00e+00 | 20260506 11:08:10.614 | INFO | PerfReport: 33.48%, Compute -- 3.00e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260506 11:08:10.614 | INFO | PerfReport: 0.57%, LockUpload -- 5.07e-01 / 0.00e+00, 3.62e+00 / 0.00e+00 | 20260506 11:08:10.614 | INFO | PerfReport: 0.00%, LockDownload -- 4.95e-04 / 0.00e+00, 1.28e-04 / 0.00e+00 | 20260506 11:08:10.614 | INFO | PerfReport: 29.16%, LockCompute -- 2.61e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260506 11:08:10.614 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260506 11:08:10.710 | INFO | Starting a round of Baz IO | 20260506 11:08:13.619 | INFO | Populating RTMetrics report | 20260506 11:08:13.619 | INFO | RTMetrics report populated | 20260506 11:08:13.619 | INFO | Opening tmp RTMetrics json file | 20260506 11:08:13.656 | INFO | Serializing RTMetrics to json | 20260506 11:08:13.658 | INFO | Serialization done, swapping tmp file now... | 20260506 11:08:13.659 | INFO | RTMetrics done | 20260506 11:08:14.091 | INFO | Finished a round of Baz IO: Wrote 2.038 GiB in 3.38 seconds | 20260506 11:08:14.091 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.18% / 0.04% / 54.78% | 20260506 11:08:14.091 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.60909% / 8.34465e-05% / 6.75519e-05% | 20260506 11:08:14.091 | INFO | Baz IO overall: | 20260506 11:08:14.091 | INFO | BytesWritten : 413433805085 | 20260506 11:08:14.091 | INFO | NumEvents : 199283546741 | 20260506 11:08:15.752 | INFO | Analyzing chunk frames = [876544,877056) | 20260506 11:08:15.752 | INFO | MemInfo: { | 20260506 11:08:15.752 | INFO | "gpu" : " 17 GB", | 20260506 11:08:15.752 | INFO | "huge" : "", | 20260506 11:08:15.752 | INFO | "huge_shared" : " 27 GB", | 20260506 11:08:15.752 | INFO | "malloc" : " 14 GB", | 20260506 11:08:15.752 | INFO | "pinned" : " 29 GB" | 20260506 11:08:15.752 | INFO | } | 20260506 11:08:15.752 | INFO | Perf report for the last 5.13922 seconds: | 20260506 11:08:15.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:08:15.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:15.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:15.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:15.752 | INFO | ANALYSIS : 57.5%, 2.8, 85.4ms, 0 | 20260506 11:08:15.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:15.752 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 11:08:15.752 | INFO | PRE_HQ : 56.5%, 2.1, 60.6ms, 0 | 20260506 11:08:15.752 | INFO | BAZWRITER : 65.3%, 4.8, 160.5ms, 0 | 20260506 11:08:15.752 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 11:08:15.752 | INFO | Uploaded 36GiB to GPU in 1.67905s (21.4407GiB/s) | 20260506 11:08:15.752 | INFO | Downloaded 38GiB to GPU in 1.74447s (21.7831GiB/s) | 20260506 11:08:20.891 | INFO | Analyzing chunk frames = [877056,877568) | 20260506 11:08:20.891 | INFO | MemInfo: { | 20260506 11:08:20.891 | INFO | "gpu" : " 17 GB", | 20260506 11:08:20.891 | INFO | "huge" : "", | 20260506 11:08:20.891 | INFO | "huge_shared" : " 27 GB", | 20260506 11:08:20.891 | INFO | "malloc" : " 14 GB", | 20260506 11:08:20.891 | INFO | "pinned" : " 29 GB" | 20260506 11:08:20.891 | INFO | } | 20260506 11:08:20.891 | INFO | Perf report for the last 5.13951 seconds: | 20260506 11:08:20.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:08:20.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:20.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:20.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:20.892 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 11:08:20.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:20.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:20.892 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:08:20.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:20.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:20.892 | INFO | Uploaded 36GiB to GPU in 1.6044s (22.4382GiB/s) | 20260506 11:08:20.892 | INFO | Downloaded 34GiB to GPU in 1.52166s (22.344GiB/s) | 20260506 11:08:26.031 | INFO | Analyzing chunk frames = [877568,878080) | 20260506 11:08:26.031 | INFO | MemInfo: { | 20260506 11:08:26.031 | INFO | "gpu" : " 17 GB", | 20260506 11:08:26.031 | INFO | "huge" : "", | 20260506 11:08:26.031 | INFO | "huge_shared" : " 27 GB", | 20260506 11:08:26.031 | INFO | "malloc" : " 14 GB", | 20260506 11:08:26.031 | INFO | "pinned" : " 29 GB" | 20260506 11:08:26.031 | INFO | } | 20260506 11:08:26.031 | INFO | Perf report for the last 5.13928 seconds: | 20260506 11:08:26.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:08:26.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:26.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:26.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:26.031 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260506 11:08:26.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:26.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:26.031 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:08:26.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:26.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:26.031 | INFO | Uploaded 36GiB to GPU in 1.60758s (22.3939GiB/s) | 20260506 11:08:26.031 | INFO | Downloaded 34GiB to GPU in 1.52559s (22.2865GiB/s) | 20260506 11:08:31.170 | INFO | Analyzing chunk frames = [878080,878592) | 20260506 11:08:31.170 | INFO | MemInfo: { | 20260506 11:08:31.170 | INFO | "gpu" : " 17 GB", | 20260506 11:08:31.170 | INFO | "huge" : "", | 20260506 11:08:31.170 | INFO | "huge_shared" : " 27 GB", | 20260506 11:08:31.170 | INFO | "malloc" : " 14 GB", | 20260506 11:08:31.170 | INFO | "pinned" : " 29 GB" | 20260506 11:08:31.170 | INFO | } | 20260506 11:08:31.170 | INFO | Perf report for the last 5.13942 seconds: | 20260506 11:08:31.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:08:31.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:31.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:31.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:31.170 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 11:08:31.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:31.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:31.170 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:08:31.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:31.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:31.171 | INFO | Uploaded 36GiB to GPU in 1.60809s (22.3867GiB/s) | 20260506 11:08:31.171 | INFO | Downloaded 34GiB to GPU in 1.52587s (22.2824GiB/s) | 20260506 11:08:36.309 | INFO | Analyzing chunk frames = [878592,879104) | 20260506 11:08:36.309 | INFO | MemInfo: { | 20260506 11:08:36.309 | INFO | "gpu" : " 17 GB", | 20260506 11:08:36.309 | INFO | "huge" : "", | 20260506 11:08:36.309 | INFO | "huge_shared" : " 27 GB", | 20260506 11:08:36.309 | INFO | "malloc" : " 14 GB", | 20260506 11:08:36.309 | INFO | "pinned" : " 29 GB" | 20260506 11:08:36.309 | INFO | } | 20260506 11:08:36.309 | INFO | Perf report for the last 5.13929 seconds: | 20260506 11:08:36.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:08:36.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:36.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:36.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:36.310 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260506 11:08:36.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:36.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:36.310 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:08:36.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:36.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:36.310 | INFO | Uploaded 36GiB to GPU in 1.60576s (22.4193GiB/s) | 20260506 11:08:36.310 | INFO | Downloaded 34GiB to GPU in 1.52321s (22.3212GiB/s) | 20260506 11:08:41.449 | INFO | Analyzing chunk frames = [879104,879616) | 20260506 11:08:41.449 | INFO | MemInfo: { | 20260506 11:08:41.449 | INFO | "gpu" : " 17 GB", | 20260506 11:08:41.449 | INFO | "huge" : "", | 20260506 11:08:41.449 | INFO | "huge_shared" : " 27 GB", | 20260506 11:08:41.449 | INFO | "malloc" : " 14 GB", | 20260506 11:08:41.449 | INFO | "pinned" : " 29 GB" | 20260506 11:08:41.449 | INFO | } | 20260506 11:08:41.449 | INFO | Perf report for the last 5.13921 seconds: | 20260506 11:08:41.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:08:41.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:41.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:41.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:41.449 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 11:08:41.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:41.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:41.449 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:08:41.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:41.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:41.449 | INFO | Uploaded 36GiB to GPU in 1.60642s (22.4101GiB/s) | 20260506 11:08:41.449 | INFO | Downloaded 34GiB to GPU in 1.52373s (22.3137GiB/s) | 20260506 11:08:46.588 | INFO | Analyzing chunk frames = [879616,880128) | 20260506 11:08:46.588 | INFO | MemInfo: { | 20260506 11:08:46.588 | INFO | "gpu" : " 17 GB", | 20260506 11:08:46.588 | INFO | "huge" : "", | 20260506 11:08:46.588 | INFO | "huge_shared" : " 27 GB", | 20260506 11:08:46.588 | INFO | "malloc" : " 14 GB", | 20260506 11:08:46.588 | INFO | "pinned" : " 29 GB" | 20260506 11:08:46.588 | INFO | } | 20260506 11:08:46.588 | INFO | Perf report for the last 5.1393 seconds: | 20260506 11:08:46.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:08:46.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:46.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:46.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:46.588 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 11:08:46.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:46.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:46.588 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 11:08:46.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:46.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:46.588 | INFO | Uploaded 36GiB to GPU in 1.60739s (22.3966GiB/s) | 20260506 11:08:46.588 | INFO | Downloaded 34GiB to GPU in 1.5258s (22.2834GiB/s) | 20260506 11:08:51.727 | INFO | Analyzing chunk frames = [880128,880640) | 20260506 11:08:51.727 | INFO | MemInfo: { | 20260506 11:08:51.727 | INFO | "gpu" : " 17 GB", | 20260506 11:08:51.727 | INFO | "huge" : "", | 20260506 11:08:51.727 | INFO | "huge_shared" : " 27 GB", | 20260506 11:08:51.727 | INFO | "malloc" : " 14 GB", | 20260506 11:08:51.727 | INFO | "pinned" : " 29 GB" | 20260506 11:08:51.727 | INFO | } | 20260506 11:08:51.727 | INFO | Perf report for the last 5.13923 seconds: | 20260506 11:08:51.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:08:51.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:51.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:51.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:51.727 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 11:08:51.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:51.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:51.727 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:08:51.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:51.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:51.728 | INFO | Uploaded 36GiB to GPU in 1.60731s (22.3977GiB/s) | 20260506 11:08:51.728 | INFO | Downloaded 34GiB to GPU in 1.52511s (22.2935GiB/s) | 20260506 11:08:51.825 | INFO | Starting a round of Baz IO | 20260506 11:08:54.727 | INFO | Populating RTMetrics report | 20260506 11:08:54.727 | INFO | RTMetrics report populated | 20260506 11:08:54.727 | INFO | Opening tmp RTMetrics json file | 20260506 11:08:54.728 | INFO | Serializing RTMetrics to json | 20260506 11:08:54.732 | INFO | Serialization done, swapping tmp file now... | 20260506 11:08:54.735 | INFO | RTMetrics done | 20260506 11:08:55.145 | INFO | Finished a round of Baz IO: Wrote 2.037 GiB in 3.319 seconds | 20260506 11:08:55.145 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.14% / 0.04% / 54.82% | 20260506 11:08:55.145 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.58538% / 6.75519e-05% / 4.76837e-05% | 20260506 11:08:55.145 | INFO | Baz IO overall: | 20260506 11:08:55.145 | INFO | BytesWritten : 415620904576 | 20260506 11:08:55.145 | INFO | NumEvents : 200344675950 | 20260506 11:08:56.867 | INFO | Analyzing chunk frames = [880640,881152) | 20260506 11:08:56.867 | INFO | MemInfo: { | 20260506 11:08:56.867 | INFO | "gpu" : " 17 GB", | 20260506 11:08:56.867 | INFO | "huge" : "", | 20260506 11:08:56.867 | INFO | "huge_shared" : " 27 GB", | 20260506 11:08:56.867 | INFO | "malloc" : " 14 GB", | 20260506 11:08:56.867 | INFO | "pinned" : " 29 GB" | 20260506 11:08:56.867 | INFO | } | 20260506 11:08:56.867 | INFO | Perf report for the last 5.1395 seconds: | 20260506 11:08:56.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:08:56.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:56.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:56.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:56.867 | INFO | ANALYSIS : 57.4%, 2.8, 85.1ms, 0 | 20260506 11:08:56.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:08:56.867 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 11:08:56.867 | INFO | PRE_HQ : 56.5%, 2.1, 60.4ms, 0 | 20260506 11:08:56.867 | INFO | BAZWRITER : 64.2%, 4.8, 157.9ms, 0 | 20260506 11:08:56.867 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260506 11:08:56.867 | INFO | Uploaded 36GiB to GPU in 1.67556s (21.4853GiB/s) | 20260506 11:08:56.867 | INFO | Downloaded 38GiB to GPU in 1.74094s (21.8273GiB/s) | 20260506 11:09:02.006 | INFO | Analyzing chunk frames = [881152,881664) | 20260506 11:09:02.006 | INFO | MemInfo: { | 20260506 11:09:02.006 | INFO | "gpu" : " 17 GB", | 20260506 11:09:02.006 | INFO | "huge" : "", | 20260506 11:09:02.006 | INFO | "huge_shared" : " 27 GB", | 20260506 11:09:02.006 | INFO | "malloc" : " 14 GB", | 20260506 11:09:02.006 | INFO | "pinned" : " 29 GB" | 20260506 11:09:02.006 | INFO | } | 20260506 11:09:02.006 | INFO | Perf report for the last 5.13941 seconds: | 20260506 11:09:02.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:09:02.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:02.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:02.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:02.006 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 11:09:02.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:02.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:02.006 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 11:09:02.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:02.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:02.006 | INFO | Uploaded 36GiB to GPU in 1.60536s (22.4248GiB/s) | 20260506 11:09:02.007 | INFO | Downloaded 34GiB to GPU in 1.52222s (22.3359GiB/s) | 20260506 11:09:07.145 | INFO | Analyzing chunk frames = [881664,882176) | 20260506 11:09:07.145 | INFO | MemInfo: { | 20260506 11:09:07.145 | INFO | "gpu" : " 17 GB", | 20260506 11:09:07.145 | INFO | "huge" : "", | 20260506 11:09:07.145 | INFO | "huge_shared" : " 27 GB", | 20260506 11:09:07.145 | INFO | "malloc" : " 14 GB", | 20260506 11:09:07.145 | INFO | "pinned" : " 29 GB" | 20260506 11:09:07.146 | INFO | } | 20260506 11:09:07.146 | INFO | Perf report for the last 5.13935 seconds: | 20260506 11:09:07.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:09:07.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:07.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:07.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:07.146 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260506 11:09:07.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:07.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:07.146 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:09:07.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:07.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:07.146 | INFO | Uploaded 36GiB to GPU in 1.60768s (22.3926GiB/s) | 20260506 11:09:07.146 | INFO | Downloaded 34GiB to GPU in 1.52618s (22.2778GiB/s) | 20260506 11:09:12.285 | INFO | Analyzing chunk frames = [882176,882688) | 20260506 11:09:12.285 | INFO | MemInfo: { | 20260506 11:09:12.285 | INFO | "gpu" : " 17 GB", | 20260506 11:09:12.285 | INFO | "huge" : "", | 20260506 11:09:12.285 | INFO | "huge_shared" : " 27 GB", | 20260506 11:09:12.285 | INFO | "malloc" : " 14 GB", | 20260506 11:09:12.285 | INFO | "pinned" : " 29 GB" | 20260506 11:09:12.285 | INFO | } | 20260506 11:09:12.285 | INFO | Perf report for the last 5.13928 seconds: | 20260506 11:09:12.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:09:12.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:12.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:12.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:12.285 | INFO | ANALYSIS : 56.3%, 3.0, 87.2ms, 0 | 20260506 11:09:12.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:12.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:12.285 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:09:12.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:12.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:12.286 | INFO | Uploaded 36GiB to GPU in 1.60814s (22.3861GiB/s) | 20260506 11:09:12.286 | INFO | Downloaded 34GiB to GPU in 1.52555s (22.287GiB/s) | 20260506 11:09:17.424 | INFO | Analyzing chunk frames = [882688,883200) | 20260506 11:09:17.424 | INFO | MemInfo: { | 20260506 11:09:17.424 | INFO | "gpu" : " 17 GB", | 20260506 11:09:17.424 | INFO | "huge" : "", | 20260506 11:09:17.424 | INFO | "huge_shared" : " 27 GB", | 20260506 11:09:17.424 | INFO | "malloc" : " 14 GB", | 20260506 11:09:17.424 | INFO | "pinned" : " 29 GB" | 20260506 11:09:17.424 | INFO | } | 20260506 11:09:17.424 | INFO | Perf report for the last 5.13943 seconds: | 20260506 11:09:17.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:09:17.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:17.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:17.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:17.424 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260506 11:09:17.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:17.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:17.424 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:09:17.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:17.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:17.425 | INFO | Uploaded 36GiB to GPU in 1.60704s (22.4014GiB/s) | 20260506 11:09:17.425 | INFO | Downloaded 34GiB to GPU in 1.52418s (22.3071GiB/s) | 20260506 11:09:22.563 | INFO | Analyzing chunk frames = [883200,883712) | 20260506 11:09:22.563 | INFO | MemInfo: { | 20260506 11:09:22.563 | INFO | "gpu" : " 17 GB", | 20260506 11:09:22.563 | INFO | "huge" : "", | 20260506 11:09:22.563 | INFO | "huge_shared" : " 27 GB", | 20260506 11:09:22.563 | INFO | "malloc" : " 14 GB", | 20260506 11:09:22.563 | INFO | "pinned" : " 29 GB" | 20260506 11:09:22.564 | INFO | } | 20260506 11:09:22.564 | INFO | Perf report for the last 5.13929 seconds: | 20260506 11:09:22.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:09:22.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:22.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:22.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:22.564 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 11:09:22.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:22.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:22.564 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:09:22.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:22.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:22.564 | INFO | Uploaded 36GiB to GPU in 1.60612s (22.4143GiB/s) | 20260506 11:09:22.564 | INFO | Downloaded 34GiB to GPU in 1.52371s (22.314GiB/s) | 20260506 11:09:27.703 | INFO | Analyzing chunk frames = [883712,884224) | 20260506 11:09:27.703 | INFO | MemInfo: { | 20260506 11:09:27.703 | INFO | "gpu" : " 17 GB", | 20260506 11:09:27.703 | INFO | "huge" : "", | 20260506 11:09:27.703 | INFO | "huge_shared" : " 27 GB", | 20260506 11:09:27.703 | INFO | "malloc" : " 14 GB", | 20260506 11:09:27.703 | INFO | "pinned" : " 29 GB" | 20260506 11:09:27.703 | INFO | } | 20260506 11:09:27.703 | INFO | Perf report for the last 5.13938 seconds: | 20260506 11:09:27.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:09:27.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:27.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:27.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:27.703 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 11:09:27.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:27.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:27.703 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:09:27.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:27.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:27.703 | INFO | Uploaded 36GiB to GPU in 1.60575s (22.4194GiB/s) | 20260506 11:09:27.703 | INFO | Downloaded 34GiB to GPU in 1.5248s (22.298GiB/s) | 20260506 11:09:32.842 | INFO | Analyzing chunk frames = [884224,884736) | 20260506 11:09:32.842 | INFO | MemInfo: { | 20260506 11:09:32.842 | INFO | "gpu" : " 17 GB", | 20260506 11:09:32.842 | INFO | "huge" : "", | 20260506 11:09:32.842 | INFO | "huge_shared" : " 27 GB", | 20260506 11:09:32.842 | INFO | "malloc" : " 14 GB", | 20260506 11:09:32.842 | INFO | "pinned" : " 29 GB" | 20260506 11:09:32.842 | INFO | } | 20260506 11:09:32.842 | INFO | Perf report for the last 5.13929 seconds: | 20260506 11:09:32.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:09:32.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:32.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:32.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:32.842 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 11:09:32.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:32.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:32.842 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:09:32.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:32.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:32.843 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:09:32.843 | INFO | Total mem usage: 16.8 GiB | 20260506 11:09:32.843 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:09:32.843 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -7.68%) | 20260506 11:09:32.843 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:09:32.843 | INFO | High Water Breakdown: | 20260506 11:09:32.843 | INFO | 60.6% : 9.46 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:09:32.843 | INFO | 39.3% : 6.13 GiB ( 91.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:09:32.843 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:09:32.843 | INFO | ------------------------------------------ | 20260506 11:09:32.843 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:09:32.843 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:09:32.843 | INFO | Total mem usage: 26.5 GiB | 20260506 11:09:32.843 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 11:09:32.843 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:09:32.843 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 11:09:32.843 | INFO | High Water Breakdown: | 20260506 11:09:32.843 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:09:32.843 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:09:32.843 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:09:32.843 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:09:32.843 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:09:32.843 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:09:32.843 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:09:32.843 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:09:32.843 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:09:32.843 | INFO | Uploaded 36GiB to GPU in 1.60511s (22.4283GiB/s) | 20260506 11:09:32.843 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:09:32.843 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:09:32.843 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:09:32.843 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:09:32.843 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:09:32.843 | INFO | Downloaded 34GiB to GPU in 1.52549s (22.2879GiB/s) | 20260506 11:09:32.843 | INFO | ------------------------------------------ | 20260506 11:09:32.843 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:09:32.843 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:09:32.843 | INFO | Total mem usage: 25.6 GiB | 20260506 11:09:32.843 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:09:32.843 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 11:09:32.843 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 11:09:32.843 | INFO | High Water Breakdown: | 20260506 11:09:32.843 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 11:09:32.843 | INFO | ------------------------------------------ | 20260506 11:09:32.843 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:09:32.843 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:09:32.843 | INFO | Total mem usage: 15.7 GiB | 20260506 11:09:32.843 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:09:32.843 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:09:32.843 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:09:32.843 | INFO | High Water Breakdown: | 20260506 11:09:32.843 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:09:32.843 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:09:32.843 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:09:32.843 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:09:32.843 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:09:32.843 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:09:32.843 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:09:32.843 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:09:32.843 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:09:32.843 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:09:32.843 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:09:32.844 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:09:32.844 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:09:32.844 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:09:32.844 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:09:32.844 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:09:32.844 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:09:32.844 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:09:32.844 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:09:32.844 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:09:32.844 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:09:32.844 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:09:32.844 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:09:32.844 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:09:32.844 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:09:32.844 | INFO | ------------------------------------------ | 20260506 11:09:32.844 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:09:32.844 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:09:32.844 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:09:32.844 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:09:32.844 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 11:09:32.844 | INFO | PerfReport: 0.29%, AggregateStats -- 8.65e-02 / -nan, 4.56e-03 / 0.00e+00 | 20260506 11:09:32.844 | INFO | PerfReport: 10.98%, Baseline -- 3.30e+00 / -nan, 1.89e-02 / 0.00e+00 | 20260506 11:09:32.844 | INFO | PerfReport: 3.07%, Histogram -- 9.21e-01 / -nan, 5.21e-02 / 0.00e+00 | 20260506 11:09:32.844 | INFO | PerfReport: 16.07%, DME -- 3.86e+01 / -nan, 2.97e+00 / 0.00e+00 | 20260506 11:09:32.844 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.72e+01 / -nan, 6.67e-02 / 0.00e+00 | 20260506 11:09:32.844 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.20e+00 / -nan, 1.24e-01 / 0.00e+00 | 20260506 11:09:32.844 | INFO | PerfReport: 3.59%, Metrics -- 1.08e+00 / -nan, 1.68e-01 / 0.00e+00 | 20260506 11:09:32.844 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.15e-01 / -nan, 5.76e-02 / 0.00e+00 | 20260506 11:09:32.844 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:09:32.844 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:09:32.844 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:09:32.844 | INFO | PerfReport: Full Execution -- 8.96e+01 / -nan, 1.87e+01 / 0.00e+00 | 20260506 11:09:32.844 | INFO | PerfReport: 18.78%, Upload -- 1.68e+01 / 0.00e+00, 5.05e-01 / 0.00e+00 | 20260506 11:09:32.844 | INFO | PerfReport: 18.04%, Download -- 1.62e+01 / 0.00e+00, 8.57e-01 / 0.00e+00 | 20260506 11:09:32.844 | INFO | PerfReport: 33.49%, Compute -- 3.00e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 11:09:32.844 | INFO | PerfReport: 0.56%, LockUpload -- 5.06e-01 / 0.00e+00, 3.60e+00 / 0.00e+00 | 20260506 11:09:32.844 | INFO | PerfReport: 0.00%, LockDownload -- 4.89e-04 / 0.00e+00, 1.31e-04 / 0.00e+00 | 20260506 11:09:32.844 | INFO | PerfReport: 29.11%, LockCompute -- 2.61e+01 / 0.00e+00, 1.70e+01 / 0.00e+00 | 20260506 11:09:32.844 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260506 11:09:32.940 | INFO | Starting a round of Baz IO | 20260506 11:09:35.846 | INFO | Populating RTMetrics report | 20260506 11:09:35.846 | INFO | RTMetrics report populated | 20260506 11:09:35.846 | INFO | Opening tmp RTMetrics json file | 20260506 11:09:36.168 | INFO | Serializing RTMetrics to json | 20260506 11:09:36.223 | INFO | Serialization done, swapping tmp file now... | 20260506 11:09:36.240 | INFO | RTMetrics done | 20260506 11:09:36.496 | INFO | Finished a round of Baz IO: Wrote 2.038 GiB in 3.555 seconds | 20260506 11:09:36.496 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.10% / 0.04% / 54.86% | 20260506 11:09:36.497 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.57393% / 7.94729e-05% / 5.16574e-05% | 20260506 11:09:36.497 | INFO | Baz IO overall: | 20260506 11:09:36.497 | INFO | BytesWritten : 417809510404 | 20260506 11:09:36.497 | INFO | NumEvents : 201406633912 | 20260506 11:09:37.981 | INFO | Analyzing chunk frames = [884736,885248) | 20260506 11:09:37.981 | INFO | MemInfo: { | 20260506 11:09:37.981 | INFO | "gpu" : " 17 GB", | 20260506 11:09:37.981 | INFO | "huge" : "", | 20260506 11:09:37.981 | INFO | "huge_shared" : " 27 GB", | 20260506 11:09:37.981 | INFO | "malloc" : " 14 GB", | 20260506 11:09:37.981 | INFO | "pinned" : " 29 GB" | 20260506 11:09:37.981 | INFO | } | 20260506 11:09:37.981 | INFO | Perf report for the last 5.13919 seconds: | 20260506 11:09:37.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:09:37.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:37.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:37.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:37.982 | INFO | ANALYSIS : 57.4%, 2.8, 84.9ms, 0 | 20260506 11:09:37.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:37.982 | INFO | RT_METRICS : 51.8%, 1.0, 27.2ms, 0 | 20260506 11:09:37.982 | INFO | PRE_HQ : 56.4%, 2.1, 60.5ms, 0 | 20260506 11:09:37.982 | INFO | BAZWRITER : 68.8%, 4.8, 169.6ms, 0 | 20260506 11:09:37.982 | INFO | BAZ_SAMPLING : 6.5%, 1.0, 3.4ms, 0 | 20260506 11:09:37.982 | INFO | Uploaded 36GiB to GPU in 1.67309s (21.5171GiB/s) | 20260506 11:09:37.982 | INFO | Downloaded 38GiB to GPU in 1.73818s (21.8619GiB/s) | 20260506 11:09:43.121 | INFO | Analyzing chunk frames = [885248,885760) | 20260506 11:09:43.121 | INFO | MemInfo: { | 20260506 11:09:43.121 | INFO | "gpu" : " 17 GB", | 20260506 11:09:43.121 | INFO | "huge" : "", | 20260506 11:09:43.121 | INFO | "huge_shared" : " 27 GB", | 20260506 11:09:43.121 | INFO | "malloc" : " 14 GB", | 20260506 11:09:43.121 | INFO | "pinned" : " 29 GB" | 20260506 11:09:43.121 | INFO | } | 20260506 11:09:43.121 | INFO | Perf report for the last 5.13958 seconds: | 20260506 11:09:43.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:09:43.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:43.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:43.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:43.121 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 11:09:43.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:43.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:43.121 | INFO | PRE_HQ : 28.7%, 1.0, 15.0ms, 0 | 20260506 11:09:43.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:43.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:43.121 | INFO | Uploaded 36GiB to GPU in 1.60313s (22.456GiB/s) | 20260506 11:09:43.121 | INFO | Downloaded 34GiB to GPU in 1.52015s (22.3662GiB/s) | 20260506 11:09:48.260 | INFO | Analyzing chunk frames = [885760,886272) | 20260506 11:09:48.260 | INFO | MemInfo: { | 20260506 11:09:48.260 | INFO | "gpu" : " 17 GB", | 20260506 11:09:48.260 | INFO | "huge" : "", | 20260506 11:09:48.260 | INFO | "huge_shared" : " 27 GB", | 20260506 11:09:48.260 | INFO | "malloc" : " 14 GB", | 20260506 11:09:48.260 | INFO | "pinned" : " 29 GB" | 20260506 11:09:48.260 | INFO | } | 20260506 11:09:48.260 | INFO | Perf report for the last 5.13939 seconds: | 20260506 11:09:48.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:09:48.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:48.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:48.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:48.261 | INFO | ANALYSIS : 56.5%, 3.0, 87.9ms, 0 | 20260506 11:09:48.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:48.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:48.261 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:09:48.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:48.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:48.261 | INFO | Uploaded 36GiB to GPU in 1.60842s (22.3823GiB/s) | 20260506 11:09:48.261 | INFO | Downloaded 34GiB to GPU in 1.5268s (22.2688GiB/s) | 20260506 11:09:53.399 | INFO | Analyzing chunk frames = [886272,886784) | 20260506 11:09:53.399 | INFO | MemInfo: { | 20260506 11:09:53.399 | INFO | "gpu" : " 17 GB", | 20260506 11:09:53.399 | INFO | "huge" : "", | 20260506 11:09:53.399 | INFO | "huge_shared" : " 27 GB", | 20260506 11:09:53.400 | INFO | "malloc" : " 14 GB", | 20260506 11:09:53.400 | INFO | "pinned" : " 29 GB" | 20260506 11:09:53.400 | INFO | } | 20260506 11:09:53.400 | INFO | Perf report for the last 5.1391 seconds: | 20260506 11:09:53.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:09:53.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:53.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:53.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:53.400 | INFO | ANALYSIS : 56.3%, 3.0, 87.4ms, 0 | 20260506 11:09:53.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:53.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:53.400 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:09:53.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:53.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:53.400 | INFO | Uploaded 36GiB to GPU in 1.60698s (22.4022GiB/s) | 20260506 11:09:53.400 | INFO | Downloaded 34GiB to GPU in 1.52459s (22.3011GiB/s) | 20260506 11:09:58.539 | INFO | Analyzing chunk frames = [886784,887296) | 20260506 11:09:58.539 | INFO | MemInfo: { | 20260506 11:09:58.539 | INFO | "gpu" : " 17 GB", | 20260506 11:09:58.539 | INFO | "huge" : "", | 20260506 11:09:58.539 | INFO | "huge_shared" : " 27 GB", | 20260506 11:09:58.539 | INFO | "malloc" : " 14 GB", | 20260506 11:09:58.539 | INFO | "pinned" : " 29 GB" | 20260506 11:09:58.539 | INFO | } | 20260506 11:09:58.539 | INFO | Perf report for the last 5.13941 seconds: | 20260506 11:09:58.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:09:58.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:58.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:58.539 | INFO | SAVE_TRACE : 0.1%, 1.0, 0.0ms, 0 | 20260506 11:09:58.539 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260506 11:09:58.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:58.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:58.539 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:09:58.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:58.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:09:58.539 | INFO | Uploaded 36GiB to GPU in 1.60612s (22.4143GiB/s) | 20260506 11:09:58.539 | INFO | Downloaded 34GiB to GPU in 1.52295s (22.3251GiB/s) | 20260506 11:10:03.678 | INFO | Analyzing chunk frames = [887296,887808) | 20260506 11:10:03.678 | INFO | MemInfo: { | 20260506 11:10:03.678 | INFO | "gpu" : " 17 GB", | 20260506 11:10:03.678 | INFO | "huge" : "", | 20260506 11:10:03.678 | INFO | "huge_shared" : " 27 GB", | 20260506 11:10:03.678 | INFO | "malloc" : " 14 GB", | 20260506 11:10:03.678 | INFO | "pinned" : " 29 GB" | 20260506 11:10:03.678 | INFO | } | 20260506 11:10:03.678 | INFO | Perf report for the last 5.1394 seconds: | 20260506 11:10:03.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:10:03.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:03.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:03.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:03.679 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 11:10:03.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:03.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:03.679 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:10:03.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:03.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:03.679 | INFO | Uploaded 36GiB to GPU in 1.60621s (22.413GiB/s) | 20260506 11:10:03.679 | INFO | Downloaded 34GiB to GPU in 1.52304s (22.3238GiB/s) | 20260506 11:10:08.818 | INFO | Analyzing chunk frames = [887808,888320) | 20260506 11:10:08.818 | INFO | MemInfo: { | 20260506 11:10:08.818 | INFO | "gpu" : " 17 GB", | 20260506 11:10:08.818 | INFO | "huge" : "", | 20260506 11:10:08.818 | INFO | "huge_shared" : " 27 GB", | 20260506 11:10:08.818 | INFO | "malloc" : " 14 GB", | 20260506 11:10:08.818 | INFO | "pinned" : " 29 GB" | 20260506 11:10:08.818 | INFO | } | 20260506 11:10:08.818 | INFO | Perf report for the last 5.13937 seconds: | 20260506 11:10:08.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:10:08.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:08.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:08.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:08.818 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 11:10:08.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:08.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:08.818 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:10:08.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:08.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:08.818 | INFO | Uploaded 36GiB to GPU in 1.60645s (22.4096GiB/s) | 20260506 11:10:08.818 | INFO | Downloaded 34GiB to GPU in 1.52422s (22.3064GiB/s) | 20260506 11:10:13.957 | INFO | Analyzing chunk frames = [888320,888832) | 20260506 11:10:13.957 | INFO | MemInfo: { | 20260506 11:10:13.957 | INFO | "gpu" : " 17 GB", | 20260506 11:10:13.957 | INFO | "huge" : "", | 20260506 11:10:13.957 | INFO | "huge_shared" : " 27 GB", | 20260506 11:10:13.957 | INFO | "malloc" : " 14 GB", | 20260506 11:10:13.957 | INFO | "pinned" : " 29 GB" | 20260506 11:10:13.957 | INFO | } | 20260506 11:10:13.957 | INFO | Perf report for the last 5.13951 seconds: | 20260506 11:10:13.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:10:13.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:13.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:13.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:13.957 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 11:10:13.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:13.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:13.957 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 11:10:13.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:13.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:13.958 | INFO | Uploaded 36GiB to GPU in 1.60634s (22.4112GiB/s) | 20260506 11:10:13.958 | INFO | Downloaded 34GiB to GPU in 1.5237s (22.3141GiB/s) | 20260506 11:10:14.055 | INFO | Starting a round of Baz IO | 20260506 11:10:16.958 | INFO | Populating RTMetrics report | 20260506 11:10:16.958 | INFO | RTMetrics report populated | 20260506 11:10:16.958 | INFO | Opening tmp RTMetrics json file | 20260506 11:10:16.994 | INFO | Serializing RTMetrics to json | 20260506 11:10:17.019 | INFO | Serialization done, swapping tmp file now... | 20260506 11:10:17.020 | INFO | RTMetrics done | 20260506 11:10:17.157 | INFO | Finished a round of Baz IO: Wrote 2.038 GiB in 3.102 seconds | 20260506 11:10:17.157 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.06% / 0.04% / 54.90% | 20260506 11:10:17.157 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.55658% / 7.94729e-05% / 5.16574e-05% | 20260506 11:10:17.157 | INFO | Baz IO overall: | 20260506 11:10:17.158 | INFO | BytesWritten : 419998148730 | 20260506 11:10:17.158 | INFO | NumEvents : 202468676892 | 20260506 11:10:19.096 | INFO | Analyzing chunk frames = [888832,889344) | 20260506 11:10:19.096 | INFO | MemInfo: { | 20260506 11:10:19.096 | INFO | "gpu" : " 17 GB", | 20260506 11:10:19.096 | INFO | "huge" : "", | 20260506 11:10:19.096 | INFO | "huge_shared" : " 27 GB", | 20260506 11:10:19.096 | INFO | "malloc" : " 14 GB", | 20260506 11:10:19.096 | INFO | "pinned" : " 29 GB" | 20260506 11:10:19.096 | INFO | } | 20260506 11:10:19.096 | INFO | Perf report for the last 5.1391 seconds: | 20260506 11:10:19.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:10:19.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:19.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:19.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:19.097 | INFO | ANALYSIS : 57.4%, 2.8, 85.2ms, 0 | 20260506 11:10:19.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:19.097 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 11:10:19.097 | INFO | PRE_HQ : 56.4%, 2.1, 60.2ms, 0 | 20260506 11:10:19.097 | INFO | BAZWRITER : 59.9%, 4.8, 146.5ms, 0 | 20260506 11:10:19.097 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 11:10:19.097 | INFO | Uploaded 36GiB to GPU in 1.67582s (21.482GiB/s) | 20260506 11:10:19.097 | INFO | Downloaded 38GiB to GPU in 1.74247s (21.8082GiB/s) | 20260506 11:10:24.235 | INFO | Analyzing chunk frames = [889344,889856) | 20260506 11:10:24.235 | INFO | MemInfo: { | 20260506 11:10:24.235 | INFO | "gpu" : " 17 GB", | 20260506 11:10:24.235 | INFO | "huge" : "", | 20260506 11:10:24.236 | INFO | "huge_shared" : " 27 GB", | 20260506 11:10:24.236 | INFO | "malloc" : " 14 GB", | 20260506 11:10:24.236 | INFO | "pinned" : " 29 GB" | 20260506 11:10:24.236 | INFO | } | 20260506 11:10:24.236 | INFO | Perf report for the last 5.13928 seconds: | 20260506 11:10:24.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:10:24.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:24.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:24.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:24.236 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 11:10:24.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:24.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:24.236 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:10:24.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:24.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:24.236 | INFO | Uploaded 36GiB to GPU in 1.60365s (22.4488GiB/s) | 20260506 11:10:24.236 | INFO | Downloaded 34GiB to GPU in 1.52108s (22.3525GiB/s) | 20260506 11:10:29.375 | INFO | Analyzing chunk frames = [889856,890368) | 20260506 11:10:29.375 | INFO | MemInfo: { | 20260506 11:10:29.375 | INFO | "gpu" : " 17 GB", | 20260506 11:10:29.375 | INFO | "huge" : "", | 20260506 11:10:29.375 | INFO | "huge_shared" : " 27 GB", | 20260506 11:10:29.375 | INFO | "malloc" : " 14 GB", | 20260506 11:10:29.375 | INFO | "pinned" : " 29 GB" | 20260506 11:10:29.375 | INFO | } | 20260506 11:10:29.375 | INFO | Perf report for the last 5.13943 seconds: | 20260506 11:10:29.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:10:29.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:29.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:29.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:29.375 | INFO | ANALYSIS : 56.5%, 3.0, 87.8ms, 0 | 20260506 11:10:29.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:29.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:29.375 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:10:29.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:29.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:29.376 | INFO | Uploaded 36GiB to GPU in 1.60856s (22.3802GiB/s) | 20260506 11:10:29.376 | INFO | Downloaded 34GiB to GPU in 1.52673s (22.2698GiB/s) | 20260506 11:10:34.514 | INFO | Analyzing chunk frames = [890368,890880) | 20260506 11:10:34.514 | INFO | MemInfo: { | 20260506 11:10:34.514 | INFO | "gpu" : " 17 GB", | 20260506 11:10:34.514 | INFO | "huge" : "", | 20260506 11:10:34.514 | INFO | "huge_shared" : " 27 GB", | 20260506 11:10:34.514 | INFO | "malloc" : " 14 GB", | 20260506 11:10:34.514 | INFO | "pinned" : " 29 GB" | 20260506 11:10:34.514 | INFO | } | 20260506 11:10:34.514 | INFO | Perf report for the last 5.13927 seconds: | 20260506 11:10:34.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:10:34.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:34.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:34.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:34.515 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 11:10:34.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:34.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:34.515 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:10:34.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:34.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:34.515 | INFO | Uploaded 36GiB to GPU in 1.6078s (22.3908GiB/s) | 20260506 11:10:34.515 | INFO | Downloaded 34GiB to GPU in 1.5255s (22.2878GiB/s) | 20260506 11:10:39.653 | INFO | Analyzing chunk frames = [890880,891392) | 20260506 11:10:39.653 | INFO | MemInfo: { | 20260506 11:10:39.653 | INFO | "gpu" : " 17 GB", | 20260506 11:10:39.653 | INFO | "huge" : "", | 20260506 11:10:39.654 | INFO | "huge_shared" : " 27 GB", | 20260506 11:10:39.654 | INFO | "malloc" : " 14 GB", | 20260506 11:10:39.654 | INFO | "pinned" : " 29 GB" | 20260506 11:10:39.654 | INFO | } | 20260506 11:10:39.654 | INFO | Perf report for the last 5.13926 seconds: | 20260506 11:10:39.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:10:39.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:39.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:39.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:39.654 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260506 11:10:39.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:39.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:39.654 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:10:39.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:39.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:39.655 | INFO | Uploaded 36GiB to GPU in 1.60634s (22.4111GiB/s) | 20260506 11:10:39.655 | INFO | Downloaded 34GiB to GPU in 1.52302s (22.3241GiB/s) | 20260506 11:10:44.793 | INFO | Analyzing chunk frames = [891392,891904) | 20260506 11:10:44.793 | INFO | MemInfo: { | 20260506 11:10:44.793 | INFO | "gpu" : " 17 GB", | 20260506 11:10:44.793 | INFO | "huge" : "", | 20260506 11:10:44.793 | INFO | "huge_shared" : " 27 GB", | 20260506 11:10:44.793 | INFO | "malloc" : " 14 GB", | 20260506 11:10:44.793 | INFO | "pinned" : " 29 GB" | 20260506 11:10:44.793 | INFO | } | 20260506 11:10:44.793 | INFO | Perf report for the last 5.13916 seconds: | 20260506 11:10:44.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:10:44.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:44.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:44.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:44.793 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 11:10:44.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:44.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:44.793 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:10:44.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:44.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:44.793 | INFO | Uploaded 36GiB to GPU in 1.6065s (22.409GiB/s) | 20260506 11:10:44.793 | INFO | Downloaded 34GiB to GPU in 1.52323s (22.321GiB/s) | 20260506 11:10:49.932 | INFO | Analyzing chunk frames = [891904,892416) | 20260506 11:10:49.932 | INFO | MemInfo: { | 20260506 11:10:49.932 | INFO | "gpu" : " 17 GB", | 20260506 11:10:49.932 | INFO | "huge" : "", | 20260506 11:10:49.932 | INFO | "huge_shared" : " 27 GB", | 20260506 11:10:49.932 | INFO | "malloc" : " 14 GB", | 20260506 11:10:49.932 | INFO | "pinned" : " 29 GB" | 20260506 11:10:49.932 | INFO | } | 20260506 11:10:49.932 | INFO | Perf report for the last 5.13943 seconds: | 20260506 11:10:49.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:10:49.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:49.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:49.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:49.932 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 11:10:49.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:49.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:49.933 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 11:10:49.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:49.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:49.933 | INFO | Uploaded 36GiB to GPU in 1.60702s (22.4017GiB/s) | 20260506 11:10:49.933 | INFO | Downloaded 34GiB to GPU in 1.52484s (22.2974GiB/s) | 20260506 11:10:55.071 | INFO | Analyzing chunk frames = [892416,892928) | 20260506 11:10:55.071 | INFO | MemInfo: { | 20260506 11:10:55.071 | INFO | "gpu" : " 17 GB", | 20260506 11:10:55.071 | INFO | "huge" : "", | 20260506 11:10:55.072 | INFO | "huge_shared" : " 27 GB", | 20260506 11:10:55.072 | INFO | "malloc" : " 14 GB", | 20260506 11:10:55.072 | INFO | "pinned" : " 29 GB" | 20260506 11:10:55.072 | INFO | } | 20260506 11:10:55.072 | INFO | Perf report for the last 5.13923 seconds: | 20260506 11:10:55.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:10:55.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:55.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:55.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:55.072 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 11:10:55.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:55.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:55.072 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 11:10:55.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:55.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:10:55.072 | INFO | Uploaded 36GiB to GPU in 1.60682s (22.4044GiB/s) | 20260506 11:10:55.072 | INFO | Downloaded 34GiB to GPU in 1.52396s (22.3103GiB/s) | 20260506 11:10:55.072 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:10:55.072 | INFO | Total mem usage: 16.8 GiB | 20260506 11:10:55.072 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:10:55.072 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -8.61%) | 20260506 11:10:55.072 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:10:55.072 | INFO | High Water Breakdown: | 20260506 11:10:55.072 | INFO | 60.4% : 9.35 GiB ( 90.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:10:55.072 | INFO | 39.5% : 6.11 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:10:55.072 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:10:55.072 | INFO | ------------------------------------------ | 20260506 11:10:55.072 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:10:55.072 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:10:55.072 | INFO | Total mem usage: 26.5 GiB | 20260506 11:10:55.072 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 11:10:55.073 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:10:55.073 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 11:10:55.073 | INFO | High Water Breakdown: | 20260506 11:10:55.073 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:10:55.073 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:10:55.073 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:10:55.073 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:10:55.073 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:10:55.073 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:10:55.073 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:10:55.073 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:10:55.073 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:10:55.073 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:10:55.073 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:10:55.073 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:10:55.073 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:10:55.073 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:10:55.073 | INFO | ------------------------------------------ | 20260506 11:10:55.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:10:55.073 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:10:55.073 | INFO | Total mem usage: 25.6 GiB | 20260506 11:10:55.073 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:10:55.073 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 11:10:55.073 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 11:10:55.073 | INFO | High Water Breakdown: | 20260506 11:10:55.073 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 11:10:55.073 | INFO | ------------------------------------------ | 20260506 11:10:55.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:10:55.073 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:10:55.073 | INFO | Total mem usage: 15.7 GiB | 20260506 11:10:55.073 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:10:55.073 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:10:55.073 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:10:55.073 | INFO | High Water Breakdown: | 20260506 11:10:55.073 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:10:55.073 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:10:55.073 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:10:55.073 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:10:55.073 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:10:55.073 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:10:55.073 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:10:55.073 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:10:55.073 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:10:55.073 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:10:55.073 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:10:55.073 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:10:55.073 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:10:55.073 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:10:55.073 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:10:55.073 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:10:55.073 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:10:55.073 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:10:55.073 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:10:55.073 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:10:55.073 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:10:55.073 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:10:55.073 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:10:55.073 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:10:55.073 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:10:55.073 | INFO | ------------------------------------------ | 20260506 11:10:55.073 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:10:55.074 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:10:55.074 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:10:55.074 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:10:55.074 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.25e+01 / 0.00e+00 | 20260506 11:10:55.074 | INFO | PerfReport: 0.29%, AggregateStats -- 8.67e-02 / -nan, 6.81e-03 / 0.00e+00 | 20260506 11:10:55.074 | INFO | PerfReport: 10.98%, Baseline -- 3.30e+00 / -nan, 1.75e-02 / 0.00e+00 | 20260506 11:10:55.074 | INFO | PerfReport: 3.07%, Histogram -- 9.21e-01 / -nan, 5.20e-02 / 0.00e+00 | 20260506 11:10:55.074 | INFO | PerfReport: 16.07%, DME -- 3.86e+01 / -nan, 3.04e+00 / 0.00e+00 | 20260506 11:10:55.074 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.72e+01 / -nan, 4.22e-02 / 0.00e+00 | 20260506 11:10:55.074 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.20e+00 / -nan, 1.23e-01 / 0.00e+00 | 20260506 11:10:55.074 | INFO | PerfReport: 3.59%, Metrics -- 1.08e+00 / -nan, 1.68e-01 / 0.00e+00 | 20260506 11:10:55.074 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.14e-01 / -nan, 5.75e-02 / 0.00e+00 | 20260506 11:10:55.074 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:10:55.074 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:10:55.074 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:10:55.074 | INFO | PerfReport: Full Execution -- 8.96e+01 / -nan, 1.84e+01 / 0.00e+00 | 20260506 11:10:55.074 | INFO | PerfReport: 18.78%, Upload -- 1.68e+01 / 0.00e+00, 4.92e-01 / 0.00e+00 | 20260506 11:10:55.074 | INFO | PerfReport: 18.03%, Download -- 1.62e+01 / 0.00e+00, 8.48e-01 / 0.00e+00 | 20260506 11:10:55.074 | INFO | PerfReport: 33.50%, Compute -- 3.00e+01 / 0.00e+00, 1.25e+01 / 0.00e+00 | 20260506 11:10:55.074 | INFO | PerfReport: 0.56%, LockUpload -- 5.05e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 11:10:55.074 | INFO | PerfReport: 0.00%, LockDownload -- 4.96e-04 / 0.00e+00, 2.18e-04 / 0.00e+00 | 20260506 11:10:55.074 | INFO | PerfReport: 29.11%, LockCompute -- 2.61e+01 / 0.00e+00, 1.68e+01 / 0.00e+00 | 20260506 11:10:55.074 | INFO | PerfReport: 0.02%, Misc -- 1.37e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260506 11:10:55.169 | INFO | Starting a round of Baz IO | 20260506 11:10:58.069 | INFO | Populating RTMetrics report | 20260506 11:10:58.070 | INFO | RTMetrics report populated | 20260506 11:10:58.070 | INFO | Opening tmp RTMetrics json file | 20260506 11:10:58.090 | INFO | Serializing RTMetrics to json | 20260506 11:10:58.091 | INFO | Serialization done, swapping tmp file now... | 20260506 11:10:58.092 | INFO | RTMetrics done | 20260506 11:10:58.646 | INFO | Finished a round of Baz IO: Wrote 2.043 GiB in 3.477 seconds | 20260506 11:10:58.646 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 45.02% / 0.04% / 54.94% | 20260506 11:10:58.646 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.57007% / 6.75519e-05% / 6.75519e-05% | 20260506 11:10:58.646 | INFO | Baz IO overall: | 20260506 11:10:58.647 | INFO | BytesWritten : 422191675855 | 20260506 11:10:58.647 | INFO | NumEvents : 203534310019 | 20260506 11:11:00.211 | INFO | Analyzing chunk frames = [892928,893440) | 20260506 11:11:00.211 | INFO | MemInfo: { | 20260506 11:11:00.211 | INFO | "gpu" : " 17 GB", | 20260506 11:11:00.211 | INFO | "huge" : "", | 20260506 11:11:00.211 | INFO | "huge_shared" : " 27 GB", | 20260506 11:11:00.211 | INFO | "malloc" : " 14 GB", | 20260506 11:11:00.211 | INFO | "pinned" : " 29 GB" | 20260506 11:11:00.211 | INFO | } | 20260506 11:11:00.211 | INFO | Perf report for the last 5.13919 seconds: | 20260506 11:11:00.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:11:00.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:00.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:00.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:00.211 | INFO | ANALYSIS : 57.3%, 2.8, 85.4ms, 0 | 20260506 11:11:00.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:00.211 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 11:11:00.211 | INFO | PRE_HQ : 56.3%, 2.0, 59.7ms, 0 | 20260506 11:11:00.211 | INFO | BAZWRITER : 67.2%, 4.8, 165.6ms, 0 | 20260506 11:11:00.211 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 11:11:00.212 | INFO | Uploaded 36GiB to GPU in 1.68166s (21.4074GiB/s) | 20260506 11:11:00.212 | INFO | Downloaded 38GiB to GPU in 1.74727s (21.7482GiB/s) | 20260506 11:11:05.350 | INFO | Analyzing chunk frames = [893440,893952) | 20260506 11:11:05.350 | INFO | MemInfo: { | 20260506 11:11:05.350 | INFO | "gpu" : " 17 GB", | 20260506 11:11:05.350 | INFO | "huge" : "", | 20260506 11:11:05.350 | INFO | "huge_shared" : " 27 GB", | 20260506 11:11:05.350 | INFO | "malloc" : " 14 GB", | 20260506 11:11:05.350 | INFO | "pinned" : " 29 GB" | 20260506 11:11:05.350 | INFO | } | 20260506 11:11:05.350 | INFO | Perf report for the last 5.1394 seconds: | 20260506 11:11:05.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:11:05.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:05.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:05.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:05.350 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 11:11:05.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:05.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:05.350 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:11:05.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:05.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:05.350 | INFO | Uploaded 36GiB to GPU in 1.60409s (22.4427GiB/s) | 20260506 11:11:05.351 | INFO | Downloaded 34GiB to GPU in 1.52146s (22.347GiB/s) | 20260506 11:11:10.489 | INFO | Analyzing chunk frames = [893952,894464) | 20260506 11:11:10.489 | INFO | MemInfo: { | 20260506 11:11:10.489 | INFO | "gpu" : " 17 GB", | 20260506 11:11:10.489 | INFO | "huge" : "", | 20260506 11:11:10.489 | INFO | "huge_shared" : " 27 GB", | 20260506 11:11:10.489 | INFO | "malloc" : " 14 GB", | 20260506 11:11:10.489 | INFO | "pinned" : " 29 GB" | 20260506 11:11:10.489 | INFO | } | 20260506 11:11:10.489 | INFO | Perf report for the last 5.13921 seconds: | 20260506 11:11:10.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:11:10.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:10.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:10.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:10.490 | INFO | ANALYSIS : 56.5%, 3.0, 87.8ms, 0 | 20260506 11:11:10.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:10.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:10.490 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:11:10.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:10.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:10.490 | INFO | Uploaded 36GiB to GPU in 1.60796s (22.3886GiB/s) | 20260506 11:11:10.490 | INFO | Downloaded 34GiB to GPU in 1.52593s (22.2815GiB/s) | 20260506 11:11:15.629 | INFO | Analyzing chunk frames = [894464,894976) | 20260506 11:11:15.629 | INFO | MemInfo: { | 20260506 11:11:15.629 | INFO | "gpu" : " 17 GB", | 20260506 11:11:15.629 | INFO | "huge" : "", | 20260506 11:11:15.629 | INFO | "huge_shared" : " 27 GB", | 20260506 11:11:15.629 | INFO | "malloc" : " 14 GB", | 20260506 11:11:15.629 | INFO | "pinned" : " 29 GB" | 20260506 11:11:15.629 | INFO | } | 20260506 11:11:15.629 | INFO | Perf report for the last 5.13962 seconds: | 20260506 11:11:15.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:11:15.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:15.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:15.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:15.629 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 11:11:15.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:15.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:15.629 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:11:15.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:15.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:15.630 | INFO | Uploaded 36GiB to GPU in 1.60742s (22.3961GiB/s) | 20260506 11:11:15.630 | INFO | Downloaded 34GiB to GPU in 1.5247s (22.2995GiB/s) | 20260506 11:11:20.768 | INFO | Analyzing chunk frames = [894976,895488) | 20260506 11:11:20.768 | INFO | MemInfo: { | 20260506 11:11:20.768 | INFO | "gpu" : " 17 GB", | 20260506 11:11:20.768 | INFO | "huge" : "", | 20260506 11:11:20.768 | INFO | "huge_shared" : " 27 GB", | 20260506 11:11:20.768 | INFO | "malloc" : " 14 GB", | 20260506 11:11:20.768 | INFO | "pinned" : " 29 GB" | 20260506 11:11:20.768 | INFO | } | 20260506 11:11:20.768 | INFO | Perf report for the last 5.13931 seconds: | 20260506 11:11:20.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:11:20.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:20.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:20.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:20.769 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260506 11:11:20.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:20.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:20.769 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:11:20.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:20.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:20.769 | INFO | Uploaded 36GiB to GPU in 1.60625s (22.4124GiB/s) | 20260506 11:11:20.769 | INFO | Downloaded 34GiB to GPU in 1.52324s (22.3208GiB/s) | 20260506 11:11:25.907 | INFO | Analyzing chunk frames = [895488,896000) | 20260506 11:11:25.907 | INFO | MemInfo: { | 20260506 11:11:25.907 | INFO | "gpu" : " 17 GB", | 20260506 11:11:25.907 | INFO | "huge" : "", | 20260506 11:11:25.907 | INFO | "huge_shared" : " 27 GB", | 20260506 11:11:25.907 | INFO | "malloc" : " 14 GB", | 20260506 11:11:25.907 | INFO | "pinned" : " 29 GB" | 20260506 11:11:25.908 | INFO | } | 20260506 11:11:25.908 | INFO | Perf report for the last 5.13915 seconds: | 20260506 11:11:25.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:11:25.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:25.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:25.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:25.908 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 11:11:25.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:25.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:25.908 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 11:11:25.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:25.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:25.908 | INFO | Uploaded 36GiB to GPU in 1.60617s (22.4136GiB/s) | 20260506 11:11:25.908 | INFO | Downloaded 34GiB to GPU in 1.52362s (22.3153GiB/s) | 20260506 11:11:31.047 | INFO | Analyzing chunk frames = [896000,896512) | 20260506 11:11:31.047 | INFO | MemInfo: { | 20260506 11:11:31.047 | INFO | "gpu" : " 17 GB", | 20260506 11:11:31.047 | INFO | "huge" : "", | 20260506 11:11:31.047 | INFO | "huge_shared" : " 27 GB", | 20260506 11:11:31.047 | INFO | "malloc" : " 14 GB", | 20260506 11:11:31.047 | INFO | "pinned" : " 29 GB" | 20260506 11:11:31.047 | INFO | } | 20260506 11:11:31.047 | INFO | Perf report for the last 5.13951 seconds: | 20260506 11:11:31.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:11:31.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:31.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:31.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:31.047 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 11:11:31.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:31.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:31.047 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:11:31.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:31.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:31.047 | INFO | Uploaded 36GiB to GPU in 1.6066s (22.4076GiB/s) | 20260506 11:11:31.048 | INFO | Downloaded 34GiB to GPU in 1.5241s (22.3082GiB/s) | 20260506 11:11:36.186 | INFO | Analyzing chunk frames = [896512,897024) | 20260506 11:11:36.186 | INFO | MemInfo: { | 20260506 11:11:36.186 | INFO | "gpu" : " 17 GB", | 20260506 11:11:36.186 | INFO | "huge" : "", | 20260506 11:11:36.186 | INFO | "huge_shared" : " 27 GB", | 20260506 11:11:36.186 | INFO | "malloc" : " 14 GB", | 20260506 11:11:36.186 | INFO | "pinned" : " 29 GB" | 20260506 11:11:36.186 | INFO | } | 20260506 11:11:36.186 | INFO | Perf report for the last 5.13922 seconds: | 20260506 11:11:36.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:11:36.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:36.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:36.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:36.186 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 11:11:36.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:36.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:36.186 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 11:11:36.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:36.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:36.187 | INFO | Uploaded 36GiB to GPU in 1.60682s (22.4046GiB/s) | 20260506 11:11:36.187 | INFO | Downloaded 34GiB to GPU in 1.52517s (22.2926GiB/s) | 20260506 11:11:36.284 | INFO | Starting a round of Baz IO | 20260506 11:11:39.186 | INFO | Populating RTMetrics report | 20260506 11:11:39.187 | INFO | RTMetrics report populated | 20260506 11:11:39.187 | INFO | Opening tmp RTMetrics json file | 20260506 11:11:39.238 | INFO | Serializing RTMetrics to json | 20260506 11:11:39.291 | INFO | Serialization done, swapping tmp file now... | 20260506 11:11:39.294 | INFO | RTMetrics done | 20260506 11:11:39.615 | INFO | Finished a round of Baz IO: Wrote 2.043 GiB in 3.331 seconds | 20260506 11:11:39.615 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.99% / 0.04% / 54.98% | 20260506 11:11:39.615 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.5481% / 5.5631e-05% / 5.5631e-05% | 20260506 11:11:39.615 | INFO | Baz IO overall: | 20260506 11:11:39.615 | INFO | BytesWritten : 424385268275 | 20260506 11:11:39.615 | INFO | NumEvents : 204599422946 | 20260506 11:11:41.325 | INFO | Analyzing chunk frames = [897024,897536) | 20260506 11:11:41.325 | INFO | MemInfo: { | 20260506 11:11:41.325 | INFO | "gpu" : " 17 GB", | 20260506 11:11:41.325 | INFO | "huge" : "", | 20260506 11:11:41.325 | INFO | "huge_shared" : " 27 GB", | 20260506 11:11:41.325 | INFO | "malloc" : " 14 GB", | 20260506 11:11:41.325 | INFO | "pinned" : " 29 GB" | 20260506 11:11:41.326 | INFO | } | 20260506 11:11:41.326 | INFO | Perf report for the last 5.13922 seconds: | 20260506 11:11:41.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:11:41.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:41.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:41.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:41.326 | INFO | ANALYSIS : 57.4%, 2.9, 85.8ms, 0 | 20260506 11:11:41.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:41.326 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260506 11:11:41.326 | INFO | PRE_HQ : 56.4%, 2.0, 59.9ms, 0 | 20260506 11:11:41.326 | INFO | BAZWRITER : 64.4%, 4.7, 155.6ms, 0 | 20260506 11:11:41.326 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 11:11:41.326 | INFO | Uploaded 36GiB to GPU in 1.67897s (21.4418GiB/s) | 20260506 11:11:41.326 | INFO | Downloaded 38GiB to GPU in 1.74323s (21.7986GiB/s) | 20260506 11:11:46.465 | INFO | Analyzing chunk frames = [897536,898048) | 20260506 11:11:46.465 | INFO | MemInfo: { | 20260506 11:11:46.465 | INFO | "gpu" : " 17 GB", | 20260506 11:11:46.465 | INFO | "huge" : "", | 20260506 11:11:46.465 | INFO | "huge_shared" : " 27 GB", | 20260506 11:11:46.465 | INFO | "malloc" : " 14 GB", | 20260506 11:11:46.465 | INFO | "pinned" : " 29 GB" | 20260506 11:11:46.465 | INFO | } | 20260506 11:11:46.465 | INFO | Perf report for the last 5.13959 seconds: | 20260506 11:11:46.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:11:46.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:46.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:46.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:46.465 | INFO | ANALYSIS : 57.2%, 3.0, 89.2ms, 0 | 20260506 11:11:46.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:46.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:46.465 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:11:46.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:46.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:46.465 | INFO | Uploaded 36GiB to GPU in 1.60451s (22.4367GiB/s) | 20260506 11:11:46.466 | INFO | Downloaded 34GiB to GPU in 1.52296s (22.3249GiB/s) | 20260506 11:11:51.604 | INFO | Analyzing chunk frames = [898048,898560) | 20260506 11:11:51.604 | INFO | MemInfo: { | 20260506 11:11:51.604 | INFO | "gpu" : " 17 GB", | 20260506 11:11:51.604 | INFO | "huge" : "", | 20260506 11:11:51.604 | INFO | "huge_shared" : " 27 GB", | 20260506 11:11:51.604 | INFO | "malloc" : " 14 GB", | 20260506 11:11:51.604 | INFO | "pinned" : " 29 GB" | 20260506 11:11:51.604 | INFO | } | 20260506 11:11:51.604 | INFO | Perf report for the last 5.13924 seconds: | 20260506 11:11:51.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:11:51.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:51.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:51.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:51.605 | INFO | ANALYSIS : 56.6%, 3.0, 87.9ms, 0 | 20260506 11:11:51.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:51.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:51.605 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:11:51.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:51.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:51.605 | INFO | Uploaded 36GiB to GPU in 1.60788s (22.3897GiB/s) | 20260506 11:11:51.605 | INFO | Downloaded 34GiB to GPU in 1.52558s (22.2867GiB/s) | 20260506 11:11:56.744 | INFO | Analyzing chunk frames = [898560,899072) | 20260506 11:11:56.744 | INFO | MemInfo: { | 20260506 11:11:56.744 | INFO | "gpu" : " 17 GB", | 20260506 11:11:56.744 | INFO | "huge" : "", | 20260506 11:11:56.744 | INFO | "huge_shared" : " 27 GB", | 20260506 11:11:56.744 | INFO | "malloc" : " 14 GB", | 20260506 11:11:56.745 | INFO | "pinned" : " 29 GB" | 20260506 11:11:56.745 | INFO | } | 20260506 11:11:56.745 | INFO | Perf report for the last 5.14023 seconds: | 20260506 11:11:56.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:11:56.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:56.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:56.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:56.745 | INFO | ANALYSIS : 56.3%, 3.0, 87.2ms, 0 | 20260506 11:11:56.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:56.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:56.745 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:11:56.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:56.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:11:56.745 | INFO | Uploaded 36GiB to GPU in 1.60776s (22.3914GiB/s) | 20260506 11:11:56.745 | INFO | Downloaded 34GiB to GPU in 1.52619s (22.2777GiB/s) | 20260506 11:12:01.883 | INFO | Analyzing chunk frames = [899072,899584) | 20260506 11:12:01.883 | INFO | MemInfo: { | 20260506 11:12:01.883 | INFO | "gpu" : " 17 GB", | 20260506 11:12:01.883 | INFO | "huge" : "", | 20260506 11:12:01.883 | INFO | "huge_shared" : " 27 GB", | 20260506 11:12:01.883 | INFO | "malloc" : " 14 GB", | 20260506 11:12:01.883 | INFO | "pinned" : " 29 GB" | 20260506 11:12:01.883 | INFO | } | 20260506 11:12:01.883 | INFO | Perf report for the last 5.13836 seconds: | 20260506 11:12:01.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:12:01.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:01.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:01.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:01.883 | INFO | ANALYSIS : 56.2%, 3.0, 87.7ms, 0 | 20260506 11:12:01.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:01.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:01.883 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 11:12:01.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:01.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:01.883 | INFO | Uploaded 36GiB to GPU in 1.60732s (22.3976GiB/s) | 20260506 11:12:01.883 | INFO | Downloaded 34GiB to GPU in 1.52483s (22.2975GiB/s) | 20260506 11:12:07.022 | INFO | Analyzing chunk frames = [899584,900096) | 20260506 11:12:07.022 | INFO | MemInfo: { | 20260506 11:12:07.022 | INFO | "gpu" : " 17 GB", | 20260506 11:12:07.022 | INFO | "huge" : "", | 20260506 11:12:07.022 | INFO | "huge_shared" : " 27 GB", | 20260506 11:12:07.022 | INFO | "malloc" : " 14 GB", | 20260506 11:12:07.022 | INFO | "pinned" : " 29 GB" | 20260506 11:12:07.022 | INFO | } | 20260506 11:12:07.022 | INFO | Perf report for the last 5.13922 seconds: | 20260506 11:12:07.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:12:07.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:07.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:07.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:07.022 | INFO | ANALYSIS : 56.4%, 3.0, 88.0ms, 0 | 20260506 11:12:07.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:07.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:07.022 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:12:07.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:07.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:07.023 | INFO | Uploaded 36GiB to GPU in 1.60657s (22.4079GiB/s) | 20260506 11:12:07.023 | INFO | Downloaded 34GiB to GPU in 1.52556s (22.2869GiB/s) | 20260506 11:12:12.161 | INFO | Analyzing chunk frames = [900096,900608) | 20260506 11:12:12.161 | INFO | MemInfo: { | 20260506 11:12:12.161 | INFO | "gpu" : " 17 GB", | 20260506 11:12:12.161 | INFO | "huge" : "", | 20260506 11:12:12.162 | INFO | "huge_shared" : " 27 GB", | 20260506 11:12:12.162 | INFO | "malloc" : " 14 GB", | 20260506 11:12:12.162 | INFO | "pinned" : " 29 GB" | 20260506 11:12:12.162 | INFO | } | 20260506 11:12:12.162 | INFO | Perf report for the last 5.13948 seconds: | 20260506 11:12:12.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:12:12.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:12.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:12.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:12.162 | INFO | ANALYSIS : 56.7%, 3.0, 88.5ms, 0 | 20260506 11:12:12.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:12.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:12.162 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:12:12.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:12.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:12.162 | INFO | Uploaded 36GiB to GPU in 1.60472s (22.4339GiB/s) | 20260506 11:12:12.162 | INFO | Downloaded 34GiB to GPU in 1.52467s (22.3GiB/s) | 20260506 11:12:17.301 | INFO | Analyzing chunk frames = [900608,901120) | 20260506 11:12:17.301 | INFO | MemInfo: { | 20260506 11:12:17.301 | INFO | "gpu" : " 17 GB", | 20260506 11:12:17.301 | INFO | "huge" : "", | 20260506 11:12:17.301 | INFO | "huge_shared" : " 27 GB", | 20260506 11:12:17.301 | INFO | "malloc" : " 14 GB", | 20260506 11:12:17.301 | INFO | "pinned" : " 29 GB" | 20260506 11:12:17.301 | INFO | } | 20260506 11:12:17.301 | INFO | Perf report for the last 5.13925 seconds: | 20260506 11:12:17.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:12:17.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:17.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:17.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:17.301 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 11:12:17.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:17.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:17.301 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 11:12:17.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:17.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:17.301 | INFO | Uploaded 36GiB to GPU in 1.60727s (22.3982GiB/s) | 20260506 11:12:17.301 | INFO | Downloaded 34GiB to GPU in 1.52597s (22.2808GiB/s) | 20260506 11:12:17.302 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:12:17.302 | INFO | Total mem usage: 16.8 GiB | 20260506 11:12:17.302 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:12:17.302 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -5.59%) | 20260506 11:12:17.302 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:12:17.302 | INFO | High Water Breakdown: | 20260506 11:12:17.302 | INFO | 60.9% : 9.69 GiB ( 93.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:12:17.302 | INFO | 39% : 6.21 GiB ( 92.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:12:17.302 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:12:17.302 | INFO | ------------------------------------------ | 20260506 11:12:17.302 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:12:17.302 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:12:17.302 | INFO | Total mem usage: 26.5 GiB | 20260506 11:12:17.302 | INFO | All Time High Water Marks sum: 27.1 GiB (Savings: 2.04%) | 20260506 11:12:17.302 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:12:17.302 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 11:12:17.302 | INFO | High Water Breakdown: | 20260506 11:12:17.302 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:12:17.302 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:12:17.302 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:12:17.302 | INFO | 1.61% : 420 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:12:17.302 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:12:17.302 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:12:17.302 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:12:17.302 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:12:17.302 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:12:17.302 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:12:17.302 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:12:17.302 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:12:17.302 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:12:17.302 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:12:17.302 | INFO | ------------------------------------------ | 20260506 11:12:17.302 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:12:17.302 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:12:17.302 | INFO | Total mem usage: 25.6 GiB | 20260506 11:12:17.302 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:12:17.302 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 11:12:17.302 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 11:12:17.302 | INFO | High Water Breakdown: | 20260506 11:12:17.302 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 11:12:17.302 | INFO | ------------------------------------------ | 20260506 11:12:17.302 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:12:17.302 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:12:17.302 | INFO | Total mem usage: 15.7 GiB | 20260506 11:12:17.302 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:12:17.302 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:12:17.302 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:12:17.302 | INFO | High Water Breakdown: | 20260506 11:12:17.302 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:12:17.302 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:12:17.302 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:12:17.302 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:12:17.302 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:12:17.302 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:12:17.302 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:12:17.302 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:12:17.302 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:12:17.302 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:12:17.302 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:12:17.302 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:12:17.302 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:12:17.302 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:12:17.302 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:12:17.302 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:12:17.303 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:12:17.303 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:12:17.303 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:12:17.303 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:12:17.303 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:12:17.303 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:12:17.303 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:12:17.303 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:12:17.303 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:12:17.303 | INFO | ------------------------------------------ | 20260506 11:12:17.303 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:12:17.303 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:12:17.303 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:12:17.303 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:12:17.303 | INFO | PerfReport: Full Execution -- 3.00e+01 / -nan, 1.24e+01 / 0.00e+00 | 20260506 11:12:17.303 | INFO | PerfReport: 0.29%, AggregateStats -- 8.67e-02 / -nan, 5.05e-03 / 0.00e+00 | 20260506 11:12:17.303 | INFO | PerfReport: 10.98%, Baseline -- 3.30e+00 / -nan, 1.70e-02 / 0.00e+00 | 20260506 11:12:17.303 | INFO | PerfReport: 3.07%, Histogram -- 9.21e-01 / -nan, 5.14e-02 / 0.00e+00 | 20260506 11:12:17.303 | INFO | PerfReport: 16.07%, DME -- 3.86e+01 / -nan, 3.05e+00 / 0.00e+00 | 20260506 11:12:17.303 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.72e+01 / -nan, 4.21e-02 / 0.00e+00 | 20260506 11:12:17.303 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.20e+00 / -nan, 1.24e-01 / 0.00e+00 | 20260506 11:12:17.303 | INFO | PerfReport: 3.59%, Metrics -- 1.08e+00 / -nan, 1.68e-01 / 0.00e+00 | 20260506 11:12:17.303 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.14e-01 / -nan, 5.80e-02 / 0.00e+00 | 20260506 11:12:17.303 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:12:17.303 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:12:17.303 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:12:17.303 | INFO | PerfReport: Full Execution -- 8.97e+01 / -nan, 1.86e+01 / 0.00e+00 | 20260506 11:12:17.303 | INFO | PerfReport: 18.77%, Upload -- 1.68e+01 / 0.00e+00, 5.14e-01 / 0.00e+00 | 20260506 11:12:17.303 | INFO | PerfReport: 18.03%, Download -- 1.62e+01 / 0.00e+00, 8.68e-01 / 0.00e+00 | 20260506 11:12:17.303 | INFO | PerfReport: 33.47%, Compute -- 3.00e+01 / 0.00e+00, 1.24e+01 / 0.00e+00 | 20260506 11:12:17.303 | INFO | PerfReport: 0.57%, LockUpload -- 5.08e-01 / 0.00e+00, 3.61e+00 / 0.00e+00 | 20260506 11:12:17.303 | INFO | PerfReport: 0.00%, LockDownload -- 4.87e-04 / 0.00e+00, 1.61e-04 / 0.00e+00 | 20260506 11:12:17.303 | INFO | PerfReport: 29.14%, LockCompute -- 2.61e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 11:12:17.303 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.13e-02 / 0.00e+00 | 20260506 11:12:17.402 | INFO | Starting a round of Baz IO | 20260506 11:12:20.317 | INFO | Populating RTMetrics report | 20260506 11:12:20.317 | INFO | RTMetrics report populated | 20260506 11:12:20.317 | INFO | Opening tmp RTMetrics json file | 20260506 11:12:20.325 | INFO | Serializing RTMetrics to json | 20260506 11:12:20.383 | INFO | Serialization done, swapping tmp file now... | 20260506 11:12:20.449 | INFO | RTMetrics done | 20260506 11:12:20.693 | INFO | Finished a round of Baz IO: Wrote 2.044 GiB in 3.291 seconds | 20260506 11:12:20.693 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.95% / 0.04% / 55.01% | 20260506 11:12:20.694 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.5244% / 7.54992e-05% / 6.75519e-05% | 20260506 11:12:20.694 | INFO | Baz IO overall: | 20260506 11:12:20.694 | INFO | BytesWritten : 426579566464 | 20260506 11:12:20.694 | INFO | NumEvents : 205664864293 | 20260506 11:12:22.440 | INFO | Analyzing chunk frames = [901120,901632) | 20260506 11:12:22.440 | INFO | MemInfo: { | 20260506 11:12:22.440 | INFO | "gpu" : " 17 GB", | 20260506 11:12:22.440 | INFO | "huge" : "", | 20260506 11:12:22.440 | INFO | "huge_shared" : " 27 GB", | 20260506 11:12:22.440 | INFO | "malloc" : " 14 GB", | 20260506 11:12:22.440 | INFO | "pinned" : " 29 GB" | 20260506 11:12:22.441 | INFO | } | 20260506 11:12:22.441 | INFO | Perf report for the last 5.13953 seconds: | 20260506 11:12:22.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:12:22.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:22.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:22.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:22.441 | INFO | ANALYSIS : 57.6%, 2.8, 85.2ms, 0 | 20260506 11:12:22.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:22.441 | INFO | RT_METRICS : 47.4%, 1.0, 24.9ms, 0 | 20260506 11:12:22.441 | INFO | PRE_HQ : 56.8%, 2.1, 61.6ms, 0 | 20260506 11:12:22.441 | INFO | BAZWRITER : 63.5%, 4.8, 155.0ms, 0 | 20260506 11:12:22.441 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 11:12:22.441 | INFO | Uploaded 36GiB to GPU in 1.82411s (19.7357GiB/s) | 20260506 11:12:22.441 | INFO | Downloaded 38GiB to GPU in 1.75067s (21.706GiB/s) | 20260506 11:12:27.580 | INFO | Analyzing chunk frames = [901632,902144) | 20260506 11:12:27.580 | INFO | MemInfo: { | 20260506 11:12:27.580 | INFO | "gpu" : " 17 GB", | 20260506 11:12:27.580 | INFO | "huge" : "", | 20260506 11:12:27.580 | INFO | "huge_shared" : " 27 GB", | 20260506 11:12:27.580 | INFO | "malloc" : " 14 GB", | 20260506 11:12:27.580 | INFO | "pinned" : " 29 GB" | 20260506 11:12:27.580 | INFO | } | 20260506 11:12:27.580 | INFO | Perf report for the last 5.13911 seconds: | 20260506 11:12:27.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:12:27.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:27.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:27.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:27.580 | INFO | ANALYSIS : 57.1%, 3.0, 89.1ms, 0 | 20260506 11:12:27.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:27.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:27.580 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:12:27.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:27.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:27.580 | INFO | Uploaded 36GiB to GPU in 1.60436s (22.4389GiB/s) | 20260506 11:12:27.580 | INFO | Downloaded 34GiB to GPU in 1.52261s (22.3301GiB/s) | 20260506 11:12:32.719 | INFO | Analyzing chunk frames = [902144,902656) | 20260506 11:12:32.719 | INFO | MemInfo: { | 20260506 11:12:32.719 | INFO | "gpu" : " 17 GB", | 20260506 11:12:32.719 | INFO | "huge" : "", | 20260506 11:12:32.719 | INFO | "huge_shared" : " 27 GB", | 20260506 11:12:32.719 | INFO | "malloc" : " 14 GB", | 20260506 11:12:32.719 | INFO | "pinned" : " 29 GB" | 20260506 11:12:32.719 | INFO | } | 20260506 11:12:32.719 | INFO | Perf report for the last 5.13931 seconds: | 20260506 11:12:32.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:12:32.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:32.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:32.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:32.719 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260506 11:12:32.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:32.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:32.719 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 11:12:32.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:32.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:32.720 | INFO | Uploaded 36GiB to GPU in 1.63237s (22.0538GiB/s) | 20260506 11:12:32.720 | INFO | Downloaded 34GiB to GPU in 1.60113s (21.235GiB/s) | 20260506 11:12:37.858 | INFO | Analyzing chunk frames = [902656,903168) | 20260506 11:12:37.858 | INFO | MemInfo: { | 20260506 11:12:37.858 | INFO | "gpu" : " 17 GB", | 20260506 11:12:37.858 | INFO | "huge" : "", | 20260506 11:12:37.858 | INFO | "huge_shared" : " 27 GB", | 20260506 11:12:37.858 | INFO | "malloc" : " 14 GB", | 20260506 11:12:37.858 | INFO | "pinned" : " 29 GB" | 20260506 11:12:37.858 | INFO | } | 20260506 11:12:37.859 | INFO | Perf report for the last 5.13946 seconds: | 20260506 11:12:37.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:12:37.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:37.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:37.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:37.859 | INFO | ANALYSIS : 56.4%, 3.0, 87.4ms, 0 | 20260506 11:12:37.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:37.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:37.859 | INFO | PRE_HQ : 34.0%, 1.0, 17.7ms, 0 | 20260506 11:12:37.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:37.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:37.859 | INFO | Uploaded 36GiB to GPU in 1.63745s (21.9854GiB/s) | 20260506 11:12:37.859 | INFO | Downloaded 34GiB to GPU in 1.65153s (20.5869GiB/s) | 20260506 11:12:42.997 | INFO | Analyzing chunk frames = [903168,903680) | 20260506 11:12:42.998 | INFO | MemInfo: { | 20260506 11:12:42.998 | INFO | "gpu" : " 17 GB", | 20260506 11:12:42.998 | INFO | "huge" : "", | 20260506 11:12:42.998 | INFO | "huge_shared" : " 27 GB", | 20260506 11:12:42.998 | INFO | "malloc" : " 14 GB", | 20260506 11:12:42.998 | INFO | "pinned" : " 29 GB" | 20260506 11:12:42.998 | INFO | } | 20260506 11:12:42.998 | INFO | Perf report for the last 5.13926 seconds: | 20260506 11:12:42.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:12:42.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:42.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:42.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:42.998 | INFO | ANALYSIS : 56.3%, 3.0, 87.8ms, 0 | 20260506 11:12:42.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:42.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:42.998 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260506 11:12:42.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:42.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:42.999 | INFO | Uploaded 36GiB to GPU in 1.63231s (22.0546GiB/s) | 20260506 11:12:42.999 | INFO | Downloaded 34GiB to GPU in 1.62817s (20.8824GiB/s) | 20260506 11:12:48.137 | INFO | Analyzing chunk frames = [903680,904192) | 20260506 11:12:48.137 | INFO | MemInfo: { | 20260506 11:12:48.137 | INFO | "gpu" : " 17 GB", | 20260506 11:12:48.137 | INFO | "huge" : "", | 20260506 11:12:48.137 | INFO | "huge_shared" : " 27 GB", | 20260506 11:12:48.137 | INFO | "malloc" : " 14 GB", | 20260506 11:12:48.137 | INFO | "pinned" : " 29 GB" | 20260506 11:12:48.137 | INFO | } | 20260506 11:12:48.137 | INFO | Perf report for the last 5.13934 seconds: | 20260506 11:12:48.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:12:48.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:48.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:48.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:48.137 | INFO | ANALYSIS : 56.5%, 3.0, 88.1ms, 0 | 20260506 11:12:48.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:48.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:48.137 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 11:12:48.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:48.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:48.138 | INFO | Uploaded 36GiB to GPU in 1.6359s (22.0062GiB/s) | 20260506 11:12:48.138 | INFO | Downloaded 34GiB to GPU in 1.64298s (20.6941GiB/s) | 20260506 11:12:53.276 | INFO | Analyzing chunk frames = [904192,904704) | 20260506 11:12:53.276 | INFO | MemInfo: { | 20260506 11:12:53.276 | INFO | "gpu" : " 17 GB", | 20260506 11:12:53.276 | INFO | "huge" : "", | 20260506 11:12:53.276 | INFO | "huge_shared" : " 27 GB", | 20260506 11:12:53.276 | INFO | "malloc" : " 14 GB", | 20260506 11:12:53.276 | INFO | "pinned" : " 29 GB" | 20260506 11:12:53.276 | INFO | } | 20260506 11:12:53.276 | INFO | Perf report for the last 5.13932 seconds: | 20260506 11:12:53.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:12:53.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:53.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:53.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:53.277 | INFO | ANALYSIS : 56.8%, 3.0, 88.5ms, 0 | 20260506 11:12:53.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:53.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:53.277 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260506 11:12:53.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:53.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:53.277 | INFO | Uploaded 36GiB to GPU in 1.60903s (22.3737GiB/s) | 20260506 11:12:53.277 | INFO | Downloaded 34GiB to GPU in 1.52723s (22.2626GiB/s) | 20260506 11:12:58.416 | INFO | Analyzing chunk frames = [904704,905216) | 20260506 11:12:58.416 | INFO | MemInfo: { | 20260506 11:12:58.416 | INFO | "gpu" : " 17 GB", | 20260506 11:12:58.416 | INFO | "huge" : "", | 20260506 11:12:58.416 | INFO | "huge_shared" : " 27 GB", | 20260506 11:12:58.416 | INFO | "malloc" : " 14 GB", | 20260506 11:12:58.416 | INFO | "pinned" : " 29 GB" | 20260506 11:12:58.416 | INFO | } | 20260506 11:12:58.416 | INFO | Perf report for the last 5.1395 seconds: | 20260506 11:12:58.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:12:58.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:58.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:58.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:58.416 | INFO | ANALYSIS : 56.8%, 3.0, 88.6ms, 0 | 20260506 11:12:58.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:58.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:58.416 | INFO | PRE_HQ : 34.2%, 1.0, 17.8ms, 0 | 20260506 11:12:58.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:58.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:12:58.417 | INFO | Uploaded 36GiB to GPU in 1.60979s (22.3632GiB/s) | 20260506 11:12:58.417 | INFO | Downloaded 34GiB to GPU in 1.52702s (22.2656GiB/s) | 20260506 11:12:58.516 | INFO | Starting a round of Baz IO | 20260506 11:13:01.547 | INFO | Populating RTMetrics report | 20260506 11:13:01.548 | INFO | RTMetrics report populated | 20260506 11:13:01.548 | INFO | Opening tmp RTMetrics json file | 20260506 11:13:01.573 | INFO | Serializing RTMetrics to json | 20260506 11:13:01.577 | INFO | Serialization done, swapping tmp file now... | 20260506 11:13:01.580 | INFO | RTMetrics done | 20260506 11:13:01.911 | INFO | Finished a round of Baz IO: Wrote 2.046 GiB in 3.395 seconds | 20260506 11:13:01.911 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.91% / 0.04% / 55.05% | 20260506 11:13:01.911 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.52781% / 7.54992e-05% / 4.37101e-05% | 20260506 11:13:01.912 | INFO | Baz IO overall: | 20260506 11:13:01.912 | INFO | BytesWritten : 428776171108 | 20260506 11:13:01.912 | INFO | NumEvents : 206731673970 | 20260506 11:13:03.555 | INFO | Analyzing chunk frames = [905216,905728) | 20260506 11:13:03.555 | INFO | MemInfo: { | 20260506 11:13:03.555 | INFO | "gpu" : " 17 GB", | 20260506 11:13:03.555 | INFO | "huge" : "", | 20260506 11:13:03.555 | INFO | "huge_shared" : " 27 GB", | 20260506 11:13:03.555 | INFO | "malloc" : " 14 GB", | 20260506 11:13:03.555 | INFO | "pinned" : " 29 GB" | 20260506 11:13:03.555 | INFO | } | 20260506 11:13:03.555 | INFO | Perf report for the last 5.13899 seconds: | 20260506 11:13:03.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:13:03.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:03.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:03.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:03.555 | INFO | ANALYSIS : 59.7%, 2.7, 83.5ms, 0 | 20260506 11:13:03.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:03.555 | INFO | RT_METRICS : 49.1%, 1.0, 25.7ms, 0 | 20260506 11:13:03.555 | INFO | PRE_HQ : 58.7%, 2.2, 68.0ms, 0 | 20260506 11:13:03.555 | INFO | BAZWRITER : 65.7%, 4.8, 161.7ms, 0 | 20260506 11:13:03.555 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 11:13:03.555 | INFO | Uploaded 36GiB to GPU in 1.66585s (21.6106GiB/s) | 20260506 11:13:03.555 | INFO | Downloaded 38GiB to GPU in 1.7873s (21.2612GiB/s) | 20260506 11:13:08.694 | INFO | Analyzing chunk frames = [905728,906240) | 20260506 11:13:08.694 | INFO | MemInfo: { | 20260506 11:13:08.694 | INFO | "gpu" : " 17 GB", | 20260506 11:13:08.694 | INFO | "huge" : "", | 20260506 11:13:08.694 | INFO | "huge_shared" : " 27 GB", | 20260506 11:13:08.694 | INFO | "malloc" : " 14 GB", | 20260506 11:13:08.694 | INFO | "pinned" : " 29 GB" | 20260506 11:13:08.694 | INFO | } | 20260506 11:13:08.694 | INFO | Perf report for the last 5.13946 seconds: | 20260506 11:13:08.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:13:08.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:08.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:08.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:08.695 | INFO | ANALYSIS : 57.2%, 3.0, 89.3ms, 0 | 20260506 11:13:08.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:08.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:08.695 | INFO | PRE_HQ : 33.4%, 1.0, 17.3ms, 0 | 20260506 11:13:08.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:08.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:08.695 | INFO | Uploaded 36GiB to GPU in 1.61548s (22.2844GiB/s) | 20260506 11:13:08.695 | INFO | Downloaded 34GiB to GPU in 1.54968s (21.94GiB/s) | 20260506 11:13:13.834 | INFO | Analyzing chunk frames = [906240,906752) | 20260506 11:13:13.834 | INFO | MemInfo: { | 20260506 11:13:13.834 | INFO | "gpu" : " 17 GB", | 20260506 11:13:13.834 | INFO | "huge" : "", | 20260506 11:13:13.834 | INFO | "huge_shared" : " 27 GB", | 20260506 11:13:13.834 | INFO | "malloc" : " 14 GB", | 20260506 11:13:13.834 | INFO | "pinned" : " 29 GB" | 20260506 11:13:13.834 | INFO | } | 20260506 11:13:13.834 | INFO | Perf report for the last 5.13953 seconds: | 20260506 11:13:13.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:13:13.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:13.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:13.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:13.834 | INFO | ANALYSIS : 56.6%, 3.0, 88.0ms, 0 | 20260506 11:13:13.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:13.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:13.834 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260506 11:13:13.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:13.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:13.834 | INFO | Uploaded 36GiB to GPU in 1.62565s (22.145GiB/s) | 20260506 11:13:13.834 | INFO | Downloaded 34GiB to GPU in 1.56219s (21.7643GiB/s) | 20260506 11:13:18.973 | INFO | Analyzing chunk frames = [906752,907264) | 20260506 11:13:18.973 | INFO | MemInfo: { | 20260506 11:13:18.973 | INFO | "gpu" : " 17 GB", | 20260506 11:13:18.973 | INFO | "huge" : "", | 20260506 11:13:18.973 | INFO | "huge_shared" : " 27 GB", | 20260506 11:13:18.973 | INFO | "malloc" : " 14 GB", | 20260506 11:13:18.973 | INFO | "pinned" : " 29 GB" | 20260506 11:13:18.973 | INFO | } | 20260506 11:13:18.973 | INFO | Perf report for the last 5.13911 seconds: | 20260506 11:13:18.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:13:18.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:18.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:18.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:18.973 | INFO | ANALYSIS : 56.3%, 3.0, 87.3ms, 0 | 20260506 11:13:18.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:18.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:18.973 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260506 11:13:18.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:18.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:18.973 | INFO | Uploaded 36GiB to GPU in 1.62252s (22.1877GiB/s) | 20260506 11:13:18.974 | INFO | Downloaded 34GiB to GPU in 1.55567s (21.8555GiB/s) | 20260506 11:13:24.112 | INFO | Analyzing chunk frames = [907264,907776) | 20260506 11:13:24.112 | INFO | MemInfo: { | 20260506 11:13:24.112 | INFO | "gpu" : " 17 GB", | 20260506 11:13:24.112 | INFO | "huge" : "", | 20260506 11:13:24.112 | INFO | "huge_shared" : " 27 GB", | 20260506 11:13:24.112 | INFO | "malloc" : " 14 GB", | 20260506 11:13:24.112 | INFO | "pinned" : " 29 GB" | 20260506 11:13:24.112 | INFO | } | 20260506 11:13:24.113 | INFO | Perf report for the last 5.13945 seconds: | 20260506 11:13:24.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:13:24.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:24.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:24.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:24.113 | INFO | ANALYSIS : 56.2%, 3.0, 87.6ms, 0 | 20260506 11:13:24.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:24.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:24.113 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260506 11:13:24.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:24.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:24.113 | INFO | Uploaded 36GiB to GPU in 1.62314s (22.1792GiB/s) | 20260506 11:13:24.113 | INFO | Downloaded 34GiB to GPU in 1.55877s (21.8121GiB/s) | 20260506 11:13:29.252 | INFO | Analyzing chunk frames = [907776,908288) | 20260506 11:13:29.252 | INFO | MemInfo: { | 20260506 11:13:29.252 | INFO | "gpu" : " 17 GB", | 20260506 11:13:29.252 | INFO | "huge" : "", | 20260506 11:13:29.252 | INFO | "huge_shared" : " 27 GB", | 20260506 11:13:29.252 | INFO | "malloc" : " 14 GB", | 20260506 11:13:29.252 | INFO | "pinned" : " 29 GB" | 20260506 11:13:29.252 | INFO | } | 20260506 11:13:29.252 | INFO | Perf report for the last 5.13961 seconds: | 20260506 11:13:29.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:13:29.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:29.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:29.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:29.252 | INFO | ANALYSIS : 56.5%, 3.0, 88.2ms, 0 | 20260506 11:13:29.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:29.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:29.252 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 11:13:29.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:29.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:29.253 | INFO | Uploaded 36GiB to GPU in 1.60865s (22.379GiB/s) | 20260506 11:13:29.253 | INFO | Downloaded 34GiB to GPU in 1.53007s (22.2211GiB/s) | 20260506 11:13:34.392 | INFO | Analyzing chunk frames = [908288,908800) | 20260506 11:13:34.392 | INFO | MemInfo: { | 20260506 11:13:34.392 | INFO | "gpu" : " 17 GB", | 20260506 11:13:34.392 | INFO | "huge" : "", | 20260506 11:13:34.392 | INFO | "huge_shared" : " 27 GB", | 20260506 11:13:34.392 | INFO | "malloc" : " 14 GB", | 20260506 11:13:34.392 | INFO | "pinned" : " 29 GB" | 20260506 11:13:34.392 | INFO | } | 20260506 11:13:34.392 | INFO | Perf report for the last 5.13955 seconds: | 20260506 11:13:34.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:13:34.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:34.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:34.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:34.392 | INFO | ANALYSIS : 56.7%, 3.0, 88.4ms, 0 | 20260506 11:13:34.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:34.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:34.392 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:13:34.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:34.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:34.392 | INFO | Uploaded 36GiB to GPU in 1.60668s (22.4065GiB/s) | 20260506 11:13:34.392 | INFO | Downloaded 34GiB to GPU in 1.52426s (22.306GiB/s) | 20260506 11:13:39.531 | INFO | Analyzing chunk frames = [908800,909312) | 20260506 11:13:39.531 | INFO | MemInfo: { | 20260506 11:13:39.531 | INFO | "gpu" : " 17 GB", | 20260506 11:13:39.531 | INFO | "huge" : "", | 20260506 11:13:39.531 | INFO | "huge_shared" : " 27 GB", | 20260506 11:13:39.531 | INFO | "malloc" : " 14 GB", | 20260506 11:13:39.531 | INFO | "pinned" : " 29 GB" | 20260506 11:13:39.531 | INFO | } | 20260506 11:13:39.531 | INFO | Perf report for the last 5.13958 seconds: | 20260506 11:13:39.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:13:39.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:39.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:39.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:39.531 | INFO | ANALYSIS : 57.4%, 3.0, 89.3ms, 0 | 20260506 11:13:39.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:39.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:39.532 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:13:39.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:39.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:39.532 | INFO | Uploaded 36GiB to GPU in 1.70205s (21.151GiB/s) | 20260506 11:13:39.532 | INFO | Downloaded 34GiB to GPU in 1.5939s (21.3314GiB/s) | 20260506 11:13:39.532 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:13:39.532 | INFO | Total mem usage: 16.8 GiB | 20260506 11:13:39.532 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:13:39.532 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -10.1%) | 20260506 11:13:39.532 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:13:39.532 | INFO | High Water Breakdown: | 20260506 11:13:39.532 | INFO | 60.2% : 9.19 GiB ( 88.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:13:39.532 | INFO | 39.7% : 6.05 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:13:39.532 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:13:39.532 | INFO | ------------------------------------------ | 20260506 11:13:39.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:13:39.532 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:13:39.532 | INFO | Total mem usage: 26.5 GiB | 20260506 11:13:39.532 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:13:39.532 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 11:13:39.532 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 11:13:39.532 | INFO | High Water Breakdown: | 20260506 11:13:39.532 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:13:39.532 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:13:39.532 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:13:39.532 | INFO | 2% : 523 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:13:39.532 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:13:39.532 | INFO | 0.557% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:13:39.532 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:13:39.532 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:13:39.532 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:13:39.532 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:13:39.532 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:13:39.532 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:13:39.532 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:13:39.532 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:13:39.532 | INFO | ------------------------------------------ | 20260506 11:13:39.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:13:39.532 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:13:39.532 | INFO | Total mem usage: 25.6 GiB | 20260506 11:13:39.532 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:13:39.532 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 11:13:39.532 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 11:13:39.532 | INFO | High Water Breakdown: | 20260506 11:13:39.532 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 11:13:39.532 | INFO | ------------------------------------------ | 20260506 11:13:39.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:13:39.532 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:13:39.532 | INFO | Total mem usage: 15.7 GiB | 20260506 11:13:39.532 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:13:39.532 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:13:39.532 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:13:39.532 | INFO | High Water Breakdown: | 20260506 11:13:39.532 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:13:39.532 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:13:39.532 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:13:39.532 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:13:39.532 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:13:39.532 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:13:39.532 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:13:39.532 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:13:39.532 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:13:39.532 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:13:39.532 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:13:39.532 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:13:39.532 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:13:39.533 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:13:39.533 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:13:39.533 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:13:39.533 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:13:39.533 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:13:39.533 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:13:39.533 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:13:39.533 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:13:39.533 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:13:39.533 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:13:39.533 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:13:39.533 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:13:39.533 | INFO | ------------------------------------------ | 20260506 11:13:39.533 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:13:39.533 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:13:39.533 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:13:39.533 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:13:39.533 | INFO | PerfReport: Full Execution -- 3.01e+01 / 9.52e+01, 1.26e+01 / 0.00e+00 | 20260506 11:13:39.533 | INFO | PerfReport: 0.29%, AggregateStats -- 8.75e-02 / 8.83e-02, 5.71e-03 / 0.00e+00 | 20260506 11:13:39.533 | INFO | PerfReport: 11.13%, Baseline -- 3.35e+00 / 7.28e+01, 1.75e+00 / 0.00e+00 | 20260506 11:13:39.533 | INFO | PerfReport: 3.07%, Histogram -- 9.22e-01 / 9.08e-01, 5.37e-02 / 0.00e+00 | 20260506 11:13:39.533 | INFO | PerfReport: 16.03%, DME -- 3.86e+01 / -nan, 3.06e+00 / 0.00e+00 | 20260506 11:13:39.533 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.72e+01 / 1.73e+01, 4.87e-02 / 0.00e+00 | 20260506 11:13:39.533 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.20e+00 / 2.26e+00, 1.24e-01 / 0.00e+00 | 20260506 11:13:39.533 | INFO | PerfReport: 3.59%, Metrics -- 1.08e+00 / 1.37e+00, 1.69e-01 / 0.00e+00 | 20260506 11:13:39.533 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.16e-01 / 4.64e-01, 5.88e-02 / 0.00e+00 | 20260506 11:13:39.533 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:13:39.533 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:13:39.533 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:13:39.533 | INFO | PerfReport: Full Execution -- 8.96e+01 / -nan, 1.88e+01 / 0.00e+00 | 20260506 11:13:39.533 | INFO | PerfReport: 19.08%, Upload -- 1.71e+01 / 0.00e+00, 1.34e+00 / 0.00e+00 | 20260506 11:13:39.533 | INFO | PerfReport: 18.54%, Download -- 1.66e+01 / 0.00e+00, 2.29e+00 / 0.00e+00 | 20260506 11:13:39.533 | INFO | PerfReport: 33.57%, Compute -- 3.01e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 11:13:39.533 | INFO | PerfReport: 0.60%, LockUpload -- 5.38e-01 / 0.00e+00, 3.67e+00 / 0.00e+00 | 20260506 11:13:39.533 | INFO | PerfReport: 0.00%, LockDownload -- 1.21e-03 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260506 11:13:39.533 | INFO | PerfReport: 28.19%, LockCompute -- 2.53e+01 / 0.00e+00, 1.69e+01 / 0.00e+00 | 20260506 11:13:39.533 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260506 11:13:39.642 | INFO | Starting a round of Baz IO | 20260506 11:13:43.039 | INFO | Populating RTMetrics report | 20260506 11:13:43.039 | INFO | RTMetrics report populated | 20260506 11:13:43.039 | INFO | Opening tmp RTMetrics json file | 20260506 11:13:43.049 | INFO | Serializing RTMetrics to json | 20260506 11:13:43.054 | INFO | Serialization done, swapping tmp file now... | 20260506 11:13:43.059 | INFO | RTMetrics done | 20260506 11:13:43.195 | INFO | Finished a round of Baz IO: Wrote 2.049 GiB in 3.552 seconds | 20260506 11:13:43.195 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.87% / 0.04% / 55.09% | 20260506 11:13:43.195 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.53151% / 6.75519e-05% / 3.97364e-05% | 20260506 11:13:43.195 | INFO | Baz IO overall: | 20260506 11:13:43.195 | INFO | BytesWritten : 430975992062 | 20260506 11:13:43.195 | INFO | NumEvents : 207800509659 | 20260506 11:13:44.670 | INFO | Analyzing chunk frames = [909312,909824) | 20260506 11:13:44.670 | INFO | MemInfo: { | 20260506 11:13:44.670 | INFO | "gpu" : " 17 GB", | 20260506 11:13:44.670 | INFO | "huge" : "", | 20260506 11:13:44.670 | INFO | "huge_shared" : " 27 GB", | 20260506 11:13:44.670 | INFO | "malloc" : " 14 GB", | 20260506 11:13:44.670 | INFO | "pinned" : " 29 GB" | 20260506 11:13:44.670 | INFO | } | 20260506 11:13:44.670 | INFO | Perf report for the last 5.13906 seconds: | 20260506 11:13:44.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:13:44.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:44.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:44.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:44.671 | INFO | ANALYSIS : 67.7%, 2.9, 104.3ms, 0 | 20260506 11:13:44.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:44.671 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 11:13:44.671 | INFO | PRE_HQ : 66.3%, 1.9, 66.2ms, 0 | 20260506 11:13:44.671 | INFO | BAZWRITER : 68.7%, 4.6, 162.1ms, 0 | 20260506 11:13:44.671 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 11:13:44.671 | INFO | Uploaded 36GiB to GPU in 2.84153s (12.6692GiB/s) | 20260506 11:13:44.671 | INFO | Downloaded 38GiB to GPU in 2.57162s (14.7767GiB/s) | 20260506 11:13:49.810 | INFO | Analyzing chunk frames = [909824,910336) | 20260506 11:13:49.810 | INFO | MemInfo: { | 20260506 11:13:49.810 | INFO | "gpu" : " 17 GB", | 20260506 11:13:49.810 | INFO | "huge" : "", | 20260506 11:13:49.810 | INFO | "huge_shared" : " 27 GB", | 20260506 11:13:49.810 | INFO | "malloc" : " 14 GB", | 20260506 11:13:49.810 | INFO | "pinned" : " 29 GB" | 20260506 11:13:49.810 | INFO | } | 20260506 11:13:49.810 | INFO | Perf report for the last 5.13954 seconds: | 20260506 11:13:49.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:13:49.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:49.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:49.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:49.810 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 11:13:49.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:49.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:49.810 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:13:49.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:49.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:49.811 | INFO | Uploaded 36GiB to GPU in 2.06018s (17.4742GiB/s) | 20260506 11:13:49.811 | INFO | Downloaded 34GiB to GPU in 1.83799s (18.4985GiB/s) | 20260506 11:13:54.949 | INFO | Analyzing chunk frames = [910336,910848) | 20260506 11:13:54.949 | INFO | MemInfo: { | 20260506 11:13:54.949 | INFO | "gpu" : " 17 GB", | 20260506 11:13:54.949 | INFO | "huge" : "", | 20260506 11:13:54.949 | INFO | "huge_shared" : " 27 GB", | 20260506 11:13:54.949 | INFO | "malloc" : " 14 GB", | 20260506 11:13:54.949 | INFO | "pinned" : " 29 GB" | 20260506 11:13:54.949 | INFO | } | 20260506 11:13:54.949 | INFO | Perf report for the last 5.13909 seconds: | 20260506 11:13:54.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:13:54.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:54.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:54.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:54.949 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 11:13:54.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:54.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:54.949 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:13:54.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:54.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:13:54.949 | INFO | Uploaded 36GiB to GPU in 2.05641s (17.5063GiB/s) | 20260506 11:13:54.949 | INFO | Downloaded 34GiB to GPU in 1.82353s (18.6452GiB/s) | 20260506 11:14:00.088 | INFO | Analyzing chunk frames = [910848,911360) | 20260506 11:14:00.088 | INFO | MemInfo: { | 20260506 11:14:00.088 | INFO | "gpu" : " 17 GB", | 20260506 11:14:00.088 | INFO | "huge" : "", | 20260506 11:14:00.088 | INFO | "huge_shared" : " 27 GB", | 20260506 11:14:00.088 | INFO | "malloc" : " 14 GB", | 20260506 11:14:00.088 | INFO | "pinned" : " 29 GB" | 20260506 11:14:00.088 | INFO | } | 20260506 11:14:00.088 | INFO | Perf report for the last 5.1392 seconds: | 20260506 11:14:00.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:14:00.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:00.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:00.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:00.088 | INFO | ANALYSIS : 60.3%, 2.9, 93.0ms, 0 | 20260506 11:14:00.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:00.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:00.088 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 11:14:00.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:00.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:00.089 | INFO | Uploaded 36GiB to GPU in 2.04879s (17.5713GiB/s) | 20260506 11:14:00.089 | INFO | Downloaded 34GiB to GPU in 1.7672s (19.2395GiB/s) | 20260506 11:14:05.227 | INFO | Analyzing chunk frames = [911360,911872) | 20260506 11:14:05.227 | INFO | MemInfo: { | 20260506 11:14:05.227 | INFO | "gpu" : " 17 GB", | 20260506 11:14:05.227 | INFO | "huge" : "", | 20260506 11:14:05.227 | INFO | "huge_shared" : " 27 GB", | 20260506 11:14:05.227 | INFO | "malloc" : " 14 GB", | 20260506 11:14:05.227 | INFO | "pinned" : " 29 GB" | 20260506 11:14:05.228 | INFO | } | 20260506 11:14:05.228 | INFO | Perf report for the last 5.13937 seconds: | 20260506 11:14:05.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:14:05.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:05.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:05.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:05.228 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260506 11:14:05.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:05.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:05.228 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 11:14:05.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:05.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:05.228 | INFO | Uploaded 36GiB to GPU in 2.06844s (17.4044GiB/s) | 20260506 11:14:05.228 | INFO | Downloaded 34GiB to GPU in 1.75503s (19.3729GiB/s) | 20260506 11:14:10.367 | INFO | Analyzing chunk frames = [911872,912384) | 20260506 11:14:10.367 | INFO | MemInfo: { | 20260506 11:14:10.367 | INFO | "gpu" : " 17 GB", | 20260506 11:14:10.367 | INFO | "huge" : "", | 20260506 11:14:10.367 | INFO | "huge_shared" : " 27 GB", | 20260506 11:14:10.367 | INFO | "malloc" : " 14 GB", | 20260506 11:14:10.367 | INFO | "pinned" : " 29 GB" | 20260506 11:14:10.367 | INFO | } | 20260506 11:14:10.367 | INFO | Perf report for the last 5.13933 seconds: | 20260506 11:14:10.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:14:10.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:10.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:10.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:10.367 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 11:14:10.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:10.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:10.367 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 11:14:10.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:10.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:10.368 | INFO | Uploaded 36GiB to GPU in 2.08435s (17.2716GiB/s) | 20260506 11:14:10.368 | INFO | Downloaded 34GiB to GPU in 1.79411s (18.9509GiB/s) | 20260506 11:14:15.506 | INFO | Analyzing chunk frames = [912384,912896) | 20260506 11:14:15.506 | INFO | MemInfo: { | 20260506 11:14:15.506 | INFO | "gpu" : " 17 GB", | 20260506 11:14:15.506 | INFO | "huge" : "", | 20260506 11:14:15.506 | INFO | "huge_shared" : " 27 GB", | 20260506 11:14:15.506 | INFO | "malloc" : " 14 GB", | 20260506 11:14:15.506 | INFO | "pinned" : " 29 GB" | 20260506 11:14:15.506 | INFO | } | 20260506 11:14:15.506 | INFO | Perf report for the last 5.1394 seconds: | 20260506 11:14:15.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:14:15.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:15.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:15.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:15.507 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260506 11:14:15.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:15.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:15.507 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 11:14:15.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:15.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:15.507 | INFO | Uploaded 36GiB to GPU in 2.08849s (17.2373GiB/s) | 20260506 11:14:15.507 | INFO | Downloaded 34GiB to GPU in 1.76717s (19.2398GiB/s) | 20260506 11:14:20.646 | INFO | Analyzing chunk frames = [912896,913408) | 20260506 11:14:20.646 | INFO | MemInfo: { | 20260506 11:14:20.646 | INFO | "gpu" : " 17 GB", | 20260506 11:14:20.646 | INFO | "huge" : "", | 20260506 11:14:20.646 | INFO | "huge_shared" : " 27 GB", | 20260506 11:14:20.646 | INFO | "malloc" : " 14 GB", | 20260506 11:14:20.646 | INFO | "pinned" : " 29 GB" | 20260506 11:14:20.646 | INFO | } | 20260506 11:14:20.646 | INFO | Perf report for the last 5.13958 seconds: | 20260506 11:14:20.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:14:20.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:20.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:20.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:20.646 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 11:14:20.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:20.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:20.646 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 11:14:20.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:20.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:20.646 | INFO | Uploaded 36GiB to GPU in 2.08196s (17.2914GiB/s) | 20260506 11:14:20.646 | INFO | Downloaded 34GiB to GPU in 1.75791s (19.3412GiB/s) | 20260506 11:14:20.764 | INFO | Starting a round of Baz IO | 20260506 11:14:23.906 | INFO | Populating RTMetrics report | 20260506 11:14:23.907 | INFO | RTMetrics report populated | 20260506 11:14:23.907 | INFO | Opening tmp RTMetrics json file | 20260506 11:14:23.911 | INFO | Serializing RTMetrics to json | 20260506 11:14:23.915 | INFO | Serialization done, swapping tmp file now... | 20260506 11:14:23.919 | INFO | RTMetrics done | 20260506 11:14:24.376 | INFO | Finished a round of Baz IO: Wrote 2.048 GiB in 3.611 seconds | 20260506 11:14:24.376 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.83% / 0.04% / 55.13% | 20260506 11:14:24.376 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.50001% / 6.35783e-05% / 2.38419e-05% | 20260506 11:14:24.376 | INFO | Baz IO overall: | 20260506 11:14:24.376 | INFO | BytesWritten : 433174828024 | 20260506 11:14:24.376 | INFO | NumEvents : 208868630279 | 20260506 11:14:25.785 | INFO | Analyzing chunk frames = [913408,913920) | 20260506 11:14:25.785 | INFO | MemInfo: { | 20260506 11:14:25.785 | INFO | "gpu" : " 17 GB", | 20260506 11:14:25.785 | INFO | "huge" : "", | 20260506 11:14:25.785 | INFO | "huge_shared" : " 27 GB", | 20260506 11:14:25.785 | INFO | "malloc" : " 14 GB", | 20260506 11:14:25.785 | INFO | "pinned" : " 29 GB" | 20260506 11:14:25.785 | INFO | } | 20260506 11:14:25.785 | INFO | Perf report for the last 5.13905 seconds: | 20260506 11:14:25.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:14:25.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:25.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:25.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:25.785 | INFO | ANALYSIS : 62.9%, 2.9, 94.6ms, 0 | 20260506 11:14:25.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:25.785 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 11:14:25.785 | INFO | PRE_HQ : 61.1%, 2.0, 62.8ms, 0 | 20260506 11:14:25.785 | INFO | BAZWRITER : 70.3%, 4.8, 173.7ms, 0 | 20260506 11:14:25.785 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 11:14:25.786 | INFO | Uploaded 36GiB to GPU in 2.15357s (16.7164GiB/s) | 20260506 11:14:25.786 | INFO | Downloaded 38GiB to GPU in 2.12594s (17.8744GiB/s) | 20260506 11:14:30.925 | INFO | Analyzing chunk frames = [913920,914432) | 20260506 11:14:30.925 | INFO | MemInfo: { | 20260506 11:14:30.925 | INFO | "gpu" : " 17 GB", | 20260506 11:14:30.925 | INFO | "huge" : "", | 20260506 11:14:30.925 | INFO | "huge_shared" : " 27 GB", | 20260506 11:14:30.925 | INFO | "malloc" : " 14 GB", | 20260506 11:14:30.925 | INFO | "pinned" : " 29 GB" | 20260506 11:14:30.925 | INFO | } | 20260506 11:14:30.925 | INFO | Perf report for the last 5.1397 seconds: | 20260506 11:14:30.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:14:30.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:30.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:30.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:30.925 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 11:14:30.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:30.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:30.925 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 11:14:30.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:30.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:30.925 | INFO | Uploaded 36GiB to GPU in 2.06119s (17.4656GiB/s) | 20260506 11:14:30.925 | INFO | Downloaded 34GiB to GPU in 1.83609s (18.5176GiB/s) | 20260506 11:14:36.064 | INFO | Analyzing chunk frames = [914432,914944) | 20260506 11:14:36.064 | INFO | MemInfo: { | 20260506 11:14:36.064 | INFO | "gpu" : " 17 GB", | 20260506 11:14:36.064 | INFO | "huge" : "", | 20260506 11:14:36.064 | INFO | "huge_shared" : " 27 GB", | 20260506 11:14:36.064 | INFO | "malloc" : " 14 GB", | 20260506 11:14:36.064 | INFO | "pinned" : " 29 GB" | 20260506 11:14:36.064 | INFO | } | 20260506 11:14:36.064 | INFO | Perf report for the last 5.1394 seconds: | 20260506 11:14:36.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:14:36.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:36.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:36.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:36.064 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 11:14:36.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:36.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:36.064 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 11:14:36.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:36.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:36.065 | INFO | Uploaded 36GiB to GPU in 2.05823s (17.4908GiB/s) | 20260506 11:14:36.065 | INFO | Downloaded 34GiB to GPU in 1.83675s (18.511GiB/s) | 20260506 11:14:41.203 | INFO | Analyzing chunk frames = [914944,915456) | 20260506 11:14:41.203 | INFO | MemInfo: { | 20260506 11:14:41.203 | INFO | "gpu" : " 17 GB", | 20260506 11:14:41.203 | INFO | "huge" : "", | 20260506 11:14:41.203 | INFO | "huge_shared" : " 27 GB", | 20260506 11:14:41.203 | INFO | "malloc" : " 14 GB", | 20260506 11:14:41.203 | INFO | "pinned" : " 29 GB" | 20260506 11:14:41.203 | INFO | } | 20260506 11:14:41.203 | INFO | Perf report for the last 5.13894 seconds: | 20260506 11:14:41.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:14:41.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:41.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:41.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:41.203 | INFO | ANALYSIS : 60.3%, 2.9, 93.1ms, 0 | 20260506 11:14:41.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:41.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:41.203 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 11:14:41.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:41.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:41.204 | INFO | Uploaded 36GiB to GPU in 2.04965s (17.564GiB/s) | 20260506 11:14:41.204 | INFO | Downloaded 34GiB to GPU in 1.77311s (19.1754GiB/s) | 20260506 11:14:46.343 | INFO | Analyzing chunk frames = [915456,915968) | 20260506 11:14:46.343 | INFO | MemInfo: { | 20260506 11:14:46.343 | INFO | "gpu" : " 17 GB", | 20260506 11:14:46.343 | INFO | "huge" : "", | 20260506 11:14:46.343 | INFO | "huge_shared" : " 27 GB", | 20260506 11:14:46.343 | INFO | "malloc" : " 14 GB", | 20260506 11:14:46.343 | INFO | "pinned" : " 29 GB" | 20260506 11:14:46.343 | INFO | } | 20260506 11:14:46.343 | INFO | Perf report for the last 5.14021 seconds: | 20260506 11:14:46.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:14:46.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:46.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:46.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:46.343 | INFO | ANALYSIS : 60.0%, 3.0, 93.2ms, 0 | 20260506 11:14:46.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:46.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:46.344 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 11:14:46.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:46.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:46.344 | INFO | Uploaded 36GiB to GPU in 2.0673s (17.414GiB/s) | 20260506 11:14:46.344 | INFO | Downloaded 34GiB to GPU in 1.73341s (19.6146GiB/s) | 20260506 11:14:51.482 | INFO | Analyzing chunk frames = [915968,916480) | 20260506 11:14:51.482 | INFO | MemInfo: { | 20260506 11:14:51.482 | INFO | "gpu" : " 17 GB", | 20260506 11:14:51.482 | INFO | "huge" : "", | 20260506 11:14:51.482 | INFO | "huge_shared" : " 27 GB", | 20260506 11:14:51.482 | INFO | "malloc" : " 14 GB", | 20260506 11:14:51.482 | INFO | "pinned" : " 29 GB" | 20260506 11:14:51.482 | INFO | } | 20260506 11:14:51.482 | INFO | Perf report for the last 5.13896 seconds: | 20260506 11:14:51.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:14:51.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:51.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:51.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:51.482 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260506 11:14:51.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:51.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:51.482 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 11:14:51.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:51.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:51.483 | INFO | Uploaded 36GiB to GPU in 2.08201s (17.2909GiB/s) | 20260506 11:14:51.483 | INFO | Downloaded 34GiB to GPU in 1.79493s (18.9423GiB/s) | 20260506 11:14:56.622 | INFO | Analyzing chunk frames = [916480,916992) | 20260506 11:14:56.622 | INFO | MemInfo: { | 20260506 11:14:56.622 | INFO | "gpu" : " 17 GB", | 20260506 11:14:56.622 | INFO | "huge" : "", | 20260506 11:14:56.622 | INFO | "huge_shared" : " 27 GB", | 20260506 11:14:56.622 | INFO | "malloc" : " 14 GB", | 20260506 11:14:56.622 | INFO | "pinned" : " 29 GB" | 20260506 11:14:56.622 | INFO | } | 20260506 11:14:56.622 | INFO | Perf report for the last 5.13965 seconds: | 20260506 11:14:56.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:14:56.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:56.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:56.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:56.622 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260506 11:14:56.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:56.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:56.622 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 11:14:56.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:56.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:14:56.622 | INFO | Uploaded 36GiB to GPU in 2.08806s (17.2409GiB/s) | 20260506 11:14:56.622 | INFO | Downloaded 34GiB to GPU in 1.73701s (19.5739GiB/s) | 20260506 11:15:01.761 | INFO | Analyzing chunk frames = [916992,917504) | 20260506 11:15:01.761 | INFO | MemInfo: { | 20260506 11:15:01.761 | INFO | "gpu" : " 17 GB", | 20260506 11:15:01.761 | INFO | "huge" : "", | 20260506 11:15:01.761 | INFO | "huge_shared" : " 27 GB", | 20260506 11:15:01.761 | INFO | "malloc" : " 14 GB", | 20260506 11:15:01.761 | INFO | "pinned" : " 29 GB" | 20260506 11:15:01.761 | INFO | } | 20260506 11:15:01.761 | INFO | Perf report for the last 5.13941 seconds: | 20260506 11:15:01.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:15:01.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:01.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:01.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:01.761 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 11:15:01.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:01.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:01.762 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 11:15:01.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:01.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:01.762 | INFO | Uploaded 36GiB to GPU in 2.08948s (17.2291GiB/s) | 20260506 11:15:01.762 | INFO | Downloaded 34GiB to GPU in 1.79106s (18.9832GiB/s) | 20260506 11:15:01.762 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:15:01.762 | INFO | Total mem usage: 16.8 GiB | 20260506 11:15:01.762 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:15:01.762 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -6.31%) | 20260506 11:15:01.762 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:15:01.762 | INFO | High Water Breakdown: | 20260506 11:15:01.762 | INFO | 60.8% : 9.61 GiB ( 92.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:15:01.762 | INFO | 39.1% : 6.18 GiB ( 92.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:15:01.762 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:15:01.762 | INFO | ------------------------------------------ | 20260506 11:15:01.762 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:15:01.762 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:15:01.762 | INFO | Total mem usage: 26.5 GiB | 20260506 11:15:01.762 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:15:01.762 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:15:01.762 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:15:01.762 | INFO | High Water Breakdown: | 20260506 11:15:01.762 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:15:01.762 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:15:01.762 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:15:01.762 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:15:01.762 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:15:01.762 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:15:01.762 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:15:01.762 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:15:01.762 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:15:01.762 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:15:01.762 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:15:01.762 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:15:01.762 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:15:01.763 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:15:01.763 | INFO | ------------------------------------------ | 20260506 11:15:01.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:15:01.763 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:15:01.763 | INFO | Total mem usage: 25.6 GiB | 20260506 11:15:01.763 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:15:01.763 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 11:15:01.763 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 11:15:01.763 | INFO | High Water Breakdown: | 20260506 11:15:01.763 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 11:15:01.763 | INFO | ------------------------------------------ | 20260506 11:15:01.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:15:01.763 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:15:01.763 | INFO | Total mem usage: 15.7 GiB | 20260506 11:15:01.763 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:15:01.763 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:15:01.763 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 11:15:01.763 | INFO | High Water Breakdown: | 20260506 11:15:01.763 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:15:01.763 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:15:01.763 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:15:01.763 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:15:01.763 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:15:01.763 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:15:01.763 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:15:01.763 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:15:01.763 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:15:01.763 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:15:01.763 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:15:01.763 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:15:01.763 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:15:01.763 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:15:01.763 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:15:01.763 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:15:01.763 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:15:01.763 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:15:01.763 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:15:01.763 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:15:01.763 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:15:01.763 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:15:01.763 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:15:01.763 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:15:01.763 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:15:01.763 | INFO | ------------------------------------------ | 20260506 11:15:01.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:15:01.763 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:15:01.763 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:15:01.763 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:15:01.763 | INFO | PerfReport: Full Execution -- 3.02e+01 / -nan, 1.26e+01 / 0.00e+00 | 20260506 11:15:01.763 | INFO | PerfReport: 0.29%, AggregateStats -- 8.82e-02 / -nan, 6.33e-03 / 0.00e+00 | 20260506 11:15:01.763 | INFO | PerfReport: 11.03%, Baseline -- 3.33e+00 / -nan, 1.27e-01 / 0.00e+00 | 20260506 11:15:01.763 | INFO | PerfReport: 3.09%, Histogram -- 9.31e-01 / -nan, 7.58e-02 / 0.00e+00 | 20260506 11:15:01.763 | INFO | PerfReport: 16.04%, DME -- 3.87e+01 / -nan, 3.04e+00 / 0.00e+00 | 20260506 11:15:01.763 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.72e+01 / -nan, 2.34e-01 / 0.00e+00 | 20260506 11:15:01.763 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.21e+00 / -nan, 1.48e-01 / 0.00e+00 | 20260506 11:15:01.763 | INFO | PerfReport: 3.65%, Metrics -- 1.10e+00 / -nan, 2.04e-01 / 0.00e+00 | 20260506 11:15:01.763 | INFO | PerfReport: 1.39%, PulseSerializing -- 4.19e-01 / -nan, 6.30e-02 / 0.00e+00 | 20260506 11:15:01.763 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:15:01.763 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:15:01.763 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:15:01.763 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 11:15:01.763 | INFO | PerfReport: 22.83%, Upload -- 2.21e+01 / 0.00e+00, 8.19e+00 / 0.00e+00 | 20260506 11:15:01.763 | INFO | PerfReport: 19.95%, Download -- 1.93e+01 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260506 11:15:01.763 | INFO | PerfReport: 31.13%, Compute -- 3.02e+01 / 0.00e+00, 1.26e+01 / 0.00e+00 | 20260506 11:15:01.763 | INFO | PerfReport: 4.16%, LockUpload -- 4.03e+00 / 0.00e+00, 8.18e+00 / 0.00e+00 | 20260506 11:15:01.763 | INFO | PerfReport: 0.71%, LockDownload -- 6.88e-01 / 0.00e+00, 2.50e+00 / 0.00e+00 | 20260506 11:15:01.763 | INFO | PerfReport: 21.20%, LockCompute -- 2.06e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 11:15:01.763 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260506 11:15:01.858 | INFO | Starting a round of Baz IO | 20260506 11:15:05.011 | INFO | Populating RTMetrics report | 20260506 11:15:05.011 | INFO | RTMetrics report populated | 20260506 11:15:05.011 | INFO | Opening tmp RTMetrics json file | 20260506 11:15:05.061 | INFO | Serializing RTMetrics to json | 20260506 11:15:05.122 | INFO | Serialization done, swapping tmp file now... | 20260506 11:15:05.160 | INFO | RTMetrics done | 20260506 11:15:05.208 | INFO | Finished a round of Baz IO: Wrote 2.047 GiB in 3.349 seconds | 20260506 11:15:05.208 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.79% / 0.04% / 55.17% | 20260506 11:15:05.208 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.46035% / 7.54992e-05% / 3.57628e-05% | 20260506 11:15:05.208 | INFO | Baz IO overall: | 20260506 11:15:05.208 | INFO | BytesWritten : 435372298460 | 20260506 11:15:05.208 | INFO | NumEvents : 209935678205 | 20260506 11:15:06.900 | INFO | Analyzing chunk frames = [917504,918016) | 20260506 11:15:06.900 | INFO | MemInfo: { | 20260506 11:15:06.900 | INFO | "gpu" : " 17 GB", | 20260506 11:15:06.900 | INFO | "huge" : "", | 20260506 11:15:06.900 | INFO | "huge_shared" : " 27 GB", | 20260506 11:15:06.900 | INFO | "malloc" : " 14 GB", | 20260506 11:15:06.900 | INFO | "pinned" : " 29 GB" | 20260506 11:15:06.900 | INFO | } | 20260506 11:15:06.900 | INFO | Perf report for the last 5.13851 seconds: | 20260506 11:15:06.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:15:06.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:06.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:06.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:06.900 | INFO | ANALYSIS : 62.6%, 2.9, 94.2ms, 0 | 20260506 11:15:06.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:06.900 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260506 11:15:06.900 | INFO | PRE_HQ : 61.3%, 2.0, 63.0ms, 0 | 20260506 11:15:06.900 | INFO | BAZWRITER : 64.8%, 4.8, 158.2ms, 0 | 20260506 11:15:06.900 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 11:15:06.901 | INFO | Uploaded 36GiB to GPU in 2.14715s (16.7664GiB/s) | 20260506 11:15:06.901 | INFO | Downloaded 38GiB to GPU in 2.10898s (18.0182GiB/s) | 20260506 11:15:12.039 | INFO | Analyzing chunk frames = [918016,918528) | 20260506 11:15:12.039 | INFO | MemInfo: { | 20260506 11:15:12.039 | INFO | "gpu" : " 17 GB", | 20260506 11:15:12.039 | INFO | "huge" : "", | 20260506 11:15:12.039 | INFO | "huge_shared" : " 27 GB", | 20260506 11:15:12.040 | INFO | "malloc" : " 14 GB", | 20260506 11:15:12.040 | INFO | "pinned" : " 29 GB" | 20260506 11:15:12.040 | INFO | } | 20260506 11:15:12.040 | INFO | Perf report for the last 5.13973 seconds: | 20260506 11:15:12.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:15:12.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:12.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:12.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:12.040 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 11:15:12.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:12.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:12.040 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 11:15:12.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:12.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:12.040 | INFO | Uploaded 36GiB to GPU in 2.06222s (17.457GiB/s) | 20260506 11:15:12.040 | INFO | Downloaded 34GiB to GPU in 1.78354s (19.0632GiB/s) | 20260506 11:15:17.179 | INFO | Analyzing chunk frames = [918528,919040) | 20260506 11:15:17.179 | INFO | MemInfo: { | 20260506 11:15:17.179 | INFO | "gpu" : " 17 GB", | 20260506 11:15:17.179 | INFO | "huge" : "", | 20260506 11:15:17.179 | INFO | "huge_shared" : " 27 GB", | 20260506 11:15:17.179 | INFO | "malloc" : " 14 GB", | 20260506 11:15:17.179 | INFO | "pinned" : " 29 GB" | 20260506 11:15:17.179 | INFO | } | 20260506 11:15:17.179 | INFO | Perf report for the last 5.13911 seconds: | 20260506 11:15:17.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:15:17.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:17.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:17.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:17.179 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 11:15:17.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:17.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:17.179 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 11:15:17.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:17.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:17.180 | INFO | Uploaded 36GiB to GPU in 2.06175s (17.4609GiB/s) | 20260506 11:15:17.180 | INFO | Downloaded 34GiB to GPU in 1.78714s (19.0248GiB/s) | 20260506 11:15:22.318 | INFO | Analyzing chunk frames = [919040,919552) | 20260506 11:15:22.318 | INFO | MemInfo: { | 20260506 11:15:22.318 | INFO | "gpu" : " 17 GB", | 20260506 11:15:22.318 | INFO | "huge" : "", | 20260506 11:15:22.318 | INFO | "huge_shared" : " 27 GB", | 20260506 11:15:22.318 | INFO | "malloc" : " 14 GB", | 20260506 11:15:22.318 | INFO | "pinned" : " 29 GB" | 20260506 11:15:22.318 | INFO | } | 20260506 11:15:22.318 | INFO | Perf report for the last 5.13952 seconds: | 20260506 11:15:22.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:15:22.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:22.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:22.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:22.318 | INFO | ANALYSIS : 60.3%, 2.9, 93.1ms, 0 | 20260506 11:15:22.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:22.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:22.319 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 11:15:22.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:22.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:22.319 | INFO | Uploaded 36GiB to GPU in 2.04704s (17.5864GiB/s) | 20260506 11:15:22.319 | INFO | Downloaded 34GiB to GPU in 1.7918s (18.9753GiB/s) | 20260506 11:15:27.457 | INFO | Analyzing chunk frames = [919552,920064) | 20260506 11:15:27.457 | INFO | MemInfo: { | 20260506 11:15:27.457 | INFO | "gpu" : " 17 GB", | 20260506 11:15:27.457 | INFO | "huge" : "", | 20260506 11:15:27.458 | INFO | "huge_shared" : " 27 GB", | 20260506 11:15:27.458 | INFO | "malloc" : " 14 GB", | 20260506 11:15:27.458 | INFO | "pinned" : " 29 GB" | 20260506 11:15:27.458 | INFO | } | 20260506 11:15:27.458 | INFO | Perf report for the last 5.13933 seconds: | 20260506 11:15:27.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:15:27.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:27.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:27.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:27.458 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 11:15:27.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:27.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:27.458 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 11:15:27.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:27.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:27.458 | INFO | Uploaded 36GiB to GPU in 2.06581s (17.4266GiB/s) | 20260506 11:15:27.458 | INFO | Downloaded 34GiB to GPU in 1.72351s (19.7272GiB/s) | 20260506 11:15:32.597 | INFO | Analyzing chunk frames = [920064,920576) | 20260506 11:15:32.597 | INFO | MemInfo: { | 20260506 11:15:32.597 | INFO | "gpu" : " 17 GB", | 20260506 11:15:32.597 | INFO | "huge" : "", | 20260506 11:15:32.597 | INFO | "huge_shared" : " 27 GB", | 20260506 11:15:32.597 | INFO | "malloc" : " 14 GB", | 20260506 11:15:32.597 | INFO | "pinned" : " 29 GB" | 20260506 11:15:32.597 | INFO | } | 20260506 11:15:32.597 | INFO | Perf report for the last 5.1393 seconds: | 20260506 11:15:32.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:15:32.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:32.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:32.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:32.597 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260506 11:15:32.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:32.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:32.597 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 11:15:32.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:32.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:32.597 | INFO | Uploaded 36GiB to GPU in 2.08314s (17.2816GiB/s) | 20260506 11:15:32.597 | INFO | Downloaded 34GiB to GPU in 1.80849s (18.8003GiB/s) | 20260506 11:15:37.736 | INFO | Analyzing chunk frames = [920576,921088) | 20260506 11:15:37.736 | INFO | MemInfo: { | 20260506 11:15:37.736 | INFO | "gpu" : " 17 GB", | 20260506 11:15:37.736 | INFO | "huge" : "", | 20260506 11:15:37.736 | INFO | "huge_shared" : " 27 GB", | 20260506 11:15:37.736 | INFO | "malloc" : " 14 GB", | 20260506 11:15:37.736 | INFO | "pinned" : " 29 GB" | 20260506 11:15:37.736 | INFO | } | 20260506 11:15:37.736 | INFO | Perf report for the last 5.13924 seconds: | 20260506 11:15:37.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:15:37.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:37.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:37.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:37.736 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260506 11:15:37.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:37.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:37.736 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 11:15:37.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:37.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:37.737 | INFO | Uploaded 36GiB to GPU in 2.08656s (17.2533GiB/s) | 20260506 11:15:37.737 | INFO | Downloaded 34GiB to GPU in 1.75897s (19.3294GiB/s) | 20260506 11:15:42.876 | INFO | Analyzing chunk frames = [921088,921600) | 20260506 11:15:42.876 | INFO | MemInfo: { | 20260506 11:15:42.876 | INFO | "gpu" : " 17 GB", | 20260506 11:15:42.876 | INFO | "huge" : "", | 20260506 11:15:42.876 | INFO | "huge_shared" : " 27 GB", | 20260506 11:15:42.876 | INFO | "malloc" : " 14 GB", | 20260506 11:15:42.876 | INFO | "pinned" : " 29 GB" | 20260506 11:15:42.876 | INFO | } | 20260506 11:15:42.876 | INFO | Perf report for the last 5.13952 seconds: | 20260506 11:15:42.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:15:42.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:42.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:42.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:42.876 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 11:15:42.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:42.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:42.876 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 11:15:42.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:42.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:42.876 | INFO | Uploaded 36GiB to GPU in 2.08539s (17.263GiB/s) | 20260506 11:15:42.876 | INFO | Downloaded 34GiB to GPU in 1.72597s (19.6991GiB/s) | 20260506 11:15:42.973 | INFO | Starting a round of Baz IO | 20260506 11:15:46.115 | INFO | Populating RTMetrics report | 20260506 11:15:46.115 | INFO | RTMetrics report populated | 20260506 11:15:46.115 | INFO | Opening tmp RTMetrics json file | 20260506 11:15:46.130 | INFO | Serializing RTMetrics to json | 20260506 11:15:46.176 | INFO | Serialization done, swapping tmp file now... | 20260506 11:15:46.177 | INFO | RTMetrics done | 20260506 11:15:46.535 | INFO | Finished a round of Baz IO: Wrote 2.046 GiB in 3.561 seconds | 20260506 11:15:46.535 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.76% / 0.04% / 55.21% | 20260506 11:15:46.535 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.44244% / 5.5631e-05% / 3.97364e-05% | 20260506 11:15:46.535 | INFO | Baz IO overall: | 20260506 11:15:46.535 | INFO | BytesWritten : 437569254475 | 20260506 11:15:46.535 | INFO | NumEvents : 211002092136 | 20260506 11:15:48.015 | INFO | Analyzing chunk frames = [921600,922112) | 20260506 11:15:48.015 | INFO | MemInfo: { | 20260506 11:15:48.015 | INFO | "gpu" : " 17 GB", | 20260506 11:15:48.015 | INFO | "huge" : "", | 20260506 11:15:48.015 | INFO | "huge_shared" : " 27 GB", | 20260506 11:15:48.015 | INFO | "malloc" : " 14 GB", | 20260506 11:15:48.015 | INFO | "pinned" : " 29 GB" | 20260506 11:15:48.015 | INFO | } | 20260506 11:15:48.015 | INFO | Perf report for the last 5.13912 seconds: | 20260506 11:15:48.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:15:48.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:48.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:48.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:48.015 | INFO | ANALYSIS : 62.4%, 2.9, 93.5ms, 0 | 20260506 11:15:48.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:48.015 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 11:15:48.015 | INFO | PRE_HQ : 61.3%, 2.0, 63.4ms, 0 | 20260506 11:15:48.015 | INFO | BAZWRITER : 68.9%, 4.8, 168.6ms, 0 | 20260506 11:15:48.015 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 11:15:48.016 | INFO | Uploaded 36GiB to GPU in 2.154s (16.7131GiB/s) | 20260506 11:15:48.016 | INFO | Downloaded 38GiB to GPU in 2.09027s (18.1795GiB/s) | 20260506 11:15:53.154 | INFO | Analyzing chunk frames = [922112,922624) | 20260506 11:15:53.154 | INFO | MemInfo: { | 20260506 11:15:53.154 | INFO | "gpu" : " 17 GB", | 20260506 11:15:53.154 | INFO | "huge" : "", | 20260506 11:15:53.154 | INFO | "huge_shared" : " 27 GB", | 20260506 11:15:53.154 | INFO | "malloc" : " 14 GB", | 20260506 11:15:53.154 | INFO | "pinned" : " 29 GB" | 20260506 11:15:53.154 | INFO | } | 20260506 11:15:53.154 | INFO | Perf report for the last 5.13949 seconds: | 20260506 11:15:53.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:15:53.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:53.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:53.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:53.155 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 11:15:53.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:53.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:53.155 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:15:53.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:53.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:53.155 | INFO | Uploaded 36GiB to GPU in 2.05892s (17.4849GiB/s) | 20260506 11:15:53.155 | INFO | Downloaded 34GiB to GPU in 1.81695s (18.7127GiB/s) | 20260506 11:15:58.294 | INFO | Analyzing chunk frames = [922624,923136) | 20260506 11:15:58.294 | INFO | MemInfo: { | 20260506 11:15:58.294 | INFO | "gpu" : " 17 GB", | 20260506 11:15:58.294 | INFO | "huge" : "", | 20260506 11:15:58.294 | INFO | "huge_shared" : " 27 GB", | 20260506 11:15:58.294 | INFO | "malloc" : " 14 GB", | 20260506 11:15:58.294 | INFO | "pinned" : " 29 GB" | 20260506 11:15:58.294 | INFO | } | 20260506 11:15:58.294 | INFO | Perf report for the last 5.13952 seconds: | 20260506 11:15:58.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:15:58.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:58.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:58.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:58.294 | INFO | ANALYSIS : 59.4%, 3.0, 92.0ms, 0 | 20260506 11:15:58.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:58.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:58.294 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 11:15:58.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:58.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:15:58.294 | INFO | Uploaded 36GiB to GPU in 1.86937s (19.2579GiB/s) | 20260506 11:15:58.294 | INFO | Downloaded 34GiB to GPU in 1.70382s (19.9551GiB/s) | 20260506 11:16:03.433 | INFO | Analyzing chunk frames = [923136,923648) | 20260506 11:16:03.433 | INFO | MemInfo: { | 20260506 11:16:03.433 | INFO | "gpu" : " 17 GB", | 20260506 11:16:03.433 | INFO | "huge" : "", | 20260506 11:16:03.433 | INFO | "huge_shared" : " 27 GB", | 20260506 11:16:03.433 | INFO | "malloc" : " 14 GB", | 20260506 11:16:03.433 | INFO | "pinned" : " 29 GB" | 20260506 11:16:03.433 | INFO | } | 20260506 11:16:03.433 | INFO | Perf report for the last 5.13899 seconds: | 20260506 11:16:03.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:16:03.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:03.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:03.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:03.433 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260506 11:16:03.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:03.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:03.433 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 11:16:03.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:03.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:03.433 | INFO | Uploaded 36GiB to GPU in 2.18628s (16.4663GiB/s) | 20260506 11:16:03.433 | INFO | Downloaded 34GiB to GPU in 1.89925s (17.9018GiB/s) | 20260506 11:16:08.572 | INFO | Analyzing chunk frames = [923648,924160) | 20260506 11:16:08.572 | INFO | MemInfo: { | 20260506 11:16:08.572 | INFO | "gpu" : " 17 GB", | 20260506 11:16:08.572 | INFO | "huge" : "", | 20260506 11:16:08.572 | INFO | "huge_shared" : " 27 GB", | 20260506 11:16:08.572 | INFO | "malloc" : " 14 GB", | 20260506 11:16:08.572 | INFO | "pinned" : " 29 GB" | 20260506 11:16:08.572 | INFO | } | 20260506 11:16:08.572 | INFO | Perf report for the last 5.13944 seconds: | 20260506 11:16:08.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:16:08.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:08.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:08.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:08.573 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 11:16:08.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:08.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:08.573 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 11:16:08.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:08.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:08.573 | INFO | Uploaded 36GiB to GPU in 2.05675s (17.5034GiB/s) | 20260506 11:16:08.573 | INFO | Downloaded 34GiB to GPU in 1.897s (17.923GiB/s) | 20260506 11:16:13.711 | INFO | Analyzing chunk frames = [924160,924672) | 20260506 11:16:13.711 | INFO | MemInfo: { | 20260506 11:16:13.711 | INFO | "gpu" : " 17 GB", | 20260506 11:16:13.711 | INFO | "huge" : "", | 20260506 11:16:13.712 | INFO | "huge_shared" : " 27 GB", | 20260506 11:16:13.712 | INFO | "malloc" : " 14 GB", | 20260506 11:16:13.712 | INFO | "pinned" : " 29 GB" | 20260506 11:16:13.712 | INFO | } | 20260506 11:16:13.712 | INFO | Perf report for the last 5.13914 seconds: | 20260506 11:16:13.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:16:13.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:13.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:13.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:13.712 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260506 11:16:13.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:13.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:13.712 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 11:16:13.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:13.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:13.712 | INFO | Uploaded 36GiB to GPU in 2.17409s (16.5586GiB/s) | 20260506 11:16:13.712 | INFO | Downloaded 34GiB to GPU in 1.97155s (17.2453GiB/s) | 20260506 11:16:18.851 | INFO | Analyzing chunk frames = [924672,925184) | 20260506 11:16:18.851 | INFO | MemInfo: { | 20260506 11:16:18.851 | INFO | "gpu" : " 17 GB", | 20260506 11:16:18.851 | INFO | "huge" : "", | 20260506 11:16:18.851 | INFO | "huge_shared" : " 27 GB", | 20260506 11:16:18.851 | INFO | "malloc" : " 14 GB", | 20260506 11:16:18.851 | INFO | "pinned" : " 29 GB" | 20260506 11:16:18.851 | INFO | } | 20260506 11:16:18.851 | INFO | Perf report for the last 5.13923 seconds: | 20260506 11:16:18.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:16:18.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:18.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:18.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:18.851 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 11:16:18.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:18.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:18.851 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 11:16:18.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:18.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:18.851 | INFO | Uploaded 36GiB to GPU in 2.03313s (17.7067GiB/s) | 20260506 11:16:18.851 | INFO | Downloaded 34GiB to GPU in 1.91355s (17.768GiB/s) | 20260506 11:16:23.990 | INFO | Analyzing chunk frames = [925184,925696) | 20260506 11:16:23.990 | INFO | MemInfo: { | 20260506 11:16:23.990 | INFO | "gpu" : " 17 GB", | 20260506 11:16:23.990 | INFO | "huge" : "", | 20260506 11:16:23.990 | INFO | "huge_shared" : " 27 GB", | 20260506 11:16:23.990 | INFO | "malloc" : " 14 GB", | 20260506 11:16:23.990 | INFO | "pinned" : " 29 GB" | 20260506 11:16:23.990 | INFO | } | 20260506 11:16:23.990 | INFO | Perf report for the last 5.13959 seconds: | 20260506 11:16:23.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:16:23.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:23.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:23.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:23.990 | INFO | ANALYSIS : 64.2%, 3.0, 100.0ms, 0 | 20260506 11:16:23.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:23.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:23.991 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 11:16:23.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:23.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:23.991 | INFO | Uploaded 36GiB to GPU in 2.20674s (16.3137GiB/s) | 20260506 11:16:23.991 | INFO | Downloaded 34GiB to GPU in 2.11634s (16.0655GiB/s) | 20260506 11:16:23.991 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:16:23.991 | INFO | Total mem usage: 16.8 GiB | 20260506 11:16:23.991 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:16:23.991 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -6.2%) | 20260506 11:16:23.991 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:16:23.991 | INFO | High Water Breakdown: | 20260506 11:16:23.991 | INFO | 60.8% : 9.63 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:16:23.991 | INFO | 39% : 6.18 GiB ( 92.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:16:23.991 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:16:23.991 | INFO | ------------------------------------------ | 20260506 11:16:23.991 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:16:23.991 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:16:23.991 | INFO | Total mem usage: 26.5 GiB | 20260506 11:16:23.991 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:16:23.991 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:16:23.991 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:16:23.991 | INFO | High Water Breakdown: | 20260506 11:16:23.991 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:16:23.991 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:16:23.991 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:16:23.991 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:16:23.991 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:16:23.991 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:16:23.991 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:16:23.991 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:16:23.991 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:16:23.991 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:16:23.991 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:16:23.991 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:16:23.991 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:16:23.991 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:16:23.991 | INFO | ------------------------------------------ | 20260506 11:16:23.991 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:16:23.991 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:16:23.991 | INFO | Total mem usage: 25.6 GiB | 20260506 11:16:23.991 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:16:23.991 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:16:23.991 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:16:23.991 | INFO | High Water Breakdown: | 20260506 11:16:23.991 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:16:23.991 | INFO | ------------------------------------------ | 20260506 11:16:23.991 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:16:23.991 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:16:23.991 | INFO | Total mem usage: 15.7 GiB | 20260506 11:16:23.991 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:16:23.991 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:16:23.991 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 11:16:23.991 | INFO | High Water Breakdown: | 20260506 11:16:23.991 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:16:23.991 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:16:23.991 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:16:23.991 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:16:23.991 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:16:23.991 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:16:23.991 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:16:23.991 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:16:23.991 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:16:23.991 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:16:23.991 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:16:23.991 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:16:23.991 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:16:23.991 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:16:23.991 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:16:23.992 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:16:23.992 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:16:23.992 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:16:23.992 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:16:23.992 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:16:23.992 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:16:23.992 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:16:23.992 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:16:23.992 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:16:23.992 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:16:23.992 | INFO | ------------------------------------------ | 20260506 11:16:23.992 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:16:23.992 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:16:23.992 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:16:23.992 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:16:23.992 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.43e+01, 1.29e+01 / 3.40e+00 | 20260506 11:16:23.992 | INFO | PerfReport: 0.29%, AggregateStats -- 8.91e-02 / 9.80e-02, 7.81e-03 / 5.00e-03 | 20260506 11:16:23.992 | INFO | PerfReport: 11.12%, Baseline -- 3.41e+00 / 4.14e+00, 3.19e-01 / 3.18e-01 | 20260506 11:16:23.992 | INFO | PerfReport: 3.11%, Histogram -- 9.53e-01 / 1.35e+00, 1.17e-01 / 1.31e-01 | 20260506 11:16:23.992 | INFO | PerfReport: 15.99%, DME -- 3.92e+01 / 4.49e+01, 3.57e+00 / 2.81e+00 | 20260506 11:16:23.992 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.75e+01 / 1.93e+01, 1.04e+00 / 1.48e+00 | 20260506 11:16:23.992 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.27e+00 / 2.65e+00, 2.57e-01 / 1.89e-01 | 20260506 11:16:23.992 | INFO | PerfReport: 3.70%, Metrics -- 1.13e+00 / 1.39e+00, 2.31e-01 / 1.98e-01 | 20260506 11:16:23.992 | INFO | PerfReport: 1.38%, PulseSerializing -- 4.23e-01 / 4.57e-01, 6.33e-02 / 2.65e-02 | 20260506 11:16:23.992 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:16:23.992 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:16:23.992 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:16:23.992 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 11:16:23.992 | INFO | PerfReport: 22.37%, Upload -- 2.17e+01 / 0.00e+00, 7.92e+00 / 0.00e+00 | 20260506 11:16:23.992 | INFO | PerfReport: 20.04%, Download -- 1.95e+01 / 0.00e+00, 5.83e+00 / 0.00e+00 | 20260506 11:16:23.992 | INFO | PerfReport: 31.55%, Compute -- 3.07e+01 / 0.00e+00, 1.29e+01 / 0.00e+00 | 20260506 11:16:23.992 | INFO | PerfReport: 3.64%, LockUpload -- 3.53e+00 / 0.00e+00, 7.99e+00 / 0.00e+00 | 20260506 11:16:23.992 | INFO | PerfReport: 0.77%, LockDownload -- 7.52e-01 / 0.00e+00, 2.58e+00 / 0.00e+00 | 20260506 11:16:23.992 | INFO | PerfReport: 21.61%, LockCompute -- 2.10e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 11:16:23.992 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260506 11:16:24.114 | INFO | Starting a round of Baz IO | 20260506 11:16:27.370 | INFO | Populating RTMetrics report | 20260506 11:16:27.370 | INFO | RTMetrics report populated | 20260506 11:16:27.370 | INFO | Opening tmp RTMetrics json file | 20260506 11:16:27.372 | INFO | Serializing RTMetrics to json | 20260506 11:16:27.374 | INFO | Serialization done, swapping tmp file now... | 20260506 11:16:27.374 | INFO | RTMetrics done | 20260506 11:16:27.502 | INFO | Finished a round of Baz IO: Wrote 2.049 GiB in 3.387 seconds | 20260506 11:16:27.502 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.72% / 0.04% / 55.24% | 20260506 11:16:27.502 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.44255% / 7.54992e-05% / 3.57628e-05% | 20260506 11:16:27.502 | INFO | Baz IO overall: | 20260506 11:16:27.502 | INFO | BytesWritten : 439769822961 | 20260506 11:16:27.502 | INFO | NumEvents : 212071168999 | 20260506 11:16:29.130 | INFO | Analyzing chunk frames = [925696,926208) | 20260506 11:16:29.130 | INFO | MemInfo: { | 20260506 11:16:29.130 | INFO | "gpu" : " 17 GB", | 20260506 11:16:29.130 | INFO | "huge" : "", | 20260506 11:16:29.130 | INFO | "huge_shared" : " 27 GB", | 20260506 11:16:29.130 | INFO | "malloc" : " 14 GB", | 20260506 11:16:29.130 | INFO | "pinned" : " 29 GB" | 20260506 11:16:29.130 | INFO | } | 20260506 11:16:29.130 | INFO | Perf report for the last 5.1395 seconds: | 20260506 11:16:29.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:16:29.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:29.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:29.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:29.130 | INFO | ANALYSIS : 65.2%, 2.9, 98.3ms, 0 | 20260506 11:16:29.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:29.130 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 11:16:29.130 | INFO | PRE_HQ : 63.9%, 1.9, 62.0ms, 0 | 20260506 11:16:29.130 | INFO | BAZWRITER : 65.5%, 4.4, 149.3ms, 0 | 20260506 11:16:29.130 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 11:16:29.130 | INFO | Uploaded 36GiB to GPU in 2.11402s (17.0292GiB/s) | 20260506 11:16:29.131 | INFO | Downloaded 38GiB to GPU in 2.32283s (16.3593GiB/s) | 20260506 11:16:34.269 | INFO | Analyzing chunk frames = [926208,926720) | 20260506 11:16:34.269 | INFO | MemInfo: { | 20260506 11:16:34.269 | INFO | "gpu" : " 17 GB", | 20260506 11:16:34.269 | INFO | "huge" : "", | 20260506 11:16:34.269 | INFO | "huge_shared" : " 27 GB", | 20260506 11:16:34.269 | INFO | "malloc" : " 14 GB", | 20260506 11:16:34.269 | INFO | "pinned" : " 29 GB" | 20260506 11:16:34.269 | INFO | } | 20260506 11:16:34.269 | INFO | Perf report for the last 5.13898 seconds: | 20260506 11:16:34.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:16:34.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:34.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:34.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:34.269 | INFO | ANALYSIS : 63.7%, 3.0, 99.2ms, 0 | 20260506 11:16:34.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:34.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:34.269 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:16:34.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:34.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:34.269 | INFO | Uploaded 36GiB to GPU in 2.05092s (17.5531GiB/s) | 20260506 11:16:34.269 | INFO | Downloaded 34GiB to GPU in 1.98783s (17.1041GiB/s) | 20260506 11:16:39.409 | INFO | Analyzing chunk frames = [926720,927232) | 20260506 11:16:39.409 | INFO | MemInfo: { | 20260506 11:16:39.409 | INFO | "gpu" : " 17 GB", | 20260506 11:16:39.409 | INFO | "huge" : "", | 20260506 11:16:39.409 | INFO | "huge_shared" : " 27 GB", | 20260506 11:16:39.409 | INFO | "malloc" : " 14 GB", | 20260506 11:16:39.409 | INFO | "pinned" : " 29 GB" | 20260506 11:16:39.409 | INFO | } | 20260506 11:16:39.409 | INFO | Perf report for the last 5.14009 seconds: | 20260506 11:16:39.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:16:39.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:39.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:39.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:39.409 | INFO | ANALYSIS : 63.9%, 3.0, 99.0ms, 0 | 20260506 11:16:39.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:39.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:39.409 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:16:39.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:39.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:39.409 | INFO | Uploaded 36GiB to GPU in 2.16234s (16.6486GiB/s) | 20260506 11:16:39.409 | INFO | Downloaded 34GiB to GPU in 2.04755s (16.6052GiB/s) | 20260506 11:16:44.548 | INFO | Analyzing chunk frames = [927232,927744) | 20260506 11:16:44.548 | INFO | MemInfo: { | 20260506 11:16:44.548 | INFO | "gpu" : " 17 GB", | 20260506 11:16:44.548 | INFO | "huge" : "", | 20260506 11:16:44.548 | INFO | "huge_shared" : " 27 GB", | 20260506 11:16:44.548 | INFO | "malloc" : " 14 GB", | 20260506 11:16:44.548 | INFO | "pinned" : " 29 GB" | 20260506 11:16:44.548 | INFO | } | 20260506 11:16:44.548 | INFO | Perf report for the last 5.13873 seconds: | 20260506 11:16:44.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:16:44.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:44.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:44.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:44.548 | INFO | ANALYSIS : 63.6%, 2.9, 98.3ms, 0 | 20260506 11:16:44.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:44.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:44.548 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:16:44.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:44.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:44.548 | INFO | Uploaded 36GiB to GPU in 2.17679s (16.5381GiB/s) | 20260506 11:16:44.548 | INFO | Downloaded 34GiB to GPU in 2.03344s (16.7204GiB/s) | 20260506 11:16:49.687 | INFO | Analyzing chunk frames = [927744,928256) | 20260506 11:16:49.687 | INFO | MemInfo: { | 20260506 11:16:49.687 | INFO | "gpu" : " 17 GB", | 20260506 11:16:49.687 | INFO | "huge" : "", | 20260506 11:16:49.687 | INFO | "huge_shared" : " 27 GB", | 20260506 11:16:49.687 | INFO | "malloc" : " 14 GB", | 20260506 11:16:49.687 | INFO | "pinned" : " 29 GB" | 20260506 11:16:49.687 | INFO | } | 20260506 11:16:49.687 | INFO | Perf report for the last 5.13938 seconds: | 20260506 11:16:49.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:16:49.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:49.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:49.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:49.687 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260506 11:16:49.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:49.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:49.687 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 11:16:49.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:49.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:49.688 | INFO | Uploaded 36GiB to GPU in 2.14313s (16.7978GiB/s) | 20260506 11:16:49.688 | INFO | Downloaded 34GiB to GPU in 2.01284s (16.8915GiB/s) | 20260506 11:16:54.827 | INFO | Analyzing chunk frames = [928256,928768) | 20260506 11:16:54.827 | INFO | MemInfo: { | 20260506 11:16:54.827 | INFO | "gpu" : " 17 GB", | 20260506 11:16:54.827 | INFO | "huge" : "", | 20260506 11:16:54.827 | INFO | "huge_shared" : " 27 GB", | 20260506 11:16:54.827 | INFO | "malloc" : " 14 GB", | 20260506 11:16:54.827 | INFO | "pinned" : " 29 GB" | 20260506 11:16:54.827 | INFO | } | 20260506 11:16:54.827 | INFO | Perf report for the last 5.13969 seconds: | 20260506 11:16:54.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:16:54.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:54.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:54.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:54.827 | INFO | ANALYSIS : 63.3%, 3.0, 98.8ms, 0 | 20260506 11:16:54.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:54.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:54.827 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:16:54.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:54.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:54.827 | INFO | Uploaded 36GiB to GPU in 2.11125s (17.0515GiB/s) | 20260506 11:16:54.827 | INFO | Downloaded 34GiB to GPU in 2.04634s (16.615GiB/s) | 20260506 11:16:59.966 | INFO | Analyzing chunk frames = [928768,929280) | 20260506 11:16:59.966 | INFO | MemInfo: { | 20260506 11:16:59.966 | INFO | "gpu" : " 17 GB", | 20260506 11:16:59.966 | INFO | "huge" : "", | 20260506 11:16:59.966 | INFO | "huge_shared" : " 27 GB", | 20260506 11:16:59.966 | INFO | "malloc" : " 14 GB", | 20260506 11:16:59.966 | INFO | "pinned" : " 29 GB" | 20260506 11:16:59.966 | INFO | } | 20260506 11:16:59.966 | INFO | Perf report for the last 5.13891 seconds: | 20260506 11:16:59.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:16:59.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:59.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:59.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:59.966 | INFO | ANALYSIS : 63.4%, 3.0, 98.7ms, 0 | 20260506 11:16:59.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:59.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:59.966 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:16:59.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:59.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:16:59.966 | INFO | Uploaded 36GiB to GPU in 2.007s (17.9373GiB/s) | 20260506 11:16:59.966 | INFO | Downloaded 34GiB to GPU in 2.08505s (16.3066GiB/s) | 20260506 11:17:05.105 | INFO | Analyzing chunk frames = [929280,929792) | 20260506 11:17:05.105 | INFO | MemInfo: { | 20260506 11:17:05.105 | INFO | "gpu" : " 17 GB", | 20260506 11:17:05.105 | INFO | "huge" : "", | 20260506 11:17:05.105 | INFO | "huge_shared" : " 27 GB", | 20260506 11:17:05.105 | INFO | "malloc" : " 14 GB", | 20260506 11:17:05.105 | INFO | "pinned" : " 29 GB" | 20260506 11:17:05.105 | INFO | } | 20260506 11:17:05.105 | INFO | Perf report for the last 5.13953 seconds: | 20260506 11:17:05.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:17:05.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:05.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:05.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:05.105 | INFO | ANALYSIS : 64.3%, 3.0, 100.2ms, 0 | 20260506 11:17:05.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:05.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:05.105 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:17:05.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:05.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:05.106 | INFO | Uploaded 36GiB to GPU in 2.12373s (16.9513GiB/s) | 20260506 11:17:05.106 | INFO | Downloaded 34GiB to GPU in 2.09765s (16.2086GiB/s) | 20260506 11:17:05.216 | INFO | Starting a round of Baz IO | 20260506 11:17:08.460 | INFO | Populating RTMetrics report | 20260506 11:17:08.461 | INFO | RTMetrics report populated | 20260506 11:17:08.461 | INFO | Opening tmp RTMetrics json file | 20260506 11:17:08.469 | INFO | Serializing RTMetrics to json | 20260506 11:17:08.490 | INFO | Serialization done, swapping tmp file now... | 20260506 11:17:08.507 | INFO | RTMetrics done | 20260506 11:17:08.670 | INFO | Finished a round of Baz IO: Wrote 2.048 GiB in 3.454 seconds | 20260506 11:17:08.671 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.68% / 0.04% / 55.28% | 20260506 11:17:08.671 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.42277% / 5.96046e-05% / 5.16574e-05% | 20260506 11:17:08.671 | INFO | Baz IO overall: | 20260506 11:17:08.671 | INFO | BytesWritten : 441969266701 | 20260506 11:17:08.671 | INFO | NumEvents : 213139629110 | 20260506 11:17:10.244 | INFO | Analyzing chunk frames = [929792,930304) | 20260506 11:17:10.244 | INFO | MemInfo: { | 20260506 11:17:10.244 | INFO | "gpu" : " 17 GB", | 20260506 11:17:10.244 | INFO | "huge" : "", | 20260506 11:17:10.244 | INFO | "huge_shared" : " 27 GB", | 20260506 11:17:10.244 | INFO | "malloc" : " 14 GB", | 20260506 11:17:10.244 | INFO | "pinned" : " 29 GB" | 20260506 11:17:10.244 | INFO | } | 20260506 11:17:10.244 | INFO | Perf report for the last 5.13903 seconds: | 20260506 11:17:10.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:17:10.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:10.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:10.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:10.245 | INFO | ANALYSIS : 64.7%, 2.9, 97.4ms, 0 | 20260506 11:17:10.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:10.245 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260506 11:17:10.245 | INFO | PRE_HQ : 63.3%, 1.9, 61.6ms, 0 | 20260506 11:17:10.245 | INFO | BAZWRITER : 66.5%, 4.7, 162.3ms, 0 | 20260506 11:17:10.245 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 11:17:10.245 | INFO | Uploaded 36GiB to GPU in 2.13786s (16.8393GiB/s) | 20260506 11:17:10.245 | INFO | Downloaded 38GiB to GPU in 2.31556s (16.4107GiB/s) | 20260506 11:17:15.384 | INFO | Analyzing chunk frames = [930304,930816) | 20260506 11:17:15.384 | INFO | MemInfo: { | 20260506 11:17:15.384 | INFO | "gpu" : " 17 GB", | 20260506 11:17:15.384 | INFO | "huge" : "", | 20260506 11:17:15.384 | INFO | "huge_shared" : " 27 GB", | 20260506 11:17:15.384 | INFO | "malloc" : " 14 GB", | 20260506 11:17:15.384 | INFO | "pinned" : " 29 GB" | 20260506 11:17:15.384 | INFO | } | 20260506 11:17:15.384 | INFO | Perf report for the last 5.13954 seconds: | 20260506 11:17:15.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:17:15.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:15.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:15.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:15.384 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260506 11:17:15.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:15.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:15.384 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:17:15.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:15.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:15.384 | INFO | Uploaded 36GiB to GPU in 2.09353s (17.1958GiB/s) | 20260506 11:17:15.384 | INFO | Downloaded 34GiB to GPU in 2.0498s (16.587GiB/s) | 20260506 11:17:20.523 | INFO | Analyzing chunk frames = [930816,931328) | 20260506 11:17:20.523 | INFO | MemInfo: { | 20260506 11:17:20.523 | INFO | "gpu" : " 17 GB", | 20260506 11:17:20.523 | INFO | "huge" : "", | 20260506 11:17:20.523 | INFO | "huge_shared" : " 27 GB", | 20260506 11:17:20.523 | INFO | "malloc" : " 14 GB", | 20260506 11:17:20.523 | INFO | "pinned" : " 29 GB" | 20260506 11:17:20.523 | INFO | } | 20260506 11:17:20.523 | INFO | Perf report for the last 5.13933 seconds: | 20260506 11:17:20.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:17:20.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:20.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:20.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:20.523 | INFO | ANALYSIS : 63.0%, 3.0, 97.7ms, 0 | 20260506 11:17:20.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:20.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:20.523 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:17:20.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:20.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:20.524 | INFO | Uploaded 36GiB to GPU in 2.04717s (17.5853GiB/s) | 20260506 11:17:20.524 | INFO | Downloaded 34GiB to GPU in 1.98872s (17.0965GiB/s) | 20260506 11:17:25.663 | INFO | Analyzing chunk frames = [931328,931840) | 20260506 11:17:25.663 | INFO | MemInfo: { | 20260506 11:17:25.663 | INFO | "gpu" : " 17 GB", | 20260506 11:17:25.663 | INFO | "huge" : "", | 20260506 11:17:25.663 | INFO | "huge_shared" : " 27 GB", | 20260506 11:17:25.663 | INFO | "malloc" : " 14 GB", | 20260506 11:17:25.663 | INFO | "pinned" : " 29 GB" | 20260506 11:17:25.663 | INFO | } | 20260506 11:17:25.663 | INFO | Perf report for the last 5.13954 seconds: | 20260506 11:17:25.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:17:25.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:25.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:25.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:25.663 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 11:17:25.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:25.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:25.663 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:17:25.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:25.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:25.663 | INFO | Uploaded 36GiB to GPU in 1.98815s (18.1073GiB/s) | 20260506 11:17:25.663 | INFO | Downloaded 34GiB to GPU in 1.93092s (17.6082GiB/s) | 20260506 11:17:30.802 | INFO | Analyzing chunk frames = [931840,932352) | 20260506 11:17:30.802 | INFO | MemInfo: { | 20260506 11:17:30.802 | INFO | "gpu" : " 17 GB", | 20260506 11:17:30.802 | INFO | "huge" : "", | 20260506 11:17:30.802 | INFO | "huge_shared" : " 27 GB", | 20260506 11:17:30.802 | INFO | "malloc" : " 14 GB", | 20260506 11:17:30.802 | INFO | "pinned" : " 29 GB" | 20260506 11:17:30.802 | INFO | } | 20260506 11:17:30.802 | INFO | Perf report for the last 5.13898 seconds: | 20260506 11:17:30.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:17:30.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:30.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:30.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:30.802 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 11:17:30.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:30.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:30.802 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 11:17:30.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:30.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:30.802 | INFO | Uploaded 36GiB to GPU in 2.06511s (17.4325GiB/s) | 20260506 11:17:30.803 | INFO | Downloaded 34GiB to GPU in 1.95962s (17.3503GiB/s) | 20260506 11:17:35.941 | INFO | Analyzing chunk frames = [932352,932864) | 20260506 11:17:35.941 | INFO | MemInfo: { | 20260506 11:17:35.941 | INFO | "gpu" : " 17 GB", | 20260506 11:17:35.941 | INFO | "huge" : "", | 20260506 11:17:35.941 | INFO | "huge_shared" : " 27 GB", | 20260506 11:17:35.941 | INFO | "malloc" : " 14 GB", | 20260506 11:17:35.941 | INFO | "pinned" : " 29 GB" | 20260506 11:17:35.941 | INFO | } | 20260506 11:17:35.941 | INFO | Perf report for the last 5.13926 seconds: | 20260506 11:17:35.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:17:35.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:35.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:35.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:35.941 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 11:17:35.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:35.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:35.941 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:17:35.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:35.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:35.941 | INFO | Uploaded 36GiB to GPU in 2.0785s (17.3202GiB/s) | 20260506 11:17:35.942 | INFO | Downloaded 34GiB to GPU in 1.92938s (17.6223GiB/s) | 20260506 11:17:41.080 | INFO | Analyzing chunk frames = [932864,933376) | 20260506 11:17:41.080 | INFO | MemInfo: { | 20260506 11:17:41.080 | INFO | "gpu" : " 17 GB", | 20260506 11:17:41.081 | INFO | "huge" : "", | 20260506 11:17:41.081 | INFO | "huge_shared" : " 27 GB", | 20260506 11:17:41.081 | INFO | "malloc" : " 14 GB", | 20260506 11:17:41.081 | INFO | "pinned" : " 29 GB" | 20260506 11:17:41.081 | INFO | } | 20260506 11:17:41.081 | INFO | Perf report for the last 5.13959 seconds: | 20260506 11:17:41.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:17:41.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:41.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:41.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:41.081 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 11:17:41.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:41.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:41.081 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:17:41.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:41.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:41.081 | INFO | Uploaded 36GiB to GPU in 2.00272s (17.9756GiB/s) | 20260506 11:17:41.081 | INFO | Downloaded 34GiB to GPU in 2.0305s (16.7446GiB/s) | 20260506 11:17:46.220 | INFO | Analyzing chunk frames = [933376,933888) | 20260506 11:17:46.220 | INFO | MemInfo: { | 20260506 11:17:46.220 | INFO | "gpu" : " 17 GB", | 20260506 11:17:46.220 | INFO | "huge" : "", | 20260506 11:17:46.220 | INFO | "huge_shared" : " 27 GB", | 20260506 11:17:46.220 | INFO | "malloc" : " 14 GB", | 20260506 11:17:46.220 | INFO | "pinned" : " 29 GB" | 20260506 11:17:46.220 | INFO | } | 20260506 11:17:46.220 | INFO | Perf report for the last 5.13912 seconds: | 20260506 11:17:46.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:17:46.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:46.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:46.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:46.220 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 11:17:46.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:46.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:46.220 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 11:17:46.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:46.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:46.220 | INFO | Uploaded 36GiB to GPU in 2.04296s (17.6215GiB/s) | 20260506 11:17:46.220 | INFO | Downloaded 34GiB to GPU in 1.96748s (17.281GiB/s) | 20260506 11:17:46.220 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:17:46.220 | INFO | Total mem usage: 16.8 GiB | 20260506 11:17:46.220 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:17:46.220 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -10.8%) | 20260506 11:17:46.220 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:17:46.220 | INFO | High Water Breakdown: | 20260506 11:17:46.220 | INFO | 60.2% : 9.13 GiB ( 88% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:17:46.220 | INFO | 39.7% : 6.02 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:17:46.220 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:17:46.220 | INFO | ------------------------------------------ | 20260506 11:17:46.220 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:17:46.220 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:17:46.220 | INFO | Total mem usage: 26.5 GiB | 20260506 11:17:46.220 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:17:46.220 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:17:46.220 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:17:46.221 | INFO | High Water Breakdown: | 20260506 11:17:46.221 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:17:46.221 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:17:46.221 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:17:46.221 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:17:46.221 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:17:46.221 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:17:46.221 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:17:46.221 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:17:46.221 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:17:46.221 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:17:46.221 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:17:46.221 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:17:46.221 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:17:46.221 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:17:46.221 | INFO | ------------------------------------------ | 20260506 11:17:46.221 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:17:46.221 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:17:46.221 | INFO | Total mem usage: 25.6 GiB | 20260506 11:17:46.221 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:17:46.221 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:17:46.221 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:17:46.221 | INFO | High Water Breakdown: | 20260506 11:17:46.221 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:17:46.221 | INFO | ------------------------------------------ | 20260506 11:17:46.221 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:17:46.221 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:17:46.221 | INFO | Total mem usage: 15.7 GiB | 20260506 11:17:46.221 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:17:46.221 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:17:46.221 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 11:17:46.221 | INFO | High Water Breakdown: | 20260506 11:17:46.221 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:17:46.221 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:17:46.221 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:17:46.221 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:17:46.221 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:17:46.221 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:17:46.221 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:17:46.221 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:17:46.221 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:17:46.221 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:17:46.221 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:17:46.221 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:17:46.221 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:17:46.221 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:17:46.221 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:17:46.221 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:17:46.221 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:17:46.221 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:17:46.221 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:17:46.221 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:17:46.221 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:17:46.221 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:17:46.221 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:17:46.221 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:17:46.221 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:17:46.221 | INFO | ------------------------------------------ | 20260506 11:17:46.221 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:17:46.222 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:17:46.222 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:17:46.222 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:17:46.222 | INFO | PerfReport: Full Execution -- 3.23e+01 / 7.57e+01, 1.38e+01 / 2.27e+00 | 20260506 11:17:46.222 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.04e-01, 1.03e-02 / 7.62e-03 | 20260506 11:17:46.222 | INFO | PerfReport: 11.59%, Baseline -- 3.74e+00 / 4.43e+00, 5.89e-01 / 2.44e-01 | 20260506 11:17:46.222 | INFO | PerfReport: 3.25%, Histogram -- 1.05e+00 / 1.50e+00, 2.00e-01 / 1.20e-01 | 20260506 11:17:46.222 | INFO | PerfReport: 15.77%, DME -- 4.08e+01 / 4.50e+01, 4.12e+00 / 2.07e+00 | 20260506 11:17:46.222 | INFO | PerfReport: 56.13%, FrameLabeling -- 1.81e+01 / 1.94e+01, 1.69e+00 / 1.42e+00 | 20260506 11:17:46.222 | INFO | PerfReport: 7.70%, PulseAccumulating -- 2.49e+00 / 3.04e+00, 4.40e-01 / 2.71e-01 | 20260506 11:17:46.222 | INFO | PerfReport: 3.91%, Metrics -- 1.26e+00 / 1.66e+00, 3.19e-01 / 1.64e-01 | 20260506 11:17:46.222 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.41e-01 / 5.00e-01, 6.99e-02 / 2.30e-02 | 20260506 11:17:46.222 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:17:46.222 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:17:46.222 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:17:46.222 | INFO | PerfReport: Full Execution -- 1.00e+02 / -nan, 2.50e+01 / 0.00e+00 | 20260506 11:17:46.222 | INFO | PerfReport: 21.69%, Upload -- 2.17e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260506 11:17:46.222 | INFO | PerfReport: 21.34%, Download -- 2.14e+01 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260506 11:17:46.222 | INFO | PerfReport: 32.30%, Compute -- 3.23e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260506 11:17:46.222 | INFO | PerfReport: 2.07%, LockUpload -- 2.07e+00 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 11:17:46.222 | INFO | PerfReport: 1.07%, LockDownload -- 1.07e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260506 11:17:46.222 | INFO | PerfReport: 21.51%, LockCompute -- 2.15e+01 / 0.00e+00, 1.89e+01 / 0.00e+00 | 20260506 11:17:46.222 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 3.44e-02 / 0.00e+00 | 20260506 11:17:46.352 | INFO | Starting a round of Baz IO | 20260506 11:17:49.525 | INFO | Populating RTMetrics report | 20260506 11:17:49.525 | INFO | RTMetrics report populated | 20260506 11:17:49.525 | INFO | Opening tmp RTMetrics json file | 20260506 11:17:49.527 | INFO | Serializing RTMetrics to json | 20260506 11:17:49.529 | INFO | Serialization done, swapping tmp file now... | 20260506 11:17:49.530 | INFO | RTMetrics done | 20260506 11:17:49.676 | INFO | Finished a round of Baz IO: Wrote 2.05 GiB in 3.324 seconds | 20260506 11:17:49.676 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.64% / 0.04% / 55.32% | 20260506 11:17:49.676 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.41449% / 7.15256e-05% / 2.78155e-05% | 20260506 11:17:49.676 | INFO | Baz IO overall: | 20260506 11:17:49.676 | INFO | BytesWritten : 444170267806 | 20260506 11:17:49.676 | INFO | NumEvents : 214209201587 | 20260506 11:17:51.360 | INFO | Analyzing chunk frames = [933888,934400) | 20260506 11:17:51.360 | INFO | MemInfo: { | 20260506 11:17:51.360 | INFO | "gpu" : " 17 GB", | 20260506 11:17:51.360 | INFO | "huge" : "", | 20260506 11:17:51.360 | INFO | "huge_shared" : " 27 GB", | 20260506 11:17:51.360 | INFO | "malloc" : " 14 GB", | 20260506 11:17:51.360 | INFO | "pinned" : " 29 GB" | 20260506 11:17:51.360 | INFO | } | 20260506 11:17:51.360 | INFO | Perf report for the last 5.13973 seconds: | 20260506 11:17:51.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:17:51.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:51.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:51.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:51.360 | INFO | ANALYSIS : 63.7%, 2.9, 95.8ms, 0 | 20260506 11:17:51.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:51.360 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 11:17:51.360 | INFO | PRE_HQ : 62.0%, 1.9, 61.7ms, 0 | 20260506 11:17:51.360 | INFO | BAZWRITER : 63.8%, 4.6, 151.3ms, 0 | 20260506 11:17:51.360 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 11:17:51.360 | INFO | Uploaded 36GiB to GPU in 2.12851s (16.9133GiB/s) | 20260506 11:17:51.360 | INFO | Downloaded 38GiB to GPU in 2.23577s (16.9963GiB/s) | 20260506 11:17:56.499 | INFO | Analyzing chunk frames = [934400,934912) | 20260506 11:17:56.499 | INFO | MemInfo: { | 20260506 11:17:56.499 | INFO | "gpu" : " 17 GB", | 20260506 11:17:56.499 | INFO | "huge" : "", | 20260506 11:17:56.499 | INFO | "huge_shared" : " 27 GB", | 20260506 11:17:56.499 | INFO | "malloc" : " 14 GB", | 20260506 11:17:56.499 | INFO | "pinned" : " 29 GB" | 20260506 11:17:56.499 | INFO | } | 20260506 11:17:56.499 | INFO | Perf report for the last 5.13954 seconds: | 20260506 11:17:56.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:17:56.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:56.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:56.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:56.499 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 11:17:56.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:56.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:56.499 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:17:56.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:56.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:17:56.499 | INFO | Uploaded 36GiB to GPU in 2.02233s (17.8012GiB/s) | 20260506 11:17:56.500 | INFO | Downloaded 34GiB to GPU in 1.93992s (17.5265GiB/s) | 20260506 11:18:01.638 | INFO | Analyzing chunk frames = [934912,935424) | 20260506 11:18:01.638 | INFO | MemInfo: { | 20260506 11:18:01.638 | INFO | "gpu" : " 17 GB", | 20260506 11:18:01.638 | INFO | "huge" : "", | 20260506 11:18:01.638 | INFO | "huge_shared" : " 27 GB", | 20260506 11:18:01.638 | INFO | "malloc" : " 14 GB", | 20260506 11:18:01.638 | INFO | "pinned" : " 29 GB" | 20260506 11:18:01.638 | INFO | } | 20260506 11:18:01.638 | INFO | Perf report for the last 5.1392 seconds: | 20260506 11:18:01.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:18:01.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:01.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:01.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:01.638 | INFO | ANALYSIS : 61.9%, 3.0, 96.0ms, 0 | 20260506 11:18:01.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:01.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:01.639 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 11:18:01.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:01.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:01.639 | INFO | Uploaded 36GiB to GPU in 2.0093s (17.9167GiB/s) | 20260506 11:18:01.639 | INFO | Downloaded 34GiB to GPU in 1.93954s (17.5299GiB/s) | 20260506 11:18:06.777 | INFO | Analyzing chunk frames = [935424,935936) | 20260506 11:18:06.777 | INFO | MemInfo: { | 20260506 11:18:06.777 | INFO | "gpu" : " 17 GB", | 20260506 11:18:06.777 | INFO | "huge" : "", | 20260506 11:18:06.777 | INFO | "huge_shared" : " 27 GB", | 20260506 11:18:06.777 | INFO | "malloc" : " 14 GB", | 20260506 11:18:06.777 | INFO | "pinned" : " 29 GB" | 20260506 11:18:06.777 | INFO | } | 20260506 11:18:06.777 | INFO | Perf report for the last 5.13867 seconds: | 20260506 11:18:06.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:18:06.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:06.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:06.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:06.777 | INFO | ANALYSIS : 62.4%, 2.9, 96.4ms, 0 | 20260506 11:18:06.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:06.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:06.777 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:18:06.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:06.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:06.778 | INFO | Uploaded 36GiB to GPU in 2.06979s (17.3931GiB/s) | 20260506 11:18:06.778 | INFO | Downloaded 34GiB to GPU in 1.97991s (17.1725GiB/s) | 20260506 11:18:11.916 | INFO | Analyzing chunk frames = [935936,936448) | 20260506 11:18:11.916 | INFO | MemInfo: { | 20260506 11:18:11.916 | INFO | "gpu" : " 17 GB", | 20260506 11:18:11.916 | INFO | "huge" : "", | 20260506 11:18:11.916 | INFO | "huge_shared" : " 27 GB", | 20260506 11:18:11.916 | INFO | "malloc" : " 14 GB", | 20260506 11:18:11.916 | INFO | "pinned" : " 29 GB" | 20260506 11:18:11.916 | INFO | } | 20260506 11:18:11.917 | INFO | Perf report for the last 5.13949 seconds: | 20260506 11:18:11.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:18:11.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:11.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:11.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:11.917 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 11:18:11.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:11.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:11.917 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:18:11.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:11.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:11.917 | INFO | Uploaded 36GiB to GPU in 2.04467s (17.6067GiB/s) | 20260506 11:18:11.917 | INFO | Downloaded 34GiB to GPU in 1.95285s (17.4105GiB/s) | 20260506 11:18:17.056 | INFO | Analyzing chunk frames = [936448,936960) | 20260506 11:18:17.056 | INFO | MemInfo: { | 20260506 11:18:17.056 | INFO | "gpu" : " 17 GB", | 20260506 11:18:17.056 | INFO | "huge" : "", | 20260506 11:18:17.056 | INFO | "huge_shared" : " 27 GB", | 20260506 11:18:17.056 | INFO | "malloc" : " 14 GB", | 20260506 11:18:17.056 | INFO | "pinned" : " 29 GB" | 20260506 11:18:17.056 | INFO | } | 20260506 11:18:17.056 | INFO | Perf report for the last 5.13929 seconds: | 20260506 11:18:17.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:18:17.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:17.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:17.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:17.056 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 11:18:17.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:17.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:17.056 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 11:18:17.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:17.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:17.057 | INFO | Uploaded 36GiB to GPU in 2.08134s (17.2965GiB/s) | 20260506 11:18:17.057 | INFO | Downloaded 34GiB to GPU in 1.92855s (17.6298GiB/s) | 20260506 11:18:22.195 | INFO | Analyzing chunk frames = [936960,937472) | 20260506 11:18:22.195 | INFO | MemInfo: { | 20260506 11:18:22.195 | INFO | "gpu" : " 17 GB", | 20260506 11:18:22.195 | INFO | "huge" : "", | 20260506 11:18:22.195 | INFO | "huge_shared" : " 27 GB", | 20260506 11:18:22.195 | INFO | "malloc" : " 14 GB", | 20260506 11:18:22.195 | INFO | "pinned" : " 29 GB" | 20260506 11:18:22.195 | INFO | } | 20260506 11:18:22.196 | INFO | Perf report for the last 5.13959 seconds: | 20260506 11:18:22.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:18:22.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:22.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:22.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:22.196 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 11:18:22.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:22.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:22.196 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:18:22.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:22.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:22.196 | INFO | Uploaded 36GiB to GPU in 1.98508s (18.1353GiB/s) | 20260506 11:18:22.196 | INFO | Downloaded 34GiB to GPU in 2.02843s (16.7618GiB/s) | 20260506 11:18:27.335 | INFO | Analyzing chunk frames = [937472,937984) | 20260506 11:18:27.335 | INFO | MemInfo: { | 20260506 11:18:27.335 | INFO | "gpu" : " 17 GB", | 20260506 11:18:27.335 | INFO | "huge" : "", | 20260506 11:18:27.335 | INFO | "huge_shared" : " 27 GB", | 20260506 11:18:27.335 | INFO | "malloc" : " 14 GB", | 20260506 11:18:27.335 | INFO | "pinned" : " 29 GB" | 20260506 11:18:27.335 | INFO | } | 20260506 11:18:27.335 | INFO | Perf report for the last 5.13952 seconds: | 20260506 11:18:27.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:18:27.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:27.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:27.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:27.335 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260506 11:18:27.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:27.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:27.335 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:18:27.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:27.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:27.335 | INFO | Uploaded 36GiB to GPU in 2.05915s (17.4829GiB/s) | 20260506 11:18:27.335 | INFO | Downloaded 34GiB to GPU in 1.96597s (17.2942GiB/s) | 20260506 11:18:27.458 | INFO | Starting a round of Baz IO | 20260506 11:18:30.657 | INFO | Populating RTMetrics report | 20260506 11:18:30.657 | INFO | RTMetrics report populated | 20260506 11:18:30.657 | INFO | Opening tmp RTMetrics json file | 20260506 11:18:30.675 | INFO | Serializing RTMetrics to json | 20260506 11:18:30.678 | INFO | Serialization done, swapping tmp file now... | 20260506 11:18:30.683 | INFO | RTMetrics done | 20260506 11:18:31.087 | INFO | Finished a round of Baz IO: Wrote 2.047 GiB in 3.628 seconds | 20260506 11:18:31.087 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.61% / 0.04% / 55.36% | 20260506 11:18:31.087 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.36836% / 5.96046e-05% / 3.97364e-05% | 20260506 11:18:31.087 | INFO | Baz IO overall: | 20260506 11:18:31.087 | INFO | BytesWritten : 446367837044 | 20260506 11:18:31.087 | INFO | NumEvents : 215275854876 | 20260506 11:18:32.474 | INFO | Analyzing chunk frames = [937984,938496) | 20260506 11:18:32.474 | INFO | MemInfo: { | 20260506 11:18:32.474 | INFO | "gpu" : " 17 GB", | 20260506 11:18:32.474 | INFO | "huge" : "", | 20260506 11:18:32.474 | INFO | "huge_shared" : " 27 GB", | 20260506 11:18:32.474 | INFO | "malloc" : " 14 GB", | 20260506 11:18:32.474 | INFO | "pinned" : " 29 GB" | 20260506 11:18:32.474 | INFO | } | 20260506 11:18:32.474 | INFO | Perf report for the last 5.13918 seconds: | 20260506 11:18:32.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:18:32.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:32.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:32.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:32.474 | INFO | ANALYSIS : 64.0%, 2.9, 96.5ms, 0 | 20260506 11:18:32.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:32.474 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 11:18:32.474 | INFO | PRE_HQ : 62.0%, 1.9, 61.4ms, 0 | 20260506 11:18:32.474 | INFO | BAZWRITER : 69.8%, 4.8, 172.4ms, 0 | 20260506 11:18:32.474 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 11:18:32.474 | INFO | Uploaded 36GiB to GPU in 2.13888s (16.8313GiB/s) | 20260506 11:18:32.475 | INFO | Downloaded 38GiB to GPU in 2.2386s (16.9749GiB/s) | 20260506 11:18:37.613 | INFO | Analyzing chunk frames = [938496,939008) | 20260506 11:18:37.613 | INFO | MemInfo: { | 20260506 11:18:37.613 | INFO | "gpu" : " 17 GB", | 20260506 11:18:37.613 | INFO | "huge" : "", | 20260506 11:18:37.614 | INFO | "huge_shared" : " 27 GB", | 20260506 11:18:37.614 | INFO | "malloc" : " 14 GB", | 20260506 11:18:37.614 | INFO | "pinned" : " 29 GB" | 20260506 11:18:37.614 | INFO | } | 20260506 11:18:37.614 | INFO | Perf report for the last 5.13951 seconds: | 20260506 11:18:37.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:18:37.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:37.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:37.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:37.614 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 11:18:37.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:37.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:37.614 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:18:37.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:37.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:37.614 | INFO | Uploaded 36GiB to GPU in 2.01038s (17.9071GiB/s) | 20260506 11:18:37.614 | INFO | Downloaded 34GiB to GPU in 1.90801s (17.8196GiB/s) | 20260506 11:18:42.753 | INFO | Analyzing chunk frames = [939008,939520) | 20260506 11:18:42.753 | INFO | MemInfo: { | 20260506 11:18:42.753 | INFO | "gpu" : " 17 GB", | 20260506 11:18:42.753 | INFO | "huge" : "", | 20260506 11:18:42.753 | INFO | "huge_shared" : " 27 GB", | 20260506 11:18:42.753 | INFO | "malloc" : " 14 GB", | 20260506 11:18:42.753 | INFO | "pinned" : " 29 GB" | 20260506 11:18:42.753 | INFO | } | 20260506 11:18:42.753 | INFO | Perf report for the last 5.13896 seconds: | 20260506 11:18:42.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:18:42.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:42.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:42.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:42.753 | INFO | ANALYSIS : 62.0%, 3.0, 96.1ms, 0 | 20260506 11:18:42.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:42.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:42.753 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:18:42.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:42.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:42.753 | INFO | Uploaded 36GiB to GPU in 2.01138s (17.8982GiB/s) | 20260506 11:18:42.753 | INFO | Downloaded 34GiB to GPU in 1.93667s (17.5559GiB/s) | 20260506 11:18:47.892 | INFO | Analyzing chunk frames = [939520,940032) | 20260506 11:18:47.892 | INFO | MemInfo: { | 20260506 11:18:47.892 | INFO | "gpu" : " 17 GB", | 20260506 11:18:47.892 | INFO | "huge" : "", | 20260506 11:18:47.892 | INFO | "huge_shared" : " 27 GB", | 20260506 11:18:47.892 | INFO | "malloc" : " 14 GB", | 20260506 11:18:47.892 | INFO | "pinned" : " 29 GB" | 20260506 11:18:47.892 | INFO | } | 20260506 11:18:47.892 | INFO | Perf report for the last 5.1396 seconds: | 20260506 11:18:47.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:18:47.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:47.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:47.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:47.892 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260506 11:18:47.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:47.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:47.893 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 11:18:47.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:47.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:47.893 | INFO | Uploaded 36GiB to GPU in 2.0832s (17.2811GiB/s) | 20260506 11:18:47.893 | INFO | Downloaded 34GiB to GPU in 1.96955s (17.2628GiB/s) | 20260506 11:18:53.031 | INFO | Analyzing chunk frames = [940032,940544) | 20260506 11:18:53.031 | INFO | MemInfo: { | 20260506 11:18:53.031 | INFO | "gpu" : " 17 GB", | 20260506 11:18:53.031 | INFO | "huge" : "", | 20260506 11:18:53.031 | INFO | "huge_shared" : " 27 GB", | 20260506 11:18:53.032 | INFO | "malloc" : " 14 GB", | 20260506 11:18:53.032 | INFO | "pinned" : " 29 GB" | 20260506 11:18:53.032 | INFO | } | 20260506 11:18:53.032 | INFO | Perf report for the last 5.13931 seconds: | 20260506 11:18:53.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:18:53.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:53.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:53.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:53.032 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 11:18:53.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:53.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:53.032 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:18:53.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:53.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:53.032 | INFO | Uploaded 36GiB to GPU in 2.09232s (17.2058GiB/s) | 20260506 11:18:53.032 | INFO | Downloaded 34GiB to GPU in 1.98518s (17.1269GiB/s) | 20260506 11:18:58.171 | INFO | Analyzing chunk frames = [940544,941056) | 20260506 11:18:58.171 | INFO | MemInfo: { | 20260506 11:18:58.171 | INFO | "gpu" : " 17 GB", | 20260506 11:18:58.171 | INFO | "huge" : "", | 20260506 11:18:58.171 | INFO | "huge_shared" : " 27 GB", | 20260506 11:18:58.171 | INFO | "malloc" : " 14 GB", | 20260506 11:18:58.171 | INFO | "pinned" : " 29 GB" | 20260506 11:18:58.171 | INFO | } | 20260506 11:18:58.171 | INFO | Perf report for the last 5.13912 seconds: | 20260506 11:18:58.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:18:58.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:58.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:58.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:58.171 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 11:18:58.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:58.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:58.171 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:18:58.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:58.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:18:58.171 | INFO | Uploaded 36GiB to GPU in 2.0213s (17.8103GiB/s) | 20260506 11:18:58.171 | INFO | Downloaded 34GiB to GPU in 1.9993s (17.006GiB/s) | 20260506 11:19:03.310 | INFO | Analyzing chunk frames = [941056,941568) | 20260506 11:19:03.310 | INFO | MemInfo: { | 20260506 11:19:03.310 | INFO | "gpu" : " 17 GB", | 20260506 11:19:03.310 | INFO | "huge" : "", | 20260506 11:19:03.310 | INFO | "huge_shared" : " 27 GB", | 20260506 11:19:03.310 | INFO | "malloc" : " 14 GB", | 20260506 11:19:03.310 | INFO | "pinned" : " 29 GB" | 20260506 11:19:03.310 | INFO | } | 20260506 11:19:03.310 | INFO | Perf report for the last 5.13939 seconds: | 20260506 11:19:03.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:19:03.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:03.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:03.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:03.310 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 11:19:03.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:03.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:03.310 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 11:19:03.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:03.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:03.310 | INFO | Uploaded 36GiB to GPU in 2.0066s (17.9408GiB/s) | 20260506 11:19:03.310 | INFO | Downloaded 34GiB to GPU in 2.02731s (16.771GiB/s) | 20260506 11:19:08.449 | INFO | Analyzing chunk frames = [941568,942080) | 20260506 11:19:08.449 | INFO | MemInfo: { | 20260506 11:19:08.449 | INFO | "gpu" : " 17 GB", | 20260506 11:19:08.449 | INFO | "huge" : "", | 20260506 11:19:08.449 | INFO | "huge_shared" : " 27 GB", | 20260506 11:19:08.449 | INFO | "malloc" : " 14 GB", | 20260506 11:19:08.449 | INFO | "pinned" : " 29 GB" | 20260506 11:19:08.449 | INFO | } | 20260506 11:19:08.450 | INFO | Perf report for the last 5.13937 seconds: | 20260506 11:19:08.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:19:08.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:08.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:08.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:08.450 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260506 11:19:08.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:08.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:08.450 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:19:08.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:08.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:08.450 | INFO | Uploaded 36GiB to GPU in 2.0398s (17.6488GiB/s) | 20260506 11:19:08.450 | INFO | Downloaded 34GiB to GPU in 1.96658s (17.2889GiB/s) | 20260506 11:19:08.450 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:19:08.450 | INFO | Total mem usage: 16.8 GiB | 20260506 11:19:08.450 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:19:08.450 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -2.65%) | 20260506 11:19:08.450 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:19:08.450 | INFO | High Water Breakdown: | 20260506 11:19:08.450 | INFO | 61.4% : 10.1 GiB ( 96.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:19:08.450 | INFO | 38.5% : 6.3 GiB ( 94.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:19:08.450 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:19:08.450 | INFO | ------------------------------------------ | 20260506 11:19:08.450 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:19:08.450 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:19:08.450 | INFO | Total mem usage: 26.5 GiB | 20260506 11:19:08.450 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:19:08.450 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:19:08.450 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:19:08.450 | INFO | High Water Breakdown: | 20260506 11:19:08.450 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:19:08.450 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:19:08.450 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:19:08.450 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:19:08.450 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:19:08.450 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:19:08.450 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:19:08.450 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:19:08.450 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:19:08.450 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:19:08.450 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:19:08.450 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:19:08.450 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:19:08.450 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:19:08.450 | INFO | ------------------------------------------ | 20260506 11:19:08.450 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:19:08.450 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:19:08.450 | INFO | Total mem usage: 25.6 GiB | 20260506 11:19:08.451 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:19:08.451 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:19:08.451 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:19:08.451 | INFO | High Water Breakdown: | 20260506 11:19:08.451 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:19:08.451 | INFO | ------------------------------------------ | 20260506 11:19:08.451 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:19:08.451 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:19:08.451 | INFO | Total mem usage: 15.7 GiB | 20260506 11:19:08.451 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:19:08.451 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:19:08.451 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 11:19:08.451 | INFO | High Water Breakdown: | 20260506 11:19:08.451 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:19:08.451 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:19:08.451 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:19:08.451 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:19:08.451 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:19:08.451 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:19:08.451 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:19:08.451 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:19:08.451 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:19:08.451 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:19:08.451 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:19:08.451 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:19:08.451 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:19:08.451 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:19:08.451 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:19:08.451 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:19:08.451 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:19:08.451 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:19:08.451 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:19:08.451 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:19:08.451 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:19:08.451 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:19:08.451 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:19:08.451 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:19:08.451 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:19:08.451 | INFO | ------------------------------------------ | 20260506 11:19:08.451 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:19:08.451 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:19:08.451 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:19:08.451 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:19:08.451 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.61e+01, 1.36e+01 / 3.02e+00 | 20260506 11:19:08.451 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 9.96e-02, 1.00e-02 / 6.07e-03 | 20260506 11:19:08.451 | INFO | PerfReport: 11.52%, Baseline -- 3.67e+00 / 4.39e+00, 5.69e-01 / 3.30e-01 | 20260506 11:19:08.451 | INFO | PerfReport: 3.23%, Histogram -- 1.03e+00 / 1.46e+00, 1.89e-01 / 1.45e-01 | 20260506 11:19:08.451 | INFO | PerfReport: 15.94%, DME -- 4.06e+01 / 4.53e+01, 4.36e+00 / 3.23e+00 | 20260506 11:19:08.451 | INFO | PerfReport: 56.06%, FrameLabeling -- 1.79e+01 / 1.96e+01, 1.31e+00 / 1.49e+00 | 20260506 11:19:08.451 | INFO | PerfReport: 7.72%, PulseAccumulating -- 2.46e+00 / 3.13e+00, 4.33e-01 / 1.66e-01 | 20260506 11:19:08.451 | INFO | PerfReport: 3.87%, Metrics -- 1.23e+00 / 1.61e+00, 3.10e-01 / 1.32e-01 | 20260506 11:19:08.451 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.37e-01 / 4.93e-01, 6.81e-02 / 2.51e-02 | 20260506 11:19:08.451 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:19:08.451 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:19:08.451 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:19:08.451 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 11:19:08.451 | INFO | PerfReport: 21.61%, Upload -- 2.14e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260506 11:19:08.451 | INFO | PerfReport: 21.08%, Download -- 2.08e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 11:19:08.451 | INFO | PerfReport: 32.22%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 11:19:08.451 | INFO | PerfReport: 1.92%, LockUpload -- 1.90e+00 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 11:19:08.451 | INFO | PerfReport: 1.03%, LockDownload -- 1.02e+00 / 0.00e+00, 2.66e+00 / 0.00e+00 | 20260506 11:19:08.451 | INFO | PerfReport: 22.13%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 11:19:08.451 | INFO | PerfReport: 0.02%, Misc -- 1.54e-02 / 0.00e+00, 2.79e-02 / 0.00e+00 | 20260506 11:19:08.574 | INFO | Starting a round of Baz IO | 20260506 11:19:11.764 | INFO | Populating RTMetrics report | 20260506 11:19:11.764 | INFO | RTMetrics report populated | 20260506 11:19:11.764 | INFO | Opening tmp RTMetrics json file | 20260506 11:19:11.770 | INFO | Serializing RTMetrics to json | 20260506 11:19:11.806 | INFO | Serialization done, swapping tmp file now... | 20260506 11:19:11.855 | INFO | RTMetrics done | 20260506 11:19:11.927 | INFO | Finished a round of Baz IO: Wrote 2.051 GiB in 3.352 seconds | 20260506 11:19:11.927 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.57% / 0.04% / 55.39% | 20260506 11:19:11.927 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.37477% / 7.15256e-05% / 5.16574e-05% | 20260506 11:19:11.927 | INFO | Baz IO overall: | 20260506 11:19:11.928 | INFO | BytesWritten : 448570014443 | 20260506 11:19:11.928 | INFO | NumEvents : 216345764608 | 20260506 11:19:13.589 | INFO | Analyzing chunk frames = [942080,942592) | 20260506 11:19:13.589 | INFO | MemInfo: { | 20260506 11:19:13.589 | INFO | "gpu" : " 17 GB", | 20260506 11:19:13.589 | INFO | "huge" : "", | 20260506 11:19:13.589 | INFO | "huge_shared" : " 27 GB", | 20260506 11:19:13.589 | INFO | "malloc" : " 14 GB", | 20260506 11:19:13.589 | INFO | "pinned" : " 29 GB" | 20260506 11:19:13.589 | INFO | } | 20260506 11:19:13.589 | INFO | Perf report for the last 5.1396 seconds: | 20260506 11:19:13.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:19:13.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:13.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:13.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:13.589 | INFO | ANALYSIS : 63.9%, 2.9, 96.7ms, 0 | 20260506 11:19:13.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:13.589 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260506 11:19:13.589 | INFO | PRE_HQ : 61.9%, 1.9, 61.6ms, 0 | 20260506 11:19:13.589 | INFO | BAZWRITER : 64.4%, 4.5, 149.6ms, 0 | 20260506 11:19:13.589 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 11:19:13.589 | INFO | Uploaded 36GiB to GPU in 2.11411s (17.0285GiB/s) | 20260506 11:19:13.590 | INFO | Downloaded 38GiB to GPU in 2.22847s (17.052GiB/s) | 20260506 11:19:18.728 | INFO | Analyzing chunk frames = [942592,943104) | 20260506 11:19:18.728 | INFO | MemInfo: { | 20260506 11:19:18.728 | INFO | "gpu" : " 17 GB", | 20260506 11:19:18.728 | INFO | "huge" : "", | 20260506 11:19:18.728 | INFO | "huge_shared" : " 27 GB", | 20260506 11:19:18.728 | INFO | "malloc" : " 14 GB", | 20260506 11:19:18.728 | INFO | "pinned" : " 29 GB" | 20260506 11:19:18.728 | INFO | } | 20260506 11:19:18.728 | INFO | Perf report for the last 5.13913 seconds: | 20260506 11:19:18.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:19:18.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:18.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:18.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:18.728 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:19:18.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:18.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:18.728 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:19:18.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:18.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:18.729 | INFO | Uploaded 36GiB to GPU in 2.00975s (17.9127GiB/s) | 20260506 11:19:18.729 | INFO | Downloaded 34GiB to GPU in 1.9098s (17.803GiB/s) | 20260506 11:19:23.867 | INFO | Analyzing chunk frames = [943104,943616) | 20260506 11:19:23.867 | INFO | MemInfo: { | 20260506 11:19:23.867 | INFO | "gpu" : " 17 GB", | 20260506 11:19:23.867 | INFO | "huge" : "", | 20260506 11:19:23.867 | INFO | "huge_shared" : " 27 GB", | 20260506 11:19:23.867 | INFO | "malloc" : " 14 GB", | 20260506 11:19:23.867 | INFO | "pinned" : " 29 GB" | 20260506 11:19:23.867 | INFO | } | 20260506 11:19:23.867 | INFO | Perf report for the last 5.1391 seconds: | 20260506 11:19:23.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:19:23.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:23.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:23.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:23.867 | INFO | ANALYSIS : 62.0%, 3.0, 96.1ms, 0 | 20260506 11:19:23.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:23.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:23.868 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:19:23.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:23.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:23.868 | INFO | Uploaded 36GiB to GPU in 2.0178s (17.8413GiB/s) | 20260506 11:19:23.868 | INFO | Downloaded 34GiB to GPU in 1.93833s (17.5409GiB/s) | 20260506 11:19:29.007 | INFO | Analyzing chunk frames = [943616,944128) | 20260506 11:19:29.007 | INFO | MemInfo: { | 20260506 11:19:29.007 | INFO | "gpu" : " 17 GB", | 20260506 11:19:29.007 | INFO | "huge" : "", | 20260506 11:19:29.007 | INFO | "huge_shared" : " 27 GB", | 20260506 11:19:29.007 | INFO | "malloc" : " 14 GB", | 20260506 11:19:29.007 | INFO | "pinned" : " 29 GB" | 20260506 11:19:29.007 | INFO | } | 20260506 11:19:29.007 | INFO | Perf report for the last 5.13975 seconds: | 20260506 11:19:29.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:19:29.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:29.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:29.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:29.007 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260506 11:19:29.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:29.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:29.007 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:19:29.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:29.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:29.007 | INFO | Uploaded 36GiB to GPU in 2.08063s (17.3024GiB/s) | 20260506 11:19:29.008 | INFO | Downloaded 34GiB to GPU in 1.97087s (17.2513GiB/s) | 20260506 11:19:34.146 | INFO | Analyzing chunk frames = [944128,944640) | 20260506 11:19:34.146 | INFO | MemInfo: { | 20260506 11:19:34.146 | INFO | "gpu" : " 17 GB", | 20260506 11:19:34.146 | INFO | "huge" : "", | 20260506 11:19:34.146 | INFO | "huge_shared" : " 27 GB", | 20260506 11:19:34.146 | INFO | "malloc" : " 14 GB", | 20260506 11:19:34.146 | INFO | "pinned" : " 29 GB" | 20260506 11:19:34.146 | INFO | } | 20260506 11:19:34.146 | INFO | Perf report for the last 5.13913 seconds: | 20260506 11:19:34.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:19:34.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:34.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:34.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:34.146 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 11:19:34.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:34.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:34.146 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:19:34.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:34.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:34.147 | INFO | Uploaded 36GiB to GPU in 2.10873s (17.0719GiB/s) | 20260506 11:19:34.147 | INFO | Downloaded 34GiB to GPU in 1.98983s (17.0869GiB/s) | 20260506 11:19:39.286 | INFO | Analyzing chunk frames = [944640,945152) | 20260506 11:19:39.286 | INFO | MemInfo: { | 20260506 11:19:39.286 | INFO | "gpu" : " 17 GB", | 20260506 11:19:39.286 | INFO | "huge" : "", | 20260506 11:19:39.286 | INFO | "huge_shared" : " 27 GB", | 20260506 11:19:39.286 | INFO | "malloc" : " 14 GB", | 20260506 11:19:39.286 | INFO | "pinned" : " 29 GB" | 20260506 11:19:39.286 | INFO | } | 20260506 11:19:39.286 | INFO | Perf report for the last 5.13973 seconds: | 20260506 11:19:39.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:19:39.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:39.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:39.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:39.286 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 11:19:39.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:39.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:39.286 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:19:39.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:39.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:39.286 | INFO | Uploaded 36GiB to GPU in 2.07234s (17.3716GiB/s) | 20260506 11:19:39.286 | INFO | Downloaded 34GiB to GPU in 2.00129s (16.989GiB/s) | 20260506 11:19:44.425 | INFO | Analyzing chunk frames = [945152,945664) | 20260506 11:19:44.425 | INFO | MemInfo: { | 20260506 11:19:44.425 | INFO | "gpu" : " 17 GB", | 20260506 11:19:44.425 | INFO | "huge" : "", | 20260506 11:19:44.425 | INFO | "huge_shared" : " 27 GB", | 20260506 11:19:44.425 | INFO | "malloc" : " 14 GB", | 20260506 11:19:44.426 | INFO | "pinned" : " 29 GB" | 20260506 11:19:44.426 | INFO | } | 20260506 11:19:44.426 | INFO | Perf report for the last 5.13951 seconds: | 20260506 11:19:44.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:19:44.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:44.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:44.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:44.426 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:19:44.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:44.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:44.426 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 11:19:44.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:44.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:44.426 | INFO | Uploaded 36GiB to GPU in 1.99711s (18.026GiB/s) | 20260506 11:19:44.426 | INFO | Downloaded 34GiB to GPU in 1.99573s (17.0364GiB/s) | 20260506 11:19:49.564 | INFO | Analyzing chunk frames = [945664,946176) | 20260506 11:19:49.564 | INFO | MemInfo: { | 20260506 11:19:49.564 | INFO | "gpu" : " 17 GB", | 20260506 11:19:49.564 | INFO | "huge" : "", | 20260506 11:19:49.564 | INFO | "huge_shared" : " 27 GB", | 20260506 11:19:49.564 | INFO | "malloc" : " 14 GB", | 20260506 11:19:49.564 | INFO | "pinned" : " 29 GB" | 20260506 11:19:49.564 | INFO | } | 20260506 11:19:49.564 | INFO | Perf report for the last 5.13883 seconds: | 20260506 11:19:49.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:19:49.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:49.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:49.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:49.564 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260506 11:19:49.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:49.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:49.565 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:19:49.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:49.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:49.565 | INFO | Uploaded 36GiB to GPU in 2.06244s (17.455GiB/s) | 20260506 11:19:49.565 | INFO | Downloaded 34GiB to GPU in 1.96378s (17.3135GiB/s) | 20260506 11:19:49.690 | INFO | Starting a round of Baz IO | 20260506 11:19:52.874 | INFO | Populating RTMetrics report | 20260506 11:19:52.874 | INFO | RTMetrics report populated | 20260506 11:19:52.874 | INFO | Opening tmp RTMetrics json file | 20260506 11:19:52.879 | INFO | Serializing RTMetrics to json | 20260506 11:19:52.881 | INFO | Serialization done, swapping tmp file now... | 20260506 11:19:52.883 | INFO | RTMetrics done | 20260506 11:19:53.022 | INFO | Finished a round of Baz IO: Wrote 2.048 GiB in 3.332 seconds | 20260506 11:19:53.022 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.53% / 0.04% / 55.43% | 20260506 11:19:53.023 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.33009% / 6.35783e-05% / 6.35783e-05% | 20260506 11:19:53.023 | INFO | Baz IO overall: | 20260506 11:19:53.023 | INFO | BytesWritten : 450769100382 | 20260506 11:19:53.023 | INFO | NumEvents : 217412998414 | 20260506 11:19:54.703 | INFO | Analyzing chunk frames = [946176,946688) | 20260506 11:19:54.703 | INFO | MemInfo: { | 20260506 11:19:54.703 | INFO | "gpu" : " 17 GB", | 20260506 11:19:54.703 | INFO | "huge" : "", | 20260506 11:19:54.703 | INFO | "huge_shared" : " 27 GB", | 20260506 11:19:54.704 | INFO | "malloc" : " 14 GB", | 20260506 11:19:54.704 | INFO | "pinned" : " 29 GB" | 20260506 11:19:54.704 | INFO | } | 20260506 11:19:54.704 | INFO | Perf report for the last 5.13913 seconds: | 20260506 11:19:54.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:19:54.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:54.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:54.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:54.704 | INFO | ANALYSIS : 63.8%, 2.9, 96.1ms, 0 | 20260506 11:19:54.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:54.704 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 11:19:54.704 | INFO | PRE_HQ : 61.9%, 1.9, 61.5ms, 0 | 20260506 11:19:54.704 | INFO | BAZWRITER : 64.1%, 4.5, 149.6ms, 0 | 20260506 11:19:54.704 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 11:19:54.704 | INFO | Uploaded 36GiB to GPU in 2.15609s (16.6969GiB/s) | 20260506 11:19:54.704 | INFO | Downloaded 38GiB to GPU in 2.2342s (17.0083GiB/s) | 20260506 11:19:59.843 | INFO | Analyzing chunk frames = [946688,947200) | 20260506 11:19:59.843 | INFO | MemInfo: { | 20260506 11:19:59.843 | INFO | "gpu" : " 17 GB", | 20260506 11:19:59.843 | INFO | "huge" : "", | 20260506 11:19:59.843 | INFO | "huge_shared" : " 27 GB", | 20260506 11:19:59.843 | INFO | "malloc" : " 14 GB", | 20260506 11:19:59.843 | INFO | "pinned" : " 29 GB" | 20260506 11:19:59.843 | INFO | } | 20260506 11:19:59.843 | INFO | Perf report for the last 5.13964 seconds: | 20260506 11:19:59.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:19:59.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:59.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:59.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:59.843 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:19:59.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:59.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:59.843 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:19:59.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:59.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:19:59.844 | INFO | Uploaded 36GiB to GPU in 2.02635s (17.766GiB/s) | 20260506 11:19:59.844 | INFO | Downloaded 34GiB to GPU in 1.92284s (17.6821GiB/s) | 20260506 11:20:04.982 | INFO | Analyzing chunk frames = [947200,947712) | 20260506 11:20:04.982 | INFO | MemInfo: { | 20260506 11:20:04.982 | INFO | "gpu" : " 17 GB", | 20260506 11:20:04.982 | INFO | "huge" : "", | 20260506 11:20:04.982 | INFO | "huge_shared" : " 27 GB", | 20260506 11:20:04.982 | INFO | "malloc" : " 14 GB", | 20260506 11:20:04.982 | INFO | "pinned" : " 29 GB" | 20260506 11:20:04.982 | INFO | } | 20260506 11:20:04.982 | INFO | Perf report for the last 5.13918 seconds: | 20260506 11:20:04.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:20:04.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:04.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:04.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:04.982 | INFO | ANALYSIS : 62.3%, 3.0, 96.5ms, 0 | 20260506 11:20:04.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:04.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:04.983 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:20:04.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:04.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:04.983 | INFO | Uploaded 36GiB to GPU in 2.04065s (17.6415GiB/s) | 20260506 11:20:04.983 | INFO | Downloaded 34GiB to GPU in 1.93196s (17.5987GiB/s) | 20260506 11:20:10.122 | INFO | Analyzing chunk frames = [947712,948224) | 20260506 11:20:10.122 | INFO | MemInfo: { | 20260506 11:20:10.122 | INFO | "gpu" : " 17 GB", | 20260506 11:20:10.122 | INFO | "huge" : "", | 20260506 11:20:10.122 | INFO | "huge_shared" : " 27 GB", | 20260506 11:20:10.122 | INFO | "malloc" : " 14 GB", | 20260506 11:20:10.122 | INFO | "pinned" : " 29 GB" | 20260506 11:20:10.122 | INFO | } | 20260506 11:20:10.122 | INFO | Perf report for the last 5.13942 seconds: | 20260506 11:20:10.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:20:10.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:10.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:10.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:10.122 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 11:20:10.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:10.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:10.122 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:20:10.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:10.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:10.122 | INFO | Uploaded 36GiB to GPU in 2.07336s (17.3631GiB/s) | 20260506 11:20:10.122 | INFO | Downloaded 34GiB to GPU in 1.95846s (17.3606GiB/s) | 20260506 11:20:15.261 | INFO | Analyzing chunk frames = [948224,948736) | 20260506 11:20:15.261 | INFO | MemInfo: { | 20260506 11:20:15.261 | INFO | "gpu" : " 17 GB", | 20260506 11:20:15.261 | INFO | "huge" : "", | 20260506 11:20:15.261 | INFO | "huge_shared" : " 27 GB", | 20260506 11:20:15.261 | INFO | "malloc" : " 14 GB", | 20260506 11:20:15.261 | INFO | "pinned" : " 29 GB" | 20260506 11:20:15.261 | INFO | } | 20260506 11:20:15.261 | INFO | Perf report for the last 5.13914 seconds: | 20260506 11:20:15.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:20:15.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:15.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:15.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:15.261 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 11:20:15.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:15.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:15.261 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:20:15.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:15.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:15.261 | INFO | Uploaded 36GiB to GPU in 2.1079s (17.0786GiB/s) | 20260506 11:20:15.261 | INFO | Downloaded 34GiB to GPU in 1.98171s (17.1569GiB/s) | 20260506 11:20:20.400 | INFO | Analyzing chunk frames = [948736,949248) | 20260506 11:20:20.400 | INFO | MemInfo: { | 20260506 11:20:20.400 | INFO | "gpu" : " 17 GB", | 20260506 11:20:20.400 | INFO | "huge" : "", | 20260506 11:20:20.400 | INFO | "huge_shared" : " 27 GB", | 20260506 11:20:20.400 | INFO | "malloc" : " 14 GB", | 20260506 11:20:20.400 | INFO | "pinned" : " 29 GB" | 20260506 11:20:20.400 | INFO | } | 20260506 11:20:20.400 | INFO | Perf report for the last 5.13899 seconds: | 20260506 11:20:20.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:20:20.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:20.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:20.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:20.400 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 11:20:20.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:20.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:20.400 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:20:20.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:20.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:20.401 | INFO | Uploaded 36GiB to GPU in 2.06829s (17.4057GiB/s) | 20260506 11:20:20.401 | INFO | Downloaded 34GiB to GPU in 1.99987s (17.0011GiB/s) | 20260506 11:20:25.539 | INFO | Analyzing chunk frames = [949248,949760) | 20260506 11:20:25.539 | INFO | MemInfo: { | 20260506 11:20:25.539 | INFO | "gpu" : " 17 GB", | 20260506 11:20:25.539 | INFO | "huge" : "", | 20260506 11:20:25.539 | INFO | "huge_shared" : " 27 GB", | 20260506 11:20:25.539 | INFO | "malloc" : " 14 GB", | 20260506 11:20:25.539 | INFO | "pinned" : " 29 GB" | 20260506 11:20:25.540 | INFO | } | 20260506 11:20:25.540 | INFO | Perf report for the last 5.13952 seconds: | 20260506 11:20:25.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:20:25.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:25.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:25.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:25.540 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260506 11:20:25.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:25.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:25.540 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:20:25.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:25.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:25.540 | INFO | Uploaded 36GiB to GPU in 2.0232s (17.7936GiB/s) | 20260506 11:20:25.540 | INFO | Downloaded 34GiB to GPU in 1.98295s (17.1462GiB/s) | 20260506 11:20:30.679 | INFO | Analyzing chunk frames = [949760,950272) | 20260506 11:20:30.679 | INFO | MemInfo: { | 20260506 11:20:30.679 | INFO | "gpu" : " 17 GB", | 20260506 11:20:30.679 | INFO | "huge" : "", | 20260506 11:20:30.679 | INFO | "huge_shared" : " 27 GB", | 20260506 11:20:30.679 | INFO | "malloc" : " 14 GB", | 20260506 11:20:30.679 | INFO | "pinned" : " 29 GB" | 20260506 11:20:30.679 | INFO | } | 20260506 11:20:30.679 | INFO | Perf report for the last 5.13924 seconds: | 20260506 11:20:30.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:20:30.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:30.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:30.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:30.679 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:20:30.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:30.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:30.679 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:20:30.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:30.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:30.679 | INFO | Uploaded 36GiB to GPU in 2.03231s (17.7138GiB/s) | 20260506 11:20:30.679 | INFO | Downloaded 34GiB to GPU in 1.95961s (17.3504GiB/s) | 20260506 11:20:30.679 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:20:30.679 | INFO | Total mem usage: 16.8 GiB | 20260506 11:20:30.679 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:20:30.679 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -13.1%) | 20260506 11:20:30.679 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:20:30.679 | INFO | High Water Breakdown: | 20260506 11:20:30.679 | INFO | 59.8% : 8.88 GiB ( 85.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:20:30.679 | INFO | 40.1% : 5.96 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:20:30.679 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:20:30.679 | INFO | ------------------------------------------ | 20260506 11:20:30.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:20:30.679 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:20:30.679 | INFO | Total mem usage: 26.5 GiB | 20260506 11:20:30.679 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:20:30.679 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:20:30.680 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:20:30.680 | INFO | High Water Breakdown: | 20260506 11:20:30.680 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:20:30.680 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:20:30.680 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:20:30.680 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:20:30.680 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:20:30.680 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:20:30.680 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:20:30.680 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:20:30.680 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:20:30.680 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:20:30.680 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:20:30.680 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:20:30.680 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:20:30.680 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:20:30.680 | INFO | ------------------------------------------ | 20260506 11:20:30.680 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:20:30.680 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:20:30.680 | INFO | Total mem usage: 25.6 GiB | 20260506 11:20:30.680 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:20:30.680 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:20:30.680 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:20:30.680 | INFO | High Water Breakdown: | 20260506 11:20:30.680 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:20:30.680 | INFO | ------------------------------------------ | 20260506 11:20:30.680 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:20:30.680 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:20:30.680 | INFO | Total mem usage: 15.7 GiB | 20260506 11:20:30.680 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:20:30.680 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:20:30.680 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 11:20:30.680 | INFO | High Water Breakdown: | 20260506 11:20:30.680 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:20:30.680 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:20:30.680 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:20:30.680 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:20:30.680 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:20:30.680 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:20:30.680 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:20:30.680 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:20:30.680 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:20:30.680 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:20:30.680 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:20:30.680 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:20:30.680 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:20:30.680 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:20:30.680 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:20:30.680 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:20:30.680 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:20:30.680 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:20:30.680 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:20:30.680 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:20:30.680 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:20:30.680 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:20:30.680 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:20:30.680 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:20:30.680 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:20:30.680 | INFO | ------------------------------------------ | 20260506 11:20:30.680 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:20:30.680 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:20:30.680 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:20:30.680 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:20:30.680 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.59e+01, 1.36e+01 / 1.99e+00 | 20260506 11:20:30.680 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.01e-01, 9.75e-03 / 5.52e-03 | 20260506 11:20:30.680 | INFO | PerfReport: 11.54%, Baseline -- 3.68e+00 / 4.47e+00, 5.79e-01 / 3.01e-01 | 20260506 11:20:30.680 | INFO | PerfReport: 3.23%, Histogram -- 1.03e+00 / 1.48e+00, 1.91e-01 / 1.28e-01 | 20260506 11:20:30.680 | INFO | PerfReport: 15.87%, DME -- 4.05e+01 / 4.49e+01, 4.00e+00 / 1.92e+00 | 20260506 11:20:30.680 | INFO | PerfReport: 56.08%, FrameLabeling -- 1.79e+01 / 1.96e+01, 1.35e+00 / 1.57e+00 | 20260506 11:20:30.680 | INFO | PerfReport: 7.74%, PulseAccumulating -- 2.47e+00 / 3.11e+00, 4.40e-01 / 1.73e-01 | 20260506 11:20:30.680 | INFO | PerfReport: 3.88%, Metrics -- 1.24e+00 / 1.63e+00, 3.12e-01 / 1.71e-01 | 20260506 11:20:30.680 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.37e-01 / 5.00e-01, 6.89e-02 / 3.13e-02 | 20260506 11:20:30.680 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:20:30.680 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:20:30.680 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:20:30.680 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260506 11:20:30.680 | INFO | PerfReport: 21.71%, Upload -- 2.15e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 11:20:30.681 | INFO | PerfReport: 21.04%, Download -- 2.08e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 11:20:30.681 | INFO | PerfReport: 32.21%, Compute -- 3.19e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 11:20:30.681 | INFO | PerfReport: 2.04%, LockUpload -- 2.02e+00 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 11:20:30.681 | INFO | PerfReport: 0.85%, LockDownload -- 8.41e-01 / 0.00e+00, 2.23e+00 / 0.00e+00 | 20260506 11:20:30.681 | INFO | PerfReport: 22.13%, LockCompute -- 2.19e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260506 11:20:30.681 | INFO | PerfReport: 0.02%, Misc -- 1.52e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260506 11:20:30.805 | INFO | Starting a round of Baz IO | 20260506 11:20:34.006 | INFO | Populating RTMetrics report | 20260506 11:20:34.007 | INFO | RTMetrics report populated | 20260506 11:20:34.007 | INFO | Opening tmp RTMetrics json file | 20260506 11:20:34.084 | INFO | Serializing RTMetrics to json | 20260506 11:20:34.109 | INFO | Serialization done, swapping tmp file now... | 20260506 11:20:34.113 | INFO | RTMetrics done | 20260506 11:20:34.244 | INFO | Finished a round of Baz IO: Wrote 2.048 GiB in 3.438 seconds | 20260506 11:20:34.244 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.49% / 0.04% / 55.47% | 20260506 11:20:34.244 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.30637% / 0.000123183% / 5.5631e-05% | 20260506 11:20:34.244 | INFO | Baz IO overall: | 20260506 11:20:34.244 | INFO | BytesWritten : 452968213443 | 20260506 11:20:34.244 | INFO | NumEvents : 218480225214 | 20260506 11:20:35.818 | INFO | Analyzing chunk frames = [950272,950784) | 20260506 11:20:35.818 | INFO | MemInfo: { | 20260506 11:20:35.818 | INFO | "gpu" : " 17 GB", | 20260506 11:20:35.818 | INFO | "huge" : "", | 20260506 11:20:35.818 | INFO | "huge_shared" : " 27 GB", | 20260506 11:20:35.818 | INFO | "malloc" : " 14 GB", | 20260506 11:20:35.818 | INFO | "pinned" : " 29 GB" | 20260506 11:20:35.818 | INFO | } | 20260506 11:20:35.818 | INFO | Perf report for the last 5.13921 seconds: | 20260506 11:20:35.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:20:35.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:35.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:35.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:35.818 | INFO | ANALYSIS : 64.0%, 2.9, 96.5ms, 0 | 20260506 11:20:35.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:35.818 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260506 11:20:35.818 | INFO | PRE_HQ : 61.9%, 1.9, 61.8ms, 0 | 20260506 11:20:35.818 | INFO | BAZWRITER : 66.1%, 4.8, 162.6ms, 0 | 20260506 11:20:35.818 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 11:20:35.818 | INFO | Uploaded 36GiB to GPU in 2.1463s (16.773GiB/s) | 20260506 11:20:35.819 | INFO | Downloaded 38GiB to GPU in 2.23339s (17.0145GiB/s) | 20260506 11:20:40.957 | INFO | Analyzing chunk frames = [950784,951296) | 20260506 11:20:40.957 | INFO | MemInfo: { | 20260506 11:20:40.957 | INFO | "gpu" : " 17 GB", | 20260506 11:20:40.957 | INFO | "huge" : "", | 20260506 11:20:40.957 | INFO | "huge_shared" : " 27 GB", | 20260506 11:20:40.957 | INFO | "malloc" : " 14 GB", | 20260506 11:20:40.957 | INFO | "pinned" : " 29 GB" | 20260506 11:20:40.958 | INFO | } | 20260506 11:20:40.958 | INFO | Perf report for the last 5.13955 seconds: | 20260506 11:20:40.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:20:40.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:40.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:40.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:40.958 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:20:40.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:40.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:40.958 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:20:40.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:40.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:40.958 | INFO | Uploaded 36GiB to GPU in 2.02102s (17.8128GiB/s) | 20260506 11:20:40.958 | INFO | Downloaded 34GiB to GPU in 1.92109s (17.6983GiB/s) | 20260506 11:20:46.097 | INFO | Analyzing chunk frames = [951296,951808) | 20260506 11:20:46.097 | INFO | MemInfo: { | 20260506 11:20:46.097 | INFO | "gpu" : " 17 GB", | 20260506 11:20:46.097 | INFO | "huge" : "", | 20260506 11:20:46.097 | INFO | "huge_shared" : " 27 GB", | 20260506 11:20:46.097 | INFO | "malloc" : " 14 GB", | 20260506 11:20:46.097 | INFO | "pinned" : " 29 GB" | 20260506 11:20:46.097 | INFO | } | 20260506 11:20:46.097 | INFO | Perf report for the last 5.13963 seconds: | 20260506 11:20:46.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:20:46.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:46.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:46.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:46.097 | INFO | ANALYSIS : 62.1%, 3.0, 96.2ms, 0 | 20260506 11:20:46.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:46.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:46.097 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 11:20:46.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:46.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:46.097 | INFO | Uploaded 36GiB to GPU in 2.04314s (17.6199GiB/s) | 20260506 11:20:46.097 | INFO | Downloaded 34GiB to GPU in 1.93534s (17.568GiB/s) | 20260506 11:20:51.236 | INFO | Analyzing chunk frames = [951808,952320) | 20260506 11:20:51.236 | INFO | MemInfo: { | 20260506 11:20:51.236 | INFO | "gpu" : " 17 GB", | 20260506 11:20:51.236 | INFO | "huge" : "", | 20260506 11:20:51.236 | INFO | "huge_shared" : " 27 GB", | 20260506 11:20:51.236 | INFO | "malloc" : " 14 GB", | 20260506 11:20:51.236 | INFO | "pinned" : " 29 GB" | 20260506 11:20:51.236 | INFO | } | 20260506 11:20:51.236 | INFO | Perf report for the last 5.13925 seconds: | 20260506 11:20:51.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:20:51.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:51.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:51.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:51.237 | INFO | ANALYSIS : 62.3%, 2.9, 96.2ms, 0 | 20260506 11:20:51.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:51.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:51.237 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 11:20:51.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:51.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:51.237 | INFO | Uploaded 36GiB to GPU in 1.9916s (18.0759GiB/s) | 20260506 11:20:51.237 | INFO | Downloaded 34GiB to GPU in 1.98144s (17.1593GiB/s) | 20260506 11:20:56.375 | INFO | Analyzing chunk frames = [952320,952832) | 20260506 11:20:56.376 | INFO | MemInfo: { | 20260506 11:20:56.376 | INFO | "gpu" : " 17 GB", | 20260506 11:20:56.376 | INFO | "huge" : "", | 20260506 11:20:56.376 | INFO | "huge_shared" : " 27 GB", | 20260506 11:20:56.376 | INFO | "malloc" : " 14 GB", | 20260506 11:20:56.376 | INFO | "pinned" : " 29 GB" | 20260506 11:20:56.376 | INFO | } | 20260506 11:20:56.376 | INFO | Perf report for the last 5.13907 seconds: | 20260506 11:20:56.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:20:56.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:56.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:56.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:56.376 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 11:20:56.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:56.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:56.376 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:20:56.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:56.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:20:56.376 | INFO | Uploaded 36GiB to GPU in 2.03106s (17.7247GiB/s) | 20260506 11:20:56.376 | INFO | Downloaded 34GiB to GPU in 1.93748s (17.5485GiB/s) | 20260506 11:21:01.515 | INFO | Analyzing chunk frames = [952832,953344) | 20260506 11:21:01.515 | INFO | MemInfo: { | 20260506 11:21:01.515 | INFO | "gpu" : " 17 GB", | 20260506 11:21:01.515 | INFO | "huge" : "", | 20260506 11:21:01.515 | INFO | "huge_shared" : " 27 GB", | 20260506 11:21:01.515 | INFO | "malloc" : " 14 GB", | 20260506 11:21:01.515 | INFO | "pinned" : " 29 GB" | 20260506 11:21:01.515 | INFO | } | 20260506 11:21:01.515 | INFO | Perf report for the last 5.13925 seconds: | 20260506 11:21:01.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:21:01.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:01.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:01.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:01.515 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 11:21:01.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:01.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:01.515 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 11:21:01.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:01.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:01.515 | INFO | Uploaded 36GiB to GPU in 2.06044s (17.472GiB/s) | 20260506 11:21:01.516 | INFO | Downloaded 34GiB to GPU in 1.93274s (17.5916GiB/s) | 20260506 11:21:06.654 | INFO | Analyzing chunk frames = [953344,953856) | 20260506 11:21:06.654 | INFO | MemInfo: { | 20260506 11:21:06.654 | INFO | "gpu" : " 17 GB", | 20260506 11:21:06.654 | INFO | "huge" : "", | 20260506 11:21:06.654 | INFO | "huge_shared" : " 27 GB", | 20260506 11:21:06.654 | INFO | "malloc" : " 14 GB", | 20260506 11:21:06.654 | INFO | "pinned" : " 29 GB" | 20260506 11:21:06.654 | INFO | } | 20260506 11:21:06.654 | INFO | Perf report for the last 5.13963 seconds: | 20260506 11:21:06.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:21:06.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:06.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:06.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:06.655 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 11:21:06.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:06.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:06.655 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 11:21:06.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:06.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:06.655 | INFO | Uploaded 36GiB to GPU in 2.0097s (17.9131GiB/s) | 20260506 11:21:06.655 | INFO | Downloaded 34GiB to GPU in 2.02909s (16.7563GiB/s) | 20260506 11:21:11.793 | INFO | Analyzing chunk frames = [953856,954368) | 20260506 11:21:11.793 | INFO | MemInfo: { | 20260506 11:21:11.793 | INFO | "gpu" : " 17 GB", | 20260506 11:21:11.793 | INFO | "huge" : "", | 20260506 11:21:11.793 | INFO | "huge_shared" : " 27 GB", | 20260506 11:21:11.793 | INFO | "malloc" : " 14 GB", | 20260506 11:21:11.793 | INFO | "pinned" : " 29 GB" | 20260506 11:21:11.793 | INFO | } | 20260506 11:21:11.793 | INFO | Perf report for the last 5.13895 seconds: | 20260506 11:21:11.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:21:11.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:11.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:11.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:11.794 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260506 11:21:11.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:11.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:11.794 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 11:21:11.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:11.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:11.794 | INFO | Uploaded 36GiB to GPU in 2.00674s (17.9395GiB/s) | 20260506 11:21:11.794 | INFO | Downloaded 34GiB to GPU in 1.98396s (17.1375GiB/s) | 20260506 11:21:11.918 | INFO | Starting a round of Baz IO | 20260506 11:21:15.125 | INFO | Populating RTMetrics report | 20260506 11:21:15.125 | INFO | RTMetrics report populated | 20260506 11:21:15.125 | INFO | Opening tmp RTMetrics json file | 20260506 11:21:15.159 | INFO | Serializing RTMetrics to json | 20260506 11:21:15.251 | INFO | Serialization done, swapping tmp file now... | 20260506 11:21:15.295 | INFO | RTMetrics done | 20260506 11:21:15.432 | INFO | Finished a round of Baz IO: Wrote 2.049 GiB in 3.513 seconds | 20260506 11:21:15.432 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.46% / 0.04% / 55.51% | 20260506 11:21:15.432 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.28826% / 4.37101e-05% / 4.37101e-05% | 20260506 11:21:15.432 | INFO | Baz IO overall: | 20260506 11:21:15.432 | INFO | BytesWritten : 455168297420 | 20260506 11:21:15.432 | INFO | NumEvents : 219547749166 | 20260506 11:21:16.933 | INFO | Analyzing chunk frames = [954368,954880) | 20260506 11:21:16.933 | INFO | MemInfo: { | 20260506 11:21:16.933 | INFO | "gpu" : " 17 GB", | 20260506 11:21:16.933 | INFO | "huge" : "", | 20260506 11:21:16.933 | INFO | "huge_shared" : " 27 GB", | 20260506 11:21:16.933 | INFO | "malloc" : " 14 GB", | 20260506 11:21:16.933 | INFO | "pinned" : " 29 GB" | 20260506 11:21:16.933 | INFO | } | 20260506 11:21:16.933 | INFO | Perf report for the last 5.13965 seconds: | 20260506 11:21:16.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:21:16.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:16.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:16.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:16.933 | INFO | ANALYSIS : 63.9%, 2.8, 95.2ms, 0 | 20260506 11:21:16.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:16.933 | INFO | RT_METRICS : 48.7%, 1.0, 25.6ms, 0 | 20260506 11:21:16.933 | INFO | PRE_HQ : 61.8%, 2.0, 63.1ms, 0 | 20260506 11:21:16.933 | INFO | BAZWRITER : 67.6%, 4.8, 166.0ms, 0 | 20260506 11:21:16.933 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 11:21:16.933 | INFO | Uploaded 36GiB to GPU in 2.07631s (17.3385GiB/s) | 20260506 11:21:16.933 | INFO | Downloaded 38GiB to GPU in 2.20749s (17.2141GiB/s) | 20260506 11:21:22.072 | INFO | Analyzing chunk frames = [954880,955392) | 20260506 11:21:22.072 | INFO | MemInfo: { | 20260506 11:21:22.072 | INFO | "gpu" : " 17 GB", | 20260506 11:21:22.072 | INFO | "huge" : "", | 20260506 11:21:22.072 | INFO | "huge_shared" : " 27 GB", | 20260506 11:21:22.072 | INFO | "malloc" : " 14 GB", | 20260506 11:21:22.072 | INFO | "pinned" : " 29 GB" | 20260506 11:21:22.072 | INFO | } | 20260506 11:21:22.072 | INFO | Perf report for the last 5.13931 seconds: | 20260506 11:21:22.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:21:22.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:22.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:22.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:22.072 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:21:22.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:22.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:22.072 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:21:22.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:22.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:22.073 | INFO | Uploaded 36GiB to GPU in 2.00715s (17.9359GiB/s) | 20260506 11:21:22.073 | INFO | Downloaded 34GiB to GPU in 1.90565s (17.8417GiB/s) | 20260506 11:21:27.211 | INFO | Analyzing chunk frames = [955392,955904) | 20260506 11:21:27.211 | INFO | MemInfo: { | 20260506 11:21:27.211 | INFO | "gpu" : " 17 GB", | 20260506 11:21:27.212 | INFO | "huge" : "", | 20260506 11:21:27.212 | INFO | "huge_shared" : " 27 GB", | 20260506 11:21:27.212 | INFO | "malloc" : " 14 GB", | 20260506 11:21:27.212 | INFO | "pinned" : " 29 GB" | 20260506 11:21:27.212 | INFO | } | 20260506 11:21:27.212 | INFO | Perf report for the last 5.13921 seconds: | 20260506 11:21:27.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:21:27.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:27.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:27.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:27.212 | INFO | ANALYSIS : 62.0%, 3.0, 96.1ms, 0 | 20260506 11:21:27.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:27.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:27.212 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:21:27.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:27.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:27.212 | INFO | Uploaded 36GiB to GPU in 2.01766s (17.8425GiB/s) | 20260506 11:21:27.212 | INFO | Downloaded 34GiB to GPU in 1.94047s (17.5215GiB/s) | 20260506 11:21:32.351 | INFO | Analyzing chunk frames = [955904,956416) | 20260506 11:21:32.351 | INFO | MemInfo: { | 20260506 11:21:32.351 | INFO | "gpu" : " 17 GB", | 20260506 11:21:32.351 | INFO | "huge" : "", | 20260506 11:21:32.351 | INFO | "huge_shared" : " 27 GB", | 20260506 11:21:32.351 | INFO | "malloc" : " 14 GB", | 20260506 11:21:32.351 | INFO | "pinned" : " 29 GB" | 20260506 11:21:32.351 | INFO | } | 20260506 11:21:32.351 | INFO | Perf report for the last 5.13924 seconds: | 20260506 11:21:32.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:21:32.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:32.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:32.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:32.351 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260506 11:21:32.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:32.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:32.351 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:21:32.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:32.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:32.351 | INFO | Uploaded 36GiB to GPU in 2.07998s (17.3079GiB/s) | 20260506 11:21:32.351 | INFO | Downloaded 34GiB to GPU in 1.97223s (17.2394GiB/s) | 20260506 11:21:37.491 | INFO | Analyzing chunk frames = [956416,956928) | 20260506 11:21:37.491 | INFO | MemInfo: { | 20260506 11:21:37.491 | INFO | "gpu" : " 17 GB", | 20260506 11:21:37.491 | INFO | "huge" : "", | 20260506 11:21:37.491 | INFO | "huge_shared" : " 27 GB", | 20260506 11:21:37.491 | INFO | "malloc" : " 14 GB", | 20260506 11:21:37.491 | INFO | "pinned" : " 29 GB" | 20260506 11:21:37.491 | INFO | } | 20260506 11:21:37.491 | INFO | Perf report for the last 5.13972 seconds: | 20260506 11:21:37.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:21:37.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:37.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:37.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:37.491 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 11:21:37.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:37.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:37.491 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:21:37.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:37.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:37.491 | INFO | Uploaded 36GiB to GPU in 2.04941s (17.566GiB/s) | 20260506 11:21:37.491 | INFO | Downloaded 34GiB to GPU in 1.98193s (17.155GiB/s) | 20260506 11:21:42.630 | INFO | Analyzing chunk frames = [956928,957440) | 20260506 11:21:42.630 | INFO | MemInfo: { | 20260506 11:21:42.630 | INFO | "gpu" : " 17 GB", | 20260506 11:21:42.630 | INFO | "huge" : "", | 20260506 11:21:42.630 | INFO | "huge_shared" : " 27 GB", | 20260506 11:21:42.630 | INFO | "malloc" : " 14 GB", | 20260506 11:21:42.630 | INFO | "pinned" : " 29 GB" | 20260506 11:21:42.630 | INFO | } | 20260506 11:21:42.630 | INFO | Perf report for the last 5.13907 seconds: | 20260506 11:21:42.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:21:42.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:42.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:42.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:42.630 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 11:21:42.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:42.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:42.630 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:21:42.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:42.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:42.630 | INFO | Uploaded 36GiB to GPU in 2.05697s (17.5014GiB/s) | 20260506 11:21:42.630 | INFO | Downloaded 34GiB to GPU in 1.98817s (17.1011GiB/s) | 20260506 11:21:47.769 | INFO | Analyzing chunk frames = [957440,957952) | 20260506 11:21:47.769 | INFO | MemInfo: { | 20260506 11:21:47.769 | INFO | "gpu" : " 17 GB", | 20260506 11:21:47.769 | INFO | "huge" : "", | 20260506 11:21:47.769 | INFO | "huge_shared" : " 27 GB", | 20260506 11:21:47.769 | INFO | "malloc" : " 14 GB", | 20260506 11:21:47.769 | INFO | "pinned" : " 29 GB" | 20260506 11:21:47.769 | INFO | } | 20260506 11:21:47.769 | INFO | Perf report for the last 5.13929 seconds: | 20260506 11:21:47.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:21:47.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:47.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:47.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:47.769 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 11:21:47.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:47.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:47.769 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 11:21:47.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:47.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:47.769 | INFO | Uploaded 36GiB to GPU in 1.9936s (18.0578GiB/s) | 20260506 11:21:47.769 | INFO | Downloaded 34GiB to GPU in 1.98157s (17.1581GiB/s) | 20260506 11:21:52.908 | INFO | Analyzing chunk frames = [957952,958464) | 20260506 11:21:52.908 | INFO | MemInfo: { | 20260506 11:21:52.908 | INFO | "gpu" : " 17 GB", | 20260506 11:21:52.908 | INFO | "huge" : "", | 20260506 11:21:52.908 | INFO | "huge_shared" : " 27 GB", | 20260506 11:21:52.908 | INFO | "malloc" : " 14 GB", | 20260506 11:21:52.908 | INFO | "pinned" : " 29 GB" | 20260506 11:21:52.908 | INFO | } | 20260506 11:21:52.908 | INFO | Perf report for the last 5.1393 seconds: | 20260506 11:21:52.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:21:52.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:52.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:52.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:52.908 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260506 11:21:52.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:52.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:52.908 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:21:52.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:52.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:52.909 | INFO | Uploaded 36GiB to GPU in 2.06241s (17.4553GiB/s) | 20260506 11:21:52.909 | INFO | Downloaded 34GiB to GPU in 1.99062s (17.0801GiB/s) | 20260506 11:21:52.909 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:21:52.909 | INFO | Total mem usage: 16.8 GiB | 20260506 11:21:52.909 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:21:52.909 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -8.13%) | 20260506 11:21:52.909 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:21:52.909 | INFO | High Water Breakdown: | 20260506 11:21:52.909 | INFO | 60.6% : 9.42 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:21:52.909 | INFO | 39.3% : 6.1 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:21:52.909 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:21:52.909 | INFO | ------------------------------------------ | 20260506 11:21:52.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:21:52.909 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:21:52.909 | INFO | Total mem usage: 26.5 GiB | 20260506 11:21:52.909 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:21:52.909 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 11:21:52.909 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 11:21:52.909 | INFO | High Water Breakdown: | 20260506 11:21:52.909 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:21:52.909 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:21:52.909 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:21:52.909 | INFO | 2% : 523 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:21:52.909 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:21:52.909 | INFO | 0.557% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:21:52.909 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:21:52.909 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:21:52.909 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:21:52.909 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:21:52.909 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:21:52.909 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:21:52.909 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:21:52.909 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:21:52.909 | INFO | ------------------------------------------ | 20260506 11:21:52.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:21:52.909 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:21:52.909 | INFO | Total mem usage: 25.6 GiB | 20260506 11:21:52.909 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:21:52.909 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:21:52.909 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:21:52.909 | INFO | High Water Breakdown: | 20260506 11:21:52.909 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:21:52.909 | INFO | ------------------------------------------ | 20260506 11:21:52.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:21:52.909 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:21:52.909 | INFO | Total mem usage: 15.7 GiB | 20260506 11:21:52.909 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:21:52.909 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:21:52.909 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 11:21:52.909 | INFO | High Water Breakdown: | 20260506 11:21:52.909 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:21:52.909 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:21:52.909 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:21:52.909 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:21:52.909 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:21:52.909 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:21:52.909 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:21:52.909 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:21:52.909 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:21:52.909 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:21:52.909 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:21:52.909 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:21:52.909 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:21:52.909 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:21:52.909 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:21:52.909 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:21:52.909 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:21:52.909 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:21:52.909 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:21:52.909 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:21:52.909 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:21:52.909 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:21:52.909 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:21:52.909 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:21:52.909 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:21:52.909 | INFO | ------------------------------------------ | 20260506 11:21:52.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:21:52.909 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:21:52.909 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:21:52.909 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:21:52.909 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.63e+01, 1.36e+01 / 2.41e+00 | 20260506 11:21:52.909 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 9.87e-02, 1.01e-02 / 7.16e-03 | 20260506 11:21:52.909 | INFO | PerfReport: 11.53%, Baseline -- 3.68e+00 / 4.40e+00, 5.92e-01 / 3.73e-01 | 20260506 11:21:52.909 | INFO | PerfReport: 3.25%, Histogram -- 1.04e+00 / 1.44e+00, 1.96e-01 / 1.58e-01 | 20260506 11:21:52.909 | INFO | PerfReport: 15.88%, DME -- 4.06e+01 / 4.53e+01, 4.31e+00 / 2.68e+00 | 20260506 11:21:52.909 | INFO | PerfReport: 56.19%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.47e+00 / 1.82e+00 | 20260506 11:21:52.909 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.44e+00 / 3.11e+00, 4.22e-01 / 2.16e-01 | 20260506 11:21:52.909 | INFO | PerfReport: 3.86%, Metrics -- 1.23e+00 / 1.63e+00, 3.14e-01 / 1.57e-01 | 20260506 11:21:52.910 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.37e-01 / 4.96e-01, 6.88e-02 / 2.77e-02 | 20260506 11:21:52.910 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:21:52.910 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:21:52.910 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:21:52.910 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 11:21:52.910 | INFO | PerfReport: 21.51%, Upload -- 2.13e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 11:21:52.910 | INFO | PerfReport: 21.03%, Download -- 2.08e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 11:21:52.910 | INFO | PerfReport: 32.34%, Compute -- 3.20e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 11:21:52.910 | INFO | PerfReport: 1.94%, LockUpload -- 1.92e+00 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 11:21:52.910 | INFO | PerfReport: 0.95%, LockDownload -- 9.42e-01 / 0.00e+00, 2.52e+00 / 0.00e+00 | 20260506 11:21:52.910 | INFO | PerfReport: 22.20%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 11:21:52.910 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.66e-02 / 0.00e+00 | 20260506 11:21:53.026 | INFO | Starting a round of Baz IO | 20260506 11:21:56.304 | INFO | Populating RTMetrics report | 20260506 11:21:56.304 | INFO | RTMetrics report populated | 20260506 11:21:56.304 | INFO | Opening tmp RTMetrics json file | 20260506 11:21:56.306 | INFO | Serializing RTMetrics to json | 20260506 11:21:56.333 | INFO | Serialization done, swapping tmp file now... | 20260506 11:21:56.358 | INFO | RTMetrics done | 20260506 11:21:56.519 | INFO | Finished a round of Baz IO: Wrote 2.055 GiB in 3.493 seconds | 20260506 11:21:56.519 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.42% / 0.04% / 55.54% | 20260506 11:21:56.519 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.31548% / 5.96046e-05% / 2.78155e-05% | 20260506 11:21:56.519 | INFO | Baz IO overall: | 20260506 11:21:56.519 | INFO | BytesWritten : 457374915324 | 20260506 11:21:56.519 | INFO | NumEvents : 220620110887 | 20260506 11:21:58.048 | INFO | Analyzing chunk frames = [958464,958976) | 20260506 11:21:58.048 | INFO | MemInfo: { | 20260506 11:21:58.048 | INFO | "gpu" : " 17 GB", | 20260506 11:21:58.048 | INFO | "huge" : "", | 20260506 11:21:58.048 | INFO | "huge_shared" : " 27 GB", | 20260506 11:21:58.048 | INFO | "malloc" : " 14 GB", | 20260506 11:21:58.048 | INFO | "pinned" : " 29 GB" | 20260506 11:21:58.048 | INFO | } | 20260506 11:21:58.048 | INFO | Perf report for the last 5.13958 seconds: | 20260506 11:21:58.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:21:58.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:58.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:58.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:58.048 | INFO | ANALYSIS : 65.3%, 2.8, 95.7ms, 0 | 20260506 11:21:58.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:21:58.048 | INFO | RT_METRICS : 48.9%, 1.0, 25.7ms, 0 | 20260506 11:21:58.048 | INFO | PRE_HQ : 64.2%, 2.0, 66.6ms, 0 | 20260506 11:21:58.048 | INFO | BAZWRITER : 67.1%, 4.8, 164.4ms, 0 | 20260506 11:21:58.048 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 11:21:58.048 | INFO | Uploaded 36GiB to GPU in 2.15854s (16.678GiB/s) | 20260506 11:21:58.048 | INFO | Downloaded 38GiB to GPU in 2.22638s (17.068GiB/s) | 20260506 11:22:03.187 | INFO | Analyzing chunk frames = [958976,959488) | 20260506 11:22:03.187 | INFO | MemInfo: { | 20260506 11:22:03.187 | INFO | "gpu" : " 17 GB", | 20260506 11:22:03.187 | INFO | "huge" : "", | 20260506 11:22:03.187 | INFO | "huge_shared" : " 27 GB", | 20260506 11:22:03.187 | INFO | "malloc" : " 14 GB", | 20260506 11:22:03.187 | INFO | "pinned" : " 29 GB" | 20260506 11:22:03.187 | INFO | } | 20260506 11:22:03.187 | INFO | Perf report for the last 5.13943 seconds: | 20260506 11:22:03.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:22:03.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:03.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:03.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:03.187 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:22:03.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:03.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:03.187 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 11:22:03.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:03.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:03.188 | INFO | Uploaded 36GiB to GPU in 2.04089s (17.6394GiB/s) | 20260506 11:22:03.188 | INFO | Downloaded 34GiB to GPU in 1.93s (17.6166GiB/s) | 20260506 11:22:08.326 | INFO | Analyzing chunk frames = [959488,960000) | 20260506 11:22:08.326 | INFO | MemInfo: { | 20260506 11:22:08.326 | INFO | "gpu" : " 17 GB", | 20260506 11:22:08.326 | INFO | "huge" : "", | 20260506 11:22:08.326 | INFO | "huge_shared" : " 27 GB", | 20260506 11:22:08.326 | INFO | "malloc" : " 14 GB", | 20260506 11:22:08.326 | INFO | "pinned" : " 29 GB" | 20260506 11:22:08.326 | INFO | } | 20260506 11:22:08.326 | INFO | Perf report for the last 5.13903 seconds: | 20260506 11:22:08.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:22:08.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:08.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:08.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:08.326 | INFO | ANALYSIS : 62.4%, 3.0, 96.6ms, 0 | 20260506 11:22:08.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:08.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:08.326 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:22:08.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:08.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:08.327 | INFO | Uploaded 36GiB to GPU in 2.0583s (17.4902GiB/s) | 20260506 11:22:08.327 | INFO | Downloaded 34GiB to GPU in 1.9508s (17.4288GiB/s) | 20260506 11:22:13.466 | INFO | Analyzing chunk frames = [960000,960512) | 20260506 11:22:13.466 | INFO | MemInfo: { | 20260506 11:22:13.466 | INFO | "gpu" : " 17 GB", | 20260506 11:22:13.466 | INFO | "huge" : "", | 20260506 11:22:13.466 | INFO | "huge_shared" : " 27 GB", | 20260506 11:22:13.466 | INFO | "malloc" : " 14 GB", | 20260506 11:22:13.466 | INFO | "pinned" : " 29 GB" | 20260506 11:22:13.466 | INFO | } | 20260506 11:22:13.466 | INFO | Perf report for the last 5.13932 seconds: | 20260506 11:22:13.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:22:13.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:13.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:13.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:13.466 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260506 11:22:13.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:13.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:13.466 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:22:13.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:13.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:13.466 | INFO | Uploaded 36GiB to GPU in 2.06503s (17.4332GiB/s) | 20260506 11:22:13.466 | INFO | Downloaded 34GiB to GPU in 1.96421s (17.3097GiB/s) | 20260506 11:22:18.605 | INFO | Analyzing chunk frames = [960512,961024) | 20260506 11:22:18.605 | INFO | MemInfo: { | 20260506 11:22:18.605 | INFO | "gpu" : " 17 GB", | 20260506 11:22:18.605 | INFO | "huge" : "", | 20260506 11:22:18.605 | INFO | "huge_shared" : " 27 GB", | 20260506 11:22:18.605 | INFO | "malloc" : " 14 GB", | 20260506 11:22:18.605 | INFO | "pinned" : " 29 GB" | 20260506 11:22:18.605 | INFO | } | 20260506 11:22:18.605 | INFO | Perf report for the last 5.1393 seconds: | 20260506 11:22:18.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:22:18.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:18.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:18.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:18.605 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 11:22:18.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:18.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:18.605 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:22:18.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:18.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:18.605 | INFO | Uploaded 36GiB to GPU in 2.05059s (17.5559GiB/s) | 20260506 11:22:18.605 | INFO | Downloaded 34GiB to GPU in 1.9832s (17.144GiB/s) | 20260506 11:22:23.745 | INFO | Analyzing chunk frames = [961024,961536) | 20260506 11:22:23.745 | INFO | MemInfo: { | 20260506 11:22:23.745 | INFO | "gpu" : " 17 GB", | 20260506 11:22:23.745 | INFO | "huge" : "", | 20260506 11:22:23.745 | INFO | "huge_shared" : " 27 GB", | 20260506 11:22:23.745 | INFO | "malloc" : " 14 GB", | 20260506 11:22:23.745 | INFO | "pinned" : " 29 GB" | 20260506 11:22:23.745 | INFO | } | 20260506 11:22:23.745 | INFO | Perf report for the last 5.13971 seconds: | 20260506 11:22:23.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:22:23.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:23.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:23.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:23.745 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 11:22:23.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:23.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:23.745 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:22:23.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:23.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:23.745 | INFO | Uploaded 36GiB to GPU in 2.0909s (17.2175GiB/s) | 20260506 11:22:23.745 | INFO | Downloaded 34GiB to GPU in 1.99007s (17.0848GiB/s) | 20260506 11:22:28.885 | INFO | Analyzing chunk frames = [961536,962048) | 20260506 11:22:28.885 | INFO | MemInfo: { | 20260506 11:22:28.885 | INFO | "gpu" : " 17 GB", | 20260506 11:22:28.885 | INFO | "huge" : "", | 20260506 11:22:28.885 | INFO | "huge_shared" : " 27 GB", | 20260506 11:22:28.885 | INFO | "malloc" : " 14 GB", | 20260506 11:22:28.885 | INFO | "pinned" : " 29 GB" | 20260506 11:22:28.885 | INFO | } | 20260506 11:22:28.885 | INFO | Perf report for the last 5.14015 seconds: | 20260506 11:22:28.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:22:28.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:28.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:28.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:28.885 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260506 11:22:28.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:28.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:28.885 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 11:22:28.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:28.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:28.885 | INFO | Uploaded 36GiB to GPU in 2.07166s (17.3774GiB/s) | 20260506 11:22:28.885 | INFO | Downloaded 34GiB to GPU in 1.99278s (17.0616GiB/s) | 20260506 11:22:34.023 | INFO | Analyzing chunk frames = [962048,962560) | 20260506 11:22:34.023 | INFO | MemInfo: { | 20260506 11:22:34.023 | INFO | "gpu" : " 17 GB", | 20260506 11:22:34.023 | INFO | "huge" : "", | 20260506 11:22:34.023 | INFO | "huge_shared" : " 27 GB", | 20260506 11:22:34.023 | INFO | "malloc" : " 14 GB", | 20260506 11:22:34.023 | INFO | "pinned" : " 29 GB" | 20260506 11:22:34.023 | INFO | } | 20260506 11:22:34.023 | INFO | Perf report for the last 5.13815 seconds: | 20260506 11:22:34.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:22:34.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:34.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:34.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:34.023 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260506 11:22:34.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:34.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:34.023 | INFO | PRE_HQ : 33.4%, 1.0, 17.4ms, 0 | 20260506 11:22:34.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:34.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:34.023 | INFO | Uploaded 36GiB to GPU in 2.07066s (17.3858GiB/s) | 20260506 11:22:34.023 | INFO | Downloaded 34GiB to GPU in 1.99413s (17.0501GiB/s) | 20260506 11:22:34.140 | INFO | Starting a round of Baz IO | 20260506 11:22:37.366 | INFO | Populating RTMetrics report | 20260506 11:22:37.366 | INFO | RTMetrics report populated | 20260506 11:22:37.366 | INFO | Opening tmp RTMetrics json file | 20260506 11:22:37.387 | INFO | Serializing RTMetrics to json | 20260506 11:22:37.390 | INFO | Serialization done, swapping tmp file now... | 20260506 11:22:37.391 | INFO | RTMetrics done | 20260506 11:22:37.773 | INFO | Finished a round of Baz IO: Wrote 2.053 GiB in 3.632 seconds | 20260506 11:22:37.773 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.39% / 0.04% / 55.58% | 20260506 11:22:37.773 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.27607% / 4.76837e-05% / 3.17891e-05% | 20260506 11:22:37.773 | INFO | Baz IO overall: | 20260506 11:22:37.773 | INFO | BytesWritten : 459579004038 | 20260506 11:22:37.773 | INFO | NumEvents : 221690089799 | 20260506 11:22:39.162 | INFO | Analyzing chunk frames = [962560,963072) | 20260506 11:22:39.162 | INFO | MemInfo: { | 20260506 11:22:39.162 | INFO | "gpu" : " 17 GB", | 20260506 11:22:39.162 | INFO | "huge" : "", | 20260506 11:22:39.162 | INFO | "huge_shared" : " 27 GB", | 20260506 11:22:39.162 | INFO | "malloc" : " 14 GB", | 20260506 11:22:39.162 | INFO | "pinned" : " 29 GB" | 20260506 11:22:39.162 | INFO | } | 20260506 11:22:39.163 | INFO | Perf report for the last 5.13942 seconds: | 20260506 11:22:39.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:22:39.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:39.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:39.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:39.163 | INFO | ANALYSIS : 64.3%, 2.9, 96.2ms, 0 | 20260506 11:22:39.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:39.163 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260506 11:22:39.163 | INFO | PRE_HQ : 63.1%, 1.9, 62.7ms, 0 | 20260506 11:22:39.163 | INFO | BAZWRITER : 69.9%, 4.8, 171.4ms, 0 | 20260506 11:22:39.163 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 11:22:39.163 | INFO | Uploaded 36GiB to GPU in 2.17345s (16.5635GiB/s) | 20260506 11:22:39.163 | INFO | Downloaded 38GiB to GPU in 2.23745s (16.9836GiB/s) | 20260506 11:22:44.302 | INFO | Analyzing chunk frames = [963072,963584) | 20260506 11:22:44.302 | INFO | MemInfo: { | 20260506 11:22:44.302 | INFO | "gpu" : " 17 GB", | 20260506 11:22:44.302 | INFO | "huge" : "", | 20260506 11:22:44.302 | INFO | "huge_shared" : " 27 GB", | 20260506 11:22:44.302 | INFO | "malloc" : " 14 GB", | 20260506 11:22:44.302 | INFO | "pinned" : " 29 GB" | 20260506 11:22:44.302 | INFO | } | 20260506 11:22:44.302 | INFO | Perf report for the last 5.13963 seconds: | 20260506 11:22:44.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:22:44.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:44.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:44.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:44.302 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:22:44.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:44.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:44.302 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:22:44.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:44.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:44.302 | INFO | Uploaded 36GiB to GPU in 2.03824s (17.6623GiB/s) | 20260506 11:22:44.302 | INFO | Downloaded 34GiB to GPU in 1.93325s (17.5869GiB/s) | 20260506 11:22:49.441 | INFO | Analyzing chunk frames = [963584,964096) | 20260506 11:22:49.441 | INFO | MemInfo: { | 20260506 11:22:49.441 | INFO | "gpu" : " 17 GB", | 20260506 11:22:49.441 | INFO | "huge" : "", | 20260506 11:22:49.441 | INFO | "huge_shared" : " 27 GB", | 20260506 11:22:49.441 | INFO | "malloc" : " 14 GB", | 20260506 11:22:49.441 | INFO | "pinned" : " 29 GB" | 20260506 11:22:49.441 | INFO | } | 20260506 11:22:49.441 | INFO | Perf report for the last 5.13916 seconds: | 20260506 11:22:49.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:22:49.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:49.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:49.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:49.441 | INFO | ANALYSIS : 62.4%, 3.0, 96.7ms, 0 | 20260506 11:22:49.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:49.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:49.441 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:22:49.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:49.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:49.442 | INFO | Uploaded 36GiB to GPU in 2.06437s (17.4388GiB/s) | 20260506 11:22:49.442 | INFO | Downloaded 34GiB to GPU in 1.96063s (17.3414GiB/s) | 20260506 11:22:54.581 | INFO | Analyzing chunk frames = [964096,964608) | 20260506 11:22:54.581 | INFO | MemInfo: { | 20260506 11:22:54.581 | INFO | "gpu" : " 17 GB", | 20260506 11:22:54.581 | INFO | "huge" : "", | 20260506 11:22:54.581 | INFO | "huge_shared" : " 27 GB", | 20260506 11:22:54.581 | INFO | "malloc" : " 14 GB", | 20260506 11:22:54.581 | INFO | "pinned" : " 29 GB" | 20260506 11:22:54.581 | INFO | } | 20260506 11:22:54.581 | INFO | Perf report for the last 5.13933 seconds: | 20260506 11:22:54.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:22:54.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:54.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:54.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:54.581 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 11:22:54.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:54.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:54.581 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:22:54.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:54.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:54.581 | INFO | Uploaded 36GiB to GPU in 2.08087s (17.3004GiB/s) | 20260506 11:22:54.581 | INFO | Downloaded 34GiB to GPU in 1.97373s (17.2263GiB/s) | 20260506 11:22:59.720 | INFO | Analyzing chunk frames = [964608,965120) | 20260506 11:22:59.720 | INFO | MemInfo: { | 20260506 11:22:59.720 | INFO | "gpu" : " 17 GB", | 20260506 11:22:59.720 | INFO | "huge" : "", | 20260506 11:22:59.720 | INFO | "huge_shared" : " 27 GB", | 20260506 11:22:59.720 | INFO | "malloc" : " 14 GB", | 20260506 11:22:59.720 | INFO | "pinned" : " 29 GB" | 20260506 11:22:59.720 | INFO | } | 20260506 11:22:59.720 | INFO | Perf report for the last 5.13932 seconds: | 20260506 11:22:59.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:22:59.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:59.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:59.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:59.720 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 11:22:59.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:59.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:59.720 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:22:59.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:59.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:22:59.720 | INFO | Uploaded 36GiB to GPU in 2.1008s (17.1363GiB/s) | 20260506 11:22:59.720 | INFO | Downloaded 34GiB to GPU in 1.99357s (17.0549GiB/s) | 20260506 11:23:04.859 | INFO | Analyzing chunk frames = [965120,965632) | 20260506 11:23:04.859 | INFO | MemInfo: { | 20260506 11:23:04.859 | INFO | "gpu" : " 17 GB", | 20260506 11:23:04.859 | INFO | "huge" : "", | 20260506 11:23:04.859 | INFO | "huge_shared" : " 27 GB", | 20260506 11:23:04.859 | INFO | "malloc" : " 14 GB", | 20260506 11:23:04.859 | INFO | "pinned" : " 29 GB" | 20260506 11:23:04.859 | INFO | } | 20260506 11:23:04.859 | INFO | Perf report for the last 5.13923 seconds: | 20260506 11:23:04.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:23:04.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:04.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:04.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:04.859 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 11:23:04.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:04.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:04.859 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:23:04.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:04.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:04.860 | INFO | Uploaded 36GiB to GPU in 2.05523s (17.5163GiB/s) | 20260506 11:23:04.860 | INFO | Downloaded 34GiB to GPU in 2.00319s (16.973GiB/s) | 20260506 11:23:10.000 | INFO | Analyzing chunk frames = [965632,966144) | 20260506 11:23:10.000 | INFO | MemInfo: { | 20260506 11:23:10.000 | INFO | "gpu" : " 17 GB", | 20260506 11:23:10.000 | INFO | "huge" : "", | 20260506 11:23:10.000 | INFO | "huge_shared" : " 27 GB", | 20260506 11:23:10.000 | INFO | "malloc" : " 14 GB", | 20260506 11:23:10.000 | INFO | "pinned" : " 29 GB" | 20260506 11:23:10.000 | INFO | } | 20260506 11:23:10.000 | INFO | Perf report for the last 5.14088 seconds: | 20260506 11:23:10.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:23:10.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:10.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:10.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:10.000 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 11:23:10.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:10.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:10.000 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 11:23:10.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:10.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:10.000 | INFO | Uploaded 36GiB to GPU in 2.01666s (17.8513GiB/s) | 20260506 11:23:10.001 | INFO | Downloaded 34GiB to GPU in 2.00581s (16.9508GiB/s) | 20260506 11:23:15.138 | INFO | Analyzing chunk frames = [966144,966656) | 20260506 11:23:15.138 | INFO | MemInfo: { | 20260506 11:23:15.138 | INFO | "gpu" : " 17 GB", | 20260506 11:23:15.138 | INFO | "huge" : "", | 20260506 11:23:15.138 | INFO | "huge_shared" : " 27 GB", | 20260506 11:23:15.138 | INFO | "malloc" : " 14 GB", | 20260506 11:23:15.138 | INFO | "pinned" : " 29 GB" | 20260506 11:23:15.138 | INFO | } | 20260506 11:23:15.138 | INFO | Perf report for the last 5.13795 seconds: | 20260506 11:23:15.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:23:15.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:15.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:15.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:15.138 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260506 11:23:15.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:15.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:15.138 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260506 11:23:15.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:15.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:15.138 | INFO | Uploaded 36GiB to GPU in 2.01938s (17.8272GiB/s) | 20260506 11:23:15.138 | INFO | Downloaded 34GiB to GPU in 1.9671s (17.2844GiB/s) | 20260506 11:23:15.139 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:23:15.139 | INFO | Total mem usage: 16.8 GiB | 20260506 11:23:15.139 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:23:15.139 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -7.75%) | 20260506 11:23:15.139 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:23:15.139 | INFO | High Water Breakdown: | 20260506 11:23:15.139 | INFO | 60.6% : 9.46 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:23:15.139 | INFO | 39.2% : 6.12 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:23:15.139 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:23:15.139 | INFO | ------------------------------------------ | 20260506 11:23:15.139 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:23:15.139 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:23:15.139 | INFO | Total mem usage: 26.5 GiB | 20260506 11:23:15.139 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:23:15.139 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:23:15.139 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:23:15.139 | INFO | High Water Breakdown: | 20260506 11:23:15.139 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:23:15.139 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:23:15.139 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:23:15.139 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:23:15.139 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:23:15.139 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:23:15.139 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:23:15.139 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:23:15.139 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:23:15.139 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:23:15.139 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:23:15.139 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:23:15.139 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:23:15.139 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:23:15.139 | INFO | ------------------------------------------ | 20260506 11:23:15.139 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:23:15.139 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:23:15.139 | INFO | Total mem usage: 25.6 GiB | 20260506 11:23:15.139 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:23:15.139 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:23:15.139 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:23:15.139 | INFO | High Water Breakdown: | 20260506 11:23:15.139 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:23:15.139 | INFO | ------------------------------------------ | 20260506 11:23:15.139 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:23:15.139 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:23:15.139 | INFO | Total mem usage: 15.7 GiB | 20260506 11:23:15.139 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:23:15.139 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:23:15.139 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 11:23:15.139 | INFO | High Water Breakdown: | 20260506 11:23:15.139 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:23:15.139 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:23:15.139 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:23:15.139 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:23:15.139 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:23:15.139 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:23:15.139 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:23:15.139 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:23:15.139 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:23:15.139 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:23:15.139 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:23:15.139 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:23:15.139 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:23:15.139 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:23:15.139 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:23:15.139 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:23:15.139 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:23:15.139 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:23:15.139 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:23:15.139 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:23:15.139 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:23:15.139 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:23:15.139 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:23:15.139 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:23:15.139 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:23:15.139 | INFO | ------------------------------------------ | 20260506 11:23:15.139 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:23:15.139 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:23:15.139 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:23:15.139 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:23:15.139 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.58e+01, 1.35e+01 / 2.32e+00 | 20260506 11:23:15.139 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.04e-01, 1.02e-02 / 9.17e-03 | 20260506 11:23:15.139 | INFO | PerfReport: 11.54%, Baseline -- 3.69e+00 / 4.41e+00, 5.82e-01 / 3.27e-01 | 20260506 11:23:15.139 | INFO | PerfReport: 3.23%, Histogram -- 1.03e+00 / 1.49e+00, 1.93e-01 / 1.43e-01 | 20260506 11:23:15.139 | INFO | PerfReport: 15.87%, DME -- 4.05e+01 / 4.51e+01, 3.99e+00 / 2.32e+00 | 20260506 11:23:15.139 | INFO | PerfReport: 56.09%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.39e+00 / 1.53e+00 | 20260506 11:23:15.139 | INFO | PerfReport: 7.74%, PulseAccumulating -- 2.47e+00 / 3.05e+00, 4.40e-01 / 2.73e-01 | 20260506 11:23:15.139 | INFO | PerfReport: 3.87%, Metrics -- 1.24e+00 / 1.63e+00, 3.14e-01 / 1.70e-01 | 20260506 11:23:15.139 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.38e-01 / 5.02e-01, 6.87e-02 / 3.40e-02 | 20260506 11:23:15.139 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:23:15.139 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:23:15.139 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:23:15.139 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260506 11:23:15.139 | INFO | PerfReport: 21.81%, Upload -- 2.16e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 11:23:15.139 | INFO | PerfReport: 21.12%, Download -- 2.09e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 11:23:15.139 | INFO | PerfReport: 32.27%, Compute -- 3.19e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 11:23:15.139 | INFO | PerfReport: 2.07%, LockUpload -- 2.05e+00 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260506 11:23:15.140 | INFO | PerfReport: 0.86%, LockDownload -- 8.55e-01 / 0.00e+00, 2.24e+00 / 0.00e+00 | 20260506 11:23:15.140 | INFO | PerfReport: 21.84%, LockCompute -- 2.16e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 11:23:15.140 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260506 11:23:15.261 | INFO | Starting a round of Baz IO | 20260506 11:23:18.442 | INFO | Populating RTMetrics report | 20260506 11:23:18.442 | INFO | RTMetrics report populated | 20260506 11:23:18.442 | INFO | Opening tmp RTMetrics json file | 20260506 11:23:18.444 | INFO | Serializing RTMetrics to json | 20260506 11:23:18.449 | INFO | Serialization done, swapping tmp file now... | 20260506 11:23:18.460 | INFO | RTMetrics done | 20260506 11:23:18.589 | INFO | Finished a round of Baz IO: Wrote 2.051 GiB in 3.327 seconds | 20260506 11:23:18.589 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.35% / 0.04% / 55.61% | 20260506 11:23:18.589 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.24707% / 4.37101e-05% / 3.17891e-05% | 20260506 11:23:18.589 | INFO | Baz IO overall: | 20260506 11:23:18.589 | INFO | BytesWritten : 461781485356 | 20260506 11:23:18.589 | INFO | NumEvents : 222758836980 | 20260506 11:23:20.277 | INFO | Analyzing chunk frames = [966656,967168) | 20260506 11:23:20.277 | INFO | MemInfo: { | 20260506 11:23:20.277 | INFO | "gpu" : " 17 GB", | 20260506 11:23:20.277 | INFO | "huge" : "", | 20260506 11:23:20.277 | INFO | "huge_shared" : " 27 GB", | 20260506 11:23:20.277 | INFO | "malloc" : " 14 GB", | 20260506 11:23:20.277 | INFO | "pinned" : " 29 GB" | 20260506 11:23:20.277 | INFO | } | 20260506 11:23:20.277 | INFO | Perf report for the last 5.13927 seconds: | 20260506 11:23:20.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:23:20.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:20.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:20.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:20.277 | INFO | ANALYSIS : 63.7%, 2.9, 95.8ms, 0 | 20260506 11:23:20.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:20.277 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260506 11:23:20.277 | INFO | PRE_HQ : 61.9%, 1.9, 62.0ms, 0 | 20260506 11:23:20.277 | INFO | BAZWRITER : 64.0%, 4.6, 152.7ms, 0 | 20260506 11:23:20.277 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 11:23:20.278 | INFO | Uploaded 36GiB to GPU in 2.13737s (16.8431GiB/s) | 20260506 11:23:20.278 | INFO | Downloaded 38GiB to GPU in 2.21774s (17.1346GiB/s) | 20260506 11:23:25.416 | INFO | Analyzing chunk frames = [967168,967680) | 20260506 11:23:25.416 | INFO | MemInfo: { | 20260506 11:23:25.416 | INFO | "gpu" : " 17 GB", | 20260506 11:23:25.416 | INFO | "huge" : "", | 20260506 11:23:25.417 | INFO | "huge_shared" : " 27 GB", | 20260506 11:23:25.417 | INFO | "malloc" : " 14 GB", | 20260506 11:23:25.417 | INFO | "pinned" : " 29 GB" | 20260506 11:23:25.417 | INFO | } | 20260506 11:23:25.417 | INFO | Perf report for the last 5.13919 seconds: | 20260506 11:23:25.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:23:25.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:25.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:25.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:25.417 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260506 11:23:25.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:25.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:25.417 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260506 11:23:25.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:25.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:25.417 | INFO | Uploaded 36GiB to GPU in 2.04359s (17.616GiB/s) | 20260506 11:23:25.417 | INFO | Downloaded 34GiB to GPU in 1.93034s (17.6135GiB/s) | 20260506 11:23:30.556 | INFO | Analyzing chunk frames = [967680,968192) | 20260506 11:23:30.556 | INFO | MemInfo: { | 20260506 11:23:30.556 | INFO | "gpu" : " 17 GB", | 20260506 11:23:30.556 | INFO | "huge" : "", | 20260506 11:23:30.556 | INFO | "huge_shared" : " 27 GB", | 20260506 11:23:30.556 | INFO | "malloc" : " 14 GB", | 20260506 11:23:30.556 | INFO | "pinned" : " 29 GB" | 20260506 11:23:30.556 | INFO | } | 20260506 11:23:30.556 | INFO | Perf report for the last 5.13929 seconds: | 20260506 11:23:30.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:23:30.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:30.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:30.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:30.556 | INFO | ANALYSIS : 62.5%, 3.0, 96.8ms, 0 | 20260506 11:23:30.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:30.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:30.556 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:23:30.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:30.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:30.557 | INFO | Uploaded 36GiB to GPU in 2.07962s (17.3109GiB/s) | 20260506 11:23:30.557 | INFO | Downloaded 34GiB to GPU in 1.96093s (17.3387GiB/s) | 20260506 11:23:35.695 | INFO | Analyzing chunk frames = [968192,968704) | 20260506 11:23:35.695 | INFO | MemInfo: { | 20260506 11:23:35.695 | INFO | "gpu" : " 17 GB", | 20260506 11:23:35.695 | INFO | "huge" : "", | 20260506 11:23:35.695 | INFO | "huge_shared" : " 27 GB", | 20260506 11:23:35.695 | INFO | "malloc" : " 14 GB", | 20260506 11:23:35.695 | INFO | "pinned" : " 29 GB" | 20260506 11:23:35.695 | INFO | } | 20260506 11:23:35.695 | INFO | Perf report for the last 5.13952 seconds: | 20260506 11:23:35.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:23:35.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:35.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:35.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:35.696 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 11:23:35.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:35.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:35.696 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:23:35.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:35.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:35.696 | INFO | Uploaded 36GiB to GPU in 2.08025s (17.3056GiB/s) | 20260506 11:23:35.696 | INFO | Downloaded 34GiB to GPU in 1.97445s (17.22GiB/s) | 20260506 11:23:40.835 | INFO | Analyzing chunk frames = [968704,969216) | 20260506 11:23:40.835 | INFO | MemInfo: { | 20260506 11:23:40.835 | INFO | "gpu" : " 17 GB", | 20260506 11:23:40.835 | INFO | "huge" : "", | 20260506 11:23:40.835 | INFO | "huge_shared" : " 27 GB", | 20260506 11:23:40.835 | INFO | "malloc" : " 14 GB", | 20260506 11:23:40.835 | INFO | "pinned" : " 29 GB" | 20260506 11:23:40.835 | INFO | } | 20260506 11:23:40.835 | INFO | Perf report for the last 5.13958 seconds: | 20260506 11:23:40.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:23:40.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:40.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:40.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:40.835 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 11:23:40.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:40.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:40.835 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:23:40.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:40.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:40.835 | INFO | Uploaded 36GiB to GPU in 2.05058s (17.556GiB/s) | 20260506 11:23:40.835 | INFO | Downloaded 34GiB to GPU in 1.98198s (17.1546GiB/s) | 20260506 11:23:45.974 | INFO | Analyzing chunk frames = [969216,969728) | 20260506 11:23:45.974 | INFO | MemInfo: { | 20260506 11:23:45.974 | INFO | "gpu" : " 17 GB", | 20260506 11:23:45.974 | INFO | "huge" : "", | 20260506 11:23:45.974 | INFO | "huge_shared" : " 27 GB", | 20260506 11:23:45.974 | INFO | "malloc" : " 14 GB", | 20260506 11:23:45.974 | INFO | "pinned" : " 29 GB" | 20260506 11:23:45.974 | INFO | } | 20260506 11:23:45.974 | INFO | Perf report for the last 5.13895 seconds: | 20260506 11:23:45.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:23:45.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:45.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:45.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:45.974 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 11:23:45.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:45.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:45.974 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:23:45.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:45.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:45.974 | INFO | Uploaded 36GiB to GPU in 2.09605s (17.1752GiB/s) | 20260506 11:23:45.974 | INFO | Downloaded 34GiB to GPU in 1.99888s (17.0095GiB/s) | 20260506 11:23:51.114 | INFO | Analyzing chunk frames = [969728,970240) | 20260506 11:23:51.114 | INFO | MemInfo: { | 20260506 11:23:51.114 | INFO | "gpu" : " 17 GB", | 20260506 11:23:51.114 | INFO | "huge" : "", | 20260506 11:23:51.114 | INFO | "huge_shared" : " 27 GB", | 20260506 11:23:51.114 | INFO | "malloc" : " 14 GB", | 20260506 11:23:51.114 | INFO | "pinned" : " 29 GB" | 20260506 11:23:51.114 | INFO | } | 20260506 11:23:51.114 | INFO | Perf report for the last 5.14038 seconds: | 20260506 11:23:51.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:23:51.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:51.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:51.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:51.115 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:23:51.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:51.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:51.115 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 11:23:51.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:51.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:51.115 | INFO | Uploaded 36GiB to GPU in 2.01151s (17.897GiB/s) | 20260506 11:23:51.115 | INFO | Downloaded 34GiB to GPU in 2.02839s (16.7621GiB/s) | 20260506 11:23:56.253 | INFO | Analyzing chunk frames = [970240,970752) | 20260506 11:23:56.253 | INFO | MemInfo: { | 20260506 11:23:56.253 | INFO | "gpu" : " 17 GB", | 20260506 11:23:56.253 | INFO | "huge" : "", | 20260506 11:23:56.253 | INFO | "huge_shared" : " 27 GB", | 20260506 11:23:56.253 | INFO | "malloc" : " 14 GB", | 20260506 11:23:56.253 | INFO | "pinned" : " 29 GB" | 20260506 11:23:56.253 | INFO | } | 20260506 11:23:56.253 | INFO | Perf report for the last 5.13865 seconds: | 20260506 11:23:56.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:23:56.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:56.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:56.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:56.253 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260506 11:23:56.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:56.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:56.253 | INFO | PRE_HQ : 33.6%, 1.0, 17.5ms, 0 | 20260506 11:23:56.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:56.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:23:56.253 | INFO | Uploaded 36GiB to GPU in 2.01528s (17.8635GiB/s) | 20260506 11:23:56.253 | INFO | Downloaded 34GiB to GPU in 1.98051s (17.1673GiB/s) | 20260506 11:23:56.376 | INFO | Starting a round of Baz IO | 20260506 11:23:59.572 | INFO | Populating RTMetrics report | 20260506 11:23:59.573 | INFO | RTMetrics report populated | 20260506 11:23:59.573 | INFO | Opening tmp RTMetrics json file | 20260506 11:23:59.574 | INFO | Serializing RTMetrics to json | 20260506 11:23:59.577 | INFO | Serialization done, swapping tmp file now... | 20260506 11:23:59.579 | INFO | RTMetrics done | 20260506 11:23:59.685 | INFO | Finished a round of Baz IO: Wrote 2.054 GiB in 3.308 seconds | 20260506 11:23:59.685 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.31% / 0.04% / 55.65% | 20260506 11:23:59.685 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.24154% / 5.96046e-05% / 5.96046e-05% | 20260506 11:23:59.685 | INFO | Baz IO overall: | 20260506 11:23:59.685 | INFO | BytesWritten : 463987283044 | 20260506 11:23:59.685 | INFO | NumEvents : 223829624879 | 20260506 11:24:01.392 | INFO | Analyzing chunk frames = [970752,971264) | 20260506 11:24:01.392 | INFO | MemInfo: { | 20260506 11:24:01.392 | INFO | "gpu" : " 17 GB", | 20260506 11:24:01.392 | INFO | "huge" : "", | 20260506 11:24:01.392 | INFO | "huge_shared" : " 27 GB", | 20260506 11:24:01.392 | INFO | "malloc" : " 14 GB", | 20260506 11:24:01.393 | INFO | "pinned" : " 29 GB" | 20260506 11:24:01.393 | INFO | } | 20260506 11:24:01.393 | INFO | Perf report for the last 5.13952 seconds: | 20260506 11:24:01.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:24:01.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:01.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:01.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:01.393 | INFO | ANALYSIS : 64.0%, 2.9, 96.1ms, 0 | 20260506 11:24:01.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:01.393 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 11:24:01.393 | INFO | PRE_HQ : 62.0%, 1.9, 61.7ms, 0 | 20260506 11:24:01.393 | INFO | BAZWRITER : 63.6%, 4.7, 152.4ms, 0 | 20260506 11:24:01.393 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 11:24:01.393 | INFO | Uploaded 36GiB to GPU in 2.12778s (16.919GiB/s) | 20260506 11:24:01.393 | INFO | Downloaded 38GiB to GPU in 2.23218s (17.0237GiB/s) | 20260506 11:24:06.532 | INFO | Analyzing chunk frames = [971264,971776) | 20260506 11:24:06.532 | INFO | MemInfo: { | 20260506 11:24:06.532 | INFO | "gpu" : " 17 GB", | 20260506 11:24:06.532 | INFO | "huge" : "", | 20260506 11:24:06.532 | INFO | "huge_shared" : " 27 GB", | 20260506 11:24:06.532 | INFO | "malloc" : " 14 GB", | 20260506 11:24:06.532 | INFO | "pinned" : " 29 GB" | 20260506 11:24:06.532 | INFO | } | 20260506 11:24:06.532 | INFO | Perf report for the last 5.13902 seconds: | 20260506 11:24:06.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:24:06.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:06.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:06.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:06.532 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:24:06.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:06.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:06.532 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260506 11:24:06.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:06.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:06.532 | INFO | Uploaded 36GiB to GPU in 2.03221s (17.7147GiB/s) | 20260506 11:24:06.532 | INFO | Downloaded 34GiB to GPU in 1.93596s (17.5624GiB/s) | 20260506 11:24:11.671 | INFO | Analyzing chunk frames = [971776,972288) | 20260506 11:24:11.671 | INFO | MemInfo: { | 20260506 11:24:11.671 | INFO | "gpu" : " 17 GB", | 20260506 11:24:11.671 | INFO | "huge" : "", | 20260506 11:24:11.671 | INFO | "huge_shared" : " 27 GB", | 20260506 11:24:11.671 | INFO | "malloc" : " 14 GB", | 20260506 11:24:11.671 | INFO | "pinned" : " 29 GB" | 20260506 11:24:11.671 | INFO | } | 20260506 11:24:11.671 | INFO | Perf report for the last 5.13919 seconds: | 20260506 11:24:11.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:24:11.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:11.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:11.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:11.671 | INFO | ANALYSIS : 62.3%, 3.0, 96.5ms, 0 | 20260506 11:24:11.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:11.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:11.671 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:24:11.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:11.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:11.671 | INFO | Uploaded 36GiB to GPU in 2.0038s (17.9659GiB/s) | 20260506 11:24:11.671 | INFO | Downloaded 34GiB to GPU in 1.95679s (17.3754GiB/s) | 20260506 11:24:16.810 | INFO | Analyzing chunk frames = [972288,972800) | 20260506 11:24:16.810 | INFO | MemInfo: { | 20260506 11:24:16.810 | INFO | "gpu" : " 17 GB", | 20260506 11:24:16.810 | INFO | "huge" : "", | 20260506 11:24:16.810 | INFO | "huge_shared" : " 27 GB", | 20260506 11:24:16.810 | INFO | "malloc" : " 14 GB", | 20260506 11:24:16.810 | INFO | "pinned" : " 29 GB" | 20260506 11:24:16.810 | INFO | } | 20260506 11:24:16.810 | INFO | Perf report for the last 5.13958 seconds: | 20260506 11:24:16.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:24:16.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:16.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:16.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:16.810 | INFO | ANALYSIS : 61.9%, 3.0, 95.9ms, 0 | 20260506 11:24:16.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:16.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:16.810 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:24:16.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:16.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:16.811 | INFO | Uploaded 36GiB to GPU in 2.07975s (17.3098GiB/s) | 20260506 11:24:16.811 | INFO | Downloaded 34GiB to GPU in 1.96758s (17.2801GiB/s) | 20260506 11:24:21.949 | INFO | Analyzing chunk frames = [972800,973312) | 20260506 11:24:21.949 | INFO | MemInfo: { | 20260506 11:24:21.949 | INFO | "gpu" : " 17 GB", | 20260506 11:24:21.949 | INFO | "huge" : "", | 20260506 11:24:21.949 | INFO | "huge_shared" : " 27 GB", | 20260506 11:24:21.950 | INFO | "malloc" : " 14 GB", | 20260506 11:24:21.950 | INFO | "pinned" : " 29 GB" | 20260506 11:24:21.950 | INFO | } | 20260506 11:24:21.950 | INFO | Perf report for the last 5.13912 seconds: | 20260506 11:24:21.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:24:21.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:21.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:21.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:21.950 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 11:24:21.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:21.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:21.950 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:24:21.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:21.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:21.950 | INFO | Uploaded 36GiB to GPU in 2.08101s (17.2993GiB/s) | 20260506 11:24:21.950 | INFO | Downloaded 34GiB to GPU in 1.99853s (17.0125GiB/s) | 20260506 11:24:27.089 | INFO | Analyzing chunk frames = [973312,973824) | 20260506 11:24:27.089 | INFO | MemInfo: { | 20260506 11:24:27.089 | INFO | "gpu" : " 17 GB", | 20260506 11:24:27.089 | INFO | "huge" : "", | 20260506 11:24:27.089 | INFO | "huge_shared" : " 27 GB", | 20260506 11:24:27.089 | INFO | "malloc" : " 14 GB", | 20260506 11:24:27.089 | INFO | "pinned" : " 29 GB" | 20260506 11:24:27.089 | INFO | } | 20260506 11:24:27.089 | INFO | Perf report for the last 5.13962 seconds: | 20260506 11:24:27.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:24:27.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:27.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:27.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:27.089 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 11:24:27.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:27.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:27.089 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:24:27.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:27.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:27.089 | INFO | Uploaded 36GiB to GPU in 1.98339s (18.1507GiB/s) | 20260506 11:24:27.090 | INFO | Downloaded 34GiB to GPU in 1.99098s (17.077GiB/s) | 20260506 11:24:32.229 | INFO | Analyzing chunk frames = [973824,974336) | 20260506 11:24:32.229 | INFO | MemInfo: { | 20260506 11:24:32.229 | INFO | "gpu" : " 17 GB", | 20260506 11:24:32.229 | INFO | "huge" : "", | 20260506 11:24:32.229 | INFO | "huge_shared" : " 27 GB", | 20260506 11:24:32.229 | INFO | "malloc" : " 14 GB", | 20260506 11:24:32.229 | INFO | "pinned" : " 29 GB" | 20260506 11:24:32.229 | INFO | } | 20260506 11:24:32.229 | INFO | Perf report for the last 5.13987 seconds: | 20260506 11:24:32.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:24:32.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:32.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:32.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:32.229 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:24:32.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:32.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:32.229 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 11:24:32.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:32.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:32.229 | INFO | Uploaded 36GiB to GPU in 2.00274s (17.9754GiB/s) | 20260506 11:24:32.229 | INFO | Downloaded 34GiB to GPU in 2.0288s (16.7587GiB/s) | 20260506 11:24:37.367 | INFO | Analyzing chunk frames = [974336,974848) | 20260506 11:24:37.368 | INFO | MemInfo: { | 20260506 11:24:37.368 | INFO | "gpu" : " 17 GB", | 20260506 11:24:37.368 | INFO | "huge" : "", | 20260506 11:24:37.368 | INFO | "huge_shared" : " 27 GB", | 20260506 11:24:37.368 | INFO | "malloc" : " 14 GB", | 20260506 11:24:37.368 | INFO | "pinned" : " 29 GB" | 20260506 11:24:37.368 | INFO | } | 20260506 11:24:37.368 | INFO | Perf report for the last 5.13876 seconds: | 20260506 11:24:37.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:24:37.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:37.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:37.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:37.368 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260506 11:24:37.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:37.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:37.368 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260506 11:24:37.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:37.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:37.368 | INFO | Uploaded 36GiB to GPU in 2.01474s (17.8683GiB/s) | 20260506 11:24:37.368 | INFO | Downloaded 34GiB to GPU in 1.97623s (17.2045GiB/s) | 20260506 11:24:37.368 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:24:37.368 | INFO | Total mem usage: 16.8 GiB | 20260506 11:24:37.368 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:24:37.368 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -10%) | 20260506 11:24:37.368 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:24:37.368 | INFO | High Water Breakdown: | 20260506 11:24:37.368 | INFO | 60.3% : 9.22 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:24:37.368 | INFO | 39.5% : 6.04 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:24:37.368 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:24:37.368 | INFO | ------------------------------------------ | 20260506 11:24:37.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:24:37.368 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:24:37.368 | INFO | Total mem usage: 26.5 GiB | 20260506 11:24:37.368 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:24:37.368 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:24:37.369 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:24:37.369 | INFO | High Water Breakdown: | 20260506 11:24:37.369 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:24:37.369 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:24:37.369 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:24:37.369 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:24:37.369 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:24:37.369 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:24:37.369 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:24:37.369 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:24:37.369 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:24:37.369 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:24:37.369 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:24:37.369 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:24:37.369 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:24:37.369 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:24:37.369 | INFO | ------------------------------------------ | 20260506 11:24:37.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:24:37.369 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:24:37.369 | INFO | Total mem usage: 25.6 GiB | 20260506 11:24:37.369 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:24:37.369 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:24:37.369 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:24:37.369 | INFO | High Water Breakdown: | 20260506 11:24:37.369 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:24:37.369 | INFO | ------------------------------------------ | 20260506 11:24:37.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:24:37.369 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:24:37.369 | INFO | Total mem usage: 15.7 GiB | 20260506 11:24:37.369 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:24:37.369 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:24:37.369 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 11:24:37.369 | INFO | High Water Breakdown: | 20260506 11:24:37.369 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:24:37.369 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:24:37.369 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:24:37.369 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:24:37.369 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:24:37.369 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:24:37.369 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:24:37.369 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:24:37.369 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:24:37.369 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:24:37.369 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:24:37.369 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:24:37.369 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:24:37.369 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:24:37.369 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:24:37.369 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:24:37.369 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:24:37.369 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:24:37.369 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:24:37.369 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:24:37.369 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:24:37.369 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:24:37.369 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:24:37.369 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:24:37.369 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:24:37.369 | INFO | ------------------------------------------ | 20260506 11:24:37.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:24:37.369 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:24:37.369 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:24:37.369 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:24:37.369 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.63e+01, 1.37e+01 / 2.76e+00 | 20260506 11:24:37.369 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 9.99e-02, 1.05e-02 / 5.61e-03 | 20260506 11:24:37.369 | INFO | PerfReport: 11.57%, Baseline -- 3.69e+00 / 4.43e+00, 5.86e-01 / 3.65e-01 | 20260506 11:24:37.369 | INFO | PerfReport: 3.25%, Histogram -- 1.04e+00 / 1.46e+00, 1.95e-01 / 1.42e-01 | 20260506 11:24:37.369 | INFO | PerfReport: 15.93%, DME -- 4.07e+01 / 4.56e+01, 4.53e+00 / 2.79e+00 | 20260506 11:24:37.369 | INFO | PerfReport: 56.03%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.32e+00 / 1.40e+00 | 20260506 11:24:37.369 | INFO | PerfReport: 7.70%, PulseAccumulating -- 2.46e+00 / 3.13e+00, 4.27e-01 / 1.78e-01 | 20260506 11:24:37.369 | INFO | PerfReport: 3.87%, Metrics -- 1.23e+00 / 1.63e+00, 3.09e-01 / 1.61e-01 | 20260506 11:24:37.369 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.37e-01 / 4.97e-01, 6.78e-02 / 2.70e-02 | 20260506 11:24:37.369 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:24:37.369 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:24:37.370 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:24:37.370 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 11:24:37.370 | INFO | PerfReport: 21.62%, Upload -- 2.14e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260506 11:24:37.370 | INFO | PerfReport: 21.17%, Download -- 2.09e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 11:24:37.370 | INFO | PerfReport: 32.29%, Compute -- 3.19e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 11:24:37.370 | INFO | PerfReport: 1.93%, LockUpload -- 1.91e+00 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260506 11:24:37.370 | INFO | PerfReport: 0.95%, LockDownload -- 9.35e-01 / 0.00e+00, 2.44e+00 / 0.00e+00 | 20260506 11:24:37.370 | INFO | PerfReport: 22.03%, LockCompute -- 2.18e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 11:24:37.370 | INFO | PerfReport: 0.02%, Misc -- 1.55e-02 / 0.00e+00, 2.72e-02 / 0.00e+00 | 20260506 11:24:37.506 | INFO | Starting a round of Baz IO | 20260506 11:24:40.672 | INFO | Populating RTMetrics report | 20260506 11:24:40.673 | INFO | RTMetrics report populated | 20260506 11:24:40.673 | INFO | Opening tmp RTMetrics json file | 20260506 11:24:40.740 | INFO | Serializing RTMetrics to json | 20260506 11:24:40.789 | INFO | Serialization done, swapping tmp file now... | 20260506 11:24:40.792 | INFO | RTMetrics done | 20260506 11:24:41.034 | INFO | Finished a round of Baz IO: Wrote 2.053 GiB in 3.527 seconds | 20260506 11:24:41.034 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.28% / 0.03% / 55.69% | 20260506 11:24:41.034 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.21566% / 3.97364e-05% / 6.75519e-05% | 20260506 11:24:41.034 | INFO | Baz IO overall: | 20260506 11:24:41.034 | INFO | BytesWritten : 466191934734 | 20260506 11:24:41.034 | INFO | NumEvents : 224899561409 | 20260506 11:24:42.507 | INFO | Analyzing chunk frames = [974848,975360) | 20260506 11:24:42.507 | INFO | MemInfo: { | 20260506 11:24:42.507 | INFO | "gpu" : " 17 GB", | 20260506 11:24:42.507 | INFO | "huge" : "", | 20260506 11:24:42.507 | INFO | "huge_shared" : " 27 GB", | 20260506 11:24:42.507 | INFO | "malloc" : " 14 GB", | 20260506 11:24:42.507 | INFO | "pinned" : " 29 GB" | 20260506 11:24:42.507 | INFO | } | 20260506 11:24:42.507 | INFO | Perf report for the last 5.13926 seconds: | 20260506 11:24:42.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:24:42.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:42.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:42.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:42.507 | INFO | ANALYSIS : 63.7%, 2.9, 95.5ms, 0 | 20260506 11:24:42.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:42.507 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260506 11:24:42.507 | INFO | PRE_HQ : 62.0%, 1.9, 62.3ms, 0 | 20260506 11:24:42.507 | INFO | BAZWRITER : 67.9%, 4.8, 168.5ms, 0 | 20260506 11:24:42.507 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 11:24:42.508 | INFO | Uploaded 36GiB to GPU in 2.13926s (16.8282GiB/s) | 20260506 11:24:42.508 | INFO | Downloaded 38GiB to GPU in 2.22588s (17.0719GiB/s) | 20260506 11:24:47.646 | INFO | Analyzing chunk frames = [975360,975872) | 20260506 11:24:47.646 | INFO | MemInfo: { | 20260506 11:24:47.646 | INFO | "gpu" : " 17 GB", | 20260506 11:24:47.646 | INFO | "huge" : "", | 20260506 11:24:47.646 | INFO | "huge_shared" : " 27 GB", | 20260506 11:24:47.646 | INFO | "malloc" : " 14 GB", | 20260506 11:24:47.646 | INFO | "pinned" : " 29 GB" | 20260506 11:24:47.646 | INFO | } | 20260506 11:24:47.646 | INFO | Perf report for the last 5.13919 seconds: | 20260506 11:24:47.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:24:47.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:47.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:47.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:47.647 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260506 11:24:47.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:47.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:47.647 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:24:47.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:47.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:47.647 | INFO | Uploaded 36GiB to GPU in 2.06021s (17.474GiB/s) | 20260506 11:24:47.647 | INFO | Downloaded 34GiB to GPU in 1.93199s (17.5984GiB/s) | 20260506 11:24:52.786 | INFO | Analyzing chunk frames = [975872,976384) | 20260506 11:24:52.786 | INFO | MemInfo: { | 20260506 11:24:52.786 | INFO | "gpu" : " 17 GB", | 20260506 11:24:52.786 | INFO | "huge" : "", | 20260506 11:24:52.786 | INFO | "huge_shared" : " 27 GB", | 20260506 11:24:52.786 | INFO | "malloc" : " 14 GB", | 20260506 11:24:52.786 | INFO | "pinned" : " 29 GB" | 20260506 11:24:52.786 | INFO | } | 20260506 11:24:52.786 | INFO | Perf report for the last 5.13947 seconds: | 20260506 11:24:52.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:24:52.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:52.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:52.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:52.786 | INFO | ANALYSIS : 62.3%, 3.0, 96.5ms, 0 | 20260506 11:24:52.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:52.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:52.786 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:24:52.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:52.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:52.786 | INFO | Uploaded 36GiB to GPU in 2.01026s (17.9081GiB/s) | 20260506 11:24:52.786 | INFO | Downloaded 34GiB to GPU in 1.95712s (17.3724GiB/s) | 20260506 11:24:57.925 | INFO | Analyzing chunk frames = [976384,976896) | 20260506 11:24:57.925 | INFO | MemInfo: { | 20260506 11:24:57.925 | INFO | "gpu" : " 17 GB", | 20260506 11:24:57.925 | INFO | "huge" : "", | 20260506 11:24:57.925 | INFO | "huge_shared" : " 27 GB", | 20260506 11:24:57.925 | INFO | "malloc" : " 14 GB", | 20260506 11:24:57.925 | INFO | "pinned" : " 29 GB" | 20260506 11:24:57.925 | INFO | } | 20260506 11:24:57.925 | INFO | Perf report for the last 5.13929 seconds: | 20260506 11:24:57.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:24:57.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:57.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:57.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:57.925 | INFO | ANALYSIS : 62.2%, 2.9, 96.2ms, 0 | 20260506 11:24:57.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:57.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:57.925 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:24:57.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:57.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:24:57.925 | INFO | Uploaded 36GiB to GPU in 1.97355s (18.2412GiB/s) | 20260506 11:24:57.925 | INFO | Downloaded 34GiB to GPU in 1.96455s (17.3068GiB/s) | 20260506 11:25:03.064 | INFO | Analyzing chunk frames = [976896,977408) | 20260506 11:25:03.064 | INFO | MemInfo: { | 20260506 11:25:03.064 | INFO | "gpu" : " 17 GB", | 20260506 11:25:03.064 | INFO | "huge" : "", | 20260506 11:25:03.064 | INFO | "huge_shared" : " 27 GB", | 20260506 11:25:03.064 | INFO | "malloc" : " 14 GB", | 20260506 11:25:03.064 | INFO | "pinned" : " 29 GB" | 20260506 11:25:03.064 | INFO | } | 20260506 11:25:03.064 | INFO | Perf report for the last 5.13925 seconds: | 20260506 11:25:03.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:25:03.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:03.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:03.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:03.065 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 11:25:03.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:03.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:03.065 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:25:03.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:03.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:03.065 | INFO | Uploaded 36GiB to GPU in 2.02985s (17.7353GiB/s) | 20260506 11:25:03.065 | INFO | Downloaded 34GiB to GPU in 1.97049s (17.2546GiB/s) | 20260506 11:25:08.204 | INFO | Analyzing chunk frames = [977408,977920) | 20260506 11:25:08.204 | INFO | MemInfo: { | 20260506 11:25:08.204 | INFO | "gpu" : " 17 GB", | 20260506 11:25:08.204 | INFO | "huge" : "", | 20260506 11:25:08.204 | INFO | "huge_shared" : " 27 GB", | 20260506 11:25:08.204 | INFO | "malloc" : " 14 GB", | 20260506 11:25:08.204 | INFO | "pinned" : " 29 GB" | 20260506 11:25:08.204 | INFO | } | 20260506 11:25:08.204 | INFO | Perf report for the last 5.13939 seconds: | 20260506 11:25:08.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:25:08.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:08.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:08.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:08.204 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 11:25:08.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:08.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:08.204 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 11:25:08.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:08.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:08.204 | INFO | Uploaded 36GiB to GPU in 2.07165s (17.3775GiB/s) | 20260506 11:25:08.204 | INFO | Downloaded 34GiB to GPU in 1.99016s (17.0841GiB/s) | 20260506 11:25:13.343 | INFO | Analyzing chunk frames = [977920,978432) | 20260506 11:25:13.343 | INFO | MemInfo: { | 20260506 11:25:13.343 | INFO | "gpu" : " 17 GB", | 20260506 11:25:13.343 | INFO | "huge" : "", | 20260506 11:25:13.343 | INFO | "huge_shared" : " 27 GB", | 20260506 11:25:13.343 | INFO | "malloc" : " 14 GB", | 20260506 11:25:13.343 | INFO | "pinned" : " 29 GB" | 20260506 11:25:13.343 | INFO | } | 20260506 11:25:13.343 | INFO | Perf report for the last 5.13945 seconds: | 20260506 11:25:13.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:25:13.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:13.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:13.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:13.343 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260506 11:25:13.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:13.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:13.343 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 11:25:13.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:13.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:13.344 | INFO | Uploaded 36GiB to GPU in 2.03747s (17.669GiB/s) | 20260506 11:25:13.344 | INFO | Downloaded 34GiB to GPU in 2.0211s (16.8225GiB/s) | 20260506 11:25:18.482 | INFO | Analyzing chunk frames = [978432,978944) | 20260506 11:25:18.482 | INFO | MemInfo: { | 20260506 11:25:18.482 | INFO | "gpu" : " 17 GB", | 20260506 11:25:18.482 | INFO | "huge" : "", | 20260506 11:25:18.482 | INFO | "huge_shared" : " 27 GB", | 20260506 11:25:18.482 | INFO | "malloc" : " 14 GB", | 20260506 11:25:18.483 | INFO | "pinned" : " 29 GB" | 20260506 11:25:18.483 | INFO | } | 20260506 11:25:18.483 | INFO | Perf report for the last 5.13942 seconds: | 20260506 11:25:18.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:25:18.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:18.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:18.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:18.483 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260506 11:25:18.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:18.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:18.483 | INFO | PRE_HQ : 32.8%, 1.0, 17.1ms, 0 | 20260506 11:25:18.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:18.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:18.483 | INFO | Uploaded 36GiB to GPU in 2.03832s (17.6616GiB/s) | 20260506 11:25:18.483 | INFO | Downloaded 34GiB to GPU in 1.96975s (17.2611GiB/s) | 20260506 11:25:18.600 | INFO | Starting a round of Baz IO | 20260506 11:25:21.822 | INFO | Populating RTMetrics report | 20260506 11:25:21.822 | INFO | RTMetrics report populated | 20260506 11:25:21.822 | INFO | Opening tmp RTMetrics json file | 20260506 11:25:21.823 | INFO | Serializing RTMetrics to json | 20260506 11:25:21.832 | INFO | Serialization done, swapping tmp file now... | 20260506 11:25:21.853 | INFO | RTMetrics done | 20260506 11:25:22.055 | INFO | Finished a round of Baz IO: Wrote 2.057 GiB in 3.454 seconds | 20260506 11:25:22.055 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.24% / 0.03% / 55.72% | 20260506 11:25:22.055 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.22083% / 4.37101e-05% / 4.37101e-05% | 20260506 11:25:22.055 | INFO | Baz IO overall: | 20260506 11:25:22.055 | INFO | BytesWritten : 468400136511 | 20260506 11:25:22.055 | INFO | NumEvents : 225971892436 | 20260506 11:25:23.622 | INFO | Analyzing chunk frames = [978944,979456) | 20260506 11:25:23.622 | INFO | MemInfo: { | 20260506 11:25:23.622 | INFO | "gpu" : " 17 GB", | 20260506 11:25:23.622 | INFO | "huge" : "", | 20260506 11:25:23.622 | INFO | "huge_shared" : " 27 GB", | 20260506 11:25:23.622 | INFO | "malloc" : " 14 GB", | 20260506 11:25:23.622 | INFO | "pinned" : " 29 GB" | 20260506 11:25:23.622 | INFO | } | 20260506 11:25:23.622 | INFO | Perf report for the last 5.13908 seconds: | 20260506 11:25:23.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:25:23.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:23.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:23.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:23.622 | INFO | ANALYSIS : 64.0%, 2.9, 96.2ms, 0 | 20260506 11:25:23.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:23.622 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 11:25:23.622 | INFO | PRE_HQ : 61.8%, 1.9, 62.0ms, 0 | 20260506 11:25:23.622 | INFO | BAZWRITER : 66.4%, 4.8, 163.2ms, 0 | 20260506 11:25:23.622 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 11:25:23.622 | INFO | Uploaded 36GiB to GPU in 2.11045s (17.058GiB/s) | 20260506 11:25:23.622 | INFO | Downloaded 38GiB to GPU in 2.20263s (17.2521GiB/s) | 20260506 11:25:28.761 | INFO | Analyzing chunk frames = [979456,979968) | 20260506 11:25:28.761 | INFO | MemInfo: { | 20260506 11:25:28.761 | INFO | "gpu" : " 17 GB", | 20260506 11:25:28.761 | INFO | "huge" : "", | 20260506 11:25:28.761 | INFO | "huge_shared" : " 27 GB", | 20260506 11:25:28.761 | INFO | "malloc" : " 14 GB", | 20260506 11:25:28.761 | INFO | "pinned" : " 29 GB" | 20260506 11:25:28.761 | INFO | } | 20260506 11:25:28.761 | INFO | Perf report for the last 5.13968 seconds: | 20260506 11:25:28.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:25:28.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:28.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:28.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:28.761 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 11:25:28.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:28.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:28.762 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:25:28.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:28.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:28.762 | INFO | Uploaded 36GiB to GPU in 2.00541s (17.9515GiB/s) | 20260506 11:25:28.762 | INFO | Downloaded 34GiB to GPU in 1.91134s (17.7885GiB/s) | 20260506 11:25:33.901 | INFO | Analyzing chunk frames = [979968,980480) | 20260506 11:25:33.901 | INFO | MemInfo: { | 20260506 11:25:33.901 | INFO | "gpu" : " 17 GB", | 20260506 11:25:33.901 | INFO | "huge" : "", | 20260506 11:25:33.901 | INFO | "huge_shared" : " 27 GB", | 20260506 11:25:33.901 | INFO | "malloc" : " 14 GB", | 20260506 11:25:33.901 | INFO | "pinned" : " 29 GB" | 20260506 11:25:33.901 | INFO | } | 20260506 11:25:33.901 | INFO | Perf report for the last 5.13948 seconds: | 20260506 11:25:33.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:25:33.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:33.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:33.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:33.901 | INFO | ANALYSIS : 62.3%, 3.0, 96.6ms, 0 | 20260506 11:25:33.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:33.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:33.901 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:25:33.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:33.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:33.901 | INFO | Uploaded 36GiB to GPU in 2.05519s (17.5167GiB/s) | 20260506 11:25:33.901 | INFO | Downloaded 34GiB to GPU in 1.94731s (17.46GiB/s) | 20260506 11:25:39.040 | INFO | Analyzing chunk frames = [980480,980992) | 20260506 11:25:39.040 | INFO | MemInfo: { | 20260506 11:25:39.040 | INFO | "gpu" : " 17 GB", | 20260506 11:25:39.040 | INFO | "huge" : "", | 20260506 11:25:39.040 | INFO | "huge_shared" : " 27 GB", | 20260506 11:25:39.040 | INFO | "malloc" : " 14 GB", | 20260506 11:25:39.040 | INFO | "pinned" : " 29 GB" | 20260506 11:25:39.040 | INFO | } | 20260506 11:25:39.040 | INFO | Perf report for the last 5.13893 seconds: | 20260506 11:25:39.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:25:39.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:39.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:39.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:39.040 | INFO | ANALYSIS : 62.3%, 2.9, 96.2ms, 0 | 20260506 11:25:39.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:39.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:39.040 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:25:39.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:39.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:39.040 | INFO | Uploaded 36GiB to GPU in 1.98761s (18.1122GiB/s) | 20260506 11:25:39.040 | INFO | Downloaded 34GiB to GPU in 1.97781s (17.1907GiB/s) | 20260506 11:25:44.180 | INFO | Analyzing chunk frames = [980992,981504) | 20260506 11:25:44.180 | INFO | MemInfo: { | 20260506 11:25:44.180 | INFO | "gpu" : " 17 GB", | 20260506 11:25:44.180 | INFO | "huge" : "", | 20260506 11:25:44.180 | INFO | "huge_shared" : " 27 GB", | 20260506 11:25:44.180 | INFO | "malloc" : " 14 GB", | 20260506 11:25:44.180 | INFO | "pinned" : " 29 GB" | 20260506 11:25:44.180 | INFO | } | 20260506 11:25:44.180 | INFO | Perf report for the last 5.14001 seconds: | 20260506 11:25:44.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:25:44.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:44.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:44.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:44.180 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 11:25:44.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:44.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:44.180 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:25:44.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:44.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:44.180 | INFO | Uploaded 36GiB to GPU in 2.00546s (17.951GiB/s) | 20260506 11:25:44.180 | INFO | Downloaded 34GiB to GPU in 1.95937s (17.3525GiB/s) | 20260506 11:25:49.319 | INFO | Analyzing chunk frames = [981504,982016) | 20260506 11:25:49.319 | INFO | MemInfo: { | 20260506 11:25:49.319 | INFO | "gpu" : " 17 GB", | 20260506 11:25:49.319 | INFO | "huge" : "", | 20260506 11:25:49.319 | INFO | "huge_shared" : " 27 GB", | 20260506 11:25:49.319 | INFO | "malloc" : " 14 GB", | 20260506 11:25:49.319 | INFO | "pinned" : " 29 GB" | 20260506 11:25:49.319 | INFO | } | 20260506 11:25:49.319 | INFO | Perf report for the last 5.13872 seconds: | 20260506 11:25:49.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:25:49.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:49.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:49.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:49.319 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 11:25:49.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:49.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:49.319 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:25:49.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:49.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:49.319 | INFO | Uploaded 36GiB to GPU in 2.0182s (17.8377GiB/s) | 20260506 11:25:49.319 | INFO | Downloaded 34GiB to GPU in 1.97571s (17.209GiB/s) | 20260506 11:25:54.459 | INFO | Analyzing chunk frames = [982016,982528) | 20260506 11:25:54.459 | INFO | MemInfo: { | 20260506 11:25:54.459 | INFO | "gpu" : " 17 GB", | 20260506 11:25:54.459 | INFO | "huge" : "", | 20260506 11:25:54.459 | INFO | "huge_shared" : " 27 GB", | 20260506 11:25:54.459 | INFO | "malloc" : " 14 GB", | 20260506 11:25:54.459 | INFO | "pinned" : " 29 GB" | 20260506 11:25:54.459 | INFO | } | 20260506 11:25:54.459 | INFO | Perf report for the last 5.14035 seconds: | 20260506 11:25:54.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:25:54.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:54.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:54.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:54.459 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260506 11:25:54.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:54.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:54.459 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 11:25:54.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:54.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:54.459 | INFO | Uploaded 36GiB to GPU in 2.0337s (17.7017GiB/s) | 20260506 11:25:54.459 | INFO | Downloaded 34GiB to GPU in 2.00509s (16.9568GiB/s) | 20260506 11:25:59.598 | INFO | Analyzing chunk frames = [982528,983040) | 20260506 11:25:59.598 | INFO | MemInfo: { | 20260506 11:25:59.598 | INFO | "gpu" : " 17 GB", | 20260506 11:25:59.598 | INFO | "huge" : "", | 20260506 11:25:59.598 | INFO | "huge_shared" : " 27 GB", | 20260506 11:25:59.598 | INFO | "malloc" : " 14 GB", | 20260506 11:25:59.598 | INFO | "pinned" : " 29 GB" | 20260506 11:25:59.598 | INFO | } | 20260506 11:25:59.598 | INFO | Perf report for the last 5.13882 seconds: | 20260506 11:25:59.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:25:59.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:59.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:59.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:59.598 | INFO | ANALYSIS : 63.6%, 3.0, 99.3ms, 0 | 20260506 11:25:59.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:59.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:59.598 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260506 11:25:59.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:59.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:25:59.598 | INFO | Uploaded 36GiB to GPU in 2.10894s (17.0702GiB/s) | 20260506 11:25:59.598 | INFO | Downloaded 34GiB to GPU in 2.04227s (16.6481GiB/s) | 20260506 11:25:59.598 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:25:59.598 | INFO | Total mem usage: 16.8 GiB | 20260506 11:25:59.598 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:25:59.598 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -7.04%) | 20260506 11:25:59.598 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:25:59.598 | INFO | High Water Breakdown: | 20260506 11:25:59.598 | INFO | 60.8% : 9.54 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:25:59.598 | INFO | 39.1% : 6.14 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:25:59.598 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:25:59.598 | INFO | ------------------------------------------ | 20260506 11:25:59.598 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:25:59.598 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:25:59.598 | INFO | Total mem usage: 26.5 GiB | 20260506 11:25:59.598 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:25:59.598 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:25:59.599 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:25:59.599 | INFO | High Water Breakdown: | 20260506 11:25:59.599 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:25:59.599 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:25:59.599 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:25:59.599 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:25:59.599 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:25:59.599 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:25:59.599 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:25:59.599 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:25:59.599 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:25:59.599 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:25:59.599 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:25:59.599 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:25:59.599 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:25:59.599 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:25:59.599 | INFO | ------------------------------------------ | 20260506 11:25:59.599 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:25:59.599 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:25:59.599 | INFO | Total mem usage: 25.6 GiB | 20260506 11:25:59.599 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:25:59.599 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:25:59.599 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:25:59.599 | INFO | High Water Breakdown: | 20260506 11:25:59.599 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:25:59.599 | INFO | ------------------------------------------ | 20260506 11:25:59.599 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:25:59.599 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:25:59.599 | INFO | Total mem usage: 15.7 GiB | 20260506 11:25:59.599 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:25:59.599 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:25:59.599 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 11:25:59.599 | INFO | High Water Breakdown: | 20260506 11:25:59.599 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:25:59.599 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:25:59.599 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:25:59.599 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:25:59.599 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:25:59.599 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:25:59.599 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:25:59.599 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:25:59.599 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:25:59.599 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:25:59.599 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:25:59.599 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:25:59.599 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:25:59.599 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:25:59.599 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:25:59.599 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:25:59.599 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:25:59.599 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:25:59.599 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:25:59.599 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:25:59.599 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:25:59.599 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:25:59.599 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:25:59.599 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:25:59.599 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:25:59.599 | INFO | ------------------------------------------ | 20260506 11:25:59.599 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:25:59.599 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:25:59.599 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:25:59.599 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:25:59.599 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.68e+01, 1.38e+01 / 2.17e+00 | 20260506 11:25:59.599 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 9.94e-02, 1.10e-02 / 5.51e-03 | 20260506 11:25:59.600 | INFO | PerfReport: 11.56%, Baseline -- 3.70e+00 / 4.50e+00, 5.98e-01 / 2.96e-01 | 20260506 11:25:59.600 | INFO | PerfReport: 3.25%, Histogram -- 1.04e+00 / 1.45e+00, 1.96e-01 / 1.40e-01 | 20260506 11:25:59.600 | INFO | PerfReport: 16.01%, DME -- 4.10e+01 / 4.64e+01, 4.51e+00 / 2.34e+00 | 20260506 11:25:59.600 | INFO | PerfReport: 56.02%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.42e+00 / 1.41e+00 | 20260506 11:25:59.600 | INFO | PerfReport: 7.66%, PulseAccumulating -- 2.45e+00 / 3.07e+00, 4.28e-01 / 2.54e-01 | 20260506 11:25:59.600 | INFO | PerfReport: 3.85%, Metrics -- 1.23e+00 / 1.65e+00, 3.13e-01 / 1.79e-01 | 20260506 11:25:59.600 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.37e-01 / 5.00e-01, 6.87e-02 / 3.27e-02 | 20260506 11:25:59.600 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:25:59.600 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:25:59.600 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:25:59.600 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260506 11:25:59.600 | INFO | PerfReport: 21.49%, Upload -- 2.13e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 11:25:59.600 | INFO | PerfReport: 21.08%, Download -- 2.09e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260506 11:25:59.600 | INFO | PerfReport: 32.35%, Compute -- 3.20e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260506 11:25:59.600 | INFO | PerfReport: 1.94%, LockUpload -- 1.92e+00 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260506 11:25:59.600 | INFO | PerfReport: 0.98%, LockDownload -- 9.72e-01 / 0.00e+00, 2.57e+00 / 0.00e+00 | 20260506 11:25:59.600 | INFO | PerfReport: 22.15%, LockCompute -- 2.19e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 11:25:59.600 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260506 11:25:59.718 | INFO | Starting a round of Baz IO | 20260506 11:26:02.924 | INFO | Populating RTMetrics report | 20260506 11:26:02.925 | INFO | RTMetrics report populated | 20260506 11:26:02.925 | INFO | Opening tmp RTMetrics json file | 20260506 11:26:02.926 | INFO | Serializing RTMetrics to json | 20260506 11:26:02.933 | INFO | Serialization done, swapping tmp file now... | 20260506 11:26:02.991 | INFO | RTMetrics done | 20260506 11:26:03.542 | INFO | Finished a round of Baz IO: Wrote 2.056 GiB in 3.823 seconds | 20260506 11:26:03.542 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.21% / 0.04% / 55.76% | 20260506 11:26:03.542 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.20484% / 6.75519e-05% / 3.17891e-05% | 20260506 11:26:03.542 | INFO | Baz IO overall: | 20260506 11:26:03.542 | INFO | BytesWritten : 470608204014 | 20260506 11:26:03.542 | INFO | NumEvents : 227043921809 | 20260506 11:26:04.736 | INFO | Analyzing chunk frames = [983040,983552) | 20260506 11:26:04.736 | INFO | MemInfo: { | 20260506 11:26:04.736 | INFO | "gpu" : " 17 GB", | 20260506 11:26:04.736 | INFO | "huge" : "", | 20260506 11:26:04.736 | INFO | "huge_shared" : " 27 GB", | 20260506 11:26:04.736 | INFO | "malloc" : " 14 GB", | 20260506 11:26:04.736 | INFO | "pinned" : " 29 GB" | 20260506 11:26:04.736 | INFO | } | 20260506 11:26:04.737 | INFO | Perf report for the last 5.13867 seconds: | 20260506 11:26:04.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:26:04.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:04.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:04.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:04.737 | INFO | ANALYSIS : 63.8%, 2.9, 96.3ms, 0 | 20260506 11:26:04.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:04.737 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 11:26:04.737 | INFO | PRE_HQ : 61.8%, 1.9, 61.5ms, 0 | 20260506 11:26:04.737 | INFO | BAZWRITER : 73.6%, 4.8, 181.7ms, 0 | 20260506 11:26:04.737 | INFO | BAZ_SAMPLING : 3.4%, 1.0, 1.8ms, 0 | 20260506 11:26:04.737 | INFO | Uploaded 36GiB to GPU in 2.11611s (17.0123GiB/s) | 20260506 11:26:04.737 | INFO | Downloaded 38GiB to GPU in 2.21674s (17.1423GiB/s) | 20260506 11:26:09.876 | INFO | Analyzing chunk frames = [983552,984064) | 20260506 11:26:09.876 | INFO | MemInfo: { | 20260506 11:26:09.876 | INFO | "gpu" : " 17 GB", | 20260506 11:26:09.876 | INFO | "huge" : "", | 20260506 11:26:09.876 | INFO | "huge_shared" : " 27 GB", | 20260506 11:26:09.876 | INFO | "malloc" : " 14 GB", | 20260506 11:26:09.876 | INFO | "pinned" : " 29 GB" | 20260506 11:26:09.876 | INFO | } | 20260506 11:26:09.876 | INFO | Perf report for the last 5.13945 seconds: | 20260506 11:26:09.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:26:09.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:09.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:09.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:09.876 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 11:26:09.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:09.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:09.876 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 11:26:09.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:09.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:09.876 | INFO | Uploaded 36GiB to GPU in 2.0179s (17.8403GiB/s) | 20260506 11:26:09.876 | INFO | Downloaded 34GiB to GPU in 1.90361s (17.8608GiB/s) | 20260506 11:26:15.015 | INFO | Analyzing chunk frames = [984064,984576) | 20260506 11:26:15.015 | INFO | MemInfo: { | 20260506 11:26:15.015 | INFO | "gpu" : " 17 GB", | 20260506 11:26:15.015 | INFO | "huge" : "", | 20260506 11:26:15.015 | INFO | "huge_shared" : " 27 GB", | 20260506 11:26:15.015 | INFO | "malloc" : " 14 GB", | 20260506 11:26:15.015 | INFO | "pinned" : " 29 GB" | 20260506 11:26:15.015 | INFO | } | 20260506 11:26:15.015 | INFO | Perf report for the last 5.13932 seconds: | 20260506 11:26:15.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:26:15.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:15.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:15.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:15.016 | INFO | ANALYSIS : 62.4%, 3.0, 96.6ms, 0 | 20260506 11:26:15.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:15.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:15.016 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:26:15.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:15.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:15.016 | INFO | Uploaded 36GiB to GPU in 2.04433s (17.6097GiB/s) | 20260506 11:26:15.016 | INFO | Downloaded 34GiB to GPU in 1.95796s (17.365GiB/s) | 20260506 11:26:20.154 | INFO | Analyzing chunk frames = [984576,985088) | 20260506 11:26:20.155 | INFO | MemInfo: { | 20260506 11:26:20.155 | INFO | "gpu" : " 17 GB", | 20260506 11:26:20.155 | INFO | "huge" : "", | 20260506 11:26:20.155 | INFO | "huge_shared" : " 27 GB", | 20260506 11:26:20.155 | INFO | "malloc" : " 14 GB", | 20260506 11:26:20.155 | INFO | "pinned" : " 29 GB" | 20260506 11:26:20.155 | INFO | } | 20260506 11:26:20.155 | INFO | Perf report for the last 5.13925 seconds: | 20260506 11:26:20.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:26:20.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:20.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:20.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:20.155 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260506 11:26:20.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:20.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:20.155 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:26:20.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:20.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:20.155 | INFO | Uploaded 36GiB to GPU in 2.08313s (17.2817GiB/s) | 20260506 11:26:20.155 | INFO | Downloaded 34GiB to GPU in 1.96465s (17.3059GiB/s) | 20260506 11:26:25.294 | INFO | Analyzing chunk frames = [985088,985600) | 20260506 11:26:25.294 | INFO | MemInfo: { | 20260506 11:26:25.294 | INFO | "gpu" : " 17 GB", | 20260506 11:26:25.294 | INFO | "huge" : "", | 20260506 11:26:25.294 | INFO | "huge_shared" : " 27 GB", | 20260506 11:26:25.294 | INFO | "malloc" : " 14 GB", | 20260506 11:26:25.294 | INFO | "pinned" : " 29 GB" | 20260506 11:26:25.294 | INFO | } | 20260506 11:26:25.294 | INFO | Perf report for the last 5.13938 seconds: | 20260506 11:26:25.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:26:25.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:25.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:25.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:25.294 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 11:26:25.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:25.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:25.294 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:26:25.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:25.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:25.294 | INFO | Uploaded 36GiB to GPU in 2.09844s (17.1556GiB/s) | 20260506 11:26:25.295 | INFO | Downloaded 34GiB to GPU in 1.99995s (17.0005GiB/s) | 20260506 11:26:30.433 | INFO | Analyzing chunk frames = [985600,986112) | 20260506 11:26:30.433 | INFO | MemInfo: { | 20260506 11:26:30.433 | INFO | "gpu" : " 17 GB", | 20260506 11:26:30.433 | INFO | "huge" : "", | 20260506 11:26:30.433 | INFO | "huge_shared" : " 27 GB", | 20260506 11:26:30.433 | INFO | "malloc" : " 14 GB", | 20260506 11:26:30.434 | INFO | "pinned" : " 29 GB" | 20260506 11:26:30.434 | INFO | } | 20260506 11:26:30.434 | INFO | Perf report for the last 5.13957 seconds: | 20260506 11:26:30.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:26:30.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:30.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:30.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:30.434 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 11:26:30.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:30.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:30.434 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:26:30.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:30.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:30.434 | INFO | Uploaded 36GiB to GPU in 1.98639s (18.1234GiB/s) | 20260506 11:26:30.434 | INFO | Downloaded 34GiB to GPU in 1.98278s (17.1477GiB/s) | 20260506 11:26:35.574 | INFO | Analyzing chunk frames = [986112,986624) | 20260506 11:26:35.574 | INFO | MemInfo: { | 20260506 11:26:35.574 | INFO | "gpu" : " 17 GB", | 20260506 11:26:35.574 | INFO | "huge" : "", | 20260506 11:26:35.574 | INFO | "huge_shared" : " 27 GB", | 20260506 11:26:35.574 | INFO | "malloc" : " 14 GB", | 20260506 11:26:35.574 | INFO | "pinned" : " 29 GB" | 20260506 11:26:35.574 | INFO | } | 20260506 11:26:35.574 | INFO | Perf report for the last 5.14052 seconds: | 20260506 11:26:35.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:26:35.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:35.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:35.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:35.574 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 11:26:35.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:35.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:35.574 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 11:26:35.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:35.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:35.574 | INFO | Uploaded 36GiB to GPU in 2.0271s (17.7594GiB/s) | 20260506 11:26:35.574 | INFO | Downloaded 34GiB to GPU in 2.00696s (16.9411GiB/s) | 20260506 11:26:40.712 | INFO | Analyzing chunk frames = [986624,987136) | 20260506 11:26:40.712 | INFO | MemInfo: { | 20260506 11:26:40.712 | INFO | "gpu" : " 17 GB", | 20260506 11:26:40.712 | INFO | "huge" : "", | 20260506 11:26:40.712 | INFO | "huge_shared" : " 27 GB", | 20260506 11:26:40.712 | INFO | "malloc" : " 14 GB", | 20260506 11:26:40.712 | INFO | "pinned" : " 29 GB" | 20260506 11:26:40.712 | INFO | } | 20260506 11:26:40.713 | INFO | Perf report for the last 5.1384 seconds: | 20260506 11:26:40.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:26:40.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:40.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:40.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:40.713 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260506 11:26:40.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:40.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:40.713 | INFO | PRE_HQ : 34.5%, 1.0, 18.0ms, 0 | 20260506 11:26:40.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:40.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:40.713 | INFO | Uploaded 36GiB to GPU in 2.00605s (17.9457GiB/s) | 20260506 11:26:40.713 | INFO | Downloaded 34GiB to GPU in 1.98245s (17.1505GiB/s) | 20260506 11:26:40.844 | INFO | Starting a round of Baz IO | 20260506 11:26:44.026 | INFO | Populating RTMetrics report | 20260506 11:26:44.026 | INFO | RTMetrics report populated | 20260506 11:26:44.026 | INFO | Opening tmp RTMetrics json file | 20260506 11:26:44.027 | INFO | Serializing RTMetrics to json | 20260506 11:26:44.029 | INFO | Serialization done, swapping tmp file now... | 20260506 11:26:44.029 | INFO | RTMetrics done | 20260506 11:26:44.196 | INFO | Finished a round of Baz IO: Wrote 2.056 GiB in 3.351 seconds | 20260506 11:26:44.196 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.17% / 0.04% / 55.79% | 20260506 11:26:44.196 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.18261% / 7.54992e-05% / 4.37101e-05% | 20260506 11:26:44.196 | INFO | Baz IO overall: | 20260506 11:26:44.196 | INFO | BytesWritten : 472816151929 | 20260506 11:26:44.196 | INFO | NumEvents : 228115574720 | 20260506 11:26:45.851 | INFO | Analyzing chunk frames = [987136,987648) | 20260506 11:26:45.851 | INFO | MemInfo: { | 20260506 11:26:45.851 | INFO | "gpu" : " 17 GB", | 20260506 11:26:45.851 | INFO | "huge" : "", | 20260506 11:26:45.851 | INFO | "huge_shared" : " 27 GB", | 20260506 11:26:45.851 | INFO | "malloc" : " 14 GB", | 20260506 11:26:45.852 | INFO | "pinned" : " 29 GB" | 20260506 11:26:45.852 | INFO | } | 20260506 11:26:45.852 | INFO | Perf report for the last 5.13908 seconds: | 20260506 11:26:45.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:26:45.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:45.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:45.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:45.852 | INFO | ANALYSIS : 63.8%, 2.9, 95.8ms, 0 | 20260506 11:26:45.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:45.852 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260506 11:26:45.852 | INFO | PRE_HQ : 62.0%, 1.9, 62.1ms, 0 | 20260506 11:26:45.852 | INFO | BAZWRITER : 64.4%, 4.5, 149.4ms, 0 | 20260506 11:26:45.852 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 11:26:45.852 | INFO | Uploaded 36GiB to GPU in 2.14453s (16.7869GiB/s) | 20260506 11:26:45.852 | INFO | Downloaded 38GiB to GPU in 2.22781s (17.0571GiB/s) | 20260506 11:26:50.991 | INFO | Analyzing chunk frames = [987648,988160) | 20260506 11:26:50.991 | INFO | MemInfo: { | 20260506 11:26:50.991 | INFO | "gpu" : " 17 GB", | 20260506 11:26:50.991 | INFO | "huge" : "", | 20260506 11:26:50.991 | INFO | "huge_shared" : " 27 GB", | 20260506 11:26:50.991 | INFO | "malloc" : " 14 GB", | 20260506 11:26:50.991 | INFO | "pinned" : " 29 GB" | 20260506 11:26:50.991 | INFO | } | 20260506 11:26:50.991 | INFO | Perf report for the last 5.13927 seconds: | 20260506 11:26:50.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:26:50.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:50.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:50.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:50.991 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260506 11:26:50.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:50.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:50.991 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 11:26:50.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:50.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:50.991 | INFO | Uploaded 36GiB to GPU in 2.07494s (17.3499GiB/s) | 20260506 11:26:50.991 | INFO | Downloaded 34GiB to GPU in 1.94397s (17.49GiB/s) | 20260506 11:26:56.130 | INFO | Analyzing chunk frames = [988160,988672) | 20260506 11:26:56.130 | INFO | MemInfo: { | 20260506 11:26:56.130 | INFO | "gpu" : " 17 GB", | 20260506 11:26:56.130 | INFO | "huge" : "", | 20260506 11:26:56.130 | INFO | "huge_shared" : " 27 GB", | 20260506 11:26:56.130 | INFO | "malloc" : " 14 GB", | 20260506 11:26:56.130 | INFO | "pinned" : " 29 GB" | 20260506 11:26:56.130 | INFO | } | 20260506 11:26:56.130 | INFO | Perf report for the last 5.13941 seconds: | 20260506 11:26:56.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:26:56.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:56.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:56.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:56.130 | INFO | ANALYSIS : 62.6%, 3.0, 96.9ms, 0 | 20260506 11:26:56.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:56.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:56.130 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:26:56.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:56.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:26:56.130 | INFO | Uploaded 36GiB to GPU in 2.08608s (17.2572GiB/s) | 20260506 11:26:56.131 | INFO | Downloaded 34GiB to GPU in 1.96048s (17.3427GiB/s) | 20260506 11:27:01.269 | INFO | Analyzing chunk frames = [988672,989184) | 20260506 11:27:01.269 | INFO | MemInfo: { | 20260506 11:27:01.269 | INFO | "gpu" : " 17 GB", | 20260506 11:27:01.269 | INFO | "huge" : "", | 20260506 11:27:01.269 | INFO | "huge_shared" : " 27 GB", | 20260506 11:27:01.269 | INFO | "malloc" : " 14 GB", | 20260506 11:27:01.269 | INFO | "pinned" : " 29 GB" | 20260506 11:27:01.269 | INFO | } | 20260506 11:27:01.269 | INFO | Perf report for the last 5.13918 seconds: | 20260506 11:27:01.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:27:01.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:01.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:01.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:01.270 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260506 11:27:01.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:01.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:01.270 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:27:01.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:01.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:01.270 | INFO | Uploaded 36GiB to GPU in 2.09047s (17.221GiB/s) | 20260506 11:27:01.270 | INFO | Downloaded 34GiB to GPU in 1.97778s (17.191GiB/s) | 20260506 11:27:06.409 | INFO | Analyzing chunk frames = [989184,989696) | 20260506 11:27:06.409 | INFO | MemInfo: { | 20260506 11:27:06.409 | INFO | "gpu" : " 17 GB", | 20260506 11:27:06.409 | INFO | "huge" : "", | 20260506 11:27:06.409 | INFO | "huge_shared" : " 27 GB", | 20260506 11:27:06.409 | INFO | "malloc" : " 14 GB", | 20260506 11:27:06.409 | INFO | "pinned" : " 29 GB" | 20260506 11:27:06.409 | INFO | } | 20260506 11:27:06.409 | INFO | Perf report for the last 5.13923 seconds: | 20260506 11:27:06.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:27:06.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:06.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:06.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:06.409 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 11:27:06.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:06.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:06.409 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:27:06.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:06.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:06.409 | INFO | Uploaded 36GiB to GPU in 2.09486s (17.1849GiB/s) | 20260506 11:27:06.409 | INFO | Downloaded 34GiB to GPU in 1.99934s (17.0056GiB/s) | 20260506 11:27:11.548 | INFO | Analyzing chunk frames = [989696,990208) | 20260506 11:27:11.548 | INFO | MemInfo: { | 20260506 11:27:11.548 | INFO | "gpu" : " 17 GB", | 20260506 11:27:11.548 | INFO | "huge" : "", | 20260506 11:27:11.548 | INFO | "huge_shared" : " 27 GB", | 20260506 11:27:11.548 | INFO | "malloc" : " 14 GB", | 20260506 11:27:11.548 | INFO | "pinned" : " 29 GB" | 20260506 11:27:11.548 | INFO | } | 20260506 11:27:11.548 | INFO | Perf report for the last 5.13933 seconds: | 20260506 11:27:11.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:27:11.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:11.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:11.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:11.548 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 11:27:11.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:11.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:11.548 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:27:11.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:11.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:11.549 | INFO | Uploaded 36GiB to GPU in 2.06926s (17.3975GiB/s) | 20260506 11:27:11.549 | INFO | Downloaded 34GiB to GPU in 2.00034s (16.9971GiB/s) | 20260506 11:27:16.688 | INFO | Analyzing chunk frames = [990208,990720) | 20260506 11:27:16.688 | INFO | MemInfo: { | 20260506 11:27:16.688 | INFO | "gpu" : " 17 GB", | 20260506 11:27:16.688 | INFO | "huge" : "", | 20260506 11:27:16.688 | INFO | "huge_shared" : " 27 GB", | 20260506 11:27:16.689 | INFO | "malloc" : " 14 GB", | 20260506 11:27:16.689 | INFO | "pinned" : " 29 GB" | 20260506 11:27:16.689 | INFO | } | 20260506 11:27:16.689 | INFO | Perf report for the last 5.14055 seconds: | 20260506 11:27:16.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:27:16.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:16.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:16.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:16.689 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:27:16.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:16.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:16.689 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 11:27:16.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:16.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:16.689 | INFO | Uploaded 36GiB to GPU in 2.02999s (17.7341GiB/s) | 20260506 11:27:16.689 | INFO | Downloaded 34GiB to GPU in 2.01556s (16.8687GiB/s) | 20260506 11:27:21.827 | INFO | Analyzing chunk frames = [990720,991232) | 20260506 11:27:21.827 | INFO | MemInfo: { | 20260506 11:27:21.827 | INFO | "gpu" : " 17 GB", | 20260506 11:27:21.827 | INFO | "huge" : "", | 20260506 11:27:21.827 | INFO | "huge_shared" : " 27 GB", | 20260506 11:27:21.827 | INFO | "malloc" : " 14 GB", | 20260506 11:27:21.827 | INFO | "pinned" : " 29 GB" | 20260506 11:27:21.827 | INFO | } | 20260506 11:27:21.827 | INFO | Perf report for the last 5.13813 seconds: | 20260506 11:27:21.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:27:21.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:21.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:21.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:21.827 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 11:27:21.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:21.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:21.827 | INFO | PRE_HQ : 33.2%, 1.0, 17.2ms, 0 | 20260506 11:27:21.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:21.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:21.827 | INFO | Uploaded 36GiB to GPU in 1.98712s (18.1166GiB/s) | 20260506 11:27:21.827 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:27:21.827 | INFO | Total mem usage: 16.8 GiB | 20260506 11:27:21.827 | INFO | Downloaded 34GiB to GPU in 1.9593s (17.3532GiB/s) | 20260506 11:27:21.827 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:27:21.827 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -5.56%) | 20260506 11:27:21.827 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:27:21.827 | INFO | High Water Breakdown: | 20260506 11:27:21.827 | INFO | 61% : 9.71 GiB ( 93.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:27:21.827 | INFO | 38.9% : 6.19 GiB ( 92.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:27:21.827 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:27:21.827 | INFO | ------------------------------------------ | 20260506 11:27:21.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:27:21.827 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:27:21.827 | INFO | Total mem usage: 26.5 GiB | 20260506 11:27:21.827 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:27:21.827 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:27:21.827 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:27:21.827 | INFO | High Water Breakdown: | 20260506 11:27:21.827 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:27:21.827 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:27:21.827 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:27:21.827 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:27:21.827 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:27:21.827 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:27:21.827 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:27:21.827 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:27:21.827 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:27:21.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:27:21.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:27:21.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:27:21.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:27:21.827 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:27:21.827 | INFO | ------------------------------------------ | 20260506 11:27:21.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:27:21.827 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:27:21.827 | INFO | Total mem usage: 25.6 GiB | 20260506 11:27:21.827 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:27:21.827 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:27:21.827 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:27:21.827 | INFO | High Water Breakdown: | 20260506 11:27:21.827 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:27:21.827 | INFO | ------------------------------------------ | 20260506 11:27:21.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:27:21.827 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:27:21.827 | INFO | Total mem usage: 15.7 GiB | 20260506 11:27:21.827 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:27:21.827 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:27:21.827 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 11:27:21.828 | INFO | High Water Breakdown: | 20260506 11:27:21.828 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:27:21.828 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:27:21.828 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:27:21.828 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:27:21.828 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:27:21.828 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:27:21.828 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:27:21.828 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:27:21.828 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:27:21.828 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:27:21.828 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:27:21.828 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:27:21.828 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:27:21.828 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:27:21.828 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:27:21.828 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:27:21.828 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:27:21.828 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:27:21.828 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:27:21.828 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:27:21.828 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:27:21.828 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:27:21.828 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:27:21.828 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:27:21.828 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:27:21.828 | INFO | ------------------------------------------ | 20260506 11:27:21.828 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:27:21.828 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:27:21.828 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:27:21.828 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:27:21.828 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.63e+01, 1.36e+01 / 2.24e+00 | 20260506 11:27:21.828 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.07e-02 / 6.89e-03 | 20260506 11:27:21.828 | INFO | PerfReport: 11.59%, Baseline -- 3.70e+00 / 4.45e+00, 5.97e-01 / 3.46e-01 | 20260506 11:27:21.828 | INFO | PerfReport: 3.24%, Histogram -- 1.04e+00 / 1.47e+00, 1.93e-01 / 1.52e-01 | 20260506 11:27:21.828 | INFO | PerfReport: 15.91%, DME -- 4.07e+01 / 4.56e+01, 4.23e+00 / 2.32e+00 | 20260506 11:27:21.828 | INFO | PerfReport: 56.05%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.37e+00 / 1.41e+00 | 20260506 11:27:21.828 | INFO | PerfReport: 7.68%, PulseAccumulating -- 2.45e+00 / 3.10e+00, 4.27e-01 / 2.09e-01 | 20260506 11:27:21.828 | INFO | PerfReport: 3.86%, Metrics -- 1.23e+00 / 1.65e+00, 3.11e-01 / 1.79e-01 | 20260506 11:27:21.828 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.37e-01 / 5.07e-01, 6.95e-02 / 6.22e-02 | 20260506 11:27:21.828 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:27:21.828 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:27:21.828 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:27:21.828 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.49e+01 / 0.00e+00 | 20260506 11:27:21.828 | INFO | PerfReport: 21.69%, Upload -- 2.15e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 11:27:21.828 | INFO | PerfReport: 21.12%, Download -- 2.09e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 11:27:21.828 | INFO | PerfReport: 32.29%, Compute -- 3.19e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 11:27:21.828 | INFO | PerfReport: 1.99%, LockUpload -- 1.97e+00 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 11:27:21.828 | INFO | PerfReport: 0.88%, LockDownload -- 8.74e-01 / 0.00e+00, 2.36e+00 / 0.00e+00 | 20260506 11:27:21.828 | INFO | PerfReport: 22.01%, LockCompute -- 2.18e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260506 11:27:21.828 | INFO | PerfReport: 0.02%, Misc -- 1.58e-02 / 0.00e+00, 2.55e-02 / 0.00e+00 | 20260506 11:27:21.951 | INFO | Starting a round of Baz IO | 20260506 11:27:25.141 | INFO | Populating RTMetrics report | 20260506 11:27:25.141 | INFO | RTMetrics report populated | 20260506 11:27:25.141 | INFO | Opening tmp RTMetrics json file | 20260506 11:27:25.160 | INFO | Serializing RTMetrics to json | 20260506 11:27:25.162 | INFO | Serialization done, swapping tmp file now... | 20260506 11:27:25.163 | INFO | RTMetrics done | 20260506 11:27:25.375 | INFO | Finished a round of Baz IO: Wrote 2.057 GiB in 3.423 seconds | 20260506 11:27:25.375 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.14% / 0.04% / 55.83% | 20260506 11:27:25.375 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.16738% / 4.76837e-05% / 4.37101e-05% | 20260506 11:27:25.375 | INFO | Baz IO overall: | 20260506 11:27:25.375 | INFO | BytesWritten : 475024638261 | 20260506 11:27:25.375 | INFO | NumEvents : 229187208623 | 20260506 11:27:26.966 | INFO | Analyzing chunk frames = [991232,991744) | 20260506 11:27:26.966 | INFO | MemInfo: { | 20260506 11:27:26.966 | INFO | "gpu" : " 17 GB", | 20260506 11:27:26.966 | INFO | "huge" : "", | 20260506 11:27:26.966 | INFO | "huge_shared" : " 27 GB", | 20260506 11:27:26.966 | INFO | "malloc" : " 14 GB", | 20260506 11:27:26.966 | INFO | "pinned" : " 29 GB" | 20260506 11:27:26.966 | INFO | } | 20260506 11:27:26.966 | INFO | Perf report for the last 5.13952 seconds: | 20260506 11:27:26.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:27:26.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:26.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:26.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:26.966 | INFO | ANALYSIS : 63.9%, 2.9, 96.5ms, 0 | 20260506 11:27:26.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:26.966 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 11:27:26.966 | INFO | PRE_HQ : 62.0%, 1.9, 61.5ms, 0 | 20260506 11:27:26.966 | INFO | BAZWRITER : 65.6%, 4.7, 159.0ms, 0 | 20260506 11:27:26.966 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 11:27:26.967 | INFO | Uploaded 36GiB to GPU in 2.15488s (16.7063GiB/s) | 20260506 11:27:26.967 | INFO | Downloaded 38GiB to GPU in 2.22904s (17.0477GiB/s) | 20260506 11:27:32.105 | INFO | Analyzing chunk frames = [991744,992256) | 20260506 11:27:32.105 | INFO | MemInfo: { | 20260506 11:27:32.105 | INFO | "gpu" : " 17 GB", | 20260506 11:27:32.105 | INFO | "huge" : "", | 20260506 11:27:32.105 | INFO | "huge_shared" : " 27 GB", | 20260506 11:27:32.105 | INFO | "malloc" : " 14 GB", | 20260506 11:27:32.105 | INFO | "pinned" : " 29 GB" | 20260506 11:27:32.105 | INFO | } | 20260506 11:27:32.106 | INFO | Perf report for the last 5.13932 seconds: | 20260506 11:27:32.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:27:32.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:32.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:32.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:32.106 | INFO | ANALYSIS : 63.2%, 3.0, 98.4ms, 0 | 20260506 11:27:32.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:32.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:32.106 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260506 11:27:32.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:32.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:32.106 | INFO | Uploaded 36GiB to GPU in 2.0374s (17.6696GiB/s) | 20260506 11:27:32.106 | INFO | Downloaded 34GiB to GPU in 1.96292s (17.3212GiB/s) | 20260506 11:27:37.245 | INFO | Analyzing chunk frames = [992256,992768) | 20260506 11:27:37.245 | INFO | MemInfo: { | 20260506 11:27:37.245 | INFO | "gpu" : " 17 GB", | 20260506 11:27:37.245 | INFO | "huge" : "", | 20260506 11:27:37.245 | INFO | "huge_shared" : " 27 GB", | 20260506 11:27:37.245 | INFO | "malloc" : " 14 GB", | 20260506 11:27:37.245 | INFO | "pinned" : " 29 GB" | 20260506 11:27:37.245 | INFO | } | 20260506 11:27:37.245 | INFO | Perf report for the last 5.13928 seconds: | 20260506 11:27:37.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:27:37.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:37.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:37.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:37.245 | INFO | ANALYSIS : 62.4%, 3.0, 96.7ms, 0 | 20260506 11:27:37.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:37.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:37.245 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:27:37.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:37.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:37.245 | INFO | Uploaded 36GiB to GPU in 2.07166s (17.3774GiB/s) | 20260506 11:27:37.245 | INFO | Downloaded 34GiB to GPU in 1.94967s (17.4389GiB/s) | 20260506 11:27:42.384 | INFO | Analyzing chunk frames = [992768,993280) | 20260506 11:27:42.384 | INFO | MemInfo: { | 20260506 11:27:42.384 | INFO | "gpu" : " 17 GB", | 20260506 11:27:42.384 | INFO | "huge" : "", | 20260506 11:27:42.384 | INFO | "huge_shared" : " 27 GB", | 20260506 11:27:42.384 | INFO | "malloc" : " 14 GB", | 20260506 11:27:42.384 | INFO | "pinned" : " 29 GB" | 20260506 11:27:42.384 | INFO | } | 20260506 11:27:42.384 | INFO | Perf report for the last 5.13926 seconds: | 20260506 11:27:42.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:27:42.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:42.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:42.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:42.384 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260506 11:27:42.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:42.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:42.384 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:27:42.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:42.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:42.384 | INFO | Uploaded 36GiB to GPU in 2.08378s (17.2763GiB/s) | 20260506 11:27:42.384 | INFO | Downloaded 34GiB to GPU in 1.95827s (17.3623GiB/s) | 20260506 11:27:47.523 | INFO | Analyzing chunk frames = [993280,993792) | 20260506 11:27:47.523 | INFO | MemInfo: { | 20260506 11:27:47.523 | INFO | "gpu" : " 17 GB", | 20260506 11:27:47.523 | INFO | "huge" : "", | 20260506 11:27:47.523 | INFO | "huge_shared" : " 27 GB", | 20260506 11:27:47.523 | INFO | "malloc" : " 14 GB", | 20260506 11:27:47.523 | INFO | "pinned" : " 29 GB" | 20260506 11:27:47.523 | INFO | } | 20260506 11:27:47.523 | INFO | Perf report for the last 5.1392 seconds: | 20260506 11:27:47.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:27:47.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:47.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:47.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:47.523 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 11:27:47.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:47.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:47.523 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:27:47.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:47.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:47.524 | INFO | Uploaded 36GiB to GPU in 2.10021s (17.1412GiB/s) | 20260506 11:27:47.524 | INFO | Downloaded 34GiB to GPU in 1.99964s (17.0031GiB/s) | 20260506 11:27:52.663 | INFO | Analyzing chunk frames = [993792,994304) | 20260506 11:27:52.663 | INFO | MemInfo: { | 20260506 11:27:52.663 | INFO | "gpu" : " 17 GB", | 20260506 11:27:52.663 | INFO | "huge" : "", | 20260506 11:27:52.663 | INFO | "huge_shared" : " 27 GB", | 20260506 11:27:52.663 | INFO | "malloc" : " 14 GB", | 20260506 11:27:52.663 | INFO | "pinned" : " 29 GB" | 20260506 11:27:52.663 | INFO | } | 20260506 11:27:52.663 | INFO | Perf report for the last 5.13926 seconds: | 20260506 11:27:52.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:27:52.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:52.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:52.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:52.663 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 11:27:52.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:52.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:52.663 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:27:52.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:52.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:52.663 | INFO | Uploaded 36GiB to GPU in 2.06714s (17.4154GiB/s) | 20260506 11:27:52.663 | INFO | Downloaded 34GiB to GPU in 1.99809s (17.0163GiB/s) | 20260506 11:27:57.803 | INFO | Analyzing chunk frames = [994304,994816) | 20260506 11:27:57.803 | INFO | MemInfo: { | 20260506 11:27:57.803 | INFO | "gpu" : " 17 GB", | 20260506 11:27:57.803 | INFO | "huge" : "", | 20260506 11:27:57.803 | INFO | "huge_shared" : " 27 GB", | 20260506 11:27:57.803 | INFO | "malloc" : " 14 GB", | 20260506 11:27:57.803 | INFO | "pinned" : " 29 GB" | 20260506 11:27:57.803 | INFO | } | 20260506 11:27:57.803 | INFO | Perf report for the last 5.14064 seconds: | 20260506 11:27:57.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:27:57.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:57.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:57.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:57.803 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:27:57.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:57.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:57.803 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 11:27:57.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:57.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:27:57.804 | INFO | Uploaded 36GiB to GPU in 2.03207s (17.716GiB/s) | 20260506 11:27:57.804 | INFO | Downloaded 34GiB to GPU in 2.01278s (16.8921GiB/s) | 20260506 11:28:02.942 | INFO | Analyzing chunk frames = [994816,995328) | 20260506 11:28:02.942 | INFO | MemInfo: { | 20260506 11:28:02.942 | INFO | "gpu" : " 17 GB", | 20260506 11:28:02.942 | INFO | "huge" : "", | 20260506 11:28:02.942 | INFO | "huge_shared" : " 27 GB", | 20260506 11:28:02.942 | INFO | "malloc" : " 14 GB", | 20260506 11:28:02.942 | INFO | "pinned" : " 29 GB" | 20260506 11:28:02.942 | INFO | } | 20260506 11:28:02.942 | INFO | Perf report for the last 5.13827 seconds: | 20260506 11:28:02.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:28:02.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:02.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:02.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:02.942 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260506 11:28:02.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:02.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:02.942 | INFO | PRE_HQ : 33.4%, 1.0, 17.4ms, 0 | 20260506 11:28:02.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:02.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:02.942 | INFO | Uploaded 36GiB to GPU in 2.04267s (17.624GiB/s) | 20260506 11:28:02.942 | INFO | Downloaded 34GiB to GPU in 1.96869s (17.2704GiB/s) | 20260506 11:28:03.059 | INFO | Starting a round of Baz IO | 20260506 11:28:06.265 | INFO | Populating RTMetrics report | 20260506 11:28:06.265 | INFO | RTMetrics report populated | 20260506 11:28:06.265 | INFO | Opening tmp RTMetrics json file | 20260506 11:28:06.283 | INFO | Serializing RTMetrics to json | 20260506 11:28:06.291 | INFO | Serialization done, swapping tmp file now... | 20260506 11:28:06.292 | INFO | RTMetrics done | 20260506 11:28:06.548 | INFO | Finished a round of Baz IO: Wrote 2.059 GiB in 3.488 seconds | 20260506 11:28:06.548 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.10% / 0.03% / 55.86% | 20260506 11:28:06.548 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.16044% / 5.16574e-05% / 4.37101e-05% | 20260506 11:28:06.548 | INFO | Baz IO overall: | 20260506 11:28:06.548 | INFO | BytesWritten : 477235460306 | 20260506 11:28:06.548 | INFO | NumEvents : 230261043107 | 20260506 11:28:08.081 | INFO | Analyzing chunk frames = [995328,995840) | 20260506 11:28:08.081 | INFO | MemInfo: { | 20260506 11:28:08.081 | INFO | "gpu" : " 17 GB", | 20260506 11:28:08.081 | INFO | "huge" : "", | 20260506 11:28:08.081 | INFO | "huge_shared" : " 27 GB", | 20260506 11:28:08.081 | INFO | "malloc" : " 14 GB", | 20260506 11:28:08.081 | INFO | "pinned" : " 29 GB" | 20260506 11:28:08.081 | INFO | } | 20260506 11:28:08.081 | INFO | Perf report for the last 5.13953 seconds: | 20260506 11:28:08.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:28:08.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:08.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:08.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:08.081 | INFO | ANALYSIS : 64.1%, 2.9, 96.0ms, 0 | 20260506 11:28:08.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:08.081 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260506 11:28:08.081 | INFO | PRE_HQ : 62.0%, 1.9, 61.5ms, 0 | 20260506 11:28:08.081 | INFO | BAZWRITER : 67.1%, 4.8, 164.4ms, 0 | 20260506 11:28:08.081 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 11:28:08.082 | INFO | Uploaded 36GiB to GPU in 2.14561s (16.7784GiB/s) | 20260506 11:28:08.082 | INFO | Downloaded 38GiB to GPU in 2.2223s (17.0994GiB/s) | 20260506 11:28:13.220 | INFO | Analyzing chunk frames = [995840,996352) | 20260506 11:28:13.220 | INFO | MemInfo: { | 20260506 11:28:13.220 | INFO | "gpu" : " 17 GB", | 20260506 11:28:13.220 | INFO | "huge" : "", | 20260506 11:28:13.220 | INFO | "huge_shared" : " 27 GB", | 20260506 11:28:13.220 | INFO | "malloc" : " 14 GB", | 20260506 11:28:13.220 | INFO | "pinned" : " 29 GB" | 20260506 11:28:13.220 | INFO | } | 20260506 11:28:13.220 | INFO | Perf report for the last 5.13897 seconds: | 20260506 11:28:13.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:28:13.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:13.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:13.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:13.220 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260506 11:28:13.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:13.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:13.220 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 11:28:13.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:13.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:13.220 | INFO | Uploaded 36GiB to GPU in 2.02964s (17.7371GiB/s) | 20260506 11:28:13.220 | INFO | Downloaded 34GiB to GPU in 1.92961s (17.6202GiB/s) | 20260506 11:28:18.359 | INFO | Analyzing chunk frames = [996352,996864) | 20260506 11:28:18.359 | INFO | MemInfo: { | 20260506 11:28:18.359 | INFO | "gpu" : " 17 GB", | 20260506 11:28:18.359 | INFO | "huge" : "", | 20260506 11:28:18.359 | INFO | "huge_shared" : " 27 GB", | 20260506 11:28:18.359 | INFO | "malloc" : " 14 GB", | 20260506 11:28:18.359 | INFO | "pinned" : " 29 GB" | 20260506 11:28:18.359 | INFO | } | 20260506 11:28:18.359 | INFO | Perf report for the last 5.13927 seconds: | 20260506 11:28:18.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:28:18.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:18.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:18.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:18.359 | INFO | ANALYSIS : 62.2%, 3.0, 96.4ms, 0 | 20260506 11:28:18.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:18.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:18.359 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:28:18.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:18.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:18.360 | INFO | Uploaded 36GiB to GPU in 1.998s (18.018GiB/s) | 20260506 11:28:18.360 | INFO | Downloaded 34GiB to GPU in 1.95508s (17.3906GiB/s) | 20260506 11:28:23.499 | INFO | Analyzing chunk frames = [996864,997376) | 20260506 11:28:23.499 | INFO | MemInfo: { | 20260506 11:28:23.499 | INFO | "gpu" : " 17 GB", | 20260506 11:28:23.499 | INFO | "huge" : "", | 20260506 11:28:23.499 | INFO | "huge_shared" : " 27 GB", | 20260506 11:28:23.499 | INFO | "malloc" : " 14 GB", | 20260506 11:28:23.499 | INFO | "pinned" : " 29 GB" | 20260506 11:28:23.499 | INFO | } | 20260506 11:28:23.499 | INFO | Perf report for the last 5.13926 seconds: | 20260506 11:28:23.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:28:23.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:23.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:23.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:23.499 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 11:28:23.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:23.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:23.499 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:28:23.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:23.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:23.499 | INFO | Uploaded 36GiB to GPU in 2.07453s (17.3533GiB/s) | 20260506 11:28:23.499 | INFO | Downloaded 34GiB to GPU in 1.97142s (17.2464GiB/s) | 20260506 11:28:28.639 | INFO | Analyzing chunk frames = [997376,997888) | 20260506 11:28:28.639 | INFO | MemInfo: { | 20260506 11:28:28.639 | INFO | "gpu" : " 17 GB", | 20260506 11:28:28.639 | INFO | "huge" : "", | 20260506 11:28:28.639 | INFO | "huge_shared" : " 27 GB", | 20260506 11:28:28.639 | INFO | "malloc" : " 14 GB", | 20260506 11:28:28.639 | INFO | "pinned" : " 29 GB" | 20260506 11:28:28.639 | INFO | } | 20260506 11:28:28.639 | INFO | Perf report for the last 5.13997 seconds: | 20260506 11:28:28.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:28:28.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:28.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:28.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:28.639 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 11:28:28.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:28.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:28.639 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:28:28.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:28.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:28.639 | INFO | Uploaded 36GiB to GPU in 2.04368s (17.6153GiB/s) | 20260506 11:28:28.639 | INFO | Downloaded 34GiB to GPU in 1.98804s (17.1023GiB/s) | 20260506 11:28:33.777 | INFO | Analyzing chunk frames = [997888,998400) | 20260506 11:28:33.777 | INFO | MemInfo: { | 20260506 11:28:33.777 | INFO | "gpu" : " 17 GB", | 20260506 11:28:33.777 | INFO | "huge" : "", | 20260506 11:28:33.777 | INFO | "huge_shared" : " 27 GB", | 20260506 11:28:33.777 | INFO | "malloc" : " 14 GB", | 20260506 11:28:33.778 | INFO | "pinned" : " 29 GB" | 20260506 11:28:33.778 | INFO | } | 20260506 11:28:33.778 | INFO | Perf report for the last 5.13882 seconds: | 20260506 11:28:33.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:28:33.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:33.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:33.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:33.778 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 11:28:33.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:33.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:33.778 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 11:28:33.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:33.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:33.778 | INFO | Uploaded 36GiB to GPU in 2.07752s (17.3284GiB/s) | 20260506 11:28:33.778 | INFO | Downloaded 34GiB to GPU in 2.00047s (16.996GiB/s) | 20260506 11:28:38.917 | INFO | Analyzing chunk frames = [998400,998912) | 20260506 11:28:38.917 | INFO | MemInfo: { | 20260506 11:28:38.917 | INFO | "gpu" : " 17 GB", | 20260506 11:28:38.917 | INFO | "huge" : "", | 20260506 11:28:38.917 | INFO | "huge_shared" : " 27 GB", | 20260506 11:28:38.917 | INFO | "malloc" : " 14 GB", | 20260506 11:28:38.917 | INFO | "pinned" : " 29 GB" | 20260506 11:28:38.917 | INFO | } | 20260506 11:28:38.917 | INFO | Perf report for the last 5.13968 seconds: | 20260506 11:28:38.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:28:38.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:38.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:38.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:38.917 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 11:28:38.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:38.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:38.917 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 11:28:38.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:38.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:38.917 | INFO | Uploaded 36GiB to GPU in 2.03966s (17.65GiB/s) | 20260506 11:28:38.918 | INFO | Downloaded 34GiB to GPU in 2.00969s (16.9181GiB/s) | 20260506 11:28:44.056 | INFO | Analyzing chunk frames = [998912,999424) | 20260506 11:28:44.056 | INFO | MemInfo: { | 20260506 11:28:44.056 | INFO | "gpu" : " 17 GB", | 20260506 11:28:44.056 | INFO | "huge" : "", | 20260506 11:28:44.056 | INFO | "huge_shared" : " 27 GB", | 20260506 11:28:44.056 | INFO | "malloc" : " 14 GB", | 20260506 11:28:44.056 | INFO | "pinned" : " 29 GB" | 20260506 11:28:44.056 | INFO | } | 20260506 11:28:44.056 | INFO | Perf report for the last 5.13918 seconds: | 20260506 11:28:44.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:28:44.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:44.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:44.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:44.056 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260506 11:28:44.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:44.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:44.057 | INFO | PRE_HQ : 34.2%, 1.0, 17.8ms, 0 | 20260506 11:28:44.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:44.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:44.057 | INFO | Uploaded 36GiB to GPU in 2.02488s (17.7788GiB/s) | 20260506 11:28:44.057 | INFO | Downloaded 34GiB to GPU in 1.96179s (17.3311GiB/s) | 20260506 11:28:44.057 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:28:44.057 | INFO | Total mem usage: 16.8 GiB | 20260506 11:28:44.057 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:28:44.057 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -6.26%) | 20260506 11:28:44.057 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:28:44.057 | INFO | High Water Breakdown: | 20260506 11:28:44.057 | INFO | 60.9% : 9.63 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:28:44.057 | INFO | 39% : 6.17 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:28:44.057 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:28:44.057 | INFO | ------------------------------------------ | 20260506 11:28:44.057 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:28:44.057 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:28:44.057 | INFO | Total mem usage: 26.5 GiB | 20260506 11:28:44.057 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:28:44.057 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:28:44.057 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:28:44.057 | INFO | High Water Breakdown: | 20260506 11:28:44.057 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:28:44.057 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:28:44.057 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:28:44.057 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:28:44.057 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:28:44.057 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:28:44.057 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:28:44.057 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:28:44.057 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:28:44.057 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:28:44.057 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:28:44.057 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:28:44.057 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:28:44.057 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:28:44.057 | INFO | ------------------------------------------ | 20260506 11:28:44.057 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:28:44.057 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:28:44.057 | INFO | Total mem usage: 25.6 GiB | 20260506 11:28:44.057 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:28:44.057 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:28:44.057 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:28:44.057 | INFO | High Water Breakdown: | 20260506 11:28:44.057 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:28:44.057 | INFO | ------------------------------------------ | 20260506 11:28:44.057 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:28:44.057 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:28:44.057 | INFO | Total mem usage: 15.7 GiB | 20260506 11:28:44.057 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:28:44.057 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:28:44.057 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 11:28:44.057 | INFO | High Water Breakdown: | 20260506 11:28:44.057 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:28:44.057 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:28:44.057 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:28:44.057 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:28:44.057 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:28:44.057 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:28:44.057 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:28:44.057 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:28:44.057 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:28:44.057 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:28:44.057 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:28:44.057 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:28:44.057 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:28:44.057 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:28:44.057 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:28:44.057 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:28:44.057 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:28:44.057 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:28:44.057 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:28:44.057 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:28:44.057 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:28:44.057 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:28:44.057 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:28:44.057 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:28:44.058 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:28:44.058 | INFO | ------------------------------------------ | 20260506 11:28:44.058 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:28:44.058 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:28:44.058 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:28:44.058 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:28:44.058 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.60e+01, 1.37e+01 / 2.09e+00 | 20260506 11:28:44.058 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 9.97e-02, 1.00e-02 / 6.62e-03 | 20260506 11:28:44.058 | INFO | PerfReport: 11.56%, Baseline -- 3.69e+00 / 4.44e+00, 5.99e-01 / 3.69e-01 | 20260506 11:28:44.058 | INFO | PerfReport: 3.23%, Histogram -- 1.03e+00 / 1.44e+00, 1.90e-01 / 1.61e-01 | 20260506 11:28:44.058 | INFO | PerfReport: 15.88%, DME -- 4.06e+01 / 4.53e+01, 4.19e+00 / 1.91e+00 | 20260506 11:28:44.058 | INFO | PerfReport: 56.12%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.40e+00 / 1.31e+00 | 20260506 11:28:44.058 | INFO | PerfReport: 7.70%, PulseAccumulating -- 2.46e+00 / 3.11e+00, 4.35e-01 / 1.84e-01 | 20260506 11:28:44.058 | INFO | PerfReport: 3.86%, Metrics -- 1.23e+00 / 1.63e+00, 3.11e-01 / 1.32e-01 | 20260506 11:28:44.058 | INFO | PerfReport: 1.37%, PulseSerializing -- 4.37e-01 / 5.00e-01, 6.85e-02 / 2.72e-02 | 20260506 11:28:44.058 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:28:44.058 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:28:44.058 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:28:44.058 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260506 11:28:44.058 | INFO | PerfReport: 21.71%, Upload -- 2.15e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 11:28:44.058 | INFO | PerfReport: 21.12%, Download -- 2.09e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 11:28:44.058 | INFO | PerfReport: 32.27%, Compute -- 3.20e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 11:28:44.058 | INFO | PerfReport: 2.03%, LockUpload -- 2.01e+00 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260506 11:28:44.058 | INFO | PerfReport: 0.86%, LockDownload -- 8.56e-01 / 0.00e+00, 2.30e+00 / 0.00e+00 | 20260506 11:28:44.058 | INFO | PerfReport: 21.99%, LockCompute -- 2.18e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 11:28:44.058 | INFO | PerfReport: 0.02%, Misc -- 1.53e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260506 11:28:44.187 | INFO | Starting a round of Baz IO | 20260506 11:28:47.399 | INFO | Populating RTMetrics report | 20260506 11:28:47.399 | INFO | RTMetrics report populated | 20260506 11:28:47.399 | INFO | Opening tmp RTMetrics json file | 20260506 11:28:47.420 | INFO | Serializing RTMetrics to json | 20260506 11:28:47.424 | INFO | Serialization done, swapping tmp file now... | 20260506 11:28:47.426 | INFO | RTMetrics done | 20260506 11:28:47.857 | INFO | Finished a round of Baz IO: Wrote 2.057 GiB in 3.67 seconds | 20260506 11:28:47.857 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.07% / 0.04% / 55.90% | 20260506 11:28:47.857 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.13522% / 9.13938e-05% / 4.37101e-05% | 20260506 11:28:47.857 | INFO | Baz IO overall: | 20260506 11:28:47.857 | INFO | BytesWritten : 479444672667 | 20260506 11:28:47.857 | INFO | NumEvents : 231333848143 | 20260506 11:28:49.196 | INFO | Analyzing chunk frames = [999424,999936) | 20260506 11:28:49.196 | INFO | MemInfo: { | 20260506 11:28:49.196 | INFO | "gpu" : " 17 GB", | 20260506 11:28:49.196 | INFO | "huge" : "", | 20260506 11:28:49.196 | INFO | "huge_shared" : " 27 GB", | 20260506 11:28:49.196 | INFO | "malloc" : " 14 GB", | 20260506 11:28:49.196 | INFO | "pinned" : " 29 GB" | 20260506 11:28:49.196 | INFO | } | 20260506 11:28:49.196 | INFO | Perf report for the last 5.13921 seconds: | 20260506 11:28:49.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:28:49.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:49.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:49.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:49.196 | INFO | ANALYSIS : 64.2%, 2.8, 95.2ms, 0 | 20260506 11:28:49.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:49.196 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260506 11:28:49.196 | INFO | PRE_HQ : 62.6%, 2.0, 64.4ms, 0 | 20260506 11:28:49.196 | INFO | BAZWRITER : 70.6%, 4.8, 174.1ms, 0 | 20260506 11:28:49.196 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 11:28:49.196 | INFO | Uploaded 36GiB to GPU in 2.11614s (17.0121GiB/s) | 20260506 11:28:49.196 | INFO | Downloaded 38GiB to GPU in 2.21573s (17.1501GiB/s) | 20260506 11:28:54.335 | INFO | Analyzing chunk frames = [999936,1000448) | 20260506 11:28:54.335 | INFO | MemInfo: { | 20260506 11:28:54.335 | INFO | "gpu" : " 17 GB", | 20260506 11:28:54.335 | INFO | "huge" : "", | 20260506 11:28:54.335 | INFO | "huge_shared" : " 27 GB", | 20260506 11:28:54.335 | INFO | "malloc" : " 14 GB", | 20260506 11:28:54.335 | INFO | "pinned" : " 29 GB" | 20260506 11:28:54.335 | INFO | } | 20260506 11:28:54.335 | INFO | Perf report for the last 5.13931 seconds: | 20260506 11:28:54.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:28:54.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:54.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:54.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:54.335 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:28:54.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:54.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:54.335 | INFO | PRE_HQ : 32.3%, 1.0, 17.3ms, 0 | 20260506 11:28:54.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:54.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:54.335 | INFO | Uploaded 36GiB to GPU in 2.02256s (17.7992GiB/s) | 20260506 11:28:54.335 | INFO | Downloaded 34GiB to GPU in 1.9249s (17.6633GiB/s) | 20260506 11:28:59.474 | INFO | Analyzing chunk frames = [1000448,1000960) | 20260506 11:28:59.474 | INFO | MemInfo: { | 20260506 11:28:59.474 | INFO | "gpu" : " 17 GB", | 20260506 11:28:59.474 | INFO | "huge" : "", | 20260506 11:28:59.474 | INFO | "huge_shared" : " 27 GB", | 20260506 11:28:59.474 | INFO | "malloc" : " 14 GB", | 20260506 11:28:59.474 | INFO | "pinned" : " 29 GB" | 20260506 11:28:59.474 | INFO | } | 20260506 11:28:59.474 | INFO | Perf report for the last 5.13942 seconds: | 20260506 11:28:59.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:28:59.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:59.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:59.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:59.475 | INFO | ANALYSIS : 62.5%, 3.0, 96.8ms, 0 | 20260506 11:28:59.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:59.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:59.475 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:28:59.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:59.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:28:59.475 | INFO | Uploaded 36GiB to GPU in 2.07851s (17.3201GiB/s) | 20260506 11:28:59.475 | INFO | Downloaded 34GiB to GPU in 1.9521s (17.4171GiB/s) | 20260506 11:29:04.614 | INFO | Analyzing chunk frames = [1000960,1001472) | 20260506 11:29:04.614 | INFO | MemInfo: { | 20260506 11:29:04.614 | INFO | "gpu" : " 17 GB", | 20260506 11:29:04.614 | INFO | "huge" : "", | 20260506 11:29:04.614 | INFO | "huge_shared" : " 27 GB", | 20260506 11:29:04.614 | INFO | "malloc" : " 14 GB", | 20260506 11:29:04.614 | INFO | "pinned" : " 29 GB" | 20260506 11:29:04.614 | INFO | } | 20260506 11:29:04.614 | INFO | Perf report for the last 5.13933 seconds: | 20260506 11:29:04.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:29:04.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:04.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:04.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:04.614 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 11:29:04.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:04.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:04.614 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:29:04.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:04.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:04.614 | INFO | Uploaded 36GiB to GPU in 2.07912s (17.315GiB/s) | 20260506 11:29:04.614 | INFO | Downloaded 34GiB to GPU in 1.96155s (17.3332GiB/s) | 20260506 11:29:09.753 | INFO | Analyzing chunk frames = [1001472,1001984) | 20260506 11:29:09.753 | INFO | MemInfo: { | 20260506 11:29:09.753 | INFO | "gpu" : " 17 GB", | 20260506 11:29:09.753 | INFO | "huge" : "", | 20260506 11:29:09.753 | INFO | "huge_shared" : " 27 GB", | 20260506 11:29:09.753 | INFO | "malloc" : " 14 GB", | 20260506 11:29:09.753 | INFO | "pinned" : " 29 GB" | 20260506 11:29:09.753 | INFO | } | 20260506 11:29:09.753 | INFO | Perf report for the last 5.13908 seconds: | 20260506 11:29:09.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:29:09.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:09.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:09.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:09.753 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 11:29:09.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:09.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:09.753 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:29:09.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:09.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:09.753 | INFO | Uploaded 36GiB to GPU in 2.11194s (17.0459GiB/s) | 20260506 11:29:09.753 | INFO | Downloaded 34GiB to GPU in 1.99017s (17.0839GiB/s) | 20260506 11:29:14.892 | INFO | Analyzing chunk frames = [1001984,1002496) | 20260506 11:29:14.892 | INFO | MemInfo: { | 20260506 11:29:14.892 | INFO | "gpu" : " 17 GB", | 20260506 11:29:14.892 | INFO | "huge" : "", | 20260506 11:29:14.892 | INFO | "huge_shared" : " 27 GB", | 20260506 11:29:14.892 | INFO | "malloc" : " 14 GB", | 20260506 11:29:14.892 | INFO | "pinned" : " 29 GB" | 20260506 11:29:14.892 | INFO | } | 20260506 11:29:14.892 | INFO | Perf report for the last 5.13945 seconds: | 20260506 11:29:14.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:29:14.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:14.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:14.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:14.892 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 11:29:14.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:14.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:14.892 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:29:14.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:14.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:14.893 | INFO | Uploaded 36GiB to GPU in 2.00299s (17.9731GiB/s) | 20260506 11:29:14.893 | INFO | Downloaded 34GiB to GPU in 1.9885s (17.0983GiB/s) | 20260506 11:29:20.032 | INFO | Analyzing chunk frames = [1002496,1003008) | 20260506 11:29:20.032 | INFO | MemInfo: { | 20260506 11:29:20.032 | INFO | "gpu" : " 17 GB", | 20260506 11:29:20.032 | INFO | "huge" : "", | 20260506 11:29:20.033 | INFO | "huge_shared" : " 27 GB", | 20260506 11:29:20.033 | INFO | "malloc" : " 14 GB", | 20260506 11:29:20.033 | INFO | "pinned" : " 29 GB" | 20260506 11:29:20.033 | INFO | } | 20260506 11:29:20.033 | INFO | Perf report for the last 5.14028 seconds: | 20260506 11:29:20.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:29:20.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:20.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:20.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:20.033 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:29:20.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:20.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:20.033 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 11:29:20.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:20.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:20.033 | INFO | Uploaded 36GiB to GPU in 2.03368s (17.7019GiB/s) | 20260506 11:29:20.033 | INFO | Downloaded 34GiB to GPU in 2.01148s (16.9029GiB/s) | 20260506 11:29:25.171 | INFO | Analyzing chunk frames = [1003008,1003520) | 20260506 11:29:25.171 | INFO | MemInfo: { | 20260506 11:29:25.171 | INFO | "gpu" : " 17 GB", | 20260506 11:29:25.171 | INFO | "huge" : "", | 20260506 11:29:25.171 | INFO | "huge_shared" : " 27 GB", | 20260506 11:29:25.171 | INFO | "malloc" : " 14 GB", | 20260506 11:29:25.171 | INFO | "pinned" : " 29 GB" | 20260506 11:29:25.171 | INFO | } | 20260506 11:29:25.171 | INFO | Perf report for the last 5.13841 seconds: | 20260506 11:29:25.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:29:25.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:25.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:25.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:25.171 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260506 11:29:25.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:25.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:25.171 | INFO | PRE_HQ : 33.3%, 1.0, 17.3ms, 0 | 20260506 11:29:25.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:25.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:25.171 | INFO | Uploaded 36GiB to GPU in 1.98947s (18.0953GiB/s) | 20260506 11:29:25.171 | INFO | Downloaded 34GiB to GPU in 1.98483s (17.1299GiB/s) | 20260506 11:29:25.306 | INFO | Starting a round of Baz IO | 20260506 11:29:28.495 | INFO | Populating RTMetrics report | 20260506 11:29:28.495 | INFO | RTMetrics report populated | 20260506 11:29:28.495 | INFO | Opening tmp RTMetrics json file | 20260506 11:29:28.499 | INFO | Serializing RTMetrics to json | 20260506 11:29:28.503 | INFO | Serialization done, swapping tmp file now... | 20260506 11:29:28.510 | INFO | RTMetrics done | 20260506 11:29:28.641 | INFO | Finished a round of Baz IO: Wrote 2.058 GiB in 3.334 seconds | 20260506 11:29:28.641 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.03% / 0.03% / 55.93% | 20260506 11:29:28.641 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.12453% / 4.76837e-05% / 4.76837e-05% | 20260506 11:29:28.641 | INFO | Baz IO overall: | 20260506 11:29:28.641 | INFO | BytesWritten : 481654756366 | 20260506 11:29:28.641 | INFO | NumEvents : 232406979455 | 20260506 11:29:30.310 | INFO | Analyzing chunk frames = [1003520,1004032) | 20260506 11:29:30.310 | INFO | MemInfo: { | 20260506 11:29:30.310 | INFO | "gpu" : " 17 GB", | 20260506 11:29:30.310 | INFO | "huge" : "", | 20260506 11:29:30.310 | INFO | "huge_shared" : " 27 GB", | 20260506 11:29:30.310 | INFO | "malloc" : " 14 GB", | 20260506 11:29:30.310 | INFO | "pinned" : " 29 GB" | 20260506 11:29:30.310 | INFO | } | 20260506 11:29:30.310 | INFO | Perf report for the last 5.13944 seconds: | 20260506 11:29:30.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:29:30.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:30.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:30.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:30.311 | INFO | ANALYSIS : 64.1%, 2.9, 95.8ms, 0 | 20260506 11:29:30.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:30.311 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 11:29:30.311 | INFO | PRE_HQ : 62.3%, 2.0, 63.2ms, 0 | 20260506 11:29:30.311 | INFO | BAZWRITER : 64.0%, 4.6, 151.8ms, 0 | 20260506 11:29:30.311 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 11:29:30.311 | INFO | Uploaded 36GiB to GPU in 2.1117s (17.0479GiB/s) | 20260506 11:29:30.311 | INFO | Downloaded 38GiB to GPU in 2.23179s (17.0267GiB/s) | 20260506 11:29:35.449 | INFO | Analyzing chunk frames = [1004032,1004544) | 20260506 11:29:35.449 | INFO | MemInfo: { | 20260506 11:29:35.449 | INFO | "gpu" : " 17 GB", | 20260506 11:29:35.449 | INFO | "huge" : "", | 20260506 11:29:35.449 | INFO | "huge_shared" : " 27 GB", | 20260506 11:29:35.450 | INFO | "malloc" : " 14 GB", | 20260506 11:29:35.450 | INFO | "pinned" : " 29 GB" | 20260506 11:29:35.450 | INFO | } | 20260506 11:29:35.450 | INFO | Perf report for the last 5.13914 seconds: | 20260506 11:29:35.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:29:35.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:35.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:35.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:35.450 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:29:35.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:35.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:35.450 | INFO | PRE_HQ : 33.4%, 1.0, 17.3ms, 0 | 20260506 11:29:35.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:35.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:35.450 | INFO | Uploaded 36GiB to GPU in 2.02541s (17.7742GiB/s) | 20260506 11:29:35.450 | INFO | Downloaded 34GiB to GPU in 1.94074s (17.5191GiB/s) | 20260506 11:29:40.589 | INFO | Analyzing chunk frames = [1004544,1005056) | 20260506 11:29:40.589 | INFO | MemInfo: { | 20260506 11:29:40.589 | INFO | "gpu" : " 17 GB", | 20260506 11:29:40.589 | INFO | "huge" : "", | 20260506 11:29:40.589 | INFO | "huge_shared" : " 27 GB", | 20260506 11:29:40.589 | INFO | "malloc" : " 14 GB", | 20260506 11:29:40.589 | INFO | "pinned" : " 29 GB" | 20260506 11:29:40.589 | INFO | } | 20260506 11:29:40.589 | INFO | Perf report for the last 5.13921 seconds: | 20260506 11:29:40.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:29:40.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:40.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:40.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:40.589 | INFO | ANALYSIS : 62.3%, 3.0, 96.5ms, 0 | 20260506 11:29:40.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:40.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:40.589 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 11:29:40.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:40.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:40.589 | INFO | Uploaded 36GiB to GPU in 2.03169s (17.7192GiB/s) | 20260506 11:29:40.589 | INFO | Downloaded 34GiB to GPU in 1.95466s (17.3943GiB/s) | 20260506 11:29:45.728 | INFO | Analyzing chunk frames = [1005056,1005568) | 20260506 11:29:45.728 | INFO | MemInfo: { | 20260506 11:29:45.728 | INFO | "gpu" : " 17 GB", | 20260506 11:29:45.728 | INFO | "huge" : "", | 20260506 11:29:45.728 | INFO | "huge_shared" : " 27 GB", | 20260506 11:29:45.728 | INFO | "malloc" : " 14 GB", | 20260506 11:29:45.728 | INFO | "pinned" : " 29 GB" | 20260506 11:29:45.728 | INFO | } | 20260506 11:29:45.728 | INFO | Perf report for the last 5.13949 seconds: | 20260506 11:29:45.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:29:45.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:45.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:45.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:45.728 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 11:29:45.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:45.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:45.728 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:29:45.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:45.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:45.729 | INFO | Uploaded 36GiB to GPU in 2.0926s (17.2034GiB/s) | 20260506 11:29:45.729 | INFO | Downloaded 34GiB to GPU in 1.96735s (17.2821GiB/s) | 20260506 11:29:50.867 | INFO | Analyzing chunk frames = [1005568,1006080) | 20260506 11:29:50.867 | INFO | MemInfo: { | 20260506 11:29:50.867 | INFO | "gpu" : " 17 GB", | 20260506 11:29:50.867 | INFO | "huge" : "", | 20260506 11:29:50.867 | INFO | "huge_shared" : " 27 GB", | 20260506 11:29:50.867 | INFO | "malloc" : " 14 GB", | 20260506 11:29:50.867 | INFO | "pinned" : " 29 GB" | 20260506 11:29:50.868 | INFO | } | 20260506 11:29:50.868 | INFO | Perf report for the last 5.13919 seconds: | 20260506 11:29:50.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:29:50.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:50.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:50.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:50.868 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 11:29:50.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:50.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:50.868 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 11:29:50.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:50.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:50.868 | INFO | Uploaded 36GiB to GPU in 2.10264s (17.1213GiB/s) | 20260506 11:29:50.868 | INFO | Downloaded 34GiB to GPU in 1.98961s (17.0888GiB/s) | 20260506 11:29:56.007 | INFO | Analyzing chunk frames = [1006080,1006592) | 20260506 11:29:56.007 | INFO | MemInfo: { | 20260506 11:29:56.007 | INFO | "gpu" : " 17 GB", | 20260506 11:29:56.007 | INFO | "huge" : "", | 20260506 11:29:56.007 | INFO | "huge_shared" : " 27 GB", | 20260506 11:29:56.007 | INFO | "malloc" : " 14 GB", | 20260506 11:29:56.007 | INFO | "pinned" : " 29 GB" | 20260506 11:29:56.007 | INFO | } | 20260506 11:29:56.007 | INFO | Perf report for the last 5.13934 seconds: | 20260506 11:29:56.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:29:56.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:56.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:56.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:56.007 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 11:29:56.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:56.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:56.007 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:29:56.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:56.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:29:56.007 | INFO | Uploaded 36GiB to GPU in 2.02716s (17.7589GiB/s) | 20260506 11:29:56.007 | INFO | Downloaded 34GiB to GPU in 1.98625s (17.1177GiB/s) | 20260506 11:30:01.147 | INFO | Analyzing chunk frames = [1006592,1007104) | 20260506 11:30:01.147 | INFO | MemInfo: { | 20260506 11:30:01.147 | INFO | "gpu" : " 17 GB", | 20260506 11:30:01.148 | INFO | "huge" : "", | 20260506 11:30:01.148 | INFO | "huge_shared" : " 27 GB", | 20260506 11:30:01.148 | INFO | "malloc" : " 14 GB", | 20260506 11:30:01.148 | INFO | "pinned" : " 29 GB" | 20260506 11:30:01.148 | INFO | } | 20260506 11:30:01.148 | INFO | Perf report for the last 5.14072 seconds: | 20260506 11:30:01.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:30:01.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:01.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:01.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:01.148 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 11:30:01.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:01.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:01.148 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 11:30:01.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:01.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:01.148 | INFO | Uploaded 36GiB to GPU in 2.01907s (17.83GiB/s) | 20260506 11:30:01.148 | INFO | Downloaded 34GiB to GPU in 2.01014s (16.9143GiB/s) | 20260506 11:30:06.285 | INFO | Analyzing chunk frames = [1007104,1007616) | 20260506 11:30:06.285 | INFO | MemInfo: { | 20260506 11:30:06.285 | INFO | "gpu" : " 17 GB", | 20260506 11:30:06.286 | INFO | "huge" : "", | 20260506 11:30:06.286 | INFO | "huge_shared" : " 27 GB", | 20260506 11:30:06.286 | INFO | "malloc" : " 14 GB", | 20260506 11:30:06.286 | INFO | "pinned" : " 29 GB" | 20260506 11:30:06.286 | INFO | } | 20260506 11:30:06.286 | INFO | Perf report for the last 5.13807 seconds: | 20260506 11:30:06.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:30:06.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:06.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:06.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:06.286 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260506 11:30:06.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:06.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:06.286 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260506 11:30:06.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:06.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:06.286 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:30:06.286 | INFO | Total mem usage: 16.8 GiB | 20260506 11:30:06.286 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:30:06.286 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -7.16%) | 20260506 11:30:06.286 | INFO | Uploaded 36GiB to GPU in 1.99315s (18.0619GiB/s) | 20260506 11:30:06.286 | INFO | Downloaded 34GiB to GPU in 1.98196s (17.1547GiB/s) | 20260506 11:30:06.286 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:30:06.286 | INFO | High Water Breakdown: | 20260506 11:30:06.286 | INFO | 60.7% : 9.53 GiB ( 91.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:30:06.286 | INFO | 39.1% : 6.14 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:30:06.286 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:30:06.286 | INFO | ------------------------------------------ | 20260506 11:30:06.286 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:30:06.286 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:30:06.286 | INFO | Total mem usage: 26.5 GiB | 20260506 11:30:06.286 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:30:06.286 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 11:30:06.286 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 11:30:06.286 | INFO | High Water Breakdown: | 20260506 11:30:06.286 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:30:06.286 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:30:06.286 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:30:06.286 | INFO | 2% : 523 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:30:06.286 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:30:06.286 | INFO | 0.557% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:30:06.286 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:30:06.286 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:30:06.286 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:30:06.286 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:30:06.286 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:30:06.287 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:30:06.287 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:30:06.287 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:30:06.287 | INFO | ------------------------------------------ | 20260506 11:30:06.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:30:06.287 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:30:06.287 | INFO | Total mem usage: 25.6 GiB | 20260506 11:30:06.287 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:30:06.287 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:30:06.287 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:30:06.287 | INFO | High Water Breakdown: | 20260506 11:30:06.287 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:30:06.287 | INFO | ------------------------------------------ | 20260506 11:30:06.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:30:06.287 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:30:06.287 | INFO | Total mem usage: 15.7 GiB | 20260506 11:30:06.287 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:30:06.287 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:30:06.287 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:30:06.287 | INFO | High Water Breakdown: | 20260506 11:30:06.287 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:30:06.287 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:30:06.287 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:30:06.287 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:30:06.287 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:30:06.287 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:30:06.287 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:30:06.287 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:30:06.287 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:30:06.287 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:30:06.287 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:30:06.287 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:30:06.287 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:30:06.287 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:30:06.287 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:30:06.287 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:30:06.287 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:30:06.287 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:30:06.287 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:30:06.287 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:30:06.287 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:30:06.287 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:30:06.287 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:30:06.287 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:30:06.287 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:30:06.287 | INFO | ------------------------------------------ | 20260506 11:30:06.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:30:06.287 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:30:06.287 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:30:06.287 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:30:06.287 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.68e+01, 1.38e+01 / 2.32e+00 | 20260506 11:30:06.287 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.01e-02 / 6.00e-03 | 20260506 11:30:06.287 | INFO | PerfReport: 11.58%, Baseline -- 3.70e+00 / 4.47e+00, 6.02e-01 / 3.21e-01 | 20260506 11:30:06.287 | INFO | PerfReport: 3.24%, Histogram -- 1.04e+00 / 1.49e+00, 1.95e-01 / 1.36e-01 | 20260506 11:30:06.287 | INFO | PerfReport: 15.93%, DME -- 4.07e+01 / 4.60e+01, 4.37e+00 / 2.41e+00 | 20260506 11:30:06.287 | INFO | PerfReport: 56.09%, FrameLabeling -- 1.79e+01 / 1.95e+01, 1.41e+00 / 1.40e+00 | 20260506 11:30:06.287 | INFO | PerfReport: 7.66%, PulseAccumulating -- 2.45e+00 / 3.10e+00, 4.29e-01 / 2.13e-01 | 20260506 11:30:06.287 | INFO | PerfReport: 3.85%, Metrics -- 1.23e+00 / 1.65e+00, 3.16e-01 / 1.73e-01 | 20260506 11:30:06.287 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.36e-01 / 4.94e-01, 6.83e-02 / 2.36e-02 | 20260506 11:30:06.287 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:30:06.287 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:30:06.287 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:30:06.287 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260506 11:30:06.287 | INFO | PerfReport: 21.64%, Upload -- 2.14e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260506 11:30:06.287 | INFO | PerfReport: 21.15%, Download -- 2.09e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 11:30:06.287 | INFO | PerfReport: 32.37%, Compute -- 3.20e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260506 11:30:06.287 | INFO | PerfReport: 1.94%, LockUpload -- 1.92e+00 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 11:30:06.287 | INFO | PerfReport: 0.91%, LockDownload -- 8.98e-01 / 0.00e+00, 2.42e+00 / 0.00e+00 | 20260506 11:30:06.287 | INFO | PerfReport: 21.96%, LockCompute -- 2.17e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 11:30:06.287 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260506 11:30:06.408 | INFO | Starting a round of Baz IO | 20260506 11:30:09.601 | INFO | Populating RTMetrics report | 20260506 11:30:09.602 | INFO | RTMetrics report populated | 20260506 11:30:09.602 | INFO | Opening tmp RTMetrics json file | 20260506 11:30:09.608 | INFO | Serializing RTMetrics to json | 20260506 11:30:09.613 | INFO | Serialization done, swapping tmp file now... | 20260506 11:30:09.616 | INFO | RTMetrics done | 20260506 11:30:09.953 | INFO | Finished a round of Baz IO: Wrote 2.058 GiB in 3.545 seconds | 20260506 11:30:09.953 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 44.00% / 0.03% / 55.97% | 20260506 11:30:09.953 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.10366% / 6.35783e-05% / 3.17891e-05% | 20260506 11:30:09.954 | INFO | Baz IO overall: | 20260506 11:30:09.954 | INFO | BytesWritten : 483864502869 | 20260506 11:30:09.954 | INFO | NumEvents : 233479577781 | 20260506 11:30:11.425 | INFO | Analyzing chunk frames = [1007616,1008128) | 20260506 11:30:11.425 | INFO | MemInfo: { | 20260506 11:30:11.425 | INFO | "gpu" : " 17 GB", | 20260506 11:30:11.425 | INFO | "huge" : "", | 20260506 11:30:11.425 | INFO | "huge_shared" : " 27 GB", | 20260506 11:30:11.425 | INFO | "malloc" : " 14 GB", | 20260506 11:30:11.425 | INFO | "pinned" : " 29 GB" | 20260506 11:30:11.425 | INFO | } | 20260506 11:30:11.425 | INFO | Perf report for the last 5.13948 seconds: | 20260506 11:30:11.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:30:11.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:11.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:11.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:11.425 | INFO | ANALYSIS : 63.9%, 2.9, 96.3ms, 0 | 20260506 11:30:11.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:11.425 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 11:30:11.425 | INFO | PRE_HQ : 62.0%, 1.9, 61.2ms, 0 | 20260506 11:30:11.425 | INFO | BAZWRITER : 67.9%, 4.9, 169.3ms, 0 | 20260506 11:30:11.425 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 11:30:11.426 | INFO | Uploaded 36GiB to GPU in 2.0971s (17.1666GiB/s) | 20260506 11:30:11.426 | INFO | Downloaded 38GiB to GPU in 2.22524s (17.0768GiB/s) | 20260506 11:30:16.564 | INFO | Analyzing chunk frames = [1008128,1008640) | 20260506 11:30:16.564 | INFO | MemInfo: { | 20260506 11:30:16.564 | INFO | "gpu" : " 17 GB", | 20260506 11:30:16.564 | INFO | "huge" : "", | 20260506 11:30:16.564 | INFO | "huge_shared" : " 27 GB", | 20260506 11:30:16.564 | INFO | "malloc" : " 14 GB", | 20260506 11:30:16.564 | INFO | "pinned" : " 29 GB" | 20260506 11:30:16.564 | INFO | } | 20260506 11:30:16.564 | INFO | Perf report for the last 5.13896 seconds: | 20260506 11:30:16.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:30:16.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:16.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:16.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:16.564 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:30:16.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:16.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:16.564 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 11:30:16.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:16.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:16.565 | INFO | Uploaded 36GiB to GPU in 2.01585s (17.8585GiB/s) | 20260506 11:30:16.565 | INFO | Downloaded 34GiB to GPU in 1.92314s (17.6794GiB/s) | 20260506 11:30:21.704 | INFO | Analyzing chunk frames = [1008640,1009152) | 20260506 11:30:21.704 | INFO | MemInfo: { | 20260506 11:30:21.704 | INFO | "gpu" : " 17 GB", | 20260506 11:30:21.704 | INFO | "huge" : "", | 20260506 11:30:21.704 | INFO | "huge_shared" : " 27 GB", | 20260506 11:30:21.704 | INFO | "malloc" : " 14 GB", | 20260506 11:30:21.704 | INFO | "pinned" : " 29 GB" | 20260506 11:30:21.704 | INFO | } | 20260506 11:30:21.704 | INFO | Perf report for the last 5.13964 seconds: | 20260506 11:30:21.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:30:21.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:21.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:21.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:21.704 | INFO | ANALYSIS : 62.3%, 3.0, 96.5ms, 0 | 20260506 11:30:21.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:21.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:21.704 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:30:21.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:21.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:21.704 | INFO | Uploaded 36GiB to GPU in 2.02811s (17.7505GiB/s) | 20260506 11:30:21.704 | INFO | Downloaded 34GiB to GPU in 1.95288s (17.4102GiB/s) | 20260506 11:30:26.843 | INFO | Analyzing chunk frames = [1009152,1009664) | 20260506 11:30:26.843 | INFO | MemInfo: { | 20260506 11:30:26.843 | INFO | "gpu" : " 17 GB", | 20260506 11:30:26.843 | INFO | "huge" : "", | 20260506 11:30:26.843 | INFO | "huge_shared" : " 27 GB", | 20260506 11:30:26.843 | INFO | "malloc" : " 14 GB", | 20260506 11:30:26.843 | INFO | "pinned" : " 29 GB" | 20260506 11:30:26.843 | INFO | } | 20260506 11:30:26.843 | INFO | Perf report for the last 5.13926 seconds: | 20260506 11:30:26.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:30:26.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:26.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:26.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:26.843 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 11:30:26.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:26.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:26.843 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:30:26.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:26.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:26.843 | INFO | Uploaded 36GiB to GPU in 2.07337s (17.363GiB/s) | 20260506 11:30:26.843 | INFO | Downloaded 34GiB to GPU in 1.97804s (17.1887GiB/s) | 20260506 11:30:31.982 | INFO | Analyzing chunk frames = [1009664,1010176) | 20260506 11:30:31.982 | INFO | MemInfo: { | 20260506 11:30:31.982 | INFO | "gpu" : " 17 GB", | 20260506 11:30:31.982 | INFO | "huge" : "", | 20260506 11:30:31.982 | INFO | "huge_shared" : " 27 GB", | 20260506 11:30:31.982 | INFO | "malloc" : " 14 GB", | 20260506 11:30:31.982 | INFO | "pinned" : " 29 GB" | 20260506 11:30:31.982 | INFO | } | 20260506 11:30:31.982 | INFO | Perf report for the last 5.1393 seconds: | 20260506 11:30:31.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:30:31.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:31.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:31.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:31.982 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 11:30:31.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:31.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:31.982 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:30:31.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:31.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:31.983 | INFO | Uploaded 36GiB to GPU in 2.09217s (17.207GiB/s) | 20260506 11:30:31.983 | INFO | Downloaded 34GiB to GPU in 1.99469s (17.0453GiB/s) | 20260506 11:30:37.122 | INFO | Analyzing chunk frames = [1010176,1010688) | 20260506 11:30:37.122 | INFO | MemInfo: { | 20260506 11:30:37.122 | INFO | "gpu" : " 17 GB", | 20260506 11:30:37.122 | INFO | "huge" : "", | 20260506 11:30:37.122 | INFO | "huge_shared" : " 27 GB", | 20260506 11:30:37.122 | INFO | "malloc" : " 14 GB", | 20260506 11:30:37.122 | INFO | "pinned" : " 29 GB" | 20260506 11:30:37.122 | INFO | } | 20260506 11:30:37.122 | INFO | Perf report for the last 5.13957 seconds: | 20260506 11:30:37.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:30:37.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:37.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:37.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:37.122 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 11:30:37.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:37.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:37.122 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:30:37.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:37.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:37.122 | INFO | Uploaded 36GiB to GPU in 2.0571s (17.5003GiB/s) | 20260506 11:30:37.122 | INFO | Downloaded 34GiB to GPU in 2.00454s (16.9615GiB/s) | 20260506 11:30:42.262 | INFO | Analyzing chunk frames = [1010688,1011200) | 20260506 11:30:42.262 | INFO | MemInfo: { | 20260506 11:30:42.262 | INFO | "gpu" : " 17 GB", | 20260506 11:30:42.262 | INFO | "huge" : "", | 20260506 11:30:42.262 | INFO | "huge_shared" : " 27 GB", | 20260506 11:30:42.262 | INFO | "malloc" : " 14 GB", | 20260506 11:30:42.262 | INFO | "pinned" : " 29 GB" | 20260506 11:30:42.262 | INFO | } | 20260506 11:30:42.262 | INFO | Perf report for the last 5.13985 seconds: | 20260506 11:30:42.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:30:42.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:42.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:42.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:42.262 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:30:42.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:42.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:42.262 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 11:30:42.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:42.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:42.262 | INFO | Uploaded 36GiB to GPU in 1.99178s (18.0743GiB/s) | 20260506 11:30:42.262 | INFO | Downloaded 34GiB to GPU in 2.01147s (16.9031GiB/s) | 20260506 11:30:47.400 | INFO | Analyzing chunk frames = [1011200,1011712) | 20260506 11:30:47.400 | INFO | MemInfo: { | 20260506 11:30:47.400 | INFO | "gpu" : " 17 GB", | 20260506 11:30:47.400 | INFO | "huge" : "", | 20260506 11:30:47.400 | INFO | "huge_shared" : " 27 GB", | 20260506 11:30:47.400 | INFO | "malloc" : " 14 GB", | 20260506 11:30:47.400 | INFO | "pinned" : " 29 GB" | 20260506 11:30:47.400 | INFO | } | 20260506 11:30:47.400 | INFO | Perf report for the last 5.13838 seconds: | 20260506 11:30:47.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:30:47.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:47.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:47.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:47.400 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260506 11:30:47.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:47.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:47.400 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260506 11:30:47.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:47.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:47.401 | INFO | Uploaded 36GiB to GPU in 2.03582s (17.6833GiB/s) | 20260506 11:30:47.401 | INFO | Downloaded 34GiB to GPU in 1.98663s (17.1144GiB/s) | 20260506 11:30:47.525 | INFO | Starting a round of Baz IO | 20260506 11:30:50.735 | INFO | Populating RTMetrics report | 20260506 11:30:50.735 | INFO | RTMetrics report populated | 20260506 11:30:50.735 | INFO | Opening tmp RTMetrics json file | 20260506 11:30:50.748 | INFO | Serializing RTMetrics to json | 20260506 11:30:50.751 | INFO | Serialization done, swapping tmp file now... | 20260506 11:30:50.754 | INFO | RTMetrics done | 20260506 11:30:51.108 | INFO | Finished a round of Baz IO: Wrote 2.062 GiB in 3.582 seconds | 20260506 11:30:51.108 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.96% / 0.03% / 56.00% | 20260506 11:30:51.108 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.10318% / 5.5631e-05% / 6.75519e-05% | 20260506 11:30:51.108 | INFO | Baz IO overall: | 20260506 11:30:51.108 | INFO | BytesWritten : 486078311877 | 20260506 11:30:51.108 | INFO | NumEvents : 234555139156 | 20260506 11:30:52.540 | INFO | Analyzing chunk frames = [1011712,1012224) | 20260506 11:30:52.540 | INFO | MemInfo: { | 20260506 11:30:52.540 | INFO | "gpu" : " 17 GB", | 20260506 11:30:52.540 | INFO | "huge" : "", | 20260506 11:30:52.540 | INFO | "huge_shared" : " 27 GB", | 20260506 11:30:52.540 | INFO | "malloc" : " 14 GB", | 20260506 11:30:52.540 | INFO | "pinned" : " 29 GB" | 20260506 11:30:52.540 | INFO | } | 20260506 11:30:52.540 | INFO | Perf report for the last 5.13959 seconds: | 20260506 11:30:52.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:30:52.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:52.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:52.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:52.540 | INFO | ANALYSIS : 64.0%, 2.9, 95.9ms, 0 | 20260506 11:30:52.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:52.540 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260506 11:30:52.540 | INFO | PRE_HQ : 61.9%, 1.9, 62.3ms, 0 | 20260506 11:30:52.540 | INFO | BAZWRITER : 68.9%, 4.8, 170.0ms, 0 | 20260506 11:30:52.540 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 11:30:52.540 | INFO | Uploaded 36GiB to GPU in 2.11342s (17.034GiB/s) | 20260506 11:30:52.540 | INFO | Downloaded 38GiB to GPU in 2.20181s (17.2585GiB/s) | 20260506 11:30:57.679 | INFO | Analyzing chunk frames = [1012224,1012736) | 20260506 11:30:57.679 | INFO | MemInfo: { | 20260506 11:30:57.679 | INFO | "gpu" : " 17 GB", | 20260506 11:30:57.679 | INFO | "huge" : "", | 20260506 11:30:57.679 | INFO | "huge_shared" : " 27 GB", | 20260506 11:30:57.679 | INFO | "malloc" : " 14 GB", | 20260506 11:30:57.679 | INFO | "pinned" : " 29 GB" | 20260506 11:30:57.679 | INFO | } | 20260506 11:30:57.679 | INFO | Perf report for the last 5.13914 seconds: | 20260506 11:30:57.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:30:57.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:57.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:57.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:57.679 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:30:57.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:57.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:57.679 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:30:57.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:57.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:30:57.679 | INFO | Uploaded 36GiB to GPU in 1.99501s (18.045GiB/s) | 20260506 11:30:57.679 | INFO | Downloaded 34GiB to GPU in 1.90203s (17.8757GiB/s) | 20260506 11:31:02.818 | INFO | Analyzing chunk frames = [1012736,1013248) | 20260506 11:31:02.818 | INFO | MemInfo: { | 20260506 11:31:02.818 | INFO | "gpu" : " 17 GB", | 20260506 11:31:02.818 | INFO | "huge" : "", | 20260506 11:31:02.818 | INFO | "huge_shared" : " 27 GB", | 20260506 11:31:02.818 | INFO | "malloc" : " 14 GB", | 20260506 11:31:02.818 | INFO | "pinned" : " 29 GB" | 20260506 11:31:02.818 | INFO | } | 20260506 11:31:02.818 | INFO | Perf report for the last 5.13942 seconds: | 20260506 11:31:02.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:31:02.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:02.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:02.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:02.818 | INFO | ANALYSIS : 62.0%, 3.0, 96.1ms, 0 | 20260506 11:31:02.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:02.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:02.819 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:31:02.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:02.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:02.819 | INFO | Uploaded 36GiB to GPU in 2.03223s (17.7145GiB/s) | 20260506 11:31:02.819 | INFO | Downloaded 34GiB to GPU in 1.94393s (17.4903GiB/s) | 20260506 11:31:07.957 | INFO | Analyzing chunk frames = [1013248,1013760) | 20260506 11:31:07.957 | INFO | MemInfo: { | 20260506 11:31:07.957 | INFO | "gpu" : " 17 GB", | 20260506 11:31:07.957 | INFO | "huge" : "", | 20260506 11:31:07.957 | INFO | "huge_shared" : " 27 GB", | 20260506 11:31:07.957 | INFO | "malloc" : " 14 GB", | 20260506 11:31:07.957 | INFO | "pinned" : " 29 GB" | 20260506 11:31:07.957 | INFO | } | 20260506 11:31:07.958 | INFO | Perf report for the last 5.13906 seconds: | 20260506 11:31:07.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:31:07.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:07.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:07.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:07.958 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260506 11:31:07.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:07.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:07.958 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:31:07.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:07.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:07.958 | INFO | Uploaded 36GiB to GPU in 2.07736s (17.3297GiB/s) | 20260506 11:31:07.958 | INFO | Downloaded 34GiB to GPU in 1.97368s (17.2267GiB/s) | 20260506 11:31:13.097 | INFO | Analyzing chunk frames = [1013760,1014272) | 20260506 11:31:13.097 | INFO | MemInfo: { | 20260506 11:31:13.097 | INFO | "gpu" : " 17 GB", | 20260506 11:31:13.097 | INFO | "huge" : "", | 20260506 11:31:13.097 | INFO | "huge_shared" : " 27 GB", | 20260506 11:31:13.097 | INFO | "malloc" : " 14 GB", | 20260506 11:31:13.097 | INFO | "pinned" : " 29 GB" | 20260506 11:31:13.097 | INFO | } | 20260506 11:31:13.097 | INFO | Perf report for the last 5.13944 seconds: | 20260506 11:31:13.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:31:13.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:13.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:13.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:13.097 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 11:31:13.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:13.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:13.097 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:31:13.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:13.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:13.098 | INFO | Uploaded 36GiB to GPU in 1.99253s (18.0675GiB/s) | 20260506 11:31:13.098 | INFO | Downloaded 34GiB to GPU in 1.9792s (17.1787GiB/s) | 20260506 11:31:18.236 | INFO | Analyzing chunk frames = [1014272,1014784) | 20260506 11:31:18.236 | INFO | MemInfo: { | 20260506 11:31:18.236 | INFO | "gpu" : " 17 GB", | 20260506 11:31:18.236 | INFO | "huge" : "", | 20260506 11:31:18.236 | INFO | "huge_shared" : " 27 GB", | 20260506 11:31:18.236 | INFO | "malloc" : " 14 GB", | 20260506 11:31:18.236 | INFO | "pinned" : " 29 GB" | 20260506 11:31:18.236 | INFO | } | 20260506 11:31:18.236 | INFO | Perf report for the last 5.13948 seconds: | 20260506 11:31:18.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:31:18.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:18.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:18.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:18.236 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 11:31:18.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:18.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:18.237 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:31:18.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:18.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:18.237 | INFO | Uploaded 36GiB to GPU in 2.05381s (17.5284GiB/s) | 20260506 11:31:18.237 | INFO | Downloaded 34GiB to GPU in 1.92377s (17.6737GiB/s) | 20260506 11:31:23.376 | INFO | Analyzing chunk frames = [1014784,1015296) | 20260506 11:31:23.376 | INFO | MemInfo: { | 20260506 11:31:23.376 | INFO | "gpu" : " 17 GB", | 20260506 11:31:23.376 | INFO | "huge" : "", | 20260506 11:31:23.376 | INFO | "huge_shared" : " 27 GB", | 20260506 11:31:23.376 | INFO | "malloc" : " 14 GB", | 20260506 11:31:23.376 | INFO | "pinned" : " 29 GB" | 20260506 11:31:23.376 | INFO | } | 20260506 11:31:23.377 | INFO | Perf report for the last 5.14003 seconds: | 20260506 11:31:23.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:31:23.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:23.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:23.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:23.377 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:31:23.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:23.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:23.377 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 11:31:23.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:23.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:23.377 | INFO | Uploaded 36GiB to GPU in 2.01109s (17.9008GiB/s) | 20260506 11:31:23.377 | INFO | Downloaded 34GiB to GPU in 2.00969s (16.918GiB/s) | 20260506 11:31:28.515 | INFO | Analyzing chunk frames = [1015296,1015808) | 20260506 11:31:28.515 | INFO | MemInfo: { | 20260506 11:31:28.515 | INFO | "gpu" : " 17 GB", | 20260506 11:31:28.515 | INFO | "huge" : "", | 20260506 11:31:28.515 | INFO | "huge_shared" : " 27 GB", | 20260506 11:31:28.515 | INFO | "malloc" : " 14 GB", | 20260506 11:31:28.515 | INFO | "pinned" : " 29 GB" | 20260506 11:31:28.515 | INFO | } | 20260506 11:31:28.515 | INFO | Perf report for the last 5.13853 seconds: | 20260506 11:31:28.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:31:28.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:28.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:28.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:28.515 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260506 11:31:28.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:28.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:28.515 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260506 11:31:28.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:28.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:28.515 | INFO | Uploaded 36GiB to GPU in 2.01613s (17.856GiB/s) | 20260506 11:31:28.515 | INFO | Downloaded 34GiB to GPU in 1.98483s (17.1299GiB/s) | 20260506 11:31:28.515 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:31:28.515 | INFO | Total mem usage: 16.8 GiB | 20260506 11:31:28.515 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:31:28.515 | INFO | Recent High Water Marks Sum: 16.3 GiB (Savings: -3.43%) | 20260506 11:31:28.515 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:31:28.515 | INFO | High Water Breakdown: | 20260506 11:31:28.515 | INFO | 61.3% : 9.97 GiB ( 96% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:31:28.515 | INFO | 38.6% : 6.27 GiB ( 93.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:31:28.515 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:31:28.515 | INFO | ------------------------------------------ | 20260506 11:31:28.516 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:31:28.516 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:31:28.516 | INFO | Total mem usage: 26.5 GiB | 20260506 11:31:28.516 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:31:28.516 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:31:28.516 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:31:28.516 | INFO | High Water Breakdown: | 20260506 11:31:28.516 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:31:28.516 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:31:28.516 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:31:28.516 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:31:28.516 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:31:28.516 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:31:28.516 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:31:28.516 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:31:28.516 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:31:28.516 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:31:28.516 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:31:28.516 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:31:28.516 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:31:28.516 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:31:28.516 | INFO | ------------------------------------------ | 20260506 11:31:28.516 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:31:28.516 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:31:28.516 | INFO | Total mem usage: 25.6 GiB | 20260506 11:31:28.516 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:31:28.516 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:31:28.516 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:31:28.516 | INFO | High Water Breakdown: | 20260506 11:31:28.516 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:31:28.516 | INFO | ------------------------------------------ | 20260506 11:31:28.516 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:31:28.516 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:31:28.516 | INFO | Total mem usage: 15.7 GiB | 20260506 11:31:28.516 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:31:28.516 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:31:28.516 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 11:31:28.516 | INFO | High Water Breakdown: | 20260506 11:31:28.516 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:31:28.516 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:31:28.516 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:31:28.516 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:31:28.516 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:31:28.516 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:31:28.516 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:31:28.516 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:31:28.516 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:31:28.516 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:31:28.516 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:31:28.516 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:31:28.516 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:31:28.516 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:31:28.516 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:31:28.516 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:31:28.516 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:31:28.516 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:31:28.516 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:31:28.516 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:31:28.516 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:31:28.516 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:31:28.516 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:31:28.516 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:31:28.516 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:31:28.516 | INFO | ------------------------------------------ | 20260506 11:31:28.516 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:31:28.516 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:31:28.516 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:31:28.516 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:31:28.516 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.62e+01, 1.38e+01 / 2.51e+00 | 20260506 11:31:28.516 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.02e-02 / 8.14e-03 | 20260506 11:31:28.516 | INFO | PerfReport: 11.54%, Baseline -- 3.69e+00 / 4.45e+00, 5.99e-01 / 3.68e-01 | 20260506 11:31:28.516 | INFO | PerfReport: 3.24%, Histogram -- 1.04e+00 / 1.46e+00, 1.96e-01 / 1.57e-01 | 20260506 11:31:28.516 | INFO | PerfReport: 15.82%, DME -- 4.05e+01 / 4.50e+01, 4.10e+00 / 2.73e+00 | 20260506 11:31:28.516 | INFO | PerfReport: 56.29%, FrameLabeling -- 1.80e+01 / 2.00e+01, 1.56e+00 / 1.66e+00 | 20260506 11:31:28.516 | INFO | PerfReport: 7.62%, PulseAccumulating -- 2.44e+00 / 3.08e+00, 4.24e-01 / 2.31e-01 | 20260506 11:31:28.516 | INFO | PerfReport: 3.84%, Metrics -- 1.23e+00 / 1.63e+00, 3.15e-01 / 2.10e-01 | 20260506 11:31:28.516 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.35e-01 / 4.93e-01, 6.83e-02 / 3.49e-02 | 20260506 11:31:28.516 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:31:28.516 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:31:28.516 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:31:28.516 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260506 11:31:28.516 | INFO | PerfReport: 21.52%, Upload -- 2.13e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 11:31:28.516 | INFO | PerfReport: 21.07%, Download -- 2.08e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 11:31:28.516 | INFO | PerfReport: 32.37%, Compute -- 3.20e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260506 11:31:28.516 | INFO | PerfReport: 1.89%, LockUpload -- 1.87e+00 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 11:31:28.517 | INFO | PerfReport: 0.96%, LockDownload -- 9.47e-01 / 0.00e+00, 2.53e+00 / 0.00e+00 | 20260506 11:31:28.517 | INFO | PerfReport: 22.17%, LockCompute -- 2.19e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260506 11:31:28.517 | INFO | PerfReport: 0.02%, Misc -- 1.59e-02 / 0.00e+00, 2.36e-02 / 0.00e+00 | 20260506 11:31:28.638 | INFO | Starting a round of Baz IO | 20260506 11:31:31.821 | INFO | Populating RTMetrics report | 20260506 11:31:31.821 | INFO | RTMetrics report populated | 20260506 11:31:31.821 | INFO | Opening tmp RTMetrics json file | 20260506 11:31:31.824 | INFO | Serializing RTMetrics to json | 20260506 11:31:31.843 | INFO | Serialization done, swapping tmp file now... | 20260506 11:31:31.852 | INFO | RTMetrics done | 20260506 11:31:32.026 | INFO | Finished a round of Baz IO: Wrote 2.057 GiB in 3.387 seconds | 20260506 11:31:32.026 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.93% / 0.03% / 56.04% | 20260506 11:31:32.026 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.0582% / 4.76837e-05% / 5.16574e-05% | 20260506 11:31:32.026 | INFO | Baz IO overall: | 20260506 11:31:32.026 | INFO | BytesWritten : 488287499439 | 20260506 11:31:32.026 | INFO | NumEvents : 235626915413 | 20260506 11:31:33.654 | INFO | Analyzing chunk frames = [1015808,1016320) | 20260506 11:31:33.654 | INFO | MemInfo: { | 20260506 11:31:33.654 | INFO | "gpu" : " 17 GB", | 20260506 11:31:33.654 | INFO | "huge" : "", | 20260506 11:31:33.654 | INFO | "huge_shared" : " 27 GB", | 20260506 11:31:33.654 | INFO | "malloc" : " 14 GB", | 20260506 11:31:33.654 | INFO | "pinned" : " 29 GB" | 20260506 11:31:33.654 | INFO | } | 20260506 11:31:33.655 | INFO | Perf report for the last 5.13947 seconds: | 20260506 11:31:33.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:31:33.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:33.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:33.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:33.655 | INFO | ANALYSIS : 63.7%, 2.9, 96.1ms, 0 | 20260506 11:31:33.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:33.655 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 11:31:33.655 | INFO | PRE_HQ : 62.0%, 1.9, 61.8ms, 0 | 20260506 11:31:33.655 | INFO | BAZWRITER : 65.1%, 4.7, 158.1ms, 0 | 20260506 11:31:33.655 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 11:31:33.655 | INFO | Uploaded 36GiB to GPU in 2.12619s (16.9317GiB/s) | 20260506 11:31:33.655 | INFO | Downloaded 38GiB to GPU in 2.22046s (17.1135GiB/s) | 20260506 11:31:38.794 | INFO | Analyzing chunk frames = [1016320,1016832) | 20260506 11:31:38.794 | INFO | MemInfo: { | 20260506 11:31:38.794 | INFO | "gpu" : " 17 GB", | 20260506 11:31:38.794 | INFO | "huge" : "", | 20260506 11:31:38.794 | INFO | "huge_shared" : " 27 GB", | 20260506 11:31:38.794 | INFO | "malloc" : " 14 GB", | 20260506 11:31:38.794 | INFO | "pinned" : " 29 GB" | 20260506 11:31:38.794 | INFO | } | 20260506 11:31:38.794 | INFO | Perf report for the last 5.13946 seconds: | 20260506 11:31:38.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:31:38.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:38.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:38.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:38.794 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:31:38.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:38.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:38.794 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260506 11:31:38.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:38.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:38.794 | INFO | Uploaded 36GiB to GPU in 2.02758s (17.7551GiB/s) | 20260506 11:31:38.794 | INFO | Downloaded 34GiB to GPU in 1.94484s (17.4821GiB/s) | 20260506 11:31:43.933 | INFO | Analyzing chunk frames = [1016832,1017344) | 20260506 11:31:43.933 | INFO | MemInfo: { | 20260506 11:31:43.933 | INFO | "gpu" : " 17 GB", | 20260506 11:31:43.933 | INFO | "huge" : "", | 20260506 11:31:43.933 | INFO | "huge_shared" : " 27 GB", | 20260506 11:31:43.933 | INFO | "malloc" : " 14 GB", | 20260506 11:31:43.933 | INFO | "pinned" : " 29 GB" | 20260506 11:31:43.933 | INFO | } | 20260506 11:31:43.933 | INFO | Perf report for the last 5.13924 seconds: | 20260506 11:31:43.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:31:43.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:43.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:43.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:43.933 | INFO | ANALYSIS : 62.2%, 3.0, 96.5ms, 0 | 20260506 11:31:43.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:43.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:43.933 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:31:43.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:43.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:43.933 | INFO | Uploaded 36GiB to GPU in 2.01397s (17.8751GiB/s) | 20260506 11:31:43.933 | INFO | Downloaded 34GiB to GPU in 1.95728s (17.3711GiB/s) | 20260506 11:31:49.072 | INFO | Analyzing chunk frames = [1017344,1017856) | 20260506 11:31:49.072 | INFO | MemInfo: { | 20260506 11:31:49.072 | INFO | "gpu" : " 17 GB", | 20260506 11:31:49.072 | INFO | "huge" : "", | 20260506 11:31:49.072 | INFO | "huge_shared" : " 27 GB", | 20260506 11:31:49.072 | INFO | "malloc" : " 14 GB", | 20260506 11:31:49.072 | INFO | "pinned" : " 29 GB" | 20260506 11:31:49.072 | INFO | } | 20260506 11:31:49.072 | INFO | Perf report for the last 5.13889 seconds: | 20260506 11:31:49.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:31:49.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:49.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:49.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:49.072 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 11:31:49.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:49.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:49.072 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:31:49.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:49.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:49.072 | INFO | Uploaded 36GiB to GPU in 2.06637s (17.4218GiB/s) | 20260506 11:31:49.073 | INFO | Downloaded 34GiB to GPU in 1.97266s (17.2356GiB/s) | 20260506 11:31:54.212 | INFO | Analyzing chunk frames = [1017856,1018368) | 20260506 11:31:54.212 | INFO | MemInfo: { | 20260506 11:31:54.212 | INFO | "gpu" : " 17 GB", | 20260506 11:31:54.212 | INFO | "huge" : "", | 20260506 11:31:54.212 | INFO | "huge_shared" : " 27 GB", | 20260506 11:31:54.212 | INFO | "malloc" : " 14 GB", | 20260506 11:31:54.212 | INFO | "pinned" : " 29 GB" | 20260506 11:31:54.212 | INFO | } | 20260506 11:31:54.212 | INFO | Perf report for the last 5.13947 seconds: | 20260506 11:31:54.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:31:54.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:54.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:54.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:54.212 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 11:31:54.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:54.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:54.212 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:31:54.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:54.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:54.212 | INFO | Uploaded 36GiB to GPU in 2.03179s (17.7184GiB/s) | 20260506 11:31:54.212 | INFO | Downloaded 34GiB to GPU in 1.99008s (17.0847GiB/s) | 20260506 11:31:59.351 | INFO | Analyzing chunk frames = [1018368,1018880) | 20260506 11:31:59.351 | INFO | MemInfo: { | 20260506 11:31:59.351 | INFO | "gpu" : " 17 GB", | 20260506 11:31:59.351 | INFO | "huge" : "", | 20260506 11:31:59.351 | INFO | "huge_shared" : " 27 GB", | 20260506 11:31:59.351 | INFO | "malloc" : " 14 GB", | 20260506 11:31:59.351 | INFO | "pinned" : " 29 GB" | 20260506 11:31:59.351 | INFO | } | 20260506 11:31:59.352 | INFO | Perf report for the last 5.13989 seconds: | 20260506 11:31:59.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:31:59.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:59.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:59.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:59.352 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 11:31:59.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:59.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:59.352 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:31:59.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:59.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:31:59.352 | INFO | Uploaded 36GiB to GPU in 2.05907s (17.4836GiB/s) | 20260506 11:31:59.352 | INFO | Downloaded 34GiB to GPU in 2.00751s (16.9364GiB/s) | 20260506 11:32:04.492 | INFO | Analyzing chunk frames = [1018880,1019392) | 20260506 11:32:04.492 | INFO | MemInfo: { | 20260506 11:32:04.492 | INFO | "gpu" : " 17 GB", | 20260506 11:32:04.492 | INFO | "huge" : "", | 20260506 11:32:04.492 | INFO | "huge_shared" : " 27 GB", | 20260506 11:32:04.492 | INFO | "malloc" : " 14 GB", | 20260506 11:32:04.492 | INFO | "pinned" : " 29 GB" | 20260506 11:32:04.492 | INFO | } | 20260506 11:32:04.492 | INFO | Perf report for the last 5.14052 seconds: | 20260506 11:32:04.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:32:04.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:04.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:04.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:04.492 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 11:32:04.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:04.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:04.492 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260506 11:32:04.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:04.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:04.492 | INFO | Uploaded 36GiB to GPU in 1.99648s (18.0317GiB/s) | 20260506 11:32:04.492 | INFO | Downloaded 34GiB to GPU in 2.01802s (16.8482GiB/s) | 20260506 11:32:09.630 | INFO | Analyzing chunk frames = [1019392,1019904) | 20260506 11:32:09.630 | INFO | MemInfo: { | 20260506 11:32:09.630 | INFO | "gpu" : " 17 GB", | 20260506 11:32:09.630 | INFO | "huge" : "", | 20260506 11:32:09.630 | INFO | "huge_shared" : " 27 GB", | 20260506 11:32:09.630 | INFO | "malloc" : " 14 GB", | 20260506 11:32:09.630 | INFO | "pinned" : " 29 GB" | 20260506 11:32:09.630 | INFO | } | 20260506 11:32:09.630 | INFO | Perf report for the last 5.13797 seconds: | 20260506 11:32:09.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:32:09.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:09.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:09.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:09.630 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260506 11:32:09.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:09.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:09.630 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260506 11:32:09.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:09.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:09.630 | INFO | Uploaded 36GiB to GPU in 2.01624s (17.855GiB/s) | 20260506 11:32:09.630 | INFO | Downloaded 34GiB to GPU in 1.98804s (17.1023GiB/s) | 20260506 11:32:09.753 | INFO | Starting a round of Baz IO | 20260506 11:32:12.952 | INFO | Populating RTMetrics report | 20260506 11:32:12.952 | INFO | RTMetrics report populated | 20260506 11:32:12.952 | INFO | Opening tmp RTMetrics json file | 20260506 11:32:12.957 | INFO | Serializing RTMetrics to json | 20260506 11:32:13.012 | INFO | Serialization done, swapping tmp file now... | 20260506 11:32:13.041 | INFO | RTMetrics done | 20260506 11:32:13.207 | INFO | Finished a round of Baz IO: Wrote 2.061 GiB in 3.453 seconds | 20260506 11:32:13.207 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.90% / 0.03% / 56.07% | 20260506 11:32:13.207 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.05921% / 6.75519e-05% / 5.16574e-05% | 20260506 11:32:13.207 | INFO | Baz IO overall: | 20260506 11:32:13.207 | INFO | BytesWritten : 490499946664 | 20260506 11:32:13.207 | INFO | NumEvents : 236700843367 | 20260506 11:32:14.769 | INFO | Analyzing chunk frames = [1019904,1020416) | 20260506 11:32:14.769 | INFO | MemInfo: { | 20260506 11:32:14.769 | INFO | "gpu" : " 17 GB", | 20260506 11:32:14.769 | INFO | "huge" : "", | 20260506 11:32:14.769 | INFO | "huge_shared" : " 27 GB", | 20260506 11:32:14.769 | INFO | "malloc" : " 14 GB", | 20260506 11:32:14.769 | INFO | "pinned" : " 29 GB" | 20260506 11:32:14.769 | INFO | } | 20260506 11:32:14.769 | INFO | Perf report for the last 5.13936 seconds: | 20260506 11:32:14.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:32:14.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:14.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:14.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:14.769 | INFO | ANALYSIS : 64.0%, 2.9, 95.9ms, 0 | 20260506 11:32:14.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:14.769 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260506 11:32:14.770 | INFO | PRE_HQ : 61.9%, 1.9, 61.8ms, 0 | 20260506 11:32:14.770 | INFO | BAZWRITER : 66.4%, 4.8, 164.5ms, 0 | 20260506 11:32:14.770 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 11:32:14.770 | INFO | Uploaded 36GiB to GPU in 2.12377s (16.951GiB/s) | 20260506 11:32:14.770 | INFO | Downloaded 38GiB to GPU in 2.21853s (17.1284GiB/s) | 20260506 11:32:19.909 | INFO | Analyzing chunk frames = [1020416,1020928) | 20260506 11:32:19.909 | INFO | MemInfo: { | 20260506 11:32:19.909 | INFO | "gpu" : " 17 GB", | 20260506 11:32:19.909 | INFO | "huge" : "", | 20260506 11:32:19.909 | INFO | "huge_shared" : " 27 GB", | 20260506 11:32:19.909 | INFO | "malloc" : " 14 GB", | 20260506 11:32:19.909 | INFO | "pinned" : " 29 GB" | 20260506 11:32:19.909 | INFO | } | 20260506 11:32:19.909 | INFO | Perf report for the last 5.13961 seconds: | 20260506 11:32:19.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:32:19.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:19.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:19.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:19.909 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:32:19.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:19.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:19.909 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 11:32:19.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:19.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:19.909 | INFO | Uploaded 36GiB to GPU in 2.00785s (17.9296GiB/s) | 20260506 11:32:19.909 | INFO | Downloaded 34GiB to GPU in 1.93381s (17.5819GiB/s) | 20260506 11:32:25.048 | INFO | Analyzing chunk frames = [1020928,1021440) | 20260506 11:32:25.048 | INFO | MemInfo: { | 20260506 11:32:25.048 | INFO | "gpu" : " 17 GB", | 20260506 11:32:25.048 | INFO | "huge" : "", | 20260506 11:32:25.048 | INFO | "huge_shared" : " 27 GB", | 20260506 11:32:25.048 | INFO | "malloc" : " 14 GB", | 20260506 11:32:25.048 | INFO | "pinned" : " 29 GB" | 20260506 11:32:25.048 | INFO | } | 20260506 11:32:25.048 | INFO | Perf report for the last 5.13901 seconds: | 20260506 11:32:25.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:32:25.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:25.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:25.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:25.048 | INFO | ANALYSIS : 62.4%, 3.0, 96.6ms, 0 | 20260506 11:32:25.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:25.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:25.048 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260506 11:32:25.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:25.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:25.048 | INFO | Uploaded 36GiB to GPU in 2.0511s (17.5516GiB/s) | 20260506 11:32:25.048 | INFO | Downloaded 34GiB to GPU in 1.95666s (17.3765GiB/s) | 20260506 11:32:30.187 | INFO | Analyzing chunk frames = [1021440,1021952) | 20260506 11:32:30.187 | INFO | MemInfo: { | 20260506 11:32:30.187 | INFO | "gpu" : " 17 GB", | 20260506 11:32:30.187 | INFO | "huge" : "", | 20260506 11:32:30.187 | INFO | "huge_shared" : " 27 GB", | 20260506 11:32:30.187 | INFO | "malloc" : " 14 GB", | 20260506 11:32:30.187 | INFO | "pinned" : " 29 GB" | 20260506 11:32:30.187 | INFO | } | 20260506 11:32:30.187 | INFO | Perf report for the last 5.13941 seconds: | 20260506 11:32:30.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:32:30.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:30.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:30.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:30.188 | INFO | ANALYSIS : 62.2%, 2.9, 96.2ms, 0 | 20260506 11:32:30.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:30.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:30.188 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:32:30.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:30.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:30.188 | INFO | Uploaded 36GiB to GPU in 2.0013s (17.9883GiB/s) | 20260506 11:32:30.188 | INFO | Downloaded 34GiB to GPU in 1.97932s (17.1776GiB/s) | 20260506 11:32:35.326 | INFO | Analyzing chunk frames = [1021952,1022464) | 20260506 11:32:35.326 | INFO | MemInfo: { | 20260506 11:32:35.326 | INFO | "gpu" : " 17 GB", | 20260506 11:32:35.326 | INFO | "huge" : "", | 20260506 11:32:35.326 | INFO | "huge_shared" : " 27 GB", | 20260506 11:32:35.327 | INFO | "malloc" : " 14 GB", | 20260506 11:32:35.327 | INFO | "pinned" : " 29 GB" | 20260506 11:32:35.327 | INFO | } | 20260506 11:32:35.327 | INFO | Perf report for the last 5.13915 seconds: | 20260506 11:32:35.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:32:35.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:35.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:35.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:35.327 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 11:32:35.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:35.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:35.327 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 11:32:35.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:35.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:35.327 | INFO | Uploaded 36GiB to GPU in 1.96442s (18.326GiB/s) | 20260506 11:32:35.327 | INFO | Downloaded 34GiB to GPU in 1.95665s (17.3766GiB/s) | 20260506 11:32:40.466 | INFO | Analyzing chunk frames = [1022464,1022976) | 20260506 11:32:40.466 | INFO | MemInfo: { | 20260506 11:32:40.466 | INFO | "gpu" : " 17 GB", | 20260506 11:32:40.466 | INFO | "huge" : "", | 20260506 11:32:40.467 | INFO | "huge_shared" : " 27 GB", | 20260506 11:32:40.467 | INFO | "malloc" : " 14 GB", | 20260506 11:32:40.467 | INFO | "pinned" : " 29 GB" | 20260506 11:32:40.467 | INFO | } | 20260506 11:32:40.467 | INFO | Perf report for the last 5.13995 seconds: | 20260506 11:32:40.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:32:40.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:40.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:40.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:40.467 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 11:32:40.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:40.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:40.467 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:32:40.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:40.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:40.467 | INFO | Uploaded 36GiB to GPU in 2.08293s (17.2834GiB/s) | 20260506 11:32:40.467 | INFO | Downloaded 34GiB to GPU in 1.98578s (17.1217GiB/s) | 20260506 11:32:45.606 | INFO | Analyzing chunk frames = [1022976,1023488) | 20260506 11:32:45.606 | INFO | MemInfo: { | 20260506 11:32:45.606 | INFO | "gpu" : " 17 GB", | 20260506 11:32:45.606 | INFO | "huge" : "", | 20260506 11:32:45.606 | INFO | "huge_shared" : " 27 GB", | 20260506 11:32:45.606 | INFO | "malloc" : " 14 GB", | 20260506 11:32:45.606 | INFO | "pinned" : " 29 GB" | 20260506 11:32:45.606 | INFO | } | 20260506 11:32:45.606 | INFO | Perf report for the last 5.13919 seconds: | 20260506 11:32:45.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:32:45.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:45.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:45.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:45.606 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260506 11:32:45.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:45.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:45.606 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 11:32:45.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:45.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:45.606 | INFO | Uploaded 36GiB to GPU in 2.0344s (17.6956GiB/s) | 20260506 11:32:45.606 | INFO | Downloaded 34GiB to GPU in 2.02335s (16.8038GiB/s) | 20260506 11:32:50.745 | INFO | Analyzing chunk frames = [1023488,1024000) | 20260506 11:32:50.745 | INFO | MemInfo: { | 20260506 11:32:50.745 | INFO | "gpu" : " 17 GB", | 20260506 11:32:50.745 | INFO | "huge" : "", | 20260506 11:32:50.745 | INFO | "huge_shared" : " 27 GB", | 20260506 11:32:50.745 | INFO | "malloc" : " 14 GB", | 20260506 11:32:50.745 | INFO | "pinned" : " 29 GB" | 20260506 11:32:50.745 | INFO | } | 20260506 11:32:50.745 | INFO | Perf report for the last 5.13919 seconds: | 20260506 11:32:50.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:32:50.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:50.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:50.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:50.745 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260506 11:32:50.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:50.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:50.745 | INFO | PRE_HQ : 33.2%, 1.0, 17.2ms, 0 | 20260506 11:32:50.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:50.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:50.745 | INFO | Uploaded 36GiB to GPU in 2.0288s (17.7445GiB/s) | 20260506 11:32:50.745 | INFO | Downloaded 34GiB to GPU in 1.97195s (17.2418GiB/s) | 20260506 11:32:50.745 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:32:50.745 | INFO | Total mem usage: 16.8 GiB | 20260506 11:32:50.745 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:32:50.745 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -5.52%) | 20260506 11:32:50.745 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:32:50.745 | INFO | High Water Breakdown: | 20260506 11:32:50.745 | INFO | 61% : 9.72 GiB ( 93.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:32:50.745 | INFO | 38.9% : 6.19 GiB ( 92.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:32:50.745 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:32:50.745 | INFO | ------------------------------------------ | 20260506 11:32:50.745 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:32:50.745 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:32:50.745 | INFO | Total mem usage: 26.5 GiB | 20260506 11:32:50.745 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:32:50.745 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:32:50.745 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:32:50.745 | INFO | High Water Breakdown: | 20260506 11:32:50.745 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:32:50.745 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:32:50.746 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:32:50.746 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:32:50.746 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:32:50.746 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:32:50.746 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:32:50.746 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:32:50.746 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:32:50.746 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:32:50.746 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:32:50.746 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:32:50.746 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:32:50.746 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:32:50.746 | INFO | ------------------------------------------ | 20260506 11:32:50.746 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:32:50.746 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:32:50.746 | INFO | Total mem usage: 25.6 GiB | 20260506 11:32:50.746 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:32:50.746 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:32:50.746 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:32:50.746 | INFO | High Water Breakdown: | 20260506 11:32:50.746 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:32:50.746 | INFO | ------------------------------------------ | 20260506 11:32:50.746 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:32:50.746 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:32:50.746 | INFO | Total mem usage: 15.7 GiB | 20260506 11:32:50.746 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:32:50.746 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:32:50.746 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:32:50.746 | INFO | High Water Breakdown: | 20260506 11:32:50.746 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:32:50.746 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:32:50.746 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:32:50.746 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:32:50.746 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:32:50.746 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:32:50.746 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:32:50.746 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:32:50.746 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:32:50.746 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:32:50.746 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:32:50.746 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:32:50.746 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:32:50.746 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:32:50.746 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:32:50.746 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:32:50.746 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:32:50.746 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:32:50.746 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:32:50.746 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:32:50.746 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:32:50.746 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:32:50.746 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:32:50.746 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:32:50.746 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:32:50.746 | INFO | ------------------------------------------ | 20260506 11:32:50.746 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:32:50.746 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:32:50.746 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:32:50.746 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:32:50.746 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.63e+01, 1.37e+01 / 2.02e+00 | 20260506 11:32:50.746 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 9.97e-02, 1.02e-02 / 5.83e-03 | 20260506 11:32:50.746 | INFO | PerfReport: 11.54%, Baseline -- 3.68e+00 / 4.47e+00, 5.86e-01 / 3.32e-01 | 20260506 11:32:50.746 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.48e+00, 1.99e-01 / 1.45e-01 | 20260506 11:32:50.746 | INFO | PerfReport: 15.96%, DME -- 4.07e+01 / 4.57e+01, 4.24e+00 / 1.80e+00 | 20260506 11:32:50.746 | INFO | PerfReport: 56.07%, FrameLabeling -- 1.79e+01 / 1.92e+01, 1.35e+00 / 1.14e+00 | 20260506 11:32:50.746 | INFO | PerfReport: 7.69%, PulseAccumulating -- 2.45e+00 / 3.12e+00, 4.37e-01 / 1.79e-01 | 20260506 11:32:50.746 | INFO | PerfReport: 3.85%, Metrics -- 1.23e+00 / 1.63e+00, 3.11e-01 / 1.49e-01 | 20260506 11:32:50.746 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.35e-01 / 4.94e-01, 6.85e-02 / 2.61e-02 | 20260506 11:32:50.746 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:32:50.746 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:32:50.746 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:32:50.746 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260506 11:32:50.747 | INFO | PerfReport: 21.48%, Upload -- 2.12e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 11:32:50.747 | INFO | PerfReport: 21.15%, Download -- 2.09e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 11:32:50.747 | INFO | PerfReport: 32.27%, Compute -- 3.19e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 11:32:50.747 | INFO | PerfReport: 1.85%, LockUpload -- 1.82e+00 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260506 11:32:50.747 | INFO | PerfReport: 1.12%, LockDownload -- 1.11e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260506 11:32:50.747 | INFO | PerfReport: 22.11%, LockCompute -- 2.19e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260506 11:32:50.747 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.43e-02 / 0.00e+00 | 20260506 11:32:50.862 | INFO | Starting a round of Baz IO | 20260506 11:32:54.068 | INFO | Populating RTMetrics report | 20260506 11:32:54.069 | INFO | RTMetrics report populated | 20260506 11:32:54.069 | INFO | Opening tmp RTMetrics json file | 20260506 11:32:54.072 | INFO | Serializing RTMetrics to json | 20260506 11:32:54.079 | INFO | Serialization done, swapping tmp file now... | 20260506 11:32:54.080 | INFO | RTMetrics done | 20260506 11:32:54.399 | INFO | Finished a round of Baz IO: Wrote 2.06 GiB in 3.536 seconds | 20260506 11:32:54.399 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.86% / 0.03% / 56.10% | 20260506 11:32:54.399 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.03411% / 6.75519e-05% / 5.5631e-05% | 20260506 11:32:54.399 | INFO | Baz IO overall: | 20260506 11:32:54.399 | INFO | BytesWritten : 492711639869 | 20260506 11:32:54.399 | INFO | NumEvents : 237774175240 | 20260506 11:32:55.884 | INFO | Analyzing chunk frames = [1024000,1024512) | 20260506 11:32:55.884 | INFO | MemInfo: { | 20260506 11:32:55.884 | INFO | "gpu" : " 17 GB", | 20260506 11:32:55.884 | INFO | "huge" : "", | 20260506 11:32:55.884 | INFO | "huge_shared" : " 27 GB", | 20260506 11:32:55.884 | INFO | "malloc" : " 14 GB", | 20260506 11:32:55.884 | INFO | "pinned" : " 29 GB" | 20260506 11:32:55.884 | INFO | } | 20260506 11:32:55.884 | INFO | Perf report for the last 5.13904 seconds: | 20260506 11:32:55.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:32:55.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:55.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:55.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:55.884 | INFO | ANALYSIS : 63.8%, 2.9, 96.2ms, 0 | 20260506 11:32:55.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:32:55.884 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 11:32:55.884 | INFO | PRE_HQ : 61.6%, 1.9, 61.5ms, 0 | 20260506 11:32:55.884 | INFO | BAZWRITER : 67.9%, 4.8, 166.5ms, 0 | 20260506 11:32:55.884 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.5ms, 0 | 20260506 11:32:55.884 | INFO | Uploaded 36GiB to GPU in 2.10533s (17.0994GiB/s) | 20260506 11:32:55.884 | INFO | Downloaded 38GiB to GPU in 2.22447s (17.0827GiB/s) | 20260506 11:33:01.023 | INFO | Analyzing chunk frames = [1024512,1025024) | 20260506 11:33:01.023 | INFO | MemInfo: { | 20260506 11:33:01.023 | INFO | "gpu" : " 17 GB", | 20260506 11:33:01.023 | INFO | "huge" : "", | 20260506 11:33:01.023 | INFO | "huge_shared" : " 27 GB", | 20260506 11:33:01.023 | INFO | "malloc" : " 14 GB", | 20260506 11:33:01.023 | INFO | "pinned" : " 29 GB" | 20260506 11:33:01.023 | INFO | } | 20260506 11:33:01.023 | INFO | Perf report for the last 5.13931 seconds: | 20260506 11:33:01.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:33:01.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:01.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:01.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:01.023 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:33:01.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:01.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:01.024 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 11:33:01.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:01.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:01.024 | INFO | Uploaded 36GiB to GPU in 2.00859s (17.923GiB/s) | 20260506 11:33:01.024 | INFO | Downloaded 34GiB to GPU in 1.9235s (17.6761GiB/s) | 20260506 11:33:06.163 | INFO | Analyzing chunk frames = [1025024,1025536) | 20260506 11:33:06.163 | INFO | MemInfo: { | 20260506 11:33:06.163 | INFO | "gpu" : " 17 GB", | 20260506 11:33:06.163 | INFO | "huge" : "", | 20260506 11:33:06.163 | INFO | "huge_shared" : " 27 GB", | 20260506 11:33:06.163 | INFO | "malloc" : " 14 GB", | 20260506 11:33:06.163 | INFO | "pinned" : " 29 GB" | 20260506 11:33:06.163 | INFO | } | 20260506 11:33:06.163 | INFO | Perf report for the last 5.13978 seconds: | 20260506 11:33:06.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:33:06.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:06.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:06.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:06.163 | INFO | ANALYSIS : 62.0%, 3.0, 96.0ms, 0 | 20260506 11:33:06.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:06.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:06.163 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:33:06.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:06.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:06.163 | INFO | Uploaded 36GiB to GPU in 2.02402s (17.7864GiB/s) | 20260506 11:33:06.163 | INFO | Downloaded 34GiB to GPU in 1.94737s (17.4594GiB/s) | 20260506 11:33:11.302 | INFO | Analyzing chunk frames = [1025536,1026048) | 20260506 11:33:11.302 | INFO | MemInfo: { | 20260506 11:33:11.302 | INFO | "gpu" : " 17 GB", | 20260506 11:33:11.302 | INFO | "huge" : "", | 20260506 11:33:11.302 | INFO | "huge_shared" : " 27 GB", | 20260506 11:33:11.302 | INFO | "malloc" : " 14 GB", | 20260506 11:33:11.302 | INFO | "pinned" : " 29 GB" | 20260506 11:33:11.302 | INFO | } | 20260506 11:33:11.302 | INFO | Perf report for the last 5.13899 seconds: | 20260506 11:33:11.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:33:11.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:11.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:11.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:11.302 | INFO | ANALYSIS : 62.1%, 2.9, 96.0ms, 0 | 20260506 11:33:11.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:11.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:11.302 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:33:11.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:11.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:11.302 | INFO | Uploaded 36GiB to GPU in 2.00572s (17.9487GiB/s) | 20260506 11:33:11.302 | INFO | Downloaded 34GiB to GPU in 1.98777s (17.1046GiB/s) | 20260506 11:33:16.441 | INFO | Analyzing chunk frames = [1026048,1026560) | 20260506 11:33:16.441 | INFO | MemInfo: { | 20260506 11:33:16.441 | INFO | "gpu" : " 17 GB", | 20260506 11:33:16.441 | INFO | "huge" : "", | 20260506 11:33:16.441 | INFO | "huge_shared" : " 27 GB", | 20260506 11:33:16.441 | INFO | "malloc" : " 14 GB", | 20260506 11:33:16.441 | INFO | "pinned" : " 29 GB" | 20260506 11:33:16.441 | INFO | } | 20260506 11:33:16.442 | INFO | Perf report for the last 5.13932 seconds: | 20260506 11:33:16.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:33:16.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:16.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:16.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:16.442 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 11:33:16.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:16.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:16.442 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:33:16.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:16.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:16.442 | INFO | Uploaded 36GiB to GPU in 2.00773s (17.9307GiB/s) | 20260506 11:33:16.442 | INFO | Downloaded 34GiB to GPU in 1.95865s (17.3589GiB/s) | 20260506 11:33:21.581 | INFO | Analyzing chunk frames = [1026560,1027072) | 20260506 11:33:21.581 | INFO | MemInfo: { | 20260506 11:33:21.581 | INFO | "gpu" : " 17 GB", | 20260506 11:33:21.581 | INFO | "huge" : "", | 20260506 11:33:21.581 | INFO | "huge_shared" : " 27 GB", | 20260506 11:33:21.581 | INFO | "malloc" : " 14 GB", | 20260506 11:33:21.581 | INFO | "pinned" : " 29 GB" | 20260506 11:33:21.581 | INFO | } | 20260506 11:33:21.581 | INFO | Perf report for the last 5.13939 seconds: | 20260506 11:33:21.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:33:21.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:21.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:21.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:21.581 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 11:33:21.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:21.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:21.581 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:33:21.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:21.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:21.581 | INFO | Uploaded 36GiB to GPU in 2.09391s (17.1927GiB/s) | 20260506 11:33:21.581 | INFO | Downloaded 34GiB to GPU in 1.99068s (17.0796GiB/s) | 20260506 11:33:26.721 | INFO | Analyzing chunk frames = [1027072,1027584) | 20260506 11:33:26.721 | INFO | MemInfo: { | 20260506 11:33:26.721 | INFO | "gpu" : " 17 GB", | 20260506 11:33:26.721 | INFO | "huge" : "", | 20260506 11:33:26.721 | INFO | "huge_shared" : " 27 GB", | 20260506 11:33:26.721 | INFO | "malloc" : " 14 GB", | 20260506 11:33:26.721 | INFO | "pinned" : " 29 GB" | 20260506 11:33:26.721 | INFO | } | 20260506 11:33:26.721 | INFO | Perf report for the last 5.14032 seconds: | 20260506 11:33:26.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:33:26.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:26.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:26.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:26.721 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:33:26.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:26.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:26.721 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 11:33:26.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:26.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:26.721 | INFO | Uploaded 36GiB to GPU in 2.02643s (17.7652GiB/s) | 20260506 11:33:26.722 | INFO | Downloaded 34GiB to GPU in 2.03284s (16.7254GiB/s) | 20260506 11:33:31.859 | INFO | Analyzing chunk frames = [1027584,1028096) | 20260506 11:33:31.859 | INFO | MemInfo: { | 20260506 11:33:31.859 | INFO | "gpu" : " 17 GB", | 20260506 11:33:31.859 | INFO | "huge" : "", | 20260506 11:33:31.860 | INFO | "huge_shared" : " 27 GB", | 20260506 11:33:31.860 | INFO | "malloc" : " 14 GB", | 20260506 11:33:31.860 | INFO | "pinned" : " 29 GB" | 20260506 11:33:31.860 | INFO | } | 20260506 11:33:31.860 | INFO | Perf report for the last 5.13838 seconds: | 20260506 11:33:31.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:33:31.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:31.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:31.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:31.860 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 11:33:31.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:31.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:31.860 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260506 11:33:31.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:31.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:31.860 | INFO | Uploaded 36GiB to GPU in 2.00599s (17.9463GiB/s) | 20260506 11:33:31.860 | INFO | Downloaded 34GiB to GPU in 1.9647s (17.3055GiB/s) | 20260506 11:33:31.986 | INFO | Starting a round of Baz IO | 20260506 11:33:35.183 | INFO | Populating RTMetrics report | 20260506 11:33:35.184 | INFO | RTMetrics report populated | 20260506 11:33:35.184 | INFO | Opening tmp RTMetrics json file | 20260506 11:33:35.186 | INFO | Serializing RTMetrics to json | 20260506 11:33:35.190 | INFO | Serialization done, swapping tmp file now... | 20260506 11:33:35.190 | INFO | RTMetrics done | 20260506 11:33:35.292 | INFO | Finished a round of Baz IO: Wrote 2.063 GiB in 3.306 seconds | 20260506 11:33:35.292 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.83% / 0.03% / 56.14% | 20260506 11:33:35.292 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.03563% / 2.38419e-05% / 5.96046e-05% | 20260506 11:33:35.292 | INFO | Baz IO overall: | 20260506 11:33:35.292 | INFO | BytesWritten : 494926789118 | 20260506 11:33:35.292 | INFO | NumEvents : 238849832801 | 20260506 11:33:36.999 | INFO | Analyzing chunk frames = [1028096,1028608) | 20260506 11:33:36.999 | INFO | MemInfo: { | 20260506 11:33:36.999 | INFO | "gpu" : " 17 GB", | 20260506 11:33:36.999 | INFO | "huge" : "", | 20260506 11:33:36.999 | INFO | "huge_shared" : " 27 GB", | 20260506 11:33:36.999 | INFO | "malloc" : " 14 GB", | 20260506 11:33:36.999 | INFO | "pinned" : " 29 GB" | 20260506 11:33:36.999 | INFO | } | 20260506 11:33:36.999 | INFO | Perf report for the last 5.13935 seconds: | 20260506 11:33:36.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:33:36.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:36.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:36.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:36.999 | INFO | ANALYSIS : 64.0%, 2.9, 96.2ms, 0 | 20260506 11:33:36.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:36.999 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 11:33:36.999 | INFO | PRE_HQ : 62.2%, 1.9, 61.9ms, 0 | 20260506 11:33:36.999 | INFO | BAZWRITER : 63.4%, 4.7, 154.1ms, 0 | 20260506 11:33:36.999 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 11:33:36.999 | INFO | Uploaded 36GiB to GPU in 2.14443s (16.7877GiB/s) | 20260506 11:33:36.999 | INFO | Downloaded 38GiB to GPU in 2.23617s (16.9933GiB/s) | 20260506 11:33:42.138 | INFO | Analyzing chunk frames = [1028608,1029120) | 20260506 11:33:42.138 | INFO | MemInfo: { | 20260506 11:33:42.138 | INFO | "gpu" : " 17 GB", | 20260506 11:33:42.138 | INFO | "huge" : "", | 20260506 11:33:42.138 | INFO | "huge_shared" : " 27 GB", | 20260506 11:33:42.138 | INFO | "malloc" : " 14 GB", | 20260506 11:33:42.138 | INFO | "pinned" : " 29 GB" | 20260506 11:33:42.138 | INFO | } | 20260506 11:33:42.138 | INFO | Perf report for the last 5.13902 seconds: | 20260506 11:33:42.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:33:42.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:42.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:42.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:42.138 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:33:42.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:42.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:42.138 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:33:42.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:42.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:42.138 | INFO | Uploaded 36GiB to GPU in 2.01219s (17.891GiB/s) | 20260506 11:33:42.138 | INFO | Downloaded 34GiB to GPU in 1.91851s (17.7221GiB/s) | 20260506 11:33:47.277 | INFO | Analyzing chunk frames = [1029120,1029632) | 20260506 11:33:47.277 | INFO | MemInfo: { | 20260506 11:33:47.277 | INFO | "gpu" : " 17 GB", | 20260506 11:33:47.277 | INFO | "huge" : "", | 20260506 11:33:47.277 | INFO | "huge_shared" : " 27 GB", | 20260506 11:33:47.277 | INFO | "malloc" : " 14 GB", | 20260506 11:33:47.277 | INFO | "pinned" : " 29 GB" | 20260506 11:33:47.277 | INFO | } | 20260506 11:33:47.277 | INFO | Perf report for the last 5.13941 seconds: | 20260506 11:33:47.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:33:47.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:47.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:47.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:47.277 | INFO | ANALYSIS : 62.3%, 3.0, 96.5ms, 0 | 20260506 11:33:47.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:47.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:47.278 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:33:47.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:47.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:47.278 | INFO | Uploaded 36GiB to GPU in 2.0012s (17.9892GiB/s) | 20260506 11:33:47.278 | INFO | Downloaded 34GiB to GPU in 1.94904s (17.4445GiB/s) | 20260506 11:33:52.417 | INFO | Analyzing chunk frames = [1029632,1030144) | 20260506 11:33:52.417 | INFO | MemInfo: { | 20260506 11:33:52.417 | INFO | "gpu" : " 17 GB", | 20260506 11:33:52.417 | INFO | "huge" : "", | 20260506 11:33:52.417 | INFO | "huge_shared" : " 27 GB", | 20260506 11:33:52.417 | INFO | "malloc" : " 14 GB", | 20260506 11:33:52.417 | INFO | "pinned" : " 29 GB" | 20260506 11:33:52.417 | INFO | } | 20260506 11:33:52.417 | INFO | Perf report for the last 5.13941 seconds: | 20260506 11:33:52.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:33:52.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:52.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:52.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:52.417 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 11:33:52.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:52.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:52.417 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:33:52.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:52.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:52.417 | INFO | Uploaded 36GiB to GPU in 2.07341s (17.3627GiB/s) | 20260506 11:33:52.417 | INFO | Downloaded 34GiB to GPU in 1.96516s (17.3014GiB/s) | 20260506 11:33:57.556 | INFO | Analyzing chunk frames = [1030144,1030656) | 20260506 11:33:57.556 | INFO | MemInfo: { | 20260506 11:33:57.556 | INFO | "gpu" : " 17 GB", | 20260506 11:33:57.556 | INFO | "huge" : "", | 20260506 11:33:57.556 | INFO | "huge_shared" : " 27 GB", | 20260506 11:33:57.556 | INFO | "malloc" : " 14 GB", | 20260506 11:33:57.556 | INFO | "pinned" : " 29 GB" | 20260506 11:33:57.556 | INFO | } | 20260506 11:33:57.556 | INFO | Perf report for the last 5.13944 seconds: | 20260506 11:33:57.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:33:57.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:57.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:57.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:57.556 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 11:33:57.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:57.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:57.556 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 11:33:57.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:57.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:33:57.557 | INFO | Uploaded 36GiB to GPU in 2.09219s (17.2068GiB/s) | 20260506 11:33:57.557 | INFO | Downloaded 34GiB to GPU in 1.98362s (17.1403GiB/s) | 20260506 11:34:02.696 | INFO | Analyzing chunk frames = [1030656,1031168) | 20260506 11:34:02.696 | INFO | MemInfo: { | 20260506 11:34:02.696 | INFO | "gpu" : " 17 GB", | 20260506 11:34:02.696 | INFO | "huge" : "", | 20260506 11:34:02.696 | INFO | "huge_shared" : " 27 GB", | 20260506 11:34:02.696 | INFO | "malloc" : " 14 GB", | 20260506 11:34:02.696 | INFO | "pinned" : " 29 GB" | 20260506 11:34:02.696 | INFO | } | 20260506 11:34:02.696 | INFO | Perf report for the last 5.13979 seconds: | 20260506 11:34:02.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:34:02.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:02.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:02.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:02.696 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 11:34:02.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:02.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:02.696 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:34:02.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:02.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:02.696 | INFO | Uploaded 36GiB to GPU in 2.0619s (17.4596GiB/s) | 20260506 11:34:02.696 | INFO | Downloaded 34GiB to GPU in 1.99908s (17.0078GiB/s) | 20260506 11:34:07.835 | INFO | Analyzing chunk frames = [1031168,1031680) | 20260506 11:34:07.835 | INFO | MemInfo: { | 20260506 11:34:07.835 | INFO | "gpu" : " 17 GB", | 20260506 11:34:07.835 | INFO | "huge" : "", | 20260506 11:34:07.835 | INFO | "huge_shared" : " 27 GB", | 20260506 11:34:07.835 | INFO | "malloc" : " 14 GB", | 20260506 11:34:07.835 | INFO | "pinned" : " 29 GB" | 20260506 11:34:07.835 | INFO | } | 20260506 11:34:07.836 | INFO | Perf report for the last 5.13938 seconds: | 20260506 11:34:07.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:34:07.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:07.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:07.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:07.836 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:34:07.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:07.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:07.836 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 11:34:07.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:07.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:07.836 | INFO | Uploaded 36GiB to GPU in 1.99794s (18.0186GiB/s) | 20260506 11:34:07.836 | INFO | Downloaded 34GiB to GPU in 1.99755s (17.0209GiB/s) | 20260506 11:34:12.974 | INFO | Analyzing chunk frames = [1031680,1032192) | 20260506 11:34:12.974 | INFO | MemInfo: { | 20260506 11:34:12.974 | INFO | "gpu" : " 17 GB", | 20260506 11:34:12.974 | INFO | "huge" : "", | 20260506 11:34:12.974 | INFO | "huge_shared" : " 27 GB", | 20260506 11:34:12.974 | INFO | "malloc" : " 14 GB", | 20260506 11:34:12.974 | INFO | "pinned" : " 29 GB" | 20260506 11:34:12.974 | INFO | } | 20260506 11:34:12.974 | INFO | Perf report for the last 5.13892 seconds: | 20260506 11:34:12.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:34:12.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:12.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:12.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:12.974 | INFO | ANALYSIS : 63.0%, 3.0, 98.0ms, 0 | 20260506 11:34:12.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:12.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:12.975 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:34:12.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:12.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:12.975 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:34:12.975 | INFO | Total mem usage: 16.8 GiB | 20260506 11:34:12.975 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:34:12.975 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -9.95%) | 20260506 11:34:12.975 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:34:12.975 | INFO | High Water Breakdown: | 20260506 11:34:12.975 | INFO | 60.3% : 9.23 GiB ( 88.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:34:12.975 | INFO | 39.5% : 6.04 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:34:12.975 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:34:12.975 | INFO | ------------------------------------------ | 20260506 11:34:12.975 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:34:12.975 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:34:12.975 | INFO | Total mem usage: 26.5 GiB | 20260506 11:34:12.975 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:34:12.975 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:34:12.975 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:34:12.975 | INFO | High Water Breakdown: | 20260506 11:34:12.975 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:34:12.975 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:34:12.975 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:34:12.975 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:34:12.975 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:34:12.975 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:34:12.975 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:34:12.975 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:34:12.975 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:34:12.975 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:34:12.975 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:34:12.975 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:34:12.975 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:34:12.975 | INFO | Uploaded 36GiB to GPU in 2.04624s (17.5932GiB/s) | 20260506 11:34:12.975 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:34:12.975 | INFO | Downloaded 34GiB to GPU in 1.97456s (17.219GiB/s) | 20260506 11:34:12.975 | INFO | ------------------------------------------ | 20260506 11:34:12.975 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:34:12.975 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:34:12.975 | INFO | Total mem usage: 25.6 GiB | 20260506 11:34:12.975 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:34:12.975 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:34:12.975 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:34:12.975 | INFO | High Water Breakdown: | 20260506 11:34:12.975 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:34:12.975 | INFO | ------------------------------------------ | 20260506 11:34:12.975 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:34:12.975 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:34:12.975 | INFO | Total mem usage: 15.7 GiB | 20260506 11:34:12.975 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:34:12.975 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:34:12.975 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 11:34:12.975 | INFO | High Water Breakdown: | 20260506 11:34:12.975 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:34:12.975 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:34:12.975 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:34:12.975 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:34:12.975 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:34:12.975 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:34:12.975 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:34:12.975 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:34:12.975 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:34:12.975 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:34:12.975 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:34:12.975 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:34:12.975 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:34:12.975 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:34:12.975 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:34:12.975 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:34:12.975 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:34:12.975 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:34:12.975 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:34:12.975 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:34:12.975 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:34:12.976 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:34:12.976 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:34:12.976 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:34:12.976 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:34:12.976 | INFO | ------------------------------------------ | 20260506 11:34:12.976 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:34:12.976 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:34:12.976 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:34:12.976 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:34:12.976 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.62e+01, 1.37e+01 / 2.23e+00 | 20260506 11:34:12.976 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 9.93e-02, 1.02e-02 / 6.74e-03 | 20260506 11:34:12.976 | INFO | PerfReport: 11.56%, Baseline -- 3.69e+00 / 4.47e+00, 5.92e-01 / 3.17e-01 | 20260506 11:34:12.976 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.46e+00, 1.93e-01 / 1.34e-01 | 20260506 11:34:12.976 | INFO | PerfReport: 15.90%, DME -- 4.06e+01 / 4.53e+01, 4.18e+00 / 2.42e+00 | 20260506 11:34:12.976 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.44e+00 / 1.64e+00 | 20260506 11:34:12.976 | INFO | PerfReport: 7.66%, PulseAccumulating -- 2.45e+00 / 3.10e+00, 4.27e-01 / 2.24e-01 | 20260506 11:34:12.976 | INFO | PerfReport: 3.84%, Metrics -- 1.23e+00 / 1.61e+00, 3.09e-01 / 1.70e-01 | 20260506 11:34:12.976 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.34e-01 / 4.94e-01, 6.77e-02 / 3.04e-02 | 20260506 11:34:12.976 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:34:12.976 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:34:12.976 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:34:12.976 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260506 11:34:12.976 | INFO | PerfReport: 21.54%, Upload -- 2.13e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260506 11:34:12.976 | INFO | PerfReport: 21.11%, Download -- 2.09e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 11:34:12.976 | INFO | PerfReport: 32.30%, Compute -- 3.19e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260506 11:34:12.976 | INFO | PerfReport: 1.91%, LockUpload -- 1.89e+00 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 11:34:12.976 | INFO | PerfReport: 1.00%, LockDownload -- 9.86e-01 / 0.00e+00, 2.60e+00 / 0.00e+00 | 20260506 11:34:12.976 | INFO | PerfReport: 22.12%, LockCompute -- 2.19e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260506 11:34:12.976 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260506 11:34:13.093 | INFO | Starting a round of Baz IO | 20260506 11:34:16.317 | INFO | Populating RTMetrics report | 20260506 11:34:16.317 | INFO | RTMetrics report populated | 20260506 11:34:16.317 | INFO | Opening tmp RTMetrics json file | 20260506 11:34:16.436 | INFO | Serializing RTMetrics to json | 20260506 11:34:16.443 | INFO | Serialization done, swapping tmp file now... | 20260506 11:34:16.448 | INFO | RTMetrics done | 20260506 11:34:16.756 | INFO | Finished a round of Baz IO: Wrote 2.064 GiB in 3.663 seconds | 20260506 11:34:16.756 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.80% / 0.03% / 56.17% | 20260506 11:34:16.756 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.02472% / 7.94729e-05% / 2.78155e-05% | 20260506 11:34:16.756 | INFO | Baz IO overall: | 20260506 11:34:16.756 | INFO | BytesWritten : 497143326016 | 20260506 11:34:16.756 | INFO | NumEvents : 239926167601 | 20260506 11:34:18.114 | INFO | Analyzing chunk frames = [1032192,1032704) | 20260506 11:34:18.114 | INFO | MemInfo: { | 20260506 11:34:18.114 | INFO | "gpu" : " 17 GB", | 20260506 11:34:18.114 | INFO | "huge" : "", | 20260506 11:34:18.114 | INFO | "huge_shared" : " 27 GB", | 20260506 11:34:18.114 | INFO | "malloc" : " 14 GB", | 20260506 11:34:18.114 | INFO | "pinned" : " 29 GB" | 20260506 11:34:18.114 | INFO | } | 20260506 11:34:18.114 | INFO | Perf report for the last 5.13942 seconds: | 20260506 11:34:18.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:34:18.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:18.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:18.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:18.114 | INFO | ANALYSIS : 64.1%, 2.9, 96.1ms, 0 | 20260506 11:34:18.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:18.114 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260506 11:34:18.114 | INFO | PRE_HQ : 61.6%, 2.0, 62.9ms, 0 | 20260506 11:34:18.114 | INFO | BAZWRITER : 70.2%, 4.8, 172.8ms, 0 | 20260506 11:34:18.114 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 11:34:18.114 | INFO | Uploaded 36GiB to GPU in 2.09723s (17.1655GiB/s) | 20260506 11:34:18.114 | INFO | Downloaded 38GiB to GPU in 2.20442s (17.2381GiB/s) | 20260506 11:34:23.253 | INFO | Analyzing chunk frames = [1032704,1033216) | 20260506 11:34:23.253 | INFO | MemInfo: { | 20260506 11:34:23.253 | INFO | "gpu" : " 17 GB", | 20260506 11:34:23.253 | INFO | "huge" : "", | 20260506 11:34:23.253 | INFO | "huge_shared" : " 27 GB", | 20260506 11:34:23.253 | INFO | "malloc" : " 14 GB", | 20260506 11:34:23.253 | INFO | "pinned" : " 29 GB" | 20260506 11:34:23.253 | INFO | } | 20260506 11:34:23.253 | INFO | Perf report for the last 5.13925 seconds: | 20260506 11:34:23.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:34:23.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:23.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:23.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:23.253 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:34:23.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:23.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:23.253 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260506 11:34:23.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:23.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:23.254 | INFO | Uploaded 36GiB to GPU in 1.99057s (18.0853GiB/s) | 20260506 11:34:23.254 | INFO | Downloaded 34GiB to GPU in 1.91782s (17.7285GiB/s) | 20260506 11:34:28.393 | INFO | Analyzing chunk frames = [1033216,1033728) | 20260506 11:34:28.393 | INFO | MemInfo: { | 20260506 11:34:28.393 | INFO | "gpu" : " 17 GB", | 20260506 11:34:28.393 | INFO | "huge" : "", | 20260506 11:34:28.393 | INFO | "huge_shared" : " 27 GB", | 20260506 11:34:28.393 | INFO | "malloc" : " 14 GB", | 20260506 11:34:28.393 | INFO | "pinned" : " 29 GB" | 20260506 11:34:28.393 | INFO | } | 20260506 11:34:28.393 | INFO | Perf report for the last 5.13949 seconds: | 20260506 11:34:28.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:34:28.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:28.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:28.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:28.393 | INFO | ANALYSIS : 62.0%, 3.0, 96.0ms, 0 | 20260506 11:34:28.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:28.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:28.393 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:34:28.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:28.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:28.393 | INFO | Uploaded 36GiB to GPU in 2.03479s (17.6923GiB/s) | 20260506 11:34:28.393 | INFO | Downloaded 34GiB to GPU in 1.94626s (17.4694GiB/s) | 20260506 11:34:33.532 | INFO | Analyzing chunk frames = [1033728,1034240) | 20260506 11:34:33.532 | INFO | MemInfo: { | 20260506 11:34:33.532 | INFO | "gpu" : " 17 GB", | 20260506 11:34:33.532 | INFO | "huge" : "", | 20260506 11:34:33.532 | INFO | "huge_shared" : " 27 GB", | 20260506 11:34:33.532 | INFO | "malloc" : " 14 GB", | 20260506 11:34:33.532 | INFO | "pinned" : " 29 GB" | 20260506 11:34:33.532 | INFO | } | 20260506 11:34:33.532 | INFO | Perf report for the last 5.13902 seconds: | 20260506 11:34:33.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:34:33.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:33.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:33.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:33.532 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 11:34:33.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:33.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:33.532 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:34:33.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:33.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:33.532 | INFO | Uploaded 36GiB to GPU in 2.0666s (17.4199GiB/s) | 20260506 11:34:33.532 | INFO | Downloaded 34GiB to GPU in 1.98124s (17.1609GiB/s) | 20260506 11:34:38.671 | INFO | Analyzing chunk frames = [1034240,1034752) | 20260506 11:34:38.671 | INFO | MemInfo: { | 20260506 11:34:38.671 | INFO | "gpu" : " 17 GB", | 20260506 11:34:38.671 | INFO | "huge" : "", | 20260506 11:34:38.671 | INFO | "huge_shared" : " 27 GB", | 20260506 11:34:38.671 | INFO | "malloc" : " 14 GB", | 20260506 11:34:38.671 | INFO | "pinned" : " 29 GB" | 20260506 11:34:38.671 | INFO | } | 20260506 11:34:38.671 | INFO | Perf report for the last 5.13952 seconds: | 20260506 11:34:38.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:34:38.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:38.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:38.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:38.671 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 11:34:38.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:38.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:38.671 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 11:34:38.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:38.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:38.671 | INFO | Uploaded 36GiB to GPU in 2.0917s (17.2109GiB/s) | 20260506 11:34:38.671 | INFO | Downloaded 34GiB to GPU in 1.99433s (17.0484GiB/s) | 20260506 11:34:43.810 | INFO | Analyzing chunk frames = [1034752,1035264) | 20260506 11:34:43.810 | INFO | MemInfo: { | 20260506 11:34:43.810 | INFO | "gpu" : " 17 GB", | 20260506 11:34:43.810 | INFO | "huge" : "", | 20260506 11:34:43.810 | INFO | "huge_shared" : " 27 GB", | 20260506 11:34:43.810 | INFO | "malloc" : " 14 GB", | 20260506 11:34:43.810 | INFO | "pinned" : " 29 GB" | 20260506 11:34:43.810 | INFO | } | 20260506 11:34:43.811 | INFO | Perf report for the last 5.13929 seconds: | 20260506 11:34:43.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:34:43.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:43.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:43.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:43.811 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 11:34:43.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:43.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:43.811 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 11:34:43.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:43.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:43.811 | INFO | Uploaded 36GiB to GPU in 2.05133s (17.5496GiB/s) | 20260506 11:34:43.811 | INFO | Downloaded 34GiB to GPU in 2.012s (16.8986GiB/s) | 20260506 11:34:48.950 | INFO | Analyzing chunk frames = [1035264,1035776) | 20260506 11:34:48.950 | INFO | MemInfo: { | 20260506 11:34:48.950 | INFO | "gpu" : " 17 GB", | 20260506 11:34:48.951 | INFO | "huge" : "", | 20260506 11:34:48.951 | INFO | "huge_shared" : " 27 GB", | 20260506 11:34:48.951 | INFO | "malloc" : " 14 GB", | 20260506 11:34:48.951 | INFO | "pinned" : " 29 GB" | 20260506 11:34:48.951 | INFO | } | 20260506 11:34:48.951 | INFO | Perf report for the last 5.14009 seconds: | 20260506 11:34:48.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:34:48.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:48.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:48.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:48.951 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:34:48.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:48.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:48.951 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 11:34:48.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:48.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:48.951 | INFO | Uploaded 36GiB to GPU in 1.99682s (18.0286GiB/s) | 20260506 11:34:48.951 | INFO | Downloaded 34GiB to GPU in 2.00234s (16.9801GiB/s) | 20260506 11:34:54.089 | INFO | Analyzing chunk frames = [1035776,1036288) | 20260506 11:34:54.089 | INFO | MemInfo: { | 20260506 11:34:54.089 | INFO | "gpu" : " 17 GB", | 20260506 11:34:54.089 | INFO | "huge" : "", | 20260506 11:34:54.089 | INFO | "huge_shared" : " 27 GB", | 20260506 11:34:54.089 | INFO | "malloc" : " 14 GB", | 20260506 11:34:54.089 | INFO | "pinned" : " 29 GB" | 20260506 11:34:54.089 | INFO | } | 20260506 11:34:54.089 | INFO | Perf report for the last 5.13843 seconds: | 20260506 11:34:54.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:34:54.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:54.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:54.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:54.089 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260506 11:34:54.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:54.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:54.089 | INFO | PRE_HQ : 33.4%, 1.0, 17.4ms, 0 | 20260506 11:34:54.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:54.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:54.090 | INFO | Uploaded 36GiB to GPU in 2.026s (17.769GiB/s) | 20260506 11:34:54.090 | INFO | Downloaded 34GiB to GPU in 1.98498s (17.1286GiB/s) | 20260506 11:34:54.212 | INFO | Starting a round of Baz IO | 20260506 11:34:57.397 | INFO | Populating RTMetrics report | 20260506 11:34:57.397 | INFO | RTMetrics report populated | 20260506 11:34:57.397 | INFO | Opening tmp RTMetrics json file | 20260506 11:34:57.402 | INFO | Serializing RTMetrics to json | 20260506 11:34:57.416 | INFO | Serialization done, swapping tmp file now... | 20260506 11:34:57.430 | INFO | RTMetrics done | 20260506 11:34:57.562 | INFO | Finished a round of Baz IO: Wrote 2.065 GiB in 3.35 seconds | 20260506 11:34:57.562 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.76% / 0.03% / 56.20% | 20260506 11:34:57.562 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 5.00593% / 4.37101e-05% / 5.96046e-05% | 20260506 11:34:57.562 | INFO | Baz IO overall: | 20260506 11:34:57.562 | INFO | BytesWritten : 499360132399 | 20260506 11:34:57.563 | INFO | NumEvents : 241002445710 | 20260506 11:34:59.229 | INFO | Analyzing chunk frames = [1036288,1036800) | 20260506 11:34:59.229 | INFO | MemInfo: { | 20260506 11:34:59.229 | INFO | "gpu" : " 17 GB", | 20260506 11:34:59.229 | INFO | "huge" : "", | 20260506 11:34:59.229 | INFO | "huge_shared" : " 27 GB", | 20260506 11:34:59.229 | INFO | "malloc" : " 14 GB", | 20260506 11:34:59.229 | INFO | "pinned" : " 29 GB" | 20260506 11:34:59.229 | INFO | } | 20260506 11:34:59.229 | INFO | Perf report for the last 5.13973 seconds: | 20260506 11:34:59.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:34:59.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:59.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:59.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:59.229 | INFO | ANALYSIS : 63.7%, 2.9, 95.6ms, 0 | 20260506 11:34:59.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:34:59.229 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 11:34:59.229 | INFO | PRE_HQ : 61.9%, 1.9, 61.6ms, 0 | 20260506 11:34:59.229 | INFO | BAZWRITER : 64.4%, 4.8, 158.7ms, 0 | 20260506 11:34:59.229 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 11:34:59.229 | INFO | Uploaded 36GiB to GPU in 2.14026s (16.8204GiB/s) | 20260506 11:34:59.229 | INFO | Downloaded 38GiB to GPU in 2.22498s (17.0788GiB/s) | 20260506 11:35:04.367 | INFO | Analyzing chunk frames = [1036800,1037312) | 20260506 11:35:04.367 | INFO | MemInfo: { | 20260506 11:35:04.367 | INFO | "gpu" : " 17 GB", | 20260506 11:35:04.367 | INFO | "huge" : "", | 20260506 11:35:04.368 | INFO | "huge_shared" : " 27 GB", | 20260506 11:35:04.368 | INFO | "malloc" : " 14 GB", | 20260506 11:35:04.368 | INFO | "pinned" : " 29 GB" | 20260506 11:35:04.368 | INFO | } | 20260506 11:35:04.368 | INFO | Perf report for the last 5.13881 seconds: | 20260506 11:35:04.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:35:04.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:04.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:04.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:04.368 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260506 11:35:04.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:04.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:04.368 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 11:35:04.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:04.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:04.368 | INFO | Uploaded 36GiB to GPU in 2.07068s (17.3856GiB/s) | 20260506 11:35:04.368 | INFO | Downloaded 34GiB to GPU in 1.93872s (17.5373GiB/s) | 20260506 11:35:09.507 | INFO | Analyzing chunk frames = [1037312,1037824) | 20260506 11:35:09.507 | INFO | MemInfo: { | 20260506 11:35:09.507 | INFO | "gpu" : " 17 GB", | 20260506 11:35:09.507 | INFO | "huge" : "", | 20260506 11:35:09.507 | INFO | "huge_shared" : " 27 GB", | 20260506 11:35:09.507 | INFO | "malloc" : " 14 GB", | 20260506 11:35:09.507 | INFO | "pinned" : " 29 GB" | 20260506 11:35:09.507 | INFO | } | 20260506 11:35:09.507 | INFO | Perf report for the last 5.13942 seconds: | 20260506 11:35:09.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:35:09.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:09.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:09.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:09.507 | INFO | ANALYSIS : 62.3%, 3.0, 96.6ms, 0 | 20260506 11:35:09.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:09.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:09.507 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:35:09.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:09.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:09.507 | INFO | Uploaded 36GiB to GPU in 2.05264s (17.5384GiB/s) | 20260506 11:35:09.507 | INFO | Downloaded 34GiB to GPU in 1.95625s (17.3802GiB/s) | 20260506 11:35:14.646 | INFO | Analyzing chunk frames = [1037824,1038336) | 20260506 11:35:14.646 | INFO | MemInfo: { | 20260506 11:35:14.646 | INFO | "gpu" : " 17 GB", | 20260506 11:35:14.646 | INFO | "huge" : "", | 20260506 11:35:14.646 | INFO | "huge_shared" : " 27 GB", | 20260506 11:35:14.646 | INFO | "malloc" : " 14 GB", | 20260506 11:35:14.646 | INFO | "pinned" : " 29 GB" | 20260506 11:35:14.646 | INFO | } | 20260506 11:35:14.646 | INFO | Perf report for the last 5.13936 seconds: | 20260506 11:35:14.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:35:14.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:14.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:14.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:14.646 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 11:35:14.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:14.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:14.646 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:35:14.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:14.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:14.647 | INFO | Uploaded 36GiB to GPU in 2.07052s (17.387GiB/s) | 20260506 11:35:14.647 | INFO | Downloaded 34GiB to GPU in 1.96987s (17.26GiB/s) | 20260506 11:35:19.786 | INFO | Analyzing chunk frames = [1038336,1038848) | 20260506 11:35:19.786 | INFO | MemInfo: { | 20260506 11:35:19.786 | INFO | "gpu" : " 17 GB", | 20260506 11:35:19.786 | INFO | "huge" : "", | 20260506 11:35:19.786 | INFO | "huge_shared" : " 27 GB", | 20260506 11:35:19.786 | INFO | "malloc" : " 14 GB", | 20260506 11:35:19.786 | INFO | "pinned" : " 29 GB" | 20260506 11:35:19.786 | INFO | } | 20260506 11:35:19.786 | INFO | Perf report for the last 5.13931 seconds: | 20260506 11:35:19.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:35:19.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:19.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:19.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:19.786 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 11:35:19.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:19.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:19.786 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:35:19.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:19.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:19.786 | INFO | Uploaded 36GiB to GPU in 2.1005s (17.1388GiB/s) | 20260506 11:35:19.786 | INFO | Downloaded 34GiB to GPU in 1.99151s (17.0725GiB/s) | 20260506 11:35:24.925 | INFO | Analyzing chunk frames = [1038848,1039360) | 20260506 11:35:24.925 | INFO | MemInfo: { | 20260506 11:35:24.925 | INFO | "gpu" : " 17 GB", | 20260506 11:35:24.925 | INFO | "huge" : "", | 20260506 11:35:24.925 | INFO | "huge_shared" : " 27 GB", | 20260506 11:35:24.925 | INFO | "malloc" : " 14 GB", | 20260506 11:35:24.925 | INFO | "pinned" : " 29 GB" | 20260506 11:35:24.925 | INFO | } | 20260506 11:35:24.925 | INFO | Perf report for the last 5.13949 seconds: | 20260506 11:35:24.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:35:24.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:24.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:24.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:24.925 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 11:35:24.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:24.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:24.925 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 11:35:24.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:24.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:24.925 | INFO | Uploaded 36GiB to GPU in 2.03298s (17.708GiB/s) | 20260506 11:35:24.925 | INFO | Downloaded 34GiB to GPU in 1.99098s (17.077GiB/s) | 20260506 11:35:30.065 | INFO | Analyzing chunk frames = [1039360,1039872) | 20260506 11:35:30.065 | INFO | MemInfo: { | 20260506 11:35:30.065 | INFO | "gpu" : " 17 GB", | 20260506 11:35:30.065 | INFO | "huge" : "", | 20260506 11:35:30.065 | INFO | "huge_shared" : " 27 GB", | 20260506 11:35:30.065 | INFO | "malloc" : " 14 GB", | 20260506 11:35:30.065 | INFO | "pinned" : " 29 GB" | 20260506 11:35:30.065 | INFO | } | 20260506 11:35:30.065 | INFO | Perf report for the last 5.14008 seconds: | 20260506 11:35:30.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:35:30.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:30.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:30.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:30.065 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 11:35:30.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:30.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:30.065 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 11:35:30.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:30.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:30.066 | INFO | Uploaded 36GiB to GPU in 2.01423s (17.8729GiB/s) | 20260506 11:35:30.066 | INFO | Downloaded 34GiB to GPU in 1.96845s (17.2725GiB/s) | 20260506 11:35:35.204 | INFO | Analyzing chunk frames = [1039872,1040384) | 20260506 11:35:35.204 | INFO | MemInfo: { | 20260506 11:35:35.204 | INFO | "gpu" : " 17 GB", | 20260506 11:35:35.204 | INFO | "huge" : "", | 20260506 11:35:35.204 | INFO | "huge_shared" : " 27 GB", | 20260506 11:35:35.204 | INFO | "malloc" : " 14 GB", | 20260506 11:35:35.204 | INFO | "pinned" : " 29 GB" | 20260506 11:35:35.204 | INFO | } | 20260506 11:35:35.204 | INFO | Perf report for the last 5.13861 seconds: | 20260506 11:35:35.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:35:35.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:35.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:35.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:35.204 | INFO | ANALYSIS : 62.9%, 3.0, 98.2ms, 0 | 20260506 11:35:35.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:35.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:35.204 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260506 11:35:35.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:35.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:35.204 | INFO | Uploaded 36GiB to GPU in 2.03004s (17.7337GiB/s) | 20260506 11:35:35.204 | INFO | Downloaded 34GiB to GPU in 2.02738s (16.7704GiB/s) | 20260506 11:35:35.204 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:35:35.204 | INFO | Total mem usage: 16.8 GiB | 20260506 11:35:35.204 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:35:35.205 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -6.24%) | 20260506 11:35:35.205 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:35:35.205 | INFO | High Water Breakdown: | 20260506 11:35:35.205 | INFO | 60.9% : 9.64 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:35:35.205 | INFO | 38.9% : 6.16 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:35:35.205 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:35:35.205 | INFO | ------------------------------------------ | 20260506 11:35:35.205 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:35:35.205 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:35:35.205 | INFO | Total mem usage: 26.5 GiB | 20260506 11:35:35.205 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:35:35.205 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:35:35.205 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:35:35.205 | INFO | High Water Breakdown: | 20260506 11:35:35.205 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:35:35.205 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:35:35.205 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:35:35.205 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:35:35.205 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:35:35.205 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:35:35.205 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:35:35.205 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:35:35.205 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:35:35.205 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:35:35.205 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:35:35.205 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:35:35.205 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:35:35.205 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:35:35.205 | INFO | ------------------------------------------ | 20260506 11:35:35.205 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:35:35.205 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:35:35.205 | INFO | Total mem usage: 25.6 GiB | 20260506 11:35:35.205 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:35:35.205 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260506 11:35:35.205 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260506 11:35:35.205 | INFO | High Water Breakdown: | 20260506 11:35:35.205 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260506 11:35:35.205 | INFO | ------------------------------------------ | 20260506 11:35:35.205 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:35:35.205 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:35:35.205 | INFO | Total mem usage: 15.7 GiB | 20260506 11:35:35.205 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:35:35.205 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:35:35.205 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 11:35:35.205 | INFO | High Water Breakdown: | 20260506 11:35:35.205 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:35:35.205 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:35:35.205 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:35:35.205 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:35:35.205 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:35:35.205 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:35:35.205 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:35:35.205 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:35:35.205 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:35:35.205 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:35:35.205 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:35:35.205 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:35:35.205 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:35:35.205 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:35:35.205 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:35:35.205 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:35:35.205 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:35:35.205 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:35:35.205 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:35:35.205 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:35:35.205 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:35:35.205 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:35:35.206 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:35:35.206 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:35:35.206 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:35:35.206 | INFO | ------------------------------------------ | 20260506 11:35:35.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:35:35.206 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:35:35.206 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:35:35.206 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:35:35.206 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.58e+01, 1.36e+01 / 2.31e+00 | 20260506 11:35:35.206 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.05e-02 / 7.46e-03 | 20260506 11:35:35.206 | INFO | PerfReport: 11.55%, Baseline -- 3.69e+00 / 4.36e+00, 5.92e-01 / 4.15e-01 | 20260506 11:35:35.206 | INFO | PerfReport: 3.23%, Histogram -- 1.03e+00 / 1.45e+00, 1.93e-01 / 1.51e-01 | 20260506 11:35:35.206 | INFO | PerfReport: 15.84%, DME -- 4.05e+01 / 4.49e+01, 4.07e+00 / 2.00e+00 | 20260506 11:35:35.206 | INFO | PerfReport: 56.18%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.48e+00 / 1.68e+00 | 20260506 11:35:35.206 | INFO | PerfReport: 7.69%, PulseAccumulating -- 2.46e+00 / 3.07e+00, 4.35e-01 / 2.26e-01 | 20260506 11:35:35.206 | INFO | PerfReport: 3.85%, Metrics -- 1.23e+00 / 1.64e+00, 3.13e-01 / 1.68e-01 | 20260506 11:35:35.206 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.35e-01 / 4.96e-01, 6.81e-02 / 3.17e-02 | 20260506 11:35:35.206 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:35:35.206 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:35:35.206 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:35:35.206 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260506 11:35:35.206 | INFO | PerfReport: 21.64%, Upload -- 2.14e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260506 11:35:35.206 | INFO | PerfReport: 21.14%, Download -- 2.09e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 11:35:35.206 | INFO | PerfReport: 32.31%, Compute -- 3.20e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 11:35:35.206 | INFO | PerfReport: 1.99%, LockUpload -- 1.97e+00 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 11:35:35.206 | INFO | PerfReport: 0.91%, LockDownload -- 8.96e-01 / 0.00e+00, 2.38e+00 / 0.00e+00 | 20260506 11:35:35.206 | INFO | PerfReport: 22.00%, LockCompute -- 2.18e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 11:35:35.206 | INFO | PerfReport: 0.02%, Misc -- 1.50e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260506 11:35:35.327 | INFO | Starting a round of Baz IO | 20260506 11:35:38.531 | INFO | Populating RTMetrics report | 20260506 11:35:38.531 | INFO | RTMetrics report populated | 20260506 11:35:38.531 | INFO | Opening tmp RTMetrics json file | 20260506 11:35:38.533 | INFO | Serializing RTMetrics to json | 20260506 11:35:38.534 | INFO | Serialization done, swapping tmp file now... | 20260506 11:35:38.535 | INFO | RTMetrics done | 20260506 11:35:39.092 | INFO | Finished a round of Baz IO: Wrote 2.065 GiB in 3.764 seconds | 20260506 11:35:39.092 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.73% / 0.03% / 56.24% | 20260506 11:35:39.092 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.9959% / 7.54992e-05% / 3.97364e-05% | 20260506 11:35:39.092 | INFO | Baz IO overall: | 20260506 11:35:39.092 | INFO | BytesWritten : 501577759395 | 20260506 11:35:39.092 | INFO | NumEvents : 242078783521 | 20260506 11:35:40.343 | INFO | Analyzing chunk frames = [1040384,1040896) | 20260506 11:35:40.343 | INFO | MemInfo: { | 20260506 11:35:40.343 | INFO | "gpu" : " 17 GB", | 20260506 11:35:40.343 | INFO | "huge" : "", | 20260506 11:35:40.343 | INFO | "huge_shared" : " 27 GB", | 20260506 11:35:40.343 | INFO | "malloc" : " 14 GB", | 20260506 11:35:40.343 | INFO | "pinned" : " 29 GB" | 20260506 11:35:40.343 | INFO | } | 20260506 11:35:40.343 | INFO | Perf report for the last 5.13912 seconds: | 20260506 11:35:40.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:35:40.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:40.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:40.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:40.343 | INFO | ANALYSIS : 63.8%, 2.8, 95.2ms, 0 | 20260506 11:35:40.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:40.343 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260506 11:35:40.343 | INFO | PRE_HQ : 61.8%, 2.0, 63.4ms, 0 | 20260506 11:35:40.343 | INFO | BAZWRITER : 72.3%, 4.8, 178.4ms, 0 | 20260506 11:35:40.343 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 11:35:40.343 | INFO | Uploaded 36GiB to GPU in 2.06929s (17.3973GiB/s) | 20260506 11:35:40.343 | INFO | Downloaded 38GiB to GPU in 2.20839s (17.2071GiB/s) | 20260506 11:35:45.483 | INFO | Analyzing chunk frames = [1040896,1041408) | 20260506 11:35:45.483 | INFO | MemInfo: { | 20260506 11:35:45.483 | INFO | "gpu" : " 17 GB", | 20260506 11:35:45.483 | INFO | "huge" : "", | 20260506 11:35:45.483 | INFO | "huge_shared" : " 27 GB", | 20260506 11:35:45.483 | INFO | "malloc" : " 14 GB", | 20260506 11:35:45.483 | INFO | "pinned" : " 29 GB" | 20260506 11:35:45.483 | INFO | } | 20260506 11:35:45.483 | INFO | Perf report for the last 5.13974 seconds: | 20260506 11:35:45.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:35:45.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:45.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:45.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:45.483 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:35:45.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:45.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:45.483 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260506 11:35:45.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:45.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:45.483 | INFO | Uploaded 36GiB to GPU in 1.99854s (18.0132GiB/s) | 20260506 11:35:45.483 | INFO | Downloaded 34GiB to GPU in 1.92047s (17.704GiB/s) | 20260506 11:35:50.622 | INFO | Analyzing chunk frames = [1041408,1041920) | 20260506 11:35:50.622 | INFO | MemInfo: { | 20260506 11:35:50.622 | INFO | "gpu" : " 17 GB", | 20260506 11:35:50.622 | INFO | "huge" : "", | 20260506 11:35:50.622 | INFO | "huge_shared" : " 27 GB", | 20260506 11:35:50.622 | INFO | "malloc" : " 14 GB", | 20260506 11:35:50.622 | INFO | "pinned" : " 29 GB" | 20260506 11:35:50.622 | INFO | } | 20260506 11:35:50.622 | INFO | Perf report for the last 5.13932 seconds: | 20260506 11:35:50.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:35:50.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:50.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:50.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:50.622 | INFO | ANALYSIS : 62.3%, 3.0, 96.5ms, 0 | 20260506 11:35:50.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:50.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:50.622 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:35:50.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:50.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:50.623 | INFO | Uploaded 36GiB to GPU in 2.03795s (17.6648GiB/s) | 20260506 11:35:50.623 | INFO | Downloaded 34GiB to GPU in 1.94894s (17.4454GiB/s) | 20260506 11:35:55.761 | INFO | Analyzing chunk frames = [1041920,1042432) | 20260506 11:35:55.761 | INFO | MemInfo: { | 20260506 11:35:55.761 | INFO | "gpu" : " 17 GB", | 20260506 11:35:55.761 | INFO | "huge" : "", | 20260506 11:35:55.761 | INFO | "huge_shared" : " 27 GB", | 20260506 11:35:55.761 | INFO | "malloc" : " 14 GB", | 20260506 11:35:55.761 | INFO | "pinned" : " 29 GB" | 20260506 11:35:55.761 | INFO | } | 20260506 11:35:55.761 | INFO | Perf report for the last 5.1391 seconds: | 20260506 11:35:55.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:35:55.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:55.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:55.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:55.761 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 11:35:55.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:55.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:55.761 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:35:55.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:55.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:35:55.762 | INFO | Uploaded 36GiB to GPU in 2.07986s (17.3088GiB/s) | 20260506 11:35:55.762 | INFO | Downloaded 34GiB to GPU in 1.96976s (17.261GiB/s) | 20260506 11:36:00.901 | INFO | Analyzing chunk frames = [1042432,1042944) | 20260506 11:36:00.901 | INFO | MemInfo: { | 20260506 11:36:00.901 | INFO | "gpu" : " 17 GB", | 20260506 11:36:00.901 | INFO | "huge" : "", | 20260506 11:36:00.901 | INFO | "huge_shared" : " 27 GB", | 20260506 11:36:00.901 | INFO | "malloc" : " 14 GB", | 20260506 11:36:00.901 | INFO | "pinned" : " 29 GB" | 20260506 11:36:00.901 | INFO | } | 20260506 11:36:00.901 | INFO | Perf report for the last 5.1395 seconds: | 20260506 11:36:00.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:36:00.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:00.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:00.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:00.901 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 11:36:00.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:00.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:00.901 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:36:00.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:00.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:00.901 | INFO | Uploaded 36GiB to GPU in 2.09358s (17.1954GiB/s) | 20260506 11:36:00.901 | INFO | Downloaded 34GiB to GPU in 1.98617s (17.1184GiB/s) | 20260506 11:36:06.040 | INFO | Analyzing chunk frames = [1042944,1043456) | 20260506 11:36:06.040 | INFO | MemInfo: { | 20260506 11:36:06.040 | INFO | "gpu" : " 17 GB", | 20260506 11:36:06.040 | INFO | "huge" : "", | 20260506 11:36:06.040 | INFO | "huge_shared" : " 27 GB", | 20260506 11:36:06.040 | INFO | "malloc" : " 14 GB", | 20260506 11:36:06.040 | INFO | "pinned" : " 29 GB" | 20260506 11:36:06.040 | INFO | } | 20260506 11:36:06.040 | INFO | Perf report for the last 5.13909 seconds: | 20260506 11:36:06.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:36:06.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:06.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:06.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:06.040 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 11:36:06.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:06.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:06.040 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:36:06.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:06.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:06.040 | INFO | Uploaded 36GiB to GPU in 2.03117s (17.7237GiB/s) | 20260506 11:36:06.040 | INFO | Downloaded 34GiB to GPU in 2.00646s (16.9453GiB/s) | 20260506 11:36:11.180 | INFO | Analyzing chunk frames = [1043456,1043968) | 20260506 11:36:11.180 | INFO | MemInfo: { | 20260506 11:36:11.180 | INFO | "gpu" : " 17 GB", | 20260506 11:36:11.181 | INFO | "huge" : "", | 20260506 11:36:11.181 | INFO | "huge_shared" : " 27 GB", | 20260506 11:36:11.181 | INFO | "malloc" : " 14 GB", | 20260506 11:36:11.181 | INFO | "pinned" : " 29 GB" | 20260506 11:36:11.181 | INFO | } | 20260506 11:36:11.181 | INFO | Perf report for the last 5.14072 seconds: | 20260506 11:36:11.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:36:11.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:11.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:11.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:11.181 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:36:11.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:11.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:11.181 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:36:11.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:11.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:11.181 | INFO | Uploaded 36GiB to GPU in 2.0035s (17.9686GiB/s) | 20260506 11:36:11.181 | INFO | Downloaded 34GiB to GPU in 1.99431s (17.0485GiB/s) | 20260506 11:36:16.319 | INFO | Analyzing chunk frames = [1043968,1044480) | 20260506 11:36:16.319 | INFO | MemInfo: { | 20260506 11:36:16.319 | INFO | "gpu" : " 17 GB", | 20260506 11:36:16.319 | INFO | "huge" : "", | 20260506 11:36:16.319 | INFO | "huge_shared" : " 27 GB", | 20260506 11:36:16.319 | INFO | "malloc" : " 14 GB", | 20260506 11:36:16.319 | INFO | "pinned" : " 29 GB" | 20260506 11:36:16.319 | INFO | } | 20260506 11:36:16.319 | INFO | Perf report for the last 5.13811 seconds: | 20260506 11:36:16.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:36:16.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:16.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:16.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:16.319 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260506 11:36:16.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:16.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:16.319 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260506 11:36:16.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:16.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:16.319 | INFO | Uploaded 36GiB to GPU in 2.01612s (17.8561GiB/s) | 20260506 11:36:16.319 | INFO | Downloaded 34GiB to GPU in 1.99319s (17.0581GiB/s) | 20260506 11:36:16.453 | INFO | Starting a round of Baz IO | 20260506 11:36:19.623 | INFO | Populating RTMetrics report | 20260506 11:36:19.623 | INFO | RTMetrics report populated | 20260506 11:36:19.623 | INFO | Opening tmp RTMetrics json file | 20260506 11:36:19.626 | INFO | Serializing RTMetrics to json | 20260506 11:36:19.628 | INFO | Serialization done, swapping tmp file now... | 20260506 11:36:19.629 | INFO | RTMetrics done | 20260506 11:36:19.770 | INFO | Finished a round of Baz IO: Wrote 2.068 GiB in 3.316 seconds | 20260506 11:36:19.770 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.70% / 0.03% / 56.27% | 20260506 11:36:19.770 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.99213% / 3.97364e-05% / 6.75519e-05% | 20260506 11:36:19.770 | INFO | Baz IO overall: | 20260506 11:36:19.770 | INFO | BytesWritten : 503797813786 | 20260506 11:36:19.770 | INFO | NumEvents : 243157022069 | 20260506 11:36:21.458 | INFO | Analyzing chunk frames = [1044480,1044992) | 20260506 11:36:21.458 | INFO | MemInfo: { | 20260506 11:36:21.458 | INFO | "gpu" : " 17 GB", | 20260506 11:36:21.458 | INFO | "huge" : "", | 20260506 11:36:21.458 | INFO | "huge_shared" : " 27 GB", | 20260506 11:36:21.458 | INFO | "malloc" : " 14 GB", | 20260506 11:36:21.458 | INFO | "pinned" : " 29 GB" | 20260506 11:36:21.458 | INFO | } | 20260506 11:36:21.458 | INFO | Perf report for the last 5.13918 seconds: | 20260506 11:36:21.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:36:21.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:21.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:21.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:21.458 | INFO | ANALYSIS : 63.7%, 2.9, 96.3ms, 0 | 20260506 11:36:21.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:21.458 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 11:36:21.458 | INFO | PRE_HQ : 62.1%, 1.9, 61.5ms, 0 | 20260506 11:36:21.458 | INFO | BAZWRITER : 64.4%, 4.3, 143.5ms, 0 | 20260506 11:36:21.458 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 11:36:21.458 | INFO | Uploaded 36GiB to GPU in 2.13138s (16.8904GiB/s) | 20260506 11:36:21.458 | INFO | Downloaded 38GiB to GPU in 2.23016s (17.0391GiB/s) | 20260506 11:36:26.597 | INFO | Analyzing chunk frames = [1044992,1045504) | 20260506 11:36:26.597 | INFO | MemInfo: { | 20260506 11:36:26.597 | INFO | "gpu" : " 17 GB", | 20260506 11:36:26.597 | INFO | "huge" : "", | 20260506 11:36:26.597 | INFO | "huge_shared" : " 27 GB", | 20260506 11:36:26.597 | INFO | "malloc" : " 14 GB", | 20260506 11:36:26.597 | INFO | "pinned" : " 29 GB" | 20260506 11:36:26.597 | INFO | } | 20260506 11:36:26.597 | INFO | Perf report for the last 5.13928 seconds: | 20260506 11:36:26.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:36:26.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:26.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:26.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:26.597 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:36:26.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:26.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:26.597 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:36:26.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:26.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:26.597 | INFO | Uploaded 36GiB to GPU in 2.0187s (17.8333GiB/s) | 20260506 11:36:26.598 | INFO | Downloaded 34GiB to GPU in 1.92658s (17.6479GiB/s) | 20260506 11:36:31.737 | INFO | Analyzing chunk frames = [1045504,1046016) | 20260506 11:36:31.737 | INFO | MemInfo: { | 20260506 11:36:31.737 | INFO | "gpu" : " 17 GB", | 20260506 11:36:31.737 | INFO | "huge" : "", | 20260506 11:36:31.737 | INFO | "huge_shared" : " 27 GB", | 20260506 11:36:31.737 | INFO | "malloc" : " 14 GB", | 20260506 11:36:31.737 | INFO | "pinned" : " 29 GB" | 20260506 11:36:31.737 | INFO | } | 20260506 11:36:31.737 | INFO | Perf report for the last 5.13953 seconds: | 20260506 11:36:31.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:36:31.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:31.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:31.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:31.737 | INFO | ANALYSIS : 62.0%, 3.0, 96.0ms, 0 | 20260506 11:36:31.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:31.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:31.737 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:36:31.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:31.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:31.737 | INFO | Uploaded 36GiB to GPU in 2.02058s (17.8167GiB/s) | 20260506 11:36:31.737 | INFO | Downloaded 34GiB to GPU in 1.932s (17.5983GiB/s) | 20260506 11:36:36.876 | INFO | Analyzing chunk frames = [1046016,1046528) | 20260506 11:36:36.876 | INFO | MemInfo: { | 20260506 11:36:36.876 | INFO | "gpu" : " 17 GB", | 20260506 11:36:36.876 | INFO | "huge" : "", | 20260506 11:36:36.876 | INFO | "huge_shared" : " 27 GB", | 20260506 11:36:36.876 | INFO | "malloc" : " 14 GB", | 20260506 11:36:36.876 | INFO | "pinned" : " 29 GB" | 20260506 11:36:36.876 | INFO | } | 20260506 11:36:36.877 | INFO | Perf report for the last 5.13973 seconds: | 20260506 11:36:36.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:36:36.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:36.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:36.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:36.877 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 11:36:36.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:36.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:36.877 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:36:36.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:36.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:36.877 | INFO | Uploaded 36GiB to GPU in 2.06035s (17.4728GiB/s) | 20260506 11:36:36.877 | INFO | Downloaded 34GiB to GPU in 1.97286s (17.2339GiB/s) | 20260506 11:36:42.015 | INFO | Analyzing chunk frames = [1046528,1047040) | 20260506 11:36:42.015 | INFO | MemInfo: { | 20260506 11:36:42.015 | INFO | "gpu" : " 17 GB", | 20260506 11:36:42.015 | INFO | "huge" : "", | 20260506 11:36:42.015 | INFO | "huge_shared" : " 27 GB", | 20260506 11:36:42.015 | INFO | "malloc" : " 14 GB", | 20260506 11:36:42.015 | INFO | "pinned" : " 29 GB" | 20260506 11:36:42.015 | INFO | } | 20260506 11:36:42.015 | INFO | Perf report for the last 5.13874 seconds: | 20260506 11:36:42.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:36:42.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:42.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:42.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:42.015 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 11:36:42.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:42.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:42.015 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:36:42.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:42.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:42.015 | INFO | Uploaded 36GiB to GPU in 2.08102s (17.2992GiB/s) | 20260506 11:36:42.016 | INFO | Downloaded 34GiB to GPU in 1.99331s (17.057GiB/s) | 20260506 11:36:47.155 | INFO | Analyzing chunk frames = [1047040,1047552) | 20260506 11:36:47.155 | INFO | MemInfo: { | 20260506 11:36:47.155 | INFO | "gpu" : " 17 GB", | 20260506 11:36:47.155 | INFO | "huge" : "", | 20260506 11:36:47.155 | INFO | "huge_shared" : " 27 GB", | 20260506 11:36:47.155 | INFO | "malloc" : " 14 GB", | 20260506 11:36:47.155 | INFO | "pinned" : " 29 GB" | 20260506 11:36:47.155 | INFO | } | 20260506 11:36:47.155 | INFO | Perf report for the last 5.13958 seconds: | 20260506 11:36:47.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:36:47.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:47.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:47.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:47.155 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 11:36:47.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:47.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:47.155 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:36:47.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:47.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:47.155 | INFO | Uploaded 36GiB to GPU in 2.0144s (17.8714GiB/s) | 20260506 11:36:47.155 | INFO | Downloaded 34GiB to GPU in 2.00178s (16.9848GiB/s) | 20260506 11:36:52.295 | INFO | Analyzing chunk frames = [1047552,1048064) | 20260506 11:36:52.295 | INFO | MemInfo: { | 20260506 11:36:52.295 | INFO | "gpu" : " 17 GB", | 20260506 11:36:52.295 | INFO | "huge" : "", | 20260506 11:36:52.295 | INFO | "huge_shared" : " 27 GB", | 20260506 11:36:52.295 | INFO | "malloc" : " 14 GB", | 20260506 11:36:52.295 | INFO | "pinned" : " 29 GB" | 20260506 11:36:52.295 | INFO | } | 20260506 11:36:52.295 | INFO | Perf report for the last 5.1399 seconds: | 20260506 11:36:52.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:36:52.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:52.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:52.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:52.295 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:36:52.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:52.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:52.295 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 11:36:52.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:52.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:52.295 | INFO | Uploaded 36GiB to GPU in 2.01117s (17.9GiB/s) | 20260506 11:36:52.295 | INFO | Downloaded 34GiB to GPU in 2.01671s (16.8592GiB/s) | 20260506 11:36:57.433 | INFO | Analyzing chunk frames = [1048064,1048576) | 20260506 11:36:57.433 | INFO | MemInfo: { | 20260506 11:36:57.433 | INFO | "gpu" : " 17 GB", | 20260506 11:36:57.433 | INFO | "huge" : "", | 20260506 11:36:57.433 | INFO | "huge_shared" : " 27 GB", | 20260506 11:36:57.433 | INFO | "malloc" : " 14 GB", | 20260506 11:36:57.433 | INFO | "pinned" : " 29 GB" | 20260506 11:36:57.433 | INFO | } | 20260506 11:36:57.433 | INFO | Perf report for the last 5.13858 seconds: | 20260506 11:36:57.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:36:57.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:57.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:57.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:57.433 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260506 11:36:57.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:57.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:57.433 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260506 11:36:57.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:57.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:36:57.434 | INFO | Uploaded 36GiB to GPU in 2.00876s (17.9215GiB/s) | 20260506 11:36:57.434 | INFO | Downloaded 34GiB to GPU in 1.97477s (17.2172GiB/s) | 20260506 11:36:57.434 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:36:57.434 | INFO | Total mem usage: 16.8 GiB | 20260506 11:36:57.434 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:36:57.434 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -6.99%) | 20260506 11:36:57.434 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:36:57.434 | INFO | High Water Breakdown: | 20260506 11:36:57.434 | INFO | 60.8% : 9.56 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:36:57.434 | INFO | 39.1% : 6.14 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:36:57.434 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:36:57.434 | INFO | ------------------------------------------ | 20260506 11:36:57.434 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:36:57.434 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:36:57.434 | INFO | Total mem usage: 26.5 GiB | 20260506 11:36:57.434 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:36:57.434 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:36:57.434 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:36:57.434 | INFO | High Water Breakdown: | 20260506 11:36:57.434 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:36:57.434 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:36:57.434 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:36:57.434 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:36:57.434 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:36:57.434 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:36:57.434 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:36:57.434 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:36:57.434 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:36:57.434 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:36:57.434 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:36:57.434 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:36:57.434 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:36:57.434 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:36:57.434 | INFO | ------------------------------------------ | 20260506 11:36:57.434 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:36:57.434 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:36:57.434 | INFO | Total mem usage: 25.6 GiB | 20260506 11:36:57.434 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:36:57.434 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:36:57.434 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:36:57.434 | INFO | High Water Breakdown: | 20260506 11:36:57.434 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:36:57.434 | INFO | ------------------------------------------ | 20260506 11:36:57.434 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:36:57.434 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:36:57.434 | INFO | Total mem usage: 15.7 GiB | 20260506 11:36:57.434 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:36:57.434 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:36:57.434 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:36:57.434 | INFO | High Water Breakdown: | 20260506 11:36:57.434 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:36:57.434 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:36:57.434 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:36:57.434 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:36:57.434 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:36:57.434 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:36:57.434 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:36:57.434 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:36:57.434 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:36:57.434 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:36:57.434 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:36:57.434 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:36:57.434 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:36:57.434 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:36:57.434 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:36:57.434 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:36:57.434 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:36:57.434 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:36:57.434 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:36:57.435 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:36:57.435 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:36:57.435 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:36:57.435 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:36:57.435 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:36:57.435 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:36:57.435 | INFO | ------------------------------------------ | 20260506 11:36:57.435 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:36:57.435 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:36:57.435 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:36:57.435 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:36:57.435 | INFO | PerfReport: Full Execution -- 3.20e+01 / 7.64e+01, 1.37e+01 / 2.42e+00 | 20260506 11:36:57.435 | INFO | PerfReport: 0.29%, AggregateStats -- 9.30e-02 / 1.01e-01, 1.17e-02 / 7.67e-03 | 20260506 11:36:57.435 | INFO | PerfReport: 11.52%, Baseline -- 3.68e+00 / 4.43e+00, 5.83e-01 / 3.35e-01 | 20260506 11:36:57.435 | INFO | PerfReport: 3.24%, Histogram -- 1.04e+00 / 1.49e+00, 1.96e-01 / 1.33e-01 | 20260506 11:36:57.435 | INFO | PerfReport: 15.87%, DME -- 4.06e+01 / 4.53e+01, 4.18e+00 / 2.44e+00 | 20260506 11:36:57.435 | INFO | PerfReport: 56.21%, FrameLabeling -- 1.80e+01 / 1.98e+01, 1.47e+00 / 1.73e+00 | 20260506 11:36:57.435 | INFO | PerfReport: 7.67%, PulseAccumulating -- 2.45e+00 / 3.11e+00, 4.30e-01 / 2.13e-01 | 20260506 11:36:57.435 | INFO | PerfReport: 3.84%, Metrics -- 1.23e+00 / 1.62e+00, 3.08e-01 / 1.74e-01 | 20260506 11:36:57.435 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.34e-01 / 4.87e-01, 6.73e-02 / 2.85e-02 | 20260506 11:36:57.435 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:36:57.435 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:36:57.435 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:36:57.435 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260506 11:36:57.435 | INFO | PerfReport: 21.53%, Upload -- 2.13e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 11:36:57.435 | INFO | PerfReport: 21.13%, Download -- 2.09e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 11:36:57.435 | INFO | PerfReport: 32.34%, Compute -- 3.20e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 11:36:57.435 | INFO | PerfReport: 1.84%, LockUpload -- 1.82e+00 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 11:36:57.435 | INFO | PerfReport: 1.02%, LockDownload -- 1.01e+00 / 0.00e+00, 2.61e+00 / 0.00e+00 | 20260506 11:36:57.435 | INFO | PerfReport: 22.12%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 11:36:57.435 | INFO | PerfReport: 0.02%, Misc -- 1.59e-02 / 0.00e+00, 2.40e-02 / 0.00e+00 | 20260506 11:36:57.572 | INFO | Starting a round of Baz IO | 20260506 11:37:00.760 | INFO | Populating RTMetrics report | 20260506 11:37:00.761 | INFO | RTMetrics report populated | 20260506 11:37:00.761 | INFO | Opening tmp RTMetrics json file | 20260506 11:37:00.889 | INFO | Serializing RTMetrics to json | 20260506 11:37:00.894 | INFO | Serialization done, swapping tmp file now... | 20260506 11:37:00.894 | INFO | RTMetrics done | 20260506 11:37:01.212 | INFO | Finished a round of Baz IO: Wrote 2.065 GiB in 3.639 seconds | 20260506 11:37:01.212 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.67% / 0.03% / 56.30% | 20260506 11:37:01.212 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.95553% / 5.16574e-05% / 5.16574e-05% | 20260506 11:37:01.212 | INFO | Baz IO overall: | 20260506 11:37:01.212 | INFO | BytesWritten : 506015407283 | 20260506 11:37:01.212 | INFO | NumEvents : 244232836111 | 20260506 11:37:02.573 | INFO | Analyzing chunk frames = [1048576,1049088) | 20260506 11:37:02.573 | INFO | MemInfo: { | 20260506 11:37:02.573 | INFO | "gpu" : " 17 GB", | 20260506 11:37:02.573 | INFO | "huge" : "", | 20260506 11:37:02.573 | INFO | "huge_shared" : " 27 GB", | 20260506 11:37:02.573 | INFO | "malloc" : " 14 GB", | 20260506 11:37:02.573 | INFO | "pinned" : " 29 GB" | 20260506 11:37:02.573 | INFO | } | 20260506 11:37:02.573 | INFO | Perf report for the last 5.13956 seconds: | 20260506 11:37:02.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:37:02.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:02.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:02.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:02.573 | INFO | ANALYSIS : 64.0%, 2.9, 96.2ms, 0 | 20260506 11:37:02.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:02.573 | INFO | RT_METRICS : 47.9%, 1.0, 25.1ms, 0 | 20260506 11:37:02.573 | INFO | PRE_HQ : 62.2%, 2.0, 63.1ms, 0 | 20260506 11:37:02.573 | INFO | BAZWRITER : 70.0%, 4.8, 171.6ms, 0 | 20260506 11:37:02.573 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.2ms, 0 | 20260506 11:37:02.573 | INFO | Uploaded 36GiB to GPU in 2.1224s (16.9619GiB/s) | 20260506 11:37:02.573 | INFO | Downloaded 38GiB to GPU in 2.23409s (17.0091GiB/s) | 20260506 11:37:07.712 | INFO | Analyzing chunk frames = [1049088,1049600) | 20260506 11:37:07.712 | INFO | MemInfo: { | 20260506 11:37:07.712 | INFO | "gpu" : " 17 GB", | 20260506 11:37:07.712 | INFO | "huge" : "", | 20260506 11:37:07.712 | INFO | "huge_shared" : " 27 GB", | 20260506 11:37:07.712 | INFO | "malloc" : " 14 GB", | 20260506 11:37:07.712 | INFO | "pinned" : " 29 GB" | 20260506 11:37:07.712 | INFO | } | 20260506 11:37:07.712 | INFO | Perf report for the last 5.13918 seconds: | 20260506 11:37:07.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:37:07.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:07.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:07.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:07.712 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:37:07.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:07.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:07.712 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260506 11:37:07.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:07.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:07.712 | INFO | Uploaded 36GiB to GPU in 2.0133s (17.8811GiB/s) | 20260506 11:37:07.712 | INFO | Downloaded 34GiB to GPU in 1.92337s (17.6773GiB/s) | 20260506 11:37:12.851 | INFO | Analyzing chunk frames = [1049600,1050112) | 20260506 11:37:12.851 | INFO | MemInfo: { | 20260506 11:37:12.851 | INFO | "gpu" : " 17 GB", | 20260506 11:37:12.851 | INFO | "huge" : "", | 20260506 11:37:12.852 | INFO | "huge_shared" : " 27 GB", | 20260506 11:37:12.852 | INFO | "malloc" : " 14 GB", | 20260506 11:37:12.852 | INFO | "pinned" : " 29 GB" | 20260506 11:37:12.852 | INFO | } | 20260506 11:37:12.852 | INFO | Perf report for the last 5.13944 seconds: | 20260506 11:37:12.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:37:12.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:12.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:12.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:12.852 | INFO | ANALYSIS : 62.3%, 3.0, 96.5ms, 0 | 20260506 11:37:12.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:12.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:12.852 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:37:12.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:12.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:12.852 | INFO | Uploaded 36GiB to GPU in 2.01862s (17.8339GiB/s) | 20260506 11:37:12.852 | INFO | Downloaded 34GiB to GPU in 1.95142s (17.4232GiB/s) | 20260506 11:37:17.991 | INFO | Analyzing chunk frames = [1050112,1050624) | 20260506 11:37:17.991 | INFO | MemInfo: { | 20260506 11:37:17.991 | INFO | "gpu" : " 17 GB", | 20260506 11:37:17.991 | INFO | "huge" : "", | 20260506 11:37:17.991 | INFO | "huge_shared" : " 27 GB", | 20260506 11:37:17.991 | INFO | "malloc" : " 14 GB", | 20260506 11:37:17.991 | INFO | "pinned" : " 29 GB" | 20260506 11:37:17.991 | INFO | } | 20260506 11:37:17.991 | INFO | Perf report for the last 5.13913 seconds: | 20260506 11:37:17.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:37:17.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:17.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:17.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:17.991 | INFO | ANALYSIS : 61.4%, 3.0, 95.2ms, 0 | 20260506 11:37:17.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:17.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:17.991 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:37:17.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:17.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:17.991 | INFO | Uploaded 36GiB to GPU in 2.01057s (17.9054GiB/s) | 20260506 11:37:17.991 | INFO | Downloaded 34GiB to GPU in 1.95901s (17.3557GiB/s) | 20260506 11:37:23.130 | INFO | Analyzing chunk frames = [1050624,1051136) | 20260506 11:37:23.130 | INFO | MemInfo: { | 20260506 11:37:23.130 | INFO | "gpu" : " 17 GB", | 20260506 11:37:23.130 | INFO | "huge" : "", | 20260506 11:37:23.130 | INFO | "huge_shared" : " 27 GB", | 20260506 11:37:23.130 | INFO | "malloc" : " 14 GB", | 20260506 11:37:23.130 | INFO | "pinned" : " 29 GB" | 20260506 11:37:23.130 | INFO | } | 20260506 11:37:23.130 | INFO | Perf report for the last 5.13941 seconds: | 20260506 11:37:23.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:37:23.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:23.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:23.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:23.130 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260506 11:37:23.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:23.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:23.130 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:37:23.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:23.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:23.130 | INFO | Uploaded 36GiB to GPU in 1.98014s (18.1805GiB/s) | 20260506 11:37:23.130 | INFO | Downloaded 34GiB to GPU in 1.9713s (17.2475GiB/s) | 20260506 11:37:28.269 | INFO | Analyzing chunk frames = [1051136,1051648) | 20260506 11:37:28.269 | INFO | MemInfo: { | 20260506 11:37:28.269 | INFO | "gpu" : " 17 GB", | 20260506 11:37:28.269 | INFO | "huge" : "", | 20260506 11:37:28.269 | INFO | "huge_shared" : " 27 GB", | 20260506 11:37:28.269 | INFO | "malloc" : " 14 GB", | 20260506 11:37:28.269 | INFO | "pinned" : " 29 GB" | 20260506 11:37:28.269 | INFO | } | 20260506 11:37:28.269 | INFO | Perf report for the last 5.13931 seconds: | 20260506 11:37:28.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:37:28.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:28.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:28.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:28.269 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 11:37:28.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:28.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:28.270 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:37:28.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:28.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:28.270 | INFO | Uploaded 36GiB to GPU in 2.08306s (17.2822GiB/s) | 20260506 11:37:28.270 | INFO | Downloaded 34GiB to GPU in 1.98934s (17.0911GiB/s) | 20260506 11:37:33.409 | INFO | Analyzing chunk frames = [1051648,1052160) | 20260506 11:37:33.409 | INFO | MemInfo: { | 20260506 11:37:33.409 | INFO | "gpu" : " 17 GB", | 20260506 11:37:33.409 | INFO | "huge" : "", | 20260506 11:37:33.409 | INFO | "huge_shared" : " 27 GB", | 20260506 11:37:33.409 | INFO | "malloc" : " 14 GB", | 20260506 11:37:33.409 | INFO | "pinned" : " 29 GB" | 20260506 11:37:33.409 | INFO | } | 20260506 11:37:33.409 | INFO | Perf report for the last 5.14 seconds: | 20260506 11:37:33.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:37:33.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:33.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:33.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:33.410 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 11:37:33.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:33.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:33.410 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 11:37:33.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:33.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:33.410 | INFO | Uploaded 36GiB to GPU in 2.02868s (17.7455GiB/s) | 20260506 11:37:33.410 | INFO | Downloaded 34GiB to GPU in 2.01107s (16.9064GiB/s) | 20260506 11:37:38.548 | INFO | Analyzing chunk frames = [1052160,1052672) | 20260506 11:37:38.548 | INFO | MemInfo: { | 20260506 11:37:38.548 | INFO | "gpu" : " 17 GB", | 20260506 11:37:38.548 | INFO | "huge" : "", | 20260506 11:37:38.548 | INFO | "huge_shared" : " 27 GB", | 20260506 11:37:38.548 | INFO | "malloc" : " 14 GB", | 20260506 11:37:38.548 | INFO | "pinned" : " 29 GB" | 20260506 11:37:38.548 | INFO | } | 20260506 11:37:38.548 | INFO | Perf report for the last 5.13874 seconds: | 20260506 11:37:38.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:37:38.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:38.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:38.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:38.548 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260506 11:37:38.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:38.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:38.548 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 11:37:38.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:38.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:38.548 | INFO | Uploaded 36GiB to GPU in 2.06783s (17.4096GiB/s) | 20260506 11:37:38.548 | INFO | Downloaded 34GiB to GPU in 1.9701s (17.258GiB/s) | 20260506 11:37:38.667 | INFO | Starting a round of Baz IO | 20260506 11:37:41.890 | INFO | Populating RTMetrics report | 20260506 11:37:41.890 | INFO | RTMetrics report populated | 20260506 11:37:41.890 | INFO | Opening tmp RTMetrics json file | 20260506 11:37:41.905 | INFO | Serializing RTMetrics to json | 20260506 11:37:41.937 | INFO | Serialization done, swapping tmp file now... | 20260506 11:37:41.941 | INFO | RTMetrics done | 20260506 11:37:42.174 | INFO | Finished a round of Baz IO: Wrote 2.067 GiB in 3.506 seconds | 20260506 11:37:42.174 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.63% / 0.03% / 56.33% | 20260506 11:37:42.174 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.9471% / 6.75519e-05% / 3.97364e-05% | 20260506 11:37:42.174 | INFO | Baz IO overall: | 20260506 11:37:42.174 | INFO | BytesWritten : 508234585173 | 20260506 11:37:42.174 | INFO | NumEvents : 245309618557 | 20260506 11:37:43.687 | INFO | Analyzing chunk frames = [1052672,1053184) | 20260506 11:37:43.687 | INFO | MemInfo: { | 20260506 11:37:43.687 | INFO | "gpu" : " 17 GB", | 20260506 11:37:43.687 | INFO | "huge" : "", | 20260506 11:37:43.688 | INFO | "huge_shared" : " 27 GB", | 20260506 11:37:43.688 | INFO | "malloc" : " 14 GB", | 20260506 11:37:43.688 | INFO | "pinned" : " 29 GB" | 20260506 11:37:43.688 | INFO | } | 20260506 11:37:43.688 | INFO | Perf report for the last 5.13938 seconds: | 20260506 11:37:43.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:37:43.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:43.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:43.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:43.688 | INFO | ANALYSIS : 64.0%, 2.9, 95.8ms, 0 | 20260506 11:37:43.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:43.688 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260506 11:37:43.688 | INFO | PRE_HQ : 62.1%, 2.0, 63.6ms, 0 | 20260506 11:37:43.688 | INFO | BAZWRITER : 67.4%, 4.8, 165.6ms, 0 | 20260506 11:37:43.688 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.6ms, 0 | 20260506 11:37:43.688 | INFO | Uploaded 36GiB to GPU in 2.10967s (17.0643GiB/s) | 20260506 11:37:43.688 | INFO | Downloaded 38GiB to GPU in 2.20647s (17.2221GiB/s) | 20260506 11:37:48.827 | INFO | Analyzing chunk frames = [1053184,1053696) | 20260506 11:37:48.827 | INFO | MemInfo: { | 20260506 11:37:48.827 | INFO | "gpu" : " 17 GB", | 20260506 11:37:48.827 | INFO | "huge" : "", | 20260506 11:37:48.827 | INFO | "huge_shared" : " 27 GB", | 20260506 11:37:48.827 | INFO | "malloc" : " 14 GB", | 20260506 11:37:48.827 | INFO | "pinned" : " 29 GB" | 20260506 11:37:48.827 | INFO | } | 20260506 11:37:48.827 | INFO | Perf report for the last 5.13968 seconds: | 20260506 11:37:48.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:37:48.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:48.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:48.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:48.827 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:37:48.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:48.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:48.827 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 11:37:48.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:48.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:48.828 | INFO | Uploaded 36GiB to GPU in 2.0131s (17.8828GiB/s) | 20260506 11:37:48.828 | INFO | Downloaded 34GiB to GPU in 1.90654s (17.8333GiB/s) | 20260506 11:37:53.966 | INFO | Analyzing chunk frames = [1053696,1054208) | 20260506 11:37:53.966 | INFO | MemInfo: { | 20260506 11:37:53.966 | INFO | "gpu" : " 17 GB", | 20260506 11:37:53.966 | INFO | "huge" : "", | 20260506 11:37:53.966 | INFO | "huge_shared" : " 27 GB", | 20260506 11:37:53.966 | INFO | "malloc" : " 14 GB", | 20260506 11:37:53.966 | INFO | "pinned" : " 29 GB" | 20260506 11:37:53.966 | INFO | } | 20260506 11:37:53.966 | INFO | Perf report for the last 5.13891 seconds: | 20260506 11:37:53.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:37:53.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:53.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:53.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:53.966 | INFO | ANALYSIS : 62.3%, 3.0, 96.4ms, 0 | 20260506 11:37:53.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:53.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:53.966 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:37:53.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:53.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:53.966 | INFO | Uploaded 36GiB to GPU in 2.00143s (17.9871GiB/s) | 20260506 11:37:53.967 | INFO | Downloaded 34GiB to GPU in 1.94328s (17.4962GiB/s) | 20260506 11:37:59.106 | INFO | Analyzing chunk frames = [1054208,1054720) | 20260506 11:37:59.106 | INFO | MemInfo: { | 20260506 11:37:59.106 | INFO | "gpu" : " 17 GB", | 20260506 11:37:59.106 | INFO | "huge" : "", | 20260506 11:37:59.106 | INFO | "huge_shared" : " 27 GB", | 20260506 11:37:59.106 | INFO | "malloc" : " 14 GB", | 20260506 11:37:59.106 | INFO | "pinned" : " 29 GB" | 20260506 11:37:59.106 | INFO | } | 20260506 11:37:59.106 | INFO | Perf report for the last 5.13958 seconds: | 20260506 11:37:59.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:37:59.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:59.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:59.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:59.106 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 11:37:59.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:59.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:59.106 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:37:59.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:59.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:37:59.106 | INFO | Uploaded 36GiB to GPU in 2.08545s (17.2625GiB/s) | 20260506 11:37:59.106 | INFO | Downloaded 34GiB to GPU in 1.95907s (17.3552GiB/s) | 20260506 11:38:04.245 | INFO | Analyzing chunk frames = [1054720,1055232) | 20260506 11:38:04.245 | INFO | MemInfo: { | 20260506 11:38:04.245 | INFO | "gpu" : " 17 GB", | 20260506 11:38:04.245 | INFO | "huge" : "", | 20260506 11:38:04.245 | INFO | "huge_shared" : " 27 GB", | 20260506 11:38:04.245 | INFO | "malloc" : " 14 GB", | 20260506 11:38:04.245 | INFO | "pinned" : " 29 GB" | 20260506 11:38:04.245 | INFO | } | 20260506 11:38:04.245 | INFO | Perf report for the last 5.13918 seconds: | 20260506 11:38:04.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:38:04.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:04.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:04.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:04.245 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 11:38:04.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:04.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:04.245 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:38:04.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:04.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:04.245 | INFO | Uploaded 36GiB to GPU in 2.09669s (17.1699GiB/s) | 20260506 11:38:04.245 | INFO | Downloaded 34GiB to GPU in 1.98492s (17.1292GiB/s) | 20260506 11:38:09.384 | INFO | Analyzing chunk frames = [1055232,1055744) | 20260506 11:38:09.384 | INFO | MemInfo: { | 20260506 11:38:09.384 | INFO | "gpu" : " 17 GB", | 20260506 11:38:09.384 | INFO | "huge" : "", | 20260506 11:38:09.384 | INFO | "huge_shared" : " 27 GB", | 20260506 11:38:09.384 | INFO | "malloc" : " 14 GB", | 20260506 11:38:09.384 | INFO | "pinned" : " 29 GB" | 20260506 11:38:09.384 | INFO | } | 20260506 11:38:09.384 | INFO | Perf report for the last 5.1392 seconds: | 20260506 11:38:09.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:38:09.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:09.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:09.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:09.384 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 11:38:09.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:09.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:09.384 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 11:38:09.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:09.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:09.384 | INFO | Uploaded 36GiB to GPU in 2.09397s (17.1922GiB/s) | 20260506 11:38:09.385 | INFO | Downloaded 34GiB to GPU in 2.00681s (16.9423GiB/s) | 20260506 11:38:14.524 | INFO | Analyzing chunk frames = [1055744,1056256) | 20260506 11:38:14.524 | INFO | MemInfo: { | 20260506 11:38:14.524 | INFO | "gpu" : " 17 GB", | 20260506 11:38:14.524 | INFO | "huge" : "", | 20260506 11:38:14.524 | INFO | "huge_shared" : " 27 GB", | 20260506 11:38:14.524 | INFO | "malloc" : " 14 GB", | 20260506 11:38:14.524 | INFO | "pinned" : " 29 GB" | 20260506 11:38:14.524 | INFO | } | 20260506 11:38:14.524 | INFO | Perf report for the last 5.1396 seconds: | 20260506 11:38:14.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:38:14.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:14.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:14.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:14.524 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260506 11:38:14.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:14.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:14.524 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 11:38:14.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:14.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:14.524 | INFO | Uploaded 36GiB to GPU in 2.02458s (17.7815GiB/s) | 20260506 11:38:14.524 | INFO | Downloaded 34GiB to GPU in 2.01662s (16.8599GiB/s) | 20260506 11:38:19.663 | INFO | Analyzing chunk frames = [1056256,1056768) | 20260506 11:38:19.663 | INFO | MemInfo: { | 20260506 11:38:19.663 | INFO | "gpu" : " 17 GB", | 20260506 11:38:19.663 | INFO | "huge" : "", | 20260506 11:38:19.663 | INFO | "huge_shared" : " 27 GB", | 20260506 11:38:19.663 | INFO | "malloc" : " 14 GB", | 20260506 11:38:19.663 | INFO | "pinned" : " 29 GB" | 20260506 11:38:19.663 | INFO | } | 20260506 11:38:19.663 | INFO | Perf report for the last 5.13923 seconds: | 20260506 11:38:19.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:38:19.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:19.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:19.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:19.663 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260506 11:38:19.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:19.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:19.663 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260506 11:38:19.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:19.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:19.663 | INFO | Uploaded 36GiB to GPU in 2.0756s (17.3444GiB/s) | 20260506 11:38:19.663 | INFO | Downloaded 34GiB to GPU in 1.97382s (17.2255GiB/s) | 20260506 11:38:19.663 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:38:19.663 | INFO | Total mem usage: 16.8 GiB | 20260506 11:38:19.663 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:38:19.663 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -6.25%) | 20260506 11:38:19.663 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:38:19.663 | INFO | High Water Breakdown: | 20260506 11:38:19.663 | INFO | 60.9% : 9.64 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:38:19.663 | INFO | 38.9% : 6.16 GiB ( 92% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:38:19.663 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:38:19.663 | INFO | ------------------------------------------ | 20260506 11:38:19.663 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:38:19.664 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:38:19.664 | INFO | Total mem usage: 26.5 GiB | 20260506 11:38:19.664 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:38:19.664 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:38:19.664 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:38:19.664 | INFO | High Water Breakdown: | 20260506 11:38:19.664 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:38:19.664 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:38:19.664 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:38:19.664 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:38:19.664 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:38:19.664 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:38:19.664 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:38:19.664 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:38:19.664 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:38:19.664 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:38:19.664 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:38:19.664 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:38:19.664 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:38:19.664 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:38:19.664 | INFO | ------------------------------------------ | 20260506 11:38:19.664 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:38:19.664 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:38:19.664 | INFO | Total mem usage: 25.6 GiB | 20260506 11:38:19.664 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:38:19.664 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:38:19.664 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:38:19.664 | INFO | High Water Breakdown: | 20260506 11:38:19.664 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:38:19.664 | INFO | ------------------------------------------ | 20260506 11:38:19.664 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:38:19.664 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:38:19.664 | INFO | Total mem usage: 15.7 GiB | 20260506 11:38:19.664 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:38:19.664 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:38:19.664 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 11:38:19.664 | INFO | High Water Breakdown: | 20260506 11:38:19.664 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:38:19.664 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:38:19.664 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:38:19.664 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:38:19.664 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:38:19.664 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:38:19.664 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:38:19.664 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:38:19.664 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:38:19.664 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:38:19.664 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:38:19.664 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:38:19.664 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:38:19.664 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:38:19.664 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:38:19.664 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:38:19.664 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:38:19.664 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:38:19.664 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:38:19.664 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:38:19.664 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:38:19.664 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:38:19.664 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:38:19.664 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:38:19.664 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:38:19.664 | INFO | ------------------------------------------ | 20260506 11:38:19.664 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:38:19.664 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:38:19.664 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:38:19.664 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:38:19.664 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.60e+01, 1.36e+01 / 2.14e+00 | 20260506 11:38:19.664 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.02e-01, 1.04e-02 / 6.31e-03 | 20260506 11:38:19.664 | INFO | PerfReport: 11.55%, Baseline -- 3.69e+00 / 4.47e+00, 5.89e-01 / 3.52e-01 | 20260506 11:38:19.664 | INFO | PerfReport: 3.25%, Histogram -- 1.04e+00 / 1.50e+00, 1.95e-01 / 1.30e-01 | 20260506 11:38:19.664 | INFO | PerfReport: 15.90%, DME -- 4.06e+01 / 4.52e+01, 4.15e+00 / 2.18e+00 | 20260506 11:38:19.664 | INFO | PerfReport: 56.15%, FrameLabeling -- 1.79e+01 / 1.96e+01, 1.42e+00 / 1.77e+00 | 20260506 11:38:19.664 | INFO | PerfReport: 7.66%, PulseAccumulating -- 2.45e+00 / 3.05e+00, 4.28e-01 / 2.60e-01 | 20260506 11:38:19.664 | INFO | PerfReport: 3.84%, Metrics -- 1.23e+00 / 1.61e+00, 3.12e-01 / 2.09e-01 | 20260506 11:38:19.664 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.34e-01 / 4.91e-01, 6.76e-02 / 3.16e-02 | 20260506 11:38:19.664 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:38:19.664 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:38:19.664 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:38:19.664 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260506 11:38:19.664 | INFO | PerfReport: 21.62%, Upload -- 2.14e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260506 11:38:19.664 | INFO | PerfReport: 21.08%, Download -- 2.08e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 11:38:19.664 | INFO | PerfReport: 32.30%, Compute -- 3.19e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 11:38:19.664 | INFO | PerfReport: 1.92%, LockUpload -- 1.89e+00 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260506 11:38:19.664 | INFO | PerfReport: 0.98%, LockDownload -- 9.65e-01 / 0.00e+00, 2.54e+00 / 0.00e+00 | 20260506 11:38:19.664 | INFO | PerfReport: 22.09%, LockCompute -- 2.18e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260506 11:38:19.665 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260506 11:38:19.781 | INFO | Starting a round of Baz IO | 20260506 11:38:22.992 | INFO | Populating RTMetrics report | 20260506 11:38:22.992 | INFO | RTMetrics report populated | 20260506 11:38:22.992 | INFO | Opening tmp RTMetrics json file | 20260506 11:38:23.148 | INFO | Serializing RTMetrics to json | 20260506 11:38:23.263 | INFO | Serialization done, swapping tmp file now... | 20260506 11:38:23.307 | INFO | RTMetrics done | 20260506 11:38:23.663 | INFO | Finished a round of Baz IO: Wrote 2.069 GiB in 3.881 seconds | 20260506 11:38:23.663 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.60% / 0.03% / 56.37% | 20260506 11:38:23.663 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.94188% / 6.75519e-05% / 3.17891e-05% | 20260506 11:38:23.663 | INFO | Baz IO overall: | 20260506 11:38:23.663 | INFO | BytesWritten : 510456030935 | 20260506 11:38:23.663 | INFO | NumEvents : 246387989305 | 20260506 11:38:24.802 | INFO | Analyzing chunk frames = [1056768,1057280) | 20260506 11:38:24.802 | INFO | MemInfo: { | 20260506 11:38:24.802 | INFO | "gpu" : " 17 GB", | 20260506 11:38:24.802 | INFO | "huge" : "", | 20260506 11:38:24.803 | INFO | "huge_shared" : " 27 GB", | 20260506 11:38:24.803 | INFO | "malloc" : " 14 GB", | 20260506 11:38:24.803 | INFO | "pinned" : " 29 GB" | 20260506 11:38:24.803 | INFO | } | 20260506 11:38:24.803 | INFO | Perf report for the last 5.13948 seconds: | 20260506 11:38:24.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:38:24.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:24.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:24.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:24.803 | INFO | ANALYSIS : 63.9%, 2.9, 96.6ms, 0 | 20260506 11:38:24.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:24.803 | INFO | RT_METRICS : 50.4%, 1.0, 26.4ms, 0 | 20260506 11:38:24.803 | INFO | PRE_HQ : 61.8%, 1.9, 61.1ms, 0 | 20260506 11:38:24.803 | INFO | BAZWRITER : 74.7%, 4.8, 184.5ms, 0 | 20260506 11:38:24.803 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 11:38:24.803 | INFO | Uploaded 36GiB to GPU in 2.13664s (16.8489GiB/s) | 20260506 11:38:24.803 | INFO | Downloaded 38GiB to GPU in 2.21958s (17.1204GiB/s) | 20260506 11:38:29.941 | INFO | Analyzing chunk frames = [1057280,1057792) | 20260506 11:38:29.942 | INFO | MemInfo: { | 20260506 11:38:29.942 | INFO | "gpu" : " 17 GB", | 20260506 11:38:29.942 | INFO | "huge" : "", | 20260506 11:38:29.942 | INFO | "huge_shared" : " 27 GB", | 20260506 11:38:29.942 | INFO | "malloc" : " 14 GB", | 20260506 11:38:29.942 | INFO | "pinned" : " 29 GB" | 20260506 11:38:29.942 | INFO | } | 20260506 11:38:29.942 | INFO | Perf report for the last 5.13904 seconds: | 20260506 11:38:29.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:38:29.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:29.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:29.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:29.942 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 11:38:29.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:29.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:29.942 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:38:29.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:29.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:29.942 | INFO | Uploaded 36GiB to GPU in 2.013s (17.8837GiB/s) | 20260506 11:38:29.942 | INFO | Downloaded 34GiB to GPU in 1.91326s (17.7708GiB/s) | 20260506 11:38:35.081 | INFO | Analyzing chunk frames = [1057792,1058304) | 20260506 11:38:35.081 | INFO | MemInfo: { | 20260506 11:38:35.081 | INFO | "gpu" : " 17 GB", | 20260506 11:38:35.081 | INFO | "huge" : "", | 20260506 11:38:35.081 | INFO | "huge_shared" : " 27 GB", | 20260506 11:38:35.081 | INFO | "malloc" : " 14 GB", | 20260506 11:38:35.081 | INFO | "pinned" : " 29 GB" | 20260506 11:38:35.081 | INFO | } | 20260506 11:38:35.081 | INFO | Perf report for the last 5.1394 seconds: | 20260506 11:38:35.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:38:35.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:35.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:35.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:35.081 | INFO | ANALYSIS : 62.0%, 3.0, 96.0ms, 0 | 20260506 11:38:35.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:35.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:35.081 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:38:35.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:35.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:35.081 | INFO | Uploaded 36GiB to GPU in 2.04462s (17.6072GiB/s) | 20260506 11:38:35.081 | INFO | Downloaded 34GiB to GPU in 1.93688s (17.554GiB/s) | 20260506 11:38:40.220 | INFO | Analyzing chunk frames = [1058304,1058816) | 20260506 11:38:40.220 | INFO | MemInfo: { | 20260506 11:38:40.220 | INFO | "gpu" : " 17 GB", | 20260506 11:38:40.220 | INFO | "huge" : "", | 20260506 11:38:40.221 | INFO | "huge_shared" : " 27 GB", | 20260506 11:38:40.221 | INFO | "malloc" : " 14 GB", | 20260506 11:38:40.221 | INFO | "pinned" : " 29 GB" | 20260506 11:38:40.221 | INFO | } | 20260506 11:38:40.221 | INFO | Perf report for the last 5.13952 seconds: | 20260506 11:38:40.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:38:40.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:40.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:40.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:40.221 | INFO | ANALYSIS : 62.2%, 2.9, 96.2ms, 0 | 20260506 11:38:40.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:40.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:40.221 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:38:40.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:40.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:40.221 | INFO | Uploaded 36GiB to GPU in 1.99381s (18.0559GiB/s) | 20260506 11:38:40.221 | INFO | Downloaded 34GiB to GPU in 1.9804s (17.1682GiB/s) | 20260506 11:38:45.360 | INFO | Analyzing chunk frames = [1058816,1059328) | 20260506 11:38:45.360 | INFO | MemInfo: { | 20260506 11:38:45.360 | INFO | "gpu" : " 17 GB", | 20260506 11:38:45.360 | INFO | "huge" : "", | 20260506 11:38:45.360 | INFO | "huge_shared" : " 27 GB", | 20260506 11:38:45.360 | INFO | "malloc" : " 14 GB", | 20260506 11:38:45.360 | INFO | "pinned" : " 29 GB" | 20260506 11:38:45.360 | INFO | } | 20260506 11:38:45.360 | INFO | Perf report for the last 5.13909 seconds: | 20260506 11:38:45.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:38:45.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:45.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:45.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:45.360 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 11:38:45.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:45.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:45.360 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:38:45.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:45.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:45.360 | INFO | Uploaded 36GiB to GPU in 2.01905s (17.8302GiB/s) | 20260506 11:38:45.360 | INFO | Downloaded 34GiB to GPU in 1.95191s (17.4189GiB/s) | 20260506 11:38:50.499 | INFO | Analyzing chunk frames = [1059328,1059840) | 20260506 11:38:50.499 | INFO | MemInfo: { | 20260506 11:38:50.499 | INFO | "gpu" : " 17 GB", | 20260506 11:38:50.499 | INFO | "huge" : "", | 20260506 11:38:50.499 | INFO | "huge_shared" : " 27 GB", | 20260506 11:38:50.499 | INFO | "malloc" : " 14 GB", | 20260506 11:38:50.499 | INFO | "pinned" : " 29 GB" | 20260506 11:38:50.499 | INFO | } | 20260506 11:38:50.499 | INFO | Perf report for the last 5.13913 seconds: | 20260506 11:38:50.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:38:50.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:50.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:50.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:50.499 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 11:38:50.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:50.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:50.499 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:38:50.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:50.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:50.499 | INFO | Uploaded 36GiB to GPU in 2.05823s (17.4907GiB/s) | 20260506 11:38:50.499 | INFO | Downloaded 34GiB to GPU in 1.92552s (17.6575GiB/s) | 20260506 11:38:55.638 | INFO | Analyzing chunk frames = [1059840,1060352) | 20260506 11:38:55.638 | INFO | MemInfo: { | 20260506 11:38:55.638 | INFO | "gpu" : " 17 GB", | 20260506 11:38:55.639 | INFO | "huge" : "", | 20260506 11:38:55.639 | INFO | "huge_shared" : " 27 GB", | 20260506 11:38:55.639 | INFO | "malloc" : " 14 GB", | 20260506 11:38:55.639 | INFO | "pinned" : " 29 GB" | 20260506 11:38:55.639 | INFO | } | 20260506 11:38:55.639 | INFO | Perf report for the last 5.13979 seconds: | 20260506 11:38:55.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:38:55.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:55.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:55.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:55.639 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 11:38:55.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:55.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:55.639 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:38:55.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:55.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:38:55.639 | INFO | Uploaded 36GiB to GPU in 1.97944s (18.187GiB/s) | 20260506 11:38:55.639 | INFO | Downloaded 34GiB to GPU in 2.02277s (16.8087GiB/s) | 20260506 11:39:00.778 | INFO | Analyzing chunk frames = [1060352,1060864) | 20260506 11:39:00.778 | INFO | MemInfo: { | 20260506 11:39:00.778 | INFO | "gpu" : " 17 GB", | 20260506 11:39:00.778 | INFO | "huge" : "", | 20260506 11:39:00.778 | INFO | "huge_shared" : " 27 GB", | 20260506 11:39:00.778 | INFO | "malloc" : " 14 GB", | 20260506 11:39:00.778 | INFO | "pinned" : " 29 GB" | 20260506 11:39:00.778 | INFO | } | 20260506 11:39:00.778 | INFO | Perf report for the last 5.13909 seconds: | 20260506 11:39:00.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:39:00.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:00.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:00.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:00.778 | INFO | ANALYSIS : 63.0%, 3.0, 98.2ms, 0 | 20260506 11:39:00.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:00.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:00.778 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 11:39:00.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:00.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:00.778 | INFO | Uploaded 36GiB to GPU in 2.03858s (17.6594GiB/s) | 20260506 11:39:00.778 | INFO | Downloaded 34GiB to GPU in 1.98091s (17.1638GiB/s) | 20260506 11:39:00.902 | INFO | Starting a round of Baz IO | 20260506 11:39:04.082 | INFO | Populating RTMetrics report | 20260506 11:39:04.082 | INFO | RTMetrics report populated | 20260506 11:39:04.082 | INFO | Opening tmp RTMetrics json file | 20260506 11:39:04.095 | INFO | Serializing RTMetrics to json | 20260506 11:39:04.158 | INFO | Serialization done, swapping tmp file now... | 20260506 11:39:04.187 | INFO | RTMetrics done | 20260506 11:39:04.521 | INFO | Finished a round of Baz IO: Wrote 2.067 GiB in 3.618 seconds | 20260506 11:39:04.521 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.57% / 0.03% / 56.40% | 20260506 11:39:04.521 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.91859% / 4.37101e-05% / 4.37101e-05% | 20260506 11:39:04.521 | INFO | Baz IO overall: | 20260506 11:39:04.521 | INFO | BytesWritten : 512675580349 | 20260506 11:39:04.521 | INFO | NumEvents : 247465257158 | 20260506 11:39:05.917 | INFO | Analyzing chunk frames = [1060864,1061376) | 20260506 11:39:05.917 | INFO | MemInfo: { | 20260506 11:39:05.917 | INFO | "gpu" : " 17 GB", | 20260506 11:39:05.917 | INFO | "huge" : "", | 20260506 11:39:05.917 | INFO | "huge_shared" : " 27 GB", | 20260506 11:39:05.917 | INFO | "malloc" : " 14 GB", | 20260506 11:39:05.917 | INFO | "pinned" : " 29 GB" | 20260506 11:39:05.917 | INFO | } | 20260506 11:39:05.917 | INFO | Perf report for the last 5.13951 seconds: | 20260506 11:39:05.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:39:05.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:05.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:05.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:05.917 | INFO | ANALYSIS : 63.7%, 2.9, 95.7ms, 0 | 20260506 11:39:05.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:05.917 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260506 11:39:05.917 | INFO | PRE_HQ : 62.0%, 1.9, 61.8ms, 0 | 20260506 11:39:05.917 | INFO | BAZWRITER : 69.5%, 4.8, 170.3ms, 0 | 20260506 11:39:05.917 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 11:39:05.918 | INFO | Uploaded 36GiB to GPU in 2.14246s (16.8031GiB/s) | 20260506 11:39:05.918 | INFO | Downloaded 38GiB to GPU in 2.22233s (17.0992GiB/s) | 20260506 11:39:11.056 | INFO | Analyzing chunk frames = [1061376,1061888) | 20260506 11:39:11.056 | INFO | MemInfo: { | 20260506 11:39:11.056 | INFO | "gpu" : " 17 GB", | 20260506 11:39:11.056 | INFO | "huge" : "", | 20260506 11:39:11.056 | INFO | "huge_shared" : " 27 GB", | 20260506 11:39:11.056 | INFO | "malloc" : " 14 GB", | 20260506 11:39:11.056 | INFO | "pinned" : " 29 GB" | 20260506 11:39:11.056 | INFO | } | 20260506 11:39:11.056 | INFO | Perf report for the last 5.13906 seconds: | 20260506 11:39:11.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:39:11.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:11.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:11.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:11.056 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:39:11.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:11.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:11.056 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 11:39:11.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:11.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:11.056 | INFO | Uploaded 36GiB to GPU in 2.02901s (17.7426GiB/s) | 20260506 11:39:11.056 | INFO | Downloaded 34GiB to GPU in 1.92456s (17.6664GiB/s) | 20260506 11:39:16.196 | INFO | Analyzing chunk frames = [1061888,1062400) | 20260506 11:39:16.196 | INFO | MemInfo: { | 20260506 11:39:16.196 | INFO | "gpu" : " 17 GB", | 20260506 11:39:16.196 | INFO | "huge" : "", | 20260506 11:39:16.196 | INFO | "huge_shared" : " 27 GB", | 20260506 11:39:16.196 | INFO | "malloc" : " 14 GB", | 20260506 11:39:16.196 | INFO | "pinned" : " 29 GB" | 20260506 11:39:16.196 | INFO | } | 20260506 11:39:16.196 | INFO | Perf report for the last 5.13939 seconds: | 20260506 11:39:16.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:39:16.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:16.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:16.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:16.196 | INFO | ANALYSIS : 62.0%, 3.0, 96.1ms, 0 | 20260506 11:39:16.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:16.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:16.196 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:39:16.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:16.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:16.196 | INFO | Uploaded 36GiB to GPU in 2.02136s (17.8098GiB/s) | 20260506 11:39:16.196 | INFO | Downloaded 34GiB to GPU in 1.94099s (17.5168GiB/s) | 20260506 11:39:21.335 | INFO | Analyzing chunk frames = [1062400,1062912) | 20260506 11:39:21.335 | INFO | MemInfo: { | 20260506 11:39:21.335 | INFO | "gpu" : " 17 GB", | 20260506 11:39:21.335 | INFO | "huge" : "", | 20260506 11:39:21.335 | INFO | "huge_shared" : " 27 GB", | 20260506 11:39:21.335 | INFO | "malloc" : " 14 GB", | 20260506 11:39:21.335 | INFO | "pinned" : " 29 GB" | 20260506 11:39:21.335 | INFO | } | 20260506 11:39:21.335 | INFO | Perf report for the last 5.13915 seconds: | 20260506 11:39:21.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:39:21.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:21.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:21.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:21.335 | INFO | ANALYSIS : 62.3%, 2.9, 96.2ms, 0 | 20260506 11:39:21.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:21.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:21.335 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:39:21.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:21.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:21.335 | INFO | Uploaded 36GiB to GPU in 2.02333s (17.7925GiB/s) | 20260506 11:39:21.335 | INFO | Downloaded 34GiB to GPU in 1.97379s (17.2257GiB/s) | 20260506 11:39:26.474 | INFO | Analyzing chunk frames = [1062912,1063424) | 20260506 11:39:26.474 | INFO | MemInfo: { | 20260506 11:39:26.474 | INFO | "gpu" : " 17 GB", | 20260506 11:39:26.474 | INFO | "huge" : "", | 20260506 11:39:26.474 | INFO | "huge_shared" : " 27 GB", | 20260506 11:39:26.474 | INFO | "malloc" : " 14 GB", | 20260506 11:39:26.474 | INFO | "pinned" : " 29 GB" | 20260506 11:39:26.474 | INFO | } | 20260506 11:39:26.474 | INFO | Perf report for the last 5.13943 seconds: | 20260506 11:39:26.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:39:26.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:26.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:26.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:26.475 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 11:39:26.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:26.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:26.475 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:39:26.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:26.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:26.475 | INFO | Uploaded 36GiB to GPU in 2.04073s (17.6408GiB/s) | 20260506 11:39:26.475 | INFO | Downloaded 34GiB to GPU in 1.95416s (17.3988GiB/s) | 20260506 11:39:31.613 | INFO | Analyzing chunk frames = [1063424,1063936) | 20260506 11:39:31.613 | INFO | MemInfo: { | 20260506 11:39:31.613 | INFO | "gpu" : " 17 GB", | 20260506 11:39:31.613 | INFO | "huge" : "", | 20260506 11:39:31.613 | INFO | "huge_shared" : " 27 GB", | 20260506 11:39:31.613 | INFO | "malloc" : " 14 GB", | 20260506 11:39:31.613 | INFO | "pinned" : " 29 GB" | 20260506 11:39:31.613 | INFO | } | 20260506 11:39:31.614 | INFO | Perf report for the last 5.13916 seconds: | 20260506 11:39:31.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:39:31.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:31.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:31.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:31.614 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 11:39:31.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:31.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:31.614 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:39:31.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:31.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:31.614 | INFO | Uploaded 36GiB to GPU in 2.10178s (17.1284GiB/s) | 20260506 11:39:31.614 | INFO | Downloaded 34GiB to GPU in 1.9884s (17.0992GiB/s) | 20260506 11:39:36.753 | INFO | Analyzing chunk frames = [1063936,1064448) | 20260506 11:39:36.753 | INFO | MemInfo: { | 20260506 11:39:36.753 | INFO | "gpu" : " 17 GB", | 20260506 11:39:36.753 | INFO | "huge" : "", | 20260506 11:39:36.753 | INFO | "huge_shared" : " 27 GB", | 20260506 11:39:36.753 | INFO | "malloc" : " 14 GB", | 20260506 11:39:36.753 | INFO | "pinned" : " 29 GB" | 20260506 11:39:36.753 | INFO | } | 20260506 11:39:36.753 | INFO | Perf report for the last 5.13953 seconds: | 20260506 11:39:36.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:39:36.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:36.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:36.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:36.753 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:39:36.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:36.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:36.753 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 11:39:36.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:36.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:36.754 | INFO | Uploaded 36GiB to GPU in 2.0224s (17.8007GiB/s) | 20260506 11:39:36.754 | INFO | Downloaded 34GiB to GPU in 2.00944s (16.9201GiB/s) | 20260506 11:39:41.892 | INFO | Analyzing chunk frames = [1064448,1064960) | 20260506 11:39:41.892 | INFO | MemInfo: { | 20260506 11:39:41.892 | INFO | "gpu" : " 17 GB", | 20260506 11:39:41.892 | INFO | "huge" : "", | 20260506 11:39:41.892 | INFO | "huge_shared" : " 27 GB", | 20260506 11:39:41.892 | INFO | "malloc" : " 14 GB", | 20260506 11:39:41.892 | INFO | "pinned" : " 29 GB" | 20260506 11:39:41.892 | INFO | } | 20260506 11:39:41.892 | INFO | Perf report for the last 5.1393 seconds: | 20260506 11:39:41.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:39:41.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:41.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:41.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:41.892 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260506 11:39:41.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:41.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:41.892 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260506 11:39:41.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:41.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:41.893 | INFO | Uploaded 36GiB to GPU in 2.01728s (17.8458GiB/s) | 20260506 11:39:41.893 | INFO | Downloaded 34GiB to GPU in 1.96562s (17.2973GiB/s) | 20260506 11:39:41.893 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:39:41.893 | INFO | Total mem usage: 16.8 GiB | 20260506 11:39:41.893 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:39:41.893 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -5.48%) | 20260506 11:39:41.893 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:39:41.893 | INFO | High Water Breakdown: | 20260506 11:39:41.893 | INFO | 61.1% : 9.73 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:39:41.893 | INFO | 38.8% : 6.18 GiB ( 92.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:39:41.893 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:39:41.893 | INFO | ------------------------------------------ | 20260506 11:39:41.893 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:39:41.893 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:39:41.893 | INFO | Total mem usage: 26.5 GiB | 20260506 11:39:41.893 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:39:41.893 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:39:41.893 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:39:41.893 | INFO | High Water Breakdown: | 20260506 11:39:41.893 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:39:41.893 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:39:41.893 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:39:41.893 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:39:41.893 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:39:41.893 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:39:41.893 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:39:41.893 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:39:41.893 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:39:41.893 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:39:41.893 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:39:41.893 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:39:41.893 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:39:41.893 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:39:41.893 | INFO | ------------------------------------------ | 20260506 11:39:41.893 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:39:41.893 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:39:41.893 | INFO | Total mem usage: 25.6 GiB | 20260506 11:39:41.893 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:39:41.893 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:39:41.893 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:39:41.894 | INFO | High Water Breakdown: | 20260506 11:39:41.894 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:39:41.894 | INFO | ------------------------------------------ | 20260506 11:39:41.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:39:41.894 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:39:41.894 | INFO | Total mem usage: 15.7 GiB | 20260506 11:39:41.894 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:39:41.894 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:39:41.894 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:39:41.894 | INFO | High Water Breakdown: | 20260506 11:39:41.894 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:39:41.894 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:39:41.894 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:39:41.894 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:39:41.894 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:39:41.894 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:39:41.894 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:39:41.894 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:39:41.894 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:39:41.894 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:39:41.894 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:39:41.894 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:39:41.894 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:39:41.894 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:39:41.894 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:39:41.894 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:39:41.894 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:39:41.894 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:39:41.894 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:39:41.894 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:39:41.894 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:39:41.894 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:39:41.894 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:39:41.894 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:39:41.894 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:39:41.894 | INFO | ------------------------------------------ | 20260506 11:39:41.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:39:41.894 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:39:41.894 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:39:41.894 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:39:41.894 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.60e+01, 1.38e+01 / 2.62e+00 | 20260506 11:39:41.894 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.01e-01, 1.01e-02 / 6.64e-03 | 20260506 11:39:41.894 | INFO | PerfReport: 11.55%, Baseline -- 3.69e+00 / 4.38e+00, 5.97e-01 / 4.17e-01 | 20260506 11:39:41.894 | INFO | PerfReport: 3.24%, Histogram -- 1.04e+00 / 1.45e+00, 1.94e-01 / 1.49e-01 | 20260506 11:39:41.894 | INFO | PerfReport: 15.91%, DME -- 4.07e+01 / 4.51e+01, 4.31e+00 / 2.68e+00 | 20260506 11:39:41.894 | INFO | PerfReport: 56.17%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.42e+00 / 1.69e+00 | 20260506 11:39:41.894 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.44e+00 / 3.11e+00, 4.22e-01 / 2.01e-01 | 20260506 11:39:41.894 | INFO | PerfReport: 3.84%, Metrics -- 1.23e+00 / 1.62e+00, 3.12e-01 / 1.64e-01 | 20260506 11:39:41.894 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.34e-01 / 4.88e-01, 6.75e-02 / 3.26e-02 | 20260506 11:39:41.894 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:39:41.894 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:39:41.894 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:39:41.894 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 11:39:41.894 | INFO | PerfReport: 21.53%, Upload -- 2.13e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 11:39:41.894 | INFO | PerfReport: 21.02%, Download -- 2.08e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260506 11:39:41.894 | INFO | PerfReport: 32.33%, Compute -- 3.19e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260506 11:39:41.894 | INFO | PerfReport: 1.95%, LockUpload -- 1.92e+00 / 0.00e+00, 7.55e+00 / 0.00e+00 | 20260506 11:39:41.894 | INFO | PerfReport: 0.94%, LockDownload -- 9.33e-01 / 0.00e+00, 2.54e+00 / 0.00e+00 | 20260506 11:39:41.894 | INFO | PerfReport: 22.21%, LockCompute -- 2.20e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 11:39:41.894 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260506 11:39:42.019 | INFO | Starting a round of Baz IO | 20260506 11:39:45.205 | INFO | Populating RTMetrics report | 20260506 11:39:45.206 | INFO | RTMetrics report populated | 20260506 11:39:45.206 | INFO | Opening tmp RTMetrics json file | 20260506 11:39:45.344 | INFO | Serializing RTMetrics to json | 20260506 11:39:45.363 | INFO | Serialization done, swapping tmp file now... | 20260506 11:39:45.365 | INFO | RTMetrics done | 20260506 11:39:45.382 | INFO | Finished a round of Baz IO: Wrote 2.069 GiB in 3.362 seconds | 20260506 11:39:45.382 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.54% / 0.03% / 56.43% | 20260506 11:39:45.382 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.9168% / 7.15256e-05% / 7.15256e-05% | 20260506 11:39:45.382 | INFO | Baz IO overall: | 20260506 11:39:45.382 | INFO | BytesWritten : 514897020726 | 20260506 11:39:45.382 | INFO | NumEvents : 248543947602 | 20260506 11:39:47.032 | INFO | Analyzing chunk frames = [1064960,1065472) | 20260506 11:39:47.032 | INFO | MemInfo: { | 20260506 11:39:47.032 | INFO | "gpu" : " 17 GB", | 20260506 11:39:47.032 | INFO | "huge" : "", | 20260506 11:39:47.032 | INFO | "huge_shared" : " 27 GB", | 20260506 11:39:47.032 | INFO | "malloc" : " 14 GB", | 20260506 11:39:47.032 | INFO | "pinned" : " 29 GB" | 20260506 11:39:47.032 | INFO | } | 20260506 11:39:47.032 | INFO | Perf report for the last 5.1392 seconds: | 20260506 11:39:47.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:39:47.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:47.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:47.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:47.032 | INFO | ANALYSIS : 63.9%, 2.9, 96.4ms, 0 | 20260506 11:39:47.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:47.032 | INFO | RT_METRICS : 47.6%, 1.0, 25.0ms, 0 | 20260506 11:39:47.032 | INFO | PRE_HQ : 62.0%, 1.9, 62.1ms, 0 | 20260506 11:39:47.032 | INFO | BAZWRITER : 64.7%, 4.6, 153.5ms, 0 | 20260506 11:39:47.032 | INFO | BAZ_SAMPLING : 2.9%, 1.0, 1.5ms, 0 | 20260506 11:39:47.032 | INFO | Uploaded 36GiB to GPU in 2.10508s (17.1015GiB/s) | 20260506 11:39:47.032 | INFO | Downloaded 38GiB to GPU in 2.22332s (17.0916GiB/s) | 20260506 11:39:52.171 | INFO | Analyzing chunk frames = [1065472,1065984) | 20260506 11:39:52.171 | INFO | MemInfo: { | 20260506 11:39:52.171 | INFO | "gpu" : " 17 GB", | 20260506 11:39:52.171 | INFO | "huge" : "", | 20260506 11:39:52.171 | INFO | "huge_shared" : " 27 GB", | 20260506 11:39:52.171 | INFO | "malloc" : " 14 GB", | 20260506 11:39:52.171 | INFO | "pinned" : " 29 GB" | 20260506 11:39:52.171 | INFO | } | 20260506 11:39:52.171 | INFO | Perf report for the last 5.13945 seconds: | 20260506 11:39:52.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:39:52.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:52.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:52.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:52.171 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 11:39:52.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:52.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:52.171 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 11:39:52.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:52.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:52.171 | INFO | Uploaded 36GiB to GPU in 2.01801s (17.8394GiB/s) | 20260506 11:39:52.172 | INFO | Downloaded 34GiB to GPU in 1.91669s (17.739GiB/s) | 20260506 11:39:57.311 | INFO | Analyzing chunk frames = [1065984,1066496) | 20260506 11:39:57.311 | INFO | MemInfo: { | 20260506 11:39:57.311 | INFO | "gpu" : " 17 GB", | 20260506 11:39:57.311 | INFO | "huge" : "", | 20260506 11:39:57.311 | INFO | "huge_shared" : " 27 GB", | 20260506 11:39:57.311 | INFO | "malloc" : " 14 GB", | 20260506 11:39:57.311 | INFO | "pinned" : " 29 GB" | 20260506 11:39:57.311 | INFO | } | 20260506 11:39:57.311 | INFO | Perf report for the last 5.13994 seconds: | 20260506 11:39:57.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:39:57.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:57.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:57.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:57.311 | INFO | ANALYSIS : 62.2%, 3.0, 96.4ms, 0 | 20260506 11:39:57.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:57.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:57.311 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:39:57.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:57.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:39:57.311 | INFO | Uploaded 36GiB to GPU in 1.99597s (18.0364GiB/s) | 20260506 11:39:57.311 | INFO | Downloaded 34GiB to GPU in 1.9456s (17.4754GiB/s) | 20260506 11:40:02.450 | INFO | Analyzing chunk frames = [1066496,1067008) | 20260506 11:40:02.450 | INFO | MemInfo: { | 20260506 11:40:02.450 | INFO | "gpu" : " 17 GB", | 20260506 11:40:02.450 | INFO | "huge" : "", | 20260506 11:40:02.450 | INFO | "huge_shared" : " 27 GB", | 20260506 11:40:02.450 | INFO | "malloc" : " 14 GB", | 20260506 11:40:02.450 | INFO | "pinned" : " 29 GB" | 20260506 11:40:02.450 | INFO | } | 20260506 11:40:02.450 | INFO | Perf report for the last 5.13902 seconds: | 20260506 11:40:02.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:40:02.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:02.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:02.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:02.450 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 11:40:02.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:02.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:02.450 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:40:02.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:02.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:02.450 | INFO | Uploaded 36GiB to GPU in 2.08326s (17.2806GiB/s) | 20260506 11:40:02.450 | INFO | Downloaded 34GiB to GPU in 1.96404s (17.3112GiB/s) | 20260506 11:40:07.589 | INFO | Analyzing chunk frames = [1067008,1067520) | 20260506 11:40:07.589 | INFO | MemInfo: { | 20260506 11:40:07.589 | INFO | "gpu" : " 17 GB", | 20260506 11:40:07.589 | INFO | "huge" : "", | 20260506 11:40:07.589 | INFO | "huge_shared" : " 27 GB", | 20260506 11:40:07.589 | INFO | "malloc" : " 14 GB", | 20260506 11:40:07.589 | INFO | "pinned" : " 29 GB" | 20260506 11:40:07.589 | INFO | } | 20260506 11:40:07.589 | INFO | Perf report for the last 5.1393 seconds: | 20260506 11:40:07.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:40:07.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:07.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:07.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:07.589 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 11:40:07.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:07.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:07.589 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 11:40:07.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:07.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:07.590 | INFO | Uploaded 36GiB to GPU in 2.10965s (17.0644GiB/s) | 20260506 11:40:07.590 | INFO | Downloaded 34GiB to GPU in 1.99004s (17.0851GiB/s) | 20260506 11:40:12.729 | INFO | Analyzing chunk frames = [1067520,1068032) | 20260506 11:40:12.729 | INFO | MemInfo: { | 20260506 11:40:12.729 | INFO | "gpu" : " 17 GB", | 20260506 11:40:12.729 | INFO | "huge" : "", | 20260506 11:40:12.729 | INFO | "huge_shared" : " 27 GB", | 20260506 11:40:12.729 | INFO | "malloc" : " 14 GB", | 20260506 11:40:12.729 | INFO | "pinned" : " 29 GB" | 20260506 11:40:12.729 | INFO | } | 20260506 11:40:12.729 | INFO | Perf report for the last 5.13948 seconds: | 20260506 11:40:12.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:40:12.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:12.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:12.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:12.729 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 11:40:12.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:12.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:12.729 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:40:12.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:12.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:12.729 | INFO | Uploaded 36GiB to GPU in 2.06484s (17.4348GiB/s) | 20260506 11:40:12.729 | INFO | Downloaded 34GiB to GPU in 1.99992s (17.0007GiB/s) | 20260506 11:40:17.868 | INFO | Analyzing chunk frames = [1068032,1068544) | 20260506 11:40:17.868 | INFO | MemInfo: { | 20260506 11:40:17.868 | INFO | "gpu" : " 17 GB", | 20260506 11:40:17.868 | INFO | "huge" : "", | 20260506 11:40:17.868 | INFO | "huge_shared" : " 27 GB", | 20260506 11:40:17.868 | INFO | "malloc" : " 14 GB", | 20260506 11:40:17.869 | INFO | "pinned" : " 29 GB" | 20260506 11:40:17.869 | INFO | } | 20260506 11:40:17.869 | INFO | Perf report for the last 5.13972 seconds: | 20260506 11:40:17.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:40:17.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:17.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:17.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:17.869 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:40:17.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:17.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:17.869 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260506 11:40:17.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:17.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:17.869 | INFO | Uploaded 36GiB to GPU in 2.03445s (17.6952GiB/s) | 20260506 11:40:17.869 | INFO | Downloaded 34GiB to GPU in 2.0069s (16.9416GiB/s) | 20260506 11:40:23.007 | INFO | Analyzing chunk frames = [1068544,1069056) | 20260506 11:40:23.007 | INFO | MemInfo: { | 20260506 11:40:23.007 | INFO | "gpu" : " 17 GB", | 20260506 11:40:23.007 | INFO | "huge" : "", | 20260506 11:40:23.007 | INFO | "huge_shared" : " 27 GB", | 20260506 11:40:23.007 | INFO | "malloc" : " 14 GB", | 20260506 11:40:23.007 | INFO | "pinned" : " 29 GB" | 20260506 11:40:23.007 | INFO | } | 20260506 11:40:23.007 | INFO | Perf report for the last 5.13876 seconds: | 20260506 11:40:23.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:40:23.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:23.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:23.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:23.007 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260506 11:40:23.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:23.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:23.007 | INFO | PRE_HQ : 33.7%, 1.0, 17.6ms, 0 | 20260506 11:40:23.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:23.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:23.008 | INFO | Uploaded 36GiB to GPU in 2.07134s (17.3801GiB/s) | 20260506 11:40:23.008 | INFO | Downloaded 34GiB to GPU in 1.99696s (17.0259GiB/s) | 20260506 11:40:23.128 | INFO | Starting a round of Baz IO | 20260506 11:40:26.327 | INFO | Populating RTMetrics report | 20260506 11:40:26.328 | INFO | RTMetrics report populated | 20260506 11:40:26.328 | INFO | Opening tmp RTMetrics json file | 20260506 11:40:26.344 | INFO | Serializing RTMetrics to json | 20260506 11:40:26.449 | INFO | Serialization done, swapping tmp file now... | 20260506 11:40:26.484 | INFO | RTMetrics done | 20260506 11:40:26.588 | INFO | Finished a round of Baz IO: Wrote 2.071 GiB in 3.46 seconds | 20260506 11:40:26.588 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.50% / 0.03% / 56.46% | 20260506 11:40:26.588 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.91398% / 4.37101e-05% / 3.57628e-05% | 20260506 11:40:26.588 | INFO | Baz IO overall: | 20260506 11:40:26.588 | INFO | BytesWritten : 517120954043 | 20260506 11:40:26.588 | INFO | NumEvents : 249624308588 | 20260506 11:40:28.146 | INFO | Analyzing chunk frames = [1069056,1069568) | 20260506 11:40:28.146 | INFO | MemInfo: { | 20260506 11:40:28.146 | INFO | "gpu" : " 17 GB", | 20260506 11:40:28.146 | INFO | "huge" : "", | 20260506 11:40:28.146 | INFO | "huge_shared" : " 27 GB", | 20260506 11:40:28.146 | INFO | "malloc" : " 14 GB", | 20260506 11:40:28.146 | INFO | "pinned" : " 29 GB" | 20260506 11:40:28.146 | INFO | } | 20260506 11:40:28.146 | INFO | Perf report for the last 5.1391 seconds: | 20260506 11:40:28.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:40:28.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:28.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:28.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:28.147 | INFO | ANALYSIS : 64.0%, 2.9, 96.3ms, 0 | 20260506 11:40:28.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:28.147 | INFO | RT_METRICS : 47.6%, 1.0, 24.9ms, 0 | 20260506 11:40:28.147 | INFO | PRE_HQ : 62.2%, 1.9, 61.5ms, 0 | 20260506 11:40:28.147 | INFO | BAZWRITER : 66.6%, 4.7, 161.7ms, 0 | 20260506 11:40:28.147 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.2ms, 0 | 20260506 11:40:28.147 | INFO | Uploaded 36GiB to GPU in 2.13493s (16.8624GiB/s) | 20260506 11:40:28.147 | INFO | Downloaded 38GiB to GPU in 2.22323s (17.0922GiB/s) | 20260506 11:40:33.286 | INFO | Analyzing chunk frames = [1069568,1070080) | 20260506 11:40:33.286 | INFO | MemInfo: { | 20260506 11:40:33.286 | INFO | "gpu" : " 17 GB", | 20260506 11:40:33.286 | INFO | "huge" : "", | 20260506 11:40:33.286 | INFO | "huge_shared" : " 27 GB", | 20260506 11:40:33.286 | INFO | "malloc" : " 14 GB", | 20260506 11:40:33.286 | INFO | "pinned" : " 29 GB" | 20260506 11:40:33.286 | INFO | } | 20260506 11:40:33.286 | INFO | Perf report for the last 5.13951 seconds: | 20260506 11:40:33.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:40:33.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:33.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:33.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:33.286 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:40:33.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:33.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:33.286 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260506 11:40:33.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:33.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:33.286 | INFO | Uploaded 36GiB to GPU in 2.02412s (17.7855GiB/s) | 20260506 11:40:33.286 | INFO | Downloaded 34GiB to GPU in 1.92932s (17.6228GiB/s) | 20260506 11:40:38.425 | INFO | Analyzing chunk frames = [1070080,1070592) | 20260506 11:40:38.425 | INFO | MemInfo: { | 20260506 11:40:38.425 | INFO | "gpu" : " 17 GB", | 20260506 11:40:38.425 | INFO | "huge" : "", | 20260506 11:40:38.425 | INFO | "huge_shared" : " 27 GB", | 20260506 11:40:38.425 | INFO | "malloc" : " 14 GB", | 20260506 11:40:38.425 | INFO | "pinned" : " 29 GB" | 20260506 11:40:38.425 | INFO | } | 20260506 11:40:38.425 | INFO | Perf report for the last 5.13931 seconds: | 20260506 11:40:38.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:40:38.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:38.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:38.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:38.425 | INFO | ANALYSIS : 62.4%, 3.0, 96.6ms, 0 | 20260506 11:40:38.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:38.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:38.425 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:40:38.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:38.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:38.426 | INFO | Uploaded 36GiB to GPU in 2.06789s (17.409GiB/s) | 20260506 11:40:38.426 | INFO | Downloaded 34GiB to GPU in 1.948s (17.4538GiB/s) | 20260506 11:40:43.564 | INFO | Analyzing chunk frames = [1070592,1071104) | 20260506 11:40:43.564 | INFO | MemInfo: { | 20260506 11:40:43.564 | INFO | "gpu" : " 17 GB", | 20260506 11:40:43.564 | INFO | "huge" : "", | 20260506 11:40:43.564 | INFO | "huge_shared" : " 27 GB", | 20260506 11:40:43.564 | INFO | "malloc" : " 14 GB", | 20260506 11:40:43.564 | INFO | "pinned" : " 29 GB" | 20260506 11:40:43.564 | INFO | } | 20260506 11:40:43.564 | INFO | Perf report for the last 5.13916 seconds: | 20260506 11:40:43.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:40:43.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:43.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:43.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:43.565 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 11:40:43.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:43.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:43.565 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:40:43.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:43.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:43.565 | INFO | Uploaded 36GiB to GPU in 2.08065s (17.3023GiB/s) | 20260506 11:40:43.565 | INFO | Downloaded 34GiB to GPU in 1.96945s (17.2637GiB/s) | 20260506 11:40:48.704 | INFO | Analyzing chunk frames = [1071104,1071616) | 20260506 11:40:48.704 | INFO | MemInfo: { | 20260506 11:40:48.704 | INFO | "gpu" : " 17 GB", | 20260506 11:40:48.704 | INFO | "huge" : "", | 20260506 11:40:48.704 | INFO | "huge_shared" : " 27 GB", | 20260506 11:40:48.704 | INFO | "malloc" : " 14 GB", | 20260506 11:40:48.704 | INFO | "pinned" : " 29 GB" | 20260506 11:40:48.704 | INFO | } | 20260506 11:40:48.704 | INFO | Perf report for the last 5.13985 seconds: | 20260506 11:40:48.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:40:48.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:48.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:48.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:48.704 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 11:40:48.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:48.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:48.704 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:40:48.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:48.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:48.705 | INFO | Uploaded 36GiB to GPU in 2.09272s (17.2025GiB/s) | 20260506 11:40:48.705 | INFO | Downloaded 34GiB to GPU in 1.99628s (17.0317GiB/s) | 20260506 11:40:53.843 | INFO | Analyzing chunk frames = [1071616,1072128) | 20260506 11:40:53.843 | INFO | MemInfo: { | 20260506 11:40:53.843 | INFO | "gpu" : " 17 GB", | 20260506 11:40:53.843 | INFO | "huge" : "", | 20260506 11:40:53.843 | INFO | "huge_shared" : " 27 GB", | 20260506 11:40:53.843 | INFO | "malloc" : " 14 GB", | 20260506 11:40:53.843 | INFO | "pinned" : " 29 GB" | 20260506 11:40:53.843 | INFO | } | 20260506 11:40:53.843 | INFO | Perf report for the last 5.13881 seconds: | 20260506 11:40:53.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:40:53.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:53.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:53.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:53.843 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 11:40:53.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:53.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:53.843 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:40:53.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:53.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:53.844 | INFO | Uploaded 36GiB to GPU in 2.0896s (17.2282GiB/s) | 20260506 11:40:53.844 | INFO | Downloaded 34GiB to GPU in 1.98915s (17.0927GiB/s) | 20260506 11:40:58.983 | INFO | Analyzing chunk frames = [1072128,1072640) | 20260506 11:40:58.983 | INFO | MemInfo: { | 20260506 11:40:58.983 | INFO | "gpu" : " 17 GB", | 20260506 11:40:58.983 | INFO | "huge" : "", | 20260506 11:40:58.983 | INFO | "huge_shared" : " 27 GB", | 20260506 11:40:58.983 | INFO | "malloc" : " 14 GB", | 20260506 11:40:58.983 | INFO | "pinned" : " 29 GB" | 20260506 11:40:58.983 | INFO | } | 20260506 11:40:58.983 | INFO | Perf report for the last 5.14005 seconds: | 20260506 11:40:58.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:40:58.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:58.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:58.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:58.983 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:40:58.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:58.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:58.983 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 11:40:58.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:58.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:40:58.984 | INFO | Uploaded 36GiB to GPU in 2.00276s (17.9752GiB/s) | 20260506 11:40:58.984 | INFO | Downloaded 34GiB to GPU in 2.00699s (16.9408GiB/s) | 20260506 11:41:04.122 | INFO | Analyzing chunk frames = [1072640,1073152) | 20260506 11:41:04.122 | INFO | MemInfo: { | 20260506 11:41:04.122 | INFO | "gpu" : " 17 GB", | 20260506 11:41:04.122 | INFO | "huge" : "", | 20260506 11:41:04.122 | INFO | "huge_shared" : " 27 GB", | 20260506 11:41:04.122 | INFO | "malloc" : " 14 GB", | 20260506 11:41:04.122 | INFO | "pinned" : " 29 GB" | 20260506 11:41:04.122 | INFO | } | 20260506 11:41:04.122 | INFO | Perf report for the last 5.13919 seconds: | 20260506 11:41:04.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:41:04.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:04.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:04.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:04.122 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260506 11:41:04.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:04.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:04.123 | INFO | PRE_HQ : 33.6%, 1.0, 17.5ms, 0 | 20260506 11:41:04.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:04.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:04.123 | INFO | Uploaded 36GiB to GPU in 2.01014s (17.9092GiB/s) | 20260506 11:41:04.123 | INFO | Downloaded 34GiB to GPU in 1.96379s (17.3135GiB/s) | 20260506 11:41:04.123 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:41:04.123 | INFO | Total mem usage: 16.8 GiB | 20260506 11:41:04.123 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:41:04.123 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -10.1%) | 20260506 11:41:04.123 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:41:04.123 | INFO | High Water Breakdown: | 20260506 11:41:04.123 | INFO | 60.4% : 9.22 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:41:04.123 | INFO | 39.5% : 6.03 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:41:04.123 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:41:04.123 | INFO | ------------------------------------------ | 20260506 11:41:04.123 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:41:04.123 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:41:04.123 | INFO | Total mem usage: 26.5 GiB | 20260506 11:41:04.123 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:41:04.123 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:41:04.123 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:41:04.123 | INFO | High Water Breakdown: | 20260506 11:41:04.123 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:41:04.123 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:41:04.123 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:41:04.123 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:41:04.123 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:41:04.123 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:41:04.123 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:41:04.123 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:41:04.123 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:41:04.123 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:41:04.123 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:41:04.123 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:41:04.123 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:41:04.123 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:41:04.124 | INFO | ------------------------------------------ | 20260506 11:41:04.124 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:41:04.124 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:41:04.124 | INFO | Total mem usage: 25.6 GiB | 20260506 11:41:04.124 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:41:04.124 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260506 11:41:04.124 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260506 11:41:04.124 | INFO | High Water Breakdown: | 20260506 11:41:04.124 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260506 11:41:04.124 | INFO | ------------------------------------------ | 20260506 11:41:04.124 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:41:04.124 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:41:04.124 | INFO | Total mem usage: 15.7 GiB | 20260506 11:41:04.124 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:41:04.124 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:41:04.124 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 11:41:04.124 | INFO | High Water Breakdown: | 20260506 11:41:04.124 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:41:04.124 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:41:04.124 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:41:04.124 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:41:04.124 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:41:04.124 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:41:04.124 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:41:04.124 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:41:04.124 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:41:04.124 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:41:04.124 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:41:04.124 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:41:04.124 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:41:04.124 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:41:04.124 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:41:04.124 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:41:04.124 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:41:04.124 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:41:04.124 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:41:04.124 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:41:04.124 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:41:04.124 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:41:04.124 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:41:04.124 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:41:04.124 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:41:04.124 | INFO | ------------------------------------------ | 20260506 11:41:04.124 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:41:04.124 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:41:04.124 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:41:04.124 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:41:04.124 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.60e+01, 1.36e+01 / 2.00e+00 | 20260506 11:41:04.124 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.11e-02 / 6.39e-03 | 20260506 11:41:04.124 | INFO | PerfReport: 11.57%, Baseline -- 3.69e+00 / 4.49e+00, 5.90e-01 / 3.49e-01 | 20260506 11:41:04.124 | INFO | PerfReport: 3.23%, Histogram -- 1.03e+00 / 1.48e+00, 1.92e-01 / 1.33e-01 | 20260506 11:41:04.124 | INFO | PerfReport: 15.91%, DME -- 4.06e+01 / 4.53e+01, 4.19e+00 / 2.05e+00 | 20260506 11:41:04.124 | INFO | PerfReport: 56.10%, FrameLabeling -- 1.79e+01 / 1.94e+01, 1.35e+00 / 1.31e+00 | 20260506 11:41:04.124 | INFO | PerfReport: 7.69%, PulseAccumulating -- 2.45e+00 / 3.10e+00, 4.35e-01 / 2.00e-01 | 20260506 11:41:04.124 | INFO | PerfReport: 3.85%, Metrics -- 1.23e+00 / 1.62e+00, 3.10e-01 / 1.81e-01 | 20260506 11:41:04.124 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.33e-01 / 4.93e-01, 6.74e-02 / 3.03e-02 | 20260506 11:41:04.124 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:41:04.124 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:41:04.124 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:41:04.124 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260506 11:41:04.124 | INFO | PerfReport: 21.70%, Upload -- 2.15e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 11:41:04.124 | INFO | PerfReport: 21.10%, Download -- 2.09e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 11:41:04.124 | INFO | PerfReport: 32.24%, Compute -- 3.19e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 11:41:04.124 | INFO | PerfReport: 2.04%, LockUpload -- 2.01e+00 / 0.00e+00, 7.68e+00 / 0.00e+00 | 20260506 11:41:04.124 | INFO | PerfReport: 0.88%, LockDownload -- 8.67e-01 / 0.00e+00, 2.34e+00 / 0.00e+00 | 20260506 11:41:04.124 | INFO | PerfReport: 22.03%, LockCompute -- 2.18e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 11:41:04.124 | INFO | PerfReport: 0.02%, Misc -- 1.68e-02 / 0.00e+00, 3.65e-02 / 0.00e+00 | 20260506 11:41:04.262 | INFO | Starting a round of Baz IO | 20260506 11:41:07.427 | INFO | Populating RTMetrics report | 20260506 11:41:07.427 | INFO | RTMetrics report populated | 20260506 11:41:07.427 | INFO | Opening tmp RTMetrics json file | 20260506 11:41:07.430 | INFO | Serializing RTMetrics to json | 20260506 11:41:07.444 | INFO | Serialization done, swapping tmp file now... | 20260506 11:41:07.460 | INFO | RTMetrics done | 20260506 11:41:07.541 | INFO | Finished a round of Baz IO: Wrote 2.07 GiB in 3.279 seconds | 20260506 11:41:07.541 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.47% / 0.03% / 56.50% | 20260506 11:41:07.541 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.88826% / 6.35783e-05% / 3.97364e-05% | 20260506 11:41:07.541 | INFO | Baz IO overall: | 20260506 11:41:07.541 | INFO | BytesWritten : 519343732547 | 20260506 11:41:07.541 | INFO | NumEvents : 250703713617 | 20260506 11:41:09.261 | INFO | Analyzing chunk frames = [1073152,1073664) | 20260506 11:41:09.261 | INFO | MemInfo: { | 20260506 11:41:09.261 | INFO | "gpu" : " 17 GB", | 20260506 11:41:09.261 | INFO | "huge" : "", | 20260506 11:41:09.261 | INFO | "huge_shared" : " 27 GB", | 20260506 11:41:09.261 | INFO | "malloc" : " 14 GB", | 20260506 11:41:09.261 | INFO | "pinned" : " 29 GB" | 20260506 11:41:09.261 | INFO | } | 20260506 11:41:09.261 | INFO | Perf report for the last 5.13859 seconds: | 20260506 11:41:09.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:41:09.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:09.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:09.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:09.261 | INFO | ANALYSIS : 63.7%, 2.8, 95.2ms, 0 | 20260506 11:41:09.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:09.261 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260506 11:41:09.261 | INFO | PRE_HQ : 62.0%, 2.0, 63.3ms, 0 | 20260506 11:41:09.261 | INFO | BAZWRITER : 62.9%, 4.6, 148.6ms, 0 | 20260506 11:41:09.261 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 11:41:09.261 | INFO | Uploaded 36GiB to GPU in 2.13761s (16.8413GiB/s) | 20260506 11:41:09.261 | INFO | Downloaded 38GiB to GPU in 2.222s (17.1017GiB/s) | 20260506 11:41:14.401 | INFO | Analyzing chunk frames = [1073664,1074176) | 20260506 11:41:14.401 | INFO | MemInfo: { | 20260506 11:41:14.401 | INFO | "gpu" : " 17 GB", | 20260506 11:41:14.401 | INFO | "huge" : "", | 20260506 11:41:14.401 | INFO | "huge_shared" : " 27 GB", | 20260506 11:41:14.401 | INFO | "malloc" : " 14 GB", | 20260506 11:41:14.401 | INFO | "pinned" : " 29 GB" | 20260506 11:41:14.401 | INFO | } | 20260506 11:41:14.401 | INFO | Perf report for the last 5.13995 seconds: | 20260506 11:41:14.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:41:14.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:14.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:14.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:14.401 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:41:14.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:14.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:14.401 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260506 11:41:14.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:14.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:14.401 | INFO | Uploaded 36GiB to GPU in 2.03575s (17.6839GiB/s) | 20260506 11:41:14.401 | INFO | Downloaded 34GiB to GPU in 1.92526s (17.6599GiB/s) | 20260506 11:41:19.540 | INFO | Analyzing chunk frames = [1074176,1074688) | 20260506 11:41:19.540 | INFO | MemInfo: { | 20260506 11:41:19.540 | INFO | "gpu" : " 17 GB", | 20260506 11:41:19.540 | INFO | "huge" : "", | 20260506 11:41:19.540 | INFO | "huge_shared" : " 27 GB", | 20260506 11:41:19.540 | INFO | "malloc" : " 14 GB", | 20260506 11:41:19.540 | INFO | "pinned" : " 29 GB" | 20260506 11:41:19.540 | INFO | } | 20260506 11:41:19.540 | INFO | Perf report for the last 5.13881 seconds: | 20260506 11:41:19.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:41:19.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:19.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:19.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:19.540 | INFO | ANALYSIS : 62.4%, 3.0, 96.6ms, 0 | 20260506 11:41:19.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:19.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:19.540 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:41:19.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:19.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:19.540 | INFO | Uploaded 36GiB to GPU in 2.0686s (17.4031GiB/s) | 20260506 11:41:19.540 | INFO | Downloaded 34GiB to GPU in 1.95147s (17.4228GiB/s) | 20260506 11:41:24.679 | INFO | Analyzing chunk frames = [1074688,1075200) | 20260506 11:41:24.679 | INFO | MemInfo: { | 20260506 11:41:24.679 | INFO | "gpu" : " 17 GB", | 20260506 11:41:24.679 | INFO | "huge" : "", | 20260506 11:41:24.679 | INFO | "huge_shared" : " 27 GB", | 20260506 11:41:24.680 | INFO | "malloc" : " 14 GB", | 20260506 11:41:24.680 | INFO | "pinned" : " 29 GB" | 20260506 11:41:24.680 | INFO | } | 20260506 11:41:24.680 | INFO | Perf report for the last 5.13977 seconds: | 20260506 11:41:24.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:41:24.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:24.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:24.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:24.680 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260506 11:41:24.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:24.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:24.680 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:41:24.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:24.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:24.680 | INFO | Uploaded 36GiB to GPU in 2.08276s (17.2848GiB/s) | 20260506 11:41:24.680 | INFO | Downloaded 34GiB to GPU in 1.96882s (17.2692GiB/s) | 20260506 11:41:29.819 | INFO | Analyzing chunk frames = [1075200,1075712) | 20260506 11:41:29.819 | INFO | MemInfo: { | 20260506 11:41:29.819 | INFO | "gpu" : " 17 GB", | 20260506 11:41:29.819 | INFO | "huge" : "", | 20260506 11:41:29.819 | INFO | "huge_shared" : " 27 GB", | 20260506 11:41:29.819 | INFO | "malloc" : " 14 GB", | 20260506 11:41:29.819 | INFO | "pinned" : " 29 GB" | 20260506 11:41:29.819 | INFO | } | 20260506 11:41:29.819 | INFO | Perf report for the last 5.13957 seconds: | 20260506 11:41:29.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:41:29.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:29.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:29.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:29.819 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 11:41:29.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:29.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:29.819 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:41:29.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:29.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:29.819 | INFO | Uploaded 36GiB to GPU in 2.05753s (17.4967GiB/s) | 20260506 11:41:29.820 | INFO | Downloaded 34GiB to GPU in 1.98048s (17.1676GiB/s) | 20260506 11:41:34.958 | INFO | Analyzing chunk frames = [1075712,1076224) | 20260506 11:41:34.958 | INFO | MemInfo: { | 20260506 11:41:34.958 | INFO | "gpu" : " 17 GB", | 20260506 11:41:34.958 | INFO | "huge" : "", | 20260506 11:41:34.958 | INFO | "huge_shared" : " 27 GB", | 20260506 11:41:34.958 | INFO | "malloc" : " 14 GB", | 20260506 11:41:34.958 | INFO | "pinned" : " 29 GB" | 20260506 11:41:34.958 | INFO | } | 20260506 11:41:34.958 | INFO | Perf report for the last 5.13917 seconds: | 20260506 11:41:34.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:41:34.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:34.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:34.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:34.958 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 11:41:34.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:34.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:34.958 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:41:34.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:34.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:34.959 | INFO | Uploaded 36GiB to GPU in 2.09568s (17.1782GiB/s) | 20260506 11:41:34.959 | INFO | Downloaded 34GiB to GPU in 1.98898s (17.0942GiB/s) | 20260506 11:41:40.097 | INFO | Analyzing chunk frames = [1076224,1076736) | 20260506 11:41:40.097 | INFO | MemInfo: { | 20260506 11:41:40.097 | INFO | "gpu" : " 17 GB", | 20260506 11:41:40.098 | INFO | "huge" : "", | 20260506 11:41:40.098 | INFO | "huge_shared" : " 27 GB", | 20260506 11:41:40.098 | INFO | "malloc" : " 14 GB", | 20260506 11:41:40.098 | INFO | "pinned" : " 29 GB" | 20260506 11:41:40.098 | INFO | } | 20260506 11:41:40.098 | INFO | Perf report for the last 5.13944 seconds: | 20260506 11:41:40.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:41:40.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:40.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:40.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:40.098 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:41:40.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:40.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:40.098 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 11:41:40.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:40.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:40.098 | INFO | Uploaded 36GiB to GPU in 2.02312s (17.7943GiB/s) | 20260506 11:41:40.098 | INFO | Downloaded 34GiB to GPU in 1.9996s (17.0034GiB/s) | 20260506 11:41:45.237 | INFO | Analyzing chunk frames = [1076736,1077248) | 20260506 11:41:45.237 | INFO | MemInfo: { | 20260506 11:41:45.237 | INFO | "gpu" : " 17 GB", | 20260506 11:41:45.237 | INFO | "huge" : "", | 20260506 11:41:45.237 | INFO | "huge_shared" : " 27 GB", | 20260506 11:41:45.237 | INFO | "malloc" : " 14 GB", | 20260506 11:41:45.237 | INFO | "pinned" : " 29 GB" | 20260506 11:41:45.237 | INFO | } | 20260506 11:41:45.237 | INFO | Perf report for the last 5.13928 seconds: | 20260506 11:41:45.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:41:45.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:45.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:45.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:45.237 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260506 11:41:45.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:45.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:45.237 | INFO | PRE_HQ : 33.4%, 1.0, 17.4ms, 0 | 20260506 11:41:45.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:45.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:45.237 | INFO | Uploaded 36GiB to GPU in 2.07432s (17.3551GiB/s) | 20260506 11:41:45.237 | INFO | Downloaded 34GiB to GPU in 2.00636s (16.9461GiB/s) | 20260506 11:41:45.354 | INFO | Starting a round of Baz IO | 20260506 11:41:48.570 | INFO | Populating RTMetrics report | 20260506 11:41:48.571 | INFO | RTMetrics report populated | 20260506 11:41:48.571 | INFO | Opening tmp RTMetrics json file | 20260506 11:41:48.633 | INFO | Serializing RTMetrics to json | 20260506 11:41:48.705 | INFO | Serialization done, swapping tmp file now... | 20260506 11:41:48.706 | INFO | RTMetrics done | 20260506 11:41:49.156 | INFO | Finished a round of Baz IO: Wrote 2.071 GiB in 3.802 seconds | 20260506 11:41:49.156 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.44% / 0.03% / 56.53% | 20260506 11:41:49.156 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.87566% / 7.15256e-05% / 4.76837e-05% | 20260506 11:41:49.156 | INFO | Baz IO overall: | 20260506 11:41:49.156 | INFO | BytesWritten : 521567036166 | 20260506 11:41:49.157 | INFO | NumEvents : 251783213062 | 20260506 11:41:50.376 | INFO | Analyzing chunk frames = [1077248,1077760) | 20260506 11:41:50.376 | INFO | MemInfo: { | 20260506 11:41:50.376 | INFO | "gpu" : " 17 GB", | 20260506 11:41:50.376 | INFO | "huge" : "", | 20260506 11:41:50.376 | INFO | "huge_shared" : " 27 GB", | 20260506 11:41:50.376 | INFO | "malloc" : " 14 GB", | 20260506 11:41:50.376 | INFO | "pinned" : " 29 GB" | 20260506 11:41:50.376 | INFO | } | 20260506 11:41:50.376 | INFO | Perf report for the last 5.13906 seconds: | 20260506 11:41:50.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:41:50.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:50.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:50.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:50.376 | INFO | ANALYSIS : 63.9%, 2.9, 96.4ms, 0 | 20260506 11:41:50.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:50.376 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260506 11:41:50.376 | INFO | PRE_HQ : 61.8%, 1.9, 61.2ms, 0 | 20260506 11:41:50.376 | INFO | BAZWRITER : 73.2%, 4.8, 180.7ms, 0 | 20260506 11:41:50.376 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.5ms, 0 | 20260506 11:41:50.377 | INFO | Uploaded 36GiB to GPU in 2.0925s (17.2043GiB/s) | 20260506 11:41:50.377 | INFO | Downloaded 38GiB to GPU in 2.21786s (17.1336GiB/s) | 20260506 11:41:55.515 | INFO | Analyzing chunk frames = [1077760,1078272) | 20260506 11:41:55.515 | INFO | MemInfo: { | 20260506 11:41:55.515 | INFO | "gpu" : " 17 GB", | 20260506 11:41:55.515 | INFO | "huge" : "", | 20260506 11:41:55.515 | INFO | "huge_shared" : " 27 GB", | 20260506 11:41:55.515 | INFO | "malloc" : " 14 GB", | 20260506 11:41:55.515 | INFO | "pinned" : " 29 GB" | 20260506 11:41:55.515 | INFO | } | 20260506 11:41:55.515 | INFO | Perf report for the last 5.13914 seconds: | 20260506 11:41:55.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:41:55.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:55.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:55.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:55.515 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:41:55.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:55.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:55.515 | INFO | PRE_HQ : 32.4%, 1.0, 17.0ms, 0 | 20260506 11:41:55.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:55.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:41:55.516 | INFO | Uploaded 36GiB to GPU in 1.99688s (18.0282GiB/s) | 20260506 11:41:55.516 | INFO | Downloaded 34GiB to GPU in 1.92048s (17.7039GiB/s) | 20260506 11:42:00.654 | INFO | Analyzing chunk frames = [1078272,1078784) | 20260506 11:42:00.654 | INFO | MemInfo: { | 20260506 11:42:00.654 | INFO | "gpu" : " 17 GB", | 20260506 11:42:00.655 | INFO | "huge" : "", | 20260506 11:42:00.655 | INFO | "huge_shared" : " 27 GB", | 20260506 11:42:00.655 | INFO | "malloc" : " 14 GB", | 20260506 11:42:00.655 | INFO | "pinned" : " 29 GB" | 20260506 11:42:00.655 | INFO | } | 20260506 11:42:00.655 | INFO | Perf report for the last 5.13939 seconds: | 20260506 11:42:00.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:42:00.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:00.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:00.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:00.655 | INFO | ANALYSIS : 61.9%, 3.0, 95.9ms, 0 | 20260506 11:42:00.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:00.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:00.655 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:42:00.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:00.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:00.655 | INFO | Uploaded 36GiB to GPU in 2.02808s (17.7508GiB/s) | 20260506 11:42:00.655 | INFO | Downloaded 34GiB to GPU in 1.94535s (17.4776GiB/s) | 20260506 11:42:05.794 | INFO | Analyzing chunk frames = [1078784,1079296) | 20260506 11:42:05.794 | INFO | MemInfo: { | 20260506 11:42:05.794 | INFO | "gpu" : " 17 GB", | 20260506 11:42:05.794 | INFO | "huge" : "", | 20260506 11:42:05.794 | INFO | "huge_shared" : " 27 GB", | 20260506 11:42:05.794 | INFO | "malloc" : " 14 GB", | 20260506 11:42:05.794 | INFO | "pinned" : " 29 GB" | 20260506 11:42:05.794 | INFO | } | 20260506 11:42:05.794 | INFO | Perf report for the last 5.13962 seconds: | 20260506 11:42:05.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:42:05.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:05.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:05.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:05.794 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 11:42:05.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:05.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:05.795 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:42:05.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:05.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:05.795 | INFO | Uploaded 36GiB to GPU in 2.06223s (17.4569GiB/s) | 20260506 11:42:05.795 | INFO | Downloaded 34GiB to GPU in 1.96992s (17.2596GiB/s) | 20260506 11:42:10.933 | INFO | Analyzing chunk frames = [1079296,1079808) | 20260506 11:42:10.933 | INFO | MemInfo: { | 20260506 11:42:10.933 | INFO | "gpu" : " 17 GB", | 20260506 11:42:10.933 | INFO | "huge" : "", | 20260506 11:42:10.933 | INFO | "huge_shared" : " 27 GB", | 20260506 11:42:10.933 | INFO | "malloc" : " 14 GB", | 20260506 11:42:10.933 | INFO | "pinned" : " 29 GB" | 20260506 11:42:10.934 | INFO | } | 20260506 11:42:10.934 | INFO | Perf report for the last 5.13925 seconds: | 20260506 11:42:10.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:42:10.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:10.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:10.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:10.934 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 11:42:10.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:10.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:10.934 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 11:42:10.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:10.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:10.934 | INFO | Uploaded 36GiB to GPU in 2.06472s (17.4358GiB/s) | 20260506 11:42:10.934 | INFO | Downloaded 34GiB to GPU in 1.9853s (17.1259GiB/s) | 20260506 11:42:16.072 | INFO | Analyzing chunk frames = [1079808,1080320) | 20260506 11:42:16.072 | INFO | MemInfo: { | 20260506 11:42:16.072 | INFO | "gpu" : " 17 GB", | 20260506 11:42:16.072 | INFO | "huge" : "", | 20260506 11:42:16.072 | INFO | "huge_shared" : " 27 GB", | 20260506 11:42:16.072 | INFO | "malloc" : " 14 GB", | 20260506 11:42:16.073 | INFO | "pinned" : " 29 GB" | 20260506 11:42:16.073 | INFO | } | 20260506 11:42:16.073 | INFO | Perf report for the last 5.13901 seconds: | 20260506 11:42:16.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:42:16.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:16.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:16.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:16.073 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 11:42:16.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:16.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:16.073 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:42:16.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:16.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:16.073 | INFO | Uploaded 36GiB to GPU in 2.08181s (17.2926GiB/s) | 20260506 11:42:16.073 | INFO | Downloaded 34GiB to GPU in 2.00311s (16.9736GiB/s) | 20260506 11:42:21.213 | INFO | Analyzing chunk frames = [1080320,1080832) | 20260506 11:42:21.213 | INFO | MemInfo: { | 20260506 11:42:21.213 | INFO | "gpu" : " 17 GB", | 20260506 11:42:21.213 | INFO | "huge" : "", | 20260506 11:42:21.213 | INFO | "huge_shared" : " 27 GB", | 20260506 11:42:21.213 | INFO | "malloc" : " 14 GB", | 20260506 11:42:21.213 | INFO | "pinned" : " 29 GB" | 20260506 11:42:21.213 | INFO | } | 20260506 11:42:21.213 | INFO | Perf report for the last 5.14072 seconds: | 20260506 11:42:21.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:42:21.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:21.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:21.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:21.213 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 11:42:21.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:21.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:21.214 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 11:42:21.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:21.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:21.214 | INFO | Uploaded 36GiB to GPU in 2.00133s (17.988GiB/s) | 20260506 11:42:21.214 | INFO | Downloaded 34GiB to GPU in 2.01875s (16.8421GiB/s) | 20260506 11:42:26.351 | INFO | Analyzing chunk frames = [1080832,1081344) | 20260506 11:42:26.351 | INFO | MemInfo: { | 20260506 11:42:26.351 | INFO | "gpu" : " 17 GB", | 20260506 11:42:26.351 | INFO | "huge" : "", | 20260506 11:42:26.352 | INFO | "huge_shared" : " 27 GB", | 20260506 11:42:26.352 | INFO | "malloc" : " 14 GB", | 20260506 11:42:26.352 | INFO | "pinned" : " 29 GB" | 20260506 11:42:26.352 | INFO | } | 20260506 11:42:26.352 | INFO | Perf report for the last 5.13835 seconds: | 20260506 11:42:26.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:42:26.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:26.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:26.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:26.352 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260506 11:42:26.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:26.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:26.352 | INFO | PRE_HQ : 33.2%, 1.0, 17.2ms, 0 | 20260506 11:42:26.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:26.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:26.352 | INFO | Uploaded 36GiB to GPU in 1.97135s (18.2616GiB/s) | 20260506 11:42:26.352 | INFO | Downloaded 34GiB to GPU in 2.0056s (16.9525GiB/s) | 20260506 11:42:26.352 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:42:26.352 | INFO | Total mem usage: 16.8 GiB | 20260506 11:42:26.352 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:42:26.352 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: -2.04%) | 20260506 11:42:26.352 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:42:26.352 | INFO | High Water Breakdown: | 20260506 11:42:26.352 | INFO | 61.6% : 10.1 GiB ( 97.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:42:26.352 | INFO | 38.3% : 6.31 GiB ( 94.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:42:26.352 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:42:26.352 | INFO | ------------------------------------------ | 20260506 11:42:26.352 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:42:26.353 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:42:26.353 | INFO | Total mem usage: 26.5 GiB | 20260506 11:42:26.353 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:42:26.353 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:42:26.353 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:42:26.353 | INFO | High Water Breakdown: | 20260506 11:42:26.353 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:42:26.353 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:42:26.353 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:42:26.353 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:42:26.353 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:42:26.353 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:42:26.353 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:42:26.353 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:42:26.353 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:42:26.353 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:42:26.353 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:42:26.353 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:42:26.353 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:42:26.353 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:42:26.353 | INFO | ------------------------------------------ | 20260506 11:42:26.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:42:26.353 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:42:26.353 | INFO | Total mem usage: 25.6 GiB | 20260506 11:42:26.353 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:42:26.353 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:42:26.353 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:42:26.353 | INFO | High Water Breakdown: | 20260506 11:42:26.353 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:42:26.353 | INFO | ------------------------------------------ | 20260506 11:42:26.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:42:26.353 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:42:26.353 | INFO | Total mem usage: 15.7 GiB | 20260506 11:42:26.353 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:42:26.353 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:42:26.353 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:42:26.353 | INFO | High Water Breakdown: | 20260506 11:42:26.353 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:42:26.353 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:42:26.353 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:42:26.353 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:42:26.353 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:42:26.353 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:42:26.353 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:42:26.353 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:42:26.353 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:42:26.353 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:42:26.353 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:42:26.353 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:42:26.353 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:42:26.353 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:42:26.353 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:42:26.353 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:42:26.353 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:42:26.353 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:42:26.353 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:42:26.353 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:42:26.353 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:42:26.353 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:42:26.353 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:42:26.353 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:42:26.353 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:42:26.353 | INFO | ------------------------------------------ | 20260506 11:42:26.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:42:26.353 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:42:26.353 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:42:26.353 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:42:26.353 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.58e+01, 1.38e+01 / 2.23e+00 | 20260506 11:42:26.353 | INFO | PerfReport: 0.29%, AggregateStats -- 9.31e-02 / 1.01e-01, 1.03e-02 / 6.51e-03 | 20260506 11:42:26.353 | INFO | PerfReport: 11.57%, Baseline -- 3.69e+00 / 4.48e+00, 5.87e-01 / 3.30e-01 | 20260506 11:42:26.353 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.45e+00, 1.94e-01 / 1.51e-01 | 20260506 11:42:26.353 | INFO | PerfReport: 15.85%, DME -- 4.04e+01 / 4.49e+01, 4.16e+00 / 2.17e+00 | 20260506 11:42:26.353 | INFO | PerfReport: 56.13%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.38e+00 / 1.52e+00 | 20260506 11:42:26.353 | INFO | PerfReport: 7.70%, PulseAccumulating -- 2.46e+00 / 3.12e+00, 4.34e-01 / 1.98e-01 | 20260506 11:42:26.353 | INFO | PerfReport: 3.85%, Metrics -- 1.23e+00 / 1.63e+00, 3.13e-01 / 1.67e-01 | 20260506 11:42:26.353 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.34e-01 / 4.94e-01, 6.79e-02 / 2.83e-02 | 20260506 11:42:26.353 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:42:26.353 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:42:26.353 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:42:26.353 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 11:42:26.353 | INFO | PerfReport: 21.66%, Upload -- 2.14e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260506 11:42:26.353 | INFO | PerfReport: 21.16%, Download -- 2.09e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 11:42:26.353 | INFO | PerfReport: 32.28%, Compute -- 3.19e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260506 11:42:26.354 | INFO | PerfReport: 1.93%, LockUpload -- 1.91e+00 / 0.00e+00, 7.56e+00 / 0.00e+00 | 20260506 11:42:26.354 | INFO | PerfReport: 0.97%, LockDownload -- 9.63e-01 / 0.00e+00, 2.48e+00 / 0.00e+00 | 20260506 11:42:26.354 | INFO | PerfReport: 21.98%, LockCompute -- 2.17e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 11:42:26.354 | INFO | PerfReport: 0.02%, Misc -- 1.51e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260506 11:42:26.482 | INFO | Starting a round of Baz IO | 20260506 11:42:29.653 | INFO | Populating RTMetrics report | 20260506 11:42:29.653 | INFO | RTMetrics report populated | 20260506 11:42:29.653 | INFO | Opening tmp RTMetrics json file | 20260506 11:42:29.668 | INFO | Serializing RTMetrics to json | 20260506 11:42:29.672 | INFO | Serialization done, swapping tmp file now... | 20260506 11:42:29.676 | INFO | RTMetrics done | 20260506 11:42:30.054 | INFO | Finished a round of Baz IO: Wrote 2.072 GiB in 3.571 seconds | 20260506 11:42:30.054 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.41% / 0.03% / 56.56% | 20260506 11:42:30.054 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.86983% / 7.54992e-05% / 5.96046e-05% | 20260506 11:42:30.054 | INFO | Baz IO overall: | 20260506 11:42:30.054 | INFO | BytesWritten : 523792290809 | 20260506 11:42:30.054 | INFO | NumEvents : 252864133664 | 20260506 11:42:31.491 | INFO | Analyzing chunk frames = [1081344,1081856) | 20260506 11:42:31.491 | INFO | MemInfo: { | 20260506 11:42:31.491 | INFO | "gpu" : " 17 GB", | 20260506 11:42:31.491 | INFO | "huge" : "", | 20260506 11:42:31.491 | INFO | "huge_shared" : " 27 GB", | 20260506 11:42:31.491 | INFO | "malloc" : " 14 GB", | 20260506 11:42:31.491 | INFO | "pinned" : " 29 GB" | 20260506 11:42:31.491 | INFO | } | 20260506 11:42:31.491 | INFO | Perf report for the last 5.139 seconds: | 20260506 11:42:31.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:42:31.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:31.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:31.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:31.491 | INFO | ANALYSIS : 63.6%, 2.9, 95.1ms, 0 | 20260506 11:42:31.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:31.491 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260506 11:42:31.491 | INFO | PRE_HQ : 61.9%, 2.0, 62.7ms, 0 | 20260506 11:42:31.491 | INFO | BAZWRITER : 68.8%, 4.8, 169.4ms, 0 | 20260506 11:42:31.491 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 11:42:31.491 | INFO | Uploaded 36GiB to GPU in 2.12236s (16.9622GiB/s) | 20260506 11:42:31.491 | INFO | Downloaded 38GiB to GPU in 2.23008s (17.0398GiB/s) | 20260506 11:42:36.630 | INFO | Analyzing chunk frames = [1081856,1082368) | 20260506 11:42:36.630 | INFO | MemInfo: { | 20260506 11:42:36.630 | INFO | "gpu" : " 17 GB", | 20260506 11:42:36.630 | INFO | "huge" : "", | 20260506 11:42:36.630 | INFO | "huge_shared" : " 27 GB", | 20260506 11:42:36.630 | INFO | "malloc" : " 14 GB", | 20260506 11:42:36.630 | INFO | "pinned" : " 29 GB" | 20260506 11:42:36.630 | INFO | } | 20260506 11:42:36.630 | INFO | Perf report for the last 5.13969 seconds: | 20260506 11:42:36.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:42:36.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:36.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:36.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:36.631 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:42:36.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:36.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:36.631 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 11:42:36.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:36.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:36.631 | INFO | Uploaded 36GiB to GPU in 2.02107s (17.8123GiB/s) | 20260506 11:42:36.631 | INFO | Downloaded 34GiB to GPU in 1.93575s (17.5643GiB/s) | 20260506 11:42:41.770 | INFO | Analyzing chunk frames = [1082368,1082880) | 20260506 11:42:41.770 | INFO | MemInfo: { | 20260506 11:42:41.770 | INFO | "gpu" : " 17 GB", | 20260506 11:42:41.770 | INFO | "huge" : "", | 20260506 11:42:41.770 | INFO | "huge_shared" : " 27 GB", | 20260506 11:42:41.770 | INFO | "malloc" : " 14 GB", | 20260506 11:42:41.770 | INFO | "pinned" : " 29 GB" | 20260506 11:42:41.770 | INFO | } | 20260506 11:42:41.770 | INFO | Perf report for the last 5.13925 seconds: | 20260506 11:42:41.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:42:41.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:41.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:41.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:41.770 | INFO | ANALYSIS : 61.9%, 3.0, 95.9ms, 0 | 20260506 11:42:41.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:41.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:41.770 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:42:41.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:41.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:41.770 | INFO | Uploaded 36GiB to GPU in 2.00708s (17.9365GiB/s) | 20260506 11:42:41.770 | INFO | Downloaded 34GiB to GPU in 1.94277s (17.5008GiB/s) | 20260506 11:42:46.909 | INFO | Analyzing chunk frames = [1082880,1083392) | 20260506 11:42:46.909 | INFO | MemInfo: { | 20260506 11:42:46.909 | INFO | "gpu" : " 17 GB", | 20260506 11:42:46.909 | INFO | "huge" : "", | 20260506 11:42:46.909 | INFO | "huge_shared" : " 27 GB", | 20260506 11:42:46.909 | INFO | "malloc" : " 14 GB", | 20260506 11:42:46.909 | INFO | "pinned" : " 29 GB" | 20260506 11:42:46.909 | INFO | } | 20260506 11:42:46.909 | INFO | Perf report for the last 5.13914 seconds: | 20260506 11:42:46.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:42:46.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:46.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:46.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:46.909 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 11:42:46.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:46.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:46.909 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:42:46.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:46.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:46.909 | INFO | Uploaded 36GiB to GPU in 2.0702s (17.3896GiB/s) | 20260506 11:42:46.909 | INFO | Downloaded 34GiB to GPU in 1.97255s (17.2365GiB/s) | 20260506 11:42:52.048 | INFO | Analyzing chunk frames = [1083392,1083904) | 20260506 11:42:52.048 | INFO | MemInfo: { | 20260506 11:42:52.048 | INFO | "gpu" : " 17 GB", | 20260506 11:42:52.048 | INFO | "huge" : "", | 20260506 11:42:52.048 | INFO | "huge_shared" : " 27 GB", | 20260506 11:42:52.048 | INFO | "malloc" : " 14 GB", | 20260506 11:42:52.048 | INFO | "pinned" : " 29 GB" | 20260506 11:42:52.048 | INFO | } | 20260506 11:42:52.048 | INFO | Perf report for the last 5.13925 seconds: | 20260506 11:42:52.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:42:52.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:52.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:52.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:52.048 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 11:42:52.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:52.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:52.048 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:42:52.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:52.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:52.048 | INFO | Uploaded 36GiB to GPU in 2.08781s (17.2429GiB/s) | 20260506 11:42:52.049 | INFO | Downloaded 34GiB to GPU in 1.99426s (17.0489GiB/s) | 20260506 11:42:57.187 | INFO | Analyzing chunk frames = [1083904,1084416) | 20260506 11:42:57.187 | INFO | MemInfo: { | 20260506 11:42:57.187 | INFO | "gpu" : " 17 GB", | 20260506 11:42:57.187 | INFO | "huge" : "", | 20260506 11:42:57.187 | INFO | "huge_shared" : " 27 GB", | 20260506 11:42:57.187 | INFO | "malloc" : " 14 GB", | 20260506 11:42:57.187 | INFO | "pinned" : " 29 GB" | 20260506 11:42:57.187 | INFO | } | 20260506 11:42:57.188 | INFO | Perf report for the last 5.1394 seconds: | 20260506 11:42:57.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:42:57.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:57.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:57.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:57.188 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 11:42:57.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:57.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:57.188 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:42:57.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:57.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:42:57.188 | INFO | Uploaded 36GiB to GPU in 2.07369s (17.3603GiB/s) | 20260506 11:42:57.188 | INFO | Downloaded 34GiB to GPU in 2.00288s (16.9756GiB/s) | 20260506 11:43:02.327 | INFO | Analyzing chunk frames = [1084416,1084928) | 20260506 11:43:02.327 | INFO | MemInfo: { | 20260506 11:43:02.327 | INFO | "gpu" : " 17 GB", | 20260506 11:43:02.327 | INFO | "huge" : "", | 20260506 11:43:02.327 | INFO | "huge_shared" : " 27 GB", | 20260506 11:43:02.327 | INFO | "malloc" : " 14 GB", | 20260506 11:43:02.327 | INFO | "pinned" : " 29 GB" | 20260506 11:43:02.327 | INFO | } | 20260506 11:43:02.328 | INFO | Perf report for the last 5.13996 seconds: | 20260506 11:43:02.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:43:02.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:02.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:02.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:02.328 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260506 11:43:02.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:02.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:02.328 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 11:43:02.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:02.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:02.328 | INFO | Uploaded 36GiB to GPU in 1.99607s (18.0355GiB/s) | 20260506 11:43:02.328 | INFO | Downloaded 34GiB to GPU in 2.01916s (16.8387GiB/s) | 20260506 11:43:07.467 | INFO | Analyzing chunk frames = [1084928,1085440) | 20260506 11:43:07.467 | INFO | MemInfo: { | 20260506 11:43:07.467 | INFO | "gpu" : " 17 GB", | 20260506 11:43:07.467 | INFO | "huge" : "", | 20260506 11:43:07.467 | INFO | "huge_shared" : " 27 GB", | 20260506 11:43:07.467 | INFO | "malloc" : " 14 GB", | 20260506 11:43:07.467 | INFO | "pinned" : " 29 GB" | 20260506 11:43:07.467 | INFO | } | 20260506 11:43:07.467 | INFO | Perf report for the last 5.13941 seconds: | 20260506 11:43:07.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:43:07.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:07.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:07.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:07.467 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:43:07.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:07.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:07.467 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 11:43:07.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:07.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:07.467 | INFO | Uploaded 36GiB to GPU in 2.02922s (17.7408GiB/s) | 20260506 11:43:07.467 | INFO | Downloaded 34GiB to GPU in 1.96938s (17.2643GiB/s) | 20260506 11:43:07.591 | INFO | Starting a round of Baz IO | 20260506 11:43:10.789 | INFO | Populating RTMetrics report | 20260506 11:43:10.789 | INFO | RTMetrics report populated | 20260506 11:43:10.789 | INFO | Opening tmp RTMetrics json file | 20260506 11:43:10.828 | INFO | Serializing RTMetrics to json | 20260506 11:43:10.863 | INFO | Serialization done, swapping tmp file now... | 20260506 11:43:10.872 | INFO | RTMetrics done | 20260506 11:43:10.961 | INFO | Finished a round of Baz IO: Wrote 2.071 GiB in 3.369 seconds | 20260506 11:43:10.961 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.38% / 0.03% / 56.59% | 20260506 11:43:10.961 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.84095% / 7.15256e-05% / 3.17891e-05% | 20260506 11:43:10.961 | INFO | Baz IO overall: | 20260506 11:43:10.961 | INFO | BytesWritten : 526016000119 | 20260506 11:43:10.961 | INFO | NumEvents : 253943463235 | 20260506 11:43:12.606 | INFO | Analyzing chunk frames = [1085440,1085952) | 20260506 11:43:12.606 | INFO | MemInfo: { | 20260506 11:43:12.606 | INFO | "gpu" : " 17 GB", | 20260506 11:43:12.606 | INFO | "huge" : "", | 20260506 11:43:12.606 | INFO | "huge_shared" : " 27 GB", | 20260506 11:43:12.606 | INFO | "malloc" : " 14 GB", | 20260506 11:43:12.606 | INFO | "pinned" : " 29 GB" | 20260506 11:43:12.606 | INFO | } | 20260506 11:43:12.606 | INFO | Perf report for the last 5.13902 seconds: | 20260506 11:43:12.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:43:12.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:12.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:12.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:12.606 | INFO | ANALYSIS : 64.0%, 2.9, 96.4ms, 0 | 20260506 11:43:12.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:12.606 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260506 11:43:12.606 | INFO | PRE_HQ : 62.1%, 1.9, 61.3ms, 0 | 20260506 11:43:12.606 | INFO | BAZWRITER : 64.7%, 4.8, 158.9ms, 0 | 20260506 11:43:12.606 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 11:43:12.606 | INFO | Uploaded 36GiB to GPU in 2.15043s (16.7408GiB/s) | 20260506 11:43:12.607 | INFO | Downloaded 38GiB to GPU in 2.2372s (16.9855GiB/s) | 20260506 11:43:17.745 | INFO | Analyzing chunk frames = [1085952,1086464) | 20260506 11:43:17.745 | INFO | MemInfo: { | 20260506 11:43:17.745 | INFO | "gpu" : " 17 GB", | 20260506 11:43:17.745 | INFO | "huge" : "", | 20260506 11:43:17.745 | INFO | "huge_shared" : " 27 GB", | 20260506 11:43:17.745 | INFO | "malloc" : " 14 GB", | 20260506 11:43:17.745 | INFO | "pinned" : " 29 GB" | 20260506 11:43:17.745 | INFO | } | 20260506 11:43:17.745 | INFO | Perf report for the last 5.13875 seconds: | 20260506 11:43:17.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:43:17.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:17.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:17.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:17.745 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260506 11:43:17.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:17.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:17.745 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 11:43:17.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:17.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:17.745 | INFO | Uploaded 36GiB to GPU in 2.06483s (17.4348GiB/s) | 20260506 11:43:17.745 | INFO | Downloaded 34GiB to GPU in 1.95832s (17.3618GiB/s) | 20260506 11:43:22.884 | INFO | Analyzing chunk frames = [1086464,1086976) | 20260506 11:43:22.884 | INFO | MemInfo: { | 20260506 11:43:22.884 | INFO | "gpu" : " 17 GB", | 20260506 11:43:22.884 | INFO | "huge" : "", | 20260506 11:43:22.884 | INFO | "huge_shared" : " 27 GB", | 20260506 11:43:22.884 | INFO | "malloc" : " 14 GB", | 20260506 11:43:22.884 | INFO | "pinned" : " 29 GB" | 20260506 11:43:22.884 | INFO | } | 20260506 11:43:22.884 | INFO | Perf report for the last 5.13952 seconds: | 20260506 11:43:22.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:43:22.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:22.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:22.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:22.884 | INFO | ANALYSIS : 62.4%, 3.0, 96.7ms, 0 | 20260506 11:43:22.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:22.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:22.884 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:43:22.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:22.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:22.885 | INFO | Uploaded 36GiB to GPU in 2.07291s (17.3669GiB/s) | 20260506 11:43:22.885 | INFO | Downloaded 34GiB to GPU in 1.96627s (17.2916GiB/s) | 20260506 11:43:28.023 | INFO | Analyzing chunk frames = [1086976,1087488) | 20260506 11:43:28.023 | INFO | MemInfo: { | 20260506 11:43:28.023 | INFO | "gpu" : " 17 GB", | 20260506 11:43:28.023 | INFO | "huge" : "", | 20260506 11:43:28.023 | INFO | "huge_shared" : " 27 GB", | 20260506 11:43:28.023 | INFO | "malloc" : " 14 GB", | 20260506 11:43:28.024 | INFO | "pinned" : " 29 GB" | 20260506 11:43:28.024 | INFO | } | 20260506 11:43:28.024 | INFO | Perf report for the last 5.13939 seconds: | 20260506 11:43:28.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:43:28.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:28.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:28.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:28.024 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 11:43:28.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:28.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:28.024 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:43:28.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:28.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:28.024 | INFO | Uploaded 36GiB to GPU in 2.05872s (17.4866GiB/s) | 20260506 11:43:28.024 | INFO | Downloaded 34GiB to GPU in 1.97616s (17.2051GiB/s) | 20260506 11:43:33.163 | INFO | Analyzing chunk frames = [1087488,1088000) | 20260506 11:43:33.163 | INFO | MemInfo: { | 20260506 11:43:33.163 | INFO | "gpu" : " 17 GB", | 20260506 11:43:33.163 | INFO | "huge" : "", | 20260506 11:43:33.163 | INFO | "huge_shared" : " 27 GB", | 20260506 11:43:33.163 | INFO | "malloc" : " 14 GB", | 20260506 11:43:33.163 | INFO | "pinned" : " 29 GB" | 20260506 11:43:33.163 | INFO | } | 20260506 11:43:33.163 | INFO | Perf report for the last 5.13945 seconds: | 20260506 11:43:33.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:43:33.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:33.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:33.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:33.163 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 11:43:33.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:33.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:33.163 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 11:43:33.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:33.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:33.164 | INFO | Uploaded 36GiB to GPU in 2.08352s (17.2784GiB/s) | 20260506 11:43:33.164 | INFO | Downloaded 34GiB to GPU in 1.99214s (17.067GiB/s) | 20260506 11:43:38.302 | INFO | Analyzing chunk frames = [1088000,1088512) | 20260506 11:43:38.302 | INFO | MemInfo: { | 20260506 11:43:38.302 | INFO | "gpu" : " 17 GB", | 20260506 11:43:38.302 | INFO | "huge" : "", | 20260506 11:43:38.302 | INFO | "huge_shared" : " 27 GB", | 20260506 11:43:38.302 | INFO | "malloc" : " 14 GB", | 20260506 11:43:38.302 | INFO | "pinned" : " 29 GB" | 20260506 11:43:38.302 | INFO | } | 20260506 11:43:38.302 | INFO | Perf report for the last 5.13906 seconds: | 20260506 11:43:38.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:43:38.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:38.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:38.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:38.302 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 11:43:38.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:38.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:38.302 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:43:38.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:38.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:38.302 | INFO | Uploaded 36GiB to GPU in 2.06719s (17.4149GiB/s) | 20260506 11:43:38.303 | INFO | Downloaded 34GiB to GPU in 2.00792s (16.9329GiB/s) | 20260506 11:43:43.443 | INFO | Analyzing chunk frames = [1088512,1089024) | 20260506 11:43:43.443 | INFO | MemInfo: { | 20260506 11:43:43.443 | INFO | "gpu" : " 17 GB", | 20260506 11:43:43.443 | INFO | "huge" : "", | 20260506 11:43:43.443 | INFO | "huge_shared" : " 27 GB", | 20260506 11:43:43.443 | INFO | "malloc" : " 14 GB", | 20260506 11:43:43.443 | INFO | "pinned" : " 29 GB" | 20260506 11:43:43.443 | INFO | } | 20260506 11:43:43.443 | INFO | Perf report for the last 5.1406 seconds: | 20260506 11:43:43.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:43:43.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:43.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:43.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:43.443 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:43:43.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:43.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:43.443 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 11:43:43.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:43.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:43.443 | INFO | Uploaded 36GiB to GPU in 2.00532s (17.9522GiB/s) | 20260506 11:43:43.443 | INFO | Downloaded 34GiB to GPU in 2.02121s (16.8216GiB/s) | 20260506 11:43:48.581 | INFO | Analyzing chunk frames = [1089024,1089536) | 20260506 11:43:48.581 | INFO | MemInfo: { | 20260506 11:43:48.581 | INFO | "gpu" : " 17 GB", | 20260506 11:43:48.581 | INFO | "huge" : "", | 20260506 11:43:48.581 | INFO | "huge_shared" : " 27 GB", | 20260506 11:43:48.581 | INFO | "malloc" : " 14 GB", | 20260506 11:43:48.581 | INFO | "pinned" : " 29 GB" | 20260506 11:43:48.581 | INFO | } | 20260506 11:43:48.581 | INFO | Perf report for the last 5.13816 seconds: | 20260506 11:43:48.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:43:48.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:48.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:48.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:48.581 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260506 11:43:48.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:48.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:48.581 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260506 11:43:48.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:48.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:48.581 | INFO | Uploaded 36GiB to GPU in 2.04158s (17.6334GiB/s) | 20260506 11:43:48.581 | INFO | Downloaded 34GiB to GPU in 1.97327s (17.2303GiB/s) | 20260506 11:43:48.582 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:43:48.582 | INFO | Total mem usage: 16.8 GiB | 20260506 11:43:48.582 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:43:48.582 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -9.24%) | 20260506 11:43:48.582 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:43:48.582 | INFO | High Water Breakdown: | 20260506 11:43:48.582 | INFO | 60.5% : 9.31 GiB ( 89.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:43:48.582 | INFO | 39.4% : 6.06 GiB ( 90.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:43:48.582 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:43:48.582 | INFO | ------------------------------------------ | 20260506 11:43:48.582 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:43:48.582 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:43:48.582 | INFO | Total mem usage: 26.5 GiB | 20260506 11:43:48.582 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:43:48.582 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 11:43:48.582 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 11:43:48.582 | INFO | High Water Breakdown: | 20260506 11:43:48.582 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:43:48.582 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:43:48.582 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:43:48.582 | INFO | 2% : 523 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:43:48.582 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:43:48.582 | INFO | 0.557% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:43:48.582 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:43:48.582 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:43:48.582 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:43:48.582 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:43:48.582 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:43:48.582 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:43:48.582 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:43:48.582 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:43:48.582 | INFO | ------------------------------------------ | 20260506 11:43:48.582 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:43:48.582 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:43:48.582 | INFO | Total mem usage: 25.6 GiB | 20260506 11:43:48.583 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:43:48.583 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260506 11:43:48.583 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260506 11:43:48.583 | INFO | High Water Breakdown: | 20260506 11:43:48.583 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260506 11:43:48.583 | INFO | ------------------------------------------ | 20260506 11:43:48.583 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:43:48.583 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:43:48.583 | INFO | Total mem usage: 15.7 GiB | 20260506 11:43:48.583 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:43:48.583 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:43:48.583 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 11:43:48.583 | INFO | High Water Breakdown: | 20260506 11:43:48.583 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:43:48.583 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:43:48.583 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:43:48.583 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:43:48.583 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:43:48.583 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:43:48.583 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:43:48.583 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:43:48.583 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:43:48.583 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:43:48.583 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:43:48.583 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:43:48.583 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:43:48.583 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:43:48.583 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:43:48.583 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:43:48.583 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:43:48.583 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:43:48.583 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:43:48.583 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:43:48.583 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:43:48.583 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:43:48.583 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:43:48.583 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:43:48.583 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:43:48.583 | INFO | ------------------------------------------ | 20260506 11:43:48.583 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:43:48.583 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:43:48.583 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:43:48.583 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:43:48.583 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.60e+01, 1.35e+01 / 1.93e+00 | 20260506 11:43:48.583 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.02e-01, 1.07e-02 / 5.73e-03 | 20260506 11:43:48.583 | INFO | PerfReport: 11.52%, Baseline -- 3.67e+00 / 4.51e+00, 5.75e-01 / 3.09e-01 | 20260506 11:43:48.583 | INFO | PerfReport: 3.23%, Histogram -- 1.03e+00 / 1.48e+00, 1.89e-01 / 1.37e-01 | 20260506 11:43:48.583 | INFO | PerfReport: 15.83%, DME -- 4.04e+01 / 4.49e+01, 3.82e+00 / 1.97e+00 | 20260506 11:43:48.583 | INFO | PerfReport: 56.20%, FrameLabeling -- 1.79e+01 / 1.97e+01, 1.41e+00 / 1.53e+00 | 20260506 11:43:48.583 | INFO | PerfReport: 7.72%, PulseAccumulating -- 2.46e+00 / 3.15e+00, 4.43e-01 / 1.69e-01 | 20260506 11:43:48.583 | INFO | PerfReport: 3.84%, Metrics -- 1.23e+00 / 1.65e+00, 3.07e-01 / 1.52e-01 | 20260506 11:43:48.583 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.33e-01 / 4.95e-01, 6.69e-02 / 2.66e-02 | 20260506 11:43:48.583 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:43:48.583 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:43:48.583 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:43:48.583 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260506 11:43:48.583 | INFO | PerfReport: 21.69%, Upload -- 2.15e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 11:43:48.583 | INFO | PerfReport: 21.20%, Download -- 2.10e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 11:43:48.583 | INFO | PerfReport: 32.23%, Compute -- 3.19e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 11:43:48.583 | INFO | PerfReport: 1.86%, LockUpload -- 1.84e+00 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 11:43:48.583 | INFO | PerfReport: 1.01%, LockDownload -- 1.00e+00 / 0.00e+00, 2.47e+00 / 0.00e+00 | 20260506 11:43:48.583 | INFO | PerfReport: 21.99%, LockCompute -- 2.17e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 11:43:48.583 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260506 11:43:48.705 | INFO | Starting a round of Baz IO | 20260506 11:43:51.914 | INFO | Populating RTMetrics report | 20260506 11:43:51.915 | INFO | RTMetrics report populated | 20260506 11:43:51.915 | INFO | Opening tmp RTMetrics json file | 20260506 11:43:51.925 | INFO | Serializing RTMetrics to json | 20260506 11:43:51.931 | INFO | Serialization done, swapping tmp file now... | 20260506 11:43:51.937 | INFO | RTMetrics done | 20260506 11:43:52.086 | INFO | Finished a round of Baz IO: Wrote 2.073 GiB in 3.38 seconds | 20260506 11:43:52.086 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.34% / 0.03% / 56.62% | 20260506 11:43:52.086 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.84093% / 7.15256e-05% / 2.38419e-05% | 20260506 11:43:52.086 | INFO | Baz IO overall: | 20260506 11:43:52.086 | INFO | BytesWritten : 528242090866 | 20260506 11:43:52.086 | INFO | NumEvents : 255024735663 | 20260506 11:43:53.720 | INFO | Analyzing chunk frames = [1089536,1090048) | 20260506 11:43:53.720 | INFO | MemInfo: { | 20260506 11:43:53.720 | INFO | "gpu" : " 17 GB", | 20260506 11:43:53.720 | INFO | "huge" : "", | 20260506 11:43:53.720 | INFO | "huge_shared" : " 27 GB", | 20260506 11:43:53.720 | INFO | "malloc" : " 14 GB", | 20260506 11:43:53.720 | INFO | "pinned" : " 29 GB" | 20260506 11:43:53.720 | INFO | } | 20260506 11:43:53.720 | INFO | Perf report for the last 5.13916 seconds: | 20260506 11:43:53.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:43:53.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:53.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:53.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:53.720 | INFO | ANALYSIS : 64.0%, 2.9, 96.7ms, 0 | 20260506 11:43:53.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:53.720 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 11:43:53.720 | INFO | PRE_HQ : 61.7%, 1.9, 61.1ms, 0 | 20260506 11:43:53.720 | INFO | BAZWRITER : 65.0%, 4.8, 159.1ms, 0 | 20260506 11:43:53.720 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 11:43:53.720 | INFO | Uploaded 36GiB to GPU in 2.12829s (16.915GiB/s) | 20260506 11:43:53.721 | INFO | Downloaded 38GiB to GPU in 2.23499s (17.0023GiB/s) | 20260506 11:43:58.860 | INFO | Analyzing chunk frames = [1090048,1090560) | 20260506 11:43:58.860 | INFO | MemInfo: { | 20260506 11:43:58.860 | INFO | "gpu" : " 17 GB", | 20260506 11:43:58.860 | INFO | "huge" : "", | 20260506 11:43:58.860 | INFO | "huge_shared" : " 27 GB", | 20260506 11:43:58.860 | INFO | "malloc" : " 14 GB", | 20260506 11:43:58.860 | INFO | "pinned" : " 29 GB" | 20260506 11:43:58.860 | INFO | } | 20260506 11:43:58.860 | INFO | Perf report for the last 5.13978 seconds: | 20260506 11:43:58.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:43:58.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:58.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:58.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:58.860 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:43:58.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:58.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:58.860 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 11:43:58.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:58.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:43:58.860 | INFO | Uploaded 36GiB to GPU in 2.00109s (17.9902GiB/s) | 20260506 11:43:58.860 | INFO | Downloaded 34GiB to GPU in 1.9258s (17.655GiB/s) | 20260506 11:44:03.999 | INFO | Analyzing chunk frames = [1090560,1091072) | 20260506 11:44:03.999 | INFO | MemInfo: { | 20260506 11:44:03.999 | INFO | "gpu" : " 17 GB", | 20260506 11:44:03.999 | INFO | "huge" : "", | 20260506 11:44:03.999 | INFO | "huge_shared" : " 27 GB", | 20260506 11:44:03.999 | INFO | "malloc" : " 14 GB", | 20260506 11:44:03.999 | INFO | "pinned" : " 29 GB" | 20260506 11:44:03.999 | INFO | } | 20260506 11:44:03.999 | INFO | Perf report for the last 5.13921 seconds: | 20260506 11:44:03.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:44:03.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:03.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:03.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:03.999 | INFO | ANALYSIS : 61.9%, 3.0, 95.8ms, 0 | 20260506 11:44:03.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:03.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:03.999 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:44:03.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:03.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:04.000 | INFO | Uploaded 36GiB to GPU in 2.0142s (17.8731GiB/s) | 20260506 11:44:04.000 | INFO | Downloaded 34GiB to GPU in 1.92469s (17.6652GiB/s) | 20260506 11:44:09.138 | INFO | Analyzing chunk frames = [1091072,1091584) | 20260506 11:44:09.138 | INFO | MemInfo: { | 20260506 11:44:09.138 | INFO | "gpu" : " 17 GB", | 20260506 11:44:09.138 | INFO | "huge" : "", | 20260506 11:44:09.138 | INFO | "huge_shared" : " 27 GB", | 20260506 11:44:09.138 | INFO | "malloc" : " 14 GB", | 20260506 11:44:09.138 | INFO | "pinned" : " 29 GB" | 20260506 11:44:09.138 | INFO | } | 20260506 11:44:09.138 | INFO | Perf report for the last 5.13918 seconds: | 20260506 11:44:09.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:44:09.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:09.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:09.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:09.138 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 11:44:09.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:09.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:09.139 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:44:09.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:09.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:09.139 | INFO | Uploaded 36GiB to GPU in 2.06491s (17.4342GiB/s) | 20260506 11:44:09.139 | INFO | Downloaded 34GiB to GPU in 1.97706s (17.1973GiB/s) | 20260506 11:44:14.278 | INFO | Analyzing chunk frames = [1091584,1092096) | 20260506 11:44:14.278 | INFO | MemInfo: { | 20260506 11:44:14.278 | INFO | "gpu" : " 17 GB", | 20260506 11:44:14.278 | INFO | "huge" : "", | 20260506 11:44:14.278 | INFO | "huge_shared" : " 27 GB", | 20260506 11:44:14.278 | INFO | "malloc" : " 14 GB", | 20260506 11:44:14.278 | INFO | "pinned" : " 29 GB" | 20260506 11:44:14.278 | INFO | } | 20260506 11:44:14.279 | INFO | Perf report for the last 5.14009 seconds: | 20260506 11:44:14.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:44:14.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:14.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:14.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:14.279 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 11:44:14.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:14.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:14.279 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:44:14.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:14.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:14.279 | INFO | Uploaded 36GiB to GPU in 2.07389s (17.3587GiB/s) | 20260506 11:44:14.279 | INFO | Downloaded 34GiB to GPU in 1.98412s (17.1361GiB/s) | 20260506 11:44:19.417 | INFO | Analyzing chunk frames = [1092096,1092608) | 20260506 11:44:19.417 | INFO | MemInfo: { | 20260506 11:44:19.417 | INFO | "gpu" : " 17 GB", | 20260506 11:44:19.417 | INFO | "huge" : "", | 20260506 11:44:19.417 | INFO | "huge_shared" : " 27 GB", | 20260506 11:44:19.417 | INFO | "malloc" : " 14 GB", | 20260506 11:44:19.417 | INFO | "pinned" : " 29 GB" | 20260506 11:44:19.417 | INFO | } | 20260506 11:44:19.417 | INFO | Perf report for the last 5.13857 seconds: | 20260506 11:44:19.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:44:19.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:19.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:19.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:19.417 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 11:44:19.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:19.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:19.417 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 11:44:19.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:19.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:19.417 | INFO | Uploaded 36GiB to GPU in 2.09101s (17.2166GiB/s) | 20260506 11:44:19.417 | INFO | Downloaded 34GiB to GPU in 2.00964s (16.9185GiB/s) | 20260506 11:44:24.556 | INFO | Analyzing chunk frames = [1092608,1093120) | 20260506 11:44:24.556 | INFO | MemInfo: { | 20260506 11:44:24.556 | INFO | "gpu" : " 17 GB", | 20260506 11:44:24.556 | INFO | "huge" : "", | 20260506 11:44:24.556 | INFO | "huge_shared" : " 27 GB", | 20260506 11:44:24.556 | INFO | "malloc" : " 14 GB", | 20260506 11:44:24.557 | INFO | "pinned" : " 29 GB" | 20260506 11:44:24.557 | INFO | } | 20260506 11:44:24.557 | INFO | Perf report for the last 5.13945 seconds: | 20260506 11:44:24.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:44:24.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:24.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:24.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:24.557 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 11:44:24.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:24.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:24.557 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 11:44:24.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:24.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:24.557 | INFO | Uploaded 36GiB to GPU in 2.00012s (17.9989GiB/s) | 20260506 11:44:24.557 | INFO | Downloaded 34GiB to GPU in 2.02895s (16.7574GiB/s) | 20260506 11:44:29.696 | INFO | Analyzing chunk frames = [1093120,1093632) | 20260506 11:44:29.696 | INFO | MemInfo: { | 20260506 11:44:29.696 | INFO | "gpu" : " 17 GB", | 20260506 11:44:29.696 | INFO | "huge" : "", | 20260506 11:44:29.696 | INFO | "huge_shared" : " 27 GB", | 20260506 11:44:29.696 | INFO | "malloc" : " 14 GB", | 20260506 11:44:29.696 | INFO | "pinned" : " 29 GB" | 20260506 11:44:29.696 | INFO | } | 20260506 11:44:29.696 | INFO | Perf report for the last 5.13931 seconds: | 20260506 11:44:29.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:44:29.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:29.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:29.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:29.696 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260506 11:44:29.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:29.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:29.696 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:44:29.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:29.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:29.696 | INFO | Uploaded 36GiB to GPU in 2.06163s (17.4619GiB/s) | 20260506 11:44:29.696 | INFO | Downloaded 34GiB to GPU in 1.99557s (17.0378GiB/s) | 20260506 11:44:29.814 | INFO | Starting a round of Baz IO | 20260506 11:44:33.043 | INFO | Populating RTMetrics report | 20260506 11:44:33.044 | INFO | RTMetrics report populated | 20260506 11:44:33.044 | INFO | Opening tmp RTMetrics json file | 20260506 11:44:33.057 | INFO | Serializing RTMetrics to json | 20260506 11:44:33.061 | INFO | Serialization done, swapping tmp file now... | 20260506 11:44:33.064 | INFO | RTMetrics done | 20260506 11:44:33.289 | INFO | Finished a round of Baz IO: Wrote 2.072 GiB in 3.475 seconds | 20260506 11:44:33.289 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.31% / 0.03% / 56.66% | 20260506 11:44:33.289 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.82174% / 8.34465e-05% / 4.37101e-05% | 20260506 11:44:33.289 | INFO | Baz IO overall: | 20260506 11:44:33.289 | INFO | BytesWritten : 530467404053 | 20260506 11:44:33.290 | INFO | NumEvents : 256105211848 | 20260506 11:44:34.835 | INFO | Analyzing chunk frames = [1093632,1094144) | 20260506 11:44:34.835 | INFO | MemInfo: { | 20260506 11:44:34.835 | INFO | "gpu" : " 17 GB", | 20260506 11:44:34.835 | INFO | "huge" : "", | 20260506 11:44:34.835 | INFO | "huge_shared" : " 27 GB", | 20260506 11:44:34.835 | INFO | "malloc" : " 14 GB", | 20260506 11:44:34.835 | INFO | "pinned" : " 29 GB" | 20260506 11:44:34.835 | INFO | } | 20260506 11:44:34.836 | INFO | Perf report for the last 5.13964 seconds: | 20260506 11:44:34.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:44:34.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:34.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:34.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:34.836 | INFO | ANALYSIS : 64.2%, 2.9, 96.3ms, 0 | 20260506 11:44:34.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:34.836 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260506 11:44:34.836 | INFO | PRE_HQ : 62.7%, 1.9, 62.8ms, 0 | 20260506 11:44:34.836 | INFO | BAZWRITER : 66.7%, 4.8, 164.5ms, 0 | 20260506 11:44:34.836 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.5ms, 0 | 20260506 11:44:34.836 | INFO | Uploaded 36GiB to GPU in 2.15745s (16.6864GiB/s) | 20260506 11:44:34.836 | INFO | Downloaded 38GiB to GPU in 2.23622s (16.993GiB/s) | 20260506 11:44:39.974 | INFO | Analyzing chunk frames = [1094144,1094656) | 20260506 11:44:39.974 | INFO | MemInfo: { | 20260506 11:44:39.974 | INFO | "gpu" : " 17 GB", | 20260506 11:44:39.974 | INFO | "huge" : "", | 20260506 11:44:39.974 | INFO | "huge_shared" : " 27 GB", | 20260506 11:44:39.974 | INFO | "malloc" : " 14 GB", | 20260506 11:44:39.975 | INFO | "pinned" : " 29 GB" | 20260506 11:44:39.975 | INFO | } | 20260506 11:44:39.975 | INFO | Perf report for the last 5.13917 seconds: | 20260506 11:44:39.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:44:39.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:39.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:39.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:39.975 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:44:39.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:39.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:39.975 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:44:39.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:39.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:39.975 | INFO | Uploaded 36GiB to GPU in 2.0172s (17.8465GiB/s) | 20260506 11:44:39.975 | INFO | Downloaded 34GiB to GPU in 1.92415s (17.6701GiB/s) | 20260506 11:44:45.114 | INFO | Analyzing chunk frames = [1094656,1095168) | 20260506 11:44:45.114 | INFO | MemInfo: { | 20260506 11:44:45.114 | INFO | "gpu" : " 17 GB", | 20260506 11:44:45.114 | INFO | "huge" : "", | 20260506 11:44:45.114 | INFO | "huge_shared" : " 27 GB", | 20260506 11:44:45.114 | INFO | "malloc" : " 14 GB", | 20260506 11:44:45.114 | INFO | "pinned" : " 29 GB" | 20260506 11:44:45.114 | INFO | } | 20260506 11:44:45.114 | INFO | Perf report for the last 5.13908 seconds: | 20260506 11:44:45.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:44:45.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:45.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:45.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:45.114 | INFO | ANALYSIS : 62.0%, 3.0, 96.1ms, 0 | 20260506 11:44:45.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:45.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:45.114 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:44:45.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:45.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:45.114 | INFO | Uploaded 36GiB to GPU in 2.04668s (17.5895GiB/s) | 20260506 11:44:45.114 | INFO | Downloaded 34GiB to GPU in 1.93849s (17.5394GiB/s) | 20260506 11:44:50.253 | INFO | Analyzing chunk frames = [1095168,1095680) | 20260506 11:44:50.253 | INFO | MemInfo: { | 20260506 11:44:50.253 | INFO | "gpu" : " 17 GB", | 20260506 11:44:50.253 | INFO | "huge" : "", | 20260506 11:44:50.253 | INFO | "huge_shared" : " 27 GB", | 20260506 11:44:50.253 | INFO | "malloc" : " 14 GB", | 20260506 11:44:50.253 | INFO | "pinned" : " 29 GB" | 20260506 11:44:50.253 | INFO | } | 20260506 11:44:50.253 | INFO | Perf report for the last 5.13954 seconds: | 20260506 11:44:50.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:44:50.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:50.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:50.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:50.253 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 11:44:50.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:50.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:50.253 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:44:50.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:50.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:50.254 | INFO | Uploaded 36GiB to GPU in 2.08183s (17.2925GiB/s) | 20260506 11:44:50.254 | INFO | Downloaded 34GiB to GPU in 1.97408s (17.2232GiB/s) | 20260506 11:44:55.392 | INFO | Analyzing chunk frames = [1095680,1096192) | 20260506 11:44:55.392 | INFO | MemInfo: { | 20260506 11:44:55.392 | INFO | "gpu" : " 17 GB", | 20260506 11:44:55.392 | INFO | "huge" : "", | 20260506 11:44:55.392 | INFO | "huge_shared" : " 27 GB", | 20260506 11:44:55.392 | INFO | "malloc" : " 14 GB", | 20260506 11:44:55.392 | INFO | "pinned" : " 29 GB" | 20260506 11:44:55.392 | INFO | } | 20260506 11:44:55.392 | INFO | Perf report for the last 5.13896 seconds: | 20260506 11:44:55.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:44:55.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:55.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:55.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:55.392 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 11:44:55.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:55.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:55.392 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:44:55.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:55.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:44:55.393 | INFO | Uploaded 36GiB to GPU in 2.09381s (17.1936GiB/s) | 20260506 11:44:55.393 | INFO | Downloaded 34GiB to GPU in 1.98756s (17.1064GiB/s) | 20260506 11:45:00.532 | INFO | Analyzing chunk frames = [1096192,1096704) | 20260506 11:45:00.532 | INFO | MemInfo: { | 20260506 11:45:00.532 | INFO | "gpu" : " 17 GB", | 20260506 11:45:00.532 | INFO | "huge" : "", | 20260506 11:45:00.532 | INFO | "huge_shared" : " 27 GB", | 20260506 11:45:00.532 | INFO | "malloc" : " 14 GB", | 20260506 11:45:00.532 | INFO | "pinned" : " 29 GB" | 20260506 11:45:00.532 | INFO | } | 20260506 11:45:00.532 | INFO | Perf report for the last 5.13964 seconds: | 20260506 11:45:00.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:45:00.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:00.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:00.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:00.532 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 11:45:00.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:00.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:00.532 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:45:00.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:00.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:00.533 | INFO | Uploaded 36GiB to GPU in 2.06859s (17.4032GiB/s) | 20260506 11:45:00.533 | INFO | Downloaded 34GiB to GPU in 1.99721s (17.0237GiB/s) | 20260506 11:45:05.672 | INFO | Analyzing chunk frames = [1096704,1097216) | 20260506 11:45:05.673 | INFO | MemInfo: { | 20260506 11:45:05.673 | INFO | "gpu" : " 17 GB", | 20260506 11:45:05.673 | INFO | "huge" : "", | 20260506 11:45:05.673 | INFO | "huge_shared" : " 27 GB", | 20260506 11:45:05.673 | INFO | "malloc" : " 14 GB", | 20260506 11:45:05.673 | INFO | "pinned" : " 29 GB" | 20260506 11:45:05.673 | INFO | } | 20260506 11:45:05.673 | INFO | Perf report for the last 5.14072 seconds: | 20260506 11:45:05.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:45:05.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:05.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:05.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:05.673 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:45:05.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:05.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:05.673 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:45:05.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:05.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:05.673 | INFO | Uploaded 36GiB to GPU in 2.02615s (17.7676GiB/s) | 20260506 11:45:05.673 | INFO | Downloaded 34GiB to GPU in 1.98882s (17.0955GiB/s) | 20260506 11:45:10.811 | INFO | Analyzing chunk frames = [1097216,1097728) | 20260506 11:45:10.811 | INFO | MemInfo: { | 20260506 11:45:10.811 | INFO | "gpu" : " 17 GB", | 20260506 11:45:10.811 | INFO | "huge" : "", | 20260506 11:45:10.811 | INFO | "huge_shared" : " 27 GB", | 20260506 11:45:10.811 | INFO | "malloc" : " 14 GB", | 20260506 11:45:10.811 | INFO | "pinned" : " 29 GB" | 20260506 11:45:10.811 | INFO | } | 20260506 11:45:10.811 | INFO | Perf report for the last 5.13795 seconds: | 20260506 11:45:10.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:45:10.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:10.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:10.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:10.811 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260506 11:45:10.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:10.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:10.811 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:45:10.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:10.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:10.811 | INFO | Uploaded 36GiB to GPU in 2.01945s (17.8266GiB/s) | 20260506 11:45:10.811 | INFO | Downloaded 34GiB to GPU in 1.98657s (17.1149GiB/s) | 20260506 11:45:10.811 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:45:10.811 | INFO | Total mem usage: 16.8 GiB | 20260506 11:45:10.811 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:45:10.811 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -7.04%) | 20260506 11:45:10.811 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:45:10.811 | INFO | High Water Breakdown: | 20260506 11:45:10.811 | INFO | 60.8% : 9.55 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:45:10.811 | INFO | 39% : 6.13 GiB ( 91.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:45:10.811 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:45:10.811 | INFO | ------------------------------------------ | 20260506 11:45:10.811 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:45:10.811 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:45:10.811 | INFO | Total mem usage: 26.5 GiB | 20260506 11:45:10.811 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:45:10.811 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 11:45:10.811 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 11:45:10.811 | INFO | High Water Breakdown: | 20260506 11:45:10.811 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:45:10.811 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:45:10.812 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:45:10.812 | INFO | 2% : 523 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:45:10.812 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:45:10.812 | INFO | 0.557% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:45:10.812 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:45:10.812 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:45:10.812 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:45:10.812 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:45:10.812 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:45:10.812 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:45:10.812 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:45:10.812 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:45:10.812 | INFO | ------------------------------------------ | 20260506 11:45:10.812 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:45:10.812 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:45:10.812 | INFO | Total mem usage: 25.6 GiB | 20260506 11:45:10.812 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:45:10.812 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:45:10.812 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:45:10.812 | INFO | High Water Breakdown: | 20260506 11:45:10.812 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:45:10.812 | INFO | ------------------------------------------ | 20260506 11:45:10.812 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:45:10.812 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:45:10.812 | INFO | Total mem usage: 15.7 GiB | 20260506 11:45:10.812 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:45:10.812 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:45:10.812 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 11:45:10.812 | INFO | High Water Breakdown: | 20260506 11:45:10.812 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:45:10.812 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:45:10.812 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:45:10.812 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:45:10.812 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:45:10.812 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:45:10.812 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:45:10.812 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:45:10.812 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:45:10.812 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:45:10.812 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:45:10.812 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:45:10.812 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:45:10.812 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:45:10.812 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:45:10.812 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:45:10.812 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:45:10.812 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:45:10.812 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:45:10.812 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:45:10.812 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:45:10.812 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:45:10.812 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:45:10.812 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:45:10.812 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:45:10.812 | INFO | ------------------------------------------ | 20260506 11:45:10.812 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:45:10.812 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:45:10.812 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:45:10.812 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:45:10.812 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.59e+01, 1.33e+01 / 2.43e+00 | 20260506 11:45:10.812 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 9.96e-02, 1.04e-02 / 7.42e-03 | 20260506 11:45:10.812 | INFO | PerfReport: 11.52%, Baseline -- 3.68e+00 / 4.42e+00, 5.87e-01 / 3.75e-01 | 20260506 11:45:10.812 | INFO | PerfReport: 3.23%, Histogram -- 1.03e+00 / 1.45e+00, 1.93e-01 / 1.56e-01 | 20260506 11:45:10.812 | INFO | PerfReport: 15.80%, DME -- 4.03e+01 / 4.44e+01, 3.99e+00 / 2.44e+00 | 20260506 11:45:10.812 | INFO | PerfReport: 56.32%, FrameLabeling -- 1.80e+01 / 2.03e+01, 1.54e+00 / 1.85e+00 | 20260506 11:45:10.812 | INFO | PerfReport: 7.65%, PulseAccumulating -- 2.44e+00 / 3.04e+00, 4.32e-01 / 2.67e-01 | 20260506 11:45:10.812 | INFO | PerfReport: 3.84%, Metrics -- 1.23e+00 / 1.62e+00, 3.14e-01 / 2.23e-01 | 20260506 11:45:10.812 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.31e-01 / 4.90e-01, 6.76e-02 / 3.17e-02 | 20260506 11:45:10.812 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:45:10.812 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:45:10.812 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:45:10.812 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260506 11:45:10.812 | INFO | PerfReport: 21.69%, Upload -- 2.14e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 11:45:10.812 | INFO | PerfReport: 21.14%, Download -- 2.09e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 11:45:10.812 | INFO | PerfReport: 32.29%, Compute -- 3.19e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 11:45:10.812 | INFO | PerfReport: 2.00%, LockUpload -- 1.98e+00 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260506 11:45:10.812 | INFO | PerfReport: 0.86%, LockDownload -- 8.48e-01 / 0.00e+00, 2.32e+00 / 0.00e+00 | 20260506 11:45:10.812 | INFO | PerfReport: 22.01%, LockCompute -- 2.18e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 11:45:10.812 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.55e-02 / 0.00e+00 | 20260506 11:45:10.938 | INFO | Starting a round of Baz IO | 20260506 11:45:14.108 | INFO | Populating RTMetrics report | 20260506 11:45:14.109 | INFO | RTMetrics report populated | 20260506 11:45:14.109 | INFO | Opening tmp RTMetrics json file | 20260506 11:45:14.112 | INFO | Serializing RTMetrics to json | 20260506 11:45:14.116 | INFO | Serialization done, swapping tmp file now... | 20260506 11:45:14.120 | INFO | RTMetrics done | 20260506 11:45:14.418 | INFO | Finished a round of Baz IO: Wrote 2.074 GiB in 3.48 seconds | 20260506 11:45:14.418 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.28% / 0.03% / 56.69% | 20260506 11:45:14.418 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.81182% / 5.16574e-05% / 4.76837e-05% | 20260506 11:45:14.418 | INFO | Baz IO overall: | 20260506 11:45:14.418 | INFO | BytesWritten : 532694528251 | 20260506 11:45:14.418 | INFO | NumEvents : 257186652212 | 20260506 11:45:15.950 | INFO | Analyzing chunk frames = [1097728,1098240) | 20260506 11:45:15.950 | INFO | MemInfo: { | 20260506 11:45:15.950 | INFO | "gpu" : " 17 GB", | 20260506 11:45:15.950 | INFO | "huge" : "", | 20260506 11:45:15.950 | INFO | "huge_shared" : " 27 GB", | 20260506 11:45:15.950 | INFO | "malloc" : " 14 GB", | 20260506 11:45:15.950 | INFO | "pinned" : " 29 GB" | 20260506 11:45:15.950 | INFO | } | 20260506 11:45:15.950 | INFO | Perf report for the last 5.13895 seconds: | 20260506 11:45:15.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:45:15.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:15.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:15.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:15.950 | INFO | ANALYSIS : 63.6%, 2.9, 96.1ms, 0 | 20260506 11:45:15.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:15.950 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 11:45:15.950 | INFO | PRE_HQ : 61.8%, 1.9, 61.2ms, 0 | 20260506 11:45:15.950 | INFO | BAZWRITER : 67.0%, 4.7, 162.9ms, 0 | 20260506 11:45:15.950 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260506 11:45:15.950 | INFO | Uploaded 36GiB to GPU in 2.13069s (16.8959GiB/s) | 20260506 11:45:15.950 | INFO | Downloaded 38GiB to GPU in 2.22785s (17.0568GiB/s) | 20260506 11:45:21.089 | INFO | Analyzing chunk frames = [1098240,1098752) | 20260506 11:45:21.089 | INFO | MemInfo: { | 20260506 11:45:21.089 | INFO | "gpu" : " 17 GB", | 20260506 11:45:21.089 | INFO | "huge" : "", | 20260506 11:45:21.089 | INFO | "huge_shared" : " 27 GB", | 20260506 11:45:21.090 | INFO | "malloc" : " 14 GB", | 20260506 11:45:21.090 | INFO | "pinned" : " 29 GB" | 20260506 11:45:21.090 | INFO | } | 20260506 11:45:21.090 | INFO | Perf report for the last 5.14007 seconds: | 20260506 11:45:21.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:45:21.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:21.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:21.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:21.090 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:45:21.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:21.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:21.090 | INFO | PRE_HQ : 29.6%, 1.0, 15.7ms, 0 | 20260506 11:45:21.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:21.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:21.090 | INFO | Uploaded 36GiB to GPU in 2.02547s (17.7737GiB/s) | 20260506 11:45:21.090 | INFO | Downloaded 34GiB to GPU in 1.92462s (17.6658GiB/s) | 20260506 11:45:26.228 | INFO | Analyzing chunk frames = [1098752,1099264) | 20260506 11:45:26.228 | INFO | MemInfo: { | 20260506 11:45:26.228 | INFO | "gpu" : " 17 GB", | 20260506 11:45:26.228 | INFO | "huge" : "", | 20260506 11:45:26.228 | INFO | "huge_shared" : " 27 GB", | 20260506 11:45:26.228 | INFO | "malloc" : " 14 GB", | 20260506 11:45:26.228 | INFO | "pinned" : " 29 GB" | 20260506 11:45:26.228 | INFO | } | 20260506 11:45:26.228 | INFO | Perf report for the last 5.13871 seconds: | 20260506 11:45:26.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:45:26.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:26.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:26.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:26.228 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260506 11:45:26.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:26.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:26.229 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:45:26.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:26.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:26.229 | INFO | Uploaded 36GiB to GPU in 2.0303s (17.7313GiB/s) | 20260506 11:45:26.229 | INFO | Downloaded 34GiB to GPU in 1.92718s (17.6424GiB/s) | 20260506 11:45:31.368 | INFO | Analyzing chunk frames = [1099264,1099776) | 20260506 11:45:31.368 | INFO | MemInfo: { | 20260506 11:45:31.368 | INFO | "gpu" : " 17 GB", | 20260506 11:45:31.368 | INFO | "huge" : "", | 20260506 11:45:31.368 | INFO | "huge_shared" : " 27 GB", | 20260506 11:45:31.368 | INFO | "malloc" : " 14 GB", | 20260506 11:45:31.368 | INFO | "pinned" : " 29 GB" | 20260506 11:45:31.368 | INFO | } | 20260506 11:45:31.368 | INFO | Perf report for the last 5.13994 seconds: | 20260506 11:45:31.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:45:31.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:31.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:31.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:31.368 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260506 11:45:31.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:31.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:31.368 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:45:31.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:31.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:31.369 | INFO | Uploaded 36GiB to GPU in 2.09292s (17.2009GiB/s) | 20260506 11:45:31.369 | INFO | Downloaded 34GiB to GPU in 1.97592s (17.2071GiB/s) | 20260506 11:45:36.507 | INFO | Analyzing chunk frames = [1099776,1100288) | 20260506 11:45:36.507 | INFO | MemInfo: { | 20260506 11:45:36.507 | INFO | "gpu" : " 17 GB", | 20260506 11:45:36.507 | INFO | "huge" : "", | 20260506 11:45:36.507 | INFO | "huge_shared" : " 27 GB", | 20260506 11:45:36.507 | INFO | "malloc" : " 14 GB", | 20260506 11:45:36.507 | INFO | "pinned" : " 29 GB" | 20260506 11:45:36.507 | INFO | } | 20260506 11:45:36.507 | INFO | Perf report for the last 5.13877 seconds: | 20260506 11:45:36.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:45:36.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:36.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:36.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:36.507 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 11:45:36.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:36.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:36.507 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:45:36.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:36.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:36.507 | INFO | Uploaded 36GiB to GPU in 2.10115s (17.1335GiB/s) | 20260506 11:45:36.508 | INFO | Downloaded 34GiB to GPU in 1.98509s (17.1277GiB/s) | 20260506 11:45:41.647 | INFO | Analyzing chunk frames = [1100288,1100800) | 20260506 11:45:41.647 | INFO | MemInfo: { | 20260506 11:45:41.647 | INFO | "gpu" : " 17 GB", | 20260506 11:45:41.647 | INFO | "huge" : "", | 20260506 11:45:41.647 | INFO | "huge_shared" : " 27 GB", | 20260506 11:45:41.647 | INFO | "malloc" : " 14 GB", | 20260506 11:45:41.647 | INFO | "pinned" : " 29 GB" | 20260506 11:45:41.647 | INFO | } | 20260506 11:45:41.647 | INFO | Perf report for the last 5.13972 seconds: | 20260506 11:45:41.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:45:41.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:41.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:41.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:41.647 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 11:45:41.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:41.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:41.647 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:45:41.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:41.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:41.647 | INFO | Uploaded 36GiB to GPU in 2.08758s (17.2448GiB/s) | 20260506 11:45:41.647 | INFO | Downloaded 34GiB to GPU in 1.99725s (17.0234GiB/s) | 20260506 11:45:46.786 | INFO | Analyzing chunk frames = [1100800,1101312) | 20260506 11:45:46.786 | INFO | MemInfo: { | 20260506 11:45:46.786 | INFO | "gpu" : " 17 GB", | 20260506 11:45:46.786 | INFO | "huge" : "", | 20260506 11:45:46.786 | INFO | "huge_shared" : " 27 GB", | 20260506 11:45:46.786 | INFO | "malloc" : " 14 GB", | 20260506 11:45:46.786 | INFO | "pinned" : " 29 GB" | 20260506 11:45:46.786 | INFO | } | 20260506 11:45:46.786 | INFO | Perf report for the last 5.13928 seconds: | 20260506 11:45:46.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:45:46.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:46.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:46.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:46.786 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:45:46.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:46.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:46.786 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:45:46.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:46.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:46.786 | INFO | Uploaded 36GiB to GPU in 2.00292s (17.9737GiB/s) | 20260506 11:45:46.787 | INFO | Downloaded 34GiB to GPU in 1.97909s (17.1797GiB/s) | 20260506 11:45:51.925 | INFO | Analyzing chunk frames = [1101312,1101824) | 20260506 11:45:51.925 | INFO | MemInfo: { | 20260506 11:45:51.925 | INFO | "gpu" : " 17 GB", | 20260506 11:45:51.925 | INFO | "huge" : "", | 20260506 11:45:51.925 | INFO | "huge_shared" : " 27 GB", | 20260506 11:45:51.925 | INFO | "malloc" : " 14 GB", | 20260506 11:45:51.926 | INFO | "pinned" : " 29 GB" | 20260506 11:45:51.926 | INFO | } | 20260506 11:45:51.926 | INFO | Perf report for the last 5.13948 seconds: | 20260506 11:45:51.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:45:51.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:51.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:51.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:51.926 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260506 11:45:51.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:51.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:51.926 | INFO | PRE_HQ : 33.8%, 1.0, 17.6ms, 0 | 20260506 11:45:51.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:51.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:51.926 | INFO | Uploaded 36GiB to GPU in 1.99938s (18.0056GiB/s) | 20260506 11:45:51.926 | INFO | Downloaded 34GiB to GPU in 1.96692s (17.2859GiB/s) | 20260506 11:45:52.056 | INFO | Starting a round of Baz IO | 20260506 11:45:55.234 | INFO | Populating RTMetrics report | 20260506 11:45:55.235 | INFO | RTMetrics report populated | 20260506 11:45:55.235 | INFO | Opening tmp RTMetrics json file | 20260506 11:45:55.257 | INFO | Serializing RTMetrics to json | 20260506 11:45:55.286 | INFO | Serialization done, swapping tmp file now... | 20260506 11:45:55.289 | INFO | RTMetrics done | 20260506 11:45:55.667 | INFO | Finished a round of Baz IO: Wrote 2.074 GiB in 3.61 seconds | 20260506 11:45:55.667 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.25% / 0.03% / 56.72% | 20260506 11:45:55.667 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.79505% / 6.75519e-05% / 3.97364e-05% | 20260506 11:45:55.667 | INFO | Baz IO overall: | 20260506 11:45:55.667 | INFO | BytesWritten : 534921990604 | 20260506 11:45:55.667 | INFO | NumEvents : 258268170977 | 20260506 11:45:57.064 | INFO | Analyzing chunk frames = [1101824,1102336) | 20260506 11:45:57.064 | INFO | MemInfo: { | 20260506 11:45:57.064 | INFO | "gpu" : " 17 GB", | 20260506 11:45:57.064 | INFO | "huge" : "", | 20260506 11:45:57.064 | INFO | "huge_shared" : " 27 GB", | 20260506 11:45:57.064 | INFO | "malloc" : " 14 GB", | 20260506 11:45:57.064 | INFO | "pinned" : " 29 GB" | 20260506 11:45:57.065 | INFO | } | 20260506 11:45:57.065 | INFO | Perf report for the last 5.13886 seconds: | 20260506 11:45:57.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:45:57.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:57.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:57.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:57.065 | INFO | ANALYSIS : 63.7%, 2.8, 95.1ms, 0 | 20260506 11:45:57.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:45:57.065 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260506 11:45:57.065 | INFO | PRE_HQ : 62.1%, 2.0, 63.1ms, 0 | 20260506 11:45:57.065 | INFO | BAZWRITER : 69.5%, 4.8, 170.3ms, 0 | 20260506 11:45:57.065 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 11:45:57.065 | INFO | Uploaded 36GiB to GPU in 2.13527s (16.8597GiB/s) | 20260506 11:45:57.065 | INFO | Downloaded 38GiB to GPU in 2.22979s (17.042GiB/s) | 20260506 11:46:02.204 | INFO | Analyzing chunk frames = [1102336,1102848) | 20260506 11:46:02.204 | INFO | MemInfo: { | 20260506 11:46:02.204 | INFO | "gpu" : " 17 GB", | 20260506 11:46:02.204 | INFO | "huge" : "", | 20260506 11:46:02.204 | INFO | "huge_shared" : " 27 GB", | 20260506 11:46:02.204 | INFO | "malloc" : " 14 GB", | 20260506 11:46:02.204 | INFO | "pinned" : " 29 GB" | 20260506 11:46:02.204 | INFO | } | 20260506 11:46:02.204 | INFO | Perf report for the last 5.1394 seconds: | 20260506 11:46:02.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:46:02.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:02.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:02.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:02.204 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:46:02.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:02.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:02.204 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 11:46:02.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:02.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:02.204 | INFO | Uploaded 36GiB to GPU in 2.02184s (17.8055GiB/s) | 20260506 11:46:02.204 | INFO | Downloaded 34GiB to GPU in 1.93136s (17.6041GiB/s) | 20260506 11:46:07.343 | INFO | Analyzing chunk frames = [1102848,1103360) | 20260506 11:46:07.343 | INFO | MemInfo: { | 20260506 11:46:07.343 | INFO | "gpu" : " 17 GB", | 20260506 11:46:07.343 | INFO | "huge" : "", | 20260506 11:46:07.343 | INFO | "huge_shared" : " 27 GB", | 20260506 11:46:07.343 | INFO | "malloc" : " 14 GB", | 20260506 11:46:07.343 | INFO | "pinned" : " 29 GB" | 20260506 11:46:07.343 | INFO | } | 20260506 11:46:07.343 | INFO | Perf report for the last 5.13924 seconds: | 20260506 11:46:07.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:46:07.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:07.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:07.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:07.343 | INFO | ANALYSIS : 62.0%, 3.0, 96.0ms, 0 | 20260506 11:46:07.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:07.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:07.343 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:46:07.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:07.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:07.344 | INFO | Uploaded 36GiB to GPU in 2.01711s (17.8473GiB/s) | 20260506 11:46:07.344 | INFO | Downloaded 34GiB to GPU in 1.94533s (17.4778GiB/s) | 20260506 11:46:12.482 | INFO | Analyzing chunk frames = [1103360,1103872) | 20260506 11:46:12.482 | INFO | MemInfo: { | 20260506 11:46:12.482 | INFO | "gpu" : " 17 GB", | 20260506 11:46:12.482 | INFO | "huge" : "", | 20260506 11:46:12.482 | INFO | "huge_shared" : " 27 GB", | 20260506 11:46:12.483 | INFO | "malloc" : " 14 GB", | 20260506 11:46:12.483 | INFO | "pinned" : " 29 GB" | 20260506 11:46:12.483 | INFO | } | 20260506 11:46:12.483 | INFO | Perf report for the last 5.13934 seconds: | 20260506 11:46:12.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:46:12.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:12.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:12.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:12.483 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 11:46:12.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:12.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:12.483 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:46:12.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:12.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:12.483 | INFO | Uploaded 36GiB to GPU in 2.06772s (17.4105GiB/s) | 20260506 11:46:12.483 | INFO | Downloaded 34GiB to GPU in 1.98123s (17.161GiB/s) | 20260506 11:46:17.622 | INFO | Analyzing chunk frames = [1103872,1104384) | 20260506 11:46:17.622 | INFO | MemInfo: { | 20260506 11:46:17.622 | INFO | "gpu" : " 17 GB", | 20260506 11:46:17.622 | INFO | "huge" : "", | 20260506 11:46:17.622 | INFO | "huge_shared" : " 27 GB", | 20260506 11:46:17.622 | INFO | "malloc" : " 14 GB", | 20260506 11:46:17.622 | INFO | "pinned" : " 29 GB" | 20260506 11:46:17.622 | INFO | } | 20260506 11:46:17.622 | INFO | Perf report for the last 5.1394 seconds: | 20260506 11:46:17.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:46:17.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:17.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:17.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:17.622 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 11:46:17.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:17.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:17.622 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:46:17.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:17.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:17.622 | INFO | Uploaded 36GiB to GPU in 2.09851s (17.155GiB/s) | 20260506 11:46:17.622 | INFO | Downloaded 34GiB to GPU in 1.98683s (17.1127GiB/s) | 20260506 11:46:22.761 | INFO | Analyzing chunk frames = [1104384,1104896) | 20260506 11:46:22.761 | INFO | MemInfo: { | 20260506 11:46:22.761 | INFO | "gpu" : " 17 GB", | 20260506 11:46:22.761 | INFO | "huge" : "", | 20260506 11:46:22.761 | INFO | "huge_shared" : " 27 GB", | 20260506 11:46:22.761 | INFO | "malloc" : " 14 GB", | 20260506 11:46:22.761 | INFO | "pinned" : " 29 GB" | 20260506 11:46:22.761 | INFO | } | 20260506 11:46:22.762 | INFO | Perf report for the last 5.13956 seconds: | 20260506 11:46:22.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:46:22.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:22.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:22.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:22.762 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 11:46:22.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:22.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:22.762 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:46:22.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:22.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:22.762 | INFO | Uploaded 36GiB to GPU in 2.00944s (17.9155GiB/s) | 20260506 11:46:22.762 | INFO | Downloaded 34GiB to GPU in 1.99985s (17.0012GiB/s) | 20260506 11:46:27.901 | INFO | Analyzing chunk frames = [1104896,1105408) | 20260506 11:46:27.901 | INFO | MemInfo: { | 20260506 11:46:27.901 | INFO | "gpu" : " 17 GB", | 20260506 11:46:27.901 | INFO | "huge" : "", | 20260506 11:46:27.901 | INFO | "huge_shared" : " 27 GB", | 20260506 11:46:27.901 | INFO | "malloc" : " 14 GB", | 20260506 11:46:27.901 | INFO | "pinned" : " 29 GB" | 20260506 11:46:27.901 | INFO | } | 20260506 11:46:27.901 | INFO | Perf report for the last 5.13988 seconds: | 20260506 11:46:27.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:46:27.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:27.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:27.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:27.902 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:46:27.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:27.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:27.902 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:46:27.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:27.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:27.902 | INFO | Uploaded 36GiB to GPU in 1.99157s (18.0762GiB/s) | 20260506 11:46:27.902 | INFO | Downloaded 34GiB to GPU in 2.0374s (16.6879GiB/s) | 20260506 11:46:33.040 | INFO | Analyzing chunk frames = [1105408,1105920) | 20260506 11:46:33.040 | INFO | MemInfo: { | 20260506 11:46:33.040 | INFO | "gpu" : " 17 GB", | 20260506 11:46:33.040 | INFO | "huge" : "", | 20260506 11:46:33.040 | INFO | "huge_shared" : " 27 GB", | 20260506 11:46:33.040 | INFO | "malloc" : " 14 GB", | 20260506 11:46:33.040 | INFO | "pinned" : " 29 GB" | 20260506 11:46:33.040 | INFO | } | 20260506 11:46:33.040 | INFO | Perf report for the last 5.13856 seconds: | 20260506 11:46:33.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:46:33.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:33.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:33.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:33.040 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260506 11:46:33.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:33.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:33.040 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:46:33.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:33.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:33.040 | INFO | Uploaded 36GiB to GPU in 2.0252s (17.776GiB/s) | 20260506 11:46:33.040 | INFO | Downloaded 34GiB to GPU in 1.96254s (17.3245GiB/s) | 20260506 11:46:33.041 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:46:33.041 | INFO | Total mem usage: 16.8 GiB | 20260506 11:46:33.041 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:46:33.041 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -7.8%) | 20260506 11:46:33.041 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:46:33.041 | INFO | High Water Breakdown: | 20260506 11:46:33.041 | INFO | 60.7% : 9.47 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:46:33.041 | INFO | 39.1% : 6.1 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:46:33.041 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:46:33.041 | INFO | ------------------------------------------ | 20260506 11:46:33.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:46:33.041 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:46:33.041 | INFO | Total mem usage: 26.5 GiB | 20260506 11:46:33.041 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:46:33.041 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:46:33.041 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:46:33.041 | INFO | High Water Breakdown: | 20260506 11:46:33.041 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:46:33.041 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:46:33.041 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:46:33.041 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:46:33.041 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:46:33.041 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:46:33.041 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:46:33.041 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:46:33.041 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:46:33.041 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:46:33.041 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:46:33.041 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:46:33.041 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:46:33.041 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:46:33.041 | INFO | ------------------------------------------ | 20260506 11:46:33.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:46:33.041 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:46:33.041 | INFO | Total mem usage: 25.6 GiB | 20260506 11:46:33.041 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:46:33.041 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260506 11:46:33.041 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260506 11:46:33.041 | INFO | High Water Breakdown: | 20260506 11:46:33.041 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260506 11:46:33.041 | INFO | ------------------------------------------ | 20260506 11:46:33.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:46:33.041 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:46:33.041 | INFO | Total mem usage: 15.7 GiB | 20260506 11:46:33.041 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:46:33.041 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:46:33.041 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:46:33.041 | INFO | High Water Breakdown: | 20260506 11:46:33.041 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:46:33.041 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:46:33.041 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:46:33.041 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:46:33.041 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:46:33.041 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:46:33.041 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:46:33.041 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:46:33.041 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:46:33.041 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:46:33.041 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:46:33.041 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:46:33.041 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:46:33.041 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:46:33.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:46:33.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:46:33.041 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:46:33.041 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:46:33.041 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:46:33.042 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:46:33.042 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:46:33.042 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:46:33.042 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:46:33.042 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:46:33.042 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:46:33.042 | INFO | ------------------------------------------ | 20260506 11:46:33.042 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:46:33.042 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:46:33.042 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:46:33.042 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:46:33.042 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.62e+01, 1.38e+01 / 2.48e+00 | 20260506 11:46:33.042 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 9.96e-02, 1.05e-02 / 6.38e-03 | 20260506 11:46:33.042 | INFO | PerfReport: 11.54%, Baseline -- 3.68e+00 / 4.40e+00, 5.94e-01 / 3.87e-01 | 20260506 11:46:33.042 | INFO | PerfReport: 3.23%, Histogram -- 1.03e+00 / 1.45e+00, 1.92e-01 / 1.51e-01 | 20260506 11:46:33.042 | INFO | PerfReport: 15.85%, DME -- 4.05e+01 / 4.51e+01, 4.15e+00 / 2.44e+00 | 20260506 11:46:33.042 | INFO | PerfReport: 56.18%, FrameLabeling -- 1.79e+01 / 1.99e+01, 1.38e+00 / 1.52e+00 | 20260506 11:46:33.042 | INFO | PerfReport: 7.70%, PulseAccumulating -- 2.46e+00 / 3.13e+00, 4.39e-01 / 1.99e-01 | 20260506 11:46:33.042 | INFO | PerfReport: 3.84%, Metrics -- 1.23e+00 / 1.65e+00, 3.12e-01 / 1.50e-01 | 20260506 11:46:33.042 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.33e-01 / 4.98e-01, 6.80e-02 / 3.01e-02 | 20260506 11:46:33.042 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:46:33.042 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:46:33.042 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:46:33.042 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 11:46:33.042 | INFO | PerfReport: 21.64%, Upload -- 2.14e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260506 11:46:33.042 | INFO | PerfReport: 21.13%, Download -- 2.09e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 11:46:33.042 | INFO | PerfReport: 32.30%, Compute -- 3.19e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260506 11:46:33.042 | INFO | PerfReport: 1.94%, LockUpload -- 1.91e+00 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 11:46:33.042 | INFO | PerfReport: 0.91%, LockDownload -- 9.00e-01 / 0.00e+00, 2.42e+00 / 0.00e+00 | 20260506 11:46:33.042 | INFO | PerfReport: 22.07%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 11:46:33.042 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 2.15e-02 / 0.00e+00 | 20260506 11:46:33.167 | INFO | Starting a round of Baz IO | 20260506 11:46:36.368 | INFO | Populating RTMetrics report | 20260506 11:46:36.368 | INFO | RTMetrics report populated | 20260506 11:46:36.368 | INFO | Opening tmp RTMetrics json file | 20260506 11:46:36.457 | INFO | Serializing RTMetrics to json | 20260506 11:46:36.480 | INFO | Serialization done, swapping tmp file now... | 20260506 11:46:36.483 | INFO | RTMetrics done | 20260506 11:46:36.697 | INFO | Finished a round of Baz IO: Wrote 2.076 GiB in 3.529 seconds | 20260506 11:46:36.697 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.22% / 0.03% / 56.75% | 20260506 11:46:36.697 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.78868% / 4.76837e-05% / 7.15256e-05% | 20260506 11:46:36.697 | INFO | Baz IO overall: | 20260506 11:46:36.697 | INFO | BytesWritten : 537151328093 | 20260506 11:46:36.697 | INFO | NumEvents : 259350797125 | 20260506 11:46:38.179 | INFO | Analyzing chunk frames = [1105920,1106432) | 20260506 11:46:38.179 | INFO | MemInfo: { | 20260506 11:46:38.179 | INFO | "gpu" : " 17 GB", | 20260506 11:46:38.179 | INFO | "huge" : "", | 20260506 11:46:38.179 | INFO | "huge_shared" : " 27 GB", | 20260506 11:46:38.179 | INFO | "malloc" : " 14 GB", | 20260506 11:46:38.179 | INFO | "pinned" : " 29 GB" | 20260506 11:46:38.179 | INFO | } | 20260506 11:46:38.179 | INFO | Perf report for the last 5.13912 seconds: | 20260506 11:46:38.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:46:38.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:38.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:38.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:38.179 | INFO | ANALYSIS : 63.9%, 2.9, 95.4ms, 0 | 20260506 11:46:38.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:38.179 | INFO | RT_METRICS : 47.9%, 1.0, 25.1ms, 0 | 20260506 11:46:38.179 | INFO | PRE_HQ : 61.9%, 2.0, 62.9ms, 0 | 20260506 11:46:38.179 | INFO | BAZWRITER : 68.0%, 4.8, 167.5ms, 0 | 20260506 11:46:38.180 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 11:46:38.180 | INFO | Uploaded 36GiB to GPU in 2.04874s (17.5718GiB/s) | 20260506 11:46:38.180 | INFO | Downloaded 38GiB to GPU in 2.20075s (17.2668GiB/s) | 20260506 11:46:43.319 | INFO | Analyzing chunk frames = [1106432,1106944) | 20260506 11:46:43.319 | INFO | MemInfo: { | 20260506 11:46:43.319 | INFO | "gpu" : " 17 GB", | 20260506 11:46:43.319 | INFO | "huge" : "", | 20260506 11:46:43.319 | INFO | "huge_shared" : " 27 GB", | 20260506 11:46:43.319 | INFO | "malloc" : " 14 GB", | 20260506 11:46:43.319 | INFO | "pinned" : " 29 GB" | 20260506 11:46:43.319 | INFO | } | 20260506 11:46:43.319 | INFO | Perf report for the last 5.13987 seconds: | 20260506 11:46:43.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:46:43.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:43.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:43.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:43.319 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:46:43.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:43.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:43.319 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 11:46:43.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:43.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:43.319 | INFO | Uploaded 36GiB to GPU in 2.00386s (17.9654GiB/s) | 20260506 11:46:43.319 | INFO | Downloaded 34GiB to GPU in 1.92125s (17.6968GiB/s) | 20260506 11:46:48.458 | INFO | Analyzing chunk frames = [1106944,1107456) | 20260506 11:46:48.458 | INFO | MemInfo: { | 20260506 11:46:48.458 | INFO | "gpu" : " 17 GB", | 20260506 11:46:48.458 | INFO | "huge" : "", | 20260506 11:46:48.458 | INFO | "huge_shared" : " 27 GB", | 20260506 11:46:48.458 | INFO | "malloc" : " 14 GB", | 20260506 11:46:48.458 | INFO | "pinned" : " 29 GB" | 20260506 11:46:48.458 | INFO | } | 20260506 11:46:48.458 | INFO | Perf report for the last 5.1391 seconds: | 20260506 11:46:48.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:46:48.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:48.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:48.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:48.458 | INFO | ANALYSIS : 62.1%, 3.0, 96.1ms, 0 | 20260506 11:46:48.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:48.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:48.458 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260506 11:46:48.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:48.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:48.458 | INFO | Uploaded 36GiB to GPU in 2.03488s (17.6914GiB/s) | 20260506 11:46:48.459 | INFO | Downloaded 34GiB to GPU in 1.94769s (17.4566GiB/s) | 20260506 11:46:53.597 | INFO | Analyzing chunk frames = [1107456,1107968) | 20260506 11:46:53.597 | INFO | MemInfo: { | 20260506 11:46:53.597 | INFO | "gpu" : " 17 GB", | 20260506 11:46:53.597 | INFO | "huge" : "", | 20260506 11:46:53.597 | INFO | "huge_shared" : " 27 GB", | 20260506 11:46:53.597 | INFO | "malloc" : " 14 GB", | 20260506 11:46:53.597 | INFO | "pinned" : " 29 GB" | 20260506 11:46:53.597 | INFO | } | 20260506 11:46:53.597 | INFO | Perf report for the last 5.13917 seconds: | 20260506 11:46:53.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:46:53.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:53.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:53.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:53.597 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 11:46:53.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:53.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:53.597 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:46:53.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:53.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:53.598 | INFO | Uploaded 36GiB to GPU in 2.06951s (17.3954GiB/s) | 20260506 11:46:53.598 | INFO | Downloaded 34GiB to GPU in 1.97154s (17.2454GiB/s) | 20260506 11:46:58.737 | INFO | Analyzing chunk frames = [1107968,1108480) | 20260506 11:46:58.737 | INFO | MemInfo: { | 20260506 11:46:58.737 | INFO | "gpu" : " 17 GB", | 20260506 11:46:58.737 | INFO | "huge" : "", | 20260506 11:46:58.737 | INFO | "huge_shared" : " 27 GB", | 20260506 11:46:58.737 | INFO | "malloc" : " 14 GB", | 20260506 11:46:58.737 | INFO | "pinned" : " 29 GB" | 20260506 11:46:58.737 | INFO | } | 20260506 11:46:58.737 | INFO | Perf report for the last 5.13942 seconds: | 20260506 11:46:58.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:46:58.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:58.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:58.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:58.737 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 11:46:58.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:58.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:58.737 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:46:58.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:58.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:46:58.737 | INFO | Uploaded 36GiB to GPU in 2.08626s (17.2558GiB/s) | 20260506 11:46:58.737 | INFO | Downloaded 34GiB to GPU in 1.98982s (17.087GiB/s) | 20260506 11:47:03.876 | INFO | Analyzing chunk frames = [1108480,1108992) | 20260506 11:47:03.876 | INFO | MemInfo: { | 20260506 11:47:03.876 | INFO | "gpu" : " 17 GB", | 20260506 11:47:03.876 | INFO | "huge" : "", | 20260506 11:47:03.876 | INFO | "huge_shared" : " 27 GB", | 20260506 11:47:03.876 | INFO | "malloc" : " 14 GB", | 20260506 11:47:03.876 | INFO | "pinned" : " 29 GB" | 20260506 11:47:03.876 | INFO | } | 20260506 11:47:03.876 | INFO | Perf report for the last 5.13933 seconds: | 20260506 11:47:03.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:47:03.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:03.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:03.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:03.876 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 11:47:03.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:03.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:03.876 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:47:03.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:03.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:03.876 | INFO | Uploaded 36GiB to GPU in 2.01554s (17.8612GiB/s) | 20260506 11:47:03.877 | INFO | Downloaded 34GiB to GPU in 1.99981s (17.0016GiB/s) | 20260506 11:47:09.015 | INFO | Analyzing chunk frames = [1108992,1109504) | 20260506 11:47:09.015 | INFO | MemInfo: { | 20260506 11:47:09.015 | INFO | "gpu" : " 17 GB", | 20260506 11:47:09.015 | INFO | "huge" : "", | 20260506 11:47:09.015 | INFO | "huge_shared" : " 27 GB", | 20260506 11:47:09.016 | INFO | "malloc" : " 14 GB", | 20260506 11:47:09.016 | INFO | "pinned" : " 29 GB" | 20260506 11:47:09.016 | INFO | } | 20260506 11:47:09.016 | INFO | Perf report for the last 5.13954 seconds: | 20260506 11:47:09.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:47:09.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:09.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:09.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:09.016 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 11:47:09.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:09.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:09.016 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 11:47:09.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:09.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:09.016 | INFO | Uploaded 36GiB to GPU in 1.98248s (18.159GiB/s) | 20260506 11:47:09.016 | INFO | Downloaded 34GiB to GPU in 2.03605s (16.699GiB/s) | 20260506 11:47:14.155 | INFO | Analyzing chunk frames = [1109504,1110016) | 20260506 11:47:14.155 | INFO | MemInfo: { | 20260506 11:47:14.155 | INFO | "gpu" : " 17 GB", | 20260506 11:47:14.155 | INFO | "huge" : "", | 20260506 11:47:14.155 | INFO | "huge_shared" : " 27 GB", | 20260506 11:47:14.155 | INFO | "malloc" : " 14 GB", | 20260506 11:47:14.155 | INFO | "pinned" : " 29 GB" | 20260506 11:47:14.155 | INFO | } | 20260506 11:47:14.155 | INFO | Perf report for the last 5.13919 seconds: | 20260506 11:47:14.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:47:14.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:14.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:14.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:14.155 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260506 11:47:14.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:14.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:14.155 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260506 11:47:14.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:14.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:14.155 | INFO | Uploaded 36GiB to GPU in 2.05895s (17.4846GiB/s) | 20260506 11:47:14.155 | INFO | Downloaded 34GiB to GPU in 2.00257s (16.9782GiB/s) | 20260506 11:47:14.272 | INFO | Starting a round of Baz IO | 20260506 11:47:17.482 | INFO | Populating RTMetrics report | 20260506 11:47:17.482 | INFO | RTMetrics report populated | 20260506 11:47:17.482 | INFO | Opening tmp RTMetrics json file | 20260506 11:47:17.520 | INFO | Serializing RTMetrics to json | 20260506 11:47:17.530 | INFO | Serialization done, swapping tmp file now... | 20260506 11:47:17.537 | INFO | RTMetrics done | 20260506 11:47:17.735 | INFO | Finished a round of Baz IO: Wrote 2.076 GiB in 3.462 seconds | 20260506 11:47:17.735 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.19% / 0.03% / 56.78% | 20260506 11:47:17.735 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.77351% / 6.35783e-05% / 4.37101e-05% | 20260506 11:47:17.735 | INFO | Baz IO overall: | 20260506 11:47:17.735 | INFO | BytesWritten : 539380846246 | 20260506 11:47:17.735 | INFO | NumEvents : 260433418226 | 20260506 11:47:19.294 | INFO | Analyzing chunk frames = [1110016,1110528) | 20260506 11:47:19.294 | INFO | MemInfo: { | 20260506 11:47:19.294 | INFO | "gpu" : " 17 GB", | 20260506 11:47:19.294 | INFO | "huge" : "", | 20260506 11:47:19.294 | INFO | "huge_shared" : " 27 GB", | 20260506 11:47:19.294 | INFO | "malloc" : " 14 GB", | 20260506 11:47:19.294 | INFO | "pinned" : " 29 GB" | 20260506 11:47:19.294 | INFO | } | 20260506 11:47:19.294 | INFO | Perf report for the last 5.13905 seconds: | 20260506 11:47:19.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:47:19.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:19.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:19.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:19.294 | INFO | ANALYSIS : 63.8%, 2.9, 96.1ms, 0 | 20260506 11:47:19.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:19.294 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 11:47:19.294 | INFO | PRE_HQ : 61.7%, 1.9, 61.5ms, 0 | 20260506 11:47:19.294 | INFO | BAZWRITER : 66.5%, 4.7, 162.1ms, 0 | 20260506 11:47:19.294 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 11:47:19.294 | INFO | Uploaded 36GiB to GPU in 2.0934s (17.1969GiB/s) | 20260506 11:47:19.294 | INFO | Downloaded 38GiB to GPU in 2.22445s (17.0829GiB/s) | 20260506 11:47:24.433 | INFO | Analyzing chunk frames = [1110528,1111040) | 20260506 11:47:24.433 | INFO | MemInfo: { | 20260506 11:47:24.433 | INFO | "gpu" : " 17 GB", | 20260506 11:47:24.433 | INFO | "huge" : "", | 20260506 11:47:24.433 | INFO | "huge_shared" : " 27 GB", | 20260506 11:47:24.433 | INFO | "malloc" : " 14 GB", | 20260506 11:47:24.433 | INFO | "pinned" : " 29 GB" | 20260506 11:47:24.433 | INFO | } | 20260506 11:47:24.433 | INFO | Perf report for the last 5.13934 seconds: | 20260506 11:47:24.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:47:24.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:24.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:24.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:24.433 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:47:24.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:24.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:24.433 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 11:47:24.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:24.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:24.434 | INFO | Uploaded 36GiB to GPU in 2.01018s (17.9089GiB/s) | 20260506 11:47:24.434 | INFO | Downloaded 34GiB to GPU in 1.92414s (17.6702GiB/s) | 20260506 11:47:29.573 | INFO | Analyzing chunk frames = [1111040,1111552) | 20260506 11:47:29.573 | INFO | MemInfo: { | 20260506 11:47:29.573 | INFO | "gpu" : " 17 GB", | 20260506 11:47:29.573 | INFO | "huge" : "", | 20260506 11:47:29.573 | INFO | "huge_shared" : " 27 GB", | 20260506 11:47:29.573 | INFO | "malloc" : " 14 GB", | 20260506 11:47:29.573 | INFO | "pinned" : " 29 GB" | 20260506 11:47:29.573 | INFO | } | 20260506 11:47:29.573 | INFO | Perf report for the last 5.13962 seconds: | 20260506 11:47:29.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:47:29.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:29.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:29.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:29.573 | INFO | ANALYSIS : 61.9%, 3.0, 95.8ms, 0 | 20260506 11:47:29.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:29.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:29.573 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:47:29.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:29.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:29.573 | INFO | Uploaded 36GiB to GPU in 2.01931s (17.8279GiB/s) | 20260506 11:47:29.574 | INFO | Downloaded 34GiB to GPU in 1.94599s (17.4718GiB/s) | 20260506 11:47:34.712 | INFO | Analyzing chunk frames = [1111552,1112064) | 20260506 11:47:34.712 | INFO | MemInfo: { | 20260506 11:47:34.712 | INFO | "gpu" : " 17 GB", | 20260506 11:47:34.712 | INFO | "huge" : "", | 20260506 11:47:34.712 | INFO | "huge_shared" : " 27 GB", | 20260506 11:47:34.712 | INFO | "malloc" : " 14 GB", | 20260506 11:47:34.712 | INFO | "pinned" : " 29 GB" | 20260506 11:47:34.712 | INFO | } | 20260506 11:47:34.712 | INFO | Perf report for the last 5.13909 seconds: | 20260506 11:47:34.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:47:34.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:34.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:34.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:34.712 | INFO | ANALYSIS : 62.2%, 2.9, 96.2ms, 0 | 20260506 11:47:34.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:34.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:34.712 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:47:34.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:34.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:34.712 | INFO | Uploaded 36GiB to GPU in 2.00755s (17.9323GiB/s) | 20260506 11:47:34.712 | INFO | Downloaded 34GiB to GPU in 1.97901s (17.1803GiB/s) | 20260506 11:47:39.851 | INFO | Analyzing chunk frames = [1112064,1112576) | 20260506 11:47:39.851 | INFO | MemInfo: { | 20260506 11:47:39.851 | INFO | "gpu" : " 17 GB", | 20260506 11:47:39.851 | INFO | "huge" : "", | 20260506 11:47:39.851 | INFO | "huge_shared" : " 27 GB", | 20260506 11:47:39.851 | INFO | "malloc" : " 14 GB", | 20260506 11:47:39.851 | INFO | "pinned" : " 29 GB" | 20260506 11:47:39.851 | INFO | } | 20260506 11:47:39.851 | INFO | Perf report for the last 5.1392 seconds: | 20260506 11:47:39.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:47:39.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:39.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:39.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:39.851 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 11:47:39.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:39.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:39.851 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:47:39.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:39.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:39.852 | INFO | Uploaded 36GiB to GPU in 2.00706s (17.9367GiB/s) | 20260506 11:47:39.852 | INFO | Downloaded 34GiB to GPU in 1.96031s (17.3442GiB/s) | 20260506 11:47:44.990 | INFO | Analyzing chunk frames = [1112576,1113088) | 20260506 11:47:44.990 | INFO | MemInfo: { | 20260506 11:47:44.990 | INFO | "gpu" : " 17 GB", | 20260506 11:47:44.990 | INFO | "huge" : "", | 20260506 11:47:44.990 | INFO | "huge_shared" : " 27 GB", | 20260506 11:47:44.991 | INFO | "malloc" : " 14 GB", | 20260506 11:47:44.991 | INFO | "pinned" : " 29 GB" | 20260506 11:47:44.991 | INFO | } | 20260506 11:47:44.991 | INFO | Perf report for the last 5.13944 seconds: | 20260506 11:47:44.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:47:44.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:44.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:44.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:44.991 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 11:47:44.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:44.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:44.991 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 11:47:44.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:44.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:44.991 | INFO | Uploaded 36GiB to GPU in 2.09213s (17.2074GiB/s) | 20260506 11:47:44.991 | INFO | Downloaded 34GiB to GPU in 1.97721s (17.1959GiB/s) | 20260506 11:47:50.131 | INFO | Analyzing chunk frames = [1113088,1113600) | 20260506 11:47:50.131 | INFO | MemInfo: { | 20260506 11:47:50.131 | INFO | "gpu" : " 17 GB", | 20260506 11:47:50.131 | INFO | "huge" : "", | 20260506 11:47:50.131 | INFO | "huge_shared" : " 27 GB", | 20260506 11:47:50.131 | INFO | "malloc" : " 14 GB", | 20260506 11:47:50.131 | INFO | "pinned" : " 29 GB" | 20260506 11:47:50.131 | INFO | } | 20260506 11:47:50.131 | INFO | Perf report for the last 5.1407 seconds: | 20260506 11:47:50.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:47:50.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:50.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:50.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:50.131 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260506 11:47:50.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:50.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:50.132 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 11:47:50.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:50.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:50.132 | INFO | Uploaded 36GiB to GPU in 2.04306s (17.6206GiB/s) | 20260506 11:47:50.132 | INFO | Downloaded 34GiB to GPU in 2.00182s (16.9845GiB/s) | 20260506 11:47:55.270 | INFO | Analyzing chunk frames = [1113600,1114112) | 20260506 11:47:55.270 | INFO | MemInfo: { | 20260506 11:47:55.270 | INFO | "gpu" : " 17 GB", | 20260506 11:47:55.270 | INFO | "huge" : "", | 20260506 11:47:55.270 | INFO | "huge_shared" : " 27 GB", | 20260506 11:47:55.270 | INFO | "malloc" : " 14 GB", | 20260506 11:47:55.270 | INFO | "pinned" : " 29 GB" | 20260506 11:47:55.270 | INFO | } | 20260506 11:47:55.270 | INFO | Perf report for the last 5.13846 seconds: | 20260506 11:47:55.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:47:55.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:55.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:55.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:55.270 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260506 11:47:55.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:55.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:55.270 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260506 11:47:55.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:55.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:47:55.270 | INFO | Uploaded 36GiB to GPU in 2.03911s (17.6548GiB/s) | 20260506 11:47:55.270 | INFO | Downloaded 34GiB to GPU in 1.9622s (17.3275GiB/s) | 20260506 11:47:55.270 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:47:55.270 | INFO | Total mem usage: 16.8 GiB | 20260506 11:47:55.270 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:47:55.270 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -8.72%) | 20260506 11:47:55.270 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:47:55.270 | INFO | High Water Breakdown: | 20260506 11:47:55.270 | INFO | 60.7% : 9.38 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:47:55.271 | INFO | 39.2% : 6.06 GiB ( 90.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:47:55.271 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:47:55.271 | INFO | ------------------------------------------ | 20260506 11:47:55.271 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:47:55.271 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:47:55.271 | INFO | Total mem usage: 26.5 GiB | 20260506 11:47:55.271 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:47:55.271 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:47:55.271 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:47:55.271 | INFO | High Water Breakdown: | 20260506 11:47:55.271 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:47:55.271 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:47:55.271 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:47:55.271 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:47:55.271 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:47:55.271 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:47:55.271 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:47:55.271 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:47:55.271 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:47:55.271 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:47:55.271 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:47:55.271 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:47:55.271 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:47:55.271 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:47:55.271 | INFO | ------------------------------------------ | 20260506 11:47:55.271 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:47:55.271 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:47:55.271 | INFO | Total mem usage: 25.6 GiB | 20260506 11:47:55.271 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:47:55.271 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:47:55.271 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:47:55.271 | INFO | High Water Breakdown: | 20260506 11:47:55.271 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:47:55.271 | INFO | ------------------------------------------ | 20260506 11:47:55.271 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:47:55.271 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:47:55.271 | INFO | Total mem usage: 15.7 GiB | 20260506 11:47:55.271 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:47:55.271 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:47:55.271 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:47:55.271 | INFO | High Water Breakdown: | 20260506 11:47:55.271 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:47:55.271 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:47:55.271 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:47:55.271 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:47:55.271 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:47:55.271 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:47:55.271 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:47:55.271 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:47:55.271 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:47:55.271 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:47:55.271 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:47:55.271 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:47:55.271 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:47:55.271 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:47:55.271 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:47:55.271 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:47:55.271 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:47:55.271 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:47:55.271 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:47:55.271 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:47:55.271 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:47:55.271 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:47:55.271 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:47:55.271 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:47:55.271 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:47:55.271 | INFO | ------------------------------------------ | 20260506 11:47:55.271 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:47:55.271 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:47:55.271 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:47:55.271 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:47:55.271 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.68e+01, 1.35e+01 / 2.57e+00 | 20260506 11:47:55.271 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.02e-01, 1.03e-02 / 6.40e-03 | 20260506 11:47:55.271 | INFO | PerfReport: 11.55%, Baseline -- 3.69e+00 / 4.46e+00, 5.96e-01 / 3.67e-01 | 20260506 11:47:55.271 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.47e+00, 1.91e-01 / 1.38e-01 | 20260506 11:47:55.271 | INFO | PerfReport: 15.89%, DME -- 4.06e+01 / 4.54e+01, 4.31e+00 / 2.92e+00 | 20260506 11:47:55.271 | INFO | PerfReport: 56.17%, FrameLabeling -- 1.79e+01 / 2.01e+01, 1.41e+00 / 1.88e+00 | 20260506 11:47:55.271 | INFO | PerfReport: 7.67%, PulseAccumulating -- 2.45e+00 / 3.11e+00, 4.32e-01 / 2.11e-01 | 20260506 11:47:55.271 | INFO | PerfReport: 3.83%, Metrics -- 1.22e+00 / 1.64e+00, 3.11e-01 / 2.00e-01 | 20260506 11:47:55.271 | INFO | PerfReport: 1.36%, PulseSerializing -- 4.33e-01 / 4.96e-01, 6.83e-02 / 2.83e-02 | 20260506 11:47:55.271 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:47:55.271 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:47:55.271 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:47:55.271 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 11:47:55.271 | INFO | PerfReport: 21.50%, Upload -- 2.12e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 11:47:55.272 | INFO | PerfReport: 21.13%, Download -- 2.09e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 11:47:55.272 | INFO | PerfReport: 32.32%, Compute -- 3.19e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 11:47:55.272 | INFO | PerfReport: 1.85%, LockUpload -- 1.83e+00 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 11:47:55.272 | INFO | PerfReport: 1.06%, LockDownload -- 1.04e+00 / 0.00e+00, 2.67e+00 / 0.00e+00 | 20260506 11:47:55.272 | INFO | PerfReport: 22.12%, LockCompute -- 2.18e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260506 11:47:55.272 | INFO | PerfReport: 0.02%, Misc -- 1.52e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260506 11:47:55.395 | INFO | Starting a round of Baz IO | 20260506 11:47:58.592 | INFO | Populating RTMetrics report | 20260506 11:47:58.592 | INFO | RTMetrics report populated | 20260506 11:47:58.592 | INFO | Opening tmp RTMetrics json file | 20260506 11:47:58.606 | INFO | Serializing RTMetrics to json | 20260506 11:47:58.608 | INFO | Serialization done, swapping tmp file now... | 20260506 11:47:58.609 | INFO | RTMetrics done | 20260506 11:47:58.781 | INFO | Finished a round of Baz IO: Wrote 2.073 GiB in 3.386 seconds | 20260506 11:47:58.781 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.16% / 0.03% / 56.81% | 20260506 11:47:58.781 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.73356% / 6.35783e-05% / 3.57628e-05% | 20260506 11:47:58.781 | INFO | Baz IO overall: | 20260506 11:47:58.781 | INFO | BytesWritten : 541606734939 | 20260506 11:47:58.781 | INFO | NumEvents : 261513220103 | 20260506 11:48:00.408 | INFO | Analyzing chunk frames = [1114112,1114624) | 20260506 11:48:00.409 | INFO | MemInfo: { | 20260506 11:48:00.409 | INFO | "gpu" : " 17 GB", | 20260506 11:48:00.409 | INFO | "huge" : "", | 20260506 11:48:00.409 | INFO | "huge_shared" : " 27 GB", | 20260506 11:48:00.409 | INFO | "malloc" : " 14 GB", | 20260506 11:48:00.409 | INFO | "pinned" : " 29 GB" | 20260506 11:48:00.409 | INFO | } | 20260506 11:48:00.409 | INFO | Perf report for the last 5.13871 seconds: | 20260506 11:48:00.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:48:00.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:00.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:00.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:00.409 | INFO | ANALYSIS : 64.0%, 2.9, 96.7ms, 0 | 20260506 11:48:00.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:00.409 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 11:48:00.409 | INFO | PRE_HQ : 61.8%, 1.9, 61.0ms, 0 | 20260506 11:48:00.409 | INFO | BAZWRITER : 65.0%, 4.8, 160.2ms, 0 | 20260506 11:48:00.409 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260506 11:48:00.409 | INFO | Uploaded 36GiB to GPU in 2.14882s (16.7534GiB/s) | 20260506 11:48:00.409 | INFO | Downloaded 38GiB to GPU in 2.23658s (16.9903GiB/s) | 20260506 11:48:05.548 | INFO | Analyzing chunk frames = [1114624,1115136) | 20260506 11:48:05.548 | INFO | MemInfo: { | 20260506 11:48:05.548 | INFO | "gpu" : " 17 GB", | 20260506 11:48:05.548 | INFO | "huge" : "", | 20260506 11:48:05.548 | INFO | "huge_shared" : " 27 GB", | 20260506 11:48:05.548 | INFO | "malloc" : " 14 GB", | 20260506 11:48:05.548 | INFO | "pinned" : " 29 GB" | 20260506 11:48:05.548 | INFO | } | 20260506 11:48:05.548 | INFO | Perf report for the last 5.13973 seconds: | 20260506 11:48:05.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:48:05.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:05.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:05.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:05.548 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:48:05.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:05.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:05.548 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 11:48:05.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:05.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:05.549 | INFO | Uploaded 36GiB to GPU in 2.01524s (17.8639GiB/s) | 20260506 11:48:05.549 | INFO | Downloaded 34GiB to GPU in 1.91083s (17.7933GiB/s) | 20260506 11:48:10.687 | INFO | Analyzing chunk frames = [1115136,1115648) | 20260506 11:48:10.687 | INFO | MemInfo: { | 20260506 11:48:10.687 | INFO | "gpu" : " 17 GB", | 20260506 11:48:10.687 | INFO | "huge" : "", | 20260506 11:48:10.687 | INFO | "huge_shared" : " 27 GB", | 20260506 11:48:10.687 | INFO | "malloc" : " 14 GB", | 20260506 11:48:10.687 | INFO | "pinned" : " 29 GB" | 20260506 11:48:10.687 | INFO | } | 20260506 11:48:10.687 | INFO | Perf report for the last 5.13898 seconds: | 20260506 11:48:10.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:48:10.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:10.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:10.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:10.687 | INFO | ANALYSIS : 62.1%, 3.0, 96.1ms, 0 | 20260506 11:48:10.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:10.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:10.687 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:48:10.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:10.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:10.688 | INFO | Uploaded 36GiB to GPU in 2.04249s (17.6256GiB/s) | 20260506 11:48:10.688 | INFO | Downloaded 34GiB to GPU in 1.93666s (17.556GiB/s) | 20260506 11:48:15.827 | INFO | Analyzing chunk frames = [1115648,1116160) | 20260506 11:48:15.827 | INFO | MemInfo: { | 20260506 11:48:15.827 | INFO | "gpu" : " 17 GB", | 20260506 11:48:15.827 | INFO | "huge" : "", | 20260506 11:48:15.827 | INFO | "huge_shared" : " 27 GB", | 20260506 11:48:15.827 | INFO | "malloc" : " 14 GB", | 20260506 11:48:15.827 | INFO | "pinned" : " 29 GB" | 20260506 11:48:15.827 | INFO | } | 20260506 11:48:15.827 | INFO | Perf report for the last 5.13947 seconds: | 20260506 11:48:15.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:48:15.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:15.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:15.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:15.827 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 11:48:15.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:15.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:15.827 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:48:15.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:15.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:15.827 | INFO | Uploaded 36GiB to GPU in 2.08499s (17.2662GiB/s) | 20260506 11:48:15.827 | INFO | Downloaded 34GiB to GPU in 1.96416s (17.3102GiB/s) | 20260506 11:48:20.966 | INFO | Analyzing chunk frames = [1116160,1116672) | 20260506 11:48:20.966 | INFO | MemInfo: { | 20260506 11:48:20.966 | INFO | "gpu" : " 17 GB", | 20260506 11:48:20.966 | INFO | "huge" : "", | 20260506 11:48:20.966 | INFO | "huge_shared" : " 27 GB", | 20260506 11:48:20.966 | INFO | "malloc" : " 14 GB", | 20260506 11:48:20.966 | INFO | "pinned" : " 29 GB" | 20260506 11:48:20.966 | INFO | } | 20260506 11:48:20.966 | INFO | Perf report for the last 5.13928 seconds: | 20260506 11:48:20.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:48:20.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:20.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:20.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:20.966 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 11:48:20.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:20.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:20.966 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:48:20.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:20.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:20.966 | INFO | Uploaded 36GiB to GPU in 2.09469s (17.1863GiB/s) | 20260506 11:48:20.966 | INFO | Downloaded 34GiB to GPU in 1.99356s (17.0549GiB/s) | 20260506 11:48:26.105 | INFO | Analyzing chunk frames = [1116672,1117184) | 20260506 11:48:26.105 | INFO | MemInfo: { | 20260506 11:48:26.105 | INFO | "gpu" : " 17 GB", | 20260506 11:48:26.105 | INFO | "huge" : "", | 20260506 11:48:26.105 | INFO | "huge_shared" : " 27 GB", | 20260506 11:48:26.105 | INFO | "malloc" : " 14 GB", | 20260506 11:48:26.105 | INFO | "pinned" : " 29 GB" | 20260506 11:48:26.105 | INFO | } | 20260506 11:48:26.106 | INFO | Perf report for the last 5.13939 seconds: | 20260506 11:48:26.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:48:26.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:26.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:26.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:26.106 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 11:48:26.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:26.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:26.106 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:48:26.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:26.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:26.106 | INFO | Uploaded 36GiB to GPU in 2.10202s (17.1264GiB/s) | 20260506 11:48:26.106 | INFO | Downloaded 34GiB to GPU in 2.00896s (16.9242GiB/s) | 20260506 11:48:31.245 | INFO | Analyzing chunk frames = [1117184,1117696) | 20260506 11:48:31.245 | INFO | MemInfo: { | 20260506 11:48:31.245 | INFO | "gpu" : " 17 GB", | 20260506 11:48:31.245 | INFO | "huge" : "", | 20260506 11:48:31.245 | INFO | "huge_shared" : " 27 GB", | 20260506 11:48:31.245 | INFO | "malloc" : " 14 GB", | 20260506 11:48:31.245 | INFO | "pinned" : " 29 GB" | 20260506 11:48:31.245 | INFO | } | 20260506 11:48:31.245 | INFO | Perf report for the last 5.13973 seconds: | 20260506 11:48:31.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:48:31.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:31.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:31.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:31.245 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:48:31.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:31.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:31.245 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 11:48:31.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:31.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:31.246 | INFO | Uploaded 36GiB to GPU in 2.0123s (17.8899GiB/s) | 20260506 11:48:31.246 | INFO | Downloaded 34GiB to GPU in 2.01986s (16.8329GiB/s) | 20260506 11:48:36.384 | INFO | Analyzing chunk frames = [1117696,1118208) | 20260506 11:48:36.384 | INFO | MemInfo: { | 20260506 11:48:36.384 | INFO | "gpu" : " 17 GB", | 20260506 11:48:36.384 | INFO | "huge" : "", | 20260506 11:48:36.384 | INFO | "huge_shared" : " 27 GB", | 20260506 11:48:36.384 | INFO | "malloc" : " 14 GB", | 20260506 11:48:36.384 | INFO | "pinned" : " 29 GB" | 20260506 11:48:36.384 | INFO | } | 20260506 11:48:36.384 | INFO | Perf report for the last 5.13902 seconds: | 20260506 11:48:36.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:48:36.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:36.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:36.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:36.384 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260506 11:48:36.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:36.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:36.384 | INFO | PRE_HQ : 32.6%, 1.0, 17.0ms, 0 | 20260506 11:48:36.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:36.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:36.385 | INFO | Uploaded 36GiB to GPU in 2.02994s (17.7345GiB/s) | 20260506 11:48:36.385 | INFO | Downloaded 34GiB to GPU in 2.00608s (16.9485GiB/s) | 20260506 11:48:36.511 | INFO | Starting a round of Baz IO | 20260506 11:48:39.688 | INFO | Populating RTMetrics report | 20260506 11:48:39.688 | INFO | RTMetrics report populated | 20260506 11:48:39.688 | INFO | Opening tmp RTMetrics json file | 20260506 11:48:39.693 | INFO | Serializing RTMetrics to json | 20260506 11:48:39.720 | INFO | Serialization done, swapping tmp file now... | 20260506 11:48:39.730 | INFO | RTMetrics done | 20260506 11:48:39.897 | INFO | Finished a round of Baz IO: Wrote 2.077 GiB in 3.386 seconds | 20260506 11:48:39.897 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.12% / 0.03% / 56.84% | 20260506 11:48:39.897 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.74166% / 4.76837e-05% / 6.35783e-05% | 20260506 11:48:39.897 | INFO | Baz IO overall: | 20260506 11:48:39.897 | INFO | BytesWritten : 543836894569 | 20260506 11:48:39.897 | INFO | NumEvents : 262596047664 | 20260506 11:48:41.524 | INFO | Analyzing chunk frames = [1118208,1118720) | 20260506 11:48:41.524 | INFO | MemInfo: { | 20260506 11:48:41.524 | INFO | "gpu" : " 17 GB", | 20260506 11:48:41.524 | INFO | "huge" : "", | 20260506 11:48:41.524 | INFO | "huge_shared" : " 27 GB", | 20260506 11:48:41.524 | INFO | "malloc" : " 14 GB", | 20260506 11:48:41.524 | INFO | "pinned" : " 29 GB" | 20260506 11:48:41.524 | INFO | } | 20260506 11:48:41.524 | INFO | Perf report for the last 5.13941 seconds: | 20260506 11:48:41.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:48:41.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:41.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:41.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:41.524 | INFO | ANALYSIS : 63.7%, 2.9, 95.8ms, 0 | 20260506 11:48:41.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:41.524 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260506 11:48:41.524 | INFO | PRE_HQ : 62.0%, 1.9, 61.1ms, 0 | 20260506 11:48:41.524 | INFO | BAZWRITER : 65.1%, 4.8, 160.7ms, 0 | 20260506 11:48:41.524 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 11:48:41.524 | INFO | Uploaded 36GiB to GPU in 2.12355s (16.9527GiB/s) | 20260506 11:48:41.524 | INFO | Downloaded 38GiB to GPU in 2.23422s (17.0082GiB/s) | 20260506 11:48:46.663 | INFO | Analyzing chunk frames = [1118720,1119232) | 20260506 11:48:46.663 | INFO | MemInfo: { | 20260506 11:48:46.663 | INFO | "gpu" : " 17 GB", | 20260506 11:48:46.663 | INFO | "huge" : "", | 20260506 11:48:46.663 | INFO | "huge_shared" : " 27 GB", | 20260506 11:48:46.663 | INFO | "malloc" : " 14 GB", | 20260506 11:48:46.663 | INFO | "pinned" : " 29 GB" | 20260506 11:48:46.663 | INFO | } | 20260506 11:48:46.663 | INFO | Perf report for the last 5.13972 seconds: | 20260506 11:48:46.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:48:46.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:46.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:46.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:46.664 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:48:46.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:46.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:46.664 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 11:48:46.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:46.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:46.664 | INFO | Uploaded 36GiB to GPU in 2.03198s (17.7167GiB/s) | 20260506 11:48:46.664 | INFO | Downloaded 34GiB to GPU in 1.93661s (17.5565GiB/s) | 20260506 11:48:51.802 | INFO | Analyzing chunk frames = [1119232,1119744) | 20260506 11:48:51.802 | INFO | MemInfo: { | 20260506 11:48:51.802 | INFO | "gpu" : " 17 GB", | 20260506 11:48:51.802 | INFO | "huge" : "", | 20260506 11:48:51.802 | INFO | "huge_shared" : " 27 GB", | 20260506 11:48:51.802 | INFO | "malloc" : " 14 GB", | 20260506 11:48:51.802 | INFO | "pinned" : " 29 GB" | 20260506 11:48:51.802 | INFO | } | 20260506 11:48:51.802 | INFO | Perf report for the last 5.13891 seconds: | 20260506 11:48:51.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:48:51.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:51.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:51.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:51.802 | INFO | ANALYSIS : 62.0%, 3.0, 96.0ms, 0 | 20260506 11:48:51.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:51.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:51.802 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:48:51.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:51.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:51.803 | INFO | Uploaded 36GiB to GPU in 2.03245s (17.7126GiB/s) | 20260506 11:48:51.803 | INFO | Downloaded 34GiB to GPU in 1.93414s (17.5789GiB/s) | 20260506 11:48:56.942 | INFO | Analyzing chunk frames = [1119744,1120256) | 20260506 11:48:56.942 | INFO | MemInfo: { | 20260506 11:48:56.942 | INFO | "gpu" : " 17 GB", | 20260506 11:48:56.942 | INFO | "huge" : "", | 20260506 11:48:56.942 | INFO | "huge_shared" : " 27 GB", | 20260506 11:48:56.942 | INFO | "malloc" : " 14 GB", | 20260506 11:48:56.942 | INFO | "pinned" : " 29 GB" | 20260506 11:48:56.942 | INFO | } | 20260506 11:48:56.942 | INFO | Perf report for the last 5.13976 seconds: | 20260506 11:48:56.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:48:56.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:56.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:56.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:56.942 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260506 11:48:56.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:56.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:56.942 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:48:56.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:56.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:48:56.942 | INFO | Uploaded 36GiB to GPU in 2.08401s (17.2744GiB/s) | 20260506 11:48:56.942 | INFO | Downloaded 34GiB to GPU in 1.96066s (17.3411GiB/s) | 20260506 11:49:02.081 | INFO | Analyzing chunk frames = [1120256,1120768) | 20260506 11:49:02.081 | INFO | MemInfo: { | 20260506 11:49:02.081 | INFO | "gpu" : " 17 GB", | 20260506 11:49:02.081 | INFO | "huge" : "", | 20260506 11:49:02.081 | INFO | "huge_shared" : " 27 GB", | 20260506 11:49:02.081 | INFO | "malloc" : " 14 GB", | 20260506 11:49:02.081 | INFO | "pinned" : " 29 GB" | 20260506 11:49:02.081 | INFO | } | 20260506 11:49:02.081 | INFO | Perf report for the last 5.1389 seconds: | 20260506 11:49:02.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:49:02.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:02.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:02.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:02.081 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 11:49:02.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:02.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:02.081 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:49:02.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:02.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:02.081 | INFO | Uploaded 36GiB to GPU in 2.10081s (17.1363GiB/s) | 20260506 11:49:02.081 | INFO | Downloaded 34GiB to GPU in 1.98502s (17.1283GiB/s) | 20260506 11:49:07.220 | INFO | Analyzing chunk frames = [1120768,1121280) | 20260506 11:49:07.220 | INFO | MemInfo: { | 20260506 11:49:07.220 | INFO | "gpu" : " 17 GB", | 20260506 11:49:07.220 | INFO | "huge" : "", | 20260506 11:49:07.220 | INFO | "huge_shared" : " 27 GB", | 20260506 11:49:07.220 | INFO | "malloc" : " 14 GB", | 20260506 11:49:07.220 | INFO | "pinned" : " 29 GB" | 20260506 11:49:07.220 | INFO | } | 20260506 11:49:07.220 | INFO | Perf report for the last 5.13906 seconds: | 20260506 11:49:07.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:49:07.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:07.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:07.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:07.220 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 11:49:07.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:07.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:07.220 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:49:07.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:07.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:07.220 | INFO | Uploaded 36GiB to GPU in 2.08208s (17.2904GiB/s) | 20260506 11:49:07.220 | INFO | Downloaded 34GiB to GPU in 1.9994s (17.0051GiB/s) | 20260506 11:49:12.360 | INFO | Analyzing chunk frames = [1121280,1121792) | 20260506 11:49:12.360 | INFO | MemInfo: { | 20260506 11:49:12.360 | INFO | "gpu" : " 17 GB", | 20260506 11:49:12.360 | INFO | "huge" : "", | 20260506 11:49:12.360 | INFO | "huge_shared" : " 27 GB", | 20260506 11:49:12.360 | INFO | "malloc" : " 14 GB", | 20260506 11:49:12.360 | INFO | "pinned" : " 29 GB" | 20260506 11:49:12.360 | INFO | } | 20260506 11:49:12.360 | INFO | Perf report for the last 5.14022 seconds: | 20260506 11:49:12.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:49:12.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:12.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:12.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:12.360 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:49:12.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:12.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:12.360 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:49:12.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:12.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:12.361 | INFO | Uploaded 36GiB to GPU in 1.99722s (18.025GiB/s) | 20260506 11:49:12.361 | INFO | Downloaded 34GiB to GPU in 1.99591s (17.0349GiB/s) | 20260506 11:49:17.499 | INFO | Analyzing chunk frames = [1121792,1122304) | 20260506 11:49:17.499 | INFO | MemInfo: { | 20260506 11:49:17.499 | INFO | "gpu" : " 17 GB", | 20260506 11:49:17.499 | INFO | "huge" : "", | 20260506 11:49:17.499 | INFO | "huge_shared" : " 27 GB", | 20260506 11:49:17.499 | INFO | "malloc" : " 14 GB", | 20260506 11:49:17.499 | INFO | "pinned" : " 29 GB" | 20260506 11:49:17.499 | INFO | } | 20260506 11:49:17.499 | INFO | Perf report for the last 5.13851 seconds: | 20260506 11:49:17.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:49:17.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:17.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:17.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:17.499 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260506 11:49:17.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:17.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:17.499 | INFO | PRE_HQ : 33.2%, 1.0, 17.3ms, 0 | 20260506 11:49:17.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:17.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:17.499 | INFO | Uploaded 36GiB to GPU in 2.04861s (17.5729GiB/s) | 20260506 11:49:17.499 | INFO | Downloaded 34GiB to GPU in 1.98752s (17.1067GiB/s) | 20260506 11:49:17.499 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:49:17.499 | INFO | Total mem usage: 16.8 GiB | 20260506 11:49:17.499 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:49:17.499 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -10.1%) | 20260506 11:49:17.499 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:49:17.499 | INFO | High Water Breakdown: | 20260506 11:49:17.499 | INFO | 60.4% : 9.22 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:49:17.500 | INFO | 39.5% : 6.02 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:49:17.500 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:49:17.500 | INFO | ------------------------------------------ | 20260506 11:49:17.500 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:49:17.500 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:49:17.500 | INFO | Total mem usage: 26.5 GiB | 20260506 11:49:17.500 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:49:17.500 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:49:17.500 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:49:17.500 | INFO | High Water Breakdown: | 20260506 11:49:17.500 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:49:17.500 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:49:17.500 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:49:17.500 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:49:17.500 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:49:17.500 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:49:17.500 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:49:17.500 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:49:17.500 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:49:17.500 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:49:17.500 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:49:17.500 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:49:17.500 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:49:17.500 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:49:17.500 | INFO | ------------------------------------------ | 20260506 11:49:17.500 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:49:17.500 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:49:17.500 | INFO | Total mem usage: 25.6 GiB | 20260506 11:49:17.500 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:49:17.500 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:49:17.500 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:49:17.500 | INFO | High Water Breakdown: | 20260506 11:49:17.500 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:49:17.500 | INFO | ------------------------------------------ | 20260506 11:49:17.500 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:49:17.500 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:49:17.500 | INFO | Total mem usage: 15.7 GiB | 20260506 11:49:17.500 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:49:17.500 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:49:17.500 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 11:49:17.500 | INFO | High Water Breakdown: | 20260506 11:49:17.500 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:49:17.500 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:49:17.500 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:49:17.500 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:49:17.500 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:49:17.500 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:49:17.500 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:49:17.500 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:49:17.500 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:49:17.500 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:49:17.500 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:49:17.500 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:49:17.500 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:49:17.500 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:49:17.500 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:49:17.500 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:49:17.500 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:49:17.500 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:49:17.500 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:49:17.500 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:49:17.500 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:49:17.500 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:49:17.500 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:49:17.500 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:49:17.500 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:49:17.500 | INFO | ------------------------------------------ | 20260506 11:49:17.500 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:49:17.500 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:49:17.500 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:49:17.500 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:49:17.500 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.57e+01, 1.37e+01 / 2.20e+00 | 20260506 11:49:17.500 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.00e-01, 1.05e-02 / 7.27e-03 | 20260506 11:49:17.500 | INFO | PerfReport: 11.55%, Baseline -- 3.68e+00 / 4.40e+00, 5.85e-01 / 3.81e-01 | 20260506 11:49:17.500 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.46e+00, 1.91e-01 / 1.60e-01 | 20260506 11:49:17.500 | INFO | PerfReport: 15.83%, DME -- 4.04e+01 / 4.46e+01, 4.00e+00 / 2.13e+00 | 20260506 11:49:17.500 | INFO | PerfReport: 56.23%, FrameLabeling -- 1.79e+01 / 1.99e+01, 1.42e+00 / 1.66e+00 | 20260506 11:49:17.501 | INFO | PerfReport: 7.67%, PulseAccumulating -- 2.45e+00 / 3.12e+00, 4.34e-01 / 2.24e-01 | 20260506 11:49:17.501 | INFO | PerfReport: 3.83%, Metrics -- 1.22e+00 / 1.60e+00, 3.06e-01 / 1.67e-01 | 20260506 11:49:17.501 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.32e-01 / 4.92e-01, 6.74e-02 / 3.16e-02 | 20260506 11:49:17.501 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:49:17.501 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:49:17.501 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:49:17.501 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260506 11:49:17.501 | INFO | PerfReport: 21.74%, Upload -- 2.15e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 11:49:17.501 | INFO | PerfReport: 21.14%, Download -- 2.09e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 11:49:17.501 | INFO | PerfReport: 32.23%, Compute -- 3.19e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 11:49:17.501 | INFO | PerfReport: 2.04%, LockUpload -- 2.01e+00 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 11:49:17.501 | INFO | PerfReport: 0.87%, LockDownload -- 8.56e-01 / 0.00e+00, 2.33e+00 / 0.00e+00 | 20260506 11:49:17.501 | INFO | PerfReport: 21.98%, LockCompute -- 2.17e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 11:49:17.501 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260506 11:49:17.626 | INFO | Starting a round of Baz IO | 20260506 11:49:20.801 | INFO | Populating RTMetrics report | 20260506 11:49:20.802 | INFO | RTMetrics report populated | 20260506 11:49:20.802 | INFO | Opening tmp RTMetrics json file | 20260506 11:49:20.907 | INFO | Serializing RTMetrics to json | 20260506 11:49:20.914 | INFO | Serialization done, swapping tmp file now... | 20260506 11:49:20.919 | INFO | RTMetrics done | 20260506 11:49:21.348 | INFO | Finished a round of Baz IO: Wrote 2.075 GiB in 3.721 seconds | 20260506 11:49:21.348 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.09% / 0.03% / 56.88% | 20260506 11:49:21.348 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.71581% / 4.76837e-05% / 6.75519e-05% | 20260506 11:49:21.348 | INFO | Baz IO overall: | 20260506 11:49:21.348 | INFO | BytesWritten : 546064941118 | 20260506 11:49:21.348 | INFO | NumEvents : 263677519111 | 20260506 11:49:22.638 | INFO | Analyzing chunk frames = [1122304,1122816) | 20260506 11:49:22.638 | INFO | MemInfo: { | 20260506 11:49:22.638 | INFO | "gpu" : " 17 GB", | 20260506 11:49:22.638 | INFO | "huge" : "", | 20260506 11:49:22.638 | INFO | "huge_shared" : " 27 GB", | 20260506 11:49:22.638 | INFO | "malloc" : " 14 GB", | 20260506 11:49:22.638 | INFO | "pinned" : " 29 GB" | 20260506 11:49:22.638 | INFO | } | 20260506 11:49:22.638 | INFO | Perf report for the last 5.13923 seconds: | 20260506 11:49:22.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:49:22.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:22.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:22.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:22.638 | INFO | ANALYSIS : 63.6%, 2.9, 95.8ms, 0 | 20260506 11:49:22.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:22.638 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260506 11:49:22.638 | INFO | PRE_HQ : 61.7%, 1.9, 61.2ms, 0 | 20260506 11:49:22.638 | INFO | BAZWRITER : 71.7%, 4.8, 176.3ms, 0 | 20260506 11:49:22.638 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.5ms, 0 | 20260506 11:49:22.639 | INFO | Uploaded 36GiB to GPU in 2.127s (16.9252GiB/s) | 20260506 11:49:22.639 | INFO | Downloaded 38GiB to GPU in 2.22799s (17.0557GiB/s) | 20260506 11:49:27.777 | INFO | Analyzing chunk frames = [1122816,1123328) | 20260506 11:49:27.777 | INFO | MemInfo: { | 20260506 11:49:27.777 | INFO | "gpu" : " 17 GB", | 20260506 11:49:27.777 | INFO | "huge" : "", | 20260506 11:49:27.777 | INFO | "huge_shared" : " 27 GB", | 20260506 11:49:27.777 | INFO | "malloc" : " 14 GB", | 20260506 11:49:27.777 | INFO | "pinned" : " 29 GB" | 20260506 11:49:27.777 | INFO | } | 20260506 11:49:27.777 | INFO | Perf report for the last 5.13929 seconds: | 20260506 11:49:27.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:49:27.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:27.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:27.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:27.777 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:49:27.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:27.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:27.778 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:49:27.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:27.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:27.778 | INFO | Uploaded 36GiB to GPU in 2.03123s (17.7232GiB/s) | 20260506 11:49:27.778 | INFO | Downloaded 34GiB to GPU in 1.92951s (17.6211GiB/s) | 20260506 11:49:32.917 | INFO | Analyzing chunk frames = [1123328,1123840) | 20260506 11:49:32.917 | INFO | MemInfo: { | 20260506 11:49:32.917 | INFO | "gpu" : " 17 GB", | 20260506 11:49:32.917 | INFO | "huge" : "", | 20260506 11:49:32.917 | INFO | "huge_shared" : " 27 GB", | 20260506 11:49:32.917 | INFO | "malloc" : " 14 GB", | 20260506 11:49:32.917 | INFO | "pinned" : " 29 GB" | 20260506 11:49:32.917 | INFO | } | 20260506 11:49:32.917 | INFO | Perf report for the last 5.13996 seconds: | 20260506 11:49:32.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:49:32.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:32.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:32.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:32.917 | INFO | ANALYSIS : 62.0%, 3.0, 96.0ms, 0 | 20260506 11:49:32.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:32.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:32.918 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:49:32.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:32.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:32.918 | INFO | Uploaded 36GiB to GPU in 2.02558s (17.7727GiB/s) | 20260506 11:49:32.918 | INFO | Downloaded 34GiB to GPU in 1.94927s (17.4424GiB/s) | 20260506 11:49:38.056 | INFO | Analyzing chunk frames = [1123840,1124352) | 20260506 11:49:38.056 | INFO | MemInfo: { | 20260506 11:49:38.056 | INFO | "gpu" : " 17 GB", | 20260506 11:49:38.056 | INFO | "huge" : "", | 20260506 11:49:38.056 | INFO | "huge_shared" : " 27 GB", | 20260506 11:49:38.056 | INFO | "malloc" : " 14 GB", | 20260506 11:49:38.056 | INFO | "pinned" : " 29 GB" | 20260506 11:49:38.056 | INFO | } | 20260506 11:49:38.056 | INFO | Perf report for the last 5.13902 seconds: | 20260506 11:49:38.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:49:38.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:38.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:38.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:38.056 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 11:49:38.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:38.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:38.057 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:49:38.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:38.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:38.057 | INFO | Uploaded 36GiB to GPU in 2.08092s (17.3001GiB/s) | 20260506 11:49:38.057 | INFO | Downloaded 34GiB to GPU in 1.97398s (17.2241GiB/s) | 20260506 11:49:43.196 | INFO | Analyzing chunk frames = [1124352,1124864) | 20260506 11:49:43.196 | INFO | MemInfo: { | 20260506 11:49:43.196 | INFO | "gpu" : " 17 GB", | 20260506 11:49:43.196 | INFO | "huge" : "", | 20260506 11:49:43.196 | INFO | "huge_shared" : " 27 GB", | 20260506 11:49:43.196 | INFO | "malloc" : " 14 GB", | 20260506 11:49:43.196 | INFO | "pinned" : " 29 GB" | 20260506 11:49:43.196 | INFO | } | 20260506 11:49:43.196 | INFO | Perf report for the last 5.1393 seconds: | 20260506 11:49:43.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:49:43.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:43.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:43.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:43.196 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 11:49:43.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:43.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:43.196 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:49:43.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:43.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:43.196 | INFO | Uploaded 36GiB to GPU in 2.08864s (17.2361GiB/s) | 20260506 11:49:43.196 | INFO | Downloaded 34GiB to GPU in 1.98054s (17.1671GiB/s) | 20260506 11:49:48.335 | INFO | Analyzing chunk frames = [1124864,1125376) | 20260506 11:49:48.335 | INFO | MemInfo: { | 20260506 11:49:48.335 | INFO | "gpu" : " 17 GB", | 20260506 11:49:48.335 | INFO | "huge" : "", | 20260506 11:49:48.335 | INFO | "huge_shared" : " 27 GB", | 20260506 11:49:48.335 | INFO | "malloc" : " 14 GB", | 20260506 11:49:48.335 | INFO | "pinned" : " 29 GB" | 20260506 11:49:48.335 | INFO | } | 20260506 11:49:48.335 | INFO | Perf report for the last 5.1395 seconds: | 20260506 11:49:48.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:49:48.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:48.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:48.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:48.335 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 11:49:48.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:48.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:48.335 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:49:48.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:48.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:48.336 | INFO | Uploaded 36GiB to GPU in 2.10449s (17.1063GiB/s) | 20260506 11:49:48.336 | INFO | Downloaded 34GiB to GPU in 1.99378s (17.0531GiB/s) | 20260506 11:49:53.475 | INFO | Analyzing chunk frames = [1125376,1125888) | 20260506 11:49:53.475 | INFO | MemInfo: { | 20260506 11:49:53.475 | INFO | "gpu" : " 17 GB", | 20260506 11:49:53.475 | INFO | "huge" : "", | 20260506 11:49:53.475 | INFO | "huge_shared" : " 27 GB", | 20260506 11:49:53.475 | INFO | "malloc" : " 14 GB", | 20260506 11:49:53.475 | INFO | "pinned" : " 29 GB" | 20260506 11:49:53.475 | INFO | } | 20260506 11:49:53.475 | INFO | Perf report for the last 5.13989 seconds: | 20260506 11:49:53.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:49:53.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:53.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:53.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:53.475 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260506 11:49:53.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:53.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:53.475 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 11:49:53.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:53.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:53.475 | INFO | Uploaded 36GiB to GPU in 2.04684s (17.5881GiB/s) | 20260506 11:49:53.476 | INFO | Downloaded 34GiB to GPU in 2.01031s (16.9128GiB/s) | 20260506 11:49:58.613 | INFO | Analyzing chunk frames = [1125888,1126400) | 20260506 11:49:58.613 | INFO | MemInfo: { | 20260506 11:49:58.613 | INFO | "gpu" : " 17 GB", | 20260506 11:49:58.613 | INFO | "huge" : "", | 20260506 11:49:58.613 | INFO | "huge_shared" : " 27 GB", | 20260506 11:49:58.613 | INFO | "malloc" : " 14 GB", | 20260506 11:49:58.613 | INFO | "pinned" : " 29 GB" | 20260506 11:49:58.613 | INFO | } | 20260506 11:49:58.614 | INFO | Perf report for the last 5.13836 seconds: | 20260506 11:49:58.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:49:58.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:58.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:58.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:58.614 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260506 11:49:58.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:58.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:58.614 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260506 11:49:58.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:58.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:49:58.614 | INFO | Uploaded 36GiB to GPU in 2.06091s (17.468GiB/s) | 20260506 11:49:58.614 | INFO | Downloaded 34GiB to GPU in 1.9598s (17.3487GiB/s) | 20260506 11:49:58.741 | INFO | Starting a round of Baz IO | 20260506 11:50:01.930 | INFO | Populating RTMetrics report | 20260506 11:50:01.930 | INFO | RTMetrics report populated | 20260506 11:50:01.930 | INFO | Opening tmp RTMetrics json file | 20260506 11:50:01.979 | INFO | Serializing RTMetrics to json | 20260506 11:50:02.013 | INFO | Serialization done, swapping tmp file now... | 20260506 11:50:02.018 | INFO | RTMetrics done | 20260506 11:50:02.171 | INFO | Finished a round of Baz IO: Wrote 2.076 GiB in 3.43 seconds | 20260506 11:50:02.171 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.06% / 0.03% / 56.91% | 20260506 11:50:02.171 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.70311% / 6.35783e-05% / 4.37101e-05% | 20260506 11:50:02.171 | INFO | Baz IO overall: | 20260506 11:50:02.171 | INFO | BytesWritten : 548293764901 | 20260506 11:50:02.171 | INFO | NumEvents : 264759160614 | 20260506 11:50:03.753 | INFO | Analyzing chunk frames = [1126400,1126912) | 20260506 11:50:03.753 | INFO | MemInfo: { | 20260506 11:50:03.753 | INFO | "gpu" : " 17 GB", | 20260506 11:50:03.753 | INFO | "huge" : "", | 20260506 11:50:03.753 | INFO | "huge_shared" : " 27 GB", | 20260506 11:50:03.753 | INFO | "malloc" : " 14 GB", | 20260506 11:50:03.753 | INFO | "pinned" : " 29 GB" | 20260506 11:50:03.753 | INFO | } | 20260506 11:50:03.753 | INFO | Perf report for the last 5.13954 seconds: | 20260506 11:50:03.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:50:03.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:03.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:03.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:03.753 | INFO | ANALYSIS : 63.9%, 2.9, 96.0ms, 0 | 20260506 11:50:03.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:03.753 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260506 11:50:03.753 | INFO | PRE_HQ : 62.2%, 1.9, 62.9ms, 0 | 20260506 11:50:03.753 | INFO | BAZWRITER : 66.0%, 4.8, 161.4ms, 0 | 20260506 11:50:03.753 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 11:50:03.753 | INFO | Uploaded 36GiB to GPU in 2.16317s (16.6422GiB/s) | 20260506 11:50:03.753 | INFO | Downloaded 38GiB to GPU in 2.2387s (16.9741GiB/s) | 20260506 11:50:08.892 | INFO | Analyzing chunk frames = [1126912,1127424) | 20260506 11:50:08.892 | INFO | MemInfo: { | 20260506 11:50:08.892 | INFO | "gpu" : " 17 GB", | 20260506 11:50:08.892 | INFO | "huge" : "", | 20260506 11:50:08.892 | INFO | "huge_shared" : " 27 GB", | 20260506 11:50:08.892 | INFO | "malloc" : " 14 GB", | 20260506 11:50:08.893 | INFO | "pinned" : " 29 GB" | 20260506 11:50:08.893 | INFO | } | 20260506 11:50:08.893 | INFO | Perf report for the last 5.13945 seconds: | 20260506 11:50:08.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:50:08.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:08.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:08.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:08.893 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260506 11:50:08.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:08.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:08.893 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 11:50:08.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:08.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:08.893 | INFO | Uploaded 36GiB to GPU in 2.0444s (17.6091GiB/s) | 20260506 11:50:08.893 | INFO | Downloaded 34GiB to GPU in 1.9336s (17.5838GiB/s) | 20260506 11:50:14.032 | INFO | Analyzing chunk frames = [1127424,1127936) | 20260506 11:50:14.032 | INFO | MemInfo: { | 20260506 11:50:14.032 | INFO | "gpu" : " 17 GB", | 20260506 11:50:14.032 | INFO | "huge" : "", | 20260506 11:50:14.032 | INFO | "huge_shared" : " 27 GB", | 20260506 11:50:14.032 | INFO | "malloc" : " 14 GB", | 20260506 11:50:14.032 | INFO | "pinned" : " 29 GB" | 20260506 11:50:14.032 | INFO | } | 20260506 11:50:14.032 | INFO | Perf report for the last 5.1396 seconds: | 20260506 11:50:14.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:50:14.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:14.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:14.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:14.032 | INFO | ANALYSIS : 62.3%, 3.0, 96.5ms, 0 | 20260506 11:50:14.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:14.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:14.032 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 11:50:14.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:14.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:14.032 | INFO | Uploaded 36GiB to GPU in 2.05692s (17.5019GiB/s) | 20260506 11:50:14.033 | INFO | Downloaded 34GiB to GPU in 1.95058s (17.4307GiB/s) | 20260506 11:50:19.171 | INFO | Analyzing chunk frames = [1127936,1128448) | 20260506 11:50:19.171 | INFO | MemInfo: { | 20260506 11:50:19.171 | INFO | "gpu" : " 17 GB", | 20260506 11:50:19.171 | INFO | "huge" : "", | 20260506 11:50:19.171 | INFO | "huge_shared" : " 27 GB", | 20260506 11:50:19.171 | INFO | "malloc" : " 14 GB", | 20260506 11:50:19.171 | INFO | "pinned" : " 29 GB" | 20260506 11:50:19.171 | INFO | } | 20260506 11:50:19.172 | INFO | Perf report for the last 5.13939 seconds: | 20260506 11:50:19.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:50:19.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:19.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:19.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:19.172 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260506 11:50:19.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:19.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:19.172 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:50:19.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:19.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:19.172 | INFO | Uploaded 36GiB to GPU in 2.07319s (17.3645GiB/s) | 20260506 11:50:19.172 | INFO | Downloaded 34GiB to GPU in 1.96006s (17.3464GiB/s) | 20260506 11:50:24.310 | INFO | Analyzing chunk frames = [1128448,1128960) | 20260506 11:50:24.310 | INFO | MemInfo: { | 20260506 11:50:24.310 | INFO | "gpu" : " 17 GB", | 20260506 11:50:24.310 | INFO | "huge" : "", | 20260506 11:50:24.310 | INFO | "huge_shared" : " 27 GB", | 20260506 11:50:24.310 | INFO | "malloc" : " 14 GB", | 20260506 11:50:24.310 | INFO | "pinned" : " 29 GB" | 20260506 11:50:24.310 | INFO | } | 20260506 11:50:24.310 | INFO | Perf report for the last 5.13869 seconds: | 20260506 11:50:24.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:50:24.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:24.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:24.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:24.310 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 11:50:24.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:24.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:24.310 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:50:24.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:24.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:24.311 | INFO | Uploaded 36GiB to GPU in 2.04016s (17.6456GiB/s) | 20260506 11:50:24.311 | INFO | Downloaded 34GiB to GPU in 1.97555s (17.2104GiB/s) | 20260506 11:50:29.450 | INFO | Analyzing chunk frames = [1128960,1129472) | 20260506 11:50:29.450 | INFO | MemInfo: { | 20260506 11:50:29.450 | INFO | "gpu" : " 17 GB", | 20260506 11:50:29.450 | INFO | "huge" : "", | 20260506 11:50:29.450 | INFO | "huge_shared" : " 27 GB", | 20260506 11:50:29.450 | INFO | "malloc" : " 14 GB", | 20260506 11:50:29.450 | INFO | "pinned" : " 29 GB" | 20260506 11:50:29.450 | INFO | } | 20260506 11:50:29.450 | INFO | Perf report for the last 5.13969 seconds: | 20260506 11:50:29.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:50:29.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:29.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:29.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:29.450 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 11:50:29.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:29.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:29.450 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 11:50:29.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:29.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:29.450 | INFO | Uploaded 36GiB to GPU in 2.09386s (17.1931GiB/s) | 20260506 11:50:29.450 | INFO | Downloaded 34GiB to GPU in 1.98029s (17.1692GiB/s) | 20260506 11:50:34.590 | INFO | Analyzing chunk frames = [1129472,1129984) | 20260506 11:50:34.590 | INFO | MemInfo: { | 20260506 11:50:34.590 | INFO | "gpu" : " 17 GB", | 20260506 11:50:34.590 | INFO | "huge" : "", | 20260506 11:50:34.590 | INFO | "huge_shared" : " 27 GB", | 20260506 11:50:34.590 | INFO | "malloc" : " 14 GB", | 20260506 11:50:34.590 | INFO | "pinned" : " 29 GB" | 20260506 11:50:34.590 | INFO | } | 20260506 11:50:34.590 | INFO | Perf report for the last 5.13984 seconds: | 20260506 11:50:34.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:50:34.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:34.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:34.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:34.590 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:50:34.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:34.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:34.590 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 11:50:34.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:34.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:34.590 | INFO | Uploaded 36GiB to GPU in 1.96228s (18.346GiB/s) | 20260506 11:50:34.590 | INFO | Downloaded 34GiB to GPU in 1.9997s (17.0026GiB/s) | 20260506 11:50:39.728 | INFO | Analyzing chunk frames = [1129984,1130496) | 20260506 11:50:39.728 | INFO | MemInfo: { | 20260506 11:50:39.728 | INFO | "gpu" : " 17 GB", | 20260506 11:50:39.728 | INFO | "huge" : "", | 20260506 11:50:39.728 | INFO | "huge_shared" : " 27 GB", | 20260506 11:50:39.728 | INFO | "malloc" : " 14 GB", | 20260506 11:50:39.728 | INFO | "pinned" : " 29 GB" | 20260506 11:50:39.728 | INFO | } | 20260506 11:50:39.728 | INFO | Perf report for the last 5.13859 seconds: | 20260506 11:50:39.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:50:39.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:39.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:39.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:39.729 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 11:50:39.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:39.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:39.729 | INFO | PRE_HQ : 32.3%, 1.0, 16.7ms, 0 | 20260506 11:50:39.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:39.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:39.729 | INFO | Uploaded 36GiB to GPU in 1.98508s (18.1353GiB/s) | 20260506 11:50:39.729 | INFO | Downloaded 34GiB to GPU in 1.96466s (17.3058GiB/s) | 20260506 11:50:39.729 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:50:39.729 | INFO | Total mem usage: 16.8 GiB | 20260506 11:50:39.729 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:50:39.729 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -9.48%) | 20260506 11:50:39.729 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:50:39.729 | INFO | High Water Breakdown: | 20260506 11:50:39.729 | INFO | 60.5% : 9.29 GiB ( 89.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:50:39.729 | INFO | 39.4% : 6.05 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:50:39.729 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:50:39.729 | INFO | ------------------------------------------ | 20260506 11:50:39.729 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:50:39.729 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:50:39.729 | INFO | Total mem usage: 26.5 GiB | 20260506 11:50:39.729 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:50:39.729 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:50:39.729 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:50:39.729 | INFO | High Water Breakdown: | 20260506 11:50:39.729 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:50:39.729 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:50:39.729 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:50:39.729 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:50:39.729 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:50:39.729 | INFO | 0.559% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:50:39.729 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:50:39.729 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:50:39.729 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:50:39.729 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:50:39.729 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:50:39.729 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:50:39.729 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:50:39.729 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:50:39.729 | INFO | ------------------------------------------ | 20260506 11:50:39.729 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:50:39.729 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:50:39.729 | INFO | Total mem usage: 25.6 GiB | 20260506 11:50:39.729 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:50:39.729 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:50:39.729 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:50:39.729 | INFO | High Water Breakdown: | 20260506 11:50:39.729 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:50:39.729 | INFO | ------------------------------------------ | 20260506 11:50:39.729 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:50:39.729 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:50:39.729 | INFO | Total mem usage: 15.7 GiB | 20260506 11:50:39.729 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:50:39.729 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:50:39.729 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 11:50:39.729 | INFO | High Water Breakdown: | 20260506 11:50:39.729 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:50:39.729 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:50:39.729 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:50:39.729 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:50:39.729 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:50:39.729 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:50:39.729 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:50:39.729 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:50:39.729 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:50:39.729 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:50:39.729 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:50:39.729 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:50:39.729 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:50:39.729 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:50:39.729 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:50:39.729 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:50:39.730 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:50:39.730 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:50:39.730 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:50:39.730 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:50:39.730 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:50:39.730 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:50:39.730 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:50:39.730 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:50:39.730 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:50:39.730 | INFO | ------------------------------------------ | 20260506 11:50:39.730 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:50:39.730 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:50:39.730 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:50:39.730 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:50:39.730 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.61e+01, 1.37e+01 / 2.11e+00 | 20260506 11:50:39.730 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.00e-01, 1.04e-02 / 5.55e-03 | 20260506 11:50:39.730 | INFO | PerfReport: 11.54%, Baseline -- 3.68e+00 / 4.48e+00, 5.80e-01 / 3.20e-01 | 20260506 11:50:39.730 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.47e+00, 1.91e-01 / 1.49e-01 | 20260506 11:50:39.730 | INFO | PerfReport: 15.89%, DME -- 4.05e+01 / 4.52e+01, 3.99e+00 / 2.07e+00 | 20260506 11:50:39.730 | INFO | PerfReport: 56.14%, FrameLabeling -- 1.79e+01 / 1.96e+01, 1.35e+00 / 1.48e+00 | 20260506 11:50:39.730 | INFO | PerfReport: 7.70%, PulseAccumulating -- 2.46e+00 / 3.13e+00, 4.40e-01 / 1.85e-01 | 20260506 11:50:39.730 | INFO | PerfReport: 3.84%, Metrics -- 1.22e+00 / 1.62e+00, 3.07e-01 / 1.56e-01 | 20260506 11:50:39.730 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.31e-01 / 4.88e-01, 6.69e-02 / 2.97e-02 | 20260506 11:50:39.730 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:50:39.730 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:50:39.730 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:50:39.730 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 11:50:39.730 | INFO | PerfReport: 21.73%, Upload -- 2.15e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 11:50:39.730 | INFO | PerfReport: 21.10%, Download -- 2.09e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260506 11:50:39.730 | INFO | PerfReport: 32.25%, Compute -- 3.19e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 11:50:39.730 | INFO | PerfReport: 1.95%, LockUpload -- 1.93e+00 / 0.00e+00, 7.65e+00 / 0.00e+00 | 20260506 11:50:39.730 | INFO | PerfReport: 0.88%, LockDownload -- 8.68e-01 / 0.00e+00, 2.29e+00 / 0.00e+00 | 20260506 11:50:39.730 | INFO | PerfReport: 22.07%, LockCompute -- 2.18e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260506 11:50:39.730 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.53e-02 / 0.00e+00 | 20260506 11:50:39.856 | INFO | Starting a round of Baz IO | 20260506 11:50:43.062 | INFO | Populating RTMetrics report | 20260506 11:50:43.063 | INFO | RTMetrics report populated | 20260506 11:50:43.063 | INFO | Opening tmp RTMetrics json file | 20260506 11:50:43.105 | INFO | Serializing RTMetrics to json | 20260506 11:50:43.111 | INFO | Serialization done, swapping tmp file now... | 20260506 11:50:43.115 | INFO | RTMetrics done | 20260506 11:50:43.216 | INFO | Finished a round of Baz IO: Wrote 2.078 GiB in 3.36 seconds | 20260506 11:50:43.216 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.03% / 0.03% / 56.94% | 20260506 11:50:43.216 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.70013% / 3.17891e-05% / 5.5631e-05% | 20260506 11:50:43.216 | INFO | Baz IO overall: | 20260506 11:50:43.216 | INFO | BytesWritten : 550525045873 | 20260506 11:50:43.216 | INFO | NumEvents : 265842857823 | 20260506 11:50:44.867 | INFO | Analyzing chunk frames = [1130496,1131008) | 20260506 11:50:44.867 | INFO | MemInfo: { | 20260506 11:50:44.867 | INFO | "gpu" : " 17 GB", | 20260506 11:50:44.867 | INFO | "huge" : "", | 20260506 11:50:44.867 | INFO | "huge_shared" : " 27 GB", | 20260506 11:50:44.867 | INFO | "malloc" : " 14 GB", | 20260506 11:50:44.867 | INFO | "pinned" : " 29 GB" | 20260506 11:50:44.867 | INFO | } | 20260506 11:50:44.868 | INFO | Perf report for the last 5.13909 seconds: | 20260506 11:50:44.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:50:44.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:44.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:44.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:44.868 | INFO | ANALYSIS : 64.0%, 2.8, 95.4ms, 0 | 20260506 11:50:44.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:44.868 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260506 11:50:44.868 | INFO | PRE_HQ : 62.0%, 1.9, 62.6ms, 0 | 20260506 11:50:44.868 | INFO | BAZWRITER : 64.8%, 4.8, 159.7ms, 0 | 20260506 11:50:44.868 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 11:50:44.868 | INFO | Uploaded 36GiB to GPU in 2.09171s (17.2108GiB/s) | 20260506 11:50:44.868 | INFO | Downloaded 38GiB to GPU in 2.21355s (17.167GiB/s) | 20260506 11:50:50.007 | INFO | Analyzing chunk frames = [1131008,1131520) | 20260506 11:50:50.007 | INFO | MemInfo: { | 20260506 11:50:50.007 | INFO | "gpu" : " 17 GB", | 20260506 11:50:50.007 | INFO | "huge" : "", | 20260506 11:50:50.007 | INFO | "huge_shared" : " 27 GB", | 20260506 11:50:50.007 | INFO | "malloc" : " 14 GB", | 20260506 11:50:50.007 | INFO | "pinned" : " 29 GB" | 20260506 11:50:50.007 | INFO | } | 20260506 11:50:50.007 | INFO | Perf report for the last 5.13943 seconds: | 20260506 11:50:50.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:50:50.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:50.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:50.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:50.007 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:50:50.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:50.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:50.007 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:50:50.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:50.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:50.007 | INFO | Uploaded 36GiB to GPU in 2.01683s (17.8498GiB/s) | 20260506 11:50:50.007 | INFO | Downloaded 34GiB to GPU in 1.91741s (17.7323GiB/s) | 20260506 11:50:55.146 | INFO | Analyzing chunk frames = [1131520,1132032) | 20260506 11:50:55.146 | INFO | MemInfo: { | 20260506 11:50:55.146 | INFO | "gpu" : " 17 GB", | 20260506 11:50:55.146 | INFO | "huge" : "", | 20260506 11:50:55.146 | INFO | "huge_shared" : " 27 GB", | 20260506 11:50:55.146 | INFO | "malloc" : " 14 GB", | 20260506 11:50:55.146 | INFO | "pinned" : " 29 GB" | 20260506 11:50:55.146 | INFO | } | 20260506 11:50:55.146 | INFO | Perf report for the last 5.13939 seconds: | 20260506 11:50:55.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:50:55.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:55.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:55.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:55.146 | INFO | ANALYSIS : 62.1%, 3.0, 96.1ms, 0 | 20260506 11:50:55.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:55.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:55.146 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:50:55.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:55.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:50:55.147 | INFO | Uploaded 36GiB to GPU in 2.04726s (17.5845GiB/s) | 20260506 11:50:55.147 | INFO | Downloaded 34GiB to GPU in 1.93799s (17.5439GiB/s) | 20260506 11:51:00.285 | INFO | Analyzing chunk frames = [1132032,1132544) | 20260506 11:51:00.285 | INFO | MemInfo: { | 20260506 11:51:00.285 | INFO | "gpu" : " 17 GB", | 20260506 11:51:00.285 | INFO | "huge" : "", | 20260506 11:51:00.285 | INFO | "huge_shared" : " 27 GB", | 20260506 11:51:00.285 | INFO | "malloc" : " 14 GB", | 20260506 11:51:00.286 | INFO | "pinned" : " 29 GB" | 20260506 11:51:00.286 | INFO | } | 20260506 11:51:00.286 | INFO | Perf report for the last 5.13927 seconds: | 20260506 11:51:00.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:51:00.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:00.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:00.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:00.286 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 11:51:00.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:00.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:00.286 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:51:00.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:00.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:00.286 | INFO | Uploaded 36GiB to GPU in 2.08031s (17.3051GiB/s) | 20260506 11:51:00.286 | INFO | Downloaded 34GiB to GPU in 1.96145s (17.3341GiB/s) | 20260506 11:51:05.425 | INFO | Analyzing chunk frames = [1132544,1133056) | 20260506 11:51:05.425 | INFO | MemInfo: { | 20260506 11:51:05.425 | INFO | "gpu" : " 17 GB", | 20260506 11:51:05.425 | INFO | "huge" : "", | 20260506 11:51:05.425 | INFO | "huge_shared" : " 27 GB", | 20260506 11:51:05.425 | INFO | "malloc" : " 14 GB", | 20260506 11:51:05.425 | INFO | "pinned" : " 29 GB" | 20260506 11:51:05.425 | INFO | } | 20260506 11:51:05.425 | INFO | Perf report for the last 5.13943 seconds: | 20260506 11:51:05.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:51:05.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:05.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:05.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:05.425 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 11:51:05.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:05.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:05.425 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:51:05.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:05.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:05.425 | INFO | Uploaded 36GiB to GPU in 2.10251s (17.1224GiB/s) | 20260506 11:51:05.425 | INFO | Downloaded 34GiB to GPU in 1.9884s (17.0992GiB/s) | 20260506 11:51:10.565 | INFO | Analyzing chunk frames = [1133056,1133568) | 20260506 11:51:10.565 | INFO | MemInfo: { | 20260506 11:51:10.565 | INFO | "gpu" : " 17 GB", | 20260506 11:51:10.565 | INFO | "huge" : "", | 20260506 11:51:10.565 | INFO | "huge_shared" : " 27 GB", | 20260506 11:51:10.565 | INFO | "malloc" : " 14 GB", | 20260506 11:51:10.565 | INFO | "pinned" : " 29 GB" | 20260506 11:51:10.565 | INFO | } | 20260506 11:51:10.565 | INFO | Perf report for the last 5.13964 seconds: | 20260506 11:51:10.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:51:10.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:10.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:10.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:10.565 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 11:51:10.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:10.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:10.565 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:51:10.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:10.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:10.565 | INFO | Uploaded 36GiB to GPU in 1.95213s (18.4414GiB/s) | 20260506 11:51:10.565 | INFO | Downloaded 34GiB to GPU in 1.97522s (17.2133GiB/s) | 20260506 11:51:15.704 | INFO | Analyzing chunk frames = [1133568,1134080) | 20260506 11:51:15.704 | INFO | MemInfo: { | 20260506 11:51:15.704 | INFO | "gpu" : " 17 GB", | 20260506 11:51:15.704 | INFO | "huge" : "", | 20260506 11:51:15.704 | INFO | "huge_shared" : " 27 GB", | 20260506 11:51:15.704 | INFO | "malloc" : " 14 GB", | 20260506 11:51:15.704 | INFO | "pinned" : " 29 GB" | 20260506 11:51:15.704 | INFO | } | 20260506 11:51:15.704 | INFO | Perf report for the last 5.13964 seconds: | 20260506 11:51:15.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:51:15.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:15.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:15.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:15.704 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260506 11:51:15.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:15.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:15.705 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 11:51:15.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:15.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:15.705 | INFO | Uploaded 36GiB to GPU in 1.98958s (18.0943GiB/s) | 20260506 11:51:15.705 | INFO | Downloaded 34GiB to GPU in 2.02826s (16.7631GiB/s) | 20260506 11:51:20.843 | INFO | Analyzing chunk frames = [1134080,1134592) | 20260506 11:51:20.843 | INFO | MemInfo: { | 20260506 11:51:20.843 | INFO | "gpu" : " 17 GB", | 20260506 11:51:20.843 | INFO | "huge" : "", | 20260506 11:51:20.843 | INFO | "huge_shared" : " 27 GB", | 20260506 11:51:20.843 | INFO | "malloc" : " 14 GB", | 20260506 11:51:20.843 | INFO | "pinned" : " 29 GB" | 20260506 11:51:20.843 | INFO | } | 20260506 11:51:20.843 | INFO | Perf report for the last 5.13872 seconds: | 20260506 11:51:20.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:51:20.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:20.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:20.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:20.843 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260506 11:51:20.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:20.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:20.843 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260506 11:51:20.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:20.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:20.843 | INFO | Uploaded 36GiB to GPU in 2.019s (17.8306GiB/s) | 20260506 11:51:20.843 | INFO | Downloaded 34GiB to GPU in 1.9652s (17.301GiB/s) | 20260506 11:51:20.968 | INFO | Starting a round of Baz IO | 20260506 11:51:24.154 | INFO | Populating RTMetrics report | 20260506 11:51:24.154 | INFO | RTMetrics report populated | 20260506 11:51:24.154 | INFO | Opening tmp RTMetrics json file | 20260506 11:51:24.175 | INFO | Serializing RTMetrics to json | 20260506 11:51:24.258 | INFO | Serialization done, swapping tmp file now... | 20260506 11:51:24.261 | INFO | RTMetrics done | 20260506 11:51:24.507 | INFO | Finished a round of Baz IO: Wrote 2.075 GiB in 3.539 seconds | 20260506 11:51:24.507 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 43.00% / 0.03% / 56.97% | 20260506 11:51:24.507 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.66911% / 7.15256e-05% / 5.5631e-05% | 20260506 11:51:24.507 | INFO | Baz IO overall: | 20260506 11:51:24.507 | INFO | BytesWritten : 552753311709 | 20260506 11:51:24.507 | INFO | NumEvents : 266924053705 | 20260506 11:51:25.982 | INFO | Analyzing chunk frames = [1134592,1135104) | 20260506 11:51:25.982 | INFO | MemInfo: { | 20260506 11:51:25.982 | INFO | "gpu" : " 17 GB", | 20260506 11:51:25.982 | INFO | "huge" : "", | 20260506 11:51:25.982 | INFO | "huge_shared" : " 27 GB", | 20260506 11:51:25.982 | INFO | "malloc" : " 14 GB", | 20260506 11:51:25.982 | INFO | "pinned" : " 29 GB" | 20260506 11:51:25.983 | INFO | } | 20260506 11:51:25.983 | INFO | Perf report for the last 5.13943 seconds: | 20260506 11:51:25.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:51:25.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:25.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:25.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:25.983 | INFO | ANALYSIS : 63.7%, 2.9, 95.7ms, 0 | 20260506 11:51:25.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:25.983 | INFO | RT_METRICS : 47.4%, 1.0, 24.8ms, 0 | 20260506 11:51:25.983 | INFO | PRE_HQ : 62.2%, 2.0, 63.2ms, 0 | 20260506 11:51:25.983 | INFO | BAZWRITER : 68.1%, 4.8, 167.4ms, 0 | 20260506 11:51:25.983 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 11:51:25.983 | INFO | Uploaded 36GiB to GPU in 2.12793s (16.9179GiB/s) | 20260506 11:51:25.983 | INFO | Downloaded 38GiB to GPU in 2.2175s (17.1364GiB/s) | 20260506 11:51:31.122 | INFO | Analyzing chunk frames = [1135104,1135616) | 20260506 11:51:31.122 | INFO | MemInfo: { | 20260506 11:51:31.122 | INFO | "gpu" : " 17 GB", | 20260506 11:51:31.122 | INFO | "huge" : "", | 20260506 11:51:31.122 | INFO | "huge_shared" : " 27 GB", | 20260506 11:51:31.122 | INFO | "malloc" : " 14 GB", | 20260506 11:51:31.122 | INFO | "pinned" : " 29 GB" | 20260506 11:51:31.122 | INFO | } | 20260506 11:51:31.122 | INFO | Perf report for the last 5.13926 seconds: | 20260506 11:51:31.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:51:31.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:31.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:31.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:31.122 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:51:31.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:31.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:31.122 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:51:31.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:31.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:31.122 | INFO | Uploaded 36GiB to GPU in 2.01089s (17.9025GiB/s) | 20260506 11:51:31.122 | INFO | Downloaded 34GiB to GPU in 1.92848s (17.6304GiB/s) | 20260506 11:51:36.261 | INFO | Analyzing chunk frames = [1135616,1136128) | 20260506 11:51:36.261 | INFO | MemInfo: { | 20260506 11:51:36.261 | INFO | "gpu" : " 17 GB", | 20260506 11:51:36.261 | INFO | "huge" : "", | 20260506 11:51:36.261 | INFO | "huge_shared" : " 27 GB", | 20260506 11:51:36.261 | INFO | "malloc" : " 14 GB", | 20260506 11:51:36.261 | INFO | "pinned" : " 29 GB" | 20260506 11:51:36.261 | INFO | } | 20260506 11:51:36.261 | INFO | Perf report for the last 5.13935 seconds: | 20260506 11:51:36.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:51:36.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:36.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:36.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:36.261 | INFO | ANALYSIS : 62.0%, 3.0, 96.0ms, 0 | 20260506 11:51:36.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:36.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:36.261 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:51:36.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:36.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:36.262 | INFO | Uploaded 36GiB to GPU in 2.04324s (17.6191GiB/s) | 20260506 11:51:36.262 | INFO | Downloaded 34GiB to GPU in 1.93746s (17.5487GiB/s) | 20260506 11:51:41.400 | INFO | Analyzing chunk frames = [1136128,1136640) | 20260506 11:51:41.400 | INFO | MemInfo: { | 20260506 11:51:41.400 | INFO | "gpu" : " 17 GB", | 20260506 11:51:41.400 | INFO | "huge" : "", | 20260506 11:51:41.400 | INFO | "huge_shared" : " 27 GB", | 20260506 11:51:41.400 | INFO | "malloc" : " 14 GB", | 20260506 11:51:41.400 | INFO | "pinned" : " 29 GB" | 20260506 11:51:41.400 | INFO | } | 20260506 11:51:41.400 | INFO | Perf report for the last 5.13923 seconds: | 20260506 11:51:41.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:51:41.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:41.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:41.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:41.401 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 11:51:41.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:41.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:41.401 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:51:41.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:41.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:41.401 | INFO | Uploaded 36GiB to GPU in 2.07965s (17.3106GiB/s) | 20260506 11:51:41.401 | INFO | Downloaded 34GiB to GPU in 1.96461s (17.3062GiB/s) | 20260506 11:51:46.539 | INFO | Analyzing chunk frames = [1136640,1137152) | 20260506 11:51:46.539 | INFO | MemInfo: { | 20260506 11:51:46.539 | INFO | "gpu" : " 17 GB", | 20260506 11:51:46.540 | INFO | "huge" : "", | 20260506 11:51:46.540 | INFO | "huge_shared" : " 27 GB", | 20260506 11:51:46.540 | INFO | "malloc" : " 14 GB", | 20260506 11:51:46.540 | INFO | "pinned" : " 29 GB" | 20260506 11:51:46.540 | INFO | } | 20260506 11:51:46.540 | INFO | Perf report for the last 5.13924 seconds: | 20260506 11:51:46.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:51:46.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:46.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:46.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:46.540 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 11:51:46.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:46.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:46.540 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 11:51:46.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:46.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:46.540 | INFO | Uploaded 36GiB to GPU in 2.09802s (17.1591GiB/s) | 20260506 11:51:46.540 | INFO | Downloaded 34GiB to GPU in 1.98164s (17.1575GiB/s) | 20260506 11:51:51.679 | INFO | Analyzing chunk frames = [1137152,1137664) | 20260506 11:51:51.679 | INFO | MemInfo: { | 20260506 11:51:51.679 | INFO | "gpu" : " 17 GB", | 20260506 11:51:51.679 | INFO | "huge" : "", | 20260506 11:51:51.679 | INFO | "huge_shared" : " 27 GB", | 20260506 11:51:51.679 | INFO | "malloc" : " 14 GB", | 20260506 11:51:51.679 | INFO | "pinned" : " 29 GB" | 20260506 11:51:51.679 | INFO | } | 20260506 11:51:51.679 | INFO | Perf report for the last 5.13929 seconds: | 20260506 11:51:51.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:51:51.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:51.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:51.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:51.679 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 11:51:51.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:51.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:51.679 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 11:51:51.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:51.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:51.679 | INFO | Uploaded 36GiB to GPU in 2.07117s (17.3815GiB/s) | 20260506 11:51:51.679 | INFO | Downloaded 34GiB to GPU in 2.00219s (16.9814GiB/s) | 20260506 11:51:56.819 | INFO | Analyzing chunk frames = [1137664,1138176) | 20260506 11:51:56.819 | INFO | MemInfo: { | 20260506 11:51:56.819 | INFO | "gpu" : " 17 GB", | 20260506 11:51:56.819 | INFO | "huge" : "", | 20260506 11:51:56.819 | INFO | "huge_shared" : " 27 GB", | 20260506 11:51:56.819 | INFO | "malloc" : " 14 GB", | 20260506 11:51:56.819 | INFO | "pinned" : " 29 GB" | 20260506 11:51:56.819 | INFO | } | 20260506 11:51:56.819 | INFO | Perf report for the last 5.13989 seconds: | 20260506 11:51:56.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:51:56.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:56.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:56.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:56.819 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:51:56.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:56.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:56.819 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 11:51:56.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:56.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:51:56.819 | INFO | Uploaded 36GiB to GPU in 2.03371s (17.7017GiB/s) | 20260506 11:51:56.819 | INFO | Downloaded 34GiB to GPU in 1.97773s (17.1914GiB/s) | 20260506 11:52:01.958 | INFO | Analyzing chunk frames = [1138176,1138688) | 20260506 11:52:01.958 | INFO | MemInfo: { | 20260506 11:52:01.958 | INFO | "gpu" : " 17 GB", | 20260506 11:52:01.958 | INFO | "huge" : "", | 20260506 11:52:01.958 | INFO | "huge_shared" : " 27 GB", | 20260506 11:52:01.958 | INFO | "malloc" : " 14 GB", | 20260506 11:52:01.958 | INFO | "pinned" : " 29 GB" | 20260506 11:52:01.958 | INFO | } | 20260506 11:52:01.958 | INFO | Perf report for the last 5.13916 seconds: | 20260506 11:52:01.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:52:01.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:01.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:01.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:01.958 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 11:52:01.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:01.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:01.958 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:52:01.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:01.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:01.958 | INFO | Uploaded 36GiB to GPU in 2.02873s (17.7451GiB/s) | 20260506 11:52:01.958 | INFO | Downloaded 34GiB to GPU in 1.95929s (17.3532GiB/s) | 20260506 11:52:01.958 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:52:01.958 | INFO | Total mem usage: 16.8 GiB | 20260506 11:52:01.958 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:52:01.958 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -6.36%) | 20260506 11:52:01.958 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:52:01.958 | INFO | High Water Breakdown: | 20260506 11:52:01.958 | INFO | 61% : 9.63 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:52:01.958 | INFO | 38.9% : 6.15 GiB ( 91.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:52:01.958 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:52:01.958 | INFO | ------------------------------------------ | 20260506 11:52:01.958 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:52:01.958 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:52:01.958 | INFO | Total mem usage: 26.5 GiB | 20260506 11:52:01.958 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.41%) | 20260506 11:52:01.958 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 11:52:01.958 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 11:52:01.959 | INFO | High Water Breakdown: | 20260506 11:52:01.959 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:52:01.959 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:52:01.959 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:52:01.959 | INFO | 2% : 523 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:52:01.959 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:52:01.959 | INFO | 0.557% : 146 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:52:01.959 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:52:01.959 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:52:01.959 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:52:01.959 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:52:01.959 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:52:01.959 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:52:01.959 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:52:01.959 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:52:01.959 | INFO | ------------------------------------------ | 20260506 11:52:01.959 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:52:01.959 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:52:01.959 | INFO | Total mem usage: 25.6 GiB | 20260506 11:52:01.959 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:52:01.959 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:52:01.959 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:52:01.959 | INFO | High Water Breakdown: | 20260506 11:52:01.959 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:52:01.959 | INFO | ------------------------------------------ | 20260506 11:52:01.959 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:52:01.959 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:52:01.959 | INFO | Total mem usage: 15.7 GiB | 20260506 11:52:01.959 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:52:01.959 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:52:01.959 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:52:01.959 | INFO | High Water Breakdown: | 20260506 11:52:01.959 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:52:01.959 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:52:01.959 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:52:01.959 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:52:01.959 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:52:01.959 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:52:01.959 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:52:01.959 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:52:01.959 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:52:01.959 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:52:01.959 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:52:01.959 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:52:01.959 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:52:01.959 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:52:01.959 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:52:01.959 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:52:01.959 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:52:01.959 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:52:01.959 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:52:01.959 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:52:01.959 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:52:01.959 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:52:01.959 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:52:01.959 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:52:01.959 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:52:01.959 | INFO | ------------------------------------------ | 20260506 11:52:01.959 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:52:01.959 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:52:01.959 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:52:01.959 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:52:01.959 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.65e+01, 1.36e+01 / 2.47e+00 | 20260506 11:52:01.959 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 9.93e-02, 1.01e-02 / 7.58e-03 | 20260506 11:52:01.959 | INFO | PerfReport: 11.56%, Baseline -- 3.69e+00 / 4.45e+00, 5.92e-01 / 3.64e-01 | 20260506 11:52:01.959 | INFO | PerfReport: 3.23%, Histogram -- 1.03e+00 / 1.44e+00, 1.91e-01 / 1.56e-01 | 20260506 11:52:01.959 | INFO | PerfReport: 15.87%, DME -- 4.05e+01 / 4.53e+01, 4.13e+00 / 2.52e+00 | 20260506 11:52:01.959 | INFO | PerfReport: 56.23%, FrameLabeling -- 1.79e+01 / 2.00e+01, 1.45e+00 / 1.58e+00 | 20260506 11:52:01.959 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.44e+00 / 3.07e+00, 4.27e-01 / 2.79e-01 | 20260506 11:52:01.959 | INFO | PerfReport: 3.82%, Metrics -- 1.22e+00 / 1.63e+00, 3.09e-01 / 1.68e-01 | 20260506 11:52:01.959 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.31e-01 / 4.92e-01, 6.79e-02 / 2.95e-02 | 20260506 11:52:01.959 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:52:01.959 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:52:01.959 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:52:01.959 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 11:52:01.959 | INFO | PerfReport: 21.62%, Upload -- 2.13e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 11:52:01.959 | INFO | PerfReport: 21.07%, Download -- 2.08e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 11:52:01.959 | INFO | PerfReport: 32.32%, Compute -- 3.19e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 11:52:01.959 | INFO | PerfReport: 1.97%, LockUpload -- 1.94e+00 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 11:52:01.959 | INFO | PerfReport: 0.87%, LockDownload -- 8.60e-01 / 0.00e+00, 2.38e+00 / 0.00e+00 | 20260506 11:52:01.959 | INFO | PerfReport: 22.13%, LockCompute -- 2.18e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 11:52:01.959 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260506 11:52:02.080 | INFO | Starting a round of Baz IO | 20260506 11:52:05.267 | INFO | Populating RTMetrics report | 20260506 11:52:05.268 | INFO | RTMetrics report populated | 20260506 11:52:05.268 | INFO | Opening tmp RTMetrics json file | 20260506 11:52:05.382 | INFO | Serializing RTMetrics to json | 20260506 11:52:05.411 | INFO | Serialization done, swapping tmp file now... | 20260506 11:52:05.416 | INFO | RTMetrics done | 20260506 11:52:05.591 | INFO | Finished a round of Baz IO: Wrote 2.079 GiB in 3.511 seconds | 20260506 11:52:05.591 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.97% / 0.03% / 57.00% | 20260506 11:52:05.591 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.67197% / 3.97364e-05% / 5.5631e-05% | 20260506 11:52:05.591 | INFO | Baz IO overall: | 20260506 11:52:05.591 | INFO | BytesWritten : 554985549621 | 20260506 11:52:05.591 | INFO | NumEvents : 268008168766 | 20260506 11:52:07.097 | INFO | Analyzing chunk frames = [1138688,1139200) | 20260506 11:52:07.097 | INFO | MemInfo: { | 20260506 11:52:07.097 | INFO | "gpu" : " 17 GB", | 20260506 11:52:07.097 | INFO | "huge" : "", | 20260506 11:52:07.097 | INFO | "huge_shared" : " 27 GB", | 20260506 11:52:07.097 | INFO | "malloc" : " 14 GB", | 20260506 11:52:07.097 | INFO | "pinned" : " 29 GB" | 20260506 11:52:07.097 | INFO | } | 20260506 11:52:07.097 | INFO | Perf report for the last 5.13892 seconds: | 20260506 11:52:07.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:52:07.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:07.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:07.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:07.097 | INFO | ANALYSIS : 63.8%, 2.9, 96.4ms, 0 | 20260506 11:52:07.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:07.097 | INFO | RT_METRICS : 47.2%, 1.0, 24.7ms, 0 | 20260506 11:52:07.097 | INFO | PRE_HQ : 61.8%, 1.9, 60.9ms, 0 | 20260506 11:52:07.097 | INFO | BAZWRITER : 67.5%, 4.8, 167.0ms, 0 | 20260506 11:52:07.097 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.3ms, 0 | 20260506 11:52:07.097 | INFO | Uploaded 36GiB to GPU in 2.15003s (16.7439GiB/s) | 20260506 11:52:07.097 | INFO | Downloaded 38GiB to GPU in 2.23029s (17.0381GiB/s) | 20260506 11:52:12.236 | INFO | Analyzing chunk frames = [1139200,1139712) | 20260506 11:52:12.236 | INFO | MemInfo: { | 20260506 11:52:12.236 | INFO | "gpu" : " 17 GB", | 20260506 11:52:12.236 | INFO | "huge" : "", | 20260506 11:52:12.236 | INFO | "huge_shared" : " 27 GB", | 20260506 11:52:12.236 | INFO | "malloc" : " 14 GB", | 20260506 11:52:12.236 | INFO | "pinned" : " 29 GB" | 20260506 11:52:12.236 | INFO | } | 20260506 11:52:12.236 | INFO | Perf report for the last 5.13949 seconds: | 20260506 11:52:12.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:52:12.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:12.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:12.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:12.237 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 11:52:12.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:12.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:12.237 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:52:12.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:12.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:12.237 | INFO | Uploaded 36GiB to GPU in 2.02629s (17.7664GiB/s) | 20260506 11:52:12.237 | INFO | Downloaded 34GiB to GPU in 1.93217s (17.5968GiB/s) | 20260506 11:52:17.376 | INFO | Analyzing chunk frames = [1139712,1140224) | 20260506 11:52:17.376 | INFO | MemInfo: { | 20260506 11:52:17.376 | INFO | "gpu" : " 17 GB", | 20260506 11:52:17.376 | INFO | "huge" : "", | 20260506 11:52:17.376 | INFO | "huge_shared" : " 27 GB", | 20260506 11:52:17.376 | INFO | "malloc" : " 14 GB", | 20260506 11:52:17.376 | INFO | "pinned" : " 29 GB" | 20260506 11:52:17.376 | INFO | } | 20260506 11:52:17.376 | INFO | Perf report for the last 5.13951 seconds: | 20260506 11:52:17.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:52:17.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:17.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:17.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:17.376 | INFO | ANALYSIS : 62.0%, 3.0, 96.0ms, 0 | 20260506 11:52:17.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:17.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:17.376 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:52:17.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:17.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:17.377 | INFO | Uploaded 36GiB to GPU in 2.04333s (17.6183GiB/s) | 20260506 11:52:17.377 | INFO | Downloaded 34GiB to GPU in 1.93507s (17.5704GiB/s) | 20260506 11:52:22.515 | INFO | Analyzing chunk frames = [1140224,1140736) | 20260506 11:52:22.515 | INFO | MemInfo: { | 20260506 11:52:22.515 | INFO | "gpu" : " 17 GB", | 20260506 11:52:22.515 | INFO | "huge" : "", | 20260506 11:52:22.515 | INFO | "huge_shared" : " 27 GB", | 20260506 11:52:22.515 | INFO | "malloc" : " 14 GB", | 20260506 11:52:22.515 | INFO | "pinned" : " 29 GB" | 20260506 11:52:22.515 | INFO | } | 20260506 11:52:22.515 | INFO | Perf report for the last 5.13925 seconds: | 20260506 11:52:22.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:52:22.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:22.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:22.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:22.515 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260506 11:52:22.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:22.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:22.515 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:52:22.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:22.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:22.516 | INFO | Uploaded 36GiB to GPU in 2.08347s (17.2788GiB/s) | 20260506 11:52:22.516 | INFO | Downloaded 34GiB to GPU in 1.95392s (17.4009GiB/s) | 20260506 11:52:27.654 | INFO | Analyzing chunk frames = [1140736,1141248) | 20260506 11:52:27.654 | INFO | MemInfo: { | 20260506 11:52:27.654 | INFO | "gpu" : " 17 GB", | 20260506 11:52:27.654 | INFO | "huge" : "", | 20260506 11:52:27.654 | INFO | "huge_shared" : " 27 GB", | 20260506 11:52:27.654 | INFO | "malloc" : " 14 GB", | 20260506 11:52:27.654 | INFO | "pinned" : " 29 GB" | 20260506 11:52:27.654 | INFO | } | 20260506 11:52:27.654 | INFO | Perf report for the last 5.13901 seconds: | 20260506 11:52:27.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:52:27.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:27.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:27.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:27.654 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 11:52:27.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:27.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:27.655 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:52:27.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:27.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:27.655 | INFO | Uploaded 36GiB to GPU in 2.10856s (17.0732GiB/s) | 20260506 11:52:27.655 | INFO | Downloaded 34GiB to GPU in 1.98984s (17.0868GiB/s) | 20260506 11:52:32.794 | INFO | Analyzing chunk frames = [1141248,1141760) | 20260506 11:52:32.794 | INFO | MemInfo: { | 20260506 11:52:32.794 | INFO | "gpu" : " 17 GB", | 20260506 11:52:32.794 | INFO | "huge" : "", | 20260506 11:52:32.794 | INFO | "huge_shared" : " 27 GB", | 20260506 11:52:32.794 | INFO | "malloc" : " 14 GB", | 20260506 11:52:32.794 | INFO | "pinned" : " 29 GB" | 20260506 11:52:32.794 | INFO | } | 20260506 11:52:32.794 | INFO | Perf report for the last 5.13986 seconds: | 20260506 11:52:32.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:52:32.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:32.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:32.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:32.794 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 11:52:32.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:32.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:32.794 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:52:32.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:32.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:32.794 | INFO | Uploaded 36GiB to GPU in 2.06718s (17.4151GiB/s) | 20260506 11:52:32.794 | INFO | Downloaded 34GiB to GPU in 1.99618s (17.0325GiB/s) | 20260506 11:52:37.933 | INFO | Analyzing chunk frames = [1141760,1142272) | 20260506 11:52:37.933 | INFO | MemInfo: { | 20260506 11:52:37.933 | INFO | "gpu" : " 17 GB", | 20260506 11:52:37.933 | INFO | "huge" : "", | 20260506 11:52:37.933 | INFO | "huge_shared" : " 27 GB", | 20260506 11:52:37.933 | INFO | "malloc" : " 14 GB", | 20260506 11:52:37.933 | INFO | "pinned" : " 29 GB" | 20260506 11:52:37.933 | INFO | } | 20260506 11:52:37.934 | INFO | Perf report for the last 5.13936 seconds: | 20260506 11:52:37.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:52:37.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:37.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:37.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:37.934 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:52:37.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:37.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:37.934 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 11:52:37.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:37.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:37.934 | INFO | Uploaded 36GiB to GPU in 2.02629s (17.7664GiB/s) | 20260506 11:52:37.934 | INFO | Downloaded 34GiB to GPU in 2.00819s (16.9307GiB/s) | 20260506 11:52:43.072 | INFO | Analyzing chunk frames = [1142272,1142784) | 20260506 11:52:43.072 | INFO | MemInfo: { | 20260506 11:52:43.072 | INFO | "gpu" : " 17 GB", | 20260506 11:52:43.072 | INFO | "huge" : "", | 20260506 11:52:43.072 | INFO | "huge_shared" : " 27 GB", | 20260506 11:52:43.072 | INFO | "malloc" : " 14 GB", | 20260506 11:52:43.072 | INFO | "pinned" : " 29 GB" | 20260506 11:52:43.073 | INFO | } | 20260506 11:52:43.073 | INFO | Perf report for the last 5.13901 seconds: | 20260506 11:52:43.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:52:43.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:43.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:43.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:43.073 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260506 11:52:43.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:43.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:43.073 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260506 11:52:43.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:43.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:43.073 | INFO | Uploaded 36GiB to GPU in 2.01329s (17.8812GiB/s) | 20260506 11:52:43.073 | INFO | Downloaded 34GiB to GPU in 1.96533s (17.2999GiB/s) | 20260506 11:52:43.201 | INFO | Starting a round of Baz IO | 20260506 11:52:46.455 | INFO | Populating RTMetrics report | 20260506 11:52:46.455 | INFO | RTMetrics report populated | 20260506 11:52:46.455 | INFO | Opening tmp RTMetrics json file | 20260506 11:52:46.480 | INFO | Serializing RTMetrics to json | 20260506 11:52:46.487 | INFO | Serialization done, swapping tmp file now... | 20260506 11:52:46.490 | INFO | RTMetrics done | 20260506 11:52:46.745 | INFO | Finished a round of Baz IO: Wrote 2.08 GiB in 3.543 seconds | 20260506 11:52:46.745 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.94% / 0.03% / 57.03% | 20260506 11:52:46.745 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.66332% / 4.76837e-05% / 2.78155e-05% | 20260506 11:52:46.745 | INFO | Baz IO overall: | 20260506 11:52:46.745 | INFO | BytesWritten : 557218808697 | 20260506 11:52:46.745 | INFO | NumEvents : 269092738423 | 20260506 11:52:48.212 | INFO | Analyzing chunk frames = [1142784,1143296) | 20260506 11:52:48.212 | INFO | MemInfo: { | 20260506 11:52:48.212 | INFO | "gpu" : " 17 GB", | 20260506 11:52:48.212 | INFO | "huge" : "", | 20260506 11:52:48.212 | INFO | "huge_shared" : " 27 GB", | 20260506 11:52:48.212 | INFO | "malloc" : " 14 GB", | 20260506 11:52:48.212 | INFO | "pinned" : " 29 GB" | 20260506 11:52:48.212 | INFO | } | 20260506 11:52:48.212 | INFO | Perf report for the last 5.13922 seconds: | 20260506 11:52:48.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:52:48.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:48.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:48.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:48.212 | INFO | ANALYSIS : 64.8%, 2.8, 96.1ms, 0 | 20260506 11:52:48.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:48.212 | INFO | RT_METRICS : 47.6%, 1.0, 25.0ms, 0 | 20260506 11:52:48.212 | INFO | PRE_HQ : 63.2%, 2.0, 64.3ms, 0 | 20260506 11:52:48.212 | INFO | BAZWRITER : 68.2%, 4.8, 168.1ms, 0 | 20260506 11:52:48.212 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 11:52:48.212 | INFO | Uploaded 36GiB to GPU in 2.1096s (17.0649GiB/s) | 20260506 11:52:48.212 | INFO | Downloaded 38GiB to GPU in 2.22643s (17.0677GiB/s) | 20260506 11:52:53.351 | INFO | Analyzing chunk frames = [1143296,1143808) | 20260506 11:52:53.351 | INFO | MemInfo: { | 20260506 11:52:53.351 | INFO | "gpu" : " 17 GB", | 20260506 11:52:53.351 | INFO | "huge" : "", | 20260506 11:52:53.351 | INFO | "huge_shared" : " 27 GB", | 20260506 11:52:53.352 | INFO | "malloc" : " 14 GB", | 20260506 11:52:53.352 | INFO | "pinned" : " 29 GB" | 20260506 11:52:53.352 | INFO | } | 20260506 11:52:53.352 | INFO | Perf report for the last 5.13979 seconds: | 20260506 11:52:53.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:52:53.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:53.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:53.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:53.352 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:52:53.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:53.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:53.352 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260506 11:52:53.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:53.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:53.352 | INFO | Uploaded 36GiB to GPU in 2.04012s (17.6461GiB/s) | 20260506 11:52:53.352 | INFO | Downloaded 34GiB to GPU in 1.93894s (17.5354GiB/s) | 20260506 11:52:58.491 | INFO | Analyzing chunk frames = [1143808,1144320) | 20260506 11:52:58.491 | INFO | MemInfo: { | 20260506 11:52:58.491 | INFO | "gpu" : " 17 GB", | 20260506 11:52:58.491 | INFO | "huge" : "", | 20260506 11:52:58.491 | INFO | "huge_shared" : " 27 GB", | 20260506 11:52:58.491 | INFO | "malloc" : " 14 GB", | 20260506 11:52:58.491 | INFO | "pinned" : " 29 GB" | 20260506 11:52:58.491 | INFO | } | 20260506 11:52:58.491 | INFO | Perf report for the last 5.13936 seconds: | 20260506 11:52:58.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:52:58.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:58.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:58.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:58.491 | INFO | ANALYSIS : 62.0%, 3.0, 96.0ms, 0 | 20260506 11:52:58.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:58.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:58.491 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:52:58.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:58.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:52:58.491 | INFO | Uploaded 36GiB to GPU in 2.04342s (17.6176GiB/s) | 20260506 11:52:58.491 | INFO | Downloaded 34GiB to GPU in 1.93573s (17.5644GiB/s) | 20260506 11:53:03.630 | INFO | Analyzing chunk frames = [1144320,1144832) | 20260506 11:53:03.630 | INFO | MemInfo: { | 20260506 11:53:03.630 | INFO | "gpu" : " 17 GB", | 20260506 11:53:03.630 | INFO | "huge" : "", | 20260506 11:53:03.630 | INFO | "huge_shared" : " 27 GB", | 20260506 11:53:03.630 | INFO | "malloc" : " 14 GB", | 20260506 11:53:03.630 | INFO | "pinned" : " 29 GB" | 20260506 11:53:03.630 | INFO | } | 20260506 11:53:03.630 | INFO | Perf report for the last 5.13932 seconds: | 20260506 11:53:03.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:53:03.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:03.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:03.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:03.630 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 11:53:03.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:03.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:03.630 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:53:03.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:03.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:03.631 | INFO | Uploaded 36GiB to GPU in 2.0451s (17.6031GiB/s) | 20260506 11:53:03.631 | INFO | Downloaded 34GiB to GPU in 1.96777s (17.2785GiB/s) | 20260506 11:53:08.769 | INFO | Analyzing chunk frames = [1144832,1145344) | 20260506 11:53:08.769 | INFO | MemInfo: { | 20260506 11:53:08.769 | INFO | "gpu" : " 17 GB", | 20260506 11:53:08.769 | INFO | "huge" : "", | 20260506 11:53:08.769 | INFO | "huge_shared" : " 27 GB", | 20260506 11:53:08.769 | INFO | "malloc" : " 14 GB", | 20260506 11:53:08.769 | INFO | "pinned" : " 29 GB" | 20260506 11:53:08.769 | INFO | } | 20260506 11:53:08.769 | INFO | Perf report for the last 5.13916 seconds: | 20260506 11:53:08.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:53:08.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:08.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:08.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:08.770 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 11:53:08.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:08.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:08.770 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:53:08.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:08.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:08.770 | INFO | Uploaded 36GiB to GPU in 2.0921s (17.2076GiB/s) | 20260506 11:53:08.770 | INFO | Downloaded 34GiB to GPU in 1.97865s (17.1834GiB/s) | 20260506 11:53:13.909 | INFO | Analyzing chunk frames = [1145344,1145856) | 20260506 11:53:13.909 | INFO | MemInfo: { | 20260506 11:53:13.909 | INFO | "gpu" : " 17 GB", | 20260506 11:53:13.909 | INFO | "huge" : "", | 20260506 11:53:13.909 | INFO | "huge_shared" : " 27 GB", | 20260506 11:53:13.909 | INFO | "malloc" : " 14 GB", | 20260506 11:53:13.909 | INFO | "pinned" : " 29 GB" | 20260506 11:53:13.909 | INFO | } | 20260506 11:53:13.909 | INFO | Perf report for the last 5.13946 seconds: | 20260506 11:53:13.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:53:13.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:13.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:13.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:13.909 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 11:53:13.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:13.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:13.909 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:53:13.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:13.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:13.909 | INFO | Uploaded 36GiB to GPU in 2.09255s (17.2039GiB/s) | 20260506 11:53:13.909 | INFO | Downloaded 34GiB to GPU in 1.99977s (17.002GiB/s) | 20260506 11:53:19.049 | INFO | Analyzing chunk frames = [1145856,1146368) | 20260506 11:53:19.049 | INFO | MemInfo: { | 20260506 11:53:19.049 | INFO | "gpu" : " 17 GB", | 20260506 11:53:19.049 | INFO | "huge" : "", | 20260506 11:53:19.049 | INFO | "huge_shared" : " 27 GB", | 20260506 11:53:19.049 | INFO | "malloc" : " 14 GB", | 20260506 11:53:19.049 | INFO | "pinned" : " 29 GB" | 20260506 11:53:19.049 | INFO | } | 20260506 11:53:19.049 | INFO | Perf report for the last 5.1402 seconds: | 20260506 11:53:19.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:53:19.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:19.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:19.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:19.049 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260506 11:53:19.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:19.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:19.049 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:53:19.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:19.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:19.049 | INFO | Uploaded 36GiB to GPU in 2.04646s (17.5914GiB/s) | 20260506 11:53:19.049 | INFO | Downloaded 34GiB to GPU in 2.00365s (16.969GiB/s) | 20260506 11:53:24.187 | INFO | Analyzing chunk frames = [1146368,1146880) | 20260506 11:53:24.187 | INFO | MemInfo: { | 20260506 11:53:24.187 | INFO | "gpu" : " 17 GB", | 20260506 11:53:24.187 | INFO | "huge" : "", | 20260506 11:53:24.188 | INFO | "huge_shared" : " 27 GB", | 20260506 11:53:24.188 | INFO | "malloc" : " 14 GB", | 20260506 11:53:24.188 | INFO | "pinned" : " 29 GB" | 20260506 11:53:24.188 | INFO | } | 20260506 11:53:24.188 | INFO | Perf report for the last 5.13849 seconds: | 20260506 11:53:24.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:53:24.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:24.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:24.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:24.188 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:53:24.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:24.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:24.188 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260506 11:53:24.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:24.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:24.188 | INFO | Uploaded 36GiB to GPU in 2.01834s (17.8364GiB/s) | 20260506 11:53:24.188 | INFO | Downloaded 34GiB to GPU in 2.027s (16.7736GiB/s) | 20260506 11:53:24.188 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:53:24.188 | INFO | Total mem usage: 16.8 GiB | 20260506 11:53:24.188 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:53:24.188 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -8.63%) | 20260506 11:53:24.188 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:53:24.188 | INFO | High Water Breakdown: | 20260506 11:53:24.188 | INFO | 60.6% : 9.38 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:53:24.188 | INFO | 39.2% : 6.07 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:53:24.188 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:53:24.188 | INFO | ------------------------------------------ | 20260506 11:53:24.188 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:53:24.188 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:53:24.189 | INFO | Total mem usage: 26.5 GiB | 20260506 11:53:24.189 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 11:53:24.189 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 11:53:24.189 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 11:53:24.189 | INFO | High Water Breakdown: | 20260506 11:53:24.189 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:53:24.189 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:53:24.189 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:53:24.189 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:53:24.189 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:53:24.189 | INFO | 0.698% : 182 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:53:24.189 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:53:24.189 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:53:24.189 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:53:24.189 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:53:24.189 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:53:24.189 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:53:24.189 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:53:24.189 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:53:24.189 | INFO | ------------------------------------------ | 20260506 11:53:24.189 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:53:24.189 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:53:24.189 | INFO | Total mem usage: 25.6 GiB | 20260506 11:53:24.189 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:53:24.189 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:53:24.189 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:53:24.189 | INFO | High Water Breakdown: | 20260506 11:53:24.189 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:53:24.189 | INFO | ------------------------------------------ | 20260506 11:53:24.189 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:53:24.189 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:53:24.189 | INFO | Total mem usage: 15.7 GiB | 20260506 11:53:24.189 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:53:24.189 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:53:24.189 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:53:24.189 | INFO | High Water Breakdown: | 20260506 11:53:24.189 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:53:24.189 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:53:24.189 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:53:24.189 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:53:24.189 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:53:24.189 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:53:24.189 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:53:24.189 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:53:24.189 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:53:24.189 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:53:24.189 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:53:24.189 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:53:24.189 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:53:24.189 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:53:24.189 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:53:24.189 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:53:24.189 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:53:24.189 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:53:24.189 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:53:24.189 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:53:24.189 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:53:24.189 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:53:24.189 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:53:24.189 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:53:24.189 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:53:24.189 | INFO | ------------------------------------------ | 20260506 11:53:24.189 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:53:24.189 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:53:24.189 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:53:24.189 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:53:24.189 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.63e+01, 1.35e+01 / 3.32e+00 | 20260506 11:53:24.189 | INFO | PerfReport: 0.35%, AggregateStats -- 1.12e-01 / 5.92e-01, 7.48e-01 / 3.25e+00 | 20260506 11:53:24.189 | INFO | PerfReport: 11.54%, Baseline -- 3.68e+00 / 4.44e+00, 5.83e-01 / 3.56e-01 | 20260506 11:53:24.189 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.45e+00, 1.93e-01 / 1.61e-01 | 20260506 11:53:24.189 | INFO | PerfReport: 15.82%, DME -- 4.04e+01 / 4.47e+01, 4.14e+00 / 2.70e+00 | 20260506 11:53:24.189 | INFO | PerfReport: 56.21%, FrameLabeling -- 1.80e+01 / 1.99e+01, 1.47e+00 / 1.65e+00 | 20260506 11:53:24.189 | INFO | PerfReport: 7.66%, PulseAccumulating -- 2.45e+00 / 3.08e+00, 4.37e-01 / 2.51e-01 | 20260506 11:53:24.189 | INFO | PerfReport: 3.82%, Metrics -- 1.22e+00 / 1.62e+00, 3.13e-01 / 1.89e-01 | 20260506 11:53:24.189 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.31e-01 / 4.90e-01, 6.69e-02 / 3.29e-02 | 20260506 11:53:24.189 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:53:24.189 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:53:24.189 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:53:24.189 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.48e+01 / 0.00e+00 | 20260506 11:53:24.189 | INFO | PerfReport: 21.73%, Upload -- 2.15e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 11:53:24.189 | INFO | PerfReport: 21.13%, Download -- 2.09e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 11:53:24.189 | INFO | PerfReport: 32.30%, Compute -- 3.19e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 11:53:24.189 | INFO | PerfReport: 2.02%, LockUpload -- 2.00e+00 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 11:53:24.189 | INFO | PerfReport: 0.82%, LockDownload -- 8.11e-01 / 0.00e+00, 2.23e+00 / 0.00e+00 | 20260506 11:53:24.189 | INFO | PerfReport: 21.99%, LockCompute -- 2.17e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 11:53:24.189 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260506 11:53:24.313 | INFO | Starting a round of Baz IO | 20260506 11:53:27.511 | INFO | Populating RTMetrics report | 20260506 11:53:27.512 | INFO | RTMetrics report populated | 20260506 11:53:27.512 | INFO | Opening tmp RTMetrics json file | 20260506 11:53:27.524 | INFO | Serializing RTMetrics to json | 20260506 11:53:27.529 | INFO | Serialization done, swapping tmp file now... | 20260506 11:53:27.534 | INFO | RTMetrics done | 20260506 11:53:27.784 | INFO | Finished a round of Baz IO: Wrote 2.077 GiB in 3.471 seconds | 20260506 11:53:27.784 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.91% / 0.03% / 57.06% | 20260506 11:53:27.784 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.62893% / 9.93411e-05% / 5.96046e-05% | 20260506 11:53:27.784 | INFO | Baz IO overall: | 20260506 11:53:27.784 | INFO | BytesWritten : 559449094547 | 20260506 11:53:27.784 | INFO | NumEvents : 270174706299 | 20260506 11:53:29.327 | INFO | Analyzing chunk frames = [1146880,1147392) | 20260506 11:53:29.327 | INFO | MemInfo: { | 20260506 11:53:29.327 | INFO | "gpu" : " 17 GB", | 20260506 11:53:29.327 | INFO | "huge" : "", | 20260506 11:53:29.327 | INFO | "huge_shared" : " 27 GB", | 20260506 11:53:29.327 | INFO | "malloc" : " 14 GB", | 20260506 11:53:29.327 | INFO | "pinned" : " 29 GB" | 20260506 11:53:29.327 | INFO | } | 20260506 11:53:29.327 | INFO | Perf report for the last 5.1395 seconds: | 20260506 11:53:29.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:53:29.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:29.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:29.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:29.327 | INFO | ANALYSIS : 63.8%, 2.9, 96.1ms, 0 | 20260506 11:53:29.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:29.327 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 11:53:29.327 | INFO | PRE_HQ : 61.6%, 1.9, 61.8ms, 0 | 20260506 11:53:29.327 | INFO | BAZWRITER : 66.8%, 4.8, 164.6ms, 0 | 20260506 11:53:29.327 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 11:53:29.328 | INFO | Uploaded 36GiB to GPU in 2.0791s (17.3152GiB/s) | 20260506 11:53:29.328 | INFO | Downloaded 38GiB to GPU in 2.22037s (17.1142GiB/s) | 20260506 11:53:34.466 | INFO | Analyzing chunk frames = [1147392,1147904) | 20260506 11:53:34.466 | INFO | MemInfo: { | 20260506 11:53:34.466 | INFO | "gpu" : " 17 GB", | 20260506 11:53:34.466 | INFO | "huge" : "", | 20260506 11:53:34.466 | INFO | "huge_shared" : " 27 GB", | 20260506 11:53:34.466 | INFO | "malloc" : " 14 GB", | 20260506 11:53:34.466 | INFO | "pinned" : " 29 GB" | 20260506 11:53:34.466 | INFO | } | 20260506 11:53:34.466 | INFO | Perf report for the last 5.13887 seconds: | 20260506 11:53:34.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:53:34.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:34.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:34.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:34.466 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:53:34.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:34.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:34.466 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:53:34.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:34.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:34.466 | INFO | Uploaded 36GiB to GPU in 2.05137s (17.5493GiB/s) | 20260506 11:53:34.466 | INFO | Downloaded 34GiB to GPU in 1.93407s (17.5795GiB/s) | 20260506 11:53:39.605 | INFO | Analyzing chunk frames = [1147904,1148416) | 20260506 11:53:39.605 | INFO | MemInfo: { | 20260506 11:53:39.605 | INFO | "gpu" : " 17 GB", | 20260506 11:53:39.605 | INFO | "huge" : "", | 20260506 11:53:39.605 | INFO | "huge_shared" : " 27 GB", | 20260506 11:53:39.605 | INFO | "malloc" : " 14 GB", | 20260506 11:53:39.605 | INFO | "pinned" : " 29 GB" | 20260506 11:53:39.605 | INFO | } | 20260506 11:53:39.606 | INFO | Perf report for the last 5.13945 seconds: | 20260506 11:53:39.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:53:39.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:39.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:39.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:39.606 | INFO | ANALYSIS : 62.3%, 3.0, 96.5ms, 0 | 20260506 11:53:39.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:39.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:39.606 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:53:39.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:39.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:39.606 | INFO | Uploaded 36GiB to GPU in 2.0475s (17.5825GiB/s) | 20260506 11:53:39.606 | INFO | Downloaded 34GiB to GPU in 1.96805s (17.2759GiB/s) | 20260506 11:53:44.745 | INFO | Analyzing chunk frames = [1148416,1148928) | 20260506 11:53:44.745 | INFO | MemInfo: { | 20260506 11:53:44.745 | INFO | "gpu" : " 17 GB", | 20260506 11:53:44.745 | INFO | "huge" : "", | 20260506 11:53:44.745 | INFO | "huge_shared" : " 27 GB", | 20260506 11:53:44.745 | INFO | "malloc" : " 14 GB", | 20260506 11:53:44.745 | INFO | "pinned" : " 29 GB" | 20260506 11:53:44.745 | INFO | } | 20260506 11:53:44.745 | INFO | Perf report for the last 5.13941 seconds: | 20260506 11:53:44.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:53:44.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:44.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:44.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:44.745 | INFO | ANALYSIS : 61.4%, 3.0, 95.3ms, 0 | 20260506 11:53:44.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:44.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:44.745 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:53:44.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:44.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:44.745 | INFO | Uploaded 36GiB to GPU in 2.02255s (17.7993GiB/s) | 20260506 11:53:44.745 | INFO | Downloaded 34GiB to GPU in 1.95265s (17.4122GiB/s) | 20260506 11:53:49.884 | INFO | Analyzing chunk frames = [1148928,1149440) | 20260506 11:53:49.884 | INFO | MemInfo: { | 20260506 11:53:49.884 | INFO | "gpu" : " 17 GB", | 20260506 11:53:49.884 | INFO | "huge" : "", | 20260506 11:53:49.884 | INFO | "huge_shared" : " 27 GB", | 20260506 11:53:49.884 | INFO | "malloc" : " 14 GB", | 20260506 11:53:49.884 | INFO | "pinned" : " 29 GB" | 20260506 11:53:49.884 | INFO | } | 20260506 11:53:49.884 | INFO | Perf report for the last 5.13915 seconds: | 20260506 11:53:49.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:53:49.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:49.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:49.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:49.884 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 11:53:49.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:49.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:49.884 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:53:49.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:49.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:49.884 | INFO | Uploaded 36GiB to GPU in 2.02572s (17.7715GiB/s) | 20260506 11:53:49.884 | INFO | Downloaded 34GiB to GPU in 1.96115s (17.3367GiB/s) | 20260506 11:53:55.024 | INFO | Analyzing chunk frames = [1149440,1149952) | 20260506 11:53:55.024 | INFO | MemInfo: { | 20260506 11:53:55.024 | INFO | "gpu" : " 17 GB", | 20260506 11:53:55.024 | INFO | "huge" : "", | 20260506 11:53:55.024 | INFO | "huge_shared" : " 27 GB", | 20260506 11:53:55.024 | INFO | "malloc" : " 14 GB", | 20260506 11:53:55.024 | INFO | "pinned" : " 29 GB" | 20260506 11:53:55.024 | INFO | } | 20260506 11:53:55.024 | INFO | Perf report for the last 5.13981 seconds: | 20260506 11:53:55.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:53:55.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:55.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:55.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:55.024 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 11:53:55.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:55.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:55.024 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:53:55.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:55.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:53:55.024 | INFO | Uploaded 36GiB to GPU in 2.07574s (17.3432GiB/s) | 20260506 11:53:55.024 | INFO | Downloaded 34GiB to GPU in 1.98241s (17.1508GiB/s) | 20260506 11:54:00.163 | INFO | Analyzing chunk frames = [1149952,1150464) | 20260506 11:54:00.163 | INFO | MemInfo: { | 20260506 11:54:00.163 | INFO | "gpu" : " 17 GB", | 20260506 11:54:00.163 | INFO | "huge" : "", | 20260506 11:54:00.163 | INFO | "huge_shared" : " 27 GB", | 20260506 11:54:00.163 | INFO | "malloc" : " 14 GB", | 20260506 11:54:00.163 | INFO | "pinned" : " 29 GB" | 20260506 11:54:00.164 | INFO | } | 20260506 11:54:00.164 | INFO | Perf report for the last 5.13975 seconds: | 20260506 11:54:00.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:54:00.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:00.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:00.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:00.164 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:54:00.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:00.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:00.164 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 11:54:00.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:00.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:00.164 | INFO | Uploaded 36GiB to GPU in 1.98249s (18.159GiB/s) | 20260506 11:54:00.164 | INFO | Downloaded 34GiB to GPU in 2.0033s (16.972GiB/s) | 20260506 11:54:05.302 | INFO | Analyzing chunk frames = [1150464,1150976) | 20260506 11:54:05.302 | INFO | MemInfo: { | 20260506 11:54:05.302 | INFO | "gpu" : " 17 GB", | 20260506 11:54:05.302 | INFO | "huge" : "", | 20260506 11:54:05.302 | INFO | "huge_shared" : " 27 GB", | 20260506 11:54:05.302 | INFO | "malloc" : " 14 GB", | 20260506 11:54:05.302 | INFO | "pinned" : " 29 GB" | 20260506 11:54:05.302 | INFO | } | 20260506 11:54:05.303 | INFO | Perf report for the last 5.1388 seconds: | 20260506 11:54:05.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:54:05.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:05.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:05.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:05.303 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260506 11:54:05.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:05.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:05.303 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260506 11:54:05.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:05.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:05.303 | INFO | Uploaded 36GiB to GPU in 2.01426s (17.8726GiB/s) | 20260506 11:54:05.303 | INFO | Downloaded 34GiB to GPU in 1.98495s (17.1289GiB/s) | 20260506 11:54:05.430 | INFO | Starting a round of Baz IO | 20260506 11:54:08.615 | INFO | Populating RTMetrics report | 20260506 11:54:08.615 | INFO | RTMetrics report populated | 20260506 11:54:08.615 | INFO | Opening tmp RTMetrics json file | 20260506 11:54:08.627 | INFO | Serializing RTMetrics to json | 20260506 11:54:08.661 | INFO | Serialization done, swapping tmp file now... | 20260506 11:54:08.701 | INFO | RTMetrics done | 20260506 11:54:08.786 | INFO | Finished a round of Baz IO: Wrote 2.08 GiB in 3.355 seconds | 20260506 11:54:08.786 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.88% / 0.03% / 57.09% | 20260506 11:54:08.786 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.62167% / 4.37101e-05% / 4.76837e-05% | 20260506 11:54:08.786 | INFO | Baz IO overall: | 20260506 11:54:08.786 | INFO | BytesWritten : 561682026286 | 20260506 11:54:08.786 | INFO | NumEvents : 271258646006 | 20260506 11:54:10.441 | INFO | Analyzing chunk frames = [1150976,1151488) | 20260506 11:54:10.441 | INFO | MemInfo: { | 20260506 11:54:10.441 | INFO | "gpu" : " 17 GB", | 20260506 11:54:10.441 | INFO | "huge" : "", | 20260506 11:54:10.442 | INFO | "huge_shared" : " 27 GB", | 20260506 11:54:10.442 | INFO | "malloc" : " 14 GB", | 20260506 11:54:10.442 | INFO | "pinned" : " 29 GB" | 20260506 11:54:10.442 | INFO | } | 20260506 11:54:10.442 | INFO | Perf report for the last 5.1391 seconds: | 20260506 11:54:10.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:54:10.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:10.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:10.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:10.442 | INFO | ANALYSIS : 63.9%, 2.9, 95.7ms, 0 | 20260506 11:54:10.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:10.442 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260506 11:54:10.442 | INFO | PRE_HQ : 62.0%, 1.9, 61.8ms, 0 | 20260506 11:54:10.442 | INFO | BAZWRITER : 64.6%, 4.7, 156.9ms, 0 | 20260506 11:54:10.442 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260506 11:54:10.442 | INFO | Uploaded 36GiB to GPU in 2.13491s (16.8625GiB/s) | 20260506 11:54:10.442 | INFO | Downloaded 38GiB to GPU in 2.23243s (17.0218GiB/s) | 20260506 11:54:15.581 | INFO | Analyzing chunk frames = [1151488,1152000) | 20260506 11:54:15.581 | INFO | MemInfo: { | 20260506 11:54:15.581 | INFO | "gpu" : " 17 GB", | 20260506 11:54:15.581 | INFO | "huge" : "", | 20260506 11:54:15.581 | INFO | "huge_shared" : " 27 GB", | 20260506 11:54:15.581 | INFO | "malloc" : " 14 GB", | 20260506 11:54:15.581 | INFO | "pinned" : " 29 GB" | 20260506 11:54:15.581 | INFO | } | 20260506 11:54:15.581 | INFO | Perf report for the last 5.13942 seconds: | 20260506 11:54:15.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:54:15.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:15.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:15.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:15.581 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 11:54:15.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:15.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:15.581 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 11:54:15.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:15.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:15.581 | INFO | Uploaded 36GiB to GPU in 2.02016s (17.8203GiB/s) | 20260506 11:54:15.582 | INFO | Downloaded 34GiB to GPU in 1.93705s (17.5525GiB/s) | 20260506 11:54:20.720 | INFO | Analyzing chunk frames = [1152000,1152512) | 20260506 11:54:20.720 | INFO | MemInfo: { | 20260506 11:54:20.720 | INFO | "gpu" : " 17 GB", | 20260506 11:54:20.720 | INFO | "huge" : "", | 20260506 11:54:20.720 | INFO | "huge_shared" : " 27 GB", | 20260506 11:54:20.720 | INFO | "malloc" : " 14 GB", | 20260506 11:54:20.720 | INFO | "pinned" : " 29 GB" | 20260506 11:54:20.720 | INFO | } | 20260506 11:54:20.721 | INFO | Perf report for the last 5.13948 seconds: | 20260506 11:54:20.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:54:20.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:20.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:20.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:20.721 | INFO | ANALYSIS : 61.9%, 3.0, 95.9ms, 0 | 20260506 11:54:20.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:20.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:20.721 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:54:20.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:20.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:20.721 | INFO | Uploaded 36GiB to GPU in 2.00273s (17.9755GiB/s) | 20260506 11:54:20.721 | INFO | Downloaded 34GiB to GPU in 1.94394s (17.4902GiB/s) | 20260506 11:54:25.860 | INFO | Analyzing chunk frames = [1152512,1153024) | 20260506 11:54:25.860 | INFO | MemInfo: { | 20260506 11:54:25.860 | INFO | "gpu" : " 17 GB", | 20260506 11:54:25.860 | INFO | "huge" : "", | 20260506 11:54:25.860 | INFO | "huge_shared" : " 27 GB", | 20260506 11:54:25.860 | INFO | "malloc" : " 14 GB", | 20260506 11:54:25.860 | INFO | "pinned" : " 29 GB" | 20260506 11:54:25.860 | INFO | } | 20260506 11:54:25.860 | INFO | Perf report for the last 5.1392 seconds: | 20260506 11:54:25.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:54:25.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:25.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:25.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:25.860 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260506 11:54:25.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:25.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:25.860 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 11:54:25.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:25.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:25.860 | INFO | Uploaded 36GiB to GPU in 2.05482s (17.5198GiB/s) | 20260506 11:54:25.860 | INFO | Downloaded 34GiB to GPU in 1.97293s (17.2333GiB/s) | 20260506 11:54:30.999 | INFO | Analyzing chunk frames = [1153024,1153536) | 20260506 11:54:30.999 | INFO | MemInfo: { | 20260506 11:54:30.999 | INFO | "gpu" : " 17 GB", | 20260506 11:54:30.999 | INFO | "huge" : "", | 20260506 11:54:30.999 | INFO | "huge_shared" : " 27 GB", | 20260506 11:54:30.999 | INFO | "malloc" : " 14 GB", | 20260506 11:54:30.999 | INFO | "pinned" : " 29 GB" | 20260506 11:54:30.999 | INFO | } | 20260506 11:54:30.999 | INFO | Perf report for the last 5.13937 seconds: | 20260506 11:54:30.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:54:30.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:30.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:30.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:30.999 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 11:54:30.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:30.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:30.999 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:54:30.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:30.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:30.999 | INFO | Uploaded 36GiB to GPU in 2.03478s (17.6923GiB/s) | 20260506 11:54:30.999 | INFO | Downloaded 34GiB to GPU in 1.97978s (17.1736GiB/s) | 20260506 11:54:36.138 | INFO | Analyzing chunk frames = [1153536,1154048) | 20260506 11:54:36.138 | INFO | MemInfo: { | 20260506 11:54:36.138 | INFO | "gpu" : " 17 GB", | 20260506 11:54:36.138 | INFO | "huge" : "", | 20260506 11:54:36.138 | INFO | "huge_shared" : " 27 GB", | 20260506 11:54:36.138 | INFO | "malloc" : " 14 GB", | 20260506 11:54:36.138 | INFO | "pinned" : " 29 GB" | 20260506 11:54:36.138 | INFO | } | 20260506 11:54:36.139 | INFO | Perf report for the last 5.13938 seconds: | 20260506 11:54:36.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:54:36.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:36.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:36.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:36.139 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 11:54:36.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:36.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:36.139 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:54:36.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:36.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:36.139 | INFO | Uploaded 36GiB to GPU in 2.07405s (17.3574GiB/s) | 20260506 11:54:36.139 | INFO | Downloaded 34GiB to GPU in 1.9847s (17.1311GiB/s) | 20260506 11:54:41.278 | INFO | Analyzing chunk frames = [1154048,1154560) | 20260506 11:54:41.278 | INFO | MemInfo: { | 20260506 11:54:41.278 | INFO | "gpu" : " 17 GB", | 20260506 11:54:41.278 | INFO | "huge" : "", | 20260506 11:54:41.278 | INFO | "huge_shared" : " 27 GB", | 20260506 11:54:41.278 | INFO | "malloc" : " 14 GB", | 20260506 11:54:41.278 | INFO | "pinned" : " 29 GB" | 20260506 11:54:41.278 | INFO | } | 20260506 11:54:41.278 | INFO | Perf report for the last 5.13971 seconds: | 20260506 11:54:41.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:54:41.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:41.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:41.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:41.278 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 11:54:41.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:41.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:41.278 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:54:41.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:41.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:41.279 | INFO | Uploaded 36GiB to GPU in 1.98775s (18.1109GiB/s) | 20260506 11:54:41.279 | INFO | Downloaded 34GiB to GPU in 2.01413s (16.8807GiB/s) | 20260506 11:54:46.417 | INFO | Analyzing chunk frames = [1154560,1155072) | 20260506 11:54:46.417 | INFO | MemInfo: { | 20260506 11:54:46.417 | INFO | "gpu" : " 17 GB", | 20260506 11:54:46.417 | INFO | "huge" : "", | 20260506 11:54:46.417 | INFO | "huge_shared" : " 27 GB", | 20260506 11:54:46.417 | INFO | "malloc" : " 14 GB", | 20260506 11:54:46.417 | INFO | "pinned" : " 29 GB" | 20260506 11:54:46.417 | INFO | } | 20260506 11:54:46.417 | INFO | Perf report for the last 5.13894 seconds: | 20260506 11:54:46.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:54:46.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:46.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:46.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:46.417 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260506 11:54:46.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:46.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:46.417 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260506 11:54:46.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:46.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:46.417 | INFO | Uploaded 36GiB to GPU in 1.99231s (18.0695GiB/s) | 20260506 11:54:46.417 | INFO | Downloaded 34GiB to GPU in 1.99118s (17.0753GiB/s) | 20260506 11:54:46.418 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:54:46.418 | INFO | Total mem usage: 16.8 GiB | 20260506 11:54:46.418 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:54:46.418 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -8.63%) | 20260506 11:54:46.418 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:54:46.418 | INFO | High Water Breakdown: | 20260506 11:54:46.418 | INFO | 60.6% : 9.38 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:54:46.418 | INFO | 39.2% : 6.07 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:54:46.418 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:54:46.418 | INFO | ------------------------------------------ | 20260506 11:54:46.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:54:46.418 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:54:46.418 | INFO | Total mem usage: 26.5 GiB | 20260506 11:54:46.418 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 11:54:46.418 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:54:46.418 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 11:54:46.418 | INFO | High Water Breakdown: | 20260506 11:54:46.418 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:54:46.418 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:54:46.418 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:54:46.418 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:54:46.418 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:54:46.418 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:54:46.418 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:54:46.418 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:54:46.418 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:54:46.418 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:54:46.418 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:54:46.418 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:54:46.418 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:54:46.418 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:54:46.418 | INFO | ------------------------------------------ | 20260506 11:54:46.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:54:46.418 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:54:46.418 | INFO | Total mem usage: 25.6 GiB | 20260506 11:54:46.418 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:54:46.418 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:54:46.418 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:54:46.418 | INFO | High Water Breakdown: | 20260506 11:54:46.418 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:54:46.418 | INFO | ------------------------------------------ | 20260506 11:54:46.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:54:46.418 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:54:46.418 | INFO | Total mem usage: 15.7 GiB | 20260506 11:54:46.418 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:54:46.418 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:54:46.418 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 11:54:46.418 | INFO | High Water Breakdown: | 20260506 11:54:46.418 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:54:46.418 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:54:46.418 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:54:46.418 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:54:46.418 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:54:46.418 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:54:46.418 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:54:46.418 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:54:46.418 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:54:46.418 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:54:46.418 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:54:46.418 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:54:46.418 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:54:46.418 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:54:46.418 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:54:46.418 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:54:46.418 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:54:46.418 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:54:46.419 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:54:46.419 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:54:46.419 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:54:46.419 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:54:46.419 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:54:46.419 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:54:46.419 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:54:46.419 | INFO | ------------------------------------------ | 20260506 11:54:46.419 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:54:46.419 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:54:46.419 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:54:46.419 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:54:46.419 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.62e+01, 1.36e+01 / 2.03e+00 | 20260506 11:54:46.419 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.00e-01, 1.03e-02 / 6.51e-03 | 20260506 11:54:46.419 | INFO | PerfReport: 11.53%, Baseline -- 3.68e+00 / 4.44e+00, 5.84e-01 / 3.58e-01 | 20260506 11:54:46.419 | INFO | PerfReport: 3.23%, Histogram -- 1.03e+00 / 1.47e+00, 1.91e-01 / 1.55e-01 | 20260506 11:54:46.419 | INFO | PerfReport: 15.86%, DME -- 4.05e+01 / 4.51e+01, 4.01e+00 / 2.32e+00 | 20260506 11:54:46.419 | INFO | PerfReport: 56.28%, FrameLabeling -- 1.80e+01 / 2.00e+01, 1.48e+00 / 1.81e+00 | 20260506 11:54:46.419 | INFO | PerfReport: 7.66%, PulseAccumulating -- 2.44e+00 / 3.11e+00, 4.36e-01 / 1.98e-01 | 20260506 11:54:46.419 | INFO | PerfReport: 3.80%, Metrics -- 1.21e+00 / 1.58e+00, 3.05e-01 / 1.76e-01 | 20260506 11:54:46.419 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.29e-01 / 4.84e-01, 6.72e-02 / 2.79e-02 | 20260506 11:54:46.419 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:54:46.419 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:54:46.419 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:54:46.419 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260506 11:54:46.419 | INFO | PerfReport: 21.50%, Upload -- 2.12e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 11:54:46.419 | INFO | PerfReport: 21.14%, Download -- 2.09e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 11:54:46.419 | INFO | PerfReport: 32.32%, Compute -- 3.19e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 11:54:46.419 | INFO | PerfReport: 1.83%, LockUpload -- 1.80e+00 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 11:54:46.419 | INFO | PerfReport: 1.08%, LockDownload -- 1.07e+00 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260506 11:54:46.419 | INFO | PerfReport: 22.12%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 11:54:46.419 | INFO | PerfReport: 0.02%, Misc -- 1.51e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260506 11:54:46.546 | INFO | Starting a round of Baz IO | 20260506 11:54:49.730 | INFO | Populating RTMetrics report | 20260506 11:54:49.731 | INFO | RTMetrics report populated | 20260506 11:54:49.731 | INFO | Opening tmp RTMetrics json file | 20260506 11:54:49.740 | INFO | Serializing RTMetrics to json | 20260506 11:54:49.743 | INFO | Serialization done, swapping tmp file now... | 20260506 11:54:49.746 | INFO | RTMetrics done | 20260506 11:54:50.204 | INFO | Finished a round of Baz IO: Wrote 2.077 GiB in 3.658 seconds | 20260506 11:54:50.204 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.85% / 0.03% / 57.12% | 20260506 11:54:50.204 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.59588% / 4.37101e-05% / 4.37101e-05% | 20260506 11:54:50.204 | INFO | Baz IO overall: | 20260506 11:54:50.204 | INFO | BytesWritten : 563912278751 | 20260506 11:54:50.204 | INFO | NumEvents : 272340752896 | 20260506 11:54:51.557 | INFO | Analyzing chunk frames = [1155072,1155584) | 20260506 11:54:51.557 | INFO | MemInfo: { | 20260506 11:54:51.557 | INFO | "gpu" : " 17 GB", | 20260506 11:54:51.557 | INFO | "huge" : "", | 20260506 11:54:51.557 | INFO | "huge_shared" : " 27 GB", | 20260506 11:54:51.557 | INFO | "malloc" : " 14 GB", | 20260506 11:54:51.557 | INFO | "pinned" : " 29 GB" | 20260506 11:54:51.557 | INFO | } | 20260506 11:54:51.557 | INFO | Perf report for the last 5.13973 seconds: | 20260506 11:54:51.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:54:51.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:51.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:51.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:51.557 | INFO | ANALYSIS : 63.9%, 2.9, 95.7ms, 0 | 20260506 11:54:51.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:51.557 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260506 11:54:51.557 | INFO | PRE_HQ : 62.2%, 2.0, 63.0ms, 0 | 20260506 11:54:51.557 | INFO | BAZWRITER : 70.6%, 4.8, 174.4ms, 0 | 20260506 11:54:51.557 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 11:54:51.557 | INFO | Uploaded 36GiB to GPU in 2.14161s (16.8098GiB/s) | 20260506 11:54:51.558 | INFO | Downloaded 38GiB to GPU in 2.24323s (16.9399GiB/s) | 20260506 11:54:56.696 | INFO | Analyzing chunk frames = [1155584,1156096) | 20260506 11:54:56.696 | INFO | MemInfo: { | 20260506 11:54:56.696 | INFO | "gpu" : " 17 GB", | 20260506 11:54:56.696 | INFO | "huge" : "", | 20260506 11:54:56.696 | INFO | "huge_shared" : " 27 GB", | 20260506 11:54:56.696 | INFO | "malloc" : " 14 GB", | 20260506 11:54:56.696 | INFO | "pinned" : " 29 GB" | 20260506 11:54:56.696 | INFO | } | 20260506 11:54:56.696 | INFO | Perf report for the last 5.13899 seconds: | 20260506 11:54:56.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:54:56.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:56.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:56.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:56.696 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 11:54:56.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:56.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:56.696 | INFO | PRE_HQ : 29.6%, 1.0, 15.5ms, 0 | 20260506 11:54:56.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:56.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:54:56.696 | INFO | Uploaded 36GiB to GPU in 2.01974s (17.8241GiB/s) | 20260506 11:54:56.696 | INFO | Downloaded 34GiB to GPU in 1.93778s (17.5458GiB/s) | 20260506 11:55:01.835 | INFO | Analyzing chunk frames = [1156096,1156608) | 20260506 11:55:01.835 | INFO | MemInfo: { | 20260506 11:55:01.835 | INFO | "gpu" : " 17 GB", | 20260506 11:55:01.835 | INFO | "huge" : "", | 20260506 11:55:01.835 | INFO | "huge_shared" : " 27 GB", | 20260506 11:55:01.835 | INFO | "malloc" : " 14 GB", | 20260506 11:55:01.835 | INFO | "pinned" : " 29 GB" | 20260506 11:55:01.835 | INFO | } | 20260506 11:55:01.835 | INFO | Perf report for the last 5.13947 seconds: | 20260506 11:55:01.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:55:01.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:01.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:01.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:01.835 | INFO | ANALYSIS : 61.9%, 3.0, 95.9ms, 0 | 20260506 11:55:01.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:01.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:01.836 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:55:01.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:01.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:01.836 | INFO | Uploaded 36GiB to GPU in 1.99643s (18.0322GiB/s) | 20260506 11:55:01.836 | INFO | Downloaded 34GiB to GPU in 1.94428s (17.4872GiB/s) | 20260506 11:55:06.974 | INFO | Analyzing chunk frames = [1156608,1157120) | 20260506 11:55:06.974 | INFO | MemInfo: { | 20260506 11:55:06.974 | INFO | "gpu" : " 17 GB", | 20260506 11:55:06.974 | INFO | "huge" : "", | 20260506 11:55:06.974 | INFO | "huge_shared" : " 27 GB", | 20260506 11:55:06.974 | INFO | "malloc" : " 14 GB", | 20260506 11:55:06.974 | INFO | "pinned" : " 29 GB" | 20260506 11:55:06.974 | INFO | } | 20260506 11:55:06.975 | INFO | Perf report for the last 5.13913 seconds: | 20260506 11:55:06.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:55:06.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:06.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:06.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:06.975 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 11:55:06.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:06.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:06.975 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 11:55:06.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:06.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:06.975 | INFO | Uploaded 36GiB to GPU in 2.06274s (17.4525GiB/s) | 20260506 11:55:06.975 | INFO | Downloaded 34GiB to GPU in 1.97821s (17.1873GiB/s) | 20260506 11:55:12.114 | INFO | Analyzing chunk frames = [1157120,1157632) | 20260506 11:55:12.114 | INFO | MemInfo: { | 20260506 11:55:12.114 | INFO | "gpu" : " 17 GB", | 20260506 11:55:12.114 | INFO | "huge" : "", | 20260506 11:55:12.114 | INFO | "huge_shared" : " 27 GB", | 20260506 11:55:12.114 | INFO | "malloc" : " 14 GB", | 20260506 11:55:12.114 | INFO | "pinned" : " 29 GB" | 20260506 11:55:12.114 | INFO | } | 20260506 11:55:12.114 | INFO | Perf report for the last 5.13936 seconds: | 20260506 11:55:12.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:55:12.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:12.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:12.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:12.114 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 11:55:12.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:12.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:12.114 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:55:12.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:12.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:12.114 | INFO | Uploaded 36GiB to GPU in 1.99495s (18.0456GiB/s) | 20260506 11:55:12.114 | INFO | Downloaded 34GiB to GPU in 1.97611s (17.2056GiB/s) | 20260506 11:55:17.253 | INFO | Analyzing chunk frames = [1157632,1158144) | 20260506 11:55:17.253 | INFO | MemInfo: { | 20260506 11:55:17.253 | INFO | "gpu" : " 17 GB", | 20260506 11:55:17.253 | INFO | "huge" : "", | 20260506 11:55:17.254 | INFO | "huge_shared" : " 27 GB", | 20260506 11:55:17.254 | INFO | "malloc" : " 14 GB", | 20260506 11:55:17.254 | INFO | "pinned" : " 29 GB" | 20260506 11:55:17.254 | INFO | } | 20260506 11:55:17.254 | INFO | Perf report for the last 5.13978 seconds: | 20260506 11:55:17.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:55:17.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:17.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:17.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:17.254 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 11:55:17.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:17.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:17.254 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 11:55:17.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:17.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:17.254 | INFO | Uploaded 36GiB to GPU in 2.0345s (17.6948GiB/s) | 20260506 11:55:17.254 | INFO | Downloaded 34GiB to GPU in 1.93171s (17.601GiB/s) | 20260506 11:55:22.393 | INFO | Analyzing chunk frames = [1158144,1158656) | 20260506 11:55:22.393 | INFO | MemInfo: { | 20260506 11:55:22.393 | INFO | "gpu" : " 17 GB", | 20260506 11:55:22.393 | INFO | "huge" : "", | 20260506 11:55:22.393 | INFO | "huge_shared" : " 27 GB", | 20260506 11:55:22.393 | INFO | "malloc" : " 14 GB", | 20260506 11:55:22.393 | INFO | "pinned" : " 29 GB" | 20260506 11:55:22.393 | INFO | } | 20260506 11:55:22.393 | INFO | Perf report for the last 5.13951 seconds: | 20260506 11:55:22.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:55:22.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:22.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:22.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:22.393 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 11:55:22.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:22.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:22.393 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 11:55:22.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:22.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:22.393 | INFO | Uploaded 36GiB to GPU in 1.96853s (18.2877GiB/s) | 20260506 11:55:22.393 | INFO | Downloaded 34GiB to GPU in 2.03095s (16.741GiB/s) | 20260506 11:55:27.532 | INFO | Analyzing chunk frames = [1158656,1159168) | 20260506 11:55:27.532 | INFO | MemInfo: { | 20260506 11:55:27.532 | INFO | "gpu" : " 17 GB", | 20260506 11:55:27.532 | INFO | "huge" : "", | 20260506 11:55:27.532 | INFO | "huge_shared" : " 27 GB", | 20260506 11:55:27.532 | INFO | "malloc" : " 14 GB", | 20260506 11:55:27.532 | INFO | "pinned" : " 29 GB" | 20260506 11:55:27.532 | INFO | } | 20260506 11:55:27.532 | INFO | Perf report for the last 5.13898 seconds: | 20260506 11:55:27.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:55:27.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:27.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:27.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:27.532 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260506 11:55:27.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:27.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:27.532 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260506 11:55:27.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:27.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:27.533 | INFO | Uploaded 36GiB to GPU in 2.02723s (17.7582GiB/s) | 20260506 11:55:27.533 | INFO | Downloaded 34GiB to GPU in 1.97972s (17.1741GiB/s) | 20260506 11:55:27.655 | INFO | Starting a round of Baz IO | 20260506 11:55:30.837 | INFO | Populating RTMetrics report | 20260506 11:55:30.837 | INFO | RTMetrics report populated | 20260506 11:55:30.838 | INFO | Opening tmp RTMetrics json file | 20260506 11:55:30.869 | INFO | Serializing RTMetrics to json | 20260506 11:55:30.952 | INFO | Serialization done, swapping tmp file now... | 20260506 11:55:30.954 | INFO | RTMetrics done | 20260506 11:55:31.255 | INFO | Finished a round of Baz IO: Wrote 2.078 GiB in 3.599 seconds | 20260506 11:55:31.255 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.82% / 0.03% / 57.15% | 20260506 11:55:31.255 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.57935% / 6.75519e-05% / 3.17891e-05% | 20260506 11:55:31.255 | INFO | Baz IO overall: | 20260506 11:55:31.255 | INFO | BytesWritten : 566143096214 | 20260506 11:55:31.255 | INFO | NumEvents : 273422751764 | 20260506 11:55:32.671 | INFO | Analyzing chunk frames = [1159168,1159680) | 20260506 11:55:32.671 | INFO | MemInfo: { | 20260506 11:55:32.671 | INFO | "gpu" : " 17 GB", | 20260506 11:55:32.671 | INFO | "huge" : "", | 20260506 11:55:32.671 | INFO | "huge_shared" : " 27 GB", | 20260506 11:55:32.672 | INFO | "malloc" : " 14 GB", | 20260506 11:55:32.672 | INFO | "pinned" : " 29 GB" | 20260506 11:55:32.672 | INFO | } | 20260506 11:55:32.672 | INFO | Perf report for the last 5.13943 seconds: | 20260506 11:55:32.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:55:32.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:32.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:32.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:32.672 | INFO | ANALYSIS : 63.7%, 2.9, 95.7ms, 0 | 20260506 11:55:32.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:32.672 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260506 11:55:32.672 | INFO | PRE_HQ : 61.7%, 1.9, 61.1ms, 0 | 20260506 11:55:32.672 | INFO | BAZWRITER : 69.1%, 4.8, 170.4ms, 0 | 20260506 11:55:32.672 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.1ms, 0 | 20260506 11:55:32.672 | INFO | Uploaded 36GiB to GPU in 2.1305s (16.8975GiB/s) | 20260506 11:55:32.672 | INFO | Downloaded 38GiB to GPU in 2.22043s (17.1138GiB/s) | 20260506 11:55:37.811 | INFO | Analyzing chunk frames = [1159680,1160192) | 20260506 11:55:37.811 | INFO | MemInfo: { | 20260506 11:55:37.811 | INFO | "gpu" : " 17 GB", | 20260506 11:55:37.811 | INFO | "huge" : "", | 20260506 11:55:37.811 | INFO | "huge_shared" : " 27 GB", | 20260506 11:55:37.811 | INFO | "malloc" : " 14 GB", | 20260506 11:55:37.811 | INFO | "pinned" : " 29 GB" | 20260506 11:55:37.811 | INFO | } | 20260506 11:55:37.811 | INFO | Perf report for the last 5.1391 seconds: | 20260506 11:55:37.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:55:37.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:37.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:37.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:37.811 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:55:37.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:37.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:37.811 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 11:55:37.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:37.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:37.811 | INFO | Uploaded 36GiB to GPU in 2.03836s (17.6612GiB/s) | 20260506 11:55:37.811 | INFO | Downloaded 34GiB to GPU in 1.92805s (17.6344GiB/s) | 20260506 11:55:42.950 | INFO | Analyzing chunk frames = [1160192,1160704) | 20260506 11:55:42.950 | INFO | MemInfo: { | 20260506 11:55:42.950 | INFO | "gpu" : " 17 GB", | 20260506 11:55:42.950 | INFO | "huge" : "", | 20260506 11:55:42.950 | INFO | "huge_shared" : " 27 GB", | 20260506 11:55:42.950 | INFO | "malloc" : " 14 GB", | 20260506 11:55:42.950 | INFO | "pinned" : " 29 GB" | 20260506 11:55:42.950 | INFO | } | 20260506 11:55:42.950 | INFO | Perf report for the last 5.13935 seconds: | 20260506 11:55:42.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:55:42.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:42.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:42.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:42.950 | INFO | ANALYSIS : 62.0%, 3.0, 96.0ms, 0 | 20260506 11:55:42.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:42.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:42.950 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 11:55:42.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:42.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:42.950 | INFO | Uploaded 36GiB to GPU in 2.04911s (17.5686GiB/s) | 20260506 11:55:42.950 | INFO | Downloaded 34GiB to GPU in 1.93399s (17.5802GiB/s) | 20260506 11:55:48.089 | INFO | Analyzing chunk frames = [1160704,1161216) | 20260506 11:55:48.089 | INFO | MemInfo: { | 20260506 11:55:48.089 | INFO | "gpu" : " 17 GB", | 20260506 11:55:48.089 | INFO | "huge" : "", | 20260506 11:55:48.089 | INFO | "huge_shared" : " 27 GB", | 20260506 11:55:48.089 | INFO | "malloc" : " 14 GB", | 20260506 11:55:48.089 | INFO | "pinned" : " 29 GB" | 20260506 11:55:48.089 | INFO | } | 20260506 11:55:48.090 | INFO | Perf report for the last 5.13935 seconds: | 20260506 11:55:48.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:55:48.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:48.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:48.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:48.090 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 11:55:48.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:48.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:48.090 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:55:48.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:48.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:48.090 | INFO | Uploaded 36GiB to GPU in 2.07583s (17.3425GiB/s) | 20260506 11:55:48.090 | INFO | Downloaded 34GiB to GPU in 1.96433s (17.3087GiB/s) | 20260506 11:55:53.229 | INFO | Analyzing chunk frames = [1161216,1161728) | 20260506 11:55:53.229 | INFO | MemInfo: { | 20260506 11:55:53.229 | INFO | "gpu" : " 17 GB", | 20260506 11:55:53.229 | INFO | "huge" : "", | 20260506 11:55:53.229 | INFO | "huge_shared" : " 27 GB", | 20260506 11:55:53.229 | INFO | "malloc" : " 14 GB", | 20260506 11:55:53.229 | INFO | "pinned" : " 29 GB" | 20260506 11:55:53.229 | INFO | } | 20260506 11:55:53.229 | INFO | Perf report for the last 5.13985 seconds: | 20260506 11:55:53.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:55:53.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:53.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:53.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:53.229 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 11:55:53.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:53.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:53.230 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:55:53.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:53.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:53.230 | INFO | Uploaded 36GiB to GPU in 2.09743s (17.1639GiB/s) | 20260506 11:55:53.230 | INFO | Downloaded 34GiB to GPU in 1.98251s (17.15GiB/s) | 20260506 11:55:58.368 | INFO | Analyzing chunk frames = [1161728,1162240) | 20260506 11:55:58.368 | INFO | MemInfo: { | 20260506 11:55:58.368 | INFO | "gpu" : " 17 GB", | 20260506 11:55:58.368 | INFO | "huge" : "", | 20260506 11:55:58.368 | INFO | "huge_shared" : " 27 GB", | 20260506 11:55:58.368 | INFO | "malloc" : " 14 GB", | 20260506 11:55:58.368 | INFO | "pinned" : " 29 GB" | 20260506 11:55:58.368 | INFO | } | 20260506 11:55:58.368 | INFO | Perf report for the last 5.13884 seconds: | 20260506 11:55:58.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:55:58.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:58.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:58.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:58.368 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 11:55:58.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:58.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:58.368 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:55:58.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:58.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:55:58.369 | INFO | Uploaded 36GiB to GPU in 2.07718s (17.3312GiB/s) | 20260506 11:55:58.369 | INFO | Downloaded 34GiB to GPU in 2.00182s (16.9846GiB/s) | 20260506 11:56:03.508 | INFO | Analyzing chunk frames = [1162240,1162752) | 20260506 11:56:03.508 | INFO | MemInfo: { | 20260506 11:56:03.508 | INFO | "gpu" : " 17 GB", | 20260506 11:56:03.508 | INFO | "huge" : "", | 20260506 11:56:03.508 | INFO | "huge_shared" : " 27 GB", | 20260506 11:56:03.508 | INFO | "malloc" : " 14 GB", | 20260506 11:56:03.508 | INFO | "pinned" : " 29 GB" | 20260506 11:56:03.508 | INFO | } | 20260506 11:56:03.508 | INFO | Perf report for the last 5.1402 seconds: | 20260506 11:56:03.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:56:03.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:03.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:03.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:03.509 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 11:56:03.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:03.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:03.509 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 11:56:03.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:03.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:03.509 | INFO | Uploaded 36GiB to GPU in 2.02391s (17.7873GiB/s) | 20260506 11:56:03.509 | INFO | Downloaded 34GiB to GPU in 2.00445s (16.9623GiB/s) | 20260506 11:56:08.647 | INFO | Analyzing chunk frames = [1162752,1163264) | 20260506 11:56:08.647 | INFO | MemInfo: { | 20260506 11:56:08.647 | INFO | "gpu" : " 17 GB", | 20260506 11:56:08.647 | INFO | "huge" : "", | 20260506 11:56:08.647 | INFO | "huge_shared" : " 27 GB", | 20260506 11:56:08.647 | INFO | "malloc" : " 14 GB", | 20260506 11:56:08.647 | INFO | "pinned" : " 29 GB" | 20260506 11:56:08.647 | INFO | } | 20260506 11:56:08.647 | INFO | Perf report for the last 5.1387 seconds: | 20260506 11:56:08.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:56:08.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:08.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:08.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:08.647 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260506 11:56:08.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:08.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:08.647 | INFO | PRE_HQ : 33.0%, 1.0, 17.2ms, 0 | 20260506 11:56:08.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:08.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:08.647 | INFO | Uploaded 36GiB to GPU in 1.99746s (18.0229GiB/s) | 20260506 11:56:08.648 | INFO | Downloaded 34GiB to GPU in 1.97722s (17.1959GiB/s) | 20260506 11:56:08.648 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:56:08.648 | INFO | Total mem usage: 16.8 GiB | 20260506 11:56:08.648 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:56:08.648 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -6.41%) | 20260506 11:56:08.648 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:56:08.648 | INFO | High Water Breakdown: | 20260506 11:56:08.648 | INFO | 61% : 9.63 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:56:08.648 | INFO | 38.9% : 6.15 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:56:08.648 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:56:08.648 | INFO | ------------------------------------------ | 20260506 11:56:08.648 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:56:08.648 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:56:08.648 | INFO | Total mem usage: 26.5 GiB | 20260506 11:56:08.648 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 11:56:08.648 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:56:08.648 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 11:56:08.648 | INFO | High Water Breakdown: | 20260506 11:56:08.648 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:56:08.648 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:56:08.648 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:56:08.648 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:56:08.648 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:56:08.648 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:56:08.648 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:56:08.648 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:56:08.648 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:56:08.648 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:56:08.648 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:56:08.648 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:56:08.648 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:56:08.648 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:56:08.648 | INFO | ------------------------------------------ | 20260506 11:56:08.648 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:56:08.648 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:56:08.648 | INFO | Total mem usage: 25.6 GiB | 20260506 11:56:08.648 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:56:08.648 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:56:08.648 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:56:08.648 | INFO | High Water Breakdown: | 20260506 11:56:08.648 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:56:08.648 | INFO | ------------------------------------------ | 20260506 11:56:08.648 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:56:08.648 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:56:08.648 | INFO | Total mem usage: 15.7 GiB | 20260506 11:56:08.648 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:56:08.648 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:56:08.648 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 11:56:08.648 | INFO | High Water Breakdown: | 20260506 11:56:08.648 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:56:08.648 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:56:08.648 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:56:08.648 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:56:08.648 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:56:08.648 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:56:08.648 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:56:08.648 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:56:08.648 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:56:08.648 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:56:08.648 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:56:08.648 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:56:08.648 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:56:08.648 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:56:08.648 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:56:08.648 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:56:08.648 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:56:08.648 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:56:08.648 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:56:08.648 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:56:08.648 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:56:08.648 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:56:08.648 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:56:08.648 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:56:08.648 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:56:08.648 | INFO | ------------------------------------------ | 20260506 11:56:08.648 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:56:08.648 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:56:08.648 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:56:08.648 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:56:08.648 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.58e+01, 1.36e+01 / 2.65e+00 | 20260506 11:56:08.648 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.03e-01, 1.19e-02 / 2.46e-02 | 20260506 11:56:08.648 | INFO | PerfReport: 11.56%, Baseline -- 3.69e+00 / 4.41e+00, 5.88e-01 / 3.86e-01 | 20260506 11:56:08.648 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.44e+00, 1.92e-01 / 1.51e-01 | 20260506 11:56:08.648 | INFO | PerfReport: 15.84%, DME -- 4.04e+01 / 4.48e+01, 4.22e+00 / 2.62e+00 | 20260506 11:56:08.648 | INFO | PerfReport: 56.26%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.47e+00 / 1.78e+00 | 20260506 11:56:08.648 | INFO | PerfReport: 7.65%, PulseAccumulating -- 2.44e+00 / 3.12e+00, 4.33e-01 / 1.94e-01 | 20260506 11:56:08.648 | INFO | PerfReport: 3.81%, Metrics -- 1.22e+00 / 1.61e+00, 3.08e-01 / 1.69e-01 | 20260506 11:56:08.648 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.30e-01 / 4.94e-01, 6.70e-02 / 3.60e-02 | 20260506 11:56:08.648 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:56:08.648 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:56:08.648 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:56:08.648 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 11:56:08.648 | INFO | PerfReport: 21.58%, Upload -- 2.13e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260506 11:56:08.648 | INFO | PerfReport: 21.12%, Download -- 2.09e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 11:56:08.648 | INFO | PerfReport: 32.31%, Compute -- 3.19e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 11:56:08.649 | INFO | PerfReport: 1.93%, LockUpload -- 1.90e+00 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 11:56:08.649 | INFO | PerfReport: 0.95%, LockDownload -- 9.36e-01 / 0.00e+00, 2.52e+00 / 0.00e+00 | 20260506 11:56:08.649 | INFO | PerfReport: 22.09%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 11:56:08.649 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.31e-02 / 0.00e+00 | 20260506 11:56:08.778 | INFO | Starting a round of Baz IO | 20260506 11:56:11.952 | INFO | Populating RTMetrics report | 20260506 11:56:11.953 | INFO | RTMetrics report populated | 20260506 11:56:11.953 | INFO | Opening tmp RTMetrics json file | 20260506 11:56:12.008 | INFO | Serializing RTMetrics to json | 20260506 11:56:12.011 | INFO | Serialization done, swapping tmp file now... | 20260506 11:56:12.012 | INFO | RTMetrics done | 20260506 11:56:12.158 | INFO | Finished a round of Baz IO: Wrote 2.078 GiB in 3.38 seconds | 20260506 11:56:12.158 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.79% / 0.03% / 57.18% | 20260506 11:56:12.159 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.56623% / 7.15256e-05% / 3.97364e-05% | 20260506 11:56:12.159 | INFO | Baz IO overall: | 20260506 11:56:12.159 | INFO | BytesWritten : 568374030890 | 20260506 11:56:12.159 | INFO | NumEvents : 274504842399 | 20260506 11:56:13.786 | INFO | Analyzing chunk frames = [1163264,1163776) | 20260506 11:56:13.786 | INFO | MemInfo: { | 20260506 11:56:13.786 | INFO | "gpu" : " 17 GB", | 20260506 11:56:13.786 | INFO | "huge" : "", | 20260506 11:56:13.786 | INFO | "huge_shared" : " 27 GB", | 20260506 11:56:13.786 | INFO | "malloc" : " 14 GB", | 20260506 11:56:13.786 | INFO | "pinned" : " 29 GB" | 20260506 11:56:13.786 | INFO | } | 20260506 11:56:13.786 | INFO | Perf report for the last 5.13905 seconds: | 20260506 11:56:13.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:56:13.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:13.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:13.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:13.786 | INFO | ANALYSIS : 63.5%, 2.9, 95.5ms, 0 | 20260506 11:56:13.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:13.786 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260506 11:56:13.786 | INFO | PRE_HQ : 61.9%, 1.9, 62.3ms, 0 | 20260506 11:56:13.786 | INFO | BAZWRITER : 65.2%, 4.8, 160.3ms, 0 | 20260506 11:56:13.786 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.5ms, 0 | 20260506 11:56:13.787 | INFO | Uploaded 36GiB to GPU in 2.11008s (17.061GiB/s) | 20260506 11:56:13.787 | INFO | Downloaded 38GiB to GPU in 2.23639s (16.9917GiB/s) | 20260506 11:56:18.925 | INFO | Analyzing chunk frames = [1163776,1164288) | 20260506 11:56:18.925 | INFO | MemInfo: { | 20260506 11:56:18.925 | INFO | "gpu" : " 17 GB", | 20260506 11:56:18.925 | INFO | "huge" : "", | 20260506 11:56:18.925 | INFO | "huge_shared" : " 27 GB", | 20260506 11:56:18.925 | INFO | "malloc" : " 14 GB", | 20260506 11:56:18.926 | INFO | "pinned" : " 29 GB" | 20260506 11:56:18.926 | INFO | } | 20260506 11:56:18.926 | INFO | Perf report for the last 5.13942 seconds: | 20260506 11:56:18.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:56:18.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:18.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:18.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:18.926 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:56:18.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:18.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:18.926 | INFO | PRE_HQ : 31.2%, 1.0, 16.4ms, 0 | 20260506 11:56:18.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:18.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:18.926 | INFO | Uploaded 36GiB to GPU in 2.02929s (17.7402GiB/s) | 20260506 11:56:18.926 | INFO | Downloaded 34GiB to GPU in 1.92102s (17.699GiB/s) | 20260506 11:56:24.065 | INFO | Analyzing chunk frames = [1164288,1164800) | 20260506 11:56:24.065 | INFO | MemInfo: { | 20260506 11:56:24.065 | INFO | "gpu" : " 17 GB", | 20260506 11:56:24.065 | INFO | "huge" : "", | 20260506 11:56:24.065 | INFO | "huge_shared" : " 27 GB", | 20260506 11:56:24.065 | INFO | "malloc" : " 14 GB", | 20260506 11:56:24.065 | INFO | "pinned" : " 29 GB" | 20260506 11:56:24.065 | INFO | } | 20260506 11:56:24.065 | INFO | Perf report for the last 5.13955 seconds: | 20260506 11:56:24.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:56:24.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:24.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:24.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:24.065 | INFO | ANALYSIS : 62.0%, 3.0, 96.0ms, 0 | 20260506 11:56:24.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:24.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:24.065 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:56:24.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:24.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:24.066 | INFO | Uploaded 36GiB to GPU in 2.04584s (17.5967GiB/s) | 20260506 11:56:24.066 | INFO | Downloaded 34GiB to GPU in 1.93632s (17.5591GiB/s) | 20260506 11:56:29.204 | INFO | Analyzing chunk frames = [1164800,1165312) | 20260506 11:56:29.204 | INFO | MemInfo: { | 20260506 11:56:29.204 | INFO | "gpu" : " 17 GB", | 20260506 11:56:29.204 | INFO | "huge" : "", | 20260506 11:56:29.205 | INFO | "huge_shared" : " 27 GB", | 20260506 11:56:29.205 | INFO | "malloc" : " 14 GB", | 20260506 11:56:29.205 | INFO | "pinned" : " 29 GB" | 20260506 11:56:29.205 | INFO | } | 20260506 11:56:29.205 | INFO | Perf report for the last 5.13948 seconds: | 20260506 11:56:29.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:56:29.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:29.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:29.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:29.205 | INFO | ANALYSIS : 62.2%, 2.9, 96.1ms, 0 | 20260506 11:56:29.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:29.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:29.205 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:56:29.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:29.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:29.205 | INFO | Uploaded 36GiB to GPU in 2.03105s (17.7249GiB/s) | 20260506 11:56:29.205 | INFO | Downloaded 34GiB to GPU in 1.98083s (17.1645GiB/s) | 20260506 11:56:34.343 | INFO | Analyzing chunk frames = [1165312,1165824) | 20260506 11:56:34.343 | INFO | MemInfo: { | 20260506 11:56:34.343 | INFO | "gpu" : " 17 GB", | 20260506 11:56:34.344 | INFO | "huge" : "", | 20260506 11:56:34.344 | INFO | "huge_shared" : " 27 GB", | 20260506 11:56:34.344 | INFO | "malloc" : " 14 GB", | 20260506 11:56:34.344 | INFO | "pinned" : " 29 GB" | 20260506 11:56:34.344 | INFO | } | 20260506 11:56:34.344 | INFO | Perf report for the last 5.13892 seconds: | 20260506 11:56:34.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:56:34.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:34.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:34.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:34.344 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 11:56:34.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:34.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:34.344 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:56:34.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:34.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:34.344 | INFO | Uploaded 36GiB to GPU in 2.03474s (17.6926GiB/s) | 20260506 11:56:34.344 | INFO | Downloaded 34GiB to GPU in 1.96289s (17.3214GiB/s) | 20260506 11:56:39.483 | INFO | Analyzing chunk frames = [1165824,1166336) | 20260506 11:56:39.483 | INFO | MemInfo: { | 20260506 11:56:39.483 | INFO | "gpu" : " 17 GB", | 20260506 11:56:39.483 | INFO | "huge" : "", | 20260506 11:56:39.483 | INFO | "huge_shared" : " 27 GB", | 20260506 11:56:39.483 | INFO | "malloc" : " 14 GB", | 20260506 11:56:39.483 | INFO | "pinned" : " 29 GB" | 20260506 11:56:39.483 | INFO | } | 20260506 11:56:39.483 | INFO | Perf report for the last 5.13944 seconds: | 20260506 11:56:39.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:56:39.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:39.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:39.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:39.483 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 11:56:39.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:39.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:39.483 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 11:56:39.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:39.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:39.483 | INFO | Uploaded 36GiB to GPU in 2.10219s (17.125GiB/s) | 20260506 11:56:39.484 | INFO | Downloaded 34GiB to GPU in 1.9804s (17.1682GiB/s) | 20260506 11:56:44.623 | INFO | Analyzing chunk frames = [1166336,1166848) | 20260506 11:56:44.623 | INFO | MemInfo: { | 20260506 11:56:44.623 | INFO | "gpu" : " 17 GB", | 20260506 11:56:44.623 | INFO | "huge" : "", | 20260506 11:56:44.623 | INFO | "huge_shared" : " 27 GB", | 20260506 11:56:44.623 | INFO | "malloc" : " 14 GB", | 20260506 11:56:44.623 | INFO | "pinned" : " 29 GB" | 20260506 11:56:44.623 | INFO | } | 20260506 11:56:44.623 | INFO | Perf report for the last 5.14007 seconds: | 20260506 11:56:44.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:56:44.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:44.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:44.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:44.623 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260506 11:56:44.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:44.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:44.623 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:56:44.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:44.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:44.624 | INFO | Uploaded 36GiB to GPU in 1.9896s (18.0941GiB/s) | 20260506 11:56:44.624 | INFO | Downloaded 34GiB to GPU in 2.0113s (16.9045GiB/s) | 20260506 11:56:49.762 | INFO | Analyzing chunk frames = [1166848,1167360) | 20260506 11:56:49.762 | INFO | MemInfo: { | 20260506 11:56:49.762 | INFO | "gpu" : " 17 GB", | 20260506 11:56:49.762 | INFO | "huge" : "", | 20260506 11:56:49.762 | INFO | "huge_shared" : " 27 GB", | 20260506 11:56:49.762 | INFO | "malloc" : " 14 GB", | 20260506 11:56:49.762 | INFO | "pinned" : " 29 GB" | 20260506 11:56:49.762 | INFO | } | 20260506 11:56:49.762 | INFO | Perf report for the last 5.13873 seconds: | 20260506 11:56:49.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:56:49.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:49.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:49.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:49.762 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 11:56:49.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:49.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:49.762 | INFO | PRE_HQ : 33.4%, 1.0, 17.3ms, 0 | 20260506 11:56:49.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:49.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:49.762 | INFO | Uploaded 36GiB to GPU in 2.00915s (17.9181GiB/s) | 20260506 11:56:49.762 | INFO | Downloaded 34GiB to GPU in 1.96581s (17.2956GiB/s) | 20260506 11:56:49.890 | INFO | Starting a round of Baz IO | 20260506 11:56:53.088 | INFO | Populating RTMetrics report | 20260506 11:56:53.088 | INFO | RTMetrics report populated | 20260506 11:56:53.088 | INFO | Opening tmp RTMetrics json file | 20260506 11:56:53.103 | INFO | Serializing RTMetrics to json | 20260506 11:56:53.105 | INFO | Serialization done, swapping tmp file now... | 20260506 11:56:53.105 | INFO | RTMetrics done | 20260506 11:56:53.426 | INFO | Finished a round of Baz IO: Wrote 2.078 GiB in 3.536 seconds | 20260506 11:56:53.426 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.76% / 0.03% / 57.21% | 20260506 11:56:53.427 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.54863% / 5.16574e-05% / 4.37101e-05% | 20260506 11:56:53.427 | INFO | Baz IO overall: | 20260506 11:56:53.427 | INFO | BytesWritten : 570605608431 | 20260506 11:56:53.427 | INFO | NumEvents : 275587172803 | 20260506 11:56:54.901 | INFO | Analyzing chunk frames = [1167360,1167872) | 20260506 11:56:54.901 | INFO | MemInfo: { | 20260506 11:56:54.901 | INFO | "gpu" : " 17 GB", | 20260506 11:56:54.901 | INFO | "huge" : "", | 20260506 11:56:54.901 | INFO | "huge_shared" : " 27 GB", | 20260506 11:56:54.901 | INFO | "malloc" : " 14 GB", | 20260506 11:56:54.901 | INFO | "pinned" : " 29 GB" | 20260506 11:56:54.901 | INFO | } | 20260506 11:56:54.901 | INFO | Perf report for the last 5.13908 seconds: | 20260506 11:56:54.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:56:54.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:54.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:54.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:54.901 | INFO | ANALYSIS : 63.8%, 2.9, 96.0ms, 0 | 20260506 11:56:54.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:56:54.901 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260506 11:56:54.901 | INFO | PRE_HQ : 62.2%, 2.0, 63.5ms, 0 | 20260506 11:56:54.901 | INFO | BAZWRITER : 68.2%, 4.8, 168.1ms, 0 | 20260506 11:56:54.901 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 11:56:54.901 | INFO | Uploaded 36GiB to GPU in 2.08002s (17.3075GiB/s) | 20260506 11:56:54.902 | INFO | Downloaded 38GiB to GPU in 2.20771s (17.2124GiB/s) | 20260506 11:57:00.040 | INFO | Analyzing chunk frames = [1167872,1168384) | 20260506 11:57:00.040 | INFO | MemInfo: { | 20260506 11:57:00.040 | INFO | "gpu" : " 17 GB", | 20260506 11:57:00.040 | INFO | "huge" : "", | 20260506 11:57:00.040 | INFO | "huge_shared" : " 27 GB", | 20260506 11:57:00.040 | INFO | "malloc" : " 14 GB", | 20260506 11:57:00.040 | INFO | "pinned" : " 29 GB" | 20260506 11:57:00.040 | INFO | } | 20260506 11:57:00.040 | INFO | Perf report for the last 5.1393 seconds: | 20260506 11:57:00.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:57:00.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:00.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:00.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:00.040 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:57:00.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:00.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:00.040 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 11:57:00.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:00.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:00.041 | INFO | Uploaded 36GiB to GPU in 1.99802s (18.0179GiB/s) | 20260506 11:57:00.041 | INFO | Downloaded 34GiB to GPU in 1.91655s (17.7402GiB/s) | 20260506 11:57:05.180 | INFO | Analyzing chunk frames = [1168384,1168896) | 20260506 11:57:05.180 | INFO | MemInfo: { | 20260506 11:57:05.180 | INFO | "gpu" : " 17 GB", | 20260506 11:57:05.180 | INFO | "huge" : "", | 20260506 11:57:05.180 | INFO | "huge_shared" : " 27 GB", | 20260506 11:57:05.180 | INFO | "malloc" : " 14 GB", | 20260506 11:57:05.180 | INFO | "pinned" : " 29 GB" | 20260506 11:57:05.180 | INFO | } | 20260506 11:57:05.180 | INFO | Perf report for the last 5.13937 seconds: | 20260506 11:57:05.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:57:05.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:05.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:05.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:05.180 | INFO | ANALYSIS : 61.9%, 3.0, 95.8ms, 0 | 20260506 11:57:05.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:05.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:05.180 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 11:57:05.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:05.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:05.180 | INFO | Uploaded 36GiB to GPU in 2.02216s (17.8027GiB/s) | 20260506 11:57:05.180 | INFO | Downloaded 34GiB to GPU in 1.94217s (17.5062GiB/s) | 20260506 11:57:10.319 | INFO | Analyzing chunk frames = [1168896,1169408) | 20260506 11:57:10.319 | INFO | MemInfo: { | 20260506 11:57:10.319 | INFO | "gpu" : " 17 GB", | 20260506 11:57:10.319 | INFO | "huge" : "", | 20260506 11:57:10.319 | INFO | "huge_shared" : " 27 GB", | 20260506 11:57:10.319 | INFO | "malloc" : " 14 GB", | 20260506 11:57:10.319 | INFO | "pinned" : " 29 GB" | 20260506 11:57:10.319 | INFO | } | 20260506 11:57:10.319 | INFO | Perf report for the last 5.13963 seconds: | 20260506 11:57:10.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:57:10.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:10.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:10.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:10.319 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260506 11:57:10.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:10.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:10.319 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:57:10.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:10.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:10.320 | INFO | Uploaded 36GiB to GPU in 2.05389s (17.5277GiB/s) | 20260506 11:57:10.320 | INFO | Downloaded 34GiB to GPU in 1.97929s (17.1779GiB/s) | 20260506 11:57:15.459 | INFO | Analyzing chunk frames = [1169408,1169920) | 20260506 11:57:15.459 | INFO | MemInfo: { | 20260506 11:57:15.459 | INFO | "gpu" : " 17 GB", | 20260506 11:57:15.459 | INFO | "huge" : "", | 20260506 11:57:15.459 | INFO | "huge_shared" : " 27 GB", | 20260506 11:57:15.459 | INFO | "malloc" : " 14 GB", | 20260506 11:57:15.459 | INFO | "pinned" : " 29 GB" | 20260506 11:57:15.459 | INFO | } | 20260506 11:57:15.459 | INFO | Perf report for the last 5.13942 seconds: | 20260506 11:57:15.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:57:15.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:15.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:15.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:15.459 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 11:57:15.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:15.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:15.459 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 11:57:15.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:15.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:15.459 | INFO | Uploaded 36GiB to GPU in 1.99252s (18.0676GiB/s) | 20260506 11:57:15.459 | INFO | Downloaded 34GiB to GPU in 1.98123s (17.161GiB/s) | 20260506 11:57:20.598 | INFO | Analyzing chunk frames = [1169920,1170432) | 20260506 11:57:20.598 | INFO | MemInfo: { | 20260506 11:57:20.598 | INFO | "gpu" : " 17 GB", | 20260506 11:57:20.598 | INFO | "huge" : "", | 20260506 11:57:20.598 | INFO | "huge_shared" : " 27 GB", | 20260506 11:57:20.598 | INFO | "malloc" : " 14 GB", | 20260506 11:57:20.598 | INFO | "pinned" : " 29 GB" | 20260506 11:57:20.598 | INFO | } | 20260506 11:57:20.598 | INFO | Perf report for the last 5.13917 seconds: | 20260506 11:57:20.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:57:20.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:20.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:20.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:20.598 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 11:57:20.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:20.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:20.598 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:57:20.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:20.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:20.598 | INFO | Uploaded 36GiB to GPU in 2.08482s (17.2677GiB/s) | 20260506 11:57:20.598 | INFO | Downloaded 34GiB to GPU in 1.98106s (17.1625GiB/s) | 20260506 11:57:25.738 | INFO | Analyzing chunk frames = [1170432,1170944) | 20260506 11:57:25.738 | INFO | MemInfo: { | 20260506 11:57:25.738 | INFO | "gpu" : " 17 GB", | 20260506 11:57:25.738 | INFO | "huge" : "", | 20260506 11:57:25.738 | INFO | "huge_shared" : " 27 GB", | 20260506 11:57:25.738 | INFO | "malloc" : " 14 GB", | 20260506 11:57:25.738 | INFO | "pinned" : " 29 GB" | 20260506 11:57:25.738 | INFO | } | 20260506 11:57:25.738 | INFO | Perf report for the last 5.1402 seconds: | 20260506 11:57:25.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:57:25.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:25.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:25.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:25.738 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 11:57:25.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:25.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:25.738 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:57:25.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:25.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:25.738 | INFO | Uploaded 36GiB to GPU in 1.9863s (18.1241GiB/s) | 20260506 11:57:25.738 | INFO | Downloaded 34GiB to GPU in 2.0119s (16.8995GiB/s) | 20260506 11:57:30.876 | INFO | Analyzing chunk frames = [1170944,1171456) | 20260506 11:57:30.876 | INFO | MemInfo: { | 20260506 11:57:30.876 | INFO | "gpu" : " 17 GB", | 20260506 11:57:30.876 | INFO | "huge" : "", | 20260506 11:57:30.876 | INFO | "huge_shared" : " 27 GB", | 20260506 11:57:30.876 | INFO | "malloc" : " 14 GB", | 20260506 11:57:30.877 | INFO | "pinned" : " 29 GB" | 20260506 11:57:30.877 | INFO | } | 20260506 11:57:30.877 | INFO | Perf report for the last 5.13846 seconds: | 20260506 11:57:30.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:57:30.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:30.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:30.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:30.877 | INFO | ANALYSIS : 63.0%, 3.0, 98.1ms, 0 | 20260506 11:57:30.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:30.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:30.877 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260506 11:57:30.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:30.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:30.877 | INFO | Uploaded 36GiB to GPU in 2.07785s (17.3256GiB/s) | 20260506 11:57:30.877 | INFO | Downloaded 34GiB to GPU in 1.99114s (17.0757GiB/s) | 20260506 11:57:30.877 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:57:30.877 | INFO | Total mem usage: 16.8 GiB | 20260506 11:57:30.877 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:57:30.877 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -7.08%) | 20260506 11:57:30.877 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:57:30.877 | INFO | High Water Breakdown: | 20260506 11:57:30.877 | INFO | 60.9% : 9.56 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:57:30.877 | INFO | 39% : 6.12 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:57:30.877 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:57:30.877 | INFO | ------------------------------------------ | 20260506 11:57:30.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:57:30.877 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:57:30.877 | INFO | Total mem usage: 26.5 GiB | 20260506 11:57:30.877 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 11:57:30.877 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:57:30.877 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 11:57:30.877 | INFO | High Water Breakdown: | 20260506 11:57:30.877 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:57:30.877 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:57:30.877 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:57:30.877 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:57:30.877 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:57:30.877 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:57:30.877 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:57:30.877 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:57:30.877 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:57:30.877 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:57:30.877 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:57:30.877 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:57:30.877 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:57:30.877 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:57:30.877 | INFO | ------------------------------------------ | 20260506 11:57:30.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:57:30.877 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:57:30.877 | INFO | Total mem usage: 25.6 GiB | 20260506 11:57:30.877 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:57:30.877 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:57:30.878 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:57:30.878 | INFO | High Water Breakdown: | 20260506 11:57:30.878 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:57:30.878 | INFO | ------------------------------------------ | 20260506 11:57:30.878 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:57:30.878 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:57:30.878 | INFO | Total mem usage: 15.7 GiB | 20260506 11:57:30.878 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:57:30.878 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:57:30.878 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 11:57:30.878 | INFO | High Water Breakdown: | 20260506 11:57:30.878 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:57:30.878 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:57:30.878 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:57:30.878 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:57:30.878 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:57:30.878 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:57:30.878 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:57:30.878 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:57:30.878 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:57:30.878 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:57:30.878 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:57:30.878 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:57:30.878 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:57:30.878 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:57:30.878 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:57:30.878 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:57:30.878 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:57:30.878 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:57:30.878 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:57:30.878 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:57:30.878 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:57:30.878 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:57:30.878 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:57:30.878 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:57:30.878 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:57:30.878 | INFO | ------------------------------------------ | 20260506 11:57:30.878 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:57:30.878 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:57:30.878 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:57:30.878 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:57:30.878 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.63e+01, 1.36e+01 / 2.06e+00 | 20260506 11:57:30.878 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.06e-02 / 7.58e-03 | 20260506 11:57:30.878 | INFO | PerfReport: 11.54%, Baseline -- 3.68e+00 / 4.47e+00, 5.82e-01 / 2.83e-01 | 20260506 11:57:30.878 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.47e+00, 1.92e-01 / 1.19e-01 | 20260506 11:57:30.878 | INFO | PerfReport: 15.90%, DME -- 4.05e+01 / 4.53e+01, 4.13e+00 / 2.49e+00 | 20260506 11:57:30.878 | INFO | PerfReport: 56.21%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.40e+00 / 1.45e+00 | 20260506 11:57:30.878 | INFO | PerfReport: 7.66%, PulseAccumulating -- 2.44e+00 / 3.12e+00, 4.33e-01 / 2.02e-01 | 20260506 11:57:30.878 | INFO | PerfReport: 3.81%, Metrics -- 1.21e+00 / 1.58e+00, 3.07e-01 / 1.86e-01 | 20260506 11:57:30.878 | INFO | PerfReport: 1.35%, PulseSerializing -- 4.29e-01 / 4.83e-01, 6.64e-02 / 2.55e-02 | 20260506 11:57:30.878 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:57:30.878 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:57:30.878 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:57:30.878 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 11:57:30.878 | INFO | PerfReport: 21.53%, Upload -- 2.13e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 11:57:30.878 | INFO | PerfReport: 21.11%, Download -- 2.08e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 11:57:30.878 | INFO | PerfReport: 32.28%, Compute -- 3.19e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 11:57:30.878 | INFO | PerfReport: 1.87%, LockUpload -- 1.85e+00 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260506 11:57:30.878 | INFO | PerfReport: 1.04%, LockDownload -- 1.03e+00 / 0.00e+00, 2.62e+00 / 0.00e+00 | 20260506 11:57:30.878 | INFO | PerfReport: 22.15%, LockCompute -- 2.19e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 11:57:30.878 | INFO | PerfReport: 0.02%, Misc -- 1.50e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260506 11:57:31.002 | INFO | Starting a round of Baz IO | 20260506 11:57:34.189 | INFO | Populating RTMetrics report | 20260506 11:57:34.189 | INFO | RTMetrics report populated | 20260506 11:57:34.189 | INFO | Opening tmp RTMetrics json file | 20260506 11:57:34.956 | INFO | Serializing RTMetrics to json | 20260506 11:57:34.961 | INFO | Serialization done, swapping tmp file now... | 20260506 11:57:34.964 | INFO | RTMetrics done | 20260506 11:57:34.975 | INFO | Finished a round of Baz IO: Wrote 2.08 GiB in 3.972 seconds | 20260506 11:57:34.975 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.73% / 0.03% / 57.24% | 20260506 11:57:34.975 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.54826% / 4.37101e-05% / 3.17891e-05% | 20260506 11:57:34.975 | INFO | Baz IO overall: | 20260506 11:57:34.975 | INFO | BytesWritten : 572839505903 | 20260506 11:57:34.975 | INFO | NumEvents : 276670920794 | 20260506 11:57:36.016 | INFO | Analyzing chunk frames = [1171456,1171968) | 20260506 11:57:36.016 | INFO | MemInfo: { | 20260506 11:57:36.016 | INFO | "gpu" : " 17 GB", | 20260506 11:57:36.016 | INFO | "huge" : "", | 20260506 11:57:36.016 | INFO | "huge_shared" : " 27 GB", | 20260506 11:57:36.016 | INFO | "malloc" : " 14 GB", | 20260506 11:57:36.016 | INFO | "pinned" : " 29 GB" | 20260506 11:57:36.016 | INFO | } | 20260506 11:57:36.016 | INFO | Perf report for the last 5.13909 seconds: | 20260506 11:57:36.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:57:36.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:36.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:36.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:36.016 | INFO | ANALYSIS : 63.8%, 2.9, 95.9ms, 0 | 20260506 11:57:36.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:36.016 | INFO | RT_METRICS : 59.3%, 1.0, 31.1ms, 0 | 20260506 11:57:36.016 | INFO | PRE_HQ : 61.9%, 1.9, 61.2ms, 0 | 20260506 11:57:36.016 | INFO | BAZWRITER : 76.5%, 4.8, 190.4ms, 0 | 20260506 11:57:36.016 | INFO | BAZ_SAMPLING : 12.8%, 1.0, 6.7ms, 0 | 20260506 11:57:36.016 | INFO | Uploaded 36GiB to GPU in 2.16335s (16.6409GiB/s) | 20260506 11:57:36.016 | INFO | Downloaded 38GiB to GPU in 2.2359s (16.9954GiB/s) | 20260506 11:57:41.155 | INFO | Analyzing chunk frames = [1171968,1172480) | 20260506 11:57:41.155 | INFO | MemInfo: { | 20260506 11:57:41.155 | INFO | "gpu" : " 17 GB", | 20260506 11:57:41.155 | INFO | "huge" : "", | 20260506 11:57:41.155 | INFO | "huge_shared" : " 27 GB", | 20260506 11:57:41.155 | INFO | "malloc" : " 14 GB", | 20260506 11:57:41.155 | INFO | "pinned" : " 29 GB" | 20260506 11:57:41.155 | INFO | } | 20260506 11:57:41.155 | INFO | Perf report for the last 5.13959 seconds: | 20260506 11:57:41.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:57:41.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:41.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:41.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:41.155 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 11:57:41.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:41.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:41.155 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:57:41.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:41.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:41.156 | INFO | Uploaded 36GiB to GPU in 2.03793s (17.665GiB/s) | 20260506 11:57:41.156 | INFO | Downloaded 34GiB to GPU in 1.92947s (17.6215GiB/s) | 20260506 11:57:46.294 | INFO | Analyzing chunk frames = [1172480,1172992) | 20260506 11:57:46.294 | INFO | MemInfo: { | 20260506 11:57:46.294 | INFO | "gpu" : " 17 GB", | 20260506 11:57:46.294 | INFO | "huge" : "", | 20260506 11:57:46.294 | INFO | "huge_shared" : " 27 GB", | 20260506 11:57:46.295 | INFO | "malloc" : " 14 GB", | 20260506 11:57:46.295 | INFO | "pinned" : " 29 GB" | 20260506 11:57:46.295 | INFO | } | 20260506 11:57:46.295 | INFO | Perf report for the last 5.13923 seconds: | 20260506 11:57:46.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:57:46.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:46.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:46.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:46.295 | INFO | ANALYSIS : 62.0%, 3.0, 96.0ms, 0 | 20260506 11:57:46.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:46.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:46.295 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:57:46.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:46.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:46.295 | INFO | Uploaded 36GiB to GPU in 2.04436s (17.6094GiB/s) | 20260506 11:57:46.295 | INFO | Downloaded 34GiB to GPU in 1.93662s (17.5563GiB/s) | 20260506 11:57:51.434 | INFO | Analyzing chunk frames = [1172992,1173504) | 20260506 11:57:51.434 | INFO | MemInfo: { | 20260506 11:57:51.434 | INFO | "gpu" : " 17 GB", | 20260506 11:57:51.434 | INFO | "huge" : "", | 20260506 11:57:51.434 | INFO | "huge_shared" : " 27 GB", | 20260506 11:57:51.434 | INFO | "malloc" : " 14 GB", | 20260506 11:57:51.434 | INFO | "pinned" : " 29 GB" | 20260506 11:57:51.434 | INFO | } | 20260506 11:57:51.434 | INFO | Perf report for the last 5.13938 seconds: | 20260506 11:57:51.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:57:51.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:51.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:51.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:51.434 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 11:57:51.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:51.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:51.434 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 11:57:51.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:51.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:51.434 | INFO | Uploaded 36GiB to GPU in 2.08258s (17.2862GiB/s) | 20260506 11:57:51.434 | INFO | Downloaded 34GiB to GPU in 1.97578s (17.2084GiB/s) | 20260506 11:57:56.573 | INFO | Analyzing chunk frames = [1173504,1174016) | 20260506 11:57:56.573 | INFO | MemInfo: { | 20260506 11:57:56.573 | INFO | "gpu" : " 17 GB", | 20260506 11:57:56.573 | INFO | "huge" : "", | 20260506 11:57:56.573 | INFO | "huge_shared" : " 27 GB", | 20260506 11:57:56.573 | INFO | "malloc" : " 14 GB", | 20260506 11:57:56.573 | INFO | "pinned" : " 29 GB" | 20260506 11:57:56.573 | INFO | } | 20260506 11:57:56.573 | INFO | Perf report for the last 5.13934 seconds: | 20260506 11:57:56.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:57:56.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:56.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:56.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:56.573 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 11:57:56.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:56.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:56.574 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 11:57:56.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:56.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:57:56.574 | INFO | Uploaded 36GiB to GPU in 2.10924s (17.0678GiB/s) | 20260506 11:57:56.574 | INFO | Downloaded 34GiB to GPU in 1.98221s (17.1525GiB/s) | 20260506 11:58:01.712 | INFO | Analyzing chunk frames = [1174016,1174528) | 20260506 11:58:01.712 | INFO | MemInfo: { | 20260506 11:58:01.712 | INFO | "gpu" : " 17 GB", | 20260506 11:58:01.712 | INFO | "huge" : "", | 20260506 11:58:01.713 | INFO | "huge_shared" : " 27 GB", | 20260506 11:58:01.713 | INFO | "malloc" : " 14 GB", | 20260506 11:58:01.713 | INFO | "pinned" : " 29 GB" | 20260506 11:58:01.713 | INFO | } | 20260506 11:58:01.713 | INFO | Perf report for the last 5.13938 seconds: | 20260506 11:58:01.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:58:01.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:01.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:01.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:01.713 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 11:58:01.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:01.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:01.713 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:58:01.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:01.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:01.713 | INFO | Uploaded 36GiB to GPU in 2.07589s (17.342GiB/s) | 20260506 11:58:01.713 | INFO | Downloaded 34GiB to GPU in 2.00047s (16.996GiB/s) | 20260506 11:58:06.853 | INFO | Analyzing chunk frames = [1174528,1175040) | 20260506 11:58:06.853 | INFO | MemInfo: { | 20260506 11:58:06.853 | INFO | "gpu" : " 17 GB", | 20260506 11:58:06.853 | INFO | "huge" : "", | 20260506 11:58:06.853 | INFO | "huge_shared" : " 27 GB", | 20260506 11:58:06.853 | INFO | "malloc" : " 14 GB", | 20260506 11:58:06.853 | INFO | "pinned" : " 29 GB" | 20260506 11:58:06.853 | INFO | } | 20260506 11:58:06.853 | INFO | Perf report for the last 5.14047 seconds: | 20260506 11:58:06.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:58:06.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:06.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:06.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:06.853 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 11:58:06.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:06.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:06.853 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 11:58:06.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:06.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:06.854 | INFO | Uploaded 36GiB to GPU in 2.00515s (17.9538GiB/s) | 20260506 11:58:06.854 | INFO | Downloaded 34GiB to GPU in 1.97626s (17.2042GiB/s) | 20260506 11:58:11.991 | INFO | Analyzing chunk frames = [1175040,1175552) | 20260506 11:58:11.991 | INFO | MemInfo: { | 20260506 11:58:11.991 | INFO | "gpu" : " 17 GB", | 20260506 11:58:11.991 | INFO | "huge" : "", | 20260506 11:58:11.991 | INFO | "huge_shared" : " 27 GB", | 20260506 11:58:11.991 | INFO | "malloc" : " 14 GB", | 20260506 11:58:11.991 | INFO | "pinned" : " 29 GB" | 20260506 11:58:11.991 | INFO | } | 20260506 11:58:11.992 | INFO | Perf report for the last 5.13832 seconds: | 20260506 11:58:11.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:58:11.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:11.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:11.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:11.992 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 11:58:11.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:11.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:11.992 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260506 11:58:11.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:11.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:11.992 | INFO | Uploaded 36GiB to GPU in 2.0126s (17.8873GiB/s) | 20260506 11:58:11.992 | INFO | Downloaded 34GiB to GPU in 1.98027s (17.1694GiB/s) | 20260506 11:58:12.113 | INFO | Starting a round of Baz IO | 20260506 11:58:15.312 | INFO | Populating RTMetrics report | 20260506 11:58:15.313 | INFO | RTMetrics report populated | 20260506 11:58:15.313 | INFO | Opening tmp RTMetrics json file | 20260506 11:58:15.329 | INFO | Serializing RTMetrics to json | 20260506 11:58:15.360 | INFO | Serialization done, swapping tmp file now... | 20260506 11:58:15.377 | INFO | RTMetrics done | 20260506 11:58:15.805 | INFO | Finished a round of Baz IO: Wrote 2.08 GiB in 3.692 seconds | 20260506 11:58:15.805 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.70% / 0.03% / 57.27% | 20260506 11:58:15.805 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.52769% / 4.37101e-05% / 2.78155e-05% | 20260506 11:58:15.805 | INFO | Baz IO overall: | 20260506 11:58:15.805 | INFO | BytesWritten : 575072954345 | 20260506 11:58:15.805 | INFO | NumEvents : 277754155811 | 20260506 11:58:17.131 | INFO | Analyzing chunk frames = [1175552,1176064) | 20260506 11:58:17.131 | INFO | MemInfo: { | 20260506 11:58:17.131 | INFO | "gpu" : " 17 GB", | 20260506 11:58:17.131 | INFO | "huge" : "", | 20260506 11:58:17.131 | INFO | "huge_shared" : " 27 GB", | 20260506 11:58:17.131 | INFO | "malloc" : " 14 GB", | 20260506 11:58:17.131 | INFO | "pinned" : " 29 GB" | 20260506 11:58:17.131 | INFO | } | 20260506 11:58:17.131 | INFO | Perf report for the last 5.13931 seconds: | 20260506 11:58:17.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:58:17.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:17.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:17.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:17.131 | INFO | ANALYSIS : 63.7%, 2.9, 95.6ms, 0 | 20260506 11:58:17.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:17.131 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260506 11:58:17.131 | INFO | PRE_HQ : 61.4%, 1.9, 61.6ms, 0 | 20260506 11:58:17.131 | INFO | BAZWRITER : 71.0%, 4.8, 175.9ms, 0 | 20260506 11:58:17.131 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 11:58:17.131 | INFO | Uploaded 36GiB to GPU in 2.08548s (17.2622GiB/s) | 20260506 11:58:17.131 | INFO | Downloaded 38GiB to GPU in 2.21133s (17.1842GiB/s) | 20260506 11:58:22.270 | INFO | Analyzing chunk frames = [1176064,1176576) | 20260506 11:58:22.270 | INFO | MemInfo: { | 20260506 11:58:22.270 | INFO | "gpu" : " 17 GB", | 20260506 11:58:22.270 | INFO | "huge" : "", | 20260506 11:58:22.270 | INFO | "huge_shared" : " 27 GB", | 20260506 11:58:22.270 | INFO | "malloc" : " 14 GB", | 20260506 11:58:22.270 | INFO | "pinned" : " 29 GB" | 20260506 11:58:22.270 | INFO | } | 20260506 11:58:22.270 | INFO | Perf report for the last 5.13904 seconds: | 20260506 11:58:22.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:58:22.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:22.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:22.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:22.270 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 11:58:22.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:22.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:22.270 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 11:58:22.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:22.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:22.270 | INFO | Uploaded 36GiB to GPU in 2.04519s (17.6023GiB/s) | 20260506 11:58:22.270 | INFO | Downloaded 34GiB to GPU in 1.93212s (17.5972GiB/s) | 20260506 11:58:27.409 | INFO | Analyzing chunk frames = [1176576,1177088) | 20260506 11:58:27.409 | INFO | MemInfo: { | 20260506 11:58:27.409 | INFO | "gpu" : " 17 GB", | 20260506 11:58:27.409 | INFO | "huge" : "", | 20260506 11:58:27.409 | INFO | "huge_shared" : " 27 GB", | 20260506 11:58:27.409 | INFO | "malloc" : " 14 GB", | 20260506 11:58:27.409 | INFO | "pinned" : " 29 GB" | 20260506 11:58:27.410 | INFO | } | 20260506 11:58:27.410 | INFO | Perf report for the last 5.13967 seconds: | 20260506 11:58:27.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:58:27.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:27.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:27.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:27.410 | INFO | ANALYSIS : 62.3%, 3.0, 96.5ms, 0 | 20260506 11:58:27.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:27.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:27.410 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 11:58:27.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:27.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:27.410 | INFO | Uploaded 36GiB to GPU in 2.05756s (17.4965GiB/s) | 20260506 11:58:27.410 | INFO | Downloaded 34GiB to GPU in 1.96743s (17.2814GiB/s) | 20260506 11:58:32.548 | INFO | Analyzing chunk frames = [1177088,1177600) | 20260506 11:58:32.548 | INFO | MemInfo: { | 20260506 11:58:32.548 | INFO | "gpu" : " 17 GB", | 20260506 11:58:32.548 | INFO | "huge" : "", | 20260506 11:58:32.549 | INFO | "huge_shared" : " 27 GB", | 20260506 11:58:32.549 | INFO | "malloc" : " 14 GB", | 20260506 11:58:32.549 | INFO | "pinned" : " 29 GB" | 20260506 11:58:32.549 | INFO | } | 20260506 11:58:32.549 | INFO | Perf report for the last 5.13914 seconds: | 20260506 11:58:32.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:58:32.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:32.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:32.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:32.549 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260506 11:58:32.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:32.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:32.549 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:58:32.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:32.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:32.549 | INFO | Uploaded 36GiB to GPU in 2.06873s (17.4019GiB/s) | 20260506 11:58:32.549 | INFO | Downloaded 34GiB to GPU in 1.96341s (17.3168GiB/s) | 20260506 11:58:37.688 | INFO | Analyzing chunk frames = [1177600,1178112) | 20260506 11:58:37.688 | INFO | MemInfo: { | 20260506 11:58:37.688 | INFO | "gpu" : " 17 GB", | 20260506 11:58:37.688 | INFO | "huge" : "", | 20260506 11:58:37.688 | INFO | "huge_shared" : " 27 GB", | 20260506 11:58:37.688 | INFO | "malloc" : " 14 GB", | 20260506 11:58:37.688 | INFO | "pinned" : " 29 GB" | 20260506 11:58:37.688 | INFO | } | 20260506 11:58:37.688 | INFO | Perf report for the last 5.13924 seconds: | 20260506 11:58:37.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:58:37.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:37.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:37.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:37.688 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 11:58:37.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:37.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:37.688 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 11:58:37.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:37.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:37.688 | INFO | Uploaded 36GiB to GPU in 2.00906s (17.9188GiB/s) | 20260506 11:58:37.688 | INFO | Downloaded 34GiB to GPU in 1.97235s (17.2383GiB/s) | 20260506 11:58:42.827 | INFO | Analyzing chunk frames = [1178112,1178624) | 20260506 11:58:42.827 | INFO | MemInfo: { | 20260506 11:58:42.827 | INFO | "gpu" : " 17 GB", | 20260506 11:58:42.827 | INFO | "huge" : "", | 20260506 11:58:42.827 | INFO | "huge_shared" : " 27 GB", | 20260506 11:58:42.827 | INFO | "malloc" : " 14 GB", | 20260506 11:58:42.827 | INFO | "pinned" : " 29 GB" | 20260506 11:58:42.827 | INFO | } | 20260506 11:58:42.827 | INFO | Perf report for the last 5.13929 seconds: | 20260506 11:58:42.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:58:42.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:42.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:42.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:42.827 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 11:58:42.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:42.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:42.827 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:58:42.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:42.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:42.828 | INFO | Uploaded 36GiB to GPU in 2.03799s (17.6645GiB/s) | 20260506 11:58:42.828 | INFO | Downloaded 34GiB to GPU in 1.92986s (17.6178GiB/s) | 20260506 11:58:47.967 | INFO | Analyzing chunk frames = [1178624,1179136) | 20260506 11:58:47.967 | INFO | MemInfo: { | 20260506 11:58:47.967 | INFO | "gpu" : " 17 GB", | 20260506 11:58:47.967 | INFO | "huge" : "", | 20260506 11:58:47.967 | INFO | "huge_shared" : " 27 GB", | 20260506 11:58:47.967 | INFO | "malloc" : " 14 GB", | 20260506 11:58:47.967 | INFO | "pinned" : " 29 GB" | 20260506 11:58:47.967 | INFO | } | 20260506 11:58:47.967 | INFO | Perf report for the last 5.14002 seconds: | 20260506 11:58:47.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:58:47.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:47.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:47.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:47.967 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 11:58:47.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:47.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:47.967 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 11:58:47.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:47.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:47.968 | INFO | Uploaded 36GiB to GPU in 1.98247s (18.1591GiB/s) | 20260506 11:58:47.968 | INFO | Downloaded 34GiB to GPU in 2.03032s (16.7462GiB/s) | 20260506 11:58:53.106 | INFO | Analyzing chunk frames = [1179136,1179648) | 20260506 11:58:53.106 | INFO | MemInfo: { | 20260506 11:58:53.106 | INFO | "gpu" : " 17 GB", | 20260506 11:58:53.106 | INFO | "huge" : "", | 20260506 11:58:53.106 | INFO | "huge_shared" : " 27 GB", | 20260506 11:58:53.106 | INFO | "malloc" : " 14 GB", | 20260506 11:58:53.106 | INFO | "pinned" : " 29 GB" | 20260506 11:58:53.107 | INFO | } | 20260506 11:58:53.107 | INFO | Perf report for the last 5.13922 seconds: | 20260506 11:58:53.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:58:53.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:53.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:53.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:53.107 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 11:58:53.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:53.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:53.107 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260506 11:58:53.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:53.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:53.107 | INFO | Uploaded 36GiB to GPU in 1.98913s (18.0983GiB/s) | 20260506 11:58:53.107 | INFO | Downloaded 34GiB to GPU in 1.96929s (17.2651GiB/s) | 20260506 11:58:53.107 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 11:58:53.107 | INFO | Total mem usage: 16.8 GiB | 20260506 11:58:53.107 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 11:58:53.107 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -6.41%) | 20260506 11:58:53.107 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 11:58:53.107 | INFO | High Water Breakdown: | 20260506 11:58:53.107 | INFO | 60.9% : 9.63 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 11:58:53.107 | INFO | 38.9% : 6.15 GiB ( 91.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 11:58:53.107 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 11:58:53.107 | INFO | ------------------------------------------ | 20260506 11:58:53.107 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:58:53.107 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 11:58:53.107 | INFO | Total mem usage: 26.5 GiB | 20260506 11:58:53.107 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 11:58:53.107 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 11:58:53.107 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 11:58:53.107 | INFO | High Water Breakdown: | 20260506 11:58:53.107 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:58:53.107 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:58:53.107 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:58:53.107 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:58:53.108 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:58:53.108 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:58:53.108 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 11:58:53.108 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 11:58:53.108 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:58:53.108 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 11:58:53.108 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 11:58:53.108 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 11:58:53.108 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 11:58:53.108 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 11:58:53.108 | INFO | ------------------------------------------ | 20260506 11:58:53.108 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:58:53.108 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 11:58:53.108 | INFO | Total mem usage: 25.6 GiB | 20260506 11:58:53.108 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 11:58:53.108 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 11:58:53.108 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 11:58:53.108 | INFO | High Water Breakdown: | 20260506 11:58:53.108 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 11:58:53.108 | INFO | ------------------------------------------ | 20260506 11:58:53.108 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:58:53.108 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 11:58:53.108 | INFO | Total mem usage: 15.7 GiB | 20260506 11:58:53.108 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 11:58:53.108 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 11:58:53.108 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 11:58:53.108 | INFO | High Water Breakdown: | 20260506 11:58:53.108 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 11:58:53.108 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 11:58:53.108 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 11:58:53.108 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 11:58:53.108 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 11:58:53.108 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 11:58:53.108 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 11:58:53.108 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 11:58:53.108 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 11:58:53.108 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 11:58:53.108 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 11:58:53.108 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 11:58:53.108 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 11:58:53.108 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 11:58:53.108 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 11:58:53.108 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 11:58:53.108 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 11:58:53.108 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 11:58:53.108 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 11:58:53.108 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 11:58:53.108 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 11:58:53.108 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 11:58:53.108 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 11:58:53.108 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 11:58:53.108 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 11:58:53.108 | INFO | ------------------------------------------ | 20260506 11:58:53.108 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 11:58:53.108 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 11:58:53.108 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:58:53.108 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:58:53.108 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.64e+01, 1.35e+01 / 2.59e+00 | 20260506 11:58:53.108 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.02e-02 / 9.33e-03 | 20260506 11:58:53.108 | INFO | PerfReport: 11.54%, Baseline -- 3.68e+00 / 4.43e+00, 5.86e-01 / 3.12e-01 | 20260506 11:58:53.108 | INFO | PerfReport: 3.23%, Histogram -- 1.03e+00 / 1.46e+00, 1.91e-01 / 1.34e-01 | 20260506 11:58:53.108 | INFO | PerfReport: 15.87%, DME -- 4.05e+01 / 4.53e+01, 4.36e+00 / 2.75e+00 | 20260506 11:58:53.108 | INFO | PerfReport: 56.28%, FrameLabeling -- 1.79e+01 / 2.00e+01, 1.46e+00 / 1.53e+00 | 20260506 11:58:53.108 | INFO | PerfReport: 7.64%, PulseAccumulating -- 2.43e+00 / 3.08e+00, 4.27e-01 / 2.10e-01 | 20260506 11:58:53.108 | INFO | PerfReport: 3.80%, Metrics -- 1.21e+00 / 1.60e+00, 3.07e-01 / 1.39e-01 | 20260506 11:58:53.108 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.29e-01 / 4.89e-01, 6.65e-02 / 2.52e-02 | 20260506 11:58:53.108 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 11:58:53.108 | INFO | PerfReport: Gathered 1536 samples. | 20260506 11:58:53.108 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 11:58:53.108 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 11:58:53.108 | INFO | PerfReport: 21.64%, Upload -- 2.14e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260506 11:58:53.108 | INFO | PerfReport: 21.10%, Download -- 2.08e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 11:58:53.108 | INFO | PerfReport: 32.31%, Compute -- 3.19e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 11:58:53.108 | INFO | PerfReport: 1.92%, LockUpload -- 1.89e+00 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 11:58:53.108 | INFO | PerfReport: 0.96%, LockDownload -- 9.51e-01 / 0.00e+00, 2.52e+00 / 0.00e+00 | 20260506 11:58:53.108 | INFO | PerfReport: 22.06%, LockCompute -- 2.18e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 11:58:53.108 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260506 11:58:53.238 | INFO | Starting a round of Baz IO | 20260506 11:58:56.404 | INFO | Populating RTMetrics report | 20260506 11:58:56.404 | INFO | RTMetrics report populated | 20260506 11:58:56.404 | INFO | Opening tmp RTMetrics json file | 20260506 11:58:56.406 | INFO | Serializing RTMetrics to json | 20260506 11:58:56.408 | INFO | Serialization done, swapping tmp file now... | 20260506 11:58:56.409 | INFO | RTMetrics done | 20260506 11:58:56.542 | INFO | Finished a round of Baz IO: Wrote 2.08 GiB in 3.304 seconds | 20260506 11:58:56.542 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.67% / 0.03% / 57.30% | 20260506 11:58:56.542 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.51205% / 5.5631e-05% / 5.5631e-05% | 20260506 11:58:56.542 | INFO | Baz IO overall: | 20260506 11:58:56.543 | INFO | BytesWritten : 577306345117 | 20260506 11:58:56.543 | INFO | NumEvents : 278837066705 | 20260506 11:58:58.245 | INFO | Analyzing chunk frames = [1179648,1180160) | 20260506 11:58:58.245 | INFO | MemInfo: { | 20260506 11:58:58.245 | INFO | "gpu" : " 17 GB", | 20260506 11:58:58.245 | INFO | "huge" : "", | 20260506 11:58:58.245 | INFO | "huge_shared" : " 27 GB", | 20260506 11:58:58.245 | INFO | "malloc" : " 14 GB", | 20260506 11:58:58.245 | INFO | "pinned" : " 29 GB" | 20260506 11:58:58.245 | INFO | } | 20260506 11:58:58.245 | INFO | Perf report for the last 5.13871 seconds: | 20260506 11:58:58.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:58:58.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:58.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:58.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:58.245 | INFO | ANALYSIS : 63.6%, 2.9, 95.3ms, 0 | 20260506 11:58:58.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:58:58.246 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 11:58:58.246 | INFO | PRE_HQ : 61.7%, 1.9, 61.6ms, 0 | 20260506 11:58:58.246 | INFO | BAZWRITER : 63.6%, 4.7, 153.3ms, 0 | 20260506 11:58:58.246 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 11:58:58.246 | INFO | Uploaded 36GiB to GPU in 2.10254s (17.1222GiB/s) | 20260506 11:58:58.246 | INFO | Downloaded 38GiB to GPU in 2.23767s (16.9819GiB/s) | 20260506 11:59:03.385 | INFO | Analyzing chunk frames = [1180160,1180672) | 20260506 11:59:03.385 | INFO | MemInfo: { | 20260506 11:59:03.385 | INFO | "gpu" : " 17 GB", | 20260506 11:59:03.385 | INFO | "huge" : "", | 20260506 11:59:03.385 | INFO | "huge_shared" : " 27 GB", | 20260506 11:59:03.385 | INFO | "malloc" : " 14 GB", | 20260506 11:59:03.385 | INFO | "pinned" : " 29 GB" | 20260506 11:59:03.385 | INFO | } | 20260506 11:59:03.385 | INFO | Perf report for the last 5.13947 seconds: | 20260506 11:59:03.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:59:03.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:03.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:03.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:03.385 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 11:59:03.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:03.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:03.385 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 11:59:03.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:03.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:03.385 | INFO | Uploaded 36GiB to GPU in 2.01217s (17.8911GiB/s) | 20260506 11:59:03.385 | INFO | Downloaded 34GiB to GPU in 1.928s (17.6348GiB/s) | 20260506 11:59:08.525 | INFO | Analyzing chunk frames = [1180672,1181184) | 20260506 11:59:08.525 | INFO | MemInfo: { | 20260506 11:59:08.525 | INFO | "gpu" : " 17 GB", | 20260506 11:59:08.525 | INFO | "huge" : "", | 20260506 11:59:08.525 | INFO | "huge_shared" : " 27 GB", | 20260506 11:59:08.525 | INFO | "malloc" : " 14 GB", | 20260506 11:59:08.525 | INFO | "pinned" : " 29 GB" | 20260506 11:59:08.525 | INFO | } | 20260506 11:59:08.525 | INFO | Perf report for the last 5.14006 seconds: | 20260506 11:59:08.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:59:08.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:08.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:08.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:08.525 | INFO | ANALYSIS : 61.9%, 3.0, 95.8ms, 0 | 20260506 11:59:08.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:08.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:08.525 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 11:59:08.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:08.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:08.525 | INFO | Uploaded 36GiB to GPU in 2.02056s (17.8169GiB/s) | 20260506 11:59:08.525 | INFO | Downloaded 34GiB to GPU in 1.94654s (17.4669GiB/s) | 20260506 11:59:13.664 | INFO | Analyzing chunk frames = [1181184,1181696) | 20260506 11:59:13.664 | INFO | MemInfo: { | 20260506 11:59:13.664 | INFO | "gpu" : " 17 GB", | 20260506 11:59:13.664 | INFO | "huge" : "", | 20260506 11:59:13.664 | INFO | "huge_shared" : " 27 GB", | 20260506 11:59:13.664 | INFO | "malloc" : " 14 GB", | 20260506 11:59:13.664 | INFO | "pinned" : " 29 GB" | 20260506 11:59:13.664 | INFO | } | 20260506 11:59:13.664 | INFO | Perf report for the last 5.13888 seconds: | 20260506 11:59:13.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:59:13.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:13.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:13.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:13.664 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 11:59:13.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:13.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:13.664 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 11:59:13.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:13.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:13.664 | INFO | Uploaded 36GiB to GPU in 2.0362s (17.68GiB/s) | 20260506 11:59:13.664 | INFO | Downloaded 34GiB to GPU in 1.97254s (17.2367GiB/s) | 20260506 11:59:18.803 | INFO | Analyzing chunk frames = [1181696,1182208) | 20260506 11:59:18.803 | INFO | MemInfo: { | 20260506 11:59:18.803 | INFO | "gpu" : " 17 GB", | 20260506 11:59:18.803 | INFO | "huge" : "", | 20260506 11:59:18.803 | INFO | "huge_shared" : " 27 GB", | 20260506 11:59:18.803 | INFO | "malloc" : " 14 GB", | 20260506 11:59:18.803 | INFO | "pinned" : " 29 GB" | 20260506 11:59:18.803 | INFO | } | 20260506 11:59:18.803 | INFO | Perf report for the last 5.13919 seconds: | 20260506 11:59:18.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:59:18.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:18.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:18.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:18.803 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 11:59:18.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:18.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:18.803 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 11:59:18.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:18.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:18.803 | INFO | Uploaded 36GiB to GPU in 2.027s (17.7602GiB/s) | 20260506 11:59:18.803 | INFO | Downloaded 34GiB to GPU in 1.96708s (17.2845GiB/s) | 20260506 11:59:23.942 | INFO | Analyzing chunk frames = [1182208,1182720) | 20260506 11:59:23.942 | INFO | MemInfo: { | 20260506 11:59:23.942 | INFO | "gpu" : " 17 GB", | 20260506 11:59:23.942 | INFO | "huge" : "", | 20260506 11:59:23.942 | INFO | "huge_shared" : " 27 GB", | 20260506 11:59:23.942 | INFO | "malloc" : " 14 GB", | 20260506 11:59:23.942 | INFO | "pinned" : " 29 GB" | 20260506 11:59:23.942 | INFO | } | 20260506 11:59:23.942 | INFO | Perf report for the last 5.13939 seconds: | 20260506 11:59:23.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:59:23.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:23.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:23.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:23.942 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 11:59:23.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:23.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:23.942 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 11:59:23.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:23.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:23.943 | INFO | Uploaded 36GiB to GPU in 2.03824s (17.6623GiB/s) | 20260506 11:59:23.943 | INFO | Downloaded 34GiB to GPU in 1.93369s (17.583GiB/s) | 20260506 11:59:29.082 | INFO | Analyzing chunk frames = [1182720,1183232) | 20260506 11:59:29.082 | INFO | MemInfo: { | 20260506 11:59:29.082 | INFO | "gpu" : " 17 GB", | 20260506 11:59:29.082 | INFO | "huge" : "", | 20260506 11:59:29.082 | INFO | "huge_shared" : " 27 GB", | 20260506 11:59:29.082 | INFO | "malloc" : " 14 GB", | 20260506 11:59:29.082 | INFO | "pinned" : " 29 GB" | 20260506 11:59:29.082 | INFO | } | 20260506 11:59:29.082 | INFO | Perf report for the last 5.13966 seconds: | 20260506 11:59:29.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:59:29.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:29.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:29.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:29.082 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 11:59:29.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:29.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:29.082 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 11:59:29.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:29.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:29.082 | INFO | Uploaded 36GiB to GPU in 1.98556s (18.1309GiB/s) | 20260506 11:59:29.082 | INFO | Downloaded 34GiB to GPU in 2.0307s (16.743GiB/s) | 20260506 11:59:34.221 | INFO | Analyzing chunk frames = [1183232,1183744) | 20260506 11:59:34.221 | INFO | MemInfo: { | 20260506 11:59:34.221 | INFO | "gpu" : " 17 GB", | 20260506 11:59:34.221 | INFO | "huge" : "", | 20260506 11:59:34.221 | INFO | "huge_shared" : " 27 GB", | 20260506 11:59:34.221 | INFO | "malloc" : " 14 GB", | 20260506 11:59:34.221 | INFO | "pinned" : " 29 GB" | 20260506 11:59:34.221 | INFO | } | 20260506 11:59:34.221 | INFO | Perf report for the last 5.13889 seconds: | 20260506 11:59:34.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:59:34.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:34.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:34.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:34.221 | INFO | ANALYSIS : 62.8%, 3.0, 97.0ms, 0 | 20260506 11:59:34.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:34.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:34.221 | INFO | PRE_HQ : 33.7%, 1.0, 17.5ms, 0 | 20260506 11:59:34.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:34.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:34.221 | INFO | Uploaded 36GiB to GPU in 2.04522s (17.602GiB/s) | 20260506 11:59:34.221 | INFO | Downloaded 34GiB to GPU in 2.03114s (16.7394GiB/s) | 20260506 11:59:34.353 | INFO | Starting a round of Baz IO | 20260506 11:59:37.559 | INFO | Populating RTMetrics report | 20260506 11:59:37.559 | INFO | RTMetrics report populated | 20260506 11:59:37.559 | INFO | Opening tmp RTMetrics json file | 20260506 11:59:37.576 | INFO | Serializing RTMetrics to json | 20260506 11:59:37.600 | INFO | Serialization done, swapping tmp file now... | 20260506 11:59:37.617 | INFO | RTMetrics done | 20260506 11:59:38.062 | INFO | Finished a round of Baz IO: Wrote 2.082 GiB in 3.709 seconds | 20260506 11:59:38.062 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.64% / 0.03% / 57.33% | 20260506 11:59:38.062 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.50796% / 5.16574e-05% / 3.57628e-05% | 20260506 11:59:38.062 | INFO | Baz IO overall: | 20260506 11:59:38.062 | INFO | BytesWritten : 579541502193 | 20260506 11:59:38.062 | INFO | NumEvents : 279921214602 | 20260506 11:59:39.360 | INFO | Analyzing chunk frames = [1183744,1184256) | 20260506 11:59:39.360 | INFO | MemInfo: { | 20260506 11:59:39.360 | INFO | "gpu" : " 17 GB", | 20260506 11:59:39.360 | INFO | "huge" : "", | 20260506 11:59:39.360 | INFO | "huge_shared" : " 27 GB", | 20260506 11:59:39.360 | INFO | "malloc" : " 14 GB", | 20260506 11:59:39.360 | INFO | "pinned" : " 29 GB" | 20260506 11:59:39.360 | INFO | } | 20260506 11:59:39.360 | INFO | Perf report for the last 5.13926 seconds: | 20260506 11:59:39.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:59:39.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:39.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:39.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:39.360 | INFO | ANALYSIS : 64.3%, 2.8, 94.6ms, 0 | 20260506 11:59:39.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:39.360 | INFO | RT_METRICS : 47.9%, 1.0, 25.1ms, 0 | 20260506 11:59:39.360 | INFO | PRE_HQ : 62.6%, 2.0, 65.5ms, 0 | 20260506 11:59:39.360 | INFO | BAZWRITER : 71.6%, 4.8, 176.4ms, 0 | 20260506 11:59:39.360 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 11:59:39.361 | INFO | Uploaded 36GiB to GPU in 2.12339s (16.954GiB/s) | 20260506 11:59:39.361 | INFO | Downloaded 38GiB to GPU in 2.22991s (17.0411GiB/s) | 20260506 11:59:44.500 | INFO | Analyzing chunk frames = [1184256,1184768) | 20260506 11:59:44.500 | INFO | MemInfo: { | 20260506 11:59:44.500 | INFO | "gpu" : " 17 GB", | 20260506 11:59:44.500 | INFO | "huge" : "", | 20260506 11:59:44.500 | INFO | "huge_shared" : " 27 GB", | 20260506 11:59:44.500 | INFO | "malloc" : " 14 GB", | 20260506 11:59:44.500 | INFO | "pinned" : " 29 GB" | 20260506 11:59:44.500 | INFO | } | 20260506 11:59:44.500 | INFO | Perf report for the last 5.13974 seconds: | 20260506 11:59:44.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:59:44.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:44.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:44.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:44.500 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 11:59:44.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:44.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:44.500 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 11:59:44.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:44.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:44.500 | INFO | Uploaded 36GiB to GPU in 2.03534s (17.6874GiB/s) | 20260506 11:59:44.500 | INFO | Downloaded 34GiB to GPU in 1.92715s (17.6426GiB/s) | 20260506 11:59:49.639 | INFO | Analyzing chunk frames = [1184768,1185280) | 20260506 11:59:49.639 | INFO | MemInfo: { | 20260506 11:59:49.639 | INFO | "gpu" : " 17 GB", | 20260506 11:59:49.639 | INFO | "huge" : "", | 20260506 11:59:49.639 | INFO | "huge_shared" : " 27 GB", | 20260506 11:59:49.639 | INFO | "malloc" : " 14 GB", | 20260506 11:59:49.639 | INFO | "pinned" : " 29 GB" | 20260506 11:59:49.639 | INFO | } | 20260506 11:59:49.639 | INFO | Perf report for the last 5.13911 seconds: | 20260506 11:59:49.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:59:49.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:49.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:49.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:49.639 | INFO | ANALYSIS : 62.0%, 3.0, 96.0ms, 0 | 20260506 11:59:49.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:49.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:49.639 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:59:49.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:49.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:49.639 | INFO | Uploaded 36GiB to GPU in 2.03904s (17.6554GiB/s) | 20260506 11:59:49.639 | INFO | Downloaded 34GiB to GPU in 1.93875s (17.5371GiB/s) | 20260506 11:59:54.779 | INFO | Analyzing chunk frames = [1185280,1185792) | 20260506 11:59:54.779 | INFO | MemInfo: { | 20260506 11:59:54.779 | INFO | "gpu" : " 17 GB", | 20260506 11:59:54.779 | INFO | "huge" : "", | 20260506 11:59:54.779 | INFO | "huge_shared" : " 27 GB", | 20260506 11:59:54.779 | INFO | "malloc" : " 14 GB", | 20260506 11:59:54.779 | INFO | "pinned" : " 29 GB" | 20260506 11:59:54.779 | INFO | } | 20260506 11:59:54.779 | INFO | Perf report for the last 5.13975 seconds: | 20260506 11:59:54.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:59:54.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:54.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:54.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:54.779 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260506 11:59:54.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:54.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:54.779 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 11:59:54.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:54.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:54.779 | INFO | Uploaded 36GiB to GPU in 2.08257s (17.2863GiB/s) | 20260506 11:59:54.779 | INFO | Downloaded 34GiB to GPU in 1.96812s (17.2754GiB/s) | 20260506 11:59:59.918 | INFO | Analyzing chunk frames = [1185792,1186304) | 20260506 11:59:59.918 | INFO | MemInfo: { | 20260506 11:59:59.918 | INFO | "gpu" : " 17 GB", | 20260506 11:59:59.918 | INFO | "huge" : "", | 20260506 11:59:59.918 | INFO | "huge_shared" : " 27 GB", | 20260506 11:59:59.918 | INFO | "malloc" : " 14 GB", | 20260506 11:59:59.918 | INFO | "pinned" : " 29 GB" | 20260506 11:59:59.918 | INFO | } | 20260506 11:59:59.918 | INFO | Perf report for the last 5.13915 seconds: | 20260506 11:59:59.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 11:59:59.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:59.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:59.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:59.918 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 11:59:59.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:59.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:59.918 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 11:59:59.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:59.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 11:59:59.918 | INFO | Uploaded 36GiB to GPU in 2.09883s (17.1524GiB/s) | 20260506 11:59:59.918 | INFO | Downloaded 34GiB to GPU in 1.98669s (17.1139GiB/s) | 20260506 12:00:05.057 | INFO | Analyzing chunk frames = [1186304,1186816) | 20260506 12:00:05.057 | INFO | MemInfo: { | 20260506 12:00:05.057 | INFO | "gpu" : " 17 GB", | 20260506 12:00:05.057 | INFO | "huge" : "", | 20260506 12:00:05.057 | INFO | "huge_shared" : " 27 GB", | 20260506 12:00:05.057 | INFO | "malloc" : " 14 GB", | 20260506 12:00:05.057 | INFO | "pinned" : " 29 GB" | 20260506 12:00:05.057 | INFO | } | 20260506 12:00:05.057 | INFO | Perf report for the last 5.13881 seconds: | 20260506 12:00:05.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:00:05.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:05.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:05.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:05.057 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 12:00:05.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:05.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:05.057 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:00:05.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:05.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:05.057 | INFO | Uploaded 36GiB to GPU in 2.07836s (17.3214GiB/s) | 20260506 12:00:05.057 | INFO | Downloaded 34GiB to GPU in 2.0042s (16.9643GiB/s) | 20260506 12:00:10.196 | INFO | Analyzing chunk frames = [1186816,1187328) | 20260506 12:00:10.196 | INFO | MemInfo: { | 20260506 12:00:10.196 | INFO | "gpu" : " 17 GB", | 20260506 12:00:10.196 | INFO | "huge" : "", | 20260506 12:00:10.196 | INFO | "huge_shared" : " 27 GB", | 20260506 12:00:10.196 | INFO | "malloc" : " 14 GB", | 20260506 12:00:10.196 | INFO | "pinned" : " 29 GB" | 20260506 12:00:10.196 | INFO | } | 20260506 12:00:10.196 | INFO | Perf report for the last 5.13945 seconds: | 20260506 12:00:10.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:00:10.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:10.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:10.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:10.196 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 12:00:10.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:10.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:10.197 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 12:00:10.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:10.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:10.197 | INFO | Uploaded 36GiB to GPU in 1.97978s (18.1838GiB/s) | 20260506 12:00:10.197 | INFO | Downloaded 34GiB to GPU in 2.01677s (16.8586GiB/s) | 20260506 12:00:15.336 | INFO | Analyzing chunk frames = [1187328,1187840) | 20260506 12:00:15.336 | INFO | MemInfo: { | 20260506 12:00:15.336 | INFO | "gpu" : " 17 GB", | 20260506 12:00:15.336 | INFO | "huge" : "", | 20260506 12:00:15.336 | INFO | "huge_shared" : " 27 GB", | 20260506 12:00:15.336 | INFO | "malloc" : " 14 GB", | 20260506 12:00:15.336 | INFO | "pinned" : " 29 GB" | 20260506 12:00:15.336 | INFO | } | 20260506 12:00:15.336 | INFO | Perf report for the last 5.1396 seconds: | 20260506 12:00:15.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:00:15.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:15.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:15.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:15.336 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260506 12:00:15.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:15.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:15.336 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260506 12:00:15.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:15.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:15.336 | INFO | Uploaded 36GiB to GPU in 2.02263s (17.7986GiB/s) | 20260506 12:00:15.336 | INFO | Downloaded 34GiB to GPU in 1.96751s (17.2807GiB/s) | 20260506 12:00:15.336 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:00:15.336 | INFO | Total mem usage: 16.8 GiB | 20260506 12:00:15.336 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:00:15.337 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -4.31%) | 20260506 12:00:15.337 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:00:15.337 | INFO | High Water Breakdown: | 20260506 12:00:15.337 | INFO | 61.3% : 9.88 GiB ( 95.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:00:15.337 | INFO | 38.6% : 6.22 GiB ( 92.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:00:15.337 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:00:15.337 | INFO | ------------------------------------------ | 20260506 12:00:15.337 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:00:15.337 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:00:15.337 | INFO | Total mem usage: 26.5 GiB | 20260506 12:00:15.337 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:00:15.337 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 12:00:15.337 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260506 12:00:15.337 | INFO | High Water Breakdown: | 20260506 12:00:15.337 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:00:15.337 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:00:15.337 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:00:15.337 | INFO | 2% : 523 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:00:15.337 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:00:15.337 | INFO | 0.557% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:00:15.337 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:00:15.337 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:00:15.337 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:00:15.337 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:00:15.337 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:00:15.337 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:00:15.337 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:00:15.337 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:00:15.337 | INFO | ------------------------------------------ | 20260506 12:00:15.337 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:00:15.337 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:00:15.337 | INFO | Total mem usage: 25.6 GiB | 20260506 12:00:15.337 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:00:15.337 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:00:15.337 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:00:15.337 | INFO | High Water Breakdown: | 20260506 12:00:15.337 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:00:15.337 | INFO | ------------------------------------------ | 20260506 12:00:15.337 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:00:15.337 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:00:15.337 | INFO | Total mem usage: 15.7 GiB | 20260506 12:00:15.337 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:00:15.337 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:00:15.337 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:00:15.337 | INFO | High Water Breakdown: | 20260506 12:00:15.337 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:00:15.337 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:00:15.337 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:00:15.337 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:00:15.337 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:00:15.337 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:00:15.337 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:00:15.337 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:00:15.337 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:00:15.337 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:00:15.337 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:00:15.337 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:00:15.337 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:00:15.337 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:00:15.337 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:00:15.337 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:00:15.337 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:00:15.338 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:00:15.338 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:00:15.338 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:00:15.338 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:00:15.338 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:00:15.338 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:00:15.338 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:00:15.338 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:00:15.338 | INFO | ------------------------------------------ | 20260506 12:00:15.338 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:00:15.338 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:00:15.338 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:00:15.338 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:00:15.338 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.61e+01, 1.36e+01 / 2.64e+00 | 20260506 12:00:15.338 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.01e-01, 1.14e-02 / 6.62e-03 | 20260506 12:00:15.338 | INFO | PerfReport: 11.56%, Baseline -- 3.69e+00 / 4.40e+00, 5.90e-01 / 3.58e-01 | 20260506 12:00:15.338 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.46e+00, 1.94e-01 / 1.51e-01 | 20260506 12:00:15.338 | INFO | PerfReport: 15.85%, DME -- 4.04e+01 / 4.49e+01, 4.29e+00 / 2.99e+00 | 20260506 12:00:15.338 | INFO | PerfReport: 56.24%, FrameLabeling -- 1.79e+01 / 2.01e+01, 1.41e+00 / 1.75e+00 | 20260506 12:00:15.338 | INFO | PerfReport: 7.66%, PulseAccumulating -- 2.44e+00 / 3.10e+00, 4.34e-01 / 2.10e-01 | 20260506 12:00:15.338 | INFO | PerfReport: 3.80%, Metrics -- 1.21e+00 / 1.59e+00, 3.08e-01 / 1.15e-01 | 20260506 12:00:15.338 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.28e-01 / 4.87e-01, 6.61e-02 / 2.60e-02 | 20260506 12:00:15.338 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:00:15.338 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:00:15.338 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:00:15.338 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 12:00:15.338 | INFO | PerfReport: 21.61%, Upload -- 2.13e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260506 12:00:15.338 | INFO | PerfReport: 21.19%, Download -- 2.09e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 12:00:15.338 | INFO | PerfReport: 32.33%, Compute -- 3.19e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 12:00:15.338 | INFO | PerfReport: 1.85%, LockUpload -- 1.82e+00 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260506 12:00:15.338 | INFO | PerfReport: 1.01%, LockDownload -- 9.92e-01 / 0.00e+00, 2.64e+00 / 0.00e+00 | 20260506 12:00:15.338 | INFO | PerfReport: 21.99%, LockCompute -- 2.17e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:00:15.338 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260506 12:00:15.465 | INFO | Starting a round of Baz IO | 20260506 12:00:18.631 | INFO | Populating RTMetrics report | 20260506 12:00:18.631 | INFO | RTMetrics report populated | 20260506 12:00:18.631 | INFO | Opening tmp RTMetrics json file | 20260506 12:00:18.633 | INFO | Serializing RTMetrics to json | 20260506 12:00:18.634 | INFO | Serialization done, swapping tmp file now... | 20260506 12:00:18.635 | INFO | RTMetrics done | 20260506 12:00:18.812 | INFO | Finished a round of Baz IO: Wrote 2.08 GiB in 3.346 seconds | 20260506 12:00:18.812 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.62% / 0.03% / 57.36% | 20260506 12:00:18.812 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.47864% / 4.37101e-05% / 3.97364e-05% | 20260506 12:00:18.812 | INFO | Baz IO overall: | 20260506 12:00:18.812 | INFO | BytesWritten : 581775114001 | 20260506 12:00:18.812 | INFO | NumEvents : 281004379216 | 20260506 12:00:20.475 | INFO | Analyzing chunk frames = [1187840,1188352) | 20260506 12:00:20.475 | INFO | MemInfo: { | 20260506 12:00:20.475 | INFO | "gpu" : " 17 GB", | 20260506 12:00:20.475 | INFO | "huge" : "", | 20260506 12:00:20.475 | INFO | "huge_shared" : " 27 GB", | 20260506 12:00:20.475 | INFO | "malloc" : " 14 GB", | 20260506 12:00:20.475 | INFO | "pinned" : " 29 GB" | 20260506 12:00:20.475 | INFO | } | 20260506 12:00:20.475 | INFO | Perf report for the last 5.13919 seconds: | 20260506 12:00:20.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:00:20.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:20.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:20.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:20.475 | INFO | ANALYSIS : 63.5%, 2.9, 95.5ms, 0 | 20260506 12:00:20.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:20.475 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:00:20.475 | INFO | PRE_HQ : 61.8%, 1.9, 61.3ms, 0 | 20260506 12:00:20.475 | INFO | BAZWRITER : 64.5%, 4.7, 154.8ms, 0 | 20260506 12:00:20.475 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 12:00:20.475 | INFO | Uploaded 36GiB to GPU in 2.12528s (16.939GiB/s) | 20260506 12:00:20.475 | INFO | Downloaded 38GiB to GPU in 2.22428s (17.0842GiB/s) | 20260506 12:00:25.614 | INFO | Analyzing chunk frames = [1188352,1188864) | 20260506 12:00:25.614 | INFO | MemInfo: { | 20260506 12:00:25.614 | INFO | "gpu" : " 17 GB", | 20260506 12:00:25.614 | INFO | "huge" : "", | 20260506 12:00:25.615 | INFO | "huge_shared" : " 27 GB", | 20260506 12:00:25.615 | INFO | "malloc" : " 14 GB", | 20260506 12:00:25.615 | INFO | "pinned" : " 29 GB" | 20260506 12:00:25.615 | INFO | } | 20260506 12:00:25.615 | INFO | Perf report for the last 5.13954 seconds: | 20260506 12:00:25.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:00:25.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:25.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:25.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:25.615 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 12:00:25.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:25.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:25.615 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:00:25.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:25.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:25.615 | INFO | Uploaded 36GiB to GPU in 2.01038s (17.9071GiB/s) | 20260506 12:00:25.615 | INFO | Downloaded 34GiB to GPU in 1.91985s (17.7098GiB/s) | 20260506 12:00:30.753 | INFO | Analyzing chunk frames = [1188864,1189376) | 20260506 12:00:30.753 | INFO | MemInfo: { | 20260506 12:00:30.753 | INFO | "gpu" : " 17 GB", | 20260506 12:00:30.753 | INFO | "huge" : "", | 20260506 12:00:30.753 | INFO | "huge_shared" : " 27 GB", | 20260506 12:00:30.753 | INFO | "malloc" : " 14 GB", | 20260506 12:00:30.754 | INFO | "pinned" : " 29 GB" | 20260506 12:00:30.754 | INFO | } | 20260506 12:00:30.754 | INFO | Perf report for the last 5.13904 seconds: | 20260506 12:00:30.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:00:30.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:30.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:30.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:30.754 | INFO | ANALYSIS : 62.0%, 3.0, 95.9ms, 0 | 20260506 12:00:30.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:30.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:30.754 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:00:30.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:30.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:30.754 | INFO | Uploaded 36GiB to GPU in 2.03637s (17.6785GiB/s) | 20260506 12:00:30.754 | INFO | Downloaded 34GiB to GPU in 1.93989s (17.5268GiB/s) | 20260506 12:00:35.893 | INFO | Analyzing chunk frames = [1189376,1189888) | 20260506 12:00:35.893 | INFO | MemInfo: { | 20260506 12:00:35.893 | INFO | "gpu" : " 17 GB", | 20260506 12:00:35.893 | INFO | "huge" : "", | 20260506 12:00:35.893 | INFO | "huge_shared" : " 27 GB", | 20260506 12:00:35.893 | INFO | "malloc" : " 14 GB", | 20260506 12:00:35.893 | INFO | "pinned" : " 29 GB" | 20260506 12:00:35.893 | INFO | } | 20260506 12:00:35.893 | INFO | Perf report for the last 5.13932 seconds: | 20260506 12:00:35.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:00:35.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:35.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:35.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:35.893 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 12:00:35.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:35.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:35.893 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:00:35.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:35.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:35.893 | INFO | Uploaded 36GiB to GPU in 2.0877s (17.2438GiB/s) | 20260506 12:00:35.893 | INFO | Downloaded 34GiB to GPU in 1.97528s (17.2128GiB/s) | 20260506 12:00:41.032 | INFO | Analyzing chunk frames = [1189888,1190400) | 20260506 12:00:41.032 | INFO | MemInfo: { | 20260506 12:00:41.032 | INFO | "gpu" : " 17 GB", | 20260506 12:00:41.032 | INFO | "huge" : "", | 20260506 12:00:41.032 | INFO | "huge_shared" : " 27 GB", | 20260506 12:00:41.033 | INFO | "malloc" : " 14 GB", | 20260506 12:00:41.033 | INFO | "pinned" : " 29 GB" | 20260506 12:00:41.033 | INFO | } | 20260506 12:00:41.033 | INFO | Perf report for the last 5.13954 seconds: | 20260506 12:00:41.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:00:41.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:41.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:41.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:41.033 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 12:00:41.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:41.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:41.033 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:00:41.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:41.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:41.033 | INFO | Uploaded 36GiB to GPU in 1.98875s (18.1018GiB/s) | 20260506 12:00:41.033 | INFO | Downloaded 34GiB to GPU in 1.96942s (17.2639GiB/s) | 20260506 12:00:46.172 | INFO | Analyzing chunk frames = [1190400,1190912) | 20260506 12:00:46.172 | INFO | MemInfo: { | 20260506 12:00:46.172 | INFO | "gpu" : " 17 GB", | 20260506 12:00:46.172 | INFO | "huge" : "", | 20260506 12:00:46.172 | INFO | "huge_shared" : " 27 GB", | 20260506 12:00:46.172 | INFO | "malloc" : " 14 GB", | 20260506 12:00:46.172 | INFO | "pinned" : " 29 GB" | 20260506 12:00:46.172 | INFO | } | 20260506 12:00:46.172 | INFO | Perf report for the last 5.13915 seconds: | 20260506 12:00:46.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:00:46.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:46.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:46.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:46.172 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:00:46.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:46.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:46.172 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:00:46.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:46.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:46.172 | INFO | Uploaded 36GiB to GPU in 2.10249s (17.1226GiB/s) | 20260506 12:00:46.172 | INFO | Downloaded 34GiB to GPU in 1.9762s (17.2047GiB/s) | 20260506 12:00:51.312 | INFO | Analyzing chunk frames = [1190912,1191424) | 20260506 12:00:51.312 | INFO | MemInfo: { | 20260506 12:00:51.312 | INFO | "gpu" : " 17 GB", | 20260506 12:00:51.312 | INFO | "huge" : "", | 20260506 12:00:51.312 | INFO | "huge_shared" : " 27 GB", | 20260506 12:00:51.312 | INFO | "malloc" : " 14 GB", | 20260506 12:00:51.312 | INFO | "pinned" : " 29 GB" | 20260506 12:00:51.312 | INFO | } | 20260506 12:00:51.312 | INFO | Perf report for the last 5.14027 seconds: | 20260506 12:00:51.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:00:51.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:51.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:51.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:51.312 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260506 12:00:51.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:51.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:51.312 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 12:00:51.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:51.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:51.312 | INFO | Uploaded 36GiB to GPU in 2.04519s (17.6022GiB/s) | 20260506 12:00:51.312 | INFO | Downloaded 34GiB to GPU in 2.00507s (16.9571GiB/s) | 20260506 12:00:56.450 | INFO | Analyzing chunk frames = [1191424,1191936) | 20260506 12:00:56.450 | INFO | MemInfo: { | 20260506 12:00:56.450 | INFO | "gpu" : " 17 GB", | 20260506 12:00:56.450 | INFO | "huge" : "", | 20260506 12:00:56.450 | INFO | "huge_shared" : " 27 GB", | 20260506 12:00:56.450 | INFO | "malloc" : " 14 GB", | 20260506 12:00:56.450 | INFO | "pinned" : " 29 GB" | 20260506 12:00:56.450 | INFO | } | 20260506 12:00:56.450 | INFO | Perf report for the last 5.13832 seconds: | 20260506 12:00:56.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:00:56.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:56.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:56.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:56.451 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 12:00:56.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:56.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:56.451 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260506 12:00:56.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:56.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:00:56.451 | INFO | Uploaded 36GiB to GPU in 2.029s (17.7427GiB/s) | 20260506 12:00:56.451 | INFO | Downloaded 34GiB to GPU in 1.95793s (17.3653GiB/s) | 20260506 12:00:56.577 | INFO | Starting a round of Baz IO | 20260506 12:00:59.784 | INFO | Populating RTMetrics report | 20260506 12:00:59.784 | INFO | RTMetrics report populated | 20260506 12:00:59.784 | INFO | Opening tmp RTMetrics json file | 20260506 12:00:59.824 | INFO | Serializing RTMetrics to json | 20260506 12:00:59.861 | INFO | Serialization done, swapping tmp file now... | 20260506 12:00:59.897 | INFO | RTMetrics done | 20260506 12:01:00.239 | INFO | Finished a round of Baz IO: Wrote 2.082 GiB in 3.661 seconds | 20260506 12:01:00.239 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.59% / 0.03% / 57.38% | 20260506 12:01:00.239 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.47357% / 7.15256e-05% / 4.37101e-05% | 20260506 12:01:00.239 | INFO | Baz IO overall: | 20260506 12:01:00.239 | INFO | BytesWritten : 584010137487 | 20260506 12:01:00.239 | INFO | NumEvents : 282089058568 | 20260506 12:01:01.590 | INFO | Analyzing chunk frames = [1191936,1192448) | 20260506 12:01:01.590 | INFO | MemInfo: { | 20260506 12:01:01.590 | INFO | "gpu" : " 17 GB", | 20260506 12:01:01.590 | INFO | "huge" : "", | 20260506 12:01:01.590 | INFO | "huge_shared" : " 27 GB", | 20260506 12:01:01.590 | INFO | "malloc" : " 14 GB", | 20260506 12:01:01.590 | INFO | "pinned" : " 29 GB" | 20260506 12:01:01.590 | INFO | } | 20260506 12:01:01.590 | INFO | Perf report for the last 5.13965 seconds: | 20260506 12:01:01.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:01:01.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:01.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:01.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:01.590 | INFO | ANALYSIS : 63.8%, 2.9, 95.9ms, 0 | 20260506 12:01:01.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:01.590 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260506 12:01:01.590 | INFO | PRE_HQ : 61.5%, 2.0, 62.6ms, 0 | 20260506 12:01:01.590 | INFO | BAZWRITER : 70.4%, 4.8, 173.7ms, 0 | 20260506 12:01:01.590 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 12:01:01.591 | INFO | Uploaded 36GiB to GPU in 2.08595s (17.2583GiB/s) | 20260506 12:01:01.591 | INFO | Downloaded 38GiB to GPU in 2.20139s (17.2619GiB/s) | 20260506 12:01:06.729 | INFO | Analyzing chunk frames = [1192448,1192960) | 20260506 12:01:06.729 | INFO | MemInfo: { | 20260506 12:01:06.729 | INFO | "gpu" : " 17 GB", | 20260506 12:01:06.729 | INFO | "huge" : "", | 20260506 12:01:06.729 | INFO | "huge_shared" : " 27 GB", | 20260506 12:01:06.729 | INFO | "malloc" : " 14 GB", | 20260506 12:01:06.729 | INFO | "pinned" : " 29 GB" | 20260506 12:01:06.729 | INFO | } | 20260506 12:01:06.729 | INFO | Perf report for the last 5.1393 seconds: | 20260506 12:01:06.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:01:06.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:06.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:06.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:06.729 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:01:06.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:06.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:06.730 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 12:01:06.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:06.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:06.730 | INFO | Uploaded 36GiB to GPU in 2.00823s (17.9262GiB/s) | 20260506 12:01:06.730 | INFO | Downloaded 34GiB to GPU in 1.91921s (17.7156GiB/s) | 20260506 12:01:11.868 | INFO | Analyzing chunk frames = [1192960,1193472) | 20260506 12:01:11.868 | INFO | MemInfo: { | 20260506 12:01:11.868 | INFO | "gpu" : " 17 GB", | 20260506 12:01:11.868 | INFO | "huge" : "", | 20260506 12:01:11.869 | INFO | "huge_shared" : " 27 GB", | 20260506 12:01:11.869 | INFO | "malloc" : " 14 GB", | 20260506 12:01:11.869 | INFO | "pinned" : " 29 GB" | 20260506 12:01:11.869 | INFO | } | 20260506 12:01:11.869 | INFO | Perf report for the last 5.13934 seconds: | 20260506 12:01:11.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:01:11.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:11.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:11.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:11.869 | INFO | ANALYSIS : 62.0%, 3.0, 95.9ms, 0 | 20260506 12:01:11.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:11.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:11.869 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:01:11.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:11.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:11.869 | INFO | Uploaded 36GiB to GPU in 2.0392s (17.654GiB/s) | 20260506 12:01:11.869 | INFO | Downloaded 34GiB to GPU in 1.93904s (17.5345GiB/s) | 20260506 12:01:17.008 | INFO | Analyzing chunk frames = [1193472,1193984) | 20260506 12:01:17.008 | INFO | MemInfo: { | 20260506 12:01:17.008 | INFO | "gpu" : " 17 GB", | 20260506 12:01:17.008 | INFO | "huge" : "", | 20260506 12:01:17.008 | INFO | "huge_shared" : " 27 GB", | 20260506 12:01:17.008 | INFO | "malloc" : " 14 GB", | 20260506 12:01:17.008 | INFO | "pinned" : " 29 GB" | 20260506 12:01:17.008 | INFO | } | 20260506 12:01:17.008 | INFO | Perf report for the last 5.13911 seconds: | 20260506 12:01:17.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:01:17.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:17.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:17.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:17.008 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260506 12:01:17.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:17.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:17.008 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:01:17.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:17.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:17.008 | INFO | Uploaded 36GiB to GPU in 2.0824s (17.2877GiB/s) | 20260506 12:01:17.008 | INFO | Downloaded 34GiB to GPU in 1.97645s (17.2025GiB/s) | 20260506 12:01:22.147 | INFO | Analyzing chunk frames = [1193984,1194496) | 20260506 12:01:22.147 | INFO | MemInfo: { | 20260506 12:01:22.147 | INFO | "gpu" : " 17 GB", | 20260506 12:01:22.147 | INFO | "huge" : "", | 20260506 12:01:22.147 | INFO | "huge_shared" : " 27 GB", | 20260506 12:01:22.147 | INFO | "malloc" : " 14 GB", | 20260506 12:01:22.147 | INFO | "pinned" : " 29 GB" | 20260506 12:01:22.147 | INFO | } | 20260506 12:01:22.147 | INFO | Perf report for the last 5.13928 seconds: | 20260506 12:01:22.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:01:22.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:22.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:22.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:22.147 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 12:01:22.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:22.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:22.147 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 12:01:22.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:22.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:22.147 | INFO | Uploaded 36GiB to GPU in 2.08935s (17.2302GiB/s) | 20260506 12:01:22.147 | INFO | Downloaded 34GiB to GPU in 1.98286s (17.1469GiB/s) | 20260506 12:01:27.286 | INFO | Analyzing chunk frames = [1194496,1195008) | 20260506 12:01:27.286 | INFO | MemInfo: { | 20260506 12:01:27.286 | INFO | "gpu" : " 17 GB", | 20260506 12:01:27.286 | INFO | "huge" : "", | 20260506 12:01:27.286 | INFO | "huge_shared" : " 27 GB", | 20260506 12:01:27.287 | INFO | "malloc" : " 14 GB", | 20260506 12:01:27.287 | INFO | "pinned" : " 29 GB" | 20260506 12:01:27.287 | INFO | } | 20260506 12:01:27.287 | INFO | Perf report for the last 5.13949 seconds: | 20260506 12:01:27.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:01:27.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:27.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:27.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:27.287 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 12:01:27.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:27.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:27.287 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 12:01:27.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:27.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:27.287 | INFO | Uploaded 36GiB to GPU in 2.0955s (17.1797GiB/s) | 20260506 12:01:27.287 | INFO | Downloaded 34GiB to GPU in 1.98239s (17.151GiB/s) | 20260506 12:01:32.427 | INFO | Analyzing chunk frames = [1195008,1195520) | 20260506 12:01:32.427 | INFO | MemInfo: { | 20260506 12:01:32.427 | INFO | "gpu" : " 17 GB", | 20260506 12:01:32.427 | INFO | "huge" : "", | 20260506 12:01:32.427 | INFO | "huge_shared" : " 27 GB", | 20260506 12:01:32.427 | INFO | "malloc" : " 14 GB", | 20260506 12:01:32.427 | INFO | "pinned" : " 29 GB" | 20260506 12:01:32.427 | INFO | } | 20260506 12:01:32.427 | INFO | Perf report for the last 5.14045 seconds: | 20260506 12:01:32.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:01:32.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:32.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:32.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:32.427 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:01:32.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:32.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:32.427 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 12:01:32.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:32.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:32.427 | INFO | Uploaded 36GiB to GPU in 2.00939s (17.9159GiB/s) | 20260506 12:01:32.427 | INFO | Downloaded 34GiB to GPU in 2.00539s (16.9543GiB/s) | 20260506 12:01:37.565 | INFO | Analyzing chunk frames = [1195520,1196032) | 20260506 12:01:37.565 | INFO | MemInfo: { | 20260506 12:01:37.565 | INFO | "gpu" : " 17 GB", | 20260506 12:01:37.565 | INFO | "huge" : "", | 20260506 12:01:37.565 | INFO | "huge_shared" : " 27 GB", | 20260506 12:01:37.565 | INFO | "malloc" : " 14 GB", | 20260506 12:01:37.565 | INFO | "pinned" : " 29 GB" | 20260506 12:01:37.565 | INFO | } | 20260506 12:01:37.565 | INFO | Perf report for the last 5.13822 seconds: | 20260506 12:01:37.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:01:37.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:37.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:37.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:37.565 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260506 12:01:37.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:37.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:37.565 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260506 12:01:37.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:37.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:37.566 | INFO | Uploaded 36GiB to GPU in 2.01608s (17.8564GiB/s) | 20260506 12:01:37.566 | INFO | Downloaded 34GiB to GPU in 1.98087s (17.1641GiB/s) | 20260506 12:01:37.566 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:01:37.566 | INFO | Total mem usage: 16.8 GiB | 20260506 12:01:37.566 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:01:37.566 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -5.72%) | 20260506 12:01:37.566 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:01:37.566 | INFO | High Water Breakdown: | 20260506 12:01:37.566 | INFO | 61.1% : 9.71 GiB ( 93.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:01:37.566 | INFO | 38.8% : 6.17 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:01:37.566 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:01:37.566 | INFO | ------------------------------------------ | 20260506 12:01:37.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:01:37.566 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:01:37.566 | INFO | Total mem usage: 26.5 GiB | 20260506 12:01:37.566 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:01:37.566 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:01:37.566 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:01:37.566 | INFO | High Water Breakdown: | 20260506 12:01:37.566 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:01:37.566 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:01:37.566 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:01:37.566 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:01:37.566 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:01:37.566 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:01:37.566 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:01:37.566 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:01:37.566 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:01:37.566 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:01:37.566 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:01:37.566 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:01:37.566 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:01:37.566 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:01:37.566 | INFO | ------------------------------------------ | 20260506 12:01:37.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:01:37.566 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:01:37.566 | INFO | Total mem usage: 25.6 GiB | 20260506 12:01:37.566 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:01:37.566 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:01:37.566 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:01:37.566 | INFO | High Water Breakdown: | 20260506 12:01:37.566 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:01:37.566 | INFO | ------------------------------------------ | 20260506 12:01:37.566 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:01:37.566 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:01:37.566 | INFO | Total mem usage: 15.7 GiB | 20260506 12:01:37.566 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:01:37.566 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:01:37.566 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 12:01:37.566 | INFO | High Water Breakdown: | 20260506 12:01:37.566 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:01:37.567 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:01:37.567 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:01:37.567 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:01:37.567 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:01:37.567 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:01:37.567 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:01:37.567 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:01:37.567 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:01:37.567 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:01:37.567 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:01:37.567 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:01:37.567 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:01:37.567 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:01:37.567 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:01:37.567 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:01:37.567 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:01:37.567 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:01:37.567 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:01:37.567 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:01:37.567 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:01:37.567 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:01:37.567 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:01:37.567 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:01:37.567 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:01:37.567 | INFO | ------------------------------------------ | 20260506 12:01:37.567 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:01:37.567 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:01:37.567 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:01:37.567 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:01:37.567 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.61e+01, 1.36e+01 / 2.51e+00 | 20260506 12:01:37.567 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.00e-01, 1.02e-02 / 6.10e-03 | 20260506 12:01:37.567 | INFO | PerfReport: 11.57%, Baseline -- 3.69e+00 / 4.44e+00, 5.94e-01 / 3.85e-01 | 20260506 12:01:37.567 | INFO | PerfReport: 3.25%, Histogram -- 1.04e+00 / 1.45e+00, 1.93e-01 / 1.45e-01 | 20260506 12:01:37.567 | INFO | PerfReport: 15.82%, DME -- 4.04e+01 / 4.49e+01, 4.15e+00 / 2.53e+00 | 20260506 12:01:37.567 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.80e+01 / 2.01e+01, 1.49e+00 / 1.40e+00 | 20260506 12:01:37.567 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.43e+00 / 3.06e+00, 4.23e-01 / 2.06e-01 | 20260506 12:01:37.567 | INFO | PerfReport: 3.79%, Metrics -- 1.21e+00 / 1.60e+00, 3.10e-01 / 1.81e-01 | 20260506 12:01:37.567 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.27e-01 / 4.86e-01, 6.61e-02 / 2.78e-02 | 20260506 12:01:37.567 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:01:37.567 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:01:37.567 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:01:37.567 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260506 12:01:37.567 | INFO | PerfReport: 21.67%, Upload -- 2.14e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 12:01:37.567 | INFO | PerfReport: 21.08%, Download -- 2.08e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 12:01:37.567 | INFO | PerfReport: 32.31%, Compute -- 3.19e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 12:01:37.567 | INFO | PerfReport: 1.99%, LockUpload -- 1.96e+00 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260506 12:01:37.567 | INFO | PerfReport: 0.86%, LockDownload -- 8.53e-01 / 0.00e+00, 2.37e+00 / 0.00e+00 | 20260506 12:01:37.567 | INFO | PerfReport: 22.07%, LockCompute -- 2.18e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260506 12:01:37.567 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260506 12:01:37.699 | INFO | Starting a round of Baz IO | 20260506 12:01:40.871 | INFO | Populating RTMetrics report | 20260506 12:01:40.871 | INFO | RTMetrics report populated | 20260506 12:01:40.871 | INFO | Opening tmp RTMetrics json file | 20260506 12:01:40.872 | INFO | Serializing RTMetrics to json | 20260506 12:01:40.873 | INFO | Serialization done, swapping tmp file now... | 20260506 12:01:40.873 | INFO | RTMetrics done | 20260506 12:01:40.971 | INFO | Finished a round of Baz IO: Wrote 2.081 GiB in 3.271 seconds | 20260506 12:01:40.971 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.56% / 0.03% / 57.41% | 20260506 12:01:40.971 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.46386% / 2.38419e-05% / 5.16574e-05% | 20260506 12:01:40.971 | INFO | Baz IO overall: | 20260506 12:01:40.971 | INFO | BytesWritten : 586245071089 | 20260506 12:01:40.971 | INFO | NumEvents : 283173526913 | 20260506 12:01:42.704 | INFO | Analyzing chunk frames = [1196032,1196544) | 20260506 12:01:42.704 | INFO | MemInfo: { | 20260506 12:01:42.704 | INFO | "gpu" : " 17 GB", | 20260506 12:01:42.704 | INFO | "huge" : "", | 20260506 12:01:42.704 | INFO | "huge_shared" : " 27 GB", | 20260506 12:01:42.704 | INFO | "malloc" : " 14 GB", | 20260506 12:01:42.704 | INFO | "pinned" : " 29 GB" | 20260506 12:01:42.704 | INFO | } | 20260506 12:01:42.704 | INFO | Perf report for the last 5.13887 seconds: | 20260506 12:01:42.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:01:42.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:42.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:42.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:42.704 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260506 12:01:42.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:42.704 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:01:42.704 | INFO | PRE_HQ : 61.7%, 1.9, 61.7ms, 0 | 20260506 12:01:42.704 | INFO | BAZWRITER : 63.0%, 3.6, 117.4ms, 0 | 20260506 12:01:42.704 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:01:42.705 | INFO | Uploaded 36GiB to GPU in 2.04718s (17.5851GiB/s) | 20260506 12:01:42.705 | INFO | Downloaded 38GiB to GPU in 2.21527s (17.1537GiB/s) | 20260506 12:01:47.844 | INFO | Analyzing chunk frames = [1196544,1197056) | 20260506 12:01:47.844 | INFO | MemInfo: { | 20260506 12:01:47.844 | INFO | "gpu" : " 17 GB", | 20260506 12:01:47.844 | INFO | "huge" : "", | 20260506 12:01:47.844 | INFO | "huge_shared" : " 27 GB", | 20260506 12:01:47.844 | INFO | "malloc" : " 14 GB", | 20260506 12:01:47.844 | INFO | "pinned" : " 29 GB" | 20260506 12:01:47.844 | INFO | } | 20260506 12:01:47.844 | INFO | Perf report for the last 5.13976 seconds: | 20260506 12:01:47.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:01:47.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:47.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:47.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:47.844 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:01:47.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:47.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:47.844 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 12:01:47.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:47.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:47.844 | INFO | Uploaded 36GiB to GPU in 2.00311s (17.972GiB/s) | 20260506 12:01:47.844 | INFO | Downloaded 34GiB to GPU in 1.92385s (17.6729GiB/s) | 20260506 12:01:52.983 | INFO | Analyzing chunk frames = [1197056,1197568) | 20260506 12:01:52.983 | INFO | MemInfo: { | 20260506 12:01:52.983 | INFO | "gpu" : " 17 GB", | 20260506 12:01:52.983 | INFO | "huge" : "", | 20260506 12:01:52.984 | INFO | "huge_shared" : " 27 GB", | 20260506 12:01:52.984 | INFO | "malloc" : " 14 GB", | 20260506 12:01:52.984 | INFO | "pinned" : " 29 GB" | 20260506 12:01:52.984 | INFO | } | 20260506 12:01:52.984 | INFO | Perf report for the last 5.13969 seconds: | 20260506 12:01:52.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:01:52.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:52.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:52.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:52.984 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:01:52.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:52.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:52.984 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:01:52.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:52.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:52.984 | INFO | Uploaded 36GiB to GPU in 1.98528s (18.1335GiB/s) | 20260506 12:01:52.984 | INFO | Downloaded 34GiB to GPU in 1.94553s (17.4759GiB/s) | 20260506 12:01:58.122 | INFO | Analyzing chunk frames = [1197568,1198080) | 20260506 12:01:58.122 | INFO | MemInfo: { | 20260506 12:01:58.122 | INFO | "gpu" : " 17 GB", | 20260506 12:01:58.122 | INFO | "huge" : "", | 20260506 12:01:58.123 | INFO | "huge_shared" : " 27 GB", | 20260506 12:01:58.123 | INFO | "malloc" : " 14 GB", | 20260506 12:01:58.123 | INFO | "pinned" : " 29 GB" | 20260506 12:01:58.123 | INFO | } | 20260506 12:01:58.123 | INFO | Perf report for the last 5.13898 seconds: | 20260506 12:01:58.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:01:58.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:58.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:58.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:58.123 | INFO | ANALYSIS : 61.4%, 3.0, 95.3ms, 0 | 20260506 12:01:58.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:58.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:58.123 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:01:58.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:58.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:01:58.123 | INFO | Uploaded 36GiB to GPU in 2.05633s (17.5069GiB/s) | 20260506 12:01:58.123 | INFO | Downloaded 34GiB to GPU in 1.94967s (17.4388GiB/s) | 20260506 12:02:03.262 | INFO | Analyzing chunk frames = [1198080,1198592) | 20260506 12:02:03.262 | INFO | MemInfo: { | 20260506 12:02:03.262 | INFO | "gpu" : " 17 GB", | 20260506 12:02:03.262 | INFO | "huge" : "", | 20260506 12:02:03.262 | INFO | "huge_shared" : " 27 GB", | 20260506 12:02:03.262 | INFO | "malloc" : " 14 GB", | 20260506 12:02:03.262 | INFO | "pinned" : " 29 GB" | 20260506 12:02:03.262 | INFO | } | 20260506 12:02:03.262 | INFO | Perf report for the last 5.13907 seconds: | 20260506 12:02:03.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:02:03.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:03.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:03.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:03.262 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 12:02:03.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:03.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:03.262 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:02:03.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:03.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:03.262 | INFO | Uploaded 36GiB to GPU in 2.01033s (17.9075GiB/s) | 20260506 12:02:03.262 | INFO | Downloaded 34GiB to GPU in 1.96752s (17.2806GiB/s) | 20260506 12:02:08.401 | INFO | Analyzing chunk frames = [1198592,1199104) | 20260506 12:02:08.401 | INFO | MemInfo: { | 20260506 12:02:08.401 | INFO | "gpu" : " 17 GB", | 20260506 12:02:08.401 | INFO | "huge" : "", | 20260506 12:02:08.401 | INFO | "huge_shared" : " 27 GB", | 20260506 12:02:08.401 | INFO | "malloc" : " 14 GB", | 20260506 12:02:08.401 | INFO | "pinned" : " 29 GB" | 20260506 12:02:08.401 | INFO | } | 20260506 12:02:08.401 | INFO | Perf report for the last 5.13959 seconds: | 20260506 12:02:08.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:02:08.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:08.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:08.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:08.402 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 12:02:08.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:08.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:08.402 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:02:08.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:08.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:08.402 | INFO | Uploaded 36GiB to GPU in 2.05873s (17.4865GiB/s) | 20260506 12:02:08.402 | INFO | Downloaded 34GiB to GPU in 1.93619s (17.5602GiB/s) | 20260506 12:02:13.541 | INFO | Analyzing chunk frames = [1199104,1199616) | 20260506 12:02:13.541 | INFO | MemInfo: { | 20260506 12:02:13.541 | INFO | "gpu" : " 17 GB", | 20260506 12:02:13.541 | INFO | "huge" : "", | 20260506 12:02:13.541 | INFO | "huge_shared" : " 27 GB", | 20260506 12:02:13.541 | INFO | "malloc" : " 14 GB", | 20260506 12:02:13.541 | INFO | "pinned" : " 29 GB" | 20260506 12:02:13.541 | INFO | } | 20260506 12:02:13.541 | INFO | Perf report for the last 5.13949 seconds: | 20260506 12:02:13.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:02:13.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:13.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:13.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:13.541 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:02:13.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:13.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:13.541 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 12:02:13.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:13.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:13.541 | INFO | Uploaded 36GiB to GPU in 1.98601s (18.1268GiB/s) | 20260506 12:02:13.541 | INFO | Downloaded 34GiB to GPU in 2.03098s (16.7407GiB/s) | 20260506 12:02:18.680 | INFO | Analyzing chunk frames = [1199616,1200128) | 20260506 12:02:18.680 | INFO | MemInfo: { | 20260506 12:02:18.680 | INFO | "gpu" : " 17 GB", | 20260506 12:02:18.680 | INFO | "huge" : "", | 20260506 12:02:18.680 | INFO | "huge_shared" : " 27 GB", | 20260506 12:02:18.680 | INFO | "malloc" : " 14 GB", | 20260506 12:02:18.680 | INFO | "pinned" : " 29 GB" | 20260506 12:02:18.680 | INFO | } | 20260506 12:02:18.680 | INFO | Perf report for the last 5.1396 seconds: | 20260506 12:02:18.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:02:18.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:18.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:18.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:18.681 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260506 12:02:18.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:18.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:18.681 | INFO | PRE_HQ : 30.2%, 1.0, 15.9ms, 0 | 20260506 12:02:18.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:18.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:18.681 | INFO | Uploaded 36GiB to GPU in 2.02004s (17.8214GiB/s) | 20260506 12:02:18.681 | INFO | Downloaded 34GiB to GPU in 2.04856s (16.597GiB/s) | 20260506 12:02:18.812 | INFO | Starting a round of Baz IO | 20260506 12:02:21.977 | INFO | Populating RTMetrics report | 20260506 12:02:21.977 | INFO | RTMetrics report populated | 20260506 12:02:21.977 | INFO | Opening tmp RTMetrics json file | 20260506 12:02:21.978 | INFO | Serializing RTMetrics to json | 20260506 12:02:21.979 | INFO | Serialization done, swapping tmp file now... | 20260506 12:02:21.979 | INFO | RTMetrics done | 20260506 12:02:22.078 | INFO | Finished a round of Baz IO: Wrote 2.084 GiB in 3.266 seconds | 20260506 12:02:22.078 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.53% / 0.03% / 57.44% | 20260506 12:02:22.078 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.46247% / 5.16574e-05% / 3.97364e-05% | 20260506 12:02:22.078 | INFO | Baz IO overall: | 20260506 12:02:22.078 | INFO | BytesWritten : 588483209246 | 20260506 12:02:22.078 | INFO | NumEvents : 284260334731 | 20260506 12:02:23.819 | INFO | Analyzing chunk frames = [1200128,1200640) | 20260506 12:02:23.819 | INFO | MemInfo: { | 20260506 12:02:23.819 | INFO | "gpu" : " 17 GB", | 20260506 12:02:23.819 | INFO | "huge" : "", | 20260506 12:02:23.819 | INFO | "huge_shared" : " 27 GB", | 20260506 12:02:23.819 | INFO | "malloc" : " 14 GB", | 20260506 12:02:23.819 | INFO | "pinned" : " 29 GB" | 20260506 12:02:23.819 | INFO | } | 20260506 12:02:23.820 | INFO | Perf report for the last 5.139 seconds: | 20260506 12:02:23.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:02:23.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:23.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:23.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:23.820 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260506 12:02:23.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:23.820 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:02:23.820 | INFO | PRE_HQ : 61.4%, 1.9, 62.0ms, 0 | 20260506 12:02:23.820 | INFO | BAZWRITER : 62.9%, 3.7, 118.5ms, 0 | 20260506 12:02:23.820 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:02:23.820 | INFO | Uploaded 36GiB to GPU in 2.07663s (17.3357GiB/s) | 20260506 12:02:23.820 | INFO | Downloaded 38GiB to GPU in 2.21103s (17.1866GiB/s) | 20260506 12:02:28.959 | INFO | Analyzing chunk frames = [1200640,1201152) | 20260506 12:02:28.959 | INFO | MemInfo: { | 20260506 12:02:28.959 | INFO | "gpu" : " 17 GB", | 20260506 12:02:28.959 | INFO | "huge" : "", | 20260506 12:02:28.959 | INFO | "huge_shared" : " 27 GB", | 20260506 12:02:28.959 | INFO | "malloc" : " 14 GB", | 20260506 12:02:28.959 | INFO | "pinned" : " 29 GB" | 20260506 12:02:28.959 | INFO | } | 20260506 12:02:28.959 | INFO | Perf report for the last 5.1397 seconds: | 20260506 12:02:28.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:02:28.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:28.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:28.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:28.959 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 12:02:28.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:28.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:28.959 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:02:28.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:28.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:28.960 | INFO | Uploaded 36GiB to GPU in 2.0299s (17.7349GiB/s) | 20260506 12:02:28.960 | INFO | Downloaded 34GiB to GPU in 1.92191s (17.6908GiB/s) | 20260506 12:02:34.098 | INFO | Analyzing chunk frames = [1201152,1201664) | 20260506 12:02:34.098 | INFO | MemInfo: { | 20260506 12:02:34.098 | INFO | "gpu" : " 17 GB", | 20260506 12:02:34.098 | INFO | "huge" : "", | 20260506 12:02:34.098 | INFO | "huge_shared" : " 27 GB", | 20260506 12:02:34.098 | INFO | "malloc" : " 14 GB", | 20260506 12:02:34.098 | INFO | "pinned" : " 29 GB" | 20260506 12:02:34.098 | INFO | } | 20260506 12:02:34.098 | INFO | Perf report for the last 5.13881 seconds: | 20260506 12:02:34.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:02:34.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:34.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:34.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:34.098 | INFO | ANALYSIS : 62.0%, 3.0, 96.0ms, 0 | 20260506 12:02:34.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:34.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:34.098 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:02:34.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:34.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:34.098 | INFO | Uploaded 36GiB to GPU in 2.04064s (17.6415GiB/s) | 20260506 12:02:34.098 | INFO | Downloaded 34GiB to GPU in 1.9335s (17.5847GiB/s) | 20260506 12:02:39.237 | INFO | Analyzing chunk frames = [1201664,1202176) | 20260506 12:02:39.237 | INFO | MemInfo: { | 20260506 12:02:39.237 | INFO | "gpu" : " 17 GB", | 20260506 12:02:39.237 | INFO | "huge" : "", | 20260506 12:02:39.237 | INFO | "huge_shared" : " 27 GB", | 20260506 12:02:39.237 | INFO | "malloc" : " 14 GB", | 20260506 12:02:39.238 | INFO | "pinned" : " 29 GB" | 20260506 12:02:39.238 | INFO | } | 20260506 12:02:39.238 | INFO | Perf report for the last 5.13962 seconds: | 20260506 12:02:39.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:02:39.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:39.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:39.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:39.238 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260506 12:02:39.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:39.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:39.238 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:02:39.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:39.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:39.238 | INFO | Uploaded 36GiB to GPU in 2.08421s (17.2727GiB/s) | 20260506 12:02:39.238 | INFO | Downloaded 34GiB to GPU in 1.96881s (17.2693GiB/s) | 20260506 12:02:44.377 | INFO | Analyzing chunk frames = [1202176,1202688) | 20260506 12:02:44.377 | INFO | MemInfo: { | 20260506 12:02:44.377 | INFO | "gpu" : " 17 GB", | 20260506 12:02:44.377 | INFO | "huge" : "", | 20260506 12:02:44.377 | INFO | "huge_shared" : " 27 GB", | 20260506 12:02:44.377 | INFO | "malloc" : " 14 GB", | 20260506 12:02:44.377 | INFO | "pinned" : " 29 GB" | 20260506 12:02:44.377 | INFO | } | 20260506 12:02:44.377 | INFO | Perf report for the last 5.1395 seconds: | 20260506 12:02:44.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:02:44.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:44.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:44.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:44.377 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 12:02:44.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:44.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:44.377 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 12:02:44.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:44.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:44.377 | INFO | Uploaded 36GiB to GPU in 2.10808s (17.0771GiB/s) | 20260506 12:02:44.378 | INFO | Downloaded 34GiB to GPU in 1.98499s (17.1286GiB/s) | 20260506 12:02:49.516 | INFO | Analyzing chunk frames = [1202688,1203200) | 20260506 12:02:49.516 | INFO | MemInfo: { | 20260506 12:02:49.516 | INFO | "gpu" : " 17 GB", | 20260506 12:02:49.516 | INFO | "huge" : "", | 20260506 12:02:49.516 | INFO | "huge_shared" : " 27 GB", | 20260506 12:02:49.516 | INFO | "malloc" : " 14 GB", | 20260506 12:02:49.516 | INFO | "pinned" : " 29 GB" | 20260506 12:02:49.516 | INFO | } | 20260506 12:02:49.516 | INFO | Perf report for the last 5.1391 seconds: | 20260506 12:02:49.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:02:49.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:49.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:49.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:49.516 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 12:02:49.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:49.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:49.516 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:02:49.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:49.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:49.517 | INFO | Uploaded 36GiB to GPU in 1.97319s (18.2446GiB/s) | 20260506 12:02:49.517 | INFO | Downloaded 34GiB to GPU in 1.9919s (17.0691GiB/s) | 20260506 12:02:54.655 | INFO | Analyzing chunk frames = [1203200,1203712) | 20260506 12:02:54.655 | INFO | MemInfo: { | 20260506 12:02:54.655 | INFO | "gpu" : " 17 GB", | 20260506 12:02:54.655 | INFO | "huge" : "", | 20260506 12:02:54.655 | INFO | "huge_shared" : " 27 GB", | 20260506 12:02:54.655 | INFO | "malloc" : " 14 GB", | 20260506 12:02:54.655 | INFO | "pinned" : " 29 GB" | 20260506 12:02:54.655 | INFO | } | 20260506 12:02:54.655 | INFO | Perf report for the last 5.13902 seconds: | 20260506 12:02:54.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:02:54.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:54.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:54.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:54.655 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:02:54.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:54.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:54.655 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 12:02:54.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:54.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:54.655 | INFO | Uploaded 36GiB to GPU in 2.01171s (17.8953GiB/s) | 20260506 12:02:54.656 | INFO | Downloaded 34GiB to GPU in 2.03282s (16.7256GiB/s) | 20260506 12:02:59.794 | INFO | Analyzing chunk frames = [1203712,1204224) | 20260506 12:02:59.795 | INFO | MemInfo: { | 20260506 12:02:59.795 | INFO | "gpu" : " 17 GB", | 20260506 12:02:59.795 | INFO | "huge" : "", | 20260506 12:02:59.795 | INFO | "huge_shared" : " 27 GB", | 20260506 12:02:59.795 | INFO | "malloc" : " 14 GB", | 20260506 12:02:59.795 | INFO | "pinned" : " 29 GB" | 20260506 12:02:59.795 | INFO | } | 20260506 12:02:59.795 | INFO | Perf report for the last 5.13936 seconds: | 20260506 12:02:59.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:02:59.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:59.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:59.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:59.795 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260506 12:02:59.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:59.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:59.795 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260506 12:02:59.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:59.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:02:59.795 | INFO | Uploaded 36GiB to GPU in 2.00733s (17.9343GiB/s) | 20260506 12:02:59.795 | INFO | Downloaded 34GiB to GPU in 1.98853s (17.098GiB/s) | 20260506 12:02:59.795 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:02:59.795 | INFO | Total mem usage: 16.8 GiB | 20260506 12:02:59.795 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:02:59.795 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.7%) | 20260506 12:02:59.795 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:02:59.795 | INFO | High Water Breakdown: | 20260506 12:02:59.795 | INFO | 59.5% : 8.57 GiB ( 82.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:02:59.795 | INFO | 40.3% : 5.81 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:02:59.795 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:02:59.795 | INFO | ------------------------------------------ | 20260506 12:02:59.795 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:02:59.795 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:02:59.795 | INFO | Total mem usage: 26.5 GiB | 20260506 12:02:59.795 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:02:59.795 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:02:59.795 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:02:59.795 | INFO | High Water Breakdown: | 20260506 12:02:59.795 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:02:59.795 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:02:59.795 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:02:59.795 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:02:59.795 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:02:59.795 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:02:59.795 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:02:59.795 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:02:59.795 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:02:59.795 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:02:59.795 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:02:59.795 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:02:59.795 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:02:59.795 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:02:59.795 | INFO | ------------------------------------------ | 20260506 12:02:59.795 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:02:59.796 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:02:59.796 | INFO | Total mem usage: 25.6 GiB | 20260506 12:02:59.796 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:02:59.796 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:02:59.796 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:02:59.796 | INFO | High Water Breakdown: | 20260506 12:02:59.796 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:02:59.796 | INFO | ------------------------------------------ | 20260506 12:02:59.796 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:02:59.796 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:02:59.796 | INFO | Total mem usage: 15.7 GiB | 20260506 12:02:59.796 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:02:59.796 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:02:59.796 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:02:59.796 | INFO | High Water Breakdown: | 20260506 12:02:59.796 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:02:59.796 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:02:59.796 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:02:59.796 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:02:59.796 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:02:59.796 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:02:59.796 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:02:59.796 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:02:59.796 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:02:59.796 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:02:59.796 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:02:59.796 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:02:59.796 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:02:59.796 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:02:59.796 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:02:59.796 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:02:59.796 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:02:59.796 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:02:59.796 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:02:59.796 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:02:59.796 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:02:59.796 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:02:59.796 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:02:59.796 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:02:59.796 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:02:59.796 | INFO | ------------------------------------------ | 20260506 12:02:59.796 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:02:59.796 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:02:59.796 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:02:59.796 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:02:59.796 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.66e+01, 1.36e+01 / 2.83e+00 | 20260506 12:02:59.796 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 9.97e-02, 1.02e-02 / 6.53e-03 | 20260506 12:02:59.796 | INFO | PerfReport: 11.56%, Baseline -- 3.69e+00 / 4.42e+00, 5.96e-01 / 3.53e-01 | 20260506 12:02:59.796 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.43e+00, 1.94e-01 / 1.54e-01 | 20260506 12:02:59.796 | INFO | PerfReport: 15.87%, DME -- 4.05e+01 / 4.52e+01, 4.34e+00 / 2.91e+00 | 20260506 12:02:59.796 | INFO | PerfReport: 56.29%, FrameLabeling -- 1.80e+01 / 2.03e+01, 1.47e+00 / 1.47e+00 | 20260506 12:02:59.796 | INFO | PerfReport: 7.61%, PulseAccumulating -- 2.43e+00 / 3.06e+00, 4.27e-01 / 2.42e-01 | 20260506 12:02:59.796 | INFO | PerfReport: 3.80%, Metrics -- 1.21e+00 / 1.61e+00, 3.10e-01 / 1.68e-01 | 20260506 12:02:59.797 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.28e-01 / 4.87e-01, 6.63e-02 / 2.77e-02 | 20260506 12:02:59.797 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:02:59.797 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:02:59.797 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:02:59.797 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 12:02:59.797 | INFO | PerfReport: 21.36%, Upload -- 2.12e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260506 12:02:59.797 | INFO | PerfReport: 21.07%, Download -- 2.09e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 12:02:59.797 | INFO | PerfReport: 32.23%, Compute -- 3.19e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 12:02:59.797 | INFO | PerfReport: 1.80%, LockUpload -- 1.78e+00 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 12:02:59.797 | INFO | PerfReport: 1.01%, LockDownload -- 1.00e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260506 12:02:59.797 | INFO | PerfReport: 22.51%, LockCompute -- 2.23e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 12:02:59.797 | INFO | PerfReport: 0.02%, Misc -- 1.51e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260506 12:02:59.927 | INFO | Starting a round of Baz IO | 20260506 12:03:03.102 | INFO | Populating RTMetrics report | 20260506 12:03:03.103 | INFO | RTMetrics report populated | 20260506 12:03:03.103 | INFO | Opening tmp RTMetrics json file | 20260506 12:03:03.103 | INFO | Serializing RTMetrics to json | 20260506 12:03:03.105 | INFO | Serialization done, swapping tmp file now... | 20260506 12:03:03.105 | INFO | RTMetrics done | 20260506 12:03:03.204 | INFO | Finished a round of Baz IO: Wrote 2.082 GiB in 3.276 seconds | 20260506 12:03:03.204 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.50% / 0.03% / 57.47% | 20260506 12:03:03.204 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.43247% / 4.76837e-05% / 3.57628e-05% | 20260506 12:03:03.204 | INFO | Baz IO overall: | 20260506 12:03:03.204 | INFO | BytesWritten : 590718957876 | 20260506 12:03:03.204 | INFO | NumEvents : 285345302256 | 20260506 12:03:04.934 | INFO | Analyzing chunk frames = [1204224,1204736) | 20260506 12:03:04.934 | INFO | MemInfo: { | 20260506 12:03:04.934 | INFO | "gpu" : " 17 GB", | 20260506 12:03:04.934 | INFO | "huge" : "", | 20260506 12:03:04.934 | INFO | "huge_shared" : " 27 GB", | 20260506 12:03:04.934 | INFO | "malloc" : " 14 GB", | 20260506 12:03:04.934 | INFO | "pinned" : " 29 GB" | 20260506 12:03:04.934 | INFO | } | 20260506 12:03:04.934 | INFO | Perf report for the last 5.13926 seconds: | 20260506 12:03:04.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:03:04.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:04.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:04.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:04.934 | INFO | ANALYSIS : 63.6%, 2.9, 98.3ms, 0 | 20260506 12:03:04.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:04.934 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 12:03:04.934 | INFO | PRE_HQ : 62.0%, 2.0, 64.2ms, 0 | 20260506 12:03:04.934 | INFO | BAZWRITER : 63.1%, 3.8, 122.8ms, 0 | 20260506 12:03:04.934 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 12:03:04.934 | INFO | Uploaded 36GiB to GPU in 2.08355s (17.2782GiB/s) | 20260506 12:03:04.934 | INFO | Downloaded 38GiB to GPU in 2.22157s (17.1051GiB/s) | 20260506 12:03:10.073 | INFO | Analyzing chunk frames = [1204736,1205248) | 20260506 12:03:10.073 | INFO | MemInfo: { | 20260506 12:03:10.073 | INFO | "gpu" : " 17 GB", | 20260506 12:03:10.073 | INFO | "huge" : "", | 20260506 12:03:10.073 | INFO | "huge_shared" : " 27 GB", | 20260506 12:03:10.073 | INFO | "malloc" : " 14 GB", | 20260506 12:03:10.073 | INFO | "pinned" : " 29 GB" | 20260506 12:03:10.073 | INFO | } | 20260506 12:03:10.074 | INFO | Perf report for the last 5.13961 seconds: | 20260506 12:03:10.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:03:10.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:10.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:10.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:10.074 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 12:03:10.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:10.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:10.074 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 12:03:10.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:10.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:10.074 | INFO | Uploaded 36GiB to GPU in 2.00906s (17.9188GiB/s) | 20260506 12:03:10.074 | INFO | Downloaded 34GiB to GPU in 1.9391s (17.5339GiB/s) | 20260506 12:03:15.213 | INFO | Analyzing chunk frames = [1205248,1205760) | 20260506 12:03:15.213 | INFO | MemInfo: { | 20260506 12:03:15.213 | INFO | "gpu" : " 17 GB", | 20260506 12:03:15.213 | INFO | "huge" : "", | 20260506 12:03:15.213 | INFO | "huge_shared" : " 27 GB", | 20260506 12:03:15.213 | INFO | "malloc" : " 14 GB", | 20260506 12:03:15.213 | INFO | "pinned" : " 29 GB" | 20260506 12:03:15.213 | INFO | } | 20260506 12:03:15.213 | INFO | Perf report for the last 5.13918 seconds: | 20260506 12:03:15.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:03:15.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:15.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:15.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:15.213 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260506 12:03:15.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:15.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:15.213 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 12:03:15.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:15.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:15.213 | INFO | Uploaded 36GiB to GPU in 2.01836s (17.8363GiB/s) | 20260506 12:03:15.213 | INFO | Downloaded 34GiB to GPU in 1.94496s (17.4811GiB/s) | 20260506 12:03:20.352 | INFO | Analyzing chunk frames = [1205760,1206272) | 20260506 12:03:20.352 | INFO | MemInfo: { | 20260506 12:03:20.352 | INFO | "gpu" : " 17 GB", | 20260506 12:03:20.352 | INFO | "huge" : "", | 20260506 12:03:20.352 | INFO | "huge_shared" : " 27 GB", | 20260506 12:03:20.352 | INFO | "malloc" : " 14 GB", | 20260506 12:03:20.352 | INFO | "pinned" : " 29 GB" | 20260506 12:03:20.352 | INFO | } | 20260506 12:03:20.352 | INFO | Perf report for the last 5.13936 seconds: | 20260506 12:03:20.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:03:20.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:20.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:20.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:20.352 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 12:03:20.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:20.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:20.352 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:03:20.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:20.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:20.352 | INFO | Uploaded 36GiB to GPU in 2.05587s (17.5109GiB/s) | 20260506 12:03:20.353 | INFO | Downloaded 34GiB to GPU in 1.97103s (17.2499GiB/s) | 20260506 12:03:25.491 | INFO | Analyzing chunk frames = [1206272,1206784) | 20260506 12:03:25.491 | INFO | MemInfo: { | 20260506 12:03:25.491 | INFO | "gpu" : " 17 GB", | 20260506 12:03:25.491 | INFO | "huge" : "", | 20260506 12:03:25.491 | INFO | "huge_shared" : " 27 GB", | 20260506 12:03:25.491 | INFO | "malloc" : " 14 GB", | 20260506 12:03:25.492 | INFO | "pinned" : " 29 GB" | 20260506 12:03:25.492 | INFO | } | 20260506 12:03:25.492 | INFO | Perf report for the last 5.13947 seconds: | 20260506 12:03:25.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:03:25.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:25.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:25.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:25.492 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 12:03:25.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:25.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:25.492 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:03:25.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:25.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:25.492 | INFO | Uploaded 36GiB to GPU in 2.02365s (17.7896GiB/s) | 20260506 12:03:25.492 | INFO | Downloaded 34GiB to GPU in 1.95785s (17.366GiB/s) | 20260506 12:03:30.631 | INFO | Analyzing chunk frames = [1206784,1207296) | 20260506 12:03:30.631 | INFO | MemInfo: { | 20260506 12:03:30.631 | INFO | "gpu" : " 17 GB", | 20260506 12:03:30.631 | INFO | "huge" : "", | 20260506 12:03:30.631 | INFO | "huge_shared" : " 27 GB", | 20260506 12:03:30.631 | INFO | "malloc" : " 14 GB", | 20260506 12:03:30.631 | INFO | "pinned" : " 29 GB" | 20260506 12:03:30.631 | INFO | } | 20260506 12:03:30.631 | INFO | Perf report for the last 5.13914 seconds: | 20260506 12:03:30.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:03:30.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:30.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:30.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:30.631 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 12:03:30.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:30.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:30.631 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:03:30.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:30.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:30.631 | INFO | Uploaded 36GiB to GPU in 2.01694s (17.8489GiB/s) | 20260506 12:03:30.631 | INFO | Downloaded 34GiB to GPU in 1.9092s (17.8085GiB/s) | 20260506 12:03:35.771 | INFO | Analyzing chunk frames = [1207296,1207808) | 20260506 12:03:35.771 | INFO | MemInfo: { | 20260506 12:03:35.771 | INFO | "gpu" : " 17 GB", | 20260506 12:03:35.771 | INFO | "huge" : "", | 20260506 12:03:35.771 | INFO | "huge_shared" : " 27 GB", | 20260506 12:03:35.771 | INFO | "malloc" : " 14 GB", | 20260506 12:03:35.771 | INFO | "pinned" : " 29 GB" | 20260506 12:03:35.771 | INFO | } | 20260506 12:03:35.771 | INFO | Perf report for the last 5.14068 seconds: | 20260506 12:03:35.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:03:35.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:35.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:35.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:35.772 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:03:35.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:35.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:35.772 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 12:03:35.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:35.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:35.772 | INFO | Uploaded 36GiB to GPU in 1.99716s (18.0256GiB/s) | 20260506 12:03:35.772 | INFO | Downloaded 34GiB to GPU in 2.03452s (16.7115GiB/s) | 20260506 12:03:40.909 | INFO | Analyzing chunk frames = [1207808,1208320) | 20260506 12:03:40.909 | INFO | MemInfo: { | 20260506 12:03:40.909 | INFO | "gpu" : " 17 GB", | 20260506 12:03:40.909 | INFO | "huge" : "", | 20260506 12:03:40.909 | INFO | "huge_shared" : " 27 GB", | 20260506 12:03:40.910 | INFO | "malloc" : " 14 GB", | 20260506 12:03:40.910 | INFO | "pinned" : " 29 GB" | 20260506 12:03:40.910 | INFO | } | 20260506 12:03:40.910 | INFO | Perf report for the last 5.13824 seconds: | 20260506 12:03:40.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:03:40.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:40.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:40.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:40.910 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 12:03:40.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:40.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:40.910 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260506 12:03:40.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:40.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:40.910 | INFO | Uploaded 36GiB to GPU in 2.00083s (17.9925GiB/s) | 20260506 12:03:40.910 | INFO | Downloaded 34GiB to GPU in 1.99934s (17.0056GiB/s) | 20260506 12:03:41.040 | INFO | Starting a round of Baz IO | 20260506 12:03:44.217 | INFO | Populating RTMetrics report | 20260506 12:03:44.217 | INFO | RTMetrics report populated | 20260506 12:03:44.217 | INFO | Opening tmp RTMetrics json file | 20260506 12:03:44.218 | INFO | Serializing RTMetrics to json | 20260506 12:03:44.219 | INFO | Serialization done, swapping tmp file now... | 20260506 12:03:44.220 | INFO | RTMetrics done | 20260506 12:03:44.324 | INFO | Finished a round of Baz IO: Wrote 2.083 GiB in 3.283 seconds | 20260506 12:03:44.324 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.47% / 0.03% / 57.50% | 20260506 12:03:44.324 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.42204% / 3.17891e-05% / 5.5631e-05% | 20260506 12:03:44.324 | INFO | Baz IO overall: | 20260506 12:03:44.324 | INFO | BytesWritten : 592955061394 | 20260506 12:03:44.324 | INFO | NumEvents : 286430276193 | 20260506 12:03:46.049 | INFO | Analyzing chunk frames = [1208320,1208832) | 20260506 12:03:46.049 | INFO | MemInfo: { | 20260506 12:03:46.049 | INFO | "gpu" : " 17 GB", | 20260506 12:03:46.049 | INFO | "huge" : "", | 20260506 12:03:46.049 | INFO | "huge_shared" : " 27 GB", | 20260506 12:03:46.049 | INFO | "malloc" : " 14 GB", | 20260506 12:03:46.049 | INFO | "pinned" : " 29 GB" | 20260506 12:03:46.049 | INFO | } | 20260506 12:03:46.049 | INFO | Perf report for the last 5.13934 seconds: | 20260506 12:03:46.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:03:46.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:46.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:46.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:46.049 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260506 12:03:46.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:46.049 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 12:03:46.049 | INFO | PRE_HQ : 61.9%, 2.0, 62.8ms, 0 | 20260506 12:03:46.049 | INFO | BAZWRITER : 63.3%, 3.7, 119.9ms, 0 | 20260506 12:03:46.049 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 12:03:46.049 | INFO | Uploaded 36GiB to GPU in 2.07931s (17.3134GiB/s) | 20260506 12:03:46.049 | INFO | Downloaded 38GiB to GPU in 2.22423s (17.0846GiB/s) | 20260506 12:03:51.188 | INFO | Analyzing chunk frames = [1208832,1209344) | 20260506 12:03:51.188 | INFO | MemInfo: { | 20260506 12:03:51.188 | INFO | "gpu" : " 17 GB", | 20260506 12:03:51.188 | INFO | "huge" : "", | 20260506 12:03:51.188 | INFO | "huge_shared" : " 27 GB", | 20260506 12:03:51.188 | INFO | "malloc" : " 14 GB", | 20260506 12:03:51.188 | INFO | "pinned" : " 29 GB" | 20260506 12:03:51.188 | INFO | } | 20260506 12:03:51.188 | INFO | Perf report for the last 5.13917 seconds: | 20260506 12:03:51.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:03:51.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:51.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:51.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:51.188 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 12:03:51.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:51.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:51.188 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 12:03:51.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:51.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:51.189 | INFO | Uploaded 36GiB to GPU in 2.01756s (17.8434GiB/s) | 20260506 12:03:51.189 | INFO | Downloaded 34GiB to GPU in 1.93794s (17.5444GiB/s) | 20260506 12:03:56.327 | INFO | Analyzing chunk frames = [1209344,1209856) | 20260506 12:03:56.328 | INFO | MemInfo: { | 20260506 12:03:56.328 | INFO | "gpu" : " 17 GB", | 20260506 12:03:56.328 | INFO | "huge" : "", | 20260506 12:03:56.328 | INFO | "huge_shared" : " 27 GB", | 20260506 12:03:56.328 | INFO | "malloc" : " 14 GB", | 20260506 12:03:56.328 | INFO | "pinned" : " 29 GB" | 20260506 12:03:56.328 | INFO | } | 20260506 12:03:56.328 | INFO | Perf report for the last 5.13942 seconds: | 20260506 12:03:56.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:03:56.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:56.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:56.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:56.328 | INFO | ANALYSIS : 61.9%, 3.0, 95.9ms, 0 | 20260506 12:03:56.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:56.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:56.328 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 12:03:56.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:56.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:03:56.328 | INFO | Uploaded 36GiB to GPU in 2.03145s (17.7214GiB/s) | 20260506 12:03:56.328 | INFO | Downloaded 34GiB to GPU in 1.94546s (17.4766GiB/s) | 20260506 12:04:01.467 | INFO | Analyzing chunk frames = [1209856,1210368) | 20260506 12:04:01.467 | INFO | MemInfo: { | 20260506 12:04:01.467 | INFO | "gpu" : " 17 GB", | 20260506 12:04:01.467 | INFO | "huge" : "", | 20260506 12:04:01.467 | INFO | "huge_shared" : " 27 GB", | 20260506 12:04:01.467 | INFO | "malloc" : " 14 GB", | 20260506 12:04:01.467 | INFO | "pinned" : " 29 GB" | 20260506 12:04:01.467 | INFO | } | 20260506 12:04:01.467 | INFO | Perf report for the last 5.13913 seconds: | 20260506 12:04:01.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:04:01.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:01.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:01.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:01.467 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 12:04:01.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:01.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:01.467 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 12:04:01.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:01.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:01.467 | INFO | Uploaded 36GiB to GPU in 2.05839s (17.4894GiB/s) | 20260506 12:04:01.467 | INFO | Downloaded 34GiB to GPU in 1.97646s (17.2025GiB/s) | 20260506 12:04:06.606 | INFO | Analyzing chunk frames = [1210368,1210880) | 20260506 12:04:06.606 | INFO | MemInfo: { | 20260506 12:04:06.606 | INFO | "gpu" : " 17 GB", | 20260506 12:04:06.606 | INFO | "huge" : "", | 20260506 12:04:06.606 | INFO | "huge_shared" : " 27 GB", | 20260506 12:04:06.606 | INFO | "malloc" : " 14 GB", | 20260506 12:04:06.606 | INFO | "pinned" : " 29 GB" | 20260506 12:04:06.606 | INFO | } | 20260506 12:04:06.606 | INFO | Perf report for the last 5.1394 seconds: | 20260506 12:04:06.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:04:06.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:06.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:06.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:06.606 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 12:04:06.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:06.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:06.606 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 12:04:06.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:06.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:06.607 | INFO | Uploaded 36GiB to GPU in 2.0319s (17.7174GiB/s) | 20260506 12:04:06.607 | INFO | Downloaded 34GiB to GPU in 1.9818s (17.1561GiB/s) | 20260506 12:04:11.746 | INFO | Analyzing chunk frames = [1210880,1211392) | 20260506 12:04:11.746 | INFO | MemInfo: { | 20260506 12:04:11.746 | INFO | "gpu" : " 17 GB", | 20260506 12:04:11.746 | INFO | "huge" : "", | 20260506 12:04:11.746 | INFO | "huge_shared" : " 27 GB", | 20260506 12:04:11.746 | INFO | "malloc" : " 14 GB", | 20260506 12:04:11.746 | INFO | "pinned" : " 29 GB" | 20260506 12:04:11.746 | INFO | } | 20260506 12:04:11.746 | INFO | Perf report for the last 5.13964 seconds: | 20260506 12:04:11.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:04:11.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:11.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:11.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:11.746 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:04:11.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:11.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:11.746 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:04:11.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:11.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:11.746 | INFO | Uploaded 36GiB to GPU in 2.106s (17.094GiB/s) | 20260506 12:04:11.746 | INFO | Downloaded 34GiB to GPU in 1.98436s (17.134GiB/s) | 20260506 12:04:16.886 | INFO | Analyzing chunk frames = [1211392,1211904) | 20260506 12:04:16.886 | INFO | MemInfo: { | 20260506 12:04:16.886 | INFO | "gpu" : " 17 GB", | 20260506 12:04:16.886 | INFO | "huge" : "", | 20260506 12:04:16.886 | INFO | "huge_shared" : " 27 GB", | 20260506 12:04:16.886 | INFO | "malloc" : " 14 GB", | 20260506 12:04:16.886 | INFO | "pinned" : " 29 GB" | 20260506 12:04:16.886 | INFO | } | 20260506 12:04:16.886 | INFO | Perf report for the last 5.14014 seconds: | 20260506 12:04:16.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:04:16.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:16.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:16.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:16.886 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 12:04:16.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:16.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:16.886 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 12:04:16.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:16.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:16.886 | INFO | Uploaded 36GiB to GPU in 1.96334s (18.3361GiB/s) | 20260506 12:04:16.886 | INFO | Downloaded 34GiB to GPU in 2.00283s (16.976GiB/s) | 20260506 12:04:22.024 | INFO | Analyzing chunk frames = [1211904,1212416) | 20260506 12:04:22.024 | INFO | MemInfo: { | 20260506 12:04:22.024 | INFO | "gpu" : " 17 GB", | 20260506 12:04:22.024 | INFO | "huge" : "", | 20260506 12:04:22.024 | INFO | "huge_shared" : " 27 GB", | 20260506 12:04:22.024 | INFO | "malloc" : " 14 GB", | 20260506 12:04:22.024 | INFO | "pinned" : " 29 GB" | 20260506 12:04:22.024 | INFO | } | 20260506 12:04:22.024 | INFO | Perf report for the last 5.13834 seconds: | 20260506 12:04:22.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:04:22.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:22.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:22.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:22.024 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260506 12:04:22.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:22.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:22.025 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260506 12:04:22.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:22.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:22.025 | INFO | Uploaded 36GiB to GPU in 2.01922s (17.8287GiB/s) | 20260506 12:04:22.025 | INFO | Downloaded 34GiB to GPU in 1.98044s (17.1679GiB/s) | 20260506 12:04:22.025 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:04:22.025 | INFO | Total mem usage: 16.8 GiB | 20260506 12:04:22.025 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:04:22.025 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -16%) | 20260506 12:04:22.025 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:04:22.025 | INFO | High Water Breakdown: | 20260506 12:04:22.025 | INFO | 59.6% : 8.63 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:04:22.025 | INFO | 40.3% : 5.83 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:04:22.025 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:04:22.025 | INFO | ------------------------------------------ | 20260506 12:04:22.025 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:04:22.025 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:04:22.025 | INFO | Total mem usage: 26.5 GiB | 20260506 12:04:22.025 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:04:22.025 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 12:04:22.025 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260506 12:04:22.025 | INFO | High Water Breakdown: | 20260506 12:04:22.025 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:04:22.025 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:04:22.025 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:04:22.025 | INFO | 2% : 523 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:04:22.025 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:04:22.025 | INFO | 0.557% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:04:22.025 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:04:22.025 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:04:22.025 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:04:22.025 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:04:22.025 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:04:22.025 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:04:22.025 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:04:22.025 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:04:22.025 | INFO | ------------------------------------------ | 20260506 12:04:22.025 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:04:22.025 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:04:22.025 | INFO | Total mem usage: 25.6 GiB | 20260506 12:04:22.025 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:04:22.025 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:04:22.025 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:04:22.025 | INFO | High Water Breakdown: | 20260506 12:04:22.025 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:04:22.025 | INFO | ------------------------------------------ | 20260506 12:04:22.025 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:04:22.025 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:04:22.025 | INFO | Total mem usage: 15.7 GiB | 20260506 12:04:22.025 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:04:22.025 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:04:22.025 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:04:22.025 | INFO | High Water Breakdown: | 20260506 12:04:22.025 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:04:22.025 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:04:22.026 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:04:22.026 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:04:22.026 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:04:22.026 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:04:22.026 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:04:22.026 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:04:22.026 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:04:22.026 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:04:22.026 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:04:22.026 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:04:22.026 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:04:22.026 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:04:22.026 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:04:22.026 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:04:22.026 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:04:22.026 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:04:22.026 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:04:22.026 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:04:22.026 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:04:22.026 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:04:22.026 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:04:22.026 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:04:22.026 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:04:22.026 | INFO | ------------------------------------------ | 20260506 12:04:22.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:04:22.026 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:04:22.026 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:04:22.026 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:04:22.026 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.59e+01, 1.35e+01 / 2.64e+00 | 20260506 12:04:22.026 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.03e-02 / 7.22e-03 | 20260506 12:04:22.026 | INFO | PerfReport: 11.54%, Baseline -- 3.68e+00 / 4.33e+00, 5.83e-01 / 3.98e-01 | 20260506 12:04:22.026 | INFO | PerfReport: 3.25%, Histogram -- 1.04e+00 / 1.46e+00, 1.94e-01 / 1.58e-01 | 20260506 12:04:22.026 | INFO | PerfReport: 15.84%, DME -- 4.04e+01 / 4.49e+01, 4.20e+00 / 2.53e+00 | 20260506 12:04:22.026 | INFO | PerfReport: 56.29%, FrameLabeling -- 1.80e+01 / 2.00e+01, 1.48e+00 / 1.20e+00 | 20260506 12:04:22.026 | INFO | PerfReport: 7.65%, PulseAccumulating -- 2.44e+00 / 3.12e+00, 4.37e-01 / 2.17e-01 | 20260506 12:04:22.026 | INFO | PerfReport: 3.79%, Metrics -- 1.21e+00 / 1.58e+00, 3.03e-01 / 1.29e-01 | 20260506 12:04:22.026 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.28e-01 / 4.85e-01, 6.62e-02 / 2.93e-02 | 20260506 12:04:22.026 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:04:22.026 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:04:22.026 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:04:22.026 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 12:04:22.026 | INFO | PerfReport: 21.37%, Upload -- 2.12e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260506 12:04:22.026 | INFO | PerfReport: 21.04%, Download -- 2.08e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 12:04:22.026 | INFO | PerfReport: 32.21%, Compute -- 3.19e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 12:04:22.026 | INFO | PerfReport: 1.77%, LockUpload -- 1.75e+00 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 12:04:22.026 | INFO | PerfReport: 1.10%, LockDownload -- 1.09e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260506 12:04:22.026 | INFO | PerfReport: 22.49%, LockCompute -- 2.23e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 12:04:22.026 | INFO | PerfReport: 0.01%, Misc -- 1.49e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260506 12:04:22.156 | INFO | Starting a round of Baz IO | 20260506 12:04:25.333 | INFO | Populating RTMetrics report | 20260506 12:04:25.333 | INFO | RTMetrics report populated | 20260506 12:04:25.333 | INFO | Opening tmp RTMetrics json file | 20260506 12:04:25.334 | INFO | Serializing RTMetrics to json | 20260506 12:04:25.335 | INFO | Serialization done, swapping tmp file now... | 20260506 12:04:25.336 | INFO | RTMetrics done | 20260506 12:04:25.433 | INFO | Finished a round of Baz IO: Wrote 2.083 GiB in 3.276 seconds | 20260506 12:04:25.433 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.44% / 0.03% / 57.53% | 20260506 12:04:25.433 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.40927% / 2.38419e-05% / 6.35783e-05% | 20260506 12:04:25.433 | INFO | Baz IO overall: | 20260506 12:04:25.433 | INFO | BytesWritten : 595192148360 | 20260506 12:04:25.433 | INFO | NumEvents : 287515706106 | 20260506 12:04:27.164 | INFO | Analyzing chunk frames = [1212416,1212928) | 20260506 12:04:27.164 | INFO | MemInfo: { | 20260506 12:04:27.164 | INFO | "gpu" : " 17 GB", | 20260506 12:04:27.164 | INFO | "huge" : "", | 20260506 12:04:27.164 | INFO | "huge_shared" : " 27 GB", | 20260506 12:04:27.164 | INFO | "malloc" : " 14 GB", | 20260506 12:04:27.164 | INFO | "pinned" : " 29 GB" | 20260506 12:04:27.164 | INFO | } | 20260506 12:04:27.164 | INFO | Perf report for the last 5.13922 seconds: | 20260506 12:04:27.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:04:27.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:27.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:27.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:27.164 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260506 12:04:27.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:27.164 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:04:27.164 | INFO | PRE_HQ : 61.7%, 1.9, 61.8ms, 0 | 20260506 12:04:27.164 | INFO | BAZWRITER : 63.1%, 3.6, 117.8ms, 0 | 20260506 12:04:27.164 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:04:27.164 | INFO | Uploaded 36GiB to GPU in 2.08072s (17.3017GiB/s) | 20260506 12:04:27.164 | INFO | Downloaded 38GiB to GPU in 2.21764s (17.1354GiB/s) | 20260506 12:04:32.303 | INFO | Analyzing chunk frames = [1212928,1213440) | 20260506 12:04:32.303 | INFO | MemInfo: { | 20260506 12:04:32.303 | INFO | "gpu" : " 17 GB", | 20260506 12:04:32.303 | INFO | "huge" : "", | 20260506 12:04:32.303 | INFO | "huge_shared" : " 27 GB", | 20260506 12:04:32.303 | INFO | "malloc" : " 14 GB", | 20260506 12:04:32.303 | INFO | "pinned" : " 29 GB" | 20260506 12:04:32.303 | INFO | } | 20260506 12:04:32.303 | INFO | Perf report for the last 5.1393 seconds: | 20260506 12:04:32.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:04:32.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:32.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:32.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:32.303 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 12:04:32.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:32.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:32.303 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 12:04:32.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:32.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:32.303 | INFO | Uploaded 36GiB to GPU in 2.01707s (17.8476GiB/s) | 20260506 12:04:32.303 | INFO | Downloaded 34GiB to GPU in 1.91382s (17.7655GiB/s) | 20260506 12:04:37.442 | INFO | Analyzing chunk frames = [1213440,1213952) | 20260506 12:04:37.442 | INFO | MemInfo: { | 20260506 12:04:37.442 | INFO | "gpu" : " 17 GB", | 20260506 12:04:37.442 | INFO | "huge" : "", | 20260506 12:04:37.442 | INFO | "huge_shared" : " 27 GB", | 20260506 12:04:37.442 | INFO | "malloc" : " 14 GB", | 20260506 12:04:37.442 | INFO | "pinned" : " 29 GB" | 20260506 12:04:37.442 | INFO | } | 20260506 12:04:37.443 | INFO | Perf report for the last 5.13956 seconds: | 20260506 12:04:37.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:04:37.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:37.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:37.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:37.443 | INFO | ANALYSIS : 61.9%, 3.0, 95.9ms, 0 | 20260506 12:04:37.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:37.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:37.443 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:04:37.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:37.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:37.443 | INFO | Uploaded 36GiB to GPU in 2.03244s (17.7127GiB/s) | 20260506 12:04:37.443 | INFO | Downloaded 34GiB to GPU in 1.93925s (17.5326GiB/s) | 20260506 12:04:42.582 | INFO | Analyzing chunk frames = [1213952,1214464) | 20260506 12:04:42.582 | INFO | MemInfo: { | 20260506 12:04:42.582 | INFO | "gpu" : " 17 GB", | 20260506 12:04:42.582 | INFO | "huge" : "", | 20260506 12:04:42.582 | INFO | "huge_shared" : " 27 GB", | 20260506 12:04:42.582 | INFO | "malloc" : " 14 GB", | 20260506 12:04:42.582 | INFO | "pinned" : " 29 GB" | 20260506 12:04:42.582 | INFO | } | 20260506 12:04:42.582 | INFO | Perf report for the last 5.13919 seconds: | 20260506 12:04:42.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:04:42.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:42.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:42.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:42.582 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 12:04:42.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:42.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:42.582 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:04:42.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:42.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:42.582 | INFO | Uploaded 36GiB to GPU in 2.07585s (17.3423GiB/s) | 20260506 12:04:42.582 | INFO | Downloaded 34GiB to GPU in 1.96645s (17.29GiB/s) | 20260506 12:04:47.721 | INFO | Analyzing chunk frames = [1214464,1214976) | 20260506 12:04:47.721 | INFO | MemInfo: { | 20260506 12:04:47.721 | INFO | "gpu" : " 17 GB", | 20260506 12:04:47.721 | INFO | "huge" : "", | 20260506 12:04:47.721 | INFO | "huge_shared" : " 27 GB", | 20260506 12:04:47.721 | INFO | "malloc" : " 14 GB", | 20260506 12:04:47.721 | INFO | "pinned" : " 29 GB" | 20260506 12:04:47.721 | INFO | } | 20260506 12:04:47.721 | INFO | Perf report for the last 5.13933 seconds: | 20260506 12:04:47.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:04:47.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:47.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:47.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:47.721 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 12:04:47.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:47.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:47.721 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:04:47.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:47.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:47.721 | INFO | Uploaded 36GiB to GPU in 2.06918s (17.3982GiB/s) | 20260506 12:04:47.721 | INFO | Downloaded 34GiB to GPU in 1.97508s (17.2145GiB/s) | 20260506 12:04:52.861 | INFO | Analyzing chunk frames = [1214976,1215488) | 20260506 12:04:52.861 | INFO | MemInfo: { | 20260506 12:04:52.861 | INFO | "gpu" : " 17 GB", | 20260506 12:04:52.861 | INFO | "huge" : "", | 20260506 12:04:52.861 | INFO | "huge_shared" : " 27 GB", | 20260506 12:04:52.861 | INFO | "malloc" : " 14 GB", | 20260506 12:04:52.861 | INFO | "pinned" : " 29 GB" | 20260506 12:04:52.861 | INFO | } | 20260506 12:04:52.861 | INFO | Perf report for the last 5.13994 seconds: | 20260506 12:04:52.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:04:52.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:52.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:52.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:52.861 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:04:52.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:52.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:52.861 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:04:52.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:52.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:52.861 | INFO | Uploaded 36GiB to GPU in 2.10174s (17.1286GiB/s) | 20260506 12:04:52.861 | INFO | Downloaded 34GiB to GPU in 1.98331s (17.1431GiB/s) | 20260506 12:04:58.000 | INFO | Analyzing chunk frames = [1215488,1216000) | 20260506 12:04:58.000 | INFO | MemInfo: { | 20260506 12:04:58.000 | INFO | "gpu" : " 17 GB", | 20260506 12:04:58.000 | INFO | "huge" : "", | 20260506 12:04:58.000 | INFO | "huge_shared" : " 27 GB", | 20260506 12:04:58.000 | INFO | "malloc" : " 14 GB", | 20260506 12:04:58.000 | INFO | "pinned" : " 29 GB" | 20260506 12:04:58.000 | INFO | } | 20260506 12:04:58.000 | INFO | Perf report for the last 5.13897 seconds: | 20260506 12:04:58.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:04:58.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:58.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:58.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:58.000 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 12:04:58.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:58.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:58.000 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 12:04:58.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:58.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:04:58.000 | INFO | Uploaded 36GiB to GPU in 2.00488s (17.9562GiB/s) | 20260506 12:04:58.000 | INFO | Downloaded 34GiB to GPU in 2.01023s (16.9135GiB/s) | 20260506 12:05:03.139 | INFO | Analyzing chunk frames = [1216000,1216512) | 20260506 12:05:03.139 | INFO | MemInfo: { | 20260506 12:05:03.139 | INFO | "gpu" : " 17 GB", | 20260506 12:05:03.139 | INFO | "huge" : "", | 20260506 12:05:03.139 | INFO | "huge_shared" : " 27 GB", | 20260506 12:05:03.139 | INFO | "malloc" : " 14 GB", | 20260506 12:05:03.139 | INFO | "pinned" : " 29 GB" | 20260506 12:05:03.139 | INFO | } | 20260506 12:05:03.139 | INFO | Perf report for the last 5.1393 seconds: | 20260506 12:05:03.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:05:03.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:03.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:03.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:03.139 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260506 12:05:03.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:03.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:03.139 | INFO | PRE_HQ : 33.7%, 1.0, 17.6ms, 0 | 20260506 12:05:03.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:03.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:03.140 | INFO | Uploaded 36GiB to GPU in 2.01921s (17.8288GiB/s) | 20260506 12:05:03.140 | INFO | Downloaded 34GiB to GPU in 1.9885s (17.0983GiB/s) | 20260506 12:05:03.271 | INFO | Starting a round of Baz IO | 20260506 12:05:06.439 | INFO | Populating RTMetrics report | 20260506 12:05:06.439 | INFO | RTMetrics report populated | 20260506 12:05:06.439 | INFO | Opening tmp RTMetrics json file | 20260506 12:05:06.440 | INFO | Serializing RTMetrics to json | 20260506 12:05:06.441 | INFO | Serialization done, swapping tmp file now... | 20260506 12:05:06.442 | INFO | RTMetrics done | 20260506 12:05:06.538 | INFO | Finished a round of Baz IO: Wrote 2.084 GiB in 3.267 seconds | 20260506 12:05:06.538 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.42% / 0.03% / 57.56% | 20260506 12:05:06.538 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.399% / 2.78155e-05% / 4.76837e-05% | 20260506 12:05:06.538 | INFO | Baz IO overall: | 20260506 12:05:06.538 | INFO | BytesWritten : 597429986752 | 20260506 12:05:06.538 | INFO | NumEvents : 288601915405 | 20260506 12:05:08.278 | INFO | Analyzing chunk frames = [1216512,1217024) | 20260506 12:05:08.278 | INFO | MemInfo: { | 20260506 12:05:08.278 | INFO | "gpu" : " 17 GB", | 20260506 12:05:08.278 | INFO | "huge" : "", | 20260506 12:05:08.278 | INFO | "huge_shared" : " 27 GB", | 20260506 12:05:08.278 | INFO | "malloc" : " 14 GB", | 20260506 12:05:08.278 | INFO | "pinned" : " 29 GB" | 20260506 12:05:08.278 | INFO | } | 20260506 12:05:08.278 | INFO | Perf report for the last 5.13911 seconds: | 20260506 12:05:08.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:05:08.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:08.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:08.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:08.278 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260506 12:05:08.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:08.279 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 12:05:08.279 | INFO | PRE_HQ : 61.6%, 1.9, 61.8ms, 0 | 20260506 12:05:08.279 | INFO | BAZWRITER : 62.9%, 3.7, 119.4ms, 0 | 20260506 12:05:08.279 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:05:08.279 | INFO | Uploaded 36GiB to GPU in 2.09154s (17.2122GiB/s) | 20260506 12:05:08.279 | INFO | Downloaded 38GiB to GPU in 2.21439s (17.1605GiB/s) | 20260506 12:05:13.418 | INFO | Analyzing chunk frames = [1217024,1217536) | 20260506 12:05:13.418 | INFO | MemInfo: { | 20260506 12:05:13.418 | INFO | "gpu" : " 17 GB", | 20260506 12:05:13.418 | INFO | "huge" : "", | 20260506 12:05:13.418 | INFO | "huge_shared" : " 27 GB", | 20260506 12:05:13.418 | INFO | "malloc" : " 14 GB", | 20260506 12:05:13.418 | INFO | "pinned" : " 29 GB" | 20260506 12:05:13.418 | INFO | } | 20260506 12:05:13.418 | INFO | Perf report for the last 5.13939 seconds: | 20260506 12:05:13.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:05:13.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:13.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:13.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:13.418 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:05:13.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:13.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:13.418 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 12:05:13.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:13.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:13.418 | INFO | Uploaded 36GiB to GPU in 2.02802s (17.7513GiB/s) | 20260506 12:05:13.418 | INFO | Downloaded 34GiB to GPU in 1.92991s (17.6174GiB/s) | 20260506 12:05:18.557 | INFO | Analyzing chunk frames = [1217536,1218048) | 20260506 12:05:18.557 | INFO | MemInfo: { | 20260506 12:05:18.557 | INFO | "gpu" : " 17 GB", | 20260506 12:05:18.557 | INFO | "huge" : "", | 20260506 12:05:18.557 | INFO | "huge_shared" : " 27 GB", | 20260506 12:05:18.557 | INFO | "malloc" : " 14 GB", | 20260506 12:05:18.557 | INFO | "pinned" : " 29 GB" | 20260506 12:05:18.557 | INFO | } | 20260506 12:05:18.557 | INFO | Perf report for the last 5.13945 seconds: | 20260506 12:05:18.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:05:18.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:18.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:18.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:18.557 | INFO | ANALYSIS : 62.0%, 3.0, 96.0ms, 0 | 20260506 12:05:18.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:18.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:18.557 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:05:18.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:18.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:18.558 | INFO | Uploaded 36GiB to GPU in 2.04536s (17.6008GiB/s) | 20260506 12:05:18.558 | INFO | Downloaded 34GiB to GPU in 1.93947s (17.5306GiB/s) | 20260506 12:05:23.696 | INFO | Analyzing chunk frames = [1218048,1218560) | 20260506 12:05:23.696 | INFO | MemInfo: { | 20260506 12:05:23.696 | INFO | "gpu" : " 17 GB", | 20260506 12:05:23.696 | INFO | "huge" : "", | 20260506 12:05:23.696 | INFO | "huge_shared" : " 27 GB", | 20260506 12:05:23.697 | INFO | "malloc" : " 14 GB", | 20260506 12:05:23.697 | INFO | "pinned" : " 29 GB" | 20260506 12:05:23.697 | INFO | } | 20260506 12:05:23.697 | INFO | Perf report for the last 5.13928 seconds: | 20260506 12:05:23.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:05:23.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:23.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:23.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:23.697 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 12:05:23.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:23.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:23.697 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:05:23.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:23.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:23.697 | INFO | Uploaded 36GiB to GPU in 2.08502s (17.2661GiB/s) | 20260506 12:05:23.697 | INFO | Downloaded 34GiB to GPU in 1.9753s (17.2126GiB/s) | 20260506 12:05:28.836 | INFO | Analyzing chunk frames = [1218560,1219072) | 20260506 12:05:28.836 | INFO | MemInfo: { | 20260506 12:05:28.836 | INFO | "gpu" : " 17 GB", | 20260506 12:05:28.836 | INFO | "huge" : "", | 20260506 12:05:28.836 | INFO | "huge_shared" : " 27 GB", | 20260506 12:05:28.836 | INFO | "malloc" : " 14 GB", | 20260506 12:05:28.836 | INFO | "pinned" : " 29 GB" | 20260506 12:05:28.836 | INFO | } | 20260506 12:05:28.836 | INFO | Perf report for the last 5.13925 seconds: | 20260506 12:05:28.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:05:28.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:28.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:28.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:28.836 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 12:05:28.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:28.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:28.836 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:05:28.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:28.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:28.836 | INFO | Uploaded 36GiB to GPU in 2.0481s (17.5772GiB/s) | 20260506 12:05:28.836 | INFO | Downloaded 34GiB to GPU in 1.97287s (17.2338GiB/s) | 20260506 12:05:33.975 | INFO | Analyzing chunk frames = [1219072,1219584) | 20260506 12:05:33.975 | INFO | MemInfo: { | 20260506 12:05:33.975 | INFO | "gpu" : " 17 GB", | 20260506 12:05:33.975 | INFO | "huge" : "", | 20260506 12:05:33.975 | INFO | "huge_shared" : " 27 GB", | 20260506 12:05:33.975 | INFO | "malloc" : " 14 GB", | 20260506 12:05:33.975 | INFO | "pinned" : " 29 GB" | 20260506 12:05:33.975 | INFO | } | 20260506 12:05:33.975 | INFO | Perf report for the last 5.13951 seconds: | 20260506 12:05:33.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:05:33.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:33.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:33.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:33.975 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 12:05:33.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:33.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:33.975 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:05:33.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:33.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:33.976 | INFO | Uploaded 36GiB to GPU in 2.03565s (17.6848GiB/s) | 20260506 12:05:33.976 | INFO | Downloaded 34GiB to GPU in 1.91261s (17.7768GiB/s) | 20260506 12:05:39.115 | INFO | Analyzing chunk frames = [1219584,1220096) | 20260506 12:05:39.115 | INFO | MemInfo: { | 20260506 12:05:39.115 | INFO | "gpu" : " 17 GB", | 20260506 12:05:39.115 | INFO | "huge" : "", | 20260506 12:05:39.115 | INFO | "huge_shared" : " 27 GB", | 20260506 12:05:39.115 | INFO | "malloc" : " 14 GB", | 20260506 12:05:39.115 | INFO | "pinned" : " 29 GB" | 20260506 12:05:39.115 | INFO | } | 20260506 12:05:39.116 | INFO | Perf report for the last 5.14014 seconds: | 20260506 12:05:39.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:05:39.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:39.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:39.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:39.116 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:05:39.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:39.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:39.116 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 12:05:39.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:39.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:39.116 | INFO | Uploaded 36GiB to GPU in 2.01473s (17.8684GiB/s) | 20260506 12:05:39.116 | INFO | Downloaded 34GiB to GPU in 2.0363s (16.6969GiB/s) | 20260506 12:05:44.254 | INFO | Analyzing chunk frames = [1220096,1220608) | 20260506 12:05:44.254 | INFO | MemInfo: { | 20260506 12:05:44.254 | INFO | "gpu" : " 17 GB", | 20260506 12:05:44.254 | INFO | "huge" : "", | 20260506 12:05:44.254 | INFO | "huge_shared" : " 27 GB", | 20260506 12:05:44.254 | INFO | "malloc" : " 14 GB", | 20260506 12:05:44.254 | INFO | "pinned" : " 29 GB" | 20260506 12:05:44.254 | INFO | } | 20260506 12:05:44.254 | INFO | Perf report for the last 5.13872 seconds: | 20260506 12:05:44.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:05:44.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:44.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:44.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:44.254 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260506 12:05:44.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:44.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:44.254 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260506 12:05:44.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:44.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:44.255 | INFO | Uploaded 36GiB to GPU in 2.02748s (17.756GiB/s) | 20260506 12:05:44.255 | INFO | Downloaded 34GiB to GPU in 1.99109s (17.0761GiB/s) | 20260506 12:05:44.255 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:05:44.255 | INFO | Total mem usage: 16.8 GiB | 20260506 12:05:44.255 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:05:44.255 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.7%) | 20260506 12:05:44.255 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:05:44.255 | INFO | High Water Breakdown: | 20260506 12:05:44.255 | INFO | 59.5% : 8.57 GiB ( 82.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:05:44.255 | INFO | 40.3% : 5.81 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:05:44.255 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:05:44.255 | INFO | ------------------------------------------ | 20260506 12:05:44.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:05:44.255 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:05:44.255 | INFO | Total mem usage: 26.5 GiB | 20260506 12:05:44.255 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:05:44.255 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:05:44.255 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:05:44.255 | INFO | High Water Breakdown: | 20260506 12:05:44.255 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:05:44.255 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:05:44.255 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:05:44.255 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:05:44.255 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:05:44.255 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:05:44.255 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:05:44.255 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:05:44.255 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:05:44.255 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:05:44.255 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:05:44.255 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:05:44.255 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:05:44.255 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:05:44.255 | INFO | ------------------------------------------ | 20260506 12:05:44.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:05:44.255 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:05:44.255 | INFO | Total mem usage: 25.6 GiB | 20260506 12:05:44.255 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:05:44.255 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:05:44.255 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:05:44.255 | INFO | High Water Breakdown: | 20260506 12:05:44.255 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:05:44.255 | INFO | ------------------------------------------ | 20260506 12:05:44.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:05:44.255 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:05:44.255 | INFO | Total mem usage: 15.7 GiB | 20260506 12:05:44.255 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:05:44.255 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:05:44.255 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 12:05:44.255 | INFO | High Water Breakdown: | 20260506 12:05:44.255 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:05:44.255 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:05:44.255 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:05:44.255 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:05:44.255 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:05:44.255 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:05:44.255 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:05:44.255 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:05:44.255 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:05:44.255 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:05:44.255 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:05:44.256 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:05:44.256 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:05:44.256 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:05:44.256 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:05:44.256 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:05:44.256 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:05:44.256 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:05:44.256 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:05:44.256 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:05:44.256 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:05:44.256 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:05:44.256 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:05:44.256 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:05:44.256 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:05:44.256 | INFO | ------------------------------------------ | 20260506 12:05:44.256 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:05:44.256 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:05:44.256 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:05:44.256 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:05:44.256 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.58e+01, 1.35e+01 / 2.67e+00 | 20260506 12:05:44.256 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 9.83e-02, 1.08e-02 / 6.50e-03 | 20260506 12:05:44.256 | INFO | PerfReport: 11.54%, Baseline -- 3.68e+00 / 4.36e+00, 5.85e-01 / 4.04e-01 | 20260506 12:05:44.256 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.43e+00, 1.93e-01 / 1.53e-01 | 20260506 12:05:44.256 | INFO | PerfReport: 15.83%, DME -- 4.04e+01 / 4.47e+01, 4.19e+00 / 2.85e+00 | 20260506 12:05:44.256 | INFO | PerfReport: 56.34%, FrameLabeling -- 1.80e+01 / 2.01e+01, 1.48e+00 / 1.86e+00 | 20260506 12:05:44.256 | INFO | PerfReport: 7.62%, PulseAccumulating -- 2.43e+00 / 3.08e+00, 4.26e-01 / 2.38e-01 | 20260506 12:05:44.256 | INFO | PerfReport: 3.79%, Metrics -- 1.21e+00 / 1.59e+00, 3.07e-01 / 1.71e-01 | 20260506 12:05:44.256 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.27e-01 / 4.87e-01, 6.69e-02 / 3.42e-02 | 20260506 12:05:44.256 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:05:44.256 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:05:44.256 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:05:44.256 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 12:05:44.256 | INFO | PerfReport: 21.53%, Upload -- 2.13e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 12:05:44.256 | INFO | PerfReport: 21.00%, Download -- 2.08e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 12:05:44.256 | INFO | PerfReport: 32.16%, Compute -- 3.19e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 12:05:44.256 | INFO | PerfReport: 1.94%, LockUpload -- 1.92e+00 / 0.00e+00, 7.59e+00 / 0.00e+00 | 20260506 12:05:44.256 | INFO | PerfReport: 0.92%, LockDownload -- 9.07e-01 / 0.00e+00, 2.48e+00 / 0.00e+00 | 20260506 12:05:44.256 | INFO | PerfReport: 22.44%, LockCompute -- 2.22e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 12:05:44.256 | INFO | PerfReport: 0.02%, Misc -- 1.50e-02 / 0.00e+00, 2.24e-02 / 0.00e+00 | 20260506 12:05:44.380 | INFO | Starting a round of Baz IO | 20260506 12:05:47.556 | INFO | Populating RTMetrics report | 20260506 12:05:47.557 | INFO | RTMetrics report populated | 20260506 12:05:47.557 | INFO | Opening tmp RTMetrics json file | 20260506 12:05:47.558 | INFO | Serializing RTMetrics to json | 20260506 12:05:47.559 | INFO | Serialization done, swapping tmp file now... | 20260506 12:05:47.559 | INFO | RTMetrics done | 20260506 12:05:47.651 | INFO | Finished a round of Baz IO: Wrote 2.084 GiB in 3.27 seconds | 20260506 12:05:47.651 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.39% / 0.03% / 57.58% | 20260506 12:05:47.651 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.38289% / 7.15256e-05% / 3.97364e-05% | 20260506 12:05:47.651 | INFO | Baz IO overall: | 20260506 12:05:47.651 | INFO | BytesWritten : 599667196601 | 20260506 12:05:47.651 | INFO | NumEvents : 289687585372 | 20260506 12:05:49.393 | INFO | Analyzing chunk frames = [1220608,1221120) | 20260506 12:05:49.393 | INFO | MemInfo: { | 20260506 12:05:49.393 | INFO | "gpu" : " 17 GB", | 20260506 12:05:49.393 | INFO | "huge" : "", | 20260506 12:05:49.393 | INFO | "huge_shared" : " 27 GB", | 20260506 12:05:49.393 | INFO | "malloc" : " 14 GB", | 20260506 12:05:49.393 | INFO | "pinned" : " 29 GB" | 20260506 12:05:49.393 | INFO | } | 20260506 12:05:49.393 | INFO | Perf report for the last 5.13898 seconds: | 20260506 12:05:49.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:05:49.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:49.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:49.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:49.393 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260506 12:05:49.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:49.393 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 12:05:49.393 | INFO | PRE_HQ : 61.8%, 1.9, 62.1ms, 0 | 20260506 12:05:49.393 | INFO | BAZWRITER : 62.8%, 3.6, 116.4ms, 0 | 20260506 12:05:49.393 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:05:49.394 | INFO | Uploaded 36GiB to GPU in 2.10883s (17.0711GiB/s) | 20260506 12:05:49.394 | INFO | Downloaded 38GiB to GPU in 2.21188s (17.18GiB/s) | 20260506 12:05:54.533 | INFO | Analyzing chunk frames = [1221120,1221632) | 20260506 12:05:54.533 | INFO | MemInfo: { | 20260506 12:05:54.533 | INFO | "gpu" : " 17 GB", | 20260506 12:05:54.533 | INFO | "huge" : "", | 20260506 12:05:54.533 | INFO | "huge_shared" : " 27 GB", | 20260506 12:05:54.533 | INFO | "malloc" : " 14 GB", | 20260506 12:05:54.533 | INFO | "pinned" : " 29 GB" | 20260506 12:05:54.533 | INFO | } | 20260506 12:05:54.533 | INFO | Perf report for the last 5.13959 seconds: | 20260506 12:05:54.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:05:54.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:54.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:54.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:54.533 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 12:05:54.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:54.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:54.533 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:05:54.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:54.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:54.533 | INFO | Uploaded 36GiB to GPU in 2.01339s (17.8803GiB/s) | 20260506 12:05:54.533 | INFO | Downloaded 34GiB to GPU in 1.93059s (17.6112GiB/s) | 20260506 12:05:59.672 | INFO | Analyzing chunk frames = [1221632,1222144) | 20260506 12:05:59.672 | INFO | MemInfo: { | 20260506 12:05:59.672 | INFO | "gpu" : " 17 GB", | 20260506 12:05:59.672 | INFO | "huge" : "", | 20260506 12:05:59.672 | INFO | "huge_shared" : " 27 GB", | 20260506 12:05:59.672 | INFO | "malloc" : " 14 GB", | 20260506 12:05:59.672 | INFO | "pinned" : " 29 GB" | 20260506 12:05:59.672 | INFO | } | 20260506 12:05:59.672 | INFO | Perf report for the last 5.139 seconds: | 20260506 12:05:59.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:05:59.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:59.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:59.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:59.672 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:05:59.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:59.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:59.672 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:05:59.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:59.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:05:59.672 | INFO | Uploaded 36GiB to GPU in 2.01817s (17.8379GiB/s) | 20260506 12:05:59.672 | INFO | Downloaded 34GiB to GPU in 1.9436s (17.4933GiB/s) | 20260506 12:06:04.811 | INFO | Analyzing chunk frames = [1222144,1222656) | 20260506 12:06:04.811 | INFO | MemInfo: { | 20260506 12:06:04.811 | INFO | "gpu" : " 17 GB", | 20260506 12:06:04.811 | INFO | "huge" : "", | 20260506 12:06:04.811 | INFO | "huge_shared" : " 27 GB", | 20260506 12:06:04.811 | INFO | "malloc" : " 14 GB", | 20260506 12:06:04.811 | INFO | "pinned" : " 29 GB" | 20260506 12:06:04.811 | INFO | } | 20260506 12:06:04.811 | INFO | Perf report for the last 5.13937 seconds: | 20260506 12:06:04.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:06:04.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:04.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:04.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:04.811 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 12:06:04.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:04.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:04.811 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:06:04.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:04.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:04.812 | INFO | Uploaded 36GiB to GPU in 2.06476s (17.4354GiB/s) | 20260506 12:06:04.812 | INFO | Downloaded 34GiB to GPU in 1.9712s (17.2484GiB/s) | 20260506 12:06:09.951 | INFO | Analyzing chunk frames = [1222656,1223168) | 20260506 12:06:09.951 | INFO | MemInfo: { | 20260506 12:06:09.951 | INFO | "gpu" : " 17 GB", | 20260506 12:06:09.951 | INFO | "huge" : "", | 20260506 12:06:09.951 | INFO | "huge_shared" : " 27 GB", | 20260506 12:06:09.951 | INFO | "malloc" : " 14 GB", | 20260506 12:06:09.951 | INFO | "pinned" : " 29 GB" | 20260506 12:06:09.951 | INFO | } | 20260506 12:06:09.951 | INFO | Perf report for the last 5.13947 seconds: | 20260506 12:06:09.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:06:09.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:09.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:09.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:09.951 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 12:06:09.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:09.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:09.951 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:06:09.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:09.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:09.951 | INFO | Uploaded 36GiB to GPU in 2.01871s (17.8332GiB/s) | 20260506 12:06:09.951 | INFO | Downloaded 34GiB to GPU in 1.96683s (17.2867GiB/s) | 20260506 12:06:15.090 | INFO | Analyzing chunk frames = [1223168,1223680) | 20260506 12:06:15.090 | INFO | MemInfo: { | 20260506 12:06:15.090 | INFO | "gpu" : " 17 GB", | 20260506 12:06:15.090 | INFO | "huge" : "", | 20260506 12:06:15.090 | INFO | "huge_shared" : " 27 GB", | 20260506 12:06:15.090 | INFO | "malloc" : " 14 GB", | 20260506 12:06:15.090 | INFO | "pinned" : " 29 GB" | 20260506 12:06:15.090 | INFO | } | 20260506 12:06:15.090 | INFO | Perf report for the last 5.13931 seconds: | 20260506 12:06:15.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:06:15.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:15.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:15.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:15.090 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 12:06:15.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:15.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:15.090 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:06:15.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:15.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:15.090 | INFO | Uploaded 36GiB to GPU in 2.06419s (17.4403GiB/s) | 20260506 12:06:15.090 | INFO | Downloaded 34GiB to GPU in 1.93227s (17.5959GiB/s) | 20260506 12:06:20.230 | INFO | Analyzing chunk frames = [1223680,1224192) | 20260506 12:06:20.230 | INFO | MemInfo: { | 20260506 12:06:20.230 | INFO | "gpu" : " 17 GB", | 20260506 12:06:20.230 | INFO | "huge" : "", | 20260506 12:06:20.230 | INFO | "huge_shared" : " 27 GB", | 20260506 12:06:20.230 | INFO | "malloc" : " 14 GB", | 20260506 12:06:20.230 | INFO | "pinned" : " 29 GB" | 20260506 12:06:20.230 | INFO | } | 20260506 12:06:20.230 | INFO | Perf report for the last 5.14012 seconds: | 20260506 12:06:20.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:06:20.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:20.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:20.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:20.230 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:06:20.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:20.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:20.230 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 12:06:20.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:20.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:20.230 | INFO | Uploaded 36GiB to GPU in 1.97948s (18.1866GiB/s) | 20260506 12:06:20.230 | INFO | Downloaded 34GiB to GPU in 2.02719s (16.772GiB/s) | 20260506 12:06:25.368 | INFO | Analyzing chunk frames = [1224192,1224704) | 20260506 12:06:25.368 | INFO | MemInfo: { | 20260506 12:06:25.368 | INFO | "gpu" : " 17 GB", | 20260506 12:06:25.368 | INFO | "huge" : "", | 20260506 12:06:25.368 | INFO | "huge_shared" : " 27 GB", | 20260506 12:06:25.369 | INFO | "malloc" : " 14 GB", | 20260506 12:06:25.369 | INFO | "pinned" : " 29 GB" | 20260506 12:06:25.369 | INFO | } | 20260506 12:06:25.369 | INFO | Perf report for the last 5.13871 seconds: | 20260506 12:06:25.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:06:25.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:25.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:25.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:25.369 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260506 12:06:25.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:25.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:25.369 | INFO | PRE_HQ : 33.8%, 1.0, 17.6ms, 0 | 20260506 12:06:25.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:25.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:25.369 | INFO | Uploaded 36GiB to GPU in 2.00842s (17.9246GiB/s) | 20260506 12:06:25.369 | INFO | Downloaded 34GiB to GPU in 1.9987s (17.0111GiB/s) | 20260506 12:06:25.500 | INFO | Starting a round of Baz IO | 20260506 12:06:28.662 | INFO | Populating RTMetrics report | 20260506 12:06:28.662 | INFO | RTMetrics report populated | 20260506 12:06:28.662 | INFO | Opening tmp RTMetrics json file | 20260506 12:06:28.663 | INFO | Serializing RTMetrics to json | 20260506 12:06:28.664 | INFO | Serialization done, swapping tmp file now... | 20260506 12:06:28.664 | INFO | RTMetrics done | 20260506 12:06:28.758 | INFO | Finished a round of Baz IO: Wrote 2.086 GiB in 3.257 seconds | 20260506 12:06:28.758 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.36% / 0.03% / 57.61% | 20260506 12:06:28.758 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.37156% / 5.16574e-05% / 6.75519e-05% | 20260506 12:06:28.758 | INFO | Baz IO overall: | 20260506 12:06:28.758 | INFO | BytesWritten : 601906531620 | 20260506 12:06:28.758 | INFO | NumEvents : 290774193419 | 20260506 12:06:30.508 | INFO | Analyzing chunk frames = [1224704,1225216) | 20260506 12:06:30.508 | INFO | MemInfo: { | 20260506 12:06:30.508 | INFO | "gpu" : " 17 GB", | 20260506 12:06:30.508 | INFO | "huge" : "", | 20260506 12:06:30.508 | INFO | "huge_shared" : " 27 GB", | 20260506 12:06:30.508 | INFO | "malloc" : " 14 GB", | 20260506 12:06:30.508 | INFO | "pinned" : " 29 GB" | 20260506 12:06:30.508 | INFO | } | 20260506 12:06:30.508 | INFO | Perf report for the last 5.139 seconds: | 20260506 12:06:30.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:06:30.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:30.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:30.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:30.508 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 12:06:30.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:30.508 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:06:30.508 | INFO | PRE_HQ : 61.5%, 1.9, 61.9ms, 0 | 20260506 12:06:30.508 | INFO | BAZWRITER : 62.6%, 3.6, 117.0ms, 0 | 20260506 12:06:30.508 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:06:30.508 | INFO | Uploaded 36GiB to GPU in 2.07176s (17.3765GiB/s) | 20260506 12:06:30.508 | INFO | Downloaded 38GiB to GPU in 2.2087s (17.2047GiB/s) | 20260506 12:06:35.647 | INFO | Analyzing chunk frames = [1225216,1225728) | 20260506 12:06:35.647 | INFO | MemInfo: { | 20260506 12:06:35.647 | INFO | "gpu" : " 17 GB", | 20260506 12:06:35.647 | INFO | "huge" : "", | 20260506 12:06:35.647 | INFO | "huge_shared" : " 27 GB", | 20260506 12:06:35.647 | INFO | "malloc" : " 14 GB", | 20260506 12:06:35.647 | INFO | "pinned" : " 29 GB" | 20260506 12:06:35.647 | INFO | } | 20260506 12:06:35.647 | INFO | Perf report for the last 5.1393 seconds: | 20260506 12:06:35.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:06:35.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:35.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:35.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:35.647 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:06:35.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:35.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:35.647 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 12:06:35.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:35.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:35.647 | INFO | Uploaded 36GiB to GPU in 2.01612s (17.8561GiB/s) | 20260506 12:06:35.647 | INFO | Downloaded 34GiB to GPU in 1.92062s (17.7026GiB/s) | 20260506 12:06:40.786 | INFO | Analyzing chunk frames = [1225728,1226240) | 20260506 12:06:40.786 | INFO | MemInfo: { | 20260506 12:06:40.786 | INFO | "gpu" : " 17 GB", | 20260506 12:06:40.786 | INFO | "huge" : "", | 20260506 12:06:40.786 | INFO | "huge_shared" : " 27 GB", | 20260506 12:06:40.786 | INFO | "malloc" : " 14 GB", | 20260506 12:06:40.786 | INFO | "pinned" : " 29 GB" | 20260506 12:06:40.786 | INFO | } | 20260506 12:06:40.786 | INFO | Perf report for the last 5.13922 seconds: | 20260506 12:06:40.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:06:40.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:40.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:40.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:40.787 | INFO | ANALYSIS : 61.9%, 3.0, 95.8ms, 0 | 20260506 12:06:40.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:40.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:40.787 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:06:40.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:40.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:40.787 | INFO | Uploaded 36GiB to GPU in 2.03203s (17.7163GiB/s) | 20260506 12:06:40.787 | INFO | Downloaded 34GiB to GPU in 1.93813s (17.5427GiB/s) | 20260506 12:06:45.925 | INFO | Analyzing chunk frames = [1226240,1226752) | 20260506 12:06:45.925 | INFO | MemInfo: { | 20260506 12:06:45.925 | INFO | "gpu" : " 17 GB", | 20260506 12:06:45.925 | INFO | "huge" : "", | 20260506 12:06:45.925 | INFO | "huge_shared" : " 27 GB", | 20260506 12:06:45.925 | INFO | "malloc" : " 14 GB", | 20260506 12:06:45.925 | INFO | "pinned" : " 29 GB" | 20260506 12:06:45.926 | INFO | } | 20260506 12:06:45.926 | INFO | Perf report for the last 5.13923 seconds: | 20260506 12:06:45.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:06:45.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:45.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:45.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:45.926 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260506 12:06:45.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:45.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:45.926 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:06:45.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:45.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:45.926 | INFO | Uploaded 36GiB to GPU in 2.08147s (17.2955GiB/s) | 20260506 12:06:45.926 | INFO | Downloaded 34GiB to GPU in 1.9737s (17.2266GiB/s) | 20260506 12:06:51.065 | INFO | Analyzing chunk frames = [1226752,1227264) | 20260506 12:06:51.065 | INFO | MemInfo: { | 20260506 12:06:51.065 | INFO | "gpu" : " 17 GB", | 20260506 12:06:51.065 | INFO | "huge" : "", | 20260506 12:06:51.065 | INFO | "huge_shared" : " 27 GB", | 20260506 12:06:51.065 | INFO | "malloc" : " 14 GB", | 20260506 12:06:51.065 | INFO | "pinned" : " 29 GB" | 20260506 12:06:51.065 | INFO | } | 20260506 12:06:51.065 | INFO | Perf report for the last 5.13967 seconds: | 20260506 12:06:51.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:06:51.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:51.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:51.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:51.065 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 12:06:51.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:51.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:51.065 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:06:51.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:51.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:51.066 | INFO | Uploaded 36GiB to GPU in 2.04171s (17.6323GiB/s) | 20260506 12:06:51.066 | INFO | Downloaded 34GiB to GPU in 1.97182s (17.2429GiB/s) | 20260506 12:06:56.208 | INFO | Analyzing chunk frames = [1227264,1227776) | 20260506 12:06:56.208 | INFO | MemInfo: { | 20260506 12:06:56.208 | INFO | "gpu" : " 17 GB", | 20260506 12:06:56.208 | INFO | "huge" : "", | 20260506 12:06:56.208 | INFO | "huge_shared" : " 27 GB", | 20260506 12:06:56.208 | INFO | "malloc" : " 14 GB", | 20260506 12:06:56.208 | INFO | "pinned" : " 29 GB" | 20260506 12:06:56.208 | INFO | } | 20260506 12:06:56.208 | INFO | Perf report for the last 5.14309 seconds: | 20260506 12:06:56.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:06:56.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:56.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:56.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:56.209 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 12:06:56.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:56.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:56.209 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:06:56.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:56.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:06:56.209 | INFO | Uploaded 36GiB to GPU in 2.02814s (17.7502GiB/s) | 20260506 12:06:56.209 | INFO | Downloaded 34GiB to GPU in 1.9524s (17.4145GiB/s) | 20260506 12:07:01.345 | INFO | Analyzing chunk frames = [1227776,1228288) | 20260506 12:07:01.345 | INFO | MemInfo: { | 20260506 12:07:01.345 | INFO | "gpu" : " 17 GB", | 20260506 12:07:01.345 | INFO | "huge" : "", | 20260506 12:07:01.345 | INFO | "huge_shared" : " 27 GB", | 20260506 12:07:01.345 | INFO | "malloc" : " 14 GB", | 20260506 12:07:01.345 | INFO | "pinned" : " 29 GB" | 20260506 12:07:01.346 | INFO | } | 20260506 12:07:01.346 | INFO | Perf report for the last 5.1371 seconds: | 20260506 12:07:01.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:07:01.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:01.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:01.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:01.346 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 12:07:01.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:01.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:01.346 | INFO | PRE_HQ : 34.2%, 1.0, 17.9ms, 0 | 20260506 12:07:01.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:01.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:01.346 | INFO | Uploaded 36GiB to GPU in 2.03319s (17.7062GiB/s) | 20260506 12:07:01.346 | INFO | Downloaded 34GiB to GPU in 2.0156s (16.8684GiB/s) | 20260506 12:07:06.483 | INFO | Analyzing chunk frames = [1228288,1228800) | 20260506 12:07:06.483 | INFO | MemInfo: { | 20260506 12:07:06.483 | INFO | "gpu" : " 17 GB", | 20260506 12:07:06.483 | INFO | "huge" : "", | 20260506 12:07:06.483 | INFO | "huge_shared" : " 27 GB", | 20260506 12:07:06.483 | INFO | "malloc" : " 14 GB", | 20260506 12:07:06.483 | INFO | "pinned" : " 29 GB" | 20260506 12:07:06.483 | INFO | } | 20260506 12:07:06.483 | INFO | Perf report for the last 5.13765 seconds: | 20260506 12:07:06.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:07:06.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:06.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:06.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:06.483 | INFO | ANALYSIS : 63.1%, 3.0, 98.4ms, 0 | 20260506 12:07:06.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:06.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:06.483 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 12:07:06.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:06.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:06.483 | INFO | Uploaded 36GiB to GPU in 2.08671s (17.252GiB/s) | 20260506 12:07:06.484 | INFO | Downloaded 34GiB to GPU in 2.03161s (16.7355GiB/s) | 20260506 12:07:06.484 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:07:06.484 | INFO | Total mem usage: 16.8 GiB | 20260506 12:07:06.484 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:07:06.484 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.7%) | 20260506 12:07:06.484 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:07:06.484 | INFO | High Water Breakdown: | 20260506 12:07:06.484 | INFO | 59.5% : 8.58 GiB ( 82.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:07:06.484 | INFO | 40.3% : 5.81 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:07:06.484 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:07:06.484 | INFO | ------------------------------------------ | 20260506 12:07:06.484 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:07:06.484 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:07:06.484 | INFO | Total mem usage: 26.5 GiB | 20260506 12:07:06.484 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:07:06.484 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:07:06.484 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:07:06.484 | INFO | High Water Breakdown: | 20260506 12:07:06.484 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:07:06.484 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:07:06.484 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:07:06.484 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:07:06.484 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:07:06.484 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:07:06.484 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:07:06.484 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:07:06.484 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:07:06.484 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:07:06.484 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:07:06.484 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:07:06.484 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:07:06.484 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:07:06.484 | INFO | ------------------------------------------ | 20260506 12:07:06.484 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:07:06.484 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:07:06.484 | INFO | Total mem usage: 25.6 GiB | 20260506 12:07:06.484 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:07:06.484 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:07:06.484 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:07:06.484 | INFO | High Water Breakdown: | 20260506 12:07:06.484 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:07:06.484 | INFO | ------------------------------------------ | 20260506 12:07:06.484 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:07:06.484 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:07:06.484 | INFO | Total mem usage: 15.7 GiB | 20260506 12:07:06.484 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:07:06.484 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:07:06.484 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:07:06.484 | INFO | High Water Breakdown: | 20260506 12:07:06.484 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:07:06.484 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:07:06.484 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:07:06.484 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:07:06.484 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:07:06.484 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:07:06.484 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:07:06.484 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:07:06.484 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:07:06.484 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:07:06.484 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:07:06.484 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:07:06.484 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:07:06.484 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:07:06.484 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:07:06.484 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:07:06.484 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:07:06.484 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:07:06.484 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:07:06.484 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:07:06.484 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:07:06.484 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:07:06.484 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:07:06.484 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:07:06.484 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:07:06.484 | INFO | ------------------------------------------ | 20260506 12:07:06.484 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:07:06.484 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:07:06.485 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:07:06.485 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:07:06.485 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.63e+01, 1.38e+01 / 2.74e+00 | 20260506 12:07:06.485 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.01e-01, 1.14e-02 / 7.86e-03 | 20260506 12:07:06.485 | INFO | PerfReport: 11.55%, Baseline -- 3.69e+00 / 4.45e+00, 5.91e-01 / 3.63e-01 | 20260506 12:07:06.485 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.44e+00, 1.94e-01 / 1.58e-01 | 20260506 12:07:06.485 | INFO | PerfReport: 15.85%, DME -- 4.04e+01 / 4.50e+01, 4.18e+00 / 2.80e+00 | 20260506 12:07:06.485 | INFO | PerfReport: 56.34%, FrameLabeling -- 1.80e+01 / 2.01e+01, 1.50e+00 / 1.64e+00 | 20260506 12:07:06.485 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.42e+00 / 3.09e+00, 4.28e-01 / 2.12e-01 | 20260506 12:07:06.485 | INFO | PerfReport: 3.79%, Metrics -- 1.21e+00 / 1.60e+00, 3.10e-01 / 1.66e-01 | 20260506 12:07:06.485 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.27e-01 / 4.83e-01, 6.67e-02 / 2.66e-02 | 20260506 12:07:06.485 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:07:06.485 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:07:06.485 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:07:06.485 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 12:07:06.485 | INFO | PerfReport: 21.47%, Upload -- 2.13e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260506 12:07:06.485 | INFO | PerfReport: 21.03%, Download -- 2.08e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 12:07:06.485 | INFO | PerfReport: 32.21%, Compute -- 3.19e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260506 12:07:06.485 | INFO | PerfReport: 1.78%, LockUpload -- 1.77e+00 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 12:07:06.485 | INFO | PerfReport: 1.03%, LockDownload -- 1.02e+00 / 0.00e+00, 2.65e+00 / 0.00e+00 | 20260506 12:07:06.485 | INFO | PerfReport: 22.46%, LockCompute -- 2.23e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 12:07:06.485 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260506 12:07:06.605 | INFO | Starting a round of Baz IO | 20260506 12:07:09.787 | INFO | Populating RTMetrics report | 20260506 12:07:09.787 | INFO | RTMetrics report populated | 20260506 12:07:09.787 | INFO | Opening tmp RTMetrics json file | 20260506 12:07:09.788 | INFO | Serializing RTMetrics to json | 20260506 12:07:09.789 | INFO | Serialization done, swapping tmp file now... | 20260506 12:07:09.789 | INFO | RTMetrics done | 20260506 12:07:09.882 | INFO | Finished a round of Baz IO: Wrote 2.083 GiB in 3.277 seconds | 20260506 12:07:09.883 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.33% / 0.03% / 57.64% | 20260506 12:07:09.883 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.34723% / 5.16574e-05% / 6.35783e-05% | 20260506 12:07:09.883 | INFO | Baz IO overall: | 20260506 12:07:09.883 | INFO | BytesWritten : 604143152387 | 20260506 12:07:09.883 | INFO | NumEvents : 291858708530 | 20260506 12:07:11.622 | INFO | Analyzing chunk frames = [1228800,1229312) | 20260506 12:07:11.622 | INFO | MemInfo: { | 20260506 12:07:11.622 | INFO | "gpu" : " 17 GB", | 20260506 12:07:11.622 | INFO | "huge" : "", | 20260506 12:07:11.622 | INFO | "huge_shared" : " 27 GB", | 20260506 12:07:11.622 | INFO | "malloc" : " 14 GB", | 20260506 12:07:11.622 | INFO | "pinned" : " 29 GB" | 20260506 12:07:11.622 | INFO | } | 20260506 12:07:11.622 | INFO | Perf report for the last 5.1391 seconds: | 20260506 12:07:11.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:07:11.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:11.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:11.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:11.622 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260506 12:07:11.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:11.622 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 12:07:11.622 | INFO | PRE_HQ : 61.8%, 1.9, 62.1ms, 0 | 20260506 12:07:11.622 | INFO | BAZWRITER : 62.9%, 3.7, 119.9ms, 0 | 20260506 12:07:11.622 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:07:11.623 | INFO | Uploaded 36GiB to GPU in 2.08746s (17.2459GiB/s) | 20260506 12:07:11.623 | INFO | Downloaded 38GiB to GPU in 2.20808s (17.2095GiB/s) | 20260506 12:07:16.762 | INFO | Analyzing chunk frames = [1229312,1229824) | 20260506 12:07:16.762 | INFO | MemInfo: { | 20260506 12:07:16.762 | INFO | "gpu" : " 17 GB", | 20260506 12:07:16.762 | INFO | "huge" : "", | 20260506 12:07:16.762 | INFO | "huge_shared" : " 27 GB", | 20260506 12:07:16.762 | INFO | "malloc" : " 14 GB", | 20260506 12:07:16.762 | INFO | "pinned" : " 29 GB" | 20260506 12:07:16.762 | INFO | } | 20260506 12:07:16.762 | INFO | Perf report for the last 5.13947 seconds: | 20260506 12:07:16.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:07:16.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:16.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:16.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:16.762 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:07:16.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:16.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:16.762 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 12:07:16.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:16.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:16.762 | INFO | Uploaded 36GiB to GPU in 2.01648s (17.8529GiB/s) | 20260506 12:07:16.762 | INFO | Downloaded 34GiB to GPU in 1.93429s (17.5775GiB/s) | 20260506 12:07:21.905 | INFO | Analyzing chunk frames = [1229824,1230336) | 20260506 12:07:21.905 | INFO | MemInfo: { | 20260506 12:07:21.905 | INFO | "gpu" : " 17 GB", | 20260506 12:07:21.905 | INFO | "huge" : "", | 20260506 12:07:21.906 | INFO | "huge_shared" : " 27 GB", | 20260506 12:07:21.906 | INFO | "malloc" : " 14 GB", | 20260506 12:07:21.906 | INFO | "pinned" : " 29 GB" | 20260506 12:07:21.906 | INFO | } | 20260506 12:07:21.906 | INFO | Perf report for the last 5.14395 seconds: | 20260506 12:07:21.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:07:21.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:21.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:21.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:21.906 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260506 12:07:21.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:21.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:21.906 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:07:21.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:21.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:21.906 | INFO | Uploaded 36GiB to GPU in 2.02043s (17.818GiB/s) | 20260506 12:07:21.906 | INFO | Downloaded 34GiB to GPU in 1.94233s (17.5048GiB/s) | 20260506 12:07:27.041 | INFO | Analyzing chunk frames = [1230336,1230848) | 20260506 12:07:27.041 | INFO | MemInfo: { | 20260506 12:07:27.041 | INFO | "gpu" : " 17 GB", | 20260506 12:07:27.041 | INFO | "huge" : "", | 20260506 12:07:27.041 | INFO | "huge_shared" : " 27 GB", | 20260506 12:07:27.041 | INFO | "malloc" : " 14 GB", | 20260506 12:07:27.041 | INFO | "pinned" : " 29 GB" | 20260506 12:07:27.041 | INFO | } | 20260506 12:07:27.041 | INFO | Perf report for the last 5.1355 seconds: | 20260506 12:07:27.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:07:27.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:27.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:27.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:27.041 | INFO | ANALYSIS : 61.5%, 3.0, 95.3ms, 0 | 20260506 12:07:27.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:27.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:27.041 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:07:27.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:27.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:27.042 | INFO | Uploaded 36GiB to GPU in 2.00645s (17.9422GiB/s) | 20260506 12:07:27.042 | INFO | Downloaded 34GiB to GPU in 1.93902s (17.5346GiB/s) | 20260506 12:07:32.180 | INFO | Analyzing chunk frames = [1230848,1231360) | 20260506 12:07:32.180 | INFO | MemInfo: { | 20260506 12:07:32.180 | INFO | "gpu" : " 17 GB", | 20260506 12:07:32.180 | INFO | "huge" : "", | 20260506 12:07:32.180 | INFO | "huge_shared" : " 27 GB", | 20260506 12:07:32.180 | INFO | "malloc" : " 14 GB", | 20260506 12:07:32.180 | INFO | "pinned" : " 29 GB" | 20260506 12:07:32.180 | INFO | } | 20260506 12:07:32.180 | INFO | Perf report for the last 5.13906 seconds: | 20260506 12:07:32.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:07:32.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:32.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:32.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:32.180 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 12:07:32.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:32.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:32.180 | INFO | PRE_HQ : 33.2%, 1.0, 17.2ms, 0 | 20260506 12:07:32.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:32.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:32.181 | INFO | Uploaded 36GiB to GPU in 2.02635s (17.7659GiB/s) | 20260506 12:07:32.181 | INFO | Downloaded 34GiB to GPU in 1.9786s (17.1839GiB/s) | 20260506 12:07:37.319 | INFO | Analyzing chunk frames = [1231360,1231872) | 20260506 12:07:37.319 | INFO | MemInfo: { | 20260506 12:07:37.319 | INFO | "gpu" : " 17 GB", | 20260506 12:07:37.319 | INFO | "huge" : "", | 20260506 12:07:37.319 | INFO | "huge_shared" : " 27 GB", | 20260506 12:07:37.319 | INFO | "malloc" : " 14 GB", | 20260506 12:07:37.319 | INFO | "pinned" : " 29 GB" | 20260506 12:07:37.319 | INFO | } | 20260506 12:07:37.319 | INFO | Perf report for the last 5.13868 seconds: | 20260506 12:07:37.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:07:37.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:37.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:37.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:37.319 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:07:37.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:37.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:37.319 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:07:37.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:37.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:37.319 | INFO | Uploaded 36GiB to GPU in 2.10151s (17.1305GiB/s) | 20260506 12:07:37.319 | INFO | Downloaded 34GiB to GPU in 1.98627s (17.1175GiB/s) | 20260506 12:07:42.459 | INFO | Analyzing chunk frames = [1231872,1232384) | 20260506 12:07:42.459 | INFO | MemInfo: { | 20260506 12:07:42.459 | INFO | "gpu" : " 17 GB", | 20260506 12:07:42.459 | INFO | "huge" : "", | 20260506 12:07:42.459 | INFO | "huge_shared" : " 27 GB", | 20260506 12:07:42.459 | INFO | "malloc" : " 14 GB", | 20260506 12:07:42.459 | INFO | "pinned" : " 29 GB" | 20260506 12:07:42.459 | INFO | } | 20260506 12:07:42.460 | INFO | Perf report for the last 5.14049 seconds: | 20260506 12:07:42.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:07:42.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:42.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:42.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:42.460 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 12:07:42.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:42.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:42.460 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 12:07:42.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:42.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:42.460 | INFO | Uploaded 36GiB to GPU in 2.00603s (17.9459GiB/s) | 20260506 12:07:42.460 | INFO | Downloaded 34GiB to GPU in 2.01732s (16.8541GiB/s) | 20260506 12:07:47.598 | INFO | Analyzing chunk frames = [1232384,1232896) | 20260506 12:07:47.598 | INFO | MemInfo: { | 20260506 12:07:47.598 | INFO | "gpu" : " 17 GB", | 20260506 12:07:47.598 | INFO | "huge" : "", | 20260506 12:07:47.598 | INFO | "huge_shared" : " 27 GB", | 20260506 12:07:47.598 | INFO | "malloc" : " 14 GB", | 20260506 12:07:47.598 | INFO | "pinned" : " 29 GB" | 20260506 12:07:47.598 | INFO | } | 20260506 12:07:47.598 | INFO | Perf report for the last 5.13823 seconds: | 20260506 12:07:47.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:07:47.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:47.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:47.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:47.598 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 12:07:47.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:47.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:47.598 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260506 12:07:47.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:47.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:47.598 | INFO | Uploaded 36GiB to GPU in 2.03109s (17.7244GiB/s) | 20260506 12:07:47.598 | INFO | Downloaded 34GiB to GPU in 1.97301s (17.2326GiB/s) | 20260506 12:07:47.724 | INFO | Starting a round of Baz IO | 20260506 12:07:50.935 | INFO | Populating RTMetrics report | 20260506 12:07:50.936 | INFO | RTMetrics report populated | 20260506 12:07:50.936 | INFO | Opening tmp RTMetrics json file | 20260506 12:07:50.937 | INFO | Serializing RTMetrics to json | 20260506 12:07:50.938 | INFO | Serialization done, swapping tmp file now... | 20260506 12:07:50.938 | INFO | RTMetrics done | 20260506 12:07:51.039 | INFO | Finished a round of Baz IO: Wrote 2.083 GiB in 3.314 seconds | 20260506 12:07:51.039 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.30% / 0.03% / 57.67% | 20260506 12:07:51.039 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.33905% / 4.76837e-05% / 3.57628e-05% | 20260506 12:07:51.039 | INFO | Baz IO overall: | 20260506 12:07:51.039 | INFO | BytesWritten : 606380089541 | 20260506 12:07:51.039 | INFO | NumEvents : 292943557051 | 20260506 12:07:52.737 | INFO | Analyzing chunk frames = [1232896,1233408) | 20260506 12:07:52.737 | INFO | MemInfo: { | 20260506 12:07:52.737 | INFO | "gpu" : " 17 GB", | 20260506 12:07:52.737 | INFO | "huge" : "", | 20260506 12:07:52.737 | INFO | "huge_shared" : " 27 GB", | 20260506 12:07:52.737 | INFO | "malloc" : " 14 GB", | 20260506 12:07:52.737 | INFO | "pinned" : " 29 GB" | 20260506 12:07:52.737 | INFO | } | 20260506 12:07:52.737 | INFO | Perf report for the last 5.13923 seconds: | 20260506 12:07:52.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:07:52.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:52.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:52.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:52.737 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260506 12:07:52.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:52.737 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260506 12:07:52.737 | INFO | PRE_HQ : 62.5%, 1.9, 63.2ms, 0 | 20260506 12:07:52.737 | INFO | BAZWRITER : 63.7%, 3.7, 121.8ms, 0 | 20260506 12:07:52.737 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:07:52.737 | INFO | Uploaded 36GiB to GPU in 2.1092s (17.0681GiB/s) | 20260506 12:07:52.737 | INFO | Downloaded 38GiB to GPU in 2.20822s (17.2085GiB/s) | 20260506 12:07:57.876 | INFO | Analyzing chunk frames = [1233408,1233920) | 20260506 12:07:57.876 | INFO | MemInfo: { | 20260506 12:07:57.876 | INFO | "gpu" : " 17 GB", | 20260506 12:07:57.876 | INFO | "huge" : "", | 20260506 12:07:57.876 | INFO | "huge_shared" : " 27 GB", | 20260506 12:07:57.876 | INFO | "malloc" : " 14 GB", | 20260506 12:07:57.876 | INFO | "pinned" : " 29 GB" | 20260506 12:07:57.876 | INFO | } | 20260506 12:07:57.876 | INFO | Perf report for the last 5.13928 seconds: | 20260506 12:07:57.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:07:57.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:57.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:57.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:57.876 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 12:07:57.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:57.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:57.876 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:07:57.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:57.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:07:57.877 | INFO | Uploaded 36GiB to GPU in 2.0129s (17.8847GiB/s) | 20260506 12:07:57.877 | INFO | Downloaded 34GiB to GPU in 1.93552s (17.5663GiB/s) | 20260506 12:08:03.016 | INFO | Analyzing chunk frames = [1233920,1234432) | 20260506 12:08:03.016 | INFO | MemInfo: { | 20260506 12:08:03.016 | INFO | "gpu" : " 17 GB", | 20260506 12:08:03.016 | INFO | "huge" : "", | 20260506 12:08:03.016 | INFO | "huge_shared" : " 27 GB", | 20260506 12:08:03.016 | INFO | "malloc" : " 14 GB", | 20260506 12:08:03.016 | INFO | "pinned" : " 29 GB" | 20260506 12:08:03.016 | INFO | } | 20260506 12:08:03.016 | INFO | Perf report for the last 5.13955 seconds: | 20260506 12:08:03.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:08:03.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:03.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:03.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:03.016 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:08:03.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:03.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:03.016 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:08:03.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:03.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:03.016 | INFO | Uploaded 36GiB to GPU in 2.01614s (17.8559GiB/s) | 20260506 12:08:03.016 | INFO | Downloaded 34GiB to GPU in 1.94745s (17.4587GiB/s) | 20260506 12:08:08.155 | INFO | Analyzing chunk frames = [1234432,1234944) | 20260506 12:08:08.155 | INFO | MemInfo: { | 20260506 12:08:08.155 | INFO | "gpu" : " 17 GB", | 20260506 12:08:08.155 | INFO | "huge" : "", | 20260506 12:08:08.155 | INFO | "huge_shared" : " 27 GB", | 20260506 12:08:08.155 | INFO | "malloc" : " 14 GB", | 20260506 12:08:08.155 | INFO | "pinned" : " 29 GB" | 20260506 12:08:08.155 | INFO | } | 20260506 12:08:08.155 | INFO | Perf report for the last 5.13927 seconds: | 20260506 12:08:08.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:08:08.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:08.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:08.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:08.155 | INFO | ANALYSIS : 61.4%, 3.0, 95.2ms, 0 | 20260506 12:08:08.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:08.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:08.155 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:08:08.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:08.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:08.155 | INFO | Uploaded 36GiB to GPU in 2.03181s (17.7182GiB/s) | 20260506 12:08:08.156 | INFO | Downloaded 34GiB to GPU in 1.96766s (17.2794GiB/s) | 20260506 12:08:13.294 | INFO | Analyzing chunk frames = [1234944,1235456) | 20260506 12:08:13.294 | INFO | MemInfo: { | 20260506 12:08:13.294 | INFO | "gpu" : " 17 GB", | 20260506 12:08:13.294 | INFO | "huge" : "", | 20260506 12:08:13.295 | INFO | "huge_shared" : " 27 GB", | 20260506 12:08:13.295 | INFO | "malloc" : " 14 GB", | 20260506 12:08:13.295 | INFO | "pinned" : " 29 GB" | 20260506 12:08:13.295 | INFO | } | 20260506 12:08:13.295 | INFO | Perf report for the last 5.1395 seconds: | 20260506 12:08:13.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:08:13.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:13.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:13.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:13.295 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 12:08:13.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:13.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:13.295 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:08:13.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:13.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:13.295 | INFO | Uploaded 36GiB to GPU in 2.01378s (17.8768GiB/s) | 20260506 12:08:13.295 | INFO | Downloaded 34GiB to GPU in 1.96235s (17.3261GiB/s) | 20260506 12:08:18.434 | INFO | Analyzing chunk frames = [1235456,1235968) | 20260506 12:08:18.434 | INFO | MemInfo: { | 20260506 12:08:18.434 | INFO | "gpu" : " 17 GB", | 20260506 12:08:18.434 | INFO | "huge" : "", | 20260506 12:08:18.434 | INFO | "huge_shared" : " 27 GB", | 20260506 12:08:18.434 | INFO | "malloc" : " 14 GB", | 20260506 12:08:18.434 | INFO | "pinned" : " 29 GB" | 20260506 12:08:18.434 | INFO | } | 20260506 12:08:18.434 | INFO | Perf report for the last 5.1392 seconds: | 20260506 12:08:18.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:08:18.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:18.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:18.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:18.434 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:08:18.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:18.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:18.434 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:08:18.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:18.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:18.434 | INFO | Uploaded 36GiB to GPU in 2.10553s (17.0979GiB/s) | 20260506 12:08:18.434 | INFO | Downloaded 34GiB to GPU in 1.99501s (17.0425GiB/s) | 20260506 12:08:23.574 | INFO | Analyzing chunk frames = [1235968,1236480) | 20260506 12:08:23.574 | INFO | MemInfo: { | 20260506 12:08:23.574 | INFO | "gpu" : " 17 GB", | 20260506 12:08:23.574 | INFO | "huge" : "", | 20260506 12:08:23.574 | INFO | "huge_shared" : " 27 GB", | 20260506 12:08:23.574 | INFO | "malloc" : " 14 GB", | 20260506 12:08:23.574 | INFO | "pinned" : " 29 GB" | 20260506 12:08:23.574 | INFO | } | 20260506 12:08:23.574 | INFO | Perf report for the last 5.14002 seconds: | 20260506 12:08:23.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:08:23.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:23.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:23.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:23.574 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:08:23.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:23.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:23.574 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 12:08:23.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:23.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:23.574 | INFO | Uploaded 36GiB to GPU in 2.02153s (17.8083GiB/s) | 20260506 12:08:23.574 | INFO | Downloaded 34GiB to GPU in 2.003s (16.9745GiB/s) | 20260506 12:08:28.712 | INFO | Analyzing chunk frames = [1236480,1236992) | 20260506 12:08:28.712 | INFO | MemInfo: { | 20260506 12:08:28.712 | INFO | "gpu" : " 17 GB", | 20260506 12:08:28.712 | INFO | "huge" : "", | 20260506 12:08:28.712 | INFO | "huge_shared" : " 27 GB", | 20260506 12:08:28.712 | INFO | "malloc" : " 14 GB", | 20260506 12:08:28.712 | INFO | "pinned" : " 29 GB" | 20260506 12:08:28.712 | INFO | } | 20260506 12:08:28.713 | INFO | Perf report for the last 5.1386 seconds: | 20260506 12:08:28.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:08:28.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:28.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:28.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:28.713 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 12:08:28.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:28.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:28.713 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:08:28.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:28.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:28.713 | INFO | Uploaded 36GiB to GPU in 2.03833s (17.6615GiB/s) | 20260506 12:08:28.713 | INFO | Downloaded 34GiB to GPU in 1.97249s (17.2371GiB/s) | 20260506 12:08:28.713 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:08:28.713 | INFO | Total mem usage: 16.8 GiB | 20260506 12:08:28.713 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:08:28.713 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.6%) | 20260506 12:08:28.713 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:08:28.713 | INFO | High Water Breakdown: | 20260506 12:08:28.713 | INFO | 59.5% : 8.58 GiB ( 82.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:08:28.713 | INFO | 40.3% : 5.81 GiB ( 86.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:08:28.713 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:08:28.713 | INFO | ------------------------------------------ | 20260506 12:08:28.713 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:08:28.713 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:08:28.713 | INFO | Total mem usage: 26.5 GiB | 20260506 12:08:28.713 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:08:28.713 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:08:28.713 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:08:28.713 | INFO | High Water Breakdown: | 20260506 12:08:28.713 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:08:28.713 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:08:28.713 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:08:28.713 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:08:28.713 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:08:28.713 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:08:28.713 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:08:28.713 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:08:28.713 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:08:28.713 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:08:28.713 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:08:28.713 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:08:28.714 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:08:28.714 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:08:28.714 | INFO | ------------------------------------------ | 20260506 12:08:28.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:08:28.714 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:08:28.714 | INFO | Total mem usage: 25.6 GiB | 20260506 12:08:28.714 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:08:28.714 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:08:28.714 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:08:28.714 | INFO | High Water Breakdown: | 20260506 12:08:28.714 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:08:28.714 | INFO | ------------------------------------------ | 20260506 12:08:28.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:08:28.714 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:08:28.714 | INFO | Total mem usage: 15.7 GiB | 20260506 12:08:28.714 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:08:28.714 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:08:28.714 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 12:08:28.714 | INFO | High Water Breakdown: | 20260506 12:08:28.714 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:08:28.714 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:08:28.714 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:08:28.714 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:08:28.714 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:08:28.714 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:08:28.714 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:08:28.714 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:08:28.714 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:08:28.714 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:08:28.714 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:08:28.714 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:08:28.714 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:08:28.714 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:08:28.714 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:08:28.714 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:08:28.714 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:08:28.714 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:08:28.714 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:08:28.714 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:08:28.714 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:08:28.714 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:08:28.714 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:08:28.714 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:08:28.714 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:08:28.714 | INFO | ------------------------------------------ | 20260506 12:08:28.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:08:28.714 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:08:28.714 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:08:28.714 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:08:28.714 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.59e+01, 1.36e+01 / 2.04e+00 | 20260506 12:08:28.714 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 9.95e-02, 1.03e-02 / 5.89e-03 | 20260506 12:08:28.714 | INFO | PerfReport: 11.55%, Baseline -- 3.67e+00 / 4.49e+00, 5.76e-01 / 3.04e-01 | 20260506 12:08:28.714 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.46e+00, 1.90e-01 / 1.52e-01 | 20260506 12:08:28.714 | INFO | PerfReport: 15.83%, DME -- 4.03e+01 / 4.46e+01, 3.94e+00 / 2.04e+00 | 20260506 12:08:28.714 | INFO | PerfReport: 56.28%, FrameLabeling -- 1.79e+01 / 2.01e+01, 1.38e+00 / 1.69e+00 | 20260506 12:08:28.714 | INFO | PerfReport: 7.67%, PulseAccumulating -- 2.44e+00 / 3.10e+00, 4.33e-01 / 2.19e-01 | 20260506 12:08:28.714 | INFO | PerfReport: 3.80%, Metrics -- 1.21e+00 / 1.59e+00, 3.04e-01 / 1.64e-01 | 20260506 12:08:28.714 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.26e-01 / 4.80e-01, 6.60e-02 / 2.36e-02 | 20260506 12:08:28.714 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:08:28.714 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:08:28.714 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:08:28.714 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 12:08:28.714 | INFO | PerfReport: 21.46%, Upload -- 2.13e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260506 12:08:28.714 | INFO | PerfReport: 21.02%, Download -- 2.08e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 12:08:28.714 | INFO | PerfReport: 32.11%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 12:08:28.714 | INFO | PerfReport: 1.81%, LockUpload -- 1.79e+00 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260506 12:08:28.714 | INFO | PerfReport: 1.07%, LockDownload -- 1.06e+00 / 0.00e+00, 2.66e+00 / 0.00e+00 | 20260506 12:08:28.714 | INFO | PerfReport: 22.51%, LockCompute -- 2.23e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260506 12:08:28.714 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260506 12:08:28.841 | INFO | Starting a round of Baz IO | 20260506 12:08:32.017 | INFO | Populating RTMetrics report | 20260506 12:08:32.017 | INFO | RTMetrics report populated | 20260506 12:08:32.017 | INFO | Opening tmp RTMetrics json file | 20260506 12:08:32.018 | INFO | Serializing RTMetrics to json | 20260506 12:08:32.019 | INFO | Serialization done, swapping tmp file now... | 20260506 12:08:32.020 | INFO | RTMetrics done | 20260506 12:08:32.112 | INFO | Finished a round of Baz IO: Wrote 2.087 GiB in 3.271 seconds | 20260506 12:08:32.112 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.28% / 0.03% / 57.70% | 20260506 12:08:32.112 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.34144% / 5.96046e-05% / 3.97364e-05% | 20260506 12:08:32.112 | INFO | Baz IO overall: | 20260506 12:08:32.112 | INFO | BytesWritten : 608621315094 | 20260506 12:08:32.112 | INFO | NumEvents : 294031416351 | 20260506 12:08:33.852 | INFO | Analyzing chunk frames = [1236992,1237504) | 20260506 12:08:33.852 | INFO | MemInfo: { | 20260506 12:08:33.852 | INFO | "gpu" : " 17 GB", | 20260506 12:08:33.852 | INFO | "huge" : "", | 20260506 12:08:33.852 | INFO | "huge_shared" : " 27 GB", | 20260506 12:08:33.852 | INFO | "malloc" : " 14 GB", | 20260506 12:08:33.852 | INFO | "pinned" : " 29 GB" | 20260506 12:08:33.852 | INFO | } | 20260506 12:08:33.852 | INFO | Perf report for the last 5.13909 seconds: | 20260506 12:08:33.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:08:33.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:33.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:33.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:33.852 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260506 12:08:33.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:33.852 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:08:33.852 | INFO | PRE_HQ : 61.9%, 1.9, 62.3ms, 0 | 20260506 12:08:33.852 | INFO | BAZWRITER : 62.8%, 3.6, 117.7ms, 0 | 20260506 12:08:33.852 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:08:33.852 | INFO | Uploaded 36GiB to GPU in 2.04854s (17.5735GiB/s) | 20260506 12:08:33.852 | INFO | Downloaded 38GiB to GPU in 2.21225s (17.1771GiB/s) | 20260506 12:08:38.991 | INFO | Analyzing chunk frames = [1237504,1238016) | 20260506 12:08:38.991 | INFO | MemInfo: { | 20260506 12:08:38.991 | INFO | "gpu" : " 17 GB", | 20260506 12:08:38.991 | INFO | "huge" : "", | 20260506 12:08:38.991 | INFO | "huge_shared" : " 27 GB", | 20260506 12:08:38.991 | INFO | "malloc" : " 14 GB", | 20260506 12:08:38.991 | INFO | "pinned" : " 29 GB" | 20260506 12:08:38.991 | INFO | } | 20260506 12:08:38.991 | INFO | Perf report for the last 5.13965 seconds: | 20260506 12:08:38.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:08:38.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:38.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:38.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:38.991 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 12:08:38.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:38.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:38.991 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:08:38.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:38.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:38.992 | INFO | Uploaded 36GiB to GPU in 2.01644s (17.8532GiB/s) | 20260506 12:08:38.992 | INFO | Downloaded 34GiB to GPU in 1.92655s (17.6482GiB/s) | 20260506 12:08:44.131 | INFO | Analyzing chunk frames = [1238016,1238528) | 20260506 12:08:44.131 | INFO | MemInfo: { | 20260506 12:08:44.131 | INFO | "gpu" : " 17 GB", | 20260506 12:08:44.131 | INFO | "huge" : "", | 20260506 12:08:44.131 | INFO | "huge_shared" : " 27 GB", | 20260506 12:08:44.131 | INFO | "malloc" : " 14 GB", | 20260506 12:08:44.131 | INFO | "pinned" : " 29 GB" | 20260506 12:08:44.131 | INFO | } | 20260506 12:08:44.131 | INFO | Perf report for the last 5.13956 seconds: | 20260506 12:08:44.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:08:44.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:44.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:44.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:44.131 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:08:44.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:44.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:44.131 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 12:08:44.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:44.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:44.131 | INFO | Uploaded 36GiB to GPU in 1.99557s (18.04GiB/s) | 20260506 12:08:44.131 | INFO | Downloaded 34GiB to GPU in 1.94532s (17.4779GiB/s) | 20260506 12:08:49.270 | INFO | Analyzing chunk frames = [1238528,1239040) | 20260506 12:08:49.270 | INFO | MemInfo: { | 20260506 12:08:49.270 | INFO | "gpu" : " 17 GB", | 20260506 12:08:49.270 | INFO | "huge" : "", | 20260506 12:08:49.270 | INFO | "huge_shared" : " 27 GB", | 20260506 12:08:49.270 | INFO | "malloc" : " 14 GB", | 20260506 12:08:49.270 | INFO | "pinned" : " 29 GB" | 20260506 12:08:49.270 | INFO | } | 20260506 12:08:49.270 | INFO | Perf report for the last 5.13911 seconds: | 20260506 12:08:49.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:08:49.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:49.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:49.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:49.270 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 12:08:49.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:49.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:49.270 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:08:49.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:49.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:49.270 | INFO | Uploaded 36GiB to GPU in 2.00617s (17.9446GiB/s) | 20260506 12:08:49.270 | INFO | Downloaded 34GiB to GPU in 1.95621s (17.3806GiB/s) | 20260506 12:08:54.409 | INFO | Analyzing chunk frames = [1239040,1239552) | 20260506 12:08:54.409 | INFO | MemInfo: { | 20260506 12:08:54.409 | INFO | "gpu" : " 17 GB", | 20260506 12:08:54.409 | INFO | "huge" : "", | 20260506 12:08:54.409 | INFO | "huge_shared" : " 27 GB", | 20260506 12:08:54.409 | INFO | "malloc" : " 14 GB", | 20260506 12:08:54.409 | INFO | "pinned" : " 29 GB" | 20260506 12:08:54.409 | INFO | } | 20260506 12:08:54.409 | INFO | Perf report for the last 5.13928 seconds: | 20260506 12:08:54.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:08:54.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:54.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:54.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:54.409 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 12:08:54.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:54.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:54.409 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:08:54.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:54.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:54.410 | INFO | Uploaded 36GiB to GPU in 2.03726s (17.6708GiB/s) | 20260506 12:08:54.410 | INFO | Downloaded 34GiB to GPU in 1.97053s (17.2543GiB/s) | 20260506 12:08:59.549 | INFO | Analyzing chunk frames = [1239552,1240064) | 20260506 12:08:59.549 | INFO | MemInfo: { | 20260506 12:08:59.549 | INFO | "gpu" : " 17 GB", | 20260506 12:08:59.549 | INFO | "huge" : "", | 20260506 12:08:59.549 | INFO | "huge_shared" : " 27 GB", | 20260506 12:08:59.549 | INFO | "malloc" : " 14 GB", | 20260506 12:08:59.549 | INFO | "pinned" : " 29 GB" | 20260506 12:08:59.549 | INFO | } | 20260506 12:08:59.549 | INFO | Perf report for the last 5.13949 seconds: | 20260506 12:08:59.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:08:59.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:59.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:59.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:59.549 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 12:08:59.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:59.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:59.549 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260506 12:08:59.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:59.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:08:59.549 | INFO | Uploaded 36GiB to GPU in 2.03909s (17.655GiB/s) | 20260506 12:08:59.549 | INFO | Downloaded 34GiB to GPU in 1.92453s (17.6667GiB/s) | 20260506 12:09:04.688 | INFO | Analyzing chunk frames = [1240064,1240576) | 20260506 12:09:04.688 | INFO | MemInfo: { | 20260506 12:09:04.688 | INFO | "gpu" : " 17 GB", | 20260506 12:09:04.688 | INFO | "huge" : "", | 20260506 12:09:04.688 | INFO | "huge_shared" : " 27 GB", | 20260506 12:09:04.688 | INFO | "malloc" : " 14 GB", | 20260506 12:09:04.688 | INFO | "pinned" : " 29 GB" | 20260506 12:09:04.688 | INFO | } | 20260506 12:09:04.688 | INFO | Perf report for the last 5.13952 seconds: | 20260506 12:09:04.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:09:04.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:04.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:04.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:04.688 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:09:04.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:04.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:04.688 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 12:09:04.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:04.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:04.689 | INFO | Uploaded 36GiB to GPU in 1.98651s (18.1222GiB/s) | 20260506 12:09:04.689 | INFO | Downloaded 34GiB to GPU in 2.03347s (16.7202GiB/s) | 20260506 12:09:09.827 | INFO | Analyzing chunk frames = [1240576,1241088) | 20260506 12:09:09.827 | INFO | MemInfo: { | 20260506 12:09:09.827 | INFO | "gpu" : " 17 GB", | 20260506 12:09:09.827 | INFO | "huge" : "", | 20260506 12:09:09.827 | INFO | "huge_shared" : " 27 GB", | 20260506 12:09:09.827 | INFO | "malloc" : " 14 GB", | 20260506 12:09:09.827 | INFO | "pinned" : " 29 GB" | 20260506 12:09:09.827 | INFO | } | 20260506 12:09:09.827 | INFO | Perf report for the last 5.13876 seconds: | 20260506 12:09:09.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:09:09.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:09.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:09.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:09.827 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 12:09:09.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:09.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:09.827 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:09:09.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:09.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:09.828 | INFO | Uploaded 36GiB to GPU in 2.00653s (17.9414GiB/s) | 20260506 12:09:09.828 | INFO | Downloaded 34GiB to GPU in 1.96252s (17.3247GiB/s) | 20260506 12:09:09.959 | INFO | Starting a round of Baz IO | 20260506 12:09:13.122 | INFO | Populating RTMetrics report | 20260506 12:09:13.123 | INFO | RTMetrics report populated | 20260506 12:09:13.123 | INFO | Opening tmp RTMetrics json file | 20260506 12:09:13.123 | INFO | Serializing RTMetrics to json | 20260506 12:09:13.124 | INFO | Serialization done, swapping tmp file now... | 20260506 12:09:13.125 | INFO | RTMetrics done | 20260506 12:09:13.228 | INFO | Finished a round of Baz IO: Wrote 2.086 GiB in 3.269 seconds | 20260506 12:09:13.228 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.25% / 0.03% / 57.72% | 20260506 12:09:13.228 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.32009% / 6.75519e-05% / 4.76837e-05% | 20260506 12:09:13.228 | INFO | Baz IO overall: | 20260506 12:09:13.228 | INFO | BytesWritten : 610860618555 | 20260506 12:09:13.228 | INFO | NumEvents : 295117536845 | 20260506 12:09:14.966 | INFO | Analyzing chunk frames = [1241088,1241600) | 20260506 12:09:14.966 | INFO | MemInfo: { | 20260506 12:09:14.966 | INFO | "gpu" : " 17 GB", | 20260506 12:09:14.966 | INFO | "huge" : "", | 20260506 12:09:14.966 | INFO | "huge_shared" : " 27 GB", | 20260506 12:09:14.966 | INFO | "malloc" : " 14 GB", | 20260506 12:09:14.967 | INFO | "pinned" : " 29 GB" | 20260506 12:09:14.967 | INFO | } | 20260506 12:09:14.967 | INFO | Perf report for the last 5.13952 seconds: | 20260506 12:09:14.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:09:14.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:14.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:14.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:14.967 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 12:09:14.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:14.967 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:09:14.967 | INFO | PRE_HQ : 61.4%, 1.9, 61.5ms, 0 | 20260506 12:09:14.967 | INFO | BAZWRITER : 63.0%, 3.7, 119.0ms, 0 | 20260506 12:09:14.967 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:09:14.967 | INFO | Uploaded 36GiB to GPU in 2.07583s (17.3424GiB/s) | 20260506 12:09:14.967 | INFO | Downloaded 38GiB to GPU in 2.21376s (17.1653GiB/s) | 20260506 12:09:20.106 | INFO | Analyzing chunk frames = [1241600,1242112) | 20260506 12:09:20.106 | INFO | MemInfo: { | 20260506 12:09:20.106 | INFO | "gpu" : " 17 GB", | 20260506 12:09:20.106 | INFO | "huge" : "", | 20260506 12:09:20.106 | INFO | "huge_shared" : " 27 GB", | 20260506 12:09:20.106 | INFO | "malloc" : " 14 GB", | 20260506 12:09:20.106 | INFO | "pinned" : " 29 GB" | 20260506 12:09:20.106 | INFO | } | 20260506 12:09:20.106 | INFO | Perf report for the last 5.13936 seconds: | 20260506 12:09:20.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:09:20.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:20.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:20.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:20.106 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:09:20.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:20.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:20.106 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:09:20.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:20.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:20.106 | INFO | Uploaded 36GiB to GPU in 2.01138s (17.8982GiB/s) | 20260506 12:09:20.106 | INFO | Downloaded 34GiB to GPU in 1.90081s (17.8872GiB/s) | 20260506 12:09:25.245 | INFO | Analyzing chunk frames = [1242112,1242624) | 20260506 12:09:25.245 | INFO | MemInfo: { | 20260506 12:09:25.245 | INFO | "gpu" : " 17 GB", | 20260506 12:09:25.245 | INFO | "huge" : "", | 20260506 12:09:25.245 | INFO | "huge_shared" : " 27 GB", | 20260506 12:09:25.245 | INFO | "malloc" : " 14 GB", | 20260506 12:09:25.245 | INFO | "pinned" : " 29 GB" | 20260506 12:09:25.245 | INFO | } | 20260506 12:09:25.245 | INFO | Perf report for the last 5.13905 seconds: | 20260506 12:09:25.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:09:25.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:25.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:25.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:25.245 | INFO | ANALYSIS : 61.9%, 3.0, 95.8ms, 0 | 20260506 12:09:25.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:25.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:25.245 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:09:25.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:25.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:25.245 | INFO | Uploaded 36GiB to GPU in 2.03552s (17.6859GiB/s) | 20260506 12:09:25.246 | INFO | Downloaded 34GiB to GPU in 1.93945s (17.5307GiB/s) | 20260506 12:09:30.384 | INFO | Analyzing chunk frames = [1242624,1243136) | 20260506 12:09:30.384 | INFO | MemInfo: { | 20260506 12:09:30.384 | INFO | "gpu" : " 17 GB", | 20260506 12:09:30.384 | INFO | "huge" : "", | 20260506 12:09:30.384 | INFO | "huge_shared" : " 27 GB", | 20260506 12:09:30.384 | INFO | "malloc" : " 14 GB", | 20260506 12:09:30.384 | INFO | "pinned" : " 29 GB" | 20260506 12:09:30.384 | INFO | } | 20260506 12:09:30.384 | INFO | Perf report for the last 5.13941 seconds: | 20260506 12:09:30.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:09:30.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:30.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:30.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:30.385 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:09:30.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:30.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:30.385 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:09:30.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:30.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:30.385 | INFO | Uploaded 36GiB to GPU in 2.07916s (17.3147GiB/s) | 20260506 12:09:30.385 | INFO | Downloaded 34GiB to GPU in 1.96925s (17.2655GiB/s) | 20260506 12:09:35.524 | INFO | Analyzing chunk frames = [1243136,1243648) | 20260506 12:09:35.524 | INFO | MemInfo: { | 20260506 12:09:35.524 | INFO | "gpu" : " 17 GB", | 20260506 12:09:35.524 | INFO | "huge" : "", | 20260506 12:09:35.524 | INFO | "huge_shared" : " 27 GB", | 20260506 12:09:35.524 | INFO | "malloc" : " 14 GB", | 20260506 12:09:35.524 | INFO | "pinned" : " 29 GB" | 20260506 12:09:35.524 | INFO | } | 20260506 12:09:35.524 | INFO | Perf report for the last 5.13935 seconds: | 20260506 12:09:35.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:09:35.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:35.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:35.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:35.524 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 12:09:35.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:35.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:35.524 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:09:35.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:35.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:35.524 | INFO | Uploaded 36GiB to GPU in 2.10533s (17.0994GiB/s) | 20260506 12:09:35.524 | INFO | Downloaded 34GiB to GPU in 1.9789s (17.1813GiB/s) | 20260506 12:09:40.663 | INFO | Analyzing chunk frames = [1243648,1244160) | 20260506 12:09:40.663 | INFO | MemInfo: { | 20260506 12:09:40.663 | INFO | "gpu" : " 17 GB", | 20260506 12:09:40.663 | INFO | "huge" : "", | 20260506 12:09:40.663 | INFO | "huge_shared" : " 27 GB", | 20260506 12:09:40.663 | INFO | "malloc" : " 14 GB", | 20260506 12:09:40.663 | INFO | "pinned" : " 29 GB" | 20260506 12:09:40.663 | INFO | } | 20260506 12:09:40.663 | INFO | Perf report for the last 5.13908 seconds: | 20260506 12:09:40.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:09:40.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:40.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:40.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:40.663 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:09:40.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:40.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:40.663 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260506 12:09:40.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:40.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:40.663 | INFO | Uploaded 36GiB to GPU in 2.09461s (17.187GiB/s) | 20260506 12:09:40.664 | INFO | Downloaded 34GiB to GPU in 2.00108s (16.9908GiB/s) | 20260506 12:09:45.803 | INFO | Analyzing chunk frames = [1244160,1244672) | 20260506 12:09:45.803 | INFO | MemInfo: { | 20260506 12:09:45.803 | INFO | "gpu" : " 17 GB", | 20260506 12:09:45.803 | INFO | "huge" : "", | 20260506 12:09:45.803 | INFO | "huge_shared" : " 27 GB", | 20260506 12:09:45.803 | INFO | "malloc" : " 14 GB", | 20260506 12:09:45.803 | INFO | "pinned" : " 29 GB" | 20260506 12:09:45.803 | INFO | } | 20260506 12:09:45.803 | INFO | Perf report for the last 5.14022 seconds: | 20260506 12:09:45.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:09:45.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:45.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:45.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:45.803 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:09:45.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:45.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:45.803 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 12:09:45.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:45.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:45.804 | INFO | Uploaded 36GiB to GPU in 2.02352s (17.7907GiB/s) | 20260506 12:09:45.804 | INFO | Downloaded 34GiB to GPU in 2.01113s (16.9059GiB/s) | 20260506 12:09:50.942 | INFO | Analyzing chunk frames = [1244672,1245184) | 20260506 12:09:50.942 | INFO | MemInfo: { | 20260506 12:09:50.942 | INFO | "gpu" : " 17 GB", | 20260506 12:09:50.942 | INFO | "huge" : "", | 20260506 12:09:50.942 | INFO | "huge_shared" : " 27 GB", | 20260506 12:09:50.942 | INFO | "malloc" : " 14 GB", | 20260506 12:09:50.942 | INFO | "pinned" : " 29 GB" | 20260506 12:09:50.942 | INFO | } | 20260506 12:09:50.942 | INFO | Perf report for the last 5.13878 seconds: | 20260506 12:09:50.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:09:50.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:50.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:50.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:50.942 | INFO | ANALYSIS : 62.9%, 3.0, 98.1ms, 0 | 20260506 12:09:50.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:50.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:50.942 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:09:50.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:50.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:50.942 | INFO | Uploaded 36GiB to GPU in 2.03828s (17.662GiB/s) | 20260506 12:09:50.942 | INFO | Downloaded 34GiB to GPU in 1.96348s (17.3162GiB/s) | 20260506 12:09:50.942 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:09:50.943 | INFO | Total mem usage: 16.8 GiB | 20260506 12:09:50.943 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:09:50.943 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.6%) | 20260506 12:09:50.943 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:09:50.943 | INFO | High Water Breakdown: | 20260506 12:09:50.943 | INFO | 59.6% : 8.59 GiB ( 82.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:09:50.943 | INFO | 40.3% : 5.81 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:09:50.943 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:09:50.943 | INFO | ------------------------------------------ | 20260506 12:09:50.943 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:09:50.943 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:09:50.943 | INFO | Total mem usage: 26.5 GiB | 20260506 12:09:50.943 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:09:50.943 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:09:50.943 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:09:50.943 | INFO | High Water Breakdown: | 20260506 12:09:50.943 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:09:50.943 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:09:50.943 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:09:50.943 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:09:50.943 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:09:50.943 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:09:50.943 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:09:50.943 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:09:50.943 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:09:50.943 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:09:50.943 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:09:50.943 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:09:50.943 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:09:50.943 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:09:50.943 | INFO | ------------------------------------------ | 20260506 12:09:50.943 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:09:50.943 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:09:50.943 | INFO | Total mem usage: 25.6 GiB | 20260506 12:09:50.943 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:09:50.943 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:09:50.943 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:09:50.943 | INFO | High Water Breakdown: | 20260506 12:09:50.943 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:09:50.943 | INFO | ------------------------------------------ | 20260506 12:09:50.943 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:09:50.943 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:09:50.943 | INFO | Total mem usage: 15.7 GiB | 20260506 12:09:50.943 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:09:50.943 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:09:50.943 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:09:50.943 | INFO | High Water Breakdown: | 20260506 12:09:50.943 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:09:50.943 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:09:50.943 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:09:50.943 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:09:50.943 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:09:50.943 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:09:50.943 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:09:50.943 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:09:50.943 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:09:50.943 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:09:50.943 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:09:50.943 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:09:50.943 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:09:50.943 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:09:50.943 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:09:50.943 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:09:50.944 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:09:50.944 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:09:50.944 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:09:50.944 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:09:50.944 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:09:50.944 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:09:50.944 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:09:50.944 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:09:50.944 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:09:50.944 | INFO | ------------------------------------------ | 20260506 12:09:50.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:09:50.944 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:09:50.944 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:09:50.944 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:09:50.944 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.60e+01, 1.36e+01 / 2.49e+00 | 20260506 12:09:50.944 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.01e-01, 1.01e-02 / 8.83e-03 | 20260506 12:09:50.944 | INFO | PerfReport: 11.56%, Baseline -- 3.68e+00 / 4.45e+00, 5.91e-01 / 3.95e-01 | 20260506 12:09:50.944 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.45e+00, 1.92e-01 / 1.67e-01 | 20260506 12:09:50.944 | INFO | PerfReport: 15.85%, DME -- 4.04e+01 / 4.47e+01, 4.09e+00 / 2.71e+00 | 20260506 12:09:50.944 | INFO | PerfReport: 56.31%, FrameLabeling -- 1.79e+01 / 2.02e+01, 1.46e+00 / 1.75e+00 | 20260506 12:09:50.944 | INFO | PerfReport: 7.63%, PulseAccumulating -- 2.43e+00 / 3.09e+00, 4.28e-01 / 2.01e-01 | 20260506 12:09:50.944 | INFO | PerfReport: 3.78%, Metrics -- 1.21e+00 / 1.59e+00, 3.05e-01 / 1.70e-01 | 20260506 12:09:50.944 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.26e-01 / 4.78e-01, 6.64e-02 / 2.53e-02 | 20260506 12:09:50.944 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:09:50.944 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:09:50.944 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:09:50.944 | INFO | PerfReport: Full Execution -- 9.91e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260506 12:09:50.944 | INFO | PerfReport: 21.42%, Upload -- 2.12e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 12:09:50.944 | INFO | PerfReport: 20.97%, Download -- 2.08e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 12:09:50.944 | INFO | PerfReport: 32.17%, Compute -- 3.19e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 12:09:50.944 | INFO | PerfReport: 1.89%, LockUpload -- 1.87e+00 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 12:09:50.944 | INFO | PerfReport: 1.00%, LockDownload -- 9.92e-01 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 12:09:50.944 | INFO | PerfReport: 22.53%, LockCompute -- 2.23e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 12:09:50.944 | INFO | PerfReport: 0.02%, Misc -- 1.50e-02 / 0.00e+00, 2.22e-02 / 0.00e+00 | 20260506 12:09:51.073 | INFO | Starting a round of Baz IO | 20260506 12:09:54.239 | INFO | Populating RTMetrics report | 20260506 12:09:54.240 | INFO | RTMetrics report populated | 20260506 12:09:54.240 | INFO | Opening tmp RTMetrics json file | 20260506 12:09:54.240 | INFO | Serializing RTMetrics to json | 20260506 12:09:54.242 | INFO | Serialization done, swapping tmp file now... | 20260506 12:09:54.242 | INFO | RTMetrics done | 20260506 12:09:54.338 | INFO | Finished a round of Baz IO: Wrote 2.085 GiB in 3.265 seconds | 20260506 12:09:54.339 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.22% / 0.03% / 57.75% | 20260506 12:09:54.339 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.30197% / 3.17891e-05% / 3.17891e-05% | 20260506 12:09:54.339 | INFO | Baz IO overall: | 20260506 12:09:54.339 | INFO | BytesWritten : 613099529952 | 20260506 12:09:54.339 | INFO | NumEvents : 296203291828 | 20260506 12:09:56.081 | INFO | Analyzing chunk frames = [1245184,1245696) | 20260506 12:09:56.081 | INFO | MemInfo: { | 20260506 12:09:56.081 | INFO | "gpu" : " 17 GB", | 20260506 12:09:56.081 | INFO | "huge" : "", | 20260506 12:09:56.081 | INFO | "huge_shared" : " 27 GB", | 20260506 12:09:56.081 | INFO | "malloc" : " 14 GB", | 20260506 12:09:56.081 | INFO | "pinned" : " 29 GB" | 20260506 12:09:56.081 | INFO | } | 20260506 12:09:56.081 | INFO | Perf report for the last 5.13934 seconds: | 20260506 12:09:56.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:09:56.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:56.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:56.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:56.081 | INFO | ANALYSIS : 63.6%, 3.0, 98.7ms, 0 | 20260506 12:09:56.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:09:56.082 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:09:56.082 | INFO | PRE_HQ : 61.5%, 1.9, 61.2ms, 0 | 20260506 12:09:56.082 | INFO | BAZWRITER : 62.9%, 3.7, 118.1ms, 0 | 20260506 12:09:56.082 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:09:56.082 | INFO | Uploaded 36GiB to GPU in 2.09057s (17.2202GiB/s) | 20260506 12:09:56.082 | INFO | Downloaded 38GiB to GPU in 2.21327s (17.1692GiB/s) | 20260506 12:10:01.220 | INFO | Analyzing chunk frames = [1245696,1246208) | 20260506 12:10:01.220 | INFO | MemInfo: { | 20260506 12:10:01.220 | INFO | "gpu" : " 17 GB", | 20260506 12:10:01.220 | INFO | "huge" : "", | 20260506 12:10:01.220 | INFO | "huge_shared" : " 27 GB", | 20260506 12:10:01.220 | INFO | "malloc" : " 14 GB", | 20260506 12:10:01.220 | INFO | "pinned" : " 29 GB" | 20260506 12:10:01.220 | INFO | } | 20260506 12:10:01.220 | INFO | Perf report for the last 5.13913 seconds: | 20260506 12:10:01.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:10:01.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:01.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:01.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:01.221 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 12:10:01.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:01.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:01.221 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:10:01.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:01.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:01.221 | INFO | Uploaded 36GiB to GPU in 2.02422s (17.7847GiB/s) | 20260506 12:10:01.221 | INFO | Downloaded 34GiB to GPU in 1.92271s (17.6834GiB/s) | 20260506 12:10:06.360 | INFO | Analyzing chunk frames = [1246208,1246720) | 20260506 12:10:06.360 | INFO | MemInfo: { | 20260506 12:10:06.360 | INFO | "gpu" : " 17 GB", | 20260506 12:10:06.360 | INFO | "huge" : "", | 20260506 12:10:06.360 | INFO | "huge_shared" : " 27 GB", | 20260506 12:10:06.360 | INFO | "malloc" : " 14 GB", | 20260506 12:10:06.360 | INFO | "pinned" : " 29 GB" | 20260506 12:10:06.360 | INFO | } | 20260506 12:10:06.360 | INFO | Perf report for the last 5.13966 seconds: | 20260506 12:10:06.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:10:06.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:06.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:06.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:06.360 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260506 12:10:06.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:06.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:06.360 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:10:06.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:06.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:06.360 | INFO | Uploaded 36GiB to GPU in 2.03573s (17.6841GiB/s) | 20260506 12:10:06.360 | INFO | Downloaded 34GiB to GPU in 1.93588s (17.5631GiB/s) | 20260506 12:10:11.499 | INFO | Analyzing chunk frames = [1246720,1247232) | 20260506 12:10:11.499 | INFO | MemInfo: { | 20260506 12:10:11.499 | INFO | "gpu" : " 17 GB", | 20260506 12:10:11.499 | INFO | "huge" : "", | 20260506 12:10:11.499 | INFO | "huge_shared" : " 27 GB", | 20260506 12:10:11.499 | INFO | "malloc" : " 14 GB", | 20260506 12:10:11.499 | INFO | "pinned" : " 29 GB" | 20260506 12:10:11.499 | INFO | } | 20260506 12:10:11.499 | INFO | Perf report for the last 5.1393 seconds: | 20260506 12:10:11.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:10:11.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:11.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:11.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:11.499 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 12:10:11.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:11.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:11.500 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:10:11.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:11.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:11.500 | INFO | Uploaded 36GiB to GPU in 2.08935s (17.2303GiB/s) | 20260506 12:10:11.500 | INFO | Downloaded 34GiB to GPU in 1.96519s (17.3011GiB/s) | 20260506 12:10:16.639 | INFO | Analyzing chunk frames = [1247232,1247744) | 20260506 12:10:16.639 | INFO | MemInfo: { | 20260506 12:10:16.639 | INFO | "gpu" : " 17 GB", | 20260506 12:10:16.639 | INFO | "huge" : "", | 20260506 12:10:16.639 | INFO | "huge_shared" : " 27 GB", | 20260506 12:10:16.639 | INFO | "malloc" : " 14 GB", | 20260506 12:10:16.639 | INFO | "pinned" : " 29 GB" | 20260506 12:10:16.639 | INFO | } | 20260506 12:10:16.639 | INFO | Perf report for the last 5.13933 seconds: | 20260506 12:10:16.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:10:16.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:16.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:16.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:16.639 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 12:10:16.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:16.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:16.639 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:10:16.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:16.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:16.639 | INFO | Uploaded 36GiB to GPU in 2.05706s (17.5007GiB/s) | 20260506 12:10:16.639 | INFO | Downloaded 34GiB to GPU in 1.975s (17.2152GiB/s) | 20260506 12:10:21.783 | INFO | Analyzing chunk frames = [1247744,1248256) | 20260506 12:10:21.783 | INFO | MemInfo: { | 20260506 12:10:21.783 | INFO | "gpu" : " 17 GB", | 20260506 12:10:21.783 | INFO | "huge" : "", | 20260506 12:10:21.783 | INFO | "huge_shared" : " 27 GB", | 20260506 12:10:21.783 | INFO | "malloc" : " 14 GB", | 20260506 12:10:21.783 | INFO | "pinned" : " 29 GB" | 20260506 12:10:21.783 | INFO | } | 20260506 12:10:21.783 | INFO | Perf report for the last 5.14469 seconds: | 20260506 12:10:21.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:10:21.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:21.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:21.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:21.784 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 12:10:21.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:21.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:21.784 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:10:21.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:21.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:21.784 | INFO | Uploaded 36GiB to GPU in 2.06823s (17.4062GiB/s) | 20260506 12:10:21.784 | INFO | Downloaded 34GiB to GPU in 1.9345s (17.5756GiB/s) | 20260506 12:10:26.918 | INFO | Analyzing chunk frames = [1248256,1248768) | 20260506 12:10:26.918 | INFO | MemInfo: { | 20260506 12:10:26.918 | INFO | "gpu" : " 17 GB", | 20260506 12:10:26.918 | INFO | "huge" : "", | 20260506 12:10:26.918 | INFO | "huge_shared" : " 27 GB", | 20260506 12:10:26.918 | INFO | "malloc" : " 14 GB", | 20260506 12:10:26.918 | INFO | "pinned" : " 29 GB" | 20260506 12:10:26.918 | INFO | } | 20260506 12:10:26.918 | INFO | Perf report for the last 5.13492 seconds: | 20260506 12:10:26.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:10:26.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:26.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:26.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:26.918 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 12:10:26.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:26.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:26.918 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 12:10:26.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:26.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:26.919 | INFO | Uploaded 36GiB to GPU in 1.97127s (18.2623GiB/s) | 20260506 12:10:26.919 | INFO | Downloaded 34GiB to GPU in 2.01576s (16.8671GiB/s) | 20260506 12:10:32.056 | INFO | Analyzing chunk frames = [1248768,1249280) | 20260506 12:10:32.056 | INFO | MemInfo: { | 20260506 12:10:32.056 | INFO | "gpu" : " 17 GB", | 20260506 12:10:32.056 | INFO | "huge" : "", | 20260506 12:10:32.056 | INFO | "huge_shared" : " 27 GB", | 20260506 12:10:32.056 | INFO | "malloc" : " 14 GB", | 20260506 12:10:32.056 | INFO | "pinned" : " 29 GB" | 20260506 12:10:32.056 | INFO | } | 20260506 12:10:32.057 | INFO | Perf report for the last 5.13811 seconds: | 20260506 12:10:32.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:10:32.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:32.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:32.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:32.057 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 12:10:32.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:32.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:32.057 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:10:32.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:32.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:32.057 | INFO | Uploaded 36GiB to GPU in 2.00478s (17.9571GiB/s) | 20260506 12:10:32.057 | INFO | Downloaded 34GiB to GPU in 2.00083s (16.993GiB/s) | 20260506 12:10:32.188 | INFO | Starting a round of Baz IO | 20260506 12:10:35.358 | INFO | Populating RTMetrics report | 20260506 12:10:35.358 | INFO | RTMetrics report populated | 20260506 12:10:35.358 | INFO | Opening tmp RTMetrics json file | 20260506 12:10:35.359 | INFO | Serializing RTMetrics to json | 20260506 12:10:35.360 | INFO | Serialization done, swapping tmp file now... | 20260506 12:10:35.360 | INFO | RTMetrics done | 20260506 12:10:35.460 | INFO | Finished a round of Baz IO: Wrote 2.086 GiB in 3.272 seconds | 20260506 12:10:35.460 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.19% / 0.03% / 57.78% | 20260506 12:10:35.460 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.29381% / 4.76837e-05% / 6.35783e-05% | 20260506 12:10:35.460 | INFO | Baz IO overall: | 20260506 12:10:35.460 | INFO | BytesWritten : 615339671259 | 20260506 12:10:35.460 | INFO | NumEvents : 297289971717 | 20260506 12:10:37.197 | INFO | Analyzing chunk frames = [1249280,1249792) | 20260506 12:10:37.197 | INFO | MemInfo: { | 20260506 12:10:37.197 | INFO | "gpu" : " 17 GB", | 20260506 12:10:37.197 | INFO | "huge" : "", | 20260506 12:10:37.197 | INFO | "huge_shared" : " 27 GB", | 20260506 12:10:37.197 | INFO | "malloc" : " 14 GB", | 20260506 12:10:37.197 | INFO | "pinned" : " 29 GB" | 20260506 12:10:37.197 | INFO | } | 20260506 12:10:37.197 | INFO | Perf report for the last 5.14028 seconds: | 20260506 12:10:37.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:10:37.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:37.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:37.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:37.197 | INFO | ANALYSIS : 63.6%, 3.0, 98.7ms, 0 | 20260506 12:10:37.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:37.197 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:10:37.197 | INFO | PRE_HQ : 61.5%, 1.9, 61.6ms, 0 | 20260506 12:10:37.197 | INFO | BAZWRITER : 63.1%, 3.7, 121.5ms, 0 | 20260506 12:10:37.197 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:10:37.197 | INFO | Uploaded 36GiB to GPU in 2.03243s (17.7128GiB/s) | 20260506 12:10:37.197 | INFO | Downloaded 38GiB to GPU in 2.21565s (17.1507GiB/s) | 20260506 12:10:42.335 | INFO | Analyzing chunk frames = [1249792,1250304) | 20260506 12:10:42.335 | INFO | MemInfo: { | 20260506 12:10:42.335 | INFO | "gpu" : " 17 GB", | 20260506 12:10:42.335 | INFO | "huge" : "", | 20260506 12:10:42.335 | INFO | "huge_shared" : " 27 GB", | 20260506 12:10:42.335 | INFO | "malloc" : " 14 GB", | 20260506 12:10:42.335 | INFO | "pinned" : " 29 GB" | 20260506 12:10:42.335 | INFO | } | 20260506 12:10:42.335 | INFO | Perf report for the last 5.13857 seconds: | 20260506 12:10:42.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:10:42.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:42.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:42.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:42.335 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:10:42.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:42.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:42.336 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:10:42.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:42.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:42.336 | INFO | Uploaded 36GiB to GPU in 1.99406s (18.0537GiB/s) | 20260506 12:10:42.336 | INFO | Downloaded 34GiB to GPU in 1.93054s (17.6117GiB/s) | 20260506 12:10:47.475 | INFO | Analyzing chunk frames = [1250304,1250816) | 20260506 12:10:47.475 | INFO | MemInfo: { | 20260506 12:10:47.475 | INFO | "gpu" : " 17 GB", | 20260506 12:10:47.475 | INFO | "huge" : "", | 20260506 12:10:47.475 | INFO | "huge_shared" : " 27 GB", | 20260506 12:10:47.475 | INFO | "malloc" : " 14 GB", | 20260506 12:10:47.475 | INFO | "pinned" : " 29 GB" | 20260506 12:10:47.475 | INFO | } | 20260506 12:10:47.475 | INFO | Perf report for the last 5.13934 seconds: | 20260506 12:10:47.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:10:47.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:47.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:47.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:47.475 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 12:10:47.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:47.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:47.475 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:10:47.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:47.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:47.475 | INFO | Uploaded 36GiB to GPU in 2.01091s (17.9024GiB/s) | 20260506 12:10:47.475 | INFO | Downloaded 34GiB to GPU in 1.94523s (17.4786GiB/s) | 20260506 12:10:52.614 | INFO | Analyzing chunk frames = [1250816,1251328) | 20260506 12:10:52.614 | INFO | MemInfo: { | 20260506 12:10:52.614 | INFO | "gpu" : " 17 GB", | 20260506 12:10:52.614 | INFO | "huge" : "", | 20260506 12:10:52.614 | INFO | "huge_shared" : " 27 GB", | 20260506 12:10:52.614 | INFO | "malloc" : " 14 GB", | 20260506 12:10:52.614 | INFO | "pinned" : " 29 GB" | 20260506 12:10:52.614 | INFO | } | 20260506 12:10:52.614 | INFO | Perf report for the last 5.13953 seconds: | 20260506 12:10:52.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:10:52.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:52.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:52.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:52.614 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 12:10:52.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:52.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:52.614 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:10:52.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:52.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:52.614 | INFO | Uploaded 36GiB to GPU in 2.01745s (17.8443GiB/s) | 20260506 12:10:52.614 | INFO | Downloaded 34GiB to GPU in 1.95266s (17.4122GiB/s) | 20260506 12:10:57.753 | INFO | Analyzing chunk frames = [1251328,1251840) | 20260506 12:10:57.753 | INFO | MemInfo: { | 20260506 12:10:57.753 | INFO | "gpu" : " 17 GB", | 20260506 12:10:57.753 | INFO | "huge" : "", | 20260506 12:10:57.753 | INFO | "huge_shared" : " 27 GB", | 20260506 12:10:57.753 | INFO | "malloc" : " 14 GB", | 20260506 12:10:57.753 | INFO | "pinned" : " 29 GB" | 20260506 12:10:57.753 | INFO | } | 20260506 12:10:57.754 | INFO | Perf report for the last 5.13924 seconds: | 20260506 12:10:57.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:10:57.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:57.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:57.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:57.754 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 12:10:57.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:57.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:57.754 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:10:57.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:57.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:10:57.754 | INFO | Uploaded 36GiB to GPU in 2.02113s (17.8118GiB/s) | 20260506 12:10:57.754 | INFO | Downloaded 34GiB to GPU in 1.9621s (17.3284GiB/s) | 20260506 12:11:02.893 | INFO | Analyzing chunk frames = [1251840,1252352) | 20260506 12:11:02.893 | INFO | MemInfo: { | 20260506 12:11:02.893 | INFO | "gpu" : " 17 GB", | 20260506 12:11:02.893 | INFO | "huge" : "", | 20260506 12:11:02.893 | INFO | "huge_shared" : " 27 GB", | 20260506 12:11:02.893 | INFO | "malloc" : " 14 GB", | 20260506 12:11:02.893 | INFO | "pinned" : " 29 GB" | 20260506 12:11:02.893 | INFO | } | 20260506 12:11:02.893 | INFO | Perf report for the last 5.13924 seconds: | 20260506 12:11:02.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:11:02.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:02.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:02.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:02.893 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:11:02.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:02.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:02.893 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 12:11:02.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:02.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:02.893 | INFO | Uploaded 36GiB to GPU in 2.0885s (17.2373GiB/s) | 20260506 12:11:02.893 | INFO | Downloaded 34GiB to GPU in 1.98573s (17.1221GiB/s) | 20260506 12:11:08.034 | INFO | Analyzing chunk frames = [1252352,1252864) | 20260506 12:11:08.034 | INFO | MemInfo: { | 20260506 12:11:08.034 | INFO | "gpu" : " 17 GB", | 20260506 12:11:08.034 | INFO | "huge" : "", | 20260506 12:11:08.034 | INFO | "huge_shared" : " 27 GB", | 20260506 12:11:08.034 | INFO | "malloc" : " 14 GB", | 20260506 12:11:08.034 | INFO | "pinned" : " 29 GB" | 20260506 12:11:08.034 | INFO | } | 20260506 12:11:08.034 | INFO | Perf report for the last 5.14087 seconds: | 20260506 12:11:08.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:11:08.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:08.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:08.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:08.034 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 12:11:08.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:08.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:08.034 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 12:11:08.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:08.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:08.034 | INFO | Uploaded 36GiB to GPU in 2.0316s (17.72GiB/s) | 20260506 12:11:08.034 | INFO | Downloaded 34GiB to GPU in 2.02213s (16.8139GiB/s) | 20260506 12:11:13.171 | INFO | Analyzing chunk frames = [1252864,1253376) | 20260506 12:11:13.172 | INFO | MemInfo: { | 20260506 12:11:13.172 | INFO | "gpu" : " 17 GB", | 20260506 12:11:13.172 | INFO | "huge" : "", | 20260506 12:11:13.172 | INFO | "huge_shared" : " 27 GB", | 20260506 12:11:13.172 | INFO | "malloc" : " 14 GB", | 20260506 12:11:13.172 | INFO | "pinned" : " 29 GB" | 20260506 12:11:13.172 | INFO | } | 20260506 12:11:13.172 | INFO | Perf report for the last 5.13797 seconds: | 20260506 12:11:13.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:11:13.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:13.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:13.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:13.172 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260506 12:11:13.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:13.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:13.172 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:11:13.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:13.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:13.172 | INFO | Uploaded 36GiB to GPU in 2.07477s (17.3513GiB/s) | 20260506 12:11:13.172 | INFO | Downloaded 34GiB to GPU in 1.9974s (17.0221GiB/s) | 20260506 12:11:13.172 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:11:13.172 | INFO | Total mem usage: 16.8 GiB | 20260506 12:11:13.172 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:11:13.172 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.6%) | 20260506 12:11:13.172 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:11:13.172 | INFO | High Water Breakdown: | 20260506 12:11:13.172 | INFO | 59.6% : 8.59 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:11:13.172 | INFO | 40.3% : 5.81 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:11:13.172 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:11:13.172 | INFO | ------------------------------------------ | 20260506 12:11:13.172 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:11:13.172 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:11:13.172 | INFO | Total mem usage: 26.5 GiB | 20260506 12:11:13.172 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:11:13.172 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:11:13.172 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:11:13.172 | INFO | High Water Breakdown: | 20260506 12:11:13.172 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:11:13.172 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:11:13.172 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:11:13.172 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:11:13.172 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:11:13.172 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:11:13.172 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:11:13.172 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:11:13.172 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:11:13.172 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:11:13.172 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:11:13.172 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:11:13.172 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:11:13.172 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:11:13.172 | INFO | ------------------------------------------ | 20260506 12:11:13.172 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:11:13.172 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:11:13.172 | INFO | Total mem usage: 25.6 GiB | 20260506 12:11:13.172 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:11:13.172 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:11:13.172 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:11:13.172 | INFO | High Water Breakdown: | 20260506 12:11:13.172 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:11:13.172 | INFO | ------------------------------------------ | 20260506 12:11:13.172 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:11:13.172 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:11:13.172 | INFO | Total mem usage: 15.7 GiB | 20260506 12:11:13.172 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:11:13.172 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:11:13.172 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:11:13.172 | INFO | High Water Breakdown: | 20260506 12:11:13.172 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:11:13.172 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:11:13.172 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:11:13.172 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:11:13.172 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:11:13.172 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:11:13.172 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:11:13.172 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:11:13.172 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:11:13.172 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:11:13.172 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:11:13.172 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:11:13.173 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:11:13.173 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:11:13.173 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:11:13.173 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:11:13.173 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:11:13.173 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:11:13.173 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:11:13.173 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:11:13.173 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:11:13.173 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:11:13.173 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:11:13.173 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:11:13.173 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:11:13.173 | INFO | ------------------------------------------ | 20260506 12:11:13.173 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:11:13.173 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:11:13.173 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:11:13.173 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:11:13.173 | INFO | PerfReport: Full Execution -- 3.19e+01 / 7.62e+01, 1.36e+01 / 2.20e+00 | 20260506 12:11:13.173 | INFO | PerfReport: 0.30%, AggregateStats -- 9.52e-02 / 1.84e-01, 1.26e-01 / 5.84e-01 | 20260506 12:11:13.173 | INFO | PerfReport: 11.55%, Baseline -- 3.68e+00 / 4.42e+00, 5.90e-01 / 3.58e-01 | 20260506 12:11:13.173 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.45e+00, 1.94e-01 / 1.43e-01 | 20260506 12:11:13.173 | INFO | PerfReport: 15.80%, DME -- 4.03e+01 / 4.46e+01, 4.05e+00 / 2.44e+00 | 20260506 12:11:13.173 | INFO | PerfReport: 56.40%, FrameLabeling -- 1.80e+01 / 2.04e+01, 1.51e+00 / 1.42e+00 | 20260506 12:11:13.173 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.42e+00 / 3.08e+00, 4.22e-01 / 2.14e-01 | 20260506 12:11:13.173 | INFO | PerfReport: 3.78%, Metrics -- 1.20e+00 / 1.60e+00, 3.06e-01 / 1.65e-01 | 20260506 12:11:13.173 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.25e-01 / 4.81e-01, 6.58e-02 / 2.86e-02 | 20260506 12:11:13.173 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:11:13.173 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:11:13.173 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:11:13.173 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 12:11:13.173 | INFO | PerfReport: 21.45%, Upload -- 2.12e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260506 12:11:13.173 | INFO | PerfReport: 21.03%, Download -- 2.08e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 12:11:13.173 | INFO | PerfReport: 32.20%, Compute -- 3.19e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 12:11:13.173 | INFO | PerfReport: 1.82%, LockUpload -- 1.80e+00 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 12:11:13.173 | INFO | PerfReport: 1.01%, LockDownload -- 9.97e-01 / 0.00e+00, 2.63e+00 / 0.00e+00 | 20260506 12:11:13.173 | INFO | PerfReport: 22.48%, LockCompute -- 2.22e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 12:11:13.173 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260506 12:11:13.297 | INFO | Starting a round of Baz IO | 20260506 12:11:16.473 | INFO | Populating RTMetrics report | 20260506 12:11:16.473 | INFO | RTMetrics report populated | 20260506 12:11:16.473 | INFO | Opening tmp RTMetrics json file | 20260506 12:11:16.474 | INFO | Serializing RTMetrics to json | 20260506 12:11:16.475 | INFO | Serialization done, swapping tmp file now... | 20260506 12:11:16.476 | INFO | RTMetrics done | 20260506 12:11:16.569 | INFO | Finished a round of Baz IO: Wrote 2.087 GiB in 3.271 seconds | 20260506 12:11:16.569 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.17% / 0.03% / 57.81% | 20260506 12:11:16.569 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.28403% / 5.96046e-05% / 2.38419e-05% | 20260506 12:11:16.569 | INFO | Baz IO overall: | 20260506 12:11:16.569 | INFO | BytesWritten : 617580591300 | 20260506 12:11:16.569 | INFO | NumEvents : 298377190174 | 20260506 12:11:18.311 | INFO | Analyzing chunk frames = [1253376,1253888) | 20260506 12:11:18.311 | INFO | MemInfo: { | 20260506 12:11:18.311 | INFO | "gpu" : " 17 GB", | 20260506 12:11:18.311 | INFO | "huge" : "", | 20260506 12:11:18.311 | INFO | "huge_shared" : " 27 GB", | 20260506 12:11:18.311 | INFO | "malloc" : " 14 GB", | 20260506 12:11:18.311 | INFO | "pinned" : " 29 GB" | 20260506 12:11:18.311 | INFO | } | 20260506 12:11:18.311 | INFO | Perf report for the last 5.13915 seconds: | 20260506 12:11:18.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:11:18.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:18.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:18.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:18.311 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260506 12:11:18.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:18.311 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 12:11:18.311 | INFO | PRE_HQ : 61.9%, 1.9, 61.8ms, 0 | 20260506 12:11:18.311 | INFO | BAZWRITER : 62.9%, 3.7, 118.2ms, 0 | 20260506 12:11:18.311 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:11:18.311 | INFO | Uploaded 36GiB to GPU in 2.11324s (17.0355GiB/s) | 20260506 12:11:18.311 | INFO | Downloaded 38GiB to GPU in 2.21788s (17.1335GiB/s) | 20260506 12:11:23.450 | INFO | Analyzing chunk frames = [1253888,1254400) | 20260506 12:11:23.450 | INFO | MemInfo: { | 20260506 12:11:23.450 | INFO | "gpu" : " 17 GB", | 20260506 12:11:23.450 | INFO | "huge" : "", | 20260506 12:11:23.450 | INFO | "huge_shared" : " 27 GB", | 20260506 12:11:23.450 | INFO | "malloc" : " 14 GB", | 20260506 12:11:23.450 | INFO | "pinned" : " 29 GB" | 20260506 12:11:23.450 | INFO | } | 20260506 12:11:23.450 | INFO | Perf report for the last 5.13949 seconds: | 20260506 12:11:23.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:11:23.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:23.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:23.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:23.450 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:11:23.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:23.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:23.450 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 12:11:23.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:23.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:23.451 | INFO | Uploaded 36GiB to GPU in 2.01044s (17.9065GiB/s) | 20260506 12:11:23.451 | INFO | Downloaded 34GiB to GPU in 1.91628s (17.7427GiB/s) | 20260506 12:11:28.589 | INFO | Analyzing chunk frames = [1254400,1254912) | 20260506 12:11:28.590 | INFO | MemInfo: { | 20260506 12:11:28.590 | INFO | "gpu" : " 17 GB", | 20260506 12:11:28.590 | INFO | "huge" : "", | 20260506 12:11:28.590 | INFO | "huge_shared" : " 27 GB", | 20260506 12:11:28.590 | INFO | "malloc" : " 14 GB", | 20260506 12:11:28.590 | INFO | "pinned" : " 29 GB" | 20260506 12:11:28.590 | INFO | } | 20260506 12:11:28.590 | INFO | Perf report for the last 5.13933 seconds: | 20260506 12:11:28.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:11:28.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:28.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:28.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:28.590 | INFO | ANALYSIS : 61.9%, 3.0, 95.8ms, 0 | 20260506 12:11:28.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:28.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:28.590 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:11:28.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:28.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:28.590 | INFO | Uploaded 36GiB to GPU in 2.03562s (17.685GiB/s) | 20260506 12:11:28.590 | INFO | Downloaded 34GiB to GPU in 1.93993s (17.5264GiB/s) | 20260506 12:11:33.729 | INFO | Analyzing chunk frames = [1254912,1255424) | 20260506 12:11:33.729 | INFO | MemInfo: { | 20260506 12:11:33.729 | INFO | "gpu" : " 17 GB", | 20260506 12:11:33.729 | INFO | "huge" : "", | 20260506 12:11:33.729 | INFO | "huge_shared" : " 27 GB", | 20260506 12:11:33.729 | INFO | "malloc" : " 14 GB", | 20260506 12:11:33.729 | INFO | "pinned" : " 29 GB" | 20260506 12:11:33.729 | INFO | } | 20260506 12:11:33.729 | INFO | Perf report for the last 5.13935 seconds: | 20260506 12:11:33.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:11:33.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:33.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:33.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:33.729 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 12:11:33.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:33.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:33.729 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:11:33.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:33.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:33.729 | INFO | Uploaded 36GiB to GPU in 2.0864s (17.2546GiB/s) | 20260506 12:11:33.729 | INFO | Downloaded 34GiB to GPU in 1.96574s (17.2962GiB/s) | 20260506 12:11:38.868 | INFO | Analyzing chunk frames = [1255424,1255936) | 20260506 12:11:38.868 | INFO | MemInfo: { | 20260506 12:11:38.868 | INFO | "gpu" : " 17 GB", | 20260506 12:11:38.868 | INFO | "huge" : "", | 20260506 12:11:38.868 | INFO | "huge_shared" : " 27 GB", | 20260506 12:11:38.868 | INFO | "malloc" : " 14 GB", | 20260506 12:11:38.868 | INFO | "pinned" : " 29 GB" | 20260506 12:11:38.868 | INFO | } | 20260506 12:11:38.868 | INFO | Perf report for the last 5.13919 seconds: | 20260506 12:11:38.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:11:38.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:38.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:38.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:38.868 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 12:11:38.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:38.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:38.868 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:11:38.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:38.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:38.868 | INFO | Uploaded 36GiB to GPU in 2.03211s (17.7156GiB/s) | 20260506 12:11:38.868 | INFO | Downloaded 34GiB to GPU in 1.95915s (17.3545GiB/s) | 20260506 12:11:44.008 | INFO | Analyzing chunk frames = [1255936,1256448) | 20260506 12:11:44.008 | INFO | MemInfo: { | 20260506 12:11:44.008 | INFO | "gpu" : " 17 GB", | 20260506 12:11:44.008 | INFO | "huge" : "", | 20260506 12:11:44.008 | INFO | "huge_shared" : " 27 GB", | 20260506 12:11:44.008 | INFO | "malloc" : " 14 GB", | 20260506 12:11:44.008 | INFO | "pinned" : " 29 GB" | 20260506 12:11:44.008 | INFO | } | 20260506 12:11:44.008 | INFO | Perf report for the last 5.13964 seconds: | 20260506 12:11:44.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:11:44.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:44.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:44.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:44.008 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:11:44.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:44.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:44.008 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260506 12:11:44.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:44.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:44.008 | INFO | Uploaded 36GiB to GPU in 2.10001s (17.1427GiB/s) | 20260506 12:11:44.008 | INFO | Downloaded 34GiB to GPU in 1.98124s (17.161GiB/s) | 20260506 12:11:49.147 | INFO | Analyzing chunk frames = [1256448,1256960) | 20260506 12:11:49.147 | INFO | MemInfo: { | 20260506 12:11:49.147 | INFO | "gpu" : " 17 GB", | 20260506 12:11:49.147 | INFO | "huge" : "", | 20260506 12:11:49.147 | INFO | "huge_shared" : " 27 GB", | 20260506 12:11:49.147 | INFO | "malloc" : " 14 GB", | 20260506 12:11:49.147 | INFO | "pinned" : " 29 GB" | 20260506 12:11:49.147 | INFO | } | 20260506 12:11:49.147 | INFO | Perf report for the last 5.13934 seconds: | 20260506 12:11:49.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:11:49.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:49.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:49.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:49.147 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:11:49.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:49.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:49.147 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:11:49.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:49.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:49.148 | INFO | Uploaded 36GiB to GPU in 1.96637s (18.3078GiB/s) | 20260506 12:11:49.148 | INFO | Downloaded 34GiB to GPU in 1.99678s (17.0274GiB/s) | 20260506 12:11:54.286 | INFO | Analyzing chunk frames = [1256960,1257472) | 20260506 12:11:54.286 | INFO | MemInfo: { | 20260506 12:11:54.286 | INFO | "gpu" : " 17 GB", | 20260506 12:11:54.286 | INFO | "huge" : "", | 20260506 12:11:54.286 | INFO | "huge_shared" : " 27 GB", | 20260506 12:11:54.286 | INFO | "malloc" : " 14 GB", | 20260506 12:11:54.286 | INFO | "pinned" : " 29 GB" | 20260506 12:11:54.286 | INFO | } | 20260506 12:11:54.286 | INFO | Perf report for the last 5.13911 seconds: | 20260506 12:11:54.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:11:54.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:54.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:54.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:54.286 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 12:11:54.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:54.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:54.286 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:11:54.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:54.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:54.287 | INFO | Uploaded 36GiB to GPU in 2.06258s (17.4539GiB/s) | 20260506 12:11:54.287 | INFO | Downloaded 34GiB to GPU in 1.96837s (17.2732GiB/s) | 20260506 12:11:54.408 | INFO | Starting a round of Baz IO | 20260506 12:11:57.585 | INFO | Populating RTMetrics report | 20260506 12:11:57.585 | INFO | RTMetrics report populated | 20260506 12:11:57.585 | INFO | Opening tmp RTMetrics json file | 20260506 12:11:57.586 | INFO | Serializing RTMetrics to json | 20260506 12:11:57.587 | INFO | Serialization done, swapping tmp file now... | 20260506 12:11:57.588 | INFO | RTMetrics done | 20260506 12:11:57.683 | INFO | Finished a round of Baz IO: Wrote 2.087 GiB in 3.275 seconds | 20260506 12:11:57.683 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.14% / 0.03% / 57.83% | 20260506 12:11:57.684 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.26908% / 3.57628e-05% / 3.57628e-05% | 20260506 12:11:57.684 | INFO | Baz IO overall: | 20260506 12:11:57.684 | INFO | BytesWritten : 619821467480 | 20260506 12:11:57.684 | INFO | NumEvents : 299464288898 | 20260506 12:11:59.426 | INFO | Analyzing chunk frames = [1257472,1257984) | 20260506 12:11:59.426 | INFO | MemInfo: { | 20260506 12:11:59.426 | INFO | "gpu" : " 17 GB", | 20260506 12:11:59.426 | INFO | "huge" : "", | 20260506 12:11:59.426 | INFO | "huge_shared" : " 27 GB", | 20260506 12:11:59.426 | INFO | "malloc" : " 14 GB", | 20260506 12:11:59.426 | INFO | "pinned" : " 29 GB" | 20260506 12:11:59.426 | INFO | } | 20260506 12:11:59.426 | INFO | Perf report for the last 5.1396 seconds: | 20260506 12:11:59.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:11:59.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:59.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:59.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:59.426 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260506 12:11:59.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:11:59.426 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:11:59.426 | INFO | PRE_HQ : 61.8%, 1.9, 61.8ms, 0 | 20260506 12:11:59.426 | INFO | BAZWRITER : 62.9%, 3.7, 118.9ms, 0 | 20260506 12:11:59.426 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:11:59.426 | INFO | Uploaded 36GiB to GPU in 2.0968s (17.169GiB/s) | 20260506 12:11:59.426 | INFO | Downloaded 38GiB to GPU in 2.20157s (17.2604GiB/s) | 20260506 12:12:04.565 | INFO | Analyzing chunk frames = [1257984,1258496) | 20260506 12:12:04.565 | INFO | MemInfo: { | 20260506 12:12:04.565 | INFO | "gpu" : " 17 GB", | 20260506 12:12:04.565 | INFO | "huge" : "", | 20260506 12:12:04.565 | INFO | "huge_shared" : " 27 GB", | 20260506 12:12:04.565 | INFO | "malloc" : " 14 GB", | 20260506 12:12:04.565 | INFO | "pinned" : " 29 GB" | 20260506 12:12:04.565 | INFO | } | 20260506 12:12:04.565 | INFO | Perf report for the last 5.13908 seconds: | 20260506 12:12:04.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:12:04.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:04.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:04.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:04.565 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:12:04.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:04.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:04.565 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:12:04.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:04.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:04.565 | INFO | Uploaded 36GiB to GPU in 2.01468s (17.8689GiB/s) | 20260506 12:12:04.565 | INFO | Downloaded 34GiB to GPU in 1.91395s (17.7643GiB/s) | 20260506 12:12:09.704 | INFO | Analyzing chunk frames = [1258496,1259008) | 20260506 12:12:09.704 | INFO | MemInfo: { | 20260506 12:12:09.704 | INFO | "gpu" : " 17 GB", | 20260506 12:12:09.704 | INFO | "huge" : "", | 20260506 12:12:09.704 | INFO | "huge_shared" : " 27 GB", | 20260506 12:12:09.704 | INFO | "malloc" : " 14 GB", | 20260506 12:12:09.704 | INFO | "pinned" : " 29 GB" | 20260506 12:12:09.704 | INFO | } | 20260506 12:12:09.704 | INFO | Perf report for the last 5.13943 seconds: | 20260506 12:12:09.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:12:09.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:09.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:09.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:09.705 | INFO | ANALYSIS : 61.9%, 3.0, 95.8ms, 0 | 20260506 12:12:09.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:09.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:09.705 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:12:09.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:09.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:09.705 | INFO | Uploaded 36GiB to GPU in 2.02736s (17.757GiB/s) | 20260506 12:12:09.705 | INFO | Downloaded 34GiB to GPU in 1.93703s (17.5526GiB/s) | 20260506 12:12:14.844 | INFO | Analyzing chunk frames = [1259008,1259520) | 20260506 12:12:14.844 | INFO | MemInfo: { | 20260506 12:12:14.844 | INFO | "gpu" : " 17 GB", | 20260506 12:12:14.844 | INFO | "huge" : "", | 20260506 12:12:14.844 | INFO | "huge_shared" : " 27 GB", | 20260506 12:12:14.844 | INFO | "malloc" : " 14 GB", | 20260506 12:12:14.844 | INFO | "pinned" : " 29 GB" | 20260506 12:12:14.844 | INFO | } | 20260506 12:12:14.844 | INFO | Perf report for the last 5.13933 seconds: | 20260506 12:12:14.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:12:14.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:14.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:14.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:14.844 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260506 12:12:14.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:14.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:14.844 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:12:14.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:14.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:14.844 | INFO | Uploaded 36GiB to GPU in 2.0755s (17.3452GiB/s) | 20260506 12:12:14.844 | INFO | Downloaded 34GiB to GPU in 1.96051s (17.3424GiB/s) | 20260506 12:12:19.983 | INFO | Analyzing chunk frames = [1259520,1260032) | 20260506 12:12:19.983 | INFO | MemInfo: { | 20260506 12:12:19.983 | INFO | "gpu" : " 17 GB", | 20260506 12:12:19.983 | INFO | "huge" : "", | 20260506 12:12:19.983 | INFO | "huge_shared" : " 27 GB", | 20260506 12:12:19.983 | INFO | "malloc" : " 14 GB", | 20260506 12:12:19.983 | INFO | "pinned" : " 29 GB" | 20260506 12:12:19.983 | INFO | } | 20260506 12:12:19.983 | INFO | Perf report for the last 5.13916 seconds: | 20260506 12:12:19.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:12:19.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:19.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:19.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:19.983 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 12:12:19.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:19.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:19.983 | INFO | PRE_HQ : 33.0%, 1.0, 17.2ms, 0 | 20260506 12:12:19.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:19.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:19.983 | INFO | Uploaded 36GiB to GPU in 2.04932s (17.5668GiB/s) | 20260506 12:12:19.983 | INFO | Downloaded 34GiB to GPU in 1.97594s (17.207GiB/s) | 20260506 12:12:25.122 | INFO | Analyzing chunk frames = [1260032,1260544) | 20260506 12:12:25.122 | INFO | MemInfo: { | 20260506 12:12:25.122 | INFO | "gpu" : " 17 GB", | 20260506 12:12:25.122 | INFO | "huge" : "", | 20260506 12:12:25.122 | INFO | "huge_shared" : " 27 GB", | 20260506 12:12:25.122 | INFO | "malloc" : " 14 GB", | 20260506 12:12:25.122 | INFO | "pinned" : " 29 GB" | 20260506 12:12:25.122 | INFO | } | 20260506 12:12:25.122 | INFO | Perf report for the last 5.13938 seconds: | 20260506 12:12:25.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:12:25.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:25.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:25.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:25.122 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:12:25.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:25.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:25.122 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:12:25.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:25.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:25.123 | INFO | Uploaded 36GiB to GPU in 2.11056s (17.0571GiB/s) | 20260506 12:12:25.123 | INFO | Downloaded 34GiB to GPU in 1.97743s (17.1941GiB/s) | 20260506 12:12:30.262 | INFO | Analyzing chunk frames = [1260544,1261056) | 20260506 12:12:30.262 | INFO | MemInfo: { | 20260506 12:12:30.262 | INFO | "gpu" : " 17 GB", | 20260506 12:12:30.262 | INFO | "huge" : "", | 20260506 12:12:30.262 | INFO | "huge_shared" : " 27 GB", | 20260506 12:12:30.262 | INFO | "malloc" : " 14 GB", | 20260506 12:12:30.262 | INFO | "pinned" : " 29 GB" | 20260506 12:12:30.262 | INFO | } | 20260506 12:12:30.263 | INFO | Perf report for the last 5.1401 seconds: | 20260506 12:12:30.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:12:30.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:30.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:30.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:30.263 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:12:30.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:30.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:30.263 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 12:12:30.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:30.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:30.263 | INFO | Uploaded 36GiB to GPU in 1.99545s (18.041GiB/s) | 20260506 12:12:30.263 | INFO | Downloaded 34GiB to GPU in 2.01088s (16.908GiB/s) | 20260506 12:12:35.401 | INFO | Analyzing chunk frames = [1261056,1261568) | 20260506 12:12:35.401 | INFO | MemInfo: { | 20260506 12:12:35.401 | INFO | "gpu" : " 17 GB", | 20260506 12:12:35.401 | INFO | "huge" : "", | 20260506 12:12:35.401 | INFO | "huge_shared" : " 27 GB", | 20260506 12:12:35.401 | INFO | "malloc" : " 14 GB", | 20260506 12:12:35.401 | INFO | "pinned" : " 29 GB" | 20260506 12:12:35.401 | INFO | } | 20260506 12:12:35.401 | INFO | Perf report for the last 5.13864 seconds: | 20260506 12:12:35.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:12:35.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:35.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:35.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:35.401 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260506 12:12:35.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:35.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:35.401 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 12:12:35.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:35.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:35.401 | INFO | Uploaded 36GiB to GPU in 2.0602s (17.4741GiB/s) | 20260506 12:12:35.401 | INFO | Downloaded 34GiB to GPU in 1.96011s (17.346GiB/s) | 20260506 12:12:35.401 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:12:35.401 | INFO | Total mem usage: 16.8 GiB | 20260506 12:12:35.402 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:12:35.402 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.5%) | 20260506 12:12:35.402 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:12:35.402 | INFO | High Water Breakdown: | 20260506 12:12:35.402 | INFO | 59.5% : 8.59 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:12:35.402 | INFO | 40.3% : 5.82 GiB ( 86.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:12:35.402 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:12:35.402 | INFO | ------------------------------------------ | 20260506 12:12:35.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:12:35.402 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:12:35.402 | INFO | Total mem usage: 26.5 GiB | 20260506 12:12:35.402 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:12:35.402 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:12:35.402 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:12:35.402 | INFO | High Water Breakdown: | 20260506 12:12:35.402 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:12:35.402 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:12:35.402 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:12:35.402 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:12:35.402 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:12:35.402 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:12:35.402 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:12:35.402 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:12:35.402 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:12:35.402 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:12:35.402 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:12:35.402 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:12:35.402 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:12:35.402 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:12:35.402 | INFO | ------------------------------------------ | 20260506 12:12:35.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:12:35.402 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:12:35.402 | INFO | Total mem usage: 25.6 GiB | 20260506 12:12:35.402 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:12:35.402 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:12:35.402 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:12:35.402 | INFO | High Water Breakdown: | 20260506 12:12:35.402 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:12:35.402 | INFO | ------------------------------------------ | 20260506 12:12:35.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:12:35.402 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:12:35.402 | INFO | Total mem usage: 15.7 GiB | 20260506 12:12:35.402 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:12:35.402 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:12:35.402 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:12:35.402 | INFO | High Water Breakdown: | 20260506 12:12:35.402 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:12:35.402 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:12:35.402 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:12:35.402 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:12:35.402 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:12:35.402 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:12:35.402 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:12:35.402 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:12:35.402 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:12:35.402 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:12:35.402 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:12:35.402 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:12:35.402 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:12:35.402 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:12:35.402 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:12:35.402 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:12:35.402 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:12:35.402 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:12:35.402 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:12:35.402 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:12:35.402 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:12:35.402 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:12:35.402 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:12:35.402 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:12:35.402 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:12:35.402 | INFO | ------------------------------------------ | 20260506 12:12:35.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:12:35.402 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:12:35.402 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:12:35.402 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:12:35.402 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.58e+01, 1.35e+01 / 2.04e+00 | 20260506 12:12:35.402 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.01e-01, 1.02e-02 / 9.10e-03 | 20260506 12:12:35.402 | INFO | PerfReport: 11.58%, Baseline -- 3.68e+00 / 4.45e+00, 5.88e-01 / 3.33e-01 | 20260506 12:12:35.402 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.46e+00, 1.92e-01 / 1.52e-01 | 20260506 12:12:35.402 | INFO | PerfReport: 15.87%, DME -- 4.04e+01 / 4.48e+01, 4.04e+00 / 2.11e+00 | 20260506 12:12:35.402 | INFO | PerfReport: 56.24%, FrameLabeling -- 1.79e+01 / 1.99e+01, 1.37e+00 / 1.71e+00 | 20260506 12:12:35.402 | INFO | PerfReport: 7.65%, PulseAccumulating -- 2.43e+00 / 3.11e+00, 4.34e-01 / 1.93e-01 | 20260506 12:12:35.402 | INFO | PerfReport: 3.78%, Metrics -- 1.20e+00 / 1.56e+00, 3.03e-01 / 1.65e-01 | 20260506 12:12:35.402 | INFO | PerfReport: 1.34%, PulseSerializing -- 4.25e-01 / 4.81e-01, 6.63e-02 / 2.91e-02 | 20260506 12:12:35.402 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:12:35.402 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:12:35.402 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:12:35.402 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 12:12:35.402 | INFO | PerfReport: 21.59%, Upload -- 2.14e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 12:12:35.402 | INFO | PerfReport: 20.96%, Download -- 2.08e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 12:12:35.402 | INFO | PerfReport: 32.11%, Compute -- 3.18e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 12:12:35.402 | INFO | PerfReport: 1.97%, LockUpload -- 1.95e+00 / 0.00e+00, 7.67e+00 / 0.00e+00 | 20260506 12:12:35.402 | INFO | PerfReport: 0.94%, LockDownload -- 9.33e-01 / 0.00e+00, 2.51e+00 / 0.00e+00 | 20260506 12:12:35.402 | INFO | PerfReport: 22.41%, LockCompute -- 2.22e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260506 12:12:35.402 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260506 12:12:35.524 | INFO | Starting a round of Baz IO | 20260506 12:12:38.702 | INFO | Populating RTMetrics report | 20260506 12:12:38.702 | INFO | RTMetrics report populated | 20260506 12:12:38.702 | INFO | Opening tmp RTMetrics json file | 20260506 12:12:38.703 | INFO | Serializing RTMetrics to json | 20260506 12:12:38.704 | INFO | Serialization done, swapping tmp file now... | 20260506 12:12:38.705 | INFO | RTMetrics done | 20260506 12:12:38.802 | INFO | Finished a round of Baz IO: Wrote 2.086 GiB in 3.277 seconds | 20260506 12:12:38.802 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.11% / 0.03% / 57.86% | 20260506 12:12:38.802 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.15299% / 3.57628e-05% / 3.97364e-05% | 20260506 12:12:38.802 | INFO | Baz IO overall: | 20260506 12:12:38.802 | INFO | BytesWritten : 622061537757 | 20260506 12:12:38.802 | INFO | NumEvents : 300546969618 | 20260506 12:12:40.540 | INFO | Analyzing chunk frames = [1261568,1262080) | 20260506 12:12:40.540 | INFO | MemInfo: { | 20260506 12:12:40.540 | INFO | "gpu" : " 17 GB", | 20260506 12:12:40.540 | INFO | "huge" : "", | 20260506 12:12:40.540 | INFO | "huge_shared" : " 27 GB", | 20260506 12:12:40.540 | INFO | "malloc" : " 14 GB", | 20260506 12:12:40.540 | INFO | "pinned" : " 29 GB" | 20260506 12:12:40.541 | INFO | } | 20260506 12:12:40.541 | INFO | Perf report for the last 5.13939 seconds: | 20260506 12:12:40.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:12:40.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:40.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:40.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:40.541 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260506 12:12:40.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:40.541 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 12:12:40.541 | INFO | PRE_HQ : 61.7%, 1.9, 61.6ms, 0 | 20260506 12:12:40.541 | INFO | BAZWRITER : 63.0%, 3.7, 119.0ms, 0 | 20260506 12:12:40.541 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:12:40.541 | INFO | Uploaded 36GiB to GPU in 2.10036s (17.1399GiB/s) | 20260506 12:12:40.541 | INFO | Downloaded 38GiB to GPU in 2.20581s (17.2272GiB/s) | 20260506 12:12:45.680 | INFO | Analyzing chunk frames = [1262080,1262592) | 20260506 12:12:45.680 | INFO | MemInfo: { | 20260506 12:12:45.680 | INFO | "gpu" : " 17 GB", | 20260506 12:12:45.680 | INFO | "huge" : "", | 20260506 12:12:45.680 | INFO | "huge_shared" : " 27 GB", | 20260506 12:12:45.680 | INFO | "malloc" : " 14 GB", | 20260506 12:12:45.680 | INFO | "pinned" : " 29 GB" | 20260506 12:12:45.680 | INFO | } | 20260506 12:12:45.680 | INFO | Perf report for the last 5.13954 seconds: | 20260506 12:12:45.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:12:45.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:45.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:45.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:45.680 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:12:45.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:45.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:45.680 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:12:45.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:45.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:45.680 | INFO | Uploaded 36GiB to GPU in 2.03704s (17.6727GiB/s) | 20260506 12:12:45.680 | INFO | Downloaded 34GiB to GPU in 1.91876s (17.7198GiB/s) | 20260506 12:12:50.819 | INFO | Analyzing chunk frames = [1262592,1263104) | 20260506 12:12:50.819 | INFO | MemInfo: { | 20260506 12:12:50.819 | INFO | "gpu" : " 17 GB", | 20260506 12:12:50.819 | INFO | "huge" : "", | 20260506 12:12:50.819 | INFO | "huge_shared" : " 27 GB", | 20260506 12:12:50.819 | INFO | "malloc" : " 14 GB", | 20260506 12:12:50.819 | INFO | "pinned" : " 29 GB" | 20260506 12:12:50.819 | INFO | } | 20260506 12:12:50.819 | INFO | Perf report for the last 5.1391 seconds: | 20260506 12:12:50.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:12:50.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:50.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:50.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:50.819 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:12:50.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:50.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:50.819 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:12:50.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:50.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:50.819 | INFO | Uploaded 36GiB to GPU in 2.00261s (17.9765GiB/s) | 20260506 12:12:50.819 | INFO | Downloaded 34GiB to GPU in 1.93605s (17.5615GiB/s) | 20260506 12:12:55.959 | INFO | Analyzing chunk frames = [1263104,1263616) | 20260506 12:12:55.959 | INFO | MemInfo: { | 20260506 12:12:55.959 | INFO | "gpu" : " 17 GB", | 20260506 12:12:55.959 | INFO | "huge" : "", | 20260506 12:12:55.959 | INFO | "huge_shared" : " 27 GB", | 20260506 12:12:55.959 | INFO | "malloc" : " 14 GB", | 20260506 12:12:55.959 | INFO | "pinned" : " 29 GB" | 20260506 12:12:55.959 | INFO | } | 20260506 12:12:55.959 | INFO | Perf report for the last 5.13938 seconds: | 20260506 12:12:55.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:12:55.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:55.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:55.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:55.959 | INFO | ANALYSIS : 61.4%, 3.0, 95.3ms, 0 | 20260506 12:12:55.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:55.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:55.959 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:12:55.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:55.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:12:55.959 | INFO | Uploaded 36GiB to GPU in 2.01791s (17.8403GiB/s) | 20260506 12:12:55.959 | INFO | Downloaded 34GiB to GPU in 1.95072s (17.4294GiB/s) | 20260506 12:13:01.098 | INFO | Analyzing chunk frames = [1263616,1264128) | 20260506 12:13:01.098 | INFO | MemInfo: { | 20260506 12:13:01.098 | INFO | "gpu" : " 17 GB", | 20260506 12:13:01.098 | INFO | "huge" : "", | 20260506 12:13:01.098 | INFO | "huge_shared" : " 27 GB", | 20260506 12:13:01.098 | INFO | "malloc" : " 14 GB", | 20260506 12:13:01.098 | INFO | "pinned" : " 29 GB" | 20260506 12:13:01.098 | INFO | } | 20260506 12:13:01.098 | INFO | Perf report for the last 5.13935 seconds: | 20260506 12:13:01.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:13:01.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:01.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:01.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:01.098 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 12:13:01.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:01.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:01.098 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:13:01.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:01.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:01.098 | INFO | Uploaded 36GiB to GPU in 1.96275s (18.3416GiB/s) | 20260506 12:13:01.098 | INFO | Downloaded 34GiB to GPU in 1.94032s (17.5229GiB/s) | 20260506 12:13:06.237 | INFO | Analyzing chunk frames = [1264128,1264640) | 20260506 12:13:06.237 | INFO | MemInfo: { | 20260506 12:13:06.237 | INFO | "gpu" : " 17 GB", | 20260506 12:13:06.237 | INFO | "huge" : "", | 20260506 12:13:06.237 | INFO | "huge_shared" : " 27 GB", | 20260506 12:13:06.237 | INFO | "malloc" : " 14 GB", | 20260506 12:13:06.237 | INFO | "pinned" : " 29 GB" | 20260506 12:13:06.237 | INFO | } | 20260506 12:13:06.237 | INFO | Perf report for the last 5.13933 seconds: | 20260506 12:13:06.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:13:06.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:06.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:06.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:06.237 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 12:13:06.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:06.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:06.237 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:13:06.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:06.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:06.238 | INFO | Uploaded 36GiB to GPU in 2.05649s (17.5056GiB/s) | 20260506 12:13:06.238 | INFO | Downloaded 34GiB to GPU in 1.92268s (17.6836GiB/s) | 20260506 12:13:11.378 | INFO | Analyzing chunk frames = [1264640,1265152) | 20260506 12:13:11.378 | INFO | MemInfo: { | 20260506 12:13:11.378 | INFO | "gpu" : " 17 GB", | 20260506 12:13:11.378 | INFO | "huge" : "", | 20260506 12:13:11.378 | INFO | "huge_shared" : " 27 GB", | 20260506 12:13:11.378 | INFO | "malloc" : " 14 GB", | 20260506 12:13:11.378 | INFO | "pinned" : " 29 GB" | 20260506 12:13:11.378 | INFO | } | 20260506 12:13:11.378 | INFO | Perf report for the last 5.14102 seconds: | 20260506 12:13:11.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:13:11.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:11.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:11.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:11.378 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 12:13:11.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:11.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:11.378 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:13:11.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:11.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:11.378 | INFO | Uploaded 36GiB to GPU in 1.97777s (18.2023GiB/s) | 20260506 12:13:11.378 | INFO | Downloaded 34GiB to GPU in 2.03291s (16.7248GiB/s) | 20260506 12:13:16.516 | INFO | Analyzing chunk frames = [1265152,1265664) | 20260506 12:13:16.516 | INFO | MemInfo: { | 20260506 12:13:16.516 | INFO | "gpu" : " 17 GB", | 20260506 12:13:16.516 | INFO | "huge" : "", | 20260506 12:13:16.516 | INFO | "huge_shared" : " 27 GB", | 20260506 12:13:16.516 | INFO | "malloc" : " 14 GB", | 20260506 12:13:16.516 | INFO | "pinned" : " 29 GB" | 20260506 12:13:16.516 | INFO | } | 20260506 12:13:16.516 | INFO | Perf report for the last 5.1377 seconds: | 20260506 12:13:16.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:13:16.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:16.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:16.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:16.516 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:13:16.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:16.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:16.516 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:13:16.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:16.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:16.516 | INFO | Uploaded 36GiB to GPU in 2.06775s (17.4102GiB/s) | 20260506 12:13:16.516 | INFO | Downloaded 34GiB to GPU in 1.9929s (17.0606GiB/s) | 20260506 12:13:16.643 | INFO | Starting a round of Baz IO | 20260506 12:13:19.809 | INFO | Populating RTMetrics report | 20260506 12:13:19.809 | INFO | RTMetrics report populated | 20260506 12:13:19.809 | INFO | Opening tmp RTMetrics json file | 20260506 12:13:19.810 | INFO | Serializing RTMetrics to json | 20260506 12:13:19.811 | INFO | Serialization done, swapping tmp file now... | 20260506 12:13:19.811 | INFO | RTMetrics done | 20260506 12:13:19.916 | INFO | Finished a round of Baz IO: Wrote 2.086 GiB in 3.272 seconds | 20260506 12:13:19.916 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.09% / 0.03% / 57.89% | 20260506 12:13:19.916 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.05911% / 5.5631e-05% / 7.15256e-05% | 20260506 12:13:19.916 | INFO | Baz IO overall: | 20260506 12:13:19.916 | INFO | BytesWritten : 624300859852 | 20260506 12:13:19.916 | INFO | NumEvents : 301627511291 | 20260506 12:13:21.659 | INFO | Analyzing chunk frames = [1265664,1266176) | 20260506 12:13:21.659 | INFO | MemInfo: { | 20260506 12:13:21.659 | INFO | "gpu" : " 17 GB", | 20260506 12:13:21.659 | INFO | "huge" : "", | 20260506 12:13:21.659 | INFO | "huge_shared" : " 27 GB", | 20260506 12:13:21.659 | INFO | "malloc" : " 14 GB", | 20260506 12:13:21.659 | INFO | "pinned" : " 29 GB" | 20260506 12:13:21.659 | INFO | } | 20260506 12:13:21.659 | INFO | Perf report for the last 5.14288 seconds: | 20260506 12:13:21.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:13:21.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:21.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:21.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:21.659 | INFO | ANALYSIS : 63.4%, 3.0, 98.6ms, 0 | 20260506 12:13:21.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:21.659 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:13:21.659 | INFO | PRE_HQ : 61.6%, 1.9, 62.1ms, 0 | 20260506 12:13:21.659 | INFO | BAZWRITER : 62.9%, 3.7, 120.2ms, 0 | 20260506 12:13:21.659 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 12:13:21.659 | INFO | Uploaded 36GiB to GPU in 2.08316s (17.2815GiB/s) | 20260506 12:13:21.659 | INFO | Downloaded 38GiB to GPU in 2.21425s (17.1616GiB/s) | 20260506 12:13:26.795 | INFO | Analyzing chunk frames = [1266176,1266688) | 20260506 12:13:26.795 | INFO | MemInfo: { | 20260506 12:13:26.795 | INFO | "gpu" : " 17 GB", | 20260506 12:13:26.795 | INFO | "huge" : "", | 20260506 12:13:26.795 | INFO | "huge_shared" : " 27 GB", | 20260506 12:13:26.795 | INFO | "malloc" : " 14 GB", | 20260506 12:13:26.795 | INFO | "pinned" : " 29 GB" | 20260506 12:13:26.795 | INFO | } | 20260506 12:13:26.795 | INFO | Perf report for the last 5.13614 seconds: | 20260506 12:13:26.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:13:26.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:26.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:26.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:26.795 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:13:26.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:26.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:26.795 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:13:26.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:26.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:26.795 | INFO | Uploaded 36GiB to GPU in 2.05675s (17.5034GiB/s) | 20260506 12:13:26.795 | INFO | Downloaded 34GiB to GPU in 1.93246s (17.5942GiB/s) | 20260506 12:13:31.934 | INFO | Analyzing chunk frames = [1266688,1267200) | 20260506 12:13:31.934 | INFO | MemInfo: { | 20260506 12:13:31.934 | INFO | "gpu" : " 17 GB", | 20260506 12:13:31.934 | INFO | "huge" : "", | 20260506 12:13:31.934 | INFO | "huge_shared" : " 27 GB", | 20260506 12:13:31.934 | INFO | "malloc" : " 14 GB", | 20260506 12:13:31.934 | INFO | "pinned" : " 29 GB" | 20260506 12:13:31.934 | INFO | } | 20260506 12:13:31.934 | INFO | Perf report for the last 5.13902 seconds: | 20260506 12:13:31.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:13:31.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:31.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:31.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:31.934 | INFO | ANALYSIS : 62.0%, 3.0, 96.0ms, 0 | 20260506 12:13:31.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:31.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:31.934 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:13:31.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:31.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:31.934 | INFO | Uploaded 36GiB to GPU in 2.04497s (17.6041GiB/s) | 20260506 12:13:31.934 | INFO | Downloaded 34GiB to GPU in 1.96744s (17.2813GiB/s) | 20260506 12:13:37.074 | INFO | Analyzing chunk frames = [1267200,1267712) | 20260506 12:13:37.074 | INFO | MemInfo: { | 20260506 12:13:37.074 | INFO | "gpu" : " 17 GB", | 20260506 12:13:37.074 | INFO | "huge" : "", | 20260506 12:13:37.074 | INFO | "huge_shared" : " 27 GB", | 20260506 12:13:37.074 | INFO | "malloc" : " 14 GB", | 20260506 12:13:37.074 | INFO | "pinned" : " 29 GB" | 20260506 12:13:37.074 | INFO | } | 20260506 12:13:37.074 | INFO | Perf report for the last 5.1398 seconds: | 20260506 12:13:37.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:13:37.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:37.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:37.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:37.074 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260506 12:13:37.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:37.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:37.074 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:13:37.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:37.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:37.074 | INFO | Uploaded 36GiB to GPU in 2.07591s (17.3418GiB/s) | 20260506 12:13:37.074 | INFO | Downloaded 34GiB to GPU in 1.96437s (17.3083GiB/s) | 20260506 12:13:42.213 | INFO | Analyzing chunk frames = [1267712,1268224) | 20260506 12:13:42.213 | INFO | MemInfo: { | 20260506 12:13:42.213 | INFO | "gpu" : " 17 GB", | 20260506 12:13:42.213 | INFO | "huge" : "", | 20260506 12:13:42.213 | INFO | "huge_shared" : " 27 GB", | 20260506 12:13:42.213 | INFO | "malloc" : " 14 GB", | 20260506 12:13:42.213 | INFO | "pinned" : " 29 GB" | 20260506 12:13:42.213 | INFO | } | 20260506 12:13:42.213 | INFO | Perf report for the last 5.13868 seconds: | 20260506 12:13:42.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:13:42.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:42.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:42.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:42.213 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 12:13:42.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:42.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:42.213 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 12:13:42.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:42.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:42.213 | INFO | Uploaded 36GiB to GPU in 2.03151s (17.7208GiB/s) | 20260506 12:13:42.213 | INFO | Downloaded 34GiB to GPU in 1.95705s (17.373GiB/s) | 20260506 12:13:47.352 | INFO | Analyzing chunk frames = [1268224,1268736) | 20260506 12:13:47.352 | INFO | MemInfo: { | 20260506 12:13:47.352 | INFO | "gpu" : " 17 GB", | 20260506 12:13:47.352 | INFO | "huge" : "", | 20260506 12:13:47.352 | INFO | "huge_shared" : " 27 GB", | 20260506 12:13:47.352 | INFO | "malloc" : " 14 GB", | 20260506 12:13:47.352 | INFO | "pinned" : " 29 GB" | 20260506 12:13:47.352 | INFO | } | 20260506 12:13:47.352 | INFO | Perf report for the last 5.1395 seconds: | 20260506 12:13:47.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:13:47.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:47.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:47.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:47.352 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 12:13:47.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:47.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:47.352 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260506 12:13:47.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:47.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:47.352 | INFO | Uploaded 36GiB to GPU in 2.05703s (17.501GiB/s) | 20260506 12:13:47.352 | INFO | Downloaded 34GiB to GPU in 1.90267s (17.8696GiB/s) | 20260506 12:13:52.492 | INFO | Analyzing chunk frames = [1268736,1269248) | 20260506 12:13:52.492 | INFO | MemInfo: { | 20260506 12:13:52.492 | INFO | "gpu" : " 17 GB", | 20260506 12:13:52.492 | INFO | "huge" : "", | 20260506 12:13:52.492 | INFO | "huge_shared" : " 27 GB", | 20260506 12:13:52.492 | INFO | "malloc" : " 14 GB", | 20260506 12:13:52.492 | INFO | "pinned" : " 29 GB" | 20260506 12:13:52.492 | INFO | } | 20260506 12:13:52.492 | INFO | Perf report for the last 5.13984 seconds: | 20260506 12:13:52.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:13:52.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:52.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:52.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:52.492 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:13:52.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:52.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:52.492 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:13:52.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:52.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:52.492 | INFO | Uploaded 36GiB to GPU in 1.98029s (18.1791GiB/s) | 20260506 12:13:52.492 | INFO | Downloaded 34GiB to GPU in 2.02702s (16.7734GiB/s) | 20260506 12:13:57.631 | INFO | Analyzing chunk frames = [1269248,1269760) | 20260506 12:13:57.631 | INFO | MemInfo: { | 20260506 12:13:57.631 | INFO | "gpu" : " 17 GB", | 20260506 12:13:57.631 | INFO | "huge" : "", | 20260506 12:13:57.631 | INFO | "huge_shared" : " 27 GB", | 20260506 12:13:57.631 | INFO | "malloc" : " 14 GB", | 20260506 12:13:57.631 | INFO | "pinned" : " 29 GB" | 20260506 12:13:57.631 | INFO | } | 20260506 12:13:57.631 | INFO | Perf report for the last 5.13913 seconds: | 20260506 12:13:57.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:13:57.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:57.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:57.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:57.631 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260506 12:13:57.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:57.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:57.631 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:13:57.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:57.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:13:57.631 | INFO | Uploaded 36GiB to GPU in 2.08809s (17.2406GiB/s) | 20260506 12:13:57.631 | INFO | Downloaded 34GiB to GPU in 1.99381s (17.0528GiB/s) | 20260506 12:13:57.632 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:13:57.632 | INFO | Total mem usage: 16.8 GiB | 20260506 12:13:57.632 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:13:57.632 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.9%) | 20260506 12:13:57.632 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:13:57.632 | INFO | High Water Breakdown: | 20260506 12:13:57.632 | INFO | 59.7% : 8.65 GiB ( 83.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:13:57.632 | INFO | 40.2% : 5.83 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:13:57.632 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:13:57.632 | INFO | ------------------------------------------ | 20260506 12:13:57.632 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:13:57.632 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:13:57.632 | INFO | Total mem usage: 26.5 GiB | 20260506 12:13:57.632 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:13:57.632 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:13:57.632 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:13:57.632 | INFO | High Water Breakdown: | 20260506 12:13:57.632 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:13:57.632 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:13:57.632 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:13:57.632 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:13:57.632 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:13:57.632 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:13:57.632 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:13:57.632 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:13:57.632 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:13:57.632 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:13:57.632 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:13:57.632 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:13:57.632 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:13:57.632 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:13:57.632 | INFO | ------------------------------------------ | 20260506 12:13:57.632 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:13:57.632 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:13:57.632 | INFO | Total mem usage: 25.6 GiB | 20260506 12:13:57.632 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:13:57.632 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:13:57.632 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:13:57.632 | INFO | High Water Breakdown: | 20260506 12:13:57.632 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:13:57.632 | INFO | ------------------------------------------ | 20260506 12:13:57.632 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:13:57.632 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:13:57.632 | INFO | Total mem usage: 15.7 GiB | 20260506 12:13:57.632 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:13:57.632 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:13:57.632 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:13:57.632 | INFO | High Water Breakdown: | 20260506 12:13:57.632 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:13:57.632 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:13:57.632 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:13:57.632 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:13:57.632 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:13:57.632 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:13:57.632 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:13:57.632 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:13:57.632 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:13:57.633 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:13:57.633 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:13:57.633 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:13:57.633 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:13:57.633 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:13:57.633 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:13:57.633 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:13:57.633 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:13:57.633 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:13:57.633 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:13:57.633 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:13:57.633 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:13:57.633 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:13:57.633 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:13:57.633 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:13:57.633 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:13:57.633 | INFO | ------------------------------------------ | 20260506 12:13:57.633 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:13:57.633 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:13:57.633 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:13:57.633 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:13:57.633 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.64e+01, 1.35e+01 / 2.88e+00 | 20260506 12:13:57.633 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 9.93e-02, 1.03e-02 / 6.80e-03 | 20260506 12:13:57.633 | INFO | PerfReport: 11.55%, Baseline -- 3.67e+00 / 4.41e+00, 5.80e-01 / 3.45e-01 | 20260506 12:13:57.633 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.43e+00, 1.88e-01 / 1.60e-01 | 20260506 12:13:57.633 | INFO | PerfReport: 15.93%, DME -- 4.05e+01 / 4.53e+01, 4.42e+00 / 2.75e+00 | 20260506 12:13:57.633 | INFO | PerfReport: 56.31%, FrameLabeling -- 1.79e+01 / 2.00e+01, 1.36e+00 / 1.56e+00 | 20260506 12:13:57.633 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.42e+00 / 3.08e+00, 4.27e-01 / 1.74e-01 | 20260506 12:13:57.633 | INFO | PerfReport: 3.75%, Metrics -- 1.19e+00 / 1.59e+00, 3.03e-01 / 1.42e-01 | 20260506 12:13:57.633 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.20e-01 / 4.77e-01, 6.71e-02 / 2.87e-02 | 20260506 12:13:57.633 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:13:57.633 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:13:57.633 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:13:57.633 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 12:13:57.633 | INFO | PerfReport: 21.48%, Upload -- 2.13e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260506 12:13:57.633 | INFO | PerfReport: 20.97%, Download -- 2.07e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260506 12:13:57.633 | INFO | PerfReport: 32.13%, Compute -- 3.18e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 12:13:57.633 | INFO | PerfReport: 1.86%, LockUpload -- 1.84e+00 / 0.00e+00, 7.57e+00 / 0.00e+00 | 20260506 12:13:57.633 | INFO | PerfReport: 1.04%, LockDownload -- 1.03e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260506 12:13:57.633 | INFO | PerfReport: 22.50%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:13:57.633 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260506 12:13:57.757 | INFO | Starting a round of Baz IO | 20260506 12:14:00.935 | INFO | Populating RTMetrics report | 20260506 12:14:00.935 | INFO | RTMetrics report populated | 20260506 12:14:00.935 | INFO | Opening tmp RTMetrics json file | 20260506 12:14:00.936 | INFO | Serializing RTMetrics to json | 20260506 12:14:00.937 | INFO | Serialization done, swapping tmp file now... | 20260506 12:14:00.937 | INFO | RTMetrics done | 20260506 12:14:01.035 | INFO | Finished a round of Baz IO: Wrote 2.087 GiB in 3.277 seconds | 20260506 12:14:01.035 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.06% / 0.03% / 57.91% | 20260506 12:14:01.035 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.09966% / 5.16574e-05% / 7.54992e-05% | 20260506 12:14:01.035 | INFO | Baz IO overall: | 20260506 12:14:01.035 | INFO | BytesWritten : 626541916496 | 20260506 12:14:01.035 | INFO | NumEvents : 302712078416 | 20260506 12:14:02.770 | INFO | Analyzing chunk frames = [1269760,1270272) | 20260506 12:14:02.770 | INFO | MemInfo: { | 20260506 12:14:02.770 | INFO | "gpu" : " 17 GB", | 20260506 12:14:02.770 | INFO | "huge" : "", | 20260506 12:14:02.770 | INFO | "huge_shared" : " 27 GB", | 20260506 12:14:02.770 | INFO | "malloc" : " 14 GB", | 20260506 12:14:02.770 | INFO | "pinned" : " 29 GB" | 20260506 12:14:02.770 | INFO | } | 20260506 12:14:02.770 | INFO | Perf report for the last 5.13882 seconds: | 20260506 12:14:02.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:14:02.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:02.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:02.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:02.770 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260506 12:14:02.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:02.770 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:14:02.770 | INFO | PRE_HQ : 61.7%, 1.9, 61.7ms, 0 | 20260506 12:14:02.770 | INFO | BAZWRITER : 62.8%, 3.7, 120.2ms, 0 | 20260506 12:14:02.770 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:14:02.770 | INFO | Uploaded 36GiB to GPU in 2.11644s (17.0097GiB/s) | 20260506 12:14:02.770 | INFO | Downloaded 38GiB to GPU in 2.20688s (17.2188GiB/s) | 20260506 12:14:07.909 | INFO | Analyzing chunk frames = [1270272,1270784) | 20260506 12:14:07.909 | INFO | MemInfo: { | 20260506 12:14:07.909 | INFO | "gpu" : " 17 GB", | 20260506 12:14:07.909 | INFO | "huge" : "", | 20260506 12:14:07.909 | INFO | "huge_shared" : " 27 GB", | 20260506 12:14:07.909 | INFO | "malloc" : " 14 GB", | 20260506 12:14:07.909 | INFO | "pinned" : " 29 GB" | 20260506 12:14:07.909 | INFO | } | 20260506 12:14:07.910 | INFO | Perf report for the last 5.13953 seconds: | 20260506 12:14:07.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:14:07.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:07.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:07.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:07.910 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 12:14:07.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:07.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:07.910 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:14:07.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:07.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:07.910 | INFO | Uploaded 36GiB to GPU in 2.01884s (17.832GiB/s) | 20260506 12:14:07.910 | INFO | Downloaded 34GiB to GPU in 1.92224s (17.6877GiB/s) | 20260506 12:14:13.049 | INFO | Analyzing chunk frames = [1270784,1271296) | 20260506 12:14:13.049 | INFO | MemInfo: { | 20260506 12:14:13.049 | INFO | "gpu" : " 17 GB", | 20260506 12:14:13.049 | INFO | "huge" : "", | 20260506 12:14:13.049 | INFO | "huge_shared" : " 27 GB", | 20260506 12:14:13.049 | INFO | "malloc" : " 14 GB", | 20260506 12:14:13.049 | INFO | "pinned" : " 29 GB" | 20260506 12:14:13.049 | INFO | } | 20260506 12:14:13.049 | INFO | Perf report for the last 5.13997 seconds: | 20260506 12:14:13.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:14:13.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:13.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:13.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:13.049 | INFO | ANALYSIS : 61.9%, 3.0, 95.8ms, 0 | 20260506 12:14:13.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:13.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:13.050 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:14:13.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:13.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:13.050 | INFO | Uploaded 36GiB to GPU in 2.03148s (17.7211GiB/s) | 20260506 12:14:13.050 | INFO | Downloaded 34GiB to GPU in 1.93908s (17.5341GiB/s) | 20260506 12:14:18.188 | INFO | Analyzing chunk frames = [1271296,1271808) | 20260506 12:14:18.188 | INFO | MemInfo: { | 20260506 12:14:18.188 | INFO | "gpu" : " 17 GB", | 20260506 12:14:18.188 | INFO | "huge" : "", | 20260506 12:14:18.188 | INFO | "huge_shared" : " 27 GB", | 20260506 12:14:18.188 | INFO | "malloc" : " 14 GB", | 20260506 12:14:18.188 | INFO | "pinned" : " 29 GB" | 20260506 12:14:18.188 | INFO | } | 20260506 12:14:18.188 | INFO | Perf report for the last 5.13874 seconds: | 20260506 12:14:18.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:14:18.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:18.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:18.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:18.188 | INFO | ANALYSIS : 61.4%, 3.0, 95.2ms, 0 | 20260506 12:14:18.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:18.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:18.188 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:14:18.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:18.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:18.189 | INFO | Uploaded 36GiB to GPU in 2.04553s (17.5993GiB/s) | 20260506 12:14:18.189 | INFO | Downloaded 34GiB to GPU in 1.96041s (17.3434GiB/s) | 20260506 12:14:23.327 | INFO | Analyzing chunk frames = [1271808,1272320) | 20260506 12:14:23.327 | INFO | MemInfo: { | 20260506 12:14:23.327 | INFO | "gpu" : " 17 GB", | 20260506 12:14:23.327 | INFO | "huge" : "", | 20260506 12:14:23.327 | INFO | "huge_shared" : " 27 GB", | 20260506 12:14:23.327 | INFO | "malloc" : " 14 GB", | 20260506 12:14:23.327 | INFO | "pinned" : " 29 GB" | 20260506 12:14:23.327 | INFO | } | 20260506 12:14:23.327 | INFO | Perf report for the last 5.13894 seconds: | 20260506 12:14:23.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:14:23.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:23.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:23.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:23.327 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 12:14:23.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:23.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:23.327 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:14:23.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:23.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:23.327 | INFO | Uploaded 36GiB to GPU in 2.05478s (17.5201GiB/s) | 20260506 12:14:23.328 | INFO | Downloaded 34GiB to GPU in 1.94745s (17.4587GiB/s) | 20260506 12:14:28.467 | INFO | Analyzing chunk frames = [1272320,1272832) | 20260506 12:14:28.467 | INFO | MemInfo: { | 20260506 12:14:28.467 | INFO | "gpu" : " 17 GB", | 20260506 12:14:28.467 | INFO | "huge" : "", | 20260506 12:14:28.467 | INFO | "huge_shared" : " 27 GB", | 20260506 12:14:28.467 | INFO | "malloc" : " 14 GB", | 20260506 12:14:28.467 | INFO | "pinned" : " 29 GB" | 20260506 12:14:28.467 | INFO | } | 20260506 12:14:28.467 | INFO | Perf report for the last 5.13972 seconds: | 20260506 12:14:28.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:14:28.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:28.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:28.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:28.467 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:14:28.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:28.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:28.467 | INFO | PRE_HQ : 31.4%, 1.0, 16.4ms, 0 | 20260506 12:14:28.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:28.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:28.467 | INFO | Uploaded 36GiB to GPU in 2.11055s (17.0572GiB/s) | 20260506 12:14:28.467 | INFO | Downloaded 34GiB to GPU in 1.97943s (17.1767GiB/s) | 20260506 12:14:33.607 | INFO | Analyzing chunk frames = [1272832,1273344) | 20260506 12:14:33.607 | INFO | MemInfo: { | 20260506 12:14:33.607 | INFO | "gpu" : " 17 GB", | 20260506 12:14:33.607 | INFO | "huge" : "", | 20260506 12:14:33.607 | INFO | "huge_shared" : " 27 GB", | 20260506 12:14:33.607 | INFO | "malloc" : " 14 GB", | 20260506 12:14:33.607 | INFO | "pinned" : " 29 GB" | 20260506 12:14:33.607 | INFO | } | 20260506 12:14:33.607 | INFO | Perf report for the last 5.14024 seconds: | 20260506 12:14:33.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:14:33.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:33.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:33.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:33.607 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:14:33.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:33.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:33.607 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 12:14:33.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:33.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:33.607 | INFO | Uploaded 36GiB to GPU in 2.02115s (17.8116GiB/s) | 20260506 12:14:33.607 | INFO | Downloaded 34GiB to GPU in 2.01232s (16.8959GiB/s) | 20260506 12:14:38.745 | INFO | Analyzing chunk frames = [1273344,1273856) | 20260506 12:14:38.745 | INFO | MemInfo: { | 20260506 12:14:38.745 | INFO | "gpu" : " 17 GB", | 20260506 12:14:38.745 | INFO | "huge" : "", | 20260506 12:14:38.745 | INFO | "huge_shared" : " 27 GB", | 20260506 12:14:38.745 | INFO | "malloc" : " 14 GB", | 20260506 12:14:38.745 | INFO | "pinned" : " 29 GB" | 20260506 12:14:38.745 | INFO | } | 20260506 12:14:38.745 | INFO | Perf report for the last 5.13819 seconds: | 20260506 12:14:38.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:14:38.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:38.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:38.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:38.746 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:14:38.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:38.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:38.746 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:14:38.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:38.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:38.746 | INFO | Uploaded 36GiB to GPU in 2.05246s (17.54GiB/s) | 20260506 12:14:38.746 | INFO | Downloaded 34GiB to GPU in 1.96273s (17.3228GiB/s) | 20260506 12:14:38.874 | INFO | Starting a round of Baz IO | 20260506 12:14:42.043 | INFO | Populating RTMetrics report | 20260506 12:14:42.044 | INFO | RTMetrics report populated | 20260506 12:14:42.044 | INFO | Opening tmp RTMetrics json file | 20260506 12:14:42.044 | INFO | Serializing RTMetrics to json | 20260506 12:14:42.045 | INFO | Serialization done, swapping tmp file now... | 20260506 12:14:42.046 | INFO | RTMetrics done | 20260506 12:14:42.146 | INFO | Finished a round of Baz IO: Wrote 2.082 GiB in 3.272 seconds | 20260506 12:14:42.146 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.04% / 0.03% / 57.94% | 20260506 12:14:42.146 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.10095% / 5.16574e-05% / 5.16574e-05% | 20260506 12:14:42.146 | INFO | Baz IO overall: | 20260506 12:14:42.146 | INFO | BytesWritten : 628777602995 | 20260506 12:14:42.146 | INFO | NumEvents : 303794773423 | 20260506 12:14:43.885 | INFO | Analyzing chunk frames = [1273856,1274368) | 20260506 12:14:43.885 | INFO | MemInfo: { | 20260506 12:14:43.885 | INFO | "gpu" : " 17 GB", | 20260506 12:14:43.885 | INFO | "huge" : "", | 20260506 12:14:43.885 | INFO | "huge_shared" : " 27 GB", | 20260506 12:14:43.885 | INFO | "malloc" : " 14 GB", | 20260506 12:14:43.885 | INFO | "pinned" : " 29 GB" | 20260506 12:14:43.885 | INFO | } | 20260506 12:14:43.885 | INFO | Perf report for the last 5.13968 seconds: | 20260506 12:14:43.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:14:43.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:43.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:43.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:43.885 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260506 12:14:43.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:43.885 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:14:43.885 | INFO | PRE_HQ : 61.6%, 1.9, 61.4ms, 0 | 20260506 12:14:43.885 | INFO | BAZWRITER : 63.0%, 3.7, 120.2ms, 0 | 20260506 12:14:43.885 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:14:43.885 | INFO | Uploaded 36GiB to GPU in 2.08316s (17.2815GiB/s) | 20260506 12:14:43.885 | INFO | Downloaded 38GiB to GPU in 2.21069s (17.1892GiB/s) | 20260506 12:14:49.024 | INFO | Analyzing chunk frames = [1274368,1274880) | 20260506 12:14:49.024 | INFO | MemInfo: { | 20260506 12:14:49.024 | INFO | "gpu" : " 17 GB", | 20260506 12:14:49.024 | INFO | "huge" : "", | 20260506 12:14:49.024 | INFO | "huge_shared" : " 27 GB", | 20260506 12:14:49.024 | INFO | "malloc" : " 14 GB", | 20260506 12:14:49.024 | INFO | "pinned" : " 29 GB" | 20260506 12:14:49.024 | INFO | } | 20260506 12:14:49.024 | INFO | Perf report for the last 5.13923 seconds: | 20260506 12:14:49.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:14:49.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:49.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:49.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:49.024 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 12:14:49.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:49.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:49.024 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 12:14:49.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:49.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:49.024 | INFO | Uploaded 36GiB to GPU in 2.01873s (17.833GiB/s) | 20260506 12:14:49.025 | INFO | Downloaded 34GiB to GPU in 1.90895s (17.8108GiB/s) | 20260506 12:14:54.163 | INFO | Analyzing chunk frames = [1274880,1275392) | 20260506 12:14:54.163 | INFO | MemInfo: { | 20260506 12:14:54.163 | INFO | "gpu" : " 17 GB", | 20260506 12:14:54.163 | INFO | "huge" : "", | 20260506 12:14:54.163 | INFO | "huge_shared" : " 27 GB", | 20260506 12:14:54.163 | INFO | "malloc" : " 14 GB", | 20260506 12:14:54.163 | INFO | "pinned" : " 29 GB" | 20260506 12:14:54.163 | INFO | } | 20260506 12:14:54.164 | INFO | Perf report for the last 5.13922 seconds: | 20260506 12:14:54.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:14:54.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:54.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:54.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:54.164 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:14:54.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:54.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:54.164 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:14:54.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:54.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:54.164 | INFO | Uploaded 36GiB to GPU in 2.00713s (17.9361GiB/s) | 20260506 12:14:54.164 | INFO | Downloaded 34GiB to GPU in 1.93825s (17.5416GiB/s) | 20260506 12:14:59.303 | INFO | Analyzing chunk frames = [1275392,1275904) | 20260506 12:14:59.303 | INFO | MemInfo: { | 20260506 12:14:59.303 | INFO | "gpu" : " 17 GB", | 20260506 12:14:59.303 | INFO | "huge" : "", | 20260506 12:14:59.303 | INFO | "huge_shared" : " 27 GB", | 20260506 12:14:59.303 | INFO | "malloc" : " 14 GB", | 20260506 12:14:59.303 | INFO | "pinned" : " 29 GB" | 20260506 12:14:59.303 | INFO | } | 20260506 12:14:59.303 | INFO | Perf report for the last 5.13954 seconds: | 20260506 12:14:59.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:14:59.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:59.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:59.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:59.303 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 12:14:59.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:59.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:59.303 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:14:59.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:59.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:14:59.303 | INFO | Uploaded 36GiB to GPU in 2.00629s (17.9436GiB/s) | 20260506 12:14:59.303 | INFO | Downloaded 34GiB to GPU in 1.93887s (17.536GiB/s) | 20260506 12:15:04.442 | INFO | Analyzing chunk frames = [1275904,1276416) | 20260506 12:15:04.442 | INFO | MemInfo: { | 20260506 12:15:04.442 | INFO | "gpu" : " 17 GB", | 20260506 12:15:04.442 | INFO | "huge" : "", | 20260506 12:15:04.442 | INFO | "huge_shared" : " 27 GB", | 20260506 12:15:04.442 | INFO | "malloc" : " 14 GB", | 20260506 12:15:04.442 | INFO | "pinned" : " 29 GB" | 20260506 12:15:04.442 | INFO | } | 20260506 12:15:04.442 | INFO | Perf report for the last 5.13936 seconds: | 20260506 12:15:04.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:15:04.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:04.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:04.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:04.442 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 12:15:04.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:04.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:04.443 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 12:15:04.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:04.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:04.443 | INFO | Uploaded 36GiB to GPU in 2.02308s (17.7947GiB/s) | 20260506 12:15:04.443 | INFO | Downloaded 34GiB to GPU in 1.94084s (17.5182GiB/s) | 20260506 12:15:09.581 | INFO | Analyzing chunk frames = [1276416,1276928) | 20260506 12:15:09.581 | INFO | MemInfo: { | 20260506 12:15:09.581 | INFO | "gpu" : " 17 GB", | 20260506 12:15:09.581 | INFO | "huge" : "", | 20260506 12:15:09.581 | INFO | "huge_shared" : " 27 GB", | 20260506 12:15:09.581 | INFO | "malloc" : " 14 GB", | 20260506 12:15:09.582 | INFO | "pinned" : " 29 GB" | 20260506 12:15:09.582 | INFO | } | 20260506 12:15:09.582 | INFO | Perf report for the last 5.13916 seconds: | 20260506 12:15:09.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:15:09.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:09.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:09.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:09.582 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:15:09.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:09.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:09.582 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:15:09.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:09.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:09.582 | INFO | Uploaded 36GiB to GPU in 2.11061s (17.0567GiB/s) | 20260506 12:15:09.582 | INFO | Downloaded 34GiB to GPU in 1.98466s (17.1314GiB/s) | 20260506 12:15:14.722 | INFO | Analyzing chunk frames = [1276928,1277440) | 20260506 12:15:14.722 | INFO | MemInfo: { | 20260506 12:15:14.722 | INFO | "gpu" : " 17 GB", | 20260506 12:15:14.722 | INFO | "huge" : "", | 20260506 12:15:14.722 | INFO | "huge_shared" : " 27 GB", | 20260506 12:15:14.722 | INFO | "malloc" : " 14 GB", | 20260506 12:15:14.722 | INFO | "pinned" : " 29 GB" | 20260506 12:15:14.722 | INFO | } | 20260506 12:15:14.722 | INFO | Perf report for the last 5.14048 seconds: | 20260506 12:15:14.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:15:14.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:14.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:14.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:14.722 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:15:14.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:14.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:14.722 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 12:15:14.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:14.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:14.722 | INFO | Uploaded 36GiB to GPU in 1.99544s (18.0411GiB/s) | 20260506 12:15:14.723 | INFO | Downloaded 34GiB to GPU in 1.99564s (17.0372GiB/s) | 20260506 12:15:19.860 | INFO | Analyzing chunk frames = [1277440,1277952) | 20260506 12:15:19.860 | INFO | MemInfo: { | 20260506 12:15:19.860 | INFO | "gpu" : " 17 GB", | 20260506 12:15:19.860 | INFO | "huge" : "", | 20260506 12:15:19.860 | INFO | "huge_shared" : " 27 GB", | 20260506 12:15:19.860 | INFO | "malloc" : " 14 GB", | 20260506 12:15:19.860 | INFO | "pinned" : " 29 GB" | 20260506 12:15:19.860 | INFO | } | 20260506 12:15:19.860 | INFO | Perf report for the last 5.13833 seconds: | 20260506 12:15:19.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:15:19.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:19.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:19.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:19.860 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 12:15:19.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:19.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:19.861 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:15:19.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:19.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:19.861 | INFO | Uploaded 36GiB to GPU in 2.0228s (17.7971GiB/s) | 20260506 12:15:19.861 | INFO | Downloaded 34GiB to GPU in 2.00401s (16.9659GiB/s) | 20260506 12:15:19.861 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:15:19.861 | INFO | Total mem usage: 16.8 GiB | 20260506 12:15:19.861 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:15:19.861 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.6%) | 20260506 12:15:19.861 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:15:19.861 | INFO | High Water Breakdown: | 20260506 12:15:19.861 | INFO | 59.6% : 8.59 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:15:19.861 | INFO | 40.3% : 5.81 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:15:19.861 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:15:19.861 | INFO | ------------------------------------------ | 20260506 12:15:19.861 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:15:19.861 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:15:19.861 | INFO | Total mem usage: 26.5 GiB | 20260506 12:15:19.861 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:15:19.861 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:15:19.861 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:15:19.861 | INFO | High Water Breakdown: | 20260506 12:15:19.861 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:15:19.861 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:15:19.861 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:15:19.861 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:15:19.861 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:15:19.861 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:15:19.861 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:15:19.861 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:15:19.861 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:15:19.861 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:15:19.861 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:15:19.861 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:15:19.861 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:15:19.861 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:15:19.861 | INFO | ------------------------------------------ | 20260506 12:15:19.861 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:15:19.861 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:15:19.861 | INFO | Total mem usage: 25.6 GiB | 20260506 12:15:19.861 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:15:19.861 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:15:19.861 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:15:19.861 | INFO | High Water Breakdown: | 20260506 12:15:19.861 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:15:19.861 | INFO | ------------------------------------------ | 20260506 12:15:19.861 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:15:19.861 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:15:19.861 | INFO | Total mem usage: 15.7 GiB | 20260506 12:15:19.861 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:15:19.861 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:15:19.861 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:15:19.861 | INFO | High Water Breakdown: | 20260506 12:15:19.861 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:15:19.861 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:15:19.861 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:15:19.862 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:15:19.862 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:15:19.862 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:15:19.862 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:15:19.862 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:15:19.862 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:15:19.862 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:15:19.862 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:15:19.862 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:15:19.862 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:15:19.862 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:15:19.862 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:15:19.862 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:15:19.862 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:15:19.862 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:15:19.862 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:15:19.862 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:15:19.862 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:15:19.862 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:15:19.862 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:15:19.862 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:15:19.862 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:15:19.862 | INFO | ------------------------------------------ | 20260506 12:15:19.862 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:15:19.862 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:15:19.862 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:15:19.862 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:15:19.862 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.58e+01, 1.36e+01 / 1.83e+00 | 20260506 12:15:19.862 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.00e-01, 1.15e-02 / 6.07e-03 | 20260506 12:15:19.862 | INFO | PerfReport: 11.56%, Baseline -- 3.67e+00 / 4.45e+00, 5.79e-01 / 3.10e-01 | 20260506 12:15:19.862 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.45e+00, 1.91e-01 / 1.30e-01 | 20260506 12:15:19.862 | INFO | PerfReport: 15.84%, DME -- 4.03e+01 / 4.46e+01, 3.93e+00 / 2.03e+00 | 20260506 12:15:19.862 | INFO | PerfReport: 56.37%, FrameLabeling -- 1.79e+01 / 2.01e+01, 1.44e+00 / 1.64e+00 | 20260506 12:15:19.862 | INFO | PerfReport: 7.62%, PulseAccumulating -- 2.42e+00 / 3.09e+00, 4.31e-01 / 2.06e-01 | 20260506 12:15:19.862 | INFO | PerfReport: 3.75%, Metrics -- 1.19e+00 / 1.55e+00, 2.99e-01 / 1.27e-01 | 20260506 12:15:19.862 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.21e-01 / 4.81e-01, 6.66e-02 / 3.34e-02 | 20260506 12:15:19.862 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:15:19.862 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:15:19.862 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:15:19.862 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 12:15:19.862 | INFO | PerfReport: 21.52%, Upload -- 2.13e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260506 12:15:19.862 | INFO | PerfReport: 20.95%, Download -- 2.07e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 12:15:19.862 | INFO | PerfReport: 32.12%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 12:15:19.862 | INFO | PerfReport: 1.94%, LockUpload -- 1.92e+00 / 0.00e+00, 7.74e+00 / 0.00e+00 | 20260506 12:15:19.862 | INFO | PerfReport: 0.94%, LockDownload -- 9.30e-01 / 0.00e+00, 2.49e+00 / 0.00e+00 | 20260506 12:15:19.862 | INFO | PerfReport: 22.52%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:15:19.862 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.29e-02 / 0.00e+00 | 20260506 12:15:19.990 | INFO | Starting a round of Baz IO | 20260506 12:15:23.160 | INFO | Populating RTMetrics report | 20260506 12:15:23.160 | INFO | RTMetrics report populated | 20260506 12:15:23.160 | INFO | Opening tmp RTMetrics json file | 20260506 12:15:23.161 | INFO | Serializing RTMetrics to json | 20260506 12:15:23.162 | INFO | Serialization done, swapping tmp file now... | 20260506 12:15:23.163 | INFO | RTMetrics done | 20260506 12:15:23.259 | INFO | Finished a round of Baz IO: Wrote 2.08 GiB in 3.268 seconds | 20260506 12:15:23.259 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 42.01% / 0.03% / 57.96% | 20260506 12:15:23.259 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.0961% / 4.37101e-05% / 5.16574e-05% | 20260506 12:15:23.259 | INFO | Baz IO overall: | 20260506 12:15:23.259 | INFO | BytesWritten : 631011382404 | 20260506 12:15:23.259 | INFO | NumEvents : 304876978098 | 20260506 12:15:25.000 | INFO | Analyzing chunk frames = [1277952,1278464) | 20260506 12:15:25.000 | INFO | MemInfo: { | 20260506 12:15:25.000 | INFO | "gpu" : " 17 GB", | 20260506 12:15:25.000 | INFO | "huge" : "", | 20260506 12:15:25.000 | INFO | "huge_shared" : " 27 GB", | 20260506 12:15:25.000 | INFO | "malloc" : " 14 GB", | 20260506 12:15:25.000 | INFO | "pinned" : " 29 GB" | 20260506 12:15:25.000 | INFO | } | 20260506 12:15:25.000 | INFO | Perf report for the last 5.13917 seconds: | 20260506 12:15:25.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:15:25.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:25.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:25.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:25.000 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260506 12:15:25.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:25.000 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 12:15:25.000 | INFO | PRE_HQ : 61.4%, 1.9, 61.3ms, 0 | 20260506 12:15:25.000 | INFO | BAZWRITER : 63.0%, 3.6, 117.5ms, 0 | 20260506 12:15:25.000 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:15:25.000 | INFO | Uploaded 36GiB to GPU in 2.06983s (17.3928GiB/s) | 20260506 12:15:25.000 | INFO | Downloaded 38GiB to GPU in 2.22066s (17.112GiB/s) | 20260506 12:15:30.139 | INFO | Analyzing chunk frames = [1278464,1278976) | 20260506 12:15:30.139 | INFO | MemInfo: { | 20260506 12:15:30.139 | INFO | "gpu" : " 17 GB", | 20260506 12:15:30.139 | INFO | "huge" : "", | 20260506 12:15:30.139 | INFO | "huge_shared" : " 27 GB", | 20260506 12:15:30.139 | INFO | "malloc" : " 14 GB", | 20260506 12:15:30.139 | INFO | "pinned" : " 29 GB" | 20260506 12:15:30.139 | INFO | } | 20260506 12:15:30.139 | INFO | Perf report for the last 5.13938 seconds: | 20260506 12:15:30.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:15:30.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:30.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:30.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:30.139 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:15:30.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:30.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:30.139 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:15:30.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:30.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:30.139 | INFO | Uploaded 36GiB to GPU in 2.01476s (17.8681GiB/s) | 20260506 12:15:30.139 | INFO | Downloaded 34GiB to GPU in 1.9195s (17.7129GiB/s) | 20260506 12:15:35.278 | INFO | Analyzing chunk frames = [1278976,1279488) | 20260506 12:15:35.278 | INFO | MemInfo: { | 20260506 12:15:35.278 | INFO | "gpu" : " 17 GB", | 20260506 12:15:35.278 | INFO | "huge" : "", | 20260506 12:15:35.278 | INFO | "huge_shared" : " 27 GB", | 20260506 12:15:35.278 | INFO | "malloc" : " 14 GB", | 20260506 12:15:35.278 | INFO | "pinned" : " 29 GB" | 20260506 12:15:35.278 | INFO | } | 20260506 12:15:35.278 | INFO | Perf report for the last 5.13909 seconds: | 20260506 12:15:35.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:15:35.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:35.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:35.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:35.278 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:15:35.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:35.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:35.278 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:15:35.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:35.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:35.278 | INFO | Uploaded 36GiB to GPU in 2.02666s (17.7632GiB/s) | 20260506 12:15:35.278 | INFO | Downloaded 34GiB to GPU in 1.94097s (17.517GiB/s) | 20260506 12:15:40.418 | INFO | Analyzing chunk frames = [1279488,1280000) | 20260506 12:15:40.418 | INFO | MemInfo: { | 20260506 12:15:40.418 | INFO | "gpu" : " 17 GB", | 20260506 12:15:40.418 | INFO | "huge" : "", | 20260506 12:15:40.418 | INFO | "huge_shared" : " 27 GB", | 20260506 12:15:40.418 | INFO | "malloc" : " 14 GB", | 20260506 12:15:40.418 | INFO | "pinned" : " 29 GB" | 20260506 12:15:40.418 | INFO | } | 20260506 12:15:40.418 | INFO | Perf report for the last 5.13998 seconds: | 20260506 12:15:40.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:15:40.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:40.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:40.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:40.418 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 12:15:40.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:40.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:40.418 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:15:40.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:40.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:40.419 | INFO | Uploaded 36GiB to GPU in 2.00799s (17.9284GiB/s) | 20260506 12:15:40.419 | INFO | Downloaded 34GiB to GPU in 1.94459s (17.4844GiB/s) | 20260506 12:15:45.557 | INFO | Analyzing chunk frames = [1280000,1280512) | 20260506 12:15:45.557 | INFO | MemInfo: { | 20260506 12:15:45.557 | INFO | "gpu" : " 17 GB", | 20260506 12:15:45.557 | INFO | "huge" : "", | 20260506 12:15:45.557 | INFO | "huge_shared" : " 27 GB", | 20260506 12:15:45.557 | INFO | "malloc" : " 14 GB", | 20260506 12:15:45.557 | INFO | "pinned" : " 29 GB" | 20260506 12:15:45.557 | INFO | } | 20260506 12:15:45.557 | INFO | Perf report for the last 5.13885 seconds: | 20260506 12:15:45.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:15:45.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:45.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:45.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:45.557 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 12:15:45.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:45.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:45.557 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:15:45.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:45.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:45.557 | INFO | Uploaded 36GiB to GPU in 2.01571s (17.8597GiB/s) | 20260506 12:15:45.557 | INFO | Downloaded 34GiB to GPU in 1.93554s (17.5661GiB/s) | 20260506 12:15:50.696 | INFO | Analyzing chunk frames = [1280512,1281024) | 20260506 12:15:50.696 | INFO | MemInfo: { | 20260506 12:15:50.696 | INFO | "gpu" : " 17 GB", | 20260506 12:15:50.696 | INFO | "huge" : "", | 20260506 12:15:50.696 | INFO | "huge_shared" : " 27 GB", | 20260506 12:15:50.696 | INFO | "malloc" : " 14 GB", | 20260506 12:15:50.696 | INFO | "pinned" : " 29 GB" | 20260506 12:15:50.696 | INFO | } | 20260506 12:15:50.696 | INFO | Perf report for the last 5.13934 seconds: | 20260506 12:15:50.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:15:50.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:50.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:50.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:50.697 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:15:50.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:50.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:50.697 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:15:50.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:50.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:50.697 | INFO | Uploaded 36GiB to GPU in 2.10377s (17.1122GiB/s) | 20260506 12:15:50.697 | INFO | Downloaded 34GiB to GPU in 1.97914s (17.1792GiB/s) | 20260506 12:15:55.837 | INFO | Analyzing chunk frames = [1281024,1281536) | 20260506 12:15:55.837 | INFO | MemInfo: { | 20260506 12:15:55.837 | INFO | "gpu" : " 17 GB", | 20260506 12:15:55.837 | INFO | "huge" : "", | 20260506 12:15:55.837 | INFO | "huge_shared" : " 27 GB", | 20260506 12:15:55.837 | INFO | "malloc" : " 14 GB", | 20260506 12:15:55.837 | INFO | "pinned" : " 29 GB" | 20260506 12:15:55.837 | INFO | } | 20260506 12:15:55.837 | INFO | Perf report for the last 5.14078 seconds: | 20260506 12:15:55.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:15:55.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:55.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:55.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:55.837 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:15:55.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:55.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:55.837 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 12:15:55.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:55.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:15:55.837 | INFO | Uploaded 36GiB to GPU in 1.99724s (18.0249GiB/s) | 20260506 12:15:55.837 | INFO | Downloaded 34GiB to GPU in 1.97024s (17.2568GiB/s) | 20260506 12:16:00.975 | INFO | Analyzing chunk frames = [1281536,1282048) | 20260506 12:16:00.975 | INFO | MemInfo: { | 20260506 12:16:00.975 | INFO | "gpu" : " 17 GB", | 20260506 12:16:00.975 | INFO | "huge" : "", | 20260506 12:16:00.975 | INFO | "huge_shared" : " 27 GB", | 20260506 12:16:00.975 | INFO | "malloc" : " 14 GB", | 20260506 12:16:00.975 | INFO | "pinned" : " 29 GB" | 20260506 12:16:00.975 | INFO | } | 20260506 12:16:00.975 | INFO | Perf report for the last 5.13825 seconds: | 20260506 12:16:00.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:16:00.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:00.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:00.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:00.975 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 12:16:00.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:00.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:00.975 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:16:00.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:00.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:00.976 | INFO | Uploaded 36GiB to GPU in 2.04678s (17.5886GiB/s) | 20260506 12:16:00.976 | INFO | Downloaded 34GiB to GPU in 1.9789s (17.1812GiB/s) | 20260506 12:16:01.098 | INFO | Starting a round of Baz IO | 20260506 12:16:04.273 | INFO | Populating RTMetrics report | 20260506 12:16:04.273 | INFO | RTMetrics report populated | 20260506 12:16:04.273 | INFO | Opening tmp RTMetrics json file | 20260506 12:16:04.274 | INFO | Serializing RTMetrics to json | 20260506 12:16:04.275 | INFO | Serialization done, swapping tmp file now... | 20260506 12:16:04.275 | INFO | RTMetrics done | 20260506 12:16:04.376 | INFO | Finished a round of Baz IO: Wrote 2.081 GiB in 3.278 seconds | 20260506 12:16:04.376 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.98% / 0.03% / 57.99% | 20260506 12:16:04.377 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.09646% / 4.37101e-05% / 5.5631e-05% | 20260506 12:16:04.377 | INFO | Baz IO overall: | 20260506 12:16:04.377 | INFO | BytesWritten : 633245926399 | 20260506 12:16:04.377 | INFO | NumEvents : 305960000379 | 20260506 12:16:06.115 | INFO | Analyzing chunk frames = [1282048,1282560) | 20260506 12:16:06.115 | INFO | MemInfo: { | 20260506 12:16:06.115 | INFO | "gpu" : " 17 GB", | 20260506 12:16:06.115 | INFO | "huge" : "", | 20260506 12:16:06.115 | INFO | "huge_shared" : " 27 GB", | 20260506 12:16:06.115 | INFO | "malloc" : " 14 GB", | 20260506 12:16:06.115 | INFO | "pinned" : " 29 GB" | 20260506 12:16:06.115 | INFO | } | 20260506 12:16:06.115 | INFO | Perf report for the last 5.13933 seconds: | 20260506 12:16:06.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:16:06.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:06.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:06.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:06.115 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260506 12:16:06.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:06.115 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:16:06.115 | INFO | PRE_HQ : 61.6%, 1.9, 61.3ms, 0 | 20260506 12:16:06.115 | INFO | BAZWRITER : 63.0%, 3.7, 118.6ms, 0 | 20260506 12:16:06.115 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:16:06.115 | INFO | Uploaded 36GiB to GPU in 2.08806s (17.2409GiB/s) | 20260506 12:16:06.115 | INFO | Downloaded 38GiB to GPU in 2.20806s (17.2097GiB/s) | 20260506 12:16:11.254 | INFO | Analyzing chunk frames = [1282560,1283072) | 20260506 12:16:11.254 | INFO | MemInfo: { | 20260506 12:16:11.254 | INFO | "gpu" : " 17 GB", | 20260506 12:16:11.254 | INFO | "huge" : "", | 20260506 12:16:11.254 | INFO | "huge_shared" : " 27 GB", | 20260506 12:16:11.254 | INFO | "malloc" : " 14 GB", | 20260506 12:16:11.254 | INFO | "pinned" : " 29 GB" | 20260506 12:16:11.254 | INFO | } | 20260506 12:16:11.254 | INFO | Perf report for the last 5.13929 seconds: | 20260506 12:16:11.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:16:11.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:11.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:11.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:11.254 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:16:11.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:11.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:11.254 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:16:11.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:11.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:11.254 | INFO | Uploaded 36GiB to GPU in 2.00626s (17.9439GiB/s) | 20260506 12:16:11.254 | INFO | Downloaded 34GiB to GPU in 1.91777s (17.729GiB/s) | 20260506 12:16:16.393 | INFO | Analyzing chunk frames = [1283072,1283584) | 20260506 12:16:16.393 | INFO | MemInfo: { | 20260506 12:16:16.393 | INFO | "gpu" : " 17 GB", | 20260506 12:16:16.393 | INFO | "huge" : "", | 20260506 12:16:16.393 | INFO | "huge_shared" : " 27 GB", | 20260506 12:16:16.393 | INFO | "malloc" : " 14 GB", | 20260506 12:16:16.393 | INFO | "pinned" : " 29 GB" | 20260506 12:16:16.393 | INFO | } | 20260506 12:16:16.393 | INFO | Perf report for the last 5.13916 seconds: | 20260506 12:16:16.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:16:16.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:16.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:16.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:16.393 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:16:16.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:16.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:16.393 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:16:16.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:16.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:16.393 | INFO | Uploaded 36GiB to GPU in 2.01116s (17.9001GiB/s) | 20260506 12:16:16.393 | INFO | Downloaded 34GiB to GPU in 1.94785s (17.4551GiB/s) | 20260506 12:16:21.540 | INFO | Analyzing chunk frames = [1283584,1284096) | 20260506 12:16:21.540 | INFO | MemInfo: { | 20260506 12:16:21.540 | INFO | "gpu" : " 17 GB", | 20260506 12:16:21.540 | INFO | "huge" : "", | 20260506 12:16:21.540 | INFO | "huge_shared" : " 27 GB", | 20260506 12:16:21.540 | INFO | "malloc" : " 14 GB", | 20260506 12:16:21.540 | INFO | "pinned" : " 29 GB" | 20260506 12:16:21.540 | INFO | } | 20260506 12:16:21.540 | INFO | Perf report for the last 5.1467 seconds: | 20260506 12:16:21.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:16:21.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:21.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:21.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:21.540 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 12:16:21.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:21.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:21.540 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 12:16:21.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:21.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:21.540 | INFO | Uploaded 36GiB to GPU in 1.98935s (18.0964GiB/s) | 20260506 12:16:21.540 | INFO | Downloaded 34GiB to GPU in 1.94588s (17.4729GiB/s) | 20260506 12:16:26.672 | INFO | Analyzing chunk frames = [1284096,1284608) | 20260506 12:16:26.672 | INFO | MemInfo: { | 20260506 12:16:26.672 | INFO | "gpu" : " 17 GB", | 20260506 12:16:26.672 | INFO | "huge" : "", | 20260506 12:16:26.672 | INFO | "huge_shared" : " 27 GB", | 20260506 12:16:26.672 | INFO | "malloc" : " 14 GB", | 20260506 12:16:26.672 | INFO | "pinned" : " 29 GB" | 20260506 12:16:26.673 | INFO | } | 20260506 12:16:26.673 | INFO | Perf report for the last 5.13259 seconds: | 20260506 12:16:26.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:16:26.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:26.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:26.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:26.673 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 12:16:26.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:26.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:26.673 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:16:26.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:26.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:26.673 | INFO | Uploaded 36GiB to GPU in 2.01722s (17.8463GiB/s) | 20260506 12:16:26.673 | INFO | Downloaded 34GiB to GPU in 1.94531s (17.4779GiB/s) | 20260506 12:16:31.811 | INFO | Analyzing chunk frames = [1284608,1285120) | 20260506 12:16:31.811 | INFO | MemInfo: { | 20260506 12:16:31.811 | INFO | "gpu" : " 17 GB", | 20260506 12:16:31.811 | INFO | "huge" : "", | 20260506 12:16:31.811 | INFO | "huge_shared" : " 27 GB", | 20260506 12:16:31.811 | INFO | "malloc" : " 14 GB", | 20260506 12:16:31.811 | INFO | "pinned" : " 29 GB" | 20260506 12:16:31.811 | INFO | } | 20260506 12:16:31.811 | INFO | Perf report for the last 5.13864 seconds: | 20260506 12:16:31.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:16:31.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:31.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:31.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:31.811 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 12:16:31.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:31.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:31.811 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 12:16:31.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:31.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:31.811 | INFO | Uploaded 36GiB to GPU in 2.07423s (17.3559GiB/s) | 20260506 12:16:31.811 | INFO | Downloaded 34GiB to GPU in 1.94285s (17.5001GiB/s) | 20260506 12:16:36.950 | INFO | Analyzing chunk frames = [1285120,1285632) | 20260506 12:16:36.950 | INFO | MemInfo: { | 20260506 12:16:36.950 | INFO | "gpu" : " 17 GB", | 20260506 12:16:36.950 | INFO | "huge" : "", | 20260506 12:16:36.951 | INFO | "huge_shared" : " 27 GB", | 20260506 12:16:36.951 | INFO | "malloc" : " 14 GB", | 20260506 12:16:36.951 | INFO | "pinned" : " 29 GB" | 20260506 12:16:36.951 | INFO | } | 20260506 12:16:36.951 | INFO | Perf report for the last 5.13943 seconds: | 20260506 12:16:36.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:16:36.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:36.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:36.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:36.951 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 12:16:36.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:36.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:36.951 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:16:36.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:36.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:36.951 | INFO | Uploaded 36GiB to GPU in 1.97348s (18.2419GiB/s) | 20260506 12:16:36.951 | INFO | Downloaded 34GiB to GPU in 2.01728s (16.8543GiB/s) | 20260506 12:16:42.090 | INFO | Analyzing chunk frames = [1285632,1286144) | 20260506 12:16:42.090 | INFO | MemInfo: { | 20260506 12:16:42.090 | INFO | "gpu" : " 17 GB", | 20260506 12:16:42.090 | INFO | "huge" : "", | 20260506 12:16:42.090 | INFO | "huge_shared" : " 27 GB", | 20260506 12:16:42.090 | INFO | "malloc" : " 14 GB", | 20260506 12:16:42.090 | INFO | "pinned" : " 29 GB" | 20260506 12:16:42.090 | INFO | } | 20260506 12:16:42.090 | INFO | Perf report for the last 5.13909 seconds: | 20260506 12:16:42.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:16:42.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:42.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:42.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:42.090 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 12:16:42.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:42.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:42.090 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 12:16:42.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:42.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:42.090 | INFO | Uploaded 36GiB to GPU in 2.06148s (17.4632GiB/s) | 20260506 12:16:42.090 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:16:42.090 | INFO | Total mem usage: 16.8 GiB | 20260506 12:16:42.090 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:16:42.090 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.6%) | 20260506 12:16:42.090 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:16:42.090 | INFO | Downloaded 34GiB to GPU in 1.96953s (17.263GiB/s) | 20260506 12:16:42.090 | INFO | High Water Breakdown: | 20260506 12:16:42.090 | INFO | 59.6% : 8.59 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:16:42.090 | INFO | 40.3% : 5.8 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:16:42.090 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:16:42.090 | INFO | ------------------------------------------ | 20260506 12:16:42.090 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:16:42.090 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:16:42.090 | INFO | Total mem usage: 26.5 GiB | 20260506 12:16:42.090 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:16:42.090 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:16:42.090 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:16:42.090 | INFO | High Water Breakdown: | 20260506 12:16:42.090 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:16:42.090 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:16:42.090 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:16:42.090 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:16:42.090 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:16:42.090 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:16:42.090 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:16:42.091 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:16:42.091 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:16:42.091 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:16:42.091 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:16:42.091 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:16:42.091 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:16:42.091 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:16:42.091 | INFO | ------------------------------------------ | 20260506 12:16:42.091 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:16:42.091 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:16:42.091 | INFO | Total mem usage: 25.6 GiB | 20260506 12:16:42.091 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:16:42.091 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:16:42.091 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:16:42.091 | INFO | High Water Breakdown: | 20260506 12:16:42.091 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:16:42.091 | INFO | ------------------------------------------ | 20260506 12:16:42.091 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:16:42.091 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:16:42.091 | INFO | Total mem usage: 15.7 GiB | 20260506 12:16:42.091 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:16:42.091 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:16:42.091 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 12:16:42.091 | INFO | High Water Breakdown: | 20260506 12:16:42.091 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:16:42.091 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:16:42.091 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:16:42.091 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:16:42.091 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:16:42.091 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:16:42.091 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:16:42.091 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:16:42.091 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:16:42.091 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:16:42.091 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:16:42.091 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:16:42.091 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:16:42.091 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:16:42.091 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:16:42.091 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:16:42.091 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:16:42.091 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:16:42.091 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:16:42.091 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:16:42.091 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:16:42.091 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:16:42.091 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:16:42.091 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:16:42.091 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:16:42.091 | INFO | ------------------------------------------ | 20260506 12:16:42.091 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:16:42.091 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:16:42.091 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:16:42.091 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:16:42.091 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.60e+01, 1.35e+01 / 1.90e+00 | 20260506 12:16:42.091 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.03e-01, 1.12e-02 / 6.40e-03 | 20260506 12:16:42.091 | INFO | PerfReport: 11.56%, Baseline -- 3.68e+00 / 4.38e+00, 5.83e-01 / 3.61e-01 | 20260506 12:16:42.091 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.50e+00, 1.93e-01 / 1.22e-01 | 20260506 12:16:42.091 | INFO | PerfReport: 15.80%, DME -- 4.02e+01 / 4.44e+01, 4.00e+00 / 2.26e+00 | 20260506 12:16:42.091 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.50e+00 / 1.89e+00 | 20260506 12:16:42.091 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.41e+00 / 3.08e+00, 4.23e-01 / 2.25e-01 | 20260506 12:16:42.091 | INFO | PerfReport: 3.75%, Metrics -- 1.19e+00 / 1.59e+00, 3.00e-01 / 1.82e-01 | 20260506 12:16:42.091 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.21e-01 / 4.82e-01, 6.67e-02 / 3.02e-02 | 20260506 12:16:42.091 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:16:42.091 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:16:42.091 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:16:42.091 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 12:16:42.091 | INFO | PerfReport: 21.41%, Upload -- 2.12e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260506 12:16:42.091 | INFO | PerfReport: 20.94%, Download -- 2.07e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 12:16:42.091 | INFO | PerfReport: 32.18%, Compute -- 3.18e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 12:16:42.091 | INFO | PerfReport: 1.79%, LockUpload -- 1.77e+00 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 12:16:42.091 | INFO | PerfReport: 1.03%, LockDownload -- 1.02e+00 / 0.00e+00, 2.68e+00 / 0.00e+00 | 20260506 12:16:42.091 | INFO | PerfReport: 22.63%, LockCompute -- 2.24e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:16:42.091 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260506 12:16:42.218 | INFO | Starting a round of Baz IO | 20260506 12:16:45.387 | INFO | Populating RTMetrics report | 20260506 12:16:45.387 | INFO | RTMetrics report populated | 20260506 12:16:45.387 | INFO | Opening tmp RTMetrics json file | 20260506 12:16:45.388 | INFO | Serializing RTMetrics to json | 20260506 12:16:45.389 | INFO | Serialization done, swapping tmp file now... | 20260506 12:16:45.390 | INFO | RTMetrics done | 20260506 12:16:45.485 | INFO | Finished a round of Baz IO: Wrote 2.082 GiB in 3.267 seconds | 20260506 12:16:45.486 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.96% / 0.03% / 58.02% | 20260506 12:16:45.486 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.09694% / 4.76837e-05% / 5.96046e-05% | 20260506 12:16:45.486 | INFO | Baz IO overall: | 20260506 12:16:45.486 | INFO | BytesWritten : 635481381213 | 20260506 12:16:45.486 | INFO | NumEvents : 307043743670 | 20260506 12:16:47.229 | INFO | Analyzing chunk frames = [1286144,1286656) | 20260506 12:16:47.229 | INFO | MemInfo: { | 20260506 12:16:47.229 | INFO | "gpu" : " 17 GB", | 20260506 12:16:47.229 | INFO | "huge" : "", | 20260506 12:16:47.229 | INFO | "huge_shared" : " 27 GB", | 20260506 12:16:47.229 | INFO | "malloc" : " 14 GB", | 20260506 12:16:47.229 | INFO | "pinned" : " 29 GB" | 20260506 12:16:47.229 | INFO | } | 20260506 12:16:47.229 | INFO | Perf report for the last 5.13937 seconds: | 20260506 12:16:47.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:16:47.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:47.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:47.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:47.229 | INFO | ANALYSIS : 63.5%, 3.0, 98.8ms, 0 | 20260506 12:16:47.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:47.229 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:16:47.229 | INFO | PRE_HQ : 61.6%, 1.9, 61.8ms, 0 | 20260506 12:16:47.229 | INFO | BAZWRITER : 62.8%, 3.7, 118.2ms, 0 | 20260506 12:16:47.229 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:16:47.229 | INFO | Uploaded 36GiB to GPU in 2.097s (17.1674GiB/s) | 20260506 12:16:47.229 | INFO | Downloaded 38GiB to GPU in 2.20961s (17.1976GiB/s) | 20260506 12:16:52.368 | INFO | Analyzing chunk frames = [1286656,1287168) | 20260506 12:16:52.368 | INFO | MemInfo: { | 20260506 12:16:52.368 | INFO | "gpu" : " 17 GB", | 20260506 12:16:52.368 | INFO | "huge" : "", | 20260506 12:16:52.368 | INFO | "huge_shared" : " 27 GB", | 20260506 12:16:52.368 | INFO | "malloc" : " 14 GB", | 20260506 12:16:52.368 | INFO | "pinned" : " 29 GB" | 20260506 12:16:52.368 | INFO | } | 20260506 12:16:52.369 | INFO | Perf report for the last 5.13942 seconds: | 20260506 12:16:52.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:16:52.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:52.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:52.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:52.369 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:16:52.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:52.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:52.369 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:16:52.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:52.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:52.369 | INFO | Uploaded 36GiB to GPU in 2.01702s (17.8481GiB/s) | 20260506 12:16:52.369 | INFO | Downloaded 34GiB to GPU in 1.91731s (17.7332GiB/s) | 20260506 12:16:57.508 | INFO | Analyzing chunk frames = [1287168,1287680) | 20260506 12:16:57.508 | INFO | MemInfo: { | 20260506 12:16:57.508 | INFO | "gpu" : " 17 GB", | 20260506 12:16:57.508 | INFO | "huge" : "", | 20260506 12:16:57.508 | INFO | "huge_shared" : " 27 GB", | 20260506 12:16:57.508 | INFO | "malloc" : " 14 GB", | 20260506 12:16:57.508 | INFO | "pinned" : " 29 GB" | 20260506 12:16:57.508 | INFO | } | 20260506 12:16:57.508 | INFO | Perf report for the last 5.13942 seconds: | 20260506 12:16:57.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:16:57.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:57.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:57.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:57.508 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 12:16:57.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:57.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:57.508 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:16:57.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:57.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:16:57.508 | INFO | Uploaded 36GiB to GPU in 2.01073s (17.904GiB/s) | 20260506 12:16:57.508 | INFO | Downloaded 34GiB to GPU in 1.94316s (17.4973GiB/s) | 20260506 12:17:02.647 | INFO | Analyzing chunk frames = [1287680,1288192) | 20260506 12:17:02.647 | INFO | MemInfo: { | 20260506 12:17:02.647 | INFO | "gpu" : " 17 GB", | 20260506 12:17:02.647 | INFO | "huge" : "", | 20260506 12:17:02.659 | INFO | "huge_shared" : " 27 GB", | 20260506 12:17:02.659 | INFO | "malloc" : " 14 GB", | 20260506 12:17:02.659 | INFO | "pinned" : " 29 GB" | 20260506 12:17:02.659 | INFO | } | 20260506 12:17:02.659 | INFO | Perf report for the last 5.15112 seconds: | 20260506 12:17:02.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:17:02.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:02.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:02.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:02.659 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 12:17:02.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:02.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:02.659 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 12:17:02.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:02.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:02.659 | INFO | Uploaded 36GiB to GPU in 2.01623s (17.8551GiB/s) | 20260506 12:17:02.659 | INFO | Downloaded 34GiB to GPU in 1.94678s (17.4648GiB/s) | 20260506 12:17:07.786 | INFO | Analyzing chunk frames = [1288192,1288704) | 20260506 12:17:07.786 | INFO | MemInfo: { | 20260506 12:17:07.786 | INFO | "gpu" : " 17 GB", | 20260506 12:17:07.786 | INFO | "huge" : "", | 20260506 12:17:07.786 | INFO | "huge_shared" : " 27 GB", | 20260506 12:17:07.787 | INFO | "malloc" : " 14 GB", | 20260506 12:17:07.787 | INFO | "pinned" : " 29 GB" | 20260506 12:17:07.787 | INFO | } | 20260506 12:17:07.787 | INFO | Perf report for the last 5.12744 seconds: | 20260506 12:17:07.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:17:07.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:07.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:07.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:07.787 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 12:17:07.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:07.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:07.787 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:17:07.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:07.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:07.787 | INFO | Uploaded 36GiB to GPU in 1.96761s (18.2963GiB/s) | 20260506 12:17:07.787 | INFO | Downloaded 34GiB to GPU in 1.94384s (17.4911GiB/s) | 20260506 12:17:12.926 | INFO | Analyzing chunk frames = [1288704,1289216) | 20260506 12:17:12.926 | INFO | MemInfo: { | 20260506 12:17:12.926 | INFO | "gpu" : " 17 GB", | 20260506 12:17:12.926 | INFO | "huge" : "", | 20260506 12:17:12.926 | INFO | "huge_shared" : " 27 GB", | 20260506 12:17:12.926 | INFO | "malloc" : " 14 GB", | 20260506 12:17:12.926 | INFO | "pinned" : " 29 GB" | 20260506 12:17:12.926 | INFO | } | 20260506 12:17:12.926 | INFO | Perf report for the last 5.13944 seconds: | 20260506 12:17:12.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:17:12.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:12.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:12.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:12.926 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 12:17:12.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:12.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:12.926 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:17:12.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:12.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:12.926 | INFO | Uploaded 36GiB to GPU in 2.04036s (17.644GiB/s) | 20260506 12:17:12.926 | INFO | Downloaded 34GiB to GPU in 1.9069s (17.8299GiB/s) | 20260506 12:17:18.066 | INFO | Analyzing chunk frames = [1289216,1289728) | 20260506 12:17:18.066 | INFO | MemInfo: { | 20260506 12:17:18.066 | INFO | "gpu" : " 17 GB", | 20260506 12:17:18.066 | INFO | "huge" : "", | 20260506 12:17:18.066 | INFO | "huge_shared" : " 27 GB", | 20260506 12:17:18.066 | INFO | "malloc" : " 14 GB", | 20260506 12:17:18.066 | INFO | "pinned" : " 29 GB" | 20260506 12:17:18.066 | INFO | } | 20260506 12:17:18.066 | INFO | Perf report for the last 5.13968 seconds: | 20260506 12:17:18.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:17:18.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:18.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:18.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:18.066 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:17:18.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:18.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:18.066 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 12:17:18.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:18.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:18.066 | INFO | Uploaded 36GiB to GPU in 2.00155s (17.9861GiB/s) | 20260506 12:17:18.066 | INFO | Downloaded 34GiB to GPU in 2.03264s (16.7271GiB/s) | 20260506 12:17:23.204 | INFO | Analyzing chunk frames = [1289728,1290240) | 20260506 12:17:23.204 | INFO | MemInfo: { | 20260506 12:17:23.204 | INFO | "gpu" : " 17 GB", | 20260506 12:17:23.204 | INFO | "huge" : "", | 20260506 12:17:23.205 | INFO | "huge_shared" : " 27 GB", | 20260506 12:17:23.205 | INFO | "malloc" : " 14 GB", | 20260506 12:17:23.205 | INFO | "pinned" : " 29 GB" | 20260506 12:17:23.205 | INFO | } | 20260506 12:17:23.205 | INFO | Perf report for the last 5.13885 seconds: | 20260506 12:17:23.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:17:23.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:23.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:23.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:23.205 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260506 12:17:23.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:23.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:23.205 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:17:23.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:23.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:23.205 | INFO | Uploaded 36GiB to GPU in 2.09329s (17.1978GiB/s) | 20260506 12:17:23.205 | INFO | Downloaded 34GiB to GPU in 1.99913s (17.0074GiB/s) | 20260506 12:17:23.330 | INFO | Starting a round of Baz IO | 20260506 12:17:26.516 | INFO | Populating RTMetrics report | 20260506 12:17:26.516 | INFO | RTMetrics report populated | 20260506 12:17:26.516 | INFO | Opening tmp RTMetrics json file | 20260506 12:17:26.517 | INFO | Serializing RTMetrics to json | 20260506 12:17:26.527 | INFO | Serialization done, swapping tmp file now... | 20260506 12:17:26.539 | INFO | RTMetrics done | 20260506 12:17:26.610 | INFO | Finished a round of Baz IO: Wrote 2.085 GiB in 3.28 seconds | 20260506 12:17:26.610 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.93% / 0.03% / 58.04% | 20260506 12:17:26.610 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.10755% / 4.76837e-05% / 3.17891e-05% | 20260506 12:17:26.610 | INFO | Baz IO overall: | 20260506 12:17:26.610 | INFO | BytesWritten : 637719859262 | 20260506 12:17:26.610 | INFO | NumEvents : 308129690801 | 20260506 12:17:28.344 | INFO | Analyzing chunk frames = [1290240,1290752) | 20260506 12:17:28.344 | INFO | MemInfo: { | 20260506 12:17:28.344 | INFO | "gpu" : " 17 GB", | 20260506 12:17:28.344 | INFO | "huge" : "", | 20260506 12:17:28.345 | INFO | "huge_shared" : " 27 GB", | 20260506 12:17:28.345 | INFO | "malloc" : " 14 GB", | 20260506 12:17:28.345 | INFO | "pinned" : " 29 GB" | 20260506 12:17:28.345 | INFO | } | 20260506 12:17:28.345 | INFO | Perf report for the last 5.14003 seconds: | 20260506 12:17:28.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:17:28.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:28.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:28.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:28.345 | INFO | ANALYSIS : 63.8%, 3.0, 99.2ms, 0 | 20260506 12:17:28.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:28.345 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 12:17:28.345 | INFO | PRE_HQ : 61.8%, 1.9, 61.7ms, 0 | 20260506 12:17:28.345 | INFO | BAZWRITER : 63.0%, 3.6, 117.8ms, 0 | 20260506 12:17:28.345 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:17:28.345 | INFO | Uploaded 36GiB to GPU in 2.11711s (17.0043GiB/s) | 20260506 12:17:28.345 | INFO | Downloaded 38GiB to GPU in 2.21623s (17.1462GiB/s) | 20260506 12:17:33.483 | INFO | Analyzing chunk frames = [1290752,1291264) | 20260506 12:17:33.483 | INFO | MemInfo: { | 20260506 12:17:33.483 | INFO | "gpu" : " 17 GB", | 20260506 12:17:33.483 | INFO | "huge" : "", | 20260506 12:17:33.483 | INFO | "huge_shared" : " 27 GB", | 20260506 12:17:33.483 | INFO | "malloc" : " 14 GB", | 20260506 12:17:33.483 | INFO | "pinned" : " 29 GB" | 20260506 12:17:33.483 | INFO | } | 20260506 12:17:33.483 | INFO | Perf report for the last 5.13885 seconds: | 20260506 12:17:33.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:17:33.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:33.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:33.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:33.484 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:17:33.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:33.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:33.484 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:17:33.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:33.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:33.484 | INFO | Uploaded 36GiB to GPU in 2.0219s (17.805GiB/s) | 20260506 12:17:33.484 | INFO | Downloaded 34GiB to GPU in 1.91856s (17.7216GiB/s) | 20260506 12:17:38.623 | INFO | Analyzing chunk frames = [1291264,1291776) | 20260506 12:17:38.623 | INFO | MemInfo: { | 20260506 12:17:38.623 | INFO | "gpu" : " 17 GB", | 20260506 12:17:38.623 | INFO | "huge" : "", | 20260506 12:17:38.623 | INFO | "huge_shared" : " 27 GB", | 20260506 12:17:38.623 | INFO | "malloc" : " 14 GB", | 20260506 12:17:38.623 | INFO | "pinned" : " 29 GB" | 20260506 12:17:38.623 | INFO | } | 20260506 12:17:38.623 | INFO | Perf report for the last 5.13942 seconds: | 20260506 12:17:38.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:17:38.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:38.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:38.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:38.623 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:17:38.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:38.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:38.623 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:17:38.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:38.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:38.623 | INFO | Uploaded 36GiB to GPU in 2.01836s (17.8363GiB/s) | 20260506 12:17:38.623 | INFO | Downloaded 34GiB to GPU in 1.94205s (17.5073GiB/s) | 20260506 12:17:43.762 | INFO | Analyzing chunk frames = [1291776,1292288) | 20260506 12:17:43.762 | INFO | MemInfo: { | 20260506 12:17:43.762 | INFO | "gpu" : " 17 GB", | 20260506 12:17:43.762 | INFO | "huge" : "", | 20260506 12:17:43.762 | INFO | "huge_shared" : " 27 GB", | 20260506 12:17:43.762 | INFO | "malloc" : " 14 GB", | 20260506 12:17:43.762 | INFO | "pinned" : " 29 GB" | 20260506 12:17:43.762 | INFO | } | 20260506 12:17:43.762 | INFO | Perf report for the last 5.13933 seconds: | 20260506 12:17:43.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:17:43.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:43.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:43.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:43.762 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 12:17:43.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:43.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:43.762 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 12:17:43.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:43.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:43.762 | INFO | Uploaded 36GiB to GPU in 2.02147s (17.8088GiB/s) | 20260506 12:17:43.763 | INFO | Downloaded 34GiB to GPU in 1.94552s (17.476GiB/s) | 20260506 12:17:48.901 | INFO | Analyzing chunk frames = [1292288,1292800) | 20260506 12:17:48.901 | INFO | MemInfo: { | 20260506 12:17:48.901 | INFO | "gpu" : " 17 GB", | 20260506 12:17:48.902 | INFO | "huge" : "", | 20260506 12:17:48.902 | INFO | "huge_shared" : " 27 GB", | 20260506 12:17:48.902 | INFO | "malloc" : " 14 GB", | 20260506 12:17:48.902 | INFO | "pinned" : " 29 GB" | 20260506 12:17:48.902 | INFO | } | 20260506 12:17:48.902 | INFO | Perf report for the last 5.13937 seconds: | 20260506 12:17:48.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:17:48.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:48.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:48.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:48.902 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 12:17:48.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:48.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:48.902 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 12:17:48.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:48.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:48.902 | INFO | Uploaded 36GiB to GPU in 2.05385s (17.528GiB/s) | 20260506 12:17:48.902 | INFO | Downloaded 34GiB to GPU in 1.95605s (17.382GiB/s) | 20260506 12:17:54.041 | INFO | Analyzing chunk frames = [1292800,1293312) | 20260506 12:17:54.041 | INFO | MemInfo: { | 20260506 12:17:54.041 | INFO | "gpu" : " 17 GB", | 20260506 12:17:54.041 | INFO | "huge" : "", | 20260506 12:17:54.041 | INFO | "huge_shared" : " 27 GB", | 20260506 12:17:54.041 | INFO | "malloc" : " 14 GB", | 20260506 12:17:54.041 | INFO | "pinned" : " 29 GB" | 20260506 12:17:54.041 | INFO | } | 20260506 12:17:54.041 | INFO | Perf report for the last 5.13903 seconds: | 20260506 12:17:54.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:17:54.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:54.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:54.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:54.041 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 12:17:54.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:54.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:54.041 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260506 12:17:54.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:54.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:54.041 | INFO | Uploaded 36GiB to GPU in 2.02685s (17.7616GiB/s) | 20260506 12:17:54.041 | INFO | Downloaded 34GiB to GPU in 1.92602s (17.653GiB/s) | 20260506 12:17:59.181 | INFO | Analyzing chunk frames = [1293312,1293824) | 20260506 12:17:59.181 | INFO | MemInfo: { | 20260506 12:17:59.181 | INFO | "gpu" : " 17 GB", | 20260506 12:17:59.181 | INFO | "huge" : "", | 20260506 12:17:59.181 | INFO | "huge_shared" : " 27 GB", | 20260506 12:17:59.181 | INFO | "malloc" : " 14 GB", | 20260506 12:17:59.181 | INFO | "pinned" : " 29 GB" | 20260506 12:17:59.181 | INFO | } | 20260506 12:17:59.181 | INFO | Perf report for the last 5.14038 seconds: | 20260506 12:17:59.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:17:59.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:59.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:59.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:59.181 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:17:59.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:59.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:59.181 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 12:17:59.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:59.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:17:59.181 | INFO | Uploaded 36GiB to GPU in 2.00127s (17.9886GiB/s) | 20260506 12:17:59.181 | INFO | Downloaded 34GiB to GPU in 2.02768s (16.768GiB/s) | 20260506 12:18:04.319 | INFO | Analyzing chunk frames = [1293824,1294336) | 20260506 12:18:04.319 | INFO | MemInfo: { | 20260506 12:18:04.319 | INFO | "gpu" : " 17 GB", | 20260506 12:18:04.319 | INFO | "huge" : "", | 20260506 12:18:04.319 | INFO | "huge_shared" : " 27 GB", | 20260506 12:18:04.319 | INFO | "malloc" : " 14 GB", | 20260506 12:18:04.319 | INFO | "pinned" : " 29 GB" | 20260506 12:18:04.319 | INFO | } | 20260506 12:18:04.319 | INFO | Perf report for the last 5.13834 seconds: | 20260506 12:18:04.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:18:04.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:04.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:04.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:04.319 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260506 12:18:04.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:04.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:04.319 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 12:18:04.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:04.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:04.320 | INFO | Uploaded 36GiB to GPU in 2.0432s (17.6194GiB/s) | 20260506 12:18:04.320 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:18:04.320 | INFO | Total mem usage: 16.8 GiB | 20260506 12:18:04.320 | INFO | Downloaded 34GiB to GPU in 2.00317s (16.9731GiB/s) | 20260506 12:18:04.320 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:18:04.320 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.6%) | 20260506 12:18:04.320 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:18:04.320 | INFO | High Water Breakdown: | 20260506 12:18:04.320 | INFO | 59.6% : 8.59 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:18:04.320 | INFO | 40.3% : 5.8 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:18:04.320 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:18:04.320 | INFO | ------------------------------------------ | 20260506 12:18:04.320 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:18:04.320 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:18:04.320 | INFO | Total mem usage: 26.5 GiB | 20260506 12:18:04.320 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:18:04.320 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:18:04.320 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:18:04.320 | INFO | High Water Breakdown: | 20260506 12:18:04.320 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:18:04.320 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:18:04.320 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:18:04.320 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:18:04.320 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:18:04.320 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:18:04.320 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:18:04.320 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:18:04.320 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:18:04.320 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:18:04.320 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:18:04.320 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:18:04.320 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:18:04.320 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:18:04.320 | INFO | ------------------------------------------ | 20260506 12:18:04.320 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:18:04.320 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:18:04.320 | INFO | Total mem usage: 25.6 GiB | 20260506 12:18:04.320 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:18:04.320 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:18:04.320 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:18:04.320 | INFO | High Water Breakdown: | 20260506 12:18:04.320 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:18:04.320 | INFO | ------------------------------------------ | 20260506 12:18:04.320 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:18:04.320 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:18:04.320 | INFO | Total mem usage: 15.7 GiB | 20260506 12:18:04.320 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:18:04.320 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:18:04.320 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 12:18:04.320 | INFO | High Water Breakdown: | 20260506 12:18:04.320 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:18:04.320 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:18:04.320 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:18:04.320 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:18:04.320 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:18:04.320 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:18:04.320 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:18:04.320 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:18:04.320 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:18:04.320 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:18:04.320 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:18:04.320 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:18:04.320 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:18:04.320 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:18:04.320 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:18:04.320 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:18:04.320 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:18:04.320 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:18:04.320 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:18:04.320 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:18:04.320 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:18:04.320 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:18:04.320 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:18:04.320 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:18:04.320 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:18:04.320 | INFO | ------------------------------------------ | 20260506 12:18:04.320 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:18:04.320 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:18:04.320 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:18:04.320 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:18:04.320 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.63e+01, 1.37e+01 / 2.69e+00 | 20260506 12:18:04.320 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.01e-01, 1.10e-02 / 5.91e-03 | 20260506 12:18:04.320 | INFO | PerfReport: 11.56%, Baseline -- 3.68e+00 / 4.41e+00, 5.90e-01 / 3.72e-01 | 20260506 12:18:04.320 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.45e+00, 1.93e-01 / 1.39e-01 | 20260506 12:18:04.321 | INFO | PerfReport: 15.86%, DME -- 4.04e+01 / 4.50e+01, 4.40e+00 / 2.80e+00 | 20260506 12:18:04.321 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.79e+01 / 2.02e+01, 1.45e+00 / 1.74e+00 | 20260506 12:18:04.321 | INFO | PerfReport: 7.59%, PulseAccumulating -- 2.41e+00 / 3.05e+00, 4.23e-01 / 2.14e-01 | 20260506 12:18:04.321 | INFO | PerfReport: 3.76%, Metrics -- 1.20e+00 / 1.57e+00, 3.05e-01 / 1.67e-01 | 20260506 12:18:04.321 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.21e-01 / 4.80e-01, 6.68e-02 / 3.39e-02 | 20260506 12:18:04.321 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:18:04.321 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:18:04.321 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:18:04.321 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 12:18:04.321 | INFO | PerfReport: 21.42%, Upload -- 2.12e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 12:18:04.321 | INFO | PerfReport: 20.95%, Download -- 2.07e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 12:18:04.321 | INFO | PerfReport: 32.18%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 12:18:04.321 | INFO | PerfReport: 1.84%, LockUpload -- 1.82e+00 / 0.00e+00, 7.58e+00 / 0.00e+00 | 20260506 12:18:04.321 | INFO | PerfReport: 1.02%, LockDownload -- 1.01e+00 / 0.00e+00, 2.67e+00 / 0.00e+00 | 20260506 12:18:04.321 | INFO | PerfReport: 22.57%, LockCompute -- 2.23e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 12:18:04.321 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260506 12:18:04.450 | INFO | Starting a round of Baz IO | 20260506 12:18:07.609 | INFO | Populating RTMetrics report | 20260506 12:18:07.610 | INFO | RTMetrics report populated | 20260506 12:18:07.610 | INFO | Opening tmp RTMetrics json file | 20260506 12:18:07.611 | INFO | Serializing RTMetrics to json | 20260506 12:18:07.612 | INFO | Serialization done, swapping tmp file now... | 20260506 12:18:07.612 | INFO | RTMetrics done | 20260506 12:18:07.712 | INFO | Finished a round of Baz IO: Wrote 2.084 GiB in 3.262 seconds | 20260506 12:18:07.713 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.91% / 0.03% / 58.07% | 20260506 12:18:07.713 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.09489% / 7.54992e-05% / 3.57628e-05% | 20260506 12:18:07.713 | INFO | Baz IO overall: | 20260506 12:18:07.713 | INFO | BytesWritten : 639957824252 | 20260506 12:18:07.713 | INFO | NumEvents : 309215113520 | 20260506 12:18:09.459 | INFO | Analyzing chunk frames = [1294336,1294848) | 20260506 12:18:09.459 | INFO | MemInfo: { | 20260506 12:18:09.459 | INFO | "gpu" : " 17 GB", | 20260506 12:18:09.459 | INFO | "huge" : "", | 20260506 12:18:09.459 | INFO | "huge_shared" : " 27 GB", | 20260506 12:18:09.459 | INFO | "malloc" : " 14 GB", | 20260506 12:18:09.459 | INFO | "pinned" : " 29 GB" | 20260506 12:18:09.459 | INFO | } | 20260506 12:18:09.459 | INFO | Perf report for the last 5.13919 seconds: | 20260506 12:18:09.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:18:09.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:09.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:09.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:09.459 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260506 12:18:09.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:09.459 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:18:09.459 | INFO | PRE_HQ : 61.4%, 1.9, 61.6ms, 0 | 20260506 12:18:09.459 | INFO | BAZWRITER : 62.8%, 3.7, 118.7ms, 0 | 20260506 12:18:09.459 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:18:09.459 | INFO | Uploaded 36GiB to GPU in 2.07508s (17.3488GiB/s) | 20260506 12:18:09.459 | INFO | Downloaded 38GiB to GPU in 2.21456s (17.1592GiB/s) | 20260506 12:18:14.598 | INFO | Analyzing chunk frames = [1294848,1295360) | 20260506 12:18:14.598 | INFO | MemInfo: { | 20260506 12:18:14.598 | INFO | "gpu" : " 17 GB", | 20260506 12:18:14.598 | INFO | "huge" : "", | 20260506 12:18:14.598 | INFO | "huge_shared" : " 27 GB", | 20260506 12:18:14.598 | INFO | "malloc" : " 14 GB", | 20260506 12:18:14.598 | INFO | "pinned" : " 29 GB" | 20260506 12:18:14.598 | INFO | } | 20260506 12:18:14.598 | INFO | Perf report for the last 5.13981 seconds: | 20260506 12:18:14.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:18:14.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:14.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:14.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:14.598 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:18:14.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:14.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:14.599 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:18:14.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:14.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:14.599 | INFO | Uploaded 36GiB to GPU in 2.02071s (17.8156GiB/s) | 20260506 12:18:14.599 | INFO | Downloaded 34GiB to GPU in 1.91774s (17.7292GiB/s) | 20260506 12:18:19.737 | INFO | Analyzing chunk frames = [1295360,1295872) | 20260506 12:18:19.737 | INFO | MemInfo: { | 20260506 12:18:19.737 | INFO | "gpu" : " 17 GB", | 20260506 12:18:19.737 | INFO | "huge" : "", | 20260506 12:18:19.737 | INFO | "huge_shared" : " 27 GB", | 20260506 12:18:19.737 | INFO | "malloc" : " 14 GB", | 20260506 12:18:19.737 | INFO | "pinned" : " 29 GB" | 20260506 12:18:19.737 | INFO | } | 20260506 12:18:19.737 | INFO | Perf report for the last 5.13897 seconds: | 20260506 12:18:19.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:18:19.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:19.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:19.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:19.737 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:18:19.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:19.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:19.737 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:18:19.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:19.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:19.738 | INFO | Uploaded 36GiB to GPU in 2.02594s (17.7695GiB/s) | 20260506 12:18:19.738 | INFO | Downloaded 34GiB to GPU in 1.93991s (17.5265GiB/s) | 20260506 12:18:24.877 | INFO | Analyzing chunk frames = [1295872,1296384) | 20260506 12:18:24.877 | INFO | MemInfo: { | 20260506 12:18:24.877 | INFO | "gpu" : " 17 GB", | 20260506 12:18:24.877 | INFO | "huge" : "", | 20260506 12:18:24.877 | INFO | "huge_shared" : " 27 GB", | 20260506 12:18:24.877 | INFO | "malloc" : " 14 GB", | 20260506 12:18:24.877 | INFO | "pinned" : " 29 GB" | 20260506 12:18:24.877 | INFO | } | 20260506 12:18:24.877 | INFO | Perf report for the last 5.13927 seconds: | 20260506 12:18:24.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:18:24.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:24.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:24.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:24.877 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 12:18:24.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:24.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:24.877 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:18:24.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:24.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:24.877 | INFO | Uploaded 36GiB to GPU in 2.02552s (17.7732GiB/s) | 20260506 12:18:24.877 | INFO | Downloaded 34GiB to GPU in 1.93017s (17.615GiB/s) | 20260506 12:18:30.016 | INFO | Analyzing chunk frames = [1296384,1296896) | 20260506 12:18:30.016 | INFO | MemInfo: { | 20260506 12:18:30.016 | INFO | "gpu" : " 17 GB", | 20260506 12:18:30.016 | INFO | "huge" : "", | 20260506 12:18:30.016 | INFO | "huge_shared" : " 27 GB", | 20260506 12:18:30.016 | INFO | "malloc" : " 14 GB", | 20260506 12:18:30.016 | INFO | "pinned" : " 29 GB" | 20260506 12:18:30.016 | INFO | } | 20260506 12:18:30.016 | INFO | Perf report for the last 5.13965 seconds: | 20260506 12:18:30.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:18:30.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:30.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:30.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:30.016 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260506 12:18:30.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:30.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:30.016 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:18:30.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:30.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:30.017 | INFO | Uploaded 36GiB to GPU in 2.04076s (17.6405GiB/s) | 20260506 12:18:30.017 | INFO | Downloaded 34GiB to GPU in 1.95594s (17.383GiB/s) | 20260506 12:18:35.155 | INFO | Analyzing chunk frames = [1296896,1297408) | 20260506 12:18:35.155 | INFO | MemInfo: { | 20260506 12:18:35.155 | INFO | "gpu" : " 17 GB", | 20260506 12:18:35.155 | INFO | "huge" : "", | 20260506 12:18:35.155 | INFO | "huge_shared" : " 27 GB", | 20260506 12:18:35.155 | INFO | "malloc" : " 14 GB", | 20260506 12:18:35.155 | INFO | "pinned" : " 29 GB" | 20260506 12:18:35.155 | INFO | } | 20260506 12:18:35.155 | INFO | Perf report for the last 5.13901 seconds: | 20260506 12:18:35.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:18:35.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:35.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:35.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:35.155 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 12:18:35.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:35.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:35.155 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260506 12:18:35.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:35.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:35.156 | INFO | Uploaded 36GiB to GPU in 2.0732s (17.3645GiB/s) | 20260506 12:18:35.156 | INFO | Downloaded 34GiB to GPU in 1.93558s (17.5658GiB/s) | 20260506 12:18:40.296 | INFO | Analyzing chunk frames = [1297408,1297920) | 20260506 12:18:40.296 | INFO | MemInfo: { | 20260506 12:18:40.296 | INFO | "gpu" : " 17 GB", | 20260506 12:18:40.296 | INFO | "huge" : "", | 20260506 12:18:40.296 | INFO | "huge_shared" : " 27 GB", | 20260506 12:18:40.296 | INFO | "malloc" : " 14 GB", | 20260506 12:18:40.296 | INFO | "pinned" : " 29 GB" | 20260506 12:18:40.296 | INFO | } | 20260506 12:18:40.296 | INFO | Perf report for the last 5.14041 seconds: | 20260506 12:18:40.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:18:40.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:40.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:40.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:40.296 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:18:40.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:40.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:40.296 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:18:40.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:40.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:40.296 | INFO | Uploaded 36GiB to GPU in 1.99625s (18.0338GiB/s) | 20260506 12:18:40.296 | INFO | Downloaded 34GiB to GPU in 2.02872s (16.7593GiB/s) | 20260506 12:18:45.434 | INFO | Analyzing chunk frames = [1297920,1298432) | 20260506 12:18:45.434 | INFO | MemInfo: { | 20260506 12:18:45.434 | INFO | "gpu" : " 17 GB", | 20260506 12:18:45.434 | INFO | "huge" : "", | 20260506 12:18:45.434 | INFO | "huge_shared" : " 27 GB", | 20260506 12:18:45.434 | INFO | "malloc" : " 14 GB", | 20260506 12:18:45.434 | INFO | "pinned" : " 29 GB" | 20260506 12:18:45.434 | INFO | } | 20260506 12:18:45.434 | INFO | Perf report for the last 5.13839 seconds: | 20260506 12:18:45.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:18:45.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:45.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:45.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:45.434 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 12:18:45.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:45.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:45.434 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:18:45.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:45.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:45.434 | INFO | Uploaded 36GiB to GPU in 2.05621s (17.5079GiB/s) | 20260506 12:18:45.434 | INFO | Downloaded 34GiB to GPU in 1.96407s (17.311GiB/s) | 20260506 12:18:45.557 | INFO | Starting a round of Baz IO | 20260506 12:18:48.733 | INFO | Populating RTMetrics report | 20260506 12:18:48.733 | INFO | RTMetrics report populated | 20260506 12:18:48.733 | INFO | Opening tmp RTMetrics json file | 20260506 12:18:48.734 | INFO | Serializing RTMetrics to json | 20260506 12:18:48.735 | INFO | Serialization done, swapping tmp file now... | 20260506 12:18:48.735 | INFO | RTMetrics done | 20260506 12:18:48.829 | INFO | Finished a round of Baz IO: Wrote 2.084 GiB in 3.272 seconds | 20260506 12:18:48.829 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.88% / 0.03% / 58.09% | 20260506 12:18:48.830 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.08245% / 3.97364e-05% / 1.98682e-05% | 20260506 12:18:48.830 | INFO | Baz IO overall: | 20260506 12:18:48.830 | INFO | BytesWritten : 642195257553 | 20260506 12:18:48.830 | INFO | NumEvents : 310299977615 | 20260506 12:18:50.573 | INFO | Analyzing chunk frames = [1298432,1298944) | 20260506 12:18:50.573 | INFO | MemInfo: { | 20260506 12:18:50.573 | INFO | "gpu" : " 17 GB", | 20260506 12:18:50.573 | INFO | "huge" : "", | 20260506 12:18:50.573 | INFO | "huge_shared" : " 27 GB", | 20260506 12:18:50.573 | INFO | "malloc" : " 14 GB", | 20260506 12:18:50.573 | INFO | "pinned" : " 29 GB" | 20260506 12:18:50.573 | INFO | } | 20260506 12:18:50.574 | INFO | Perf report for the last 5.1393 seconds: | 20260506 12:18:50.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:18:50.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:50.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:50.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:50.574 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260506 12:18:50.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:50.574 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:18:50.574 | INFO | PRE_HQ : 61.6%, 1.9, 61.4ms, 0 | 20260506 12:18:50.574 | INFO | BAZWRITER : 62.9%, 3.7, 118.8ms, 0 | 20260506 12:18:50.574 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:18:50.574 | INFO | Uploaded 36GiB to GPU in 2.09426s (17.1898GiB/s) | 20260506 12:18:50.574 | INFO | Downloaded 38GiB to GPU in 2.20752s (17.2139GiB/s) | 20260506 12:18:55.713 | INFO | Analyzing chunk frames = [1298944,1299456) | 20260506 12:18:55.713 | INFO | MemInfo: { | 20260506 12:18:55.713 | INFO | "gpu" : " 17 GB", | 20260506 12:18:55.713 | INFO | "huge" : "", | 20260506 12:18:55.713 | INFO | "huge_shared" : " 27 GB", | 20260506 12:18:55.713 | INFO | "malloc" : " 14 GB", | 20260506 12:18:55.713 | INFO | "pinned" : " 29 GB" | 20260506 12:18:55.713 | INFO | } | 20260506 12:18:55.713 | INFO | Perf report for the last 5.13937 seconds: | 20260506 12:18:55.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:18:55.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:55.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:55.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:55.713 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:18:55.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:55.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:55.713 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 12:18:55.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:55.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:18:55.713 | INFO | Uploaded 36GiB to GPU in 2.03392s (17.6998GiB/s) | 20260506 12:18:55.713 | INFO | Downloaded 34GiB to GPU in 1.92173s (17.6924GiB/s) | 20260506 12:19:00.852 | INFO | Analyzing chunk frames = [1299456,1299968) | 20260506 12:19:00.852 | INFO | MemInfo: { | 20260506 12:19:00.852 | INFO | "gpu" : " 17 GB", | 20260506 12:19:00.852 | INFO | "huge" : "", | 20260506 12:19:00.852 | INFO | "huge_shared" : " 27 GB", | 20260506 12:19:00.852 | INFO | "malloc" : " 14 GB", | 20260506 12:19:00.852 | INFO | "pinned" : " 29 GB" | 20260506 12:19:00.852 | INFO | } | 20260506 12:19:00.852 | INFO | Perf report for the last 5.13939 seconds: | 20260506 12:19:00.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:19:00.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:00.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:00.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:00.852 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:19:00.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:00.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:00.852 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:19:00.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:00.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:00.852 | INFO | Uploaded 36GiB to GPU in 2.00037s (17.9966GiB/s) | 20260506 12:19:00.853 | INFO | Downloaded 34GiB to GPU in 1.94393s (17.4903GiB/s) | 20260506 12:19:05.991 | INFO | Analyzing chunk frames = [1299968,1300480) | 20260506 12:19:05.991 | INFO | MemInfo: { | 20260506 12:19:05.991 | INFO | "gpu" : " 17 GB", | 20260506 12:19:05.991 | INFO | "huge" : "", | 20260506 12:19:05.991 | INFO | "huge_shared" : " 27 GB", | 20260506 12:19:05.991 | INFO | "malloc" : " 14 GB", | 20260506 12:19:05.991 | INFO | "pinned" : " 29 GB" | 20260506 12:19:05.991 | INFO | } | 20260506 12:19:05.991 | INFO | Perf report for the last 5.13914 seconds: | 20260506 12:19:05.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:19:05.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:05.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:05.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:05.991 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 12:19:05.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:05.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:05.992 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:19:05.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:05.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:05.992 | INFO | Uploaded 36GiB to GPU in 2.01536s (17.8628GiB/s) | 20260506 12:19:05.992 | INFO | Downloaded 34GiB to GPU in 1.94374s (17.492GiB/s) | 20260506 12:19:11.131 | INFO | Analyzing chunk frames = [1300480,1300992) | 20260506 12:19:11.131 | INFO | MemInfo: { | 20260506 12:19:11.131 | INFO | "gpu" : " 17 GB", | 20260506 12:19:11.131 | INFO | "huge" : "", | 20260506 12:19:11.131 | INFO | "huge_shared" : " 27 GB", | 20260506 12:19:11.131 | INFO | "malloc" : " 14 GB", | 20260506 12:19:11.131 | INFO | "pinned" : " 29 GB" | 20260506 12:19:11.131 | INFO | } | 20260506 12:19:11.131 | INFO | Perf report for the last 5.13945 seconds: | 20260506 12:19:11.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:19:11.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:11.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:11.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:11.131 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 12:19:11.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:11.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:11.131 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 12:19:11.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:11.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:11.131 | INFO | Uploaded 36GiB to GPU in 2.06142s (17.4637GiB/s) | 20260506 12:19:11.131 | INFO | Downloaded 34GiB to GPU in 1.94133s (17.5138GiB/s) | 20260506 12:19:16.270 | INFO | Analyzing chunk frames = [1300992,1301504) | 20260506 12:19:16.270 | INFO | MemInfo: { | 20260506 12:19:16.270 | INFO | "gpu" : " 17 GB", | 20260506 12:19:16.270 | INFO | "huge" : "", | 20260506 12:19:16.270 | INFO | "huge_shared" : " 27 GB", | 20260506 12:19:16.270 | INFO | "malloc" : " 14 GB", | 20260506 12:19:16.270 | INFO | "pinned" : " 29 GB" | 20260506 12:19:16.270 | INFO | } | 20260506 12:19:16.270 | INFO | Perf report for the last 5.13928 seconds: | 20260506 12:19:16.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:19:16.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:16.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:16.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:16.270 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:19:16.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:16.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:16.270 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:19:16.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:16.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:16.270 | INFO | Uploaded 36GiB to GPU in 2.10445s (17.1066GiB/s) | 20260506 12:19:16.271 | INFO | Downloaded 34GiB to GPU in 1.97101s (17.25GiB/s) | 20260506 12:19:21.412 | INFO | Analyzing chunk frames = [1301504,1302016) | 20260506 12:19:21.412 | INFO | MemInfo: { | 20260506 12:19:21.412 | INFO | "gpu" : " 17 GB", | 20260506 12:19:21.412 | INFO | "huge" : "", | 20260506 12:19:21.413 | INFO | "huge_shared" : " 27 GB", | 20260506 12:19:21.413 | INFO | "malloc" : " 14 GB", | 20260506 12:19:21.413 | INFO | "pinned" : " 29 GB" | 20260506 12:19:21.413 | INFO | } | 20260506 12:19:21.413 | INFO | Perf report for the last 5.14239 seconds: | 20260506 12:19:21.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:19:21.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:21.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:21.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:21.413 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:19:21.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:21.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:21.413 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:19:21.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:21.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:21.413 | INFO | Uploaded 36GiB to GPU in 2.01721s (17.8465GiB/s) | 20260506 12:19:21.413 | INFO | Downloaded 34GiB to GPU in 1.98475s (17.1306GiB/s) | 20260506 12:19:26.550 | INFO | Analyzing chunk frames = [1302016,1302528) | 20260506 12:19:26.550 | INFO | MemInfo: { | 20260506 12:19:26.550 | INFO | "gpu" : " 17 GB", | 20260506 12:19:26.550 | INFO | "huge" : "", | 20260506 12:19:26.550 | INFO | "huge_shared" : " 27 GB", | 20260506 12:19:26.550 | INFO | "malloc" : " 14 GB", | 20260506 12:19:26.550 | INFO | "pinned" : " 29 GB" | 20260506 12:19:26.550 | INFO | } | 20260506 12:19:26.550 | INFO | Perf report for the last 5.13715 seconds: | 20260506 12:19:26.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:19:26.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:26.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:26.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:26.550 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:19:26.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:26.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:26.550 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:19:26.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:26.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:26.550 | INFO | Uploaded 36GiB to GPU in 2.03159s (17.7201GiB/s) | 20260506 12:19:26.550 | INFO | Downloaded 34GiB to GPU in 1.95935s (17.3527GiB/s) | 20260506 12:19:26.550 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:19:26.550 | INFO | Total mem usage: 16.8 GiB | 20260506 12:19:26.550 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:19:26.550 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.6%) | 20260506 12:19:26.550 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:19:26.550 | INFO | High Water Breakdown: | 20260506 12:19:26.550 | INFO | 59.6% : 8.59 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:19:26.550 | INFO | 40.3% : 5.8 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:19:26.550 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:19:26.550 | INFO | ------------------------------------------ | 20260506 12:19:26.550 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:19:26.550 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:19:26.550 | INFO | Total mem usage: 26.5 GiB | 20260506 12:19:26.550 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:19:26.550 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:19:26.550 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:19:26.550 | INFO | High Water Breakdown: | 20260506 12:19:26.550 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:19:26.550 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:19:26.550 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:19:26.550 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:19:26.550 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:19:26.550 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:19:26.550 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:19:26.550 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:19:26.550 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:19:26.550 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:19:26.550 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:19:26.550 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:19:26.551 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:19:26.551 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:19:26.551 | INFO | ------------------------------------------ | 20260506 12:19:26.551 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:19:26.551 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:19:26.551 | INFO | Total mem usage: 25.6 GiB | 20260506 12:19:26.551 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:19:26.551 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:19:26.551 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:19:26.551 | INFO | High Water Breakdown: | 20260506 12:19:26.551 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:19:26.551 | INFO | ------------------------------------------ | 20260506 12:19:26.551 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:19:26.551 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:19:26.551 | INFO | Total mem usage: 15.7 GiB | 20260506 12:19:26.551 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:19:26.551 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:19:26.551 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:19:26.551 | INFO | High Water Breakdown: | 20260506 12:19:26.551 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:19:26.551 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:19:26.551 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:19:26.551 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:19:26.551 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:19:26.551 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:19:26.551 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:19:26.551 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:19:26.551 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:19:26.551 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:19:26.551 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:19:26.551 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:19:26.551 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:19:26.551 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:19:26.551 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:19:26.551 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:19:26.551 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:19:26.551 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:19:26.551 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:19:26.551 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:19:26.551 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:19:26.551 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:19:26.551 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:19:26.551 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:19:26.551 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:19:26.551 | INFO | ------------------------------------------ | 20260506 12:19:26.551 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:19:26.551 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:19:26.551 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:19:26.551 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:19:26.551 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.61e+01, 1.34e+01 / 2.58e+00 | 20260506 12:19:26.551 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.01e-01, 1.06e-02 / 5.98e-03 | 20260506 12:19:26.551 | INFO | PerfReport: 11.58%, Baseline -- 3.68e+00 / 4.46e+00, 5.88e-01 / 3.65e-01 | 20260506 12:19:26.551 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.46e+00, 1.91e-01 / 1.55e-01 | 20260506 12:19:26.551 | INFO | PerfReport: 15.85%, DME -- 4.03e+01 / 4.49e+01, 4.25e+00 / 2.65e+00 | 20260506 12:19:26.551 | INFO | PerfReport: 56.35%, FrameLabeling -- 1.79e+01 / 2.01e+01, 1.41e+00 / 1.79e+00 | 20260506 12:19:26.551 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.41e+00 / 3.07e+00, 4.23e-01 / 1.95e-01 | 20260506 12:19:26.551 | INFO | PerfReport: 3.76%, Metrics -- 1.19e+00 / 1.56e+00, 3.04e-01 / 1.75e-01 | 20260506 12:19:26.551 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.21e-01 / 4.80e-01, 6.57e-02 / 2.79e-02 | 20260506 12:19:26.551 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:19:26.551 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:19:26.551 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:19:26.551 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 12:19:26.551 | INFO | PerfReport: 21.51%, Upload -- 2.13e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260506 12:19:26.551 | INFO | PerfReport: 20.91%, Download -- 2.07e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 12:19:26.551 | INFO | PerfReport: 32.14%, Compute -- 3.18e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 12:19:26.551 | INFO | PerfReport: 1.91%, LockUpload -- 1.89e+00 / 0.00e+00, 7.61e+00 / 0.00e+00 | 20260506 12:19:26.551 | INFO | PerfReport: 0.92%, LockDownload -- 9.12e-01 / 0.00e+00, 2.46e+00 / 0.00e+00 | 20260506 12:19:26.551 | INFO | PerfReport: 22.60%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:19:26.551 | INFO | PerfReport: 0.02%, Misc -- 1.51e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260506 12:19:26.678 | INFO | Starting a round of Baz IO | 20260506 12:19:29.844 | INFO | Populating RTMetrics report | 20260506 12:19:29.845 | INFO | RTMetrics report populated | 20260506 12:19:29.845 | INFO | Opening tmp RTMetrics json file | 20260506 12:19:29.846 | INFO | Serializing RTMetrics to json | 20260506 12:19:29.847 | INFO | Serialization done, swapping tmp file now... | 20260506 12:19:29.847 | INFO | RTMetrics done | 20260506 12:19:29.950 | INFO | Finished a round of Baz IO: Wrote 2.084 GiB in 3.271 seconds | 20260506 12:19:29.950 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.85% / 0.03% / 58.12% | 20260506 12:19:29.950 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.07289% / 5.96046e-05% / 4.37101e-05% | 20260506 12:19:29.950 | INFO | Baz IO overall: | 20260506 12:19:29.950 | INFO | BytesWritten : 644432435880 | 20260506 12:19:29.950 | INFO | NumEvents : 311384449415 | 20260506 12:19:31.688 | INFO | Analyzing chunk frames = [1302528,1303040) | 20260506 12:19:31.688 | INFO | MemInfo: { | 20260506 12:19:31.688 | INFO | "gpu" : " 17 GB", | 20260506 12:19:31.688 | INFO | "huge" : "", | 20260506 12:19:31.688 | INFO | "huge_shared" : " 27 GB", | 20260506 12:19:31.688 | INFO | "malloc" : " 14 GB", | 20260506 12:19:31.688 | INFO | "pinned" : " 29 GB" | 20260506 12:19:31.688 | INFO | } | 20260506 12:19:31.688 | INFO | Perf report for the last 5.13846 seconds: | 20260506 12:19:31.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:19:31.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:31.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:31.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:31.688 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 12:19:31.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:31.688 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 12:19:31.688 | INFO | PRE_HQ : 61.8%, 2.0, 62.6ms, 0 | 20260506 12:19:31.688 | INFO | BAZWRITER : 63.0%, 3.8, 122.2ms, 0 | 20260506 12:19:31.688 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:19:31.689 | INFO | Uploaded 36GiB to GPU in 2.08396s (17.2748GiB/s) | 20260506 12:19:31.689 | INFO | Downloaded 38GiB to GPU in 2.21189s (17.1799GiB/s) | 20260506 12:19:36.828 | INFO | Analyzing chunk frames = [1303040,1303552) | 20260506 12:19:36.828 | INFO | MemInfo: { | 20260506 12:19:36.828 | INFO | "gpu" : " 17 GB", | 20260506 12:19:36.828 | INFO | "huge" : "", | 20260506 12:19:36.828 | INFO | "huge_shared" : " 27 GB", | 20260506 12:19:36.828 | INFO | "malloc" : " 14 GB", | 20260506 12:19:36.828 | INFO | "pinned" : " 29 GB" | 20260506 12:19:36.828 | INFO | } | 20260506 12:19:36.828 | INFO | Perf report for the last 5.13982 seconds: | 20260506 12:19:36.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:19:36.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:36.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:36.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:36.828 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 12:19:36.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:36.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:36.828 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:19:36.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:36.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:36.828 | INFO | Uploaded 36GiB to GPU in 2.01789s (17.8404GiB/s) | 20260506 12:19:36.828 | INFO | Downloaded 34GiB to GPU in 1.91701s (17.7359GiB/s) | 20260506 12:19:41.967 | INFO | Analyzing chunk frames = [1303552,1304064) | 20260506 12:19:41.967 | INFO | MemInfo: { | 20260506 12:19:41.967 | INFO | "gpu" : " 17 GB", | 20260506 12:19:41.967 | INFO | "huge" : "", | 20260506 12:19:41.967 | INFO | "huge_shared" : " 27 GB", | 20260506 12:19:41.967 | INFO | "malloc" : " 14 GB", | 20260506 12:19:41.967 | INFO | "pinned" : " 29 GB" | 20260506 12:19:41.967 | INFO | } | 20260506 12:19:41.967 | INFO | Perf report for the last 5.13939 seconds: | 20260506 12:19:41.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:19:41.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:41.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:41.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:41.967 | INFO | ANALYSIS : 61.8%, 3.0, 95.6ms, 0 | 20260506 12:19:41.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:41.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:41.968 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:19:41.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:41.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:41.968 | INFO | Uploaded 36GiB to GPU in 1.99975s (18.0023GiB/s) | 20260506 12:19:41.968 | INFO | Downloaded 34GiB to GPU in 1.943s (17.4987GiB/s) | 20260506 12:19:47.106 | INFO | Analyzing chunk frames = [1304064,1304576) | 20260506 12:19:47.106 | INFO | MemInfo: { | 20260506 12:19:47.106 | INFO | "gpu" : " 17 GB", | 20260506 12:19:47.106 | INFO | "huge" : "", | 20260506 12:19:47.106 | INFO | "huge_shared" : " 27 GB", | 20260506 12:19:47.106 | INFO | "malloc" : " 14 GB", | 20260506 12:19:47.106 | INFO | "pinned" : " 29 GB" | 20260506 12:19:47.106 | INFO | } | 20260506 12:19:47.106 | INFO | Perf report for the last 5.13876 seconds: | 20260506 12:19:47.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:19:47.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:47.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:47.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:47.106 | INFO | ANALYSIS : 61.4%, 3.0, 95.1ms, 0 | 20260506 12:19:47.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:47.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:47.106 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 12:19:47.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:47.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:47.107 | INFO | Uploaded 36GiB to GPU in 2.01458s (17.8697GiB/s) | 20260506 12:19:47.107 | INFO | Downloaded 34GiB to GPU in 1.93451s (17.5755GiB/s) | 20260506 12:19:52.246 | INFO | Analyzing chunk frames = [1304576,1305088) | 20260506 12:19:52.246 | INFO | MemInfo: { | 20260506 12:19:52.246 | INFO | "gpu" : " 17 GB", | 20260506 12:19:52.246 | INFO | "huge" : "", | 20260506 12:19:52.246 | INFO | "huge_shared" : " 27 GB", | 20260506 12:19:52.246 | INFO | "malloc" : " 14 GB", | 20260506 12:19:52.246 | INFO | "pinned" : " 29 GB" | 20260506 12:19:52.246 | INFO | } | 20260506 12:19:52.246 | INFO | Perf report for the last 5.13955 seconds: | 20260506 12:19:52.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:19:52.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:52.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:52.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:52.246 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 12:19:52.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:52.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:52.246 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:19:52.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:52.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:52.246 | INFO | Uploaded 36GiB to GPU in 2.02592s (17.7697GiB/s) | 20260506 12:19:52.246 | INFO | Downloaded 34GiB to GPU in 1.95656s (17.3775GiB/s) | 20260506 12:19:57.385 | INFO | Analyzing chunk frames = [1305088,1305600) | 20260506 12:19:57.385 | INFO | MemInfo: { | 20260506 12:19:57.385 | INFO | "gpu" : " 17 GB", | 20260506 12:19:57.385 | INFO | "huge" : "", | 20260506 12:19:57.385 | INFO | "huge_shared" : " 27 GB", | 20260506 12:19:57.385 | INFO | "malloc" : " 14 GB", | 20260506 12:19:57.385 | INFO | "pinned" : " 29 GB" | 20260506 12:19:57.385 | INFO | } | 20260506 12:19:57.385 | INFO | Perf report for the last 5.13915 seconds: | 20260506 12:19:57.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:19:57.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:57.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:57.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:57.385 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 12:19:57.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:57.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:57.385 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 12:19:57.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:57.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:19:57.386 | INFO | Uploaded 36GiB to GPU in 2.05856s (17.488GiB/s) | 20260506 12:19:57.386 | INFO | Downloaded 34GiB to GPU in 1.89942s (17.9002GiB/s) | 20260506 12:20:02.525 | INFO | Analyzing chunk frames = [1305600,1306112) | 20260506 12:20:02.525 | INFO | MemInfo: { | 20260506 12:20:02.525 | INFO | "gpu" : " 17 GB", | 20260506 12:20:02.525 | INFO | "huge" : "", | 20260506 12:20:02.525 | INFO | "huge_shared" : " 27 GB", | 20260506 12:20:02.525 | INFO | "malloc" : " 14 GB", | 20260506 12:20:02.525 | INFO | "pinned" : " 29 GB" | 20260506 12:20:02.525 | INFO | } | 20260506 12:20:02.525 | INFO | Perf report for the last 5.1401 seconds: | 20260506 12:20:02.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:20:02.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:02.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:02.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:02.525 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:20:02.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:02.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:02.525 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 12:20:02.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:02.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:02.525 | INFO | Uploaded 36GiB to GPU in 2.01663s (17.8515GiB/s) | 20260506 12:20:02.525 | INFO | Downloaded 34GiB to GPU in 2.03203s (16.732GiB/s) | 20260506 12:20:07.664 | INFO | Analyzing chunk frames = [1306112,1306624) | 20260506 12:20:07.664 | INFO | MemInfo: { | 20260506 12:20:07.664 | INFO | "gpu" : " 17 GB", | 20260506 12:20:07.664 | INFO | "huge" : "", | 20260506 12:20:07.664 | INFO | "huge_shared" : " 27 GB", | 20260506 12:20:07.664 | INFO | "malloc" : " 14 GB", | 20260506 12:20:07.664 | INFO | "pinned" : " 29 GB" | 20260506 12:20:07.664 | INFO | } | 20260506 12:20:07.664 | INFO | Perf report for the last 5.1387 seconds: | 20260506 12:20:07.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:20:07.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:07.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:07.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:07.664 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 12:20:07.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:07.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:07.664 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:20:07.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:07.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:07.664 | INFO | Uploaded 36GiB to GPU in 2.05549s (17.514GiB/s) | 20260506 12:20:07.664 | INFO | Downloaded 34GiB to GPU in 1.96434s (17.3086GiB/s) | 20260506 12:20:07.787 | INFO | Starting a round of Baz IO | 20260506 12:20:10.968 | INFO | Populating RTMetrics report | 20260506 12:20:10.968 | INFO | RTMetrics report populated | 20260506 12:20:10.968 | INFO | Opening tmp RTMetrics json file | 20260506 12:20:10.969 | INFO | Serializing RTMetrics to json | 20260506 12:20:10.970 | INFO | Serialization done, swapping tmp file now... | 20260506 12:20:10.970 | INFO | RTMetrics done | 20260506 12:20:11.071 | INFO | Finished a round of Baz IO: Wrote 2.086 GiB in 3.283 seconds | 20260506 12:20:11.071 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.83% / 0.03% / 58.15% | 20260506 12:20:11.071 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.0704% / 3.57628e-05% / 4.37101e-05% | 20260506 12:20:11.071 | INFO | Baz IO overall: | 20260506 12:20:11.071 | INFO | BytesWritten : 646671847520 | 20260506 12:20:11.071 | INFO | NumEvents : 312470520783 | 20260506 12:20:12.803 | INFO | Analyzing chunk frames = [1306624,1307136) | 20260506 12:20:12.803 | INFO | MemInfo: { | 20260506 12:20:12.803 | INFO | "gpu" : " 17 GB", | 20260506 12:20:12.803 | INFO | "huge" : "", | 20260506 12:20:12.803 | INFO | "huge_shared" : " 27 GB", | 20260506 12:20:12.803 | INFO | "malloc" : " 14 GB", | 20260506 12:20:12.803 | INFO | "pinned" : " 29 GB" | 20260506 12:20:12.803 | INFO | } | 20260506 12:20:12.803 | INFO | Perf report for the last 5.13922 seconds: | 20260506 12:20:12.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:20:12.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:12.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:12.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:12.803 | INFO | ANALYSIS : 63.7%, 3.0, 99.0ms, 0 | 20260506 12:20:12.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:12.803 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:20:12.803 | INFO | PRE_HQ : 61.7%, 1.9, 61.4ms, 0 | 20260506 12:20:12.803 | INFO | BAZWRITER : 63.1%, 3.6, 118.0ms, 0 | 20260506 12:20:12.803 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:20:12.803 | INFO | Uploaded 36GiB to GPU in 2.06445s (17.4381GiB/s) | 20260506 12:20:12.803 | INFO | Downloaded 38GiB to GPU in 2.21525s (17.1538GiB/s) | 20260506 12:20:17.942 | INFO | Analyzing chunk frames = [1307136,1307648) | 20260506 12:20:17.942 | INFO | MemInfo: { | 20260506 12:20:17.942 | INFO | "gpu" : " 17 GB", | 20260506 12:20:17.942 | INFO | "huge" : "", | 20260506 12:20:17.942 | INFO | "huge_shared" : " 27 GB", | 20260506 12:20:17.942 | INFO | "malloc" : " 14 GB", | 20260506 12:20:17.942 | INFO | "pinned" : " 29 GB" | 20260506 12:20:17.942 | INFO | } | 20260506 12:20:17.942 | INFO | Perf report for the last 5.13944 seconds: | 20260506 12:20:17.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:20:17.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:17.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:17.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:17.943 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:20:17.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:17.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:17.943 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:20:17.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:17.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:17.943 | INFO | Uploaded 36GiB to GPU in 2.01529s (17.8634GiB/s) | 20260506 12:20:17.943 | INFO | Downloaded 34GiB to GPU in 1.92206s (17.6893GiB/s) | 20260506 12:20:23.082 | INFO | Analyzing chunk frames = [1307648,1308160) | 20260506 12:20:23.082 | INFO | MemInfo: { | 20260506 12:20:23.082 | INFO | "gpu" : " 17 GB", | 20260506 12:20:23.082 | INFO | "huge" : "", | 20260506 12:20:23.082 | INFO | "huge_shared" : " 27 GB", | 20260506 12:20:23.082 | INFO | "malloc" : " 14 GB", | 20260506 12:20:23.082 | INFO | "pinned" : " 29 GB" | 20260506 12:20:23.082 | INFO | } | 20260506 12:20:23.082 | INFO | Perf report for the last 5.13942 seconds: | 20260506 12:20:23.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:20:23.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:23.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:23.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:23.082 | INFO | ANALYSIS : 61.8%, 3.0, 95.6ms, 0 | 20260506 12:20:23.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:23.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:23.082 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:20:23.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:23.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:23.082 | INFO | Uploaded 36GiB to GPU in 2.02845s (17.7475GiB/s) | 20260506 12:20:23.082 | INFO | Downloaded 34GiB to GPU in 1.94224s (17.5055GiB/s) | 20260506 12:20:28.221 | INFO | Analyzing chunk frames = [1308160,1308672) | 20260506 12:20:28.221 | INFO | MemInfo: { | 20260506 12:20:28.221 | INFO | "gpu" : " 17 GB", | 20260506 12:20:28.221 | INFO | "huge" : "", | 20260506 12:20:28.221 | INFO | "huge_shared" : " 27 GB", | 20260506 12:20:28.221 | INFO | "malloc" : " 14 GB", | 20260506 12:20:28.221 | INFO | "pinned" : " 29 GB" | 20260506 12:20:28.221 | INFO | } | 20260506 12:20:28.221 | INFO | Perf report for the last 5.13932 seconds: | 20260506 12:20:28.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:20:28.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:28.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:28.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:28.221 | INFO | ANALYSIS : 61.5%, 3.0, 95.4ms, 0 | 20260506 12:20:28.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:28.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:28.221 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:20:28.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:28.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:28.221 | INFO | Uploaded 36GiB to GPU in 2.07684s (17.334GiB/s) | 20260506 12:20:28.222 | INFO | Downloaded 34GiB to GPU in 1.95024s (17.4337GiB/s) | 20260506 12:20:33.360 | INFO | Analyzing chunk frames = [1308672,1309184) | 20260506 12:20:33.360 | INFO | MemInfo: { | 20260506 12:20:33.360 | INFO | "gpu" : " 17 GB", | 20260506 12:20:33.360 | INFO | "huge" : "", | 20260506 12:20:33.360 | INFO | "huge_shared" : " 27 GB", | 20260506 12:20:33.360 | INFO | "malloc" : " 14 GB", | 20260506 12:20:33.360 | INFO | "pinned" : " 29 GB" | 20260506 12:20:33.360 | INFO | } | 20260506 12:20:33.361 | INFO | Perf report for the last 5.13925 seconds: | 20260506 12:20:33.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:20:33.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:33.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:33.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:33.361 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 12:20:33.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:33.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:33.361 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:20:33.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:33.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:33.361 | INFO | Uploaded 36GiB to GPU in 2.06163s (17.4619GiB/s) | 20260506 12:20:33.361 | INFO | Downloaded 34GiB to GPU in 1.96091s (17.3389GiB/s) | 20260506 12:20:38.500 | INFO | Analyzing chunk frames = [1309184,1309696) | 20260506 12:20:38.500 | INFO | MemInfo: { | 20260506 12:20:38.500 | INFO | "gpu" : " 17 GB", | 20260506 12:20:38.500 | INFO | "huge" : "", | 20260506 12:20:38.500 | INFO | "huge_shared" : " 27 GB", | 20260506 12:20:38.500 | INFO | "malloc" : " 14 GB", | 20260506 12:20:38.500 | INFO | "pinned" : " 29 GB" | 20260506 12:20:38.500 | INFO | } | 20260506 12:20:38.500 | INFO | Perf report for the last 5.13936 seconds: | 20260506 12:20:38.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:20:38.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:38.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:38.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:38.500 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 12:20:38.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:38.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:38.500 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 12:20:38.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:38.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:38.500 | INFO | Uploaded 36GiB to GPU in 2.03996s (17.6474GiB/s) | 20260506 12:20:38.500 | INFO | Downloaded 34GiB to GPU in 1.90088s (17.8864GiB/s) | 20260506 12:20:43.639 | INFO | Analyzing chunk frames = [1309696,1310208) | 20260506 12:20:43.639 | INFO | MemInfo: { | 20260506 12:20:43.639 | INFO | "gpu" : " 17 GB", | 20260506 12:20:43.639 | INFO | "huge" : "", | 20260506 12:20:43.639 | INFO | "huge_shared" : " 27 GB", | 20260506 12:20:43.639 | INFO | "malloc" : " 14 GB", | 20260506 12:20:43.640 | INFO | "pinned" : " 29 GB" | 20260506 12:20:43.640 | INFO | } | 20260506 12:20:43.640 | INFO | Perf report for the last 5.13966 seconds: | 20260506 12:20:43.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:20:43.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:43.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:43.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:43.640 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:20:43.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:43.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:43.640 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 12:20:43.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:43.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:43.640 | INFO | Uploaded 36GiB to GPU in 2.00514s (17.9538GiB/s) | 20260506 12:20:43.640 | INFO | Downloaded 34GiB to GPU in 2.03041s (16.7454GiB/s) | 20260506 12:20:48.778 | INFO | Analyzing chunk frames = [1310208,1310720) | 20260506 12:20:48.778 | INFO | MemInfo: { | 20260506 12:20:48.778 | INFO | "gpu" : " 17 GB", | 20260506 12:20:48.778 | INFO | "huge" : "", | 20260506 12:20:48.778 | INFO | "huge_shared" : " 27 GB", | 20260506 12:20:48.778 | INFO | "malloc" : " 14 GB", | 20260506 12:20:48.778 | INFO | "pinned" : " 29 GB" | 20260506 12:20:48.778 | INFO | } | 20260506 12:20:48.778 | INFO | Perf report for the last 5.13885 seconds: | 20260506 12:20:48.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:20:48.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:48.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:48.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:48.778 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260506 12:20:48.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:48.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:48.779 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 12:20:48.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:48.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:48.779 | INFO | Uploaded 36GiB to GPU in 2.05632s (17.507GiB/s) | 20260506 12:20:48.779 | INFO | Downloaded 34GiB to GPU in 1.99425s (17.049GiB/s) | 20260506 12:20:48.779 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:20:48.779 | INFO | Total mem usage: 16.8 GiB | 20260506 12:20:48.779 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:20:48.779 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -16.1%) | 20260506 12:20:48.779 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:20:48.779 | INFO | High Water Breakdown: | 20260506 12:20:48.779 | INFO | 59.6% : 8.63 GiB ( 83.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:20:48.779 | INFO | 40.2% : 5.83 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:20:48.779 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:20:48.779 | INFO | ------------------------------------------ | 20260506 12:20:48.779 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:20:48.779 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:20:48.779 | INFO | Total mem usage: 26.5 GiB | 20260506 12:20:48.779 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:20:48.779 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:20:48.779 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:20:48.779 | INFO | High Water Breakdown: | 20260506 12:20:48.779 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:20:48.779 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:20:48.779 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:20:48.779 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:20:48.779 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:20:48.779 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:20:48.779 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:20:48.779 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:20:48.779 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:20:48.779 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:20:48.779 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:20:48.779 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:20:48.779 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:20:48.779 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:20:48.779 | INFO | ------------------------------------------ | 20260506 12:20:48.779 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:20:48.779 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:20:48.779 | INFO | Total mem usage: 25.6 GiB | 20260506 12:20:48.779 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:20:48.779 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:20:48.779 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:20:48.779 | INFO | High Water Breakdown: | 20260506 12:20:48.779 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:20:48.779 | INFO | ------------------------------------------ | 20260506 12:20:48.779 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:20:48.779 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:20:48.779 | INFO | Total mem usage: 15.7 GiB | 20260506 12:20:48.779 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:20:48.779 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:20:48.779 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:20:48.779 | INFO | High Water Breakdown: | 20260506 12:20:48.779 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:20:48.779 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:20:48.779 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:20:48.779 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:20:48.779 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:20:48.779 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:20:48.779 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:20:48.779 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:20:48.779 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:20:48.779 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:20:48.779 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:20:48.779 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:20:48.779 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:20:48.779 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:20:48.779 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:20:48.779 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:20:48.779 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:20:48.779 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:20:48.779 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:20:48.779 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:20:48.779 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:20:48.779 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:20:48.779 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:20:48.779 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:20:48.779 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:20:48.779 | INFO | ------------------------------------------ | 20260506 12:20:48.779 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:20:48.780 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:20:48.780 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:20:48.780 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:20:48.780 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.62e+01, 1.37e+01 / 2.58e+00 | 20260506 12:20:48.780 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.17e-02 / 8.09e-03 | 20260506 12:20:48.780 | INFO | PerfReport: 11.57%, Baseline -- 3.68e+00 / 4.41e+00, 5.90e-01 / 3.67e-01 | 20260506 12:20:48.780 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.47e+00, 1.94e-01 / 1.35e-01 | 20260506 12:20:48.780 | INFO | PerfReport: 15.87%, DME -- 4.04e+01 / 4.50e+01, 4.32e+00 / 3.13e+00 | 20260506 12:20:48.780 | INFO | PerfReport: 56.35%, FrameLabeling -- 1.79e+01 / 2.02e+01, 1.44e+00 / 1.84e+00 | 20260506 12:20:48.780 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.41e+00 / 3.04e+00, 4.20e-01 / 2.45e-01 | 20260506 12:20:48.780 | INFO | PerfReport: 3.76%, Metrics -- 1.19e+00 / 1.56e+00, 3.03e-01 / 1.57e-01 | 20260506 12:20:48.780 | INFO | PerfReport: 1.33%, PulseSerializing -- 4.22e-01 / 4.76e-01, 6.68e-02 / 3.19e-02 | 20260506 12:20:48.780 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:20:48.780 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:20:48.780 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:20:48.780 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 12:20:48.780 | INFO | PerfReport: 21.47%, Upload -- 2.12e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 12:20:48.780 | INFO | PerfReport: 20.92%, Download -- 2.07e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260506 12:20:48.780 | INFO | PerfReport: 32.17%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 12:20:48.780 | INFO | PerfReport: 1.90%, LockUpload -- 1.88e+00 / 0.00e+00, 7.62e+00 / 0.00e+00 | 20260506 12:20:48.780 | INFO | PerfReport: 0.99%, LockDownload -- 9.80e-01 / 0.00e+00, 2.63e+00 / 0.00e+00 | 20260506 12:20:48.780 | INFO | PerfReport: 22.54%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:20:48.780 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260506 12:20:48.910 | INFO | Starting a round of Baz IO | 20260506 12:20:52.072 | INFO | Populating RTMetrics report | 20260506 12:20:52.072 | INFO | RTMetrics report populated | 20260506 12:20:52.072 | INFO | Opening tmp RTMetrics json file | 20260506 12:20:52.073 | INFO | Serializing RTMetrics to json | 20260506 12:20:52.074 | INFO | Serialization done, swapping tmp file now... | 20260506 12:20:52.075 | INFO | RTMetrics done | 20260506 12:20:52.170 | INFO | Finished a round of Baz IO: Wrote 2.087 GiB in 3.26 seconds | 20260506 12:20:52.170 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.80% / 0.03% / 58.17% | 20260506 12:20:52.170 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.06514% / 5.5631e-05% / 4.76837e-05% | 20260506 12:20:52.170 | INFO | Baz IO overall: | 20260506 12:20:52.170 | INFO | BytesWritten : 648912383953 | 20260506 12:20:52.170 | INFO | NumEvents : 313557397519 | 20260506 12:20:53.918 | INFO | Analyzing chunk frames = [1310720,1311232) | 20260506 12:20:53.918 | INFO | MemInfo: { | 20260506 12:20:53.918 | INFO | "gpu" : " 17 GB", | 20260506 12:20:53.918 | INFO | "huge" : "", | 20260506 12:20:53.918 | INFO | "huge_shared" : " 27 GB", | 20260506 12:20:53.918 | INFO | "malloc" : " 14 GB", | 20260506 12:20:53.918 | INFO | "pinned" : " 29 GB" | 20260506 12:20:53.918 | INFO | } | 20260506 12:20:53.918 | INFO | Perf report for the last 5.13936 seconds: | 20260506 12:20:53.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:20:53.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:53.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:53.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:53.918 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 12:20:53.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:53.918 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:20:53.918 | INFO | PRE_HQ : 61.5%, 1.9, 61.5ms, 0 | 20260506 12:20:53.918 | INFO | BAZWRITER : 62.8%, 3.7, 119.5ms, 0 | 20260506 12:20:53.918 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:20:53.918 | INFO | Uploaded 36GiB to GPU in 2.08607s (17.2573GiB/s) | 20260506 12:20:53.918 | INFO | Downloaded 38GiB to GPU in 2.22356s (17.0897GiB/s) | 20260506 12:20:59.057 | INFO | Analyzing chunk frames = [1311232,1311744) | 20260506 12:20:59.057 | INFO | MemInfo: { | 20260506 12:20:59.057 | INFO | "gpu" : " 17 GB", | 20260506 12:20:59.057 | INFO | "huge" : "", | 20260506 12:20:59.057 | INFO | "huge_shared" : " 27 GB", | 20260506 12:20:59.057 | INFO | "malloc" : " 14 GB", | 20260506 12:20:59.057 | INFO | "pinned" : " 29 GB" | 20260506 12:20:59.057 | INFO | } | 20260506 12:20:59.057 | INFO | Perf report for the last 5.13959 seconds: | 20260506 12:20:59.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:20:59.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:59.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:59.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:59.057 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:20:59.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:59.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:59.057 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 12:20:59.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:59.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:20:59.058 | INFO | Uploaded 36GiB to GPU in 2.02203s (17.8038GiB/s) | 20260506 12:20:59.058 | INFO | Downloaded 34GiB to GPU in 1.91722s (17.734GiB/s) | 20260506 12:21:04.196 | INFO | Analyzing chunk frames = [1311744,1312256) | 20260506 12:21:04.196 | INFO | MemInfo: { | 20260506 12:21:04.196 | INFO | "gpu" : " 17 GB", | 20260506 12:21:04.196 | INFO | "huge" : "", | 20260506 12:21:04.196 | INFO | "huge_shared" : " 27 GB", | 20260506 12:21:04.196 | INFO | "malloc" : " 14 GB", | 20260506 12:21:04.196 | INFO | "pinned" : " 29 GB" | 20260506 12:21:04.196 | INFO | } | 20260506 12:21:04.196 | INFO | Perf report for the last 5.1391 seconds: | 20260506 12:21:04.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:21:04.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:04.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:04.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:04.197 | INFO | ANALYSIS : 61.8%, 3.0, 95.6ms, 0 | 20260506 12:21:04.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:04.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:04.197 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 12:21:04.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:04.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:04.197 | INFO | Uploaded 36GiB to GPU in 2.02628s (17.7665GiB/s) | 20260506 12:21:04.197 | INFO | Downloaded 34GiB to GPU in 1.93852s (17.5391GiB/s) | 20260506 12:21:09.336 | INFO | Analyzing chunk frames = [1312256,1312768) | 20260506 12:21:09.336 | INFO | MemInfo: { | 20260506 12:21:09.336 | INFO | "gpu" : " 17 GB", | 20260506 12:21:09.336 | INFO | "huge" : "", | 20260506 12:21:09.336 | INFO | "huge_shared" : " 27 GB", | 20260506 12:21:09.336 | INFO | "malloc" : " 14 GB", | 20260506 12:21:09.336 | INFO | "pinned" : " 29 GB" | 20260506 12:21:09.336 | INFO | } | 20260506 12:21:09.336 | INFO | Perf report for the last 5.1394 seconds: | 20260506 12:21:09.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:21:09.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:09.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:09.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:09.336 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 12:21:09.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:09.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:09.336 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:21:09.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:09.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:09.336 | INFO | Uploaded 36GiB to GPU in 2.02101s (17.8128GiB/s) | 20260506 12:21:09.336 | INFO | Downloaded 34GiB to GPU in 1.94307s (17.4981GiB/s) | 20260506 12:21:14.475 | INFO | Analyzing chunk frames = [1312768,1313280) | 20260506 12:21:14.475 | INFO | MemInfo: { | 20260506 12:21:14.475 | INFO | "gpu" : " 17 GB", | 20260506 12:21:14.475 | INFO | "huge" : "", | 20260506 12:21:14.475 | INFO | "huge_shared" : " 27 GB", | 20260506 12:21:14.475 | INFO | "malloc" : " 14 GB", | 20260506 12:21:14.475 | INFO | "pinned" : " 29 GB" | 20260506 12:21:14.475 | INFO | } | 20260506 12:21:14.475 | INFO | Perf report for the last 5.13955 seconds: | 20260506 12:21:14.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:21:14.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:14.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:14.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:14.476 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 12:21:14.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:14.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:14.476 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:21:14.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:14.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:14.476 | INFO | Uploaded 36GiB to GPU in 2.02409s (17.7858GiB/s) | 20260506 12:21:14.476 | INFO | Downloaded 34GiB to GPU in 1.93426s (17.5778GiB/s) | 20260506 12:21:19.614 | INFO | Analyzing chunk frames = [1313280,1313792) | 20260506 12:21:19.614 | INFO | MemInfo: { | 20260506 12:21:19.614 | INFO | "gpu" : " 17 GB", | 20260506 12:21:19.615 | INFO | "huge" : "", | 20260506 12:21:19.615 | INFO | "huge_shared" : " 27 GB", | 20260506 12:21:19.615 | INFO | "malloc" : " 14 GB", | 20260506 12:21:19.615 | INFO | "pinned" : " 29 GB" | 20260506 12:21:19.615 | INFO | } | 20260506 12:21:19.615 | INFO | Perf report for the last 5.13916 seconds: | 20260506 12:21:19.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:21:19.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:19.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:19.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:19.615 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:21:19.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:19.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:19.615 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 12:21:19.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:19.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:19.615 | INFO | Uploaded 36GiB to GPU in 2.11123s (17.0517GiB/s) | 20260506 12:21:19.615 | INFO | Downloaded 34GiB to GPU in 1.98131s (17.1603GiB/s) | 20260506 12:21:24.755 | INFO | Analyzing chunk frames = [1313792,1314304) | 20260506 12:21:24.755 | INFO | MemInfo: { | 20260506 12:21:24.755 | INFO | "gpu" : " 17 GB", | 20260506 12:21:24.755 | INFO | "huge" : "", | 20260506 12:21:24.755 | INFO | "huge_shared" : " 27 GB", | 20260506 12:21:24.755 | INFO | "malloc" : " 14 GB", | 20260506 12:21:24.755 | INFO | "pinned" : " 29 GB" | 20260506 12:21:24.755 | INFO | } | 20260506 12:21:24.755 | INFO | Perf report for the last 5.1402 seconds: | 20260506 12:21:24.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:21:24.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:24.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:24.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:24.755 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:21:24.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:24.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:24.755 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 12:21:24.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:24.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:24.755 | INFO | Uploaded 36GiB to GPU in 2.00925s (17.9171GiB/s) | 20260506 12:21:24.755 | INFO | Downloaded 34GiB to GPU in 1.9991s (17.0077GiB/s) | 20260506 12:21:29.893 | INFO | Analyzing chunk frames = [1314304,1314816) | 20260506 12:21:29.893 | INFO | MemInfo: { | 20260506 12:21:29.893 | INFO | "gpu" : " 17 GB", | 20260506 12:21:29.893 | INFO | "huge" : "", | 20260506 12:21:29.893 | INFO | "huge_shared" : " 27 GB", | 20260506 12:21:29.893 | INFO | "malloc" : " 14 GB", | 20260506 12:21:29.893 | INFO | "pinned" : " 29 GB" | 20260506 12:21:29.893 | INFO | } | 20260506 12:21:29.893 | INFO | Perf report for the last 5.13841 seconds: | 20260506 12:21:29.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:21:29.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:29.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:29.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:29.893 | INFO | ANALYSIS : 62.9%, 3.0, 97.9ms, 0 | 20260506 12:21:29.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:29.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:29.893 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:21:29.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:29.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:29.893 | INFO | Uploaded 36GiB to GPU in 2.07426s (17.3556GiB/s) | 20260506 12:21:29.893 | INFO | Downloaded 34GiB to GPU in 1.99238s (17.0651GiB/s) | 20260506 12:21:30.018 | INFO | Starting a round of Baz IO | 20260506 12:21:33.204 | INFO | Populating RTMetrics report | 20260506 12:21:33.205 | INFO | RTMetrics report populated | 20260506 12:21:33.205 | INFO | Opening tmp RTMetrics json file | 20260506 12:21:33.205 | INFO | Serializing RTMetrics to json | 20260506 12:21:33.206 | INFO | Serialization done, swapping tmp file now... | 20260506 12:21:33.207 | INFO | RTMetrics done | 20260506 12:21:33.307 | INFO | Finished a round of Baz IO: Wrote 2.086 GiB in 3.289 seconds | 20260506 12:21:33.308 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.77% / 0.03% / 58.20% | 20260506 12:21:33.308 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.05313% / 3.97364e-05% / 4.76837e-05% | 20260506 12:21:33.308 | INFO | Baz IO overall: | 20260506 12:21:33.308 | INFO | BytesWritten : 651152362906 | 20260506 12:21:33.308 | INFO | NumEvents : 314643737376 | 20260506 12:21:35.032 | INFO | Analyzing chunk frames = [1314816,1315328) | 20260506 12:21:35.033 | INFO | MemInfo: { | 20260506 12:21:35.033 | INFO | "gpu" : " 17 GB", | 20260506 12:21:35.033 | INFO | "huge" : "", | 20260506 12:21:35.033 | INFO | "huge_shared" : " 27 GB", | 20260506 12:21:35.033 | INFO | "malloc" : " 14 GB", | 20260506 12:21:35.033 | INFO | "pinned" : " 29 GB" | 20260506 12:21:35.033 | INFO | } | 20260506 12:21:35.033 | INFO | Perf report for the last 5.13934 seconds: | 20260506 12:21:35.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:21:35.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:35.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:35.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:35.033 | INFO | ANALYSIS : 63.8%, 3.0, 99.1ms, 0 | 20260506 12:21:35.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:35.033 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:21:35.033 | INFO | PRE_HQ : 61.7%, 1.9, 61.2ms, 0 | 20260506 12:21:35.033 | INFO | BAZWRITER : 63.1%, 3.6, 117.7ms, 0 | 20260506 12:21:35.033 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:21:35.033 | INFO | Uploaded 36GiB to GPU in 2.08619s (17.2563GiB/s) | 20260506 12:21:35.033 | INFO | Downloaded 38GiB to GPU in 2.20786s (17.2113GiB/s) | 20260506 12:21:40.172 | INFO | Analyzing chunk frames = [1315328,1315840) | 20260506 12:21:40.172 | INFO | MemInfo: { | 20260506 12:21:40.172 | INFO | "gpu" : " 17 GB", | 20260506 12:21:40.172 | INFO | "huge" : "", | 20260506 12:21:40.172 | INFO | "huge_shared" : " 27 GB", | 20260506 12:21:40.172 | INFO | "malloc" : " 14 GB", | 20260506 12:21:40.172 | INFO | "pinned" : " 29 GB" | 20260506 12:21:40.172 | INFO | } | 20260506 12:21:40.172 | INFO | Perf report for the last 5.13928 seconds: | 20260506 12:21:40.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:21:40.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:40.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:40.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:40.172 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:21:40.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:40.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:40.172 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:21:40.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:40.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:40.172 | INFO | Uploaded 36GiB to GPU in 2.01439s (17.8714GiB/s) | 20260506 12:21:40.172 | INFO | Downloaded 34GiB to GPU in 1.91948s (17.7131GiB/s) | 20260506 12:21:45.311 | INFO | Analyzing chunk frames = [1315840,1316352) | 20260506 12:21:45.311 | INFO | MemInfo: { | 20260506 12:21:45.311 | INFO | "gpu" : " 17 GB", | 20260506 12:21:45.311 | INFO | "huge" : "", | 20260506 12:21:45.311 | INFO | "huge_shared" : " 27 GB", | 20260506 12:21:45.311 | INFO | "malloc" : " 14 GB", | 20260506 12:21:45.311 | INFO | "pinned" : " 29 GB" | 20260506 12:21:45.312 | INFO | } | 20260506 12:21:45.312 | INFO | Perf report for the last 5.13962 seconds: | 20260506 12:21:45.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:21:45.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:45.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:45.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:45.312 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:21:45.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:45.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:45.312 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:21:45.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:45.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:45.312 | INFO | Uploaded 36GiB to GPU in 2.02672s (17.7627GiB/s) | 20260506 12:21:45.312 | INFO | Downloaded 34GiB to GPU in 1.93955s (17.5298GiB/s) | 20260506 12:21:50.451 | INFO | Analyzing chunk frames = [1316352,1316864) | 20260506 12:21:50.451 | INFO | MemInfo: { | 20260506 12:21:50.451 | INFO | "gpu" : " 17 GB", | 20260506 12:21:50.451 | INFO | "huge" : "", | 20260506 12:21:50.451 | INFO | "huge_shared" : " 27 GB", | 20260506 12:21:50.451 | INFO | "malloc" : " 14 GB", | 20260506 12:21:50.451 | INFO | "pinned" : " 29 GB" | 20260506 12:21:50.451 | INFO | } | 20260506 12:21:50.451 | INFO | Perf report for the last 5.13917 seconds: | 20260506 12:21:50.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:21:50.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:50.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:50.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:50.451 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 12:21:50.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:50.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:50.451 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:21:50.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:50.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:50.451 | INFO | Uploaded 36GiB to GPU in 2.02701s (17.7601GiB/s) | 20260506 12:21:50.451 | INFO | Downloaded 34GiB to GPU in 1.92791s (17.6357GiB/s) | 20260506 12:21:55.590 | INFO | Analyzing chunk frames = [1316864,1317376) | 20260506 12:21:55.590 | INFO | MemInfo: { | 20260506 12:21:55.590 | INFO | "gpu" : " 17 GB", | 20260506 12:21:55.590 | INFO | "huge" : "", | 20260506 12:21:55.590 | INFO | "huge_shared" : " 27 GB", | 20260506 12:21:55.590 | INFO | "malloc" : " 14 GB", | 20260506 12:21:55.590 | INFO | "pinned" : " 29 GB" | 20260506 12:21:55.590 | INFO | } | 20260506 12:21:55.590 | INFO | Perf report for the last 5.1397 seconds: | 20260506 12:21:55.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:21:55.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:55.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:55.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:55.590 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 12:21:55.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:55.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:55.590 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:21:55.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:55.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:21:55.591 | INFO | Uploaded 36GiB to GPU in 2.03139s (17.7218GiB/s) | 20260506 12:21:55.591 | INFO | Downloaded 34GiB to GPU in 1.93849s (17.5394GiB/s) | 20260506 12:22:00.729 | INFO | Analyzing chunk frames = [1317376,1317888) | 20260506 12:22:00.729 | INFO | MemInfo: { | 20260506 12:22:00.729 | INFO | "gpu" : " 17 GB", | 20260506 12:22:00.729 | INFO | "huge" : "", | 20260506 12:22:00.729 | INFO | "huge_shared" : " 27 GB", | 20260506 12:22:00.729 | INFO | "malloc" : " 14 GB", | 20260506 12:22:00.729 | INFO | "pinned" : " 29 GB" | 20260506 12:22:00.729 | INFO | } | 20260506 12:22:00.729 | INFO | Perf report for the last 5.13894 seconds: | 20260506 12:22:00.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:22:00.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:00.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:00.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:00.729 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 12:22:00.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:00.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:00.729 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 12:22:00.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:00.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:00.730 | INFO | Uploaded 36GiB to GPU in 2.0836s (17.2778GiB/s) | 20260506 12:22:00.730 | INFO | Downloaded 34GiB to GPU in 1.92124s (17.6969GiB/s) | 20260506 12:22:05.869 | INFO | Analyzing chunk frames = [1317888,1318400) | 20260506 12:22:05.869 | INFO | MemInfo: { | 20260506 12:22:05.869 | INFO | "gpu" : " 17 GB", | 20260506 12:22:05.869 | INFO | "huge" : "", | 20260506 12:22:05.869 | INFO | "huge_shared" : " 27 GB", | 20260506 12:22:05.869 | INFO | "malloc" : " 14 GB", | 20260506 12:22:05.869 | INFO | "pinned" : " 29 GB" | 20260506 12:22:05.869 | INFO | } | 20260506 12:22:05.869 | INFO | Perf report for the last 5.14004 seconds: | 20260506 12:22:05.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:22:05.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:05.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:05.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:05.869 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:22:05.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:05.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:05.869 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 12:22:05.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:05.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:05.870 | INFO | Uploaded 36GiB to GPU in 2.00887s (17.9206GiB/s) | 20260506 12:22:05.870 | INFO | Downloaded 34GiB to GPU in 2.03318s (16.7226GiB/s) | 20260506 12:22:11.008 | INFO | Analyzing chunk frames = [1318400,1318912) | 20260506 12:22:11.008 | INFO | MemInfo: { | 20260506 12:22:11.008 | INFO | "gpu" : " 17 GB", | 20260506 12:22:11.008 | INFO | "huge" : "", | 20260506 12:22:11.008 | INFO | "huge_shared" : " 27 GB", | 20260506 12:22:11.008 | INFO | "malloc" : " 14 GB", | 20260506 12:22:11.008 | INFO | "pinned" : " 29 GB" | 20260506 12:22:11.008 | INFO | } | 20260506 12:22:11.008 | INFO | Perf report for the last 5.13873 seconds: | 20260506 12:22:11.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:22:11.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:11.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:11.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:11.008 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 12:22:11.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:11.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:11.008 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 12:22:11.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:11.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:11.009 | INFO | Uploaded 36GiB to GPU in 2.05619s (17.5081GiB/s) | 20260506 12:22:11.009 | INFO | Downloaded 34GiB to GPU in 1.96686s (17.2864GiB/s) | 20260506 12:22:11.009 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:22:11.009 | INFO | Total mem usage: 16.8 GiB | 20260506 12:22:11.009 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:22:11.009 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.6%) | 20260506 12:22:11.009 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:22:11.009 | INFO | High Water Breakdown: | 20260506 12:22:11.009 | INFO | 59.6% : 8.59 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:22:11.009 | INFO | 40.3% : 5.8 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:22:11.009 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:22:11.009 | INFO | ------------------------------------------ | 20260506 12:22:11.009 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:22:11.009 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:22:11.009 | INFO | Total mem usage: 26.5 GiB | 20260506 12:22:11.009 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:22:11.009 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:22:11.009 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:22:11.009 | INFO | High Water Breakdown: | 20260506 12:22:11.009 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:22:11.009 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:22:11.009 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:22:11.009 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:22:11.009 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:22:11.009 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:22:11.009 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:22:11.009 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:22:11.009 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:22:11.009 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:22:11.009 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:22:11.009 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:22:11.009 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:22:11.009 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:22:11.009 | INFO | ------------------------------------------ | 20260506 12:22:11.009 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:22:11.009 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:22:11.009 | INFO | Total mem usage: 25.6 GiB | 20260506 12:22:11.009 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:22:11.009 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:22:11.009 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:22:11.009 | INFO | High Water Breakdown: | 20260506 12:22:11.009 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:22:11.009 | INFO | ------------------------------------------ | 20260506 12:22:11.009 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:22:11.009 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:22:11.009 | INFO | Total mem usage: 15.7 GiB | 20260506 12:22:11.009 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:22:11.009 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:22:11.009 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 12:22:11.009 | INFO | High Water Breakdown: | 20260506 12:22:11.009 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:22:11.009 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:22:11.009 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:22:11.009 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:22:11.009 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:22:11.009 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:22:11.009 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:22:11.009 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:22:11.009 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:22:11.009 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:22:11.009 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:22:11.009 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:22:11.009 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:22:11.009 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:22:11.009 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:22:11.009 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:22:11.009 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:22:11.009 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:22:11.009 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:22:11.009 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:22:11.009 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:22:11.009 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:22:11.009 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:22:11.009 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:22:11.009 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:22:11.009 | INFO | ------------------------------------------ | 20260506 12:22:11.009 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:22:11.009 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:22:11.009 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:22:11.009 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:22:11.009 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.62e+01, 1.36e+01 / 2.42e+00 | 20260506 12:22:11.009 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.02e-01, 1.04e-02 / 8.07e-03 | 20260506 12:22:11.009 | INFO | PerfReport: 11.57%, Baseline -- 3.68e+00 / 4.44e+00, 5.91e-01 / 3.79e-01 | 20260506 12:22:11.009 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.47e+00, 1.93e-01 / 1.44e-01 | 20260506 12:22:11.009 | INFO | PerfReport: 15.87%, DME -- 4.03e+01 / 4.49e+01, 4.19e+00 / 2.84e+00 | 20260506 12:22:11.009 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.79e+01 / 2.02e+01, 1.43e+00 / 1.83e+00 | 20260506 12:22:11.009 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.40e+00 / 3.03e+00, 4.14e-01 / 2.47e-01 | 20260506 12:22:11.009 | INFO | PerfReport: 3.75%, Metrics -- 1.19e+00 / 1.57e+00, 3.02e-01 / 1.80e-01 | 20260506 12:22:11.010 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.20e-01 / 4.80e-01, 6.63e-02 / 2.74e-02 | 20260506 12:22:11.010 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:22:11.010 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:22:11.010 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:22:11.010 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 12:22:11.010 | INFO | PerfReport: 21.53%, Upload -- 2.13e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 12:22:11.010 | INFO | PerfReport: 20.92%, Download -- 2.07e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 12:22:11.010 | INFO | PerfReport: 32.12%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 12:22:11.010 | INFO | PerfReport: 1.94%, LockUpload -- 1.92e+00 / 0.00e+00, 7.60e+00 / 0.00e+00 | 20260506 12:22:11.010 | INFO | PerfReport: 0.92%, LockDownload -- 9.13e-01 / 0.00e+00, 2.48e+00 / 0.00e+00 | 20260506 12:22:11.010 | INFO | PerfReport: 22.56%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:22:11.010 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260506 12:22:11.140 | INFO | Starting a round of Baz IO | 20260506 12:22:14.309 | INFO | Populating RTMetrics report | 20260506 12:22:14.309 | INFO | RTMetrics report populated | 20260506 12:22:14.309 | INFO | Opening tmp RTMetrics json file | 20260506 12:22:14.310 | INFO | Serializing RTMetrics to json | 20260506 12:22:14.311 | INFO | Serialization done, swapping tmp file now... | 20260506 12:22:14.311 | INFO | RTMetrics done | 20260506 12:22:14.408 | INFO | Finished a round of Baz IO: Wrote 2.087 GiB in 3.267 seconds | 20260506 12:22:14.408 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.75% / 0.03% / 58.23% | 20260506 12:22:14.408 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.04471% / 5.96046e-05% / 2.78155e-05% | 20260506 12:22:14.408 | INFO | Baz IO overall: | 20260506 12:22:14.408 | INFO | BytesWritten : 653392751378 | 20260506 12:22:14.408 | INFO | NumEvents : 315730416023 | 20260506 12:22:16.147 | INFO | Analyzing chunk frames = [1318912,1319424) | 20260506 12:22:16.147 | INFO | MemInfo: { | 20260506 12:22:16.147 | INFO | "gpu" : " 17 GB", | 20260506 12:22:16.147 | INFO | "huge" : "", | 20260506 12:22:16.147 | INFO | "huge_shared" : " 27 GB", | 20260506 12:22:16.147 | INFO | "malloc" : " 14 GB", | 20260506 12:22:16.147 | INFO | "pinned" : " 29 GB" | 20260506 12:22:16.147 | INFO | } | 20260506 12:22:16.147 | INFO | Perf report for the last 5.13928 seconds: | 20260506 12:22:16.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:22:16.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:16.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:16.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:16.148 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260506 12:22:16.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:16.148 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 12:22:16.148 | INFO | PRE_HQ : 61.8%, 1.9, 61.6ms, 0 | 20260506 12:22:16.148 | INFO | BAZWRITER : 62.9%, 3.7, 119.4ms, 0 | 20260506 12:22:16.148 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:22:16.148 | INFO | Uploaded 36GiB to GPU in 2.09462s (17.1869GiB/s) | 20260506 12:22:16.148 | INFO | Downloaded 38GiB to GPU in 2.21241s (17.1758GiB/s) | 20260506 12:22:21.291 | INFO | Analyzing chunk frames = [1319424,1319936) | 20260506 12:22:21.291 | INFO | MemInfo: { | 20260506 12:22:21.291 | INFO | "gpu" : " 17 GB", | 20260506 12:22:21.291 | INFO | "huge" : "", | 20260506 12:22:21.291 | INFO | "huge_shared" : " 27 GB", | 20260506 12:22:21.291 | INFO | "malloc" : " 14 GB", | 20260506 12:22:21.291 | INFO | "pinned" : " 29 GB" | 20260506 12:22:21.291 | INFO | } | 20260506 12:22:21.291 | INFO | Perf report for the last 5.14381 seconds: | 20260506 12:22:21.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:22:21.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:21.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:21.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:21.291 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:22:21.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:21.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:21.291 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:22:21.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:21.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:21.292 | INFO | Uploaded 36GiB to GPU in 2.02214s (17.8029GiB/s) | 20260506 12:22:21.292 | INFO | Downloaded 34GiB to GPU in 1.9177s (17.7295GiB/s) | 20260506 12:22:26.426 | INFO | Analyzing chunk frames = [1319936,1320448) | 20260506 12:22:26.426 | INFO | MemInfo: { | 20260506 12:22:26.426 | INFO | "gpu" : " 17 GB", | 20260506 12:22:26.426 | INFO | "huge" : "", | 20260506 12:22:26.426 | INFO | "huge_shared" : " 27 GB", | 20260506 12:22:26.426 | INFO | "malloc" : " 14 GB", | 20260506 12:22:26.426 | INFO | "pinned" : " 29 GB" | 20260506 12:22:26.426 | INFO | } | 20260506 12:22:26.426 | INFO | Perf report for the last 5.13489 seconds: | 20260506 12:22:26.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:22:26.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:26.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:26.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:26.426 | INFO | ANALYSIS : 61.8%, 3.0, 95.6ms, 0 | 20260506 12:22:26.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:26.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:26.426 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:22:26.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:26.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:26.426 | INFO | Uploaded 36GiB to GPU in 2.01901s (17.8305GiB/s) | 20260506 12:22:26.427 | INFO | Downloaded 34GiB to GPU in 1.93516s (17.5696GiB/s) | 20260506 12:22:31.565 | INFO | Analyzing chunk frames = [1320448,1320960) | 20260506 12:22:31.565 | INFO | MemInfo: { | 20260506 12:22:31.565 | INFO | "gpu" : " 17 GB", | 20260506 12:22:31.565 | INFO | "huge" : "", | 20260506 12:22:31.566 | INFO | "huge_shared" : " 27 GB", | 20260506 12:22:31.566 | INFO | "malloc" : " 14 GB", | 20260506 12:22:31.566 | INFO | "pinned" : " 29 GB" | 20260506 12:22:31.566 | INFO | } | 20260506 12:22:31.566 | INFO | Perf report for the last 5.13938 seconds: | 20260506 12:22:31.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:22:31.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:31.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:31.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:31.566 | INFO | ANALYSIS : 61.5%, 3.0, 95.4ms, 0 | 20260506 12:22:31.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:31.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:31.566 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 12:22:31.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:31.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:31.566 | INFO | Uploaded 36GiB to GPU in 2.07402s (17.3576GiB/s) | 20260506 12:22:31.566 | INFO | Downloaded 34GiB to GPU in 1.96037s (17.3436GiB/s) | 20260506 12:22:36.705 | INFO | Analyzing chunk frames = [1320960,1321472) | 20260506 12:22:36.705 | INFO | MemInfo: { | 20260506 12:22:36.705 | INFO | "gpu" : " 17 GB", | 20260506 12:22:36.705 | INFO | "huge" : "", | 20260506 12:22:36.705 | INFO | "huge_shared" : " 27 GB", | 20260506 12:22:36.705 | INFO | "malloc" : " 14 GB", | 20260506 12:22:36.705 | INFO | "pinned" : " 29 GB" | 20260506 12:22:36.705 | INFO | } | 20260506 12:22:36.705 | INFO | Perf report for the last 5.13908 seconds: | 20260506 12:22:36.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:22:36.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:36.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:36.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:36.705 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 12:22:36.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:36.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:36.705 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 12:22:36.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:36.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:36.705 | INFO | Uploaded 36GiB to GPU in 2.05293s (17.5359GiB/s) | 20260506 12:22:36.705 | INFO | Downloaded 34GiB to GPU in 1.97244s (17.2376GiB/s) | 20260506 12:22:41.844 | INFO | Analyzing chunk frames = [1321472,1321984) | 20260506 12:22:41.844 | INFO | MemInfo: { | 20260506 12:22:41.844 | INFO | "gpu" : " 17 GB", | 20260506 12:22:41.844 | INFO | "huge" : "", | 20260506 12:22:41.844 | INFO | "huge_shared" : " 27 GB", | 20260506 12:22:41.844 | INFO | "malloc" : " 14 GB", | 20260506 12:22:41.844 | INFO | "pinned" : " 29 GB" | 20260506 12:22:41.844 | INFO | } | 20260506 12:22:41.844 | INFO | Perf report for the last 5.13941 seconds: | 20260506 12:22:41.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:22:41.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:41.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:41.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:41.844 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:22:41.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:41.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:41.844 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:22:41.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:41.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:41.844 | INFO | Uploaded 36GiB to GPU in 2.103s (17.1184GiB/s) | 20260506 12:22:41.844 | INFO | Downloaded 34GiB to GPU in 1.98309s (17.145GiB/s) | 20260506 12:22:46.984 | INFO | Analyzing chunk frames = [1321984,1322496) | 20260506 12:22:46.984 | INFO | MemInfo: { | 20260506 12:22:46.984 | INFO | "gpu" : " 17 GB", | 20260506 12:22:46.984 | INFO | "huge" : "", | 20260506 12:22:46.984 | INFO | "huge_shared" : " 27 GB", | 20260506 12:22:46.984 | INFO | "malloc" : " 14 GB", | 20260506 12:22:46.984 | INFO | "pinned" : " 29 GB" | 20260506 12:22:46.984 | INFO | } | 20260506 12:22:46.984 | INFO | Perf report for the last 5.13971 seconds: | 20260506 12:22:46.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:22:46.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:46.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:46.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:46.984 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:22:46.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:46.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:46.984 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 12:22:46.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:46.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:46.984 | INFO | Uploaded 36GiB to GPU in 2.00587s (17.9473GiB/s) | 20260506 12:22:46.984 | INFO | Downloaded 34GiB to GPU in 1.99499s (17.0427GiB/s) | 20260506 12:22:52.123 | INFO | Analyzing chunk frames = [1322496,1323008) | 20260506 12:22:52.123 | INFO | MemInfo: { | 20260506 12:22:52.123 | INFO | "gpu" : " 17 GB", | 20260506 12:22:52.123 | INFO | "huge" : "", | 20260506 12:22:52.123 | INFO | "huge_shared" : " 27 GB", | 20260506 12:22:52.123 | INFO | "malloc" : " 14 GB", | 20260506 12:22:52.123 | INFO | "pinned" : " 29 GB" | 20260506 12:22:52.123 | INFO | } | 20260506 12:22:52.123 | INFO | Perf report for the last 5.13904 seconds: | 20260506 12:22:52.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:22:52.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:52.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:52.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:52.123 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260506 12:22:52.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:52.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:52.123 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:22:52.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:52.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:52.124 | INFO | Uploaded 36GiB to GPU in 2.04519s (17.6023GiB/s) | 20260506 12:22:52.124 | INFO | Downloaded 34GiB to GPU in 2.00296s (16.9748GiB/s) | 20260506 12:22:52.253 | INFO | Starting a round of Baz IO | 20260506 12:22:55.415 | INFO | Populating RTMetrics report | 20260506 12:22:55.416 | INFO | RTMetrics report populated | 20260506 12:22:55.416 | INFO | Opening tmp RTMetrics json file | 20260506 12:22:55.417 | INFO | Serializing RTMetrics to json | 20260506 12:22:55.418 | INFO | Serialization done, swapping tmp file now... | 20260506 12:22:55.418 | INFO | RTMetrics done | 20260506 12:22:55.511 | INFO | Finished a round of Baz IO: Wrote 2.088 GiB in 3.257 seconds | 20260506 12:22:55.511 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.72% / 0.03% / 58.25% | 20260506 12:22:55.511 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.04155% / 4.37101e-05% / 3.57628e-05% | 20260506 12:22:55.511 | INFO | Baz IO overall: | 20260506 12:22:55.511 | INFO | BytesWritten : 655634576126 | 20260506 12:22:55.511 | INFO | NumEvents : 316818511677 | 20260506 12:22:57.262 | INFO | Analyzing chunk frames = [1323008,1323520) | 20260506 12:22:57.262 | INFO | MemInfo: { | 20260506 12:22:57.262 | INFO | "gpu" : " 17 GB", | 20260506 12:22:57.262 | INFO | "huge" : "", | 20260506 12:22:57.262 | INFO | "huge_shared" : " 27 GB", | 20260506 12:22:57.262 | INFO | "malloc" : " 14 GB", | 20260506 12:22:57.262 | INFO | "pinned" : " 29 GB" | 20260506 12:22:57.262 | INFO | } | 20260506 12:22:57.262 | INFO | Perf report for the last 5.13935 seconds: | 20260506 12:22:57.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:22:57.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:57.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:57.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:57.262 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 12:22:57.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:22:57.262 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:22:57.262 | INFO | PRE_HQ : 61.3%, 1.9, 61.0ms, 0 | 20260506 12:22:57.262 | INFO | BAZWRITER : 62.8%, 3.7, 118.3ms, 0 | 20260506 12:22:57.262 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:22:57.262 | INFO | Uploaded 36GiB to GPU in 2.07491s (17.3502GiB/s) | 20260506 12:22:57.263 | INFO | Downloaded 38GiB to GPU in 2.21876s (17.1267GiB/s) | 20260506 12:23:02.401 | INFO | Analyzing chunk frames = [1323520,1324032) | 20260506 12:23:02.401 | INFO | MemInfo: { | 20260506 12:23:02.401 | INFO | "gpu" : " 17 GB", | 20260506 12:23:02.401 | INFO | "huge" : "", | 20260506 12:23:02.401 | INFO | "huge_shared" : " 27 GB", | 20260506 12:23:02.401 | INFO | "malloc" : " 14 GB", | 20260506 12:23:02.401 | INFO | "pinned" : " 29 GB" | 20260506 12:23:02.401 | INFO | } | 20260506 12:23:02.401 | INFO | Perf report for the last 5.13917 seconds: | 20260506 12:23:02.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:23:02.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:02.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:02.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:02.401 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:23:02.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:02.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:02.401 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:23:02.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:02.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:02.402 | INFO | Uploaded 36GiB to GPU in 2.01543s (17.8622GiB/s) | 20260506 12:23:02.402 | INFO | Downloaded 34GiB to GPU in 1.92039s (17.7047GiB/s) | 20260506 12:23:07.541 | INFO | Analyzing chunk frames = [1324032,1324544) | 20260506 12:23:07.541 | INFO | MemInfo: { | 20260506 12:23:07.541 | INFO | "gpu" : " 17 GB", | 20260506 12:23:07.541 | INFO | "huge" : "", | 20260506 12:23:07.541 | INFO | "huge_shared" : " 27 GB", | 20260506 12:23:07.541 | INFO | "malloc" : " 14 GB", | 20260506 12:23:07.541 | INFO | "pinned" : " 29 GB" | 20260506 12:23:07.541 | INFO | } | 20260506 12:23:07.541 | INFO | Perf report for the last 5.13953 seconds: | 20260506 12:23:07.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:23:07.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:07.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:07.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:07.541 | INFO | ANALYSIS : 61.8%, 3.0, 95.6ms, 0 | 20260506 12:23:07.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:07.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:07.541 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:23:07.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:07.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:07.541 | INFO | Uploaded 36GiB to GPU in 2.02416s (17.7851GiB/s) | 20260506 12:23:07.541 | INFO | Downloaded 34GiB to GPU in 1.94139s (17.5132GiB/s) | 20260506 12:23:12.680 | INFO | Analyzing chunk frames = [1324544,1325056) | 20260506 12:23:12.680 | INFO | MemInfo: { | 20260506 12:23:12.680 | INFO | "gpu" : " 17 GB", | 20260506 12:23:12.680 | INFO | "huge" : "", | 20260506 12:23:12.680 | INFO | "huge_shared" : " 27 GB", | 20260506 12:23:12.680 | INFO | "malloc" : " 14 GB", | 20260506 12:23:12.680 | INFO | "pinned" : " 29 GB" | 20260506 12:23:12.680 | INFO | } | 20260506 12:23:12.680 | INFO | Perf report for the last 5.1392 seconds: | 20260506 12:23:12.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:23:12.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:12.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:12.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:12.680 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 12:23:12.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:12.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:12.680 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:23:12.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:12.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:12.680 | INFO | Uploaded 36GiB to GPU in 2.01679s (17.8501GiB/s) | 20260506 12:23:12.680 | INFO | Downloaded 34GiB to GPU in 1.94189s (17.5087GiB/s) | 20260506 12:23:17.819 | INFO | Analyzing chunk frames = [1325056,1325568) | 20260506 12:23:17.819 | INFO | MemInfo: { | 20260506 12:23:17.819 | INFO | "gpu" : " 17 GB", | 20260506 12:23:17.819 | INFO | "huge" : "", | 20260506 12:23:17.819 | INFO | "huge_shared" : " 27 GB", | 20260506 12:23:17.819 | INFO | "malloc" : " 14 GB", | 20260506 12:23:17.819 | INFO | "pinned" : " 29 GB" | 20260506 12:23:17.819 | INFO | } | 20260506 12:23:17.820 | INFO | Perf report for the last 5.13932 seconds: | 20260506 12:23:17.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:23:17.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:17.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:17.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:17.820 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 12:23:17.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:17.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:17.820 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:23:17.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:17.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:17.820 | INFO | Uploaded 36GiB to GPU in 2.03438s (17.6958GiB/s) | 20260506 12:23:17.820 | INFO | Downloaded 34GiB to GPU in 1.94671s (17.4654GiB/s) | 20260506 12:23:22.959 | INFO | Analyzing chunk frames = [1325568,1326080) | 20260506 12:23:22.959 | INFO | MemInfo: { | 20260506 12:23:22.959 | INFO | "gpu" : " 17 GB", | 20260506 12:23:22.959 | INFO | "huge" : "", | 20260506 12:23:22.959 | INFO | "huge_shared" : " 27 GB", | 20260506 12:23:22.959 | INFO | "malloc" : " 14 GB", | 20260506 12:23:22.959 | INFO | "pinned" : " 29 GB" | 20260506 12:23:22.959 | INFO | } | 20260506 12:23:22.959 | INFO | Perf report for the last 5.13948 seconds: | 20260506 12:23:22.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:23:22.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:22.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:22.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:22.959 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 12:23:22.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:22.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:22.959 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260506 12:23:22.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:22.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:22.959 | INFO | Uploaded 36GiB to GPU in 2.06201s (17.4587GiB/s) | 20260506 12:23:22.959 | INFO | Downloaded 34GiB to GPU in 1.93458s (17.5749GiB/s) | 20260506 12:23:28.099 | INFO | Analyzing chunk frames = [1326080,1326592) | 20260506 12:23:28.099 | INFO | MemInfo: { | 20260506 12:23:28.099 | INFO | "gpu" : " 17 GB", | 20260506 12:23:28.099 | INFO | "huge" : "", | 20260506 12:23:28.099 | INFO | "huge_shared" : " 27 GB", | 20260506 12:23:28.099 | INFO | "malloc" : " 14 GB", | 20260506 12:23:28.099 | INFO | "pinned" : " 29 GB" | 20260506 12:23:28.099 | INFO | } | 20260506 12:23:28.099 | INFO | Perf report for the last 5.13981 seconds: | 20260506 12:23:28.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:23:28.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:28.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:28.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:28.099 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 12:23:28.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:28.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:28.099 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:23:28.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:28.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:28.099 | INFO | Uploaded 36GiB to GPU in 1.98852s (18.1039GiB/s) | 20260506 12:23:28.099 | INFO | Downloaded 34GiB to GPU in 2.01951s (16.8357GiB/s) | 20260506 12:23:33.238 | INFO | Analyzing chunk frames = [1326592,1327104) | 20260506 12:23:33.238 | INFO | MemInfo: { | 20260506 12:23:33.238 | INFO | "gpu" : " 17 GB", | 20260506 12:23:33.238 | INFO | "huge" : "", | 20260506 12:23:33.238 | INFO | "huge_shared" : " 27 GB", | 20260506 12:23:33.238 | INFO | "malloc" : " 14 GB", | 20260506 12:23:33.238 | INFO | "pinned" : " 29 GB" | 20260506 12:23:33.238 | INFO | } | 20260506 12:23:33.238 | INFO | Perf report for the last 5.13882 seconds: | 20260506 12:23:33.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:23:33.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:33.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:33.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:33.238 | INFO | ANALYSIS : 62.9%, 3.0, 98.0ms, 0 | 20260506 12:23:33.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:33.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:33.238 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:23:33.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:33.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:33.238 | INFO | Uploaded 36GiB to GPU in 2.07642s (17.3375GiB/s) | 20260506 12:23:33.238 | INFO | Downloaded 34GiB to GPU in 1.98286s (17.1469GiB/s) | 20260506 12:23:33.238 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:23:33.238 | INFO | Total mem usage: 16.8 GiB | 20260506 12:23:33.238 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:23:33.238 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.6%) | 20260506 12:23:33.238 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:23:33.238 | INFO | High Water Breakdown: | 20260506 12:23:33.238 | INFO | 59.6% : 8.59 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:23:33.238 | INFO | 40.2% : 5.8 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:23:33.238 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:23:33.238 | INFO | ------------------------------------------ | 20260506 12:23:33.238 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:23:33.238 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:23:33.238 | INFO | Total mem usage: 26.5 GiB | 20260506 12:23:33.238 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:23:33.238 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:23:33.238 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:23:33.238 | INFO | High Water Breakdown: | 20260506 12:23:33.238 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:23:33.238 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:23:33.238 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:23:33.238 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:23:33.238 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:23:33.238 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:23:33.238 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:23:33.238 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:23:33.238 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:23:33.238 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:23:33.238 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:23:33.238 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:23:33.238 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:23:33.238 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:23:33.238 | INFO | ------------------------------------------ | 20260506 12:23:33.238 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:23:33.238 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:23:33.238 | INFO | Total mem usage: 25.6 GiB | 20260506 12:23:33.238 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:23:33.238 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:23:33.238 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:23:33.238 | INFO | High Water Breakdown: | 20260506 12:23:33.238 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:23:33.238 | INFO | ------------------------------------------ | 20260506 12:23:33.238 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:23:33.238 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:23:33.238 | INFO | Total mem usage: 15.7 GiB | 20260506 12:23:33.238 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:23:33.238 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:23:33.238 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:23:33.238 | INFO | High Water Breakdown: | 20260506 12:23:33.238 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:23:33.238 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:23:33.238 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:23:33.238 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:23:33.238 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:23:33.238 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:23:33.238 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:23:33.238 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:23:33.238 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:23:33.238 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:23:33.238 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:23:33.239 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:23:33.239 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:23:33.239 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:23:33.239 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:23:33.239 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:23:33.239 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:23:33.239 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:23:33.239 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:23:33.239 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:23:33.239 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:23:33.239 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:23:33.239 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:23:33.239 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:23:33.239 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:23:33.239 | INFO | ------------------------------------------ | 20260506 12:23:33.239 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:23:33.239 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:23:33.239 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:23:33.239 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:23:33.239 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.60e+01, 1.37e+01 / 2.22e+00 | 20260506 12:23:33.239 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 9.96e-02, 1.01e-02 / 5.98e-03 | 20260506 12:23:33.239 | INFO | PerfReport: 11.58%, Baseline -- 3.68e+00 / 4.40e+00, 5.85e-01 / 4.02e-01 | 20260506 12:23:33.239 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.45e+00, 1.94e-01 / 1.50e-01 | 20260506 12:23:33.239 | INFO | PerfReport: 15.83%, DME -- 4.02e+01 / 4.46e+01, 4.18e+00 / 2.52e+00 | 20260506 12:23:33.239 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.79e+01 / 2.03e+01, 1.42e+00 / 1.71e+00 | 20260506 12:23:33.239 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.41e+00 / 3.03e+00, 4.16e-01 / 2.31e-01 | 20260506 12:23:33.239 | INFO | PerfReport: 3.76%, Metrics -- 1.20e+00 / 1.57e+00, 3.02e-01 / 1.62e-01 | 20260506 12:23:33.239 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.21e-01 / 4.78e-01, 6.58e-02 / 2.80e-02 | 20260506 12:23:33.239 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:23:33.239 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:23:33.239 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:23:33.239 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.47e+01 / 0.00e+00 | 20260506 12:23:33.239 | INFO | PerfReport: 21.53%, Upload -- 2.13e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260506 12:23:33.239 | INFO | PerfReport: 20.99%, Download -- 2.08e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 12:23:33.239 | INFO | PerfReport: 32.13%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 12:23:33.239 | INFO | PerfReport: 1.91%, LockUpload -- 1.89e+00 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260506 12:23:33.239 | INFO | PerfReport: 0.97%, LockDownload -- 9.55e-01 / 0.00e+00, 2.52e+00 / 0.00e+00 | 20260506 12:23:33.239 | INFO | PerfReport: 22.46%, LockCompute -- 2.22e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:23:33.239 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260506 12:23:33.361 | INFO | Starting a round of Baz IO | 20260506 12:23:36.532 | INFO | Populating RTMetrics report | 20260506 12:23:36.532 | INFO | RTMetrics report populated | 20260506 12:23:36.532 | INFO | Opening tmp RTMetrics json file | 20260506 12:23:36.533 | INFO | Serializing RTMetrics to json | 20260506 12:23:36.534 | INFO | Serialization done, swapping tmp file now... | 20260506 12:23:36.535 | INFO | RTMetrics done | 20260506 12:23:36.630 | INFO | Finished a round of Baz IO: Wrote 2.085 GiB in 3.269 seconds | 20260506 12:23:36.630 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.70% / 0.03% / 58.28% | 20260506 12:23:36.630 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.01392% / 5.16574e-05% / 4.37101e-05% | 20260506 12:23:36.630 | INFO | Baz IO overall: | 20260506 12:23:36.630 | INFO | BytesWritten : 657873439269 | 20260506 12:23:36.630 | INFO | NumEvents : 317903976524 | 20260506 12:23:38.377 | INFO | Analyzing chunk frames = [1327104,1327616) | 20260506 12:23:38.377 | INFO | MemInfo: { | 20260506 12:23:38.377 | INFO | "gpu" : " 17 GB", | 20260506 12:23:38.377 | INFO | "huge" : "", | 20260506 12:23:38.377 | INFO | "huge_shared" : " 27 GB", | 20260506 12:23:38.377 | INFO | "malloc" : " 14 GB", | 20260506 12:23:38.377 | INFO | "pinned" : " 29 GB" | 20260506 12:23:38.377 | INFO | } | 20260506 12:23:38.377 | INFO | Perf report for the last 5.13903 seconds: | 20260506 12:23:38.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:23:38.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:38.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:38.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:38.377 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260506 12:23:38.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:38.377 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:23:38.377 | INFO | PRE_HQ : 61.5%, 1.9, 61.2ms, 0 | 20260506 12:23:38.377 | INFO | BAZWRITER : 62.8%, 3.7, 118.2ms, 0 | 20260506 12:23:38.377 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:23:38.377 | INFO | Uploaded 36GiB to GPU in 2.0846s (17.2695GiB/s) | 20260506 12:23:38.377 | INFO | Downloaded 38GiB to GPU in 2.20678s (17.2197GiB/s) | 20260506 12:23:43.516 | INFO | Analyzing chunk frames = [1327616,1328128) | 20260506 12:23:43.516 | INFO | MemInfo: { | 20260506 12:23:43.516 | INFO | "gpu" : " 17 GB", | 20260506 12:23:43.516 | INFO | "huge" : "", | 20260506 12:23:43.516 | INFO | "huge_shared" : " 27 GB", | 20260506 12:23:43.516 | INFO | "malloc" : " 14 GB", | 20260506 12:23:43.516 | INFO | "pinned" : " 29 GB" | 20260506 12:23:43.516 | INFO | } | 20260506 12:23:43.516 | INFO | Perf report for the last 5.13974 seconds: | 20260506 12:23:43.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:23:43.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:43.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:43.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:43.516 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:23:43.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:43.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:43.517 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:23:43.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:43.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:43.517 | INFO | Uploaded 36GiB to GPU in 2.01929s (17.8281GiB/s) | 20260506 12:23:43.517 | INFO | Downloaded 34GiB to GPU in 1.9169s (17.7369GiB/s) | 20260506 12:23:48.656 | INFO | Analyzing chunk frames = [1328128,1328640) | 20260506 12:23:48.656 | INFO | MemInfo: { | 20260506 12:23:48.656 | INFO | "gpu" : " 17 GB", | 20260506 12:23:48.656 | INFO | "huge" : "", | 20260506 12:23:48.656 | INFO | "huge_shared" : " 27 GB", | 20260506 12:23:48.656 | INFO | "malloc" : " 14 GB", | 20260506 12:23:48.656 | INFO | "pinned" : " 29 GB" | 20260506 12:23:48.656 | INFO | } | 20260506 12:23:48.656 | INFO | Perf report for the last 5.13926 seconds: | 20260506 12:23:48.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:23:48.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:48.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:48.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:48.656 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 12:23:48.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:48.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:48.656 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:23:48.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:48.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:48.656 | INFO | Uploaded 36GiB to GPU in 2.02519s (17.7761GiB/s) | 20260506 12:23:48.656 | INFO | Downloaded 34GiB to GPU in 1.9405s (17.5212GiB/s) | 20260506 12:23:53.795 | INFO | Analyzing chunk frames = [1328640,1329152) | 20260506 12:23:53.795 | INFO | MemInfo: { | 20260506 12:23:53.795 | INFO | "gpu" : " 17 GB", | 20260506 12:23:53.795 | INFO | "huge" : "", | 20260506 12:23:53.795 | INFO | "huge_shared" : " 27 GB", | 20260506 12:23:53.795 | INFO | "malloc" : " 14 GB", | 20260506 12:23:53.795 | INFO | "pinned" : " 29 GB" | 20260506 12:23:53.795 | INFO | } | 20260506 12:23:53.795 | INFO | Perf report for the last 5.13926 seconds: | 20260506 12:23:53.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:23:53.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:53.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:53.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:53.795 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 12:23:53.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:53.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:53.795 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:23:53.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:53.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:53.795 | INFO | Uploaded 36GiB to GPU in 2.01124s (17.8994GiB/s) | 20260506 12:23:53.795 | INFO | Downloaded 34GiB to GPU in 1.92782s (17.6365GiB/s) | 20260506 12:23:58.934 | INFO | Analyzing chunk frames = [1329152,1329664) | 20260506 12:23:58.934 | INFO | MemInfo: { | 20260506 12:23:58.934 | INFO | "gpu" : " 17 GB", | 20260506 12:23:58.934 | INFO | "huge" : "", | 20260506 12:23:58.935 | INFO | "huge_shared" : " 27 GB", | 20260506 12:23:58.935 | INFO | "malloc" : " 14 GB", | 20260506 12:23:58.935 | INFO | "pinned" : " 29 GB" | 20260506 12:23:58.935 | INFO | } | 20260506 12:23:58.935 | INFO | Perf report for the last 5.13958 seconds: | 20260506 12:23:58.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:23:58.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:58.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:58.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:58.935 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 12:23:58.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:58.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:58.935 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:23:58.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:58.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:23:58.935 | INFO | Uploaded 36GiB to GPU in 1.96161s (18.3522GiB/s) | 20260506 12:23:58.935 | INFO | Downloaded 34GiB to GPU in 1.93852s (17.5392GiB/s) | 20260506 12:24:04.073 | INFO | Analyzing chunk frames = [1329664,1330176) | 20260506 12:24:04.073 | INFO | MemInfo: { | 20260506 12:24:04.073 | INFO | "gpu" : " 17 GB", | 20260506 12:24:04.074 | INFO | "huge" : "", | 20260506 12:24:04.074 | INFO | "huge_shared" : " 27 GB", | 20260506 12:24:04.074 | INFO | "malloc" : " 14 GB", | 20260506 12:24:04.074 | INFO | "pinned" : " 29 GB" | 20260506 12:24:04.074 | INFO | } | 20260506 12:24:04.074 | INFO | Perf report for the last 5.13904 seconds: | 20260506 12:24:04.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:24:04.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:04.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:04.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:04.074 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 12:24:04.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:04.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:04.074 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260506 12:24:04.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:04.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:04.074 | INFO | Uploaded 36GiB to GPU in 2.05019s (17.5593GiB/s) | 20260506 12:24:04.074 | INFO | Downloaded 34GiB to GPU in 1.92744s (17.64GiB/s) | 20260506 12:24:09.214 | INFO | Analyzing chunk frames = [1330176,1330688) | 20260506 12:24:09.214 | INFO | MemInfo: { | 20260506 12:24:09.214 | INFO | "gpu" : " 17 GB", | 20260506 12:24:09.214 | INFO | "huge" : "", | 20260506 12:24:09.214 | INFO | "huge_shared" : " 27 GB", | 20260506 12:24:09.214 | INFO | "malloc" : " 14 GB", | 20260506 12:24:09.214 | INFO | "pinned" : " 29 GB" | 20260506 12:24:09.214 | INFO | } | 20260506 12:24:09.214 | INFO | Perf report for the last 5.14008 seconds: | 20260506 12:24:09.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:24:09.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:09.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:09.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:09.214 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 12:24:09.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:09.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:09.214 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 12:24:09.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:09.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:09.214 | INFO | Uploaded 36GiB to GPU in 1.9928s (18.0651GiB/s) | 20260506 12:24:09.214 | INFO | Downloaded 34GiB to GPU in 2.03281s (16.7256GiB/s) | 20260506 12:24:14.353 | INFO | Analyzing chunk frames = [1330688,1331200) | 20260506 12:24:14.353 | INFO | MemInfo: { | 20260506 12:24:14.353 | INFO | "gpu" : " 17 GB", | 20260506 12:24:14.353 | INFO | "huge" : "", | 20260506 12:24:14.353 | INFO | "huge_shared" : " 27 GB", | 20260506 12:24:14.353 | INFO | "malloc" : " 14 GB", | 20260506 12:24:14.353 | INFO | "pinned" : " 29 GB" | 20260506 12:24:14.353 | INFO | } | 20260506 12:24:14.353 | INFO | Perf report for the last 5.1391 seconds: | 20260506 12:24:14.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:24:14.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:14.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:14.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:14.353 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 12:24:14.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:14.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:14.353 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:24:14.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:14.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:14.353 | INFO | Uploaded 36GiB to GPU in 2.02684s (17.7617GiB/s) | 20260506 12:24:14.353 | INFO | Downloaded 34GiB to GPU in 1.96722s (17.2832GiB/s) | 20260506 12:24:14.485 | INFO | Starting a round of Baz IO | 20260506 12:24:17.646 | INFO | Populating RTMetrics report | 20260506 12:24:17.646 | INFO | RTMetrics report populated | 20260506 12:24:17.646 | INFO | Opening tmp RTMetrics json file | 20260506 12:24:17.647 | INFO | Serializing RTMetrics to json | 20260506 12:24:17.648 | INFO | Serialization done, swapping tmp file now... | 20260506 12:24:17.648 | INFO | RTMetrics done | 20260506 12:24:17.745 | INFO | Finished a round of Baz IO: Wrote 2.089 GiB in 3.26 seconds | 20260506 12:24:17.745 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.67% / 0.03% / 58.30% | 20260506 12:24:17.745 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.0229% / 4.76837e-05% / 4.37101e-05% | 20260506 12:24:17.745 | INFO | Baz IO overall: | 20260506 12:24:17.745 | INFO | BytesWritten : 660116987741 | 20260506 12:24:17.745 | INFO | NumEvents : 318993262731 | 20260506 12:24:19.492 | INFO | Analyzing chunk frames = [1331200,1331712) | 20260506 12:24:19.492 | INFO | MemInfo: { | 20260506 12:24:19.492 | INFO | "gpu" : " 17 GB", | 20260506 12:24:19.492 | INFO | "huge" : "", | 20260506 12:24:19.492 | INFO | "huge_shared" : " 27 GB", | 20260506 12:24:19.492 | INFO | "malloc" : " 14 GB", | 20260506 12:24:19.492 | INFO | "pinned" : " 29 GB" | 20260506 12:24:19.492 | INFO | } | 20260506 12:24:19.492 | INFO | Perf report for the last 5.13882 seconds: | 20260506 12:24:19.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:24:19.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:19.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:19.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:19.492 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260506 12:24:19.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:19.492 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:24:19.492 | INFO | PRE_HQ : 61.5%, 1.9, 61.2ms, 0 | 20260506 12:24:19.492 | INFO | BAZWRITER : 62.8%, 3.6, 117.6ms, 0 | 20260506 12:24:19.492 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260506 12:24:19.492 | INFO | Uploaded 36GiB to GPU in 2.04816s (17.5768GiB/s) | 20260506 12:24:19.492 | INFO | Downloaded 38GiB to GPU in 2.22541s (17.0755GiB/s) | 20260506 12:24:24.631 | INFO | Analyzing chunk frames = [1331712,1332224) | 20260506 12:24:24.631 | INFO | MemInfo: { | 20260506 12:24:24.631 | INFO | "gpu" : " 17 GB", | 20260506 12:24:24.631 | INFO | "huge" : "", | 20260506 12:24:24.631 | INFO | "huge_shared" : " 27 GB", | 20260506 12:24:24.631 | INFO | "malloc" : " 14 GB", | 20260506 12:24:24.631 | INFO | "pinned" : " 29 GB" | 20260506 12:24:24.631 | INFO | } | 20260506 12:24:24.631 | INFO | Perf report for the last 5.13942 seconds: | 20260506 12:24:24.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:24:24.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:24.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:24.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:24.631 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:24:24.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:24.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:24.631 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:24:24.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:24.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:24.631 | INFO | Uploaded 36GiB to GPU in 2.02331s (17.7926GiB/s) | 20260506 12:24:24.631 | INFO | Downloaded 34GiB to GPU in 1.93433s (17.5771GiB/s) | 20260506 12:24:29.770 | INFO | Analyzing chunk frames = [1332224,1332736) | 20260506 12:24:29.770 | INFO | MemInfo: { | 20260506 12:24:29.770 | INFO | "gpu" : " 17 GB", | 20260506 12:24:29.770 | INFO | "huge" : "", | 20260506 12:24:29.770 | INFO | "huge_shared" : " 27 GB", | 20260506 12:24:29.770 | INFO | "malloc" : " 14 GB", | 20260506 12:24:29.770 | INFO | "pinned" : " 29 GB" | 20260506 12:24:29.770 | INFO | } | 20260506 12:24:29.770 | INFO | Perf report for the last 5.13924 seconds: | 20260506 12:24:29.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:24:29.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:29.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:29.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:29.770 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:24:29.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:29.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:29.770 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:24:29.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:29.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:29.771 | INFO | Uploaded 36GiB to GPU in 2.02883s (17.7443GiB/s) | 20260506 12:24:29.771 | INFO | Downloaded 34GiB to GPU in 1.94361s (17.4932GiB/s) | 20260506 12:24:34.910 | INFO | Analyzing chunk frames = [1332736,1333248) | 20260506 12:24:34.910 | INFO | MemInfo: { | 20260506 12:24:34.910 | INFO | "gpu" : " 17 GB", | 20260506 12:24:34.910 | INFO | "huge" : "", | 20260506 12:24:34.910 | INFO | "huge_shared" : " 27 GB", | 20260506 12:24:34.910 | INFO | "malloc" : " 14 GB", | 20260506 12:24:34.910 | INFO | "pinned" : " 29 GB" | 20260506 12:24:34.910 | INFO | } | 20260506 12:24:34.910 | INFO | Perf report for the last 5.13935 seconds: | 20260506 12:24:34.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:24:34.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:34.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:34.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:34.910 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 12:24:34.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:34.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:34.910 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:24:34.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:34.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:34.910 | INFO | Uploaded 36GiB to GPU in 2.00674s (17.9395GiB/s) | 20260506 12:24:34.910 | INFO | Downloaded 34GiB to GPU in 1.95428s (17.3977GiB/s) | 20260506 12:24:40.049 | INFO | Analyzing chunk frames = [1333248,1333760) | 20260506 12:24:40.049 | INFO | MemInfo: { | 20260506 12:24:40.049 | INFO | "gpu" : " 17 GB", | 20260506 12:24:40.049 | INFO | "huge" : "", | 20260506 12:24:40.049 | INFO | "huge_shared" : " 27 GB", | 20260506 12:24:40.049 | INFO | "malloc" : " 14 GB", | 20260506 12:24:40.049 | INFO | "pinned" : " 29 GB" | 20260506 12:24:40.049 | INFO | } | 20260506 12:24:40.049 | INFO | Perf report for the last 5.13926 seconds: | 20260506 12:24:40.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:24:40.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:40.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:40.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:40.049 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 12:24:40.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:40.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:40.049 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260506 12:24:40.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:40.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:40.049 | INFO | Uploaded 36GiB to GPU in 2.01593s (17.8578GiB/s) | 20260506 12:24:40.049 | INFO | Downloaded 34GiB to GPU in 1.95293s (17.4097GiB/s) | 20260506 12:24:45.188 | INFO | Analyzing chunk frames = [1333760,1334272) | 20260506 12:24:45.188 | INFO | MemInfo: { | 20260506 12:24:45.188 | INFO | "gpu" : " 17 GB", | 20260506 12:24:45.188 | INFO | "huge" : "", | 20260506 12:24:45.188 | INFO | "huge_shared" : " 27 GB", | 20260506 12:24:45.188 | INFO | "malloc" : " 14 GB", | 20260506 12:24:45.188 | INFO | "pinned" : " 29 GB" | 20260506 12:24:45.188 | INFO | } | 20260506 12:24:45.188 | INFO | Perf report for the last 5.13944 seconds: | 20260506 12:24:45.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:24:45.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:45.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:45.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:45.188 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:24:45.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:45.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:45.189 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:24:45.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:45.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:45.189 | INFO | Uploaded 36GiB to GPU in 2.10799s (17.0779GiB/s) | 20260506 12:24:45.189 | INFO | Downloaded 34GiB to GPU in 1.98746s (17.1073GiB/s) | 20260506 12:24:50.329 | INFO | Analyzing chunk frames = [1334272,1334784) | 20260506 12:24:50.329 | INFO | MemInfo: { | 20260506 12:24:50.329 | INFO | "gpu" : " 17 GB", | 20260506 12:24:50.329 | INFO | "huge" : "", | 20260506 12:24:50.329 | INFO | "huge_shared" : " 27 GB", | 20260506 12:24:50.329 | INFO | "malloc" : " 14 GB", | 20260506 12:24:50.329 | INFO | "pinned" : " 29 GB" | 20260506 12:24:50.329 | INFO | } | 20260506 12:24:50.329 | INFO | Perf report for the last 5.1405 seconds: | 20260506 12:24:50.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:24:50.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:50.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:50.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:50.329 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:24:50.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:50.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:50.329 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:24:50.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:50.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:50.329 | INFO | Uploaded 36GiB to GPU in 2.01492s (17.8667GiB/s) | 20260506 12:24:50.329 | INFO | Downloaded 34GiB to GPU in 2.0109s (16.9079GiB/s) | 20260506 12:24:55.467 | INFO | Analyzing chunk frames = [1334784,1335296) | 20260506 12:24:55.467 | INFO | MemInfo: { | 20260506 12:24:55.467 | INFO | "gpu" : " 17 GB", | 20260506 12:24:55.467 | INFO | "huge" : "", | 20260506 12:24:55.467 | INFO | "huge_shared" : " 27 GB", | 20260506 12:24:55.467 | INFO | "malloc" : " 14 GB", | 20260506 12:24:55.467 | INFO | "pinned" : " 29 GB" | 20260506 12:24:55.467 | INFO | } | 20260506 12:24:55.467 | INFO | Perf report for the last 5.13818 seconds: | 20260506 12:24:55.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:24:55.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:55.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:55.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:55.467 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260506 12:24:55.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:55.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:55.467 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260506 12:24:55.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:55.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:24:55.467 | INFO | Uploaded 36GiB to GPU in 2.03392s (17.6998GiB/s) | 20260506 12:24:55.467 | INFO | Downloaded 34GiB to GPU in 1.96522s (17.3009GiB/s) | 20260506 12:24:55.467 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:24:55.467 | INFO | Total mem usage: 16.8 GiB | 20260506 12:24:55.467 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:24:55.467 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.6%) | 20260506 12:24:55.467 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:24:55.467 | INFO | High Water Breakdown: | 20260506 12:24:55.468 | INFO | 59.6% : 8.6 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:24:55.468 | INFO | 40.2% : 5.8 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:24:55.468 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:24:55.468 | INFO | ------------------------------------------ | 20260506 12:24:55.468 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:24:55.468 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:24:55.468 | INFO | Total mem usage: 26.5 GiB | 20260506 12:24:55.468 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:24:55.468 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:24:55.468 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:24:55.468 | INFO | High Water Breakdown: | 20260506 12:24:55.468 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:24:55.468 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:24:55.468 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:24:55.468 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:24:55.468 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:24:55.468 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:24:55.468 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:24:55.468 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:24:55.468 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:24:55.468 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:24:55.468 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:24:55.468 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:24:55.468 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:24:55.468 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:24:55.468 | INFO | ------------------------------------------ | 20260506 12:24:55.468 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:24:55.468 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:24:55.468 | INFO | Total mem usage: 25.6 GiB | 20260506 12:24:55.468 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:24:55.468 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:24:55.468 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:24:55.468 | INFO | High Water Breakdown: | 20260506 12:24:55.468 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:24:55.468 | INFO | ------------------------------------------ | 20260506 12:24:55.468 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:24:55.468 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:24:55.468 | INFO | Total mem usage: 15.7 GiB | 20260506 12:24:55.468 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:24:55.468 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:24:55.468 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:24:55.468 | INFO | High Water Breakdown: | 20260506 12:24:55.468 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:24:55.468 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:24:55.468 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:24:55.468 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:24:55.468 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:24:55.468 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:24:55.468 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:24:55.468 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:24:55.468 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:24:55.468 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:24:55.468 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:24:55.468 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:24:55.468 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:24:55.468 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:24:55.468 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:24:55.468 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:24:55.468 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:24:55.468 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:24:55.468 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:24:55.468 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:24:55.468 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:24:55.468 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:24:55.468 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:24:55.468 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:24:55.468 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:24:55.468 | INFO | ------------------------------------------ | 20260506 12:24:55.468 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:24:55.468 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:24:55.468 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:24:55.468 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:24:55.468 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.64e+01, 1.36e+01 / 2.77e+00 | 20260506 12:24:55.468 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 9.99e-02, 9.95e-03 / 7.04e-03 | 20260506 12:24:55.468 | INFO | PerfReport: 11.56%, Baseline -- 3.67e+00 / 4.45e+00, 5.82e-01 / 3.57e-01 | 20260506 12:24:55.468 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.48e+00, 1.95e-01 / 1.37e-01 | 20260506 12:24:55.468 | INFO | PerfReport: 15.87%, DME -- 4.04e+01 / 4.51e+01, 4.33e+00 / 3.10e+00 | 20260506 12:24:55.468 | INFO | PerfReport: 56.37%, FrameLabeling -- 1.79e+01 / 2.01e+01, 1.42e+00 / 1.83e+00 | 20260506 12:24:55.468 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.41e+00 / 3.05e+00, 4.19e-01 / 2.32e-01 | 20260506 12:24:55.468 | INFO | PerfReport: 3.75%, Metrics -- 1.19e+00 / 1.59e+00, 3.03e-01 / 1.69e-01 | 20260506 12:24:55.468 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.21e-01 / 4.78e-01, 6.53e-02 / 2.67e-02 | 20260506 12:24:55.468 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:24:55.468 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:24:55.468 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:24:55.468 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 12:24:55.468 | INFO | PerfReport: 21.38%, Upload -- 2.11e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260506 12:24:55.468 | INFO | PerfReport: 20.97%, Download -- 2.07e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 12:24:55.468 | INFO | PerfReport: 32.17%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 12:24:55.468 | INFO | PerfReport: 1.76%, LockUpload -- 1.74e+00 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 12:24:55.468 | INFO | PerfReport: 1.07%, LockDownload -- 1.06e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260506 12:24:55.468 | INFO | PerfReport: 22.62%, LockCompute -- 2.24e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 12:24:55.468 | INFO | PerfReport: 0.02%, Misc -- 1.50e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260506 12:24:55.597 | INFO | Starting a round of Baz IO | 20260506 12:24:58.768 | INFO | Populating RTMetrics report | 20260506 12:24:58.768 | INFO | RTMetrics report populated | 20260506 12:24:58.768 | INFO | Opening tmp RTMetrics json file | 20260506 12:24:58.769 | INFO | Serializing RTMetrics to json | 20260506 12:24:58.770 | INFO | Serialization done, swapping tmp file now... | 20260506 12:24:58.771 | INFO | RTMetrics done | 20260506 12:24:58.868 | INFO | Finished a round of Baz IO: Wrote 2.088 GiB in 3.271 seconds | 20260506 12:24:58.868 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.65% / 0.03% / 58.33% | 20260506 12:24:58.868 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 4.00258% / 5.96046e-05% / 2.78155e-05% | 20260506 12:24:58.868 | INFO | Baz IO overall: | 20260506 12:24:58.868 | INFO | BytesWritten : 662358916667 | 20260506 12:24:58.868 | INFO | NumEvents : 320080963057 | 20260506 12:25:00.606 | INFO | Analyzing chunk frames = [1335296,1335808) | 20260506 12:25:00.606 | INFO | MemInfo: { | 20260506 12:25:00.606 | INFO | "gpu" : " 17 GB", | 20260506 12:25:00.606 | INFO | "huge" : "", | 20260506 12:25:00.606 | INFO | "huge_shared" : " 27 GB", | 20260506 12:25:00.606 | INFO | "malloc" : " 14 GB", | 20260506 12:25:00.606 | INFO | "pinned" : " 29 GB" | 20260506 12:25:00.606 | INFO | } | 20260506 12:25:00.606 | INFO | Perf report for the last 5.13917 seconds: | 20260506 12:25:00.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:25:00.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:00.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:00.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:00.607 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260506 12:25:00.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:00.607 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:25:00.607 | INFO | PRE_HQ : 61.5%, 1.9, 61.7ms, 0 | 20260506 12:25:00.607 | INFO | BAZWRITER : 63.0%, 3.7, 118.2ms, 0 | 20260506 12:25:00.607 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 12:25:00.607 | INFO | Uploaded 36GiB to GPU in 2.05769s (17.4953GiB/s) | 20260506 12:25:00.607 | INFO | Downloaded 38GiB to GPU in 2.21688s (17.1412GiB/s) | 20260506 12:25:05.746 | INFO | Analyzing chunk frames = [1335808,1336320) | 20260506 12:25:05.746 | INFO | MemInfo: { | 20260506 12:25:05.746 | INFO | "gpu" : " 17 GB", | 20260506 12:25:05.746 | INFO | "huge" : "", | 20260506 12:25:05.746 | INFO | "huge_shared" : " 27 GB", | 20260506 12:25:05.746 | INFO | "malloc" : " 14 GB", | 20260506 12:25:05.746 | INFO | "pinned" : " 29 GB" | 20260506 12:25:05.746 | INFO | } | 20260506 12:25:05.746 | INFO | Perf report for the last 5.13942 seconds: | 20260506 12:25:05.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:25:05.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:05.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:05.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:05.746 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:25:05.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:05.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:05.746 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:25:05.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:05.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:05.746 | INFO | Uploaded 36GiB to GPU in 2.06393s (17.4425GiB/s) | 20260506 12:25:05.746 | INFO | Downloaded 34GiB to GPU in 1.93617s (17.5605GiB/s) | 20260506 12:25:10.885 | INFO | Analyzing chunk frames = [1336320,1336832) | 20260506 12:25:10.885 | INFO | MemInfo: { | 20260506 12:25:10.885 | INFO | "gpu" : " 17 GB", | 20260506 12:25:10.885 | INFO | "huge" : "", | 20260506 12:25:10.885 | INFO | "huge_shared" : " 27 GB", | 20260506 12:25:10.885 | INFO | "malloc" : " 14 GB", | 20260506 12:25:10.885 | INFO | "pinned" : " 29 GB" | 20260506 12:25:10.885 | INFO | } | 20260506 12:25:10.885 | INFO | Perf report for the last 5.1395 seconds: | 20260506 12:25:10.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:25:10.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:10.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:10.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:10.885 | INFO | ANALYSIS : 62.2%, 3.0, 96.3ms, 0 | 20260506 12:25:10.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:10.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:10.885 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:25:10.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:10.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:10.886 | INFO | Uploaded 36GiB to GPU in 2.01172s (17.8951GiB/s) | 20260506 12:25:10.886 | INFO | Downloaded 34GiB to GPU in 1.95053s (17.4312GiB/s) | 20260506 12:25:16.024 | INFO | Analyzing chunk frames = [1336832,1337344) | 20260506 12:25:16.024 | INFO | MemInfo: { | 20260506 12:25:16.024 | INFO | "gpu" : " 17 GB", | 20260506 12:25:16.024 | INFO | "huge" : "", | 20260506 12:25:16.024 | INFO | "huge_shared" : " 27 GB", | 20260506 12:25:16.024 | INFO | "malloc" : " 14 GB", | 20260506 12:25:16.025 | INFO | "pinned" : " 29 GB" | 20260506 12:25:16.025 | INFO | } | 20260506 12:25:16.025 | INFO | Perf report for the last 5.1393 seconds: | 20260506 12:25:16.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:25:16.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:16.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:16.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:16.025 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 12:25:16.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:16.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:16.025 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:25:16.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:16.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:16.025 | INFO | Uploaded 36GiB to GPU in 2.00627s (17.9438GiB/s) | 20260506 12:25:16.025 | INFO | Downloaded 34GiB to GPU in 1.94141s (17.5131GiB/s) | 20260506 12:25:21.166 | INFO | Analyzing chunk frames = [1337344,1337856) | 20260506 12:25:21.166 | INFO | MemInfo: { | 20260506 12:25:21.166 | INFO | "gpu" : " 17 GB", | 20260506 12:25:21.166 | INFO | "huge" : "", | 20260506 12:25:21.166 | INFO | "huge_shared" : " 27 GB", | 20260506 12:25:21.166 | INFO | "malloc" : " 14 GB", | 20260506 12:25:21.166 | INFO | "pinned" : " 29 GB" | 20260506 12:25:21.166 | INFO | } | 20260506 12:25:21.166 | INFO | Perf report for the last 5.14175 seconds: | 20260506 12:25:21.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:25:21.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:21.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:21.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:21.166 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 12:25:21.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:21.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:21.167 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 12:25:21.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:21.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:21.167 | INFO | Uploaded 36GiB to GPU in 2.03258s (17.7115GiB/s) | 20260506 12:25:21.167 | INFO | Downloaded 34GiB to GPU in 1.94006s (17.5252GiB/s) | 20260506 12:25:26.305 | INFO | Analyzing chunk frames = [1337856,1338368) | 20260506 12:25:26.305 | INFO | MemInfo: { | 20260506 12:25:26.305 | INFO | "gpu" : " 17 GB", | 20260506 12:25:26.305 | INFO | "huge" : "", | 20260506 12:25:26.305 | INFO | "huge_shared" : " 27 GB", | 20260506 12:25:26.305 | INFO | "malloc" : " 14 GB", | 20260506 12:25:26.305 | INFO | "pinned" : " 29 GB" | 20260506 12:25:26.305 | INFO | } | 20260506 12:25:26.306 | INFO | Perf report for the last 5.13914 seconds: | 20260506 12:25:26.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:25:26.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:26.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:26.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:26.306 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:25:26.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:26.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:26.306 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:25:26.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:26.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:26.306 | INFO | Uploaded 36GiB to GPU in 2.08497s (17.2665GiB/s) | 20260506 12:25:26.306 | INFO | Downloaded 34GiB to GPU in 1.96749s (17.2809GiB/s) | 20260506 12:25:31.443 | INFO | Analyzing chunk frames = [1338368,1338880) | 20260506 12:25:31.443 | INFO | MemInfo: { | 20260506 12:25:31.443 | INFO | "gpu" : " 17 GB", | 20260506 12:25:31.443 | INFO | "huge" : "", | 20260506 12:25:31.443 | INFO | "huge_shared" : " 27 GB", | 20260506 12:25:31.443 | INFO | "malloc" : " 14 GB", | 20260506 12:25:31.443 | INFO | "pinned" : " 29 GB" | 20260506 12:25:31.443 | INFO | } | 20260506 12:25:31.443 | INFO | Perf report for the last 5.13759 seconds: | 20260506 12:25:31.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:25:31.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:31.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:31.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:31.443 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:25:31.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:31.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:31.443 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260506 12:25:31.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:31.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:31.443 | INFO | Uploaded 36GiB to GPU in 2.05529s (17.5158GiB/s) | 20260506 12:25:31.443 | INFO | Downloaded 34GiB to GPU in 2.00768s (16.935GiB/s) | 20260506 12:25:36.582 | INFO | Analyzing chunk frames = [1338880,1339392) | 20260506 12:25:36.582 | INFO | MemInfo: { | 20260506 12:25:36.582 | INFO | "gpu" : " 17 GB", | 20260506 12:25:36.582 | INFO | "huge" : "", | 20260506 12:25:36.582 | INFO | "huge_shared" : " 27 GB", | 20260506 12:25:36.582 | INFO | "malloc" : " 14 GB", | 20260506 12:25:36.582 | INFO | "pinned" : " 29 GB" | 20260506 12:25:36.582 | INFO | } | 20260506 12:25:36.582 | INFO | Perf report for the last 5.13896 seconds: | 20260506 12:25:36.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:25:36.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:36.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:36.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:36.582 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260506 12:25:36.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:36.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:36.582 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 12:25:36.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:36.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:36.582 | INFO | Uploaded 36GiB to GPU in 2.05884s (17.4856GiB/s) | 20260506 12:25:36.582 | INFO | Downloaded 34GiB to GPU in 2.01922s (16.8382GiB/s) | 20260506 12:25:36.713 | INFO | Starting a round of Baz IO | 20260506 12:25:39.880 | INFO | Populating RTMetrics report | 20260506 12:25:39.880 | INFO | RTMetrics report populated | 20260506 12:25:39.880 | INFO | Opening tmp RTMetrics json file | 20260506 12:25:39.881 | INFO | Serializing RTMetrics to json | 20260506 12:25:39.882 | INFO | Serialization done, swapping tmp file now... | 20260506 12:25:39.882 | INFO | RTMetrics done | 20260506 12:25:39.976 | INFO | Finished a round of Baz IO: Wrote 2.088 GiB in 3.263 seconds | 20260506 12:25:39.976 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.62% / 0.02% / 58.35% | 20260506 12:25:39.976 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.99295% / 4.37101e-05% / 3.97364e-05% | 20260506 12:25:39.976 | INFO | Baz IO overall: | 20260506 12:25:39.976 | INFO | BytesWritten : 664601251607 | 20260506 12:25:39.976 | INFO | NumEvents : 321168839101 | 20260506 12:25:41.721 | INFO | Analyzing chunk frames = [1339392,1339904) | 20260506 12:25:41.721 | INFO | MemInfo: { | 20260506 12:25:41.721 | INFO | "gpu" : " 17 GB", | 20260506 12:25:41.721 | INFO | "huge" : "", | 20260506 12:25:41.721 | INFO | "huge_shared" : " 27 GB", | 20260506 12:25:41.721 | INFO | "malloc" : " 14 GB", | 20260506 12:25:41.721 | INFO | "pinned" : " 29 GB" | 20260506 12:25:41.721 | INFO | } | 20260506 12:25:41.721 | INFO | Perf report for the last 5.13921 seconds: | 20260506 12:25:41.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:25:41.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:41.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:41.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:41.721 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260506 12:25:41.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:41.721 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:25:41.721 | INFO | PRE_HQ : 61.5%, 1.9, 61.4ms, 0 | 20260506 12:25:41.721 | INFO | BAZWRITER : 62.9%, 3.6, 116.0ms, 0 | 20260506 12:25:41.721 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:25:41.722 | INFO | Uploaded 36GiB to GPU in 2.0637s (17.4444GiB/s) | 20260506 12:25:41.722 | INFO | Downloaded 38GiB to GPU in 2.21851s (17.1286GiB/s) | 20260506 12:25:46.860 | INFO | Analyzing chunk frames = [1339904,1340416) | 20260506 12:25:46.860 | INFO | MemInfo: { | 20260506 12:25:46.860 | INFO | "gpu" : " 17 GB", | 20260506 12:25:46.861 | INFO | "huge" : "", | 20260506 12:25:46.861 | INFO | "huge_shared" : " 27 GB", | 20260506 12:25:46.861 | INFO | "malloc" : " 14 GB", | 20260506 12:25:46.861 | INFO | "pinned" : " 29 GB" | 20260506 12:25:46.861 | INFO | } | 20260506 12:25:46.861 | INFO | Perf report for the last 5.13927 seconds: | 20260506 12:25:46.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:25:46.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:46.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:46.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:46.861 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:25:46.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:46.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:46.861 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:25:46.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:46.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:46.861 | INFO | Uploaded 36GiB to GPU in 2.04857s (17.5732GiB/s) | 20260506 12:25:46.861 | INFO | Downloaded 34GiB to GPU in 1.93277s (17.5914GiB/s) | 20260506 12:25:52.000 | INFO | Analyzing chunk frames = [1340416,1340928) | 20260506 12:25:52.000 | INFO | MemInfo: { | 20260506 12:25:52.000 | INFO | "gpu" : " 17 GB", | 20260506 12:25:52.000 | INFO | "huge" : "", | 20260506 12:25:52.000 | INFO | "huge_shared" : " 27 GB", | 20260506 12:25:52.000 | INFO | "malloc" : " 14 GB", | 20260506 12:25:52.000 | INFO | "pinned" : " 29 GB" | 20260506 12:25:52.000 | INFO | } | 20260506 12:25:52.000 | INFO | Perf report for the last 5.13932 seconds: | 20260506 12:25:52.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:25:52.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:52.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:52.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:52.000 | INFO | ANALYSIS : 61.9%, 3.0, 95.9ms, 0 | 20260506 12:25:52.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:52.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:52.000 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:25:52.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:52.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:52.000 | INFO | Uploaded 36GiB to GPU in 2.02213s (17.803GiB/s) | 20260506 12:25:52.000 | INFO | Downloaded 34GiB to GPU in 1.94545s (17.4767GiB/s) | 20260506 12:25:57.140 | INFO | Analyzing chunk frames = [1340928,1341440) | 20260506 12:25:57.140 | INFO | MemInfo: { | 20260506 12:25:57.140 | INFO | "gpu" : " 17 GB", | 20260506 12:25:57.140 | INFO | "huge" : "", | 20260506 12:25:57.140 | INFO | "huge_shared" : " 27 GB", | 20260506 12:25:57.140 | INFO | "malloc" : " 14 GB", | 20260506 12:25:57.140 | INFO | "pinned" : " 29 GB" | 20260506 12:25:57.140 | INFO | } | 20260506 12:25:57.140 | INFO | Perf report for the last 5.13976 seconds: | 20260506 12:25:57.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:25:57.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:57.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:57.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:57.140 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 12:25:57.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:57.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:57.140 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:25:57.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:57.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:25:57.140 | INFO | Uploaded 36GiB to GPU in 2.03133s (17.7223GiB/s) | 20260506 12:25:57.140 | INFO | Downloaded 34GiB to GPU in 1.94867s (17.4478GiB/s) | 20260506 12:26:02.278 | INFO | Analyzing chunk frames = [1341440,1341952) | 20260506 12:26:02.278 | INFO | MemInfo: { | 20260506 12:26:02.278 | INFO | "gpu" : " 17 GB", | 20260506 12:26:02.278 | INFO | "huge" : "", | 20260506 12:26:02.278 | INFO | "huge_shared" : " 27 GB", | 20260506 12:26:02.278 | INFO | "malloc" : " 14 GB", | 20260506 12:26:02.278 | INFO | "pinned" : " 29 GB" | 20260506 12:26:02.278 | INFO | } | 20260506 12:26:02.279 | INFO | Perf report for the last 5.13879 seconds: | 20260506 12:26:02.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:26:02.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:02.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:02.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:02.279 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 12:26:02.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:02.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:02.279 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:26:02.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:02.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:02.279 | INFO | Uploaded 36GiB to GPU in 1.97237s (18.2521GiB/s) | 20260506 12:26:02.279 | INFO | Downloaded 34GiB to GPU in 1.94155s (17.5118GiB/s) | 20260506 12:26:07.418 | INFO | Analyzing chunk frames = [1341952,1342464) | 20260506 12:26:07.418 | INFO | MemInfo: { | 20260506 12:26:07.418 | INFO | "gpu" : " 17 GB", | 20260506 12:26:07.423 | INFO | "huge" : "", | 20260506 12:26:07.423 | INFO | "huge_shared" : " 27 GB", | 20260506 12:26:07.423 | INFO | "malloc" : " 14 GB", | 20260506 12:26:07.423 | INFO | "pinned" : " 29 GB" | 20260506 12:26:07.423 | INFO | } | 20260506 12:26:07.423 | INFO | Perf report for the last 5.1449 seconds: | 20260506 12:26:07.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:26:07.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:07.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:07.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:07.423 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:26:07.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:07.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:07.424 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260506 12:26:07.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:07.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:07.424 | INFO | Uploaded 36GiB to GPU in 2.10223s (17.1247GiB/s) | 20260506 12:26:07.424 | INFO | Downloaded 34GiB to GPU in 1.9901s (17.0846GiB/s) | 20260506 12:26:12.558 | INFO | Analyzing chunk frames = [1342464,1342976) | 20260506 12:26:12.558 | INFO | MemInfo: { | 20260506 12:26:12.558 | INFO | "gpu" : " 17 GB", | 20260506 12:26:12.558 | INFO | "huge" : "", | 20260506 12:26:12.558 | INFO | "huge_shared" : " 27 GB", | 20260506 12:26:12.558 | INFO | "malloc" : " 14 GB", | 20260506 12:26:12.558 | INFO | "pinned" : " 29 GB" | 20260506 12:26:12.558 | INFO | } | 20260506 12:26:12.558 | INFO | Perf report for the last 5.13422 seconds: | 20260506 12:26:12.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:26:12.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:12.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:12.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:12.558 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260506 12:26:12.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:12.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:12.558 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 12:26:12.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:12.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:12.558 | INFO | Uploaded 36GiB to GPU in 2.00484s (17.9565GiB/s) | 20260506 12:26:12.558 | INFO | Downloaded 34GiB to GPU in 2.00351s (16.9702GiB/s) | 20260506 12:26:17.696 | INFO | Analyzing chunk frames = [1342976,1343488) | 20260506 12:26:17.696 | INFO | MemInfo: { | 20260506 12:26:17.696 | INFO | "gpu" : " 17 GB", | 20260506 12:26:17.696 | INFO | "huge" : "", | 20260506 12:26:17.697 | INFO | "huge_shared" : " 27 GB", | 20260506 12:26:17.697 | INFO | "malloc" : " 14 GB", | 20260506 12:26:17.697 | INFO | "pinned" : " 29 GB" | 20260506 12:26:17.697 | INFO | } | 20260506 12:26:17.697 | INFO | Perf report for the last 5.13894 seconds: | 20260506 12:26:17.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:26:17.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:17.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:17.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:17.697 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:26:17.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:17.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:17.697 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 12:26:17.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:17.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:17.697 | INFO | Uploaded 36GiB to GPU in 2.06015s (17.4745GiB/s) | 20260506 12:26:17.697 | INFO | Downloaded 34GiB to GPU in 1.95755s (17.3687GiB/s) | 20260506 12:26:17.697 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:26:17.697 | INFO | Total mem usage: 16.8 GiB | 20260506 12:26:17.697 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:26:17.697 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.6%) | 20260506 12:26:17.697 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:26:17.697 | INFO | High Water Breakdown: | 20260506 12:26:17.697 | INFO | 59.6% : 8.6 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:26:17.697 | INFO | 40.2% : 5.8 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:26:17.697 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:26:17.697 | INFO | ------------------------------------------ | 20260506 12:26:17.697 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:26:17.697 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:26:17.697 | INFO | Total mem usage: 26.5 GiB | 20260506 12:26:17.697 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:26:17.697 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:26:17.697 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:26:17.697 | INFO | High Water Breakdown: | 20260506 12:26:17.697 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:26:17.697 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:26:17.697 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:26:17.697 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:26:17.697 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:26:17.697 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:26:17.697 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:26:17.697 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:26:17.697 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:26:17.697 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:26:17.697 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:26:17.697 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:26:17.697 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:26:17.698 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:26:17.698 | INFO | ------------------------------------------ | 20260506 12:26:17.698 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:26:17.698 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:26:17.698 | INFO | Total mem usage: 25.6 GiB | 20260506 12:26:17.698 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:26:17.698 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:26:17.698 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:26:17.698 | INFO | High Water Breakdown: | 20260506 12:26:17.698 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:26:17.698 | INFO | ------------------------------------------ | 20260506 12:26:17.698 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:26:17.698 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:26:17.698 | INFO | Total mem usage: 15.7 GiB | 20260506 12:26:17.698 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:26:17.698 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:26:17.698 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:26:17.698 | INFO | High Water Breakdown: | 20260506 12:26:17.698 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:26:17.698 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:26:17.698 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:26:17.698 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:26:17.698 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:26:17.698 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:26:17.698 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:26:17.698 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:26:17.698 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:26:17.698 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:26:17.698 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:26:17.698 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:26:17.698 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:26:17.698 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:26:17.698 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:26:17.698 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:26:17.698 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:26:17.698 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:26:17.698 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:26:17.698 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:26:17.698 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:26:17.698 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:26:17.698 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:26:17.698 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:26:17.698 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:26:17.698 | INFO | ------------------------------------------ | 20260506 12:26:17.698 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:26:17.698 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:26:17.698 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:26:17.698 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:26:17.698 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.60e+01, 1.35e+01 / 2.02e+00 | 20260506 12:26:17.698 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.01e-01, 1.14e-02 / 6.29e-03 | 20260506 12:26:17.698 | INFO | PerfReport: 11.57%, Baseline -- 3.67e+00 / 4.42e+00, 5.86e-01 / 3.86e-01 | 20260506 12:26:17.698 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.49e+00, 1.94e-01 / 1.39e-01 | 20260506 12:26:17.698 | INFO | PerfReport: 15.86%, DME -- 4.03e+01 / 4.48e+01, 4.00e+00 / 2.28e+00 | 20260506 12:26:17.698 | INFO | PerfReport: 56.38%, FrameLabeling -- 1.79e+01 / 2.01e+01, 1.44e+00 / 1.64e+00 | 20260506 12:26:17.698 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.41e+00 / 3.05e+00, 4.23e-01 / 2.05e-01 | 20260506 12:26:17.698 | INFO | PerfReport: 3.75%, Metrics -- 1.19e+00 / 1.58e+00, 3.05e-01 / 2.02e-01 | 20260506 12:26:17.698 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.19e-01 / 4.76e-01, 6.60e-02 / 2.44e-02 | 20260506 12:26:17.698 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:26:17.698 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:26:17.698 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:26:17.698 | INFO | PerfReport: Full Execution -- 9.90e+01 / -nan, 2.46e+01 / 0.00e+00 | 20260506 12:26:17.698 | INFO | PerfReport: 21.49%, Upload -- 2.13e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260506 12:26:17.698 | INFO | PerfReport: 20.99%, Download -- 2.08e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 12:26:17.698 | INFO | PerfReport: 32.09%, Compute -- 3.18e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 12:26:17.698 | INFO | PerfReport: 1.92%, LockUpload -- 1.90e+00 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 12:26:17.698 | INFO | PerfReport: 1.00%, LockDownload -- 9.90e-01 / 0.00e+00, 2.60e+00 / 0.00e+00 | 20260506 12:26:17.698 | INFO | PerfReport: 22.49%, LockCompute -- 2.23e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 12:26:17.698 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260506 12:26:17.825 | INFO | Starting a round of Baz IO | 20260506 12:26:20.992 | INFO | Populating RTMetrics report | 20260506 12:26:20.993 | INFO | RTMetrics report populated | 20260506 12:26:20.993 | INFO | Opening tmp RTMetrics json file | 20260506 12:26:20.994 | INFO | Serializing RTMetrics to json | 20260506 12:26:20.995 | INFO | Serialization done, swapping tmp file now... | 20260506 12:26:20.995 | INFO | RTMetrics done | 20260506 12:26:21.095 | INFO | Finished a round of Baz IO: Wrote 2.087 GiB in 3.27 seconds | 20260506 12:26:21.095 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.60% / 0.02% / 58.38% | 20260506 12:26:21.095 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.97332% / 4.76837e-05% / 1.98682e-05% | 20260506 12:26:21.095 | INFO | Baz IO overall: | 20260506 12:26:21.095 | INFO | BytesWritten : 666842125630 | 20260506 12:26:21.095 | INFO | NumEvents : 322255592558 | 20260506 12:26:22.836 | INFO | Analyzing chunk frames = [1343488,1344000) | 20260506 12:26:22.836 | INFO | MemInfo: { | 20260506 12:26:22.836 | INFO | "gpu" : " 17 GB", | 20260506 12:26:22.836 | INFO | "huge" : "", | 20260506 12:26:22.836 | INFO | "huge_shared" : " 27 GB", | 20260506 12:26:22.836 | INFO | "malloc" : " 14 GB", | 20260506 12:26:22.836 | INFO | "pinned" : " 29 GB" | 20260506 12:26:22.836 | INFO | } | 20260506 12:26:22.836 | INFO | Perf report for the last 5.13952 seconds: | 20260506 12:26:22.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:26:22.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:22.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:22.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:22.836 | INFO | ANALYSIS : 63.6%, 3.0, 98.7ms, 0 | 20260506 12:26:22.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:22.836 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:26:22.836 | INFO | PRE_HQ : 61.6%, 1.9, 61.5ms, 0 | 20260506 12:26:22.836 | INFO | BAZWRITER : 62.9%, 3.7, 118.6ms, 0 | 20260506 12:26:22.836 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:26:22.836 | INFO | Uploaded 36GiB to GPU in 2.08324s (17.2808GiB/s) | 20260506 12:26:22.837 | INFO | Downloaded 38GiB to GPU in 2.20885s (17.2035GiB/s) | 20260506 12:26:27.975 | INFO | Analyzing chunk frames = [1344000,1344512) | 20260506 12:26:27.975 | INFO | MemInfo: { | 20260506 12:26:27.975 | INFO | "gpu" : " 17 GB", | 20260506 12:26:27.975 | INFO | "huge" : "", | 20260506 12:26:27.975 | INFO | "huge_shared" : " 27 GB", | 20260506 12:26:27.975 | INFO | "malloc" : " 14 GB", | 20260506 12:26:27.975 | INFO | "pinned" : " 29 GB" | 20260506 12:26:27.975 | INFO | } | 20260506 12:26:27.975 | INFO | Perf report for the last 5.13917 seconds: | 20260506 12:26:27.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:26:27.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:27.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:27.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:27.975 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:26:27.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:27.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:27.975 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:26:27.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:27.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:27.976 | INFO | Uploaded 36GiB to GPU in 2.01885s (17.8319GiB/s) | 20260506 12:26:27.976 | INFO | Downloaded 34GiB to GPU in 1.91173s (17.7849GiB/s) | 20260506 12:26:33.115 | INFO | Analyzing chunk frames = [1344512,1345024) | 20260506 12:26:33.115 | INFO | MemInfo: { | 20260506 12:26:33.115 | INFO | "gpu" : " 17 GB", | 20260506 12:26:33.115 | INFO | "huge" : "", | 20260506 12:26:33.115 | INFO | "huge_shared" : " 27 GB", | 20260506 12:26:33.115 | INFO | "malloc" : " 14 GB", | 20260506 12:26:33.115 | INFO | "pinned" : " 29 GB" | 20260506 12:26:33.115 | INFO | } | 20260506 12:26:33.115 | INFO | Perf report for the last 5.13952 seconds: | 20260506 12:26:33.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:26:33.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:33.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:33.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:33.115 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 12:26:33.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:33.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:33.115 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:26:33.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:33.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:33.115 | INFO | Uploaded 36GiB to GPU in 2.02324s (17.7932GiB/s) | 20260506 12:26:33.115 | INFO | Downloaded 34GiB to GPU in 1.9381s (17.543GiB/s) | 20260506 12:26:38.254 | INFO | Analyzing chunk frames = [1345024,1345536) | 20260506 12:26:38.254 | INFO | MemInfo: { | 20260506 12:26:38.254 | INFO | "gpu" : " 17 GB", | 20260506 12:26:38.254 | INFO | "huge" : "", | 20260506 12:26:38.254 | INFO | "huge_shared" : " 27 GB", | 20260506 12:26:38.254 | INFO | "malloc" : " 14 GB", | 20260506 12:26:38.254 | INFO | "pinned" : " 29 GB" | 20260506 12:26:38.254 | INFO | } | 20260506 12:26:38.254 | INFO | Perf report for the last 5.13923 seconds: | 20260506 12:26:38.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:26:38.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:38.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:38.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:38.254 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 12:26:38.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:38.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:38.254 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:26:38.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:38.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:38.254 | INFO | Uploaded 36GiB to GPU in 2.01646s (17.8531GiB/s) | 20260506 12:26:38.254 | INFO | Downloaded 34GiB to GPU in 1.94028s (17.5232GiB/s) | 20260506 12:26:43.393 | INFO | Analyzing chunk frames = [1345536,1346048) | 20260506 12:26:43.393 | INFO | MemInfo: { | 20260506 12:26:43.393 | INFO | "gpu" : " 17 GB", | 20260506 12:26:43.393 | INFO | "huge" : "", | 20260506 12:26:43.393 | INFO | "huge_shared" : " 27 GB", | 20260506 12:26:43.393 | INFO | "malloc" : " 14 GB", | 20260506 12:26:43.393 | INFO | "pinned" : " 29 GB" | 20260506 12:26:43.393 | INFO | } | 20260506 12:26:43.393 | INFO | Perf report for the last 5.13932 seconds: | 20260506 12:26:43.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:26:43.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:43.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:43.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:43.393 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 12:26:43.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:43.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:43.393 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 12:26:43.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:43.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:43.394 | INFO | Uploaded 36GiB to GPU in 2.05789s (17.4937GiB/s) | 20260506 12:26:43.394 | INFO | Downloaded 34GiB to GPU in 1.93636s (17.5587GiB/s) | 20260506 12:26:48.533 | INFO | Analyzing chunk frames = [1346048,1346560) | 20260506 12:26:48.533 | INFO | MemInfo: { | 20260506 12:26:48.533 | INFO | "gpu" : " 17 GB", | 20260506 12:26:48.533 | INFO | "huge" : "", | 20260506 12:26:48.533 | INFO | "huge_shared" : " 27 GB", | 20260506 12:26:48.533 | INFO | "malloc" : " 14 GB", | 20260506 12:26:48.533 | INFO | "pinned" : " 29 GB" | 20260506 12:26:48.533 | INFO | } | 20260506 12:26:48.533 | INFO | Perf report for the last 5.13934 seconds: | 20260506 12:26:48.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:26:48.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:48.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:48.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:48.533 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:26:48.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:48.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:48.533 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 12:26:48.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:48.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:48.533 | INFO | Uploaded 36GiB to GPU in 2.11129s (17.0512GiB/s) | 20260506 12:26:48.533 | INFO | Downloaded 34GiB to GPU in 1.97915s (17.1791GiB/s) | 20260506 12:26:53.673 | INFO | Analyzing chunk frames = [1346560,1347072) | 20260506 12:26:53.673 | INFO | MemInfo: { | 20260506 12:26:53.673 | INFO | "gpu" : " 17 GB", | 20260506 12:26:53.673 | INFO | "huge" : "", | 20260506 12:26:53.673 | INFO | "huge_shared" : " 27 GB", | 20260506 12:26:53.673 | INFO | "malloc" : " 14 GB", | 20260506 12:26:53.673 | INFO | "pinned" : " 29 GB" | 20260506 12:26:53.673 | INFO | } | 20260506 12:26:53.673 | INFO | Perf report for the last 5.14048 seconds: | 20260506 12:26:53.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:26:53.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:53.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:53.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:53.673 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:26:53.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:53.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:53.673 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:26:53.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:53.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:53.674 | INFO | Uploaded 36GiB to GPU in 2.01158s (17.8964GiB/s) | 20260506 12:26:53.674 | INFO | Downloaded 34GiB to GPU in 1.97419s (17.2222GiB/s) | 20260506 12:26:58.812 | INFO | Analyzing chunk frames = [1347072,1347584) | 20260506 12:26:58.812 | INFO | MemInfo: { | 20260506 12:26:58.812 | INFO | "gpu" : " 17 GB", | 20260506 12:26:58.812 | INFO | "huge" : "", | 20260506 12:26:58.812 | INFO | "huge_shared" : " 27 GB", | 20260506 12:26:58.812 | INFO | "malloc" : " 14 GB", | 20260506 12:26:58.812 | INFO | "pinned" : " 29 GB" | 20260506 12:26:58.812 | INFO | } | 20260506 12:26:58.812 | INFO | Perf report for the last 5.13862 seconds: | 20260506 12:26:58.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:26:58.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:58.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:58.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:58.812 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 12:26:58.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:58.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:58.812 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:26:58.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:58.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:26:58.812 | INFO | Uploaded 36GiB to GPU in 2.05184s (17.5452GiB/s) | 20260506 12:26:58.812 | INFO | Downloaded 34GiB to GPU in 2.02285s (16.808GiB/s) | 20260506 12:26:58.940 | INFO | Starting a round of Baz IO | 20260506 12:27:02.116 | INFO | Populating RTMetrics report | 20260506 12:27:02.116 | INFO | RTMetrics report populated | 20260506 12:27:02.116 | INFO | Opening tmp RTMetrics json file | 20260506 12:27:02.117 | INFO | Serializing RTMetrics to json | 20260506 12:27:02.118 | INFO | Serialization done, swapping tmp file now... | 20260506 12:27:02.118 | INFO | RTMetrics done | 20260506 12:27:02.213 | INFO | Finished a round of Baz IO: Wrote 2.089 GiB in 3.273 seconds | 20260506 12:27:02.213 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.57% / 0.02% / 58.40% | 20260506 12:27:02.213 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.97148% / 3.97364e-05% / 3.97364e-05% | 20260506 12:27:02.213 | INFO | Baz IO overall: | 20260506 12:27:02.213 | INFO | BytesWritten : 669085387192 | 20260506 12:27:02.213 | INFO | NumEvents : 323343599832 | 20260506 12:27:03.951 | INFO | Analyzing chunk frames = [1347584,1348096) | 20260506 12:27:03.951 | INFO | MemInfo: { | 20260506 12:27:03.951 | INFO | "gpu" : " 17 GB", | 20260506 12:27:03.951 | INFO | "huge" : "", | 20260506 12:27:03.951 | INFO | "huge_shared" : " 27 GB", | 20260506 12:27:03.951 | INFO | "malloc" : " 14 GB", | 20260506 12:27:03.951 | INFO | "pinned" : " 29 GB" | 20260506 12:27:03.951 | INFO | } | 20260506 12:27:03.951 | INFO | Perf report for the last 5.13905 seconds: | 20260506 12:27:03.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:27:03.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:03.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:03.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:03.951 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260506 12:27:03.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:03.951 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:27:03.951 | INFO | PRE_HQ : 61.7%, 1.9, 61.5ms, 0 | 20260506 12:27:03.951 | INFO | BAZWRITER : 62.9%, 3.7, 118.1ms, 0 | 20260506 12:27:03.951 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:27:03.951 | INFO | Uploaded 36GiB to GPU in 2.06281s (17.4519GiB/s) | 20260506 12:27:03.951 | INFO | Downloaded 38GiB to GPU in 2.21401s (17.1635GiB/s) | 20260506 12:27:09.090 | INFO | Analyzing chunk frames = [1348096,1348608) | 20260506 12:27:09.090 | INFO | MemInfo: { | 20260506 12:27:09.090 | INFO | "gpu" : " 17 GB", | 20260506 12:27:09.090 | INFO | "huge" : "", | 20260506 12:27:09.090 | INFO | "huge_shared" : " 27 GB", | 20260506 12:27:09.090 | INFO | "malloc" : " 14 GB", | 20260506 12:27:09.090 | INFO | "pinned" : " 29 GB" | 20260506 12:27:09.090 | INFO | } | 20260506 12:27:09.090 | INFO | Perf report for the last 5.13942 seconds: | 20260506 12:27:09.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:27:09.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:09.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:09.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:09.090 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:27:09.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:09.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:09.090 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 12:27:09.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:09.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:09.091 | INFO | Uploaded 36GiB to GPU in 1.99963s (18.0033GiB/s) | 20260506 12:27:09.091 | INFO | Downloaded 34GiB to GPU in 1.92316s (17.6792GiB/s) | 20260506 12:27:14.230 | INFO | Analyzing chunk frames = [1348608,1349120) | 20260506 12:27:14.230 | INFO | MemInfo: { | 20260506 12:27:14.230 | INFO | "gpu" : " 17 GB", | 20260506 12:27:14.230 | INFO | "huge" : "", | 20260506 12:27:14.230 | INFO | "huge_shared" : " 27 GB", | 20260506 12:27:14.230 | INFO | "malloc" : " 14 GB", | 20260506 12:27:14.230 | INFO | "pinned" : " 29 GB" | 20260506 12:27:14.230 | INFO | } | 20260506 12:27:14.230 | INFO | Perf report for the last 5.13959 seconds: | 20260506 12:27:14.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:27:14.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:14.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:14.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:14.230 | INFO | ANALYSIS : 61.5%, 3.0, 95.3ms, 0 | 20260506 12:27:14.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:14.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:14.230 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:27:14.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:14.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:14.230 | INFO | Uploaded 36GiB to GPU in 1.9898s (18.0923GiB/s) | 20260506 12:27:14.230 | INFO | Downloaded 34GiB to GPU in 1.94251s (17.5032GiB/s) | 20260506 12:27:19.369 | INFO | Analyzing chunk frames = [1349120,1349632) | 20260506 12:27:19.369 | INFO | MemInfo: { | 20260506 12:27:19.369 | INFO | "gpu" : " 17 GB", | 20260506 12:27:19.369 | INFO | "huge" : "", | 20260506 12:27:19.369 | INFO | "huge_shared" : " 27 GB", | 20260506 12:27:19.369 | INFO | "malloc" : " 14 GB", | 20260506 12:27:19.369 | INFO | "pinned" : " 29 GB" | 20260506 12:27:19.369 | INFO | } | 20260506 12:27:19.369 | INFO | Perf report for the last 5.13895 seconds: | 20260506 12:27:19.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:27:19.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:19.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:19.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:19.369 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 12:27:19.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:19.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:19.369 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:27:19.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:19.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:19.369 | INFO | Uploaded 36GiB to GPU in 2.01758s (17.8431GiB/s) | 20260506 12:27:19.369 | INFO | Downloaded 34GiB to GPU in 1.94124s (17.5146GiB/s) | 20260506 12:27:24.508 | INFO | Analyzing chunk frames = [1349632,1350144) | 20260506 12:27:24.508 | INFO | MemInfo: { | 20260506 12:27:24.508 | INFO | "gpu" : " 17 GB", | 20260506 12:27:24.508 | INFO | "huge" : "", | 20260506 12:27:24.508 | INFO | "huge_shared" : " 27 GB", | 20260506 12:27:24.508 | INFO | "malloc" : " 14 GB", | 20260506 12:27:24.508 | INFO | "pinned" : " 29 GB" | 20260506 12:27:24.508 | INFO | } | 20260506 12:27:24.508 | INFO | Perf report for the last 5.13921 seconds: | 20260506 12:27:24.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:27:24.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:24.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:24.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:24.508 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 12:27:24.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:24.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:24.508 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:27:24.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:24.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:24.508 | INFO | Uploaded 36GiB to GPU in 1.98859s (18.1033GiB/s) | 20260506 12:27:24.508 | INFO | Downloaded 34GiB to GPU in 1.95248s (17.4138GiB/s) | 20260506 12:27:29.648 | INFO | Analyzing chunk frames = [1350144,1350656) | 20260506 12:27:29.648 | INFO | MemInfo: { | 20260506 12:27:29.648 | INFO | "gpu" : " 17 GB", | 20260506 12:27:29.648 | INFO | "huge" : "", | 20260506 12:27:29.648 | INFO | "huge_shared" : " 27 GB", | 20260506 12:27:29.648 | INFO | "malloc" : " 14 GB", | 20260506 12:27:29.648 | INFO | "pinned" : " 29 GB" | 20260506 12:27:29.648 | INFO | } | 20260506 12:27:29.648 | INFO | Perf report for the last 5.13963 seconds: | 20260506 12:27:29.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:27:29.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:29.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:29.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:29.648 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 12:27:29.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:29.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:29.648 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260506 12:27:29.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:29.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:29.648 | INFO | Uploaded 36GiB to GPU in 2.03146s (17.7212GiB/s) | 20260506 12:27:29.648 | INFO | Downloaded 34GiB to GPU in 1.92691s (17.6448GiB/s) | 20260506 12:27:34.787 | INFO | Analyzing chunk frames = [1350656,1351168) | 20260506 12:27:34.787 | INFO | MemInfo: { | 20260506 12:27:34.787 | INFO | "gpu" : " 17 GB", | 20260506 12:27:34.787 | INFO | "huge" : "", | 20260506 12:27:34.787 | INFO | "huge_shared" : " 27 GB", | 20260506 12:27:34.787 | INFO | "malloc" : " 14 GB", | 20260506 12:27:34.787 | INFO | "pinned" : " 29 GB" | 20260506 12:27:34.787 | INFO | } | 20260506 12:27:34.787 | INFO | Perf report for the last 5.13926 seconds: | 20260506 12:27:34.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:27:34.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:34.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:34.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:34.787 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:27:34.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:34.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:34.787 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:27:34.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:34.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:34.787 | INFO | Uploaded 36GiB to GPU in 1.98098s (18.1728GiB/s) | 20260506 12:27:34.787 | INFO | Downloaded 34GiB to GPU in 2.01752s (16.8524GiB/s) | 20260506 12:27:39.926 | INFO | Analyzing chunk frames = [1351168,1351680) | 20260506 12:27:39.926 | INFO | MemInfo: { | 20260506 12:27:39.926 | INFO | "gpu" : " 17 GB", | 20260506 12:27:39.926 | INFO | "huge" : "", | 20260506 12:27:39.926 | INFO | "huge_shared" : " 27 GB", | 20260506 12:27:39.926 | INFO | "malloc" : " 14 GB", | 20260506 12:27:39.926 | INFO | "pinned" : " 29 GB" | 20260506 12:27:39.926 | INFO | } | 20260506 12:27:39.926 | INFO | Perf report for the last 5.13906 seconds: | 20260506 12:27:39.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:27:39.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:39.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:39.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:39.926 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 12:27:39.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:39.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:39.926 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:27:39.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:39.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:39.927 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:27:39.927 | INFO | Total mem usage: 16.8 GiB | 20260506 12:27:39.927 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:27:39.927 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.6%) | 20260506 12:27:39.927 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:27:39.927 | INFO | High Water Breakdown: | 20260506 12:27:39.927 | INFO | 59.6% : 8.59 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:27:39.927 | INFO | 40.2% : 5.8 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:27:39.927 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:27:39.927 | INFO | ------------------------------------------ | 20260506 12:27:39.927 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:27:39.927 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:27:39.927 | INFO | Uploaded 36GiB to GPU in 2.06207s (17.4581GiB/s) | 20260506 12:27:39.927 | INFO | Total mem usage: 26.5 GiB | 20260506 12:27:39.927 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:27:39.927 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:27:39.927 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:27:39.927 | INFO | High Water Breakdown: | 20260506 12:27:39.927 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:27:39.927 | INFO | Downloaded 34GiB to GPU in 1.99232s (17.0655GiB/s) | 20260506 12:27:39.927 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:27:39.927 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:27:39.927 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:27:39.927 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:27:39.927 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:27:39.927 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:27:39.927 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:27:39.927 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:27:39.927 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:27:39.927 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:27:39.927 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:27:39.927 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:27:39.927 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:27:39.927 | INFO | ------------------------------------------ | 20260506 12:27:39.927 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:27:39.927 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:27:39.927 | INFO | Total mem usage: 25.6 GiB | 20260506 12:27:39.927 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:27:39.927 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:27:39.927 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:27:39.927 | INFO | High Water Breakdown: | 20260506 12:27:39.927 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:27:39.927 | INFO | ------------------------------------------ | 20260506 12:27:39.927 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:27:39.927 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:27:39.927 | INFO | Total mem usage: 15.7 GiB | 20260506 12:27:39.927 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:27:39.927 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:27:39.927 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:27:39.927 | INFO | High Water Breakdown: | 20260506 12:27:39.927 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:27:39.927 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:27:39.927 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:27:39.927 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:27:39.927 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:27:39.927 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:27:39.927 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:27:39.927 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:27:39.927 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:27:39.927 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:27:39.927 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:27:39.927 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:27:39.927 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:27:39.927 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:27:39.927 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:27:39.927 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:27:39.927 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:27:39.927 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:27:39.927 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:27:39.927 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:27:39.927 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:27:39.927 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:27:39.927 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:27:39.927 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:27:39.927 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:27:39.927 | INFO | ------------------------------------------ | 20260506 12:27:39.927 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:27:39.927 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:27:39.927 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:27:39.927 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:27:39.927 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.61e+01, 1.37e+01 / 2.07e+00 | 20260506 12:27:39.927 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.00e-01, 1.01e-02 / 6.42e-03 | 20260506 12:27:39.927 | INFO | PerfReport: 11.56%, Baseline -- 3.67e+00 / 4.44e+00, 5.84e-01 / 3.56e-01 | 20260506 12:27:39.927 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.46e+00, 1.94e-01 / 1.44e-01 | 20260506 12:27:39.927 | INFO | PerfReport: 15.81%, DME -- 4.02e+01 / 4.47e+01, 4.13e+00 / 2.15e+00 | 20260506 12:27:39.927 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.80e+01 / 2.03e+01, 1.48e+00 / 1.60e+00 | 20260506 12:27:39.927 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.41e+00 / 3.05e+00, 4.20e-01 / 2.11e-01 | 20260506 12:27:39.927 | INFO | PerfReport: 3.74%, Metrics -- 1.19e+00 / 1.54e+00, 3.01e-01 / 1.70e-01 | 20260506 12:27:39.927 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.19e-01 / 4.76e-01, 6.58e-02 / 2.81e-02 | 20260506 12:27:39.927 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:27:39.927 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:27:39.927 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:27:39.927 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 12:27:39.927 | INFO | PerfReport: 21.41%, Upload -- 2.12e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 12:27:39.927 | INFO | PerfReport: 20.96%, Download -- 2.07e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 12:27:39.927 | INFO | PerfReport: 32.17%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 12:27:39.927 | INFO | PerfReport: 1.83%, LockUpload -- 1.81e+00 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 12:27:39.927 | INFO | PerfReport: 1.04%, LockDownload -- 1.03e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260506 12:27:39.927 | INFO | PerfReport: 22.58%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:27:39.927 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260506 12:27:40.053 | INFO | Starting a round of Baz IO | 20260506 12:27:43.223 | INFO | Populating RTMetrics report | 20260506 12:27:43.223 | INFO | RTMetrics report populated | 20260506 12:27:43.223 | INFO | Opening tmp RTMetrics json file | 20260506 12:27:43.224 | INFO | Serializing RTMetrics to json | 20260506 12:27:43.225 | INFO | Serialization done, swapping tmp file now... | 20260506 12:27:43.225 | INFO | RTMetrics done | 20260506 12:27:43.319 | INFO | Finished a round of Baz IO: Wrote 2.089 GiB in 3.265 seconds | 20260506 12:27:43.319 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.55% / 0.02% / 58.43% | 20260506 12:27:43.319 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.95993% / 3.17891e-05% / 6.35783e-05% | 20260506 12:27:43.319 | INFO | Baz IO overall: | 20260506 12:27:43.319 | INFO | BytesWritten : 671328880414 | 20260506 12:27:43.319 | INFO | NumEvents : 324431896132 | 20260506 12:27:45.066 | INFO | Analyzing chunk frames = [1351680,1352192) | 20260506 12:27:45.066 | INFO | MemInfo: { | 20260506 12:27:45.066 | INFO | "gpu" : " 17 GB", | 20260506 12:27:45.066 | INFO | "huge" : "", | 20260506 12:27:45.066 | INFO | "huge_shared" : " 27 GB", | 20260506 12:27:45.066 | INFO | "malloc" : " 14 GB", | 20260506 12:27:45.066 | INFO | "pinned" : " 29 GB" | 20260506 12:27:45.066 | INFO | } | 20260506 12:27:45.066 | INFO | Perf report for the last 5.13949 seconds: | 20260506 12:27:45.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:27:45.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:45.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:45.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:45.066 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260506 12:27:45.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:45.066 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:27:45.066 | INFO | PRE_HQ : 61.6%, 1.9, 61.4ms, 0 | 20260506 12:27:45.066 | INFO | BAZWRITER : 62.8%, 3.6, 117.7ms, 0 | 20260506 12:27:45.066 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:27:45.066 | INFO | Uploaded 36GiB to GPU in 2.09669s (17.1699GiB/s) | 20260506 12:27:45.066 | INFO | Downloaded 38GiB to GPU in 2.20484s (17.2348GiB/s) | 20260506 12:27:50.205 | INFO | Analyzing chunk frames = [1352192,1352704) | 20260506 12:27:50.205 | INFO | MemInfo: { | 20260506 12:27:50.205 | INFO | "gpu" : " 17 GB", | 20260506 12:27:50.205 | INFO | "huge" : "", | 20260506 12:27:50.205 | INFO | "huge_shared" : " 27 GB", | 20260506 12:27:50.205 | INFO | "malloc" : " 14 GB", | 20260506 12:27:50.205 | INFO | "pinned" : " 29 GB" | 20260506 12:27:50.205 | INFO | } | 20260506 12:27:50.205 | INFO | Perf report for the last 5.13931 seconds: | 20260506 12:27:50.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:27:50.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:50.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:50.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:50.205 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:27:50.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:50.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:50.205 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:27:50.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:50.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:50.205 | INFO | Uploaded 36GiB to GPU in 2.03744s (17.6692GiB/s) | 20260506 12:27:50.205 | INFO | Downloaded 34GiB to GPU in 1.91858s (17.7214GiB/s) | 20260506 12:27:55.344 | INFO | Analyzing chunk frames = [1352704,1353216) | 20260506 12:27:55.344 | INFO | MemInfo: { | 20260506 12:27:55.344 | INFO | "gpu" : " 17 GB", | 20260506 12:27:55.344 | INFO | "huge" : "", | 20260506 12:27:55.344 | INFO | "huge_shared" : " 27 GB", | 20260506 12:27:55.344 | INFO | "malloc" : " 14 GB", | 20260506 12:27:55.344 | INFO | "pinned" : " 29 GB" | 20260506 12:27:55.344 | INFO | } | 20260506 12:27:55.345 | INFO | Perf report for the last 5.13947 seconds: | 20260506 12:27:55.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:27:55.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:55.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:55.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:55.345 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:27:55.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:55.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:55.345 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:27:55.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:55.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:27:55.345 | INFO | Uploaded 36GiB to GPU in 2.03544s (17.6866GiB/s) | 20260506 12:27:55.345 | INFO | Downloaded 34GiB to GPU in 1.94206s (17.5072GiB/s) | 20260506 12:28:00.483 | INFO | Analyzing chunk frames = [1353216,1353728) | 20260506 12:28:00.483 | INFO | MemInfo: { | 20260506 12:28:00.483 | INFO | "gpu" : " 17 GB", | 20260506 12:28:00.483 | INFO | "huge" : "", | 20260506 12:28:00.484 | INFO | "huge_shared" : " 27 GB", | 20260506 12:28:00.484 | INFO | "malloc" : " 14 GB", | 20260506 12:28:00.484 | INFO | "pinned" : " 29 GB" | 20260506 12:28:00.484 | INFO | } | 20260506 12:28:00.484 | INFO | Perf report for the last 5.13909 seconds: | 20260506 12:28:00.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:28:00.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:00.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:00.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:00.484 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 12:28:00.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:00.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:00.484 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:28:00.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:00.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:00.484 | INFO | Uploaded 36GiB to GPU in 2.03019s (17.7323GiB/s) | 20260506 12:28:00.484 | INFO | Downloaded 34GiB to GPU in 1.94672s (17.4653GiB/s) | 20260506 12:28:05.623 | INFO | Analyzing chunk frames = [1353728,1354240) | 20260506 12:28:05.623 | INFO | MemInfo: { | 20260506 12:28:05.623 | INFO | "gpu" : " 17 GB", | 20260506 12:28:05.623 | INFO | "huge" : "", | 20260506 12:28:05.623 | INFO | "huge_shared" : " 27 GB", | 20260506 12:28:05.623 | INFO | "malloc" : " 14 GB", | 20260506 12:28:05.623 | INFO | "pinned" : " 29 GB" | 20260506 12:28:05.623 | INFO | } | 20260506 12:28:05.623 | INFO | Perf report for the last 5.1397 seconds: | 20260506 12:28:05.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:28:05.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:05.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:05.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:05.623 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260506 12:28:05.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:05.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:05.623 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:28:05.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:05.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:05.624 | INFO | Uploaded 36GiB to GPU in 2.05005s (17.5606GiB/s) | 20260506 12:28:05.624 | INFO | Downloaded 34GiB to GPU in 1.93935s (17.5316GiB/s) | 20260506 12:28:10.762 | INFO | Analyzing chunk frames = [1354240,1354752) | 20260506 12:28:10.762 | INFO | MemInfo: { | 20260506 12:28:10.762 | INFO | "gpu" : " 17 GB", | 20260506 12:28:10.762 | INFO | "huge" : "", | 20260506 12:28:10.762 | INFO | "huge_shared" : " 27 GB", | 20260506 12:28:10.763 | INFO | "malloc" : " 14 GB", | 20260506 12:28:10.763 | INFO | "pinned" : " 29 GB" | 20260506 12:28:10.763 | INFO | } | 20260506 12:28:10.763 | INFO | Perf report for the last 5.13925 seconds: | 20260506 12:28:10.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:28:10.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:10.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:10.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:10.763 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:28:10.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:10.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:10.763 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260506 12:28:10.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:10.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:10.763 | INFO | Uploaded 36GiB to GPU in 2.11157s (17.0489GiB/s) | 20260506 12:28:10.763 | INFO | Downloaded 34GiB to GPU in 1.98124s (17.161GiB/s) | 20260506 12:28:15.903 | INFO | Analyzing chunk frames = [1354752,1355264) | 20260506 12:28:15.903 | INFO | MemInfo: { | 20260506 12:28:15.903 | INFO | "gpu" : " 17 GB", | 20260506 12:28:15.903 | INFO | "huge" : "", | 20260506 12:28:15.903 | INFO | "huge_shared" : " 27 GB", | 20260506 12:28:15.903 | INFO | "malloc" : " 14 GB", | 20260506 12:28:15.903 | INFO | "pinned" : " 29 GB" | 20260506 12:28:15.903 | INFO | } | 20260506 12:28:15.903 | INFO | Perf report for the last 5.14021 seconds: | 20260506 12:28:15.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:28:15.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:15.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:15.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:15.903 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260506 12:28:15.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:15.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:15.903 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:28:15.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:15.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:15.903 | INFO | Uploaded 36GiB to GPU in 2.02101s (17.8129GiB/s) | 20260506 12:28:15.903 | INFO | Downloaded 34GiB to GPU in 2.00801s (16.9322GiB/s) | 20260506 12:28:21.045 | INFO | Analyzing chunk frames = [1355264,1355776) | 20260506 12:28:21.045 | INFO | MemInfo: { | 20260506 12:28:21.045 | INFO | "gpu" : " 17 GB", | 20260506 12:28:21.045 | INFO | "huge" : "", | 20260506 12:28:21.045 | INFO | "huge_shared" : " 27 GB", | 20260506 12:28:21.045 | INFO | "malloc" : " 14 GB", | 20260506 12:28:21.045 | INFO | "pinned" : " 29 GB" | 20260506 12:28:21.045 | INFO | } | 20260506 12:28:21.045 | INFO | Perf report for the last 5.1423 seconds: | 20260506 12:28:21.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:28:21.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:21.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:21.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:21.045 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260506 12:28:21.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:21.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:21.045 | INFO | PRE_HQ : 33.4%, 1.0, 17.4ms, 0 | 20260506 12:28:21.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:21.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:21.045 | INFO | Uploaded 36GiB to GPU in 2.04301s (17.6211GiB/s) | 20260506 12:28:21.045 | INFO | Downloaded 34GiB to GPU in 1.99155s (17.0721GiB/s) | 20260506 12:28:21.175 | INFO | Starting a round of Baz IO | 20260506 12:28:24.336 | INFO | Populating RTMetrics report | 20260506 12:28:24.336 | INFO | RTMetrics report populated | 20260506 12:28:24.336 | INFO | Opening tmp RTMetrics json file | 20260506 12:28:24.337 | INFO | Serializing RTMetrics to json | 20260506 12:28:24.338 | INFO | Serialization done, swapping tmp file now... | 20260506 12:28:24.338 | INFO | RTMetrics done | 20260506 12:28:24.439 | INFO | Finished a round of Baz IO: Wrote 2.09 GiB in 3.263 seconds | 20260506 12:28:24.439 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.52% / 0.02% / 58.45% | 20260506 12:28:24.439 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.94911% / 5.16574e-05% / 5.5631e-05% | 20260506 12:28:24.439 | INFO | Baz IO overall: | 20260506 12:28:24.439 | INFO | BytesWritten : 673572780783 | 20260506 12:28:24.439 | INFO | NumEvents : 325520234871 | 20260506 12:28:26.181 | INFO | Analyzing chunk frames = [1355776,1356288) | 20260506 12:28:26.181 | INFO | MemInfo: { | 20260506 12:28:26.181 | INFO | "gpu" : " 17 GB", | 20260506 12:28:26.181 | INFO | "huge" : "", | 20260506 12:28:26.181 | INFO | "huge_shared" : " 27 GB", | 20260506 12:28:26.181 | INFO | "malloc" : " 14 GB", | 20260506 12:28:26.181 | INFO | "pinned" : " 29 GB" | 20260506 12:28:26.181 | INFO | } | 20260506 12:28:26.181 | INFO | Perf report for the last 5.13606 seconds: | 20260506 12:28:26.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:28:26.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:26.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:26.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:26.181 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 12:28:26.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:26.181 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:28:26.181 | INFO | PRE_HQ : 61.5%, 1.9, 61.2ms, 0 | 20260506 12:28:26.181 | INFO | BAZWRITER : 62.9%, 3.7, 120.3ms, 0 | 20260506 12:28:26.181 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:28:26.181 | INFO | Uploaded 36GiB to GPU in 2.06304s (17.45GiB/s) | 20260506 12:28:26.181 | INFO | Downloaded 38GiB to GPU in 2.21205s (17.1787GiB/s) | 20260506 12:28:31.320 | INFO | Analyzing chunk frames = [1356288,1356800) | 20260506 12:28:31.320 | INFO | MemInfo: { | 20260506 12:28:31.320 | INFO | "gpu" : " 17 GB", | 20260506 12:28:31.320 | INFO | "huge" : "", | 20260506 12:28:31.320 | INFO | "huge_shared" : " 27 GB", | 20260506 12:28:31.320 | INFO | "malloc" : " 14 GB", | 20260506 12:28:31.320 | INFO | "pinned" : " 29 GB" | 20260506 12:28:31.320 | INFO | } | 20260506 12:28:31.320 | INFO | Perf report for the last 5.13892 seconds: | 20260506 12:28:31.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:28:31.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:31.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:31.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:31.320 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:28:31.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:31.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:31.320 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:28:31.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:31.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:31.320 | INFO | Uploaded 36GiB to GPU in 2.02292s (17.7961GiB/s) | 20260506 12:28:31.320 | INFO | Downloaded 34GiB to GPU in 1.92079s (17.701GiB/s) | 20260506 12:28:36.459 | INFO | Analyzing chunk frames = [1356800,1357312) | 20260506 12:28:36.459 | INFO | MemInfo: { | 20260506 12:28:36.459 | INFO | "gpu" : " 17 GB", | 20260506 12:28:36.459 | INFO | "huge" : "", | 20260506 12:28:36.459 | INFO | "huge_shared" : " 27 GB", | 20260506 12:28:36.459 | INFO | "malloc" : " 14 GB", | 20260506 12:28:36.459 | INFO | "pinned" : " 29 GB" | 20260506 12:28:36.459 | INFO | } | 20260506 12:28:36.459 | INFO | Perf report for the last 5.13914 seconds: | 20260506 12:28:36.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:28:36.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:36.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:36.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:36.459 | INFO | ANALYSIS : 61.8%, 3.0, 95.6ms, 0 | 20260506 12:28:36.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:36.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:36.459 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:28:36.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:36.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:36.459 | INFO | Uploaded 36GiB to GPU in 2.02112s (17.8119GiB/s) | 20260506 12:28:36.460 | INFO | Downloaded 34GiB to GPU in 1.94151s (17.5121GiB/s) | 20260506 12:28:41.598 | INFO | Analyzing chunk frames = [1357312,1357824) | 20260506 12:28:41.598 | INFO | MemInfo: { | 20260506 12:28:41.598 | INFO | "gpu" : " 17 GB", | 20260506 12:28:41.598 | INFO | "huge" : "", | 20260506 12:28:41.598 | INFO | "huge_shared" : " 27 GB", | 20260506 12:28:41.598 | INFO | "malloc" : " 14 GB", | 20260506 12:28:41.598 | INFO | "pinned" : " 29 GB" | 20260506 12:28:41.598 | INFO | } | 20260506 12:28:41.598 | INFO | Perf report for the last 5.13889 seconds: | 20260506 12:28:41.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:28:41.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:41.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:41.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:41.598 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 12:28:41.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:41.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:41.598 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:28:41.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:41.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:41.598 | INFO | Uploaded 36GiB to GPU in 2.01518s (17.8644GiB/s) | 20260506 12:28:41.598 | INFO | Downloaded 34GiB to GPU in 1.93969s (17.5286GiB/s) | 20260506 12:28:46.738 | INFO | Analyzing chunk frames = [1357824,1358336) | 20260506 12:28:46.738 | INFO | MemInfo: { | 20260506 12:28:46.738 | INFO | "gpu" : " 17 GB", | 20260506 12:28:46.738 | INFO | "huge" : "", | 20260506 12:28:46.738 | INFO | "huge_shared" : " 27 GB", | 20260506 12:28:46.738 | INFO | "malloc" : " 14 GB", | 20260506 12:28:46.738 | INFO | "pinned" : " 29 GB" | 20260506 12:28:46.738 | INFO | } | 20260506 12:28:46.738 | INFO | Perf report for the last 5.13971 seconds: | 20260506 12:28:46.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:28:46.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:46.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:46.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:46.738 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 12:28:46.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:46.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:46.738 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:28:46.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:46.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:46.738 | INFO | Uploaded 36GiB to GPU in 2.01516s (17.8646GiB/s) | 20260506 12:28:46.738 | INFO | Downloaded 34GiB to GPU in 1.94583s (17.4733GiB/s) | 20260506 12:28:51.877 | INFO | Analyzing chunk frames = [1358336,1358848) | 20260506 12:28:51.877 | INFO | MemInfo: { | 20260506 12:28:51.877 | INFO | "gpu" : " 17 GB", | 20260506 12:28:51.877 | INFO | "huge" : "", | 20260506 12:28:51.877 | INFO | "huge_shared" : " 27 GB", | 20260506 12:28:51.877 | INFO | "malloc" : " 14 GB", | 20260506 12:28:51.877 | INFO | "pinned" : " 29 GB" | 20260506 12:28:51.877 | INFO | } | 20260506 12:28:51.877 | INFO | Perf report for the last 5.13945 seconds: | 20260506 12:28:51.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:28:51.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:51.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:51.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:51.877 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 12:28:51.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:51.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:51.877 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:28:51.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:51.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:51.878 | INFO | Uploaded 36GiB to GPU in 2.04738s (17.5835GiB/s) | 20260506 12:28:51.878 | INFO | Downloaded 34GiB to GPU in 1.92666s (17.6471GiB/s) | 20260506 12:28:57.017 | INFO | Analyzing chunk frames = [1358848,1359360) | 20260506 12:28:57.017 | INFO | MemInfo: { | 20260506 12:28:57.017 | INFO | "gpu" : " 17 GB", | 20260506 12:28:57.017 | INFO | "huge" : "", | 20260506 12:28:57.018 | INFO | "huge_shared" : " 27 GB", | 20260506 12:28:57.018 | INFO | "malloc" : " 14 GB", | 20260506 12:28:57.018 | INFO | "pinned" : " 29 GB" | 20260506 12:28:57.018 | INFO | } | 20260506 12:28:57.018 | INFO | Perf report for the last 5.14025 seconds: | 20260506 12:28:57.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:28:57.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:57.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:57.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:57.018 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:28:57.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:57.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:57.018 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 12:28:57.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:57.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:28:57.018 | INFO | Uploaded 36GiB to GPU in 2.02275s (17.7976GiB/s) | 20260506 12:28:57.018 | INFO | Downloaded 34GiB to GPU in 2.03265s (16.7269GiB/s) | 20260506 12:29:02.156 | INFO | Analyzing chunk frames = [1359360,1359872) | 20260506 12:29:02.156 | INFO | MemInfo: { | 20260506 12:29:02.156 | INFO | "gpu" : " 17 GB", | 20260506 12:29:02.156 | INFO | "huge" : "", | 20260506 12:29:02.156 | INFO | "huge_shared" : " 27 GB", | 20260506 12:29:02.156 | INFO | "malloc" : " 14 GB", | 20260506 12:29:02.156 | INFO | "pinned" : " 29 GB" | 20260506 12:29:02.156 | INFO | } | 20260506 12:29:02.156 | INFO | Perf report for the last 5.13822 seconds: | 20260506 12:29:02.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:29:02.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:02.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:02.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:02.156 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260506 12:29:02.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:02.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:02.156 | INFO | PRE_HQ : 33.0%, 1.0, 17.2ms, 0 | 20260506 12:29:02.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:02.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:02.156 | INFO | Uploaded 36GiB to GPU in 2.06624s (17.423GiB/s) | 20260506 12:29:02.156 | INFO | Downloaded 34GiB to GPU in 2.0446s (16.6292GiB/s) | 20260506 12:29:02.156 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:29:02.156 | INFO | Total mem usage: 16.8 GiB | 20260506 12:29:02.156 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:29:02.156 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.6%) | 20260506 12:29:02.156 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:29:02.156 | INFO | High Water Breakdown: | 20260506 12:29:02.156 | INFO | 59.6% : 8.59 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:29:02.156 | INFO | 40.3% : 5.8 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:29:02.156 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:29:02.156 | INFO | ------------------------------------------ | 20260506 12:29:02.156 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:29:02.156 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:29:02.156 | INFO | Total mem usage: 26.5 GiB | 20260506 12:29:02.156 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:29:02.156 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:29:02.156 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:29:02.156 | INFO | High Water Breakdown: | 20260506 12:29:02.156 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:29:02.156 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:29:02.156 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:29:02.156 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:29:02.156 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:29:02.156 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:29:02.156 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:29:02.156 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:29:02.156 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:29:02.156 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:29:02.156 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:29:02.156 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:29:02.156 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:29:02.156 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:29:02.156 | INFO | ------------------------------------------ | 20260506 12:29:02.156 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:29:02.157 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:29:02.157 | INFO | Total mem usage: 25.6 GiB | 20260506 12:29:02.157 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:29:02.157 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:29:02.157 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:29:02.157 | INFO | High Water Breakdown: | 20260506 12:29:02.157 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:29:02.157 | INFO | ------------------------------------------ | 20260506 12:29:02.157 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:29:02.157 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:29:02.157 | INFO | Total mem usage: 15.7 GiB | 20260506 12:29:02.157 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:29:02.157 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:29:02.157 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:29:02.157 | INFO | High Water Breakdown: | 20260506 12:29:02.157 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:29:02.157 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:29:02.157 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:29:02.157 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:29:02.157 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:29:02.157 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:29:02.157 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:29:02.157 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:29:02.157 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:29:02.157 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:29:02.157 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:29:02.157 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:29:02.157 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:29:02.157 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:29:02.157 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:29:02.157 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:29:02.157 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:29:02.157 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:29:02.157 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:29:02.157 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:29:02.157 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:29:02.157 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:29:02.157 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:29:02.157 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:29:02.157 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:29:02.157 | INFO | ------------------------------------------ | 20260506 12:29:02.157 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:29:02.157 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:29:02.157 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:29:02.157 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:29:02.157 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.62e+01, 1.37e+01 / 2.46e+00 | 20260506 12:29:02.157 | INFO | PerfReport: 0.29%, AggregateStats -- 9.27e-02 / 1.02e-01, 1.12e-02 / 7.88e-03 | 20260506 12:29:02.157 | INFO | PerfReport: 11.56%, Baseline -- 3.68e+00 / 4.38e+00, 5.81e-01 / 4.08e-01 | 20260506 12:29:02.157 | INFO | PerfReport: 3.25%, Histogram -- 1.04e+00 / 1.47e+00, 1.94e-01 / 1.45e-01 | 20260506 12:29:02.157 | INFO | PerfReport: 15.83%, DME -- 4.03e+01 / 4.48e+01, 4.07e+00 / 2.93e+00 | 20260506 12:29:02.157 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.80e+01 / 2.04e+01, 1.47e+00 / 1.97e+00 | 20260506 12:29:02.157 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.41e+00 / 3.04e+00, 4.17e-01 / 2.43e-01 | 20260506 12:29:02.157 | INFO | PerfReport: 3.74%, Metrics -- 1.19e+00 / 1.55e+00, 3.01e-01 / 1.58e-01 | 20260506 12:29:02.157 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.19e-01 / 4.74e-01, 6.55e-02 / 3.00e-02 | 20260506 12:29:02.157 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:29:02.157 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:29:02.157 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:29:02.157 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 12:29:02.157 | INFO | PerfReport: 21.52%, Upload -- 2.13e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260506 12:29:02.157 | INFO | PerfReport: 20.99%, Download -- 2.08e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 12:29:02.157 | INFO | PerfReport: 32.17%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 12:29:02.157 | INFO | PerfReport: 1.85%, LockUpload -- 1.83e+00 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 12:29:02.157 | INFO | PerfReport: 0.96%, LockDownload -- 9.45e-01 / 0.00e+00, 2.51e+00 / 0.00e+00 | 20260506 12:29:02.157 | INFO | PerfReport: 22.50%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:29:02.157 | INFO | PerfReport: 0.02%, Misc -- 1.52e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260506 12:29:02.286 | INFO | Starting a round of Baz IO | 20260506 12:29:05.448 | INFO | Populating RTMetrics report | 20260506 12:29:05.448 | INFO | RTMetrics report populated | 20260506 12:29:05.448 | INFO | Opening tmp RTMetrics json file | 20260506 12:29:05.449 | INFO | Serializing RTMetrics to json | 20260506 12:29:05.450 | INFO | Serialization done, swapping tmp file now... | 20260506 12:29:05.450 | INFO | RTMetrics done | 20260506 12:29:05.543 | INFO | Finished a round of Baz IO: Wrote 2.09 GiB in 3.256 seconds | 20260506 12:29:05.543 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.50% / 0.02% / 58.48% | 20260506 12:29:05.544 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.9427% / 6.75519e-05% / 5.96046e-05% | 20260506 12:29:05.544 | INFO | Baz IO overall: | 20260506 12:29:05.544 | INFO | BytesWritten : 675817161289 | 20260506 12:29:05.544 | INFO | NumEvents : 326609175879 | 20260506 12:29:07.295 | INFO | Analyzing chunk frames = [1359872,1360384) | 20260506 12:29:07.295 | INFO | MemInfo: { | 20260506 12:29:07.295 | INFO | "gpu" : " 17 GB", | 20260506 12:29:07.295 | INFO | "huge" : "", | 20260506 12:29:07.295 | INFO | "huge_shared" : " 27 GB", | 20260506 12:29:07.295 | INFO | "malloc" : " 14 GB", | 20260506 12:29:07.295 | INFO | "pinned" : " 29 GB" | 20260506 12:29:07.295 | INFO | } | 20260506 12:29:07.296 | INFO | Perf report for the last 5.13969 seconds: | 20260506 12:29:07.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:29:07.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:07.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:07.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:07.296 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 12:29:07.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:07.296 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:29:07.296 | INFO | PRE_HQ : 61.4%, 1.9, 61.7ms, 0 | 20260506 12:29:07.296 | INFO | BAZWRITER : 62.8%, 3.7, 118.2ms, 0 | 20260506 12:29:07.296 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:29:07.296 | INFO | Uploaded 36GiB to GPU in 2.08285s (17.284GiB/s) | 20260506 12:29:07.296 | INFO | Downloaded 38GiB to GPU in 2.21669s (17.1427GiB/s) | 20260506 12:29:12.435 | INFO | Analyzing chunk frames = [1360384,1360896) | 20260506 12:29:12.435 | INFO | MemInfo: { | 20260506 12:29:12.435 | INFO | "gpu" : " 17 GB", | 20260506 12:29:12.435 | INFO | "huge" : "", | 20260506 12:29:12.435 | INFO | "huge_shared" : " 27 GB", | 20260506 12:29:12.435 | INFO | "malloc" : " 14 GB", | 20260506 12:29:12.435 | INFO | "pinned" : " 29 GB" | 20260506 12:29:12.435 | INFO | } | 20260506 12:29:12.435 | INFO | Perf report for the last 5.13913 seconds: | 20260506 12:29:12.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:29:12.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:12.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:12.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:12.435 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:29:12.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:12.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:12.435 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:29:12.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:12.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:12.435 | INFO | Uploaded 36GiB to GPU in 2.01817s (17.838GiB/s) | 20260506 12:29:12.435 | INFO | Downloaded 34GiB to GPU in 1.92369s (17.6744GiB/s) | 20260506 12:29:17.574 | INFO | Analyzing chunk frames = [1360896,1361408) | 20260506 12:29:17.574 | INFO | MemInfo: { | 20260506 12:29:17.574 | INFO | "gpu" : " 17 GB", | 20260506 12:29:17.574 | INFO | "huge" : "", | 20260506 12:29:17.574 | INFO | "huge_shared" : " 27 GB", | 20260506 12:29:17.574 | INFO | "malloc" : " 14 GB", | 20260506 12:29:17.574 | INFO | "pinned" : " 29 GB" | 20260506 12:29:17.574 | INFO | } | 20260506 12:29:17.574 | INFO | Perf report for the last 5.139 seconds: | 20260506 12:29:17.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:29:17.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:17.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:17.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:17.574 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 12:29:17.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:17.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:17.574 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:29:17.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:17.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:17.574 | INFO | Uploaded 36GiB to GPU in 2.02798s (17.7517GiB/s) | 20260506 12:29:17.574 | INFO | Downloaded 34GiB to GPU in 1.94076s (17.5189GiB/s) | 20260506 12:29:22.713 | INFO | Analyzing chunk frames = [1361408,1361920) | 20260506 12:29:22.713 | INFO | MemInfo: { | 20260506 12:29:22.713 | INFO | "gpu" : " 17 GB", | 20260506 12:29:22.713 | INFO | "huge" : "", | 20260506 12:29:22.713 | INFO | "huge_shared" : " 27 GB", | 20260506 12:29:22.713 | INFO | "malloc" : " 14 GB", | 20260506 12:29:22.713 | INFO | "pinned" : " 29 GB" | 20260506 12:29:22.713 | INFO | } | 20260506 12:29:22.713 | INFO | Perf report for the last 5.13954 seconds: | 20260506 12:29:22.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:29:22.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:22.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:22.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:22.713 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 12:29:22.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:22.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:22.713 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:29:22.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:22.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:22.713 | INFO | Uploaded 36GiB to GPU in 2.01658s (17.852GiB/s) | 20260506 12:29:22.714 | INFO | Downloaded 34GiB to GPU in 1.93645s (17.5579GiB/s) | 20260506 12:29:27.852 | INFO | Analyzing chunk frames = [1361920,1362432) | 20260506 12:29:27.852 | INFO | MemInfo: { | 20260506 12:29:27.852 | INFO | "gpu" : " 17 GB", | 20260506 12:29:27.852 | INFO | "huge" : "", | 20260506 12:29:27.852 | INFO | "huge_shared" : " 27 GB", | 20260506 12:29:27.852 | INFO | "malloc" : " 14 GB", | 20260506 12:29:27.852 | INFO | "pinned" : " 29 GB" | 20260506 12:29:27.852 | INFO | } | 20260506 12:29:27.852 | INFO | Perf report for the last 5.13921 seconds: | 20260506 12:29:27.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:29:27.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:27.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:27.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:27.853 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 12:29:27.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:27.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:27.853 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:29:27.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:27.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:27.853 | INFO | Uploaded 36GiB to GPU in 1.96472s (18.3232GiB/s) | 20260506 12:29:27.853 | INFO | Downloaded 34GiB to GPU in 1.93571s (17.5646GiB/s) | 20260506 12:29:32.992 | INFO | Analyzing chunk frames = [1362432,1362944) | 20260506 12:29:32.992 | INFO | MemInfo: { | 20260506 12:29:32.992 | INFO | "gpu" : " 17 GB", | 20260506 12:29:32.992 | INFO | "huge" : "", | 20260506 12:29:32.992 | INFO | "huge_shared" : " 27 GB", | 20260506 12:29:32.992 | INFO | "malloc" : " 14 GB", | 20260506 12:29:32.992 | INFO | "pinned" : " 29 GB" | 20260506 12:29:32.992 | INFO | } | 20260506 12:29:32.992 | INFO | Perf report for the last 5.13947 seconds: | 20260506 12:29:32.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:29:32.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:32.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:32.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:32.992 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 12:29:32.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:32.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:32.992 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 12:29:32.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:32.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:32.993 | INFO | Uploaded 36GiB to GPU in 2.09195s (17.2088GiB/s) | 20260506 12:29:32.993 | INFO | Downloaded 34GiB to GPU in 1.97202s (17.2412GiB/s) | 20260506 12:29:38.132 | INFO | Analyzing chunk frames = [1362944,1363456) | 20260506 12:29:38.132 | INFO | MemInfo: { | 20260506 12:29:38.132 | INFO | "gpu" : " 17 GB", | 20260506 12:29:38.132 | INFO | "huge" : "", | 20260506 12:29:38.132 | INFO | "huge_shared" : " 27 GB", | 20260506 12:29:38.132 | INFO | "malloc" : " 14 GB", | 20260506 12:29:38.132 | INFO | "pinned" : " 29 GB" | 20260506 12:29:38.132 | INFO | } | 20260506 12:29:38.132 | INFO | Perf report for the last 5.1398 seconds: | 20260506 12:29:38.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:29:38.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:38.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:38.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:38.132 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:29:38.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:38.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:38.132 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 12:29:38.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:38.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:38.132 | INFO | Uploaded 36GiB to GPU in 2.0064s (17.9426GiB/s) | 20260506 12:29:38.132 | INFO | Downloaded 34GiB to GPU in 1.97195s (17.2418GiB/s) | 20260506 12:29:43.270 | INFO | Analyzing chunk frames = [1363456,1363968) | 20260506 12:29:43.270 | INFO | MemInfo: { | 20260506 12:29:43.270 | INFO | "gpu" : " 17 GB", | 20260506 12:29:43.270 | INFO | "huge" : "", | 20260506 12:29:43.270 | INFO | "huge_shared" : " 27 GB", | 20260506 12:29:43.270 | INFO | "malloc" : " 14 GB", | 20260506 12:29:43.270 | INFO | "pinned" : " 29 GB" | 20260506 12:29:43.270 | INFO | } | 20260506 12:29:43.270 | INFO | Perf report for the last 5.13868 seconds: | 20260506 12:29:43.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:29:43.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:43.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:43.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:43.270 | INFO | ANALYSIS : 62.8%, 3.0, 98.1ms, 0 | 20260506 12:29:43.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:43.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:43.271 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 12:29:43.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:43.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:43.271 | INFO | Uploaded 36GiB to GPU in 2.04374s (17.6148GiB/s) | 20260506 12:29:43.271 | INFO | Downloaded 34GiB to GPU in 2.00873s (16.9261GiB/s) | 20260506 12:29:43.402 | INFO | Starting a round of Baz IO | 20260506 12:29:46.563 | INFO | Populating RTMetrics report | 20260506 12:29:46.563 | INFO | RTMetrics report populated | 20260506 12:29:46.563 | INFO | Opening tmp RTMetrics json file | 20260506 12:29:46.564 | INFO | Serializing RTMetrics to json | 20260506 12:29:46.565 | INFO | Serialization done, swapping tmp file now... | 20260506 12:29:46.565 | INFO | RTMetrics done | 20260506 12:29:46.664 | INFO | Finished a round of Baz IO: Wrote 2.091 GiB in 3.262 seconds | 20260506 12:29:46.664 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.48% / 0.02% / 58.50% | 20260506 12:29:46.664 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.93464% / 2.78155e-05% / 4.76837e-05% | 20260506 12:29:46.664 | INFO | Baz IO overall: | 20260506 12:29:46.664 | INFO | BytesWritten : 678062781743 | 20260506 12:29:46.664 | INFO | NumEvents : 327698515557 | 20260506 12:29:48.410 | INFO | Analyzing chunk frames = [1363968,1364480) | 20260506 12:29:48.410 | INFO | MemInfo: { | 20260506 12:29:48.410 | INFO | "gpu" : " 17 GB", | 20260506 12:29:48.410 | INFO | "huge" : "", | 20260506 12:29:48.410 | INFO | "huge_shared" : " 27 GB", | 20260506 12:29:48.410 | INFO | "malloc" : " 14 GB", | 20260506 12:29:48.410 | INFO | "pinned" : " 29 GB" | 20260506 12:29:48.410 | INFO | } | 20260506 12:29:48.410 | INFO | Perf report for the last 5.1393 seconds: | 20260506 12:29:48.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:29:48.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:48.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:48.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:48.410 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 12:29:48.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:48.410 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260506 12:29:48.410 | INFO | PRE_HQ : 61.3%, 1.9, 61.6ms, 0 | 20260506 12:29:48.410 | INFO | BAZWRITER : 62.8%, 3.7, 117.9ms, 0 | 20260506 12:29:48.410 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:29:48.410 | INFO | Uploaded 36GiB to GPU in 2.07234s (17.3716GiB/s) | 20260506 12:29:48.410 | INFO | Downloaded 38GiB to GPU in 2.21427s (17.1614GiB/s) | 20260506 12:29:53.549 | INFO | Analyzing chunk frames = [1364480,1364992) | 20260506 12:29:53.549 | INFO | MemInfo: { | 20260506 12:29:53.549 | INFO | "gpu" : " 17 GB", | 20260506 12:29:53.549 | INFO | "huge" : "", | 20260506 12:29:53.549 | INFO | "huge_shared" : " 27 GB", | 20260506 12:29:53.549 | INFO | "malloc" : " 14 GB", | 20260506 12:29:53.549 | INFO | "pinned" : " 29 GB" | 20260506 12:29:53.549 | INFO | } | 20260506 12:29:53.549 | INFO | Perf report for the last 5.13939 seconds: | 20260506 12:29:53.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:29:53.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:53.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:53.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:53.549 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:29:53.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:53.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:53.549 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 12:29:53.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:53.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:53.549 | INFO | Uploaded 36GiB to GPU in 2.03542s (17.6867GiB/s) | 20260506 12:29:53.550 | INFO | Downloaded 34GiB to GPU in 1.92137s (17.6957GiB/s) | 20260506 12:29:58.689 | INFO | Analyzing chunk frames = [1364992,1365504) | 20260506 12:29:58.689 | INFO | MemInfo: { | 20260506 12:29:58.689 | INFO | "gpu" : " 17 GB", | 20260506 12:29:58.689 | INFO | "huge" : "", | 20260506 12:29:58.689 | INFO | "huge_shared" : " 27 GB", | 20260506 12:29:58.689 | INFO | "malloc" : " 14 GB", | 20260506 12:29:58.689 | INFO | "pinned" : " 29 GB" | 20260506 12:29:58.689 | INFO | } | 20260506 12:29:58.689 | INFO | Perf report for the last 5.13949 seconds: | 20260506 12:29:58.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:29:58.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:58.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:58.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:58.689 | INFO | ANALYSIS : 61.7%, 3.0, 95.5ms, 0 | 20260506 12:29:58.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:58.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:58.689 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:29:58.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:58.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:29:58.689 | INFO | Uploaded 36GiB to GPU in 2.02058s (17.8167GiB/s) | 20260506 12:29:58.689 | INFO | Downloaded 34GiB to GPU in 1.94011s (17.5248GiB/s) | 20260506 12:30:03.828 | INFO | Analyzing chunk frames = [1365504,1366016) | 20260506 12:30:03.828 | INFO | MemInfo: { | 20260506 12:30:03.828 | INFO | "gpu" : " 17 GB", | 20260506 12:30:03.828 | INFO | "huge" : "", | 20260506 12:30:03.828 | INFO | "huge_shared" : " 27 GB", | 20260506 12:30:03.828 | INFO | "malloc" : " 14 GB", | 20260506 12:30:03.828 | INFO | "pinned" : " 29 GB" | 20260506 12:30:03.828 | INFO | } | 20260506 12:30:03.828 | INFO | Perf report for the last 5.13934 seconds: | 20260506 12:30:03.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:30:03.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:03.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:03.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:03.828 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 12:30:03.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:03.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:03.828 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:30:03.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:03.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:03.828 | INFO | Uploaded 36GiB to GPU in 2.00726s (17.9349GiB/s) | 20260506 12:30:03.828 | INFO | Downloaded 34GiB to GPU in 1.9358s (17.5638GiB/s) | 20260506 12:30:08.967 | INFO | Analyzing chunk frames = [1366016,1366528) | 20260506 12:30:08.967 | INFO | MemInfo: { | 20260506 12:30:08.967 | INFO | "gpu" : " 17 GB", | 20260506 12:30:08.967 | INFO | "huge" : "", | 20260506 12:30:08.967 | INFO | "huge_shared" : " 27 GB", | 20260506 12:30:08.967 | INFO | "malloc" : " 14 GB", | 20260506 12:30:08.967 | INFO | "pinned" : " 29 GB" | 20260506 12:30:08.967 | INFO | } | 20260506 12:30:08.967 | INFO | Perf report for the last 5.13916 seconds: | 20260506 12:30:08.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:30:08.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:08.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:08.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:08.967 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 12:30:08.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:08.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:08.967 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 12:30:08.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:08.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:08.967 | INFO | Uploaded 36GiB to GPU in 2.01583s (17.8587GiB/s) | 20260506 12:30:08.967 | INFO | Downloaded 34GiB to GPU in 1.94672s (17.4653GiB/s) | 20260506 12:30:14.107 | INFO | Analyzing chunk frames = [1366528,1367040) | 20260506 12:30:14.107 | INFO | MemInfo: { | 20260506 12:30:14.107 | INFO | "gpu" : " 17 GB", | 20260506 12:30:14.107 | INFO | "huge" : "", | 20260506 12:30:14.107 | INFO | "huge_shared" : " 27 GB", | 20260506 12:30:14.107 | INFO | "malloc" : " 14 GB", | 20260506 12:30:14.107 | INFO | "pinned" : " 29 GB" | 20260506 12:30:14.107 | INFO | } | 20260506 12:30:14.107 | INFO | Perf report for the last 5.13963 seconds: | 20260506 12:30:14.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:30:14.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:14.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:14.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:14.107 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 12:30:14.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:14.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:14.107 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 12:30:14.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:14.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:14.107 | INFO | Uploaded 36GiB to GPU in 2.04909s (17.5688GiB/s) | 20260506 12:30:14.107 | INFO | Downloaded 34GiB to GPU in 1.9236s (17.6752GiB/s) | 20260506 12:30:19.246 | INFO | Analyzing chunk frames = [1367040,1367552) | 20260506 12:30:19.246 | INFO | MemInfo: { | 20260506 12:30:19.246 | INFO | "gpu" : " 17 GB", | 20260506 12:30:19.246 | INFO | "huge" : "", | 20260506 12:30:19.246 | INFO | "huge_shared" : " 27 GB", | 20260506 12:30:19.246 | INFO | "malloc" : " 14 GB", | 20260506 12:30:19.246 | INFO | "pinned" : " 29 GB" | 20260506 12:30:19.246 | INFO | } | 20260506 12:30:19.246 | INFO | Perf report for the last 5.13919 seconds: | 20260506 12:30:19.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:30:19.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:19.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:19.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:19.246 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:30:19.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:19.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:19.246 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 12:30:19.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:19.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:19.246 | INFO | Uploaded 36GiB to GPU in 1.97189s (18.2566GiB/s) | 20260506 12:30:19.246 | INFO | Downloaded 34GiB to GPU in 2.02317s (16.8053GiB/s) | 20260506 12:30:24.385 | INFO | Analyzing chunk frames = [1367552,1368064) | 20260506 12:30:24.385 | INFO | MemInfo: { | 20260506 12:30:24.385 | INFO | "gpu" : " 17 GB", | 20260506 12:30:24.385 | INFO | "huge" : "", | 20260506 12:30:24.385 | INFO | "huge_shared" : " 27 GB", | 20260506 12:30:24.385 | INFO | "malloc" : " 14 GB", | 20260506 12:30:24.385 | INFO | "pinned" : " 29 GB" | 20260506 12:30:24.385 | INFO | } | 20260506 12:30:24.385 | INFO | Perf report for the last 5.13919 seconds: | 20260506 12:30:24.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:30:24.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:24.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:24.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:24.385 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:30:24.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:24.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:24.385 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 12:30:24.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:24.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:24.386 | INFO | Uploaded 36GiB to GPU in 2.04793s (17.5788GiB/s) | 20260506 12:30:24.386 | INFO | Downloaded 34GiB to GPU in 1.96504s (17.3025GiB/s) | 20260506 12:30:24.386 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:30:24.386 | INFO | Total mem usage: 16.8 GiB | 20260506 12:30:24.386 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:30:24.386 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.9%) | 20260506 12:30:24.386 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:30:24.386 | INFO | High Water Breakdown: | 20260506 12:30:24.386 | INFO | 59.7% : 8.67 GiB ( 83.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:30:24.386 | INFO | 40.1% : 5.82 GiB ( 87% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:30:24.386 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:30:24.386 | INFO | ------------------------------------------ | 20260506 12:30:24.386 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:30:24.386 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:30:24.386 | INFO | Total mem usage: 26.5 GiB | 20260506 12:30:24.386 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:30:24.386 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:30:24.386 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:30:24.386 | INFO | High Water Breakdown: | 20260506 12:30:24.386 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:30:24.386 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:30:24.386 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:30:24.386 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:30:24.386 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:30:24.386 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:30:24.386 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:30:24.386 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:30:24.386 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:30:24.386 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:30:24.386 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:30:24.386 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:30:24.386 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:30:24.386 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:30:24.386 | INFO | ------------------------------------------ | 20260506 12:30:24.386 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:30:24.386 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:30:24.386 | INFO | Total mem usage: 25.6 GiB | 20260506 12:30:24.386 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:30:24.386 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:30:24.386 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:30:24.386 | INFO | High Water Breakdown: | 20260506 12:30:24.386 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:30:24.386 | INFO | ------------------------------------------ | 20260506 12:30:24.386 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:30:24.386 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:30:24.386 | INFO | Total mem usage: 15.7 GiB | 20260506 12:30:24.387 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:30:24.387 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:30:24.387 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 12:30:24.387 | INFO | High Water Breakdown: | 20260506 12:30:24.387 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:30:24.387 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:30:24.387 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:30:24.387 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:30:24.387 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:30:24.387 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:30:24.387 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:30:24.387 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:30:24.387 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:30:24.387 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:30:24.387 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:30:24.387 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:30:24.387 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:30:24.387 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:30:24.387 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:30:24.387 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:30:24.387 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:30:24.387 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:30:24.387 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:30:24.387 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:30:24.387 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:30:24.387 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:30:24.387 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:30:24.387 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:30:24.387 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:30:24.387 | INFO | ------------------------------------------ | 20260506 12:30:24.387 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:30:24.387 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:30:24.387 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:30:24.387 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:30:24.387 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.64e+01, 1.38e+01 / 2.30e+00 | 20260506 12:30:24.387 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.01e-01, 1.01e-02 / 6.47e-03 | 20260506 12:30:24.387 | INFO | PerfReport: 11.57%, Baseline -- 3.68e+00 / 4.39e+00, 5.88e-01 / 3.78e-01 | 20260506 12:30:24.387 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.46e+00, 1.95e-01 / 1.44e-01 | 20260506 12:30:24.387 | INFO | PerfReport: 15.87%, DME -- 4.03e+01 / 4.50e+01, 4.23e+00 / 2.52e+00 | 20260506 12:30:24.387 | INFO | PerfReport: 56.41%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.44e+00 / 1.59e+00 | 20260506 12:30:24.387 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.40e+00 / 3.04e+00, 4.19e-01 / 2.25e-01 | 20260506 12:30:24.387 | INFO | PerfReport: 3.74%, Metrics -- 1.19e+00 / 1.56e+00, 3.03e-01 / 1.53e-01 | 20260506 12:30:24.387 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.18e-01 / 4.75e-01, 6.56e-02 / 2.88e-02 | 20260506 12:30:24.387 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:30:24.387 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:30:24.387 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:30:24.387 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 12:30:24.387 | INFO | PerfReport: 21.39%, Upload -- 2.11e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260506 12:30:24.387 | INFO | PerfReport: 20.93%, Download -- 2.07e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 12:30:24.387 | INFO | PerfReport: 32.16%, Compute -- 3.18e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260506 12:30:24.387 | INFO | PerfReport: 1.83%, LockUpload -- 1.81e+00 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 12:30:24.387 | INFO | PerfReport: 1.00%, LockDownload -- 9.93e-01 / 0.00e+00, 2.66e+00 / 0.00e+00 | 20260506 12:30:24.387 | INFO | PerfReport: 22.66%, LockCompute -- 2.24e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 12:30:24.387 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260506 12:30:24.516 | INFO | Starting a round of Baz IO | 20260506 12:30:27.682 | INFO | Populating RTMetrics report | 20260506 12:30:27.682 | INFO | RTMetrics report populated | 20260506 12:30:27.682 | INFO | Opening tmp RTMetrics json file | 20260506 12:30:27.683 | INFO | Serializing RTMetrics to json | 20260506 12:30:27.684 | INFO | Serialization done, swapping tmp file now... | 20260506 12:30:27.685 | INFO | RTMetrics done | 20260506 12:30:27.779 | INFO | Finished a round of Baz IO: Wrote 2.09 GiB in 3.263 seconds | 20260506 12:30:27.779 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.45% / 0.02% / 58.52% | 20260506 12:30:27.779 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.91341% / 3.57628e-05% / 5.16574e-05% | 20260506 12:30:27.779 | INFO | Baz IO overall: | 20260506 12:30:27.780 | INFO | BytesWritten : 680307143924 | 20260506 12:30:27.780 | INFO | NumEvents : 328786665181 | 20260506 12:30:29.525 | INFO | Analyzing chunk frames = [1368064,1368576) | 20260506 12:30:29.525 | INFO | MemInfo: { | 20260506 12:30:29.525 | INFO | "gpu" : " 17 GB", | 20260506 12:30:29.525 | INFO | "huge" : "", | 20260506 12:30:29.525 | INFO | "huge_shared" : " 27 GB", | 20260506 12:30:29.525 | INFO | "malloc" : " 14 GB", | 20260506 12:30:29.525 | INFO | "pinned" : " 29 GB" | 20260506 12:30:29.525 | INFO | } | 20260506 12:30:29.525 | INFO | Perf report for the last 5.13968 seconds: | 20260506 12:30:29.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:30:29.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:29.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:29.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:29.525 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260506 12:30:29.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:29.525 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:30:29.525 | INFO | PRE_HQ : 61.5%, 1.9, 61.1ms, 0 | 20260506 12:30:29.525 | INFO | BAZWRITER : 62.8%, 3.8, 122.0ms, 0 | 20260506 12:30:29.525 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:30:29.525 | INFO | Uploaded 36GiB to GPU in 2.06334s (17.4474GiB/s) | 20260506 12:30:29.525 | INFO | Downloaded 38GiB to GPU in 2.2167s (17.1426GiB/s) | 20260506 12:30:34.664 | INFO | Analyzing chunk frames = [1368576,1369088) | 20260506 12:30:34.664 | INFO | MemInfo: { | 20260506 12:30:34.664 | INFO | "gpu" : " 17 GB", | 20260506 12:30:34.664 | INFO | "huge" : "", | 20260506 12:30:34.664 | INFO | "huge_shared" : " 27 GB", | 20260506 12:30:34.664 | INFO | "malloc" : " 14 GB", | 20260506 12:30:34.664 | INFO | "pinned" : " 29 GB" | 20260506 12:30:34.664 | INFO | } | 20260506 12:30:34.664 | INFO | Perf report for the last 5.13893 seconds: | 20260506 12:30:34.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:30:34.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:34.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:34.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:34.664 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:30:34.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:34.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:34.664 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:30:34.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:34.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:34.664 | INFO | Uploaded 36GiB to GPU in 2.04355s (17.6164GiB/s) | 20260506 12:30:34.664 | INFO | Downloaded 34GiB to GPU in 1.92666s (17.6471GiB/s) | 20260506 12:30:39.803 | INFO | Analyzing chunk frames = [1369088,1369600) | 20260506 12:30:39.803 | INFO | MemInfo: { | 20260506 12:30:39.803 | INFO | "gpu" : " 17 GB", | 20260506 12:30:39.803 | INFO | "huge" : "", | 20260506 12:30:39.803 | INFO | "huge_shared" : " 27 GB", | 20260506 12:30:39.803 | INFO | "malloc" : " 14 GB", | 20260506 12:30:39.803 | INFO | "pinned" : " 29 GB" | 20260506 12:30:39.803 | INFO | } | 20260506 12:30:39.804 | INFO | Perf report for the last 5.13961 seconds: | 20260506 12:30:39.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:30:39.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:39.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:39.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:39.804 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:30:39.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:39.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:39.804 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:30:39.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:39.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:39.804 | INFO | Uploaded 36GiB to GPU in 2.01421s (17.8731GiB/s) | 20260506 12:30:39.804 | INFO | Downloaded 34GiB to GPU in 1.94233s (17.5047GiB/s) | 20260506 12:30:44.942 | INFO | Analyzing chunk frames = [1369600,1370112) | 20260506 12:30:44.942 | INFO | MemInfo: { | 20260506 12:30:44.942 | INFO | "gpu" : " 17 GB", | 20260506 12:30:44.942 | INFO | "huge" : "", | 20260506 12:30:44.942 | INFO | "huge_shared" : " 27 GB", | 20260506 12:30:44.942 | INFO | "malloc" : " 14 GB", | 20260506 12:30:44.943 | INFO | "pinned" : " 29 GB" | 20260506 12:30:44.943 | INFO | } | 20260506 12:30:44.943 | INFO | Perf report for the last 5.13907 seconds: | 20260506 12:30:44.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:30:44.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:44.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:44.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:44.943 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 12:30:44.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:44.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:44.943 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 12:30:44.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:44.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:44.943 | INFO | Uploaded 36GiB to GPU in 2.02511s (17.7768GiB/s) | 20260506 12:30:44.943 | INFO | Downloaded 34GiB to GPU in 1.94244s (17.5038GiB/s) | 20260506 12:30:50.082 | INFO | Analyzing chunk frames = [1370112,1370624) | 20260506 12:30:50.082 | INFO | MemInfo: { | 20260506 12:30:50.082 | INFO | "gpu" : " 17 GB", | 20260506 12:30:50.082 | INFO | "huge" : "", | 20260506 12:30:50.082 | INFO | "huge_shared" : " 27 GB", | 20260506 12:30:50.082 | INFO | "malloc" : " 14 GB", | 20260506 12:30:50.082 | INFO | "pinned" : " 29 GB" | 20260506 12:30:50.082 | INFO | } | 20260506 12:30:50.082 | INFO | Perf report for the last 5.13942 seconds: | 20260506 12:30:50.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:30:50.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:50.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:50.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:50.082 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 12:30:50.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:50.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:50.082 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 12:30:50.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:50.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:50.082 | INFO | Uploaded 36GiB to GPU in 2.04006s (17.6466GiB/s) | 20260506 12:30:50.082 | INFO | Downloaded 34GiB to GPU in 1.94443s (17.4858GiB/s) | 20260506 12:30:55.221 | INFO | Analyzing chunk frames = [1370624,1371136) | 20260506 12:30:55.221 | INFO | MemInfo: { | 20260506 12:30:55.221 | INFO | "gpu" : " 17 GB", | 20260506 12:30:55.221 | INFO | "huge" : "", | 20260506 12:30:55.221 | INFO | "huge_shared" : " 27 GB", | 20260506 12:30:55.221 | INFO | "malloc" : " 14 GB", | 20260506 12:30:55.221 | INFO | "pinned" : " 29 GB" | 20260506 12:30:55.221 | INFO | } | 20260506 12:30:55.221 | INFO | Perf report for the last 5.13925 seconds: | 20260506 12:30:55.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:30:55.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:55.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:55.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:55.221 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 12:30:55.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:55.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:55.221 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260506 12:30:55.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:55.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:30:55.222 | INFO | Uploaded 36GiB to GPU in 2.05973s (17.478GiB/s) | 20260506 12:30:55.222 | INFO | Downloaded 34GiB to GPU in 1.93211s (17.5974GiB/s) | 20260506 12:31:00.361 | INFO | Analyzing chunk frames = [1371136,1371648) | 20260506 12:31:00.361 | INFO | MemInfo: { | 20260506 12:31:00.361 | INFO | "gpu" : " 17 GB", | 20260506 12:31:00.361 | INFO | "huge" : "", | 20260506 12:31:00.361 | INFO | "huge_shared" : " 27 GB", | 20260506 12:31:00.361 | INFO | "malloc" : " 14 GB", | 20260506 12:31:00.361 | INFO | "pinned" : " 29 GB" | 20260506 12:31:00.361 | INFO | } | 20260506 12:31:00.361 | INFO | Perf report for the last 5.13976 seconds: | 20260506 12:31:00.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:31:00.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:00.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:00.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:00.361 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:31:00.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:00.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:00.361 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 12:31:00.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:00.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:00.361 | INFO | Uploaded 36GiB to GPU in 2.00195s (17.9824GiB/s) | 20260506 12:31:00.361 | INFO | Downloaded 34GiB to GPU in 2.02633s (16.7791GiB/s) | 20260506 12:31:05.500 | INFO | Analyzing chunk frames = [1371648,1372160) | 20260506 12:31:05.500 | INFO | MemInfo: { | 20260506 12:31:05.500 | INFO | "gpu" : " 17 GB", | 20260506 12:31:05.500 | INFO | "huge" : "", | 20260506 12:31:05.500 | INFO | "huge_shared" : " 27 GB", | 20260506 12:31:05.500 | INFO | "malloc" : " 14 GB", | 20260506 12:31:05.500 | INFO | "pinned" : " 29 GB" | 20260506 12:31:05.500 | INFO | } | 20260506 12:31:05.500 | INFO | Perf report for the last 5.13906 seconds: | 20260506 12:31:05.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:31:05.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:05.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:05.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:05.500 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 12:31:05.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:05.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:05.500 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:31:05.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:05.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:05.500 | INFO | Uploaded 36GiB to GPU in 1.99338s (18.0598GiB/s) | 20260506 12:31:05.500 | INFO | Downloaded 34GiB to GPU in 1.97629s (17.2039GiB/s) | 20260506 12:31:05.633 | INFO | Starting a round of Baz IO | 20260506 12:31:08.793 | INFO | Populating RTMetrics report | 20260506 12:31:08.793 | INFO | RTMetrics report populated | 20260506 12:31:08.793 | INFO | Opening tmp RTMetrics json file | 20260506 12:31:08.794 | INFO | Serializing RTMetrics to json | 20260506 12:31:08.795 | INFO | Serialization done, swapping tmp file now... | 20260506 12:31:08.795 | INFO | RTMetrics done | 20260506 12:31:08.892 | INFO | Finished a round of Baz IO: Wrote 2.091 GiB in 3.259 seconds | 20260506 12:31:08.892 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.43% / 0.02% / 58.55% | 20260506 12:31:08.892 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.90339% / 4.37101e-05% / 5.16574e-05% | 20260506 12:31:08.892 | INFO | Baz IO overall: | 20260506 12:31:08.892 | INFO | BytesWritten : 682552042863 | 20260506 12:31:08.892 | INFO | NumEvents : 329875327860 | 20260506 12:31:10.639 | INFO | Analyzing chunk frames = [1372160,1372672) | 20260506 12:31:10.639 | INFO | MemInfo: { | 20260506 12:31:10.639 | INFO | "gpu" : " 17 GB", | 20260506 12:31:10.639 | INFO | "huge" : "", | 20260506 12:31:10.639 | INFO | "huge_shared" : " 27 GB", | 20260506 12:31:10.639 | INFO | "malloc" : " 14 GB", | 20260506 12:31:10.639 | INFO | "pinned" : " 29 GB" | 20260506 12:31:10.639 | INFO | } | 20260506 12:31:10.639 | INFO | Perf report for the last 5.13903 seconds: | 20260506 12:31:10.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:31:10.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:10.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:10.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:10.639 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 12:31:10.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:10.639 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:31:10.639 | INFO | PRE_HQ : 61.5%, 1.9, 61.5ms, 0 | 20260506 12:31:10.639 | INFO | BAZWRITER : 62.7%, 3.7, 119.5ms, 0 | 20260506 12:31:10.639 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 12:31:10.640 | INFO | Uploaded 36GiB to GPU in 2.05955s (17.4796GiB/s) | 20260506 12:31:10.640 | INFO | Downloaded 38GiB to GPU in 2.20954s (17.1981GiB/s) | 20260506 12:31:15.778 | INFO | Analyzing chunk frames = [1372672,1373184) | 20260506 12:31:15.778 | INFO | MemInfo: { | 20260506 12:31:15.778 | INFO | "gpu" : " 17 GB", | 20260506 12:31:15.778 | INFO | "huge" : "", | 20260506 12:31:15.778 | INFO | "huge_shared" : " 27 GB", | 20260506 12:31:15.778 | INFO | "malloc" : " 14 GB", | 20260506 12:31:15.779 | INFO | "pinned" : " 29 GB" | 20260506 12:31:15.779 | INFO | } | 20260506 12:31:15.779 | INFO | Perf report for the last 5.13934 seconds: | 20260506 12:31:15.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:31:15.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:15.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:15.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:15.779 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:31:15.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:15.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:15.779 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:31:15.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:15.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:15.779 | INFO | Uploaded 36GiB to GPU in 2.01403s (17.8746GiB/s) | 20260506 12:31:15.779 | INFO | Downloaded 34GiB to GPU in 1.92975s (17.6188GiB/s) | 20260506 12:31:20.920 | INFO | Analyzing chunk frames = [1373184,1373696) | 20260506 12:31:20.920 | INFO | MemInfo: { | 20260506 12:31:20.920 | INFO | "gpu" : " 17 GB", | 20260506 12:31:20.920 | INFO | "huge" : "", | 20260506 12:31:20.921 | INFO | "huge_shared" : " 27 GB", | 20260506 12:31:20.921 | INFO | "malloc" : " 14 GB", | 20260506 12:31:20.921 | INFO | "pinned" : " 29 GB" | 20260506 12:31:20.921 | INFO | } | 20260506 12:31:20.921 | INFO | Perf report for the last 5.14201 seconds: | 20260506 12:31:20.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:31:20.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:20.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:20.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:20.921 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 12:31:20.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:20.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:20.921 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:31:20.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:20.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:20.921 | INFO | Uploaded 36GiB to GPU in 1.98345s (18.1502GiB/s) | 20260506 12:31:20.921 | INFO | Downloaded 34GiB to GPU in 1.93779s (17.5458GiB/s) | 20260506 12:31:26.060 | INFO | Analyzing chunk frames = [1373696,1374208) | 20260506 12:31:26.060 | INFO | MemInfo: { | 20260506 12:31:26.060 | INFO | "gpu" : " 17 GB", | 20260506 12:31:26.060 | INFO | "huge" : "", | 20260506 12:31:26.061 | INFO | "huge_shared" : " 27 GB", | 20260506 12:31:26.061 | INFO | "malloc" : " 14 GB", | 20260506 12:31:26.061 | INFO | "pinned" : " 29 GB" | 20260506 12:31:26.061 | INFO | } | 20260506 12:31:26.061 | INFO | Perf report for the last 5.14 seconds: | 20260506 12:31:26.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:31:26.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:26.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:26.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:26.061 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260506 12:31:26.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:26.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:26.061 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:31:26.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:26.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:26.061 | INFO | Uploaded 36GiB to GPU in 2.05067s (17.5552GiB/s) | 20260506 12:31:26.061 | INFO | Downloaded 34GiB to GPU in 1.93274s (17.5916GiB/s) | 20260506 12:31:31.197 | INFO | Analyzing chunk frames = [1374208,1374720) | 20260506 12:31:31.197 | INFO | MemInfo: { | 20260506 12:31:31.197 | INFO | "gpu" : " 17 GB", | 20260506 12:31:31.197 | INFO | "huge" : "", | 20260506 12:31:31.197 | INFO | "huge_shared" : " 27 GB", | 20260506 12:31:31.197 | INFO | "malloc" : " 14 GB", | 20260506 12:31:31.197 | INFO | "pinned" : " 29 GB" | 20260506 12:31:31.197 | INFO | } | 20260506 12:31:31.197 | INFO | Perf report for the last 5.13615 seconds: | 20260506 12:31:31.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:31:31.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:31.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:31.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:31.197 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 12:31:31.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:31.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:31.197 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:31:31.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:31.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:31.197 | INFO | Uploaded 36GiB to GPU in 2.01131s (17.8987GiB/s) | 20260506 12:31:31.197 | INFO | Downloaded 34GiB to GPU in 1.9688s (17.2694GiB/s) | 20260506 12:31:36.336 | INFO | Analyzing chunk frames = [1374720,1375232) | 20260506 12:31:36.336 | INFO | MemInfo: { | 20260506 12:31:36.336 | INFO | "gpu" : " 17 GB", | 20260506 12:31:36.336 | INFO | "huge" : "", | 20260506 12:31:36.336 | INFO | "huge_shared" : " 27 GB", | 20260506 12:31:36.336 | INFO | "malloc" : " 14 GB", | 20260506 12:31:36.336 | INFO | "pinned" : " 29 GB" | 20260506 12:31:36.336 | INFO | } | 20260506 12:31:36.336 | INFO | Perf report for the last 5.13905 seconds: | 20260506 12:31:36.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:31:36.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:36.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:36.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:36.336 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:31:36.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:36.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:36.336 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260506 12:31:36.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:36.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:36.336 | INFO | Uploaded 36GiB to GPU in 2.08413s (17.2734GiB/s) | 20260506 12:31:36.336 | INFO | Downloaded 34GiB to GPU in 1.97836s (17.1859GiB/s) | 20260506 12:31:41.476 | INFO | Analyzing chunk frames = [1375232,1375744) | 20260506 12:31:41.476 | INFO | MemInfo: { | 20260506 12:31:41.476 | INFO | "gpu" : " 17 GB", | 20260506 12:31:41.476 | INFO | "huge" : "", | 20260506 12:31:41.476 | INFO | "huge_shared" : " 27 GB", | 20260506 12:31:41.476 | INFO | "malloc" : " 14 GB", | 20260506 12:31:41.476 | INFO | "pinned" : " 29 GB" | 20260506 12:31:41.476 | INFO | } | 20260506 12:31:41.476 | INFO | Perf report for the last 5.14006 seconds: | 20260506 12:31:41.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:31:41.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:41.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:41.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:41.476 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 12:31:41.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:41.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:41.476 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:31:41.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:41.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:41.476 | INFO | Uploaded 36GiB to GPU in 2.01471s (17.8685GiB/s) | 20260506 12:31:41.476 | INFO | Downloaded 34GiB to GPU in 2.01294s (16.8907GiB/s) | 20260506 12:31:46.615 | INFO | Analyzing chunk frames = [1375744,1376256) | 20260506 12:31:46.615 | INFO | MemInfo: { | 20260506 12:31:46.615 | INFO | "gpu" : " 17 GB", | 20260506 12:31:46.615 | INFO | "huge" : "", | 20260506 12:31:46.615 | INFO | "huge_shared" : " 27 GB", | 20260506 12:31:46.615 | INFO | "malloc" : " 14 GB", | 20260506 12:31:46.615 | INFO | "pinned" : " 29 GB" | 20260506 12:31:46.615 | INFO | } | 20260506 12:31:46.615 | INFO | Perf report for the last 5.13886 seconds: | 20260506 12:31:46.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:31:46.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:46.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:46.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:46.615 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 12:31:46.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:46.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:46.615 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 12:31:46.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:46.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:46.615 | INFO | Uploaded 36GiB to GPU in 2.06559s (17.4284GiB/s) | 20260506 12:31:46.615 | INFO | Downloaded 34GiB to GPU in 1.99581s (17.0356GiB/s) | 20260506 12:31:46.615 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:31:46.615 | INFO | Total mem usage: 16.8 GiB | 20260506 12:31:46.615 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:31:46.615 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.7%) | 20260506 12:31:46.615 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:31:46.615 | INFO | High Water Breakdown: | 20260506 12:31:46.615 | INFO | 59.6% : 8.59 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:31:46.615 | INFO | 40.2% : 5.8 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:31:46.615 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:31:46.615 | INFO | ------------------------------------------ | 20260506 12:31:46.615 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:31:46.615 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:31:46.615 | INFO | Total mem usage: 26.5 GiB | 20260506 12:31:46.615 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:31:46.615 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:31:46.615 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:31:46.615 | INFO | High Water Breakdown: | 20260506 12:31:46.615 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:31:46.615 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:31:46.615 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:31:46.615 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:31:46.615 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:31:46.615 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:31:46.615 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:31:46.615 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:31:46.615 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:31:46.615 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:31:46.615 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:31:46.615 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:31:46.615 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:31:46.615 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:31:46.615 | INFO | ------------------------------------------ | 20260506 12:31:46.615 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:31:46.615 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:31:46.615 | INFO | Total mem usage: 25.6 GiB | 20260506 12:31:46.615 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:31:46.616 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:31:46.616 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:31:46.616 | INFO | High Water Breakdown: | 20260506 12:31:46.616 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:31:46.616 | INFO | ------------------------------------------ | 20260506 12:31:46.616 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:31:46.616 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:31:46.616 | INFO | Total mem usage: 15.7 GiB | 20260506 12:31:46.616 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:31:46.616 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:31:46.616 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:31:46.616 | INFO | High Water Breakdown: | 20260506 12:31:46.616 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:31:46.616 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:31:46.616 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:31:46.616 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:31:46.616 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:31:46.616 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:31:46.616 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:31:46.616 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:31:46.616 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:31:46.616 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:31:46.616 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:31:46.616 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:31:46.616 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:31:46.616 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:31:46.616 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:31:46.616 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:31:46.616 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:31:46.616 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:31:46.616 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:31:46.616 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:31:46.616 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:31:46.616 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:31:46.616 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:31:46.616 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:31:46.616 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:31:46.616 | INFO | ------------------------------------------ | 20260506 12:31:46.616 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:31:46.616 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:31:46.616 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:31:46.616 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:31:46.616 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.65e+01, 1.36e+01 / 2.65e+00 | 20260506 12:31:46.616 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.01e-01, 1.10e-02 / 8.73e-03 | 20260506 12:31:46.616 | INFO | PerfReport: 11.54%, Baseline -- 3.67e+00 / 4.40e+00, 5.78e-01 / 3.67e-01 | 20260506 12:31:46.616 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.45e+00, 1.92e-01 / 1.59e-01 | 20260506 12:31:46.616 | INFO | PerfReport: 15.85%, DME -- 4.03e+01 / 4.50e+01, 4.25e+00 / 3.12e+00 | 20260506 12:31:46.616 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.80e+01 / 2.05e+01, 1.51e+00 / 1.88e+00 | 20260506 12:31:46.616 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.39e+00 / 3.03e+00, 4.14e-01 / 2.11e-01 | 20260506 12:31:46.616 | INFO | PerfReport: 3.73%, Metrics -- 1.19e+00 / 1.56e+00, 3.06e-01 / 2.15e-01 | 20260506 12:31:46.616 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.18e-01 / 4.75e-01, 6.61e-02 / 3.37e-02 | 20260506 12:31:46.616 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:31:46.616 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:31:46.616 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:31:46.616 | INFO | PerfReport: Full Execution -- 9.89e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 12:31:46.616 | INFO | PerfReport: 21.42%, Upload -- 2.12e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260506 12:31:46.616 | INFO | PerfReport: 20.99%, Download -- 2.08e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 12:31:46.616 | INFO | PerfReport: 32.17%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 12:31:46.616 | INFO | PerfReport: 1.80%, LockUpload -- 1.78e+00 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 12:31:46.616 | INFO | PerfReport: 1.05%, LockDownload -- 1.04e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260506 12:31:46.616 | INFO | PerfReport: 22.56%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:31:46.616 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260506 12:31:46.743 | INFO | Starting a round of Baz IO | 20260506 12:31:49.910 | INFO | Populating RTMetrics report | 20260506 12:31:49.910 | INFO | RTMetrics report populated | 20260506 12:31:49.910 | INFO | Opening tmp RTMetrics json file | 20260506 12:31:49.912 | INFO | Serializing RTMetrics to json | 20260506 12:31:49.913 | INFO | Serialization done, swapping tmp file now... | 20260506 12:31:49.913 | INFO | RTMetrics done | 20260506 12:31:50.010 | INFO | Finished a round of Baz IO: Wrote 2.092 GiB in 3.267 seconds | 20260506 12:31:50.010 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.40% / 0.02% / 58.57% | 20260506 12:31:50.010 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.90118% / 6.35783e-05% / 6.35783e-05% | 20260506 12:31:50.010 | INFO | Baz IO overall: | 20260506 12:31:50.010 | INFO | BytesWritten : 684798756360 | 20260506 12:31:50.010 | INFO | NumEvents : 330965182164 | 20260506 12:31:51.754 | INFO | Analyzing chunk frames = [1376256,1376768) | 20260506 12:31:51.754 | INFO | MemInfo: { | 20260506 12:31:51.754 | INFO | "gpu" : " 17 GB", | 20260506 12:31:51.754 | INFO | "huge" : "", | 20260506 12:31:51.754 | INFO | "huge_shared" : " 27 GB", | 20260506 12:31:51.754 | INFO | "malloc" : " 14 GB", | 20260506 12:31:51.754 | INFO | "pinned" : " 29 GB" | 20260506 12:31:51.754 | INFO | } | 20260506 12:31:51.754 | INFO | Perf report for the last 5.13934 seconds: | 20260506 12:31:51.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:31:51.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:51.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:51.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:51.754 | INFO | ANALYSIS : 63.5%, 3.0, 98.5ms, 0 | 20260506 12:31:51.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:51.754 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 12:31:51.754 | INFO | PRE_HQ : 61.6%, 1.9, 61.5ms, 0 | 20260506 12:31:51.754 | INFO | BAZWRITER : 62.5%, 3.7, 118.6ms, 0 | 20260506 12:31:51.754 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:31:51.754 | INFO | Uploaded 36GiB to GPU in 2.0926s (17.2035GiB/s) | 20260506 12:31:51.754 | INFO | Downloaded 38GiB to GPU in 2.22012s (17.1162GiB/s) | 20260506 12:31:56.893 | INFO | Analyzing chunk frames = [1376768,1377280) | 20260506 12:31:56.893 | INFO | MemInfo: { | 20260506 12:31:56.893 | INFO | "gpu" : " 17 GB", | 20260506 12:31:56.893 | INFO | "huge" : "", | 20260506 12:31:56.894 | INFO | "huge_shared" : " 27 GB", | 20260506 12:31:56.894 | INFO | "malloc" : " 14 GB", | 20260506 12:31:56.894 | INFO | "pinned" : " 29 GB" | 20260506 12:31:56.894 | INFO | } | 20260506 12:31:56.894 | INFO | Perf report for the last 5.13941 seconds: | 20260506 12:31:56.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:31:56.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:56.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:56.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:56.894 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 12:31:56.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:56.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:56.894 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:31:56.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:56.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:31:56.894 | INFO | Uploaded 36GiB to GPU in 2.01104s (17.9012GiB/s) | 20260506 12:31:56.894 | INFO | Downloaded 34GiB to GPU in 1.92743s (17.6401GiB/s) | 20260506 12:32:02.033 | INFO | Analyzing chunk frames = [1377280,1377792) | 20260506 12:32:02.033 | INFO | MemInfo: { | 20260506 12:32:02.033 | INFO | "gpu" : " 17 GB", | 20260506 12:32:02.033 | INFO | "huge" : "", | 20260506 12:32:02.033 | INFO | "huge_shared" : " 27 GB", | 20260506 12:32:02.033 | INFO | "malloc" : " 14 GB", | 20260506 12:32:02.033 | INFO | "pinned" : " 29 GB" | 20260506 12:32:02.033 | INFO | } | 20260506 12:32:02.033 | INFO | Perf report for the last 5.13925 seconds: | 20260506 12:32:02.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:32:02.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:02.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:02.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:02.033 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 12:32:02.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:02.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:02.033 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:32:02.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:02.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:02.033 | INFO | Uploaded 36GiB to GPU in 2.00847s (17.9241GiB/s) | 20260506 12:32:02.033 | INFO | Downloaded 34GiB to GPU in 1.94348s (17.4944GiB/s) | 20260506 12:32:07.172 | INFO | Analyzing chunk frames = [1377792,1378304) | 20260506 12:32:07.172 | INFO | MemInfo: { | 20260506 12:32:07.172 | INFO | "gpu" : " 17 GB", | 20260506 12:32:07.172 | INFO | "huge" : "", | 20260506 12:32:07.173 | INFO | "huge_shared" : " 27 GB", | 20260506 12:32:07.173 | INFO | "malloc" : " 14 GB", | 20260506 12:32:07.173 | INFO | "pinned" : " 29 GB" | 20260506 12:32:07.173 | INFO | } | 20260506 12:32:07.173 | INFO | Perf report for the last 5.13983 seconds: | 20260506 12:32:07.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:32:07.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:07.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:07.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:07.173 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 12:32:07.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:07.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:07.173 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:32:07.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:07.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:07.173 | INFO | Uploaded 36GiB to GPU in 1.9891s (18.0986GiB/s) | 20260506 12:32:07.173 | INFO | Downloaded 34GiB to GPU in 1.94112s (17.5156GiB/s) | 20260506 12:32:12.311 | INFO | Analyzing chunk frames = [1378304,1378816) | 20260506 12:32:12.311 | INFO | MemInfo: { | 20260506 12:32:12.311 | INFO | "gpu" : " 17 GB", | 20260506 12:32:12.311 | INFO | "huge" : "", | 20260506 12:32:12.312 | INFO | "huge_shared" : " 27 GB", | 20260506 12:32:12.312 | INFO | "malloc" : " 14 GB", | 20260506 12:32:12.312 | INFO | "pinned" : " 29 GB" | 20260506 12:32:12.312 | INFO | } | 20260506 12:32:12.312 | INFO | Perf report for the last 5.1389 seconds: | 20260506 12:32:12.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:32:12.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:12.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:12.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:12.312 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 12:32:12.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:12.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:12.312 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260506 12:32:12.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:12.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:12.312 | INFO | Uploaded 36GiB to GPU in 1.97425s (18.2348GiB/s) | 20260506 12:32:12.312 | INFO | Downloaded 34GiB to GPU in 1.95027s (17.4334GiB/s) | 20260506 12:32:17.451 | INFO | Analyzing chunk frames = [1378816,1379328) | 20260506 12:32:17.451 | INFO | MemInfo: { | 20260506 12:32:17.451 | INFO | "gpu" : " 17 GB", | 20260506 12:32:17.451 | INFO | "huge" : "", | 20260506 12:32:17.451 | INFO | "huge_shared" : " 27 GB", | 20260506 12:32:17.451 | INFO | "malloc" : " 14 GB", | 20260506 12:32:17.451 | INFO | "pinned" : " 29 GB" | 20260506 12:32:17.451 | INFO | } | 20260506 12:32:17.451 | INFO | Perf report for the last 5.13933 seconds: | 20260506 12:32:17.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:32:17.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:17.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:17.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:17.451 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 12:32:17.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:17.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:17.451 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:32:17.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:17.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:17.451 | INFO | Uploaded 36GiB to GPU in 2.06982s (17.3928GiB/s) | 20260506 12:32:17.451 | INFO | Downloaded 34GiB to GPU in 1.97281s (17.2343GiB/s) | 20260506 12:32:22.591 | INFO | Analyzing chunk frames = [1379328,1379840) | 20260506 12:32:22.591 | INFO | MemInfo: { | 20260506 12:32:22.591 | INFO | "gpu" : " 17 GB", | 20260506 12:32:22.591 | INFO | "huge" : "", | 20260506 12:32:22.591 | INFO | "huge_shared" : " 27 GB", | 20260506 12:32:22.591 | INFO | "malloc" : " 14 GB", | 20260506 12:32:22.591 | INFO | "pinned" : " 29 GB" | 20260506 12:32:22.591 | INFO | } | 20260506 12:32:22.591 | INFO | Perf report for the last 5.13981 seconds: | 20260506 12:32:22.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:32:22.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:22.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:22.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:22.591 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:32:22.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:22.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:22.591 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 12:32:22.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:22.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:22.591 | INFO | Uploaded 36GiB to GPU in 1.96391s (18.3308GiB/s) | 20260506 12:32:22.591 | INFO | Downloaded 34GiB to GPU in 2.03154s (16.736GiB/s) | 20260506 12:32:27.729 | INFO | Analyzing chunk frames = [1379840,1380352) | 20260506 12:32:27.729 | INFO | MemInfo: { | 20260506 12:32:27.729 | INFO | "gpu" : " 17 GB", | 20260506 12:32:27.729 | INFO | "huge" : "", | 20260506 12:32:27.729 | INFO | "huge_shared" : " 27 GB", | 20260506 12:32:27.729 | INFO | "malloc" : " 14 GB", | 20260506 12:32:27.730 | INFO | "pinned" : " 29 GB" | 20260506 12:32:27.730 | INFO | } | 20260506 12:32:27.730 | INFO | Perf report for the last 5.1388 seconds: | 20260506 12:32:27.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:32:27.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:27.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:27.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:27.730 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260506 12:32:27.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:27.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:27.730 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:32:27.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:27.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:27.730 | INFO | Uploaded 36GiB to GPU in 2.02816s (17.75GiB/s) | 20260506 12:32:27.730 | INFO | Downloaded 34GiB to GPU in 1.96298s (17.3206GiB/s) | 20260506 12:32:27.856 | INFO | Starting a round of Baz IO | 20260506 12:32:31.018 | INFO | Populating RTMetrics report | 20260506 12:32:31.018 | INFO | RTMetrics report populated | 20260506 12:32:31.018 | INFO | Opening tmp RTMetrics json file | 20260506 12:32:31.019 | INFO | Serializing RTMetrics to json | 20260506 12:32:31.020 | INFO | Serialization done, swapping tmp file now... | 20260506 12:32:31.020 | INFO | RTMetrics done | 20260506 12:32:31.124 | INFO | Finished a round of Baz IO: Wrote 2.092 GiB in 3.267 seconds | 20260506 12:32:31.124 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.38% / 0.02% / 58.60% | 20260506 12:32:31.124 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.88531% / 3.17891e-05% / 4.37101e-05% | 20260506 12:32:31.124 | INFO | Baz IO overall: | 20260506 12:32:31.124 | INFO | BytesWritten : 687044906319 | 20260506 12:32:31.124 | INFO | NumEvents : 332054438749 | 20260506 12:32:32.869 | INFO | Analyzing chunk frames = [1380352,1380864) | 20260506 12:32:32.869 | INFO | MemInfo: { | 20260506 12:32:32.869 | INFO | "gpu" : " 17 GB", | 20260506 12:32:32.869 | INFO | "huge" : "", | 20260506 12:32:32.869 | INFO | "huge_shared" : " 27 GB", | 20260506 12:32:32.869 | INFO | "malloc" : " 14 GB", | 20260506 12:32:32.869 | INFO | "pinned" : " 29 GB" | 20260506 12:32:32.869 | INFO | } | 20260506 12:32:32.869 | INFO | Perf report for the last 5.13952 seconds: | 20260506 12:32:32.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:32:32.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:32.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:32.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:32.869 | INFO | ANALYSIS : 63.4%, 3.0, 98.2ms, 0 | 20260506 12:32:32.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:32.869 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:32:32.869 | INFO | PRE_HQ : 61.7%, 1.9, 61.7ms, 0 | 20260506 12:32:32.869 | INFO | BAZWRITER : 63.0%, 3.8, 122.4ms, 0 | 20260506 12:32:32.869 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:32:32.870 | INFO | Uploaded 36GiB to GPU in 2.07492s (17.3501GiB/s) | 20260506 12:32:32.870 | INFO | Downloaded 38GiB to GPU in 2.21207s (17.1785GiB/s) | 20260506 12:32:38.008 | INFO | Analyzing chunk frames = [1380864,1381376) | 20260506 12:32:38.008 | INFO | MemInfo: { | 20260506 12:32:38.008 | INFO | "gpu" : " 17 GB", | 20260506 12:32:38.008 | INFO | "huge" : "", | 20260506 12:32:38.008 | INFO | "huge_shared" : " 27 GB", | 20260506 12:32:38.008 | INFO | "malloc" : " 14 GB", | 20260506 12:32:38.008 | INFO | "pinned" : " 29 GB" | 20260506 12:32:38.008 | INFO | } | 20260506 12:32:38.008 | INFO | Perf report for the last 5.13916 seconds: | 20260506 12:32:38.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:32:38.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:38.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:38.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:38.008 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:32:38.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:38.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:38.008 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:32:38.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:38.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:38.008 | INFO | Uploaded 36GiB to GPU in 2.07447s (17.3538GiB/s) | 20260506 12:32:38.009 | INFO | Downloaded 34GiB to GPU in 1.94647s (17.4675GiB/s) | 20260506 12:32:43.148 | INFO | Analyzing chunk frames = [1381376,1381888) | 20260506 12:32:43.148 | INFO | MemInfo: { | 20260506 12:32:43.148 | INFO | "gpu" : " 17 GB", | 20260506 12:32:43.148 | INFO | "huge" : "", | 20260506 12:32:43.148 | INFO | "huge_shared" : " 27 GB", | 20260506 12:32:43.148 | INFO | "malloc" : " 14 GB", | 20260506 12:32:43.148 | INFO | "pinned" : " 29 GB" | 20260506 12:32:43.148 | INFO | } | 20260506 12:32:43.148 | INFO | Perf report for the last 5.13968 seconds: | 20260506 12:32:43.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:32:43.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:43.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:43.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:43.148 | INFO | ANALYSIS : 62.4%, 3.0, 96.6ms, 0 | 20260506 12:32:43.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:43.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:43.148 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:32:43.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:43.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:43.148 | INFO | Uploaded 36GiB to GPU in 2.0806s (17.3027GiB/s) | 20260506 12:32:43.148 | INFO | Downloaded 34GiB to GPU in 1.95435s (17.3971GiB/s) | 20260506 12:32:48.287 | INFO | Analyzing chunk frames = [1381888,1382400) | 20260506 12:32:48.287 | INFO | MemInfo: { | 20260506 12:32:48.287 | INFO | "gpu" : " 17 GB", | 20260506 12:32:48.287 | INFO | "huge" : "", | 20260506 12:32:48.287 | INFO | "huge_shared" : " 27 GB", | 20260506 12:32:48.287 | INFO | "malloc" : " 14 GB", | 20260506 12:32:48.287 | INFO | "pinned" : " 29 GB" | 20260506 12:32:48.287 | INFO | } | 20260506 12:32:48.287 | INFO | Perf report for the last 5.1391 seconds: | 20260506 12:32:48.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:32:48.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:48.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:48.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:48.287 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 12:32:48.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:48.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:48.287 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 12:32:48.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:48.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:48.287 | INFO | Uploaded 36GiB to GPU in 2.01666s (17.8513GiB/s) | 20260506 12:32:48.287 | INFO | Downloaded 34GiB to GPU in 1.93845s (17.5398GiB/s) | 20260506 12:32:53.426 | INFO | Analyzing chunk frames = [1382400,1382912) | 20260506 12:32:53.426 | INFO | MemInfo: { | 20260506 12:32:53.426 | INFO | "gpu" : " 17 GB", | 20260506 12:32:53.426 | INFO | "huge" : "", | 20260506 12:32:53.426 | INFO | "huge_shared" : " 27 GB", | 20260506 12:32:53.426 | INFO | "malloc" : " 14 GB", | 20260506 12:32:53.426 | INFO | "pinned" : " 29 GB" | 20260506 12:32:53.426 | INFO | } | 20260506 12:32:53.426 | INFO | Perf report for the last 5.13912 seconds: | 20260506 12:32:53.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:32:53.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:53.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:53.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:53.426 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 12:32:53.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:53.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:53.426 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 12:32:53.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:53.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:53.426 | INFO | Uploaded 36GiB to GPU in 2.01178s (17.8946GiB/s) | 20260506 12:32:53.426 | INFO | Downloaded 34GiB to GPU in 1.9438s (17.4915GiB/s) | 20260506 12:32:58.566 | INFO | Analyzing chunk frames = [1382912,1383424) | 20260506 12:32:58.566 | INFO | MemInfo: { | 20260506 12:32:58.566 | INFO | "gpu" : " 17 GB", | 20260506 12:32:58.566 | INFO | "huge" : "", | 20260506 12:32:58.566 | INFO | "huge_shared" : " 27 GB", | 20260506 12:32:58.566 | INFO | "malloc" : " 14 GB", | 20260506 12:32:58.566 | INFO | "pinned" : " 29 GB" | 20260506 12:32:58.566 | INFO | } | 20260506 12:32:58.566 | INFO | Perf report for the last 5.13959 seconds: | 20260506 12:32:58.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:32:58.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:58.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:58.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:58.566 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 12:32:58.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:58.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:58.566 | INFO | PRE_HQ : 34.0%, 1.0, 17.7ms, 0 | 20260506 12:32:58.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:58.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:32:58.566 | INFO | Uploaded 36GiB to GPU in 2.05165s (17.5469GiB/s) | 20260506 12:32:58.566 | INFO | Downloaded 34GiB to GPU in 1.92728s (17.6415GiB/s) | 20260506 12:33:03.705 | INFO | Analyzing chunk frames = [1383424,1383936) | 20260506 12:33:03.705 | INFO | MemInfo: { | 20260506 12:33:03.705 | INFO | "gpu" : " 17 GB", | 20260506 12:33:03.705 | INFO | "huge" : "", | 20260506 12:33:03.705 | INFO | "huge_shared" : " 27 GB", | 20260506 12:33:03.705 | INFO | "malloc" : " 14 GB", | 20260506 12:33:03.705 | INFO | "pinned" : " 29 GB" | 20260506 12:33:03.705 | INFO | } | 20260506 12:33:03.705 | INFO | Perf report for the last 5.13912 seconds: | 20260506 12:33:03.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:33:03.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:03.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:03.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:03.705 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:33:03.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:03.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:03.705 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 12:33:03.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:03.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:03.705 | INFO | Uploaded 36GiB to GPU in 1.98557s (18.1308GiB/s) | 20260506 12:33:03.705 | INFO | Downloaded 34GiB to GPU in 2.02813s (16.7642GiB/s) | 20260506 12:33:08.844 | INFO | Analyzing chunk frames = [1383936,1384448) | 20260506 12:33:08.844 | INFO | MemInfo: { | 20260506 12:33:08.844 | INFO | "gpu" : " 17 GB", | 20260506 12:33:08.844 | INFO | "huge" : "", | 20260506 12:33:08.844 | INFO | "huge_shared" : " 27 GB", | 20260506 12:33:08.844 | INFO | "malloc" : " 14 GB", | 20260506 12:33:08.844 | INFO | "pinned" : " 29 GB" | 20260506 12:33:08.844 | INFO | } | 20260506 12:33:08.844 | INFO | Perf report for the last 5.1394 seconds: | 20260506 12:33:08.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:33:08.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:08.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:08.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:08.844 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:33:08.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:08.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:08.844 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:33:08.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:08.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:08.845 | INFO | Uploaded 36GiB to GPU in 2.01575s (17.8593GiB/s) | 20260506 12:33:08.845 | INFO | Downloaded 34GiB to GPU in 1.96323s (17.3184GiB/s) | 20260506 12:33:08.845 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:33:08.845 | INFO | Total mem usage: 16.8 GiB | 20260506 12:33:08.845 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:33:08.845 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.4%) | 20260506 12:33:08.845 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:33:08.845 | INFO | High Water Breakdown: | 20260506 12:33:08.845 | INFO | 59.7% : 8.62 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:33:08.845 | INFO | 40.2% : 5.8 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:33:08.845 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:33:08.845 | INFO | ------------------------------------------ | 20260506 12:33:08.845 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:33:08.845 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:33:08.845 | INFO | Total mem usage: 26.5 GiB | 20260506 12:33:08.845 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:33:08.845 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:33:08.845 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:33:08.845 | INFO | High Water Breakdown: | 20260506 12:33:08.845 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:33:08.845 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:33:08.845 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:33:08.845 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:33:08.845 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:33:08.845 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:33:08.845 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:33:08.845 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:33:08.845 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:33:08.845 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:33:08.845 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:33:08.845 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:33:08.845 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:33:08.845 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:33:08.845 | INFO | ------------------------------------------ | 20260506 12:33:08.845 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:33:08.845 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:33:08.845 | INFO | Total mem usage: 25.6 GiB | 20260506 12:33:08.845 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:33:08.845 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:33:08.845 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:33:08.846 | INFO | High Water Breakdown: | 20260506 12:33:08.846 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:33:08.846 | INFO | ------------------------------------------ | 20260506 12:33:08.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:33:08.846 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:33:08.846 | INFO | Total mem usage: 15.7 GiB | 20260506 12:33:08.846 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:33:08.846 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:33:08.846 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 12:33:08.846 | INFO | High Water Breakdown: | 20260506 12:33:08.846 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:33:08.846 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:33:08.846 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:33:08.846 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:33:08.846 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:33:08.846 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:33:08.846 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:33:08.846 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:33:08.846 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:33:08.846 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:33:08.846 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:33:08.846 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:33:08.846 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:33:08.846 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:33:08.846 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:33:08.846 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:33:08.846 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:33:08.846 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:33:08.846 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:33:08.846 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:33:08.846 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:33:08.846 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:33:08.846 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:33:08.846 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:33:08.846 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:33:08.846 | INFO | ------------------------------------------ | 20260506 12:33:08.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:33:08.846 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:33:08.846 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:33:08.846 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:33:08.846 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.65e+01, 1.38e+01 / 2.53e+00 | 20260506 12:33:08.846 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.04e-02 / 1.12e-02 | 20260506 12:33:08.846 | INFO | PerfReport: 11.55%, Baseline -- 3.67e+00 / 4.43e+00, 5.77e-01 / 3.43e-01 | 20260506 12:33:08.846 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.48e+00, 1.94e-01 / 1.34e-01 | 20260506 12:33:08.846 | INFO | PerfReport: 15.95%, DME -- 4.05e+01 / 4.56e+01, 4.59e+00 / 2.92e+00 | 20260506 12:33:08.846 | INFO | PerfReport: 56.33%, FrameLabeling -- 1.79e+01 / 1.98e+01, 1.38e+00 / 1.56e+00 | 20260506 12:33:08.846 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.41e+00 / 3.07e+00, 4.24e-01 / 1.97e-01 | 20260506 12:33:08.846 | INFO | PerfReport: 3.74%, Metrics -- 1.19e+00 / 1.56e+00, 3.05e-01 / 1.40e-01 | 20260506 12:33:08.846 | INFO | PerfReport: 1.32%, PulseSerializing -- 4.19e-01 / 4.77e-01, 6.74e-02 / 2.80e-02 | 20260506 12:33:08.846 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:33:08.846 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:33:08.846 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:33:08.846 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 12:33:08.846 | INFO | PerfReport: 21.39%, Upload -- 2.11e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 12:33:08.847 | INFO | PerfReport: 21.00%, Download -- 2.07e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 12:33:08.847 | INFO | PerfReport: 32.15%, Compute -- 3.18e+01 / 0.00e+00, 1.38e+01 / 0.00e+00 | 20260506 12:33:08.847 | INFO | PerfReport: 1.82%, LockUpload -- 1.79e+00 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 12:33:08.847 | INFO | PerfReport: 1.09%, LockDownload -- 1.08e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260506 12:33:08.847 | INFO | PerfReport: 22.53%, LockCompute -- 2.23e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 12:33:08.847 | INFO | PerfReport: 0.02%, Misc -- 1.52e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260506 12:33:08.975 | INFO | Starting a round of Baz IO | 20260506 12:33:12.133 | INFO | Populating RTMetrics report | 20260506 12:33:12.133 | INFO | RTMetrics report populated | 20260506 12:33:12.133 | INFO | Opening tmp RTMetrics json file | 20260506 12:33:12.134 | INFO | Serializing RTMetrics to json | 20260506 12:33:12.135 | INFO | Serialization done, swapping tmp file now... | 20260506 12:33:12.135 | INFO | RTMetrics done | 20260506 12:33:12.232 | INFO | Finished a round of Baz IO: Wrote 2.092 GiB in 3.257 seconds | 20260506 12:33:12.232 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.35% / 0.02% / 58.62% | 20260506 12:33:12.232 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.87254% / 3.97364e-05% / 4.76837e-05% | 20260506 12:33:12.232 | INFO | Baz IO overall: | 20260506 12:33:12.232 | INFO | BytesWritten : 689290988891 | 20260506 12:33:12.232 | INFO | NumEvents : 333143624375 | 20260506 12:33:13.984 | INFO | Analyzing chunk frames = [1384448,1384960) | 20260506 12:33:13.984 | INFO | MemInfo: { | 20260506 12:33:13.984 | INFO | "gpu" : " 17 GB", | 20260506 12:33:13.984 | INFO | "huge" : "", | 20260506 12:33:13.984 | INFO | "huge_shared" : " 27 GB", | 20260506 12:33:13.984 | INFO | "malloc" : " 14 GB", | 20260506 12:33:13.984 | INFO | "pinned" : " 29 GB" | 20260506 12:33:13.984 | INFO | } | 20260506 12:33:13.984 | INFO | Perf report for the last 5.13971 seconds: | 20260506 12:33:13.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:33:13.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:13.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:13.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:13.984 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260506 12:33:13.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:13.984 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:33:13.984 | INFO | PRE_HQ : 61.3%, 1.9, 61.3ms, 0 | 20260506 12:33:13.984 | INFO | BAZWRITER : 62.8%, 3.7, 119.3ms, 0 | 20260506 12:33:13.984 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:33:13.984 | INFO | Uploaded 36GiB to GPU in 2.06654s (17.4204GiB/s) | 20260506 12:33:13.984 | INFO | Downloaded 38GiB to GPU in 2.21763s (17.1354GiB/s) | 20260506 12:33:19.123 | INFO | Analyzing chunk frames = [1384960,1385472) | 20260506 12:33:19.123 | INFO | MemInfo: { | 20260506 12:33:19.124 | INFO | "gpu" : " 17 GB", | 20260506 12:33:19.124 | INFO | "huge" : "", | 20260506 12:33:19.124 | INFO | "huge_shared" : " 27 GB", | 20260506 12:33:19.124 | INFO | "malloc" : " 14 GB", | 20260506 12:33:19.124 | INFO | "pinned" : " 29 GB" | 20260506 12:33:19.124 | INFO | } | 20260506 12:33:19.124 | INFO | Perf report for the last 5.13963 seconds: | 20260506 12:33:19.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:33:19.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:19.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:19.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:19.124 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:33:19.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:19.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:19.124 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:33:19.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:19.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:19.124 | INFO | Uploaded 36GiB to GPU in 2.01342s (17.88GiB/s) | 20260506 12:33:19.124 | INFO | Downloaded 34GiB to GPU in 1.93094s (17.608GiB/s) | 20260506 12:33:24.262 | INFO | Analyzing chunk frames = [1385472,1385984) | 20260506 12:33:24.262 | INFO | MemInfo: { | 20260506 12:33:24.262 | INFO | "gpu" : " 17 GB", | 20260506 12:33:24.262 | INFO | "huge" : "", | 20260506 12:33:24.262 | INFO | "huge_shared" : " 27 GB", | 20260506 12:33:24.262 | INFO | "malloc" : " 14 GB", | 20260506 12:33:24.262 | INFO | "pinned" : " 29 GB" | 20260506 12:33:24.262 | INFO | } | 20260506 12:33:24.262 | INFO | Perf report for the last 5.13871 seconds: | 20260506 12:33:24.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:33:24.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:24.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:24.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:24.262 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 12:33:24.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:24.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:24.263 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:33:24.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:24.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:24.263 | INFO | Uploaded 36GiB to GPU in 1.99883s (18.0105GiB/s) | 20260506 12:33:24.263 | INFO | Downloaded 34GiB to GPU in 1.93315s (17.5879GiB/s) | 20260506 12:33:29.402 | INFO | Analyzing chunk frames = [1385984,1386496) | 20260506 12:33:29.402 | INFO | MemInfo: { | 20260506 12:33:29.402 | INFO | "gpu" : " 17 GB", | 20260506 12:33:29.402 | INFO | "huge" : "", | 20260506 12:33:29.402 | INFO | "huge_shared" : " 27 GB", | 20260506 12:33:29.402 | INFO | "malloc" : " 14 GB", | 20260506 12:33:29.402 | INFO | "pinned" : " 29 GB" | 20260506 12:33:29.402 | INFO | } | 20260506 12:33:29.402 | INFO | Perf report for the last 5.1398 seconds: | 20260506 12:33:29.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:33:29.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:29.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:29.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:29.402 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 12:33:29.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:29.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:29.402 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:33:29.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:29.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:29.403 | INFO | Uploaded 36GiB to GPU in 1.99127s (18.0789GiB/s) | 20260506 12:33:29.403 | INFO | Downloaded 34GiB to GPU in 1.94496s (17.4811GiB/s) | 20260506 12:33:34.541 | INFO | Analyzing chunk frames = [1386496,1387008) | 20260506 12:33:34.541 | INFO | MemInfo: { | 20260506 12:33:34.541 | INFO | "gpu" : " 17 GB", | 20260506 12:33:34.541 | INFO | "huge" : "", | 20260506 12:33:34.541 | INFO | "huge_shared" : " 27 GB", | 20260506 12:33:34.541 | INFO | "malloc" : " 14 GB", | 20260506 12:33:34.541 | INFO | "pinned" : " 29 GB" | 20260506 12:33:34.541 | INFO | } | 20260506 12:33:34.541 | INFO | Perf report for the last 5.13879 seconds: | 20260506 12:33:34.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:33:34.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:34.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:34.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:34.541 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 12:33:34.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:34.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:34.541 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260506 12:33:34.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:34.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:34.541 | INFO | Uploaded 36GiB to GPU in 2.03162s (17.7199GiB/s) | 20260506 12:33:34.541 | INFO | Downloaded 34GiB to GPU in 1.96002s (17.3467GiB/s) | 20260506 12:33:39.680 | INFO | Analyzing chunk frames = [1387008,1387520) | 20260506 12:33:39.680 | INFO | MemInfo: { | 20260506 12:33:39.680 | INFO | "gpu" : " 17 GB", | 20260506 12:33:39.681 | INFO | "huge" : "", | 20260506 12:33:39.681 | INFO | "huge_shared" : " 27 GB", | 20260506 12:33:39.681 | INFO | "malloc" : " 14 GB", | 20260506 12:33:39.681 | INFO | "pinned" : " 29 GB" | 20260506 12:33:39.681 | INFO | } | 20260506 12:33:39.681 | INFO | Perf report for the last 5.1395 seconds: | 20260506 12:33:39.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:33:39.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:39.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:39.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:39.681 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:33:39.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:39.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:39.681 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 12:33:39.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:39.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:39.681 | INFO | Uploaded 36GiB to GPU in 2.10538s (17.0991GiB/s) | 20260506 12:33:39.681 | INFO | Downloaded 34GiB to GPU in 1.99449s (17.0469GiB/s) | 20260506 12:33:44.821 | INFO | Analyzing chunk frames = [1387520,1388032) | 20260506 12:33:44.821 | INFO | MemInfo: { | 20260506 12:33:44.821 | INFO | "gpu" : " 17 GB", | 20260506 12:33:44.821 | INFO | "huge" : "", | 20260506 12:33:44.821 | INFO | "huge_shared" : " 27 GB", | 20260506 12:33:44.821 | INFO | "malloc" : " 14 GB", | 20260506 12:33:44.821 | INFO | "pinned" : " 29 GB" | 20260506 12:33:44.821 | INFO | } | 20260506 12:33:44.821 | INFO | Perf report for the last 5.14023 seconds: | 20260506 12:33:44.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:33:44.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:44.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:44.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:44.821 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:33:44.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:44.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:44.821 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 12:33:44.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:44.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:44.821 | INFO | Uploaded 36GiB to GPU in 1.99408s (18.0534GiB/s) | 20260506 12:33:44.821 | INFO | Downloaded 34GiB to GPU in 2.02796s (16.7656GiB/s) | 20260506 12:33:49.959 | INFO | Analyzing chunk frames = [1388032,1388544) | 20260506 12:33:49.959 | INFO | MemInfo: { | 20260506 12:33:49.959 | INFO | "gpu" : " 17 GB", | 20260506 12:33:49.959 | INFO | "huge" : "", | 20260506 12:33:49.959 | INFO | "huge_shared" : " 27 GB", | 20260506 12:33:49.959 | INFO | "malloc" : " 14 GB", | 20260506 12:33:49.959 | INFO | "pinned" : " 29 GB" | 20260506 12:33:49.959 | INFO | } | 20260506 12:33:49.959 | INFO | Perf report for the last 5.1386 seconds: | 20260506 12:33:49.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:33:49.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:49.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:49.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:49.959 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260506 12:33:49.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:49.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:49.960 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 12:33:49.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:49.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:49.960 | INFO | Uploaded 36GiB to GPU in 2.06768s (17.4108GiB/s) | 20260506 12:33:49.960 | INFO | Downloaded 34GiB to GPU in 2.0012s (16.9898GiB/s) | 20260506 12:33:50.082 | INFO | Starting a round of Baz IO | 20260506 12:33:53.257 | INFO | Populating RTMetrics report | 20260506 12:33:53.258 | INFO | RTMetrics report populated | 20260506 12:33:53.258 | INFO | Opening tmp RTMetrics json file | 20260506 12:33:53.258 | INFO | Serializing RTMetrics to json | 20260506 12:33:53.259 | INFO | Serialization done, swapping tmp file now... | 20260506 12:33:53.260 | INFO | RTMetrics done | 20260506 12:33:53.354 | INFO | Finished a round of Baz IO: Wrote 2.093 GiB in 3.272 seconds | 20260506 12:33:53.354 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.33% / 0.02% / 58.65% | 20260506 12:33:53.354 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.87062% / 3.17891e-05% / 3.97364e-05% | 20260506 12:33:53.354 | INFO | Baz IO overall: | 20260506 12:33:53.354 | INFO | BytesWritten : 691538827760 | 20260506 12:33:53.354 | INFO | NumEvents : 334234363502 | 20260506 12:33:55.098 | INFO | Analyzing chunk frames = [1388544,1389056) | 20260506 12:33:55.098 | INFO | MemInfo: { | 20260506 12:33:55.098 | INFO | "gpu" : " 17 GB", | 20260506 12:33:55.098 | INFO | "huge" : "", | 20260506 12:33:55.098 | INFO | "huge_shared" : " 27 GB", | 20260506 12:33:55.098 | INFO | "malloc" : " 14 GB", | 20260506 12:33:55.098 | INFO | "pinned" : " 29 GB" | 20260506 12:33:55.098 | INFO | } | 20260506 12:33:55.098 | INFO | Perf report for the last 5.13897 seconds: | 20260506 12:33:55.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:33:55.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:55.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:55.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:55.099 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260506 12:33:55.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:33:55.099 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:33:55.099 | INFO | PRE_HQ : 61.6%, 1.9, 61.3ms, 0 | 20260506 12:33:55.099 | INFO | BAZWRITER : 62.9%, 3.7, 118.2ms, 0 | 20260506 12:33:55.099 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:33:55.099 | INFO | Uploaded 36GiB to GPU in 2.08581s (17.2595GiB/s) | 20260506 12:33:55.099 | INFO | Downloaded 38GiB to GPU in 2.20896s (17.2027GiB/s) | 20260506 12:34:00.238 | INFO | Analyzing chunk frames = [1389056,1389568) | 20260506 12:34:00.238 | INFO | MemInfo: { | 20260506 12:34:00.238 | INFO | "gpu" : " 17 GB", | 20260506 12:34:00.238 | INFO | "huge" : "", | 20260506 12:34:00.238 | INFO | "huge_shared" : " 27 GB", | 20260506 12:34:00.238 | INFO | "malloc" : " 14 GB", | 20260506 12:34:00.238 | INFO | "pinned" : " 29 GB" | 20260506 12:34:00.238 | INFO | } | 20260506 12:34:00.238 | INFO | Perf report for the last 5.13938 seconds: | 20260506 12:34:00.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:34:00.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:00.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:00.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:00.238 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:34:00.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:00.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:00.238 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:34:00.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:00.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:00.238 | INFO | Uploaded 36GiB to GPU in 2.0266s (17.7638GiB/s) | 20260506 12:34:00.238 | INFO | Downloaded 34GiB to GPU in 1.92699s (17.6441GiB/s) | 20260506 12:34:05.377 | INFO | Analyzing chunk frames = [1389568,1390080) | 20260506 12:34:05.377 | INFO | MemInfo: { | 20260506 12:34:05.377 | INFO | "gpu" : " 17 GB", | 20260506 12:34:05.377 | INFO | "huge" : "", | 20260506 12:34:05.377 | INFO | "huge_shared" : " 27 GB", | 20260506 12:34:05.377 | INFO | "malloc" : " 14 GB", | 20260506 12:34:05.377 | INFO | "pinned" : " 29 GB" | 20260506 12:34:05.377 | INFO | } | 20260506 12:34:05.377 | INFO | Perf report for the last 5.13922 seconds: | 20260506 12:34:05.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:34:05.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:05.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:05.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:05.377 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 12:34:05.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:05.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:05.377 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:34:05.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:05.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:05.377 | INFO | Uploaded 36GiB to GPU in 1.99685s (18.0284GiB/s) | 20260506 12:34:05.377 | INFO | Downloaded 34GiB to GPU in 1.94397s (17.4899GiB/s) | 20260506 12:34:10.516 | INFO | Analyzing chunk frames = [1390080,1390592) | 20260506 12:34:10.516 | INFO | MemInfo: { | 20260506 12:34:10.516 | INFO | "gpu" : " 17 GB", | 20260506 12:34:10.516 | INFO | "huge" : "", | 20260506 12:34:10.516 | INFO | "huge_shared" : " 27 GB", | 20260506 12:34:10.516 | INFO | "malloc" : " 14 GB", | 20260506 12:34:10.516 | INFO | "pinned" : " 29 GB" | 20260506 12:34:10.516 | INFO | } | 20260506 12:34:10.516 | INFO | Perf report for the last 5.13941 seconds: | 20260506 12:34:10.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:34:10.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:10.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:10.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:10.516 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 12:34:10.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:10.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:10.517 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:34:10.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:10.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:10.517 | INFO | Uploaded 36GiB to GPU in 1.9875s (18.1132GiB/s) | 20260506 12:34:10.517 | INFO | Downloaded 34GiB to GPU in 1.94219s (17.506GiB/s) | 20260506 12:34:15.656 | INFO | Analyzing chunk frames = [1390592,1391104) | 20260506 12:34:15.656 | INFO | MemInfo: { | 20260506 12:34:15.656 | INFO | "gpu" : " 17 GB", | 20260506 12:34:15.656 | INFO | "huge" : "", | 20260506 12:34:15.656 | INFO | "huge_shared" : " 27 GB", | 20260506 12:34:15.656 | INFO | "malloc" : " 14 GB", | 20260506 12:34:15.656 | INFO | "pinned" : " 29 GB" | 20260506 12:34:15.656 | INFO | } | 20260506 12:34:15.656 | INFO | Perf report for the last 5.13947 seconds: | 20260506 12:34:15.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:34:15.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:15.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:15.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:15.656 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 12:34:15.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:15.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:15.656 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:34:15.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:15.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:15.656 | INFO | Uploaded 36GiB to GPU in 2.06713s (17.4155GiB/s) | 20260506 12:34:15.656 | INFO | Downloaded 34GiB to GPU in 1.95645s (17.3784GiB/s) | 20260506 12:34:20.799 | INFO | Analyzing chunk frames = [1391104,1391616) | 20260506 12:34:20.799 | INFO | MemInfo: { | 20260506 12:34:20.799 | INFO | "gpu" : " 17 GB", | 20260506 12:34:20.799 | INFO | "huge" : "", | 20260506 12:34:20.799 | INFO | "huge_shared" : " 27 GB", | 20260506 12:34:20.799 | INFO | "malloc" : " 14 GB", | 20260506 12:34:20.799 | INFO | "pinned" : " 29 GB" | 20260506 12:34:20.799 | INFO | } | 20260506 12:34:20.799 | INFO | Perf report for the last 5.14351 seconds: | 20260506 12:34:20.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:34:20.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:20.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:20.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:20.800 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 12:34:20.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:20.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:20.800 | INFO | PRE_HQ : 32.6%, 1.0, 17.0ms, 0 | 20260506 12:34:20.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:20.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:20.800 | INFO | Uploaded 36GiB to GPU in 2.01041s (17.9068GiB/s) | 20260506 12:34:20.800 | INFO | Downloaded 34GiB to GPU in 1.92972s (17.6191GiB/s) | 20260506 12:34:25.936 | INFO | Analyzing chunk frames = [1391616,1392128) | 20260506 12:34:25.936 | INFO | MemInfo: { | 20260506 12:34:25.936 | INFO | "gpu" : " 17 GB", | 20260506 12:34:25.937 | INFO | "huge" : "", | 20260506 12:34:25.937 | INFO | "huge_shared" : " 27 GB", | 20260506 12:34:25.937 | INFO | "malloc" : " 14 GB", | 20260506 12:34:25.937 | INFO | "pinned" : " 29 GB" | 20260506 12:34:25.937 | INFO | } | 20260506 12:34:25.937 | INFO | Perf report for the last 5.1371 seconds: | 20260506 12:34:25.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:34:25.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:25.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:25.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:25.937 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260506 12:34:25.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:25.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:25.937 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 12:34:25.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:25.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:25.937 | INFO | Uploaded 36GiB to GPU in 1.99114s (18.0801GiB/s) | 20260506 12:34:25.937 | INFO | Downloaded 34GiB to GPU in 2.0158s (16.8668GiB/s) | 20260506 12:34:31.074 | INFO | Analyzing chunk frames = [1392128,1392640) | 20260506 12:34:31.074 | INFO | MemInfo: { | 20260506 12:34:31.074 | INFO | "gpu" : " 17 GB", | 20260506 12:34:31.074 | INFO | "huge" : "", | 20260506 12:34:31.074 | INFO | "huge_shared" : " 27 GB", | 20260506 12:34:31.074 | INFO | "malloc" : " 14 GB", | 20260506 12:34:31.074 | INFO | "pinned" : " 29 GB" | 20260506 12:34:31.074 | INFO | } | 20260506 12:34:31.074 | INFO | Perf report for the last 5.13787 seconds: | 20260506 12:34:31.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:34:31.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:31.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:31.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:31.075 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 12:34:31.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:31.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:31.075 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 12:34:31.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:31.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:31.075 | INFO | Uploaded 36GiB to GPU in 2.01884s (17.832GiB/s) | 20260506 12:34:31.075 | INFO | Downloaded 34GiB to GPU in 1.99161s (17.0716GiB/s) | 20260506 12:34:31.075 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:34:31.075 | INFO | Total mem usage: 16.8 GiB | 20260506 12:34:31.075 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:34:31.075 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.7%) | 20260506 12:34:31.075 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:34:31.075 | INFO | High Water Breakdown: | 20260506 12:34:31.075 | INFO | 59.6% : 8.59 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:34:31.075 | INFO | 40.2% : 5.79 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:34:31.075 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:34:31.075 | INFO | ------------------------------------------ | 20260506 12:34:31.075 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:34:31.075 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:34:31.075 | INFO | Total mem usage: 26.5 GiB | 20260506 12:34:31.075 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:34:31.075 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:34:31.075 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:34:31.075 | INFO | High Water Breakdown: | 20260506 12:34:31.075 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:34:31.075 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:34:31.075 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:34:31.075 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:34:31.075 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:34:31.075 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:34:31.075 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:34:31.075 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:34:31.075 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:34:31.075 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:34:31.075 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:34:31.075 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:34:31.075 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:34:31.075 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:34:31.075 | INFO | ------------------------------------------ | 20260506 12:34:31.075 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:34:31.075 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:34:31.075 | INFO | Total mem usage: 25.6 GiB | 20260506 12:34:31.075 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:34:31.075 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:34:31.075 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:34:31.075 | INFO | High Water Breakdown: | 20260506 12:34:31.075 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:34:31.075 | INFO | ------------------------------------------ | 20260506 12:34:31.075 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:34:31.075 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:34:31.075 | INFO | Total mem usage: 15.7 GiB | 20260506 12:34:31.075 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:34:31.075 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:34:31.076 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 12:34:31.076 | INFO | High Water Breakdown: | 20260506 12:34:31.076 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:34:31.076 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:34:31.076 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:34:31.076 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:34:31.076 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:34:31.076 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:34:31.076 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:34:31.076 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:34:31.076 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:34:31.076 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:34:31.076 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:34:31.076 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:34:31.076 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:34:31.076 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:34:31.076 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:34:31.076 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:34:31.076 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:34:31.076 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:34:31.076 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:34:31.076 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:34:31.076 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:34:31.076 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:34:31.076 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:34:31.076 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:34:31.076 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:34:31.076 | INFO | ------------------------------------------ | 20260506 12:34:31.076 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:34:31.076 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:34:31.076 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:34:31.076 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:34:31.076 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.61e+01, 1.37e+01 / 2.58e+00 | 20260506 12:34:31.076 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.03e-01, 1.07e-02 / 6.80e-03 | 20260506 12:34:31.076 | INFO | PerfReport: 11.55%, Baseline -- 3.67e+00 / 4.37e+00, 5.77e-01 / 4.10e-01 | 20260506 12:34:31.076 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.43e+00, 1.90e-01 / 1.51e-01 | 20260506 12:34:31.076 | INFO | PerfReport: 15.80%, DME -- 4.01e+01 / 4.43e+01, 4.12e+00 / 2.81e+00 | 20260506 12:34:31.076 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.80e+01 / 2.09e+01, 1.50e+00 / 1.97e+00 | 20260506 12:34:31.076 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.40e+00 / 2.99e+00, 4.13e-01 / 2.58e-01 | 20260506 12:34:31.076 | INFO | PerfReport: 3.72%, Metrics -- 1.18e+00 / 1.55e+00, 3.02e-01 / 1.90e-01 | 20260506 12:34:31.076 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.18e-01 / 4.75e-01, 6.63e-02 / 2.92e-02 | 20260506 12:34:31.076 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:34:31.076 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:34:31.076 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:34:31.076 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 12:34:31.076 | INFO | PerfReport: 21.40%, Upload -- 2.11e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 12:34:31.076 | INFO | PerfReport: 21.05%, Download -- 2.08e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 12:34:31.076 | INFO | PerfReport: 32.18%, Compute -- 3.18e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 12:34:31.076 | INFO | PerfReport: 1.65%, LockUpload -- 1.63e+00 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260506 12:34:31.076 | INFO | PerfReport: 1.10%, LockDownload -- 1.09e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260506 12:34:31.076 | INFO | PerfReport: 22.61%, LockCompute -- 2.23e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 12:34:31.076 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260506 12:34:31.208 | INFO | Starting a round of Baz IO | 20260506 12:34:34.365 | INFO | Populating RTMetrics report | 20260506 12:34:34.365 | INFO | RTMetrics report populated | 20260506 12:34:34.365 | INFO | Opening tmp RTMetrics json file | 20260506 12:34:34.366 | INFO | Serializing RTMetrics to json | 20260506 12:34:34.367 | INFO | Serialization done, swapping tmp file now... | 20260506 12:34:34.367 | INFO | RTMetrics done | 20260506 12:34:34.469 | INFO | Finished a round of Baz IO: Wrote 2.091 GiB in 3.261 seconds | 20260506 12:34:34.470 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.31% / 0.02% / 58.67% | 20260506 12:34:34.470 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.8447% / 1.58946e-05% / 5.16574e-05% | 20260506 12:34:34.470 | INFO | Baz IO overall: | 20260506 12:34:34.470 | INFO | BytesWritten : 693784232056 | 20260506 12:34:34.470 | INFO | NumEvents : 335322998659 | 20260506 12:34:36.213 | INFO | Analyzing chunk frames = [1392640,1393152) | 20260506 12:34:36.213 | INFO | MemInfo: { | 20260506 12:34:36.213 | INFO | "gpu" : " 17 GB", | 20260506 12:34:36.213 | INFO | "huge" : "", | 20260506 12:34:36.213 | INFO | "huge_shared" : " 27 GB", | 20260506 12:34:36.213 | INFO | "malloc" : " 14 GB", | 20260506 12:34:36.213 | INFO | "pinned" : " 29 GB" | 20260506 12:34:36.213 | INFO | } | 20260506 12:34:36.213 | INFO | Perf report for the last 5.13882 seconds: | 20260506 12:34:36.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:34:36.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:36.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:36.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:36.213 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260506 12:34:36.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:36.214 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:34:36.214 | INFO | PRE_HQ : 61.2%, 1.9, 61.2ms, 0 | 20260506 12:34:36.214 | INFO | BAZWRITER : 62.8%, 3.7, 120.8ms, 0 | 20260506 12:34:36.214 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:34:36.214 | INFO | Uploaded 36GiB to GPU in 2.07432s (17.3551GiB/s) | 20260506 12:34:36.214 | INFO | Downloaded 38GiB to GPU in 2.20712s (17.217GiB/s) | 20260506 12:34:41.353 | INFO | Analyzing chunk frames = [1393152,1393664) | 20260506 12:34:41.353 | INFO | MemInfo: { | 20260506 12:34:41.353 | INFO | "gpu" : " 17 GB", | 20260506 12:34:41.353 | INFO | "huge" : "", | 20260506 12:34:41.353 | INFO | "huge_shared" : " 27 GB", | 20260506 12:34:41.353 | INFO | "malloc" : " 14 GB", | 20260506 12:34:41.353 | INFO | "pinned" : " 29 GB" | 20260506 12:34:41.353 | INFO | } | 20260506 12:34:41.353 | INFO | Perf report for the last 5.13948 seconds: | 20260506 12:34:41.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:34:41.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:41.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:41.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:41.353 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:34:41.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:41.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:41.353 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:34:41.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:41.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:41.353 | INFO | Uploaded 36GiB to GPU in 2.01753s (17.8436GiB/s) | 20260506 12:34:41.353 | INFO | Downloaded 34GiB to GPU in 1.92618s (17.6515GiB/s) | 20260506 12:34:46.492 | INFO | Analyzing chunk frames = [1393664,1394176) | 20260506 12:34:46.492 | INFO | MemInfo: { | 20260506 12:34:46.492 | INFO | "gpu" : " 17 GB", | 20260506 12:34:46.492 | INFO | "huge" : "", | 20260506 12:34:46.492 | INFO | "huge_shared" : " 27 GB", | 20260506 12:34:46.492 | INFO | "malloc" : " 14 GB", | 20260506 12:34:46.492 | INFO | "pinned" : " 29 GB" | 20260506 12:34:46.492 | INFO | } | 20260506 12:34:46.492 | INFO | Perf report for the last 5.13953 seconds: | 20260506 12:34:46.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:34:46.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:46.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:46.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:46.492 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 12:34:46.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:46.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:46.492 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:34:46.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:46.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:46.493 | INFO | Uploaded 36GiB to GPU in 2.01437s (17.8716GiB/s) | 20260506 12:34:46.493 | INFO | Downloaded 34GiB to GPU in 1.93895s (17.5353GiB/s) | 20260506 12:34:51.631 | INFO | Analyzing chunk frames = [1394176,1394688) | 20260506 12:34:51.631 | INFO | MemInfo: { | 20260506 12:34:51.631 | INFO | "gpu" : " 17 GB", | 20260506 12:34:51.631 | INFO | "huge" : "", | 20260506 12:34:51.631 | INFO | "huge_shared" : " 27 GB", | 20260506 12:34:51.631 | INFO | "malloc" : " 14 GB", | 20260506 12:34:51.631 | INFO | "pinned" : " 29 GB" | 20260506 12:34:51.631 | INFO | } | 20260506 12:34:51.631 | INFO | Perf report for the last 5.13882 seconds: | 20260506 12:34:51.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:34:51.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:51.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:51.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:51.631 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 12:34:51.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:51.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:51.631 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:34:51.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:51.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:51.631 | INFO | Uploaded 36GiB to GPU in 2.01435s (17.8718GiB/s) | 20260506 12:34:51.631 | INFO | Downloaded 34GiB to GPU in 1.92585s (17.6545GiB/s) | 20260506 12:34:56.771 | INFO | Analyzing chunk frames = [1394688,1395200) | 20260506 12:34:56.771 | INFO | MemInfo: { | 20260506 12:34:56.771 | INFO | "gpu" : " 17 GB", | 20260506 12:34:56.771 | INFO | "huge" : "", | 20260506 12:34:56.771 | INFO | "huge_shared" : " 27 GB", | 20260506 12:34:56.771 | INFO | "malloc" : " 14 GB", | 20260506 12:34:56.771 | INFO | "pinned" : " 29 GB" | 20260506 12:34:56.771 | INFO | } | 20260506 12:34:56.771 | INFO | Perf report for the last 5.13968 seconds: | 20260506 12:34:56.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:34:56.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:56.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:56.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:56.771 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 12:34:56.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:56.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:56.771 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:34:56.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:56.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:34:56.771 | INFO | Uploaded 36GiB to GPU in 2.05482s (17.5198GiB/s) | 20260506 12:34:56.771 | INFO | Downloaded 34GiB to GPU in 1.9537s (17.4029GiB/s) | 20260506 12:35:01.910 | INFO | Analyzing chunk frames = [1395200,1395712) | 20260506 12:35:01.910 | INFO | MemInfo: { | 20260506 12:35:01.910 | INFO | "gpu" : " 17 GB", | 20260506 12:35:01.910 | INFO | "huge" : "", | 20260506 12:35:01.910 | INFO | "huge_shared" : " 27 GB", | 20260506 12:35:01.910 | INFO | "malloc" : " 14 GB", | 20260506 12:35:01.910 | INFO | "pinned" : " 29 GB" | 20260506 12:35:01.910 | INFO | } | 20260506 12:35:01.910 | INFO | Perf report for the last 5.13929 seconds: | 20260506 12:35:01.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:35:01.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:01.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:01.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:01.910 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 12:35:01.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:01.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:01.910 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 12:35:01.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:01.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:01.910 | INFO | Uploaded 36GiB to GPU in 2.07097s (17.3832GiB/s) | 20260506 12:35:01.910 | INFO | Downloaded 34GiB to GPU in 1.933s (17.5893GiB/s) | 20260506 12:35:07.049 | INFO | Analyzing chunk frames = [1395712,1396224) | 20260506 12:35:07.049 | INFO | MemInfo: { | 20260506 12:35:07.049 | INFO | "gpu" : " 17 GB", | 20260506 12:35:07.049 | INFO | "huge" : "", | 20260506 12:35:07.049 | INFO | "huge_shared" : " 27 GB", | 20260506 12:35:07.049 | INFO | "malloc" : " 14 GB", | 20260506 12:35:07.049 | INFO | "pinned" : " 29 GB" | 20260506 12:35:07.049 | INFO | } | 20260506 12:35:07.049 | INFO | Perf report for the last 5.1392 seconds: | 20260506 12:35:07.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:35:07.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:07.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:07.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:07.049 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:35:07.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:07.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:07.049 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:35:07.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:07.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:07.050 | INFO | Uploaded 36GiB to GPU in 2.01086s (17.9028GiB/s) | 20260506 12:35:07.050 | INFO | Downloaded 34GiB to GPU in 2.02392s (16.7991GiB/s) | 20260506 12:35:12.188 | INFO | Analyzing chunk frames = [1396224,1396736) | 20260506 12:35:12.188 | INFO | MemInfo: { | 20260506 12:35:12.188 | INFO | "gpu" : " 17 GB", | 20260506 12:35:12.188 | INFO | "huge" : "", | 20260506 12:35:12.188 | INFO | "huge_shared" : " 27 GB", | 20260506 12:35:12.189 | INFO | "malloc" : " 14 GB", | 20260506 12:35:12.189 | INFO | "pinned" : " 29 GB" | 20260506 12:35:12.189 | INFO | } | 20260506 12:35:12.189 | INFO | Perf report for the last 5.13923 seconds: | 20260506 12:35:12.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:35:12.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:12.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:12.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:12.189 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:35:12.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:12.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:12.189 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:35:12.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:12.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:12.189 | INFO | Uploaded 36GiB to GPU in 2.03609s (17.681GiB/s) | 20260506 12:35:12.189 | INFO | Downloaded 34GiB to GPU in 1.9709s (17.251GiB/s) | 20260506 12:35:12.320 | INFO | Starting a round of Baz IO | 20260506 12:35:15.483 | INFO | Populating RTMetrics report | 20260506 12:35:15.483 | INFO | RTMetrics report populated | 20260506 12:35:15.483 | INFO | Opening tmp RTMetrics json file | 20260506 12:35:15.483 | INFO | Serializing RTMetrics to json | 20260506 12:35:15.485 | INFO | Serialization done, swapping tmp file now... | 20260506 12:35:15.485 | INFO | RTMetrics done | 20260506 12:35:15.582 | INFO | Finished a round of Baz IO: Wrote 2.093 GiB in 3.262 seconds | 20260506 12:35:15.583 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.28% / 0.02% / 58.69% | 20260506 12:35:15.583 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.8436% / 5.5631e-05% / 3.97364e-05% | 20260506 12:35:15.583 | INFO | Baz IO overall: | 20260506 12:35:15.583 | INFO | BytesWritten : 696031720389 | 20260506 12:35:15.583 | INFO | NumEvents : 336413223446 | 20260506 12:35:17.328 | INFO | Analyzing chunk frames = [1396736,1397248) | 20260506 12:35:17.328 | INFO | MemInfo: { | 20260506 12:35:17.328 | INFO | "gpu" : " 17 GB", | 20260506 12:35:17.328 | INFO | "huge" : "", | 20260506 12:35:17.328 | INFO | "huge_shared" : " 27 GB", | 20260506 12:35:17.328 | INFO | "malloc" : " 14 GB", | 20260506 12:35:17.328 | INFO | "pinned" : " 29 GB" | 20260506 12:35:17.328 | INFO | } | 20260506 12:35:17.328 | INFO | Perf report for the last 5.13952 seconds: | 20260506 12:35:17.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:35:17.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:17.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:17.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:17.328 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 12:35:17.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:17.328 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:35:17.328 | INFO | PRE_HQ : 61.6%, 1.9, 61.6ms, 0 | 20260506 12:35:17.328 | INFO | BAZWRITER : 62.8%, 3.7, 119.6ms, 0 | 20260506 12:35:17.328 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:35:17.328 | INFO | Uploaded 36GiB to GPU in 2.08422s (17.2727GiB/s) | 20260506 12:35:17.328 | INFO | Downloaded 38GiB to GPU in 2.22529s (17.0764GiB/s) | 20260506 12:35:22.467 | INFO | Analyzing chunk frames = [1397248,1397760) | 20260506 12:35:22.467 | INFO | MemInfo: { | 20260506 12:35:22.467 | INFO | "gpu" : " 17 GB", | 20260506 12:35:22.467 | INFO | "huge" : "", | 20260506 12:35:22.467 | INFO | "huge_shared" : " 27 GB", | 20260506 12:35:22.468 | INFO | "malloc" : " 14 GB", | 20260506 12:35:22.468 | INFO | "pinned" : " 29 GB" | 20260506 12:35:22.468 | INFO | } | 20260506 12:35:22.468 | INFO | Perf report for the last 5.1396 seconds: | 20260506 12:35:22.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:35:22.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:22.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:22.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:22.468 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 12:35:22.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:22.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:22.468 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:35:22.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:22.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:22.468 | INFO | Uploaded 36GiB to GPU in 2.01554s (17.8612GiB/s) | 20260506 12:35:22.468 | INFO | Downloaded 34GiB to GPU in 1.92554s (17.6574GiB/s) | 20260506 12:35:27.607 | INFO | Analyzing chunk frames = [1397760,1398272) | 20260506 12:35:27.607 | INFO | MemInfo: { | 20260506 12:35:27.607 | INFO | "gpu" : " 17 GB", | 20260506 12:35:27.607 | INFO | "huge" : "", | 20260506 12:35:27.607 | INFO | "huge_shared" : " 27 GB", | 20260506 12:35:27.607 | INFO | "malloc" : " 14 GB", | 20260506 12:35:27.607 | INFO | "pinned" : " 29 GB" | 20260506 12:35:27.607 | INFO | } | 20260506 12:35:27.607 | INFO | Perf report for the last 5.13897 seconds: | 20260506 12:35:27.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:35:27.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:27.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:27.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:27.607 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 12:35:27.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:27.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:27.607 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 12:35:27.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:27.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:27.607 | INFO | Uploaded 36GiB to GPU in 1.98146s (18.1684GiB/s) | 20260506 12:35:27.607 | INFO | Downloaded 34GiB to GPU in 1.93412s (17.5791GiB/s) | 20260506 12:35:32.746 | INFO | Analyzing chunk frames = [1398272,1398784) | 20260506 12:35:32.746 | INFO | MemInfo: { | 20260506 12:35:32.746 | INFO | "gpu" : " 17 GB", | 20260506 12:35:32.746 | INFO | "huge" : "", | 20260506 12:35:32.746 | INFO | "huge_shared" : " 27 GB", | 20260506 12:35:32.746 | INFO | "malloc" : " 14 GB", | 20260506 12:35:32.746 | INFO | "pinned" : " 29 GB" | 20260506 12:35:32.746 | INFO | } | 20260506 12:35:32.746 | INFO | Perf report for the last 5.13938 seconds: | 20260506 12:35:32.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:35:32.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:32.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:32.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:32.746 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 12:35:32.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:32.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:32.746 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:35:32.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:32.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:32.746 | INFO | Uploaded 36GiB to GPU in 1.97545s (18.2237GiB/s) | 20260506 12:35:32.747 | INFO | Downloaded 34GiB to GPU in 1.93976s (17.5279GiB/s) | 20260506 12:35:37.885 | INFO | Analyzing chunk frames = [1398784,1399296) | 20260506 12:35:37.885 | INFO | MemInfo: { | 20260506 12:35:37.885 | INFO | "gpu" : " 17 GB", | 20260506 12:35:37.885 | INFO | "huge" : "", | 20260506 12:35:37.885 | INFO | "huge_shared" : " 27 GB", | 20260506 12:35:37.886 | INFO | "malloc" : " 14 GB", | 20260506 12:35:37.886 | INFO | "pinned" : " 29 GB" | 20260506 12:35:37.886 | INFO | } | 20260506 12:35:37.886 | INFO | Perf report for the last 5.13962 seconds: | 20260506 12:35:37.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:35:37.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:37.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:37.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:37.886 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 12:35:37.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:37.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:37.886 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:35:37.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:37.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:37.886 | INFO | Uploaded 36GiB to GPU in 2.01923s (17.8286GiB/s) | 20260506 12:35:37.886 | INFO | Downloaded 34GiB to GPU in 1.96052s (17.3424GiB/s) | 20260506 12:35:43.025 | INFO | Analyzing chunk frames = [1399296,1399808) | 20260506 12:35:43.025 | INFO | MemInfo: { | 20260506 12:35:43.025 | INFO | "gpu" : " 17 GB", | 20260506 12:35:43.025 | INFO | "huge" : "", | 20260506 12:35:43.025 | INFO | "huge_shared" : " 27 GB", | 20260506 12:35:43.025 | INFO | "malloc" : " 14 GB", | 20260506 12:35:43.025 | INFO | "pinned" : " 29 GB" | 20260506 12:35:43.025 | INFO | } | 20260506 12:35:43.025 | INFO | Perf report for the last 5.1391 seconds: | 20260506 12:35:43.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:35:43.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:43.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:43.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:43.025 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 12:35:43.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:43.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:43.025 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260506 12:35:43.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:43.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:43.025 | INFO | Uploaded 36GiB to GPU in 2.03212s (17.7155GiB/s) | 20260506 12:35:43.025 | INFO | Downloaded 34GiB to GPU in 1.92609s (17.6524GiB/s) | 20260506 12:35:48.165 | INFO | Analyzing chunk frames = [1399808,1400320) | 20260506 12:35:48.165 | INFO | MemInfo: { | 20260506 12:35:48.165 | INFO | "gpu" : " 17 GB", | 20260506 12:35:48.165 | INFO | "huge" : "", | 20260506 12:35:48.165 | INFO | "huge_shared" : " 27 GB", | 20260506 12:35:48.165 | INFO | "malloc" : " 14 GB", | 20260506 12:35:48.165 | INFO | "pinned" : " 29 GB" | 20260506 12:35:48.165 | INFO | } | 20260506 12:35:48.165 | INFO | Perf report for the last 5.13987 seconds: | 20260506 12:35:48.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:35:48.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:48.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:48.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:48.165 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260506 12:35:48.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:48.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:48.165 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 12:35:48.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:48.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:48.165 | INFO | Uploaded 36GiB to GPU in 2.00739s (17.9338GiB/s) | 20260506 12:35:48.165 | INFO | Downloaded 34GiB to GPU in 2.02794s (16.7658GiB/s) | 20260506 12:35:53.303 | INFO | Analyzing chunk frames = [1400320,1400832) | 20260506 12:35:53.303 | INFO | MemInfo: { | 20260506 12:35:53.303 | INFO | "gpu" : " 17 GB", | 20260506 12:35:53.303 | INFO | "huge" : "", | 20260506 12:35:53.303 | INFO | "huge_shared" : " 27 GB", | 20260506 12:35:53.303 | INFO | "malloc" : " 14 GB", | 20260506 12:35:53.304 | INFO | "pinned" : " 29 GB" | 20260506 12:35:53.304 | INFO | } | 20260506 12:35:53.304 | INFO | Perf report for the last 5.1388 seconds: | 20260506 12:35:53.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:35:53.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:53.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:53.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:53.304 | INFO | ANALYSIS : 62.7%, 3.0, 97.9ms, 0 | 20260506 12:35:53.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:53.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:53.304 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:35:53.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:53.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:53.304 | INFO | Uploaded 36GiB to GPU in 2.03616s (17.6803GiB/s) | 20260506 12:35:53.304 | INFO | Downloaded 34GiB to GPU in 2.00788s (16.9333GiB/s) | 20260506 12:35:53.304 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:35:53.304 | INFO | Total mem usage: 16.8 GiB | 20260506 12:35:53.304 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:35:53.304 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.6%) | 20260506 12:35:53.304 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:35:53.304 | INFO | High Water Breakdown: | 20260506 12:35:53.304 | INFO | 59.6% : 8.59 GiB ( 82.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:35:53.304 | INFO | 40.3% : 5.81 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:35:53.304 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:35:53.304 | INFO | ------------------------------------------ | 20260506 12:35:53.304 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:35:53.304 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:35:53.304 | INFO | Total mem usage: 26.5 GiB | 20260506 12:35:53.304 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:35:53.304 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:35:53.304 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:35:53.304 | INFO | High Water Breakdown: | 20260506 12:35:53.304 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:35:53.304 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:35:53.304 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:35:53.304 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:35:53.304 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:35:53.304 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:35:53.304 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:35:53.304 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:35:53.304 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:35:53.304 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:35:53.304 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:35:53.304 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:35:53.304 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:35:53.304 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:35:53.304 | INFO | ------------------------------------------ | 20260506 12:35:53.304 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:35:53.304 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:35:53.304 | INFO | Total mem usage: 25.6 GiB | 20260506 12:35:53.304 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:35:53.304 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:35:53.304 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:35:53.304 | INFO | High Water Breakdown: | 20260506 12:35:53.304 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:35:53.304 | INFO | ------------------------------------------ | 20260506 12:35:53.304 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:35:53.304 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:35:53.304 | INFO | Total mem usage: 15.7 GiB | 20260506 12:35:53.304 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:35:53.304 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:35:53.304 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 12:35:53.304 | INFO | High Water Breakdown: | 20260506 12:35:53.304 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:35:53.304 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:35:53.304 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:35:53.304 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:35:53.304 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:35:53.304 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:35:53.305 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:35:53.305 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:35:53.305 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:35:53.305 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:35:53.305 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:35:53.305 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:35:53.305 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:35:53.305 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:35:53.305 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:35:53.305 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:35:53.305 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:35:53.305 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:35:53.305 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:35:53.305 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:35:53.305 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:35:53.305 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:35:53.305 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:35:53.305 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:35:53.305 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:35:53.305 | INFO | ------------------------------------------ | 20260506 12:35:53.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:35:53.305 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:35:53.305 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:35:53.305 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:35:53.305 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.59e+01, 1.34e+01 / 2.37e+00 | 20260506 12:35:53.305 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.01e-01, 1.03e-02 / 8.18e-03 | 20260506 12:35:53.305 | INFO | PerfReport: 11.55%, Baseline -- 3.67e+00 / 4.36e+00, 5.89e-01 / 4.22e-01 | 20260506 12:35:53.305 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.44e+00, 1.94e-01 / 1.53e-01 | 20260506 12:35:53.305 | INFO | PerfReport: 15.77%, DME -- 4.01e+01 / 4.42e+01, 3.85e+00 / 2.81e+00 | 20260506 12:35:53.305 | INFO | PerfReport: 56.59%, FrameLabeling -- 1.80e+01 / 2.08e+01, 1.53e+00 / 1.89e+00 | 20260506 12:35:53.305 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.39e+00 / 2.95e+00, 4.10e-01 / 2.73e-01 | 20260506 12:35:53.305 | INFO | PerfReport: 3.73%, Metrics -- 1.18e+00 / 1.54e+00, 3.05e-01 / 1.84e-01 | 20260506 12:35:53.305 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.18e-01 / 4.73e-01, 6.64e-02 / 3.25e-02 | 20260506 12:35:53.305 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:35:53.305 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:35:53.305 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:35:53.305 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 12:35:53.305 | INFO | PerfReport: 21.40%, Upload -- 2.11e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 12:35:53.305 | INFO | PerfReport: 21.00%, Download -- 2.07e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 12:35:53.305 | INFO | PerfReport: 32.21%, Compute -- 3.18e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 12:35:53.305 | INFO | PerfReport: 1.71%, LockUpload -- 1.69e+00 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260506 12:35:53.305 | INFO | PerfReport: 1.09%, LockDownload -- 1.07e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260506 12:35:53.305 | INFO | PerfReport: 22.58%, LockCompute -- 2.23e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 12:35:53.305 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260506 12:35:53.438 | INFO | Starting a round of Baz IO | 20260506 12:35:56.594 | INFO | Populating RTMetrics report | 20260506 12:35:56.594 | INFO | RTMetrics report populated | 20260506 12:35:56.594 | INFO | Opening tmp RTMetrics json file | 20260506 12:35:56.595 | INFO | Serializing RTMetrics to json | 20260506 12:35:56.596 | INFO | Serialization done, swapping tmp file now... | 20260506 12:35:56.596 | INFO | RTMetrics done | 20260506 12:35:56.691 | INFO | Finished a round of Baz IO: Wrote 2.092 GiB in 3.252 seconds | 20260506 12:35:56.691 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.26% / 0.02% / 58.72% | 20260506 12:35:56.691 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.82557% / 3.17891e-05% / 5.96046e-05% | 20260506 12:35:56.691 | INFO | Baz IO overall: | 20260506 12:35:56.691 | INFO | BytesWritten : 698277958630 | 20260506 12:35:56.691 | INFO | NumEvents : 337502693668 | 20260506 12:35:58.443 | INFO | Analyzing chunk frames = [1400832,1401344) | 20260506 12:35:58.443 | INFO | MemInfo: { | 20260506 12:35:58.443 | INFO | "gpu" : " 17 GB", | 20260506 12:35:58.443 | INFO | "huge" : "", | 20260506 12:35:58.443 | INFO | "huge_shared" : " 27 GB", | 20260506 12:35:58.443 | INFO | "malloc" : " 14 GB", | 20260506 12:35:58.443 | INFO | "pinned" : " 29 GB" | 20260506 12:35:58.443 | INFO | } | 20260506 12:35:58.443 | INFO | Perf report for the last 5.13926 seconds: | 20260506 12:35:58.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:35:58.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:58.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:58.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:58.443 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260506 12:35:58.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:35:58.443 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:35:58.443 | INFO | PRE_HQ : 61.3%, 1.9, 61.2ms, 0 | 20260506 12:35:58.443 | INFO | BAZWRITER : 62.6%, 3.7, 118.3ms, 0 | 20260506 12:35:58.443 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:35:58.443 | INFO | Uploaded 36GiB to GPU in 2.07779s (17.3261GiB/s) | 20260506 12:35:58.443 | INFO | Downloaded 38GiB to GPU in 2.2186s (17.1279GiB/s) | 20260506 12:36:03.582 | INFO | Analyzing chunk frames = [1401344,1401856) | 20260506 12:36:03.582 | INFO | MemInfo: { | 20260506 12:36:03.582 | INFO | "gpu" : " 17 GB", | 20260506 12:36:03.582 | INFO | "huge" : "", | 20260506 12:36:03.582 | INFO | "huge_shared" : " 27 GB", | 20260506 12:36:03.582 | INFO | "malloc" : " 14 GB", | 20260506 12:36:03.582 | INFO | "pinned" : " 29 GB" | 20260506 12:36:03.582 | INFO | } | 20260506 12:36:03.582 | INFO | Perf report for the last 5.13927 seconds: | 20260506 12:36:03.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:36:03.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:03.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:03.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:03.582 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 12:36:03.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:03.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:03.582 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:36:03.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:03.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:03.582 | INFO | Uploaded 36GiB to GPU in 2.01837s (17.8362GiB/s) | 20260506 12:36:03.582 | INFO | Downloaded 34GiB to GPU in 1.92765s (17.6381GiB/s) | 20260506 12:36:08.721 | INFO | Analyzing chunk frames = [1401856,1402368) | 20260506 12:36:08.721 | INFO | MemInfo: { | 20260506 12:36:08.721 | INFO | "gpu" : " 17 GB", | 20260506 12:36:08.721 | INFO | "huge" : "", | 20260506 12:36:08.721 | INFO | "huge_shared" : " 27 GB", | 20260506 12:36:08.721 | INFO | "malloc" : " 14 GB", | 20260506 12:36:08.721 | INFO | "pinned" : " 29 GB" | 20260506 12:36:08.721 | INFO | } | 20260506 12:36:08.721 | INFO | Perf report for the last 5.13935 seconds: | 20260506 12:36:08.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:36:08.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:08.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:08.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:08.722 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 12:36:08.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:08.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:08.722 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:36:08.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:08.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:08.722 | INFO | Uploaded 36GiB to GPU in 2.00517s (17.9536GiB/s) | 20260506 12:36:08.722 | INFO | Downloaded 34GiB to GPU in 1.9368s (17.5547GiB/s) | 20260506 12:36:13.863 | INFO | Analyzing chunk frames = [1402368,1402880) | 20260506 12:36:13.863 | INFO | MemInfo: { | 20260506 12:36:13.863 | INFO | "gpu" : " 17 GB", | 20260506 12:36:13.863 | INFO | "huge" : "", | 20260506 12:36:13.863 | INFO | "huge_shared" : " 27 GB", | 20260506 12:36:13.863 | INFO | "malloc" : " 14 GB", | 20260506 12:36:13.863 | INFO | "pinned" : " 29 GB" | 20260506 12:36:13.863 | INFO | } | 20260506 12:36:13.863 | INFO | Perf report for the last 5.14144 seconds: | 20260506 12:36:13.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:36:13.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:13.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:13.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:13.863 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 12:36:13.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:13.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:13.863 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 12:36:13.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:13.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:13.863 | INFO | Uploaded 36GiB to GPU in 2.01746s (17.8442GiB/s) | 20260506 12:36:13.863 | INFO | Downloaded 34GiB to GPU in 1.93191s (17.5992GiB/s) | 20260506 12:36:19.000 | INFO | Analyzing chunk frames = [1402880,1403392) | 20260506 12:36:19.000 | INFO | MemInfo: { | 20260506 12:36:19.000 | INFO | "gpu" : " 17 GB", | 20260506 12:36:19.000 | INFO | "huge" : "", | 20260506 12:36:19.000 | INFO | "huge_shared" : " 27 GB", | 20260506 12:36:19.000 | INFO | "malloc" : " 14 GB", | 20260506 12:36:19.000 | INFO | "pinned" : " 29 GB" | 20260506 12:36:19.000 | INFO | } | 20260506 12:36:19.000 | INFO | Perf report for the last 5.13729 seconds: | 20260506 12:36:19.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:36:19.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:19.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:19.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:19.000 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 12:36:19.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:19.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:19.000 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:36:19.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:19.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:19.000 | INFO | Uploaded 36GiB to GPU in 2.06726s (17.4144GiB/s) | 20260506 12:36:19.000 | INFO | Downloaded 34GiB to GPU in 1.93737s (17.5496GiB/s) | 20260506 12:36:24.139 | INFO | Analyzing chunk frames = [1403392,1403904) | 20260506 12:36:24.139 | INFO | MemInfo: { | 20260506 12:36:24.139 | INFO | "gpu" : " 17 GB", | 20260506 12:36:24.139 | INFO | "huge" : "", | 20260506 12:36:24.139 | INFO | "huge_shared" : " 27 GB", | 20260506 12:36:24.139 | INFO | "malloc" : " 14 GB", | 20260506 12:36:24.139 | INFO | "pinned" : " 29 GB" | 20260506 12:36:24.139 | INFO | } | 20260506 12:36:24.139 | INFO | Perf report for the last 5.13914 seconds: | 20260506 12:36:24.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:36:24.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:24.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:24.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:24.139 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 12:36:24.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:24.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:24.140 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 12:36:24.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:24.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:24.140 | INFO | Uploaded 36GiB to GPU in 2.03682s (17.6746GiB/s) | 20260506 12:36:24.140 | INFO | Downloaded 34GiB to GPU in 1.90873s (17.8129GiB/s) | 20260506 12:36:29.280 | INFO | Analyzing chunk frames = [1403904,1404416) | 20260506 12:36:29.280 | INFO | MemInfo: { | 20260506 12:36:29.280 | INFO | "gpu" : " 17 GB", | 20260506 12:36:29.280 | INFO | "huge" : "", | 20260506 12:36:29.280 | INFO | "huge_shared" : " 27 GB", | 20260506 12:36:29.280 | INFO | "malloc" : " 14 GB", | 20260506 12:36:29.280 | INFO | "pinned" : " 29 GB" | 20260506 12:36:29.280 | INFO | } | 20260506 12:36:29.280 | INFO | Perf report for the last 5.14038 seconds: | 20260506 12:36:29.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:36:29.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:29.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:29.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:29.280 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260506 12:36:29.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:29.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:29.280 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 12:36:29.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:29.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:29.280 | INFO | Uploaded 36GiB to GPU in 2.01011s (17.9095GiB/s) | 20260506 12:36:29.280 | INFO | Downloaded 34GiB to GPU in 2.03444s (16.7122GiB/s) | 20260506 12:36:34.418 | INFO | Analyzing chunk frames = [1404416,1404928) | 20260506 12:36:34.418 | INFO | MemInfo: { | 20260506 12:36:34.418 | INFO | "gpu" : " 17 GB", | 20260506 12:36:34.418 | INFO | "huge" : "", | 20260506 12:36:34.418 | INFO | "huge_shared" : " 27 GB", | 20260506 12:36:34.418 | INFO | "malloc" : " 14 GB", | 20260506 12:36:34.418 | INFO | "pinned" : " 29 GB" | 20260506 12:36:34.418 | INFO | } | 20260506 12:36:34.418 | INFO | Perf report for the last 5.13853 seconds: | 20260506 12:36:34.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:36:34.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:34.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:34.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:34.418 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 12:36:34.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:34.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:34.418 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:36:34.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:34.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:34.419 | INFO | Uploaded 36GiB to GPU in 2.0717s (17.377GiB/s) | 20260506 12:36:34.419 | INFO | Downloaded 34GiB to GPU in 1.98325s (17.1436GiB/s) | 20260506 12:36:34.548 | INFO | Starting a round of Baz IO | 20260506 12:36:37.718 | INFO | Populating RTMetrics report | 20260506 12:36:37.718 | INFO | RTMetrics report populated | 20260506 12:36:37.718 | INFO | Opening tmp RTMetrics json file | 20260506 12:36:37.719 | INFO | Serializing RTMetrics to json | 20260506 12:36:37.720 | INFO | Serialization done, swapping tmp file now... | 20260506 12:36:37.720 | INFO | RTMetrics done | 20260506 12:36:37.818 | INFO | Finished a round of Baz IO: Wrote 2.092 GiB in 3.269 seconds | 20260506 12:36:37.818 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.24% / 0.02% / 58.74% | 20260506 12:36:37.818 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.80952% / 3.17891e-05% / 5.16574e-05% | 20260506 12:36:37.818 | INFO | Baz IO overall: | 20260506 12:36:37.818 | INFO | BytesWritten : 700524263380 | 20260506 12:36:37.818 | INFO | NumEvents : 338591909733 | 20260506 12:36:39.558 | INFO | Analyzing chunk frames = [1404928,1405440) | 20260506 12:36:39.558 | INFO | MemInfo: { | 20260506 12:36:39.558 | INFO | "gpu" : " 17 GB", | 20260506 12:36:39.558 | INFO | "huge" : "", | 20260506 12:36:39.558 | INFO | "huge_shared" : " 27 GB", | 20260506 12:36:39.558 | INFO | "malloc" : " 14 GB", | 20260506 12:36:39.558 | INFO | "pinned" : " 29 GB" | 20260506 12:36:39.558 | INFO | } | 20260506 12:36:39.558 | INFO | Perf report for the last 5.13942 seconds: | 20260506 12:36:39.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:36:39.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:39.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:39.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:39.558 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260506 12:36:39.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:39.558 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:36:39.558 | INFO | PRE_HQ : 61.7%, 1.9, 61.7ms, 0 | 20260506 12:36:39.558 | INFO | BAZWRITER : 62.9%, 3.7, 120.6ms, 0 | 20260506 12:36:39.558 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:36:39.558 | INFO | Uploaded 36GiB to GPU in 2.06451s (17.4375GiB/s) | 20260506 12:36:39.558 | INFO | Downloaded 38GiB to GPU in 2.211s (17.1868GiB/s) | 20260506 12:36:44.697 | INFO | Analyzing chunk frames = [1405440,1405952) | 20260506 12:36:44.697 | INFO | MemInfo: { | 20260506 12:36:44.697 | INFO | "gpu" : " 17 GB", | 20260506 12:36:44.697 | INFO | "huge" : "", | 20260506 12:36:44.697 | INFO | "huge_shared" : " 27 GB", | 20260506 12:36:44.697 | INFO | "malloc" : " 14 GB", | 20260506 12:36:44.697 | INFO | "pinned" : " 29 GB" | 20260506 12:36:44.697 | INFO | } | 20260506 12:36:44.697 | INFO | Perf report for the last 5.13909 seconds: | 20260506 12:36:44.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:36:44.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:44.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:44.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:44.697 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:36:44.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:44.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:44.697 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:36:44.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:44.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:44.697 | INFO | Uploaded 36GiB to GPU in 2.04803s (17.5779GiB/s) | 20260506 12:36:44.697 | INFO | Downloaded 34GiB to GPU in 1.9326s (17.5929GiB/s) | 20260506 12:36:49.836 | INFO | Analyzing chunk frames = [1405952,1406464) | 20260506 12:36:49.836 | INFO | MemInfo: { | 20260506 12:36:49.836 | INFO | "gpu" : " 17 GB", | 20260506 12:36:49.836 | INFO | "huge" : "", | 20260506 12:36:49.836 | INFO | "huge_shared" : " 27 GB", | 20260506 12:36:49.836 | INFO | "malloc" : " 14 GB", | 20260506 12:36:49.836 | INFO | "pinned" : " 29 GB" | 20260506 12:36:49.836 | INFO | } | 20260506 12:36:49.836 | INFO | Perf report for the last 5.13942 seconds: | 20260506 12:36:49.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:36:49.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:49.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:49.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:49.836 | INFO | ANALYSIS : 61.9%, 3.0, 95.8ms, 0 | 20260506 12:36:49.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:49.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:49.836 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:36:49.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:49.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:49.837 | INFO | Uploaded 36GiB to GPU in 2.01894s (17.8311GiB/s) | 20260506 12:36:49.837 | INFO | Downloaded 34GiB to GPU in 1.9419s (17.5086GiB/s) | 20260506 12:36:54.976 | INFO | Analyzing chunk frames = [1406464,1406976) | 20260506 12:36:54.976 | INFO | MemInfo: { | 20260506 12:36:54.976 | INFO | "gpu" : " 17 GB", | 20260506 12:36:54.976 | INFO | "huge" : "", | 20260506 12:36:54.976 | INFO | "huge_shared" : " 27 GB", | 20260506 12:36:54.976 | INFO | "malloc" : " 14 GB", | 20260506 12:36:54.976 | INFO | "pinned" : " 29 GB" | 20260506 12:36:54.976 | INFO | } | 20260506 12:36:54.976 | INFO | Perf report for the last 5.13948 seconds: | 20260506 12:36:54.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:36:54.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:54.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:54.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:54.976 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 12:36:54.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:54.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:54.976 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:36:54.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:54.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:36:54.976 | INFO | Uploaded 36GiB to GPU in 2.01621s (17.8553GiB/s) | 20260506 12:36:54.976 | INFO | Downloaded 34GiB to GPU in 1.94399s (17.4898GiB/s) | 20260506 12:37:00.115 | INFO | Analyzing chunk frames = [1406976,1407488) | 20260506 12:37:00.115 | INFO | MemInfo: { | 20260506 12:37:00.115 | INFO | "gpu" : " 17 GB", | 20260506 12:37:00.115 | INFO | "huge" : "", | 20260506 12:37:00.115 | INFO | "huge_shared" : " 27 GB", | 20260506 12:37:00.115 | INFO | "malloc" : " 14 GB", | 20260506 12:37:00.115 | INFO | "pinned" : " 29 GB" | 20260506 12:37:00.115 | INFO | } | 20260506 12:37:00.115 | INFO | Perf report for the last 5.13911 seconds: | 20260506 12:37:00.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:37:00.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:00.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:00.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:00.115 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 12:37:00.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:00.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:00.115 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:37:00.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:00.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:00.115 | INFO | Uploaded 36GiB to GPU in 2.03262s (17.7111GiB/s) | 20260506 12:37:00.115 | INFO | Downloaded 34GiB to GPU in 1.94357s (17.4936GiB/s) | 20260506 12:37:05.254 | INFO | Analyzing chunk frames = [1407488,1408000) | 20260506 12:37:05.254 | INFO | MemInfo: { | 20260506 12:37:05.254 | INFO | "gpu" : " 17 GB", | 20260506 12:37:05.254 | INFO | "huge" : "", | 20260506 12:37:05.254 | INFO | "huge_shared" : " 27 GB", | 20260506 12:37:05.254 | INFO | "malloc" : " 14 GB", | 20260506 12:37:05.254 | INFO | "pinned" : " 29 GB" | 20260506 12:37:05.254 | INFO | } | 20260506 12:37:05.254 | INFO | Perf report for the last 5.13951 seconds: | 20260506 12:37:05.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:37:05.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:05.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:05.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:05.254 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 12:37:05.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:05.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:05.255 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 12:37:05.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:05.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:05.255 | INFO | Uploaded 36GiB to GPU in 2.05837s (17.4896GiB/s) | 20260506 12:37:05.255 | INFO | Downloaded 34GiB to GPU in 1.92132s (17.6961GiB/s) | 20260506 12:37:10.394 | INFO | Analyzing chunk frames = [1408000,1408512) | 20260506 12:37:10.394 | INFO | MemInfo: { | 20260506 12:37:10.394 | INFO | "gpu" : " 17 GB", | 20260506 12:37:10.394 | INFO | "huge" : "", | 20260506 12:37:10.394 | INFO | "huge_shared" : " 27 GB", | 20260506 12:37:10.394 | INFO | "malloc" : " 14 GB", | 20260506 12:37:10.394 | INFO | "pinned" : " 29 GB" | 20260506 12:37:10.394 | INFO | } | 20260506 12:37:10.394 | INFO | Perf report for the last 5.1393 seconds: | 20260506 12:37:10.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:37:10.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:10.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:10.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:10.394 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:37:10.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:10.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:10.394 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 12:37:10.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:10.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:10.394 | INFO | Uploaded 36GiB to GPU in 2.00856s (17.9233GiB/s) | 20260506 12:37:10.394 | INFO | Downloaded 34GiB to GPU in 2.02145s (16.8196GiB/s) | 20260506 12:37:15.533 | INFO | Analyzing chunk frames = [1408512,1409024) | 20260506 12:37:15.533 | INFO | MemInfo: { | 20260506 12:37:15.533 | INFO | "gpu" : " 17 GB", | 20260506 12:37:15.533 | INFO | "huge" : "", | 20260506 12:37:15.533 | INFO | "huge_shared" : " 27 GB", | 20260506 12:37:15.533 | INFO | "malloc" : " 14 GB", | 20260506 12:37:15.533 | INFO | "pinned" : " 29 GB" | 20260506 12:37:15.533 | INFO | } | 20260506 12:37:15.533 | INFO | Perf report for the last 5.13908 seconds: | 20260506 12:37:15.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:37:15.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:15.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:15.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:15.533 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:37:15.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:15.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:15.533 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 12:37:15.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:15.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:15.533 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:37:15.533 | INFO | Total mem usage: 16.8 GiB | 20260506 12:37:15.533 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:37:15.533 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.7%) | 20260506 12:37:15.533 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:37:15.533 | INFO | High Water Breakdown: | 20260506 12:37:15.533 | INFO | 59.6% : 8.59 GiB ( 82.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:37:15.533 | INFO | 40.2% : 5.79 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:37:15.533 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:37:15.533 | INFO | ------------------------------------------ | 20260506 12:37:15.533 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:37:15.533 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:37:15.533 | INFO | Total mem usage: 26.5 GiB | 20260506 12:37:15.533 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:37:15.533 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:37:15.533 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:37:15.533 | INFO | High Water Breakdown: | 20260506 12:37:15.533 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:37:15.533 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:37:15.533 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:37:15.533 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:37:15.533 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:37:15.533 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:37:15.533 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:37:15.533 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:37:15.533 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:37:15.533 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:37:15.533 | INFO | Uploaded 36GiB to GPU in 2.03566s (17.6847GiB/s) | 20260506 12:37:15.533 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:37:15.533 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:37:15.533 | INFO | Downloaded 34GiB to GPU in 1.96697s (17.2854GiB/s) | 20260506 12:37:15.533 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:37:15.534 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:37:15.534 | INFO | ------------------------------------------ | 20260506 12:37:15.534 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:37:15.534 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:37:15.534 | INFO | Total mem usage: 25.6 GiB | 20260506 12:37:15.534 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:37:15.534 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:37:15.534 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:37:15.534 | INFO | High Water Breakdown: | 20260506 12:37:15.534 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:37:15.534 | INFO | ------------------------------------------ | 20260506 12:37:15.534 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:37:15.534 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:37:15.534 | INFO | Total mem usage: 15.7 GiB | 20260506 12:37:15.534 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:37:15.534 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:37:15.534 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:37:15.534 | INFO | High Water Breakdown: | 20260506 12:37:15.534 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:37:15.534 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:37:15.534 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:37:15.534 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:37:15.534 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:37:15.534 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:37:15.534 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:37:15.534 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:37:15.534 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:37:15.534 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:37:15.534 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:37:15.534 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:37:15.534 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:37:15.534 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:37:15.534 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:37:15.534 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:37:15.534 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:37:15.534 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:37:15.534 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:37:15.534 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:37:15.534 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:37:15.534 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:37:15.534 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:37:15.534 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:37:15.534 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:37:15.534 | INFO | ------------------------------------------ | 20260506 12:37:15.534 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:37:15.534 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:37:15.534 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:37:15.534 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:37:15.534 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.65e+01, 1.35e+01 / 2.41e+00 | 20260506 12:37:15.534 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.02e-01, 1.01e-02 / 5.62e-03 | 20260506 12:37:15.534 | INFO | PerfReport: 11.55%, Baseline -- 3.67e+00 / 4.42e+00, 5.79e-01 / 3.14e-01 | 20260506 12:37:15.534 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.48e+00, 1.93e-01 / 1.38e-01 | 20260506 12:37:15.534 | INFO | PerfReport: 15.87%, DME -- 4.03e+01 / 4.50e+01, 4.16e+00 / 2.72e+00 | 20260506 12:37:15.534 | INFO | PerfReport: 56.44%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.46e+00 / 1.80e+00 | 20260506 12:37:15.534 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.40e+00 / 3.02e+00, 4.21e-01 / 2.67e-01 | 20260506 12:37:15.534 | INFO | PerfReport: 3.73%, Metrics -- 1.18e+00 / 1.57e+00, 3.02e-01 / 1.99e-01 | 20260506 12:37:15.534 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.17e-01 / 4.74e-01, 6.57e-02 / 3.44e-02 | 20260506 12:37:15.534 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:37:15.534 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:37:15.534 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:37:15.534 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 12:37:15.534 | INFO | PerfReport: 21.48%, Upload -- 2.12e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 12:37:15.534 | INFO | PerfReport: 20.93%, Download -- 2.07e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 12:37:15.534 | INFO | PerfReport: 32.16%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 12:37:15.535 | INFO | PerfReport: 1.83%, LockUpload -- 1.81e+00 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 12:37:15.535 | INFO | PerfReport: 1.01%, LockDownload -- 9.99e-01 / 0.00e+00, 2.62e+00 / 0.00e+00 | 20260506 12:37:15.535 | INFO | PerfReport: 22.58%, LockCompute -- 2.23e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 12:37:15.535 | INFO | PerfReport: 0.02%, Misc -- 1.86e-02 / 0.00e+00, 1.07e-01 / 0.00e+00 | 20260506 12:37:15.665 | INFO | Starting a round of Baz IO | 20260506 12:37:18.830 | INFO | Populating RTMetrics report | 20260506 12:37:18.830 | INFO | RTMetrics report populated | 20260506 12:37:18.830 | INFO | Opening tmp RTMetrics json file | 20260506 12:37:18.831 | INFO | Serializing RTMetrics to json | 20260506 12:37:18.832 | INFO | Serialization done, swapping tmp file now... | 20260506 12:37:18.833 | INFO | RTMetrics done | 20260506 12:37:18.922 | INFO | Finished a round of Baz IO: Wrote 2.095 GiB in 3.256 seconds | 20260506 12:37:18.922 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.21% / 0.02% / 58.76% | 20260506 12:37:18.922 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.81431% / 4.37101e-05% / 3.97364e-05% | 20260506 12:37:18.922 | INFO | Baz IO overall: | 20260506 12:37:18.922 | INFO | BytesWritten : 702773972137 | 20260506 12:37:18.922 | INFO | NumEvents : 339683332173 | 20260506 12:37:20.672 | INFO | Analyzing chunk frames = [1409024,1409536) | 20260506 12:37:20.672 | INFO | MemInfo: { | 20260506 12:37:20.672 | INFO | "gpu" : " 17 GB", | 20260506 12:37:20.672 | INFO | "huge" : "", | 20260506 12:37:20.672 | INFO | "huge_shared" : " 27 GB", | 20260506 12:37:20.672 | INFO | "malloc" : " 14 GB", | 20260506 12:37:20.672 | INFO | "pinned" : " 29 GB" | 20260506 12:37:20.673 | INFO | } | 20260506 12:37:20.673 | INFO | Perf report for the last 5.13967 seconds: | 20260506 12:37:20.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:37:20.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:20.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:20.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:20.673 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 12:37:20.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:20.673 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:37:20.673 | INFO | PRE_HQ : 61.5%, 1.9, 61.7ms, 0 | 20260506 12:37:20.673 | INFO | BAZWRITER : 62.6%, 3.7, 118.4ms, 0 | 20260506 12:37:20.673 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:37:20.673 | INFO | Uploaded 36GiB to GPU in 2.07825s (17.3223GiB/s) | 20260506 12:37:20.673 | INFO | Downloaded 38GiB to GPU in 2.22244s (17.0983GiB/s) | 20260506 12:37:25.811 | INFO | Analyzing chunk frames = [1409536,1410048) | 20260506 12:37:25.811 | INFO | MemInfo: { | 20260506 12:37:25.811 | INFO | "gpu" : " 17 GB", | 20260506 12:37:25.811 | INFO | "huge" : "", | 20260506 12:37:25.811 | INFO | "huge_shared" : " 27 GB", | 20260506 12:37:25.811 | INFO | "malloc" : " 14 GB", | 20260506 12:37:25.812 | INFO | "pinned" : " 29 GB" | 20260506 12:37:25.812 | INFO | } | 20260506 12:37:25.812 | INFO | Perf report for the last 5.13909 seconds: | 20260506 12:37:25.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:37:25.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:25.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:25.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:25.812 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:37:25.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:25.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:25.812 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:37:25.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:25.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:25.812 | INFO | Uploaded 36GiB to GPU in 2.02202s (17.8039GiB/s) | 20260506 12:37:25.812 | INFO | Downloaded 34GiB to GPU in 1.92076s (17.7013GiB/s) | 20260506 12:37:30.951 | INFO | Analyzing chunk frames = [1410048,1410560) | 20260506 12:37:30.951 | INFO | MemInfo: { | 20260506 12:37:30.951 | INFO | "gpu" : " 17 GB", | 20260506 12:37:30.951 | INFO | "huge" : "", | 20260506 12:37:30.951 | INFO | "huge_shared" : " 27 GB", | 20260506 12:37:30.951 | INFO | "malloc" : " 14 GB", | 20260506 12:37:30.951 | INFO | "pinned" : " 29 GB" | 20260506 12:37:30.951 | INFO | } | 20260506 12:37:30.951 | INFO | Perf report for the last 5.13951 seconds: | 20260506 12:37:30.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:37:30.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:30.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:30.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:30.951 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260506 12:37:30.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:30.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:30.951 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:37:30.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:30.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:30.951 | INFO | Uploaded 36GiB to GPU in 2.00909s (17.9185GiB/s) | 20260506 12:37:30.951 | INFO | Downloaded 34GiB to GPU in 1.93946s (17.5306GiB/s) | 20260506 12:37:36.090 | INFO | Analyzing chunk frames = [1410560,1411072) | 20260506 12:37:36.090 | INFO | MemInfo: { | 20260506 12:37:36.090 | INFO | "gpu" : " 17 GB", | 20260506 12:37:36.090 | INFO | "huge" : "", | 20260506 12:37:36.090 | INFO | "huge_shared" : " 27 GB", | 20260506 12:37:36.090 | INFO | "malloc" : " 14 GB", | 20260506 12:37:36.090 | INFO | "pinned" : " 29 GB" | 20260506 12:37:36.090 | INFO | } | 20260506 12:37:36.090 | INFO | Perf report for the last 5.13892 seconds: | 20260506 12:37:36.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:37:36.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:36.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:36.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:36.090 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 12:37:36.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:36.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:36.090 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:37:36.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:36.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:36.090 | INFO | Uploaded 36GiB to GPU in 2.01618s (17.8556GiB/s) | 20260506 12:37:36.090 | INFO | Downloaded 34GiB to GPU in 1.94051s (17.5212GiB/s) | 20260506 12:37:41.229 | INFO | Analyzing chunk frames = [1411072,1411584) | 20260506 12:37:41.229 | INFO | MemInfo: { | 20260506 12:37:41.229 | INFO | "gpu" : " 17 GB", | 20260506 12:37:41.229 | INFO | "huge" : "", | 20260506 12:37:41.229 | INFO | "huge_shared" : " 27 GB", | 20260506 12:37:41.229 | INFO | "malloc" : " 14 GB", | 20260506 12:37:41.229 | INFO | "pinned" : " 29 GB" | 20260506 12:37:41.229 | INFO | } | 20260506 12:37:41.229 | INFO | Perf report for the last 5.13934 seconds: | 20260506 12:37:41.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:37:41.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:41.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:41.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:41.230 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 12:37:41.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:41.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:41.230 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:37:41.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:41.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:41.230 | INFO | Uploaded 36GiB to GPU in 2.01361s (17.8783GiB/s) | 20260506 12:37:41.230 | INFO | Downloaded 34GiB to GPU in 1.95185s (17.4193GiB/s) | 20260506 12:37:46.369 | INFO | Analyzing chunk frames = [1411584,1412096) | 20260506 12:37:46.369 | INFO | MemInfo: { | 20260506 12:37:46.369 | INFO | "gpu" : " 17 GB", | 20260506 12:37:46.369 | INFO | "huge" : "", | 20260506 12:37:46.369 | INFO | "huge_shared" : " 27 GB", | 20260506 12:37:46.369 | INFO | "malloc" : " 14 GB", | 20260506 12:37:46.369 | INFO | "pinned" : " 29 GB" | 20260506 12:37:46.369 | INFO | } | 20260506 12:37:46.369 | INFO | Perf report for the last 5.1394 seconds: | 20260506 12:37:46.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:37:46.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:46.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:46.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:46.369 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 12:37:46.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:46.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:46.369 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 12:37:46.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:46.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:46.369 | INFO | Uploaded 36GiB to GPU in 2.06954s (17.3952GiB/s) | 20260506 12:37:46.369 | INFO | Downloaded 34GiB to GPU in 1.92857s (17.6296GiB/s) | 20260506 12:37:51.509 | INFO | Analyzing chunk frames = [1412096,1412608) | 20260506 12:37:51.509 | INFO | MemInfo: { | 20260506 12:37:51.509 | INFO | "gpu" : " 17 GB", | 20260506 12:37:51.509 | INFO | "huge" : "", | 20260506 12:37:51.509 | INFO | "huge_shared" : " 27 GB", | 20260506 12:37:51.509 | INFO | "malloc" : " 14 GB", | 20260506 12:37:51.509 | INFO | "pinned" : " 29 GB" | 20260506 12:37:51.509 | INFO | } | 20260506 12:37:51.509 | INFO | Perf report for the last 5.14017 seconds: | 20260506 12:37:51.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:37:51.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:51.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:51.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:51.509 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:37:51.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:51.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:51.509 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:37:51.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:51.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:51.509 | INFO | Uploaded 36GiB to GPU in 2.0101s (17.9095GiB/s) | 20260506 12:37:51.509 | INFO | Downloaded 34GiB to GPU in 2.0346s (16.7109GiB/s) | 20260506 12:37:56.648 | INFO | Analyzing chunk frames = [1412608,1413120) | 20260506 12:37:56.648 | INFO | MemInfo: { | 20260506 12:37:56.648 | INFO | "gpu" : " 17 GB", | 20260506 12:37:56.648 | INFO | "huge" : "", | 20260506 12:37:56.648 | INFO | "huge_shared" : " 27 GB", | 20260506 12:37:56.648 | INFO | "malloc" : " 14 GB", | 20260506 12:37:56.648 | INFO | "pinned" : " 29 GB" | 20260506 12:37:56.648 | INFO | } | 20260506 12:37:56.648 | INFO | Perf report for the last 5.13877 seconds: | 20260506 12:37:56.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:37:56.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:56.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:56.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:56.648 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:37:56.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:56.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:56.648 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:37:56.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:56.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:37:56.648 | INFO | Uploaded 36GiB to GPU in 2.05817s (17.4913GiB/s) | 20260506 12:37:56.648 | INFO | Downloaded 34GiB to GPU in 1.96273s (17.3228GiB/s) | 20260506 12:37:56.780 | INFO | Starting a round of Baz IO | 20260506 12:37:59.949 | INFO | Populating RTMetrics report | 20260506 12:37:59.949 | INFO | RTMetrics report populated | 20260506 12:37:59.949 | INFO | Opening tmp RTMetrics json file | 20260506 12:37:59.950 | INFO | Serializing RTMetrics to json | 20260506 12:37:59.951 | INFO | Serialization done, swapping tmp file now... | 20260506 12:37:59.951 | INFO | RTMetrics done | 20260506 12:38:00.044 | INFO | Finished a round of Baz IO: Wrote 2.096 GiB in 3.264 seconds | 20260506 12:38:00.044 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.19% / 0.02% / 58.79% | 20260506 12:38:00.045 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.80813% / 4.37101e-05% / 5.16574e-05% | 20260506 12:38:00.045 | INFO | Baz IO overall: | 20260506 12:38:00.045 | INFO | BytesWritten : 705024256226 | 20260506 12:38:00.045 | INFO | NumEvents : 340775172069 | 20260506 12:38:01.787 | INFO | Analyzing chunk frames = [1413120,1413632) | 20260506 12:38:01.787 | INFO | MemInfo: { | 20260506 12:38:01.787 | INFO | "gpu" : " 17 GB", | 20260506 12:38:01.787 | INFO | "huge" : "", | 20260506 12:38:01.787 | INFO | "huge_shared" : " 27 GB", | 20260506 12:38:01.787 | INFO | "malloc" : " 14 GB", | 20260506 12:38:01.787 | INFO | "pinned" : " 29 GB" | 20260506 12:38:01.787 | INFO | } | 20260506 12:38:01.787 | INFO | Perf report for the last 5.13952 seconds: | 20260506 12:38:01.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:38:01.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:01.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:01.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:01.787 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260506 12:38:01.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:01.787 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:38:01.787 | INFO | PRE_HQ : 61.4%, 1.9, 61.3ms, 0 | 20260506 12:38:01.787 | INFO | BAZWRITER : 62.9%, 3.7, 118.2ms, 0 | 20260506 12:38:01.787 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:38:01.788 | INFO | Uploaded 36GiB to GPU in 2.09161s (17.2116GiB/s) | 20260506 12:38:01.788 | INFO | Downloaded 38GiB to GPU in 2.22748s (17.0596GiB/s) | 20260506 12:38:06.926 | INFO | Analyzing chunk frames = [1413632,1414144) | 20260506 12:38:06.926 | INFO | MemInfo: { | 20260506 12:38:06.926 | INFO | "gpu" : " 17 GB", | 20260506 12:38:06.926 | INFO | "huge" : "", | 20260506 12:38:06.926 | INFO | "huge_shared" : " 27 GB", | 20260506 12:38:06.926 | INFO | "malloc" : " 14 GB", | 20260506 12:38:06.926 | INFO | "pinned" : " 29 GB" | 20260506 12:38:06.926 | INFO | } | 20260506 12:38:06.926 | INFO | Perf report for the last 5.13901 seconds: | 20260506 12:38:06.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:38:06.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:06.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:06.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:06.926 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:38:06.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:06.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:06.926 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:38:06.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:06.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:06.927 | INFO | Uploaded 36GiB to GPU in 2.02082s (17.8145GiB/s) | 20260506 12:38:06.927 | INFO | Downloaded 34GiB to GPU in 1.9244s (17.6679GiB/s) | 20260506 12:38:12.066 | INFO | Analyzing chunk frames = [1414144,1414656) | 20260506 12:38:12.066 | INFO | MemInfo: { | 20260506 12:38:12.066 | INFO | "gpu" : " 17 GB", | 20260506 12:38:12.066 | INFO | "huge" : "", | 20260506 12:38:12.066 | INFO | "huge_shared" : " 27 GB", | 20260506 12:38:12.066 | INFO | "malloc" : " 14 GB", | 20260506 12:38:12.066 | INFO | "pinned" : " 29 GB" | 20260506 12:38:12.066 | INFO | } | 20260506 12:38:12.066 | INFO | Perf report for the last 5.13952 seconds: | 20260506 12:38:12.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:38:12.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:12.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:12.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:12.066 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 12:38:12.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:12.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:12.066 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 12:38:12.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:12.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:12.066 | INFO | Uploaded 36GiB to GPU in 2.01503s (17.8658GiB/s) | 20260506 12:38:12.066 | INFO | Downloaded 34GiB to GPU in 1.93444s (17.5762GiB/s) | 20260506 12:38:17.205 | INFO | Analyzing chunk frames = [1414656,1415168) | 20260506 12:38:17.205 | INFO | MemInfo: { | 20260506 12:38:17.205 | INFO | "gpu" : " 17 GB", | 20260506 12:38:17.205 | INFO | "huge" : "", | 20260506 12:38:17.205 | INFO | "huge_shared" : " 27 GB", | 20260506 12:38:17.205 | INFO | "malloc" : " 14 GB", | 20260506 12:38:17.205 | INFO | "pinned" : " 29 GB" | 20260506 12:38:17.205 | INFO | } | 20260506 12:38:17.205 | INFO | Perf report for the last 5.13907 seconds: | 20260506 12:38:17.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:38:17.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:17.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:17.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:17.205 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 12:38:17.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:17.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:17.205 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:38:17.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:17.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:17.205 | INFO | Uploaded 36GiB to GPU in 2.01949s (17.8263GiB/s) | 20260506 12:38:17.205 | INFO | Downloaded 34GiB to GPU in 1.93496s (17.5714GiB/s) | 20260506 12:38:22.344 | INFO | Analyzing chunk frames = [1415168,1415680) | 20260506 12:38:22.344 | INFO | MemInfo: { | 20260506 12:38:22.344 | INFO | "gpu" : " 17 GB", | 20260506 12:38:22.344 | INFO | "huge" : "", | 20260506 12:38:22.344 | INFO | "huge_shared" : " 27 GB", | 20260506 12:38:22.344 | INFO | "malloc" : " 14 GB", | 20260506 12:38:22.344 | INFO | "pinned" : " 29 GB" | 20260506 12:38:22.344 | INFO | } | 20260506 12:38:22.344 | INFO | Perf report for the last 5.13943 seconds: | 20260506 12:38:22.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:38:22.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:22.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:22.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:22.345 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 12:38:22.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:22.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:22.345 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:38:22.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:22.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:22.345 | INFO | Uploaded 36GiB to GPU in 2.00848s (17.924GiB/s) | 20260506 12:38:22.345 | INFO | Downloaded 34GiB to GPU in 1.94676s (17.4649GiB/s) | 20260506 12:38:27.484 | INFO | Analyzing chunk frames = [1415680,1416192) | 20260506 12:38:27.484 | INFO | MemInfo: { | 20260506 12:38:27.484 | INFO | "gpu" : " 17 GB", | 20260506 12:38:27.484 | INFO | "huge" : "", | 20260506 12:38:27.484 | INFO | "huge_shared" : " 27 GB", | 20260506 12:38:27.484 | INFO | "malloc" : " 14 GB", | 20260506 12:38:27.484 | INFO | "pinned" : " 29 GB" | 20260506 12:38:27.484 | INFO | } | 20260506 12:38:27.484 | INFO | Perf report for the last 5.13947 seconds: | 20260506 12:38:27.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:38:27.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:27.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:27.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:27.484 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 12:38:27.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:27.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:27.484 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260506 12:38:27.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:27.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:27.484 | INFO | Uploaded 36GiB to GPU in 2.05059s (17.5559GiB/s) | 20260506 12:38:27.484 | INFO | Downloaded 34GiB to GPU in 1.93172s (17.6009GiB/s) | 20260506 12:38:32.624 | INFO | Analyzing chunk frames = [1416192,1416704) | 20260506 12:38:32.624 | INFO | MemInfo: { | 20260506 12:38:32.624 | INFO | "gpu" : " 17 GB", | 20260506 12:38:32.624 | INFO | "huge" : "", | 20260506 12:38:32.624 | INFO | "huge_shared" : " 27 GB", | 20260506 12:38:32.624 | INFO | "malloc" : " 14 GB", | 20260506 12:38:32.624 | INFO | "pinned" : " 29 GB" | 20260506 12:38:32.624 | INFO | } | 20260506 12:38:32.624 | INFO | Perf report for the last 5.14043 seconds: | 20260506 12:38:32.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:38:32.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:32.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:32.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:32.624 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:38:32.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:32.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:32.624 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:38:32.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:32.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:32.625 | INFO | Uploaded 36GiB to GPU in 2.0128s (17.8855GiB/s) | 20260506 12:38:32.625 | INFO | Downloaded 34GiB to GPU in 2.03137s (16.7375GiB/s) | 20260506 12:38:37.763 | INFO | Analyzing chunk frames = [1416704,1417216) | 20260506 12:38:37.763 | INFO | MemInfo: { | 20260506 12:38:37.763 | INFO | "gpu" : " 17 GB", | 20260506 12:38:37.763 | INFO | "huge" : "", | 20260506 12:38:37.763 | INFO | "huge_shared" : " 27 GB", | 20260506 12:38:37.763 | INFO | "malloc" : " 14 GB", | 20260506 12:38:37.763 | INFO | "pinned" : " 29 GB" | 20260506 12:38:37.763 | INFO | } | 20260506 12:38:37.763 | INFO | Perf report for the last 5.13842 seconds: | 20260506 12:38:37.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:38:37.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:37.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:37.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:37.763 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:38:37.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:37.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:37.763 | INFO | PRE_HQ : 33.5%, 1.0, 17.4ms, 0 | 20260506 12:38:37.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:37.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:37.763 | INFO | Uploaded 36GiB to GPU in 2.03209s (17.7158GiB/s) | 20260506 12:38:37.763 | INFO | Downloaded 34GiB to GPU in 1.96532s (17.3GiB/s) | 20260506 12:38:37.763 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:38:37.763 | INFO | Total mem usage: 16.8 GiB | 20260506 12:38:37.763 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:38:37.763 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.5%) | 20260506 12:38:37.763 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:38:37.763 | INFO | High Water Breakdown: | 20260506 12:38:37.763 | INFO | 59.7% : 8.62 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:38:37.763 | INFO | 40.2% : 5.8 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:38:37.763 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:38:37.763 | INFO | ------------------------------------------ | 20260506 12:38:37.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:38:37.763 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:38:37.763 | INFO | Total mem usage: 26.5 GiB | 20260506 12:38:37.763 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:38:37.763 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:38:37.763 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:38:37.763 | INFO | High Water Breakdown: | 20260506 12:38:37.763 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:38:37.763 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:38:37.763 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:38:37.763 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:38:37.763 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:38:37.763 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:38:37.763 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:38:37.763 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:38:37.763 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:38:37.763 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:38:37.763 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:38:37.763 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:38:37.763 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:38:37.763 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:38:37.763 | INFO | ------------------------------------------ | 20260506 12:38:37.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:38:37.763 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:38:37.763 | INFO | Total mem usage: 25.6 GiB | 20260506 12:38:37.763 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:38:37.763 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:38:37.763 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:38:37.763 | INFO | High Water Breakdown: | 20260506 12:38:37.764 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:38:37.764 | INFO | ------------------------------------------ | 20260506 12:38:37.764 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:38:37.764 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:38:37.764 | INFO | Total mem usage: 15.7 GiB | 20260506 12:38:37.764 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:38:37.764 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:38:37.764 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 12:38:37.764 | INFO | High Water Breakdown: | 20260506 12:38:37.764 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:38:37.764 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:38:37.764 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:38:37.764 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:38:37.764 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:38:37.764 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:38:37.764 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:38:37.764 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:38:37.764 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:38:37.764 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:38:37.764 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:38:37.764 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:38:37.764 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:38:37.764 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:38:37.764 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:38:37.764 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:38:37.764 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:38:37.764 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:38:37.764 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:38:37.764 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:38:37.764 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:38:37.764 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:38:37.764 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:38:37.764 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:38:37.764 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:38:37.764 | INFO | ------------------------------------------ | 20260506 12:38:37.764 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:38:37.764 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:38:37.764 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:38:37.764 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:38:37.764 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.66e+01, 1.36e+01 / 2.76e+00 | 20260506 12:38:37.764 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.02e-01, 1.04e-02 / 7.46e-03 | 20260506 12:38:37.764 | INFO | PerfReport: 11.54%, Baseline -- 3.67e+00 / 4.37e+00, 5.82e-01 / 3.84e-01 | 20260506 12:38:37.764 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.48e+00, 1.96e-01 / 1.42e-01 | 20260506 12:38:37.764 | INFO | PerfReport: 15.87%, DME -- 4.03e+01 / 4.51e+01, 4.49e+00 / 3.26e+00 | 20260506 12:38:37.764 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.48e+00 / 1.74e+00 | 20260506 12:38:37.764 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.39e+00 / 2.97e+00, 4.14e-01 / 2.71e-01 | 20260506 12:38:37.764 | INFO | PerfReport: 3.72%, Metrics -- 1.18e+00 / 1.56e+00, 2.98e-01 / 1.68e-01 | 20260506 12:38:37.764 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.17e-01 / 4.75e-01, 6.54e-02 / 3.73e-02 | 20260506 12:38:37.764 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:38:37.764 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:38:37.764 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:38:37.764 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 12:38:37.764 | INFO | PerfReport: 21.44%, Upload -- 2.12e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260506 12:38:37.764 | INFO | PerfReport: 20.96%, Download -- 2.07e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 12:38:37.764 | INFO | PerfReport: 32.18%, Compute -- 3.18e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 12:38:37.764 | INFO | PerfReport: 1.76%, LockUpload -- 1.74e+00 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 12:38:37.764 | INFO | PerfReport: 1.06%, LockDownload -- 1.05e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260506 12:38:37.764 | INFO | PerfReport: 22.59%, LockCompute -- 2.23e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 12:38:37.764 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260506 12:38:37.895 | INFO | Starting a round of Baz IO | 20260506 12:38:41.057 | INFO | Populating RTMetrics report | 20260506 12:38:41.058 | INFO | RTMetrics report populated | 20260506 12:38:41.058 | INFO | Opening tmp RTMetrics json file | 20260506 12:38:41.058 | INFO | Serializing RTMetrics to json | 20260506 12:38:41.059 | INFO | Serialization done, swapping tmp file now... | 20260506 12:38:41.060 | INFO | RTMetrics done | 20260506 12:38:41.153 | INFO | Finished a round of Baz IO: Wrote 2.095 GiB in 3.258 seconds | 20260506 12:38:41.153 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.17% / 0.02% / 58.81% | 20260506 12:38:41.153 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.79637% / 4.76837e-05% / 3.57628e-05% | 20260506 12:38:41.153 | INFO | Baz IO overall: | 20260506 12:38:41.153 | INFO | BytesWritten : 707274032451 | 20260506 12:38:41.153 | INFO | NumEvents : 341866409474 | 20260506 12:38:42.902 | INFO | Analyzing chunk frames = [1417216,1417728) | 20260506 12:38:42.902 | INFO | MemInfo: { | 20260506 12:38:42.902 | INFO | "gpu" : " 17 GB", | 20260506 12:38:42.902 | INFO | "huge" : "", | 20260506 12:38:42.902 | INFO | "huge_shared" : " 27 GB", | 20260506 12:38:42.902 | INFO | "malloc" : " 14 GB", | 20260506 12:38:42.902 | INFO | "pinned" : " 29 GB" | 20260506 12:38:42.902 | INFO | } | 20260506 12:38:42.902 | INFO | Perf report for the last 5.13922 seconds: | 20260506 12:38:42.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:38:42.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:42.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:42.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:42.902 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260506 12:38:42.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:42.902 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:38:42.902 | INFO | PRE_HQ : 61.4%, 1.9, 61.0ms, 0 | 20260506 12:38:42.902 | INFO | BAZWRITER : 62.8%, 3.6, 117.6ms, 0 | 20260506 12:38:42.902 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:38:42.902 | INFO | Uploaded 36GiB to GPU in 2.04265s (17.6242GiB/s) | 20260506 12:38:42.902 | INFO | Downloaded 38GiB to GPU in 2.21922s (17.1232GiB/s) | 20260506 12:38:48.041 | INFO | Analyzing chunk frames = [1417728,1418240) | 20260506 12:38:48.041 | INFO | MemInfo: { | 20260506 12:38:48.041 | INFO | "gpu" : " 17 GB", | 20260506 12:38:48.041 | INFO | "huge" : "", | 20260506 12:38:48.041 | INFO | "huge_shared" : " 27 GB", | 20260506 12:38:48.041 | INFO | "malloc" : " 14 GB", | 20260506 12:38:48.041 | INFO | "pinned" : " 29 GB" | 20260506 12:38:48.041 | INFO | } | 20260506 12:38:48.041 | INFO | Perf report for the last 5.13913 seconds: | 20260506 12:38:48.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:38:48.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:48.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:48.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:48.041 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 12:38:48.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:48.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:48.041 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:38:48.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:48.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:48.041 | INFO | Uploaded 36GiB to GPU in 2.0277s (17.7541GiB/s) | 20260506 12:38:48.041 | INFO | Downloaded 34GiB to GPU in 1.93911s (17.5338GiB/s) | 20260506 12:38:53.180 | INFO | Analyzing chunk frames = [1418240,1418752) | 20260506 12:38:53.180 | INFO | MemInfo: { | 20260506 12:38:53.180 | INFO | "gpu" : " 17 GB", | 20260506 12:38:53.180 | INFO | "huge" : "", | 20260506 12:38:53.180 | INFO | "huge_shared" : " 27 GB", | 20260506 12:38:53.180 | INFO | "malloc" : " 14 GB", | 20260506 12:38:53.180 | INFO | "pinned" : " 29 GB" | 20260506 12:38:53.180 | INFO | } | 20260506 12:38:53.180 | INFO | Perf report for the last 5.1393 seconds: | 20260506 12:38:53.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:38:53.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:53.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:53.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:53.181 | INFO | ANALYSIS : 61.7%, 3.0, 95.5ms, 0 | 20260506 12:38:53.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:53.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:53.181 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:38:53.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:53.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:53.181 | INFO | Uploaded 36GiB to GPU in 2.0154s (17.8625GiB/s) | 20260506 12:38:53.181 | INFO | Downloaded 34GiB to GPU in 1.94357s (17.4936GiB/s) | 20260506 12:38:58.320 | INFO | Analyzing chunk frames = [1418752,1419264) | 20260506 12:38:58.320 | INFO | MemInfo: { | 20260506 12:38:58.320 | INFO | "gpu" : " 17 GB", | 20260506 12:38:58.320 | INFO | "huge" : "", | 20260506 12:38:58.320 | INFO | "huge_shared" : " 27 GB", | 20260506 12:38:58.320 | INFO | "malloc" : " 14 GB", | 20260506 12:38:58.320 | INFO | "pinned" : " 29 GB" | 20260506 12:38:58.320 | INFO | } | 20260506 12:38:58.320 | INFO | Perf report for the last 5.13949 seconds: | 20260506 12:38:58.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:38:58.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:58.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:58.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:58.320 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 12:38:58.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:58.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:58.320 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:38:58.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:58.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:38:58.321 | INFO | Uploaded 36GiB to GPU in 1.99082s (18.083GiB/s) | 20260506 12:38:58.321 | INFO | Downloaded 34GiB to GPU in 1.93762s (17.5473GiB/s) | 20260506 12:39:03.459 | INFO | Analyzing chunk frames = [1419264,1419776) | 20260506 12:39:03.459 | INFO | MemInfo: { | 20260506 12:39:03.459 | INFO | "gpu" : " 17 GB", | 20260506 12:39:03.459 | INFO | "huge" : "", | 20260506 12:39:03.459 | INFO | "huge_shared" : " 27 GB", | 20260506 12:39:03.459 | INFO | "malloc" : " 14 GB", | 20260506 12:39:03.459 | INFO | "pinned" : " 29 GB" | 20260506 12:39:03.459 | INFO | } | 20260506 12:39:03.459 | INFO | Perf report for the last 5.13903 seconds: | 20260506 12:39:03.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:39:03.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:03.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:03.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:03.459 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 12:39:03.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:03.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:03.459 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:39:03.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:03.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:03.459 | INFO | Uploaded 36GiB to GPU in 1.99482s (18.0467GiB/s) | 20260506 12:39:03.459 | INFO | Downloaded 34GiB to GPU in 1.94719s (17.461GiB/s) | 20260506 12:39:08.598 | INFO | Analyzing chunk frames = [1419776,1420288) | 20260506 12:39:08.598 | INFO | MemInfo: { | 20260506 12:39:08.598 | INFO | "gpu" : " 17 GB", | 20260506 12:39:08.598 | INFO | "huge" : "", | 20260506 12:39:08.599 | INFO | "huge_shared" : " 27 GB", | 20260506 12:39:08.599 | INFO | "malloc" : " 14 GB", | 20260506 12:39:08.599 | INFO | "pinned" : " 29 GB" | 20260506 12:39:08.599 | INFO | } | 20260506 12:39:08.599 | INFO | Perf report for the last 5.13959 seconds: | 20260506 12:39:08.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:39:08.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:08.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:08.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:08.599 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 12:39:08.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:08.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:08.599 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:39:08.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:08.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:08.599 | INFO | Uploaded 36GiB to GPU in 2.09037s (17.2219GiB/s) | 20260506 12:39:08.599 | INFO | Downloaded 34GiB to GPU in 1.97962s (17.175GiB/s) | 20260506 12:39:13.738 | INFO | Analyzing chunk frames = [1420288,1420800) | 20260506 12:39:13.738 | INFO | MemInfo: { | 20260506 12:39:13.738 | INFO | "gpu" : " 17 GB", | 20260506 12:39:13.738 | INFO | "huge" : "", | 20260506 12:39:13.738 | INFO | "huge_shared" : " 27 GB", | 20260506 12:39:13.738 | INFO | "malloc" : " 14 GB", | 20260506 12:39:13.738 | INFO | "pinned" : " 29 GB" | 20260506 12:39:13.739 | INFO | } | 20260506 12:39:13.739 | INFO | Perf report for the last 5.13988 seconds: | 20260506 12:39:13.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:39:13.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:13.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:13.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:13.739 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:39:13.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:13.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:13.739 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 12:39:13.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:13.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:13.739 | INFO | Uploaded 36GiB to GPU in 1.99243s (18.0684GiB/s) | 20260506 12:39:13.739 | INFO | Downloaded 34GiB to GPU in 1.99662s (17.0288GiB/s) | 20260506 12:39:18.877 | INFO | Analyzing chunk frames = [1420800,1421312) | 20260506 12:39:18.877 | INFO | MemInfo: { | 20260506 12:39:18.877 | INFO | "gpu" : " 17 GB", | 20260506 12:39:18.877 | INFO | "huge" : "", | 20260506 12:39:18.877 | INFO | "huge_shared" : " 27 GB", | 20260506 12:39:18.877 | INFO | "malloc" : " 14 GB", | 20260506 12:39:18.877 | INFO | "pinned" : " 29 GB" | 20260506 12:39:18.877 | INFO | } | 20260506 12:39:18.877 | INFO | Perf report for the last 5.13847 seconds: | 20260506 12:39:18.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:39:18.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:18.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:18.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:18.877 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:39:18.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:18.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:18.877 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 12:39:18.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:18.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:18.877 | INFO | Uploaded 36GiB to GPU in 2.06308s (17.4497GiB/s) | 20260506 12:39:18.877 | INFO | Downloaded 34GiB to GPU in 1.97105s (17.2497GiB/s) | 20260506 12:39:19.000 | INFO | Starting a round of Baz IO | 20260506 12:39:22.172 | INFO | Populating RTMetrics report | 20260506 12:39:22.172 | INFO | RTMetrics report populated | 20260506 12:39:22.172 | INFO | Opening tmp RTMetrics json file | 20260506 12:39:22.173 | INFO | Serializing RTMetrics to json | 20260506 12:39:22.174 | INFO | Serialization done, swapping tmp file now... | 20260506 12:39:22.174 | INFO | RTMetrics done | 20260506 12:39:22.273 | INFO | Finished a round of Baz IO: Wrote 2.094 GiB in 3.273 seconds | 20260506 12:39:22.274 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.14% / 0.02% / 58.83% | 20260506 12:39:22.274 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.78221% / 4.76837e-05% / 3.97364e-05% | 20260506 12:39:22.274 | INFO | Baz IO overall: | 20260506 12:39:22.274 | INFO | BytesWritten : 709522531129 | 20260506 12:39:22.274 | INFO | NumEvents : 342956506906 | 20260506 12:39:24.017 | INFO | Analyzing chunk frames = [1421312,1421824) | 20260506 12:39:24.017 | INFO | MemInfo: { | 20260506 12:39:24.017 | INFO | "gpu" : " 17 GB", | 20260506 12:39:24.017 | INFO | "huge" : "", | 20260506 12:39:24.017 | INFO | "huge_shared" : " 27 GB", | 20260506 12:39:24.017 | INFO | "malloc" : " 14 GB", | 20260506 12:39:24.017 | INFO | "pinned" : " 29 GB" | 20260506 12:39:24.017 | INFO | } | 20260506 12:39:24.017 | INFO | Perf report for the last 5.13965 seconds: | 20260506 12:39:24.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:39:24.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:24.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:24.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:24.017 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260506 12:39:24.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:24.017 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:39:24.017 | INFO | PRE_HQ : 61.5%, 1.9, 61.1ms, 0 | 20260506 12:39:24.017 | INFO | BAZWRITER : 62.9%, 3.7, 119.6ms, 0 | 20260506 12:39:24.017 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:39:24.017 | INFO | Uploaded 36GiB to GPU in 2.09679s (17.1691GiB/s) | 20260506 12:39:24.017 | INFO | Downloaded 38GiB to GPU in 2.20827s (17.208GiB/s) | 20260506 12:39:29.156 | INFO | Analyzing chunk frames = [1421824,1422336) | 20260506 12:39:29.156 | INFO | MemInfo: { | 20260506 12:39:29.156 | INFO | "gpu" : " 17 GB", | 20260506 12:39:29.156 | INFO | "huge" : "", | 20260506 12:39:29.156 | INFO | "huge_shared" : " 27 GB", | 20260506 12:39:29.156 | INFO | "malloc" : " 14 GB", | 20260506 12:39:29.156 | INFO | "pinned" : " 29 GB" | 20260506 12:39:29.156 | INFO | } | 20260506 12:39:29.156 | INFO | Perf report for the last 5.13944 seconds: | 20260506 12:39:29.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:39:29.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:29.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:29.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:29.156 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 12:39:29.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:29.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:29.156 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:39:29.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:29.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:29.157 | INFO | Uploaded 36GiB to GPU in 2.02147s (17.8088GiB/s) | 20260506 12:39:29.157 | INFO | Downloaded 34GiB to GPU in 1.93328s (17.5867GiB/s) | 20260506 12:39:34.295 | INFO | Analyzing chunk frames = [1422336,1422848) | 20260506 12:39:34.295 | INFO | MemInfo: { | 20260506 12:39:34.295 | INFO | "gpu" : " 17 GB", | 20260506 12:39:34.295 | INFO | "huge" : "", | 20260506 12:39:34.295 | INFO | "huge_shared" : " 27 GB", | 20260506 12:39:34.295 | INFO | "malloc" : " 14 GB", | 20260506 12:39:34.295 | INFO | "pinned" : " 29 GB" | 20260506 12:39:34.295 | INFO | } | 20260506 12:39:34.296 | INFO | Perf report for the last 5.13936 seconds: | 20260506 12:39:34.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:39:34.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:34.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:34.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:34.296 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 12:39:34.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:34.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:34.296 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:39:34.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:34.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:34.296 | INFO | Uploaded 36GiB to GPU in 2.01574s (17.8594GiB/s) | 20260506 12:39:34.296 | INFO | Downloaded 34GiB to GPU in 1.94015s (17.5244GiB/s) | 20260506 12:39:39.435 | INFO | Analyzing chunk frames = [1422848,1423360) | 20260506 12:39:39.435 | INFO | MemInfo: { | 20260506 12:39:39.435 | INFO | "gpu" : " 17 GB", | 20260506 12:39:39.435 | INFO | "huge" : "", | 20260506 12:39:39.435 | INFO | "huge_shared" : " 27 GB", | 20260506 12:39:39.435 | INFO | "malloc" : " 14 GB", | 20260506 12:39:39.435 | INFO | "pinned" : " 29 GB" | 20260506 12:39:39.435 | INFO | } | 20260506 12:39:39.435 | INFO | Perf report for the last 5.13924 seconds: | 20260506 12:39:39.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:39:39.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:39.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:39.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:39.435 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 12:39:39.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:39.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:39.435 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:39:39.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:39.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:39.435 | INFO | Uploaded 36GiB to GPU in 1.9993s (18.0063GiB/s) | 20260506 12:39:39.435 | INFO | Downloaded 34GiB to GPU in 1.93956s (17.5297GiB/s) | 20260506 12:39:44.574 | INFO | Analyzing chunk frames = [1423360,1423872) | 20260506 12:39:44.574 | INFO | MemInfo: { | 20260506 12:39:44.574 | INFO | "gpu" : " 17 GB", | 20260506 12:39:44.574 | INFO | "huge" : "", | 20260506 12:39:44.574 | INFO | "huge_shared" : " 27 GB", | 20260506 12:39:44.574 | INFO | "malloc" : " 14 GB", | 20260506 12:39:44.574 | INFO | "pinned" : " 29 GB" | 20260506 12:39:44.574 | INFO | } | 20260506 12:39:44.574 | INFO | Perf report for the last 5.13956 seconds: | 20260506 12:39:44.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:39:44.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:44.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:44.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:44.574 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 12:39:44.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:44.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:44.574 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:39:44.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:44.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:44.575 | INFO | Uploaded 36GiB to GPU in 2.01657s (17.8521GiB/s) | 20260506 12:39:44.575 | INFO | Downloaded 34GiB to GPU in 1.9537s (17.4028GiB/s) | 20260506 12:39:49.713 | INFO | Analyzing chunk frames = [1423872,1424384) | 20260506 12:39:49.713 | INFO | MemInfo: { | 20260506 12:39:49.713 | INFO | "gpu" : " 17 GB", | 20260506 12:39:49.713 | INFO | "huge" : "", | 20260506 12:39:49.713 | INFO | "huge_shared" : " 27 GB", | 20260506 12:39:49.713 | INFO | "malloc" : " 14 GB", | 20260506 12:39:49.713 | INFO | "pinned" : " 29 GB" | 20260506 12:39:49.713 | INFO | } | 20260506 12:39:49.713 | INFO | Perf report for the last 5.13901 seconds: | 20260506 12:39:49.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:39:49.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:49.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:49.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:49.713 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:39:49.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:49.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:49.713 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260506 12:39:49.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:49.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:49.714 | INFO | Uploaded 36GiB to GPU in 2.09426s (17.1899GiB/s) | 20260506 12:39:49.714 | INFO | Downloaded 34GiB to GPU in 1.97534s (17.2122GiB/s) | 20260506 12:39:54.852 | INFO | Analyzing chunk frames = [1424384,1424896) | 20260506 12:39:54.853 | INFO | MemInfo: { | 20260506 12:39:54.853 | INFO | "gpu" : " 17 GB", | 20260506 12:39:54.853 | INFO | "huge" : "", | 20260506 12:39:54.853 | INFO | "huge_shared" : " 27 GB", | 20260506 12:39:54.853 | INFO | "malloc" : " 14 GB", | 20260506 12:39:54.853 | INFO | "pinned" : " 29 GB" | 20260506 12:39:54.853 | INFO | } | 20260506 12:39:54.853 | INFO | Perf report for the last 5.13929 seconds: | 20260506 12:39:54.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:39:54.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:54.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:54.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:54.853 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 12:39:54.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:54.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:54.853 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 12:39:54.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:54.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:54.853 | INFO | Uploaded 36GiB to GPU in 1.99856s (18.013GiB/s) | 20260506 12:39:54.853 | INFO | Downloaded 34GiB to GPU in 2.00492s (16.9583GiB/s) | 20260506 12:39:59.992 | INFO | Analyzing chunk frames = [1424896,1425408) | 20260506 12:39:59.992 | INFO | MemInfo: { | 20260506 12:39:59.992 | INFO | "gpu" : " 17 GB", | 20260506 12:39:59.992 | INFO | "huge" : "", | 20260506 12:39:59.992 | INFO | "huge_shared" : " 27 GB", | 20260506 12:39:59.992 | INFO | "malloc" : " 14 GB", | 20260506 12:39:59.992 | INFO | "pinned" : " 29 GB" | 20260506 12:39:59.992 | INFO | } | 20260506 12:39:59.992 | INFO | Perf report for the last 5.13941 seconds: | 20260506 12:39:59.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:39:59.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:59.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:59.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:59.992 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:39:59.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:59.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:59.992 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:39:59.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:59.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:39:59.992 | INFO | Uploaded 36GiB to GPU in 2.03131s (17.7225GiB/s) | 20260506 12:39:59.992 | INFO | Downloaded 34GiB to GPU in 1.97722s (17.1959GiB/s) | 20260506 12:39:59.993 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:39:59.993 | INFO | Total mem usage: 16.8 GiB | 20260506 12:39:59.993 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:39:59.993 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -16.2%) | 20260506 12:39:59.993 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:39:59.993 | INFO | High Water Breakdown: | 20260506 12:39:59.993 | INFO | 59.7% : 8.63 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:39:59.993 | INFO | 40.2% : 5.81 GiB ( 86.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:39:59.993 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:39:59.993 | INFO | ------------------------------------------ | 20260506 12:39:59.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:39:59.993 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:39:59.993 | INFO | Total mem usage: 26.5 GiB | 20260506 12:39:59.993 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:39:59.993 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:39:59.993 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:39:59.993 | INFO | High Water Breakdown: | 20260506 12:39:59.993 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:39:59.993 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:39:59.993 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:39:59.993 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:39:59.993 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:39:59.993 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:39:59.993 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:39:59.993 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:39:59.993 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:39:59.993 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:39:59.993 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:39:59.993 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:39:59.993 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:39:59.993 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:39:59.993 | INFO | ------------------------------------------ | 20260506 12:39:59.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:39:59.993 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:39:59.993 | INFO | Total mem usage: 25.6 GiB | 20260506 12:39:59.993 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:39:59.993 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:39:59.993 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:39:59.993 | INFO | High Water Breakdown: | 20260506 12:39:59.993 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:39:59.993 | INFO | ------------------------------------------ | 20260506 12:39:59.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:39:59.993 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:39:59.993 | INFO | Total mem usage: 15.7 GiB | 20260506 12:39:59.993 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:39:59.993 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:39:59.993 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:39:59.993 | INFO | High Water Breakdown: | 20260506 12:39:59.993 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:39:59.993 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:39:59.993 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:39:59.993 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:39:59.993 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:39:59.993 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:39:59.993 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:39:59.993 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:39:59.993 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:39:59.993 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:39:59.993 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:39:59.993 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:39:59.993 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:39:59.993 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:39:59.993 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:39:59.993 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:39:59.993 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:39:59.993 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:39:59.993 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:39:59.993 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:39:59.993 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:39:59.993 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:39:59.993 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:39:59.993 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:39:59.993 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:39:59.993 | INFO | ------------------------------------------ | 20260506 12:39:59.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:39:59.993 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:39:59.993 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:39:59.993 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:39:59.993 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.61e+01, 1.37e+01 / 2.33e+00 | 20260506 12:39:59.993 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.01e-01, 9.96e-03 / 6.88e-03 | 20260506 12:39:59.993 | INFO | PerfReport: 11.56%, Baseline -- 3.67e+00 / 4.42e+00, 5.75e-01 / 3.48e-01 | 20260506 12:39:59.993 | INFO | PerfReport: 3.23%, Histogram -- 1.03e+00 / 1.48e+00, 1.89e-01 / 1.43e-01 | 20260506 12:39:59.993 | INFO | PerfReport: 15.83%, DME -- 4.02e+01 / 4.47e+01, 4.18e+00 / 2.80e+00 | 20260506 12:39:59.993 | INFO | PerfReport: 56.47%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.45e+00 / 1.65e+00 | 20260506 12:39:59.993 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.40e+00 / 3.04e+00, 4.18e-01 / 2.44e-01 | 20260506 12:39:59.993 | INFO | PerfReport: 3.72%, Metrics -- 1.18e+00 / 1.56e+00, 2.98e-01 / 1.86e-01 | 20260506 12:39:59.993 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.17e-01 / 4.74e-01, 6.53e-02 / 3.10e-02 | 20260506 12:39:59.993 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:39:59.993 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:39:59.993 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:39:59.993 | INFO | PerfReport: Full Execution -- 9.88e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 12:39:59.993 | INFO | PerfReport: 21.42%, Upload -- 2.12e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260506 12:39:59.993 | INFO | PerfReport: 21.01%, Download -- 2.07e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 12:39:59.993 | INFO | PerfReport: 32.14%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 12:39:59.993 | INFO | PerfReport: 1.79%, LockUpload -- 1.76e+00 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 12:39:59.993 | INFO | PerfReport: 1.04%, LockDownload -- 1.03e+00 / 0.00e+00, 2.67e+00 / 0.00e+00 | 20260506 12:39:59.993 | INFO | PerfReport: 22.59%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:39:59.994 | INFO | PerfReport: 0.02%, Misc -- 1.51e-02 / 0.00e+00, 2.65e-02 / 0.00e+00 | 20260506 12:40:00.126 | INFO | Starting a round of Baz IO | 20260506 12:40:03.295 | INFO | Populating RTMetrics report | 20260506 12:40:03.295 | INFO | RTMetrics report populated | 20260506 12:40:03.295 | INFO | Opening tmp RTMetrics json file | 20260506 12:40:03.296 | INFO | Serializing RTMetrics to json | 20260506 12:40:03.297 | INFO | Serialization done, swapping tmp file now... | 20260506 12:40:03.298 | INFO | RTMetrics done | 20260506 12:40:03.396 | INFO | Finished a round of Baz IO: Wrote 2.094 GiB in 3.269 seconds | 20260506 12:40:03.396 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.12% / 0.02% / 58.86% | 20260506 12:40:03.396 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.76823% / 4.37101e-05% / 4.37101e-05% | 20260506 12:40:03.396 | INFO | Baz IO overall: | 20260506 12:40:03.396 | INFO | BytesWritten : 711770975155 | 20260506 12:40:03.396 | INFO | NumEvents : 344046356384 | 20260506 12:40:05.131 | INFO | Analyzing chunk frames = [1425408,1425920) | 20260506 12:40:05.131 | INFO | MemInfo: { | 20260506 12:40:05.131 | INFO | "gpu" : " 17 GB", | 20260506 12:40:05.131 | INFO | "huge" : "", | 20260506 12:40:05.131 | INFO | "huge_shared" : " 27 GB", | 20260506 12:40:05.131 | INFO | "malloc" : " 14 GB", | 20260506 12:40:05.131 | INFO | "pinned" : " 29 GB" | 20260506 12:40:05.131 | INFO | } | 20260506 12:40:05.131 | INFO | Perf report for the last 5.13936 seconds: | 20260506 12:40:05.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:40:05.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:05.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:05.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:05.132 | INFO | ANALYSIS : 63.5%, 3.0, 98.9ms, 0 | 20260506 12:40:05.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:05.132 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:40:05.132 | INFO | PRE_HQ : 61.2%, 1.9, 61.1ms, 0 | 20260506 12:40:05.132 | INFO | BAZWRITER : 63.0%, 3.7, 118.6ms, 0 | 20260506 12:40:05.132 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260506 12:40:05.132 | INFO | Uploaded 36GiB to GPU in 2.04375s (17.6147GiB/s) | 20260506 12:40:05.132 | INFO | Downloaded 38GiB to GPU in 2.22651s (17.0671GiB/s) | 20260506 12:40:10.270 | INFO | Analyzing chunk frames = [1425920,1426432) | 20260506 12:40:10.270 | INFO | MemInfo: { | 20260506 12:40:10.270 | INFO | "gpu" : " 17 GB", | 20260506 12:40:10.270 | INFO | "huge" : "", | 20260506 12:40:10.270 | INFO | "huge_shared" : " 27 GB", | 20260506 12:40:10.271 | INFO | "malloc" : " 14 GB", | 20260506 12:40:10.271 | INFO | "pinned" : " 29 GB" | 20260506 12:40:10.271 | INFO | } | 20260506 12:40:10.271 | INFO | Perf report for the last 5.13909 seconds: | 20260506 12:40:10.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:40:10.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:10.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:10.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:10.271 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 12:40:10.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:10.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:10.271 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:40:10.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:10.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:10.271 | INFO | Uploaded 36GiB to GPU in 2.00516s (17.9536GiB/s) | 20260506 12:40:10.271 | INFO | Downloaded 34GiB to GPU in 1.94053s (17.521GiB/s) | 20260506 12:40:15.410 | INFO | Analyzing chunk frames = [1426432,1426944) | 20260506 12:40:15.410 | INFO | MemInfo: { | 20260506 12:40:15.410 | INFO | "gpu" : " 17 GB", | 20260506 12:40:15.410 | INFO | "huge" : "", | 20260506 12:40:15.410 | INFO | "huge_shared" : " 27 GB", | 20260506 12:40:15.410 | INFO | "malloc" : " 14 GB", | 20260506 12:40:15.410 | INFO | "pinned" : " 29 GB" | 20260506 12:40:15.410 | INFO | } | 20260506 12:40:15.411 | INFO | Perf report for the last 5.13993 seconds: | 20260506 12:40:15.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:40:15.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:15.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:15.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:15.411 | INFO | ANALYSIS : 61.7%, 3.0, 95.5ms, 0 | 20260506 12:40:15.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:15.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:15.411 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:40:15.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:15.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:15.411 | INFO | Uploaded 36GiB to GPU in 2.01232s (17.8898GiB/s) | 20260506 12:40:15.411 | INFO | Downloaded 34GiB to GPU in 1.94934s (17.4418GiB/s) | 20260506 12:40:20.550 | INFO | Analyzing chunk frames = [1426944,1427456) | 20260506 12:40:20.550 | INFO | MemInfo: { | 20260506 12:40:20.550 | INFO | "gpu" : " 17 GB", | 20260506 12:40:20.550 | INFO | "huge" : "", | 20260506 12:40:20.550 | INFO | "huge_shared" : " 27 GB", | 20260506 12:40:20.550 | INFO | "malloc" : " 14 GB", | 20260506 12:40:20.550 | INFO | "pinned" : " 29 GB" | 20260506 12:40:20.550 | INFO | } | 20260506 12:40:20.550 | INFO | Perf report for the last 5.13922 seconds: | 20260506 12:40:20.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:40:20.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:20.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:20.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:20.550 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 12:40:20.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:20.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:20.550 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:40:20.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:20.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:20.550 | INFO | Uploaded 36GiB to GPU in 1.96679s (18.3039GiB/s) | 20260506 12:40:20.550 | INFO | Downloaded 34GiB to GPU in 1.92571s (17.6558GiB/s) | 20260506 12:40:25.689 | INFO | Analyzing chunk frames = [1427456,1427968) | 20260506 12:40:25.689 | INFO | MemInfo: { | 20260506 12:40:25.689 | INFO | "gpu" : " 17 GB", | 20260506 12:40:25.689 | INFO | "huge" : "", | 20260506 12:40:25.689 | INFO | "huge_shared" : " 27 GB", | 20260506 12:40:25.689 | INFO | "malloc" : " 14 GB", | 20260506 12:40:25.689 | INFO | "pinned" : " 29 GB" | 20260506 12:40:25.689 | INFO | } | 20260506 12:40:25.689 | INFO | Perf report for the last 5.13931 seconds: | 20260506 12:40:25.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:40:25.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:25.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:25.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:25.689 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 12:40:25.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:25.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:25.689 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:40:25.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:25.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:25.689 | INFO | Uploaded 36GiB to GPU in 2.01179s (17.8945GiB/s) | 20260506 12:40:25.689 | INFO | Downloaded 34GiB to GPU in 1.9488s (17.4467GiB/s) | 20260506 12:40:30.828 | INFO | Analyzing chunk frames = [1427968,1428480) | 20260506 12:40:30.828 | INFO | MemInfo: { | 20260506 12:40:30.828 | INFO | "gpu" : " 17 GB", | 20260506 12:40:30.828 | INFO | "huge" : "", | 20260506 12:40:30.828 | INFO | "huge_shared" : " 27 GB", | 20260506 12:40:30.828 | INFO | "malloc" : " 14 GB", | 20260506 12:40:30.828 | INFO | "pinned" : " 29 GB" | 20260506 12:40:30.828 | INFO | } | 20260506 12:40:30.828 | INFO | Perf report for the last 5.13923 seconds: | 20260506 12:40:30.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:40:30.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:30.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:30.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:30.828 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 12:40:30.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:30.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:30.828 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 12:40:30.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:30.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:30.829 | INFO | Uploaded 36GiB to GPU in 2.01316s (17.8823GiB/s) | 20260506 12:40:30.829 | INFO | Downloaded 34GiB to GPU in 1.91802s (17.7266GiB/s) | 20260506 12:40:35.968 | INFO | Analyzing chunk frames = [1428480,1428992) | 20260506 12:40:35.968 | INFO | MemInfo: { | 20260506 12:40:35.968 | INFO | "gpu" : " 17 GB", | 20260506 12:40:35.968 | INFO | "huge" : "", | 20260506 12:40:35.969 | INFO | "huge_shared" : " 27 GB", | 20260506 12:40:35.969 | INFO | "malloc" : " 14 GB", | 20260506 12:40:35.969 | INFO | "pinned" : " 29 GB" | 20260506 12:40:35.969 | INFO | } | 20260506 12:40:35.969 | INFO | Perf report for the last 5.14025 seconds: | 20260506 12:40:35.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:40:35.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:35.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:35.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:35.969 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:40:35.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:35.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:35.969 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:40:35.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:35.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:35.969 | INFO | Uploaded 36GiB to GPU in 2.01865s (17.8337GiB/s) | 20260506 12:40:35.969 | INFO | Downloaded 34GiB to GPU in 2.01332s (16.8875GiB/s) | 20260506 12:40:41.107 | INFO | Analyzing chunk frames = [1428992,1429504) | 20260506 12:40:41.107 | INFO | MemInfo: { | 20260506 12:40:41.107 | INFO | "gpu" : " 17 GB", | 20260506 12:40:41.107 | INFO | "huge" : "", | 20260506 12:40:41.107 | INFO | "huge_shared" : " 27 GB", | 20260506 12:40:41.107 | INFO | "malloc" : " 14 GB", | 20260506 12:40:41.107 | INFO | "pinned" : " 29 GB" | 20260506 12:40:41.107 | INFO | } | 20260506 12:40:41.107 | INFO | Perf report for the last 5.13825 seconds: | 20260506 12:40:41.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:40:41.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:41.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:41.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:41.107 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260506 12:40:41.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:41.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:41.107 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:40:41.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:41.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:41.107 | INFO | Uploaded 36GiB to GPU in 2.04357s (17.6163GiB/s) | 20260506 12:40:41.107 | INFO | Downloaded 34GiB to GPU in 1.98939s (17.0907GiB/s) | 20260506 12:40:41.234 | INFO | Starting a round of Baz IO | 20260506 12:40:44.400 | INFO | Populating RTMetrics report | 20260506 12:40:44.400 | INFO | RTMetrics report populated | 20260506 12:40:44.400 | INFO | Opening tmp RTMetrics json file | 20260506 12:40:44.401 | INFO | Serializing RTMetrics to json | 20260506 12:40:44.402 | INFO | Serialization done, swapping tmp file now... | 20260506 12:40:44.402 | INFO | RTMetrics done | 20260506 12:40:44.498 | INFO | Finished a round of Baz IO: Wrote 2.096 GiB in 3.264 seconds | 20260506 12:40:44.499 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.10% / 0.02% / 58.88% | 20260506 12:40:44.499 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.76132% / 4.37101e-05% / 3.57628e-05% | 20260506 12:40:44.499 | INFO | Baz IO overall: | 20260506 12:40:44.499 | INFO | BytesWritten : 714021457275 | 20260506 12:40:44.499 | INFO | NumEvents : 345137681307 | 20260506 12:40:46.246 | INFO | Analyzing chunk frames = [1429504,1430016) | 20260506 12:40:46.246 | INFO | MemInfo: { | 20260506 12:40:46.246 | INFO | "gpu" : " 17 GB", | 20260506 12:40:46.246 | INFO | "huge" : "", | 20260506 12:40:46.246 | INFO | "huge_shared" : " 27 GB", | 20260506 12:40:46.246 | INFO | "malloc" : " 14 GB", | 20260506 12:40:46.246 | INFO | "pinned" : " 29 GB" | 20260506 12:40:46.246 | INFO | } | 20260506 12:40:46.246 | INFO | Perf report for the last 5.13939 seconds: | 20260506 12:40:46.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:40:46.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:46.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:46.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:46.246 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 12:40:46.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:46.246 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:40:46.246 | INFO | PRE_HQ : 61.6%, 1.9, 61.6ms, 0 | 20260506 12:40:46.246 | INFO | BAZWRITER : 62.7%, 3.7, 118.2ms, 0 | 20260506 12:40:46.246 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:40:46.247 | INFO | Uploaded 36GiB to GPU in 2.06229s (17.4563GiB/s) | 20260506 12:40:46.247 | INFO | Downloaded 38GiB to GPU in 2.2182s (17.131GiB/s) | 20260506 12:40:51.385 | INFO | Analyzing chunk frames = [1430016,1430528) | 20260506 12:40:51.385 | INFO | MemInfo: { | 20260506 12:40:51.385 | INFO | "gpu" : " 17 GB", | 20260506 12:40:51.385 | INFO | "huge" : "", | 20260506 12:40:51.385 | INFO | "huge_shared" : " 27 GB", | 20260506 12:40:51.385 | INFO | "malloc" : " 14 GB", | 20260506 12:40:51.385 | INFO | "pinned" : " 29 GB" | 20260506 12:40:51.385 | INFO | } | 20260506 12:40:51.386 | INFO | Perf report for the last 5.13932 seconds: | 20260506 12:40:51.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:40:51.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:51.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:51.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:51.386 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:40:51.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:51.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:51.386 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 12:40:51.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:51.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:51.386 | INFO | Uploaded 36GiB to GPU in 2.06245s (17.455GiB/s) | 20260506 12:40:51.386 | INFO | Downloaded 34GiB to GPU in 1.93118s (17.6058GiB/s) | 20260506 12:40:56.525 | INFO | Analyzing chunk frames = [1430528,1431040) | 20260506 12:40:56.525 | INFO | MemInfo: { | 20260506 12:40:56.525 | INFO | "gpu" : " 17 GB", | 20260506 12:40:56.525 | INFO | "huge" : "", | 20260506 12:40:56.525 | INFO | "huge_shared" : " 27 GB", | 20260506 12:40:56.525 | INFO | "malloc" : " 14 GB", | 20260506 12:40:56.525 | INFO | "pinned" : " 29 GB" | 20260506 12:40:56.525 | INFO | } | 20260506 12:40:56.525 | INFO | Perf report for the last 5.13973 seconds: | 20260506 12:40:56.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:40:56.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:56.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:56.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:56.525 | INFO | ANALYSIS : 61.9%, 3.0, 95.8ms, 0 | 20260506 12:40:56.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:56.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:56.526 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:40:56.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:56.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:40:56.526 | INFO | Uploaded 36GiB to GPU in 2.02583s (17.7705GiB/s) | 20260506 12:40:56.526 | INFO | Downloaded 34GiB to GPU in 1.94434s (17.4867GiB/s) | 20260506 12:41:01.664 | INFO | Analyzing chunk frames = [1431040,1431552) | 20260506 12:41:01.664 | INFO | MemInfo: { | 20260506 12:41:01.664 | INFO | "gpu" : " 17 GB", | 20260506 12:41:01.664 | INFO | "huge" : "", | 20260506 12:41:01.664 | INFO | "huge_shared" : " 27 GB", | 20260506 12:41:01.664 | INFO | "malloc" : " 14 GB", | 20260506 12:41:01.664 | INFO | "pinned" : " 29 GB" | 20260506 12:41:01.664 | INFO | } | 20260506 12:41:01.664 | INFO | Perf report for the last 5.13896 seconds: | 20260506 12:41:01.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:41:01.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:01.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:01.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:01.664 | INFO | ANALYSIS : 61.3%, 3.0, 95.1ms, 0 | 20260506 12:41:01.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:01.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:01.664 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:41:01.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:01.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:01.665 | INFO | Uploaded 36GiB to GPU in 2.02875s (17.7449GiB/s) | 20260506 12:41:01.665 | INFO | Downloaded 34GiB to GPU in 1.94868s (17.4477GiB/s) | 20260506 12:41:06.803 | INFO | Analyzing chunk frames = [1431552,1432064) | 20260506 12:41:06.803 | INFO | MemInfo: { | 20260506 12:41:06.803 | INFO | "gpu" : " 17 GB", | 20260506 12:41:06.803 | INFO | "huge" : "", | 20260506 12:41:06.803 | INFO | "huge_shared" : " 27 GB", | 20260506 12:41:06.803 | INFO | "malloc" : " 14 GB", | 20260506 12:41:06.804 | INFO | "pinned" : " 29 GB" | 20260506 12:41:06.804 | INFO | } | 20260506 12:41:06.804 | INFO | Perf report for the last 5.13933 seconds: | 20260506 12:41:06.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:41:06.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:06.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:06.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:06.804 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 12:41:06.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:06.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:06.804 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:41:06.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:06.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:06.804 | INFO | Uploaded 36GiB to GPU in 2.01521s (17.8642GiB/s) | 20260506 12:41:06.804 | INFO | Downloaded 34GiB to GPU in 1.94431s (17.4869GiB/s) | 20260506 12:41:11.943 | INFO | Analyzing chunk frames = [1432064,1432576) | 20260506 12:41:11.943 | INFO | MemInfo: { | 20260506 12:41:11.943 | INFO | "gpu" : " 17 GB", | 20260506 12:41:11.943 | INFO | "huge" : "", | 20260506 12:41:11.943 | INFO | "huge_shared" : " 27 GB", | 20260506 12:41:11.943 | INFO | "malloc" : " 14 GB", | 20260506 12:41:11.943 | INFO | "pinned" : " 29 GB" | 20260506 12:41:11.943 | INFO | } | 20260506 12:41:11.943 | INFO | Perf report for the last 5.13921 seconds: | 20260506 12:41:11.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:41:11.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:11.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:11.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:11.943 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 12:41:11.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:11.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:11.943 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 12:41:11.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:11.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:11.943 | INFO | Uploaded 36GiB to GPU in 2.0833s (17.2803GiB/s) | 20260506 12:41:11.943 | INFO | Downloaded 34GiB to GPU in 1.94084s (17.5182GiB/s) | 20260506 12:41:17.083 | INFO | Analyzing chunk frames = [1432576,1433088) | 20260506 12:41:17.083 | INFO | MemInfo: { | 20260506 12:41:17.083 | INFO | "gpu" : " 17 GB", | 20260506 12:41:17.083 | INFO | "huge" : "", | 20260506 12:41:17.083 | INFO | "huge_shared" : " 27 GB", | 20260506 12:41:17.083 | INFO | "malloc" : " 14 GB", | 20260506 12:41:17.083 | INFO | "pinned" : " 29 GB" | 20260506 12:41:17.083 | INFO | } | 20260506 12:41:17.083 | INFO | Perf report for the last 5.13999 seconds: | 20260506 12:41:17.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:41:17.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:17.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:17.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:17.083 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 12:41:17.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:17.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:17.083 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 12:41:17.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:17.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:17.083 | INFO | Uploaded 36GiB to GPU in 1.98687s (18.119GiB/s) | 20260506 12:41:17.083 | INFO | Downloaded 34GiB to GPU in 2.03442s (16.7124GiB/s) | 20260506 12:41:22.222 | INFO | Analyzing chunk frames = [1433088,1433600) | 20260506 12:41:22.222 | INFO | MemInfo: { | 20260506 12:41:22.222 | INFO | "gpu" : " 17 GB", | 20260506 12:41:22.222 | INFO | "huge" : "", | 20260506 12:41:22.222 | INFO | "huge_shared" : " 27 GB", | 20260506 12:41:22.222 | INFO | "malloc" : " 14 GB", | 20260506 12:41:22.222 | INFO | "pinned" : " 29 GB" | 20260506 12:41:22.222 | INFO | } | 20260506 12:41:22.222 | INFO | Perf report for the last 5.13889 seconds: | 20260506 12:41:22.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:41:22.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:22.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:22.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:22.222 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:41:22.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:22.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:22.222 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:41:22.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:22.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:22.222 | INFO | Uploaded 36GiB to GPU in 2.02566s (17.772GiB/s) | 20260506 12:41:22.222 | INFO | Downloaded 34GiB to GPU in 1.96549s (17.2984GiB/s) | 20260506 12:41:22.222 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:41:22.222 | INFO | Total mem usage: 16.8 GiB | 20260506 12:41:22.222 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:41:22.222 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.8%) | 20260506 12:41:22.222 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:41:22.222 | INFO | High Water Breakdown: | 20260506 12:41:22.222 | INFO | 59.6% : 8.59 GiB ( 82.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:41:22.222 | INFO | 40.2% : 5.79 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:41:22.222 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:41:22.222 | INFO | ------------------------------------------ | 20260506 12:41:22.222 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:41:22.222 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:41:22.222 | INFO | Total mem usage: 26.5 GiB | 20260506 12:41:22.222 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:41:22.222 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:41:22.222 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:41:22.222 | INFO | High Water Breakdown: | 20260506 12:41:22.222 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:41:22.222 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:41:22.222 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:41:22.222 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:41:22.222 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:41:22.222 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:41:22.222 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:41:22.222 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:41:22.222 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:41:22.222 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:41:22.222 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:41:22.223 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:41:22.223 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:41:22.223 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:41:22.223 | INFO | ------------------------------------------ | 20260506 12:41:22.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:41:22.223 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:41:22.223 | INFO | Total mem usage: 25.6 GiB | 20260506 12:41:22.223 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:41:22.223 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:41:22.223 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:41:22.223 | INFO | High Water Breakdown: | 20260506 12:41:22.223 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:41:22.223 | INFO | ------------------------------------------ | 20260506 12:41:22.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:41:22.223 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:41:22.223 | INFO | Total mem usage: 15.7 GiB | 20260506 12:41:22.223 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:41:22.223 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:41:22.223 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 12:41:22.223 | INFO | High Water Breakdown: | 20260506 12:41:22.223 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:41:22.223 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:41:22.223 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:41:22.223 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:41:22.223 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:41:22.223 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:41:22.223 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:41:22.223 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:41:22.223 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:41:22.223 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:41:22.223 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:41:22.223 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:41:22.223 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:41:22.223 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:41:22.223 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:41:22.223 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:41:22.223 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:41:22.223 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:41:22.223 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:41:22.223 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:41:22.223 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:41:22.223 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:41:22.223 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:41:22.223 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:41:22.223 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:41:22.223 | INFO | ------------------------------------------ | 20260506 12:41:22.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:41:22.223 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:41:22.223 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:41:22.223 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:41:22.223 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.64e+01, 1.33e+01 / 3.10e+00 | 20260506 12:41:22.223 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 9.99e-02, 1.12e-02 / 7.36e-03 | 20260506 12:41:22.223 | INFO | PerfReport: 11.56%, Baseline -- 3.67e+00 / 4.41e+00, 5.78e-01 / 3.29e-01 | 20260506 12:41:22.223 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.44e+00, 1.90e-01 / 1.59e-01 | 20260506 12:41:22.223 | INFO | PerfReport: 15.83%, DME -- 4.02e+01 / 4.48e+01, 4.24e+00 / 3.34e+00 | 20260506 12:41:22.223 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.49e+00 / 1.95e+00 | 20260506 12:41:22.223 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.39e+00 / 3.00e+00, 4.16e-01 / 2.72e-01 | 20260506 12:41:22.223 | INFO | PerfReport: 3.72%, Metrics -- 1.18e+00 / 1.58e+00, 3.00e-01 / 2.02e-01 | 20260506 12:41:22.223 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.16e-01 / 4.71e-01, 6.47e-02 / 3.28e-02 | 20260506 12:41:22.223 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:41:22.223 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:41:22.223 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:41:22.223 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 12:41:22.223 | INFO | PerfReport: 21.37%, Upload -- 2.11e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260506 12:41:22.223 | INFO | PerfReport: 21.00%, Download -- 2.07e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 12:41:22.223 | INFO | PerfReport: 32.16%, Compute -- 3.17e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 12:41:22.223 | INFO | PerfReport: 1.73%, LockUpload -- 1.71e+00 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 12:41:22.223 | INFO | PerfReport: 1.12%, LockDownload -- 1.10e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 12:41:22.223 | INFO | PerfReport: 22.61%, LockCompute -- 2.23e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 12:41:22.223 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260506 12:41:22.354 | INFO | Starting a round of Baz IO | 20260506 12:41:25.515 | INFO | Populating RTMetrics report | 20260506 12:41:25.516 | INFO | RTMetrics report populated | 20260506 12:41:25.516 | INFO | Opening tmp RTMetrics json file | 20260506 12:41:25.517 | INFO | Serializing RTMetrics to json | 20260506 12:41:25.518 | INFO | Serialization done, swapping tmp file now... | 20260506 12:41:25.518 | INFO | RTMetrics done | 20260506 12:41:25.612 | INFO | Finished a round of Baz IO: Wrote 2.097 GiB in 3.257 seconds | 20260506 12:41:25.612 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.07% / 0.02% / 58.90% | 20260506 12:41:25.612 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.75585% / 3.57628e-05% / 2.78155e-05% | 20260506 12:41:25.612 | INFO | Baz IO overall: | 20260506 12:41:25.612 | INFO | BytesWritten : 716273143285 | 20260506 12:41:25.612 | INFO | NumEvents : 346229826271 | 20260506 12:41:27.361 | INFO | Analyzing chunk frames = [1433600,1434112) | 20260506 12:41:27.361 | INFO | MemInfo: { | 20260506 12:41:27.361 | INFO | "gpu" : " 17 GB", | 20260506 12:41:27.361 | INFO | "huge" : "", | 20260506 12:41:27.361 | INFO | "huge_shared" : " 27 GB", | 20260506 12:41:27.361 | INFO | "malloc" : " 14 GB", | 20260506 12:41:27.361 | INFO | "pinned" : " 29 GB" | 20260506 12:41:27.361 | INFO | } | 20260506 12:41:27.361 | INFO | Perf report for the last 5.13956 seconds: | 20260506 12:41:27.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:41:27.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:27.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:27.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:27.361 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260506 12:41:27.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:27.362 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:41:27.362 | INFO | PRE_HQ : 61.3%, 1.9, 61.2ms, 0 | 20260506 12:41:27.362 | INFO | BAZWRITER : 62.6%, 3.7, 118.7ms, 0 | 20260506 12:41:27.362 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:41:27.362 | INFO | Uploaded 36GiB to GPU in 2.04322s (17.6192GiB/s) | 20260506 12:41:27.362 | INFO | Downloaded 38GiB to GPU in 2.22561s (17.074GiB/s) | 20260506 12:41:32.500 | INFO | Analyzing chunk frames = [1434112,1434624) | 20260506 12:41:32.500 | INFO | MemInfo: { | 20260506 12:41:32.500 | INFO | "gpu" : " 17 GB", | 20260506 12:41:32.500 | INFO | "huge" : "", | 20260506 12:41:32.500 | INFO | "huge_shared" : " 27 GB", | 20260506 12:41:32.500 | INFO | "malloc" : " 14 GB", | 20260506 12:41:32.500 | INFO | "pinned" : " 29 GB" | 20260506 12:41:32.500 | INFO | } | 20260506 12:41:32.500 | INFO | Perf report for the last 5.13908 seconds: | 20260506 12:41:32.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:41:32.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:32.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:32.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:32.500 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:41:32.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:32.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:32.500 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:41:32.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:32.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:32.501 | INFO | Uploaded 36GiB to GPU in 2.03833s (17.6615GiB/s) | 20260506 12:41:32.501 | INFO | Downloaded 34GiB to GPU in 1.93766s (17.5469GiB/s) | 20260506 12:41:37.640 | INFO | Analyzing chunk frames = [1434624,1435136) | 20260506 12:41:37.640 | INFO | MemInfo: { | 20260506 12:41:37.640 | INFO | "gpu" : " 17 GB", | 20260506 12:41:37.640 | INFO | "huge" : "", | 20260506 12:41:37.640 | INFO | "huge_shared" : " 27 GB", | 20260506 12:41:37.640 | INFO | "malloc" : " 14 GB", | 20260506 12:41:37.640 | INFO | "pinned" : " 29 GB" | 20260506 12:41:37.640 | INFO | } | 20260506 12:41:37.640 | INFO | Perf report for the last 5.13937 seconds: | 20260506 12:41:37.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:41:37.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:37.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:37.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:37.640 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 12:41:37.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:37.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:37.640 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:41:37.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:37.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:37.640 | INFO | Uploaded 36GiB to GPU in 2.02084s (17.8144GiB/s) | 20260506 12:41:37.640 | INFO | Downloaded 34GiB to GPU in 1.94685s (17.4641GiB/s) | 20260506 12:41:42.779 | INFO | Analyzing chunk frames = [1435136,1435648) | 20260506 12:41:42.779 | INFO | MemInfo: { | 20260506 12:41:42.779 | INFO | "gpu" : " 17 GB", | 20260506 12:41:42.779 | INFO | "huge" : "", | 20260506 12:41:42.779 | INFO | "huge_shared" : " 27 GB", | 20260506 12:41:42.779 | INFO | "malloc" : " 14 GB", | 20260506 12:41:42.779 | INFO | "pinned" : " 29 GB" | 20260506 12:41:42.779 | INFO | } | 20260506 12:41:42.779 | INFO | Perf report for the last 5.13932 seconds: | 20260506 12:41:42.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:41:42.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:42.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:42.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:42.779 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 12:41:42.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:42.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:42.779 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:41:42.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:42.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:42.779 | INFO | Uploaded 36GiB to GPU in 1.99404s (18.0538GiB/s) | 20260506 12:41:42.779 | INFO | Downloaded 34GiB to GPU in 1.93845s (17.5398GiB/s) | 20260506 12:41:47.918 | INFO | Analyzing chunk frames = [1435648,1436160) | 20260506 12:41:47.918 | INFO | MemInfo: { | 20260506 12:41:47.918 | INFO | "gpu" : " 17 GB", | 20260506 12:41:47.918 | INFO | "huge" : "", | 20260506 12:41:47.918 | INFO | "huge_shared" : " 27 GB", | 20260506 12:41:47.918 | INFO | "malloc" : " 14 GB", | 20260506 12:41:47.918 | INFO | "pinned" : " 29 GB" | 20260506 12:41:47.918 | INFO | } | 20260506 12:41:47.918 | INFO | Perf report for the last 5.13923 seconds: | 20260506 12:41:47.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:41:47.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:47.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:47.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:47.918 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 12:41:47.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:47.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:47.918 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:41:47.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:47.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:47.919 | INFO | Uploaded 36GiB to GPU in 2.02886s (17.7439GiB/s) | 20260506 12:41:47.919 | INFO | Downloaded 34GiB to GPU in 1.94996s (17.4363GiB/s) | 20260506 12:41:53.057 | INFO | Analyzing chunk frames = [1436160,1436672) | 20260506 12:41:53.057 | INFO | MemInfo: { | 20260506 12:41:53.057 | INFO | "gpu" : " 17 GB", | 20260506 12:41:53.057 | INFO | "huge" : "", | 20260506 12:41:53.057 | INFO | "huge_shared" : " 27 GB", | 20260506 12:41:53.057 | INFO | "malloc" : " 14 GB", | 20260506 12:41:53.057 | INFO | "pinned" : " 29 GB" | 20260506 12:41:53.057 | INFO | } | 20260506 12:41:53.058 | INFO | Perf report for the last 5.13917 seconds: | 20260506 12:41:53.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:41:53.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:53.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:53.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:53.058 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 12:41:53.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:53.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:53.058 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:41:53.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:53.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:53.058 | INFO | Uploaded 36GiB to GPU in 2.09179s (17.2102GiB/s) | 20260506 12:41:53.058 | INFO | Downloaded 34GiB to GPU in 1.98734s (17.1083GiB/s) | 20260506 12:41:58.197 | INFO | Analyzing chunk frames = [1436672,1437184) | 20260506 12:41:58.197 | INFO | MemInfo: { | 20260506 12:41:58.197 | INFO | "gpu" : " 17 GB", | 20260506 12:41:58.197 | INFO | "huge" : "", | 20260506 12:41:58.197 | INFO | "huge_shared" : " 27 GB", | 20260506 12:41:58.197 | INFO | "malloc" : " 14 GB", | 20260506 12:41:58.197 | INFO | "pinned" : " 29 GB" | 20260506 12:41:58.197 | INFO | } | 20260506 12:41:58.197 | INFO | Perf report for the last 5.13979 seconds: | 20260506 12:41:58.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:41:58.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:58.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:58.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:58.197 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:41:58.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:58.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:58.197 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 12:41:58.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:58.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:41:58.198 | INFO | Uploaded 36GiB to GPU in 1.9737s (18.2398GiB/s) | 20260506 12:41:58.198 | INFO | Downloaded 34GiB to GPU in 2.03578s (16.7012GiB/s) | 20260506 12:42:03.336 | INFO | Analyzing chunk frames = [1437184,1437696) | 20260506 12:42:03.336 | INFO | MemInfo: { | 20260506 12:42:03.336 | INFO | "gpu" : " 17 GB", | 20260506 12:42:03.336 | INFO | "huge" : "", | 20260506 12:42:03.336 | INFO | "huge_shared" : " 27 GB", | 20260506 12:42:03.336 | INFO | "malloc" : " 14 GB", | 20260506 12:42:03.336 | INFO | "pinned" : " 29 GB" | 20260506 12:42:03.336 | INFO | } | 20260506 12:42:03.336 | INFO | Perf report for the last 5.13887 seconds: | 20260506 12:42:03.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:42:03.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:03.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:03.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:03.336 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:42:03.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:03.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:03.336 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 12:42:03.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:03.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:03.337 | INFO | Uploaded 36GiB to GPU in 2.03852s (17.6599GiB/s) | 20260506 12:42:03.337 | INFO | Downloaded 34GiB to GPU in 1.9641s (17.3108GiB/s) | 20260506 12:42:03.467 | INFO | Starting a round of Baz IO | 20260506 12:42:06.630 | INFO | Populating RTMetrics report | 20260506 12:42:06.630 | INFO | RTMetrics report populated | 20260506 12:42:06.630 | INFO | Opening tmp RTMetrics json file | 20260506 12:42:06.631 | INFO | Serializing RTMetrics to json | 20260506 12:42:06.632 | INFO | Serialization done, swapping tmp file now... | 20260506 12:42:06.632 | INFO | RTMetrics done | 20260506 12:42:06.731 | INFO | Finished a round of Baz IO: Wrote 2.097 GiB in 3.263 seconds | 20260506 12:42:06.731 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.05% / 0.02% / 58.93% | 20260506 12:42:06.731 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.74766% / 3.57628e-05% / 4.76837e-05% | 20260506 12:42:06.731 | INFO | Baz IO overall: | 20260506 12:42:06.731 | INFO | BytesWritten : 718525309925 | 20260506 12:42:06.731 | INFO | NumEvents : 347322318420 | 20260506 12:42:08.475 | INFO | Analyzing chunk frames = [1437696,1438208) | 20260506 12:42:08.475 | INFO | MemInfo: { | 20260506 12:42:08.475 | INFO | "gpu" : " 17 GB", | 20260506 12:42:08.475 | INFO | "huge" : "", | 20260506 12:42:08.475 | INFO | "huge_shared" : " 27 GB", | 20260506 12:42:08.476 | INFO | "malloc" : " 14 GB", | 20260506 12:42:08.476 | INFO | "pinned" : " 29 GB" | 20260506 12:42:08.476 | INFO | } | 20260506 12:42:08.476 | INFO | Perf report for the last 5.13934 seconds: | 20260506 12:42:08.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:42:08.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:08.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:08.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:08.476 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 12:42:08.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:08.476 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:42:08.476 | INFO | PRE_HQ : 61.3%, 1.9, 61.4ms, 0 | 20260506 12:42:08.476 | INFO | BAZWRITER : 62.7%, 3.7, 120.5ms, 0 | 20260506 12:42:08.476 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:42:08.476 | INFO | Uploaded 36GiB to GPU in 2.07732s (17.33GiB/s) | 20260506 12:42:08.476 | INFO | Downloaded 38GiB to GPU in 2.22043s (17.1138GiB/s) | 20260506 12:42:13.615 | INFO | Analyzing chunk frames = [1438208,1438720) | 20260506 12:42:13.615 | INFO | MemInfo: { | 20260506 12:42:13.615 | INFO | "gpu" : " 17 GB", | 20260506 12:42:13.615 | INFO | "huge" : "", | 20260506 12:42:13.615 | INFO | "huge_shared" : " 27 GB", | 20260506 12:42:13.615 | INFO | "malloc" : " 14 GB", | 20260506 12:42:13.615 | INFO | "pinned" : " 29 GB" | 20260506 12:42:13.615 | INFO | } | 20260506 12:42:13.615 | INFO | Perf report for the last 5.13964 seconds: | 20260506 12:42:13.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:42:13.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:13.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:13.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:13.615 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:42:13.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:13.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:13.615 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:42:13.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:13.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:13.615 | INFO | Uploaded 36GiB to GPU in 2.01091s (17.9023GiB/s) | 20260506 12:42:13.616 | INFO | Downloaded 34GiB to GPU in 1.92868s (17.6287GiB/s) | 20260506 12:42:18.754 | INFO | Analyzing chunk frames = [1438720,1439232) | 20260506 12:42:18.754 | INFO | MemInfo: { | 20260506 12:42:18.754 | INFO | "gpu" : " 17 GB", | 20260506 12:42:18.754 | INFO | "huge" : "", | 20260506 12:42:18.754 | INFO | "huge_shared" : " 27 GB", | 20260506 12:42:18.754 | INFO | "malloc" : " 14 GB", | 20260506 12:42:18.754 | INFO | "pinned" : " 29 GB" | 20260506 12:42:18.754 | INFO | } | 20260506 12:42:18.754 | INFO | Perf report for the last 5.13925 seconds: | 20260506 12:42:18.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:42:18.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:18.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:18.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:18.755 | INFO | ANALYSIS : 61.5%, 3.0, 95.3ms, 0 | 20260506 12:42:18.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:18.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:18.755 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:42:18.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:18.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:18.755 | INFO | Uploaded 36GiB to GPU in 1.98363s (18.1485GiB/s) | 20260506 12:42:18.755 | INFO | Downloaded 34GiB to GPU in 1.93282s (17.5909GiB/s) | 20260506 12:42:23.894 | INFO | Analyzing chunk frames = [1439232,1439744) | 20260506 12:42:23.894 | INFO | MemInfo: { | 20260506 12:42:23.894 | INFO | "gpu" : " 17 GB", | 20260506 12:42:23.894 | INFO | "huge" : "", | 20260506 12:42:23.894 | INFO | "huge_shared" : " 27 GB", | 20260506 12:42:23.894 | INFO | "malloc" : " 14 GB", | 20260506 12:42:23.894 | INFO | "pinned" : " 29 GB" | 20260506 12:42:23.894 | INFO | } | 20260506 12:42:23.894 | INFO | Perf report for the last 5.13935 seconds: | 20260506 12:42:23.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:42:23.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:23.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:23.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:23.894 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 12:42:23.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:23.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:23.894 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:42:23.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:23.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:23.894 | INFO | Uploaded 36GiB to GPU in 1.97671s (18.2121GiB/s) | 20260506 12:42:23.894 | INFO | Downloaded 34GiB to GPU in 1.94271s (17.5013GiB/s) | 20260506 12:42:29.033 | INFO | Analyzing chunk frames = [1439744,1440256) | 20260506 12:42:29.033 | INFO | MemInfo: { | 20260506 12:42:29.033 | INFO | "gpu" : " 17 GB", | 20260506 12:42:29.033 | INFO | "huge" : "", | 20260506 12:42:29.033 | INFO | "huge_shared" : " 27 GB", | 20260506 12:42:29.033 | INFO | "malloc" : " 14 GB", | 20260506 12:42:29.033 | INFO | "pinned" : " 29 GB" | 20260506 12:42:29.033 | INFO | } | 20260506 12:42:29.033 | INFO | Perf report for the last 5.13954 seconds: | 20260506 12:42:29.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:42:29.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:29.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:29.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:29.033 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 12:42:29.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:29.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:29.034 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 12:42:29.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:29.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:29.034 | INFO | Uploaded 36GiB to GPU in 2.04231s (17.6271GiB/s) | 20260506 12:42:29.034 | INFO | Downloaded 34GiB to GPU in 1.9596s (17.3505GiB/s) | 20260506 12:42:34.172 | INFO | Analyzing chunk frames = [1440256,1440768) | 20260506 12:42:34.172 | INFO | MemInfo: { | 20260506 12:42:34.172 | INFO | "gpu" : " 17 GB", | 20260506 12:42:34.172 | INFO | "huge" : "", | 20260506 12:42:34.172 | INFO | "huge_shared" : " 27 GB", | 20260506 12:42:34.172 | INFO | "malloc" : " 14 GB", | 20260506 12:42:34.172 | INFO | "pinned" : " 29 GB" | 20260506 12:42:34.172 | INFO | } | 20260506 12:42:34.172 | INFO | Perf report for the last 5.13897 seconds: | 20260506 12:42:34.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:42:34.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:34.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:34.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:34.172 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 12:42:34.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:34.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:34.173 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260506 12:42:34.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:34.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:34.173 | INFO | Uploaded 36GiB to GPU in 2.07081s (17.3845GiB/s) | 20260506 12:42:34.173 | INFO | Downloaded 34GiB to GPU in 1.93841s (17.5401GiB/s) | 20260506 12:42:39.312 | INFO | Analyzing chunk frames = [1440768,1441280) | 20260506 12:42:39.312 | INFO | MemInfo: { | 20260506 12:42:39.312 | INFO | "gpu" : " 17 GB", | 20260506 12:42:39.312 | INFO | "huge" : "", | 20260506 12:42:39.312 | INFO | "huge_shared" : " 27 GB", | 20260506 12:42:39.312 | INFO | "malloc" : " 14 GB", | 20260506 12:42:39.312 | INFO | "pinned" : " 29 GB" | 20260506 12:42:39.312 | INFO | } | 20260506 12:42:39.312 | INFO | Perf report for the last 5.13987 seconds: | 20260506 12:42:39.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:42:39.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:39.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:39.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:39.312 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 12:42:39.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:39.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:39.312 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:42:39.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:39.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:39.313 | INFO | Uploaded 36GiB to GPU in 1.97663s (18.2128GiB/s) | 20260506 12:42:39.313 | INFO | Downloaded 34GiB to GPU in 2.01775s (16.8505GiB/s) | 20260506 12:42:44.451 | INFO | Analyzing chunk frames = [1441280,1441792) | 20260506 12:42:44.451 | INFO | MemInfo: { | 20260506 12:42:44.451 | INFO | "gpu" : " 17 GB", | 20260506 12:42:44.451 | INFO | "huge" : "", | 20260506 12:42:44.451 | INFO | "huge_shared" : " 27 GB", | 20260506 12:42:44.451 | INFO | "malloc" : " 14 GB", | 20260506 12:42:44.451 | INFO | "pinned" : " 29 GB" | 20260506 12:42:44.451 | INFO | } | 20260506 12:42:44.452 | INFO | Perf report for the last 5.13924 seconds: | 20260506 12:42:44.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:42:44.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:44.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:44.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:44.452 | INFO | ANALYSIS : 62.8%, 3.0, 98.0ms, 0 | 20260506 12:42:44.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:44.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:44.452 | INFO | PRE_HQ : 29.2%, 1.0, 15.3ms, 0 | 20260506 12:42:44.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:44.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:44.452 | INFO | Uploaded 36GiB to GPU in 2.03982s (17.6486GiB/s) | 20260506 12:42:44.452 | INFO | Downloaded 34GiB to GPU in 2.00846s (16.9284GiB/s) | 20260506 12:42:44.452 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:42:44.452 | INFO | Total mem usage: 16.8 GiB | 20260506 12:42:44.452 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:42:44.452 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.8%) | 20260506 12:42:44.452 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:42:44.452 | INFO | High Water Breakdown: | 20260506 12:42:44.452 | INFO | 59.7% : 8.59 GiB ( 82.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:42:44.452 | INFO | 40.2% : 5.79 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:42:44.452 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:42:44.452 | INFO | ------------------------------------------ | 20260506 12:42:44.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:42:44.452 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:42:44.452 | INFO | Total mem usage: 26.5 GiB | 20260506 12:42:44.452 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:42:44.452 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:42:44.452 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:42:44.452 | INFO | High Water Breakdown: | 20260506 12:42:44.452 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:42:44.452 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:42:44.452 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:42:44.452 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:42:44.452 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:42:44.452 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:42:44.452 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:42:44.452 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:42:44.452 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:42:44.452 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:42:44.452 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:42:44.452 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:42:44.452 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:42:44.452 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:42:44.452 | INFO | ------------------------------------------ | 20260506 12:42:44.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:42:44.452 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:42:44.452 | INFO | Total mem usage: 25.6 GiB | 20260506 12:42:44.452 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:42:44.452 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:42:44.452 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:42:44.452 | INFO | High Water Breakdown: | 20260506 12:42:44.452 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:42:44.452 | INFO | ------------------------------------------ | 20260506 12:42:44.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:42:44.452 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:42:44.452 | INFO | Total mem usage: 15.7 GiB | 20260506 12:42:44.452 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:42:44.452 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:42:44.452 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:42:44.452 | INFO | High Water Breakdown: | 20260506 12:42:44.452 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:42:44.452 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:42:44.452 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:42:44.452 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:42:44.452 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:42:44.452 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:42:44.452 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:42:44.452 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:42:44.452 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:42:44.452 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:42:44.452 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:42:44.453 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:42:44.453 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:42:44.453 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:42:44.453 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:42:44.453 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:42:44.453 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:42:44.453 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:42:44.453 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:42:44.453 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:42:44.453 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:42:44.453 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:42:44.453 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:42:44.453 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:42:44.453 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:42:44.453 | INFO | ------------------------------------------ | 20260506 12:42:44.453 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:42:44.453 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:42:44.453 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:42:44.453 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:42:44.453 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.58e+01, 1.36e+01 / 2.41e+00 | 20260506 12:42:44.453 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.00e-01, 1.11e-02 / 6.30e-03 | 20260506 12:42:44.453 | INFO | PerfReport: 11.56%, Baseline -- 3.67e+00 / 4.40e+00, 5.84e-01 / 3.88e-01 | 20260506 12:42:44.453 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.45e+00, 1.94e-01 / 1.50e-01 | 20260506 12:42:44.453 | INFO | PerfReport: 15.77%, DME -- 4.00e+01 / 4.41e+01, 4.02e+00 / 2.85e+00 | 20260506 12:42:44.453 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.47e+00 / 2.03e+00 | 20260506 12:42:44.453 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.40e+00 / 2.97e+00, 4.21e-01 / 2.84e-01 | 20260506 12:42:44.453 | INFO | PerfReport: 3.73%, Metrics -- 1.18e+00 / 1.56e+00, 3.04e-01 / 1.88e-01 | 20260506 12:42:44.453 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.17e-01 / 4.70e-01, 6.49e-02 / 3.08e-02 | 20260506 12:42:44.453 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:42:44.453 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:42:44.453 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:42:44.453 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 12:42:44.453 | INFO | PerfReport: 21.39%, Upload -- 2.11e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260506 12:42:44.453 | INFO | PerfReport: 21.08%, Download -- 2.08e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 12:42:44.453 | INFO | PerfReport: 32.17%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 12:42:44.453 | INFO | PerfReport: 1.67%, LockUpload -- 1.64e+00 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260506 12:42:44.453 | INFO | PerfReport: 1.13%, LockDownload -- 1.12e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 12:42:44.453 | INFO | PerfReport: 22.55%, LockCompute -- 2.22e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 12:42:44.453 | INFO | PerfReport: 0.02%, Misc -- 1.55e-02 / 0.00e+00, 2.56e-02 / 0.00e+00 | 20260506 12:42:44.578 | INFO | Starting a round of Baz IO | 20260506 12:42:47.745 | INFO | Populating RTMetrics report | 20260506 12:42:47.745 | INFO | RTMetrics report populated | 20260506 12:42:47.745 | INFO | Opening tmp RTMetrics json file | 20260506 12:42:47.746 | INFO | Serializing RTMetrics to json | 20260506 12:42:47.747 | INFO | Serialization done, swapping tmp file now... | 20260506 12:42:47.747 | INFO | RTMetrics done | 20260506 12:42:47.844 | INFO | Finished a round of Baz IO: Wrote 2.098 GiB in 3.265 seconds | 20260506 12:42:47.844 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.03% / 0.02% / 58.95% | 20260506 12:42:47.844 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.73764% / 3.17891e-05% / 3.57628e-05% | 20260506 12:42:47.844 | INFO | Baz IO overall: | 20260506 12:42:47.844 | INFO | BytesWritten : 720777748184 | 20260506 12:42:47.844 | INFO | NumEvents : 348414874140 | 20260506 12:42:49.590 | INFO | Analyzing chunk frames = [1441792,1442304) | 20260506 12:42:49.590 | INFO | MemInfo: { | 20260506 12:42:49.590 | INFO | "gpu" : " 17 GB", | 20260506 12:42:49.590 | INFO | "huge" : "", | 20260506 12:42:49.590 | INFO | "huge_shared" : " 27 GB", | 20260506 12:42:49.590 | INFO | "malloc" : " 14 GB", | 20260506 12:42:49.590 | INFO | "pinned" : " 29 GB" | 20260506 12:42:49.590 | INFO | } | 20260506 12:42:49.591 | INFO | Perf report for the last 5.13897 seconds: | 20260506 12:42:49.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:42:49.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:49.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:49.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:49.591 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 12:42:49.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:49.591 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:42:49.591 | INFO | PRE_HQ : 61.3%, 1.9, 61.2ms, 0 | 20260506 12:42:49.591 | INFO | BAZWRITER : 62.9%, 3.7, 119.7ms, 0 | 20260506 12:42:49.591 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:42:49.591 | INFO | Uploaded 36GiB to GPU in 2.03601s (17.6817GiB/s) | 20260506 12:42:49.591 | INFO | Downloaded 38GiB to GPU in 2.22447s (17.0827GiB/s) | 20260506 12:42:54.730 | INFO | Analyzing chunk frames = [1442304,1442816) | 20260506 12:42:54.730 | INFO | MemInfo: { | 20260506 12:42:54.730 | INFO | "gpu" : " 17 GB", | 20260506 12:42:54.730 | INFO | "huge" : "", | 20260506 12:42:54.730 | INFO | "huge_shared" : " 27 GB", | 20260506 12:42:54.730 | INFO | "malloc" : " 14 GB", | 20260506 12:42:54.730 | INFO | "pinned" : " 29 GB" | 20260506 12:42:54.730 | INFO | } | 20260506 12:42:54.730 | INFO | Perf report for the last 5.13957 seconds: | 20260506 12:42:54.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:42:54.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:54.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:54.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:54.730 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 12:42:54.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:54.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:54.730 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:42:54.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:54.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:54.730 | INFO | Uploaded 36GiB to GPU in 2.02047s (17.8177GiB/s) | 20260506 12:42:54.730 | INFO | Downloaded 34GiB to GPU in 1.93174s (17.6007GiB/s) | 20260506 12:42:59.869 | INFO | Analyzing chunk frames = [1442816,1443328) | 20260506 12:42:59.869 | INFO | MemInfo: { | 20260506 12:42:59.869 | INFO | "gpu" : " 17 GB", | 20260506 12:42:59.869 | INFO | "huge" : "", | 20260506 12:42:59.869 | INFO | "huge_shared" : " 27 GB", | 20260506 12:42:59.869 | INFO | "malloc" : " 14 GB", | 20260506 12:42:59.869 | INFO | "pinned" : " 29 GB" | 20260506 12:42:59.869 | INFO | } | 20260506 12:42:59.869 | INFO | Perf report for the last 5.13925 seconds: | 20260506 12:42:59.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:42:59.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:59.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:59.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:59.869 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 12:42:59.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:59.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:59.869 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:42:59.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:59.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:42:59.870 | INFO | Uploaded 36GiB to GPU in 2.01736s (17.8451GiB/s) | 20260506 12:42:59.870 | INFO | Downloaded 34GiB to GPU in 1.94618s (17.4701GiB/s) | 20260506 12:43:05.008 | INFO | Analyzing chunk frames = [1443328,1443840) | 20260506 12:43:05.008 | INFO | MemInfo: { | 20260506 12:43:05.008 | INFO | "gpu" : " 17 GB", | 20260506 12:43:05.008 | INFO | "huge" : "", | 20260506 12:43:05.009 | INFO | "huge_shared" : " 27 GB", | 20260506 12:43:05.009 | INFO | "malloc" : " 14 GB", | 20260506 12:43:05.009 | INFO | "pinned" : " 29 GB" | 20260506 12:43:05.009 | INFO | } | 20260506 12:43:05.009 | INFO | Perf report for the last 5.13921 seconds: | 20260506 12:43:05.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:43:05.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:05.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:05.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:05.009 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 12:43:05.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:05.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:05.009 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 12:43:05.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:05.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:05.009 | INFO | Uploaded 36GiB to GPU in 2.00537s (17.9518GiB/s) | 20260506 12:43:05.009 | INFO | Downloaded 34GiB to GPU in 1.94266s (17.5017GiB/s) | 20260506 12:43:10.148 | INFO | Analyzing chunk frames = [1443840,1444352) | 20260506 12:43:10.148 | INFO | MemInfo: { | 20260506 12:43:10.148 | INFO | "gpu" : " 17 GB", | 20260506 12:43:10.148 | INFO | "huge" : "", | 20260506 12:43:10.148 | INFO | "huge_shared" : " 27 GB", | 20260506 12:43:10.148 | INFO | "malloc" : " 14 GB", | 20260506 12:43:10.148 | INFO | "pinned" : " 29 GB" | 20260506 12:43:10.148 | INFO | } | 20260506 12:43:10.148 | INFO | Perf report for the last 5.13943 seconds: | 20260506 12:43:10.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:43:10.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:10.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:10.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:10.148 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 12:43:10.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:10.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:10.148 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:43:10.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:10.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:10.148 | INFO | Uploaded 36GiB to GPU in 2.03807s (17.6637GiB/s) | 20260506 12:43:10.148 | INFO | Downloaded 34GiB to GPU in 1.95664s (17.3768GiB/s) | 20260506 12:43:15.287 | INFO | Analyzing chunk frames = [1444352,1444864) | 20260506 12:43:15.287 | INFO | MemInfo: { | 20260506 12:43:15.287 | INFO | "gpu" : " 17 GB", | 20260506 12:43:15.287 | INFO | "huge" : "", | 20260506 12:43:15.287 | INFO | "huge_shared" : " 27 GB", | 20260506 12:43:15.287 | INFO | "malloc" : " 14 GB", | 20260506 12:43:15.287 | INFO | "pinned" : " 29 GB" | 20260506 12:43:15.287 | INFO | } | 20260506 12:43:15.287 | INFO | Perf report for the last 5.13928 seconds: | 20260506 12:43:15.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:43:15.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:15.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:15.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:15.287 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:43:15.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:15.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:15.287 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 12:43:15.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:15.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:15.288 | INFO | Uploaded 36GiB to GPU in 2.10495s (17.1026GiB/s) | 20260506 12:43:15.288 | INFO | Downloaded 34GiB to GPU in 1.99047s (17.0814GiB/s) | 20260506 12:43:20.427 | INFO | Analyzing chunk frames = [1444864,1445376) | 20260506 12:43:20.427 | INFO | MemInfo: { | 20260506 12:43:20.427 | INFO | "gpu" : " 17 GB", | 20260506 12:43:20.427 | INFO | "huge" : "", | 20260506 12:43:20.427 | INFO | "huge_shared" : " 27 GB", | 20260506 12:43:20.427 | INFO | "malloc" : " 14 GB", | 20260506 12:43:20.427 | INFO | "pinned" : " 29 GB" | 20260506 12:43:20.427 | INFO | } | 20260506 12:43:20.427 | INFO | Perf report for the last 5.14008 seconds: | 20260506 12:43:20.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:43:20.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:20.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:20.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:20.427 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:43:20.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:20.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:20.427 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:43:20.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:20.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:20.428 | INFO | Uploaded 36GiB to GPU in 1.99844s (18.0141GiB/s) | 20260506 12:43:20.428 | INFO | Downloaded 34GiB to GPU in 1.99001s (17.0854GiB/s) | 20260506 12:43:25.566 | INFO | Analyzing chunk frames = [1445376,1445888) | 20260506 12:43:25.566 | INFO | MemInfo: { | 20260506 12:43:25.566 | INFO | "gpu" : " 17 GB", | 20260506 12:43:25.566 | INFO | "huge" : "", | 20260506 12:43:25.566 | INFO | "huge_shared" : " 27 GB", | 20260506 12:43:25.566 | INFO | "malloc" : " 14 GB", | 20260506 12:43:25.566 | INFO | "pinned" : " 29 GB" | 20260506 12:43:25.566 | INFO | } | 20260506 12:43:25.566 | INFO | Perf report for the last 5.13845 seconds: | 20260506 12:43:25.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:43:25.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:25.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:25.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:25.566 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:43:25.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:25.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:25.566 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:43:25.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:25.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:25.566 | INFO | Uploaded 36GiB to GPU in 2.03079s (17.7271GiB/s) | 20260506 12:43:25.566 | INFO | Downloaded 34GiB to GPU in 1.98116s (17.1617GiB/s) | 20260506 12:43:25.699 | INFO | Starting a round of Baz IO | 20260506 12:43:28.864 | INFO | Populating RTMetrics report | 20260506 12:43:28.864 | INFO | RTMetrics report populated | 20260506 12:43:28.864 | INFO | Opening tmp RTMetrics json file | 20260506 12:43:28.865 | INFO | Serializing RTMetrics to json | 20260506 12:43:28.866 | INFO | Serialization done, swapping tmp file now... | 20260506 12:43:28.867 | INFO | RTMetrics done | 20260506 12:43:28.964 | INFO | Finished a round of Baz IO: Wrote 2.098 GiB in 3.265 seconds | 20260506 12:43:28.964 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 41.01% / 0.02% / 58.97% | 20260506 12:43:28.964 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.72773% / 4.37101e-05% / 3.97364e-05% | 20260506 12:43:28.964 | INFO | Baz IO overall: | 20260506 12:43:28.964 | INFO | BytesWritten : 723030602897 | 20260506 12:43:28.964 | INFO | NumEvents : 349507539398 | 20260506 12:43:30.705 | INFO | Analyzing chunk frames = [1445888,1446400) | 20260506 12:43:30.705 | INFO | MemInfo: { | 20260506 12:43:30.705 | INFO | "gpu" : " 17 GB", | 20260506 12:43:30.705 | INFO | "huge" : "", | 20260506 12:43:30.705 | INFO | "huge_shared" : " 27 GB", | 20260506 12:43:30.705 | INFO | "malloc" : " 14 GB", | 20260506 12:43:30.705 | INFO | "pinned" : " 29 GB" | 20260506 12:43:30.705 | INFO | } | 20260506 12:43:30.705 | INFO | Perf report for the last 5.13947 seconds: | 20260506 12:43:30.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:43:30.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:30.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:30.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:30.705 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 12:43:30.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:30.705 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:43:30.705 | INFO | PRE_HQ : 61.7%, 1.9, 61.6ms, 0 | 20260506 12:43:30.705 | INFO | BAZWRITER : 62.8%, 3.7, 119.8ms, 0 | 20260506 12:43:30.705 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:43:30.706 | INFO | Uploaded 36GiB to GPU in 2.06617s (17.4236GiB/s) | 20260506 12:43:30.706 | INFO | Downloaded 38GiB to GPU in 2.2173s (17.138GiB/s) | 20260506 12:43:35.844 | INFO | Analyzing chunk frames = [1446400,1446912) | 20260506 12:43:35.844 | INFO | MemInfo: { | 20260506 12:43:35.844 | INFO | "gpu" : " 17 GB", | 20260506 12:43:35.844 | INFO | "huge" : "", | 20260506 12:43:35.844 | INFO | "huge_shared" : " 27 GB", | 20260506 12:43:35.844 | INFO | "malloc" : " 14 GB", | 20260506 12:43:35.844 | INFO | "pinned" : " 29 GB" | 20260506 12:43:35.844 | INFO | } | 20260506 12:43:35.844 | INFO | Perf report for the last 5.13906 seconds: | 20260506 12:43:35.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:43:35.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:35.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:35.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:35.844 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:43:35.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:35.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:35.844 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:43:35.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:35.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:35.845 | INFO | Uploaded 36GiB to GPU in 2.06733s (17.4138GiB/s) | 20260506 12:43:35.845 | INFO | Downloaded 34GiB to GPU in 1.94615s (17.4704GiB/s) | 20260506 12:43:40.984 | INFO | Analyzing chunk frames = [1446912,1447424) | 20260506 12:43:40.984 | INFO | MemInfo: { | 20260506 12:43:40.984 | INFO | "gpu" : " 17 GB", | 20260506 12:43:40.984 | INFO | "huge" : "", | 20260506 12:43:40.984 | INFO | "huge_shared" : " 27 GB", | 20260506 12:43:40.984 | INFO | "malloc" : " 14 GB", | 20260506 12:43:40.984 | INFO | "pinned" : " 29 GB" | 20260506 12:43:40.984 | INFO | } | 20260506 12:43:40.984 | INFO | Perf report for the last 5.13938 seconds: | 20260506 12:43:40.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:43:40.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:40.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:40.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:40.984 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:43:40.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:40.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:40.984 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:43:40.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:40.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:40.984 | INFO | Uploaded 36GiB to GPU in 2.00736s (17.934GiB/s) | 20260506 12:43:40.984 | INFO | Downloaded 34GiB to GPU in 1.94765s (17.457GiB/s) | 20260506 12:43:46.123 | INFO | Analyzing chunk frames = [1447424,1447936) | 20260506 12:43:46.123 | INFO | MemInfo: { | 20260506 12:43:46.123 | INFO | "gpu" : " 17 GB", | 20260506 12:43:46.123 | INFO | "huge" : "", | 20260506 12:43:46.123 | INFO | "huge_shared" : " 27 GB", | 20260506 12:43:46.123 | INFO | "malloc" : " 14 GB", | 20260506 12:43:46.123 | INFO | "pinned" : " 29 GB" | 20260506 12:43:46.123 | INFO | } | 20260506 12:43:46.123 | INFO | Perf report for the last 5.13933 seconds: | 20260506 12:43:46.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:43:46.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:46.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:46.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:46.123 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 12:43:46.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:46.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:46.123 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:43:46.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:46.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:46.123 | INFO | Uploaded 36GiB to GPU in 1.98816s (18.1072GiB/s) | 20260506 12:43:46.123 | INFO | Downloaded 34GiB to GPU in 1.94223s (17.5057GiB/s) | 20260506 12:43:51.262 | INFO | Analyzing chunk frames = [1447936,1448448) | 20260506 12:43:51.262 | INFO | MemInfo: { | 20260506 12:43:51.262 | INFO | "gpu" : " 17 GB", | 20260506 12:43:51.262 | INFO | "huge" : "", | 20260506 12:43:51.263 | INFO | "huge_shared" : " 27 GB", | 20260506 12:43:51.263 | INFO | "malloc" : " 14 GB", | 20260506 12:43:51.263 | INFO | "pinned" : " 29 GB" | 20260506 12:43:51.263 | INFO | } | 20260506 12:43:51.263 | INFO | Perf report for the last 5.13949 seconds: | 20260506 12:43:51.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:43:51.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:51.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:51.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:51.263 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 12:43:51.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:51.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:51.263 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:43:51.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:51.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:51.263 | INFO | Uploaded 36GiB to GPU in 2.02557s (17.7728GiB/s) | 20260506 12:43:51.263 | INFO | Downloaded 34GiB to GPU in 1.94842s (17.45GiB/s) | 20260506 12:43:56.402 | INFO | Analyzing chunk frames = [1448448,1448960) | 20260506 12:43:56.402 | INFO | MemInfo: { | 20260506 12:43:56.402 | INFO | "gpu" : " 17 GB", | 20260506 12:43:56.402 | INFO | "huge" : "", | 20260506 12:43:56.402 | INFO | "huge_shared" : " 27 GB", | 20260506 12:43:56.402 | INFO | "malloc" : " 14 GB", | 20260506 12:43:56.402 | INFO | "pinned" : " 29 GB" | 20260506 12:43:56.402 | INFO | } | 20260506 12:43:56.402 | INFO | Perf report for the last 5.13919 seconds: | 20260506 12:43:56.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:43:56.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:56.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:56.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:56.402 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 12:43:56.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:56.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:56.402 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:43:56.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:56.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:43:56.402 | INFO | Uploaded 36GiB to GPU in 2.02705s (17.7598GiB/s) | 20260506 12:43:56.402 | INFO | Downloaded 34GiB to GPU in 1.93167s (17.6014GiB/s) | 20260506 12:44:01.541 | INFO | Analyzing chunk frames = [1448960,1449472) | 20260506 12:44:01.541 | INFO | MemInfo: { | 20260506 12:44:01.541 | INFO | "gpu" : " 17 GB", | 20260506 12:44:01.541 | INFO | "huge" : "", | 20260506 12:44:01.542 | INFO | "huge_shared" : " 27 GB", | 20260506 12:44:01.542 | INFO | "malloc" : " 14 GB", | 20260506 12:44:01.542 | INFO | "pinned" : " 29 GB" | 20260506 12:44:01.542 | INFO | } | 20260506 12:44:01.542 | INFO | Perf report for the last 5.13972 seconds: | 20260506 12:44:01.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:44:01.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:01.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:01.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:01.542 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:44:01.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:01.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:01.542 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:44:01.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:01.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:01.542 | INFO | Uploaded 36GiB to GPU in 2.02018s (17.8202GiB/s) | 20260506 12:44:01.542 | INFO | Downloaded 34GiB to GPU in 2.02891s (16.7577GiB/s) | 20260506 12:44:06.680 | INFO | Analyzing chunk frames = [1449472,1449984) | 20260506 12:44:06.680 | INFO | MemInfo: { | 20260506 12:44:06.680 | INFO | "gpu" : " 17 GB", | 20260506 12:44:06.680 | INFO | "huge" : "", | 20260506 12:44:06.681 | INFO | "huge_shared" : " 27 GB", | 20260506 12:44:06.681 | INFO | "malloc" : " 14 GB", | 20260506 12:44:06.681 | INFO | "pinned" : " 29 GB" | 20260506 12:44:06.681 | INFO | } | 20260506 12:44:06.681 | INFO | Perf report for the last 5.13899 seconds: | 20260506 12:44:06.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:44:06.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:06.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:06.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:06.681 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:44:06.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:06.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:06.681 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:44:06.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:06.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:06.681 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:44:06.681 | INFO | Total mem usage: 16.8 GiB | 20260506 12:44:06.681 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:44:06.681 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.8%) | 20260506 12:44:06.681 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:44:06.681 | INFO | High Water Breakdown: | 20260506 12:44:06.681 | INFO | 59.7% : 8.59 GiB ( 82.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:44:06.681 | INFO | 40.2% : 5.79 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:44:06.681 | INFO | Uploaded 36GiB to GPU in 2.04065s (17.6414GiB/s) | 20260506 12:44:06.681 | INFO | Downloaded 34GiB to GPU in 1.96087s (17.3393GiB/s) | 20260506 12:44:06.681 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:44:06.681 | INFO | ------------------------------------------ | 20260506 12:44:06.681 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:44:06.681 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:44:06.681 | INFO | Total mem usage: 26.5 GiB | 20260506 12:44:06.681 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:44:06.681 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:44:06.681 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:44:06.681 | INFO | High Water Breakdown: | 20260506 12:44:06.681 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:44:06.681 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:44:06.681 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:44:06.681 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:44:06.681 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:44:06.681 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:44:06.681 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:44:06.681 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:44:06.681 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:44:06.681 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:44:06.681 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:44:06.681 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:44:06.681 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:44:06.681 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:44:06.681 | INFO | ------------------------------------------ | 20260506 12:44:06.681 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:44:06.681 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:44:06.681 | INFO | Total mem usage: 25.6 GiB | 20260506 12:44:06.681 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:44:06.681 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:44:06.681 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:44:06.681 | INFO | High Water Breakdown: | 20260506 12:44:06.681 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:44:06.681 | INFO | ------------------------------------------ | 20260506 12:44:06.681 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:44:06.681 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:44:06.681 | INFO | Total mem usage: 15.7 GiB | 20260506 12:44:06.681 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:44:06.681 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:44:06.681 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 12:44:06.681 | INFO | High Water Breakdown: | 20260506 12:44:06.681 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:44:06.681 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:44:06.681 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:44:06.681 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:44:06.681 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:44:06.681 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:44:06.681 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:44:06.681 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:44:06.681 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:44:06.681 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:44:06.681 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:44:06.681 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:44:06.681 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:44:06.681 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:44:06.681 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:44:06.681 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:44:06.681 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:44:06.681 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:44:06.681 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:44:06.681 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:44:06.681 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:44:06.682 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:44:06.682 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:44:06.682 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:44:06.682 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:44:06.682 | INFO | ------------------------------------------ | 20260506 12:44:06.682 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:44:06.682 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:44:06.682 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:44:06.682 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:44:06.682 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.58e+01, 1.35e+01 / 2.11e+00 | 20260506 12:44:06.682 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.02e-01, 1.01e-02 / 7.04e-03 | 20260506 12:44:06.682 | INFO | PerfReport: 11.56%, Baseline -- 3.67e+00 / 4.45e+00, 5.81e-01 / 3.55e-01 | 20260506 12:44:06.682 | INFO | PerfReport: 3.23%, Histogram -- 1.02e+00 / 1.45e+00, 1.88e-01 / 1.41e-01 | 20260506 12:44:06.682 | INFO | PerfReport: 15.74%, DME -- 3.99e+01 / 4.39e+01, 3.81e+00 / 2.29e+00 | 20260506 12:44:06.682 | INFO | PerfReport: 56.56%, FrameLabeling -- 1.79e+01 / 2.09e+01, 1.48e+00 / 1.60e+00 | 20260506 12:44:06.682 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.40e+00 / 3.03e+00, 4.23e-01 / 2.19e-01 | 20260506 12:44:06.682 | INFO | PerfReport: 3.73%, Metrics -- 1.18e+00 / 1.57e+00, 3.01e-01 / 1.66e-01 | 20260506 12:44:06.682 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.15e-01 / 4.66e-01, 6.38e-02 / 2.60e-02 | 20260506 12:44:06.682 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:44:06.682 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:44:06.682 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:44:06.682 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 12:44:06.682 | INFO | PerfReport: 21.44%, Upload -- 2.12e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260506 12:44:06.682 | INFO | PerfReport: 21.04%, Download -- 2.08e+01 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260506 12:44:06.682 | INFO | PerfReport: 32.13%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 12:44:06.682 | INFO | PerfReport: 1.69%, LockUpload -- 1.67e+00 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260506 12:44:06.682 | INFO | PerfReport: 1.14%, LockDownload -- 1.12e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 12:44:06.682 | INFO | PerfReport: 22.56%, LockCompute -- 2.23e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 12:44:06.682 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260506 12:44:06.812 | INFO | Starting a round of Baz IO | 20260506 12:44:09.981 | INFO | Populating RTMetrics report | 20260506 12:44:09.981 | INFO | RTMetrics report populated | 20260506 12:44:09.981 | INFO | Opening tmp RTMetrics json file | 20260506 12:44:09.982 | INFO | Serializing RTMetrics to json | 20260506 12:44:09.984 | INFO | Serialization done, swapping tmp file now... | 20260506 12:44:09.984 | INFO | RTMetrics done | 20260506 12:44:10.078 | INFO | Finished a round of Baz IO: Wrote 2.099 GiB in 3.265 seconds | 20260506 12:44:10.078 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.98% / 0.02% / 58.99% | 20260506 12:44:10.078 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.72363% / 5.5631e-05% / 4.76837e-05% | 20260506 12:44:10.078 | INFO | Baz IO overall: | 20260506 12:44:10.078 | INFO | BytesWritten : 725284628557 | 20260506 12:44:10.078 | INFO | NumEvents : 350601209861 | 20260506 12:44:11.820 | INFO | Analyzing chunk frames = [1449984,1450496) | 20260506 12:44:11.820 | INFO | MemInfo: { | 20260506 12:44:11.820 | INFO | "gpu" : " 17 GB", | 20260506 12:44:11.820 | INFO | "huge" : "", | 20260506 12:44:11.820 | INFO | "huge_shared" : " 27 GB", | 20260506 12:44:11.820 | INFO | "malloc" : " 14 GB", | 20260506 12:44:11.820 | INFO | "pinned" : " 29 GB" | 20260506 12:44:11.820 | INFO | } | 20260506 12:44:11.820 | INFO | Perf report for the last 5.13953 seconds: | 20260506 12:44:11.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:44:11.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:11.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:11.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:11.820 | INFO | ANALYSIS : 63.6%, 3.0, 98.9ms, 0 | 20260506 12:44:11.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:11.820 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:44:11.820 | INFO | PRE_HQ : 61.4%, 1.9, 61.2ms, 0 | 20260506 12:44:11.820 | INFO | BAZWRITER : 62.9%, 3.6, 117.7ms, 0 | 20260506 12:44:11.820 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:44:11.820 | INFO | Uploaded 36GiB to GPU in 2.07865s (17.3189GiB/s) | 20260506 12:44:11.820 | INFO | Downloaded 38GiB to GPU in 2.22766s (17.0583GiB/s) | 20260506 12:44:16.959 | INFO | Analyzing chunk frames = [1450496,1451008) | 20260506 12:44:16.959 | INFO | MemInfo: { | 20260506 12:44:16.959 | INFO | "gpu" : " 17 GB", | 20260506 12:44:16.959 | INFO | "huge" : "", | 20260506 12:44:16.959 | INFO | "huge_shared" : " 27 GB", | 20260506 12:44:16.959 | INFO | "malloc" : " 14 GB", | 20260506 12:44:16.959 | INFO | "pinned" : " 29 GB" | 20260506 12:44:16.959 | INFO | } | 20260506 12:44:16.959 | INFO | Perf report for the last 5.1392 seconds: | 20260506 12:44:16.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:44:16.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:16.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:16.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:16.959 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:44:16.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:16.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:16.959 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:44:16.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:16.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:16.960 | INFO | Uploaded 36GiB to GPU in 2.01209s (17.8918GiB/s) | 20260506 12:44:16.960 | INFO | Downloaded 34GiB to GPU in 1.92946s (17.6215GiB/s) | 20260506 12:44:22.098 | INFO | Analyzing chunk frames = [1451008,1451520) | 20260506 12:44:22.098 | INFO | MemInfo: { | 20260506 12:44:22.098 | INFO | "gpu" : " 17 GB", | 20260506 12:44:22.098 | INFO | "huge" : "", | 20260506 12:44:22.098 | INFO | "huge_shared" : " 27 GB", | 20260506 12:44:22.099 | INFO | "malloc" : " 14 GB", | 20260506 12:44:22.099 | INFO | "pinned" : " 29 GB" | 20260506 12:44:22.099 | INFO | } | 20260506 12:44:22.099 | INFO | Perf report for the last 5.13926 seconds: | 20260506 12:44:22.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:44:22.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:22.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:22.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:22.099 | INFO | ANALYSIS : 61.4%, 3.0, 95.0ms, 0 | 20260506 12:44:22.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:22.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:22.099 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:44:22.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:22.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:22.099 | INFO | Uploaded 36GiB to GPU in 1.9583s (18.3833GiB/s) | 20260506 12:44:22.099 | INFO | Downloaded 34GiB to GPU in 1.93424s (17.578GiB/s) | 20260506 12:44:27.238 | INFO | Analyzing chunk frames = [1451520,1452032) | 20260506 12:44:27.238 | INFO | MemInfo: { | 20260506 12:44:27.238 | INFO | "gpu" : " 17 GB", | 20260506 12:44:27.238 | INFO | "huge" : "", | 20260506 12:44:27.238 | INFO | "huge_shared" : " 27 GB", | 20260506 12:44:27.238 | INFO | "malloc" : " 14 GB", | 20260506 12:44:27.238 | INFO | "pinned" : " 29 GB" | 20260506 12:44:27.238 | INFO | } | 20260506 12:44:27.238 | INFO | Perf report for the last 5.13983 seconds: | 20260506 12:44:27.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:44:27.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:27.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:27.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:27.239 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 12:44:27.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:27.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:27.239 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:44:27.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:27.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:27.239 | INFO | Uploaded 36GiB to GPU in 1.99562s (18.0395GiB/s) | 20260506 12:44:27.239 | INFO | Downloaded 34GiB to GPU in 1.91547s (17.7502GiB/s) | 20260506 12:44:32.377 | INFO | Analyzing chunk frames = [1452032,1452544) | 20260506 12:44:32.377 | INFO | MemInfo: { | 20260506 12:44:32.377 | INFO | "gpu" : " 17 GB", | 20260506 12:44:32.377 | INFO | "huge" : "", | 20260506 12:44:32.377 | INFO | "huge_shared" : " 27 GB", | 20260506 12:44:32.377 | INFO | "malloc" : " 14 GB", | 20260506 12:44:32.377 | INFO | "pinned" : " 29 GB" | 20260506 12:44:32.377 | INFO | } | 20260506 12:44:32.377 | INFO | Perf report for the last 5.13888 seconds: | 20260506 12:44:32.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:44:32.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:32.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:32.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:32.377 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 12:44:32.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:32.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:32.377 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:44:32.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:32.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:32.377 | INFO | Uploaded 36GiB to GPU in 2.02274s (17.7977GiB/s) | 20260506 12:44:32.378 | INFO | Downloaded 34GiB to GPU in 1.95085s (17.4283GiB/s) | 20260506 12:44:37.516 | INFO | Analyzing chunk frames = [1452544,1453056) | 20260506 12:44:37.516 | INFO | MemInfo: { | 20260506 12:44:37.516 | INFO | "gpu" : " 17 GB", | 20260506 12:44:37.516 | INFO | "huge" : "", | 20260506 12:44:37.517 | INFO | "huge_shared" : " 27 GB", | 20260506 12:44:37.517 | INFO | "malloc" : " 14 GB", | 20260506 12:44:37.517 | INFO | "pinned" : " 29 GB" | 20260506 12:44:37.517 | INFO | } | 20260506 12:44:37.517 | INFO | Perf report for the last 5.13924 seconds: | 20260506 12:44:37.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:44:37.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:37.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:37.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:37.517 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 12:44:37.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:37.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:37.517 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260506 12:44:37.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:37.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:37.517 | INFO | Uploaded 36GiB to GPU in 2.06445s (17.4381GiB/s) | 20260506 12:44:37.517 | INFO | Downloaded 34GiB to GPU in 1.93986s (17.527GiB/s) | 20260506 12:44:42.657 | INFO | Analyzing chunk frames = [1453056,1453568) | 20260506 12:44:42.657 | INFO | MemInfo: { | 20260506 12:44:42.657 | INFO | "gpu" : " 17 GB", | 20260506 12:44:42.657 | INFO | "huge" : "", | 20260506 12:44:42.657 | INFO | "huge_shared" : " 27 GB", | 20260506 12:44:42.657 | INFO | "malloc" : " 14 GB", | 20260506 12:44:42.657 | INFO | "pinned" : " 29 GB" | 20260506 12:44:42.657 | INFO | } | 20260506 12:44:42.657 | INFO | Perf report for the last 5.14029 seconds: | 20260506 12:44:42.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:44:42.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:42.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:42.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:42.657 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:44:42.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:42.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:42.657 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:44:42.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:42.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:42.657 | INFO | Uploaded 36GiB to GPU in 2.00922s (17.9174GiB/s) | 20260506 12:44:42.657 | INFO | Downloaded 34GiB to GPU in 2.03075s (16.7426GiB/s) | 20260506 12:44:47.795 | INFO | Analyzing chunk frames = [1453568,1454080) | 20260506 12:44:47.795 | INFO | MemInfo: { | 20260506 12:44:47.795 | INFO | "gpu" : " 17 GB", | 20260506 12:44:47.795 | INFO | "huge" : "", | 20260506 12:44:47.795 | INFO | "huge_shared" : " 27 GB", | 20260506 12:44:47.795 | INFO | "malloc" : " 14 GB", | 20260506 12:44:47.795 | INFO | "pinned" : " 29 GB" | 20260506 12:44:47.795 | INFO | } | 20260506 12:44:47.795 | INFO | Perf report for the last 5.13847 seconds: | 20260506 12:44:47.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:44:47.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:47.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:47.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:47.795 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 12:44:47.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:47.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:47.795 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:44:47.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:47.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:47.796 | INFO | Uploaded 36GiB to GPU in 2.00508s (17.9544GiB/s) | 20260506 12:44:47.796 | INFO | Downloaded 34GiB to GPU in 1.99325s (17.0576GiB/s) | 20260506 12:44:47.928 | INFO | Starting a round of Baz IO | 20260506 12:44:51.087 | INFO | Populating RTMetrics report | 20260506 12:44:51.087 | INFO | RTMetrics report populated | 20260506 12:44:51.087 | INFO | Opening tmp RTMetrics json file | 20260506 12:44:51.087 | INFO | Serializing RTMetrics to json | 20260506 12:44:51.089 | INFO | Serialization done, swapping tmp file now... | 20260506 12:44:51.089 | INFO | RTMetrics done | 20260506 12:44:51.183 | INFO | Finished a round of Baz IO: Wrote 2.098 GiB in 3.254 seconds | 20260506 12:44:51.183 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.96% / 0.02% / 59.02% | 20260506 12:44:51.183 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.70889% / 3.17891e-05% / 3.17891e-05% | 20260506 12:44:51.183 | INFO | Baz IO overall: | 20260506 12:44:51.183 | INFO | BytesWritten : 727537624360 | 20260506 12:44:51.183 | INFO | NumEvents : 351694437916 | 20260506 12:44:52.935 | INFO | Analyzing chunk frames = [1454080,1454592) | 20260506 12:44:52.935 | INFO | MemInfo: { | 20260506 12:44:52.935 | INFO | "gpu" : " 17 GB", | 20260506 12:44:52.935 | INFO | "huge" : "", | 20260506 12:44:52.935 | INFO | "huge_shared" : " 27 GB", | 20260506 12:44:52.935 | INFO | "malloc" : " 14 GB", | 20260506 12:44:52.935 | INFO | "pinned" : " 29 GB" | 20260506 12:44:52.935 | INFO | } | 20260506 12:44:52.935 | INFO | Perf report for the last 5.13971 seconds: | 20260506 12:44:52.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:44:52.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:52.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:52.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:52.935 | INFO | ANALYSIS : 63.5%, 3.0, 98.5ms, 0 | 20260506 12:44:52.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:52.935 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:44:52.935 | INFO | PRE_HQ : 61.4%, 1.9, 61.3ms, 0 | 20260506 12:44:52.935 | INFO | BAZWRITER : 62.7%, 3.7, 119.6ms, 0 | 20260506 12:44:52.935 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:44:52.935 | INFO | Uploaded 36GiB to GPU in 2.03252s (17.712GiB/s) | 20260506 12:44:52.935 | INFO | Downloaded 38GiB to GPU in 2.21155s (17.1825GiB/s) | 20260506 12:44:58.074 | INFO | Analyzing chunk frames = [1454592,1455104) | 20260506 12:44:58.074 | INFO | MemInfo: { | 20260506 12:44:58.074 | INFO | "gpu" : " 17 GB", | 20260506 12:44:58.074 | INFO | "huge" : "", | 20260506 12:44:58.074 | INFO | "huge_shared" : " 27 GB", | 20260506 12:44:58.074 | INFO | "malloc" : " 14 GB", | 20260506 12:44:58.074 | INFO | "pinned" : " 29 GB" | 20260506 12:44:58.074 | INFO | } | 20260506 12:44:58.074 | INFO | Perf report for the last 5.13913 seconds: | 20260506 12:44:58.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:44:58.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:58.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:58.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:58.074 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:44:58.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:58.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:58.074 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:44:58.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:58.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:44:58.074 | INFO | Uploaded 36GiB to GPU in 2.02876s (17.7449GiB/s) | 20260506 12:44:58.075 | INFO | Downloaded 34GiB to GPU in 1.94387s (17.4909GiB/s) | 20260506 12:45:03.214 | INFO | Analyzing chunk frames = [1455104,1455616) | 20260506 12:45:03.214 | INFO | MemInfo: { | 20260506 12:45:03.214 | INFO | "gpu" : " 17 GB", | 20260506 12:45:03.214 | INFO | "huge" : "", | 20260506 12:45:03.214 | INFO | "huge_shared" : " 27 GB", | 20260506 12:45:03.214 | INFO | "malloc" : " 14 GB", | 20260506 12:45:03.214 | INFO | "pinned" : " 29 GB" | 20260506 12:45:03.214 | INFO | } | 20260506 12:45:03.214 | INFO | Perf report for the last 5.13975 seconds: | 20260506 12:45:03.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:45:03.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:03.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:03.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:03.214 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260506 12:45:03.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:03.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:03.214 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:45:03.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:03.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:03.214 | INFO | Uploaded 36GiB to GPU in 2.01064s (17.9047GiB/s) | 20260506 12:45:03.214 | INFO | Downloaded 34GiB to GPU in 1.94267s (17.5016GiB/s) | 20260506 12:45:08.352 | INFO | Analyzing chunk frames = [1455616,1456128) | 20260506 12:45:08.353 | INFO | MemInfo: { | 20260506 12:45:08.353 | INFO | "gpu" : " 17 GB", | 20260506 12:45:08.353 | INFO | "huge" : "", | 20260506 12:45:08.353 | INFO | "huge_shared" : " 27 GB", | 20260506 12:45:08.353 | INFO | "malloc" : " 14 GB", | 20260506 12:45:08.353 | INFO | "pinned" : " 29 GB" | 20260506 12:45:08.353 | INFO | } | 20260506 12:45:08.353 | INFO | Perf report for the last 5.13864 seconds: | 20260506 12:45:08.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:45:08.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:08.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:08.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:08.353 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 12:45:08.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:08.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:08.353 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:45:08.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:08.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:08.353 | INFO | Uploaded 36GiB to GPU in 1.99156s (18.0763GiB/s) | 20260506 12:45:08.353 | INFO | Downloaded 34GiB to GPU in 1.94192s (17.5085GiB/s) | 20260506 12:45:13.492 | INFO | Analyzing chunk frames = [1456128,1456640) | 20260506 12:45:13.492 | INFO | MemInfo: { | 20260506 12:45:13.492 | INFO | "gpu" : " 17 GB", | 20260506 12:45:13.492 | INFO | "huge" : "", | 20260506 12:45:13.492 | INFO | "huge_shared" : " 27 GB", | 20260506 12:45:13.492 | INFO | "malloc" : " 14 GB", | 20260506 12:45:13.492 | INFO | "pinned" : " 29 GB" | 20260506 12:45:13.492 | INFO | } | 20260506 12:45:13.492 | INFO | Perf report for the last 5.13934 seconds: | 20260506 12:45:13.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:45:13.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:13.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:13.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:13.492 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 12:45:13.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:13.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:13.492 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:45:13.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:13.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:13.492 | INFO | Uploaded 36GiB to GPU in 2.02747s (17.7561GiB/s) | 20260506 12:45:13.492 | INFO | Downloaded 34GiB to GPU in 1.95074s (17.4293GiB/s) | 20260506 12:45:18.631 | INFO | Analyzing chunk frames = [1456640,1457152) | 20260506 12:45:18.631 | INFO | MemInfo: { | 20260506 12:45:18.631 | INFO | "gpu" : " 17 GB", | 20260506 12:45:18.631 | INFO | "huge" : "", | 20260506 12:45:18.631 | INFO | "huge_shared" : " 27 GB", | 20260506 12:45:18.631 | INFO | "malloc" : " 14 GB", | 20260506 12:45:18.631 | INFO | "pinned" : " 29 GB" | 20260506 12:45:18.631 | INFO | } | 20260506 12:45:18.631 | INFO | Perf report for the last 5.13943 seconds: | 20260506 12:45:18.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:45:18.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:18.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:18.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:18.632 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 12:45:18.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:18.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:18.632 | INFO | PRE_HQ : 33.2%, 1.0, 17.2ms, 0 | 20260506 12:45:18.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:18.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:18.632 | INFO | Uploaded 36GiB to GPU in 2.05951s (17.4799GiB/s) | 20260506 12:45:18.632 | INFO | Downloaded 34GiB to GPU in 1.94059s (17.5205GiB/s) | 20260506 12:45:23.771 | INFO | Analyzing chunk frames = [1457152,1457664) | 20260506 12:45:23.771 | INFO | MemInfo: { | 20260506 12:45:23.771 | INFO | "gpu" : " 17 GB", | 20260506 12:45:23.771 | INFO | "huge" : "", | 20260506 12:45:23.771 | INFO | "huge_shared" : " 27 GB", | 20260506 12:45:23.771 | INFO | "malloc" : " 14 GB", | 20260506 12:45:23.771 | INFO | "pinned" : " 29 GB" | 20260506 12:45:23.771 | INFO | } | 20260506 12:45:23.771 | INFO | Perf report for the last 5.13972 seconds: | 20260506 12:45:23.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:45:23.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:23.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:23.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:23.771 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:45:23.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:23.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:23.771 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:45:23.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:23.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:23.771 | INFO | Uploaded 36GiB to GPU in 2.0023s (17.9794GiB/s) | 20260506 12:45:23.772 | INFO | Downloaded 34GiB to GPU in 2.02486s (16.7913GiB/s) | 20260506 12:45:28.910 | INFO | Analyzing chunk frames = [1457664,1458176) | 20260506 12:45:28.910 | INFO | MemInfo: { | 20260506 12:45:28.910 | INFO | "gpu" : " 17 GB", | 20260506 12:45:28.910 | INFO | "huge" : "", | 20260506 12:45:28.910 | INFO | "huge_shared" : " 27 GB", | 20260506 12:45:28.910 | INFO | "malloc" : " 14 GB", | 20260506 12:45:28.910 | INFO | "pinned" : " 29 GB" | 20260506 12:45:28.910 | INFO | } | 20260506 12:45:28.910 | INFO | Perf report for the last 5.13919 seconds: | 20260506 12:45:28.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:45:28.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:28.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:28.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:28.910 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 12:45:28.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:28.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:28.910 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 12:45:28.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:28.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:28.911 | INFO | Uploaded 36GiB to GPU in 2.04867s (17.5724GiB/s) | 20260506 12:45:28.911 | INFO | Downloaded 34GiB to GPU in 1.96305s (17.32GiB/s) | 20260506 12:45:28.911 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:45:28.911 | INFO | Total mem usage: 16.8 GiB | 20260506 12:45:28.911 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:45:28.911 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.8%) | 20260506 12:45:28.911 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:45:28.911 | INFO | High Water Breakdown: | 20260506 12:45:28.911 | INFO | 59.7% : 8.59 GiB ( 82.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:45:28.911 | INFO | 40.2% : 5.79 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:45:28.911 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:45:28.911 | INFO | ------------------------------------------ | 20260506 12:45:28.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:45:28.911 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:45:28.911 | INFO | Total mem usage: 26.5 GiB | 20260506 12:45:28.911 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:45:28.911 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:45:28.911 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:45:28.911 | INFO | High Water Breakdown: | 20260506 12:45:28.911 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:45:28.911 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:45:28.911 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:45:28.911 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:45:28.911 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:45:28.911 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:45:28.911 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:45:28.911 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:45:28.911 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:45:28.911 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:45:28.911 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:45:28.911 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:45:28.911 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:45:28.911 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:45:28.911 | INFO | ------------------------------------------ | 20260506 12:45:28.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:45:28.911 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:45:28.911 | INFO | Total mem usage: 25.6 GiB | 20260506 12:45:28.911 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:45:28.911 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:45:28.911 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:45:28.911 | INFO | High Water Breakdown: | 20260506 12:45:28.911 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:45:28.911 | INFO | ------------------------------------------ | 20260506 12:45:28.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:45:28.911 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:45:28.911 | INFO | Total mem usage: 15.7 GiB | 20260506 12:45:28.911 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:45:28.911 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:45:28.911 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 12:45:28.911 | INFO | High Water Breakdown: | 20260506 12:45:28.911 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:45:28.911 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:45:28.911 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:45:28.911 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:45:28.911 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:45:28.911 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:45:28.911 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:45:28.911 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:45:28.911 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:45:28.911 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:45:28.911 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:45:28.912 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:45:28.912 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:45:28.912 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:45:28.912 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:45:28.912 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:45:28.912 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:45:28.912 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:45:28.912 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:45:28.912 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:45:28.912 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:45:28.912 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:45:28.912 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:45:28.912 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:45:28.912 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:45:28.912 | INFO | ------------------------------------------ | 20260506 12:45:28.912 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:45:28.912 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:45:28.912 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:45:28.912 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:45:28.912 | INFO | PerfReport: Full Execution -- 3.18e+01 / 7.62e+01, 1.35e+01 / 2.21e+00 | 20260506 12:45:28.912 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.01e-01, 1.05e-02 / 8.69e-03 | 20260506 12:45:28.912 | INFO | PerfReport: 11.56%, Baseline -- 3.67e+00 / 4.39e+00, 5.84e-01 / 4.08e-01 | 20260506 12:45:28.912 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.45e+00, 1.92e-01 / 1.40e-01 | 20260506 12:45:28.912 | INFO | PerfReport: 15.76%, DME -- 4.00e+01 / 4.43e+01, 3.96e+00 / 2.56e+00 | 20260506 12:45:28.912 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.80e+01 / 2.10e+01, 1.52e+00 / 1.95e+00 | 20260506 12:45:28.912 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.39e+00 / 2.93e+00, 4.05e-01 / 2.69e-01 | 20260506 12:45:28.912 | INFO | PerfReport: 3.72%, Metrics -- 1.18e+00 / 1.57e+00, 3.02e-01 / 1.87e-01 | 20260506 12:45:28.912 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.15e-01 / 4.66e-01, 6.42e-02 / 2.65e-02 | 20260506 12:45:28.912 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:45:28.912 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:45:28.912 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:45:28.912 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 12:45:28.912 | INFO | PerfReport: 21.35%, Upload -- 2.11e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 12:45:28.912 | INFO | PerfReport: 21.02%, Download -- 2.07e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 12:45:28.912 | INFO | PerfReport: 32.20%, Compute -- 3.18e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 12:45:28.912 | INFO | PerfReport: 1.64%, LockUpload -- 1.62e+00 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 12:45:28.912 | INFO | PerfReport: 1.13%, LockDownload -- 1.12e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 12:45:28.912 | INFO | PerfReport: 22.63%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:45:28.912 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260506 12:45:29.042 | INFO | Starting a round of Baz IO | 20260506 12:45:32.202 | INFO | Populating RTMetrics report | 20260506 12:45:32.202 | INFO | RTMetrics report populated | 20260506 12:45:32.202 | INFO | Opening tmp RTMetrics json file | 20260506 12:45:32.203 | INFO | Serializing RTMetrics to json | 20260506 12:45:32.204 | INFO | Serialization done, swapping tmp file now... | 20260506 12:45:32.204 | INFO | RTMetrics done | 20260506 12:45:32.299 | INFO | Finished a round of Baz IO: Wrote 2.098 GiB in 3.257 seconds | 20260506 12:45:32.299 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.94% / 0.02% / 59.04% | 20260506 12:45:32.299 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.69523% / 3.97364e-05% / 4.76837e-05% | 20260506 12:45:32.299 | INFO | Baz IO overall: | 20260506 12:45:32.299 | INFO | BytesWritten : 729789900395 | 20260506 12:45:32.299 | INFO | NumEvents : 352787108276 | 20260506 12:45:34.050 | INFO | Analyzing chunk frames = [1458176,1458688) | 20260506 12:45:34.050 | INFO | MemInfo: { | 20260506 12:45:34.050 | INFO | "gpu" : " 17 GB", | 20260506 12:45:34.050 | INFO | "huge" : "", | 20260506 12:45:34.050 | INFO | "huge_shared" : " 27 GB", | 20260506 12:45:34.050 | INFO | "malloc" : " 14 GB", | 20260506 12:45:34.050 | INFO | "pinned" : " 29 GB" | 20260506 12:45:34.050 | INFO | } | 20260506 12:45:34.050 | INFO | Perf report for the last 5.13933 seconds: | 20260506 12:45:34.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:45:34.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:34.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:34.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:34.050 | INFO | ANALYSIS : 63.5%, 3.0, 98.5ms, 0 | 20260506 12:45:34.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:34.050 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:45:34.050 | INFO | PRE_HQ : 61.4%, 1.9, 61.0ms, 0 | 20260506 12:45:34.050 | INFO | BAZWRITER : 62.8%, 3.7, 119.8ms, 0 | 20260506 12:45:34.050 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:45:34.050 | INFO | Uploaded 36GiB to GPU in 2.08393s (17.2751GiB/s) | 20260506 12:45:34.050 | INFO | Downloaded 38GiB to GPU in 2.22844s (17.0523GiB/s) | 20260506 12:45:39.189 | INFO | Analyzing chunk frames = [1458688,1459200) | 20260506 12:45:39.189 | INFO | MemInfo: { | 20260506 12:45:39.189 | INFO | "gpu" : " 17 GB", | 20260506 12:45:39.189 | INFO | "huge" : "", | 20260506 12:45:39.189 | INFO | "huge_shared" : " 27 GB", | 20260506 12:45:39.189 | INFO | "malloc" : " 14 GB", | 20260506 12:45:39.189 | INFO | "pinned" : " 29 GB" | 20260506 12:45:39.189 | INFO | } | 20260506 12:45:39.189 | INFO | Perf report for the last 5.13962 seconds: | 20260506 12:45:39.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:45:39.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:39.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:39.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:39.189 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:45:39.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:39.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:39.189 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 12:45:39.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:39.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:39.190 | INFO | Uploaded 36GiB to GPU in 2.02146s (17.8089GiB/s) | 20260506 12:45:39.190 | INFO | Downloaded 34GiB to GPU in 1.92591s (17.654GiB/s) | 20260506 12:45:44.328 | INFO | Analyzing chunk frames = [1459200,1459712) | 20260506 12:45:44.328 | INFO | MemInfo: { | 20260506 12:45:44.328 | INFO | "gpu" : " 17 GB", | 20260506 12:45:44.328 | INFO | "huge" : "", | 20260506 12:45:44.329 | INFO | "huge_shared" : " 27 GB", | 20260506 12:45:44.329 | INFO | "malloc" : " 14 GB", | 20260506 12:45:44.329 | INFO | "pinned" : " 29 GB" | 20260506 12:45:44.329 | INFO | } | 20260506 12:45:44.329 | INFO | Perf report for the last 5.13924 seconds: | 20260506 12:45:44.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:45:44.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:44.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:44.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:44.329 | INFO | ANALYSIS : 61.5%, 3.0, 95.2ms, 0 | 20260506 12:45:44.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:44.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:44.329 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:45:44.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:44.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:44.329 | INFO | Uploaded 36GiB to GPU in 1.98645s (18.1228GiB/s) | 20260506 12:45:44.329 | INFO | Downloaded 34GiB to GPU in 1.92984s (17.618GiB/s) | 20260506 12:45:49.467 | INFO | Analyzing chunk frames = [1459712,1460224) | 20260506 12:45:49.468 | INFO | MemInfo: { | 20260506 12:45:49.468 | INFO | "gpu" : " 17 GB", | 20260506 12:45:49.468 | INFO | "huge" : "", | 20260506 12:45:49.468 | INFO | "huge_shared" : " 27 GB", | 20260506 12:45:49.468 | INFO | "malloc" : " 14 GB", | 20260506 12:45:49.468 | INFO | "pinned" : " 29 GB" | 20260506 12:45:49.468 | INFO | } | 20260506 12:45:49.468 | INFO | Perf report for the last 5.1391 seconds: | 20260506 12:45:49.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:45:49.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:49.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:49.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:49.468 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 12:45:49.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:49.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:49.468 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:45:49.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:49.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:49.468 | INFO | Uploaded 36GiB to GPU in 1.99206s (18.0718GiB/s) | 20260506 12:45:49.468 | INFO | Downloaded 34GiB to GPU in 1.91883s (17.7191GiB/s) | 20260506 12:45:54.607 | INFO | Analyzing chunk frames = [1460224,1460736) | 20260506 12:45:54.607 | INFO | MemInfo: { | 20260506 12:45:54.607 | INFO | "gpu" : " 17 GB", | 20260506 12:45:54.607 | INFO | "huge" : "", | 20260506 12:45:54.607 | INFO | "huge_shared" : " 27 GB", | 20260506 12:45:54.607 | INFO | "malloc" : " 14 GB", | 20260506 12:45:54.607 | INFO | "pinned" : " 29 GB" | 20260506 12:45:54.607 | INFO | } | 20260506 12:45:54.607 | INFO | Perf report for the last 5.13914 seconds: | 20260506 12:45:54.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:45:54.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:54.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:54.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:54.607 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 12:45:54.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:54.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:54.607 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:45:54.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:54.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:54.607 | INFO | Uploaded 36GiB to GPU in 2.06024s (17.4737GiB/s) | 20260506 12:45:54.607 | INFO | Downloaded 34GiB to GPU in 1.9525s (17.4136GiB/s) | 20260506 12:45:59.747 | INFO | Analyzing chunk frames = [1460736,1461248) | 20260506 12:45:59.747 | INFO | MemInfo: { | 20260506 12:45:59.747 | INFO | "gpu" : " 17 GB", | 20260506 12:45:59.747 | INFO | "huge" : "", | 20260506 12:45:59.747 | INFO | "huge_shared" : " 27 GB", | 20260506 12:45:59.747 | INFO | "malloc" : " 14 GB", | 20260506 12:45:59.747 | INFO | "pinned" : " 29 GB" | 20260506 12:45:59.747 | INFO | } | 20260506 12:45:59.747 | INFO | Perf report for the last 5.14012 seconds: | 20260506 12:45:59.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:45:59.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:59.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:59.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:59.747 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:45:59.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:59.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:59.747 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 12:45:59.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:59.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:45:59.747 | INFO | Uploaded 36GiB to GPU in 2.11324s (17.0354GiB/s) | 20260506 12:45:59.747 | INFO | Downloaded 34GiB to GPU in 1.97733s (17.1949GiB/s) | 20260506 12:46:04.887 | INFO | Analyzing chunk frames = [1461248,1461760) | 20260506 12:46:04.887 | INFO | MemInfo: { | 20260506 12:46:04.887 | INFO | "gpu" : " 17 GB", | 20260506 12:46:04.887 | INFO | "huge" : "", | 20260506 12:46:04.887 | INFO | "huge_shared" : " 27 GB", | 20260506 12:46:04.887 | INFO | "malloc" : " 14 GB", | 20260506 12:46:04.887 | INFO | "pinned" : " 29 GB" | 20260506 12:46:04.887 | INFO | } | 20260506 12:46:04.887 | INFO | Perf report for the last 5.13966 seconds: | 20260506 12:46:04.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:46:04.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:04.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:04.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:04.887 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:46:04.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:04.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:04.887 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 12:46:04.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:04.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:04.887 | INFO | Uploaded 36GiB to GPU in 2.00842s (17.9245GiB/s) | 20260506 12:46:04.887 | INFO | Downloaded 34GiB to GPU in 2.01207s (16.898GiB/s) | 20260506 12:46:10.025 | INFO | Analyzing chunk frames = [1461760,1462272) | 20260506 12:46:10.025 | INFO | MemInfo: { | 20260506 12:46:10.025 | INFO | "gpu" : " 17 GB", | 20260506 12:46:10.025 | INFO | "huge" : "", | 20260506 12:46:10.025 | INFO | "huge_shared" : " 27 GB", | 20260506 12:46:10.025 | INFO | "malloc" : " 14 GB", | 20260506 12:46:10.025 | INFO | "pinned" : " 29 GB" | 20260506 12:46:10.025 | INFO | } | 20260506 12:46:10.025 | INFO | Perf report for the last 5.13831 seconds: | 20260506 12:46:10.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:46:10.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:10.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:10.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:10.025 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:46:10.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:10.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:10.025 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:46:10.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:10.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:10.025 | INFO | Uploaded 36GiB to GPU in 2.06885s (17.4009GiB/s) | 20260506 12:46:10.025 | INFO | Downloaded 34GiB to GPU in 1.96347s (17.3163GiB/s) | 20260506 12:46:10.156 | INFO | Starting a round of Baz IO | 20260506 12:46:13.325 | INFO | Populating RTMetrics report | 20260506 12:46:13.325 | INFO | RTMetrics report populated | 20260506 12:46:13.325 | INFO | Opening tmp RTMetrics json file | 20260506 12:46:13.326 | INFO | Serializing RTMetrics to json | 20260506 12:46:13.327 | INFO | Serialization done, swapping tmp file now... | 20260506 12:46:13.327 | INFO | RTMetrics done | 20260506 12:46:13.431 | INFO | Finished a round of Baz IO: Wrote 2.099 GiB in 3.274 seconds | 20260506 12:46:13.431 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.92% / 0.02% / 59.06% | 20260506 12:46:13.431 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.69219% / 3.57628e-05% / 3.97364e-05% | 20260506 12:46:13.431 | INFO | Baz IO overall: | 20260506 12:46:13.431 | INFO | BytesWritten : 732043761617 | 20260506 12:46:13.431 | INFO | NumEvents : 353881132621 | 20260506 12:46:15.164 | INFO | Analyzing chunk frames = [1462272,1462784) | 20260506 12:46:15.164 | INFO | MemInfo: { | 20260506 12:46:15.164 | INFO | "gpu" : " 17 GB", | 20260506 12:46:15.164 | INFO | "huge" : "", | 20260506 12:46:15.164 | INFO | "huge_shared" : " 27 GB", | 20260506 12:46:15.164 | INFO | "malloc" : " 14 GB", | 20260506 12:46:15.164 | INFO | "pinned" : " 29 GB" | 20260506 12:46:15.164 | INFO | } | 20260506 12:46:15.164 | INFO | Perf report for the last 5.1393 seconds: | 20260506 12:46:15.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:46:15.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:15.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:15.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:15.164 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260506 12:46:15.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:15.164 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:46:15.164 | INFO | PRE_HQ : 61.3%, 1.9, 61.2ms, 0 | 20260506 12:46:15.164 | INFO | BAZWRITER : 63.0%, 3.7, 118.3ms, 0 | 20260506 12:46:15.164 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:46:15.165 | INFO | Uploaded 36GiB to GPU in 2.10033s (17.1401GiB/s) | 20260506 12:46:15.165 | INFO | Downloaded 38GiB to GPU in 2.22195s (17.1021GiB/s) | 20260506 12:46:20.304 | INFO | Analyzing chunk frames = [1462784,1463296) | 20260506 12:46:20.304 | INFO | MemInfo: { | 20260506 12:46:20.304 | INFO | "gpu" : " 17 GB", | 20260506 12:46:20.304 | INFO | "huge" : "", | 20260506 12:46:20.304 | INFO | "huge_shared" : " 27 GB", | 20260506 12:46:20.304 | INFO | "malloc" : " 14 GB", | 20260506 12:46:20.304 | INFO | "pinned" : " 29 GB" | 20260506 12:46:20.304 | INFO | } | 20260506 12:46:20.304 | INFO | Perf report for the last 5.13966 seconds: | 20260506 12:46:20.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:46:20.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:20.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:20.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:20.304 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:46:20.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:20.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:20.304 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:46:20.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:20.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:20.304 | INFO | Uploaded 36GiB to GPU in 2.02192s (17.8049GiB/s) | 20260506 12:46:20.304 | INFO | Downloaded 34GiB to GPU in 1.92429s (17.6689GiB/s) | 20260506 12:46:25.443 | INFO | Analyzing chunk frames = [1463296,1463808) | 20260506 12:46:25.443 | INFO | MemInfo: { | 20260506 12:46:25.443 | INFO | "gpu" : " 17 GB", | 20260506 12:46:25.443 | INFO | "huge" : "", | 20260506 12:46:25.443 | INFO | "huge_shared" : " 27 GB", | 20260506 12:46:25.443 | INFO | "malloc" : " 14 GB", | 20260506 12:46:25.443 | INFO | "pinned" : " 29 GB" | 20260506 12:46:25.443 | INFO | } | 20260506 12:46:25.443 | INFO | Perf report for the last 5.13922 seconds: | 20260506 12:46:25.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:46:25.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:25.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:25.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:25.443 | INFO | ANALYSIS : 61.6%, 3.0, 95.3ms, 0 | 20260506 12:46:25.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:25.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:25.443 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:46:25.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:25.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:25.443 | INFO | Uploaded 36GiB to GPU in 2.01136s (17.8983GiB/s) | 20260506 12:46:25.444 | INFO | Downloaded 34GiB to GPU in 1.93805s (17.5434GiB/s) | 20260506 12:46:30.582 | INFO | Analyzing chunk frames = [1463808,1464320) | 20260506 12:46:30.582 | INFO | MemInfo: { | 20260506 12:46:30.582 | INFO | "gpu" : " 17 GB", | 20260506 12:46:30.582 | INFO | "huge" : "", | 20260506 12:46:30.582 | INFO | "huge_shared" : " 27 GB", | 20260506 12:46:30.582 | INFO | "malloc" : " 14 GB", | 20260506 12:46:30.582 | INFO | "pinned" : " 29 GB" | 20260506 12:46:30.582 | INFO | } | 20260506 12:46:30.582 | INFO | Perf report for the last 5.1389 seconds: | 20260506 12:46:30.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:46:30.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:30.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:30.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:30.582 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 12:46:30.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:30.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:30.582 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:46:30.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:30.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:30.582 | INFO | Uploaded 36GiB to GPU in 2.01081s (17.9032GiB/s) | 20260506 12:46:30.582 | INFO | Downloaded 34GiB to GPU in 1.9351s (17.5701GiB/s) | 20260506 12:46:35.721 | INFO | Analyzing chunk frames = [1464320,1464832) | 20260506 12:46:35.721 | INFO | MemInfo: { | 20260506 12:46:35.721 | INFO | "gpu" : " 17 GB", | 20260506 12:46:35.722 | INFO | "huge" : "", | 20260506 12:46:35.722 | INFO | "huge_shared" : " 27 GB", | 20260506 12:46:35.722 | INFO | "malloc" : " 14 GB", | 20260506 12:46:35.722 | INFO | "pinned" : " 29 GB" | 20260506 12:46:35.722 | INFO | } | 20260506 12:46:35.722 | INFO | Perf report for the last 5.13948 seconds: | 20260506 12:46:35.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:46:35.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:35.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:35.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:35.722 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 12:46:35.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:35.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:35.722 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 12:46:35.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:35.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:35.722 | INFO | Uploaded 36GiB to GPU in 2.01721s (17.8465GiB/s) | 20260506 12:46:35.722 | INFO | Downloaded 34GiB to GPU in 1.94919s (17.4432GiB/s) | 20260506 12:46:40.861 | INFO | Analyzing chunk frames = [1464832,1465344) | 20260506 12:46:40.861 | INFO | MemInfo: { | 20260506 12:46:40.861 | INFO | "gpu" : " 17 GB", | 20260506 12:46:40.861 | INFO | "huge" : "", | 20260506 12:46:40.861 | INFO | "huge_shared" : " 27 GB", | 20260506 12:46:40.861 | INFO | "malloc" : " 14 GB", | 20260506 12:46:40.861 | INFO | "pinned" : " 29 GB" | 20260506 12:46:40.861 | INFO | } | 20260506 12:46:40.862 | INFO | Perf report for the last 5.13987 seconds: | 20260506 12:46:40.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:46:40.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:40.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:40.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:40.862 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 12:46:40.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:40.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:40.862 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:46:40.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:40.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:40.862 | INFO | Uploaded 36GiB to GPU in 2.04534s (17.601GiB/s) | 20260506 12:46:40.862 | INFO | Downloaded 34GiB to GPU in 1.93197s (17.5986GiB/s) | 20260506 12:46:46.001 | INFO | Analyzing chunk frames = [1465344,1465856) | 20260506 12:46:46.001 | INFO | MemInfo: { | 20260506 12:46:46.001 | INFO | "gpu" : " 17 GB", | 20260506 12:46:46.001 | INFO | "huge" : "", | 20260506 12:46:46.001 | INFO | "huge_shared" : " 27 GB", | 20260506 12:46:46.001 | INFO | "malloc" : " 14 GB", | 20260506 12:46:46.001 | INFO | "pinned" : " 29 GB" | 20260506 12:46:46.001 | INFO | } | 20260506 12:46:46.002 | INFO | Perf report for the last 5.13995 seconds: | 20260506 12:46:46.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:46:46.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:46.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:46.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:46.002 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:46:46.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:46.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:46.002 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:46:46.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:46.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:46.002 | INFO | Uploaded 36GiB to GPU in 2.01458s (17.8698GiB/s) | 20260506 12:46:46.002 | INFO | Downloaded 34GiB to GPU in 2.02787s (16.7664GiB/s) | 20260506 12:46:51.140 | INFO | Analyzing chunk frames = [1465856,1466368) | 20260506 12:46:51.140 | INFO | MemInfo: { | 20260506 12:46:51.140 | INFO | "gpu" : " 17 GB", | 20260506 12:46:51.140 | INFO | "huge" : "", | 20260506 12:46:51.140 | INFO | "huge_shared" : " 27 GB", | 20260506 12:46:51.140 | INFO | "malloc" : " 14 GB", | 20260506 12:46:51.140 | INFO | "pinned" : " 29 GB" | 20260506 12:46:51.140 | INFO | } | 20260506 12:46:51.140 | INFO | Perf report for the last 5.13823 seconds: | 20260506 12:46:51.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:46:51.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:51.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:51.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:51.140 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:46:51.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:51.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:51.140 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:46:51.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:51.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:51.140 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:46:51.140 | INFO | Total mem usage: 16.8 GiB | 20260506 12:46:51.140 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:46:51.140 | INFO | Uploaded 36GiB to GPU in 2.04701s (17.5866GiB/s) | 20260506 12:46:51.140 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.7%) | 20260506 12:46:51.140 | INFO | Downloaded 34GiB to GPU in 1.97046s (17.2549GiB/s) | 20260506 12:46:51.140 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:46:51.140 | INFO | High Water Breakdown: | 20260506 12:46:51.140 | INFO | 59.7% : 8.59 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:46:51.140 | INFO | 40.2% : 5.79 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:46:51.140 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:46:51.140 | INFO | ------------------------------------------ | 20260506 12:46:51.140 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:46:51.140 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:46:51.140 | INFO | Total mem usage: 26.5 GiB | 20260506 12:46:51.140 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:46:51.140 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:46:51.140 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:46:51.140 | INFO | High Water Breakdown: | 20260506 12:46:51.140 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:46:51.140 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:46:51.140 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:46:51.140 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:46:51.140 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:46:51.140 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:46:51.140 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:46:51.140 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:46:51.140 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:46:51.140 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:46:51.140 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:46:51.140 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:46:51.140 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:46:51.140 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:46:51.140 | INFO | ------------------------------------------ | 20260506 12:46:51.140 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:46:51.140 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:46:51.140 | INFO | Total mem usage: 25.6 GiB | 20260506 12:46:51.140 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:46:51.140 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:46:51.140 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:46:51.140 | INFO | High Water Breakdown: | 20260506 12:46:51.140 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:46:51.140 | INFO | ------------------------------------------ | 20260506 12:46:51.140 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:46:51.140 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:46:51.140 | INFO | Total mem usage: 15.7 GiB | 20260506 12:46:51.140 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:46:51.140 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:46:51.140 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 12:46:51.140 | INFO | High Water Breakdown: | 20260506 12:46:51.140 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:46:51.140 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:46:51.140 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:46:51.140 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:46:51.140 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:46:51.140 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:46:51.140 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:46:51.140 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:46:51.140 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:46:51.141 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:46:51.141 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:46:51.141 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:46:51.141 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:46:51.141 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:46:51.141 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:46:51.141 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:46:51.141 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:46:51.141 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:46:51.141 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:46:51.141 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:46:51.141 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:46:51.141 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:46:51.141 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:46:51.141 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:46:51.141 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:46:51.141 | INFO | ------------------------------------------ | 20260506 12:46:51.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:46:51.141 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:46:51.141 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:46:51.141 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:46:51.141 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.62e+01, 1.37e+01 / 2.61e+00 | 20260506 12:46:51.141 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.01e-01, 9.67e-03 / 6.61e-03 | 20260506 12:46:51.141 | INFO | PerfReport: 11.56%, Baseline -- 3.67e+00 / 4.38e+00, 5.77e-01 / 3.78e-01 | 20260506 12:46:51.141 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.47e+00, 1.91e-01 / 1.40e-01 | 20260506 12:46:51.141 | INFO | PerfReport: 15.82%, DME -- 4.02e+01 / 4.46e+01, 4.13e+00 / 2.86e+00 | 20260506 12:46:51.141 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.49e+00 / 1.62e+00 | 20260506 12:46:51.141 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.39e+00 / 3.03e+00, 4.14e-01 / 2.34e-01 | 20260506 12:46:51.141 | INFO | PerfReport: 3.71%, Metrics -- 1.18e+00 / 1.53e+00, 2.95e-01 / 1.43e-01 | 20260506 12:46:51.141 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.16e-01 / 4.75e-01, 6.54e-02 / 3.17e-02 | 20260506 12:46:51.141 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:46:51.141 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:46:51.141 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:46:51.141 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 12:46:51.141 | INFO | PerfReport: 21.50%, Upload -- 2.12e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260506 12:46:51.141 | INFO | PerfReport: 20.98%, Download -- 2.07e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 12:46:51.141 | INFO | PerfReport: 32.16%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 12:46:51.141 | INFO | PerfReport: 1.79%, LockUpload -- 1.77e+00 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260506 12:46:51.141 | INFO | PerfReport: 1.01%, LockDownload -- 9.94e-01 / 0.00e+00, 2.63e+00 / 0.00e+00 | 20260506 12:46:51.141 | INFO | PerfReport: 22.55%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:46:51.141 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260506 12:46:51.270 | INFO | Starting a round of Baz IO | 20260506 12:46:54.444 | INFO | Populating RTMetrics report | 20260506 12:46:54.444 | INFO | RTMetrics report populated | 20260506 12:46:54.444 | INFO | Opening tmp RTMetrics json file | 20260506 12:46:54.445 | INFO | Serializing RTMetrics to json | 20260506 12:46:54.446 | INFO | Serialization done, swapping tmp file now... | 20260506 12:46:54.446 | INFO | RTMetrics done | 20260506 12:46:54.541 | INFO | Finished a round of Baz IO: Wrote 2.1 GiB in 3.271 seconds | 20260506 12:46:54.541 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.89% / 0.02% / 59.08% | 20260506 12:46:54.541 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.68488% / 4.76837e-05% / 5.96046e-05% | 20260506 12:46:54.541 | INFO | Baz IO overall: | 20260506 12:46:54.541 | INFO | BytesWritten : 734298517344 | 20260506 12:46:54.541 | INFO | NumEvents : 354975514053 | 20260506 12:46:56.279 | INFO | Analyzing chunk frames = [1466368,1466880) | 20260506 12:46:56.279 | INFO | MemInfo: { | 20260506 12:46:56.279 | INFO | "gpu" : " 17 GB", | 20260506 12:46:56.279 | INFO | "huge" : "", | 20260506 12:46:56.279 | INFO | "huge_shared" : " 27 GB", | 20260506 12:46:56.279 | INFO | "malloc" : " 14 GB", | 20260506 12:46:56.279 | INFO | "pinned" : " 29 GB" | 20260506 12:46:56.279 | INFO | } | 20260506 12:46:56.279 | INFO | Perf report for the last 5.13912 seconds: | 20260506 12:46:56.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:46:56.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:56.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:56.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:56.279 | INFO | ANALYSIS : 63.7%, 3.0, 98.9ms, 0 | 20260506 12:46:56.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:46:56.279 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:46:56.279 | INFO | PRE_HQ : 61.6%, 1.9, 61.3ms, 0 | 20260506 12:46:56.279 | INFO | BAZWRITER : 63.0%, 3.7, 119.2ms, 0 | 20260506 12:46:56.279 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:46:56.279 | INFO | Uploaded 36GiB to GPU in 2.06526s (17.4312GiB/s) | 20260506 12:46:56.279 | INFO | Downloaded 38GiB to GPU in 2.21881s (17.1263GiB/s) | 20260506 12:47:01.419 | INFO | Analyzing chunk frames = [1466880,1467392) | 20260506 12:47:01.419 | INFO | MemInfo: { | 20260506 12:47:01.419 | INFO | "gpu" : " 17 GB", | 20260506 12:47:01.419 | INFO | "huge" : "", | 20260506 12:47:01.419 | INFO | "huge_shared" : " 27 GB", | 20260506 12:47:01.419 | INFO | "malloc" : " 14 GB", | 20260506 12:47:01.419 | INFO | "pinned" : " 29 GB" | 20260506 12:47:01.419 | INFO | } | 20260506 12:47:01.419 | INFO | Perf report for the last 5.14015 seconds: | 20260506 12:47:01.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:47:01.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:01.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:01.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:01.419 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:47:01.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:01.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:01.419 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 12:47:01.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:01.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:01.419 | INFO | Uploaded 36GiB to GPU in 2.06s (17.4757GiB/s) | 20260506 12:47:01.419 | INFO | Downloaded 34GiB to GPU in 1.93382s (17.5818GiB/s) | 20260506 12:47:06.557 | INFO | Analyzing chunk frames = [1467392,1467904) | 20260506 12:47:06.557 | INFO | MemInfo: { | 20260506 12:47:06.557 | INFO | "gpu" : " 17 GB", | 20260506 12:47:06.558 | INFO | "huge" : "", | 20260506 12:47:06.558 | INFO | "huge_shared" : " 27 GB", | 20260506 12:47:06.558 | INFO | "malloc" : " 14 GB", | 20260506 12:47:06.558 | INFO | "pinned" : " 29 GB" | 20260506 12:47:06.558 | INFO | } | 20260506 12:47:06.558 | INFO | Perf report for the last 5.13861 seconds: | 20260506 12:47:06.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:47:06.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:06.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:06.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:06.558 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260506 12:47:06.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:06.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:06.558 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:47:06.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:06.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:06.558 | INFO | Uploaded 36GiB to GPU in 2.01963s (17.8251GiB/s) | 20260506 12:47:06.558 | INFO | Downloaded 34GiB to GPU in 1.9395s (17.5303GiB/s) | 20260506 12:47:11.697 | INFO | Analyzing chunk frames = [1467904,1468416) | 20260506 12:47:11.697 | INFO | MemInfo: { | 20260506 12:47:11.697 | INFO | "gpu" : " 17 GB", | 20260506 12:47:11.697 | INFO | "huge" : "", | 20260506 12:47:11.697 | INFO | "huge_shared" : " 27 GB", | 20260506 12:47:11.697 | INFO | "malloc" : " 14 GB", | 20260506 12:47:11.697 | INFO | "pinned" : " 29 GB" | 20260506 12:47:11.697 | INFO | } | 20260506 12:47:11.697 | INFO | Perf report for the last 5.13935 seconds: | 20260506 12:47:11.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:47:11.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:11.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:11.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:11.697 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 12:47:11.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:11.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:11.697 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 12:47:11.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:11.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:11.697 | INFO | Uploaded 36GiB to GPU in 2.01199s (17.8928GiB/s) | 20260506 12:47:11.697 | INFO | Downloaded 34GiB to GPU in 1.94116s (17.5153GiB/s) | 20260506 12:47:16.836 | INFO | Analyzing chunk frames = [1468416,1468928) | 20260506 12:47:16.836 | INFO | MemInfo: { | 20260506 12:47:16.836 | INFO | "gpu" : " 17 GB", | 20260506 12:47:16.836 | INFO | "huge" : "", | 20260506 12:47:16.836 | INFO | "huge_shared" : " 27 GB", | 20260506 12:47:16.836 | INFO | "malloc" : " 14 GB", | 20260506 12:47:16.836 | INFO | "pinned" : " 29 GB" | 20260506 12:47:16.836 | INFO | } | 20260506 12:47:16.836 | INFO | Perf report for the last 5.13926 seconds: | 20260506 12:47:16.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:47:16.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:16.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:16.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:16.836 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 12:47:16.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:16.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:16.836 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:47:16.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:16.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:16.836 | INFO | Uploaded 36GiB to GPU in 1.97267s (18.2494GiB/s) | 20260506 12:47:16.837 | INFO | Downloaded 34GiB to GPU in 1.94571s (17.4743GiB/s) | 20260506 12:47:21.976 | INFO | Analyzing chunk frames = [1468928,1469440) | 20260506 12:47:21.976 | INFO | MemInfo: { | 20260506 12:47:21.976 | INFO | "gpu" : " 17 GB", | 20260506 12:47:21.976 | INFO | "huge" : "", | 20260506 12:47:21.976 | INFO | "huge_shared" : " 27 GB", | 20260506 12:47:21.976 | INFO | "malloc" : " 14 GB", | 20260506 12:47:21.976 | INFO | "pinned" : " 29 GB" | 20260506 12:47:21.976 | INFO | } | 20260506 12:47:21.976 | INFO | Perf report for the last 5.13968 seconds: | 20260506 12:47:21.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:47:21.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:21.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:21.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:21.976 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:47:21.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:21.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:21.976 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:47:21.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:21.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:21.976 | INFO | Uploaded 36GiB to GPU in 2.09334s (17.1974GiB/s) | 20260506 12:47:21.976 | INFO | Downloaded 34GiB to GPU in 1.96148s (17.3338GiB/s) | 20260506 12:47:27.116 | INFO | Analyzing chunk frames = [1469440,1469952) | 20260506 12:47:27.116 | INFO | MemInfo: { | 20260506 12:47:27.116 | INFO | "gpu" : " 17 GB", | 20260506 12:47:27.116 | INFO | "huge" : "", | 20260506 12:47:27.116 | INFO | "huge_shared" : " 27 GB", | 20260506 12:47:27.116 | INFO | "malloc" : " 14 GB", | 20260506 12:47:27.116 | INFO | "pinned" : " 29 GB" | 20260506 12:47:27.116 | INFO | } | 20260506 12:47:27.116 | INFO | Perf report for the last 5.14028 seconds: | 20260506 12:47:27.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:47:27.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:27.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:27.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:27.116 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 12:47:27.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:27.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:27.116 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 12:47:27.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:27.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:27.116 | INFO | Uploaded 36GiB to GPU in 1.97692s (18.2101GiB/s) | 20260506 12:47:27.117 | INFO | Downloaded 34GiB to GPU in 2.02528s (16.7878GiB/s) | 20260506 12:47:32.254 | INFO | Analyzing chunk frames = [1469952,1470464) | 20260506 12:47:32.254 | INFO | MemInfo: { | 20260506 12:47:32.254 | INFO | "gpu" : " 17 GB", | 20260506 12:47:32.254 | INFO | "huge" : "", | 20260506 12:47:32.254 | INFO | "huge_shared" : " 27 GB", | 20260506 12:47:32.254 | INFO | "malloc" : " 14 GB", | 20260506 12:47:32.254 | INFO | "pinned" : " 29 GB" | 20260506 12:47:32.254 | INFO | } | 20260506 12:47:32.254 | INFO | Perf report for the last 5.13808 seconds: | 20260506 12:47:32.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:47:32.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:32.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:32.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:32.254 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 12:47:32.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:32.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:32.254 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:47:32.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:32.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:32.255 | INFO | Uploaded 36GiB to GPU in 2.03609s (17.681GiB/s) | 20260506 12:47:32.255 | INFO | Downloaded 34GiB to GPU in 1.97472s (17.2176GiB/s) | 20260506 12:47:32.386 | INFO | Starting a round of Baz IO | 20260506 12:47:35.541 | INFO | Populating RTMetrics report | 20260506 12:47:35.542 | INFO | RTMetrics report populated | 20260506 12:47:35.542 | INFO | Opening tmp RTMetrics json file | 20260506 12:47:35.542 | INFO | Serializing RTMetrics to json | 20260506 12:47:35.543 | INFO | Serialization done, swapping tmp file now... | 20260506 12:47:35.544 | INFO | RTMetrics done | 20260506 12:47:35.640 | INFO | Finished a round of Baz IO: Wrote 2.098 GiB in 3.254 seconds | 20260506 12:47:35.640 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.87% / 0.02% / 59.11% | 20260506 12:47:35.640 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.66869% / 5.16574e-05% / 5.16574e-05% | 20260506 12:47:35.640 | INFO | Baz IO overall: | 20260506 12:47:35.640 | INFO | BytesWritten : 736551134588 | 20260506 12:47:35.640 | INFO | NumEvents : 356068199875 | 20260506 12:47:37.393 | INFO | Analyzing chunk frames = [1470464,1470976) | 20260506 12:47:37.393 | INFO | MemInfo: { | 20260506 12:47:37.393 | INFO | "gpu" : " 17 GB", | 20260506 12:47:37.393 | INFO | "huge" : "", | 20260506 12:47:37.393 | INFO | "huge_shared" : " 27 GB", | 20260506 12:47:37.393 | INFO | "malloc" : " 14 GB", | 20260506 12:47:37.393 | INFO | "pinned" : " 29 GB" | 20260506 12:47:37.393 | INFO | } | 20260506 12:47:37.394 | INFO | Perf report for the last 5.13915 seconds: | 20260506 12:47:37.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:47:37.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:37.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:37.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:37.394 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260506 12:47:37.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:37.394 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:47:37.394 | INFO | PRE_HQ : 61.3%, 1.9, 61.2ms, 0 | 20260506 12:47:37.394 | INFO | BAZWRITER : 62.7%, 3.7, 120.0ms, 0 | 20260506 12:47:37.394 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:47:37.394 | INFO | Uploaded 36GiB to GPU in 2.0443s (17.61GiB/s) | 20260506 12:47:37.394 | INFO | Downloaded 38GiB to GPU in 2.22932s (17.0456GiB/s) | 20260506 12:47:42.533 | INFO | Analyzing chunk frames = [1470976,1471488) | 20260506 12:47:42.533 | INFO | MemInfo: { | 20260506 12:47:42.533 | INFO | "gpu" : " 17 GB", | 20260506 12:47:42.533 | INFO | "huge" : "", | 20260506 12:47:42.533 | INFO | "huge_shared" : " 27 GB", | 20260506 12:47:42.533 | INFO | "malloc" : " 14 GB", | 20260506 12:47:42.533 | INFO | "pinned" : " 29 GB" | 20260506 12:47:42.533 | INFO | } | 20260506 12:47:42.533 | INFO | Perf report for the last 5.13953 seconds: | 20260506 12:47:42.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:47:42.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:42.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:42.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:42.533 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 12:47:42.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:42.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:42.533 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 12:47:42.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:42.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:42.533 | INFO | Uploaded 36GiB to GPU in 1.99437s (18.0508GiB/s) | 20260506 12:47:42.533 | INFO | Downloaded 34GiB to GPU in 1.93028s (17.6141GiB/s) | 20260506 12:47:47.672 | INFO | Analyzing chunk frames = [1471488,1472000) | 20260506 12:47:47.672 | INFO | MemInfo: { | 20260506 12:47:47.672 | INFO | "gpu" : " 17 GB", | 20260506 12:47:47.672 | INFO | "huge" : "", | 20260506 12:47:47.672 | INFO | "huge_shared" : " 27 GB", | 20260506 12:47:47.672 | INFO | "malloc" : " 14 GB", | 20260506 12:47:47.672 | INFO | "pinned" : " 29 GB" | 20260506 12:47:47.672 | INFO | } | 20260506 12:47:47.673 | INFO | Perf report for the last 5.13944 seconds: | 20260506 12:47:47.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:47:47.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:47.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:47.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:47.673 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 12:47:47.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:47.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:47.673 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260506 12:47:47.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:47.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:47.673 | INFO | Uploaded 36GiB to GPU in 2.01368s (17.8777GiB/s) | 20260506 12:47:47.673 | INFO | Downloaded 34GiB to GPU in 1.9401s (17.5249GiB/s) | 20260506 12:47:52.812 | INFO | Analyzing chunk frames = [1472000,1472512) | 20260506 12:47:52.812 | INFO | MemInfo: { | 20260506 12:47:52.812 | INFO | "gpu" : " 17 GB", | 20260506 12:47:52.812 | INFO | "huge" : "", | 20260506 12:47:52.812 | INFO | "huge_shared" : " 27 GB", | 20260506 12:47:52.812 | INFO | "malloc" : " 14 GB", | 20260506 12:47:52.812 | INFO | "pinned" : " 29 GB" | 20260506 12:47:52.812 | INFO | } | 20260506 12:47:52.812 | INFO | Perf report for the last 5.13927 seconds: | 20260506 12:47:52.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:47:52.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:52.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:52.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:52.812 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 12:47:52.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:52.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:52.812 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:47:52.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:52.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:52.812 | INFO | Uploaded 36GiB to GPU in 2.01406s (17.8743GiB/s) | 20260506 12:47:52.812 | INFO | Downloaded 34GiB to GPU in 1.95106s (17.4265GiB/s) | 20260506 12:47:57.951 | INFO | Analyzing chunk frames = [1472512,1473024) | 20260506 12:47:57.951 | INFO | MemInfo: { | 20260506 12:47:57.951 | INFO | "gpu" : " 17 GB", | 20260506 12:47:57.951 | INFO | "huge" : "", | 20260506 12:47:57.951 | INFO | "huge_shared" : " 27 GB", | 20260506 12:47:57.951 | INFO | "malloc" : " 14 GB", | 20260506 12:47:57.951 | INFO | "pinned" : " 29 GB" | 20260506 12:47:57.951 | INFO | } | 20260506 12:47:57.951 | INFO | Perf report for the last 5.1395 seconds: | 20260506 12:47:57.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:47:57.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:57.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:57.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:57.951 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 12:47:57.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:57.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:57.951 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 12:47:57.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:57.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:47:57.952 | INFO | Uploaded 36GiB to GPU in 2.03698s (17.6732GiB/s) | 20260506 12:47:57.952 | INFO | Downloaded 34GiB to GPU in 1.95651s (17.3779GiB/s) | 20260506 12:48:03.090 | INFO | Analyzing chunk frames = [1473024,1473536) | 20260506 12:48:03.090 | INFO | MemInfo: { | 20260506 12:48:03.090 | INFO | "gpu" : " 17 GB", | 20260506 12:48:03.090 | INFO | "huge" : "", | 20260506 12:48:03.090 | INFO | "huge_shared" : " 27 GB", | 20260506 12:48:03.090 | INFO | "malloc" : " 14 GB", | 20260506 12:48:03.090 | INFO | "pinned" : " 29 GB" | 20260506 12:48:03.090 | INFO | } | 20260506 12:48:03.090 | INFO | Perf report for the last 5.13913 seconds: | 20260506 12:48:03.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:48:03.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:03.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:03.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:03.090 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:48:03.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:03.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:03.091 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 12:48:03.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:03.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:03.091 | INFO | Uploaded 36GiB to GPU in 2.10439s (17.1071GiB/s) | 20260506 12:48:03.091 | INFO | Downloaded 34GiB to GPU in 1.99282s (17.0613GiB/s) | 20260506 12:48:08.230 | INFO | Analyzing chunk frames = [1473536,1474048) | 20260506 12:48:08.230 | INFO | MemInfo: { | 20260506 12:48:08.230 | INFO | "gpu" : " 17 GB", | 20260506 12:48:08.230 | INFO | "huge" : "", | 20260506 12:48:08.230 | INFO | "huge_shared" : " 27 GB", | 20260506 12:48:08.230 | INFO | "malloc" : " 14 GB", | 20260506 12:48:08.230 | INFO | "pinned" : " 29 GB" | 20260506 12:48:08.230 | INFO | } | 20260506 12:48:08.230 | INFO | Perf report for the last 5.14002 seconds: | 20260506 12:48:08.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:48:08.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:08.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:08.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:08.231 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:48:08.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:08.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:08.231 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:48:08.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:08.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:08.231 | INFO | Uploaded 36GiB to GPU in 2.02667s (17.7631GiB/s) | 20260506 12:48:08.231 | INFO | Downloaded 34GiB to GPU in 2.01887s (16.8411GiB/s) | 20260506 12:48:13.369 | INFO | Analyzing chunk frames = [1474048,1474560) | 20260506 12:48:13.369 | INFO | MemInfo: { | 20260506 12:48:13.369 | INFO | "gpu" : " 17 GB", | 20260506 12:48:13.369 | INFO | "huge" : "", | 20260506 12:48:13.369 | INFO | "huge_shared" : " 27 GB", | 20260506 12:48:13.369 | INFO | "malloc" : " 14 GB", | 20260506 12:48:13.369 | INFO | "pinned" : " 29 GB" | 20260506 12:48:13.369 | INFO | } | 20260506 12:48:13.369 | INFO | Perf report for the last 5.13861 seconds: | 20260506 12:48:13.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:48:13.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:13.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:13.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:13.369 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:48:13.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:13.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:13.369 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:48:13.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:13.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:13.369 | INFO | Uploaded 36GiB to GPU in 2.068s (17.4082GiB/s) | 20260506 12:48:13.369 | INFO | Downloaded 34GiB to GPU in 1.99363s (17.0543GiB/s) | 20260506 12:48:13.369 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:48:13.369 | INFO | Total mem usage: 16.8 GiB | 20260506 12:48:13.369 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:48:13.369 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.6%) | 20260506 12:48:13.369 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:48:13.369 | INFO | High Water Breakdown: | 20260506 12:48:13.369 | INFO | 59.6% : 8.59 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:48:13.369 | INFO | 40.2% : 5.8 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:48:13.369 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:48:13.369 | INFO | ------------------------------------------ | 20260506 12:48:13.370 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:48:13.370 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:48:13.370 | INFO | Total mem usage: 26.5 GiB | 20260506 12:48:13.370 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:48:13.370 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:48:13.370 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:48:13.370 | INFO | High Water Breakdown: | 20260506 12:48:13.370 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:48:13.370 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:48:13.370 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:48:13.370 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:48:13.370 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:48:13.370 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:48:13.370 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:48:13.370 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:48:13.370 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:48:13.370 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:48:13.370 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:48:13.370 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:48:13.370 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:48:13.370 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:48:13.370 | INFO | ------------------------------------------ | 20260506 12:48:13.370 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:48:13.370 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:48:13.370 | INFO | Total mem usage: 25.6 GiB | 20260506 12:48:13.370 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:48:13.370 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:48:13.370 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:48:13.370 | INFO | High Water Breakdown: | 20260506 12:48:13.370 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:48:13.370 | INFO | ------------------------------------------ | 20260506 12:48:13.370 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:48:13.370 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:48:13.370 | INFO | Total mem usage: 15.7 GiB | 20260506 12:48:13.370 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:48:13.370 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:48:13.370 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:48:13.370 | INFO | High Water Breakdown: | 20260506 12:48:13.370 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:48:13.370 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:48:13.370 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:48:13.370 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:48:13.370 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:48:13.370 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:48:13.370 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:48:13.370 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:48:13.370 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:48:13.370 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:48:13.370 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:48:13.370 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:48:13.370 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:48:13.370 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:48:13.370 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:48:13.370 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:48:13.370 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:48:13.370 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:48:13.370 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:48:13.370 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:48:13.370 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:48:13.370 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:48:13.370 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:48:13.370 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:48:13.370 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:48:13.370 | INFO | ------------------------------------------ | 20260506 12:48:13.370 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:48:13.370 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:48:13.370 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:48:13.370 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:48:13.370 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.60e+01, 1.35e+01 / 2.25e+00 | 20260506 12:48:13.370 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.02e-01, 1.09e-02 / 7.57e-03 | 20260506 12:48:13.370 | INFO | PerfReport: 11.55%, Baseline -- 3.66e+00 / 4.41e+00, 5.74e-01 / 3.66e-01 | 20260506 12:48:13.370 | INFO | PerfReport: 3.23%, Histogram -- 1.02e+00 / 1.48e+00, 1.90e-01 / 1.43e-01 | 20260506 12:48:13.370 | INFO | PerfReport: 15.81%, DME -- 4.01e+01 / 4.43e+01, 3.99e+00 / 2.65e+00 | 20260506 12:48:13.370 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.38e+00 / 1.67e+00 | 20260506 12:48:13.370 | INFO | PerfReport: 7.60%, PulseAccumulating -- 2.41e+00 / 3.05e+00, 4.30e-01 / 2.09e-01 | 20260506 12:48:13.370 | INFO | PerfReport: 3.74%, Metrics -- 1.18e+00 / 1.60e+00, 3.05e-01 / 1.48e-01 | 20260506 12:48:13.370 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.16e-01 / 4.76e-01, 6.53e-02 / 3.25e-02 | 20260506 12:48:13.370 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:48:13.370 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:48:13.370 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:48:13.370 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 12:48:13.370 | INFO | PerfReport: 21.46%, Upload -- 2.12e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 12:48:13.370 | INFO | PerfReport: 21.08%, Download -- 2.08e+01 / 0.00e+00, 7.54e+00 / 0.00e+00 | 20260506 12:48:13.370 | INFO | PerfReport: 32.10%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 12:48:13.371 | INFO | PerfReport: 1.73%, LockUpload -- 1.71e+00 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260506 12:48:13.371 | INFO | PerfReport: 1.10%, LockDownload -- 1.08e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260506 12:48:13.371 | INFO | PerfReport: 22.51%, LockCompute -- 2.22e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:48:13.371 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260506 12:48:13.500 | INFO | Starting a round of Baz IO | 20260506 12:48:16.662 | INFO | Populating RTMetrics report | 20260506 12:48:16.662 | INFO | RTMetrics report populated | 20260506 12:48:16.662 | INFO | Opening tmp RTMetrics json file | 20260506 12:48:16.663 | INFO | Serializing RTMetrics to json | 20260506 12:48:16.664 | INFO | Serialization done, swapping tmp file now... | 20260506 12:48:16.665 | INFO | RTMetrics done | 20260506 12:48:16.763 | INFO | Finished a round of Baz IO: Wrote 2.1 GiB in 3.262 seconds | 20260506 12:48:16.763 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.85% / 0.02% / 59.13% | 20260506 12:48:16.763 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.66495% / 3.97364e-05% / 7.15256e-05% | 20260506 12:48:16.763 | INFO | Baz IO overall: | 20260506 12:48:16.763 | INFO | BytesWritten : 738806152064 | 20260506 12:48:16.763 | INFO | NumEvents : 357162783514 | 20260506 12:48:18.509 | INFO | Analyzing chunk frames = [1474560,1475072) | 20260506 12:48:18.509 | INFO | MemInfo: { | 20260506 12:48:18.509 | INFO | "gpu" : " 17 GB", | 20260506 12:48:18.509 | INFO | "huge" : "", | 20260506 12:48:18.509 | INFO | "huge_shared" : " 27 GB", | 20260506 12:48:18.509 | INFO | "malloc" : " 14 GB", | 20260506 12:48:18.509 | INFO | "pinned" : " 29 GB" | 20260506 12:48:18.509 | INFO | } | 20260506 12:48:18.509 | INFO | Perf report for the last 5.13962 seconds: | 20260506 12:48:18.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:48:18.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:18.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:18.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:18.509 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260506 12:48:18.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:18.509 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:48:18.509 | INFO | PRE_HQ : 61.5%, 1.9, 61.3ms, 0 | 20260506 12:48:18.509 | INFO | BAZWRITER : 62.8%, 3.7, 120.4ms, 0 | 20260506 12:48:18.509 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:48:18.509 | INFO | Uploaded 36GiB to GPU in 2.06647s (17.421GiB/s) | 20260506 12:48:18.509 | INFO | Downloaded 38GiB to GPU in 2.22503s (17.0784GiB/s) | 20260506 12:48:23.648 | INFO | Analyzing chunk frames = [1475072,1475584) | 20260506 12:48:23.648 | INFO | MemInfo: { | 20260506 12:48:23.648 | INFO | "gpu" : " 17 GB", | 20260506 12:48:23.648 | INFO | "huge" : "", | 20260506 12:48:23.648 | INFO | "huge_shared" : " 27 GB", | 20260506 12:48:23.648 | INFO | "malloc" : " 14 GB", | 20260506 12:48:23.648 | INFO | "pinned" : " 29 GB" | 20260506 12:48:23.648 | INFO | } | 20260506 12:48:23.648 | INFO | Perf report for the last 5.13928 seconds: | 20260506 12:48:23.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:48:23.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:23.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:23.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:23.648 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:48:23.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:23.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:23.648 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:48:23.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:23.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:23.648 | INFO | Uploaded 36GiB to GPU in 2.01432s (17.872GiB/s) | 20260506 12:48:23.648 | INFO | Downloaded 34GiB to GPU in 1.92822s (17.6328GiB/s) | 20260506 12:48:28.787 | INFO | Analyzing chunk frames = [1475584,1476096) | 20260506 12:48:28.787 | INFO | MemInfo: { | 20260506 12:48:28.787 | INFO | "gpu" : " 17 GB", | 20260506 12:48:28.787 | INFO | "huge" : "", | 20260506 12:48:28.787 | INFO | "huge_shared" : " 27 GB", | 20260506 12:48:28.787 | INFO | "malloc" : " 14 GB", | 20260506 12:48:28.787 | INFO | "pinned" : " 29 GB" | 20260506 12:48:28.787 | INFO | } | 20260506 12:48:28.787 | INFO | Perf report for the last 5.13923 seconds: | 20260506 12:48:28.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:48:28.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:28.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:28.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:28.787 | INFO | ANALYSIS : 61.4%, 3.0, 95.1ms, 0 | 20260506 12:48:28.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:28.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:28.787 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:48:28.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:28.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:28.787 | INFO | Uploaded 36GiB to GPU in 1.96774s (18.2951GiB/s) | 20260506 12:48:28.787 | INFO | Downloaded 34GiB to GPU in 1.93296s (17.5896GiB/s) | 20260506 12:48:33.926 | INFO | Analyzing chunk frames = [1476096,1476608) | 20260506 12:48:33.926 | INFO | MemInfo: { | 20260506 12:48:33.926 | INFO | "gpu" : " 17 GB", | 20260506 12:48:33.926 | INFO | "huge" : "", | 20260506 12:48:33.926 | INFO | "huge_shared" : " 27 GB", | 20260506 12:48:33.927 | INFO | "malloc" : " 14 GB", | 20260506 12:48:33.927 | INFO | "pinned" : " 29 GB" | 20260506 12:48:33.927 | INFO | } | 20260506 12:48:33.927 | INFO | Perf report for the last 5.13926 seconds: | 20260506 12:48:33.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:48:33.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:33.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:33.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:33.927 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 12:48:33.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:33.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:33.927 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 12:48:33.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:33.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:33.927 | INFO | Uploaded 36GiB to GPU in 1.98597s (18.1272GiB/s) | 20260506 12:48:33.927 | INFO | Downloaded 34GiB to GPU in 1.91914s (17.7163GiB/s) | 20260506 12:48:39.066 | INFO | Analyzing chunk frames = [1476608,1477120) | 20260506 12:48:39.066 | INFO | MemInfo: { | 20260506 12:48:39.066 | INFO | "gpu" : " 17 GB", | 20260506 12:48:39.066 | INFO | "huge" : "", | 20260506 12:48:39.066 | INFO | "huge_shared" : " 27 GB", | 20260506 12:48:39.066 | INFO | "malloc" : " 14 GB", | 20260506 12:48:39.066 | INFO | "pinned" : " 29 GB" | 20260506 12:48:39.066 | INFO | } | 20260506 12:48:39.066 | INFO | Perf report for the last 5.1396 seconds: | 20260506 12:48:39.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:48:39.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:39.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:39.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:39.066 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 12:48:39.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:39.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:39.066 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:48:39.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:39.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:39.066 | INFO | Uploaded 36GiB to GPU in 2.00524s (17.953GiB/s) | 20260506 12:48:39.067 | INFO | Downloaded 34GiB to GPU in 1.95211s (17.4171GiB/s) | 20260506 12:48:44.206 | INFO | Analyzing chunk frames = [1477120,1477632) | 20260506 12:48:44.206 | INFO | MemInfo: { | 20260506 12:48:44.206 | INFO | "gpu" : " 17 GB", | 20260506 12:48:44.206 | INFO | "huge" : "", | 20260506 12:48:44.206 | INFO | "huge_shared" : " 27 GB", | 20260506 12:48:44.206 | INFO | "malloc" : " 14 GB", | 20260506 12:48:44.206 | INFO | "pinned" : " 29 GB" | 20260506 12:48:44.206 | INFO | } | 20260506 12:48:44.206 | INFO | Perf report for the last 5.13971 seconds: | 20260506 12:48:44.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:48:44.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:44.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:44.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:44.206 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 12:48:44.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:44.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:44.206 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260506 12:48:44.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:44.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:44.206 | INFO | Uploaded 36GiB to GPU in 2.00396s (17.9644GiB/s) | 20260506 12:48:44.206 | INFO | Downloaded 34GiB to GPU in 1.90582s (17.8401GiB/s) | 20260506 12:48:49.346 | INFO | Analyzing chunk frames = [1477632,1478144) | 20260506 12:48:49.346 | INFO | MemInfo: { | 20260506 12:48:49.346 | INFO | "gpu" : " 17 GB", | 20260506 12:48:49.346 | INFO | "huge" : "", | 20260506 12:48:49.346 | INFO | "huge_shared" : " 27 GB", | 20260506 12:48:49.346 | INFO | "malloc" : " 14 GB", | 20260506 12:48:49.346 | INFO | "pinned" : " 29 GB" | 20260506 12:48:49.346 | INFO | } | 20260506 12:48:49.346 | INFO | Perf report for the last 5.13997 seconds: | 20260506 12:48:49.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:48:49.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:49.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:49.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:49.346 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 12:48:49.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:49.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:49.346 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:48:49.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:49.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:49.346 | INFO | Uploaded 36GiB to GPU in 2.01859s (17.8342GiB/s) | 20260506 12:48:49.346 | INFO | Downloaded 34GiB to GPU in 2.01182s (16.9001GiB/s) | 20260506 12:48:54.484 | INFO | Analyzing chunk frames = [1478144,1478656) | 20260506 12:48:54.484 | INFO | MemInfo: { | 20260506 12:48:54.484 | INFO | "gpu" : " 17 GB", | 20260506 12:48:54.484 | INFO | "huge" : "", | 20260506 12:48:54.484 | INFO | "huge_shared" : " 27 GB", | 20260506 12:48:54.484 | INFO | "malloc" : " 14 GB", | 20260506 12:48:54.484 | INFO | "pinned" : " 29 GB" | 20260506 12:48:54.484 | INFO | } | 20260506 12:48:54.485 | INFO | Perf report for the last 5.13865 seconds: | 20260506 12:48:54.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:48:54.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:54.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:54.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:54.485 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 12:48:54.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:54.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:54.485 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 12:48:54.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:54.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:54.485 | INFO | Uploaded 36GiB to GPU in 2.06908s (17.399GiB/s) | 20260506 12:48:54.485 | INFO | Downloaded 34GiB to GPU in 1.98709s (17.1104GiB/s) | 20260506 12:48:54.615 | INFO | Starting a round of Baz IO | 20260506 12:48:57.776 | INFO | Populating RTMetrics report | 20260506 12:48:57.776 | INFO | RTMetrics report populated | 20260506 12:48:57.776 | INFO | Opening tmp RTMetrics json file | 20260506 12:48:57.777 | INFO | Serializing RTMetrics to json | 20260506 12:48:57.778 | INFO | Serialization done, swapping tmp file now... | 20260506 12:48:57.778 | INFO | RTMetrics done | 20260506 12:48:57.875 | INFO | Finished a round of Baz IO: Wrote 2.099 GiB in 3.26 seconds | 20260506 12:48:57.875 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.83% / 0.02% / 59.15% | 20260506 12:48:57.875 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.64988% / 1.58946e-05% / 3.17891e-05% | 20260506 12:48:57.876 | INFO | Baz IO overall: | 20260506 12:48:57.876 | INFO | BytesWritten : 741059674768 | 20260506 12:48:57.876 | INFO | NumEvents : 358256008618 | 20260506 12:48:59.623 | INFO | Analyzing chunk frames = [1478656,1479168) | 20260506 12:48:59.623 | INFO | MemInfo: { | 20260506 12:48:59.623 | INFO | "gpu" : " 17 GB", | 20260506 12:48:59.623 | INFO | "huge" : "", | 20260506 12:48:59.624 | INFO | "huge_shared" : " 27 GB", | 20260506 12:48:59.624 | INFO | "malloc" : " 14 GB", | 20260506 12:48:59.624 | INFO | "pinned" : " 29 GB" | 20260506 12:48:59.624 | INFO | } | 20260506 12:48:59.624 | INFO | Perf report for the last 5.13901 seconds: | 20260506 12:48:59.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:48:59.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:59.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:59.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:59.624 | INFO | ANALYSIS : 63.5%, 3.0, 98.5ms, 0 | 20260506 12:48:59.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:48:59.624 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 12:48:59.624 | INFO | PRE_HQ : 61.3%, 1.9, 61.0ms, 0 | 20260506 12:48:59.624 | INFO | BAZWRITER : 62.8%, 3.7, 119.6ms, 0 | 20260506 12:48:59.624 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:48:59.624 | INFO | Uploaded 36GiB to GPU in 2.08032s (17.305GiB/s) | 20260506 12:48:59.624 | INFO | Downloaded 38GiB to GPU in 2.22277s (17.0958GiB/s) | 20260506 12:49:04.763 | INFO | Analyzing chunk frames = [1479168,1479680) | 20260506 12:49:04.763 | INFO | MemInfo: { | 20260506 12:49:04.763 | INFO | "gpu" : " 17 GB", | 20260506 12:49:04.763 | INFO | "huge" : "", | 20260506 12:49:04.763 | INFO | "huge_shared" : " 27 GB", | 20260506 12:49:04.763 | INFO | "malloc" : " 14 GB", | 20260506 12:49:04.763 | INFO | "pinned" : " 29 GB" | 20260506 12:49:04.763 | INFO | } | 20260506 12:49:04.763 | INFO | Perf report for the last 5.13919 seconds: | 20260506 12:49:04.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:49:04.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:04.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:04.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:04.763 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:49:04.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:04.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:04.763 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:49:04.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:04.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:04.763 | INFO | Uploaded 36GiB to GPU in 2.07466s (17.3523GiB/s) | 20260506 12:49:04.763 | INFO | Downloaded 34GiB to GPU in 1.95403s (17.3999GiB/s) | 20260506 12:49:09.902 | INFO | Analyzing chunk frames = [1479680,1480192) | 20260506 12:49:09.902 | INFO | MemInfo: { | 20260506 12:49:09.902 | INFO | "gpu" : " 17 GB", | 20260506 12:49:09.902 | INFO | "huge" : "", | 20260506 12:49:09.902 | INFO | "huge_shared" : " 27 GB", | 20260506 12:49:09.902 | INFO | "malloc" : " 14 GB", | 20260506 12:49:09.902 | INFO | "pinned" : " 29 GB" | 20260506 12:49:09.902 | INFO | } | 20260506 12:49:09.902 | INFO | Perf report for the last 5.13923 seconds: | 20260506 12:49:09.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:49:09.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:09.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:09.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:09.902 | INFO | ANALYSIS : 61.9%, 3.0, 95.8ms, 0 | 20260506 12:49:09.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:09.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:09.902 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:49:09.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:09.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:09.902 | INFO | Uploaded 36GiB to GPU in 2.03494s (17.6909GiB/s) | 20260506 12:49:09.902 | INFO | Downloaded 34GiB to GPU in 1.94764s (17.4571GiB/s) | 20260506 12:49:15.041 | INFO | Analyzing chunk frames = [1480192,1480704) | 20260506 12:49:15.041 | INFO | MemInfo: { | 20260506 12:49:15.041 | INFO | "gpu" : " 17 GB", | 20260506 12:49:15.041 | INFO | "huge" : "", | 20260506 12:49:15.041 | INFO | "huge_shared" : " 27 GB", | 20260506 12:49:15.041 | INFO | "malloc" : " 14 GB", | 20260506 12:49:15.041 | INFO | "pinned" : " 29 GB" | 20260506 12:49:15.041 | INFO | } | 20260506 12:49:15.041 | INFO | Perf report for the last 5.13939 seconds: | 20260506 12:49:15.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:49:15.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:15.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:15.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:15.041 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 12:49:15.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:15.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:15.041 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:49:15.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:15.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:15.042 | INFO | Uploaded 36GiB to GPU in 1.97058s (18.2687GiB/s) | 20260506 12:49:15.042 | INFO | Downloaded 34GiB to GPU in 1.93081s (17.6092GiB/s) | 20260506 12:49:20.180 | INFO | Analyzing chunk frames = [1480704,1481216) | 20260506 12:49:20.180 | INFO | MemInfo: { | 20260506 12:49:20.180 | INFO | "gpu" : " 17 GB", | 20260506 12:49:20.181 | INFO | "huge" : "", | 20260506 12:49:20.181 | INFO | "huge_shared" : " 27 GB", | 20260506 12:49:20.181 | INFO | "malloc" : " 14 GB", | 20260506 12:49:20.181 | INFO | "pinned" : " 29 GB" | 20260506 12:49:20.181 | INFO | } | 20260506 12:49:20.181 | INFO | Perf report for the last 5.13924 seconds: | 20260506 12:49:20.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:49:20.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:20.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:20.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:20.181 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 12:49:20.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:20.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:20.181 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:49:20.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:20.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:20.181 | INFO | Uploaded 36GiB to GPU in 1.98995s (18.0909GiB/s) | 20260506 12:49:20.181 | INFO | Downloaded 34GiB to GPU in 1.94616s (17.4703GiB/s) | 20260506 12:49:25.320 | INFO | Analyzing chunk frames = [1481216,1481728) | 20260506 12:49:25.320 | INFO | MemInfo: { | 20260506 12:49:25.320 | INFO | "gpu" : " 17 GB", | 20260506 12:49:25.320 | INFO | "huge" : "", | 20260506 12:49:25.320 | INFO | "huge_shared" : " 27 GB", | 20260506 12:49:25.320 | INFO | "malloc" : " 14 GB", | 20260506 12:49:25.320 | INFO | "pinned" : " 29 GB" | 20260506 12:49:25.320 | INFO | } | 20260506 12:49:25.320 | INFO | Perf report for the last 5.13925 seconds: | 20260506 12:49:25.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:49:25.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:25.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:25.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:25.320 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 12:49:25.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:25.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:25.320 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260506 12:49:25.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:25.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:25.320 | INFO | Uploaded 36GiB to GPU in 2.07418s (17.3562GiB/s) | 20260506 12:49:25.320 | INFO | Downloaded 34GiB to GPU in 1.97284s (17.234GiB/s) | 20260506 12:49:30.459 | INFO | Analyzing chunk frames = [1481728,1482240) | 20260506 12:49:30.459 | INFO | MemInfo: { | 20260506 12:49:30.459 | INFO | "gpu" : " 17 GB", | 20260506 12:49:30.459 | INFO | "huge" : "", | 20260506 12:49:30.459 | INFO | "huge_shared" : " 27 GB", | 20260506 12:49:30.459 | INFO | "malloc" : " 14 GB", | 20260506 12:49:30.459 | INFO | "pinned" : " 29 GB" | 20260506 12:49:30.459 | INFO | } | 20260506 12:49:30.459 | INFO | Perf report for the last 5.13925 seconds: | 20260506 12:49:30.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:49:30.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:30.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:30.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:30.459 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 12:49:30.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:30.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:30.459 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 12:49:30.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:30.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:30.459 | INFO | Uploaded 36GiB to GPU in 1.96991s (18.2749GiB/s) | 20260506 12:49:30.459 | INFO | Downloaded 34GiB to GPU in 2.02411s (16.7975GiB/s) | 20260506 12:49:35.599 | INFO | Analyzing chunk frames = [1482240,1482752) | 20260506 12:49:35.599 | INFO | MemInfo: { | 20260506 12:49:35.599 | INFO | "gpu" : " 17 GB", | 20260506 12:49:35.599 | INFO | "huge" : "", | 20260506 12:49:35.599 | INFO | "huge_shared" : " 27 GB", | 20260506 12:49:35.599 | INFO | "malloc" : " 14 GB", | 20260506 12:49:35.599 | INFO | "pinned" : " 29 GB" | 20260506 12:49:35.599 | INFO | } | 20260506 12:49:35.599 | INFO | Perf report for the last 5.13992 seconds: | 20260506 12:49:35.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:49:35.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:35.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:35.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:35.599 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 12:49:35.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:35.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:35.599 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 12:49:35.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:35.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:35.599 | INFO | Uploaded 36GiB to GPU in 2.06267s (17.4531GiB/s) | 20260506 12:49:35.599 | INFO | Downloaded 34GiB to GPU in 1.99501s (17.0425GiB/s) | 20260506 12:49:35.599 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:49:35.599 | INFO | Total mem usage: 16.8 GiB | 20260506 12:49:35.599 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:49:35.599 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.8%) | 20260506 12:49:35.599 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:49:35.599 | INFO | High Water Breakdown: | 20260506 12:49:35.599 | INFO | 59.7% : 8.59 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:49:35.599 | INFO | 40.2% : 5.78 GiB ( 86.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:49:35.599 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:49:35.599 | INFO | ------------------------------------------ | 20260506 12:49:35.599 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:49:35.600 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:49:35.600 | INFO | Total mem usage: 26.5 GiB | 20260506 12:49:35.600 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:49:35.600 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:49:35.600 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:49:35.600 | INFO | High Water Breakdown: | 20260506 12:49:35.600 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:49:35.600 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:49:35.600 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:49:35.600 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:49:35.600 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:49:35.600 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:49:35.600 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:49:35.600 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:49:35.600 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:49:35.600 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:49:35.600 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:49:35.600 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:49:35.600 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:49:35.600 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:49:35.600 | INFO | ------------------------------------------ | 20260506 12:49:35.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:49:35.600 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:49:35.600 | INFO | Total mem usage: 25.6 GiB | 20260506 12:49:35.600 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:49:35.600 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:49:35.600 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:49:35.600 | INFO | High Water Breakdown: | 20260506 12:49:35.600 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:49:35.600 | INFO | ------------------------------------------ | 20260506 12:49:35.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:49:35.600 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:49:35.600 | INFO | Total mem usage: 15.7 GiB | 20260506 12:49:35.600 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:49:35.600 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:49:35.600 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 12:49:35.600 | INFO | High Water Breakdown: | 20260506 12:49:35.600 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:49:35.600 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:49:35.600 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:49:35.600 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:49:35.600 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:49:35.600 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:49:35.600 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:49:35.600 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:49:35.600 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:49:35.600 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:49:35.600 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:49:35.600 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:49:35.600 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:49:35.600 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:49:35.600 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:49:35.600 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:49:35.600 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:49:35.600 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:49:35.600 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:49:35.600 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:49:35.600 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:49:35.600 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:49:35.600 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:49:35.600 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:49:35.600 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:49:35.600 | INFO | ------------------------------------------ | 20260506 12:49:35.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:49:35.600 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:49:35.600 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:49:35.600 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:49:35.600 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.59e+01, 1.34e+01 / 2.32e+00 | 20260506 12:49:35.600 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.01e-01, 1.09e-02 / 6.14e-03 | 20260506 12:49:35.600 | INFO | PerfReport: 11.55%, Baseline -- 3.66e+00 / 4.36e+00, 5.73e-01 / 3.60e-01 | 20260506 12:49:35.600 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.47e+00, 1.90e-01 / 1.38e-01 | 20260506 12:49:35.600 | INFO | PerfReport: 15.80%, DME -- 4.01e+01 / 4.43e+01, 3.94e+00 / 2.48e+00 | 20260506 12:49:35.600 | INFO | PerfReport: 56.53%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.44e+00 / 1.68e+00 | 20260506 12:49:35.600 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.39e+00 / 3.03e+00, 4.21e-01 / 2.19e-01 | 20260506 12:49:35.600 | INFO | PerfReport: 3.72%, Metrics -- 1.18e+00 / 1.54e+00, 2.98e-01 / 1.67e-01 | 20260506 12:49:35.600 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.16e-01 / 4.76e-01, 6.58e-02 / 2.94e-02 | 20260506 12:49:35.600 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:49:35.600 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:49:35.600 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:49:35.600 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 12:49:35.600 | INFO | PerfReport: 21.38%, Upload -- 2.11e+01 / 0.00e+00, 7.04e+00 / 0.00e+00 | 20260506 12:49:35.600 | INFO | PerfReport: 21.02%, Download -- 2.07e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 12:49:35.600 | INFO | PerfReport: 32.13%, Compute -- 3.17e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 12:49:35.600 | INFO | PerfReport: 1.73%, LockUpload -- 1.70e+00 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 12:49:35.600 | INFO | PerfReport: 1.15%, LockDownload -- 1.14e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 12:49:35.600 | INFO | PerfReport: 22.57%, LockCompute -- 2.23e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 12:49:35.600 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260506 12:49:35.731 | INFO | Starting a round of Baz IO | 20260506 12:49:38.899 | INFO | Populating RTMetrics report | 20260506 12:49:38.900 | INFO | RTMetrics report populated | 20260506 12:49:38.900 | INFO | Opening tmp RTMetrics json file | 20260506 12:49:38.901 | INFO | Serializing RTMetrics to json | 20260506 12:49:38.902 | INFO | Serialization done, swapping tmp file now... | 20260506 12:49:38.902 | INFO | RTMetrics done | 20260506 12:49:38.996 | INFO | Finished a round of Baz IO: Wrote 2.1 GiB in 3.265 seconds | 20260506 12:49:38.996 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.81% / 0.02% / 59.17% | 20260506 12:49:38.996 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.64593% / 2.78155e-05% / 4.76837e-05% | 20260506 12:49:38.999 | INFO | Baz IO overall: | 20260506 12:49:38.999 | INFO | BytesWritten : 743314072810 | 20260506 12:49:38.999 | INFO | NumEvents : 359349801490 | 20260506 12:49:40.738 | INFO | Analyzing chunk frames = [1482752,1483264) | 20260506 12:49:40.738 | INFO | MemInfo: { | 20260506 12:49:40.738 | INFO | "gpu" : " 17 GB", | 20260506 12:49:40.738 | INFO | "huge" : "", | 20260506 12:49:40.738 | INFO | "huge_shared" : " 27 GB", | 20260506 12:49:40.738 | INFO | "malloc" : " 14 GB", | 20260506 12:49:40.738 | INFO | "pinned" : " 29 GB" | 20260506 12:49:40.738 | INFO | } | 20260506 12:49:40.738 | INFO | Perf report for the last 5.13875 seconds: | 20260506 12:49:40.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:49:40.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:40.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:40.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:40.738 | INFO | ANALYSIS : 63.6%, 3.0, 98.8ms, 0 | 20260506 12:49:40.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:40.738 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:49:40.738 | INFO | PRE_HQ : 61.5%, 1.9, 61.2ms, 0 | 20260506 12:49:40.738 | INFO | BAZWRITER : 62.9%, 3.7, 119.2ms, 0 | 20260506 12:49:40.738 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:49:40.738 | INFO | Uploaded 36GiB to GPU in 2.07742s (17.3291GiB/s) | 20260506 12:49:40.738 | INFO | Downloaded 38GiB to GPU in 2.21933s (17.1223GiB/s) | 20260506 12:49:45.877 | INFO | Analyzing chunk frames = [1483264,1483776) | 20260506 12:49:45.877 | INFO | MemInfo: { | 20260506 12:49:45.877 | INFO | "gpu" : " 17 GB", | 20260506 12:49:45.877 | INFO | "huge" : "", | 20260506 12:49:45.877 | INFO | "huge_shared" : " 27 GB", | 20260506 12:49:45.877 | INFO | "malloc" : " 14 GB", | 20260506 12:49:45.877 | INFO | "pinned" : " 29 GB" | 20260506 12:49:45.877 | INFO | } | 20260506 12:49:45.877 | INFO | Perf report for the last 5.13942 seconds: | 20260506 12:49:45.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:49:45.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:45.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:45.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:45.877 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:49:45.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:45.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:45.877 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:49:45.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:45.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:45.878 | INFO | Uploaded 36GiB to GPU in 2.04257s (17.6249GiB/s) | 20260506 12:49:45.878 | INFO | Downloaded 34GiB to GPU in 1.92741s (17.6402GiB/s) | 20260506 12:49:51.017 | INFO | Analyzing chunk frames = [1483776,1484288) | 20260506 12:49:51.017 | INFO | MemInfo: { | 20260506 12:49:51.017 | INFO | "gpu" : " 17 GB", | 20260506 12:49:51.017 | INFO | "huge" : "", | 20260506 12:49:51.017 | INFO | "huge_shared" : " 27 GB", | 20260506 12:49:51.017 | INFO | "malloc" : " 14 GB", | 20260506 12:49:51.017 | INFO | "pinned" : " 29 GB" | 20260506 12:49:51.017 | INFO | } | 20260506 12:49:51.017 | INFO | Perf report for the last 5.13958 seconds: | 20260506 12:49:51.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:49:51.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:51.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:51.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:51.017 | INFO | ANALYSIS : 61.9%, 3.0, 95.8ms, 0 | 20260506 12:49:51.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:51.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:51.017 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:49:51.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:51.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:51.017 | INFO | Uploaded 36GiB to GPU in 2.04631s (17.5926GiB/s) | 20260506 12:49:51.017 | INFO | Downloaded 34GiB to GPU in 1.93681s (17.5547GiB/s) | 20260506 12:49:56.156 | INFO | Analyzing chunk frames = [1484288,1484800) | 20260506 12:49:56.156 | INFO | MemInfo: { | 20260506 12:49:56.156 | INFO | "gpu" : " 17 GB", | 20260506 12:49:56.156 | INFO | "huge" : "", | 20260506 12:49:56.156 | INFO | "huge_shared" : " 27 GB", | 20260506 12:49:56.156 | INFO | "malloc" : " 14 GB", | 20260506 12:49:56.156 | INFO | "pinned" : " 29 GB" | 20260506 12:49:56.156 | INFO | } | 20260506 12:49:56.156 | INFO | Perf report for the last 5.13933 seconds: | 20260506 12:49:56.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:49:56.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:56.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:56.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:56.156 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 12:49:56.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:56.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:56.156 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:49:56.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:56.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:49:56.156 | INFO | Uploaded 36GiB to GPU in 2.01522s (17.8641GiB/s) | 20260506 12:49:56.156 | INFO | Downloaded 34GiB to GPU in 1.94094s (17.5173GiB/s) | 20260506 12:50:01.295 | INFO | Analyzing chunk frames = [1484800,1485312) | 20260506 12:50:01.295 | INFO | MemInfo: { | 20260506 12:50:01.295 | INFO | "gpu" : " 17 GB", | 20260506 12:50:01.295 | INFO | "huge" : "", | 20260506 12:50:01.295 | INFO | "huge_shared" : " 27 GB", | 20260506 12:50:01.295 | INFO | "malloc" : " 14 GB", | 20260506 12:50:01.296 | INFO | "pinned" : " 29 GB" | 20260506 12:50:01.296 | INFO | } | 20260506 12:50:01.296 | INFO | Perf report for the last 5.13934 seconds: | 20260506 12:50:01.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:50:01.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:01.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:01.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:01.296 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 12:50:01.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:01.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:01.296 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:50:01.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:01.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:01.296 | INFO | Uploaded 36GiB to GPU in 1.96343s (18.3352GiB/s) | 20260506 12:50:01.296 | INFO | Downloaded 34GiB to GPU in 1.93378s (17.5821GiB/s) | 20260506 12:50:06.435 | INFO | Analyzing chunk frames = [1485312,1485824) | 20260506 12:50:06.435 | INFO | MemInfo: { | 20260506 12:50:06.435 | INFO | "gpu" : " 17 GB", | 20260506 12:50:06.435 | INFO | "huge" : "", | 20260506 12:50:06.435 | INFO | "huge_shared" : " 27 GB", | 20260506 12:50:06.435 | INFO | "malloc" : " 14 GB", | 20260506 12:50:06.435 | INFO | "pinned" : " 29 GB" | 20260506 12:50:06.435 | INFO | } | 20260506 12:50:06.435 | INFO | Perf report for the last 5.13959 seconds: | 20260506 12:50:06.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:50:06.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:06.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:06.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:06.435 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 12:50:06.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:06.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:06.435 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260506 12:50:06.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:06.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:06.435 | INFO | Uploaded 36GiB to GPU in 2.08449s (17.2704GiB/s) | 20260506 12:50:06.435 | INFO | Downloaded 34GiB to GPU in 1.97287s (17.2337GiB/s) | 20260506 12:50:11.574 | INFO | Analyzing chunk frames = [1485824,1486336) | 20260506 12:50:11.574 | INFO | MemInfo: { | 20260506 12:50:11.574 | INFO | "gpu" : " 17 GB", | 20260506 12:50:11.574 | INFO | "huge" : "", | 20260506 12:50:11.574 | INFO | "huge_shared" : " 27 GB", | 20260506 12:50:11.575 | INFO | "malloc" : " 14 GB", | 20260506 12:50:11.575 | INFO | "pinned" : " 29 GB" | 20260506 12:50:11.575 | INFO | } | 20260506 12:50:11.575 | INFO | Perf report for the last 5.13938 seconds: | 20260506 12:50:11.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:50:11.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:11.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:11.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:11.575 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 12:50:11.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:11.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:11.575 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:50:11.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:11.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:11.575 | INFO | Uploaded 36GiB to GPU in 2.0094s (17.9158GiB/s) | 20260506 12:50:11.575 | INFO | Downloaded 34GiB to GPU in 2.02081s (16.8249GiB/s) | 20260506 12:50:16.713 | INFO | Analyzing chunk frames = [1486336,1486848) | 20260506 12:50:16.713 | INFO | MemInfo: { | 20260506 12:50:16.713 | INFO | "gpu" : " 17 GB", | 20260506 12:50:16.713 | INFO | "huge" : "", | 20260506 12:50:16.713 | INFO | "huge_shared" : " 27 GB", | 20260506 12:50:16.713 | INFO | "malloc" : " 14 GB", | 20260506 12:50:16.713 | INFO | "pinned" : " 29 GB" | 20260506 12:50:16.713 | INFO | } | 20260506 12:50:16.713 | INFO | Perf report for the last 5.13886 seconds: | 20260506 12:50:16.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:50:16.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:16.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:16.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:16.714 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 12:50:16.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:16.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:16.714 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:50:16.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:16.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:16.714 | INFO | Uploaded 36GiB to GPU in 2.0584s (17.4893GiB/s) | 20260506 12:50:16.714 | INFO | Downloaded 34GiB to GPU in 1.96987s (17.26GiB/s) | 20260506 12:50:16.847 | INFO | Starting a round of Baz IO | 20260506 12:50:20.024 | INFO | Populating RTMetrics report | 20260506 12:50:20.024 | INFO | RTMetrics report populated | 20260506 12:50:20.024 | INFO | Opening tmp RTMetrics json file | 20260506 12:50:20.037 | INFO | Serializing RTMetrics to json | 20260506 12:50:20.063 | INFO | Serialization done, swapping tmp file now... | 20260506 12:50:20.064 | INFO | RTMetrics done | 20260506 12:50:20.125 | INFO | Finished a round of Baz IO: Wrote 2.099 GiB in 3.278 seconds | 20260506 12:50:20.125 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.78% / 0.02% / 59.19% | 20260506 12:50:20.125 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.63158% / 4.76837e-05% / 6.75519e-05% | 20260506 12:50:20.125 | INFO | Baz IO overall: | 20260506 12:50:20.125 | INFO | BytesWritten : 745567933141 | 20260506 12:50:20.125 | INFO | NumEvents : 360443107384 | 20260506 12:50:21.853 | INFO | Analyzing chunk frames = [1486848,1487360) | 20260506 12:50:21.853 | INFO | MemInfo: { | 20260506 12:50:21.853 | INFO | "gpu" : " 17 GB", | 20260506 12:50:21.853 | INFO | "huge" : "", | 20260506 12:50:21.853 | INFO | "huge_shared" : " 27 GB", | 20260506 12:50:21.853 | INFO | "malloc" : " 14 GB", | 20260506 12:50:21.853 | INFO | "pinned" : " 29 GB" | 20260506 12:50:21.853 | INFO | } | 20260506 12:50:21.853 | INFO | Perf report for the last 5.13939 seconds: | 20260506 12:50:21.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:50:21.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:21.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:21.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:21.853 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260506 12:50:21.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:21.853 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 12:50:21.853 | INFO | PRE_HQ : 61.3%, 1.9, 61.2ms, 0 | 20260506 12:50:21.853 | INFO | BAZWRITER : 63.2%, 3.7, 119.6ms, 0 | 20260506 12:50:21.853 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 12:50:21.853 | INFO | Uploaded 36GiB to GPU in 2.05622s (17.5078GiB/s) | 20260506 12:50:21.853 | INFO | Downloaded 38GiB to GPU in 2.21811s (17.1317GiB/s) | 20260506 12:50:26.992 | INFO | Analyzing chunk frames = [1487360,1487872) | 20260506 12:50:26.992 | INFO | MemInfo: { | 20260506 12:50:26.992 | INFO | "gpu" : " 17 GB", | 20260506 12:50:26.992 | INFO | "huge" : "", | 20260506 12:50:26.992 | INFO | "huge_shared" : " 27 GB", | 20260506 12:50:26.992 | INFO | "malloc" : " 14 GB", | 20260506 12:50:26.992 | INFO | "pinned" : " 29 GB" | 20260506 12:50:26.992 | INFO | } | 20260506 12:50:26.992 | INFO | Perf report for the last 5.13945 seconds: | 20260506 12:50:26.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:50:26.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:26.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:26.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:26.992 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:50:26.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:26.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:26.992 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:50:26.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:26.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:26.993 | INFO | Uploaded 36GiB to GPU in 2.04068s (17.6411GiB/s) | 20260506 12:50:26.993 | INFO | Downloaded 34GiB to GPU in 1.93356s (17.5842GiB/s) | 20260506 12:50:32.132 | INFO | Analyzing chunk frames = [1487872,1488384) | 20260506 12:50:32.132 | INFO | MemInfo: { | 20260506 12:50:32.132 | INFO | "gpu" : " 17 GB", | 20260506 12:50:32.132 | INFO | "huge" : "", | 20260506 12:50:32.132 | INFO | "huge_shared" : " 27 GB", | 20260506 12:50:32.132 | INFO | "malloc" : " 14 GB", | 20260506 12:50:32.132 | INFO | "pinned" : " 29 GB" | 20260506 12:50:32.132 | INFO | } | 20260506 12:50:32.132 | INFO | Perf report for the last 5.13944 seconds: | 20260506 12:50:32.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:50:32.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:32.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:32.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:32.132 | INFO | ANALYSIS : 61.9%, 3.0, 95.8ms, 0 | 20260506 12:50:32.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:32.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:32.132 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:50:32.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:32.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:32.132 | INFO | Uploaded 36GiB to GPU in 2.03821s (17.6625GiB/s) | 20260506 12:50:32.132 | INFO | Downloaded 34GiB to GPU in 1.93954s (17.53GiB/s) | 20260506 12:50:37.271 | INFO | Analyzing chunk frames = [1488384,1488896) | 20260506 12:50:37.271 | INFO | MemInfo: { | 20260506 12:50:37.271 | INFO | "gpu" : " 17 GB", | 20260506 12:50:37.271 | INFO | "huge" : "", | 20260506 12:50:37.271 | INFO | "huge_shared" : " 27 GB", | 20260506 12:50:37.271 | INFO | "malloc" : " 14 GB", | 20260506 12:50:37.271 | INFO | "pinned" : " 29 GB" | 20260506 12:50:37.271 | INFO | } | 20260506 12:50:37.271 | INFO | Perf report for the last 5.13926 seconds: | 20260506 12:50:37.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:50:37.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:37.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:37.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:37.271 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 12:50:37.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:37.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:37.271 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:50:37.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:37.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:37.271 | INFO | Uploaded 36GiB to GPU in 2.00428s (17.9615GiB/s) | 20260506 12:50:37.271 | INFO | Downloaded 34GiB to GPU in 1.94371s (17.4923GiB/s) | 20260506 12:50:42.411 | INFO | Analyzing chunk frames = [1488896,1489408) | 20260506 12:50:42.411 | INFO | MemInfo: { | 20260506 12:50:42.411 | INFO | "gpu" : " 17 GB", | 20260506 12:50:42.411 | INFO | "huge" : "", | 20260506 12:50:42.411 | INFO | "huge_shared" : " 27 GB", | 20260506 12:50:42.411 | INFO | "malloc" : " 14 GB", | 20260506 12:50:42.411 | INFO | "pinned" : " 29 GB" | 20260506 12:50:42.411 | INFO | } | 20260506 12:50:42.411 | INFO | Perf report for the last 5.13976 seconds: | 20260506 12:50:42.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:50:42.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:42.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:42.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:42.411 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 12:50:42.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:42.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:42.411 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:50:42.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:42.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:42.411 | INFO | Uploaded 36GiB to GPU in 1.97626s (18.2163GiB/s) | 20260506 12:50:42.411 | INFO | Downloaded 34GiB to GPU in 1.94121s (17.5149GiB/s) | 20260506 12:50:47.550 | INFO | Analyzing chunk frames = [1489408,1489920) | 20260506 12:50:47.550 | INFO | MemInfo: { | 20260506 12:50:47.550 | INFO | "gpu" : " 17 GB", | 20260506 12:50:47.550 | INFO | "huge" : "", | 20260506 12:50:47.550 | INFO | "huge_shared" : " 27 GB", | 20260506 12:50:47.550 | INFO | "malloc" : " 14 GB", | 20260506 12:50:47.550 | INFO | "pinned" : " 29 GB" | 20260506 12:50:47.550 | INFO | } | 20260506 12:50:47.550 | INFO | Perf report for the last 5.13947 seconds: | 20260506 12:50:47.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:50:47.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:47.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:47.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:47.550 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 12:50:47.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:47.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:47.550 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:50:47.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:47.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:47.551 | INFO | Uploaded 36GiB to GPU in 2.08357s (17.2781GiB/s) | 20260506 12:50:47.551 | INFO | Downloaded 34GiB to GPU in 1.96346s (17.3164GiB/s) | 20260506 12:50:52.690 | INFO | Analyzing chunk frames = [1489920,1490432) | 20260506 12:50:52.690 | INFO | MemInfo: { | 20260506 12:50:52.690 | INFO | "gpu" : " 17 GB", | 20260506 12:50:52.690 | INFO | "huge" : "", | 20260506 12:50:52.690 | INFO | "huge_shared" : " 27 GB", | 20260506 12:50:52.690 | INFO | "malloc" : " 14 GB", | 20260506 12:50:52.690 | INFO | "pinned" : " 29 GB" | 20260506 12:50:52.690 | INFO | } | 20260506 12:50:52.690 | INFO | Perf report for the last 5.13982 seconds: | 20260506 12:50:52.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:50:52.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:52.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:52.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:52.690 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:50:52.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:52.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:52.690 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:50:52.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:52.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:52.690 | INFO | Uploaded 36GiB to GPU in 1.98278s (18.1563GiB/s) | 20260506 12:50:52.690 | INFO | Downloaded 34GiB to GPU in 2.03239s (16.7291GiB/s) | 20260506 12:50:57.828 | INFO | Analyzing chunk frames = [1490432,1490944) | 20260506 12:50:57.829 | INFO | MemInfo: { | 20260506 12:50:57.829 | INFO | "gpu" : " 17 GB", | 20260506 12:50:57.829 | INFO | "huge" : "", | 20260506 12:50:57.829 | INFO | "huge_shared" : " 27 GB", | 20260506 12:50:57.829 | INFO | "malloc" : " 14 GB", | 20260506 12:50:57.829 | INFO | "pinned" : " 29 GB" | 20260506 12:50:57.829 | INFO | } | 20260506 12:50:57.829 | INFO | Perf report for the last 5.13861 seconds: | 20260506 12:50:57.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:50:57.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:57.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:57.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:57.829 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 12:50:57.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:57.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:57.829 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:50:57.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:57.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:50:57.829 | INFO | Uploaded 36GiB to GPU in 2.03554s (17.6857GiB/s) | 20260506 12:50:57.829 | INFO | Downloaded 34GiB to GPU in 1.96254s (17.3245GiB/s) | 20260506 12:50:57.829 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:50:57.829 | INFO | Total mem usage: 16.8 GiB | 20260506 12:50:57.829 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:50:57.829 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.4%) | 20260506 12:50:57.829 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:50:57.829 | INFO | High Water Breakdown: | 20260506 12:50:57.829 | INFO | 59.8% : 8.63 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:50:57.829 | INFO | 40.1% : 5.79 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:50:57.829 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:50:57.829 | INFO | ------------------------------------------ | 20260506 12:50:57.829 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:50:57.829 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:50:57.829 | INFO | Total mem usage: 26.5 GiB | 20260506 12:50:57.829 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:50:57.829 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:50:57.829 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:50:57.829 | INFO | High Water Breakdown: | 20260506 12:50:57.829 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:50:57.829 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:50:57.829 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:50:57.829 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:50:57.829 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:50:57.829 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:50:57.829 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:50:57.829 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:50:57.829 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:50:57.829 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:50:57.829 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:50:57.829 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:50:57.829 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:50:57.829 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:50:57.829 | INFO | ------------------------------------------ | 20260506 12:50:57.829 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:50:57.829 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:50:57.829 | INFO | Total mem usage: 25.6 GiB | 20260506 12:50:57.829 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:50:57.829 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:50:57.829 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:50:57.829 | INFO | High Water Breakdown: | 20260506 12:50:57.829 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:50:57.829 | INFO | ------------------------------------------ | 20260506 12:50:57.829 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:50:57.829 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:50:57.829 | INFO | Total mem usage: 15.7 GiB | 20260506 12:50:57.829 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:50:57.829 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:50:57.829 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:50:57.830 | INFO | High Water Breakdown: | 20260506 12:50:57.830 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:50:57.830 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:50:57.830 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:50:57.830 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:50:57.830 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:50:57.830 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:50:57.830 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:50:57.830 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:50:57.830 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:50:57.830 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:50:57.830 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:50:57.830 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:50:57.830 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:50:57.830 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:50:57.830 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:50:57.830 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:50:57.830 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:50:57.830 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:50:57.830 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:50:57.830 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:50:57.830 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:50:57.830 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:50:57.830 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:50:57.830 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:50:57.830 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:50:57.830 | INFO | ------------------------------------------ | 20260506 12:50:57.830 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:50:57.830 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:50:57.830 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:50:57.830 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:50:57.830 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.67e+01, 1.37e+01 / 2.60e+00 | 20260506 12:50:57.830 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.01e-01, 1.01e-02 / 4.24e-03 | 20260506 12:50:57.830 | INFO | PerfReport: 11.59%, Baseline -- 3.67e+00 / 4.51e+00, 5.83e-01 / 2.52e-01 | 20260506 12:50:57.830 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.48e+00, 1.91e-01 / 1.20e-01 | 20260506 12:50:57.830 | INFO | PerfReport: 15.91%, DME -- 4.03e+01 / 4.53e+01, 4.43e+00 / 3.08e+00 | 20260506 12:50:57.830 | INFO | PerfReport: 56.36%, FrameLabeling -- 1.79e+01 / 2.02e+01, 1.33e+00 / 1.81e+00 | 20260506 12:50:57.830 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.40e+00 / 3.03e+00, 4.23e-01 / 2.40e-01 | 20260506 12:50:57.830 | INFO | PerfReport: 3.73%, Metrics -- 1.18e+00 / 1.58e+00, 3.02e-01 / 1.61e-01 | 20260506 12:50:57.830 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.15e-01 / 4.74e-01, 6.46e-02 / 2.55e-02 | 20260506 12:50:57.830 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:50:57.830 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:50:57.830 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:50:57.830 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 12:50:57.830 | INFO | PerfReport: 21.44%, Upload -- 2.12e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260506 12:50:57.830 | INFO | PerfReport: 21.01%, Download -- 2.07e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 12:50:57.830 | INFO | PerfReport: 32.09%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 12:50:57.830 | INFO | PerfReport: 1.84%, LockUpload -- 1.82e+00 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260506 12:50:57.830 | INFO | PerfReport: 0.99%, LockDownload -- 9.77e-01 / 0.00e+00, 2.64e+00 / 0.00e+00 | 20260506 12:50:57.830 | INFO | PerfReport: 22.62%, LockCompute -- 2.23e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 12:50:57.830 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260506 12:50:57.963 | INFO | Starting a round of Baz IO | 20260506 12:51:01.121 | INFO | Populating RTMetrics report | 20260506 12:51:01.121 | INFO | RTMetrics report populated | 20260506 12:51:01.121 | INFO | Opening tmp RTMetrics json file | 20260506 12:51:01.122 | INFO | Serializing RTMetrics to json | 20260506 12:51:01.123 | INFO | Serialization done, swapping tmp file now... | 20260506 12:51:01.123 | INFO | RTMetrics done | 20260506 12:51:01.220 | INFO | Finished a round of Baz IO: Wrote 2.098 GiB in 3.256 seconds | 20260506 12:51:01.220 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.76% / 0.02% / 59.22% | 20260506 12:51:01.220 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.61491% / 1.98682e-05% / 4.76837e-05% | 20260506 12:51:01.220 | INFO | Baz IO overall: | 20260506 12:51:01.220 | INFO | BytesWritten : 747820934062 | 20260506 12:51:01.220 | INFO | NumEvents : 361535565319 | 20260506 12:51:02.967 | INFO | Analyzing chunk frames = [1490944,1491456) | 20260506 12:51:02.967 | INFO | MemInfo: { | 20260506 12:51:02.968 | INFO | "gpu" : " 17 GB", | 20260506 12:51:02.968 | INFO | "huge" : "", | 20260506 12:51:02.968 | INFO | "huge_shared" : " 27 GB", | 20260506 12:51:02.968 | INFO | "malloc" : " 14 GB", | 20260506 12:51:02.968 | INFO | "pinned" : " 29 GB" | 20260506 12:51:02.968 | INFO | } | 20260506 12:51:02.968 | INFO | Perf report for the last 5.13884 seconds: | 20260506 12:51:02.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:51:02.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:02.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:02.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:02.968 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 12:51:02.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:02.968 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:51:02.968 | INFO | PRE_HQ : 61.4%, 1.9, 61.4ms, 0 | 20260506 12:51:02.968 | INFO | BAZWRITER : 62.6%, 3.7, 119.3ms, 0 | 20260506 12:51:02.968 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:51:02.968 | INFO | Uploaded 36GiB to GPU in 2.03909s (17.6549GiB/s) | 20260506 12:51:02.968 | INFO | Downloaded 38GiB to GPU in 2.21954s (17.1206GiB/s) | 20260506 12:51:08.107 | INFO | Analyzing chunk frames = [1491456,1491968) | 20260506 12:51:08.107 | INFO | MemInfo: { | 20260506 12:51:08.107 | INFO | "gpu" : " 17 GB", | 20260506 12:51:08.107 | INFO | "huge" : "", | 20260506 12:51:08.107 | INFO | "huge_shared" : " 27 GB", | 20260506 12:51:08.107 | INFO | "malloc" : " 14 GB", | 20260506 12:51:08.107 | INFO | "pinned" : " 29 GB" | 20260506 12:51:08.107 | INFO | } | 20260506 12:51:08.107 | INFO | Perf report for the last 5.13945 seconds: | 20260506 12:51:08.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:51:08.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:08.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:08.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:08.107 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 12:51:08.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:08.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:08.107 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:51:08.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:08.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:08.107 | INFO | Uploaded 36GiB to GPU in 2.03391s (17.6999GiB/s) | 20260506 12:51:08.107 | INFO | Downloaded 34GiB to GPU in 1.94864s (17.448GiB/s) | 20260506 12:51:13.246 | INFO | Analyzing chunk frames = [1491968,1492480) | 20260506 12:51:13.246 | INFO | MemInfo: { | 20260506 12:51:13.246 | INFO | "gpu" : " 17 GB", | 20260506 12:51:13.246 | INFO | "huge" : "", | 20260506 12:51:13.246 | INFO | "huge_shared" : " 27 GB", | 20260506 12:51:13.246 | INFO | "malloc" : " 14 GB", | 20260506 12:51:13.246 | INFO | "pinned" : " 29 GB" | 20260506 12:51:13.246 | INFO | } | 20260506 12:51:13.246 | INFO | Perf report for the last 5.13916 seconds: | 20260506 12:51:13.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:51:13.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:13.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:13.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:13.246 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 12:51:13.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:13.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:13.246 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:51:13.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:13.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:13.246 | INFO | Uploaded 36GiB to GPU in 2.02019s (17.8201GiB/s) | 20260506 12:51:13.247 | INFO | Downloaded 34GiB to GPU in 1.94477s (17.4828GiB/s) | 20260506 12:51:18.385 | INFO | Analyzing chunk frames = [1492480,1492992) | 20260506 12:51:18.385 | INFO | MemInfo: { | 20260506 12:51:18.385 | INFO | "gpu" : " 17 GB", | 20260506 12:51:18.385 | INFO | "huge" : "", | 20260506 12:51:18.385 | INFO | "huge_shared" : " 27 GB", | 20260506 12:51:18.385 | INFO | "malloc" : " 14 GB", | 20260506 12:51:18.385 | INFO | "pinned" : " 29 GB" | 20260506 12:51:18.385 | INFO | } | 20260506 12:51:18.385 | INFO | Perf report for the last 5.13913 seconds: | 20260506 12:51:18.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:51:18.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:18.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:18.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:18.385 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 12:51:18.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:18.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:18.385 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:51:18.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:18.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:18.386 | INFO | Uploaded 36GiB to GPU in 1.98716s (18.1163GiB/s) | 20260506 12:51:18.386 | INFO | Downloaded 34GiB to GPU in 1.94108s (17.516GiB/s) | 20260506 12:51:23.525 | INFO | Analyzing chunk frames = [1492992,1493504) | 20260506 12:51:23.525 | INFO | MemInfo: { | 20260506 12:51:23.525 | INFO | "gpu" : " 17 GB", | 20260506 12:51:23.525 | INFO | "huge" : "", | 20260506 12:51:23.525 | INFO | "huge_shared" : " 27 GB", | 20260506 12:51:23.525 | INFO | "malloc" : " 14 GB", | 20260506 12:51:23.525 | INFO | "pinned" : " 29 GB" | 20260506 12:51:23.525 | INFO | } | 20260506 12:51:23.525 | INFO | Perf report for the last 5.14 seconds: | 20260506 12:51:23.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:51:23.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:23.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:23.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:23.525 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 12:51:23.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:23.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:23.525 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:51:23.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:23.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:23.526 | INFO | Uploaded 36GiB to GPU in 2.03011s (17.733GiB/s) | 20260506 12:51:23.526 | INFO | Downloaded 34GiB to GPU in 1.94973s (17.4383GiB/s) | 20260506 12:51:28.664 | INFO | Analyzing chunk frames = [1493504,1494016) | 20260506 12:51:28.664 | INFO | MemInfo: { | 20260506 12:51:28.664 | INFO | "gpu" : " 17 GB", | 20260506 12:51:28.664 | INFO | "huge" : "", | 20260506 12:51:28.664 | INFO | "huge_shared" : " 27 GB", | 20260506 12:51:28.664 | INFO | "malloc" : " 14 GB", | 20260506 12:51:28.664 | INFO | "pinned" : " 29 GB" | 20260506 12:51:28.664 | INFO | } | 20260506 12:51:28.664 | INFO | Perf report for the last 5.13899 seconds: | 20260506 12:51:28.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:51:28.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:28.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:28.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:28.664 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 12:51:28.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:28.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:28.664 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260506 12:51:28.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:28.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:28.665 | INFO | Uploaded 36GiB to GPU in 2.03264s (17.711GiB/s) | 20260506 12:51:28.665 | INFO | Downloaded 34GiB to GPU in 1.9264s (17.6495GiB/s) | 20260506 12:51:33.804 | INFO | Analyzing chunk frames = [1494016,1494528) | 20260506 12:51:33.804 | INFO | MemInfo: { | 20260506 12:51:33.804 | INFO | "gpu" : " 17 GB", | 20260506 12:51:33.805 | INFO | "huge" : "", | 20260506 12:51:33.805 | INFO | "huge_shared" : " 27 GB", | 20260506 12:51:33.805 | INFO | "malloc" : " 14 GB", | 20260506 12:51:33.805 | INFO | "pinned" : " 29 GB" | 20260506 12:51:33.805 | INFO | } | 20260506 12:51:33.805 | INFO | Perf report for the last 5.14024 seconds: | 20260506 12:51:33.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:51:33.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:33.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:33.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:33.805 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:51:33.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:33.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:33.805 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:51:33.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:33.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:33.805 | INFO | Uploaded 36GiB to GPU in 2.00322s (17.971GiB/s) | 20260506 12:51:33.805 | INFO | Downloaded 34GiB to GPU in 2.0227s (16.8092GiB/s) | 20260506 12:51:38.943 | INFO | Analyzing chunk frames = [1494528,1495040) | 20260506 12:51:38.943 | INFO | MemInfo: { | 20260506 12:51:38.943 | INFO | "gpu" : " 17 GB", | 20260506 12:51:38.943 | INFO | "huge" : "", | 20260506 12:51:38.943 | INFO | "huge_shared" : " 27 GB", | 20260506 12:51:38.943 | INFO | "malloc" : " 14 GB", | 20260506 12:51:38.943 | INFO | "pinned" : " 29 GB" | 20260506 12:51:38.943 | INFO | } | 20260506 12:51:38.943 | INFO | Perf report for the last 5.13839 seconds: | 20260506 12:51:38.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:51:38.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:38.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:38.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:38.943 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:51:38.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:38.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:38.943 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:51:38.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:38.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:38.943 | INFO | Uploaded 36GiB to GPU in 2.05181s (17.5455GiB/s) | 20260506 12:51:38.943 | INFO | Downloaded 34GiB to GPU in 1.96755s (17.2804GiB/s) | 20260506 12:51:39.076 | INFO | Starting a round of Baz IO | 20260506 12:51:42.237 | INFO | Populating RTMetrics report | 20260506 12:51:42.237 | INFO | RTMetrics report populated | 20260506 12:51:42.237 | INFO | Opening tmp RTMetrics json file | 20260506 12:51:42.238 | INFO | Serializing RTMetrics to json | 20260506 12:51:42.239 | INFO | Serialization done, swapping tmp file now... | 20260506 12:51:42.239 | INFO | RTMetrics done | 20260506 12:51:42.336 | INFO | Finished a round of Baz IO: Wrote 2.099 GiB in 3.26 seconds | 20260506 12:51:42.336 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.74% / 0.02% / 59.24% | 20260506 12:51:42.336 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.60626% / 3.97364e-05% / 5.96046e-05% | 20260506 12:51:42.336 | INFO | Baz IO overall: | 20260506 12:51:42.336 | INFO | BytesWritten : 750075059511 | 20260506 12:51:42.336 | INFO | NumEvents : 362628599330 | 20260506 12:51:44.082 | INFO | Analyzing chunk frames = [1495040,1495552) | 20260506 12:51:44.082 | INFO | MemInfo: { | 20260506 12:51:44.082 | INFO | "gpu" : " 17 GB", | 20260506 12:51:44.083 | INFO | "huge" : "", | 20260506 12:51:44.083 | INFO | "huge_shared" : " 27 GB", | 20260506 12:51:44.083 | INFO | "malloc" : " 14 GB", | 20260506 12:51:44.083 | INFO | "pinned" : " 29 GB" | 20260506 12:51:44.083 | INFO | } | 20260506 12:51:44.083 | INFO | Perf report for the last 5.13963 seconds: | 20260506 12:51:44.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:51:44.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:44.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:44.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:44.083 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260506 12:51:44.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:44.083 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:51:44.083 | INFO | PRE_HQ : 61.2%, 1.9, 61.1ms, 0 | 20260506 12:51:44.083 | INFO | BAZWRITER : 62.8%, 3.7, 118.0ms, 0 | 20260506 12:51:44.083 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:51:44.083 | INFO | Uploaded 36GiB to GPU in 2.03848s (17.6602GiB/s) | 20260506 12:51:44.083 | INFO | Downloaded 38GiB to GPU in 2.2231s (17.0933GiB/s) | 20260506 12:51:49.222 | INFO | Analyzing chunk frames = [1495552,1496064) | 20260506 12:51:49.222 | INFO | MemInfo: { | 20260506 12:51:49.222 | INFO | "gpu" : " 17 GB", | 20260506 12:51:49.222 | INFO | "huge" : "", | 20260506 12:51:49.222 | INFO | "huge_shared" : " 27 GB", | 20260506 12:51:49.222 | INFO | "malloc" : " 14 GB", | 20260506 12:51:49.222 | INFO | "pinned" : " 29 GB" | 20260506 12:51:49.222 | INFO | } | 20260506 12:51:49.222 | INFO | Perf report for the last 5.13904 seconds: | 20260506 12:51:49.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:51:49.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:49.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:49.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:49.222 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:51:49.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:49.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:49.222 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:51:49.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:49.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:49.222 | INFO | Uploaded 36GiB to GPU in 2.04712s (17.5857GiB/s) | 20260506 12:51:49.222 | INFO | Downloaded 34GiB to GPU in 1.94445s (17.4856GiB/s) | 20260506 12:51:54.361 | INFO | Analyzing chunk frames = [1496064,1496576) | 20260506 12:51:54.361 | INFO | MemInfo: { | 20260506 12:51:54.361 | INFO | "gpu" : " 17 GB", | 20260506 12:51:54.361 | INFO | "huge" : "", | 20260506 12:51:54.361 | INFO | "huge_shared" : " 27 GB", | 20260506 12:51:54.361 | INFO | "malloc" : " 14 GB", | 20260506 12:51:54.361 | INFO | "pinned" : " 29 GB" | 20260506 12:51:54.361 | INFO | } | 20260506 12:51:54.361 | INFO | Perf report for the last 5.13955 seconds: | 20260506 12:51:54.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:51:54.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:54.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:54.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:54.361 | INFO | ANALYSIS : 61.8%, 3.0, 95.6ms, 0 | 20260506 12:51:54.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:54.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:54.361 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:51:54.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:54.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:54.362 | INFO | Uploaded 36GiB to GPU in 2.03265s (17.7109GiB/s) | 20260506 12:51:54.362 | INFO | Downloaded 34GiB to GPU in 1.94632s (17.4688GiB/s) | 20260506 12:51:59.501 | INFO | Analyzing chunk frames = [1496576,1497088) | 20260506 12:51:59.501 | INFO | MemInfo: { | 20260506 12:51:59.501 | INFO | "gpu" : " 17 GB", | 20260506 12:51:59.501 | INFO | "huge" : "", | 20260506 12:51:59.501 | INFO | "huge_shared" : " 27 GB", | 20260506 12:51:59.501 | INFO | "malloc" : " 14 GB", | 20260506 12:51:59.501 | INFO | "pinned" : " 29 GB" | 20260506 12:51:59.501 | INFO | } | 20260506 12:51:59.501 | INFO | Perf report for the last 5.13955 seconds: | 20260506 12:51:59.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:51:59.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:59.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:59.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:59.501 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 12:51:59.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:59.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:59.501 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:51:59.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:59.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:51:59.501 | INFO | Uploaded 36GiB to GPU in 1.98648s (18.1225GiB/s) | 20260506 12:51:59.501 | INFO | Downloaded 34GiB to GPU in 1.93903s (17.5345GiB/s) | 20260506 12:52:04.640 | INFO | Analyzing chunk frames = [1497088,1497600) | 20260506 12:52:04.640 | INFO | MemInfo: { | 20260506 12:52:04.640 | INFO | "gpu" : " 17 GB", | 20260506 12:52:04.640 | INFO | "huge" : "", | 20260506 12:52:04.640 | INFO | "huge_shared" : " 27 GB", | 20260506 12:52:04.640 | INFO | "malloc" : " 14 GB", | 20260506 12:52:04.640 | INFO | "pinned" : " 29 GB" | 20260506 12:52:04.640 | INFO | } | 20260506 12:52:04.640 | INFO | Perf report for the last 5.13901 seconds: | 20260506 12:52:04.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:52:04.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:04.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:04.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:04.640 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 12:52:04.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:04.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:04.640 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:52:04.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:04.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:04.640 | INFO | Uploaded 36GiB to GPU in 2.01287s (17.8849GiB/s) | 20260506 12:52:04.640 | INFO | Downloaded 34GiB to GPU in 1.9453s (17.4781GiB/s) | 20260506 12:52:09.779 | INFO | Analyzing chunk frames = [1497600,1498112) | 20260506 12:52:09.779 | INFO | MemInfo: { | 20260506 12:52:09.779 | INFO | "gpu" : " 17 GB", | 20260506 12:52:09.779 | INFO | "huge" : "", | 20260506 12:52:09.779 | INFO | "huge_shared" : " 27 GB", | 20260506 12:52:09.779 | INFO | "malloc" : " 14 GB", | 20260506 12:52:09.779 | INFO | "pinned" : " 29 GB" | 20260506 12:52:09.779 | INFO | } | 20260506 12:52:09.779 | INFO | Perf report for the last 5.13933 seconds: | 20260506 12:52:09.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:52:09.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:09.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:09.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:09.779 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 12:52:09.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:09.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:09.779 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:52:09.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:09.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:09.779 | INFO | Uploaded 36GiB to GPU in 2.07801s (17.3242GiB/s) | 20260506 12:52:09.780 | INFO | Downloaded 34GiB to GPU in 1.98122s (17.1612GiB/s) | 20260506 12:52:14.919 | INFO | Analyzing chunk frames = [1498112,1498624) | 20260506 12:52:14.919 | INFO | MemInfo: { | 20260506 12:52:14.919 | INFO | "gpu" : " 17 GB", | 20260506 12:52:14.919 | INFO | "huge" : "", | 20260506 12:52:14.919 | INFO | "huge_shared" : " 27 GB", | 20260506 12:52:14.919 | INFO | "malloc" : " 14 GB", | 20260506 12:52:14.919 | INFO | "pinned" : " 29 GB" | 20260506 12:52:14.919 | INFO | } | 20260506 12:52:14.920 | INFO | Perf report for the last 5.14028 seconds: | 20260506 12:52:14.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:52:14.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:14.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:14.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:14.920 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:52:14.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:14.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:14.920 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 12:52:14.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:14.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:14.920 | INFO | Uploaded 36GiB to GPU in 1.96719s (18.3002GiB/s) | 20260506 12:52:14.920 | INFO | Downloaded 34GiB to GPU in 2.0352s (16.706GiB/s) | 20260506 12:52:20.058 | INFO | Analyzing chunk frames = [1498624,1499136) | 20260506 12:52:20.058 | INFO | MemInfo: { | 20260506 12:52:20.058 | INFO | "gpu" : " 17 GB", | 20260506 12:52:20.058 | INFO | "huge" : "", | 20260506 12:52:20.058 | INFO | "huge_shared" : " 27 GB", | 20260506 12:52:20.058 | INFO | "malloc" : " 14 GB", | 20260506 12:52:20.058 | INFO | "pinned" : " 29 GB" | 20260506 12:52:20.058 | INFO | } | 20260506 12:52:20.058 | INFO | Perf report for the last 5.13834 seconds: | 20260506 12:52:20.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:52:20.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:20.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:20.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:20.058 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:52:20.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:20.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:20.058 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:52:20.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:20.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:20.058 | INFO | Uploaded 36GiB to GPU in 2.065s (17.4334GiB/s) | 20260506 12:52:20.058 | INFO | Downloaded 34GiB to GPU in 1.97465s (17.2182GiB/s) | 20260506 12:52:20.058 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:52:20.058 | INFO | Total mem usage: 16.8 GiB | 20260506 12:52:20.058 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:52:20.058 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -16.2%) | 20260506 12:52:20.058 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:52:20.058 | INFO | High Water Breakdown: | 20260506 12:52:20.058 | INFO | 59.7% : 8.64 GiB ( 83.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:52:20.058 | INFO | 40.1% : 5.81 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:52:20.058 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:52:20.058 | INFO | ------------------------------------------ | 20260506 12:52:20.058 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:52:20.058 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:52:20.058 | INFO | Total mem usage: 26.5 GiB | 20260506 12:52:20.058 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:52:20.058 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:52:20.058 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:52:20.058 | INFO | High Water Breakdown: | 20260506 12:52:20.058 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:52:20.058 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:52:20.058 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:52:20.058 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:52:20.058 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:52:20.058 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:52:20.058 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:52:20.058 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:52:20.058 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:52:20.058 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:52:20.058 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:52:20.059 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:52:20.059 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:52:20.059 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:52:20.059 | INFO | ------------------------------------------ | 20260506 12:52:20.059 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:52:20.059 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:52:20.059 | INFO | Total mem usage: 25.6 GiB | 20260506 12:52:20.059 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:52:20.059 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:52:20.059 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:52:20.059 | INFO | High Water Breakdown: | 20260506 12:52:20.059 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:52:20.059 | INFO | ------------------------------------------ | 20260506 12:52:20.059 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:52:20.059 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:52:20.059 | INFO | Total mem usage: 15.7 GiB | 20260506 12:52:20.059 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:52:20.059 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:52:20.059 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 12:52:20.059 | INFO | High Water Breakdown: | 20260506 12:52:20.059 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:52:20.059 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:52:20.059 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:52:20.059 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:52:20.059 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:52:20.059 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:52:20.059 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:52:20.059 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:52:20.059 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:52:20.059 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:52:20.059 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:52:20.059 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:52:20.059 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:52:20.059 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:52:20.059 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:52:20.059 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:52:20.059 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:52:20.059 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:52:20.059 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:52:20.059 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:52:20.059 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:52:20.059 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:52:20.059 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:52:20.059 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:52:20.059 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:52:20.059 | INFO | ------------------------------------------ | 20260506 12:52:20.059 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:52:20.059 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:52:20.059 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:52:20.059 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:52:20.059 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.62e+01, 1.35e+01 / 2.62e+00 | 20260506 12:52:20.059 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.02e-01, 1.00e-02 / 7.17e-03 | 20260506 12:52:20.059 | INFO | PerfReport: 11.56%, Baseline -- 3.66e+00 / 4.43e+00, 5.76e-01 / 3.64e-01 | 20260506 12:52:20.059 | INFO | PerfReport: 3.23%, Histogram -- 1.02e+00 / 1.48e+00, 1.90e-01 / 1.39e-01 | 20260506 12:52:20.059 | INFO | PerfReport: 15.80%, DME -- 4.00e+01 / 4.43e+01, 4.09e+00 / 2.99e+00 | 20260506 12:52:20.059 | INFO | PerfReport: 56.52%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.42e+00 / 1.99e+00 | 20260506 12:52:20.059 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.40e+00 / 3.01e+00, 4.20e-01 / 2.70e-01 | 20260506 12:52:20.059 | INFO | PerfReport: 3.72%, Metrics -- 1.18e+00 / 1.58e+00, 3.02e-01 / 1.69e-01 | 20260506 12:52:20.059 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.14e-01 / 4.68e-01, 6.49e-02 / 2.60e-02 | 20260506 12:52:20.059 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:52:20.059 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:52:20.059 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:52:20.059 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 12:52:20.059 | INFO | PerfReport: 21.40%, Upload -- 2.11e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 12:52:20.059 | INFO | PerfReport: 21.06%, Download -- 2.08e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 12:52:20.059 | INFO | PerfReport: 32.12%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 12:52:20.059 | INFO | PerfReport: 1.66%, LockUpload -- 1.64e+00 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260506 12:52:20.059 | INFO | PerfReport: 1.18%, LockDownload -- 1.16e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260506 12:52:20.059 | INFO | PerfReport: 22.57%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:52:20.059 | INFO | PerfReport: 0.02%, Misc -- 1.54e-02 / 0.00e+00, 2.35e-02 / 0.00e+00 | 20260506 12:52:20.190 | INFO | Starting a round of Baz IO | 20260506 12:52:23.350 | INFO | Populating RTMetrics report | 20260506 12:52:23.351 | INFO | RTMetrics report populated | 20260506 12:52:23.351 | INFO | Opening tmp RTMetrics json file | 20260506 12:52:23.362 | INFO | Serializing RTMetrics to json | 20260506 12:52:23.373 | INFO | Serialization done, swapping tmp file now... | 20260506 12:52:23.377 | INFO | RTMetrics done | 20260506 12:52:23.441 | INFO | Finished a round of Baz IO: Wrote 2.1 GiB in 3.251 seconds | 20260506 12:52:23.441 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.72% / 0.02% / 59.26% | 20260506 12:52:23.441 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.60299% / 4.76837e-05% / 5.5631e-05% | 20260506 12:52:23.442 | INFO | Baz IO overall: | 20260506 12:52:23.442 | INFO | BytesWritten : 752330120659 | 20260506 12:52:23.442 | INFO | NumEvents : 363722310044 | 20260506 12:52:25.197 | INFO | Analyzing chunk frames = [1499136,1499648) | 20260506 12:52:25.197 | INFO | MemInfo: { | 20260506 12:52:25.197 | INFO | "gpu" : " 17 GB", | 20260506 12:52:25.197 | INFO | "huge" : "", | 20260506 12:52:25.197 | INFO | "huge_shared" : " 27 GB", | 20260506 12:52:25.197 | INFO | "malloc" : " 14 GB", | 20260506 12:52:25.197 | INFO | "pinned" : " 29 GB" | 20260506 12:52:25.197 | INFO | } | 20260506 12:52:25.197 | INFO | Perf report for the last 5.13924 seconds: | 20260506 12:52:25.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:52:25.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:25.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:25.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:25.197 | INFO | ANALYSIS : 63.5%, 3.0, 98.3ms, 0 | 20260506 12:52:25.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:25.197 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260506 12:52:25.197 | INFO | PRE_HQ : 61.6%, 2.0, 64.2ms, 0 | 20260506 12:52:25.197 | INFO | BAZWRITER : 62.6%, 3.8, 122.0ms, 0 | 20260506 12:52:25.197 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 12:52:25.197 | INFO | Uploaded 36GiB to GPU in 2.03544s (17.6866GiB/s) | 20260506 12:52:25.197 | INFO | Downloaded 38GiB to GPU in 2.218s (17.1326GiB/s) | 20260506 12:52:30.336 | INFO | Analyzing chunk frames = [1499648,1500160) | 20260506 12:52:30.336 | INFO | MemInfo: { | 20260506 12:52:30.336 | INFO | "gpu" : " 17 GB", | 20260506 12:52:30.336 | INFO | "huge" : "", | 20260506 12:52:30.336 | INFO | "huge_shared" : " 27 GB", | 20260506 12:52:30.336 | INFO | "malloc" : " 14 GB", | 20260506 12:52:30.336 | INFO | "pinned" : " 29 GB" | 20260506 12:52:30.336 | INFO | } | 20260506 12:52:30.336 | INFO | Perf report for the last 5.13924 seconds: | 20260506 12:52:30.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:52:30.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:30.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:30.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:30.336 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 12:52:30.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:30.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:30.336 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:52:30.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:30.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:30.337 | INFO | Uploaded 36GiB to GPU in 2.02408s (17.7859GiB/s) | 20260506 12:52:30.337 | INFO | Downloaded 34GiB to GPU in 1.94022s (17.5238GiB/s) | 20260506 12:52:35.476 | INFO | Analyzing chunk frames = [1500160,1500672) | 20260506 12:52:35.476 | INFO | MemInfo: { | 20260506 12:52:35.476 | INFO | "gpu" : " 17 GB", | 20260506 12:52:35.476 | INFO | "huge" : "", | 20260506 12:52:35.476 | INFO | "huge_shared" : " 27 GB", | 20260506 12:52:35.476 | INFO | "malloc" : " 14 GB", | 20260506 12:52:35.476 | INFO | "pinned" : " 29 GB" | 20260506 12:52:35.476 | INFO | } | 20260506 12:52:35.476 | INFO | Perf report for the last 5.13981 seconds: | 20260506 12:52:35.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:52:35.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:35.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:35.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:35.476 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 12:52:35.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:35.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:35.476 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:52:35.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:35.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:35.476 | INFO | Uploaded 36GiB to GPU in 2.0225s (17.7998GiB/s) | 20260506 12:52:35.476 | INFO | Downloaded 34GiB to GPU in 1.94336s (17.4955GiB/s) | 20260506 12:52:40.615 | INFO | Analyzing chunk frames = [1500672,1501184) | 20260506 12:52:40.615 | INFO | MemInfo: { | 20260506 12:52:40.615 | INFO | "gpu" : " 17 GB", | 20260506 12:52:40.615 | INFO | "huge" : "", | 20260506 12:52:40.615 | INFO | "huge_shared" : " 27 GB", | 20260506 12:52:40.615 | INFO | "malloc" : " 14 GB", | 20260506 12:52:40.615 | INFO | "pinned" : " 29 GB" | 20260506 12:52:40.615 | INFO | } | 20260506 12:52:40.615 | INFO | Perf report for the last 5.13888 seconds: | 20260506 12:52:40.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:52:40.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:40.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:40.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:40.615 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 12:52:40.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:40.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:40.615 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:52:40.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:40.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:40.616 | INFO | Uploaded 36GiB to GPU in 1.99713s (18.0258GiB/s) | 20260506 12:52:40.616 | INFO | Downloaded 34GiB to GPU in 1.93924s (17.5326GiB/s) | 20260506 12:52:45.754 | INFO | Analyzing chunk frames = [1501184,1501696) | 20260506 12:52:45.754 | INFO | MemInfo: { | 20260506 12:52:45.754 | INFO | "gpu" : " 17 GB", | 20260506 12:52:45.755 | INFO | "huge" : "", | 20260506 12:52:45.755 | INFO | "huge_shared" : " 27 GB", | 20260506 12:52:45.755 | INFO | "malloc" : " 14 GB", | 20260506 12:52:45.755 | INFO | "pinned" : " 29 GB" | 20260506 12:52:45.755 | INFO | } | 20260506 12:52:45.755 | INFO | Perf report for the last 5.1395 seconds: | 20260506 12:52:45.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:52:45.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:45.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:45.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:45.755 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 12:52:45.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:45.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:45.755 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:52:45.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:45.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:45.755 | INFO | Uploaded 36GiB to GPU in 2.03539s (17.687GiB/s) | 20260506 12:52:45.755 | INFO | Downloaded 34GiB to GPU in 1.95469s (17.394GiB/s) | 20260506 12:52:50.894 | INFO | Analyzing chunk frames = [1501696,1502208) | 20260506 12:52:50.894 | INFO | MemInfo: { | 20260506 12:52:50.894 | INFO | "gpu" : " 17 GB", | 20260506 12:52:50.894 | INFO | "huge" : "", | 20260506 12:52:50.894 | INFO | "huge_shared" : " 27 GB", | 20260506 12:52:50.894 | INFO | "malloc" : " 14 GB", | 20260506 12:52:50.894 | INFO | "pinned" : " 29 GB" | 20260506 12:52:50.894 | INFO | } | 20260506 12:52:50.894 | INFO | Perf report for the last 5.13948 seconds: | 20260506 12:52:50.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:52:50.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:50.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:50.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:50.894 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 12:52:50.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:50.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:50.894 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:52:50.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:50.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:50.895 | INFO | Uploaded 36GiB to GPU in 2.0375s (17.6687GiB/s) | 20260506 12:52:50.895 | INFO | Downloaded 34GiB to GPU in 1.93157s (17.6023GiB/s) | 20260506 12:52:56.035 | INFO | Analyzing chunk frames = [1502208,1502720) | 20260506 12:52:56.035 | INFO | MemInfo: { | 20260506 12:52:56.035 | INFO | "gpu" : " 17 GB", | 20260506 12:52:56.035 | INFO | "huge" : "", | 20260506 12:52:56.035 | INFO | "huge_shared" : " 27 GB", | 20260506 12:52:56.035 | INFO | "malloc" : " 14 GB", | 20260506 12:52:56.035 | INFO | "pinned" : " 29 GB" | 20260506 12:52:56.035 | INFO | } | 20260506 12:52:56.035 | INFO | Perf report for the last 5.14072 seconds: | 20260506 12:52:56.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:52:56.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:56.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:56.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:56.035 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:52:56.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:56.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:56.035 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:52:56.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:56.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:52:56.035 | INFO | Uploaded 36GiB to GPU in 2.03132s (17.7225GiB/s) | 20260506 12:52:56.035 | INFO | Downloaded 34GiB to GPU in 1.99513s (17.0415GiB/s) | 20260506 12:53:01.172 | INFO | Analyzing chunk frames = [1502720,1503232) | 20260506 12:53:01.172 | INFO | MemInfo: { | 20260506 12:53:01.172 | INFO | "gpu" : " 17 GB", | 20260506 12:53:01.173 | INFO | "huge" : "", | 20260506 12:53:01.173 | INFO | "huge_shared" : " 27 GB", | 20260506 12:53:01.173 | INFO | "malloc" : " 14 GB", | 20260506 12:53:01.173 | INFO | "pinned" : " 29 GB" | 20260506 12:53:01.173 | INFO | } | 20260506 12:53:01.173 | INFO | Perf report for the last 5.13805 seconds: | 20260506 12:53:01.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:53:01.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:01.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:01.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:01.173 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 12:53:01.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:01.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:01.173 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260506 12:53:01.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:01.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:01.173 | INFO | Uploaded 36GiB to GPU in 2.0461s (17.5944GiB/s) | 20260506 12:53:01.173 | INFO | Downloaded 34GiB to GPU in 2.01662s (16.8599GiB/s) | 20260506 12:53:01.308 | INFO | Starting a round of Baz IO | 20260506 12:53:04.463 | INFO | Populating RTMetrics report | 20260506 12:53:04.464 | INFO | RTMetrics report populated | 20260506 12:53:04.464 | INFO | Opening tmp RTMetrics json file | 20260506 12:53:04.464 | INFO | Serializing RTMetrics to json | 20260506 12:53:04.465 | INFO | Serialization done, swapping tmp file now... | 20260506 12:53:04.465 | INFO | RTMetrics done | 20260506 12:53:04.560 | INFO | Finished a round of Baz IO: Wrote 2.101 GiB in 3.251 seconds | 20260506 12:53:04.560 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.70% / 0.02% / 59.28% | 20260506 12:53:04.560 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.59268% / 2.78155e-05% / 7.94729e-05% | 20260506 12:53:04.560 | INFO | Baz IO overall: | 20260506 12:53:04.560 | INFO | BytesWritten : 754586180721 | 20260506 12:53:04.561 | INFO | NumEvents : 364816688554 | 20260506 12:53:06.312 | INFO | Analyzing chunk frames = [1503232,1503744) | 20260506 12:53:06.312 | INFO | MemInfo: { | 20260506 12:53:06.312 | INFO | "gpu" : " 17 GB", | 20260506 12:53:06.312 | INFO | "huge" : "", | 20260506 12:53:06.312 | INFO | "huge_shared" : " 27 GB", | 20260506 12:53:06.312 | INFO | "malloc" : " 14 GB", | 20260506 12:53:06.312 | INFO | "pinned" : " 29 GB" | 20260506 12:53:06.312 | INFO | } | 20260506 12:53:06.312 | INFO | Perf report for the last 5.13908 seconds: | 20260506 12:53:06.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:53:06.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:06.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:06.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:06.312 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260506 12:53:06.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:06.312 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 12:53:06.312 | INFO | PRE_HQ : 61.3%, 1.9, 61.4ms, 0 | 20260506 12:53:06.312 | INFO | BAZWRITER : 62.7%, 3.7, 119.4ms, 0 | 20260506 12:53:06.312 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:53:06.312 | INFO | Uploaded 36GiB to GPU in 2.03612s (17.6807GiB/s) | 20260506 12:53:06.312 | INFO | Downloaded 38GiB to GPU in 2.21721s (17.1387GiB/s) | 20260506 12:53:11.451 | INFO | Analyzing chunk frames = [1503744,1504256) | 20260506 12:53:11.451 | INFO | MemInfo: { | 20260506 12:53:11.451 | INFO | "gpu" : " 17 GB", | 20260506 12:53:11.451 | INFO | "huge" : "", | 20260506 12:53:11.451 | INFO | "huge_shared" : " 27 GB", | 20260506 12:53:11.451 | INFO | "malloc" : " 14 GB", | 20260506 12:53:11.451 | INFO | "pinned" : " 29 GB" | 20260506 12:53:11.451 | INFO | } | 20260506 12:53:11.451 | INFO | Perf report for the last 5.13921 seconds: | 20260506 12:53:11.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:53:11.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:11.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:11.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:11.451 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:53:11.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:11.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:11.451 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:53:11.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:11.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:11.451 | INFO | Uploaded 36GiB to GPU in 2.02826s (17.7492GiB/s) | 20260506 12:53:11.452 | INFO | Downloaded 34GiB to GPU in 1.94026s (17.5234GiB/s) | 20260506 12:53:16.590 | INFO | Analyzing chunk frames = [1504256,1504768) | 20260506 12:53:16.590 | INFO | MemInfo: { | 20260506 12:53:16.590 | INFO | "gpu" : " 17 GB", | 20260506 12:53:16.590 | INFO | "huge" : "", | 20260506 12:53:16.590 | INFO | "huge_shared" : " 27 GB", | 20260506 12:53:16.590 | INFO | "malloc" : " 14 GB", | 20260506 12:53:16.590 | INFO | "pinned" : " 29 GB" | 20260506 12:53:16.590 | INFO | } | 20260506 12:53:16.590 | INFO | Perf report for the last 5.13931 seconds: | 20260506 12:53:16.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:53:16.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:16.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:16.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:16.591 | INFO | ANALYSIS : 61.7%, 3.0, 95.5ms, 0 | 20260506 12:53:16.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:16.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:16.591 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:53:16.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:16.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:16.591 | INFO | Uploaded 36GiB to GPU in 1.99952s (18.0043GiB/s) | 20260506 12:53:16.591 | INFO | Downloaded 34GiB to GPU in 1.93007s (17.6159GiB/s) | 20260506 12:53:21.730 | INFO | Analyzing chunk frames = [1504768,1505280) | 20260506 12:53:21.730 | INFO | MemInfo: { | 20260506 12:53:21.730 | INFO | "gpu" : " 17 GB", | 20260506 12:53:21.730 | INFO | "huge" : "", | 20260506 12:53:21.730 | INFO | "huge_shared" : " 27 GB", | 20260506 12:53:21.730 | INFO | "malloc" : " 14 GB", | 20260506 12:53:21.730 | INFO | "pinned" : " 29 GB" | 20260506 12:53:21.730 | INFO | } | 20260506 12:53:21.730 | INFO | Perf report for the last 5.13966 seconds: | 20260506 12:53:21.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:53:21.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:21.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:21.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:21.730 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 12:53:21.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:21.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:21.730 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:53:21.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:21.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:21.731 | INFO | Uploaded 36GiB to GPU in 1.98719s (18.1161GiB/s) | 20260506 12:53:21.731 | INFO | Downloaded 34GiB to GPU in 1.94175s (17.51GiB/s) | 20260506 12:53:26.869 | INFO | Analyzing chunk frames = [1505280,1505792) | 20260506 12:53:26.869 | INFO | MemInfo: { | 20260506 12:53:26.869 | INFO | "gpu" : " 17 GB", | 20260506 12:53:26.869 | INFO | "huge" : "", | 20260506 12:53:26.869 | INFO | "huge_shared" : " 27 GB", | 20260506 12:53:26.869 | INFO | "malloc" : " 14 GB", | 20260506 12:53:26.869 | INFO | "pinned" : " 29 GB" | 20260506 12:53:26.869 | INFO | } | 20260506 12:53:26.869 | INFO | Perf report for the last 5.13912 seconds: | 20260506 12:53:26.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:53:26.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:26.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:26.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:26.869 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 12:53:26.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:26.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:26.869 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:53:26.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:26.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:26.870 | INFO | Uploaded 36GiB to GPU in 2.04007s (17.6464GiB/s) | 20260506 12:53:26.870 | INFO | Downloaded 34GiB to GPU in 1.95713s (17.3723GiB/s) | 20260506 12:53:32.009 | INFO | Analyzing chunk frames = [1505792,1506304) | 20260506 12:53:32.009 | INFO | MemInfo: { | 20260506 12:53:32.009 | INFO | "gpu" : " 17 GB", | 20260506 12:53:32.009 | INFO | "huge" : "", | 20260506 12:53:32.009 | INFO | "huge_shared" : " 27 GB", | 20260506 12:53:32.009 | INFO | "malloc" : " 14 GB", | 20260506 12:53:32.009 | INFO | "pinned" : " 29 GB" | 20260506 12:53:32.009 | INFO | } | 20260506 12:53:32.009 | INFO | Perf report for the last 5.13945 seconds: | 20260506 12:53:32.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:53:32.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:32.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:32.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:32.009 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 12:53:32.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:32.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:32.009 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:53:32.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:32.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:32.009 | INFO | Uploaded 36GiB to GPU in 2.07969s (17.3103GiB/s) | 20260506 12:53:32.009 | INFO | Downloaded 34GiB to GPU in 1.98601s (17.1198GiB/s) | 20260506 12:53:37.149 | INFO | Analyzing chunk frames = [1506304,1506816) | 20260506 12:53:37.149 | INFO | MemInfo: { | 20260506 12:53:37.149 | INFO | "gpu" : " 17 GB", | 20260506 12:53:37.149 | INFO | "huge" : "", | 20260506 12:53:37.149 | INFO | "huge_shared" : " 27 GB", | 20260506 12:53:37.149 | INFO | "malloc" : " 14 GB", | 20260506 12:53:37.149 | INFO | "pinned" : " 29 GB" | 20260506 12:53:37.149 | INFO | } | 20260506 12:53:37.149 | INFO | Perf report for the last 5.14014 seconds: | 20260506 12:53:37.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:53:37.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:37.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:37.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:37.149 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:53:37.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:37.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:37.149 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 12:53:37.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:37.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:37.149 | INFO | Uploaded 36GiB to GPU in 1.9899s (18.0913GiB/s) | 20260506 12:53:37.149 | INFO | Downloaded 34GiB to GPU in 2.04126s (16.6564GiB/s) | 20260506 12:53:42.287 | INFO | Analyzing chunk frames = [1506816,1507328) | 20260506 12:53:42.287 | INFO | MemInfo: { | 20260506 12:53:42.287 | INFO | "gpu" : " 17 GB", | 20260506 12:53:42.287 | INFO | "huge" : "", | 20260506 12:53:42.287 | INFO | "huge_shared" : " 27 GB", | 20260506 12:53:42.287 | INFO | "malloc" : " 14 GB", | 20260506 12:53:42.287 | INFO | "pinned" : " 29 GB" | 20260506 12:53:42.287 | INFO | } | 20260506 12:53:42.288 | INFO | Perf report for the last 5.13859 seconds: | 20260506 12:53:42.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:53:42.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:42.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:42.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:42.288 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:53:42.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:42.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:42.288 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:53:42.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:42.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:42.288 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:53:42.288 | INFO | Total mem usage: 16.8 GiB | 20260506 12:53:42.288 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:53:42.288 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -16.3%) | 20260506 12:53:42.288 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:53:42.288 | INFO | High Water Breakdown: | 20260506 12:53:42.288 | INFO | 59.7% : 8.63 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:53:42.288 | INFO | 40.2% : 5.81 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:53:42.288 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:53:42.288 | INFO | ------------------------------------------ | 20260506 12:53:42.288 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:53:42.288 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:53:42.288 | INFO | Total mem usage: 26.5 GiB | 20260506 12:53:42.288 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:53:42.288 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:53:42.288 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:53:42.288 | INFO | High Water Breakdown: | 20260506 12:53:42.288 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:53:42.288 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:53:42.288 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:53:42.288 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:53:42.288 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:53:42.288 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:53:42.288 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:53:42.288 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:53:42.288 | INFO | Uploaded 36GiB to GPU in 2.02115s (17.8116GiB/s) | 20260506 12:53:42.288 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:53:42.288 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:53:42.288 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:53:42.288 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:53:42.288 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:53:42.288 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:53:42.288 | INFO | ------------------------------------------ | 20260506 12:53:42.288 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:53:42.288 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:53:42.288 | INFO | Downloaded 34GiB to GPU in 1.963s (17.3204GiB/s) | 20260506 12:53:42.288 | INFO | Total mem usage: 25.6 GiB | 20260506 12:53:42.288 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:53:42.288 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:53:42.288 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:53:42.288 | INFO | High Water Breakdown: | 20260506 12:53:42.288 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:53:42.288 | INFO | ------------------------------------------ | 20260506 12:53:42.288 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:53:42.288 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:53:42.288 | INFO | Total mem usage: 15.7 GiB | 20260506 12:53:42.288 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:53:42.288 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:53:42.288 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 12:53:42.288 | INFO | High Water Breakdown: | 20260506 12:53:42.288 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:53:42.288 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:53:42.288 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:53:42.288 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:53:42.288 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:53:42.288 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:53:42.288 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:53:42.288 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:53:42.288 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:53:42.288 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:53:42.288 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:53:42.288 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:53:42.289 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:53:42.289 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:53:42.289 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:53:42.289 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:53:42.289 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:53:42.289 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:53:42.289 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:53:42.289 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:53:42.289 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:53:42.289 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:53:42.289 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:53:42.289 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:53:42.289 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:53:42.289 | INFO | ------------------------------------------ | 20260506 12:53:42.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:53:42.289 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:53:42.289 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:53:42.289 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:53:42.289 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.57e+01, 1.36e+01 / 2.51e+00 | 20260506 12:53:42.289 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.00e-01, 1.03e-02 / 7.40e-03 | 20260506 12:53:42.289 | INFO | PerfReport: 11.55%, Baseline -- 3.66e+00 / 4.37e+00, 5.77e-01 / 4.06e-01 | 20260506 12:53:42.289 | INFO | PerfReport: 3.23%, Histogram -- 1.03e+00 / 1.45e+00, 1.89e-01 / 1.59e-01 | 20260506 12:53:42.289 | INFO | PerfReport: 15.78%, DME -- 4.00e+01 / 4.40e+01, 3.90e+00 / 2.42e+00 | 20260506 12:53:42.289 | INFO | PerfReport: 56.58%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.50e+00 / 1.82e+00 | 20260506 12:53:42.289 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.39e+00 / 2.99e+00, 4.18e-01 / 2.74e-01 | 20260506 12:53:42.289 | INFO | PerfReport: 3.71%, Metrics -- 1.18e+00 / 1.57e+00, 3.04e-01 / 2.11e-01 | 20260506 12:53:42.289 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.15e-01 / 4.71e-01, 6.49e-02 / 3.38e-02 | 20260506 12:53:42.289 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:53:42.289 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:53:42.289 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:53:42.289 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 12:53:42.289 | INFO | PerfReport: 21.40%, Upload -- 2.11e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260506 12:53:42.289 | INFO | PerfReport: 21.07%, Download -- 2.08e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 12:53:42.289 | INFO | PerfReport: 32.17%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 12:53:42.289 | INFO | PerfReport: 1.64%, LockUpload -- 1.61e+00 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260506 12:53:42.289 | INFO | PerfReport: 1.17%, LockDownload -- 1.15e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260506 12:53:42.289 | INFO | PerfReport: 22.55%, LockCompute -- 2.22e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 12:53:42.289 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260506 12:53:42.415 | INFO | Starting a round of Baz IO | 20260506 12:53:45.577 | INFO | Populating RTMetrics report | 20260506 12:53:45.577 | INFO | RTMetrics report populated | 20260506 12:53:45.577 | INFO | Opening tmp RTMetrics json file | 20260506 12:53:45.578 | INFO | Serializing RTMetrics to json | 20260506 12:53:45.579 | INFO | Serialization done, swapping tmp file now... | 20260506 12:53:45.579 | INFO | RTMetrics done | 20260506 12:53:45.676 | INFO | Finished a round of Baz IO: Wrote 2.103 GiB in 3.26 seconds | 20260506 12:53:45.676 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.68% / 0.02% / 59.30% | 20260506 12:53:45.676 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.58643% / 2.38419e-05% / 5.16574e-05% | 20260506 12:53:45.676 | INFO | Baz IO overall: | 20260506 12:53:45.676 | INFO | BytesWritten : 756843915920 | 20260506 12:53:45.676 | INFO | NumEvents : 365912042672 | 20260506 12:53:47.427 | INFO | Analyzing chunk frames = [1507328,1507840) | 20260506 12:53:47.427 | INFO | MemInfo: { | 20260506 12:53:47.427 | INFO | "gpu" : " 17 GB", | 20260506 12:53:47.427 | INFO | "huge" : "", | 20260506 12:53:47.427 | INFO | "huge_shared" : " 27 GB", | 20260506 12:53:47.427 | INFO | "malloc" : " 14 GB", | 20260506 12:53:47.427 | INFO | "pinned" : " 29 GB" | 20260506 12:53:47.427 | INFO | } | 20260506 12:53:47.427 | INFO | Perf report for the last 5.13973 seconds: | 20260506 12:53:47.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:53:47.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:47.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:47.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:47.427 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260506 12:53:47.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:47.427 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:53:47.427 | INFO | PRE_HQ : 61.3%, 1.9, 61.3ms, 0 | 20260506 12:53:47.427 | INFO | BAZWRITER : 62.8%, 3.7, 118.8ms, 0 | 20260506 12:53:47.427 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:53:47.427 | INFO | Uploaded 36GiB to GPU in 2.03942s (17.6521GiB/s) | 20260506 12:53:47.428 | INFO | Downloaded 38GiB to GPU in 2.22064s (17.1122GiB/s) | 20260506 12:53:52.566 | INFO | Analyzing chunk frames = [1507840,1508352) | 20260506 12:53:52.566 | INFO | MemInfo: { | 20260506 12:53:52.566 | INFO | "gpu" : " 17 GB", | 20260506 12:53:52.566 | INFO | "huge" : "", | 20260506 12:53:52.566 | INFO | "huge_shared" : " 27 GB", | 20260506 12:53:52.566 | INFO | "malloc" : " 14 GB", | 20260506 12:53:52.566 | INFO | "pinned" : " 29 GB" | 20260506 12:53:52.566 | INFO | } | 20260506 12:53:52.566 | INFO | Perf report for the last 5.13885 seconds: | 20260506 12:53:52.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:53:52.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:52.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:52.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:52.566 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:53:52.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:52.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:52.566 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:53:52.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:52.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:52.566 | INFO | Uploaded 36GiB to GPU in 2.03622s (17.6798GiB/s) | 20260506 12:53:52.567 | INFO | Downloaded 34GiB to GPU in 1.94593s (17.4723GiB/s) | 20260506 12:53:57.706 | INFO | Analyzing chunk frames = [1508352,1508864) | 20260506 12:53:57.706 | INFO | MemInfo: { | 20260506 12:53:57.706 | INFO | "gpu" : " 17 GB", | 20260506 12:53:57.706 | INFO | "huge" : "", | 20260506 12:53:57.706 | INFO | "huge_shared" : " 27 GB", | 20260506 12:53:57.706 | INFO | "malloc" : " 14 GB", | 20260506 12:53:57.706 | INFO | "pinned" : " 29 GB" | 20260506 12:53:57.706 | INFO | } | 20260506 12:53:57.706 | INFO | Perf report for the last 5.13993 seconds: | 20260506 12:53:57.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:53:57.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:57.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:57.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:57.706 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 12:53:57.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:57.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:57.706 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:53:57.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:57.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:53:57.706 | INFO | Uploaded 36GiB to GPU in 2.02359s (17.7901GiB/s) | 20260506 12:53:57.706 | INFO | Downloaded 34GiB to GPU in 1.94509s (17.4799GiB/s) | 20260506 12:54:02.845 | INFO | Analyzing chunk frames = [1508864,1509376) | 20260506 12:54:02.845 | INFO | MemInfo: { | 20260506 12:54:02.845 | INFO | "gpu" : " 17 GB", | 20260506 12:54:02.845 | INFO | "huge" : "", | 20260506 12:54:02.845 | INFO | "huge_shared" : " 27 GB", | 20260506 12:54:02.845 | INFO | "malloc" : " 14 GB", | 20260506 12:54:02.845 | INFO | "pinned" : " 29 GB" | 20260506 12:54:02.845 | INFO | } | 20260506 12:54:02.845 | INFO | Perf report for the last 5.13887 seconds: | 20260506 12:54:02.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:54:02.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:02.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:02.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:02.845 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 12:54:02.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:02.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:02.845 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:54:02.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:02.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:02.845 | INFO | Uploaded 36GiB to GPU in 1.98027s (18.1793GiB/s) | 20260506 12:54:02.845 | INFO | Downloaded 34GiB to GPU in 1.94472s (17.4832GiB/s) | 20260506 12:54:07.984 | INFO | Analyzing chunk frames = [1509376,1509888) | 20260506 12:54:07.984 | INFO | MemInfo: { | 20260506 12:54:07.984 | INFO | "gpu" : " 17 GB", | 20260506 12:54:07.984 | INFO | "huge" : "", | 20260506 12:54:07.984 | INFO | "huge_shared" : " 27 GB", | 20260506 12:54:07.984 | INFO | "malloc" : " 14 GB", | 20260506 12:54:07.984 | INFO | "pinned" : " 29 GB" | 20260506 12:54:07.984 | INFO | } | 20260506 12:54:07.984 | INFO | Perf report for the last 5.13935 seconds: | 20260506 12:54:07.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:54:07.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:07.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:07.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:07.984 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 12:54:07.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:07.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:07.984 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 12:54:07.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:07.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:07.985 | INFO | Uploaded 36GiB to GPU in 2.03209s (17.7157GiB/s) | 20260506 12:54:07.985 | INFO | Downloaded 34GiB to GPU in 1.94977s (17.4379GiB/s) | 20260506 12:54:13.124 | INFO | Analyzing chunk frames = [1509888,1510400) | 20260506 12:54:13.124 | INFO | MemInfo: { | 20260506 12:54:13.124 | INFO | "gpu" : " 17 GB", | 20260506 12:54:13.124 | INFO | "huge" : "", | 20260506 12:54:13.124 | INFO | "huge_shared" : " 27 GB", | 20260506 12:54:13.124 | INFO | "malloc" : " 14 GB", | 20260506 12:54:13.124 | INFO | "pinned" : " 29 GB" | 20260506 12:54:13.124 | INFO | } | 20260506 12:54:13.124 | INFO | Perf report for the last 5.13953 seconds: | 20260506 12:54:13.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:54:13.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:13.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:13.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:13.124 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 12:54:13.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:13.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:13.124 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260506 12:54:13.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:13.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:13.124 | INFO | Uploaded 36GiB to GPU in 2.01837s (17.8362GiB/s) | 20260506 12:54:13.124 | INFO | Downloaded 34GiB to GPU in 1.90883s (17.8119GiB/s) | 20260506 12:54:18.264 | INFO | Analyzing chunk frames = [1510400,1510912) | 20260506 12:54:18.264 | INFO | MemInfo: { | 20260506 12:54:18.264 | INFO | "gpu" : " 17 GB", | 20260506 12:54:18.264 | INFO | "huge" : "", | 20260506 12:54:18.264 | INFO | "huge_shared" : " 27 GB", | 20260506 12:54:18.264 | INFO | "malloc" : " 14 GB", | 20260506 12:54:18.264 | INFO | "pinned" : " 29 GB" | 20260506 12:54:18.264 | INFO | } | 20260506 12:54:18.264 | INFO | Perf report for the last 5.1399 seconds: | 20260506 12:54:18.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:54:18.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:18.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:18.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:18.264 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 12:54:18.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:18.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:18.264 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 12:54:18.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:18.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:18.264 | INFO | Uploaded 36GiB to GPU in 2.00575s (17.9484GiB/s) | 20260506 12:54:18.264 | INFO | Downloaded 34GiB to GPU in 2.01242s (16.8951GiB/s) | 20260506 12:54:23.402 | INFO | Analyzing chunk frames = [1510912,1511424) | 20260506 12:54:23.402 | INFO | MemInfo: { | 20260506 12:54:23.402 | INFO | "gpu" : " 17 GB", | 20260506 12:54:23.402 | INFO | "huge" : "", | 20260506 12:54:23.402 | INFO | "huge_shared" : " 27 GB", | 20260506 12:54:23.402 | INFO | "malloc" : " 14 GB", | 20260506 12:54:23.402 | INFO | "pinned" : " 29 GB" | 20260506 12:54:23.402 | INFO | } | 20260506 12:54:23.402 | INFO | Perf report for the last 5.13856 seconds: | 20260506 12:54:23.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:54:23.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:23.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:23.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:23.402 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260506 12:54:23.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:23.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:23.402 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:54:23.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:23.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:23.403 | INFO | Uploaded 36GiB to GPU in 2.07013s (17.3902GiB/s) | 20260506 12:54:23.403 | INFO | Downloaded 34GiB to GPU in 1.99093s (17.0775GiB/s) | 20260506 12:54:23.533 | INFO | Starting a round of Baz IO | 20260506 12:54:26.699 | INFO | Populating RTMetrics report | 20260506 12:54:26.699 | INFO | RTMetrics report populated | 20260506 12:54:26.699 | INFO | Opening tmp RTMetrics json file | 20260506 12:54:26.700 | INFO | Serializing RTMetrics to json | 20260506 12:54:26.701 | INFO | Serialization done, swapping tmp file now... | 20260506 12:54:26.701 | INFO | RTMetrics done | 20260506 12:54:26.804 | INFO | Finished a round of Baz IO: Wrote 2.1 GiB in 3.271 seconds | 20260506 12:54:26.804 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.66% / 0.02% / 59.32% | 20260506 12:54:26.804 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.56559% / 3.57628e-05% / 3.57628e-05% | 20260506 12:54:26.804 | INFO | Baz IO overall: | 20260506 12:54:26.804 | INFO | BytesWritten : 759098855940 | 20260506 12:54:26.804 | INFO | NumEvents : 367005387350 | 20260506 12:54:28.542 | INFO | Analyzing chunk frames = [1511424,1511936) | 20260506 12:54:28.542 | INFO | MemInfo: { | 20260506 12:54:28.542 | INFO | "gpu" : " 17 GB", | 20260506 12:54:28.542 | INFO | "huge" : "", | 20260506 12:54:28.542 | INFO | "huge_shared" : " 27 GB", | 20260506 12:54:28.542 | INFO | "malloc" : " 14 GB", | 20260506 12:54:28.542 | INFO | "pinned" : " 29 GB" | 20260506 12:54:28.542 | INFO | } | 20260506 12:54:28.542 | INFO | Perf report for the last 5.13929 seconds: | 20260506 12:54:28.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:54:28.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:28.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:28.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:28.542 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260506 12:54:28.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:28.542 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 12:54:28.542 | INFO | PRE_HQ : 61.3%, 1.9, 60.9ms, 0 | 20260506 12:54:28.542 | INFO | BAZWRITER : 63.0%, 3.7, 119.1ms, 0 | 20260506 12:54:28.542 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:54:28.542 | INFO | Uploaded 36GiB to GPU in 2.06214s (17.4576GiB/s) | 20260506 12:54:28.542 | INFO | Downloaded 38GiB to GPU in 2.20952s (17.1983GiB/s) | 20260506 12:54:33.681 | INFO | Analyzing chunk frames = [1511936,1512448) | 20260506 12:54:33.681 | INFO | MemInfo: { | 20260506 12:54:33.681 | INFO | "gpu" : " 17 GB", | 20260506 12:54:33.681 | INFO | "huge" : "", | 20260506 12:54:33.681 | INFO | "huge_shared" : " 27 GB", | 20260506 12:54:33.681 | INFO | "malloc" : " 14 GB", | 20260506 12:54:33.681 | INFO | "pinned" : " 29 GB" | 20260506 12:54:33.681 | INFO | } | 20260506 12:54:33.681 | INFO | Perf report for the last 5.13962 seconds: | 20260506 12:54:33.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:54:33.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:33.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:33.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:33.681 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:54:33.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:33.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:33.681 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:54:33.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:33.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:33.682 | INFO | Uploaded 36GiB to GPU in 2.05225s (17.5417GiB/s) | 20260506 12:54:33.682 | INFO | Downloaded 34GiB to GPU in 1.9521s (17.4172GiB/s) | 20260506 12:54:38.820 | INFO | Analyzing chunk frames = [1512448,1512960) | 20260506 12:54:38.820 | INFO | MemInfo: { | 20260506 12:54:38.820 | INFO | "gpu" : " 17 GB", | 20260506 12:54:38.820 | INFO | "huge" : "", | 20260506 12:54:38.820 | INFO | "huge_shared" : " 27 GB", | 20260506 12:54:38.820 | INFO | "malloc" : " 14 GB", | 20260506 12:54:38.820 | INFO | "pinned" : " 29 GB" | 20260506 12:54:38.820 | INFO | } | 20260506 12:54:38.820 | INFO | Perf report for the last 5.13901 seconds: | 20260506 12:54:38.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:54:38.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:38.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:38.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:38.820 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260506 12:54:38.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:38.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:38.820 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:54:38.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:38.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:38.821 | INFO | Uploaded 36GiB to GPU in 2.04541s (17.6004GiB/s) | 20260506 12:54:38.821 | INFO | Downloaded 34GiB to GPU in 1.9381s (17.543GiB/s) | 20260506 12:54:43.960 | INFO | Analyzing chunk frames = [1512960,1513472) | 20260506 12:54:43.960 | INFO | MemInfo: { | 20260506 12:54:43.960 | INFO | "gpu" : " 17 GB", | 20260506 12:54:43.960 | INFO | "huge" : "", | 20260506 12:54:43.960 | INFO | "huge_shared" : " 27 GB", | 20260506 12:54:43.960 | INFO | "malloc" : " 14 GB", | 20260506 12:54:43.960 | INFO | "pinned" : " 29 GB" | 20260506 12:54:43.960 | INFO | } | 20260506 12:54:43.960 | INFO | Perf report for the last 5.13946 seconds: | 20260506 12:54:43.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:54:43.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:43.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:43.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:43.960 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 12:54:43.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:43.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:43.960 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 12:54:43.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:43.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:43.960 | INFO | Uploaded 36GiB to GPU in 2.02318s (17.7937GiB/s) | 20260506 12:54:43.960 | INFO | Downloaded 34GiB to GPU in 1.94074s (17.5191GiB/s) | 20260506 12:54:49.099 | INFO | Analyzing chunk frames = [1513472,1513984) | 20260506 12:54:49.099 | INFO | MemInfo: { | 20260506 12:54:49.099 | INFO | "gpu" : " 17 GB", | 20260506 12:54:49.099 | INFO | "huge" : "", | 20260506 12:54:49.099 | INFO | "huge_shared" : " 27 GB", | 20260506 12:54:49.099 | INFO | "malloc" : " 14 GB", | 20260506 12:54:49.099 | INFO | "pinned" : " 29 GB" | 20260506 12:54:49.099 | INFO | } | 20260506 12:54:49.099 | INFO | Perf report for the last 5.13944 seconds: | 20260506 12:54:49.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:54:49.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:49.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:49.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:49.099 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 12:54:49.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:49.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:49.099 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 12:54:49.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:49.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:49.100 | INFO | Uploaded 36GiB to GPU in 2.06941s (17.3963GiB/s) | 20260506 12:54:49.100 | INFO | Downloaded 34GiB to GPU in 1.95039s (17.4324GiB/s) | 20260506 12:54:54.239 | INFO | Analyzing chunk frames = [1513984,1514496) | 20260506 12:54:54.239 | INFO | MemInfo: { | 20260506 12:54:54.239 | INFO | "gpu" : " 17 GB", | 20260506 12:54:54.239 | INFO | "huge" : "", | 20260506 12:54:54.239 | INFO | "huge_shared" : " 27 GB", | 20260506 12:54:54.239 | INFO | "malloc" : " 14 GB", | 20260506 12:54:54.239 | INFO | "pinned" : " 29 GB" | 20260506 12:54:54.239 | INFO | } | 20260506 12:54:54.239 | INFO | Perf report for the last 5.13972 seconds: | 20260506 12:54:54.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:54:54.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:54.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:54.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:54.239 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 12:54:54.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:54.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:54.239 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260506 12:54:54.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:54.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:54.239 | INFO | Uploaded 36GiB to GPU in 2.06657s (17.4202GiB/s) | 20260506 12:54:54.239 | INFO | Downloaded 34GiB to GPU in 1.93269s (17.592GiB/s) | 20260506 12:54:59.378 | INFO | Analyzing chunk frames = [1514496,1515008) | 20260506 12:54:59.378 | INFO | MemInfo: { | 20260506 12:54:59.378 | INFO | "gpu" : " 17 GB", | 20260506 12:54:59.379 | INFO | "huge" : "", | 20260506 12:54:59.379 | INFO | "huge_shared" : " 27 GB", | 20260506 12:54:59.379 | INFO | "malloc" : " 14 GB", | 20260506 12:54:59.379 | INFO | "pinned" : " 29 GB" | 20260506 12:54:59.379 | INFO | } | 20260506 12:54:59.379 | INFO | Perf report for the last 5.13968 seconds: | 20260506 12:54:59.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:54:59.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:59.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:59.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:59.379 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 12:54:59.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:59.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:59.379 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:54:59.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:59.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:54:59.379 | INFO | Uploaded 36GiB to GPU in 2.01911s (17.8296GiB/s) | 20260506 12:54:59.379 | INFO | Downloaded 34GiB to GPU in 2.02596s (16.7822GiB/s) | 20260506 12:55:04.517 | INFO | Analyzing chunk frames = [1515008,1515520) | 20260506 12:55:04.517 | INFO | MemInfo: { | 20260506 12:55:04.517 | INFO | "gpu" : " 17 GB", | 20260506 12:55:04.517 | INFO | "huge" : "", | 20260506 12:55:04.517 | INFO | "huge_shared" : " 27 GB", | 20260506 12:55:04.517 | INFO | "malloc" : " 14 GB", | 20260506 12:55:04.517 | INFO | "pinned" : " 29 GB" | 20260506 12:55:04.517 | INFO | } | 20260506 12:55:04.517 | INFO | Perf report for the last 5.13856 seconds: | 20260506 12:55:04.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:55:04.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:04.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:04.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:04.517 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:55:04.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:04.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:04.517 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:55:04.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:04.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:04.517 | INFO | Uploaded 36GiB to GPU in 2.02734s (17.7572GiB/s) | 20260506 12:55:04.518 | INFO | Downloaded 34GiB to GPU in 1.96824s (17.2743GiB/s) | 20260506 12:55:04.518 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:55:04.518 | INFO | Total mem usage: 16.8 GiB | 20260506 12:55:04.518 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:55:04.518 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.8%) | 20260506 12:55:04.518 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:55:04.518 | INFO | High Water Breakdown: | 20260506 12:55:04.518 | INFO | 59.7% : 8.59 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:55:04.518 | INFO | 40.2% : 5.78 GiB ( 86.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:55:04.518 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:55:04.518 | INFO | ------------------------------------------ | 20260506 12:55:04.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:55:04.518 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:55:04.518 | INFO | Total mem usage: 26.5 GiB | 20260506 12:55:04.518 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:55:04.518 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:55:04.518 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:55:04.518 | INFO | High Water Breakdown: | 20260506 12:55:04.518 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:55:04.518 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:55:04.518 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:55:04.518 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:55:04.518 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:55:04.518 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:55:04.518 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:55:04.518 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:55:04.518 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:55:04.518 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:55:04.518 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:55:04.518 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:55:04.518 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:55:04.518 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:55:04.518 | INFO | ------------------------------------------ | 20260506 12:55:04.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:55:04.518 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:55:04.518 | INFO | Total mem usage: 25.6 GiB | 20260506 12:55:04.518 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:55:04.518 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:55:04.518 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:55:04.518 | INFO | High Water Breakdown: | 20260506 12:55:04.518 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:55:04.518 | INFO | ------------------------------------------ | 20260506 12:55:04.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:55:04.518 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:55:04.518 | INFO | Total mem usage: 15.7 GiB | 20260506 12:55:04.518 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:55:04.518 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:55:04.518 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 12:55:04.518 | INFO | High Water Breakdown: | 20260506 12:55:04.518 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:55:04.518 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:55:04.518 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:55:04.518 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:55:04.518 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:55:04.518 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:55:04.518 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:55:04.518 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:55:04.518 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:55:04.518 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:55:04.518 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:55:04.518 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:55:04.518 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:55:04.518 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:55:04.518 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:55:04.518 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:55:04.518 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:55:04.518 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:55:04.518 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:55:04.518 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:55:04.518 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:55:04.518 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:55:04.518 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:55:04.518 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:55:04.518 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:55:04.518 | INFO | ------------------------------------------ | 20260506 12:55:04.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:55:04.518 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:55:04.518 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:55:04.518 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:55:04.518 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.60e+01, 1.35e+01 / 2.21e+00 | 20260506 12:55:04.518 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.04e-01, 1.03e-02 / 8.86e-03 | 20260506 12:55:04.518 | INFO | PerfReport: 11.58%, Baseline -- 3.67e+00 / 4.40e+00, 5.78e-01 / 3.51e-01 | 20260506 12:55:04.518 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.49e+00, 1.88e-01 / 1.19e-01 | 20260506 12:55:04.518 | INFO | PerfReport: 15.75%, DME -- 3.99e+01 / 4.40e+01, 3.90e+00 / 2.57e+00 | 20260506 12:55:04.518 | INFO | PerfReport: 56.57%, FrameLabeling -- 1.79e+01 / 2.10e+01, 1.46e+00 / 1.70e+00 | 20260506 12:55:04.518 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.39e+00 / 2.98e+00, 4.12e-01 / 2.47e-01 | 20260506 12:55:04.518 | INFO | PerfReport: 3.72%, Metrics -- 1.18e+00 / 1.56e+00, 3.03e-01 / 1.14e-01 | 20260506 12:55:04.518 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.15e-01 / 4.79e-01, 6.46e-02 / 3.07e-02 | 20260506 12:55:04.518 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:55:04.518 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:55:04.518 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:55:04.518 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 12:55:04.518 | INFO | PerfReport: 21.50%, Upload -- 2.12e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260506 12:55:04.518 | INFO | PerfReport: 21.01%, Download -- 2.07e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 12:55:04.518 | INFO | PerfReport: 32.12%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 12:55:04.519 | INFO | PerfReport: 1.77%, LockUpload -- 1.75e+00 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260506 12:55:04.519 | INFO | PerfReport: 1.05%, LockDownload -- 1.03e+00 / 0.00e+00, 2.70e+00 / 0.00e+00 | 20260506 12:55:04.519 | INFO | PerfReport: 22.53%, LockCompute -- 2.22e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:55:04.519 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260506 12:55:04.644 | INFO | Starting a round of Baz IO | 20260506 12:55:07.803 | INFO | Populating RTMetrics report | 20260506 12:55:07.803 | INFO | RTMetrics report populated | 20260506 12:55:07.803 | INFO | Opening tmp RTMetrics json file | 20260506 12:55:07.804 | INFO | Serializing RTMetrics to json | 20260506 12:55:07.805 | INFO | Serialization done, swapping tmp file now... | 20260506 12:55:07.806 | INFO | RTMetrics done | 20260506 12:55:07.897 | INFO | Finished a round of Baz IO: Wrote 2.1 GiB in 3.252 seconds | 20260506 12:55:07.897 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.63% / 0.02% / 59.34% | 20260506 12:55:07.897 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.55528% / 2.78155e-05% / 4.37101e-05% | 20260506 12:55:07.897 | INFO | Baz IO overall: | 20260506 12:55:07.897 | INFO | BytesWritten : 761353335331 | 20260506 12:55:07.897 | INFO | NumEvents : 368098248537 | 20260506 12:55:09.657 | INFO | Analyzing chunk frames = [1515520,1516032) | 20260506 12:55:09.657 | INFO | MemInfo: { | 20260506 12:55:09.657 | INFO | "gpu" : " 17 GB", | 20260506 12:55:09.657 | INFO | "huge" : "", | 20260506 12:55:09.657 | INFO | "huge_shared" : " 27 GB", | 20260506 12:55:09.657 | INFO | "malloc" : " 14 GB", | 20260506 12:55:09.657 | INFO | "pinned" : " 29 GB" | 20260506 12:55:09.657 | INFO | } | 20260506 12:55:09.657 | INFO | Perf report for the last 5.13936 seconds: | 20260506 12:55:09.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:55:09.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:09.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:09.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:09.657 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260506 12:55:09.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:09.657 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:55:09.657 | INFO | PRE_HQ : 61.4%, 1.9, 61.6ms, 0 | 20260506 12:55:09.657 | INFO | BAZWRITER : 62.7%, 3.7, 118.9ms, 0 | 20260506 12:55:09.657 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:55:09.657 | INFO | Uploaded 36GiB to GPU in 2.06437s (17.4387GiB/s) | 20260506 12:55:09.657 | INFO | Downloaded 38GiB to GPU in 2.22359s (17.0894GiB/s) | 20260506 12:55:14.796 | INFO | Analyzing chunk frames = [1516032,1516544) | 20260506 12:55:14.796 | INFO | MemInfo: { | 20260506 12:55:14.796 | INFO | "gpu" : " 17 GB", | 20260506 12:55:14.796 | INFO | "huge" : "", | 20260506 12:55:14.796 | INFO | "huge_shared" : " 27 GB", | 20260506 12:55:14.796 | INFO | "malloc" : " 14 GB", | 20260506 12:55:14.796 | INFO | "pinned" : " 29 GB" | 20260506 12:55:14.796 | INFO | } | 20260506 12:55:14.796 | INFO | Perf report for the last 5.13928 seconds: | 20260506 12:55:14.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:55:14.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:14.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:14.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:14.796 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:55:14.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:14.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:14.796 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:55:14.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:14.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:14.796 | INFO | Uploaded 36GiB to GPU in 2.01126s (17.8992GiB/s) | 20260506 12:55:14.796 | INFO | Downloaded 34GiB to GPU in 1.93017s (17.615GiB/s) | 20260506 12:55:19.935 | INFO | Analyzing chunk frames = [1516544,1517056) | 20260506 12:55:19.935 | INFO | MemInfo: { | 20260506 12:55:19.935 | INFO | "gpu" : " 17 GB", | 20260506 12:55:19.935 | INFO | "huge" : "", | 20260506 12:55:19.936 | INFO | "huge_shared" : " 27 GB", | 20260506 12:55:19.936 | INFO | "malloc" : " 14 GB", | 20260506 12:55:19.936 | INFO | "pinned" : " 29 GB" | 20260506 12:55:19.936 | INFO | } | 20260506 12:55:19.936 | INFO | Perf report for the last 5.13976 seconds: | 20260506 12:55:19.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:55:19.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:19.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:19.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:19.936 | INFO | ANALYSIS : 61.4%, 3.0, 95.2ms, 0 | 20260506 12:55:19.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:19.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:19.936 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:55:19.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:19.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:19.936 | INFO | Uploaded 36GiB to GPU in 1.96723s (18.2999GiB/s) | 20260506 12:55:19.936 | INFO | Downloaded 34GiB to GPU in 1.92969s (17.6194GiB/s) | 20260506 12:55:25.074 | INFO | Analyzing chunk frames = [1517056,1517568) | 20260506 12:55:25.074 | INFO | MemInfo: { | 20260506 12:55:25.074 | INFO | "gpu" : " 17 GB", | 20260506 12:55:25.075 | INFO | "huge" : "", | 20260506 12:55:25.075 | INFO | "huge_shared" : " 27 GB", | 20260506 12:55:25.075 | INFO | "malloc" : " 14 GB", | 20260506 12:55:25.075 | INFO | "pinned" : " 29 GB" | 20260506 12:55:25.075 | INFO | } | 20260506 12:55:25.075 | INFO | Perf report for the last 5.13899 seconds: | 20260506 12:55:25.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:55:25.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:25.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:25.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:25.075 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 12:55:25.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:25.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:25.075 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:55:25.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:25.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:25.075 | INFO | Uploaded 36GiB to GPU in 1.97817s (18.1986GiB/s) | 20260506 12:55:25.075 | INFO | Downloaded 34GiB to GPU in 1.90703s (17.8288GiB/s) | 20260506 12:55:30.214 | INFO | Analyzing chunk frames = [1517568,1518080) | 20260506 12:55:30.214 | INFO | MemInfo: { | 20260506 12:55:30.214 | INFO | "gpu" : " 17 GB", | 20260506 12:55:30.214 | INFO | "huge" : "", | 20260506 12:55:30.214 | INFO | "huge_shared" : " 27 GB", | 20260506 12:55:30.214 | INFO | "malloc" : " 14 GB", | 20260506 12:55:30.214 | INFO | "pinned" : " 29 GB" | 20260506 12:55:30.214 | INFO | } | 20260506 12:55:30.214 | INFO | Perf report for the last 5.13914 seconds: | 20260506 12:55:30.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:55:30.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:30.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:30.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:30.214 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 12:55:30.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:30.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:30.214 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:55:30.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:30.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:30.214 | INFO | Uploaded 36GiB to GPU in 1.99388s (18.0553GiB/s) | 20260506 12:55:30.214 | INFO | Downloaded 34GiB to GPU in 1.94412s (17.4886GiB/s) | 20260506 12:55:35.353 | INFO | Analyzing chunk frames = [1518080,1518592) | 20260506 12:55:35.353 | INFO | MemInfo: { | 20260506 12:55:35.353 | INFO | "gpu" : " 17 GB", | 20260506 12:55:35.353 | INFO | "huge" : "", | 20260506 12:55:35.353 | INFO | "huge_shared" : " 27 GB", | 20260506 12:55:35.353 | INFO | "malloc" : " 14 GB", | 20260506 12:55:35.353 | INFO | "pinned" : " 29 GB" | 20260506 12:55:35.353 | INFO | } | 20260506 12:55:35.353 | INFO | Perf report for the last 5.13933 seconds: | 20260506 12:55:35.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:55:35.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:35.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:35.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:35.353 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 12:55:35.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:35.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:35.353 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260506 12:55:35.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:35.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:35.353 | INFO | Uploaded 36GiB to GPU in 2.02529s (17.7753GiB/s) | 20260506 12:55:35.353 | INFO | Downloaded 34GiB to GPU in 1.92509s (17.6615GiB/s) | 20260506 12:55:40.493 | INFO | Analyzing chunk frames = [1518592,1519104) | 20260506 12:55:40.493 | INFO | MemInfo: { | 20260506 12:55:40.493 | INFO | "gpu" : " 17 GB", | 20260506 12:55:40.493 | INFO | "huge" : "", | 20260506 12:55:40.493 | INFO | "huge_shared" : " 27 GB", | 20260506 12:55:40.493 | INFO | "malloc" : " 14 GB", | 20260506 12:55:40.493 | INFO | "pinned" : " 29 GB" | 20260506 12:55:40.493 | INFO | } | 20260506 12:55:40.493 | INFO | Perf report for the last 5.13964 seconds: | 20260506 12:55:40.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:55:40.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:40.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:40.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:40.493 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:55:40.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:40.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:40.493 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 12:55:40.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:40.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:40.493 | INFO | Uploaded 36GiB to GPU in 2.00558s (17.9499GiB/s) | 20260506 12:55:40.493 | INFO | Downloaded 34GiB to GPU in 2.01016s (16.9141GiB/s) | 20260506 12:55:45.632 | INFO | Analyzing chunk frames = [1519104,1519616) | 20260506 12:55:45.632 | INFO | MemInfo: { | 20260506 12:55:45.632 | INFO | "gpu" : " 17 GB", | 20260506 12:55:45.632 | INFO | "huge" : "", | 20260506 12:55:45.632 | INFO | "huge_shared" : " 27 GB", | 20260506 12:55:45.632 | INFO | "malloc" : " 14 GB", | 20260506 12:55:45.632 | INFO | "pinned" : " 29 GB" | 20260506 12:55:45.632 | INFO | } | 20260506 12:55:45.632 | INFO | Perf report for the last 5.13906 seconds: | 20260506 12:55:45.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:55:45.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:45.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:45.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:45.632 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:55:45.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:45.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:45.632 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 12:55:45.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:45.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:45.632 | INFO | Uploaded 36GiB to GPU in 2.0398s (17.6488GiB/s) | 20260506 12:55:45.632 | INFO | Downloaded 34GiB to GPU in 1.96357s (17.3154GiB/s) | 20260506 12:55:45.759 | INFO | Starting a round of Baz IO | 20260506 12:55:48.920 | INFO | Populating RTMetrics report | 20260506 12:55:48.920 | INFO | RTMetrics report populated | 20260506 12:55:48.920 | INFO | Opening tmp RTMetrics json file | 20260506 12:55:48.921 | INFO | Serializing RTMetrics to json | 20260506 12:55:48.922 | INFO | Serialization done, swapping tmp file now... | 20260506 12:55:48.922 | INFO | RTMetrics done | 20260506 12:55:49.022 | INFO | Finished a round of Baz IO: Wrote 2.101 GiB in 3.263 seconds | 20260506 12:55:49.022 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.61% / 0.02% / 59.37% | 20260506 12:55:49.022 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.55403% / 2.38419e-05% / 4.76837e-05% | 20260506 12:55:49.022 | INFO | Baz IO overall: | 20260506 12:55:49.022 | INFO | BytesWritten : 763608954415 | 20260506 12:55:49.022 | INFO | NumEvents : 369192289889 | 20260506 12:55:50.771 | INFO | Analyzing chunk frames = [1519616,1520128) | 20260506 12:55:50.771 | INFO | MemInfo: { | 20260506 12:55:50.771 | INFO | "gpu" : " 17 GB", | 20260506 12:55:50.771 | INFO | "huge" : "", | 20260506 12:55:50.771 | INFO | "huge_shared" : " 27 GB", | 20260506 12:55:50.771 | INFO | "malloc" : " 14 GB", | 20260506 12:55:50.771 | INFO | "pinned" : " 29 GB" | 20260506 12:55:50.771 | INFO | } | 20260506 12:55:50.771 | INFO | Perf report for the last 5.13929 seconds: | 20260506 12:55:50.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:55:50.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:50.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:50.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:50.771 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260506 12:55:50.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:50.771 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:55:50.771 | INFO | PRE_HQ : 61.4%, 1.9, 61.0ms, 0 | 20260506 12:55:50.771 | INFO | BAZWRITER : 62.9%, 3.7, 118.4ms, 0 | 20260506 12:55:50.771 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 12:55:50.772 | INFO | Uploaded 36GiB to GPU in 2.06472s (17.4358GiB/s) | 20260506 12:55:50.772 | INFO | Downloaded 38GiB to GPU in 2.21036s (17.1918GiB/s) | 20260506 12:55:55.910 | INFO | Analyzing chunk frames = [1520128,1520640) | 20260506 12:55:55.910 | INFO | MemInfo: { | 20260506 12:55:55.910 | INFO | "gpu" : " 17 GB", | 20260506 12:55:55.910 | INFO | "huge" : "", | 20260506 12:55:55.911 | INFO | "huge_shared" : " 27 GB", | 20260506 12:55:55.911 | INFO | "malloc" : " 14 GB", | 20260506 12:55:55.911 | INFO | "pinned" : " 29 GB" | 20260506 12:55:55.911 | INFO | } | 20260506 12:55:55.911 | INFO | Perf report for the last 5.13942 seconds: | 20260506 12:55:55.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:55:55.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:55.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:55.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:55.911 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:55:55.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:55.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:55.911 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:55:55.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:55.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:55:55.911 | INFO | Uploaded 36GiB to GPU in 2.06776s (17.4101GiB/s) | 20260506 12:55:55.911 | INFO | Downloaded 34GiB to GPU in 1.94685s (17.4641GiB/s) | 20260506 12:56:01.050 | INFO | Analyzing chunk frames = [1520640,1521152) | 20260506 12:56:01.050 | INFO | MemInfo: { | 20260506 12:56:01.050 | INFO | "gpu" : " 17 GB", | 20260506 12:56:01.050 | INFO | "huge" : "", | 20260506 12:56:01.050 | INFO | "huge_shared" : " 27 GB", | 20260506 12:56:01.050 | INFO | "malloc" : " 14 GB", | 20260506 12:56:01.050 | INFO | "pinned" : " 29 GB" | 20260506 12:56:01.050 | INFO | } | 20260506 12:56:01.050 | INFO | Perf report for the last 5.13921 seconds: | 20260506 12:56:01.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:56:01.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:01.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:01.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:01.050 | INFO | ANALYSIS : 61.9%, 3.0, 95.8ms, 0 | 20260506 12:56:01.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:01.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:01.050 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:56:01.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:01.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:01.050 | INFO | Uploaded 36GiB to GPU in 2.02509s (17.777GiB/s) | 20260506 12:56:01.050 | INFO | Downloaded 34GiB to GPU in 1.93816s (17.5424GiB/s) | 20260506 12:56:06.189 | INFO | Analyzing chunk frames = [1521152,1521664) | 20260506 12:56:06.189 | INFO | MemInfo: { | 20260506 12:56:06.189 | INFO | "gpu" : " 17 GB", | 20260506 12:56:06.189 | INFO | "huge" : "", | 20260506 12:56:06.189 | INFO | "huge_shared" : " 27 GB", | 20260506 12:56:06.189 | INFO | "malloc" : " 14 GB", | 20260506 12:56:06.189 | INFO | "pinned" : " 29 GB" | 20260506 12:56:06.189 | INFO | } | 20260506 12:56:06.189 | INFO | Perf report for the last 5.13938 seconds: | 20260506 12:56:06.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:56:06.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:06.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:06.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:06.189 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 12:56:06.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:06.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:06.189 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:56:06.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:06.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:06.189 | INFO | Uploaded 36GiB to GPU in 1.99065s (18.0846GiB/s) | 20260506 12:56:06.190 | INFO | Downloaded 34GiB to GPU in 1.94547s (17.4765GiB/s) | 20260506 12:56:11.328 | INFO | Analyzing chunk frames = [1521664,1522176) | 20260506 12:56:11.328 | INFO | MemInfo: { | 20260506 12:56:11.328 | INFO | "gpu" : " 17 GB", | 20260506 12:56:11.328 | INFO | "huge" : "", | 20260506 12:56:11.328 | INFO | "huge_shared" : " 27 GB", | 20260506 12:56:11.328 | INFO | "malloc" : " 14 GB", | 20260506 12:56:11.328 | INFO | "pinned" : " 29 GB" | 20260506 12:56:11.328 | INFO | } | 20260506 12:56:11.328 | INFO | Perf report for the last 5.13923 seconds: | 20260506 12:56:11.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:56:11.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:11.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:11.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:11.329 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 12:56:11.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:11.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:11.329 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:56:11.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:11.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:11.329 | INFO | Uploaded 36GiB to GPU in 2.06126s (17.465GiB/s) | 20260506 12:56:11.329 | INFO | Downloaded 34GiB to GPU in 1.9503s (17.4332GiB/s) | 20260506 12:56:16.468 | INFO | Analyzing chunk frames = [1522176,1522688) | 20260506 12:56:16.468 | INFO | MemInfo: { | 20260506 12:56:16.468 | INFO | "gpu" : " 17 GB", | 20260506 12:56:16.468 | INFO | "huge" : "", | 20260506 12:56:16.468 | INFO | "huge_shared" : " 27 GB", | 20260506 12:56:16.468 | INFO | "malloc" : " 14 GB", | 20260506 12:56:16.468 | INFO | "pinned" : " 29 GB" | 20260506 12:56:16.468 | INFO | } | 20260506 12:56:16.468 | INFO | Perf report for the last 5.13942 seconds: | 20260506 12:56:16.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:56:16.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:16.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:16.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:16.468 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:56:16.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:16.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:16.468 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 12:56:16.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:16.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:16.468 | INFO | Uploaded 36GiB to GPU in 2.1125s (17.0414GiB/s) | 20260506 12:56:16.468 | INFO | Downloaded 34GiB to GPU in 1.98423s (17.1351GiB/s) | 20260506 12:56:21.608 | INFO | Analyzing chunk frames = [1522688,1523200) | 20260506 12:56:21.608 | INFO | MemInfo: { | 20260506 12:56:21.608 | INFO | "gpu" : " 17 GB", | 20260506 12:56:21.608 | INFO | "huge" : "", | 20260506 12:56:21.609 | INFO | "huge_shared" : " 27 GB", | 20260506 12:56:21.609 | INFO | "malloc" : " 14 GB", | 20260506 12:56:21.609 | INFO | "pinned" : " 29 GB" | 20260506 12:56:21.609 | INFO | } | 20260506 12:56:21.609 | INFO | Perf report for the last 5.14066 seconds: | 20260506 12:56:21.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:56:21.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:21.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:21.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:21.609 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:56:21.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:21.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:21.609 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 12:56:21.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:21.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:21.609 | INFO | Uploaded 36GiB to GPU in 2.02385s (17.7879GiB/s) | 20260506 12:56:21.609 | INFO | Downloaded 34GiB to GPU in 2.04268s (16.6448GiB/s) | 20260506 12:56:26.746 | INFO | Analyzing chunk frames = [1523200,1523712) | 20260506 12:56:26.746 | INFO | MemInfo: { | 20260506 12:56:26.746 | INFO | "gpu" : " 17 GB", | 20260506 12:56:26.746 | INFO | "huge" : "", | 20260506 12:56:26.746 | INFO | "huge_shared" : " 27 GB", | 20260506 12:56:26.746 | INFO | "malloc" : " 14 GB", | 20260506 12:56:26.747 | INFO | "pinned" : " 29 GB" | 20260506 12:56:26.747 | INFO | } | 20260506 12:56:26.747 | INFO | Perf report for the last 5.13792 seconds: | 20260506 12:56:26.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:56:26.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:26.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:26.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:26.747 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:56:26.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:26.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:26.747 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:56:26.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:26.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:26.747 | INFO | Uploaded 36GiB to GPU in 2.0436s (17.616GiB/s) | 20260506 12:56:26.747 | INFO | Downloaded 34GiB to GPU in 1.97063s (17.2534GiB/s) | 20260506 12:56:26.747 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:56:26.747 | INFO | Total mem usage: 16.8 GiB | 20260506 12:56:26.747 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:56:26.747 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.7%) | 20260506 12:56:26.747 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:56:26.747 | INFO | High Water Breakdown: | 20260506 12:56:26.747 | INFO | 59.7% : 8.6 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:56:26.747 | INFO | 40.2% : 5.78 GiB ( 86.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:56:26.747 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:56:26.747 | INFO | ------------------------------------------ | 20260506 12:56:26.747 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:56:26.747 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:56:26.747 | INFO | Total mem usage: 26.5 GiB | 20260506 12:56:26.747 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:56:26.747 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:56:26.747 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:56:26.747 | INFO | High Water Breakdown: | 20260506 12:56:26.747 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:56:26.747 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:56:26.747 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:56:26.747 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:56:26.747 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:56:26.747 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:56:26.747 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:56:26.747 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:56:26.747 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:56:26.747 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:56:26.747 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:56:26.747 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:56:26.747 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:56:26.747 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:56:26.747 | INFO | ------------------------------------------ | 20260506 12:56:26.747 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:56:26.747 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:56:26.747 | INFO | Total mem usage: 25.6 GiB | 20260506 12:56:26.747 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:56:26.747 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:56:26.747 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:56:26.747 | INFO | High Water Breakdown: | 20260506 12:56:26.747 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:56:26.747 | INFO | ------------------------------------------ | 20260506 12:56:26.747 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:56:26.747 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:56:26.747 | INFO | Total mem usage: 15.7 GiB | 20260506 12:56:26.747 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:56:26.747 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:56:26.747 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 12:56:26.747 | INFO | High Water Breakdown: | 20260506 12:56:26.747 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:56:26.747 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:56:26.747 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:56:26.747 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:56:26.747 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:56:26.747 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:56:26.747 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:56:26.747 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:56:26.747 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:56:26.747 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:56:26.747 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:56:26.747 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:56:26.747 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:56:26.747 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:56:26.747 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:56:26.747 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:56:26.747 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:56:26.747 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:56:26.747 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:56:26.747 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:56:26.747 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:56:26.747 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:56:26.747 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:56:26.748 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:56:26.748 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:56:26.748 | INFO | ------------------------------------------ | 20260506 12:56:26.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:56:26.748 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:56:26.748 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:56:26.748 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:56:26.748 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.66e+01, 1.37e+01 / 2.74e+00 | 20260506 12:56:26.748 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.00e-01, 1.04e-02 / 5.40e-03 | 20260506 12:56:26.748 | INFO | PerfReport: 11.55%, Baseline -- 3.66e+00 / 4.49e+00, 5.73e-01 / 2.67e-01 | 20260506 12:56:26.748 | INFO | PerfReport: 3.23%, Histogram -- 1.02e+00 / 1.46e+00, 1.87e-01 / 1.43e-01 | 20260506 12:56:26.748 | INFO | PerfReport: 15.87%, DME -- 4.02e+01 / 4.50e+01, 4.34e+00 / 3.16e+00 | 20260506 12:56:26.748 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.41e+00 / 1.88e+00 | 20260506 12:56:26.748 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.40e+00 / 3.05e+00, 4.21e-01 / 1.91e-01 | 20260506 12:56:26.748 | INFO | PerfReport: 3.71%, Metrics -- 1.17e+00 / 1.56e+00, 2.96e-01 / 1.61e-01 | 20260506 12:56:26.748 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.13e-01 / 4.68e-01, 6.42e-02 / 2.67e-02 | 20260506 12:56:26.748 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:56:26.748 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:56:26.748 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:56:26.748 | INFO | PerfReport: Full Execution -- 9.87e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 12:56:26.748 | INFO | PerfReport: 21.43%, Upload -- 2.11e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260506 12:56:26.748 | INFO | PerfReport: 21.00%, Download -- 2.07e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 12:56:26.748 | INFO | PerfReport: 32.13%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 12:56:26.748 | INFO | PerfReport: 1.76%, LockUpload -- 1.74e+00 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 12:56:26.748 | INFO | PerfReport: 1.08%, LockDownload -- 1.06e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260506 12:56:26.748 | INFO | PerfReport: 22.59%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:56:26.748 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260506 12:56:26.876 | INFO | Starting a round of Baz IO | 20260506 12:56:30.032 | INFO | Populating RTMetrics report | 20260506 12:56:30.033 | INFO | RTMetrics report populated | 20260506 12:56:30.033 | INFO | Opening tmp RTMetrics json file | 20260506 12:56:30.033 | INFO | Serializing RTMetrics to json | 20260506 12:56:30.034 | INFO | Serialization done, swapping tmp file now... | 20260506 12:56:30.035 | INFO | RTMetrics done | 20260506 12:56:30.131 | INFO | Finished a round of Baz IO: Wrote 2.1 GiB in 3.254 seconds | 20260506 12:56:30.131 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.59% / 0.02% / 59.39% | 20260506 12:56:30.131 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.5417% / 4.76837e-05% / 5.5631e-05% | 20260506 12:56:30.131 | INFO | Baz IO overall: | 20260506 12:56:30.131 | INFO | BytesWritten : 765864345988 | 20260506 12:56:30.131 | INFO | NumEvents : 370286090702 | 20260506 12:56:31.886 | INFO | Analyzing chunk frames = [1523712,1524224) | 20260506 12:56:31.886 | INFO | MemInfo: { | 20260506 12:56:31.886 | INFO | "gpu" : " 17 GB", | 20260506 12:56:31.886 | INFO | "huge" : "", | 20260506 12:56:31.886 | INFO | "huge_shared" : " 27 GB", | 20260506 12:56:31.886 | INFO | "malloc" : " 14 GB", | 20260506 12:56:31.886 | INFO | "pinned" : " 29 GB" | 20260506 12:56:31.886 | INFO | } | 20260506 12:56:31.886 | INFO | Perf report for the last 5.13955 seconds: | 20260506 12:56:31.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:56:31.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:31.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:31.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:31.886 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260506 12:56:31.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:31.886 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:56:31.886 | INFO | PRE_HQ : 61.3%, 1.9, 61.2ms, 0 | 20260506 12:56:31.886 | INFO | BAZWRITER : 62.7%, 3.7, 119.2ms, 0 | 20260506 12:56:31.886 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:56:31.886 | INFO | Uploaded 36GiB to GPU in 2.02304s (17.795GiB/s) | 20260506 12:56:31.886 | INFO | Downloaded 38GiB to GPU in 2.22045s (17.1137GiB/s) | 20260506 12:56:37.025 | INFO | Analyzing chunk frames = [1524224,1524736) | 20260506 12:56:37.025 | INFO | MemInfo: { | 20260506 12:56:37.025 | INFO | "gpu" : " 17 GB", | 20260506 12:56:37.025 | INFO | "huge" : "", | 20260506 12:56:37.025 | INFO | "huge_shared" : " 27 GB", | 20260506 12:56:37.025 | INFO | "malloc" : " 14 GB", | 20260506 12:56:37.025 | INFO | "pinned" : " 29 GB" | 20260506 12:56:37.025 | INFO | } | 20260506 12:56:37.025 | INFO | Perf report for the last 5.13904 seconds: | 20260506 12:56:37.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:56:37.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:37.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:37.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:37.025 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 12:56:37.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:37.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:37.025 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:56:37.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:37.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:37.025 | INFO | Uploaded 36GiB to GPU in 2.00002s (17.9999GiB/s) | 20260506 12:56:37.025 | INFO | Downloaded 34GiB to GPU in 1.93344s (17.5852GiB/s) | 20260506 12:56:42.165 | INFO | Analyzing chunk frames = [1524736,1525248) | 20260506 12:56:42.165 | INFO | MemInfo: { | 20260506 12:56:42.165 | INFO | "gpu" : " 17 GB", | 20260506 12:56:42.165 | INFO | "huge" : "", | 20260506 12:56:42.165 | INFO | "huge_shared" : " 27 GB", | 20260506 12:56:42.165 | INFO | "malloc" : " 14 GB", | 20260506 12:56:42.165 | INFO | "pinned" : " 29 GB" | 20260506 12:56:42.165 | INFO | } | 20260506 12:56:42.165 | INFO | Perf report for the last 5.13992 seconds: | 20260506 12:56:42.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:56:42.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:42.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:42.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:42.165 | INFO | ANALYSIS : 61.7%, 3.0, 95.5ms, 0 | 20260506 12:56:42.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:42.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:42.165 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:56:42.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:42.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:42.165 | INFO | Uploaded 36GiB to GPU in 2.01716s (17.8469GiB/s) | 20260506 12:56:42.165 | INFO | Downloaded 34GiB to GPU in 1.94588s (17.4728GiB/s) | 20260506 12:56:47.304 | INFO | Analyzing chunk frames = [1525248,1525760) | 20260506 12:56:47.304 | INFO | MemInfo: { | 20260506 12:56:47.304 | INFO | "gpu" : " 17 GB", | 20260506 12:56:47.304 | INFO | "huge" : "", | 20260506 12:56:47.304 | INFO | "huge_shared" : " 27 GB", | 20260506 12:56:47.304 | INFO | "malloc" : " 14 GB", | 20260506 12:56:47.304 | INFO | "pinned" : " 29 GB" | 20260506 12:56:47.304 | INFO | } | 20260506 12:56:47.304 | INFO | Perf report for the last 5.13878 seconds: | 20260506 12:56:47.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:56:47.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:47.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:47.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:47.304 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 12:56:47.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:47.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:47.304 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:56:47.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:47.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:47.304 | INFO | Uploaded 36GiB to GPU in 1.98493s (18.1367GiB/s) | 20260506 12:56:47.304 | INFO | Downloaded 34GiB to GPU in 1.93976s (17.5279GiB/s) | 20260506 12:56:52.443 | INFO | Analyzing chunk frames = [1525760,1526272) | 20260506 12:56:52.443 | INFO | MemInfo: { | 20260506 12:56:52.443 | INFO | "gpu" : " 17 GB", | 20260506 12:56:52.443 | INFO | "huge" : "", | 20260506 12:56:52.443 | INFO | "huge_shared" : " 27 GB", | 20260506 12:56:52.443 | INFO | "malloc" : " 14 GB", | 20260506 12:56:52.443 | INFO | "pinned" : " 29 GB" | 20260506 12:56:52.443 | INFO | } | 20260506 12:56:52.443 | INFO | Perf report for the last 5.13934 seconds: | 20260506 12:56:52.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:56:52.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:52.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:52.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:52.443 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 12:56:52.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:52.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:52.443 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:56:52.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:52.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:52.444 | INFO | Uploaded 36GiB to GPU in 2.05653s (17.5052GiB/s) | 20260506 12:56:52.444 | INFO | Downloaded 34GiB to GPU in 1.95503s (17.391GiB/s) | 20260506 12:56:57.582 | INFO | Analyzing chunk frames = [1526272,1526784) | 20260506 12:56:57.582 | INFO | MemInfo: { | 20260506 12:56:57.582 | INFO | "gpu" : " 17 GB", | 20260506 12:56:57.582 | INFO | "huge" : "", | 20260506 12:56:57.583 | INFO | "huge_shared" : " 27 GB", | 20260506 12:56:57.583 | INFO | "malloc" : " 14 GB", | 20260506 12:56:57.583 | INFO | "pinned" : " 29 GB" | 20260506 12:56:57.583 | INFO | } | 20260506 12:56:57.583 | INFO | Perf report for the last 5.13939 seconds: | 20260506 12:56:57.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:56:57.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:57.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:57.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:57.583 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 12:56:57.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:57.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:57.583 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260506 12:56:57.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:57.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:56:57.583 | INFO | Uploaded 36GiB to GPU in 2.03951s (17.6513GiB/s) | 20260506 12:56:57.583 | INFO | Downloaded 34GiB to GPU in 1.9387s (17.5376GiB/s) | 20260506 12:57:02.722 | INFO | Analyzing chunk frames = [1526784,1527296) | 20260506 12:57:02.722 | INFO | MemInfo: { | 20260506 12:57:02.722 | INFO | "gpu" : " 17 GB", | 20260506 12:57:02.722 | INFO | "huge" : "", | 20260506 12:57:02.722 | INFO | "huge_shared" : " 27 GB", | 20260506 12:57:02.722 | INFO | "malloc" : " 14 GB", | 20260506 12:57:02.722 | INFO | "pinned" : " 29 GB" | 20260506 12:57:02.722 | INFO | } | 20260506 12:57:02.722 | INFO | Perf report for the last 5.13947 seconds: | 20260506 12:57:02.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:57:02.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:02.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:02.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:02.722 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:57:02.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:02.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:02.722 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 12:57:02.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:02.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:02.722 | INFO | Uploaded 36GiB to GPU in 2.0259s (17.7699GiB/s) | 20260506 12:57:02.722 | INFO | Downloaded 34GiB to GPU in 2.01579s (16.8669GiB/s) | 20260506 12:57:07.861 | INFO | Analyzing chunk frames = [1527296,1527808) | 20260506 12:57:07.861 | INFO | MemInfo: { | 20260506 12:57:07.861 | INFO | "gpu" : " 17 GB", | 20260506 12:57:07.861 | INFO | "huge" : "", | 20260506 12:57:07.861 | INFO | "huge_shared" : " 27 GB", | 20260506 12:57:07.861 | INFO | "malloc" : " 14 GB", | 20260506 12:57:07.861 | INFO | "pinned" : " 29 GB" | 20260506 12:57:07.861 | INFO | } | 20260506 12:57:07.861 | INFO | Perf report for the last 5.13904 seconds: | 20260506 12:57:07.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:57:07.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:07.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:07.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:07.861 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 12:57:07.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:07.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:07.861 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 12:57:07.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:07.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:07.861 | INFO | Uploaded 36GiB to GPU in 2.05963s (17.4788GiB/s) | 20260506 12:57:07.861 | INFO | Downloaded 34GiB to GPU in 1.98865s (17.097GiB/s) | 20260506 12:57:07.991 | INFO | Starting a round of Baz IO | 20260506 12:57:11.156 | INFO | Populating RTMetrics report | 20260506 12:57:11.157 | INFO | RTMetrics report populated | 20260506 12:57:11.157 | INFO | Opening tmp RTMetrics json file | 20260506 12:57:11.157 | INFO | Serializing RTMetrics to json | 20260506 12:57:11.158 | INFO | Serialization done, swapping tmp file now... | 20260506 12:57:11.159 | INFO | RTMetrics done | 20260506 12:57:11.255 | INFO | Finished a round of Baz IO: Wrote 2.099 GiB in 3.263 seconds | 20260506 12:57:11.255 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.57% / 0.02% / 59.41% | 20260506 12:57:11.255 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.5281% / 2.78155e-05% / 4.37101e-05% | 20260506 12:57:11.255 | INFO | Baz IO overall: | 20260506 12:57:11.255 | INFO | BytesWritten : 768118391351 | 20260506 12:57:11.255 | INFO | NumEvents : 371378870089 | 20260506 12:57:13.000 | INFO | Analyzing chunk frames = [1527808,1528320) | 20260506 12:57:13.001 | INFO | MemInfo: { | 20260506 12:57:13.001 | INFO | "gpu" : " 17 GB", | 20260506 12:57:13.001 | INFO | "huge" : "", | 20260506 12:57:13.001 | INFO | "huge_shared" : " 27 GB", | 20260506 12:57:13.001 | INFO | "malloc" : " 14 GB", | 20260506 12:57:13.001 | INFO | "pinned" : " 29 GB" | 20260506 12:57:13.001 | INFO | } | 20260506 12:57:13.001 | INFO | Perf report for the last 5.13944 seconds: | 20260506 12:57:13.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:57:13.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:13.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:13.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:13.001 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260506 12:57:13.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:13.001 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:57:13.001 | INFO | PRE_HQ : 61.7%, 1.9, 61.6ms, 0 | 20260506 12:57:13.001 | INFO | BAZWRITER : 62.8%, 3.7, 119.0ms, 0 | 20260506 12:57:13.001 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:57:13.001 | INFO | Uploaded 36GiB to GPU in 2.04292s (17.6219GiB/s) | 20260506 12:57:13.001 | INFO | Downloaded 38GiB to GPU in 2.219s (17.1248GiB/s) | 20260506 12:57:18.140 | INFO | Analyzing chunk frames = [1528320,1528832) | 20260506 12:57:18.140 | INFO | MemInfo: { | 20260506 12:57:18.140 | INFO | "gpu" : " 17 GB", | 20260506 12:57:18.140 | INFO | "huge" : "", | 20260506 12:57:18.140 | INFO | "huge_shared" : " 27 GB", | 20260506 12:57:18.140 | INFO | "malloc" : " 14 GB", | 20260506 12:57:18.140 | INFO | "pinned" : " 29 GB" | 20260506 12:57:18.140 | INFO | } | 20260506 12:57:18.140 | INFO | Perf report for the last 5.13919 seconds: | 20260506 12:57:18.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:57:18.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:18.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:18.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:18.140 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 12:57:18.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:18.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:18.140 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:57:18.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:18.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:18.140 | INFO | Uploaded 36GiB to GPU in 2.04029s (17.6445GiB/s) | 20260506 12:57:18.140 | INFO | Downloaded 34GiB to GPU in 1.94142s (17.5129GiB/s) | 20260506 12:57:23.279 | INFO | Analyzing chunk frames = [1528832,1529344) | 20260506 12:57:23.279 | INFO | MemInfo: { | 20260506 12:57:23.279 | INFO | "gpu" : " 17 GB", | 20260506 12:57:23.279 | INFO | "huge" : "", | 20260506 12:57:23.279 | INFO | "huge_shared" : " 27 GB", | 20260506 12:57:23.279 | INFO | "malloc" : " 14 GB", | 20260506 12:57:23.279 | INFO | "pinned" : " 29 GB" | 20260506 12:57:23.279 | INFO | } | 20260506 12:57:23.279 | INFO | Perf report for the last 5.13952 seconds: | 20260506 12:57:23.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:57:23.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:23.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:23.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:23.279 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 12:57:23.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:23.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:23.279 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:57:23.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:23.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:23.280 | INFO | Uploaded 36GiB to GPU in 2.02063s (17.8163GiB/s) | 20260506 12:57:23.280 | INFO | Downloaded 34GiB to GPU in 1.9473s (17.4601GiB/s) | 20260506 12:57:28.419 | INFO | Analyzing chunk frames = [1529344,1529856) | 20260506 12:57:28.419 | INFO | MemInfo: { | 20260506 12:57:28.419 | INFO | "gpu" : " 17 GB", | 20260506 12:57:28.419 | INFO | "huge" : "", | 20260506 12:57:28.419 | INFO | "huge_shared" : " 27 GB", | 20260506 12:57:28.419 | INFO | "malloc" : " 14 GB", | 20260506 12:57:28.419 | INFO | "pinned" : " 29 GB" | 20260506 12:57:28.419 | INFO | } | 20260506 12:57:28.419 | INFO | Perf report for the last 5.13931 seconds: | 20260506 12:57:28.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:57:28.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:28.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:28.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:28.419 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 12:57:28.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:28.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:28.419 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:57:28.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:28.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:28.419 | INFO | Uploaded 36GiB to GPU in 1.97889s (18.192GiB/s) | 20260506 12:57:28.419 | INFO | Downloaded 34GiB to GPU in 1.93931s (17.532GiB/s) | 20260506 12:57:33.558 | INFO | Analyzing chunk frames = [1529856,1530368) | 20260506 12:57:33.558 | INFO | MemInfo: { | 20260506 12:57:33.558 | INFO | "gpu" : " 17 GB", | 20260506 12:57:33.558 | INFO | "huge" : "", | 20260506 12:57:33.558 | INFO | "huge_shared" : " 27 GB", | 20260506 12:57:33.558 | INFO | "malloc" : " 14 GB", | 20260506 12:57:33.558 | INFO | "pinned" : " 29 GB" | 20260506 12:57:33.558 | INFO | } | 20260506 12:57:33.558 | INFO | Perf report for the last 5.13939 seconds: | 20260506 12:57:33.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:57:33.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:33.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:33.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:33.558 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 12:57:33.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:33.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:33.558 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 12:57:33.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:33.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:33.558 | INFO | Uploaded 36GiB to GPU in 2.02388s (17.7876GiB/s) | 20260506 12:57:33.558 | INFO | Downloaded 34GiB to GPU in 1.95539s (17.3878GiB/s) | 20260506 12:57:38.697 | INFO | Analyzing chunk frames = [1530368,1530880) | 20260506 12:57:38.697 | INFO | MemInfo: { | 20260506 12:57:38.697 | INFO | "gpu" : " 17 GB", | 20260506 12:57:38.697 | INFO | "huge" : "", | 20260506 12:57:38.697 | INFO | "huge_shared" : " 27 GB", | 20260506 12:57:38.697 | INFO | "malloc" : " 14 GB", | 20260506 12:57:38.697 | INFO | "pinned" : " 29 GB" | 20260506 12:57:38.697 | INFO | } | 20260506 12:57:38.697 | INFO | Perf report for the last 5.1391 seconds: | 20260506 12:57:38.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:57:38.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:38.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:38.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:38.697 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 12:57:38.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:38.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:38.697 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260506 12:57:38.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:38.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:38.698 | INFO | Uploaded 36GiB to GPU in 2.09306s (17.1997GiB/s) | 20260506 12:57:38.698 | INFO | Downloaded 34GiB to GPU in 1.9783s (17.1865GiB/s) | 20260506 12:57:43.837 | INFO | Analyzing chunk frames = [1530880,1531392) | 20260506 12:57:43.837 | INFO | MemInfo: { | 20260506 12:57:43.837 | INFO | "gpu" : " 17 GB", | 20260506 12:57:43.837 | INFO | "huge" : "", | 20260506 12:57:43.837 | INFO | "huge_shared" : " 27 GB", | 20260506 12:57:43.837 | INFO | "malloc" : " 14 GB", | 20260506 12:57:43.837 | INFO | "pinned" : " 29 GB" | 20260506 12:57:43.837 | INFO | } | 20260506 12:57:43.837 | INFO | Perf report for the last 5.13955 seconds: | 20260506 12:57:43.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:57:43.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:43.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:43.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:43.837 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 12:57:43.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:43.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:43.837 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 12:57:43.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:43.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:43.837 | INFO | Uploaded 36GiB to GPU in 1.98491s (18.1368GiB/s) | 20260506 12:57:43.837 | INFO | Downloaded 34GiB to GPU in 2.03684s (16.6925GiB/s) | 20260506 12:57:48.976 | INFO | Analyzing chunk frames = [1531392,1531904) | 20260506 12:57:48.976 | INFO | MemInfo: { | 20260506 12:57:48.976 | INFO | "gpu" : " 17 GB", | 20260506 12:57:48.976 | INFO | "huge" : "", | 20260506 12:57:48.976 | INFO | "huge_shared" : " 27 GB", | 20260506 12:57:48.976 | INFO | "malloc" : " 14 GB", | 20260506 12:57:48.976 | INFO | "pinned" : " 29 GB" | 20260506 12:57:48.976 | INFO | } | 20260506 12:57:48.976 | INFO | Perf report for the last 5.13958 seconds: | 20260506 12:57:48.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:57:48.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:48.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:48.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:48.976 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 12:57:48.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:48.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:48.976 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:57:48.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:48.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:48.977 | INFO | Uploaded 36GiB to GPU in 2.0455s (17.5996GiB/s) | 20260506 12:57:48.977 | INFO | Downloaded 34GiB to GPU in 1.96928s (17.2652GiB/s) | 20260506 12:57:48.977 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:57:48.977 | INFO | Total mem usage: 16.8 GiB | 20260506 12:57:48.977 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:57:48.977 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -16.3%) | 20260506 12:57:48.977 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:57:48.977 | INFO | High Water Breakdown: | 20260506 12:57:48.977 | INFO | 59.7% : 8.63 GiB ( 83.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:57:48.977 | INFO | 40.2% : 5.81 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:57:48.977 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:57:48.977 | INFO | ------------------------------------------ | 20260506 12:57:48.977 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:57:48.977 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:57:48.977 | INFO | Total mem usage: 26.5 GiB | 20260506 12:57:48.977 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:57:48.977 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:57:48.977 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:57:48.977 | INFO | High Water Breakdown: | 20260506 12:57:48.977 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:57:48.977 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:57:48.977 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:57:48.977 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:57:48.977 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:57:48.977 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:57:48.977 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:57:48.977 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:57:48.977 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:57:48.977 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:57:48.977 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:57:48.977 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:57:48.977 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:57:48.977 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:57:48.977 | INFO | ------------------------------------------ | 20260506 12:57:48.977 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:57:48.977 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:57:48.977 | INFO | Total mem usage: 25.6 GiB | 20260506 12:57:48.977 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:57:48.977 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:57:48.977 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:57:48.977 | INFO | High Water Breakdown: | 20260506 12:57:48.977 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:57:48.977 | INFO | ------------------------------------------ | 20260506 12:57:48.977 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:57:48.977 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:57:48.977 | INFO | Total mem usage: 15.7 GiB | 20260506 12:57:48.977 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:57:48.977 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:57:48.977 | INFO | Unused Cache: 830 MiB (Waste: 5.17%) | 20260506 12:57:48.977 | INFO | High Water Breakdown: | 20260506 12:57:48.977 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:57:48.977 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:57:48.977 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:57:48.977 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:57:48.977 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:57:48.977 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:57:48.977 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:57:48.977 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:57:48.977 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:57:48.977 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:57:48.977 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:57:48.977 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:57:48.977 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:57:48.977 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:57:48.977 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:57:48.977 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:57:48.977 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:57:48.977 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:57:48.977 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:57:48.977 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:57:48.977 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:57:48.977 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:57:48.977 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:57:48.977 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:57:48.977 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:57:48.977 | INFO | ------------------------------------------ | 20260506 12:57:48.977 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:57:48.977 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:57:48.977 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:57:48.977 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:57:48.977 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.60e+01, 1.36e+01 / 2.53e+00 | 20260506 12:57:48.977 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.02e-01, 1.09e-02 / 7.22e-03 | 20260506 12:57:48.977 | INFO | PerfReport: 11.57%, Baseline -- 3.66e+00 / 4.43e+00, 5.71e-01 / 3.48e-01 | 20260506 12:57:48.977 | INFO | PerfReport: 3.23%, Histogram -- 1.02e+00 / 1.46e+00, 1.87e-01 / 1.39e-01 | 20260506 12:57:48.977 | INFO | PerfReport: 15.79%, DME -- 4.00e+01 / 4.42e+01, 3.94e+00 / 2.75e+00 | 20260506 12:57:48.977 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.79e+01 / 2.09e+01, 1.41e+00 / 1.81e+00 | 20260506 12:57:48.977 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.39e+00 / 2.98e+00, 4.14e-01 / 2.47e-01 | 20260506 12:57:48.978 | INFO | PerfReport: 3.71%, Metrics -- 1.18e+00 / 1.56e+00, 2.99e-01 / 1.85e-01 | 20260506 12:57:48.978 | INFO | PerfReport: 1.31%, PulseSerializing -- 4.13e-01 / 4.72e-01, 6.45e-02 / 3.00e-02 | 20260506 12:57:48.978 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:57:48.978 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:57:48.978 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:57:48.978 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 12:57:48.978 | INFO | PerfReport: 21.42%, Upload -- 2.11e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 12:57:48.978 | INFO | PerfReport: 21.08%, Download -- 2.08e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 12:57:48.978 | INFO | PerfReport: 32.12%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 12:57:48.978 | INFO | PerfReport: 1.67%, LockUpload -- 1.65e+00 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260506 12:57:48.978 | INFO | PerfReport: 1.17%, LockDownload -- 1.15e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 12:57:48.978 | INFO | PerfReport: 22.53%, LockCompute -- 2.22e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 12:57:48.978 | INFO | PerfReport: 0.02%, Misc -- 1.50e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260506 12:57:49.105 | INFO | Starting a round of Baz IO | 20260506 12:57:52.261 | INFO | Populating RTMetrics report | 20260506 12:57:52.261 | INFO | RTMetrics report populated | 20260506 12:57:52.261 | INFO | Opening tmp RTMetrics json file | 20260506 12:57:52.262 | INFO | Serializing RTMetrics to json | 20260506 12:57:52.263 | INFO | Serialization done, swapping tmp file now... | 20260506 12:57:52.264 | INFO | RTMetrics done | 20260506 12:57:52.365 | INFO | Finished a round of Baz IO: Wrote 2.103 GiB in 3.259 seconds | 20260506 12:57:52.365 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.55% / 0.02% / 59.43% | 20260506 12:57:52.365 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.53393% / 5.5631e-05% / 5.16574e-05% | 20260506 12:57:52.365 | INFO | Baz IO overall: | 20260506 12:57:52.365 | INFO | BytesWritten : 770376523393 | 20260506 12:57:52.365 | INFO | NumEvents : 372474706626 | 20260506 12:57:54.115 | INFO | Analyzing chunk frames = [1531904,1532416) | 20260506 12:57:54.115 | INFO | MemInfo: { | 20260506 12:57:54.115 | INFO | "gpu" : " 17 GB", | 20260506 12:57:54.115 | INFO | "huge" : "", | 20260506 12:57:54.115 | INFO | "huge_shared" : " 27 GB", | 20260506 12:57:54.115 | INFO | "malloc" : " 14 GB", | 20260506 12:57:54.115 | INFO | "pinned" : " 29 GB" | 20260506 12:57:54.115 | INFO | } | 20260506 12:57:54.115 | INFO | Perf report for the last 5.13906 seconds: | 20260506 12:57:54.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:57:54.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:54.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:54.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:54.115 | INFO | ANALYSIS : 63.4%, 3.0, 98.3ms, 0 | 20260506 12:57:54.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:54.116 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:57:54.116 | INFO | PRE_HQ : 61.5%, 1.9, 61.4ms, 0 | 20260506 12:57:54.116 | INFO | BAZWRITER : 62.8%, 3.7, 118.1ms, 0 | 20260506 12:57:54.116 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 12:57:54.116 | INFO | Uploaded 36GiB to GPU in 2.07055s (17.3867GiB/s) | 20260506 12:57:54.116 | INFO | Downloaded 38GiB to GPU in 2.21665s (17.143GiB/s) | 20260506 12:57:59.255 | INFO | Analyzing chunk frames = [1532416,1532928) | 20260506 12:57:59.255 | INFO | MemInfo: { | 20260506 12:57:59.255 | INFO | "gpu" : " 17 GB", | 20260506 12:57:59.255 | INFO | "huge" : "", | 20260506 12:57:59.255 | INFO | "huge_shared" : " 27 GB", | 20260506 12:57:59.255 | INFO | "malloc" : " 14 GB", | 20260506 12:57:59.255 | INFO | "pinned" : " 29 GB" | 20260506 12:57:59.255 | INFO | } | 20260506 12:57:59.255 | INFO | Perf report for the last 5.13967 seconds: | 20260506 12:57:59.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:57:59.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:59.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:59.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:59.255 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 12:57:59.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:59.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:59.255 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:57:59.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:59.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:57:59.255 | INFO | Uploaded 36GiB to GPU in 2.0133s (17.8811GiB/s) | 20260506 12:57:59.255 | INFO | Downloaded 34GiB to GPU in 1.92114s (17.6978GiB/s) | 20260506 12:58:04.394 | INFO | Analyzing chunk frames = [1532928,1533440) | 20260506 12:58:04.394 | INFO | MemInfo: { | 20260506 12:58:04.394 | INFO | "gpu" : " 17 GB", | 20260506 12:58:04.394 | INFO | "huge" : "", | 20260506 12:58:04.394 | INFO | "huge_shared" : " 27 GB", | 20260506 12:58:04.394 | INFO | "malloc" : " 14 GB", | 20260506 12:58:04.394 | INFO | "pinned" : " 29 GB" | 20260506 12:58:04.394 | INFO | } | 20260506 12:58:04.394 | INFO | Perf report for the last 5.13931 seconds: | 20260506 12:58:04.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:58:04.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:04.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:04.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:04.394 | INFO | ANALYSIS : 61.4%, 3.0, 95.1ms, 0 | 20260506 12:58:04.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:04.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:04.394 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:58:04.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:04.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:04.395 | INFO | Uploaded 36GiB to GPU in 1.98647s (18.1226GiB/s) | 20260506 12:58:04.395 | INFO | Downloaded 34GiB to GPU in 1.9298s (17.6185GiB/s) | 20260506 12:58:09.533 | INFO | Analyzing chunk frames = [1533440,1533952) | 20260506 12:58:09.533 | INFO | MemInfo: { | 20260506 12:58:09.533 | INFO | "gpu" : " 17 GB", | 20260506 12:58:09.533 | INFO | "huge" : "", | 20260506 12:58:09.533 | INFO | "huge_shared" : " 27 GB", | 20260506 12:58:09.533 | INFO | "malloc" : " 14 GB", | 20260506 12:58:09.533 | INFO | "pinned" : " 29 GB" | 20260506 12:58:09.533 | INFO | } | 20260506 12:58:09.533 | INFO | Perf report for the last 5.13909 seconds: | 20260506 12:58:09.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:58:09.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:09.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:09.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:09.534 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 12:58:09.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:09.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:09.534 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:58:09.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:09.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:09.534 | INFO | Uploaded 36GiB to GPU in 1.98413s (18.144GiB/s) | 20260506 12:58:09.534 | INFO | Downloaded 34GiB to GPU in 1.9149s (17.7555GiB/s) | 20260506 12:58:14.673 | INFO | Analyzing chunk frames = [1533952,1534464) | 20260506 12:58:14.673 | INFO | MemInfo: { | 20260506 12:58:14.673 | INFO | "gpu" : " 17 GB", | 20260506 12:58:14.673 | INFO | "huge" : "", | 20260506 12:58:14.673 | INFO | "huge_shared" : " 27 GB", | 20260506 12:58:14.673 | INFO | "malloc" : " 14 GB", | 20260506 12:58:14.673 | INFO | "pinned" : " 29 GB" | 20260506 12:58:14.673 | INFO | } | 20260506 12:58:14.673 | INFO | Perf report for the last 5.13951 seconds: | 20260506 12:58:14.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:58:14.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:14.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:14.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:14.673 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 12:58:14.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:14.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:14.673 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 12:58:14.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:14.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:14.673 | INFO | Uploaded 36GiB to GPU in 2.01681s (17.85GiB/s) | 20260506 12:58:14.673 | INFO | Downloaded 34GiB to GPU in 1.94179s (17.5096GiB/s) | 20260506 12:58:19.812 | INFO | Analyzing chunk frames = [1534464,1534976) | 20260506 12:58:19.812 | INFO | MemInfo: { | 20260506 12:58:19.812 | INFO | "gpu" : " 17 GB", | 20260506 12:58:19.812 | INFO | "huge" : "", | 20260506 12:58:19.812 | INFO | "huge_shared" : " 27 GB", | 20260506 12:58:19.812 | INFO | "malloc" : " 14 GB", | 20260506 12:58:19.813 | INFO | "pinned" : " 29 GB" | 20260506 12:58:19.813 | INFO | } | 20260506 12:58:19.813 | INFO | Perf report for the last 5.13954 seconds: | 20260506 12:58:19.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:58:19.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:19.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:19.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:19.813 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 12:58:19.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:19.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:19.813 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260506 12:58:19.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:19.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:19.813 | INFO | Uploaded 36GiB to GPU in 2.03528s (17.688GiB/s) | 20260506 12:58:19.813 | INFO | Downloaded 34GiB to GPU in 1.91907s (17.717GiB/s) | 20260506 12:58:24.952 | INFO | Analyzing chunk frames = [1534976,1535488) | 20260506 12:58:24.952 | INFO | MemInfo: { | 20260506 12:58:24.952 | INFO | "gpu" : " 17 GB", | 20260506 12:58:24.952 | INFO | "huge" : "", | 20260506 12:58:24.952 | INFO | "huge_shared" : " 27 GB", | 20260506 12:58:24.952 | INFO | "malloc" : " 14 GB", | 20260506 12:58:24.952 | INFO | "pinned" : " 29 GB" | 20260506 12:58:24.952 | INFO | } | 20260506 12:58:24.952 | INFO | Perf report for the last 5.13967 seconds: | 20260506 12:58:24.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:58:24.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:24.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:24.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:24.952 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:58:24.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:24.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:24.952 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 12:58:24.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:24.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:24.953 | INFO | Uploaded 36GiB to GPU in 2.0447s (17.6065GiB/s) | 20260506 12:58:24.953 | INFO | Downloaded 34GiB to GPU in 1.98551s (17.1241GiB/s) | 20260506 12:58:30.091 | INFO | Analyzing chunk frames = [1535488,1536000) | 20260506 12:58:30.091 | INFO | MemInfo: { | 20260506 12:58:30.091 | INFO | "gpu" : " 17 GB", | 20260506 12:58:30.091 | INFO | "huge" : "", | 20260506 12:58:30.091 | INFO | "huge_shared" : " 27 GB", | 20260506 12:58:30.091 | INFO | "malloc" : " 14 GB", | 20260506 12:58:30.091 | INFO | "pinned" : " 29 GB" | 20260506 12:58:30.091 | INFO | } | 20260506 12:58:30.091 | INFO | Perf report for the last 5.13849 seconds: | 20260506 12:58:30.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:58:30.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:30.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:30.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:30.091 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 12:58:30.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:30.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:30.091 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:58:30.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:30.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:30.091 | INFO | Uploaded 36GiB to GPU in 2.0456s (17.5987GiB/s) | 20260506 12:58:30.091 | INFO | Downloaded 34GiB to GPU in 1.97167s (17.2442GiB/s) | 20260506 12:58:30.219 | INFO | Starting a round of Baz IO | 20260506 12:58:33.390 | INFO | Populating RTMetrics report | 20260506 12:58:33.390 | INFO | RTMetrics report populated | 20260506 12:58:33.390 | INFO | Opening tmp RTMetrics json file | 20260506 12:58:33.391 | INFO | Serializing RTMetrics to json | 20260506 12:58:33.392 | INFO | Serialization done, swapping tmp file now... | 20260506 12:58:33.392 | INFO | RTMetrics done | 20260506 12:58:33.486 | INFO | Finished a round of Baz IO: Wrote 2.102 GiB in 3.266 seconds | 20260506 12:58:33.486 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.53% / 0.02% / 59.45% | 20260506 12:58:33.486 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.51539% / 5.5631e-05% / 6.75519e-05% | 20260506 12:58:33.486 | INFO | Baz IO overall: | 20260506 12:58:33.486 | INFO | BytesWritten : 772633388173 | 20260506 12:58:33.486 | INFO | NumEvents : 373569492647 | 20260506 12:58:35.230 | INFO | Analyzing chunk frames = [1536000,1536512) | 20260506 12:58:35.230 | INFO | MemInfo: { | 20260506 12:58:35.230 | INFO | "gpu" : " 17 GB", | 20260506 12:58:35.230 | INFO | "huge" : "", | 20260506 12:58:35.230 | INFO | "huge_shared" : " 27 GB", | 20260506 12:58:35.230 | INFO | "malloc" : " 14 GB", | 20260506 12:58:35.230 | INFO | "pinned" : " 29 GB" | 20260506 12:58:35.230 | INFO | } | 20260506 12:58:35.230 | INFO | Perf report for the last 5.13925 seconds: | 20260506 12:58:35.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:58:35.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:35.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:35.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:35.230 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260506 12:58:35.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:35.230 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:58:35.230 | INFO | PRE_HQ : 61.5%, 1.9, 61.5ms, 0 | 20260506 12:58:35.230 | INFO | BAZWRITER : 62.9%, 3.7, 119.1ms, 0 | 20260506 12:58:35.230 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 12:58:35.230 | INFO | Uploaded 36GiB to GPU in 2.06662s (17.4198GiB/s) | 20260506 12:58:35.230 | INFO | Downloaded 38GiB to GPU in 2.21828s (17.1304GiB/s) | 20260506 12:58:40.370 | INFO | Analyzing chunk frames = [1536512,1537024) | 20260506 12:58:40.370 | INFO | MemInfo: { | 20260506 12:58:40.370 | INFO | "gpu" : " 17 GB", | 20260506 12:58:40.370 | INFO | "huge" : "", | 20260506 12:58:40.370 | INFO | "huge_shared" : " 27 GB", | 20260506 12:58:40.370 | INFO | "malloc" : " 14 GB", | 20260506 12:58:40.370 | INFO | "pinned" : " 29 GB" | 20260506 12:58:40.370 | INFO | } | 20260506 12:58:40.370 | INFO | Perf report for the last 5.13996 seconds: | 20260506 12:58:40.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:58:40.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:40.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:40.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:40.370 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 12:58:40.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:40.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:40.370 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:58:40.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:40.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:40.370 | INFO | Uploaded 36GiB to GPU in 2.04968s (17.5637GiB/s) | 20260506 12:58:40.370 | INFO | Downloaded 34GiB to GPU in 1.9363s (17.5593GiB/s) | 20260506 12:58:45.509 | INFO | Analyzing chunk frames = [1537024,1537536) | 20260506 12:58:45.509 | INFO | MemInfo: { | 20260506 12:58:45.509 | INFO | "gpu" : " 17 GB", | 20260506 12:58:45.509 | INFO | "huge" : "", | 20260506 12:58:45.509 | INFO | "huge_shared" : " 27 GB", | 20260506 12:58:45.509 | INFO | "malloc" : " 14 GB", | 20260506 12:58:45.509 | INFO | "pinned" : " 29 GB" | 20260506 12:58:45.509 | INFO | } | 20260506 12:58:45.509 | INFO | Perf report for the last 5.13896 seconds: | 20260506 12:58:45.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:58:45.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:45.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:45.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:45.509 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 12:58:45.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:45.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:45.509 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 12:58:45.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:45.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:45.509 | INFO | Uploaded 36GiB to GPU in 2.03195s (17.717GiB/s) | 20260506 12:58:45.509 | INFO | Downloaded 34GiB to GPU in 1.9433s (17.496GiB/s) | 20260506 12:58:50.648 | INFO | Analyzing chunk frames = [1537536,1538048) | 20260506 12:58:50.648 | INFO | MemInfo: { | 20260506 12:58:50.648 | INFO | "gpu" : " 17 GB", | 20260506 12:58:50.648 | INFO | "huge" : "", | 20260506 12:58:50.648 | INFO | "huge_shared" : " 27 GB", | 20260506 12:58:50.648 | INFO | "malloc" : " 14 GB", | 20260506 12:58:50.648 | INFO | "pinned" : " 29 GB" | 20260506 12:58:50.648 | INFO | } | 20260506 12:58:50.648 | INFO | Perf report for the last 5.13943 seconds: | 20260506 12:58:50.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:58:50.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:50.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:50.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:50.648 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 12:58:50.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:50.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:50.648 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 12:58:50.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:50.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:50.649 | INFO | Uploaded 36GiB to GPU in 2.01719s (17.8466GiB/s) | 20260506 12:58:50.649 | INFO | Downloaded 34GiB to GPU in 1.94123s (17.5146GiB/s) | 20260506 12:58:55.787 | INFO | Analyzing chunk frames = [1538048,1538560) | 20260506 12:58:55.787 | INFO | MemInfo: { | 20260506 12:58:55.788 | INFO | "gpu" : " 17 GB", | 20260506 12:58:55.788 | INFO | "huge" : "", | 20260506 12:58:55.788 | INFO | "huge_shared" : " 27 GB", | 20260506 12:58:55.788 | INFO | "malloc" : " 14 GB", | 20260506 12:58:55.788 | INFO | "pinned" : " 29 GB" | 20260506 12:58:55.788 | INFO | } | 20260506 12:58:55.788 | INFO | Perf report for the last 5.13942 seconds: | 20260506 12:58:55.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:58:55.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:55.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:55.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:55.788 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 12:58:55.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:55.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:55.788 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 12:58:55.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:55.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:58:55.788 | INFO | Uploaded 36GiB to GPU in 2.02726s (17.758GiB/s) | 20260506 12:58:55.788 | INFO | Downloaded 34GiB to GPU in 1.9513s (17.4243GiB/s) | 20260506 12:59:00.927 | INFO | Analyzing chunk frames = [1538560,1539072) | 20260506 12:59:00.927 | INFO | MemInfo: { | 20260506 12:59:00.927 | INFO | "gpu" : " 17 GB", | 20260506 12:59:00.927 | INFO | "huge" : "", | 20260506 12:59:00.927 | INFO | "huge_shared" : " 27 GB", | 20260506 12:59:00.927 | INFO | "malloc" : " 14 GB", | 20260506 12:59:00.927 | INFO | "pinned" : " 29 GB" | 20260506 12:59:00.927 | INFO | } | 20260506 12:59:00.927 | INFO | Perf report for the last 5.13922 seconds: | 20260506 12:59:00.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:59:00.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:00.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:00.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:00.927 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 12:59:00.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:00.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:00.927 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260506 12:59:00.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:00.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:00.927 | INFO | Uploaded 36GiB to GPU in 2.07037s (17.3882GiB/s) | 20260506 12:59:00.927 | INFO | Downloaded 34GiB to GPU in 1.93712s (17.5518GiB/s) | 20260506 12:59:06.067 | INFO | Analyzing chunk frames = [1539072,1539584) | 20260506 12:59:06.067 | INFO | MemInfo: { | 20260506 12:59:06.067 | INFO | "gpu" : " 17 GB", | 20260506 12:59:06.067 | INFO | "huge" : "", | 20260506 12:59:06.067 | INFO | "huge_shared" : " 27 GB", | 20260506 12:59:06.067 | INFO | "malloc" : " 14 GB", | 20260506 12:59:06.068 | INFO | "pinned" : " 29 GB" | 20260506 12:59:06.068 | INFO | } | 20260506 12:59:06.068 | INFO | Perf report for the last 5.14054 seconds: | 20260506 12:59:06.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:59:06.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:06.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:06.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:06.068 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 12:59:06.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:06.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:06.068 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 12:59:06.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:06.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:06.068 | INFO | Uploaded 36GiB to GPU in 2.02187s (17.8053GiB/s) | 20260506 12:59:06.068 | INFO | Downloaded 34GiB to GPU in 2.03085s (16.7417GiB/s) | 20260506 12:59:11.205 | INFO | Analyzing chunk frames = [1539584,1540096) | 20260506 12:59:11.205 | INFO | MemInfo: { | 20260506 12:59:11.205 | INFO | "gpu" : " 17 GB", | 20260506 12:59:11.205 | INFO | "huge" : "", | 20260506 12:59:11.205 | INFO | "huge_shared" : " 27 GB", | 20260506 12:59:11.205 | INFO | "malloc" : " 14 GB", | 20260506 12:59:11.205 | INFO | "pinned" : " 29 GB" | 20260506 12:59:11.205 | INFO | } | 20260506 12:59:11.206 | INFO | Perf report for the last 5.13806 seconds: | 20260506 12:59:11.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:59:11.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:11.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:11.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:11.206 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 12:59:11.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:11.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:11.206 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 12:59:11.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:11.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:11.206 | INFO | Uploaded 36GiB to GPU in 2.04894s (17.5701GiB/s) | 20260506 12:59:11.206 | INFO | Downloaded 34GiB to GPU in 1.95377s (17.4022GiB/s) | 20260506 12:59:11.206 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 12:59:11.206 | INFO | Total mem usage: 16.8 GiB | 20260506 12:59:11.206 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 12:59:11.206 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.6%) | 20260506 12:59:11.206 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 12:59:11.206 | INFO | High Water Breakdown: | 20260506 12:59:11.206 | INFO | 59.7% : 8.6 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 12:59:11.206 | INFO | 40.2% : 5.8 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 12:59:11.206 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 12:59:11.206 | INFO | ------------------------------------------ | 20260506 12:59:11.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:59:11.206 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 12:59:11.206 | INFO | Total mem usage: 26.5 GiB | 20260506 12:59:11.206 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 12:59:11.206 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 12:59:11.206 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 12:59:11.206 | INFO | High Water Breakdown: | 20260506 12:59:11.206 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:59:11.206 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:59:11.206 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:59:11.206 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:59:11.206 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:59:11.206 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:59:11.206 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 12:59:11.206 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 12:59:11.206 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:59:11.206 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 12:59:11.206 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 12:59:11.207 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 12:59:11.207 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 12:59:11.207 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 12:59:11.207 | INFO | ------------------------------------------ | 20260506 12:59:11.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:59:11.207 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 12:59:11.207 | INFO | Total mem usage: 25.6 GiB | 20260506 12:59:11.207 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 12:59:11.207 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 12:59:11.207 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 12:59:11.207 | INFO | High Water Breakdown: | 20260506 12:59:11.207 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 12:59:11.207 | INFO | ------------------------------------------ | 20260506 12:59:11.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:59:11.207 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 12:59:11.207 | INFO | Total mem usage: 15.7 GiB | 20260506 12:59:11.207 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 12:59:11.207 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 12:59:11.207 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 12:59:11.207 | INFO | High Water Breakdown: | 20260506 12:59:11.207 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 12:59:11.207 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 12:59:11.207 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 12:59:11.207 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 12:59:11.207 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 12:59:11.207 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 12:59:11.207 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 12:59:11.207 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 12:59:11.207 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 12:59:11.207 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 12:59:11.207 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 12:59:11.207 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 12:59:11.207 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 12:59:11.207 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 12:59:11.207 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 12:59:11.207 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 12:59:11.207 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 12:59:11.207 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 12:59:11.207 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 12:59:11.207 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 12:59:11.207 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 12:59:11.207 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 12:59:11.207 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 12:59:11.207 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 12:59:11.207 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 12:59:11.207 | INFO | ------------------------------------------ | 20260506 12:59:11.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 12:59:11.207 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 12:59:11.207 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:59:11.207 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:59:11.207 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.60e+01, 1.33e+01 / 2.44e+00 | 20260506 12:59:11.207 | INFO | PerfReport: 0.29%, AggregateStats -- 9.29e-02 / 1.01e-01, 1.09e-02 / 6.82e-03 | 20260506 12:59:11.207 | INFO | PerfReport: 11.54%, Baseline -- 3.66e+00 / 4.41e+00, 5.71e-01 / 3.80e-01 | 20260506 12:59:11.207 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.46e+00, 1.89e-01 / 1.54e-01 | 20260506 12:59:11.207 | INFO | PerfReport: 15.77%, DME -- 4.00e+01 / 4.41e+01, 3.96e+00 / 2.57e+00 | 20260506 12:59:11.207 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.79e+01 / 2.09e+01, 1.49e+00 / 1.70e+00 | 20260506 12:59:11.207 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.38e+00 / 3.02e+00, 4.13e-01 / 2.46e-01 | 20260506 12:59:11.207 | INFO | PerfReport: 3.70%, Metrics -- 1.17e+00 / 1.57e+00, 2.98e-01 / 1.74e-01 | 20260506 12:59:11.207 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.13e-01 / 4.71e-01, 6.56e-02 / 2.68e-02 | 20260506 12:59:11.207 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 12:59:11.207 | INFO | PerfReport: Gathered 1536 samples. | 20260506 12:59:11.207 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 12:59:11.207 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 12:59:11.207 | INFO | PerfReport: 21.48%, Upload -- 2.12e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 12:59:11.207 | INFO | PerfReport: 20.94%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 12:59:11.207 | INFO | PerfReport: 32.14%, Compute -- 3.17e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 12:59:11.207 | INFO | PerfReport: 1.73%, LockUpload -- 1.71e+00 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260506 12:59:11.207 | INFO | PerfReport: 1.06%, LockDownload -- 1.05e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260506 12:59:11.207 | INFO | PerfReport: 22.63%, LockCompute -- 2.23e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 12:59:11.207 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260506 12:59:11.336 | INFO | Starting a round of Baz IO | 20260506 12:59:14.501 | INFO | Populating RTMetrics report | 20260506 12:59:14.501 | INFO | RTMetrics report populated | 20260506 12:59:14.501 | INFO | Opening tmp RTMetrics json file | 20260506 12:59:14.502 | INFO | Serializing RTMetrics to json | 20260506 12:59:14.503 | INFO | Serialization done, swapping tmp file now... | 20260506 12:59:14.504 | INFO | RTMetrics done | 20260506 12:59:14.607 | INFO | Finished a round of Baz IO: Wrote 2.101 GiB in 3.27 seconds | 20260506 12:59:14.607 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.51% / 0.02% / 59.47% | 20260506 12:59:14.607 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.49869% / 3.97364e-05% / 5.16574e-05% | 20260506 12:59:14.607 | INFO | Baz IO overall: | 20260506 12:59:14.607 | INFO | BytesWritten : 774888972344 | 20260506 12:59:14.607 | INFO | NumEvents : 374663128604 | 20260506 12:59:16.345 | INFO | Analyzing chunk frames = [1540096,1540608) | 20260506 12:59:16.345 | INFO | MemInfo: { | 20260506 12:59:16.345 | INFO | "gpu" : " 17 GB", | 20260506 12:59:16.345 | INFO | "huge" : "", | 20260506 12:59:16.345 | INFO | "huge_shared" : " 27 GB", | 20260506 12:59:16.345 | INFO | "malloc" : " 14 GB", | 20260506 12:59:16.345 | INFO | "pinned" : " 29 GB" | 20260506 12:59:16.345 | INFO | } | 20260506 12:59:16.345 | INFO | Perf report for the last 5.1391 seconds: | 20260506 12:59:16.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:59:16.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:16.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:16.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:16.345 | INFO | ANALYSIS : 63.5%, 3.0, 98.4ms, 0 | 20260506 12:59:16.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:16.345 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 12:59:16.345 | INFO | PRE_HQ : 61.4%, 1.9, 61.3ms, 0 | 20260506 12:59:16.345 | INFO | BAZWRITER : 63.0%, 3.8, 122.6ms, 0 | 20260506 12:59:16.345 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 12:59:16.345 | INFO | Uploaded 36GiB to GPU in 2.06834s (17.4053GiB/s) | 20260506 12:59:16.345 | INFO | Downloaded 38GiB to GPU in 2.22801s (17.0556GiB/s) | 20260506 12:59:21.484 | INFO | Analyzing chunk frames = [1540608,1541120) | 20260506 12:59:21.484 | INFO | MemInfo: { | 20260506 12:59:21.484 | INFO | "gpu" : " 17 GB", | 20260506 12:59:21.484 | INFO | "huge" : "", | 20260506 12:59:21.484 | INFO | "huge_shared" : " 27 GB", | 20260506 12:59:21.484 | INFO | "malloc" : " 14 GB", | 20260506 12:59:21.484 | INFO | "pinned" : " 29 GB" | 20260506 12:59:21.484 | INFO | } | 20260506 12:59:21.484 | INFO | Perf report for the last 5.13939 seconds: | 20260506 12:59:21.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:59:21.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:21.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:21.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:21.484 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 12:59:21.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:21.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:21.484 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 12:59:21.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:21.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:21.485 | INFO | Uploaded 36GiB to GPU in 2.04522s (17.602GiB/s) | 20260506 12:59:21.485 | INFO | Downloaded 34GiB to GPU in 1.9315s (17.6029GiB/s) | 20260506 12:59:26.623 | INFO | Analyzing chunk frames = [1541120,1541632) | 20260506 12:59:26.623 | INFO | MemInfo: { | 20260506 12:59:26.623 | INFO | "gpu" : " 17 GB", | 20260506 12:59:26.623 | INFO | "huge" : "", | 20260506 12:59:26.623 | INFO | "huge_shared" : " 27 GB", | 20260506 12:59:26.623 | INFO | "malloc" : " 14 GB", | 20260506 12:59:26.623 | INFO | "pinned" : " 29 GB" | 20260506 12:59:26.623 | INFO | } | 20260506 12:59:26.624 | INFO | Perf report for the last 5.13929 seconds: | 20260506 12:59:26.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:59:26.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:26.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:26.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:26.624 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 12:59:26.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:26.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:26.624 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 12:59:26.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:26.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:26.624 | INFO | Uploaded 36GiB to GPU in 2.0371s (17.6722GiB/s) | 20260506 12:59:26.624 | INFO | Downloaded 34GiB to GPU in 1.93916s (17.5333GiB/s) | 20260506 12:59:31.763 | INFO | Analyzing chunk frames = [1541632,1542144) | 20260506 12:59:31.763 | INFO | MemInfo: { | 20260506 12:59:31.763 | INFO | "gpu" : " 17 GB", | 20260506 12:59:31.763 | INFO | "huge" : "", | 20260506 12:59:31.763 | INFO | "huge_shared" : " 27 GB", | 20260506 12:59:31.763 | INFO | "malloc" : " 14 GB", | 20260506 12:59:31.763 | INFO | "pinned" : " 29 GB" | 20260506 12:59:31.763 | INFO | } | 20260506 12:59:31.763 | INFO | Perf report for the last 5.13945 seconds: | 20260506 12:59:31.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:59:31.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:31.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:31.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:31.763 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 12:59:31.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:31.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:31.763 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 12:59:31.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:31.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:31.763 | INFO | Uploaded 36GiB to GPU in 2.00923s (17.9173GiB/s) | 20260506 12:59:31.763 | INFO | Downloaded 34GiB to GPU in 1.93573s (17.5645GiB/s) | 20260506 12:59:36.902 | INFO | Analyzing chunk frames = [1542144,1542656) | 20260506 12:59:36.902 | INFO | MemInfo: { | 20260506 12:59:36.902 | INFO | "gpu" : " 17 GB", | 20260506 12:59:36.902 | INFO | "huge" : "", | 20260506 12:59:36.902 | INFO | "huge_shared" : " 27 GB", | 20260506 12:59:36.902 | INFO | "malloc" : " 14 GB", | 20260506 12:59:36.902 | INFO | "pinned" : " 29 GB" | 20260506 12:59:36.902 | INFO | } | 20260506 12:59:36.902 | INFO | Perf report for the last 5.13926 seconds: | 20260506 12:59:36.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:59:36.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:36.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:36.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:36.902 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 12:59:36.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:36.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:36.903 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 12:59:36.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:36.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:36.903 | INFO | Uploaded 36GiB to GPU in 2.05346s (17.5314GiB/s) | 20260506 12:59:36.903 | INFO | Downloaded 34GiB to GPU in 1.94892s (17.4456GiB/s) | 20260506 12:59:42.041 | INFO | Analyzing chunk frames = [1542656,1543168) | 20260506 12:59:42.042 | INFO | MemInfo: { | 20260506 12:59:42.042 | INFO | "gpu" : " 17 GB", | 20260506 12:59:42.042 | INFO | "huge" : "", | 20260506 12:59:42.042 | INFO | "huge_shared" : " 27 GB", | 20260506 12:59:42.042 | INFO | "malloc" : " 14 GB", | 20260506 12:59:42.042 | INFO | "pinned" : " 29 GB" | 20260506 12:59:42.042 | INFO | } | 20260506 12:59:42.042 | INFO | Perf report for the last 5.13946 seconds: | 20260506 12:59:42.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:59:42.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:42.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:42.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:42.042 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 12:59:42.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:42.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:42.042 | INFO | PRE_HQ : 33.3%, 1.0, 17.3ms, 0 | 20260506 12:59:42.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:42.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:42.042 | INFO | Uploaded 36GiB to GPU in 2.06933s (17.3969GiB/s) | 20260506 12:59:42.042 | INFO | Downloaded 34GiB to GPU in 1.9159s (17.7462GiB/s) | 20260506 12:59:47.182 | INFO | Analyzing chunk frames = [1543168,1543680) | 20260506 12:59:47.182 | INFO | MemInfo: { | 20260506 12:59:47.182 | INFO | "gpu" : " 17 GB", | 20260506 12:59:47.182 | INFO | "huge" : "", | 20260506 12:59:47.182 | INFO | "huge_shared" : " 27 GB", | 20260506 12:59:47.182 | INFO | "malloc" : " 14 GB", | 20260506 12:59:47.182 | INFO | "pinned" : " 29 GB" | 20260506 12:59:47.182 | INFO | } | 20260506 12:59:47.182 | INFO | Perf report for the last 5.14036 seconds: | 20260506 12:59:47.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:59:47.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:47.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:47.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:47.182 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 12:59:47.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:47.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:47.182 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 12:59:47.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:47.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:47.183 | INFO | Uploaded 36GiB to GPU in 2.0049s (17.956GiB/s) | 20260506 12:59:47.183 | INFO | Downloaded 34GiB to GPU in 2.03132s (16.7379GiB/s) | 20260506 12:59:52.320 | INFO | Analyzing chunk frames = [1543680,1544192) | 20260506 12:59:52.320 | INFO | MemInfo: { | 20260506 12:59:52.320 | INFO | "gpu" : " 17 GB", | 20260506 12:59:52.320 | INFO | "huge" : "", | 20260506 12:59:52.320 | INFO | "huge_shared" : " 27 GB", | 20260506 12:59:52.320 | INFO | "malloc" : " 14 GB", | 20260506 12:59:52.320 | INFO | "pinned" : " 29 GB" | 20260506 12:59:52.320 | INFO | } | 20260506 12:59:52.321 | INFO | Perf report for the last 5.13846 seconds: | 20260506 12:59:52.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:59:52.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:52.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:52.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:52.321 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 12:59:52.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:52.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:52.321 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 12:59:52.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:52.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:52.321 | INFO | Uploaded 36GiB to GPU in 2.0492s (17.5678GiB/s) | 20260506 12:59:52.321 | INFO | Downloaded 34GiB to GPU in 1.95963s (17.3502GiB/s) | 20260506 12:59:52.475 | INFO | Starting a round of Baz IO | 20260506 12:59:55.614 | INFO | Populating RTMetrics report | 20260506 12:59:55.615 | INFO | RTMetrics report populated | 20260506 12:59:55.615 | INFO | Opening tmp RTMetrics json file | 20260506 12:59:55.615 | INFO | Serializing RTMetrics to json | 20260506 12:59:55.616 | INFO | Serialization done, swapping tmp file now... | 20260506 12:59:55.617 | INFO | RTMetrics done | 20260506 12:59:55.717 | INFO | Finished a round of Baz IO: Wrote 2.103 GiB in 3.242 seconds | 20260506 12:59:55.717 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.49% / 0.02% / 59.49% | 20260506 12:59:55.717 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.49698% / 2.38419e-05% / 4.76837e-05% | 20260506 12:59:55.717 | INFO | Baz IO overall: | 20260506 12:59:55.717 | INFO | BytesWritten : 777146829227 | 20260506 12:59:55.717 | INFO | NumEvents : 375758549495 | 20260506 12:59:57.459 | INFO | Analyzing chunk frames = [1544192,1544704) | 20260506 12:59:57.459 | INFO | MemInfo: { | 20260506 12:59:57.459 | INFO | "gpu" : " 17 GB", | 20260506 12:59:57.459 | INFO | "huge" : "", | 20260506 12:59:57.460 | INFO | "huge_shared" : " 27 GB", | 20260506 12:59:57.460 | INFO | "malloc" : " 14 GB", | 20260506 12:59:57.460 | INFO | "pinned" : " 29 GB" | 20260506 12:59:57.460 | INFO | } | 20260506 12:59:57.460 | INFO | Perf report for the last 5.13898 seconds: | 20260506 12:59:57.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 12:59:57.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:57.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:57.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:57.460 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260506 12:59:57.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 12:59:57.460 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 12:59:57.460 | INFO | PRE_HQ : 61.3%, 1.9, 61.2ms, 0 | 20260506 12:59:57.460 | INFO | BAZWRITER : 63.1%, 3.7, 118.5ms, 0 | 20260506 12:59:57.460 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 12:59:57.460 | INFO | Uploaded 36GiB to GPU in 2.05178s (17.5457GiB/s) | 20260506 12:59:57.460 | INFO | Downloaded 38GiB to GPU in 2.21936s (17.1221GiB/s) | 20260506 13:00:02.599 | INFO | Analyzing chunk frames = [1544704,1545216) | 20260506 13:00:02.599 | INFO | MemInfo: { | 20260506 13:00:02.599 | INFO | "gpu" : " 17 GB", | 20260506 13:00:02.599 | INFO | "huge" : "", | 20260506 13:00:02.599 | INFO | "huge_shared" : " 27 GB", | 20260506 13:00:02.599 | INFO | "malloc" : " 14 GB", | 20260506 13:00:02.599 | INFO | "pinned" : " 29 GB" | 20260506 13:00:02.599 | INFO | } | 20260506 13:00:02.599 | INFO | Perf report for the last 5.13926 seconds: | 20260506 13:00:02.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:00:02.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:02.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:02.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:02.599 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:00:02.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:02.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:02.599 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:00:02.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:02.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:02.599 | INFO | Uploaded 36GiB to GPU in 2.0342s (17.6973GiB/s) | 20260506 13:00:02.599 | INFO | Downloaded 34GiB to GPU in 1.94636s (17.4685GiB/s) | 20260506 13:00:07.738 | INFO | Analyzing chunk frames = [1545216,1545728) | 20260506 13:00:07.738 | INFO | MemInfo: { | 20260506 13:00:07.738 | INFO | "gpu" : " 17 GB", | 20260506 13:00:07.738 | INFO | "huge" : "", | 20260506 13:00:07.738 | INFO | "huge_shared" : " 27 GB", | 20260506 13:00:07.738 | INFO | "malloc" : " 14 GB", | 20260506 13:00:07.738 | INFO | "pinned" : " 29 GB" | 20260506 13:00:07.738 | INFO | } | 20260506 13:00:07.738 | INFO | Perf report for the last 5.13942 seconds: | 20260506 13:00:07.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:00:07.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:07.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:07.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:07.738 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 13:00:07.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:07.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:07.738 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:00:07.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:07.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:07.738 | INFO | Uploaded 36GiB to GPU in 2.03475s (17.6926GiB/s) | 20260506 13:00:07.739 | INFO | Downloaded 34GiB to GPU in 1.93769s (17.5467GiB/s) | 20260506 13:00:12.878 | INFO | Analyzing chunk frames = [1545728,1546240) | 20260506 13:00:12.878 | INFO | MemInfo: { | 20260506 13:00:12.878 | INFO | "gpu" : " 17 GB", | 20260506 13:00:12.878 | INFO | "huge" : "", | 20260506 13:00:12.878 | INFO | "huge_shared" : " 27 GB", | 20260506 13:00:12.878 | INFO | "malloc" : " 14 GB", | 20260506 13:00:12.878 | INFO | "pinned" : " 29 GB" | 20260506 13:00:12.878 | INFO | } | 20260506 13:00:12.878 | INFO | Perf report for the last 5.13939 seconds: | 20260506 13:00:12.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:00:12.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:12.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:12.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:12.878 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 13:00:12.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:12.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:12.878 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:00:12.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:12.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:12.878 | INFO | Uploaded 36GiB to GPU in 1.98419s (18.1435GiB/s) | 20260506 13:00:12.878 | INFO | Downloaded 34GiB to GPU in 1.9385s (17.5394GiB/s) | 20260506 13:00:18.017 | INFO | Analyzing chunk frames = [1546240,1546752) | 20260506 13:00:18.017 | INFO | MemInfo: { | 20260506 13:00:18.017 | INFO | "gpu" : " 17 GB", | 20260506 13:00:18.017 | INFO | "huge" : "", | 20260506 13:00:18.017 | INFO | "huge_shared" : " 27 GB", | 20260506 13:00:18.017 | INFO | "malloc" : " 14 GB", | 20260506 13:00:18.017 | INFO | "pinned" : " 29 GB" | 20260506 13:00:18.017 | INFO | } | 20260506 13:00:18.017 | INFO | Perf report for the last 5.13932 seconds: | 20260506 13:00:18.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:00:18.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:18.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:18.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:18.017 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 13:00:18.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:18.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:18.017 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 13:00:18.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:18.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:18.017 | INFO | Uploaded 36GiB to GPU in 2.06249s (17.4546GiB/s) | 20260506 13:00:18.017 | INFO | Downloaded 34GiB to GPU in 1.95171s (17.4206GiB/s) | 20260506 13:00:23.156 | INFO | Analyzing chunk frames = [1546752,1547264) | 20260506 13:00:23.156 | INFO | MemInfo: { | 20260506 13:00:23.156 | INFO | "gpu" : " 17 GB", | 20260506 13:00:23.156 | INFO | "huge" : "", | 20260506 13:00:23.156 | INFO | "huge_shared" : " 27 GB", | 20260506 13:00:23.156 | INFO | "malloc" : " 14 GB", | 20260506 13:00:23.156 | INFO | "pinned" : " 29 GB" | 20260506 13:00:23.156 | INFO | } | 20260506 13:00:23.156 | INFO | Perf report for the last 5.13939 seconds: | 20260506 13:00:23.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:00:23.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:23.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:23.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:23.156 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:00:23.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:23.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:23.156 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 13:00:23.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:23.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:23.157 | INFO | Uploaded 36GiB to GPU in 2.10179s (17.1283GiB/s) | 20260506 13:00:23.157 | INFO | Downloaded 34GiB to GPU in 1.98598s (17.12GiB/s) | 20260506 13:00:28.297 | INFO | Analyzing chunk frames = [1547264,1547776) | 20260506 13:00:28.297 | INFO | MemInfo: { | 20260506 13:00:28.297 | INFO | "gpu" : " 17 GB", | 20260506 13:00:28.297 | INFO | "huge" : "", | 20260506 13:00:28.297 | INFO | "huge_shared" : " 27 GB", | 20260506 13:00:28.297 | INFO | "malloc" : " 14 GB", | 20260506 13:00:28.297 | INFO | "pinned" : " 29 GB" | 20260506 13:00:28.297 | INFO | } | 20260506 13:00:28.297 | INFO | Perf report for the last 5.14077 seconds: | 20260506 13:00:28.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:00:28.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:28.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:28.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:28.297 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:00:28.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:28.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:28.297 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 13:00:28.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:28.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:28.297 | INFO | Uploaded 36GiB to GPU in 1.99558s (18.0399GiB/s) | 20260506 13:00:28.297 | INFO | Downloaded 34GiB to GPU in 2.03232s (16.7296GiB/s) | 20260506 13:00:33.435 | INFO | Analyzing chunk frames = [1547776,1548288) | 20260506 13:00:33.435 | INFO | MemInfo: { | 20260506 13:00:33.435 | INFO | "gpu" : " 17 GB", | 20260506 13:00:33.435 | INFO | "huge" : "", | 20260506 13:00:33.435 | INFO | "huge_shared" : " 27 GB", | 20260506 13:00:33.435 | INFO | "malloc" : " 14 GB", | 20260506 13:00:33.435 | INFO | "pinned" : " 29 GB" | 20260506 13:00:33.435 | INFO | } | 20260506 13:00:33.436 | INFO | Perf report for the last 5.1383 seconds: | 20260506 13:00:33.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:00:33.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:33.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:33.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:33.436 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:00:33.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:33.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:33.436 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:00:33.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:33.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:33.436 | INFO | Uploaded 36GiB to GPU in 2.03819s (17.6628GiB/s) | 20260506 13:00:33.436 | INFO | Downloaded 34GiB to GPU in 1.96752s (17.2807GiB/s) | 20260506 13:00:33.436 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:00:33.436 | INFO | Total mem usage: 16.8 GiB | 20260506 13:00:33.436 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 13:00:33.436 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -16.2%) | 20260506 13:00:33.436 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:00:33.436 | INFO | High Water Breakdown: | 20260506 13:00:33.436 | INFO | 59.7% : 8.64 GiB ( 83.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:00:33.436 | INFO | 40.1% : 5.81 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:00:33.436 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:00:33.436 | INFO | ------------------------------------------ | 20260506 13:00:33.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:00:33.436 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:00:33.436 | INFO | Total mem usage: 26.5 GiB | 20260506 13:00:33.436 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:00:33.436 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:00:33.436 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:00:33.436 | INFO | High Water Breakdown: | 20260506 13:00:33.436 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:00:33.436 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:00:33.436 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:00:33.436 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:00:33.436 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:00:33.436 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:00:33.436 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:00:33.436 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:00:33.436 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:00:33.436 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:00:33.436 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:00:33.436 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:00:33.436 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:00:33.436 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:00:33.436 | INFO | ------------------------------------------ | 20260506 13:00:33.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:00:33.436 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:00:33.436 | INFO | Total mem usage: 25.6 GiB | 20260506 13:00:33.436 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:00:33.436 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:00:33.436 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:00:33.436 | INFO | High Water Breakdown: | 20260506 13:00:33.436 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:00:33.436 | INFO | ------------------------------------------ | 20260506 13:00:33.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:00:33.436 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:00:33.436 | INFO | Total mem usage: 15.7 GiB | 20260506 13:00:33.436 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:00:33.436 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:00:33.436 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:00:33.436 | INFO | High Water Breakdown: | 20260506 13:00:33.436 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:00:33.436 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:00:33.436 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:00:33.436 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:00:33.436 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:00:33.436 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:00:33.436 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:00:33.436 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:00:33.436 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:00:33.436 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:00:33.436 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:00:33.437 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:00:33.437 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:00:33.437 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:00:33.437 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:00:33.437 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:00:33.437 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:00:33.437 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:00:33.437 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:00:33.437 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:00:33.437 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:00:33.437 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:00:33.437 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:00:33.437 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:00:33.437 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:00:33.437 | INFO | ------------------------------------------ | 20260506 13:00:33.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:00:33.437 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:00:33.437 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:00:33.437 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:00:33.437 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.61e+01, 1.36e+01 / 2.54e+00 | 20260506 13:00:33.437 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.01e-01, 1.04e-02 / 6.66e-03 | 20260506 13:00:33.437 | INFO | PerfReport: 11.58%, Baseline -- 3.66e+00 / 4.47e+00, 5.67e-01 / 3.06e-01 | 20260506 13:00:33.437 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.48e+00, 1.90e-01 / 1.36e-01 | 20260506 13:00:33.437 | INFO | PerfReport: 15.82%, DME -- 4.00e+01 / 4.43e+01, 4.06e+00 / 2.76e+00 | 20260506 13:00:33.437 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.36e+00 / 1.81e+00 | 20260506 13:00:33.437 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.39e+00 / 2.98e+00, 4.14e-01 / 2.86e-01 | 20260506 13:00:33.437 | INFO | PerfReport: 3.72%, Metrics -- 1.18e+00 / 1.55e+00, 3.01e-01 / 2.13e-01 | 20260506 13:00:33.437 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.13e-01 / 4.66e-01, 6.44e-02 / 3.29e-02 | 20260506 13:00:33.437 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:00:33.437 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:00:33.437 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:00:33.437 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 13:00:33.437 | INFO | PerfReport: 21.54%, Upload -- 2.13e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260506 13:00:33.437 | INFO | PerfReport: 21.03%, Download -- 2.07e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 13:00:33.437 | INFO | PerfReport: 32.07%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 13:00:33.437 | INFO | PerfReport: 1.82%, LockUpload -- 1.80e+00 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 13:00:33.437 | INFO | PerfReport: 1.02%, LockDownload -- 1.00e+00 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260506 13:00:33.437 | INFO | PerfReport: 22.50%, LockCompute -- 2.22e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 13:00:33.437 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260506 13:00:33.570 | INFO | Starting a round of Baz IO | 20260506 13:00:36.730 | INFO | Populating RTMetrics report | 20260506 13:00:36.731 | INFO | RTMetrics report populated | 20260506 13:00:36.731 | INFO | Opening tmp RTMetrics json file | 20260506 13:00:36.732 | INFO | Serializing RTMetrics to json | 20260506 13:00:36.733 | INFO | Serialization done, swapping tmp file now... | 20260506 13:00:36.733 | INFO | RTMetrics done | 20260506 13:00:36.835 | INFO | Finished a round of Baz IO: Wrote 2.102 GiB in 3.265 seconds | 20260506 13:00:36.835 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.47% / 0.02% / 59.51% | 20260506 13:00:36.835 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.48443% / 2.78155e-05% / 4.76837e-05% | 20260506 13:00:36.835 | INFO | Baz IO overall: | 20260506 13:00:36.835 | INFO | BytesWritten : 779403785507 | 20260506 13:00:36.835 | INFO | NumEvents : 376853074241 | 20260506 13:00:38.574 | INFO | Analyzing chunk frames = [1548288,1548800) | 20260506 13:00:38.574 | INFO | MemInfo: { | 20260506 13:00:38.574 | INFO | "gpu" : " 17 GB", | 20260506 13:00:38.574 | INFO | "huge" : "", | 20260506 13:00:38.574 | INFO | "huge_shared" : " 27 GB", | 20260506 13:00:38.574 | INFO | "malloc" : " 14 GB", | 20260506 13:00:38.574 | INFO | "pinned" : " 29 GB" | 20260506 13:00:38.574 | INFO | } | 20260506 13:00:38.575 | INFO | Perf report for the last 5.13902 seconds: | 20260506 13:00:38.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:00:38.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:38.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:38.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:38.575 | INFO | ANALYSIS : 63.5%, 3.0, 98.3ms, 0 | 20260506 13:00:38.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:38.575 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 13:00:38.575 | INFO | PRE_HQ : 61.4%, 1.9, 61.3ms, 0 | 20260506 13:00:38.575 | INFO | BAZWRITER : 62.7%, 3.8, 121.1ms, 0 | 20260506 13:00:38.575 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 13:00:38.575 | INFO | Uploaded 36GiB to GPU in 2.04316s (17.6198GiB/s) | 20260506 13:00:38.575 | INFO | Downloaded 38GiB to GPU in 2.21324s (17.1694GiB/s) | 20260506 13:00:43.714 | INFO | Analyzing chunk frames = [1548800,1549312) | 20260506 13:00:43.714 | INFO | MemInfo: { | 20260506 13:00:43.714 | INFO | "gpu" : " 17 GB", | 20260506 13:00:43.714 | INFO | "huge" : "", | 20260506 13:00:43.714 | INFO | "huge_shared" : " 27 GB", | 20260506 13:00:43.714 | INFO | "malloc" : " 14 GB", | 20260506 13:00:43.714 | INFO | "pinned" : " 29 GB" | 20260506 13:00:43.714 | INFO | } | 20260506 13:00:43.714 | INFO | Perf report for the last 5.13917 seconds: | 20260506 13:00:43.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:00:43.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:43.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:43.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:43.714 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:00:43.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:43.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:43.714 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:00:43.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:43.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:43.714 | INFO | Uploaded 36GiB to GPU in 2.01751s (17.8438GiB/s) | 20260506 13:00:43.714 | INFO | Downloaded 34GiB to GPU in 1.93651s (17.5574GiB/s) | 20260506 13:00:48.853 | INFO | Analyzing chunk frames = [1549312,1549824) | 20260506 13:00:48.853 | INFO | MemInfo: { | 20260506 13:00:48.853 | INFO | "gpu" : " 17 GB", | 20260506 13:00:48.853 | INFO | "huge" : "", | 20260506 13:00:48.853 | INFO | "huge_shared" : " 27 GB", | 20260506 13:00:48.853 | INFO | "malloc" : " 14 GB", | 20260506 13:00:48.853 | INFO | "pinned" : " 29 GB" | 20260506 13:00:48.853 | INFO | } | 20260506 13:00:48.853 | INFO | Perf report for the last 5.13958 seconds: | 20260506 13:00:48.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:00:48.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:48.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:48.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:48.853 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 13:00:48.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:48.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:48.853 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:00:48.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:48.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:48.854 | INFO | Uploaded 36GiB to GPU in 2.02251s (17.7996GiB/s) | 20260506 13:00:48.854 | INFO | Downloaded 34GiB to GPU in 1.94779s (17.4557GiB/s) | 20260506 13:00:53.992 | INFO | Analyzing chunk frames = [1549824,1550336) | 20260506 13:00:53.992 | INFO | MemInfo: { | 20260506 13:00:53.992 | INFO | "gpu" : " 17 GB", | 20260506 13:00:53.992 | INFO | "huge" : "", | 20260506 13:00:53.992 | INFO | "huge_shared" : " 27 GB", | 20260506 13:00:53.992 | INFO | "malloc" : " 14 GB", | 20260506 13:00:53.992 | INFO | "pinned" : " 29 GB" | 20260506 13:00:53.992 | INFO | } | 20260506 13:00:53.992 | INFO | Perf report for the last 5.13908 seconds: | 20260506 13:00:53.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:00:53.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:53.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:53.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:53.993 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:00:53.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:53.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:53.993 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 13:00:53.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:53.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:53.993 | INFO | Uploaded 36GiB to GPU in 1.99911s (18.008GiB/s) | 20260506 13:00:53.993 | INFO | Downloaded 34GiB to GPU in 1.94304s (17.4984GiB/s) | 20260506 13:00:59.132 | INFO | Analyzing chunk frames = [1550336,1550848) | 20260506 13:00:59.132 | INFO | MemInfo: { | 20260506 13:00:59.132 | INFO | "gpu" : " 17 GB", | 20260506 13:00:59.132 | INFO | "huge" : "", | 20260506 13:00:59.132 | INFO | "huge_shared" : " 27 GB", | 20260506 13:00:59.132 | INFO | "malloc" : " 14 GB", | 20260506 13:00:59.132 | INFO | "pinned" : " 29 GB" | 20260506 13:00:59.132 | INFO | } | 20260506 13:00:59.132 | INFO | Perf report for the last 5.13961 seconds: | 20260506 13:00:59.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:00:59.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:59.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:59.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:59.132 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 13:00:59.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:59.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:59.132 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:00:59.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:59.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:00:59.132 | INFO | Uploaded 36GiB to GPU in 2.03676s (17.6752GiB/s) | 20260506 13:00:59.132 | INFO | Downloaded 34GiB to GPU in 1.93969s (17.5286GiB/s) | 20260506 13:01:04.271 | INFO | Analyzing chunk frames = [1550848,1551360) | 20260506 13:01:04.271 | INFO | MemInfo: { | 20260506 13:01:04.271 | INFO | "gpu" : " 17 GB", | 20260506 13:01:04.271 | INFO | "huge" : "", | 20260506 13:01:04.271 | INFO | "huge_shared" : " 27 GB", | 20260506 13:01:04.271 | INFO | "malloc" : " 14 GB", | 20260506 13:01:04.271 | INFO | "pinned" : " 29 GB" | 20260506 13:01:04.271 | INFO | } | 20260506 13:01:04.271 | INFO | Perf report for the last 5.13916 seconds: | 20260506 13:01:04.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:01:04.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:04.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:04.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:04.271 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:01:04.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:04.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:04.271 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 13:01:04.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:04.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:04.271 | INFO | Uploaded 36GiB to GPU in 2.08052s (17.3034GiB/s) | 20260506 13:01:04.271 | INFO | Downloaded 34GiB to GPU in 1.98375s (17.1393GiB/s) | 20260506 13:01:09.412 | INFO | Analyzing chunk frames = [1551360,1551872) | 20260506 13:01:09.412 | INFO | MemInfo: { | 20260506 13:01:09.412 | INFO | "gpu" : " 17 GB", | 20260506 13:01:09.412 | INFO | "huge" : "", | 20260506 13:01:09.412 | INFO | "huge_shared" : " 27 GB", | 20260506 13:01:09.412 | INFO | "malloc" : " 14 GB", | 20260506 13:01:09.412 | INFO | "pinned" : " 29 GB" | 20260506 13:01:09.412 | INFO | } | 20260506 13:01:09.412 | INFO | Perf report for the last 5.14056 seconds: | 20260506 13:01:09.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:01:09.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:09.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:09.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:09.412 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 13:01:09.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:09.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:09.412 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 13:01:09.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:09.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:09.412 | INFO | Uploaded 36GiB to GPU in 1.9678s (18.2946GiB/s) | 20260506 13:01:09.412 | INFO | Downloaded 34GiB to GPU in 2.02852s (16.761GiB/s) | 20260506 13:01:14.550 | INFO | Analyzing chunk frames = [1551872,1552384) | 20260506 13:01:14.550 | INFO | MemInfo: { | 20260506 13:01:14.550 | INFO | "gpu" : " 17 GB", | 20260506 13:01:14.550 | INFO | "huge" : "", | 20260506 13:01:14.550 | INFO | "huge_shared" : " 27 GB", | 20260506 13:01:14.550 | INFO | "malloc" : " 14 GB", | 20260506 13:01:14.550 | INFO | "pinned" : " 29 GB" | 20260506 13:01:14.550 | INFO | } | 20260506 13:01:14.550 | INFO | Perf report for the last 5.13852 seconds: | 20260506 13:01:14.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:01:14.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:14.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:14.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:14.550 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:01:14.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:14.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:14.550 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:01:14.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:14.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:14.551 | INFO | Uploaded 36GiB to GPU in 2.04816s (17.5768GiB/s) | 20260506 13:01:14.551 | INFO | Downloaded 34GiB to GPU in 1.95732s (17.3707GiB/s) | 20260506 13:01:14.678 | INFO | Starting a round of Baz IO | 20260506 13:01:17.840 | INFO | Populating RTMetrics report | 20260506 13:01:17.841 | INFO | RTMetrics report populated | 20260506 13:01:17.841 | INFO | Opening tmp RTMetrics json file | 20260506 13:01:17.841 | INFO | Serializing RTMetrics to json | 20260506 13:01:17.842 | INFO | Serialization done, swapping tmp file now... | 20260506 13:01:17.843 | INFO | RTMetrics done | 20260506 13:01:17.937 | INFO | Finished a round of Baz IO: Wrote 2.105 GiB in 3.258 seconds | 20260506 13:01:17.937 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.44% / 0.02% / 59.53% | 20260506 13:01:17.937 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.4903% / 3.97364e-05% / 4.76837e-05% | 20260506 13:01:17.937 | INFO | Baz IO overall: | 20260506 13:01:17.937 | INFO | BytesWritten : 781663782257 | 20260506 13:01:17.937 | INFO | NumEvents : 377949868460 | 20260506 13:01:19.689 | INFO | Analyzing chunk frames = [1552384,1552896) | 20260506 13:01:19.689 | INFO | MemInfo: { | 20260506 13:01:19.689 | INFO | "gpu" : " 17 GB", | 20260506 13:01:19.689 | INFO | "huge" : "", | 20260506 13:01:19.689 | INFO | "huge_shared" : " 27 GB", | 20260506 13:01:19.689 | INFO | "malloc" : " 14 GB", | 20260506 13:01:19.689 | INFO | "pinned" : " 29 GB" | 20260506 13:01:19.689 | INFO | } | 20260506 13:01:19.689 | INFO | Perf report for the last 5.13885 seconds: | 20260506 13:01:19.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:01:19.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:19.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:19.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:19.689 | INFO | ANALYSIS : 63.4%, 3.0, 98.3ms, 0 | 20260506 13:01:19.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:19.689 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 13:01:19.689 | INFO | PRE_HQ : 61.4%, 1.9, 61.4ms, 0 | 20260506 13:01:19.689 | INFO | BAZWRITER : 62.8%, 3.7, 118.7ms, 0 | 20260506 13:01:19.689 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 13:01:19.689 | INFO | Uploaded 36GiB to GPU in 2.07417s (17.3563GiB/s) | 20260506 13:01:19.689 | INFO | Downloaded 38GiB to GPU in 2.22328s (17.0919GiB/s) | 20260506 13:01:24.829 | INFO | Analyzing chunk frames = [1552896,1553408) | 20260506 13:01:24.829 | INFO | MemInfo: { | 20260506 13:01:24.829 | INFO | "gpu" : " 17 GB", | 20260506 13:01:24.829 | INFO | "huge" : "", | 20260506 13:01:24.829 | INFO | "huge_shared" : " 27 GB", | 20260506 13:01:24.829 | INFO | "malloc" : " 14 GB", | 20260506 13:01:24.829 | INFO | "pinned" : " 29 GB" | 20260506 13:01:24.829 | INFO | } | 20260506 13:01:24.829 | INFO | Perf report for the last 5.13979 seconds: | 20260506 13:01:24.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:01:24.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:24.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:24.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:24.829 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 13:01:24.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:24.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:24.829 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 13:01:24.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:24.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:24.829 | INFO | Uploaded 36GiB to GPU in 2.0133s (17.8811GiB/s) | 20260506 13:01:24.829 | INFO | Downloaded 34GiB to GPU in 1.92015s (17.707GiB/s) | 20260506 13:01:29.968 | INFO | Analyzing chunk frames = [1553408,1553920) | 20260506 13:01:29.968 | INFO | MemInfo: { | 20260506 13:01:29.968 | INFO | "gpu" : " 17 GB", | 20260506 13:01:29.968 | INFO | "huge" : "", | 20260506 13:01:29.968 | INFO | "huge_shared" : " 27 GB", | 20260506 13:01:29.968 | INFO | "malloc" : " 14 GB", | 20260506 13:01:29.968 | INFO | "pinned" : " 29 GB" | 20260506 13:01:29.968 | INFO | } | 20260506 13:01:29.968 | INFO | Perf report for the last 5.13892 seconds: | 20260506 13:01:29.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:01:29.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:29.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:29.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:29.968 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 13:01:29.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:29.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:29.968 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 13:01:29.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:29.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:29.968 | INFO | Uploaded 36GiB to GPU in 2.0143s (17.8722GiB/s) | 20260506 13:01:29.968 | INFO | Downloaded 34GiB to GPU in 1.94092s (17.5175GiB/s) | 20260506 13:01:35.107 | INFO | Analyzing chunk frames = [1553920,1554432) | 20260506 13:01:35.107 | INFO | MemInfo: { | 20260506 13:01:35.107 | INFO | "gpu" : " 17 GB", | 20260506 13:01:35.107 | INFO | "huge" : "", | 20260506 13:01:35.107 | INFO | "huge_shared" : " 27 GB", | 20260506 13:01:35.107 | INFO | "malloc" : " 14 GB", | 20260506 13:01:35.107 | INFO | "pinned" : " 29 GB" | 20260506 13:01:35.107 | INFO | } | 20260506 13:01:35.107 | INFO | Perf report for the last 5.13935 seconds: | 20260506 13:01:35.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:01:35.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:35.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:35.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:35.107 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:01:35.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:35.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:35.107 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:01:35.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:35.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:35.107 | INFO | Uploaded 36GiB to GPU in 2.00591s (17.947GiB/s) | 20260506 13:01:35.108 | INFO | Downloaded 34GiB to GPU in 1.91884s (17.719GiB/s) | 20260506 13:01:40.247 | INFO | Analyzing chunk frames = [1554432,1554944) | 20260506 13:01:40.247 | INFO | MemInfo: { | 20260506 13:01:40.247 | INFO | "gpu" : " 17 GB", | 20260506 13:01:40.247 | INFO | "huge" : "", | 20260506 13:01:40.247 | INFO | "huge_shared" : " 27 GB", | 20260506 13:01:40.247 | INFO | "malloc" : " 14 GB", | 20260506 13:01:40.247 | INFO | "pinned" : " 29 GB" | 20260506 13:01:40.247 | INFO | } | 20260506 13:01:40.247 | INFO | Perf report for the last 5.13944 seconds: | 20260506 13:01:40.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:01:40.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:40.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:40.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:40.247 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 13:01:40.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:40.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:40.247 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:01:40.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:40.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:40.247 | INFO | Uploaded 36GiB to GPU in 2.031s (17.7253GiB/s) | 20260506 13:01:40.247 | INFO | Downloaded 34GiB to GPU in 1.94449s (17.4853GiB/s) | 20260506 13:01:45.386 | INFO | Analyzing chunk frames = [1554944,1555456) | 20260506 13:01:45.386 | INFO | MemInfo: { | 20260506 13:01:45.386 | INFO | "gpu" : " 17 GB", | 20260506 13:01:45.386 | INFO | "huge" : "", | 20260506 13:01:45.386 | INFO | "huge_shared" : " 27 GB", | 20260506 13:01:45.386 | INFO | "malloc" : " 14 GB", | 20260506 13:01:45.386 | INFO | "pinned" : " 29 GB" | 20260506 13:01:45.386 | INFO | } | 20260506 13:01:45.386 | INFO | Perf report for the last 5.13929 seconds: | 20260506 13:01:45.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:01:45.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:45.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:45.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:45.386 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 13:01:45.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:45.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:45.386 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:01:45.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:45.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:45.386 | INFO | Uploaded 36GiB to GPU in 2.04978s (17.5629GiB/s) | 20260506 13:01:45.386 | INFO | Downloaded 34GiB to GPU in 1.92708s (17.6433GiB/s) | 20260506 13:01:50.526 | INFO | Analyzing chunk frames = [1555456,1555968) | 20260506 13:01:50.526 | INFO | MemInfo: { | 20260506 13:01:50.526 | INFO | "gpu" : " 17 GB", | 20260506 13:01:50.526 | INFO | "huge" : "", | 20260506 13:01:50.526 | INFO | "huge_shared" : " 27 GB", | 20260506 13:01:50.526 | INFO | "malloc" : " 14 GB", | 20260506 13:01:50.526 | INFO | "pinned" : " 29 GB" | 20260506 13:01:50.526 | INFO | } | 20260506 13:01:50.526 | INFO | Perf report for the last 5.13992 seconds: | 20260506 13:01:50.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:01:50.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:50.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:50.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:50.526 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 13:01:50.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:50.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:50.526 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:01:50.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:50.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:50.526 | INFO | Uploaded 36GiB to GPU in 2.03186s (17.7178GiB/s) | 20260506 13:01:50.526 | INFO | Downloaded 34GiB to GPU in 2.02363s (16.8015GiB/s) | 20260506 13:01:55.665 | INFO | Analyzing chunk frames = [1555968,1556480) | 20260506 13:01:55.665 | INFO | MemInfo: { | 20260506 13:01:55.665 | INFO | "gpu" : " 17 GB", | 20260506 13:01:55.665 | INFO | "huge" : "", | 20260506 13:01:55.665 | INFO | "huge_shared" : " 27 GB", | 20260506 13:01:55.665 | INFO | "malloc" : " 14 GB", | 20260506 13:01:55.665 | INFO | "pinned" : " 29 GB" | 20260506 13:01:55.665 | INFO | } | 20260506 13:01:55.665 | INFO | Perf report for the last 5.13913 seconds: | 20260506 13:01:55.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:01:55.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:55.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:55.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:55.665 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 13:01:55.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:55.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:55.665 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 13:01:55.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:55.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:01:55.665 | INFO | Uploaded 36GiB to GPU in 2.02611s (17.768GiB/s) | 20260506 13:01:55.665 | INFO | Downloaded 34GiB to GPU in 2.01616s (16.8637GiB/s) | 20260506 13:01:55.666 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:01:55.666 | INFO | Total mem usage: 16.8 GiB | 20260506 13:01:55.666 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 13:01:55.666 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -15.9%) | 20260506 13:01:55.666 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:01:55.666 | INFO | High Water Breakdown: | 20260506 13:01:55.666 | INFO | 59.8% : 8.68 GiB ( 83.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:01:55.666 | INFO | 40% : 5.81 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:01:55.666 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:01:55.666 | INFO | ------------------------------------------ | 20260506 13:01:55.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:01:55.666 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:01:55.666 | INFO | Total mem usage: 26.5 GiB | 20260506 13:01:55.666 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:01:55.666 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:01:55.666 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:01:55.666 | INFO | High Water Breakdown: | 20260506 13:01:55.666 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:01:55.666 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:01:55.666 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:01:55.666 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:01:55.666 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:01:55.666 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:01:55.666 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:01:55.666 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:01:55.666 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:01:55.666 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:01:55.666 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:01:55.666 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:01:55.666 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:01:55.666 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:01:55.666 | INFO | ------------------------------------------ | 20260506 13:01:55.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:01:55.666 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:01:55.666 | INFO | Total mem usage: 25.6 GiB | 20260506 13:01:55.666 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:01:55.666 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:01:55.666 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:01:55.666 | INFO | High Water Breakdown: | 20260506 13:01:55.666 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:01:55.666 | INFO | ------------------------------------------ | 20260506 13:01:55.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:01:55.666 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:01:55.666 | INFO | Total mem usage: 15.7 GiB | 20260506 13:01:55.666 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:01:55.666 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:01:55.666 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:01:55.666 | INFO | High Water Breakdown: | 20260506 13:01:55.666 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:01:55.666 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:01:55.666 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:01:55.666 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:01:55.666 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:01:55.666 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:01:55.666 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:01:55.666 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:01:55.666 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:01:55.666 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:01:55.666 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:01:55.666 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:01:55.666 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:01:55.666 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:01:55.666 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:01:55.666 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:01:55.666 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:01:55.666 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:01:55.666 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:01:55.666 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:01:55.666 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:01:55.666 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:01:55.666 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:01:55.666 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:01:55.666 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:01:55.666 | INFO | ------------------------------------------ | 20260506 13:01:55.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:01:55.666 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:01:55.666 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:01:55.666 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:01:55.666 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.58e+01, 1.36e+01 / 2.29e+00 | 20260506 13:01:55.666 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.00e-01, 1.09e-02 / 7.65e-03 | 20260506 13:01:55.666 | INFO | PerfReport: 11.55%, Baseline -- 3.66e+00 / 4.39e+00, 5.72e-01 / 3.89e-01 | 20260506 13:01:55.666 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.44e+00, 1.90e-01 / 1.47e-01 | 20260506 13:01:55.666 | INFO | PerfReport: 15.77%, DME -- 4.00e+01 / 4.41e+01, 3.86e+00 / 2.33e+00 | 20260506 13:01:55.666 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.80e+01 / 2.08e+01, 1.51e+00 / 1.68e+00 | 20260506 13:01:55.666 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.38e+00 / 2.99e+00, 4.18e-01 / 2.48e-01 | 20260506 13:01:55.666 | INFO | PerfReport: 3.70%, Metrics -- 1.17e+00 / 1.56e+00, 3.00e-01 / 1.66e-01 | 20260506 13:01:55.666 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.13e-01 / 4.65e-01, 6.45e-02 / 2.98e-02 | 20260506 13:01:55.667 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:01:55.667 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:01:55.667 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:01:55.667 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 13:01:55.667 | INFO | PerfReport: 21.45%, Upload -- 2.11e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260506 13:01:55.667 | INFO | PerfReport: 21.05%, Download -- 2.07e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 13:01:55.667 | INFO | PerfReport: 32.18%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 13:01:55.667 | INFO | PerfReport: 1.68%, LockUpload -- 1.66e+00 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 13:01:55.667 | INFO | PerfReport: 1.07%, LockDownload -- 1.05e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 13:01:55.667 | INFO | PerfReport: 22.56%, LockCompute -- 2.22e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 13:01:55.667 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260506 13:01:55.794 | INFO | Starting a round of Baz IO | 20260506 13:01:58.954 | INFO | Populating RTMetrics report | 20260506 13:01:58.954 | INFO | RTMetrics report populated | 20260506 13:01:58.954 | INFO | Opening tmp RTMetrics json file | 20260506 13:01:58.955 | INFO | Serializing RTMetrics to json | 20260506 13:01:58.956 | INFO | Serialization done, swapping tmp file now... | 20260506 13:01:58.957 | INFO | RTMetrics done | 20260506 13:01:59.056 | INFO | Finished a round of Baz IO: Wrote 2.105 GiB in 3.262 seconds | 20260506 13:01:59.056 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.42% / 0.02% / 59.56% | 20260506 13:01:59.056 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.4731% / 1.98682e-05% / 2.78155e-05% | 20260506 13:01:59.056 | INFO | Baz IO overall: | 20260506 13:01:59.056 | INFO | BytesWritten : 783923478431 | 20260506 13:01:59.056 | INFO | NumEvents : 379046296187 | 20260506 13:02:00.804 | INFO | Analyzing chunk frames = [1556480,1556992) | 20260506 13:02:00.804 | INFO | MemInfo: { | 20260506 13:02:00.804 | INFO | "gpu" : " 17 GB", | 20260506 13:02:00.804 | INFO | "huge" : "", | 20260506 13:02:00.804 | INFO | "huge_shared" : " 27 GB", | 20260506 13:02:00.804 | INFO | "malloc" : " 14 GB", | 20260506 13:02:00.804 | INFO | "pinned" : " 29 GB" | 20260506 13:02:00.804 | INFO | } | 20260506 13:02:00.804 | INFO | Perf report for the last 5.13893 seconds: | 20260506 13:02:00.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:02:00.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:00.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:00.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:00.804 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260506 13:02:00.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:00.804 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260506 13:02:00.804 | INFO | PRE_HQ : 61.2%, 1.9, 60.8ms, 0 | 20260506 13:02:00.804 | INFO | BAZWRITER : 62.8%, 3.7, 118.2ms, 0 | 20260506 13:02:00.804 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 13:02:00.804 | INFO | Uploaded 36GiB to GPU in 2.04039s (17.6437GiB/s) | 20260506 13:02:00.804 | INFO | Downloaded 38GiB to GPU in 2.2171s (17.1395GiB/s) | 20260506 13:02:05.943 | INFO | Analyzing chunk frames = [1556992,1557504) | 20260506 13:02:05.943 | INFO | MemInfo: { | 20260506 13:02:05.943 | INFO | "gpu" : " 17 GB", | 20260506 13:02:05.943 | INFO | "huge" : "", | 20260506 13:02:05.943 | INFO | "huge_shared" : " 27 GB", | 20260506 13:02:05.943 | INFO | "malloc" : " 14 GB", | 20260506 13:02:05.943 | INFO | "pinned" : " 29 GB" | 20260506 13:02:05.943 | INFO | } | 20260506 13:02:05.943 | INFO | Perf report for the last 5.13922 seconds: | 20260506 13:02:05.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:02:05.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:05.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:05.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:05.944 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:02:05.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:05.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:05.944 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:02:05.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:05.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:05.944 | INFO | Uploaded 36GiB to GPU in 2.01356s (17.8788GiB/s) | 20260506 13:02:05.944 | INFO | Downloaded 34GiB to GPU in 1.93912s (17.5337GiB/s) | 20260506 13:02:11.083 | INFO | Analyzing chunk frames = [1557504,1558016) | 20260506 13:02:11.083 | INFO | MemInfo: { | 20260506 13:02:11.083 | INFO | "gpu" : " 17 GB", | 20260506 13:02:11.083 | INFO | "huge" : "", | 20260506 13:02:11.083 | INFO | "huge_shared" : " 27 GB", | 20260506 13:02:11.083 | INFO | "malloc" : " 14 GB", | 20260506 13:02:11.083 | INFO | "pinned" : " 29 GB" | 20260506 13:02:11.083 | INFO | } | 20260506 13:02:11.083 | INFO | Perf report for the last 5.13941 seconds: | 20260506 13:02:11.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:02:11.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:11.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:11.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:11.083 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 13:02:11.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:11.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:11.083 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:02:11.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:11.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:11.083 | INFO | Uploaded 36GiB to GPU in 2.01144s (17.8976GiB/s) | 20260506 13:02:11.083 | INFO | Downloaded 34GiB to GPU in 1.94358s (17.4935GiB/s) | 20260506 13:02:16.222 | INFO | Analyzing chunk frames = [1558016,1558528) | 20260506 13:02:16.222 | INFO | MemInfo: { | 20260506 13:02:16.222 | INFO | "gpu" : " 17 GB", | 20260506 13:02:16.222 | INFO | "huge" : "", | 20260506 13:02:16.222 | INFO | "huge_shared" : " 27 GB", | 20260506 13:02:16.222 | INFO | "malloc" : " 14 GB", | 20260506 13:02:16.222 | INFO | "pinned" : " 29 GB" | 20260506 13:02:16.222 | INFO | } | 20260506 13:02:16.222 | INFO | Perf report for the last 5.13928 seconds: | 20260506 13:02:16.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:02:16.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:16.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:16.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:16.222 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 13:02:16.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:16.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:16.222 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:02:16.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:16.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:16.222 | INFO | Uploaded 36GiB to GPU in 1.99964s (18.0033GiB/s) | 20260506 13:02:16.222 | INFO | Downloaded 34GiB to GPU in 1.94523s (17.4786GiB/s) | 20260506 13:02:21.361 | INFO | Analyzing chunk frames = [1558528,1559040) | 20260506 13:02:21.361 | INFO | MemInfo: { | 20260506 13:02:21.361 | INFO | "gpu" : " 17 GB", | 20260506 13:02:21.361 | INFO | "huge" : "", | 20260506 13:02:21.361 | INFO | "huge_shared" : " 27 GB", | 20260506 13:02:21.361 | INFO | "malloc" : " 14 GB", | 20260506 13:02:21.361 | INFO | "pinned" : " 29 GB" | 20260506 13:02:21.362 | INFO | } | 20260506 13:02:21.362 | INFO | Perf report for the last 5.13955 seconds: | 20260506 13:02:21.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:02:21.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:21.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:21.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:21.362 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 13:02:21.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:21.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:21.362 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:02:21.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:21.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:21.362 | INFO | Uploaded 36GiB to GPU in 2.00689s (17.9382GiB/s) | 20260506 13:02:21.362 | INFO | Downloaded 34GiB to GPU in 1.94675s (17.465GiB/s) | 20260506 13:02:26.501 | INFO | Analyzing chunk frames = [1559040,1559552) | 20260506 13:02:26.501 | INFO | MemInfo: { | 20260506 13:02:26.501 | INFO | "gpu" : " 17 GB", | 20260506 13:02:26.501 | INFO | "huge" : "", | 20260506 13:02:26.501 | INFO | "huge_shared" : " 27 GB", | 20260506 13:02:26.501 | INFO | "malloc" : " 14 GB", | 20260506 13:02:26.501 | INFO | "pinned" : " 29 GB" | 20260506 13:02:26.501 | INFO | } | 20260506 13:02:26.501 | INFO | Perf report for the last 5.13934 seconds: | 20260506 13:02:26.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:02:26.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:26.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:26.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:26.501 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 13:02:26.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:26.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:26.501 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260506 13:02:26.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:26.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:26.501 | INFO | Uploaded 36GiB to GPU in 2.01993s (17.8224GiB/s) | 20260506 13:02:26.501 | INFO | Downloaded 34GiB to GPU in 1.93567s (17.565GiB/s) | 20260506 13:02:31.641 | INFO | Analyzing chunk frames = [1559552,1560064) | 20260506 13:02:31.641 | INFO | MemInfo: { | 20260506 13:02:31.641 | INFO | "gpu" : " 17 GB", | 20260506 13:02:31.641 | INFO | "huge" : "", | 20260506 13:02:31.641 | INFO | "huge_shared" : " 27 GB", | 20260506 13:02:31.641 | INFO | "malloc" : " 14 GB", | 20260506 13:02:31.641 | INFO | "pinned" : " 29 GB" | 20260506 13:02:31.641 | INFO | } | 20260506 13:02:31.641 | INFO | Perf report for the last 5.13983 seconds: | 20260506 13:02:31.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:02:31.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:31.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:31.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:31.641 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 13:02:31.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:31.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:31.641 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 13:02:31.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:31.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:31.641 | INFO | Uploaded 36GiB to GPU in 1.99496s (18.0455GiB/s) | 20260506 13:02:31.641 | INFO | Downloaded 34GiB to GPU in 2.01159s (16.902GiB/s) | 20260506 13:02:36.779 | INFO | Analyzing chunk frames = [1560064,1560576) | 20260506 13:02:36.779 | INFO | MemInfo: { | 20260506 13:02:36.779 | INFO | "gpu" : " 17 GB", | 20260506 13:02:36.779 | INFO | "huge" : "", | 20260506 13:02:36.779 | INFO | "huge_shared" : " 27 GB", | 20260506 13:02:36.779 | INFO | "malloc" : " 14 GB", | 20260506 13:02:36.779 | INFO | "pinned" : " 29 GB" | 20260506 13:02:36.779 | INFO | } | 20260506 13:02:36.779 | INFO | Perf report for the last 5.13867 seconds: | 20260506 13:02:36.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:02:36.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:36.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:36.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:36.779 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 13:02:36.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:36.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:36.780 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:02:36.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:36.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:36.780 | INFO | Uploaded 36GiB to GPU in 2.05235s (17.5409GiB/s) | 20260506 13:02:36.780 | INFO | Downloaded 34GiB to GPU in 1.96698s (17.2854GiB/s) | 20260506 13:02:36.913 | INFO | Starting a round of Baz IO | 20260506 13:02:40.064 | INFO | Populating RTMetrics report | 20260506 13:02:40.064 | INFO | RTMetrics report populated | 20260506 13:02:40.064 | INFO | Opening tmp RTMetrics json file | 20260506 13:02:40.065 | INFO | Serializing RTMetrics to json | 20260506 13:02:40.066 | INFO | Serialization done, swapping tmp file now... | 20260506 13:02:40.066 | INFO | RTMetrics done | 20260506 13:02:40.163 | INFO | Finished a round of Baz IO: Wrote 2.104 GiB in 3.249 seconds | 20260506 13:02:40.163 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.40% / 0.02% / 59.58% | 20260506 13:02:40.163 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.46425% / 3.57628e-05% / 6.35783e-05% | 20260506 13:02:40.163 | INFO | Baz IO overall: | 20260506 13:02:40.163 | INFO | BytesWritten : 786183058084 | 20260506 13:02:40.163 | INFO | NumEvents : 380142551875 | 20260506 13:02:41.919 | INFO | Analyzing chunk frames = [1560576,1561088) | 20260506 13:02:41.919 | INFO | MemInfo: { | 20260506 13:02:41.919 | INFO | "gpu" : " 17 GB", | 20260506 13:02:41.919 | INFO | "huge" : "", | 20260506 13:02:41.919 | INFO | "huge_shared" : " 27 GB", | 20260506 13:02:41.919 | INFO | "malloc" : " 14 GB", | 20260506 13:02:41.919 | INFO | "pinned" : " 29 GB" | 20260506 13:02:41.919 | INFO | } | 20260506 13:02:41.919 | INFO | Perf report for the last 5.13976 seconds: | 20260506 13:02:41.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:02:41.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:41.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:41.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:41.919 | INFO | ANALYSIS : 63.3%, 3.0, 98.4ms, 0 | 20260506 13:02:41.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:41.919 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:02:41.919 | INFO | PRE_HQ : 61.3%, 1.9, 61.0ms, 0 | 20260506 13:02:41.919 | INFO | BAZWRITER : 62.7%, 3.7, 118.8ms, 0 | 20260506 13:02:41.919 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 13:02:41.919 | INFO | Uploaded 36GiB to GPU in 2.07684s (17.334GiB/s) | 20260506 13:02:41.920 | INFO | Downloaded 38GiB to GPU in 2.21884s (17.1261GiB/s) | 20260506 13:02:47.058 | INFO | Analyzing chunk frames = [1561088,1561600) | 20260506 13:02:47.058 | INFO | MemInfo: { | 20260506 13:02:47.058 | INFO | "gpu" : " 17 GB", | 20260506 13:02:47.058 | INFO | "huge" : "", | 20260506 13:02:47.058 | INFO | "huge_shared" : " 27 GB", | 20260506 13:02:47.058 | INFO | "malloc" : " 14 GB", | 20260506 13:02:47.058 | INFO | "pinned" : " 29 GB" | 20260506 13:02:47.058 | INFO | } | 20260506 13:02:47.058 | INFO | Perf report for the last 5.13886 seconds: | 20260506 13:02:47.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:02:47.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:47.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:47.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:47.058 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 13:02:47.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:47.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:47.058 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:02:47.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:47.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:47.058 | INFO | Uploaded 36GiB to GPU in 2.01577s (17.8592GiB/s) | 20260506 13:02:47.058 | INFO | Downloaded 34GiB to GPU in 1.92768s (17.6377GiB/s) | 20260506 13:02:52.198 | INFO | Analyzing chunk frames = [1561600,1562112) | 20260506 13:02:52.198 | INFO | MemInfo: { | 20260506 13:02:52.198 | INFO | "gpu" : " 17 GB", | 20260506 13:02:52.198 | INFO | "huge" : "", | 20260506 13:02:52.198 | INFO | "huge_shared" : " 27 GB", | 20260506 13:02:52.198 | INFO | "malloc" : " 14 GB", | 20260506 13:02:52.198 | INFO | "pinned" : " 29 GB" | 20260506 13:02:52.198 | INFO | } | 20260506 13:02:52.198 | INFO | Perf report for the last 5.13968 seconds: | 20260506 13:02:52.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:02:52.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:52.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:52.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:52.198 | INFO | ANALYSIS : 61.4%, 3.0, 95.1ms, 0 | 20260506 13:02:52.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:52.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:52.198 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:02:52.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:52.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:52.198 | INFO | Uploaded 36GiB to GPU in 1.98162s (18.1669GiB/s) | 20260506 13:02:52.198 | INFO | Downloaded 34GiB to GPU in 1.92707s (17.6433GiB/s) | 20260506 13:02:57.337 | INFO | Analyzing chunk frames = [1562112,1562624) | 20260506 13:02:57.337 | INFO | MemInfo: { | 20260506 13:02:57.337 | INFO | "gpu" : " 17 GB", | 20260506 13:02:57.337 | INFO | "huge" : "", | 20260506 13:02:57.337 | INFO | "huge_shared" : " 27 GB", | 20260506 13:02:57.337 | INFO | "malloc" : " 14 GB", | 20260506 13:02:57.337 | INFO | "pinned" : " 29 GB" | 20260506 13:02:57.337 | INFO | } | 20260506 13:02:57.337 | INFO | Perf report for the last 5.1393 seconds: | 20260506 13:02:57.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:02:57.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:57.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:57.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:57.337 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:02:57.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:57.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:57.337 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:02:57.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:57.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:02:57.337 | INFO | Uploaded 36GiB to GPU in 2.00419s (17.9624GiB/s) | 20260506 13:02:57.337 | INFO | Downloaded 34GiB to GPU in 1.91341s (17.7693GiB/s) | 20260506 13:03:02.476 | INFO | Analyzing chunk frames = [1562624,1563136) | 20260506 13:03:02.476 | INFO | MemInfo: { | 20260506 13:03:02.476 | INFO | "gpu" : " 17 GB", | 20260506 13:03:02.476 | INFO | "huge" : "", | 20260506 13:03:02.476 | INFO | "huge_shared" : " 27 GB", | 20260506 13:03:02.476 | INFO | "malloc" : " 14 GB", | 20260506 13:03:02.476 | INFO | "pinned" : " 29 GB" | 20260506 13:03:02.476 | INFO | } | 20260506 13:03:02.476 | INFO | Perf report for the last 5.13916 seconds: | 20260506 13:03:02.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:03:02.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:02.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:02.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:02.476 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 13:03:02.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:02.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:02.476 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:03:02.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:02.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:02.477 | INFO | Uploaded 36GiB to GPU in 2.01542s (17.8623GiB/s) | 20260506 13:03:02.477 | INFO | Downloaded 34GiB to GPU in 1.94352s (17.494GiB/s) | 20260506 13:03:07.615 | INFO | Analyzing chunk frames = [1563136,1563648) | 20260506 13:03:07.615 | INFO | MemInfo: { | 20260506 13:03:07.615 | INFO | "gpu" : " 17 GB", | 20260506 13:03:07.615 | INFO | "huge" : "", | 20260506 13:03:07.615 | INFO | "huge_shared" : " 27 GB", | 20260506 13:03:07.615 | INFO | "malloc" : " 14 GB", | 20260506 13:03:07.615 | INFO | "pinned" : " 29 GB" | 20260506 13:03:07.615 | INFO | } | 20260506 13:03:07.615 | INFO | Perf report for the last 5.13912 seconds: | 20260506 13:03:07.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:03:07.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:07.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:07.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:07.615 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 13:03:07.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:07.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:07.615 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 13:03:07.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:07.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:07.616 | INFO | Uploaded 36GiB to GPU in 2.04051s (17.6426GiB/s) | 20260506 13:03:07.616 | INFO | Downloaded 34GiB to GPU in 1.93328s (17.5867GiB/s) | 20260506 13:03:12.755 | INFO | Analyzing chunk frames = [1563648,1564160) | 20260506 13:03:12.755 | INFO | MemInfo: { | 20260506 13:03:12.755 | INFO | "gpu" : " 17 GB", | 20260506 13:03:12.755 | INFO | "huge" : "", | 20260506 13:03:12.755 | INFO | "huge_shared" : " 27 GB", | 20260506 13:03:12.755 | INFO | "malloc" : " 14 GB", | 20260506 13:03:12.755 | INFO | "pinned" : " 29 GB" | 20260506 13:03:12.755 | INFO | } | 20260506 13:03:12.755 | INFO | Perf report for the last 5.13997 seconds: | 20260506 13:03:12.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:03:12.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:12.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:12.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:12.755 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:03:12.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:12.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:12.755 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:03:12.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:12.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:12.756 | INFO | Uploaded 36GiB to GPU in 2.03061s (17.7287GiB/s) | 20260506 13:03:12.756 | INFO | Downloaded 34GiB to GPU in 1.96723s (17.2832GiB/s) | 20260506 13:03:17.894 | INFO | Analyzing chunk frames = [1564160,1564672) | 20260506 13:03:17.894 | INFO | MemInfo: { | 20260506 13:03:17.894 | INFO | "gpu" : " 17 GB", | 20260506 13:03:17.894 | INFO | "huge" : "", | 20260506 13:03:17.894 | INFO | "huge_shared" : " 27 GB", | 20260506 13:03:17.894 | INFO | "malloc" : " 14 GB", | 20260506 13:03:17.894 | INFO | "pinned" : " 29 GB" | 20260506 13:03:17.894 | INFO | } | 20260506 13:03:17.894 | INFO | Perf report for the last 5.13881 seconds: | 20260506 13:03:17.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:03:17.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:17.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:17.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:17.894 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 13:03:17.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:17.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:17.894 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 13:03:17.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:17.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:17.894 | INFO | Uploaded 36GiB to GPU in 2.07984s (17.309GiB/s) | 20260506 13:03:17.894 | INFO | Downloaded 34GiB to GPU in 1.97996s (17.172GiB/s) | 20260506 13:03:17.895 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:03:17.895 | INFO | Total mem usage: 16.8 GiB | 20260506 13:03:17.895 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 13:03:17.895 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.7%) | 20260506 13:03:17.895 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:03:17.895 | INFO | High Water Breakdown: | 20260506 13:03:17.895 | INFO | 59.7% : 8.61 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:03:17.895 | INFO | 40.1% : 5.78 GiB ( 86.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:03:17.895 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:03:17.895 | INFO | ------------------------------------------ | 20260506 13:03:17.895 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:03:17.895 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:03:17.895 | INFO | Total mem usage: 26.5 GiB | 20260506 13:03:17.895 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:03:17.895 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:03:17.895 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:03:17.895 | INFO | High Water Breakdown: | 20260506 13:03:17.895 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:03:17.895 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:03:17.895 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:03:17.895 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:03:17.895 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:03:17.895 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:03:17.895 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:03:17.895 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:03:17.895 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:03:17.895 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:03:17.895 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:03:17.895 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:03:17.895 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:03:17.895 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:03:17.895 | INFO | ------------------------------------------ | 20260506 13:03:17.895 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:03:17.895 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:03:17.895 | INFO | Total mem usage: 25.6 GiB | 20260506 13:03:17.895 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:03:17.895 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:03:17.895 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:03:17.895 | INFO | High Water Breakdown: | 20260506 13:03:17.895 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:03:17.895 | INFO | ------------------------------------------ | 20260506 13:03:17.895 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:03:17.895 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:03:17.895 | INFO | Total mem usage: 15.7 GiB | 20260506 13:03:17.895 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:03:17.895 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:03:17.895 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:03:17.895 | INFO | High Water Breakdown: | 20260506 13:03:17.895 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:03:17.895 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:03:17.895 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:03:17.895 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:03:17.895 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:03:17.895 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:03:17.895 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:03:17.895 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:03:17.895 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:03:17.895 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:03:17.895 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:03:17.895 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:03:17.895 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:03:17.895 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:03:17.895 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:03:17.895 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:03:17.895 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:03:17.895 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:03:17.895 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:03:17.895 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:03:17.895 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:03:17.895 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:03:17.895 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:03:17.895 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:03:17.895 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:03:17.895 | INFO | ------------------------------------------ | 20260506 13:03:17.895 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:03:17.895 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:03:17.895 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:03:17.895 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:03:17.895 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.56e+01, 1.36e+01 / 2.06e+00 | 20260506 13:03:17.895 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.01e-01, 1.02e-02 / 6.20e-03 | 20260506 13:03:17.895 | INFO | PerfReport: 11.56%, Baseline -- 3.66e+00 / 4.32e+00, 5.73e-01 / 4.16e-01 | 20260506 13:03:17.895 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.47e+00, 1.89e-01 / 1.45e-01 | 20260506 13:03:17.895 | INFO | PerfReport: 15.71%, DME -- 3.98e+01 / 4.36e+01, 3.63e+00 / 2.30e+00 | 20260506 13:03:17.896 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.80e+01 / 2.11e+01, 1.53e+00 / 2.12e+00 | 20260506 13:03:17.896 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.38e+00 / 2.99e+00, 4.09e-01 / 2.71e-01 | 20260506 13:03:17.896 | INFO | PerfReport: 3.71%, Metrics -- 1.17e+00 / 1.57e+00, 2.99e-01 / 1.67e-01 | 20260506 13:03:17.896 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.13e-01 / 4.76e-01, 6.49e-02 / 3.38e-02 | 20260506 13:03:17.896 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:03:17.896 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:03:17.896 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:03:17.896 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 13:03:17.896 | INFO | PerfReport: 21.40%, Upload -- 2.11e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260506 13:03:17.896 | INFO | PerfReport: 20.96%, Download -- 2.07e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 13:03:17.896 | INFO | PerfReport: 32.16%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 13:03:17.896 | INFO | PerfReport: 1.69%, LockUpload -- 1.67e+00 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 13:03:17.896 | INFO | PerfReport: 1.08%, LockDownload -- 1.06e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260506 13:03:17.896 | INFO | PerfReport: 22.69%, LockCompute -- 2.24e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 13:03:17.896 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260506 13:03:18.025 | INFO | Starting a round of Baz IO | 20260506 13:03:21.183 | INFO | Populating RTMetrics report | 20260506 13:03:21.183 | INFO | RTMetrics report populated | 20260506 13:03:21.183 | INFO | Opening tmp RTMetrics json file | 20260506 13:03:21.184 | INFO | Serializing RTMetrics to json | 20260506 13:03:21.185 | INFO | Serialization done, swapping tmp file now... | 20260506 13:03:21.185 | INFO | RTMetrics done | 20260506 13:03:21.281 | INFO | Finished a round of Baz IO: Wrote 2.103 GiB in 3.255 seconds | 20260506 13:03:21.281 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.38% / 0.02% / 59.60% | 20260506 13:03:21.281 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.45226% / 5.96046e-05% / 3.97364e-05% | 20260506 13:03:21.281 | INFO | Baz IO overall: | 20260506 13:03:21.281 | INFO | BytesWritten : 788441573484 | 20260506 13:03:21.281 | INFO | NumEvents : 381237891490 | 20260506 13:03:23.034 | INFO | Analyzing chunk frames = [1564672,1565184) | 20260506 13:03:23.034 | INFO | MemInfo: { | 20260506 13:03:23.034 | INFO | "gpu" : " 17 GB", | 20260506 13:03:23.034 | INFO | "huge" : "", | 20260506 13:03:23.034 | INFO | "huge_shared" : " 27 GB", | 20260506 13:03:23.034 | INFO | "malloc" : " 14 GB", | 20260506 13:03:23.034 | INFO | "pinned" : " 29 GB" | 20260506 13:03:23.034 | INFO | } | 20260506 13:03:23.034 | INFO | Perf report for the last 5.13974 seconds: | 20260506 13:03:23.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:03:23.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:23.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:23.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:23.034 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260506 13:03:23.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:23.034 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 13:03:23.034 | INFO | PRE_HQ : 61.4%, 1.9, 61.2ms, 0 | 20260506 13:03:23.034 | INFO | BAZWRITER : 62.6%, 3.7, 119.0ms, 0 | 20260506 13:03:23.034 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 13:03:23.034 | INFO | Uploaded 36GiB to GPU in 2.05513s (17.5171GiB/s) | 20260506 13:03:23.034 | INFO | Downloaded 38GiB to GPU in 2.22383s (17.0877GiB/s) | 20260506 13:03:28.173 | INFO | Analyzing chunk frames = [1565184,1565696) | 20260506 13:03:28.173 | INFO | MemInfo: { | 20260506 13:03:28.173 | INFO | "gpu" : " 17 GB", | 20260506 13:03:28.173 | INFO | "huge" : "", | 20260506 13:03:28.173 | INFO | "huge_shared" : " 27 GB", | 20260506 13:03:28.173 | INFO | "malloc" : " 14 GB", | 20260506 13:03:28.173 | INFO | "pinned" : " 29 GB" | 20260506 13:03:28.173 | INFO | } | 20260506 13:03:28.173 | INFO | Perf report for the last 5.13906 seconds: | 20260506 13:03:28.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:03:28.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:28.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:28.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:28.173 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 13:03:28.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:28.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:28.173 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:03:28.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:28.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:28.173 | INFO | Uploaded 36GiB to GPU in 2.00954s (17.9146GiB/s) | 20260506 13:03:28.173 | INFO | Downloaded 34GiB to GPU in 1.93507s (17.5704GiB/s) | 20260506 13:03:33.312 | INFO | Analyzing chunk frames = [1565696,1566208) | 20260506 13:03:33.312 | INFO | MemInfo: { | 20260506 13:03:33.312 | INFO | "gpu" : " 17 GB", | 20260506 13:03:33.312 | INFO | "huge" : "", | 20260506 13:03:33.312 | INFO | "huge_shared" : " 27 GB", | 20260506 13:03:33.312 | INFO | "malloc" : " 14 GB", | 20260506 13:03:33.312 | INFO | "pinned" : " 29 GB" | 20260506 13:03:33.312 | INFO | } | 20260506 13:03:33.312 | INFO | Perf report for the last 5.13908 seconds: | 20260506 13:03:33.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:03:33.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:33.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:33.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:33.312 | INFO | ANALYSIS : 61.7%, 3.0, 95.5ms, 0 | 20260506 13:03:33.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:33.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:33.312 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:03:33.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:33.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:33.312 | INFO | Uploaded 36GiB to GPU in 2.01703s (17.848GiB/s) | 20260506 13:03:33.312 | INFO | Downloaded 34GiB to GPU in 1.94601s (17.4716GiB/s) | 20260506 13:03:38.451 | INFO | Analyzing chunk frames = [1566208,1566720) | 20260506 13:03:38.451 | INFO | MemInfo: { | 20260506 13:03:38.451 | INFO | "gpu" : " 17 GB", | 20260506 13:03:38.454 | INFO | "huge" : "", | 20260506 13:03:38.454 | INFO | "huge_shared" : " 27 GB", | 20260506 13:03:38.454 | INFO | "malloc" : " 14 GB", | 20260506 13:03:38.454 | INFO | "pinned" : " 29 GB" | 20260506 13:03:38.454 | INFO | } | 20260506 13:03:38.455 | INFO | Perf report for the last 5.14236 seconds: | 20260506 13:03:38.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:03:38.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:38.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:38.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:38.455 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:03:38.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:38.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:38.455 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:03:38.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:38.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:38.455 | INFO | Uploaded 36GiB to GPU in 2.01277s (17.8858GiB/s) | 20260506 13:03:38.455 | INFO | Downloaded 34GiB to GPU in 1.9301s (17.6157GiB/s) | 20260506 13:03:43.592 | INFO | Analyzing chunk frames = [1566720,1567232) | 20260506 13:03:43.592 | INFO | MemInfo: { | 20260506 13:03:43.592 | INFO | "gpu" : " 17 GB", | 20260506 13:03:43.592 | INFO | "huge" : "", | 20260506 13:03:43.592 | INFO | "huge_shared" : " 27 GB", | 20260506 13:03:43.592 | INFO | "malloc" : " 14 GB", | 20260506 13:03:43.592 | INFO | "pinned" : " 29 GB" | 20260506 13:03:43.592 | INFO | } | 20260506 13:03:43.592 | INFO | Perf report for the last 5.1375 seconds: | 20260506 13:03:43.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:03:43.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:43.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:43.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:43.592 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 13:03:43.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:43.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:43.592 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:03:43.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:43.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:43.592 | INFO | Uploaded 36GiB to GPU in 1.98521s (18.1341GiB/s) | 20260506 13:03:43.592 | INFO | Downloaded 34GiB to GPU in 1.95159s (17.4217GiB/s) | 20260506 13:03:48.730 | INFO | Analyzing chunk frames = [1567232,1567744) | 20260506 13:03:48.730 | INFO | MemInfo: { | 20260506 13:03:48.730 | INFO | "gpu" : " 17 GB", | 20260506 13:03:48.730 | INFO | "huge" : "", | 20260506 13:03:48.730 | INFO | "huge_shared" : " 27 GB", | 20260506 13:03:48.730 | INFO | "malloc" : " 14 GB", | 20260506 13:03:48.730 | INFO | "pinned" : " 29 GB" | 20260506 13:03:48.730 | INFO | } | 20260506 13:03:48.730 | INFO | Perf report for the last 5.13818 seconds: | 20260506 13:03:48.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:03:48.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:48.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:48.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:48.730 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 13:03:48.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:48.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:48.730 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260506 13:03:48.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:48.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:48.730 | INFO | Uploaded 36GiB to GPU in 2.05421s (17.525GiB/s) | 20260506 13:03:48.730 | INFO | Downloaded 34GiB to GPU in 1.9728s (17.2344GiB/s) | 20260506 13:03:53.871 | INFO | Analyzing chunk frames = [1567744,1568256) | 20260506 13:03:53.871 | INFO | MemInfo: { | 20260506 13:03:53.871 | INFO | "gpu" : " 17 GB", | 20260506 13:03:53.871 | INFO | "huge" : "", | 20260506 13:03:53.871 | INFO | "huge_shared" : " 27 GB", | 20260506 13:03:53.871 | INFO | "malloc" : " 14 GB", | 20260506 13:03:53.871 | INFO | "pinned" : " 29 GB" | 20260506 13:03:53.871 | INFO | } | 20260506 13:03:53.871 | INFO | Perf report for the last 5.14096 seconds: | 20260506 13:03:53.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:03:53.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:53.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:53.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:53.871 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:03:53.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:53.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:53.871 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 13:03:53.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:53.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:53.871 | INFO | Uploaded 36GiB to GPU in 1.98535s (18.1328GiB/s) | 20260506 13:03:53.871 | INFO | Downloaded 34GiB to GPU in 2.03393s (16.7164GiB/s) | 20260506 13:03:59.012 | INFO | Analyzing chunk frames = [1568256,1568768) | 20260506 13:03:59.012 | INFO | MemInfo: { | 20260506 13:03:59.012 | INFO | "gpu" : " 17 GB", | 20260506 13:03:59.012 | INFO | "huge" : "", | 20260506 13:03:59.012 | INFO | "huge_shared" : " 27 GB", | 20260506 13:03:59.012 | INFO | "malloc" : " 14 GB", | 20260506 13:03:59.012 | INFO | "pinned" : " 29 GB" | 20260506 13:03:59.012 | INFO | } | 20260506 13:03:59.012 | INFO | Perf report for the last 5.14104 seconds: | 20260506 13:03:59.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:03:59.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:59.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:59.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:59.012 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 13:03:59.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:59.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:59.012 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:03:59.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:59.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:03:59.012 | INFO | Uploaded 36GiB to GPU in 2.04436s (17.6094GiB/s) | 20260506 13:03:59.012 | INFO | Downloaded 34GiB to GPU in 1.96631s (17.2913GiB/s) | 20260506 13:03:59.146 | INFO | Starting a round of Baz IO | 20260506 13:04:02.306 | INFO | Populating RTMetrics report | 20260506 13:04:02.307 | INFO | RTMetrics report populated | 20260506 13:04:02.307 | INFO | Opening tmp RTMetrics json file | 20260506 13:04:02.307 | INFO | Serializing RTMetrics to json | 20260506 13:04:02.308 | INFO | Serialization done, swapping tmp file now... | 20260506 13:04:02.309 | INFO | RTMetrics done | 20260506 13:04:02.406 | INFO | Finished a round of Baz IO: Wrote 2.105 GiB in 3.26 seconds | 20260506 13:04:02.406 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.36% / 0.02% / 59.62% | 20260506 13:04:02.406 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.44665% / 2.78155e-05% / 5.5631e-05% | 20260506 13:04:02.406 | INFO | Baz IO overall: | 20260506 13:04:02.406 | INFO | BytesWritten : 790701419918 | 20260506 13:04:02.406 | INFO | NumEvents : 382334259987 | 20260506 13:04:04.148 | INFO | Analyzing chunk frames = [1568768,1569280) | 20260506 13:04:04.148 | INFO | MemInfo: { | 20260506 13:04:04.148 | INFO | "gpu" : " 17 GB", | 20260506 13:04:04.148 | INFO | "huge" : "", | 20260506 13:04:04.148 | INFO | "huge_shared" : " 27 GB", | 20260506 13:04:04.148 | INFO | "malloc" : " 14 GB", | 20260506 13:04:04.148 | INFO | "pinned" : " 29 GB" | 20260506 13:04:04.148 | INFO | } | 20260506 13:04:04.148 | INFO | Perf report for the last 5.13599 seconds: | 20260506 13:04:04.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:04:04.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:04.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:04.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:04.148 | INFO | ANALYSIS : 63.6%, 3.0, 98.6ms, 0 | 20260506 13:04:04.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:04.148 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 13:04:04.148 | INFO | PRE_HQ : 61.4%, 1.9, 61.3ms, 0 | 20260506 13:04:04.148 | INFO | BAZWRITER : 62.8%, 3.7, 120.6ms, 0 | 20260506 13:04:04.148 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:04:04.148 | INFO | Uploaded 36GiB to GPU in 2.05414s (17.5255GiB/s) | 20260506 13:04:04.148 | INFO | Downloaded 38GiB to GPU in 2.21232s (17.1765GiB/s) | 20260506 13:04:09.287 | INFO | Analyzing chunk frames = [1569280,1569792) | 20260506 13:04:09.287 | INFO | MemInfo: { | 20260506 13:04:09.287 | INFO | "gpu" : " 17 GB", | 20260506 13:04:09.287 | INFO | "huge" : "", | 20260506 13:04:09.287 | INFO | "huge_shared" : " 27 GB", | 20260506 13:04:09.287 | INFO | "malloc" : " 14 GB", | 20260506 13:04:09.287 | INFO | "pinned" : " 29 GB" | 20260506 13:04:09.287 | INFO | } | 20260506 13:04:09.288 | INFO | Perf report for the last 5.13932 seconds: | 20260506 13:04:09.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:04:09.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:09.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:09.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:09.288 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:04:09.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:09.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:09.288 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:04:09.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:09.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:09.288 | INFO | Uploaded 36GiB to GPU in 2.04031s (17.6444GiB/s) | 20260506 13:04:09.288 | INFO | Downloaded 34GiB to GPU in 1.93148s (17.6031GiB/s) | 20260506 13:04:14.427 | INFO | Analyzing chunk frames = [1569792,1570304) | 20260506 13:04:14.427 | INFO | MemInfo: { | 20260506 13:04:14.427 | INFO | "gpu" : " 17 GB", | 20260506 13:04:14.427 | INFO | "huge" : "", | 20260506 13:04:14.427 | INFO | "huge_shared" : " 27 GB", | 20260506 13:04:14.427 | INFO | "malloc" : " 14 GB", | 20260506 13:04:14.427 | INFO | "pinned" : " 29 GB" | 20260506 13:04:14.427 | INFO | } | 20260506 13:04:14.427 | INFO | Perf report for the last 5.1398 seconds: | 20260506 13:04:14.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:04:14.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:14.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:14.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:14.428 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 13:04:14.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:14.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:14.428 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:04:14.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:14.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:14.428 | INFO | Uploaded 36GiB to GPU in 2.03512s (17.6893GiB/s) | 20260506 13:04:14.428 | INFO | Downloaded 34GiB to GPU in 1.93664s (17.5562GiB/s) | 20260506 13:04:19.567 | INFO | Analyzing chunk frames = [1570304,1570816) | 20260506 13:04:19.567 | INFO | MemInfo: { | 20260506 13:04:19.567 | INFO | "gpu" : " 17 GB", | 20260506 13:04:19.567 | INFO | "huge" : "", | 20260506 13:04:19.567 | INFO | "huge_shared" : " 27 GB", | 20260506 13:04:19.567 | INFO | "malloc" : " 14 GB", | 20260506 13:04:19.567 | INFO | "pinned" : " 29 GB" | 20260506 13:04:19.567 | INFO | } | 20260506 13:04:19.567 | INFO | Perf report for the last 5.13997 seconds: | 20260506 13:04:19.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:04:19.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:19.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:19.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:19.567 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 13:04:19.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:19.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:19.567 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:04:19.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:19.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:19.568 | INFO | Uploaded 36GiB to GPU in 2.02545s (17.7738GiB/s) | 20260506 13:04:19.568 | INFO | Downloaded 34GiB to GPU in 1.94094s (17.5173GiB/s) | 20260506 13:04:24.709 | INFO | Analyzing chunk frames = [1570816,1571328) | 20260506 13:04:24.709 | INFO | MemInfo: { | 20260506 13:04:24.709 | INFO | "gpu" : " 17 GB", | 20260506 13:04:24.709 | INFO | "huge" : "", | 20260506 13:04:24.709 | INFO | "huge_shared" : " 27 GB", | 20260506 13:04:24.709 | INFO | "malloc" : " 14 GB", | 20260506 13:04:24.709 | INFO | "pinned" : " 29 GB" | 20260506 13:04:24.709 | INFO | } | 20260506 13:04:24.709 | INFO | Perf report for the last 5.14135 seconds: | 20260506 13:04:24.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:04:24.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:24.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:24.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:24.709 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 13:04:24.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:24.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:24.709 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 13:04:24.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:24.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:24.709 | INFO | Uploaded 36GiB to GPU in 2.01429s (17.8723GiB/s) | 20260506 13:04:24.709 | INFO | Downloaded 34GiB to GPU in 1.94819s (17.4521GiB/s) | 20260506 13:04:29.845 | INFO | Analyzing chunk frames = [1571328,1571840) | 20260506 13:04:29.845 | INFO | MemInfo: { | 20260506 13:04:29.845 | INFO | "gpu" : " 17 GB", | 20260506 13:04:29.845 | INFO | "huge" : "", | 20260506 13:04:29.845 | INFO | "huge_shared" : " 27 GB", | 20260506 13:04:29.845 | INFO | "malloc" : " 14 GB", | 20260506 13:04:29.845 | INFO | "pinned" : " 29 GB" | 20260506 13:04:29.845 | INFO | } | 20260506 13:04:29.845 | INFO | Perf report for the last 5.13649 seconds: | 20260506 13:04:29.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:04:29.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:29.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:29.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:29.845 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:04:29.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:29.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:29.845 | INFO | PRE_HQ : 33.2%, 1.0, 17.2ms, 0 | 20260506 13:04:29.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:29.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:29.845 | INFO | Uploaded 36GiB to GPU in 2.09202s (17.2083GiB/s) | 20260506 13:04:29.845 | INFO | Downloaded 34GiB to GPU in 1.97478s (17.2171GiB/s) | 20260506 13:04:34.984 | INFO | Analyzing chunk frames = [1571840,1572352) | 20260506 13:04:34.984 | INFO | MemInfo: { | 20260506 13:04:34.984 | INFO | "gpu" : " 17 GB", | 20260506 13:04:34.984 | INFO | "huge" : "", | 20260506 13:04:34.984 | INFO | "huge_shared" : " 27 GB", | 20260506 13:04:34.984 | INFO | "malloc" : " 14 GB", | 20260506 13:04:34.984 | INFO | "pinned" : " 29 GB" | 20260506 13:04:34.984 | INFO | } | 20260506 13:04:34.984 | INFO | Perf report for the last 5.13913 seconds: | 20260506 13:04:34.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:04:34.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:34.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:34.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:34.984 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 13:04:34.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:34.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:34.984 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:04:34.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:34.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:34.985 | INFO | Uploaded 36GiB to GPU in 1.98525s (18.1338GiB/s) | 20260506 13:04:34.985 | INFO | Downloaded 34GiB to GPU in 2.02886s (16.7582GiB/s) | 20260506 13:04:40.123 | INFO | Analyzing chunk frames = [1572352,1572864) | 20260506 13:04:40.123 | INFO | MemInfo: { | 20260506 13:04:40.123 | INFO | "gpu" : " 17 GB", | 20260506 13:04:40.123 | INFO | "huge" : "", | 20260506 13:04:40.124 | INFO | "huge_shared" : " 27 GB", | 20260506 13:04:40.124 | INFO | "malloc" : " 14 GB", | 20260506 13:04:40.124 | INFO | "pinned" : " 29 GB" | 20260506 13:04:40.124 | INFO | } | 20260506 13:04:40.124 | INFO | Perf report for the last 5.13925 seconds: | 20260506 13:04:40.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:04:40.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:40.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:40.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:40.124 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 13:04:40.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:40.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:40.124 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:04:40.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:40.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:40.124 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:04:40.124 | INFO | Total mem usage: 16.8 GiB | 20260506 13:04:40.124 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 13:04:40.124 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.5%) | 20260506 13:04:40.124 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:04:40.124 | INFO | High Water Breakdown: | 20260506 13:04:40.124 | INFO | 59.8% : 8.62 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:04:40.124 | INFO | 40.1% : 5.78 GiB ( 86.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:04:40.124 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:04:40.124 | INFO | Uploaded 36GiB to GPU in 2.0509s (17.5533GiB/s) | 20260506 13:04:40.124 | INFO | ------------------------------------------ | 20260506 13:04:40.124 | INFO | Downloaded 34GiB to GPU in 1.97289s (17.2336GiB/s) | 20260506 13:04:40.124 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:04:40.124 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:04:40.124 | INFO | Total mem usage: 26.5 GiB | 20260506 13:04:40.124 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:04:40.124 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:04:40.124 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:04:40.124 | INFO | High Water Breakdown: | 20260506 13:04:40.124 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:04:40.124 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:04:40.124 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:04:40.124 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:04:40.124 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:04:40.124 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:04:40.124 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:04:40.124 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:04:40.124 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:04:40.124 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:04:40.124 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:04:40.124 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:04:40.124 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:04:40.124 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:04:40.124 | INFO | ------------------------------------------ | 20260506 13:04:40.124 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:04:40.124 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:04:40.124 | INFO | Total mem usage: 25.6 GiB | 20260506 13:04:40.124 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:04:40.124 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:04:40.124 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:04:40.124 | INFO | High Water Breakdown: | 20260506 13:04:40.124 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:04:40.124 | INFO | ------------------------------------------ | 20260506 13:04:40.124 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:04:40.124 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:04:40.124 | INFO | Total mem usage: 15.7 GiB | 20260506 13:04:40.124 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:04:40.124 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:04:40.124 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:04:40.124 | INFO | High Water Breakdown: | 20260506 13:04:40.124 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:04:40.124 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:04:40.124 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:04:40.124 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:04:40.124 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:04:40.124 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:04:40.124 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:04:40.124 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:04:40.124 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:04:40.124 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:04:40.124 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:04:40.124 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:04:40.124 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:04:40.125 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:04:40.125 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:04:40.125 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:04:40.125 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:04:40.125 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:04:40.125 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:04:40.125 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:04:40.125 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:04:40.125 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:04:40.125 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:04:40.125 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:04:40.125 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:04:40.125 | INFO | ------------------------------------------ | 20260506 13:04:40.125 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:04:40.125 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:04:40.125 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:04:40.125 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:04:40.125 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.63e+01, 1.35e+01 / 2.97e+00 | 20260506 13:04:40.125 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.02e-01, 1.03e-02 / 5.59e-03 | 20260506 13:04:40.125 | INFO | PerfReport: 11.59%, Baseline -- 3.67e+00 / 4.47e+00, 5.77e-01 / 2.82e-01 | 20260506 13:04:40.125 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.49e+00, 1.91e-01 / 1.29e-01 | 20260506 13:04:40.125 | INFO | PerfReport: 15.89%, DME -- 4.02e+01 / 4.50e+01, 4.42e+00 / 3.25e+00 | 20260506 13:04:40.125 | INFO | PerfReport: 56.39%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.33e+00 / 1.73e+00 | 20260506 13:04:40.125 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.40e+00 / 3.04e+00, 4.24e-01 / 1.98e-01 | 20260506 13:04:40.125 | INFO | PerfReport: 3.71%, Metrics -- 1.17e+00 / 1.57e+00, 3.01e-01 / 1.52e-01 | 20260506 13:04:40.125 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.13e-01 / 4.71e-01, 6.34e-02 / 2.89e-02 | 20260506 13:04:40.125 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:04:40.125 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:04:40.125 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:04:40.125 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 13:04:40.125 | INFO | PerfReport: 21.44%, Upload -- 2.11e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260506 13:04:40.125 | INFO | PerfReport: 21.07%, Download -- 2.08e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 13:04:40.125 | INFO | PerfReport: 32.10%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 13:04:40.125 | INFO | PerfReport: 1.71%, LockUpload -- 1.69e+00 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 13:04:40.125 | INFO | PerfReport: 1.10%, LockDownload -- 1.08e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260506 13:04:40.125 | INFO | PerfReport: 22.57%, LockCompute -- 2.22e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 13:04:40.125 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260506 13:04:40.251 | INFO | Starting a round of Baz IO | 20260506 13:04:43.411 | INFO | Populating RTMetrics report | 20260506 13:04:43.411 | INFO | RTMetrics report populated | 20260506 13:04:43.411 | INFO | Opening tmp RTMetrics json file | 20260506 13:04:43.412 | INFO | Serializing RTMetrics to json | 20260506 13:04:43.413 | INFO | Serialization done, swapping tmp file now... | 20260506 13:04:43.414 | INFO | RTMetrics done | 20260506 13:04:43.508 | INFO | Finished a round of Baz IO: Wrote 2.105 GiB in 3.257 seconds | 20260506 13:04:43.508 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.34% / 0.02% / 59.64% | 20260506 13:04:43.508 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.43859% / 3.57628e-05% / 6.35783e-05% | 20260506 13:04:43.508 | INFO | Baz IO overall: | 20260506 13:04:43.508 | INFO | BytesWritten : 792961861765 | 20260506 13:04:43.509 | INFO | NumEvents : 383430732541 | 20260506 13:04:45.263 | INFO | Analyzing chunk frames = [1572864,1573376) | 20260506 13:04:45.263 | INFO | MemInfo: { | 20260506 13:04:45.263 | INFO | "gpu" : " 17 GB", | 20260506 13:04:45.263 | INFO | "huge" : "", | 20260506 13:04:45.263 | INFO | "huge_shared" : " 27 GB", | 20260506 13:04:45.263 | INFO | "malloc" : " 14 GB", | 20260506 13:04:45.263 | INFO | "pinned" : " 29 GB" | 20260506 13:04:45.263 | INFO | } | 20260506 13:04:45.263 | INFO | Perf report for the last 5.13941 seconds: | 20260506 13:04:45.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:04:45.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:45.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:45.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:45.263 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260506 13:04:45.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:45.263 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260506 13:04:45.263 | INFO | PRE_HQ : 61.4%, 1.9, 60.9ms, 0 | 20260506 13:04:45.263 | INFO | BAZWRITER : 62.7%, 3.7, 119.2ms, 0 | 20260506 13:04:45.263 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 13:04:45.263 | INFO | Uploaded 36GiB to GPU in 2.06534s (17.4305GiB/s) | 20260506 13:04:45.263 | INFO | Downloaded 38GiB to GPU in 2.21014s (17.1935GiB/s) | 20260506 13:04:50.402 | INFO | Analyzing chunk frames = [1573376,1573888) | 20260506 13:04:50.402 | INFO | MemInfo: { | 20260506 13:04:50.402 | INFO | "gpu" : " 17 GB", | 20260506 13:04:50.402 | INFO | "huge" : "", | 20260506 13:04:50.402 | INFO | "huge_shared" : " 27 GB", | 20260506 13:04:50.402 | INFO | "malloc" : " 14 GB", | 20260506 13:04:50.402 | INFO | "pinned" : " 29 GB" | 20260506 13:04:50.402 | INFO | } | 20260506 13:04:50.402 | INFO | Perf report for the last 5.13936 seconds: | 20260506 13:04:50.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:04:50.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:50.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:50.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:50.402 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:04:50.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:50.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:50.402 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:04:50.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:50.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:50.403 | INFO | Uploaded 36GiB to GPU in 1.99005s (18.09GiB/s) | 20260506 13:04:50.403 | INFO | Downloaded 34GiB to GPU in 1.92596s (17.6535GiB/s) | 20260506 13:04:55.542 | INFO | Analyzing chunk frames = [1573888,1574400) | 20260506 13:04:55.542 | INFO | MemInfo: { | 20260506 13:04:55.542 | INFO | "gpu" : " 17 GB", | 20260506 13:04:55.542 | INFO | "huge" : "", | 20260506 13:04:55.542 | INFO | "huge_shared" : " 27 GB", | 20260506 13:04:55.542 | INFO | "malloc" : " 14 GB", | 20260506 13:04:55.542 | INFO | "pinned" : " 29 GB" | 20260506 13:04:55.542 | INFO | } | 20260506 13:04:55.542 | INFO | Perf report for the last 5.13934 seconds: | 20260506 13:04:55.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:04:55.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:55.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:55.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:55.542 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 13:04:55.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:55.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:55.542 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:04:55.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:55.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:04:55.542 | INFO | Uploaded 36GiB to GPU in 1.96064s (18.3614GiB/s) | 20260506 13:04:55.542 | INFO | Downloaded 34GiB to GPU in 1.92522s (17.6603GiB/s) | 20260506 13:05:00.681 | INFO | Analyzing chunk frames = [1574400,1574912) | 20260506 13:05:00.681 | INFO | MemInfo: { | 20260506 13:05:00.681 | INFO | "gpu" : " 17 GB", | 20260506 13:05:00.681 | INFO | "huge" : "", | 20260506 13:05:00.681 | INFO | "huge_shared" : " 27 GB", | 20260506 13:05:00.681 | INFO | "malloc" : " 14 GB", | 20260506 13:05:00.681 | INFO | "pinned" : " 29 GB" | 20260506 13:05:00.681 | INFO | } | 20260506 13:05:00.681 | INFO | Perf report for the last 5.13924 seconds: | 20260506 13:05:00.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:05:00.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:00.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:00.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:00.681 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:05:00.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:00.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:00.681 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:05:00.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:00.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:00.681 | INFO | Uploaded 36GiB to GPU in 2.02131s (17.8102GiB/s) | 20260506 13:05:00.681 | INFO | Downloaded 34GiB to GPU in 1.91937s (17.7141GiB/s) | 20260506 13:05:05.820 | INFO | Analyzing chunk frames = [1574912,1575424) | 20260506 13:05:05.820 | INFO | MemInfo: { | 20260506 13:05:05.820 | INFO | "gpu" : " 17 GB", | 20260506 13:05:05.820 | INFO | "huge" : "", | 20260506 13:05:05.820 | INFO | "huge_shared" : " 27 GB", | 20260506 13:05:05.820 | INFO | "malloc" : " 14 GB", | 20260506 13:05:05.820 | INFO | "pinned" : " 29 GB" | 20260506 13:05:05.820 | INFO | } | 20260506 13:05:05.820 | INFO | Perf report for the last 5.1394 seconds: | 20260506 13:05:05.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:05:05.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:05.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:05.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:05.821 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 13:05:05.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:05.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:05.821 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:05:05.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:05.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:05.821 | INFO | Uploaded 36GiB to GPU in 2.03226s (17.7143GiB/s) | 20260506 13:05:05.821 | INFO | Downloaded 34GiB to GPU in 1.95198s (17.4182GiB/s) | 20260506 13:05:10.960 | INFO | Analyzing chunk frames = [1575424,1575936) | 20260506 13:05:10.960 | INFO | MemInfo: { | 20260506 13:05:10.960 | INFO | "gpu" : " 17 GB", | 20260506 13:05:10.960 | INFO | "huge" : "", | 20260506 13:05:10.960 | INFO | "huge_shared" : " 27 GB", | 20260506 13:05:10.960 | INFO | "malloc" : " 14 GB", | 20260506 13:05:10.960 | INFO | "pinned" : " 29 GB" | 20260506 13:05:10.960 | INFO | } | 20260506 13:05:10.960 | INFO | Perf report for the last 5.13949 seconds: | 20260506 13:05:10.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:05:10.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:10.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:10.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:10.960 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 13:05:10.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:10.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:10.960 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 13:05:10.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:10.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:10.960 | INFO | Uploaded 36GiB to GPU in 2.04023s (17.6451GiB/s) | 20260506 13:05:10.960 | INFO | Downloaded 34GiB to GPU in 1.93391s (17.5809GiB/s) | 20260506 13:05:16.100 | INFO | Analyzing chunk frames = [1575936,1576448) | 20260506 13:05:16.100 | INFO | MemInfo: { | 20260506 13:05:16.100 | INFO | "gpu" : " 17 GB", | 20260506 13:05:16.100 | INFO | "huge" : "", | 20260506 13:05:16.100 | INFO | "huge_shared" : " 27 GB", | 20260506 13:05:16.100 | INFO | "malloc" : " 14 GB", | 20260506 13:05:16.100 | INFO | "pinned" : " 29 GB" | 20260506 13:05:16.100 | INFO | } | 20260506 13:05:16.100 | INFO | Perf report for the last 5.14031 seconds: | 20260506 13:05:16.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:05:16.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:16.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:16.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:16.100 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:05:16.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:16.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:16.100 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 13:05:16.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:16.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:16.100 | INFO | Uploaded 36GiB to GPU in 2.02613s (17.7678GiB/s) | 20260506 13:05:16.100 | INFO | Downloaded 34GiB to GPU in 2.00692s (16.9414GiB/s) | 20260506 13:05:21.239 | INFO | Analyzing chunk frames = [1576448,1576960) | 20260506 13:05:21.239 | INFO | MemInfo: { | 20260506 13:05:21.239 | INFO | "gpu" : " 17 GB", | 20260506 13:05:21.239 | INFO | "huge" : "", | 20260506 13:05:21.239 | INFO | "huge_shared" : " 27 GB", | 20260506 13:05:21.239 | INFO | "malloc" : " 14 GB", | 20260506 13:05:21.239 | INFO | "pinned" : " 29 GB" | 20260506 13:05:21.239 | INFO | } | 20260506 13:05:21.239 | INFO | Perf report for the last 5.13877 seconds: | 20260506 13:05:21.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:05:21.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:21.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:21.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:21.239 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 13:05:21.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:21.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:21.239 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:05:21.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:21.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:21.239 | INFO | Uploaded 36GiB to GPU in 2.05249s (17.5396GiB/s) | 20260506 13:05:21.239 | INFO | Downloaded 34GiB to GPU in 1.97187s (17.2425GiB/s) | 20260506 13:05:21.368 | INFO | Starting a round of Baz IO | 20260506 13:05:24.520 | INFO | Populating RTMetrics report | 20260506 13:05:24.521 | INFO | RTMetrics report populated | 20260506 13:05:24.521 | INFO | Opening tmp RTMetrics json file | 20260506 13:05:24.521 | INFO | Serializing RTMetrics to json | 20260506 13:05:24.522 | INFO | Serialization done, swapping tmp file now... | 20260506 13:05:24.523 | INFO | RTMetrics done | 20260506 13:05:24.619 | INFO | Finished a round of Baz IO: Wrote 2.103 GiB in 3.25 seconds | 20260506 13:05:24.619 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.32% / 0.02% / 59.66% | 20260506 13:05:24.619 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.42278% / 3.57628e-05% / 7.15256e-05% | 20260506 13:05:24.619 | INFO | Baz IO overall: | 20260506 13:05:24.619 | INFO | BytesWritten : 795220194184 | 20260506 13:05:24.619 | INFO | NumEvents : 384525660520 | 20260506 13:05:26.378 | INFO | Analyzing chunk frames = [1576960,1577472) | 20260506 13:05:26.378 | INFO | MemInfo: { | 20260506 13:05:26.378 | INFO | "gpu" : " 17 GB", | 20260506 13:05:26.378 | INFO | "huge" : "", | 20260506 13:05:26.378 | INFO | "huge_shared" : " 27 GB", | 20260506 13:05:26.378 | INFO | "malloc" : " 14 GB", | 20260506 13:05:26.378 | INFO | "pinned" : " 29 GB" | 20260506 13:05:26.378 | INFO | } | 20260506 13:05:26.378 | INFO | Perf report for the last 5.13875 seconds: | 20260506 13:05:26.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:05:26.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:26.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:26.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:26.378 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260506 13:05:26.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:26.378 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:05:26.378 | INFO | PRE_HQ : 61.3%, 1.9, 60.9ms, 0 | 20260506 13:05:26.378 | INFO | BAZWRITER : 62.5%, 3.7, 118.7ms, 0 | 20260506 13:05:26.378 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 13:05:26.378 | INFO | Uploaded 36GiB to GPU in 2.05566s (17.5126GiB/s) | 20260506 13:05:26.378 | INFO | Downloaded 38GiB to GPU in 2.21502s (17.1556GiB/s) | 20260506 13:05:31.517 | INFO | Analyzing chunk frames = [1577472,1577984) | 20260506 13:05:31.517 | INFO | MemInfo: { | 20260506 13:05:31.517 | INFO | "gpu" : " 17 GB", | 20260506 13:05:31.517 | INFO | "huge" : "", | 20260506 13:05:31.517 | INFO | "huge_shared" : " 27 GB", | 20260506 13:05:31.517 | INFO | "malloc" : " 14 GB", | 20260506 13:05:31.517 | INFO | "pinned" : " 29 GB" | 20260506 13:05:31.517 | INFO | } | 20260506 13:05:31.517 | INFO | Perf report for the last 5.13934 seconds: | 20260506 13:05:31.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:05:31.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:31.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:31.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:31.517 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 13:05:31.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:31.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:31.517 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 13:05:31.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:31.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:31.517 | INFO | Uploaded 36GiB to GPU in 2.05654s (17.5051GiB/s) | 20260506 13:05:31.518 | INFO | Downloaded 34GiB to GPU in 1.94522s (17.4788GiB/s) | 20260506 13:05:36.656 | INFO | Analyzing chunk frames = [1577984,1578496) | 20260506 13:05:36.656 | INFO | MemInfo: { | 20260506 13:05:36.656 | INFO | "gpu" : " 17 GB", | 20260506 13:05:36.656 | INFO | "huge" : "", | 20260506 13:05:36.656 | INFO | "huge_shared" : " 27 GB", | 20260506 13:05:36.656 | INFO | "malloc" : " 14 GB", | 20260506 13:05:36.656 | INFO | "pinned" : " 29 GB" | 20260506 13:05:36.656 | INFO | } | 20260506 13:05:36.657 | INFO | Perf report for the last 5.13951 seconds: | 20260506 13:05:36.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:05:36.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:36.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:36.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:36.657 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260506 13:05:36.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:36.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:36.657 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:05:36.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:36.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:36.657 | INFO | Uploaded 36GiB to GPU in 2.00509s (17.9543GiB/s) | 20260506 13:05:36.657 | INFO | Downloaded 34GiB to GPU in 1.94303s (17.4985GiB/s) | 20260506 13:05:41.796 | INFO | Analyzing chunk frames = [1578496,1579008) | 20260506 13:05:41.796 | INFO | MemInfo: { | 20260506 13:05:41.796 | INFO | "gpu" : " 17 GB", | 20260506 13:05:41.796 | INFO | "huge" : "", | 20260506 13:05:41.796 | INFO | "huge_shared" : " 27 GB", | 20260506 13:05:41.796 | INFO | "malloc" : " 14 GB", | 20260506 13:05:41.796 | INFO | "pinned" : " 29 GB" | 20260506 13:05:41.796 | INFO | } | 20260506 13:05:41.796 | INFO | Perf report for the last 5.13923 seconds: | 20260506 13:05:41.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:05:41.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:41.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:41.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:41.796 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:05:41.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:41.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:41.796 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:05:41.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:41.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:41.796 | INFO | Uploaded 36GiB to GPU in 1.98863s (18.1029GiB/s) | 20260506 13:05:41.796 | INFO | Downloaded 34GiB to GPU in 1.92726s (17.6416GiB/s) | 20260506 13:05:46.935 | INFO | Analyzing chunk frames = [1579008,1579520) | 20260506 13:05:46.935 | INFO | MemInfo: { | 20260506 13:05:46.935 | INFO | "gpu" : " 17 GB", | 20260506 13:05:46.935 | INFO | "huge" : "", | 20260506 13:05:46.936 | INFO | "huge_shared" : " 27 GB", | 20260506 13:05:46.936 | INFO | "malloc" : " 14 GB", | 20260506 13:05:46.936 | INFO | "pinned" : " 29 GB" | 20260506 13:05:46.936 | INFO | } | 20260506 13:05:46.936 | INFO | Perf report for the last 5.13974 seconds: | 20260506 13:05:46.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:05:46.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:46.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:46.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:46.936 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 13:05:46.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:46.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:46.936 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:05:46.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:46.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:46.936 | INFO | Uploaded 36GiB to GPU in 2.01663s (17.8515GiB/s) | 20260506 13:05:46.936 | INFO | Downloaded 34GiB to GPU in 1.9417s (17.5104GiB/s) | 20260506 13:05:52.074 | INFO | Analyzing chunk frames = [1579520,1580032) | 20260506 13:05:52.074 | INFO | MemInfo: { | 20260506 13:05:52.074 | INFO | "gpu" : " 17 GB", | 20260506 13:05:52.074 | INFO | "huge" : "", | 20260506 13:05:52.074 | INFO | "huge_shared" : " 27 GB", | 20260506 13:05:52.074 | INFO | "malloc" : " 14 GB", | 20260506 13:05:52.074 | INFO | "pinned" : " 29 GB" | 20260506 13:05:52.074 | INFO | } | 20260506 13:05:52.075 | INFO | Perf report for the last 5.13883 seconds: | 20260506 13:05:52.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:05:52.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:52.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:52.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:52.075 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 13:05:52.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:52.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:52.075 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260506 13:05:52.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:52.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:52.075 | INFO | Uploaded 36GiB to GPU in 2.09317s (17.1988GiB/s) | 20260506 13:05:52.075 | INFO | Downloaded 34GiB to GPU in 1.97622s (17.2046GiB/s) | 20260506 13:05:57.215 | INFO | Analyzing chunk frames = [1580032,1580544) | 20260506 13:05:57.215 | INFO | MemInfo: { | 20260506 13:05:57.215 | INFO | "gpu" : " 17 GB", | 20260506 13:05:57.215 | INFO | "huge" : "", | 20260506 13:05:57.215 | INFO | "huge_shared" : " 27 GB", | 20260506 13:05:57.215 | INFO | "malloc" : " 14 GB", | 20260506 13:05:57.215 | INFO | "pinned" : " 29 GB" | 20260506 13:05:57.215 | INFO | } | 20260506 13:05:57.215 | INFO | Perf report for the last 5.14015 seconds: | 20260506 13:05:57.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:05:57.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:57.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:57.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:57.215 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:05:57.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:57.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:57.215 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:05:57.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:57.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:05:57.215 | INFO | Uploaded 36GiB to GPU in 1.99501s (18.045GiB/s) | 20260506 13:05:57.215 | INFO | Downloaded 34GiB to GPU in 2.03662s (16.6943GiB/s) | 20260506 13:06:02.353 | INFO | Analyzing chunk frames = [1580544,1581056) | 20260506 13:06:02.353 | INFO | MemInfo: { | 20260506 13:06:02.353 | INFO | "gpu" : " 17 GB", | 20260506 13:06:02.353 | INFO | "huge" : "", | 20260506 13:06:02.353 | INFO | "huge_shared" : " 27 GB", | 20260506 13:06:02.353 | INFO | "malloc" : " 14 GB", | 20260506 13:06:02.353 | INFO | "pinned" : " 29 GB" | 20260506 13:06:02.353 | INFO | } | 20260506 13:06:02.353 | INFO | Perf report for the last 5.13861 seconds: | 20260506 13:06:02.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:06:02.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:02.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:02.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:02.353 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 13:06:02.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:02.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:02.353 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:06:02.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:02.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:02.354 | INFO | Uploaded 36GiB to GPU in 2.02744s (17.7564GiB/s) | 20260506 13:06:02.354 | INFO | Downloaded 34GiB to GPU in 2.02398s (16.7986GiB/s) | 20260506 13:06:02.354 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:06:02.354 | INFO | Total mem usage: 16.8 GiB | 20260506 13:06:02.354 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 13:06:02.354 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.7%) | 20260506 13:06:02.354 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:06:02.354 | INFO | High Water Breakdown: | 20260506 13:06:02.354 | INFO | 59.7% : 8.6 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:06:02.354 | INFO | 40.1% : 5.78 GiB ( 86.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:06:02.354 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:06:02.354 | INFO | ------------------------------------------ | 20260506 13:06:02.354 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:06:02.354 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:06:02.354 | INFO | Total mem usage: 26.5 GiB | 20260506 13:06:02.354 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:06:02.354 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:06:02.354 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:06:02.354 | INFO | High Water Breakdown: | 20260506 13:06:02.354 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:06:02.354 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:06:02.354 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:06:02.354 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:06:02.354 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:06:02.354 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:06:02.354 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:06:02.354 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:06:02.354 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:06:02.354 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:06:02.354 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:06:02.354 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:06:02.354 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:06:02.354 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:06:02.354 | INFO | ------------------------------------------ | 20260506 13:06:02.354 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:06:02.354 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:06:02.354 | INFO | Total mem usage: 25.6 GiB | 20260506 13:06:02.354 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:06:02.354 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:06:02.354 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:06:02.354 | INFO | High Water Breakdown: | 20260506 13:06:02.354 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:06:02.354 | INFO | ------------------------------------------ | 20260506 13:06:02.354 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:06:02.354 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:06:02.354 | INFO | Total mem usage: 15.7 GiB | 20260506 13:06:02.354 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:06:02.354 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:06:02.354 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:06:02.354 | INFO | High Water Breakdown: | 20260506 13:06:02.354 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:06:02.354 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:06:02.354 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:06:02.354 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:06:02.354 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:06:02.354 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:06:02.354 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:06:02.354 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:06:02.354 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:06:02.354 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:06:02.354 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:06:02.354 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:06:02.354 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:06:02.354 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:06:02.354 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:06:02.354 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:06:02.354 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:06:02.354 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:06:02.354 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:06:02.354 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:06:02.354 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:06:02.354 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:06:02.354 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:06:02.354 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:06:02.354 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:06:02.354 | INFO | ------------------------------------------ | 20260506 13:06:02.354 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:06:02.354 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:06:02.354 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:06:02.354 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:06:02.354 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.62e+01, 1.36e+01 / 1.89e+00 | 20260506 13:06:02.354 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.02e-01, 1.03e-02 / 6.24e-03 | 20260506 13:06:02.354 | INFO | PerfReport: 11.52%, Baseline -- 3.65e+00 / 4.46e+00, 5.66e-01 / 3.36e-01 | 20260506 13:06:02.354 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.47e+00, 1.88e-01 / 1.30e-01 | 20260506 13:06:02.354 | INFO | PerfReport: 15.82%, DME -- 4.01e+01 / 4.46e+01, 3.89e+00 / 2.27e+00 | 20260506 13:06:02.354 | INFO | PerfReport: 56.67%, FrameLabeling -- 1.80e+01 / 2.06e+01, 1.55e+00 / 1.90e+00 | 20260506 13:06:02.354 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.37e+00 / 2.99e+00, 4.09e-01 / 2.37e-01 | 20260506 13:06:02.354 | INFO | PerfReport: 3.68%, Metrics -- 1.17e+00 / 1.53e+00, 2.95e-01 / 1.64e-01 | 20260506 13:06:02.354 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.13e-01 / 4.77e-01, 6.54e-02 / 3.02e-02 | 20260506 13:06:02.355 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:06:02.355 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:06:02.355 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:06:02.355 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 13:06:02.355 | INFO | PerfReport: 21.40%, Upload -- 2.11e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 13:06:02.355 | INFO | PerfReport: 21.03%, Download -- 2.07e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 13:06:02.355 | INFO | PerfReport: 32.16%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 13:06:02.355 | INFO | PerfReport: 1.70%, LockUpload -- 1.68e+00 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260506 13:06:02.355 | INFO | PerfReport: 1.10%, LockDownload -- 1.08e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260506 13:06:02.355 | INFO | PerfReport: 22.59%, LockCompute -- 2.23e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 13:06:02.355 | INFO | PerfReport: 0.02%, Misc -- 1.50e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260506 13:06:02.483 | INFO | Starting a round of Baz IO | 20260506 13:06:05.640 | INFO | Populating RTMetrics report | 20260506 13:06:05.641 | INFO | RTMetrics report populated | 20260506 13:06:05.641 | INFO | Opening tmp RTMetrics json file | 20260506 13:06:05.641 | INFO | Serializing RTMetrics to json | 20260506 13:06:05.642 | INFO | Serialization done, swapping tmp file now... | 20260506 13:06:05.643 | INFO | RTMetrics done | 20260506 13:06:05.742 | INFO | Finished a round of Baz IO: Wrote 2.104 GiB in 3.259 seconds | 20260506 13:06:05.742 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.30% / 0.02% / 59.68% | 20260506 13:06:05.742 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.4162% / 1.19209e-05% / 5.5631e-05% | 20260506 13:06:05.742 | INFO | Baz IO overall: | 20260506 13:06:05.742 | INFO | BytesWritten : 797479749797 | 20260506 13:06:05.742 | INFO | NumEvents : 385621612401 | 20260506 13:06:07.492 | INFO | Analyzing chunk frames = [1581056,1581568) | 20260506 13:06:07.492 | INFO | MemInfo: { | 20260506 13:06:07.492 | INFO | "gpu" : " 17 GB", | 20260506 13:06:07.492 | INFO | "huge" : "", | 20260506 13:06:07.492 | INFO | "huge_shared" : " 27 GB", | 20260506 13:06:07.492 | INFO | "malloc" : " 14 GB", | 20260506 13:06:07.492 | INFO | "pinned" : " 29 GB" | 20260506 13:06:07.492 | INFO | } | 20260506 13:06:07.493 | INFO | Perf report for the last 5.13926 seconds: | 20260506 13:06:07.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:06:07.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:07.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:07.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:07.493 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260506 13:06:07.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:07.493 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:06:07.493 | INFO | PRE_HQ : 61.3%, 1.9, 61.0ms, 0 | 20260506 13:06:07.493 | INFO | BAZWRITER : 62.6%, 3.7, 118.7ms, 0 | 20260506 13:06:07.493 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 13:06:07.493 | INFO | Uploaded 36GiB to GPU in 2.07242s (17.371GiB/s) | 20260506 13:06:07.493 | INFO | Downloaded 38GiB to GPU in 2.21275s (17.1732GiB/s) | 20260506 13:06:12.633 | INFO | Analyzing chunk frames = [1581568,1582080) | 20260506 13:06:12.633 | INFO | MemInfo: { | 20260506 13:06:12.633 | INFO | "gpu" : " 17 GB", | 20260506 13:06:12.633 | INFO | "huge" : "", | 20260506 13:06:12.633 | INFO | "huge_shared" : " 27 GB", | 20260506 13:06:12.633 | INFO | "malloc" : " 14 GB", | 20260506 13:06:12.633 | INFO | "pinned" : " 29 GB" | 20260506 13:06:12.633 | INFO | } | 20260506 13:06:12.633 | INFO | Perf report for the last 5.14072 seconds: | 20260506 13:06:12.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:06:12.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:12.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:12.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:12.633 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 13:06:12.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:12.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:12.633 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:06:12.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:12.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:12.634 | INFO | Uploaded 36GiB to GPU in 2.00305s (17.9726GiB/s) | 20260506 13:06:12.634 | INFO | Downloaded 34GiB to GPU in 1.93221s (17.5964GiB/s) | 20260506 13:06:17.771 | INFO | Analyzing chunk frames = [1582080,1582592) | 20260506 13:06:17.771 | INFO | MemInfo: { | 20260506 13:06:17.771 | INFO | "gpu" : " 17 GB", | 20260506 13:06:17.771 | INFO | "huge" : "", | 20260506 13:06:17.771 | INFO | "huge_shared" : " 27 GB", | 20260506 13:06:17.771 | INFO | "malloc" : " 14 GB", | 20260506 13:06:17.772 | INFO | "pinned" : " 29 GB" | 20260506 13:06:17.772 | INFO | } | 20260506 13:06:17.772 | INFO | Perf report for the last 5.13827 seconds: | 20260506 13:06:17.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:06:17.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:17.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:17.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:17.772 | INFO | ANALYSIS : 61.4%, 3.0, 95.1ms, 0 | 20260506 13:06:17.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:17.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:17.772 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 13:06:17.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:17.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:17.772 | INFO | Uploaded 36GiB to GPU in 1.97646s (18.2143GiB/s) | 20260506 13:06:17.772 | INFO | Downloaded 34GiB to GPU in 1.92913s (17.6245GiB/s) | 20260506 13:06:22.911 | INFO | Analyzing chunk frames = [1582592,1583104) | 20260506 13:06:22.911 | INFO | MemInfo: { | 20260506 13:06:22.911 | INFO | "gpu" : " 17 GB", | 20260506 13:06:22.911 | INFO | "huge" : "", | 20260506 13:06:22.911 | INFO | "huge_shared" : " 27 GB", | 20260506 13:06:22.911 | INFO | "malloc" : " 14 GB", | 20260506 13:06:22.911 | INFO | "pinned" : " 29 GB" | 20260506 13:06:22.911 | INFO | } | 20260506 13:06:22.911 | INFO | Perf report for the last 5.13984 seconds: | 20260506 13:06:22.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:06:22.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:22.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:22.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:22.911 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 13:06:22.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:22.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:22.911 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:06:22.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:22.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:22.912 | INFO | Uploaded 36GiB to GPU in 1.95986s (18.3687GiB/s) | 20260506 13:06:22.912 | INFO | Downloaded 34GiB to GPU in 1.92202s (17.6897GiB/s) | 20260506 13:06:28.050 | INFO | Analyzing chunk frames = [1583104,1583616) | 20260506 13:06:28.050 | INFO | MemInfo: { | 20260506 13:06:28.050 | INFO | "gpu" : " 17 GB", | 20260506 13:06:28.050 | INFO | "huge" : "", | 20260506 13:06:28.050 | INFO | "huge_shared" : " 27 GB", | 20260506 13:06:28.050 | INFO | "malloc" : " 14 GB", | 20260506 13:06:28.050 | INFO | "pinned" : " 29 GB" | 20260506 13:06:28.050 | INFO | } | 20260506 13:06:28.050 | INFO | Perf report for the last 5.13854 seconds: | 20260506 13:06:28.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:06:28.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:28.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:28.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:28.050 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 13:06:28.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:28.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:28.050 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 13:06:28.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:28.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:28.050 | INFO | Uploaded 36GiB to GPU in 2.00399s (17.9641GiB/s) | 20260506 13:06:28.050 | INFO | Downloaded 34GiB to GPU in 1.94129s (17.5141GiB/s) | 20260506 13:06:33.189 | INFO | Analyzing chunk frames = [1583616,1584128) | 20260506 13:06:33.189 | INFO | MemInfo: { | 20260506 13:06:33.189 | INFO | "gpu" : " 17 GB", | 20260506 13:06:33.189 | INFO | "huge" : "", | 20260506 13:06:33.189 | INFO | "huge_shared" : " 27 GB", | 20260506 13:06:33.189 | INFO | "malloc" : " 14 GB", | 20260506 13:06:33.189 | INFO | "pinned" : " 29 GB" | 20260506 13:06:33.189 | INFO | } | 20260506 13:06:33.189 | INFO | Perf report for the last 5.13935 seconds: | 20260506 13:06:33.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:06:33.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:33.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:33.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:33.189 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 13:06:33.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:33.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:33.189 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 13:06:33.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:33.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:33.190 | INFO | Uploaded 36GiB to GPU in 2.06359s (17.4453GiB/s) | 20260506 13:06:33.190 | INFO | Downloaded 34GiB to GPU in 1.9681s (17.2755GiB/s) | 20260506 13:06:38.329 | INFO | Analyzing chunk frames = [1584128,1584640) | 20260506 13:06:38.329 | INFO | MemInfo: { | 20260506 13:06:38.329 | INFO | "gpu" : " 17 GB", | 20260506 13:06:38.329 | INFO | "huge" : "", | 20260506 13:06:38.329 | INFO | "huge_shared" : " 27 GB", | 20260506 13:06:38.329 | INFO | "malloc" : " 14 GB", | 20260506 13:06:38.329 | INFO | "pinned" : " 29 GB" | 20260506 13:06:38.329 | INFO | } | 20260506 13:06:38.329 | INFO | Perf report for the last 5.13955 seconds: | 20260506 13:06:38.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:06:38.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:38.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:38.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:38.329 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:06:38.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:38.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:38.329 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 13:06:38.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:38.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:38.329 | INFO | Uploaded 36GiB to GPU in 1.98921s (18.0976GiB/s) | 20260506 13:06:38.329 | INFO | Downloaded 34GiB to GPU in 2.03654s (16.695GiB/s) | 20260506 13:06:43.468 | INFO | Analyzing chunk frames = [1584640,1585152) | 20260506 13:06:43.468 | INFO | MemInfo: { | 20260506 13:06:43.468 | INFO | "gpu" : " 17 GB", | 20260506 13:06:43.468 | INFO | "huge" : "", | 20260506 13:06:43.468 | INFO | "huge_shared" : " 27 GB", | 20260506 13:06:43.468 | INFO | "malloc" : " 14 GB", | 20260506 13:06:43.468 | INFO | "pinned" : " 29 GB" | 20260506 13:06:43.468 | INFO | } | 20260506 13:06:43.468 | INFO | Perf report for the last 5.13899 seconds: | 20260506 13:06:43.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:06:43.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:43.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:43.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:43.468 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 13:06:43.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:43.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:43.468 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 13:06:43.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:43.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:43.468 | INFO | Uploaded 36GiB to GPU in 2.06722s (17.4147GiB/s) | 20260506 13:06:43.468 | INFO | Downloaded 34GiB to GPU in 1.96951s (17.2631GiB/s) | 20260506 13:06:43.619 | INFO | Starting a round of Baz IO | 20260506 13:06:46.759 | INFO | Populating RTMetrics report | 20260506 13:06:46.759 | INFO | RTMetrics report populated | 20260506 13:06:46.759 | INFO | Opening tmp RTMetrics json file | 20260506 13:06:46.760 | INFO | Serializing RTMetrics to json | 20260506 13:06:46.761 | INFO | Serialization done, swapping tmp file now... | 20260506 13:06:46.761 | INFO | RTMetrics done | 20260506 13:06:46.857 | INFO | Finished a round of Baz IO: Wrote 2.102 GiB in 3.238 seconds | 20260506 13:06:46.857 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.28% / 0.02% / 59.70% | 20260506 13:06:46.858 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.40006% / 4.37101e-05% / 5.16574e-05% | 20260506 13:06:46.858 | INFO | Baz IO overall: | 20260506 13:06:46.858 | INFO | BytesWritten : 799736733715 | 20260506 13:06:46.858 | INFO | NumEvents : 386715950511 | 20260506 13:06:48.608 | INFO | Analyzing chunk frames = [1585152,1585664) | 20260506 13:06:48.608 | INFO | MemInfo: { | 20260506 13:06:48.608 | INFO | "gpu" : " 17 GB", | 20260506 13:06:48.608 | INFO | "huge" : "", | 20260506 13:06:48.608 | INFO | "huge_shared" : " 27 GB", | 20260506 13:06:48.608 | INFO | "malloc" : " 14 GB", | 20260506 13:06:48.608 | INFO | "pinned" : " 29 GB" | 20260506 13:06:48.608 | INFO | } | 20260506 13:06:48.608 | INFO | Perf report for the last 5.13993 seconds: | 20260506 13:06:48.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:06:48.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:48.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:48.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:48.608 | INFO | ANALYSIS : 63.4%, 3.0, 98.5ms, 0 | 20260506 13:06:48.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:48.608 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 13:06:48.608 | INFO | PRE_HQ : 61.3%, 1.9, 61.6ms, 0 | 20260506 13:06:48.608 | INFO | BAZWRITER : 62.7%, 3.7, 118.9ms, 0 | 20260506 13:06:48.608 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:06:48.608 | INFO | Uploaded 36GiB to GPU in 2.0477s (17.5807GiB/s) | 20260506 13:06:48.608 | INFO | Downloaded 38GiB to GPU in 2.23124s (17.0309GiB/s) | 20260506 13:06:53.746 | INFO | Analyzing chunk frames = [1585664,1586176) | 20260506 13:06:53.746 | INFO | MemInfo: { | 20260506 13:06:53.746 | INFO | "gpu" : " 17 GB", | 20260506 13:06:53.746 | INFO | "huge" : "", | 20260506 13:06:53.746 | INFO | "huge_shared" : " 27 GB", | 20260506 13:06:53.747 | INFO | "malloc" : " 14 GB", | 20260506 13:06:53.747 | INFO | "pinned" : " 29 GB" | 20260506 13:06:53.747 | INFO | } | 20260506 13:06:53.747 | INFO | Perf report for the last 5.13871 seconds: | 20260506 13:06:53.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:06:53.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:53.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:53.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:53.747 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 13:06:53.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:53.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:53.747 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:06:53.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:53.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:53.747 | INFO | Uploaded 36GiB to GPU in 2.01318s (17.8822GiB/s) | 20260506 13:06:53.747 | INFO | Downloaded 34GiB to GPU in 1.94109s (17.5159GiB/s) | 20260506 13:06:58.886 | INFO | Analyzing chunk frames = [1586176,1586688) | 20260506 13:06:58.886 | INFO | MemInfo: { | 20260506 13:06:58.886 | INFO | "gpu" : " 17 GB", | 20260506 13:06:58.886 | INFO | "huge" : "", | 20260506 13:06:58.886 | INFO | "huge_shared" : " 27 GB", | 20260506 13:06:58.886 | INFO | "malloc" : " 14 GB", | 20260506 13:06:58.886 | INFO | "pinned" : " 29 GB" | 20260506 13:06:58.886 | INFO | } | 20260506 13:06:58.886 | INFO | Perf report for the last 5.13951 seconds: | 20260506 13:06:58.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:06:58.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:58.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:58.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:58.886 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 13:06:58.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:58.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:58.886 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:06:58.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:58.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:06:58.886 | INFO | Uploaded 36GiB to GPU in 1.9988s (18.0108GiB/s) | 20260506 13:06:58.886 | INFO | Downloaded 34GiB to GPU in 1.93537s (17.5677GiB/s) | 20260506 13:07:04.025 | INFO | Analyzing chunk frames = [1586688,1587200) | 20260506 13:07:04.025 | INFO | MemInfo: { | 20260506 13:07:04.025 | INFO | "gpu" : " 17 GB", | 20260506 13:07:04.025 | INFO | "huge" : "", | 20260506 13:07:04.025 | INFO | "huge_shared" : " 27 GB", | 20260506 13:07:04.025 | INFO | "malloc" : " 14 GB", | 20260506 13:07:04.025 | INFO | "pinned" : " 29 GB" | 20260506 13:07:04.025 | INFO | } | 20260506 13:07:04.025 | INFO | Perf report for the last 5.13914 seconds: | 20260506 13:07:04.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:07:04.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:04.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:04.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:04.025 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:07:04.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:04.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:04.025 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 13:07:04.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:04.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:04.026 | INFO | Uploaded 36GiB to GPU in 2.01345s (17.8797GiB/s) | 20260506 13:07:04.026 | INFO | Downloaded 34GiB to GPU in 1.93524s (17.5689GiB/s) | 20260506 13:07:09.165 | INFO | Analyzing chunk frames = [1587200,1587712) | 20260506 13:07:09.165 | INFO | MemInfo: { | 20260506 13:07:09.165 | INFO | "gpu" : " 17 GB", | 20260506 13:07:09.165 | INFO | "huge" : "", | 20260506 13:07:09.165 | INFO | "huge_shared" : " 27 GB", | 20260506 13:07:09.165 | INFO | "malloc" : " 14 GB", | 20260506 13:07:09.165 | INFO | "pinned" : " 29 GB" | 20260506 13:07:09.165 | INFO | } | 20260506 13:07:09.165 | INFO | Perf report for the last 5.13959 seconds: | 20260506 13:07:09.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:07:09.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:09.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:09.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:09.165 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 13:07:09.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:09.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:09.165 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:07:09.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:09.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:09.165 | INFO | Uploaded 36GiB to GPU in 2.03082s (17.7268GiB/s) | 20260506 13:07:09.165 | INFO | Downloaded 34GiB to GPU in 1.954s (17.4002GiB/s) | 20260506 13:07:14.304 | INFO | Analyzing chunk frames = [1587712,1588224) | 20260506 13:07:14.304 | INFO | MemInfo: { | 20260506 13:07:14.304 | INFO | "gpu" : " 17 GB", | 20260506 13:07:14.304 | INFO | "huge" : "", | 20260506 13:07:14.304 | INFO | "huge_shared" : " 27 GB", | 20260506 13:07:14.304 | INFO | "malloc" : " 14 GB", | 20260506 13:07:14.304 | INFO | "pinned" : " 29 GB" | 20260506 13:07:14.304 | INFO | } | 20260506 13:07:14.304 | INFO | Perf report for the last 5.13955 seconds: | 20260506 13:07:14.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:07:14.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:14.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:14.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:14.305 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:07:14.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:14.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:14.305 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:07:14.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:14.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:14.305 | INFO | Uploaded 36GiB to GPU in 2.07908s (17.3154GiB/s) | 20260506 13:07:14.305 | INFO | Downloaded 34GiB to GPU in 1.97819s (17.1874GiB/s) | 20260506 13:07:19.443 | INFO | Analyzing chunk frames = [1588224,1588736) | 20260506 13:07:19.444 | INFO | MemInfo: { | 20260506 13:07:19.444 | INFO | "gpu" : " 17 GB", | 20260506 13:07:19.444 | INFO | "huge" : "", | 20260506 13:07:19.444 | INFO | "huge_shared" : " 27 GB", | 20260506 13:07:19.444 | INFO | "malloc" : " 14 GB", | 20260506 13:07:19.444 | INFO | "pinned" : " 29 GB" | 20260506 13:07:19.444 | INFO | } | 20260506 13:07:19.444 | INFO | Perf report for the last 5.13928 seconds: | 20260506 13:07:19.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:07:19.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:19.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:19.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:19.444 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 13:07:19.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:19.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:19.444 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:07:19.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:19.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:19.444 | INFO | Uploaded 36GiB to GPU in 1.97268s (18.2493GiB/s) | 20260506 13:07:19.444 | INFO | Downloaded 34GiB to GPU in 2.02172s (16.8173GiB/s) | 20260506 13:07:24.583 | INFO | Analyzing chunk frames = [1588736,1589248) | 20260506 13:07:24.583 | INFO | MemInfo: { | 20260506 13:07:24.583 | INFO | "gpu" : " 17 GB", | 20260506 13:07:24.583 | INFO | "huge" : "", | 20260506 13:07:24.583 | INFO | "huge_shared" : " 27 GB", | 20260506 13:07:24.583 | INFO | "malloc" : " 14 GB", | 20260506 13:07:24.583 | INFO | "pinned" : " 29 GB" | 20260506 13:07:24.583 | INFO | } | 20260506 13:07:24.583 | INFO | Perf report for the last 5.13903 seconds: | 20260506 13:07:24.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:07:24.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:24.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:24.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:24.583 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 13:07:24.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:24.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:24.583 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:07:24.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:24.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:24.583 | INFO | Uploaded 36GiB to GPU in 2.04983s (17.5624GiB/s) | 20260506 13:07:24.583 | INFO | Downloaded 34GiB to GPU in 1.96202s (17.3291GiB/s) | 20260506 13:07:24.583 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:07:24.583 | INFO | Total mem usage: 16.8 GiB | 20260506 13:07:24.583 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 13:07:24.583 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.7%) | 20260506 13:07:24.583 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:07:24.583 | INFO | High Water Breakdown: | 20260506 13:07:24.583 | INFO | 59.7% : 8.6 GiB ( 82.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:07:24.583 | INFO | 40.1% : 5.78 GiB ( 86.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:07:24.583 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:07:24.583 | INFO | ------------------------------------------ | 20260506 13:07:24.583 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:07:24.583 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:07:24.583 | INFO | Total mem usage: 26.5 GiB | 20260506 13:07:24.583 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:07:24.583 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:07:24.583 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:07:24.583 | INFO | High Water Breakdown: | 20260506 13:07:24.583 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:07:24.583 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:07:24.584 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:07:24.584 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:07:24.584 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:07:24.584 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:07:24.584 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:07:24.584 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:07:24.584 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:07:24.584 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:07:24.584 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:07:24.584 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:07:24.584 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:07:24.584 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:07:24.584 | INFO | ------------------------------------------ | 20260506 13:07:24.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:07:24.584 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:07:24.584 | INFO | Total mem usage: 25.6 GiB | 20260506 13:07:24.584 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:07:24.584 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:07:24.584 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:07:24.584 | INFO | High Water Breakdown: | 20260506 13:07:24.584 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:07:24.584 | INFO | ------------------------------------------ | 20260506 13:07:24.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:07:24.584 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:07:24.584 | INFO | Total mem usage: 15.7 GiB | 20260506 13:07:24.584 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:07:24.584 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:07:24.584 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:07:24.584 | INFO | High Water Breakdown: | 20260506 13:07:24.584 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:07:24.584 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:07:24.584 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:07:24.584 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:07:24.584 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:07:24.584 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:07:24.584 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:07:24.584 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:07:24.584 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:07:24.584 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:07:24.584 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:07:24.584 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:07:24.584 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:07:24.584 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:07:24.584 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:07:24.584 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:07:24.584 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:07:24.584 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:07:24.584 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:07:24.584 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:07:24.584 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:07:24.584 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:07:24.584 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:07:24.584 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:07:24.584 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:07:24.584 | INFO | ------------------------------------------ | 20260506 13:07:24.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:07:24.584 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:07:24.584 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:07:24.584 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:07:24.584 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.63e+01, 1.36e+01 / 2.14e+00 | 20260506 13:07:24.584 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.01e-01, 1.02e-02 / 5.42e-03 | 20260506 13:07:24.584 | INFO | PerfReport: 11.57%, Baseline -- 3.66e+00 / 4.45e+00, 5.75e-01 / 3.51e-01 | 20260506 13:07:24.584 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.48e+00, 1.93e-01 / 1.17e-01 | 20260506 13:07:24.584 | INFO | PerfReport: 15.86%, DME -- 4.01e+01 / 4.46e+01, 4.14e+00 / 2.46e+00 | 20260506 13:07:24.584 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.41e+00 / 1.55e+00 | 20260506 13:07:24.584 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.38e+00 / 3.01e+00, 4.17e-01 / 2.40e-01 | 20260506 13:07:24.584 | INFO | PerfReport: 3.70%, Metrics -- 1.17e+00 / 1.54e+00, 3.01e-01 / 1.72e-01 | 20260506 13:07:24.584 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.12e-01 / 4.72e-01, 6.45e-02 / 2.58e-02 | 20260506 13:07:24.584 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:07:24.584 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:07:24.584 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:07:24.584 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 13:07:24.584 | INFO | PerfReport: 21.37%, Upload -- 2.11e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260506 13:07:24.584 | INFO | PerfReport: 21.06%, Download -- 2.08e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 13:07:24.584 | INFO | PerfReport: 32.12%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 13:07:24.584 | INFO | PerfReport: 1.63%, LockUpload -- 1.60e+00 / 0.00e+00, 6.93e+00 / 0.00e+00 | 20260506 13:07:24.584 | INFO | PerfReport: 1.16%, LockDownload -- 1.14e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 13:07:24.584 | INFO | PerfReport: 22.65%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 13:07:24.584 | INFO | PerfReport: 0.01%, Misc -- 1.47e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260506 13:07:24.712 | INFO | Starting a round of Baz IO | 20260506 13:07:27.878 | INFO | Populating RTMetrics report | 20260506 13:07:27.878 | INFO | RTMetrics report populated | 20260506 13:07:27.878 | INFO | Opening tmp RTMetrics json file | 20260506 13:07:27.879 | INFO | Serializing RTMetrics to json | 20260506 13:07:27.880 | INFO | Serialization done, swapping tmp file now... | 20260506 13:07:27.880 | INFO | RTMetrics done | 20260506 13:07:27.975 | INFO | Finished a round of Baz IO: Wrote 2.103 GiB in 3.262 seconds | 20260506 13:07:27.975 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.26% / 0.02% / 59.72% | 20260506 13:07:27.975 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.39346% / 4.76837e-05% / 6.75519e-05% | 20260506 13:07:27.975 | INFO | Baz IO overall: | 20260506 13:07:27.975 | INFO | BytesWritten : 801994444595 | 20260506 13:07:27.975 | INFO | NumEvents : 387810865316 | 20260506 13:07:29.722 | INFO | Analyzing chunk frames = [1589248,1589760) | 20260506 13:07:29.722 | INFO | MemInfo: { | 20260506 13:07:29.722 | INFO | "gpu" : " 17 GB", | 20260506 13:07:29.722 | INFO | "huge" : "", | 20260506 13:07:29.722 | INFO | "huge_shared" : " 27 GB", | 20260506 13:07:29.722 | INFO | "malloc" : " 14 GB", | 20260506 13:07:29.722 | INFO | "pinned" : " 29 GB" | 20260506 13:07:29.722 | INFO | } | 20260506 13:07:29.722 | INFO | Perf report for the last 5.13939 seconds: | 20260506 13:07:29.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:07:29.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:29.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:29.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:29.722 | INFO | ANALYSIS : 63.5%, 3.0, 98.7ms, 0 | 20260506 13:07:29.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:29.722 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 13:07:29.722 | INFO | PRE_HQ : 61.4%, 1.9, 60.9ms, 0 | 20260506 13:07:29.722 | INFO | BAZWRITER : 62.9%, 3.6, 117.7ms, 0 | 20260506 13:07:29.722 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:07:29.722 | INFO | Uploaded 36GiB to GPU in 2.06067s (17.4701GiB/s) | 20260506 13:07:29.723 | INFO | Downloaded 38GiB to GPU in 2.22261s (17.097GiB/s) | 20260506 13:07:34.861 | INFO | Analyzing chunk frames = [1589760,1590272) | 20260506 13:07:34.861 | INFO | MemInfo: { | 20260506 13:07:34.861 | INFO | "gpu" : " 17 GB", | 20260506 13:07:34.861 | INFO | "huge" : "", | 20260506 13:07:34.861 | INFO | "huge_shared" : " 27 GB", | 20260506 13:07:34.861 | INFO | "malloc" : " 14 GB", | 20260506 13:07:34.861 | INFO | "pinned" : " 29 GB" | 20260506 13:07:34.861 | INFO | } | 20260506 13:07:34.861 | INFO | Perf report for the last 5.13934 seconds: | 20260506 13:07:34.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:07:34.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:34.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:34.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:34.862 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:07:34.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:34.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:34.862 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:07:34.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:34.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:34.862 | INFO | Uploaded 36GiB to GPU in 2.03473s (17.6928GiB/s) | 20260506 13:07:34.862 | INFO | Downloaded 34GiB to GPU in 1.93588s (17.563GiB/s) | 20260506 13:07:40.001 | INFO | Analyzing chunk frames = [1590272,1590784) | 20260506 13:07:40.001 | INFO | MemInfo: { | 20260506 13:07:40.001 | INFO | "gpu" : " 17 GB", | 20260506 13:07:40.001 | INFO | "huge" : "", | 20260506 13:07:40.001 | INFO | "huge_shared" : " 27 GB", | 20260506 13:07:40.001 | INFO | "malloc" : " 14 GB", | 20260506 13:07:40.001 | INFO | "pinned" : " 29 GB" | 20260506 13:07:40.001 | INFO | } | 20260506 13:07:40.001 | INFO | Perf report for the last 5.13935 seconds: | 20260506 13:07:40.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:07:40.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:40.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:40.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:40.001 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 13:07:40.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:40.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:40.001 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:07:40.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:40.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:40.001 | INFO | Uploaded 36GiB to GPU in 2.03129s (17.7227GiB/s) | 20260506 13:07:40.001 | INFO | Downloaded 34GiB to GPU in 1.94082s (17.5184GiB/s) | 20260506 13:07:45.140 | INFO | Analyzing chunk frames = [1590784,1591296) | 20260506 13:07:45.140 | INFO | MemInfo: { | 20260506 13:07:45.140 | INFO | "gpu" : " 17 GB", | 20260506 13:07:45.140 | INFO | "huge" : "", | 20260506 13:07:45.140 | INFO | "huge_shared" : " 27 GB", | 20260506 13:07:45.140 | INFO | "malloc" : " 14 GB", | 20260506 13:07:45.140 | INFO | "pinned" : " 29 GB" | 20260506 13:07:45.140 | INFO | } | 20260506 13:07:45.140 | INFO | Perf report for the last 5.13932 seconds: | 20260506 13:07:45.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:07:45.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:45.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:45.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:45.140 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 13:07:45.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:45.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:45.140 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 13:07:45.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:45.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:45.140 | INFO | Uploaded 36GiB to GPU in 2.01249s (17.8883GiB/s) | 20260506 13:07:45.140 | INFO | Downloaded 34GiB to GPU in 1.92976s (17.6188GiB/s) | 20260506 13:07:50.279 | INFO | Analyzing chunk frames = [1591296,1591808) | 20260506 13:07:50.280 | INFO | MemInfo: { | 20260506 13:07:50.280 | INFO | "gpu" : " 17 GB", | 20260506 13:07:50.280 | INFO | "huge" : "", | 20260506 13:07:50.280 | INFO | "huge_shared" : " 27 GB", | 20260506 13:07:50.280 | INFO | "malloc" : " 14 GB", | 20260506 13:07:50.280 | INFO | "pinned" : " 29 GB" | 20260506 13:07:50.280 | INFO | } | 20260506 13:07:50.280 | INFO | Perf report for the last 5.13948 seconds: | 20260506 13:07:50.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:07:50.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:50.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:50.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:50.280 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 13:07:50.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:50.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:50.280 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 13:07:50.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:50.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:50.280 | INFO | Uploaded 36GiB to GPU in 2.02758s (17.7552GiB/s) | 20260506 13:07:50.280 | INFO | Downloaded 34GiB to GPU in 1.95119s (17.4253GiB/s) | 20260506 13:07:55.419 | INFO | Analyzing chunk frames = [1591808,1592320) | 20260506 13:07:55.419 | INFO | MemInfo: { | 20260506 13:07:55.419 | INFO | "gpu" : " 17 GB", | 20260506 13:07:55.419 | INFO | "huge" : "", | 20260506 13:07:55.419 | INFO | "huge_shared" : " 27 GB", | 20260506 13:07:55.419 | INFO | "malloc" : " 14 GB", | 20260506 13:07:55.419 | INFO | "pinned" : " 29 GB" | 20260506 13:07:55.419 | INFO | } | 20260506 13:07:55.419 | INFO | Perf report for the last 5.13956 seconds: | 20260506 13:07:55.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:07:55.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:55.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:55.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:55.419 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:07:55.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:55.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:55.419 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 13:07:55.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:55.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:07:55.419 | INFO | Uploaded 36GiB to GPU in 2.10101s (17.1346GiB/s) | 20260506 13:07:55.420 | INFO | Downloaded 34GiB to GPU in 1.97377s (17.2259GiB/s) | 20260506 13:08:00.559 | INFO | Analyzing chunk frames = [1592320,1592832) | 20260506 13:08:00.559 | INFO | MemInfo: { | 20260506 13:08:00.559 | INFO | "gpu" : " 17 GB", | 20260506 13:08:00.559 | INFO | "huge" : "", | 20260506 13:08:00.559 | INFO | "huge_shared" : " 27 GB", | 20260506 13:08:00.559 | INFO | "malloc" : " 14 GB", | 20260506 13:08:00.559 | INFO | "pinned" : " 29 GB" | 20260506 13:08:00.559 | INFO | } | 20260506 13:08:00.559 | INFO | Perf report for the last 5.14004 seconds: | 20260506 13:08:00.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:08:00.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:00.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:00.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:00.559 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:08:00.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:00.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:00.559 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 13:08:00.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:00.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:00.560 | INFO | Uploaded 36GiB to GPU in 1.98914s (18.0983GiB/s) | 20260506 13:08:00.560 | INFO | Downloaded 34GiB to GPU in 2.03234s (16.7294GiB/s) | 20260506 13:08:05.697 | INFO | Analyzing chunk frames = [1592832,1593344) | 20260506 13:08:05.697 | INFO | MemInfo: { | 20260506 13:08:05.697 | INFO | "gpu" : " 17 GB", | 20260506 13:08:05.697 | INFO | "huge" : "", | 20260506 13:08:05.697 | INFO | "huge_shared" : " 27 GB", | 20260506 13:08:05.697 | INFO | "malloc" : " 14 GB", | 20260506 13:08:05.697 | INFO | "pinned" : " 29 GB" | 20260506 13:08:05.697 | INFO | } | 20260506 13:08:05.698 | INFO | Perf report for the last 5.1382 seconds: | 20260506 13:08:05.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:08:05.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:05.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:05.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:05.698 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:08:05.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:05.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:05.698 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:08:05.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:05.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:05.698 | INFO | Uploaded 36GiB to GPU in 2.04244s (17.626GiB/s) | 20260506 13:08:05.698 | INFO | Downloaded 34GiB to GPU in 1.97781s (17.1907GiB/s) | 20260506 13:08:05.830 | INFO | Starting a round of Baz IO | 20260506 13:08:08.994 | INFO | Populating RTMetrics report | 20260506 13:08:08.994 | INFO | RTMetrics report populated | 20260506 13:08:08.994 | INFO | Opening tmp RTMetrics json file | 20260506 13:08:08.995 | INFO | Serializing RTMetrics to json | 20260506 13:08:08.996 | INFO | Serialization done, swapping tmp file now... | 20260506 13:08:08.997 | INFO | RTMetrics done | 20260506 13:08:09.091 | INFO | Finished a round of Baz IO: Wrote 2.103 GiB in 3.26 seconds | 20260506 13:08:09.091 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.24% / 0.02% / 59.74% | 20260506 13:08:09.091 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.38703% / 3.97364e-05% / 6.75519e-05% | 20260506 13:08:09.091 | INFO | Baz IO overall: | 20260506 13:08:09.091 | INFO | BytesWritten : 804252548393 | 20260506 13:08:09.091 | INFO | NumEvents : 388906069858 | 20260506 13:08:10.837 | INFO | Analyzing chunk frames = [1593344,1593856) | 20260506 13:08:10.837 | INFO | MemInfo: { | 20260506 13:08:10.837 | INFO | "gpu" : " 17 GB", | 20260506 13:08:10.837 | INFO | "huge" : "", | 20260506 13:08:10.837 | INFO | "huge_shared" : " 27 GB", | 20260506 13:08:10.837 | INFO | "malloc" : " 14 GB", | 20260506 13:08:10.837 | INFO | "pinned" : " 29 GB" | 20260506 13:08:10.837 | INFO | } | 20260506 13:08:10.837 | INFO | Perf report for the last 5.13924 seconds: | 20260506 13:08:10.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:08:10.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:10.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:10.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:10.837 | INFO | ANALYSIS : 63.6%, 3.0, 98.7ms, 0 | 20260506 13:08:10.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:10.837 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:08:10.837 | INFO | PRE_HQ : 61.2%, 1.9, 60.8ms, 0 | 20260506 13:08:10.837 | INFO | BAZWRITER : 62.8%, 3.6, 117.7ms, 0 | 20260506 13:08:10.837 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 13:08:10.837 | INFO | Uploaded 36GiB to GPU in 2.06623s (17.423GiB/s) | 20260506 13:08:10.837 | INFO | Downloaded 38GiB to GPU in 2.2184s (17.1294GiB/s) | 20260506 13:08:15.976 | INFO | Analyzing chunk frames = [1593856,1594368) | 20260506 13:08:15.976 | INFO | MemInfo: { | 20260506 13:08:15.976 | INFO | "gpu" : " 17 GB", | 20260506 13:08:15.976 | INFO | "huge" : "", | 20260506 13:08:15.976 | INFO | "huge_shared" : " 27 GB", | 20260506 13:08:15.976 | INFO | "malloc" : " 14 GB", | 20260506 13:08:15.976 | INFO | "pinned" : " 29 GB" | 20260506 13:08:15.976 | INFO | } | 20260506 13:08:15.977 | INFO | Perf report for the last 5.13973 seconds: | 20260506 13:08:15.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:08:15.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:15.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:15.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:15.977 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:08:15.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:15.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:15.977 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:08:15.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:15.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:15.977 | INFO | Uploaded 36GiB to GPU in 2.0064s (17.9426GiB/s) | 20260506 13:08:15.977 | INFO | Downloaded 34GiB to GPU in 1.92818s (17.6332GiB/s) | 20260506 13:08:21.116 | INFO | Analyzing chunk frames = [1594368,1594880) | 20260506 13:08:21.116 | INFO | MemInfo: { | 20260506 13:08:21.116 | INFO | "gpu" : " 17 GB", | 20260506 13:08:21.116 | INFO | "huge" : "", | 20260506 13:08:21.116 | INFO | "huge_shared" : " 27 GB", | 20260506 13:08:21.116 | INFO | "malloc" : " 14 GB", | 20260506 13:08:21.116 | INFO | "pinned" : " 29 GB" | 20260506 13:08:21.116 | INFO | } | 20260506 13:08:21.116 | INFO | Perf report for the last 5.13913 seconds: | 20260506 13:08:21.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:08:21.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:21.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:21.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:21.116 | INFO | ANALYSIS : 61.4%, 3.0, 95.1ms, 0 | 20260506 13:08:21.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:21.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:21.116 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:08:21.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:21.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:21.116 | INFO | Uploaded 36GiB to GPU in 1.97929s (18.1884GiB/s) | 20260506 13:08:21.116 | INFO | Downloaded 34GiB to GPU in 1.92908s (17.625GiB/s) | 20260506 13:08:26.255 | INFO | Analyzing chunk frames = [1594880,1595392) | 20260506 13:08:26.255 | INFO | MemInfo: { | 20260506 13:08:26.255 | INFO | "gpu" : " 17 GB", | 20260506 13:08:26.255 | INFO | "huge" : "", | 20260506 13:08:26.255 | INFO | "huge_shared" : " 27 GB", | 20260506 13:08:26.255 | INFO | "malloc" : " 14 GB", | 20260506 13:08:26.255 | INFO | "pinned" : " 29 GB" | 20260506 13:08:26.255 | INFO | } | 20260506 13:08:26.255 | INFO | Perf report for the last 5.13958 seconds: | 20260506 13:08:26.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:08:26.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:26.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:26.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:26.255 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:08:26.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:26.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:26.255 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:08:26.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:26.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:26.255 | INFO | Uploaded 36GiB to GPU in 2.03319s (17.7061GiB/s) | 20260506 13:08:26.255 | INFO | Downloaded 34GiB to GPU in 1.92264s (17.684GiB/s) | 20260506 13:08:31.394 | INFO | Analyzing chunk frames = [1595392,1595904) | 20260506 13:08:31.394 | INFO | MemInfo: { | 20260506 13:08:31.394 | INFO | "gpu" : " 17 GB", | 20260506 13:08:31.394 | INFO | "huge" : "", | 20260506 13:08:31.394 | INFO | "huge_shared" : " 27 GB", | 20260506 13:08:31.394 | INFO | "malloc" : " 14 GB", | 20260506 13:08:31.394 | INFO | "pinned" : " 29 GB" | 20260506 13:08:31.394 | INFO | } | 20260506 13:08:31.394 | INFO | Perf report for the last 5.13899 seconds: | 20260506 13:08:31.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:08:31.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:31.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:31.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:31.394 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 13:08:31.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:31.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:31.394 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 13:08:31.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:31.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:31.394 | INFO | Uploaded 36GiB to GPU in 2.02627s (17.7666GiB/s) | 20260506 13:08:31.395 | INFO | Downloaded 34GiB to GPU in 1.95096s (17.4273GiB/s) | 20260506 13:08:36.533 | INFO | Analyzing chunk frames = [1595904,1596416) | 20260506 13:08:36.533 | INFO | MemInfo: { | 20260506 13:08:36.533 | INFO | "gpu" : " 17 GB", | 20260506 13:08:36.533 | INFO | "huge" : "", | 20260506 13:08:36.533 | INFO | "huge_shared" : " 27 GB", | 20260506 13:08:36.533 | INFO | "malloc" : " 14 GB", | 20260506 13:08:36.533 | INFO | "pinned" : " 29 GB" | 20260506 13:08:36.533 | INFO | } | 20260506 13:08:36.533 | INFO | Perf report for the last 5.13923 seconds: | 20260506 13:08:36.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:08:36.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:36.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:36.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:36.533 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 13:08:36.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:36.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:36.534 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260506 13:08:36.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:36.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:36.534 | INFO | Uploaded 36GiB to GPU in 2.05358s (17.5304GiB/s) | 20260506 13:08:36.534 | INFO | Downloaded 34GiB to GPU in 1.93338s (17.5858GiB/s) | 20260506 13:08:41.673 | INFO | Analyzing chunk frames = [1596416,1596928) | 20260506 13:08:41.673 | INFO | MemInfo: { | 20260506 13:08:41.673 | INFO | "gpu" : " 17 GB", | 20260506 13:08:41.673 | INFO | "huge" : "", | 20260506 13:08:41.673 | INFO | "huge_shared" : " 27 GB", | 20260506 13:08:41.673 | INFO | "malloc" : " 14 GB", | 20260506 13:08:41.673 | INFO | "pinned" : " 29 GB" | 20260506 13:08:41.673 | INFO | } | 20260506 13:08:41.673 | INFO | Perf report for the last 5.13951 seconds: | 20260506 13:08:41.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:08:41.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:41.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:41.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:41.673 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 13:08:41.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:41.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:41.673 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 13:08:41.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:41.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:41.673 | INFO | Uploaded 36GiB to GPU in 2.01078s (17.9035GiB/s) | 20260506 13:08:41.673 | INFO | Downloaded 34GiB to GPU in 2.02706s (16.7731GiB/s) | 20260506 13:08:46.812 | INFO | Analyzing chunk frames = [1596928,1597440) | 20260506 13:08:46.812 | INFO | MemInfo: { | 20260506 13:08:46.812 | INFO | "gpu" : " 17 GB", | 20260506 13:08:46.812 | INFO | "huge" : "", | 20260506 13:08:46.812 | INFO | "huge_shared" : " 27 GB", | 20260506 13:08:46.812 | INFO | "malloc" : " 14 GB", | 20260506 13:08:46.812 | INFO | "pinned" : " 29 GB" | 20260506 13:08:46.812 | INFO | } | 20260506 13:08:46.812 | INFO | Perf report for the last 5.13923 seconds: | 20260506 13:08:46.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:08:46.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:46.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:46.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:46.812 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:08:46.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:46.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:46.812 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 13:08:46.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:46.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:46.812 | INFO | Uploaded 36GiB to GPU in 2.06047s (17.4718GiB/s) | 20260506 13:08:46.812 | INFO | Downloaded 34GiB to GPU in 1.95806s (17.3642GiB/s) | 20260506 13:08:46.813 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:08:46.813 | INFO | Total mem usage: 16.8 GiB | 20260506 13:08:46.813 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 13:08:46.813 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -16.4%) | 20260506 13:08:46.813 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:08:46.813 | INFO | High Water Breakdown: | 20260506 13:08:46.813 | INFO | 59.8% : 8.64 GiB ( 83.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:08:46.813 | INFO | 40% : 5.78 GiB ( 86.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:08:46.813 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:08:46.813 | INFO | ------------------------------------------ | 20260506 13:08:46.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:08:46.813 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:08:46.813 | INFO | Total mem usage: 26.5 GiB | 20260506 13:08:46.813 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:08:46.813 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:08:46.813 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:08:46.813 | INFO | High Water Breakdown: | 20260506 13:08:46.813 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:08:46.813 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:08:46.813 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:08:46.813 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:08:46.813 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:08:46.813 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:08:46.813 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:08:46.813 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:08:46.813 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:08:46.813 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:08:46.813 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:08:46.813 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:08:46.813 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:08:46.813 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:08:46.813 | INFO | ------------------------------------------ | 20260506 13:08:46.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:08:46.813 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:08:46.813 | INFO | Total mem usage: 25.6 GiB | 20260506 13:08:46.813 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:08:46.813 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:08:46.813 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:08:46.813 | INFO | High Water Breakdown: | 20260506 13:08:46.813 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:08:46.813 | INFO | ------------------------------------------ | 20260506 13:08:46.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:08:46.813 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:08:46.813 | INFO | Total mem usage: 15.7 GiB | 20260506 13:08:46.813 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:08:46.813 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:08:46.813 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 13:08:46.813 | INFO | High Water Breakdown: | 20260506 13:08:46.813 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:08:46.813 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:08:46.813 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:08:46.813 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:08:46.813 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:08:46.813 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:08:46.813 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:08:46.813 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:08:46.813 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:08:46.813 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:08:46.813 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:08:46.813 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:08:46.813 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:08:46.813 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:08:46.813 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:08:46.813 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:08:46.813 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:08:46.813 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:08:46.813 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:08:46.813 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:08:46.813 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:08:46.813 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:08:46.813 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:08:46.813 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:08:46.813 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:08:46.813 | INFO | ------------------------------------------ | 20260506 13:08:46.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:08:46.813 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:08:46.813 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:08:46.813 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:08:46.813 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.66e+01, 1.37e+01 / 2.51e+00 | 20260506 13:08:46.813 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.02e-01, 1.08e-02 / 6.12e-03 | 20260506 13:08:46.813 | INFO | PerfReport: 11.56%, Baseline -- 3.66e+00 / 4.43e+00, 5.76e-01 / 3.57e-01 | 20260506 13:08:46.813 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.48e+00, 1.92e-01 / 1.35e-01 | 20260506 13:08:46.813 | INFO | PerfReport: 15.85%, DME -- 4.02e+01 / 4.48e+01, 4.21e+00 / 2.66e+00 | 20260506 13:08:46.813 | INFO | PerfReport: 56.55%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.47e+00 / 1.85e+00 | 20260506 13:08:46.813 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.38e+00 / 2.97e+00, 4.10e-01 / 2.89e-01 | 20260506 13:08:46.813 | INFO | PerfReport: 3.69%, Metrics -- 1.17e+00 / 1.55e+00, 2.98e-01 / 1.72e-01 | 20260506 13:08:46.813 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.11e-01 / 4.71e-01, 6.39e-02 / 2.89e-02 | 20260506 13:08:46.813 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:08:46.813 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:08:46.813 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:08:46.813 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 13:08:46.813 | INFO | PerfReport: 21.48%, Upload -- 2.12e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260506 13:08:46.813 | INFO | PerfReport: 21.01%, Download -- 2.07e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 13:08:46.813 | INFO | PerfReport: 32.13%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 13:08:46.813 | INFO | PerfReport: 1.73%, LockUpload -- 1.71e+00 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 13:08:46.813 | INFO | PerfReport: 1.03%, LockDownload -- 1.02e+00 / 0.00e+00, 2.68e+00 / 0.00e+00 | 20260506 13:08:46.813 | INFO | PerfReport: 22.61%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 13:08:46.813 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260506 13:08:46.941 | INFO | Starting a round of Baz IO | 20260506 13:08:50.096 | INFO | Populating RTMetrics report | 20260506 13:08:50.097 | INFO | RTMetrics report populated | 20260506 13:08:50.097 | INFO | Opening tmp RTMetrics json file | 20260506 13:08:50.098 | INFO | Serializing RTMetrics to json | 20260506 13:08:50.099 | INFO | Serialization done, swapping tmp file now... | 20260506 13:08:50.100 | INFO | RTMetrics done | 20260506 13:08:50.202 | INFO | Finished a round of Baz IO: Wrote 2.106 GiB in 3.261 seconds | 20260506 13:08:50.202 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.22% / 0.02% / 59.76% | 20260506 13:08:50.202 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.38957% / 6.75519e-05% / 4.37101e-05% | 20260506 13:08:50.202 | INFO | Baz IO overall: | 20260506 13:08:50.202 | INFO | BytesWritten : 806514177277 | 20260506 13:08:50.202 | INFO | NumEvents : 390003711822 | 20260506 13:08:51.951 | INFO | Analyzing chunk frames = [1597440,1597952) | 20260506 13:08:51.951 | INFO | MemInfo: { | 20260506 13:08:51.951 | INFO | "gpu" : " 17 GB", | 20260506 13:08:51.951 | INFO | "huge" : "", | 20260506 13:08:51.951 | INFO | "huge_shared" : " 27 GB", | 20260506 13:08:51.951 | INFO | "malloc" : " 14 GB", | 20260506 13:08:51.951 | INFO | "pinned" : " 29 GB" | 20260506 13:08:51.951 | INFO | } | 20260506 13:08:51.951 | INFO | Perf report for the last 5.13916 seconds: | 20260506 13:08:51.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:08:51.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:51.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:51.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:51.951 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260506 13:08:51.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:51.951 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:08:51.951 | INFO | PRE_HQ : 61.5%, 1.9, 61.2ms, 0 | 20260506 13:08:51.951 | INFO | BAZWRITER : 62.9%, 3.7, 118.1ms, 0 | 20260506 13:08:51.951 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:08:51.952 | INFO | Uploaded 36GiB to GPU in 2.06409s (17.4411GiB/s) | 20260506 13:08:51.952 | INFO | Downloaded 38GiB to GPU in 2.22106s (17.1089GiB/s) | 20260506 13:08:57.091 | INFO | Analyzing chunk frames = [1597952,1598464) | 20260506 13:08:57.091 | INFO | MemInfo: { | 20260506 13:08:57.091 | INFO | "gpu" : " 17 GB", | 20260506 13:08:57.091 | INFO | "huge" : "", | 20260506 13:08:57.091 | INFO | "huge_shared" : " 27 GB", | 20260506 13:08:57.091 | INFO | "malloc" : " 14 GB", | 20260506 13:08:57.091 | INFO | "pinned" : " 29 GB" | 20260506 13:08:57.091 | INFO | } | 20260506 13:08:57.091 | INFO | Perf report for the last 5.13955 seconds: | 20260506 13:08:57.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:08:57.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:57.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:57.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:57.091 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 13:08:57.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:57.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:57.091 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:08:57.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:57.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:08:57.091 | INFO | Uploaded 36GiB to GPU in 2.05946s (17.4803GiB/s) | 20260506 13:08:57.091 | INFO | Downloaded 34GiB to GPU in 1.94487s (17.4819GiB/s) | 20260506 13:09:02.230 | INFO | Analyzing chunk frames = [1598464,1598976) | 20260506 13:09:02.230 | INFO | MemInfo: { | 20260506 13:09:02.230 | INFO | "gpu" : " 17 GB", | 20260506 13:09:02.230 | INFO | "huge" : "", | 20260506 13:09:02.230 | INFO | "huge_shared" : " 27 GB", | 20260506 13:09:02.231 | INFO | "malloc" : " 14 GB", | 20260506 13:09:02.231 | INFO | "pinned" : " 29 GB" | 20260506 13:09:02.231 | INFO | } | 20260506 13:09:02.231 | INFO | Perf report for the last 5.13966 seconds: | 20260506 13:09:02.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:09:02.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:02.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:02.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:02.231 | INFO | ANALYSIS : 61.7%, 3.0, 95.5ms, 0 | 20260506 13:09:02.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:02.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:02.231 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:09:02.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:02.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:02.231 | INFO | Uploaded 36GiB to GPU in 2.02105s (17.8126GiB/s) | 20260506 13:09:02.231 | INFO | Downloaded 34GiB to GPU in 1.94656s (17.4667GiB/s) | 20260506 13:09:07.370 | INFO | Analyzing chunk frames = [1598976,1599488) | 20260506 13:09:07.370 | INFO | MemInfo: { | 20260506 13:09:07.370 | INFO | "gpu" : " 17 GB", | 20260506 13:09:07.370 | INFO | "huge" : "", | 20260506 13:09:07.370 | INFO | "huge_shared" : " 27 GB", | 20260506 13:09:07.370 | INFO | "malloc" : " 14 GB", | 20260506 13:09:07.370 | INFO | "pinned" : " 29 GB" | 20260506 13:09:07.370 | INFO | } | 20260506 13:09:07.370 | INFO | Perf report for the last 5.13899 seconds: | 20260506 13:09:07.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:09:07.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:07.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:07.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:07.370 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 13:09:07.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:07.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:07.370 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:09:07.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:07.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:07.370 | INFO | Uploaded 36GiB to GPU in 1.9842s (18.1434GiB/s) | 20260506 13:09:07.370 | INFO | Downloaded 34GiB to GPU in 1.93934s (17.5317GiB/s) | 20260506 13:09:12.509 | INFO | Analyzing chunk frames = [1599488,1600000) | 20260506 13:09:12.509 | INFO | MemInfo: { | 20260506 13:09:12.509 | INFO | "gpu" : " 17 GB", | 20260506 13:09:12.509 | INFO | "huge" : "", | 20260506 13:09:12.509 | INFO | "huge_shared" : " 27 GB", | 20260506 13:09:12.509 | INFO | "malloc" : " 14 GB", | 20260506 13:09:12.509 | INFO | "pinned" : " 29 GB" | 20260506 13:09:12.509 | INFO | } | 20260506 13:09:12.509 | INFO | Perf report for the last 5.13978 seconds: | 20260506 13:09:12.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:09:12.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:12.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:12.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:12.510 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 13:09:12.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:12.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:12.510 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260506 13:09:12.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:12.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:12.510 | INFO | Uploaded 36GiB to GPU in 1.98073s (18.1751GiB/s) | 20260506 13:09:12.510 | INFO | Downloaded 34GiB to GPU in 1.93923s (17.5328GiB/s) | 20260506 13:09:17.648 | INFO | Analyzing chunk frames = [1600000,1600512) | 20260506 13:09:17.648 | INFO | MemInfo: { | 20260506 13:09:17.648 | INFO | "gpu" : " 17 GB", | 20260506 13:09:17.648 | INFO | "huge" : "", | 20260506 13:09:17.648 | INFO | "huge_shared" : " 27 GB", | 20260506 13:09:17.648 | INFO | "malloc" : " 14 GB", | 20260506 13:09:17.648 | INFO | "pinned" : " 29 GB" | 20260506 13:09:17.648 | INFO | } | 20260506 13:09:17.648 | INFO | Perf report for the last 5.13904 seconds: | 20260506 13:09:17.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:09:17.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:17.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:17.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:17.649 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:09:17.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:17.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:17.649 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:09:17.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:17.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:17.649 | INFO | Uploaded 36GiB to GPU in 2.06844s (17.4044GiB/s) | 20260506 13:09:17.649 | INFO | Downloaded 34GiB to GPU in 1.97324s (17.2306GiB/s) | 20260506 13:09:22.788 | INFO | Analyzing chunk frames = [1600512,1601024) | 20260506 13:09:22.788 | INFO | MemInfo: { | 20260506 13:09:22.788 | INFO | "gpu" : " 17 GB", | 20260506 13:09:22.788 | INFO | "huge" : "", | 20260506 13:09:22.788 | INFO | "huge_shared" : " 27 GB", | 20260506 13:09:22.788 | INFO | "malloc" : " 14 GB", | 20260506 13:09:22.788 | INFO | "pinned" : " 29 GB" | 20260506 13:09:22.788 | INFO | } | 20260506 13:09:22.788 | INFO | Perf report for the last 5.13967 seconds: | 20260506 13:09:22.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:09:22.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:22.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:22.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:22.788 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 13:09:22.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:22.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:22.788 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:09:22.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:22.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:22.788 | INFO | Uploaded 36GiB to GPU in 1.96469s (18.3235GiB/s) | 20260506 13:09:22.789 | INFO | Downloaded 34GiB to GPU in 2.02505s (16.7897GiB/s) | 20260506 13:09:27.927 | INFO | Analyzing chunk frames = [1601024,1601536) | 20260506 13:09:27.927 | INFO | MemInfo: { | 20260506 13:09:27.927 | INFO | "gpu" : " 17 GB", | 20260506 13:09:27.927 | INFO | "huge" : "", | 20260506 13:09:27.927 | INFO | "huge_shared" : " 27 GB", | 20260506 13:09:27.927 | INFO | "malloc" : " 14 GB", | 20260506 13:09:27.927 | INFO | "pinned" : " 29 GB" | 20260506 13:09:27.927 | INFO | } | 20260506 13:09:27.927 | INFO | Perf report for the last 5.13878 seconds: | 20260506 13:09:27.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:09:27.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:27.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:27.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:27.927 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 13:09:27.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:27.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:27.927 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:09:27.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:27.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:27.927 | INFO | Uploaded 36GiB to GPU in 2.05484s (17.5196GiB/s) | 20260506 13:09:27.927 | INFO | Downloaded 34GiB to GPU in 1.9693s (17.265GiB/s) | 20260506 13:09:28.056 | INFO | Starting a round of Baz IO | 20260506 13:09:31.217 | INFO | Populating RTMetrics report | 20260506 13:09:31.217 | INFO | RTMetrics report populated | 20260506 13:09:31.217 | INFO | Opening tmp RTMetrics json file | 20260506 13:09:31.219 | INFO | Serializing RTMetrics to json | 20260506 13:09:31.221 | INFO | Serialization done, swapping tmp file now... | 20260506 13:09:31.222 | INFO | RTMetrics done | 20260506 13:09:31.324 | INFO | Finished a round of Baz IO: Wrote 2.105 GiB in 3.267 seconds | 20260506 13:09:31.324 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.20% / 0.02% / 59.78% | 20260506 13:09:31.324 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.37706% / 4.76837e-05% / 5.16574e-05% | 20260506 13:09:31.324 | INFO | Baz IO overall: | 20260506 13:09:31.324 | INFO | BytesWritten : 808774788529 | 20260506 13:09:31.324 | INFO | NumEvents : 391100565893 | 20260506 13:09:33.066 | INFO | Analyzing chunk frames = [1601536,1602048) | 20260506 13:09:33.066 | INFO | MemInfo: { | 20260506 13:09:33.066 | INFO | "gpu" : " 17 GB", | 20260506 13:09:33.066 | INFO | "huge" : "", | 20260506 13:09:33.066 | INFO | "huge_shared" : " 27 GB", | 20260506 13:09:33.066 | INFO | "malloc" : " 14 GB", | 20260506 13:09:33.066 | INFO | "pinned" : " 29 GB" | 20260506 13:09:33.066 | INFO | } | 20260506 13:09:33.066 | INFO | Perf report for the last 5.13934 seconds: | 20260506 13:09:33.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:09:33.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:33.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:33.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:33.066 | INFO | ANALYSIS : 63.4%, 2.9, 95.5ms, 0 | 20260506 13:09:33.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:33.066 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 13:09:33.066 | INFO | PRE_HQ : 61.5%, 1.9, 61.0ms, 0 | 20260506 13:09:33.066 | INFO | BAZWRITER : 63.0%, 4.4, 142.8ms, 0 | 20260506 13:09:33.066 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:09:33.067 | INFO | Uploaded 36GiB to GPU in 2.05165s (17.5468GiB/s) | 20260506 13:09:33.067 | INFO | Downloaded 38GiB to GPU in 2.24366s (16.9366GiB/s) | 20260506 13:09:38.206 | INFO | Analyzing chunk frames = [1602048,1602560) | 20260506 13:09:38.206 | INFO | MemInfo: { | 20260506 13:09:38.206 | INFO | "gpu" : " 17 GB", | 20260506 13:09:38.206 | INFO | "huge" : "", | 20260506 13:09:38.206 | INFO | "huge_shared" : " 27 GB", | 20260506 13:09:38.206 | INFO | "malloc" : " 14 GB", | 20260506 13:09:38.206 | INFO | "pinned" : " 29 GB" | 20260506 13:09:38.206 | INFO | } | 20260506 13:09:38.206 | INFO | Perf report for the last 5.13951 seconds: | 20260506 13:09:38.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:09:38.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:38.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:38.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:38.206 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:09:38.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:38.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:38.206 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:09:38.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:38.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:38.206 | INFO | Uploaded 36GiB to GPU in 2.01529s (17.8634GiB/s) | 20260506 13:09:38.206 | INFO | Downloaded 34GiB to GPU in 1.94279s (17.5006GiB/s) | 20260506 13:09:43.345 | INFO | Analyzing chunk frames = [1602560,1603072) | 20260506 13:09:43.345 | INFO | MemInfo: { | 20260506 13:09:43.345 | INFO | "gpu" : " 17 GB", | 20260506 13:09:43.345 | INFO | "huge" : "", | 20260506 13:09:43.345 | INFO | "huge_shared" : " 27 GB", | 20260506 13:09:43.345 | INFO | "malloc" : " 14 GB", | 20260506 13:09:43.345 | INFO | "pinned" : " 29 GB" | 20260506 13:09:43.345 | INFO | } | 20260506 13:09:43.345 | INFO | Perf report for the last 5.13918 seconds: | 20260506 13:09:43.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:09:43.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:43.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:43.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:43.345 | INFO | ANALYSIS : 61.5%, 3.0, 95.2ms, 0 | 20260506 13:09:43.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:43.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:43.345 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:09:43.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:43.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:43.345 | INFO | Uploaded 36GiB to GPU in 1.99079s (18.0833GiB/s) | 20260506 13:09:43.345 | INFO | Downloaded 34GiB to GPU in 1.94474s (17.4831GiB/s) | 20260506 13:09:48.484 | INFO | Analyzing chunk frames = [1603072,1603584) | 20260506 13:09:48.484 | INFO | MemInfo: { | 20260506 13:09:48.484 | INFO | "gpu" : " 17 GB", | 20260506 13:09:48.484 | INFO | "huge" : "", | 20260506 13:09:48.484 | INFO | "huge_shared" : " 27 GB", | 20260506 13:09:48.484 | INFO | "malloc" : " 14 GB", | 20260506 13:09:48.484 | INFO | "pinned" : " 29 GB" | 20260506 13:09:48.484 | INFO | } | 20260506 13:09:48.484 | INFO | Perf report for the last 5.13938 seconds: | 20260506 13:09:48.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:09:48.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:48.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:48.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:48.484 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 13:09:48.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:48.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:48.485 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:09:48.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:48.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:48.485 | INFO | Uploaded 36GiB to GPU in 1.9937s (18.0569GiB/s) | 20260506 13:09:48.485 | INFO | Downloaded 34GiB to GPU in 1.92319s (17.6789GiB/s) | 20260506 13:09:53.624 | INFO | Analyzing chunk frames = [1603584,1604096) | 20260506 13:09:53.624 | INFO | MemInfo: { | 20260506 13:09:53.624 | INFO | "gpu" : " 17 GB", | 20260506 13:09:53.624 | INFO | "huge" : "", | 20260506 13:09:53.624 | INFO | "huge_shared" : " 27 GB", | 20260506 13:09:53.624 | INFO | "malloc" : " 14 GB", | 20260506 13:09:53.624 | INFO | "pinned" : " 29 GB" | 20260506 13:09:53.624 | INFO | } | 20260506 13:09:53.624 | INFO | Perf report for the last 5.13945 seconds: | 20260506 13:09:53.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:09:53.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:53.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:53.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:53.624 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 13:09:53.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:53.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:53.624 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:09:53.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:53.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:53.624 | INFO | Uploaded 36GiB to GPU in 2.02817s (17.75GiB/s) | 20260506 13:09:53.624 | INFO | Downloaded 34GiB to GPU in 1.95469s (17.3941GiB/s) | 20260506 13:09:58.763 | INFO | Analyzing chunk frames = [1604096,1604608) | 20260506 13:09:58.763 | INFO | MemInfo: { | 20260506 13:09:58.763 | INFO | "gpu" : " 17 GB", | 20260506 13:09:58.763 | INFO | "huge" : "", | 20260506 13:09:58.763 | INFO | "huge_shared" : " 27 GB", | 20260506 13:09:58.763 | INFO | "malloc" : " 14 GB", | 20260506 13:09:58.763 | INFO | "pinned" : " 29 GB" | 20260506 13:09:58.763 | INFO | } | 20260506 13:09:58.763 | INFO | Perf report for the last 5.13908 seconds: | 20260506 13:09:58.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:09:58.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:58.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:58.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:58.763 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 13:09:58.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:58.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:58.763 | INFO | PRE_HQ : 33.9%, 1.0, 17.6ms, 0 | 20260506 13:09:58.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:58.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:09:58.763 | INFO | Uploaded 36GiB to GPU in 2.06245s (17.455GiB/s) | 20260506 13:09:58.763 | INFO | Downloaded 34GiB to GPU in 1.93274s (17.5916GiB/s) | 20260506 13:10:03.903 | INFO | Analyzing chunk frames = [1604608,1605120) | 20260506 13:10:03.903 | INFO | MemInfo: { | 20260506 13:10:03.903 | INFO | "gpu" : " 17 GB", | 20260506 13:10:03.903 | INFO | "huge" : "", | 20260506 13:10:03.903 | INFO | "huge_shared" : " 27 GB", | 20260506 13:10:03.903 | INFO | "malloc" : " 14 GB", | 20260506 13:10:03.903 | INFO | "pinned" : " 29 GB" | 20260506 13:10:03.903 | INFO | } | 20260506 13:10:03.904 | INFO | Perf report for the last 5.14051 seconds: | 20260506 13:10:03.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:10:03.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:03.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:03.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:03.904 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:10:03.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:03.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:03.904 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:10:03.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:03.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:03.904 | INFO | Uploaded 36GiB to GPU in 2.01396s (17.8753GiB/s) | 20260506 13:10:03.904 | INFO | Downloaded 34GiB to GPU in 2.0263s (16.7793GiB/s) | 20260506 13:10:09.042 | INFO | Analyzing chunk frames = [1605120,1605632) | 20260506 13:10:09.042 | INFO | MemInfo: { | 20260506 13:10:09.042 | INFO | "gpu" : " 17 GB", | 20260506 13:10:09.042 | INFO | "huge" : "", | 20260506 13:10:09.042 | INFO | "huge_shared" : " 27 GB", | 20260506 13:10:09.042 | INFO | "malloc" : " 14 GB", | 20260506 13:10:09.042 | INFO | "pinned" : " 29 GB" | 20260506 13:10:09.042 | INFO | } | 20260506 13:10:09.042 | INFO | Perf report for the last 5.13827 seconds: | 20260506 13:10:09.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:10:09.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:09.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:09.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:09.042 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:10:09.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:09.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:09.042 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 13:10:09.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:09.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:09.042 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:10:09.042 | INFO | Total mem usage: 16.8 GiB | 20260506 13:10:09.042 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 13:10:09.042 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -13.7%) | 20260506 13:10:09.042 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:10:09.042 | INFO | High Water Breakdown: | 20260506 13:10:09.042 | INFO | 60.1% : 8.89 GiB ( 85.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:10:09.042 | INFO | 39.7% : 5.88 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:10:09.042 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:10:09.042 | INFO | ------------------------------------------ | 20260506 13:10:09.042 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:10:09.042 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:10:09.042 | INFO | Total mem usage: 26.5 GiB | 20260506 13:10:09.042 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:10:09.042 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:10:09.042 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:10:09.042 | INFO | Uploaded 36GiB to GPU in 2.05033s (17.5582GiB/s) | 20260506 13:10:09.042 | INFO | High Water Breakdown: | 20260506 13:10:09.042 | INFO | Downloaded 34GiB to GPU in 1.97415s (17.2226GiB/s) | 20260506 13:10:09.042 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:10:09.042 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:10:09.042 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:10:09.042 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:10:09.042 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:10:09.042 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:10:09.042 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:10:09.042 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:10:09.042 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:10:09.042 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:10:09.042 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:10:09.043 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:10:09.043 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:10:09.043 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:10:09.043 | INFO | ------------------------------------------ | 20260506 13:10:09.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:10:09.043 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:10:09.043 | INFO | Total mem usage: 25.6 GiB | 20260506 13:10:09.043 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:10:09.043 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:10:09.043 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:10:09.043 | INFO | High Water Breakdown: | 20260506 13:10:09.043 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:10:09.043 | INFO | ------------------------------------------ | 20260506 13:10:09.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:10:09.043 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:10:09.043 | INFO | Total mem usage: 15.7 GiB | 20260506 13:10:09.043 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:10:09.043 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:10:09.043 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:10:09.043 | INFO | High Water Breakdown: | 20260506 13:10:09.043 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:10:09.043 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:10:09.043 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:10:09.043 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:10:09.043 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:10:09.043 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:10:09.043 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:10:09.043 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:10:09.043 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:10:09.043 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:10:09.043 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:10:09.043 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:10:09.043 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:10:09.043 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:10:09.043 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:10:09.043 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:10:09.043 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:10:09.043 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:10:09.043 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:10:09.043 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:10:09.043 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:10:09.043 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:10:09.043 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:10:09.043 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:10:09.043 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:10:09.043 | INFO | ------------------------------------------ | 20260506 13:10:09.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:10:09.043 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:10:09.043 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:10:09.043 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:10:09.043 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.61e+01, 1.35e+01 / 2.34e+00 | 20260506 13:10:09.043 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.01e-01, 1.01e-02 / 6.35e-03 | 20260506 13:10:09.043 | INFO | PerfReport: 11.56%, Baseline -- 3.66e+00 / 4.47e+00, 5.71e-01 / 3.11e-01 | 20260506 13:10:09.043 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.47e+00, 1.89e-01 / 1.44e-01 | 20260506 13:10:09.043 | INFO | PerfReport: 15.85%, DME -- 4.01e+01 / 4.46e+01, 4.10e+00 / 2.99e+00 | 20260506 13:10:09.043 | INFO | PerfReport: 56.52%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.40e+00 / 2.03e+00 | 20260506 13:10:09.043 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.38e+00 / 2.99e+00, 4.17e-01 / 2.77e-01 | 20260506 13:10:09.043 | INFO | PerfReport: 3.69%, Metrics -- 1.17e+00 / 1.54e+00, 2.98e-01 / 1.56e-01 | 20260506 13:10:09.043 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.11e-01 / 4.69e-01, 6.40e-02 / 2.68e-02 | 20260506 13:10:09.043 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:10:09.043 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:10:09.043 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:10:09.043 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 13:10:09.043 | INFO | PerfReport: 21.45%, Upload -- 2.11e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 13:10:09.043 | INFO | PerfReport: 21.12%, Download -- 2.08e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 13:10:09.043 | INFO | PerfReport: 32.17%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 13:10:09.043 | INFO | PerfReport: 1.67%, LockUpload -- 1.64e+00 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260506 13:10:09.043 | INFO | PerfReport: 1.18%, LockDownload -- 1.16e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260506 13:10:09.043 | INFO | PerfReport: 22.40%, LockCompute -- 2.20e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 13:10:09.043 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260506 13:10:09.176 | INFO | Starting a round of Baz IO | 20260506 13:10:12.332 | INFO | Populating RTMetrics report | 20260506 13:10:12.332 | INFO | RTMetrics report populated | 20260506 13:10:12.332 | INFO | Opening tmp RTMetrics json file | 20260506 13:10:12.345 | INFO | Serializing RTMetrics to json | 20260506 13:10:12.369 | INFO | Serialization done, swapping tmp file now... | 20260506 13:10:12.372 | INFO | RTMetrics done | 20260506 13:10:12.435 | INFO | Finished a round of Baz IO: Wrote 2.104 GiB in 3.259 seconds | 20260506 13:10:12.435 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.18% / 0.02% / 59.80% | 20260506 13:10:12.435 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.36077% / 3.57628e-05% / 7.54992e-05% | 20260506 13:10:12.435 | INFO | Baz IO overall: | 20260506 13:10:12.435 | INFO | BytesWritten : 811034327788 | 20260506 13:10:12.435 | INFO | NumEvents : 392196701633 | 20260506 13:10:14.181 | INFO | Analyzing chunk frames = [1605632,1606144) | 20260506 13:10:14.181 | INFO | MemInfo: { | 20260506 13:10:14.181 | INFO | "gpu" : " 17 GB", | 20260506 13:10:14.181 | INFO | "huge" : "", | 20260506 13:10:14.181 | INFO | "huge_shared" : " 27 GB", | 20260506 13:10:14.181 | INFO | "malloc" : " 14 GB", | 20260506 13:10:14.181 | INFO | "pinned" : " 29 GB" | 20260506 13:10:14.181 | INFO | } | 20260506 13:10:14.181 | INFO | Perf report for the last 5.13949 seconds: | 20260506 13:10:14.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:10:14.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:14.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:14.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:14.181 | INFO | ANALYSIS : 63.5%, 2.9, 96.8ms, 0 | 20260506 13:10:14.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:14.181 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 13:10:14.181 | INFO | PRE_HQ : 61.1%, 1.9, 60.4ms, 0 | 20260506 13:10:14.181 | INFO | BAZWRITER : 62.7%, 4.3, 140.1ms, 0 | 20260506 13:10:14.182 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 13:10:14.182 | INFO | Uploaded 36GiB to GPU in 2.10367s (17.1129GiB/s) | 20260506 13:10:14.182 | INFO | Downloaded 38GiB to GPU in 2.23492s (17.0028GiB/s) | 20260506 13:10:19.320 | INFO | Analyzing chunk frames = [1606144,1606656) | 20260506 13:10:19.320 | INFO | MemInfo: { | 20260506 13:10:19.320 | INFO | "gpu" : " 17 GB", | 20260506 13:10:19.320 | INFO | "huge" : "", | 20260506 13:10:19.320 | INFO | "huge_shared" : " 27 GB", | 20260506 13:10:19.320 | INFO | "malloc" : " 14 GB", | 20260506 13:10:19.320 | INFO | "pinned" : " 29 GB" | 20260506 13:10:19.320 | INFO | } | 20260506 13:10:19.321 | INFO | Perf report for the last 5.13934 seconds: | 20260506 13:10:19.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:10:19.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:19.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:19.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:19.321 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:10:19.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:19.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:19.321 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:10:19.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:19.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:19.321 | INFO | Uploaded 36GiB to GPU in 2.01782s (17.841GiB/s) | 20260506 13:10:19.321 | INFO | Downloaded 34GiB to GPU in 1.9263s (17.6504GiB/s) | 20260506 13:10:24.460 | INFO | Analyzing chunk frames = [1606656,1607168) | 20260506 13:10:24.460 | INFO | MemInfo: { | 20260506 13:10:24.460 | INFO | "gpu" : " 17 GB", | 20260506 13:10:24.460 | INFO | "huge" : "", | 20260506 13:10:24.460 | INFO | "huge_shared" : " 27 GB", | 20260506 13:10:24.460 | INFO | "malloc" : " 14 GB", | 20260506 13:10:24.460 | INFO | "pinned" : " 29 GB" | 20260506 13:10:24.460 | INFO | } | 20260506 13:10:24.460 | INFO | Perf report for the last 5.13926 seconds: | 20260506 13:10:24.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:10:24.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:24.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:24.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:24.460 | INFO | ANALYSIS : 61.4%, 3.0, 95.1ms, 0 | 20260506 13:10:24.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:24.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:24.460 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:10:24.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:24.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:24.460 | INFO | Uploaded 36GiB to GPU in 1.97582s (18.2203GiB/s) | 20260506 13:10:24.460 | INFO | Downloaded 34GiB to GPU in 1.92868s (17.6286GiB/s) | 20260506 13:10:29.599 | INFO | Analyzing chunk frames = [1607168,1607680) | 20260506 13:10:29.599 | INFO | MemInfo: { | 20260506 13:10:29.599 | INFO | "gpu" : " 17 GB", | 20260506 13:10:29.599 | INFO | "huge" : "", | 20260506 13:10:29.599 | INFO | "huge_shared" : " 27 GB", | 20260506 13:10:29.600 | INFO | "malloc" : " 14 GB", | 20260506 13:10:29.600 | INFO | "pinned" : " 29 GB" | 20260506 13:10:29.600 | INFO | } | 20260506 13:10:29.600 | INFO | Perf report for the last 5.13964 seconds: | 20260506 13:10:29.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:10:29.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:29.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:29.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:29.600 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:10:29.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:29.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:29.600 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:10:29.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:29.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:29.600 | INFO | Uploaded 36GiB to GPU in 2.01662s (17.8517GiB/s) | 20260506 13:10:29.600 | INFO | Downloaded 34GiB to GPU in 1.91641s (17.7415GiB/s) | 20260506 13:10:34.738 | INFO | Analyzing chunk frames = [1607680,1608192) | 20260506 13:10:34.738 | INFO | MemInfo: { | 20260506 13:10:34.738 | INFO | "gpu" : " 17 GB", | 20260506 13:10:34.738 | INFO | "huge" : "", | 20260506 13:10:34.738 | INFO | "huge_shared" : " 27 GB", | 20260506 13:10:34.738 | INFO | "malloc" : " 14 GB", | 20260506 13:10:34.738 | INFO | "pinned" : " 29 GB" | 20260506 13:10:34.738 | INFO | } | 20260506 13:10:34.738 | INFO | Perf report for the last 5.13889 seconds: | 20260506 13:10:34.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:10:34.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:34.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:34.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:34.739 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 13:10:34.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:34.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:34.739 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 13:10:34.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:34.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:34.739 | INFO | Uploaded 36GiB to GPU in 2.01425s (17.8726GiB/s) | 20260506 13:10:34.739 | INFO | Downloaded 34GiB to GPU in 1.94293s (17.4993GiB/s) | 20260506 13:10:39.878 | INFO | Analyzing chunk frames = [1608192,1608704) | 20260506 13:10:39.878 | INFO | MemInfo: { | 20260506 13:10:39.878 | INFO | "gpu" : " 17 GB", | 20260506 13:10:39.878 | INFO | "huge" : "", | 20260506 13:10:39.878 | INFO | "huge_shared" : " 27 GB", | 20260506 13:10:39.878 | INFO | "malloc" : " 14 GB", | 20260506 13:10:39.878 | INFO | "pinned" : " 29 GB" | 20260506 13:10:39.878 | INFO | } | 20260506 13:10:39.878 | INFO | Perf report for the last 5.13985 seconds: | 20260506 13:10:39.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:10:39.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:39.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:39.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:39.878 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:10:39.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:39.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:39.878 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:10:39.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:39.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:39.879 | INFO | Uploaded 36GiB to GPU in 2.10106s (17.1342GiB/s) | 20260506 13:10:39.879 | INFO | Downloaded 34GiB to GPU in 1.97239s (17.238GiB/s) | 20260506 13:10:45.017 | INFO | Analyzing chunk frames = [1608704,1609216) | 20260506 13:10:45.017 | INFO | MemInfo: { | 20260506 13:10:45.017 | INFO | "gpu" : " 17 GB", | 20260506 13:10:45.017 | INFO | "huge" : "", | 20260506 13:10:45.017 | INFO | "huge_shared" : " 27 GB", | 20260506 13:10:45.017 | INFO | "malloc" : " 14 GB", | 20260506 13:10:45.017 | INFO | "pinned" : " 29 GB" | 20260506 13:10:45.017 | INFO | } | 20260506 13:10:45.017 | INFO | Perf report for the last 5.13881 seconds: | 20260506 13:10:45.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:10:45.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:45.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:45.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:45.017 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:10:45.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:45.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:45.017 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 13:10:45.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:45.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:45.017 | INFO | Uploaded 36GiB to GPU in 1.99853s (18.0132GiB/s) | 20260506 13:10:45.017 | INFO | Downloaded 34GiB to GPU in 2.03179s (16.734GiB/s) | 20260506 13:10:50.157 | INFO | Analyzing chunk frames = [1609216,1609728) | 20260506 13:10:50.157 | INFO | MemInfo: { | 20260506 13:10:50.157 | INFO | "gpu" : " 17 GB", | 20260506 13:10:50.157 | INFO | "huge" : "", | 20260506 13:10:50.157 | INFO | "huge_shared" : " 27 GB", | 20260506 13:10:50.157 | INFO | "malloc" : " 14 GB", | 20260506 13:10:50.157 | INFO | "pinned" : " 29 GB" | 20260506 13:10:50.157 | INFO | } | 20260506 13:10:50.157 | INFO | Perf report for the last 5.13947 seconds: | 20260506 13:10:50.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:10:50.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:50.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:50.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:50.157 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 13:10:50.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:50.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:50.157 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 13:10:50.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:50.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:50.157 | INFO | Uploaded 36GiB to GPU in 2.08s (17.3077GiB/s) | 20260506 13:10:50.157 | INFO | Downloaded 34GiB to GPU in 1.9881s (17.1018GiB/s) | 20260506 13:10:50.286 | INFO | Starting a round of Baz IO | 20260506 13:10:53.459 | INFO | Populating RTMetrics report | 20260506 13:10:53.460 | INFO | RTMetrics report populated | 20260506 13:10:53.460 | INFO | Opening tmp RTMetrics json file | 20260506 13:10:53.461 | INFO | Serializing RTMetrics to json | 20260506 13:10:53.462 | INFO | Serialization done, swapping tmp file now... | 20260506 13:10:53.462 | INFO | RTMetrics done | 20260506 13:10:53.560 | INFO | Finished a round of Baz IO: Wrote 2.107 GiB in 3.274 seconds | 20260506 13:10:53.560 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.16% / 0.02% / 59.82% | 20260506 13:10:53.560 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.36318% / 4.76837e-05% / 5.96046e-05% | 20260506 13:10:53.560 | INFO | Baz IO overall: | 20260506 13:10:53.560 | INFO | BytesWritten : 813296607191 | 20260506 13:10:53.560 | INFO | NumEvents : 393294814535 | 20260506 13:10:55.296 | INFO | Analyzing chunk frames = [1609728,1610240) | 20260506 13:10:55.296 | INFO | MemInfo: { | 20260506 13:10:55.296 | INFO | "gpu" : " 17 GB", | 20260506 13:10:55.296 | INFO | "huge" : "", | 20260506 13:10:55.296 | INFO | "huge_shared" : " 27 GB", | 20260506 13:10:55.296 | INFO | "malloc" : " 14 GB", | 20260506 13:10:55.296 | INFO | "pinned" : " 29 GB" | 20260506 13:10:55.296 | INFO | } | 20260506 13:10:55.296 | INFO | Perf report for the last 5.13927 seconds: | 20260506 13:10:55.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:10:55.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:55.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:55.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:55.296 | INFO | ANALYSIS : 63.7%, 2.9, 95.7ms, 0 | 20260506 13:10:55.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:10:55.296 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 13:10:55.296 | INFO | PRE_HQ : 61.8%, 1.9, 61.2ms, 0 | 20260506 13:10:55.296 | INFO | BAZWRITER : 63.1%, 4.5, 144.7ms, 0 | 20260506 13:10:55.296 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 13:10:55.296 | INFO | Uploaded 36GiB to GPU in 2.0893s (17.2306GiB/s) | 20260506 13:10:55.296 | INFO | Downloaded 38GiB to GPU in 2.23554s (16.9981GiB/s) | 20260506 13:11:00.435 | INFO | Analyzing chunk frames = [1610240,1610752) | 20260506 13:11:00.435 | INFO | MemInfo: { | 20260506 13:11:00.435 | INFO | "gpu" : " 17 GB", | 20260506 13:11:00.435 | INFO | "huge" : "", | 20260506 13:11:00.435 | INFO | "huge_shared" : " 27 GB", | 20260506 13:11:00.435 | INFO | "malloc" : " 14 GB", | 20260506 13:11:00.435 | INFO | "pinned" : " 29 GB" | 20260506 13:11:00.435 | INFO | } | 20260506 13:11:00.435 | INFO | Perf report for the last 5.13938 seconds: | 20260506 13:11:00.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:11:00.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:00.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:00.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:00.435 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:11:00.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:00.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:00.435 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:11:00.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:00.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:00.436 | INFO | Uploaded 36GiB to GPU in 2.04453s (17.608GiB/s) | 20260506 13:11:00.436 | INFO | Downloaded 34GiB to GPU in 1.94213s (17.5066GiB/s) | 20260506 13:11:05.574 | INFO | Analyzing chunk frames = [1610752,1611264) | 20260506 13:11:05.574 | INFO | MemInfo: { | 20260506 13:11:05.574 | INFO | "gpu" : " 17 GB", | 20260506 13:11:05.574 | INFO | "huge" : "", | 20260506 13:11:05.574 | INFO | "huge_shared" : " 27 GB", | 20260506 13:11:05.574 | INFO | "malloc" : " 14 GB", | 20260506 13:11:05.575 | INFO | "pinned" : " 29 GB" | 20260506 13:11:05.575 | INFO | } | 20260506 13:11:05.575 | INFO | Perf report for the last 5.13923 seconds: | 20260506 13:11:05.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:11:05.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:05.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:05.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:05.575 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260506 13:11:05.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:05.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:05.575 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:11:05.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:05.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:05.575 | INFO | Uploaded 36GiB to GPU in 2.01093s (17.9022GiB/s) | 20260506 13:11:05.575 | INFO | Downloaded 34GiB to GPU in 1.94722s (17.4608GiB/s) | 20260506 13:11:10.714 | INFO | Analyzing chunk frames = [1611264,1611776) | 20260506 13:11:10.714 | INFO | MemInfo: { | 20260506 13:11:10.714 | INFO | "gpu" : " 17 GB", | 20260506 13:11:10.714 | INFO | "huge" : "", | 20260506 13:11:10.714 | INFO | "huge_shared" : " 27 GB", | 20260506 13:11:10.714 | INFO | "malloc" : " 14 GB", | 20260506 13:11:10.714 | INFO | "pinned" : " 29 GB" | 20260506 13:11:10.714 | INFO | } | 20260506 13:11:10.714 | INFO | Perf report for the last 5.13964 seconds: | 20260506 13:11:10.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:11:10.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:10.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:10.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:10.714 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:11:10.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:10.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:10.714 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:11:10.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:10.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:10.714 | INFO | Uploaded 36GiB to GPU in 2.00068s (17.9938GiB/s) | 20260506 13:11:10.715 | INFO | Downloaded 34GiB to GPU in 1.93051s (17.6119GiB/s) | 20260506 13:11:15.853 | INFO | Analyzing chunk frames = [1611776,1612288) | 20260506 13:11:15.853 | INFO | MemInfo: { | 20260506 13:11:15.853 | INFO | "gpu" : " 17 GB", | 20260506 13:11:15.853 | INFO | "huge" : "", | 20260506 13:11:15.853 | INFO | "huge_shared" : " 27 GB", | 20260506 13:11:15.853 | INFO | "malloc" : " 14 GB", | 20260506 13:11:15.853 | INFO | "pinned" : " 29 GB" | 20260506 13:11:15.853 | INFO | } | 20260506 13:11:15.853 | INFO | Perf report for the last 5.13908 seconds: | 20260506 13:11:15.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:11:15.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:15.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:15.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:15.853 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 13:11:15.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:15.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:15.853 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:11:15.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:15.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:15.854 | INFO | Uploaded 36GiB to GPU in 2.01416s (17.8735GiB/s) | 20260506 13:11:15.854 | INFO | Downloaded 34GiB to GPU in 1.95029s (17.4333GiB/s) | 20260506 13:11:20.993 | INFO | Analyzing chunk frames = [1612288,1612800) | 20260506 13:11:20.993 | INFO | MemInfo: { | 20260506 13:11:20.993 | INFO | "gpu" : " 17 GB", | 20260506 13:11:20.993 | INFO | "huge" : "", | 20260506 13:11:20.993 | INFO | "huge_shared" : " 27 GB", | 20260506 13:11:20.993 | INFO | "malloc" : " 14 GB", | 20260506 13:11:20.993 | INFO | "pinned" : " 29 GB" | 20260506 13:11:20.993 | INFO | } | 20260506 13:11:20.993 | INFO | Perf report for the last 5.13956 seconds: | 20260506 13:11:20.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:11:20.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:20.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:20.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:20.993 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 13:11:20.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:20.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:20.993 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260506 13:11:20.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:20.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:20.993 | INFO | Uploaded 36GiB to GPU in 2.04591s (17.5961GiB/s) | 20260506 13:11:20.993 | INFO | Downloaded 34GiB to GPU in 1.93862s (17.5382GiB/s) | 20260506 13:11:26.132 | INFO | Analyzing chunk frames = [1612800,1613312) | 20260506 13:11:26.132 | INFO | MemInfo: { | 20260506 13:11:26.132 | INFO | "gpu" : " 17 GB", | 20260506 13:11:26.132 | INFO | "huge" : "", | 20260506 13:11:26.133 | INFO | "huge_shared" : " 27 GB", | 20260506 13:11:26.133 | INFO | "malloc" : " 14 GB", | 20260506 13:11:26.133 | INFO | "pinned" : " 29 GB" | 20260506 13:11:26.133 | INFO | } | 20260506 13:11:26.133 | INFO | Perf report for the last 5.13972 seconds: | 20260506 13:11:26.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:11:26.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:26.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:26.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:26.133 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 13:11:26.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:26.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:26.133 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 13:11:26.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:26.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:26.133 | INFO | Uploaded 36GiB to GPU in 1.96195s (18.3491GiB/s) | 20260506 13:11:26.133 | INFO | Downloaded 34GiB to GPU in 2.01802s (16.8482GiB/s) | 20260506 13:11:31.271 | INFO | Analyzing chunk frames = [1613312,1613824) | 20260506 13:11:31.271 | INFO | MemInfo: { | 20260506 13:11:31.271 | INFO | "gpu" : " 17 GB", | 20260506 13:11:31.271 | INFO | "huge" : "", | 20260506 13:11:31.271 | INFO | "huge_shared" : " 27 GB", | 20260506 13:11:31.271 | INFO | "malloc" : " 14 GB", | 20260506 13:11:31.271 | INFO | "pinned" : " 29 GB" | 20260506 13:11:31.271 | INFO | } | 20260506 13:11:31.272 | INFO | Perf report for the last 5.13892 seconds: | 20260506 13:11:31.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:11:31.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:31.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:31.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:31.272 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 13:11:31.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:31.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:31.272 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 13:11:31.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:31.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:31.272 | INFO | Uploaded 36GiB to GPU in 2.05758s (17.4963GiB/s) | 20260506 13:11:31.272 | INFO | Downloaded 34GiB to GPU in 1.99719s (17.0239GiB/s) | 20260506 13:11:31.272 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:11:31.272 | INFO | Total mem usage: 16.8 GiB | 20260506 13:11:31.272 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 13:11:31.272 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -13.7%) | 20260506 13:11:31.272 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:11:31.272 | INFO | High Water Breakdown: | 20260506 13:11:31.272 | INFO | 60.1% : 8.89 GiB ( 85.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:11:31.272 | INFO | 39.7% : 5.87 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:11:31.272 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:11:31.272 | INFO | ------------------------------------------ | 20260506 13:11:31.272 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:11:31.272 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:11:31.272 | INFO | Total mem usage: 26.5 GiB | 20260506 13:11:31.272 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:11:31.272 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:11:31.272 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:11:31.272 | INFO | High Water Breakdown: | 20260506 13:11:31.272 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:11:31.272 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:11:31.272 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:11:31.272 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:11:31.272 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:11:31.272 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:11:31.272 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:11:31.272 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:11:31.272 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:11:31.272 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:11:31.272 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:11:31.272 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:11:31.272 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:11:31.272 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:11:31.272 | INFO | ------------------------------------------ | 20260506 13:11:31.272 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:11:31.272 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:11:31.272 | INFO | Total mem usage: 25.6 GiB | 20260506 13:11:31.272 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:11:31.272 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:11:31.272 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:11:31.272 | INFO | High Water Breakdown: | 20260506 13:11:31.272 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:11:31.272 | INFO | ------------------------------------------ | 20260506 13:11:31.272 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:11:31.272 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:11:31.272 | INFO | Total mem usage: 15.7 GiB | 20260506 13:11:31.272 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:11:31.272 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:11:31.272 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:11:31.272 | INFO | High Water Breakdown: | 20260506 13:11:31.272 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:11:31.272 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:11:31.272 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:11:31.272 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:11:31.272 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:11:31.272 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:11:31.272 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:11:31.272 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:11:31.272 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:11:31.272 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:11:31.272 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:11:31.272 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:11:31.272 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:11:31.272 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:11:31.272 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:11:31.272 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:11:31.272 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:11:31.272 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:11:31.272 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:11:31.272 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:11:31.272 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:11:31.272 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:11:31.273 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:11:31.273 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:11:31.273 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:11:31.273 | INFO | ------------------------------------------ | 20260506 13:11:31.273 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:11:31.273 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:11:31.273 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:11:31.273 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:11:31.273 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.65e+01, 1.35e+01 / 2.50e+00 | 20260506 13:11:31.273 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.01e-01, 1.05e-02 / 6.82e-03 | 20260506 13:11:31.273 | INFO | PerfReport: 11.54%, Baseline -- 3.66e+00 / 4.43e+00, 5.69e-01 / 2.96e-01 | 20260506 13:11:31.273 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.46e+00, 1.88e-01 / 1.39e-01 | 20260506 13:11:31.273 | INFO | PerfReport: 15.87%, DME -- 4.02e+01 / 4.50e+01, 4.21e+00 / 3.00e+00 | 20260506 13:11:31.273 | INFO | PerfReport: 56.58%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.46e+00 / 1.76e+00 | 20260506 13:11:31.273 | INFO | PerfReport: 7.50%, PulseAccumulating -- 2.37e+00 / 2.97e+00, 4.10e-01 / 2.75e-01 | 20260506 13:11:31.273 | INFO | PerfReport: 3.68%, Metrics -- 1.17e+00 / 1.55e+00, 2.97e-01 / 1.43e-01 | 20260506 13:11:31.273 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.11e-01 / 4.73e-01, 6.44e-02 / 3.35e-02 | 20260506 13:11:31.273 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:11:31.273 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:11:31.273 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:11:31.273 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 13:11:31.273 | INFO | PerfReport: 21.54%, Upload -- 2.12e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260506 13:11:31.273 | INFO | PerfReport: 21.12%, Download -- 2.08e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 13:11:31.273 | INFO | PerfReport: 32.21%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 13:11:31.273 | INFO | PerfReport: 1.75%, LockUpload -- 1.72e+00 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 13:11:31.273 | INFO | PerfReport: 1.06%, LockDownload -- 1.04e+00 / 0.00e+00, 2.69e+00 / 0.00e+00 | 20260506 13:11:31.273 | INFO | PerfReport: 22.31%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 13:11:31.273 | INFO | PerfReport: 0.02%, Misc -- 1.57e-02 / 0.00e+00, 2.73e-02 / 0.00e+00 | 20260506 13:11:31.402 | INFO | Starting a round of Baz IO | 20260506 13:11:34.570 | INFO | Populating RTMetrics report | 20260506 13:11:34.570 | INFO | RTMetrics report populated | 20260506 13:11:34.570 | INFO | Opening tmp RTMetrics json file | 20260506 13:11:34.571 | INFO | Serializing RTMetrics to json | 20260506 13:11:34.573 | INFO | Serialization done, swapping tmp file now... | 20260506 13:11:34.573 | INFO | RTMetrics done | 20260506 13:11:34.714 | INFO | Finished a round of Baz IO: Wrote 2.107 GiB in 3.312 seconds | 20260506 13:11:34.714 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.14% / 0.02% / 59.84% | 20260506 13:11:34.715 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.35498% / 3.17891e-05% / 5.96046e-05% | 20260506 13:11:34.715 | INFO | Baz IO overall: | 20260506 13:11:34.715 | INFO | BytesWritten : 815559043143 | 20260506 13:11:34.715 | INFO | NumEvents : 394392978321 | 20260506 13:11:36.411 | INFO | Analyzing chunk frames = [1613824,1614336) | 20260506 13:11:36.411 | INFO | MemInfo: { | 20260506 13:11:36.411 | INFO | "gpu" : " 17 GB", | 20260506 13:11:36.411 | INFO | "huge" : "", | 20260506 13:11:36.411 | INFO | "huge_shared" : " 27 GB", | 20260506 13:11:36.411 | INFO | "malloc" : " 14 GB", | 20260506 13:11:36.411 | INFO | "pinned" : " 29 GB" | 20260506 13:11:36.411 | INFO | } | 20260506 13:11:36.411 | INFO | Perf report for the last 5.13954 seconds: | 20260506 13:11:36.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:11:36.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:36.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:36.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:36.411 | INFO | ANALYSIS : 63.6%, 2.9, 97.0ms, 0 | 20260506 13:11:36.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:36.411 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:11:36.411 | INFO | PRE_HQ : 61.3%, 1.9, 60.3ms, 0 | 20260506 13:11:36.411 | INFO | BAZWRITER : 63.9%, 4.3, 140.6ms, 0 | 20260506 13:11:36.411 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 13:11:36.411 | INFO | Uploaded 36GiB to GPU in 2.08749s (17.2456GiB/s) | 20260506 13:11:36.411 | INFO | Downloaded 38GiB to GPU in 2.24502s (16.9264GiB/s) | 20260506 13:11:41.550 | INFO | Analyzing chunk frames = [1614336,1614848) | 20260506 13:11:41.550 | INFO | MemInfo: { | 20260506 13:11:41.550 | INFO | "gpu" : " 17 GB", | 20260506 13:11:41.550 | INFO | "huge" : "", | 20260506 13:11:41.550 | INFO | "huge_shared" : " 27 GB", | 20260506 13:11:41.550 | INFO | "malloc" : " 14 GB", | 20260506 13:11:41.550 | INFO | "pinned" : " 29 GB" | 20260506 13:11:41.550 | INFO | } | 20260506 13:11:41.550 | INFO | Perf report for the last 5.13908 seconds: | 20260506 13:11:41.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:11:41.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:41.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:41.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:41.550 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:11:41.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:41.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:41.550 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:11:41.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:41.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:41.550 | INFO | Uploaded 36GiB to GPU in 2.00962s (17.9138GiB/s) | 20260506 13:11:41.550 | INFO | Downloaded 34GiB to GPU in 1.92343s (17.6767GiB/s) | 20260506 13:11:46.689 | INFO | Analyzing chunk frames = [1614848,1615360) | 20260506 13:11:46.689 | INFO | MemInfo: { | 20260506 13:11:46.689 | INFO | "gpu" : " 17 GB", | 20260506 13:11:46.689 | INFO | "huge" : "", | 20260506 13:11:46.689 | INFO | "huge_shared" : " 27 GB", | 20260506 13:11:46.689 | INFO | "malloc" : " 14 GB", | 20260506 13:11:46.690 | INFO | "pinned" : " 29 GB" | 20260506 13:11:46.690 | INFO | } | 20260506 13:11:46.690 | INFO | Perf report for the last 5.13936 seconds: | 20260506 13:11:46.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:11:46.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:46.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:46.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:46.690 | INFO | ANALYSIS : 61.4%, 3.0, 95.0ms, 0 | 20260506 13:11:46.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:46.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:46.690 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:11:46.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:46.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:46.690 | INFO | Uploaded 36GiB to GPU in 1.97967s (18.1849GiB/s) | 20260506 13:11:46.690 | INFO | Downloaded 34GiB to GPU in 1.93043s (17.6126GiB/s) | 20260506 13:11:51.829 | INFO | Analyzing chunk frames = [1615360,1615872) | 20260506 13:11:51.829 | INFO | MemInfo: { | 20260506 13:11:51.829 | INFO | "gpu" : " 17 GB", | 20260506 13:11:51.829 | INFO | "huge" : "", | 20260506 13:11:51.829 | INFO | "huge_shared" : " 27 GB", | 20260506 13:11:51.829 | INFO | "malloc" : " 14 GB", | 20260506 13:11:51.829 | INFO | "pinned" : " 29 GB" | 20260506 13:11:51.829 | INFO | } | 20260506 13:11:51.829 | INFO | Perf report for the last 5.13929 seconds: | 20260506 13:11:51.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:11:51.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:51.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:51.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:51.829 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:11:51.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:51.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:51.829 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:11:51.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:51.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:51.829 | INFO | Uploaded 36GiB to GPU in 2.03274s (17.7101GiB/s) | 20260506 13:11:51.829 | INFO | Downloaded 34GiB to GPU in 1.91688s (17.7372GiB/s) | 20260506 13:11:56.968 | INFO | Analyzing chunk frames = [1615872,1616384) | 20260506 13:11:56.968 | INFO | MemInfo: { | 20260506 13:11:56.968 | INFO | "gpu" : " 17 GB", | 20260506 13:11:56.968 | INFO | "huge" : "", | 20260506 13:11:56.968 | INFO | "huge_shared" : " 27 GB", | 20260506 13:11:56.968 | INFO | "malloc" : " 14 GB", | 20260506 13:11:56.968 | INFO | "pinned" : " 29 GB" | 20260506 13:11:56.968 | INFO | } | 20260506 13:11:56.968 | INFO | Perf report for the last 5.13919 seconds: | 20260506 13:11:56.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:11:56.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:56.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:56.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:56.968 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 13:11:56.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:56.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:56.968 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:11:56.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:56.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:11:56.968 | INFO | Uploaded 36GiB to GPU in 2.03364s (17.7023GiB/s) | 20260506 13:11:56.968 | INFO | Downloaded 34GiB to GPU in 1.94873s (17.4473GiB/s) | 20260506 13:12:02.107 | INFO | Analyzing chunk frames = [1616384,1616896) | 20260506 13:12:02.107 | INFO | MemInfo: { | 20260506 13:12:02.107 | INFO | "gpu" : " 17 GB", | 20260506 13:12:02.107 | INFO | "huge" : "", | 20260506 13:12:02.107 | INFO | "huge_shared" : " 27 GB", | 20260506 13:12:02.107 | INFO | "malloc" : " 14 GB", | 20260506 13:12:02.108 | INFO | "pinned" : " 29 GB" | 20260506 13:12:02.108 | INFO | } | 20260506 13:12:02.108 | INFO | Perf report for the last 5.13947 seconds: | 20260506 13:12:02.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:12:02.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:02.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:02.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:02.108 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 13:12:02.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:02.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:02.108 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260506 13:12:02.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:02.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:02.108 | INFO | Uploaded 36GiB to GPU in 2.05264s (17.5384GiB/s) | 20260506 13:12:02.108 | INFO | Downloaded 34GiB to GPU in 1.92482s (17.6639GiB/s) | 20260506 13:12:07.247 | INFO | Analyzing chunk frames = [1616896,1617408) | 20260506 13:12:07.247 | INFO | MemInfo: { | 20260506 13:12:07.247 | INFO | "gpu" : " 17 GB", | 20260506 13:12:07.247 | INFO | "huge" : "", | 20260506 13:12:07.247 | INFO | "huge_shared" : " 27 GB", | 20260506 13:12:07.247 | INFO | "malloc" : " 14 GB", | 20260506 13:12:07.247 | INFO | "pinned" : " 29 GB" | 20260506 13:12:07.247 | INFO | } | 20260506 13:12:07.247 | INFO | Perf report for the last 5.13946 seconds: | 20260506 13:12:07.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:12:07.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:07.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:07.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:07.247 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:12:07.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:07.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:07.247 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 13:12:07.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:07.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:07.247 | INFO | Uploaded 36GiB to GPU in 2.01155s (17.8966GiB/s) | 20260506 13:12:07.247 | INFO | Downloaded 34GiB to GPU in 2.023s (16.8067GiB/s) | 20260506 13:12:12.386 | INFO | Analyzing chunk frames = [1617408,1617920) | 20260506 13:12:12.386 | INFO | MemInfo: { | 20260506 13:12:12.386 | INFO | "gpu" : " 17 GB", | 20260506 13:12:12.386 | INFO | "huge" : "", | 20260506 13:12:12.386 | INFO | "huge_shared" : " 27 GB", | 20260506 13:12:12.386 | INFO | "malloc" : " 14 GB", | 20260506 13:12:12.386 | INFO | "pinned" : " 29 GB" | 20260506 13:12:12.386 | INFO | } | 20260506 13:12:12.386 | INFO | Perf report for the last 5.13901 seconds: | 20260506 13:12:12.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:12:12.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:12.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:12.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:12.386 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 13:12:12.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:12.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:12.386 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 13:12:12.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:12.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:12.386 | INFO | Uploaded 36GiB to GPU in 2.05367s (17.5296GiB/s) | 20260506 13:12:12.386 | INFO | Downloaded 34GiB to GPU in 1.96991s (17.2597GiB/s) | 20260506 13:12:12.517 | INFO | Starting a round of Baz IO | 20260506 13:12:15.679 | INFO | Populating RTMetrics report | 20260506 13:12:15.680 | INFO | RTMetrics report populated | 20260506 13:12:15.680 | INFO | Opening tmp RTMetrics json file | 20260506 13:12:15.680 | INFO | Serializing RTMetrics to json | 20260506 13:12:15.682 | INFO | Serialization done, swapping tmp file now... | 20260506 13:12:15.682 | INFO | RTMetrics done | 20260506 13:12:15.791 | INFO | Finished a round of Baz IO: Wrote 2.108 GiB in 3.273 seconds | 20260506 13:12:15.791 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.12% / 0.02% / 59.86% | 20260506 13:12:15.791 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.34427% / 2.38419e-05% / 4.37101e-05% | 20260506 13:12:15.791 | INFO | Baz IO overall: | 20260506 13:12:15.791 | INFO | BytesWritten : 817822198763 | 20260506 13:12:15.791 | INFO | NumEvents : 395491013883 | 20260506 13:12:17.525 | INFO | Analyzing chunk frames = [1617920,1618432) | 20260506 13:12:17.525 | INFO | MemInfo: { | 20260506 13:12:17.525 | INFO | "gpu" : " 17 GB", | 20260506 13:12:17.525 | INFO | "huge" : "", | 20260506 13:12:17.525 | INFO | "huge_shared" : " 27 GB", | 20260506 13:12:17.525 | INFO | "malloc" : " 14 GB", | 20260506 13:12:17.525 | INFO | "pinned" : " 29 GB" | 20260506 13:12:17.525 | INFO | } | 20260506 13:12:17.526 | INFO | Perf report for the last 5.13945 seconds: | 20260506 13:12:17.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:12:17.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:17.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:17.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:17.526 | INFO | ANALYSIS : 63.5%, 2.9, 96.1ms, 0 | 20260506 13:12:17.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:17.526 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 13:12:17.526 | INFO | PRE_HQ : 61.5%, 1.9, 60.7ms, 0 | 20260506 13:12:17.526 | INFO | BAZWRITER : 62.9%, 4.7, 151.5ms, 0 | 20260506 13:12:17.526 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 13:12:17.526 | INFO | Uploaded 36GiB to GPU in 2.09982s (17.1443GiB/s) | 20260506 13:12:17.526 | INFO | Downloaded 38GiB to GPU in 2.24252s (16.9452GiB/s) | 20260506 13:12:22.665 | INFO | Analyzing chunk frames = [1618432,1618944) | 20260506 13:12:22.665 | INFO | MemInfo: { | 20260506 13:12:22.665 | INFO | "gpu" : " 17 GB", | 20260506 13:12:22.665 | INFO | "huge" : "", | 20260506 13:12:22.665 | INFO | "huge_shared" : " 27 GB", | 20260506 13:12:22.665 | INFO | "malloc" : " 14 GB", | 20260506 13:12:22.665 | INFO | "pinned" : " 29 GB" | 20260506 13:12:22.665 | INFO | } | 20260506 13:12:22.665 | INFO | Perf report for the last 5.13921 seconds: | 20260506 13:12:22.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:12:22.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:22.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:22.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:22.665 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:12:22.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:22.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:22.665 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:12:22.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:22.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:22.665 | INFO | Uploaded 36GiB to GPU in 2.01572s (17.8597GiB/s) | 20260506 13:12:22.665 | INFO | Downloaded 34GiB to GPU in 1.92778s (17.6368GiB/s) | 20260506 13:12:27.804 | INFO | Analyzing chunk frames = [1618944,1619456) | 20260506 13:12:27.804 | INFO | MemInfo: { | 20260506 13:12:27.804 | INFO | "gpu" : " 17 GB", | 20260506 13:12:27.804 | INFO | "huge" : "", | 20260506 13:12:27.804 | INFO | "huge_shared" : " 27 GB", | 20260506 13:12:27.804 | INFO | "malloc" : " 14 GB", | 20260506 13:12:27.804 | INFO | "pinned" : " 29 GB" | 20260506 13:12:27.804 | INFO | } | 20260506 13:12:27.804 | INFO | Perf report for the last 5.13938 seconds: | 20260506 13:12:27.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:12:27.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:27.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:27.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:27.804 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 13:12:27.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:27.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:27.804 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:12:27.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:27.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:27.804 | INFO | Uploaded 36GiB to GPU in 1.99103s (18.0811GiB/s) | 20260506 13:12:27.804 | INFO | Downloaded 34GiB to GPU in 1.93816s (17.5424GiB/s) | 20260506 13:12:32.944 | INFO | Analyzing chunk frames = [1619456,1619968) | 20260506 13:12:32.944 | INFO | MemInfo: { | 20260506 13:12:32.944 | INFO | "gpu" : " 17 GB", | 20260506 13:12:32.944 | INFO | "huge" : "", | 20260506 13:12:32.944 | INFO | "huge_shared" : " 27 GB", | 20260506 13:12:32.944 | INFO | "malloc" : " 14 GB", | 20260506 13:12:32.944 | INFO | "pinned" : " 29 GB" | 20260506 13:12:32.944 | INFO | } | 20260506 13:12:32.944 | INFO | Perf report for the last 5.13956 seconds: | 20260506 13:12:32.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:12:32.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:32.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:32.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:32.944 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:12:32.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:32.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:32.944 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 13:12:32.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:32.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:32.944 | INFO | Uploaded 36GiB to GPU in 1.98675s (18.12GiB/s) | 20260506 13:12:32.944 | INFO | Downloaded 34GiB to GPU in 1.94339s (17.4952GiB/s) | 20260506 13:12:38.083 | INFO | Analyzing chunk frames = [1619968,1620480) | 20260506 13:12:38.083 | INFO | MemInfo: { | 20260506 13:12:38.083 | INFO | "gpu" : " 17 GB", | 20260506 13:12:38.083 | INFO | "huge" : "", | 20260506 13:12:38.083 | INFO | "huge_shared" : " 27 GB", | 20260506 13:12:38.083 | INFO | "malloc" : " 14 GB", | 20260506 13:12:38.083 | INFO | "pinned" : " 29 GB" | 20260506 13:12:38.083 | INFO | } | 20260506 13:12:38.083 | INFO | Perf report for the last 5.1392 seconds: | 20260506 13:12:38.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:12:38.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:38.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:38.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:38.083 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 13:12:38.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:38.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:38.083 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:12:38.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:38.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:38.083 | INFO | Uploaded 36GiB to GPU in 2.0285s (17.7471GiB/s) | 20260506 13:12:38.083 | INFO | Downloaded 34GiB to GPU in 1.95081s (17.4286GiB/s) | 20260506 13:12:43.222 | INFO | Analyzing chunk frames = [1620480,1620992) | 20260506 13:12:43.222 | INFO | MemInfo: { | 20260506 13:12:43.222 | INFO | "gpu" : " 17 GB", | 20260506 13:12:43.222 | INFO | "huge" : "", | 20260506 13:12:43.222 | INFO | "huge_shared" : " 27 GB", | 20260506 13:12:43.222 | INFO | "malloc" : " 14 GB", | 20260506 13:12:43.222 | INFO | "pinned" : " 29 GB" | 20260506 13:12:43.222 | INFO | } | 20260506 13:12:43.222 | INFO | Perf report for the last 5.13947 seconds: | 20260506 13:12:43.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:12:43.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:43.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:43.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:43.222 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 13:12:43.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:43.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:43.222 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260506 13:12:43.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:43.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:43.223 | INFO | Uploaded 36GiB to GPU in 2.03997s (17.6473GiB/s) | 20260506 13:12:43.223 | INFO | Downloaded 34GiB to GPU in 1.93152s (17.6027GiB/s) | 20260506 13:12:48.362 | INFO | Analyzing chunk frames = [1620992,1621504) | 20260506 13:12:48.362 | INFO | MemInfo: { | 20260506 13:12:48.362 | INFO | "gpu" : " 17 GB", | 20260506 13:12:48.362 | INFO | "huge" : "", | 20260506 13:12:48.362 | INFO | "huge_shared" : " 27 GB", | 20260506 13:12:48.362 | INFO | "malloc" : " 14 GB", | 20260506 13:12:48.362 | INFO | "pinned" : " 29 GB" | 20260506 13:12:48.362 | INFO | } | 20260506 13:12:48.362 | INFO | Perf report for the last 5.13982 seconds: | 20260506 13:12:48.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:12:48.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:48.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:48.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:48.362 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:12:48.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:48.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:48.362 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 13:12:48.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:48.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:48.363 | INFO | Uploaded 36GiB to GPU in 2.0117s (17.8953GiB/s) | 20260506 13:12:48.363 | INFO | Downloaded 34GiB to GPU in 1.96789s (17.2774GiB/s) | 20260506 13:12:53.501 | INFO | Analyzing chunk frames = [1621504,1622016) | 20260506 13:12:53.501 | INFO | MemInfo: { | 20260506 13:12:53.501 | INFO | "gpu" : " 17 GB", | 20260506 13:12:53.501 | INFO | "huge" : "", | 20260506 13:12:53.501 | INFO | "huge_shared" : " 27 GB", | 20260506 13:12:53.501 | INFO | "malloc" : " 14 GB", | 20260506 13:12:53.501 | INFO | "pinned" : " 29 GB" | 20260506 13:12:53.501 | INFO | } | 20260506 13:12:53.501 | INFO | Perf report for the last 5.13866 seconds: | 20260506 13:12:53.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:12:53.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:53.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:53.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:53.501 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 13:12:53.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:53.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:53.501 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 13:12:53.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:53.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:53.501 | INFO | Uploaded 36GiB to GPU in 2.06611s (17.424GiB/s) | 20260506 13:12:53.501 | INFO | Downloaded 34GiB to GPU in 1.96935s (17.2646GiB/s) | 20260506 13:12:53.501 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:12:53.501 | INFO | Total mem usage: 16.8 GiB | 20260506 13:12:53.501 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 13:12:53.501 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -11.2%) | 20260506 13:12:53.501 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:12:53.501 | INFO | High Water Breakdown: | 20260506 13:12:53.501 | INFO | 60.5% : 9.14 GiB ( 88% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:12:53.501 | INFO | 39.4% : 5.95 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:12:53.501 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:12:53.501 | INFO | ------------------------------------------ | 20260506 13:12:53.501 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:12:53.501 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:12:53.501 | INFO | Total mem usage: 26.5 GiB | 20260506 13:12:53.501 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:12:53.501 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:12:53.501 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:12:53.501 | INFO | High Water Breakdown: | 20260506 13:12:53.501 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:12:53.501 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:12:53.501 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:12:53.501 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:12:53.501 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:12:53.501 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:12:53.501 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:12:53.501 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:12:53.501 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:12:53.501 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:12:53.501 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:12:53.501 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:12:53.501 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:12:53.502 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:12:53.502 | INFO | ------------------------------------------ | 20260506 13:12:53.502 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:12:53.502 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:12:53.502 | INFO | Total mem usage: 25.6 GiB | 20260506 13:12:53.502 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:12:53.502 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:12:53.502 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:12:53.502 | INFO | High Water Breakdown: | 20260506 13:12:53.502 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:12:53.502 | INFO | ------------------------------------------ | 20260506 13:12:53.502 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:12:53.502 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:12:53.502 | INFO | Total mem usage: 15.7 GiB | 20260506 13:12:53.502 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:12:53.502 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:12:53.502 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 13:12:53.502 | INFO | High Water Breakdown: | 20260506 13:12:53.502 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:12:53.502 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:12:53.502 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:12:53.502 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:12:53.502 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:12:53.502 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:12:53.502 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:12:53.502 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:12:53.502 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:12:53.502 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:12:53.502 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:12:53.502 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:12:53.502 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:12:53.502 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:12:53.502 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:12:53.502 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:12:53.502 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:12:53.502 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:12:53.502 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:12:53.502 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:12:53.502 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:12:53.502 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:12:53.502 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:12:53.502 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:12:53.502 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:12:53.502 | INFO | ------------------------------------------ | 20260506 13:12:53.502 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:12:53.502 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:12:53.502 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:12:53.502 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:12:53.502 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.60e+01, 1.36e+01 / 1.97e+00 | 20260506 13:12:53.502 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.03e-01, 1.02e-02 / 7.68e-03 | 20260506 13:12:53.502 | INFO | PerfReport: 11.56%, Baseline -- 3.67e+00 / 4.37e+00, 5.81e-01 / 4.19e-01 | 20260506 13:12:53.502 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.46e+00, 1.91e-01 / 1.48e-01 | 20260506 13:12:53.502 | INFO | PerfReport: 15.77%, DME -- 4.00e+01 / 4.40e+01, 3.65e+00 / 2.39e+00 | 20260506 13:12:53.502 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.79e+01 / 2.11e+01, 1.52e+00 / 1.80e+00 | 20260506 13:12:53.502 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.38e+00 / 3.00e+00, 4.08e-01 / 2.52e-01 | 20260506 13:12:53.502 | INFO | PerfReport: 3.70%, Metrics -- 1.17e+00 / 1.55e+00, 3.01e-01 / 1.70e-01 | 20260506 13:12:53.502 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.12e-01 / 4.69e-01, 6.36e-02 / 2.82e-02 | 20260506 13:12:53.502 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:12:53.502 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:12:53.502 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:12:53.502 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 13:12:53.502 | INFO | PerfReport: 21.52%, Upload -- 2.12e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 13:12:53.502 | INFO | PerfReport: 21.02%, Download -- 2.07e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 13:12:53.502 | INFO | PerfReport: 32.25%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 13:12:53.502 | INFO | PerfReport: 1.72%, LockUpload -- 1.70e+00 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260506 13:12:53.502 | INFO | PerfReport: 1.10%, LockDownload -- 1.08e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260506 13:12:53.502 | INFO | PerfReport: 22.37%, LockCompute -- 2.20e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 13:12:53.502 | INFO | PerfReport: 0.02%, Misc -- 1.52e-02 / 0.00e+00, 2.24e-02 / 0.00e+00 | 20260506 13:12:53.632 | INFO | Starting a round of Baz IO | 20260506 13:12:56.789 | INFO | Populating RTMetrics report | 20260506 13:12:56.789 | INFO | RTMetrics report populated | 20260506 13:12:56.789 | INFO | Opening tmp RTMetrics json file | 20260506 13:12:56.796 | INFO | Serializing RTMetrics to json | 20260506 13:12:56.799 | INFO | Serialization done, swapping tmp file now... | 20260506 13:12:56.802 | INFO | RTMetrics done | 20260506 13:12:56.981 | INFO | Finished a round of Baz IO: Wrote 2.124 GiB in 3.349 seconds | 20260506 13:12:56.982 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.10% / 0.02% / 59.88% | 20260506 13:12:56.982 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.38493% / 2.38419e-05% / 5.96046e-05% | 20260506 13:12:56.982 | INFO | Baz IO overall: | 20260506 13:12:56.982 | INFO | BytesWritten : 820102440585 | 20260506 13:12:56.982 | INFO | NumEvents : 396595694780 | 20260506 13:12:58.640 | INFO | Analyzing chunk frames = [1622016,1622528) | 20260506 13:12:58.640 | INFO | MemInfo: { | 20260506 13:12:58.640 | INFO | "gpu" : " 17 GB", | 20260506 13:12:58.640 | INFO | "huge" : "", | 20260506 13:12:58.640 | INFO | "huge_shared" : " 27 GB", | 20260506 13:12:58.640 | INFO | "malloc" : " 14 GB", | 20260506 13:12:58.640 | INFO | "pinned" : " 29 GB" | 20260506 13:12:58.640 | INFO | } | 20260506 13:12:58.640 | INFO | Perf report for the last 5.13917 seconds: | 20260506 13:12:58.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:12:58.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:58.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:58.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:58.640 | INFO | ANALYSIS : 63.4%, 2.9, 95.5ms, 0 | 20260506 13:12:58.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:12:58.640 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 13:12:58.640 | INFO | PRE_HQ : 61.3%, 1.9, 60.4ms, 0 | 20260506 13:12:58.640 | INFO | BAZWRITER : 64.6%, 4.8, 159.6ms, 0 | 20260506 13:12:58.640 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 13:12:58.641 | INFO | Uploaded 36GiB to GPU in 2.11379s (17.031GiB/s) | 20260506 13:12:58.641 | INFO | Downloaded 38GiB to GPU in 2.24904s (16.8961GiB/s) | 20260506 13:13:03.779 | INFO | Analyzing chunk frames = [1622528,1623040) | 20260506 13:13:03.779 | INFO | MemInfo: { | 20260506 13:13:03.779 | INFO | "gpu" : " 17 GB", | 20260506 13:13:03.779 | INFO | "huge" : "", | 20260506 13:13:03.780 | INFO | "huge_shared" : " 27 GB", | 20260506 13:13:03.780 | INFO | "malloc" : " 14 GB", | 20260506 13:13:03.780 | INFO | "pinned" : " 29 GB" | 20260506 13:13:03.780 | INFO | } | 20260506 13:13:03.780 | INFO | Perf report for the last 5.13952 seconds: | 20260506 13:13:03.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:13:03.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:03.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:03.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:03.780 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 13:13:03.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:03.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:03.780 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:13:03.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:03.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:03.780 | INFO | Uploaded 36GiB to GPU in 2.03182s (17.7181GiB/s) | 20260506 13:13:03.780 | INFO | Downloaded 34GiB to GPU in 1.92106s (17.6985GiB/s) | 20260506 13:13:08.919 | INFO | Analyzing chunk frames = [1623040,1623552) | 20260506 13:13:08.919 | INFO | MemInfo: { | 20260506 13:13:08.919 | INFO | "gpu" : " 17 GB", | 20260506 13:13:08.919 | INFO | "huge" : "", | 20260506 13:13:08.919 | INFO | "huge_shared" : " 27 GB", | 20260506 13:13:08.919 | INFO | "malloc" : " 14 GB", | 20260506 13:13:08.919 | INFO | "pinned" : " 29 GB" | 20260506 13:13:08.919 | INFO | } | 20260506 13:13:08.919 | INFO | Perf report for the last 5.13983 seconds: | 20260506 13:13:08.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:13:08.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:08.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:08.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:08.919 | INFO | ANALYSIS : 61.5%, 3.0, 95.1ms, 0 | 20260506 13:13:08.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:08.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:08.920 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:13:08.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:08.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:08.920 | INFO | Uploaded 36GiB to GPU in 1.99453s (18.0494GiB/s) | 20260506 13:13:08.920 | INFO | Downloaded 34GiB to GPU in 1.9328s (17.591GiB/s) | 20260506 13:13:14.059 | INFO | Analyzing chunk frames = [1623552,1624064) | 20260506 13:13:14.059 | INFO | MemInfo: { | 20260506 13:13:14.059 | INFO | "gpu" : " 17 GB", | 20260506 13:13:14.059 | INFO | "huge" : "", | 20260506 13:13:14.059 | INFO | "huge_shared" : " 27 GB", | 20260506 13:13:14.059 | INFO | "malloc" : " 14 GB", | 20260506 13:13:14.059 | INFO | "pinned" : " 29 GB" | 20260506 13:13:14.059 | INFO | } | 20260506 13:13:14.059 | INFO | Perf report for the last 5.13925 seconds: | 20260506 13:13:14.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:13:14.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:14.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:14.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:14.059 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:13:14.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:14.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:14.059 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 13:13:14.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:14.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:14.059 | INFO | Uploaded 36GiB to GPU in 1.97377s (18.2392GiB/s) | 20260506 13:13:14.059 | INFO | Downloaded 34GiB to GPU in 1.91471s (17.7573GiB/s) | 20260506 13:13:19.197 | INFO | Analyzing chunk frames = [1624064,1624576) | 20260506 13:13:19.197 | INFO | MemInfo: { | 20260506 13:13:19.197 | INFO | "gpu" : " 17 GB", | 20260506 13:13:19.197 | INFO | "huge" : "", | 20260506 13:13:19.197 | INFO | "huge_shared" : " 27 GB", | 20260506 13:13:19.197 | INFO | "malloc" : " 14 GB", | 20260506 13:13:19.197 | INFO | "pinned" : " 29 GB" | 20260506 13:13:19.197 | INFO | } | 20260506 13:13:19.197 | INFO | Perf report for the last 5.13873 seconds: | 20260506 13:13:19.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:13:19.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:19.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:19.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:19.197 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 13:13:19.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:19.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:19.198 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 13:13:19.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:19.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:19.198 | INFO | Uploaded 36GiB to GPU in 2.01634s (17.8542GiB/s) | 20260506 13:13:19.198 | INFO | Downloaded 34GiB to GPU in 1.94834s (17.4507GiB/s) | 20260506 13:13:24.337 | INFO | Analyzing chunk frames = [1624576,1625088) | 20260506 13:13:24.337 | INFO | MemInfo: { | 20260506 13:13:24.337 | INFO | "gpu" : " 17 GB", | 20260506 13:13:24.337 | INFO | "huge" : "", | 20260506 13:13:24.337 | INFO | "huge_shared" : " 27 GB", | 20260506 13:13:24.337 | INFO | "malloc" : " 14 GB", | 20260506 13:13:24.337 | INFO | "pinned" : " 29 GB" | 20260506 13:13:24.337 | INFO | } | 20260506 13:13:24.337 | INFO | Perf report for the last 5.13969 seconds: | 20260506 13:13:24.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:13:24.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:24.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:24.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:24.337 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:13:24.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:24.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:24.337 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:13:24.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:24.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:24.337 | INFO | Uploaded 36GiB to GPU in 2.09323s (17.1983GiB/s) | 20260506 13:13:24.337 | INFO | Downloaded 34GiB to GPU in 1.96362s (17.3149GiB/s) | 20260506 13:13:29.477 | INFO | Analyzing chunk frames = [1625088,1625600) | 20260506 13:13:29.477 | INFO | MemInfo: { | 20260506 13:13:29.477 | INFO | "gpu" : " 17 GB", | 20260506 13:13:29.477 | INFO | "huge" : "", | 20260506 13:13:29.477 | INFO | "huge_shared" : " 27 GB", | 20260506 13:13:29.477 | INFO | "malloc" : " 14 GB", | 20260506 13:13:29.477 | INFO | "pinned" : " 29 GB" | 20260506 13:13:29.477 | INFO | } | 20260506 13:13:29.477 | INFO | Perf report for the last 5.13985 seconds: | 20260506 13:13:29.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:13:29.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:29.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:29.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:29.477 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:13:29.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:29.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:29.477 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 13:13:29.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:29.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:29.477 | INFO | Uploaded 36GiB to GPU in 1.98556s (18.1309GiB/s) | 20260506 13:13:29.477 | INFO | Downloaded 34GiB to GPU in 2.02997s (16.749GiB/s) | 20260506 13:13:34.615 | INFO | Analyzing chunk frames = [1625600,1626112) | 20260506 13:13:34.615 | INFO | MemInfo: { | 20260506 13:13:34.615 | INFO | "gpu" : " 17 GB", | 20260506 13:13:34.615 | INFO | "huge" : "", | 20260506 13:13:34.616 | INFO | "huge_shared" : " 27 GB", | 20260506 13:13:34.616 | INFO | "malloc" : " 14 GB", | 20260506 13:13:34.616 | INFO | "pinned" : " 29 GB" | 20260506 13:13:34.616 | INFO | } | 20260506 13:13:34.616 | INFO | Perf report for the last 5.13854 seconds: | 20260506 13:13:34.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:13:34.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:34.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:34.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:34.616 | INFO | ANALYSIS : 62.8%, 3.0, 97.7ms, 0 | 20260506 13:13:34.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:34.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:34.616 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:13:34.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:34.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:34.616 | INFO | Uploaded 36GiB to GPU in 2.07492s (17.3501GiB/s) | 20260506 13:13:34.616 | INFO | Downloaded 34GiB to GPU in 1.98114s (17.1618GiB/s) | 20260506 13:13:34.747 | INFO | Starting a round of Baz IO | 20260506 13:13:37.905 | INFO | Populating RTMetrics report | 20260506 13:13:37.906 | INFO | RTMetrics report populated | 20260506 13:13:37.906 | INFO | Opening tmp RTMetrics json file | 20260506 13:13:37.912 | INFO | Serializing RTMetrics to json | 20260506 13:13:37.934 | INFO | Serialization done, swapping tmp file now... | 20260506 13:13:37.960 | INFO | RTMetrics done | 20260506 13:13:38.078 | INFO | Finished a round of Baz IO: Wrote 2.118 GiB in 3.33 seconds | 20260506 13:13:38.078 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.08% / 0.02% / 59.90% | 20260506 13:13:38.078 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.36525% / 3.17891e-05% / 4.37101e-05% | 20260506 13:13:38.078 | INFO | Baz IO overall: | 20260506 13:13:38.078 | INFO | BytesWritten : 822376703906 | 20260506 13:13:38.078 | INFO | NumEvents : 397696890655 | 20260506 13:13:39.755 | INFO | Analyzing chunk frames = [1626112,1626624) | 20260506 13:13:39.755 | INFO | MemInfo: { | 20260506 13:13:39.755 | INFO | "gpu" : " 17 GB", | 20260506 13:13:39.755 | INFO | "huge" : "", | 20260506 13:13:39.755 | INFO | "huge_shared" : " 27 GB", | 20260506 13:13:39.755 | INFO | "malloc" : " 14 GB", | 20260506 13:13:39.755 | INFO | "pinned" : " 29 GB" | 20260506 13:13:39.755 | INFO | } | 20260506 13:13:39.755 | INFO | Perf report for the last 5.13955 seconds: | 20260506 13:13:39.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:13:39.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:39.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:39.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:39.755 | INFO | ANALYSIS : 63.4%, 2.9, 95.7ms, 0 | 20260506 13:13:39.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:39.755 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260506 13:13:39.755 | INFO | PRE_HQ : 61.5%, 1.9, 60.7ms, 0 | 20260506 13:13:39.755 | INFO | BAZWRITER : 64.2%, 4.8, 157.3ms, 0 | 20260506 13:13:39.755 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 13:13:39.755 | INFO | Uploaded 36GiB to GPU in 2.05922s (17.4823GiB/s) | 20260506 13:13:39.755 | INFO | Downloaded 38GiB to GPU in 2.23875s (16.9737GiB/s) | 20260506 13:13:44.894 | INFO | Analyzing chunk frames = [1626624,1627136) | 20260506 13:13:44.894 | INFO | MemInfo: { | 20260506 13:13:44.894 | INFO | "gpu" : " 17 GB", | 20260506 13:13:44.894 | INFO | "huge" : "", | 20260506 13:13:44.894 | INFO | "huge_shared" : " 27 GB", | 20260506 13:13:44.894 | INFO | "malloc" : " 14 GB", | 20260506 13:13:44.894 | INFO | "pinned" : " 29 GB" | 20260506 13:13:44.894 | INFO | } | 20260506 13:13:44.894 | INFO | Perf report for the last 5.13924 seconds: | 20260506 13:13:44.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:13:44.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:44.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:44.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:44.894 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 13:13:44.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:44.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:44.895 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:13:44.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:44.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:44.895 | INFO | Uploaded 36GiB to GPU in 2.00781s (17.93GiB/s) | 20260506 13:13:44.895 | INFO | Downloaded 34GiB to GPU in 1.93715s (17.5516GiB/s) | 20260506 13:13:50.034 | INFO | Analyzing chunk frames = [1627136,1627648) | 20260506 13:13:50.034 | INFO | MemInfo: { | 20260506 13:13:50.034 | INFO | "gpu" : " 17 GB", | 20260506 13:13:50.034 | INFO | "huge" : "", | 20260506 13:13:50.034 | INFO | "huge_shared" : " 27 GB", | 20260506 13:13:50.034 | INFO | "malloc" : " 14 GB", | 20260506 13:13:50.034 | INFO | "pinned" : " 29 GB" | 20260506 13:13:50.034 | INFO | } | 20260506 13:13:50.034 | INFO | Perf report for the last 5.13937 seconds: | 20260506 13:13:50.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:13:50.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:50.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:50.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:50.034 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 13:13:50.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:50.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:50.034 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:13:50.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:50.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:50.034 | INFO | Uploaded 36GiB to GPU in 2.00118s (17.9894GiB/s) | 20260506 13:13:50.034 | INFO | Downloaded 34GiB to GPU in 1.93725s (17.5507GiB/s) | 20260506 13:13:55.173 | INFO | Analyzing chunk frames = [1627648,1628160) | 20260506 13:13:55.173 | INFO | MemInfo: { | 20260506 13:13:55.173 | INFO | "gpu" : " 17 GB", | 20260506 13:13:55.173 | INFO | "huge" : "", | 20260506 13:13:55.173 | INFO | "huge_shared" : " 27 GB", | 20260506 13:13:55.173 | INFO | "malloc" : " 14 GB", | 20260506 13:13:55.173 | INFO | "pinned" : " 29 GB" | 20260506 13:13:55.173 | INFO | } | 20260506 13:13:55.173 | INFO | Perf report for the last 5.13952 seconds: | 20260506 13:13:55.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:13:55.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:55.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:55.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:55.173 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:13:55.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:55.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:55.173 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:13:55.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:55.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:13:55.174 | INFO | Uploaded 36GiB to GPU in 2.00491s (17.9559GiB/s) | 20260506 13:13:55.174 | INFO | Downloaded 34GiB to GPU in 1.94309s (17.4979GiB/s) | 20260506 13:14:00.312 | INFO | Analyzing chunk frames = [1628160,1628672) | 20260506 13:14:00.312 | INFO | MemInfo: { | 20260506 13:14:00.312 | INFO | "gpu" : " 17 GB", | 20260506 13:14:00.313 | INFO | "huge" : "", | 20260506 13:14:00.313 | INFO | "huge_shared" : " 27 GB", | 20260506 13:14:00.313 | INFO | "malloc" : " 14 GB", | 20260506 13:14:00.313 | INFO | "pinned" : " 29 GB" | 20260506 13:14:00.313 | INFO | } | 20260506 13:14:00.313 | INFO | Perf report for the last 5.1393 seconds: | 20260506 13:14:00.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:14:00.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:00.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:00.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:00.313 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 13:14:00.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:00.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:00.313 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:14:00.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:00.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:00.313 | INFO | Uploaded 36GiB to GPU in 1.97862s (18.1945GiB/s) | 20260506 13:14:00.313 | INFO | Downloaded 34GiB to GPU in 1.94283s (17.5003GiB/s) | 20260506 13:14:05.452 | INFO | Analyzing chunk frames = [1628672,1629184) | 20260506 13:14:05.452 | INFO | MemInfo: { | 20260506 13:14:05.452 | INFO | "gpu" : " 17 GB", | 20260506 13:14:05.452 | INFO | "huge" : "", | 20260506 13:14:05.452 | INFO | "huge_shared" : " 27 GB", | 20260506 13:14:05.452 | INFO | "malloc" : " 14 GB", | 20260506 13:14:05.452 | INFO | "pinned" : " 29 GB" | 20260506 13:14:05.452 | INFO | } | 20260506 13:14:05.452 | INFO | Perf report for the last 5.13935 seconds: | 20260506 13:14:05.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:14:05.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:05.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:05.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:05.452 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 13:14:05.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:05.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:05.452 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260506 13:14:05.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:05.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:05.452 | INFO | Uploaded 36GiB to GPU in 2.07036s (17.3883GiB/s) | 20260506 13:14:05.452 | INFO | Downloaded 34GiB to GPU in 1.96661s (17.2886GiB/s) | 20260506 13:14:10.591 | INFO | Analyzing chunk frames = [1629184,1629696) | 20260506 13:14:10.591 | INFO | MemInfo: { | 20260506 13:14:10.591 | INFO | "gpu" : " 17 GB", | 20260506 13:14:10.591 | INFO | "huge" : "", | 20260506 13:14:10.591 | INFO | "huge_shared" : " 27 GB", | 20260506 13:14:10.591 | INFO | "malloc" : " 14 GB", | 20260506 13:14:10.591 | INFO | "pinned" : " 29 GB" | 20260506 13:14:10.591 | INFO | } | 20260506 13:14:10.591 | INFO | Perf report for the last 5.13918 seconds: | 20260506 13:14:10.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:14:10.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:10.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:10.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:10.591 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:14:10.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:10.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:10.591 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:14:10.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:10.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:10.591 | INFO | Uploaded 36GiB to GPU in 1.97491s (18.2287GiB/s) | 20260506 13:14:10.591 | INFO | Downloaded 34GiB to GPU in 2.03278s (16.7258GiB/s) | 20260506 13:14:15.730 | INFO | Analyzing chunk frames = [1629696,1630208) | 20260506 13:14:15.730 | INFO | MemInfo: { | 20260506 13:14:15.730 | INFO | "gpu" : " 17 GB", | 20260506 13:14:15.730 | INFO | "huge" : "", | 20260506 13:14:15.730 | INFO | "huge_shared" : " 27 GB", | 20260506 13:14:15.730 | INFO | "malloc" : " 14 GB", | 20260506 13:14:15.730 | INFO | "pinned" : " 29 GB" | 20260506 13:14:15.730 | INFO | } | 20260506 13:14:15.730 | INFO | Perf report for the last 5.13931 seconds: | 20260506 13:14:15.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:14:15.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:15.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:15.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:15.731 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:14:15.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:15.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:15.731 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:14:15.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:15.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:15.731 | INFO | Uploaded 36GiB to GPU in 2.02898s (17.7429GiB/s) | 20260506 13:14:15.731 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:14:15.731 | INFO | Total mem usage: 16.8 GiB | 20260506 13:14:15.731 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 13:14:15.731 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -8.93%) | 20260506 13:14:15.731 | INFO | Downloaded 34GiB to GPU in 1.97526s (17.2129GiB/s) | 20260506 13:14:15.731 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:14:15.731 | INFO | High Water Breakdown: | 20260506 13:14:15.731 | INFO | 60.8% : 9.38 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:14:15.731 | INFO | 39.1% : 6.03 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:14:15.731 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:14:15.731 | INFO | ------------------------------------------ | 20260506 13:14:15.731 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:14:15.731 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:14:15.731 | INFO | Total mem usage: 26.5 GiB | 20260506 13:14:15.731 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:14:15.731 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:14:15.731 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:14:15.731 | INFO | High Water Breakdown: | 20260506 13:14:15.731 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:14:15.731 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:14:15.731 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:14:15.731 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:14:15.731 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:14:15.731 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:14:15.731 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:14:15.731 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:14:15.731 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:14:15.731 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:14:15.731 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:14:15.731 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:14:15.731 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:14:15.731 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:14:15.731 | INFO | ------------------------------------------ | 20260506 13:14:15.731 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:14:15.731 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:14:15.731 | INFO | Total mem usage: 25.6 GiB | 20260506 13:14:15.731 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:14:15.731 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:14:15.731 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:14:15.731 | INFO | High Water Breakdown: | 20260506 13:14:15.731 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:14:15.731 | INFO | ------------------------------------------ | 20260506 13:14:15.731 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:14:15.731 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:14:15.731 | INFO | Total mem usage: 15.7 GiB | 20260506 13:14:15.732 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:14:15.732 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:14:15.732 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:14:15.732 | INFO | High Water Breakdown: | 20260506 13:14:15.732 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:14:15.732 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:14:15.732 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:14:15.732 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:14:15.732 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:14:15.732 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:14:15.732 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:14:15.732 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:14:15.732 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:14:15.732 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:14:15.732 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:14:15.732 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:14:15.732 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:14:15.732 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:14:15.732 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:14:15.732 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:14:15.732 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:14:15.732 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:14:15.732 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:14:15.732 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:14:15.732 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:14:15.732 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:14:15.732 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:14:15.732 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:14:15.732 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:14:15.732 | INFO | ------------------------------------------ | 20260506 13:14:15.732 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:14:15.732 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:14:15.732 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:14:15.732 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:14:15.732 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.65e+01, 1.34e+01 / 2.61e+00 | 20260506 13:14:15.732 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.02e-01, 1.05e-02 / 6.67e-03 | 20260506 13:14:15.732 | INFO | PerfReport: 11.57%, Baseline -- 3.66e+00 / 4.42e+00, 5.76e-01 / 3.76e-01 | 20260506 13:14:15.732 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.47e+00, 1.90e-01 / 1.51e-01 | 20260506 13:14:15.732 | INFO | PerfReport: 15.93%, DME -- 4.03e+01 / 4.52e+01, 4.43e+00 / 3.07e+00 | 20260506 13:14:15.732 | INFO | PerfReport: 56.40%, FrameLabeling -- 1.79e+01 / 2.02e+01, 1.34e+00 / 1.62e+00 | 20260506 13:14:15.732 | INFO | PerfReport: 7.56%, PulseAccumulating -- 2.39e+00 / 3.03e+00, 4.20e-01 / 2.29e-01 | 20260506 13:14:15.732 | INFO | PerfReport: 3.71%, Metrics -- 1.17e+00 / 1.58e+00, 3.00e-01 / 1.61e-01 | 20260506 13:14:15.732 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.13e-01 / 4.76e-01, 6.50e-02 / 3.86e-02 | 20260506 13:14:15.732 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:14:15.732 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:14:15.732 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:14:15.732 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 13:14:15.732 | INFO | PerfReport: 21.49%, Upload -- 2.11e+01 / 0.00e+00, 7.04e+00 / 0.00e+00 | 20260506 13:14:15.732 | INFO | PerfReport: 21.16%, Download -- 2.08e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 13:14:15.732 | INFO | PerfReport: 32.24%, Compute -- 3.17e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 13:14:15.732 | INFO | PerfReport: 1.72%, LockUpload -- 1.68e+00 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 13:14:15.732 | INFO | PerfReport: 1.12%, LockDownload -- 1.10e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260506 13:14:15.732 | INFO | PerfReport: 22.27%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 13:14:15.732 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260506 13:14:15.864 | INFO | Starting a round of Baz IO | 20260506 13:14:19.025 | INFO | Populating RTMetrics report | 20260506 13:14:19.025 | INFO | RTMetrics report populated | 20260506 13:14:19.025 | INFO | Opening tmp RTMetrics json file | 20260506 13:14:19.208 | INFO | Serializing RTMetrics to json | 20260506 13:14:19.272 | INFO | Serialization done, swapping tmp file now... | 20260506 13:14:19.282 | INFO | RTMetrics done | 20260506 13:14:19.398 | INFO | Finished a round of Baz IO: Wrote 2.115 GiB in 3.534 seconds | 20260506 13:14:19.398 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.06% / 0.02% / 59.92% | 20260506 13:14:19.398 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.36068% / 3.17891e-05% / 5.16574e-05% | 20260506 13:14:19.398 | INFO | Baz IO overall: | 20260506 13:14:19.398 | INFO | BytesWritten : 824648203358 | 20260506 13:14:19.398 | INFO | NumEvents : 398798298734 | 20260506 13:14:20.870 | INFO | Analyzing chunk frames = [1630208,1630720) | 20260506 13:14:20.870 | INFO | MemInfo: { | 20260506 13:14:20.870 | INFO | "gpu" : " 17 GB", | 20260506 13:14:20.870 | INFO | "huge" : "", | 20260506 13:14:20.870 | INFO | "huge_shared" : " 27 GB", | 20260506 13:14:20.870 | INFO | "malloc" : " 14 GB", | 20260506 13:14:20.870 | INFO | "pinned" : " 29 GB" | 20260506 13:14:20.870 | INFO | } | 20260506 13:14:20.870 | INFO | Perf report for the last 5.13953 seconds: | 20260506 13:14:20.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:14:20.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:20.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:20.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:20.870 | INFO | ANALYSIS : 63.5%, 2.9, 96.0ms, 0 | 20260506 13:14:20.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:20.870 | INFO | RT_METRICS : 49.1%, 1.0, 25.7ms, 0 | 20260506 13:14:20.870 | INFO | PRE_HQ : 61.2%, 1.9, 60.1ms, 0 | 20260506 13:14:20.870 | INFO | BAZWRITER : 68.1%, 4.8, 168.7ms, 0 | 20260506 13:14:20.870 | INFO | BAZ_SAMPLING : 3.4%, 1.0, 1.8ms, 0 | 20260506 13:14:20.870 | INFO | Uploaded 36GiB to GPU in 2.09264s (17.2032GiB/s) | 20260506 13:14:20.870 | INFO | Downloaded 38GiB to GPU in 2.24485s (16.9276GiB/s) | 20260506 13:14:26.009 | INFO | Analyzing chunk frames = [1630720,1631232) | 20260506 13:14:26.009 | INFO | MemInfo: { | 20260506 13:14:26.009 | INFO | "gpu" : " 17 GB", | 20260506 13:14:26.009 | INFO | "huge" : "", | 20260506 13:14:26.009 | INFO | "huge_shared" : " 27 GB", | 20260506 13:14:26.009 | INFO | "malloc" : " 14 GB", | 20260506 13:14:26.009 | INFO | "pinned" : " 29 GB" | 20260506 13:14:26.009 | INFO | } | 20260506 13:14:26.010 | INFO | Perf report for the last 5.13947 seconds: | 20260506 13:14:26.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:14:26.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:26.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:26.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:26.010 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:14:26.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:26.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:26.010 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:14:26.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:26.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:26.010 | INFO | Uploaded 36GiB to GPU in 2.0445s (17.6083GiB/s) | 20260506 13:14:26.010 | INFO | Downloaded 34GiB to GPU in 1.92724s (17.6418GiB/s) | 20260506 13:14:31.148 | INFO | Analyzing chunk frames = [1631232,1631744) | 20260506 13:14:31.148 | INFO | MemInfo: { | 20260506 13:14:31.148 | INFO | "gpu" : " 17 GB", | 20260506 13:14:31.148 | INFO | "huge" : "", | 20260506 13:14:31.148 | INFO | "huge_shared" : " 27 GB", | 20260506 13:14:31.148 | INFO | "malloc" : " 14 GB", | 20260506 13:14:31.148 | INFO | "pinned" : " 29 GB" | 20260506 13:14:31.148 | INFO | } | 20260506 13:14:31.148 | INFO | Perf report for the last 5.13894 seconds: | 20260506 13:14:31.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:14:31.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:31.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:31.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:31.149 | INFO | ANALYSIS : 61.8%, 3.0, 95.6ms, 0 | 20260506 13:14:31.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:31.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:31.149 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:14:31.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:31.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:31.149 | INFO | Uploaded 36GiB to GPU in 2.02571s (17.7716GiB/s) | 20260506 13:14:31.149 | INFO | Downloaded 34GiB to GPU in 1.94668s (17.4656GiB/s) | 20260506 13:14:36.288 | INFO | Analyzing chunk frames = [1631744,1632256) | 20260506 13:14:36.288 | INFO | MemInfo: { | 20260506 13:14:36.288 | INFO | "gpu" : " 17 GB", | 20260506 13:14:36.288 | INFO | "huge" : "", | 20260506 13:14:36.288 | INFO | "huge_shared" : " 27 GB", | 20260506 13:14:36.288 | INFO | "malloc" : " 14 GB", | 20260506 13:14:36.288 | INFO | "pinned" : " 29 GB" | 20260506 13:14:36.288 | INFO | } | 20260506 13:14:36.288 | INFO | Perf report for the last 5.13933 seconds: | 20260506 13:14:36.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:14:36.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:36.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:36.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:36.288 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 13:14:36.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:36.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:36.288 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:14:36.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:36.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:36.288 | INFO | Uploaded 36GiB to GPU in 2.02314s (17.7942GiB/s) | 20260506 13:14:36.288 | INFO | Downloaded 34GiB to GPU in 1.94224s (17.5055GiB/s) | 20260506 13:14:41.427 | INFO | Analyzing chunk frames = [1632256,1632768) | 20260506 13:14:41.427 | INFO | MemInfo: { | 20260506 13:14:41.427 | INFO | "gpu" : " 17 GB", | 20260506 13:14:41.427 | INFO | "huge" : "", | 20260506 13:14:41.427 | INFO | "huge_shared" : " 27 GB", | 20260506 13:14:41.427 | INFO | "malloc" : " 14 GB", | 20260506 13:14:41.427 | INFO | "pinned" : " 29 GB" | 20260506 13:14:41.427 | INFO | } | 20260506 13:14:41.427 | INFO | Perf report for the last 5.1395 seconds: | 20260506 13:14:41.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:14:41.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:41.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:41.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:41.427 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 13:14:41.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:41.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:41.428 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:14:41.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:41.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:41.428 | INFO | Uploaded 36GiB to GPU in 2.05996s (17.4761GiB/s) | 20260506 13:14:41.428 | INFO | Downloaded 34GiB to GPU in 1.95141s (17.4233GiB/s) | 20260506 13:14:46.566 | INFO | Analyzing chunk frames = [1632768,1633280) | 20260506 13:14:46.566 | INFO | MemInfo: { | 20260506 13:14:46.566 | INFO | "gpu" : " 17 GB", | 20260506 13:14:46.566 | INFO | "huge" : "", | 20260506 13:14:46.566 | INFO | "huge_shared" : " 27 GB", | 20260506 13:14:46.566 | INFO | "malloc" : " 14 GB", | 20260506 13:14:46.566 | INFO | "pinned" : " 29 GB" | 20260506 13:14:46.566 | INFO | } | 20260506 13:14:46.566 | INFO | Perf report for the last 5.13902 seconds: | 20260506 13:14:46.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:14:46.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:46.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:46.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:46.566 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 13:14:46.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:46.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:46.566 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260506 13:14:46.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:46.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:46.567 | INFO | Uploaded 36GiB to GPU in 2.05554s (17.5137GiB/s) | 20260506 13:14:46.567 | INFO | Downloaded 34GiB to GPU in 1.92091s (17.6999GiB/s) | 20260506 13:14:51.706 | INFO | Analyzing chunk frames = [1633280,1633792) | 20260506 13:14:51.706 | INFO | MemInfo: { | 20260506 13:14:51.706 | INFO | "gpu" : " 17 GB", | 20260506 13:14:51.706 | INFO | "huge" : "", | 20260506 13:14:51.706 | INFO | "huge_shared" : " 27 GB", | 20260506 13:14:51.706 | INFO | "malloc" : " 14 GB", | 20260506 13:14:51.706 | INFO | "pinned" : " 29 GB" | 20260506 13:14:51.706 | INFO | } | 20260506 13:14:51.706 | INFO | Perf report for the last 5.13984 seconds: | 20260506 13:14:51.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:14:51.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:51.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:51.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:51.706 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 13:14:51.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:51.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:51.706 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 13:14:51.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:51.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:51.706 | INFO | Uploaded 36GiB to GPU in 2.01989s (17.8227GiB/s) | 20260506 13:14:51.706 | INFO | Downloaded 34GiB to GPU in 2.02654s (16.7773GiB/s) | 20260506 13:14:56.845 | INFO | Analyzing chunk frames = [1633792,1634304) | 20260506 13:14:56.845 | INFO | MemInfo: { | 20260506 13:14:56.845 | INFO | "gpu" : " 17 GB", | 20260506 13:14:56.845 | INFO | "huge" : "", | 20260506 13:14:56.845 | INFO | "huge_shared" : " 27 GB", | 20260506 13:14:56.845 | INFO | "malloc" : " 14 GB", | 20260506 13:14:56.845 | INFO | "pinned" : " 29 GB" | 20260506 13:14:56.845 | INFO | } | 20260506 13:14:56.845 | INFO | Perf report for the last 5.13918 seconds: | 20260506 13:14:56.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:14:56.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:56.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:56.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:56.845 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:14:56.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:56.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:56.846 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 13:14:56.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:56.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:14:56.846 | INFO | Uploaded 36GiB to GPU in 2.0471s (17.5858GiB/s) | 20260506 13:14:56.846 | INFO | Downloaded 34GiB to GPU in 1.96908s (17.2669GiB/s) | 20260506 13:14:56.979 | INFO | Starting a round of Baz IO | 20260506 13:15:00.154 | INFO | Populating RTMetrics report | 20260506 13:15:00.155 | INFO | RTMetrics report populated | 20260506 13:15:00.155 | INFO | Opening tmp RTMetrics json file | 20260506 13:15:00.183 | INFO | Serializing RTMetrics to json | 20260506 13:15:00.233 | INFO | Serialization done, swapping tmp file now... | 20260506 13:15:00.236 | INFO | RTMetrics done | 20260506 13:15:00.359 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.379 seconds | 20260506 13:15:00.359 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.05% / 0.02% / 59.94% | 20260506 13:15:00.359 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.34769% / 4.76837e-05% / 3.97364e-05% | 20260506 13:15:00.359 | INFO | Baz IO overall: | 20260506 13:15:00.359 | INFO | BytesWritten : 826917658343 | 20260506 13:15:00.359 | INFO | NumEvents : 399899078311 | 20260506 13:15:01.985 | INFO | Analyzing chunk frames = [1634304,1634816) | 20260506 13:15:01.985 | INFO | MemInfo: { | 20260506 13:15:01.985 | INFO | "gpu" : " 17 GB", | 20260506 13:15:01.985 | INFO | "huge" : "", | 20260506 13:15:01.985 | INFO | "huge_shared" : " 27 GB", | 20260506 13:15:01.985 | INFO | "malloc" : " 14 GB", | 20260506 13:15:01.985 | INFO | "pinned" : " 29 GB" | 20260506 13:15:01.985 | INFO | } | 20260506 13:15:01.985 | INFO | Perf report for the last 5.13939 seconds: | 20260506 13:15:01.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:15:01.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:01.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:01.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:01.985 | INFO | ANALYSIS : 63.7%, 2.9, 95.4ms, 0 | 20260506 13:15:01.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:01.985 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260506 13:15:01.985 | INFO | PRE_HQ : 61.8%, 1.9, 61.1ms, 0 | 20260506 13:15:01.985 | INFO | BAZWRITER : 65.4%, 4.8, 161.3ms, 0 | 20260506 13:15:01.985 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 13:15:01.985 | INFO | Uploaded 36GiB to GPU in 2.10793s (17.0784GiB/s) | 20260506 13:15:01.985 | INFO | Downloaded 38GiB to GPU in 2.2319s (17.0259GiB/s) | 20260506 13:15:07.124 | INFO | Analyzing chunk frames = [1634816,1635328) | 20260506 13:15:07.124 | INFO | MemInfo: { | 20260506 13:15:07.124 | INFO | "gpu" : " 17 GB", | 20260506 13:15:07.124 | INFO | "huge" : "", | 20260506 13:15:07.124 | INFO | "huge_shared" : " 27 GB", | 20260506 13:15:07.124 | INFO | "malloc" : " 14 GB", | 20260506 13:15:07.124 | INFO | "pinned" : " 29 GB" | 20260506 13:15:07.124 | INFO | } | 20260506 13:15:07.124 | INFO | Perf report for the last 5.13901 seconds: | 20260506 13:15:07.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:15:07.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:07.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:07.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:07.124 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:15:07.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:07.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:07.124 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:15:07.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:07.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:07.124 | INFO | Uploaded 36GiB to GPU in 2.05096s (17.5528GiB/s) | 20260506 13:15:07.124 | INFO | Downloaded 34GiB to GPU in 1.937s (17.553GiB/s) | 20260506 13:15:12.263 | INFO | Analyzing chunk frames = [1635328,1635840) | 20260506 13:15:12.263 | INFO | MemInfo: { | 20260506 13:15:12.263 | INFO | "gpu" : " 17 GB", | 20260506 13:15:12.263 | INFO | "huge" : "", | 20260506 13:15:12.263 | INFO | "huge_shared" : " 27 GB", | 20260506 13:15:12.263 | INFO | "malloc" : " 14 GB", | 20260506 13:15:12.263 | INFO | "pinned" : " 29 GB" | 20260506 13:15:12.263 | INFO | } | 20260506 13:15:12.263 | INFO | Perf report for the last 5.13932 seconds: | 20260506 13:15:12.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:15:12.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:12.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:12.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:12.263 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 13:15:12.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:12.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:12.263 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 13:15:12.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:12.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:12.263 | INFO | Uploaded 36GiB to GPU in 2.03608s (17.681GiB/s) | 20260506 13:15:12.263 | INFO | Downloaded 34GiB to GPU in 1.94166s (17.5108GiB/s) | 20260506 13:15:17.403 | INFO | Analyzing chunk frames = [1635840,1636352) | 20260506 13:15:17.403 | INFO | MemInfo: { | 20260506 13:15:17.403 | INFO | "gpu" : " 17 GB", | 20260506 13:15:17.403 | INFO | "huge" : "", | 20260506 13:15:17.403 | INFO | "huge_shared" : " 27 GB", | 20260506 13:15:17.403 | INFO | "malloc" : " 14 GB", | 20260506 13:15:17.403 | INFO | "pinned" : " 29 GB" | 20260506 13:15:17.403 | INFO | } | 20260506 13:15:17.403 | INFO | Perf report for the last 5.13952 seconds: | 20260506 13:15:17.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:15:17.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:17.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:17.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:17.403 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 13:15:17.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:17.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:17.403 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 13:15:17.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:17.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:17.403 | INFO | Uploaded 36GiB to GPU in 2.01365s (17.878GiB/s) | 20260506 13:15:17.403 | INFO | Downloaded 34GiB to GPU in 1.94047s (17.5216GiB/s) | 20260506 13:15:22.542 | INFO | Analyzing chunk frames = [1636352,1636864) | 20260506 13:15:22.542 | INFO | MemInfo: { | 20260506 13:15:22.542 | INFO | "gpu" : " 17 GB", | 20260506 13:15:22.542 | INFO | "huge" : "", | 20260506 13:15:22.542 | INFO | "huge_shared" : " 27 GB", | 20260506 13:15:22.542 | INFO | "malloc" : " 14 GB", | 20260506 13:15:22.542 | INFO | "pinned" : " 29 GB" | 20260506 13:15:22.542 | INFO | } | 20260506 13:15:22.542 | INFO | Perf report for the last 5.13917 seconds: | 20260506 13:15:22.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:15:22.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:22.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:22.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:22.542 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 13:15:22.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:22.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:22.542 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:15:22.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:22.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:22.542 | INFO | Uploaded 36GiB to GPU in 2.04002s (17.6469GiB/s) | 20260506 13:15:22.542 | INFO | Downloaded 34GiB to GPU in 1.94934s (17.4418GiB/s) | 20260506 13:15:27.681 | INFO | Analyzing chunk frames = [1636864,1637376) | 20260506 13:15:27.681 | INFO | MemInfo: { | 20260506 13:15:27.681 | INFO | "gpu" : " 17 GB", | 20260506 13:15:27.681 | INFO | "huge" : "", | 20260506 13:15:27.681 | INFO | "huge_shared" : " 27 GB", | 20260506 13:15:27.681 | INFO | "malloc" : " 14 GB", | 20260506 13:15:27.681 | INFO | "pinned" : " 29 GB" | 20260506 13:15:27.681 | INFO | } | 20260506 13:15:27.681 | INFO | Perf report for the last 5.13938 seconds: | 20260506 13:15:27.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:15:27.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:27.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:27.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:27.681 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:15:27.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:27.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:27.681 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260506 13:15:27.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:27.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:27.682 | INFO | Uploaded 36GiB to GPU in 2.04743s (17.583GiB/s) | 20260506 13:15:27.682 | INFO | Downloaded 34GiB to GPU in 1.92258s (17.6845GiB/s) | 20260506 13:15:32.821 | INFO | Analyzing chunk frames = [1637376,1637888) | 20260506 13:15:32.821 | INFO | MemInfo: { | 20260506 13:15:32.821 | INFO | "gpu" : " 17 GB", | 20260506 13:15:32.821 | INFO | "huge" : "", | 20260506 13:15:32.821 | INFO | "huge_shared" : " 27 GB", | 20260506 13:15:32.821 | INFO | "malloc" : " 14 GB", | 20260506 13:15:32.821 | INFO | "pinned" : " 29 GB" | 20260506 13:15:32.821 | INFO | } | 20260506 13:15:32.821 | INFO | Perf report for the last 5.13962 seconds: | 20260506 13:15:32.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:15:32.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:32.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:32.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:32.821 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:15:32.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:32.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:32.821 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 13:15:32.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:32.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:32.821 | INFO | Uploaded 36GiB to GPU in 2.03586s (17.6829GiB/s) | 20260506 13:15:32.821 | INFO | Downloaded 34GiB to GPU in 2.00663s (16.9438GiB/s) | 20260506 13:15:37.960 | INFO | Analyzing chunk frames = [1637888,1638400) | 20260506 13:15:37.960 | INFO | MemInfo: { | 20260506 13:15:37.960 | INFO | "gpu" : " 17 GB", | 20260506 13:15:37.960 | INFO | "huge" : "", | 20260506 13:15:37.960 | INFO | "huge_shared" : " 27 GB", | 20260506 13:15:37.960 | INFO | "malloc" : " 14 GB", | 20260506 13:15:37.960 | INFO | "pinned" : " 29 GB" | 20260506 13:15:37.960 | INFO | } | 20260506 13:15:37.960 | INFO | Perf report for the last 5.13897 seconds: | 20260506 13:15:37.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:15:37.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:37.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:37.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:37.960 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 13:15:37.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:37.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:37.960 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 13:15:37.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:37.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:37.960 | INFO | Uploaded 36GiB to GPU in 2.0607s (17.4698GiB/s) | 20260506 13:15:37.960 | INFO | Downloaded 34GiB to GPU in 1.98312s (17.1447GiB/s) | 20260506 13:15:37.960 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:15:37.960 | INFO | Total mem usage: 16.8 GiB | 20260506 13:15:37.960 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 13:15:37.960 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -8.87%) | 20260506 13:15:37.960 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:15:37.960 | INFO | High Water Breakdown: | 20260506 13:15:37.960 | INFO | 60.8% : 9.39 GiB ( 90.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:15:37.960 | INFO | 39% : 6.03 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:15:37.960 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:15:37.960 | INFO | ------------------------------------------ | 20260506 13:15:37.960 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:15:37.960 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:15:37.960 | INFO | Total mem usage: 26.5 GiB | 20260506 13:15:37.960 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:15:37.960 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:15:37.960 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:15:37.960 | INFO | High Water Breakdown: | 20260506 13:15:37.960 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:15:37.960 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:15:37.960 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:15:37.960 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:15:37.960 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:15:37.961 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:15:37.961 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:15:37.961 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:15:37.961 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:15:37.961 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:15:37.961 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:15:37.961 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:15:37.961 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:15:37.961 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:15:37.961 | INFO | ------------------------------------------ | 20260506 13:15:37.961 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:15:37.961 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:15:37.961 | INFO | Total mem usage: 25.6 GiB | 20260506 13:15:37.961 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:15:37.961 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:15:37.961 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:15:37.961 | INFO | High Water Breakdown: | 20260506 13:15:37.961 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:15:37.961 | INFO | ------------------------------------------ | 20260506 13:15:37.961 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:15:37.961 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:15:37.961 | INFO | Total mem usage: 15.7 GiB | 20260506 13:15:37.961 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:15:37.961 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:15:37.961 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 13:15:37.961 | INFO | High Water Breakdown: | 20260506 13:15:37.961 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:15:37.961 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:15:37.961 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:15:37.961 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:15:37.961 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:15:37.961 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:15:37.961 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:15:37.961 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:15:37.961 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:15:37.961 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:15:37.961 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:15:37.961 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:15:37.961 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:15:37.961 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:15:37.961 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:15:37.961 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:15:37.961 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:15:37.961 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:15:37.961 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:15:37.961 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:15:37.961 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:15:37.961 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:15:37.961 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:15:37.961 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:15:37.961 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:15:37.961 | INFO | ------------------------------------------ | 20260506 13:15:37.961 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:15:37.961 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:15:37.961 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:15:37.961 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:15:37.961 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.59e+01, 1.36e+01 / 1.96e+00 | 20260506 13:15:37.961 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.02e-01, 1.02e-02 / 6.35e-03 | 20260506 13:15:37.961 | INFO | PerfReport: 11.55%, Baseline -- 3.66e+00 / 4.40e+00, 5.70e-01 / 3.42e-01 | 20260506 13:15:37.961 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.47e+00, 1.89e-01 / 1.43e-01 | 20260506 13:15:37.961 | INFO | PerfReport: 15.75%, DME -- 3.99e+01 / 4.41e+01, 3.80e+00 / 2.50e+00 | 20260506 13:15:37.961 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.49e+00 / 1.78e+00 | 20260506 13:15:37.961 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.38e+00 / 2.97e+00, 4.11e-01 / 3.01e-01 | 20260506 13:15:37.961 | INFO | PerfReport: 3.70%, Metrics -- 1.17e+00 / 1.55e+00, 2.98e-01 / 1.90e-01 | 20260506 13:15:37.961 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.11e-01 / 4.66e-01, 6.38e-02 / 2.57e-02 | 20260506 13:15:37.961 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:15:37.961 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:15:37.961 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:15:37.961 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 13:15:37.961 | INFO | PerfReport: 21.70%, Upload -- 2.13e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260506 13:15:37.961 | INFO | PerfReport: 21.09%, Download -- 2.07e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 13:15:37.961 | INFO | PerfReport: 32.24%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 13:15:37.961 | INFO | PerfReport: 1.81%, LockUpload -- 1.78e+00 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 13:15:37.961 | INFO | PerfReport: 1.00%, LockDownload -- 9.82e-01 / 0.00e+00, 2.57e+00 / 0.00e+00 | 20260506 13:15:37.962 | INFO | PerfReport: 22.13%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 13:15:37.962 | INFO | PerfReport: 0.02%, Misc -- 1.50e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260506 13:15:38.091 | INFO | Starting a round of Baz IO | 20260506 13:15:41.292 | INFO | Populating RTMetrics report | 20260506 13:15:41.292 | INFO | RTMetrics report populated | 20260506 13:15:41.292 | INFO | Opening tmp RTMetrics json file | 20260506 13:15:41.293 | INFO | Serializing RTMetrics to json | 20260506 13:15:41.294 | INFO | Serialization done, swapping tmp file now... | 20260506 13:15:41.294 | INFO | RTMetrics done | 20260506 13:15:41.461 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.37 seconds | 20260506 13:15:41.461 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.03% / 0.02% / 59.95% | 20260506 13:15:41.461 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.3421% / 3.97364e-05% / 5.16574e-05% | 20260506 13:15:41.461 | INFO | Baz IO overall: | 20260506 13:15:41.461 | INFO | BytesWritten : 829187814341 | 20260506 13:15:41.461 | INFO | NumEvents : 401000607562 | 20260506 13:15:43.099 | INFO | Analyzing chunk frames = [1638400,1638912) | 20260506 13:15:43.099 | INFO | MemInfo: { | 20260506 13:15:43.099 | INFO | "gpu" : " 17 GB", | 20260506 13:15:43.099 | INFO | "huge" : "", | 20260506 13:15:43.099 | INFO | "huge_shared" : " 27 GB", | 20260506 13:15:43.099 | INFO | "malloc" : " 14 GB", | 20260506 13:15:43.099 | INFO | "pinned" : " 29 GB" | 20260506 13:15:43.099 | INFO | } | 20260506 13:15:43.099 | INFO | Perf report for the last 5.13919 seconds: | 20260506 13:15:43.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:15:43.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:43.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:43.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:43.099 | INFO | ANALYSIS : 63.8%, 2.8, 95.3ms, 0 | 20260506 13:15:43.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:43.099 | INFO | RT_METRICS : 46.2%, 1.0, 24.3ms, 0 | 20260506 13:15:43.099 | INFO | PRE_HQ : 61.9%, 2.0, 64.2ms, 0 | 20260506 13:15:43.099 | INFO | BAZWRITER : 65.1%, 4.8, 160.3ms, 0 | 20260506 13:15:43.099 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 13:15:43.099 | INFO | Uploaded 36GiB to GPU in 2.10038s (17.1398GiB/s) | 20260506 13:15:43.099 | INFO | Downloaded 38GiB to GPU in 2.22105s (17.1091GiB/s) | 20260506 13:15:48.239 | INFO | Analyzing chunk frames = [1638912,1639424) | 20260506 13:15:48.239 | INFO | MemInfo: { | 20260506 13:15:48.239 | INFO | "gpu" : " 17 GB", | 20260506 13:15:48.239 | INFO | "huge" : "", | 20260506 13:15:48.239 | INFO | "huge_shared" : " 27 GB", | 20260506 13:15:48.239 | INFO | "malloc" : " 14 GB", | 20260506 13:15:48.239 | INFO | "pinned" : " 29 GB" | 20260506 13:15:48.239 | INFO | } | 20260506 13:15:48.239 | INFO | Perf report for the last 5.13972 seconds: | 20260506 13:15:48.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:15:48.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:48.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:48.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:48.239 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 13:15:48.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:48.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:48.239 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 13:15:48.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:48.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:48.239 | INFO | Uploaded 36GiB to GPU in 2.01479s (17.8679GiB/s) | 20260506 13:15:48.239 | INFO | Downloaded 34GiB to GPU in 1.92354s (17.6757GiB/s) | 20260506 13:15:53.378 | INFO | Analyzing chunk frames = [1639424,1639936) | 20260506 13:15:53.378 | INFO | MemInfo: { | 20260506 13:15:53.378 | INFO | "gpu" : " 17 GB", | 20260506 13:15:53.378 | INFO | "huge" : "", | 20260506 13:15:53.378 | INFO | "huge_shared" : " 27 GB", | 20260506 13:15:53.378 | INFO | "malloc" : " 14 GB", | 20260506 13:15:53.378 | INFO | "pinned" : " 29 GB" | 20260506 13:15:53.378 | INFO | } | 20260506 13:15:53.378 | INFO | Perf report for the last 5.13926 seconds: | 20260506 13:15:53.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:15:53.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:53.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:53.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:53.378 | INFO | ANALYSIS : 61.4%, 3.0, 95.1ms, 0 | 20260506 13:15:53.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:53.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:53.378 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:15:53.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:53.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:53.378 | INFO | Uploaded 36GiB to GPU in 1.97959s (18.1856GiB/s) | 20260506 13:15:53.378 | INFO | Downloaded 34GiB to GPU in 1.92578s (17.6552GiB/s) | 20260506 13:15:58.517 | INFO | Analyzing chunk frames = [1639936,1640448) | 20260506 13:15:58.517 | INFO | MemInfo: { | 20260506 13:15:58.517 | INFO | "gpu" : " 17 GB", | 20260506 13:15:58.517 | INFO | "huge" : "", | 20260506 13:15:58.517 | INFO | "huge_shared" : " 27 GB", | 20260506 13:15:58.517 | INFO | "malloc" : " 14 GB", | 20260506 13:15:58.517 | INFO | "pinned" : " 29 GB" | 20260506 13:15:58.517 | INFO | } | 20260506 13:15:58.517 | INFO | Perf report for the last 5.13934 seconds: | 20260506 13:15:58.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:15:58.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:58.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:58.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:58.517 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 13:15:58.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:58.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:58.518 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:15:58.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:58.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:15:58.518 | INFO | Uploaded 36GiB to GPU in 1.98438s (18.1417GiB/s) | 20260506 13:15:58.518 | INFO | Downloaded 34GiB to GPU in 1.915s (17.7546GiB/s) | 20260506 13:16:03.657 | INFO | Analyzing chunk frames = [1640448,1640960) | 20260506 13:16:03.657 | INFO | MemInfo: { | 20260506 13:16:03.657 | INFO | "gpu" : " 17 GB", | 20260506 13:16:03.657 | INFO | "huge" : "", | 20260506 13:16:03.657 | INFO | "huge_shared" : " 27 GB", | 20260506 13:16:03.657 | INFO | "malloc" : " 14 GB", | 20260506 13:16:03.657 | INFO | "pinned" : " 29 GB" | 20260506 13:16:03.657 | INFO | } | 20260506 13:16:03.657 | INFO | Perf report for the last 5.13955 seconds: | 20260506 13:16:03.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:16:03.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:03.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:03.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:03.657 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 13:16:03.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:03.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:03.657 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 13:16:03.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:03.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:03.657 | INFO | Uploaded 36GiB to GPU in 2.01282s (17.8854GiB/s) | 20260506 13:16:03.657 | INFO | Downloaded 34GiB to GPU in 1.9427s (17.5014GiB/s) | 20260506 13:16:08.796 | INFO | Analyzing chunk frames = [1640960,1641472) | 20260506 13:16:08.796 | INFO | MemInfo: { | 20260506 13:16:08.796 | INFO | "gpu" : " 17 GB", | 20260506 13:16:08.796 | INFO | "huge" : "", | 20260506 13:16:08.796 | INFO | "huge_shared" : " 27 GB", | 20260506 13:16:08.796 | INFO | "malloc" : " 14 GB", | 20260506 13:16:08.796 | INFO | "pinned" : " 29 GB" | 20260506 13:16:08.796 | INFO | } | 20260506 13:16:08.796 | INFO | Perf report for the last 5.13928 seconds: | 20260506 13:16:08.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:16:08.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:08.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:08.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:08.796 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:16:08.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:08.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:08.796 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:16:08.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:08.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:08.797 | INFO | Uploaded 36GiB to GPU in 2.09417s (17.1906GiB/s) | 20260506 13:16:08.797 | INFO | Downloaded 34GiB to GPU in 1.96695s (17.2856GiB/s) | 20260506 13:16:13.935 | INFO | Analyzing chunk frames = [1641472,1641984) | 20260506 13:16:13.935 | INFO | MemInfo: { | 20260506 13:16:13.935 | INFO | "gpu" : " 17 GB", | 20260506 13:16:13.935 | INFO | "huge" : "", | 20260506 13:16:13.936 | INFO | "huge_shared" : " 27 GB", | 20260506 13:16:13.936 | INFO | "malloc" : " 14 GB", | 20260506 13:16:13.936 | INFO | "pinned" : " 29 GB" | 20260506 13:16:13.936 | INFO | } | 20260506 13:16:13.936 | INFO | Perf report for the last 5.13938 seconds: | 20260506 13:16:13.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:16:13.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:13.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:13.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:13.936 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 13:16:13.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:13.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:13.936 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 13:16:13.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:13.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:13.936 | INFO | Uploaded 36GiB to GPU in 2.0008s (17.9928GiB/s) | 20260506 13:16:13.936 | INFO | Downloaded 34GiB to GPU in 2.02917s (16.7556GiB/s) | 20260506 13:16:19.075 | INFO | Analyzing chunk frames = [1641984,1642496) | 20260506 13:16:19.075 | INFO | MemInfo: { | 20260506 13:16:19.075 | INFO | "gpu" : " 17 GB", | 20260506 13:16:19.075 | INFO | "huge" : "", | 20260506 13:16:19.075 | INFO | "huge_shared" : " 27 GB", | 20260506 13:16:19.075 | INFO | "malloc" : " 14 GB", | 20260506 13:16:19.075 | INFO | "pinned" : " 29 GB" | 20260506 13:16:19.075 | INFO | } | 20260506 13:16:19.075 | INFO | Perf report for the last 5.13905 seconds: | 20260506 13:16:19.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:16:19.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:19.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:19.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:19.075 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 13:16:19.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:19.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:19.075 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 13:16:19.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:19.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:19.075 | INFO | Uploaded 36GiB to GPU in 2.0382s (17.6626GiB/s) | 20260506 13:16:19.075 | INFO | Downloaded 34GiB to GPU in 1.97783s (17.1906GiB/s) | 20260506 13:16:19.206 | INFO | Starting a round of Baz IO | 20260506 13:16:22.393 | INFO | Populating RTMetrics report | 20260506 13:16:22.393 | INFO | RTMetrics report populated | 20260506 13:16:22.393 | INFO | Opening tmp RTMetrics json file | 20260506 13:16:22.394 | INFO | Serializing RTMetrics to json | 20260506 13:16:22.395 | INFO | Serialization done, swapping tmp file now... | 20260506 13:16:22.396 | INFO | RTMetrics done | 20260506 13:16:22.743 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.537 seconds | 20260506 13:16:22.743 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 40.01% / 0.02% / 59.97% | 20260506 13:16:22.743 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.32456% / 3.17891e-05% / 3.57628e-05% | 20260506 13:16:22.743 | INFO | Baz IO overall: | 20260506 13:16:22.743 | INFO | BytesWritten : 831455795145 | 20260506 13:16:22.743 | INFO | NumEvents : 402100539343 | 20260506 13:16:24.214 | INFO | Analyzing chunk frames = [1642496,1643008) | 20260506 13:16:24.214 | INFO | MemInfo: { | 20260506 13:16:24.214 | INFO | "gpu" : " 17 GB", | 20260506 13:16:24.214 | INFO | "huge" : "", | 20260506 13:16:24.214 | INFO | "huge_shared" : " 27 GB", | 20260506 13:16:24.214 | INFO | "malloc" : " 14 GB", | 20260506 13:16:24.214 | INFO | "pinned" : " 29 GB" | 20260506 13:16:24.214 | INFO | } | 20260506 13:16:24.214 | INFO | Perf report for the last 5.13929 seconds: | 20260506 13:16:24.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:16:24.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:24.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:24.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:24.214 | INFO | ANALYSIS : 63.7%, 2.9, 95.5ms, 0 | 20260506 13:16:24.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:24.214 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260506 13:16:24.214 | INFO | PRE_HQ : 61.5%, 1.9, 62.2ms, 0 | 20260506 13:16:24.214 | INFO | BAZWRITER : 68.2%, 4.8, 169.1ms, 0 | 20260506 13:16:24.214 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 13:16:24.214 | INFO | Uploaded 36GiB to GPU in 2.07234s (17.3717GiB/s) | 20260506 13:16:24.214 | INFO | Downloaded 38GiB to GPU in 2.22601s (17.0709GiB/s) | 20260506 13:16:29.354 | INFO | Analyzing chunk frames = [1643008,1643520) | 20260506 13:16:29.354 | INFO | MemInfo: { | 20260506 13:16:29.354 | INFO | "gpu" : " 17 GB", | 20260506 13:16:29.354 | INFO | "huge" : "", | 20260506 13:16:29.354 | INFO | "huge_shared" : " 27 GB", | 20260506 13:16:29.354 | INFO | "malloc" : " 14 GB", | 20260506 13:16:29.354 | INFO | "pinned" : " 29 GB" | 20260506 13:16:29.354 | INFO | } | 20260506 13:16:29.354 | INFO | Perf report for the last 5.13971 seconds: | 20260506 13:16:29.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:16:29.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:29.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:29.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:29.354 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:16:29.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:29.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:29.354 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 13:16:29.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:29.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:29.354 | INFO | Uploaded 36GiB to GPU in 2.03238s (17.7132GiB/s) | 20260506 13:16:29.354 | INFO | Downloaded 34GiB to GPU in 1.94542s (17.477GiB/s) | 20260506 13:16:34.493 | INFO | Analyzing chunk frames = [1643520,1644032) | 20260506 13:16:34.493 | INFO | MemInfo: { | 20260506 13:16:34.493 | INFO | "gpu" : " 17 GB", | 20260506 13:16:34.493 | INFO | "huge" : "", | 20260506 13:16:34.493 | INFO | "huge_shared" : " 27 GB", | 20260506 13:16:34.493 | INFO | "malloc" : " 14 GB", | 20260506 13:16:34.493 | INFO | "pinned" : " 29 GB" | 20260506 13:16:34.493 | INFO | } | 20260506 13:16:34.493 | INFO | Perf report for the last 5.13919 seconds: | 20260506 13:16:34.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:16:34.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:34.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:34.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:34.493 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 13:16:34.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:34.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:34.493 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 13:16:34.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:34.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:34.493 | INFO | Uploaded 36GiB to GPU in 2.01441s (17.8713GiB/s) | 20260506 13:16:34.493 | INFO | Downloaded 34GiB to GPU in 1.94842s (17.4501GiB/s) | 20260506 13:16:39.632 | INFO | Analyzing chunk frames = [1644032,1644544) | 20260506 13:16:39.632 | INFO | MemInfo: { | 20260506 13:16:39.632 | INFO | "gpu" : " 17 GB", | 20260506 13:16:39.632 | INFO | "huge" : "", | 20260506 13:16:39.632 | INFO | "huge_shared" : " 27 GB", | 20260506 13:16:39.632 | INFO | "malloc" : " 14 GB", | 20260506 13:16:39.632 | INFO | "pinned" : " 29 GB" | 20260506 13:16:39.632 | INFO | } | 20260506 13:16:39.633 | INFO | Perf report for the last 5.1395 seconds: | 20260506 13:16:39.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:16:39.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:39.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:39.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:39.633 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 13:16:39.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:39.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:39.633 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 13:16:39.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:39.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:39.633 | INFO | Uploaded 36GiB to GPU in 1.97378s (18.2391GiB/s) | 20260506 13:16:39.633 | INFO | Downloaded 34GiB to GPU in 1.94809s (17.453GiB/s) | 20260506 13:16:44.771 | INFO | Analyzing chunk frames = [1644544,1645056) | 20260506 13:16:44.771 | INFO | MemInfo: { | 20260506 13:16:44.771 | INFO | "gpu" : " 17 GB", | 20260506 13:16:44.772 | INFO | "huge" : "", | 20260506 13:16:44.772 | INFO | "huge_shared" : " 27 GB", | 20260506 13:16:44.772 | INFO | "malloc" : " 14 GB", | 20260506 13:16:44.772 | INFO | "pinned" : " 29 GB" | 20260506 13:16:44.772 | INFO | } | 20260506 13:16:44.772 | INFO | Perf report for the last 5.13924 seconds: | 20260506 13:16:44.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:16:44.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:44.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:44.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:44.772 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 13:16:44.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:44.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:44.772 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:16:44.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:44.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:44.772 | INFO | Uploaded 36GiB to GPU in 2.02104s (17.8126GiB/s) | 20260506 13:16:44.772 | INFO | Downloaded 34GiB to GPU in 1.95252s (17.4134GiB/s) | 20260506 13:16:49.911 | INFO | Analyzing chunk frames = [1645056,1645568) | 20260506 13:16:49.911 | INFO | MemInfo: { | 20260506 13:16:49.911 | INFO | "gpu" : " 17 GB", | 20260506 13:16:49.911 | INFO | "huge" : "", | 20260506 13:16:49.911 | INFO | "huge_shared" : " 27 GB", | 20260506 13:16:49.911 | INFO | "malloc" : " 14 GB", | 20260506 13:16:49.911 | INFO | "pinned" : " 29 GB" | 20260506 13:16:49.911 | INFO | } | 20260506 13:16:49.911 | INFO | Perf report for the last 5.1394 seconds: | 20260506 13:16:49.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:16:49.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:49.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:49.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:49.911 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 13:16:49.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:49.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:49.911 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:16:49.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:49.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:49.911 | INFO | Uploaded 36GiB to GPU in 2.04486s (17.6052GiB/s) | 20260506 13:16:49.912 | INFO | Downloaded 34GiB to GPU in 1.91846s (17.7226GiB/s) | 20260506 13:16:55.051 | INFO | Analyzing chunk frames = [1645568,1646080) | 20260506 13:16:55.051 | INFO | MemInfo: { | 20260506 13:16:55.051 | INFO | "gpu" : " 17 GB", | 20260506 13:16:55.051 | INFO | "huge" : "", | 20260506 13:16:55.051 | INFO | "huge_shared" : " 27 GB", | 20260506 13:16:55.051 | INFO | "malloc" : " 14 GB", | 20260506 13:16:55.051 | INFO | "pinned" : " 29 GB" | 20260506 13:16:55.051 | INFO | } | 20260506 13:16:55.051 | INFO | Perf report for the last 5.13982 seconds: | 20260506 13:16:55.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:16:55.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:55.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:55.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:55.051 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:16:55.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:55.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:55.051 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:16:55.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:55.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:16:55.051 | INFO | Uploaded 36GiB to GPU in 2.00084s (17.9925GiB/s) | 20260506 13:16:55.051 | INFO | Downloaded 34GiB to GPU in 1.97858s (17.184GiB/s) | 20260506 13:17:00.189 | INFO | Analyzing chunk frames = [1646080,1646592) | 20260506 13:17:00.189 | INFO | MemInfo: { | 20260506 13:17:00.189 | INFO | "gpu" : " 17 GB", | 20260506 13:17:00.189 | INFO | "huge" : "", | 20260506 13:17:00.190 | INFO | "huge_shared" : " 27 GB", | 20260506 13:17:00.190 | INFO | "malloc" : " 14 GB", | 20260506 13:17:00.190 | INFO | "pinned" : " 29 GB" | 20260506 13:17:00.190 | INFO | } | 20260506 13:17:00.190 | INFO | Perf report for the last 5.13879 seconds: | 20260506 13:17:00.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:17:00.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:00.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:00.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:00.190 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 13:17:00.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:00.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:00.190 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:17:00.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:00.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:00.190 | INFO | Uploaded 36GiB to GPU in 2.06727s (17.4143GiB/s) | 20260506 13:17:00.190 | INFO | Downloaded 34GiB to GPU in 1.9816s (17.1578GiB/s) | 20260506 13:17:00.190 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:17:00.190 | INFO | Total mem usage: 16.8 GiB | 20260506 13:17:00.190 | INFO | All Time High Water Marks sum: 17.1 GiB (Savings: 1.66%) | 20260506 13:17:00.190 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -8.13%) | 20260506 13:17:00.190 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:17:00.190 | INFO | High Water Breakdown: | 20260506 13:17:00.190 | INFO | 61% : 9.48 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:17:00.190 | INFO | 38.9% : 6.05 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:17:00.190 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:17:00.190 | INFO | ------------------------------------------ | 20260506 13:17:00.190 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:17:00.190 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:17:00.190 | INFO | Total mem usage: 26.5 GiB | 20260506 13:17:00.190 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:17:00.190 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 13:17:00.190 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 13:17:00.190 | INFO | High Water Breakdown: | 20260506 13:17:00.191 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:17:00.191 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:17:00.191 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:17:00.191 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:17:00.191 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:17:00.191 | INFO | 0.698% : 182 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:17:00.191 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:17:00.191 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:17:00.191 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:17:00.191 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:17:00.191 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:17:00.191 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:17:00.191 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:17:00.191 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:17:00.191 | INFO | ------------------------------------------ | 20260506 13:17:00.191 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:17:00.191 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:17:00.191 | INFO | Total mem usage: 25.6 GiB | 20260506 13:17:00.191 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:17:00.191 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:17:00.191 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:17:00.191 | INFO | High Water Breakdown: | 20260506 13:17:00.191 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:17:00.191 | INFO | ------------------------------------------ | 20260506 13:17:00.191 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:17:00.191 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:17:00.191 | INFO | Total mem usage: 15.7 GiB | 20260506 13:17:00.191 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:17:00.191 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:17:00.191 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 13:17:00.191 | INFO | High Water Breakdown: | 20260506 13:17:00.191 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:17:00.191 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:17:00.191 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:17:00.191 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:17:00.191 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:17:00.191 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:17:00.191 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:17:00.191 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:17:00.191 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:17:00.191 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:17:00.191 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:17:00.191 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:17:00.191 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:17:00.191 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:17:00.191 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:17:00.191 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:17:00.191 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:17:00.191 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:17:00.191 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:17:00.191 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:17:00.191 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:17:00.191 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:17:00.191 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:17:00.191 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:17:00.191 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:17:00.191 | INFO | ------------------------------------------ | 20260506 13:17:00.191 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:17:00.191 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:17:00.191 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:17:00.191 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:17:00.191 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.62e+01, 1.35e+01 / 2.23e+00 | 20260506 13:17:00.191 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.02e-01, 1.05e-02 / 6.68e-03 | 20260506 13:17:00.191 | INFO | PerfReport: 11.55%, Baseline -- 3.66e+00 / 4.42e+00, 5.74e-01 / 3.27e-01 | 20260506 13:17:00.192 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.47e+00, 1.90e-01 / 1.47e-01 | 20260506 13:17:00.192 | INFO | PerfReport: 15.85%, DME -- 4.01e+01 / 4.47e+01, 4.12e+00 / 2.36e+00 | 20260506 13:17:00.192 | INFO | PerfReport: 56.52%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.43e+00 / 1.97e+00 | 20260506 13:17:00.192 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.39e+00 / 3.03e+00, 4.24e-01 / 2.43e-01 | 20260506 13:17:00.192 | INFO | PerfReport: 3.70%, Metrics -- 1.17e+00 / 1.55e+00, 3.03e-01 / 1.64e-01 | 20260506 13:17:00.192 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.11e-01 / 4.66e-01, 6.40e-02 / 2.40e-02 | 20260506 13:17:00.192 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:17:00.192 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:17:00.192 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:17:00.192 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260506 13:17:00.192 | INFO | PerfReport: 21.51%, Upload -- 2.11e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 13:17:00.192 | INFO | PerfReport: 21.08%, Download -- 2.07e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 13:17:00.192 | INFO | PerfReport: 32.24%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 13:17:00.192 | INFO | PerfReport: 1.81%, LockUpload -- 1.78e+00 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260506 13:17:00.192 | INFO | PerfReport: 1.11%, LockDownload -- 1.09e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260506 13:17:00.192 | INFO | PerfReport: 22.22%, LockCompute -- 2.18e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 13:17:00.192 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260506 13:17:00.322 | INFO | Starting a round of Baz IO | 20260506 13:17:03.476 | INFO | Populating RTMetrics report | 20260506 13:17:03.476 | INFO | RTMetrics report populated | 20260506 13:17:03.476 | INFO | Opening tmp RTMetrics json file | 20260506 13:17:03.658 | INFO | Serializing RTMetrics to json | 20260506 13:17:03.690 | INFO | Serialization done, swapping tmp file now... | 20260506 13:17:03.695 | INFO | RTMetrics done | 20260506 13:17:04.388 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 4.065 seconds | 20260506 13:17:04.388 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.99% / 0.02% / 59.99% | 20260506 13:17:04.388 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.31712% / 4.37101e-05% / 3.97364e-05% | 20260506 13:17:04.388 | INFO | Baz IO overall: | 20260506 13:17:04.388 | INFO | BytesWritten : 833723696953 | 20260506 13:17:04.388 | INFO | NumEvents : 403200856929 | 20260506 13:17:05.329 | INFO | Analyzing chunk frames = [1646592,1647104) | 20260506 13:17:05.329 | INFO | MemInfo: { | 20260506 13:17:05.329 | INFO | "gpu" : " 17 GB", | 20260506 13:17:05.329 | INFO | "huge" : "", | 20260506 13:17:05.329 | INFO | "huge_shared" : " 27 GB", | 20260506 13:17:05.329 | INFO | "malloc" : " 14 GB", | 20260506 13:17:05.329 | INFO | "pinned" : " 29 GB" | 20260506 13:17:05.329 | INFO | } | 20260506 13:17:05.329 | INFO | Perf report for the last 5.13899 seconds: | 20260506 13:17:05.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:17:05.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:05.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:05.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:05.329 | INFO | ANALYSIS : 63.3%, 2.9, 95.7ms, 0 | 20260506 13:17:05.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:05.329 | INFO | RT_METRICS : 48.2%, 1.0, 25.3ms, 0 | 20260506 13:17:05.329 | INFO | PRE_HQ : 61.3%, 1.9, 59.9ms, 0 | 20260506 13:17:05.329 | INFO | BAZWRITER : 78.6%, 4.8, 195.4ms, 0 | 20260506 13:17:05.329 | INFO | BAZ_SAMPLING : 4.0%, 1.0, 2.1ms, 0 | 20260506 13:17:05.329 | INFO | Uploaded 36GiB to GPU in 2.0918s (17.21GiB/s) | 20260506 13:17:05.329 | INFO | Downloaded 38GiB to GPU in 2.24214s (16.9481GiB/s) | 20260506 13:17:10.468 | INFO | Analyzing chunk frames = [1647104,1647616) | 20260506 13:17:10.468 | INFO | MemInfo: { | 20260506 13:17:10.468 | INFO | "gpu" : " 17 GB", | 20260506 13:17:10.468 | INFO | "huge" : "", | 20260506 13:17:10.468 | INFO | "huge_shared" : " 27 GB", | 20260506 13:17:10.468 | INFO | "malloc" : " 14 GB", | 20260506 13:17:10.468 | INFO | "pinned" : " 29 GB" | 20260506 13:17:10.468 | INFO | } | 20260506 13:17:10.468 | INFO | Perf report for the last 5.13948 seconds: | 20260506 13:17:10.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:17:10.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:10.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:10.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:10.468 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:17:10.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:10.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:10.468 | INFO | PRE_HQ : 28.4%, 1.0, 15.0ms, 0 | 20260506 13:17:10.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:10.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:10.469 | INFO | Uploaded 36GiB to GPU in 2.00116s (17.9895GiB/s) | 20260506 13:17:10.469 | INFO | Downloaded 34GiB to GPU in 1.92724s (17.6418GiB/s) | 20260506 13:17:15.607 | INFO | Analyzing chunk frames = [1647616,1648128) | 20260506 13:17:15.607 | INFO | MemInfo: { | 20260506 13:17:15.607 | INFO | "gpu" : " 17 GB", | 20260506 13:17:15.607 | INFO | "huge" : "", | 20260506 13:17:15.608 | INFO | "huge_shared" : " 27 GB", | 20260506 13:17:15.608 | INFO | "malloc" : " 14 GB", | 20260506 13:17:15.608 | INFO | "pinned" : " 29 GB" | 20260506 13:17:15.608 | INFO | } | 20260506 13:17:15.608 | INFO | Perf report for the last 5.13939 seconds: | 20260506 13:17:15.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:17:15.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:15.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:15.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:15.608 | INFO | ANALYSIS : 61.4%, 3.0, 95.0ms, 0 | 20260506 13:17:15.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:15.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:15.608 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:17:15.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:15.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:15.608 | INFO | Uploaded 36GiB to GPU in 1.96381s (18.3317GiB/s) | 20260506 13:17:15.608 | INFO | Downloaded 34GiB to GPU in 1.92389s (17.6725GiB/s) | 20260506 13:17:20.747 | INFO | Analyzing chunk frames = [1648128,1648640) | 20260506 13:17:20.747 | INFO | MemInfo: { | 20260506 13:17:20.747 | INFO | "gpu" : " 17 GB", | 20260506 13:17:20.747 | INFO | "huge" : "", | 20260506 13:17:20.747 | INFO | "huge_shared" : " 27 GB", | 20260506 13:17:20.747 | INFO | "malloc" : " 14 GB", | 20260506 13:17:20.747 | INFO | "pinned" : " 29 GB" | 20260506 13:17:20.747 | INFO | } | 20260506 13:17:20.747 | INFO | Perf report for the last 5.13938 seconds: | 20260506 13:17:20.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:17:20.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:20.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:20.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:20.747 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:17:20.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:20.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:20.747 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:17:20.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:20.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:20.747 | INFO | Uploaded 36GiB to GPU in 2.0262s (17.7673GiB/s) | 20260506 13:17:20.747 | INFO | Downloaded 34GiB to GPU in 1.92353s (17.6758GiB/s) | 20260506 13:17:25.886 | INFO | Analyzing chunk frames = [1648640,1649152) | 20260506 13:17:25.886 | INFO | MemInfo: { | 20260506 13:17:25.886 | INFO | "gpu" : " 17 GB", | 20260506 13:17:25.886 | INFO | "huge" : "", | 20260506 13:17:25.886 | INFO | "huge_shared" : " 27 GB", | 20260506 13:17:25.886 | INFO | "malloc" : " 14 GB", | 20260506 13:17:25.887 | INFO | "pinned" : " 29 GB" | 20260506 13:17:25.887 | INFO | } | 20260506 13:17:25.887 | INFO | Perf report for the last 5.1395 seconds: | 20260506 13:17:25.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:17:25.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:25.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:25.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:25.887 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 13:17:25.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:25.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:25.887 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:17:25.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:25.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:25.887 | INFO | Uploaded 36GiB to GPU in 2.00084s (17.9924GiB/s) | 20260506 13:17:25.887 | INFO | Downloaded 34GiB to GPU in 1.94735s (17.4596GiB/s) | 20260506 13:17:31.026 | INFO | Analyzing chunk frames = [1649152,1649664) | 20260506 13:17:31.026 | INFO | MemInfo: { | 20260506 13:17:31.026 | INFO | "gpu" : " 17 GB", | 20260506 13:17:31.026 | INFO | "huge" : "", | 20260506 13:17:31.026 | INFO | "huge_shared" : " 27 GB", | 20260506 13:17:31.026 | INFO | "malloc" : " 14 GB", | 20260506 13:17:31.026 | INFO | "pinned" : " 29 GB" | 20260506 13:17:31.026 | INFO | } | 20260506 13:17:31.026 | INFO | Perf report for the last 5.13921 seconds: | 20260506 13:17:31.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:17:31.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:31.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:31.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:31.026 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:17:31.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:31.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:31.026 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260506 13:17:31.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:31.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:31.026 | INFO | Uploaded 36GiB to GPU in 2.06354s (17.4458GiB/s) | 20260506 13:17:31.026 | INFO | Downloaded 34GiB to GPU in 1.96418s (17.31GiB/s) | 20260506 13:17:36.165 | INFO | Analyzing chunk frames = [1649664,1650176) | 20260506 13:17:36.165 | INFO | MemInfo: { | 20260506 13:17:36.165 | INFO | "gpu" : " 17 GB", | 20260506 13:17:36.165 | INFO | "huge" : "", | 20260506 13:17:36.165 | INFO | "huge_shared" : " 27 GB", | 20260506 13:17:36.166 | INFO | "malloc" : " 14 GB", | 20260506 13:17:36.166 | INFO | "pinned" : " 29 GB" | 20260506 13:17:36.166 | INFO | } | 20260506 13:17:36.166 | INFO | Perf report for the last 5.14004 seconds: | 20260506 13:17:36.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:17:36.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:36.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:36.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:36.166 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 13:17:36.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:36.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:36.166 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 13:17:36.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:36.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:36.166 | INFO | Uploaded 36GiB to GPU in 2.01135s (17.8984GiB/s) | 20260506 13:17:36.166 | INFO | Downloaded 34GiB to GPU in 2.03179s (16.734GiB/s) | 20260506 13:17:41.304 | INFO | Analyzing chunk frames = [1650176,1650688) | 20260506 13:17:41.304 | INFO | MemInfo: { | 20260506 13:17:41.304 | INFO | "gpu" : " 17 GB", | 20260506 13:17:41.304 | INFO | "huge" : "", | 20260506 13:17:41.304 | INFO | "huge_shared" : " 27 GB", | 20260506 13:17:41.304 | INFO | "malloc" : " 14 GB", | 20260506 13:17:41.304 | INFO | "pinned" : " 29 GB" | 20260506 13:17:41.304 | INFO | } | 20260506 13:17:41.304 | INFO | Perf report for the last 5.13852 seconds: | 20260506 13:17:41.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:17:41.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:41.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:41.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:41.305 | INFO | ANALYSIS : 62.8%, 3.0, 97.9ms, 0 | 20260506 13:17:41.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:41.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:41.305 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260506 13:17:41.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:41.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:41.305 | INFO | Uploaded 36GiB to GPU in 2.05094s (17.5529GiB/s) | 20260506 13:17:41.305 | INFO | Downloaded 34GiB to GPU in 1.99325s (17.0576GiB/s) | 20260506 13:17:41.432 | INFO | Starting a round of Baz IO | 20260506 13:17:44.627 | INFO | Populating RTMetrics report | 20260506 13:17:44.628 | INFO | RTMetrics report populated | 20260506 13:17:44.628 | INFO | Opening tmp RTMetrics json file | 20260506 13:17:44.658 | INFO | Serializing RTMetrics to json | 20260506 13:17:44.662 | INFO | Serialization done, swapping tmp file now... | 20260506 13:17:44.666 | INFO | RTMetrics done | 20260506 13:17:44.853 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.421 seconds | 20260506 13:17:44.854 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.97% / 0.02% / 60.01% | 20260506 13:17:44.854 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.29837% / 1.98682e-05% / 4.76837e-05% | 20260506 13:17:44.854 | INFO | Baz IO overall: | 20260506 13:17:44.854 | INFO | BytesWritten : 835990415420 | 20260506 13:17:44.854 | INFO | NumEvents : 404300055925 | 20260506 13:17:46.443 | INFO | Analyzing chunk frames = [1650688,1651200) | 20260506 13:17:46.443 | INFO | MemInfo: { | 20260506 13:17:46.443 | INFO | "gpu" : " 17 GB", | 20260506 13:17:46.443 | INFO | "huge" : "", | 20260506 13:17:46.444 | INFO | "huge_shared" : " 27 GB", | 20260506 13:17:46.444 | INFO | "malloc" : " 14 GB", | 20260506 13:17:46.444 | INFO | "pinned" : " 29 GB" | 20260506 13:17:46.444 | INFO | } | 20260506 13:17:46.444 | INFO | Perf report for the last 5.13924 seconds: | 20260506 13:17:46.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:17:46.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:46.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:46.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:46.444 | INFO | ANALYSIS : 63.8%, 2.8, 94.3ms, 0 | 20260506 13:17:46.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:46.444 | INFO | RT_METRICS : 47.2%, 1.0, 24.7ms, 0 | 20260506 13:17:46.444 | INFO | PRE_HQ : 61.8%, 2.0, 63.9ms, 0 | 20260506 13:17:46.444 | INFO | BAZWRITER : 66.0%, 4.8, 163.7ms, 0 | 20260506 13:17:46.444 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 13:17:46.444 | INFO | Uploaded 36GiB to GPU in 2.07315s (17.3648GiB/s) | 20260506 13:17:46.444 | INFO | Downloaded 38GiB to GPU in 2.21379s (17.1651GiB/s) | 20260506 13:17:51.583 | INFO | Analyzing chunk frames = [1651200,1651712) | 20260506 13:17:51.583 | INFO | MemInfo: { | 20260506 13:17:51.583 | INFO | "gpu" : " 17 GB", | 20260506 13:17:51.583 | INFO | "huge" : "", | 20260506 13:17:51.583 | INFO | "huge_shared" : " 27 GB", | 20260506 13:17:51.583 | INFO | "malloc" : " 14 GB", | 20260506 13:17:51.583 | INFO | "pinned" : " 29 GB" | 20260506 13:17:51.583 | INFO | } | 20260506 13:17:51.583 | INFO | Perf report for the last 5.13925 seconds: | 20260506 13:17:51.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:17:51.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:51.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:51.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:51.583 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:17:51.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:51.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:51.583 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 13:17:51.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:51.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:51.583 | INFO | Uploaded 36GiB to GPU in 2.04052s (17.6426GiB/s) | 20260506 13:17:51.583 | INFO | Downloaded 34GiB to GPU in 1.94179s (17.5096GiB/s) | 20260506 13:17:56.722 | INFO | Analyzing chunk frames = [1651712,1652224) | 20260506 13:17:56.722 | INFO | MemInfo: { | 20260506 13:17:56.722 | INFO | "gpu" : " 17 GB", | 20260506 13:17:56.722 | INFO | "huge" : "", | 20260506 13:17:56.722 | INFO | "huge_shared" : " 27 GB", | 20260506 13:17:56.722 | INFO | "malloc" : " 14 GB", | 20260506 13:17:56.722 | INFO | "pinned" : " 29 GB" | 20260506 13:17:56.722 | INFO | } | 20260506 13:17:56.722 | INFO | Perf report for the last 5.13949 seconds: | 20260506 13:17:56.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:17:56.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:56.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:56.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:56.722 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 13:17:56.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:56.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:56.723 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:17:56.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:56.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:17:56.723 | INFO | Uploaded 36GiB to GPU in 2.03889s (17.6567GiB/s) | 20260506 13:17:56.723 | INFO | Downloaded 34GiB to GPU in 1.9424s (17.5042GiB/s) | 20260506 13:18:01.862 | INFO | Analyzing chunk frames = [1652224,1652736) | 20260506 13:18:01.862 | INFO | MemInfo: { | 20260506 13:18:01.862 | INFO | "gpu" : " 17 GB", | 20260506 13:18:01.862 | INFO | "huge" : "", | 20260506 13:18:01.862 | INFO | "huge_shared" : " 27 GB", | 20260506 13:18:01.862 | INFO | "malloc" : " 14 GB", | 20260506 13:18:01.862 | INFO | "pinned" : " 29 GB" | 20260506 13:18:01.862 | INFO | } | 20260506 13:18:01.862 | INFO | Perf report for the last 5.13923 seconds: | 20260506 13:18:01.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:18:01.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:01.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:01.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:01.862 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:18:01.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:01.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:01.862 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:18:01.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:01.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:01.862 | INFO | Uploaded 36GiB to GPU in 2.01051s (17.9059GiB/s) | 20260506 13:18:01.863 | INFO | Downloaded 34GiB to GPU in 1.94113s (17.5156GiB/s) | 20260506 13:18:07.001 | INFO | Analyzing chunk frames = [1652736,1653248) | 20260506 13:18:07.001 | INFO | MemInfo: { | 20260506 13:18:07.001 | INFO | "gpu" : " 17 GB", | 20260506 13:18:07.001 | INFO | "huge" : "", | 20260506 13:18:07.001 | INFO | "huge_shared" : " 27 GB", | 20260506 13:18:07.001 | INFO | "malloc" : " 14 GB", | 20260506 13:18:07.001 | INFO | "pinned" : " 29 GB" | 20260506 13:18:07.001 | INFO | } | 20260506 13:18:07.001 | INFO | Perf report for the last 5.13936 seconds: | 20260506 13:18:07.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:18:07.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:07.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:07.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:07.001 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 13:18:07.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:07.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:07.001 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:18:07.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:07.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:07.001 | INFO | Uploaded 36GiB to GPU in 2.05645s (17.5059GiB/s) | 20260506 13:18:07.001 | INFO | Downloaded 34GiB to GPU in 1.94942s (17.4411GiB/s) | 20260506 13:18:12.140 | INFO | Analyzing chunk frames = [1653248,1653760) | 20260506 13:18:12.140 | INFO | MemInfo: { | 20260506 13:18:12.140 | INFO | "gpu" : " 17 GB", | 20260506 13:18:12.140 | INFO | "huge" : "", | 20260506 13:18:12.140 | INFO | "huge_shared" : " 27 GB", | 20260506 13:18:12.140 | INFO | "malloc" : " 14 GB", | 20260506 13:18:12.140 | INFO | "pinned" : " 29 GB" | 20260506 13:18:12.140 | INFO | } | 20260506 13:18:12.140 | INFO | Perf report for the last 5.13923 seconds: | 20260506 13:18:12.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:18:12.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:12.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:12.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:12.140 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:18:12.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:12.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:12.140 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:18:12.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:12.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:12.141 | INFO | Uploaded 36GiB to GPU in 2.05675s (17.5033GiB/s) | 20260506 13:18:12.141 | INFO | Downloaded 34GiB to GPU in 1.93091s (17.6083GiB/s) | 20260506 13:18:17.281 | INFO | Analyzing chunk frames = [1653760,1654272) | 20260506 13:18:17.281 | INFO | MemInfo: { | 20260506 13:18:17.281 | INFO | "gpu" : " 17 GB", | 20260506 13:18:17.281 | INFO | "huge" : "", | 20260506 13:18:17.281 | INFO | "huge_shared" : " 27 GB", | 20260506 13:18:17.281 | INFO | "malloc" : " 14 GB", | 20260506 13:18:17.281 | INFO | "pinned" : " 29 GB" | 20260506 13:18:17.281 | INFO | } | 20260506 13:18:17.281 | INFO | Perf report for the last 5.14054 seconds: | 20260506 13:18:17.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:18:17.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:17.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:17.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:17.281 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:18:17.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:17.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:17.281 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 13:18:17.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:17.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:17.281 | INFO | Uploaded 36GiB to GPU in 2.0393s (17.6531GiB/s) | 20260506 13:18:17.281 | INFO | Downloaded 34GiB to GPU in 2.01768s (16.851GiB/s) | 20260506 13:18:22.419 | INFO | Analyzing chunk frames = [1654272,1654784) | 20260506 13:18:22.419 | INFO | MemInfo: { | 20260506 13:18:22.419 | INFO | "gpu" : " 17 GB", | 20260506 13:18:22.419 | INFO | "huge" : "", | 20260506 13:18:22.419 | INFO | "huge_shared" : " 27 GB", | 20260506 13:18:22.419 | INFO | "malloc" : " 14 GB", | 20260506 13:18:22.419 | INFO | "pinned" : " 29 GB" | 20260506 13:18:22.419 | INFO | } | 20260506 13:18:22.419 | INFO | Perf report for the last 5.13857 seconds: | 20260506 13:18:22.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:18:22.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:22.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:22.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:22.419 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 13:18:22.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:22.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:22.420 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 13:18:22.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:22.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:22.420 | INFO | Uploaded 36GiB to GPU in 2.08171s (17.2935GiB/s) | 20260506 13:18:22.420 | INFO | Downloaded 34GiB to GPU in 2.00133s (16.9887GiB/s) | 20260506 13:18:22.420 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:18:22.420 | INFO | Total mem usage: 16.8 GiB | 20260506 13:18:22.420 | INFO | All Time High Water Marks sum: 17.2 GiB (Savings: 2.01%) | 20260506 13:18:22.420 | INFO | Recent High Water Marks Sum: 16.9 GiB (Savings: 0.0456%) | 20260506 13:18:22.420 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:18:22.420 | INFO | High Water Breakdown: | 20260506 13:18:22.420 | INFO | 62.2% : 10.5 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:18:22.420 | INFO | 37.7% : 6.35 GiB ( 94.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:18:22.420 | INFO | 0.118% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:18:22.420 | INFO | ------------------------------------------ | 20260506 13:18:22.420 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:18:22.420 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:18:22.420 | INFO | Total mem usage: 26.5 GiB | 20260506 13:18:22.420 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:18:22.420 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 13:18:22.420 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 13:18:22.420 | INFO | High Water Breakdown: | 20260506 13:18:22.420 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:18:22.420 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:18:22.420 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:18:22.420 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:18:22.420 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:18:22.420 | INFO | 0.698% : 182 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:18:22.420 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:18:22.420 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:18:22.420 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:18:22.420 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:18:22.420 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:18:22.420 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:18:22.420 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:18:22.420 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:18:22.420 | INFO | ------------------------------------------ | 20260506 13:18:22.420 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:18:22.420 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:18:22.420 | INFO | Total mem usage: 25.6 GiB | 20260506 13:18:22.420 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:18:22.420 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:18:22.420 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:18:22.420 | INFO | High Water Breakdown: | 20260506 13:18:22.420 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:18:22.420 | INFO | ------------------------------------------ | 20260506 13:18:22.420 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:18:22.420 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:18:22.420 | INFO | Total mem usage: 15.7 GiB | 20260506 13:18:22.420 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:18:22.420 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:18:22.420 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 13:18:22.420 | INFO | High Water Breakdown: | 20260506 13:18:22.420 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:18:22.420 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:18:22.420 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:18:22.420 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:18:22.420 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:18:22.420 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:18:22.420 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:18:22.420 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:18:22.420 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:18:22.420 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:18:22.420 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:18:22.420 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:18:22.420 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:18:22.420 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:18:22.421 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:18:22.421 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:18:22.421 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:18:22.421 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:18:22.421 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:18:22.421 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:18:22.421 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:18:22.421 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:18:22.421 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:18:22.421 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:18:22.421 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:18:22.421 | INFO | ------------------------------------------ | 20260506 13:18:22.421 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:18:22.421 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:18:22.421 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:18:22.421 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:18:22.421 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.64e+01, 1.37e+01 / 2.90e+00 | 20260506 13:18:22.421 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.02e-01, 1.03e-02 / 6.94e-03 | 20260506 13:18:22.421 | INFO | PerfReport: 11.56%, Baseline -- 3.66e+00 / 4.40e+00, 5.74e-01 / 3.77e-01 | 20260506 13:18:22.421 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.45e+00, 1.91e-01 / 1.46e-01 | 20260506 13:18:22.421 | INFO | PerfReport: 15.86%, DME -- 4.02e+01 / 4.48e+01, 4.36e+00 / 3.34e+00 | 20260506 13:18:22.421 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.44e+00 / 1.91e+00 | 20260506 13:18:22.421 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.38e+00 / 2.98e+00, 4.11e-01 / 2.46e-01 | 20260506 13:18:22.421 | INFO | PerfReport: 3.69%, Metrics -- 1.17e+00 / 1.55e+00, 2.99e-01 / 1.80e-01 | 20260506 13:18:22.421 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.11e-01 / 4.67e-01, 6.49e-02 / 2.76e-02 | 20260506 13:18:22.421 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:18:22.421 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:18:22.421 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:18:22.421 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 13:18:22.421 | INFO | PerfReport: 21.62%, Upload -- 2.12e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260506 13:18:22.421 | INFO | PerfReport: 21.15%, Download -- 2.08e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 13:18:22.421 | INFO | PerfReport: 32.28%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 13:18:22.421 | INFO | PerfReport: 1.79%, LockUpload -- 1.76e+00 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 13:18:22.421 | INFO | PerfReport: 1.05%, LockDownload -- 1.03e+00 / 0.00e+00, 2.70e+00 / 0.00e+00 | 20260506 13:18:22.421 | INFO | PerfReport: 22.10%, LockCompute -- 2.17e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 13:18:22.421 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260506 13:18:22.551 | INFO | Starting a round of Baz IO | 20260506 13:18:25.715 | INFO | Populating RTMetrics report | 20260506 13:18:25.715 | INFO | RTMetrics report populated | 20260506 13:18:25.715 | INFO | Opening tmp RTMetrics json file | 20260506 13:18:25.718 | INFO | Serializing RTMetrics to json | 20260506 13:18:25.720 | INFO | Serialization done, swapping tmp file now... | 20260506 13:18:25.722 | INFO | RTMetrics done | 20260506 13:18:26.311 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.759 seconds | 20260506 13:18:26.311 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.95% / 0.02% / 60.03% | 20260506 13:18:26.311 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.29387% / 3.97364e-05% / 7.15256e-05% | 20260506 13:18:26.311 | INFO | Baz IO overall: | 20260506 13:18:26.311 | INFO | BytesWritten : 838257488812 | 20260506 13:18:26.311 | INFO | NumEvents : 405399831877 | 20260506 13:18:27.558 | INFO | Analyzing chunk frames = [1654784,1655296) | 20260506 13:18:27.558 | INFO | MemInfo: { | 20260506 13:18:27.558 | INFO | "gpu" : " 17 GB", | 20260506 13:18:27.558 | INFO | "huge" : "", | 20260506 13:18:27.558 | INFO | "huge_shared" : " 27 GB", | 20260506 13:18:27.558 | INFO | "malloc" : " 14 GB", | 20260506 13:18:27.558 | INFO | "pinned" : " 29 GB" | 20260506 13:18:27.558 | INFO | } | 20260506 13:18:27.558 | INFO | Perf report for the last 5.13904 seconds: | 20260506 13:18:27.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:18:27.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:27.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:27.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:27.559 | INFO | ANALYSIS : 63.5%, 2.9, 96.0ms, 0 | 20260506 13:18:27.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:27.559 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:18:27.559 | INFO | PRE_HQ : 61.3%, 1.9, 60.1ms, 0 | 20260506 13:18:27.559 | INFO | BAZWRITER : 72.5%, 4.8, 178.8ms, 0 | 20260506 13:18:27.559 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 13:18:27.559 | INFO | Uploaded 36GiB to GPU in 2.07536s (17.3464GiB/s) | 20260506 13:18:27.559 | INFO | Downloaded 38GiB to GPU in 2.24235s (16.9465GiB/s) | 20260506 13:18:32.698 | INFO | Analyzing chunk frames = [1655296,1655808) | 20260506 13:18:32.698 | INFO | MemInfo: { | 20260506 13:18:32.698 | INFO | "gpu" : " 17 GB", | 20260506 13:18:32.698 | INFO | "huge" : "", | 20260506 13:18:32.698 | INFO | "huge_shared" : " 27 GB", | 20260506 13:18:32.698 | INFO | "malloc" : " 14 GB", | 20260506 13:18:32.698 | INFO | "pinned" : " 29 GB" | 20260506 13:18:32.698 | INFO | } | 20260506 13:18:32.698 | INFO | Perf report for the last 5.13963 seconds: | 20260506 13:18:32.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:18:32.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:32.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:32.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:32.698 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 13:18:32.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:32.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:32.698 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 13:18:32.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:32.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:32.698 | INFO | Uploaded 36GiB to GPU in 2.03677s (17.675GiB/s) | 20260506 13:18:32.698 | INFO | Downloaded 34GiB to GPU in 1.93173s (17.6008GiB/s) | 20260506 13:18:37.837 | INFO | Analyzing chunk frames = [1655808,1656320) | 20260506 13:18:37.837 | INFO | MemInfo: { | 20260506 13:18:37.837 | INFO | "gpu" : " 17 GB", | 20260506 13:18:37.837 | INFO | "huge" : "", | 20260506 13:18:37.837 | INFO | "huge_shared" : " 27 GB", | 20260506 13:18:37.837 | INFO | "malloc" : " 14 GB", | 20260506 13:18:37.837 | INFO | "pinned" : " 29 GB" | 20260506 13:18:37.837 | INFO | } | 20260506 13:18:37.837 | INFO | Perf report for the last 5.1392 seconds: | 20260506 13:18:37.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:18:37.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:37.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:37.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:37.837 | INFO | ANALYSIS : 61.7%, 3.0, 95.5ms, 0 | 20260506 13:18:37.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:37.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:37.837 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:18:37.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:37.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:37.838 | INFO | Uploaded 36GiB to GPU in 2.0036s (17.9676GiB/s) | 20260506 13:18:37.838 | INFO | Downloaded 34GiB to GPU in 1.94432s (17.4868GiB/s) | 20260506 13:18:42.977 | INFO | Analyzing chunk frames = [1656320,1656832) | 20260506 13:18:42.977 | INFO | MemInfo: { | 20260506 13:18:42.977 | INFO | "gpu" : " 17 GB", | 20260506 13:18:42.977 | INFO | "huge" : "", | 20260506 13:18:42.977 | INFO | "huge_shared" : " 27 GB", | 20260506 13:18:42.977 | INFO | "malloc" : " 14 GB", | 20260506 13:18:42.977 | INFO | "pinned" : " 29 GB" | 20260506 13:18:42.977 | INFO | } | 20260506 13:18:42.977 | INFO | Perf report for the last 5.13975 seconds: | 20260506 13:18:42.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:18:42.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:42.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:42.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:42.977 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:18:42.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:42.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:42.977 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:18:42.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:42.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:42.977 | INFO | Uploaded 36GiB to GPU in 2.00099s (17.9911GiB/s) | 20260506 13:18:42.977 | INFO | Downloaded 34GiB to GPU in 1.93639s (17.5585GiB/s) | 20260506 13:18:48.116 | INFO | Analyzing chunk frames = [1656832,1657344) | 20260506 13:18:48.116 | INFO | MemInfo: { | 20260506 13:18:48.116 | INFO | "gpu" : " 17 GB", | 20260506 13:18:48.116 | INFO | "huge" : "", | 20260506 13:18:48.116 | INFO | "huge_shared" : " 27 GB", | 20260506 13:18:48.116 | INFO | "malloc" : " 14 GB", | 20260506 13:18:48.116 | INFO | "pinned" : " 29 GB" | 20260506 13:18:48.116 | INFO | } | 20260506 13:18:48.116 | INFO | Perf report for the last 5.13883 seconds: | 20260506 13:18:48.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:18:48.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:48.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:48.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:48.116 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 13:18:48.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:48.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:48.116 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:18:48.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:48.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:48.116 | INFO | Uploaded 36GiB to GPU in 2.01024s (17.9083GiB/s) | 20260506 13:18:48.116 | INFO | Downloaded 34GiB to GPU in 1.94425s (17.4874GiB/s) | 20260506 13:18:53.255 | INFO | Analyzing chunk frames = [1657344,1657856) | 20260506 13:18:53.255 | INFO | MemInfo: { | 20260506 13:18:53.255 | INFO | "gpu" : " 17 GB", | 20260506 13:18:53.255 | INFO | "huge" : "", | 20260506 13:18:53.255 | INFO | "huge_shared" : " 27 GB", | 20260506 13:18:53.255 | INFO | "malloc" : " 14 GB", | 20260506 13:18:53.255 | INFO | "pinned" : " 29 GB" | 20260506 13:18:53.255 | INFO | } | 20260506 13:18:53.255 | INFO | Perf report for the last 5.1392 seconds: | 20260506 13:18:53.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:18:53.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:53.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:53.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:53.255 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:18:53.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:53.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:53.255 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:18:53.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:53.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:53.256 | INFO | Uploaded 36GiB to GPU in 2.09154s (17.2122GiB/s) | 20260506 13:18:53.256 | INFO | Downloaded 34GiB to GPU in 1.96131s (17.3354GiB/s) | 20260506 13:18:58.395 | INFO | Analyzing chunk frames = [1657856,1658368) | 20260506 13:18:58.395 | INFO | MemInfo: { | 20260506 13:18:58.395 | INFO | "gpu" : " 17 GB", | 20260506 13:18:58.395 | INFO | "huge" : "", | 20260506 13:18:58.395 | INFO | "huge_shared" : " 27 GB", | 20260506 13:18:58.395 | INFO | "malloc" : " 14 GB", | 20260506 13:18:58.395 | INFO | "pinned" : " 29 GB" | 20260506 13:18:58.395 | INFO | } | 20260506 13:18:58.395 | INFO | Perf report for the last 5.13988 seconds: | 20260506 13:18:58.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:18:58.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:58.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:58.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:58.395 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:18:58.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:58.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:58.395 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 13:18:58.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:58.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:18:58.395 | INFO | Uploaded 36GiB to GPU in 1.96916s (18.2819GiB/s) | 20260506 13:18:58.395 | INFO | Downloaded 34GiB to GPU in 2.02668s (16.7762GiB/s) | 20260506 13:19:03.534 | INFO | Analyzing chunk frames = [1658368,1658880) | 20260506 13:19:03.534 | INFO | MemInfo: { | 20260506 13:19:03.534 | INFO | "gpu" : " 17 GB", | 20260506 13:19:03.534 | INFO | "huge" : "", | 20260506 13:19:03.534 | INFO | "huge_shared" : " 27 GB", | 20260506 13:19:03.534 | INFO | "malloc" : " 14 GB", | 20260506 13:19:03.534 | INFO | "pinned" : " 29 GB" | 20260506 13:19:03.534 | INFO | } | 20260506 13:19:03.534 | INFO | Perf report for the last 5.13866 seconds: | 20260506 13:19:03.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:19:03.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:03.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:03.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:03.534 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 13:19:03.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:03.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:03.534 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260506 13:19:03.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:03.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:03.534 | INFO | Uploaded 36GiB to GPU in 2.04661s (17.5901GiB/s) | 20260506 13:19:03.534 | INFO | Downloaded 34GiB to GPU in 1.96874s (17.2699GiB/s) | 20260506 13:19:03.671 | INFO | Starting a round of Baz IO | 20260506 13:19:06.829 | INFO | Populating RTMetrics report | 20260506 13:19:06.829 | INFO | RTMetrics report populated | 20260506 13:19:06.829 | INFO | Opening tmp RTMetrics json file | 20260506 13:19:06.833 | INFO | Serializing RTMetrics to json | 20260506 13:19:06.834 | INFO | Serialization done, swapping tmp file now... | 20260506 13:19:06.834 | INFO | RTMetrics done | 20260506 13:19:07.346 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.675 seconds | 20260506 13:19:07.346 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.93% / 0.02% / 60.05% | 20260506 13:19:07.346 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.28122% / 3.17891e-05% / 5.5631e-05% | 20260506 13:19:07.346 | INFO | Baz IO overall: | 20260506 13:19:07.346 | INFO | BytesWritten : 840523220457 | 20260506 13:19:07.346 | INFO | NumEvents : 406498929928 | 20260506 13:19:08.673 | INFO | Analyzing chunk frames = [1658880,1659392) | 20260506 13:19:08.673 | INFO | MemInfo: { | 20260506 13:19:08.673 | INFO | "gpu" : " 17 GB", | 20260506 13:19:08.673 | INFO | "huge" : "", | 20260506 13:19:08.673 | INFO | "huge_shared" : " 27 GB", | 20260506 13:19:08.673 | INFO | "malloc" : " 14 GB", | 20260506 13:19:08.673 | INFO | "pinned" : " 29 GB" | 20260506 13:19:08.673 | INFO | } | 20260506 13:19:08.673 | INFO | Perf report for the last 5.13954 seconds: | 20260506 13:19:08.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:19:08.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:08.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:08.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:08.673 | INFO | ANALYSIS : 63.5%, 2.9, 95.5ms, 0 | 20260506 13:19:08.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:08.673 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 13:19:08.673 | INFO | PRE_HQ : 61.3%, 1.9, 60.9ms, 0 | 20260506 13:19:08.673 | INFO | BAZWRITER : 70.8%, 4.8, 175.7ms, 0 | 20260506 13:19:08.673 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 13:19:08.674 | INFO | Uploaded 36GiB to GPU in 2.09918s (17.1495GiB/s) | 20260506 13:19:08.674 | INFO | Downloaded 38GiB to GPU in 2.24524s (16.9247GiB/s) | 20260506 13:19:13.812 | INFO | Analyzing chunk frames = [1659392,1659904) | 20260506 13:19:13.812 | INFO | MemInfo: { | 20260506 13:19:13.812 | INFO | "gpu" : " 17 GB", | 20260506 13:19:13.812 | INFO | "huge" : "", | 20260506 13:19:13.812 | INFO | "huge_shared" : " 27 GB", | 20260506 13:19:13.812 | INFO | "malloc" : " 14 GB", | 20260506 13:19:13.812 | INFO | "pinned" : " 29 GB" | 20260506 13:19:13.812 | INFO | } | 20260506 13:19:13.812 | INFO | Perf report for the last 5.1392 seconds: | 20260506 13:19:13.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:19:13.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:13.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:13.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:13.813 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 13:19:13.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:13.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:13.813 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:19:13.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:13.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:13.813 | INFO | Uploaded 36GiB to GPU in 2.05906s (17.4837GiB/s) | 20260506 13:19:13.813 | INFO | Downloaded 34GiB to GPU in 1.93633s (17.559GiB/s) | 20260506 13:19:18.952 | INFO | Analyzing chunk frames = [1659904,1660416) | 20260506 13:19:18.952 | INFO | MemInfo: { | 20260506 13:19:18.952 | INFO | "gpu" : " 17 GB", | 20260506 13:19:18.952 | INFO | "huge" : "", | 20260506 13:19:18.952 | INFO | "huge_shared" : " 27 GB", | 20260506 13:19:18.952 | INFO | "malloc" : " 14 GB", | 20260506 13:19:18.952 | INFO | "pinned" : " 29 GB" | 20260506 13:19:18.952 | INFO | } | 20260506 13:19:18.952 | INFO | Perf report for the last 5.13936 seconds: | 20260506 13:19:18.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:19:18.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:18.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:18.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:18.952 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 13:19:18.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:18.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:18.952 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:19:18.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:18.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:18.952 | INFO | Uploaded 36GiB to GPU in 2.03721s (17.6713GiB/s) | 20260506 13:19:18.952 | INFO | Downloaded 34GiB to GPU in 1.94172s (17.5102GiB/s) | 20260506 13:19:24.091 | INFO | Analyzing chunk frames = [1660416,1660928) | 20260506 13:19:24.091 | INFO | MemInfo: { | 20260506 13:19:24.091 | INFO | "gpu" : " 17 GB", | 20260506 13:19:24.091 | INFO | "huge" : "", | 20260506 13:19:24.091 | INFO | "huge_shared" : " 27 GB", | 20260506 13:19:24.091 | INFO | "malloc" : " 14 GB", | 20260506 13:19:24.091 | INFO | "pinned" : " 29 GB" | 20260506 13:19:24.091 | INFO | } | 20260506 13:19:24.091 | INFO | Perf report for the last 5.13921 seconds: | 20260506 13:19:24.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:19:24.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:24.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:24.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:24.091 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 13:19:24.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:24.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:24.091 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:19:24.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:24.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:24.091 | INFO | Uploaded 36GiB to GPU in 2.01612s (17.8561GiB/s) | 20260506 13:19:24.092 | INFO | Downloaded 34GiB to GPU in 1.93831s (17.541GiB/s) | 20260506 13:19:29.231 | INFO | Analyzing chunk frames = [1660928,1661440) | 20260506 13:19:29.231 | INFO | MemInfo: { | 20260506 13:19:29.231 | INFO | "gpu" : " 17 GB", | 20260506 13:19:29.231 | INFO | "huge" : "", | 20260506 13:19:29.231 | INFO | "huge_shared" : " 27 GB", | 20260506 13:19:29.231 | INFO | "malloc" : " 14 GB", | 20260506 13:19:29.231 | INFO | "pinned" : " 29 GB" | 20260506 13:19:29.231 | INFO | } | 20260506 13:19:29.231 | INFO | Perf report for the last 5.13985 seconds: | 20260506 13:19:29.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:19:29.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:29.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:29.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:29.231 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 13:19:29.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:29.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:29.231 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:19:29.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:29.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:29.231 | INFO | Uploaded 36GiB to GPU in 1.99269s (18.0661GiB/s) | 20260506 13:19:29.231 | INFO | Downloaded 34GiB to GPU in 1.93618s (17.5603GiB/s) | 20260506 13:19:34.370 | INFO | Analyzing chunk frames = [1661440,1661952) | 20260506 13:19:34.370 | INFO | MemInfo: { | 20260506 13:19:34.370 | INFO | "gpu" : " 17 GB", | 20260506 13:19:34.370 | INFO | "huge" : "", | 20260506 13:19:34.370 | INFO | "huge_shared" : " 27 GB", | 20260506 13:19:34.370 | INFO | "malloc" : " 14 GB", | 20260506 13:19:34.370 | INFO | "pinned" : " 29 GB" | 20260506 13:19:34.370 | INFO | } | 20260506 13:19:34.370 | INFO | Perf report for the last 5.13911 seconds: | 20260506 13:19:34.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:19:34.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:34.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:34.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:34.370 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:19:34.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:34.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:34.370 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:19:34.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:34.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:34.370 | INFO | Uploaded 36GiB to GPU in 2.0918s (17.2101GiB/s) | 20260506 13:19:34.370 | INFO | Downloaded 34GiB to GPU in 1.95821s (17.3628GiB/s) | 20260506 13:19:39.509 | INFO | Analyzing chunk frames = [1661952,1662464) | 20260506 13:19:39.509 | INFO | MemInfo: { | 20260506 13:19:39.509 | INFO | "gpu" : " 17 GB", | 20260506 13:19:39.509 | INFO | "huge" : "", | 20260506 13:19:39.509 | INFO | "huge_shared" : " 27 GB", | 20260506 13:19:39.509 | INFO | "malloc" : " 14 GB", | 20260506 13:19:39.509 | INFO | "pinned" : " 29 GB" | 20260506 13:19:39.509 | INFO | } | 20260506 13:19:39.509 | INFO | Perf report for the last 5.13942 seconds: | 20260506 13:19:39.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:19:39.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:39.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:39.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:39.510 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:19:39.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:39.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:39.510 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 13:19:39.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:39.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:39.510 | INFO | Uploaded 36GiB to GPU in 1.99385s (18.0555GiB/s) | 20260506 13:19:39.510 | INFO | Downloaded 34GiB to GPU in 2.03155s (16.736GiB/s) | 20260506 13:19:44.648 | INFO | Analyzing chunk frames = [1662464,1662976) | 20260506 13:19:44.648 | INFO | MemInfo: { | 20260506 13:19:44.648 | INFO | "gpu" : " 17 GB", | 20260506 13:19:44.648 | INFO | "huge" : "", | 20260506 13:19:44.648 | INFO | "huge_shared" : " 27 GB", | 20260506 13:19:44.648 | INFO | "malloc" : " 14 GB", | 20260506 13:19:44.648 | INFO | "pinned" : " 29 GB" | 20260506 13:19:44.648 | INFO | } | 20260506 13:19:44.649 | INFO | Perf report for the last 5.13914 seconds: | 20260506 13:19:44.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:19:44.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:44.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:44.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:44.649 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 13:19:44.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:44.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:44.649 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 13:19:44.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:44.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:44.649 | INFO | Uploaded 36GiB to GPU in 2.06386s (17.4431GiB/s) | 20260506 13:19:44.649 | INFO | Downloaded 34GiB to GPU in 1.97097s (17.2504GiB/s) | 20260506 13:19:44.649 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:19:44.649 | INFO | Total mem usage: 16.8 GiB | 20260506 13:19:44.649 | INFO | All Time High Water Marks sum: 17.2 GiB (Savings: 2.01%) | 20260506 13:19:44.649 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -2.94%) | 20260506 13:19:44.649 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:19:44.649 | INFO | High Water Breakdown: | 20260506 13:19:44.649 | INFO | 61.8% : 10.1 GiB ( 96.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:19:44.649 | INFO | 38.1% : 6.23 GiB ( 93.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:19:44.649 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:19:44.649 | INFO | ------------------------------------------ | 20260506 13:19:44.649 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:19:44.649 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:19:44.649 | INFO | Total mem usage: 26.5 GiB | 20260506 13:19:44.649 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:19:44.649 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:19:44.649 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:19:44.649 | INFO | High Water Breakdown: | 20260506 13:19:44.649 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:19:44.649 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:19:44.649 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:19:44.649 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:19:44.649 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:19:44.649 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:19:44.649 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:19:44.649 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:19:44.649 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:19:44.649 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:19:44.650 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:19:44.650 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:19:44.650 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:19:44.650 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:19:44.650 | INFO | ------------------------------------------ | 20260506 13:19:44.650 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:19:44.650 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:19:44.650 | INFO | Total mem usage: 25.6 GiB | 20260506 13:19:44.650 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:19:44.650 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:19:44.650 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:19:44.650 | INFO | High Water Breakdown: | 20260506 13:19:44.650 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:19:44.650 | INFO | ------------------------------------------ | 20260506 13:19:44.650 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:19:44.650 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:19:44.650 | INFO | Total mem usage: 15.7 GiB | 20260506 13:19:44.650 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:19:44.650 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:19:44.650 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 13:19:44.650 | INFO | High Water Breakdown: | 20260506 13:19:44.650 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:19:44.650 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:19:44.650 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:19:44.650 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:19:44.650 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:19:44.650 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:19:44.650 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:19:44.650 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:19:44.650 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:19:44.650 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:19:44.650 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:19:44.650 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:19:44.650 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:19:44.650 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:19:44.650 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:19:44.650 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:19:44.650 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:19:44.650 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:19:44.650 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:19:44.650 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:19:44.650 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:19:44.650 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:19:44.650 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:19:44.650 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:19:44.650 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:19:44.650 | INFO | ------------------------------------------ | 20260506 13:19:44.650 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:19:44.650 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:19:44.650 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:19:44.650 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:19:44.650 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.65e+01, 1.36e+01 / 2.67e+00 | 20260506 13:19:44.650 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.03e-01, 1.03e-02 / 7.28e-03 | 20260506 13:19:44.650 | INFO | PerfReport: 11.56%, Baseline -- 3.66e+00 / 4.46e+00, 5.72e-01 / 2.42e-01 | 20260506 13:19:44.650 | INFO | PerfReport: 3.23%, Histogram -- 1.02e+00 / 1.50e+00, 1.88e-01 / 1.12e-01 | 20260506 13:19:44.650 | INFO | PerfReport: 15.92%, DME -- 4.03e+01 / 4.52e+01, 4.29e+00 / 3.12e+00 | 20260506 13:19:44.650 | INFO | PerfReport: 56.42%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.33e+00 / 1.86e+00 | 20260506 13:19:44.650 | INFO | PerfReport: 7.57%, PulseAccumulating -- 2.40e+00 / 3.02e+00, 4.26e-01 / 2.27e-01 | 20260506 13:19:44.650 | INFO | PerfReport: 3.70%, Metrics -- 1.17e+00 / 1.57e+00, 3.02e-01 / 1.59e-01 | 20260506 13:19:44.650 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.10e-01 / 4.71e-01, 6.47e-02 / 3.43e-02 | 20260506 13:19:44.650 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:19:44.650 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:19:44.650 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:19:44.650 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 13:19:44.650 | INFO | PerfReport: 21.58%, Upload -- 2.12e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260506 13:19:44.650 | INFO | PerfReport: 21.14%, Download -- 2.08e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 13:19:44.650 | INFO | PerfReport: 32.19%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 13:19:44.650 | INFO | PerfReport: 1.86%, LockUpload -- 1.83e+00 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 13:19:44.650 | INFO | PerfReport: 1.00%, LockDownload -- 9.79e-01 / 0.00e+00, 2.66e+00 / 0.00e+00 | 20260506 13:19:44.650 | INFO | PerfReport: 22.22%, LockCompute -- 2.18e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 13:19:44.650 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260506 13:19:44.779 | INFO | Starting a round of Baz IO | 20260506 13:19:47.930 | INFO | Populating RTMetrics report | 20260506 13:19:47.931 | INFO | RTMetrics report populated | 20260506 13:19:47.931 | INFO | Opening tmp RTMetrics json file | 20260506 13:19:48.145 | INFO | Serializing RTMetrics to json | 20260506 13:19:48.162 | INFO | Serialization done, swapping tmp file now... | 20260506 13:19:48.166 | INFO | RTMetrics done | 20260506 13:19:48.776 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.996 seconds | 20260506 13:19:48.776 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.91% / 0.02% / 60.07% | 20260506 13:19:48.776 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.27221% / 1.19209e-05% / 6.75519e-05% | 20260506 13:19:48.776 | INFO | Baz IO overall: | 20260506 13:19:48.777 | INFO | BytesWritten : 842788620535 | 20260506 13:19:48.777 | INFO | NumEvents : 407597763851 | 20260506 13:19:49.788 | INFO | Analyzing chunk frames = [1662976,1663488) | 20260506 13:19:49.788 | INFO | MemInfo: { | 20260506 13:19:49.788 | INFO | "gpu" : " 17 GB", | 20260506 13:19:49.788 | INFO | "huge" : "", | 20260506 13:19:49.788 | INFO | "huge_shared" : " 27 GB", | 20260506 13:19:49.788 | INFO | "malloc" : " 14 GB", | 20260506 13:19:49.788 | INFO | "pinned" : " 29 GB" | 20260506 13:19:49.788 | INFO | } | 20260506 13:19:49.788 | INFO | Perf report for the last 5.13922 seconds: | 20260506 13:19:49.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:19:49.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:49.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:49.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:49.788 | INFO | ANALYSIS : 63.3%, 2.9, 95.6ms, 0 | 20260506 13:19:49.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:49.788 | INFO | RT_METRICS : 48.5%, 1.0, 25.4ms, 0 | 20260506 13:19:49.788 | INFO | PRE_HQ : 61.0%, 1.9, 59.8ms, 0 | 20260506 13:19:49.788 | INFO | BAZWRITER : 77.2%, 4.8, 191.8ms, 0 | 20260506 13:19:49.788 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 13:19:49.788 | INFO | Uploaded 36GiB to GPU in 2.10035s (17.14GiB/s) | 20260506 13:19:49.789 | INFO | Downloaded 38GiB to GPU in 2.23664s (16.9897GiB/s) | 20260506 13:19:54.927 | INFO | Analyzing chunk frames = [1663488,1664000) | 20260506 13:19:54.927 | INFO | MemInfo: { | 20260506 13:19:54.927 | INFO | "gpu" : " 17 GB", | 20260506 13:19:54.927 | INFO | "huge" : "", | 20260506 13:19:54.927 | INFO | "huge_shared" : " 27 GB", | 20260506 13:19:54.927 | INFO | "malloc" : " 14 GB", | 20260506 13:19:54.927 | INFO | "pinned" : " 29 GB" | 20260506 13:19:54.927 | INFO | } | 20260506 13:19:54.927 | INFO | Perf report for the last 5.13936 seconds: | 20260506 13:19:54.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:19:54.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:54.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:54.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:54.927 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:19:54.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:54.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:54.927 | INFO | PRE_HQ : 28.9%, 1.0, 15.5ms, 0 | 20260506 13:19:54.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:54.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:19:54.928 | INFO | Uploaded 36GiB to GPU in 2.01116s (17.9001GiB/s) | 20260506 13:19:54.928 | INFO | Downloaded 34GiB to GPU in 1.91717s (17.7345GiB/s) | 20260506 13:20:00.066 | INFO | Analyzing chunk frames = [1664000,1664512) | 20260506 13:20:00.066 | INFO | MemInfo: { | 20260506 13:20:00.066 | INFO | "gpu" : " 17 GB", | 20260506 13:20:00.066 | INFO | "huge" : "", | 20260506 13:20:00.066 | INFO | "huge_shared" : " 27 GB", | 20260506 13:20:00.066 | INFO | "malloc" : " 14 GB", | 20260506 13:20:00.066 | INFO | "pinned" : " 29 GB" | 20260506 13:20:00.066 | INFO | } | 20260506 13:20:00.067 | INFO | Perf report for the last 5.13944 seconds: | 20260506 13:20:00.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:20:00.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:00.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:00.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:00.067 | INFO | ANALYSIS : 61.4%, 3.0, 95.1ms, 0 | 20260506 13:20:00.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:00.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:00.067 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:20:00.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:00.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:00.067 | INFO | Uploaded 36GiB to GPU in 1.97813s (18.199GiB/s) | 20260506 13:20:00.067 | INFO | Downloaded 34GiB to GPU in 1.92174s (17.6923GiB/s) | 20260506 13:20:05.206 | INFO | Analyzing chunk frames = [1664512,1665024) | 20260506 13:20:05.206 | INFO | MemInfo: { | 20260506 13:20:05.206 | INFO | "gpu" : " 17 GB", | 20260506 13:20:05.206 | INFO | "huge" : "", | 20260506 13:20:05.206 | INFO | "huge_shared" : " 27 GB", | 20260506 13:20:05.206 | INFO | "malloc" : " 14 GB", | 20260506 13:20:05.206 | INFO | "pinned" : " 29 GB" | 20260506 13:20:05.206 | INFO | } | 20260506 13:20:05.206 | INFO | Perf report for the last 5.1393 seconds: | 20260506 13:20:05.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:20:05.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:05.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:05.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:05.206 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:20:05.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:05.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:05.206 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 13:20:05.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:05.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:05.207 | INFO | Uploaded 36GiB to GPU in 2.00254s (17.9771GiB/s) | 20260506 13:20:05.207 | INFO | Downloaded 34GiB to GPU in 1.90655s (17.8333GiB/s) | 20260506 13:20:10.345 | INFO | Analyzing chunk frames = [1665024,1665536) | 20260506 13:20:10.345 | INFO | MemInfo: { | 20260506 13:20:10.345 | INFO | "gpu" : " 17 GB", | 20260506 13:20:10.345 | INFO | "huge" : "", | 20260506 13:20:10.345 | INFO | "huge_shared" : " 27 GB", | 20260506 13:20:10.345 | INFO | "malloc" : " 14 GB", | 20260506 13:20:10.345 | INFO | "pinned" : " 29 GB" | 20260506 13:20:10.345 | INFO | } | 20260506 13:20:10.345 | INFO | Perf report for the last 5.13944 seconds: | 20260506 13:20:10.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:20:10.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:10.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:10.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:10.345 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 13:20:10.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:10.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:10.346 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:20:10.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:10.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:10.346 | INFO | Uploaded 36GiB to GPU in 2.0785s (17.3202GiB/s) | 20260506 13:20:10.346 | INFO | Downloaded 34GiB to GPU in 1.94633s (17.4688GiB/s) | 20260506 13:20:15.485 | INFO | Analyzing chunk frames = [1665536,1666048) | 20260506 13:20:15.485 | INFO | MemInfo: { | 20260506 13:20:15.485 | INFO | "gpu" : " 17 GB", | 20260506 13:20:15.485 | INFO | "huge" : "", | 20260506 13:20:15.485 | INFO | "huge_shared" : " 27 GB", | 20260506 13:20:15.485 | INFO | "malloc" : " 14 GB", | 20260506 13:20:15.485 | INFO | "pinned" : " 29 GB" | 20260506 13:20:15.485 | INFO | } | 20260506 13:20:15.485 | INFO | Perf report for the last 5.13939 seconds: | 20260506 13:20:15.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:20:15.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:15.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:15.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:15.485 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:20:15.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:15.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:15.485 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 13:20:15.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:15.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:15.485 | INFO | Uploaded 36GiB to GPU in 2.06098s (17.4674GiB/s) | 20260506 13:20:15.486 | INFO | Downloaded 34GiB to GPU in 1.92818s (17.6332GiB/s) | 20260506 13:20:20.625 | INFO | Analyzing chunk frames = [1666048,1666560) | 20260506 13:20:20.625 | INFO | MemInfo: { | 20260506 13:20:20.625 | INFO | "gpu" : " 17 GB", | 20260506 13:20:20.625 | INFO | "huge" : "", | 20260506 13:20:20.625 | INFO | "huge_shared" : " 27 GB", | 20260506 13:20:20.625 | INFO | "malloc" : " 14 GB", | 20260506 13:20:20.625 | INFO | "pinned" : " 29 GB" | 20260506 13:20:20.625 | INFO | } | 20260506 13:20:20.625 | INFO | Perf report for the last 5.13989 seconds: | 20260506 13:20:20.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:20:20.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:20.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:20.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:20.625 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:20:20.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:20.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:20.625 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:20:20.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:20.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:20.625 | INFO | Uploaded 36GiB to GPU in 2.03522s (17.6885GiB/s) | 20260506 13:20:20.625 | INFO | Downloaded 34GiB to GPU in 1.98697s (17.1115GiB/s) | 20260506 13:20:25.763 | INFO | Analyzing chunk frames = [1666560,1667072) | 20260506 13:20:25.763 | INFO | MemInfo: { | 20260506 13:20:25.763 | INFO | "gpu" : " 17 GB", | 20260506 13:20:25.763 | INFO | "huge" : "", | 20260506 13:20:25.763 | INFO | "huge_shared" : " 27 GB", | 20260506 13:20:25.763 | INFO | "malloc" : " 14 GB", | 20260506 13:20:25.763 | INFO | "pinned" : " 29 GB" | 20260506 13:20:25.764 | INFO | } | 20260506 13:20:25.764 | INFO | Perf report for the last 5.13885 seconds: | 20260506 13:20:25.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:20:25.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:25.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:25.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:25.764 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 13:20:25.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:25.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:25.764 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:20:25.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:25.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:25.764 | INFO | Uploaded 36GiB to GPU in 2.06295s (17.4507GiB/s) | 20260506 13:20:25.764 | INFO | Downloaded 34GiB to GPU in 1.9602s (17.3452GiB/s) | 20260506 13:20:25.892 | INFO | Starting a round of Baz IO | 20260506 13:20:29.047 | INFO | Populating RTMetrics report | 20260506 13:20:29.047 | INFO | RTMetrics report populated | 20260506 13:20:29.047 | INFO | Opening tmp RTMetrics json file | 20260506 13:20:29.091 | INFO | Serializing RTMetrics to json | 20260506 13:20:29.094 | INFO | Serialization done, swapping tmp file now... | 20260506 13:20:29.097 | INFO | RTMetrics done | 20260506 13:20:29.565 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.672 seconds | 20260506 13:20:29.565 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.89% / 0.02% / 60.09% | 20260506 13:20:29.565 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.25857% / 2.78155e-05% / 5.16574e-05% | 20260506 13:20:29.565 | INFO | Baz IO overall: | 20260506 13:20:29.565 | INFO | BytesWritten : 845053039295 | 20260506 13:20:29.565 | INFO | NumEvents : 408696012175 | 20260506 13:20:30.903 | INFO | Analyzing chunk frames = [1667072,1667584) | 20260506 13:20:30.903 | INFO | MemInfo: { | 20260506 13:20:30.903 | INFO | "gpu" : " 17 GB", | 20260506 13:20:30.903 | INFO | "huge" : "", | 20260506 13:20:30.903 | INFO | "huge_shared" : " 27 GB", | 20260506 13:20:30.903 | INFO | "malloc" : " 14 GB", | 20260506 13:20:30.903 | INFO | "pinned" : " 29 GB" | 20260506 13:20:30.903 | INFO | } | 20260506 13:20:30.903 | INFO | Perf report for the last 5.13925 seconds: | 20260506 13:20:30.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:20:30.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:30.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:30.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:30.903 | INFO | ANALYSIS : 63.3%, 2.9, 95.9ms, 0 | 20260506 13:20:30.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:30.903 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260506 13:20:30.903 | INFO | PRE_HQ : 61.1%, 1.9, 60.1ms, 0 | 20260506 13:20:30.903 | INFO | BAZWRITER : 70.9%, 4.8, 173.7ms, 0 | 20260506 13:20:30.903 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 13:20:30.903 | INFO | Uploaded 36GiB to GPU in 2.09818s (17.1577GiB/s) | 20260506 13:20:30.903 | INFO | Downloaded 38GiB to GPU in 2.23889s (16.9727GiB/s) | 20260506 13:20:36.042 | INFO | Analyzing chunk frames = [1667584,1668096) | 20260506 13:20:36.042 | INFO | MemInfo: { | 20260506 13:20:36.042 | INFO | "gpu" : " 17 GB", | 20260506 13:20:36.042 | INFO | "huge" : "", | 20260506 13:20:36.042 | INFO | "huge_shared" : " 27 GB", | 20260506 13:20:36.042 | INFO | "malloc" : " 14 GB", | 20260506 13:20:36.042 | INFO | "pinned" : " 29 GB" | 20260506 13:20:36.042 | INFO | } | 20260506 13:20:36.042 | INFO | Perf report for the last 5.13922 seconds: | 20260506 13:20:36.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:20:36.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:36.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:36.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:36.042 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 13:20:36.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:36.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:36.042 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:20:36.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:36.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:36.042 | INFO | Uploaded 36GiB to GPU in 2.06579s (17.4268GiB/s) | 20260506 13:20:36.042 | INFO | Downloaded 34GiB to GPU in 1.94032s (17.5229GiB/s) | 20260506 13:20:41.181 | INFO | Analyzing chunk frames = [1668096,1668608) | 20260506 13:20:41.181 | INFO | MemInfo: { | 20260506 13:20:41.181 | INFO | "gpu" : " 17 GB", | 20260506 13:20:41.181 | INFO | "huge" : "", | 20260506 13:20:41.181 | INFO | "huge_shared" : " 27 GB", | 20260506 13:20:41.181 | INFO | "malloc" : " 14 GB", | 20260506 13:20:41.181 | INFO | "pinned" : " 29 GB" | 20260506 13:20:41.182 | INFO | } | 20260506 13:20:41.182 | INFO | Perf report for the last 5.1396 seconds: | 20260506 13:20:41.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:20:41.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:41.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:41.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:41.182 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260506 13:20:41.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:41.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:41.182 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:20:41.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:41.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:41.182 | INFO | Uploaded 36GiB to GPU in 2.03137s (17.7221GiB/s) | 20260506 13:20:41.182 | INFO | Downloaded 34GiB to GPU in 1.9438s (17.4915GiB/s) | 20260506 13:20:46.321 | INFO | Analyzing chunk frames = [1668608,1669120) | 20260506 13:20:46.321 | INFO | MemInfo: { | 20260506 13:20:46.321 | INFO | "gpu" : " 17 GB", | 20260506 13:20:46.321 | INFO | "huge" : "", | 20260506 13:20:46.321 | INFO | "huge_shared" : " 27 GB", | 20260506 13:20:46.321 | INFO | "malloc" : " 14 GB", | 20260506 13:20:46.321 | INFO | "pinned" : " 29 GB" | 20260506 13:20:46.321 | INFO | } | 20260506 13:20:46.321 | INFO | Perf report for the last 5.13899 seconds: | 20260506 13:20:46.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:20:46.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:46.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:46.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:46.321 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:20:46.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:46.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:46.321 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:20:46.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:46.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:46.321 | INFO | Uploaded 36GiB to GPU in 2.00832s (17.9254GiB/s) | 20260506 13:20:46.321 | INFO | Downloaded 34GiB to GPU in 1.9404s (17.5222GiB/s) | 20260506 13:20:51.460 | INFO | Analyzing chunk frames = [1669120,1669632) | 20260506 13:20:51.460 | INFO | MemInfo: { | 20260506 13:20:51.460 | INFO | "gpu" : " 17 GB", | 20260506 13:20:51.460 | INFO | "huge" : "", | 20260506 13:20:51.460 | INFO | "huge_shared" : " 27 GB", | 20260506 13:20:51.460 | INFO | "malloc" : " 14 GB", | 20260506 13:20:51.460 | INFO | "pinned" : " 29 GB" | 20260506 13:20:51.460 | INFO | } | 20260506 13:20:51.460 | INFO | Perf report for the last 5.1393 seconds: | 20260506 13:20:51.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:20:51.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:51.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:51.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:51.460 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 13:20:51.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:51.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:51.460 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:20:51.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:51.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:51.460 | INFO | Uploaded 36GiB to GPU in 1.99053s (18.0856GiB/s) | 20260506 13:20:51.460 | INFO | Downloaded 34GiB to GPU in 1.94559s (17.4754GiB/s) | 20260506 13:20:56.599 | INFO | Analyzing chunk frames = [1669632,1670144) | 20260506 13:20:56.599 | INFO | MemInfo: { | 20260506 13:20:56.599 | INFO | "gpu" : " 17 GB", | 20260506 13:20:56.599 | INFO | "huge" : "", | 20260506 13:20:56.599 | INFO | "huge_shared" : " 27 GB", | 20260506 13:20:56.599 | INFO | "malloc" : " 14 GB", | 20260506 13:20:56.599 | INFO | "pinned" : " 29 GB" | 20260506 13:20:56.599 | INFO | } | 20260506 13:20:56.599 | INFO | Perf report for the last 5.13928 seconds: | 20260506 13:20:56.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:20:56.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:56.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:56.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:56.600 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:20:56.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:56.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:56.600 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260506 13:20:56.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:56.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:20:56.600 | INFO | Uploaded 36GiB to GPU in 2.085s (17.2662GiB/s) | 20260506 13:20:56.600 | INFO | Downloaded 34GiB to GPU in 1.95939s (17.3523GiB/s) | 20260506 13:21:01.739 | INFO | Analyzing chunk frames = [1670144,1670656) | 20260506 13:21:01.739 | INFO | MemInfo: { | 20260506 13:21:01.739 | INFO | "gpu" : " 17 GB", | 20260506 13:21:01.739 | INFO | "huge" : "", | 20260506 13:21:01.739 | INFO | "huge_shared" : " 27 GB", | 20260506 13:21:01.739 | INFO | "malloc" : " 14 GB", | 20260506 13:21:01.739 | INFO | "pinned" : " 29 GB" | 20260506 13:21:01.739 | INFO | } | 20260506 13:21:01.739 | INFO | Perf report for the last 5.1395 seconds: | 20260506 13:21:01.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:21:01.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:01.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:01.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:01.739 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 13:21:01.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:01.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:01.739 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:21:01.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:01.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:01.739 | INFO | Uploaded 36GiB to GPU in 1.97779s (18.2021GiB/s) | 20260506 13:21:01.740 | INFO | Downloaded 34GiB to GPU in 2.02367s (16.8012GiB/s) | 20260506 13:21:06.878 | INFO | Analyzing chunk frames = [1670656,1671168) | 20260506 13:21:06.878 | INFO | MemInfo: { | 20260506 13:21:06.878 | INFO | "gpu" : " 17 GB", | 20260506 13:21:06.878 | INFO | "huge" : "", | 20260506 13:21:06.878 | INFO | "huge_shared" : " 27 GB", | 20260506 13:21:06.878 | INFO | "malloc" : " 14 GB", | 20260506 13:21:06.878 | INFO | "pinned" : " 29 GB" | 20260506 13:21:06.878 | INFO | } | 20260506 13:21:06.878 | INFO | Perf report for the last 5.13946 seconds: | 20260506 13:21:06.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:21:06.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:06.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:06.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:06.878 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 13:21:06.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:06.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:06.878 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 13:21:06.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:06.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:06.879 | INFO | Uploaded 36GiB to GPU in 2.0623s (17.4562GiB/s) | 20260506 13:21:06.879 | INFO | Downloaded 34GiB to GPU in 1.96419s (17.3099GiB/s) | 20260506 13:21:06.879 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:21:06.879 | INFO | Total mem usage: 17.4 GiB | 20260506 13:21:06.879 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:21:06.879 | INFO | Recent High Water Marks Sum: 17.4 GiB (Savings: 0.0442%) | 20260506 13:21:06.879 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:21:06.879 | INFO | High Water Breakdown: | 20260506 13:21:06.879 | INFO | 62.7% : 10.9 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:21:06.879 | INFO | 37.2% : 6.48 GiB ( 96.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:21:06.879 | INFO | 0.114% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:21:06.879 | INFO | ------------------------------------------ | 20260506 13:21:06.879 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:21:06.879 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:21:06.879 | INFO | Total mem usage: 26.5 GiB | 20260506 13:21:06.879 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:21:06.879 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:21:06.879 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:21:06.879 | INFO | High Water Breakdown: | 20260506 13:21:06.879 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:21:06.879 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:21:06.879 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:21:06.879 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:21:06.879 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:21:06.879 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:21:06.879 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:21:06.879 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:21:06.879 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:21:06.879 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:21:06.879 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:21:06.879 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:21:06.879 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:21:06.879 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:21:06.879 | INFO | ------------------------------------------ | 20260506 13:21:06.879 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:21:06.879 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:21:06.879 | INFO | Total mem usage: 25.6 GiB | 20260506 13:21:06.879 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:21:06.879 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:21:06.879 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:21:06.879 | INFO | High Water Breakdown: | 20260506 13:21:06.879 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:21:06.879 | INFO | ------------------------------------------ | 20260506 13:21:06.879 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:21:06.879 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:21:06.879 | INFO | Total mem usage: 15.7 GiB | 20260506 13:21:06.879 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:21:06.879 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:21:06.879 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:21:06.879 | INFO | High Water Breakdown: | 20260506 13:21:06.879 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:21:06.879 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:21:06.879 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:21:06.879 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:21:06.879 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:21:06.879 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:21:06.879 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:21:06.879 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:21:06.879 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:21:06.879 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:21:06.879 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:21:06.879 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:21:06.879 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:21:06.879 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:21:06.879 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:21:06.879 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:21:06.879 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:21:06.879 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:21:06.879 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:21:06.879 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:21:06.879 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:21:06.879 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:21:06.879 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:21:06.879 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:21:06.879 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:21:06.879 | INFO | ------------------------------------------ | 20260506 13:21:06.879 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:21:06.879 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:21:06.879 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:21:06.879 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:21:06.879 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.59e+01, 1.32e+01 / 2.43e+00 | 20260506 13:21:06.879 | INFO | PerfReport: 0.29%, AggregateStats -- 9.32e-02 / 1.01e-01, 3.16e-02 / 5.29e-03 | 20260506 13:21:06.879 | INFO | PerfReport: 11.56%, Baseline -- 3.66e+00 / 4.44e+00, 5.73e-01 / 3.76e-01 | 20260506 13:21:06.879 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.44e+00, 1.88e-01 / 1.57e-01 | 20260506 13:21:06.880 | INFO | PerfReport: 15.81%, DME -- 4.00e+01 / 4.43e+01, 4.01e+00 / 2.66e+00 | 20260506 13:21:06.880 | INFO | PerfReport: 56.58%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.44e+00 / 1.81e+00 | 20260506 13:21:06.880 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.38e+00 / 3.01e+00, 4.14e-01 / 2.56e-01 | 20260506 13:21:06.880 | INFO | PerfReport: 3.69%, Metrics -- 1.17e+00 / 1.56e+00, 3.00e-01 / 1.78e-01 | 20260506 13:21:06.880 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.11e-01 / 4.68e-01, 6.48e-02 / 2.73e-02 | 20260506 13:21:06.880 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:21:06.880 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:21:06.880 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:21:06.880 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 13:21:06.880 | INFO | PerfReport: 21.63%, Upload -- 2.13e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260506 13:21:06.880 | INFO | PerfReport: 21.04%, Download -- 2.07e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 13:21:06.880 | INFO | PerfReport: 32.21%, Compute -- 3.17e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 13:21:06.880 | INFO | PerfReport: 1.85%, LockUpload -- 1.82e+00 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260506 13:21:06.880 | INFO | PerfReport: 0.96%, LockDownload -- 9.46e-01 / 0.00e+00, 2.57e+00 / 0.00e+00 | 20260506 13:21:06.880 | INFO | PerfReport: 22.28%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 13:21:06.880 | INFO | PerfReport: 0.02%, Misc -- 1.57e-02 / 0.00e+00, 2.75e-02 / 0.00e+00 | 20260506 13:21:07.007 | INFO | Starting a round of Baz IO | 20260506 13:21:10.166 | INFO | Populating RTMetrics report | 20260506 13:21:10.167 | INFO | RTMetrics report populated | 20260506 13:21:10.167 | INFO | Opening tmp RTMetrics json file | 20260506 13:21:10.217 | INFO | Serializing RTMetrics to json | 20260506 13:21:10.219 | INFO | Serialization done, swapping tmp file now... | 20260506 13:21:10.219 | INFO | RTMetrics done | 20260506 13:21:10.655 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.648 seconds | 20260506 13:21:10.655 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.87% / 0.02% / 60.11% | 20260506 13:21:10.655 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.25548% / 4.37101e-05% / 3.97364e-05% | 20260506 13:21:10.655 | INFO | Baz IO overall: | 20260506 13:21:10.655 | INFO | BytesWritten : 847320335285 | 20260506 13:21:10.655 | INFO | NumEvents : 409796118611 | 20260506 13:21:12.017 | INFO | Analyzing chunk frames = [1671168,1671680) | 20260506 13:21:12.017 | INFO | MemInfo: { | 20260506 13:21:12.017 | INFO | "gpu" : " 17 GB", | 20260506 13:21:12.017 | INFO | "huge" : "", | 20260506 13:21:12.017 | INFO | "huge_shared" : " 27 GB", | 20260506 13:21:12.017 | INFO | "malloc" : " 14 GB", | 20260506 13:21:12.017 | INFO | "pinned" : " 29 GB" | 20260506 13:21:12.017 | INFO | } | 20260506 13:21:12.017 | INFO | Perf report for the last 5.13916 seconds: | 20260506 13:21:12.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:21:12.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:12.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:12.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:12.018 | INFO | ANALYSIS : 63.4%, 2.9, 95.6ms, 0 | 20260506 13:21:12.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:12.018 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 13:21:12.018 | INFO | PRE_HQ : 61.2%, 1.9, 59.9ms, 0 | 20260506 13:21:12.018 | INFO | BAZWRITER : 70.9%, 4.8, 176.1ms, 0 | 20260506 13:21:12.018 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 13:21:12.018 | INFO | Uploaded 36GiB to GPU in 2.11297s (17.0376GiB/s) | 20260506 13:21:12.018 | INFO | Downloaded 38GiB to GPU in 2.24752s (16.9075GiB/s) | 20260506 13:21:17.157 | INFO | Analyzing chunk frames = [1671680,1672192) | 20260506 13:21:17.157 | INFO | MemInfo: { | 20260506 13:21:17.157 | INFO | "gpu" : " 17 GB", | 20260506 13:21:17.157 | INFO | "huge" : "", | 20260506 13:21:17.157 | INFO | "huge_shared" : " 27 GB", | 20260506 13:21:17.157 | INFO | "malloc" : " 14 GB", | 20260506 13:21:17.157 | INFO | "pinned" : " 29 GB" | 20260506 13:21:17.157 | INFO | } | 20260506 13:21:17.157 | INFO | Perf report for the last 5.13953 seconds: | 20260506 13:21:17.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:21:17.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:17.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:17.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:17.157 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 13:21:17.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:17.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:17.157 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:21:17.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:17.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:17.157 | INFO | Uploaded 36GiB to GPU in 2.06897s (17.4GiB/s) | 20260506 13:21:17.157 | INFO | Downloaded 34GiB to GPU in 1.94365s (17.4929GiB/s) | 20260506 13:21:22.296 | INFO | Analyzing chunk frames = [1672192,1672704) | 20260506 13:21:22.296 | INFO | MemInfo: { | 20260506 13:21:22.296 | INFO | "gpu" : " 17 GB", | 20260506 13:21:22.296 | INFO | "huge" : "", | 20260506 13:21:22.296 | INFO | "huge_shared" : " 27 GB", | 20260506 13:21:22.296 | INFO | "malloc" : " 14 GB", | 20260506 13:21:22.296 | INFO | "pinned" : " 29 GB" | 20260506 13:21:22.296 | INFO | } | 20260506 13:21:22.296 | INFO | Perf report for the last 5.13931 seconds: | 20260506 13:21:22.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:21:22.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:22.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:22.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:22.296 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 13:21:22.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:22.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:22.296 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 13:21:22.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:22.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:22.297 | INFO | Uploaded 36GiB to GPU in 2.01503s (17.8657GiB/s) | 20260506 13:21:22.297 | INFO | Downloaded 34GiB to GPU in 1.94673s (17.4652GiB/s) | 20260506 13:21:27.435 | INFO | Analyzing chunk frames = [1672704,1673216) | 20260506 13:21:27.436 | INFO | MemInfo: { | 20260506 13:21:27.436 | INFO | "gpu" : " 17 GB", | 20260506 13:21:27.436 | INFO | "huge" : "", | 20260506 13:21:27.436 | INFO | "huge_shared" : " 27 GB", | 20260506 13:21:27.436 | INFO | "malloc" : " 14 GB", | 20260506 13:21:27.436 | INFO | "pinned" : " 29 GB" | 20260506 13:21:27.436 | INFO | } | 20260506 13:21:27.436 | INFO | Perf report for the last 5.1394 seconds: | 20260506 13:21:27.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:21:27.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:27.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:27.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:27.436 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:21:27.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:27.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:27.436 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:21:27.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:27.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:27.436 | INFO | Uploaded 36GiB to GPU in 2.00976s (17.9126GiB/s) | 20260506 13:21:27.436 | INFO | Downloaded 34GiB to GPU in 1.93702s (17.5528GiB/s) | 20260506 13:21:32.575 | INFO | Analyzing chunk frames = [1673216,1673728) | 20260506 13:21:32.575 | INFO | MemInfo: { | 20260506 13:21:32.575 | INFO | "gpu" : " 17 GB", | 20260506 13:21:32.575 | INFO | "huge" : "", | 20260506 13:21:32.575 | INFO | "huge_shared" : " 27 GB", | 20260506 13:21:32.575 | INFO | "malloc" : " 14 GB", | 20260506 13:21:32.575 | INFO | "pinned" : " 29 GB" | 20260506 13:21:32.575 | INFO | } | 20260506 13:21:32.575 | INFO | Perf report for the last 5.13892 seconds: | 20260506 13:21:32.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:21:32.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:32.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:32.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:32.575 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 13:21:32.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:32.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:32.575 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260506 13:21:32.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:32.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:32.575 | INFO | Uploaded 36GiB to GPU in 2.01581s (17.8588GiB/s) | 20260506 13:21:32.575 | INFO | Downloaded 34GiB to GPU in 1.94613s (17.4705GiB/s) | 20260506 13:21:37.714 | INFO | Analyzing chunk frames = [1673728,1674240) | 20260506 13:21:37.714 | INFO | MemInfo: { | 20260506 13:21:37.714 | INFO | "gpu" : " 17 GB", | 20260506 13:21:37.714 | INFO | "huge" : "", | 20260506 13:21:37.714 | INFO | "huge_shared" : " 27 GB", | 20260506 13:21:37.714 | INFO | "malloc" : " 14 GB", | 20260506 13:21:37.714 | INFO | "pinned" : " 29 GB" | 20260506 13:21:37.714 | INFO | } | 20260506 13:21:37.714 | INFO | Perf report for the last 5.13956 seconds: | 20260506 13:21:37.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:21:37.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:37.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:37.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:37.714 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 13:21:37.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:37.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:37.714 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:21:37.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:37.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:37.715 | INFO | Uploaded 36GiB to GPU in 2.09215s (17.2072GiB/s) | 20260506 13:21:37.715 | INFO | Downloaded 34GiB to GPU in 1.96131s (17.3354GiB/s) | 20260506 13:21:42.854 | INFO | Analyzing chunk frames = [1674240,1674752) | 20260506 13:21:42.854 | INFO | MemInfo: { | 20260506 13:21:42.854 | INFO | "gpu" : " 17 GB", | 20260506 13:21:42.854 | INFO | "huge" : "", | 20260506 13:21:42.854 | INFO | "huge_shared" : " 27 GB", | 20260506 13:21:42.854 | INFO | "malloc" : " 14 GB", | 20260506 13:21:42.854 | INFO | "pinned" : " 29 GB" | 20260506 13:21:42.854 | INFO | } | 20260506 13:21:42.854 | INFO | Perf report for the last 5.14012 seconds: | 20260506 13:21:42.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:21:42.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:42.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:42.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:42.854 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:21:42.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:42.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:42.854 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 13:21:42.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:42.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:42.855 | INFO | Uploaded 36GiB to GPU in 1.9875s (18.1133GiB/s) | 20260506 13:21:42.855 | INFO | Downloaded 34GiB to GPU in 2.02985s (16.75GiB/s) | 20260506 13:21:47.993 | INFO | Analyzing chunk frames = [1674752,1675264) | 20260506 13:21:47.993 | INFO | MemInfo: { | 20260506 13:21:47.993 | INFO | "gpu" : " 17 GB", | 20260506 13:21:47.993 | INFO | "huge" : "", | 20260506 13:21:47.993 | INFO | "huge_shared" : " 27 GB", | 20260506 13:21:47.993 | INFO | "malloc" : " 14 GB", | 20260506 13:21:47.993 | INFO | "pinned" : " 29 GB" | 20260506 13:21:47.993 | INFO | } | 20260506 13:21:47.993 | INFO | Perf report for the last 5.1387 seconds: | 20260506 13:21:47.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:21:47.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:47.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:47.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:47.993 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 13:21:47.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:47.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:47.993 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 13:21:47.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:47.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:47.993 | INFO | Uploaded 36GiB to GPU in 2.04181s (17.6314GiB/s) | 20260506 13:21:47.993 | INFO | Downloaded 34GiB to GPU in 1.97335s (17.2296GiB/s) | 20260506 13:21:48.128 | INFO | Starting a round of Baz IO | 20260506 13:21:51.293 | INFO | Populating RTMetrics report | 20260506 13:21:51.293 | INFO | RTMetrics report populated | 20260506 13:21:51.293 | INFO | Opening tmp RTMetrics json file | 20260506 13:21:51.324 | INFO | Serializing RTMetrics to json | 20260506 13:21:51.347 | INFO | Serialization done, swapping tmp file now... | 20260506 13:21:51.363 | INFO | RTMetrics done | 20260506 13:21:52.043 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.914 seconds | 20260506 13:21:52.043 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.85% / 0.02% / 60.13% | 20260506 13:21:52.043 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.23914% / 1.19209e-05% / 3.97364e-05% | 20260506 13:21:52.044 | INFO | Baz IO overall: | 20260506 13:21:52.044 | INFO | BytesWritten : 849587032052 | 20260506 13:21:52.044 | INFO | NumEvents : 410895426017 | 20260506 13:21:53.132 | INFO | Analyzing chunk frames = [1675264,1675776) | 20260506 13:21:53.132 | INFO | MemInfo: { | 20260506 13:21:53.132 | INFO | "gpu" : " 17 GB", | 20260506 13:21:53.132 | INFO | "huge" : "", | 20260506 13:21:53.132 | INFO | "huge_shared" : " 27 GB", | 20260506 13:21:53.132 | INFO | "malloc" : " 14 GB", | 20260506 13:21:53.132 | INFO | "pinned" : " 29 GB" | 20260506 13:21:53.132 | INFO | } | 20260506 13:21:53.132 | INFO | Perf report for the last 5.13917 seconds: | 20260506 13:21:53.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:21:53.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:53.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:53.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:53.132 | INFO | ANALYSIS : 63.6%, 2.9, 95.7ms, 0 | 20260506 13:21:53.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:53.132 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260506 13:21:53.132 | INFO | PRE_HQ : 61.3%, 1.9, 60.5ms, 0 | 20260506 13:21:53.132 | INFO | BAZWRITER : 75.6%, 4.8, 186.1ms, 0 | 20260506 13:21:53.132 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 13:21:53.133 | INFO | Uploaded 36GiB to GPU in 2.12693s (16.9258GiB/s) | 20260506 13:21:53.133 | INFO | Downloaded 38GiB to GPU in 2.22682s (17.0647GiB/s) | 20260506 13:21:58.271 | INFO | Analyzing chunk frames = [1675776,1676288) | 20260506 13:21:58.271 | INFO | MemInfo: { | 20260506 13:21:58.271 | INFO | "gpu" : " 17 GB", | 20260506 13:21:58.271 | INFO | "huge" : "", | 20260506 13:21:58.272 | INFO | "huge_shared" : " 27 GB", | 20260506 13:21:58.272 | INFO | "malloc" : " 14 GB", | 20260506 13:21:58.272 | INFO | "pinned" : " 29 GB" | 20260506 13:21:58.272 | INFO | } | 20260506 13:21:58.272 | INFO | Perf report for the last 5.13936 seconds: | 20260506 13:21:58.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:21:58.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:58.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:58.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:58.272 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:21:58.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:58.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:58.272 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:21:58.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:58.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:21:58.272 | INFO | Uploaded 36GiB to GPU in 2.02034s (17.8188GiB/s) | 20260506 13:21:58.272 | INFO | Downloaded 34GiB to GPU in 1.92812s (17.6338GiB/s) | 20260506 13:22:03.411 | INFO | Analyzing chunk frames = [1676288,1676800) | 20260506 13:22:03.411 | INFO | MemInfo: { | 20260506 13:22:03.411 | INFO | "gpu" : " 17 GB", | 20260506 13:22:03.411 | INFO | "huge" : "", | 20260506 13:22:03.411 | INFO | "huge_shared" : " 27 GB", | 20260506 13:22:03.411 | INFO | "malloc" : " 14 GB", | 20260506 13:22:03.411 | INFO | "pinned" : " 29 GB" | 20260506 13:22:03.411 | INFO | } | 20260506 13:22:03.411 | INFO | Perf report for the last 5.13957 seconds: | 20260506 13:22:03.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:22:03.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:03.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:03.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:03.411 | INFO | ANALYSIS : 61.4%, 3.0, 95.0ms, 0 | 20260506 13:22:03.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:03.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:03.411 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:22:03.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:03.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:03.412 | INFO | Uploaded 36GiB to GPU in 1.97493s (18.2285GiB/s) | 20260506 13:22:03.412 | INFO | Downloaded 34GiB to GPU in 1.93141s (17.6038GiB/s) | 20260506 13:22:08.550 | INFO | Analyzing chunk frames = [1676800,1677312) | 20260506 13:22:08.550 | INFO | MemInfo: { | 20260506 13:22:08.550 | INFO | "gpu" : " 17 GB", | 20260506 13:22:08.550 | INFO | "huge" : "", | 20260506 13:22:08.550 | INFO | "huge_shared" : " 27 GB", | 20260506 13:22:08.550 | INFO | "malloc" : " 14 GB", | 20260506 13:22:08.550 | INFO | "pinned" : " 29 GB" | 20260506 13:22:08.550 | INFO | } | 20260506 13:22:08.551 | INFO | Perf report for the last 5.13926 seconds: | 20260506 13:22:08.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:22:08.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:08.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:08.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:08.551 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:22:08.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:08.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:08.551 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 13:22:08.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:08.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:08.551 | INFO | Uploaded 36GiB to GPU in 2.0009s (17.9919GiB/s) | 20260506 13:22:08.551 | INFO | Downloaded 34GiB to GPU in 1.92303s (17.6804GiB/s) | 20260506 13:22:13.690 | INFO | Analyzing chunk frames = [1677312,1677824) | 20260506 13:22:13.690 | INFO | MemInfo: { | 20260506 13:22:13.690 | INFO | "gpu" : " 17 GB", | 20260506 13:22:13.690 | INFO | "huge" : "", | 20260506 13:22:13.690 | INFO | "huge_shared" : " 27 GB", | 20260506 13:22:13.690 | INFO | "malloc" : " 14 GB", | 20260506 13:22:13.690 | INFO | "pinned" : " 29 GB" | 20260506 13:22:13.690 | INFO | } | 20260506 13:22:13.690 | INFO | Perf report for the last 5.13941 seconds: | 20260506 13:22:13.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:22:13.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:13.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:13.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:13.690 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 13:22:13.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:13.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:13.690 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 13:22:13.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:13.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:13.690 | INFO | Uploaded 36GiB to GPU in 2.00923s (17.9173GiB/s) | 20260506 13:22:13.690 | INFO | Downloaded 34GiB to GPU in 1.94117s (17.5152GiB/s) | 20260506 13:22:18.829 | INFO | Analyzing chunk frames = [1677824,1678336) | 20260506 13:22:18.829 | INFO | MemInfo: { | 20260506 13:22:18.829 | INFO | "gpu" : " 17 GB", | 20260506 13:22:18.829 | INFO | "huge" : "", | 20260506 13:22:18.829 | INFO | "huge_shared" : " 27 GB", | 20260506 13:22:18.829 | INFO | "malloc" : " 14 GB", | 20260506 13:22:18.829 | INFO | "pinned" : " 29 GB" | 20260506 13:22:18.829 | INFO | } | 20260506 13:22:18.829 | INFO | Perf report for the last 5.13935 seconds: | 20260506 13:22:18.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:22:18.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:18.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:18.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:18.829 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:22:18.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:18.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:18.829 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:22:18.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:18.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:18.830 | INFO | Uploaded 36GiB to GPU in 2.0934s (17.1969GiB/s) | 20260506 13:22:18.830 | INFO | Downloaded 34GiB to GPU in 1.97697s (17.1981GiB/s) | 20260506 13:22:23.968 | INFO | Analyzing chunk frames = [1678336,1678848) | 20260506 13:22:23.968 | INFO | MemInfo: { | 20260506 13:22:23.968 | INFO | "gpu" : " 17 GB", | 20260506 13:22:23.968 | INFO | "huge" : "", | 20260506 13:22:23.968 | INFO | "huge_shared" : " 27 GB", | 20260506 13:22:23.968 | INFO | "malloc" : " 14 GB", | 20260506 13:22:23.968 | INFO | "pinned" : " 29 GB" | 20260506 13:22:23.968 | INFO | } | 20260506 13:22:23.969 | INFO | Perf report for the last 5.1392 seconds: | 20260506 13:22:23.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:22:23.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:23.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:23.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:23.969 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:22:23.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:23.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:23.969 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:22:23.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:23.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:23.969 | INFO | Uploaded 36GiB to GPU in 1.98514s (18.1347GiB/s) | 20260506 13:22:23.969 | INFO | Downloaded 34GiB to GPU in 2.03315s (16.7228GiB/s) | 20260506 13:22:29.108 | INFO | Analyzing chunk frames = [1678848,1679360) | 20260506 13:22:29.108 | INFO | MemInfo: { | 20260506 13:22:29.108 | INFO | "gpu" : " 17 GB", | 20260506 13:22:29.108 | INFO | "huge" : "", | 20260506 13:22:29.108 | INFO | "huge_shared" : " 27 GB", | 20260506 13:22:29.108 | INFO | "malloc" : " 14 GB", | 20260506 13:22:29.108 | INFO | "pinned" : " 29 GB" | 20260506 13:22:29.108 | INFO | } | 20260506 13:22:29.108 | INFO | Perf report for the last 5.13948 seconds: | 20260506 13:22:29.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:22:29.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:29.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:29.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:29.108 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:22:29.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:29.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:29.108 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:22:29.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:29.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:29.108 | INFO | Uploaded 36GiB to GPU in 2.04455s (17.6078GiB/s) | 20260506 13:22:29.108 | INFO | Downloaded 34GiB to GPU in 1.97897s (17.1807GiB/s) | 20260506 13:22:29.109 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:22:29.109 | INFO | Total mem usage: 17.4 GiB | 20260506 13:22:29.109 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:22:29.109 | INFO | Recent High Water Marks Sum: 16.9 GiB (Savings: -3.28%) | 20260506 13:22:29.109 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:22:29.109 | INFO | High Water Breakdown: | 20260506 13:22:29.109 | INFO | 62.2% : 10.5 GiB ( 96% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:22:29.109 | INFO | 37.7% : 6.35 GiB ( 94.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:22:29.109 | INFO | 0.118% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:22:29.109 | INFO | ------------------------------------------ | 20260506 13:22:29.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:22:29.109 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:22:29.109 | INFO | Total mem usage: 26.5 GiB | 20260506 13:22:29.109 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:22:29.109 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:22:29.109 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:22:29.109 | INFO | High Water Breakdown: | 20260506 13:22:29.109 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:22:29.109 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:22:29.109 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:22:29.109 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:22:29.109 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:22:29.109 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:22:29.109 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:22:29.109 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:22:29.109 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:22:29.109 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:22:29.109 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:22:29.109 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:22:29.109 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:22:29.109 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:22:29.109 | INFO | ------------------------------------------ | 20260506 13:22:29.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:22:29.109 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:22:29.109 | INFO | Total mem usage: 25.6 GiB | 20260506 13:22:29.109 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:22:29.109 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:22:29.109 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:22:29.109 | INFO | High Water Breakdown: | 20260506 13:22:29.109 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:22:29.109 | INFO | ------------------------------------------ | 20260506 13:22:29.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:22:29.109 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:22:29.109 | INFO | Total mem usage: 15.7 GiB | 20260506 13:22:29.109 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:22:29.109 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:22:29.109 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:22:29.109 | INFO | High Water Breakdown: | 20260506 13:22:29.109 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:22:29.109 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:22:29.109 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:22:29.109 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:22:29.109 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:22:29.109 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:22:29.109 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:22:29.109 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:22:29.109 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:22:29.109 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:22:29.109 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:22:29.109 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:22:29.109 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:22:29.109 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:22:29.109 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:22:29.109 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:22:29.109 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:22:29.109 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:22:29.109 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:22:29.109 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:22:29.109 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:22:29.109 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:22:29.109 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:22:29.109 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:22:29.109 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:22:29.109 | INFO | ------------------------------------------ | 20260506 13:22:29.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:22:29.109 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:22:29.109 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:22:29.109 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:22:29.109 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.66e+01, 1.35e+01 / 2.66e+00 | 20260506 13:22:29.109 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.01e-01, 1.03e-02 / 5.75e-03 | 20260506 13:22:29.109 | INFO | PerfReport: 11.55%, Baseline -- 3.65e+00 / 4.44e+00, 5.66e-01 / 3.30e-01 | 20260506 13:22:29.109 | INFO | PerfReport: 3.23%, Histogram -- 1.02e+00 / 1.45e+00, 1.86e-01 / 1.55e-01 | 20260506 13:22:29.109 | INFO | PerfReport: 15.95%, DME -- 4.03e+01 / 4.53e+01, 4.51e+00 / 3.17e+00 | 20260506 13:22:29.109 | INFO | PerfReport: 56.40%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.29e+00 / 1.49e+00 | 20260506 13:22:29.109 | INFO | PerfReport: 7.58%, PulseAccumulating -- 2.40e+00 / 3.07e+00, 4.30e-01 / 1.74e-01 | 20260506 13:22:29.109 | INFO | PerfReport: 3.69%, Metrics -- 1.17e+00 / 1.54e+00, 3.01e-01 / 1.57e-01 | 20260506 13:22:29.109 | INFO | PerfReport: 1.30%, PulseSerializing -- 4.10e-01 / 4.69e-01, 6.49e-02 / 3.61e-02 | 20260506 13:22:29.109 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:22:29.109 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:22:29.109 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:22:29.109 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 13:22:29.109 | INFO | PerfReport: 21.60%, Upload -- 2.12e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260506 13:22:29.110 | INFO | PerfReport: 21.16%, Download -- 2.08e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 13:22:29.110 | INFO | PerfReport: 32.18%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 13:22:29.110 | INFO | PerfReport: 1.83%, LockUpload -- 1.79e+00 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260506 13:22:29.110 | INFO | PerfReport: 1.03%, LockDownload -- 1.02e+00 / 0.00e+00, 2.63e+00 / 0.00e+00 | 20260506 13:22:29.110 | INFO | PerfReport: 22.19%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 13:22:29.110 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260506 13:22:29.241 | INFO | Starting a round of Baz IO | 20260506 13:22:32.406 | INFO | Populating RTMetrics report | 20260506 13:22:32.406 | INFO | RTMetrics report populated | 20260506 13:22:32.406 | INFO | Opening tmp RTMetrics json file | 20260506 13:22:32.468 | INFO | Serializing RTMetrics to json | 20260506 13:22:32.549 | INFO | Serialization done, swapping tmp file now... | 20260506 13:22:32.552 | INFO | RTMetrics done | 20260506 13:22:32.769 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.528 seconds | 20260506 13:22:32.769 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.84% / 0.02% / 60.15% | 20260506 13:22:32.769 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.23427% / 2.38419e-05% / 5.5631e-05% | 20260506 13:22:32.769 | INFO | Baz IO overall: | 20260506 13:22:32.769 | INFO | BytesWritten : 851853326175 | 20260506 13:22:32.769 | INFO | NumEvents : 411994667114 | 20260506 13:22:34.247 | INFO | Analyzing chunk frames = [1679360,1679872) | 20260506 13:22:34.247 | INFO | MemInfo: { | 20260506 13:22:34.247 | INFO | "gpu" : " 17 GB", | 20260506 13:22:34.247 | INFO | "huge" : "", | 20260506 13:22:34.247 | INFO | "huge_shared" : " 27 GB", | 20260506 13:22:34.247 | INFO | "malloc" : " 14 GB", | 20260506 13:22:34.247 | INFO | "pinned" : " 29 GB" | 20260506 13:22:34.247 | INFO | } | 20260506 13:22:34.247 | INFO | Perf report for the last 5.13945 seconds: | 20260506 13:22:34.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:22:34.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:34.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:34.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:34.248 | INFO | ANALYSIS : 63.6%, 2.9, 96.1ms, 0 | 20260506 13:22:34.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:34.248 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260506 13:22:34.248 | INFO | PRE_HQ : 61.2%, 1.9, 60.5ms, 0 | 20260506 13:22:34.248 | INFO | BAZWRITER : 68.2%, 4.8, 168.0ms, 0 | 20260506 13:22:34.248 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 13:22:34.248 | INFO | Uploaded 36GiB to GPU in 2.07616s (17.3397GiB/s) | 20260506 13:22:34.248 | INFO | Downloaded 38GiB to GPU in 2.23421s (17.0083GiB/s) | 20260506 13:22:39.386 | INFO | Analyzing chunk frames = [1679872,1680384) | 20260506 13:22:39.386 | INFO | MemInfo: { | 20260506 13:22:39.386 | INFO | "gpu" : " 17 GB", | 20260506 13:22:39.386 | INFO | "huge" : "", | 20260506 13:22:39.386 | INFO | "huge_shared" : " 27 GB", | 20260506 13:22:39.386 | INFO | "malloc" : " 14 GB", | 20260506 13:22:39.386 | INFO | "pinned" : " 29 GB" | 20260506 13:22:39.387 | INFO | } | 20260506 13:22:39.387 | INFO | Perf report for the last 5.13906 seconds: | 20260506 13:22:39.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:22:39.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:39.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:39.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:39.387 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 13:22:39.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:39.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:39.387 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:22:39.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:39.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:39.387 | INFO | Uploaded 36GiB to GPU in 2.04562s (17.5986GiB/s) | 20260506 13:22:39.387 | INFO | Downloaded 34GiB to GPU in 1.94354s (17.4939GiB/s) | 20260506 13:22:44.526 | INFO | Analyzing chunk frames = [1680384,1680896) | 20260506 13:22:44.526 | INFO | MemInfo: { | 20260506 13:22:44.526 | INFO | "gpu" : " 17 GB", | 20260506 13:22:44.526 | INFO | "huge" : "", | 20260506 13:22:44.526 | INFO | "huge_shared" : " 27 GB", | 20260506 13:22:44.526 | INFO | "malloc" : " 14 GB", | 20260506 13:22:44.526 | INFO | "pinned" : " 29 GB" | 20260506 13:22:44.526 | INFO | } | 20260506 13:22:44.526 | INFO | Perf report for the last 5.13946 seconds: | 20260506 13:22:44.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:22:44.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:44.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:44.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:44.526 | INFO | ANALYSIS : 61.7%, 3.0, 95.5ms, 0 | 20260506 13:22:44.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:44.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:44.526 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:22:44.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:44.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:44.526 | INFO | Uploaded 36GiB to GPU in 2.03168s (17.7193GiB/s) | 20260506 13:22:44.526 | INFO | Downloaded 34GiB to GPU in 1.94102s (17.5165GiB/s) | 20260506 13:22:49.665 | INFO | Analyzing chunk frames = [1680896,1681408) | 20260506 13:22:49.665 | INFO | MemInfo: { | 20260506 13:22:49.665 | INFO | "gpu" : " 17 GB", | 20260506 13:22:49.665 | INFO | "huge" : "", | 20260506 13:22:49.665 | INFO | "huge_shared" : " 27 GB", | 20260506 13:22:49.665 | INFO | "malloc" : " 14 GB", | 20260506 13:22:49.665 | INFO | "pinned" : " 29 GB" | 20260506 13:22:49.665 | INFO | } | 20260506 13:22:49.665 | INFO | Perf report for the last 5.13916 seconds: | 20260506 13:22:49.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:22:49.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:49.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:49.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:49.665 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:22:49.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:49.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:49.665 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:22:49.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:49.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:49.665 | INFO | Uploaded 36GiB to GPU in 2.01149s (17.8972GiB/s) | 20260506 13:22:49.665 | INFO | Downloaded 34GiB to GPU in 1.94252s (17.503GiB/s) | 20260506 13:22:54.804 | INFO | Analyzing chunk frames = [1681408,1681920) | 20260506 13:22:54.804 | INFO | MemInfo: { | 20260506 13:22:54.804 | INFO | "gpu" : " 17 GB", | 20260506 13:22:54.804 | INFO | "huge" : "", | 20260506 13:22:54.805 | INFO | "huge_shared" : " 27 GB", | 20260506 13:22:54.805 | INFO | "malloc" : " 14 GB", | 20260506 13:22:54.805 | INFO | "pinned" : " 29 GB" | 20260506 13:22:54.805 | INFO | } | 20260506 13:22:54.805 | INFO | Perf report for the last 5.13959 seconds: | 20260506 13:22:54.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:22:54.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:54.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:54.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:54.805 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 13:22:54.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:54.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:54.805 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:22:54.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:54.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:54.805 | INFO | Uploaded 36GiB to GPU in 2.02319s (17.7937GiB/s) | 20260506 13:22:54.805 | INFO | Downloaded 34GiB to GPU in 1.94951s (17.4402GiB/s) | 20260506 13:22:59.944 | INFO | Analyzing chunk frames = [1681920,1682432) | 20260506 13:22:59.944 | INFO | MemInfo: { | 20260506 13:22:59.944 | INFO | "gpu" : " 17 GB", | 20260506 13:22:59.944 | INFO | "huge" : "", | 20260506 13:22:59.944 | INFO | "huge_shared" : " 27 GB", | 20260506 13:22:59.944 | INFO | "malloc" : " 14 GB", | 20260506 13:22:59.944 | INFO | "pinned" : " 29 GB" | 20260506 13:22:59.944 | INFO | } | 20260506 13:22:59.944 | INFO | Perf report for the last 5.13923 seconds: | 20260506 13:22:59.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:22:59.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:59.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:59.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:59.944 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:22:59.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:59.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:59.944 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260506 13:22:59.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:59.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:22:59.944 | INFO | Uploaded 36GiB to GPU in 2.05331s (17.5327GiB/s) | 20260506 13:22:59.944 | INFO | Downloaded 34GiB to GPU in 1.92026s (17.706GiB/s) | 20260506 13:23:05.084 | INFO | Analyzing chunk frames = [1682432,1682944) | 20260506 13:23:05.084 | INFO | MemInfo: { | 20260506 13:23:05.084 | INFO | "gpu" : " 17 GB", | 20260506 13:23:05.084 | INFO | "huge" : "", | 20260506 13:23:05.084 | INFO | "huge_shared" : " 27 GB", | 20260506 13:23:05.084 | INFO | "malloc" : " 14 GB", | 20260506 13:23:05.084 | INFO | "pinned" : " 29 GB" | 20260506 13:23:05.084 | INFO | } | 20260506 13:23:05.084 | INFO | Perf report for the last 5.14036 seconds: | 20260506 13:23:05.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:23:05.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:05.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:05.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:05.085 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:23:05.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:05.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:05.085 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 13:23:05.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:05.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:05.085 | INFO | Uploaded 36GiB to GPU in 2.02153s (17.8083GiB/s) | 20260506 13:23:05.085 | INFO | Downloaded 34GiB to GPU in 2.01977s (16.8336GiB/s) | 20260506 13:23:10.223 | INFO | Analyzing chunk frames = [1682944,1683456) | 20260506 13:23:10.223 | INFO | MemInfo: { | 20260506 13:23:10.223 | INFO | "gpu" : " 17 GB", | 20260506 13:23:10.223 | INFO | "huge" : "", | 20260506 13:23:10.223 | INFO | "huge_shared" : " 27 GB", | 20260506 13:23:10.223 | INFO | "malloc" : " 14 GB", | 20260506 13:23:10.223 | INFO | "pinned" : " 29 GB" | 20260506 13:23:10.223 | INFO | } | 20260506 13:23:10.223 | INFO | Perf report for the last 5.13822 seconds: | 20260506 13:23:10.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:23:10.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:10.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:10.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:10.223 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 13:23:10.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:10.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:10.223 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:23:10.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:10.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:10.223 | INFO | Uploaded 36GiB to GPU in 2.05421s (17.525GiB/s) | 20260506 13:23:10.223 | INFO | Downloaded 34GiB to GPU in 1.97292s (17.2333GiB/s) | 20260506 13:23:10.356 | INFO | Starting a round of Baz IO | 20260506 13:23:13.510 | INFO | Populating RTMetrics report | 20260506 13:23:13.510 | INFO | RTMetrics report populated | 20260506 13:23:13.510 | INFO | Opening tmp RTMetrics json file | 20260506 13:23:13.523 | INFO | Serializing RTMetrics to json | 20260506 13:23:13.525 | INFO | Serialization done, swapping tmp file now... | 20260506 13:23:13.527 | INFO | RTMetrics done | 20260506 13:23:13.790 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.433 seconds | 20260506 13:23:13.790 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.82% / 0.02% / 60.16% | 20260506 13:23:13.790 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.22191% / 2.78155e-05% / 3.57628e-05% | 20260506 13:23:13.790 | INFO | Baz IO overall: | 20260506 13:23:13.790 | INFO | BytesWritten : 854118429873 | 20260506 13:23:13.790 | INFO | NumEvents : 413093173172 | 20260506 13:23:15.362 | INFO | Analyzing chunk frames = [1683456,1683968) | 20260506 13:23:15.362 | INFO | MemInfo: { | 20260506 13:23:15.362 | INFO | "gpu" : " 17 GB", | 20260506 13:23:15.362 | INFO | "huge" : "", | 20260506 13:23:15.362 | INFO | "huge_shared" : " 27 GB", | 20260506 13:23:15.362 | INFO | "malloc" : " 14 GB", | 20260506 13:23:15.362 | INFO | "pinned" : " 29 GB" | 20260506 13:23:15.362 | INFO | } | 20260506 13:23:15.362 | INFO | Perf report for the last 5.13925 seconds: | 20260506 13:23:15.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:23:15.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:15.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:15.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:15.362 | INFO | ANALYSIS : 63.4%, 2.9, 95.8ms, 0 | 20260506 13:23:15.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:15.362 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 13:23:15.362 | INFO | PRE_HQ : 61.3%, 1.9, 59.9ms, 0 | 20260506 13:23:15.362 | INFO | BAZWRITER : 66.4%, 4.8, 163.2ms, 0 | 20260506 13:23:15.362 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 13:23:15.362 | INFO | Uploaded 36GiB to GPU in 2.05827s (17.4904GiB/s) | 20260506 13:23:15.362 | INFO | Downloaded 38GiB to GPU in 2.23788s (16.9803GiB/s) | 20260506 13:23:20.501 | INFO | Analyzing chunk frames = [1683968,1684480) | 20260506 13:23:20.501 | INFO | MemInfo: { | 20260506 13:23:20.501 | INFO | "gpu" : " 17 GB", | 20260506 13:23:20.501 | INFO | "huge" : "", | 20260506 13:23:20.501 | INFO | "huge_shared" : " 27 GB", | 20260506 13:23:20.501 | INFO | "malloc" : " 14 GB", | 20260506 13:23:20.501 | INFO | "pinned" : " 29 GB" | 20260506 13:23:20.501 | INFO | } | 20260506 13:23:20.501 | INFO | Perf report for the last 5.13922 seconds: | 20260506 13:23:20.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:23:20.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:20.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:20.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:20.501 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:23:20.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:20.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:20.501 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 13:23:20.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:20.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:20.501 | INFO | Uploaded 36GiB to GPU in 2.01939s (17.8272GiB/s) | 20260506 13:23:20.501 | INFO | Downloaded 34GiB to GPU in 1.93807s (17.5433GiB/s) | 20260506 13:23:25.640 | INFO | Analyzing chunk frames = [1684480,1684992) | 20260506 13:23:25.640 | INFO | MemInfo: { | 20260506 13:23:25.640 | INFO | "gpu" : " 17 GB", | 20260506 13:23:25.641 | INFO | "huge" : "", | 20260506 13:23:25.641 | INFO | "huge_shared" : " 27 GB", | 20260506 13:23:25.641 | INFO | "malloc" : " 14 GB", | 20260506 13:23:25.641 | INFO | "pinned" : " 29 GB" | 20260506 13:23:25.641 | INFO | } | 20260506 13:23:25.641 | INFO | Perf report for the last 5.13962 seconds: | 20260506 13:23:25.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:23:25.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:25.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:25.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:25.641 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 13:23:25.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:25.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:25.641 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:23:25.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:25.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:25.641 | INFO | Uploaded 36GiB to GPU in 2.0148s (17.8678GiB/s) | 20260506 13:23:25.641 | INFO | Downloaded 34GiB to GPU in 1.94172s (17.5102GiB/s) | 20260506 13:23:30.780 | INFO | Analyzing chunk frames = [1684992,1685504) | 20260506 13:23:30.780 | INFO | MemInfo: { | 20260506 13:23:30.780 | INFO | "gpu" : " 17 GB", | 20260506 13:23:30.780 | INFO | "huge" : "", | 20260506 13:23:30.780 | INFO | "huge_shared" : " 27 GB", | 20260506 13:23:30.780 | INFO | "malloc" : " 14 GB", | 20260506 13:23:30.780 | INFO | "pinned" : " 29 GB" | 20260506 13:23:30.780 | INFO | } | 20260506 13:23:30.781 | INFO | Perf report for the last 5.13977 seconds: | 20260506 13:23:30.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:23:30.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:30.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:30.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:30.781 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 13:23:30.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:30.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:30.781 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:23:30.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:30.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:30.781 | INFO | Uploaded 36GiB to GPU in 2.00103s (17.9908GiB/s) | 20260506 13:23:30.781 | INFO | Downloaded 34GiB to GPU in 1.91247s (17.7781GiB/s) | 20260506 13:23:35.919 | INFO | Analyzing chunk frames = [1685504,1686016) | 20260506 13:23:35.919 | INFO | MemInfo: { | 20260506 13:23:35.919 | INFO | "gpu" : " 17 GB", | 20260506 13:23:35.919 | INFO | "huge" : "", | 20260506 13:23:35.919 | INFO | "huge_shared" : " 27 GB", | 20260506 13:23:35.919 | INFO | "malloc" : " 14 GB", | 20260506 13:23:35.919 | INFO | "pinned" : " 29 GB" | 20260506 13:23:35.919 | INFO | } | 20260506 13:23:35.920 | INFO | Perf report for the last 5.13902 seconds: | 20260506 13:23:35.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:23:35.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:35.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:35.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:35.920 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 13:23:35.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:35.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:35.920 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:23:35.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:35.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:35.920 | INFO | Uploaded 36GiB to GPU in 1.96525s (18.3183GiB/s) | 20260506 13:23:35.920 | INFO | Downloaded 34GiB to GPU in 1.93889s (17.5358GiB/s) | 20260506 13:23:41.059 | INFO | Analyzing chunk frames = [1686016,1686528) | 20260506 13:23:41.059 | INFO | MemInfo: { | 20260506 13:23:41.059 | INFO | "gpu" : " 17 GB", | 20260506 13:23:41.059 | INFO | "huge" : "", | 20260506 13:23:41.059 | INFO | "huge_shared" : " 27 GB", | 20260506 13:23:41.059 | INFO | "malloc" : " 14 GB", | 20260506 13:23:41.059 | INFO | "pinned" : " 29 GB" | 20260506 13:23:41.059 | INFO | } | 20260506 13:23:41.059 | INFO | Perf report for the last 5.13915 seconds: | 20260506 13:23:41.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:23:41.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:41.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:41.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:41.059 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:23:41.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:41.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:41.059 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 13:23:41.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:41.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:41.059 | INFO | Uploaded 36GiB to GPU in 2.09301s (17.2001GiB/s) | 20260506 13:23:41.059 | INFO | Downloaded 34GiB to GPU in 1.96965s (17.262GiB/s) | 20260506 13:23:46.199 | INFO | Analyzing chunk frames = [1686528,1687040) | 20260506 13:23:46.199 | INFO | MemInfo: { | 20260506 13:23:46.199 | INFO | "gpu" : " 17 GB", | 20260506 13:23:46.199 | INFO | "huge" : "", | 20260506 13:23:46.199 | INFO | "huge_shared" : " 27 GB", | 20260506 13:23:46.199 | INFO | "malloc" : " 14 GB", | 20260506 13:23:46.199 | INFO | "pinned" : " 29 GB" | 20260506 13:23:46.199 | INFO | } | 20260506 13:23:46.199 | INFO | Perf report for the last 5.1406 seconds: | 20260506 13:23:46.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:23:46.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:46.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:46.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:46.199 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:23:46.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:46.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:46.200 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 13:23:46.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:46.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:46.200 | INFO | Uploaded 36GiB to GPU in 1.99269s (18.0661GiB/s) | 20260506 13:23:46.200 | INFO | Downloaded 34GiB to GPU in 2.03367s (16.7186GiB/s) | 20260506 13:23:51.337 | INFO | Analyzing chunk frames = [1687040,1687552) | 20260506 13:23:51.337 | INFO | MemInfo: { | 20260506 13:23:51.337 | INFO | "gpu" : " 17 GB", | 20260506 13:23:51.337 | INFO | "huge" : "", | 20260506 13:23:51.337 | INFO | "huge_shared" : " 27 GB", | 20260506 13:23:51.338 | INFO | "malloc" : " 14 GB", | 20260506 13:23:51.338 | INFO | "pinned" : " 29 GB" | 20260506 13:23:51.338 | INFO | } | 20260506 13:23:51.338 | INFO | Perf report for the last 5.13824 seconds: | 20260506 13:23:51.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:23:51.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:51.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:51.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:51.338 | INFO | ANALYSIS : 62.5%, 3.0, 97.6ms, 0 | 20260506 13:23:51.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:51.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:51.338 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:23:51.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:51.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:51.338 | INFO | Uploaded 36GiB to GPU in 2.0842s (17.2728GiB/s) | 20260506 13:23:51.338 | INFO | Downloaded 34GiB to GPU in 1.98143s (17.1593GiB/s) | 20260506 13:23:51.338 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:23:51.338 | INFO | Total mem usage: 17.4 GiB | 20260506 13:23:51.338 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:23:51.338 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -11.1%) | 20260506 13:23:51.338 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:23:51.338 | INFO | High Water Breakdown: | 20260506 13:23:51.338 | INFO | 61.1% : 9.57 GiB ( 87.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:23:51.338 | INFO | 38.8% : 6.07 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:23:51.338 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:23:51.338 | INFO | ------------------------------------------ | 20260506 13:23:51.338 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:23:51.338 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:23:51.338 | INFO | Total mem usage: 26.5 GiB | 20260506 13:23:51.338 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:23:51.338 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:23:51.338 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:23:51.338 | INFO | High Water Breakdown: | 20260506 13:23:51.338 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:23:51.338 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:23:51.338 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:23:51.338 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:23:51.338 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:23:51.338 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:23:51.338 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:23:51.338 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:23:51.339 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:23:51.339 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:23:51.339 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:23:51.339 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:23:51.339 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:23:51.339 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:23:51.339 | INFO | ------------------------------------------ | 20260506 13:23:51.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:23:51.339 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:23:51.339 | INFO | Total mem usage: 25.6 GiB | 20260506 13:23:51.339 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:23:51.339 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:23:51.339 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:23:51.339 | INFO | High Water Breakdown: | 20260506 13:23:51.339 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:23:51.339 | INFO | ------------------------------------------ | 20260506 13:23:51.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:23:51.339 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:23:51.339 | INFO | Total mem usage: 15.7 GiB | 20260506 13:23:51.339 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:23:51.339 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:23:51.339 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 13:23:51.339 | INFO | High Water Breakdown: | 20260506 13:23:51.339 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:23:51.339 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:23:51.339 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:23:51.339 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:23:51.339 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:23:51.339 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:23:51.339 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:23:51.339 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:23:51.339 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:23:51.339 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:23:51.339 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:23:51.339 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:23:51.339 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:23:51.339 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:23:51.339 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:23:51.339 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:23:51.339 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:23:51.339 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:23:51.339 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:23:51.339 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:23:51.339 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:23:51.339 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:23:51.339 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:23:51.339 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:23:51.339 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:23:51.339 | INFO | ------------------------------------------ | 20260506 13:23:51.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:23:51.339 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:23:51.339 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:23:51.339 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:23:51.339 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.64e+01, 1.35e+01 / 2.37e+00 | 20260506 13:23:51.339 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.08e-02 / 8.02e-03 | 20260506 13:23:51.339 | INFO | PerfReport: 11.57%, Baseline -- 3.66e+00 / 4.47e+00, 5.77e-01 / 3.35e-01 | 20260506 13:23:51.339 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.50e+00, 1.91e-01 / 1.22e-01 | 20260506 13:23:51.339 | INFO | PerfReport: 15.90%, DME -- 4.03e+01 / 4.51e+01, 4.33e+00 / 2.94e+00 | 20260506 13:23:51.339 | INFO | PerfReport: 56.48%, FrameLabeling -- 1.79e+01 / 2.02e+01, 1.39e+00 / 1.77e+00 | 20260506 13:23:51.339 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.38e+00 / 3.00e+00, 4.15e-01 / 2.37e-01 | 20260506 13:23:51.339 | INFO | PerfReport: 3.68%, Metrics -- 1.17e+00 / 1.56e+00, 3.01e-01 / 1.83e-01 | 20260506 13:23:51.339 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.09e-01 / 4.66e-01, 6.47e-02 / 2.87e-02 | 20260506 13:23:51.339 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:23:51.339 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:23:51.339 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:23:51.339 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 13:23:51.339 | INFO | PerfReport: 21.56%, Upload -- 2.12e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260506 13:23:51.339 | INFO | PerfReport: 21.11%, Download -- 2.08e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 13:23:51.339 | INFO | PerfReport: 32.19%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 13:23:51.339 | INFO | PerfReport: 1.80%, LockUpload -- 1.77e+00 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 13:23:51.339 | INFO | PerfReport: 1.02%, LockDownload -- 1.01e+00 / 0.00e+00, 2.69e+00 / 0.00e+00 | 20260506 13:23:51.339 | INFO | PerfReport: 22.29%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 13:23:51.339 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260506 13:23:51.469 | INFO | Starting a round of Baz IO | 20260506 13:23:54.667 | INFO | Populating RTMetrics report | 20260506 13:23:54.667 | INFO | RTMetrics report populated | 20260506 13:23:54.667 | INFO | Opening tmp RTMetrics json file | 20260506 13:23:54.699 | INFO | Serializing RTMetrics to json | 20260506 13:23:54.702 | INFO | Serialization done, swapping tmp file now... | 20260506 13:23:54.702 | INFO | RTMetrics done | 20260506 13:23:54.853 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.383 seconds | 20260506 13:23:54.853 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.80% / 0.02% / 60.18% | 20260506 13:23:54.853 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.22164% / 4.37101e-05% / 6.35783e-05% | 20260506 13:23:54.853 | INFO | Baz IO overall: | 20260506 13:23:54.853 | INFO | BytesWritten : 856386504337 | 20260506 13:23:54.853 | INFO | NumEvents : 414193723348 | 20260506 13:23:56.477 | INFO | Analyzing chunk frames = [1687552,1688064) | 20260506 13:23:56.477 | INFO | MemInfo: { | 20260506 13:23:56.477 | INFO | "gpu" : " 17 GB", | 20260506 13:23:56.477 | INFO | "huge" : "", | 20260506 13:23:56.477 | INFO | "huge_shared" : " 27 GB", | 20260506 13:23:56.477 | INFO | "malloc" : " 14 GB", | 20260506 13:23:56.477 | INFO | "pinned" : " 29 GB" | 20260506 13:23:56.477 | INFO | } | 20260506 13:23:56.477 | INFO | Perf report for the last 5.13944 seconds: | 20260506 13:23:56.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:23:56.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:56.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:56.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:56.477 | INFO | ANALYSIS : 63.9%, 2.8, 95.4ms, 0 | 20260506 13:23:56.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:23:56.477 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260506 13:23:56.477 | INFO | PRE_HQ : 61.4%, 2.0, 64.5ms, 0 | 20260506 13:23:56.477 | INFO | BAZWRITER : 65.4%, 4.7, 158.6ms, 0 | 20260506 13:23:56.477 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 13:23:56.478 | INFO | Uploaded 36GiB to GPU in 2.07677s (17.3346GiB/s) | 20260506 13:23:56.478 | INFO | Downloaded 38GiB to GPU in 2.2071s (17.2171GiB/s) | 20260506 13:24:01.616 | INFO | Analyzing chunk frames = [1688064,1688576) | 20260506 13:24:01.616 | INFO | MemInfo: { | 20260506 13:24:01.616 | INFO | "gpu" : " 17 GB", | 20260506 13:24:01.616 | INFO | "huge" : "", | 20260506 13:24:01.616 | INFO | "huge_shared" : " 27 GB", | 20260506 13:24:01.616 | INFO | "malloc" : " 14 GB", | 20260506 13:24:01.616 | INFO | "pinned" : " 29 GB" | 20260506 13:24:01.616 | INFO | } | 20260506 13:24:01.616 | INFO | Perf report for the last 5.1389 seconds: | 20260506 13:24:01.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:24:01.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:01.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:01.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:01.616 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:24:01.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:01.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:01.616 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 13:24:01.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:01.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:01.616 | INFO | Uploaded 36GiB to GPU in 2.06632s (17.4222GiB/s) | 20260506 13:24:01.616 | INFO | Downloaded 34GiB to GPU in 1.9381s (17.5429GiB/s) | 20260506 13:24:06.755 | INFO | Analyzing chunk frames = [1688576,1689088) | 20260506 13:24:06.755 | INFO | MemInfo: { | 20260506 13:24:06.755 | INFO | "gpu" : " 17 GB", | 20260506 13:24:06.755 | INFO | "huge" : "", | 20260506 13:24:06.755 | INFO | "huge_shared" : " 27 GB", | 20260506 13:24:06.755 | INFO | "malloc" : " 14 GB", | 20260506 13:24:06.755 | INFO | "pinned" : " 29 GB" | 20260506 13:24:06.755 | INFO | } | 20260506 13:24:06.755 | INFO | Perf report for the last 5.13931 seconds: | 20260506 13:24:06.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:24:06.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:06.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:06.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:06.756 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 13:24:06.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:06.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:06.756 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:24:06.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:06.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:06.756 | INFO | Uploaded 36GiB to GPU in 2.01165s (17.8957GiB/s) | 20260506 13:24:06.756 | INFO | Downloaded 34GiB to GPU in 1.94274s (17.5011GiB/s) | 20260506 13:24:11.895 | INFO | Analyzing chunk frames = [1689088,1689600) | 20260506 13:24:11.895 | INFO | MemInfo: { | 20260506 13:24:11.895 | INFO | "gpu" : " 17 GB", | 20260506 13:24:11.895 | INFO | "huge" : "", | 20260506 13:24:11.895 | INFO | "huge_shared" : " 27 GB", | 20260506 13:24:11.895 | INFO | "malloc" : " 14 GB", | 20260506 13:24:11.895 | INFO | "pinned" : " 29 GB" | 20260506 13:24:11.895 | INFO | } | 20260506 13:24:11.895 | INFO | Perf report for the last 5.13932 seconds: | 20260506 13:24:11.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:24:11.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:11.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:11.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:11.895 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:24:11.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:11.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:11.895 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:24:11.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:11.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:11.895 | INFO | Uploaded 36GiB to GPU in 2.00635s (17.943GiB/s) | 20260506 13:24:11.895 | INFO | Downloaded 34GiB to GPU in 1.93997s (17.526GiB/s) | 20260506 13:24:17.034 | INFO | Analyzing chunk frames = [1689600,1690112) | 20260506 13:24:17.034 | INFO | MemInfo: { | 20260506 13:24:17.034 | INFO | "gpu" : " 17 GB", | 20260506 13:24:17.034 | INFO | "huge" : "", | 20260506 13:24:17.034 | INFO | "huge_shared" : " 27 GB", | 20260506 13:24:17.034 | INFO | "malloc" : " 14 GB", | 20260506 13:24:17.034 | INFO | "pinned" : " 29 GB" | 20260506 13:24:17.034 | INFO | } | 20260506 13:24:17.034 | INFO | Perf report for the last 5.13955 seconds: | 20260506 13:24:17.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:24:17.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:17.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:17.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:17.034 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 13:24:17.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:17.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:17.034 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 13:24:17.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:17.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:17.034 | INFO | Uploaded 36GiB to GPU in 2.07242s (17.371GiB/s) | 20260506 13:24:17.034 | INFO | Downloaded 34GiB to GPU in 1.9517s (17.4207GiB/s) | 20260506 13:24:22.173 | INFO | Analyzing chunk frames = [1690112,1690624) | 20260506 13:24:22.173 | INFO | MemInfo: { | 20260506 13:24:22.173 | INFO | "gpu" : " 17 GB", | 20260506 13:24:22.173 | INFO | "huge" : "", | 20260506 13:24:22.173 | INFO | "huge_shared" : " 27 GB", | 20260506 13:24:22.173 | INFO | "malloc" : " 14 GB", | 20260506 13:24:22.173 | INFO | "pinned" : " 29 GB" | 20260506 13:24:22.173 | INFO | } | 20260506 13:24:22.173 | INFO | Perf report for the last 5.1388 seconds: | 20260506 13:24:22.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:24:22.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:22.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:22.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:22.174 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:24:22.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:22.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:22.174 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:24:22.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:22.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:22.174 | INFO | Uploaded 36GiB to GPU in 2.11288s (17.0383GiB/s) | 20260506 13:24:22.174 | INFO | Downloaded 34GiB to GPU in 1.98702s (17.111GiB/s) | 20260506 13:24:27.312 | INFO | Analyzing chunk frames = [1690624,1691136) | 20260506 13:24:27.312 | INFO | MemInfo: { | 20260506 13:24:27.312 | INFO | "gpu" : " 17 GB", | 20260506 13:24:27.312 | INFO | "huge" : "", | 20260506 13:24:27.312 | INFO | "huge_shared" : " 27 GB", | 20260506 13:24:27.312 | INFO | "malloc" : " 14 GB", | 20260506 13:24:27.312 | INFO | "pinned" : " 29 GB" | 20260506 13:24:27.313 | INFO | } | 20260506 13:24:27.313 | INFO | Perf report for the last 5.13952 seconds: | 20260506 13:24:27.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:24:27.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:27.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:27.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:27.313 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:24:27.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:27.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:27.313 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:24:27.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:27.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:27.313 | INFO | Uploaded 36GiB to GPU in 2.0408s (17.6402GiB/s) | 20260506 13:24:27.313 | INFO | Downloaded 34GiB to GPU in 2.01482s (16.8749GiB/s) | 20260506 13:24:32.452 | INFO | Analyzing chunk frames = [1691136,1691648) | 20260506 13:24:32.452 | INFO | MemInfo: { | 20260506 13:24:32.452 | INFO | "gpu" : " 17 GB", | 20260506 13:24:32.452 | INFO | "huge" : "", | 20260506 13:24:32.452 | INFO | "huge_shared" : " 27 GB", | 20260506 13:24:32.452 | INFO | "malloc" : " 14 GB", | 20260506 13:24:32.452 | INFO | "pinned" : " 29 GB" | 20260506 13:24:32.452 | INFO | } | 20260506 13:24:32.452 | INFO | Perf report for the last 5.13955 seconds: | 20260506 13:24:32.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:24:32.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:32.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:32.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:32.452 | INFO | ANALYSIS : 62.8%, 3.0, 97.8ms, 0 | 20260506 13:24:32.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:32.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:32.452 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260506 13:24:32.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:32.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:32.452 | INFO | Uploaded 36GiB to GPU in 2.07132s (17.3803GiB/s) | 20260506 13:24:32.452 | INFO | Downloaded 34GiB to GPU in 1.96732s (17.2824GiB/s) | 20260506 13:24:32.580 | INFO | Starting a round of Baz IO | 20260506 13:24:35.755 | INFO | Populating RTMetrics report | 20260506 13:24:35.755 | INFO | RTMetrics report populated | 20260506 13:24:35.755 | INFO | Opening tmp RTMetrics json file | 20260506 13:24:35.861 | INFO | Serializing RTMetrics to json | 20260506 13:24:35.992 | INFO | Serialization done, swapping tmp file now... | 20260506 13:24:35.993 | INFO | RTMetrics done | 20260506 13:24:36.663 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 4.082 seconds | 20260506 13:24:36.663 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.78% / 0.02% / 60.20% | 20260506 13:24:36.663 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.21738% / 2.78155e-05% / 4.76837e-05% | 20260506 13:24:36.663 | INFO | Baz IO overall: | 20260506 13:24:36.663 | INFO | BytesWritten : 858655665917 | 20260506 13:24:36.663 | INFO | NumEvents : 415294836550 | 20260506 13:24:37.591 | INFO | Analyzing chunk frames = [1691648,1692160) | 20260506 13:24:37.591 | INFO | MemInfo: { | 20260506 13:24:37.591 | INFO | "gpu" : " 17 GB", | 20260506 13:24:37.591 | INFO | "huge" : "", | 20260506 13:24:37.591 | INFO | "huge_shared" : " 27 GB", | 20260506 13:24:37.591 | INFO | "malloc" : " 14 GB", | 20260506 13:24:37.591 | INFO | "pinned" : " 29 GB" | 20260506 13:24:37.591 | INFO | } | 20260506 13:24:37.591 | INFO | Perf report for the last 5.1392 seconds: | 20260506 13:24:37.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:24:37.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:37.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:37.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:37.591 | INFO | ANALYSIS : 63.6%, 2.9, 96.0ms, 0 | 20260506 13:24:37.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:37.591 | INFO | RT_METRICS : 49.6%, 1.0, 26.0ms, 0 | 20260506 13:24:37.591 | INFO | PRE_HQ : 61.3%, 1.9, 60.7ms, 0 | 20260506 13:24:37.591 | INFO | BAZWRITER : 78.8%, 4.8, 195.3ms, 0 | 20260506 13:24:37.591 | INFO | BAZ_SAMPLING : 4.5%, 1.0, 2.3ms, 0 | 20260506 13:24:37.592 | INFO | Uploaded 36GiB to GPU in 2.09754s (17.163GiB/s) | 20260506 13:24:37.592 | INFO | Downloaded 38GiB to GPU in 2.22249s (17.098GiB/s) | 20260506 13:24:42.730 | INFO | Analyzing chunk frames = [1692160,1692672) | 20260506 13:24:42.730 | INFO | MemInfo: { | 20260506 13:24:42.730 | INFO | "gpu" : " 17 GB", | 20260506 13:24:42.730 | INFO | "huge" : "", | 20260506 13:24:42.730 | INFO | "huge_shared" : " 27 GB", | 20260506 13:24:42.730 | INFO | "malloc" : " 14 GB", | 20260506 13:24:42.730 | INFO | "pinned" : " 29 GB" | 20260506 13:24:42.730 | INFO | } | 20260506 13:24:42.730 | INFO | Perf report for the last 5.13908 seconds: | 20260506 13:24:42.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:24:42.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:42.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:42.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:42.731 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:24:42.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:42.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:42.731 | INFO | PRE_HQ : 28.0%, 1.0, 14.7ms, 0 | 20260506 13:24:42.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:42.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:42.731 | INFO | Uploaded 36GiB to GPU in 2.04601s (17.5952GiB/s) | 20260506 13:24:42.731 | INFO | Downloaded 34GiB to GPU in 1.93161s (17.6019GiB/s) | 20260506 13:24:47.870 | INFO | Analyzing chunk frames = [1692672,1693184) | 20260506 13:24:47.870 | INFO | MemInfo: { | 20260506 13:24:47.870 | INFO | "gpu" : " 17 GB", | 20260506 13:24:47.870 | INFO | "huge" : "", | 20260506 13:24:47.870 | INFO | "huge_shared" : " 27 GB", | 20260506 13:24:47.870 | INFO | "malloc" : " 14 GB", | 20260506 13:24:47.870 | INFO | "pinned" : " 29 GB" | 20260506 13:24:47.870 | INFO | } | 20260506 13:24:47.870 | INFO | Perf report for the last 5.13988 seconds: | 20260506 13:24:47.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:24:47.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:47.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:47.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:47.870 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 13:24:47.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:47.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:47.870 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:24:47.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:47.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:47.871 | INFO | Uploaded 36GiB to GPU in 2.03363s (17.7024GiB/s) | 20260506 13:24:47.871 | INFO | Downloaded 34GiB to GPU in 1.94248s (17.5034GiB/s) | 20260506 13:24:53.009 | INFO | Analyzing chunk frames = [1693184,1693696) | 20260506 13:24:53.009 | INFO | MemInfo: { | 20260506 13:24:53.009 | INFO | "gpu" : " 17 GB", | 20260506 13:24:53.009 | INFO | "huge" : "", | 20260506 13:24:53.009 | INFO | "huge_shared" : " 27 GB", | 20260506 13:24:53.009 | INFO | "malloc" : " 14 GB", | 20260506 13:24:53.009 | INFO | "pinned" : " 29 GB" | 20260506 13:24:53.009 | INFO | } | 20260506 13:24:53.009 | INFO | Perf report for the last 5.13909 seconds: | 20260506 13:24:53.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:24:53.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:53.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:53.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:53.009 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:24:53.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:53.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:53.010 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:24:53.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:53.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:53.010 | INFO | Uploaded 36GiB to GPU in 2.0011s (17.9901GiB/s) | 20260506 13:24:53.010 | INFO | Downloaded 34GiB to GPU in 1.93682s (17.5546GiB/s) | 20260506 13:24:58.148 | INFO | Analyzing chunk frames = [1693696,1694208) | 20260506 13:24:58.148 | INFO | MemInfo: { | 20260506 13:24:58.148 | INFO | "gpu" : " 17 GB", | 20260506 13:24:58.148 | INFO | "huge" : "", | 20260506 13:24:58.148 | INFO | "huge_shared" : " 27 GB", | 20260506 13:24:58.149 | INFO | "malloc" : " 14 GB", | 20260506 13:24:58.149 | INFO | "pinned" : " 29 GB" | 20260506 13:24:58.149 | INFO | } | 20260506 13:24:58.149 | INFO | Perf report for the last 5.1393 seconds: | 20260506 13:24:58.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:24:58.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:58.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:58.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:58.149 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 13:24:58.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:58.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:58.149 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:24:58.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:58.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:24:58.149 | INFO | Uploaded 36GiB to GPU in 2.05901s (17.4841GiB/s) | 20260506 13:24:58.149 | INFO | Downloaded 34GiB to GPU in 1.95067s (17.4299GiB/s) | 20260506 13:25:03.288 | INFO | Analyzing chunk frames = [1694208,1694720) | 20260506 13:25:03.288 | INFO | MemInfo: { | 20260506 13:25:03.288 | INFO | "gpu" : " 17 GB", | 20260506 13:25:03.288 | INFO | "huge" : "", | 20260506 13:25:03.288 | INFO | "huge_shared" : " 27 GB", | 20260506 13:25:03.288 | INFO | "malloc" : " 14 GB", | 20260506 13:25:03.288 | INFO | "pinned" : " 29 GB" | 20260506 13:25:03.288 | INFO | } | 20260506 13:25:03.288 | INFO | Perf report for the last 5.13964 seconds: | 20260506 13:25:03.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:25:03.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:03.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:03.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:03.288 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:25:03.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:03.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:03.288 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:25:03.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:03.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:03.289 | INFO | Uploaded 36GiB to GPU in 2.0572s (17.4995GiB/s) | 20260506 13:25:03.289 | INFO | Downloaded 34GiB to GPU in 1.9291s (17.6248GiB/s) | 20260506 13:25:08.428 | INFO | Analyzing chunk frames = [1694720,1695232) | 20260506 13:25:08.428 | INFO | MemInfo: { | 20260506 13:25:08.428 | INFO | "gpu" : " 17 GB", | 20260506 13:25:08.428 | INFO | "huge" : "", | 20260506 13:25:08.428 | INFO | "huge_shared" : " 27 GB", | 20260506 13:25:08.428 | INFO | "malloc" : " 14 GB", | 20260506 13:25:08.428 | INFO | "pinned" : " 29 GB" | 20260506 13:25:08.428 | INFO | } | 20260506 13:25:08.428 | INFO | Perf report for the last 5.13958 seconds: | 20260506 13:25:08.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:25:08.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:08.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:08.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:08.428 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 13:25:08.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:08.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:08.428 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 13:25:08.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:08.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:08.428 | INFO | Uploaded 36GiB to GPU in 2.02219s (17.8025GiB/s) | 20260506 13:25:08.428 | INFO | Downloaded 34GiB to GPU in 2.00068s (16.9942GiB/s) | 20260506 13:25:13.567 | INFO | Analyzing chunk frames = [1695232,1695744) | 20260506 13:25:13.567 | INFO | MemInfo: { | 20260506 13:25:13.567 | INFO | "gpu" : " 17 GB", | 20260506 13:25:13.567 | INFO | "huge" : "", | 20260506 13:25:13.567 | INFO | "huge_shared" : " 27 GB", | 20260506 13:25:13.567 | INFO | "malloc" : " 14 GB", | 20260506 13:25:13.567 | INFO | "pinned" : " 29 GB" | 20260506 13:25:13.567 | INFO | } | 20260506 13:25:13.567 | INFO | Perf report for the last 5.1388 seconds: | 20260506 13:25:13.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:25:13.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:13.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:13.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:13.567 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 13:25:13.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:13.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:13.567 | INFO | PRE_HQ : 33.2%, 1.0, 17.3ms, 0 | 20260506 13:25:13.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:13.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:13.567 | INFO | Uploaded 36GiB to GPU in 2.04847s (17.5741GiB/s) | 20260506 13:25:13.567 | INFO | Downloaded 34GiB to GPU in 1.9624s (17.3257GiB/s) | 20260506 13:25:13.567 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:25:13.567 | INFO | Total mem usage: 17.4 GiB | 20260506 13:25:13.567 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:25:13.567 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -6.02%) | 20260506 13:25:13.567 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:25:13.567 | INFO | High Water Breakdown: | 20260506 13:25:13.567 | INFO | 61.8% : 10.1 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:25:13.567 | INFO | 38.1% : 6.26 GiB ( 93.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:25:13.567 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:25:13.567 | INFO | ------------------------------------------ | 20260506 13:25:13.567 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:25:13.567 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:25:13.567 | INFO | Total mem usage: 26.5 GiB | 20260506 13:25:13.567 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:25:13.567 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260506 13:25:13.567 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260506 13:25:13.567 | INFO | High Water Breakdown: | 20260506 13:25:13.567 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:25:13.567 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:25:13.567 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:25:13.567 | INFO | 1.99% : 523 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:25:13.567 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:25:13.567 | INFO | 0.695% : 182 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:25:13.567 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:25:13.567 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:25:13.567 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:25:13.567 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:25:13.567 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:25:13.567 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:25:13.567 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:25:13.567 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:25:13.567 | INFO | ------------------------------------------ | 20260506 13:25:13.567 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:25:13.568 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:25:13.568 | INFO | Total mem usage: 25.6 GiB | 20260506 13:25:13.568 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:25:13.568 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:25:13.568 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:25:13.568 | INFO | High Water Breakdown: | 20260506 13:25:13.568 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:25:13.568 | INFO | ------------------------------------------ | 20260506 13:25:13.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:25:13.568 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:25:13.568 | INFO | Total mem usage: 15.7 GiB | 20260506 13:25:13.568 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:25:13.568 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:25:13.568 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 13:25:13.568 | INFO | High Water Breakdown: | 20260506 13:25:13.568 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:25:13.568 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:25:13.568 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:25:13.568 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:25:13.568 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:25:13.568 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:25:13.568 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:25:13.568 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:25:13.568 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:25:13.568 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:25:13.568 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:25:13.568 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:25:13.568 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:25:13.568 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:25:13.568 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:25:13.568 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:25:13.568 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:25:13.568 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:25:13.568 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:25:13.568 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:25:13.568 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:25:13.568 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:25:13.568 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:25:13.568 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:25:13.568 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:25:13.568 | INFO | ------------------------------------------ | 20260506 13:25:13.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:25:13.568 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:25:13.568 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:25:13.568 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:25:13.568 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.59e+01, 1.35e+01 / 2.40e+00 | 20260506 13:25:13.568 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.03e-01, 1.04e-02 / 6.50e-03 | 20260506 13:25:13.568 | INFO | PerfReport: 11.55%, Baseline -- 3.66e+00 / 4.40e+00, 5.58e-01 / 3.45e-01 | 20260506 13:25:13.568 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.46e+00, 1.87e-01 / 1.37e-01 | 20260506 13:25:13.568 | INFO | PerfReport: 15.80%, DME -- 4.00e+01 / 4.42e+01, 3.96e+00 / 2.67e+00 | 20260506 13:25:13.568 | INFO | PerfReport: 56.63%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.47e+00 / 2.00e+00 | 20260506 13:25:13.568 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.38e+00 / 2.99e+00, 4.09e-01 / 2.54e-01 | 20260506 13:25:13.568 | INFO | PerfReport: 3.69%, Metrics -- 1.17e+00 / 1.54e+00, 3.00e-01 / 1.66e-01 | 20260506 13:25:13.568 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.09e-01 / 4.57e-01, 6.37e-02 / 2.84e-02 | 20260506 13:25:13.568 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:25:13.568 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:25:13.568 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:25:13.568 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 13:25:13.568 | INFO | PerfReport: 21.74%, Upload -- 2.14e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260506 13:25:13.568 | INFO | PerfReport: 21.08%, Download -- 2.07e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260506 13:25:13.568 | INFO | PerfReport: 32.21%, Compute -- 3.17e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 13:25:13.568 | INFO | PerfReport: 1.89%, LockUpload -- 1.85e+00 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 13:25:13.568 | INFO | PerfReport: 0.93%, LockDownload -- 9.15e-01 / 0.00e+00, 2.44e+00 / 0.00e+00 | 20260506 13:25:13.568 | INFO | PerfReport: 22.14%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 13:25:13.568 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260506 13:25:13.697 | INFO | Starting a round of Baz IO | 20260506 13:25:16.887 | INFO | Populating RTMetrics report | 20260506 13:25:16.888 | INFO | RTMetrics report populated | 20260506 13:25:16.888 | INFO | Opening tmp RTMetrics json file | 20260506 13:25:16.927 | INFO | Serializing RTMetrics to json | 20260506 13:25:16.930 | INFO | Serialization done, swapping tmp file now... | 20260506 13:25:16.930 | INFO | RTMetrics done | 20260506 13:25:17.895 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 4.197 seconds | 20260506 13:25:17.895 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.76% / 0.02% / 60.22% | 20260506 13:25:17.895 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.20688% / 3.97364e-05% / 5.16574e-05% | 20260506 13:25:17.895 | INFO | Baz IO overall: | 20260506 13:25:17.895 | INFO | BytesWritten : 860924115881 | 20260506 13:25:17.895 | INFO | NumEvents : 416395292658 | 20260506 13:25:18.706 | INFO | Analyzing chunk frames = [1695744,1696256) | 20260506 13:25:18.706 | INFO | MemInfo: { | 20260506 13:25:18.706 | INFO | "gpu" : " 17 GB", | 20260506 13:25:18.706 | INFO | "huge" : "", | 20260506 13:25:18.706 | INFO | "huge_shared" : " 27 GB", | 20260506 13:25:18.706 | INFO | "malloc" : " 14 GB", | 20260506 13:25:18.706 | INFO | "pinned" : " 29 GB" | 20260506 13:25:18.706 | INFO | } | 20260506 13:25:18.706 | INFO | Perf report for the last 5.13937 seconds: | 20260506 13:25:18.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:25:18.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:18.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:18.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:18.706 | INFO | ANALYSIS : 63.7%, 2.8, 95.2ms, 0 | 20260506 13:25:18.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:18.706 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260506 13:25:18.706 | INFO | PRE_HQ : 61.8%, 2.0, 62.8ms, 0 | 20260506 13:25:18.706 | INFO | BAZWRITER : 81.0%, 4.8, 201.2ms, 0 | 20260506 13:25:18.706 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 13:25:18.707 | INFO | Uploaded 36GiB to GPU in 2.10634s (17.0912GiB/s) | 20260506 13:25:18.707 | INFO | Downloaded 38GiB to GPU in 2.2385s (16.9757GiB/s) | 20260506 13:25:23.845 | INFO | Analyzing chunk frames = [1696256,1696768) | 20260506 13:25:23.845 | INFO | MemInfo: { | 20260506 13:25:23.845 | INFO | "gpu" : " 17 GB", | 20260506 13:25:23.845 | INFO | "huge" : "", | 20260506 13:25:23.845 | INFO | "huge_shared" : " 27 GB", | 20260506 13:25:23.846 | INFO | "malloc" : " 14 GB", | 20260506 13:25:23.846 | INFO | "pinned" : " 29 GB" | 20260506 13:25:23.846 | INFO | } | 20260506 13:25:23.846 | INFO | Perf report for the last 5.13945 seconds: | 20260506 13:25:23.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:25:23.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:23.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:23.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:23.846 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:25:23.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:23.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:23.846 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:25:23.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:23.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:23.846 | INFO | Uploaded 36GiB to GPU in 2.04488s (17.6049GiB/s) | 20260506 13:25:23.846 | INFO | Downloaded 34GiB to GPU in 1.93529s (17.5684GiB/s) | 20260506 13:25:28.985 | INFO | Analyzing chunk frames = [1696768,1697280) | 20260506 13:25:28.985 | INFO | MemInfo: { | 20260506 13:25:28.985 | INFO | "gpu" : " 17 GB", | 20260506 13:25:28.985 | INFO | "huge" : "", | 20260506 13:25:28.985 | INFO | "huge_shared" : " 27 GB", | 20260506 13:25:28.985 | INFO | "malloc" : " 14 GB", | 20260506 13:25:28.985 | INFO | "pinned" : " 29 GB" | 20260506 13:25:28.985 | INFO | } | 20260506 13:25:28.985 | INFO | Perf report for the last 5.13963 seconds: | 20260506 13:25:28.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:25:28.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:28.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:28.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:28.985 | INFO | ANALYSIS : 61.7%, 3.0, 95.5ms, 0 | 20260506 13:25:28.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:28.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:28.985 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 13:25:28.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:28.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:28.985 | INFO | Uploaded 36GiB to GPU in 2.00764s (17.9315GiB/s) | 20260506 13:25:28.985 | INFO | Downloaded 34GiB to GPU in 1.93857s (17.5387GiB/s) | 20260506 13:25:34.124 | INFO | Analyzing chunk frames = [1697280,1697792) | 20260506 13:25:34.124 | INFO | MemInfo: { | 20260506 13:25:34.124 | INFO | "gpu" : " 17 GB", | 20260506 13:25:34.124 | INFO | "huge" : "", | 20260506 13:25:34.124 | INFO | "huge_shared" : " 27 GB", | 20260506 13:25:34.124 | INFO | "malloc" : " 14 GB", | 20260506 13:25:34.124 | INFO | "pinned" : " 29 GB" | 20260506 13:25:34.124 | INFO | } | 20260506 13:25:34.124 | INFO | Perf report for the last 5.13884 seconds: | 20260506 13:25:34.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:25:34.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:34.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:34.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:34.124 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:25:34.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:34.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:34.124 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:25:34.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:34.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:34.124 | INFO | Uploaded 36GiB to GPU in 2.00556s (17.9501GiB/s) | 20260506 13:25:34.124 | INFO | Downloaded 34GiB to GPU in 1.93115s (17.6061GiB/s) | 20260506 13:25:39.264 | INFO | Analyzing chunk frames = [1697792,1698304) | 20260506 13:25:39.264 | INFO | MemInfo: { | 20260506 13:25:39.264 | INFO | "gpu" : " 17 GB", | 20260506 13:25:39.264 | INFO | "huge" : "", | 20260506 13:25:39.264 | INFO | "huge_shared" : " 27 GB", | 20260506 13:25:39.264 | INFO | "malloc" : " 14 GB", | 20260506 13:25:39.264 | INFO | "pinned" : " 29 GB" | 20260506 13:25:39.264 | INFO | } | 20260506 13:25:39.264 | INFO | Perf report for the last 5.13952 seconds: | 20260506 13:25:39.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:25:39.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:39.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:39.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:39.264 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 13:25:39.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:39.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:39.264 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:25:39.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:39.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:39.264 | INFO | Uploaded 36GiB to GPU in 2.02792s (17.7522GiB/s) | 20260506 13:25:39.264 | INFO | Downloaded 34GiB to GPU in 1.94444s (17.4858GiB/s) | 20260506 13:25:44.403 | INFO | Analyzing chunk frames = [1698304,1698816) | 20260506 13:25:44.403 | INFO | MemInfo: { | 20260506 13:25:44.403 | INFO | "gpu" : " 17 GB", | 20260506 13:25:44.403 | INFO | "huge" : "", | 20260506 13:25:44.403 | INFO | "huge_shared" : " 27 GB", | 20260506 13:25:44.403 | INFO | "malloc" : " 14 GB", | 20260506 13:25:44.403 | INFO | "pinned" : " 29 GB" | 20260506 13:25:44.403 | INFO | } | 20260506 13:25:44.403 | INFO | Perf report for the last 5.13954 seconds: | 20260506 13:25:44.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:25:44.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:44.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:44.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:44.403 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:25:44.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:44.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:44.403 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:25:44.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:44.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:44.403 | INFO | Uploaded 36GiB to GPU in 2.05607s (17.5091GiB/s) | 20260506 13:25:44.403 | INFO | Downloaded 34GiB to GPU in 1.94777s (17.4558GiB/s) | 20260506 13:25:49.543 | INFO | Analyzing chunk frames = [1698816,1699328) | 20260506 13:25:49.543 | INFO | MemInfo: { | 20260506 13:25:49.543 | INFO | "gpu" : " 17 GB", | 20260506 13:25:49.543 | INFO | "huge" : "", | 20260506 13:25:49.543 | INFO | "huge_shared" : " 27 GB", | 20260506 13:25:49.543 | INFO | "malloc" : " 14 GB", | 20260506 13:25:49.543 | INFO | "pinned" : " 29 GB" | 20260506 13:25:49.543 | INFO | } | 20260506 13:25:49.544 | INFO | Perf report for the last 5.14044 seconds: | 20260506 13:25:49.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:25:49.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:49.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:49.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:49.544 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:25:49.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:49.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:49.544 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 13:25:49.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:49.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:49.544 | INFO | Uploaded 36GiB to GPU in 2.01048s (17.9062GiB/s) | 20260506 13:25:49.544 | INFO | Downloaded 34GiB to GPU in 2.0263s (16.7793GiB/s) | 20260506 13:25:54.681 | INFO | Analyzing chunk frames = [1699328,1699840) | 20260506 13:25:54.682 | INFO | MemInfo: { | 20260506 13:25:54.682 | INFO | "gpu" : " 17 GB", | 20260506 13:25:54.682 | INFO | "huge" : "", | 20260506 13:25:54.682 | INFO | "huge_shared" : " 27 GB", | 20260506 13:25:54.682 | INFO | "malloc" : " 14 GB", | 20260506 13:25:54.682 | INFO | "pinned" : " 29 GB" | 20260506 13:25:54.682 | INFO | } | 20260506 13:25:54.682 | INFO | Perf report for the last 5.13813 seconds: | 20260506 13:25:54.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:25:54.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:54.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:54.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:54.682 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 13:25:54.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:54.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:54.682 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:25:54.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:54.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:54.682 | INFO | Uploaded 36GiB to GPU in 2.0192s (17.8288GiB/s) | 20260506 13:25:54.682 | INFO | Downloaded 34GiB to GPU in 1.96792s (17.2771GiB/s) | 20260506 13:25:54.807 | INFO | Starting a round of Baz IO | 20260506 13:25:57.975 | INFO | Populating RTMetrics report | 20260506 13:25:57.975 | INFO | RTMetrics report populated | 20260506 13:25:57.975 | INFO | Opening tmp RTMetrics json file | 20260506 13:25:57.996 | INFO | Serializing RTMetrics to json | 20260506 13:25:58.006 | INFO | Serialization done, swapping tmp file now... | 20260506 13:25:58.007 | INFO | RTMetrics done | 20260506 13:25:58.693 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 3.885 seconds | 20260506 13:25:58.693 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.74% / 0.02% / 60.24% | 20260506 13:25:58.693 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.19889% / 2.78155e-05% / 5.96046e-05% | 20260506 13:25:58.693 | INFO | Baz IO overall: | 20260506 13:25:58.693 | INFO | BytesWritten : 863192874649 | 20260506 13:25:58.693 | INFO | NumEvents : 417496010619 | 20260506 13:25:59.821 | INFO | Analyzing chunk frames = [1699840,1700352) | 20260506 13:25:59.821 | INFO | MemInfo: { | 20260506 13:25:59.821 | INFO | "gpu" : " 17 GB", | 20260506 13:25:59.821 | INFO | "huge" : "", | 20260506 13:25:59.821 | INFO | "huge_shared" : " 27 GB", | 20260506 13:25:59.821 | INFO | "malloc" : " 14 GB", | 20260506 13:25:59.821 | INFO | "pinned" : " 29 GB" | 20260506 13:25:59.821 | INFO | } | 20260506 13:25:59.821 | INFO | Perf report for the last 5.13904 seconds: | 20260506 13:25:59.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:25:59.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:59.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:59.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:59.821 | INFO | ANALYSIS : 63.5%, 2.9, 95.7ms, 0 | 20260506 13:25:59.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:25:59.821 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 13:25:59.821 | INFO | PRE_HQ : 61.4%, 1.9, 60.3ms, 0 | 20260506 13:25:59.821 | INFO | BAZWRITER : 74.9%, 4.8, 185.7ms, 0 | 20260506 13:25:59.821 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 13:25:59.821 | INFO | Uploaded 36GiB to GPU in 2.07527s (17.3471GiB/s) | 20260506 13:25:59.821 | INFO | Downloaded 38GiB to GPU in 2.22858s (17.0512GiB/s) | 20260506 13:26:04.960 | INFO | Analyzing chunk frames = [1700352,1700864) | 20260506 13:26:04.960 | INFO | MemInfo: { | 20260506 13:26:04.960 | INFO | "gpu" : " 17 GB", | 20260506 13:26:04.960 | INFO | "huge" : "", | 20260506 13:26:04.960 | INFO | "huge_shared" : " 27 GB", | 20260506 13:26:04.960 | INFO | "malloc" : " 14 GB", | 20260506 13:26:04.960 | INFO | "pinned" : " 29 GB" | 20260506 13:26:04.960 | INFO | } | 20260506 13:26:04.961 | INFO | Perf report for the last 5.13965 seconds: | 20260506 13:26:04.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:26:04.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:04.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:04.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:04.961 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:26:04.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:04.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:04.961 | INFO | PRE_HQ : 28.7%, 1.0, 15.1ms, 0 | 20260506 13:26:04.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:04.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:04.961 | INFO | Uploaded 36GiB to GPU in 2.0414s (17.6349GiB/s) | 20260506 13:26:04.961 | INFO | Downloaded 34GiB to GPU in 1.9392s (17.533GiB/s) | 20260506 13:26:10.099 | INFO | Analyzing chunk frames = [1700864,1701376) | 20260506 13:26:10.100 | INFO | MemInfo: { | 20260506 13:26:10.100 | INFO | "gpu" : " 17 GB", | 20260506 13:26:10.100 | INFO | "huge" : "", | 20260506 13:26:10.100 | INFO | "huge_shared" : " 27 GB", | 20260506 13:26:10.100 | INFO | "malloc" : " 14 GB", | 20260506 13:26:10.100 | INFO | "pinned" : " 29 GB" | 20260506 13:26:10.100 | INFO | } | 20260506 13:26:10.100 | INFO | Perf report for the last 5.13968 seconds: | 20260506 13:26:10.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:26:10.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:10.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:10.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:10.100 | INFO | ANALYSIS : 61.7%, 3.0, 95.5ms, 0 | 20260506 13:26:10.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:10.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:10.100 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:26:10.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:10.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:10.100 | INFO | Uploaded 36GiB to GPU in 2.0298s (17.7357GiB/s) | 20260506 13:26:10.100 | INFO | Downloaded 34GiB to GPU in 1.94017s (17.5242GiB/s) | 20260506 13:26:15.239 | INFO | Analyzing chunk frames = [1701376,1701888) | 20260506 13:26:15.239 | INFO | MemInfo: { | 20260506 13:26:15.239 | INFO | "gpu" : " 17 GB", | 20260506 13:26:15.239 | INFO | "huge" : "", | 20260506 13:26:15.239 | INFO | "huge_shared" : " 27 GB", | 20260506 13:26:15.239 | INFO | "malloc" : " 14 GB", | 20260506 13:26:15.239 | INFO | "pinned" : " 29 GB" | 20260506 13:26:15.239 | INFO | } | 20260506 13:26:15.239 | INFO | Perf report for the last 5.1389 seconds: | 20260506 13:26:15.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:26:15.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:15.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:15.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:15.239 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:26:15.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:15.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:15.239 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:26:15.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:15.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:15.239 | INFO | Uploaded 36GiB to GPU in 2.01486s (17.8673GiB/s) | 20260506 13:26:15.239 | INFO | Downloaded 34GiB to GPU in 1.94014s (17.5245GiB/s) | 20260506 13:26:20.378 | INFO | Analyzing chunk frames = [1701888,1702400) | 20260506 13:26:20.378 | INFO | MemInfo: { | 20260506 13:26:20.378 | INFO | "gpu" : " 17 GB", | 20260506 13:26:20.378 | INFO | "huge" : "", | 20260506 13:26:20.378 | INFO | "huge_shared" : " 27 GB", | 20260506 13:26:20.378 | INFO | "malloc" : " 14 GB", | 20260506 13:26:20.378 | INFO | "pinned" : " 29 GB" | 20260506 13:26:20.378 | INFO | } | 20260506 13:26:20.379 | INFO | Perf report for the last 5.13947 seconds: | 20260506 13:26:20.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:26:20.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:20.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:20.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:20.379 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 13:26:20.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:20.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:20.379 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:26:20.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:20.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:20.379 | INFO | Uploaded 36GiB to GPU in 2.01034s (17.9074GiB/s) | 20260506 13:26:20.379 | INFO | Downloaded 34GiB to GPU in 1.93812s (17.5428GiB/s) | 20260506 13:26:25.517 | INFO | Analyzing chunk frames = [1702400,1702912) | 20260506 13:26:25.517 | INFO | MemInfo: { | 20260506 13:26:25.517 | INFO | "gpu" : " 17 GB", | 20260506 13:26:25.517 | INFO | "huge" : "", | 20260506 13:26:25.518 | INFO | "huge_shared" : " 27 GB", | 20260506 13:26:25.518 | INFO | "malloc" : " 14 GB", | 20260506 13:26:25.518 | INFO | "pinned" : " 29 GB" | 20260506 13:26:25.518 | INFO | } | 20260506 13:26:25.518 | INFO | Perf report for the last 5.13914 seconds: | 20260506 13:26:25.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:26:25.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:25.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:25.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:25.518 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 13:26:25.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:25.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:25.518 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 13:26:25.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:25.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:25.518 | INFO | Uploaded 36GiB to GPU in 2.09619s (17.174GiB/s) | 20260506 13:26:25.518 | INFO | Downloaded 34GiB to GPU in 1.96864s (17.2708GiB/s) | 20260506 13:26:30.657 | INFO | Analyzing chunk frames = [1702912,1703424) | 20260506 13:26:30.657 | INFO | MemInfo: { | 20260506 13:26:30.657 | INFO | "gpu" : " 17 GB", | 20260506 13:26:30.657 | INFO | "huge" : "", | 20260506 13:26:30.657 | INFO | "huge_shared" : " 27 GB", | 20260506 13:26:30.657 | INFO | "malloc" : " 14 GB", | 20260506 13:26:30.657 | INFO | "pinned" : " 29 GB" | 20260506 13:26:30.657 | INFO | } | 20260506 13:26:30.657 | INFO | Perf report for the last 5.13958 seconds: | 20260506 13:26:30.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:26:30.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:30.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:30.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:30.658 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:26:30.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:30.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:30.658 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:26:30.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:30.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:30.658 | INFO | Uploaded 36GiB to GPU in 2.00409s (17.9633GiB/s) | 20260506 13:26:30.658 | INFO | Downloaded 34GiB to GPU in 2.03426s (16.7137GiB/s) | 20260506 13:26:35.796 | INFO | Analyzing chunk frames = [1703424,1703936) | 20260506 13:26:35.796 | INFO | MemInfo: { | 20260506 13:26:35.796 | INFO | "gpu" : " 17 GB", | 20260506 13:26:35.796 | INFO | "huge" : "", | 20260506 13:26:35.796 | INFO | "huge_shared" : " 27 GB", | 20260506 13:26:35.796 | INFO | "malloc" : " 14 GB", | 20260506 13:26:35.796 | INFO | "pinned" : " 29 GB" | 20260506 13:26:35.796 | INFO | } | 20260506 13:26:35.796 | INFO | Perf report for the last 5.13909 seconds: | 20260506 13:26:35.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:26:35.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:35.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:35.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:35.797 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:26:35.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:35.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:35.797 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 13:26:35.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:35.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:35.797 | INFO | Uploaded 36GiB to GPU in 2.0537s (17.5293GiB/s) | 20260506 13:26:35.797 | INFO | Downloaded 34GiB to GPU in 1.96494s (17.3033GiB/s) | 20260506 13:26:35.797 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:26:35.797 | INFO | Total mem usage: 17.4 GiB | 20260506 13:26:35.797 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:26:35.797 | INFO | Recent High Water Marks Sum: 16.9 GiB (Savings: -3.31%) | 20260506 13:26:35.797 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:26:35.797 | INFO | High Water Breakdown: | 20260506 13:26:35.797 | INFO | 62.2% : 10.5 GiB ( 96% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:26:35.797 | INFO | 37.7% : 6.35 GiB ( 94.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:26:35.797 | INFO | 0.118% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:26:35.797 | INFO | ------------------------------------------ | 20260506 13:26:35.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:26:35.797 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:26:35.797 | INFO | Total mem usage: 26.5 GiB | 20260506 13:26:35.797 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:26:35.797 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:26:35.797 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:26:35.797 | INFO | High Water Breakdown: | 20260506 13:26:35.797 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:26:35.797 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:26:35.797 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:26:35.797 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:26:35.797 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:26:35.797 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:26:35.797 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:26:35.797 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:26:35.797 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:26:35.797 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:26:35.797 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:26:35.797 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:26:35.797 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:26:35.797 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:26:35.797 | INFO | ------------------------------------------ | 20260506 13:26:35.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:26:35.797 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:26:35.798 | INFO | Total mem usage: 25.6 GiB | 20260506 13:26:35.798 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:26:35.798 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:26:35.798 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:26:35.798 | INFO | High Water Breakdown: | 20260506 13:26:35.798 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:26:35.798 | INFO | ------------------------------------------ | 20260506 13:26:35.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:26:35.798 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:26:35.798 | INFO | Total mem usage: 15.7 GiB | 20260506 13:26:35.798 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:26:35.798 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:26:35.798 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 13:26:35.798 | INFO | High Water Breakdown: | 20260506 13:26:35.798 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:26:35.798 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:26:35.798 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:26:35.798 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:26:35.798 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:26:35.798 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:26:35.798 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:26:35.798 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:26:35.798 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:26:35.798 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:26:35.798 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:26:35.798 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:26:35.798 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:26:35.798 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:26:35.798 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:26:35.798 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:26:35.798 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:26:35.798 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:26:35.798 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:26:35.798 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:26:35.798 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:26:35.798 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:26:35.798 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:26:35.798 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:26:35.798 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:26:35.798 | INFO | ------------------------------------------ | 20260506 13:26:35.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:26:35.798 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:26:35.798 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:26:35.798 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:26:35.798 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.64e+01, 1.34e+01 / 2.71e+00 | 20260506 13:26:35.798 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.04e-01, 1.01e-02 / 7.76e-03 | 20260506 13:26:35.798 | INFO | PerfReport: 11.56%, Baseline -- 3.66e+00 / 4.45e+00, 5.78e-01 / 3.11e-01 | 20260506 13:26:35.798 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.47e+00, 1.90e-01 / 1.40e-01 | 20260506 13:26:35.798 | INFO | PerfReport: 15.83%, DME -- 4.01e+01 / 4.47e+01, 4.27e+00 / 3.37e+00 | 20260506 13:26:35.798 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.44e+00 / 1.82e+00 | 20260506 13:26:35.798 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.38e+00 / 2.98e+00, 4.15e-01 / 2.73e-01 | 20260506 13:26:35.798 | INFO | PerfReport: 3.67%, Metrics -- 1.16e+00 / 1.52e+00, 2.98e-01 / 1.43e-01 | 20260506 13:26:35.798 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.08e-01 / 4.67e-01, 6.45e-02 / 2.77e-02 | 20260506 13:26:35.798 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:26:35.798 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:26:35.798 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:26:35.798 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 13:26:35.798 | INFO | PerfReport: 21.60%, Upload -- 2.12e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260506 13:26:35.798 | INFO | PerfReport: 21.13%, Download -- 2.08e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 13:26:35.798 | INFO | PerfReport: 32.23%, Compute -- 3.17e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 13:26:35.798 | INFO | PerfReport: 1.82%, LockUpload -- 1.79e+00 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260506 13:26:35.798 | INFO | PerfReport: 0.99%, LockDownload -- 9.75e-01 / 0.00e+00, 2.59e+00 / 0.00e+00 | 20260506 13:26:35.798 | INFO | PerfReport: 22.21%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 13:26:35.798 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260506 13:26:35.931 | INFO | Starting a round of Baz IO | 20260506 13:26:39.089 | INFO | Populating RTMetrics report | 20260506 13:26:39.089 | INFO | RTMetrics report populated | 20260506 13:26:39.089 | INFO | Opening tmp RTMetrics json file | 20260506 13:26:39.149 | INFO | Serializing RTMetrics to json | 20260506 13:26:39.201 | INFO | Serialization done, swapping tmp file now... | 20260506 13:26:39.203 | INFO | RTMetrics done | 20260506 13:26:39.829 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.898 seconds | 20260506 13:26:39.829 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.72% / 0.02% / 60.26% | 20260506 13:26:39.829 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.18595% / 2.78155e-05% / 3.57628e-05% | 20260506 13:26:39.829 | INFO | Baz IO overall: | 20260506 13:26:39.829 | INFO | BytesWritten : 865459839082 | 20260506 13:26:39.829 | INFO | NumEvents : 418595321335 | 20260506 13:26:40.936 | INFO | Analyzing chunk frames = [1703936,1704448) | 20260506 13:26:40.936 | INFO | MemInfo: { | 20260506 13:26:40.936 | INFO | "gpu" : " 17 GB", | 20260506 13:26:40.936 | INFO | "huge" : "", | 20260506 13:26:40.936 | INFO | "huge_shared" : " 27 GB", | 20260506 13:26:40.936 | INFO | "malloc" : " 14 GB", | 20260506 13:26:40.936 | INFO | "pinned" : " 29 GB" | 20260506 13:26:40.936 | INFO | } | 20260506 13:26:40.936 | INFO | Perf report for the last 5.13931 seconds: | 20260506 13:26:40.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:26:40.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:40.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:40.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:40.936 | INFO | ANALYSIS : 63.5%, 2.9, 95.9ms, 0 | 20260506 13:26:40.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:40.936 | INFO | RT_METRICS : 46.2%, 1.0, 24.3ms, 0 | 20260506 13:26:40.936 | INFO | PRE_HQ : 61.3%, 1.9, 60.2ms, 0 | 20260506 13:26:40.936 | INFO | BAZWRITER : 75.6%, 4.8, 184.8ms, 0 | 20260506 13:26:40.936 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 13:26:40.936 | INFO | Uploaded 36GiB to GPU in 2.06309s (17.4496GiB/s) | 20260506 13:26:40.936 | INFO | Downloaded 38GiB to GPU in 2.23377s (17.0116GiB/s) | 20260506 13:26:46.075 | INFO | Analyzing chunk frames = [1704448,1704960) | 20260506 13:26:46.075 | INFO | MemInfo: { | 20260506 13:26:46.075 | INFO | "gpu" : " 17 GB", | 20260506 13:26:46.075 | INFO | "huge" : "", | 20260506 13:26:46.075 | INFO | "huge_shared" : " 27 GB", | 20260506 13:26:46.075 | INFO | "malloc" : " 14 GB", | 20260506 13:26:46.075 | INFO | "pinned" : " 29 GB" | 20260506 13:26:46.075 | INFO | } | 20260506 13:26:46.075 | INFO | Perf report for the last 5.1393 seconds: | 20260506 13:26:46.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:26:46.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:46.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:46.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:46.075 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:26:46.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:46.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:46.075 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:26:46.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:46.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:46.075 | INFO | Uploaded 36GiB to GPU in 2.03987s (17.6482GiB/s) | 20260506 13:26:46.075 | INFO | Downloaded 34GiB to GPU in 1.9473s (17.4601GiB/s) | 20260506 13:26:51.214 | INFO | Analyzing chunk frames = [1704960,1705472) | 20260506 13:26:51.214 | INFO | MemInfo: { | 20260506 13:26:51.214 | INFO | "gpu" : " 17 GB", | 20260506 13:26:51.214 | INFO | "huge" : "", | 20260506 13:26:51.214 | INFO | "huge_shared" : " 27 GB", | 20260506 13:26:51.214 | INFO | "malloc" : " 14 GB", | 20260506 13:26:51.214 | INFO | "pinned" : " 29 GB" | 20260506 13:26:51.214 | INFO | } | 20260506 13:26:51.214 | INFO | Perf report for the last 5.13934 seconds: | 20260506 13:26:51.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:26:51.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:51.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:51.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:51.214 | INFO | ANALYSIS : 61.7%, 3.0, 95.5ms, 0 | 20260506 13:26:51.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:51.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:51.215 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:26:51.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:51.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:51.215 | INFO | Uploaded 36GiB to GPU in 2.02933s (17.7398GiB/s) | 20260506 13:26:51.215 | INFO | Downloaded 34GiB to GPU in 1.93935s (17.5317GiB/s) | 20260506 13:26:56.354 | INFO | Analyzing chunk frames = [1705472,1705984) | 20260506 13:26:56.354 | INFO | MemInfo: { | 20260506 13:26:56.354 | INFO | "gpu" : " 17 GB", | 20260506 13:26:56.354 | INFO | "huge" : "", | 20260506 13:26:56.354 | INFO | "huge_shared" : " 27 GB", | 20260506 13:26:56.354 | INFO | "malloc" : " 14 GB", | 20260506 13:26:56.354 | INFO | "pinned" : " 29 GB" | 20260506 13:26:56.354 | INFO | } | 20260506 13:26:56.354 | INFO | Perf report for the last 5.13973 seconds: | 20260506 13:26:56.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:26:56.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:56.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:56.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:56.354 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:26:56.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:56.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:56.354 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:26:56.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:56.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:26:56.355 | INFO | Uploaded 36GiB to GPU in 1.99231s (18.0695GiB/s) | 20260506 13:26:56.355 | INFO | Downloaded 34GiB to GPU in 1.92265s (17.6839GiB/s) | 20260506 13:27:01.493 | INFO | Analyzing chunk frames = [1705984,1706496) | 20260506 13:27:01.493 | INFO | MemInfo: { | 20260506 13:27:01.493 | INFO | "gpu" : " 17 GB", | 20260506 13:27:01.493 | INFO | "huge" : "", | 20260506 13:27:01.493 | INFO | "huge_shared" : " 27 GB", | 20260506 13:27:01.493 | INFO | "malloc" : " 14 GB", | 20260506 13:27:01.493 | INFO | "pinned" : " 29 GB" | 20260506 13:27:01.493 | INFO | } | 20260506 13:27:01.493 | INFO | Perf report for the last 5.13912 seconds: | 20260506 13:27:01.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:27:01.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:01.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:01.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:01.493 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 13:27:01.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:01.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:01.493 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 13:27:01.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:01.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:01.494 | INFO | Uploaded 36GiB to GPU in 2.0492s (17.5679GiB/s) | 20260506 13:27:01.494 | INFO | Downloaded 34GiB to GPU in 1.95339s (17.4056GiB/s) | 20260506 13:27:06.632 | INFO | Analyzing chunk frames = [1706496,1707008) | 20260506 13:27:06.632 | INFO | MemInfo: { | 20260506 13:27:06.632 | INFO | "gpu" : " 17 GB", | 20260506 13:27:06.632 | INFO | "huge" : "", | 20260506 13:27:06.632 | INFO | "huge_shared" : " 27 GB", | 20260506 13:27:06.632 | INFO | "malloc" : " 14 GB", | 20260506 13:27:06.632 | INFO | "pinned" : " 29 GB" | 20260506 13:27:06.632 | INFO | } | 20260506 13:27:06.633 | INFO | Perf report for the last 5.13928 seconds: | 20260506 13:27:06.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:27:06.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:06.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:06.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:06.633 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:27:06.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:06.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:06.633 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 13:27:06.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:06.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:06.633 | INFO | Uploaded 36GiB to GPU in 2.05847s (17.4887GiB/s) | 20260506 13:27:06.633 | INFO | Downloaded 34GiB to GPU in 1.92525s (17.66GiB/s) | 20260506 13:27:11.772 | INFO | Analyzing chunk frames = [1707008,1707520) | 20260506 13:27:11.772 | INFO | MemInfo: { | 20260506 13:27:11.772 | INFO | "gpu" : " 17 GB", | 20260506 13:27:11.772 | INFO | "huge" : "", | 20260506 13:27:11.772 | INFO | "huge_shared" : " 27 GB", | 20260506 13:27:11.772 | INFO | "malloc" : " 14 GB", | 20260506 13:27:11.772 | INFO | "pinned" : " 29 GB" | 20260506 13:27:11.772 | INFO | } | 20260506 13:27:11.772 | INFO | Perf report for the last 5.13986 seconds: | 20260506 13:27:11.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:27:11.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:11.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:11.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:11.773 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:27:11.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:11.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:11.773 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:27:11.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:11.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:11.773 | INFO | Uploaded 36GiB to GPU in 2.03056s (17.7291GiB/s) | 20260506 13:27:11.773 | INFO | Downloaded 34GiB to GPU in 1.95637s (17.3791GiB/s) | 20260506 13:27:16.911 | INFO | Analyzing chunk frames = [1707520,1708032) | 20260506 13:27:16.911 | INFO | MemInfo: { | 20260506 13:27:16.911 | INFO | "gpu" : " 17 GB", | 20260506 13:27:16.911 | INFO | "huge" : "", | 20260506 13:27:16.911 | INFO | "huge_shared" : " 27 GB", | 20260506 13:27:16.911 | INFO | "malloc" : " 14 GB", | 20260506 13:27:16.911 | INFO | "pinned" : " 29 GB" | 20260506 13:27:16.911 | INFO | } | 20260506 13:27:16.911 | INFO | Perf report for the last 5.13866 seconds: | 20260506 13:27:16.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:27:16.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:16.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:16.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:16.911 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 13:27:16.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:16.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:16.911 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:27:16.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:16.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:16.912 | INFO | Uploaded 36GiB to GPU in 2.03244s (17.7127GiB/s) | 20260506 13:27:16.912 | INFO | Downloaded 34GiB to GPU in 1.9871s (17.1104GiB/s) | 20260506 13:27:17.039 | INFO | Starting a round of Baz IO | 20260506 13:27:20.212 | INFO | Populating RTMetrics report | 20260506 13:27:20.212 | INFO | RTMetrics report populated | 20260506 13:27:20.212 | INFO | Opening tmp RTMetrics json file | 20260506 13:27:20.233 | INFO | Serializing RTMetrics to json | 20260506 13:27:20.324 | INFO | Serialization done, swapping tmp file now... | 20260506 13:27:20.348 | INFO | RTMetrics done | 20260506 13:27:20.933 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.894 seconds | 20260506 13:27:20.933 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.71% / 0.02% / 60.28% | 20260506 13:27:20.933 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.17785% / 3.17891e-05% / 6.35783e-05% | 20260506 13:27:20.934 | INFO | Baz IO overall: | 20260506 13:27:20.934 | INFO | BytesWritten : 867727108341 | 20260506 13:27:20.934 | INFO | NumEvents : 419694977956 | 20260506 13:27:22.050 | INFO | Analyzing chunk frames = [1708032,1708544) | 20260506 13:27:22.050 | INFO | MemInfo: { | 20260506 13:27:22.050 | INFO | "gpu" : " 17 GB", | 20260506 13:27:22.050 | INFO | "huge" : "", | 20260506 13:27:22.050 | INFO | "huge_shared" : " 27 GB", | 20260506 13:27:22.050 | INFO | "malloc" : " 14 GB", | 20260506 13:27:22.050 | INFO | "pinned" : " 29 GB" | 20260506 13:27:22.050 | INFO | } | 20260506 13:27:22.050 | INFO | Perf report for the last 5.13929 seconds: | 20260506 13:27:22.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:27:22.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:22.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:22.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:22.050 | INFO | ANALYSIS : 63.5%, 2.9, 96.3ms, 0 | 20260506 13:27:22.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:22.051 | INFO | RT_METRICS : 47.2%, 1.0, 24.8ms, 0 | 20260506 13:27:22.051 | INFO | PRE_HQ : 61.1%, 1.9, 59.9ms, 0 | 20260506 13:27:22.051 | INFO | BAZWRITER : 75.1%, 4.8, 187.1ms, 0 | 20260506 13:27:22.051 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.2ms, 0 | 20260506 13:27:22.051 | INFO | Uploaded 36GiB to GPU in 2.08712s (17.2487GiB/s) | 20260506 13:27:22.051 | INFO | Downloaded 38GiB to GPU in 2.24028s (16.9622GiB/s) | 20260506 13:27:27.190 | INFO | Analyzing chunk frames = [1708544,1709056) | 20260506 13:27:27.190 | INFO | MemInfo: { | 20260506 13:27:27.190 | INFO | "gpu" : " 17 GB", | 20260506 13:27:27.190 | INFO | "huge" : "", | 20260506 13:27:27.190 | INFO | "huge_shared" : " 27 GB", | 20260506 13:27:27.190 | INFO | "malloc" : " 14 GB", | 20260506 13:27:27.190 | INFO | "pinned" : " 29 GB" | 20260506 13:27:27.190 | INFO | } | 20260506 13:27:27.190 | INFO | Perf report for the last 5.13959 seconds: | 20260506 13:27:27.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:27:27.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:27.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:27.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:27.190 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 13:27:27.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:27.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:27.190 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 13:27:27.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:27.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:27.190 | INFO | Uploaded 36GiB to GPU in 2.02217s (17.8026GiB/s) | 20260506 13:27:27.190 | INFO | Downloaded 34GiB to GPU in 1.93147s (17.6032GiB/s) | 20260506 13:27:32.329 | INFO | Analyzing chunk frames = [1709056,1709568) | 20260506 13:27:32.329 | INFO | MemInfo: { | 20260506 13:27:32.329 | INFO | "gpu" : " 17 GB", | 20260506 13:27:32.329 | INFO | "huge" : "", | 20260506 13:27:32.329 | INFO | "huge_shared" : " 27 GB", | 20260506 13:27:32.329 | INFO | "malloc" : " 14 GB", | 20260506 13:27:32.329 | INFO | "pinned" : " 29 GB" | 20260506 13:27:32.329 | INFO | } | 20260506 13:27:32.329 | INFO | Perf report for the last 5.13912 seconds: | 20260506 13:27:32.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:27:32.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:32.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:32.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:32.329 | INFO | ANALYSIS : 61.6%, 3.0, 95.3ms, 0 | 20260506 13:27:32.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:32.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:32.329 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:27:32.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:32.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:32.329 | INFO | Uploaded 36GiB to GPU in 2.01613s (17.856GiB/s) | 20260506 13:27:32.329 | INFO | Downloaded 34GiB to GPU in 1.94128s (17.5143GiB/s) | 20260506 13:27:37.468 | INFO | Analyzing chunk frames = [1709568,1710080) | 20260506 13:27:37.468 | INFO | MemInfo: { | 20260506 13:27:37.468 | INFO | "gpu" : " 17 GB", | 20260506 13:27:37.468 | INFO | "huge" : "", | 20260506 13:27:37.468 | INFO | "huge_shared" : " 27 GB", | 20260506 13:27:37.468 | INFO | "malloc" : " 14 GB", | 20260506 13:27:37.468 | INFO | "pinned" : " 29 GB" | 20260506 13:27:37.468 | INFO | } | 20260506 13:27:37.468 | INFO | Perf report for the last 5.13922 seconds: | 20260506 13:27:37.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:27:37.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:37.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:37.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:37.469 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 13:27:37.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:37.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:37.469 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:27:37.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:37.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:37.469 | INFO | Uploaded 36GiB to GPU in 2.02346s (17.7913GiB/s) | 20260506 13:27:37.469 | INFO | Downloaded 34GiB to GPU in 1.91961s (17.7119GiB/s) | 20260506 13:27:42.608 | INFO | Analyzing chunk frames = [1710080,1710592) | 20260506 13:27:42.608 | INFO | MemInfo: { | 20260506 13:27:42.608 | INFO | "gpu" : " 17 GB", | 20260506 13:27:42.608 | INFO | "huge" : "", | 20260506 13:27:42.608 | INFO | "huge_shared" : " 27 GB", | 20260506 13:27:42.608 | INFO | "malloc" : " 14 GB", | 20260506 13:27:42.608 | INFO | "pinned" : " 29 GB" | 20260506 13:27:42.608 | INFO | } | 20260506 13:27:42.608 | INFO | Perf report for the last 5.13933 seconds: | 20260506 13:27:42.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:27:42.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:42.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:42.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:42.608 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 13:27:42.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:42.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:42.608 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:27:42.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:42.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:42.608 | INFO | Uploaded 36GiB to GPU in 2.06058s (17.4708GiB/s) | 20260506 13:27:42.608 | INFO | Downloaded 34GiB to GPU in 1.95513s (17.3901GiB/s) | 20260506 13:27:47.747 | INFO | Analyzing chunk frames = [1710592,1711104) | 20260506 13:27:47.747 | INFO | MemInfo: { | 20260506 13:27:47.747 | INFO | "gpu" : " 17 GB", | 20260506 13:27:47.747 | INFO | "huge" : "", | 20260506 13:27:47.747 | INFO | "huge_shared" : " 27 GB", | 20260506 13:27:47.747 | INFO | "malloc" : " 14 GB", | 20260506 13:27:47.747 | INFO | "pinned" : " 29 GB" | 20260506 13:27:47.747 | INFO | } | 20260506 13:27:47.747 | INFO | Perf report for the last 5.13965 seconds: | 20260506 13:27:47.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:27:47.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:47.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:47.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:47.747 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:27:47.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:47.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:47.748 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 13:27:47.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:47.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:47.748 | INFO | Uploaded 36GiB to GPU in 2.06039s (17.4724GiB/s) | 20260506 13:27:47.748 | INFO | Downloaded 34GiB to GPU in 1.93152s (17.6027GiB/s) | 20260506 13:27:52.887 | INFO | Analyzing chunk frames = [1711104,1711616) | 20260506 13:27:52.887 | INFO | MemInfo: { | 20260506 13:27:52.887 | INFO | "gpu" : " 17 GB", | 20260506 13:27:52.887 | INFO | "huge" : "", | 20260506 13:27:52.887 | INFO | "huge_shared" : " 27 GB", | 20260506 13:27:52.887 | INFO | "malloc" : " 14 GB", | 20260506 13:27:52.887 | INFO | "pinned" : " 29 GB" | 20260506 13:27:52.887 | INFO | } | 20260506 13:27:52.887 | INFO | Perf report for the last 5.13978 seconds: | 20260506 13:27:52.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:27:52.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:52.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:52.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:52.887 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:27:52.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:52.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:52.888 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 13:27:52.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:52.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:52.888 | INFO | Uploaded 36GiB to GPU in 2.02363s (17.7898GiB/s) | 20260506 13:27:52.888 | INFO | Downloaded 34GiB to GPU in 2.02674s (16.7757GiB/s) | 20260506 13:27:58.026 | INFO | Analyzing chunk frames = [1711616,1712128) | 20260506 13:27:58.026 | INFO | MemInfo: { | 20260506 13:27:58.026 | INFO | "gpu" : " 17 GB", | 20260506 13:27:58.026 | INFO | "huge" : "", | 20260506 13:27:58.026 | INFO | "huge_shared" : " 27 GB", | 20260506 13:27:58.026 | INFO | "malloc" : " 14 GB", | 20260506 13:27:58.026 | INFO | "pinned" : " 29 GB" | 20260506 13:27:58.026 | INFO | } | 20260506 13:27:58.026 | INFO | Perf report for the last 5.13878 seconds: | 20260506 13:27:58.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:27:58.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:58.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:58.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:58.026 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:27:58.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:58.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:58.026 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:27:58.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:58.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:27:58.026 | INFO | Uploaded 36GiB to GPU in 2.028s (17.7514GiB/s) | 20260506 13:27:58.026 | INFO | Downloaded 34GiB to GPU in 1.97729s (17.1953GiB/s) | 20260506 13:27:58.026 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:27:58.026 | INFO | Total mem usage: 17.4 GiB | 20260506 13:27:58.026 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:27:58.026 | INFO | Recent High Water Marks Sum: 16.7 GiB (Savings: -3.98%) | 20260506 13:27:58.026 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:27:58.026 | INFO | High Water Breakdown: | 20260506 13:27:58.026 | INFO | 62.1% : 10.4 GiB ( 95.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:27:58.026 | INFO | 37.8% : 6.32 GiB ( 94.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:27:58.026 | INFO | 0.119% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:27:58.026 | INFO | ------------------------------------------ | 20260506 13:27:58.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:27:58.027 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:27:58.027 | INFO | Total mem usage: 26.5 GiB | 20260506 13:27:58.027 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:27:58.027 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:27:58.027 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:27:58.027 | INFO | High Water Breakdown: | 20260506 13:27:58.027 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:27:58.027 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:27:58.027 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:27:58.027 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:27:58.027 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:27:58.027 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:27:58.027 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:27:58.027 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:27:58.027 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:27:58.027 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:27:58.027 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:27:58.027 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:27:58.027 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:27:58.027 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:27:58.027 | INFO | ------------------------------------------ | 20260506 13:27:58.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:27:58.027 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:27:58.027 | INFO | Total mem usage: 25.6 GiB | 20260506 13:27:58.027 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:27:58.027 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:27:58.027 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:27:58.027 | INFO | High Water Breakdown: | 20260506 13:27:58.027 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:27:58.027 | INFO | ------------------------------------------ | 20260506 13:27:58.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:27:58.027 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:27:58.027 | INFO | Total mem usage: 15.7 GiB | 20260506 13:27:58.027 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:27:58.027 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:27:58.027 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 13:27:58.027 | INFO | High Water Breakdown: | 20260506 13:27:58.027 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:27:58.027 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:27:58.027 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:27:58.027 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:27:58.027 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:27:58.027 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:27:58.027 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:27:58.027 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:27:58.027 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:27:58.027 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:27:58.027 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:27:58.027 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:27:58.027 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:27:58.027 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:27:58.027 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:27:58.027 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:27:58.027 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:27:58.027 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:27:58.027 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:27:58.027 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:27:58.027 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:27:58.027 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:27:58.027 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:27:58.027 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:27:58.027 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:27:58.027 | INFO | ------------------------------------------ | 20260506 13:27:58.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:27:58.027 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:27:58.027 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:27:58.027 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:27:58.027 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.59e+01, 1.34e+01 / 2.32e+00 | 20260506 13:27:58.027 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.02e-01, 1.10e-02 / 5.88e-03 | 20260506 13:27:58.027 | INFO | PerfReport: 11.57%, Baseline -- 3.67e+00 / 4.41e+00, 5.82e-01 / 3.51e-01 | 20260506 13:27:58.027 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.47e+00, 1.89e-01 / 1.40e-01 | 20260506 13:27:58.027 | INFO | PerfReport: 15.80%, DME -- 4.01e+01 / 4.42e+01, 3.99e+00 / 2.54e+00 | 20260506 13:27:58.027 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.80e+01 / 2.07e+01, 1.53e+00 / 2.06e+00 | 20260506 13:27:58.027 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.37e+00 / 2.94e+00, 4.07e-01 / 2.84e-01 | 20260506 13:27:58.027 | INFO | PerfReport: 3.68%, Metrics -- 1.17e+00 / 1.56e+00, 2.99e-01 / 1.82e-01 | 20260506 13:27:58.027 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.08e-01 / 4.66e-01, 6.42e-02 / 2.51e-02 | 20260506 13:27:58.027 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:27:58.027 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:27:58.027 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:27:58.027 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 13:27:58.027 | INFO | PerfReport: 21.60%, Upload -- 2.12e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260506 13:27:58.027 | INFO | PerfReport: 21.06%, Download -- 2.07e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 13:27:58.028 | INFO | PerfReport: 32.27%, Compute -- 3.17e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 13:27:58.028 | INFO | PerfReport: 1.79%, LockUpload -- 1.76e+00 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260506 13:27:58.028 | INFO | PerfReport: 0.98%, LockDownload -- 9.59e-01 / 0.00e+00, 2.58e+00 / 0.00e+00 | 20260506 13:27:58.028 | INFO | PerfReport: 22.29%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 13:27:58.028 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260506 13:27:58.156 | INFO | Starting a round of Baz IO | 20260506 13:28:01.317 | INFO | Populating RTMetrics report | 20260506 13:28:01.318 | INFO | RTMetrics report populated | 20260506 13:28:01.318 | INFO | Opening tmp RTMetrics json file | 20260506 13:28:01.419 | INFO | Serializing RTMetrics to json | 20260506 13:28:01.504 | INFO | Serialization done, swapping tmp file now... | 20260506 13:28:01.507 | INFO | RTMetrics done | 20260506 13:28:01.637 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.48 seconds | 20260506 13:28:01.637 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.69% / 0.02% / 60.29% | 20260506 13:28:01.637 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.16079% / 3.17891e-05% / 4.76837e-05% | 20260506 13:28:01.637 | INFO | Baz IO overall: | 20260506 13:28:01.637 | INFO | BytesWritten : 869992580877 | 20260506 13:28:01.637 | INFO | NumEvents : 420793225802 | 20260506 13:28:03.165 | INFO | Analyzing chunk frames = [1712128,1712640) | 20260506 13:28:03.165 | INFO | MemInfo: { | 20260506 13:28:03.165 | INFO | "gpu" : " 17 GB", | 20260506 13:28:03.165 | INFO | "huge" : "", | 20260506 13:28:03.165 | INFO | "huge_shared" : " 27 GB", | 20260506 13:28:03.165 | INFO | "malloc" : " 14 GB", | 20260506 13:28:03.165 | INFO | "pinned" : " 29 GB" | 20260506 13:28:03.165 | INFO | } | 20260506 13:28:03.165 | INFO | Perf report for the last 5.13926 seconds: | 20260506 13:28:03.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:28:03.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:03.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:03.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:03.165 | INFO | ANALYSIS : 63.4%, 2.9, 95.8ms, 0 | 20260506 13:28:03.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:03.165 | INFO | RT_METRICS : 47.9%, 1.0, 25.1ms, 0 | 20260506 13:28:03.165 | INFO | PRE_HQ : 61.3%, 1.9, 60.5ms, 0 | 20260506 13:28:03.165 | INFO | BAZWRITER : 67.0%, 4.8, 165.9ms, 0 | 20260506 13:28:03.165 | INFO | BAZ_SAMPLING : 3.0%, 1.0, 1.6ms, 0 | 20260506 13:28:03.165 | INFO | Uploaded 36GiB to GPU in 2.05672s (17.5036GiB/s) | 20260506 13:28:03.166 | INFO | Downloaded 38GiB to GPU in 2.22005s (17.1168GiB/s) | 20260506 13:28:08.305 | INFO | Analyzing chunk frames = [1712640,1713152) | 20260506 13:28:08.305 | INFO | MemInfo: { | 20260506 13:28:08.305 | INFO | "gpu" : " 17 GB", | 20260506 13:28:08.305 | INFO | "huge" : "", | 20260506 13:28:08.305 | INFO | "huge_shared" : " 27 GB", | 20260506 13:28:08.305 | INFO | "malloc" : " 14 GB", | 20260506 13:28:08.305 | INFO | "pinned" : " 29 GB" | 20260506 13:28:08.305 | INFO | } | 20260506 13:28:08.305 | INFO | Perf report for the last 5.13945 seconds: | 20260506 13:28:08.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:28:08.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:08.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:08.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:08.305 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:28:08.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:08.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:08.305 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:28:08.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:08.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:08.305 | INFO | Uploaded 36GiB to GPU in 2.01211s (17.8917GiB/s) | 20260506 13:28:08.305 | INFO | Downloaded 34GiB to GPU in 1.93588s (17.5631GiB/s) | 20260506 13:28:13.444 | INFO | Analyzing chunk frames = [1713152,1713664) | 20260506 13:28:13.444 | INFO | MemInfo: { | 20260506 13:28:13.444 | INFO | "gpu" : " 17 GB", | 20260506 13:28:13.444 | INFO | "huge" : "", | 20260506 13:28:13.444 | INFO | "huge_shared" : " 27 GB", | 20260506 13:28:13.444 | INFO | "malloc" : " 14 GB", | 20260506 13:28:13.444 | INFO | "pinned" : " 29 GB" | 20260506 13:28:13.444 | INFO | } | 20260506 13:28:13.444 | INFO | Perf report for the last 5.13936 seconds: | 20260506 13:28:13.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:28:13.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:13.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:13.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:13.444 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 13:28:13.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:13.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:13.444 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:28:13.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:13.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:13.444 | INFO | Uploaded 36GiB to GPU in 2.01767s (17.8423GiB/s) | 20260506 13:28:13.444 | INFO | Downloaded 34GiB to GPU in 1.9408s (17.5186GiB/s) | 20260506 13:28:18.583 | INFO | Analyzing chunk frames = [1713664,1714176) | 20260506 13:28:18.583 | INFO | MemInfo: { | 20260506 13:28:18.583 | INFO | "gpu" : " 17 GB", | 20260506 13:28:18.583 | INFO | "huge" : "", | 20260506 13:28:18.583 | INFO | "huge_shared" : " 27 GB", | 20260506 13:28:18.583 | INFO | "malloc" : " 14 GB", | 20260506 13:28:18.583 | INFO | "pinned" : " 29 GB" | 20260506 13:28:18.583 | INFO | } | 20260506 13:28:18.583 | INFO | Perf report for the last 5.13901 seconds: | 20260506 13:28:18.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:28:18.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:18.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:18.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:18.583 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:28:18.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:18.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:18.583 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:28:18.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:18.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:18.584 | INFO | Uploaded 36GiB to GPU in 1.99704s (18.0267GiB/s) | 20260506 13:28:18.584 | INFO | Downloaded 34GiB to GPU in 1.92279s (17.6826GiB/s) | 20260506 13:28:23.722 | INFO | Analyzing chunk frames = [1714176,1714688) | 20260506 13:28:23.722 | INFO | MemInfo: { | 20260506 13:28:23.722 | INFO | "gpu" : " 17 GB", | 20260506 13:28:23.722 | INFO | "huge" : "", | 20260506 13:28:23.722 | INFO | "huge_shared" : " 27 GB", | 20260506 13:28:23.722 | INFO | "malloc" : " 14 GB", | 20260506 13:28:23.723 | INFO | "pinned" : " 29 GB" | 20260506 13:28:23.723 | INFO | } | 20260506 13:28:23.723 | INFO | Perf report for the last 5.13964 seconds: | 20260506 13:28:23.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:28:23.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:23.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:23.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:23.723 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 13:28:23.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:23.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:23.723 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:28:23.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:23.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:23.723 | INFO | Uploaded 36GiB to GPU in 2.00668s (17.9401GiB/s) | 20260506 13:28:23.723 | INFO | Downloaded 34GiB to GPU in 1.93775s (17.5461GiB/s) | 20260506 13:28:28.862 | INFO | Analyzing chunk frames = [1714688,1715200) | 20260506 13:28:28.862 | INFO | MemInfo: { | 20260506 13:28:28.862 | INFO | "gpu" : " 17 GB", | 20260506 13:28:28.862 | INFO | "huge" : "", | 20260506 13:28:28.862 | INFO | "huge_shared" : " 27 GB", | 20260506 13:28:28.862 | INFO | "malloc" : " 14 GB", | 20260506 13:28:28.862 | INFO | "pinned" : " 29 GB" | 20260506 13:28:28.862 | INFO | } | 20260506 13:28:28.862 | INFO | Perf report for the last 5.13944 seconds: | 20260506 13:28:28.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:28:28.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:28.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:28.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:28.862 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:28:28.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:28.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:28.862 | INFO | PRE_HQ : 32.4%, 1.0, 16.9ms, 0 | 20260506 13:28:28.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:28.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:28.862 | INFO | Uploaded 36GiB to GPU in 2.09439s (17.1888GiB/s) | 20260506 13:28:28.862 | INFO | Downloaded 34GiB to GPU in 1.96958s (17.2625GiB/s) | 20260506 13:28:34.002 | INFO | Analyzing chunk frames = [1715200,1715712) | 20260506 13:28:34.002 | INFO | MemInfo: { | 20260506 13:28:34.002 | INFO | "gpu" : " 17 GB", | 20260506 13:28:34.002 | INFO | "huge" : "", | 20260506 13:28:34.002 | INFO | "huge_shared" : " 27 GB", | 20260506 13:28:34.002 | INFO | "malloc" : " 14 GB", | 20260506 13:28:34.002 | INFO | "pinned" : " 29 GB" | 20260506 13:28:34.002 | INFO | } | 20260506 13:28:34.002 | INFO | Perf report for the last 5.14021 seconds: | 20260506 13:28:34.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:28:34.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:34.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:34.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:34.003 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:28:34.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:34.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:34.003 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:28:34.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:34.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:34.003 | INFO | Uploaded 36GiB to GPU in 2.01183s (17.8941GiB/s) | 20260506 13:28:34.003 | INFO | Downloaded 34GiB to GPU in 2.03645s (16.6957GiB/s) | 20260506 13:28:39.141 | INFO | Analyzing chunk frames = [1715712,1716224) | 20260506 13:28:39.141 | INFO | MemInfo: { | 20260506 13:28:39.141 | INFO | "gpu" : " 17 GB", | 20260506 13:28:39.141 | INFO | "huge" : "", | 20260506 13:28:39.141 | INFO | "huge_shared" : " 27 GB", | 20260506 13:28:39.141 | INFO | "malloc" : " 14 GB", | 20260506 13:28:39.141 | INFO | "pinned" : " 29 GB" | 20260506 13:28:39.141 | INFO | } | 20260506 13:28:39.141 | INFO | Perf report for the last 5.13858 seconds: | 20260506 13:28:39.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:28:39.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:39.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:39.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:39.141 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:28:39.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:39.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:39.141 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:28:39.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:39.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:39.141 | INFO | Uploaded 36GiB to GPU in 2.04459s (17.6074GiB/s) | 20260506 13:28:39.141 | INFO | Downloaded 34GiB to GPU in 1.97061s (17.2536GiB/s) | 20260506 13:28:39.269 | INFO | Starting a round of Baz IO | 20260506 13:28:42.437 | INFO | Populating RTMetrics report | 20260506 13:28:42.438 | INFO | RTMetrics report populated | 20260506 13:28:42.438 | INFO | Opening tmp RTMetrics json file | 20260506 13:28:42.492 | INFO | Serializing RTMetrics to json | 20260506 13:28:42.581 | INFO | Serialization done, swapping tmp file now... | 20260506 13:28:42.616 | INFO | RTMetrics done | 20260506 13:28:43.292 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 4.023 seconds | 20260506 13:28:43.292 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.67% / 0.02% / 60.31% | 20260506 13:28:43.292 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.14935% / 1.19209e-05% / 5.5631e-05% | 20260506 13:28:43.292 | INFO | Baz IO overall: | 20260506 13:28:43.292 | INFO | BytesWritten : 872257596990 | 20260506 13:28:43.292 | INFO | NumEvents : 421891648127 | 20260506 13:28:44.280 | INFO | Analyzing chunk frames = [1716224,1716736) | 20260506 13:28:44.280 | INFO | MemInfo: { | 20260506 13:28:44.280 | INFO | "gpu" : " 17 GB", | 20260506 13:28:44.280 | INFO | "huge" : "", | 20260506 13:28:44.280 | INFO | "huge_shared" : " 27 GB", | 20260506 13:28:44.280 | INFO | "malloc" : " 14 GB", | 20260506 13:28:44.280 | INFO | "pinned" : " 29 GB" | 20260506 13:28:44.280 | INFO | } | 20260506 13:28:44.280 | INFO | Perf report for the last 5.13941 seconds: | 20260506 13:28:44.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:28:44.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:44.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:44.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:44.281 | INFO | ANALYSIS : 63.4%, 2.9, 96.2ms, 0 | 20260506 13:28:44.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:44.281 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260506 13:28:44.281 | INFO | PRE_HQ : 61.0%, 1.9, 59.3ms, 0 | 20260506 13:28:44.281 | INFO | BAZWRITER : 77.7%, 4.8, 192.8ms, 0 | 20260506 13:28:44.281 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 13:28:44.281 | INFO | Uploaded 36GiB to GPU in 2.08536s (17.2632GiB/s) | 20260506 13:28:44.281 | INFO | Downloaded 38GiB to GPU in 2.24614s (16.9179GiB/s) | 20260506 13:28:49.419 | INFO | Analyzing chunk frames = [1716736,1717248) | 20260506 13:28:49.419 | INFO | MemInfo: { | 20260506 13:28:49.419 | INFO | "gpu" : " 17 GB", | 20260506 13:28:49.419 | INFO | "huge" : "", | 20260506 13:28:49.419 | INFO | "huge_shared" : " 27 GB", | 20260506 13:28:49.419 | INFO | "malloc" : " 14 GB", | 20260506 13:28:49.419 | INFO | "pinned" : " 29 GB" | 20260506 13:28:49.419 | INFO | } | 20260506 13:28:49.419 | INFO | Perf report for the last 5.13871 seconds: | 20260506 13:28:49.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:28:49.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:49.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:49.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:49.419 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:28:49.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:49.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:49.419 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:28:49.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:49.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:49.420 | INFO | Uploaded 36GiB to GPU in 2.00249s (17.9777GiB/s) | 20260506 13:28:49.420 | INFO | Downloaded 34GiB to GPU in 1.94041s (17.5221GiB/s) | 20260506 13:28:54.559 | INFO | Analyzing chunk frames = [1717248,1717760) | 20260506 13:28:54.559 | INFO | MemInfo: { | 20260506 13:28:54.559 | INFO | "gpu" : " 17 GB", | 20260506 13:28:54.559 | INFO | "huge" : "", | 20260506 13:28:54.559 | INFO | "huge_shared" : " 27 GB", | 20260506 13:28:54.559 | INFO | "malloc" : " 14 GB", | 20260506 13:28:54.559 | INFO | "pinned" : " 29 GB" | 20260506 13:28:54.559 | INFO | } | 20260506 13:28:54.559 | INFO | Perf report for the last 5.13998 seconds: | 20260506 13:28:54.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:28:54.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:54.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:54.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:54.559 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 13:28:54.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:54.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:54.559 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:28:54.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:54.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:54.559 | INFO | Uploaded 36GiB to GPU in 2.01591s (17.8579GiB/s) | 20260506 13:28:54.559 | INFO | Downloaded 34GiB to GPU in 1.94207s (17.5071GiB/s) | 20260506 13:28:59.698 | INFO | Analyzing chunk frames = [1717760,1718272) | 20260506 13:28:59.698 | INFO | MemInfo: { | 20260506 13:28:59.698 | INFO | "gpu" : " 17 GB", | 20260506 13:28:59.698 | INFO | "huge" : "", | 20260506 13:28:59.698 | INFO | "huge_shared" : " 27 GB", | 20260506 13:28:59.698 | INFO | "malloc" : " 14 GB", | 20260506 13:28:59.698 | INFO | "pinned" : " 29 GB" | 20260506 13:28:59.699 | INFO | } | 20260506 13:28:59.699 | INFO | Perf report for the last 5.13937 seconds: | 20260506 13:28:59.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:28:59.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:59.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:59.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:59.699 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:28:59.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:59.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:59.699 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 13:28:59.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:59.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:28:59.699 | INFO | Uploaded 36GiB to GPU in 2.01195s (17.8931GiB/s) | 20260506 13:28:59.699 | INFO | Downloaded 34GiB to GPU in 1.91997s (17.7086GiB/s) | 20260506 13:29:04.837 | INFO | Analyzing chunk frames = [1718272,1718784) | 20260506 13:29:04.837 | INFO | MemInfo: { | 20260506 13:29:04.837 | INFO | "gpu" : " 17 GB", | 20260506 13:29:04.837 | INFO | "huge" : "", | 20260506 13:29:04.837 | INFO | "huge_shared" : " 27 GB", | 20260506 13:29:04.837 | INFO | "malloc" : " 14 GB", | 20260506 13:29:04.837 | INFO | "pinned" : " 29 GB" | 20260506 13:29:04.837 | INFO | } | 20260506 13:29:04.837 | INFO | Perf report for the last 5.1386 seconds: | 20260506 13:29:04.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:29:04.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:04.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:04.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:04.837 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 13:29:04.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:04.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:04.837 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:29:04.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:04.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:04.838 | INFO | Uploaded 36GiB to GPU in 2.07123s (17.381GiB/s) | 20260506 13:29:04.838 | INFO | Downloaded 34GiB to GPU in 1.95099s (17.4271GiB/s) | 20260506 13:29:09.976 | INFO | Analyzing chunk frames = [1718784,1719296) | 20260506 13:29:09.976 | INFO | MemInfo: { | 20260506 13:29:09.976 | INFO | "gpu" : " 17 GB", | 20260506 13:29:09.976 | INFO | "huge" : "", | 20260506 13:29:09.976 | INFO | "huge_shared" : " 27 GB", | 20260506 13:29:09.976 | INFO | "malloc" : " 14 GB", | 20260506 13:29:09.976 | INFO | "pinned" : " 29 GB" | 20260506 13:29:09.976 | INFO | } | 20260506 13:29:09.976 | INFO | Perf report for the last 5.13927 seconds: | 20260506 13:29:09.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:29:09.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:09.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:09.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:09.977 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 13:29:09.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:09.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:09.977 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 13:29:09.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:09.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:09.977 | INFO | Uploaded 36GiB to GPU in 2.05654s (17.5051GiB/s) | 20260506 13:29:09.977 | INFO | Downloaded 34GiB to GPU in 1.91775s (17.7291GiB/s) | 20260506 13:29:15.116 | INFO | Analyzing chunk frames = [1719296,1719808) | 20260506 13:29:15.116 | INFO | MemInfo: { | 20260506 13:29:15.117 | INFO | "gpu" : " 17 GB", | 20260506 13:29:15.117 | INFO | "huge" : "", | 20260506 13:29:15.117 | INFO | "huge_shared" : " 27 GB", | 20260506 13:29:15.117 | INFO | "malloc" : " 14 GB", | 20260506 13:29:15.117 | INFO | "pinned" : " 29 GB" | 20260506 13:29:15.117 | INFO | } | 20260506 13:29:15.117 | INFO | Perf report for the last 5.1401 seconds: | 20260506 13:29:15.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:29:15.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:15.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:15.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:15.117 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:29:15.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:15.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:15.117 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 13:29:15.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:15.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:15.117 | INFO | Uploaded 36GiB to GPU in 2.03218s (17.715GiB/s) | 20260506 13:29:15.117 | INFO | Downloaded 34GiB to GPU in 2.01638s (16.8619GiB/s) | 20260506 13:29:20.255 | INFO | Analyzing chunk frames = [1719808,1720320) | 20260506 13:29:20.255 | INFO | MemInfo: { | 20260506 13:29:20.255 | INFO | "gpu" : " 17 GB", | 20260506 13:29:20.255 | INFO | "huge" : "", | 20260506 13:29:20.255 | INFO | "huge_shared" : " 27 GB", | 20260506 13:29:20.255 | INFO | "malloc" : " 14 GB", | 20260506 13:29:20.255 | INFO | "pinned" : " 29 GB" | 20260506 13:29:20.255 | INFO | } | 20260506 13:29:20.255 | INFO | Perf report for the last 5.13868 seconds: | 20260506 13:29:20.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:29:20.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:20.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:20.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:20.256 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 13:29:20.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:20.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:20.256 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 13:29:20.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:20.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:20.256 | INFO | Uploaded 36GiB to GPU in 2.05676s (17.5033GiB/s) | 20260506 13:29:20.256 | INFO | Downloaded 34GiB to GPU in 1.96755s (17.2804GiB/s) | 20260506 13:29:20.256 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:29:20.256 | INFO | Total mem usage: 17.4 GiB | 20260506 13:29:20.256 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:29:20.256 | INFO | Recent High Water Marks Sum: 16.6 GiB (Savings: -4.64%) | 20260506 13:29:20.256 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:29:20.256 | INFO | High Water Breakdown: | 20260506 13:29:20.256 | INFO | 62% : 10.3 GiB ( 94.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:29:20.256 | INFO | 37.9% : 6.3 GiB ( 94.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:29:20.256 | INFO | 0.12% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:29:20.256 | INFO | ------------------------------------------ | 20260506 13:29:20.256 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:29:20.256 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:29:20.256 | INFO | Total mem usage: 26.5 GiB | 20260506 13:29:20.256 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:29:20.256 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:29:20.256 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:29:20.256 | INFO | High Water Breakdown: | 20260506 13:29:20.256 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:29:20.256 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:29:20.256 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:29:20.256 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:29:20.256 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:29:20.256 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:29:20.256 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:29:20.256 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:29:20.256 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:29:20.256 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:29:20.256 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:29:20.257 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:29:20.257 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:29:20.257 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:29:20.257 | INFO | ------------------------------------------ | 20260506 13:29:20.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:29:20.257 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:29:20.257 | INFO | Total mem usage: 25.6 GiB | 20260506 13:29:20.257 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:29:20.257 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:29:20.257 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:29:20.257 | INFO | High Water Breakdown: | 20260506 13:29:20.257 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:29:20.257 | INFO | ------------------------------------------ | 20260506 13:29:20.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:29:20.257 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:29:20.257 | INFO | Total mem usage: 15.7 GiB | 20260506 13:29:20.257 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:29:20.257 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:29:20.257 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 13:29:20.257 | INFO | High Water Breakdown: | 20260506 13:29:20.257 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:29:20.257 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:29:20.257 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:29:20.257 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:29:20.257 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:29:20.257 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:29:20.257 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:29:20.257 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:29:20.257 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:29:20.257 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:29:20.257 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:29:20.257 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:29:20.257 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:29:20.257 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:29:20.257 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:29:20.257 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:29:20.257 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:29:20.257 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:29:20.257 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:29:20.257 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:29:20.257 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:29:20.257 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:29:20.257 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:29:20.257 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:29:20.257 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:29:20.257 | INFO | ------------------------------------------ | 20260506 13:29:20.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:29:20.257 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:29:20.257 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:29:20.257 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:29:20.257 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.62e+01, 1.36e+01 / 2.94e+00 | 20260506 13:29:20.257 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.02e-01, 1.03e-02 / 6.73e-03 | 20260506 13:29:20.257 | INFO | PerfReport: 11.60%, Baseline -- 3.67e+00 / 4.40e+00, 5.89e-01 / 3.85e-01 | 20260506 13:29:20.257 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.48e+00, 1.92e-01 / 1.35e-01 | 20260506 13:29:20.257 | INFO | PerfReport: 15.86%, DME -- 4.02e+01 / 4.46e+01, 4.13e+00 / 3.11e+00 | 20260506 13:29:20.257 | INFO | PerfReport: 56.50%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.38e+00 / 1.54e+00 | 20260506 13:29:20.257 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.38e+00 / 2.99e+00, 4.18e-01 / 2.52e-01 | 20260506 13:29:20.257 | INFO | PerfReport: 3.68%, Metrics -- 1.16e+00 / 1.56e+00, 3.01e-01 / 1.69e-01 | 20260506 13:29:20.257 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.08e-01 / 4.60e-01, 6.38e-02 / 2.24e-02 | 20260506 13:29:20.257 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:29:20.257 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:29:20.257 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:29:20.257 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 13:29:20.257 | INFO | PerfReport: 21.59%, Upload -- 2.12e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260506 13:29:20.257 | INFO | PerfReport: 21.10%, Download -- 2.07e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 13:29:20.257 | INFO | PerfReport: 32.22%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 13:29:20.257 | INFO | PerfReport: 1.78%, LockUpload -- 1.75e+00 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 13:29:20.258 | INFO | PerfReport: 0.98%, LockDownload -- 9.58e-01 / 0.00e+00, 2.56e+00 / 0.00e+00 | 20260506 13:29:20.258 | INFO | PerfReport: 22.32%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 13:29:20.258 | INFO | PerfReport: 0.02%, Misc -- 1.55e-02 / 0.00e+00, 2.78e-02 / 0.00e+00 | 20260506 13:29:20.387 | INFO | Starting a round of Baz IO | 20260506 13:29:23.551 | INFO | Populating RTMetrics report | 20260506 13:29:23.551 | INFO | RTMetrics report populated | 20260506 13:29:23.551 | INFO | Opening tmp RTMetrics json file | 20260506 13:29:23.599 | INFO | Serializing RTMetrics to json | 20260506 13:29:23.601 | INFO | Serialization done, swapping tmp file now... | 20260506 13:29:23.607 | INFO | RTMetrics done | 20260506 13:29:24.009 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.622 seconds | 20260506 13:29:24.009 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.65% / 0.02% / 60.33% | 20260506 13:29:24.009 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.13953% / 1.19209e-05% / 5.16574e-05% | 20260506 13:29:24.009 | INFO | Baz IO overall: | 20260506 13:29:24.009 | INFO | BytesWritten : 874521757855 | 20260506 13:29:24.009 | INFO | NumEvents : 422989556834 | 20260506 13:29:25.395 | INFO | Analyzing chunk frames = [1720320,1720832) | 20260506 13:29:25.395 | INFO | MemInfo: { | 20260506 13:29:25.395 | INFO | "gpu" : " 17 GB", | 20260506 13:29:25.395 | INFO | "huge" : "", | 20260506 13:29:25.395 | INFO | "huge_shared" : " 27 GB", | 20260506 13:29:25.395 | INFO | "malloc" : " 14 GB", | 20260506 13:29:25.395 | INFO | "pinned" : " 29 GB" | 20260506 13:29:25.395 | INFO | } | 20260506 13:29:25.395 | INFO | Perf report for the last 5.13942 seconds: | 20260506 13:29:25.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:29:25.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:25.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:25.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:25.395 | INFO | ANALYSIS : 63.5%, 2.9, 96.3ms, 0 | 20260506 13:29:25.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:25.395 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260506 13:29:25.395 | INFO | PRE_HQ : 61.4%, 1.9, 59.9ms, 0 | 20260506 13:29:25.395 | INFO | BAZWRITER : 69.9%, 4.8, 171.6ms, 0 | 20260506 13:29:25.395 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 13:29:25.395 | INFO | Uploaded 36GiB to GPU in 2.08467s (17.2689GiB/s) | 20260506 13:29:25.395 | INFO | Downloaded 38GiB to GPU in 2.24485s (16.9276GiB/s) | 20260506 13:29:30.534 | INFO | Analyzing chunk frames = [1720832,1721344) | 20260506 13:29:30.534 | INFO | MemInfo: { | 20260506 13:29:30.534 | INFO | "gpu" : " 17 GB", | 20260506 13:29:30.534 | INFO | "huge" : "", | 20260506 13:29:30.534 | INFO | "huge_shared" : " 27 GB", | 20260506 13:29:30.534 | INFO | "malloc" : " 14 GB", | 20260506 13:29:30.534 | INFO | "pinned" : " 29 GB" | 20260506 13:29:30.534 | INFO | } | 20260506 13:29:30.534 | INFO | Perf report for the last 5.13948 seconds: | 20260506 13:29:30.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:29:30.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:30.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:30.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:30.534 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:29:30.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:30.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:30.534 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:29:30.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:30.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:30.535 | INFO | Uploaded 36GiB to GPU in 2.04233s (17.6269GiB/s) | 20260506 13:29:30.535 | INFO | Downloaded 34GiB to GPU in 1.94788s (17.4548GiB/s) | 20260506 13:29:35.673 | INFO | Analyzing chunk frames = [1721344,1721856) | 20260506 13:29:35.673 | INFO | MemInfo: { | 20260506 13:29:35.673 | INFO | "gpu" : " 17 GB", | 20260506 13:29:35.673 | INFO | "huge" : "", | 20260506 13:29:35.673 | INFO | "huge_shared" : " 27 GB", | 20260506 13:29:35.673 | INFO | "malloc" : " 14 GB", | 20260506 13:29:35.673 | INFO | "pinned" : " 29 GB" | 20260506 13:29:35.673 | INFO | } | 20260506 13:29:35.673 | INFO | Perf report for the last 5.1392 seconds: | 20260506 13:29:35.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:29:35.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:35.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:35.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:35.674 | INFO | ANALYSIS : 61.7%, 3.0, 95.5ms, 0 | 20260506 13:29:35.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:35.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:35.674 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:29:35.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:35.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:35.674 | INFO | Uploaded 36GiB to GPU in 2.03321s (17.706GiB/s) | 20260506 13:29:35.674 | INFO | Downloaded 34GiB to GPU in 1.93801s (17.5438GiB/s) | 20260506 13:29:40.813 | INFO | Analyzing chunk frames = [1721856,1722368) | 20260506 13:29:40.813 | INFO | MemInfo: { | 20260506 13:29:40.813 | INFO | "gpu" : " 17 GB", | 20260506 13:29:40.813 | INFO | "huge" : "", | 20260506 13:29:40.813 | INFO | "huge_shared" : " 27 GB", | 20260506 13:29:40.813 | INFO | "malloc" : " 14 GB", | 20260506 13:29:40.813 | INFO | "pinned" : " 29 GB" | 20260506 13:29:40.813 | INFO | } | 20260506 13:29:40.813 | INFO | Perf report for the last 5.13961 seconds: | 20260506 13:29:40.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:29:40.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:40.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:40.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:40.813 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260506 13:29:40.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:40.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:40.813 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:29:40.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:40.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:40.813 | INFO | Uploaded 36GiB to GPU in 2.01046s (17.9063GiB/s) | 20260506 13:29:40.813 | INFO | Downloaded 34GiB to GPU in 1.9401s (17.5249GiB/s) | 20260506 13:29:45.952 | INFO | Analyzing chunk frames = [1722368,1722880) | 20260506 13:29:45.952 | INFO | MemInfo: { | 20260506 13:29:45.952 | INFO | "gpu" : " 17 GB", | 20260506 13:29:45.952 | INFO | "huge" : "", | 20260506 13:29:45.952 | INFO | "huge_shared" : " 27 GB", | 20260506 13:29:45.952 | INFO | "malloc" : " 14 GB", | 20260506 13:29:45.952 | INFO | "pinned" : " 29 GB" | 20260506 13:29:45.952 | INFO | } | 20260506 13:29:45.953 | INFO | Perf report for the last 5.13945 seconds: | 20260506 13:29:45.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:29:45.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:45.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:45.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:45.953 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 13:29:45.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:45.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:45.953 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:29:45.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:45.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:45.953 | INFO | Uploaded 36GiB to GPU in 2.0486s (17.5729GiB/s) | 20260506 13:29:45.953 | INFO | Downloaded 34GiB to GPU in 1.95107s (17.4263GiB/s) | 20260506 13:29:51.091 | INFO | Analyzing chunk frames = [1722880,1723392) | 20260506 13:29:51.091 | INFO | MemInfo: { | 20260506 13:29:51.091 | INFO | "gpu" : " 17 GB", | 20260506 13:29:51.091 | INFO | "huge" : "", | 20260506 13:29:51.091 | INFO | "huge_shared" : " 27 GB", | 20260506 13:29:51.091 | INFO | "malloc" : " 14 GB", | 20260506 13:29:51.091 | INFO | "pinned" : " 29 GB" | 20260506 13:29:51.091 | INFO | } | 20260506 13:29:51.092 | INFO | Perf report for the last 5.13906 seconds: | 20260506 13:29:51.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:29:51.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:51.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:51.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:51.092 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 13:29:51.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:51.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:51.092 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260506 13:29:51.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:51.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:51.092 | INFO | Uploaded 36GiB to GPU in 2.08443s (17.2709GiB/s) | 20260506 13:29:51.092 | INFO | Downloaded 34GiB to GPU in 1.92278s (17.6827GiB/s) | 20260506 13:29:56.231 | INFO | Analyzing chunk frames = [1723392,1723904) | 20260506 13:29:56.231 | INFO | MemInfo: { | 20260506 13:29:56.231 | INFO | "gpu" : " 17 GB", | 20260506 13:29:56.231 | INFO | "huge" : "", | 20260506 13:29:56.231 | INFO | "huge_shared" : " 27 GB", | 20260506 13:29:56.231 | INFO | "malloc" : " 14 GB", | 20260506 13:29:56.231 | INFO | "pinned" : " 29 GB" | 20260506 13:29:56.231 | INFO | } | 20260506 13:29:56.231 | INFO | Perf report for the last 5.1399 seconds: | 20260506 13:29:56.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:29:56.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:56.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:56.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:56.232 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:29:56.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:56.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:56.232 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:29:56.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:56.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:29:56.232 | INFO | Uploaded 36GiB to GPU in 2.02296s (17.7957GiB/s) | 20260506 13:29:56.232 | INFO | Downloaded 34GiB to GPU in 2.02426s (16.7963GiB/s) | 20260506 13:30:01.370 | INFO | Analyzing chunk frames = [1723904,1724416) | 20260506 13:30:01.370 | INFO | MemInfo: { | 20260506 13:30:01.370 | INFO | "gpu" : " 17 GB", | 20260506 13:30:01.370 | INFO | "huge" : "", | 20260506 13:30:01.370 | INFO | "huge_shared" : " 27 GB", | 20260506 13:30:01.371 | INFO | "malloc" : " 14 GB", | 20260506 13:30:01.371 | INFO | "pinned" : " 29 GB" | 20260506 13:30:01.371 | INFO | } | 20260506 13:30:01.371 | INFO | Perf report for the last 5.13913 seconds: | 20260506 13:30:01.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:30:01.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:01.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:01.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:01.371 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 13:30:01.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:01.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:01.371 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 13:30:01.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:01.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:01.371 | INFO | Uploaded 36GiB to GPU in 2.06855s (17.4035GiB/s) | 20260506 13:30:01.371 | INFO | Downloaded 34GiB to GPU in 1.96561s (17.2975GiB/s) | 20260506 13:30:01.507 | INFO | Starting a round of Baz IO | 20260506 13:30:04.703 | INFO | Populating RTMetrics report | 20260506 13:30:04.703 | INFO | RTMetrics report populated | 20260506 13:30:04.703 | INFO | Opening tmp RTMetrics json file | 20260506 13:30:04.732 | INFO | Serializing RTMetrics to json | 20260506 13:30:04.735 | INFO | Serialization done, swapping tmp file now... | 20260506 13:30:04.739 | INFO | RTMetrics done | 20260506 13:30:05.278 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.77 seconds | 20260506 13:30:05.278 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.63% / 0.02% / 60.35% | 20260506 13:30:05.278 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.14115% / 2.38419e-05% / 3.57628e-05% | 20260506 13:30:05.278 | INFO | Baz IO overall: | 20260506 13:30:05.278 | INFO | BytesWritten : 876788124237 | 20260506 13:30:05.278 | INFO | NumEvents : 424089119576 | 20260506 13:30:06.510 | INFO | Analyzing chunk frames = [1724416,1724928) | 20260506 13:30:06.510 | INFO | MemInfo: { | 20260506 13:30:06.510 | INFO | "gpu" : " 17 GB", | 20260506 13:30:06.510 | INFO | "huge" : "", | 20260506 13:30:06.510 | INFO | "huge_shared" : " 27 GB", | 20260506 13:30:06.510 | INFO | "malloc" : " 14 GB", | 20260506 13:30:06.510 | INFO | "pinned" : " 29 GB" | 20260506 13:30:06.510 | INFO | } | 20260506 13:30:06.510 | INFO | Perf report for the last 5.13915 seconds: | 20260506 13:30:06.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:30:06.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:06.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:06.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:06.510 | INFO | ANALYSIS : 63.5%, 2.9, 95.8ms, 0 | 20260506 13:30:06.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:06.510 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260506 13:30:06.510 | INFO | PRE_HQ : 61.7%, 1.9, 61.7ms, 0 | 20260506 13:30:06.510 | INFO | BAZWRITER : 72.7%, 4.8, 179.5ms, 0 | 20260506 13:30:06.510 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 13:30:06.510 | INFO | Uploaded 36GiB to GPU in 2.09147s (17.2128GiB/s) | 20260506 13:30:06.510 | INFO | Downloaded 38GiB to GPU in 2.23906s (16.9714GiB/s) | 20260506 13:30:11.649 | INFO | Analyzing chunk frames = [1724928,1725440) | 20260506 13:30:11.649 | INFO | MemInfo: { | 20260506 13:30:11.649 | INFO | "gpu" : " 17 GB", | 20260506 13:30:11.649 | INFO | "huge" : "", | 20260506 13:30:11.649 | INFO | "huge_shared" : " 27 GB", | 20260506 13:30:11.649 | INFO | "malloc" : " 14 GB", | 20260506 13:30:11.649 | INFO | "pinned" : " 29 GB" | 20260506 13:30:11.649 | INFO | } | 20260506 13:30:11.649 | INFO | Perf report for the last 5.13929 seconds: | 20260506 13:30:11.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:30:11.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:11.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:11.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:11.649 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:30:11.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:11.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:11.649 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 13:30:11.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:11.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:11.649 | INFO | Uploaded 36GiB to GPU in 2.05632s (17.507GiB/s) | 20260506 13:30:11.650 | INFO | Downloaded 34GiB to GPU in 1.93161s (17.6019GiB/s) | 20260506 13:30:16.788 | INFO | Analyzing chunk frames = [1725440,1725952) | 20260506 13:30:16.788 | INFO | MemInfo: { | 20260506 13:30:16.788 | INFO | "gpu" : " 17 GB", | 20260506 13:30:16.788 | INFO | "huge" : "", | 20260506 13:30:16.788 | INFO | "huge_shared" : " 27 GB", | 20260506 13:30:16.788 | INFO | "malloc" : " 14 GB", | 20260506 13:30:16.788 | INFO | "pinned" : " 29 GB" | 20260506 13:30:16.788 | INFO | } | 20260506 13:30:16.788 | INFO | Perf report for the last 5.13924 seconds: | 20260506 13:30:16.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:30:16.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:16.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:16.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:16.789 | INFO | ANALYSIS : 61.7%, 3.0, 95.5ms, 0 | 20260506 13:30:16.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:16.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:16.789 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 13:30:16.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:16.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:16.789 | INFO | Uploaded 36GiB to GPU in 2.0324s (17.7131GiB/s) | 20260506 13:30:16.789 | INFO | Downloaded 34GiB to GPU in 1.93856s (17.5388GiB/s) | 20260506 13:30:21.927 | INFO | Analyzing chunk frames = [1725952,1726464) | 20260506 13:30:21.928 | INFO | MemInfo: { | 20260506 13:30:21.928 | INFO | "gpu" : " 17 GB", | 20260506 13:30:21.928 | INFO | "huge" : "", | 20260506 13:30:21.928 | INFO | "huge_shared" : " 27 GB", | 20260506 13:30:21.928 | INFO | "malloc" : " 14 GB", | 20260506 13:30:21.928 | INFO | "pinned" : " 29 GB" | 20260506 13:30:21.928 | INFO | } | 20260506 13:30:21.928 | INFO | Perf report for the last 5.13945 seconds: | 20260506 13:30:21.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:30:21.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:21.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:21.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:21.928 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:30:21.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:21.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:21.928 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 13:30:21.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:21.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:21.928 | INFO | Uploaded 36GiB to GPU in 2.01013s (17.9093GiB/s) | 20260506 13:30:21.928 | INFO | Downloaded 34GiB to GPU in 1.93744s (17.5489GiB/s) | 20260506 13:30:27.067 | INFO | Analyzing chunk frames = [1726464,1726976) | 20260506 13:30:27.067 | INFO | MemInfo: { | 20260506 13:30:27.067 | INFO | "gpu" : " 17 GB", | 20260506 13:30:27.067 | INFO | "huge" : "", | 20260506 13:30:27.067 | INFO | "huge_shared" : " 27 GB", | 20260506 13:30:27.067 | INFO | "malloc" : " 14 GB", | 20260506 13:30:27.067 | INFO | "pinned" : " 29 GB" | 20260506 13:30:27.067 | INFO | } | 20260506 13:30:27.067 | INFO | Perf report for the last 5.13943 seconds: | 20260506 13:30:27.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:30:27.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:27.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:27.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:27.067 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 13:30:27.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:27.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:27.067 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 13:30:27.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:27.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:27.067 | INFO | Uploaded 36GiB to GPU in 2.00794s (17.9288GiB/s) | 20260506 13:30:27.068 | INFO | Downloaded 34GiB to GPU in 1.93779s (17.5457GiB/s) | 20260506 13:30:32.206 | INFO | Analyzing chunk frames = [1726976,1727488) | 20260506 13:30:32.206 | INFO | MemInfo: { | 20260506 13:30:32.206 | INFO | "gpu" : " 17 GB", | 20260506 13:30:32.206 | INFO | "huge" : "", | 20260506 13:30:32.206 | INFO | "huge_shared" : " 27 GB", | 20260506 13:30:32.206 | INFO | "malloc" : " 14 GB", | 20260506 13:30:32.206 | INFO | "pinned" : " 29 GB" | 20260506 13:30:32.206 | INFO | } | 20260506 13:30:32.206 | INFO | Perf report for the last 5.13913 seconds: | 20260506 13:30:32.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:30:32.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:32.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:32.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:32.206 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 13:30:32.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:32.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:32.206 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:30:32.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:32.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:32.207 | INFO | Uploaded 36GiB to GPU in 2.09277s (17.2021GiB/s) | 20260506 13:30:32.207 | INFO | Downloaded 34GiB to GPU in 1.96723s (17.2832GiB/s) | 20260506 13:30:37.346 | INFO | Analyzing chunk frames = [1727488,1728000) | 20260506 13:30:37.346 | INFO | MemInfo: { | 20260506 13:30:37.346 | INFO | "gpu" : " 17 GB", | 20260506 13:30:37.346 | INFO | "huge" : "", | 20260506 13:30:37.346 | INFO | "huge_shared" : " 27 GB", | 20260506 13:30:37.346 | INFO | "malloc" : " 14 GB", | 20260506 13:30:37.346 | INFO | "pinned" : " 29 GB" | 20260506 13:30:37.346 | INFO | } | 20260506 13:30:37.346 | INFO | Perf report for the last 5.13976 seconds: | 20260506 13:30:37.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:30:37.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:37.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:37.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:37.346 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:30:37.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:37.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:37.346 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 13:30:37.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:37.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:37.347 | INFO | Uploaded 36GiB to GPU in 1.98698s (18.118GiB/s) | 20260506 13:30:37.347 | INFO | Downloaded 34GiB to GPU in 2.03345s (16.7204GiB/s) | 20260506 13:30:42.485 | INFO | Analyzing chunk frames = [1728000,1728512) | 20260506 13:30:42.485 | INFO | MemInfo: { | 20260506 13:30:42.485 | INFO | "gpu" : " 17 GB", | 20260506 13:30:42.485 | INFO | "huge" : "", | 20260506 13:30:42.485 | INFO | "huge_shared" : " 27 GB", | 20260506 13:30:42.485 | INFO | "malloc" : " 14 GB", | 20260506 13:30:42.485 | INFO | "pinned" : " 29 GB" | 20260506 13:30:42.485 | INFO | } | 20260506 13:30:42.485 | INFO | Perf report for the last 5.13894 seconds: | 20260506 13:30:42.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:30:42.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:42.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:42.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:42.485 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 13:30:42.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:42.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:42.485 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 13:30:42.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:42.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:42.486 | INFO | Uploaded 36GiB to GPU in 2.07854s (17.3198GiB/s) | 20260506 13:30:42.486 | INFO | Downloaded 34GiB to GPU in 1.99905s (17.0081GiB/s) | 20260506 13:30:42.486 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:30:42.486 | INFO | Total mem usage: 17.4 GiB | 20260506 13:30:42.486 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:30:42.486 | INFO | Recent High Water Marks Sum: 16.6 GiB (Savings: -4.63%) | 20260506 13:30:42.486 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:30:42.486 | INFO | High Water Breakdown: | 20260506 13:30:42.486 | INFO | 62% : 10.3 GiB ( 94.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:30:42.486 | INFO | 37.9% : 6.3 GiB ( 94.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:30:42.486 | INFO | 0.12% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:30:42.486 | INFO | ------------------------------------------ | 20260506 13:30:42.486 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:30:42.486 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:30:42.486 | INFO | Total mem usage: 26.5 GiB | 20260506 13:30:42.486 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:30:42.486 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:30:42.486 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:30:42.486 | INFO | High Water Breakdown: | 20260506 13:30:42.486 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:30:42.486 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:30:42.486 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:30:42.486 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:30:42.486 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:30:42.486 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:30:42.486 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:30:42.486 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:30:42.486 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:30:42.486 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:30:42.486 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:30:42.486 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:30:42.486 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:30:42.486 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:30:42.486 | INFO | ------------------------------------------ | 20260506 13:30:42.486 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:30:42.486 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:30:42.486 | INFO | Total mem usage: 25.6 GiB | 20260506 13:30:42.487 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:30:42.487 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:30:42.487 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:30:42.487 | INFO | High Water Breakdown: | 20260506 13:30:42.487 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:30:42.487 | INFO | ------------------------------------------ | 20260506 13:30:42.487 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:30:42.487 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:30:42.487 | INFO | Total mem usage: 15.7 GiB | 20260506 13:30:42.487 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:30:42.487 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:30:42.487 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 13:30:42.487 | INFO | High Water Breakdown: | 20260506 13:30:42.487 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:30:42.487 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:30:42.487 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:30:42.487 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:30:42.487 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:30:42.487 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:30:42.487 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:30:42.487 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:30:42.487 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:30:42.487 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:30:42.487 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:30:42.487 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:30:42.487 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:30:42.487 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:30:42.487 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:30:42.487 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:30:42.487 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:30:42.487 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:30:42.487 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:30:42.487 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:30:42.487 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:30:42.487 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:30:42.487 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:30:42.487 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:30:42.487 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:30:42.487 | INFO | ------------------------------------------ | 20260506 13:30:42.487 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:30:42.487 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:30:42.487 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:30:42.487 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:30:42.487 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.64e+01, 1.36e+01 / 2.88e+00 | 20260506 13:30:42.487 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.02e-01, 1.01e-02 / 4.57e-03 | 20260506 13:30:42.487 | INFO | PerfReport: 11.56%, Baseline -- 3.66e+00 / 4.46e+00, 5.69e-01 / 2.90e-01 | 20260506 13:30:42.487 | INFO | PerfReport: 3.23%, Histogram -- 1.02e+00 / 1.48e+00, 1.87e-01 / 1.24e-01 | 20260506 13:30:42.487 | INFO | PerfReport: 15.87%, DME -- 4.01e+01 / 4.47e+01, 4.23e+00 / 3.16e+00 | 20260506 13:30:42.487 | INFO | PerfReport: 56.53%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.38e+00 / 1.65e+00 | 20260506 13:30:42.487 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.39e+00 / 3.02e+00, 4.21e-01 / 2.57e-01 | 20260506 13:30:42.487 | INFO | PerfReport: 3.68%, Metrics -- 1.16e+00 / 1.56e+00, 2.98e-01 / 1.48e-01 | 20260506 13:30:42.487 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.09e-01 / 4.66e-01, 6.38e-02 / 2.98e-02 | 20260506 13:30:42.487 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:30:42.487 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:30:42.487 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:30:42.487 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 13:30:42.487 | INFO | PerfReport: 21.68%, Upload -- 2.13e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260506 13:30:42.487 | INFO | PerfReport: 21.13%, Download -- 2.08e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 13:30:42.487 | INFO | PerfReport: 32.17%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 13:30:42.487 | INFO | PerfReport: 1.83%, LockUpload -- 1.80e+00 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260506 13:30:42.487 | INFO | PerfReport: 1.01%, LockDownload -- 9.89e-01 / 0.00e+00, 2.60e+00 / 0.00e+00 | 20260506 13:30:42.487 | INFO | PerfReport: 22.16%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 13:30:42.487 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260506 13:30:42.616 | INFO | Starting a round of Baz IO | 20260506 13:30:45.777 | INFO | Populating RTMetrics report | 20260506 13:30:45.777 | INFO | RTMetrics report populated | 20260506 13:30:45.777 | INFO | Opening tmp RTMetrics json file | 20260506 13:30:45.899 | INFO | Serializing RTMetrics to json | 20260506 13:30:46.016 | INFO | Serialization done, swapping tmp file now... | 20260506 13:30:46.040 | INFO | RTMetrics done | 20260506 13:30:46.080 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.463 seconds | 20260506 13:30:46.080 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.61% / 0.02% / 60.37% | 20260506 13:30:46.080 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.13019% / 4.37101e-05% / 8.34465e-05% | 20260506 13:30:46.080 | INFO | Baz IO overall: | 20260506 13:30:46.080 | INFO | BytesWritten : 879053671929 | 20260506 13:30:46.080 | INFO | NumEvents : 425188237400 | 20260506 13:30:47.624 | INFO | Analyzing chunk frames = [1728512,1729024) | 20260506 13:30:47.624 | INFO | MemInfo: { | 20260506 13:30:47.624 | INFO | "gpu" : " 17 GB", | 20260506 13:30:47.624 | INFO | "huge" : "", | 20260506 13:30:47.624 | INFO | "huge_shared" : " 27 GB", | 20260506 13:30:47.625 | INFO | "malloc" : " 14 GB", | 20260506 13:30:47.625 | INFO | "pinned" : " 29 GB" | 20260506 13:30:47.625 | INFO | } | 20260506 13:30:47.625 | INFO | Perf report for the last 5.13943 seconds: | 20260506 13:30:47.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:30:47.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:47.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:47.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:47.625 | INFO | ANALYSIS : 63.4%, 2.9, 95.6ms, 0 | 20260506 13:30:47.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:47.625 | INFO | RT_METRICS : 49.3%, 1.0, 25.8ms, 0 | 20260506 13:30:47.625 | INFO | PRE_HQ : 61.4%, 1.9, 60.5ms, 0 | 20260506 13:30:47.625 | INFO | BAZWRITER : 66.7%, 4.8, 165.2ms, 0 | 20260506 13:30:47.625 | INFO | BAZ_SAMPLING : 4.0%, 1.0, 2.1ms, 0 | 20260506 13:30:47.625 | INFO | Uploaded 36GiB to GPU in 2.0627s (17.4528GiB/s) | 20260506 13:30:47.625 | INFO | Downloaded 38GiB to GPU in 2.23912s (16.9709GiB/s) | 20260506 13:30:52.764 | INFO | Analyzing chunk frames = [1729024,1729536) | 20260506 13:30:52.764 | INFO | MemInfo: { | 20260506 13:30:52.764 | INFO | "gpu" : " 17 GB", | 20260506 13:30:52.764 | INFO | "huge" : "", | 20260506 13:30:52.764 | INFO | "huge_shared" : " 27 GB", | 20260506 13:30:52.764 | INFO | "malloc" : " 14 GB", | 20260506 13:30:52.764 | INFO | "pinned" : " 29 GB" | 20260506 13:30:52.764 | INFO | } | 20260506 13:30:52.764 | INFO | Perf report for the last 5.13936 seconds: | 20260506 13:30:52.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:30:52.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:52.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:52.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:52.764 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:30:52.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:52.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:52.764 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:30:52.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:52.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:52.764 | INFO | Uploaded 36GiB to GPU in 2.02284s (17.7968GiB/s) | 20260506 13:30:52.764 | INFO | Downloaded 34GiB to GPU in 1.94054s (17.5209GiB/s) | 20260506 13:30:57.903 | INFO | Analyzing chunk frames = [1729536,1730048) | 20260506 13:30:57.903 | INFO | MemInfo: { | 20260506 13:30:57.903 | INFO | "gpu" : " 17 GB", | 20260506 13:30:57.903 | INFO | "huge" : "", | 20260506 13:30:57.903 | INFO | "huge_shared" : " 27 GB", | 20260506 13:30:57.903 | INFO | "malloc" : " 14 GB", | 20260506 13:30:57.903 | INFO | "pinned" : " 29 GB" | 20260506 13:30:57.903 | INFO | } | 20260506 13:30:57.903 | INFO | Perf report for the last 5.13906 seconds: | 20260506 13:30:57.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:30:57.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:57.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:57.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:57.903 | INFO | ANALYSIS : 61.5%, 3.0, 95.3ms, 0 | 20260506 13:30:57.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:57.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:57.903 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:30:57.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:57.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:30:57.903 | INFO | Uploaded 36GiB to GPU in 2.01197s (17.8929GiB/s) | 20260506 13:30:57.903 | INFO | Downloaded 34GiB to GPU in 1.94117s (17.5152GiB/s) | 20260506 13:31:03.042 | INFO | Analyzing chunk frames = [1730048,1730560) | 20260506 13:31:03.042 | INFO | MemInfo: { | 20260506 13:31:03.042 | INFO | "gpu" : " 17 GB", | 20260506 13:31:03.042 | INFO | "huge" : "", | 20260506 13:31:03.042 | INFO | "huge_shared" : " 27 GB", | 20260506 13:31:03.042 | INFO | "malloc" : " 14 GB", | 20260506 13:31:03.042 | INFO | "pinned" : " 29 GB" | 20260506 13:31:03.042 | INFO | } | 20260506 13:31:03.042 | INFO | Perf report for the last 5.1394 seconds: | 20260506 13:31:03.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:31:03.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:03.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:03.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:03.042 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:31:03.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:03.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:03.042 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:31:03.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:03.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:03.043 | INFO | Uploaded 36GiB to GPU in 1.98727s (18.1153GiB/s) | 20260506 13:31:03.043 | INFO | Downloaded 34GiB to GPU in 1.92187s (17.6911GiB/s) | 20260506 13:31:08.182 | INFO | Analyzing chunk frames = [1730560,1731072) | 20260506 13:31:08.182 | INFO | MemInfo: { | 20260506 13:31:08.182 | INFO | "gpu" : " 17 GB", | 20260506 13:31:08.182 | INFO | "huge" : "", | 20260506 13:31:08.182 | INFO | "huge_shared" : " 27 GB", | 20260506 13:31:08.182 | INFO | "malloc" : " 14 GB", | 20260506 13:31:08.182 | INFO | "pinned" : " 29 GB" | 20260506 13:31:08.182 | INFO | } | 20260506 13:31:08.182 | INFO | Perf report for the last 5.13963 seconds: | 20260506 13:31:08.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:31:08.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:08.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:08.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:08.182 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 13:31:08.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:08.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:08.182 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 13:31:08.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:08.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:08.183 | INFO | Uploaded 36GiB to GPU in 2.00692s (17.9379GiB/s) | 20260506 13:31:08.183 | INFO | Downloaded 34GiB to GPU in 1.93898s (17.5349GiB/s) | 20260506 13:31:13.321 | INFO | Analyzing chunk frames = [1731072,1731584) | 20260506 13:31:13.321 | INFO | MemInfo: { | 20260506 13:31:13.321 | INFO | "gpu" : " 17 GB", | 20260506 13:31:13.321 | INFO | "huge" : "", | 20260506 13:31:13.321 | INFO | "huge_shared" : " 27 GB", | 20260506 13:31:13.321 | INFO | "malloc" : " 14 GB", | 20260506 13:31:13.321 | INFO | "pinned" : " 29 GB" | 20260506 13:31:13.321 | INFO | } | 20260506 13:31:13.321 | INFO | Perf report for the last 5.13914 seconds: | 20260506 13:31:13.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:31:13.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:13.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:13.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:13.321 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:31:13.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:13.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:13.321 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 13:31:13.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:13.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:13.322 | INFO | Uploaded 36GiB to GPU in 2.09304s (17.1999GiB/s) | 20260506 13:31:13.322 | INFO | Downloaded 34GiB to GPU in 1.96279s (17.3223GiB/s) | 20260506 13:31:18.461 | INFO | Analyzing chunk frames = [1731584,1732096) | 20260506 13:31:18.461 | INFO | MemInfo: { | 20260506 13:31:18.461 | INFO | "gpu" : " 17 GB", | 20260506 13:31:18.461 | INFO | "huge" : "", | 20260506 13:31:18.461 | INFO | "huge_shared" : " 27 GB", | 20260506 13:31:18.461 | INFO | "malloc" : " 14 GB", | 20260506 13:31:18.461 | INFO | "pinned" : " 29 GB" | 20260506 13:31:18.461 | INFO | } | 20260506 13:31:18.461 | INFO | Perf report for the last 5.13957 seconds: | 20260506 13:31:18.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:31:18.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:18.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:18.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:18.461 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 13:31:18.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:18.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:18.461 | INFO | PRE_HQ : 29.5%, 1.0, 15.4ms, 0 | 20260506 13:31:18.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:18.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:18.461 | INFO | Uploaded 36GiB to GPU in 2.00638s (17.9427GiB/s) | 20260506 13:31:18.461 | INFO | Downloaded 34GiB to GPU in 2.02639s (16.7786GiB/s) | 20260506 13:31:23.600 | INFO | Analyzing chunk frames = [1732096,1732608) | 20260506 13:31:23.600 | INFO | MemInfo: { | 20260506 13:31:23.600 | INFO | "gpu" : " 17 GB", | 20260506 13:31:23.600 | INFO | "huge" : "", | 20260506 13:31:23.600 | INFO | "huge_shared" : " 27 GB", | 20260506 13:31:23.600 | INFO | "malloc" : " 14 GB", | 20260506 13:31:23.600 | INFO | "pinned" : " 29 GB" | 20260506 13:31:23.600 | INFO | } | 20260506 13:31:23.600 | INFO | Perf report for the last 5.13936 seconds: | 20260506 13:31:23.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:31:23.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:23.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:23.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:23.600 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 13:31:23.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:23.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:23.600 | INFO | PRE_HQ : 34.3%, 1.0, 17.8ms, 0 | 20260506 13:31:23.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:23.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:23.600 | INFO | Uploaded 36GiB to GPU in 2.02425s (17.7844GiB/s) | 20260506 13:31:23.600 | INFO | Downloaded 34GiB to GPU in 1.98157s (17.1581GiB/s) | 20260506 13:31:23.730 | INFO | Starting a round of Baz IO | 20260506 13:31:26.920 | INFO | Populating RTMetrics report | 20260506 13:31:26.921 | INFO | RTMetrics report populated | 20260506 13:31:26.921 | INFO | Opening tmp RTMetrics json file | 20260506 13:31:26.951 | INFO | Serializing RTMetrics to json | 20260506 13:31:26.974 | INFO | Serialization done, swapping tmp file now... | 20260506 13:31:26.977 | INFO | RTMetrics done | 20260506 13:31:27.317 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.586 seconds | 20260506 13:31:27.317 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.60% / 0.02% / 60.39% | 20260506 13:31:27.317 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.12198% / 4.37101e-05% / 5.96046e-05% | 20260506 13:31:27.317 | INFO | Baz IO overall: | 20260506 13:31:27.317 | INFO | BytesWritten : 881319196315 | 20260506 13:31:27.317 | INFO | NumEvents : 426287301091 | 20260506 13:31:28.739 | INFO | Analyzing chunk frames = [1732608,1733120) | 20260506 13:31:28.739 | INFO | MemInfo: { | 20260506 13:31:28.739 | INFO | "gpu" : " 17 GB", | 20260506 13:31:28.739 | INFO | "huge" : "", | 20260506 13:31:28.739 | INFO | "huge_shared" : " 27 GB", | 20260506 13:31:28.739 | INFO | "malloc" : " 14 GB", | 20260506 13:31:28.739 | INFO | "pinned" : " 29 GB" | 20260506 13:31:28.739 | INFO | } | 20260506 13:31:28.739 | INFO | Perf report for the last 5.13913 seconds: | 20260506 13:31:28.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:31:28.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:28.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:28.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:28.739 | INFO | ANALYSIS : 63.7%, 2.8, 94.5ms, 0 | 20260506 13:31:28.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:28.739 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260506 13:31:28.739 | INFO | PRE_HQ : 61.6%, 2.0, 63.3ms, 0 | 20260506 13:31:28.739 | INFO | BAZWRITER : 69.1%, 4.7, 168.2ms, 0 | 20260506 13:31:28.739 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 13:31:28.739 | INFO | Uploaded 36GiB to GPU in 2.05935s (17.4813GiB/s) | 20260506 13:31:28.740 | INFO | Downloaded 38GiB to GPU in 2.19898s (17.2807GiB/s) | 20260506 13:31:33.879 | INFO | Analyzing chunk frames = [1733120,1733632) | 20260506 13:31:33.879 | INFO | MemInfo: { | 20260506 13:31:33.879 | INFO | "gpu" : " 17 GB", | 20260506 13:31:33.879 | INFO | "huge" : "", | 20260506 13:31:33.879 | INFO | "huge_shared" : " 27 GB", | 20260506 13:31:33.879 | INFO | "malloc" : " 14 GB", | 20260506 13:31:33.879 | INFO | "pinned" : " 29 GB" | 20260506 13:31:33.879 | INFO | } | 20260506 13:31:33.879 | INFO | Perf report for the last 5.1394 seconds: | 20260506 13:31:33.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:31:33.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:33.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:33.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:33.879 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:31:33.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:33.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:33.879 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 13:31:33.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:33.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:33.879 | INFO | Uploaded 36GiB to GPU in 2.03861s (17.6591GiB/s) | 20260506 13:31:33.879 | INFO | Downloaded 34GiB to GPU in 1.94383s (17.4913GiB/s) | 20260506 13:31:39.018 | INFO | Analyzing chunk frames = [1733632,1734144) | 20260506 13:31:39.018 | INFO | MemInfo: { | 20260506 13:31:39.018 | INFO | "gpu" : " 17 GB", | 20260506 13:31:39.018 | INFO | "huge" : "", | 20260506 13:31:39.018 | INFO | "huge_shared" : " 27 GB", | 20260506 13:31:39.018 | INFO | "malloc" : " 14 GB", | 20260506 13:31:39.018 | INFO | "pinned" : " 29 GB" | 20260506 13:31:39.018 | INFO | } | 20260506 13:31:39.018 | INFO | Perf report for the last 5.13931 seconds: | 20260506 13:31:39.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:31:39.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:39.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:39.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:39.018 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 13:31:39.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:39.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:39.018 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:31:39.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:39.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:39.019 | INFO | Uploaded 36GiB to GPU in 2.01304s (17.8834GiB/s) | 20260506 13:31:39.019 | INFO | Downloaded 34GiB to GPU in 1.94631s (17.469GiB/s) | 20260506 13:31:44.157 | INFO | Analyzing chunk frames = [1734144,1734656) | 20260506 13:31:44.157 | INFO | MemInfo: { | 20260506 13:31:44.157 | INFO | "gpu" : " 17 GB", | 20260506 13:31:44.157 | INFO | "huge" : "", | 20260506 13:31:44.158 | INFO | "huge_shared" : " 27 GB", | 20260506 13:31:44.158 | INFO | "malloc" : " 14 GB", | 20260506 13:31:44.158 | INFO | "pinned" : " 29 GB" | 20260506 13:31:44.158 | INFO | } | 20260506 13:31:44.158 | INFO | Perf report for the last 5.13971 seconds: | 20260506 13:31:44.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:31:44.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:44.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:44.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:44.158 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 13:31:44.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:44.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:44.158 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:31:44.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:44.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:44.158 | INFO | Uploaded 36GiB to GPU in 2.0018s (17.9838GiB/s) | 20260506 13:31:44.158 | INFO | Downloaded 34GiB to GPU in 1.92402s (17.6714GiB/s) | 20260506 13:31:49.296 | INFO | Analyzing chunk frames = [1734656,1735168) | 20260506 13:31:49.296 | INFO | MemInfo: { | 20260506 13:31:49.297 | INFO | "gpu" : " 17 GB", | 20260506 13:31:49.297 | INFO | "huge" : "", | 20260506 13:31:49.297 | INFO | "huge_shared" : " 27 GB", | 20260506 13:31:49.297 | INFO | "malloc" : " 14 GB", | 20260506 13:31:49.297 | INFO | "pinned" : " 29 GB" | 20260506 13:31:49.297 | INFO | } | 20260506 13:31:49.297 | INFO | Perf report for the last 5.13895 seconds: | 20260506 13:31:49.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:31:49.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:49.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:49.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:49.297 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 13:31:49.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:49.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:49.297 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:31:49.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:49.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:49.297 | INFO | Uploaded 36GiB to GPU in 1.96626s (18.3089GiB/s) | 20260506 13:31:49.297 | INFO | Downloaded 34GiB to GPU in 1.93702s (17.5528GiB/s) | 20260506 13:31:54.436 | INFO | Analyzing chunk frames = [1735168,1735680) | 20260506 13:31:54.436 | INFO | MemInfo: { | 20260506 13:31:54.436 | INFO | "gpu" : " 17 GB", | 20260506 13:31:54.436 | INFO | "huge" : "", | 20260506 13:31:54.436 | INFO | "huge_shared" : " 27 GB", | 20260506 13:31:54.436 | INFO | "malloc" : " 14 GB", | 20260506 13:31:54.436 | INFO | "pinned" : " 29 GB" | 20260506 13:31:54.436 | INFO | } | 20260506 13:31:54.436 | INFO | Perf report for the last 5.13931 seconds: | 20260506 13:31:54.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:31:54.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:54.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:54.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:54.436 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 13:31:54.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:54.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:54.436 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:31:54.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:54.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:54.436 | INFO | Uploaded 36GiB to GPU in 2.07058s (17.3864GiB/s) | 20260506 13:31:54.436 | INFO | Downloaded 34GiB to GPU in 1.96327s (17.3181GiB/s) | 20260506 13:31:59.577 | INFO | Analyzing chunk frames = [1735680,1736192) | 20260506 13:31:59.577 | INFO | MemInfo: { | 20260506 13:31:59.577 | INFO | "gpu" : " 17 GB", | 20260506 13:31:59.577 | INFO | "huge" : "", | 20260506 13:31:59.577 | INFO | "huge_shared" : " 27 GB", | 20260506 13:31:59.577 | INFO | "malloc" : " 14 GB", | 20260506 13:31:59.577 | INFO | "pinned" : " 29 GB" | 20260506 13:31:59.577 | INFO | } | 20260506 13:31:59.577 | INFO | Perf report for the last 5.14072 seconds: | 20260506 13:31:59.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:31:59.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:59.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:59.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:59.577 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 13:31:59.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:59.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:59.577 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 13:31:59.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:59.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:31:59.577 | INFO | Uploaded 36GiB to GPU in 1.99933s (18.0061GiB/s) | 20260506 13:31:59.577 | INFO | Downloaded 34GiB to GPU in 2.0259s (16.7826GiB/s) | 20260506 13:32:04.714 | INFO | Analyzing chunk frames = [1736192,1736704) | 20260506 13:32:04.714 | INFO | MemInfo: { | 20260506 13:32:04.714 | INFO | "gpu" : " 17 GB", | 20260506 13:32:04.714 | INFO | "huge" : "", | 20260506 13:32:04.714 | INFO | "huge_shared" : " 27 GB", | 20260506 13:32:04.715 | INFO | "malloc" : " 14 GB", | 20260506 13:32:04.715 | INFO | "pinned" : " 29 GB" | 20260506 13:32:04.715 | INFO | } | 20260506 13:32:04.715 | INFO | Perf report for the last 5.13781 seconds: | 20260506 13:32:04.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:32:04.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:04.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:04.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:04.715 | INFO | ANALYSIS : 62.6%, 3.0, 97.6ms, 0 | 20260506 13:32:04.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:04.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:04.715 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:32:04.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:04.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:04.715 | INFO | Uploaded 36GiB to GPU in 2.01804s (17.8391GiB/s) | 20260506 13:32:04.715 | INFO | Downloaded 34GiB to GPU in 2.00561s (16.9524GiB/s) | 20260506 13:32:04.715 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:32:04.715 | INFO | Total mem usage: 17.4 GiB | 20260506 13:32:04.715 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:32:04.715 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -11.9%) | 20260506 13:32:04.715 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:32:04.715 | INFO | High Water Breakdown: | 20260506 13:32:04.715 | INFO | 61% : 9.5 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:32:04.715 | INFO | 38.8% : 6.04 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:32:04.715 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:32:04.715 | INFO | ------------------------------------------ | 20260506 13:32:04.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:32:04.715 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:32:04.715 | INFO | Total mem usage: 26.5 GiB | 20260506 13:32:04.715 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:32:04.715 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 13:32:04.715 | INFO | Unused Cache: 1.67 GiB (Waste: 6.29%) | 20260506 13:32:04.715 | INFO | High Water Breakdown: | 20260506 13:32:04.715 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:32:04.715 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:32:04.715 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:32:04.715 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:32:04.715 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:32:04.715 | INFO | 0.698% : 182 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:32:04.715 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:32:04.715 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:32:04.715 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:32:04.715 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:32:04.715 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:32:04.715 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:32:04.715 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:32:04.715 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:32:04.715 | INFO | ------------------------------------------ | 20260506 13:32:04.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:32:04.715 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:32:04.715 | INFO | Total mem usage: 25.6 GiB | 20260506 13:32:04.715 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:32:04.715 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:32:04.715 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:32:04.715 | INFO | High Water Breakdown: | 20260506 13:32:04.715 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:32:04.715 | INFO | ------------------------------------------ | 20260506 13:32:04.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:32:04.715 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:32:04.715 | INFO | Total mem usage: 15.7 GiB | 20260506 13:32:04.715 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:32:04.715 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:32:04.715 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 13:32:04.715 | INFO | High Water Breakdown: | 20260506 13:32:04.715 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:32:04.715 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:32:04.715 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:32:04.715 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:32:04.716 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:32:04.716 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:32:04.716 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:32:04.716 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:32:04.716 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:32:04.716 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:32:04.716 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:32:04.716 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:32:04.716 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:32:04.716 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:32:04.716 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:32:04.716 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:32:04.716 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:32:04.716 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:32:04.716 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:32:04.716 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:32:04.716 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:32:04.716 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:32:04.716 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:32:04.716 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:32:04.716 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:32:04.716 | INFO | ------------------------------------------ | 20260506 13:32:04.716 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:32:04.716 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:32:04.716 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:32:04.716 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:32:04.716 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.64e+01, 1.37e+01 / 2.65e+00 | 20260506 13:32:04.716 | INFO | PerfReport: 0.29%, AggregateStats -- 9.28e-02 / 1.03e-01, 1.18e-02 / 8.01e-03 | 20260506 13:32:04.716 | INFO | PerfReport: 11.58%, Baseline -- 3.67e+00 / 4.43e+00, 5.77e-01 / 3.75e-01 | 20260506 13:32:04.716 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.49e+00, 1.93e-01 / 1.41e-01 | 20260506 13:32:04.716 | INFO | PerfReport: 15.93%, DME -- 4.04e+01 / 4.51e+01, 4.43e+00 / 2.90e+00 | 20260506 13:32:04.716 | INFO | PerfReport: 56.46%, FrameLabeling -- 1.79e+01 / 2.03e+01, 1.37e+00 / 1.70e+00 | 20260506 13:32:04.716 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.38e+00 / 2.99e+00, 4.14e-01 / 2.62e-01 | 20260506 13:32:04.716 | INFO | PerfReport: 3.67%, Metrics -- 1.16e+00 / 1.52e+00, 3.00e-01 / 1.89e-01 | 20260506 13:32:04.716 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.08e-01 / 4.64e-01, 6.36e-02 / 3.03e-02 | 20260506 13:32:04.716 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:32:04.716 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:32:04.716 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:32:04.716 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 13:32:04.716 | INFO | PerfReport: 21.48%, Upload -- 2.11e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260506 13:32:04.716 | INFO | PerfReport: 21.16%, Download -- 2.08e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 13:32:04.716 | INFO | PerfReport: 32.26%, Compute -- 3.17e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 13:32:04.716 | INFO | PerfReport: 1.76%, LockUpload -- 1.73e+00 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260506 13:32:04.716 | INFO | PerfReport: 1.11%, LockDownload -- 1.09e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260506 13:32:04.716 | INFO | PerfReport: 22.20%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 13:32:04.716 | INFO | PerfReport: 0.02%, Misc -- 1.54e-02 / 0.00e+00, 2.51e-02 / 0.00e+00 | 20260506 13:32:04.841 | INFO | Starting a round of Baz IO | 20260506 13:32:08.003 | INFO | Populating RTMetrics report | 20260506 13:32:08.003 | INFO | RTMetrics report populated | 20260506 13:32:08.003 | INFO | Opening tmp RTMetrics json file | 20260506 13:32:08.007 | INFO | Serializing RTMetrics to json | 20260506 13:32:08.011 | INFO | Serialization done, swapping tmp file now... | 20260506 13:32:08.014 | INFO | RTMetrics done | 20260506 13:32:08.125 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.284 seconds | 20260506 13:32:08.125 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.58% / 0.02% / 60.40% | 20260506 13:32:08.126 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.10789% / 2.78155e-05% / 6.35783e-05% | 20260506 13:32:08.126 | INFO | Baz IO overall: | 20260506 13:32:08.126 | INFO | BytesWritten : 883584158856 | 20260506 13:32:08.126 | INFO | NumEvents : 427385735126 | 20260506 13:32:09.854 | INFO | Analyzing chunk frames = [1736704,1737216) | 20260506 13:32:09.854 | INFO | MemInfo: { | 20260506 13:32:09.854 | INFO | "gpu" : " 17 GB", | 20260506 13:32:09.854 | INFO | "huge" : "", | 20260506 13:32:09.854 | INFO | "huge_shared" : " 27 GB", | 20260506 13:32:09.854 | INFO | "malloc" : " 14 GB", | 20260506 13:32:09.854 | INFO | "pinned" : " 29 GB" | 20260506 13:32:09.854 | INFO | } | 20260506 13:32:09.854 | INFO | Perf report for the last 5.13929 seconds: | 20260506 13:32:09.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:32:09.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:09.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:09.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:09.854 | INFO | ANALYSIS : 63.4%, 2.9, 95.8ms, 0 | 20260506 13:32:09.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:09.854 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 13:32:09.854 | INFO | PRE_HQ : 61.4%, 1.9, 60.4ms, 0 | 20260506 13:32:09.854 | INFO | BAZWRITER : 63.3%, 4.6, 149.3ms, 0 | 20260506 13:32:09.854 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 13:32:09.854 | INFO | Uploaded 36GiB to GPU in 2.05825s (17.4906GiB/s) | 20260506 13:32:09.854 | INFO | Downloaded 38GiB to GPU in 2.23476s (17.004GiB/s) | 20260506 13:32:14.993 | INFO | Analyzing chunk frames = [1737216,1737728) | 20260506 13:32:14.993 | INFO | MemInfo: { | 20260506 13:32:14.993 | INFO | "gpu" : " 17 GB", | 20260506 13:32:14.993 | INFO | "huge" : "", | 20260506 13:32:14.993 | INFO | "huge_shared" : " 27 GB", | 20260506 13:32:14.993 | INFO | "malloc" : " 14 GB", | 20260506 13:32:14.993 | INFO | "pinned" : " 29 GB" | 20260506 13:32:14.993 | INFO | } | 20260506 13:32:14.993 | INFO | Perf report for the last 5.13934 seconds: | 20260506 13:32:14.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:32:14.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:14.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:14.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:14.993 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 13:32:14.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:14.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:14.993 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 13:32:14.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:14.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:14.993 | INFO | Uploaded 36GiB to GPU in 1.98858s (18.1034GiB/s) | 20260506 13:32:14.993 | INFO | Downloaded 34GiB to GPU in 1.9433s (17.496GiB/s) | 20260506 13:32:20.133 | INFO | Analyzing chunk frames = [1737728,1738240) | 20260506 13:32:20.133 | INFO | MemInfo: { | 20260506 13:32:20.133 | INFO | "gpu" : " 17 GB", | 20260506 13:32:20.133 | INFO | "huge" : "", | 20260506 13:32:20.133 | INFO | "huge_shared" : " 27 GB", | 20260506 13:32:20.133 | INFO | "malloc" : " 14 GB", | 20260506 13:32:20.133 | INFO | "pinned" : " 29 GB" | 20260506 13:32:20.133 | INFO | } | 20260506 13:32:20.133 | INFO | Perf report for the last 5.13961 seconds: | 20260506 13:32:20.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:32:20.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:20.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:20.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:20.133 | INFO | ANALYSIS : 61.4%, 3.0, 95.0ms, 0 | 20260506 13:32:20.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:20.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:20.133 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:32:20.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:20.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:20.133 | INFO | Uploaded 36GiB to GPU in 1.96695s (18.3025GiB/s) | 20260506 13:32:20.133 | INFO | Downloaded 34GiB to GPU in 1.93946s (17.5307GiB/s) | 20260506 13:32:25.272 | INFO | Analyzing chunk frames = [1738240,1738752) | 20260506 13:32:25.272 | INFO | MemInfo: { | 20260506 13:32:25.272 | INFO | "gpu" : " 17 GB", | 20260506 13:32:25.272 | INFO | "huge" : "", | 20260506 13:32:25.272 | INFO | "huge_shared" : " 27 GB", | 20260506 13:32:25.272 | INFO | "malloc" : " 14 GB", | 20260506 13:32:25.272 | INFO | "pinned" : " 29 GB" | 20260506 13:32:25.272 | INFO | } | 20260506 13:32:25.272 | INFO | Perf report for the last 5.1393 seconds: | 20260506 13:32:25.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:32:25.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:25.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:25.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:25.272 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:32:25.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:25.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:25.272 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:32:25.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:25.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:25.272 | INFO | Uploaded 36GiB to GPU in 2.02905s (17.7423GiB/s) | 20260506 13:32:25.272 | INFO | Downloaded 34GiB to GPU in 1.91671s (17.7388GiB/s) | 20260506 13:32:30.411 | INFO | Analyzing chunk frames = [1738752,1739264) | 20260506 13:32:30.411 | INFO | MemInfo: { | 20260506 13:32:30.411 | INFO | "gpu" : " 17 GB", | 20260506 13:32:30.411 | INFO | "huge" : "", | 20260506 13:32:30.411 | INFO | "huge_shared" : " 27 GB", | 20260506 13:32:30.411 | INFO | "malloc" : " 14 GB", | 20260506 13:32:30.411 | INFO | "pinned" : " 29 GB" | 20260506 13:32:30.412 | INFO | } | 20260506 13:32:30.412 | INFO | Perf report for the last 5.13941 seconds: | 20260506 13:32:30.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:32:30.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:30.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:30.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:30.412 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 13:32:30.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:30.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:30.412 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:32:30.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:30.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:30.412 | INFO | Uploaded 36GiB to GPU in 2.02871s (17.7453GiB/s) | 20260506 13:32:30.412 | INFO | Downloaded 34GiB to GPU in 1.94877s (17.4469GiB/s) | 20260506 13:32:35.550 | INFO | Analyzing chunk frames = [1739264,1739776) | 20260506 13:32:35.550 | INFO | MemInfo: { | 20260506 13:32:35.550 | INFO | "gpu" : " 17 GB", | 20260506 13:32:35.550 | INFO | "huge" : "", | 20260506 13:32:35.551 | INFO | "huge_shared" : " 27 GB", | 20260506 13:32:35.551 | INFO | "malloc" : " 14 GB", | 20260506 13:32:35.551 | INFO | "pinned" : " 29 GB" | 20260506 13:32:35.551 | INFO | } | 20260506 13:32:35.551 | INFO | Perf report for the last 5.13899 seconds: | 20260506 13:32:35.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:32:35.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:35.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:35.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:35.551 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 13:32:35.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:35.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:35.551 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 13:32:35.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:35.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:35.551 | INFO | Uploaded 36GiB to GPU in 2.02301s (17.7953GiB/s) | 20260506 13:32:35.551 | INFO | Downloaded 34GiB to GPU in 1.93019s (17.6149GiB/s) | 20260506 13:32:40.691 | INFO | Analyzing chunk frames = [1739776,1740288) | 20260506 13:32:40.691 | INFO | MemInfo: { | 20260506 13:32:40.691 | INFO | "gpu" : " 17 GB", | 20260506 13:32:40.691 | INFO | "huge" : "", | 20260506 13:32:40.691 | INFO | "huge_shared" : " 27 GB", | 20260506 13:32:40.691 | INFO | "malloc" : " 14 GB", | 20260506 13:32:40.691 | INFO | "pinned" : " 29 GB" | 20260506 13:32:40.691 | INFO | } | 20260506 13:32:40.692 | INFO | Perf report for the last 5.14092 seconds: | 20260506 13:32:40.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:32:40.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:40.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:40.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:40.692 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 13:32:40.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:40.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:40.692 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 13:32:40.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:40.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:40.692 | INFO | Uploaded 36GiB to GPU in 2.02916s (17.7413GiB/s) | 20260506 13:32:40.692 | INFO | Downloaded 34GiB to GPU in 1.99787s (17.0182GiB/s) | 20260506 13:32:45.829 | INFO | Analyzing chunk frames = [1740288,1740800) | 20260506 13:32:45.829 | INFO | MemInfo: { | 20260506 13:32:45.829 | INFO | "gpu" : " 17 GB", | 20260506 13:32:45.829 | INFO | "huge" : "", | 20260506 13:32:45.830 | INFO | "huge_shared" : " 27 GB", | 20260506 13:32:45.830 | INFO | "malloc" : " 14 GB", | 20260506 13:32:45.830 | INFO | "pinned" : " 29 GB" | 20260506 13:32:45.830 | INFO | } | 20260506 13:32:45.830 | INFO | Perf report for the last 5.13812 seconds: | 20260506 13:32:45.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:32:45.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:45.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:45.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:45.830 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 13:32:45.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:45.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:45.830 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:32:45.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:45.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:45.830 | INFO | Uploaded 36GiB to GPU in 2.04785s (17.5795GiB/s) | 20260506 13:32:45.830 | INFO | Downloaded 34GiB to GPU in 1.96259s (17.324GiB/s) | 20260506 13:32:45.961 | INFO | Starting a round of Baz IO | 20260506 13:32:49.114 | INFO | Populating RTMetrics report | 20260506 13:32:49.115 | INFO | RTMetrics report populated | 20260506 13:32:49.115 | INFO | Opening tmp RTMetrics json file | 20260506 13:32:49.177 | INFO | Serializing RTMetrics to json | 20260506 13:32:49.226 | INFO | Serialization done, swapping tmp file now... | 20260506 13:32:49.263 | INFO | RTMetrics done | 20260506 13:32:49.505 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.543 seconds | 20260506 13:32:49.505 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.56% / 0.02% / 60.42% | 20260506 13:32:49.505 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.10031% / 1.58946e-05% / 4.37101e-05% | 20260506 13:32:49.505 | INFO | Baz IO overall: | 20260506 13:32:49.505 | INFO | BytesWritten : 885849153506 | 20260506 13:32:49.505 | INFO | NumEvents : 428483961112 | 20260506 13:32:50.969 | INFO | Analyzing chunk frames = [1740800,1741312) | 20260506 13:32:50.969 | INFO | MemInfo: { | 20260506 13:32:50.969 | INFO | "gpu" : " 17 GB", | 20260506 13:32:50.969 | INFO | "huge" : "", | 20260506 13:32:50.969 | INFO | "huge_shared" : " 27 GB", | 20260506 13:32:50.969 | INFO | "malloc" : " 14 GB", | 20260506 13:32:50.969 | INFO | "pinned" : " 29 GB" | 20260506 13:32:50.969 | INFO | } | 20260506 13:32:50.969 | INFO | Perf report for the last 5.13922 seconds: | 20260506 13:32:50.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:32:50.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:50.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:50.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:50.969 | INFO | ANALYSIS : 63.3%, 2.9, 95.6ms, 0 | 20260506 13:32:50.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:50.969 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260506 13:32:50.969 | INFO | PRE_HQ : 61.2%, 1.9, 60.2ms, 0 | 20260506 13:32:50.969 | INFO | BAZWRITER : 68.3%, 4.8, 169.2ms, 0 | 20260506 13:32:50.969 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 13:32:50.969 | INFO | Uploaded 36GiB to GPU in 2.11209s (17.0448GiB/s) | 20260506 13:32:50.969 | INFO | Downloaded 38GiB to GPU in 2.247s (16.9114GiB/s) | 20260506 13:32:56.108 | INFO | Analyzing chunk frames = [1741312,1741824) | 20260506 13:32:56.108 | INFO | MemInfo: { | 20260506 13:32:56.108 | INFO | "gpu" : " 17 GB", | 20260506 13:32:56.109 | INFO | "huge" : "", | 20260506 13:32:56.109 | INFO | "huge_shared" : " 27 GB", | 20260506 13:32:56.109 | INFO | "malloc" : " 14 GB", | 20260506 13:32:56.109 | INFO | "pinned" : " 29 GB" | 20260506 13:32:56.109 | INFO | } | 20260506 13:32:56.109 | INFO | Perf report for the last 5.13977 seconds: | 20260506 13:32:56.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:32:56.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:56.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:56.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:56.109 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 13:32:56.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:56.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:56.109 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:32:56.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:56.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:32:56.109 | INFO | Uploaded 36GiB to GPU in 2.06964s (17.3943GiB/s) | 20260506 13:32:56.109 | INFO | Downloaded 34GiB to GPU in 1.94282s (17.5003GiB/s) | 20260506 13:33:01.248 | INFO | Analyzing chunk frames = [1741824,1742336) | 20260506 13:33:01.248 | INFO | MemInfo: { | 20260506 13:33:01.248 | INFO | "gpu" : " 17 GB", | 20260506 13:33:01.248 | INFO | "huge" : "", | 20260506 13:33:01.248 | INFO | "huge_shared" : " 27 GB", | 20260506 13:33:01.248 | INFO | "malloc" : " 14 GB", | 20260506 13:33:01.248 | INFO | "pinned" : " 29 GB" | 20260506 13:33:01.248 | INFO | } | 20260506 13:33:01.248 | INFO | Perf report for the last 5.13981 seconds: | 20260506 13:33:01.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:33:01.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:01.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:01.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:01.248 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 13:33:01.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:01.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:01.248 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:33:01.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:01.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:01.249 | INFO | Uploaded 36GiB to GPU in 2.03142s (17.7216GiB/s) | 20260506 13:33:01.249 | INFO | Downloaded 34GiB to GPU in 1.94271s (17.5013GiB/s) | 20260506 13:33:06.387 | INFO | Analyzing chunk frames = [1742336,1742848) | 20260506 13:33:06.387 | INFO | MemInfo: { | 20260506 13:33:06.387 | INFO | "gpu" : " 17 GB", | 20260506 13:33:06.387 | INFO | "huge" : "", | 20260506 13:33:06.387 | INFO | "huge_shared" : " 27 GB", | 20260506 13:33:06.387 | INFO | "malloc" : " 14 GB", | 20260506 13:33:06.387 | INFO | "pinned" : " 29 GB" | 20260506 13:33:06.387 | INFO | } | 20260506 13:33:06.387 | INFO | Perf report for the last 5.13897 seconds: | 20260506 13:33:06.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:33:06.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:06.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:06.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:06.387 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 13:33:06.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:06.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:06.388 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:33:06.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:06.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:06.388 | INFO | Uploaded 36GiB to GPU in 2.0326s (17.7113GiB/s) | 20260506 13:33:06.388 | INFO | Downloaded 34GiB to GPU in 1.94428s (17.4872GiB/s) | 20260506 13:33:11.526 | INFO | Analyzing chunk frames = [1742848,1743360) | 20260506 13:33:11.526 | INFO | MemInfo: { | 20260506 13:33:11.526 | INFO | "gpu" : " 17 GB", | 20260506 13:33:11.526 | INFO | "huge" : "", | 20260506 13:33:11.526 | INFO | "huge_shared" : " 27 GB", | 20260506 13:33:11.526 | INFO | "malloc" : " 14 GB", | 20260506 13:33:11.526 | INFO | "pinned" : " 29 GB" | 20260506 13:33:11.526 | INFO | } | 20260506 13:33:11.526 | INFO | Perf report for the last 5.139 seconds: | 20260506 13:33:11.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:33:11.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:11.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:11.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:11.526 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 13:33:11.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:11.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:11.527 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:33:11.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:11.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:11.527 | INFO | Uploaded 36GiB to GPU in 2.05023s (17.559GiB/s) | 20260506 13:33:11.527 | INFO | Downloaded 34GiB to GPU in 1.95407s (17.3996GiB/s) | 20260506 13:33:16.665 | INFO | Analyzing chunk frames = [1743360,1743872) | 20260506 13:33:16.665 | INFO | MemInfo: { | 20260506 13:33:16.665 | INFO | "gpu" : " 17 GB", | 20260506 13:33:16.665 | INFO | "huge" : "", | 20260506 13:33:16.665 | INFO | "huge_shared" : " 27 GB", | 20260506 13:33:16.666 | INFO | "malloc" : " 14 GB", | 20260506 13:33:16.666 | INFO | "pinned" : " 29 GB" | 20260506 13:33:16.666 | INFO | } | 20260506 13:33:16.666 | INFO | Perf report for the last 5.13911 seconds: | 20260506 13:33:16.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:33:16.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:16.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:16.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:16.666 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 13:33:16.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:16.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:16.666 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260506 13:33:16.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:16.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:16.666 | INFO | Uploaded 36GiB to GPU in 2.05285s (17.5366GiB/s) | 20260506 13:33:16.666 | INFO | Downloaded 34GiB to GPU in 1.91075s (17.7941GiB/s) | 20260506 13:33:21.805 | INFO | Analyzing chunk frames = [1743872,1744384) | 20260506 13:33:21.805 | INFO | MemInfo: { | 20260506 13:33:21.805 | INFO | "gpu" : " 17 GB", | 20260506 13:33:21.805 | INFO | "huge" : "", | 20260506 13:33:21.805 | INFO | "huge_shared" : " 27 GB", | 20260506 13:33:21.805 | INFO | "malloc" : " 14 GB", | 20260506 13:33:21.805 | INFO | "pinned" : " 29 GB" | 20260506 13:33:21.805 | INFO | } | 20260506 13:33:21.805 | INFO | Perf report for the last 5.13934 seconds: | 20260506 13:33:21.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:33:21.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:21.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:21.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:21.805 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 13:33:21.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:21.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:21.805 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:33:21.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:21.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:21.805 | INFO | Uploaded 36GiB to GPU in 1.99995s (18.0005GiB/s) | 20260506 13:33:21.805 | INFO | Downloaded 34GiB to GPU in 1.96018s (17.3453GiB/s) | 20260506 13:33:26.944 | INFO | Analyzing chunk frames = [1744384,1744896) | 20260506 13:33:26.944 | INFO | MemInfo: { | 20260506 13:33:26.944 | INFO | "gpu" : " 17 GB", | 20260506 13:33:26.944 | INFO | "huge" : "", | 20260506 13:33:26.944 | INFO | "huge_shared" : " 27 GB", | 20260506 13:33:26.944 | INFO | "malloc" : " 14 GB", | 20260506 13:33:26.944 | INFO | "pinned" : " 29 GB" | 20260506 13:33:26.944 | INFO | } | 20260506 13:33:26.944 | INFO | Perf report for the last 5.13923 seconds: | 20260506 13:33:26.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:33:26.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:26.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:26.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:26.944 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 13:33:26.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:26.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:26.944 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:33:26.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:26.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:26.944 | INFO | Uploaded 36GiB to GPU in 2.0464s (17.5919GiB/s) | 20260506 13:33:26.944 | INFO | Downloaded 34GiB to GPU in 1.98106s (17.1625GiB/s) | 20260506 13:33:26.944 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:33:26.944 | INFO | Total mem usage: 17.4 GiB | 20260506 13:33:26.944 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:33:26.944 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -13.7%) | 20260506 13:33:26.944 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:33:26.944 | INFO | High Water Breakdown: | 20260506 13:33:26.944 | INFO | 60.7% : 9.3 GiB ( 85.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:33:26.944 | INFO | 39.1% : 5.99 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:33:26.944 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:33:26.945 | INFO | ------------------------------------------ | 20260506 13:33:26.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:33:26.945 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:33:26.945 | INFO | Total mem usage: 26.5 GiB | 20260506 13:33:26.945 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:33:26.945 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:33:26.945 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:33:26.945 | INFO | High Water Breakdown: | 20260506 13:33:26.945 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:33:26.945 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:33:26.945 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:33:26.945 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:33:26.945 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:33:26.945 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:33:26.945 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:33:26.945 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:33:26.945 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:33:26.945 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:33:26.945 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:33:26.945 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:33:26.945 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:33:26.945 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:33:26.945 | INFO | ------------------------------------------ | 20260506 13:33:26.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:33:26.945 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:33:26.945 | INFO | Total mem usage: 25.6 GiB | 20260506 13:33:26.945 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:33:26.945 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:33:26.945 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:33:26.945 | INFO | High Water Breakdown: | 20260506 13:33:26.945 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:33:26.945 | INFO | ------------------------------------------ | 20260506 13:33:26.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:33:26.945 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:33:26.945 | INFO | Total mem usage: 15.7 GiB | 20260506 13:33:26.945 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:33:26.945 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:33:26.945 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:33:26.945 | INFO | High Water Breakdown: | 20260506 13:33:26.945 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:33:26.945 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:33:26.945 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:33:26.945 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:33:26.945 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:33:26.945 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:33:26.945 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:33:26.945 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:33:26.945 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:33:26.945 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:33:26.945 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:33:26.945 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:33:26.945 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:33:26.945 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:33:26.945 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:33:26.945 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:33:26.945 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:33:26.945 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:33:26.945 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:33:26.945 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:33:26.945 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:33:26.945 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:33:26.945 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:33:26.945 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:33:26.945 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:33:26.945 | INFO | ------------------------------------------ | 20260506 13:33:26.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:33:26.945 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:33:26.945 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:33:26.945 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:33:26.945 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.56e+01, 1.35e+01 / 2.13e+00 | 20260506 13:33:26.945 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 9.93e-02, 1.04e-02 / 5.95e-03 | 20260506 13:33:26.945 | INFO | PerfReport: 11.58%, Baseline -- 3.66e+00 / 4.42e+00, 5.80e-01 / 3.93e-01 | 20260506 13:33:26.945 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.43e+00, 1.91e-01 / 1.51e-01 | 20260506 13:33:26.945 | INFO | PerfReport: 15.75%, DME -- 3.99e+01 / 4.36e+01, 3.73e+00 / 2.18e+00 | 20260506 13:33:26.945 | INFO | PerfReport: 56.67%, FrameLabeling -- 1.79e+01 / 2.10e+01, 1.49e+00 / 1.65e+00 | 20260506 13:33:26.945 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.37e+00 / 2.99e+00, 4.08e-01 / 2.40e-01 | 20260506 13:33:26.945 | INFO | PerfReport: 3.67%, Metrics -- 1.16e+00 / 1.55e+00, 3.00e-01 / 1.48e-01 | 20260506 13:33:26.945 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.08e-01 / 4.63e-01, 6.32e-02 / 2.13e-02 | 20260506 13:33:26.945 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:33:26.945 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:33:26.945 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:33:26.945 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 13:33:26.945 | INFO | PerfReport: 21.59%, Upload -- 2.12e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260506 13:33:26.945 | INFO | PerfReport: 21.06%, Download -- 2.07e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 13:33:26.945 | INFO | PerfReport: 32.23%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 13:33:26.945 | INFO | PerfReport: 1.78%, LockUpload -- 1.74e+00 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260506 13:33:26.945 | INFO | PerfReport: 1.09%, LockDownload -- 1.07e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260506 13:33:26.945 | INFO | PerfReport: 22.24%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 13:33:26.945 | INFO | PerfReport: 0.02%, Misc -- 1.53e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260506 13:33:27.078 | INFO | Starting a round of Baz IO | 20260506 13:33:30.239 | INFO | Populating RTMetrics report | 20260506 13:33:30.239 | INFO | RTMetrics report populated | 20260506 13:33:30.239 | INFO | Opening tmp RTMetrics json file | 20260506 13:33:30.245 | INFO | Serializing RTMetrics to json | 20260506 13:33:30.256 | INFO | Serialization done, swapping tmp file now... | 20260506 13:33:30.271 | INFO | RTMetrics done | 20260506 13:33:30.356 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.277 seconds | 20260506 13:33:30.356 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.54% / 0.02% / 60.44% | 20260506 13:33:30.356 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.10043% / 1.98682e-05% / 5.5631e-05% | 20260506 13:33:30.356 | INFO | Baz IO overall: | 20260506 13:33:30.356 | INFO | BytesWritten : 888115579396 | 20260506 13:33:30.356 | INFO | NumEvents : 429583329096 | 20260506 13:33:32.084 | INFO | Analyzing chunk frames = [1744896,1745408) | 20260506 13:33:32.084 | INFO | MemInfo: { | 20260506 13:33:32.084 | INFO | "gpu" : " 17 GB", | 20260506 13:33:32.084 | INFO | "huge" : "", | 20260506 13:33:32.084 | INFO | "huge_shared" : " 27 GB", | 20260506 13:33:32.084 | INFO | "malloc" : " 14 GB", | 20260506 13:33:32.084 | INFO | "pinned" : " 29 GB" | 20260506 13:33:32.084 | INFO | } | 20260506 13:33:32.084 | INFO | Perf report for the last 5.13969 seconds: | 20260506 13:33:32.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:33:32.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:32.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:32.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:32.084 | INFO | ANALYSIS : 63.5%, 2.9, 96.0ms, 0 | 20260506 13:33:32.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:32.084 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 13:33:32.084 | INFO | PRE_HQ : 61.3%, 1.9, 60.2ms, 0 | 20260506 13:33:32.084 | INFO | BAZWRITER : 63.1%, 4.4, 143.5ms, 0 | 20260506 13:33:32.084 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 13:33:32.084 | INFO | Uploaded 36GiB to GPU in 2.06506s (17.4329GiB/s) | 20260506 13:33:32.084 | INFO | Downloaded 38GiB to GPU in 2.23978s (16.966GiB/s) | 20260506 13:33:37.223 | INFO | Analyzing chunk frames = [1745408,1745920) | 20260506 13:33:37.223 | INFO | MemInfo: { | 20260506 13:33:37.223 | INFO | "gpu" : " 17 GB", | 20260506 13:33:37.223 | INFO | "huge" : "", | 20260506 13:33:37.223 | INFO | "huge_shared" : " 27 GB", | 20260506 13:33:37.223 | INFO | "malloc" : " 14 GB", | 20260506 13:33:37.223 | INFO | "pinned" : " 29 GB" | 20260506 13:33:37.223 | INFO | } | 20260506 13:33:37.223 | INFO | Perf report for the last 5.13906 seconds: | 20260506 13:33:37.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:33:37.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:37.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:37.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:37.223 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:33:37.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:37.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:37.223 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:33:37.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:37.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:37.223 | INFO | Uploaded 36GiB to GPU in 2.03888s (17.6568GiB/s) | 20260506 13:33:37.223 | INFO | Downloaded 34GiB to GPU in 1.94819s (17.4521GiB/s) | 20260506 13:33:42.362 | INFO | Analyzing chunk frames = [1745920,1746432) | 20260506 13:33:42.362 | INFO | MemInfo: { | 20260506 13:33:42.362 | INFO | "gpu" : " 17 GB", | 20260506 13:33:42.362 | INFO | "huge" : "", | 20260506 13:33:42.362 | INFO | "huge_shared" : " 27 GB", | 20260506 13:33:42.362 | INFO | "malloc" : " 14 GB", | 20260506 13:33:42.362 | INFO | "pinned" : " 29 GB" | 20260506 13:33:42.362 | INFO | } | 20260506 13:33:42.362 | INFO | Perf report for the last 5.1394 seconds: | 20260506 13:33:42.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:33:42.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:42.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:42.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:42.362 | INFO | ANALYSIS : 61.6%, 3.0, 95.3ms, 0 | 20260506 13:33:42.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:42.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:42.362 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:33:42.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:42.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:42.363 | INFO | Uploaded 36GiB to GPU in 2.01048s (17.9062GiB/s) | 20260506 13:33:42.363 | INFO | Downloaded 34GiB to GPU in 1.94837s (17.4505GiB/s) | 20260506 13:33:47.502 | INFO | Analyzing chunk frames = [1746432,1746944) | 20260506 13:33:47.502 | INFO | MemInfo: { | 20260506 13:33:47.502 | INFO | "gpu" : " 17 GB", | 20260506 13:33:47.502 | INFO | "huge" : "", | 20260506 13:33:47.502 | INFO | "huge_shared" : " 27 GB", | 20260506 13:33:47.502 | INFO | "malloc" : " 14 GB", | 20260506 13:33:47.502 | INFO | "pinned" : " 29 GB" | 20260506 13:33:47.502 | INFO | } | 20260506 13:33:47.502 | INFO | Perf report for the last 5.13953 seconds: | 20260506 13:33:47.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:33:47.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:47.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:47.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:47.502 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:33:47.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:47.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:47.502 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:33:47.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:47.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:47.502 | INFO | Uploaded 36GiB to GPU in 1.99096s (18.0817GiB/s) | 20260506 13:33:47.502 | INFO | Downloaded 34GiB to GPU in 1.92328s (17.6781GiB/s) | 20260506 13:33:52.641 | INFO | Analyzing chunk frames = [1746944,1747456) | 20260506 13:33:52.641 | INFO | MemInfo: { | 20260506 13:33:52.641 | INFO | "gpu" : " 17 GB", | 20260506 13:33:52.641 | INFO | "huge" : "", | 20260506 13:33:52.641 | INFO | "huge_shared" : " 27 GB", | 20260506 13:33:52.641 | INFO | "malloc" : " 14 GB", | 20260506 13:33:52.641 | INFO | "pinned" : " 29 GB" | 20260506 13:33:52.641 | INFO | } | 20260506 13:33:52.641 | INFO | Perf report for the last 5.13921 seconds: | 20260506 13:33:52.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:33:52.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:52.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:52.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:52.641 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 13:33:52.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:52.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:52.641 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:33:52.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:52.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:52.641 | INFO | Uploaded 36GiB to GPU in 2.03753s (17.6685GiB/s) | 20260506 13:33:52.641 | INFO | Downloaded 34GiB to GPU in 1.94753s (17.4581GiB/s) | 20260506 13:33:57.780 | INFO | Analyzing chunk frames = [1747456,1747968) | 20260506 13:33:57.780 | INFO | MemInfo: { | 20260506 13:33:57.780 | INFO | "gpu" : " 17 GB", | 20260506 13:33:57.780 | INFO | "huge" : "", | 20260506 13:33:57.780 | INFO | "huge_shared" : " 27 GB", | 20260506 13:33:57.780 | INFO | "malloc" : " 14 GB", | 20260506 13:33:57.780 | INFO | "pinned" : " 29 GB" | 20260506 13:33:57.780 | INFO | } | 20260506 13:33:57.780 | INFO | Perf report for the last 5.13934 seconds: | 20260506 13:33:57.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:33:57.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:57.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:57.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:57.780 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 13:33:57.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:57.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:57.781 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 13:33:57.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:57.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:33:57.781 | INFO | Uploaded 36GiB to GPU in 2.05367s (17.5296GiB/s) | 20260506 13:33:57.781 | INFO | Downloaded 34GiB to GPU in 1.93016s (17.6151GiB/s) | 20260506 13:34:02.920 | INFO | Analyzing chunk frames = [1747968,1748480) | 20260506 13:34:02.920 | INFO | MemInfo: { | 20260506 13:34:02.920 | INFO | "gpu" : " 17 GB", | 20260506 13:34:02.920 | INFO | "huge" : "", | 20260506 13:34:02.920 | INFO | "huge_shared" : " 27 GB", | 20260506 13:34:02.920 | INFO | "malloc" : " 14 GB", | 20260506 13:34:02.920 | INFO | "pinned" : " 29 GB" | 20260506 13:34:02.920 | INFO | } | 20260506 13:34:02.920 | INFO | Perf report for the last 5.13932 seconds: | 20260506 13:34:02.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:34:02.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:02.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:02.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:02.920 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 13:34:02.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:02.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:02.920 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 13:34:02.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:02.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:02.920 | INFO | Uploaded 36GiB to GPU in 2.00632s (17.9433GiB/s) | 20260506 13:34:02.920 | INFO | Downloaded 34GiB to GPU in 1.98122s (17.1611GiB/s) | 20260506 13:34:08.059 | INFO | Analyzing chunk frames = [1748480,1748992) | 20260506 13:34:08.059 | INFO | MemInfo: { | 20260506 13:34:08.059 | INFO | "gpu" : " 17 GB", | 20260506 13:34:08.059 | INFO | "huge" : "", | 20260506 13:34:08.059 | INFO | "huge_shared" : " 27 GB", | 20260506 13:34:08.059 | INFO | "malloc" : " 14 GB", | 20260506 13:34:08.059 | INFO | "pinned" : " 29 GB" | 20260506 13:34:08.059 | INFO | } | 20260506 13:34:08.059 | INFO | Perf report for the last 5.1395 seconds: | 20260506 13:34:08.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:34:08.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:08.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:08.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:08.059 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:34:08.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:08.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:08.059 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:34:08.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:08.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:08.060 | INFO | Uploaded 36GiB to GPU in 2.00693s (17.9379GiB/s) | 20260506 13:34:08.060 | INFO | Downloaded 34GiB to GPU in 1.97146s (17.2461GiB/s) | 20260506 13:34:08.181 | INFO | Starting a round of Baz IO | 20260506 13:34:11.347 | INFO | Populating RTMetrics report | 20260506 13:34:11.347 | INFO | RTMetrics report populated | 20260506 13:34:11.347 | INFO | Opening tmp RTMetrics json file | 20260506 13:34:11.356 | INFO | Serializing RTMetrics to json | 20260506 13:34:11.410 | INFO | Serialization done, swapping tmp file now... | 20260506 13:34:11.442 | INFO | RTMetrics done | 20260506 13:34:11.559 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.377 seconds | 20260506 13:34:11.559 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.52% / 0.02% / 60.46% | 20260506 13:34:11.559 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.08556% / 4.37101e-05% / 6.35783e-05% | 20260506 13:34:11.559 | INFO | Baz IO overall: | 20260506 13:34:11.559 | INFO | BytesWritten : 890381217502 | 20260506 13:34:11.559 | INFO | NumEvents : 430682186993 | 20260506 13:34:13.198 | INFO | Analyzing chunk frames = [1748992,1749504) | 20260506 13:34:13.198 | INFO | MemInfo: { | 20260506 13:34:13.198 | INFO | "gpu" : " 17 GB", | 20260506 13:34:13.198 | INFO | "huge" : "", | 20260506 13:34:13.198 | INFO | "huge_shared" : " 27 GB", | 20260506 13:34:13.198 | INFO | "malloc" : " 14 GB", | 20260506 13:34:13.198 | INFO | "pinned" : " 29 GB" | 20260506 13:34:13.198 | INFO | } | 20260506 13:34:13.199 | INFO | Perf report for the last 5.13921 seconds: | 20260506 13:34:13.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:34:13.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:13.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:13.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:13.199 | INFO | ANALYSIS : 63.4%, 2.9, 95.7ms, 0 | 20260506 13:34:13.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:13.199 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260506 13:34:13.199 | INFO | PRE_HQ : 61.2%, 1.9, 60.2ms, 0 | 20260506 13:34:13.199 | INFO | BAZWRITER : 65.1%, 4.8, 160.0ms, 0 | 20260506 13:34:13.199 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 13:34:13.199 | INFO | Uploaded 36GiB to GPU in 2.07762s (17.3276GiB/s) | 20260506 13:34:13.199 | INFO | Downloaded 38GiB to GPU in 2.24445s (16.9307GiB/s) | 20260506 13:34:18.338 | INFO | Analyzing chunk frames = [1749504,1750016) | 20260506 13:34:18.338 | INFO | MemInfo: { | 20260506 13:34:18.338 | INFO | "gpu" : " 17 GB", | 20260506 13:34:18.338 | INFO | "huge" : "", | 20260506 13:34:18.338 | INFO | "huge_shared" : " 27 GB", | 20260506 13:34:18.338 | INFO | "malloc" : " 14 GB", | 20260506 13:34:18.338 | INFO | "pinned" : " 29 GB" | 20260506 13:34:18.338 | INFO | } | 20260506 13:34:18.338 | INFO | Perf report for the last 5.13925 seconds: | 20260506 13:34:18.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:34:18.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:18.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:18.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:18.338 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:34:18.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:18.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:18.338 | INFO | PRE_HQ : 28.2%, 1.0, 14.8ms, 0 | 20260506 13:34:18.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:18.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:18.338 | INFO | Uploaded 36GiB to GPU in 2.03537s (17.6872GiB/s) | 20260506 13:34:18.338 | INFO | Downloaded 34GiB to GPU in 1.93717s (17.5514GiB/s) | 20260506 13:34:23.477 | INFO | Analyzing chunk frames = [1750016,1750528) | 20260506 13:34:23.477 | INFO | MemInfo: { | 20260506 13:34:23.477 | INFO | "gpu" : " 17 GB", | 20260506 13:34:23.477 | INFO | "huge" : "", | 20260506 13:34:23.477 | INFO | "huge_shared" : " 27 GB", | 20260506 13:34:23.477 | INFO | "malloc" : " 14 GB", | 20260506 13:34:23.477 | INFO | "pinned" : " 29 GB" | 20260506 13:34:23.477 | INFO | } | 20260506 13:34:23.477 | INFO | Perf report for the last 5.1392 seconds: | 20260506 13:34:23.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:34:23.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:23.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:23.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:23.477 | INFO | ANALYSIS : 61.6%, 3.0, 95.3ms, 0 | 20260506 13:34:23.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:23.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:23.477 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:34:23.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:23.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:23.477 | INFO | Uploaded 36GiB to GPU in 2.00567s (17.9491GiB/s) | 20260506 13:34:23.477 | INFO | Downloaded 34GiB to GPU in 1.94703s (17.4625GiB/s) | 20260506 13:34:28.616 | INFO | Analyzing chunk frames = [1750528,1751040) | 20260506 13:34:28.616 | INFO | MemInfo: { | 20260506 13:34:28.616 | INFO | "gpu" : " 17 GB", | 20260506 13:34:28.616 | INFO | "huge" : "", | 20260506 13:34:28.616 | INFO | "huge_shared" : " 27 GB", | 20260506 13:34:28.616 | INFO | "malloc" : " 14 GB", | 20260506 13:34:28.616 | INFO | "pinned" : " 29 GB" | 20260506 13:34:28.616 | INFO | } | 20260506 13:34:28.616 | INFO | Perf report for the last 5.13934 seconds: | 20260506 13:34:28.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:34:28.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:28.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:28.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:28.616 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:34:28.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:28.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:28.616 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:34:28.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:28.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:28.617 | INFO | Uploaded 36GiB to GPU in 1.99191s (18.0731GiB/s) | 20260506 13:34:28.617 | INFO | Downloaded 34GiB to GPU in 1.92436s (17.6682GiB/s) | 20260506 13:34:33.756 | INFO | Analyzing chunk frames = [1751040,1751552) | 20260506 13:34:33.756 | INFO | MemInfo: { | 20260506 13:34:33.756 | INFO | "gpu" : " 17 GB", | 20260506 13:34:33.756 | INFO | "huge" : "", | 20260506 13:34:33.756 | INFO | "huge_shared" : " 27 GB", | 20260506 13:34:33.756 | INFO | "malloc" : " 14 GB", | 20260506 13:34:33.756 | INFO | "pinned" : " 29 GB" | 20260506 13:34:33.756 | INFO | } | 20260506 13:34:33.756 | INFO | Perf report for the last 5.13948 seconds: | 20260506 13:34:33.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:34:33.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:33.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:33.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:33.756 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 13:34:33.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:33.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:33.756 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 13:34:33.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:33.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:33.756 | INFO | Uploaded 36GiB to GPU in 2.04717s (17.5852GiB/s) | 20260506 13:34:33.756 | INFO | Downloaded 34GiB to GPU in 1.95747s (17.3694GiB/s) | 20260506 13:34:38.895 | INFO | Analyzing chunk frames = [1751552,1752064) | 20260506 13:34:38.895 | INFO | MemInfo: { | 20260506 13:34:38.895 | INFO | "gpu" : " 17 GB", | 20260506 13:34:38.895 | INFO | "huge" : "", | 20260506 13:34:38.895 | INFO | "huge_shared" : " 27 GB", | 20260506 13:34:38.895 | INFO | "malloc" : " 14 GB", | 20260506 13:34:38.895 | INFO | "pinned" : " 29 GB" | 20260506 13:34:38.895 | INFO | } | 20260506 13:34:38.895 | INFO | Perf report for the last 5.13933 seconds: | 20260506 13:34:38.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:34:38.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:38.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:38.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:38.895 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 13:34:38.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:38.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:38.895 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:34:38.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:38.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:38.895 | INFO | Uploaded 36GiB to GPU in 2.0301s (17.7331GiB/s) | 20260506 13:34:38.896 | INFO | Downloaded 34GiB to GPU in 1.94137s (17.5134GiB/s) | 20260506 13:34:44.035 | INFO | Analyzing chunk frames = [1752064,1752576) | 20260506 13:34:44.035 | INFO | MemInfo: { | 20260506 13:34:44.035 | INFO | "gpu" : " 17 GB", | 20260506 13:34:44.035 | INFO | "huge" : "", | 20260506 13:34:44.035 | INFO | "huge_shared" : " 27 GB", | 20260506 13:34:44.035 | INFO | "malloc" : " 14 GB", | 20260506 13:34:44.035 | INFO | "pinned" : " 29 GB" | 20260506 13:34:44.035 | INFO | } | 20260506 13:34:44.035 | INFO | Perf report for the last 5.13963 seconds: | 20260506 13:34:44.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:34:44.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:44.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:44.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:44.035 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:34:44.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:44.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:44.035 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 13:34:44.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:44.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:44.035 | INFO | Uploaded 36GiB to GPU in 2.00649s (17.9418GiB/s) | 20260506 13:34:44.035 | INFO | Downloaded 34GiB to GPU in 1.98298s (17.1459GiB/s) | 20260506 13:34:49.174 | INFO | Analyzing chunk frames = [1752576,1753088) | 20260506 13:34:49.174 | INFO | MemInfo: { | 20260506 13:34:49.174 | INFO | "gpu" : " 17 GB", | 20260506 13:34:49.174 | INFO | "huge" : "", | 20260506 13:34:49.174 | INFO | "huge_shared" : " 27 GB", | 20260506 13:34:49.174 | INFO | "malloc" : " 14 GB", | 20260506 13:34:49.174 | INFO | "pinned" : " 29 GB" | 20260506 13:34:49.174 | INFO | } | 20260506 13:34:49.174 | INFO | Perf report for the last 5.13925 seconds: | 20260506 13:34:49.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:34:49.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:49.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:49.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:49.174 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:34:49.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:49.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:49.174 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:34:49.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:49.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:49.174 | INFO | Uploaded 36GiB to GPU in 1.99917s (18.0074GiB/s) | 20260506 13:34:49.174 | INFO | Downloaded 34GiB to GPU in 1.97022s (17.2569GiB/s) | 20260506 13:34:49.175 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:34:49.175 | INFO | Total mem usage: 17.4 GiB | 20260506 13:34:49.175 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:34:49.175 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -10.8%) | 20260506 13:34:49.175 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:34:49.175 | INFO | High Water Breakdown: | 20260506 13:34:49.175 | INFO | 61.2% : 9.62 GiB ( 88.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:34:49.175 | INFO | 38.7% : 6.07 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:34:49.175 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:34:49.175 | INFO | ------------------------------------------ | 20260506 13:34:49.175 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:34:49.175 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:34:49.175 | INFO | Total mem usage: 26.5 GiB | 20260506 13:34:49.175 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:34:49.175 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:34:49.175 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:34:49.175 | INFO | High Water Breakdown: | 20260506 13:34:49.175 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:34:49.175 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:34:49.175 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:34:49.175 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:34:49.175 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:34:49.175 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:34:49.175 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:34:49.175 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:34:49.175 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:34:49.175 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:34:49.175 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:34:49.175 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:34:49.175 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:34:49.175 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:34:49.175 | INFO | ------------------------------------------ | 20260506 13:34:49.175 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:34:49.175 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:34:49.175 | INFO | Total mem usage: 25.6 GiB | 20260506 13:34:49.175 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:34:49.175 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:34:49.175 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:34:49.175 | INFO | High Water Breakdown: | 20260506 13:34:49.175 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:34:49.175 | INFO | ------------------------------------------ | 20260506 13:34:49.175 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:34:49.175 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:34:49.175 | INFO | Total mem usage: 15.7 GiB | 20260506 13:34:49.175 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:34:49.175 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:34:49.175 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 13:34:49.175 | INFO | High Water Breakdown: | 20260506 13:34:49.175 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:34:49.175 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:34:49.175 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:34:49.175 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:34:49.175 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:34:49.175 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:34:49.175 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:34:49.175 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:34:49.175 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:34:49.175 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:34:49.175 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:34:49.175 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:34:49.175 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:34:49.175 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:34:49.175 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:34:49.175 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:34:49.175 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:34:49.175 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:34:49.175 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:34:49.175 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:34:49.175 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:34:49.175 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:34:49.175 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:34:49.175 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:34:49.175 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:34:49.175 | INFO | ------------------------------------------ | 20260506 13:34:49.175 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:34:49.175 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:34:49.175 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:34:49.175 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:34:49.175 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.59e+01, 1.35e+01 / 1.82e+00 | 20260506 13:34:49.175 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.04e-01, 1.06e-02 / 6.56e-03 | 20260506 13:34:49.175 | INFO | PerfReport: 11.57%, Baseline -- 3.67e+00 / 4.47e+00, 5.82e-01 / 2.97e-01 | 20260506 13:34:49.175 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.47e+00, 1.92e-01 / 1.33e-01 | 20260506 13:34:49.175 | INFO | PerfReport: 15.72%, DME -- 3.99e+01 / 4.37e+01, 3.59e+00 / 1.77e+00 | 20260506 13:34:49.175 | INFO | PerfReport: 56.80%, FrameLabeling -- 1.80e+01 / 2.12e+01, 1.61e+00 / 1.96e+00 | 20260506 13:34:49.175 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.36e+00 / 2.89e+00, 4.02e-01 / 3.08e-01 | 20260506 13:34:49.175 | INFO | PerfReport: 3.64%, Metrics -- 1.16e+00 / 1.52e+00, 2.93e-01 / 1.97e-01 | 20260506 13:34:49.175 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.07e-01 / 4.67e-01, 6.36e-02 / 3.50e-02 | 20260506 13:34:49.175 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:34:49.175 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:34:49.175 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:34:49.176 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 13:34:49.176 | INFO | PerfReport: 21.49%, Upload -- 2.11e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260506 13:34:49.176 | INFO | PerfReport: 21.09%, Download -- 2.07e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 13:34:49.176 | INFO | PerfReport: 32.30%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 13:34:49.176 | INFO | PerfReport: 1.64%, LockUpload -- 1.61e+00 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260506 13:34:49.176 | INFO | PerfReport: 1.12%, LockDownload -- 1.10e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260506 13:34:49.176 | INFO | PerfReport: 22.36%, LockCompute -- 2.19e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 13:34:49.176 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260506 13:34:49.300 | INFO | Starting a round of Baz IO | 20260506 13:34:52.460 | INFO | Populating RTMetrics report | 20260506 13:34:52.460 | INFO | RTMetrics report populated | 20260506 13:34:52.460 | INFO | Opening tmp RTMetrics json file | 20260506 13:34:52.467 | INFO | Serializing RTMetrics to json | 20260506 13:34:52.472 | INFO | Serialization done, swapping tmp file now... | 20260506 13:34:52.494 | INFO | RTMetrics done | 20260506 13:34:52.606 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.306 seconds | 20260506 13:34:52.606 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.51% / 0.02% / 60.48% | 20260506 13:34:52.606 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.06954% / 2.78155e-05% / 4.76837e-05% | 20260506 13:34:52.606 | INFO | Baz IO overall: | 20260506 13:34:52.606 | INFO | BytesWritten : 892645693457 | 20260506 13:34:52.606 | INFO | NumEvents : 431780140723 | 20260506 13:34:54.313 | INFO | Analyzing chunk frames = [1753088,1753600) | 20260506 13:34:54.313 | INFO | MemInfo: { | 20260506 13:34:54.313 | INFO | "gpu" : " 17 GB", | 20260506 13:34:54.313 | INFO | "huge" : "", | 20260506 13:34:54.313 | INFO | "huge_shared" : " 27 GB", | 20260506 13:34:54.313 | INFO | "malloc" : " 14 GB", | 20260506 13:34:54.313 | INFO | "pinned" : " 29 GB" | 20260506 13:34:54.313 | INFO | } | 20260506 13:34:54.313 | INFO | Perf report for the last 5.13904 seconds: | 20260506 13:34:54.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:34:54.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:54.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:54.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:54.313 | INFO | ANALYSIS : 63.3%, 2.9, 95.7ms, 0 | 20260506 13:34:54.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:54.313 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 13:34:54.313 | INFO | PRE_HQ : 61.3%, 1.9, 60.5ms, 0 | 20260506 13:34:54.313 | INFO | BAZWRITER : 63.7%, 4.6, 151.5ms, 0 | 20260506 13:34:54.313 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 13:34:54.313 | INFO | Uploaded 36GiB to GPU in 2.06605s (17.4246GiB/s) | 20260506 13:34:54.313 | INFO | Downloaded 38GiB to GPU in 2.24542s (16.9233GiB/s) | 20260506 13:34:59.453 | INFO | Analyzing chunk frames = [1753600,1754112) | 20260506 13:34:59.453 | INFO | MemInfo: { | 20260506 13:34:59.453 | INFO | "gpu" : " 17 GB", | 20260506 13:34:59.453 | INFO | "huge" : "", | 20260506 13:34:59.453 | INFO | "huge_shared" : " 27 GB", | 20260506 13:34:59.453 | INFO | "malloc" : " 14 GB", | 20260506 13:34:59.453 | INFO | "pinned" : " 29 GB" | 20260506 13:34:59.453 | INFO | } | 20260506 13:34:59.453 | INFO | Perf report for the last 5.13967 seconds: | 20260506 13:34:59.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:34:59.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:59.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:59.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:59.453 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 13:34:59.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:59.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:59.453 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:34:59.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:59.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:34:59.453 | INFO | Uploaded 36GiB to GPU in 1.98352s (18.1495GiB/s) | 20260506 13:34:59.453 | INFO | Downloaded 34GiB to GPU in 1.93923s (17.5327GiB/s) | 20260506 13:35:04.592 | INFO | Analyzing chunk frames = [1754112,1754624) | 20260506 13:35:04.592 | INFO | MemInfo: { | 20260506 13:35:04.592 | INFO | "gpu" : " 17 GB", | 20260506 13:35:04.592 | INFO | "huge" : "", | 20260506 13:35:04.592 | INFO | "huge_shared" : " 27 GB", | 20260506 13:35:04.592 | INFO | "malloc" : " 14 GB", | 20260506 13:35:04.592 | INFO | "pinned" : " 29 GB" | 20260506 13:35:04.592 | INFO | } | 20260506 13:35:04.592 | INFO | Perf report for the last 5.13943 seconds: | 20260506 13:35:04.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:35:04.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:04.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:04.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:04.592 | INFO | ANALYSIS : 61.4%, 3.0, 95.0ms, 0 | 20260506 13:35:04.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:04.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:04.592 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:35:04.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:04.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:04.593 | INFO | Uploaded 36GiB to GPU in 1.96939s (18.2797GiB/s) | 20260506 13:35:04.593 | INFO | Downloaded 34GiB to GPU in 1.93843s (17.54GiB/s) | 20260506 13:35:09.731 | INFO | Analyzing chunk frames = [1754624,1755136) | 20260506 13:35:09.731 | INFO | MemInfo: { | 20260506 13:35:09.731 | INFO | "gpu" : " 17 GB", | 20260506 13:35:09.731 | INFO | "huge" : "", | 20260506 13:35:09.731 | INFO | "huge_shared" : " 27 GB", | 20260506 13:35:09.731 | INFO | "malloc" : " 14 GB", | 20260506 13:35:09.731 | INFO | "pinned" : " 29 GB" | 20260506 13:35:09.731 | INFO | } | 20260506 13:35:09.731 | INFO | Perf report for the last 5.13884 seconds: | 20260506 13:35:09.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:35:09.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:09.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:09.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:09.731 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:35:09.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:09.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:09.731 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:35:09.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:09.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:09.731 | INFO | Uploaded 36GiB to GPU in 2.02064s (17.8161GiB/s) | 20260506 13:35:09.731 | INFO | Downloaded 34GiB to GPU in 1.91675s (17.7384GiB/s) | 20260506 13:35:14.870 | INFO | Analyzing chunk frames = [1755136,1755648) | 20260506 13:35:14.870 | INFO | MemInfo: { | 20260506 13:35:14.870 | INFO | "gpu" : " 17 GB", | 20260506 13:35:14.870 | INFO | "huge" : "", | 20260506 13:35:14.870 | INFO | "huge_shared" : " 27 GB", | 20260506 13:35:14.870 | INFO | "malloc" : " 14 GB", | 20260506 13:35:14.870 | INFO | "pinned" : " 29 GB" | 20260506 13:35:14.871 | INFO | } | 20260506 13:35:14.871 | INFO | Perf report for the last 5.13942 seconds: | 20260506 13:35:14.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:35:14.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:14.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:14.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:14.871 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 13:35:14.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:14.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:14.871 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:35:14.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:14.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:14.871 | INFO | Uploaded 36GiB to GPU in 1.98695s (18.1182GiB/s) | 20260506 13:35:14.871 | INFO | Downloaded 34GiB to GPU in 1.94095s (17.5172GiB/s) | 20260506 13:35:20.010 | INFO | Analyzing chunk frames = [1755648,1756160) | 20260506 13:35:20.010 | INFO | MemInfo: { | 20260506 13:35:20.010 | INFO | "gpu" : " 17 GB", | 20260506 13:35:20.010 | INFO | "huge" : "", | 20260506 13:35:20.010 | INFO | "huge_shared" : " 27 GB", | 20260506 13:35:20.010 | INFO | "malloc" : " 14 GB", | 20260506 13:35:20.010 | INFO | "pinned" : " 29 GB" | 20260506 13:35:20.010 | INFO | } | 20260506 13:35:20.010 | INFO | Perf report for the last 5.13915 seconds: | 20260506 13:35:20.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:35:20.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:20.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:20.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:20.010 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:35:20.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:20.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:20.010 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260506 13:35:20.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:20.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:20.010 | INFO | Uploaded 36GiB to GPU in 2.08768s (17.2441GiB/s) | 20260506 13:35:20.010 | INFO | Downloaded 34GiB to GPU in 1.96863s (17.2709GiB/s) | 20260506 13:35:25.150 | INFO | Analyzing chunk frames = [1756160,1756672) | 20260506 13:35:25.150 | INFO | MemInfo: { | 20260506 13:35:25.150 | INFO | "gpu" : " 17 GB", | 20260506 13:35:25.150 | INFO | "huge" : "", | 20260506 13:35:25.150 | INFO | "huge_shared" : " 27 GB", | 20260506 13:35:25.150 | INFO | "malloc" : " 14 GB", | 20260506 13:35:25.150 | INFO | "pinned" : " 29 GB" | 20260506 13:35:25.150 | INFO | } | 20260506 13:35:25.150 | INFO | Perf report for the last 5.14026 seconds: | 20260506 13:35:25.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:35:25.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:25.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:25.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:25.150 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:35:25.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:25.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:25.150 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 13:35:25.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:25.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:25.150 | INFO | Uploaded 36GiB to GPU in 1.98904s (18.0991GiB/s) | 20260506 13:35:25.150 | INFO | Downloaded 34GiB to GPU in 2.03466s (16.7104GiB/s) | 20260506 13:35:30.288 | INFO | Analyzing chunk frames = [1756672,1757184) | 20260506 13:35:30.288 | INFO | MemInfo: { | 20260506 13:35:30.288 | INFO | "gpu" : " 17 GB", | 20260506 13:35:30.288 | INFO | "huge" : "", | 20260506 13:35:30.288 | INFO | "huge_shared" : " 27 GB", | 20260506 13:35:30.288 | INFO | "malloc" : " 14 GB", | 20260506 13:35:30.288 | INFO | "pinned" : " 29 GB" | 20260506 13:35:30.288 | INFO | } | 20260506 13:35:30.289 | INFO | Perf report for the last 5.1385 seconds: | 20260506 13:35:30.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:35:30.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:30.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:30.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:30.289 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 13:35:30.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:30.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:30.289 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 13:35:30.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:30.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:30.289 | INFO | Uploaded 36GiB to GPU in 2.05942s (17.4806GiB/s) | 20260506 13:35:30.289 | INFO | Downloaded 34GiB to GPU in 1.99283s (17.0612GiB/s) | 20260506 13:35:30.420 | INFO | Starting a round of Baz IO | 20260506 13:35:33.577 | INFO | Populating RTMetrics report | 20260506 13:35:33.577 | INFO | RTMetrics report populated | 20260506 13:35:33.577 | INFO | Opening tmp RTMetrics json file | 20260506 13:35:33.715 | INFO | Serializing RTMetrics to json | 20260506 13:35:33.748 | INFO | Serialization done, swapping tmp file now... | 20260506 13:35:33.749 | INFO | RTMetrics done | 20260506 13:35:33.962 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.542 seconds | 20260506 13:35:33.962 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.49% / 0.02% / 60.49% | 20260506 13:35:33.962 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.07123% / 2.38419e-05% / 4.37101e-05% | 20260506 13:35:33.962 | INFO | Baz IO overall: | 20260506 13:35:33.962 | INFO | BytesWritten : 894911950732 | 20260506 13:35:33.962 | INFO | NumEvents : 432879674306 | 20260506 13:35:35.428 | INFO | Analyzing chunk frames = [1757184,1757696) | 20260506 13:35:35.428 | INFO | MemInfo: { | 20260506 13:35:35.428 | INFO | "gpu" : " 17 GB", | 20260506 13:35:35.428 | INFO | "huge" : "", | 20260506 13:35:35.428 | INFO | "huge_shared" : " 27 GB", | 20260506 13:35:35.428 | INFO | "malloc" : " 14 GB", | 20260506 13:35:35.428 | INFO | "pinned" : " 29 GB" | 20260506 13:35:35.428 | INFO | } | 20260506 13:35:35.428 | INFO | Perf report for the last 5.13942 seconds: | 20260506 13:35:35.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:35:35.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:35.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:35.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:35.428 | INFO | ANALYSIS : 63.4%, 2.9, 95.9ms, 0 | 20260506 13:35:35.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:35.428 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260506 13:35:35.428 | INFO | PRE_HQ : 61.2%, 1.9, 60.1ms, 0 | 20260506 13:35:35.428 | INFO | BAZWRITER : 68.3%, 4.8, 168.7ms, 0 | 20260506 13:35:35.428 | INFO | BAZ_SAMPLING : 3.1%, 1.0, 1.6ms, 0 | 20260506 13:35:35.428 | INFO | Uploaded 36GiB to GPU in 2.06814s (17.4069GiB/s) | 20260506 13:35:35.428 | INFO | Downloaded 38GiB to GPU in 2.25115s (16.8803GiB/s) | 20260506 13:35:40.567 | INFO | Analyzing chunk frames = [1757696,1758208) | 20260506 13:35:40.567 | INFO | MemInfo: { | 20260506 13:35:40.567 | INFO | "gpu" : " 17 GB", | 20260506 13:35:40.567 | INFO | "huge" : "", | 20260506 13:35:40.567 | INFO | "huge_shared" : " 27 GB", | 20260506 13:35:40.567 | INFO | "malloc" : " 14 GB", | 20260506 13:35:40.567 | INFO | "pinned" : " 29 GB" | 20260506 13:35:40.567 | INFO | } | 20260506 13:35:40.567 | INFO | Perf report for the last 5.1393 seconds: | 20260506 13:35:40.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:35:40.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:40.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:40.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:40.567 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 13:35:40.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:40.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:40.567 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:35:40.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:40.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:40.567 | INFO | Uploaded 36GiB to GPU in 2.00139s (17.9875GiB/s) | 20260506 13:35:40.568 | INFO | Downloaded 34GiB to GPU in 1.93664s (17.5562GiB/s) | 20260506 13:35:45.706 | INFO | Analyzing chunk frames = [1758208,1758720) | 20260506 13:35:45.706 | INFO | MemInfo: { | 20260506 13:35:45.706 | INFO | "gpu" : " 17 GB", | 20260506 13:35:45.706 | INFO | "huge" : "", | 20260506 13:35:45.706 | INFO | "huge_shared" : " 27 GB", | 20260506 13:35:45.706 | INFO | "malloc" : " 14 GB", | 20260506 13:35:45.707 | INFO | "pinned" : " 29 GB" | 20260506 13:35:45.707 | INFO | } | 20260506 13:35:45.707 | INFO | Perf report for the last 5.13939 seconds: | 20260506 13:35:45.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:35:45.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:45.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:45.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:45.707 | INFO | ANALYSIS : 61.4%, 3.0, 95.2ms, 0 | 20260506 13:35:45.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:45.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:45.707 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:35:45.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:45.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:45.707 | INFO | Uploaded 36GiB to GPU in 1.97745s (18.2053GiB/s) | 20260506 13:35:45.707 | INFO | Downloaded 34GiB to GPU in 1.93696s (17.5532GiB/s) | 20260506 13:35:50.846 | INFO | Analyzing chunk frames = [1758720,1759232) | 20260506 13:35:50.846 | INFO | MemInfo: { | 20260506 13:35:50.846 | INFO | "gpu" : " 17 GB", | 20260506 13:35:50.846 | INFO | "huge" : "", | 20260506 13:35:50.847 | INFO | "huge_shared" : " 27 GB", | 20260506 13:35:50.847 | INFO | "malloc" : " 14 GB", | 20260506 13:35:50.847 | INFO | "pinned" : " 29 GB" | 20260506 13:35:50.847 | INFO | } | 20260506 13:35:50.847 | INFO | Perf report for the last 5.13995 seconds: | 20260506 13:35:50.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:35:50.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:50.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:50.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:50.847 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 13:35:50.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:50.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:50.847 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:35:50.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:50.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:50.847 | INFO | Uploaded 36GiB to GPU in 1.99958s (18.0038GiB/s) | 20260506 13:35:50.847 | INFO | Downloaded 34GiB to GPU in 1.91402s (17.7637GiB/s) | 20260506 13:35:55.985 | INFO | Analyzing chunk frames = [1759232,1759744) | 20260506 13:35:55.985 | INFO | MemInfo: { | 20260506 13:35:55.985 | INFO | "gpu" : " 17 GB", | 20260506 13:35:55.985 | INFO | "huge" : "", | 20260506 13:35:55.985 | INFO | "huge_shared" : " 27 GB", | 20260506 13:35:55.985 | INFO | "malloc" : " 14 GB", | 20260506 13:35:55.985 | INFO | "pinned" : " 29 GB" | 20260506 13:35:55.985 | INFO | } | 20260506 13:35:55.985 | INFO | Perf report for the last 5.13859 seconds: | 20260506 13:35:55.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:35:55.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:55.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:55.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:55.985 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 13:35:55.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:55.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:55.985 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:35:55.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:55.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:35:55.986 | INFO | Uploaded 36GiB to GPU in 2.03426s (17.6968GiB/s) | 20260506 13:35:55.986 | INFO | Downloaded 34GiB to GPU in 1.94203s (17.5075GiB/s) | 20260506 13:36:01.124 | INFO | Analyzing chunk frames = [1759744,1760256) | 20260506 13:36:01.124 | INFO | MemInfo: { | 20260506 13:36:01.124 | INFO | "gpu" : " 17 GB", | 20260506 13:36:01.124 | INFO | "huge" : "", | 20260506 13:36:01.124 | INFO | "huge_shared" : " 27 GB", | 20260506 13:36:01.124 | INFO | "malloc" : " 14 GB", | 20260506 13:36:01.124 | INFO | "pinned" : " 29 GB" | 20260506 13:36:01.124 | INFO | } | 20260506 13:36:01.124 | INFO | Perf report for the last 5.13917 seconds: | 20260506 13:36:01.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:36:01.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:01.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:01.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:01.124 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 13:36:01.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:01.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:01.124 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260506 13:36:01.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:01.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:01.125 | INFO | Uploaded 36GiB to GPU in 2.04021s (17.6452GiB/s) | 20260506 13:36:01.125 | INFO | Downloaded 34GiB to GPU in 1.91327s (17.7706GiB/s) | 20260506 13:36:06.265 | INFO | Analyzing chunk frames = [1760256,1760768) | 20260506 13:36:06.265 | INFO | MemInfo: { | 20260506 13:36:06.265 | INFO | "gpu" : " 17 GB", | 20260506 13:36:06.265 | INFO | "huge" : "", | 20260506 13:36:06.265 | INFO | "huge_shared" : " 27 GB", | 20260506 13:36:06.265 | INFO | "malloc" : " 14 GB", | 20260506 13:36:06.265 | INFO | "pinned" : " 29 GB" | 20260506 13:36:06.265 | INFO | } | 20260506 13:36:06.265 | INFO | Perf report for the last 5.14071 seconds: | 20260506 13:36:06.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:36:06.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:06.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:06.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:06.265 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 13:36:06.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:06.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:06.265 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 13:36:06.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:06.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:06.265 | INFO | Uploaded 36GiB to GPU in 2.02198s (17.8043GiB/s) | 20260506 13:36:06.265 | INFO | Downloaded 34GiB to GPU in 2.00052s (16.9956GiB/s) | 20260506 13:36:11.403 | INFO | Analyzing chunk frames = [1760768,1761280) | 20260506 13:36:11.403 | INFO | MemInfo: { | 20260506 13:36:11.403 | INFO | "gpu" : " 17 GB", | 20260506 13:36:11.403 | INFO | "huge" : "", | 20260506 13:36:11.403 | INFO | "huge_shared" : " 27 GB", | 20260506 13:36:11.403 | INFO | "malloc" : " 14 GB", | 20260506 13:36:11.403 | INFO | "pinned" : " 29 GB" | 20260506 13:36:11.403 | INFO | } | 20260506 13:36:11.403 | INFO | Perf report for the last 5.13824 seconds: | 20260506 13:36:11.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:36:11.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:11.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:11.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:11.403 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 13:36:11.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:11.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:11.403 | INFO | PRE_HQ : 33.1%, 1.0, 17.4ms, 0 | 20260506 13:36:11.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:11.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:11.404 | INFO | Uploaded 36GiB to GPU in 2.05279s (17.5371GiB/s) | 20260506 13:36:11.404 | INFO | Downloaded 34GiB to GPU in 2.02288s (16.8077GiB/s) | 20260506 13:36:11.404 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:36:11.404 | INFO | Total mem usage: 17.4 GiB | 20260506 13:36:11.404 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:36:11.404 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -13.7%) | 20260506 13:36:11.404 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:36:11.404 | INFO | High Water Breakdown: | 20260506 13:36:11.404 | INFO | 60.7% : 9.3 GiB ( 85.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:36:11.404 | INFO | 39.1% : 5.99 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:36:11.404 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:36:11.404 | INFO | ------------------------------------------ | 20260506 13:36:11.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:36:11.404 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:36:11.404 | INFO | Total mem usage: 26.5 GiB | 20260506 13:36:11.404 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:36:11.404 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:36:11.404 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:36:11.404 | INFO | High Water Breakdown: | 20260506 13:36:11.404 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:36:11.404 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:36:11.404 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:36:11.404 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:36:11.404 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:36:11.404 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:36:11.404 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:36:11.404 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:36:11.404 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:36:11.404 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:36:11.404 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:36:11.404 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:36:11.404 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:36:11.404 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:36:11.404 | INFO | ------------------------------------------ | 20260506 13:36:11.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:36:11.404 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:36:11.404 | INFO | Total mem usage: 25.6 GiB | 20260506 13:36:11.404 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:36:11.404 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:36:11.404 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:36:11.404 | INFO | High Water Breakdown: | 20260506 13:36:11.404 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:36:11.404 | INFO | ------------------------------------------ | 20260506 13:36:11.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:36:11.404 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:36:11.404 | INFO | Total mem usage: 15.7 GiB | 20260506 13:36:11.404 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:36:11.404 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:36:11.404 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 13:36:11.404 | INFO | High Water Breakdown: | 20260506 13:36:11.404 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:36:11.404 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:36:11.404 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:36:11.404 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:36:11.404 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:36:11.404 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:36:11.404 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:36:11.404 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:36:11.404 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:36:11.404 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:36:11.404 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:36:11.404 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:36:11.404 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:36:11.404 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:36:11.404 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:36:11.404 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:36:11.404 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:36:11.404 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:36:11.404 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:36:11.404 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:36:11.404 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:36:11.404 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:36:11.404 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:36:11.404 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:36:11.404 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:36:11.404 | INFO | ------------------------------------------ | 20260506 13:36:11.405 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:36:11.405 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:36:11.405 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:36:11.405 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:36:11.405 | INFO | PerfReport: Full Execution -- 3.17e+01 / 7.60e+01, 1.36e+01 / 2.59e+00 | 20260506 13:36:11.405 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.01e-01, 1.04e-02 / 7.57e-03 | 20260506 13:36:11.405 | INFO | PerfReport: 11.59%, Baseline -- 3.67e+00 / 4.35e+00, 5.85e-01 / 4.49e-01 | 20260506 13:36:11.405 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.45e+00, 1.94e-01 / 1.58e-01 | 20260506 13:36:11.405 | INFO | PerfReport: 15.83%, DME -- 4.01e+01 / 4.43e+01, 4.10e+00 / 2.64e+00 | 20260506 13:36:11.405 | INFO | PerfReport: 56.58%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.48e+00 / 1.51e+00 | 20260506 13:36:11.405 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.37e+00 / 2.94e+00, 4.12e-01 / 2.24e-01 | 20260506 13:36:11.405 | INFO | PerfReport: 3.66%, Metrics -- 1.16e+00 / 1.55e+00, 2.98e-01 / 1.62e-01 | 20260506 13:36:11.405 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.07e-01 / 4.65e-01, 6.35e-02 / 3.24e-02 | 20260506 13:36:11.405 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:36:11.405 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:36:11.405 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:36:11.405 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 13:36:11.405 | INFO | PerfReport: 21.47%, Upload -- 2.11e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 13:36:11.405 | INFO | PerfReport: 21.16%, Download -- 2.08e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 13:36:11.405 | INFO | PerfReport: 32.29%, Compute -- 3.17e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 13:36:11.405 | INFO | PerfReport: 1.58%, LockUpload -- 1.55e+00 / 0.00e+00, 6.59e+00 / 0.00e+00 | 20260506 13:36:11.405 | INFO | PerfReport: 1.20%, LockDownload -- 1.18e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 13:36:11.405 | INFO | PerfReport: 22.29%, LockCompute -- 2.19e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 13:36:11.405 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260506 13:36:11.535 | INFO | Starting a round of Baz IO | 20260506 13:36:14.691 | INFO | Populating RTMetrics report | 20260506 13:36:14.691 | INFO | RTMetrics report populated | 20260506 13:36:14.691 | INFO | Opening tmp RTMetrics json file | 20260506 13:36:14.692 | INFO | Serializing RTMetrics to json | 20260506 13:36:14.693 | INFO | Serialization done, swapping tmp file now... | 20260506 13:36:14.694 | INFO | RTMetrics done | 20260506 13:36:14.791 | INFO | Finished a round of Baz IO: Wrote 2.108 GiB in 3.255 seconds | 20260506 13:36:14.791 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.47% / 0.02% / 60.51% | 20260506 13:36:14.791 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.05367% / 2.78155e-05% / 5.16574e-05% | 20260506 13:36:14.791 | INFO | Baz IO overall: | 20260506 13:36:14.791 | INFO | BytesWritten : 897175912722 | 20260506 13:36:14.791 | INFO | NumEvents : 433977346206 | 20260506 13:36:16.542 | INFO | Analyzing chunk frames = [1761280,1761792) | 20260506 13:36:16.542 | INFO | MemInfo: { | 20260506 13:36:16.542 | INFO | "gpu" : " 17 GB", | 20260506 13:36:16.543 | INFO | "huge" : "", | 20260506 13:36:16.543 | INFO | "huge_shared" : " 27 GB", | 20260506 13:36:16.543 | INFO | "malloc" : " 14 GB", | 20260506 13:36:16.543 | INFO | "pinned" : " 29 GB" | 20260506 13:36:16.543 | INFO | } | 20260506 13:36:16.543 | INFO | Perf report for the last 5.13927 seconds: | 20260506 13:36:16.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:36:16.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:16.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:16.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:16.543 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260506 13:36:16.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:16.543 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 13:36:16.543 | INFO | PRE_HQ : 61.4%, 1.9, 61.2ms, 0 | 20260506 13:36:16.543 | INFO | BAZWRITER : 62.7%, 3.8, 122.0ms, 0 | 20260506 13:36:16.543 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 13:36:16.543 | INFO | Uploaded 36GiB to GPU in 2.05445s (17.5229GiB/s) | 20260506 13:36:16.543 | INFO | Downloaded 38GiB to GPU in 2.21114s (17.1857GiB/s) | 20260506 13:36:21.682 | INFO | Analyzing chunk frames = [1761792,1762304) | 20260506 13:36:21.682 | INFO | MemInfo: { | 20260506 13:36:21.682 | INFO | "gpu" : " 17 GB", | 20260506 13:36:21.682 | INFO | "huge" : "", | 20260506 13:36:21.682 | INFO | "huge_shared" : " 27 GB", | 20260506 13:36:21.682 | INFO | "malloc" : " 14 GB", | 20260506 13:36:21.682 | INFO | "pinned" : " 29 GB" | 20260506 13:36:21.682 | INFO | } | 20260506 13:36:21.682 | INFO | Perf report for the last 5.13939 seconds: | 20260506 13:36:21.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:36:21.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:21.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:21.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:21.682 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:36:21.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:21.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:21.682 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 13:36:21.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:21.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:21.682 | INFO | Uploaded 36GiB to GPU in 2.0245s (17.7822GiB/s) | 20260506 13:36:21.682 | INFO | Downloaded 34GiB to GPU in 1.93756s (17.5479GiB/s) | 20260506 13:36:26.821 | INFO | Analyzing chunk frames = [1762304,1762816) | 20260506 13:36:26.821 | INFO | MemInfo: { | 20260506 13:36:26.821 | INFO | "gpu" : " 17 GB", | 20260506 13:36:26.821 | INFO | "huge" : "", | 20260506 13:36:26.821 | INFO | "huge_shared" : " 27 GB", | 20260506 13:36:26.821 | INFO | "malloc" : " 14 GB", | 20260506 13:36:26.821 | INFO | "pinned" : " 29 GB" | 20260506 13:36:26.821 | INFO | } | 20260506 13:36:26.821 | INFO | Perf report for the last 5.13925 seconds: | 20260506 13:36:26.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:36:26.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:26.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:26.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:26.821 | INFO | ANALYSIS : 61.5%, 3.0, 95.3ms, 0 | 20260506 13:36:26.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:26.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:26.821 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:36:26.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:26.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:26.822 | INFO | Uploaded 36GiB to GPU in 2.0212s (17.8112GiB/s) | 20260506 13:36:26.822 | INFO | Downloaded 34GiB to GPU in 1.93924s (17.5327GiB/s) | 20260506 13:36:31.960 | INFO | Analyzing chunk frames = [1762816,1763328) | 20260506 13:36:31.960 | INFO | MemInfo: { | 20260506 13:36:31.960 | INFO | "gpu" : " 17 GB", | 20260506 13:36:31.960 | INFO | "huge" : "", | 20260506 13:36:31.960 | INFO | "huge_shared" : " 27 GB", | 20260506 13:36:31.960 | INFO | "malloc" : " 14 GB", | 20260506 13:36:31.960 | INFO | "pinned" : " 29 GB" | 20260506 13:36:31.960 | INFO | } | 20260506 13:36:31.960 | INFO | Perf report for the last 5.13916 seconds: | 20260506 13:36:31.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:36:31.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:31.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:31.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:31.961 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:36:31.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:31.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:31.961 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:36:31.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:31.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:31.961 | INFO | Uploaded 36GiB to GPU in 1.98762s (18.1122GiB/s) | 20260506 13:36:31.961 | INFO | Downloaded 34GiB to GPU in 1.91847s (17.7225GiB/s) | 20260506 13:36:37.100 | INFO | Analyzing chunk frames = [1763328,1763840) | 20260506 13:36:37.100 | INFO | MemInfo: { | 20260506 13:36:37.100 | INFO | "gpu" : " 17 GB", | 20260506 13:36:37.100 | INFO | "huge" : "", | 20260506 13:36:37.100 | INFO | "huge_shared" : " 27 GB", | 20260506 13:36:37.100 | INFO | "malloc" : " 14 GB", | 20260506 13:36:37.100 | INFO | "pinned" : " 29 GB" | 20260506 13:36:37.100 | INFO | } | 20260506 13:36:37.100 | INFO | Perf report for the last 5.13947 seconds: | 20260506 13:36:37.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:36:37.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:37.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:37.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:37.100 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 13:36:37.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:37.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:37.100 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:36:37.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:37.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:37.100 | INFO | Uploaded 36GiB to GPU in 2.0496s (17.5644GiB/s) | 20260506 13:36:37.100 | INFO | Downloaded 34GiB to GPU in 1.94337s (17.4954GiB/s) | 20260506 13:36:42.239 | INFO | Analyzing chunk frames = [1763840,1764352) | 20260506 13:36:42.239 | INFO | MemInfo: { | 20260506 13:36:42.239 | INFO | "gpu" : " 17 GB", | 20260506 13:36:42.239 | INFO | "huge" : "", | 20260506 13:36:42.239 | INFO | "huge_shared" : " 27 GB", | 20260506 13:36:42.239 | INFO | "malloc" : " 14 GB", | 20260506 13:36:42.239 | INFO | "pinned" : " 29 GB" | 20260506 13:36:42.239 | INFO | } | 20260506 13:36:42.239 | INFO | Perf report for the last 5.13923 seconds: | 20260506 13:36:42.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:36:42.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:42.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:42.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:42.240 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 13:36:42.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:42.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:42.240 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:36:42.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:42.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:42.240 | INFO | Uploaded 36GiB to GPU in 2.06741s (17.4131GiB/s) | 20260506 13:36:42.240 | INFO | Downloaded 34GiB to GPU in 1.92006s (17.7078GiB/s) | 20260506 13:36:47.379 | INFO | Analyzing chunk frames = [1764352,1764864) | 20260506 13:36:47.379 | INFO | MemInfo: { | 20260506 13:36:47.379 | INFO | "gpu" : " 17 GB", | 20260506 13:36:47.379 | INFO | "huge" : "", | 20260506 13:36:47.379 | INFO | "huge_shared" : " 27 GB", | 20260506 13:36:47.379 | INFO | "malloc" : " 14 GB", | 20260506 13:36:47.379 | INFO | "pinned" : " 29 GB" | 20260506 13:36:47.379 | INFO | } | 20260506 13:36:47.379 | INFO | Perf report for the last 5.14024 seconds: | 20260506 13:36:47.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:36:47.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:47.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:47.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:47.379 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:36:47.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:47.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:47.379 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 13:36:47.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:47.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:47.380 | INFO | Uploaded 36GiB to GPU in 2.00635s (17.943GiB/s) | 20260506 13:36:47.380 | INFO | Downloaded 34GiB to GPU in 1.98742s (17.1076GiB/s) | 20260506 13:36:52.518 | INFO | Analyzing chunk frames = [1764864,1765376) | 20260506 13:36:52.518 | INFO | MemInfo: { | 20260506 13:36:52.518 | INFO | "gpu" : " 17 GB", | 20260506 13:36:52.518 | INFO | "huge" : "", | 20260506 13:36:52.518 | INFO | "huge_shared" : " 27 GB", | 20260506 13:36:52.518 | INFO | "malloc" : " 14 GB", | 20260506 13:36:52.518 | INFO | "pinned" : " 29 GB" | 20260506 13:36:52.518 | INFO | } | 20260506 13:36:52.518 | INFO | Perf report for the last 5.13837 seconds: | 20260506 13:36:52.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:36:52.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:52.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:52.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:52.518 | INFO | ANALYSIS : 62.7%, 3.0, 97.7ms, 0 | 20260506 13:36:52.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:52.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:52.518 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:36:52.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:52.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:52.518 | INFO | Uploaded 36GiB to GPU in 2.08124s (17.2974GiB/s) | 20260506 13:36:52.518 | INFO | Downloaded 34GiB to GPU in 1.96909s (17.2669GiB/s) | 20260506 13:36:52.650 | INFO | Starting a round of Baz IO | 20260506 13:36:55.804 | INFO | Populating RTMetrics report | 20260506 13:36:55.805 | INFO | RTMetrics report populated | 20260506 13:36:55.805 | INFO | Opening tmp RTMetrics json file | 20260506 13:36:55.805 | INFO | Serializing RTMetrics to json | 20260506 13:36:55.806 | INFO | Serialization done, swapping tmp file now... | 20260506 13:36:55.807 | INFO | RTMetrics done | 20260506 13:36:55.906 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.256 seconds | 20260506 13:36:55.906 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.45% / 0.02% / 60.53% | 20260506 13:36:55.906 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.04825% / 4.76837e-05% / 2.78155e-05% | 20260506 13:36:55.906 | INFO | Baz IO overall: | 20260506 13:36:55.906 | INFO | BytesWritten : 899440554956 | 20260506 13:36:55.906 | INFO | NumEvents : 435075036339 | 20260506 13:36:57.657 | INFO | Analyzing chunk frames = [1765376,1765888) | 20260506 13:36:57.657 | INFO | MemInfo: { | 20260506 13:36:57.657 | INFO | "gpu" : " 17 GB", | 20260506 13:36:57.657 | INFO | "huge" : "", | 20260506 13:36:57.657 | INFO | "huge_shared" : " 27 GB", | 20260506 13:36:57.657 | INFO | "malloc" : " 14 GB", | 20260506 13:36:57.657 | INFO | "pinned" : " 29 GB" | 20260506 13:36:57.657 | INFO | } | 20260506 13:36:57.657 | INFO | Perf report for the last 5.13934 seconds: | 20260506 13:36:57.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:36:57.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:57.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:57.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:57.657 | INFO | ANALYSIS : 63.4%, 3.0, 98.3ms, 0 | 20260506 13:36:57.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:36:57.657 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:36:57.657 | INFO | PRE_HQ : 61.1%, 1.9, 60.4ms, 0 | 20260506 13:36:57.657 | INFO | BAZWRITER : 62.8%, 3.7, 119.8ms, 0 | 20260506 13:36:57.657 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 13:36:57.657 | INFO | Uploaded 36GiB to GPU in 2.07492s (17.35GiB/s) | 20260506 13:36:57.658 | INFO | Downloaded 38GiB to GPU in 2.21391s (17.1642GiB/s) | 20260506 13:37:02.796 | INFO | Analyzing chunk frames = [1765888,1766400) | 20260506 13:37:02.796 | INFO | MemInfo: { | 20260506 13:37:02.796 | INFO | "gpu" : " 17 GB", | 20260506 13:37:02.797 | INFO | "huge" : "", | 20260506 13:37:02.797 | INFO | "huge_shared" : " 27 GB", | 20260506 13:37:02.797 | INFO | "malloc" : " 14 GB", | 20260506 13:37:02.797 | INFO | "pinned" : " 29 GB" | 20260506 13:37:02.797 | INFO | } | 20260506 13:37:02.797 | INFO | Perf report for the last 5.13941 seconds: | 20260506 13:37:02.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:37:02.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:02.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:02.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:02.797 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 13:37:02.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:02.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:02.797 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:37:02.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:02.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:02.797 | INFO | Uploaded 36GiB to GPU in 2.06996s (17.3917GiB/s) | 20260506 13:37:02.797 | INFO | Downloaded 34GiB to GPU in 1.94726s (17.4604GiB/s) | 20260506 13:37:07.936 | INFO | Analyzing chunk frames = [1766400,1766912) | 20260506 13:37:07.936 | INFO | MemInfo: { | 20260506 13:37:07.936 | INFO | "gpu" : " 17 GB", | 20260506 13:37:07.936 | INFO | "huge" : "", | 20260506 13:37:07.936 | INFO | "huge_shared" : " 27 GB", | 20260506 13:37:07.936 | INFO | "malloc" : " 14 GB", | 20260506 13:37:07.936 | INFO | "pinned" : " 29 GB" | 20260506 13:37:07.936 | INFO | } | 20260506 13:37:07.936 | INFO | Perf report for the last 5.13932 seconds: | 20260506 13:37:07.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:37:07.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:07.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:07.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:07.936 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260506 13:37:07.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:07.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:07.936 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:37:07.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:07.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:07.936 | INFO | Uploaded 36GiB to GPU in 2.03945s (17.6518GiB/s) | 20260506 13:37:07.936 | INFO | Downloaded 34GiB to GPU in 1.94295s (17.4992GiB/s) | 20260506 13:37:13.075 | INFO | Analyzing chunk frames = [1766912,1767424) | 20260506 13:37:13.075 | INFO | MemInfo: { | 20260506 13:37:13.075 | INFO | "gpu" : " 17 GB", | 20260506 13:37:13.075 | INFO | "huge" : "", | 20260506 13:37:13.075 | INFO | "huge_shared" : " 27 GB", | 20260506 13:37:13.075 | INFO | "malloc" : " 14 GB", | 20260506 13:37:13.075 | INFO | "pinned" : " 29 GB" | 20260506 13:37:13.075 | INFO | } | 20260506 13:37:13.075 | INFO | Perf report for the last 5.13915 seconds: | 20260506 13:37:13.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:37:13.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:13.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:13.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:13.075 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 13:37:13.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:13.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:13.075 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:37:13.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:13.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:13.075 | INFO | Uploaded 36GiB to GPU in 2.00754s (17.9324GiB/s) | 20260506 13:37:13.075 | INFO | Downloaded 34GiB to GPU in 1.93625s (17.5597GiB/s) | 20260506 13:37:18.214 | INFO | Analyzing chunk frames = [1767424,1767936) | 20260506 13:37:18.214 | INFO | MemInfo: { | 20260506 13:37:18.214 | INFO | "gpu" : " 17 GB", | 20260506 13:37:18.214 | INFO | "huge" : "", | 20260506 13:37:18.214 | INFO | "huge_shared" : " 27 GB", | 20260506 13:37:18.214 | INFO | "malloc" : " 14 GB", | 20260506 13:37:18.214 | INFO | "pinned" : " 29 GB" | 20260506 13:37:18.215 | INFO | } | 20260506 13:37:18.215 | INFO | Perf report for the last 5.13943 seconds: | 20260506 13:37:18.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:37:18.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:18.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:18.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:18.215 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 13:37:18.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:18.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:18.215 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:37:18.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:18.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:18.215 | INFO | Uploaded 36GiB to GPU in 2.02292s (17.796GiB/s) | 20260506 13:37:18.215 | INFO | Downloaded 34GiB to GPU in 1.9482s (17.452GiB/s) | 20260506 13:37:23.354 | INFO | Analyzing chunk frames = [1767936,1768448) | 20260506 13:37:23.354 | INFO | MemInfo: { | 20260506 13:37:23.354 | INFO | "gpu" : " 17 GB", | 20260506 13:37:23.354 | INFO | "huge" : "", | 20260506 13:37:23.354 | INFO | "huge_shared" : " 27 GB", | 20260506 13:37:23.354 | INFO | "malloc" : " 14 GB", | 20260506 13:37:23.354 | INFO | "pinned" : " 29 GB" | 20260506 13:37:23.354 | INFO | } | 20260506 13:37:23.354 | INFO | Perf report for the last 5.13933 seconds: | 20260506 13:37:23.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:37:23.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:23.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:23.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:23.354 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:37:23.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:23.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:23.354 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 13:37:23.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:23.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:23.354 | INFO | Uploaded 36GiB to GPU in 2.09544s (17.1802GiB/s) | 20260506 13:37:23.354 | INFO | Downloaded 34GiB to GPU in 1.9805s (17.1674GiB/s) | 20260506 13:37:28.493 | INFO | Analyzing chunk frames = [1768448,1768960) | 20260506 13:37:28.493 | INFO | MemInfo: { | 20260506 13:37:28.493 | INFO | "gpu" : " 17 GB", | 20260506 13:37:28.493 | INFO | "huge" : "", | 20260506 13:37:28.493 | INFO | "huge_shared" : " 27 GB", | 20260506 13:37:28.493 | INFO | "malloc" : " 14 GB", | 20260506 13:37:28.493 | INFO | "pinned" : " 29 GB" | 20260506 13:37:28.493 | INFO | } | 20260506 13:37:28.493 | INFO | Perf report for the last 5.13929 seconds: | 20260506 13:37:28.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:37:28.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:28.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:28.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:28.493 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:37:28.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:28.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:28.493 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 13:37:28.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:28.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:28.493 | INFO | Uploaded 36GiB to GPU in 2.03647s (17.6776GiB/s) | 20260506 13:37:28.493 | INFO | Downloaded 34GiB to GPU in 2.01154s (16.9024GiB/s) | 20260506 13:37:33.632 | INFO | Analyzing chunk frames = [1768960,1769472) | 20260506 13:37:33.632 | INFO | MemInfo: { | 20260506 13:37:33.632 | INFO | "gpu" : " 17 GB", | 20260506 13:37:33.632 | INFO | "huge" : "", | 20260506 13:37:33.632 | INFO | "huge_shared" : " 27 GB", | 20260506 13:37:33.632 | INFO | "malloc" : " 14 GB", | 20260506 13:37:33.632 | INFO | "pinned" : " 29 GB" | 20260506 13:37:33.632 | INFO | } | 20260506 13:37:33.633 | INFO | Perf report for the last 5.13934 seconds: | 20260506 13:37:33.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:37:33.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:33.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:33.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:33.633 | INFO | ANALYSIS : 62.6%, 3.0, 96.5ms, 0 | 20260506 13:37:33.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:33.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:33.633 | INFO | PRE_HQ : 28.8%, 1.0, 14.8ms, 0 | 20260506 13:37:33.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:33.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:33.633 | INFO | Uploaded 36GiB to GPU in 2.05558s (17.5133GiB/s) | 20260506 13:37:33.633 | INFO | Downloaded 34GiB to GPU in 1.97769s (17.1918GiB/s) | 20260506 13:37:33.633 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:37:33.633 | INFO | Total mem usage: 17.4 GiB | 20260506 13:37:33.633 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:37:33.633 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -20.3%) | 20260506 13:37:33.633 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:37:33.633 | INFO | High Water Breakdown: | 20260506 13:37:33.633 | INFO | 59.9% : 8.67 GiB ( 79.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:37:33.633 | INFO | 40% : 5.79 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:37:33.633 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:37:33.633 | INFO | ------------------------------------------ | 20260506 13:37:33.633 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:37:33.633 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:37:33.633 | INFO | Total mem usage: 26.5 GiB | 20260506 13:37:33.633 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:37:33.633 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:37:33.633 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:37:33.633 | INFO | High Water Breakdown: | 20260506 13:37:33.633 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:37:33.633 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:37:33.633 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:37:33.633 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:37:33.633 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:37:33.633 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:37:33.633 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:37:33.633 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:37:33.633 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:37:33.633 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:37:33.633 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:37:33.633 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:37:33.633 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:37:33.633 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:37:33.633 | INFO | ------------------------------------------ | 20260506 13:37:33.633 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:37:33.633 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:37:33.633 | INFO | Total mem usage: 25.6 GiB | 20260506 13:37:33.633 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:37:33.633 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:37:33.633 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:37:33.633 | INFO | High Water Breakdown: | 20260506 13:37:33.633 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:37:33.633 | INFO | ------------------------------------------ | 20260506 13:37:33.633 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:37:33.633 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:37:33.633 | INFO | Total mem usage: 15.7 GiB | 20260506 13:37:33.633 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:37:33.633 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:37:33.633 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:37:33.633 | INFO | High Water Breakdown: | 20260506 13:37:33.633 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:37:33.633 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:37:33.633 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:37:33.633 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:37:33.633 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:37:33.634 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:37:33.634 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:37:33.634 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:37:33.634 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:37:33.634 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:37:33.634 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:37:33.634 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:37:33.634 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:37:33.634 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:37:33.634 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:37:33.634 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:37:33.634 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:37:33.634 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:37:33.634 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:37:33.634 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:37:33.634 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:37:33.634 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:37:33.634 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:37:33.634 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:37:33.634 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:37:33.634 | INFO | ------------------------------------------ | 20260506 13:37:33.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:37:33.634 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:37:33.634 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:37:33.634 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:37:33.634 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.57e+01, 1.35e+01 / 2.11e+00 | 20260506 13:37:33.634 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.02e-01, 1.07e-02 / 6.11e-03 | 20260506 13:37:33.634 | INFO | PerfReport: 11.58%, Baseline -- 3.66e+00 / 4.44e+00, 5.73e-01 / 3.36e-01 | 20260506 13:37:33.634 | INFO | PerfReport: 3.24%, Histogram -- 1.03e+00 / 1.46e+00, 1.90e-01 / 1.33e-01 | 20260506 13:37:33.634 | INFO | PerfReport: 15.82%, DME -- 4.00e+01 / 4.41e+01, 3.90e+00 / 2.41e+00 | 20260506 13:37:33.634 | INFO | PerfReport: 56.61%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.44e+00 / 1.88e+00 | 20260506 13:37:33.634 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.38e+00 / 3.02e+00, 4.20e-01 / 2.28e-01 | 20260506 13:37:33.634 | INFO | PerfReport: 3.66%, Metrics -- 1.16e+00 / 1.54e+00, 2.95e-01 / 1.76e-01 | 20260506 13:37:33.634 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.06e-01 / 4.65e-01, 6.26e-02 / 3.48e-02 | 20260506 13:37:33.634 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:37:33.634 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:37:33.634 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:37:33.634 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 13:37:33.634 | INFO | PerfReport: 21.58%, Upload -- 2.13e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 13:37:33.634 | INFO | PerfReport: 20.98%, Download -- 2.07e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 13:37:33.634 | INFO | PerfReport: 32.07%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 13:37:33.634 | INFO | PerfReport: 1.84%, LockUpload -- 1.81e+00 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260506 13:37:33.634 | INFO | PerfReport: 0.99%, LockDownload -- 9.80e-01 / 0.00e+00, 2.57e+00 / 0.00e+00 | 20260506 13:37:33.634 | INFO | PerfReport: 22.51%, LockCompute -- 2.22e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 13:37:33.634 | INFO | PerfReport: 0.02%, Misc -- 1.58e-02 / 0.00e+00, 2.51e-02 / 0.00e+00 | 20260506 13:37:33.766 | INFO | Starting a round of Baz IO | 20260506 13:37:36.929 | INFO | Populating RTMetrics report | 20260506 13:37:36.929 | INFO | RTMetrics report populated | 20260506 13:37:36.929 | INFO | Opening tmp RTMetrics json file | 20260506 13:37:36.930 | INFO | Serializing RTMetrics to json | 20260506 13:37:36.933 | INFO | Serialization done, swapping tmp file now... | 20260506 13:37:36.934 | INFO | RTMetrics done | 20260506 13:37:37.028 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.262 seconds | 20260506 13:37:37.028 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.43% / 0.02% / 60.55% | 20260506 13:37:37.028 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.04281% / 1.98682e-05% / 6.35783e-05% | 20260506 13:37:37.028 | INFO | Baz IO overall: | 20260506 13:37:37.028 | INFO | BytesWritten : 901705511323 | 20260506 13:37:37.028 | INFO | NumEvents : 436172717228 | 20260506 13:37:38.772 | INFO | Analyzing chunk frames = [1769472,1769984) | 20260506 13:37:38.772 | INFO | MemInfo: { | 20260506 13:37:38.772 | INFO | "gpu" : " 17 GB", | 20260506 13:37:38.772 | INFO | "huge" : "", | 20260506 13:37:38.772 | INFO | "huge_shared" : " 27 GB", | 20260506 13:37:38.772 | INFO | "malloc" : " 14 GB", | 20260506 13:37:38.772 | INFO | "pinned" : " 29 GB" | 20260506 13:37:38.772 | INFO | } | 20260506 13:37:38.772 | INFO | Perf report for the last 5.13933 seconds: | 20260506 13:37:38.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:37:38.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:38.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:38.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:38.772 | INFO | ANALYSIS : 63.5%, 3.0, 98.3ms, 0 | 20260506 13:37:38.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:38.772 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 13:37:38.772 | INFO | PRE_HQ : 61.2%, 1.9, 60.6ms, 0 | 20260506 13:37:38.772 | INFO | BAZWRITER : 62.9%, 3.8, 123.4ms, 0 | 20260506 13:37:38.772 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 13:37:38.772 | INFO | Uploaded 36GiB to GPU in 2.059s (17.4842GiB/s) | 20260506 13:37:38.772 | INFO | Downloaded 38GiB to GPU in 2.22298s (17.0941GiB/s) | 20260506 13:37:43.911 | INFO | Analyzing chunk frames = [1769984,1770496) | 20260506 13:37:43.911 | INFO | MemInfo: { | 20260506 13:37:43.911 | INFO | "gpu" : " 17 GB", | 20260506 13:37:43.911 | INFO | "huge" : "", | 20260506 13:37:43.911 | INFO | "huge_shared" : " 27 GB", | 20260506 13:37:43.911 | INFO | "malloc" : " 14 GB", | 20260506 13:37:43.911 | INFO | "pinned" : " 29 GB" | 20260506 13:37:43.911 | INFO | } | 20260506 13:37:43.911 | INFO | Perf report for the last 5.13924 seconds: | 20260506 13:37:43.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:37:43.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:43.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:43.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:43.911 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:37:43.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:43.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:43.911 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:37:43.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:43.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:43.911 | INFO | Uploaded 36GiB to GPU in 2.04492s (17.6046GiB/s) | 20260506 13:37:43.911 | INFO | Downloaded 34GiB to GPU in 1.94566s (17.4748GiB/s) | 20260506 13:37:49.050 | INFO | Analyzing chunk frames = [1770496,1771008) | 20260506 13:37:49.051 | INFO | MemInfo: { | 20260506 13:37:49.051 | INFO | "gpu" : " 17 GB", | 20260506 13:37:49.051 | INFO | "huge" : "", | 20260506 13:37:49.051 | INFO | "huge_shared" : " 27 GB", | 20260506 13:37:49.051 | INFO | "malloc" : " 14 GB", | 20260506 13:37:49.051 | INFO | "pinned" : " 29 GB" | 20260506 13:37:49.051 | INFO | } | 20260506 13:37:49.051 | INFO | Perf report for the last 5.13955 seconds: | 20260506 13:37:49.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:37:49.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:49.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:49.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:49.051 | INFO | ANALYSIS : 61.7%, 3.0, 95.5ms, 0 | 20260506 13:37:49.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:49.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:49.051 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:37:49.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:49.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:49.051 | INFO | Uploaded 36GiB to GPU in 2.03082s (17.7269GiB/s) | 20260506 13:37:49.051 | INFO | Downloaded 34GiB to GPU in 1.94158s (17.5115GiB/s) | 20260506 13:37:54.190 | INFO | Analyzing chunk frames = [1771008,1771520) | 20260506 13:37:54.190 | INFO | MemInfo: { | 20260506 13:37:54.190 | INFO | "gpu" : " 17 GB", | 20260506 13:37:54.190 | INFO | "huge" : "", | 20260506 13:37:54.190 | INFO | "huge_shared" : " 27 GB", | 20260506 13:37:54.190 | INFO | "malloc" : " 14 GB", | 20260506 13:37:54.190 | INFO | "pinned" : " 29 GB" | 20260506 13:37:54.190 | INFO | } | 20260506 13:37:54.190 | INFO | Perf report for the last 5.13945 seconds: | 20260506 13:37:54.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:37:54.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:54.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:54.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:54.190 | INFO | ANALYSIS : 61.4%, 2.9, 94.8ms, 0 | 20260506 13:37:54.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:54.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:54.190 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:37:54.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:54.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:54.190 | INFO | Uploaded 36GiB to GPU in 2.01498s (17.8662GiB/s) | 20260506 13:37:54.190 | INFO | Downloaded 34GiB to GPU in 1.92484s (17.6638GiB/s) | 20260506 13:37:59.329 | INFO | Analyzing chunk frames = [1771520,1772032) | 20260506 13:37:59.329 | INFO | MemInfo: { | 20260506 13:37:59.329 | INFO | "gpu" : " 17 GB", | 20260506 13:37:59.329 | INFO | "huge" : "", | 20260506 13:37:59.329 | INFO | "huge_shared" : " 27 GB", | 20260506 13:37:59.329 | INFO | "malloc" : " 14 GB", | 20260506 13:37:59.329 | INFO | "pinned" : " 29 GB" | 20260506 13:37:59.329 | INFO | } | 20260506 13:37:59.330 | INFO | Perf report for the last 5.13936 seconds: | 20260506 13:37:59.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:37:59.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:59.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:59.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:59.330 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 13:37:59.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:59.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:59.330 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:37:59.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:59.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:37:59.330 | INFO | Uploaded 36GiB to GPU in 2.00078s (17.993GiB/s) | 20260506 13:37:59.330 | INFO | Downloaded 34GiB to GPU in 1.93631s (17.5592GiB/s) | 20260506 13:38:04.469 | INFO | Analyzing chunk frames = [1772032,1772544) | 20260506 13:38:04.469 | INFO | MemInfo: { | 20260506 13:38:04.469 | INFO | "gpu" : " 17 GB", | 20260506 13:38:04.469 | INFO | "huge" : "", | 20260506 13:38:04.469 | INFO | "huge_shared" : " 27 GB", | 20260506 13:38:04.469 | INFO | "malloc" : " 14 GB", | 20260506 13:38:04.469 | INFO | "pinned" : " 29 GB" | 20260506 13:38:04.469 | INFO | } | 20260506 13:38:04.469 | INFO | Perf report for the last 5.13918 seconds: | 20260506 13:38:04.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:38:04.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:04.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:04.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:04.469 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:38:04.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:04.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:04.469 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260506 13:38:04.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:04.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:04.469 | INFO | Uploaded 36GiB to GPU in 2.09617s (17.1742GiB/s) | 20260506 13:38:04.469 | INFO | Downloaded 34GiB to GPU in 1.97049s (17.2546GiB/s) | 20260506 13:38:09.608 | INFO | Analyzing chunk frames = [1772544,1773056) | 20260506 13:38:09.608 | INFO | MemInfo: { | 20260506 13:38:09.608 | INFO | "gpu" : " 17 GB", | 20260506 13:38:09.608 | INFO | "huge" : "", | 20260506 13:38:09.608 | INFO | "huge_shared" : " 27 GB", | 20260506 13:38:09.608 | INFO | "malloc" : " 14 GB", | 20260506 13:38:09.608 | INFO | "pinned" : " 29 GB" | 20260506 13:38:09.608 | INFO | } | 20260506 13:38:09.608 | INFO | Perf report for the last 5.13964 seconds: | 20260506 13:38:09.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:38:09.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:09.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:09.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:09.608 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:38:09.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:09.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:09.609 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 13:38:09.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:09.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:09.609 | INFO | Uploaded 36GiB to GPU in 1.98103s (18.1724GiB/s) | 20260506 13:38:09.609 | INFO | Downloaded 34GiB to GPU in 2.02927s (16.7548GiB/s) | 20260506 13:38:14.747 | INFO | Analyzing chunk frames = [1773056,1773568) | 20260506 13:38:14.747 | INFO | MemInfo: { | 20260506 13:38:14.747 | INFO | "gpu" : " 17 GB", | 20260506 13:38:14.747 | INFO | "huge" : "", | 20260506 13:38:14.747 | INFO | "huge_shared" : " 27 GB", | 20260506 13:38:14.747 | INFO | "malloc" : " 14 GB", | 20260506 13:38:14.747 | INFO | "pinned" : " 29 GB" | 20260506 13:38:14.747 | INFO | } | 20260506 13:38:14.747 | INFO | Perf report for the last 5.13891 seconds: | 20260506 13:38:14.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:38:14.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:14.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:14.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:14.747 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:38:14.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:14.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:14.747 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:38:14.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:14.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:14.747 | INFO | Uploaded 36GiB to GPU in 2.04918s (17.568GiB/s) | 20260506 13:38:14.748 | INFO | Downloaded 34GiB to GPU in 1.97297s (17.2329GiB/s) | 20260506 13:38:14.880 | INFO | Starting a round of Baz IO | 20260506 13:38:18.033 | INFO | Populating RTMetrics report | 20260506 13:38:18.033 | INFO | RTMetrics report populated | 20260506 13:38:18.033 | INFO | Opening tmp RTMetrics json file | 20260506 13:38:18.033 | INFO | Serializing RTMetrics to json | 20260506 13:38:18.035 | INFO | Serialization done, swapping tmp file now... | 20260506 13:38:18.035 | INFO | RTMetrics done | 20260506 13:38:18.131 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 3.25 seconds | 20260506 13:38:18.131 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.42% / 0.02% / 60.57% | 20260506 13:38:18.131 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.05191% / 2.38419e-05% / 7.54992e-05% | 20260506 13:38:18.131 | INFO | Baz IO overall: | 20260506 13:38:18.131 | INFO | BytesWritten : 903974200373 | 20260506 13:38:18.131 | INFO | NumEvents : 437273161189 | 20260506 13:38:19.887 | INFO | Analyzing chunk frames = [1773568,1774080) | 20260506 13:38:19.887 | INFO | MemInfo: { | 20260506 13:38:19.887 | INFO | "gpu" : " 17 GB", | 20260506 13:38:19.887 | INFO | "huge" : "", | 20260506 13:38:19.887 | INFO | "huge_shared" : " 27 GB", | 20260506 13:38:19.887 | INFO | "malloc" : " 14 GB", | 20260506 13:38:19.887 | INFO | "pinned" : " 29 GB" | 20260506 13:38:19.887 | INFO | } | 20260506 13:38:19.887 | INFO | Perf report for the last 5.13986 seconds: | 20260506 13:38:19.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:38:19.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:19.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:19.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:19.887 | INFO | ANALYSIS : 63.3%, 3.0, 98.1ms, 0 | 20260506 13:38:19.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:19.887 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260506 13:38:19.887 | INFO | PRE_HQ : 61.1%, 1.9, 60.5ms, 0 | 20260506 13:38:19.887 | INFO | BAZWRITER : 62.5%, 3.8, 121.5ms, 0 | 20260506 13:38:19.887 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:38:19.887 | INFO | Uploaded 36GiB to GPU in 2.06197s (17.459GiB/s) | 20260506 13:38:19.887 | INFO | Downloaded 38GiB to GPU in 2.216s (17.148GiB/s) | 20260506 13:38:25.026 | INFO | Analyzing chunk frames = [1774080,1774592) | 20260506 13:38:25.026 | INFO | MemInfo: { | 20260506 13:38:25.026 | INFO | "gpu" : " 17 GB", | 20260506 13:38:25.026 | INFO | "huge" : "", | 20260506 13:38:25.026 | INFO | "huge_shared" : " 27 GB", | 20260506 13:38:25.026 | INFO | "malloc" : " 14 GB", | 20260506 13:38:25.026 | INFO | "pinned" : " 29 GB" | 20260506 13:38:25.026 | INFO | } | 20260506 13:38:25.026 | INFO | Perf report for the last 5.1387 seconds: | 20260506 13:38:25.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:38:25.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:25.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:25.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:25.026 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 13:38:25.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:25.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:25.026 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 13:38:25.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:25.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:25.026 | INFO | Uploaded 36GiB to GPU in 2.06314s (17.4491GiB/s) | 20260506 13:38:25.026 | INFO | Downloaded 34GiB to GPU in 1.94312s (17.4976GiB/s) | 20260506 13:38:30.165 | INFO | Analyzing chunk frames = [1774592,1775104) | 20260506 13:38:30.165 | INFO | MemInfo: { | 20260506 13:38:30.165 | INFO | "gpu" : " 17 GB", | 20260506 13:38:30.165 | INFO | "huge" : "", | 20260506 13:38:30.165 | INFO | "huge_shared" : " 27 GB", | 20260506 13:38:30.165 | INFO | "malloc" : " 14 GB", | 20260506 13:38:30.165 | INFO | "pinned" : " 29 GB" | 20260506 13:38:30.165 | INFO | } | 20260506 13:38:30.165 | INFO | Perf report for the last 5.13943 seconds: | 20260506 13:38:30.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:38:30.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:30.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:30.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:30.165 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260506 13:38:30.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:30.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:30.165 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:38:30.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:30.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:30.166 | INFO | Uploaded 36GiB to GPU in 2.0072s (17.9354GiB/s) | 20260506 13:38:30.166 | INFO | Downloaded 34GiB to GPU in 1.94425s (17.4875GiB/s) | 20260506 13:38:35.304 | INFO | Analyzing chunk frames = [1775104,1775616) | 20260506 13:38:35.304 | INFO | MemInfo: { | 20260506 13:38:35.304 | INFO | "gpu" : " 17 GB", | 20260506 13:38:35.304 | INFO | "huge" : "", | 20260506 13:38:35.304 | INFO | "huge_shared" : " 27 GB", | 20260506 13:38:35.304 | INFO | "malloc" : " 14 GB", | 20260506 13:38:35.304 | INFO | "pinned" : " 29 GB" | 20260506 13:38:35.304 | INFO | } | 20260506 13:38:35.305 | INFO | Perf report for the last 5.13919 seconds: | 20260506 13:38:35.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:38:35.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:35.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:35.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:35.305 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:38:35.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:35.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:35.305 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:38:35.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:35.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:35.305 | INFO | Uploaded 36GiB to GPU in 1.98987s (18.0917GiB/s) | 20260506 13:38:35.305 | INFO | Downloaded 34GiB to GPU in 1.92149s (17.6946GiB/s) | 20260506 13:38:40.444 | INFO | Analyzing chunk frames = [1775616,1776128) | 20260506 13:38:40.444 | INFO | MemInfo: { | 20260506 13:38:40.444 | INFO | "gpu" : " 17 GB", | 20260506 13:38:40.444 | INFO | "huge" : "", | 20260506 13:38:40.444 | INFO | "huge_shared" : " 27 GB", | 20260506 13:38:40.444 | INFO | "malloc" : " 14 GB", | 20260506 13:38:40.444 | INFO | "pinned" : " 29 GB" | 20260506 13:38:40.444 | INFO | } | 20260506 13:38:40.445 | INFO | Perf report for the last 5.13999 seconds: | 20260506 13:38:40.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:38:40.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:40.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:40.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:40.445 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 13:38:40.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:40.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:40.445 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:38:40.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:40.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:40.445 | INFO | Uploaded 36GiB to GPU in 2.05122s (17.5505GiB/s) | 20260506 13:38:40.445 | INFO | Downloaded 34GiB to GPU in 1.94402s (17.4896GiB/s) | 20260506 13:38:45.583 | INFO | Analyzing chunk frames = [1776128,1776640) | 20260506 13:38:45.583 | INFO | MemInfo: { | 20260506 13:38:45.583 | INFO | "gpu" : " 17 GB", | 20260506 13:38:45.583 | INFO | "huge" : "", | 20260506 13:38:45.583 | INFO | "huge_shared" : " 27 GB", | 20260506 13:38:45.583 | INFO | "malloc" : " 14 GB", | 20260506 13:38:45.583 | INFO | "pinned" : " 29 GB" | 20260506 13:38:45.583 | INFO | } | 20260506 13:38:45.583 | INFO | Perf report for the last 5.13867 seconds: | 20260506 13:38:45.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:38:45.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:45.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:45.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:45.583 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 13:38:45.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:45.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:45.583 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260506 13:38:45.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:45.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:45.584 | INFO | Uploaded 36GiB to GPU in 2.05071s (17.5549GiB/s) | 20260506 13:38:45.584 | INFO | Downloaded 34GiB to GPU in 1.92963s (17.6199GiB/s) | 20260506 13:38:50.724 | INFO | Analyzing chunk frames = [1776640,1777152) | 20260506 13:38:50.724 | INFO | MemInfo: { | 20260506 13:38:50.724 | INFO | "gpu" : " 17 GB", | 20260506 13:38:50.724 | INFO | "huge" : "", | 20260506 13:38:50.724 | INFO | "huge_shared" : " 27 GB", | 20260506 13:38:50.724 | INFO | "malloc" : " 14 GB", | 20260506 13:38:50.724 | INFO | "pinned" : " 29 GB" | 20260506 13:38:50.724 | INFO | } | 20260506 13:38:50.724 | INFO | Perf report for the last 5.14099 seconds: | 20260506 13:38:50.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:38:50.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:50.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:50.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:50.724 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:38:50.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:50.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:50.724 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 13:38:50.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:50.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:50.724 | INFO | Uploaded 36GiB to GPU in 2.00601s (17.9461GiB/s) | 20260506 13:38:50.725 | INFO | Downloaded 34GiB to GPU in 1.9629s (17.3213GiB/s) | 20260506 13:38:55.862 | INFO | Analyzing chunk frames = [1777152,1777664) | 20260506 13:38:55.862 | INFO | MemInfo: { | 20260506 13:38:55.862 | INFO | "gpu" : " 17 GB", | 20260506 13:38:55.862 | INFO | "huge" : "", | 20260506 13:38:55.862 | INFO | "huge_shared" : " 27 GB", | 20260506 13:38:55.862 | INFO | "malloc" : " 14 GB", | 20260506 13:38:55.862 | INFO | "pinned" : " 29 GB" | 20260506 13:38:55.862 | INFO | } | 20260506 13:38:55.862 | INFO | Perf report for the last 5.13773 seconds: | 20260506 13:38:55.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:38:55.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:55.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:55.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:55.862 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:38:55.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:55.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:55.862 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:38:55.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:55.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:38:55.862 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:38:55.862 | INFO | Total mem usage: 17.4 GiB | 20260506 13:38:55.862 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:38:55.862 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -20.7%) | 20260506 13:38:55.862 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:38:55.862 | INFO | High Water Breakdown: | 20260506 13:38:55.862 | INFO | 59.9% : 8.64 GiB ( 79.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:38:55.862 | INFO | 40% : 5.77 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:38:55.862 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:38:55.862 | INFO | ------------------------------------------ | 20260506 13:38:55.862 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:38:55.862 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:38:55.862 | INFO | Total mem usage: 26.5 GiB | 20260506 13:38:55.862 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:38:55.862 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:38:55.862 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:38:55.862 | INFO | High Water Breakdown: | 20260506 13:38:55.862 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:38:55.862 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:38:55.862 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:38:55.862 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:38:55.862 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:38:55.862 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:38:55.862 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:38:55.862 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:38:55.862 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:38:55.863 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:38:55.863 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:38:55.863 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:38:55.863 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:38:55.863 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:38:55.863 | INFO | ------------------------------------------ | 20260506 13:38:55.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:38:55.863 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:38:55.863 | INFO | Total mem usage: 25.6 GiB | 20260506 13:38:55.863 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:38:55.863 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:38:55.863 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:38:55.863 | INFO | High Water Breakdown: | 20260506 13:38:55.863 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:38:55.863 | INFO | ------------------------------------------ | 20260506 13:38:55.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:38:55.863 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:38:55.863 | INFO | Total mem usage: 15.7 GiB | 20260506 13:38:55.863 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:38:55.863 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:38:55.863 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:38:55.863 | INFO | High Water Breakdown: | 20260506 13:38:55.863 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:38:55.863 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:38:55.863 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:38:55.863 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:38:55.863 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:38:55.863 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:38:55.863 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:38:55.863 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:38:55.863 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:38:55.863 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:38:55.863 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:38:55.863 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:38:55.863 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:38:55.863 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:38:55.863 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:38:55.863 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:38:55.863 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:38:55.863 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:38:55.863 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:38:55.863 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:38:55.863 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:38:55.863 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:38:55.863 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:38:55.863 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:38:55.863 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:38:55.863 | INFO | ------------------------------------------ | 20260506 13:38:55.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:38:55.863 | INFO | Uploaded 36GiB to GPU in 2.04899s (17.5696GiB/s) | 20260506 13:38:55.863 | INFO | Downloaded 34GiB to GPU in 1.97117s (17.2486GiB/s) | 20260506 13:38:55.863 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:38:55.863 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:38:55.863 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:38:55.863 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.63e+01, 1.36e+01 / 2.32e+00 | 20260506 13:38:55.863 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.01e-01, 9.92e-03 / 8.94e-03 | 20260506 13:38:55.863 | INFO | PerfReport: 11.56%, Baseline -- 3.65e+00 / 4.48e+00, 5.66e-01 / 2.82e-01 | 20260506 13:38:55.863 | INFO | PerfReport: 3.23%, Histogram -- 1.02e+00 / 1.46e+00, 1.87e-01 / 1.37e-01 | 20260506 13:38:55.863 | INFO | PerfReport: 15.85%, DME -- 4.01e+01 / 4.47e+01, 3.98e+00 / 2.55e+00 | 20260506 13:38:55.863 | INFO | PerfReport: 56.58%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.41e+00 / 1.83e+00 | 20260506 13:38:55.863 | INFO | PerfReport: 7.54%, PulseAccumulating -- 2.38e+00 / 3.02e+00, 4.23e-01 / 2.45e-01 | 20260506 13:38:55.863 | INFO | PerfReport: 3.67%, Metrics -- 1.16e+00 / 1.55e+00, 2.95e-01 / 1.67e-01 | 20260506 13:38:55.863 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.07e-01 / 4.62e-01, 6.24e-02 / 2.17e-02 | 20260506 13:38:55.863 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:38:55.863 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:38:55.863 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:38:55.863 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 13:38:55.863 | INFO | PerfReport: 21.50%, Upload -- 2.12e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260506 13:38:55.863 | INFO | PerfReport: 20.99%, Download -- 2.07e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 13:38:55.863 | INFO | PerfReport: 32.08%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 13:38:55.863 | INFO | PerfReport: 1.86%, LockUpload -- 1.83e+00 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260506 13:38:55.863 | INFO | PerfReport: 1.00%, LockDownload -- 9.81e-01 / 0.00e+00, 2.63e+00 / 0.00e+00 | 20260506 13:38:55.863 | INFO | PerfReport: 22.56%, LockCompute -- 2.22e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 13:38:55.863 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260506 13:38:55.996 | INFO | Starting a round of Baz IO | 20260506 13:38:59.156 | INFO | Populating RTMetrics report | 20260506 13:38:59.156 | INFO | RTMetrics report populated | 20260506 13:38:59.156 | INFO | Opening tmp RTMetrics json file | 20260506 13:38:59.157 | INFO | Serializing RTMetrics to json | 20260506 13:38:59.158 | INFO | Serialization done, swapping tmp file now... | 20260506 13:38:59.159 | INFO | RTMetrics done | 20260506 13:38:59.251 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.255 seconds | 20260506 13:38:59.251 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.40% / 0.02% / 60.58% | 20260506 13:38:59.251 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.04117% / 4.37101e-05% / 4.37101e-05% | 20260506 13:38:59.251 | INFO | Baz IO overall: | 20260506 13:38:59.251 | INFO | BytesWritten : 906242177375 | 20260506 13:38:59.251 | INFO | NumEvents : 438373135998 | 20260506 13:39:01.001 | INFO | Analyzing chunk frames = [1777664,1778176) | 20260506 13:39:01.001 | INFO | MemInfo: { | 20260506 13:39:01.001 | INFO | "gpu" : " 17 GB", | 20260506 13:39:01.001 | INFO | "huge" : "", | 20260506 13:39:01.001 | INFO | "huge_shared" : " 27 GB", | 20260506 13:39:01.001 | INFO | "malloc" : " 14 GB", | 20260506 13:39:01.001 | INFO | "pinned" : " 29 GB" | 20260506 13:39:01.001 | INFO | } | 20260506 13:39:01.001 | INFO | Perf report for the last 5.13944 seconds: | 20260506 13:39:01.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:39:01.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:01.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:01.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:01.001 | INFO | ANALYSIS : 63.5%, 3.0, 98.4ms, 0 | 20260506 13:39:01.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:01.001 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 13:39:01.001 | INFO | PRE_HQ : 61.2%, 1.9, 60.5ms, 0 | 20260506 13:39:01.001 | INFO | BAZWRITER : 62.7%, 3.8, 120.8ms, 0 | 20260506 13:39:01.001 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:39:01.002 | INFO | Uploaded 36GiB to GPU in 2.05394s (17.5273GiB/s) | 20260506 13:39:01.002 | INFO | Downloaded 38GiB to GPU in 2.21972s (17.1193GiB/s) | 20260506 13:39:06.140 | INFO | Analyzing chunk frames = [1778176,1778688) | 20260506 13:39:06.140 | INFO | MemInfo: { | 20260506 13:39:06.140 | INFO | "gpu" : " 17 GB", | 20260506 13:39:06.140 | INFO | "huge" : "", | 20260506 13:39:06.140 | INFO | "huge_shared" : " 27 GB", | 20260506 13:39:06.141 | INFO | "malloc" : " 14 GB", | 20260506 13:39:06.141 | INFO | "pinned" : " 29 GB" | 20260506 13:39:06.141 | INFO | } | 20260506 13:39:06.141 | INFO | Perf report for the last 5.13922 seconds: | 20260506 13:39:06.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:39:06.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:06.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:06.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:06.141 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:39:06.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:06.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:06.141 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:39:06.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:06.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:06.141 | INFO | Uploaded 36GiB to GPU in 2.02945s (17.7388GiB/s) | 20260506 13:39:06.141 | INFO | Downloaded 34GiB to GPU in 1.94107s (17.5161GiB/s) | 20260506 13:39:11.280 | INFO | Analyzing chunk frames = [1778688,1779200) | 20260506 13:39:11.280 | INFO | MemInfo: { | 20260506 13:39:11.280 | INFO | "gpu" : " 17 GB", | 20260506 13:39:11.280 | INFO | "huge" : "", | 20260506 13:39:11.280 | INFO | "huge_shared" : " 27 GB", | 20260506 13:39:11.280 | INFO | "malloc" : " 14 GB", | 20260506 13:39:11.280 | INFO | "pinned" : " 29 GB" | 20260506 13:39:11.280 | INFO | } | 20260506 13:39:11.280 | INFO | Perf report for the last 5.13928 seconds: | 20260506 13:39:11.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:39:11.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:11.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:11.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:11.280 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 13:39:11.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:11.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:11.280 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:39:11.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:11.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:11.280 | INFO | Uploaded 36GiB to GPU in 2.02684s (17.7616GiB/s) | 20260506 13:39:11.280 | INFO | Downloaded 34GiB to GPU in 1.94134s (17.5137GiB/s) | 20260506 13:39:16.419 | INFO | Analyzing chunk frames = [1779200,1779712) | 20260506 13:39:16.419 | INFO | MemInfo: { | 20260506 13:39:16.419 | INFO | "gpu" : " 17 GB", | 20260506 13:39:16.419 | INFO | "huge" : "", | 20260506 13:39:16.419 | INFO | "huge_shared" : " 27 GB", | 20260506 13:39:16.419 | INFO | "malloc" : " 14 GB", | 20260506 13:39:16.419 | INFO | "pinned" : " 29 GB" | 20260506 13:39:16.419 | INFO | } | 20260506 13:39:16.419 | INFO | Perf report for the last 5.1394 seconds: | 20260506 13:39:16.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:39:16.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:16.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:16.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:16.419 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:39:16.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:16.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:16.419 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:39:16.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:16.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:16.420 | INFO | Uploaded 36GiB to GPU in 2.00967s (17.9134GiB/s) | 20260506 13:39:16.420 | INFO | Downloaded 34GiB to GPU in 1.92241s (17.6861GiB/s) | 20260506 13:39:21.559 | INFO | Analyzing chunk frames = [1779712,1780224) | 20260506 13:39:21.559 | INFO | MemInfo: { | 20260506 13:39:21.559 | INFO | "gpu" : " 17 GB", | 20260506 13:39:21.559 | INFO | "huge" : "", | 20260506 13:39:21.559 | INFO | "huge_shared" : " 27 GB", | 20260506 13:39:21.559 | INFO | "malloc" : " 14 GB", | 20260506 13:39:21.559 | INFO | "pinned" : " 29 GB" | 20260506 13:39:21.559 | INFO | } | 20260506 13:39:21.559 | INFO | Perf report for the last 5.13931 seconds: | 20260506 13:39:21.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:39:21.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:21.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:21.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:21.559 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 13:39:21.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:21.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:21.559 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:39:21.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:21.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:21.559 | INFO | Uploaded 36GiB to GPU in 2.0714s (17.3796GiB/s) | 20260506 13:39:21.559 | INFO | Downloaded 34GiB to GPU in 1.95511s (17.3904GiB/s) | 20260506 13:39:26.698 | INFO | Analyzing chunk frames = [1780224,1780736) | 20260506 13:39:26.698 | INFO | MemInfo: { | 20260506 13:39:26.698 | INFO | "gpu" : " 17 GB", | 20260506 13:39:26.698 | INFO | "huge" : "", | 20260506 13:39:26.698 | INFO | "huge_shared" : " 27 GB", | 20260506 13:39:26.698 | INFO | "malloc" : " 14 GB", | 20260506 13:39:26.698 | INFO | "pinned" : " 29 GB" | 20260506 13:39:26.698 | INFO | } | 20260506 13:39:26.698 | INFO | Perf report for the last 5.13957 seconds: | 20260506 13:39:26.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:39:26.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:26.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:26.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:26.698 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 13:39:26.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:26.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:26.698 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 13:39:26.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:26.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:26.699 | INFO | Uploaded 36GiB to GPU in 2.04681s (17.5883GiB/s) | 20260506 13:39:26.699 | INFO | Downloaded 34GiB to GPU in 1.91319s (17.7713GiB/s) | 20260506 13:39:31.838 | INFO | Analyzing chunk frames = [1780736,1781248) | 20260506 13:39:31.838 | INFO | MemInfo: { | 20260506 13:39:31.838 | INFO | "gpu" : " 17 GB", | 20260506 13:39:31.838 | INFO | "huge" : "", | 20260506 13:39:31.838 | INFO | "huge_shared" : " 27 GB", | 20260506 13:39:31.838 | INFO | "malloc" : " 14 GB", | 20260506 13:39:31.838 | INFO | "pinned" : " 29 GB" | 20260506 13:39:31.838 | INFO | } | 20260506 13:39:31.838 | INFO | Perf report for the last 5.1401 seconds: | 20260506 13:39:31.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:39:31.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:31.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:31.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:31.838 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:39:31.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:31.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:31.838 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 13:39:31.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:31.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:31.839 | INFO | Uploaded 36GiB to GPU in 2.01976s (17.8239GiB/s) | 20260506 13:39:31.839 | INFO | Downloaded 34GiB to GPU in 1.96956s (17.2627GiB/s) | 20260506 13:39:36.977 | INFO | Analyzing chunk frames = [1781248,1781760) | 20260506 13:39:36.977 | INFO | MemInfo: { | 20260506 13:39:36.977 | INFO | "gpu" : " 17 GB", | 20260506 13:39:36.977 | INFO | "huge" : "", | 20260506 13:39:36.977 | INFO | "huge_shared" : " 27 GB", | 20260506 13:39:36.977 | INFO | "malloc" : " 14 GB", | 20260506 13:39:36.977 | INFO | "pinned" : " 29 GB" | 20260506 13:39:36.977 | INFO | } | 20260506 13:39:36.977 | INFO | Perf report for the last 5.13865 seconds: | 20260506 13:39:36.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:39:36.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:36.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:36.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:36.977 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:39:36.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:36.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:36.977 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:39:36.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:36.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:36.977 | INFO | Uploaded 36GiB to GPU in 2.06246s (17.4549GiB/s) | 20260506 13:39:36.977 | INFO | Downloaded 34GiB to GPU in 1.97776s (17.1912GiB/s) | 20260506 13:39:37.111 | INFO | Starting a round of Baz IO | 20260506 13:39:40.268 | INFO | Populating RTMetrics report | 20260506 13:39:40.268 | INFO | RTMetrics report populated | 20260506 13:39:40.268 | INFO | Opening tmp RTMetrics json file | 20260506 13:39:40.269 | INFO | Serializing RTMetrics to json | 20260506 13:39:40.270 | INFO | Serialization done, swapping tmp file now... | 20260506 13:39:40.271 | INFO | RTMetrics done | 20260506 13:39:40.372 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 3.26 seconds | 20260506 13:39:40.372 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.38% / 0.02% / 60.60% | 20260506 13:39:40.372 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.03554% / 4.37101e-05% / 4.76837e-05% | 20260506 13:39:40.372 | INFO | Baz IO overall: | 20260506 13:39:40.372 | INFO | BytesWritten : 908510875511 | 20260506 13:39:40.372 | INFO | NumEvents : 439473977090 | 20260506 13:39:42.117 | INFO | Analyzing chunk frames = [1781760,1782272) | 20260506 13:39:42.117 | INFO | MemInfo: { | 20260506 13:39:42.117 | INFO | "gpu" : " 17 GB", | 20260506 13:39:42.117 | INFO | "huge" : "", | 20260506 13:39:42.117 | INFO | "huge_shared" : " 27 GB", | 20260506 13:39:42.117 | INFO | "malloc" : " 14 GB", | 20260506 13:39:42.117 | INFO | "pinned" : " 29 GB" | 20260506 13:39:42.117 | INFO | } | 20260506 13:39:42.117 | INFO | Perf report for the last 5.13966 seconds: | 20260506 13:39:42.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:39:42.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:42.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:42.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:42.117 | INFO | ANALYSIS : 63.4%, 2.9, 98.0ms, 0 | 20260506 13:39:42.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:42.117 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 13:39:42.117 | INFO | PRE_HQ : 61.2%, 1.9, 61.1ms, 0 | 20260506 13:39:42.117 | INFO | BAZWRITER : 62.7%, 3.8, 123.3ms, 0 | 20260506 13:39:42.117 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:39:42.117 | INFO | Uploaded 36GiB to GPU in 2.04866s (17.5724GiB/s) | 20260506 13:39:42.117 | INFO | Downloaded 38GiB to GPU in 2.20719s (17.2165GiB/s) | 20260506 13:39:47.255 | INFO | Analyzing chunk frames = [1782272,1782784) | 20260506 13:39:47.255 | INFO | MemInfo: { | 20260506 13:39:47.255 | INFO | "gpu" : " 17 GB", | 20260506 13:39:47.255 | INFO | "huge" : "", | 20260506 13:39:47.255 | INFO | "huge_shared" : " 27 GB", | 20260506 13:39:47.255 | INFO | "malloc" : " 14 GB", | 20260506 13:39:47.255 | INFO | "pinned" : " 29 GB" | 20260506 13:39:47.255 | INFO | } | 20260506 13:39:47.255 | INFO | Perf report for the last 5.13879 seconds: | 20260506 13:39:47.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:39:47.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:47.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:47.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:47.255 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:39:47.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:47.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:47.256 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:39:47.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:47.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:47.256 | INFO | Uploaded 36GiB to GPU in 2.02679s (17.7621GiB/s) | 20260506 13:39:47.256 | INFO | Downloaded 34GiB to GPU in 1.93866s (17.5379GiB/s) | 20260506 13:39:52.395 | INFO | Analyzing chunk frames = [1782784,1783296) | 20260506 13:39:52.395 | INFO | MemInfo: { | 20260506 13:39:52.395 | INFO | "gpu" : " 17 GB", | 20260506 13:39:52.395 | INFO | "huge" : "", | 20260506 13:39:52.395 | INFO | "huge_shared" : " 27 GB", | 20260506 13:39:52.395 | INFO | "malloc" : " 14 GB", | 20260506 13:39:52.395 | INFO | "pinned" : " 29 GB" | 20260506 13:39:52.395 | INFO | } | 20260506 13:39:52.395 | INFO | Perf report for the last 5.13953 seconds: | 20260506 13:39:52.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:39:52.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:52.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:52.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:52.395 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 13:39:52.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:52.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:52.395 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:39:52.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:52.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:52.395 | INFO | Uploaded 36GiB to GPU in 2.01826s (17.8372GiB/s) | 20260506 13:39:52.395 | INFO | Downloaded 34GiB to GPU in 1.94438s (17.4863GiB/s) | 20260506 13:39:57.534 | INFO | Analyzing chunk frames = [1783296,1783808) | 20260506 13:39:57.534 | INFO | MemInfo: { | 20260506 13:39:57.534 | INFO | "gpu" : " 17 GB", | 20260506 13:39:57.534 | INFO | "huge" : "", | 20260506 13:39:57.534 | INFO | "huge_shared" : " 27 GB", | 20260506 13:39:57.534 | INFO | "malloc" : " 14 GB", | 20260506 13:39:57.534 | INFO | "pinned" : " 29 GB" | 20260506 13:39:57.534 | INFO | } | 20260506 13:39:57.534 | INFO | Perf report for the last 5.13931 seconds: | 20260506 13:39:57.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:39:57.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:57.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:57.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:57.534 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:39:57.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:57.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:57.534 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:39:57.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:57.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:39:57.535 | INFO | Uploaded 36GiB to GPU in 2.0096s (17.914GiB/s) | 20260506 13:39:57.535 | INFO | Downloaded 34GiB to GPU in 1.93197s (17.5986GiB/s) | 20260506 13:40:02.674 | INFO | Analyzing chunk frames = [1783808,1784320) | 20260506 13:40:02.674 | INFO | MemInfo: { | 20260506 13:40:02.674 | INFO | "gpu" : " 17 GB", | 20260506 13:40:02.674 | INFO | "huge" : "", | 20260506 13:40:02.674 | INFO | "huge_shared" : " 27 GB", | 20260506 13:40:02.674 | INFO | "malloc" : " 14 GB", | 20260506 13:40:02.674 | INFO | "pinned" : " 29 GB" | 20260506 13:40:02.674 | INFO | } | 20260506 13:40:02.674 | INFO | Perf report for the last 5.1393 seconds: | 20260506 13:40:02.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:40:02.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:02.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:02.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:02.674 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 13:40:02.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:02.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:02.674 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 13:40:02.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:02.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:02.674 | INFO | Uploaded 36GiB to GPU in 2.00777s (17.9303GiB/s) | 20260506 13:40:02.674 | INFO | Downloaded 34GiB to GPU in 1.94256s (17.5026GiB/s) | 20260506 13:40:07.813 | INFO | Analyzing chunk frames = [1784320,1784832) | 20260506 13:40:07.813 | INFO | MemInfo: { | 20260506 13:40:07.813 | INFO | "gpu" : " 17 GB", | 20260506 13:40:07.813 | INFO | "huge" : "", | 20260506 13:40:07.813 | INFO | "huge_shared" : " 27 GB", | 20260506 13:40:07.813 | INFO | "malloc" : " 14 GB", | 20260506 13:40:07.813 | INFO | "pinned" : " 29 GB" | 20260506 13:40:07.813 | INFO | } | 20260506 13:40:07.813 | INFO | Perf report for the last 5.13942 seconds: | 20260506 13:40:07.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:40:07.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:07.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:07.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:07.813 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:40:07.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:07.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:07.813 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 13:40:07.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:07.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:07.813 | INFO | Uploaded 36GiB to GPU in 2.09387s (17.193GiB/s) | 20260506 13:40:07.813 | INFO | Downloaded 34GiB to GPU in 1.97073s (17.2525GiB/s) | 20260506 13:40:12.953 | INFO | Analyzing chunk frames = [1784832,1785344) | 20260506 13:40:12.953 | INFO | MemInfo: { | 20260506 13:40:12.953 | INFO | "gpu" : " 17 GB", | 20260506 13:40:12.953 | INFO | "huge" : "", | 20260506 13:40:12.953 | INFO | "huge_shared" : " 27 GB", | 20260506 13:40:12.953 | INFO | "malloc" : " 14 GB", | 20260506 13:40:12.953 | INFO | "pinned" : " 29 GB" | 20260506 13:40:12.953 | INFO | } | 20260506 13:40:12.953 | INFO | Perf report for the last 5.13955 seconds: | 20260506 13:40:12.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:40:12.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:12.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:12.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:12.953 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:40:12.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:12.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:12.953 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:40:12.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:12.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:12.953 | INFO | Uploaded 36GiB to GPU in 1.98361s (18.1487GiB/s) | 20260506 13:40:12.953 | INFO | Downloaded 34GiB to GPU in 2.02821s (16.7636GiB/s) | 20260506 13:40:18.091 | INFO | Analyzing chunk frames = [1785344,1785856) | 20260506 13:40:18.091 | INFO | MemInfo: { | 20260506 13:40:18.091 | INFO | "gpu" : " 17 GB", | 20260506 13:40:18.091 | INFO | "huge" : "", | 20260506 13:40:18.092 | INFO | "huge_shared" : " 27 GB", | 20260506 13:40:18.092 | INFO | "malloc" : " 14 GB", | 20260506 13:40:18.092 | INFO | "pinned" : " 29 GB" | 20260506 13:40:18.092 | INFO | } | 20260506 13:40:18.092 | INFO | Perf report for the last 5.13896 seconds: | 20260506 13:40:18.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:40:18.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:18.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:18.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:18.092 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:40:18.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:18.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:18.092 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 13:40:18.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:18.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:18.092 | INFO | Uploaded 36GiB to GPU in 2.04076s (17.6405GiB/s) | 20260506 13:40:18.092 | INFO | Downloaded 34GiB to GPU in 1.97516s (17.2138GiB/s) | 20260506 13:40:18.092 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:40:18.092 | INFO | Total mem usage: 17.4 GiB | 20260506 13:40:18.092 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:40:18.092 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -20.2%) | 20260506 13:40:18.092 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:40:18.092 | INFO | High Water Breakdown: | 20260506 13:40:18.092 | INFO | 59.9% : 8.67 GiB ( 79.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:40:18.092 | INFO | 40% : 5.79 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:40:18.092 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:40:18.092 | INFO | ------------------------------------------ | 20260506 13:40:18.092 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:40:18.092 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:40:18.092 | INFO | Total mem usage: 26.5 GiB | 20260506 13:40:18.092 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:40:18.092 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:40:18.092 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:40:18.092 | INFO | High Water Breakdown: | 20260506 13:40:18.092 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:40:18.092 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:40:18.092 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:40:18.092 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:40:18.092 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:40:18.092 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:40:18.092 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:40:18.092 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:40:18.092 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:40:18.092 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:40:18.092 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:40:18.092 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:40:18.092 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:40:18.092 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:40:18.092 | INFO | ------------------------------------------ | 20260506 13:40:18.092 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:40:18.092 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:40:18.092 | INFO | Total mem usage: 25.6 GiB | 20260506 13:40:18.092 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:40:18.092 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:40:18.092 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:40:18.092 | INFO | High Water Breakdown: | 20260506 13:40:18.092 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:40:18.092 | INFO | ------------------------------------------ | 20260506 13:40:18.092 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:40:18.092 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:40:18.092 | INFO | Total mem usage: 15.7 GiB | 20260506 13:40:18.093 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:40:18.093 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:40:18.093 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 13:40:18.093 | INFO | High Water Breakdown: | 20260506 13:40:18.093 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:40:18.093 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:40:18.093 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:40:18.093 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:40:18.093 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:40:18.093 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:40:18.093 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:40:18.093 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:40:18.093 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:40:18.093 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:40:18.093 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:40:18.093 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:40:18.093 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:40:18.093 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:40:18.093 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:40:18.093 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:40:18.093 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:40:18.093 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:40:18.093 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:40:18.093 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:40:18.093 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:40:18.093 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:40:18.093 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:40:18.093 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:40:18.093 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:40:18.093 | INFO | ------------------------------------------ | 20260506 13:40:18.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:40:18.093 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:40:18.093 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:40:18.093 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:40:18.093 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.61e+01, 1.34e+01 / 2.36e+00 | 20260506 13:40:18.093 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.02e-01, 1.11e-02 / 6.55e-03 | 20260506 13:40:18.093 | INFO | PerfReport: 11.61%, Baseline -- 3.67e+00 / 4.43e+00, 5.85e-01 / 3.37e-01 | 20260506 13:40:18.093 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.48e+00, 1.93e-01 / 1.25e-01 | 20260506 13:40:18.093 | INFO | PerfReport: 15.84%, DME -- 4.01e+01 / 4.45e+01, 4.01e+00 / 2.33e+00 | 20260506 13:40:18.093 | INFO | PerfReport: 56.54%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.41e+00 / 1.55e+00 | 20260506 13:40:18.093 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.38e+00 / 3.00e+00, 4.16e-01 / 2.27e-01 | 20260506 13:40:18.093 | INFO | PerfReport: 3.67%, Metrics -- 1.16e+00 / 1.54e+00, 2.99e-01 / 2.02e-01 | 20260506 13:40:18.093 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.07e-01 / 4.64e-01, 6.25e-02 / 2.74e-02 | 20260506 13:40:18.093 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:40:18.093 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:40:18.093 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:40:18.093 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 13:40:18.093 | INFO | PerfReport: 21.51%, Upload -- 2.12e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260506 13:40:18.093 | INFO | PerfReport: 21.00%, Download -- 2.07e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 13:40:18.093 | INFO | PerfReport: 32.11%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 13:40:18.093 | INFO | PerfReport: 1.82%, LockUpload -- 1.80e+00 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260506 13:40:18.093 | INFO | PerfReport: 1.00%, LockDownload -- 9.89e-01 / 0.00e+00, 2.61e+00 / 0.00e+00 | 20260506 13:40:18.093 | INFO | PerfReport: 22.53%, LockCompute -- 2.22e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260506 13:40:18.093 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260506 13:40:18.226 | INFO | Starting a round of Baz IO | 20260506 13:40:21.383 | INFO | Populating RTMetrics report | 20260506 13:40:21.383 | INFO | RTMetrics report populated | 20260506 13:40:21.383 | INFO | Opening tmp RTMetrics json file | 20260506 13:40:21.385 | INFO | Serializing RTMetrics to json | 20260506 13:40:21.386 | INFO | Serialization done, swapping tmp file now... | 20260506 13:40:21.387 | INFO | RTMetrics done | 20260506 13:40:21.485 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 3.259 seconds | 20260506 13:40:21.485 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.36% / 0.02% / 60.62% | 20260506 13:40:21.485 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.02812% / 1.98682e-05% / 6.35783e-05% | 20260506 13:40:21.485 | INFO | Baz IO overall: | 20260506 13:40:21.485 | INFO | BytesWritten : 910779353707 | 20260506 13:40:21.485 | INFO | NumEvents : 440574682702 | 20260506 13:40:23.231 | INFO | Analyzing chunk frames = [1785856,1786368) | 20260506 13:40:23.231 | INFO | MemInfo: { | 20260506 13:40:23.231 | INFO | "gpu" : " 17 GB", | 20260506 13:40:23.231 | INFO | "huge" : "", | 20260506 13:40:23.231 | INFO | "huge_shared" : " 27 GB", | 20260506 13:40:23.231 | INFO | "malloc" : " 14 GB", | 20260506 13:40:23.231 | INFO | "pinned" : " 29 GB" | 20260506 13:40:23.231 | INFO | } | 20260506 13:40:23.231 | INFO | Perf report for the last 5.13931 seconds: | 20260506 13:40:23.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:40:23.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:23.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:23.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:23.231 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 13:40:23.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:23.231 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:40:23.231 | INFO | PRE_HQ : 61.3%, 1.9, 61.0ms, 0 | 20260506 13:40:23.231 | INFO | BAZWRITER : 62.8%, 3.7, 121.0ms, 0 | 20260506 13:40:23.231 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 13:40:23.231 | INFO | Uploaded 36GiB to GPU in 2.07422s (17.3559GiB/s) | 20260506 13:40:23.231 | INFO | Downloaded 38GiB to GPU in 2.21328s (17.1691GiB/s) | 20260506 13:40:28.370 | INFO | Analyzing chunk frames = [1786368,1786880) | 20260506 13:40:28.370 | INFO | MemInfo: { | 20260506 13:40:28.370 | INFO | "gpu" : " 17 GB", | 20260506 13:40:28.370 | INFO | "huge" : "", | 20260506 13:40:28.370 | INFO | "huge_shared" : " 27 GB", | 20260506 13:40:28.370 | INFO | "malloc" : " 14 GB", | 20260506 13:40:28.370 | INFO | "pinned" : " 29 GB" | 20260506 13:40:28.370 | INFO | } | 20260506 13:40:28.370 | INFO | Perf report for the last 5.13955 seconds: | 20260506 13:40:28.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:40:28.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:28.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:28.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:28.371 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:40:28.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:28.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:28.371 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:40:28.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:28.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:28.371 | INFO | Uploaded 36GiB to GPU in 2.01569s (17.8599GiB/s) | 20260506 13:40:28.371 | INFO | Downloaded 34GiB to GPU in 1.92312s (17.6796GiB/s) | 20260506 13:40:33.509 | INFO | Analyzing chunk frames = [1786880,1787392) | 20260506 13:40:33.510 | INFO | MemInfo: { | 20260506 13:40:33.510 | INFO | "gpu" : " 17 GB", | 20260506 13:40:33.510 | INFO | "huge" : "", | 20260506 13:40:33.510 | INFO | "huge_shared" : " 27 GB", | 20260506 13:40:33.510 | INFO | "malloc" : " 14 GB", | 20260506 13:40:33.510 | INFO | "pinned" : " 29 GB" | 20260506 13:40:33.510 | INFO | } | 20260506 13:40:33.510 | INFO | Perf report for the last 5.13912 seconds: | 20260506 13:40:33.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:40:33.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:33.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:33.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:33.510 | INFO | ANALYSIS : 61.4%, 3.0, 95.0ms, 0 | 20260506 13:40:33.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:33.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:33.510 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:40:33.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:33.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:33.510 | INFO | Uploaded 36GiB to GPU in 1.97035s (18.2709GiB/s) | 20260506 13:40:33.510 | INFO | Downloaded 34GiB to GPU in 1.91996s (17.7087GiB/s) | 20260506 13:40:38.649 | INFO | Analyzing chunk frames = [1787392,1787904) | 20260506 13:40:38.649 | INFO | MemInfo: { | 20260506 13:40:38.649 | INFO | "gpu" : " 17 GB", | 20260506 13:40:38.649 | INFO | "huge" : "", | 20260506 13:40:38.649 | INFO | "huge_shared" : " 27 GB", | 20260506 13:40:38.649 | INFO | "malloc" : " 14 GB", | 20260506 13:40:38.649 | INFO | "pinned" : " 29 GB" | 20260506 13:40:38.649 | INFO | } | 20260506 13:40:38.649 | INFO | Perf report for the last 5.1393 seconds: | 20260506 13:40:38.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:40:38.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:38.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:38.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:38.649 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 13:40:38.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:38.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:38.649 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:40:38.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:38.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:38.649 | INFO | Uploaded 36GiB to GPU in 1.96089s (18.359GiB/s) | 20260506 13:40:38.649 | INFO | Downloaded 34GiB to GPU in 1.91357s (17.7678GiB/s) | 20260506 13:40:43.788 | INFO | Analyzing chunk frames = [1787904,1788416) | 20260506 13:40:43.788 | INFO | MemInfo: { | 20260506 13:40:43.788 | INFO | "gpu" : " 17 GB", | 20260506 13:40:43.788 | INFO | "huge" : "", | 20260506 13:40:43.788 | INFO | "huge_shared" : " 27 GB", | 20260506 13:40:43.788 | INFO | "malloc" : " 14 GB", | 20260506 13:40:43.788 | INFO | "pinned" : " 29 GB" | 20260506 13:40:43.788 | INFO | } | 20260506 13:40:43.788 | INFO | Perf report for the last 5.1393 seconds: | 20260506 13:40:43.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:40:43.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:43.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:43.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:43.788 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 13:40:43.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:43.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:43.788 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:40:43.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:43.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:43.789 | INFO | Uploaded 36GiB to GPU in 2.04638s (17.592GiB/s) | 20260506 13:40:43.789 | INFO | Downloaded 34GiB to GPU in 1.93656s (17.5569GiB/s) | 20260506 13:40:48.927 | INFO | Analyzing chunk frames = [1788416,1788928) | 20260506 13:40:48.927 | INFO | MemInfo: { | 20260506 13:40:48.927 | INFO | "gpu" : " 17 GB", | 20260506 13:40:48.927 | INFO | "huge" : "", | 20260506 13:40:48.928 | INFO | "huge_shared" : " 27 GB", | 20260506 13:40:48.928 | INFO | "malloc" : " 14 GB", | 20260506 13:40:48.928 | INFO | "pinned" : " 29 GB" | 20260506 13:40:48.928 | INFO | } | 20260506 13:40:48.928 | INFO | Perf report for the last 5.13931 seconds: | 20260506 13:40:48.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:40:48.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:48.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:48.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:48.928 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 13:40:48.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:48.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:48.928 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 13:40:48.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:48.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:48.928 | INFO | Uploaded 36GiB to GPU in 2.06519s (17.4318GiB/s) | 20260506 13:40:48.928 | INFO | Downloaded 34GiB to GPU in 1.91346s (17.7688GiB/s) | 20260506 13:40:54.067 | INFO | Analyzing chunk frames = [1788928,1789440) | 20260506 13:40:54.067 | INFO | MemInfo: { | 20260506 13:40:54.067 | INFO | "gpu" : " 17 GB", | 20260506 13:40:54.067 | INFO | "huge" : "", | 20260506 13:40:54.067 | INFO | "huge_shared" : " 27 GB", | 20260506 13:40:54.067 | INFO | "malloc" : " 14 GB", | 20260506 13:40:54.067 | INFO | "pinned" : " 29 GB" | 20260506 13:40:54.067 | INFO | } | 20260506 13:40:54.067 | INFO | Perf report for the last 5.13982 seconds: | 20260506 13:40:54.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:40:54.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:54.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:54.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:54.068 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 13:40:54.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:54.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:54.068 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 13:40:54.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:54.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:54.068 | INFO | Uploaded 36GiB to GPU in 2.04742s (17.5831GiB/s) | 20260506 13:40:54.068 | INFO | Downloaded 34GiB to GPU in 2.00116s (16.9902GiB/s) | 20260506 13:40:59.206 | INFO | Analyzing chunk frames = [1789440,1789952) | 20260506 13:40:59.206 | INFO | MemInfo: { | 20260506 13:40:59.206 | INFO | "gpu" : " 17 GB", | 20260506 13:40:59.206 | INFO | "huge" : "", | 20260506 13:40:59.206 | INFO | "huge_shared" : " 27 GB", | 20260506 13:40:59.206 | INFO | "malloc" : " 14 GB", | 20260506 13:40:59.206 | INFO | "pinned" : " 29 GB" | 20260506 13:40:59.206 | INFO | } | 20260506 13:40:59.206 | INFO | Perf report for the last 5.13906 seconds: | 20260506 13:40:59.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:40:59.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:59.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:59.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:59.207 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:40:59.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:59.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:59.207 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:40:59.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:59.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:40:59.207 | INFO | Uploaded 36GiB to GPU in 2.0446s (17.6074GiB/s) | 20260506 13:40:59.207 | INFO | Downloaded 34GiB to GPU in 1.96889s (17.2686GiB/s) | 20260506 13:40:59.339 | INFO | Starting a round of Baz IO | 20260506 13:41:02.500 | INFO | Populating RTMetrics report | 20260506 13:41:02.500 | INFO | RTMetrics report populated | 20260506 13:41:02.500 | INFO | Opening tmp RTMetrics json file | 20260506 13:41:02.501 | INFO | Serializing RTMetrics to json | 20260506 13:41:02.502 | INFO | Serialization done, swapping tmp file now... | 20260506 13:41:02.503 | INFO | RTMetrics done | 20260506 13:41:02.605 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.266 seconds | 20260506 13:41:02.606 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.35% / 0.02% / 60.64% | 20260506 13:41:02.606 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 3.00967% / 3.57628e-05% / 6.35783e-05% | 20260506 13:41:02.606 | INFO | Baz IO overall: | 20260506 13:41:02.606 | INFO | BytesWritten : 913044813079 | 20260506 13:41:02.606 | INFO | NumEvents : 441673198461 | 20260506 13:41:04.345 | INFO | Analyzing chunk frames = [1789952,1790464) | 20260506 13:41:04.345 | INFO | MemInfo: { | 20260506 13:41:04.345 | INFO | "gpu" : " 17 GB", | 20260506 13:41:04.345 | INFO | "huge" : "", | 20260506 13:41:04.345 | INFO | "huge_shared" : " 27 GB", | 20260506 13:41:04.345 | INFO | "malloc" : " 14 GB", | 20260506 13:41:04.345 | INFO | "pinned" : " 29 GB" | 20260506 13:41:04.345 | INFO | } | 20260506 13:41:04.345 | INFO | Perf report for the last 5.13894 seconds: | 20260506 13:41:04.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:41:04.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:04.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:04.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:04.345 | INFO | ANALYSIS : 63.5%, 3.0, 98.6ms, 0 | 20260506 13:41:04.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:04.346 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 13:41:04.346 | INFO | PRE_HQ : 61.2%, 1.9, 60.8ms, 0 | 20260506 13:41:04.346 | INFO | BAZWRITER : 62.9%, 3.8, 122.6ms, 0 | 20260506 13:41:04.346 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 13:41:04.346 | INFO | Uploaded 36GiB to GPU in 2.05305s (17.5349GiB/s) | 20260506 13:41:04.346 | INFO | Downloaded 38GiB to GPU in 2.21582s (17.1494GiB/s) | 20260506 13:41:09.485 | INFO | Analyzing chunk frames = [1790464,1790976) | 20260506 13:41:09.485 | INFO | MemInfo: { | 20260506 13:41:09.485 | INFO | "gpu" : " 17 GB", | 20260506 13:41:09.485 | INFO | "huge" : "", | 20260506 13:41:09.485 | INFO | "huge_shared" : " 27 GB", | 20260506 13:41:09.485 | INFO | "malloc" : " 14 GB", | 20260506 13:41:09.485 | INFO | "pinned" : " 29 GB" | 20260506 13:41:09.485 | INFO | } | 20260506 13:41:09.485 | INFO | Perf report for the last 5.13944 seconds: | 20260506 13:41:09.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:41:09.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:09.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:09.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:09.485 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:41:09.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:09.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:09.485 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:41:09.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:09.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:09.485 | INFO | Uploaded 36GiB to GPU in 2.03467s (17.6933GiB/s) | 20260506 13:41:09.485 | INFO | Downloaded 34GiB to GPU in 1.94428s (17.4872GiB/s) | 20260506 13:41:14.624 | INFO | Analyzing chunk frames = [1790976,1791488) | 20260506 13:41:14.624 | INFO | MemInfo: { | 20260506 13:41:14.624 | INFO | "gpu" : " 17 GB", | 20260506 13:41:14.624 | INFO | "huge" : "", | 20260506 13:41:14.624 | INFO | "huge_shared" : " 27 GB", | 20260506 13:41:14.624 | INFO | "malloc" : " 14 GB", | 20260506 13:41:14.624 | INFO | "pinned" : " 29 GB" | 20260506 13:41:14.624 | INFO | } | 20260506 13:41:14.625 | INFO | Perf report for the last 5.13961 seconds: | 20260506 13:41:14.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:41:14.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:14.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:14.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:14.625 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 13:41:14.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:14.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:14.625 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:41:14.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:14.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:14.625 | INFO | Uploaded 36GiB to GPU in 2.02035s (17.8187GiB/s) | 20260506 13:41:14.625 | INFO | Downloaded 34GiB to GPU in 1.94104s (17.5164GiB/s) | 20260506 13:41:19.764 | INFO | Analyzing chunk frames = [1791488,1792000) | 20260506 13:41:19.764 | INFO | MemInfo: { | 20260506 13:41:19.764 | INFO | "gpu" : " 17 GB", | 20260506 13:41:19.764 | INFO | "huge" : "", | 20260506 13:41:19.764 | INFO | "huge_shared" : " 27 GB", | 20260506 13:41:19.764 | INFO | "malloc" : " 14 GB", | 20260506 13:41:19.764 | INFO | "pinned" : " 29 GB" | 20260506 13:41:19.764 | INFO | } | 20260506 13:41:19.764 | INFO | Perf report for the last 5.13917 seconds: | 20260506 13:41:19.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:41:19.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:19.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:19.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:19.764 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 13:41:19.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:19.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:19.764 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:41:19.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:19.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:19.764 | INFO | Uploaded 36GiB to GPU in 1.99246s (18.0682GiB/s) | 20260506 13:41:19.764 | INFO | Downloaded 34GiB to GPU in 1.92252s (17.6851GiB/s) | 20260506 13:41:24.903 | INFO | Analyzing chunk frames = [1792000,1792512) | 20260506 13:41:24.903 | INFO | MemInfo: { | 20260506 13:41:24.903 | INFO | "gpu" : " 17 GB", | 20260506 13:41:24.903 | INFO | "huge" : "", | 20260506 13:41:24.903 | INFO | "huge_shared" : " 27 GB", | 20260506 13:41:24.903 | INFO | "malloc" : " 14 GB", | 20260506 13:41:24.903 | INFO | "pinned" : " 29 GB" | 20260506 13:41:24.903 | INFO | } | 20260506 13:41:24.903 | INFO | Perf report for the last 5.13931 seconds: | 20260506 13:41:24.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:41:24.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:24.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:24.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:24.903 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 13:41:24.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:24.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:24.903 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:41:24.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:24.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:24.903 | INFO | Uploaded 36GiB to GPU in 2.00313s (17.9719GiB/s) | 20260506 13:41:24.903 | INFO | Downloaded 34GiB to GPU in 1.93729s (17.5503GiB/s) | 20260506 13:41:30.042 | INFO | Analyzing chunk frames = [1792512,1793024) | 20260506 13:41:30.042 | INFO | MemInfo: { | 20260506 13:41:30.042 | INFO | "gpu" : " 17 GB", | 20260506 13:41:30.042 | INFO | "huge" : "", | 20260506 13:41:30.042 | INFO | "huge_shared" : " 27 GB", | 20260506 13:41:30.042 | INFO | "malloc" : " 14 GB", | 20260506 13:41:30.042 | INFO | "pinned" : " 29 GB" | 20260506 13:41:30.042 | INFO | } | 20260506 13:41:30.042 | INFO | Perf report for the last 5.13915 seconds: | 20260506 13:41:30.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:41:30.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:30.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:30.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:30.042 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:41:30.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:30.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:30.042 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260506 13:41:30.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:30.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:30.042 | INFO | Uploaded 36GiB to GPU in 2.09343s (17.1967GiB/s) | 20260506 13:41:30.042 | INFO | Downloaded 34GiB to GPU in 1.96255s (17.3244GiB/s) | 20260506 13:41:35.183 | INFO | Analyzing chunk frames = [1793024,1793536) | 20260506 13:41:35.183 | INFO | MemInfo: { | 20260506 13:41:35.183 | INFO | "gpu" : " 17 GB", | 20260506 13:41:35.183 | INFO | "huge" : "", | 20260506 13:41:35.183 | INFO | "huge_shared" : " 27 GB", | 20260506 13:41:35.183 | INFO | "malloc" : " 14 GB", | 20260506 13:41:35.183 | INFO | "pinned" : " 29 GB" | 20260506 13:41:35.183 | INFO | } | 20260506 13:41:35.183 | INFO | Perf report for the last 5.14074 seconds: | 20260506 13:41:35.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:41:35.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:35.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:35.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:35.183 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 13:41:35.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:35.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:35.183 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:41:35.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:35.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:35.183 | INFO | Uploaded 36GiB to GPU in 1.99173s (18.0747GiB/s) | 20260506 13:41:35.183 | INFO | Downloaded 34GiB to GPU in 2.00661s (16.944GiB/s) | 20260506 13:41:40.321 | INFO | Analyzing chunk frames = [1793536,1794048) | 20260506 13:41:40.321 | INFO | MemInfo: { | 20260506 13:41:40.321 | INFO | "gpu" : " 17 GB", | 20260506 13:41:40.321 | INFO | "huge" : "", | 20260506 13:41:40.321 | INFO | "huge_shared" : " 27 GB", | 20260506 13:41:40.321 | INFO | "malloc" : " 14 GB", | 20260506 13:41:40.321 | INFO | "pinned" : " 29 GB" | 20260506 13:41:40.321 | INFO | } | 20260506 13:41:40.321 | INFO | Perf report for the last 5.13814 seconds: | 20260506 13:41:40.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:41:40.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:40.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:40.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:40.321 | INFO | ANALYSIS : 62.7%, 3.0, 97.6ms, 0 | 20260506 13:41:40.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:40.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:40.321 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:41:40.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:40.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:40.321 | INFO | Uploaded 36GiB to GPU in 2.07724s (17.3307GiB/s) | 20260506 13:41:40.321 | INFO | Downloaded 34GiB to GPU in 1.99671s (17.028GiB/s) | 20260506 13:41:40.321 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:41:40.321 | INFO | Total mem usage: 17.4 GiB | 20260506 13:41:40.321 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:41:40.321 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -20.9%) | 20260506 13:41:40.321 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:41:40.321 | INFO | High Water Breakdown: | 20260506 13:41:40.321 | INFO | 59.8% : 8.61 GiB ( 78.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:41:40.321 | INFO | 40.1% : 5.78 GiB ( 86.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:41:40.321 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:41:40.321 | INFO | ------------------------------------------ | 20260506 13:41:40.321 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:41:40.321 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:41:40.321 | INFO | Total mem usage: 26.5 GiB | 20260506 13:41:40.321 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:41:40.321 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:41:40.322 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:41:40.322 | INFO | High Water Breakdown: | 20260506 13:41:40.322 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:41:40.322 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:41:40.322 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:41:40.322 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:41:40.322 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:41:40.322 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:41:40.322 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:41:40.322 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:41:40.322 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:41:40.322 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:41:40.322 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:41:40.322 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:41:40.322 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:41:40.322 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:41:40.322 | INFO | ------------------------------------------ | 20260506 13:41:40.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:41:40.322 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:41:40.322 | INFO | Total mem usage: 25.6 GiB | 20260506 13:41:40.322 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:41:40.322 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:41:40.322 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:41:40.322 | INFO | High Water Breakdown: | 20260506 13:41:40.322 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:41:40.322 | INFO | ------------------------------------------ | 20260506 13:41:40.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:41:40.322 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:41:40.322 | INFO | Total mem usage: 15.7 GiB | 20260506 13:41:40.322 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:41:40.322 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:41:40.322 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:41:40.322 | INFO | High Water Breakdown: | 20260506 13:41:40.322 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:41:40.322 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:41:40.322 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:41:40.322 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:41:40.322 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:41:40.322 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:41:40.322 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:41:40.322 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:41:40.322 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:41:40.322 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:41:40.322 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:41:40.322 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:41:40.322 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:41:40.322 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:41:40.322 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:41:40.322 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:41:40.322 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:41:40.322 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:41:40.322 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:41:40.322 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:41:40.322 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:41:40.322 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:41:40.322 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:41:40.322 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:41:40.322 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:41:40.322 | INFO | ------------------------------------------ | 20260506 13:41:40.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:41:40.322 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:41:40.322 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:41:40.322 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:41:40.322 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.57e+01, 1.37e+01 / 2.33e+00 | 20260506 13:41:40.322 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.02e-01, 1.29e-02 / 7.03e-03 | 20260506 13:41:40.322 | INFO | PerfReport: 11.56%, Baseline -- 3.66e+00 / 4.42e+00, 5.71e-01 / 3.90e-01 | 20260506 13:41:40.322 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.45e+00, 1.88e-01 / 1.57e-01 | 20260506 13:41:40.322 | INFO | PerfReport: 15.83%, DME -- 4.00e+01 / 4.41e+01, 3.89e+00 / 2.60e+00 | 20260506 13:41:40.322 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.44e+00 / 1.99e+00 | 20260506 13:41:40.322 | INFO | PerfReport: 7.53%, PulseAccumulating -- 2.38e+00 / 3.03e+00, 4.22e-01 / 2.37e-01 | 20260506 13:41:40.322 | INFO | PerfReport: 3.66%, Metrics -- 1.16e+00 / 1.54e+00, 2.94e-01 / 1.52e-01 | 20260506 13:41:40.322 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.06e-01 / 4.61e-01, 6.28e-02 / 3.40e-02 | 20260506 13:41:40.322 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:41:40.322 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:41:40.322 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:41:40.322 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 13:41:40.322 | INFO | PerfReport: 21.47%, Upload -- 2.12e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260506 13:41:40.322 | INFO | PerfReport: 20.96%, Download -- 2.07e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 13:41:40.322 | INFO | PerfReport: 32.10%, Compute -- 3.16e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 13:41:40.322 | INFO | PerfReport: 1.78%, LockUpload -- 1.76e+00 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 13:41:40.322 | INFO | PerfReport: 1.02%, LockDownload -- 1.01e+00 / 0.00e+00, 2.62e+00 / 0.00e+00 | 20260506 13:41:40.322 | INFO | PerfReport: 22.65%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 13:41:40.322 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260506 13:41:40.451 | INFO | Starting a round of Baz IO | 20260506 13:41:43.617 | INFO | Populating RTMetrics report | 20260506 13:41:43.618 | INFO | RTMetrics report populated | 20260506 13:41:43.618 | INFO | Opening tmp RTMetrics json file | 20260506 13:41:43.618 | INFO | Serializing RTMetrics to json | 20260506 13:41:43.620 | INFO | Serialization done, swapping tmp file now... | 20260506 13:41:43.620 | INFO | RTMetrics done | 20260506 13:41:43.715 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.264 seconds | 20260506 13:41:43.715 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.33% / 0.02% / 60.65% | 20260506 13:41:43.716 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.99959% / 2.38419e-05% / 5.96046e-05% | 20260506 13:41:43.716 | INFO | Baz IO overall: | 20260506 13:41:43.716 | INFO | BytesWritten : 915310047260 | 20260506 13:41:43.716 | INFO | NumEvents : 442771576243 | 20260506 13:41:45.460 | INFO | Analyzing chunk frames = [1794048,1794560) | 20260506 13:41:45.460 | INFO | MemInfo: { | 20260506 13:41:45.460 | INFO | "gpu" : " 17 GB", | 20260506 13:41:45.460 | INFO | "huge" : "", | 20260506 13:41:45.460 | INFO | "huge_shared" : " 27 GB", | 20260506 13:41:45.460 | INFO | "malloc" : " 14 GB", | 20260506 13:41:45.460 | INFO | "pinned" : " 29 GB" | 20260506 13:41:45.460 | INFO | } | 20260506 13:41:45.460 | INFO | Perf report for the last 5.13916 seconds: | 20260506 13:41:45.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:41:45.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:45.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:45.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:45.460 | INFO | ANALYSIS : 63.6%, 3.0, 98.7ms, 0 | 20260506 13:41:45.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:45.460 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:41:45.460 | INFO | PRE_HQ : 61.4%, 1.9, 60.7ms, 0 | 20260506 13:41:45.460 | INFO | BAZWRITER : 62.9%, 3.7, 120.7ms, 0 | 20260506 13:41:45.460 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 13:41:45.460 | INFO | Uploaded 36GiB to GPU in 2.07093s (17.3835GiB/s) | 20260506 13:41:45.461 | INFO | Downloaded 38GiB to GPU in 2.22875s (17.0499GiB/s) | 20260506 13:41:50.600 | INFO | Analyzing chunk frames = [1794560,1795072) | 20260506 13:41:50.600 | INFO | MemInfo: { | 20260506 13:41:50.600 | INFO | "gpu" : " 17 GB", | 20260506 13:41:50.600 | INFO | "huge" : "", | 20260506 13:41:50.600 | INFO | "huge_shared" : " 27 GB", | 20260506 13:41:50.600 | INFO | "malloc" : " 14 GB", | 20260506 13:41:50.600 | INFO | "pinned" : " 29 GB" | 20260506 13:41:50.600 | INFO | } | 20260506 13:41:50.600 | INFO | Perf report for the last 5.13953 seconds: | 20260506 13:41:50.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:41:50.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:50.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:50.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:50.600 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:41:50.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:50.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:50.600 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:41:50.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:50.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:50.600 | INFO | Uploaded 36GiB to GPU in 2.03762s (17.6676GiB/s) | 20260506 13:41:50.600 | INFO | Downloaded 34GiB to GPU in 1.93981s (17.5274GiB/s) | 20260506 13:41:55.739 | INFO | Analyzing chunk frames = [1795072,1795584) | 20260506 13:41:55.739 | INFO | MemInfo: { | 20260506 13:41:55.739 | INFO | "gpu" : " 17 GB", | 20260506 13:41:55.739 | INFO | "huge" : "", | 20260506 13:41:55.739 | INFO | "huge_shared" : " 27 GB", | 20260506 13:41:55.739 | INFO | "malloc" : " 14 GB", | 20260506 13:41:55.739 | INFO | "pinned" : " 29 GB" | 20260506 13:41:55.739 | INFO | } | 20260506 13:41:55.739 | INFO | Perf report for the last 5.13964 seconds: | 20260506 13:41:55.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:41:55.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:55.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:55.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:55.739 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 13:41:55.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:55.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:55.739 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:41:55.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:55.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:41:55.740 | INFO | Uploaded 36GiB to GPU in 2.02207s (17.8035GiB/s) | 20260506 13:41:55.740 | INFO | Downloaded 34GiB to GPU in 1.94331s (17.4959GiB/s) | 20260506 13:42:00.879 | INFO | Analyzing chunk frames = [1795584,1796096) | 20260506 13:42:00.879 | INFO | MemInfo: { | 20260506 13:42:00.879 | INFO | "gpu" : " 17 GB", | 20260506 13:42:00.879 | INFO | "huge" : "", | 20260506 13:42:00.879 | INFO | "huge_shared" : " 27 GB", | 20260506 13:42:00.879 | INFO | "malloc" : " 14 GB", | 20260506 13:42:00.879 | INFO | "pinned" : " 29 GB" | 20260506 13:42:00.879 | INFO | } | 20260506 13:42:00.879 | INFO | Perf report for the last 5.13919 seconds: | 20260506 13:42:00.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:42:00.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:00.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:00.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:00.879 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 13:42:00.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:00.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:00.879 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:42:00.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:00.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:00.879 | INFO | Uploaded 36GiB to GPU in 1.98846s (18.1044GiB/s) | 20260506 13:42:00.879 | INFO | Downloaded 34GiB to GPU in 1.92162s (17.6934GiB/s) | 20260506 13:42:06.018 | INFO | Analyzing chunk frames = [1796096,1796608) | 20260506 13:42:06.018 | INFO | MemInfo: { | 20260506 13:42:06.018 | INFO | "gpu" : " 17 GB", | 20260506 13:42:06.018 | INFO | "huge" : "", | 20260506 13:42:06.018 | INFO | "huge_shared" : " 27 GB", | 20260506 13:42:06.018 | INFO | "malloc" : " 14 GB", | 20260506 13:42:06.018 | INFO | "pinned" : " 29 GB" | 20260506 13:42:06.018 | INFO | } | 20260506 13:42:06.018 | INFO | Perf report for the last 5.13921 seconds: | 20260506 13:42:06.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:42:06.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:06.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:06.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:06.018 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 13:42:06.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:06.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:06.018 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 13:42:06.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:06.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:06.018 | INFO | Uploaded 36GiB to GPU in 1.97233s (18.2525GiB/s) | 20260506 13:42:06.018 | INFO | Downloaded 34GiB to GPU in 1.93266s (17.5924GiB/s) | 20260506 13:42:11.157 | INFO | Analyzing chunk frames = [1796608,1797120) | 20260506 13:42:11.157 | INFO | MemInfo: { | 20260506 13:42:11.157 | INFO | "gpu" : " 17 GB", | 20260506 13:42:11.157 | INFO | "huge" : "", | 20260506 13:42:11.157 | INFO | "huge_shared" : " 27 GB", | 20260506 13:42:11.157 | INFO | "malloc" : " 14 GB", | 20260506 13:42:11.157 | INFO | "pinned" : " 29 GB" | 20260506 13:42:11.157 | INFO | } | 20260506 13:42:11.157 | INFO | Perf report for the last 5.13932 seconds: | 20260506 13:42:11.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:42:11.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:11.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:11.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:11.157 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:42:11.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:11.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:11.157 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 13:42:11.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:11.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:11.157 | INFO | Uploaded 36GiB to GPU in 2.09228s (17.2061GiB/s) | 20260506 13:42:11.157 | INFO | Downloaded 34GiB to GPU in 1.96424s (17.3095GiB/s) | 20260506 13:42:16.296 | INFO | Analyzing chunk frames = [1797120,1797632) | 20260506 13:42:16.296 | INFO | MemInfo: { | 20260506 13:42:16.296 | INFO | "gpu" : " 17 GB", | 20260506 13:42:16.296 | INFO | "huge" : "", | 20260506 13:42:16.296 | INFO | "huge_shared" : " 27 GB", | 20260506 13:42:16.296 | INFO | "malloc" : " 14 GB", | 20260506 13:42:16.296 | INFO | "pinned" : " 29 GB" | 20260506 13:42:16.296 | INFO | } | 20260506 13:42:16.296 | INFO | Perf report for the last 5.13921 seconds: | 20260506 13:42:16.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:42:16.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:16.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:16.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:16.296 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 13:42:16.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:16.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:16.297 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 13:42:16.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:16.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:16.297 | INFO | Uploaded 36GiB to GPU in 1.99755s (18.022GiB/s) | 20260506 13:42:16.297 | INFO | Downloaded 34GiB to GPU in 2.03789s (16.6839GiB/s) | 20260506 13:42:21.436 | INFO | Analyzing chunk frames = [1797632,1798144) | 20260506 13:42:21.436 | INFO | MemInfo: { | 20260506 13:42:21.436 | INFO | "gpu" : " 17 GB", | 20260506 13:42:21.436 | INFO | "huge" : "", | 20260506 13:42:21.436 | INFO | "huge_shared" : " 27 GB", | 20260506 13:42:21.436 | INFO | "malloc" : " 14 GB", | 20260506 13:42:21.436 | INFO | "pinned" : " 29 GB" | 20260506 13:42:21.436 | INFO | } | 20260506 13:42:21.436 | INFO | Perf report for the last 5.13983 seconds: | 20260506 13:42:21.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:42:21.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:21.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:21.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:21.436 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 13:42:21.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:21.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:21.436 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:42:21.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:21.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:21.437 | INFO | Uploaded 36GiB to GPU in 2.0602s (17.474GiB/s) | 20260506 13:42:21.437 | INFO | Downloaded 34GiB to GPU in 1.97903s (17.1801GiB/s) | 20260506 13:42:21.571 | INFO | Starting a round of Baz IO | 20260506 13:42:24.721 | INFO | Populating RTMetrics report | 20260506 13:42:24.722 | INFO | RTMetrics report populated | 20260506 13:42:24.722 | INFO | Opening tmp RTMetrics json file | 20260506 13:42:24.722 | INFO | Serializing RTMetrics to json | 20260506 13:42:24.723 | INFO | Serialization done, swapping tmp file now... | 20260506 13:42:24.724 | INFO | RTMetrics done | 20260506 13:42:24.828 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.257 seconds | 20260506 13:42:24.828 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.31% / 0.02% / 60.67% | 20260506 13:42:24.828 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.99426% / 1.98682e-05% / 4.76837e-05% | 20260506 13:42:24.828 | INFO | Baz IO overall: | 20260506 13:42:24.828 | INFO | BytesWritten : 917575967201 | 20260506 13:42:24.828 | INFO | NumEvents : 443870583637 | 20260506 13:42:26.575 | INFO | Analyzing chunk frames = [1798144,1798656) | 20260506 13:42:26.575 | INFO | MemInfo: { | 20260506 13:42:26.575 | INFO | "gpu" : " 17 GB", | 20260506 13:42:26.576 | INFO | "huge" : "", | 20260506 13:42:26.576 | INFO | "huge_shared" : " 27 GB", | 20260506 13:42:26.576 | INFO | "malloc" : " 14 GB", | 20260506 13:42:26.576 | INFO | "pinned" : " 29 GB" | 20260506 13:42:26.576 | INFO | } | 20260506 13:42:26.576 | INFO | Perf report for the last 5.13956 seconds: | 20260506 13:42:26.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:42:26.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:26.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:26.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:26.576 | INFO | ANALYSIS : 63.3%, 3.0, 98.1ms, 0 | 20260506 13:42:26.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:26.576 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:42:26.576 | INFO | PRE_HQ : 61.1%, 1.9, 60.7ms, 0 | 20260506 13:42:26.576 | INFO | BAZWRITER : 62.7%, 3.8, 122.5ms, 0 | 20260506 13:42:26.576 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:42:26.576 | INFO | Uploaded 36GiB to GPU in 2.04545s (17.6001GiB/s) | 20260506 13:42:26.576 | INFO | Downloaded 38GiB to GPU in 2.20636s (17.2229GiB/s) | 20260506 13:42:31.714 | INFO | Analyzing chunk frames = [1798656,1799168) | 20260506 13:42:31.715 | INFO | MemInfo: { | 20260506 13:42:31.715 | INFO | "gpu" : " 17 GB", | 20260506 13:42:31.715 | INFO | "huge" : "", | 20260506 13:42:31.715 | INFO | "huge_shared" : " 27 GB", | 20260506 13:42:31.715 | INFO | "malloc" : " 14 GB", | 20260506 13:42:31.715 | INFO | "pinned" : " 29 GB" | 20260506 13:42:31.715 | INFO | } | 20260506 13:42:31.715 | INFO | Perf report for the last 5.13881 seconds: | 20260506 13:42:31.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:42:31.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:31.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:31.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:31.715 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 13:42:31.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:31.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:31.715 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:42:31.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:31.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:31.715 | INFO | Uploaded 36GiB to GPU in 2.00636s (17.943GiB/s) | 20260506 13:42:31.715 | INFO | Downloaded 34GiB to GPU in 1.93817s (17.5424GiB/s) | 20260506 13:42:36.854 | INFO | Analyzing chunk frames = [1799168,1799680) | 20260506 13:42:36.854 | INFO | MemInfo: { | 20260506 13:42:36.854 | INFO | "gpu" : " 17 GB", | 20260506 13:42:36.854 | INFO | "huge" : "", | 20260506 13:42:36.854 | INFO | "huge_shared" : " 27 GB", | 20260506 13:42:36.854 | INFO | "malloc" : " 14 GB", | 20260506 13:42:36.854 | INFO | "pinned" : " 29 GB" | 20260506 13:42:36.854 | INFO | } | 20260506 13:42:36.854 | INFO | Perf report for the last 5.13975 seconds: | 20260506 13:42:36.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:42:36.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:36.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:36.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:36.854 | INFO | ANALYSIS : 61.5%, 3.0, 95.2ms, 0 | 20260506 13:42:36.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:36.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:36.855 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:42:36.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:36.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:36.855 | INFO | Uploaded 36GiB to GPU in 2.02015s (17.8204GiB/s) | 20260506 13:42:36.855 | INFO | Downloaded 34GiB to GPU in 1.93768s (17.5468GiB/s) | 20260506 13:42:41.993 | INFO | Analyzing chunk frames = [1799680,1800192) | 20260506 13:42:41.993 | INFO | MemInfo: { | 20260506 13:42:41.993 | INFO | "gpu" : " 17 GB", | 20260506 13:42:41.993 | INFO | "huge" : "", | 20260506 13:42:41.993 | INFO | "huge_shared" : " 27 GB", | 20260506 13:42:41.993 | INFO | "malloc" : " 14 GB", | 20260506 13:42:41.993 | INFO | "pinned" : " 29 GB" | 20260506 13:42:41.993 | INFO | } | 20260506 13:42:41.993 | INFO | Perf report for the last 5.13873 seconds: | 20260506 13:42:41.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:42:41.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:41.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:41.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:41.993 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 13:42:41.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:41.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:41.993 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:42:41.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:41.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:41.993 | INFO | Uploaded 36GiB to GPU in 1.98448s (18.1408GiB/s) | 20260506 13:42:41.993 | INFO | Downloaded 34GiB to GPU in 1.91698s (17.7362GiB/s) | 20260506 13:42:47.133 | INFO | Analyzing chunk frames = [1800192,1800704) | 20260506 13:42:47.133 | INFO | MemInfo: { | 20260506 13:42:47.133 | INFO | "gpu" : " 17 GB", | 20260506 13:42:47.133 | INFO | "huge" : "", | 20260506 13:42:47.133 | INFO | "huge_shared" : " 27 GB", | 20260506 13:42:47.133 | INFO | "malloc" : " 14 GB", | 20260506 13:42:47.133 | INFO | "pinned" : " 29 GB" | 20260506 13:42:47.133 | INFO | } | 20260506 13:42:47.133 | INFO | Perf report for the last 5.13955 seconds: | 20260506 13:42:47.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:42:47.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:47.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:47.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:47.133 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 13:42:47.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:47.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:47.133 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:42:47.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:47.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:47.133 | INFO | Uploaded 36GiB to GPU in 2.03909s (17.6549GiB/s) | 20260506 13:42:47.133 | INFO | Downloaded 34GiB to GPU in 1.94642s (17.468GiB/s) | 20260506 13:42:52.272 | INFO | Analyzing chunk frames = [1800704,1801216) | 20260506 13:42:52.272 | INFO | MemInfo: { | 20260506 13:42:52.272 | INFO | "gpu" : " 17 GB", | 20260506 13:42:52.272 | INFO | "huge" : "", | 20260506 13:42:52.272 | INFO | "huge_shared" : " 27 GB", | 20260506 13:42:52.272 | INFO | "malloc" : " 14 GB", | 20260506 13:42:52.272 | INFO | "pinned" : " 29 GB" | 20260506 13:42:52.272 | INFO | } | 20260506 13:42:52.272 | INFO | Perf report for the last 5.13924 seconds: | 20260506 13:42:52.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:42:52.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:52.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:52.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:52.272 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:42:52.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:52.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:52.272 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:42:52.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:52.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:52.272 | INFO | Uploaded 36GiB to GPU in 2.1038s (17.1119GiB/s) | 20260506 13:42:52.272 | INFO | Downloaded 34GiB to GPU in 1.98036s (17.1686GiB/s) | 20260506 13:42:57.412 | INFO | Analyzing chunk frames = [1801216,1801728) | 20260506 13:42:57.412 | INFO | MemInfo: { | 20260506 13:42:57.412 | INFO | "gpu" : " 17 GB", | 20260506 13:42:57.412 | INFO | "huge" : "", | 20260506 13:42:57.412 | INFO | "huge_shared" : " 27 GB", | 20260506 13:42:57.412 | INFO | "malloc" : " 14 GB", | 20260506 13:42:57.412 | INFO | "pinned" : " 29 GB" | 20260506 13:42:57.412 | INFO | } | 20260506 13:42:57.412 | INFO | Perf report for the last 5.14032 seconds: | 20260506 13:42:57.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:42:57.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:57.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:57.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:57.412 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:42:57.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:57.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:57.412 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 13:42:57.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:57.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:42:57.413 | INFO | Uploaded 36GiB to GPU in 2.0001s (17.9991GiB/s) | 20260506 13:42:57.413 | INFO | Downloaded 34GiB to GPU in 2.03518s (16.7061GiB/s) | 20260506 13:43:02.551 | INFO | Analyzing chunk frames = [1801728,1802240) | 20260506 13:43:02.551 | INFO | MemInfo: { | 20260506 13:43:02.551 | INFO | "gpu" : " 17 GB", | 20260506 13:43:02.551 | INFO | "huge" : "", | 20260506 13:43:02.551 | INFO | "huge_shared" : " 27 GB", | 20260506 13:43:02.551 | INFO | "malloc" : " 14 GB", | 20260506 13:43:02.551 | INFO | "pinned" : " 29 GB" | 20260506 13:43:02.551 | INFO | } | 20260506 13:43:02.551 | INFO | Perf report for the last 5.13866 seconds: | 20260506 13:43:02.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:43:02.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:02.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:02.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:02.551 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 13:43:02.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:02.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:02.551 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 13:43:02.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:02.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:02.551 | INFO | Uploaded 36GiB to GPU in 2.03351s (17.7034GiB/s) | 20260506 13:43:02.551 | INFO | Downloaded 34GiB to GPU in 1.97623s (17.2044GiB/s) | 20260506 13:43:02.552 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:43:02.552 | INFO | Total mem usage: 17.4 GiB | 20260506 13:43:02.552 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:43:02.552 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -20.4%) | 20260506 13:43:02.552 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:43:02.552 | INFO | High Water Breakdown: | 20260506 13:43:02.552 | INFO | 59.9% : 8.66 GiB ( 79.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:43:02.552 | INFO | 40% : 5.79 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:43:02.552 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:43:02.552 | INFO | ------------------------------------------ | 20260506 13:43:02.552 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:43:02.552 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:43:02.552 | INFO | Total mem usage: 26.5 GiB | 20260506 13:43:02.552 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:43:02.552 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:43:02.552 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:43:02.552 | INFO | High Water Breakdown: | 20260506 13:43:02.552 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:43:02.552 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:43:02.552 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:43:02.552 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:43:02.552 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:43:02.552 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:43:02.552 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:43:02.552 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:43:02.552 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:43:02.552 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:43:02.552 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:43:02.552 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:43:02.552 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:43:02.552 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:43:02.552 | INFO | ------------------------------------------ | 20260506 13:43:02.552 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:43:02.552 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:43:02.552 | INFO | Total mem usage: 25.6 GiB | 20260506 13:43:02.552 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:43:02.552 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:43:02.552 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:43:02.552 | INFO | High Water Breakdown: | 20260506 13:43:02.552 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:43:02.552 | INFO | ------------------------------------------ | 20260506 13:43:02.552 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:43:02.552 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:43:02.552 | INFO | Total mem usage: 15.7 GiB | 20260506 13:43:02.552 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:43:02.552 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:43:02.552 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 13:43:02.552 | INFO | High Water Breakdown: | 20260506 13:43:02.552 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:43:02.552 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:43:02.552 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:43:02.552 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:43:02.552 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:43:02.552 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:43:02.552 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:43:02.552 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:43:02.552 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:43:02.552 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:43:02.552 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:43:02.552 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:43:02.552 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:43:02.552 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:43:02.552 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:43:02.552 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:43:02.552 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:43:02.552 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:43:02.552 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:43:02.552 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:43:02.552 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:43:02.552 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:43:02.552 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:43:02.552 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:43:02.552 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:43:02.552 | INFO | ------------------------------------------ | 20260506 13:43:02.552 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:43:02.552 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:43:02.552 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:43:02.552 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:43:02.552 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.65e+01, 1.33e+01 / 3.00e+00 | 20260506 13:43:02.552 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.01e-01, 1.05e-02 / 5.27e-03 | 20260506 13:43:02.552 | INFO | PerfReport: 11.57%, Baseline -- 3.65e+00 / 4.45e+00, 5.71e-01 / 3.36e-01 | 20260506 13:43:02.552 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.47e+00, 1.88e-01 / 1.47e-01 | 20260506 13:43:02.552 | INFO | PerfReport: 15.91%, DME -- 4.02e+01 / 4.48e+01, 4.38e+00 / 3.21e+00 | 20260506 13:43:02.552 | INFO | PerfReport: 56.51%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.34e+00 / 1.85e+00 | 20260506 13:43:02.552 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.37e+00 / 3.00e+00, 4.17e-01 / 2.49e-01 | 20260506 13:43:02.552 | INFO | PerfReport: 3.67%, Metrics -- 1.16e+00 / 1.55e+00, 2.99e-01 / 1.87e-01 | 20260506 13:43:02.552 | INFO | PerfReport: 1.29%, PulseSerializing -- 4.06e-01 / 4.65e-01, 6.32e-02 / 3.40e-02 | 20260506 13:43:02.553 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:43:02.553 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:43:02.553 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:43:02.553 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 13:43:02.553 | INFO | PerfReport: 21.47%, Upload -- 2.11e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260506 13:43:02.553 | INFO | PerfReport: 21.08%, Download -- 2.08e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 13:43:02.553 | INFO | PerfReport: 32.07%, Compute -- 3.16e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 13:43:02.553 | INFO | PerfReport: 1.77%, LockUpload -- 1.74e+00 / 0.00e+00, 6.91e+00 / 0.00e+00 | 20260506 13:43:02.553 | INFO | PerfReport: 1.01%, LockDownload -- 9.98e-01 / 0.00e+00, 2.64e+00 / 0.00e+00 | 20260506 13:43:02.553 | INFO | PerfReport: 22.59%, LockCompute -- 2.23e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 13:43:02.553 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260506 13:43:02.685 | INFO | Starting a round of Baz IO | 20260506 13:43:05.843 | INFO | Populating RTMetrics report | 20260506 13:43:05.844 | INFO | RTMetrics report populated | 20260506 13:43:05.844 | INFO | Opening tmp RTMetrics json file | 20260506 13:43:05.844 | INFO | Serializing RTMetrics to json | 20260506 13:43:05.846 | INFO | Serialization done, swapping tmp file now... | 20260506 13:43:05.846 | INFO | RTMetrics done | 20260506 13:43:05.947 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.262 seconds | 20260506 13:43:05.947 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.29% / 0.02% / 60.69% | 20260506 13:43:05.947 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.9824% / 5.5631e-05% / 5.5631e-05% | 20260506 13:43:05.947 | INFO | Baz IO overall: | 20260506 13:43:05.947 | INFO | BytesWritten : 919842117893 | 20260506 13:43:05.947 | INFO | NumEvents : 444969474449 | 20260506 13:43:07.690 | INFO | Analyzing chunk frames = [1802240,1802752) | 20260506 13:43:07.690 | INFO | MemInfo: { | 20260506 13:43:07.690 | INFO | "gpu" : " 17 GB", | 20260506 13:43:07.690 | INFO | "huge" : "", | 20260506 13:43:07.690 | INFO | "huge_shared" : " 27 GB", | 20260506 13:43:07.690 | INFO | "malloc" : " 14 GB", | 20260506 13:43:07.690 | INFO | "pinned" : " 29 GB" | 20260506 13:43:07.690 | INFO | } | 20260506 13:43:07.690 | INFO | Perf report for the last 5.13887 seconds: | 20260506 13:43:07.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:43:07.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:07.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:07.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:07.690 | INFO | ANALYSIS : 63.5%, 3.0, 98.5ms, 0 | 20260506 13:43:07.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:07.690 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 13:43:07.690 | INFO | PRE_HQ : 61.0%, 1.9, 60.4ms, 0 | 20260506 13:43:07.690 | INFO | BAZWRITER : 62.8%, 3.8, 121.8ms, 0 | 20260506 13:43:07.690 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:43:07.690 | INFO | Uploaded 36GiB to GPU in 2.04499s (17.604GiB/s) | 20260506 13:43:07.690 | INFO | Downloaded 38GiB to GPU in 2.21025s (17.1926GiB/s) | 20260506 13:43:12.829 | INFO | Analyzing chunk frames = [1802752,1803264) | 20260506 13:43:12.829 | INFO | MemInfo: { | 20260506 13:43:12.829 | INFO | "gpu" : " 17 GB", | 20260506 13:43:12.829 | INFO | "huge" : "", | 20260506 13:43:12.829 | INFO | "huge_shared" : " 27 GB", | 20260506 13:43:12.829 | INFO | "malloc" : " 14 GB", | 20260506 13:43:12.829 | INFO | "pinned" : " 29 GB" | 20260506 13:43:12.829 | INFO | } | 20260506 13:43:12.830 | INFO | Perf report for the last 5.13965 seconds: | 20260506 13:43:12.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:43:12.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:12.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:12.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:12.830 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:43:12.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:12.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:12.830 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 13:43:12.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:12.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:12.830 | INFO | Uploaded 36GiB to GPU in 2.02874s (17.745GiB/s) | 20260506 13:43:12.830 | INFO | Downloaded 34GiB to GPU in 1.93915s (17.5334GiB/s) | 20260506 13:43:17.969 | INFO | Analyzing chunk frames = [1803264,1803776) | 20260506 13:43:17.969 | INFO | MemInfo: { | 20260506 13:43:17.969 | INFO | "gpu" : " 17 GB", | 20260506 13:43:17.969 | INFO | "huge" : "", | 20260506 13:43:17.969 | INFO | "huge_shared" : " 27 GB", | 20260506 13:43:17.969 | INFO | "malloc" : " 14 GB", | 20260506 13:43:17.969 | INFO | "pinned" : " 29 GB" | 20260506 13:43:17.969 | INFO | } | 20260506 13:43:17.969 | INFO | Perf report for the last 5.13937 seconds: | 20260506 13:43:17.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:43:17.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:17.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:17.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:17.969 | INFO | ANALYSIS : 61.5%, 3.0, 95.2ms, 0 | 20260506 13:43:17.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:17.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:17.969 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 13:43:17.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:17.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:17.969 | INFO | Uploaded 36GiB to GPU in 2.01323s (17.8817GiB/s) | 20260506 13:43:17.969 | INFO | Downloaded 34GiB to GPU in 1.94224s (17.5056GiB/s) | 20260506 13:43:23.108 | INFO | Analyzing chunk frames = [1803776,1804288) | 20260506 13:43:23.108 | INFO | MemInfo: { | 20260506 13:43:23.108 | INFO | "gpu" : " 17 GB", | 20260506 13:43:23.108 | INFO | "huge" : "", | 20260506 13:43:23.108 | INFO | "huge_shared" : " 27 GB", | 20260506 13:43:23.108 | INFO | "malloc" : " 14 GB", | 20260506 13:43:23.108 | INFO | "pinned" : " 29 GB" | 20260506 13:43:23.108 | INFO | } | 20260506 13:43:23.108 | INFO | Perf report for the last 5.1395 seconds: | 20260506 13:43:23.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:43:23.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:23.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:23.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:23.109 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 13:43:23.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:23.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:23.109 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260506 13:43:23.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:23.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:23.109 | INFO | Uploaded 36GiB to GPU in 1.98135s (18.1694GiB/s) | 20260506 13:43:23.109 | INFO | Downloaded 34GiB to GPU in 1.91476s (17.7568GiB/s) | 20260506 13:43:28.247 | INFO | Analyzing chunk frames = [1804288,1804800) | 20260506 13:43:28.247 | INFO | MemInfo: { | 20260506 13:43:28.247 | INFO | "gpu" : " 17 GB", | 20260506 13:43:28.247 | INFO | "huge" : "", | 20260506 13:43:28.247 | INFO | "huge_shared" : " 27 GB", | 20260506 13:43:28.247 | INFO | "malloc" : " 14 GB", | 20260506 13:43:28.248 | INFO | "pinned" : " 29 GB" | 20260506 13:43:28.248 | INFO | } | 20260506 13:43:28.248 | INFO | Perf report for the last 5.13913 seconds: | 20260506 13:43:28.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:43:28.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:28.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:28.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:28.248 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260506 13:43:28.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:28.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:28.248 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:43:28.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:28.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:28.248 | INFO | Uploaded 36GiB to GPU in 2.03245s (17.7126GiB/s) | 20260506 13:43:28.248 | INFO | Downloaded 34GiB to GPU in 1.94164s (17.5109GiB/s) | 20260506 13:43:33.387 | INFO | Analyzing chunk frames = [1804800,1805312) | 20260506 13:43:33.387 | INFO | MemInfo: { | 20260506 13:43:33.387 | INFO | "gpu" : " 17 GB", | 20260506 13:43:33.387 | INFO | "huge" : "", | 20260506 13:43:33.387 | INFO | "huge_shared" : " 27 GB", | 20260506 13:43:33.387 | INFO | "malloc" : " 14 GB", | 20260506 13:43:33.387 | INFO | "pinned" : " 29 GB" | 20260506 13:43:33.387 | INFO | } | 20260506 13:43:33.387 | INFO | Perf report for the last 5.13951 seconds: | 20260506 13:43:33.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:43:33.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:33.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:33.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:33.387 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:43:33.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:33.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:33.387 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:43:33.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:33.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:33.387 | INFO | Uploaded 36GiB to GPU in 2.09561s (17.1788GiB/s) | 20260506 13:43:33.387 | INFO | Downloaded 34GiB to GPU in 1.96853s (17.2718GiB/s) | 20260506 13:43:38.526 | INFO | Analyzing chunk frames = [1805312,1805824) | 20260506 13:43:38.526 | INFO | MemInfo: { | 20260506 13:43:38.526 | INFO | "gpu" : " 17 GB", | 20260506 13:43:38.526 | INFO | "huge" : "", | 20260506 13:43:38.526 | INFO | "huge_shared" : " 27 GB", | 20260506 13:43:38.526 | INFO | "malloc" : " 14 GB", | 20260506 13:43:38.526 | INFO | "pinned" : " 29 GB" | 20260506 13:43:38.526 | INFO | } | 20260506 13:43:38.527 | INFO | Perf report for the last 5.13939 seconds: | 20260506 13:43:38.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:43:38.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:38.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:38.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:38.527 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 13:43:38.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:38.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:38.527 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:43:38.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:38.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:38.527 | INFO | Uploaded 36GiB to GPU in 1.99367s (18.0572GiB/s) | 20260506 13:43:38.527 | INFO | Downloaded 34GiB to GPU in 2.02886s (16.7582GiB/s) | 20260506 13:43:43.669 | INFO | Analyzing chunk frames = [1805824,1806336) | 20260506 13:43:43.669 | INFO | MemInfo: { | 20260506 13:43:43.669 | INFO | "gpu" : " 17 GB", | 20260506 13:43:43.669 | INFO | "huge" : "", | 20260506 13:43:43.669 | INFO | "huge_shared" : " 27 GB", | 20260506 13:43:43.669 | INFO | "malloc" : " 14 GB", | 20260506 13:43:43.669 | INFO | "pinned" : " 29 GB" | 20260506 13:43:43.669 | INFO | } | 20260506 13:43:43.669 | INFO | Perf report for the last 5.14247 seconds: | 20260506 13:43:43.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:43:43.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:43.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:43.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:43.669 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:43:43.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:43.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:43.669 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:43:43.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:43.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:43.669 | INFO | Uploaded 36GiB to GPU in 2.0395s (17.6514GiB/s) | 20260506 13:43:43.669 | INFO | Downloaded 34GiB to GPU in 1.97701s (17.1977GiB/s) | 20260506 13:43:43.807 | INFO | Starting a round of Baz IO | 20260506 13:43:46.951 | INFO | Populating RTMetrics report | 20260506 13:43:46.951 | INFO | RTMetrics report populated | 20260506 13:43:46.951 | INFO | Opening tmp RTMetrics json file | 20260506 13:43:46.952 | INFO | Serializing RTMetrics to json | 20260506 13:43:46.953 | INFO | Serialization done, swapping tmp file now... | 20260506 13:43:46.954 | INFO | RTMetrics done | 20260506 13:43:47.049 | INFO | Finished a round of Baz IO: Wrote 2.108 GiB in 3.242 seconds | 20260506 13:43:47.049 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.28% / 0.02% / 60.71% | 20260506 13:43:47.049 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.96965% / 3.17891e-05% / 8.34465e-05% | 20260506 13:43:47.049 | INFO | Baz IO overall: | 20260506 13:43:47.049 | INFO | BytesWritten : 922105839625 | 20260506 13:43:47.049 | INFO | NumEvents : 446066317692 | 20260506 13:43:48.805 | INFO | Analyzing chunk frames = [1806336,1806848) | 20260506 13:43:48.805 | INFO | MemInfo: { | 20260506 13:43:48.805 | INFO | "gpu" : " 17 GB", | 20260506 13:43:48.805 | INFO | "huge" : "", | 20260506 13:43:48.805 | INFO | "huge_shared" : " 27 GB", | 20260506 13:43:48.805 | INFO | "malloc" : " 14 GB", | 20260506 13:43:48.805 | INFO | "pinned" : " 29 GB" | 20260506 13:43:48.805 | INFO | } | 20260506 13:43:48.805 | INFO | Perf report for the last 5.13598 seconds: | 20260506 13:43:48.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:43:48.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:48.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:48.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:48.805 | INFO | ANALYSIS : 63.4%, 3.0, 98.2ms, 0 | 20260506 13:43:48.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:48.805 | INFO | RT_METRICS : 44.2%, 1.0, 23.1ms, 0 | 20260506 13:43:48.805 | INFO | PRE_HQ : 61.3%, 1.9, 61.2ms, 0 | 20260506 13:43:48.805 | INFO | BAZWRITER : 62.6%, 3.8, 122.6ms, 0 | 20260506 13:43:48.805 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 13:43:48.805 | INFO | Uploaded 36GiB to GPU in 2.0687s (17.4022GiB/s) | 20260506 13:43:48.805 | INFO | Downloaded 38GiB to GPU in 2.2147s (17.158GiB/s) | 20260506 13:43:53.945 | INFO | Analyzing chunk frames = [1806848,1807360) | 20260506 13:43:53.945 | INFO | MemInfo: { | 20260506 13:43:53.945 | INFO | "gpu" : " 17 GB", | 20260506 13:43:53.945 | INFO | "huge" : "", | 20260506 13:43:53.945 | INFO | "huge_shared" : " 27 GB", | 20260506 13:43:53.945 | INFO | "malloc" : " 14 GB", | 20260506 13:43:53.945 | INFO | "pinned" : " 29 GB" | 20260506 13:43:53.945 | INFO | } | 20260506 13:43:53.945 | INFO | Perf report for the last 5.13987 seconds: | 20260506 13:43:53.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:43:53.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:53.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:53.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:53.945 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 13:43:53.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:53.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:53.945 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 13:43:53.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:53.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:53.945 | INFO | Uploaded 36GiB to GPU in 2.06274s (17.4525GiB/s) | 20260506 13:43:53.945 | INFO | Downloaded 34GiB to GPU in 1.94018s (17.5241GiB/s) | 20260506 13:43:59.084 | INFO | Analyzing chunk frames = [1807360,1807872) | 20260506 13:43:59.084 | INFO | MemInfo: { | 20260506 13:43:59.084 | INFO | "gpu" : " 17 GB", | 20260506 13:43:59.084 | INFO | "huge" : "", | 20260506 13:43:59.084 | INFO | "huge_shared" : " 27 GB", | 20260506 13:43:59.084 | INFO | "malloc" : " 14 GB", | 20260506 13:43:59.084 | INFO | "pinned" : " 29 GB" | 20260506 13:43:59.084 | INFO | } | 20260506 13:43:59.084 | INFO | Perf report for the last 5.1391 seconds: | 20260506 13:43:59.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:43:59.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:59.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:59.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:59.084 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 13:43:59.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:59.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:59.084 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:43:59.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:59.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:43:59.084 | INFO | Uploaded 36GiB to GPU in 2.03126s (17.723GiB/s) | 20260506 13:43:59.084 | INFO | Downloaded 34GiB to GPU in 1.93942s (17.531GiB/s) | 20260506 13:44:04.223 | INFO | Analyzing chunk frames = [1807872,1808384) | 20260506 13:44:04.223 | INFO | MemInfo: { | 20260506 13:44:04.223 | INFO | "gpu" : " 17 GB", | 20260506 13:44:04.223 | INFO | "huge" : "", | 20260506 13:44:04.223 | INFO | "huge_shared" : " 27 GB", | 20260506 13:44:04.223 | INFO | "malloc" : " 14 GB", | 20260506 13:44:04.223 | INFO | "pinned" : " 29 GB" | 20260506 13:44:04.223 | INFO | } | 20260506 13:44:04.223 | INFO | Perf report for the last 5.13892 seconds: | 20260506 13:44:04.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:44:04.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:04.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:04.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:04.223 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:44:04.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:04.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:04.223 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:44:04.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:04.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:04.223 | INFO | Uploaded 36GiB to GPU in 2.01026s (17.9081GiB/s) | 20260506 13:44:04.223 | INFO | Downloaded 34GiB to GPU in 1.93856s (17.5388GiB/s) | 20260506 13:44:09.362 | INFO | Analyzing chunk frames = [1808384,1808896) | 20260506 13:44:09.362 | INFO | MemInfo: { | 20260506 13:44:09.362 | INFO | "gpu" : " 17 GB", | 20260506 13:44:09.362 | INFO | "huge" : "", | 20260506 13:44:09.362 | INFO | "huge_shared" : " 27 GB", | 20260506 13:44:09.362 | INFO | "malloc" : " 14 GB", | 20260506 13:44:09.362 | INFO | "pinned" : " 29 GB" | 20260506 13:44:09.362 | INFO | } | 20260506 13:44:09.362 | INFO | Perf report for the last 5.13953 seconds: | 20260506 13:44:09.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:44:09.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:09.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:09.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:09.363 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 13:44:09.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:09.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:09.363 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 13:44:09.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:09.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:09.363 | INFO | Uploaded 36GiB to GPU in 2.06927s (17.3975GiB/s) | 20260506 13:44:09.363 | INFO | Downloaded 34GiB to GPU in 1.95187s (17.4192GiB/s) | 20260506 13:44:14.502 | INFO | Analyzing chunk frames = [1808896,1809408) | 20260506 13:44:14.502 | INFO | MemInfo: { | 20260506 13:44:14.502 | INFO | "gpu" : " 17 GB", | 20260506 13:44:14.502 | INFO | "huge" : "", | 20260506 13:44:14.502 | INFO | "huge_shared" : " 27 GB", | 20260506 13:44:14.502 | INFO | "malloc" : " 14 GB", | 20260506 13:44:14.502 | INFO | "pinned" : " 29 GB" | 20260506 13:44:14.502 | INFO | } | 20260506 13:44:14.503 | INFO | Perf report for the last 5.14005 seconds: | 20260506 13:44:14.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:44:14.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:14.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:14.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:14.503 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 13:44:14.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:14.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:14.503 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260506 13:44:14.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:14.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:14.503 | INFO | Uploaded 36GiB to GPU in 2.04816s (17.5768GiB/s) | 20260506 13:44:14.503 | INFO | Downloaded 34GiB to GPU in 1.90861s (17.814GiB/s) | 20260506 13:44:19.642 | INFO | Analyzing chunk frames = [1809408,1809920) | 20260506 13:44:19.642 | INFO | MemInfo: { | 20260506 13:44:19.642 | INFO | "gpu" : " 17 GB", | 20260506 13:44:19.642 | INFO | "huge" : "", | 20260506 13:44:19.642 | INFO | "huge_shared" : " 27 GB", | 20260506 13:44:19.642 | INFO | "malloc" : " 14 GB", | 20260506 13:44:19.642 | INFO | "pinned" : " 29 GB" | 20260506 13:44:19.642 | INFO | } | 20260506 13:44:19.642 | INFO | Perf report for the last 5.13964 seconds: | 20260506 13:44:19.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:44:19.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:19.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:19.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:19.642 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:44:19.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:19.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:19.642 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 13:44:19.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:19.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:19.642 | INFO | Uploaded 36GiB to GPU in 2.02131s (17.8103GiB/s) | 20260506 13:44:19.642 | INFO | Downloaded 34GiB to GPU in 1.96679s (17.2871GiB/s) | 20260506 13:44:24.780 | INFO | Analyzing chunk frames = [1809920,1810432) | 20260506 13:44:24.780 | INFO | MemInfo: { | 20260506 13:44:24.780 | INFO | "gpu" : " 17 GB", | 20260506 13:44:24.780 | INFO | "huge" : "", | 20260506 13:44:24.780 | INFO | "huge_shared" : " 27 GB", | 20260506 13:44:24.780 | INFO | "malloc" : " 14 GB", | 20260506 13:44:24.780 | INFO | "pinned" : " 29 GB" | 20260506 13:44:24.780 | INFO | } | 20260506 13:44:24.780 | INFO | Perf report for the last 5.1383 seconds: | 20260506 13:44:24.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:44:24.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:24.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:24.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:24.781 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 13:44:24.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:24.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:24.781 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:44:24.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:24.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:24.781 | INFO | Uploaded 36GiB to GPU in 2.04195s (17.6302GiB/s) | 20260506 13:44:24.781 | INFO | Downloaded 34GiB to GPU in 1.97164s (17.2445GiB/s) | 20260506 13:44:24.781 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:44:24.781 | INFO | Total mem usage: 17.4 GiB | 20260506 13:44:24.781 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:44:24.781 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -20.3%) | 20260506 13:44:24.781 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:44:24.781 | INFO | High Water Breakdown: | 20260506 13:44:24.781 | INFO | 59.9% : 8.67 GiB ( 79.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:44:24.781 | INFO | 40% : 5.79 GiB ( 86.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:44:24.781 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:44:24.781 | INFO | ------------------------------------------ | 20260506 13:44:24.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:44:24.781 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:44:24.781 | INFO | Total mem usage: 26.5 GiB | 20260506 13:44:24.781 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:44:24.781 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:44:24.781 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:44:24.781 | INFO | High Water Breakdown: | 20260506 13:44:24.781 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:44:24.781 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:44:24.781 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:44:24.781 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:44:24.781 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:44:24.781 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:44:24.781 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:44:24.781 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:44:24.781 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:44:24.781 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:44:24.781 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:44:24.781 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:44:24.781 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:44:24.781 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:44:24.781 | INFO | ------------------------------------------ | 20260506 13:44:24.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:44:24.781 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:44:24.781 | INFO | Total mem usage: 25.6 GiB | 20260506 13:44:24.781 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:44:24.781 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:44:24.781 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:44:24.781 | INFO | High Water Breakdown: | 20260506 13:44:24.781 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:44:24.781 | INFO | ------------------------------------------ | 20260506 13:44:24.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:44:24.781 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:44:24.781 | INFO | Total mem usage: 15.7 GiB | 20260506 13:44:24.781 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:44:24.781 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:44:24.781 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:44:24.781 | INFO | High Water Breakdown: | 20260506 13:44:24.781 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:44:24.781 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:44:24.781 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:44:24.781 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:44:24.781 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:44:24.781 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:44:24.781 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:44:24.781 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:44:24.781 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:44:24.781 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:44:24.781 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:44:24.781 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:44:24.781 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:44:24.781 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:44:24.781 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:44:24.781 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:44:24.781 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:44:24.781 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:44:24.781 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:44:24.781 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:44:24.781 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:44:24.781 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:44:24.781 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:44:24.781 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:44:24.781 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:44:24.781 | INFO | ------------------------------------------ | 20260506 13:44:24.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:44:24.782 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:44:24.782 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:44:24.782 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:44:24.782 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.59e+01, 1.37e+01 / 2.63e+00 | 20260506 13:44:24.782 | INFO | PerfReport: 0.29%, AggregateStats -- 9.24e-02 / 1.01e-01, 9.98e-03 / 6.33e-03 | 20260506 13:44:24.782 | INFO | PerfReport: 11.58%, Baseline -- 3.66e+00 / 4.43e+00, 5.76e-01 / 3.69e-01 | 20260506 13:44:24.782 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.45e+00, 1.91e-01 / 1.46e-01 | 20260506 13:44:24.782 | INFO | PerfReport: 15.86%, DME -- 4.01e+01 / 4.44e+01, 4.00e+00 / 2.80e+00 | 20260506 13:44:24.782 | INFO | PerfReport: 56.56%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.41e+00 / 1.53e+00 | 20260506 13:44:24.782 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.37e+00 / 3.00e+00, 4.18e-01 / 2.44e-01 | 20260506 13:44:24.782 | INFO | PerfReport: 3.66%, Metrics -- 1.16e+00 / 1.53e+00, 2.93e-01 / 1.55e-01 | 20260506 13:44:24.782 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.06e-01 / 4.63e-01, 6.28e-02 / 3.03e-02 | 20260506 13:44:24.782 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:44:24.782 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:44:24.782 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:44:24.782 | INFO | PerfReport: Full Execution -- 9.85e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 13:44:24.782 | INFO | PerfReport: 21.53%, Upload -- 2.12e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260506 13:44:24.782 | INFO | PerfReport: 20.98%, Download -- 2.07e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 13:44:24.782 | INFO | PerfReport: 32.09%, Compute -- 3.16e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 13:44:24.782 | INFO | PerfReport: 1.85%, LockUpload -- 1.82e+00 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 13:44:24.782 | INFO | PerfReport: 1.00%, LockDownload -- 9.84e-01 / 0.00e+00, 2.63e+00 / 0.00e+00 | 20260506 13:44:24.782 | INFO | PerfReport: 22.54%, LockCompute -- 2.22e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 13:44:24.782 | INFO | PerfReport: 0.01%, Misc -- 1.48e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260506 13:44:24.907 | INFO | Starting a round of Baz IO | 20260506 13:44:28.069 | INFO | Populating RTMetrics report | 20260506 13:44:28.069 | INFO | RTMetrics report populated | 20260506 13:44:28.069 | INFO | Opening tmp RTMetrics json file | 20260506 13:44:28.070 | INFO | Serializing RTMetrics to json | 20260506 13:44:28.071 | INFO | Serialization done, swapping tmp file now... | 20260506 13:44:28.072 | INFO | RTMetrics done | 20260506 13:44:28.178 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.27 seconds | 20260506 13:44:28.178 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.26% / 0.02% / 60.72% | 20260506 13:44:28.178 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.97% / 3.17891e-05% / 7.15256e-05% | 20260506 13:44:28.178 | INFO | Baz IO overall: | 20260506 13:44:28.178 | INFO | BytesWritten : 924371789609 | 20260506 13:44:28.178 | INFO | NumEvents : 447164959036 | 20260506 13:44:29.919 | INFO | Analyzing chunk frames = [1810432,1810944) | 20260506 13:44:29.920 | INFO | MemInfo: { | 20260506 13:44:29.920 | INFO | "gpu" : " 17 GB", | 20260506 13:44:29.920 | INFO | "huge" : "", | 20260506 13:44:29.920 | INFO | "huge_shared" : " 27 GB", | 20260506 13:44:29.920 | INFO | "malloc" : " 14 GB", | 20260506 13:44:29.920 | INFO | "pinned" : " 29 GB" | 20260506 13:44:29.920 | INFO | } | 20260506 13:44:29.920 | INFO | Perf report for the last 5.13918 seconds: | 20260506 13:44:29.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:44:29.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:29.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:29.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:29.920 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260506 13:44:29.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:29.920 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 13:44:29.920 | INFO | PRE_HQ : 61.2%, 1.9, 60.5ms, 0 | 20260506 13:44:29.920 | INFO | BAZWRITER : 63.0%, 3.7, 120.6ms, 0 | 20260506 13:44:29.920 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 13:44:29.920 | INFO | Uploaded 36GiB to GPU in 2.06579s (17.4268GiB/s) | 20260506 13:44:29.920 | INFO | Downloaded 38GiB to GPU in 2.21848s (17.1288GiB/s) | 20260506 13:44:35.059 | INFO | Analyzing chunk frames = [1810944,1811456) | 20260506 13:44:35.059 | INFO | MemInfo: { | 20260506 13:44:35.059 | INFO | "gpu" : " 17 GB", | 20260506 13:44:35.059 | INFO | "huge" : "", | 20260506 13:44:35.059 | INFO | "huge_shared" : " 27 GB", | 20260506 13:44:35.059 | INFO | "malloc" : " 14 GB", | 20260506 13:44:35.059 | INFO | "pinned" : " 29 GB" | 20260506 13:44:35.059 | INFO | } | 20260506 13:44:35.059 | INFO | Perf report for the last 5.13916 seconds: | 20260506 13:44:35.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:44:35.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:35.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:35.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:35.059 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:44:35.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:35.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:35.059 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 13:44:35.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:35.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:35.059 | INFO | Uploaded 36GiB to GPU in 2.04698s (17.5869GiB/s) | 20260506 13:44:35.059 | INFO | Downloaded 34GiB to GPU in 1.94377s (17.4917GiB/s) | 20260506 13:44:40.198 | INFO | Analyzing chunk frames = [1811456,1811968) | 20260506 13:44:40.198 | INFO | MemInfo: { | 20260506 13:44:40.198 | INFO | "gpu" : " 17 GB", | 20260506 13:44:40.198 | INFO | "huge" : "", | 20260506 13:44:40.198 | INFO | "huge_shared" : " 27 GB", | 20260506 13:44:40.198 | INFO | "malloc" : " 14 GB", | 20260506 13:44:40.198 | INFO | "pinned" : " 29 GB" | 20260506 13:44:40.198 | INFO | } | 20260506 13:44:40.198 | INFO | Perf report for the last 5.13952 seconds: | 20260506 13:44:40.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:44:40.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:40.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:40.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:40.198 | INFO | ANALYSIS : 61.6%, 3.0, 95.3ms, 0 | 20260506 13:44:40.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:40.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:40.198 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:44:40.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:40.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:40.199 | INFO | Uploaded 36GiB to GPU in 2.02537s (17.7745GiB/s) | 20260506 13:44:40.199 | INFO | Downloaded 34GiB to GPU in 1.9419s (17.5086GiB/s) | 20260506 13:44:45.337 | INFO | Analyzing chunk frames = [1811968,1812480) | 20260506 13:44:45.337 | INFO | MemInfo: { | 20260506 13:44:45.337 | INFO | "gpu" : " 17 GB", | 20260506 13:44:45.337 | INFO | "huge" : "", | 20260506 13:44:45.338 | INFO | "huge_shared" : " 27 GB", | 20260506 13:44:45.338 | INFO | "malloc" : " 14 GB", | 20260506 13:44:45.338 | INFO | "pinned" : " 29 GB" | 20260506 13:44:45.338 | INFO | } | 20260506 13:44:45.338 | INFO | Perf report for the last 5.13923 seconds: | 20260506 13:44:45.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:44:45.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:45.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:45.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:45.338 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:44:45.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:45.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:45.338 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:44:45.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:45.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:45.338 | INFO | Uploaded 36GiB to GPU in 1.9892s (18.0977GiB/s) | 20260506 13:44:45.338 | INFO | Downloaded 34GiB to GPU in 1.92083s (17.7007GiB/s) | 20260506 13:44:50.477 | INFO | Analyzing chunk frames = [1812480,1812992) | 20260506 13:44:50.477 | INFO | MemInfo: { | 20260506 13:44:50.477 | INFO | "gpu" : " 17 GB", | 20260506 13:44:50.477 | INFO | "huge" : "", | 20260506 13:44:50.478 | INFO | "huge_shared" : " 27 GB", | 20260506 13:44:50.478 | INFO | "malloc" : " 14 GB", | 20260506 13:44:50.478 | INFO | "pinned" : " 29 GB" | 20260506 13:44:50.478 | INFO | } | 20260506 13:44:50.478 | INFO | Perf report for the last 5.14002 seconds: | 20260506 13:44:50.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:44:50.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:50.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:50.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:50.478 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 13:44:50.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:50.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:50.478 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:44:50.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:50.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:50.478 | INFO | Uploaded 36GiB to GPU in 2.00586s (17.9474GiB/s) | 20260506 13:44:50.478 | INFO | Downloaded 34GiB to GPU in 1.94341s (17.495GiB/s) | 20260506 13:44:55.617 | INFO | Analyzing chunk frames = [1812992,1813504) | 20260506 13:44:55.617 | INFO | MemInfo: { | 20260506 13:44:55.617 | INFO | "gpu" : " 17 GB", | 20260506 13:44:55.617 | INFO | "huge" : "", | 20260506 13:44:55.617 | INFO | "huge_shared" : " 27 GB", | 20260506 13:44:55.617 | INFO | "malloc" : " 14 GB", | 20260506 13:44:55.617 | INFO | "pinned" : " 29 GB" | 20260506 13:44:55.617 | INFO | } | 20260506 13:44:55.617 | INFO | Perf report for the last 5.13944 seconds: | 20260506 13:44:55.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:44:55.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:55.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:55.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:55.617 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:44:55.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:55.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:55.617 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260506 13:44:55.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:55.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:44:55.617 | INFO | Uploaded 36GiB to GPU in 2.09267s (17.2029GiB/s) | 20260506 13:44:55.617 | INFO | Downloaded 34GiB to GPU in 1.9667s (17.2879GiB/s) | 20260506 13:45:00.757 | INFO | Analyzing chunk frames = [1813504,1814016) | 20260506 13:45:00.757 | INFO | MemInfo: { | 20260506 13:45:00.757 | INFO | "gpu" : " 17 GB", | 20260506 13:45:00.757 | INFO | "huge" : "", | 20260506 13:45:00.757 | INFO | "huge_shared" : " 27 GB", | 20260506 13:45:00.757 | INFO | "malloc" : " 14 GB", | 20260506 13:45:00.757 | INFO | "pinned" : " 29 GB" | 20260506 13:45:00.757 | INFO | } | 20260506 13:45:00.757 | INFO | Perf report for the last 5.13967 seconds: | 20260506 13:45:00.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:45:00.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:00.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:00.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:00.757 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:45:00.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:00.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:00.757 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:45:00.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:00.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:00.757 | INFO | Uploaded 36GiB to GPU in 2.00445s (17.9601GiB/s) | 20260506 13:45:00.757 | INFO | Downloaded 34GiB to GPU in 2.03745s (16.6875GiB/s) | 20260506 13:45:05.895 | INFO | Analyzing chunk frames = [1814016,1814528) | 20260506 13:45:05.895 | INFO | MemInfo: { | 20260506 13:45:05.895 | INFO | "gpu" : " 17 GB", | 20260506 13:45:05.895 | INFO | "huge" : "", | 20260506 13:45:05.895 | INFO | "huge_shared" : " 27 GB", | 20260506 13:45:05.895 | INFO | "malloc" : " 14 GB", | 20260506 13:45:05.895 | INFO | "pinned" : " 29 GB" | 20260506 13:45:05.895 | INFO | } | 20260506 13:45:05.895 | INFO | Perf report for the last 5.13814 seconds: | 20260506 13:45:05.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:45:05.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:05.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:05.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:05.895 | INFO | ANALYSIS : 62.6%, 3.0, 97.7ms, 0 | 20260506 13:45:05.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:05.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:05.895 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260506 13:45:05.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:05.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:05.895 | INFO | Uploaded 36GiB to GPU in 2.05854s (17.4881GiB/s) | 20260506 13:45:05.895 | INFO | Downloaded 34GiB to GPU in 1.99724s (17.0235GiB/s) | 20260506 13:45:06.027 | INFO | Starting a round of Baz IO | 20260506 13:45:09.188 | INFO | Populating RTMetrics report | 20260506 13:45:09.188 | INFO | RTMetrics report populated | 20260506 13:45:09.188 | INFO | Opening tmp RTMetrics json file | 20260506 13:45:09.189 | INFO | Serializing RTMetrics to json | 20260506 13:45:09.190 | INFO | Serialization done, swapping tmp file now... | 20260506 13:45:09.190 | INFO | RTMetrics done | 20260506 13:45:09.501 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.474 seconds | 20260506 13:45:09.501 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.24% / 0.02% / 60.74% | 20260506 13:45:09.501 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.96516% / 5.5631e-05% / 7.15256e-05% | 20260506 13:45:09.501 | INFO | Baz IO overall: | 20260506 13:45:09.501 | INFO | BytesWritten : 926638563900 | 20260506 13:45:09.501 | INFO | NumEvents : 448264660942 | 20260506 13:45:11.034 | INFO | Analyzing chunk frames = [1814528,1815040) | 20260506 13:45:11.034 | INFO | MemInfo: { | 20260506 13:45:11.034 | INFO | "gpu" : " 17 GB", | 20260506 13:45:11.034 | INFO | "huge" : "", | 20260506 13:45:11.034 | INFO | "huge_shared" : " 27 GB", | 20260506 13:45:11.034 | INFO | "malloc" : " 14 GB", | 20260506 13:45:11.034 | INFO | "pinned" : " 29 GB" | 20260506 13:45:11.034 | INFO | } | 20260506 13:45:11.034 | INFO | Perf report for the last 5.13943 seconds: | 20260506 13:45:11.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:45:11.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:11.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:11.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:11.034 | INFO | ANALYSIS : 63.5%, 3.0, 98.5ms, 0 | 20260506 13:45:11.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:11.034 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:45:11.034 | INFO | PRE_HQ : 61.1%, 1.9, 60.7ms, 0 | 20260506 13:45:11.034 | INFO | BAZWRITER : 66.9%, 4.0, 138.7ms, 0 | 20260506 13:45:11.035 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 13:45:11.035 | INFO | Uploaded 36GiB to GPU in 2.0593s (17.4817GiB/s) | 20260506 13:45:11.035 | INFO | Downloaded 38GiB to GPU in 2.21169s (17.1814GiB/s) | 20260506 13:45:16.174 | INFO | Analyzing chunk frames = [1815040,1815552) | 20260506 13:45:16.174 | INFO | MemInfo: { | 20260506 13:45:16.174 | INFO | "gpu" : " 17 GB", | 20260506 13:45:16.174 | INFO | "huge" : "", | 20260506 13:45:16.174 | INFO | "huge_shared" : " 27 GB", | 20260506 13:45:16.174 | INFO | "malloc" : " 14 GB", | 20260506 13:45:16.174 | INFO | "pinned" : " 29 GB" | 20260506 13:45:16.174 | INFO | } | 20260506 13:45:16.174 | INFO | Perf report for the last 5.1393 seconds: | 20260506 13:45:16.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:45:16.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:16.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:16.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:16.174 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 13:45:16.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:16.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:16.174 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:45:16.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:16.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:16.174 | INFO | Uploaded 36GiB to GPU in 2.03147s (17.7211GiB/s) | 20260506 13:45:16.174 | INFO | Downloaded 34GiB to GPU in 1.93875s (17.537GiB/s) | 20260506 13:45:21.313 | INFO | Analyzing chunk frames = [1815552,1816064) | 20260506 13:45:21.313 | INFO | MemInfo: { | 20260506 13:45:21.313 | INFO | "gpu" : " 17 GB", | 20260506 13:45:21.313 | INFO | "huge" : "", | 20260506 13:45:21.313 | INFO | "huge_shared" : " 27 GB", | 20260506 13:45:21.313 | INFO | "malloc" : " 14 GB", | 20260506 13:45:21.313 | INFO | "pinned" : " 29 GB" | 20260506 13:45:21.313 | INFO | } | 20260506 13:45:21.313 | INFO | Perf report for the last 5.13945 seconds: | 20260506 13:45:21.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:45:21.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:21.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:21.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:21.313 | INFO | ANALYSIS : 61.5%, 3.0, 95.2ms, 0 | 20260506 13:45:21.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:21.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:21.313 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:45:21.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:21.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:21.313 | INFO | Uploaded 36GiB to GPU in 2.00848s (17.924GiB/s) | 20260506 13:45:21.313 | INFO | Downloaded 34GiB to GPU in 1.94027s (17.5233GiB/s) | 20260506 13:45:26.452 | INFO | Analyzing chunk frames = [1816064,1816576) | 20260506 13:45:26.452 | INFO | MemInfo: { | 20260506 13:45:26.452 | INFO | "gpu" : " 17 GB", | 20260506 13:45:26.452 | INFO | "huge" : "", | 20260506 13:45:26.452 | INFO | "huge_shared" : " 27 GB", | 20260506 13:45:26.452 | INFO | "malloc" : " 14 GB", | 20260506 13:45:26.452 | INFO | "pinned" : " 29 GB" | 20260506 13:45:26.452 | INFO | } | 20260506 13:45:26.452 | INFO | Perf report for the last 5.13921 seconds: | 20260506 13:45:26.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:45:26.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:26.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:26.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:26.452 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 13:45:26.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:26.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:26.452 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:45:26.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:26.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:26.453 | INFO | Uploaded 36GiB to GPU in 2.01523s (17.8639GiB/s) | 20260506 13:45:26.453 | INFO | Downloaded 34GiB to GPU in 1.92025s (17.706GiB/s) | 20260506 13:45:31.592 | INFO | Analyzing chunk frames = [1816576,1817088) | 20260506 13:45:31.592 | INFO | MemInfo: { | 20260506 13:45:31.592 | INFO | "gpu" : " 17 GB", | 20260506 13:45:31.592 | INFO | "huge" : "", | 20260506 13:45:31.592 | INFO | "huge_shared" : " 27 GB", | 20260506 13:45:31.592 | INFO | "malloc" : " 14 GB", | 20260506 13:45:31.592 | INFO | "pinned" : " 29 GB" | 20260506 13:45:31.592 | INFO | } | 20260506 13:45:31.592 | INFO | Perf report for the last 5.13932 seconds: | 20260506 13:45:31.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:45:31.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:31.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:31.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:31.592 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 13:45:31.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:31.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:31.592 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:45:31.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:31.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:31.592 | INFO | Uploaded 36GiB to GPU in 2.00944s (17.9155GiB/s) | 20260506 13:45:31.592 | INFO | Downloaded 34GiB to GPU in 1.9382s (17.5421GiB/s) | 20260506 13:45:36.731 | INFO | Analyzing chunk frames = [1817088,1817600) | 20260506 13:45:36.731 | INFO | MemInfo: { | 20260506 13:45:36.731 | INFO | "gpu" : " 17 GB", | 20260506 13:45:36.731 | INFO | "huge" : "", | 20260506 13:45:36.731 | INFO | "huge_shared" : " 27 GB", | 20260506 13:45:36.731 | INFO | "malloc" : " 14 GB", | 20260506 13:45:36.731 | INFO | "pinned" : " 29 GB" | 20260506 13:45:36.731 | INFO | } | 20260506 13:45:36.731 | INFO | Perf report for the last 5.13966 seconds: | 20260506 13:45:36.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:45:36.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:36.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:36.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:36.731 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 13:45:36.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:36.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:36.731 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 13:45:36.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:36.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:36.732 | INFO | Uploaded 36GiB to GPU in 2.08993s (17.2255GiB/s) | 20260506 13:45:36.732 | INFO | Downloaded 34GiB to GPU in 1.95503s (17.391GiB/s) | 20260506 13:45:41.871 | INFO | Analyzing chunk frames = [1817600,1818112) | 20260506 13:45:41.871 | INFO | MemInfo: { | 20260506 13:45:41.871 | INFO | "gpu" : " 17 GB", | 20260506 13:45:41.871 | INFO | "huge" : "", | 20260506 13:45:41.871 | INFO | "huge_shared" : " 27 GB", | 20260506 13:45:41.871 | INFO | "malloc" : " 14 GB", | 20260506 13:45:41.871 | INFO | "pinned" : " 29 GB" | 20260506 13:45:41.871 | INFO | } | 20260506 13:45:41.871 | INFO | Perf report for the last 5.13977 seconds: | 20260506 13:45:41.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:45:41.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:41.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:41.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:41.871 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 13:45:41.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:41.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:41.871 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 13:45:41.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:41.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:41.871 | INFO | Uploaded 36GiB to GPU in 1.99344s (18.0592GiB/s) | 20260506 13:45:41.871 | INFO | Downloaded 34GiB to GPU in 2.03132s (16.7379GiB/s) | 20260506 13:45:47.010 | INFO | Analyzing chunk frames = [1818112,1818624) | 20260506 13:45:47.010 | INFO | MemInfo: { | 20260506 13:45:47.010 | INFO | "gpu" : " 17 GB", | 20260506 13:45:47.010 | INFO | "huge" : "", | 20260506 13:45:47.010 | INFO | "huge_shared" : " 27 GB", | 20260506 13:45:47.010 | INFO | "malloc" : " 14 GB", | 20260506 13:45:47.010 | INFO | "pinned" : " 29 GB" | 20260506 13:45:47.010 | INFO | } | 20260506 13:45:47.010 | INFO | Perf report for the last 5.13862 seconds: | 20260506 13:45:47.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:45:47.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:47.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:47.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:47.010 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:45:47.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:47.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:47.010 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:45:47.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:47.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:47.010 | INFO | Uploaded 36GiB to GPU in 2.06131s (17.4646GiB/s) | 20260506 13:45:47.010 | INFO | Downloaded 34GiB to GPU in 1.9601s (17.346GiB/s) | 20260506 13:45:47.010 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:45:47.010 | INFO | Total mem usage: 17.4 GiB | 20260506 13:45:47.010 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:45:47.010 | INFO | Recent High Water Marks Sum: 14.7 GiB (Savings: -18.5%) | 20260506 13:45:47.010 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:45:47.010 | INFO | High Water Breakdown: | 20260506 13:45:47.010 | INFO | 60.1% : 8.84 GiB ( 80.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:45:47.010 | INFO | 39.7% : 5.84 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:45:47.010 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:45:47.010 | INFO | ------------------------------------------ | 20260506 13:45:47.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:45:47.010 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:45:47.010 | INFO | Total mem usage: 26.5 GiB | 20260506 13:45:47.010 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:45:47.010 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:45:47.010 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:45:47.010 | INFO | High Water Breakdown: | 20260506 13:45:47.010 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:45:47.010 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:45:47.010 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:45:47.010 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:45:47.010 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:45:47.010 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:45:47.010 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:45:47.010 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:45:47.010 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:45:47.010 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:45:47.010 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:45:47.010 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:45:47.010 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:45:47.010 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:45:47.010 | INFO | ------------------------------------------ | 20260506 13:45:47.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:45:47.010 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:45:47.010 | INFO | Total mem usage: 25.6 GiB | 20260506 13:45:47.010 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:45:47.010 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:45:47.010 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:45:47.010 | INFO | High Water Breakdown: | 20260506 13:45:47.010 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:45:47.010 | INFO | ------------------------------------------ | 20260506 13:45:47.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:45:47.011 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:45:47.011 | INFO | Total mem usage: 15.7 GiB | 20260506 13:45:47.011 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:45:47.011 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:45:47.011 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 13:45:47.011 | INFO | High Water Breakdown: | 20260506 13:45:47.011 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:45:47.011 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:45:47.011 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:45:47.011 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:45:47.011 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:45:47.011 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:45:47.011 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:45:47.011 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:45:47.011 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:45:47.011 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:45:47.011 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:45:47.011 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:45:47.011 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:45:47.011 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:45:47.011 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:45:47.011 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:45:47.011 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:45:47.011 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:45:47.011 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:45:47.011 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:45:47.011 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:45:47.011 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:45:47.011 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:45:47.011 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:45:47.011 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:45:47.011 | INFO | ------------------------------------------ | 20260506 13:45:47.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:45:47.011 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:45:47.011 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:45:47.011 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:45:47.011 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.68e+01, 1.36e+01 / 2.92e+00 | 20260506 13:45:47.011 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.02e-01, 1.05e-02 / 5.75e-03 | 20260506 13:45:47.011 | INFO | PerfReport: 11.57%, Baseline -- 3.66e+00 / 4.46e+00, 5.72e-01 / 3.02e-01 | 20260506 13:45:47.011 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.48e+00, 1.92e-01 / 1.40e-01 | 20260506 13:45:47.011 | INFO | PerfReport: 15.98%, DME -- 4.04e+01 / 4.55e+01, 4.45e+00 / 3.28e+00 | 20260506 13:45:47.011 | INFO | PerfReport: 56.45%, FrameLabeling -- 1.79e+01 / 2.03e+01, 1.35e+00 / 1.73e+00 | 20260506 13:45:47.011 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.37e+00 / 2.95e+00, 4.16e-01 / 2.85e-01 | 20260506 13:45:47.011 | INFO | PerfReport: 3.67%, Metrics -- 1.16e+00 / 1.53e+00, 2.98e-01 / 1.52e-01 | 20260506 13:45:47.011 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.06e-01 / 4.65e-01, 6.33e-02 / 2.73e-02 | 20260506 13:45:47.011 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:45:47.011 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:45:47.011 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:45:47.011 | INFO | PerfReport: Full Execution -- 9.86e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260506 13:45:47.011 | INFO | PerfReport: 21.51%, Upload -- 2.12e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260506 13:45:47.011 | INFO | PerfReport: 21.05%, Download -- 2.07e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 13:45:47.011 | INFO | PerfReport: 32.09%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 13:45:47.011 | INFO | PerfReport: 1.78%, LockUpload -- 1.75e+00 / 0.00e+00, 6.91e+00 / 0.00e+00 | 20260506 13:45:47.011 | INFO | PerfReport: 1.03%, LockDownload -- 1.01e+00 / 0.00e+00, 2.62e+00 / 0.00e+00 | 20260506 13:45:47.011 | INFO | PerfReport: 22.53%, LockCompute -- 2.22e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 13:45:47.011 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260506 13:45:47.165 | INFO | Starting a round of Baz IO | 20260506 13:45:50.299 | INFO | Populating RTMetrics report | 20260506 13:45:50.299 | INFO | RTMetrics report populated | 20260506 13:45:50.299 | INFO | Opening tmp RTMetrics json file | 20260506 13:45:50.300 | INFO | Serializing RTMetrics to json | 20260506 13:45:50.301 | INFO | Serialization done, swapping tmp file now... | 20260506 13:45:50.301 | INFO | RTMetrics done | 20260506 13:45:50.397 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.232 seconds | 20260506 13:45:50.397 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.23% / 0.02% / 60.76% | 20260506 13:45:50.397 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.95911% / 2.38419e-05% / 9.53674e-05% | 20260506 13:45:50.397 | INFO | Baz IO overall: | 20260506 13:45:50.397 | INFO | BytesWritten : 928906281416 | 20260506 13:45:50.397 | INFO | NumEvents : 449364879160 | 20260506 13:45:52.149 | INFO | Analyzing chunk frames = [1818624,1819136) | 20260506 13:45:52.149 | INFO | MemInfo: { | 20260506 13:45:52.149 | INFO | "gpu" : " 17 GB", | 20260506 13:45:52.149 | INFO | "huge" : "", | 20260506 13:45:52.149 | INFO | "huge_shared" : " 27 GB", | 20260506 13:45:52.149 | INFO | "malloc" : " 14 GB", | 20260506 13:45:52.149 | INFO | "pinned" : " 29 GB" | 20260506 13:45:52.149 | INFO | } | 20260506 13:45:52.149 | INFO | Perf report for the last 5.13939 seconds: | 20260506 13:45:52.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:45:52.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:52.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:52.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:52.149 | INFO | ANALYSIS : 63.4%, 2.9, 97.2ms, 0 | 20260506 13:45:52.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:52.149 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:45:52.149 | INFO | PRE_HQ : 61.2%, 1.9, 60.2ms, 0 | 20260506 13:45:52.149 | INFO | BAZWRITER : 62.9%, 4.1, 133.9ms, 0 | 20260506 13:45:52.149 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:45:52.150 | INFO | Uploaded 36GiB to GPU in 2.04996s (17.5613GiB/s) | 20260506 13:45:52.150 | INFO | Downloaded 38GiB to GPU in 2.22648s (17.0673GiB/s) | 20260506 13:45:57.289 | INFO | Analyzing chunk frames = [1819136,1819648) | 20260506 13:45:57.289 | INFO | MemInfo: { | 20260506 13:45:57.289 | INFO | "gpu" : " 17 GB", | 20260506 13:45:57.289 | INFO | "huge" : "", | 20260506 13:45:57.289 | INFO | "huge_shared" : " 27 GB", | 20260506 13:45:57.289 | INFO | "malloc" : " 14 GB", | 20260506 13:45:57.289 | INFO | "pinned" : " 29 GB" | 20260506 13:45:57.289 | INFO | } | 20260506 13:45:57.289 | INFO | Perf report for the last 5.13992 seconds: | 20260506 13:45:57.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:45:57.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:57.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:57.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:57.289 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 13:45:57.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:57.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:57.289 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 13:45:57.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:57.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:45:57.289 | INFO | Uploaded 36GiB to GPU in 1.99949s (18.0046GiB/s) | 20260506 13:45:57.289 | INFO | Downloaded 34GiB to GPU in 1.93624s (17.5598GiB/s) | 20260506 13:46:02.445 | INFO | Analyzing chunk frames = [1819648,1820160) | 20260506 13:46:02.445 | INFO | MemInfo: { | 20260506 13:46:02.445 | INFO | "gpu" : " 17 GB", | 20260506 13:46:02.445 | INFO | "huge" : "", | 20260506 13:46:02.445 | INFO | "huge_shared" : " 27 GB", | 20260506 13:46:02.445 | INFO | "malloc" : " 14 GB", | 20260506 13:46:02.445 | INFO | "pinned" : " 29 GB" | 20260506 13:46:02.445 | INFO | } | 20260506 13:46:02.446 | INFO | Perf report for the last 5.1564 seconds: | 20260506 13:46:02.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:46:02.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:02.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:02.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:02.446 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 13:46:02.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:02.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:02.446 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 13:46:02.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:02.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:02.446 | INFO | Uploaded 36GiB to GPU in 2.00998s (17.9106GiB/s) | 20260506 13:46:02.446 | INFO | Downloaded 34GiB to GPU in 1.9284s (17.6312GiB/s) | 20260506 13:46:07.567 | INFO | Analyzing chunk frames = [1820160,1820672) | 20260506 13:46:07.567 | INFO | MemInfo: { | 20260506 13:46:07.567 | INFO | "gpu" : " 17 GB", | 20260506 13:46:07.567 | INFO | "huge" : "", | 20260506 13:46:07.567 | INFO | "huge_shared" : " 27 GB", | 20260506 13:46:07.567 | INFO | "malloc" : " 14 GB", | 20260506 13:46:07.568 | INFO | "pinned" : " 29 GB" | 20260506 13:46:07.568 | INFO | } | 20260506 13:46:07.568 | INFO | Perf report for the last 5.12202 seconds: | 20260506 13:46:07.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:46:07.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:07.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:07.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:07.568 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 13:46:07.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:07.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:07.568 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 13:46:07.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:07.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:07.568 | INFO | Uploaded 36GiB to GPU in 2.01335s (17.8807GiB/s) | 20260506 13:46:07.568 | INFO | Downloaded 34GiB to GPU in 1.92536s (17.659GiB/s) | 20260506 13:46:12.706 | INFO | Analyzing chunk frames = [1820672,1821184) | 20260506 13:46:12.706 | INFO | MemInfo: { | 20260506 13:46:12.706 | INFO | "gpu" : " 17 GB", | 20260506 13:46:12.706 | INFO | "huge" : "", | 20260506 13:46:12.706 | INFO | "huge_shared" : " 27 GB", | 20260506 13:46:12.707 | INFO | "malloc" : " 14 GB", | 20260506 13:46:12.707 | INFO | "pinned" : " 29 GB" | 20260506 13:46:12.707 | INFO | } | 20260506 13:46:12.707 | INFO | Perf report for the last 5.13895 seconds: | 20260506 13:46:12.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:46:12.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:12.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:12.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:12.707 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 13:46:12.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:12.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:12.707 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:46:12.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:12.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:12.707 | INFO | Uploaded 36GiB to GPU in 2.06333s (17.4475GiB/s) | 20260506 13:46:12.707 | INFO | Downloaded 34GiB to GPU in 1.94842s (17.45GiB/s) | 20260506 13:46:17.846 | INFO | Analyzing chunk frames = [1821184,1821696) | 20260506 13:46:17.846 | INFO | MemInfo: { | 20260506 13:46:17.846 | INFO | "gpu" : " 17 GB", | 20260506 13:46:17.846 | INFO | "huge" : "", | 20260506 13:46:17.846 | INFO | "huge_shared" : " 27 GB", | 20260506 13:46:17.846 | INFO | "malloc" : " 14 GB", | 20260506 13:46:17.846 | INFO | "pinned" : " 29 GB" | 20260506 13:46:17.846 | INFO | } | 20260506 13:46:17.846 | INFO | Perf report for the last 5.13933 seconds: | 20260506 13:46:17.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:46:17.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:17.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:17.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:17.846 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 13:46:17.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:17.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:17.846 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 13:46:17.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:17.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:17.846 | INFO | Uploaded 36GiB to GPU in 2.0846s (17.2695GiB/s) | 20260506 13:46:17.846 | INFO | Downloaded 34GiB to GPU in 1.94591s (17.4726GiB/s) | 20260506 13:46:22.985 | INFO | Analyzing chunk frames = [1821696,1822208) | 20260506 13:46:22.985 | INFO | MemInfo: { | 20260506 13:46:22.985 | INFO | "gpu" : " 17 GB", | 20260506 13:46:22.985 | INFO | "huge" : "", | 20260506 13:46:22.985 | INFO | "huge_shared" : " 27 GB", | 20260506 13:46:22.985 | INFO | "malloc" : " 14 GB", | 20260506 13:46:22.985 | INFO | "pinned" : " 29 GB" | 20260506 13:46:22.985 | INFO | } | 20260506 13:46:22.985 | INFO | Perf report for the last 5.13951 seconds: | 20260506 13:46:22.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:46:22.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:22.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:22.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:22.985 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 13:46:22.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:22.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:22.985 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 13:46:22.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:22.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:22.986 | INFO | Uploaded 36GiB to GPU in 2.00437s (17.9608GiB/s) | 20260506 13:46:22.986 | INFO | Downloaded 34GiB to GPU in 2.02694s (16.774GiB/s) | 20260506 13:46:28.124 | INFO | Analyzing chunk frames = [1822208,1822720) | 20260506 13:46:28.124 | INFO | MemInfo: { | 20260506 13:46:28.124 | INFO | "gpu" : " 17 GB", | 20260506 13:46:28.124 | INFO | "huge" : "", | 20260506 13:46:28.124 | INFO | "huge_shared" : " 27 GB", | 20260506 13:46:28.124 | INFO | "malloc" : " 14 GB", | 20260506 13:46:28.124 | INFO | "pinned" : " 29 GB" | 20260506 13:46:28.124 | INFO | } | 20260506 13:46:28.125 | INFO | Perf report for the last 5.13912 seconds: | 20260506 13:46:28.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:46:28.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:28.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:28.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:28.125 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:46:28.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:28.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:28.125 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 13:46:28.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:28.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:28.125 | INFO | Uploaded 36GiB to GPU in 2.05982s (17.4773GiB/s) | 20260506 13:46:28.125 | INFO | Downloaded 34GiB to GPU in 1.98471s (17.1309GiB/s) | 20260506 13:46:28.259 | INFO | Starting a round of Baz IO | 20260506 13:46:31.412 | INFO | Populating RTMetrics report | 20260506 13:46:31.412 | INFO | RTMetrics report populated | 20260506 13:46:31.412 | INFO | Opening tmp RTMetrics json file | 20260506 13:46:31.413 | INFO | Serializing RTMetrics to json | 20260506 13:46:31.415 | INFO | Serialization done, swapping tmp file now... | 20260506 13:46:31.415 | INFO | RTMetrics done | 20260506 13:46:31.689 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.429 seconds | 20260506 13:46:31.689 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.21% / 0.02% / 60.77% | 20260506 13:46:31.689 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.95446% / 3.57628e-05% / 7.54992e-05% | 20260506 13:46:31.689 | INFO | Baz IO overall: | 20260506 13:46:31.689 | INFO | BytesWritten : 931174326730 | 20260506 13:46:31.689 | INFO | NumEvents : 450465269642 | 20260506 13:46:33.264 | INFO | Analyzing chunk frames = [1822720,1823232) | 20260506 13:46:33.264 | INFO | MemInfo: { | 20260506 13:46:33.264 | INFO | "gpu" : " 17 GB", | 20260506 13:46:33.264 | INFO | "huge" : "", | 20260506 13:46:33.264 | INFO | "huge_shared" : " 27 GB", | 20260506 13:46:33.264 | INFO | "malloc" : " 14 GB", | 20260506 13:46:33.264 | INFO | "pinned" : " 29 GB" | 20260506 13:46:33.264 | INFO | } | 20260506 13:46:33.264 | INFO | Perf report for the last 5.13927 seconds: | 20260506 13:46:33.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:46:33.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:33.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:33.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:33.264 | INFO | ANALYSIS : 63.4%, 2.9, 97.4ms, 0 | 20260506 13:46:33.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:33.264 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:46:33.264 | INFO | PRE_HQ : 61.2%, 1.9, 60.5ms, 0 | 20260506 13:46:33.264 | INFO | BAZWRITER : 66.0%, 4.7, 158.3ms, 0 | 20260506 13:46:33.264 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 13:46:33.264 | INFO | Uploaded 36GiB to GPU in 2.08442s (17.271GiB/s) | 20260506 13:46:33.264 | INFO | Downloaded 38GiB to GPU in 2.23003s (17.0401GiB/s) | 20260506 13:46:38.403 | INFO | Analyzing chunk frames = [1823232,1823744) | 20260506 13:46:38.403 | INFO | MemInfo: { | 20260506 13:46:38.403 | INFO | "gpu" : " 17 GB", | 20260506 13:46:38.403 | INFO | "huge" : "", | 20260506 13:46:38.403 | INFO | "huge_shared" : " 27 GB", | 20260506 13:46:38.403 | INFO | "malloc" : " 14 GB", | 20260506 13:46:38.403 | INFO | "pinned" : " 29 GB" | 20260506 13:46:38.403 | INFO | } | 20260506 13:46:38.403 | INFO | Perf report for the last 5.13946 seconds: | 20260506 13:46:38.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:46:38.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:38.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:38.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:38.403 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:46:38.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:38.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:38.403 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:46:38.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:38.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:38.404 | INFO | Uploaded 36GiB to GPU in 2.06485s (17.4346GiB/s) | 20260506 13:46:38.404 | INFO | Downloaded 34GiB to GPU in 1.94382s (17.4914GiB/s) | 20260506 13:46:43.543 | INFO | Analyzing chunk frames = [1823744,1824256) | 20260506 13:46:43.543 | INFO | MemInfo: { | 20260506 13:46:43.543 | INFO | "gpu" : " 17 GB", | 20260506 13:46:43.543 | INFO | "huge" : "", | 20260506 13:46:43.543 | INFO | "huge_shared" : " 27 GB", | 20260506 13:46:43.543 | INFO | "malloc" : " 14 GB", | 20260506 13:46:43.543 | INFO | "pinned" : " 29 GB" | 20260506 13:46:43.543 | INFO | } | 20260506 13:46:43.543 | INFO | Perf report for the last 5.13953 seconds: | 20260506 13:46:43.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:46:43.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:43.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:43.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:43.543 | INFO | ANALYSIS : 61.7%, 3.0, 95.5ms, 0 | 20260506 13:46:43.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:43.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:43.543 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:46:43.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:43.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:43.543 | INFO | Uploaded 36GiB to GPU in 2.0326s (17.7113GiB/s) | 20260506 13:46:43.543 | INFO | Downloaded 34GiB to GPU in 1.94011s (17.5247GiB/s) | 20260506 13:46:48.682 | INFO | Analyzing chunk frames = [1824256,1824768) | 20260506 13:46:48.682 | INFO | MemInfo: { | 20260506 13:46:48.682 | INFO | "gpu" : " 17 GB", | 20260506 13:46:48.682 | INFO | "huge" : "", | 20260506 13:46:48.682 | INFO | "huge_shared" : " 27 GB", | 20260506 13:46:48.682 | INFO | "malloc" : " 14 GB", | 20260506 13:46:48.682 | INFO | "pinned" : " 29 GB" | 20260506 13:46:48.682 | INFO | } | 20260506 13:46:48.682 | INFO | Perf report for the last 5.13905 seconds: | 20260506 13:46:48.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:46:48.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:48.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:48.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:48.682 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:46:48.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:48.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:48.682 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:46:48.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:48.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:48.682 | INFO | Uploaded 36GiB to GPU in 1.99255s (18.0673GiB/s) | 20260506 13:46:48.682 | INFO | Downloaded 34GiB to GPU in 1.9289s (17.6266GiB/s) | 20260506 13:46:53.821 | INFO | Analyzing chunk frames = [1824768,1825280) | 20260506 13:46:53.821 | INFO | MemInfo: { | 20260506 13:46:53.821 | INFO | "gpu" : " 17 GB", | 20260506 13:46:53.821 | INFO | "huge" : "", | 20260506 13:46:53.821 | INFO | "huge_shared" : " 27 GB", | 20260506 13:46:53.821 | INFO | "malloc" : " 14 GB", | 20260506 13:46:53.821 | INFO | "pinned" : " 29 GB" | 20260506 13:46:53.821 | INFO | } | 20260506 13:46:53.821 | INFO | Perf report for the last 5.13921 seconds: | 20260506 13:46:53.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:46:53.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:53.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:53.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:53.821 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 13:46:53.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:53.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:53.821 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:46:53.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:53.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:53.821 | INFO | Uploaded 36GiB to GPU in 2.05505s (17.5178GiB/s) | 20260506 13:46:53.821 | INFO | Downloaded 34GiB to GPU in 1.94653s (17.467GiB/s) | 20260506 13:46:58.960 | INFO | Analyzing chunk frames = [1825280,1825792) | 20260506 13:46:58.960 | INFO | MemInfo: { | 20260506 13:46:58.960 | INFO | "gpu" : " 17 GB", | 20260506 13:46:58.960 | INFO | "huge" : "", | 20260506 13:46:58.960 | INFO | "huge_shared" : " 27 GB", | 20260506 13:46:58.960 | INFO | "malloc" : " 14 GB", | 20260506 13:46:58.961 | INFO | "pinned" : " 29 GB" | 20260506 13:46:58.961 | INFO | } | 20260506 13:46:58.961 | INFO | Perf report for the last 5.13941 seconds: | 20260506 13:46:58.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:46:58.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:58.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:58.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:58.961 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 13:46:58.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:58.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:58.961 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 13:46:58.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:58.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:46:58.961 | INFO | Uploaded 36GiB to GPU in 2.07123s (17.381GiB/s) | 20260506 13:46:58.961 | INFO | Downloaded 34GiB to GPU in 1.92962s (17.62GiB/s) | 20260506 13:47:04.100 | INFO | Analyzing chunk frames = [1825792,1826304) | 20260506 13:47:04.100 | INFO | MemInfo: { | 20260506 13:47:04.100 | INFO | "gpu" : " 17 GB", | 20260506 13:47:04.100 | INFO | "huge" : "", | 20260506 13:47:04.100 | INFO | "huge_shared" : " 27 GB", | 20260506 13:47:04.100 | INFO | "malloc" : " 14 GB", | 20260506 13:47:04.100 | INFO | "pinned" : " 29 GB" | 20260506 13:47:04.100 | INFO | } | 20260506 13:47:04.100 | INFO | Perf report for the last 5.13973 seconds: | 20260506 13:47:04.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:47:04.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:04.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:04.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:04.100 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 13:47:04.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:04.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:04.100 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 13:47:04.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:04.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:04.101 | INFO | Uploaded 36GiB to GPU in 2.02475s (17.78GiB/s) | 20260506 13:47:04.101 | INFO | Downloaded 34GiB to GPU in 2.01857s (16.8436GiB/s) | 20260506 13:47:09.239 | INFO | Analyzing chunk frames = [1826304,1826816) | 20260506 13:47:09.239 | INFO | MemInfo: { | 20260506 13:47:09.239 | INFO | "gpu" : " 17 GB", | 20260506 13:47:09.239 | INFO | "huge" : "", | 20260506 13:47:09.239 | INFO | "huge_shared" : " 27 GB", | 20260506 13:47:09.239 | INFO | "malloc" : " 14 GB", | 20260506 13:47:09.239 | INFO | "pinned" : " 29 GB" | 20260506 13:47:09.239 | INFO | } | 20260506 13:47:09.239 | INFO | Perf report for the last 5.13889 seconds: | 20260506 13:47:09.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:47:09.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:09.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:09.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:09.239 | INFO | ANALYSIS : 62.7%, 3.0, 97.8ms, 0 | 20260506 13:47:09.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:09.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:09.239 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260506 13:47:09.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:09.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:09.239 | INFO | Uploaded 36GiB to GPU in 2.03803s (17.6641GiB/s) | 20260506 13:47:09.240 | INFO | Downloaded 34GiB to GPU in 1.99395s (17.0515GiB/s) | 20260506 13:47:09.240 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:47:09.240 | INFO | Total mem usage: 17.4 GiB | 20260506 13:47:09.240 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:47:09.240 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -10.5%) | 20260506 13:47:09.240 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:47:09.240 | INFO | High Water Breakdown: | 20260506 13:47:09.240 | INFO | 61.2% : 9.65 GiB ( 88.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:47:09.240 | INFO | 38.6% : 6.09 GiB ( 91% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:47:09.240 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:47:09.240 | INFO | ------------------------------------------ | 20260506 13:47:09.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:47:09.240 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:47:09.240 | INFO | Total mem usage: 26.5 GiB | 20260506 13:47:09.240 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:47:09.240 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:47:09.240 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:47:09.240 | INFO | High Water Breakdown: | 20260506 13:47:09.240 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:47:09.240 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:47:09.240 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:47:09.240 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:47:09.240 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:47:09.240 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:47:09.240 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:47:09.240 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:47:09.240 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:47:09.240 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:47:09.240 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:47:09.240 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:47:09.240 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:47:09.240 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:47:09.240 | INFO | ------------------------------------------ | 20260506 13:47:09.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:47:09.240 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:47:09.240 | INFO | Total mem usage: 25.6 GiB | 20260506 13:47:09.240 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:47:09.240 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:47:09.240 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:47:09.240 | INFO | High Water Breakdown: | 20260506 13:47:09.240 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:47:09.240 | INFO | ------------------------------------------ | 20260506 13:47:09.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:47:09.240 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:47:09.240 | INFO | Total mem usage: 15.7 GiB | 20260506 13:47:09.240 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:47:09.240 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:47:09.240 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:47:09.240 | INFO | High Water Breakdown: | 20260506 13:47:09.240 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:47:09.240 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:47:09.240 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:47:09.240 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:47:09.240 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:47:09.240 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:47:09.240 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:47:09.241 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:47:09.241 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:47:09.241 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:47:09.241 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:47:09.241 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:47:09.241 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:47:09.241 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:47:09.241 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:47:09.241 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:47:09.241 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:47:09.241 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:47:09.241 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:47:09.241 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:47:09.241 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:47:09.241 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:47:09.241 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:47:09.241 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:47:09.241 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:47:09.241 | INFO | ------------------------------------------ | 20260506 13:47:09.241 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:47:09.241 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:47:09.241 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:47:09.241 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:47:09.241 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.59e+01, 1.34e+01 / 2.67e+00 | 20260506 13:47:09.241 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.03e-01, 1.08e-02 / 7.17e-03 | 20260506 13:47:09.241 | INFO | PerfReport: 11.56%, Baseline -- 3.66e+00 / 4.39e+00, 5.68e-01 / 3.37e-01 | 20260506 13:47:09.241 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.43e+00, 1.89e-01 / 1.58e-01 | 20260506 13:47:09.241 | INFO | PerfReport: 15.80%, DME -- 4.00e+01 / 4.41e+01, 4.00e+00 / 2.81e+00 | 20260506 13:47:09.241 | INFO | PerfReport: 56.73%, FrameLabeling -- 1.79e+01 / 2.10e+01, 1.50e+00 / 1.78e+00 | 20260506 13:47:09.241 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.36e+00 / 2.92e+00, 4.05e-01 / 3.24e-01 | 20260506 13:47:09.241 | INFO | PerfReport: 3.64%, Metrics -- 1.15e+00 / 1.52e+00, 2.95e-01 / 1.72e-01 | 20260506 13:47:09.241 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.04e-01 / 4.58e-01, 6.31e-02 / 3.47e-02 | 20260506 13:47:09.241 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:47:09.241 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:47:09.241 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:47:09.241 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 13:47:09.241 | INFO | PerfReport: 21.60%, Upload -- 2.13e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260506 13:47:09.241 | INFO | PerfReport: 21.08%, Download -- 2.07e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 13:47:09.241 | INFO | PerfReport: 32.15%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 13:47:09.241 | INFO | PerfReport: 1.75%, LockUpload -- 1.73e+00 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 13:47:09.241 | INFO | PerfReport: 1.05%, LockDownload -- 1.04e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260506 13:47:09.241 | INFO | PerfReport: 22.35%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 13:47:09.241 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.13e-02 / 0.00e+00 | 20260506 13:47:09.366 | INFO | Starting a round of Baz IO | 20260506 13:47:12.516 | INFO | Populating RTMetrics report | 20260506 13:47:12.517 | INFO | RTMetrics report populated | 20260506 13:47:12.517 | INFO | Opening tmp RTMetrics json file | 20260506 13:47:12.652 | INFO | Serializing RTMetrics to json | 20260506 13:47:12.662 | INFO | Serialization done, swapping tmp file now... | 20260506 13:47:12.667 | INFO | RTMetrics done | 20260506 13:47:12.813 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.446 seconds | 20260506 13:47:12.813 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.19% / 0.02% / 60.79% | 20260506 13:47:12.813 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.94451% / 4.37101e-05% / 8.74201e-05% | 20260506 13:47:12.813 | INFO | Baz IO overall: | 20260506 13:47:12.813 | INFO | BytesWritten : 933442447921 | 20260506 13:47:12.813 | INFO | NumEvents : 451565369048 | 20260506 13:47:14.379 | INFO | Analyzing chunk frames = [1826816,1827328) | 20260506 13:47:14.379 | INFO | MemInfo: { | 20260506 13:47:14.379 | INFO | "gpu" : " 17 GB", | 20260506 13:47:14.379 | INFO | "huge" : "", | 20260506 13:47:14.379 | INFO | "huge_shared" : " 27 GB", | 20260506 13:47:14.379 | INFO | "malloc" : " 14 GB", | 20260506 13:47:14.379 | INFO | "pinned" : " 29 GB" | 20260506 13:47:14.379 | INFO | } | 20260506 13:47:14.379 | INFO | Perf report for the last 5.13971 seconds: | 20260506 13:47:14.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:47:14.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:14.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:14.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:14.379 | INFO | ANALYSIS : 63.2%, 2.9, 95.9ms, 0 | 20260506 13:47:14.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:14.379 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260506 13:47:14.379 | INFO | PRE_HQ : 61.2%, 1.9, 60.2ms, 0 | 20260506 13:47:14.379 | INFO | BAZWRITER : 66.4%, 4.3, 147.1ms, 0 | 20260506 13:47:14.379 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 13:47:14.379 | INFO | Uploaded 36GiB to GPU in 2.08999s (17.225GiB/s) | 20260506 13:47:14.379 | INFO | Downloaded 38GiB to GPU in 2.23081s (17.0342GiB/s) | 20260506 13:47:19.518 | INFO | Analyzing chunk frames = [1827328,1827840) | 20260506 13:47:19.518 | INFO | MemInfo: { | 20260506 13:47:19.518 | INFO | "gpu" : " 17 GB", | 20260506 13:47:19.518 | INFO | "huge" : "", | 20260506 13:47:19.518 | INFO | "huge_shared" : " 27 GB", | 20260506 13:47:19.518 | INFO | "malloc" : " 14 GB", | 20260506 13:47:19.518 | INFO | "pinned" : " 29 GB" | 20260506 13:47:19.518 | INFO | } | 20260506 13:47:19.518 | INFO | Perf report for the last 5.13889 seconds: | 20260506 13:47:19.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:47:19.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:19.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:19.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:19.518 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 13:47:19.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:19.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:19.518 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:47:19.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:19.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:19.518 | INFO | Uploaded 36GiB to GPU in 2.04886s (17.5707GiB/s) | 20260506 13:47:19.518 | INFO | Downloaded 34GiB to GPU in 1.93207s (17.5977GiB/s) | 20260506 13:47:24.657 | INFO | Analyzing chunk frames = [1827840,1828352) | 20260506 13:47:24.657 | INFO | MemInfo: { | 20260506 13:47:24.657 | INFO | "gpu" : " 17 GB", | 20260506 13:47:24.657 | INFO | "huge" : "", | 20260506 13:47:24.657 | INFO | "huge_shared" : " 27 GB", | 20260506 13:47:24.657 | INFO | "malloc" : " 14 GB", | 20260506 13:47:24.657 | INFO | "pinned" : " 29 GB" | 20260506 13:47:24.657 | INFO | } | 20260506 13:47:24.657 | INFO | Perf report for the last 5.13955 seconds: | 20260506 13:47:24.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:47:24.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:24.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:24.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:24.657 | INFO | ANALYSIS : 61.6%, 3.0, 95.4ms, 0 | 20260506 13:47:24.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:24.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:24.657 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:47:24.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:24.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:24.658 | INFO | Uploaded 36GiB to GPU in 2.02837s (17.7482GiB/s) | 20260506 13:47:24.658 | INFO | Downloaded 34GiB to GPU in 1.94373s (17.4922GiB/s) | 20260506 13:47:29.799 | INFO | Analyzing chunk frames = [1828352,1828864) | 20260506 13:47:29.799 | INFO | MemInfo: { | 20260506 13:47:29.799 | INFO | "gpu" : " 17 GB", | 20260506 13:47:29.799 | INFO | "huge" : "", | 20260506 13:47:29.799 | INFO | "huge_shared" : " 27 GB", | 20260506 13:47:29.799 | INFO | "malloc" : " 14 GB", | 20260506 13:47:29.799 | INFO | "pinned" : " 29 GB" | 20260506 13:47:29.799 | INFO | } | 20260506 13:47:29.799 | INFO | Perf report for the last 5.14142 seconds: | 20260506 13:47:29.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:47:29.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:29.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:29.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:29.799 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 13:47:29.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:29.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:29.799 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:47:29.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:29.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:29.799 | INFO | Uploaded 36GiB to GPU in 1.99211s (18.0713GiB/s) | 20260506 13:47:29.799 | INFO | Downloaded 34GiB to GPU in 1.91932s (17.7146GiB/s) | 20260506 13:47:34.950 | INFO | Analyzing chunk frames = [1828864,1829376) | 20260506 13:47:34.950 | INFO | MemInfo: { | 20260506 13:47:34.950 | INFO | "gpu" : " 17 GB", | 20260506 13:47:34.951 | INFO | "huge" : "", | 20260506 13:47:34.951 | INFO | "huge_shared" : " 27 GB", | 20260506 13:47:34.951 | INFO | "malloc" : " 14 GB", | 20260506 13:47:34.951 | INFO | "pinned" : " 29 GB" | 20260506 13:47:34.951 | INFO | } | 20260506 13:47:34.951 | INFO | Perf report for the last 5.15182 seconds: | 20260506 13:47:34.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:47:34.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:34.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:34.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:34.951 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 13:47:34.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:34.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:34.951 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 13:47:34.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:34.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:34.951 | INFO | Uploaded 36GiB to GPU in 2.00228s (17.9795GiB/s) | 20260506 13:47:34.951 | INFO | Downloaded 34GiB to GPU in 1.93741s (17.5492GiB/s) | 20260506 13:47:40.089 | INFO | Analyzing chunk frames = [1829376,1829888) | 20260506 13:47:40.089 | INFO | MemInfo: { | 20260506 13:47:40.089 | INFO | "gpu" : " 17 GB", | 20260506 13:47:40.089 | INFO | "huge" : "", | 20260506 13:47:40.089 | INFO | "huge_shared" : " 27 GB", | 20260506 13:47:40.089 | INFO | "malloc" : " 14 GB", | 20260506 13:47:40.090 | INFO | "pinned" : " 29 GB" | 20260506 13:47:40.090 | INFO | } | 20260506 13:47:40.090 | INFO | Perf report for the last 5.1389 seconds: | 20260506 13:47:40.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:47:40.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:40.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:40.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:40.090 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 13:47:40.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:40.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:40.090 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:47:40.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:40.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:40.090 | INFO | Uploaded 36GiB to GPU in 2.09315s (17.199GiB/s) | 20260506 13:47:40.090 | INFO | Downloaded 34GiB to GPU in 1.96722s (17.2832GiB/s) | 20260506 13:47:45.229 | INFO | Analyzing chunk frames = [1829888,1830400) | 20260506 13:47:45.229 | INFO | MemInfo: { | 20260506 13:47:45.229 | INFO | "gpu" : " 17 GB", | 20260506 13:47:45.229 | INFO | "huge" : "", | 20260506 13:47:45.229 | INFO | "huge_shared" : " 27 GB", | 20260506 13:47:45.229 | INFO | "malloc" : " 14 GB", | 20260506 13:47:45.229 | INFO | "pinned" : " 29 GB" | 20260506 13:47:45.229 | INFO | } | 20260506 13:47:45.229 | INFO | Perf report for the last 5.13993 seconds: | 20260506 13:47:45.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:47:45.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:45.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:45.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:45.230 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 13:47:45.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:45.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:45.230 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:47:45.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:45.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:45.230 | INFO | Uploaded 36GiB to GPU in 2.02099s (17.8131GiB/s) | 20260506 13:47:45.230 | INFO | Downloaded 34GiB to GPU in 2.03826s (16.6809GiB/s) | 20260506 13:47:50.369 | INFO | Analyzing chunk frames = [1830400,1830912) | 20260506 13:47:50.369 | INFO | MemInfo: { | 20260506 13:47:50.369 | INFO | "gpu" : " 17 GB", | 20260506 13:47:50.369 | INFO | "huge" : "", | 20260506 13:47:50.369 | INFO | "huge_shared" : " 27 GB", | 20260506 13:47:50.369 | INFO | "malloc" : " 14 GB", | 20260506 13:47:50.369 | INFO | "pinned" : " 29 GB" | 20260506 13:47:50.369 | INFO | } | 20260506 13:47:50.369 | INFO | Perf report for the last 5.13995 seconds: | 20260506 13:47:50.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:47:50.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:50.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:50.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:50.370 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 13:47:50.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:50.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:50.370 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:47:50.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:50.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:50.370 | INFO | Uploaded 36GiB to GPU in 2.06125s (17.4651GiB/s) | 20260506 13:47:50.370 | INFO | Downloaded 34GiB to GPU in 1.97396s (17.2243GiB/s) | 20260506 13:47:50.503 | INFO | Starting a round of Baz IO | 20260506 13:47:53.647 | INFO | Populating RTMetrics report | 20260506 13:47:53.648 | INFO | RTMetrics report populated | 20260506 13:47:53.648 | INFO | Opening tmp RTMetrics json file | 20260506 13:47:53.648 | INFO | Serializing RTMetrics to json | 20260506 13:47:53.649 | INFO | Serialization done, swapping tmp file now... | 20260506 13:47:53.650 | INFO | RTMetrics done | 20260506 13:47:53.747 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 3.244 seconds | 20260506 13:47:53.748 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.18% / 0.02% / 60.81% | 20260506 13:47:53.748 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.93785% / 1.58946e-05% / 5.5631e-05% | 20260506 13:47:53.748 | INFO | Baz IO overall: | 20260506 13:47:53.748 | INFO | BytesWritten : 935711382028 | 20260506 13:47:53.748 | INFO | NumEvents : 452665928790 | 20260506 13:47:55.509 | INFO | Analyzing chunk frames = [1830912,1831424) | 20260506 13:47:55.509 | INFO | MemInfo: { | 20260506 13:47:55.509 | INFO | "gpu" : " 17 GB", | 20260506 13:47:55.509 | INFO | "huge" : "", | 20260506 13:47:55.509 | INFO | "huge_shared" : " 27 GB", | 20260506 13:47:55.509 | INFO | "malloc" : " 14 GB", | 20260506 13:47:55.509 | INFO | "pinned" : " 29 GB" | 20260506 13:47:55.509 | INFO | } | 20260506 13:47:55.509 | INFO | Perf report for the last 5.13966 seconds: | 20260506 13:47:55.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:47:55.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:55.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:55.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:55.509 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260506 13:47:55.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:47:55.509 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 13:47:55.509 | INFO | PRE_HQ : 61.0%, 1.9, 60.5ms, 0 | 20260506 13:47:55.509 | INFO | BAZWRITER : 62.5%, 3.7, 120.0ms, 0 | 20260506 13:47:55.509 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 13:47:55.509 | INFO | Uploaded 36GiB to GPU in 2.09116s (17.2153GiB/s) | 20260506 13:47:55.509 | INFO | Downloaded 38GiB to GPU in 2.20897s (17.2026GiB/s) | 20260506 13:48:00.647 | INFO | Analyzing chunk frames = [1831424,1831936) | 20260506 13:48:00.648 | INFO | MemInfo: { | 20260506 13:48:00.648 | INFO | "gpu" : " 17 GB", | 20260506 13:48:00.648 | INFO | "huge" : "", | 20260506 13:48:00.648 | INFO | "huge_shared" : " 27 GB", | 20260506 13:48:00.648 | INFO | "malloc" : " 14 GB", | 20260506 13:48:00.648 | INFO | "pinned" : " 29 GB" | 20260506 13:48:00.648 | INFO | } | 20260506 13:48:00.648 | INFO | Perf report for the last 5.13866 seconds: | 20260506 13:48:00.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:48:00.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:00.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:00.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:00.648 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 13:48:00.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:00.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:00.648 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:48:00.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:00.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:00.648 | INFO | Uploaded 36GiB to GPU in 2.00551s (17.9505GiB/s) | 20260506 13:48:00.648 | INFO | Downloaded 34GiB to GPU in 1.91961s (17.712GiB/s) | 20260506 13:48:05.786 | INFO | Analyzing chunk frames = [1831936,1832448) | 20260506 13:48:05.786 | INFO | MemInfo: { | 20260506 13:48:05.786 | INFO | "gpu" : " 17 GB", | 20260506 13:48:05.786 | INFO | "huge" : "", | 20260506 13:48:05.786 | INFO | "huge_shared" : " 27 GB", | 20260506 13:48:05.786 | INFO | "malloc" : " 14 GB", | 20260506 13:48:05.786 | INFO | "pinned" : " 29 GB" | 20260506 13:48:05.786 | INFO | } | 20260506 13:48:05.787 | INFO | Perf report for the last 5.13871 seconds: | 20260506 13:48:05.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:48:05.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:05.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:05.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:05.787 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 13:48:05.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:05.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:05.787 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:48:05.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:05.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:05.787 | INFO | Uploaded 36GiB to GPU in 1.99102s (18.0812GiB/s) | 20260506 13:48:05.787 | INFO | Downloaded 34GiB to GPU in 1.93782s (17.5455GiB/s) | 20260506 13:48:10.926 | INFO | Analyzing chunk frames = [1832448,1832960) | 20260506 13:48:10.926 | INFO | MemInfo: { | 20260506 13:48:10.926 | INFO | "gpu" : " 17 GB", | 20260506 13:48:10.926 | INFO | "huge" : "", | 20260506 13:48:10.926 | INFO | "huge_shared" : " 27 GB", | 20260506 13:48:10.926 | INFO | "malloc" : " 14 GB", | 20260506 13:48:10.926 | INFO | "pinned" : " 29 GB" | 20260506 13:48:10.926 | INFO | } | 20260506 13:48:10.926 | INFO | Perf report for the last 5.13941 seconds: | 20260506 13:48:10.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:48:10.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:10.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:10.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:10.926 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 13:48:10.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:10.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:10.926 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 13:48:10.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:10.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:10.926 | INFO | Uploaded 36GiB to GPU in 1.9952s (18.0433GiB/s) | 20260506 13:48:10.926 | INFO | Downloaded 34GiB to GPU in 1.92786s (17.6362GiB/s) | 20260506 13:48:16.065 | INFO | Analyzing chunk frames = [1832960,1833472) | 20260506 13:48:16.065 | INFO | MemInfo: { | 20260506 13:48:16.065 | INFO | "gpu" : " 17 GB", | 20260506 13:48:16.065 | INFO | "huge" : "", | 20260506 13:48:16.065 | INFO | "huge_shared" : " 27 GB", | 20260506 13:48:16.065 | INFO | "malloc" : " 14 GB", | 20260506 13:48:16.065 | INFO | "pinned" : " 29 GB" | 20260506 13:48:16.065 | INFO | } | 20260506 13:48:16.065 | INFO | Perf report for the last 5.139 seconds: | 20260506 13:48:16.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:48:16.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:16.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:16.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:16.065 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 13:48:16.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:16.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:16.065 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:48:16.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:16.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:16.065 | INFO | Uploaded 36GiB to GPU in 1.96391s (18.3308GiB/s) | 20260506 13:48:16.065 | INFO | Downloaded 34GiB to GPU in 1.93073s (17.61GiB/s) | 20260506 13:48:21.204 | INFO | Analyzing chunk frames = [1833472,1833984) | 20260506 13:48:21.204 | INFO | MemInfo: { | 20260506 13:48:21.204 | INFO | "gpu" : " 17 GB", | 20260506 13:48:21.204 | INFO | "huge" : "", | 20260506 13:48:21.204 | INFO | "huge_shared" : " 27 GB", | 20260506 13:48:21.204 | INFO | "malloc" : " 14 GB", | 20260506 13:48:21.204 | INFO | "pinned" : " 29 GB" | 20260506 13:48:21.205 | INFO | } | 20260506 13:48:21.205 | INFO | Perf report for the last 5.13963 seconds: | 20260506 13:48:21.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:48:21.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:21.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:21.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:21.205 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 13:48:21.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:21.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:21.205 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 13:48:21.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:21.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:21.205 | INFO | Uploaded 36GiB to GPU in 2.0426s (17.6246GiB/s) | 20260506 13:48:21.205 | INFO | Downloaded 34GiB to GPU in 1.92439s (17.6679GiB/s) | 20260506 13:48:26.345 | INFO | Analyzing chunk frames = [1833984,1834496) | 20260506 13:48:26.345 | INFO | MemInfo: { | 20260506 13:48:26.345 | INFO | "gpu" : " 17 GB", | 20260506 13:48:26.345 | INFO | "huge" : "", | 20260506 13:48:26.345 | INFO | "huge_shared" : " 27 GB", | 20260506 13:48:26.345 | INFO | "malloc" : " 14 GB", | 20260506 13:48:26.345 | INFO | "pinned" : " 29 GB" | 20260506 13:48:26.345 | INFO | } | 20260506 13:48:26.345 | INFO | Perf report for the last 5.14047 seconds: | 20260506 13:48:26.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:48:26.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:26.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:26.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:26.345 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:48:26.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:26.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:26.345 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:48:26.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:26.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:26.346 | INFO | Uploaded 36GiB to GPU in 2.02068s (17.8158GiB/s) | 20260506 13:48:26.346 | INFO | Downloaded 34GiB to GPU in 1.99593s (17.0346GiB/s) | 20260506 13:48:31.484 | INFO | Analyzing chunk frames = [1834496,1835008) | 20260506 13:48:31.484 | INFO | MemInfo: { | 20260506 13:48:31.484 | INFO | "gpu" : " 17 GB", | 20260506 13:48:31.484 | INFO | "huge" : "", | 20260506 13:48:31.484 | INFO | "huge_shared" : " 27 GB", | 20260506 13:48:31.484 | INFO | "malloc" : " 14 GB", | 20260506 13:48:31.484 | INFO | "pinned" : " 29 GB" | 20260506 13:48:31.484 | INFO | } | 20260506 13:48:31.484 | INFO | Perf report for the last 5.1393 seconds: | 20260506 13:48:31.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:48:31.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:31.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:31.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:31.484 | INFO | ANALYSIS : 62.4%, 3.0, 97.5ms, 0 | 20260506 13:48:31.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:31.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:31.484 | INFO | PRE_HQ : 31.9%, 1.0, 16.7ms, 0 | 20260506 13:48:31.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:31.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:31.485 | INFO | Uploaded 36GiB to GPU in 2.06764s (17.4112GiB/s) | 20260506 13:48:31.485 | INFO | Downloaded 34GiB to GPU in 1.99178s (17.0701GiB/s) | 20260506 13:48:31.485 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:48:31.485 | INFO | Total mem usage: 17.4 GiB | 20260506 13:48:31.485 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:48:31.485 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -16.1%) | 20260506 13:48:31.485 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:48:31.485 | INFO | High Water Breakdown: | 20260506 13:48:31.485 | INFO | 60.5% : 9.07 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:48:31.485 | INFO | 39.4% : 5.91 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:48:31.485 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:48:31.485 | INFO | ------------------------------------------ | 20260506 13:48:31.485 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:48:31.485 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:48:31.485 | INFO | Total mem usage: 26.5 GiB | 20260506 13:48:31.485 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:48:31.485 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:48:31.485 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:48:31.485 | INFO | High Water Breakdown: | 20260506 13:48:31.485 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:48:31.485 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:48:31.485 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:48:31.485 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:48:31.485 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:48:31.485 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:48:31.485 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:48:31.485 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:48:31.485 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:48:31.485 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:48:31.485 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:48:31.485 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:48:31.485 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:48:31.485 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:48:31.485 | INFO | ------------------------------------------ | 20260506 13:48:31.485 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:48:31.485 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:48:31.485 | INFO | Total mem usage: 25.6 GiB | 20260506 13:48:31.485 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:48:31.485 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:48:31.485 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:48:31.485 | INFO | High Water Breakdown: | 20260506 13:48:31.485 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:48:31.485 | INFO | ------------------------------------------ | 20260506 13:48:31.485 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:48:31.485 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:48:31.485 | INFO | Total mem usage: 15.7 GiB | 20260506 13:48:31.485 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:48:31.485 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:48:31.485 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:48:31.485 | INFO | High Water Breakdown: | 20260506 13:48:31.485 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:48:31.485 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:48:31.485 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:48:31.485 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:48:31.485 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:48:31.485 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:48:31.485 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:48:31.485 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:48:31.485 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:48:31.485 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:48:31.485 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:48:31.485 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:48:31.485 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:48:31.485 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:48:31.485 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:48:31.485 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:48:31.485 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:48:31.486 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:48:31.486 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:48:31.486 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:48:31.486 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:48:31.486 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:48:31.486 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:48:31.486 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:48:31.486 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:48:31.486 | INFO | ------------------------------------------ | 20260506 13:48:31.486 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:48:31.486 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:48:31.486 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:48:31.486 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:48:31.486 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.58e+01, 1.34e+01 / 2.63e+00 | 20260506 13:48:31.486 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.01e-01, 1.06e-02 / 6.01e-03 | 20260506 13:48:31.486 | INFO | PerfReport: 11.55%, Baseline -- 3.65e+00 / 4.42e+00, 5.59e-01 / 3.46e-01 | 20260506 13:48:31.486 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.44e+00, 1.85e-01 / 1.50e-01 | 20260506 13:48:31.486 | INFO | PerfReport: 15.86%, DME -- 4.01e+01 / 4.43e+01, 3.99e+00 / 2.90e+00 | 20260506 13:48:31.486 | INFO | PerfReport: 56.62%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.41e+00 / 1.69e+00 | 20260506 13:48:31.486 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.37e+00 / 3.01e+00, 4.16e-01 / 2.23e-01 | 20260506 13:48:31.486 | INFO | PerfReport: 3.65%, Metrics -- 1.15e+00 / 1.53e+00, 2.92e-01 / 1.70e-01 | 20260506 13:48:31.486 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.04e-01 / 4.64e-01, 6.32e-02 / 3.12e-02 | 20260506 13:48:31.486 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:48:31.486 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:48:31.486 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:48:31.486 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 13:48:31.486 | INFO | PerfReport: 21.54%, Upload -- 2.12e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 13:48:31.486 | INFO | PerfReport: 21.06%, Download -- 2.07e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 13:48:31.486 | INFO | PerfReport: 32.14%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 13:48:31.486 | INFO | PerfReport: 1.77%, LockUpload -- 1.74e+00 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 13:48:31.486 | INFO | PerfReport: 1.04%, LockDownload -- 1.02e+00 / 0.00e+00, 2.60e+00 / 0.00e+00 | 20260506 13:48:31.486 | INFO | PerfReport: 22.44%, LockCompute -- 2.21e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 13:48:31.486 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260506 13:48:31.609 | INFO | Starting a round of Baz IO | 20260506 13:48:34.763 | INFO | Populating RTMetrics report | 20260506 13:48:34.763 | INFO | RTMetrics report populated | 20260506 13:48:34.763 | INFO | Opening tmp RTMetrics json file | 20260506 13:48:34.764 | INFO | Serializing RTMetrics to json | 20260506 13:48:34.765 | INFO | Serialization done, swapping tmp file now... | 20260506 13:48:34.765 | INFO | RTMetrics done | 20260506 13:48:34.870 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.261 seconds | 20260506 13:48:34.870 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.16% / 0.02% / 60.82% | 20260506 13:48:34.870 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.92991% / 1.58946e-05% / 4.37101e-05% | 20260506 13:48:34.870 | INFO | Baz IO overall: | 20260506 13:48:34.870 | INFO | BytesWritten : 937979203954 | 20260506 13:48:34.870 | INFO | NumEvents : 453765704889 | 20260506 13:48:36.624 | INFO | Analyzing chunk frames = [1835008,1835520) | 20260506 13:48:36.624 | INFO | MemInfo: { | 20260506 13:48:36.624 | INFO | "gpu" : " 17 GB", | 20260506 13:48:36.624 | INFO | "huge" : "", | 20260506 13:48:36.624 | INFO | "huge_shared" : " 27 GB", | 20260506 13:48:36.624 | INFO | "malloc" : " 14 GB", | 20260506 13:48:36.624 | INFO | "pinned" : " 29 GB" | 20260506 13:48:36.624 | INFO | } | 20260506 13:48:36.624 | INFO | Perf report for the last 5.13924 seconds: | 20260506 13:48:36.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:48:36.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:36.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:36.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:36.624 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260506 13:48:36.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:36.624 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 13:48:36.624 | INFO | PRE_HQ : 61.1%, 1.9, 61.1ms, 0 | 20260506 13:48:36.624 | INFO | BAZWRITER : 62.8%, 3.7, 120.6ms, 0 | 20260506 13:48:36.624 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:48:36.624 | INFO | Uploaded 36GiB to GPU in 2.04319s (17.6195GiB/s) | 20260506 13:48:36.624 | INFO | Downloaded 38GiB to GPU in 2.20315s (17.248GiB/s) | 20260506 13:48:41.762 | INFO | Analyzing chunk frames = [1835520,1836032) | 20260506 13:48:41.762 | INFO | MemInfo: { | 20260506 13:48:41.762 | INFO | "gpu" : " 17 GB", | 20260506 13:48:41.762 | INFO | "huge" : "", | 20260506 13:48:41.762 | INFO | "huge_shared" : " 27 GB", | 20260506 13:48:41.762 | INFO | "malloc" : " 14 GB", | 20260506 13:48:41.763 | INFO | "pinned" : " 29 GB" | 20260506 13:48:41.763 | INFO | } | 20260506 13:48:41.763 | INFO | Perf report for the last 5.13906 seconds: | 20260506 13:48:41.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:48:41.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:41.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:41.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:41.763 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 13:48:41.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:41.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:41.763 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 13:48:41.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:41.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:41.763 | INFO | Uploaded 36GiB to GPU in 2.00898s (17.9195GiB/s) | 20260506 13:48:41.763 | INFO | Downloaded 34GiB to GPU in 1.94335s (17.4956GiB/s) | 20260506 13:48:46.902 | INFO | Analyzing chunk frames = [1836032,1836544) | 20260506 13:48:46.902 | INFO | MemInfo: { | 20260506 13:48:46.902 | INFO | "gpu" : " 17 GB", | 20260506 13:48:46.902 | INFO | "huge" : "", | 20260506 13:48:46.902 | INFO | "huge_shared" : " 27 GB", | 20260506 13:48:46.902 | INFO | "malloc" : " 14 GB", | 20260506 13:48:46.902 | INFO | "pinned" : " 29 GB" | 20260506 13:48:46.902 | INFO | } | 20260506 13:48:46.902 | INFO | Perf report for the last 5.13923 seconds: | 20260506 13:48:46.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:48:46.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:46.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:46.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:46.902 | INFO | ANALYSIS : 61.4%, 3.0, 95.1ms, 0 | 20260506 13:48:46.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:46.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:46.902 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:48:46.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:46.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:46.902 | INFO | Uploaded 36GiB to GPU in 2.00867s (17.9224GiB/s) | 20260506 13:48:46.903 | INFO | Downloaded 34GiB to GPU in 1.93303s (17.589GiB/s) | 20260506 13:48:52.041 | INFO | Analyzing chunk frames = [1836544,1837056) | 20260506 13:48:52.041 | INFO | MemInfo: { | 20260506 13:48:52.041 | INFO | "gpu" : " 17 GB", | 20260506 13:48:52.041 | INFO | "huge" : "", | 20260506 13:48:52.041 | INFO | "huge_shared" : " 27 GB", | 20260506 13:48:52.041 | INFO | "malloc" : " 14 GB", | 20260506 13:48:52.041 | INFO | "pinned" : " 29 GB" | 20260506 13:48:52.041 | INFO | } | 20260506 13:48:52.041 | INFO | Perf report for the last 5.13882 seconds: | 20260506 13:48:52.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:48:52.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:52.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:52.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:52.041 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 13:48:52.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:52.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:52.041 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:48:52.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:52.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:52.041 | INFO | Uploaded 36GiB to GPU in 1.98261s (18.1578GiB/s) | 20260506 13:48:52.041 | INFO | Downloaded 34GiB to GPU in 1.91509s (17.7537GiB/s) | 20260506 13:48:57.180 | INFO | Analyzing chunk frames = [1837056,1837568) | 20260506 13:48:57.180 | INFO | MemInfo: { | 20260506 13:48:57.180 | INFO | "gpu" : " 17 GB", | 20260506 13:48:57.180 | INFO | "huge" : "", | 20260506 13:48:57.180 | INFO | "huge_shared" : " 27 GB", | 20260506 13:48:57.180 | INFO | "malloc" : " 14 GB", | 20260506 13:48:57.180 | INFO | "pinned" : " 29 GB" | 20260506 13:48:57.180 | INFO | } | 20260506 13:48:57.180 | INFO | Perf report for the last 5.13956 seconds: | 20260506 13:48:57.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:48:57.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:57.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:57.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:57.180 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 13:48:57.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:57.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:57.180 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 13:48:57.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:57.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:48:57.181 | INFO | Uploaded 36GiB to GPU in 2.03146s (17.7213GiB/s) | 20260506 13:48:57.181 | INFO | Downloaded 34GiB to GPU in 1.92642s (17.6493GiB/s) | 20260506 13:49:02.320 | INFO | Analyzing chunk frames = [1837568,1838080) | 20260506 13:49:02.320 | INFO | MemInfo: { | 20260506 13:49:02.320 | INFO | "gpu" : " 17 GB", | 20260506 13:49:02.320 | INFO | "huge" : "", | 20260506 13:49:02.320 | INFO | "huge_shared" : " 27 GB", | 20260506 13:49:02.320 | INFO | "malloc" : " 14 GB", | 20260506 13:49:02.320 | INFO | "pinned" : " 29 GB" | 20260506 13:49:02.320 | INFO | } | 20260506 13:49:02.320 | INFO | Perf report for the last 5.14001 seconds: | 20260506 13:49:02.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:49:02.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:02.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:02.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:02.320 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 13:49:02.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:02.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:02.321 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:49:02.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:02.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:02.321 | INFO | Uploaded 36GiB to GPU in 2.06171s (17.4612GiB/s) | 20260506 13:49:02.321 | INFO | Downloaded 34GiB to GPU in 1.92942s (17.6219GiB/s) | 20260506 13:49:07.460 | INFO | Analyzing chunk frames = [1838080,1838592) | 20260506 13:49:07.460 | INFO | MemInfo: { | 20260506 13:49:07.460 | INFO | "gpu" : " 17 GB", | 20260506 13:49:07.460 | INFO | "huge" : "", | 20260506 13:49:07.460 | INFO | "huge_shared" : " 27 GB", | 20260506 13:49:07.460 | INFO | "malloc" : " 14 GB", | 20260506 13:49:07.460 | INFO | "pinned" : " 29 GB" | 20260506 13:49:07.460 | INFO | } | 20260506 13:49:07.460 | INFO | Perf report for the last 5.13985 seconds: | 20260506 13:49:07.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:49:07.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:07.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:07.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:07.460 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 13:49:07.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:07.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:07.460 | INFO | PRE_HQ : 29.8%, 1.0, 15.6ms, 0 | 20260506 13:49:07.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:07.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:07.461 | INFO | Uploaded 36GiB to GPU in 2.00885s (17.9207GiB/s) | 20260506 13:49:07.461 | INFO | Downloaded 34GiB to GPU in 1.98281s (17.1474GiB/s) | 20260506 13:49:12.598 | INFO | Analyzing chunk frames = [1838592,1839104) | 20260506 13:49:12.598 | INFO | MemInfo: { | 20260506 13:49:12.598 | INFO | "gpu" : " 17 GB", | 20260506 13:49:12.598 | INFO | "huge" : "", | 20260506 13:49:12.598 | INFO | "huge_shared" : " 27 GB", | 20260506 13:49:12.598 | INFO | "malloc" : " 14 GB", | 20260506 13:49:12.598 | INFO | "pinned" : " 29 GB" | 20260506 13:49:12.598 | INFO | } | 20260506 13:49:12.598 | INFO | Perf report for the last 5.13755 seconds: | 20260506 13:49:12.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:49:12.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:12.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:12.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:12.598 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 13:49:12.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:12.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:12.598 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:49:12.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:12.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:12.598 | INFO | Uploaded 36GiB to GPU in 2.04771s (17.5806GiB/s) | 20260506 13:49:12.598 | INFO | Downloaded 34GiB to GPU in 1.98588s (17.1209GiB/s) | 20260506 13:49:12.714 | INFO | Starting a round of Baz IO | 20260506 13:49:15.873 | INFO | Populating RTMetrics report | 20260506 13:49:15.873 | INFO | RTMetrics report populated | 20260506 13:49:15.873 | INFO | Opening tmp RTMetrics json file | 20260506 13:49:15.874 | INFO | Serializing RTMetrics to json | 20260506 13:49:15.875 | INFO | Serialization done, swapping tmp file now... | 20260506 13:49:15.875 | INFO | RTMetrics done | 20260506 13:49:15.974 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.259 seconds | 20260506 13:49:15.974 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.14% / 0.02% / 60.84% | 20260506 13:49:15.974 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.9162% / 7.94729e-06% / 9.13938e-05% | 20260506 13:49:15.974 | INFO | Baz IO overall: | 20260506 13:49:15.974 | INFO | BytesWritten : 940245099392 | 20260506 13:49:15.974 | INFO | NumEvents : 454864037614 | 20260506 13:49:17.737 | INFO | Analyzing chunk frames = [1839104,1839616) | 20260506 13:49:17.737 | INFO | MemInfo: { | 20260506 13:49:17.737 | INFO | "gpu" : " 17 GB", | 20260506 13:49:17.737 | INFO | "huge" : "", | 20260506 13:49:17.737 | INFO | "huge_shared" : " 27 GB", | 20260506 13:49:17.737 | INFO | "malloc" : " 14 GB", | 20260506 13:49:17.738 | INFO | "pinned" : " 29 GB" | 20260506 13:49:17.738 | INFO | } | 20260506 13:49:17.738 | INFO | Perf report for the last 5.13987 seconds: | 20260506 13:49:17.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:49:17.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:17.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:17.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:17.738 | INFO | ANALYSIS : 63.2%, 2.9, 95.4ms, 0 | 20260506 13:49:17.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:17.738 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 13:49:17.738 | INFO | PRE_HQ : 61.1%, 1.9, 60.2ms, 0 | 20260506 13:49:17.738 | INFO | BAZWRITER : 62.8%, 4.4, 142.4ms, 0 | 20260506 13:49:17.738 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 13:49:17.738 | INFO | Uploaded 36GiB to GPU in 2.08803s (17.2411GiB/s) | 20260506 13:49:17.738 | INFO | Downloaded 38GiB to GPU in 2.23304s (17.0172GiB/s) | 20260506 13:49:22.877 | INFO | Analyzing chunk frames = [1839616,1840128) | 20260506 13:49:22.877 | INFO | MemInfo: { | 20260506 13:49:22.877 | INFO | "gpu" : " 17 GB", | 20260506 13:49:22.877 | INFO | "huge" : "", | 20260506 13:49:22.877 | INFO | "huge_shared" : " 27 GB", | 20260506 13:49:22.877 | INFO | "malloc" : " 14 GB", | 20260506 13:49:22.877 | INFO | "pinned" : " 29 GB" | 20260506 13:49:22.877 | INFO | } | 20260506 13:49:22.877 | INFO | Perf report for the last 5.13905 seconds: | 20260506 13:49:22.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:49:22.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:22.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:22.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:22.877 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 13:49:22.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:22.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:22.877 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:49:22.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:22.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:22.877 | INFO | Uploaded 36GiB to GPU in 2.02278s (17.7973GiB/s) | 20260506 13:49:22.877 | INFO | Downloaded 34GiB to GPU in 1.93941s (17.5311GiB/s) | 20260506 13:49:28.017 | INFO | Analyzing chunk frames = [1840128,1840640) | 20260506 13:49:28.017 | INFO | MemInfo: { | 20260506 13:49:28.017 | INFO | "gpu" : " 17 GB", | 20260506 13:49:28.017 | INFO | "huge" : "", | 20260506 13:49:28.017 | INFO | "huge_shared" : " 27 GB", | 20260506 13:49:28.017 | INFO | "malloc" : " 14 GB", | 20260506 13:49:28.017 | INFO | "pinned" : " 29 GB" | 20260506 13:49:28.017 | INFO | } | 20260506 13:49:28.017 | INFO | Perf report for the last 5.14008 seconds: | 20260506 13:49:28.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:49:28.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:28.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:28.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:28.017 | INFO | ANALYSIS : 61.5%, 3.0, 95.3ms, 0 | 20260506 13:49:28.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:28.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:28.017 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:49:28.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:28.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:28.017 | INFO | Uploaded 36GiB to GPU in 2.0178s (17.8412GiB/s) | 20260506 13:49:28.018 | INFO | Downloaded 34GiB to GPU in 1.93366s (17.5832GiB/s) | 20260506 13:49:33.155 | INFO | Analyzing chunk frames = [1840640,1841152) | 20260506 13:49:33.155 | INFO | MemInfo: { | 20260506 13:49:33.155 | INFO | "gpu" : " 17 GB", | 20260506 13:49:33.155 | INFO | "huge" : "", | 20260506 13:49:33.155 | INFO | "huge_shared" : " 27 GB", | 20260506 13:49:33.155 | INFO | "malloc" : " 14 GB", | 20260506 13:49:33.155 | INFO | "pinned" : " 29 GB" | 20260506 13:49:33.155 | INFO | } | 20260506 13:49:33.156 | INFO | Perf report for the last 5.13868 seconds: | 20260506 13:49:33.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:49:33.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:33.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:33.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:33.156 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 13:49:33.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:33.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:33.156 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:49:33.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:33.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:33.156 | INFO | Uploaded 36GiB to GPU in 1.98908s (18.0988GiB/s) | 20260506 13:49:33.156 | INFO | Downloaded 34GiB to GPU in 1.93102s (17.6073GiB/s) | 20260506 13:49:38.295 | INFO | Analyzing chunk frames = [1841152,1841664) | 20260506 13:49:38.295 | INFO | MemInfo: { | 20260506 13:49:38.295 | INFO | "gpu" : " 17 GB", | 20260506 13:49:38.295 | INFO | "huge" : "", | 20260506 13:49:38.295 | INFO | "huge_shared" : " 27 GB", | 20260506 13:49:38.295 | INFO | "malloc" : " 14 GB", | 20260506 13:49:38.295 | INFO | "pinned" : " 29 GB" | 20260506 13:49:38.295 | INFO | } | 20260506 13:49:38.295 | INFO | Perf report for the last 5.13922 seconds: | 20260506 13:49:38.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:49:38.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:38.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:38.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:38.295 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 13:49:38.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:38.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:38.295 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:49:38.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:38.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:38.295 | INFO | Uploaded 36GiB to GPU in 2.03252s (17.712GiB/s) | 20260506 13:49:38.295 | INFO | Downloaded 34GiB to GPU in 1.92659s (17.6477GiB/s) | 20260506 13:49:43.434 | INFO | Analyzing chunk frames = [1841664,1842176) | 20260506 13:49:43.434 | INFO | MemInfo: { | 20260506 13:49:43.434 | INFO | "gpu" : " 17 GB", | 20260506 13:49:43.434 | INFO | "huge" : "", | 20260506 13:49:43.434 | INFO | "huge_shared" : " 27 GB", | 20260506 13:49:43.434 | INFO | "malloc" : " 14 GB", | 20260506 13:49:43.435 | INFO | "pinned" : " 29 GB" | 20260506 13:49:43.435 | INFO | } | 20260506 13:49:43.435 | INFO | Perf report for the last 5.13998 seconds: | 20260506 13:49:43.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:49:43.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:43.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:43.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:43.435 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 13:49:43.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:43.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:43.435 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 13:49:43.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:43.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:43.435 | INFO | Uploaded 36GiB to GPU in 2.04466s (17.6069GiB/s) | 20260506 13:49:43.435 | INFO | Downloaded 34GiB to GPU in 1.91493s (17.7552GiB/s) | 20260506 13:49:48.575 | INFO | Analyzing chunk frames = [1842176,1842688) | 20260506 13:49:48.575 | INFO | MemInfo: { | 20260506 13:49:48.575 | INFO | "gpu" : " 17 GB", | 20260506 13:49:48.575 | INFO | "huge" : "", | 20260506 13:49:48.575 | INFO | "huge_shared" : " 27 GB", | 20260506 13:49:48.575 | INFO | "malloc" : " 14 GB", | 20260506 13:49:48.575 | INFO | "pinned" : " 29 GB" | 20260506 13:49:48.575 | INFO | } | 20260506 13:49:48.575 | INFO | Perf report for the last 5.14016 seconds: | 20260506 13:49:48.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:49:48.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:48.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:48.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:48.575 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:49:48.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:48.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:48.575 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 13:49:48.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:48.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:48.575 | INFO | Uploaded 36GiB to GPU in 2.00953s (17.9147GiB/s) | 20260506 13:49:48.575 | INFO | Downloaded 34GiB to GPU in 2.0077s (16.9348GiB/s) | 20260506 13:49:53.713 | INFO | Analyzing chunk frames = [1842688,1843200) | 20260506 13:49:53.713 | INFO | MemInfo: { | 20260506 13:49:53.713 | INFO | "gpu" : " 17 GB", | 20260506 13:49:53.713 | INFO | "huge" : "", | 20260506 13:49:53.713 | INFO | "huge_shared" : " 27 GB", | 20260506 13:49:53.713 | INFO | "malloc" : " 14 GB", | 20260506 13:49:53.713 | INFO | "pinned" : " 29 GB" | 20260506 13:49:53.713 | INFO | } | 20260506 13:49:53.713 | INFO | Perf report for the last 5.13803 seconds: | 20260506 13:49:53.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:49:53.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:53.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:53.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:53.713 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 13:49:53.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:53.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:53.713 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:49:53.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:53.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:53.713 | INFO | Uploaded 36GiB to GPU in 2.03743s (17.6693GiB/s) | 20260506 13:49:53.713 | INFO | Downloaded 34GiB to GPU in 1.96721s (17.2834GiB/s) | 20260506 13:49:53.714 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:49:53.714 | INFO | Total mem usage: 17.4 GiB | 20260506 13:49:53.714 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:49:53.714 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -13.5%) | 20260506 13:49:53.714 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:49:53.714 | INFO | High Water Breakdown: | 20260506 13:49:53.714 | INFO | 60.9% : 9.34 GiB ( 85.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:49:53.714 | INFO | 39% : 5.99 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:49:53.714 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:49:53.714 | INFO | ------------------------------------------ | 20260506 13:49:53.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:49:53.714 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:49:53.714 | INFO | Total mem usage: 26.5 GiB | 20260506 13:49:53.714 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:49:53.714 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:49:53.714 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:49:53.714 | INFO | High Water Breakdown: | 20260506 13:49:53.714 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:49:53.714 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:49:53.714 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:49:53.714 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:49:53.714 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:49:53.714 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:49:53.714 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:49:53.714 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:49:53.714 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:49:53.714 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:49:53.714 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:49:53.714 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:49:53.714 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:49:53.714 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:49:53.714 | INFO | ------------------------------------------ | 20260506 13:49:53.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:49:53.714 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:49:53.714 | INFO | Total mem usage: 25.6 GiB | 20260506 13:49:53.714 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:49:53.714 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:49:53.714 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:49:53.714 | INFO | High Water Breakdown: | 20260506 13:49:53.714 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:49:53.714 | INFO | ------------------------------------------ | 20260506 13:49:53.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:49:53.714 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:49:53.714 | INFO | Total mem usage: 15.7 GiB | 20260506 13:49:53.714 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:49:53.714 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:49:53.714 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 13:49:53.714 | INFO | High Water Breakdown: | 20260506 13:49:53.714 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:49:53.714 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:49:53.714 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:49:53.714 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:49:53.714 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:49:53.714 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:49:53.714 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:49:53.714 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:49:53.714 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:49:53.714 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:49:53.714 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:49:53.714 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:49:53.714 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:49:53.714 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:49:53.714 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:49:53.714 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:49:53.714 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:49:53.714 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:49:53.714 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:49:53.714 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:49:53.714 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:49:53.714 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:49:53.714 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:49:53.714 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:49:53.714 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:49:53.714 | INFO | ------------------------------------------ | 20260506 13:49:53.714 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:49:53.714 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:49:53.714 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:49:53.714 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:49:53.714 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.55e+01, 1.35e+01 / 2.03e+00 | 20260506 13:49:53.714 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.01e-01, 1.05e-02 / 6.55e-03 | 20260506 13:49:53.715 | INFO | PerfReport: 11.59%, Baseline -- 3.66e+00 / 4.43e+00, 5.79e-01 / 3.33e-01 | 20260506 13:49:53.715 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.46e+00, 1.90e-01 / 1.50e-01 | 20260506 13:49:53.715 | INFO | PerfReport: 15.80%, DME -- 4.00e+01 / 4.41e+01, 3.91e+00 / 2.29e+00 | 20260506 13:49:53.715 | INFO | PerfReport: 56.68%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.48e+00 / 1.95e+00 | 20260506 13:49:53.715 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.36e+00 / 2.94e+00, 4.11e-01 / 2.87e-01 | 20260506 13:49:53.715 | INFO | PerfReport: 3.64%, Metrics -- 1.15e+00 / 1.53e+00, 2.96e-01 / 1.94e-01 | 20260506 13:49:53.715 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.03e-01 / 4.58e-01, 6.31e-02 / 3.32e-02 | 20260506 13:49:53.715 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:49:53.715 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:49:53.715 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:49:53.715 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260506 13:49:53.715 | INFO | PerfReport: 21.51%, Upload -- 2.11e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260506 13:49:53.715 | INFO | PerfReport: 21.01%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 13:49:53.715 | INFO | PerfReport: 32.20%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 13:49:53.715 | INFO | PerfReport: 1.74%, LockUpload -- 1.70e+00 / 0.00e+00, 6.70e+00 / 0.00e+00 | 20260506 13:49:53.715 | INFO | PerfReport: 1.02%, LockDownload -- 1.00e+00 / 0.00e+00, 2.62e+00 / 0.00e+00 | 20260506 13:49:53.715 | INFO | PerfReport: 22.51%, LockCompute -- 2.21e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 13:49:53.715 | INFO | PerfReport: 0.02%, Misc -- 1.55e-02 / 0.00e+00, 3.16e-02 / 0.00e+00 | 20260506 13:49:53.837 | INFO | Starting a round of Baz IO | 20260506 13:49:56.992 | INFO | Populating RTMetrics report | 20260506 13:49:56.992 | INFO | RTMetrics report populated | 20260506 13:49:56.992 | INFO | Opening tmp RTMetrics json file | 20260506 13:49:56.993 | INFO | Serializing RTMetrics to json | 20260506 13:49:56.994 | INFO | Serialization done, swapping tmp file now... | 20260506 13:49:56.995 | INFO | RTMetrics done | 20260506 13:49:57.093 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.255 seconds | 20260506 13:49:57.093 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.13% / 0.02% / 60.86% | 20260506 13:49:57.093 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.91656% / 1.98682e-05% / 3.97364e-05% | 20260506 13:49:57.093 | INFO | Baz IO overall: | 20260506 13:49:57.093 | INFO | BytesWritten : 942513159599 | 20260506 13:49:57.093 | INFO | NumEvents : 455963959589 | 20260506 13:49:58.853 | INFO | Analyzing chunk frames = [1843200,1843712) | 20260506 13:49:58.853 | INFO | MemInfo: { | 20260506 13:49:58.853 | INFO | "gpu" : " 17 GB", | 20260506 13:49:58.853 | INFO | "huge" : "", | 20260506 13:49:58.853 | INFO | "huge_shared" : " 27 GB", | 20260506 13:49:58.853 | INFO | "malloc" : " 14 GB", | 20260506 13:49:58.853 | INFO | "pinned" : " 29 GB" | 20260506 13:49:58.853 | INFO | } | 20260506 13:49:58.853 | INFO | Perf report for the last 5.14004 seconds: | 20260506 13:49:58.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:49:58.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:58.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:58.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:58.853 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260506 13:49:58.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:49:58.853 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 13:49:58.853 | INFO | PRE_HQ : 61.2%, 1.9, 60.9ms, 0 | 20260506 13:49:58.853 | INFO | BAZWRITER : 62.7%, 3.7, 119.6ms, 0 | 20260506 13:49:58.853 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:49:58.853 | INFO | Uploaded 36GiB to GPU in 2.04876s (17.5716GiB/s) | 20260506 13:49:58.853 | INFO | Downloaded 38GiB to GPU in 2.20376s (17.2433GiB/s) | 20260506 13:50:03.992 | INFO | Analyzing chunk frames = [1843712,1844224) | 20260506 13:50:03.992 | INFO | MemInfo: { | 20260506 13:50:03.992 | INFO | "gpu" : " 17 GB", | 20260506 13:50:03.992 | INFO | "huge" : "", | 20260506 13:50:03.992 | INFO | "huge_shared" : " 27 GB", | 20260506 13:50:03.992 | INFO | "malloc" : " 14 GB", | 20260506 13:50:03.992 | INFO | "pinned" : " 29 GB" | 20260506 13:50:03.992 | INFO | } | 20260506 13:50:03.992 | INFO | Perf report for the last 5.13881 seconds: | 20260506 13:50:03.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:50:03.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:03.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:03.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:03.992 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 13:50:03.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:03.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:03.992 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:50:03.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:03.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:03.992 | INFO | Uploaded 36GiB to GPU in 2.00449s (17.9597GiB/s) | 20260506 13:50:03.992 | INFO | Downloaded 34GiB to GPU in 1.93935s (17.5316GiB/s) | 20260506 13:50:09.132 | INFO | Analyzing chunk frames = [1844224,1844736) | 20260506 13:50:09.132 | INFO | MemInfo: { | 20260506 13:50:09.132 | INFO | "gpu" : " 17 GB", | 20260506 13:50:09.132 | INFO | "huge" : "", | 20260506 13:50:09.132 | INFO | "huge_shared" : " 27 GB", | 20260506 13:50:09.132 | INFO | "malloc" : " 14 GB", | 20260506 13:50:09.132 | INFO | "pinned" : " 29 GB" | 20260506 13:50:09.132 | INFO | } | 20260506 13:50:09.132 | INFO | Perf report for the last 5.14018 seconds: | 20260506 13:50:09.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:50:09.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:09.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:09.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:09.132 | INFO | ANALYSIS : 61.4%, 3.0, 95.0ms, 0 | 20260506 13:50:09.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:09.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:09.132 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:50:09.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:09.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:09.132 | INFO | Uploaded 36GiB to GPU in 2.00115s (17.9896GiB/s) | 20260506 13:50:09.132 | INFO | Downloaded 34GiB to GPU in 1.93134s (17.6044GiB/s) | 20260506 13:50:14.271 | INFO | Analyzing chunk frames = [1844736,1845248) | 20260506 13:50:14.271 | INFO | MemInfo: { | 20260506 13:50:14.271 | INFO | "gpu" : " 17 GB", | 20260506 13:50:14.271 | INFO | "huge" : "", | 20260506 13:50:14.271 | INFO | "huge_shared" : " 27 GB", | 20260506 13:50:14.272 | INFO | "malloc" : " 14 GB", | 20260506 13:50:14.272 | INFO | "pinned" : " 29 GB" | 20260506 13:50:14.272 | INFO | } | 20260506 13:50:14.272 | INFO | Perf report for the last 5.13954 seconds: | 20260506 13:50:14.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:50:14.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:14.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:14.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:14.272 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 13:50:14.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:14.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:14.272 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:50:14.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:14.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:14.272 | INFO | Uploaded 36GiB to GPU in 1.99149s (18.0769GiB/s) | 20260506 13:50:14.272 | INFO | Downloaded 34GiB to GPU in 1.92756s (17.6389GiB/s) | 20260506 13:50:19.409 | INFO | Analyzing chunk frames = [1845248,1845760) | 20260506 13:50:19.409 | INFO | MemInfo: { | 20260506 13:50:19.409 | INFO | "gpu" : " 17 GB", | 20260506 13:50:19.409 | INFO | "huge" : "", | 20260506 13:50:19.409 | INFO | "huge_shared" : " 27 GB", | 20260506 13:50:19.410 | INFO | "malloc" : " 14 GB", | 20260506 13:50:19.410 | INFO | "pinned" : " 29 GB" | 20260506 13:50:19.410 | INFO | } | 20260506 13:50:19.410 | INFO | Perf report for the last 5.13802 seconds: | 20260506 13:50:19.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:50:19.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:19.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:19.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:19.410 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 13:50:19.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:19.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:19.410 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 13:50:19.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:19.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:19.410 | INFO | Uploaded 36GiB to GPU in 2.05033s (17.5582GiB/s) | 20260506 13:50:19.410 | INFO | Downloaded 34GiB to GPU in 1.93855s (17.5389GiB/s) | 20260506 13:50:24.548 | INFO | Analyzing chunk frames = [1845760,1846272) | 20260506 13:50:24.548 | INFO | MemInfo: { | 20260506 13:50:24.548 | INFO | "gpu" : " 17 GB", | 20260506 13:50:24.548 | INFO | "huge" : "", | 20260506 13:50:24.548 | INFO | "huge_shared" : " 27 GB", | 20260506 13:50:24.548 | INFO | "malloc" : " 14 GB", | 20260506 13:50:24.548 | INFO | "pinned" : " 29 GB" | 20260506 13:50:24.548 | INFO | } | 20260506 13:50:24.548 | INFO | Perf report for the last 5.13881 seconds: | 20260506 13:50:24.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:50:24.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:24.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:24.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:24.548 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 13:50:24.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:24.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:24.548 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 13:50:24.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:24.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:24.549 | INFO | Uploaded 36GiB to GPU in 2.05483s (17.5197GiB/s) | 20260506 13:50:24.549 | INFO | Downloaded 34GiB to GPU in 1.88673s (18.0206GiB/s) | 20260506 13:50:29.689 | INFO | Analyzing chunk frames = [1846272,1846784) | 20260506 13:50:29.689 | INFO | MemInfo: { | 20260506 13:50:29.689 | INFO | "gpu" : " 17 GB", | 20260506 13:50:29.689 | INFO | "huge" : "", | 20260506 13:50:29.689 | INFO | "huge_shared" : " 27 GB", | 20260506 13:50:29.689 | INFO | "malloc" : " 14 GB", | 20260506 13:50:29.689 | INFO | "pinned" : " 29 GB" | 20260506 13:50:29.689 | INFO | } | 20260506 13:50:29.689 | INFO | Perf report for the last 5.14085 seconds: | 20260506 13:50:29.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:50:29.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:29.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:29.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:29.689 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:50:29.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:29.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:29.689 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 13:50:29.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:29.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:29.690 | INFO | Uploaded 36GiB to GPU in 2.00613s (17.945GiB/s) | 20260506 13:50:29.690 | INFO | Downloaded 34GiB to GPU in 2.00384s (16.9674GiB/s) | 20260506 13:50:34.829 | INFO | Analyzing chunk frames = [1846784,1847296) | 20260506 13:50:34.829 | INFO | MemInfo: { | 20260506 13:50:34.829 | INFO | "gpu" : " 17 GB", | 20260506 13:50:34.829 | INFO | "huge" : "", | 20260506 13:50:34.829 | INFO | "huge_shared" : " 27 GB", | 20260506 13:50:34.829 | INFO | "malloc" : " 14 GB", | 20260506 13:50:34.829 | INFO | "pinned" : " 29 GB" | 20260506 13:50:34.829 | INFO | } | 20260506 13:50:34.829 | INFO | Perf report for the last 5.1394 seconds: | 20260506 13:50:34.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:50:34.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:34.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:34.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:34.829 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:50:34.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:34.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:34.829 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 13:50:34.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:34.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:34.829 | INFO | Uploaded 36GiB to GPU in 2.04498s (17.6041GiB/s) | 20260506 13:50:34.829 | INFO | Downloaded 34GiB to GPU in 1.97951s (17.1759GiB/s) | 20260506 13:50:34.953 | INFO | Starting a round of Baz IO | 20260506 13:50:38.113 | INFO | Populating RTMetrics report | 20260506 13:50:38.113 | INFO | RTMetrics report populated | 20260506 13:50:38.113 | INFO | Opening tmp RTMetrics json file | 20260506 13:50:38.114 | INFO | Serializing RTMetrics to json | 20260506 13:50:38.115 | INFO | Serialization done, swapping tmp file now... | 20260506 13:50:38.115 | INFO | RTMetrics done | 20260506 13:50:38.214 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.26 seconds | 20260506 13:50:38.214 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.11% / 0.02% / 60.87% | 20260506 13:50:38.214 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.91161% / 5.16574e-05% / 4.37101e-05% | 20260506 13:50:38.214 | INFO | Baz IO overall: | 20260506 13:50:38.214 | INFO | BytesWritten : 944781282144 | 20260506 13:50:38.214 | INFO | NumEvents : 457064581320 | 20260506 13:50:39.967 | INFO | Analyzing chunk frames = [1847296,1847808) | 20260506 13:50:39.967 | INFO | MemInfo: { | 20260506 13:50:39.967 | INFO | "gpu" : " 17 GB", | 20260506 13:50:39.967 | INFO | "huge" : "", | 20260506 13:50:39.967 | INFO | "huge_shared" : " 27 GB", | 20260506 13:50:39.967 | INFO | "malloc" : " 14 GB", | 20260506 13:50:39.967 | INFO | "pinned" : " 29 GB" | 20260506 13:50:39.967 | INFO | } | 20260506 13:50:39.967 | INFO | Perf report for the last 5.13856 seconds: | 20260506 13:50:39.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:50:39.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:39.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:39.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:39.967 | INFO | ANALYSIS : 63.3%, 3.0, 98.3ms, 0 | 20260506 13:50:39.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:39.967 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 13:50:39.967 | INFO | PRE_HQ : 61.1%, 1.9, 60.6ms, 0 | 20260506 13:50:39.967 | INFO | BAZWRITER : 62.8%, 3.7, 119.8ms, 0 | 20260506 13:50:39.967 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:50:39.967 | INFO | Uploaded 36GiB to GPU in 2.05266s (17.5382GiB/s) | 20260506 13:50:39.968 | INFO | Downloaded 38GiB to GPU in 2.21663s (17.1432GiB/s) | 20260506 13:50:45.106 | INFO | Analyzing chunk frames = [1847808,1848320) | 20260506 13:50:45.106 | INFO | MemInfo: { | 20260506 13:50:45.106 | INFO | "gpu" : " 17 GB", | 20260506 13:50:45.106 | INFO | "huge" : "", | 20260506 13:50:45.106 | INFO | "huge_shared" : " 27 GB", | 20260506 13:50:45.106 | INFO | "malloc" : " 14 GB", | 20260506 13:50:45.106 | INFO | "pinned" : " 29 GB" | 20260506 13:50:45.106 | INFO | } | 20260506 13:50:45.106 | INFO | Perf report for the last 5.13898 seconds: | 20260506 13:50:45.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:50:45.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:45.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:45.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:45.106 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 13:50:45.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:45.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:45.106 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 13:50:45.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:45.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:45.107 | INFO | Uploaded 36GiB to GPU in 2.01078s (17.9035GiB/s) | 20260506 13:50:45.107 | INFO | Downloaded 34GiB to GPU in 1.93886s (17.536GiB/s) | 20260506 13:50:50.245 | INFO | Analyzing chunk frames = [1848320,1848832) | 20260506 13:50:50.245 | INFO | MemInfo: { | 20260506 13:50:50.245 | INFO | "gpu" : " 17 GB", | 20260506 13:50:50.245 | INFO | "huge" : "", | 20260506 13:50:50.245 | INFO | "huge_shared" : " 27 GB", | 20260506 13:50:50.245 | INFO | "malloc" : " 14 GB", | 20260506 13:50:50.245 | INFO | "pinned" : " 29 GB" | 20260506 13:50:50.245 | INFO | } | 20260506 13:50:50.245 | INFO | Perf report for the last 5.13919 seconds: | 20260506 13:50:50.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:50:50.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:50.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:50.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:50.246 | INFO | ANALYSIS : 61.4%, 3.0, 95.0ms, 0 | 20260506 13:50:50.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:50.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:50.246 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 13:50:50.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:50.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:50.246 | INFO | Uploaded 36GiB to GPU in 2.00807s (17.9277GiB/s) | 20260506 13:50:50.246 | INFO | Downloaded 34GiB to GPU in 1.93158s (17.6022GiB/s) | 20260506 13:50:55.385 | INFO | Analyzing chunk frames = [1848832,1849344) | 20260506 13:50:55.385 | INFO | MemInfo: { | 20260506 13:50:55.385 | INFO | "gpu" : " 17 GB", | 20260506 13:50:55.385 | INFO | "huge" : "", | 20260506 13:50:55.385 | INFO | "huge_shared" : " 27 GB", | 20260506 13:50:55.385 | INFO | "malloc" : " 14 GB", | 20260506 13:50:55.386 | INFO | "pinned" : " 29 GB" | 20260506 13:50:55.386 | INFO | } | 20260506 13:50:55.386 | INFO | Perf report for the last 5.14005 seconds: | 20260506 13:50:55.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:50:55.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:55.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:55.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:55.386 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 13:50:55.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:55.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:55.386 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:50:55.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:55.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:50:55.386 | INFO | Uploaded 36GiB to GPU in 1.9881s (18.1077GiB/s) | 20260506 13:50:55.386 | INFO | Downloaded 34GiB to GPU in 1.91174s (17.7849GiB/s) | 20260506 13:51:00.525 | INFO | Analyzing chunk frames = [1849344,1849856) | 20260506 13:51:00.525 | INFO | MemInfo: { | 20260506 13:51:00.525 | INFO | "gpu" : " 17 GB", | 20260506 13:51:00.525 | INFO | "huge" : "", | 20260506 13:51:00.525 | INFO | "huge_shared" : " 27 GB", | 20260506 13:51:00.525 | INFO | "malloc" : " 14 GB", | 20260506 13:51:00.525 | INFO | "pinned" : " 29 GB" | 20260506 13:51:00.525 | INFO | } | 20260506 13:51:00.525 | INFO | Perf report for the last 5.13975 seconds: | 20260506 13:51:00.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:51:00.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:00.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:00.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:00.525 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 13:51:00.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:00.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:00.525 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:51:00.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:00.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:00.525 | INFO | Uploaded 36GiB to GPU in 1.99795s (18.0185GiB/s) | 20260506 13:51:00.526 | INFO | Downloaded 34GiB to GPU in 1.92892s (17.6265GiB/s) | 20260506 13:51:05.665 | INFO | Analyzing chunk frames = [1849856,1850368) | 20260506 13:51:05.665 | INFO | MemInfo: { | 20260506 13:51:05.665 | INFO | "gpu" : " 17 GB", | 20260506 13:51:05.665 | INFO | "huge" : "", | 20260506 13:51:05.665 | INFO | "huge_shared" : " 27 GB", | 20260506 13:51:05.665 | INFO | "malloc" : " 14 GB", | 20260506 13:51:05.665 | INFO | "pinned" : " 29 GB" | 20260506 13:51:05.665 | INFO | } | 20260506 13:51:05.665 | INFO | Perf report for the last 5.13936 seconds: | 20260506 13:51:05.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:51:05.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:05.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:05.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:05.665 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 13:51:05.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:05.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:05.665 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260506 13:51:05.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:05.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:05.665 | INFO | Uploaded 36GiB to GPU in 2.04755s (17.582GiB/s) | 20260506 13:51:05.665 | INFO | Downloaded 34GiB to GPU in 1.91323s (17.771GiB/s) | 20260506 13:51:10.803 | INFO | Analyzing chunk frames = [1850368,1850880) | 20260506 13:51:10.803 | INFO | MemInfo: { | 20260506 13:51:10.803 | INFO | "gpu" : " 17 GB", | 20260506 13:51:10.803 | INFO | "huge" : "", | 20260506 13:51:10.803 | INFO | "huge_shared" : " 27 GB", | 20260506 13:51:10.803 | INFO | "malloc" : " 14 GB", | 20260506 13:51:10.803 | INFO | "pinned" : " 29 GB" | 20260506 13:51:10.803 | INFO | } | 20260506 13:51:10.803 | INFO | Perf report for the last 5.13833 seconds: | 20260506 13:51:10.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:51:10.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:10.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:10.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:10.803 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:51:10.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:10.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:10.803 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 13:51:10.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:10.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:10.803 | INFO | Uploaded 36GiB to GPU in 2.03206s (17.716GiB/s) | 20260506 13:51:10.803 | INFO | Downloaded 34GiB to GPU in 1.99497s (17.0428GiB/s) | 20260506 13:51:15.942 | INFO | Analyzing chunk frames = [1850880,1851392) | 20260506 13:51:15.942 | INFO | MemInfo: { | 20260506 13:51:15.942 | INFO | "gpu" : " 17 GB", | 20260506 13:51:15.942 | INFO | "huge" : "", | 20260506 13:51:15.942 | INFO | "huge_shared" : " 27 GB", | 20260506 13:51:15.942 | INFO | "malloc" : " 14 GB", | 20260506 13:51:15.942 | INFO | "pinned" : " 29 GB" | 20260506 13:51:15.942 | INFO | } | 20260506 13:51:15.942 | INFO | Perf report for the last 5.13936 seconds: | 20260506 13:51:15.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:51:15.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:15.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:15.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:15.942 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:51:15.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:15.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:15.942 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:51:15.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:15.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:15.943 | INFO | Uploaded 36GiB to GPU in 2.02715s (17.7589GiB/s) | 20260506 13:51:15.943 | INFO | Downloaded 34GiB to GPU in 1.98231s (17.1517GiB/s) | 20260506 13:51:15.943 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:51:15.943 | INFO | Total mem usage: 17.4 GiB | 20260506 13:51:15.943 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:51:15.943 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -20.8%) | 20260506 13:51:15.943 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:51:15.943 | INFO | High Water Breakdown: | 20260506 13:51:15.943 | INFO | 59.9% : 8.63 GiB ( 79% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:51:15.943 | INFO | 40% : 5.76 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:51:15.943 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:51:15.943 | INFO | ------------------------------------------ | 20260506 13:51:15.943 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:51:15.943 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:51:15.943 | INFO | Total mem usage: 26.5 GiB | 20260506 13:51:15.943 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:51:15.943 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:51:15.943 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:51:15.943 | INFO | High Water Breakdown: | 20260506 13:51:15.943 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:51:15.943 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:51:15.943 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:51:15.943 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:51:15.943 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:51:15.943 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:51:15.943 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:51:15.943 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:51:15.943 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:51:15.943 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:51:15.943 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:51:15.943 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:51:15.943 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:51:15.943 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:51:15.943 | INFO | ------------------------------------------ | 20260506 13:51:15.943 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:51:15.943 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:51:15.943 | INFO | Total mem usage: 25.6 GiB | 20260506 13:51:15.943 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:51:15.943 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:51:15.943 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:51:15.943 | INFO | High Water Breakdown: | 20260506 13:51:15.943 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:51:15.943 | INFO | ------------------------------------------ | 20260506 13:51:15.943 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:51:15.943 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:51:15.943 | INFO | Total mem usage: 15.7 GiB | 20260506 13:51:15.943 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:51:15.943 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:51:15.943 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 13:51:15.943 | INFO | High Water Breakdown: | 20260506 13:51:15.943 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:51:15.943 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:51:15.943 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:51:15.943 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:51:15.943 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:51:15.943 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:51:15.943 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:51:15.943 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:51:15.943 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:51:15.943 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:51:15.943 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:51:15.943 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:51:15.943 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:51:15.943 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:51:15.943 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:51:15.943 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:51:15.943 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:51:15.943 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:51:15.943 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:51:15.943 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:51:15.943 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:51:15.943 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:51:15.943 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:51:15.943 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:51:15.943 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:51:15.943 | INFO | ------------------------------------------ | 20260506 13:51:15.943 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:51:15.943 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:51:15.943 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:51:15.943 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:51:15.943 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.54e+01, 1.35e+01 / 2.23e+00 | 20260506 13:51:15.943 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.00e-01, 1.01e-02 / 6.40e-03 | 20260506 13:51:15.943 | INFO | PerfReport: 11.60%, Baseline -- 3.67e+00 / 4.43e+00, 5.81e-01 / 3.97e-01 | 20260506 13:51:15.943 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.46e+00, 1.90e-01 / 1.49e-01 | 20260506 13:51:15.943 | INFO | PerfReport: 15.80%, DME -- 4.00e+01 / 4.41e+01, 3.78e+00 / 2.40e+00 | 20260506 13:51:15.943 | INFO | PerfReport: 56.60%, FrameLabeling -- 1.79e+01 / 2.03e+01, 1.45e+00 / 1.84e+00 | 20260506 13:51:15.943 | INFO | PerfReport: 7.51%, PulseAccumulating -- 2.37e+00 / 2.99e+00, 4.36e-01 / 1.74e-01 | 20260506 13:51:15.943 | INFO | PerfReport: 3.66%, Metrics -- 1.16e+00 / 1.55e+00, 2.99e-01 / 1.75e-01 | 20260506 13:51:15.943 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.03e-01 / 4.55e-01, 6.26e-02 / 2.22e-02 | 20260506 13:51:15.944 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:51:15.944 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:51:15.944 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:51:15.944 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 13:51:15.944 | INFO | PerfReport: 21.44%, Upload -- 2.11e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260506 13:51:15.944 | INFO | PerfReport: 20.96%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 13:51:15.944 | INFO | PerfReport: 32.16%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 13:51:15.944 | INFO | PerfReport: 1.67%, LockUpload -- 1.64e+00 / 0.00e+00, 6.73e+00 / 0.00e+00 | 20260506 13:51:15.944 | INFO | PerfReport: 1.04%, LockDownload -- 1.02e+00 / 0.00e+00, 2.65e+00 / 0.00e+00 | 20260506 13:51:15.944 | INFO | PerfReport: 22.71%, LockCompute -- 2.23e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 13:51:15.944 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.12e-02 / 0.00e+00 | 20260506 13:51:16.060 | INFO | Starting a round of Baz IO | 20260506 13:51:19.212 | INFO | Populating RTMetrics report | 20260506 13:51:19.213 | INFO | RTMetrics report populated | 20260506 13:51:19.213 | INFO | Opening tmp RTMetrics json file | 20260506 13:51:19.214 | INFO | Serializing RTMetrics to json | 20260506 13:51:19.215 | INFO | Serialization done, swapping tmp file now... | 20260506 13:51:19.215 | INFO | RTMetrics done | 20260506 13:51:19.319 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.259 seconds | 20260506 13:51:19.319 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.09% / 0.02% / 60.89% | 20260506 13:51:19.319 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.89391% / 3.97364e-05% / 5.16574e-05% | 20260506 13:51:19.319 | INFO | Baz IO overall: | 20260506 13:51:19.319 | INFO | BytesWritten : 947047165030 | 20260506 13:51:19.319 | INFO | NumEvents : 458163465122 | 20260506 13:51:21.081 | INFO | Analyzing chunk frames = [1851392,1851904) | 20260506 13:51:21.081 | INFO | MemInfo: { | 20260506 13:51:21.081 | INFO | "gpu" : " 17 GB", | 20260506 13:51:21.082 | INFO | "huge" : "", | 20260506 13:51:21.082 | INFO | "huge_shared" : " 27 GB", | 20260506 13:51:21.082 | INFO | "malloc" : " 14 GB", | 20260506 13:51:21.082 | INFO | "pinned" : " 29 GB" | 20260506 13:51:21.082 | INFO | } | 20260506 13:51:21.082 | INFO | Perf report for the last 5.1392 seconds: | 20260506 13:51:21.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:51:21.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:21.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:21.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:21.082 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260506 13:51:21.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:21.082 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:51:21.082 | INFO | PRE_HQ : 61.3%, 1.9, 60.6ms, 0 | 20260506 13:51:21.082 | INFO | BAZWRITER : 62.7%, 3.7, 120.1ms, 0 | 20260506 13:51:21.082 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:51:21.082 | INFO | Uploaded 36GiB to GPU in 2.04487s (17.605GiB/s) | 20260506 13:51:21.082 | INFO | Downloaded 38GiB to GPU in 2.20185s (17.2582GiB/s) | 20260506 13:51:26.222 | INFO | Analyzing chunk frames = [1851904,1852416) | 20260506 13:51:26.222 | INFO | MemInfo: { | 20260506 13:51:26.222 | INFO | "gpu" : " 17 GB", | 20260506 13:51:26.222 | INFO | "huge" : "", | 20260506 13:51:26.222 | INFO | "huge_shared" : " 27 GB", | 20260506 13:51:26.222 | INFO | "malloc" : " 14 GB", | 20260506 13:51:26.222 | INFO | "pinned" : " 29 GB" | 20260506 13:51:26.222 | INFO | } | 20260506 13:51:26.222 | INFO | Perf report for the last 5.14024 seconds: | 20260506 13:51:26.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:51:26.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:26.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:26.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:26.222 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 13:51:26.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:26.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:26.222 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:51:26.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:26.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:26.222 | INFO | Uploaded 36GiB to GPU in 2.05029s (17.5585GiB/s) | 20260506 13:51:26.222 | INFO | Downloaded 34GiB to GPU in 1.94885s (17.4462GiB/s) | 20260506 13:51:31.360 | INFO | Analyzing chunk frames = [1852416,1852928) | 20260506 13:51:31.360 | INFO | MemInfo: { | 20260506 13:51:31.360 | INFO | "gpu" : " 17 GB", | 20260506 13:51:31.360 | INFO | "huge" : "", | 20260506 13:51:31.360 | INFO | "huge_shared" : " 27 GB", | 20260506 13:51:31.360 | INFO | "malloc" : " 14 GB", | 20260506 13:51:31.360 | INFO | "pinned" : " 29 GB" | 20260506 13:51:31.360 | INFO | } | 20260506 13:51:31.360 | INFO | Perf report for the last 5.1384 seconds: | 20260506 13:51:31.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:51:31.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:31.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:31.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:31.360 | INFO | ANALYSIS : 61.6%, 3.0, 95.3ms, 0 | 20260506 13:51:31.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:31.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:31.360 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:51:31.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:31.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:31.360 | INFO | Uploaded 36GiB to GPU in 2.02465s (17.7809GiB/s) | 20260506 13:51:31.361 | INFO | Downloaded 34GiB to GPU in 1.93992s (17.5265GiB/s) | 20260506 13:51:36.500 | INFO | Analyzing chunk frames = [1852928,1853440) | 20260506 13:51:36.500 | INFO | MemInfo: { | 20260506 13:51:36.500 | INFO | "gpu" : " 17 GB", | 20260506 13:51:36.500 | INFO | "huge" : "", | 20260506 13:51:36.500 | INFO | "huge_shared" : " 27 GB", | 20260506 13:51:36.500 | INFO | "malloc" : " 14 GB", | 20260506 13:51:36.500 | INFO | "pinned" : " 29 GB" | 20260506 13:51:36.500 | INFO | } | 20260506 13:51:36.500 | INFO | Perf report for the last 5.1398 seconds: | 20260506 13:51:36.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:51:36.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:36.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:36.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:36.500 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 13:51:36.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:36.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:36.500 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:51:36.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:36.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:36.500 | INFO | Uploaded 36GiB to GPU in 2.01434s (17.8719GiB/s) | 20260506 13:51:36.500 | INFO | Downloaded 34GiB to GPU in 1.9381s (17.543GiB/s) | 20260506 13:51:41.639 | INFO | Analyzing chunk frames = [1853440,1853952) | 20260506 13:51:41.639 | INFO | MemInfo: { | 20260506 13:51:41.639 | INFO | "gpu" : " 17 GB", | 20260506 13:51:41.639 | INFO | "huge" : "", | 20260506 13:51:41.639 | INFO | "huge_shared" : " 27 GB", | 20260506 13:51:41.639 | INFO | "malloc" : " 14 GB", | 20260506 13:51:41.639 | INFO | "pinned" : " 29 GB" | 20260506 13:51:41.639 | INFO | } | 20260506 13:51:41.639 | INFO | Perf report for the last 5.13937 seconds: | 20260506 13:51:41.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:51:41.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:41.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:41.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:41.639 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 13:51:41.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:41.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:41.640 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 13:51:41.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:41.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:41.640 | INFO | Uploaded 36GiB to GPU in 2.0281s (17.7506GiB/s) | 20260506 13:51:41.640 | INFO | Downloaded 34GiB to GPU in 1.92569s (17.656GiB/s) | 20260506 13:51:46.779 | INFO | Analyzing chunk frames = [1853952,1854464) | 20260506 13:51:46.779 | INFO | MemInfo: { | 20260506 13:51:46.779 | INFO | "gpu" : " 17 GB", | 20260506 13:51:46.779 | INFO | "huge" : "", | 20260506 13:51:46.779 | INFO | "huge_shared" : " 27 GB", | 20260506 13:51:46.779 | INFO | "malloc" : " 14 GB", | 20260506 13:51:46.779 | INFO | "pinned" : " 29 GB" | 20260506 13:51:46.779 | INFO | } | 20260506 13:51:46.779 | INFO | Perf report for the last 5.13972 seconds: | 20260506 13:51:46.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:51:46.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:46.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:46.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:46.779 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 13:51:46.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:46.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:46.779 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260506 13:51:46.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:46.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:46.779 | INFO | Uploaded 36GiB to GPU in 2.04102s (17.6382GiB/s) | 20260506 13:51:46.779 | INFO | Downloaded 34GiB to GPU in 1.91264s (17.7764GiB/s) | 20260506 13:51:51.918 | INFO | Analyzing chunk frames = [1854464,1854976) | 20260506 13:51:51.918 | INFO | MemInfo: { | 20260506 13:51:51.918 | INFO | "gpu" : " 17 GB", | 20260506 13:51:51.918 | INFO | "huge" : "", | 20260506 13:51:51.918 | INFO | "huge_shared" : " 27 GB", | 20260506 13:51:51.918 | INFO | "malloc" : " 14 GB", | 20260506 13:51:51.918 | INFO | "pinned" : " 29 GB" | 20260506 13:51:51.918 | INFO | } | 20260506 13:51:51.918 | INFO | Perf report for the last 5.13916 seconds: | 20260506 13:51:51.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:51:51.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:51.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:51.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:51.918 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 13:51:51.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:51.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:51.918 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:51:51.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:51.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:51.919 | INFO | Uploaded 36GiB to GPU in 2.02243s (17.8004GiB/s) | 20260506 13:51:51.919 | INFO | Downloaded 34GiB to GPU in 2.00052s (16.9955GiB/s) | 20260506 13:51:57.057 | INFO | Analyzing chunk frames = [1854976,1855488) | 20260506 13:51:57.057 | INFO | MemInfo: { | 20260506 13:51:57.057 | INFO | "gpu" : " 17 GB", | 20260506 13:51:57.057 | INFO | "huge" : "", | 20260506 13:51:57.057 | INFO | "huge_shared" : " 27 GB", | 20260506 13:51:57.057 | INFO | "malloc" : " 14 GB", | 20260506 13:51:57.057 | INFO | "pinned" : " 29 GB" | 20260506 13:51:57.057 | INFO | } | 20260506 13:51:57.057 | INFO | Perf report for the last 5.13828 seconds: | 20260506 13:51:57.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:51:57.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:57.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:57.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:57.057 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 13:51:57.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:57.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:57.057 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:51:57.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:57.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:51:57.057 | INFO | Uploaded 36GiB to GPU in 2.04735s (17.5837GiB/s) | 20260506 13:51:57.057 | INFO | Downloaded 34GiB to GPU in 1.9698s (17.2606GiB/s) | 20260506 13:51:57.183 | INFO | Starting a round of Baz IO | 20260506 13:52:00.334 | INFO | Populating RTMetrics report | 20260506 13:52:00.334 | INFO | RTMetrics report populated | 20260506 13:52:00.334 | INFO | Opening tmp RTMetrics json file | 20260506 13:52:00.335 | INFO | Serializing RTMetrics to json | 20260506 13:52:00.336 | INFO | Serialization done, swapping tmp file now... | 20260506 13:52:00.337 | INFO | RTMetrics done | 20260506 13:52:00.436 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.253 seconds | 20260506 13:52:00.436 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.08% / 0.02% / 60.91% | 20260506 13:52:00.436 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.88506% / 2.78155e-05% / 5.16574e-05% | 20260506 13:52:00.436 | INFO | Baz IO overall: | 20260506 13:52:00.436 | INFO | BytesWritten : 949311774910 | 20260506 13:52:00.436 | INFO | NumEvents : 459261354210 | 20260506 13:52:02.196 | INFO | Analyzing chunk frames = [1855488,1856000) | 20260506 13:52:02.196 | INFO | MemInfo: { | 20260506 13:52:02.196 | INFO | "gpu" : " 17 GB", | 20260506 13:52:02.196 | INFO | "huge" : "", | 20260506 13:52:02.197 | INFO | "huge_shared" : " 27 GB", | 20260506 13:52:02.197 | INFO | "malloc" : " 14 GB", | 20260506 13:52:02.197 | INFO | "pinned" : " 29 GB" | 20260506 13:52:02.197 | INFO | } | 20260506 13:52:02.197 | INFO | Perf report for the last 5.13992 seconds: | 20260506 13:52:02.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:52:02.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:02.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:02.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:02.197 | INFO | ANALYSIS : 63.2%, 3.0, 98.1ms, 0 | 20260506 13:52:02.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:02.197 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 13:52:02.197 | INFO | PRE_HQ : 61.1%, 1.9, 60.7ms, 0 | 20260506 13:52:02.197 | INFO | BAZWRITER : 62.6%, 3.7, 120.2ms, 0 | 20260506 13:52:02.197 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 13:52:02.197 | INFO | Uploaded 36GiB to GPU in 2.07668s (17.3354GiB/s) | 20260506 13:52:02.197 | INFO | Downloaded 38GiB to GPU in 2.21195s (17.1794GiB/s) | 20260506 13:52:07.335 | INFO | Analyzing chunk frames = [1856000,1856512) | 20260506 13:52:07.335 | INFO | MemInfo: { | 20260506 13:52:07.335 | INFO | "gpu" : " 17 GB", | 20260506 13:52:07.335 | INFO | "huge" : "", | 20260506 13:52:07.335 | INFO | "huge_shared" : " 27 GB", | 20260506 13:52:07.335 | INFO | "malloc" : " 14 GB", | 20260506 13:52:07.335 | INFO | "pinned" : " 29 GB" | 20260506 13:52:07.335 | INFO | } | 20260506 13:52:07.335 | INFO | Perf report for the last 5.13881 seconds: | 20260506 13:52:07.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:52:07.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:07.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:07.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:07.335 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 13:52:07.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:07.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:07.335 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 13:52:07.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:07.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:07.336 | INFO | Uploaded 36GiB to GPU in 2.02157s (17.808GiB/s) | 20260506 13:52:07.336 | INFO | Downloaded 34GiB to GPU in 1.92495s (17.6628GiB/s) | 20260506 13:52:12.475 | INFO | Analyzing chunk frames = [1856512,1857024) | 20260506 13:52:12.475 | INFO | MemInfo: { | 20260506 13:52:12.475 | INFO | "gpu" : " 17 GB", | 20260506 13:52:12.475 | INFO | "huge" : "", | 20260506 13:52:12.475 | INFO | "huge_shared" : " 27 GB", | 20260506 13:52:12.475 | INFO | "malloc" : " 14 GB", | 20260506 13:52:12.475 | INFO | "pinned" : " 29 GB" | 20260506 13:52:12.475 | INFO | } | 20260506 13:52:12.475 | INFO | Perf report for the last 5.13975 seconds: | 20260506 13:52:12.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:52:12.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:12.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:12.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:12.475 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 13:52:12.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:12.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:12.475 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:52:12.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:12.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:12.475 | INFO | Uploaded 36GiB to GPU in 1.99511s (18.0441GiB/s) | 20260506 13:52:12.475 | INFO | Downloaded 34GiB to GPU in 1.93221s (17.5964GiB/s) | 20260506 13:52:17.615 | INFO | Analyzing chunk frames = [1857024,1857536) | 20260506 13:52:17.615 | INFO | MemInfo: { | 20260506 13:52:17.615 | INFO | "gpu" : " 17 GB", | 20260506 13:52:17.615 | INFO | "huge" : "", | 20260506 13:52:17.615 | INFO | "huge_shared" : " 27 GB", | 20260506 13:52:17.615 | INFO | "malloc" : " 14 GB", | 20260506 13:52:17.615 | INFO | "pinned" : " 29 GB" | 20260506 13:52:17.615 | INFO | } | 20260506 13:52:17.615 | INFO | Perf report for the last 5.13951 seconds: | 20260506 13:52:17.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:52:17.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:17.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:17.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:17.615 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 13:52:17.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:17.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:17.615 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:52:17.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:17.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:17.615 | INFO | Uploaded 36GiB to GPU in 1.99993s (18.0007GiB/s) | 20260506 13:52:17.615 | INFO | Downloaded 34GiB to GPU in 1.92481s (17.6641GiB/s) | 20260506 13:52:22.753 | INFO | Analyzing chunk frames = [1857536,1858048) | 20260506 13:52:22.753 | INFO | MemInfo: { | 20260506 13:52:22.753 | INFO | "gpu" : " 17 GB", | 20260506 13:52:22.753 | INFO | "huge" : "", | 20260506 13:52:22.753 | INFO | "huge_shared" : " 27 GB", | 20260506 13:52:22.753 | INFO | "malloc" : " 14 GB", | 20260506 13:52:22.753 | INFO | "pinned" : " 29 GB" | 20260506 13:52:22.753 | INFO | } | 20260506 13:52:22.753 | INFO | Perf report for the last 5.13866 seconds: | 20260506 13:52:22.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:52:22.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:22.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:22.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:22.753 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 13:52:22.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:22.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:22.753 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:52:22.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:22.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:22.754 | INFO | Uploaded 36GiB to GPU in 2.01633s (17.8542GiB/s) | 20260506 13:52:22.754 | INFO | Downloaded 34GiB to GPU in 1.92941s (17.6219GiB/s) | 20260506 13:52:27.893 | INFO | Analyzing chunk frames = [1858048,1858560) | 20260506 13:52:27.893 | INFO | MemInfo: { | 20260506 13:52:27.893 | INFO | "gpu" : " 17 GB", | 20260506 13:52:27.893 | INFO | "huge" : "", | 20260506 13:52:27.893 | INFO | "huge_shared" : " 27 GB", | 20260506 13:52:27.893 | INFO | "malloc" : " 14 GB", | 20260506 13:52:27.893 | INFO | "pinned" : " 29 GB" | 20260506 13:52:27.893 | INFO | } | 20260506 13:52:27.893 | INFO | Perf report for the last 5.13939 seconds: | 20260506 13:52:27.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:52:27.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:27.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:27.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:27.893 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 13:52:27.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:27.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:27.893 | INFO | PRE_HQ : 33.1%, 1.0, 17.2ms, 0 | 20260506 13:52:27.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:27.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:27.893 | INFO | Uploaded 36GiB to GPU in 2.05389s (17.5277GiB/s) | 20260506 13:52:27.893 | INFO | Downloaded 34GiB to GPU in 1.91895s (17.718GiB/s) | 20260506 13:52:33.032 | INFO | Analyzing chunk frames = [1858560,1859072) | 20260506 13:52:33.032 | INFO | MemInfo: { | 20260506 13:52:33.032 | INFO | "gpu" : " 17 GB", | 20260506 13:52:33.032 | INFO | "huge" : "", | 20260506 13:52:33.032 | INFO | "huge_shared" : " 27 GB", | 20260506 13:52:33.032 | INFO | "malloc" : " 14 GB", | 20260506 13:52:33.032 | INFO | "pinned" : " 29 GB" | 20260506 13:52:33.032 | INFO | } | 20260506 13:52:33.032 | INFO | Perf report for the last 5.1397 seconds: | 20260506 13:52:33.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:52:33.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:33.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:33.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:33.033 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 13:52:33.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:33.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:33.033 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 13:52:33.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:33.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:33.033 | INFO | Uploaded 36GiB to GPU in 1.98333s (18.1513GiB/s) | 20260506 13:52:33.033 | INFO | Downloaded 34GiB to GPU in 2.00302s (16.9744GiB/s) | 20260506 13:52:38.171 | INFO | Analyzing chunk frames = [1859072,1859584) | 20260506 13:52:38.171 | INFO | MemInfo: { | 20260506 13:52:38.171 | INFO | "gpu" : " 17 GB", | 20260506 13:52:38.171 | INFO | "huge" : "", | 20260506 13:52:38.171 | INFO | "huge_shared" : " 27 GB", | 20260506 13:52:38.171 | INFO | "malloc" : " 14 GB", | 20260506 13:52:38.171 | INFO | "pinned" : " 29 GB" | 20260506 13:52:38.172 | INFO | } | 20260506 13:52:38.172 | INFO | Perf report for the last 5.13903 seconds: | 20260506 13:52:38.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:52:38.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:38.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:38.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:38.172 | INFO | ANALYSIS : 62.4%, 3.0, 97.4ms, 0 | 20260506 13:52:38.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:38.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:38.172 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:52:38.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:38.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:38.172 | INFO | Uploaded 36GiB to GPU in 2.04728s (17.5843GiB/s) | 20260506 13:52:38.172 | INFO | Downloaded 34GiB to GPU in 1.9656s (17.2975GiB/s) | 20260506 13:52:38.172 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:52:38.172 | INFO | Total mem usage: 17.4 GiB | 20260506 13:52:38.172 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:52:38.172 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -20.8%) | 20260506 13:52:38.172 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:52:38.172 | INFO | High Water Breakdown: | 20260506 13:52:38.172 | INFO | 59.9% : 8.63 GiB ( 79% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:52:38.172 | INFO | 40% : 5.76 GiB ( 86.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:52:38.172 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:52:38.172 | INFO | ------------------------------------------ | 20260506 13:52:38.172 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:52:38.172 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:52:38.172 | INFO | Total mem usage: 26.5 GiB | 20260506 13:52:38.172 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:52:38.172 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:52:38.172 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:52:38.172 | INFO | High Water Breakdown: | 20260506 13:52:38.172 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:52:38.172 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:52:38.172 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:52:38.172 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:52:38.172 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:52:38.172 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:52:38.172 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:52:38.172 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:52:38.172 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:52:38.172 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:52:38.172 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:52:38.172 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:52:38.172 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:52:38.172 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:52:38.172 | INFO | ------------------------------------------ | 20260506 13:52:38.172 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:52:38.172 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:52:38.172 | INFO | Total mem usage: 25.6 GiB | 20260506 13:52:38.172 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:52:38.172 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:52:38.172 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:52:38.172 | INFO | High Water Breakdown: | 20260506 13:52:38.172 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:52:38.172 | INFO | ------------------------------------------ | 20260506 13:52:38.172 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:52:38.172 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:52:38.172 | INFO | Total mem usage: 15.7 GiB | 20260506 13:52:38.172 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:52:38.172 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:52:38.172 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 13:52:38.172 | INFO | High Water Breakdown: | 20260506 13:52:38.172 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:52:38.172 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:52:38.172 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:52:38.172 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:52:38.172 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:52:38.172 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:52:38.172 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:52:38.172 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:52:38.172 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:52:38.172 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:52:38.172 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:52:38.172 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:52:38.172 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:52:38.172 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:52:38.172 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:52:38.172 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:52:38.172 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:52:38.172 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:52:38.172 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:52:38.172 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:52:38.172 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:52:38.172 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:52:38.172 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:52:38.172 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:52:38.173 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:52:38.173 | INFO | ------------------------------------------ | 20260506 13:52:38.173 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:52:38.173 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:52:38.173 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:52:38.173 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:52:38.173 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.56e+01, 1.34e+01 / 1.84e+00 | 20260506 13:52:38.173 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.01e-01, 9.84e-03 / 6.17e-03 | 20260506 13:52:38.173 | INFO | PerfReport: 11.57%, Baseline -- 3.66e+00 / 4.46e+00, 5.72e-01 / 3.38e-01 | 20260506 13:52:38.173 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.47e+00, 1.90e-01 / 1.46e-01 | 20260506 13:52:38.173 | INFO | PerfReport: 15.80%, DME -- 3.99e+01 / 4.43e+01, 3.93e+00 / 2.10e+00 | 20260506 13:52:38.173 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.45e+00 / 1.57e+00 | 20260506 13:52:38.173 | INFO | PerfReport: 7.49%, PulseAccumulating -- 2.36e+00 / 3.00e+00, 4.15e-01 / 2.44e-01 | 20260506 13:52:38.173 | INFO | PerfReport: 3.65%, Metrics -- 1.15e+00 / 1.52e+00, 2.96e-01 / 1.80e-01 | 20260506 13:52:38.173 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.03e-01 / 4.60e-01, 6.20e-02 / 2.87e-02 | 20260506 13:52:38.173 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:52:38.173 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:52:38.173 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:52:38.173 | INFO | PerfReport: Full Execution -- 9.84e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 13:52:38.173 | INFO | PerfReport: 21.49%, Upload -- 2.11e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260506 13:52:38.173 | INFO | PerfReport: 20.95%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 13:52:38.173 | INFO | PerfReport: 32.11%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 13:52:38.173 | INFO | PerfReport: 1.76%, LockUpload -- 1.73e+00 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 13:52:38.173 | INFO | PerfReport: 1.06%, LockDownload -- 1.04e+00 / 0.00e+00, 2.69e+00 / 0.00e+00 | 20260506 13:52:38.173 | INFO | PerfReport: 22.61%, LockCompute -- 2.22e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 13:52:38.173 | INFO | PerfReport: 0.02%, Misc -- 1.56e-02 / 0.00e+00, 3.53e-02 / 0.00e+00 | 20260506 13:52:38.287 | INFO | Starting a round of Baz IO | 20260506 13:52:41.451 | INFO | Populating RTMetrics report | 20260506 13:52:41.451 | INFO | RTMetrics report populated | 20260506 13:52:41.451 | INFO | Opening tmp RTMetrics json file | 20260506 13:52:41.452 | INFO | Serializing RTMetrics to json | 20260506 13:52:41.453 | INFO | Serialization done, swapping tmp file now... | 20260506 13:52:41.453 | INFO | RTMetrics done | 20260506 13:52:41.555 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.268 seconds | 20260506 13:52:41.555 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.06% / 0.02% / 60.92% | 20260506 13:52:41.555 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.88683% / 1.98682e-05% / 3.17891e-05% | 20260506 13:52:41.555 | INFO | Baz IO overall: | 20260506 13:52:41.555 | INFO | BytesWritten : 951579821865 | 20260506 13:52:41.555 | INFO | NumEvents : 460361885281 | 20260506 13:52:43.311 | INFO | Analyzing chunk frames = [1859584,1860096) | 20260506 13:52:43.311 | INFO | MemInfo: { | 20260506 13:52:43.311 | INFO | "gpu" : " 17 GB", | 20260506 13:52:43.311 | INFO | "huge" : "", | 20260506 13:52:43.311 | INFO | "huge_shared" : " 27 GB", | 20260506 13:52:43.311 | INFO | "malloc" : " 14 GB", | 20260506 13:52:43.311 | INFO | "pinned" : " 29 GB" | 20260506 13:52:43.311 | INFO | } | 20260506 13:52:43.311 | INFO | Perf report for the last 5.13949 seconds: | 20260506 13:52:43.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:52:43.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:43.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:43.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:43.311 | INFO | ANALYSIS : 63.2%, 3.0, 98.2ms, 0 | 20260506 13:52:43.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:43.311 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 13:52:43.311 | INFO | PRE_HQ : 61.3%, 1.9, 60.7ms, 0 | 20260506 13:52:43.311 | INFO | BAZWRITER : 62.9%, 3.7, 120.6ms, 0 | 20260506 13:52:43.311 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 13:52:43.311 | INFO | Uploaded 36GiB to GPU in 2.028s (17.7514GiB/s) | 20260506 13:52:43.312 | INFO | Downloaded 38GiB to GPU in 2.19576s (17.3061GiB/s) | 20260506 13:52:48.451 | INFO | Analyzing chunk frames = [1860096,1860608) | 20260506 13:52:48.451 | INFO | MemInfo: { | 20260506 13:52:48.451 | INFO | "gpu" : " 17 GB", | 20260506 13:52:48.451 | INFO | "huge" : "", | 20260506 13:52:48.451 | INFO | "huge_shared" : " 27 GB", | 20260506 13:52:48.451 | INFO | "malloc" : " 14 GB", | 20260506 13:52:48.451 | INFO | "pinned" : " 29 GB" | 20260506 13:52:48.451 | INFO | } | 20260506 13:52:48.451 | INFO | Perf report for the last 5.13976 seconds: | 20260506 13:52:48.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:52:48.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:48.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:48.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:48.451 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 13:52:48.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:48.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:48.451 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:52:48.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:48.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:48.451 | INFO | Uploaded 36GiB to GPU in 1.99493s (18.0457GiB/s) | 20260506 13:52:48.451 | INFO | Downloaded 34GiB to GPU in 1.93331s (17.5864GiB/s) | 20260506 13:52:53.590 | INFO | Analyzing chunk frames = [1860608,1861120) | 20260506 13:52:53.590 | INFO | MemInfo: { | 20260506 13:52:53.590 | INFO | "gpu" : " 17 GB", | 20260506 13:52:53.590 | INFO | "huge" : "", | 20260506 13:52:53.590 | INFO | "huge_shared" : " 27 GB", | 20260506 13:52:53.590 | INFO | "malloc" : " 14 GB", | 20260506 13:52:53.590 | INFO | "pinned" : " 29 GB" | 20260506 13:52:53.591 | INFO | } | 20260506 13:52:53.591 | INFO | Perf report for the last 5.13972 seconds: | 20260506 13:52:53.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:52:53.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:53.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:53.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:53.591 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 13:52:53.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:53.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:53.591 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:52:53.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:53.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:53.591 | INFO | Uploaded 36GiB to GPU in 1.99519s (18.0434GiB/s) | 20260506 13:52:53.591 | INFO | Downloaded 34GiB to GPU in 1.92941s (17.622GiB/s) | 20260506 13:52:58.730 | INFO | Analyzing chunk frames = [1861120,1861632) | 20260506 13:52:58.730 | INFO | MemInfo: { | 20260506 13:52:58.730 | INFO | "gpu" : " 17 GB", | 20260506 13:52:58.730 | INFO | "huge" : "", | 20260506 13:52:58.730 | INFO | "huge_shared" : " 27 GB", | 20260506 13:52:58.730 | INFO | "malloc" : " 14 GB", | 20260506 13:52:58.730 | INFO | "pinned" : " 29 GB" | 20260506 13:52:58.730 | INFO | } | 20260506 13:52:58.730 | INFO | Perf report for the last 5.13933 seconds: | 20260506 13:52:58.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:52:58.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:58.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:58.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:58.730 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 13:52:58.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:58.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:58.730 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 13:52:58.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:58.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:52:58.730 | INFO | Uploaded 36GiB to GPU in 1.97847s (18.1958GiB/s) | 20260506 13:52:58.730 | INFO | Downloaded 34GiB to GPU in 1.91523s (17.7524GiB/s) | 20260506 13:53:03.868 | INFO | Analyzing chunk frames = [1861632,1862144) | 20260506 13:53:03.868 | INFO | MemInfo: { | 20260506 13:53:03.868 | INFO | "gpu" : " 17 GB", | 20260506 13:53:03.868 | INFO | "huge" : "", | 20260506 13:53:03.868 | INFO | "huge_shared" : " 27 GB", | 20260506 13:53:03.868 | INFO | "malloc" : " 14 GB", | 20260506 13:53:03.868 | INFO | "pinned" : " 29 GB" | 20260506 13:53:03.868 | INFO | } | 20260506 13:53:03.868 | INFO | Perf report for the last 5.13858 seconds: | 20260506 13:53:03.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:53:03.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:03.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:03.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:03.868 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 13:53:03.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:03.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:03.869 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:53:03.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:03.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:03.869 | INFO | Uploaded 36GiB to GPU in 2.04288s (17.6221GiB/s) | 20260506 13:53:03.869 | INFO | Downloaded 34GiB to GPU in 1.92913s (17.6245GiB/s) | 20260506 13:53:09.008 | INFO | Analyzing chunk frames = [1862144,1862656) | 20260506 13:53:09.008 | INFO | MemInfo: { | 20260506 13:53:09.008 | INFO | "gpu" : " 17 GB", | 20260506 13:53:09.008 | INFO | "huge" : "", | 20260506 13:53:09.008 | INFO | "huge_shared" : " 27 GB", | 20260506 13:53:09.008 | INFO | "malloc" : " 14 GB", | 20260506 13:53:09.008 | INFO | "pinned" : " 29 GB" | 20260506 13:53:09.008 | INFO | } | 20260506 13:53:09.008 | INFO | Perf report for the last 5.1397 seconds: | 20260506 13:53:09.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:53:09.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:09.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:09.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:09.008 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 13:53:09.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:09.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:09.008 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 13:53:09.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:09.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:09.009 | INFO | Uploaded 36GiB to GPU in 2.07466s (17.3523GiB/s) | 20260506 13:53:09.009 | INFO | Downloaded 34GiB to GPU in 1.91324s (17.7709GiB/s) | 20260506 13:53:14.148 | INFO | Analyzing chunk frames = [1862656,1863168) | 20260506 13:53:14.148 | INFO | MemInfo: { | 20260506 13:53:14.148 | INFO | "gpu" : " 17 GB", | 20260506 13:53:14.148 | INFO | "huge" : "", | 20260506 13:53:14.148 | INFO | "huge_shared" : " 27 GB", | 20260506 13:53:14.148 | INFO | "malloc" : " 14 GB", | 20260506 13:53:14.148 | INFO | "pinned" : " 29 GB" | 20260506 13:53:14.148 | INFO | } | 20260506 13:53:14.148 | INFO | Perf report for the last 5.14006 seconds: | 20260506 13:53:14.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:53:14.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:14.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:14.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:14.148 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:53:14.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:14.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:14.148 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 13:53:14.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:14.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:14.149 | INFO | Uploaded 36GiB to GPU in 2.03706s (17.6726GiB/s) | 20260506 13:53:14.149 | INFO | Downloaded 34GiB to GPU in 2.00661s (16.944GiB/s) | 20260506 13:53:19.287 | INFO | Analyzing chunk frames = [1863168,1863680) | 20260506 13:53:19.287 | INFO | MemInfo: { | 20260506 13:53:19.287 | INFO | "gpu" : " 17 GB", | 20260506 13:53:19.287 | INFO | "huge" : "", | 20260506 13:53:19.287 | INFO | "huge_shared" : " 27 GB", | 20260506 13:53:19.287 | INFO | "malloc" : " 14 GB", | 20260506 13:53:19.287 | INFO | "pinned" : " 29 GB" | 20260506 13:53:19.287 | INFO | } | 20260506 13:53:19.287 | INFO | Perf report for the last 5.13905 seconds: | 20260506 13:53:19.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:53:19.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:19.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:19.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:19.287 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 13:53:19.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:19.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:19.287 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 13:53:19.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:19.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:19.288 | INFO | Uploaded 36GiB to GPU in 2.05052s (17.5565GiB/s) | 20260506 13:53:19.288 | INFO | Downloaded 34GiB to GPU in 1.99449s (17.047GiB/s) | 20260506 13:53:19.410 | INFO | Starting a round of Baz IO | 20260506 13:53:22.562 | INFO | Populating RTMetrics report | 20260506 13:53:22.563 | INFO | RTMetrics report populated | 20260506 13:53:22.563 | INFO | Opening tmp RTMetrics json file | 20260506 13:53:22.563 | INFO | Serializing RTMetrics to json | 20260506 13:53:22.565 | INFO | Serialization done, swapping tmp file now... | 20260506 13:53:22.565 | INFO | RTMetrics done | 20260506 13:53:22.660 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.25 seconds | 20260506 13:53:22.660 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.05% / 0.02% / 60.94% | 20260506 13:53:22.660 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.88355% / 2.38419e-05% / 4.37101e-05% | 20260506 13:53:22.660 | INFO | Baz IO overall: | 20260506 13:53:22.661 | INFO | BytesWritten : 953849679678 | 20260506 13:53:22.661 | INFO | NumEvents : 461463885590 | 20260506 13:53:24.427 | INFO | Analyzing chunk frames = [1863680,1864192) | 20260506 13:53:24.427 | INFO | MemInfo: { | 20260506 13:53:24.427 | INFO | "gpu" : " 17 GB", | 20260506 13:53:24.427 | INFO | "huge" : "", | 20260506 13:53:24.427 | INFO | "huge_shared" : " 27 GB", | 20260506 13:53:24.427 | INFO | "malloc" : " 14 GB", | 20260506 13:53:24.427 | INFO | "pinned" : " 29 GB" | 20260506 13:53:24.427 | INFO | } | 20260506 13:53:24.427 | INFO | Perf report for the last 5.13949 seconds: | 20260506 13:53:24.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:53:24.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:24.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:24.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:24.427 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260506 13:53:24.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:24.427 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 13:53:24.427 | INFO | PRE_HQ : 61.0%, 1.9, 60.4ms, 0 | 20260506 13:53:24.427 | INFO | BAZWRITER : 62.6%, 3.7, 119.2ms, 0 | 20260506 13:53:24.427 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:53:24.427 | INFO | Uploaded 36GiB to GPU in 2.02643s (17.7652GiB/s) | 20260506 13:53:24.427 | INFO | Downloaded 38GiB to GPU in 2.20737s (17.2151GiB/s) | 20260506 13:53:29.565 | INFO | Analyzing chunk frames = [1864192,1864704) | 20260506 13:53:29.565 | INFO | MemInfo: { | 20260506 13:53:29.565 | INFO | "gpu" : " 17 GB", | 20260506 13:53:29.565 | INFO | "huge" : "", | 20260506 13:53:29.565 | INFO | "huge_shared" : " 27 GB", | 20260506 13:53:29.565 | INFO | "malloc" : " 14 GB", | 20260506 13:53:29.565 | INFO | "pinned" : " 29 GB" | 20260506 13:53:29.565 | INFO | } | 20260506 13:53:29.565 | INFO | Perf report for the last 5.13814 seconds: | 20260506 13:53:29.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:53:29.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:29.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:29.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:29.565 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 13:53:29.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:29.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:29.565 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:53:29.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:29.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:29.565 | INFO | Uploaded 36GiB to GPU in 1.97659s (18.2132GiB/s) | 20260506 13:53:29.565 | INFO | Downloaded 34GiB to GPU in 1.94422s (17.4877GiB/s) | 20260506 13:53:34.704 | INFO | Analyzing chunk frames = [1864704,1865216) | 20260506 13:53:34.704 | INFO | MemInfo: { | 20260506 13:53:34.704 | INFO | "gpu" : " 17 GB", | 20260506 13:53:34.704 | INFO | "huge" : "", | 20260506 13:53:34.704 | INFO | "huge_shared" : " 27 GB", | 20260506 13:53:34.704 | INFO | "malloc" : " 14 GB", | 20260506 13:53:34.704 | INFO | "pinned" : " 29 GB" | 20260506 13:53:34.704 | INFO | } | 20260506 13:53:34.704 | INFO | Perf report for the last 5.13921 seconds: | 20260506 13:53:34.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:53:34.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:34.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:34.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:34.704 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 13:53:34.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:34.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:34.704 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:53:34.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:34.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:34.704 | INFO | Uploaded 36GiB to GPU in 1.97987s (18.1831GiB/s) | 20260506 13:53:34.705 | INFO | Downloaded 34GiB to GPU in 1.93209s (17.5975GiB/s) | 20260506 13:53:39.844 | INFO | Analyzing chunk frames = [1865216,1865728) | 20260506 13:53:39.844 | INFO | MemInfo: { | 20260506 13:53:39.844 | INFO | "gpu" : " 17 GB", | 20260506 13:53:39.844 | INFO | "huge" : "", | 20260506 13:53:39.844 | INFO | "huge_shared" : " 27 GB", | 20260506 13:53:39.844 | INFO | "malloc" : " 14 GB", | 20260506 13:53:39.844 | INFO | "pinned" : " 29 GB" | 20260506 13:53:39.844 | INFO | } | 20260506 13:53:39.844 | INFO | Perf report for the last 5.13944 seconds: | 20260506 13:53:39.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:53:39.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:39.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:39.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:39.844 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 13:53:39.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:39.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:39.844 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:53:39.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:39.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:39.844 | INFO | Uploaded 36GiB to GPU in 1.98594s (18.1274GiB/s) | 20260506 13:53:39.844 | INFO | Downloaded 34GiB to GPU in 1.91711s (17.735GiB/s) | 20260506 13:53:44.984 | INFO | Analyzing chunk frames = [1865728,1866240) | 20260506 13:53:44.984 | INFO | MemInfo: { | 20260506 13:53:44.984 | INFO | "gpu" : " 17 GB", | 20260506 13:53:44.984 | INFO | "huge" : "", | 20260506 13:53:44.984 | INFO | "huge_shared" : " 27 GB", | 20260506 13:53:44.984 | INFO | "malloc" : " 14 GB", | 20260506 13:53:44.984 | INFO | "pinned" : " 29 GB" | 20260506 13:53:44.984 | INFO | } | 20260506 13:53:44.984 | INFO | Perf report for the last 5.14021 seconds: | 20260506 13:53:44.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:53:44.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:44.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:44.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:44.984 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 13:53:44.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:44.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:44.984 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:53:44.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:44.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:44.984 | INFO | Uploaded 36GiB to GPU in 2.00153s (17.9862GiB/s) | 20260506 13:53:44.984 | INFO | Downloaded 34GiB to GPU in 1.94409s (17.4889GiB/s) | 20260506 13:53:50.122 | INFO | Analyzing chunk frames = [1866240,1866752) | 20260506 13:53:50.122 | INFO | MemInfo: { | 20260506 13:53:50.122 | INFO | "gpu" : " 17 GB", | 20260506 13:53:50.122 | INFO | "huge" : "", | 20260506 13:53:50.122 | INFO | "huge_shared" : " 27 GB", | 20260506 13:53:50.122 | INFO | "malloc" : " 14 GB", | 20260506 13:53:50.122 | INFO | "pinned" : " 29 GB" | 20260506 13:53:50.122 | INFO | } | 20260506 13:53:50.123 | INFO | Perf report for the last 5.13862 seconds: | 20260506 13:53:50.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:53:50.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:50.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:50.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:50.123 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 13:53:50.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:50.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:50.123 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 13:53:50.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:50.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:50.123 | INFO | Uploaded 36GiB to GPU in 2.05813s (17.4916GiB/s) | 20260506 13:53:50.123 | INFO | Downloaded 34GiB to GPU in 1.92439s (17.668GiB/s) | 20260506 13:53:55.262 | INFO | Analyzing chunk frames = [1866752,1867264) | 20260506 13:53:55.262 | INFO | MemInfo: { | 20260506 13:53:55.262 | INFO | "gpu" : " 17 GB", | 20260506 13:53:55.262 | INFO | "huge" : "", | 20260506 13:53:55.262 | INFO | "huge_shared" : " 27 GB", | 20260506 13:53:55.262 | INFO | "malloc" : " 14 GB", | 20260506 13:53:55.262 | INFO | "pinned" : " 29 GB" | 20260506 13:53:55.262 | INFO | } | 20260506 13:53:55.262 | INFO | Perf report for the last 5.13991 seconds: | 20260506 13:53:55.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:53:55.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:55.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:55.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:55.262 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:53:55.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:55.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:55.262 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 13:53:55.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:55.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:53:55.263 | INFO | Uploaded 36GiB to GPU in 2.01348s (17.8795GiB/s) | 20260506 13:53:55.263 | INFO | Downloaded 34GiB to GPU in 2.00116s (16.9902GiB/s) | 20260506 13:54:00.401 | INFO | Analyzing chunk frames = [1867264,1867776) | 20260506 13:54:00.401 | INFO | MemInfo: { | 20260506 13:54:00.401 | INFO | "gpu" : " 17 GB", | 20260506 13:54:00.401 | INFO | "huge" : "", | 20260506 13:54:00.401 | INFO | "huge_shared" : " 27 GB", | 20260506 13:54:00.401 | INFO | "malloc" : " 14 GB", | 20260506 13:54:00.401 | INFO | "pinned" : " 29 GB" | 20260506 13:54:00.401 | INFO | } | 20260506 13:54:00.401 | INFO | Perf report for the last 5.13904 seconds: | 20260506 13:54:00.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:54:00.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:00.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:00.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:00.402 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 13:54:00.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:00.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:00.402 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260506 13:54:00.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:00.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:00.402 | INFO | Uploaded 36GiB to GPU in 2.0185s (17.8351GiB/s) | 20260506 13:54:00.402 | INFO | Downloaded 34GiB to GPU in 2.01582s (16.8666GiB/s) | 20260506 13:54:00.402 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:54:00.402 | INFO | Total mem usage: 17.4 GiB | 20260506 13:54:00.402 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:54:00.402 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -20.6%) | 20260506 13:54:00.402 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:54:00.402 | INFO | High Water Breakdown: | 20260506 13:54:00.402 | INFO | 59.8% : 8.63 GiB ( 79.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:54:00.402 | INFO | 40.1% : 5.78 GiB ( 86.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:54:00.402 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:54:00.402 | INFO | ------------------------------------------ | 20260506 13:54:00.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:54:00.402 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:54:00.402 | INFO | Total mem usage: 26.5 GiB | 20260506 13:54:00.402 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:54:00.402 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:54:00.402 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:54:00.402 | INFO | High Water Breakdown: | 20260506 13:54:00.402 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:54:00.402 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:54:00.402 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:54:00.402 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:54:00.402 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:54:00.402 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:54:00.402 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:54:00.402 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:54:00.402 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:54:00.402 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:54:00.402 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:54:00.402 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:54:00.402 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:54:00.402 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:54:00.402 | INFO | ------------------------------------------ | 20260506 13:54:00.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:54:00.402 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:54:00.402 | INFO | Total mem usage: 25.6 GiB | 20260506 13:54:00.402 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:54:00.402 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:54:00.402 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:54:00.402 | INFO | High Water Breakdown: | 20260506 13:54:00.402 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:54:00.402 | INFO | ------------------------------------------ | 20260506 13:54:00.402 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:54:00.402 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:54:00.402 | INFO | Total mem usage: 15.7 GiB | 20260506 13:54:00.402 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:54:00.402 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:54:00.402 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 13:54:00.402 | INFO | High Water Breakdown: | 20260506 13:54:00.402 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:54:00.402 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:54:00.402 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:54:00.402 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:54:00.402 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:54:00.402 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:54:00.402 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:54:00.402 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:54:00.402 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:54:00.402 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:54:00.402 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:54:00.402 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:54:00.402 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:54:00.402 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:54:00.402 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:54:00.402 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:54:00.402 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:54:00.402 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:54:00.402 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:54:00.402 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:54:00.403 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:54:00.403 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:54:00.403 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:54:00.403 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:54:00.403 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:54:00.403 | INFO | ------------------------------------------ | 20260506 13:54:00.403 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:54:00.403 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:54:00.403 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:54:00.403 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:54:00.403 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.56e+01, 1.33e+01 / 1.89e+00 | 20260506 13:54:00.403 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.01e-01, 1.04e-02 / 6.77e-03 | 20260506 13:54:00.403 | INFO | PerfReport: 11.59%, Baseline -- 3.66e+00 / 4.44e+00, 5.81e-01 / 3.75e-01 | 20260506 13:54:00.403 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.47e+00, 1.93e-01 / 1.64e-01 | 20260506 13:54:00.403 | INFO | PerfReport: 15.80%, DME -- 4.00e+01 / 4.41e+01, 3.91e+00 / 1.81e+00 | 20260506 13:54:00.403 | INFO | PerfReport: 56.67%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.51e+00 / 1.91e+00 | 20260506 13:54:00.403 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.36e+00 / 2.96e+00, 4.09e-01 / 2.52e-01 | 20260506 13:54:00.403 | INFO | PerfReport: 3.65%, Metrics -- 1.15e+00 / 1.55e+00, 2.99e-01 / 1.93e-01 | 20260506 13:54:00.403 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.03e-01 / 4.56e-01, 6.25e-02 / 2.55e-02 | 20260506 13:54:00.403 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:54:00.403 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:54:00.403 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:54:00.403 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 13:54:00.403 | INFO | PerfReport: 21.38%, Upload -- 2.10e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 13:54:00.403 | INFO | PerfReport: 21.01%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 13:54:00.403 | INFO | PerfReport: 32.19%, Compute -- 3.16e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 13:54:00.403 | INFO | PerfReport: 1.59%, LockUpload -- 1.56e+00 / 0.00e+00, 6.44e+00 / 0.00e+00 | 20260506 13:54:00.403 | INFO | PerfReport: 1.15%, LockDownload -- 1.13e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260506 13:54:00.403 | INFO | PerfReport: 22.66%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 13:54:00.403 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260506 13:54:00.516 | INFO | Starting a round of Baz IO | 20260506 13:54:03.667 | INFO | Populating RTMetrics report | 20260506 13:54:03.667 | INFO | RTMetrics report populated | 20260506 13:54:03.667 | INFO | Opening tmp RTMetrics json file | 20260506 13:54:03.668 | INFO | Serializing RTMetrics to json | 20260506 13:54:03.669 | INFO | Serialization done, swapping tmp file now... | 20260506 13:54:03.670 | INFO | RTMetrics done | 20260506 13:54:03.770 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.254 seconds | 20260506 13:54:03.770 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.03% / 0.02% / 60.95% | 20260506 13:54:03.770 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.86947% / 2.78155e-05% / 5.16574e-05% | 20260506 13:54:03.770 | INFO | Baz IO overall: | 20260506 13:54:03.770 | INFO | BytesWritten : 956117096119 | 20260506 13:54:03.770 | INFO | NumEvents : 462563694597 | 20260506 13:54:05.540 | INFO | Analyzing chunk frames = [1867776,1868288) | 20260506 13:54:05.540 | INFO | MemInfo: { | 20260506 13:54:05.540 | INFO | "gpu" : " 17 GB", | 20260506 13:54:05.540 | INFO | "huge" : "", | 20260506 13:54:05.540 | INFO | "huge_shared" : " 27 GB", | 20260506 13:54:05.540 | INFO | "malloc" : " 14 GB", | 20260506 13:54:05.540 | INFO | "pinned" : " 29 GB" | 20260506 13:54:05.540 | INFO | } | 20260506 13:54:05.541 | INFO | Perf report for the last 5.13904 seconds: | 20260506 13:54:05.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:54:05.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:05.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:05.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:05.541 | INFO | ANALYSIS : 63.0%, 3.0, 97.6ms, 0 | 20260506 13:54:05.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:05.541 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 13:54:05.541 | INFO | PRE_HQ : 61.2%, 1.9, 60.6ms, 0 | 20260506 13:54:05.541 | INFO | BAZWRITER : 62.7%, 3.7, 120.3ms, 0 | 20260506 13:54:05.541 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 13:54:05.541 | INFO | Uploaded 36GiB to GPU in 2.03514s (17.6892GiB/s) | 20260506 13:54:05.541 | INFO | Downloaded 38GiB to GPU in 2.20696s (17.2182GiB/s) | 20260506 13:54:10.679 | INFO | Analyzing chunk frames = [1868288,1868800) | 20260506 13:54:10.680 | INFO | MemInfo: { | 20260506 13:54:10.680 | INFO | "gpu" : " 17 GB", | 20260506 13:54:10.680 | INFO | "huge" : "", | 20260506 13:54:10.680 | INFO | "huge_shared" : " 27 GB", | 20260506 13:54:10.680 | INFO | "malloc" : " 14 GB", | 20260506 13:54:10.680 | INFO | "pinned" : " 29 GB" | 20260506 13:54:10.680 | INFO | } | 20260506 13:54:10.680 | INFO | Perf report for the last 5.13916 seconds: | 20260506 13:54:10.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:54:10.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:10.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:10.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:10.680 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 13:54:10.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:10.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:10.680 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:54:10.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:10.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:10.680 | INFO | Uploaded 36GiB to GPU in 2.0292s (17.741GiB/s) | 20260506 13:54:10.680 | INFO | Downloaded 34GiB to GPU in 1.95262s (17.4125GiB/s) | 20260506 13:54:15.820 | INFO | Analyzing chunk frames = [1868800,1869312) | 20260506 13:54:15.820 | INFO | MemInfo: { | 20260506 13:54:15.820 | INFO | "gpu" : " 17 GB", | 20260506 13:54:15.820 | INFO | "huge" : "", | 20260506 13:54:15.820 | INFO | "huge_shared" : " 27 GB", | 20260506 13:54:15.820 | INFO | "malloc" : " 14 GB", | 20260506 13:54:15.820 | INFO | "pinned" : " 29 GB" | 20260506 13:54:15.820 | INFO | } | 20260506 13:54:15.820 | INFO | Perf report for the last 5.14019 seconds: | 20260506 13:54:15.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:54:15.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:15.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:15.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:15.820 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:54:15.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:15.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:15.820 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:54:15.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:15.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:15.820 | INFO | Uploaded 36GiB to GPU in 1.9692s (18.2816GiB/s) | 20260506 13:54:15.820 | INFO | Downloaded 34GiB to GPU in 1.93981s (17.5275GiB/s) | 20260506 13:54:20.959 | INFO | Analyzing chunk frames = [1869312,1869824) | 20260506 13:54:20.959 | INFO | MemInfo: { | 20260506 13:54:20.959 | INFO | "gpu" : " 17 GB", | 20260506 13:54:20.959 | INFO | "huge" : "", | 20260506 13:54:20.959 | INFO | "huge_shared" : " 27 GB", | 20260506 13:54:20.959 | INFO | "malloc" : " 14 GB", | 20260506 13:54:20.959 | INFO | "pinned" : " 29 GB" | 20260506 13:54:20.959 | INFO | } | 20260506 13:54:20.959 | INFO | Perf report for the last 5.13925 seconds: | 20260506 13:54:20.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:54:20.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:20.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:20.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:20.959 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 13:54:20.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:20.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:20.959 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:54:20.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:20.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:20.959 | INFO | Uploaded 36GiB to GPU in 1.98328s (18.1517GiB/s) | 20260506 13:54:20.960 | INFO | Downloaded 34GiB to GPU in 1.92157s (17.6938GiB/s) | 20260506 13:54:26.098 | INFO | Analyzing chunk frames = [1869824,1870336) | 20260506 13:54:26.098 | INFO | MemInfo: { | 20260506 13:54:26.098 | INFO | "gpu" : " 17 GB", | 20260506 13:54:26.098 | INFO | "huge" : "", | 20260506 13:54:26.098 | INFO | "huge_shared" : " 27 GB", | 20260506 13:54:26.098 | INFO | "malloc" : " 14 GB", | 20260506 13:54:26.098 | INFO | "pinned" : " 29 GB" | 20260506 13:54:26.098 | INFO | } | 20260506 13:54:26.098 | INFO | Perf report for the last 5.13902 seconds: | 20260506 13:54:26.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:54:26.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:26.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:26.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:26.098 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 13:54:26.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:26.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:26.098 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 13:54:26.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:26.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:26.098 | INFO | Uploaded 36GiB to GPU in 1.99292s (18.064GiB/s) | 20260506 13:54:26.099 | INFO | Downloaded 34GiB to GPU in 1.93533s (17.5681GiB/s) | 20260506 13:54:31.237 | INFO | Analyzing chunk frames = [1870336,1870848) | 20260506 13:54:31.237 | INFO | MemInfo: { | 20260506 13:54:31.237 | INFO | "gpu" : " 17 GB", | 20260506 13:54:31.237 | INFO | "huge" : "", | 20260506 13:54:31.237 | INFO | "huge_shared" : " 27 GB", | 20260506 13:54:31.237 | INFO | "malloc" : " 14 GB", | 20260506 13:54:31.237 | INFO | "pinned" : " 29 GB" | 20260506 13:54:31.237 | INFO | } | 20260506 13:54:31.237 | INFO | Perf report for the last 5.13924 seconds: | 20260506 13:54:31.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:54:31.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:31.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:31.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:31.237 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 13:54:31.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:31.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:31.238 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:54:31.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:31.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:31.238 | INFO | Uploaded 36GiB to GPU in 2.04495s (17.6043GiB/s) | 20260506 13:54:31.238 | INFO | Downloaded 34GiB to GPU in 1.92559s (17.6569GiB/s) | 20260506 13:54:36.377 | INFO | Analyzing chunk frames = [1870848,1871360) | 20260506 13:54:36.377 | INFO | MemInfo: { | 20260506 13:54:36.377 | INFO | "gpu" : " 17 GB", | 20260506 13:54:36.377 | INFO | "huge" : "", | 20260506 13:54:36.377 | INFO | "huge_shared" : " 27 GB", | 20260506 13:54:36.377 | INFO | "malloc" : " 14 GB", | 20260506 13:54:36.377 | INFO | "pinned" : " 29 GB" | 20260506 13:54:36.377 | INFO | } | 20260506 13:54:36.377 | INFO | Perf report for the last 5.13925 seconds: | 20260506 13:54:36.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:54:36.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:36.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:36.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:36.377 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:54:36.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:36.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:36.377 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:54:36.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:36.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:36.377 | INFO | Uploaded 36GiB to GPU in 2.01144s (17.8976GiB/s) | 20260506 13:54:36.377 | INFO | Downloaded 34GiB to GPU in 1.99646s (17.0302GiB/s) | 20260506 13:54:41.516 | INFO | Analyzing chunk frames = [1871360,1871872) | 20260506 13:54:41.516 | INFO | MemInfo: { | 20260506 13:54:41.516 | INFO | "gpu" : " 17 GB", | 20260506 13:54:41.516 | INFO | "huge" : "", | 20260506 13:54:41.516 | INFO | "huge_shared" : " 27 GB", | 20260506 13:54:41.516 | INFO | "malloc" : " 14 GB", | 20260506 13:54:41.516 | INFO | "pinned" : " 29 GB" | 20260506 13:54:41.516 | INFO | } | 20260506 13:54:41.516 | INFO | Perf report for the last 5.13891 seconds: | 20260506 13:54:41.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:54:41.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:41.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:41.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:41.516 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 13:54:41.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:41.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:41.516 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 13:54:41.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:41.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:41.516 | INFO | Uploaded 36GiB to GPU in 2.02591s (17.7698GiB/s) | 20260506 13:54:41.516 | INFO | Downloaded 34GiB to GPU in 1.99156s (17.0721GiB/s) | 20260506 13:54:41.625 | INFO | Starting a round of Baz IO | 20260506 13:54:44.780 | INFO | Populating RTMetrics report | 20260506 13:54:44.781 | INFO | RTMetrics report populated | 20260506 13:54:44.781 | INFO | Opening tmp RTMetrics json file | 20260506 13:54:44.781 | INFO | Serializing RTMetrics to json | 20260506 13:54:44.783 | INFO | Serialization done, swapping tmp file now... | 20260506 13:54:44.783 | INFO | RTMetrics done | 20260506 13:54:44.881 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.256 seconds | 20260506 13:54:44.881 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.01% / 0.02% / 60.97% | 20260506 13:54:44.881 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.86049% / 1.58946e-05% / 6.35783e-05% | 20260506 13:54:44.881 | INFO | Baz IO overall: | 20260506 13:54:44.881 | INFO | BytesWritten : 958384379486 | 20260506 13:54:44.881 | INFO | NumEvents : 463663074432 | 20260506 13:54:46.655 | INFO | Analyzing chunk frames = [1871872,1872384) | 20260506 13:54:46.655 | INFO | MemInfo: { | 20260506 13:54:46.655 | INFO | "gpu" : " 17 GB", | 20260506 13:54:46.655 | INFO | "huge" : "", | 20260506 13:54:46.655 | INFO | "huge_shared" : " 27 GB", | 20260506 13:54:46.655 | INFO | "malloc" : " 14 GB", | 20260506 13:54:46.655 | INFO | "pinned" : " 29 GB" | 20260506 13:54:46.655 | INFO | } | 20260506 13:54:46.655 | INFO | Perf report for the last 5.1393 seconds: | 20260506 13:54:46.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:54:46.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:46.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:46.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:46.655 | INFO | ANALYSIS : 62.9%, 3.0, 97.8ms, 0 | 20260506 13:54:46.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:46.655 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:54:46.655 | INFO | PRE_HQ : 60.8%, 1.9, 60.5ms, 0 | 20260506 13:54:46.655 | INFO | BAZWRITER : 62.7%, 3.7, 119.4ms, 0 | 20260506 13:54:46.655 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 13:54:46.655 | INFO | Uploaded 36GiB to GPU in 2.03764s (17.6675GiB/s) | 20260506 13:54:46.655 | INFO | Downloaded 38GiB to GPU in 2.20506s (17.2331GiB/s) | 20260506 13:54:51.795 | INFO | Analyzing chunk frames = [1872384,1872896) | 20260506 13:54:51.795 | INFO | MemInfo: { | 20260506 13:54:51.795 | INFO | "gpu" : " 17 GB", | 20260506 13:54:51.795 | INFO | "huge" : "", | 20260506 13:54:51.795 | INFO | "huge_shared" : " 27 GB", | 20260506 13:54:51.795 | INFO | "malloc" : " 14 GB", | 20260506 13:54:51.795 | INFO | "pinned" : " 29 GB" | 20260506 13:54:51.795 | INFO | } | 20260506 13:54:51.795 | INFO | Perf report for the last 5.13974 seconds: | 20260506 13:54:51.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:54:51.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:51.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:51.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:51.795 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 13:54:51.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:51.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:51.795 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 13:54:51.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:51.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:51.795 | INFO | Uploaded 36GiB to GPU in 2.03016s (17.7326GiB/s) | 20260506 13:54:51.795 | INFO | Downloaded 34GiB to GPU in 1.94806s (17.4533GiB/s) | 20260506 13:54:56.934 | INFO | Analyzing chunk frames = [1872896,1873408) | 20260506 13:54:56.934 | INFO | MemInfo: { | 20260506 13:54:56.934 | INFO | "gpu" : " 17 GB", | 20260506 13:54:56.934 | INFO | "huge" : "", | 20260506 13:54:56.934 | INFO | "huge_shared" : " 27 GB", | 20260506 13:54:56.934 | INFO | "malloc" : " 14 GB", | 20260506 13:54:56.934 | INFO | "pinned" : " 29 GB" | 20260506 13:54:56.934 | INFO | } | 20260506 13:54:56.934 | INFO | Perf report for the last 5.13955 seconds: | 20260506 13:54:56.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:54:56.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:56.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:56.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:56.934 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 13:54:56.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:56.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:56.934 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 13:54:56.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:56.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:54:56.934 | INFO | Uploaded 36GiB to GPU in 1.99644s (18.0321GiB/s) | 20260506 13:54:56.935 | INFO | Downloaded 34GiB to GPU in 1.9384s (17.5402GiB/s) | 20260506 13:55:02.074 | INFO | Analyzing chunk frames = [1873408,1873920) | 20260506 13:55:02.074 | INFO | MemInfo: { | 20260506 13:55:02.074 | INFO | "gpu" : " 17 GB", | 20260506 13:55:02.074 | INFO | "huge" : "", | 20260506 13:55:02.074 | INFO | "huge_shared" : " 27 GB", | 20260506 13:55:02.074 | INFO | "malloc" : " 14 GB", | 20260506 13:55:02.074 | INFO | "pinned" : " 29 GB" | 20260506 13:55:02.074 | INFO | } | 20260506 13:55:02.074 | INFO | Perf report for the last 5.13965 seconds: | 20260506 13:55:02.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:55:02.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:02.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:02.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:02.074 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260506 13:55:02.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:02.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:02.074 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:55:02.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:02.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:02.074 | INFO | Uploaded 36GiB to GPU in 1.99575s (18.0383GiB/s) | 20260506 13:55:02.074 | INFO | Downloaded 34GiB to GPU in 1.93139s (17.6039GiB/s) | 20260506 13:55:07.212 | INFO | Analyzing chunk frames = [1873920,1874432) | 20260506 13:55:07.212 | INFO | MemInfo: { | 20260506 13:55:07.212 | INFO | "gpu" : " 17 GB", | 20260506 13:55:07.212 | INFO | "huge" : "", | 20260506 13:55:07.212 | INFO | "huge_shared" : " 27 GB", | 20260506 13:55:07.212 | INFO | "malloc" : " 14 GB", | 20260506 13:55:07.212 | INFO | "pinned" : " 29 GB" | 20260506 13:55:07.212 | INFO | } | 20260506 13:55:07.212 | INFO | Perf report for the last 5.13848 seconds: | 20260506 13:55:07.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:55:07.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:07.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:07.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:07.212 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 13:55:07.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:07.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:07.213 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:55:07.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:07.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:07.213 | INFO | Uploaded 36GiB to GPU in 2.00351s (17.9684GiB/s) | 20260506 13:55:07.213 | INFO | Downloaded 34GiB to GPU in 1.94531s (17.4779GiB/s) | 20260506 13:55:12.352 | INFO | Analyzing chunk frames = [1874432,1874944) | 20260506 13:55:12.352 | INFO | MemInfo: { | 20260506 13:55:12.352 | INFO | "gpu" : " 17 GB", | 20260506 13:55:12.352 | INFO | "huge" : "", | 20260506 13:55:12.352 | INFO | "huge_shared" : " 27 GB", | 20260506 13:55:12.352 | INFO | "malloc" : " 14 GB", | 20260506 13:55:12.352 | INFO | "pinned" : " 29 GB" | 20260506 13:55:12.352 | INFO | } | 20260506 13:55:12.352 | INFO | Perf report for the last 5.1399 seconds: | 20260506 13:55:12.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:55:12.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:12.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:12.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:12.352 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 13:55:12.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:12.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:12.352 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 13:55:12.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:12.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:12.353 | INFO | Uploaded 36GiB to GPU in 2.03831s (17.6616GiB/s) | 20260506 13:55:12.353 | INFO | Downloaded 34GiB to GPU in 1.91839s (17.7232GiB/s) | 20260506 13:55:17.493 | INFO | Analyzing chunk frames = [1874944,1875456) | 20260506 13:55:17.493 | INFO | MemInfo: { | 20260506 13:55:17.493 | INFO | "gpu" : " 17 GB", | 20260506 13:55:17.493 | INFO | "huge" : "", | 20260506 13:55:17.493 | INFO | "huge_shared" : " 27 GB", | 20260506 13:55:17.493 | INFO | "malloc" : " 14 GB", | 20260506 13:55:17.493 | INFO | "pinned" : " 29 GB" | 20260506 13:55:17.493 | INFO | } | 20260506 13:55:17.493 | INFO | Perf report for the last 5.14052 seconds: | 20260506 13:55:17.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:55:17.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:17.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:17.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:17.493 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 13:55:17.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:17.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:17.493 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 13:55:17.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:17.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:17.493 | INFO | Uploaded 36GiB to GPU in 2.00621s (17.9443GiB/s) | 20260506 13:55:17.493 | INFO | Downloaded 34GiB to GPU in 2.00379s (16.9679GiB/s) | 20260506 13:55:22.631 | INFO | Analyzing chunk frames = [1875456,1875968) | 20260506 13:55:22.631 | INFO | MemInfo: { | 20260506 13:55:22.631 | INFO | "gpu" : " 17 GB", | 20260506 13:55:22.631 | INFO | "huge" : "", | 20260506 13:55:22.631 | INFO | "huge_shared" : " 27 GB", | 20260506 13:55:22.631 | INFO | "malloc" : " 14 GB", | 20260506 13:55:22.631 | INFO | "pinned" : " 29 GB" | 20260506 13:55:22.631 | INFO | } | 20260506 13:55:22.631 | INFO | Perf report for the last 5.13812 seconds: | 20260506 13:55:22.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:55:22.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:22.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:22.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:22.631 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 13:55:22.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:22.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:22.631 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:55:22.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:22.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:22.631 | INFO | Uploaded 36GiB to GPU in 2.05352s (17.5309GiB/s) | 20260506 13:55:22.631 | INFO | Downloaded 34GiB to GPU in 1.98193s (17.155GiB/s) | 20260506 13:55:22.632 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:55:22.632 | INFO | Total mem usage: 17.4 GiB | 20260506 13:55:22.632 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:55:22.632 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -21%) | 20260506 13:55:22.632 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:55:22.632 | INFO | High Water Breakdown: | 20260506 13:55:22.632 | INFO | 59.8% : 8.6 GiB ( 78.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:55:22.632 | INFO | 40.1% : 5.77 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:55:22.632 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:55:22.632 | INFO | ------------------------------------------ | 20260506 13:55:22.632 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:55:22.632 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:55:22.632 | INFO | Total mem usage: 26.5 GiB | 20260506 13:55:22.632 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:55:22.632 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:55:22.632 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:55:22.632 | INFO | High Water Breakdown: | 20260506 13:55:22.632 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:55:22.632 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:55:22.632 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:55:22.632 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:55:22.632 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:55:22.632 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:55:22.632 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:55:22.632 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:55:22.632 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:55:22.632 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:55:22.632 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:55:22.632 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:55:22.632 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:55:22.632 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:55:22.632 | INFO | ------------------------------------------ | 20260506 13:55:22.632 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:55:22.632 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:55:22.632 | INFO | Total mem usage: 25.6 GiB | 20260506 13:55:22.632 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:55:22.632 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:55:22.632 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:55:22.632 | INFO | High Water Breakdown: | 20260506 13:55:22.632 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:55:22.632 | INFO | ------------------------------------------ | 20260506 13:55:22.632 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:55:22.632 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:55:22.632 | INFO | Total mem usage: 15.7 GiB | 20260506 13:55:22.632 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:55:22.632 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:55:22.632 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 13:55:22.632 | INFO | High Water Breakdown: | 20260506 13:55:22.632 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:55:22.632 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:55:22.632 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:55:22.632 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:55:22.632 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:55:22.632 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:55:22.632 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:55:22.632 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:55:22.632 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:55:22.632 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:55:22.632 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:55:22.632 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:55:22.632 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:55:22.632 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:55:22.632 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:55:22.632 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:55:22.632 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:55:22.632 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:55:22.632 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:55:22.632 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:55:22.632 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:55:22.632 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:55:22.632 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:55:22.632 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:55:22.632 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:55:22.632 | INFO | ------------------------------------------ | 20260506 13:55:22.632 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:55:22.632 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:55:22.632 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:55:22.632 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:55:22.632 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.57e+01, 1.33e+01 / 1.78e+00 | 20260506 13:55:22.632 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.03e-01, 1.05e-02 / 9.00e-03 | 20260506 13:55:22.632 | INFO | PerfReport: 11.55%, Baseline -- 3.65e+00 / 4.44e+00, 5.64e-01 / 3.09e-01 | 20260506 13:55:22.632 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.48e+00, 1.87e-01 / 1.33e-01 | 20260506 13:55:22.632 | INFO | PerfReport: 15.81%, DME -- 3.99e+01 / 4.42e+01, 3.91e+00 / 2.44e+00 | 20260506 13:55:22.632 | INFO | PerfReport: 56.71%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.50e+00 / 1.86e+00 | 20260506 13:55:22.632 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.36e+00 / 2.97e+00, 4.12e-01 / 2.64e-01 | 20260506 13:55:22.632 | INFO | PerfReport: 3.64%, Metrics -- 1.15e+00 / 1.48e+00, 2.92e-01 / 1.49e-01 | 20260506 13:55:22.632 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.02e-01 / 4.54e-01, 6.23e-02 / 3.12e-02 | 20260506 13:55:22.632 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:55:22.633 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:55:22.633 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:55:22.633 | INFO | PerfReport: Full Execution -- 9.83e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 13:55:22.633 | INFO | PerfReport: 21.37%, Upload -- 2.10e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 13:55:22.633 | INFO | PerfReport: 21.03%, Download -- 2.07e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 13:55:22.633 | INFO | PerfReport: 32.14%, Compute -- 3.16e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 13:55:22.633 | INFO | PerfReport: 1.58%, LockUpload -- 1.55e+00 / 0.00e+00, 6.62e+00 / 0.00e+00 | 20260506 13:55:22.633 | INFO | PerfReport: 1.18%, LockDownload -- 1.16e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 13:55:22.633 | INFO | PerfReport: 22.68%, LockCompute -- 2.23e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 13:55:22.633 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260506 13:55:22.764 | INFO | Starting a round of Baz IO | 20260506 13:55:25.918 | INFO | Populating RTMetrics report | 20260506 13:55:25.919 | INFO | RTMetrics report populated | 20260506 13:55:25.919 | INFO | Opening tmp RTMetrics json file | 20260506 13:55:25.920 | INFO | Serializing RTMetrics to json | 20260506 13:55:25.921 | INFO | Serialization done, swapping tmp file now... | 20260506 13:55:25.922 | INFO | RTMetrics done | 20260506 13:55:26.022 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.257 seconds | 20260506 13:55:26.022 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 39.00% / 0.02% / 60.99% | 20260506 13:55:26.022 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.85351% / 2.78155e-05% / 5.5631e-05% | 20260506 13:55:26.022 | INFO | Baz IO overall: | 20260506 13:55:26.022 | INFO | BytesWritten : 960651606781 | 20260506 13:55:26.022 | INFO | NumEvents : 464762609757 | 20260506 13:55:27.771 | INFO | Analyzing chunk frames = [1875968,1876480) | 20260506 13:55:27.771 | INFO | MemInfo: { | 20260506 13:55:27.771 | INFO | "gpu" : " 17 GB", | 20260506 13:55:27.771 | INFO | "huge" : "", | 20260506 13:55:27.771 | INFO | "huge_shared" : " 27 GB", | 20260506 13:55:27.771 | INFO | "malloc" : " 14 GB", | 20260506 13:55:27.771 | INFO | "pinned" : " 29 GB" | 20260506 13:55:27.771 | INFO | } | 20260506 13:55:27.771 | INFO | Perf report for the last 5.1399 seconds: | 20260506 13:55:27.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:55:27.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:27.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:27.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:27.771 | INFO | ANALYSIS : 63.4%, 3.0, 98.4ms, 0 | 20260506 13:55:27.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:27.771 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:55:27.771 | INFO | PRE_HQ : 61.1%, 1.9, 60.4ms, 0 | 20260506 13:55:27.771 | INFO | BAZWRITER : 62.8%, 3.7, 119.4ms, 0 | 20260506 13:55:27.771 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:55:27.771 | INFO | Uploaded 36GiB to GPU in 2.04258s (17.6247GiB/s) | 20260506 13:55:27.771 | INFO | Downloaded 38GiB to GPU in 2.21154s (17.1826GiB/s) | 20260506 13:55:32.910 | INFO | Analyzing chunk frames = [1876480,1876992) | 20260506 13:55:32.910 | INFO | MemInfo: { | 20260506 13:55:32.910 | INFO | "gpu" : " 17 GB", | 20260506 13:55:32.910 | INFO | "huge" : "", | 20260506 13:55:32.910 | INFO | "huge_shared" : " 27 GB", | 20260506 13:55:32.910 | INFO | "malloc" : " 14 GB", | 20260506 13:55:32.910 | INFO | "pinned" : " 29 GB" | 20260506 13:55:32.910 | INFO | } | 20260506 13:55:32.910 | INFO | Perf report for the last 5.13913 seconds: | 20260506 13:55:32.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:55:32.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:32.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:32.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:32.910 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 13:55:32.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:32.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:32.910 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:55:32.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:32.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:32.910 | INFO | Uploaded 36GiB to GPU in 2.00724s (17.9351GiB/s) | 20260506 13:55:32.910 | INFO | Downloaded 34GiB to GPU in 1.94144s (17.5128GiB/s) | 20260506 13:55:38.048 | INFO | Analyzing chunk frames = [1876992,1877504) | 20260506 13:55:38.048 | INFO | MemInfo: { | 20260506 13:55:38.048 | INFO | "gpu" : " 17 GB", | 20260506 13:55:38.048 | INFO | "huge" : "", | 20260506 13:55:38.048 | INFO | "huge_shared" : " 27 GB", | 20260506 13:55:38.048 | INFO | "malloc" : " 14 GB", | 20260506 13:55:38.049 | INFO | "pinned" : " 29 GB" | 20260506 13:55:38.049 | INFO | } | 20260506 13:55:38.049 | INFO | Perf report for the last 5.1385 seconds: | 20260506 13:55:38.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:55:38.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:38.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:38.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:38.049 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 13:55:38.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:38.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:38.049 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:55:38.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:38.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:38.049 | INFO | Uploaded 36GiB to GPU in 1.98304s (18.1539GiB/s) | 20260506 13:55:38.049 | INFO | Downloaded 34GiB to GPU in 1.93533s (17.5681GiB/s) | 20260506 13:55:43.188 | INFO | Analyzing chunk frames = [1877504,1878016) | 20260506 13:55:43.188 | INFO | MemInfo: { | 20260506 13:55:43.188 | INFO | "gpu" : " 17 GB", | 20260506 13:55:43.188 | INFO | "huge" : "", | 20260506 13:55:43.188 | INFO | "huge_shared" : " 27 GB", | 20260506 13:55:43.188 | INFO | "malloc" : " 14 GB", | 20260506 13:55:43.188 | INFO | "pinned" : " 29 GB" | 20260506 13:55:43.188 | INFO | } | 20260506 13:55:43.188 | INFO | Perf report for the last 5.13925 seconds: | 20260506 13:55:43.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:55:43.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:43.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:43.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:43.188 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 13:55:43.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:43.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:43.188 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:55:43.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:43.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:43.188 | INFO | Uploaded 36GiB to GPU in 1.98633s (18.1239GiB/s) | 20260506 13:55:43.188 | INFO | Downloaded 34GiB to GPU in 1.91483s (17.7561GiB/s) | 20260506 13:55:48.327 | INFO | Analyzing chunk frames = [1878016,1878528) | 20260506 13:55:48.327 | INFO | MemInfo: { | 20260506 13:55:48.327 | INFO | "gpu" : " 17 GB", | 20260506 13:55:48.327 | INFO | "huge" : "", | 20260506 13:55:48.327 | INFO | "huge_shared" : " 27 GB", | 20260506 13:55:48.327 | INFO | "malloc" : " 14 GB", | 20260506 13:55:48.327 | INFO | "pinned" : " 29 GB" | 20260506 13:55:48.327 | INFO | } | 20260506 13:55:48.327 | INFO | Perf report for the last 5.13914 seconds: | 20260506 13:55:48.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:55:48.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:48.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:48.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:48.327 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 13:55:48.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:48.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:48.327 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:55:48.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:48.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:48.327 | INFO | Uploaded 36GiB to GPU in 2.01225s (17.8905GiB/s) | 20260506 13:55:48.327 | INFO | Downloaded 34GiB to GPU in 1.94919s (17.4432GiB/s) | 20260506 13:55:53.467 | INFO | Analyzing chunk frames = [1878528,1879040) | 20260506 13:55:53.467 | INFO | MemInfo: { | 20260506 13:55:53.467 | INFO | "gpu" : " 17 GB", | 20260506 13:55:53.467 | INFO | "huge" : "", | 20260506 13:55:53.467 | INFO | "huge_shared" : " 27 GB", | 20260506 13:55:53.467 | INFO | "malloc" : " 14 GB", | 20260506 13:55:53.467 | INFO | "pinned" : " 29 GB" | 20260506 13:55:53.467 | INFO | } | 20260506 13:55:53.467 | INFO | Perf report for the last 5.14012 seconds: | 20260506 13:55:53.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:55:53.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:53.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:53.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:53.467 | INFO | ANALYSIS : 62.0%, 3.0, 96.8ms, 0 | 20260506 13:55:53.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:53.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:53.467 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:55:53.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:53.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:53.467 | INFO | Uploaded 36GiB to GPU in 2.0436s (17.6159GiB/s) | 20260506 13:55:53.467 | INFO | Downloaded 34GiB to GPU in 1.9368s (17.5547GiB/s) | 20260506 13:55:58.607 | INFO | Analyzing chunk frames = [1879040,1879552) | 20260506 13:55:58.607 | INFO | MemInfo: { | 20260506 13:55:58.607 | INFO | "gpu" : " 17 GB", | 20260506 13:55:58.607 | INFO | "huge" : "", | 20260506 13:55:58.607 | INFO | "huge_shared" : " 27 GB", | 20260506 13:55:58.607 | INFO | "malloc" : " 14 GB", | 20260506 13:55:58.607 | INFO | "pinned" : " 29 GB" | 20260506 13:55:58.607 | INFO | } | 20260506 13:55:58.607 | INFO | Perf report for the last 5.13984 seconds: | 20260506 13:55:58.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:55:58.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:58.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:58.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:58.607 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 13:55:58.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:58.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:58.607 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 13:55:58.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:58.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:55:58.607 | INFO | Uploaded 36GiB to GPU in 2.00104s (17.9906GiB/s) | 20260506 13:55:58.607 | INFO | Downloaded 34GiB to GPU in 2.00327s (16.9722GiB/s) | 20260506 13:56:03.746 | INFO | Analyzing chunk frames = [1879552,1880064) | 20260506 13:56:03.746 | INFO | MemInfo: { | 20260506 13:56:03.746 | INFO | "gpu" : " 17 GB", | 20260506 13:56:03.746 | INFO | "huge" : "", | 20260506 13:56:03.746 | INFO | "huge_shared" : " 27 GB", | 20260506 13:56:03.746 | INFO | "malloc" : " 14 GB", | 20260506 13:56:03.746 | INFO | "pinned" : " 29 GB" | 20260506 13:56:03.746 | INFO | } | 20260506 13:56:03.747 | INFO | Perf report for the last 5.13955 seconds: | 20260506 13:56:03.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:56:03.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:03.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:03.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:03.747 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 13:56:03.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:03.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:03.747 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 13:56:03.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:03.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:03.747 | INFO | Uploaded 36GiB to GPU in 2.02311s (17.7944GiB/s) | 20260506 13:56:03.747 | INFO | Downloaded 34GiB to GPU in 1.97648s (17.2023GiB/s) | 20260506 13:56:03.863 | INFO | Starting a round of Baz IO | 20260506 13:56:07.019 | INFO | Populating RTMetrics report | 20260506 13:56:07.019 | INFO | RTMetrics report populated | 20260506 13:56:07.019 | INFO | Opening tmp RTMetrics json file | 20260506 13:56:07.020 | INFO | Serializing RTMetrics to json | 20260506 13:56:07.021 | INFO | Serialization done, swapping tmp file now... | 20260506 13:56:07.021 | INFO | RTMetrics done | 20260506 13:56:07.125 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.262 seconds | 20260506 13:56:07.125 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.98% / 0.02% / 61.00% | 20260506 13:56:07.125 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.84498% / 2.78155e-05% / 5.96046e-05% | 20260506 13:56:07.125 | INFO | Baz IO overall: | 20260506 13:56:07.125 | INFO | BytesWritten : 962918786049 | 20260506 13:56:07.125 | INFO | NumEvents : 465862300971 | 20260506 13:56:08.885 | INFO | Analyzing chunk frames = [1880064,1880576) | 20260506 13:56:08.885 | INFO | MemInfo: { | 20260506 13:56:08.885 | INFO | "gpu" : " 17 GB", | 20260506 13:56:08.885 | INFO | "huge" : "", | 20260506 13:56:08.885 | INFO | "huge_shared" : " 27 GB", | 20260506 13:56:08.885 | INFO | "malloc" : " 14 GB", | 20260506 13:56:08.885 | INFO | "pinned" : " 29 GB" | 20260506 13:56:08.885 | INFO | } | 20260506 13:56:08.885 | INFO | Perf report for the last 5.13894 seconds: | 20260506 13:56:08.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:56:08.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:08.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:08.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:08.886 | INFO | ANALYSIS : 63.1%, 3.0, 97.8ms, 0 | 20260506 13:56:08.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:08.886 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 13:56:08.886 | INFO | PRE_HQ : 61.1%, 1.9, 60.5ms, 0 | 20260506 13:56:08.886 | INFO | BAZWRITER : 62.8%, 3.8, 122.8ms, 0 | 20260506 13:56:08.886 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 13:56:08.886 | INFO | Uploaded 36GiB to GPU in 2.02438s (17.7832GiB/s) | 20260506 13:56:08.886 | INFO | Downloaded 38GiB to GPU in 2.21523s (17.154GiB/s) | 20260506 13:56:14.024 | INFO | Analyzing chunk frames = [1880576,1881088) | 20260506 13:56:14.024 | INFO | MemInfo: { | 20260506 13:56:14.024 | INFO | "gpu" : " 17 GB", | 20260506 13:56:14.024 | INFO | "huge" : "", | 20260506 13:56:14.024 | INFO | "huge_shared" : " 27 GB", | 20260506 13:56:14.024 | INFO | "malloc" : " 14 GB", | 20260506 13:56:14.024 | INFO | "pinned" : " 29 GB" | 20260506 13:56:14.024 | INFO | } | 20260506 13:56:14.024 | INFO | Perf report for the last 5.13898 seconds: | 20260506 13:56:14.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:56:14.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:14.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:14.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:14.024 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 13:56:14.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:14.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:14.025 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 13:56:14.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:14.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:14.025 | INFO | Uploaded 36GiB to GPU in 1.98595s (18.1273GiB/s) | 20260506 13:56:14.025 | INFO | Downloaded 34GiB to GPU in 1.94474s (17.4831GiB/s) | 20260506 13:56:19.164 | INFO | Analyzing chunk frames = [1881088,1881600) | 20260506 13:56:19.164 | INFO | MemInfo: { | 20260506 13:56:19.164 | INFO | "gpu" : " 17 GB", | 20260506 13:56:19.164 | INFO | "huge" : "", | 20260506 13:56:19.164 | INFO | "huge_shared" : " 27 GB", | 20260506 13:56:19.164 | INFO | "malloc" : " 14 GB", | 20260506 13:56:19.164 | INFO | "pinned" : " 29 GB" | 20260506 13:56:19.164 | INFO | } | 20260506 13:56:19.164 | INFO | Perf report for the last 5.13949 seconds: | 20260506 13:56:19.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:56:19.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:19.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:19.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:19.164 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 13:56:19.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:19.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:19.164 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 13:56:19.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:19.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:19.164 | INFO | Uploaded 36GiB to GPU in 1.98279s (18.1562GiB/s) | 20260506 13:56:19.164 | INFO | Downloaded 34GiB to GPU in 1.93507s (17.5705GiB/s) | 20260506 13:56:24.303 | INFO | Analyzing chunk frames = [1881600,1882112) | 20260506 13:56:24.303 | INFO | MemInfo: { | 20260506 13:56:24.303 | INFO | "gpu" : " 17 GB", | 20260506 13:56:24.303 | INFO | "huge" : "", | 20260506 13:56:24.303 | INFO | "huge_shared" : " 27 GB", | 20260506 13:56:24.303 | INFO | "malloc" : " 14 GB", | 20260506 13:56:24.303 | INFO | "pinned" : " 29 GB" | 20260506 13:56:24.303 | INFO | } | 20260506 13:56:24.303 | INFO | Perf report for the last 5.13949 seconds: | 20260506 13:56:24.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:56:24.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:24.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:24.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:24.304 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 13:56:24.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:24.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:24.304 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 13:56:24.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:24.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:24.304 | INFO | Uploaded 36GiB to GPU in 1.99506s (18.0446GiB/s) | 20260506 13:56:24.304 | INFO | Downloaded 34GiB to GPU in 1.91897s (17.7178GiB/s) | 20260506 13:56:29.443 | INFO | Analyzing chunk frames = [1882112,1882624) | 20260506 13:56:29.443 | INFO | MemInfo: { | 20260506 13:56:29.443 | INFO | "gpu" : " 17 GB", | 20260506 13:56:29.443 | INFO | "huge" : "", | 20260506 13:56:29.443 | INFO | "huge_shared" : " 27 GB", | 20260506 13:56:29.443 | INFO | "malloc" : " 14 GB", | 20260506 13:56:29.443 | INFO | "pinned" : " 29 GB" | 20260506 13:56:29.443 | INFO | } | 20260506 13:56:29.443 | INFO | Perf report for the last 5.13949 seconds: | 20260506 13:56:29.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:56:29.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:29.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:29.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:29.443 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 13:56:29.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:29.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:29.443 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 13:56:29.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:29.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:29.443 | INFO | Uploaded 36GiB to GPU in 2.02056s (17.8169GiB/s) | 20260506 13:56:29.443 | INFO | Downloaded 34GiB to GPU in 1.95346s (17.405GiB/s) | 20260506 13:56:34.582 | INFO | Analyzing chunk frames = [1882624,1883136) | 20260506 13:56:34.582 | INFO | MemInfo: { | 20260506 13:56:34.582 | INFO | "gpu" : " 17 GB", | 20260506 13:56:34.583 | INFO | "huge" : "", | 20260506 13:56:34.583 | INFO | "huge_shared" : " 27 GB", | 20260506 13:56:34.583 | INFO | "malloc" : " 14 GB", | 20260506 13:56:34.583 | INFO | "pinned" : " 29 GB" | 20260506 13:56:34.583 | INFO | } | 20260506 13:56:34.583 | INFO | Perf report for the last 5.13966 seconds: | 20260506 13:56:34.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:56:34.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:34.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:34.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:34.583 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 13:56:34.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:34.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:34.583 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 13:56:34.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:34.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:34.583 | INFO | Uploaded 36GiB to GPU in 2.04365s (17.6155GiB/s) | 20260506 13:56:34.583 | INFO | Downloaded 34GiB to GPU in 1.92001s (17.7082GiB/s) | 20260506 13:56:39.721 | INFO | Analyzing chunk frames = [1883136,1883648) | 20260506 13:56:39.721 | INFO | MemInfo: { | 20260506 13:56:39.721 | INFO | "gpu" : " 17 GB", | 20260506 13:56:39.721 | INFO | "huge" : "", | 20260506 13:56:39.721 | INFO | "huge_shared" : " 27 GB", | 20260506 13:56:39.722 | INFO | "malloc" : " 14 GB", | 20260506 13:56:39.722 | INFO | "pinned" : " 29 GB" | 20260506 13:56:39.722 | INFO | } | 20260506 13:56:39.722 | INFO | Perf report for the last 5.13902 seconds: | 20260506 13:56:39.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:56:39.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:39.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:39.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:39.722 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 13:56:39.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:39.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:39.722 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 13:56:39.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:39.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:39.722 | INFO | Uploaded 36GiB to GPU in 1.96786s (18.294GiB/s) | 20260506 13:56:39.722 | INFO | Downloaded 34GiB to GPU in 1.99368s (17.0539GiB/s) | 20260506 13:56:44.860 | INFO | Analyzing chunk frames = [1883648,1884160) | 20260506 13:56:44.860 | INFO | MemInfo: { | 20260506 13:56:44.860 | INFO | "gpu" : " 17 GB", | 20260506 13:56:44.860 | INFO | "huge" : "", | 20260506 13:56:44.860 | INFO | "huge_shared" : " 27 GB", | 20260506 13:56:44.860 | INFO | "malloc" : " 14 GB", | 20260506 13:56:44.860 | INFO | "pinned" : " 29 GB" | 20260506 13:56:44.860 | INFO | } | 20260506 13:56:44.861 | INFO | Perf report for the last 5.13885 seconds: | 20260506 13:56:44.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:56:44.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:44.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:44.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:44.861 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 13:56:44.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:44.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:44.861 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260506 13:56:44.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:44.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:44.861 | INFO | Uploaded 36GiB to GPU in 2.00871s (17.9219GiB/s) | 20260506 13:56:44.861 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:56:44.861 | INFO | Downloaded 34GiB to GPU in 1.97895s (17.1808GiB/s) | 20260506 13:56:44.861 | INFO | Total mem usage: 17.4 GiB | 20260506 13:56:44.861 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:56:44.861 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -20.5%) | 20260506 13:56:44.861 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:56:44.861 | INFO | High Water Breakdown: | 20260506 13:56:44.861 | INFO | 59.9% : 8.65 GiB ( 79.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:56:44.861 | INFO | 40% : 5.78 GiB ( 86.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:56:44.861 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:56:44.861 | INFO | ------------------------------------------ | 20260506 13:56:44.861 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:56:44.861 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:56:44.861 | INFO | Total mem usage: 26.5 GiB | 20260506 13:56:44.861 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:56:44.861 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:56:44.861 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:56:44.861 | INFO | High Water Breakdown: | 20260506 13:56:44.861 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:56:44.861 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:56:44.861 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:56:44.861 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:56:44.861 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:56:44.861 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:56:44.861 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:56:44.861 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:56:44.861 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:56:44.861 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:56:44.861 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:56:44.861 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:56:44.861 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:56:44.861 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:56:44.861 | INFO | ------------------------------------------ | 20260506 13:56:44.861 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:56:44.861 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:56:44.861 | INFO | Total mem usage: 25.6 GiB | 20260506 13:56:44.861 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:56:44.861 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:56:44.861 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:56:44.861 | INFO | High Water Breakdown: | 20260506 13:56:44.861 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:56:44.861 | INFO | ------------------------------------------ | 20260506 13:56:44.861 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:56:44.861 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:56:44.861 | INFO | Total mem usage: 15.7 GiB | 20260506 13:56:44.862 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:56:44.862 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:56:44.862 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 13:56:44.862 | INFO | High Water Breakdown: | 20260506 13:56:44.862 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:56:44.862 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:56:44.862 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:56:44.862 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:56:44.862 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:56:44.862 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:56:44.862 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:56:44.862 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:56:44.862 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:56:44.862 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:56:44.862 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:56:44.862 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:56:44.862 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:56:44.862 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:56:44.862 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:56:44.862 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:56:44.862 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:56:44.862 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:56:44.862 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:56:44.862 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:56:44.862 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:56:44.862 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:56:44.862 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:56:44.862 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:56:44.862 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:56:44.862 | INFO | ------------------------------------------ | 20260506 13:56:44.862 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:56:44.862 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:56:44.862 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:56:44.862 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:56:44.862 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.53e+01, 1.34e+01 / 1.88e+00 | 20260506 13:56:44.862 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.01e-01, 1.03e-02 / 6.29e-03 | 20260506 13:56:44.862 | INFO | PerfReport: 11.58%, Baseline -- 3.66e+00 / 4.43e+00, 5.72e-01 / 3.22e-01 | 20260506 13:56:44.862 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.45e+00, 1.87e-01 / 1.23e-01 | 20260506 13:56:44.862 | INFO | PerfReport: 15.77%, DME -- 3.98e+01 / 4.37e+01, 3.54e+00 / 1.64e+00 | 20260506 13:56:44.862 | INFO | PerfReport: 56.72%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.48e+00 / 1.88e+00 | 20260506 13:56:44.862 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.36e+00 / 2.92e+00, 4.08e-01 / 2.49e-01 | 20260506 13:56:44.862 | INFO | PerfReport: 3.64%, Metrics -- 1.15e+00 / 1.52e+00, 2.96e-01 / 1.80e-01 | 20260506 13:56:44.862 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.01e-01 / 4.55e-01, 6.32e-02 / 2.45e-02 | 20260506 13:56:44.862 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:56:44.862 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:56:44.862 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:56:44.862 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 13:56:44.862 | INFO | PerfReport: 21.31%, Upload -- 2.09e+01 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260506 13:56:44.862 | INFO | PerfReport: 21.04%, Download -- 2.07e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 13:56:44.862 | INFO | PerfReport: 32.15%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 13:56:44.862 | INFO | PerfReport: 1.50%, LockUpload -- 1.47e+00 / 0.00e+00, 6.56e+00 / 0.00e+00 | 20260506 13:56:44.862 | INFO | PerfReport: 1.26%, LockDownload -- 1.24e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260506 13:56:44.862 | INFO | PerfReport: 22.73%, LockCompute -- 2.23e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 13:56:44.862 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.26e-02 / 0.00e+00 | 20260506 13:56:44.978 | INFO | Starting a round of Baz IO | 20260506 13:56:48.134 | INFO | Populating RTMetrics report | 20260506 13:56:48.135 | INFO | RTMetrics report populated | 20260506 13:56:48.135 | INFO | Opening tmp RTMetrics json file | 20260506 13:56:48.135 | INFO | Serializing RTMetrics to json | 20260506 13:56:48.136 | INFO | Serialization done, swapping tmp file now... | 20260506 13:56:48.137 | INFO | RTMetrics done | 20260506 13:56:48.240 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.261 seconds | 20260506 13:56:48.240 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.97% / 0.02% / 61.02% | 20260506 13:56:48.240 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.83167% / 2.38419e-05% / 4.76837e-05% | 20260506 13:56:48.240 | INFO | Baz IO overall: | 20260506 13:56:48.240 | INFO | BytesWritten : 965184284258 | 20260506 13:56:48.240 | INFO | NumEvents : 466960599794 | 20260506 13:56:50.001 | INFO | Analyzing chunk frames = [1884160,1884672) | 20260506 13:56:50.001 | INFO | MemInfo: { | 20260506 13:56:50.001 | INFO | "gpu" : " 17 GB", | 20260506 13:56:50.001 | INFO | "huge" : "", | 20260506 13:56:50.001 | INFO | "huge_shared" : " 27 GB", | 20260506 13:56:50.001 | INFO | "malloc" : " 14 GB", | 20260506 13:56:50.001 | INFO | "pinned" : " 29 GB" | 20260506 13:56:50.001 | INFO | } | 20260506 13:56:50.001 | INFO | Perf report for the last 5.14066 seconds: | 20260506 13:56:50.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:56:50.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:50.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:50.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:50.001 | INFO | ANALYSIS : 63.1%, 3.0, 97.9ms, 0 | 20260506 13:56:50.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:50.001 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:56:50.001 | INFO | PRE_HQ : 60.9%, 1.9, 60.5ms, 0 | 20260506 13:56:50.001 | INFO | BAZWRITER : 62.8%, 3.8, 121.8ms, 0 | 20260506 13:56:50.001 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:56:50.002 | INFO | Uploaded 36GiB to GPU in 2.02081s (17.8146GiB/s) | 20260506 13:56:50.002 | INFO | Downloaded 38GiB to GPU in 2.2083s (17.2078GiB/s) | 20260506 13:56:55.139 | INFO | Analyzing chunk frames = [1884672,1885184) | 20260506 13:56:55.139 | INFO | MemInfo: { | 20260506 13:56:55.139 | INFO | "gpu" : " 17 GB", | 20260506 13:56:55.139 | INFO | "huge" : "", | 20260506 13:56:55.139 | INFO | "huge_shared" : " 27 GB", | 20260506 13:56:55.139 | INFO | "malloc" : " 14 GB", | 20260506 13:56:55.139 | INFO | "pinned" : " 29 GB" | 20260506 13:56:55.139 | INFO | } | 20260506 13:56:55.139 | INFO | Perf report for the last 5.13754 seconds: | 20260506 13:56:55.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:56:55.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:55.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:55.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:55.139 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 13:56:55.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:55.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:55.139 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 13:56:55.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:55.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:56:55.139 | INFO | Uploaded 36GiB to GPU in 1.99046s (18.0863GiB/s) | 20260506 13:56:55.139 | INFO | Downloaded 34GiB to GPU in 1.94902s (17.4447GiB/s) | 20260506 13:57:00.279 | INFO | Analyzing chunk frames = [1885184,1885696) | 20260506 13:57:00.279 | INFO | MemInfo: { | 20260506 13:57:00.279 | INFO | "gpu" : " 17 GB", | 20260506 13:57:00.279 | INFO | "huge" : "", | 20260506 13:57:00.279 | INFO | "huge_shared" : " 27 GB", | 20260506 13:57:00.279 | INFO | "malloc" : " 14 GB", | 20260506 13:57:00.279 | INFO | "pinned" : " 29 GB" | 20260506 13:57:00.280 | INFO | } | 20260506 13:57:00.280 | INFO | Perf report for the last 5.1408 seconds: | 20260506 13:57:00.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:57:00.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:00.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:00.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:00.280 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 13:57:00.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:00.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:00.280 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:57:00.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:00.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:00.280 | INFO | Uploaded 36GiB to GPU in 1.96026s (18.365GiB/s) | 20260506 13:57:00.280 | INFO | Downloaded 34GiB to GPU in 1.93001s (17.6165GiB/s) | 20260506 13:57:05.418 | INFO | Analyzing chunk frames = [1885696,1886208) | 20260506 13:57:05.418 | INFO | MemInfo: { | 20260506 13:57:05.418 | INFO | "gpu" : " 17 GB", | 20260506 13:57:05.418 | INFO | "huge" : "", | 20260506 13:57:05.418 | INFO | "huge_shared" : " 27 GB", | 20260506 13:57:05.418 | INFO | "malloc" : " 14 GB", | 20260506 13:57:05.418 | INFO | "pinned" : " 29 GB" | 20260506 13:57:05.418 | INFO | } | 20260506 13:57:05.418 | INFO | Perf report for the last 5.13889 seconds: | 20260506 13:57:05.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:57:05.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:05.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:05.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:05.419 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 13:57:05.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:05.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:05.419 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:57:05.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:05.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:05.419 | INFO | Uploaded 36GiB to GPU in 1.97993s (18.1824GiB/s) | 20260506 13:57:05.419 | INFO | Downloaded 34GiB to GPU in 1.91362s (17.7674GiB/s) | 20260506 13:57:10.558 | INFO | Analyzing chunk frames = [1886208,1886720) | 20260506 13:57:10.558 | INFO | MemInfo: { | 20260506 13:57:10.558 | INFO | "gpu" : " 17 GB", | 20260506 13:57:10.558 | INFO | "huge" : "", | 20260506 13:57:10.558 | INFO | "huge_shared" : " 27 GB", | 20260506 13:57:10.558 | INFO | "malloc" : " 14 GB", | 20260506 13:57:10.558 | INFO | "pinned" : " 29 GB" | 20260506 13:57:10.558 | INFO | } | 20260506 13:57:10.558 | INFO | Perf report for the last 5.13943 seconds: | 20260506 13:57:10.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:57:10.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:10.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:10.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:10.558 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 13:57:10.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:10.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:10.558 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:57:10.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:10.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:10.558 | INFO | Uploaded 36GiB to GPU in 1.98526s (18.1336GiB/s) | 20260506 13:57:10.558 | INFO | Downloaded 34GiB to GPU in 1.94145s (17.5127GiB/s) | 20260506 13:57:15.696 | INFO | Analyzing chunk frames = [1886720,1887232) | 20260506 13:57:15.696 | INFO | MemInfo: { | 20260506 13:57:15.696 | INFO | "gpu" : " 17 GB", | 20260506 13:57:15.696 | INFO | "huge" : "", | 20260506 13:57:15.696 | INFO | "huge_shared" : " 27 GB", | 20260506 13:57:15.696 | INFO | "malloc" : " 14 GB", | 20260506 13:57:15.696 | INFO | "pinned" : " 29 GB" | 20260506 13:57:15.697 | INFO | } | 20260506 13:57:15.697 | INFO | Perf report for the last 5.13874 seconds: | 20260506 13:57:15.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:57:15.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:15.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:15.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:15.697 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 13:57:15.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:15.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:15.697 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:57:15.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:15.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:15.697 | INFO | Uploaded 36GiB to GPU in 2.01725s (17.846GiB/s) | 20260506 13:57:15.697 | INFO | Downloaded 34GiB to GPU in 1.92772s (17.6374GiB/s) | 20260506 13:57:20.837 | INFO | Analyzing chunk frames = [1887232,1887744) | 20260506 13:57:20.837 | INFO | MemInfo: { | 20260506 13:57:20.837 | INFO | "gpu" : " 17 GB", | 20260506 13:57:20.837 | INFO | "huge" : "", | 20260506 13:57:20.837 | INFO | "huge_shared" : " 27 GB", | 20260506 13:57:20.837 | INFO | "malloc" : " 14 GB", | 20260506 13:57:20.837 | INFO | "pinned" : " 29 GB" | 20260506 13:57:20.837 | INFO | } | 20260506 13:57:20.837 | INFO | Perf report for the last 5.14059 seconds: | 20260506 13:57:20.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:57:20.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:20.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:20.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:20.837 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 13:57:20.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:20.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:20.837 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 13:57:20.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:20.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:20.838 | INFO | Uploaded 36GiB to GPU in 2.00929s (17.9168GiB/s) | 20260506 13:57:20.838 | INFO | Downloaded 34GiB to GPU in 2.01128s (16.9047GiB/s) | 20260506 13:57:25.975 | INFO | Analyzing chunk frames = [1887744,1888256) | 20260506 13:57:25.975 | INFO | MemInfo: { | 20260506 13:57:25.975 | INFO | "gpu" : " 17 GB", | 20260506 13:57:25.975 | INFO | "huge" : "", | 20260506 13:57:25.975 | INFO | "huge_shared" : " 27 GB", | 20260506 13:57:25.975 | INFO | "malloc" : " 14 GB", | 20260506 13:57:25.975 | INFO | "pinned" : " 29 GB" | 20260506 13:57:25.975 | INFO | } | 20260506 13:57:25.975 | INFO | Perf report for the last 5.13786 seconds: | 20260506 13:57:25.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:57:25.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:25.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:25.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:25.975 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 13:57:25.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:25.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:25.975 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 13:57:25.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:25.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:25.976 | INFO | Uploaded 36GiB to GPU in 2.00965s (17.9136GiB/s) | 20260506 13:57:25.976 | INFO | Downloaded 34GiB to GPU in 1.98022s (17.1698GiB/s) | 20260506 13:57:26.085 | INFO | Starting a round of Baz IO | 20260506 13:57:29.248 | INFO | Populating RTMetrics report | 20260506 13:57:29.248 | INFO | RTMetrics report populated | 20260506 13:57:29.248 | INFO | Opening tmp RTMetrics json file | 20260506 13:57:29.249 | INFO | Serializing RTMetrics to json | 20260506 13:57:29.250 | INFO | Serialization done, swapping tmp file now... | 20260506 13:57:29.250 | INFO | RTMetrics done | 20260506 13:57:29.348 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.262 seconds | 20260506 13:57:29.348 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.95% / 0.02% / 61.03% | 20260506 13:57:29.348 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.82925% / 1.58946e-05% / 4.37101e-05% | 20260506 13:57:29.348 | INFO | Baz IO overall: | 20260506 13:57:29.348 | INFO | BytesWritten : 967451176471 | 20260506 13:57:29.348 | INFO | NumEvents : 468059724286 | 20260506 13:57:31.114 | INFO | Analyzing chunk frames = [1888256,1888768) | 20260506 13:57:31.114 | INFO | MemInfo: { | 20260506 13:57:31.114 | INFO | "gpu" : " 17 GB", | 20260506 13:57:31.114 | INFO | "huge" : "", | 20260506 13:57:31.114 | INFO | "huge_shared" : " 27 GB", | 20260506 13:57:31.114 | INFO | "malloc" : " 14 GB", | 20260506 13:57:31.115 | INFO | "pinned" : " 29 GB" | 20260506 13:57:31.115 | INFO | } | 20260506 13:57:31.115 | INFO | Perf report for the last 5.13953 seconds: | 20260506 13:57:31.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:57:31.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:31.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:31.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:31.115 | INFO | ANALYSIS : 63.1%, 3.0, 97.8ms, 0 | 20260506 13:57:31.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:31.115 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 13:57:31.115 | INFO | PRE_HQ : 61.3%, 1.9, 60.7ms, 0 | 20260506 13:57:31.115 | INFO | BAZWRITER : 62.8%, 3.7, 120.7ms, 0 | 20260506 13:57:31.115 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:57:31.115 | INFO | Uploaded 36GiB to GPU in 2.01467s (17.8689GiB/s) | 20260506 13:57:31.115 | INFO | Downloaded 38GiB to GPU in 2.20244s (17.2536GiB/s) | 20260506 13:57:36.257 | INFO | Analyzing chunk frames = [1888768,1889280) | 20260506 13:57:36.257 | INFO | MemInfo: { | 20260506 13:57:36.257 | INFO | "gpu" : " 17 GB", | 20260506 13:57:36.257 | INFO | "huge" : "", | 20260506 13:57:36.257 | INFO | "huge_shared" : " 27 GB", | 20260506 13:57:36.257 | INFO | "malloc" : " 14 GB", | 20260506 13:57:36.257 | INFO | "pinned" : " 29 GB" | 20260506 13:57:36.257 | INFO | } | 20260506 13:57:36.257 | INFO | Perf report for the last 5.14224 seconds: | 20260506 13:57:36.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:57:36.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:36.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:36.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:36.257 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 13:57:36.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:36.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:36.257 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 13:57:36.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:36.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:36.257 | INFO | Uploaded 36GiB to GPU in 1.97162s (18.2591GiB/s) | 20260506 13:57:36.257 | INFO | Downloaded 34GiB to GPU in 1.94431s (17.487GiB/s) | 20260506 13:57:41.394 | INFO | Analyzing chunk frames = [1889280,1889792) | 20260506 13:57:41.394 | INFO | MemInfo: { | 20260506 13:57:41.394 | INFO | "gpu" : " 17 GB", | 20260506 13:57:41.394 | INFO | "huge" : "", | 20260506 13:57:41.394 | INFO | "huge_shared" : " 27 GB", | 20260506 13:57:41.394 | INFO | "malloc" : " 14 GB", | 20260506 13:57:41.394 | INFO | "pinned" : " 29 GB" | 20260506 13:57:41.394 | INFO | } | 20260506 13:57:41.394 | INFO | Perf report for the last 5.13705 seconds: | 20260506 13:57:41.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:57:41.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:41.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:41.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:41.394 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 13:57:41.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:41.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:41.394 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 13:57:41.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:41.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:41.394 | INFO | Uploaded 36GiB to GPU in 1.96851s (18.2879GiB/s) | 20260506 13:57:41.394 | INFO | Downloaded 34GiB to GPU in 1.94129s (17.5142GiB/s) | 20260506 13:57:46.533 | INFO | Analyzing chunk frames = [1889792,1890304) | 20260506 13:57:46.533 | INFO | MemInfo: { | 20260506 13:57:46.533 | INFO | "gpu" : " 17 GB", | 20260506 13:57:46.533 | INFO | "huge" : "", | 20260506 13:57:46.533 | INFO | "huge_shared" : " 27 GB", | 20260506 13:57:46.533 | INFO | "malloc" : " 14 GB", | 20260506 13:57:46.533 | INFO | "pinned" : " 29 GB" | 20260506 13:57:46.533 | INFO | } | 20260506 13:57:46.533 | INFO | Perf report for the last 5.13898 seconds: | 20260506 13:57:46.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:57:46.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:46.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:46.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:46.533 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 13:57:46.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:46.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:46.533 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:57:46.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:46.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:46.533 | INFO | Uploaded 36GiB to GPU in 1.97478s (18.2299GiB/s) | 20260506 13:57:46.533 | INFO | Downloaded 34GiB to GPU in 1.91755s (17.7309GiB/s) | 20260506 13:57:51.672 | INFO | Analyzing chunk frames = [1890304,1890816) | 20260506 13:57:51.672 | INFO | MemInfo: { | 20260506 13:57:51.672 | INFO | "gpu" : " 17 GB", | 20260506 13:57:51.672 | INFO | "huge" : "", | 20260506 13:57:51.672 | INFO | "huge_shared" : " 27 GB", | 20260506 13:57:51.672 | INFO | "malloc" : " 14 GB", | 20260506 13:57:51.672 | INFO | "pinned" : " 29 GB" | 20260506 13:57:51.673 | INFO | } | 20260506 13:57:51.673 | INFO | Perf report for the last 5.13963 seconds: | 20260506 13:57:51.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:57:51.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:51.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:51.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:51.673 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 13:57:51.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:51.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:51.673 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:57:51.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:51.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:51.673 | INFO | Uploaded 36GiB to GPU in 2.01334s (17.8807GiB/s) | 20260506 13:57:51.673 | INFO | Downloaded 34GiB to GPU in 1.94492s (17.4814GiB/s) | 20260506 13:57:56.811 | INFO | Analyzing chunk frames = [1890816,1891328) | 20260506 13:57:56.811 | INFO | MemInfo: { | 20260506 13:57:56.811 | INFO | "gpu" : " 17 GB", | 20260506 13:57:56.811 | INFO | "huge" : "", | 20260506 13:57:56.811 | INFO | "huge_shared" : " 27 GB", | 20260506 13:57:56.811 | INFO | "malloc" : " 14 GB", | 20260506 13:57:56.811 | INFO | "pinned" : " 29 GB" | 20260506 13:57:56.811 | INFO | } | 20260506 13:57:56.811 | INFO | Perf report for the last 5.13856 seconds: | 20260506 13:57:56.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:57:56.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:56.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:56.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:56.811 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 13:57:56.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:56.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:56.811 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 13:57:56.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:56.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:57:56.812 | INFO | Uploaded 36GiB to GPU in 2.04007s (17.6465GiB/s) | 20260506 13:57:56.812 | INFO | Downloaded 34GiB to GPU in 1.91434s (17.7607GiB/s) | 20260506 13:58:01.951 | INFO | Analyzing chunk frames = [1891328,1891840) | 20260506 13:58:01.951 | INFO | MemInfo: { | 20260506 13:58:01.951 | INFO | "gpu" : " 17 GB", | 20260506 13:58:01.951 | INFO | "huge" : "", | 20260506 13:58:01.951 | INFO | "huge_shared" : " 27 GB", | 20260506 13:58:01.951 | INFO | "malloc" : " 14 GB", | 20260506 13:58:01.951 | INFO | "pinned" : " 29 GB" | 20260506 13:58:01.951 | INFO | } | 20260506 13:58:01.951 | INFO | Perf report for the last 5.13978 seconds: | 20260506 13:58:01.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:58:01.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:01.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:01.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:01.952 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:58:01.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:01.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:01.952 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 13:58:01.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:01.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:01.952 | INFO | Uploaded 36GiB to GPU in 2.00014s (17.9988GiB/s) | 20260506 13:58:01.952 | INFO | Downloaded 34GiB to GPU in 1.98177s (17.1563GiB/s) | 20260506 13:58:07.091 | INFO | Analyzing chunk frames = [1891840,1892352) | 20260506 13:58:07.091 | INFO | MemInfo: { | 20260506 13:58:07.091 | INFO | "gpu" : " 17 GB", | 20260506 13:58:07.091 | INFO | "huge" : "", | 20260506 13:58:07.091 | INFO | "huge_shared" : " 27 GB", | 20260506 13:58:07.091 | INFO | "malloc" : " 14 GB", | 20260506 13:58:07.091 | INFO | "pinned" : " 29 GB" | 20260506 13:58:07.091 | INFO | } | 20260506 13:58:07.091 | INFO | Perf report for the last 5.14037 seconds: | 20260506 13:58:07.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:58:07.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:07.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:07.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:07.091 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 13:58:07.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:07.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:07.091 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 13:58:07.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:07.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:07.092 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:58:07.092 | INFO | Total mem usage: 17.4 GiB | 20260506 13:58:07.092 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:58:07.092 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -21%) | 20260506 13:58:07.092 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:58:07.092 | INFO | High Water Breakdown: | 20260506 13:58:07.092 | INFO | 59.8% : 8.6 GiB ( 78.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:58:07.092 | INFO | 40.1% : 5.77 GiB ( 86.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:58:07.092 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:58:07.092 | INFO | ------------------------------------------ | 20260506 13:58:07.092 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:58:07.092 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:58:07.092 | INFO | Total mem usage: 26.5 GiB | 20260506 13:58:07.092 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:58:07.092 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:58:07.092 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:58:07.092 | INFO | High Water Breakdown: | 20260506 13:58:07.092 | INFO | Uploaded 36GiB to GPU in 2.0058s (17.948GiB/s) | 20260506 13:58:07.092 | INFO | Downloaded 34GiB to GPU in 1.97562s (17.2098GiB/s) | 20260506 13:58:07.092 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:58:07.092 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:58:07.092 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:58:07.092 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:58:07.092 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:58:07.092 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:58:07.092 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:58:07.092 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:58:07.092 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:58:07.092 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:58:07.092 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:58:07.092 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:58:07.092 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:58:07.092 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:58:07.092 | INFO | ------------------------------------------ | 20260506 13:58:07.092 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:58:07.092 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:58:07.092 | INFO | Total mem usage: 25.6 GiB | 20260506 13:58:07.092 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:58:07.092 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:58:07.092 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:58:07.092 | INFO | High Water Breakdown: | 20260506 13:58:07.092 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:58:07.092 | INFO | ------------------------------------------ | 20260506 13:58:07.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:58:07.093 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:58:07.093 | INFO | Total mem usage: 15.7 GiB | 20260506 13:58:07.093 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:58:07.093 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:58:07.093 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 13:58:07.093 | INFO | High Water Breakdown: | 20260506 13:58:07.093 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:58:07.093 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:58:07.093 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:58:07.093 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:58:07.093 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:58:07.093 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:58:07.093 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:58:07.093 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:58:07.093 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:58:07.093 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:58:07.093 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:58:07.093 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:58:07.093 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:58:07.093 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:58:07.093 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:58:07.093 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:58:07.093 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:58:07.093 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:58:07.093 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:58:07.093 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:58:07.093 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:58:07.093 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:58:07.093 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:58:07.093 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:58:07.093 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:58:07.093 | INFO | ------------------------------------------ | 20260506 13:58:07.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:58:07.093 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:58:07.093 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:58:07.093 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:58:07.093 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.56e+01, 1.34e+01 / 2.22e+00 | 20260506 13:58:07.093 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.01e-01, 1.13e-02 / 5.06e-03 | 20260506 13:58:07.093 | INFO | PerfReport: 11.59%, Baseline -- 3.66e+00 / 4.44e+00, 5.81e-01 / 3.80e-01 | 20260506 13:58:07.093 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.47e+00, 1.89e-01 / 1.30e-01 | 20260506 13:58:07.093 | INFO | PerfReport: 15.76%, DME -- 3.98e+01 / 4.38e+01, 3.82e+00 / 2.17e+00 | 20260506 13:58:07.093 | INFO | PerfReport: 56.76%, FrameLabeling -- 1.79e+01 / 2.09e+01, 1.54e+00 / 1.83e+00 | 20260506 13:58:07.093 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.35e+00 / 2.92e+00, 4.03e-01 / 2.57e-01 | 20260506 13:58:07.093 | INFO | PerfReport: 3.63%, Metrics -- 1.15e+00 / 1.53e+00, 2.95e-01 / 1.83e-01 | 20260506 13:58:07.093 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.01e-01 / 4.53e-01, 6.27e-02 / 2.42e-02 | 20260506 13:58:07.093 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:58:07.093 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:58:07.093 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:58:07.093 | INFO | PerfReport: Full Execution -- 9.81e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 13:58:07.093 | INFO | PerfReport: 21.22%, Upload -- 2.08e+01 / 0.00e+00, 6.84e+00 / 0.00e+00 | 20260506 13:58:07.093 | INFO | PerfReport: 21.04%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 13:58:07.093 | INFO | PerfReport: 32.23%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 13:58:07.093 | INFO | PerfReport: 1.43%, LockUpload -- 1.40e+00 / 0.00e+00, 6.20e+00 / 0.00e+00 | 20260506 13:58:07.093 | INFO | PerfReport: 1.25%, LockDownload -- 1.23e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260506 13:58:07.093 | INFO | PerfReport: 22.81%, LockCompute -- 2.24e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 13:58:07.093 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.55e-02 / 0.00e+00 | 20260506 13:58:07.205 | INFO | Starting a round of Baz IO | 20260506 13:58:10.358 | INFO | Populating RTMetrics report | 20260506 13:58:10.358 | INFO | RTMetrics report populated | 20260506 13:58:10.358 | INFO | Opening tmp RTMetrics json file | 20260506 13:58:10.359 | INFO | Serializing RTMetrics to json | 20260506 13:58:10.360 | INFO | Serialization done, swapping tmp file now... | 20260506 13:58:10.361 | INFO | RTMetrics done | 20260506 13:58:10.462 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.256 seconds | 20260506 13:58:10.462 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.93% / 0.02% / 61.05% | 20260506 13:58:10.462 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.8265% / 2.38419e-05% / 4.76837e-05% | 20260506 13:58:10.462 | INFO | Baz IO overall: | 20260506 13:58:10.462 | INFO | BytesWritten : 969718847196 | 20260506 13:58:10.462 | INFO | NumEvents : 469159448795 | 20260506 13:58:12.230 | INFO | Analyzing chunk frames = [1892352,1892864) | 20260506 13:58:12.230 | INFO | MemInfo: { | 20260506 13:58:12.230 | INFO | "gpu" : " 17 GB", | 20260506 13:58:12.230 | INFO | "huge" : "", | 20260506 13:58:12.230 | INFO | "huge_shared" : " 27 GB", | 20260506 13:58:12.230 | INFO | "malloc" : " 14 GB", | 20260506 13:58:12.230 | INFO | "pinned" : " 29 GB" | 20260506 13:58:12.230 | INFO | } | 20260506 13:58:12.230 | INFO | Perf report for the last 5.13847 seconds: | 20260506 13:58:12.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:58:12.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:12.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:12.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:12.230 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260506 13:58:12.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:12.230 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260506 13:58:12.230 | INFO | PRE_HQ : 61.1%, 1.9, 60.3ms, 0 | 20260506 13:58:12.230 | INFO | BAZWRITER : 62.7%, 3.7, 120.6ms, 0 | 20260506 13:58:12.230 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 13:58:12.230 | INFO | Uploaded 36GiB to GPU in 2.03286s (17.709GiB/s) | 20260506 13:58:12.230 | INFO | Downloaded 38GiB to GPU in 2.20583s (17.2271GiB/s) | 20260506 13:58:17.369 | INFO | Analyzing chunk frames = [1892864,1893376) | 20260506 13:58:17.369 | INFO | MemInfo: { | 20260506 13:58:17.369 | INFO | "gpu" : " 17 GB", | 20260506 13:58:17.369 | INFO | "huge" : "", | 20260506 13:58:17.369 | INFO | "huge_shared" : " 27 GB", | 20260506 13:58:17.369 | INFO | "malloc" : " 14 GB", | 20260506 13:58:17.369 | INFO | "pinned" : " 29 GB" | 20260506 13:58:17.369 | INFO | } | 20260506 13:58:17.369 | INFO | Perf report for the last 5.13886 seconds: | 20260506 13:58:17.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:58:17.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:17.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:17.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:17.369 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 13:58:17.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:17.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:17.369 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 13:58:17.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:17.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:17.369 | INFO | Uploaded 36GiB to GPU in 2.02086s (17.8142GiB/s) | 20260506 13:58:17.369 | INFO | Downloaded 34GiB to GPU in 1.94643s (17.4679GiB/s) | 20260506 13:58:22.509 | INFO | Analyzing chunk frames = [1893376,1893888) | 20260506 13:58:22.509 | INFO | MemInfo: { | 20260506 13:58:22.509 | INFO | "gpu" : " 17 GB", | 20260506 13:58:22.509 | INFO | "huge" : "", | 20260506 13:58:22.509 | INFO | "huge_shared" : " 27 GB", | 20260506 13:58:22.509 | INFO | "malloc" : " 14 GB", | 20260506 13:58:22.509 | INFO | "pinned" : " 29 GB" | 20260506 13:58:22.509 | INFO | } | 20260506 13:58:22.509 | INFO | Perf report for the last 5.14006 seconds: | 20260506 13:58:22.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:58:22.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:22.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:22.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:22.509 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 13:58:22.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:22.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:22.509 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 13:58:22.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:22.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:22.509 | INFO | Uploaded 36GiB to GPU in 1.98474s (18.1384GiB/s) | 20260506 13:58:22.509 | INFO | Downloaded 34GiB to GPU in 1.93486s (17.5723GiB/s) | 20260506 13:58:27.648 | INFO | Analyzing chunk frames = [1893888,1894400) | 20260506 13:58:27.648 | INFO | MemInfo: { | 20260506 13:58:27.648 | INFO | "gpu" : " 17 GB", | 20260506 13:58:27.648 | INFO | "huge" : "", | 20260506 13:58:27.648 | INFO | "huge_shared" : " 27 GB", | 20260506 13:58:27.648 | INFO | "malloc" : " 14 GB", | 20260506 13:58:27.648 | INFO | "pinned" : " 29 GB" | 20260506 13:58:27.648 | INFO | } | 20260506 13:58:27.648 | INFO | Perf report for the last 5.13946 seconds: | 20260506 13:58:27.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:58:27.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:27.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:27.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:27.648 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 13:58:27.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:27.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:27.648 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 13:58:27.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:27.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:27.648 | INFO | Uploaded 36GiB to GPU in 1.99423s (18.0521GiB/s) | 20260506 13:58:27.649 | INFO | Downloaded 34GiB to GPU in 1.92772s (17.6374GiB/s) | 20260506 13:58:32.788 | INFO | Analyzing chunk frames = [1894400,1894912) | 20260506 13:58:32.788 | INFO | MemInfo: { | 20260506 13:58:32.788 | INFO | "gpu" : " 17 GB", | 20260506 13:58:32.788 | INFO | "huge" : "", | 20260506 13:58:32.788 | INFO | "huge_shared" : " 27 GB", | 20260506 13:58:32.788 | INFO | "malloc" : " 14 GB", | 20260506 13:58:32.788 | INFO | "pinned" : " 29 GB" | 20260506 13:58:32.789 | INFO | } | 20260506 13:58:32.789 | INFO | Perf report for the last 5.14065 seconds: | 20260506 13:58:32.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:58:32.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:32.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:32.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:32.789 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 13:58:32.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:32.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:32.789 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:58:32.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:32.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:32.789 | INFO | Uploaded 36GiB to GPU in 1.9772s (18.2076GiB/s) | 20260506 13:58:32.789 | INFO | Downloaded 34GiB to GPU in 1.9362s (17.5602GiB/s) | 20260506 13:58:37.927 | INFO | Analyzing chunk frames = [1894912,1895424) | 20260506 13:58:37.927 | INFO | MemInfo: { | 20260506 13:58:37.927 | INFO | "gpu" : " 17 GB", | 20260506 13:58:37.927 | INFO | "huge" : "", | 20260506 13:58:37.927 | INFO | "huge_shared" : " 27 GB", | 20260506 13:58:37.927 | INFO | "malloc" : " 14 GB", | 20260506 13:58:37.927 | INFO | "pinned" : " 29 GB" | 20260506 13:58:37.927 | INFO | } | 20260506 13:58:37.927 | INFO | Perf report for the last 5.13788 seconds: | 20260506 13:58:37.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:58:37.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:37.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:37.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:37.927 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 13:58:37.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:37.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:37.927 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 13:58:37.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:37.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:37.927 | INFO | Uploaded 36GiB to GPU in 2.00783s (17.9298GiB/s) | 20260506 13:58:37.927 | INFO | Downloaded 34GiB to GPU in 1.92367s (17.6746GiB/s) | 20260506 13:58:43.067 | INFO | Analyzing chunk frames = [1895424,1895936) | 20260506 13:58:43.067 | INFO | MemInfo: { | 20260506 13:58:43.067 | INFO | "gpu" : " 17 GB", | 20260506 13:58:43.067 | INFO | "huge" : "", | 20260506 13:58:43.067 | INFO | "huge_shared" : " 27 GB", | 20260506 13:58:43.067 | INFO | "malloc" : " 14 GB", | 20260506 13:58:43.067 | INFO | "pinned" : " 29 GB" | 20260506 13:58:43.067 | INFO | } | 20260506 13:58:43.067 | INFO | Perf report for the last 5.14027 seconds: | 20260506 13:58:43.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:58:43.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:43.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:43.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:43.067 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 13:58:43.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:43.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:43.067 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 13:58:43.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:43.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:43.068 | INFO | Uploaded 36GiB to GPU in 2.00042s (17.9962GiB/s) | 20260506 13:58:43.068 | INFO | Downloaded 34GiB to GPU in 1.98555s (17.1237GiB/s) | 20260506 13:58:48.205 | INFO | Analyzing chunk frames = [1895936,1896448) | 20260506 13:58:48.206 | INFO | MemInfo: { | 20260506 13:58:48.206 | INFO | "gpu" : " 17 GB", | 20260506 13:58:48.206 | INFO | "huge" : "", | 20260506 13:58:48.206 | INFO | "huge_shared" : " 27 GB", | 20260506 13:58:48.206 | INFO | "malloc" : " 14 GB", | 20260506 13:58:48.206 | INFO | "pinned" : " 29 GB" | 20260506 13:58:48.206 | INFO | } | 20260506 13:58:48.206 | INFO | Perf report for the last 5.13863 seconds: | 20260506 13:58:48.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:58:48.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:48.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:48.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:48.206 | INFO | ANALYSIS : 62.5%, 3.0, 97.5ms, 0 | 20260506 13:58:48.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:48.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:48.206 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260506 13:58:48.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:48.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:48.206 | INFO | Uploaded 36GiB to GPU in 2.06203s (17.4586GiB/s) | 20260506 13:58:48.206 | INFO | Downloaded 34GiB to GPU in 1.97389s (17.2249GiB/s) | 20260506 13:58:48.319 | INFO | Starting a round of Baz IO | 20260506 13:58:51.473 | INFO | Populating RTMetrics report | 20260506 13:58:51.473 | INFO | RTMetrics report populated | 20260506 13:58:51.473 | INFO | Opening tmp RTMetrics json file | 20260506 13:58:51.474 | INFO | Serializing RTMetrics to json | 20260506 13:58:51.475 | INFO | Serialization done, swapping tmp file now... | 20260506 13:58:51.476 | INFO | RTMetrics done | 20260506 13:58:51.574 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.254 seconds | 20260506 13:58:51.574 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.92% / 0.02% / 61.07% | 20260506 13:58:51.574 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.81845% / 1.19209e-05% / 3.97364e-05% | 20260506 13:58:51.574 | INFO | Baz IO overall: | 20260506 13:58:51.574 | INFO | BytesWritten : 971986560164 | 20260506 13:58:51.574 | INFO | NumEvents : 470259250247 | 20260506 13:58:53.345 | INFO | Analyzing chunk frames = [1896448,1896960) | 20260506 13:58:53.345 | INFO | MemInfo: { | 20260506 13:58:53.345 | INFO | "gpu" : " 17 GB", | 20260506 13:58:53.345 | INFO | "huge" : "", | 20260506 13:58:53.345 | INFO | "huge_shared" : " 27 GB", | 20260506 13:58:53.346 | INFO | "malloc" : " 14 GB", | 20260506 13:58:53.346 | INFO | "pinned" : " 29 GB" | 20260506 13:58:53.346 | INFO | } | 20260506 13:58:53.346 | INFO | Perf report for the last 5.13985 seconds: | 20260506 13:58:53.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:58:53.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:53.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:53.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:53.346 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260506 13:58:53.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:53.346 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 13:58:53.346 | INFO | PRE_HQ : 61.2%, 1.9, 60.6ms, 0 | 20260506 13:58:53.346 | INFO | BAZWRITER : 62.7%, 3.7, 118.9ms, 0 | 20260506 13:58:53.346 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 13:58:53.346 | INFO | Uploaded 36GiB to GPU in 2.04391s (17.6133GiB/s) | 20260506 13:58:53.346 | INFO | Downloaded 38GiB to GPU in 2.20011s (17.2719GiB/s) | 20260506 13:58:58.484 | INFO | Analyzing chunk frames = [1896960,1897472) | 20260506 13:58:58.484 | INFO | MemInfo: { | 20260506 13:58:58.484 | INFO | "gpu" : " 17 GB", | 20260506 13:58:58.484 | INFO | "huge" : "", | 20260506 13:58:58.484 | INFO | "huge_shared" : " 27 GB", | 20260506 13:58:58.484 | INFO | "malloc" : " 14 GB", | 20260506 13:58:58.484 | INFO | "pinned" : " 29 GB" | 20260506 13:58:58.484 | INFO | } | 20260506 13:58:58.484 | INFO | Perf report for the last 5.13801 seconds: | 20260506 13:58:58.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:58:58.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:58.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:58.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:58.484 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 13:58:58.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:58.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:58.484 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 13:58:58.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:58.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:58:58.484 | INFO | Uploaded 36GiB to GPU in 2.03658s (17.6767GiB/s) | 20260506 13:58:58.484 | INFO | Downloaded 34GiB to GPU in 1.94824s (17.4517GiB/s) | 20260506 13:59:03.624 | INFO | Analyzing chunk frames = [1897472,1897984) | 20260506 13:59:03.624 | INFO | MemInfo: { | 20260506 13:59:03.624 | INFO | "gpu" : " 17 GB", | 20260506 13:59:03.624 | INFO | "huge" : "", | 20260506 13:59:03.624 | INFO | "huge_shared" : " 27 GB", | 20260506 13:59:03.624 | INFO | "malloc" : " 14 GB", | 20260506 13:59:03.624 | INFO | "pinned" : " 29 GB" | 20260506 13:59:03.624 | INFO | } | 20260506 13:59:03.624 | INFO | Perf report for the last 5.14009 seconds: | 20260506 13:59:03.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:59:03.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:03.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:03.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:03.624 | INFO | ANALYSIS : 61.5%, 3.0, 95.2ms, 0 | 20260506 13:59:03.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:03.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:03.624 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:59:03.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:03.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:03.624 | INFO | Uploaded 36GiB to GPU in 2.01815s (17.8381GiB/s) | 20260506 13:59:03.624 | INFO | Downloaded 34GiB to GPU in 1.9358s (17.5638GiB/s) | 20260506 13:59:08.763 | INFO | Analyzing chunk frames = [1897984,1898496) | 20260506 13:59:08.763 | INFO | MemInfo: { | 20260506 13:59:08.763 | INFO | "gpu" : " 17 GB", | 20260506 13:59:08.763 | INFO | "huge" : "", | 20260506 13:59:08.763 | INFO | "huge_shared" : " 27 GB", | 20260506 13:59:08.763 | INFO | "malloc" : " 14 GB", | 20260506 13:59:08.763 | INFO | "pinned" : " 29 GB" | 20260506 13:59:08.763 | INFO | } | 20260506 13:59:08.763 | INFO | Perf report for the last 5.13899 seconds: | 20260506 13:59:08.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:59:08.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:08.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:08.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:08.763 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 13:59:08.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:08.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:08.763 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:59:08.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:08.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:08.763 | INFO | Uploaded 36GiB to GPU in 1.9999s (18.0009GiB/s) | 20260506 13:59:08.763 | INFO | Downloaded 34GiB to GPU in 1.91808s (17.7261GiB/s) | 20260506 13:59:13.902 | INFO | Analyzing chunk frames = [1898496,1899008) | 20260506 13:59:13.903 | INFO | MemInfo: { | 20260506 13:59:13.903 | INFO | "gpu" : " 17 GB", | 20260506 13:59:13.903 | INFO | "huge" : "", | 20260506 13:59:13.903 | INFO | "huge_shared" : " 27 GB", | 20260506 13:59:13.903 | INFO | "malloc" : " 14 GB", | 20260506 13:59:13.903 | INFO | "pinned" : " 29 GB" | 20260506 13:59:13.903 | INFO | } | 20260506 13:59:13.903 | INFO | Perf report for the last 5.13979 seconds: | 20260506 13:59:13.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:59:13.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:13.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:13.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:13.903 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 13:59:13.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:13.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:13.903 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:59:13.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:13.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:13.903 | INFO | Uploaded 36GiB to GPU in 2.00359s (17.9677GiB/s) | 20260506 13:59:13.903 | INFO | Downloaded 34GiB to GPU in 1.93794s (17.5444GiB/s) | 20260506 13:59:19.041 | INFO | Analyzing chunk frames = [1899008,1899520) | 20260506 13:59:19.041 | INFO | MemInfo: { | 20260506 13:59:19.041 | INFO | "gpu" : " 17 GB", | 20260506 13:59:19.041 | INFO | "huge" : "", | 20260506 13:59:19.041 | INFO | "huge_shared" : " 27 GB", | 20260506 13:59:19.041 | INFO | "malloc" : " 14 GB", | 20260506 13:59:19.041 | INFO | "pinned" : " 29 GB" | 20260506 13:59:19.042 | INFO | } | 20260506 13:59:19.042 | INFO | Perf report for the last 5.13903 seconds: | 20260506 13:59:19.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:59:19.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:19.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:19.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:19.042 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 13:59:19.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:19.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:19.042 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 13:59:19.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:19.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:19.042 | INFO | Uploaded 36GiB to GPU in 2.05454s (17.5221GiB/s) | 20260506 13:59:19.042 | INFO | Downloaded 34GiB to GPU in 1.93231s (17.5955GiB/s) | 20260506 13:59:24.181 | INFO | Analyzing chunk frames = [1899520,1900032) | 20260506 13:59:24.181 | INFO | MemInfo: { | 20260506 13:59:24.181 | INFO | "gpu" : " 17 GB", | 20260506 13:59:24.181 | INFO | "huge" : "", | 20260506 13:59:24.181 | INFO | "huge_shared" : " 27 GB", | 20260506 13:59:24.181 | INFO | "malloc" : " 14 GB", | 20260506 13:59:24.181 | INFO | "pinned" : " 29 GB" | 20260506 13:59:24.181 | INFO | } | 20260506 13:59:24.181 | INFO | Perf report for the last 5.13951 seconds: | 20260506 13:59:24.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:59:24.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:24.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:24.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:24.181 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 13:59:24.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:24.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:24.181 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260506 13:59:24.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:24.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:24.181 | INFO | Uploaded 36GiB to GPU in 2.01313s (17.8826GiB/s) | 20260506 13:59:24.181 | INFO | Downloaded 34GiB to GPU in 2.00271s (16.977GiB/s) | 20260506 13:59:29.320 | INFO | Analyzing chunk frames = [1900032,1900544) | 20260506 13:59:29.320 | INFO | MemInfo: { | 20260506 13:59:29.320 | INFO | "gpu" : " 17 GB", | 20260506 13:59:29.321 | INFO | "huge" : "", | 20260506 13:59:29.321 | INFO | "huge_shared" : " 27 GB", | 20260506 13:59:29.321 | INFO | "malloc" : " 14 GB", | 20260506 13:59:29.321 | INFO | "pinned" : " 29 GB" | 20260506 13:59:29.321 | INFO | } | 20260506 13:59:29.321 | INFO | Perf report for the last 5.13962 seconds: | 20260506 13:59:29.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:59:29.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:29.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:29.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:29.321 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 13:59:29.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:29.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:29.321 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 13:59:29.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:29.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:29.321 | INFO | Uploaded 36GiB to GPU in 2.04702s (17.5865GiB/s) | 20260506 13:59:29.321 | INFO | Downloaded 34GiB to GPU in 1.97038s (17.2555GiB/s) | 20260506 13:59:29.321 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 13:59:29.321 | INFO | Total mem usage: 17.4 GiB | 20260506 13:59:29.321 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 13:59:29.321 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -20.5%) | 20260506 13:59:29.321 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 13:59:29.321 | INFO | High Water Breakdown: | 20260506 13:59:29.321 | INFO | 59.9% : 8.65 GiB ( 79.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 13:59:29.321 | INFO | 40% : 5.78 GiB ( 86.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 13:59:29.321 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 13:59:29.321 | INFO | ------------------------------------------ | 20260506 13:59:29.321 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:59:29.322 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 13:59:29.322 | INFO | Total mem usage: 26.5 GiB | 20260506 13:59:29.322 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 13:59:29.322 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 13:59:29.322 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 13:59:29.322 | INFO | High Water Breakdown: | 20260506 13:59:29.322 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:59:29.322 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:59:29.322 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:59:29.322 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:59:29.322 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:59:29.322 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:59:29.322 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 13:59:29.322 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 13:59:29.322 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:59:29.322 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 13:59:29.322 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 13:59:29.322 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 13:59:29.322 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 13:59:29.322 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 13:59:29.322 | INFO | ------------------------------------------ | 20260506 13:59:29.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:59:29.322 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 13:59:29.322 | INFO | Total mem usage: 25.6 GiB | 20260506 13:59:29.322 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 13:59:29.322 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 13:59:29.322 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 13:59:29.322 | INFO | High Water Breakdown: | 20260506 13:59:29.322 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 13:59:29.322 | INFO | ------------------------------------------ | 20260506 13:59:29.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:59:29.322 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 13:59:29.322 | INFO | Total mem usage: 15.7 GiB | 20260506 13:59:29.322 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 13:59:29.322 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 13:59:29.322 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 13:59:29.322 | INFO | High Water Breakdown: | 20260506 13:59:29.322 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 13:59:29.322 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 13:59:29.322 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 13:59:29.322 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 13:59:29.322 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 13:59:29.322 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 13:59:29.322 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 13:59:29.322 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 13:59:29.322 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 13:59:29.322 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 13:59:29.322 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 13:59:29.322 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 13:59:29.322 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 13:59:29.322 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 13:59:29.322 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 13:59:29.322 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 13:59:29.322 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 13:59:29.322 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 13:59:29.322 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 13:59:29.322 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 13:59:29.322 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 13:59:29.322 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 13:59:29.322 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 13:59:29.322 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 13:59:29.322 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 13:59:29.322 | INFO | ------------------------------------------ | 20260506 13:59:29.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 13:59:29.322 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 13:59:29.322 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:59:29.322 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:59:29.322 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.60e+01, 1.34e+01 / 2.05e+00 | 20260506 13:59:29.322 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.03e-01, 9.98e-03 / 6.85e-03 | 20260506 13:59:29.322 | INFO | PerfReport: 11.57%, Baseline -- 3.65e+00 / 4.49e+00, 5.60e-01 / 2.59e-01 | 20260506 13:59:29.322 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.51e+00, 1.89e-01 / 1.04e-01 | 20260506 13:59:29.322 | INFO | PerfReport: 15.84%, DME -- 4.00e+01 / 4.47e+01, 3.99e+00 / 2.51e+00 | 20260506 13:59:29.322 | INFO | PerfReport: 56.68%, FrameLabeling -- 1.79e+01 / 2.02e+01, 1.46e+00 / 1.91e+00 | 20260506 13:59:29.322 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.36e+00 / 3.01e+00, 4.07e-01 / 2.33e-01 | 20260506 13:59:29.322 | INFO | PerfReport: 3.63%, Metrics -- 1.15e+00 / 1.51e+00, 2.87e-01 / 1.71e-01 | 20260506 13:59:29.322 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.01e-01 / 4.62e-01, 6.16e-02 / 2.55e-02 | 20260506 13:59:29.322 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 13:59:29.322 | INFO | PerfReport: Gathered 1536 samples. | 20260506 13:59:29.322 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 13:59:29.322 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 13:59:29.322 | INFO | PerfReport: 21.41%, Upload -- 2.10e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260506 13:59:29.322 | INFO | PerfReport: 21.00%, Download -- 2.06e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260506 13:59:29.322 | INFO | PerfReport: 32.15%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 13:59:29.322 | INFO | PerfReport: 1.63%, LockUpload -- 1.60e+00 / 0.00e+00, 6.50e+00 / 0.00e+00 | 20260506 13:59:29.322 | INFO | PerfReport: 1.09%, LockDownload -- 1.07e+00 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260506 13:59:29.322 | INFO | PerfReport: 22.71%, LockCompute -- 2.23e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 13:59:29.323 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 3.00e-02 / 0.00e+00 | 20260506 13:59:29.444 | INFO | Starting a round of Baz IO | 20260506 13:59:32.597 | INFO | Populating RTMetrics report | 20260506 13:59:32.597 | INFO | RTMetrics report populated | 20260506 13:59:32.597 | INFO | Opening tmp RTMetrics json file | 20260506 13:59:32.598 | INFO | Serializing RTMetrics to json | 20260506 13:59:32.599 | INFO | Serialization done, swapping tmp file now... | 20260506 13:59:32.600 | INFO | RTMetrics done | 20260506 13:59:32.697 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.252 seconds | 20260506 13:59:32.697 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.90% / 0.02% / 61.08% | 20260506 13:59:32.697 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.80409% / 3.17891e-05% / 4.37101e-05% | 20260506 13:59:32.697 | INFO | Baz IO overall: | 20260506 13:59:32.697 | INFO | BytesWritten : 974252579757 | 20260506 13:59:32.697 | INFO | NumEvents : 471357668799 | 20260506 13:59:34.459 | INFO | Analyzing chunk frames = [1900544,1901056) | 20260506 13:59:34.459 | INFO | MemInfo: { | 20260506 13:59:34.459 | INFO | "gpu" : " 17 GB", | 20260506 13:59:34.459 | INFO | "huge" : "", | 20260506 13:59:34.459 | INFO | "huge_shared" : " 27 GB", | 20260506 13:59:34.459 | INFO | "malloc" : " 14 GB", | 20260506 13:59:34.459 | INFO | "pinned" : " 29 GB" | 20260506 13:59:34.459 | INFO | } | 20260506 13:59:34.460 | INFO | Perf report for the last 5.13874 seconds: | 20260506 13:59:34.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:59:34.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:34.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:34.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:34.460 | INFO | ANALYSIS : 63.2%, 3.0, 98.0ms, 0 | 20260506 13:59:34.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:34.460 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 13:59:34.460 | INFO | PRE_HQ : 60.9%, 1.9, 60.5ms, 0 | 20260506 13:59:34.460 | INFO | BAZWRITER : 62.6%, 3.7, 118.3ms, 0 | 20260506 13:59:34.460 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 13:59:34.460 | INFO | Uploaded 36GiB to GPU in 2.03266s (17.7108GiB/s) | 20260506 13:59:34.460 | INFO | Downloaded 38GiB to GPU in 2.21917s (17.1235GiB/s) | 20260506 13:59:39.600 | INFO | Analyzing chunk frames = [1901056,1901568) | 20260506 13:59:39.600 | INFO | MemInfo: { | 20260506 13:59:39.600 | INFO | "gpu" : " 17 GB", | 20260506 13:59:39.601 | INFO | "huge" : "", | 20260506 13:59:39.601 | INFO | "huge_shared" : " 27 GB", | 20260506 13:59:39.601 | INFO | "malloc" : " 14 GB", | 20260506 13:59:39.601 | INFO | "pinned" : " 29 GB" | 20260506 13:59:39.601 | INFO | } | 20260506 13:59:39.601 | INFO | Perf report for the last 5.14119 seconds: | 20260506 13:59:39.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:59:39.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:39.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:39.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:39.601 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 13:59:39.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:39.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:39.601 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 13:59:39.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:39.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:39.601 | INFO | Uploaded 36GiB to GPU in 1.97971s (18.1845GiB/s) | 20260506 13:59:39.601 | INFO | Downloaded 34GiB to GPU in 1.94596s (17.4721GiB/s) | 20260506 13:59:44.739 | INFO | Analyzing chunk frames = [1901568,1902080) | 20260506 13:59:44.739 | INFO | MemInfo: { | 20260506 13:59:44.739 | INFO | "gpu" : " 17 GB", | 20260506 13:59:44.739 | INFO | "huge" : "", | 20260506 13:59:44.739 | INFO | "huge_shared" : " 27 GB", | 20260506 13:59:44.739 | INFO | "malloc" : " 14 GB", | 20260506 13:59:44.739 | INFO | "pinned" : " 29 GB" | 20260506 13:59:44.739 | INFO | } | 20260506 13:59:44.739 | INFO | Perf report for the last 5.13827 seconds: | 20260506 13:59:44.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:59:44.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:44.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:44.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:44.739 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 13:59:44.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:44.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:44.739 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:59:44.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:44.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:44.739 | INFO | Uploaded 36GiB to GPU in 1.97456s (18.2319GiB/s) | 20260506 13:59:44.739 | INFO | Downloaded 34GiB to GPU in 1.94514s (17.4794GiB/s) | 20260506 13:59:49.878 | INFO | Analyzing chunk frames = [1902080,1902592) | 20260506 13:59:49.878 | INFO | MemInfo: { | 20260506 13:59:49.878 | INFO | "gpu" : " 17 GB", | 20260506 13:59:49.878 | INFO | "huge" : "", | 20260506 13:59:49.878 | INFO | "huge_shared" : " 27 GB", | 20260506 13:59:49.878 | INFO | "malloc" : " 14 GB", | 20260506 13:59:49.878 | INFO | "pinned" : " 29 GB" | 20260506 13:59:49.878 | INFO | } | 20260506 13:59:49.878 | INFO | Perf report for the last 5.13916 seconds: | 20260506 13:59:49.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:59:49.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:49.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:49.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:49.878 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 13:59:49.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:49.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:49.878 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 13:59:49.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:49.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:49.878 | INFO | Uploaded 36GiB to GPU in 1.98476s (18.1382GiB/s) | 20260506 13:59:49.879 | INFO | Downloaded 34GiB to GPU in 1.91202s (17.7822GiB/s) | 20260506 13:59:55.017 | INFO | Analyzing chunk frames = [1902592,1903104) | 20260506 13:59:55.017 | INFO | MemInfo: { | 20260506 13:59:55.017 | INFO | "gpu" : " 17 GB", | 20260506 13:59:55.017 | INFO | "huge" : "", | 20260506 13:59:55.017 | INFO | "huge_shared" : " 27 GB", | 20260506 13:59:55.017 | INFO | "malloc" : " 14 GB", | 20260506 13:59:55.017 | INFO | "pinned" : " 29 GB" | 20260506 13:59:55.017 | INFO | } | 20260506 13:59:55.017 | INFO | Perf report for the last 5.13924 seconds: | 20260506 13:59:55.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 13:59:55.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:55.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:55.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:55.018 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 13:59:55.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:55.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:55.018 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 13:59:55.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:55.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 13:59:55.018 | INFO | Uploaded 36GiB to GPU in 1.98919s (18.0978GiB/s) | 20260506 13:59:55.018 | INFO | Downloaded 34GiB to GPU in 1.93631s (17.5592GiB/s) | 20260506 14:00:00.157 | INFO | Analyzing chunk frames = [1903104,1903616) | 20260506 14:00:00.157 | INFO | MemInfo: { | 20260506 14:00:00.157 | INFO | "gpu" : " 17 GB", | 20260506 14:00:00.157 | INFO | "huge" : "", | 20260506 14:00:00.157 | INFO | "huge_shared" : " 27 GB", | 20260506 14:00:00.157 | INFO | "malloc" : " 14 GB", | 20260506 14:00:00.157 | INFO | "pinned" : " 29 GB" | 20260506 14:00:00.157 | INFO | } | 20260506 14:00:00.157 | INFO | Perf report for the last 5.13937 seconds: | 20260506 14:00:00.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:00:00.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:00.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:00.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:00.157 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:00:00.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:00.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:00.157 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 14:00:00.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:00.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:00.157 | INFO | Uploaded 36GiB to GPU in 2.07523s (17.3475GiB/s) | 20260506 14:00:00.157 | INFO | Downloaded 34GiB to GPU in 1.97698s (17.1979GiB/s) | 20260506 14:00:05.296 | INFO | Analyzing chunk frames = [1903616,1904128) | 20260506 14:00:05.296 | INFO | MemInfo: { | 20260506 14:00:05.296 | INFO | "gpu" : " 17 GB", | 20260506 14:00:05.296 | INFO | "huge" : "", | 20260506 14:00:05.296 | INFO | "huge_shared" : " 27 GB", | 20260506 14:00:05.296 | INFO | "malloc" : " 14 GB", | 20260506 14:00:05.296 | INFO | "pinned" : " 29 GB" | 20260506 14:00:05.296 | INFO | } | 20260506 14:00:05.296 | INFO | Perf report for the last 5.13966 seconds: | 20260506 14:00:05.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:00:05.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:05.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:05.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:05.297 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 14:00:05.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:05.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:05.297 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:00:05.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:05.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:05.297 | INFO | Uploaded 36GiB to GPU in 2.00605s (17.9457GiB/s) | 20260506 14:00:05.297 | INFO | Downloaded 34GiB to GPU in 1.98532s (17.1257GiB/s) | 20260506 14:00:10.436 | INFO | Analyzing chunk frames = [1904128,1904640) | 20260506 14:00:10.436 | INFO | MemInfo: { | 20260506 14:00:10.436 | INFO | "gpu" : " 17 GB", | 20260506 14:00:10.436 | INFO | "huge" : "", | 20260506 14:00:10.436 | INFO | "huge_shared" : " 27 GB", | 20260506 14:00:10.436 | INFO | "malloc" : " 14 GB", | 20260506 14:00:10.436 | INFO | "pinned" : " 29 GB" | 20260506 14:00:10.436 | INFO | } | 20260506 14:00:10.436 | INFO | Perf report for the last 5.13933 seconds: | 20260506 14:00:10.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:00:10.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:10.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:10.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:10.436 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:00:10.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:10.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:10.436 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:00:10.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:10.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:10.436 | INFO | Uploaded 36GiB to GPU in 2.01545s (17.862GiB/s) | 20260506 14:00:10.436 | INFO | Downloaded 34GiB to GPU in 1.97665s (17.2008GiB/s) | 20260506 14:00:10.552 | INFO | Starting a round of Baz IO | 20260506 14:00:13.710 | INFO | Populating RTMetrics report | 20260506 14:00:13.711 | INFO | RTMetrics report populated | 20260506 14:00:13.711 | INFO | Opening tmp RTMetrics json file | 20260506 14:00:13.711 | INFO | Serializing RTMetrics to json | 20260506 14:00:13.713 | INFO | Serialization done, swapping tmp file now... | 20260506 14:00:13.713 | INFO | RTMetrics done | 20260506 14:00:13.813 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.26 seconds | 20260506 14:00:13.813 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.89% / 0.02% / 61.10% | 20260506 14:00:13.813 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.80056% / 3.97364e-05% / 3.97364e-05% | 20260506 14:00:13.813 | INFO | Baz IO overall: | 20260506 14:00:13.813 | INFO | BytesWritten : 976520055008 | 20260506 14:00:13.813 | INFO | NumEvents : 472457075801 | 20260506 14:00:15.575 | INFO | Analyzing chunk frames = [1904640,1905152) | 20260506 14:00:15.575 | INFO | MemInfo: { | 20260506 14:00:15.575 | INFO | "gpu" : " 17 GB", | 20260506 14:00:15.575 | INFO | "huge" : "", | 20260506 14:00:15.575 | INFO | "huge_shared" : " 27 GB", | 20260506 14:00:15.575 | INFO | "malloc" : " 14 GB", | 20260506 14:00:15.575 | INFO | "pinned" : " 29 GB" | 20260506 14:00:15.575 | INFO | } | 20260506 14:00:15.575 | INFO | Perf report for the last 5.13955 seconds: | 20260506 14:00:15.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:00:15.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:15.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:15.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:15.575 | INFO | ANALYSIS : 63.1%, 3.0, 98.1ms, 0 | 20260506 14:00:15.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:15.576 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 14:00:15.576 | INFO | PRE_HQ : 61.1%, 1.9, 60.6ms, 0 | 20260506 14:00:15.576 | INFO | BAZWRITER : 62.8%, 3.7, 120.0ms, 0 | 20260506 14:00:15.576 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 14:00:15.576 | INFO | Uploaded 36GiB to GPU in 2.04439s (17.6092GiB/s) | 20260506 14:00:15.576 | INFO | Downloaded 38GiB to GPU in 2.21128s (17.1846GiB/s) | 20260506 14:00:20.714 | INFO | Analyzing chunk frames = [1905152,1905664) | 20260506 14:00:20.714 | INFO | MemInfo: { | 20260506 14:00:20.714 | INFO | "gpu" : " 17 GB", | 20260506 14:00:20.714 | INFO | "huge" : "", | 20260506 14:00:20.714 | INFO | "huge_shared" : " 27 GB", | 20260506 14:00:20.714 | INFO | "malloc" : " 14 GB", | 20260506 14:00:20.714 | INFO | "pinned" : " 29 GB" | 20260506 14:00:20.714 | INFO | } | 20260506 14:00:20.715 | INFO | Perf report for the last 5.1392 seconds: | 20260506 14:00:20.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:00:20.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:20.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:20.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:20.715 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 14:00:20.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:20.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:20.715 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:00:20.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:20.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:20.715 | INFO | Uploaded 36GiB to GPU in 2.02108s (17.8122GiB/s) | 20260506 14:00:20.715 | INFO | Downloaded 34GiB to GPU in 1.95003s (17.4356GiB/s) | 20260506 14:00:25.854 | INFO | Analyzing chunk frames = [1905664,1906176) | 20260506 14:00:25.854 | INFO | MemInfo: { | 20260506 14:00:25.854 | INFO | "gpu" : " 17 GB", | 20260506 14:00:25.854 | INFO | "huge" : "", | 20260506 14:00:25.854 | INFO | "huge_shared" : " 27 GB", | 20260506 14:00:25.854 | INFO | "malloc" : " 14 GB", | 20260506 14:00:25.854 | INFO | "pinned" : " 29 GB" | 20260506 14:00:25.854 | INFO | } | 20260506 14:00:25.854 | INFO | Perf report for the last 5.13935 seconds: | 20260506 14:00:25.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:00:25.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:25.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:25.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:25.854 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 14:00:25.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:25.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:25.854 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 14:00:25.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:25.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:25.854 | INFO | Uploaded 36GiB to GPU in 1.97787s (18.2014GiB/s) | 20260506 14:00:25.854 | INFO | Downloaded 34GiB to GPU in 1.93754s (17.548GiB/s) | 20260506 14:00:30.992 | INFO | Analyzing chunk frames = [1906176,1906688) | 20260506 14:00:30.992 | INFO | MemInfo: { | 20260506 14:00:30.992 | INFO | "gpu" : " 17 GB", | 20260506 14:00:30.992 | INFO | "huge" : "", | 20260506 14:00:30.992 | INFO | "huge_shared" : " 27 GB", | 20260506 14:00:30.993 | INFO | "malloc" : " 14 GB", | 20260506 14:00:30.993 | INFO | "pinned" : " 29 GB" | 20260506 14:00:30.993 | INFO | } | 20260506 14:00:30.993 | INFO | Perf report for the last 5.13876 seconds: | 20260506 14:00:30.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:00:30.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:30.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:30.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:30.993 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 14:00:30.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:30.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:30.993 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:00:30.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:30.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:30.993 | INFO | Uploaded 36GiB to GPU in 1.98658s (18.1216GiB/s) | 20260506 14:00:30.993 | INFO | Downloaded 34GiB to GPU in 1.91594s (17.7459GiB/s) | 20260506 14:00:36.132 | INFO | Analyzing chunk frames = [1906688,1907200) | 20260506 14:00:36.132 | INFO | MemInfo: { | 20260506 14:00:36.132 | INFO | "gpu" : " 17 GB", | 20260506 14:00:36.132 | INFO | "huge" : "", | 20260506 14:00:36.132 | INFO | "huge_shared" : " 27 GB", | 20260506 14:00:36.132 | INFO | "malloc" : " 14 GB", | 20260506 14:00:36.132 | INFO | "pinned" : " 29 GB" | 20260506 14:00:36.132 | INFO | } | 20260506 14:00:36.132 | INFO | Perf report for the last 5.13938 seconds: | 20260506 14:00:36.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:00:36.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:36.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:36.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:36.132 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:00:36.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:36.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:36.132 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 14:00:36.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:36.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:36.132 | INFO | Uploaded 36GiB to GPU in 1.98245s (18.1594GiB/s) | 20260506 14:00:36.133 | INFO | Downloaded 34GiB to GPU in 1.94126s (17.5144GiB/s) | 20260506 14:00:41.272 | INFO | Analyzing chunk frames = [1907200,1907712) | 20260506 14:00:41.272 | INFO | MemInfo: { | 20260506 14:00:41.272 | INFO | "gpu" : " 17 GB", | 20260506 14:00:41.272 | INFO | "huge" : "", | 20260506 14:00:41.272 | INFO | "huge_shared" : " 27 GB", | 20260506 14:00:41.272 | INFO | "malloc" : " 14 GB", | 20260506 14:00:41.272 | INFO | "pinned" : " 29 GB" | 20260506 14:00:41.272 | INFO | } | 20260506 14:00:41.272 | INFO | Perf report for the last 5.13976 seconds: | 20260506 14:00:41.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:00:41.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:41.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:41.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:41.272 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 14:00:41.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:41.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:41.272 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:00:41.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:41.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:41.272 | INFO | Uploaded 36GiB to GPU in 2.05479s (17.52GiB/s) | 20260506 14:00:41.272 | INFO | Downloaded 34GiB to GPU in 1.97416s (17.2225GiB/s) | 20260506 14:00:46.412 | INFO | Analyzing chunk frames = [1907712,1908224) | 20260506 14:00:46.412 | INFO | MemInfo: { | 20260506 14:00:46.412 | INFO | "gpu" : " 17 GB", | 20260506 14:00:46.412 | INFO | "huge" : "", | 20260506 14:00:46.412 | INFO | "huge_shared" : " 27 GB", | 20260506 14:00:46.412 | INFO | "malloc" : " 14 GB", | 20260506 14:00:46.412 | INFO | "pinned" : " 29 GB" | 20260506 14:00:46.412 | INFO | } | 20260506 14:00:46.412 | INFO | Perf report for the last 5.14 seconds: | 20260506 14:00:46.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:00:46.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:46.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:46.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:46.412 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:00:46.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:46.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:46.412 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:00:46.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:46.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:46.412 | INFO | Uploaded 36GiB to GPU in 1.99374s (18.0565GiB/s) | 20260506 14:00:46.412 | INFO | Downloaded 34GiB to GPU in 1.98251s (17.15GiB/s) | 20260506 14:00:51.551 | INFO | Analyzing chunk frames = [1908224,1908736) | 20260506 14:00:51.551 | INFO | MemInfo: { | 20260506 14:00:51.551 | INFO | "gpu" : " 17 GB", | 20260506 14:00:51.551 | INFO | "huge" : "", | 20260506 14:00:51.551 | INFO | "huge_shared" : " 27 GB", | 20260506 14:00:51.551 | INFO | "malloc" : " 14 GB", | 20260506 14:00:51.551 | INFO | "pinned" : " 29 GB" | 20260506 14:00:51.551 | INFO | } | 20260506 14:00:51.551 | INFO | Perf report for the last 5.13899 seconds: | 20260506 14:00:51.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:00:51.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:51.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:51.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:51.551 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 14:00:51.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:51.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:51.551 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:00:51.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:51.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:51.551 | INFO | Uploaded 36GiB to GPU in 2.03286s (17.709GiB/s) | 20260506 14:00:51.551 | INFO | Downloaded 34GiB to GPU in 1.98112s (17.162GiB/s) | 20260506 14:00:51.551 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:00:51.551 | INFO | Total mem usage: 17.4 GiB | 20260506 14:00:51.551 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:00:51.551 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -20.3%) | 20260506 14:00:51.551 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:00:51.551 | INFO | High Water Breakdown: | 20260506 14:00:51.551 | INFO | 59.9% : 8.67 GiB ( 79.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:00:51.551 | INFO | 39.9% : 5.78 GiB ( 86.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:00:51.551 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:00:51.552 | INFO | ------------------------------------------ | 20260506 14:00:51.552 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:00:51.552 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:00:51.552 | INFO | Total mem usage: 26.5 GiB | 20260506 14:00:51.552 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 14:00:51.552 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:00:51.552 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 14:00:51.552 | INFO | High Water Breakdown: | 20260506 14:00:51.552 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:00:51.552 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:00:51.552 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:00:51.552 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:00:51.552 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:00:51.552 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:00:51.552 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:00:51.552 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:00:51.552 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:00:51.552 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:00:51.552 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:00:51.552 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:00:51.552 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:00:51.552 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:00:51.552 | INFO | ------------------------------------------ | 20260506 14:00:51.552 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:00:51.552 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:00:51.552 | INFO | Total mem usage: 25.6 GiB | 20260506 14:00:51.552 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:00:51.552 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:00:51.552 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:00:51.552 | INFO | High Water Breakdown: | 20260506 14:00:51.552 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:00:51.552 | INFO | ------------------------------------------ | 20260506 14:00:51.552 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:00:51.552 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:00:51.552 | INFO | Total mem usage: 15.7 GiB | 20260506 14:00:51.552 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:00:51.552 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:00:51.552 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 14:00:51.552 | INFO | High Water Breakdown: | 20260506 14:00:51.552 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:00:51.552 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:00:51.552 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:00:51.552 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:00:51.552 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:00:51.552 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:00:51.552 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:00:51.552 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:00:51.552 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:00:51.552 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:00:51.552 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:00:51.552 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:00:51.552 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:00:51.552 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:00:51.552 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:00:51.552 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:00:51.552 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:00:51.552 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:00:51.552 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:00:51.552 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:00:51.552 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:00:51.552 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:00:51.552 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:00:51.552 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:00:51.552 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:00:51.552 | INFO | ------------------------------------------ | 20260506 14:00:51.552 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:00:51.552 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:00:51.552 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:00:51.552 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:00:51.552 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.60e+01, 1.36e+01 / 2.02e+00 | 20260506 14:00:51.552 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.02e-01, 1.03e-02 / 7.09e-03 | 20260506 14:00:51.552 | INFO | PerfReport: 11.58%, Baseline -- 3.65e+00 / 4.46e+00, 5.70e-01 / 2.88e-01 | 20260506 14:00:51.552 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.48e+00, 1.89e-01 / 1.22e-01 | 20260506 14:00:51.552 | INFO | PerfReport: 15.89%, DME -- 4.01e+01 / 4.48e+01, 4.19e+00 / 2.59e+00 | 20260506 14:00:51.552 | INFO | PerfReport: 56.61%, FrameLabeling -- 1.79e+01 / 2.03e+01, 1.41e+00 / 1.66e+00 | 20260506 14:00:51.552 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.36e+00 / 2.95e+00, 4.13e-01 / 2.95e-01 | 20260506 14:00:51.552 | INFO | PerfReport: 3.63%, Metrics -- 1.14e+00 / 1.50e+00, 2.93e-01 / 1.72e-01 | 20260506 14:00:51.552 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.01e-01 / 4.52e-01, 6.26e-02 / 3.39e-02 | 20260506 14:00:51.552 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:00:51.552 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:00:51.552 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:00:51.552 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 14:00:51.552 | INFO | PerfReport: 21.32%, Upload -- 2.09e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 14:00:51.552 | INFO | PerfReport: 21.09%, Download -- 2.07e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 14:00:51.552 | INFO | PerfReport: 32.14%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 14:00:51.552 | INFO | PerfReport: 1.50%, LockUpload -- 1.47e+00 / 0.00e+00, 6.04e+00 / 0.00e+00 | 20260506 14:00:51.553 | INFO | PerfReport: 1.20%, LockDownload -- 1.18e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 14:00:51.553 | INFO | PerfReport: 22.73%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:00:51.553 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.26e-02 / 0.00e+00 | 20260506 14:00:51.673 | INFO | Starting a round of Baz IO | 20260506 14:00:54.832 | INFO | Populating RTMetrics report | 20260506 14:00:54.832 | INFO | RTMetrics report populated | 20260506 14:00:54.832 | INFO | Opening tmp RTMetrics json file | 20260506 14:00:54.833 | INFO | Serializing RTMetrics to json | 20260506 14:00:54.835 | INFO | Serialization done, swapping tmp file now... | 20260506 14:00:54.835 | INFO | RTMetrics done | 20260506 14:00:54.932 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.259 seconds | 20260506 14:00:54.932 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.87% / 0.02% / 61.11% | 20260506 14:00:54.932 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.7921% / 3.97364e-05% / 5.5631e-05% | 20260506 14:00:54.932 | INFO | Baz IO overall: | 20260506 14:00:54.932 | INFO | BytesWritten : 978786601181 | 20260506 14:00:54.932 | INFO | NumEvents : 473555969690 | 20260506 14:00:56.690 | INFO | Analyzing chunk frames = [1908736,1909248) | 20260506 14:00:56.690 | INFO | MemInfo: { | 20260506 14:00:56.690 | INFO | "gpu" : " 17 GB", | 20260506 14:00:56.690 | INFO | "huge" : "", | 20260506 14:00:56.690 | INFO | "huge_shared" : " 27 GB", | 20260506 14:00:56.690 | INFO | "malloc" : " 14 GB", | 20260506 14:00:56.690 | INFO | "pinned" : " 29 GB" | 20260506 14:00:56.690 | INFO | } | 20260506 14:00:56.690 | INFO | Perf report for the last 5.13872 seconds: | 20260506 14:00:56.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:00:56.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:56.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:56.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:56.690 | INFO | ANALYSIS : 63.1%, 3.0, 98.3ms, 0 | 20260506 14:00:56.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:00:56.690 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 14:00:56.690 | INFO | PRE_HQ : 60.9%, 1.9, 60.4ms, 0 | 20260506 14:00:56.690 | INFO | BAZWRITER : 62.8%, 3.8, 121.3ms, 0 | 20260506 14:00:56.690 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260506 14:00:56.690 | INFO | Uploaded 36GiB to GPU in 2.02556s (17.7729GiB/s) | 20260506 14:00:56.690 | INFO | Downloaded 38GiB to GPU in 2.19578s (17.3059GiB/s) | 20260506 14:01:01.829 | INFO | Analyzing chunk frames = [1909248,1909760) | 20260506 14:01:01.829 | INFO | MemInfo: { | 20260506 14:01:01.829 | INFO | "gpu" : " 17 GB", | 20260506 14:01:01.829 | INFO | "huge" : "", | 20260506 14:01:01.829 | INFO | "huge_shared" : " 27 GB", | 20260506 14:01:01.829 | INFO | "malloc" : " 14 GB", | 20260506 14:01:01.829 | INFO | "pinned" : " 29 GB" | 20260506 14:01:01.829 | INFO | } | 20260506 14:01:01.829 | INFO | Perf report for the last 5.13913 seconds: | 20260506 14:01:01.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:01:01.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:01.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:01.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:01.829 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 14:01:01.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:01.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:01.829 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:01:01.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:01.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:01.829 | INFO | Uploaded 36GiB to GPU in 2.00151s (17.9864GiB/s) | 20260506 14:01:01.829 | INFO | Downloaded 34GiB to GPU in 1.9414s (17.5132GiB/s) | 20260506 14:01:06.968 | INFO | Analyzing chunk frames = [1909760,1910272) | 20260506 14:01:06.968 | INFO | MemInfo: { | 20260506 14:01:06.968 | INFO | "gpu" : " 17 GB", | 20260506 14:01:06.968 | INFO | "huge" : "", | 20260506 14:01:06.968 | INFO | "huge_shared" : " 27 GB", | 20260506 14:01:06.968 | INFO | "malloc" : " 14 GB", | 20260506 14:01:06.968 | INFO | "pinned" : " 29 GB" | 20260506 14:01:06.968 | INFO | } | 20260506 14:01:06.968 | INFO | Perf report for the last 5.13919 seconds: | 20260506 14:01:06.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:01:06.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:06.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:06.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:06.968 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:01:06.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:06.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:06.968 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:01:06.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:06.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:06.968 | INFO | Uploaded 36GiB to GPU in 1.99436s (18.0509GiB/s) | 20260506 14:01:06.968 | INFO | Downloaded 34GiB to GPU in 1.92959s (17.6203GiB/s) | 20260506 14:01:12.108 | INFO | Analyzing chunk frames = [1910272,1910784) | 20260506 14:01:12.108 | INFO | MemInfo: { | 20260506 14:01:12.108 | INFO | "gpu" : " 17 GB", | 20260506 14:01:12.108 | INFO | "huge" : "", | 20260506 14:01:12.108 | INFO | "huge_shared" : " 27 GB", | 20260506 14:01:12.108 | INFO | "malloc" : " 14 GB", | 20260506 14:01:12.108 | INFO | "pinned" : " 29 GB" | 20260506 14:01:12.108 | INFO | } | 20260506 14:01:12.108 | INFO | Perf report for the last 5.14028 seconds: | 20260506 14:01:12.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:01:12.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:12.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:12.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:12.108 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 14:01:12.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:12.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:12.108 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:01:12.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:12.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:12.109 | INFO | Uploaded 36GiB to GPU in 2.0106s (17.9051GiB/s) | 20260506 14:01:12.109 | INFO | Downloaded 34GiB to GPU in 1.91301s (17.773GiB/s) | 20260506 14:01:17.246 | INFO | Analyzing chunk frames = [1910784,1911296) | 20260506 14:01:17.246 | INFO | MemInfo: { | 20260506 14:01:17.246 | INFO | "gpu" : " 17 GB", | 20260506 14:01:17.246 | INFO | "huge" : "", | 20260506 14:01:17.246 | INFO | "huge_shared" : " 27 GB", | 20260506 14:01:17.246 | INFO | "malloc" : " 14 GB", | 20260506 14:01:17.246 | INFO | "pinned" : " 29 GB" | 20260506 14:01:17.246 | INFO | } | 20260506 14:01:17.247 | INFO | Perf report for the last 5.13819 seconds: | 20260506 14:01:17.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:01:17.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:17.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:17.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:17.247 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 14:01:17.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:17.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:17.247 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:01:17.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:17.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:17.247 | INFO | Uploaded 36GiB to GPU in 1.99838s (18.0146GiB/s) | 20260506 14:01:17.247 | INFO | Downloaded 34GiB to GPU in 1.93299s (17.5893GiB/s) | 20260506 14:01:22.386 | INFO | Analyzing chunk frames = [1911296,1911808) | 20260506 14:01:22.386 | INFO | MemInfo: { | 20260506 14:01:22.386 | INFO | "gpu" : " 17 GB", | 20260506 14:01:22.386 | INFO | "huge" : "", | 20260506 14:01:22.386 | INFO | "huge_shared" : " 27 GB", | 20260506 14:01:22.386 | INFO | "malloc" : " 14 GB", | 20260506 14:01:22.386 | INFO | "pinned" : " 29 GB" | 20260506 14:01:22.386 | INFO | } | 20260506 14:01:22.387 | INFO | Perf report for the last 5.13998 seconds: | 20260506 14:01:22.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:01:22.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:22.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:22.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:22.387 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 14:01:22.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:22.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:22.387 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 14:01:22.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:22.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:22.387 | INFO | Uploaded 36GiB to GPU in 2.03711s (17.6721GiB/s) | 20260506 14:01:22.387 | INFO | Downloaded 34GiB to GPU in 1.89686s (17.9244GiB/s) | 20260506 14:01:27.527 | INFO | Analyzing chunk frames = [1911808,1912320) | 20260506 14:01:27.527 | INFO | MemInfo: { | 20260506 14:01:27.527 | INFO | "gpu" : " 17 GB", | 20260506 14:01:27.527 | INFO | "huge" : "", | 20260506 14:01:27.527 | INFO | "huge_shared" : " 27 GB", | 20260506 14:01:27.527 | INFO | "malloc" : " 14 GB", | 20260506 14:01:27.527 | INFO | "pinned" : " 29 GB" | 20260506 14:01:27.527 | INFO | } | 20260506 14:01:27.527 | INFO | Perf report for the last 5.14032 seconds: | 20260506 14:01:27.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:01:27.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:27.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:27.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:27.527 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:01:27.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:27.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:27.527 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 14:01:27.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:27.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:27.527 | INFO | Uploaded 36GiB to GPU in 1.98397s (18.1454GiB/s) | 20260506 14:01:27.527 | INFO | Downloaded 34GiB to GPU in 2.00425s (16.964GiB/s) | 20260506 14:01:32.664 | INFO | Analyzing chunk frames = [1912320,1912832) | 20260506 14:01:32.664 | INFO | MemInfo: { | 20260506 14:01:32.664 | INFO | "gpu" : " 17 GB", | 20260506 14:01:32.665 | INFO | "huge" : "", | 20260506 14:01:32.665 | INFO | "huge_shared" : " 27 GB", | 20260506 14:01:32.665 | INFO | "malloc" : " 14 GB", | 20260506 14:01:32.665 | INFO | "pinned" : " 29 GB" | 20260506 14:01:32.665 | INFO | } | 20260506 14:01:32.665 | INFO | Perf report for the last 5.13786 seconds: | 20260506 14:01:32.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:01:32.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:32.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:32.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:32.665 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 14:01:32.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:32.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:32.665 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:01:32.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:32.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:32.665 | INFO | Uploaded 36GiB to GPU in 2.03178s (17.7185GiB/s) | 20260506 14:01:32.665 | INFO | Downloaded 34GiB to GPU in 1.97639s (17.2031GiB/s) | 20260506 14:01:32.787 | INFO | Starting a round of Baz IO | 20260506 14:01:35.934 | INFO | Populating RTMetrics report | 20260506 14:01:35.935 | INFO | RTMetrics report populated | 20260506 14:01:35.935 | INFO | Opening tmp RTMetrics json file | 20260506 14:01:35.935 | INFO | Serializing RTMetrics to json | 20260506 14:01:35.936 | INFO | Serialization done, swapping tmp file now... | 20260506 14:01:35.937 | INFO | RTMetrics done | 20260506 14:01:36.033 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.245 seconds | 20260506 14:01:36.033 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.85% / 0.02% / 61.13% | 20260506 14:01:36.033 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.79028% / 2.38419e-05% / 7.54992e-05% | 20260506 14:01:36.033 | INFO | Baz IO overall: | 20260506 14:01:36.033 | INFO | BytesWritten : 981056144937 | 20260506 14:01:36.033 | INFO | NumEvents : 474657241952 | 20260506 14:01:37.805 | INFO | Analyzing chunk frames = [1912832,1913344) | 20260506 14:01:37.805 | INFO | MemInfo: { | 20260506 14:01:37.805 | INFO | "gpu" : " 17 GB", | 20260506 14:01:37.805 | INFO | "huge" : "", | 20260506 14:01:37.805 | INFO | "huge_shared" : " 27 GB", | 20260506 14:01:37.805 | INFO | "malloc" : " 14 GB", | 20260506 14:01:37.805 | INFO | "pinned" : " 29 GB" | 20260506 14:01:37.805 | INFO | } | 20260506 14:01:37.805 | INFO | Perf report for the last 5.14003 seconds: | 20260506 14:01:37.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:01:37.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:37.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:37.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:37.805 | INFO | ANALYSIS : 63.0%, 3.0, 97.8ms, 0 | 20260506 14:01:37.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:37.805 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 14:01:37.805 | INFO | PRE_HQ : 60.9%, 1.9, 60.2ms, 0 | 20260506 14:01:37.805 | INFO | BAZWRITER : 62.5%, 3.8, 120.7ms, 0 | 20260506 14:01:37.805 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 14:01:37.805 | INFO | Uploaded 36GiB to GPU in 2.05706s (17.5007GiB/s) | 20260506 14:01:37.805 | INFO | Downloaded 38GiB to GPU in 2.20275s (17.2512GiB/s) | 20260506 14:01:42.944 | INFO | Analyzing chunk frames = [1913344,1913856) | 20260506 14:01:42.944 | INFO | MemInfo: { | 20260506 14:01:42.944 | INFO | "gpu" : " 17 GB", | 20260506 14:01:42.944 | INFO | "huge" : "", | 20260506 14:01:42.944 | INFO | "huge_shared" : " 27 GB", | 20260506 14:01:42.944 | INFO | "malloc" : " 14 GB", | 20260506 14:01:42.944 | INFO | "pinned" : " 29 GB" | 20260506 14:01:42.944 | INFO | } | 20260506 14:01:42.944 | INFO | Perf report for the last 5.13943 seconds: | 20260506 14:01:42.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:01:42.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:42.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:42.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:42.944 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:01:42.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:42.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:42.944 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:01:42.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:42.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:42.944 | INFO | Uploaded 36GiB to GPU in 2.05258s (17.5389GiB/s) | 20260506 14:01:42.944 | INFO | Downloaded 34GiB to GPU in 1.95014s (17.4346GiB/s) | 20260506 14:01:48.082 | INFO | Analyzing chunk frames = [1913856,1914368) | 20260506 14:01:48.082 | INFO | MemInfo: { | 20260506 14:01:48.082 | INFO | "gpu" : " 17 GB", | 20260506 14:01:48.082 | INFO | "huge" : "", | 20260506 14:01:48.082 | INFO | "huge_shared" : " 27 GB", | 20260506 14:01:48.082 | INFO | "malloc" : " 14 GB", | 20260506 14:01:48.083 | INFO | "pinned" : " 29 GB" | 20260506 14:01:48.083 | INFO | } | 20260506 14:01:48.083 | INFO | Perf report for the last 5.13835 seconds: | 20260506 14:01:48.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:01:48.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:48.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:48.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:48.083 | INFO | ANALYSIS : 61.5%, 3.0, 95.2ms, 0 | 20260506 14:01:48.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:48.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:48.083 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:01:48.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:48.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:48.083 | INFO | Uploaded 36GiB to GPU in 2.02511s (17.7768GiB/s) | 20260506 14:01:48.083 | INFO | Downloaded 34GiB to GPU in 1.93589s (17.563GiB/s) | 20260506 14:01:53.223 | INFO | Analyzing chunk frames = [1914368,1914880) | 20260506 14:01:53.223 | INFO | MemInfo: { | 20260506 14:01:53.223 | INFO | "gpu" : " 17 GB", | 20260506 14:01:53.223 | INFO | "huge" : "", | 20260506 14:01:53.223 | INFO | "huge_shared" : " 27 GB", | 20260506 14:01:53.223 | INFO | "malloc" : " 14 GB", | 20260506 14:01:53.223 | INFO | "pinned" : " 29 GB" | 20260506 14:01:53.223 | INFO | } | 20260506 14:01:53.223 | INFO | Perf report for the last 5.14045 seconds: | 20260506 14:01:53.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:01:53.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:53.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:53.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:53.223 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 14:01:53.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:53.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:53.223 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:01:53.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:53.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:53.223 | INFO | Uploaded 36GiB to GPU in 1.99289s (18.0642GiB/s) | 20260506 14:01:53.223 | INFO | Downloaded 34GiB to GPU in 1.9279s (17.6358GiB/s) | 20260506 14:01:58.362 | INFO | Analyzing chunk frames = [1914880,1915392) | 20260506 14:01:58.363 | INFO | MemInfo: { | 20260506 14:01:58.363 | INFO | "gpu" : " 17 GB", | 20260506 14:01:58.363 | INFO | "huge" : "", | 20260506 14:01:58.363 | INFO | "huge_shared" : " 27 GB", | 20260506 14:01:58.363 | INFO | "malloc" : " 14 GB", | 20260506 14:01:58.363 | INFO | "pinned" : " 29 GB" | 20260506 14:01:58.363 | INFO | } | 20260506 14:01:58.363 | INFO | Perf report for the last 5.13972 seconds: | 20260506 14:01:58.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:01:58.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:58.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:58.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:58.363 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 14:01:58.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:58.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:58.363 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:01:58.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:58.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:01:58.363 | INFO | Uploaded 36GiB to GPU in 2.0427s (17.6237GiB/s) | 20260506 14:01:58.363 | INFO | Downloaded 34GiB to GPU in 1.94116s (17.5153GiB/s) | 20260506 14:02:03.502 | INFO | Analyzing chunk frames = [1915392,1915904) | 20260506 14:02:03.502 | INFO | MemInfo: { | 20260506 14:02:03.502 | INFO | "gpu" : " 17 GB", | 20260506 14:02:03.502 | INFO | "huge" : "", | 20260506 14:02:03.502 | INFO | "huge_shared" : " 27 GB", | 20260506 14:02:03.502 | INFO | "malloc" : " 14 GB", | 20260506 14:02:03.502 | INFO | "pinned" : " 29 GB" | 20260506 14:02:03.502 | INFO | } | 20260506 14:02:03.502 | INFO | Perf report for the last 5.13908 seconds: | 20260506 14:02:03.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:02:03.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:03.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:03.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:03.502 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 14:02:03.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:03.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:03.502 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 14:02:03.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:03.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:03.502 | INFO | Uploaded 36GiB to GPU in 2.03681s (17.6747GiB/s) | 20260506 14:02:03.502 | INFO | Downloaded 34GiB to GPU in 1.89327s (17.9583GiB/s) | 20260506 14:02:08.640 | INFO | Analyzing chunk frames = [1915904,1916416) | 20260506 14:02:08.640 | INFO | MemInfo: { | 20260506 14:02:08.640 | INFO | "gpu" : " 17 GB", | 20260506 14:02:08.640 | INFO | "huge" : "", | 20260506 14:02:08.640 | INFO | "huge_shared" : " 27 GB", | 20260506 14:02:08.640 | INFO | "malloc" : " 14 GB", | 20260506 14:02:08.640 | INFO | "pinned" : " 29 GB" | 20260506 14:02:08.640 | INFO | } | 20260506 14:02:08.640 | INFO | Perf report for the last 5.13816 seconds: | 20260506 14:02:08.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:02:08.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:08.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:08.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:08.640 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:02:08.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:08.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:08.640 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 14:02:08.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:08.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:08.640 | INFO | Uploaded 36GiB to GPU in 1.98656s (18.1218GiB/s) | 20260506 14:02:08.640 | INFO | Downloaded 34GiB to GPU in 1.99286s (17.0609GiB/s) | 20260506 14:02:13.779 | INFO | Analyzing chunk frames = [1916416,1916928) | 20260506 14:02:13.779 | INFO | MemInfo: { | 20260506 14:02:13.779 | INFO | "gpu" : " 17 GB", | 20260506 14:02:13.779 | INFO | "huge" : "", | 20260506 14:02:13.779 | INFO | "huge_shared" : " 27 GB", | 20260506 14:02:13.779 | INFO | "malloc" : " 14 GB", | 20260506 14:02:13.779 | INFO | "pinned" : " 29 GB" | 20260506 14:02:13.779 | INFO | } | 20260506 14:02:13.779 | INFO | Perf report for the last 5.13923 seconds: | 20260506 14:02:13.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:02:13.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:13.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:13.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:13.779 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:02:13.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:13.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:13.779 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:02:13.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:13.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:13.780 | INFO | Uploaded 36GiB to GPU in 2.02313s (17.7942GiB/s) | 20260506 14:02:13.780 | INFO | Downloaded 34GiB to GPU in 1.98035s (17.1687GiB/s) | 20260506 14:02:13.780 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:02:13.780 | INFO | Total mem usage: 17.4 GiB | 20260506 14:02:13.780 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:02:13.780 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -21.2%) | 20260506 14:02:13.780 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:02:13.780 | INFO | High Water Breakdown: | 20260506 14:02:13.780 | INFO | 59.8% : 8.6 GiB ( 78.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:02:13.780 | INFO | 40% : 5.75 GiB ( 86% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:02:13.780 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:02:13.780 | INFO | ------------------------------------------ | 20260506 14:02:13.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:02:13.780 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:02:13.780 | INFO | Total mem usage: 26.5 GiB | 20260506 14:02:13.780 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 14:02:13.780 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:02:13.780 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 14:02:13.780 | INFO | High Water Breakdown: | 20260506 14:02:13.780 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:02:13.780 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:02:13.780 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:02:13.780 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:02:13.780 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:02:13.780 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:02:13.780 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:02:13.780 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:02:13.780 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:02:13.780 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:02:13.780 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:02:13.780 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:02:13.780 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:02:13.780 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:02:13.780 | INFO | ------------------------------------------ | 20260506 14:02:13.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:02:13.780 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:02:13.780 | INFO | Total mem usage: 25.6 GiB | 20260506 14:02:13.780 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:02:13.780 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:02:13.780 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:02:13.780 | INFO | High Water Breakdown: | 20260506 14:02:13.780 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:02:13.780 | INFO | ------------------------------------------ | 20260506 14:02:13.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:02:13.780 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:02:13.780 | INFO | Total mem usage: 15.7 GiB | 20260506 14:02:13.780 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:02:13.780 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:02:13.780 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 14:02:13.780 | INFO | High Water Breakdown: | 20260506 14:02:13.780 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:02:13.780 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:02:13.780 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:02:13.781 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:02:13.781 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:02:13.781 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:02:13.781 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:02:13.781 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:02:13.781 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:02:13.781 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:02:13.781 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:02:13.781 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:02:13.781 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:02:13.781 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:02:13.781 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:02:13.781 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:02:13.781 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:02:13.781 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:02:13.781 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:02:13.781 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:02:13.781 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:02:13.781 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:02:13.781 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:02:13.781 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:02:13.781 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:02:13.781 | INFO | ------------------------------------------ | 20260506 14:02:13.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:02:13.781 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:02:13.781 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:02:13.781 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:02:13.781 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.54e+01, 1.36e+01 / 1.77e+00 | 20260506 14:02:13.781 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.36e-02 / 1.28e-02 | 20260506 14:02:13.781 | INFO | PerfReport: 11.58%, Baseline -- 3.65e+00 / 4.42e+00, 5.72e-01 / 3.37e-01 | 20260506 14:02:13.781 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.48e+00, 1.90e-01 / 1.29e-01 | 20260506 14:02:13.781 | INFO | PerfReport: 15.80%, DME -- 3.99e+01 / 4.39e+01, 3.77e+00 / 2.18e+00 | 20260506 14:02:13.781 | INFO | PerfReport: 56.69%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.45e+00 / 1.96e+00 | 20260506 14:02:13.781 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.36e+00 / 2.94e+00, 4.15e-01 / 2.36e-01 | 20260506 14:02:13.781 | INFO | PerfReport: 3.64%, Metrics -- 1.15e+00 / 1.54e+00, 2.94e-01 / 1.77e-01 | 20260506 14:02:13.781 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.00e-01 / 4.55e-01, 6.25e-02 / 2.70e-02 | 20260506 14:02:13.781 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:02:13.781 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:02:13.781 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:02:13.781 | INFO | PerfReport: Full Execution -- 9.82e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 14:02:13.781 | INFO | PerfReport: 21.41%, Upload -- 2.10e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 14:02:13.781 | INFO | PerfReport: 20.96%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 14:02:13.781 | INFO | PerfReport: 32.14%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 14:02:13.781 | INFO | PerfReport: 1.63%, LockUpload -- 1.60e+00 / 0.00e+00, 6.57e+00 / 0.00e+00 | 20260506 14:02:13.781 | INFO | PerfReport: 1.12%, LockDownload -- 1.10e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260506 14:02:13.781 | INFO | PerfReport: 22.73%, LockCompute -- 2.23e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:02:13.781 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.29e-02 / 0.00e+00 | 20260506 14:02:13.903 | INFO | Starting a round of Baz IO | 20260506 14:02:17.050 | INFO | Populating RTMetrics report | 20260506 14:02:17.051 | INFO | RTMetrics report populated | 20260506 14:02:17.051 | INFO | Opening tmp RTMetrics json file | 20260506 14:02:17.053 | INFO | Serializing RTMetrics to json | 20260506 14:02:17.056 | INFO | Serialization done, swapping tmp file now... | 20260506 14:02:17.058 | INFO | RTMetrics done | 20260506 14:02:17.567 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 3.663 seconds | 20260506 14:02:17.567 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.84% / 0.02% / 61.15% | 20260506 14:02:17.567 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.78708% / 2.38419e-05% / 3.97364e-05% | 20260506 14:02:17.567 | INFO | Baz IO overall: | 20260506 14:02:17.567 | INFO | BytesWritten : 983325216191 | 20260506 14:02:17.567 | INFO | NumEvents : 475757936103 | 20260506 14:02:18.919 | INFO | Analyzing chunk frames = [1916928,1917440) | 20260506 14:02:18.919 | INFO | MemInfo: { | 20260506 14:02:18.919 | INFO | "gpu" : " 17 GB", | 20260506 14:02:18.919 | INFO | "huge" : "", | 20260506 14:02:18.919 | INFO | "huge_shared" : " 27 GB", | 20260506 14:02:18.919 | INFO | "malloc" : " 14 GB", | 20260506 14:02:18.919 | INFO | "pinned" : " 29 GB" | 20260506 14:02:18.919 | INFO | } | 20260506 14:02:18.919 | INFO | Perf report for the last 5.14002 seconds: | 20260506 14:02:18.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:02:18.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:18.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:18.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:18.919 | INFO | ANALYSIS : 63.0%, 2.9, 95.5ms, 0 | 20260506 14:02:18.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:18.920 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 14:02:18.920 | INFO | PRE_HQ : 61.0%, 1.9, 59.5ms, 0 | 20260506 14:02:18.920 | INFO | BAZWRITER : 70.8%, 4.8, 174.2ms, 0 | 20260506 14:02:18.920 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 14:02:18.920 | INFO | Uploaded 36GiB to GPU in 2.09746s (17.1636GiB/s) | 20260506 14:02:18.920 | INFO | Downloaded 38GiB to GPU in 2.23418s (17.0085GiB/s) | 20260506 14:02:24.058 | INFO | Analyzing chunk frames = [1917440,1917952) | 20260506 14:02:24.058 | INFO | MemInfo: { | 20260506 14:02:24.058 | INFO | "gpu" : " 17 GB", | 20260506 14:02:24.058 | INFO | "huge" : "", | 20260506 14:02:24.058 | INFO | "huge_shared" : " 27 GB", | 20260506 14:02:24.058 | INFO | "malloc" : " 14 GB", | 20260506 14:02:24.059 | INFO | "pinned" : " 29 GB" | 20260506 14:02:24.059 | INFO | } | 20260506 14:02:24.059 | INFO | Perf report for the last 5.13941 seconds: | 20260506 14:02:24.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:02:24.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:24.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:24.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:24.059 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 14:02:24.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:24.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:24.059 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:02:24.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:24.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:24.059 | INFO | Uploaded 36GiB to GPU in 2.0418s (17.6315GiB/s) | 20260506 14:02:24.059 | INFO | Downloaded 34GiB to GPU in 1.9461s (17.4708GiB/s) | 20260506 14:02:29.198 | INFO | Analyzing chunk frames = [1917952,1918464) | 20260506 14:02:29.198 | INFO | MemInfo: { | 20260506 14:02:29.198 | INFO | "gpu" : " 17 GB", | 20260506 14:02:29.198 | INFO | "huge" : "", | 20260506 14:02:29.198 | INFO | "huge_shared" : " 27 GB", | 20260506 14:02:29.199 | INFO | "malloc" : " 14 GB", | 20260506 14:02:29.199 | INFO | "pinned" : " 29 GB" | 20260506 14:02:29.199 | INFO | } | 20260506 14:02:29.199 | INFO | Perf report for the last 5.13993 seconds: | 20260506 14:02:29.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:02:29.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:29.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:29.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:29.199 | INFO | ANALYSIS : 61.5%, 3.0, 95.2ms, 0 | 20260506 14:02:29.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:29.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:29.199 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:02:29.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:29.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:29.199 | INFO | Uploaded 36GiB to GPU in 2.01872s (17.8331GiB/s) | 20260506 14:02:29.199 | INFO | Downloaded 34GiB to GPU in 1.93662s (17.5563GiB/s) | 20260506 14:02:34.336 | INFO | Analyzing chunk frames = [1918464,1918976) | 20260506 14:02:34.336 | INFO | MemInfo: { | 20260506 14:02:34.336 | INFO | "gpu" : " 17 GB", | 20260506 14:02:34.336 | INFO | "huge" : "", | 20260506 14:02:34.336 | INFO | "huge_shared" : " 27 GB", | 20260506 14:02:34.336 | INFO | "malloc" : " 14 GB", | 20260506 14:02:34.337 | INFO | "pinned" : " 29 GB" | 20260506 14:02:34.337 | INFO | } | 20260506 14:02:34.337 | INFO | Perf report for the last 5.13799 seconds: | 20260506 14:02:34.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:02:34.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:34.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:34.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:34.337 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 14:02:34.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:34.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:34.337 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:02:34.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:34.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:34.337 | INFO | Uploaded 36GiB to GPU in 1.99799s (18.0181GiB/s) | 20260506 14:02:34.337 | INFO | Downloaded 34GiB to GPU in 1.92395s (17.672GiB/s) | 20260506 14:02:39.476 | INFO | Analyzing chunk frames = [1918976,1919488) | 20260506 14:02:39.476 | INFO | MemInfo: { | 20260506 14:02:39.476 | INFO | "gpu" : " 17 GB", | 20260506 14:02:39.476 | INFO | "huge" : "", | 20260506 14:02:39.477 | INFO | "huge_shared" : " 27 GB", | 20260506 14:02:39.477 | INFO | "malloc" : " 14 GB", | 20260506 14:02:39.477 | INFO | "pinned" : " 29 GB" | 20260506 14:02:39.477 | INFO | } | 20260506 14:02:39.477 | INFO | Perf report for the last 5.14002 seconds: | 20260506 14:02:39.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:02:39.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:39.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:39.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:39.477 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:02:39.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:39.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:39.477 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:02:39.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:39.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:39.477 | INFO | Uploaded 36GiB to GPU in 1.99548s (18.0408GiB/s) | 20260506 14:02:39.477 | INFO | Downloaded 34GiB to GPU in 1.93562s (17.5654GiB/s) | 20260506 14:02:44.616 | INFO | Analyzing chunk frames = [1919488,1920000) | 20260506 14:02:44.616 | INFO | MemInfo: { | 20260506 14:02:44.616 | INFO | "gpu" : " 17 GB", | 20260506 14:02:44.616 | INFO | "huge" : "", | 20260506 14:02:44.616 | INFO | "huge_shared" : " 27 GB", | 20260506 14:02:44.616 | INFO | "malloc" : " 14 GB", | 20260506 14:02:44.616 | INFO | "pinned" : " 29 GB" | 20260506 14:02:44.616 | INFO | } | 20260506 14:02:44.617 | INFO | Perf report for the last 5.13982 seconds: | 20260506 14:02:44.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:02:44.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:44.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:44.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:44.617 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 14:02:44.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:44.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:44.617 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 14:02:44.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:44.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:44.617 | INFO | Uploaded 36GiB to GPU in 2.07337s (17.363GiB/s) | 20260506 14:02:44.617 | INFO | Downloaded 34GiB to GPU in 1.9388s (17.5367GiB/s) | 20260506 14:02:49.756 | INFO | Analyzing chunk frames = [1920000,1920512) | 20260506 14:02:49.756 | INFO | MemInfo: { | 20260506 14:02:49.756 | INFO | "gpu" : " 17 GB", | 20260506 14:02:49.756 | INFO | "huge" : "", | 20260506 14:02:49.756 | INFO | "huge_shared" : " 27 GB", | 20260506 14:02:49.756 | INFO | "malloc" : " 14 GB", | 20260506 14:02:49.756 | INFO | "pinned" : " 29 GB" | 20260506 14:02:49.756 | INFO | } | 20260506 14:02:49.756 | INFO | Perf report for the last 5.13948 seconds: | 20260506 14:02:49.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:02:49.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:49.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:49.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:49.756 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:02:49.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:49.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:49.756 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 14:02:49.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:49.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:49.756 | INFO | Uploaded 36GiB to GPU in 1.9853s (18.1333GiB/s) | 20260506 14:02:49.756 | INFO | Downloaded 34GiB to GPU in 1.99857s (17.0122GiB/s) | 20260506 14:02:54.894 | INFO | Analyzing chunk frames = [1920512,1921024) | 20260506 14:02:54.894 | INFO | MemInfo: { | 20260506 14:02:54.894 | INFO | "gpu" : " 17 GB", | 20260506 14:02:54.894 | INFO | "huge" : "", | 20260506 14:02:54.894 | INFO | "huge_shared" : " 27 GB", | 20260506 14:02:54.894 | INFO | "malloc" : " 14 GB", | 20260506 14:02:54.894 | INFO | "pinned" : " 29 GB" | 20260506 14:02:54.894 | INFO | } | 20260506 14:02:54.894 | INFO | Perf report for the last 5.13836 seconds: | 20260506 14:02:54.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:02:54.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:54.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:54.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:54.894 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 14:02:54.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:54.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:54.894 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 14:02:54.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:54.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:02:54.895 | INFO | Uploaded 36GiB to GPU in 2.05032s (17.5582GiB/s) | 20260506 14:02:54.895 | INFO | Downloaded 34GiB to GPU in 1.97637s (17.2033GiB/s) | 20260506 14:02:55.020 | INFO | Starting a round of Baz IO | 20260506 14:02:58.169 | INFO | Populating RTMetrics report | 20260506 14:02:58.169 | INFO | RTMetrics report populated | 20260506 14:02:58.169 | INFO | Opening tmp RTMetrics json file | 20260506 14:02:58.230 | INFO | Serializing RTMetrics to json | 20260506 14:02:58.293 | INFO | Serialization done, swapping tmp file now... | 20260506 14:02:58.321 | INFO | RTMetrics done | 20260506 14:02:58.607 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.587 seconds | 20260506 14:02:58.607 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.82% / 0.02% / 61.16% | 20260506 14:02:58.607 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.76837% / 1.58946e-05% / 6.75519e-05% | 20260506 14:02:58.607 | INFO | Baz IO overall: | 20260506 14:02:58.607 | INFO | BytesWritten : 985590686008 | 20260506 14:02:58.607 | INFO | NumEvents : 476856141246 | 20260506 14:03:00.033 | INFO | Analyzing chunk frames = [1921024,1921536) | 20260506 14:03:00.033 | INFO | MemInfo: { | 20260506 14:03:00.033 | INFO | "gpu" : " 17 GB", | 20260506 14:03:00.033 | INFO | "huge" : "", | 20260506 14:03:00.034 | INFO | "huge_shared" : " 27 GB", | 20260506 14:03:00.034 | INFO | "malloc" : " 14 GB", | 20260506 14:03:00.034 | INFO | "pinned" : " 29 GB" | 20260506 14:03:00.034 | INFO | } | 20260506 14:03:00.034 | INFO | Perf report for the last 5.13926 seconds: | 20260506 14:03:00.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:03:00.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:00.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:00.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:00.034 | INFO | ANALYSIS : 63.1%, 2.9, 95.6ms, 0 | 20260506 14:03:00.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:00.034 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260506 14:03:00.034 | INFO | PRE_HQ : 61.0%, 1.9, 59.5ms, 0 | 20260506 14:03:00.034 | INFO | BAZWRITER : 69.2%, 4.8, 171.1ms, 0 | 20260506 14:03:00.034 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.2ms, 0 | 20260506 14:03:00.034 | INFO | Uploaded 36GiB to GPU in 2.05051s (17.5566GiB/s) | 20260506 14:03:00.034 | INFO | Downloaded 38GiB to GPU in 2.22393s (17.0868GiB/s) | 20260506 14:03:05.173 | INFO | Analyzing chunk frames = [1921536,1922048) | 20260506 14:03:05.173 | INFO | MemInfo: { | 20260506 14:03:05.173 | INFO | "gpu" : " 17 GB", | 20260506 14:03:05.173 | INFO | "huge" : "", | 20260506 14:03:05.173 | INFO | "huge_shared" : " 27 GB", | 20260506 14:03:05.173 | INFO | "malloc" : " 14 GB", | 20260506 14:03:05.173 | INFO | "pinned" : " 29 GB" | 20260506 14:03:05.173 | INFO | } | 20260506 14:03:05.173 | INFO | Perf report for the last 5.13933 seconds: | 20260506 14:03:05.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:03:05.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:05.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:05.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:05.173 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:03:05.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:05.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:05.173 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:03:05.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:05.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:05.173 | INFO | Uploaded 36GiB to GPU in 2.01415s (17.8735GiB/s) | 20260506 14:03:05.173 | INFO | Downloaded 34GiB to GPU in 1.93744s (17.5489GiB/s) | 20260506 14:03:10.313 | INFO | Analyzing chunk frames = [1922048,1922560) | 20260506 14:03:10.313 | INFO | MemInfo: { | 20260506 14:03:10.313 | INFO | "gpu" : " 17 GB", | 20260506 14:03:10.313 | INFO | "huge" : "", | 20260506 14:03:10.313 | INFO | "huge_shared" : " 27 GB", | 20260506 14:03:10.313 | INFO | "malloc" : " 14 GB", | 20260506 14:03:10.313 | INFO | "pinned" : " 29 GB" | 20260506 14:03:10.313 | INFO | } | 20260506 14:03:10.313 | INFO | Perf report for the last 5.13963 seconds: | 20260506 14:03:10.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:03:10.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:10.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:10.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:10.313 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:03:10.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:10.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:10.313 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:03:10.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:10.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:10.313 | INFO | Uploaded 36GiB to GPU in 2.00089s (17.992GiB/s) | 20260506 14:03:10.313 | INFO | Downloaded 34GiB to GPU in 1.93081s (17.6092GiB/s) | 20260506 14:03:15.452 | INFO | Analyzing chunk frames = [1922560,1923072) | 20260506 14:03:15.452 | INFO | MemInfo: { | 20260506 14:03:15.452 | INFO | "gpu" : " 17 GB", | 20260506 14:03:15.452 | INFO | "huge" : "", | 20260506 14:03:15.452 | INFO | "huge_shared" : " 27 GB", | 20260506 14:03:15.452 | INFO | "malloc" : " 14 GB", | 20260506 14:03:15.452 | INFO | "pinned" : " 29 GB" | 20260506 14:03:15.452 | INFO | } | 20260506 14:03:15.452 | INFO | Perf report for the last 5.13975 seconds: | 20260506 14:03:15.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:03:15.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:15.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:15.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:15.452 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 14:03:15.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:15.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:15.453 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:03:15.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:15.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:15.453 | INFO | Uploaded 36GiB to GPU in 2.00688s (17.9383GiB/s) | 20260506 14:03:15.453 | INFO | Downloaded 34GiB to GPU in 1.91343s (17.7691GiB/s) | 20260506 14:03:20.591 | INFO | Analyzing chunk frames = [1923072,1923584) | 20260506 14:03:20.591 | INFO | MemInfo: { | 20260506 14:03:20.591 | INFO | "gpu" : " 17 GB", | 20260506 14:03:20.591 | INFO | "huge" : "", | 20260506 14:03:20.591 | INFO | "huge_shared" : " 27 GB", | 20260506 14:03:20.591 | INFO | "malloc" : " 14 GB", | 20260506 14:03:20.591 | INFO | "pinned" : " 29 GB" | 20260506 14:03:20.591 | INFO | } | 20260506 14:03:20.591 | INFO | Perf report for the last 5.13856 seconds: | 20260506 14:03:20.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:03:20.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:20.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:20.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:20.591 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 14:03:20.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:20.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:20.591 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:03:20.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:20.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:20.591 | INFO | Uploaded 36GiB to GPU in 2.00509s (17.9543GiB/s) | 20260506 14:03:20.591 | INFO | Downloaded 34GiB to GPU in 1.93144s (17.6034GiB/s) | 20260506 14:03:25.731 | INFO | Analyzing chunk frames = [1923584,1924096) | 20260506 14:03:25.731 | INFO | MemInfo: { | 20260506 14:03:25.731 | INFO | "gpu" : " 17 GB", | 20260506 14:03:25.731 | INFO | "huge" : "", | 20260506 14:03:25.731 | INFO | "huge_shared" : " 27 GB", | 20260506 14:03:25.731 | INFO | "malloc" : " 14 GB", | 20260506 14:03:25.731 | INFO | "pinned" : " 29 GB" | 20260506 14:03:25.731 | INFO | } | 20260506 14:03:25.731 | INFO | Perf report for the last 5.13999 seconds: | 20260506 14:03:25.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:03:25.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:25.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:25.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:25.731 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:03:25.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:25.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:25.731 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 14:03:25.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:25.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:25.731 | INFO | Uploaded 36GiB to GPU in 2.04756s (17.5819GiB/s) | 20260506 14:03:25.731 | INFO | Downloaded 34GiB to GPU in 1.9118s (17.7843GiB/s) | 20260506 14:03:30.870 | INFO | Analyzing chunk frames = [1924096,1924608) | 20260506 14:03:30.870 | INFO | MemInfo: { | 20260506 14:03:30.870 | INFO | "gpu" : " 17 GB", | 20260506 14:03:30.870 | INFO | "huge" : "", | 20260506 14:03:30.871 | INFO | "huge_shared" : " 27 GB", | 20260506 14:03:30.871 | INFO | "malloc" : " 14 GB", | 20260506 14:03:30.871 | INFO | "pinned" : " 29 GB" | 20260506 14:03:30.871 | INFO | } | 20260506 14:03:30.871 | INFO | Perf report for the last 5.13965 seconds: | 20260506 14:03:30.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:03:30.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:30.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:30.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:30.871 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 14:03:30.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:30.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:30.871 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 14:03:30.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:30.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:30.871 | INFO | Uploaded 36GiB to GPU in 2.0362s (17.68GiB/s) | 20260506 14:03:30.871 | INFO | Downloaded 34GiB to GPU in 1.99388s (17.0521GiB/s) | 20260506 14:03:36.009 | INFO | Analyzing chunk frames = [1924608,1925120) | 20260506 14:03:36.009 | INFO | MemInfo: { | 20260506 14:03:36.009 | INFO | "gpu" : " 17 GB", | 20260506 14:03:36.009 | INFO | "huge" : "", | 20260506 14:03:36.009 | INFO | "huge_shared" : " 27 GB", | 20260506 14:03:36.009 | INFO | "malloc" : " 14 GB", | 20260506 14:03:36.009 | INFO | "pinned" : " 29 GB" | 20260506 14:03:36.009 | INFO | } | 20260506 14:03:36.009 | INFO | Perf report for the last 5.1385 seconds: | 20260506 14:03:36.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:03:36.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:36.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:36.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:36.009 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 14:03:36.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:36.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:36.009 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 14:03:36.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:36.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:36.009 | INFO | Uploaded 36GiB to GPU in 2.04691s (17.5875GiB/s) | 20260506 14:03:36.009 | INFO | Downloaded 34GiB to GPU in 1.97194s (17.2419GiB/s) | 20260506 14:03:36.010 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:03:36.010 | INFO | Total mem usage: 17.4 GiB | 20260506 14:03:36.010 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:03:36.010 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -9.78%) | 20260506 14:03:36.010 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:03:36.010 | INFO | High Water Breakdown: | 20260506 14:03:36.010 | INFO | 61.4% : 9.73 GiB ( 89.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:03:36.010 | INFO | 38.5% : 6.11 GiB ( 91.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:03:36.010 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:03:36.010 | INFO | ------------------------------------------ | 20260506 14:03:36.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:03:36.010 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:03:36.010 | INFO | Total mem usage: 26.5 GiB | 20260506 14:03:36.010 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 14:03:36.010 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:03:36.010 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 14:03:36.010 | INFO | High Water Breakdown: | 20260506 14:03:36.010 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:03:36.010 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:03:36.010 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:03:36.010 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:03:36.010 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:03:36.010 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:03:36.010 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:03:36.010 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:03:36.010 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:03:36.010 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:03:36.010 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:03:36.010 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:03:36.010 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:03:36.010 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:03:36.010 | INFO | ------------------------------------------ | 20260506 14:03:36.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:03:36.010 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:03:36.010 | INFO | Total mem usage: 25.6 GiB | 20260506 14:03:36.010 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:03:36.010 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:03:36.010 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:03:36.010 | INFO | High Water Breakdown: | 20260506 14:03:36.010 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:03:36.010 | INFO | ------------------------------------------ | 20260506 14:03:36.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:03:36.010 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:03:36.010 | INFO | Total mem usage: 15.7 GiB | 20260506 14:03:36.010 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:03:36.010 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:03:36.010 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 14:03:36.010 | INFO | High Water Breakdown: | 20260506 14:03:36.010 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:03:36.010 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:03:36.010 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:03:36.010 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:03:36.010 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:03:36.010 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:03:36.010 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:03:36.010 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:03:36.010 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:03:36.010 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:03:36.010 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:03:36.010 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:03:36.010 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:03:36.010 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:03:36.010 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:03:36.010 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:03:36.010 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:03:36.010 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:03:36.010 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:03:36.010 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:03:36.010 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:03:36.010 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:03:36.010 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:03:36.010 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:03:36.010 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:03:36.010 | INFO | ------------------------------------------ | 20260506 14:03:36.010 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:03:36.010 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:03:36.010 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:03:36.010 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:03:36.010 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.57e+01, 1.35e+01 / 2.19e+00 | 20260506 14:03:36.010 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.02e-01, 1.01e-02 / 6.28e-03 | 20260506 14:03:36.010 | INFO | PerfReport: 11.60%, Baseline -- 3.66e+00 / 4.46e+00, 5.76e-01 / 3.11e-01 | 20260506 14:03:36.010 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.90e-01 / 1.34e-01 | 20260506 14:03:36.011 | INFO | PerfReport: 15.82%, DME -- 3.99e+01 / 4.41e+01, 3.73e+00 / 2.46e+00 | 20260506 14:03:36.011 | INFO | PerfReport: 56.71%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.48e+00 / 1.87e+00 | 20260506 14:03:36.011 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.35e+00 / 2.94e+00, 4.05e-01 / 2.75e-01 | 20260506 14:03:36.011 | INFO | PerfReport: 3.63%, Metrics -- 1.14e+00 / 1.54e+00, 2.94e-01 / 1.75e-01 | 20260506 14:03:36.011 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.00e-01 / 4.54e-01, 6.13e-02 / 2.26e-02 | 20260506 14:03:36.011 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:03:36.011 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:03:36.011 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:03:36.011 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 14:03:36.011 | INFO | PerfReport: 21.58%, Upload -- 2.11e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260506 14:03:36.011 | INFO | PerfReport: 21.08%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 14:03:36.011 | INFO | PerfReport: 32.23%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 14:03:36.011 | INFO | PerfReport: 1.69%, LockUpload -- 1.66e+00 / 0.00e+00, 6.61e+00 / 0.00e+00 | 20260506 14:03:36.011 | INFO | PerfReport: 1.07%, LockDownload -- 1.05e+00 / 0.00e+00, 2.68e+00 / 0.00e+00 | 20260506 14:03:36.011 | INFO | PerfReport: 22.33%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 14:03:36.011 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260506 14:03:36.132 | INFO | Starting a round of Baz IO | 20260506 14:03:39.288 | INFO | Populating RTMetrics report | 20260506 14:03:39.289 | INFO | RTMetrics report populated | 20260506 14:03:39.289 | INFO | Opening tmp RTMetrics json file | 20260506 14:03:39.323 | INFO | Serializing RTMetrics to json | 20260506 14:03:39.343 | INFO | Serialization done, swapping tmp file now... | 20260506 14:03:39.387 | INFO | RTMetrics done | 20260506 14:03:39.700 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.567 seconds | 20260506 14:03:39.700 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.81% / 0.02% / 61.18% | 20260506 14:03:39.700 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.76814% / 1.98682e-05% / 6.35783e-05% | 20260506 14:03:39.700 | INFO | Baz IO overall: | 20260506 14:03:39.700 | INFO | BytesWritten : 987858738624 | 20260506 14:03:39.700 | INFO | NumEvents : 477956163358 | 20260506 14:03:41.148 | INFO | Analyzing chunk frames = [1925120,1925632) | 20260506 14:03:41.148 | INFO | MemInfo: { | 20260506 14:03:41.148 | INFO | "gpu" : " 17 GB", | 20260506 14:03:41.148 | INFO | "huge" : "", | 20260506 14:03:41.148 | INFO | "huge_shared" : " 27 GB", | 20260506 14:03:41.148 | INFO | "malloc" : " 14 GB", | 20260506 14:03:41.148 | INFO | "pinned" : " 29 GB" | 20260506 14:03:41.148 | INFO | } | 20260506 14:03:41.148 | INFO | Perf report for the last 5.1389 seconds: | 20260506 14:03:41.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:03:41.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:41.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:41.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:41.148 | INFO | ANALYSIS : 63.2%, 2.9, 95.6ms, 0 | 20260506 14:03:41.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:41.148 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260506 14:03:41.148 | INFO | PRE_HQ : 61.1%, 1.9, 59.8ms, 0 | 20260506 14:03:41.148 | INFO | BAZWRITER : 68.7%, 4.8, 169.8ms, 0 | 20260506 14:03:41.148 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 14:03:41.149 | INFO | Uploaded 36GiB to GPU in 2.06572s (17.4274GiB/s) | 20260506 14:03:41.149 | INFO | Downloaded 38GiB to GPU in 2.23754s (16.9829GiB/s) | 20260506 14:03:46.287 | INFO | Analyzing chunk frames = [1925632,1926144) | 20260506 14:03:46.287 | INFO | MemInfo: { | 20260506 14:03:46.287 | INFO | "gpu" : " 17 GB", | 20260506 14:03:46.287 | INFO | "huge" : "", | 20260506 14:03:46.287 | INFO | "huge_shared" : " 27 GB", | 20260506 14:03:46.287 | INFO | "malloc" : " 14 GB", | 20260506 14:03:46.287 | INFO | "pinned" : " 29 GB" | 20260506 14:03:46.287 | INFO | } | 20260506 14:03:46.287 | INFO | Perf report for the last 5.13943 seconds: | 20260506 14:03:46.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:03:46.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:46.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:46.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:46.288 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:03:46.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:46.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:46.288 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 14:03:46.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:46.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:46.288 | INFO | Uploaded 36GiB to GPU in 1.99993s (18.0006GiB/s) | 20260506 14:03:46.288 | INFO | Downloaded 34GiB to GPU in 1.94292s (17.4994GiB/s) | 20260506 14:03:51.428 | INFO | Analyzing chunk frames = [1926144,1926656) | 20260506 14:03:51.428 | INFO | MemInfo: { | 20260506 14:03:51.428 | INFO | "gpu" : " 17 GB", | 20260506 14:03:51.428 | INFO | "huge" : "", | 20260506 14:03:51.428 | INFO | "huge_shared" : " 27 GB", | 20260506 14:03:51.428 | INFO | "malloc" : " 14 GB", | 20260506 14:03:51.428 | INFO | "pinned" : " 29 GB" | 20260506 14:03:51.428 | INFO | } | 20260506 14:03:51.428 | INFO | Perf report for the last 5.14044 seconds: | 20260506 14:03:51.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:03:51.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:51.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:51.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:51.428 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 14:03:51.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:51.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:51.428 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:03:51.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:51.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:51.428 | INFO | Uploaded 36GiB to GPU in 1.98683s (18.1193GiB/s) | 20260506 14:03:51.428 | INFO | Downloaded 34GiB to GPU in 1.93084s (17.6089GiB/s) | 20260506 14:03:56.567 | INFO | Analyzing chunk frames = [1926656,1927168) | 20260506 14:03:56.567 | INFO | MemInfo: { | 20260506 14:03:56.567 | INFO | "gpu" : " 17 GB", | 20260506 14:03:56.567 | INFO | "huge" : "", | 20260506 14:03:56.567 | INFO | "huge_shared" : " 27 GB", | 20260506 14:03:56.567 | INFO | "malloc" : " 14 GB", | 20260506 14:03:56.567 | INFO | "pinned" : " 29 GB" | 20260506 14:03:56.567 | INFO | } | 20260506 14:03:56.567 | INFO | Perf report for the last 5.1391 seconds: | 20260506 14:03:56.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:03:56.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:56.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:56.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:56.567 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 14:03:56.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:56.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:56.567 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:03:56.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:56.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:03:56.567 | INFO | Uploaded 36GiB to GPU in 1.96562s (18.3148GiB/s) | 20260506 14:03:56.567 | INFO | Downloaded 34GiB to GPU in 1.90915s (17.809GiB/s) | 20260506 14:04:01.707 | INFO | Analyzing chunk frames = [1927168,1927680) | 20260506 14:04:01.707 | INFO | MemInfo: { | 20260506 14:04:01.707 | INFO | "gpu" : " 17 GB", | 20260506 14:04:01.707 | INFO | "huge" : "", | 20260506 14:04:01.707 | INFO | "huge_shared" : " 27 GB", | 20260506 14:04:01.707 | INFO | "malloc" : " 14 GB", | 20260506 14:04:01.707 | INFO | "pinned" : " 29 GB" | 20260506 14:04:01.707 | INFO | } | 20260506 14:04:01.707 | INFO | Perf report for the last 5.13982 seconds: | 20260506 14:04:01.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:04:01.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:01.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:01.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:01.707 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 14:04:01.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:01.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:01.707 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:04:01.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:01.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:01.707 | INFO | Uploaded 36GiB to GPU in 2.03018s (17.7324GiB/s) | 20260506 14:04:01.707 | INFO | Downloaded 34GiB to GPU in 1.93837s (17.5405GiB/s) | 20260506 14:04:06.845 | INFO | Analyzing chunk frames = [1927680,1928192) | 20260506 14:04:06.845 | INFO | MemInfo: { | 20260506 14:04:06.845 | INFO | "gpu" : " 17 GB", | 20260506 14:04:06.845 | INFO | "huge" : "", | 20260506 14:04:06.845 | INFO | "huge_shared" : " 27 GB", | 20260506 14:04:06.845 | INFO | "malloc" : " 14 GB", | 20260506 14:04:06.845 | INFO | "pinned" : " 29 GB" | 20260506 14:04:06.845 | INFO | } | 20260506 14:04:06.845 | INFO | Perf report for the last 5.13808 seconds: | 20260506 14:04:06.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:04:06.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:06.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:06.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:06.845 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 14:04:06.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:06.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:06.845 | INFO | PRE_HQ : 30.0%, 1.0, 15.5ms, 0 | 20260506 14:04:06.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:06.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:06.845 | INFO | Uploaded 36GiB to GPU in 2.03842s (17.6607GiB/s) | 20260506 14:04:06.845 | INFO | Downloaded 34GiB to GPU in 1.92291s (17.6815GiB/s) | 20260506 14:04:11.984 | INFO | Analyzing chunk frames = [1928192,1928704) | 20260506 14:04:11.984 | INFO | MemInfo: { | 20260506 14:04:11.984 | INFO | "gpu" : " 17 GB", | 20260506 14:04:11.984 | INFO | "huge" : "", | 20260506 14:04:11.984 | INFO | "huge_shared" : " 27 GB", | 20260506 14:04:11.984 | INFO | "malloc" : " 14 GB", | 20260506 14:04:11.984 | INFO | "pinned" : " 29 GB" | 20260506 14:04:11.984 | INFO | } | 20260506 14:04:11.984 | INFO | Perf report for the last 5.13928 seconds: | 20260506 14:04:11.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:04:11.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:11.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:11.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:11.984 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 14:04:11.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:11.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:11.984 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 14:04:11.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:11.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:11.985 | INFO | Uploaded 36GiB to GPU in 2.01525s (17.8638GiB/s) | 20260506 14:04:11.985 | INFO | Downloaded 34GiB to GPU in 1.99784s (17.0184GiB/s) | 20260506 14:04:17.123 | INFO | Analyzing chunk frames = [1928704,1929216) | 20260506 14:04:17.123 | INFO | MemInfo: { | 20260506 14:04:17.123 | INFO | "gpu" : " 17 GB", | 20260506 14:04:17.123 | INFO | "huge" : "", | 20260506 14:04:17.123 | INFO | "huge_shared" : " 27 GB", | 20260506 14:04:17.123 | INFO | "malloc" : " 14 GB", | 20260506 14:04:17.123 | INFO | "pinned" : " 29 GB" | 20260506 14:04:17.123 | INFO | } | 20260506 14:04:17.123 | INFO | Perf report for the last 5.13918 seconds: | 20260506 14:04:17.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:04:17.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:17.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:17.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:17.124 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:04:17.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:17.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:17.124 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:04:17.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:17.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:17.124 | INFO | Uploaded 36GiB to GPU in 2.02794s (17.752GiB/s) | 20260506 14:04:17.124 | INFO | Downloaded 34GiB to GPU in 1.96545s (17.2989GiB/s) | 20260506 14:04:17.242 | INFO | Starting a round of Baz IO | 20260506 14:04:20.398 | INFO | Populating RTMetrics report | 20260506 14:04:20.399 | INFO | RTMetrics report populated | 20260506 14:04:20.399 | INFO | Opening tmp RTMetrics json file | 20260506 14:04:20.402 | INFO | Serializing RTMetrics to json | 20260506 14:04:20.438 | INFO | Serialization done, swapping tmp file now... | 20260506 14:04:20.478 | INFO | RTMetrics done | 20260506 14:04:20.808 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.565 seconds | 20260506 14:04:20.809 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.79% / 0.02% / 61.19% | 20260506 14:04:20.809 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.75525% / 2.38419e-05% / 4.76837e-05% | 20260506 14:04:20.809 | INFO | Baz IO overall: | 20260506 14:04:20.809 | INFO | BytesWritten : 990124124094 | 20260506 14:04:20.809 | INFO | NumEvents : 479054205258 | 20260506 14:04:22.263 | INFO | Analyzing chunk frames = [1929216,1929728) | 20260506 14:04:22.263 | INFO | MemInfo: { | 20260506 14:04:22.263 | INFO | "gpu" : " 17 GB", | 20260506 14:04:22.263 | INFO | "huge" : "", | 20260506 14:04:22.263 | INFO | "huge_shared" : " 27 GB", | 20260506 14:04:22.263 | INFO | "malloc" : " 14 GB", | 20260506 14:04:22.263 | INFO | "pinned" : " 29 GB" | 20260506 14:04:22.263 | INFO | } | 20260506 14:04:22.263 | INFO | Perf report for the last 5.13979 seconds: | 20260506 14:04:22.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:04:22.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:22.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:22.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:22.263 | INFO | ANALYSIS : 63.1%, 2.9, 95.5ms, 0 | 20260506 14:04:22.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:22.263 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 14:04:22.263 | INFO | PRE_HQ : 61.0%, 1.9, 59.7ms, 0 | 20260506 14:04:22.263 | INFO | BAZWRITER : 68.7%, 4.8, 169.7ms, 0 | 20260506 14:04:22.263 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 14:04:22.264 | INFO | Uploaded 36GiB to GPU in 2.05847s (17.4887GiB/s) | 20260506 14:04:22.264 | INFO | Downloaded 38GiB to GPU in 2.22734s (17.0607GiB/s) | 20260506 14:04:27.402 | INFO | Analyzing chunk frames = [1929728,1930240) | 20260506 14:04:27.402 | INFO | MemInfo: { | 20260506 14:04:27.402 | INFO | "gpu" : " 17 GB", | 20260506 14:04:27.402 | INFO | "huge" : "", | 20260506 14:04:27.402 | INFO | "huge_shared" : " 27 GB", | 20260506 14:04:27.403 | INFO | "malloc" : " 14 GB", | 20260506 14:04:27.403 | INFO | "pinned" : " 29 GB" | 20260506 14:04:27.403 | INFO | } | 20260506 14:04:27.403 | INFO | Perf report for the last 5.13938 seconds: | 20260506 14:04:27.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:04:27.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:27.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:27.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:27.403 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:04:27.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:27.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:27.403 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:04:27.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:27.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:27.403 | INFO | Uploaded 36GiB to GPU in 1.99534s (18.042GiB/s) | 20260506 14:04:27.403 | INFO | Downloaded 34GiB to GPU in 1.94335s (17.4955GiB/s) | 20260506 14:04:32.542 | INFO | Analyzing chunk frames = [1930240,1930752) | 20260506 14:04:32.542 | INFO | MemInfo: { | 20260506 14:04:32.542 | INFO | "gpu" : " 17 GB", | 20260506 14:04:32.542 | INFO | "huge" : "", | 20260506 14:04:32.542 | INFO | "huge_shared" : " 27 GB", | 20260506 14:04:32.542 | INFO | "malloc" : " 14 GB", | 20260506 14:04:32.542 | INFO | "pinned" : " 29 GB" | 20260506 14:04:32.542 | INFO | } | 20260506 14:04:32.542 | INFO | Perf report for the last 5.13966 seconds: | 20260506 14:04:32.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:04:32.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:32.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:32.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:32.542 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:04:32.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:32.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:32.542 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:04:32.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:32.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:32.543 | INFO | Uploaded 36GiB to GPU in 1.99893s (18.0096GiB/s) | 20260506 14:04:32.543 | INFO | Downloaded 34GiB to GPU in 1.92977s (17.6187GiB/s) | 20260506 14:04:37.682 | INFO | Analyzing chunk frames = [1930752,1931264) | 20260506 14:04:37.682 | INFO | MemInfo: { | 20260506 14:04:37.682 | INFO | "gpu" : " 17 GB", | 20260506 14:04:37.682 | INFO | "huge" : "", | 20260506 14:04:37.682 | INFO | "huge_shared" : " 27 GB", | 20260506 14:04:37.682 | INFO | "malloc" : " 14 GB", | 20260506 14:04:37.682 | INFO | "pinned" : " 29 GB" | 20260506 14:04:37.682 | INFO | } | 20260506 14:04:37.682 | INFO | Perf report for the last 5.13976 seconds: | 20260506 14:04:37.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:04:37.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:37.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:37.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:37.682 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 14:04:37.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:37.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:37.682 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:04:37.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:37.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:37.682 | INFO | Uploaded 36GiB to GPU in 1.99735s (18.0239GiB/s) | 20260506 14:04:37.682 | INFO | Downloaded 34GiB to GPU in 1.92584s (17.6546GiB/s) | 20260506 14:04:42.821 | INFO | Analyzing chunk frames = [1931264,1931776) | 20260506 14:04:42.821 | INFO | MemInfo: { | 20260506 14:04:42.821 | INFO | "gpu" : " 17 GB", | 20260506 14:04:42.821 | INFO | "huge" : "", | 20260506 14:04:42.821 | INFO | "huge_shared" : " 27 GB", | 20260506 14:04:42.821 | INFO | "malloc" : " 14 GB", | 20260506 14:04:42.821 | INFO | "pinned" : " 29 GB" | 20260506 14:04:42.821 | INFO | } | 20260506 14:04:42.821 | INFO | Perf report for the last 5.13888 seconds: | 20260506 14:04:42.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:04:42.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:42.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:42.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:42.821 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 14:04:42.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:42.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:42.821 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:04:42.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:42.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:42.821 | INFO | Uploaded 36GiB to GPU in 2.0325s (17.7122GiB/s) | 20260506 14:04:42.821 | INFO | Downloaded 34GiB to GPU in 1.93339s (17.5857GiB/s) | 20260506 14:04:47.960 | INFO | Analyzing chunk frames = [1931776,1932288) | 20260506 14:04:47.960 | INFO | MemInfo: { | 20260506 14:04:47.960 | INFO | "gpu" : " 17 GB", | 20260506 14:04:47.960 | INFO | "huge" : "", | 20260506 14:04:47.960 | INFO | "huge_shared" : " 27 GB", | 20260506 14:04:47.960 | INFO | "malloc" : " 14 GB", | 20260506 14:04:47.960 | INFO | "pinned" : " 29 GB" | 20260506 14:04:47.960 | INFO | } | 20260506 14:04:47.960 | INFO | Perf report for the last 5.13929 seconds: | 20260506 14:04:47.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:04:47.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:47.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:47.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:47.960 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:04:47.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:47.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:47.960 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260506 14:04:47.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:47.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:47.961 | INFO | Uploaded 36GiB to GPU in 2.05216s (17.5425GiB/s) | 20260506 14:04:47.961 | INFO | Downloaded 34GiB to GPU in 1.90592s (17.8392GiB/s) | 20260506 14:04:53.099 | INFO | Analyzing chunk frames = [1932288,1932800) | 20260506 14:04:53.099 | INFO | MemInfo: { | 20260506 14:04:53.099 | INFO | "gpu" : " 17 GB", | 20260506 14:04:53.099 | INFO | "huge" : "", | 20260506 14:04:53.099 | INFO | "huge_shared" : " 27 GB", | 20260506 14:04:53.099 | INFO | "malloc" : " 14 GB", | 20260506 14:04:53.099 | INFO | "pinned" : " 29 GB" | 20260506 14:04:53.099 | INFO | } | 20260506 14:04:53.099 | INFO | Perf report for the last 5.1388 seconds: | 20260506 14:04:53.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:04:53.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:53.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:53.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:53.099 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:04:53.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:53.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:53.099 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 14:04:53.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:53.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:53.099 | INFO | Uploaded 36GiB to GPU in 2.02027s (17.8194GiB/s) | 20260506 14:04:53.100 | INFO | Downloaded 34GiB to GPU in 2.00381s (16.9677GiB/s) | 20260506 14:04:58.238 | INFO | Analyzing chunk frames = [1932800,1933312) | 20260506 14:04:58.238 | INFO | MemInfo: { | 20260506 14:04:58.238 | INFO | "gpu" : " 17 GB", | 20260506 14:04:58.238 | INFO | "huge" : "", | 20260506 14:04:58.238 | INFO | "huge_shared" : " 27 GB", | 20260506 14:04:58.238 | INFO | "malloc" : " 14 GB", | 20260506 14:04:58.238 | INFO | "pinned" : " 29 GB" | 20260506 14:04:58.238 | INFO | } | 20260506 14:04:58.238 | INFO | Perf report for the last 5.13937 seconds: | 20260506 14:04:58.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:04:58.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:58.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:58.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:58.239 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 14:04:58.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:58.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:58.239 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:04:58.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:58.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:04:58.239 | INFO | Uploaded 36GiB to GPU in 2.04023s (17.6451GiB/s) | 20260506 14:04:58.239 | INFO | Downloaded 34GiB to GPU in 1.96767s (17.2794GiB/s) | 20260506 14:04:58.239 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:04:58.239 | INFO | Total mem usage: 17.4 GiB | 20260506 14:04:58.239 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:04:58.239 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -10.6%) | 20260506 14:04:58.239 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:04:58.239 | INFO | High Water Breakdown: | 20260506 14:04:58.239 | INFO | 61.2% : 9.64 GiB ( 88.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:04:58.239 | INFO | 38.6% : 6.08 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:04:58.239 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:04:58.239 | INFO | ------------------------------------------ | 20260506 14:04:58.239 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:04:58.239 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:04:58.239 | INFO | Total mem usage: 26.5 GiB | 20260506 14:04:58.239 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 14:04:58.239 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:04:58.239 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 14:04:58.239 | INFO | High Water Breakdown: | 20260506 14:04:58.239 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:04:58.239 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:04:58.239 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:04:58.239 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:04:58.239 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:04:58.239 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:04:58.239 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:04:58.239 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:04:58.239 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:04:58.239 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:04:58.239 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:04:58.239 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:04:58.239 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:04:58.239 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:04:58.239 | INFO | ------------------------------------------ | 20260506 14:04:58.239 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:04:58.239 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:04:58.239 | INFO | Total mem usage: 25.6 GiB | 20260506 14:04:58.239 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:04:58.239 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:04:58.239 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:04:58.239 | INFO | High Water Breakdown: | 20260506 14:04:58.239 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:04:58.239 | INFO | ------------------------------------------ | 20260506 14:04:58.239 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:04:58.239 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:04:58.239 | INFO | Total mem usage: 15.7 GiB | 20260506 14:04:58.239 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:04:58.239 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:04:58.239 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 14:04:58.239 | INFO | High Water Breakdown: | 20260506 14:04:58.239 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:04:58.239 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:04:58.239 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:04:58.239 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:04:58.239 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:04:58.239 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:04:58.239 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:04:58.239 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:04:58.239 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:04:58.239 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:04:58.239 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:04:58.239 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:04:58.239 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:04:58.239 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:04:58.239 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:04:58.239 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:04:58.239 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:04:58.239 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:04:58.240 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:04:58.240 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:04:58.240 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:04:58.240 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:04:58.240 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:04:58.240 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:04:58.240 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:04:58.240 | INFO | ------------------------------------------ | 20260506 14:04:58.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:04:58.240 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:04:58.240 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:04:58.240 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:04:58.240 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.56e+01, 1.35e+01 / 2.11e+00 | 20260506 14:04:58.240 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.02e-01, 1.01e-02 / 5.66e-03 | 20260506 14:04:58.240 | INFO | PerfReport: 11.59%, Baseline -- 3.66e+00 / 4.45e+00, 5.69e-01 / 3.45e-01 | 20260506 14:04:58.240 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.48e+00, 1.86e-01 / 1.47e-01 | 20260506 14:04:58.240 | INFO | PerfReport: 15.82%, DME -- 3.99e+01 / 4.41e+01, 3.98e+00 / 2.59e+00 | 20260506 14:04:58.240 | INFO | PerfReport: 56.67%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.43e+00 / 1.89e+00 | 20260506 14:04:58.240 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.36e+00 / 2.99e+00, 4.06e-01 / 2.24e-01 | 20260506 14:04:58.240 | INFO | PerfReport: 3.63%, Metrics -- 1.15e+00 / 1.53e+00, 2.92e-01 / 1.44e-01 | 20260506 14:04:58.240 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.01e-01 / 4.53e-01, 6.21e-02 / 3.10e-02 | 20260506 14:04:58.240 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:04:58.240 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:04:58.240 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:04:58.240 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 14:04:58.240 | INFO | PerfReport: 21.51%, Upload -- 2.10e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 14:04:58.240 | INFO | PerfReport: 21.08%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 14:04:58.240 | INFO | PerfReport: 32.25%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 14:04:58.240 | INFO | PerfReport: 1.60%, LockUpload -- 1.57e+00 / 0.00e+00, 6.43e+00 / 0.00e+00 | 20260506 14:04:58.240 | INFO | PerfReport: 1.13%, LockDownload -- 1.10e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260506 14:04:58.240 | INFO | PerfReport: 22.41%, LockCompute -- 2.19e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 14:04:58.240 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260506 14:04:58.357 | INFO | Starting a round of Baz IO | 20260506 14:05:01.517 | INFO | Populating RTMetrics report | 20260506 14:05:01.518 | INFO | RTMetrics report populated | 20260506 14:05:01.518 | INFO | Opening tmp RTMetrics json file | 20260506 14:05:01.519 | INFO | Serializing RTMetrics to json | 20260506 14:05:01.520 | INFO | Serialization done, swapping tmp file now... | 20260506 14:05:01.521 | INFO | RTMetrics done | 20260506 14:05:01.675 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.317 seconds | 20260506 14:05:01.675 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.78% / 0.02% / 61.21% | 20260506 14:05:01.675 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.74992% / 3.57628e-05% / 5.16574e-05% | 20260506 14:05:01.675 | INFO | Baz IO overall: | 20260506 14:05:01.675 | INFO | BytesWritten : 992389071544 | 20260506 14:05:01.675 | INFO | NumEvents : 480151717114 | 20260506 14:05:03.378 | INFO | Analyzing chunk frames = [1933312,1933824) | 20260506 14:05:03.378 | INFO | MemInfo: { | 20260506 14:05:03.378 | INFO | "gpu" : " 17 GB", | 20260506 14:05:03.378 | INFO | "huge" : "", | 20260506 14:05:03.378 | INFO | "huge_shared" : " 27 GB", | 20260506 14:05:03.378 | INFO | "malloc" : " 14 GB", | 20260506 14:05:03.378 | INFO | "pinned" : " 29 GB" | 20260506 14:05:03.378 | INFO | } | 20260506 14:05:03.378 | INFO | Perf report for the last 5.13936 seconds: | 20260506 14:05:03.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:05:03.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:03.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:03.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:03.378 | INFO | ANALYSIS : 63.2%, 2.9, 95.5ms, 0 | 20260506 14:05:03.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:03.378 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 14:05:03.378 | INFO | PRE_HQ : 61.2%, 1.9, 59.9ms, 0 | 20260506 14:05:03.378 | INFO | BAZWRITER : 63.9%, 4.7, 155.7ms, 0 | 20260506 14:05:03.378 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.2ms, 0 | 20260506 14:05:03.378 | INFO | Uploaded 36GiB to GPU in 2.08837s (17.2384GiB/s) | 20260506 14:05:03.378 | INFO | Downloaded 38GiB to GPU in 2.23548s (16.9986GiB/s) | 20260506 14:05:08.517 | INFO | Analyzing chunk frames = [1933824,1934336) | 20260506 14:05:08.517 | INFO | MemInfo: { | 20260506 14:05:08.517 | INFO | "gpu" : " 17 GB", | 20260506 14:05:08.517 | INFO | "huge" : "", | 20260506 14:05:08.517 | INFO | "huge_shared" : " 27 GB", | 20260506 14:05:08.517 | INFO | "malloc" : " 14 GB", | 20260506 14:05:08.517 | INFO | "pinned" : " 29 GB" | 20260506 14:05:08.517 | INFO | } | 20260506 14:05:08.518 | INFO | Perf report for the last 5.1397 seconds: | 20260506 14:05:08.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:05:08.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:08.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:08.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:08.518 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:05:08.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:08.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:08.518 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 14:05:08.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:08.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:08.518 | INFO | Uploaded 36GiB to GPU in 2.02823s (17.7495GiB/s) | 20260506 14:05:08.518 | INFO | Downloaded 34GiB to GPU in 1.945s (17.4807GiB/s) | 20260506 14:05:13.657 | INFO | Analyzing chunk frames = [1934336,1934848) | 20260506 14:05:13.657 | INFO | MemInfo: { | 20260506 14:05:13.657 | INFO | "gpu" : " 17 GB", | 20260506 14:05:13.657 | INFO | "huge" : "", | 20260506 14:05:13.657 | INFO | "huge_shared" : " 27 GB", | 20260506 14:05:13.657 | INFO | "malloc" : " 14 GB", | 20260506 14:05:13.657 | INFO | "pinned" : " 29 GB" | 20260506 14:05:13.657 | INFO | } | 20260506 14:05:13.657 | INFO | Perf report for the last 5.13966 seconds: | 20260506 14:05:13.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:05:13.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:13.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:13.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:13.657 | INFO | ANALYSIS : 61.4%, 3.0, 95.0ms, 0 | 20260506 14:05:13.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:13.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:13.657 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:05:13.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:13.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:13.657 | INFO | Uploaded 36GiB to GPU in 2.01414s (17.8736GiB/s) | 20260506 14:05:13.658 | INFO | Downloaded 34GiB to GPU in 1.93219s (17.5966GiB/s) | 20260506 14:05:18.795 | INFO | Analyzing chunk frames = [1934848,1935360) | 20260506 14:05:18.795 | INFO | MemInfo: { | 20260506 14:05:18.795 | INFO | "gpu" : " 17 GB", | 20260506 14:05:18.795 | INFO | "huge" : "", | 20260506 14:05:18.796 | INFO | "huge_shared" : " 27 GB", | 20260506 14:05:18.796 | INFO | "malloc" : " 14 GB", | 20260506 14:05:18.796 | INFO | "pinned" : " 29 GB" | 20260506 14:05:18.796 | INFO | } | 20260506 14:05:18.796 | INFO | Perf report for the last 5.13847 seconds: | 20260506 14:05:18.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:05:18.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:18.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:18.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:18.796 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:05:18.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:18.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:18.796 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 14:05:18.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:18.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:18.796 | INFO | Uploaded 36GiB to GPU in 1.98467s (18.1391GiB/s) | 20260506 14:05:18.796 | INFO | Downloaded 34GiB to GPU in 1.91278s (17.7752GiB/s) | 20260506 14:05:23.934 | INFO | Analyzing chunk frames = [1935360,1935872) | 20260506 14:05:23.934 | INFO | MemInfo: { | 20260506 14:05:23.934 | INFO | "gpu" : " 17 GB", | 20260506 14:05:23.934 | INFO | "huge" : "", | 20260506 14:05:23.935 | INFO | "huge_shared" : " 27 GB", | 20260506 14:05:23.935 | INFO | "malloc" : " 14 GB", | 20260506 14:05:23.935 | INFO | "pinned" : " 29 GB" | 20260506 14:05:23.935 | INFO | } | 20260506 14:05:23.935 | INFO | Perf report for the last 5.13893 seconds: | 20260506 14:05:23.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:05:23.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:23.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:23.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:23.935 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 14:05:23.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:23.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:23.935 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:05:23.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:23.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:23.935 | INFO | Uploaded 36GiB to GPU in 2.04238s (17.6265GiB/s) | 20260506 14:05:23.935 | INFO | Downloaded 34GiB to GPU in 1.93603s (17.5617GiB/s) | 20260506 14:05:29.075 | INFO | Analyzing chunk frames = [1935872,1936384) | 20260506 14:05:29.075 | INFO | MemInfo: { | 20260506 14:05:29.075 | INFO | "gpu" : " 17 GB", | 20260506 14:05:29.075 | INFO | "huge" : "", | 20260506 14:05:29.075 | INFO | "huge_shared" : " 27 GB", | 20260506 14:05:29.075 | INFO | "malloc" : " 14 GB", | 20260506 14:05:29.075 | INFO | "pinned" : " 29 GB" | 20260506 14:05:29.075 | INFO | } | 20260506 14:05:29.075 | INFO | Perf report for the last 5.14029 seconds: | 20260506 14:05:29.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:05:29.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:29.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:29.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:29.075 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:05:29.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:29.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:29.075 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 14:05:29.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:29.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:29.075 | INFO | Uploaded 36GiB to GPU in 2.04007s (17.6465GiB/s) | 20260506 14:05:29.075 | INFO | Downloaded 34GiB to GPU in 1.89689s (17.9241GiB/s) | 20260506 14:05:34.214 | INFO | Analyzing chunk frames = [1936384,1936896) | 20260506 14:05:34.214 | INFO | MemInfo: { | 20260506 14:05:34.214 | INFO | "gpu" : " 17 GB", | 20260506 14:05:34.214 | INFO | "huge" : "", | 20260506 14:05:34.214 | INFO | "huge_shared" : " 27 GB", | 20260506 14:05:34.214 | INFO | "malloc" : " 14 GB", | 20260506 14:05:34.214 | INFO | "pinned" : " 29 GB" | 20260506 14:05:34.215 | INFO | } | 20260506 14:05:34.215 | INFO | Perf report for the last 5.1396 seconds: | 20260506 14:05:34.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:05:34.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:34.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:34.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:34.215 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:05:34.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:34.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:34.215 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:05:34.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:34.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:34.215 | INFO | Uploaded 36GiB to GPU in 1.98099s (18.1728GiB/s) | 20260506 14:05:34.215 | INFO | Downloaded 34GiB to GPU in 2.00091s (16.9923GiB/s) | 20260506 14:05:39.355 | INFO | Analyzing chunk frames = [1936896,1937408) | 20260506 14:05:39.355 | INFO | MemInfo: { | 20260506 14:05:39.355 | INFO | "gpu" : " 17 GB", | 20260506 14:05:39.355 | INFO | "huge" : "", | 20260506 14:05:39.355 | INFO | "huge_shared" : " 27 GB", | 20260506 14:05:39.355 | INFO | "malloc" : " 14 GB", | 20260506 14:05:39.355 | INFO | "pinned" : " 29 GB" | 20260506 14:05:39.355 | INFO | } | 20260506 14:05:39.355 | INFO | Perf report for the last 5.14028 seconds: | 20260506 14:05:39.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:05:39.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:39.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:39.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:39.355 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 14:05:39.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:39.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:39.355 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:05:39.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:39.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:39.355 | INFO | Uploaded 36GiB to GPU in 2.02607s (17.7684GiB/s) | 20260506 14:05:39.355 | INFO | Downloaded 34GiB to GPU in 1.98141s (17.1595GiB/s) | 20260506 14:05:39.469 | INFO | Starting a round of Baz IO | 20260506 14:05:42.621 | INFO | Populating RTMetrics report | 20260506 14:05:42.622 | INFO | RTMetrics report populated | 20260506 14:05:42.622 | INFO | Opening tmp RTMetrics json file | 20260506 14:05:42.640 | INFO | Serializing RTMetrics to json | 20260506 14:05:42.644 | INFO | Serialization done, swapping tmp file now... | 20260506 14:05:42.645 | INFO | RTMetrics done | 20260506 14:05:42.984 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.514 seconds | 20260506 14:05:42.984 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.76% / 0.02% / 61.22% | 20260506 14:05:42.984 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.73863% / 1.98682e-05% / 7.94729e-05% | 20260506 14:05:42.984 | INFO | Baz IO overall: | 20260506 14:05:42.984 | INFO | BytesWritten : 994654768682 | 20260506 14:05:42.984 | INFO | NumEvents : 481249749765 | 20260506 14:05:44.493 | INFO | Analyzing chunk frames = [1937408,1937920) | 20260506 14:05:44.493 | INFO | MemInfo: { | 20260506 14:05:44.493 | INFO | "gpu" : " 17 GB", | 20260506 14:05:44.493 | INFO | "huge" : "", | 20260506 14:05:44.493 | INFO | "huge_shared" : " 27 GB", | 20260506 14:05:44.493 | INFO | "malloc" : " 14 GB", | 20260506 14:05:44.493 | INFO | "pinned" : " 29 GB" | 20260506 14:05:44.493 | INFO | } | 20260506 14:05:44.493 | INFO | Perf report for the last 5.1379 seconds: | 20260506 14:05:44.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:05:44.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:44.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:44.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:44.493 | INFO | ANALYSIS : 63.0%, 2.9, 95.1ms, 0 | 20260506 14:05:44.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:44.493 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 14:05:44.493 | INFO | PRE_HQ : 61.2%, 1.9, 59.8ms, 0 | 20260506 14:05:44.493 | INFO | BAZWRITER : 67.8%, 4.8, 166.5ms, 0 | 20260506 14:05:44.493 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 14:05:44.493 | INFO | Uploaded 36GiB to GPU in 2.08966s (17.2277GiB/s) | 20260506 14:05:44.493 | INFO | Downloaded 38GiB to GPU in 2.22968s (17.0428GiB/s) | 20260506 14:05:49.632 | INFO | Analyzing chunk frames = [1937920,1938432) | 20260506 14:05:49.632 | INFO | MemInfo: { | 20260506 14:05:49.632 | INFO | "gpu" : " 17 GB", | 20260506 14:05:49.632 | INFO | "huge" : "", | 20260506 14:05:49.632 | INFO | "huge_shared" : " 27 GB", | 20260506 14:05:49.632 | INFO | "malloc" : " 14 GB", | 20260506 14:05:49.632 | INFO | "pinned" : " 29 GB" | 20260506 14:05:49.632 | INFO | } | 20260506 14:05:49.632 | INFO | Perf report for the last 5.13904 seconds: | 20260506 14:05:49.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:05:49.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:49.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:49.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:49.632 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 14:05:49.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:49.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:49.632 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:05:49.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:49.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:49.632 | INFO | Uploaded 36GiB to GPU in 2.04315s (17.6198GiB/s) | 20260506 14:05:49.632 | INFO | Downloaded 34GiB to GPU in 1.94766s (17.4569GiB/s) | 20260506 14:05:54.771 | INFO | Analyzing chunk frames = [1938432,1938944) | 20260506 14:05:54.771 | INFO | MemInfo: { | 20260506 14:05:54.771 | INFO | "gpu" : " 17 GB", | 20260506 14:05:54.771 | INFO | "huge" : "", | 20260506 14:05:54.771 | INFO | "huge_shared" : " 27 GB", | 20260506 14:05:54.771 | INFO | "malloc" : " 14 GB", | 20260506 14:05:54.771 | INFO | "pinned" : " 29 GB" | 20260506 14:05:54.771 | INFO | } | 20260506 14:05:54.772 | INFO | Perf report for the last 5.13975 seconds: | 20260506 14:05:54.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:05:54.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:54.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:54.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:54.772 | INFO | ANALYSIS : 61.4%, 3.0, 95.1ms, 0 | 20260506 14:05:54.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:54.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:54.772 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:05:54.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:54.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:54.772 | INFO | Uploaded 36GiB to GPU in 2.00887s (17.9205GiB/s) | 20260506 14:05:54.772 | INFO | Downloaded 34GiB to GPU in 1.93241s (17.5946GiB/s) | 20260506 14:05:59.911 | INFO | Analyzing chunk frames = [1938944,1939456) | 20260506 14:05:59.911 | INFO | MemInfo: { | 20260506 14:05:59.911 | INFO | "gpu" : " 17 GB", | 20260506 14:05:59.911 | INFO | "huge" : "", | 20260506 14:05:59.911 | INFO | "huge_shared" : " 27 GB", | 20260506 14:05:59.911 | INFO | "malloc" : " 14 GB", | 20260506 14:05:59.911 | INFO | "pinned" : " 29 GB" | 20260506 14:05:59.911 | INFO | } | 20260506 14:05:59.911 | INFO | Perf report for the last 5.13952 seconds: | 20260506 14:05:59.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:05:59.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:59.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:59.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:59.911 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 14:05:59.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:59.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:59.911 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:05:59.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:59.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:05:59.911 | INFO | Uploaded 36GiB to GPU in 1.98026s (18.1794GiB/s) | 20260506 14:05:59.911 | INFO | Downloaded 34GiB to GPU in 1.91458s (17.7585GiB/s) | 20260506 14:06:05.050 | INFO | Analyzing chunk frames = [1939456,1939968) | 20260506 14:06:05.050 | INFO | MemInfo: { | 20260506 14:06:05.050 | INFO | "gpu" : " 17 GB", | 20260506 14:06:05.050 | INFO | "huge" : "", | 20260506 14:06:05.050 | INFO | "huge_shared" : " 27 GB", | 20260506 14:06:05.050 | INFO | "malloc" : " 14 GB", | 20260506 14:06:05.050 | INFO | "pinned" : " 29 GB" | 20260506 14:06:05.050 | INFO | } | 20260506 14:06:05.050 | INFO | Perf report for the last 5.13902 seconds: | 20260506 14:06:05.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:06:05.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:05.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:05.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:05.050 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 14:06:05.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:05.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:05.050 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:06:05.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:05.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:05.050 | INFO | Uploaded 36GiB to GPU in 2.00831s (17.9256GiB/s) | 20260506 14:06:05.050 | INFO | Downloaded 34GiB to GPU in 1.92361s (17.6751GiB/s) | 20260506 14:06:10.190 | INFO | Analyzing chunk frames = [1939968,1940480) | 20260506 14:06:10.190 | INFO | MemInfo: { | 20260506 14:06:10.190 | INFO | "gpu" : " 17 GB", | 20260506 14:06:10.190 | INFO | "huge" : "", | 20260506 14:06:10.190 | INFO | "huge_shared" : " 27 GB", | 20260506 14:06:10.190 | INFO | "malloc" : " 14 GB", | 20260506 14:06:10.190 | INFO | "pinned" : " 29 GB" | 20260506 14:06:10.190 | INFO | } | 20260506 14:06:10.190 | INFO | Perf report for the last 5.13969 seconds: | 20260506 14:06:10.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:06:10.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:10.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:10.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:10.190 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:06:10.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:10.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:10.190 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 14:06:10.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:10.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:10.190 | INFO | Uploaded 36GiB to GPU in 2.04845s (17.5743GiB/s) | 20260506 14:06:10.190 | INFO | Downloaded 34GiB to GPU in 1.92253s (17.685GiB/s) | 20260506 14:06:15.329 | INFO | Analyzing chunk frames = [1940480,1940992) | 20260506 14:06:15.329 | INFO | MemInfo: { | 20260506 14:06:15.329 | INFO | "gpu" : " 17 GB", | 20260506 14:06:15.329 | INFO | "huge" : "", | 20260506 14:06:15.329 | INFO | "huge_shared" : " 27 GB", | 20260506 14:06:15.329 | INFO | "malloc" : " 14 GB", | 20260506 14:06:15.329 | INFO | "pinned" : " 29 GB" | 20260506 14:06:15.329 | INFO | } | 20260506 14:06:15.329 | INFO | Perf report for the last 5.13962 seconds: | 20260506 14:06:15.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:06:15.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:15.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:15.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:15.330 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 14:06:15.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:15.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:15.330 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:06:15.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:15.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:15.330 | INFO | Uploaded 36GiB to GPU in 2.02108s (17.8123GiB/s) | 20260506 14:06:15.330 | INFO | Downloaded 34GiB to GPU in 2.0065s (16.945GiB/s) | 20260506 14:06:20.468 | INFO | Analyzing chunk frames = [1940992,1941504) | 20260506 14:06:20.468 | INFO | MemInfo: { | 20260506 14:06:20.468 | INFO | "gpu" : " 17 GB", | 20260506 14:06:20.468 | INFO | "huge" : "", | 20260506 14:06:20.468 | INFO | "huge_shared" : " 27 GB", | 20260506 14:06:20.468 | INFO | "malloc" : " 14 GB", | 20260506 14:06:20.468 | INFO | "pinned" : " 29 GB" | 20260506 14:06:20.468 | INFO | } | 20260506 14:06:20.468 | INFO | Perf report for the last 5.13867 seconds: | 20260506 14:06:20.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:06:20.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:20.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:20.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:20.468 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 14:06:20.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:20.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:20.468 | INFO | PRE_HQ : 32.1%, 1.0, 16.8ms, 0 | 20260506 14:06:20.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:20.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:20.468 | INFO | Uploaded 36GiB to GPU in 2.03203s (17.7162GiB/s) | 20260506 14:06:20.469 | INFO | Downloaded 34GiB to GPU in 1.99904s (17.0082GiB/s) | 20260506 14:06:20.469 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:06:20.469 | INFO | Total mem usage: 17.4 GiB | 20260506 14:06:20.469 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:06:20.469 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -12.2%) | 20260506 14:06:20.469 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:06:20.469 | INFO | High Water Breakdown: | 20260506 14:06:20.469 | INFO | 61% : 9.47 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:06:20.469 | INFO | 38.8% : 6.03 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:06:20.469 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:06:20.469 | INFO | ------------------------------------------ | 20260506 14:06:20.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:06:20.469 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:06:20.469 | INFO | Total mem usage: 26.5 GiB | 20260506 14:06:20.469 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 14:06:20.469 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:06:20.469 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 14:06:20.469 | INFO | High Water Breakdown: | 20260506 14:06:20.469 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:06:20.469 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:06:20.469 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:06:20.469 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:06:20.469 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:06:20.469 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:06:20.469 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:06:20.469 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:06:20.469 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:06:20.469 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:06:20.469 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:06:20.469 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:06:20.469 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:06:20.469 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:06:20.469 | INFO | ------------------------------------------ | 20260506 14:06:20.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:06:20.469 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:06:20.469 | INFO | Total mem usage: 25.6 GiB | 20260506 14:06:20.469 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:06:20.469 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:06:20.469 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:06:20.469 | INFO | High Water Breakdown: | 20260506 14:06:20.469 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:06:20.469 | INFO | ------------------------------------------ | 20260506 14:06:20.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:06:20.469 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:06:20.469 | INFO | Total mem usage: 15.7 GiB | 20260506 14:06:20.469 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:06:20.469 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:06:20.469 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 14:06:20.469 | INFO | High Water Breakdown: | 20260506 14:06:20.469 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:06:20.469 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:06:20.469 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:06:20.469 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:06:20.469 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:06:20.469 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:06:20.469 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:06:20.469 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:06:20.469 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:06:20.469 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:06:20.469 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:06:20.469 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:06:20.469 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:06:20.469 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:06:20.470 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:06:20.470 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:06:20.470 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:06:20.470 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:06:20.470 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:06:20.470 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:06:20.470 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:06:20.470 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:06:20.470 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:06:20.470 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:06:20.470 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:06:20.470 | INFO | ------------------------------------------ | 20260506 14:06:20.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:06:20.470 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:06:20.470 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:06:20.470 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:06:20.470 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.55e+01, 1.33e+01 / 1.95e+00 | 20260506 14:06:20.470 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.01e-01, 1.00e-02 / 6.28e-03 | 20260506 14:06:20.470 | INFO | PerfReport: 11.58%, Baseline -- 3.66e+00 / 4.52e+00, 5.70e-01 / 2.72e-01 | 20260506 14:06:20.470 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.46e+00, 1.88e-01 / 1.46e-01 | 20260506 14:06:20.470 | INFO | PerfReport: 15.78%, DME -- 3.98e+01 / 4.41e+01, 3.85e+00 / 2.25e+00 | 20260506 14:06:20.470 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.51e+00 / 2.06e+00 | 20260506 14:06:20.470 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.36e+00 / 2.99e+00, 4.15e-01 / 2.35e-01 | 20260506 14:06:20.470 | INFO | PerfReport: 3.62%, Metrics -- 1.14e+00 / 1.50e+00, 2.90e-01 / 1.54e-01 | 20260506 14:06:20.470 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.00e-01 / 4.61e-01, 6.28e-02 / 3.97e-02 | 20260506 14:06:20.470 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:06:20.470 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:06:20.470 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:06:20.470 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 14:06:20.470 | INFO | PerfReport: 21.58%, Upload -- 2.11e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 14:06:20.470 | INFO | PerfReport: 21.10%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 14:06:20.470 | INFO | PerfReport: 32.26%, Compute -- 3.16e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 14:06:20.470 | INFO | PerfReport: 1.65%, LockUpload -- 1.62e+00 / 0.00e+00, 6.58e+00 / 0.00e+00 | 20260506 14:06:20.470 | INFO | PerfReport: 1.07%, LockDownload -- 1.05e+00 / 0.00e+00, 2.67e+00 / 0.00e+00 | 20260506 14:06:20.470 | INFO | PerfReport: 22.32%, LockCompute -- 2.18e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:06:20.470 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260506 14:06:20.585 | INFO | Starting a round of Baz IO | 20260506 14:06:23.779 | INFO | Populating RTMetrics report | 20260506 14:06:23.780 | INFO | RTMetrics report populated | 20260506 14:06:23.780 | INFO | Opening tmp RTMetrics json file | 20260506 14:06:23.820 | INFO | Serializing RTMetrics to json | 20260506 14:06:23.862 | INFO | Serialization done, swapping tmp file now... | 20260506 14:06:23.920 | INFO | RTMetrics done | 20260506 14:06:24.302 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.716 seconds | 20260506 14:06:24.302 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.74% / 0.02% / 61.24% | 20260506 14:06:24.302 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.74022% / 2.38419e-05% / 5.96046e-05% | 20260506 14:06:24.302 | INFO | Baz IO overall: | 20260506 14:06:24.302 | INFO | BytesWritten : 996921616649 | 20260506 14:06:24.302 | INFO | NumEvents : 482348727813 | 20260506 14:06:25.607 | INFO | Analyzing chunk frames = [1941504,1942016) | 20260506 14:06:25.607 | INFO | MemInfo: { | 20260506 14:06:25.607 | INFO | "gpu" : " 17 GB", | 20260506 14:06:25.607 | INFO | "huge" : "", | 20260506 14:06:25.607 | INFO | "huge_shared" : " 27 GB", | 20260506 14:06:25.607 | INFO | "malloc" : " 14 GB", | 20260506 14:06:25.607 | INFO | "pinned" : " 29 GB" | 20260506 14:06:25.607 | INFO | } | 20260506 14:06:25.607 | INFO | Perf report for the last 5.13902 seconds: | 20260506 14:06:25.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:06:25.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:25.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:25.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:25.607 | INFO | ANALYSIS : 63.5%, 2.8, 94.7ms, 0 | 20260506 14:06:25.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:25.607 | INFO | RT_METRICS : 49.1%, 1.0, 25.7ms, 0 | 20260506 14:06:25.607 | INFO | PRE_HQ : 61.9%, 2.0, 63.1ms, 0 | 20260506 14:06:25.607 | INFO | BAZWRITER : 71.8%, 4.8, 176.2ms, 0 | 20260506 14:06:25.607 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 14:06:25.607 | INFO | Uploaded 36GiB to GPU in 2.07222s (17.3727GiB/s) | 20260506 14:06:25.608 | INFO | Downloaded 38GiB to GPU in 2.22538s (17.0758GiB/s) | 20260506 14:06:30.747 | INFO | Analyzing chunk frames = [1942016,1942528) | 20260506 14:06:30.747 | INFO | MemInfo: { | 20260506 14:06:30.747 | INFO | "gpu" : " 17 GB", | 20260506 14:06:30.747 | INFO | "huge" : "", | 20260506 14:06:30.747 | INFO | "huge_shared" : " 27 GB", | 20260506 14:06:30.747 | INFO | "malloc" : " 14 GB", | 20260506 14:06:30.747 | INFO | "pinned" : " 29 GB" | 20260506 14:06:30.747 | INFO | } | 20260506 14:06:30.747 | INFO | Perf report for the last 5.14018 seconds: | 20260506 14:06:30.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:06:30.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:30.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:30.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:30.747 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:06:30.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:30.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:30.747 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 14:06:30.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:30.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:30.748 | INFO | Uploaded 36GiB to GPU in 1.99327s (18.0608GiB/s) | 20260506 14:06:30.748 | INFO | Downloaded 34GiB to GPU in 1.94076s (17.5189GiB/s) | 20260506 14:06:35.886 | INFO | Analyzing chunk frames = [1942528,1943040) | 20260506 14:06:35.886 | INFO | MemInfo: { | 20260506 14:06:35.886 | INFO | "gpu" : " 17 GB", | 20260506 14:06:35.886 | INFO | "huge" : "", | 20260506 14:06:35.886 | INFO | "huge_shared" : " 27 GB", | 20260506 14:06:35.886 | INFO | "malloc" : " 14 GB", | 20260506 14:06:35.886 | INFO | "pinned" : " 29 GB" | 20260506 14:06:35.886 | INFO | } | 20260506 14:06:35.886 | INFO | Perf report for the last 5.13878 seconds: | 20260506 14:06:35.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:06:35.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:35.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:35.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:35.886 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 14:06:35.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:35.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:35.886 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:06:35.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:35.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:35.886 | INFO | Uploaded 36GiB to GPU in 1.9754s (18.2242GiB/s) | 20260506 14:06:35.886 | INFO | Downloaded 34GiB to GPU in 1.9458s (17.4735GiB/s) | 20260506 14:06:41.026 | INFO | Analyzing chunk frames = [1943040,1943552) | 20260506 14:06:41.026 | INFO | MemInfo: { | 20260506 14:06:41.026 | INFO | "gpu" : " 17 GB", | 20260506 14:06:41.026 | INFO | "huge" : "", | 20260506 14:06:41.026 | INFO | "huge_shared" : " 27 GB", | 20260506 14:06:41.026 | INFO | "malloc" : " 14 GB", | 20260506 14:06:41.026 | INFO | "pinned" : " 29 GB" | 20260506 14:06:41.026 | INFO | } | 20260506 14:06:41.026 | INFO | Perf report for the last 5.14028 seconds: | 20260506 14:06:41.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:06:41.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:41.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:41.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:41.027 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260506 14:06:41.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:41.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:41.027 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:06:41.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:41.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:41.027 | INFO | Uploaded 36GiB to GPU in 1.9906s (18.085GiB/s) | 20260506 14:06:41.027 | INFO | Downloaded 34GiB to GPU in 1.915s (17.7546GiB/s) | 20260506 14:06:46.165 | INFO | Analyzing chunk frames = [1943552,1944064) | 20260506 14:06:46.165 | INFO | MemInfo: { | 20260506 14:06:46.165 | INFO | "gpu" : " 17 GB", | 20260506 14:06:46.165 | INFO | "huge" : "", | 20260506 14:06:46.165 | INFO | "huge_shared" : " 27 GB", | 20260506 14:06:46.165 | INFO | "malloc" : " 14 GB", | 20260506 14:06:46.165 | INFO | "pinned" : " 29 GB" | 20260506 14:06:46.165 | INFO | } | 20260506 14:06:46.165 | INFO | Perf report for the last 5.13848 seconds: | 20260506 14:06:46.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:06:46.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:46.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:46.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:46.165 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:06:46.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:46.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:46.165 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:06:46.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:46.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:46.165 | INFO | Uploaded 36GiB to GPU in 1.98323s (18.1522GiB/s) | 20260506 14:06:46.165 | INFO | Downloaded 34GiB to GPU in 1.92997s (17.6168GiB/s) | 20260506 14:06:51.304 | INFO | Analyzing chunk frames = [1944064,1944576) | 20260506 14:06:51.304 | INFO | MemInfo: { | 20260506 14:06:51.304 | INFO | "gpu" : " 17 GB", | 20260506 14:06:51.304 | INFO | "huge" : "", | 20260506 14:06:51.304 | INFO | "huge_shared" : " 27 GB", | 20260506 14:06:51.304 | INFO | "malloc" : " 14 GB", | 20260506 14:06:51.304 | INFO | "pinned" : " 29 GB" | 20260506 14:06:51.304 | INFO | } | 20260506 14:06:51.304 | INFO | Perf report for the last 5.13872 seconds: | 20260506 14:06:51.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:06:51.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:51.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:51.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:51.304 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 14:06:51.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:51.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:51.304 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:06:51.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:51.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:51.304 | INFO | Uploaded 36GiB to GPU in 2.07097s (17.3832GiB/s) | 20260506 14:06:51.304 | INFO | Downloaded 34GiB to GPU in 1.97565s (17.2096GiB/s) | 20260506 14:06:56.443 | INFO | Analyzing chunk frames = [1944576,1945088) | 20260506 14:06:56.443 | INFO | MemInfo: { | 20260506 14:06:56.443 | INFO | "gpu" : " 17 GB", | 20260506 14:06:56.443 | INFO | "huge" : "", | 20260506 14:06:56.443 | INFO | "huge_shared" : " 27 GB", | 20260506 14:06:56.443 | INFO | "malloc" : " 14 GB", | 20260506 14:06:56.443 | INFO | "pinned" : " 29 GB" | 20260506 14:06:56.443 | INFO | } | 20260506 14:06:56.443 | INFO | Perf report for the last 5.13942 seconds: | 20260506 14:06:56.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:06:56.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:56.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:56.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:56.443 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:06:56.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:56.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:56.443 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 14:06:56.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:56.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:06:56.444 | INFO | Uploaded 36GiB to GPU in 2.01542s (17.8623GiB/s) | 20260506 14:06:56.444 | INFO | Downloaded 34GiB to GPU in 2.02554s (16.7857GiB/s) | 20260506 14:07:01.582 | INFO | Analyzing chunk frames = [1945088,1945600) | 20260506 14:07:01.582 | INFO | MemInfo: { | 20260506 14:07:01.582 | INFO | "gpu" : " 17 GB", | 20260506 14:07:01.582 | INFO | "huge" : "", | 20260506 14:07:01.582 | INFO | "huge_shared" : " 27 GB", | 20260506 14:07:01.582 | INFO | "malloc" : " 14 GB", | 20260506 14:07:01.582 | INFO | "pinned" : " 29 GB" | 20260506 14:07:01.582 | INFO | } | 20260506 14:07:01.582 | INFO | Perf report for the last 5.13914 seconds: | 20260506 14:07:01.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:07:01.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:01.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:01.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:01.583 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 14:07:01.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:01.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:01.583 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 14:07:01.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:01.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:01.583 | INFO | Uploaded 36GiB to GPU in 2.02976s (17.7361GiB/s) | 20260506 14:07:01.583 | INFO | Downloaded 34GiB to GPU in 1.97765s (17.1921GiB/s) | 20260506 14:07:01.712 | INFO | Starting a round of Baz IO | 20260506 14:07:04.858 | INFO | Populating RTMetrics report | 20260506 14:07:04.858 | INFO | RTMetrics report populated | 20260506 14:07:04.858 | INFO | Opening tmp RTMetrics json file | 20260506 14:07:04.883 | INFO | Serializing RTMetrics to json | 20260506 14:07:04.933 | INFO | Serialization done, swapping tmp file now... | 20260506 14:07:04.971 | INFO | RTMetrics done | 20260506 14:07:05.238 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.526 seconds | 20260506 14:07:05.238 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.73% / 0.02% / 61.26% | 20260506 14:07:05.238 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.72834% / 1.98682e-05% / 5.5631e-05% | 20260506 14:07:05.238 | INFO | Baz IO overall: | 20260506 14:07:05.238 | INFO | BytesWritten : 999187534489 | 20260506 14:07:05.238 | INFO | NumEvents : 483447022583 | 20260506 14:07:06.722 | INFO | Analyzing chunk frames = [1945600,1946112) | 20260506 14:07:06.722 | INFO | MemInfo: { | 20260506 14:07:06.722 | INFO | "gpu" : " 17 GB", | 20260506 14:07:06.722 | INFO | "huge" : "", | 20260506 14:07:06.722 | INFO | "huge_shared" : " 27 GB", | 20260506 14:07:06.722 | INFO | "malloc" : " 14 GB", | 20260506 14:07:06.722 | INFO | "pinned" : " 29 GB" | 20260506 14:07:06.722 | INFO | } | 20260506 14:07:06.722 | INFO | Perf report for the last 5.13953 seconds: | 20260506 14:07:06.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:07:06.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:06.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:06.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:06.722 | INFO | ANALYSIS : 63.1%, 2.9, 95.2ms, 0 | 20260506 14:07:06.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:06.722 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260506 14:07:06.722 | INFO | PRE_HQ : 61.3%, 1.9, 60.1ms, 0 | 20260506 14:07:06.722 | INFO | BAZWRITER : 68.0%, 4.8, 169.4ms, 0 | 20260506 14:07:06.722 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 14:07:06.722 | INFO | Uploaded 36GiB to GPU in 2.06129s (17.4648GiB/s) | 20260506 14:07:06.722 | INFO | Downloaded 38GiB to GPU in 2.22016s (17.1159GiB/s) | 20260506 14:07:11.862 | INFO | Analyzing chunk frames = [1946112,1946624) | 20260506 14:07:11.862 | INFO | MemInfo: { | 20260506 14:07:11.862 | INFO | "gpu" : " 17 GB", | 20260506 14:07:11.862 | INFO | "huge" : "", | 20260506 14:07:11.862 | INFO | "huge_shared" : " 27 GB", | 20260506 14:07:11.862 | INFO | "malloc" : " 14 GB", | 20260506 14:07:11.862 | INFO | "pinned" : " 29 GB" | 20260506 14:07:11.862 | INFO | } | 20260506 14:07:11.862 | INFO | Perf report for the last 5.14021 seconds: | 20260506 14:07:11.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:07:11.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:11.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:11.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:11.862 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 14:07:11.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:11.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:11.862 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:07:11.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:11.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:11.862 | INFO | Uploaded 36GiB to GPU in 1.99032s (18.0876GiB/s) | 20260506 14:07:11.862 | INFO | Downloaded 34GiB to GPU in 1.94715s (17.4614GiB/s) | 20260506 14:07:17.001 | INFO | Analyzing chunk frames = [1946624,1947136) | 20260506 14:07:17.001 | INFO | MemInfo: { | 20260506 14:07:17.001 | INFO | "gpu" : " 17 GB", | 20260506 14:07:17.001 | INFO | "huge" : "", | 20260506 14:07:17.001 | INFO | "huge_shared" : " 27 GB", | 20260506 14:07:17.001 | INFO | "malloc" : " 14 GB", | 20260506 14:07:17.001 | INFO | "pinned" : " 29 GB" | 20260506 14:07:17.001 | INFO | } | 20260506 14:07:17.001 | INFO | Perf report for the last 5.1393 seconds: | 20260506 14:07:17.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:07:17.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:17.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:17.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:17.001 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 14:07:17.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:17.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:17.001 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:07:17.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:17.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:17.002 | INFO | Uploaded 36GiB to GPU in 1.94873s (18.4735GiB/s) | 20260506 14:07:17.002 | INFO | Downloaded 34GiB to GPU in 1.93982s (17.5274GiB/s) | 20260506 14:07:22.140 | INFO | Analyzing chunk frames = [1947136,1947648) | 20260506 14:07:22.140 | INFO | MemInfo: { | 20260506 14:07:22.140 | INFO | "gpu" : " 17 GB", | 20260506 14:07:22.140 | INFO | "huge" : "", | 20260506 14:07:22.141 | INFO | "huge_shared" : " 27 GB", | 20260506 14:07:22.141 | INFO | "malloc" : " 14 GB", | 20260506 14:07:22.141 | INFO | "pinned" : " 29 GB" | 20260506 14:07:22.141 | INFO | } | 20260506 14:07:22.141 | INFO | Perf report for the last 5.13925 seconds: | 20260506 14:07:22.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:07:22.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:22.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:22.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:22.141 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:07:22.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:22.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:22.141 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 14:07:22.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:22.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:22.141 | INFO | Uploaded 36GiB to GPU in 2.00783s (17.9298GiB/s) | 20260506 14:07:22.141 | INFO | Downloaded 34GiB to GPU in 1.91896s (17.7179GiB/s) | 20260506 14:07:27.279 | INFO | Analyzing chunk frames = [1947648,1948160) | 20260506 14:07:27.279 | INFO | MemInfo: { | 20260506 14:07:27.279 | INFO | "gpu" : " 17 GB", | 20260506 14:07:27.279 | INFO | "huge" : "", | 20260506 14:07:27.279 | INFO | "huge_shared" : " 27 GB", | 20260506 14:07:27.279 | INFO | "malloc" : " 14 GB", | 20260506 14:07:27.279 | INFO | "pinned" : " 29 GB" | 20260506 14:07:27.279 | INFO | } | 20260506 14:07:27.280 | INFO | Perf report for the last 5.13891 seconds: | 20260506 14:07:27.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:07:27.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:27.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:27.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:27.280 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 14:07:27.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:27.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:27.280 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:07:27.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:27.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:27.280 | INFO | Uploaded 36GiB to GPU in 2.00969s (17.9132GiB/s) | 20260506 14:07:27.280 | INFO | Downloaded 34GiB to GPU in 1.94077s (17.5188GiB/s) | 20260506 14:07:32.418 | INFO | Analyzing chunk frames = [1948160,1948672) | 20260506 14:07:32.418 | INFO | MemInfo: { | 20260506 14:07:32.418 | INFO | "gpu" : " 17 GB", | 20260506 14:07:32.418 | INFO | "huge" : "", | 20260506 14:07:32.418 | INFO | "huge_shared" : " 27 GB", | 20260506 14:07:32.418 | INFO | "malloc" : " 14 GB", | 20260506 14:07:32.418 | INFO | "pinned" : " 29 GB" | 20260506 14:07:32.418 | INFO | } | 20260506 14:07:32.418 | INFO | Perf report for the last 5.13884 seconds: | 20260506 14:07:32.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:07:32.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:32.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:32.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:32.418 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 14:07:32.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:32.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:32.418 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:07:32.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:32.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:32.419 | INFO | Uploaded 36GiB to GPU in 2.03686s (17.6742GiB/s) | 20260506 14:07:32.419 | INFO | Downloaded 34GiB to GPU in 1.93259s (17.593GiB/s) | 20260506 14:07:37.558 | INFO | Analyzing chunk frames = [1948672,1949184) | 20260506 14:07:37.558 | INFO | MemInfo: { | 20260506 14:07:37.558 | INFO | "gpu" : " 17 GB", | 20260506 14:07:37.558 | INFO | "huge" : "", | 20260506 14:07:37.558 | INFO | "huge_shared" : " 27 GB", | 20260506 14:07:37.558 | INFO | "malloc" : " 14 GB", | 20260506 14:07:37.558 | INFO | "pinned" : " 29 GB" | 20260506 14:07:37.558 | INFO | } | 20260506 14:07:37.558 | INFO | Perf report for the last 5.13989 seconds: | 20260506 14:07:37.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:07:37.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:37.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:37.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:37.558 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 14:07:37.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:37.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:37.558 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 14:07:37.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:37.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:37.559 | INFO | Uploaded 36GiB to GPU in 2.00716s (17.9358GiB/s) | 20260506 14:07:37.559 | INFO | Downloaded 34GiB to GPU in 2.00426s (16.9638GiB/s) | 20260506 14:07:42.697 | INFO | Analyzing chunk frames = [1949184,1949696) | 20260506 14:07:42.697 | INFO | MemInfo: { | 20260506 14:07:42.697 | INFO | "gpu" : " 17 GB", | 20260506 14:07:42.697 | INFO | "huge" : "", | 20260506 14:07:42.697 | INFO | "huge_shared" : " 27 GB", | 20260506 14:07:42.697 | INFO | "malloc" : " 14 GB", | 20260506 14:07:42.697 | INFO | "pinned" : " 29 GB" | 20260506 14:07:42.697 | INFO | } | 20260506 14:07:42.697 | INFO | Perf report for the last 5.13892 seconds: | 20260506 14:07:42.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:07:42.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:42.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:42.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:42.697 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 14:07:42.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:42.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:42.697 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:07:42.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:42.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:42.698 | INFO | Uploaded 36GiB to GPU in 2.04034s (17.6441GiB/s) | 20260506 14:07:42.698 | INFO | Downloaded 34GiB to GPU in 1.98062s (17.1663GiB/s) | 20260506 14:07:42.698 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:07:42.698 | INFO | Total mem usage: 17.4 GiB | 20260506 14:07:42.698 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:07:42.698 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -10.6%) | 20260506 14:07:42.698 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:07:42.698 | INFO | High Water Breakdown: | 20260506 14:07:42.698 | INFO | 61.3% : 9.65 GiB ( 88.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:07:42.698 | INFO | 38.6% : 6.08 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:07:42.698 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:07:42.698 | INFO | ------------------------------------------ | 20260506 14:07:42.698 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:07:42.698 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:07:42.698 | INFO | Total mem usage: 26.5 GiB | 20260506 14:07:42.698 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 14:07:42.698 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:07:42.698 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 14:07:42.698 | INFO | High Water Breakdown: | 20260506 14:07:42.698 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:07:42.698 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:07:42.698 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:07:42.698 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:07:42.698 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:07:42.698 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:07:42.698 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:07:42.698 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:07:42.698 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:07:42.698 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:07:42.698 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:07:42.698 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:07:42.698 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:07:42.698 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:07:42.698 | INFO | ------------------------------------------ | 20260506 14:07:42.698 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:07:42.698 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:07:42.698 | INFO | Total mem usage: 25.6 GiB | 20260506 14:07:42.698 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:07:42.698 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:07:42.699 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:07:42.699 | INFO | High Water Breakdown: | 20260506 14:07:42.699 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:07:42.699 | INFO | ------------------------------------------ | 20260506 14:07:42.699 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:07:42.699 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:07:42.699 | INFO | Total mem usage: 15.7 GiB | 20260506 14:07:42.699 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:07:42.699 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:07:42.699 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 14:07:42.699 | INFO | High Water Breakdown: | 20260506 14:07:42.699 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:07:42.699 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:07:42.699 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:07:42.699 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:07:42.699 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:07:42.699 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:07:42.699 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:07:42.699 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:07:42.699 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:07:42.699 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:07:42.699 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:07:42.699 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:07:42.699 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:07:42.699 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:07:42.699 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:07:42.699 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:07:42.699 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:07:42.699 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:07:42.699 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:07:42.699 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:07:42.699 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:07:42.699 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:07:42.699 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:07:42.699 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:07:42.699 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:07:42.699 | INFO | ------------------------------------------ | 20260506 14:07:42.699 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:07:42.699 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:07:42.699 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:07:42.699 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:07:42.699 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.59e+01, 1.36e+01 / 2.18e+00 | 20260506 14:07:42.699 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.01e-01, 1.07e-02 / 5.57e-03 | 20260506 14:07:42.699 | INFO | PerfReport: 11.57%, Baseline -- 3.65e+00 / 4.40e+00, 5.68e-01 / 3.53e-01 | 20260506 14:07:42.699 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.46e+00, 1.87e-01 / 1.41e-01 | 20260506 14:07:42.699 | INFO | PerfReport: 15.83%, DME -- 3.99e+01 / 4.43e+01, 4.09e+00 / 2.44e+00 | 20260506 14:07:42.699 | INFO | PerfReport: 56.71%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.45e+00 / 2.01e+00 | 20260506 14:07:42.699 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.35e+00 / 2.91e+00, 4.09e-01 / 2.53e-01 | 20260506 14:07:42.699 | INFO | PerfReport: 3.62%, Metrics -- 1.14e+00 / 1.51e+00, 2.93e-01 / 1.92e-01 | 20260506 14:07:42.699 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.00e-01 / 4.53e-01, 6.33e-02 / 2.89e-02 | 20260506 14:07:42.699 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:07:42.699 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:07:42.699 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:07:42.699 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 14:07:42.699 | INFO | PerfReport: 21.47%, Upload -- 2.10e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 14:07:42.699 | INFO | PerfReport: 21.20%, Download -- 2.07e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 14:07:42.699 | INFO | PerfReport: 32.29%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 14:07:42.699 | INFO | PerfReport: 1.48%, LockUpload -- 1.44e+00 / 0.00e+00, 6.21e+00 / 0.00e+00 | 20260506 14:07:42.699 | INFO | PerfReport: 1.21%, LockDownload -- 1.19e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260506 14:07:42.699 | INFO | PerfReport: 22.33%, LockCompute -- 2.18e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:07:42.699 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.38e-02 / 0.00e+00 | 20260506 14:07:42.818 | INFO | Starting a round of Baz IO | 20260506 14:07:45.976 | INFO | Populating RTMetrics report | 20260506 14:07:45.976 | INFO | RTMetrics report populated | 20260506 14:07:45.976 | INFO | Opening tmp RTMetrics json file | 20260506 14:07:46.021 | INFO | Serializing RTMetrics to json | 20260506 14:07:46.024 | INFO | Serialization done, swapping tmp file now... | 20260506 14:07:46.028 | INFO | RTMetrics done | 20260506 14:07:46.335 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.516 seconds | 20260506 14:07:46.335 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.71% / 0.02% / 61.27% | 20260506 14:07:46.335 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.72046% / 1.58946e-05% / 5.16574e-05% | 20260506 14:07:46.335 | INFO | Baz IO overall: | 20260506 14:07:46.335 | INFO | BytesWritten : 1001453211683 | 20260506 14:07:46.335 | INFO | NumEvents : 484544755430 | 20260506 14:07:47.838 | INFO | Analyzing chunk frames = [1949696,1950208) | 20260506 14:07:47.838 | INFO | MemInfo: { | 20260506 14:07:47.838 | INFO | "gpu" : " 17 GB", | 20260506 14:07:47.838 | INFO | "huge" : "", | 20260506 14:07:47.838 | INFO | "huge_shared" : " 27 GB", | 20260506 14:07:47.838 | INFO | "malloc" : " 14 GB", | 20260506 14:07:47.838 | INFO | "pinned" : " 29 GB" | 20260506 14:07:47.838 | INFO | } | 20260506 14:07:47.838 | INFO | Perf report for the last 5.14108 seconds: | 20260506 14:07:47.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:07:47.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:47.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:47.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:47.838 | INFO | ANALYSIS : 63.2%, 2.9, 95.3ms, 0 | 20260506 14:07:47.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:47.839 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260506 14:07:47.839 | INFO | PRE_HQ : 60.9%, 1.9, 59.9ms, 0 | 20260506 14:07:47.839 | INFO | BAZWRITER : 67.8%, 4.8, 168.5ms, 0 | 20260506 14:07:47.839 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 14:07:47.839 | INFO | Uploaded 36GiB to GPU in 2.094s (17.192GiB/s) | 20260506 14:07:47.839 | INFO | Downloaded 38GiB to GPU in 2.23404s (17.0095GiB/s) | 20260506 14:07:52.977 | INFO | Analyzing chunk frames = [1950208,1950720) | 20260506 14:07:52.977 | INFO | MemInfo: { | 20260506 14:07:52.977 | INFO | "gpu" : " 17 GB", | 20260506 14:07:52.977 | INFO | "huge" : "", | 20260506 14:07:52.977 | INFO | "huge_shared" : " 27 GB", | 20260506 14:07:52.977 | INFO | "malloc" : " 14 GB", | 20260506 14:07:52.977 | INFO | "pinned" : " 29 GB" | 20260506 14:07:52.977 | INFO | } | 20260506 14:07:52.977 | INFO | Perf report for the last 5.13837 seconds: | 20260506 14:07:52.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:07:52.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:52.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:52.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:52.977 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:07:52.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:52.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:52.977 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:07:52.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:52.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:52.977 | INFO | Uploaded 36GiB to GPU in 2.03811s (17.6634GiB/s) | 20260506 14:07:52.977 | INFO | Downloaded 34GiB to GPU in 1.95152s (17.4223GiB/s) | 20260506 14:07:58.116 | INFO | Analyzing chunk frames = [1950720,1951232) | 20260506 14:07:58.116 | INFO | MemInfo: { | 20260506 14:07:58.116 | INFO | "gpu" : " 17 GB", | 20260506 14:07:58.116 | INFO | "huge" : "", | 20260506 14:07:58.116 | INFO | "huge_shared" : " 27 GB", | 20260506 14:07:58.116 | INFO | "malloc" : " 14 GB", | 20260506 14:07:58.116 | INFO | "pinned" : " 29 GB" | 20260506 14:07:58.116 | INFO | } | 20260506 14:07:58.116 | INFO | Perf report for the last 5.13913 seconds: | 20260506 14:07:58.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:07:58.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:58.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:58.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:58.116 | INFO | ANALYSIS : 61.4%, 3.0, 95.0ms, 0 | 20260506 14:07:58.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:58.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:58.116 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:07:58.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:58.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:07:58.116 | INFO | Uploaded 36GiB to GPU in 2.01023s (17.9084GiB/s) | 20260506 14:07:58.116 | INFO | Downloaded 34GiB to GPU in 1.93462s (17.5745GiB/s) | 20260506 14:08:03.254 | INFO | Analyzing chunk frames = [1951232,1951744) | 20260506 14:08:03.254 | INFO | MemInfo: { | 20260506 14:08:03.254 | INFO | "gpu" : " 17 GB", | 20260506 14:08:03.254 | INFO | "huge" : "", | 20260506 14:08:03.254 | INFO | "huge_shared" : " 27 GB", | 20260506 14:08:03.254 | INFO | "malloc" : " 14 GB", | 20260506 14:08:03.255 | INFO | "pinned" : " 29 GB" | 20260506 14:08:03.255 | INFO | } | 20260506 14:08:03.255 | INFO | Perf report for the last 5.13886 seconds: | 20260506 14:08:03.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:08:03.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:03.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:03.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:03.255 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 14:08:03.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:03.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:03.255 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:08:03.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:03.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:03.255 | INFO | Uploaded 36GiB to GPU in 1.99318s (18.0616GiB/s) | 20260506 14:08:03.255 | INFO | Downloaded 34GiB to GPU in 1.91399s (17.7639GiB/s) | 20260506 14:08:08.394 | INFO | Analyzing chunk frames = [1951744,1952256) | 20260506 14:08:08.394 | INFO | MemInfo: { | 20260506 14:08:08.394 | INFO | "gpu" : " 17 GB", | 20260506 14:08:08.394 | INFO | "huge" : "", | 20260506 14:08:08.394 | INFO | "huge_shared" : " 27 GB", | 20260506 14:08:08.394 | INFO | "malloc" : " 14 GB", | 20260506 14:08:08.394 | INFO | "pinned" : " 29 GB" | 20260506 14:08:08.394 | INFO | } | 20260506 14:08:08.394 | INFO | Perf report for the last 5.13911 seconds: | 20260506 14:08:08.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:08:08.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:08.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:08.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:08.394 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 14:08:08.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:08.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:08.394 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:08:08.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:08.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:08.394 | INFO | Uploaded 36GiB to GPU in 2.01129s (17.899GiB/s) | 20260506 14:08:08.394 | INFO | Downloaded 34GiB to GPU in 1.9327s (17.592GiB/s) | 20260506 14:08:13.533 | INFO | Analyzing chunk frames = [1952256,1952768) | 20260506 14:08:13.533 | INFO | MemInfo: { | 20260506 14:08:13.533 | INFO | "gpu" : " 17 GB", | 20260506 14:08:13.533 | INFO | "huge" : "", | 20260506 14:08:13.533 | INFO | "huge_shared" : " 27 GB", | 20260506 14:08:13.534 | INFO | "malloc" : " 14 GB", | 20260506 14:08:13.534 | INFO | "pinned" : " 29 GB" | 20260506 14:08:13.534 | INFO | } | 20260506 14:08:13.534 | INFO | Perf report for the last 5.13975 seconds: | 20260506 14:08:13.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:08:13.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:13.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:13.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:13.534 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:08:13.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:13.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:13.534 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260506 14:08:13.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:13.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:13.534 | INFO | Uploaded 36GiB to GPU in 2.04336s (17.618GiB/s) | 20260506 14:08:13.534 | INFO | Downloaded 34GiB to GPU in 1.91272s (17.7757GiB/s) | 20260506 14:08:18.674 | INFO | Analyzing chunk frames = [1952768,1953280) | 20260506 14:08:18.674 | INFO | MemInfo: { | 20260506 14:08:18.674 | INFO | "gpu" : " 17 GB", | 20260506 14:08:18.674 | INFO | "huge" : "", | 20260506 14:08:18.674 | INFO | "huge_shared" : " 27 GB", | 20260506 14:08:18.674 | INFO | "malloc" : " 14 GB", | 20260506 14:08:18.674 | INFO | "pinned" : " 29 GB" | 20260506 14:08:18.674 | INFO | } | 20260506 14:08:18.674 | INFO | Perf report for the last 5.14032 seconds: | 20260506 14:08:18.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:08:18.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:18.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:18.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:18.674 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:08:18.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:18.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:18.674 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 14:08:18.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:18.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:18.674 | INFO | Uploaded 36GiB to GPU in 1.98355s (18.1493GiB/s) | 20260506 14:08:18.674 | INFO | Downloaded 34GiB to GPU in 1.99743s (17.0218GiB/s) | 20260506 14:08:23.812 | INFO | Analyzing chunk frames = [1953280,1953792) | 20260506 14:08:23.812 | INFO | MemInfo: { | 20260506 14:08:23.812 | INFO | "gpu" : " 17 GB", | 20260506 14:08:23.812 | INFO | "huge" : "", | 20260506 14:08:23.812 | INFO | "huge_shared" : " 27 GB", | 20260506 14:08:23.812 | INFO | "malloc" : " 14 GB", | 20260506 14:08:23.812 | INFO | "pinned" : " 29 GB" | 20260506 14:08:23.812 | INFO | } | 20260506 14:08:23.812 | INFO | Perf report for the last 5.1383 seconds: | 20260506 14:08:23.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:08:23.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:23.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:23.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:23.812 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:08:23.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:23.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:23.812 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:08:23.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:23.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:23.813 | INFO | Uploaded 36GiB to GPU in 2.0221s (17.8033GiB/s) | 20260506 14:08:23.813 | INFO | Downloaded 34GiB to GPU in 1.98341s (17.1422GiB/s) | 20260506 14:08:23.928 | INFO | Starting a round of Baz IO | 20260506 14:08:27.078 | INFO | Populating RTMetrics report | 20260506 14:08:27.079 | INFO | RTMetrics report populated | 20260506 14:08:27.079 | INFO | Opening tmp RTMetrics json file | 20260506 14:08:27.130 | INFO | Serializing RTMetrics to json | 20260506 14:08:27.199 | INFO | Serialization done, swapping tmp file now... | 20260506 14:08:27.232 | INFO | RTMetrics done | 20260506 14:08:27.448 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.52 seconds | 20260506 14:08:27.448 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.70% / 0.02% / 61.29% | 20260506 14:08:27.448 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.71964% / 2.38419e-05% / 5.16574e-05% | 20260506 14:08:27.448 | INFO | Baz IO overall: | 20260506 14:08:27.448 | INFO | BytesWritten : 1003720884255 | 20260506 14:08:27.448 | INFO | NumEvents : 485643965769 | 20260506 14:08:28.951 | INFO | Analyzing chunk frames = [1953792,1954304) | 20260506 14:08:28.951 | INFO | MemInfo: { | 20260506 14:08:28.951 | INFO | "gpu" : " 17 GB", | 20260506 14:08:28.951 | INFO | "huge" : "", | 20260506 14:08:28.951 | INFO | "huge_shared" : " 27 GB", | 20260506 14:08:28.951 | INFO | "malloc" : " 14 GB", | 20260506 14:08:28.951 | INFO | "pinned" : " 29 GB" | 20260506 14:08:28.951 | INFO | } | 20260506 14:08:28.951 | INFO | Perf report for the last 5.13905 seconds: | 20260506 14:08:28.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:08:28.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:28.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:28.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:28.951 | INFO | ANALYSIS : 63.0%, 2.9, 95.1ms, 0 | 20260506 14:08:28.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:28.951 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260506 14:08:28.951 | INFO | PRE_HQ : 61.1%, 1.9, 59.9ms, 0 | 20260506 14:08:28.951 | INFO | BAZWRITER : 67.8%, 4.8, 168.2ms, 0 | 20260506 14:08:28.951 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 14:08:28.952 | INFO | Uploaded 36GiB to GPU in 2.07716s (17.3314GiB/s) | 20260506 14:08:28.952 | INFO | Downloaded 38GiB to GPU in 2.22259s (17.0971GiB/s) | 20260506 14:08:34.091 | INFO | Analyzing chunk frames = [1954304,1954816) | 20260506 14:08:34.091 | INFO | MemInfo: { | 20260506 14:08:34.091 | INFO | "gpu" : " 17 GB", | 20260506 14:08:34.091 | INFO | "huge" : "", | 20260506 14:08:34.091 | INFO | "huge_shared" : " 27 GB", | 20260506 14:08:34.091 | INFO | "malloc" : " 14 GB", | 20260506 14:08:34.091 | INFO | "pinned" : " 29 GB" | 20260506 14:08:34.091 | INFO | } | 20260506 14:08:34.091 | INFO | Perf report for the last 5.13948 seconds: | 20260506 14:08:34.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:08:34.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:34.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:34.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:34.091 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 14:08:34.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:34.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:34.091 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:08:34.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:34.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:34.091 | INFO | Uploaded 36GiB to GPU in 2.03051s (17.7295GiB/s) | 20260506 14:08:34.091 | INFO | Downloaded 34GiB to GPU in 1.94494s (17.4813GiB/s) | 20260506 14:08:39.230 | INFO | Analyzing chunk frames = [1954816,1955328) | 20260506 14:08:39.230 | INFO | MemInfo: { | 20260506 14:08:39.230 | INFO | "gpu" : " 17 GB", | 20260506 14:08:39.230 | INFO | "huge" : "", | 20260506 14:08:39.231 | INFO | "huge_shared" : " 27 GB", | 20260506 14:08:39.231 | INFO | "malloc" : " 14 GB", | 20260506 14:08:39.231 | INFO | "pinned" : " 29 GB" | 20260506 14:08:39.231 | INFO | } | 20260506 14:08:39.231 | INFO | Perf report for the last 5.13978 seconds: | 20260506 14:08:39.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:08:39.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:39.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:39.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:39.231 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 14:08:39.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:39.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:39.231 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:08:39.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:39.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:39.231 | INFO | Uploaded 36GiB to GPU in 2.00692s (17.9379GiB/s) | 20260506 14:08:39.231 | INFO | Downloaded 34GiB to GPU in 1.9347s (17.5738GiB/s) | 20260506 14:08:44.369 | INFO | Analyzing chunk frames = [1955328,1955840) | 20260506 14:08:44.369 | INFO | MemInfo: { | 20260506 14:08:44.369 | INFO | "gpu" : " 17 GB", | 20260506 14:08:44.369 | INFO | "huge" : "", | 20260506 14:08:44.369 | INFO | "huge_shared" : " 27 GB", | 20260506 14:08:44.369 | INFO | "malloc" : " 14 GB", | 20260506 14:08:44.369 | INFO | "pinned" : " 29 GB" | 20260506 14:08:44.370 | INFO | } | 20260506 14:08:44.370 | INFO | Perf report for the last 5.13924 seconds: | 20260506 14:08:44.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:08:44.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:44.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:44.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:44.370 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 14:08:44.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:44.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:44.370 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:08:44.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:44.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:44.370 | INFO | Uploaded 36GiB to GPU in 2.00269s (17.9758GiB/s) | 20260506 14:08:44.370 | INFO | Downloaded 34GiB to GPU in 1.92878s (17.6277GiB/s) | 20260506 14:08:49.509 | INFO | Analyzing chunk frames = [1955840,1956352) | 20260506 14:08:49.509 | INFO | MemInfo: { | 20260506 14:08:49.509 | INFO | "gpu" : " 17 GB", | 20260506 14:08:49.509 | INFO | "huge" : "", | 20260506 14:08:49.509 | INFO | "huge_shared" : " 27 GB", | 20260506 14:08:49.509 | INFO | "malloc" : " 14 GB", | 20260506 14:08:49.509 | INFO | "pinned" : " 29 GB" | 20260506 14:08:49.509 | INFO | } | 20260506 14:08:49.509 | INFO | Perf report for the last 5.13944 seconds: | 20260506 14:08:49.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:08:49.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:49.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:49.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:49.509 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 14:08:49.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:49.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:49.509 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:08:49.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:49.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:49.509 | INFO | Uploaded 36GiB to GPU in 1.98972s (18.093GiB/s) | 20260506 14:08:49.510 | INFO | Downloaded 34GiB to GPU in 1.9239s (17.6724GiB/s) | 20260506 14:08:54.648 | INFO | Analyzing chunk frames = [1956352,1956864) | 20260506 14:08:54.648 | INFO | MemInfo: { | 20260506 14:08:54.648 | INFO | "gpu" : " 17 GB", | 20260506 14:08:54.649 | INFO | "huge" : "", | 20260506 14:08:54.649 | INFO | "huge_shared" : " 27 GB", | 20260506 14:08:54.649 | INFO | "malloc" : " 14 GB", | 20260506 14:08:54.649 | INFO | "pinned" : " 29 GB" | 20260506 14:08:54.649 | INFO | } | 20260506 14:08:54.649 | INFO | Perf report for the last 5.13935 seconds: | 20260506 14:08:54.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:08:54.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:54.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:54.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:54.649 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 14:08:54.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:54.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:54.649 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 14:08:54.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:54.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:54.649 | INFO | Uploaded 36GiB to GPU in 2.03799s (17.6645GiB/s) | 20260506 14:08:54.649 | INFO | Downloaded 34GiB to GPU in 1.9266s (17.6477GiB/s) | 20260506 14:08:59.788 | INFO | Analyzing chunk frames = [1956864,1957376) | 20260506 14:08:59.788 | INFO | MemInfo: { | 20260506 14:08:59.788 | INFO | "gpu" : " 17 GB", | 20260506 14:08:59.788 | INFO | "huge" : "", | 20260506 14:08:59.788 | INFO | "huge_shared" : " 27 GB", | 20260506 14:08:59.788 | INFO | "malloc" : " 14 GB", | 20260506 14:08:59.788 | INFO | "pinned" : " 29 GB" | 20260506 14:08:59.788 | INFO | } | 20260506 14:08:59.788 | INFO | Perf report for the last 5.13965 seconds: | 20260506 14:08:59.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:08:59.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:59.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:59.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:59.788 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:08:59.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:59.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:59.788 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 14:08:59.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:59.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:08:59.789 | INFO | Uploaded 36GiB to GPU in 2.01679s (17.8501GiB/s) | 20260506 14:08:59.789 | INFO | Downloaded 34GiB to GPU in 2.00154s (16.9869GiB/s) | 20260506 14:09:04.928 | INFO | Analyzing chunk frames = [1957376,1957888) | 20260506 14:09:04.928 | INFO | MemInfo: { | 20260506 14:09:04.928 | INFO | "gpu" : " 17 GB", | 20260506 14:09:04.928 | INFO | "huge" : "", | 20260506 14:09:04.928 | INFO | "huge_shared" : " 27 GB", | 20260506 14:09:04.928 | INFO | "malloc" : " 14 GB", | 20260506 14:09:04.928 | INFO | "pinned" : " 29 GB" | 20260506 14:09:04.929 | INFO | } | 20260506 14:09:04.929 | INFO | Perf report for the last 5.14022 seconds: | 20260506 14:09:04.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:09:04.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:04.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:04.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:04.929 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 14:09:04.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:04.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:04.929 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:09:04.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:04.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:04.929 | INFO | Uploaded 36GiB to GPU in 2.02561s (17.7724GiB/s) | 20260506 14:09:04.929 | INFO | Downloaded 34GiB to GPU in 1.97303s (17.2323GiB/s) | 20260506 14:09:04.929 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:09:04.929 | INFO | Total mem usage: 17.4 GiB | 20260506 14:09:04.929 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:09:04.929 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -11.3%) | 20260506 14:09:04.929 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:09:04.929 | INFO | High Water Breakdown: | 20260506 14:09:04.929 | INFO | 61.2% : 9.57 GiB ( 87.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:09:04.929 | INFO | 38.7% : 6.05 GiB ( 90.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:09:04.929 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:09:04.929 | INFO | ------------------------------------------ | 20260506 14:09:04.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:09:04.929 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:09:04.929 | INFO | Total mem usage: 26.5 GiB | 20260506 14:09:04.929 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 14:09:04.929 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:09:04.929 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 14:09:04.929 | INFO | High Water Breakdown: | 20260506 14:09:04.929 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:09:04.929 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:09:04.929 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:09:04.929 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:09:04.929 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:09:04.929 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:09:04.929 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:09:04.929 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:09:04.929 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:09:04.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:09:04.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:09:04.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:09:04.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:09:04.929 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:09:04.929 | INFO | ------------------------------------------ | 20260506 14:09:04.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:09:04.929 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:09:04.929 | INFO | Total mem usage: 25.6 GiB | 20260506 14:09:04.929 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:09:04.930 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:09:04.930 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:09:04.930 | INFO | High Water Breakdown: | 20260506 14:09:04.930 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:09:04.930 | INFO | ------------------------------------------ | 20260506 14:09:04.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:09:04.930 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:09:04.930 | INFO | Total mem usage: 15.7 GiB | 20260506 14:09:04.930 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:09:04.930 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:09:04.930 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 14:09:04.930 | INFO | High Water Breakdown: | 20260506 14:09:04.930 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:09:04.930 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:09:04.930 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:09:04.930 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:09:04.930 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:09:04.930 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:09:04.930 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:09:04.930 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:09:04.930 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:09:04.930 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:09:04.930 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:09:04.930 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:09:04.930 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:09:04.930 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:09:04.930 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:09:04.930 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:09:04.930 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:09:04.930 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:09:04.930 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:09:04.930 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:09:04.930 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:09:04.930 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:09:04.930 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:09:04.930 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:09:04.930 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:09:04.930 | INFO | ------------------------------------------ | 20260506 14:09:04.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:09:04.930 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:09:04.930 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:09:04.930 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:09:04.930 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.54e+01, 1.35e+01 / 1.73e+00 | 20260506 14:09:04.930 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.02e-01, 1.07e-02 / 6.07e-03 | 20260506 14:09:04.930 | INFO | PerfReport: 11.58%, Baseline -- 3.65e+00 / 4.46e+00, 5.70e-01 / 3.20e-01 | 20260506 14:09:04.930 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.48e+00, 1.87e-01 / 1.19e-01 | 20260506 14:09:04.930 | INFO | PerfReport: 15.78%, DME -- 3.98e+01 / 4.40e+01, 3.92e+00 / 1.97e+00 | 20260506 14:09:04.930 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.47e+00 / 1.69e+00 | 20260506 14:09:04.930 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.36e+00 / 2.95e+00, 4.13e-01 / 2.62e-01 | 20260506 14:09:04.930 | INFO | PerfReport: 3.62%, Metrics -- 1.14e+00 / 1.51e+00, 2.91e-01 / 2.06e-01 | 20260506 14:09:04.930 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.00e-01 / 4.58e-01, 6.18e-02 / 3.02e-02 | 20260506 14:09:04.930 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:09:04.930 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:09:04.930 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:09:04.930 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 14:09:04.930 | INFO | PerfReport: 21.56%, Upload -- 2.11e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 14:09:04.930 | INFO | PerfReport: 21.11%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 14:09:04.930 | INFO | PerfReport: 32.24%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 14:09:04.930 | INFO | PerfReport: 1.60%, LockUpload -- 1.56e+00 / 0.00e+00, 6.42e+00 / 0.00e+00 | 20260506 14:09:04.930 | INFO | PerfReport: 1.06%, LockDownload -- 1.04e+00 / 0.00e+00, 2.67e+00 / 0.00e+00 | 20260506 14:09:04.930 | INFO | PerfReport: 22.41%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 14:09:04.930 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260506 14:09:05.040 | INFO | Starting a round of Baz IO | 20260506 14:09:08.203 | INFO | Populating RTMetrics report | 20260506 14:09:08.203 | INFO | RTMetrics report populated | 20260506 14:09:08.203 | INFO | Opening tmp RTMetrics json file | 20260506 14:09:08.338 | INFO | Serializing RTMetrics to json | 20260506 14:09:08.347 | INFO | Serialization done, swapping tmp file now... | 20260506 14:09:08.348 | INFO | RTMetrics done | 20260506 14:09:08.689 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.649 seconds | 20260506 14:09:08.689 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.68% / 0.02% / 61.30% | 20260506 14:09:08.689 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.716% / 4.76837e-05% / 4.76837e-05% | 20260506 14:09:08.689 | INFO | Baz IO overall: | 20260506 14:09:08.689 | INFO | BytesWritten : 1005988810816 | 20260506 14:09:08.689 | INFO | NumEvents : 486743488596 | 20260506 14:09:10.066 | INFO | Analyzing chunk frames = [1957888,1958400) | 20260506 14:09:10.066 | INFO | MemInfo: { | 20260506 14:09:10.066 | INFO | "gpu" : " 17 GB", | 20260506 14:09:10.066 | INFO | "huge" : "", | 20260506 14:09:10.066 | INFO | "huge_shared" : " 27 GB", | 20260506 14:09:10.066 | INFO | "malloc" : " 14 GB", | 20260506 14:09:10.066 | INFO | "pinned" : " 29 GB" | 20260506 14:09:10.066 | INFO | } | 20260506 14:09:10.066 | INFO | Perf report for the last 5.13714 seconds: | 20260506 14:09:10.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:09:10.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:10.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:10.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:10.066 | INFO | ANALYSIS : 63.1%, 2.9, 95.3ms, 0 | 20260506 14:09:10.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:10.066 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260506 14:09:10.066 | INFO | PRE_HQ : 61.1%, 1.9, 59.7ms, 0 | 20260506 14:09:10.066 | INFO | BAZWRITER : 70.4%, 4.8, 173.5ms, 0 | 20260506 14:09:10.066 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 14:09:10.066 | INFO | Uploaded 36GiB to GPU in 2.07051s (17.387GiB/s) | 20260506 14:09:10.066 | INFO | Downloaded 38GiB to GPU in 2.23105s (17.0323GiB/s) | 20260506 14:09:15.205 | INFO | Analyzing chunk frames = [1958400,1958912) | 20260506 14:09:15.205 | INFO | MemInfo: { | 20260506 14:09:15.205 | INFO | "gpu" : " 17 GB", | 20260506 14:09:15.205 | INFO | "huge" : "", | 20260506 14:09:15.205 | INFO | "huge_shared" : " 27 GB", | 20260506 14:09:15.205 | INFO | "malloc" : " 14 GB", | 20260506 14:09:15.205 | INFO | "pinned" : " 29 GB" | 20260506 14:09:15.205 | INFO | } | 20260506 14:09:15.205 | INFO | Perf report for the last 5.13977 seconds: | 20260506 14:09:15.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:09:15.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:15.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:15.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:15.206 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 14:09:15.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:15.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:15.206 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:09:15.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:15.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:15.206 | INFO | Uploaded 36GiB to GPU in 2.01595s (17.8576GiB/s) | 20260506 14:09:15.206 | INFO | Downloaded 34GiB to GPU in 1.94254s (17.5029GiB/s) | 20260506 14:09:20.346 | INFO | Analyzing chunk frames = [1958912,1959424) | 20260506 14:09:20.346 | INFO | MemInfo: { | 20260506 14:09:20.346 | INFO | "gpu" : " 17 GB", | 20260506 14:09:20.346 | INFO | "huge" : "", | 20260506 14:09:20.346 | INFO | "huge_shared" : " 27 GB", | 20260506 14:09:20.346 | INFO | "malloc" : " 14 GB", | 20260506 14:09:20.346 | INFO | "pinned" : " 29 GB" | 20260506 14:09:20.346 | INFO | } | 20260506 14:09:20.346 | INFO | Perf report for the last 5.14063 seconds: | 20260506 14:09:20.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:09:20.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:20.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:20.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:20.346 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:09:20.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:20.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:20.346 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:09:20.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:20.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:20.346 | INFO | Uploaded 36GiB to GPU in 1.99971s (18.0026GiB/s) | 20260506 14:09:20.346 | INFO | Downloaded 34GiB to GPU in 1.93037s (17.6132GiB/s) | 20260506 14:09:25.484 | INFO | Analyzing chunk frames = [1959424,1959936) | 20260506 14:09:25.484 | INFO | MemInfo: { | 20260506 14:09:25.484 | INFO | "gpu" : " 17 GB", | 20260506 14:09:25.485 | INFO | "huge" : "", | 20260506 14:09:25.485 | INFO | "huge_shared" : " 27 GB", | 20260506 14:09:25.485 | INFO | "malloc" : " 14 GB", | 20260506 14:09:25.485 | INFO | "pinned" : " 29 GB" | 20260506 14:09:25.485 | INFO | } | 20260506 14:09:25.485 | INFO | Perf report for the last 5.13846 seconds: | 20260506 14:09:25.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:09:25.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:25.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:25.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:25.485 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 14:09:25.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:25.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:25.485 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:09:25.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:25.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:25.485 | INFO | Uploaded 36GiB to GPU in 1.9863s (18.1241GiB/s) | 20260506 14:09:25.485 | INFO | Downloaded 34GiB to GPU in 1.91014s (17.7997GiB/s) | 20260506 14:09:30.623 | INFO | Analyzing chunk frames = [1959936,1960448) | 20260506 14:09:30.623 | INFO | MemInfo: { | 20260506 14:09:30.623 | INFO | "gpu" : " 17 GB", | 20260506 14:09:30.623 | INFO | "huge" : "", | 20260506 14:09:30.623 | INFO | "huge_shared" : " 27 GB", | 20260506 14:09:30.623 | INFO | "malloc" : " 14 GB", | 20260506 14:09:30.624 | INFO | "pinned" : " 29 GB" | 20260506 14:09:30.624 | INFO | } | 20260506 14:09:30.624 | INFO | Perf report for the last 5.13895 seconds: | 20260506 14:09:30.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:09:30.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:30.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:30.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:30.624 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:09:30.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:30.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:30.624 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:09:30.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:30.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:30.624 | INFO | Uploaded 36GiB to GPU in 2.01112s (17.9005GiB/s) | 20260506 14:09:30.624 | INFO | Downloaded 34GiB to GPU in 1.93671s (17.5555GiB/s) | 20260506 14:09:35.763 | INFO | Analyzing chunk frames = [1960448,1960960) | 20260506 14:09:35.763 | INFO | MemInfo: { | 20260506 14:09:35.763 | INFO | "gpu" : " 17 GB", | 20260506 14:09:35.763 | INFO | "huge" : "", | 20260506 14:09:35.763 | INFO | "huge_shared" : " 27 GB", | 20260506 14:09:35.763 | INFO | "malloc" : " 14 GB", | 20260506 14:09:35.763 | INFO | "pinned" : " 29 GB" | 20260506 14:09:35.763 | INFO | } | 20260506 14:09:35.763 | INFO | Perf report for the last 5.13936 seconds: | 20260506 14:09:35.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:09:35.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:35.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:35.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:35.763 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:09:35.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:35.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:35.763 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 14:09:35.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:35.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:35.763 | INFO | Uploaded 36GiB to GPU in 2.03573s (17.6841GiB/s) | 20260506 14:09:35.763 | INFO | Downloaded 34GiB to GPU in 1.92318s (17.679GiB/s) | 20260506 14:09:40.903 | INFO | Analyzing chunk frames = [1960960,1961472) | 20260506 14:09:40.903 | INFO | MemInfo: { | 20260506 14:09:40.903 | INFO | "gpu" : " 17 GB", | 20260506 14:09:40.903 | INFO | "huge" : "", | 20260506 14:09:40.903 | INFO | "huge_shared" : " 27 GB", | 20260506 14:09:40.903 | INFO | "malloc" : " 14 GB", | 20260506 14:09:40.903 | INFO | "pinned" : " 29 GB" | 20260506 14:09:40.903 | INFO | } | 20260506 14:09:40.903 | INFO | Perf report for the last 5.13999 seconds: | 20260506 14:09:40.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:09:40.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:40.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:40.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:40.903 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 14:09:40.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:40.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:40.903 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 14:09:40.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:40.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:40.903 | INFO | Uploaded 36GiB to GPU in 2.04548s (17.5998GiB/s) | 20260506 14:09:40.903 | INFO | Downloaded 34GiB to GPU in 1.99495s (17.043GiB/s) | 20260506 14:09:46.042 | INFO | Analyzing chunk frames = [1961472,1961984) | 20260506 14:09:46.042 | INFO | MemInfo: { | 20260506 14:09:46.042 | INFO | "gpu" : " 17 GB", | 20260506 14:09:46.042 | INFO | "huge" : "", | 20260506 14:09:46.042 | INFO | "huge_shared" : " 27 GB", | 20260506 14:09:46.042 | INFO | "malloc" : " 14 GB", | 20260506 14:09:46.042 | INFO | "pinned" : " 29 GB" | 20260506 14:09:46.042 | INFO | } | 20260506 14:09:46.042 | INFO | Perf report for the last 5.13885 seconds: | 20260506 14:09:46.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:09:46.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:46.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:46.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:46.042 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:09:46.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:46.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:46.042 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:09:46.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:46.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:46.042 | INFO | Uploaded 36GiB to GPU in 2.02272s (17.7978GiB/s) | 20260506 14:09:46.042 | INFO | Downloaded 34GiB to GPU in 1.98362s (17.1404GiB/s) | 20260506 14:09:46.158 | INFO | Starting a round of Baz IO | 20260506 14:09:49.310 | INFO | Populating RTMetrics report | 20260506 14:09:49.310 | INFO | RTMetrics report populated | 20260506 14:09:49.310 | INFO | Opening tmp RTMetrics json file | 20260506 14:09:49.322 | INFO | Serializing RTMetrics to json | 20260506 14:09:49.381 | INFO | Serialization done, swapping tmp file now... | 20260506 14:09:49.426 | INFO | RTMetrics done | 20260506 14:09:49.684 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.525 seconds | 20260506 14:09:49.684 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.67% / 0.02% / 61.32% | 20260506 14:09:49.684 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.70253% / 2.38419e-05% / 3.97364e-05% | 20260506 14:09:49.685 | INFO | Baz IO overall: | 20260506 14:09:49.685 | INFO | BytesWritten : 1008254594216 | 20260506 14:09:49.685 | INFO | NumEvents : 487841164371 | 20260506 14:09:51.182 | INFO | Analyzing chunk frames = [1961984,1962496) | 20260506 14:09:51.182 | INFO | MemInfo: { | 20260506 14:09:51.182 | INFO | "gpu" : " 17 GB", | 20260506 14:09:51.182 | INFO | "huge" : "", | 20260506 14:09:51.182 | INFO | "huge_shared" : " 27 GB", | 20260506 14:09:51.182 | INFO | "malloc" : " 14 GB", | 20260506 14:09:51.182 | INFO | "pinned" : " 29 GB" | 20260506 14:09:51.182 | INFO | } | 20260506 14:09:51.182 | INFO | Perf report for the last 5.13997 seconds: | 20260506 14:09:51.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:09:51.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:51.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:51.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:51.182 | INFO | ANALYSIS : 63.0%, 2.9, 95.5ms, 0 | 20260506 14:09:51.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:51.182 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260506 14:09:51.182 | INFO | PRE_HQ : 60.9%, 1.9, 59.5ms, 0 | 20260506 14:09:51.182 | INFO | BAZWRITER : 68.0%, 4.8, 168.3ms, 0 | 20260506 14:09:51.182 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 14:09:51.182 | INFO | Uploaded 36GiB to GPU in 2.09359s (17.1953GiB/s) | 20260506 14:09:51.182 | INFO | Downloaded 38GiB to GPU in 2.23714s (16.9859GiB/s) | 20260506 14:09:56.322 | INFO | Analyzing chunk frames = [1962496,1963008) | 20260506 14:09:56.322 | INFO | MemInfo: { | 20260506 14:09:56.322 | INFO | "gpu" : " 17 GB", | 20260506 14:09:56.322 | INFO | "huge" : "", | 20260506 14:09:56.322 | INFO | "huge_shared" : " 27 GB", | 20260506 14:09:56.322 | INFO | "malloc" : " 14 GB", | 20260506 14:09:56.322 | INFO | "pinned" : " 29 GB" | 20260506 14:09:56.322 | INFO | } | 20260506 14:09:56.322 | INFO | Perf report for the last 5.14064 seconds: | 20260506 14:09:56.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:09:56.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:56.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:56.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:56.322 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 14:09:56.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:56.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:56.323 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 14:09:56.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:56.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:09:56.323 | INFO | Uploaded 36GiB to GPU in 2.02777s (17.7535GiB/s) | 20260506 14:09:56.323 | INFO | Downloaded 34GiB to GPU in 1.9426s (17.5023GiB/s) | 20260506 14:10:01.460 | INFO | Analyzing chunk frames = [1963008,1963520) | 20260506 14:10:01.460 | INFO | MemInfo: { | 20260506 14:10:01.460 | INFO | "gpu" : " 17 GB", | 20260506 14:10:01.460 | INFO | "huge" : "", | 20260506 14:10:01.460 | INFO | "huge_shared" : " 27 GB", | 20260506 14:10:01.460 | INFO | "malloc" : " 14 GB", | 20260506 14:10:01.460 | INFO | "pinned" : " 29 GB" | 20260506 14:10:01.460 | INFO | } | 20260506 14:10:01.460 | INFO | Perf report for the last 5.13729 seconds: | 20260506 14:10:01.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:10:01.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:01.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:01.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:01.460 | INFO | ANALYSIS : 61.4%, 3.0, 95.0ms, 0 | 20260506 14:10:01.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:01.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:01.460 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:10:01.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:01.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:01.460 | INFO | Uploaded 36GiB to GPU in 2.00966s (17.9135GiB/s) | 20260506 14:10:01.460 | INFO | Downloaded 34GiB to GPU in 1.9344s (17.5765GiB/s) | 20260506 14:10:06.599 | INFO | Analyzing chunk frames = [1963520,1964032) | 20260506 14:10:06.599 | INFO | MemInfo: { | 20260506 14:10:06.599 | INFO | "gpu" : " 17 GB", | 20260506 14:10:06.599 | INFO | "huge" : "", | 20260506 14:10:06.599 | INFO | "huge_shared" : " 27 GB", | 20260506 14:10:06.599 | INFO | "malloc" : " 14 GB", | 20260506 14:10:06.599 | INFO | "pinned" : " 29 GB" | 20260506 14:10:06.599 | INFO | } | 20260506 14:10:06.599 | INFO | Perf report for the last 5.13945 seconds: | 20260506 14:10:06.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:10:06.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:06.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:06.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:06.599 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 14:10:06.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:06.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:06.599 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:10:06.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:06.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:06.599 | INFO | Uploaded 36GiB to GPU in 2.01146s (17.8975GiB/s) | 20260506 14:10:06.599 | INFO | Downloaded 34GiB to GPU in 1.928s (17.6349GiB/s) | 20260506 14:10:11.738 | INFO | Analyzing chunk frames = [1964032,1964544) | 20260506 14:10:11.738 | INFO | MemInfo: { | 20260506 14:10:11.738 | INFO | "gpu" : " 17 GB", | 20260506 14:10:11.738 | INFO | "huge" : "", | 20260506 14:10:11.738 | INFO | "huge_shared" : " 27 GB", | 20260506 14:10:11.738 | INFO | "malloc" : " 14 GB", | 20260506 14:10:11.738 | INFO | "pinned" : " 29 GB" | 20260506 14:10:11.738 | INFO | } | 20260506 14:10:11.738 | INFO | Perf report for the last 5.1392 seconds: | 20260506 14:10:11.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:10:11.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:11.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:11.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:11.738 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:10:11.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:11.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:11.738 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 14:10:11.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:11.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:11.739 | INFO | Uploaded 36GiB to GPU in 1.99482s (18.0467GiB/s) | 20260506 14:10:11.739 | INFO | Downloaded 34GiB to GPU in 1.93048s (17.6122GiB/s) | 20260506 14:10:16.878 | INFO | Analyzing chunk frames = [1964544,1965056) | 20260506 14:10:16.878 | INFO | MemInfo: { | 20260506 14:10:16.878 | INFO | "gpu" : " 17 GB", | 20260506 14:10:16.878 | INFO | "huge" : "", | 20260506 14:10:16.878 | INFO | "huge_shared" : " 27 GB", | 20260506 14:10:16.878 | INFO | "malloc" : " 14 GB", | 20260506 14:10:16.878 | INFO | "pinned" : " 29 GB" | 20260506 14:10:16.878 | INFO | } | 20260506 14:10:16.878 | INFO | Perf report for the last 5.13976 seconds: | 20260506 14:10:16.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:10:16.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:16.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:16.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:16.878 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:10:16.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:16.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:16.878 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 14:10:16.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:16.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:16.879 | INFO | Uploaded 36GiB to GPU in 2.08474s (17.2683GiB/s) | 20260506 14:10:16.879 | INFO | Downloaded 34GiB to GPU in 1.97829s (17.1866GiB/s) | 20260506 14:10:22.018 | INFO | Analyzing chunk frames = [1965056,1965568) | 20260506 14:10:22.018 | INFO | MemInfo: { | 20260506 14:10:22.018 | INFO | "gpu" : " 17 GB", | 20260506 14:10:22.018 | INFO | "huge" : "", | 20260506 14:10:22.018 | INFO | "huge_shared" : " 27 GB", | 20260506 14:10:22.018 | INFO | "malloc" : " 14 GB", | 20260506 14:10:22.018 | INFO | "pinned" : " 29 GB" | 20260506 14:10:22.018 | INFO | } | 20260506 14:10:22.019 | INFO | Perf report for the last 5.14036 seconds: | 20260506 14:10:22.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:10:22.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:22.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:22.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:22.019 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:10:22.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:22.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:22.019 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 14:10:22.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:22.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:22.019 | INFO | Uploaded 36GiB to GPU in 2.02326s (17.7931GiB/s) | 20260506 14:10:22.019 | INFO | Downloaded 34GiB to GPU in 1.99726s (17.0233GiB/s) | 20260506 14:10:27.157 | INFO | Analyzing chunk frames = [1965568,1966080) | 20260506 14:10:27.157 | INFO | MemInfo: { | 20260506 14:10:27.157 | INFO | "gpu" : " 17 GB", | 20260506 14:10:27.157 | INFO | "huge" : "", | 20260506 14:10:27.157 | INFO | "huge_shared" : " 27 GB", | 20260506 14:10:27.157 | INFO | "malloc" : " 14 GB", | 20260506 14:10:27.157 | INFO | "pinned" : " 29 GB" | 20260506 14:10:27.157 | INFO | } | 20260506 14:10:27.157 | INFO | Perf report for the last 5.13873 seconds: | 20260506 14:10:27.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:10:27.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:27.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:27.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:27.157 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:10:27.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:27.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:27.157 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260506 14:10:27.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:27.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:27.158 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:10:27.158 | INFO | Total mem usage: 17.4 GiB | 20260506 14:10:27.158 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:10:27.158 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -8.33%) | 20260506 14:10:27.158 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:10:27.158 | INFO | High Water Breakdown: | 20260506 14:10:27.158 | INFO | 61.6% : 9.9 GiB ( 90.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:10:27.158 | INFO | 38.3% : 6.16 GiB ( 92% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:10:27.158 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:10:27.158 | INFO | ------------------------------------------ | 20260506 14:10:27.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:10:27.158 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:10:27.158 | INFO | Total mem usage: 26.5 GiB | 20260506 14:10:27.158 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.53%) | 20260506 14:10:27.158 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:10:27.158 | INFO | Unused Cache: 1.7 GiB (Waste: 6.43%) | 20260506 14:10:27.158 | INFO | High Water Breakdown: | 20260506 14:10:27.158 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:10:27.158 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:10:27.158 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:10:27.158 | INFO | 1.61% : 420 MiB ( 80.3% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:10:27.158 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:10:27.158 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:10:27.158 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:10:27.158 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:10:27.158 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:10:27.158 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:10:27.158 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:10:27.158 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:10:27.158 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:10:27.158 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:10:27.158 | INFO | ------------------------------------------ | 20260506 14:10:27.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:10:27.158 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:10:27.158 | INFO | Total mem usage: 25.6 GiB | 20260506 14:10:27.158 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:10:27.158 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:10:27.158 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:10:27.158 | INFO | High Water Breakdown: | 20260506 14:10:27.158 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:10:27.158 | INFO | ------------------------------------------ | 20260506 14:10:27.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:10:27.158 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:10:27.158 | INFO | Total mem usage: 15.7 GiB | 20260506 14:10:27.158 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:10:27.158 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:10:27.158 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 14:10:27.158 | INFO | High Water Breakdown: | 20260506 14:10:27.158 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:10:27.158 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:10:27.158 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:10:27.158 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:10:27.158 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:10:27.158 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:10:27.158 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:10:27.158 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:10:27.158 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:10:27.158 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:10:27.158 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:10:27.158 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:10:27.158 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:10:27.158 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:10:27.158 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:10:27.158 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:10:27.158 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:10:27.158 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:10:27.158 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:10:27.158 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:10:27.158 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:10:27.158 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:10:27.158 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:10:27.158 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:10:27.158 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:10:27.158 | INFO | ------------------------------------------ | 20260506 14:10:27.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:10:27.158 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:10:27.158 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:10:27.158 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:10:27.158 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.60e+01, 1.35e+01 / 2.37e+00 | 20260506 14:10:27.158 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.02e-01, 1.01e-02 / 6.57e-03 | 20260506 14:10:27.158 | INFO | PerfReport: 11.56%, Baseline -- 3.65e+00 / 4.42e+00, 5.67e-01 / 2.87e-01 | 20260506 14:10:27.158 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.48e+00, 1.87e-01 / 1.36e-01 | 20260506 14:10:27.158 | INFO | PerfReport: 15.86%, DME -- 4.00e+01 / 4.46e+01, 4.13e+00 / 2.33e+00 | 20260506 14:10:27.158 | INFO | PerfReport: 56.72%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.47e+00 / 1.67e+00 | 20260506 14:10:27.158 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.35e+00 / 2.94e+00, 4.12e-01 / 2.61e-01 | 20260506 14:10:27.158 | INFO | PerfReport: 3.62%, Metrics -- 1.14e+00 / 1.50e+00, 2.93e-01 / 1.83e-01 | 20260506 14:10:27.158 | INFO | PerfReport: 1.27%, PulseSerializing -- 3.99e-01 / 4.54e-01, 6.22e-02 / 3.51e-02 | 20260506 14:10:27.158 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:10:27.158 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:10:27.158 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:10:27.158 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 14:10:27.158 | INFO | PerfReport: 21.60%, Upload -- 2.11e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260506 14:10:27.158 | INFO | PerfReport: 21.13%, Download -- 2.07e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 14:10:27.158 | INFO | PerfReport: 32.24%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 14:10:27.158 | INFO | PerfReport: 1.62%, LockUpload -- 1.59e+00 / 0.00e+00, 6.19e+00 / 0.00e+00 | 20260506 14:10:27.158 | INFO | PerfReport: 1.02%, LockDownload -- 9.96e-01 / 0.00e+00, 2.55e+00 / 0.00e+00 | 20260506 14:10:27.158 | INFO | PerfReport: 22.36%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 14:10:27.158 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260506 14:10:27.158 | INFO | Uploaded 36GiB to GPU in 2.0447s (17.6065GiB/s) | 20260506 14:10:27.159 | INFO | Downloaded 34GiB to GPU in 1.96717s (17.2837GiB/s) | 20260506 14:10:27.283 | INFO | Starting a round of Baz IO | 20260506 14:10:30.519 | INFO | Populating RTMetrics report | 20260506 14:10:30.519 | INFO | RTMetrics report populated | 20260506 14:10:30.519 | INFO | Opening tmp RTMetrics json file | 20260506 14:10:30.571 | INFO | Serializing RTMetrics to json | 20260506 14:10:30.574 | INFO | Serialization done, swapping tmp file now... | 20260506 14:10:30.578 | INFO | RTMetrics done | 20260506 14:10:31.279 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.996 seconds | 20260506 14:10:31.279 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.65% / 0.01% / 61.33% | 20260506 14:10:31.279 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.69105% / 7.94729e-06% / 7.54992e-05% | 20260506 14:10:31.279 | INFO | Baz IO overall: | 20260506 14:10:31.279 | INFO | BytesWritten : 1010518841739 | 20260506 14:10:31.279 | INFO | NumEvents : 488937757331 | 20260506 14:10:32.296 | INFO | Analyzing chunk frames = [1966080,1966592) | 20260506 14:10:32.296 | INFO | MemInfo: { | 20260506 14:10:32.296 | INFO | "gpu" : " 17 GB", | 20260506 14:10:32.296 | INFO | "huge" : "", | 20260506 14:10:32.296 | INFO | "huge_shared" : " 27 GB", | 20260506 14:10:32.296 | INFO | "malloc" : " 14 GB", | 20260506 14:10:32.296 | INFO | "pinned" : " 29 GB" | 20260506 14:10:32.296 | INFO | } | 20260506 14:10:32.296 | INFO | Perf report for the last 5.1385 seconds: | 20260506 14:10:32.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:10:32.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:32.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:32.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:32.296 | INFO | ANALYSIS : 63.8%, 2.9, 95.6ms, 0 | 20260506 14:10:32.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:32.296 | INFO | RT_METRICS : 48.1%, 1.0, 25.2ms, 0 | 20260506 14:10:32.296 | INFO | PRE_HQ : 62.1%, 2.0, 64.0ms, 0 | 20260506 14:10:32.296 | INFO | BAZWRITER : 77.1%, 4.8, 191.6ms, 0 | 20260506 14:10:32.296 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 14:10:32.296 | INFO | Uploaded 36GiB to GPU in 2.10046s (17.1391GiB/s) | 20260506 14:10:32.296 | INFO | Downloaded 38GiB to GPU in 2.21401s (17.1635GiB/s) | 20260506 14:10:37.436 | INFO | Analyzing chunk frames = [1966592,1967104) | 20260506 14:10:37.436 | INFO | MemInfo: { | 20260506 14:10:37.436 | INFO | "gpu" : " 17 GB", | 20260506 14:10:37.436 | INFO | "huge" : "", | 20260506 14:10:37.436 | INFO | "huge_shared" : " 27 GB", | 20260506 14:10:37.436 | INFO | "malloc" : " 14 GB", | 20260506 14:10:37.436 | INFO | "pinned" : " 29 GB" | 20260506 14:10:37.436 | INFO | } | 20260506 14:10:37.436 | INFO | Perf report for the last 5.14004 seconds: | 20260506 14:10:37.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:10:37.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:37.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:37.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:37.436 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:10:37.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:37.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:37.436 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 14:10:37.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:37.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:37.436 | INFO | Uploaded 36GiB to GPU in 1.99727s (18.0246GiB/s) | 20260506 14:10:37.436 | INFO | Downloaded 34GiB to GPU in 1.94363s (17.493GiB/s) | 20260506 14:10:42.575 | INFO | Analyzing chunk frames = [1967104,1967616) | 20260506 14:10:42.575 | INFO | MemInfo: { | 20260506 14:10:42.575 | INFO | "gpu" : " 17 GB", | 20260506 14:10:42.575 | INFO | "huge" : "", | 20260506 14:10:42.575 | INFO | "huge_shared" : " 27 GB", | 20260506 14:10:42.575 | INFO | "malloc" : " 14 GB", | 20260506 14:10:42.575 | INFO | "pinned" : " 29 GB" | 20260506 14:10:42.575 | INFO | } | 20260506 14:10:42.575 | INFO | Perf report for the last 5.13887 seconds: | 20260506 14:10:42.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:10:42.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:42.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:42.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:42.575 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:10:42.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:42.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:42.575 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:10:42.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:42.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:42.575 | INFO | Uploaded 36GiB to GPU in 1.99166s (18.0754GiB/s) | 20260506 14:10:42.575 | INFO | Downloaded 34GiB to GPU in 1.93268s (17.5922GiB/s) | 20260506 14:10:47.714 | INFO | Analyzing chunk frames = [1967616,1968128) | 20260506 14:10:47.714 | INFO | MemInfo: { | 20260506 14:10:47.714 | INFO | "gpu" : " 17 GB", | 20260506 14:10:47.714 | INFO | "huge" : "", | 20260506 14:10:47.714 | INFO | "huge_shared" : " 27 GB", | 20260506 14:10:47.714 | INFO | "malloc" : " 14 GB", | 20260506 14:10:47.714 | INFO | "pinned" : " 29 GB" | 20260506 14:10:47.714 | INFO | } | 20260506 14:10:47.714 | INFO | Perf report for the last 5.13948 seconds: | 20260506 14:10:47.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:10:47.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:47.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:47.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:47.714 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260506 14:10:47.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:47.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:47.714 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:10:47.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:47.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:47.714 | INFO | Uploaded 36GiB to GPU in 1.99095s (18.0818GiB/s) | 20260506 14:10:47.714 | INFO | Downloaded 34GiB to GPU in 1.91896s (17.718GiB/s) | 20260506 14:10:52.853 | INFO | Analyzing chunk frames = [1968128,1968640) | 20260506 14:10:52.853 | INFO | MemInfo: { | 20260506 14:10:52.853 | INFO | "gpu" : " 17 GB", | 20260506 14:10:52.853 | INFO | "huge" : "", | 20260506 14:10:52.853 | INFO | "huge_shared" : " 27 GB", | 20260506 14:10:52.853 | INFO | "malloc" : " 14 GB", | 20260506 14:10:52.853 | INFO | "pinned" : " 29 GB" | 20260506 14:10:52.853 | INFO | } | 20260506 14:10:52.853 | INFO | Perf report for the last 5.13898 seconds: | 20260506 14:10:52.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:10:52.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:52.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:52.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:52.853 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:10:52.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:52.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:52.853 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 14:10:52.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:52.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:52.853 | INFO | Uploaded 36GiB to GPU in 1.99578s (18.0381GiB/s) | 20260506 14:10:52.853 | INFO | Downloaded 34GiB to GPU in 1.93298s (17.5894GiB/s) | 20260506 14:10:57.993 | INFO | Analyzing chunk frames = [1968640,1969152) | 20260506 14:10:57.993 | INFO | MemInfo: { | 20260506 14:10:57.993 | INFO | "gpu" : " 17 GB", | 20260506 14:10:57.993 | INFO | "huge" : "", | 20260506 14:10:57.993 | INFO | "huge_shared" : " 27 GB", | 20260506 14:10:57.993 | INFO | "malloc" : " 14 GB", | 20260506 14:10:57.993 | INFO | "pinned" : " 29 GB" | 20260506 14:10:57.993 | INFO | } | 20260506 14:10:57.993 | INFO | Perf report for the last 5.1402 seconds: | 20260506 14:10:57.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:10:57.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:57.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:57.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:57.993 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:10:57.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:57.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:57.994 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:10:57.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:57.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:10:57.994 | INFO | Uploaded 36GiB to GPU in 2.081s (17.2994GiB/s) | 20260506 14:10:57.994 | INFO | Downloaded 34GiB to GPU in 1.96622s (17.292GiB/s) | 20260506 14:11:03.132 | INFO | Analyzing chunk frames = [1969152,1969664) | 20260506 14:11:03.132 | INFO | MemInfo: { | 20260506 14:11:03.132 | INFO | "gpu" : " 17 GB", | 20260506 14:11:03.132 | INFO | "huge" : "", | 20260506 14:11:03.132 | INFO | "huge_shared" : " 27 GB", | 20260506 14:11:03.132 | INFO | "malloc" : " 14 GB", | 20260506 14:11:03.132 | INFO | "pinned" : " 29 GB" | 20260506 14:11:03.133 | INFO | } | 20260506 14:11:03.133 | INFO | Perf report for the last 5.13917 seconds: | 20260506 14:11:03.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:11:03.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:03.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:03.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:03.133 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 14:11:03.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:03.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:03.133 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 14:11:03.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:03.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:03.133 | INFO | Uploaded 36GiB to GPU in 2.0661s (17.4242GiB/s) | 20260506 14:11:03.133 | INFO | Downloaded 34GiB to GPU in 1.99871s (17.011GiB/s) | 20260506 14:11:08.272 | INFO | Analyzing chunk frames = [1969664,1970176) | 20260506 14:11:08.272 | INFO | MemInfo: { | 20260506 14:11:08.272 | INFO | "gpu" : " 17 GB", | 20260506 14:11:08.272 | INFO | "huge" : "", | 20260506 14:11:08.272 | INFO | "huge_shared" : " 27 GB", | 20260506 14:11:08.272 | INFO | "malloc" : " 14 GB", | 20260506 14:11:08.272 | INFO | "pinned" : " 29 GB" | 20260506 14:11:08.272 | INFO | } | 20260506 14:11:08.272 | INFO | Perf report for the last 5.13941 seconds: | 20260506 14:11:08.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:11:08.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:08.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:08.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:08.272 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 14:11:08.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:08.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:08.272 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260506 14:11:08.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:08.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:08.272 | INFO | Uploaded 36GiB to GPU in 2.05862s (17.4875GiB/s) | 20260506 14:11:08.272 | INFO | Downloaded 34GiB to GPU in 1.97772s (17.1915GiB/s) | 20260506 14:11:08.424 | INFO | Starting a round of Baz IO | 20260506 14:11:11.557 | INFO | Populating RTMetrics report | 20260506 14:11:11.558 | INFO | RTMetrics report populated | 20260506 14:11:11.558 | INFO | Opening tmp RTMetrics json file | 20260506 14:11:11.596 | INFO | Serializing RTMetrics to json | 20260506 14:11:11.662 | INFO | Serialization done, swapping tmp file now... | 20260506 14:11:11.697 | INFO | RTMetrics done | 20260506 14:11:11.946 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.522 seconds | 20260506 14:11:11.946 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.64% / 0.01% / 61.35% | 20260506 14:11:11.946 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.69083% / 2.38419e-05% / 4.76837e-05% | 20260506 14:11:11.946 | INFO | Baz IO overall: | 20260506 14:11:11.946 | INFO | BytesWritten : 1012786279065 | 20260506 14:11:11.946 | INFO | NumEvents : 490036498524 | 20260506 14:11:13.411 | INFO | Analyzing chunk frames = [1970176,1970688) | 20260506 14:11:13.411 | INFO | MemInfo: { | 20260506 14:11:13.411 | INFO | "gpu" : " 17 GB", | 20260506 14:11:13.411 | INFO | "huge" : "", | 20260506 14:11:13.411 | INFO | "huge_shared" : " 27 GB", | 20260506 14:11:13.411 | INFO | "malloc" : " 14 GB", | 20260506 14:11:13.412 | INFO | "pinned" : " 29 GB" | 20260506 14:11:13.412 | INFO | } | 20260506 14:11:13.412 | INFO | Perf report for the last 5.13963 seconds: | 20260506 14:11:13.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:11:13.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:13.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:13.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:13.412 | INFO | ANALYSIS : 63.3%, 2.9, 95.5ms, 0 | 20260506 14:11:13.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:13.412 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260506 14:11:13.412 | INFO | PRE_HQ : 61.0%, 1.9, 59.8ms, 0 | 20260506 14:11:13.412 | INFO | BAZWRITER : 68.5%, 4.8, 168.6ms, 0 | 20260506 14:11:13.412 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.4ms, 0 | 20260506 14:11:13.412 | INFO | Uploaded 36GiB to GPU in 2.0985s (17.1551GiB/s) | 20260506 14:11:13.412 | INFO | Downloaded 38GiB to GPU in 2.22805s (17.0553GiB/s) | 20260506 14:11:18.551 | INFO | Analyzing chunk frames = [1970688,1971200) | 20260506 14:11:18.551 | INFO | MemInfo: { | 20260506 14:11:18.551 | INFO | "gpu" : " 17 GB", | 20260506 14:11:18.551 | INFO | "huge" : "", | 20260506 14:11:18.551 | INFO | "huge_shared" : " 27 GB", | 20260506 14:11:18.551 | INFO | "malloc" : " 14 GB", | 20260506 14:11:18.551 | INFO | "pinned" : " 29 GB" | 20260506 14:11:18.551 | INFO | } | 20260506 14:11:18.551 | INFO | Perf report for the last 5.13938 seconds: | 20260506 14:11:18.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:11:18.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:18.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:18.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:18.551 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:11:18.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:18.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:18.551 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:11:18.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:18.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:18.551 | INFO | Uploaded 36GiB to GPU in 2.0039s (17.9649GiB/s) | 20260506 14:11:18.551 | INFO | Downloaded 34GiB to GPU in 1.94118s (17.5151GiB/s) | 20260506 14:11:23.690 | INFO | Analyzing chunk frames = [1971200,1971712) | 20260506 14:11:23.690 | INFO | MemInfo: { | 20260506 14:11:23.690 | INFO | "gpu" : " 17 GB", | 20260506 14:11:23.690 | INFO | "huge" : "", | 20260506 14:11:23.690 | INFO | "huge_shared" : " 27 GB", | 20260506 14:11:23.690 | INFO | "malloc" : " 14 GB", | 20260506 14:11:23.690 | INFO | "pinned" : " 29 GB" | 20260506 14:11:23.690 | INFO | } | 20260506 14:11:23.690 | INFO | Perf report for the last 5.13864 seconds: | 20260506 14:11:23.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:11:23.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:23.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:23.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:23.690 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 14:11:23.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:23.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:23.690 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:11:23.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:23.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:23.690 | INFO | Uploaded 36GiB to GPU in 1.99734s (18.024GiB/s) | 20260506 14:11:23.690 | INFO | Downloaded 34GiB to GPU in 1.93448s (17.5758GiB/s) | 20260506 14:11:28.829 | INFO | Analyzing chunk frames = [1971712,1972224) | 20260506 14:11:28.829 | INFO | MemInfo: { | 20260506 14:11:28.829 | INFO | "gpu" : " 17 GB", | 20260506 14:11:28.829 | INFO | "huge" : "", | 20260506 14:11:28.829 | INFO | "huge_shared" : " 27 GB", | 20260506 14:11:28.829 | INFO | "malloc" : " 14 GB", | 20260506 14:11:28.829 | INFO | "pinned" : " 29 GB" | 20260506 14:11:28.829 | INFO | } | 20260506 14:11:28.829 | INFO | Perf report for the last 5.13937 seconds: | 20260506 14:11:28.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:11:28.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:28.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:28.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:28.829 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:11:28.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:28.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:28.829 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:11:28.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:28.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:28.829 | INFO | Uploaded 36GiB to GPU in 1.96715s (18.3006GiB/s) | 20260506 14:11:28.829 | INFO | Downloaded 34GiB to GPU in 1.91466s (17.7577GiB/s) | 20260506 14:11:33.968 | INFO | Analyzing chunk frames = [1972224,1972736) | 20260506 14:11:33.968 | INFO | MemInfo: { | 20260506 14:11:33.968 | INFO | "gpu" : " 17 GB", | 20260506 14:11:33.968 | INFO | "huge" : "", | 20260506 14:11:33.968 | INFO | "huge_shared" : " 27 GB", | 20260506 14:11:33.968 | INFO | "malloc" : " 14 GB", | 20260506 14:11:33.968 | INFO | "pinned" : " 29 GB" | 20260506 14:11:33.968 | INFO | } | 20260506 14:11:33.968 | INFO | Perf report for the last 5.139 seconds: | 20260506 14:11:33.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:11:33.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:33.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:33.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:33.968 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 14:11:33.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:33.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:33.968 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:11:33.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:33.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:33.968 | INFO | Uploaded 36GiB to GPU in 2.04701s (17.5866GiB/s) | 20260506 14:11:33.968 | INFO | Downloaded 34GiB to GPU in 1.93987s (17.527GiB/s) | 20260506 14:11:39.108 | INFO | Analyzing chunk frames = [1972736,1973248) | 20260506 14:11:39.108 | INFO | MemInfo: { | 20260506 14:11:39.108 | INFO | "gpu" : " 17 GB", | 20260506 14:11:39.108 | INFO | "huge" : "", | 20260506 14:11:39.109 | INFO | "huge_shared" : " 27 GB", | 20260506 14:11:39.109 | INFO | "malloc" : " 14 GB", | 20260506 14:11:39.109 | INFO | "pinned" : " 29 GB" | 20260506 14:11:39.109 | INFO | } | 20260506 14:11:39.109 | INFO | Perf report for the last 5.14051 seconds: | 20260506 14:11:39.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:11:39.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:39.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:39.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:39.109 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 14:11:39.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:39.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:39.109 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:11:39.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:39.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:39.109 | INFO | Uploaded 36GiB to GPU in 2.07333s (17.3634GiB/s) | 20260506 14:11:39.109 | INFO | Downloaded 34GiB to GPU in 1.9131s (17.7722GiB/s) | 20260506 14:11:44.248 | INFO | Analyzing chunk frames = [1973248,1973760) | 20260506 14:11:44.248 | INFO | MemInfo: { | 20260506 14:11:44.248 | INFO | "gpu" : " 17 GB", | 20260506 14:11:44.248 | INFO | "huge" : "", | 20260506 14:11:44.248 | INFO | "huge_shared" : " 27 GB", | 20260506 14:11:44.248 | INFO | "malloc" : " 14 GB", | 20260506 14:11:44.248 | INFO | "pinned" : " 29 GB" | 20260506 14:11:44.248 | INFO | } | 20260506 14:11:44.248 | INFO | Perf report for the last 5.13924 seconds: | 20260506 14:11:44.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:11:44.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:44.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:44.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:44.248 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:11:44.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:44.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:44.248 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 14:11:44.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:44.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:44.248 | INFO | Uploaded 36GiB to GPU in 2.00905s (17.9189GiB/s) | 20260506 14:11:44.248 | INFO | Downloaded 34GiB to GPU in 1.98796s (17.1029GiB/s) | 20260506 14:11:49.386 | INFO | Analyzing chunk frames = [1973760,1974272) | 20260506 14:11:49.386 | INFO | MemInfo: { | 20260506 14:11:49.386 | INFO | "gpu" : " 17 GB", | 20260506 14:11:49.386 | INFO | "huge" : "", | 20260506 14:11:49.386 | INFO | "huge_shared" : " 27 GB", | 20260506 14:11:49.386 | INFO | "malloc" : " 14 GB", | 20260506 14:11:49.386 | INFO | "pinned" : " 29 GB" | 20260506 14:11:49.386 | INFO | } | 20260506 14:11:49.386 | INFO | Perf report for the last 5.13808 seconds: | 20260506 14:11:49.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:11:49.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:49.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:49.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:49.386 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:11:49.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:49.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:49.386 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:11:49.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:49.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:49.386 | INFO | Uploaded 36GiB to GPU in 2.02882s (17.7443GiB/s) | 20260506 14:11:49.386 | INFO | Downloaded 34GiB to GPU in 1.96878s (17.2696GiB/s) | 20260506 14:11:49.386 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:11:49.386 | INFO | Total mem usage: 17.4 GiB | 20260506 14:11:49.386 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:11:49.386 | INFO | Recent High Water Marks Sum: 16.6 GiB (Savings: -4.77%) | 20260506 14:11:49.387 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:11:49.387 | INFO | High Water Breakdown: | 20260506 14:11:49.387 | INFO | 62.1% : 10.3 GiB ( 94.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:11:49.387 | INFO | 37.8% : 6.28 GiB ( 93.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:11:49.387 | INFO | 0.12% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:11:49.387 | INFO | ------------------------------------------ | 20260506 14:11:49.387 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:11:49.387 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:11:49.387 | INFO | Total mem usage: 26.5 GiB | 20260506 14:11:49.387 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:11:49.387 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260506 14:11:49.387 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260506 14:11:49.387 | INFO | High Water Breakdown: | 20260506 14:11:49.387 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:11:49.387 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:11:49.387 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:11:49.387 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:11:49.387 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:11:49.387 | INFO | 0.695% : 182 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:11:49.387 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:11:49.387 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:11:49.387 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:11:49.387 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:11:49.387 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:11:49.387 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:11:49.387 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:11:49.387 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:11:49.387 | INFO | ------------------------------------------ | 20260506 14:11:49.387 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:11:49.387 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:11:49.387 | INFO | Total mem usage: 25.6 GiB | 20260506 14:11:49.387 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:11:49.387 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:11:49.387 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:11:49.387 | INFO | High Water Breakdown: | 20260506 14:11:49.387 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:11:49.387 | INFO | ------------------------------------------ | 20260506 14:11:49.387 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:11:49.387 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:11:49.387 | INFO | Total mem usage: 15.7 GiB | 20260506 14:11:49.387 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:11:49.387 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:11:49.387 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 14:11:49.387 | INFO | High Water Breakdown: | 20260506 14:11:49.387 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:11:49.387 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:11:49.387 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:11:49.387 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:11:49.387 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:11:49.387 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:11:49.387 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:11:49.387 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:11:49.387 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:11:49.387 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:11:49.387 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:11:49.387 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:11:49.387 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:11:49.387 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:11:49.387 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:11:49.387 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:11:49.387 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:11:49.387 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:11:49.387 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:11:49.387 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:11:49.387 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:11:49.387 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:11:49.387 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:11:49.387 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:11:49.387 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:11:49.387 | INFO | ------------------------------------------ | 20260506 14:11:49.387 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:11:49.387 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:11:49.387 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:11:49.387 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:11:49.387 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.53e+01, 1.36e+01 / 2.24e+00 | 20260506 14:11:49.387 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.01e-01, 1.10e-02 / 5.90e-03 | 20260506 14:11:49.387 | INFO | PerfReport: 11.58%, Baseline -- 3.65e+00 / 4.42e+00, 5.74e-01 / 3.87e-01 | 20260506 14:11:49.387 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.45e+00, 1.87e-01 / 1.41e-01 | 20260506 14:11:49.387 | INFO | PerfReport: 15.81%, DME -- 3.99e+01 / 4.39e+01, 3.86e+00 / 2.41e+00 | 20260506 14:11:49.387 | INFO | PerfReport: 56.73%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.48e+00 / 1.82e+00 | 20260506 14:11:49.387 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.35e+00 / 2.91e+00, 4.08e-01 / 2.81e-01 | 20260506 14:11:49.387 | INFO | PerfReport: 3.62%, Metrics -- 1.14e+00 / 1.50e+00, 2.96e-01 / 1.91e-01 | 20260506 14:11:49.387 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.00e-01 / 4.49e-01, 6.25e-02 / 3.28e-02 | 20260506 14:11:49.387 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:11:49.387 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:11:49.387 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:11:49.387 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 14:11:49.387 | INFO | PerfReport: 21.63%, Upload -- 2.12e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260506 14:11:49.387 | INFO | PerfReport: 21.11%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 14:11:49.387 | INFO | PerfReport: 32.25%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 14:11:49.387 | INFO | PerfReport: 1.68%, LockUpload -- 1.64e+00 / 0.00e+00, 6.47e+00 / 0.00e+00 | 20260506 14:11:49.387 | INFO | PerfReport: 1.01%, LockDownload -- 9.91e-01 / 0.00e+00, 2.51e+00 / 0.00e+00 | 20260506 14:11:49.387 | INFO | PerfReport: 22.31%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 14:11:49.387 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260506 14:11:49.501 | INFO | Starting a round of Baz IO | 20260506 14:11:52.654 | INFO | Populating RTMetrics report | 20260506 14:11:52.655 | INFO | RTMetrics report populated | 20260506 14:11:52.655 | INFO | Opening tmp RTMetrics json file | 20260506 14:11:52.676 | INFO | Serializing RTMetrics to json | 20260506 14:11:52.704 | INFO | Serialization done, swapping tmp file now... | 20260506 14:11:52.720 | INFO | RTMetrics done | 20260506 14:11:52.903 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 3.402 seconds | 20260506 14:11:52.903 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.62% / 0.02% / 61.36% | 20260506 14:11:52.903 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.69169% / 1.19209e-05% / 7.54992e-05% | 20260506 14:11:52.903 | INFO | Baz IO overall: | 20260506 14:11:52.903 | INFO | BytesWritten : 1015055385963 | 20260506 14:11:52.903 | INFO | NumEvents : 491136820339 | 20260506 14:11:54.525 | INFO | Analyzing chunk frames = [1974272,1974784) | 20260506 14:11:54.526 | INFO | MemInfo: { | 20260506 14:11:54.526 | INFO | "gpu" : " 17 GB", | 20260506 14:11:54.526 | INFO | "huge" : "", | 20260506 14:11:54.526 | INFO | "huge_shared" : " 27 GB", | 20260506 14:11:54.526 | INFO | "malloc" : " 14 GB", | 20260506 14:11:54.526 | INFO | "pinned" : " 29 GB" | 20260506 14:11:54.526 | INFO | } | 20260506 14:11:54.526 | INFO | Perf report for the last 5.13958 seconds: | 20260506 14:11:54.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:11:54.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:54.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:54.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:54.526 | INFO | ANALYSIS : 63.0%, 2.9, 95.1ms, 0 | 20260506 14:11:54.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:54.526 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260506 14:11:54.526 | INFO | PRE_HQ : 61.3%, 1.9, 60.3ms, 0 | 20260506 14:11:54.526 | INFO | BAZWRITER : 65.7%, 4.8, 162.6ms, 0 | 20260506 14:11:54.526 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 14:11:54.526 | INFO | Uploaded 36GiB to GPU in 2.09177s (17.2103GiB/s) | 20260506 14:11:54.526 | INFO | Downloaded 38GiB to GPU in 2.22939s (17.045GiB/s) | 20260506 14:11:59.665 | INFO | Analyzing chunk frames = [1974784,1975296) | 20260506 14:11:59.665 | INFO | MemInfo: { | 20260506 14:11:59.665 | INFO | "gpu" : " 17 GB", | 20260506 14:11:59.665 | INFO | "huge" : "", | 20260506 14:11:59.665 | INFO | "huge_shared" : " 27 GB", | 20260506 14:11:59.665 | INFO | "malloc" : " 14 GB", | 20260506 14:11:59.665 | INFO | "pinned" : " 29 GB" | 20260506 14:11:59.665 | INFO | } | 20260506 14:11:59.665 | INFO | Perf report for the last 5.13972 seconds: | 20260506 14:11:59.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:11:59.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:59.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:59.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:59.665 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 14:11:59.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:59.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:59.666 | INFO | PRE_HQ : 27.7%, 1.0, 14.5ms, 0 | 20260506 14:11:59.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:59.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:11:59.666 | INFO | Uploaded 36GiB to GPU in 2.04418s (17.611GiB/s) | 20260506 14:11:59.666 | INFO | Downloaded 34GiB to GPU in 1.94927s (17.4424GiB/s) | 20260506 14:12:04.805 | INFO | Analyzing chunk frames = [1975296,1975808) | 20260506 14:12:04.805 | INFO | MemInfo: { | 20260506 14:12:04.805 | INFO | "gpu" : " 17 GB", | 20260506 14:12:04.805 | INFO | "huge" : "", | 20260506 14:12:04.805 | INFO | "huge_shared" : " 27 GB", | 20260506 14:12:04.805 | INFO | "malloc" : " 14 GB", | 20260506 14:12:04.805 | INFO | "pinned" : " 29 GB" | 20260506 14:12:04.805 | INFO | } | 20260506 14:12:04.805 | INFO | Perf report for the last 5.13951 seconds: | 20260506 14:12:04.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:12:04.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:04.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:04.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:04.805 | INFO | ANALYSIS : 61.4%, 3.0, 95.1ms, 0 | 20260506 14:12:04.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:04.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:04.805 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:12:04.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:04.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:04.805 | INFO | Uploaded 36GiB to GPU in 2.01964s (17.8249GiB/s) | 20260506 14:12:04.805 | INFO | Downloaded 34GiB to GPU in 1.93565s (17.5651GiB/s) | 20260506 14:12:09.943 | INFO | Analyzing chunk frames = [1975808,1976320) | 20260506 14:12:09.943 | INFO | MemInfo: { | 20260506 14:12:09.943 | INFO | "gpu" : " 17 GB", | 20260506 14:12:09.943 | INFO | "huge" : "", | 20260506 14:12:09.943 | INFO | "huge_shared" : " 27 GB", | 20260506 14:12:09.943 | INFO | "malloc" : " 14 GB", | 20260506 14:12:09.943 | INFO | "pinned" : " 29 GB" | 20260506 14:12:09.943 | INFO | } | 20260506 14:12:09.943 | INFO | Perf report for the last 5.13839 seconds: | 20260506 14:12:09.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:12:09.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:09.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:09.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:09.943 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 14:12:09.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:09.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:09.943 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:12:09.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:09.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:09.943 | INFO | Uploaded 36GiB to GPU in 2.00553s (17.9504GiB/s) | 20260506 14:12:09.944 | INFO | Downloaded 34GiB to GPU in 1.92809s (17.6341GiB/s) | 20260506 14:12:15.082 | INFO | Analyzing chunk frames = [1976320,1976832) | 20260506 14:12:15.082 | INFO | MemInfo: { | 20260506 14:12:15.082 | INFO | "gpu" : " 17 GB", | 20260506 14:12:15.082 | INFO | "huge" : "", | 20260506 14:12:15.082 | INFO | "huge_shared" : " 27 GB", | 20260506 14:12:15.082 | INFO | "malloc" : " 14 GB", | 20260506 14:12:15.082 | INFO | "pinned" : " 29 GB" | 20260506 14:12:15.082 | INFO | } | 20260506 14:12:15.082 | INFO | Perf report for the last 5.13914 seconds: | 20260506 14:12:15.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:12:15.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:15.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:15.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:15.082 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 14:12:15.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:15.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:15.082 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:12:15.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:15.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:15.083 | INFO | Uploaded 36GiB to GPU in 2.04128s (17.636GiB/s) | 20260506 14:12:15.083 | INFO | Downloaded 34GiB to GPU in 1.93696s (17.5533GiB/s) | 20260506 14:12:20.222 | INFO | Analyzing chunk frames = [1976832,1977344) | 20260506 14:12:20.222 | INFO | MemInfo: { | 20260506 14:12:20.222 | INFO | "gpu" : " 17 GB", | 20260506 14:12:20.222 | INFO | "huge" : "", | 20260506 14:12:20.222 | INFO | "huge_shared" : " 27 GB", | 20260506 14:12:20.222 | INFO | "malloc" : " 14 GB", | 20260506 14:12:20.222 | INFO | "pinned" : " 29 GB" | 20260506 14:12:20.222 | INFO | } | 20260506 14:12:20.222 | INFO | Perf report for the last 5.1401 seconds: | 20260506 14:12:20.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:12:20.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:20.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:20.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:20.223 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 14:12:20.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:20.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:20.223 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 14:12:20.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:20.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:20.223 | INFO | Uploaded 36GiB to GPU in 2.05449s (17.5226GiB/s) | 20260506 14:12:20.223 | INFO | Downloaded 34GiB to GPU in 1.89438s (17.9478GiB/s) | 20260506 14:12:25.363 | INFO | Analyzing chunk frames = [1977344,1977856) | 20260506 14:12:25.363 | INFO | MemInfo: { | 20260506 14:12:25.363 | INFO | "gpu" : " 17 GB", | 20260506 14:12:25.363 | INFO | "huge" : "", | 20260506 14:12:25.364 | INFO | "huge_shared" : " 27 GB", | 20260506 14:12:25.364 | INFO | "malloc" : " 14 GB", | 20260506 14:12:25.364 | INFO | "pinned" : " 29 GB" | 20260506 14:12:25.364 | INFO | } | 20260506 14:12:25.364 | INFO | Perf report for the last 5.14112 seconds: | 20260506 14:12:25.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:12:25.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:25.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:25.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:25.364 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 14:12:25.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:25.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:25.364 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 14:12:25.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:25.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:25.364 | INFO | Uploaded 36GiB to GPU in 2.05599s (17.5098GiB/s) | 20260506 14:12:25.364 | INFO | Downloaded 34GiB to GPU in 2.00774s (16.9345GiB/s) | 20260506 14:12:30.502 | INFO | Analyzing chunk frames = [1977856,1978368) | 20260506 14:12:30.502 | INFO | MemInfo: { | 20260506 14:12:30.502 | INFO | "gpu" : " 17 GB", | 20260506 14:12:30.502 | INFO | "huge" : "", | 20260506 14:12:30.502 | INFO | "huge_shared" : " 27 GB", | 20260506 14:12:30.502 | INFO | "malloc" : " 14 GB", | 20260506 14:12:30.502 | INFO | "pinned" : " 29 GB" | 20260506 14:12:30.502 | INFO | } | 20260506 14:12:30.502 | INFO | Perf report for the last 5.1384 seconds: | 20260506 14:12:30.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:12:30.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:30.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:30.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:30.502 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:12:30.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:30.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:30.502 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260506 14:12:30.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:30.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:30.502 | INFO | Uploaded 36GiB to GPU in 1.97195s (18.2561GiB/s) | 20260506 14:12:30.502 | INFO | Downloaded 34GiB to GPU in 1.96511s (17.3018GiB/s) | 20260506 14:12:30.615 | INFO | Starting a round of Baz IO | 20260506 14:12:33.768 | INFO | Populating RTMetrics report | 20260506 14:12:33.769 | INFO | RTMetrics report populated | 20260506 14:12:33.769 | INFO | Opening tmp RTMetrics json file | 20260506 14:12:33.924 | INFO | Serializing RTMetrics to json | 20260506 14:12:34.076 | INFO | Serialization done, swapping tmp file now... | 20260506 14:12:34.111 | INFO | RTMetrics done | 20260506 14:12:34.364 | INFO | Finished a round of Baz IO: Wrote 2.125 GiB in 3.748 seconds | 20260506 14:12:34.364 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.61% / 0.01% / 61.38% | 20260506 14:12:34.364 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.71363% / 2.38419e-05% / 6.35783e-05% | 20260506 14:12:34.364 | INFO | Baz IO overall: | 20260506 14:12:34.364 | INFO | BytesWritten : 1017336989746 | 20260506 14:12:34.364 | INFO | NumEvents : 492242220754 | 20260506 14:12:35.641 | INFO | Analyzing chunk frames = [1978368,1978880) | 20260506 14:12:35.641 | INFO | MemInfo: { | 20260506 14:12:35.641 | INFO | "gpu" : " 17 GB", | 20260506 14:12:35.641 | INFO | "huge" : "", | 20260506 14:12:35.641 | INFO | "huge_shared" : " 27 GB", | 20260506 14:12:35.641 | INFO | "malloc" : " 14 GB", | 20260506 14:12:35.641 | INFO | "pinned" : " 29 GB" | 20260506 14:12:35.641 | INFO | } | 20260506 14:12:35.641 | INFO | Perf report for the last 5.13904 seconds: | 20260506 14:12:35.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:12:35.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:35.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:35.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:35.641 | INFO | ANALYSIS : 63.0%, 2.9, 95.2ms, 0 | 20260506 14:12:35.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:35.641 | INFO | RT_METRICS : 50.7%, 1.0, 26.6ms, 0 | 20260506 14:12:35.641 | INFO | PRE_HQ : 61.1%, 1.9, 60.0ms, 0 | 20260506 14:12:35.641 | INFO | BAZWRITER : 72.4%, 4.8, 177.5ms, 0 | 20260506 14:12:35.641 | INFO | BAZ_SAMPLING : 4.8%, 1.0, 2.5ms, 0 | 20260506 14:12:35.641 | INFO | Uploaded 36GiB to GPU in 2.07727s (17.3304GiB/s) | 20260506 14:12:35.641 | INFO | Downloaded 38GiB to GPU in 2.22111s (17.1086GiB/s) | 20260506 14:12:40.779 | INFO | Analyzing chunk frames = [1978880,1979392) | 20260506 14:12:40.779 | INFO | MemInfo: { | 20260506 14:12:40.779 | INFO | "gpu" : " 17 GB", | 20260506 14:12:40.779 | INFO | "huge" : "", | 20260506 14:12:40.779 | INFO | "huge_shared" : " 27 GB", | 20260506 14:12:40.779 | INFO | "malloc" : " 14 GB", | 20260506 14:12:40.779 | INFO | "pinned" : " 29 GB" | 20260506 14:12:40.779 | INFO | } | 20260506 14:12:40.779 | INFO | Perf report for the last 5.13816 seconds: | 20260506 14:12:40.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:12:40.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:40.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:40.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:40.779 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:12:40.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:40.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:40.779 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:12:40.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:40.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:40.779 | INFO | Uploaded 36GiB to GPU in 2.03895s (17.6562GiB/s) | 20260506 14:12:40.780 | INFO | Downloaded 34GiB to GPU in 1.94357s (17.4936GiB/s) | 20260506 14:12:45.919 | INFO | Analyzing chunk frames = [1979392,1979904) | 20260506 14:12:45.919 | INFO | MemInfo: { | 20260506 14:12:45.919 | INFO | "gpu" : " 17 GB", | 20260506 14:12:45.919 | INFO | "huge" : "", | 20260506 14:12:45.919 | INFO | "huge_shared" : " 27 GB", | 20260506 14:12:45.919 | INFO | "malloc" : " 14 GB", | 20260506 14:12:45.919 | INFO | "pinned" : " 29 GB" | 20260506 14:12:45.919 | INFO | } | 20260506 14:12:45.919 | INFO | Perf report for the last 5.13974 seconds: | 20260506 14:12:45.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:12:45.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:45.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:45.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:45.919 | INFO | ANALYSIS : 61.5%, 3.0, 95.2ms, 0 | 20260506 14:12:45.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:45.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:45.919 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:12:45.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:45.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:45.919 | INFO | Uploaded 36GiB to GPU in 2.01975s (17.8239GiB/s) | 20260506 14:12:45.919 | INFO | Downloaded 34GiB to GPU in 1.93645s (17.5579GiB/s) | 20260506 14:12:51.058 | INFO | Analyzing chunk frames = [1979904,1980416) | 20260506 14:12:51.058 | INFO | MemInfo: { | 20260506 14:12:51.058 | INFO | "gpu" : " 17 GB", | 20260506 14:12:51.058 | INFO | "huge" : "", | 20260506 14:12:51.058 | INFO | "huge_shared" : " 27 GB", | 20260506 14:12:51.058 | INFO | "malloc" : " 14 GB", | 20260506 14:12:51.058 | INFO | "pinned" : " 29 GB" | 20260506 14:12:51.058 | INFO | } | 20260506 14:12:51.058 | INFO | Perf report for the last 5.1394 seconds: | 20260506 14:12:51.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:12:51.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:51.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:51.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:51.058 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 14:12:51.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:51.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:51.058 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:12:51.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:51.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:51.059 | INFO | Uploaded 36GiB to GPU in 1.98597s (18.1271GiB/s) | 20260506 14:12:51.059 | INFO | Downloaded 34GiB to GPU in 1.92953s (17.6208GiB/s) | 20260506 14:12:56.199 | INFO | Analyzing chunk frames = [1980416,1980928) | 20260506 14:12:56.199 | INFO | MemInfo: { | 20260506 14:12:56.199 | INFO | "gpu" : " 17 GB", | 20260506 14:12:56.199 | INFO | "huge" : "", | 20260506 14:12:56.199 | INFO | "huge_shared" : " 27 GB", | 20260506 14:12:56.199 | INFO | "malloc" : " 14 GB", | 20260506 14:12:56.199 | INFO | "pinned" : " 29 GB" | 20260506 14:12:56.199 | INFO | } | 20260506 14:12:56.199 | INFO | Perf report for the last 5.14098 seconds: | 20260506 14:12:56.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:12:56.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:56.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:56.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:56.199 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:12:56.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:56.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:56.199 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 14:12:56.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:56.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:12:56.200 | INFO | Uploaded 36GiB to GPU in 2.02883s (17.7442GiB/s) | 20260506 14:12:56.200 | INFO | Downloaded 34GiB to GPU in 1.928s (17.6349GiB/s) | 20260506 14:13:01.336 | INFO | Analyzing chunk frames = [1980928,1981440) | 20260506 14:13:01.336 | INFO | MemInfo: { | 20260506 14:13:01.336 | INFO | "gpu" : " 17 GB", | 20260506 14:13:01.336 | INFO | "huge" : "", | 20260506 14:13:01.336 | INFO | "huge_shared" : " 27 GB", | 20260506 14:13:01.336 | INFO | "malloc" : " 14 GB", | 20260506 14:13:01.336 | INFO | "pinned" : " 29 GB" | 20260506 14:13:01.336 | INFO | } | 20260506 14:13:01.336 | INFO | Perf report for the last 5.13701 seconds: | 20260506 14:13:01.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:13:01.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:01.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:01.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:01.336 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:13:01.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:01.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:01.336 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:13:01.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:01.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:01.337 | INFO | Uploaded 36GiB to GPU in 2.08628s (17.2556GiB/s) | 20260506 14:13:01.337 | INFO | Downloaded 34GiB to GPU in 1.97359s (17.2274GiB/s) | 20260506 14:13:06.477 | INFO | Analyzing chunk frames = [1981440,1981952) | 20260506 14:13:06.477 | INFO | MemInfo: { | 20260506 14:13:06.478 | INFO | "gpu" : " 17 GB", | 20260506 14:13:06.478 | INFO | "huge" : "", | 20260506 14:13:06.478 | INFO | "huge_shared" : " 27 GB", | 20260506 14:13:06.478 | INFO | "malloc" : " 14 GB", | 20260506 14:13:06.478 | INFO | "pinned" : " 29 GB" | 20260506 14:13:06.478 | INFO | } | 20260506 14:13:06.478 | INFO | Perf report for the last 5.1412 seconds: | 20260506 14:13:06.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:13:06.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:06.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:06.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:06.478 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 14:13:06.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:06.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:06.478 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260506 14:13:06.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:06.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:06.478 | INFO | Uploaded 36GiB to GPU in 2.03808s (17.6637GiB/s) | 20260506 14:13:06.478 | INFO | Downloaded 34GiB to GPU in 2.02053s (16.8273GiB/s) | 20260506 14:13:11.616 | INFO | Analyzing chunk frames = [1981952,1982464) | 20260506 14:13:11.616 | INFO | MemInfo: { | 20260506 14:13:11.616 | INFO | "gpu" : " 17 GB", | 20260506 14:13:11.616 | INFO | "huge" : "", | 20260506 14:13:11.616 | INFO | "huge_shared" : " 27 GB", | 20260506 14:13:11.616 | INFO | "malloc" : " 14 GB", | 20260506 14:13:11.616 | INFO | "pinned" : " 29 GB" | 20260506 14:13:11.616 | INFO | } | 20260506 14:13:11.616 | INFO | Perf report for the last 5.13825 seconds: | 20260506 14:13:11.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:13:11.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:11.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:11.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:11.616 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:13:11.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:11.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:11.616 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:13:11.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:11.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:11.616 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:13:11.616 | INFO | Total mem usage: 17.4 GiB | 20260506 14:13:11.616 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:13:11.616 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -10.6%) | 20260506 14:13:11.616 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:13:11.616 | INFO | High Water Breakdown: | 20260506 14:13:11.616 | INFO | Uploaded 36GiB to GPU in 2.03669s (17.6757GiB/s) | 20260506 14:13:11.616 | INFO | Downloaded 34GiB to GPU in 1.96744s (17.2813GiB/s) | 20260506 14:13:11.616 | INFO | 61.3% : 9.65 GiB ( 88.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:13:11.616 | INFO | 38.6% : 6.08 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:13:11.616 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:13:11.616 | INFO | ------------------------------------------ | 20260506 14:13:11.616 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:13:11.616 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:13:11.616 | INFO | Total mem usage: 26.5 GiB | 20260506 14:13:11.616 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:13:11.616 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:13:11.616 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:13:11.616 | INFO | High Water Breakdown: | 20260506 14:13:11.616 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:13:11.616 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:13:11.616 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:13:11.616 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:13:11.616 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:13:11.616 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:13:11.616 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:13:11.616 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:13:11.616 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:13:11.616 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:13:11.616 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:13:11.616 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:13:11.617 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:13:11.617 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:13:11.617 | INFO | ------------------------------------------ | 20260506 14:13:11.617 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:13:11.617 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:13:11.617 | INFO | Total mem usage: 25.6 GiB | 20260506 14:13:11.617 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:13:11.617 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:13:11.617 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:13:11.617 | INFO | High Water Breakdown: | 20260506 14:13:11.617 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:13:11.617 | INFO | ------------------------------------------ | 20260506 14:13:11.617 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:13:11.617 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:13:11.617 | INFO | Total mem usage: 15.7 GiB | 20260506 14:13:11.617 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:13:11.617 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:13:11.617 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 14:13:11.617 | INFO | High Water Breakdown: | 20260506 14:13:11.617 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:13:11.617 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:13:11.617 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:13:11.617 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:13:11.617 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:13:11.617 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:13:11.617 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:13:11.617 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:13:11.617 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:13:11.617 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:13:11.617 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:13:11.617 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:13:11.617 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:13:11.617 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:13:11.617 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:13:11.617 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:13:11.617 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:13:11.617 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:13:11.617 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:13:11.617 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:13:11.617 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:13:11.617 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:13:11.617 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:13:11.617 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:13:11.617 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:13:11.617 | INFO | ------------------------------------------ | 20260506 14:13:11.617 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:13:11.617 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:13:11.617 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:13:11.617 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:13:11.617 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.56e+01, 1.34e+01 / 1.88e+00 | 20260506 14:13:11.617 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.04e-01, 1.14e-02 / 5.39e-03 | 20260506 14:13:11.617 | INFO | PerfReport: 11.56%, Baseline -- 3.64e+00 / 4.55e+00, 5.58e-01 / 1.57e-01 | 20260506 14:13:11.617 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.49e+00, 1.84e-01 / 1.11e-01 | 20260506 14:13:11.617 | INFO | PerfReport: 15.78%, DME -- 3.98e+01 / 4.40e+01, 3.72e+00 / 2.11e+00 | 20260506 14:13:11.617 | INFO | PerfReport: 56.69%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.42e+00 / 1.74e+00 | 20260506 14:13:11.617 | INFO | PerfReport: 7.52%, PulseAccumulating -- 2.37e+00 / 2.98e+00, 4.20e-01 / 2.45e-01 | 20260506 14:13:11.617 | INFO | PerfReport: 3.64%, Metrics -- 1.15e+00 / 1.51e+00, 2.89e-01 / 2.12e-01 | 20260506 14:13:11.617 | INFO | PerfReport: 1.28%, PulseSerializing -- 4.02e-01 / 4.61e-01, 6.25e-02 / 3.20e-02 | 20260506 14:13:11.617 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:13:11.617 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:13:11.617 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:13:11.617 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 14:13:11.617 | INFO | PerfReport: 21.68%, Upload -- 2.12e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 14:13:11.617 | INFO | PerfReport: 21.13%, Download -- 2.07e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 14:13:11.617 | INFO | PerfReport: 32.21%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 14:13:11.617 | INFO | PerfReport: 1.67%, LockUpload -- 1.63e+00 / 0.00e+00, 6.34e+00 / 0.00e+00 | 20260506 14:13:11.617 | INFO | PerfReport: 0.96%, LockDownload -- 9.39e-01 / 0.00e+00, 2.41e+00 / 0.00e+00 | 20260506 14:13:11.617 | INFO | PerfReport: 22.33%, LockCompute -- 2.19e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 14:13:11.617 | INFO | PerfReport: 0.02%, Misc -- 1.51e-02 / 0.00e+00, 2.35e-02 / 0.00e+00 | 20260506 14:13:11.735 | INFO | Starting a round of Baz IO | 20260506 14:13:14.889 | INFO | Populating RTMetrics report | 20260506 14:13:14.889 | INFO | RTMetrics report populated | 20260506 14:13:14.889 | INFO | Opening tmp RTMetrics json file | 20260506 14:13:15.021 | INFO | Serializing RTMetrics to json | 20260506 14:13:15.082 | INFO | Serialization done, swapping tmp file now... | 20260506 14:13:15.121 | INFO | RTMetrics done | 20260506 14:13:15.540 | INFO | Finished a round of Baz IO: Wrote 2.13 GiB in 3.804 seconds | 20260506 14:13:15.540 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.59% / 0.01% / 61.39% | 20260506 14:13:15.540 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.72257% / 1.58946e-05% / 5.5631e-05% | 20260506 14:13:15.540 | INFO | Baz IO overall: | 20260506 14:13:15.540 | INFO | BytesWritten : 1019623872501 | 20260506 14:13:15.540 | INFO | NumEvents : 493349226751 | 20260506 14:13:16.755 | INFO | Analyzing chunk frames = [1982464,1982976) | 20260506 14:13:16.755 | INFO | MemInfo: { | 20260506 14:13:16.755 | INFO | "gpu" : " 17 GB", | 20260506 14:13:16.755 | INFO | "huge" : "", | 20260506 14:13:16.755 | INFO | "huge_shared" : " 27 GB", | 20260506 14:13:16.755 | INFO | "malloc" : " 14 GB", | 20260506 14:13:16.755 | INFO | "pinned" : " 29 GB" | 20260506 14:13:16.755 | INFO | } | 20260506 14:13:16.755 | INFO | Perf report for the last 5.13911 seconds: | 20260506 14:13:16.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:13:16.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:16.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:16.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:16.755 | INFO | ANALYSIS : 63.1%, 2.9, 95.1ms, 0 | 20260506 14:13:16.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:16.755 | INFO | RT_METRICS : 48.8%, 1.0, 25.6ms, 0 | 20260506 14:13:16.755 | INFO | PRE_HQ : 61.0%, 1.9, 60.4ms, 0 | 20260506 14:13:16.755 | INFO | BAZWRITER : 73.4%, 4.7, 177.2ms, 0 | 20260506 14:13:16.755 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 14:13:16.755 | INFO | Uploaded 36GiB to GPU in 2.05367s (17.5296GiB/s) | 20260506 14:13:16.755 | INFO | Downloaded 38GiB to GPU in 2.22726s (17.0613GiB/s) | 20260506 14:13:21.894 | INFO | Analyzing chunk frames = [1982976,1983488) | 20260506 14:13:21.894 | INFO | MemInfo: { | 20260506 14:13:21.894 | INFO | "gpu" : " 17 GB", | 20260506 14:13:21.894 | INFO | "huge" : "", | 20260506 14:13:21.894 | INFO | "huge_shared" : " 27 GB", | 20260506 14:13:21.894 | INFO | "malloc" : " 14 GB", | 20260506 14:13:21.894 | INFO | "pinned" : " 29 GB" | 20260506 14:13:21.894 | INFO | } | 20260506 14:13:21.894 | INFO | Perf report for the last 5.13944 seconds: | 20260506 14:13:21.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:13:21.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:21.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:21.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:21.894 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 14:13:21.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:21.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:21.895 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:13:21.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:21.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:21.895 | INFO | Uploaded 36GiB to GPU in 2.00797s (17.9286GiB/s) | 20260506 14:13:21.895 | INFO | Downloaded 34GiB to GPU in 1.94256s (17.5027GiB/s) | 20260506 14:13:27.033 | INFO | Analyzing chunk frames = [1983488,1984000) | 20260506 14:13:27.033 | INFO | MemInfo: { | 20260506 14:13:27.033 | INFO | "gpu" : " 17 GB", | 20260506 14:13:27.033 | INFO | "huge" : "", | 20260506 14:13:27.033 | INFO | "huge_shared" : " 27 GB", | 20260506 14:13:27.033 | INFO | "malloc" : " 14 GB", | 20260506 14:13:27.033 | INFO | "pinned" : " 29 GB" | 20260506 14:13:27.033 | INFO | } | 20260506 14:13:27.033 | INFO | Perf report for the last 5.13844 seconds: | 20260506 14:13:27.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:13:27.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:27.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:27.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:27.033 | INFO | ANALYSIS : 61.4%, 3.0, 95.0ms, 0 | 20260506 14:13:27.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:27.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:27.033 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:13:27.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:27.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:27.033 | INFO | Uploaded 36GiB to GPU in 2.00448s (17.9598GiB/s) | 20260506 14:13:27.033 | INFO | Downloaded 34GiB to GPU in 1.93402s (17.58GiB/s) | 20260506 14:13:32.174 | INFO | Analyzing chunk frames = [1984000,1984512) | 20260506 14:13:32.174 | INFO | MemInfo: { | 20260506 14:13:32.174 | INFO | "gpu" : " 17 GB", | 20260506 14:13:32.174 | INFO | "huge" : "", | 20260506 14:13:32.174 | INFO | "huge_shared" : " 27 GB", | 20260506 14:13:32.174 | INFO | "malloc" : " 14 GB", | 20260506 14:13:32.174 | INFO | "pinned" : " 29 GB" | 20260506 14:13:32.174 | INFO | } | 20260506 14:13:32.174 | INFO | Perf report for the last 5.14091 seconds: | 20260506 14:13:32.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:13:32.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:32.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:32.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:32.174 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 14:13:32.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:32.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:32.174 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 14:13:32.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:32.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:32.174 | INFO | Uploaded 36GiB to GPU in 1.98111s (18.1717GiB/s) | 20260506 14:13:32.174 | INFO | Downloaded 34GiB to GPU in 1.91637s (17.7419GiB/s) | 20260506 14:13:37.314 | INFO | Analyzing chunk frames = [1984512,1985024) | 20260506 14:13:37.314 | INFO | MemInfo: { | 20260506 14:13:37.314 | INFO | "gpu" : " 17 GB", | 20260506 14:13:37.314 | INFO | "huge" : "", | 20260506 14:13:37.314 | INFO | "huge_shared" : " 27 GB", | 20260506 14:13:37.314 | INFO | "malloc" : " 14 GB", | 20260506 14:13:37.314 | INFO | "pinned" : " 29 GB" | 20260506 14:13:37.314 | INFO | } | 20260506 14:13:37.314 | INFO | Perf report for the last 5.1399 seconds: | 20260506 14:13:37.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:13:37.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:37.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:37.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:37.314 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 14:13:37.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:37.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:37.314 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:13:37.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:37.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:37.314 | INFO | Uploaded 36GiB to GPU in 1.9958s (18.0379GiB/s) | 20260506 14:13:37.314 | INFO | Downloaded 34GiB to GPU in 1.92563s (17.6565GiB/s) | 20260506 14:13:42.453 | INFO | Analyzing chunk frames = [1985024,1985536) | 20260506 14:13:42.453 | INFO | MemInfo: { | 20260506 14:13:42.453 | INFO | "gpu" : " 17 GB", | 20260506 14:13:42.453 | INFO | "huge" : "", | 20260506 14:13:42.453 | INFO | "huge_shared" : " 27 GB", | 20260506 14:13:42.453 | INFO | "malloc" : " 14 GB", | 20260506 14:13:42.453 | INFO | "pinned" : " 29 GB" | 20260506 14:13:42.453 | INFO | } | 20260506 14:13:42.453 | INFO | Perf report for the last 5.13918 seconds: | 20260506 14:13:42.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:13:42.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:42.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:42.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:42.453 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 14:13:42.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:42.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:42.453 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260506 14:13:42.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:42.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:42.453 | INFO | Uploaded 36GiB to GPU in 2.05075s (17.5546GiB/s) | 20260506 14:13:42.453 | INFO | Downloaded 34GiB to GPU in 1.92393s (17.6722GiB/s) | 20260506 14:13:47.591 | INFO | Analyzing chunk frames = [1985536,1986048) | 20260506 14:13:47.591 | INFO | MemInfo: { | 20260506 14:13:47.591 | INFO | "gpu" : " 17 GB", | 20260506 14:13:47.591 | INFO | "huge" : "", | 20260506 14:13:47.591 | INFO | "huge_shared" : " 27 GB", | 20260506 14:13:47.591 | INFO | "malloc" : " 14 GB", | 20260506 14:13:47.591 | INFO | "pinned" : " 29 GB" | 20260506 14:13:47.591 | INFO | } | 20260506 14:13:47.591 | INFO | Perf report for the last 5.13833 seconds: | 20260506 14:13:47.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:13:47.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:47.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:47.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:47.591 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 14:13:47.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:47.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:47.591 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 14:13:47.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:47.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:47.591 | INFO | Uploaded 36GiB to GPU in 2.01432s (17.872GiB/s) | 20260506 14:13:47.592 | INFO | Downloaded 34GiB to GPU in 1.98036s (17.1686GiB/s) | 20260506 14:13:52.730 | INFO | Analyzing chunk frames = [1986048,1986560) | 20260506 14:13:52.730 | INFO | MemInfo: { | 20260506 14:13:52.730 | INFO | "gpu" : " 17 GB", | 20260506 14:13:52.730 | INFO | "huge" : "", | 20260506 14:13:52.730 | INFO | "huge_shared" : " 27 GB", | 20260506 14:13:52.730 | INFO | "malloc" : " 14 GB", | 20260506 14:13:52.730 | INFO | "pinned" : " 29 GB" | 20260506 14:13:52.730 | INFO | } | 20260506 14:13:52.730 | INFO | Perf report for the last 5.13869 seconds: | 20260506 14:13:52.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:13:52.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:52.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:52.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:52.730 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 14:13:52.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:52.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:52.730 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:13:52.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:52.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:52.730 | INFO | Uploaded 36GiB to GPU in 2.02915s (17.7414GiB/s) | 20260506 14:13:52.730 | INFO | Downloaded 34GiB to GPU in 1.98246s (17.1504GiB/s) | 20260506 14:13:52.855 | INFO | Starting a round of Baz IO | 20260506 14:13:56.005 | INFO | Populating RTMetrics report | 20260506 14:13:56.006 | INFO | RTMetrics report populated | 20260506 14:13:56.006 | INFO | Opening tmp RTMetrics json file | 20260506 14:13:56.073 | INFO | Serializing RTMetrics to json | 20260506 14:13:56.137 | INFO | Serialization done, swapping tmp file now... | 20260506 14:13:56.142 | INFO | RTMetrics done | 20260506 14:13:56.423 | INFO | Finished a round of Baz IO: Wrote 2.123 GiB in 3.568 seconds | 20260506 14:13:56.423 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.58% / 0.01% / 61.41% | 20260506 14:13:56.423 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.70645% / 3.57628e-05% / 5.16574e-05% | 20260506 14:13:56.423 | INFO | Baz IO overall: | 20260506 14:13:56.423 | INFO | BytesWritten : 1021903475016 | 20260506 14:13:56.423 | INFO | NumEvents : 494452890880 | 20260506 14:13:57.870 | INFO | Analyzing chunk frames = [1986560,1987072) | 20260506 14:13:57.870 | INFO | MemInfo: { | 20260506 14:13:57.870 | INFO | "gpu" : " 17 GB", | 20260506 14:13:57.870 | INFO | "huge" : "", | 20260506 14:13:57.870 | INFO | "huge_shared" : " 27 GB", | 20260506 14:13:57.870 | INFO | "malloc" : " 14 GB", | 20260506 14:13:57.870 | INFO | "pinned" : " 29 GB" | 20260506 14:13:57.870 | INFO | } | 20260506 14:13:57.870 | INFO | Perf report for the last 5.13983 seconds: | 20260506 14:13:57.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:13:57.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:57.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:57.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:57.870 | INFO | ANALYSIS : 63.1%, 2.9, 95.6ms, 0 | 20260506 14:13:57.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:13:57.870 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260506 14:13:57.870 | INFO | PRE_HQ : 61.2%, 1.9, 60.4ms, 0 | 20260506 14:13:57.870 | INFO | BAZWRITER : 68.7%, 4.8, 169.1ms, 0 | 20260506 14:13:57.870 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 14:13:57.870 | INFO | Uploaded 36GiB to GPU in 2.044s (17.6126GiB/s) | 20260506 14:13:57.870 | INFO | Downloaded 38GiB to GPU in 2.22329s (17.0918GiB/s) | 20260506 14:14:03.009 | INFO | Analyzing chunk frames = [1987072,1987584) | 20260506 14:14:03.009 | INFO | MemInfo: { | 20260506 14:14:03.009 | INFO | "gpu" : " 17 GB", | 20260506 14:14:03.009 | INFO | "huge" : "", | 20260506 14:14:03.009 | INFO | "huge_shared" : " 27 GB", | 20260506 14:14:03.009 | INFO | "malloc" : " 14 GB", | 20260506 14:14:03.009 | INFO | "pinned" : " 29 GB" | 20260506 14:14:03.009 | INFO | } | 20260506 14:14:03.009 | INFO | Perf report for the last 5.13943 seconds: | 20260506 14:14:03.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:14:03.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:03.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:03.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:03.009 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 14:14:03.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:03.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:03.009 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:14:03.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:03.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:03.009 | INFO | Uploaded 36GiB to GPU in 1.99881s (18.0107GiB/s) | 20260506 14:14:03.010 | INFO | Downloaded 34GiB to GPU in 1.94133s (17.5137GiB/s) | 20260506 14:14:08.148 | INFO | Analyzing chunk frames = [1987584,1988096) | 20260506 14:14:08.148 | INFO | MemInfo: { | 20260506 14:14:08.148 | INFO | "gpu" : " 17 GB", | 20260506 14:14:08.148 | INFO | "huge" : "", | 20260506 14:14:08.148 | INFO | "huge_shared" : " 27 GB", | 20260506 14:14:08.148 | INFO | "malloc" : " 14 GB", | 20260506 14:14:08.148 | INFO | "pinned" : " 29 GB" | 20260506 14:14:08.148 | INFO | } | 20260506 14:14:08.148 | INFO | Perf report for the last 5.13873 seconds: | 20260506 14:14:08.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:14:08.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:08.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:08.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:08.148 | INFO | ANALYSIS : 61.3%, 3.0, 94.8ms, 0 | 20260506 14:14:08.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:08.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:08.148 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:14:08.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:08.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:08.149 | INFO | Uploaded 36GiB to GPU in 1.98592s (18.1277GiB/s) | 20260506 14:14:08.149 | INFO | Downloaded 34GiB to GPU in 1.93306s (17.5887GiB/s) | 20260506 14:14:13.287 | INFO | Analyzing chunk frames = [1988096,1988608) | 20260506 14:14:13.287 | INFO | MemInfo: { | 20260506 14:14:13.287 | INFO | "gpu" : " 17 GB", | 20260506 14:14:13.287 | INFO | "huge" : "", | 20260506 14:14:13.288 | INFO | "huge_shared" : " 27 GB", | 20260506 14:14:13.288 | INFO | "malloc" : " 14 GB", | 20260506 14:14:13.288 | INFO | "pinned" : " 29 GB" | 20260506 14:14:13.288 | INFO | } | 20260506 14:14:13.288 | INFO | Perf report for the last 5.13957 seconds: | 20260506 14:14:13.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:14:13.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:13.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:13.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:13.288 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 14:14:13.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:13.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:13.288 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:14:13.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:13.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:13.288 | INFO | Uploaded 36GiB to GPU in 2.00396s (17.9644GiB/s) | 20260506 14:14:13.288 | INFO | Downloaded 34GiB to GPU in 1.91s (17.801GiB/s) | 20260506 14:14:18.426 | INFO | Analyzing chunk frames = [1988608,1989120) | 20260506 14:14:18.426 | INFO | MemInfo: { | 20260506 14:14:18.426 | INFO | "gpu" : " 17 GB", | 20260506 14:14:18.426 | INFO | "huge" : "", | 20260506 14:14:18.427 | INFO | "huge_shared" : " 27 GB", | 20260506 14:14:18.427 | INFO | "malloc" : " 14 GB", | 20260506 14:14:18.427 | INFO | "pinned" : " 29 GB" | 20260506 14:14:18.427 | INFO | } | 20260506 14:14:18.427 | INFO | Perf report for the last 5.13902 seconds: | 20260506 14:14:18.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:14:18.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:18.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:18.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:18.427 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:14:18.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:18.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:18.427 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:14:18.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:18.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:18.427 | INFO | Uploaded 36GiB to GPU in 2.00296s (17.9734GiB/s) | 20260506 14:14:18.427 | INFO | Downloaded 34GiB to GPU in 1.9346s (17.5747GiB/s) | 20260506 14:14:23.566 | INFO | Analyzing chunk frames = [1989120,1989632) | 20260506 14:14:23.566 | INFO | MemInfo: { | 20260506 14:14:23.566 | INFO | "gpu" : " 17 GB", | 20260506 14:14:23.566 | INFO | "huge" : "", | 20260506 14:14:23.566 | INFO | "huge_shared" : " 27 GB", | 20260506 14:14:23.566 | INFO | "malloc" : " 14 GB", | 20260506 14:14:23.566 | INFO | "pinned" : " 29 GB" | 20260506 14:14:23.566 | INFO | } | 20260506 14:14:23.566 | INFO | Perf report for the last 5.13974 seconds: | 20260506 14:14:23.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:14:23.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:23.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:23.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:23.566 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:14:23.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:23.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:23.566 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260506 14:14:23.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:23.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:23.567 | INFO | Uploaded 36GiB to GPU in 2.04664s (17.5898GiB/s) | 20260506 14:14:23.567 | INFO | Downloaded 34GiB to GPU in 1.90377s (17.8593GiB/s) | 20260506 14:14:28.706 | INFO | Analyzing chunk frames = [1989632,1990144) | 20260506 14:14:28.706 | INFO | MemInfo: { | 20260506 14:14:28.706 | INFO | "gpu" : " 17 GB", | 20260506 14:14:28.707 | INFO | "huge" : "", | 20260506 14:14:28.707 | INFO | "huge_shared" : " 27 GB", | 20260506 14:14:28.707 | INFO | "malloc" : " 14 GB", | 20260506 14:14:28.707 | INFO | "pinned" : " 29 GB" | 20260506 14:14:28.707 | INFO | } | 20260506 14:14:28.707 | INFO | Perf report for the last 5.14024 seconds: | 20260506 14:14:28.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:14:28.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:28.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:28.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:28.707 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 14:14:28.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:28.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:28.707 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 14:14:28.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:28.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:28.707 | INFO | Uploaded 36GiB to GPU in 2.03306s (17.7073GiB/s) | 20260506 14:14:28.707 | INFO | Downloaded 34GiB to GPU in 2.00593s (16.9498GiB/s) | 20260506 14:14:33.846 | INFO | Analyzing chunk frames = [1990144,1990656) | 20260506 14:14:33.846 | INFO | MemInfo: { | 20260506 14:14:33.846 | INFO | "gpu" : " 17 GB", | 20260506 14:14:33.846 | INFO | "huge" : "", | 20260506 14:14:33.846 | INFO | "huge_shared" : " 27 GB", | 20260506 14:14:33.846 | INFO | "malloc" : " 14 GB", | 20260506 14:14:33.846 | INFO | "pinned" : " 29 GB" | 20260506 14:14:33.846 | INFO | } | 20260506 14:14:33.846 | INFO | Perf report for the last 5.13902 seconds: | 20260506 14:14:33.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:14:33.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:33.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:33.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:33.846 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 14:14:33.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:33.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:33.846 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 14:14:33.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:33.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:33.846 | INFO | Uploaded 36GiB to GPU in 2.02991s (17.7348GiB/s) | 20260506 14:14:33.846 | INFO | Downloaded 34GiB to GPU in 1.98221s (17.1526GiB/s) | 20260506 14:14:33.846 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:14:33.846 | INFO | Total mem usage: 17.4 GiB | 20260506 14:14:33.846 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:14:33.846 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -12%) | 20260506 14:14:33.846 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:14:33.846 | INFO | High Water Breakdown: | 20260506 14:14:33.846 | INFO | 61.1% : 9.49 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:14:33.846 | INFO | 38.8% : 6.03 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:14:33.846 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:14:33.846 | INFO | ------------------------------------------ | 20260506 14:14:33.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:14:33.846 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:14:33.846 | INFO | Total mem usage: 26.5 GiB | 20260506 14:14:33.846 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:14:33.846 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:14:33.846 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:14:33.846 | INFO | High Water Breakdown: | 20260506 14:14:33.846 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:14:33.846 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:14:33.846 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:14:33.846 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:14:33.846 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:14:33.846 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:14:33.846 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:14:33.846 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:14:33.846 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:14:33.846 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:14:33.846 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:14:33.846 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:14:33.846 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:14:33.846 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:14:33.846 | INFO | ------------------------------------------ | 20260506 14:14:33.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:14:33.846 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:14:33.846 | INFO | Total mem usage: 25.6 GiB | 20260506 14:14:33.847 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:14:33.847 | INFO | Recent High Water Marks Sum: 24.4 GiB (Savings: -5.13%) | 20260506 14:14:33.847 | INFO | Unused Cache: 1.25 GiB (Waste: 4.88%) | 20260506 14:14:33.847 | INFO | High Water Breakdown: | 20260506 14:14:33.847 | INFO | 100% : 24.4 GiB ( 95.1% Of Peak) : WXIPCDataSourceConfig | 20260506 14:14:33.847 | INFO | ------------------------------------------ | 20260506 14:14:33.847 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:14:33.847 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:14:33.847 | INFO | Total mem usage: 15.7 GiB | 20260506 14:14:33.847 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:14:33.847 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:14:33.847 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 14:14:33.847 | INFO | High Water Breakdown: | 20260506 14:14:33.847 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:14:33.847 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:14:33.847 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:14:33.847 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:14:33.847 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:14:33.847 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:14:33.847 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:14:33.847 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:14:33.847 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:14:33.847 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:14:33.847 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:14:33.847 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:14:33.847 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:14:33.847 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:14:33.847 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:14:33.847 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:14:33.847 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:14:33.847 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:14:33.847 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:14:33.847 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:14:33.847 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:14:33.847 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:14:33.847 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:14:33.847 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:14:33.847 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:14:33.847 | INFO | ------------------------------------------ | 20260506 14:14:33.847 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:14:33.847 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:14:33.847 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:14:33.847 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:14:33.847 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.59e+01, 1.36e+01 / 2.04e+00 | 20260506 14:14:33.847 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.01e-01, 1.12e-02 / 6.61e-03 | 20260506 14:14:33.847 | INFO | PerfReport: 11.58%, Baseline -- 3.66e+00 / 4.42e+00, 5.77e-01 / 3.67e-01 | 20260506 14:14:33.847 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.46e+00, 1.92e-01 / 1.45e-01 | 20260506 14:14:33.847 | INFO | PerfReport: 15.81%, DME -- 4.00e+01 / 4.44e+01, 3.97e+00 / 2.20e+00 | 20260506 14:14:33.847 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.54e+00 / 1.69e+00 | 20260506 14:14:33.847 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.35e+00 / 2.95e+00, 4.04e-01 / 2.68e-01 | 20260506 14:14:33.847 | INFO | PerfReport: 3.62%, Metrics -- 1.15e+00 / 1.55e+00, 2.97e-01 / 2.07e-01 | 20260506 14:14:33.847 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.02e-01 / 4.54e-01, 6.41e-02 / 2.97e-02 | 20260506 14:14:33.847 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:14:33.847 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:14:33.847 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:14:33.847 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 14:14:33.847 | INFO | PerfReport: 21.48%, Upload -- 2.10e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 14:14:33.847 | INFO | PerfReport: 21.07%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 14:14:33.847 | INFO | PerfReport: 32.31%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 14:14:33.847 | INFO | PerfReport: 1.58%, LockUpload -- 1.55e+00 / 0.00e+00, 6.23e+00 / 0.00e+00 | 20260506 14:14:33.847 | INFO | PerfReport: 1.06%, LockDownload -- 1.04e+00 / 0.00e+00, 2.69e+00 / 0.00e+00 | 20260506 14:14:33.847 | INFO | PerfReport: 22.48%, LockCompute -- 2.20e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:14:33.847 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.35e-02 / 0.00e+00 | 20260506 14:14:33.968 | INFO | Starting a round of Baz IO | 20260506 14:14:37.111 | INFO | Populating RTMetrics report | 20260506 14:14:37.112 | INFO | RTMetrics report populated | 20260506 14:14:37.112 | INFO | Opening tmp RTMetrics json file | 20260506 14:14:37.207 | INFO | Serializing RTMetrics to json | 20260506 14:14:37.262 | INFO | Serialization done, swapping tmp file now... | 20260506 14:14:37.277 | INFO | RTMetrics done | 20260506 14:14:37.390 | INFO | Finished a round of Baz IO: Wrote 2.121 GiB in 3.421 seconds | 20260506 14:14:37.390 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.56% / 0.02% / 61.42% | 20260506 14:14:37.390 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.70072% / 1.19209e-05% / 3.97364e-05% | 20260506 14:14:37.390 | INFO | Baz IO overall: | 20260506 14:14:37.390 | INFO | BytesWritten : 1024180481433 | 20260506 14:14:37.390 | INFO | NumEvents : 495556320250 | 20260506 14:14:38.985 | INFO | Analyzing chunk frames = [1990656,1991168) | 20260506 14:14:38.985 | INFO | MemInfo: { | 20260506 14:14:38.985 | INFO | "gpu" : " 17 GB", | 20260506 14:14:38.985 | INFO | "huge" : "", | 20260506 14:14:38.985 | INFO | "huge_shared" : " 27 GB", | 20260506 14:14:38.985 | INFO | "malloc" : " 14 GB", | 20260506 14:14:38.985 | INFO | "pinned" : " 29 GB" | 20260506 14:14:38.985 | INFO | } | 20260506 14:14:38.985 | INFO | Perf report for the last 5.13896 seconds: | 20260506 14:14:38.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:14:38.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:38.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:38.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:38.985 | INFO | ANALYSIS : 62.9%, 2.9, 95.3ms, 0 | 20260506 14:14:38.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:38.985 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260506 14:14:38.985 | INFO | PRE_HQ : 61.0%, 1.9, 59.7ms, 0 | 20260506 14:14:38.985 | INFO | BAZWRITER : 66.0%, 4.8, 162.4ms, 0 | 20260506 14:14:38.985 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 14:14:38.985 | INFO | Uploaded 36GiB to GPU in 2.07891s (17.3168GiB/s) | 20260506 14:14:38.985 | INFO | Downloaded 38GiB to GPU in 2.22666s (17.0659GiB/s) | 20260506 14:14:44.124 | INFO | Analyzing chunk frames = [1991168,1991680) | 20260506 14:14:44.124 | INFO | MemInfo: { | 20260506 14:14:44.124 | INFO | "gpu" : " 17 GB", | 20260506 14:14:44.124 | INFO | "huge" : "", | 20260506 14:14:44.124 | INFO | "huge_shared" : " 27 GB", | 20260506 14:14:44.124 | INFO | "malloc" : " 14 GB", | 20260506 14:14:44.124 | INFO | "pinned" : " 29 GB" | 20260506 14:14:44.124 | INFO | } | 20260506 14:14:44.124 | INFO | Perf report for the last 5.13962 seconds: | 20260506 14:14:44.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:14:44.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:44.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:44.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:44.124 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:14:44.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:44.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:44.124 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:14:44.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:44.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:44.125 | INFO | Uploaded 36GiB to GPU in 2.03802s (17.6642GiB/s) | 20260506 14:14:44.125 | INFO | Downloaded 34GiB to GPU in 1.95047s (17.4317GiB/s) | 20260506 14:14:49.264 | INFO | Analyzing chunk frames = [1991680,1992192) | 20260506 14:14:49.264 | INFO | MemInfo: { | 20260506 14:14:49.264 | INFO | "gpu" : " 17 GB", | 20260506 14:14:49.265 | INFO | "huge" : "", | 20260506 14:14:49.265 | INFO | "huge_shared" : " 27 GB", | 20260506 14:14:49.265 | INFO | "malloc" : " 14 GB", | 20260506 14:14:49.265 | INFO | "pinned" : " 29 GB" | 20260506 14:14:49.265 | INFO | } | 20260506 14:14:49.265 | INFO | Perf report for the last 5.1403 seconds: | 20260506 14:14:49.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:14:49.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:49.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:49.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:49.265 | INFO | ANALYSIS : 61.4%, 3.0, 95.0ms, 0 | 20260506 14:14:49.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:49.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:49.265 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:14:49.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:49.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:49.265 | INFO | Uploaded 36GiB to GPU in 2.0108s (17.9033GiB/s) | 20260506 14:14:49.265 | INFO | Downloaded 34GiB to GPU in 1.93394s (17.5807GiB/s) | 20260506 14:14:54.402 | INFO | Analyzing chunk frames = [1992192,1992704) | 20260506 14:14:54.402 | INFO | MemInfo: { | 20260506 14:14:54.402 | INFO | "gpu" : " 17 GB", | 20260506 14:14:54.402 | INFO | "huge" : "", | 20260506 14:14:54.402 | INFO | "huge_shared" : " 27 GB", | 20260506 14:14:54.403 | INFO | "malloc" : " 14 GB", | 20260506 14:14:54.403 | INFO | "pinned" : " 29 GB" | 20260506 14:14:54.403 | INFO | } | 20260506 14:14:54.403 | INFO | Perf report for the last 5.13796 seconds: | 20260506 14:14:54.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:14:54.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:54.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:54.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:54.403 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 14:14:54.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:54.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:54.403 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:14:54.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:54.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:54.403 | INFO | Uploaded 36GiB to GPU in 1.99076s (18.0835GiB/s) | 20260506 14:14:54.403 | INFO | Downloaded 34GiB to GPU in 1.92712s (17.6429GiB/s) | 20260506 14:14:59.541 | INFO | Analyzing chunk frames = [1992704,1993216) | 20260506 14:14:59.541 | INFO | MemInfo: { | 20260506 14:14:59.541 | INFO | "gpu" : " 17 GB", | 20260506 14:14:59.541 | INFO | "huge" : "", | 20260506 14:14:59.541 | INFO | "huge_shared" : " 27 GB", | 20260506 14:14:59.541 | INFO | "malloc" : " 14 GB", | 20260506 14:14:59.541 | INFO | "pinned" : " 29 GB" | 20260506 14:14:59.541 | INFO | } | 20260506 14:14:59.541 | INFO | Perf report for the last 5.13854 seconds: | 20260506 14:14:59.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:14:59.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:59.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:59.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:59.541 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:14:59.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:59.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:59.541 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:14:59.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:59.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:14:59.541 | INFO | Uploaded 36GiB to GPU in 1.99892s (18.0098GiB/s) | 20260506 14:14:59.541 | INFO | Downloaded 34GiB to GPU in 1.9351s (17.5701GiB/s) | 20260506 14:15:04.681 | INFO | Analyzing chunk frames = [1993216,1993728) | 20260506 14:15:04.681 | INFO | MemInfo: { | 20260506 14:15:04.681 | INFO | "gpu" : " 17 GB", | 20260506 14:15:04.681 | INFO | "huge" : "", | 20260506 14:15:04.681 | INFO | "huge_shared" : " 27 GB", | 20260506 14:15:04.681 | INFO | "malloc" : " 14 GB", | 20260506 14:15:04.681 | INFO | "pinned" : " 29 GB" | 20260506 14:15:04.681 | INFO | } | 20260506 14:15:04.681 | INFO | Perf report for the last 5.13991 seconds: | 20260506 14:15:04.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:15:04.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:04.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:04.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:04.681 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 14:15:04.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:04.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:04.681 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 14:15:04.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:04.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:04.681 | INFO | Uploaded 36GiB to GPU in 2.04997s (17.5612GiB/s) | 20260506 14:15:04.681 | INFO | Downloaded 34GiB to GPU in 1.90709s (17.8282GiB/s) | 20260506 14:15:09.821 | INFO | Analyzing chunk frames = [1993728,1994240) | 20260506 14:15:09.821 | INFO | MemInfo: { | 20260506 14:15:09.821 | INFO | "gpu" : " 17 GB", | 20260506 14:15:09.821 | INFO | "huge" : "", | 20260506 14:15:09.821 | INFO | "huge_shared" : " 27 GB", | 20260506 14:15:09.821 | INFO | "malloc" : " 14 GB", | 20260506 14:15:09.821 | INFO | "pinned" : " 29 GB" | 20260506 14:15:09.821 | INFO | } | 20260506 14:15:09.821 | INFO | Perf report for the last 5.14006 seconds: | 20260506 14:15:09.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:15:09.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:09.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:09.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:09.821 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:15:09.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:09.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:09.821 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:15:09.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:09.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:09.821 | INFO | Uploaded 36GiB to GPU in 2.02963s (17.7373GiB/s) | 20260506 14:15:09.821 | INFO | Downloaded 34GiB to GPU in 1.99323s (17.0578GiB/s) | 20260506 14:15:14.960 | INFO | Analyzing chunk frames = [1994240,1994752) | 20260506 14:15:14.960 | INFO | MemInfo: { | 20260506 14:15:14.960 | INFO | "gpu" : " 17 GB", | 20260506 14:15:14.960 | INFO | "huge" : "", | 20260506 14:15:14.960 | INFO | "huge_shared" : " 27 GB", | 20260506 14:15:14.960 | INFO | "malloc" : " 14 GB", | 20260506 14:15:14.960 | INFO | "pinned" : " 29 GB" | 20260506 14:15:14.960 | INFO | } | 20260506 14:15:14.960 | INFO | Perf report for the last 5.13882 seconds: | 20260506 14:15:14.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:15:14.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:14.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:14.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:14.960 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 14:15:14.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:14.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:14.960 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:15:14.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:14.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:14.960 | INFO | Uploaded 36GiB to GPU in 2.04167s (17.6326GiB/s) | 20260506 14:15:14.960 | INFO | Downloaded 34GiB to GPU in 1.96473s (17.3052GiB/s) | 20260506 14:15:15.079 | INFO | Starting a round of Baz IO | 20260506 14:15:18.235 | INFO | Populating RTMetrics report | 20260506 14:15:18.235 | INFO | RTMetrics report populated | 20260506 14:15:18.235 | INFO | Opening tmp RTMetrics json file | 20260506 14:15:18.288 | INFO | Serializing RTMetrics to json | 20260506 14:15:18.359 | INFO | Serialization done, swapping tmp file now... | 20260506 14:15:18.362 | INFO | RTMetrics done | 20260506 14:15:18.753 | INFO | Finished a round of Baz IO: Wrote 2.119 GiB in 3.674 seconds | 20260506 14:15:18.753 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.55% / 0.01% / 61.44% | 20260506 14:15:18.753 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.68928% / 2.78155e-05% / 8.34465e-05% | 20260506 14:15:18.753 | INFO | Baz IO overall: | 20260506 14:15:18.753 | INFO | BytesWritten : 1026455267680 | 20260506 14:15:18.753 | INFO | NumEvents : 496658704377 | 20260506 14:15:20.099 | INFO | Analyzing chunk frames = [1994752,1995264) | 20260506 14:15:20.099 | INFO | MemInfo: { | 20260506 14:15:20.099 | INFO | "gpu" : " 17 GB", | 20260506 14:15:20.099 | INFO | "huge" : "", | 20260506 14:15:20.099 | INFO | "huge_shared" : " 27 GB", | 20260506 14:15:20.099 | INFO | "malloc" : " 14 GB", | 20260506 14:15:20.099 | INFO | "pinned" : " 29 GB" | 20260506 14:15:20.099 | INFO | } | 20260506 14:15:20.099 | INFO | Perf report for the last 5.13927 seconds: | 20260506 14:15:20.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:15:20.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:20.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:20.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:20.099 | INFO | ANALYSIS : 63.1%, 2.9, 95.5ms, 0 | 20260506 14:15:20.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:20.099 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260506 14:15:20.099 | INFO | PRE_HQ : 61.1%, 1.9, 59.8ms, 0 | 20260506 14:15:20.099 | INFO | BAZWRITER : 70.9%, 4.8, 174.4ms, 0 | 20260506 14:15:20.099 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 14:15:20.099 | INFO | Uploaded 36GiB to GPU in 2.05109s (17.5516GiB/s) | 20260506 14:15:20.100 | INFO | Downloaded 38GiB to GPU in 2.22876s (17.0498GiB/s) | 20260506 14:15:25.237 | INFO | Analyzing chunk frames = [1995264,1995776) | 20260506 14:15:25.237 | INFO | MemInfo: { | 20260506 14:15:25.237 | INFO | "gpu" : " 17 GB", | 20260506 14:15:25.237 | INFO | "huge" : "", | 20260506 14:15:25.237 | INFO | "huge_shared" : " 27 GB", | 20260506 14:15:25.237 | INFO | "malloc" : " 14 GB", | 20260506 14:15:25.237 | INFO | "pinned" : " 29 GB" | 20260506 14:15:25.237 | INFO | } | 20260506 14:15:25.237 | INFO | Perf report for the last 5.13819 seconds: | 20260506 14:15:25.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:15:25.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:25.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:25.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:25.237 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:15:25.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:25.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:25.237 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:15:25.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:25.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:25.238 | INFO | Uploaded 36GiB to GPU in 1.99018s (18.0888GiB/s) | 20260506 14:15:25.238 | INFO | Downloaded 34GiB to GPU in 1.94541s (17.477GiB/s) | 20260506 14:15:30.377 | INFO | Analyzing chunk frames = [1995776,1996288) | 20260506 14:15:30.377 | INFO | MemInfo: { | 20260506 14:15:30.377 | INFO | "gpu" : " 17 GB", | 20260506 14:15:30.377 | INFO | "huge" : "", | 20260506 14:15:30.377 | INFO | "huge_shared" : " 27 GB", | 20260506 14:15:30.377 | INFO | "malloc" : " 14 GB", | 20260506 14:15:30.377 | INFO | "pinned" : " 29 GB" | 20260506 14:15:30.377 | INFO | } | 20260506 14:15:30.377 | INFO | Perf report for the last 5.14002 seconds: | 20260506 14:15:30.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:15:30.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:30.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:30.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:30.377 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:15:30.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:30.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:30.378 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:15:30.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:30.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:30.378 | INFO | Uploaded 36GiB to GPU in 1.98649s (18.1224GiB/s) | 20260506 14:15:30.378 | INFO | Downloaded 34GiB to GPU in 1.93058s (17.6113GiB/s) | 20260506 14:15:35.518 | INFO | Analyzing chunk frames = [1996288,1996800) | 20260506 14:15:35.518 | INFO | MemInfo: { | 20260506 14:15:35.518 | INFO | "gpu" : " 17 GB", | 20260506 14:15:35.518 | INFO | "huge" : "", | 20260506 14:15:35.518 | INFO | "huge_shared" : " 27 GB", | 20260506 14:15:35.518 | INFO | "malloc" : " 14 GB", | 20260506 14:15:35.518 | INFO | "pinned" : " 29 GB" | 20260506 14:15:35.518 | INFO | } | 20260506 14:15:35.518 | INFO | Perf report for the last 5.14029 seconds: | 20260506 14:15:35.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:15:35.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:35.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:35.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:35.518 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 14:15:35.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:35.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:35.518 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:15:35.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:35.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:35.518 | INFO | Uploaded 36GiB to GPU in 1.98285s (18.1557GiB/s) | 20260506 14:15:35.518 | INFO | Downloaded 34GiB to GPU in 1.92437s (17.6681GiB/s) | 20260506 14:15:40.657 | INFO | Analyzing chunk frames = [1996800,1997312) | 20260506 14:15:40.657 | INFO | MemInfo: { | 20260506 14:15:40.657 | INFO | "gpu" : " 17 GB", | 20260506 14:15:40.657 | INFO | "huge" : "", | 20260506 14:15:40.657 | INFO | "huge_shared" : " 27 GB", | 20260506 14:15:40.657 | INFO | "malloc" : " 14 GB", | 20260506 14:15:40.657 | INFO | "pinned" : " 29 GB" | 20260506 14:15:40.657 | INFO | } | 20260506 14:15:40.657 | INFO | Perf report for the last 5.13934 seconds: | 20260506 14:15:40.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:15:40.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:40.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:40.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:40.657 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 14:15:40.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:40.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:40.657 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:15:40.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:40.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:40.658 | INFO | Uploaded 36GiB to GPU in 1.99225s (18.07GiB/s) | 20260506 14:15:40.658 | INFO | Downloaded 34GiB to GPU in 1.92864s (17.629GiB/s) | 20260506 14:15:45.796 | INFO | Analyzing chunk frames = [1997312,1997824) | 20260506 14:15:45.796 | INFO | MemInfo: { | 20260506 14:15:45.796 | INFO | "gpu" : " 17 GB", | 20260506 14:15:45.796 | INFO | "huge" : "", | 20260506 14:15:45.796 | INFO | "huge_shared" : " 27 GB", | 20260506 14:15:45.796 | INFO | "malloc" : " 14 GB", | 20260506 14:15:45.796 | INFO | "pinned" : " 29 GB" | 20260506 14:15:45.796 | INFO | } | 20260506 14:15:45.796 | INFO | Perf report for the last 5.1386 seconds: | 20260506 14:15:45.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:15:45.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:45.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:45.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:45.796 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 14:15:45.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:45.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:45.796 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 14:15:45.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:45.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:45.796 | INFO | Uploaded 36GiB to GPU in 2.02301s (17.7952GiB/s) | 20260506 14:15:45.796 | INFO | Downloaded 34GiB to GPU in 1.91107s (17.7911GiB/s) | 20260506 14:15:50.935 | INFO | Analyzing chunk frames = [1997824,1998336) | 20260506 14:15:50.935 | INFO | MemInfo: { | 20260506 14:15:50.935 | INFO | "gpu" : " 17 GB", | 20260506 14:15:50.935 | INFO | "huge" : "", | 20260506 14:15:50.936 | INFO | "huge_shared" : " 27 GB", | 20260506 14:15:50.936 | INFO | "malloc" : " 14 GB", | 20260506 14:15:50.936 | INFO | "pinned" : " 29 GB" | 20260506 14:15:50.936 | INFO | } | 20260506 14:15:50.936 | INFO | Perf report for the last 5.13995 seconds: | 20260506 14:15:50.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:15:50.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:50.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:50.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:50.936 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:15:50.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:50.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:50.936 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 14:15:50.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:50.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:50.936 | INFO | Uploaded 36GiB to GPU in 2.02679s (17.7621GiB/s) | 20260506 14:15:50.936 | INFO | Downloaded 34GiB to GPU in 1.98667s (17.1141GiB/s) | 20260506 14:15:56.075 | INFO | Analyzing chunk frames = [1998336,1998848) | 20260506 14:15:56.075 | INFO | MemInfo: { | 20260506 14:15:56.075 | INFO | "gpu" : " 17 GB", | 20260506 14:15:56.075 | INFO | "huge" : "", | 20260506 14:15:56.075 | INFO | "huge_shared" : " 27 GB", | 20260506 14:15:56.075 | INFO | "malloc" : " 14 GB", | 20260506 14:15:56.075 | INFO | "pinned" : " 29 GB" | 20260506 14:15:56.075 | INFO | } | 20260506 14:15:56.075 | INFO | Perf report for the last 5.1392 seconds: | 20260506 14:15:56.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:15:56.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:56.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:56.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:56.075 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:15:56.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:56.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:56.075 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 14:15:56.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:56.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:15:56.075 | INFO | Uploaded 36GiB to GPU in 2.01958s (17.8255GiB/s) | 20260506 14:15:56.075 | INFO | Downloaded 34GiB to GPU in 1.97574s (17.2088GiB/s) | 20260506 14:15:56.075 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:15:56.075 | INFO | Total mem usage: 17.4 GiB | 20260506 14:15:56.075 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:15:56.075 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -12.5%) | 20260506 14:15:56.075 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:15:56.075 | INFO | High Water Breakdown: | 20260506 14:15:56.075 | INFO | 61% : 9.45 GiB ( 86.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:15:56.075 | INFO | 38.8% : 6.01 GiB ( 89.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:15:56.075 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:15:56.075 | INFO | ------------------------------------------ | 20260506 14:15:56.075 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:15:56.075 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:15:56.075 | INFO | Total mem usage: 26.5 GiB | 20260506 14:15:56.075 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:15:56.076 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:15:56.076 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:15:56.076 | INFO | High Water Breakdown: | 20260506 14:15:56.076 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:15:56.076 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:15:56.076 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:15:56.076 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:15:56.076 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:15:56.076 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:15:56.076 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:15:56.076 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:15:56.076 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:15:56.076 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:15:56.076 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:15:56.076 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:15:56.076 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:15:56.076 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:15:56.076 | INFO | ------------------------------------------ | 20260506 14:15:56.076 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:15:56.076 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:15:56.076 | INFO | Total mem usage: 25.6 GiB | 20260506 14:15:56.076 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:15:56.076 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:15:56.076 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:15:56.076 | INFO | High Water Breakdown: | 20260506 14:15:56.076 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:15:56.076 | INFO | ------------------------------------------ | 20260506 14:15:56.076 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:15:56.076 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:15:56.076 | INFO | Total mem usage: 15.7 GiB | 20260506 14:15:56.076 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:15:56.076 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:15:56.076 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 14:15:56.076 | INFO | High Water Breakdown: | 20260506 14:15:56.076 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:15:56.076 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:15:56.076 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:15:56.076 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:15:56.076 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:15:56.076 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:15:56.076 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:15:56.076 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:15:56.076 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:15:56.076 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:15:56.076 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:15:56.076 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:15:56.076 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:15:56.076 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:15:56.076 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:15:56.076 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:15:56.076 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:15:56.076 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:15:56.076 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:15:56.076 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:15:56.076 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:15:56.076 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:15:56.076 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:15:56.076 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:15:56.076 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:15:56.076 | INFO | ------------------------------------------ | 20260506 14:15:56.076 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:15:56.076 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:15:56.076 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:15:56.076 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:15:56.076 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.54e+01, 1.35e+01 / 2.14e+00 | 20260506 14:15:56.076 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.01e-01, 1.00e-02 / 5.83e-03 | 20260506 14:15:56.076 | INFO | PerfReport: 11.58%, Baseline -- 3.65e+00 / 4.47e+00, 5.71e-01 / 3.18e-01 | 20260506 14:15:56.076 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.44e+00, 1.86e-01 / 1.54e-01 | 20260506 14:15:56.076 | INFO | PerfReport: 15.80%, DME -- 3.99e+01 / 4.42e+01, 3.87e+00 / 2.56e+00 | 20260506 14:15:56.076 | INFO | PerfReport: 56.72%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.47e+00 / 1.63e+00 | 20260506 14:15:56.076 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.36e+00 / 2.94e+00, 4.12e-01 / 2.89e-01 | 20260506 14:15:56.076 | INFO | PerfReport: 3.63%, Metrics -- 1.14e+00 / 1.50e+00, 2.90e-01 / 1.83e-01 | 20260506 14:15:56.076 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.00e-01 / 4.52e-01, 6.25e-02 / 2.96e-02 | 20260506 14:15:56.076 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:15:56.076 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:15:56.076 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:15:56.077 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 14:15:56.077 | INFO | PerfReport: 21.50%, Upload -- 2.10e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 14:15:56.077 | INFO | PerfReport: 21.07%, Download -- 2.06e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 14:15:56.077 | INFO | PerfReport: 32.27%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 14:15:56.077 | INFO | PerfReport: 1.61%, LockUpload -- 1.58e+00 / 0.00e+00, 6.38e+00 / 0.00e+00 | 20260506 14:15:56.077 | INFO | PerfReport: 1.08%, LockDownload -- 1.06e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260506 14:15:56.077 | INFO | PerfReport: 22.45%, LockCompute -- 2.20e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:15:56.077 | INFO | PerfReport: 0.02%, Misc -- 1.50e-02 / 0.00e+00, 2.39e-02 / 0.00e+00 | 20260506 14:15:56.188 | INFO | Starting a round of Baz IO | 20260506 14:15:59.358 | INFO | Populating RTMetrics report | 20260506 14:15:59.358 | INFO | RTMetrics report populated | 20260506 14:15:59.358 | INFO | Opening tmp RTMetrics json file | 20260506 14:15:59.426 | INFO | Serializing RTMetrics to json | 20260506 14:15:59.453 | INFO | Serialization done, swapping tmp file now... | 20260506 14:15:59.461 | INFO | RTMetrics done | 20260506 14:15:59.558 | INFO | Finished a round of Baz IO: Wrote 2.118 GiB in 3.369 seconds | 20260506 14:15:59.558 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.54% / 0.01% / 61.45% | 20260506 14:15:59.558 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.68148% / 1.58946e-05% / 5.16574e-05% | 20260506 14:15:59.558 | INFO | Baz IO overall: | 20260506 14:15:59.558 | INFO | BytesWritten : 1028729776377 | 20260506 14:15:59.558 | INFO | NumEvents : 497761370091 | 20260506 14:16:01.213 | INFO | Analyzing chunk frames = [1998848,1999360) | 20260506 14:16:01.213 | INFO | MemInfo: { | 20260506 14:16:01.213 | INFO | "gpu" : " 17 GB", | 20260506 14:16:01.213 | INFO | "huge" : "", | 20260506 14:16:01.213 | INFO | "huge_shared" : " 27 GB", | 20260506 14:16:01.213 | INFO | "malloc" : " 14 GB", | 20260506 14:16:01.213 | INFO | "pinned" : " 29 GB" | 20260506 14:16:01.213 | INFO | } | 20260506 14:16:01.213 | INFO | Perf report for the last 5.13817 seconds: | 20260506 14:16:01.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:16:01.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:01.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:01.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:01.213 | INFO | ANALYSIS : 63.2%, 2.9, 95.4ms, 0 | 20260506 14:16:01.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:01.213 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260506 14:16:01.213 | INFO | PRE_HQ : 61.3%, 1.9, 60.0ms, 0 | 20260506 14:16:01.213 | INFO | BAZWRITER : 65.0%, 4.8, 160.3ms, 0 | 20260506 14:16:01.213 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 14:16:01.213 | INFO | Uploaded 36GiB to GPU in 2.06493s (17.434GiB/s) | 20260506 14:16:01.213 | INFO | Downloaded 38GiB to GPU in 2.2424s (16.9461GiB/s) | 20260506 14:16:06.353 | INFO | Analyzing chunk frames = [1999360,1999872) | 20260506 14:16:06.353 | INFO | MemInfo: { | 20260506 14:16:06.353 | INFO | "gpu" : " 17 GB", | 20260506 14:16:06.353 | INFO | "huge" : "", | 20260506 14:16:06.353 | INFO | "huge_shared" : " 27 GB", | 20260506 14:16:06.353 | INFO | "malloc" : " 14 GB", | 20260506 14:16:06.353 | INFO | "pinned" : " 29 GB" | 20260506 14:16:06.354 | INFO | } | 20260506 14:16:06.354 | INFO | Perf report for the last 5.14046 seconds: | 20260506 14:16:06.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:16:06.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:06.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:06.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:06.354 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 14:16:06.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:06.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:06.354 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:16:06.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:06.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:06.354 | INFO | Uploaded 36GiB to GPU in 2.00924s (17.9173GiB/s) | 20260506 14:16:06.354 | INFO | Downloaded 34GiB to GPU in 1.94642s (17.4679GiB/s) | 20260506 14:16:11.492 | INFO | Analyzing chunk frames = [1999872,2000384) | 20260506 14:16:11.492 | INFO | MemInfo: { | 20260506 14:16:11.492 | INFO | "gpu" : " 17 GB", | 20260506 14:16:11.492 | INFO | "huge" : "", | 20260506 14:16:11.492 | INFO | "huge_shared" : " 27 GB", | 20260506 14:16:11.492 | INFO | "malloc" : " 14 GB", | 20260506 14:16:11.492 | INFO | "pinned" : " 29 GB" | 20260506 14:16:11.492 | INFO | } | 20260506 14:16:11.492 | INFO | Perf report for the last 5.13852 seconds: | 20260506 14:16:11.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:16:11.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:11.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:11.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:11.492 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:16:11.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:11.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:11.493 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:16:11.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:11.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:11.493 | INFO | Uploaded 36GiB to GPU in 1.99782s (18.0196GiB/s) | 20260506 14:16:11.493 | INFO | Downloaded 34GiB to GPU in 1.93506s (17.5705GiB/s) | 20260506 14:16:16.632 | INFO | Analyzing chunk frames = [2000384,2000896) | 20260506 14:16:16.632 | INFO | MemInfo: { | 20260506 14:16:16.632 | INFO | "gpu" : " 17 GB", | 20260506 14:16:16.632 | INFO | "huge" : "", | 20260506 14:16:16.632 | INFO | "huge_shared" : " 27 GB", | 20260506 14:16:16.632 | INFO | "malloc" : " 14 GB", | 20260506 14:16:16.632 | INFO | "pinned" : " 29 GB" | 20260506 14:16:16.632 | INFO | } | 20260506 14:16:16.632 | INFO | Perf report for the last 5.1401 seconds: | 20260506 14:16:16.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:16:16.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:16.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:16.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:16.632 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 14:16:16.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:16.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:16.632 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:16:16.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:16.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:16.632 | INFO | Uploaded 36GiB to GPU in 2.01013s (17.9093GiB/s) | 20260506 14:16:16.632 | INFO | Downloaded 34GiB to GPU in 1.92782s (17.6365GiB/s) | 20260506 14:16:21.771 | INFO | Analyzing chunk frames = [2000896,2001408) | 20260506 14:16:21.771 | INFO | MemInfo: { | 20260506 14:16:21.771 | INFO | "gpu" : " 17 GB", | 20260506 14:16:21.771 | INFO | "huge" : "", | 20260506 14:16:21.771 | INFO | "huge_shared" : " 27 GB", | 20260506 14:16:21.771 | INFO | "malloc" : " 14 GB", | 20260506 14:16:21.771 | INFO | "pinned" : " 29 GB" | 20260506 14:16:21.771 | INFO | } | 20260506 14:16:21.771 | INFO | Perf report for the last 5.1392 seconds: | 20260506 14:16:21.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:16:21.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:21.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:21.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:21.772 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:16:21.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:21.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:21.772 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 14:16:21.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:21.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:21.772 | INFO | Uploaded 36GiB to GPU in 2.00529s (17.9525GiB/s) | 20260506 14:16:21.772 | INFO | Downloaded 34GiB to GPU in 1.93303s (17.589GiB/s) | 20260506 14:16:26.911 | INFO | Analyzing chunk frames = [2001408,2001920) | 20260506 14:16:26.911 | INFO | MemInfo: { | 20260506 14:16:26.911 | INFO | "gpu" : " 17 GB", | 20260506 14:16:26.911 | INFO | "huge" : "", | 20260506 14:16:26.911 | INFO | "huge_shared" : " 27 GB", | 20260506 14:16:26.911 | INFO | "malloc" : " 14 GB", | 20260506 14:16:26.911 | INFO | "pinned" : " 29 GB" | 20260506 14:16:26.911 | INFO | } | 20260506 14:16:26.911 | INFO | Perf report for the last 5.13935 seconds: | 20260506 14:16:26.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:16:26.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:26.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:26.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:26.911 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:16:26.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:26.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:26.911 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260506 14:16:26.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:26.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:26.911 | INFO | Uploaded 36GiB to GPU in 2.029s (17.7427GiB/s) | 20260506 14:16:26.911 | INFO | Downloaded 34GiB to GPU in 1.92364s (17.6748GiB/s) | 20260506 14:16:32.050 | INFO | Analyzing chunk frames = [2001920,2002432) | 20260506 14:16:32.050 | INFO | MemInfo: { | 20260506 14:16:32.050 | INFO | "gpu" : " 17 GB", | 20260506 14:16:32.050 | INFO | "huge" : "", | 20260506 14:16:32.050 | INFO | "huge_shared" : " 27 GB", | 20260506 14:16:32.050 | INFO | "malloc" : " 14 GB", | 20260506 14:16:32.050 | INFO | "pinned" : " 29 GB" | 20260506 14:16:32.050 | INFO | } | 20260506 14:16:32.050 | INFO | Perf report for the last 5.13932 seconds: | 20260506 14:16:32.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:16:32.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:32.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:32.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:32.050 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 14:16:32.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:32.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:32.050 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 14:16:32.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:32.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:32.050 | INFO | Uploaded 36GiB to GPU in 1.96371s (18.3326GiB/s) | 20260506 14:16:32.050 | INFO | Downloaded 34GiB to GPU in 1.99473s (17.0449GiB/s) | 20260506 14:16:37.189 | INFO | Analyzing chunk frames = [2002432,2002944) | 20260506 14:16:37.189 | INFO | MemInfo: { | 20260506 14:16:37.189 | INFO | "gpu" : " 17 GB", | 20260506 14:16:37.189 | INFO | "huge" : "", | 20260506 14:16:37.189 | INFO | "huge_shared" : " 27 GB", | 20260506 14:16:37.189 | INFO | "malloc" : " 14 GB", | 20260506 14:16:37.189 | INFO | "pinned" : " 29 GB" | 20260506 14:16:37.189 | INFO | } | 20260506 14:16:37.189 | INFO | Perf report for the last 5.13878 seconds: | 20260506 14:16:37.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:16:37.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:37.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:37.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:37.189 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 14:16:37.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:37.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:37.189 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:16:37.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:37.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:37.189 | INFO | Uploaded 36GiB to GPU in 2.01356s (17.8788GiB/s) | 20260506 14:16:37.189 | INFO | Downloaded 34GiB to GPU in 1.98989s (17.0864GiB/s) | 20260506 14:16:37.318 | INFO | Starting a round of Baz IO | 20260506 14:16:40.475 | INFO | Populating RTMetrics report | 20260506 14:16:40.475 | INFO | RTMetrics report populated | 20260506 14:16:40.475 | INFO | Opening tmp RTMetrics json file | 20260506 14:16:40.476 | INFO | Serializing RTMetrics to json | 20260506 14:16:40.478 | INFO | Serialization done, swapping tmp file now... | 20260506 14:16:40.478 | INFO | RTMetrics done | 20260506 14:16:40.930 | INFO | Finished a round of Baz IO: Wrote 2.118 GiB in 3.612 seconds | 20260506 14:16:40.930 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.52% / 0.01% / 61.46% | 20260506 14:16:40.930 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.67121% / 2.38419e-05% / 8.34465e-05% | 20260506 14:16:40.930 | INFO | Baz IO overall: | 20260506 14:16:40.930 | INFO | BytesWritten : 1031004026420 | 20260506 14:16:40.930 | INFO | NumEvents : 498864142723 | 20260506 14:16:42.328 | INFO | Analyzing chunk frames = [2002944,2003456) | 20260506 14:16:42.328 | INFO | MemInfo: { | 20260506 14:16:42.328 | INFO | "gpu" : " 17 GB", | 20260506 14:16:42.328 | INFO | "huge" : "", | 20260506 14:16:42.328 | INFO | "huge_shared" : " 27 GB", | 20260506 14:16:42.328 | INFO | "malloc" : " 14 GB", | 20260506 14:16:42.328 | INFO | "pinned" : " 29 GB" | 20260506 14:16:42.328 | INFO | } | 20260506 14:16:42.328 | INFO | Perf report for the last 5.13956 seconds: | 20260506 14:16:42.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:16:42.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:42.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:42.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:42.328 | INFO | ANALYSIS : 63.4%, 2.9, 95.8ms, 0 | 20260506 14:16:42.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:42.329 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 14:16:42.329 | INFO | PRE_HQ : 61.1%, 1.9, 59.5ms, 0 | 20260506 14:16:42.329 | INFO | BAZWRITER : 69.6%, 4.8, 172.0ms, 0 | 20260506 14:16:42.329 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260506 14:16:42.329 | INFO | Uploaded 36GiB to GPU in 2.10053s (17.1385GiB/s) | 20260506 14:16:42.329 | INFO | Downloaded 38GiB to GPU in 2.24145s (16.9533GiB/s) | 20260506 14:16:47.468 | INFO | Analyzing chunk frames = [2003456,2003968) | 20260506 14:16:47.468 | INFO | MemInfo: { | 20260506 14:16:47.468 | INFO | "gpu" : " 17 GB", | 20260506 14:16:47.468 | INFO | "huge" : "", | 20260506 14:16:47.468 | INFO | "huge_shared" : " 27 GB", | 20260506 14:16:47.468 | INFO | "malloc" : " 14 GB", | 20260506 14:16:47.468 | INFO | "pinned" : " 29 GB" | 20260506 14:16:47.468 | INFO | } | 20260506 14:16:47.468 | INFO | Perf report for the last 5.13931 seconds: | 20260506 14:16:47.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:16:47.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:47.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:47.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:47.468 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 14:16:47.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:47.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:47.468 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:16:47.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:47.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:47.468 | INFO | Uploaded 36GiB to GPU in 2.02796s (17.7518GiB/s) | 20260506 14:16:47.468 | INFO | Downloaded 34GiB to GPU in 1.94739s (17.4593GiB/s) | 20260506 14:16:52.608 | INFO | Analyzing chunk frames = [2003968,2004480) | 20260506 14:16:52.608 | INFO | MemInfo: { | 20260506 14:16:52.608 | INFO | "gpu" : " 17 GB", | 20260506 14:16:52.608 | INFO | "huge" : "", | 20260506 14:16:52.608 | INFO | "huge_shared" : " 27 GB", | 20260506 14:16:52.608 | INFO | "malloc" : " 14 GB", | 20260506 14:16:52.608 | INFO | "pinned" : " 29 GB" | 20260506 14:16:52.608 | INFO | } | 20260506 14:16:52.608 | INFO | Perf report for the last 5.14072 seconds: | 20260506 14:16:52.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:16:52.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:52.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:52.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:52.609 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 14:16:52.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:52.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:52.609 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:16:52.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:52.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:52.609 | INFO | Uploaded 36GiB to GPU in 1.97559s (18.2224GiB/s) | 20260506 14:16:52.609 | INFO | Downloaded 34GiB to GPU in 1.93615s (17.5606GiB/s) | 20260506 14:16:57.746 | INFO | Analyzing chunk frames = [2004480,2004992) | 20260506 14:16:57.746 | INFO | MemInfo: { | 20260506 14:16:57.746 | INFO | "gpu" : " 17 GB", | 20260506 14:16:57.746 | INFO | "huge" : "", | 20260506 14:16:57.746 | INFO | "huge_shared" : " 27 GB", | 20260506 14:16:57.746 | INFO | "malloc" : " 14 GB", | 20260506 14:16:57.746 | INFO | "pinned" : " 29 GB" | 20260506 14:16:57.746 | INFO | } | 20260506 14:16:57.747 | INFO | Perf report for the last 5.13803 seconds: | 20260506 14:16:57.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:16:57.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:57.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:57.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:57.747 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:16:57.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:57.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:57.747 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:16:57.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:57.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:16:57.747 | INFO | Uploaded 36GiB to GPU in 1.97915s (18.1896GiB/s) | 20260506 14:16:57.747 | INFO | Downloaded 34GiB to GPU in 1.92046s (17.7041GiB/s) | 20260506 14:17:02.885 | INFO | Analyzing chunk frames = [2004992,2005504) | 20260506 14:17:02.885 | INFO | MemInfo: { | 20260506 14:17:02.885 | INFO | "gpu" : " 17 GB", | 20260506 14:17:02.885 | INFO | "huge" : "", | 20260506 14:17:02.885 | INFO | "huge_shared" : " 27 GB", | 20260506 14:17:02.885 | INFO | "malloc" : " 14 GB", | 20260506 14:17:02.885 | INFO | "pinned" : " 29 GB" | 20260506 14:17:02.885 | INFO | } | 20260506 14:17:02.885 | INFO | Perf report for the last 5.1387 seconds: | 20260506 14:17:02.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:17:02.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:02.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:02.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:02.885 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 14:17:02.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:02.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:02.885 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:17:02.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:02.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:02.885 | INFO | Uploaded 36GiB to GPU in 2.01867s (17.8335GiB/s) | 20260506 14:17:02.886 | INFO | Downloaded 34GiB to GPU in 1.93152s (17.6027GiB/s) | 20260506 14:17:08.025 | INFO | Analyzing chunk frames = [2005504,2006016) | 20260506 14:17:08.025 | INFO | MemInfo: { | 20260506 14:17:08.025 | INFO | "gpu" : " 17 GB", | 20260506 14:17:08.025 | INFO | "huge" : "", | 20260506 14:17:08.025 | INFO | "huge_shared" : " 27 GB", | 20260506 14:17:08.025 | INFO | "malloc" : " 14 GB", | 20260506 14:17:08.025 | INFO | "pinned" : " 29 GB" | 20260506 14:17:08.025 | INFO | } | 20260506 14:17:08.025 | INFO | Perf report for the last 5.13948 seconds: | 20260506 14:17:08.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:17:08.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:08.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:08.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:08.025 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:17:08.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:08.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:08.025 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260506 14:17:08.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:08.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:08.025 | INFO | Uploaded 36GiB to GPU in 2.04407s (17.6119GiB/s) | 20260506 14:17:08.025 | INFO | Downloaded 34GiB to GPU in 1.93951s (17.5302GiB/s) | 20260506 14:17:13.164 | INFO | Analyzing chunk frames = [2006016,2006528) | 20260506 14:17:13.164 | INFO | MemInfo: { | 20260506 14:17:13.164 | INFO | "gpu" : " 17 GB", | 20260506 14:17:13.165 | INFO | "huge" : "", | 20260506 14:17:13.165 | INFO | "huge_shared" : " 27 GB", | 20260506 14:17:13.165 | INFO | "malloc" : " 14 GB", | 20260506 14:17:13.165 | INFO | "pinned" : " 29 GB" | 20260506 14:17:13.165 | INFO | } | 20260506 14:17:13.165 | INFO | Perf report for the last 5.13988 seconds: | 20260506 14:17:13.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:17:13.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:13.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:13.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:13.165 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:17:13.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:13.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:13.165 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:17:13.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:13.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:13.165 | INFO | Uploaded 36GiB to GPU in 2.00347s (17.9688GiB/s) | 20260506 14:17:13.165 | INFO | Downloaded 34GiB to GPU in 1.99883s (17.01GiB/s) | 20260506 14:17:18.304 | INFO | Analyzing chunk frames = [2006528,2007040) | 20260506 14:17:18.304 | INFO | MemInfo: { | 20260506 14:17:18.304 | INFO | "gpu" : " 17 GB", | 20260506 14:17:18.304 | INFO | "huge" : "", | 20260506 14:17:18.304 | INFO | "huge_shared" : " 27 GB", | 20260506 14:17:18.304 | INFO | "malloc" : " 14 GB", | 20260506 14:17:18.304 | INFO | "pinned" : " 29 GB" | 20260506 14:17:18.304 | INFO | } | 20260506 14:17:18.304 | INFO | Perf report for the last 5.13975 seconds: | 20260506 14:17:18.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:17:18.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:18.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:18.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:18.304 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:17:18.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:18.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:18.304 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:17:18.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:18.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:18.305 | INFO | Uploaded 36GiB to GPU in 2.02663s (17.7635GiB/s) | 20260506 14:17:18.305 | INFO | Downloaded 34GiB to GPU in 1.97445s (17.22GiB/s) | 20260506 14:17:18.305 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:17:18.305 | INFO | Total mem usage: 17.4 GiB | 20260506 14:17:18.305 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:17:18.305 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -14.6%) | 20260506 14:17:18.305 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:17:18.305 | INFO | High Water Breakdown: | 20260506 14:17:18.305 | INFO | 60.7% : 9.23 GiB ( 84.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:17:18.305 | INFO | 39.2% : 5.95 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:17:18.305 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:17:18.305 | INFO | ------------------------------------------ | 20260506 14:17:18.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:17:18.305 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:17:18.305 | INFO | Total mem usage: 26.5 GiB | 20260506 14:17:18.305 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:17:18.305 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:17:18.305 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:17:18.305 | INFO | High Water Breakdown: | 20260506 14:17:18.305 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:17:18.305 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:17:18.305 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:17:18.305 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:17:18.305 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:17:18.305 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:17:18.305 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:17:18.305 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:17:18.305 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:17:18.305 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:17:18.305 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:17:18.305 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:17:18.305 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:17:18.305 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:17:18.305 | INFO | ------------------------------------------ | 20260506 14:17:18.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:17:18.305 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:17:18.305 | INFO | Total mem usage: 25.6 GiB | 20260506 14:17:18.305 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:17:18.305 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:17:18.305 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:17:18.305 | INFO | High Water Breakdown: | 20260506 14:17:18.305 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:17:18.305 | INFO | ------------------------------------------ | 20260506 14:17:18.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:17:18.305 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:17:18.305 | INFO | Total mem usage: 15.7 GiB | 20260506 14:17:18.305 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:17:18.305 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:17:18.305 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 14:17:18.305 | INFO | High Water Breakdown: | 20260506 14:17:18.305 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:17:18.305 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:17:18.305 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:17:18.305 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:17:18.305 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:17:18.305 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:17:18.305 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:17:18.305 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:17:18.305 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:17:18.305 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:17:18.305 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:17:18.305 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:17:18.305 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:17:18.305 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:17:18.305 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:17:18.305 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:17:18.305 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:17:18.305 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:17:18.305 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:17:18.305 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:17:18.306 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:17:18.306 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:17:18.306 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:17:18.306 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:17:18.306 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:17:18.306 | INFO | ------------------------------------------ | 20260506 14:17:18.306 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:17:18.306 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:17:18.306 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:17:18.306 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:17:18.306 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.57e+01, 1.34e+01 / 1.96e+00 | 20260506 14:17:18.306 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.01e-01, 1.04e-02 / 6.56e-03 | 20260506 14:17:18.306 | INFO | PerfReport: 11.57%, Baseline -- 3.65e+00 / 4.46e+00, 5.73e-01 / 3.30e-01 | 20260506 14:17:18.306 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.47e+00, 1.90e-01 / 1.43e-01 | 20260506 14:17:18.306 | INFO | PerfReport: 15.82%, DME -- 3.99e+01 / 4.43e+01, 3.94e+00 / 2.16e+00 | 20260506 14:17:18.306 | INFO | PerfReport: 56.68%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.46e+00 / 1.81e+00 | 20260506 14:17:18.306 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.36e+00 / 2.98e+00, 4.36e-01 / 2.65e-01 | 20260506 14:17:18.306 | INFO | PerfReport: 3.63%, Metrics -- 1.15e+00 / 1.54e+00, 2.99e-01 / 1.91e-01 | 20260506 14:17:18.306 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.00e-01 / 4.53e-01, 6.32e-02 / 2.51e-02 | 20260506 14:17:18.306 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:17:18.306 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:17:18.306 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:17:18.306 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 14:17:18.306 | INFO | PerfReport: 21.47%, Upload -- 2.10e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 14:17:18.306 | INFO | PerfReport: 21.15%, Download -- 2.07e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 14:17:18.306 | INFO | PerfReport: 32.26%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 14:17:18.306 | INFO | PerfReport: 1.53%, LockUpload -- 1.50e+00 / 0.00e+00, 6.29e+00 / 0.00e+00 | 20260506 14:17:18.306 | INFO | PerfReport: 1.17%, LockDownload -- 1.14e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 14:17:18.306 | INFO | PerfReport: 22.40%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:17:18.306 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260506 14:17:18.422 | INFO | Starting a round of Baz IO | 20260506 14:17:21.579 | INFO | Populating RTMetrics report | 20260506 14:17:21.579 | INFO | RTMetrics report populated | 20260506 14:17:21.579 | INFO | Opening tmp RTMetrics json file | 20260506 14:17:21.604 | INFO | Serializing RTMetrics to json | 20260506 14:17:21.607 | INFO | Serialization done, swapping tmp file now... | 20260506 14:17:21.610 | INFO | RTMetrics done | 20260506 14:17:21.924 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.501 seconds | 20260506 14:17:21.924 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.51% / 0.01% / 61.48% | 20260506 14:17:21.924 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.65437% / 1.98682e-05% / 6.35783e-05% | 20260506 14:17:21.924 | INFO | Baz IO overall: | 20260506 14:17:21.924 | INFO | BytesWritten : 1033274226511 | 20260506 14:17:21.924 | INFO | NumEvents : 499964025640 | 20260506 14:17:23.444 | INFO | Analyzing chunk frames = [2007040,2007552) | 20260506 14:17:23.444 | INFO | MemInfo: { | 20260506 14:17:23.444 | INFO | "gpu" : " 17 GB", | 20260506 14:17:23.444 | INFO | "huge" : "", | 20260506 14:17:23.444 | INFO | "huge_shared" : " 27 GB", | 20260506 14:17:23.444 | INFO | "malloc" : " 14 GB", | 20260506 14:17:23.444 | INFO | "pinned" : " 29 GB" | 20260506 14:17:23.444 | INFO | } | 20260506 14:17:23.444 | INFO | Perf report for the last 5.13958 seconds: | 20260506 14:17:23.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:17:23.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:23.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:23.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:23.444 | INFO | ANALYSIS : 63.1%, 2.9, 95.6ms, 0 | 20260506 14:17:23.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:23.444 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 14:17:23.444 | INFO | PRE_HQ : 61.0%, 1.9, 59.4ms, 0 | 20260506 14:17:23.444 | INFO | BAZWRITER : 67.6%, 4.8, 167.4ms, 0 | 20260506 14:17:23.444 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 14:17:23.444 | INFO | Uploaded 36GiB to GPU in 2.07105s (17.3825GiB/s) | 20260506 14:17:23.444 | INFO | Downloaded 38GiB to GPU in 2.23455s (17.0056GiB/s) | 20260506 14:17:28.582 | INFO | Analyzing chunk frames = [2007552,2008064) | 20260506 14:17:28.582 | INFO | MemInfo: { | 20260506 14:17:28.582 | INFO | "gpu" : " 17 GB", | 20260506 14:17:28.582 | INFO | "huge" : "", | 20260506 14:17:28.582 | INFO | "huge_shared" : " 27 GB", | 20260506 14:17:28.582 | INFO | "malloc" : " 14 GB", | 20260506 14:17:28.582 | INFO | "pinned" : " 29 GB" | 20260506 14:17:28.582 | INFO | } | 20260506 14:17:28.582 | INFO | Perf report for the last 5.13782 seconds: | 20260506 14:17:28.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:17:28.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:28.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:28.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:28.582 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:17:28.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:28.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:28.582 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:17:28.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:28.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:28.582 | INFO | Uploaded 36GiB to GPU in 2.00213s (17.9808GiB/s) | 20260506 14:17:28.582 | INFO | Downloaded 34GiB to GPU in 1.93724s (17.5507GiB/s) | 20260506 14:17:33.722 | INFO | Analyzing chunk frames = [2008064,2008576) | 20260506 14:17:33.722 | INFO | MemInfo: { | 20260506 14:17:33.722 | INFO | "gpu" : " 17 GB", | 20260506 14:17:33.722 | INFO | "huge" : "", | 20260506 14:17:33.722 | INFO | "huge_shared" : " 27 GB", | 20260506 14:17:33.722 | INFO | "malloc" : " 14 GB", | 20260506 14:17:33.722 | INFO | "pinned" : " 29 GB" | 20260506 14:17:33.722 | INFO | } | 20260506 14:17:33.722 | INFO | Perf report for the last 5.14013 seconds: | 20260506 14:17:33.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:17:33.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:33.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:33.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:33.722 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:17:33.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:33.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:33.722 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:17:33.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:33.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:33.722 | INFO | Uploaded 36GiB to GPU in 2.00504s (17.9548GiB/s) | 20260506 14:17:33.722 | INFO | Downloaded 34GiB to GPU in 1.93283s (17.5908GiB/s) | 20260506 14:17:38.861 | INFO | Analyzing chunk frames = [2008576,2009088) | 20260506 14:17:38.861 | INFO | MemInfo: { | 20260506 14:17:38.861 | INFO | "gpu" : " 17 GB", | 20260506 14:17:38.861 | INFO | "huge" : "", | 20260506 14:17:38.861 | INFO | "huge_shared" : " 27 GB", | 20260506 14:17:38.861 | INFO | "malloc" : " 14 GB", | 20260506 14:17:38.861 | INFO | "pinned" : " 29 GB" | 20260506 14:17:38.861 | INFO | } | 20260506 14:17:38.861 | INFO | Perf report for the last 5.13928 seconds: | 20260506 14:17:38.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:17:38.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:38.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:38.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:38.861 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260506 14:17:38.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:38.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:38.861 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:17:38.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:38.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:38.861 | INFO | Uploaded 36GiB to GPU in 1.97647s (18.2143GiB/s) | 20260506 14:17:38.862 | INFO | Downloaded 34GiB to GPU in 1.91333s (17.77GiB/s) | 20260506 14:17:44.001 | INFO | Analyzing chunk frames = [2009088,2009600) | 20260506 14:17:44.001 | INFO | MemInfo: { | 20260506 14:17:44.001 | INFO | "gpu" : " 17 GB", | 20260506 14:17:44.001 | INFO | "huge" : "", | 20260506 14:17:44.001 | INFO | "huge_shared" : " 27 GB", | 20260506 14:17:44.001 | INFO | "malloc" : " 14 GB", | 20260506 14:17:44.001 | INFO | "pinned" : " 29 GB" | 20260506 14:17:44.001 | INFO | } | 20260506 14:17:44.001 | INFO | Perf report for the last 5.14005 seconds: | 20260506 14:17:44.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:17:44.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:44.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:44.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:44.001 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 14:17:44.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:44.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:44.001 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 14:17:44.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:44.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:44.002 | INFO | Uploaded 36GiB to GPU in 2.03488s (17.6915GiB/s) | 20260506 14:17:44.002 | INFO | Downloaded 34GiB to GPU in 1.92527s (17.6599GiB/s) | 20260506 14:17:49.139 | INFO | Analyzing chunk frames = [2009600,2010112) | 20260506 14:17:49.139 | INFO | MemInfo: { | 20260506 14:17:49.139 | INFO | "gpu" : " 17 GB", | 20260506 14:17:49.139 | INFO | "huge" : "", | 20260506 14:17:49.139 | INFO | "huge_shared" : " 27 GB", | 20260506 14:17:49.139 | INFO | "malloc" : " 14 GB", | 20260506 14:17:49.139 | INFO | "pinned" : " 29 GB" | 20260506 14:17:49.139 | INFO | } | 20260506 14:17:49.139 | INFO | Perf report for the last 5.13813 seconds: | 20260506 14:17:49.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:17:49.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:49.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:49.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:49.140 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:17:49.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:49.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:49.140 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:17:49.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:49.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:49.140 | INFO | Uploaded 36GiB to GPU in 2.04351s (17.6167GiB/s) | 20260506 14:17:49.140 | INFO | Downloaded 34GiB to GPU in 1.91053s (17.7962GiB/s) | 20260506 14:17:54.280 | INFO | Analyzing chunk frames = [2010112,2010624) | 20260506 14:17:54.280 | INFO | MemInfo: { | 20260506 14:17:54.280 | INFO | "gpu" : " 17 GB", | 20260506 14:17:54.280 | INFO | "huge" : "", | 20260506 14:17:54.280 | INFO | "huge_shared" : " 27 GB", | 20260506 14:17:54.280 | INFO | "malloc" : " 14 GB", | 20260506 14:17:54.280 | INFO | "pinned" : " 29 GB" | 20260506 14:17:54.280 | INFO | } | 20260506 14:17:54.280 | INFO | Perf report for the last 5.14041 seconds: | 20260506 14:17:54.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:17:54.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:54.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:54.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:54.280 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:17:54.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:54.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:54.280 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 14:17:54.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:54.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:54.280 | INFO | Uploaded 36GiB to GPU in 2.00994s (17.911GiB/s) | 20260506 14:17:54.280 | INFO | Downloaded 34GiB to GPU in 2.00039s (16.9967GiB/s) | 20260506 14:17:59.418 | INFO | Analyzing chunk frames = [2010624,2011136) | 20260506 14:17:59.418 | INFO | MemInfo: { | 20260506 14:17:59.418 | INFO | "gpu" : " 17 GB", | 20260506 14:17:59.418 | INFO | "huge" : "", | 20260506 14:17:59.418 | INFO | "huge_shared" : " 27 GB", | 20260506 14:17:59.418 | INFO | "malloc" : " 14 GB", | 20260506 14:17:59.418 | INFO | "pinned" : " 29 GB" | 20260506 14:17:59.418 | INFO | } | 20260506 14:17:59.418 | INFO | Perf report for the last 5.13859 seconds: | 20260506 14:17:59.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:17:59.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:59.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:59.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:59.419 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 14:17:59.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:59.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:59.419 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:17:59.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:59.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:17:59.419 | INFO | Uploaded 36GiB to GPU in 2.04975s (17.5632GiB/s) | 20260506 14:17:59.419 | INFO | Downloaded 34GiB to GPU in 1.96905s (17.2672GiB/s) | 20260506 14:17:59.548 | INFO | Starting a round of Baz IO | 20260506 14:18:02.719 | INFO | Populating RTMetrics report | 20260506 14:18:02.720 | INFO | RTMetrics report populated | 20260506 14:18:02.720 | INFO | Opening tmp RTMetrics json file | 20260506 14:18:02.777 | INFO | Serializing RTMetrics to json | 20260506 14:18:02.918 | INFO | Serialization done, swapping tmp file now... | 20260506 14:18:02.949 | INFO | RTMetrics done | 20260506 14:18:03.505 | INFO | Finished a round of Baz IO: Wrote 2.117 GiB in 3.957 seconds | 20260506 14:18:03.505 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.49% / 0.01% / 61.49% | 20260506 14:18:03.505 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.65532% / 2.38419e-05% / 6.75519e-05% | 20260506 14:18:03.505 | INFO | Baz IO overall: | 20260506 14:18:03.505 | INFO | BytesWritten : 1035547812977 | 20260506 14:18:03.505 | INFO | NumEvents : 501066513885 | 20260506 14:18:04.557 | INFO | Analyzing chunk frames = [2011136,2011648) | 20260506 14:18:04.557 | INFO | MemInfo: { | 20260506 14:18:04.557 | INFO | "gpu" : " 17 GB", | 20260506 14:18:04.557 | INFO | "huge" : "", | 20260506 14:18:04.557 | INFO | "huge_shared" : " 27 GB", | 20260506 14:18:04.557 | INFO | "malloc" : " 14 GB", | 20260506 14:18:04.557 | INFO | "pinned" : " 29 GB" | 20260506 14:18:04.558 | INFO | } | 20260506 14:18:04.558 | INFO | Perf report for the last 5.13921 seconds: | 20260506 14:18:04.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:18:04.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:04.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:04.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:04.558 | INFO | ANALYSIS : 63.5%, 2.9, 95.8ms, 0 | 20260506 14:18:04.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:04.558 | INFO | RT_METRICS : 48.5%, 1.0, 25.5ms, 0 | 20260506 14:18:04.558 | INFO | PRE_HQ : 61.0%, 1.9, 60.2ms, 0 | 20260506 14:18:04.558 | INFO | BAZWRITER : 76.4%, 4.8, 188.0ms, 0 | 20260506 14:18:04.558 | INFO | BAZ_SAMPLING : 3.0%, 1.0, 1.5ms, 0 | 20260506 14:18:04.558 | INFO | Uploaded 36GiB to GPU in 2.10455s (17.1058GiB/s) | 20260506 14:18:04.558 | INFO | Downloaded 38GiB to GPU in 2.22601s (17.0709GiB/s) | 20260506 14:18:09.698 | INFO | Analyzing chunk frames = [2011648,2012160) | 20260506 14:18:09.698 | INFO | MemInfo: { | 20260506 14:18:09.698 | INFO | "gpu" : " 17 GB", | 20260506 14:18:09.698 | INFO | "huge" : "", | 20260506 14:18:09.698 | INFO | "huge_shared" : " 27 GB", | 20260506 14:18:09.698 | INFO | "malloc" : " 14 GB", | 20260506 14:18:09.698 | INFO | "pinned" : " 29 GB" | 20260506 14:18:09.698 | INFO | } | 20260506 14:18:09.698 | INFO | Perf report for the last 5.14013 seconds: | 20260506 14:18:09.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:18:09.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:09.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:09.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:09.698 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 14:18:09.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:09.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:09.698 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:18:09.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:09.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:09.698 | INFO | Uploaded 36GiB to GPU in 2.02268s (17.7982GiB/s) | 20260506 14:18:09.698 | INFO | Downloaded 34GiB to GPU in 1.94558s (17.4755GiB/s) | 20260506 14:18:14.837 | INFO | Analyzing chunk frames = [2012160,2012672) | 20260506 14:18:14.837 | INFO | MemInfo: { | 20260506 14:18:14.837 | INFO | "gpu" : " 17 GB", | 20260506 14:18:14.837 | INFO | "huge" : "", | 20260506 14:18:14.837 | INFO | "huge_shared" : " 27 GB", | 20260506 14:18:14.837 | INFO | "malloc" : " 14 GB", | 20260506 14:18:14.837 | INFO | "pinned" : " 29 GB" | 20260506 14:18:14.837 | INFO | } | 20260506 14:18:14.837 | INFO | Perf report for the last 5.13895 seconds: | 20260506 14:18:14.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:18:14.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:14.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:14.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:14.837 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 14:18:14.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:14.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:14.837 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:18:14.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:14.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:14.837 | INFO | Uploaded 36GiB to GPU in 2.01007s (17.9098GiB/s) | 20260506 14:18:14.837 | INFO | Downloaded 34GiB to GPU in 1.93254s (17.5934GiB/s) | 20260506 14:18:19.975 | INFO | Analyzing chunk frames = [2012672,2013184) | 20260506 14:18:19.975 | INFO | MemInfo: { | 20260506 14:18:19.975 | INFO | "gpu" : " 17 GB", | 20260506 14:18:19.975 | INFO | "huge" : "", | 20260506 14:18:19.975 | INFO | "huge_shared" : " 27 GB", | 20260506 14:18:19.975 | INFO | "malloc" : " 14 GB", | 20260506 14:18:19.975 | INFO | "pinned" : " 29 GB" | 20260506 14:18:19.975 | INFO | } | 20260506 14:18:19.975 | INFO | Perf report for the last 5.13855 seconds: | 20260506 14:18:19.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:18:19.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:19.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:19.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:19.975 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 14:18:19.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:19.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:19.975 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:18:19.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:19.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:19.976 | INFO | Uploaded 36GiB to GPU in 1.99398s (18.0543GiB/s) | 20260506 14:18:19.976 | INFO | Downloaded 34GiB to GPU in 1.92721s (17.6421GiB/s) | 20260506 14:18:25.115 | INFO | Analyzing chunk frames = [2013184,2013696) | 20260506 14:18:25.115 | INFO | MemInfo: { | 20260506 14:18:25.115 | INFO | "gpu" : " 17 GB", | 20260506 14:18:25.115 | INFO | "huge" : "", | 20260506 14:18:25.115 | INFO | "huge_shared" : " 27 GB", | 20260506 14:18:25.115 | INFO | "malloc" : " 14 GB", | 20260506 14:18:25.115 | INFO | "pinned" : " 29 GB" | 20260506 14:18:25.115 | INFO | } | 20260506 14:18:25.115 | INFO | Perf report for the last 5.13998 seconds: | 20260506 14:18:25.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:18:25.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:25.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:25.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:25.116 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 14:18:25.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:25.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:25.116 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 14:18:25.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:25.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:25.116 | INFO | Uploaded 36GiB to GPU in 2.04193s (17.6304GiB/s) | 20260506 14:18:25.116 | INFO | Downloaded 34GiB to GPU in 1.93868s (17.5377GiB/s) | 20260506 14:18:30.254 | INFO | Analyzing chunk frames = [2013696,2014208) | 20260506 14:18:30.254 | INFO | MemInfo: { | 20260506 14:18:30.254 | INFO | "gpu" : " 17 GB", | 20260506 14:18:30.254 | INFO | "huge" : "", | 20260506 14:18:30.254 | INFO | "huge_shared" : " 27 GB", | 20260506 14:18:30.254 | INFO | "malloc" : " 14 GB", | 20260506 14:18:30.254 | INFO | "pinned" : " 29 GB" | 20260506 14:18:30.254 | INFO | } | 20260506 14:18:30.254 | INFO | Perf report for the last 5.139 seconds: | 20260506 14:18:30.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:18:30.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:30.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:30.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:30.255 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 14:18:30.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:30.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:30.255 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260506 14:18:30.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:30.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:30.255 | INFO | Uploaded 36GiB to GPU in 2.07491s (17.3502GiB/s) | 20260506 14:18:30.255 | INFO | Downloaded 34GiB to GPU in 1.91198s (17.7826GiB/s) | 20260506 14:18:35.395 | INFO | Analyzing chunk frames = [2014208,2014720) | 20260506 14:18:35.395 | INFO | MemInfo: { | 20260506 14:18:35.395 | INFO | "gpu" : " 17 GB", | 20260506 14:18:35.395 | INFO | "huge" : "", | 20260506 14:18:35.395 | INFO | "huge_shared" : " 27 GB", | 20260506 14:18:35.395 | INFO | "malloc" : " 14 GB", | 20260506 14:18:35.395 | INFO | "pinned" : " 29 GB" | 20260506 14:18:35.395 | INFO | } | 20260506 14:18:35.395 | INFO | Perf report for the last 5.14038 seconds: | 20260506 14:18:35.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:18:35.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:35.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:35.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:35.395 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:18:35.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:35.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:35.395 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 14:18:35.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:35.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:35.395 | INFO | Uploaded 36GiB to GPU in 1.98098s (18.1728GiB/s) | 20260506 14:18:35.395 | INFO | Downloaded 34GiB to GPU in 1.99255s (17.0636GiB/s) | 20260506 14:18:40.533 | INFO | Analyzing chunk frames = [2014720,2015232) | 20260506 14:18:40.533 | INFO | MemInfo: { | 20260506 14:18:40.533 | INFO | "gpu" : " 17 GB", | 20260506 14:18:40.533 | INFO | "huge" : "", | 20260506 14:18:40.533 | INFO | "huge_shared" : " 27 GB", | 20260506 14:18:40.533 | INFO | "malloc" : " 14 GB", | 20260506 14:18:40.533 | INFO | "pinned" : " 29 GB" | 20260506 14:18:40.533 | INFO | } | 20260506 14:18:40.533 | INFO | Perf report for the last 5.13866 seconds: | 20260506 14:18:40.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:18:40.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:40.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:40.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:40.534 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:18:40.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:40.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:40.534 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 14:18:40.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:40.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:40.534 | INFO | Uploaded 36GiB to GPU in 1.99067s (18.0844GiB/s) | 20260506 14:18:40.534 | INFO | Downloaded 34GiB to GPU in 1.98012s (17.1707GiB/s) | 20260506 14:18:40.534 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:18:40.534 | INFO | Total mem usage: 17.4 GiB | 20260506 14:18:40.534 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:18:40.534 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -8.43%) | 20260506 14:18:40.534 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:18:40.534 | INFO | High Water Breakdown: | 20260506 14:18:40.534 | INFO | 61.6% : 9.88 GiB ( 90.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:18:40.534 | INFO | 38.3% : 6.15 GiB ( 91.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:18:40.534 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:18:40.534 | INFO | ------------------------------------------ | 20260506 14:18:40.534 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:18:40.534 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:18:40.534 | INFO | Total mem usage: 26.5 GiB | 20260506 14:18:40.534 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:18:40.534 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:18:40.534 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:18:40.534 | INFO | High Water Breakdown: | 20260506 14:18:40.534 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:18:40.534 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:18:40.534 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:18:40.534 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:18:40.534 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:18:40.534 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:18:40.534 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:18:40.534 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:18:40.534 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:18:40.534 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:18:40.534 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:18:40.534 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:18:40.534 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:18:40.534 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:18:40.534 | INFO | ------------------------------------------ | 20260506 14:18:40.534 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:18:40.534 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:18:40.534 | INFO | Total mem usage: 25.6 GiB | 20260506 14:18:40.534 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:18:40.534 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:18:40.534 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:18:40.534 | INFO | High Water Breakdown: | 20260506 14:18:40.534 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:18:40.534 | INFO | ------------------------------------------ | 20260506 14:18:40.534 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:18:40.534 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:18:40.534 | INFO | Total mem usage: 15.7 GiB | 20260506 14:18:40.534 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:18:40.535 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:18:40.535 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 14:18:40.535 | INFO | High Water Breakdown: | 20260506 14:18:40.535 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:18:40.535 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:18:40.535 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:18:40.535 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:18:40.535 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:18:40.535 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:18:40.535 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:18:40.535 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:18:40.535 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:18:40.535 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:18:40.535 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:18:40.535 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:18:40.535 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:18:40.535 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:18:40.535 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:18:40.535 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:18:40.535 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:18:40.535 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:18:40.535 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:18:40.535 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:18:40.535 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:18:40.535 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:18:40.535 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:18:40.535 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:18:40.535 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:18:40.535 | INFO | ------------------------------------------ | 20260506 14:18:40.535 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:18:40.535 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:18:40.535 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:18:40.535 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:18:40.535 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.52e+01, 1.33e+01 / 1.71e+00 | 20260506 14:18:40.535 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.03e-01, 1.08e-02 / 8.07e-03 | 20260506 14:18:40.535 | INFO | PerfReport: 11.60%, Baseline -- 3.66e+00 / 4.41e+00, 5.82e-01 / 3.93e-01 | 20260506 14:18:40.535 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.47e+00, 1.90e-01 / 1.49e-01 | 20260506 14:18:40.535 | INFO | PerfReport: 15.75%, DME -- 3.98e+01 / 4.36e+01, 3.66e+00 / 1.88e+00 | 20260506 14:18:40.535 | INFO | PerfReport: 56.71%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.48e+00 / 2.00e+00 | 20260506 14:18:40.535 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.36e+00 / 2.95e+00, 4.15e-01 / 2.58e-01 | 20260506 14:18:40.535 | INFO | PerfReport: 3.64%, Metrics -- 1.15e+00 / 1.53e+00, 2.98e-01 / 2.10e-01 | 20260506 14:18:40.535 | INFO | PerfReport: 1.27%, PulseSerializing -- 4.00e-01 / 4.53e-01, 6.25e-02 / 2.56e-02 | 20260506 14:18:40.535 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:18:40.535 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:18:40.535 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:18:40.535 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 14:18:40.535 | INFO | PerfReport: 21.57%, Upload -- 2.11e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260506 14:18:40.535 | INFO | PerfReport: 21.08%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 14:18:40.535 | INFO | PerfReport: 32.25%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 14:18:40.535 | INFO | PerfReport: 1.62%, LockUpload -- 1.59e+00 / 0.00e+00, 6.47e+00 / 0.00e+00 | 20260506 14:18:40.535 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260506 14:18:40.535 | INFO | PerfReport: 22.37%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 14:18:40.535 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.50e-02 / 0.00e+00 | 20260506 14:18:40.642 | INFO | Starting a round of Baz IO | 20260506 14:18:43.796 | INFO | Populating RTMetrics report | 20260506 14:18:43.797 | INFO | RTMetrics report populated | 20260506 14:18:43.797 | INFO | Opening tmp RTMetrics json file | 20260506 14:18:43.884 | INFO | Serializing RTMetrics to json | 20260506 14:18:43.921 | INFO | Serialization done, swapping tmp file now... | 20260506 14:18:43.922 | INFO | RTMetrics done | 20260506 14:18:44.349 | INFO | Finished a round of Baz IO: Wrote 2.117 GiB in 3.706 seconds | 20260506 14:18:44.349 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.48% / 0.01% / 61.51% | 20260506 14:18:44.349 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.64737% / 2.78155e-05% / 3.97364e-05% | 20260506 14:18:44.349 | INFO | Baz IO overall: | 20260506 14:18:44.349 | INFO | BytesWritten : 1037821364803 | 20260506 14:18:44.349 | INFO | NumEvents : 502169106774 | 20260506 14:18:45.672 | INFO | Analyzing chunk frames = [2015232,2015744) | 20260506 14:18:45.672 | INFO | MemInfo: { | 20260506 14:18:45.672 | INFO | "gpu" : " 17 GB", | 20260506 14:18:45.672 | INFO | "huge" : "", | 20260506 14:18:45.672 | INFO | "huge_shared" : " 27 GB", | 20260506 14:18:45.672 | INFO | "malloc" : " 14 GB", | 20260506 14:18:45.672 | INFO | "pinned" : " 29 GB" | 20260506 14:18:45.672 | INFO | } | 20260506 14:18:45.672 | INFO | Perf report for the last 5.13874 seconds: | 20260506 14:18:45.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:18:45.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:45.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:45.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:45.672 | INFO | ANALYSIS : 62.9%, 2.9, 95.1ms, 0 | 20260506 14:18:45.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:45.672 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260506 14:18:45.672 | INFO | PRE_HQ : 61.0%, 1.9, 59.8ms, 0 | 20260506 14:18:45.672 | INFO | BAZWRITER : 71.5%, 4.8, 177.3ms, 0 | 20260506 14:18:45.672 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 14:18:45.672 | INFO | Uploaded 36GiB to GPU in 2.08497s (17.2664GiB/s) | 20260506 14:18:45.672 | INFO | Downloaded 38GiB to GPU in 2.23157s (17.0284GiB/s) | 20260506 14:18:50.812 | INFO | Analyzing chunk frames = [2015744,2016256) | 20260506 14:18:50.812 | INFO | MemInfo: { | 20260506 14:18:50.812 | INFO | "gpu" : " 17 GB", | 20260506 14:18:50.812 | INFO | "huge" : "", | 20260506 14:18:50.812 | INFO | "huge_shared" : " 27 GB", | 20260506 14:18:50.812 | INFO | "malloc" : " 14 GB", | 20260506 14:18:50.812 | INFO | "pinned" : " 29 GB" | 20260506 14:18:50.812 | INFO | } | 20260506 14:18:50.812 | INFO | Perf report for the last 5.13991 seconds: | 20260506 14:18:50.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:18:50.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:50.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:50.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:50.812 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:18:50.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:50.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:50.812 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:18:50.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:50.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:50.812 | INFO | Uploaded 36GiB to GPU in 2.03612s (17.6807GiB/s) | 20260506 14:18:50.812 | INFO | Downloaded 34GiB to GPU in 1.95089s (17.428GiB/s) | 20260506 14:18:55.952 | INFO | Analyzing chunk frames = [2016256,2016768) | 20260506 14:18:55.952 | INFO | MemInfo: { | 20260506 14:18:55.952 | INFO | "gpu" : " 17 GB", | 20260506 14:18:55.952 | INFO | "huge" : "", | 20260506 14:18:55.952 | INFO | "huge_shared" : " 27 GB", | 20260506 14:18:55.952 | INFO | "malloc" : " 14 GB", | 20260506 14:18:55.952 | INFO | "pinned" : " 29 GB" | 20260506 14:18:55.952 | INFO | } | 20260506 14:18:55.952 | INFO | Perf report for the last 5.13984 seconds: | 20260506 14:18:55.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:18:55.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:55.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:55.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:55.952 | INFO | ANALYSIS : 61.4%, 3.0, 95.1ms, 0 | 20260506 14:18:55.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:55.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:55.952 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:18:55.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:55.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:18:55.952 | INFO | Uploaded 36GiB to GPU in 2.02154s (17.8082GiB/s) | 20260506 14:18:55.952 | INFO | Downloaded 34GiB to GPU in 1.93685s (17.5543GiB/s) | 20260506 14:19:01.091 | INFO | Analyzing chunk frames = [2016768,2017280) | 20260506 14:19:01.091 | INFO | MemInfo: { | 20260506 14:19:01.091 | INFO | "gpu" : " 17 GB", | 20260506 14:19:01.091 | INFO | "huge" : "", | 20260506 14:19:01.091 | INFO | "huge_shared" : " 27 GB", | 20260506 14:19:01.091 | INFO | "malloc" : " 14 GB", | 20260506 14:19:01.091 | INFO | "pinned" : " 29 GB" | 20260506 14:19:01.091 | INFO | } | 20260506 14:19:01.091 | INFO | Perf report for the last 5.13919 seconds: | 20260506 14:19:01.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:19:01.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:01.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:01.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:01.091 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 14:19:01.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:01.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:01.091 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 14:19:01.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:01.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:01.091 | INFO | Uploaded 36GiB to GPU in 1.99349s (18.0588GiB/s) | 20260506 14:19:01.091 | INFO | Downloaded 34GiB to GPU in 1.92937s (17.6224GiB/s) | 20260506 14:19:06.229 | INFO | Analyzing chunk frames = [2017280,2017792) | 20260506 14:19:06.229 | INFO | MemInfo: { | 20260506 14:19:06.229 | INFO | "gpu" : " 17 GB", | 20260506 14:19:06.229 | INFO | "huge" : "", | 20260506 14:19:06.229 | INFO | "huge_shared" : " 27 GB", | 20260506 14:19:06.229 | INFO | "malloc" : " 14 GB", | 20260506 14:19:06.230 | INFO | "pinned" : " 29 GB" | 20260506 14:19:06.230 | INFO | } | 20260506 14:19:06.230 | INFO | Perf report for the last 5.13841 seconds: | 20260506 14:19:06.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:19:06.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:06.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:06.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:06.230 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 14:19:06.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:06.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:06.230 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 14:19:06.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:06.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:06.230 | INFO | Uploaded 36GiB to GPU in 1.99425s (18.0519GiB/s) | 20260506 14:19:06.230 | INFO | Downloaded 34GiB to GPU in 1.93231s (17.5955GiB/s) | 20260506 14:19:11.369 | INFO | Analyzing chunk frames = [2017792,2018304) | 20260506 14:19:11.369 | INFO | MemInfo: { | 20260506 14:19:11.369 | INFO | "gpu" : " 17 GB", | 20260506 14:19:11.369 | INFO | "huge" : "", | 20260506 14:19:11.369 | INFO | "huge_shared" : " 27 GB", | 20260506 14:19:11.369 | INFO | "malloc" : " 14 GB", | 20260506 14:19:11.369 | INFO | "pinned" : " 29 GB" | 20260506 14:19:11.369 | INFO | } | 20260506 14:19:11.369 | INFO | Perf report for the last 5.13924 seconds: | 20260506 14:19:11.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:19:11.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:11.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:11.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:11.369 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 14:19:11.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:11.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:11.369 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260506 14:19:11.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:11.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:11.369 | INFO | Uploaded 36GiB to GPU in 2.04371s (17.6151GiB/s) | 20260506 14:19:11.369 | INFO | Downloaded 34GiB to GPU in 1.92822s (17.6329GiB/s) | 20260506 14:19:16.510 | INFO | Analyzing chunk frames = [2018304,2018816) | 20260506 14:19:16.510 | INFO | MemInfo: { | 20260506 14:19:16.510 | INFO | "gpu" : " 17 GB", | 20260506 14:19:16.510 | INFO | "huge" : "", | 20260506 14:19:16.510 | INFO | "huge_shared" : " 27 GB", | 20260506 14:19:16.510 | INFO | "malloc" : " 14 GB", | 20260506 14:19:16.510 | INFO | "pinned" : " 29 GB" | 20260506 14:19:16.510 | INFO | } | 20260506 14:19:16.510 | INFO | Perf report for the last 5.14146 seconds: | 20260506 14:19:16.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:19:16.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:16.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:16.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:16.510 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:19:16.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:16.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:16.510 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 14:19:16.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:16.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:16.511 | INFO | Uploaded 36GiB to GPU in 2.02018s (17.8202GiB/s) | 20260506 14:19:16.511 | INFO | Downloaded 34GiB to GPU in 2.00376s (16.9681GiB/s) | 20260506 14:19:21.648 | INFO | Analyzing chunk frames = [2018816,2019328) | 20260506 14:19:21.648 | INFO | MemInfo: { | 20260506 14:19:21.648 | INFO | "gpu" : " 17 GB", | 20260506 14:19:21.648 | INFO | "huge" : "", | 20260506 14:19:21.648 | INFO | "huge_shared" : " 27 GB", | 20260506 14:19:21.648 | INFO | "malloc" : " 14 GB", | 20260506 14:19:21.648 | INFO | "pinned" : " 29 GB" | 20260506 14:19:21.648 | INFO | } | 20260506 14:19:21.648 | INFO | Perf report for the last 5.13808 seconds: | 20260506 14:19:21.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:19:21.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:21.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:21.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:21.648 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:19:21.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:21.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:21.648 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:19:21.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:21.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:21.649 | INFO | Uploaded 36GiB to GPU in 2.02446s (17.7825GiB/s) | 20260506 14:19:21.649 | INFO | Downloaded 34GiB to GPU in 1.9634s (17.3169GiB/s) | 20260506 14:19:21.767 | INFO | Starting a round of Baz IO | 20260506 14:19:24.931 | INFO | Populating RTMetrics report | 20260506 14:19:24.931 | INFO | RTMetrics report populated | 20260506 14:19:24.931 | INFO | Opening tmp RTMetrics json file | 20260506 14:19:24.942 | INFO | Serializing RTMetrics to json | 20260506 14:19:24.968 | INFO | Serialization done, swapping tmp file now... | 20260506 14:19:24.969 | INFO | RTMetrics done | 20260506 14:19:25.277 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.51 seconds | 20260506 14:19:25.277 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.46% / 0.01% / 61.52% | 20260506 14:19:25.277 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.63063% / 1.58946e-05% / 5.5631e-05% | 20260506 14:19:25.277 | INFO | Baz IO overall: | 20260506 14:19:25.277 | INFO | BytesWritten : 1040091678562 | 20260506 14:19:25.277 | INFO | NumEvents : 503269252800 | 20260506 14:19:26.787 | INFO | Analyzing chunk frames = [2019328,2019840) | 20260506 14:19:26.787 | INFO | MemInfo: { | 20260506 14:19:26.787 | INFO | "gpu" : " 17 GB", | 20260506 14:19:26.787 | INFO | "huge" : "", | 20260506 14:19:26.788 | INFO | "huge_shared" : " 27 GB", | 20260506 14:19:26.788 | INFO | "malloc" : " 14 GB", | 20260506 14:19:26.788 | INFO | "pinned" : " 29 GB" | 20260506 14:19:26.788 | INFO | } | 20260506 14:19:26.788 | INFO | Perf report for the last 5.13914 seconds: | 20260506 14:19:26.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:19:26.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:26.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:26.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:26.788 | INFO | ANALYSIS : 63.1%, 2.9, 95.4ms, 0 | 20260506 14:19:26.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:26.788 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 14:19:26.788 | INFO | PRE_HQ : 60.9%, 1.9, 60.2ms, 0 | 20260506 14:19:26.788 | INFO | BAZWRITER : 67.7%, 4.8, 167.0ms, 0 | 20260506 14:19:26.788 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 14:19:26.788 | INFO | Uploaded 36GiB to GPU in 2.07314s (17.365GiB/s) | 20260506 14:19:26.788 | INFO | Downloaded 38GiB to GPU in 2.23925s (16.97GiB/s) | 20260506 14:19:31.927 | INFO | Analyzing chunk frames = [2019840,2020352) | 20260506 14:19:31.927 | INFO | MemInfo: { | 20260506 14:19:31.927 | INFO | "gpu" : " 17 GB", | 20260506 14:19:31.927 | INFO | "huge" : "", | 20260506 14:19:31.927 | INFO | "huge_shared" : " 27 GB", | 20260506 14:19:31.927 | INFO | "malloc" : " 14 GB", | 20260506 14:19:31.927 | INFO | "pinned" : " 29 GB" | 20260506 14:19:31.927 | INFO | } | 20260506 14:19:31.927 | INFO | Perf report for the last 5.13971 seconds: | 20260506 14:19:31.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:19:31.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:31.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:31.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:31.927 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:19:31.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:31.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:31.927 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:19:31.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:31.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:31.928 | INFO | Uploaded 36GiB to GPU in 2.00393s (17.9647GiB/s) | 20260506 14:19:31.928 | INFO | Downloaded 34GiB to GPU in 1.93327s (17.5868GiB/s) | 20260506 14:19:37.066 | INFO | Analyzing chunk frames = [2020352,2020864) | 20260506 14:19:37.066 | INFO | MemInfo: { | 20260506 14:19:37.066 | INFO | "gpu" : " 17 GB", | 20260506 14:19:37.066 | INFO | "huge" : "", | 20260506 14:19:37.066 | INFO | "huge_shared" : " 27 GB", | 20260506 14:19:37.066 | INFO | "malloc" : " 14 GB", | 20260506 14:19:37.066 | INFO | "pinned" : " 29 GB" | 20260506 14:19:37.066 | INFO | } | 20260506 14:19:37.066 | INFO | Perf report for the last 5.13869 seconds: | 20260506 14:19:37.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:19:37.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:37.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:37.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:37.066 | INFO | ANALYSIS : 61.4%, 3.0, 95.0ms, 0 | 20260506 14:19:37.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:37.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:37.066 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:19:37.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:37.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:37.066 | INFO | Uploaded 36GiB to GPU in 2.00315s (17.9717GiB/s) | 20260506 14:19:37.066 | INFO | Downloaded 34GiB to GPU in 1.93171s (17.601GiB/s) | 20260506 14:19:42.205 | INFO | Analyzing chunk frames = [2020864,2021376) | 20260506 14:19:42.205 | INFO | MemInfo: { | 20260506 14:19:42.205 | INFO | "gpu" : " 17 GB", | 20260506 14:19:42.205 | INFO | "huge" : "", | 20260506 14:19:42.205 | INFO | "huge_shared" : " 27 GB", | 20260506 14:19:42.205 | INFO | "malloc" : " 14 GB", | 20260506 14:19:42.205 | INFO | "pinned" : " 29 GB" | 20260506 14:19:42.205 | INFO | } | 20260506 14:19:42.205 | INFO | Perf report for the last 5.13908 seconds: | 20260506 14:19:42.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:19:42.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:42.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:42.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:42.205 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 14:19:42.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:42.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:42.205 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:19:42.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:42.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:42.205 | INFO | Uploaded 36GiB to GPU in 2.00863s (17.9227GiB/s) | 20260506 14:19:42.205 | INFO | Downloaded 34GiB to GPU in 1.91536s (17.7512GiB/s) | 20260506 14:19:47.344 | INFO | Analyzing chunk frames = [2021376,2021888) | 20260506 14:19:47.344 | INFO | MemInfo: { | 20260506 14:19:47.344 | INFO | "gpu" : " 17 GB", | 20260506 14:19:47.344 | INFO | "huge" : "", | 20260506 14:19:47.344 | INFO | "huge_shared" : " 27 GB", | 20260506 14:19:47.344 | INFO | "malloc" : " 14 GB", | 20260506 14:19:47.344 | INFO | "pinned" : " 29 GB" | 20260506 14:19:47.345 | INFO | } | 20260506 14:19:47.345 | INFO | Perf report for the last 5.13945 seconds: | 20260506 14:19:47.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:19:47.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:47.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:47.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:47.345 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 14:19:47.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:47.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:47.345 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 14:19:47.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:47.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:47.345 | INFO | Uploaded 36GiB to GPU in 2.00699s (17.9373GiB/s) | 20260506 14:19:47.345 | INFO | Downloaded 34GiB to GPU in 1.93169s (17.6012GiB/s) | 20260506 14:19:52.484 | INFO | Analyzing chunk frames = [2021888,2022400) | 20260506 14:19:52.484 | INFO | MemInfo: { | 20260506 14:19:52.484 | INFO | "gpu" : " 17 GB", | 20260506 14:19:52.484 | INFO | "huge" : "", | 20260506 14:19:52.484 | INFO | "huge_shared" : " 27 GB", | 20260506 14:19:52.484 | INFO | "malloc" : " 14 GB", | 20260506 14:19:52.484 | INFO | "pinned" : " 29 GB" | 20260506 14:19:52.484 | INFO | } | 20260506 14:19:52.484 | INFO | Perf report for the last 5.13934 seconds: | 20260506 14:19:52.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:19:52.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:52.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:52.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:52.484 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:19:52.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:52.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:52.484 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260506 14:19:52.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:52.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:52.484 | INFO | Uploaded 36GiB to GPU in 2.02979s (17.7358GiB/s) | 20260506 14:19:52.484 | INFO | Downloaded 34GiB to GPU in 1.9123s (17.7796GiB/s) | 20260506 14:19:57.624 | INFO | Analyzing chunk frames = [2022400,2022912) | 20260506 14:19:57.624 | INFO | MemInfo: { | 20260506 14:19:57.624 | INFO | "gpu" : " 17 GB", | 20260506 14:19:57.624 | INFO | "huge" : "", | 20260506 14:19:57.624 | INFO | "huge_shared" : " 27 GB", | 20260506 14:19:57.624 | INFO | "malloc" : " 14 GB", | 20260506 14:19:57.624 | INFO | "pinned" : " 29 GB" | 20260506 14:19:57.624 | INFO | } | 20260506 14:19:57.624 | INFO | Perf report for the last 5.1402 seconds: | 20260506 14:19:57.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:19:57.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:57.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:57.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:57.624 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:19:57.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:57.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:57.624 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 14:19:57.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:57.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:19:57.624 | INFO | Uploaded 36GiB to GPU in 1.99058s (18.0852GiB/s) | 20260506 14:19:57.624 | INFO | Downloaded 34GiB to GPU in 2.00064s (16.9945GiB/s) | 20260506 14:20:02.764 | INFO | Analyzing chunk frames = [2022912,2023424) | 20260506 14:20:02.764 | INFO | MemInfo: { | 20260506 14:20:02.764 | INFO | "gpu" : " 17 GB", | 20260506 14:20:02.764 | INFO | "huge" : "", | 20260506 14:20:02.764 | INFO | "huge_shared" : " 27 GB", | 20260506 14:20:02.764 | INFO | "malloc" : " 14 GB", | 20260506 14:20:02.764 | INFO | "pinned" : " 29 GB" | 20260506 14:20:02.764 | INFO | } | 20260506 14:20:02.764 | INFO | Perf report for the last 5.14002 seconds: | 20260506 14:20:02.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:20:02.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:02.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:02.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:02.764 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 14:20:02.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:02.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:02.764 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 14:20:02.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:02.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:02.764 | INFO | Uploaded 36GiB to GPU in 2.02587s (17.7701GiB/s) | 20260506 14:20:02.764 | INFO | Downloaded 34GiB to GPU in 1.97754s (17.1931GiB/s) | 20260506 14:20:02.765 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:20:02.765 | INFO | Total mem usage: 17.4 GiB | 20260506 14:20:02.765 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:20:02.765 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -9.89%) | 20260506 14:20:02.765 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:20:02.765 | INFO | High Water Breakdown: | 20260506 14:20:02.765 | INFO | 61.4% : 9.72 GiB ( 89% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:20:02.765 | INFO | 38.5% : 6.1 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:20:02.765 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:20:02.765 | INFO | ------------------------------------------ | 20260506 14:20:02.765 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:20:02.765 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:20:02.765 | INFO | Total mem usage: 26.5 GiB | 20260506 14:20:02.765 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:20:02.765 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:20:02.765 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:20:02.765 | INFO | High Water Breakdown: | 20260506 14:20:02.765 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:20:02.765 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:20:02.765 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:20:02.765 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:20:02.765 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:20:02.765 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:20:02.765 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:20:02.765 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:20:02.765 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:20:02.765 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:20:02.765 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:20:02.765 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:20:02.765 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:20:02.765 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:20:02.765 | INFO | ------------------------------------------ | 20260506 14:20:02.765 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:20:02.765 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:20:02.765 | INFO | Total mem usage: 25.6 GiB | 20260506 14:20:02.765 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:20:02.765 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:20:02.765 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:20:02.765 | INFO | High Water Breakdown: | 20260506 14:20:02.765 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:20:02.765 | INFO | ------------------------------------------ | 20260506 14:20:02.765 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:20:02.765 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:20:02.765 | INFO | Total mem usage: 15.7 GiB | 20260506 14:20:02.765 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:20:02.765 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:20:02.765 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 14:20:02.765 | INFO | High Water Breakdown: | 20260506 14:20:02.765 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:20:02.765 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:20:02.765 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:20:02.765 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:20:02.765 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:20:02.765 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:20:02.765 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:20:02.765 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:20:02.765 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:20:02.765 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:20:02.765 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:20:02.765 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:20:02.765 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:20:02.765 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:20:02.765 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:20:02.765 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:20:02.765 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:20:02.765 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:20:02.765 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:20:02.765 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:20:02.765 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:20:02.765 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:20:02.765 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:20:02.765 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:20:02.765 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:20:02.765 | INFO | ------------------------------------------ | 20260506 14:20:02.765 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:20:02.765 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:20:02.765 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:20:02.765 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:20:02.765 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.59e+01, 1.34e+01 / 1.83e+00 | 20260506 14:20:02.765 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.02e-01, 1.08e-02 / 5.70e-03 | 20260506 14:20:02.765 | INFO | PerfReport: 11.57%, Baseline -- 3.65e+00 / 4.48e+00, 5.70e-01 / 3.07e-01 | 20260506 14:20:02.765 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.47e+00, 1.87e-01 / 1.26e-01 | 20260506 14:20:02.765 | INFO | PerfReport: 15.83%, DME -- 4.00e+01 / 4.44e+01, 3.95e+00 / 2.04e+00 | 20260506 14:20:02.765 | INFO | PerfReport: 56.73%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.49e+00 / 1.64e+00 | 20260506 14:20:02.765 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.35e+00 / 2.93e+00, 4.08e-01 / 2.67e-01 | 20260506 14:20:02.765 | INFO | PerfReport: 3.62%, Metrics -- 1.14e+00 / 1.52e+00, 2.93e-01 / 1.78e-01 | 20260506 14:20:02.765 | INFO | PerfReport: 1.27%, PulseSerializing -- 3.99e-01 / 4.56e-01, 6.27e-02 / 3.16e-02 | 20260506 14:20:02.765 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:20:02.765 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:20:02.765 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:20:02.765 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 14:20:02.765 | INFO | PerfReport: 21.53%, Upload -- 2.11e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260506 14:20:02.766 | INFO | PerfReport: 21.10%, Download -- 2.07e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 14:20:02.766 | INFO | PerfReport: 32.25%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 14:20:02.766 | INFO | PerfReport: 1.59%, LockUpload -- 1.56e+00 / 0.00e+00, 6.37e+00 / 0.00e+00 | 20260506 14:20:02.766 | INFO | PerfReport: 1.12%, LockDownload -- 1.10e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260506 14:20:02.766 | INFO | PerfReport: 22.39%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:20:02.766 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.40e-02 / 0.00e+00 | 20260506 14:20:02.886 | INFO | Starting a round of Baz IO | 20260506 14:20:06.032 | INFO | Populating RTMetrics report | 20260506 14:20:06.033 | INFO | RTMetrics report populated | 20260506 14:20:06.033 | INFO | Opening tmp RTMetrics json file | 20260506 14:20:06.110 | INFO | Serializing RTMetrics to json | 20260506 14:20:06.138 | INFO | Serialization done, swapping tmp file now... | 20260506 14:20:06.145 | INFO | RTMetrics done | 20260506 14:20:06.801 | INFO | Finished a round of Baz IO: Wrote 2.116 GiB in 3.915 seconds | 20260506 14:20:06.801 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.45% / 0.02% / 61.54% | 20260506 14:20:06.801 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.63164% / 2.38419e-05% / 6.35783e-05% | 20260506 14:20:06.801 | INFO | Baz IO overall: | 20260506 14:20:06.801 | INFO | BytesWritten : 1042363796197 | 20260506 14:20:06.801 | INFO | NumEvents : 504370937141 | 20260506 14:20:07.903 | INFO | Analyzing chunk frames = [2023424,2023936) | 20260506 14:20:07.903 | INFO | MemInfo: { | 20260506 14:20:07.903 | INFO | "gpu" : " 17 GB", | 20260506 14:20:07.903 | INFO | "huge" : "", | 20260506 14:20:07.903 | INFO | "huge_shared" : " 27 GB", | 20260506 14:20:07.903 | INFO | "malloc" : " 14 GB", | 20260506 14:20:07.903 | INFO | "pinned" : " 29 GB" | 20260506 14:20:07.903 | INFO | } | 20260506 14:20:07.903 | INFO | Perf report for the last 5.13842 seconds: | 20260506 14:20:07.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:20:07.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:07.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:07.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:07.903 | INFO | ANALYSIS : 63.0%, 2.9, 95.5ms, 0 | 20260506 14:20:07.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:07.903 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260506 14:20:07.903 | INFO | PRE_HQ : 61.2%, 1.9, 59.5ms, 0 | 20260506 14:20:07.903 | INFO | BAZWRITER : 75.4%, 4.8, 187.2ms, 0 | 20260506 14:20:07.903 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 14:20:07.903 | INFO | Uploaded 36GiB to GPU in 2.05475s (17.5204GiB/s) | 20260506 14:20:07.903 | INFO | Downloaded 38GiB to GPU in 2.23451s (17.006GiB/s) | 20260506 14:20:13.041 | INFO | Analyzing chunk frames = [2023936,2024448) | 20260506 14:20:13.041 | INFO | MemInfo: { | 20260506 14:20:13.041 | INFO | "gpu" : " 17 GB", | 20260506 14:20:13.041 | INFO | "huge" : "", | 20260506 14:20:13.041 | INFO | "huge_shared" : " 27 GB", | 20260506 14:20:13.041 | INFO | "malloc" : " 14 GB", | 20260506 14:20:13.041 | INFO | "pinned" : " 29 GB" | 20260506 14:20:13.041 | INFO | } | 20260506 14:20:13.041 | INFO | Perf report for the last 5.1385 seconds: | 20260506 14:20:13.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:20:13.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:13.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:13.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:13.041 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 14:20:13.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:13.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:13.041 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 14:20:13.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:13.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:13.041 | INFO | Uploaded 36GiB to GPU in 1.99459s (18.0488GiB/s) | 20260506 14:20:13.042 | INFO | Downloaded 34GiB to GPU in 1.92685s (17.6454GiB/s) | 20260506 14:20:18.180 | INFO | Analyzing chunk frames = [2024448,2024960) | 20260506 14:20:18.180 | INFO | MemInfo: { | 20260506 14:20:18.180 | INFO | "gpu" : " 17 GB", | 20260506 14:20:18.180 | INFO | "huge" : "", | 20260506 14:20:18.180 | INFO | "huge_shared" : " 27 GB", | 20260506 14:20:18.180 | INFO | "malloc" : " 14 GB", | 20260506 14:20:18.180 | INFO | "pinned" : " 29 GB" | 20260506 14:20:18.180 | INFO | } | 20260506 14:20:18.180 | INFO | Perf report for the last 5.13928 seconds: | 20260506 14:20:18.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:20:18.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:18.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:18.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:18.180 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:20:18.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:18.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:18.180 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:20:18.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:18.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:18.181 | INFO | Uploaded 36GiB to GPU in 2.00411s (17.9631GiB/s) | 20260506 14:20:18.181 | INFO | Downloaded 34GiB to GPU in 1.9364s (17.5584GiB/s) | 20260506 14:20:23.320 | INFO | Analyzing chunk frames = [2024960,2025472) | 20260506 14:20:23.320 | INFO | MemInfo: { | 20260506 14:20:23.320 | INFO | "gpu" : " 17 GB", | 20260506 14:20:23.320 | INFO | "huge" : "", | 20260506 14:20:23.320 | INFO | "huge_shared" : " 27 GB", | 20260506 14:20:23.320 | INFO | "malloc" : " 14 GB", | 20260506 14:20:23.320 | INFO | "pinned" : " 29 GB" | 20260506 14:20:23.320 | INFO | } | 20260506 14:20:23.320 | INFO | Perf report for the last 5.14005 seconds: | 20260506 14:20:23.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:20:23.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:23.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:23.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:23.321 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 14:20:23.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:23.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:23.321 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:20:23.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:23.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:23.321 | INFO | Uploaded 36GiB to GPU in 2.0039s (17.965GiB/s) | 20260506 14:20:23.321 | INFO | Downloaded 34GiB to GPU in 1.91143s (17.7877GiB/s) | 20260506 14:20:28.460 | INFO | Analyzing chunk frames = [2025472,2025984) | 20260506 14:20:28.460 | INFO | MemInfo: { | 20260506 14:20:28.460 | INFO | "gpu" : " 17 GB", | 20260506 14:20:28.460 | INFO | "huge" : "", | 20260506 14:20:28.460 | INFO | "huge_shared" : " 27 GB", | 20260506 14:20:28.460 | INFO | "malloc" : " 14 GB", | 20260506 14:20:28.460 | INFO | "pinned" : " 29 GB" | 20260506 14:20:28.460 | INFO | } | 20260506 14:20:28.461 | INFO | Perf report for the last 5.14019 seconds: | 20260506 14:20:28.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:20:28.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:28.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:28.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:28.461 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 14:20:28.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:28.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:28.461 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:20:28.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:28.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:28.461 | INFO | Uploaded 36GiB to GPU in 2.00678s (17.9392GiB/s) | 20260506 14:20:28.461 | INFO | Downloaded 34GiB to GPU in 1.93855s (17.5389GiB/s) | 20260506 14:20:33.600 | INFO | Analyzing chunk frames = [2025984,2026496) | 20260506 14:20:33.600 | INFO | MemInfo: { | 20260506 14:20:33.600 | INFO | "gpu" : " 17 GB", | 20260506 14:20:33.600 | INFO | "huge" : "", | 20260506 14:20:33.600 | INFO | "huge_shared" : " 27 GB", | 20260506 14:20:33.600 | INFO | "malloc" : " 14 GB", | 20260506 14:20:33.600 | INFO | "pinned" : " 29 GB" | 20260506 14:20:33.600 | INFO | } | 20260506 14:20:33.600 | INFO | Perf report for the last 5.13928 seconds: | 20260506 14:20:33.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:20:33.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:33.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:33.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:33.600 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:20:33.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:33.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:33.600 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 14:20:33.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:33.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:33.600 | INFO | Uploaded 36GiB to GPU in 2.06582s (17.4265GiB/s) | 20260506 14:20:33.600 | INFO | Downloaded 34GiB to GPU in 1.9205s (17.7038GiB/s) | 20260506 14:20:38.739 | INFO | Analyzing chunk frames = [2026496,2027008) | 20260506 14:20:38.739 | INFO | MemInfo: { | 20260506 14:20:38.739 | INFO | "gpu" : " 17 GB", | 20260506 14:20:38.739 | INFO | "huge" : "", | 20260506 14:20:38.739 | INFO | "huge_shared" : " 27 GB", | 20260506 14:20:38.739 | INFO | "malloc" : " 14 GB", | 20260506 14:20:38.739 | INFO | "pinned" : " 29 GB" | 20260506 14:20:38.740 | INFO | } | 20260506 14:20:38.740 | INFO | Perf report for the last 5.14004 seconds: | 20260506 14:20:38.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:20:38.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:38.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:38.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:38.740 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 14:20:38.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:38.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:38.740 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 14:20:38.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:38.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:38.740 | INFO | Uploaded 36GiB to GPU in 2.04842s (17.5745GiB/s) | 20260506 14:20:38.740 | INFO | Downloaded 34GiB to GPU in 1.98745s (17.1073GiB/s) | 20260506 14:20:43.877 | INFO | Analyzing chunk frames = [2027008,2027520) | 20260506 14:20:43.877 | INFO | MemInfo: { | 20260506 14:20:43.877 | INFO | "gpu" : " 17 GB", | 20260506 14:20:43.877 | INFO | "huge" : "", | 20260506 14:20:43.877 | INFO | "huge_shared" : " 27 GB", | 20260506 14:20:43.877 | INFO | "malloc" : " 14 GB", | 20260506 14:20:43.877 | INFO | "pinned" : " 29 GB" | 20260506 14:20:43.877 | INFO | } | 20260506 14:20:43.877 | INFO | Perf report for the last 5.13743 seconds: | 20260506 14:20:43.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:20:43.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:43.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:43.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:43.877 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 14:20:43.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:43.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:43.877 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 14:20:43.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:43.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:43.878 | INFO | Uploaded 36GiB to GPU in 2.00577s (17.9482GiB/s) | 20260506 14:20:43.878 | INFO | Downloaded 34GiB to GPU in 1.98111s (17.1621GiB/s) | 20260506 14:20:43.991 | INFO | Starting a round of Baz IO | 20260506 14:20:47.153 | INFO | Populating RTMetrics report | 20260506 14:20:47.154 | INFO | RTMetrics report populated | 20260506 14:20:47.154 | INFO | Opening tmp RTMetrics json file | 20260506 14:20:47.186 | INFO | Serializing RTMetrics to json | 20260506 14:20:47.244 | INFO | Serialization done, swapping tmp file now... | 20260506 14:20:47.247 | INFO | RTMetrics done | 20260506 14:20:47.621 | INFO | Finished a round of Baz IO: Wrote 2.115 GiB in 3.629 seconds | 20260506 14:20:47.621 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.43% / 0.01% / 61.55% | 20260506 14:20:47.621 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.61972% / 3.17891e-05% / 5.5631e-05% | 20260506 14:20:47.621 | INFO | Baz IO overall: | 20260506 14:20:47.621 | INFO | BytesWritten : 1044634567192 | 20260506 14:20:47.621 | INFO | NumEvents : 505471351085 | 20260506 14:20:49.017 | INFO | Analyzing chunk frames = [2027520,2028032) | 20260506 14:20:49.017 | INFO | MemInfo: { | 20260506 14:20:49.017 | INFO | "gpu" : " 17 GB", | 20260506 14:20:49.017 | INFO | "huge" : "", | 20260506 14:20:49.017 | INFO | "huge_shared" : " 27 GB", | 20260506 14:20:49.017 | INFO | "malloc" : " 14 GB", | 20260506 14:20:49.017 | INFO | "pinned" : " 29 GB" | 20260506 14:20:49.017 | INFO | } | 20260506 14:20:49.017 | INFO | Perf report for the last 5.13963 seconds: | 20260506 14:20:49.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:20:49.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:49.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:49.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:49.017 | INFO | ANALYSIS : 63.1%, 2.9, 95.3ms, 0 | 20260506 14:20:49.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:49.017 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260506 14:20:49.017 | INFO | PRE_HQ : 61.4%, 1.9, 59.8ms, 0 | 20260506 14:20:49.017 | INFO | BAZWRITER : 69.9%, 4.9, 174.9ms, 0 | 20260506 14:20:49.017 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 14:20:49.017 | INFO | Uploaded 36GiB to GPU in 2.05575s (17.5119GiB/s) | 20260506 14:20:49.017 | INFO | Downloaded 38GiB to GPU in 2.22759s (17.0588GiB/s) | 20260506 14:20:54.157 | INFO | Analyzing chunk frames = [2028032,2028544) | 20260506 14:20:54.157 | INFO | MemInfo: { | 20260506 14:20:54.157 | INFO | "gpu" : " 17 GB", | 20260506 14:20:54.157 | INFO | "huge" : "", | 20260506 14:20:54.157 | INFO | "huge_shared" : " 27 GB", | 20260506 14:20:54.157 | INFO | "malloc" : " 14 GB", | 20260506 14:20:54.157 | INFO | "pinned" : " 29 GB" | 20260506 14:20:54.157 | INFO | } | 20260506 14:20:54.157 | INFO | Perf report for the last 5.13972 seconds: | 20260506 14:20:54.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:20:54.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:54.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:54.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:54.157 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:20:54.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:54.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:54.157 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:20:54.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:54.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:54.157 | INFO | Uploaded 36GiB to GPU in 2.00862s (17.9228GiB/s) | 20260506 14:20:54.157 | INFO | Downloaded 34GiB to GPU in 1.94311s (17.4977GiB/s) | 20260506 14:20:59.295 | INFO | Analyzing chunk frames = [2028544,2029056) | 20260506 14:20:59.295 | INFO | MemInfo: { | 20260506 14:20:59.296 | INFO | "gpu" : " 17 GB", | 20260506 14:20:59.296 | INFO | "huge" : "", | 20260506 14:20:59.296 | INFO | "huge_shared" : " 27 GB", | 20260506 14:20:59.296 | INFO | "malloc" : " 14 GB", | 20260506 14:20:59.296 | INFO | "pinned" : " 29 GB" | 20260506 14:20:59.296 | INFO | } | 20260506 14:20:59.296 | INFO | Perf report for the last 5.13892 seconds: | 20260506 14:20:59.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:20:59.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:59.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:59.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:59.296 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:20:59.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:59.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:59.296 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:20:59.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:59.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:20:59.296 | INFO | Uploaded 36GiB to GPU in 2.00674s (17.9395GiB/s) | 20260506 14:20:59.296 | INFO | Downloaded 34GiB to GPU in 1.93644s (17.558GiB/s) | 20260506 14:21:04.435 | INFO | Analyzing chunk frames = [2029056,2029568) | 20260506 14:21:04.435 | INFO | MemInfo: { | 20260506 14:21:04.435 | INFO | "gpu" : " 17 GB", | 20260506 14:21:04.435 | INFO | "huge" : "", | 20260506 14:21:04.435 | INFO | "huge_shared" : " 27 GB", | 20260506 14:21:04.435 | INFO | "malloc" : " 14 GB", | 20260506 14:21:04.435 | INFO | "pinned" : " 29 GB" | 20260506 14:21:04.435 | INFO | } | 20260506 14:21:04.435 | INFO | Perf report for the last 5.13945 seconds: | 20260506 14:21:04.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:21:04.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:04.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:04.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:04.435 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 14:21:04.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:04.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:04.436 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:21:04.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:04.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:04.436 | INFO | Uploaded 36GiB to GPU in 1.97585s (18.22GiB/s) | 20260506 14:21:04.436 | INFO | Downloaded 34GiB to GPU in 1.91013s (17.7999GiB/s) | 20260506 14:21:09.574 | INFO | Analyzing chunk frames = [2029568,2030080) | 20260506 14:21:09.574 | INFO | MemInfo: { | 20260506 14:21:09.574 | INFO | "gpu" : " 17 GB", | 20260506 14:21:09.574 | INFO | "huge" : "", | 20260506 14:21:09.574 | INFO | "huge_shared" : " 27 GB", | 20260506 14:21:09.574 | INFO | "malloc" : " 14 GB", | 20260506 14:21:09.575 | INFO | "pinned" : " 29 GB" | 20260506 14:21:09.575 | INFO | } | 20260506 14:21:09.575 | INFO | Perf report for the last 5.13963 seconds: | 20260506 14:21:09.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:21:09.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:09.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:09.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:09.575 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:21:09.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:09.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:09.575 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:21:09.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:09.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:09.575 | INFO | Uploaded 36GiB to GPU in 1.99126s (18.079GiB/s) | 20260506 14:21:09.576 | INFO | Downloaded 34GiB to GPU in 1.93325s (17.587GiB/s) | 20260506 14:21:14.714 | INFO | Analyzing chunk frames = [2030080,2030592) | 20260506 14:21:14.714 | INFO | MemInfo: { | 20260506 14:21:14.714 | INFO | "gpu" : " 17 GB", | 20260506 14:21:14.714 | INFO | "huge" : "", | 20260506 14:21:14.714 | INFO | "huge_shared" : " 27 GB", | 20260506 14:21:14.714 | INFO | "malloc" : " 14 GB", | 20260506 14:21:14.714 | INFO | "pinned" : " 29 GB" | 20260506 14:21:14.714 | INFO | } | 20260506 14:21:14.714 | INFO | Perf report for the last 5.13917 seconds: | 20260506 14:21:14.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:21:14.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:14.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:14.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:14.714 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 14:21:14.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:14.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:14.714 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260506 14:21:14.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:14.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:14.714 | INFO | Uploaded 36GiB to GPU in 2.02813s (17.7504GiB/s) | 20260506 14:21:14.714 | INFO | Downloaded 34GiB to GPU in 1.93366s (17.5832GiB/s) | 20260506 14:21:19.854 | INFO | Analyzing chunk frames = [2030592,2031104) | 20260506 14:21:19.854 | INFO | MemInfo: { | 20260506 14:21:19.854 | INFO | "gpu" : " 17 GB", | 20260506 14:21:19.854 | INFO | "huge" : "", | 20260506 14:21:19.854 | INFO | "huge_shared" : " 27 GB", | 20260506 14:21:19.854 | INFO | "malloc" : " 14 GB", | 20260506 14:21:19.854 | INFO | "pinned" : " 29 GB" | 20260506 14:21:19.854 | INFO | } | 20260506 14:21:19.854 | INFO | Perf report for the last 5.14012 seconds: | 20260506 14:21:19.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:21:19.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:19.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:19.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:19.854 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:21:19.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:19.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:19.854 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 14:21:19.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:19.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:19.855 | INFO | Uploaded 36GiB to GPU in 2.02316s (17.7939GiB/s) | 20260506 14:21:19.855 | INFO | Downloaded 34GiB to GPU in 2.00363s (16.9692GiB/s) | 20260506 14:21:24.992 | INFO | Analyzing chunk frames = [2031104,2031616) | 20260506 14:21:24.992 | INFO | MemInfo: { | 20260506 14:21:24.992 | INFO | "gpu" : " 17 GB", | 20260506 14:21:24.992 | INFO | "huge" : "", | 20260506 14:21:24.992 | INFO | "huge_shared" : " 27 GB", | 20260506 14:21:24.992 | INFO | "malloc" : " 14 GB", | 20260506 14:21:24.992 | INFO | "pinned" : " 29 GB" | 20260506 14:21:24.992 | INFO | } | 20260506 14:21:24.992 | INFO | Perf report for the last 5.13822 seconds: | 20260506 14:21:24.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:21:24.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:24.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:24.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:24.993 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:21:24.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:24.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:24.993 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 14:21:24.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:24.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:24.993 | INFO | Uploaded 36GiB to GPU in 2.03229s (17.714GiB/s) | 20260506 14:21:24.993 | INFO | Downloaded 34GiB to GPU in 1.97252s (17.2369GiB/s) | 20260506 14:21:24.993 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:21:24.993 | INFO | Total mem usage: 17.4 GiB | 20260506 14:21:24.993 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:21:24.993 | INFO | Recent High Water Marks Sum: 16.9 GiB (Savings: -3%) | 20260506 14:21:24.993 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:21:24.993 | INFO | High Water Breakdown: | 20260506 14:21:24.993 | INFO | 62.4% : 10.5 GiB ( 96.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:21:24.993 | INFO | 37.5% : 6.34 GiB ( 94.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:21:24.993 | INFO | 0.118% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:21:24.993 | INFO | ------------------------------------------ | 20260506 14:21:24.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:21:24.993 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:21:24.993 | INFO | Total mem usage: 26.5 GiB | 20260506 14:21:24.993 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:21:24.993 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:21:24.993 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:21:24.993 | INFO | High Water Breakdown: | 20260506 14:21:24.993 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:21:24.993 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:21:24.993 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:21:24.993 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:21:24.993 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:21:24.993 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:21:24.993 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:21:24.993 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:21:24.993 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:21:24.993 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:21:24.993 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:21:24.993 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:21:24.993 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:21:24.993 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:21:24.993 | INFO | ------------------------------------------ | 20260506 14:21:24.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:21:24.993 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:21:24.993 | INFO | Total mem usage: 25.6 GiB | 20260506 14:21:24.993 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:21:24.993 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:21:24.993 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:21:24.994 | INFO | High Water Breakdown: | 20260506 14:21:24.994 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:21:24.994 | INFO | ------------------------------------------ | 20260506 14:21:24.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:21:24.994 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:21:24.994 | INFO | Total mem usage: 15.7 GiB | 20260506 14:21:24.994 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:21:24.994 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:21:24.994 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 14:21:24.994 | INFO | High Water Breakdown: | 20260506 14:21:24.994 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:21:24.994 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:21:24.994 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:21:24.994 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:21:24.994 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:21:24.994 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:21:24.994 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:21:24.994 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:21:24.994 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:21:24.994 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:21:24.994 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:21:24.994 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:21:24.994 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:21:24.994 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:21:24.994 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:21:24.994 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:21:24.994 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:21:24.994 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:21:24.994 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:21:24.994 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:21:24.994 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:21:24.994 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:21:24.994 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:21:24.994 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:21:24.994 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:21:24.994 | INFO | ------------------------------------------ | 20260506 14:21:24.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:21:24.994 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:21:24.994 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:21:24.994 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:21:24.994 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.58e+01, 1.37e+01 / 2.26e+00 | 20260506 14:21:24.994 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.03e-01, 1.03e-02 / 6.29e-03 | 20260506 14:21:24.994 | INFO | PerfReport: 11.60%, Baseline -- 3.66e+00 / 4.43e+00, 5.79e-01 / 3.47e-01 | 20260506 14:21:24.994 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.48e+00, 1.93e-01 / 1.29e-01 | 20260506 14:21:24.994 | INFO | PerfReport: 15.82%, DME -- 4.00e+01 / 4.43e+01, 4.13e+00 / 2.35e+00 | 20260506 14:21:24.994 | INFO | PerfReport: 56.70%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.48e+00 / 1.79e+00 | 20260506 14:21:24.994 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.35e+00 / 2.96e+00, 4.07e-01 / 2.36e-01 | 20260506 14:21:24.994 | INFO | PerfReport: 3.62%, Metrics -- 1.14e+00 / 1.53e+00, 2.98e-01 / 1.81e-01 | 20260506 14:21:24.994 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.99e-01 / 4.54e-01, 6.16e-02 / 2.44e-02 | 20260506 14:21:24.994 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:21:24.994 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:21:24.994 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:21:24.994 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 14:21:24.994 | INFO | PerfReport: 21.50%, Upload -- 2.10e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 14:21:24.994 | INFO | PerfReport: 21.10%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 14:21:24.994 | INFO | PerfReport: 32.28%, Compute -- 3.16e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 14:21:24.994 | INFO | PerfReport: 1.57%, LockUpload -- 1.54e+00 / 0.00e+00, 6.26e+00 / 0.00e+00 | 20260506 14:21:24.994 | INFO | PerfReport: 1.08%, LockDownload -- 1.06e+00 / 0.00e+00, 2.70e+00 / 0.00e+00 | 20260506 14:21:24.994 | INFO | PerfReport: 22.45%, LockCompute -- 2.20e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 14:21:24.994 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.43e-02 / 0.00e+00 | 20260506 14:21:25.110 | INFO | Starting a round of Baz IO | 20260506 14:21:28.263 | INFO | Populating RTMetrics report | 20260506 14:21:28.264 | INFO | RTMetrics report populated | 20260506 14:21:28.264 | INFO | Opening tmp RTMetrics json file | 20260506 14:21:28.269 | INFO | Serializing RTMetrics to json | 20260506 14:21:28.294 | INFO | Serialization done, swapping tmp file now... | 20260506 14:21:28.319 | INFO | RTMetrics done | 20260506 14:21:28.413 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 3.303 seconds | 20260506 14:21:28.413 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.42% / 0.01% / 61.57% | 20260506 14:21:28.413 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.61029% / 7.94729e-06% / 9.53674e-05% | 20260506 14:21:28.413 | INFO | Baz IO overall: | 20260506 14:21:28.413 | INFO | BytesWritten : 1046903751991 | 20260506 14:21:28.413 | INFO | NumEvents : 506570673308 | 20260506 14:21:30.131 | INFO | Analyzing chunk frames = [2031616,2032128) | 20260506 14:21:30.132 | INFO | MemInfo: { | 20260506 14:21:30.132 | INFO | "gpu" : " 17 GB", | 20260506 14:21:30.132 | INFO | "huge" : "", | 20260506 14:21:30.132 | INFO | "huge_shared" : " 27 GB", | 20260506 14:21:30.132 | INFO | "malloc" : " 14 GB", | 20260506 14:21:30.132 | INFO | "pinned" : " 29 GB" | 20260506 14:21:30.132 | INFO | } | 20260506 14:21:30.132 | INFO | Perf report for the last 5.13915 seconds: | 20260506 14:21:30.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:21:30.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:30.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:30.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:30.132 | INFO | ANALYSIS : 63.1%, 2.9, 95.2ms, 0 | 20260506 14:21:30.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:30.132 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 14:21:30.132 | INFO | PRE_HQ : 61.0%, 1.9, 60.1ms, 0 | 20260506 14:21:30.132 | INFO | BAZWRITER : 63.7%, 4.8, 156.8ms, 0 | 20260506 14:21:30.132 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260506 14:21:30.132 | INFO | Uploaded 36GiB to GPU in 2.07756s (17.328GiB/s) | 20260506 14:21:30.132 | INFO | Downloaded 38GiB to GPU in 2.22542s (17.0754GiB/s) | 20260506 14:21:35.271 | INFO | Analyzing chunk frames = [2032128,2032640) | 20260506 14:21:35.271 | INFO | MemInfo: { | 20260506 14:21:35.271 | INFO | "gpu" : " 17 GB", | 20260506 14:21:35.271 | INFO | "huge" : "", | 20260506 14:21:35.271 | INFO | "huge_shared" : " 27 GB", | 20260506 14:21:35.271 | INFO | "malloc" : " 14 GB", | 20260506 14:21:35.271 | INFO | "pinned" : " 29 GB" | 20260506 14:21:35.271 | INFO | } | 20260506 14:21:35.271 | INFO | Perf report for the last 5.13948 seconds: | 20260506 14:21:35.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:21:35.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:35.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:35.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:35.271 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 14:21:35.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:35.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:35.271 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:21:35.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:35.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:35.271 | INFO | Uploaded 36GiB to GPU in 2.00786s (17.9295GiB/s) | 20260506 14:21:35.271 | INFO | Downloaded 34GiB to GPU in 1.94374s (17.492GiB/s) | 20260506 14:21:40.412 | INFO | Analyzing chunk frames = [2032640,2033152) | 20260506 14:21:40.412 | INFO | MemInfo: { | 20260506 14:21:40.412 | INFO | "gpu" : " 17 GB", | 20260506 14:21:40.412 | INFO | "huge" : "", | 20260506 14:21:40.412 | INFO | "huge_shared" : " 27 GB", | 20260506 14:21:40.412 | INFO | "malloc" : " 14 GB", | 20260506 14:21:40.412 | INFO | "pinned" : " 29 GB" | 20260506 14:21:40.414 | INFO | } | 20260506 14:21:40.414 | INFO | Perf report for the last 5.14264 seconds: | 20260506 14:21:40.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:21:40.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:40.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:40.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:40.414 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:21:40.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:40.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:40.414 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:21:40.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:40.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:40.414 | INFO | Uploaded 36GiB to GPU in 1.99882s (18.0106GiB/s) | 20260506 14:21:40.414 | INFO | Downloaded 34GiB to GPU in 1.92836s (17.6315GiB/s) | 20260506 14:21:45.550 | INFO | Analyzing chunk frames = [2033152,2033664) | 20260506 14:21:45.550 | INFO | MemInfo: { | 20260506 14:21:45.550 | INFO | "gpu" : " 17 GB", | 20260506 14:21:45.550 | INFO | "huge" : "", | 20260506 14:21:45.550 | INFO | "huge_shared" : " 27 GB", | 20260506 14:21:45.550 | INFO | "malloc" : " 14 GB", | 20260506 14:21:45.550 | INFO | "pinned" : " 29 GB" | 20260506 14:21:45.550 | INFO | } | 20260506 14:21:45.550 | INFO | Perf report for the last 5.13676 seconds: | 20260506 14:21:45.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:21:45.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:45.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:45.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:45.550 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:21:45.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:45.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:45.551 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:21:45.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:45.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:45.551 | INFO | Uploaded 36GiB to GPU in 1.98028s (18.1793GiB/s) | 20260506 14:21:45.551 | INFO | Downloaded 34GiB to GPU in 1.91205s (17.782GiB/s) | 20260506 14:21:50.689 | INFO | Analyzing chunk frames = [2033664,2034176) | 20260506 14:21:50.689 | INFO | MemInfo: { | 20260506 14:21:50.689 | INFO | "gpu" : " 17 GB", | 20260506 14:21:50.689 | INFO | "huge" : "", | 20260506 14:21:50.689 | INFO | "huge_shared" : " 27 GB", | 20260506 14:21:50.689 | INFO | "malloc" : " 14 GB", | 20260506 14:21:50.689 | INFO | "pinned" : " 29 GB" | 20260506 14:21:50.689 | INFO | } | 20260506 14:21:50.689 | INFO | Perf report for the last 5.13896 seconds: | 20260506 14:21:50.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:21:50.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:50.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:50.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:50.690 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 14:21:50.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:50.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:50.690 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:21:50.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:50.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:50.690 | INFO | Uploaded 36GiB to GPU in 2.01835s (17.8363GiB/s) | 20260506 14:21:50.690 | INFO | Downloaded 34GiB to GPU in 1.93691s (17.5537GiB/s) | 20260506 14:21:55.828 | INFO | Analyzing chunk frames = [2034176,2034688) | 20260506 14:21:55.828 | INFO | MemInfo: { | 20260506 14:21:55.828 | INFO | "gpu" : " 17 GB", | 20260506 14:21:55.828 | INFO | "huge" : "", | 20260506 14:21:55.828 | INFO | "huge_shared" : " 27 GB", | 20260506 14:21:55.828 | INFO | "malloc" : " 14 GB", | 20260506 14:21:55.828 | INFO | "pinned" : " 29 GB" | 20260506 14:21:55.828 | INFO | } | 20260506 14:21:55.828 | INFO | Perf report for the last 5.13874 seconds: | 20260506 14:21:55.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:21:55.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:55.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:55.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:55.828 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:21:55.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:55.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:55.828 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 14:21:55.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:55.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:21:55.828 | INFO | Uploaded 36GiB to GPU in 2.08699s (17.2497GiB/s) | 20260506 14:21:55.829 | INFO | Downloaded 34GiB to GPU in 1.98316s (17.1443GiB/s) | 20260506 14:22:00.969 | INFO | Analyzing chunk frames = [2034688,2035200) | 20260506 14:22:00.969 | INFO | MemInfo: { | 20260506 14:22:00.969 | INFO | "gpu" : " 17 GB", | 20260506 14:22:00.969 | INFO | "huge" : "", | 20260506 14:22:00.969 | INFO | "huge_shared" : " 27 GB", | 20260506 14:22:00.969 | INFO | "malloc" : " 14 GB", | 20260506 14:22:00.969 | INFO | "pinned" : " 29 GB" | 20260506 14:22:00.969 | INFO | } | 20260506 14:22:00.969 | INFO | Perf report for the last 5.14083 seconds: | 20260506 14:22:00.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:22:00.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:00.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:00.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:00.969 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:22:00.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:00.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:00.969 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 14:22:00.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:00.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:00.969 | INFO | Uploaded 36GiB to GPU in 2.02263s (17.7986GiB/s) | 20260506 14:22:00.969 | INFO | Downloaded 34GiB to GPU in 2.02064s (16.8264GiB/s) | 20260506 14:22:06.108 | INFO | Analyzing chunk frames = [2035200,2035712) | 20260506 14:22:06.108 | INFO | MemInfo: { | 20260506 14:22:06.108 | INFO | "gpu" : " 17 GB", | 20260506 14:22:06.108 | INFO | "huge" : "", | 20260506 14:22:06.108 | INFO | "huge_shared" : " 27 GB", | 20260506 14:22:06.108 | INFO | "malloc" : " 14 GB", | 20260506 14:22:06.108 | INFO | "pinned" : " 29 GB" | 20260506 14:22:06.108 | INFO | } | 20260506 14:22:06.109 | INFO | Perf report for the last 5.13951 seconds: | 20260506 14:22:06.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:22:06.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:06.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:06.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:06.109 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:22:06.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:06.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:06.109 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 14:22:06.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:06.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:06.109 | INFO | Uploaded 36GiB to GPU in 2.03693s (17.6736GiB/s) | 20260506 14:22:06.109 | INFO | Downloaded 34GiB to GPU in 1.96734s (17.2822GiB/s) | 20260506 14:22:06.225 | INFO | Starting a round of Baz IO | 20260506 14:22:09.376 | INFO | Populating RTMetrics report | 20260506 14:22:09.376 | INFO | RTMetrics report populated | 20260506 14:22:09.376 | INFO | Opening tmp RTMetrics json file | 20260506 14:22:09.415 | INFO | Serializing RTMetrics to json | 20260506 14:22:09.418 | INFO | Serialization done, swapping tmp file now... | 20260506 14:22:09.419 | INFO | RTMetrics done | 20260506 14:22:09.821 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.595 seconds | 20260506 14:22:09.821 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.40% / 0.01% / 61.58% | 20260506 14:22:09.821 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.6075% / 1.98682e-05% / 7.15256e-05% | 20260506 14:22:09.821 | INFO | Baz IO overall: | 20260506 14:22:09.821 | INFO | BytesWritten : 1049174177082 | 20260506 14:22:09.821 | INFO | NumEvents : 507670858360 | 20260506 14:22:11.246 | INFO | Analyzing chunk frames = [2035712,2036224) | 20260506 14:22:11.246 | INFO | MemInfo: { | 20260506 14:22:11.246 | INFO | "gpu" : " 17 GB", | 20260506 14:22:11.246 | INFO | "huge" : "", | 20260506 14:22:11.246 | INFO | "huge_shared" : " 27 GB", | 20260506 14:22:11.246 | INFO | "malloc" : " 14 GB", | 20260506 14:22:11.246 | INFO | "pinned" : " 29 GB" | 20260506 14:22:11.246 | INFO | } | 20260506 14:22:11.246 | INFO | Perf report for the last 5.13793 seconds: | 20260506 14:22:11.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:22:11.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:11.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:11.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:11.247 | INFO | ANALYSIS : 63.0%, 2.9, 95.4ms, 0 | 20260506 14:22:11.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:11.247 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260506 14:22:11.247 | INFO | PRE_HQ : 61.1%, 1.9, 59.8ms, 0 | 20260506 14:22:11.247 | INFO | BAZWRITER : 69.2%, 4.8, 172.0ms, 0 | 20260506 14:22:11.247 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 14:22:11.247 | INFO | Uploaded 36GiB to GPU in 2.06451s (17.4375GiB/s) | 20260506 14:22:11.247 | INFO | Downloaded 38GiB to GPU in 2.22781s (17.0571GiB/s) | 20260506 14:22:16.385 | INFO | Analyzing chunk frames = [2036224,2036736) | 20260506 14:22:16.385 | INFO | MemInfo: { | 20260506 14:22:16.385 | INFO | "gpu" : " 17 GB", | 20260506 14:22:16.385 | INFO | "huge" : "", | 20260506 14:22:16.385 | INFO | "huge_shared" : " 27 GB", | 20260506 14:22:16.385 | INFO | "malloc" : " 14 GB", | 20260506 14:22:16.385 | INFO | "pinned" : " 29 GB" | 20260506 14:22:16.385 | INFO | } | 20260506 14:22:16.385 | INFO | Perf report for the last 5.13877 seconds: | 20260506 14:22:16.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:22:16.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:16.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:16.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:16.385 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 14:22:16.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:16.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:16.385 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:22:16.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:16.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:16.386 | INFO | Uploaded 36GiB to GPU in 1.99321s (18.0614GiB/s) | 20260506 14:22:16.386 | INFO | Downloaded 34GiB to GPU in 1.94319s (17.497GiB/s) | 20260506 14:22:21.525 | INFO | Analyzing chunk frames = [2036736,2037248) | 20260506 14:22:21.525 | INFO | MemInfo: { | 20260506 14:22:21.525 | INFO | "gpu" : " 17 GB", | 20260506 14:22:21.525 | INFO | "huge" : "", | 20260506 14:22:21.525 | INFO | "huge_shared" : " 27 GB", | 20260506 14:22:21.525 | INFO | "malloc" : " 14 GB", | 20260506 14:22:21.525 | INFO | "pinned" : " 29 GB" | 20260506 14:22:21.525 | INFO | } | 20260506 14:22:21.525 | INFO | Perf report for the last 5.13959 seconds: | 20260506 14:22:21.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:22:21.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:21.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:21.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:21.525 | INFO | ANALYSIS : 61.3%, 3.0, 94.8ms, 0 | 20260506 14:22:21.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:21.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:21.525 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:22:21.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:21.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:21.525 | INFO | Uploaded 36GiB to GPU in 2.00265s (17.9761GiB/s) | 20260506 14:22:21.525 | INFO | Downloaded 34GiB to GPU in 1.93071s (17.6101GiB/s) | 20260506 14:22:26.665 | INFO | Analyzing chunk frames = [2037248,2037760) | 20260506 14:22:26.665 | INFO | MemInfo: { | 20260506 14:22:26.665 | INFO | "gpu" : " 17 GB", | 20260506 14:22:26.665 | INFO | "huge" : "", | 20260506 14:22:26.665 | INFO | "huge_shared" : " 27 GB", | 20260506 14:22:26.665 | INFO | "malloc" : " 14 GB", | 20260506 14:22:26.665 | INFO | "pinned" : " 29 GB" | 20260506 14:22:26.665 | INFO | } | 20260506 14:22:26.665 | INFO | Perf report for the last 5.13993 seconds: | 20260506 14:22:26.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:22:26.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:26.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:26.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:26.665 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 14:22:26.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:26.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:26.665 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:22:26.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:26.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:26.665 | INFO | Uploaded 36GiB to GPU in 1.99018s (18.0889GiB/s) | 20260506 14:22:26.665 | INFO | Downloaded 34GiB to GPU in 1.91138s (17.7882GiB/s) | 20260506 14:22:31.804 | INFO | Analyzing chunk frames = [2037760,2038272) | 20260506 14:22:31.804 | INFO | MemInfo: { | 20260506 14:22:31.804 | INFO | "gpu" : " 17 GB", | 20260506 14:22:31.804 | INFO | "huge" : "", | 20260506 14:22:31.804 | INFO | "huge_shared" : " 27 GB", | 20260506 14:22:31.804 | INFO | "malloc" : " 14 GB", | 20260506 14:22:31.804 | INFO | "pinned" : " 29 GB" | 20260506 14:22:31.804 | INFO | } | 20260506 14:22:31.804 | INFO | Perf report for the last 5.13925 seconds: | 20260506 14:22:31.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:22:31.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:31.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:31.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:31.804 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 14:22:31.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:31.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:31.804 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:22:31.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:31.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:31.805 | INFO | Uploaded 36GiB to GPU in 1.99759s (18.0218GiB/s) | 20260506 14:22:31.805 | INFO | Downloaded 34GiB to GPU in 1.93336s (17.5859GiB/s) | 20260506 14:22:36.944 | INFO | Analyzing chunk frames = [2038272,2038784) | 20260506 14:22:36.944 | INFO | MemInfo: { | 20260506 14:22:36.944 | INFO | "gpu" : " 17 GB", | 20260506 14:22:36.944 | INFO | "huge" : "", | 20260506 14:22:36.944 | INFO | "huge_shared" : " 27 GB", | 20260506 14:22:36.944 | INFO | "malloc" : " 14 GB", | 20260506 14:22:36.944 | INFO | "pinned" : " 29 GB" | 20260506 14:22:36.944 | INFO | } | 20260506 14:22:36.945 | INFO | Perf report for the last 5.1404 seconds: | 20260506 14:22:36.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:22:36.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:36.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:36.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:36.945 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 14:22:36.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:36.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:36.945 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 14:22:36.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:36.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:36.945 | INFO | Uploaded 36GiB to GPU in 2.0476s (17.5816GiB/s) | 20260506 14:22:36.945 | INFO | Downloaded 34GiB to GPU in 1.91637s (17.7419GiB/s) | 20260506 14:22:42.084 | INFO | Analyzing chunk frames = [2038784,2039296) | 20260506 14:22:42.084 | INFO | MemInfo: { | 20260506 14:22:42.084 | INFO | "gpu" : " 17 GB", | 20260506 14:22:42.084 | INFO | "huge" : "", | 20260506 14:22:42.084 | INFO | "huge_shared" : " 27 GB", | 20260506 14:22:42.084 | INFO | "malloc" : " 14 GB", | 20260506 14:22:42.084 | INFO | "pinned" : " 29 GB" | 20260506 14:22:42.084 | INFO | } | 20260506 14:22:42.084 | INFO | Perf report for the last 5.13939 seconds: | 20260506 14:22:42.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:22:42.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:42.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:42.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:42.084 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 14:22:42.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:42.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:42.084 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 14:22:42.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:42.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:42.084 | INFO | Uploaded 36GiB to GPU in 2.0645s (17.4377GiB/s) | 20260506 14:22:42.084 | INFO | Downloaded 34GiB to GPU in 1.99889s (17.0095GiB/s) | 20260506 14:22:47.222 | INFO | Analyzing chunk frames = [2039296,2039808) | 20260506 14:22:47.222 | INFO | MemInfo: { | 20260506 14:22:47.222 | INFO | "gpu" : " 17 GB", | 20260506 14:22:47.222 | INFO | "huge" : "", | 20260506 14:22:47.222 | INFO | "huge_shared" : " 27 GB", | 20260506 14:22:47.222 | INFO | "malloc" : " 14 GB", | 20260506 14:22:47.222 | INFO | "pinned" : " 29 GB" | 20260506 14:22:47.222 | INFO | } | 20260506 14:22:47.222 | INFO | Perf report for the last 5.13802 seconds: | 20260506 14:22:47.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:22:47.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:47.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:47.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:47.222 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:22:47.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:47.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:47.222 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:22:47.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:47.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:47.223 | INFO | Uploaded 36GiB to GPU in 2.00097s (17.9912GiB/s) | 20260506 14:22:47.223 | INFO | Downloaded 34GiB to GPU in 1.98175s (17.1566GiB/s) | 20260506 14:22:47.223 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:22:47.223 | INFO | Total mem usage: 17.4 GiB | 20260506 14:22:47.223 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:22:47.223 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -13.8%) | 20260506 14:22:47.223 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:22:47.223 | INFO | High Water Breakdown: | 20260506 14:22:47.223 | INFO | 60.8% : 9.3 GiB ( 85.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:22:47.223 | INFO | 39.1% : 5.97 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:22:47.223 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:22:47.223 | INFO | ------------------------------------------ | 20260506 14:22:47.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:22:47.223 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:22:47.223 | INFO | Total mem usage: 26.5 GiB | 20260506 14:22:47.223 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:22:47.223 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:22:47.223 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:22:47.223 | INFO | High Water Breakdown: | 20260506 14:22:47.223 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:22:47.223 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:22:47.223 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:22:47.223 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:22:47.223 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:22:47.223 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:22:47.223 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:22:47.223 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:22:47.223 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:22:47.223 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:22:47.223 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:22:47.223 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:22:47.223 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:22:47.223 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:22:47.223 | INFO | ------------------------------------------ | 20260506 14:22:47.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:22:47.223 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:22:47.223 | INFO | Total mem usage: 25.6 GiB | 20260506 14:22:47.223 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:22:47.223 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:22:47.223 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:22:47.223 | INFO | High Water Breakdown: | 20260506 14:22:47.223 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:22:47.223 | INFO | ------------------------------------------ | 20260506 14:22:47.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:22:47.223 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:22:47.223 | INFO | Total mem usage: 15.7 GiB | 20260506 14:22:47.223 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:22:47.223 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:22:47.223 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 14:22:47.223 | INFO | High Water Breakdown: | 20260506 14:22:47.223 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:22:47.223 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:22:47.223 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:22:47.223 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:22:47.223 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:22:47.223 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:22:47.223 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:22:47.223 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:22:47.223 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:22:47.223 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:22:47.223 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:22:47.223 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:22:47.223 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:22:47.223 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:22:47.223 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:22:47.223 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:22:47.223 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:22:47.223 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:22:47.223 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:22:47.223 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:22:47.223 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:22:47.224 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:22:47.224 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:22:47.224 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:22:47.224 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:22:47.224 | INFO | ------------------------------------------ | 20260506 14:22:47.224 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:22:47.224 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:22:47.224 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:22:47.224 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:22:47.224 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.61e+01, 1.37e+01 / 2.38e+00 | 20260506 14:22:47.224 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.03e-01, 1.07e-02 / 7.14e-03 | 20260506 14:22:47.224 | INFO | PerfReport: 11.55%, Baseline -- 3.65e+00 / 4.42e+00, 5.66e-01 / 3.27e-01 | 20260506 14:22:47.224 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.47e+00, 1.88e-01 / 1.33e-01 | 20260506 14:22:47.224 | INFO | PerfReport: 15.83%, DME -- 4.00e+01 / 4.44e+01, 4.02e+00 / 2.69e+00 | 20260506 14:22:47.224 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.51e+00 / 1.32e+00 | 20260506 14:22:47.224 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.35e+00 / 2.93e+00, 4.09e-01 / 2.54e-01 | 20260506 14:22:47.224 | INFO | PerfReport: 3.61%, Metrics -- 1.14e+00 / 1.55e+00, 2.95e-01 / 1.57e-01 | 20260506 14:22:47.224 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.99e-01 / 4.55e-01, 6.29e-02 / 3.31e-02 | 20260506 14:22:47.224 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:22:47.224 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:22:47.224 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:22:47.224 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 14:22:47.224 | INFO | PerfReport: 21.55%, Upload -- 2.11e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260506 14:22:47.224 | INFO | PerfReport: 21.13%, Download -- 2.07e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 14:22:47.224 | INFO | PerfReport: 32.28%, Compute -- 3.16e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 14:22:47.224 | INFO | PerfReport: 1.57%, LockUpload -- 1.54e+00 / 0.00e+00, 6.00e+00 / 0.00e+00 | 20260506 14:22:47.224 | INFO | PerfReport: 1.02%, LockDownload -- 9.97e-01 / 0.00e+00, 2.56e+00 / 0.00e+00 | 20260506 14:22:47.224 | INFO | PerfReport: 22.43%, LockCompute -- 2.19e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 14:22:47.224 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260506 14:22:47.333 | INFO | Starting a round of Baz IO | 20260506 14:22:50.486 | INFO | Populating RTMetrics report | 20260506 14:22:50.487 | INFO | RTMetrics report populated | 20260506 14:22:50.487 | INFO | Opening tmp RTMetrics json file | 20260506 14:22:50.888 | INFO | Serializing RTMetrics to json | 20260506 14:22:50.941 | INFO | Serialization done, swapping tmp file now... | 20260506 14:22:50.943 | INFO | RTMetrics done | 20260506 14:22:51.337 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 4.004 seconds | 20260506 14:22:51.337 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.39% / 0.01% / 61.60% | 20260506 14:22:51.337 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.59588% / 4.37101e-05% / 7.94729e-05% | 20260506 14:22:51.337 | INFO | Baz IO overall: | 20260506 14:22:51.338 | INFO | BytesWritten : 1051443157632 | 20260506 14:22:51.338 | INFO | NumEvents : 508770129406 | 20260506 14:22:52.361 | INFO | Analyzing chunk frames = [2039808,2040320) | 20260506 14:22:52.361 | INFO | MemInfo: { | 20260506 14:22:52.361 | INFO | "gpu" : " 17 GB", | 20260506 14:22:52.361 | INFO | "huge" : "", | 20260506 14:22:52.361 | INFO | "huge_shared" : " 27 GB", | 20260506 14:22:52.361 | INFO | "malloc" : " 14 GB", | 20260506 14:22:52.361 | INFO | "pinned" : " 29 GB" | 20260506 14:22:52.361 | INFO | } | 20260506 14:22:52.361 | INFO | Perf report for the last 5.1394 seconds: | 20260506 14:22:52.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:22:52.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:52.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:52.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:52.361 | INFO | ANALYSIS : 62.9%, 2.9, 95.2ms, 0 | 20260506 14:22:52.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:52.362 | INFO | RT_METRICS : 52.7%, 1.0, 27.6ms, 0 | 20260506 14:22:52.362 | INFO | PRE_HQ : 61.1%, 1.9, 59.6ms, 0 | 20260506 14:22:52.362 | INFO | BAZWRITER : 77.3%, 4.8, 192.3ms, 0 | 20260506 14:22:52.362 | INFO | BAZ_SAMPLING : 7.0%, 1.0, 3.6ms, 0 | 20260506 14:22:52.362 | INFO | Uploaded 36GiB to GPU in 2.0768s (17.3343GiB/s) | 20260506 14:22:52.362 | INFO | Downloaded 38GiB to GPU in 2.22834s (17.0531GiB/s) | 20260506 14:22:57.501 | INFO | Analyzing chunk frames = [2040320,2040832) | 20260506 14:22:57.501 | INFO | MemInfo: { | 20260506 14:22:57.501 | INFO | "gpu" : " 17 GB", | 20260506 14:22:57.501 | INFO | "huge" : "", | 20260506 14:22:57.501 | INFO | "huge_shared" : " 27 GB", | 20260506 14:22:57.501 | INFO | "malloc" : " 14 GB", | 20260506 14:22:57.501 | INFO | "pinned" : " 29 GB" | 20260506 14:22:57.501 | INFO | } | 20260506 14:22:57.501 | INFO | Perf report for the last 5.13939 seconds: | 20260506 14:22:57.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:22:57.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:57.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:57.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:57.501 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 14:22:57.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:57.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:57.501 | INFO | PRE_HQ : 28.4%, 1.0, 14.9ms, 0 | 20260506 14:22:57.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:57.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:22:57.501 | INFO | Uploaded 36GiB to GPU in 2.02236s (17.8009GiB/s) | 20260506 14:22:57.501 | INFO | Downloaded 34GiB to GPU in 1.94038s (17.5224GiB/s) | 20260506 14:23:02.639 | INFO | Analyzing chunk frames = [2040832,2041344) | 20260506 14:23:02.639 | INFO | MemInfo: { | 20260506 14:23:02.639 | INFO | "gpu" : " 17 GB", | 20260506 14:23:02.639 | INFO | "huge" : "", | 20260506 14:23:02.639 | INFO | "huge_shared" : " 27 GB", | 20260506 14:23:02.639 | INFO | "malloc" : " 14 GB", | 20260506 14:23:02.639 | INFO | "pinned" : " 29 GB" | 20260506 14:23:02.640 | INFO | } | 20260506 14:23:02.640 | INFO | Perf report for the last 5.13894 seconds: | 20260506 14:23:02.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:23:02.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:02.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:02.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:02.640 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:23:02.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:02.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:02.640 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 14:23:02.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:02.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:02.640 | INFO | Uploaded 36GiB to GPU in 1.99739s (18.0236GiB/s) | 20260506 14:23:02.640 | INFO | Downloaded 34GiB to GPU in 1.93022s (17.6145GiB/s) | 20260506 14:23:07.780 | INFO | Analyzing chunk frames = [2041344,2041856) | 20260506 14:23:07.780 | INFO | MemInfo: { | 20260506 14:23:07.780 | INFO | "gpu" : " 17 GB", | 20260506 14:23:07.780 | INFO | "huge" : "", | 20260506 14:23:07.780 | INFO | "huge_shared" : " 27 GB", | 20260506 14:23:07.780 | INFO | "malloc" : " 14 GB", | 20260506 14:23:07.780 | INFO | "pinned" : " 29 GB" | 20260506 14:23:07.780 | INFO | } | 20260506 14:23:07.780 | INFO | Perf report for the last 5.14035 seconds: | 20260506 14:23:07.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:23:07.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:07.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:07.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:07.780 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 14:23:07.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:07.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:07.780 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 14:23:07.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:07.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:07.780 | INFO | Uploaded 36GiB to GPU in 2.00087s (17.9921GiB/s) | 20260506 14:23:07.780 | INFO | Downloaded 34GiB to GPU in 1.91965s (17.7116GiB/s) | 20260506 14:23:12.919 | INFO | Analyzing chunk frames = [2041856,2042368) | 20260506 14:23:12.919 | INFO | MemInfo: { | 20260506 14:23:12.919 | INFO | "gpu" : " 17 GB", | 20260506 14:23:12.919 | INFO | "huge" : "", | 20260506 14:23:12.919 | INFO | "huge_shared" : " 27 GB", | 20260506 14:23:12.919 | INFO | "malloc" : " 14 GB", | 20260506 14:23:12.919 | INFO | "pinned" : " 29 GB" | 20260506 14:23:12.919 | INFO | } | 20260506 14:23:12.919 | INFO | Perf report for the last 5.13925 seconds: | 20260506 14:23:12.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:23:12.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:12.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:12.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:12.919 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 14:23:12.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:12.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:12.920 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 14:23:12.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:12.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:12.920 | INFO | Uploaded 36GiB to GPU in 1.96124s (18.3557GiB/s) | 20260506 14:23:12.920 | INFO | Downloaded 34GiB to GPU in 1.92635s (17.65GiB/s) | 20260506 14:23:18.058 | INFO | Analyzing chunk frames = [2042368,2042880) | 20260506 14:23:18.058 | INFO | MemInfo: { | 20260506 14:23:18.058 | INFO | "gpu" : " 17 GB", | 20260506 14:23:18.058 | INFO | "huge" : "", | 20260506 14:23:18.058 | INFO | "huge_shared" : " 27 GB", | 20260506 14:23:18.058 | INFO | "malloc" : " 14 GB", | 20260506 14:23:18.058 | INFO | "pinned" : " 29 GB" | 20260506 14:23:18.058 | INFO | } | 20260506 14:23:18.058 | INFO | Perf report for the last 5.13913 seconds: | 20260506 14:23:18.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:23:18.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:18.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:18.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:18.059 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 14:23:18.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:18.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:18.059 | INFO | PRE_HQ : 31.7%, 1.0, 16.6ms, 0 | 20260506 14:23:18.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:18.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:18.059 | INFO | Uploaded 36GiB to GPU in 2.03276s (17.7099GiB/s) | 20260506 14:23:18.059 | INFO | Downloaded 34GiB to GPU in 1.89676s (17.9253GiB/s) | 20260506 14:23:23.199 | INFO | Analyzing chunk frames = [2042880,2043392) | 20260506 14:23:23.199 | INFO | MemInfo: { | 20260506 14:23:23.199 | INFO | "gpu" : " 17 GB", | 20260506 14:23:23.199 | INFO | "huge" : "", | 20260506 14:23:23.199 | INFO | "huge_shared" : " 27 GB", | 20260506 14:23:23.199 | INFO | "malloc" : " 14 GB", | 20260506 14:23:23.199 | INFO | "pinned" : " 29 GB" | 20260506 14:23:23.199 | INFO | } | 20260506 14:23:23.199 | INFO | Perf report for the last 5.14033 seconds: | 20260506 14:23:23.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:23:23.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:23.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:23.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:23.199 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:23:23.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:23.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:23.199 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 14:23:23.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:23.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:23.199 | INFO | Uploaded 36GiB to GPU in 2.00005s (17.9996GiB/s) | 20260506 14:23:23.199 | INFO | Downloaded 34GiB to GPU in 2.00599s (16.9492GiB/s) | 20260506 14:23:28.337 | INFO | Analyzing chunk frames = [2043392,2043904) | 20260506 14:23:28.337 | INFO | MemInfo: { | 20260506 14:23:28.337 | INFO | "gpu" : " 17 GB", | 20260506 14:23:28.337 | INFO | "huge" : "", | 20260506 14:23:28.337 | INFO | "huge_shared" : " 27 GB", | 20260506 14:23:28.337 | INFO | "malloc" : " 14 GB", | 20260506 14:23:28.337 | INFO | "pinned" : " 29 GB" | 20260506 14:23:28.337 | INFO | } | 20260506 14:23:28.337 | INFO | Perf report for the last 5.13827 seconds: | 20260506 14:23:28.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:23:28.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:28.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:28.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:28.337 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:23:28.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:28.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:28.337 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:23:28.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:28.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:28.337 | INFO | Uploaded 36GiB to GPU in 2.02519s (17.7761GiB/s) | 20260506 14:23:28.337 | INFO | Downloaded 34GiB to GPU in 1.97422s (17.222GiB/s) | 20260506 14:23:28.451 | INFO | Starting a round of Baz IO | 20260506 14:23:31.601 | INFO | Populating RTMetrics report | 20260506 14:23:31.602 | INFO | RTMetrics report populated | 20260506 14:23:31.602 | INFO | Opening tmp RTMetrics json file | 20260506 14:23:31.652 | INFO | Serializing RTMetrics to json | 20260506 14:23:31.729 | INFO | Serialization done, swapping tmp file now... | 20260506 14:23:31.763 | INFO | RTMetrics done | 20260506 14:23:32.160 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.709 seconds | 20260506 14:23:32.160 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.37% / 0.01% / 61.61% | 20260506 14:23:32.160 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.59519% / 1.98682e-05% / 5.5631e-05% | 20260506 14:23:32.160 | INFO | Baz IO overall: | 20260506 14:23:32.161 | INFO | BytesWritten : 1053713380104 | 20260506 14:23:32.161 | INFO | NumEvents : 509870712916 | 20260506 14:23:33.476 | INFO | Analyzing chunk frames = [2043904,2044416) | 20260506 14:23:33.476 | INFO | MemInfo: { | 20260506 14:23:33.476 | INFO | "gpu" : " 17 GB", | 20260506 14:23:33.476 | INFO | "huge" : "", | 20260506 14:23:33.476 | INFO | "huge_shared" : " 27 GB", | 20260506 14:23:33.476 | INFO | "malloc" : " 14 GB", | 20260506 14:23:33.476 | INFO | "pinned" : " 29 GB" | 20260506 14:23:33.476 | INFO | } | 20260506 14:23:33.476 | INFO | Perf report for the last 5.13868 seconds: | 20260506 14:23:33.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:23:33.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:33.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:33.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:33.476 | INFO | ANALYSIS : 63.0%, 2.9, 94.8ms, 0 | 20260506 14:23:33.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:33.476 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260506 14:23:33.476 | INFO | PRE_HQ : 61.0%, 1.9, 59.8ms, 0 | 20260506 14:23:33.476 | INFO | BAZWRITER : 71.6%, 4.8, 177.7ms, 0 | 20260506 14:23:33.476 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 14:23:33.476 | INFO | Uploaded 36GiB to GPU in 2.08766s (17.2442GiB/s) | 20260506 14:23:33.476 | INFO | Downloaded 38GiB to GPU in 2.22777s (17.0574GiB/s) | 20260506 14:23:38.615 | INFO | Analyzing chunk frames = [2044416,2044928) | 20260506 14:23:38.615 | INFO | MemInfo: { | 20260506 14:23:38.615 | INFO | "gpu" : " 17 GB", | 20260506 14:23:38.615 | INFO | "huge" : "", | 20260506 14:23:38.615 | INFO | "huge_shared" : " 27 GB", | 20260506 14:23:38.615 | INFO | "malloc" : " 14 GB", | 20260506 14:23:38.615 | INFO | "pinned" : " 29 GB" | 20260506 14:23:38.615 | INFO | } | 20260506 14:23:38.615 | INFO | Perf report for the last 5.13962 seconds: | 20260506 14:23:38.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:23:38.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:38.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:38.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:38.616 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:23:38.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:38.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:38.616 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:23:38.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:38.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:38.616 | INFO | Uploaded 36GiB to GPU in 2.01605s (17.8567GiB/s) | 20260506 14:23:38.616 | INFO | Downloaded 34GiB to GPU in 1.94844s (17.4499GiB/s) | 20260506 14:23:43.756 | INFO | Analyzing chunk frames = [2044928,2045440) | 20260506 14:23:43.756 | INFO | MemInfo: { | 20260506 14:23:43.756 | INFO | "gpu" : " 17 GB", | 20260506 14:23:43.756 | INFO | "huge" : "", | 20260506 14:23:43.756 | INFO | "huge_shared" : " 27 GB", | 20260506 14:23:43.756 | INFO | "malloc" : " 14 GB", | 20260506 14:23:43.756 | INFO | "pinned" : " 29 GB" | 20260506 14:23:43.756 | INFO | } | 20260506 14:23:43.756 | INFO | Perf report for the last 5.14092 seconds: | 20260506 14:23:43.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:23:43.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:43.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:43.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:43.756 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 14:23:43.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:43.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:43.757 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:23:43.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:43.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:43.757 | INFO | Uploaded 36GiB to GPU in 2.00483s (17.9566GiB/s) | 20260506 14:23:43.757 | INFO | Downloaded 34GiB to GPU in 1.93246s (17.5942GiB/s) | 20260506 14:23:48.894 | INFO | Analyzing chunk frames = [2045440,2045952) | 20260506 14:23:48.894 | INFO | MemInfo: { | 20260506 14:23:48.894 | INFO | "gpu" : " 17 GB", | 20260506 14:23:48.894 | INFO | "huge" : "", | 20260506 14:23:48.894 | INFO | "huge_shared" : " 27 GB", | 20260506 14:23:48.894 | INFO | "malloc" : " 14 GB", | 20260506 14:23:48.894 | INFO | "pinned" : " 29 GB" | 20260506 14:23:48.894 | INFO | } | 20260506 14:23:48.894 | INFO | Perf report for the last 5.13798 seconds: | 20260506 14:23:48.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:23:48.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:48.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:48.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:48.894 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 14:23:48.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:48.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:48.895 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:23:48.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:48.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:48.895 | INFO | Uploaded 36GiB to GPU in 1.99737s (18.0237GiB/s) | 20260506 14:23:48.895 | INFO | Downloaded 34GiB to GPU in 1.92554s (17.6574GiB/s) | 20260506 14:23:54.034 | INFO | Analyzing chunk frames = [2045952,2046464) | 20260506 14:23:54.034 | INFO | MemInfo: { | 20260506 14:23:54.034 | INFO | "gpu" : " 17 GB", | 20260506 14:23:54.034 | INFO | "huge" : "", | 20260506 14:23:54.034 | INFO | "huge_shared" : " 27 GB", | 20260506 14:23:54.034 | INFO | "malloc" : " 14 GB", | 20260506 14:23:54.034 | INFO | "pinned" : " 29 GB" | 20260506 14:23:54.034 | INFO | } | 20260506 14:23:54.034 | INFO | Perf report for the last 5.13961 seconds: | 20260506 14:23:54.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:23:54.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:54.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:54.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:54.034 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:23:54.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:54.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:54.034 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:23:54.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:54.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:54.034 | INFO | Uploaded 36GiB to GPU in 1.99096s (18.0817GiB/s) | 20260506 14:23:54.034 | INFO | Downloaded 34GiB to GPU in 1.92413s (17.6704GiB/s) | 20260506 14:23:59.173 | INFO | Analyzing chunk frames = [2046464,2046976) | 20260506 14:23:59.173 | INFO | MemInfo: { | 20260506 14:23:59.173 | INFO | "gpu" : " 17 GB", | 20260506 14:23:59.173 | INFO | "huge" : "", | 20260506 14:23:59.173 | INFO | "huge_shared" : " 27 GB", | 20260506 14:23:59.173 | INFO | "malloc" : " 14 GB", | 20260506 14:23:59.173 | INFO | "pinned" : " 29 GB" | 20260506 14:23:59.173 | INFO | } | 20260506 14:23:59.173 | INFO | Perf report for the last 5.13885 seconds: | 20260506 14:23:59.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:23:59.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:59.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:59.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:59.173 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:23:59.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:59.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:59.173 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260506 14:23:59.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:59.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:23:59.173 | INFO | Uploaded 36GiB to GPU in 2.08339s (17.2796GiB/s) | 20260506 14:23:59.173 | INFO | Downloaded 34GiB to GPU in 1.9699s (17.2598GiB/s) | 20260506 14:24:04.313 | INFO | Analyzing chunk frames = [2046976,2047488) | 20260506 14:24:04.313 | INFO | MemInfo: { | 20260506 14:24:04.313 | INFO | "gpu" : " 17 GB", | 20260506 14:24:04.313 | INFO | "huge" : "", | 20260506 14:24:04.313 | INFO | "huge_shared" : " 27 GB", | 20260506 14:24:04.313 | INFO | "malloc" : " 14 GB", | 20260506 14:24:04.313 | INFO | "pinned" : " 29 GB" | 20260506 14:24:04.313 | INFO | } | 20260506 14:24:04.313 | INFO | Perf report for the last 5.13988 seconds: | 20260506 14:24:04.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:24:04.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:04.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:04.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:04.313 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:24:04.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:04.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:04.313 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 14:24:04.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:04.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:04.313 | INFO | Uploaded 36GiB to GPU in 2.0071s (17.9363GiB/s) | 20260506 14:24:04.313 | INFO | Downloaded 34GiB to GPU in 2.01868s (16.8427GiB/s) | 20260506 14:24:09.451 | INFO | Analyzing chunk frames = [2047488,2048000) | 20260506 14:24:09.451 | INFO | MemInfo: { | 20260506 14:24:09.451 | INFO | "gpu" : " 17 GB", | 20260506 14:24:09.451 | INFO | "huge" : "", | 20260506 14:24:09.451 | INFO | "huge_shared" : " 27 GB", | 20260506 14:24:09.451 | INFO | "malloc" : " 14 GB", | 20260506 14:24:09.451 | INFO | "pinned" : " 29 GB" | 20260506 14:24:09.451 | INFO | } | 20260506 14:24:09.451 | INFO | Perf report for the last 5.13856 seconds: | 20260506 14:24:09.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:24:09.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:09.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:09.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:09.451 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 14:24:09.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:09.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:09.451 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 14:24:09.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:09.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:09.452 | INFO | Uploaded 36GiB to GPU in 2.05398s (17.527GiB/s) | 20260506 14:24:09.452 | INFO | Downloaded 34GiB to GPU in 1.97255s (17.2365GiB/s) | 20260506 14:24:09.452 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:24:09.452 | INFO | Total mem usage: 17.4 GiB | 20260506 14:24:09.452 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:24:09.452 | INFO | Recent High Water Marks Sum: 16.8 GiB (Savings: -3.36%) | 20260506 14:24:09.452 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:24:09.452 | INFO | High Water Breakdown: | 20260506 14:24:09.452 | INFO | 62.3% : 10.5 GiB ( 96.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:24:09.452 | INFO | 37.6% : 6.33 GiB ( 94.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:24:09.452 | INFO | 0.118% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:24:09.452 | INFO | ------------------------------------------ | 20260506 14:24:09.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:24:09.452 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:24:09.452 | INFO | Total mem usage: 26.5 GiB | 20260506 14:24:09.452 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:24:09.452 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:24:09.452 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:24:09.452 | INFO | High Water Breakdown: | 20260506 14:24:09.452 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:24:09.452 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:24:09.452 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:24:09.452 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:24:09.452 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:24:09.452 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:24:09.452 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:24:09.452 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:24:09.452 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:24:09.452 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:24:09.452 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:24:09.452 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:24:09.452 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:24:09.452 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:24:09.452 | INFO | ------------------------------------------ | 20260506 14:24:09.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:24:09.452 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:24:09.452 | INFO | Total mem usage: 25.6 GiB | 20260506 14:24:09.452 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:24:09.452 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:24:09.452 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:24:09.452 | INFO | High Water Breakdown: | 20260506 14:24:09.452 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:24:09.452 | INFO | ------------------------------------------ | 20260506 14:24:09.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:24:09.453 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:24:09.453 | INFO | Total mem usage: 15.7 GiB | 20260506 14:24:09.453 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:24:09.453 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:24:09.453 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 14:24:09.453 | INFO | High Water Breakdown: | 20260506 14:24:09.453 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:24:09.453 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:24:09.453 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:24:09.453 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:24:09.453 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:24:09.453 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:24:09.453 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:24:09.453 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:24:09.453 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:24:09.453 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:24:09.453 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:24:09.453 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:24:09.453 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:24:09.453 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:24:09.453 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:24:09.453 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:24:09.453 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:24:09.453 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:24:09.453 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:24:09.453 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:24:09.453 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:24:09.453 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:24:09.453 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:24:09.453 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:24:09.453 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:24:09.453 | INFO | ------------------------------------------ | 20260506 14:24:09.453 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:24:09.453 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:24:09.453 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:24:09.453 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:24:09.453 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.57e+01, 1.36e+01 / 2.35e+00 | 20260506 14:24:09.453 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.01e-01, 1.19e-02 / 6.23e-03 | 20260506 14:24:09.453 | INFO | PerfReport: 11.58%, Baseline -- 3.65e+00 / 4.49e+00, 5.73e-01 / 3.16e-01 | 20260506 14:24:09.453 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.47e+00, 1.89e-01 / 1.17e-01 | 20260506 14:24:09.453 | INFO | PerfReport: 15.89%, DME -- 4.00e+01 / 4.45e+01, 4.05e+00 / 2.87e+00 | 20260506 14:24:09.453 | INFO | PerfReport: 56.61%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.38e+00 / 1.94e+00 | 20260506 14:24:09.453 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.36e+00 / 2.97e+00, 4.14e-01 / 2.54e-01 | 20260506 14:24:09.453 | INFO | PerfReport: 3.62%, Metrics -- 1.14e+00 / 1.50e+00, 2.92e-01 / 1.91e-01 | 20260506 14:24:09.453 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.98e-01 / 4.53e-01, 6.22e-02 / 3.24e-02 | 20260506 14:24:09.453 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:24:09.453 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:24:09.453 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:24:09.453 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 14:24:09.453 | INFO | PerfReport: 21.54%, Upload -- 2.11e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 14:24:09.453 | INFO | PerfReport: 21.13%, Download -- 2.07e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 14:24:09.453 | INFO | PerfReport: 32.22%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 14:24:09.453 | INFO | PerfReport: 1.60%, LockUpload -- 1.56e+00 / 0.00e+00, 6.25e+00 / 0.00e+00 | 20260506 14:24:09.453 | INFO | PerfReport: 1.11%, LockDownload -- 1.08e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260506 14:24:09.453 | INFO | PerfReport: 22.38%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:24:09.453 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260506 14:24:09.578 | INFO | Starting a round of Baz IO | 20260506 14:24:12.725 | INFO | Populating RTMetrics report | 20260506 14:24:12.725 | INFO | RTMetrics report populated | 20260506 14:24:12.725 | INFO | Opening tmp RTMetrics json file | 20260506 14:24:12.930 | INFO | Serializing RTMetrics to json | 20260506 14:24:13.019 | INFO | Serialization done, swapping tmp file now... | 20260506 14:24:13.030 | INFO | RTMetrics done | 20260506 14:24:13.135 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.557 seconds | 20260506 14:24:13.135 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.36% / 0.01% / 61.63% | 20260506 14:24:13.135 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.58208% / 1.19209e-05% / 5.16574e-05% | 20260506 14:24:13.135 | INFO | Baz IO overall: | 20260506 14:24:13.135 | INFO | BytesWritten : 1055981611199 | 20260506 14:24:13.135 | INFO | NumEvents : 510969696897 | 20260506 14:24:14.590 | INFO | Analyzing chunk frames = [2048000,2048512) | 20260506 14:24:14.590 | INFO | MemInfo: { | 20260506 14:24:14.590 | INFO | "gpu" : " 17 GB", | 20260506 14:24:14.590 | INFO | "huge" : "", | 20260506 14:24:14.590 | INFO | "huge_shared" : " 27 GB", | 20260506 14:24:14.590 | INFO | "malloc" : " 14 GB", | 20260506 14:24:14.590 | INFO | "pinned" : " 29 GB" | 20260506 14:24:14.590 | INFO | } | 20260506 14:24:14.591 | INFO | Perf report for the last 5.13917 seconds: | 20260506 14:24:14.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:24:14.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:14.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:14.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:14.591 | INFO | ANALYSIS : 63.1%, 2.9, 95.5ms, 0 | 20260506 14:24:14.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:14.591 | INFO | RT_METRICS : 49.8%, 1.0, 26.1ms, 0 | 20260506 14:24:14.591 | INFO | PRE_HQ : 60.9%, 1.9, 59.6ms, 0 | 20260506 14:24:14.591 | INFO | BAZWRITER : 68.6%, 4.8, 169.5ms, 0 | 20260506 14:24:14.591 | INFO | BAZ_SAMPLING : 5.5%, 1.0, 2.9ms, 0 | 20260506 14:24:14.591 | INFO | Uploaded 36GiB to GPU in 2.08619s (17.2564GiB/s) | 20260506 14:24:14.591 | INFO | Downloaded 38GiB to GPU in 2.23309s (17.0168GiB/s) | 20260506 14:24:19.730 | INFO | Analyzing chunk frames = [2048512,2049024) | 20260506 14:24:19.730 | INFO | MemInfo: { | 20260506 14:24:19.730 | INFO | "gpu" : " 17 GB", | 20260506 14:24:19.730 | INFO | "huge" : "", | 20260506 14:24:19.730 | INFO | "huge_shared" : " 27 GB", | 20260506 14:24:19.730 | INFO | "malloc" : " 14 GB", | 20260506 14:24:19.730 | INFO | "pinned" : " 29 GB" | 20260506 14:24:19.730 | INFO | } | 20260506 14:24:19.731 | INFO | Perf report for the last 5.14002 seconds: | 20260506 14:24:19.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:24:19.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:19.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:19.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:19.731 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:24:19.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:19.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:19.731 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:24:19.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:19.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:19.731 | INFO | Uploaded 36GiB to GPU in 2.05801s (17.4926GiB/s) | 20260506 14:24:19.731 | INFO | Downloaded 34GiB to GPU in 1.9521s (17.4172GiB/s) | 20260506 14:24:24.870 | INFO | Analyzing chunk frames = [2049024,2049536) | 20260506 14:24:24.870 | INFO | MemInfo: { | 20260506 14:24:24.870 | INFO | "gpu" : " 17 GB", | 20260506 14:24:24.870 | INFO | "huge" : "", | 20260506 14:24:24.870 | INFO | "huge_shared" : " 27 GB", | 20260506 14:24:24.870 | INFO | "malloc" : " 14 GB", | 20260506 14:24:24.870 | INFO | "pinned" : " 29 GB" | 20260506 14:24:24.870 | INFO | } | 20260506 14:24:24.870 | INFO | Perf report for the last 5.13932 seconds: | 20260506 14:24:24.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:24:24.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:24.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:24.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:24.870 | INFO | ANALYSIS : 61.6%, 3.0, 95.3ms, 0 | 20260506 14:24:24.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:24.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:24.870 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:24:24.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:24.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:24.870 | INFO | Uploaded 36GiB to GPU in 2.03053s (17.7294GiB/s) | 20260506 14:24:24.870 | INFO | Downloaded 34GiB to GPU in 1.93546s (17.5669GiB/s) | 20260506 14:24:30.009 | INFO | Analyzing chunk frames = [2049536,2050048) | 20260506 14:24:30.009 | INFO | MemInfo: { | 20260506 14:24:30.009 | INFO | "gpu" : " 17 GB", | 20260506 14:24:30.009 | INFO | "huge" : "", | 20260506 14:24:30.009 | INFO | "huge_shared" : " 27 GB", | 20260506 14:24:30.009 | INFO | "malloc" : " 14 GB", | 20260506 14:24:30.009 | INFO | "pinned" : " 29 GB" | 20260506 14:24:30.009 | INFO | } | 20260506 14:24:30.009 | INFO | Perf report for the last 5.13912 seconds: | 20260506 14:24:30.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:24:30.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:30.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:30.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:30.009 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 14:24:30.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:30.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:30.009 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:24:30.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:30.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:30.009 | INFO | Uploaded 36GiB to GPU in 1.988s (18.1086GiB/s) | 20260506 14:24:30.009 | INFO | Downloaded 34GiB to GPU in 1.93343s (17.5853GiB/s) | 20260506 14:24:35.149 | INFO | Analyzing chunk frames = [2050048,2050560) | 20260506 14:24:35.149 | INFO | MemInfo: { | 20260506 14:24:35.149 | INFO | "gpu" : " 17 GB", | 20260506 14:24:35.149 | INFO | "huge" : "", | 20260506 14:24:35.149 | INFO | "huge_shared" : " 27 GB", | 20260506 14:24:35.149 | INFO | "malloc" : " 14 GB", | 20260506 14:24:35.149 | INFO | "pinned" : " 29 GB" | 20260506 14:24:35.149 | INFO | } | 20260506 14:24:35.149 | INFO | Perf report for the last 5.13988 seconds: | 20260506 14:24:35.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:24:35.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:35.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:35.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:35.149 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:24:35.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:35.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:35.149 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:24:35.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:35.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:35.149 | INFO | Uploaded 36GiB to GPU in 2.00631s (17.9434GiB/s) | 20260506 14:24:35.149 | INFO | Downloaded 34GiB to GPU in 1.92238s (17.6864GiB/s) | 20260506 14:24:40.287 | INFO | Analyzing chunk frames = [2050560,2051072) | 20260506 14:24:40.287 | INFO | MemInfo: { | 20260506 14:24:40.287 | INFO | "gpu" : " 17 GB", | 20260506 14:24:40.287 | INFO | "huge" : "", | 20260506 14:24:40.287 | INFO | "huge_shared" : " 27 GB", | 20260506 14:24:40.288 | INFO | "malloc" : " 14 GB", | 20260506 14:24:40.288 | INFO | "pinned" : " 29 GB" | 20260506 14:24:40.288 | INFO | } | 20260506 14:24:40.288 | INFO | Perf report for the last 5.13869 seconds: | 20260506 14:24:40.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:24:40.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:40.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:40.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:40.288 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 14:24:40.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:40.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:40.288 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 14:24:40.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:40.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:40.288 | INFO | Uploaded 36GiB to GPU in 2.02446s (17.7825GiB/s) | 20260506 14:24:40.288 | INFO | Downloaded 34GiB to GPU in 1.91825s (17.7245GiB/s) | 20260506 14:24:45.428 | INFO | Analyzing chunk frames = [2051072,2051584) | 20260506 14:24:45.428 | INFO | MemInfo: { | 20260506 14:24:45.428 | INFO | "gpu" : " 17 GB", | 20260506 14:24:45.428 | INFO | "huge" : "", | 20260506 14:24:45.428 | INFO | "huge_shared" : " 27 GB", | 20260506 14:24:45.428 | INFO | "malloc" : " 14 GB", | 20260506 14:24:45.428 | INFO | "pinned" : " 29 GB" | 20260506 14:24:45.428 | INFO | } | 20260506 14:24:45.428 | INFO | Perf report for the last 5.14058 seconds: | 20260506 14:24:45.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:24:45.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:45.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:45.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:45.428 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 14:24:45.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:45.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:45.428 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:24:45.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:45.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:45.429 | INFO | Uploaded 36GiB to GPU in 2.00829s (17.9257GiB/s) | 20260506 14:24:45.429 | INFO | Downloaded 34GiB to GPU in 1.99666s (17.0285GiB/s) | 20260506 14:24:50.566 | INFO | Analyzing chunk frames = [2051584,2052096) | 20260506 14:24:50.566 | INFO | MemInfo: { | 20260506 14:24:50.566 | INFO | "gpu" : " 17 GB", | 20260506 14:24:50.566 | INFO | "huge" : "", | 20260506 14:24:50.567 | INFO | "huge_shared" : " 27 GB", | 20260506 14:24:50.567 | INFO | "malloc" : " 14 GB", | 20260506 14:24:50.567 | INFO | "pinned" : " 29 GB" | 20260506 14:24:50.567 | INFO | } | 20260506 14:24:50.567 | INFO | Perf report for the last 5.13846 seconds: | 20260506 14:24:50.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:24:50.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:50.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:50.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:50.567 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 14:24:50.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:50.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:50.567 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 14:24:50.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:50.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:50.567 | INFO | Uploaded 36GiB to GPU in 2.01282s (17.8854GiB/s) | 20260506 14:24:50.567 | INFO | Downloaded 34GiB to GPU in 1.98883s (17.0955GiB/s) | 20260506 14:24:50.694 | INFO | Starting a round of Baz IO | 20260506 14:24:53.842 | INFO | Populating RTMetrics report | 20260506 14:24:53.842 | INFO | RTMetrics report populated | 20260506 14:24:53.843 | INFO | Opening tmp RTMetrics json file | 20260506 14:24:53.893 | INFO | Serializing RTMetrics to json | 20260506 14:24:53.960 | INFO | Serialization done, swapping tmp file now... | 20260506 14:24:53.972 | INFO | RTMetrics done | 20260506 14:24:54.162 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 3.468 seconds | 20260506 14:24:54.163 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.35% / 0.01% / 61.64% | 20260506 14:24:54.163 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.57732% / 2.38419e-05% / 5.5631e-05% | 20260506 14:24:54.163 | INFO | Baz IO overall: | 20260506 14:24:54.163 | INFO | BytesWritten : 1058250606024 | 20260506 14:24:54.163 | INFO | NumEvents : 512069444659 | 20260506 14:24:55.705 | INFO | Analyzing chunk frames = [2052096,2052608) | 20260506 14:24:55.705 | INFO | MemInfo: { | 20260506 14:24:55.705 | INFO | "gpu" : " 17 GB", | 20260506 14:24:55.705 | INFO | "huge" : "", | 20260506 14:24:55.706 | INFO | "huge_shared" : " 27 GB", | 20260506 14:24:55.706 | INFO | "malloc" : " 14 GB", | 20260506 14:24:55.706 | INFO | "pinned" : " 29 GB" | 20260506 14:24:55.706 | INFO | } | 20260506 14:24:55.706 | INFO | Perf report for the last 5.13896 seconds: | 20260506 14:24:55.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:24:55.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:55.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:55.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:55.706 | INFO | ANALYSIS : 63.1%, 2.9, 95.6ms, 0 | 20260506 14:24:55.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:24:55.706 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260506 14:24:55.706 | INFO | PRE_HQ : 61.0%, 1.9, 59.7ms, 0 | 20260506 14:24:55.706 | INFO | BAZWRITER : 66.9%, 4.8, 164.2ms, 0 | 20260506 14:24:55.706 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 14:24:55.706 | INFO | Uploaded 36GiB to GPU in 2.07064s (17.386GiB/s) | 20260506 14:24:55.706 | INFO | Downloaded 38GiB to GPU in 2.23288s (17.0184GiB/s) | 20260506 14:25:00.846 | INFO | Analyzing chunk frames = [2052608,2053120) | 20260506 14:25:00.846 | INFO | MemInfo: { | 20260506 14:25:00.846 | INFO | "gpu" : " 17 GB", | 20260506 14:25:00.846 | INFO | "huge" : "", | 20260506 14:25:00.846 | INFO | "huge_shared" : " 27 GB", | 20260506 14:25:00.846 | INFO | "malloc" : " 14 GB", | 20260506 14:25:00.846 | INFO | "pinned" : " 29 GB" | 20260506 14:25:00.846 | INFO | } | 20260506 14:25:00.846 | INFO | Perf report for the last 5.1403 seconds: | 20260506 14:25:00.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:25:00.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:00.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:00.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:00.846 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:25:00.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:00.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:00.846 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:25:00.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:00.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:00.846 | INFO | Uploaded 36GiB to GPU in 2.02926s (17.7404GiB/s) | 20260506 14:25:00.846 | INFO | Downloaded 34GiB to GPU in 1.95544s (17.3874GiB/s) | 20260506 14:25:05.984 | INFO | Analyzing chunk frames = [2053120,2053632) | 20260506 14:25:05.984 | INFO | MemInfo: { | 20260506 14:25:05.984 | INFO | "gpu" : " 17 GB", | 20260506 14:25:05.984 | INFO | "huge" : "", | 20260506 14:25:05.984 | INFO | "huge_shared" : " 27 GB", | 20260506 14:25:05.984 | INFO | "malloc" : " 14 GB", | 20260506 14:25:05.984 | INFO | "pinned" : " 29 GB" | 20260506 14:25:05.984 | INFO | } | 20260506 14:25:05.984 | INFO | Perf report for the last 5.1383 seconds: | 20260506 14:25:05.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:25:05.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:05.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:05.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:05.984 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:25:05.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:05.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:05.984 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:25:05.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:05.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:05.985 | INFO | Uploaded 36GiB to GPU in 1.99554s (18.0402GiB/s) | 20260506 14:25:05.985 | INFO | Downloaded 34GiB to GPU in 1.93836s (17.5406GiB/s) | 20260506 14:25:11.124 | INFO | Analyzing chunk frames = [2053632,2054144) | 20260506 14:25:11.124 | INFO | MemInfo: { | 20260506 14:25:11.124 | INFO | "gpu" : " 17 GB", | 20260506 14:25:11.124 | INFO | "huge" : "", | 20260506 14:25:11.124 | INFO | "huge_shared" : " 27 GB", | 20260506 14:25:11.124 | INFO | "malloc" : " 14 GB", | 20260506 14:25:11.124 | INFO | "pinned" : " 29 GB" | 20260506 14:25:11.124 | INFO | } | 20260506 14:25:11.124 | INFO | Perf report for the last 5.13955 seconds: | 20260506 14:25:11.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:25:11.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:11.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:11.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:11.124 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260506 14:25:11.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:11.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:11.124 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:25:11.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:11.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:11.124 | INFO | Uploaded 36GiB to GPU in 1.97923s (18.1888GiB/s) | 20260506 14:25:11.124 | INFO | Downloaded 34GiB to GPU in 1.91244s (17.7783GiB/s) | 20260506 14:25:16.262 | INFO | Analyzing chunk frames = [2054144,2054656) | 20260506 14:25:16.262 | INFO | MemInfo: { | 20260506 14:25:16.262 | INFO | "gpu" : " 17 GB", | 20260506 14:25:16.262 | INFO | "huge" : "", | 20260506 14:25:16.263 | INFO | "huge_shared" : " 27 GB", | 20260506 14:25:16.263 | INFO | "malloc" : " 14 GB", | 20260506 14:25:16.263 | INFO | "pinned" : " 29 GB" | 20260506 14:25:16.263 | INFO | } | 20260506 14:25:16.263 | INFO | Perf report for the last 5.13889 seconds: | 20260506 14:25:16.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:25:16.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:16.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:16.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:16.263 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 14:25:16.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:16.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:16.263 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:25:16.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:16.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:16.263 | INFO | Uploaded 36GiB to GPU in 1.99034s (18.0873GiB/s) | 20260506 14:25:16.263 | INFO | Downloaded 34GiB to GPU in 1.94124s (17.5146GiB/s) | 20260506 14:25:21.402 | INFO | Analyzing chunk frames = [2054656,2055168) | 20260506 14:25:21.402 | INFO | MemInfo: { | 20260506 14:25:21.402 | INFO | "gpu" : " 17 GB", | 20260506 14:25:21.402 | INFO | "huge" : "", | 20260506 14:25:21.402 | INFO | "huge_shared" : " 27 GB", | 20260506 14:25:21.402 | INFO | "malloc" : " 14 GB", | 20260506 14:25:21.402 | INFO | "pinned" : " 29 GB" | 20260506 14:25:21.402 | INFO | } | 20260506 14:25:21.402 | INFO | Perf report for the last 5.13972 seconds: | 20260506 14:25:21.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:25:21.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:21.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:21.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:21.403 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 14:25:21.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:21.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:21.403 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 14:25:21.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:21.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:21.403 | INFO | Uploaded 36GiB to GPU in 2.05674s (17.5034GiB/s) | 20260506 14:25:21.403 | INFO | Downloaded 34GiB to GPU in 1.96977s (17.2609GiB/s) | 20260506 14:25:26.542 | INFO | Analyzing chunk frames = [2055168,2055680) | 20260506 14:25:26.542 | INFO | MemInfo: { | 20260506 14:25:26.542 | INFO | "gpu" : " 17 GB", | 20260506 14:25:26.542 | INFO | "huge" : "", | 20260506 14:25:26.542 | INFO | "huge_shared" : " 27 GB", | 20260506 14:25:26.542 | INFO | "malloc" : " 14 GB", | 20260506 14:25:26.543 | INFO | "pinned" : " 29 GB" | 20260506 14:25:26.543 | INFO | } | 20260506 14:25:26.543 | INFO | Perf report for the last 5.14014 seconds: | 20260506 14:25:26.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:25:26.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:26.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:26.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:26.543 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:25:26.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:26.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:26.543 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 14:25:26.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:26.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:26.543 | INFO | Uploaded 36GiB to GPU in 2.00924s (17.9172GiB/s) | 20260506 14:25:26.543 | INFO | Downloaded 34GiB to GPU in 1.9926s (17.0632GiB/s) | 20260506 14:25:31.681 | INFO | Analyzing chunk frames = [2055680,2056192) | 20260506 14:25:31.681 | INFO | MemInfo: { | 20260506 14:25:31.681 | INFO | "gpu" : " 17 GB", | 20260506 14:25:31.681 | INFO | "huge" : "", | 20260506 14:25:31.681 | INFO | "huge_shared" : " 27 GB", | 20260506 14:25:31.681 | INFO | "malloc" : " 14 GB", | 20260506 14:25:31.681 | INFO | "pinned" : " 29 GB" | 20260506 14:25:31.681 | INFO | } | 20260506 14:25:31.681 | INFO | Perf report for the last 5.13862 seconds: | 20260506 14:25:31.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:25:31.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:31.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:31.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:31.681 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:25:31.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:31.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:31.681 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 14:25:31.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:31.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:31.682 | INFO | Uploaded 36GiB to GPU in 2.01991s (17.8226GiB/s) | 20260506 14:25:31.682 | INFO | Downloaded 34GiB to GPU in 1.97686s (17.199GiB/s) | 20260506 14:25:31.682 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:25:31.682 | INFO | Total mem usage: 17.4 GiB | 20260506 14:25:31.682 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:25:31.682 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -10.6%) | 20260506 14:25:31.682 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:25:31.682 | INFO | High Water Breakdown: | 20260506 14:25:31.682 | INFO | 61.3% : 9.65 GiB ( 88.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:25:31.682 | INFO | 38.6% : 6.08 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:25:31.682 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:25:31.682 | INFO | ------------------------------------------ | 20260506 14:25:31.682 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:25:31.682 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:25:31.682 | INFO | Total mem usage: 26.5 GiB | 20260506 14:25:31.682 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:25:31.682 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:25:31.682 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:25:31.682 | INFO | High Water Breakdown: | 20260506 14:25:31.682 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:25:31.682 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:25:31.682 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:25:31.682 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:25:31.682 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:25:31.682 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:25:31.682 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:25:31.682 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:25:31.682 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:25:31.682 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:25:31.682 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:25:31.682 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:25:31.682 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:25:31.682 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:25:31.682 | INFO | ------------------------------------------ | 20260506 14:25:31.682 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:25:31.682 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:25:31.682 | INFO | Total mem usage: 25.6 GiB | 20260506 14:25:31.682 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:25:31.682 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:25:31.682 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:25:31.682 | INFO | High Water Breakdown: | 20260506 14:25:31.682 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:25:31.682 | INFO | ------------------------------------------ | 20260506 14:25:31.682 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:25:31.682 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:25:31.682 | INFO | Total mem usage: 15.7 GiB | 20260506 14:25:31.682 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:25:31.682 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:25:31.682 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 14:25:31.682 | INFO | High Water Breakdown: | 20260506 14:25:31.682 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:25:31.682 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:25:31.682 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:25:31.682 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:25:31.682 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:25:31.682 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:25:31.682 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:25:31.682 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:25:31.682 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:25:31.682 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:25:31.682 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:25:31.683 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:25:31.683 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:25:31.683 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:25:31.683 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:25:31.683 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:25:31.683 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:25:31.683 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:25:31.683 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:25:31.683 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:25:31.683 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:25:31.683 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:25:31.683 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:25:31.683 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:25:31.683 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:25:31.683 | INFO | ------------------------------------------ | 20260506 14:25:31.683 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:25:31.683 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:25:31.683 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:25:31.683 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:25:31.683 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.57e+01, 1.34e+01 / 2.08e+00 | 20260506 14:25:31.683 | INFO | PerfReport: 0.31%, AggregateStats -- 9.85e-02 / 1.02e-01, 2.64e-01 / 5.14e-03 | 20260506 14:25:31.683 | INFO | PerfReport: 11.57%, Baseline -- 3.64e+00 / 4.47e+00, 5.60e-01 / 2.59e-01 | 20260506 14:25:31.683 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.84e-01 / 1.25e-01 | 20260506 14:25:31.683 | INFO | PerfReport: 15.87%, DME -- 4.00e+01 / 4.45e+01, 3.87e+00 / 2.21e+00 | 20260506 14:25:31.683 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.40e+00 / 1.76e+00 | 20260506 14:25:31.683 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.35e+00 / 2.97e+00, 4.10e-01 / 1.94e-01 | 20260506 14:25:31.683 | INFO | PerfReport: 3.62%, Metrics -- 1.14e+00 / 1.52e+00, 2.86e-01 / 1.76e-01 | 20260506 14:25:31.683 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.98e-01 / 4.55e-01, 6.30e-02 / 3.75e-02 | 20260506 14:25:31.683 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:25:31.683 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:25:31.683 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:25:31.683 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 14:25:31.683 | INFO | PerfReport: 21.52%, Upload -- 2.11e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 14:25:31.683 | INFO | PerfReport: 21.15%, Download -- 2.07e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 14:25:31.683 | INFO | PerfReport: 32.18%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 14:25:31.683 | INFO | PerfReport: 1.66%, LockUpload -- 1.62e+00 / 0.00e+00, 6.39e+00 / 0.00e+00 | 20260506 14:25:31.683 | INFO | PerfReport: 1.15%, LockDownload -- 1.13e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 14:25:31.683 | INFO | PerfReport: 22.32%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:25:31.683 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260506 14:25:31.795 | INFO | Starting a round of Baz IO | 20260506 14:25:34.955 | INFO | Populating RTMetrics report | 20260506 14:25:34.955 | INFO | RTMetrics report populated | 20260506 14:25:34.955 | INFO | Opening tmp RTMetrics json file | 20260506 14:25:35.051 | INFO | Serializing RTMetrics to json | 20260506 14:25:35.107 | INFO | Serialization done, swapping tmp file now... | 20260506 14:25:35.120 | INFO | RTMetrics done | 20260506 14:25:35.234 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.438 seconds | 20260506 14:25:35.234 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.33% / 0.01% / 61.65% | 20260506 14:25:35.234 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.5719% / 3.97364e-06% / 4.37101e-05% | 20260506 14:25:35.234 | INFO | Baz IO overall: | 20260506 14:25:35.234 | INFO | BytesWritten : 1060520142608 | 20260506 14:25:35.234 | INFO | NumEvents : 513169507366 | 20260506 14:25:36.820 | INFO | Analyzing chunk frames = [2056192,2056704) | 20260506 14:25:36.820 | INFO | MemInfo: { | 20260506 14:25:36.820 | INFO | "gpu" : " 17 GB", | 20260506 14:25:36.820 | INFO | "huge" : "", | 20260506 14:25:36.820 | INFO | "huge_shared" : " 27 GB", | 20260506 14:25:36.820 | INFO | "malloc" : " 14 GB", | 20260506 14:25:36.820 | INFO | "pinned" : " 29 GB" | 20260506 14:25:36.820 | INFO | } | 20260506 14:25:36.820 | INFO | Perf report for the last 5.13882 seconds: | 20260506 14:25:36.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:25:36.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:36.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:36.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:36.820 | INFO | ANALYSIS : 63.1%, 2.9, 95.5ms, 0 | 20260506 14:25:36.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:36.820 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260506 14:25:36.820 | INFO | PRE_HQ : 61.3%, 1.9, 59.9ms, 0 | 20260506 14:25:36.820 | INFO | BAZWRITER : 66.3%, 4.8, 164.1ms, 0 | 20260506 14:25:36.820 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 14:25:36.820 | INFO | Uploaded 36GiB to GPU in 2.05942s (17.4806GiB/s) | 20260506 14:25:36.820 | INFO | Downloaded 38GiB to GPU in 2.22253s (17.0977GiB/s) | 20260506 14:25:41.959 | INFO | Analyzing chunk frames = [2056704,2057216) | 20260506 14:25:41.959 | INFO | MemInfo: { | 20260506 14:25:41.959 | INFO | "gpu" : " 17 GB", | 20260506 14:25:41.959 | INFO | "huge" : "", | 20260506 14:25:41.959 | INFO | "huge_shared" : " 27 GB", | 20260506 14:25:41.959 | INFO | "malloc" : " 14 GB", | 20260506 14:25:41.959 | INFO | "pinned" : " 29 GB" | 20260506 14:25:41.959 | INFO | } | 20260506 14:25:41.959 | INFO | Perf report for the last 5.13919 seconds: | 20260506 14:25:41.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:25:41.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:41.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:41.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:41.959 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:25:41.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:41.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:41.959 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:25:41.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:41.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:41.959 | INFO | Uploaded 36GiB to GPU in 1.99998s (18.0002GiB/s) | 20260506 14:25:41.960 | INFO | Downloaded 34GiB to GPU in 1.94011s (17.5248GiB/s) | 20260506 14:25:47.099 | INFO | Analyzing chunk frames = [2057216,2057728) | 20260506 14:25:47.099 | INFO | MemInfo: { | 20260506 14:25:47.099 | INFO | "gpu" : " 17 GB", | 20260506 14:25:47.099 | INFO | "huge" : "", | 20260506 14:25:47.099 | INFO | "huge_shared" : " 27 GB", | 20260506 14:25:47.099 | INFO | "malloc" : " 14 GB", | 20260506 14:25:47.099 | INFO | "pinned" : " 29 GB" | 20260506 14:25:47.099 | INFO | } | 20260506 14:25:47.099 | INFO | Perf report for the last 5.1401 seconds: | 20260506 14:25:47.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:25:47.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:47.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:47.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:47.099 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 14:25:47.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:47.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:47.099 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:25:47.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:47.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:47.100 | INFO | Uploaded 36GiB to GPU in 1.98904s (18.0992GiB/s) | 20260506 14:25:47.100 | INFO | Downloaded 34GiB to GPU in 1.93524s (17.5689GiB/s) | 20260506 14:25:52.239 | INFO | Analyzing chunk frames = [2057728,2058240) | 20260506 14:25:52.239 | INFO | MemInfo: { | 20260506 14:25:52.239 | INFO | "gpu" : " 17 GB", | 20260506 14:25:52.239 | INFO | "huge" : "", | 20260506 14:25:52.239 | INFO | "huge_shared" : " 27 GB", | 20260506 14:25:52.239 | INFO | "malloc" : " 14 GB", | 20260506 14:25:52.239 | INFO | "pinned" : " 29 GB" | 20260506 14:25:52.239 | INFO | } | 20260506 14:25:52.239 | INFO | Perf report for the last 5.14006 seconds: | 20260506 14:25:52.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:25:52.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:52.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:52.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:52.240 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 14:25:52.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:52.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:52.240 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:25:52.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:52.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:52.240 | INFO | Uploaded 36GiB to GPU in 1.97469s (18.2307GiB/s) | 20260506 14:25:52.240 | INFO | Downloaded 34GiB to GPU in 1.91129s (17.7891GiB/s) | 20260506 14:25:57.378 | INFO | Analyzing chunk frames = [2058240,2058752) | 20260506 14:25:57.378 | INFO | MemInfo: { | 20260506 14:25:57.378 | INFO | "gpu" : " 17 GB", | 20260506 14:25:57.378 | INFO | "huge" : "", | 20260506 14:25:57.378 | INFO | "huge_shared" : " 27 GB", | 20260506 14:25:57.378 | INFO | "malloc" : " 14 GB", | 20260506 14:25:57.378 | INFO | "pinned" : " 29 GB" | 20260506 14:25:57.378 | INFO | } | 20260506 14:25:57.378 | INFO | Perf report for the last 5.13883 seconds: | 20260506 14:25:57.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:25:57.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:57.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:57.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:57.378 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:25:57.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:57.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:57.378 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:25:57.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:57.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:25:57.379 | INFO | Uploaded 36GiB to GPU in 2.04194s (17.6303GiB/s) | 20260506 14:25:57.379 | INFO | Downloaded 34GiB to GPU in 1.9237s (17.6743GiB/s) | 20260506 14:26:02.517 | INFO | Analyzing chunk frames = [2058752,2059264) | 20260506 14:26:02.517 | INFO | MemInfo: { | 20260506 14:26:02.517 | INFO | "gpu" : " 17 GB", | 20260506 14:26:02.517 | INFO | "huge" : "", | 20260506 14:26:02.517 | INFO | "huge_shared" : " 27 GB", | 20260506 14:26:02.517 | INFO | "malloc" : " 14 GB", | 20260506 14:26:02.517 | INFO | "pinned" : " 29 GB" | 20260506 14:26:02.517 | INFO | } | 20260506 14:26:02.517 | INFO | Perf report for the last 5.13856 seconds: | 20260506 14:26:02.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:26:02.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:02.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:02.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:02.517 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 14:26:02.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:02.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:02.517 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260506 14:26:02.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:02.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:02.517 | INFO | Uploaded 36GiB to GPU in 2.03055s (17.7292GiB/s) | 20260506 14:26:02.517 | INFO | Downloaded 34GiB to GPU in 1.89884s (17.9057GiB/s) | 20260506 14:26:07.656 | INFO | Analyzing chunk frames = [2059264,2059776) | 20260506 14:26:07.656 | INFO | MemInfo: { | 20260506 14:26:07.656 | INFO | "gpu" : " 17 GB", | 20260506 14:26:07.656 | INFO | "huge" : "", | 20260506 14:26:07.656 | INFO | "huge_shared" : " 27 GB", | 20260506 14:26:07.656 | INFO | "malloc" : " 14 GB", | 20260506 14:26:07.657 | INFO | "pinned" : " 29 GB" | 20260506 14:26:07.657 | INFO | } | 20260506 14:26:07.657 | INFO | Perf report for the last 5.13968 seconds: | 20260506 14:26:07.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:26:07.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:07.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:07.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:07.657 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:26:07.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:07.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:07.657 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 14:26:07.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:07.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:07.657 | INFO | Uploaded 36GiB to GPU in 2.00861s (17.9229GiB/s) | 20260506 14:26:07.657 | INFO | Downloaded 34GiB to GPU in 2.0027s (16.9771GiB/s) | 20260506 14:26:12.796 | INFO | Analyzing chunk frames = [2059776,2060288) | 20260506 14:26:12.796 | INFO | MemInfo: { | 20260506 14:26:12.796 | INFO | "gpu" : " 17 GB", | 20260506 14:26:12.796 | INFO | "huge" : "", | 20260506 14:26:12.796 | INFO | "huge_shared" : " 27 GB", | 20260506 14:26:12.796 | INFO | "malloc" : " 14 GB", | 20260506 14:26:12.796 | INFO | "pinned" : " 29 GB" | 20260506 14:26:12.796 | INFO | } | 20260506 14:26:12.796 | INFO | Perf report for the last 5.13977 seconds: | 20260506 14:26:12.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:26:12.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:12.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:12.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:12.796 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 14:26:12.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:12.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:12.796 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 14:26:12.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:12.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:12.797 | INFO | Uploaded 36GiB to GPU in 2.0073s (17.9346GiB/s) | 20260506 14:26:12.797 | INFO | Downloaded 34GiB to GPU in 1.97801s (17.189GiB/s) | 20260506 14:26:12.906 | INFO | Starting a round of Baz IO | 20260506 14:26:16.061 | INFO | Populating RTMetrics report | 20260506 14:26:16.061 | INFO | RTMetrics report populated | 20260506 14:26:16.061 | INFO | Opening tmp RTMetrics json file | 20260506 14:26:16.073 | INFO | Serializing RTMetrics to json | 20260506 14:26:16.076 | INFO | Serialization done, swapping tmp file now... | 20260506 14:26:16.077 | INFO | RTMetrics done | 20260506 14:26:16.708 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.801 seconds | 20260506 14:26:16.708 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.32% / 0.01% / 61.67% | 20260506 14:26:16.708 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.56229% / 3.97364e-05% / 3.97364e-05% | 20260506 14:26:16.708 | INFO | Baz IO overall: | 20260506 14:26:16.708 | INFO | BytesWritten : 1062788059915 | 20260506 14:26:16.708 | INFO | NumEvents : 514268411549 | 20260506 14:26:17.935 | INFO | Analyzing chunk frames = [2060288,2060800) | 20260506 14:26:17.935 | INFO | MemInfo: { | 20260506 14:26:17.935 | INFO | "gpu" : " 17 GB", | 20260506 14:26:17.935 | INFO | "huge" : "", | 20260506 14:26:17.935 | INFO | "huge_shared" : " 27 GB", | 20260506 14:26:17.935 | INFO | "malloc" : " 14 GB", | 20260506 14:26:17.935 | INFO | "pinned" : " 29 GB" | 20260506 14:26:17.935 | INFO | } | 20260506 14:26:17.935 | INFO | Perf report for the last 5.13906 seconds: | 20260506 14:26:17.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:26:17.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:17.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:17.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:17.935 | INFO | ANALYSIS : 62.9%, 2.9, 94.8ms, 0 | 20260506 14:26:17.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:17.936 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 14:26:17.936 | INFO | PRE_HQ : 61.2%, 1.9, 60.0ms, 0 | 20260506 14:26:17.936 | INFO | BAZWRITER : 73.2%, 4.8, 180.3ms, 0 | 20260506 14:26:17.936 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 14:26:17.936 | INFO | Uploaded 36GiB to GPU in 2.06787s (17.4093GiB/s) | 20260506 14:26:17.936 | INFO | Downloaded 38GiB to GPU in 2.22441s (17.0832GiB/s) | 20260506 14:26:23.075 | INFO | Analyzing chunk frames = [2060800,2061312) | 20260506 14:26:23.075 | INFO | MemInfo: { | 20260506 14:26:23.075 | INFO | "gpu" : " 17 GB", | 20260506 14:26:23.075 | INFO | "huge" : "", | 20260506 14:26:23.075 | INFO | "huge_shared" : " 27 GB", | 20260506 14:26:23.075 | INFO | "malloc" : " 14 GB", | 20260506 14:26:23.075 | INFO | "pinned" : " 29 GB" | 20260506 14:26:23.075 | INFO | } | 20260506 14:26:23.075 | INFO | Perf report for the last 5.13963 seconds: | 20260506 14:26:23.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:26:23.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:23.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:23.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:23.075 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 14:26:23.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:23.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:23.075 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:26:23.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:23.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:23.075 | INFO | Uploaded 36GiB to GPU in 2.01822s (17.8375GiB/s) | 20260506 14:26:23.075 | INFO | Downloaded 34GiB to GPU in 1.95464s (17.3945GiB/s) | 20260506 14:26:28.214 | INFO | Analyzing chunk frames = [2061312,2061824) | 20260506 14:26:28.214 | INFO | MemInfo: { | 20260506 14:26:28.214 | INFO | "gpu" : " 17 GB", | 20260506 14:26:28.214 | INFO | "huge" : "", | 20260506 14:26:28.214 | INFO | "huge_shared" : " 27 GB", | 20260506 14:26:28.214 | INFO | "malloc" : " 14 GB", | 20260506 14:26:28.214 | INFO | "pinned" : " 29 GB" | 20260506 14:26:28.214 | INFO | } | 20260506 14:26:28.214 | INFO | Perf report for the last 5.13891 seconds: | 20260506 14:26:28.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:26:28.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:28.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:28.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:28.214 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 14:26:28.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:28.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:28.214 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:26:28.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:28.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:28.214 | INFO | Uploaded 36GiB to GPU in 1.96178s (18.3507GiB/s) | 20260506 14:26:28.214 | INFO | Downloaded 34GiB to GPU in 1.93301s (17.5892GiB/s) | 20260506 14:26:33.353 | INFO | Analyzing chunk frames = [2061824,2062336) | 20260506 14:26:33.353 | INFO | MemInfo: { | 20260506 14:26:33.353 | INFO | "gpu" : " 17 GB", | 20260506 14:26:33.353 | INFO | "huge" : "", | 20260506 14:26:33.353 | INFO | "huge_shared" : " 27 GB", | 20260506 14:26:33.353 | INFO | "malloc" : " 14 GB", | 20260506 14:26:33.353 | INFO | "pinned" : " 29 GB" | 20260506 14:26:33.353 | INFO | } | 20260506 14:26:33.353 | INFO | Perf report for the last 5.13937 seconds: | 20260506 14:26:33.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:26:33.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:33.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:33.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:33.353 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:26:33.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:33.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:33.353 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 14:26:33.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:33.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:33.354 | INFO | Uploaded 36GiB to GPU in 2.01286s (17.885GiB/s) | 20260506 14:26:33.354 | INFO | Downloaded 34GiB to GPU in 1.91419s (17.762GiB/s) | 20260506 14:26:38.492 | INFO | Analyzing chunk frames = [2062336,2062848) | 20260506 14:26:38.492 | INFO | MemInfo: { | 20260506 14:26:38.492 | INFO | "gpu" : " 17 GB", | 20260506 14:26:38.492 | INFO | "huge" : "", | 20260506 14:26:38.493 | INFO | "huge_shared" : " 27 GB", | 20260506 14:26:38.493 | INFO | "malloc" : " 14 GB", | 20260506 14:26:38.493 | INFO | "pinned" : " 29 GB" | 20260506 14:26:38.493 | INFO | } | 20260506 14:26:38.493 | INFO | Perf report for the last 5.13929 seconds: | 20260506 14:26:38.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:26:38.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:38.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:38.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:38.493 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 14:26:38.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:38.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:38.493 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:26:38.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:38.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:38.493 | INFO | Uploaded 36GiB to GPU in 2.01571s (17.8597GiB/s) | 20260506 14:26:38.493 | INFO | Downloaded 34GiB to GPU in 1.93892s (17.5355GiB/s) | 20260506 14:26:43.631 | INFO | Analyzing chunk frames = [2062848,2063360) | 20260506 14:26:43.631 | INFO | MemInfo: { | 20260506 14:26:43.631 | INFO | "gpu" : " 17 GB", | 20260506 14:26:43.631 | INFO | "huge" : "", | 20260506 14:26:43.631 | INFO | "huge_shared" : " 27 GB", | 20260506 14:26:43.631 | INFO | "malloc" : " 14 GB", | 20260506 14:26:43.631 | INFO | "pinned" : " 29 GB" | 20260506 14:26:43.631 | INFO | } | 20260506 14:26:43.632 | INFO | Perf report for the last 5.13885 seconds: | 20260506 14:26:43.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:26:43.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:43.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:43.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:43.632 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:26:43.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:43.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:43.632 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 14:26:43.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:43.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:43.632 | INFO | Uploaded 36GiB to GPU in 2.04855s (17.5734GiB/s) | 20260506 14:26:43.632 | INFO | Downloaded 34GiB to GPU in 1.92823s (17.6327GiB/s) | 20260506 14:26:48.772 | INFO | Analyzing chunk frames = [2063360,2063872) | 20260506 14:26:48.772 | INFO | MemInfo: { | 20260506 14:26:48.772 | INFO | "gpu" : " 17 GB", | 20260506 14:26:48.772 | INFO | "huge" : "", | 20260506 14:26:48.772 | INFO | "huge_shared" : " 27 GB", | 20260506 14:26:48.772 | INFO | "malloc" : " 14 GB", | 20260506 14:26:48.772 | INFO | "pinned" : " 29 GB" | 20260506 14:26:48.772 | INFO | } | 20260506 14:26:48.772 | INFO | Perf report for the last 5.14074 seconds: | 20260506 14:26:48.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:26:48.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:48.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:48.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:48.772 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 14:26:48.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:48.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:48.773 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:26:48.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:48.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:48.773 | INFO | Uploaded 36GiB to GPU in 1.99884s (18.0105GiB/s) | 20260506 14:26:48.773 | INFO | Downloaded 34GiB to GPU in 1.99883s (17.01GiB/s) | 20260506 14:26:53.910 | INFO | Analyzing chunk frames = [2063872,2064384) | 20260506 14:26:53.910 | INFO | MemInfo: { | 20260506 14:26:53.910 | INFO | "gpu" : " 17 GB", | 20260506 14:26:53.910 | INFO | "huge" : "", | 20260506 14:26:53.910 | INFO | "huge_shared" : " 27 GB", | 20260506 14:26:53.910 | INFO | "malloc" : " 14 GB", | 20260506 14:26:53.910 | INFO | "pinned" : " 29 GB" | 20260506 14:26:53.910 | INFO | } | 20260506 14:26:53.910 | INFO | Perf report for the last 5.13792 seconds: | 20260506 14:26:53.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:26:53.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:53.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:53.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:53.910 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:26:53.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:53.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:53.910 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:26:53.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:53.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:53.910 | INFO | Uploaded 36GiB to GPU in 2.01993s (17.8224GiB/s) | 20260506 14:26:53.910 | INFO | Downloaded 34GiB to GPU in 1.96923s (17.2657GiB/s) | 20260506 14:26:53.911 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:26:53.911 | INFO | Total mem usage: 17.4 GiB | 20260506 14:26:53.911 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:26:53.911 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -11.3%) | 20260506 14:26:53.911 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:26:53.911 | INFO | High Water Breakdown: | 20260506 14:26:53.911 | INFO | 61.2% : 9.57 GiB ( 87.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:26:53.911 | INFO | 38.7% : 6.05 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:26:53.911 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:26:53.911 | INFO | ------------------------------------------ | 20260506 14:26:53.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:26:53.911 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:26:53.911 | INFO | Total mem usage: 26.5 GiB | 20260506 14:26:53.911 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:26:53.911 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:26:53.911 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:26:53.911 | INFO | High Water Breakdown: | 20260506 14:26:53.911 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:26:53.911 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:26:53.911 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:26:53.911 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:26:53.911 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:26:53.911 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:26:53.911 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:26:53.911 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:26:53.911 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:26:53.911 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:26:53.911 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:26:53.911 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:26:53.911 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:26:53.911 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:26:53.911 | INFO | ------------------------------------------ | 20260506 14:26:53.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:26:53.911 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:26:53.911 | INFO | Total mem usage: 25.6 GiB | 20260506 14:26:53.911 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:26:53.911 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:26:53.911 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:26:53.911 | INFO | High Water Breakdown: | 20260506 14:26:53.911 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:26:53.911 | INFO | ------------------------------------------ | 20260506 14:26:53.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:26:53.911 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:26:53.911 | INFO | Total mem usage: 15.7 GiB | 20260506 14:26:53.911 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:26:53.911 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:26:53.911 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 14:26:53.911 | INFO | High Water Breakdown: | 20260506 14:26:53.911 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:26:53.911 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:26:53.911 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:26:53.911 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:26:53.911 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:26:53.911 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:26:53.911 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:26:53.911 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:26:53.911 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:26:53.912 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:26:53.912 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:26:53.912 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:26:53.912 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:26:53.912 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:26:53.912 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:26:53.912 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:26:53.912 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:26:53.912 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:26:53.912 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:26:53.912 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:26:53.912 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:26:53.912 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:26:53.912 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:26:53.912 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:26:53.912 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:26:53.912 | INFO | ------------------------------------------ | 20260506 14:26:53.912 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:26:53.912 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:26:53.912 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:26:53.912 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:26:53.912 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.56e+01, 1.35e+01 / 1.93e+00 | 20260506 14:26:53.912 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.01e-01, 1.15e-02 / 5.81e-03 | 20260506 14:26:53.912 | INFO | PerfReport: 11.57%, Baseline -- 3.65e+00 / 4.48e+00, 5.68e-01 / 3.19e-01 | 20260506 14:26:53.912 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.49e+00, 1.90e-01 / 1.47e-01 | 20260506 14:26:53.912 | INFO | PerfReport: 15.85%, DME -- 4.00e+01 / 4.43e+01, 3.71e+00 / 2.15e+00 | 20260506 14:26:53.912 | INFO | PerfReport: 56.68%, FrameLabeling -- 1.79e+01 / 2.03e+01, 1.47e+00 / 1.62e+00 | 20260506 14:26:53.912 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.35e+00 / 2.95e+00, 4.13e-01 / 2.43e-01 | 20260506 14:26:53.912 | INFO | PerfReport: 3.62%, Metrics -- 1.14e+00 / 1.50e+00, 2.91e-01 / 1.77e-01 | 20260506 14:26:53.912 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.98e-01 / 4.50e-01, 6.31e-02 / 2.52e-02 | 20260506 14:26:53.912 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:26:53.912 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:26:53.912 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:26:53.912 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 14:26:53.912 | INFO | PerfReport: 21.49%, Upload -- 2.10e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 14:26:53.912 | INFO | PerfReport: 21.10%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 14:26:53.912 | INFO | PerfReport: 32.28%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 14:26:53.912 | INFO | PerfReport: 1.53%, LockUpload -- 1.50e+00 / 0.00e+00, 6.20e+00 / 0.00e+00 | 20260506 14:26:53.912 | INFO | PerfReport: 1.15%, LockDownload -- 1.13e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260506 14:26:53.912 | INFO | PerfReport: 22.42%, LockCompute -- 2.19e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 14:26:53.912 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260506 14:26:54.023 | INFO | Starting a round of Baz IO | 20260506 14:26:57.185 | INFO | Populating RTMetrics report | 20260506 14:26:57.185 | INFO | RTMetrics report populated | 20260506 14:26:57.185 | INFO | Opening tmp RTMetrics json file | 20260506 14:26:57.195 | INFO | Serializing RTMetrics to json | 20260506 14:26:57.217 | INFO | Serialization done, swapping tmp file now... | 20260506 14:26:57.219 | INFO | RTMetrics done | 20260506 14:26:57.727 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.704 seconds | 20260506 14:26:57.727 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.30% / 0.01% / 61.68% | 20260506 14:26:57.727 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.56077% / 2.78155e-05% / 3.97364e-05% | 20260506 14:26:57.727 | INFO | Baz IO overall: | 20260506 14:26:57.728 | INFO | BytesWritten : 1065058186102 | 20260506 14:26:57.728 | INFO | NumEvents : 515369155927 | 20260506 14:26:59.051 | INFO | Analyzing chunk frames = [2064384,2064896) | 20260506 14:26:59.051 | INFO | MemInfo: { | 20260506 14:26:59.051 | INFO | "gpu" : " 17 GB", | 20260506 14:26:59.051 | INFO | "huge" : "", | 20260506 14:26:59.051 | INFO | "huge_shared" : " 27 GB", | 20260506 14:26:59.051 | INFO | "malloc" : " 14 GB", | 20260506 14:26:59.051 | INFO | "pinned" : " 29 GB" | 20260506 14:26:59.051 | INFO | } | 20260506 14:26:59.051 | INFO | Perf report for the last 5.14039 seconds: | 20260506 14:26:59.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:26:59.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:59.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:59.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:59.051 | INFO | ANALYSIS : 63.1%, 2.9, 95.6ms, 0 | 20260506 14:26:59.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:26:59.051 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 14:26:59.051 | INFO | PRE_HQ : 61.0%, 1.9, 59.1ms, 0 | 20260506 14:26:59.051 | INFO | BAZWRITER : 71.4%, 4.9, 178.2ms, 0 | 20260506 14:26:59.051 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 14:26:59.051 | INFO | Uploaded 36GiB to GPU in 2.06544s (17.4297GiB/s) | 20260506 14:26:59.051 | INFO | Downloaded 38GiB to GPU in 2.22309s (17.0933GiB/s) | 20260506 14:27:04.190 | INFO | Analyzing chunk frames = [2064896,2065408) | 20260506 14:27:04.190 | INFO | MemInfo: { | 20260506 14:27:04.190 | INFO | "gpu" : " 17 GB", | 20260506 14:27:04.190 | INFO | "huge" : "", | 20260506 14:27:04.190 | INFO | "huge_shared" : " 27 GB", | 20260506 14:27:04.190 | INFO | "malloc" : " 14 GB", | 20260506 14:27:04.190 | INFO | "pinned" : " 29 GB" | 20260506 14:27:04.190 | INFO | } | 20260506 14:27:04.190 | INFO | Perf report for the last 5.1396 seconds: | 20260506 14:27:04.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:27:04.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:04.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:04.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:04.190 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:27:04.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:04.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:04.190 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:27:04.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:04.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:04.190 | INFO | Uploaded 36GiB to GPU in 1.99957s (18.0038GiB/s) | 20260506 14:27:04.191 | INFO | Downloaded 34GiB to GPU in 1.94558s (17.4755GiB/s) | 20260506 14:27:09.328 | INFO | Analyzing chunk frames = [2065408,2065920) | 20260506 14:27:09.328 | INFO | MemInfo: { | 20260506 14:27:09.328 | INFO | "gpu" : " 17 GB", | 20260506 14:27:09.328 | INFO | "huge" : "", | 20260506 14:27:09.328 | INFO | "huge_shared" : " 27 GB", | 20260506 14:27:09.329 | INFO | "malloc" : " 14 GB", | 20260506 14:27:09.329 | INFO | "pinned" : " 29 GB" | 20260506 14:27:09.329 | INFO | } | 20260506 14:27:09.329 | INFO | Perf report for the last 5.1384 seconds: | 20260506 14:27:09.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:27:09.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:09.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:09.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:09.329 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:27:09.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:09.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:09.329 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:27:09.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:09.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:09.329 | INFO | Uploaded 36GiB to GPU in 2.01197s (17.8929GiB/s) | 20260506 14:27:09.329 | INFO | Downloaded 34GiB to GPU in 1.93468s (17.574GiB/s) | 20260506 14:27:14.467 | INFO | Analyzing chunk frames = [2065920,2066432) | 20260506 14:27:14.467 | INFO | MemInfo: { | 20260506 14:27:14.467 | INFO | "gpu" : " 17 GB", | 20260506 14:27:14.467 | INFO | "huge" : "", | 20260506 14:27:14.467 | INFO | "huge_shared" : " 27 GB", | 20260506 14:27:14.467 | INFO | "malloc" : " 14 GB", | 20260506 14:27:14.467 | INFO | "pinned" : " 29 GB" | 20260506 14:27:14.467 | INFO | } | 20260506 14:27:14.467 | INFO | Perf report for the last 5.13885 seconds: | 20260506 14:27:14.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:27:14.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:14.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:14.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:14.468 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:27:14.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:14.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:14.468 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 14:27:14.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:14.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:14.468 | INFO | Uploaded 36GiB to GPU in 1.9875s (18.1132GiB/s) | 20260506 14:27:14.468 | INFO | Downloaded 34GiB to GPU in 1.92299s (17.6808GiB/s) | 20260506 14:27:19.607 | INFO | Analyzing chunk frames = [2066432,2066944) | 20260506 14:27:19.607 | INFO | MemInfo: { | 20260506 14:27:19.607 | INFO | "gpu" : " 17 GB", | 20260506 14:27:19.607 | INFO | "huge" : "", | 20260506 14:27:19.607 | INFO | "huge_shared" : " 27 GB", | 20260506 14:27:19.607 | INFO | "malloc" : " 14 GB", | 20260506 14:27:19.607 | INFO | "pinned" : " 29 GB" | 20260506 14:27:19.607 | INFO | } | 20260506 14:27:19.607 | INFO | Perf report for the last 5.13956 seconds: | 20260506 14:27:19.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:27:19.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:19.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:19.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:19.607 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:27:19.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:19.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:19.607 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 14:27:19.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:19.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:19.607 | INFO | Uploaded 36GiB to GPU in 2.00833s (17.9253GiB/s) | 20260506 14:27:19.607 | INFO | Downloaded 34GiB to GPU in 1.93189s (17.5993GiB/s) | 20260506 14:27:24.746 | INFO | Analyzing chunk frames = [2066944,2067456) | 20260506 14:27:24.746 | INFO | MemInfo: { | 20260506 14:27:24.746 | INFO | "gpu" : " 17 GB", | 20260506 14:27:24.746 | INFO | "huge" : "", | 20260506 14:27:24.746 | INFO | "huge_shared" : " 27 GB", | 20260506 14:27:24.746 | INFO | "malloc" : " 14 GB", | 20260506 14:27:24.746 | INFO | "pinned" : " 29 GB" | 20260506 14:27:24.746 | INFO | } | 20260506 14:27:24.746 | INFO | Perf report for the last 5.13917 seconds: | 20260506 14:27:24.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:27:24.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:24.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:24.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:24.746 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:27:24.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:24.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:24.746 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:27:24.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:24.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:24.747 | INFO | Uploaded 36GiB to GPU in 2.07968s (17.3103GiB/s) | 20260506 14:27:24.747 | INFO | Downloaded 34GiB to GPU in 1.97236s (17.2383GiB/s) | 20260506 14:27:29.887 | INFO | Analyzing chunk frames = [2067456,2067968) | 20260506 14:27:29.887 | INFO | MemInfo: { | 20260506 14:27:29.887 | INFO | "gpu" : " 17 GB", | 20260506 14:27:29.887 | INFO | "huge" : "", | 20260506 14:27:29.887 | INFO | "huge_shared" : " 27 GB", | 20260506 14:27:29.887 | INFO | "malloc" : " 14 GB", | 20260506 14:27:29.887 | INFO | "pinned" : " 29 GB" | 20260506 14:27:29.887 | INFO | } | 20260506 14:27:29.887 | INFO | Perf report for the last 5.14118 seconds: | 20260506 14:27:29.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:27:29.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:29.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:29.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:29.887 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:27:29.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:29.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:29.888 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 14:27:29.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:29.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:29.888 | INFO | Uploaded 36GiB to GPU in 2.02606s (17.7684GiB/s) | 20260506 14:27:29.888 | INFO | Downloaded 34GiB to GPU in 1.97319s (17.2309GiB/s) | 20260506 14:27:35.026 | INFO | Analyzing chunk frames = [2067968,2068480) | 20260506 14:27:35.026 | INFO | MemInfo: { | 20260506 14:27:35.026 | INFO | "gpu" : " 17 GB", | 20260506 14:27:35.026 | INFO | "huge" : "", | 20260506 14:27:35.026 | INFO | "huge_shared" : " 27 GB", | 20260506 14:27:35.026 | INFO | "malloc" : " 14 GB", | 20260506 14:27:35.026 | INFO | "pinned" : " 29 GB" | 20260506 14:27:35.026 | INFO | } | 20260506 14:27:35.026 | INFO | Perf report for the last 5.13868 seconds: | 20260506 14:27:35.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:27:35.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:35.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:35.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:35.026 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:27:35.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:35.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:35.026 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:27:35.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:35.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:35.026 | INFO | Uploaded 36GiB to GPU in 2.02267s (17.7983GiB/s) | 20260506 14:27:35.026 | INFO | Downloaded 34GiB to GPU in 1.96811s (17.2755GiB/s) | 20260506 14:27:35.140 | INFO | Starting a round of Baz IO | 20260506 14:27:38.294 | INFO | Populating RTMetrics report | 20260506 14:27:38.295 | INFO | RTMetrics report populated | 20260506 14:27:38.295 | INFO | Opening tmp RTMetrics json file | 20260506 14:27:38.321 | INFO | Serializing RTMetrics to json | 20260506 14:27:38.366 | INFO | Serialization done, swapping tmp file now... | 20260506 14:27:38.395 | INFO | RTMetrics done | 20260506 14:27:38.483 | INFO | Finished a round of Baz IO: Wrote 2.115 GiB in 3.343 seconds | 20260506 14:27:38.483 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.29% / 0.01% / 61.70% | 20260506 14:27:38.483 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.55683% / 1.98682e-05% / 6.75519e-05% | 20260506 14:27:38.483 | INFO | Baz IO overall: | 20260506 14:27:38.483 | INFO | BytesWritten : 1067328706314 | 20260506 14:27:38.483 | INFO | NumEvents : 516470139874 | 20260506 14:27:40.164 | INFO | Analyzing chunk frames = [2068480,2068992) | 20260506 14:27:40.164 | INFO | MemInfo: { | 20260506 14:27:40.164 | INFO | "gpu" : " 17 GB", | 20260506 14:27:40.164 | INFO | "huge" : "", | 20260506 14:27:40.164 | INFO | "huge_shared" : " 27 GB", | 20260506 14:27:40.164 | INFO | "malloc" : " 14 GB", | 20260506 14:27:40.164 | INFO | "pinned" : " 29 GB" | 20260506 14:27:40.164 | INFO | } | 20260506 14:27:40.164 | INFO | Perf report for the last 5.13791 seconds: | 20260506 14:27:40.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:27:40.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:40.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:40.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:40.164 | INFO | ANALYSIS : 63.0%, 2.9, 95.2ms, 0 | 20260506 14:27:40.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:40.164 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260506 14:27:40.164 | INFO | PRE_HQ : 61.1%, 1.9, 59.6ms, 0 | 20260506 14:27:40.164 | INFO | BAZWRITER : 64.5%, 4.8, 158.5ms, 0 | 20260506 14:27:40.164 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 14:27:40.164 | INFO | Uploaded 36GiB to GPU in 2.08057s (17.3029GiB/s) | 20260506 14:27:40.164 | INFO | Downloaded 38GiB to GPU in 2.23161s (17.028GiB/s) | 20260506 14:27:45.304 | INFO | Analyzing chunk frames = [2068992,2069504) | 20260506 14:27:45.304 | INFO | MemInfo: { | 20260506 14:27:45.304 | INFO | "gpu" : " 17 GB", | 20260506 14:27:45.304 | INFO | "huge" : "", | 20260506 14:27:45.304 | INFO | "huge_shared" : " 27 GB", | 20260506 14:27:45.304 | INFO | "malloc" : " 14 GB", | 20260506 14:27:45.304 | INFO | "pinned" : " 29 GB" | 20260506 14:27:45.304 | INFO | } | 20260506 14:27:45.304 | INFO | Perf report for the last 5.14008 seconds: | 20260506 14:27:45.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:27:45.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:45.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:45.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:45.304 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:27:45.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:45.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:45.304 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:27:45.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:45.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:45.304 | INFO | Uploaded 36GiB to GPU in 2.04231s (17.6271GiB/s) | 20260506 14:27:45.304 | INFO | Downloaded 34GiB to GPU in 1.95058s (17.4307GiB/s) | 20260506 14:27:50.443 | INFO | Analyzing chunk frames = [2069504,2070016) | 20260506 14:27:50.443 | INFO | MemInfo: { | 20260506 14:27:50.443 | INFO | "gpu" : " 17 GB", | 20260506 14:27:50.443 | INFO | "huge" : "", | 20260506 14:27:50.443 | INFO | "huge_shared" : " 27 GB", | 20260506 14:27:50.443 | INFO | "malloc" : " 14 GB", | 20260506 14:27:50.443 | INFO | "pinned" : " 29 GB" | 20260506 14:27:50.443 | INFO | } | 20260506 14:27:50.443 | INFO | Perf report for the last 5.13931 seconds: | 20260506 14:27:50.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:27:50.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:50.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:50.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:50.444 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 14:27:50.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:50.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:50.444 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:27:50.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:50.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:50.444 | INFO | Uploaded 36GiB to GPU in 1.98741s (18.114GiB/s) | 20260506 14:27:50.444 | INFO | Downloaded 34GiB to GPU in 1.93908s (17.5341GiB/s) | 20260506 14:27:55.582 | INFO | Analyzing chunk frames = [2070016,2070528) | 20260506 14:27:55.582 | INFO | MemInfo: { | 20260506 14:27:55.582 | INFO | "gpu" : " 17 GB", | 20260506 14:27:55.582 | INFO | "huge" : "", | 20260506 14:27:55.582 | INFO | "huge_shared" : " 27 GB", | 20260506 14:27:55.582 | INFO | "malloc" : " 14 GB", | 20260506 14:27:55.582 | INFO | "pinned" : " 29 GB" | 20260506 14:27:55.582 | INFO | } | 20260506 14:27:55.583 | INFO | Perf report for the last 5.13908 seconds: | 20260506 14:27:55.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:27:55.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:55.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:55.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:55.583 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:27:55.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:55.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:55.583 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:27:55.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:55.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:27:55.583 | INFO | Uploaded 36GiB to GPU in 1.99079s (18.0833GiB/s) | 20260506 14:27:55.583 | INFO | Downloaded 34GiB to GPU in 1.90367s (17.8603GiB/s) | 20260506 14:28:00.722 | INFO | Analyzing chunk frames = [2070528,2071040) | 20260506 14:28:00.722 | INFO | MemInfo: { | 20260506 14:28:00.722 | INFO | "gpu" : " 17 GB", | 20260506 14:28:00.722 | INFO | "huge" : "", | 20260506 14:28:00.722 | INFO | "huge_shared" : " 27 GB", | 20260506 14:28:00.722 | INFO | "malloc" : " 14 GB", | 20260506 14:28:00.722 | INFO | "pinned" : " 29 GB" | 20260506 14:28:00.722 | INFO | } | 20260506 14:28:00.722 | INFO | Perf report for the last 5.13928 seconds: | 20260506 14:28:00.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:28:00.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:00.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:00.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:00.722 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:28:00.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:00.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:00.722 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:28:00.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:00.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:00.722 | INFO | Uploaded 36GiB to GPU in 2.00623s (17.9441GiB/s) | 20260506 14:28:00.722 | INFO | Downloaded 34GiB to GPU in 1.93077s (17.6095GiB/s) | 20260506 14:28:05.861 | INFO | Analyzing chunk frames = [2071040,2071552) | 20260506 14:28:05.861 | INFO | MemInfo: { | 20260506 14:28:05.861 | INFO | "gpu" : " 17 GB", | 20260506 14:28:05.861 | INFO | "huge" : "", | 20260506 14:28:05.861 | INFO | "huge_shared" : " 27 GB", | 20260506 14:28:05.861 | INFO | "malloc" : " 14 GB", | 20260506 14:28:05.861 | INFO | "pinned" : " 29 GB" | 20260506 14:28:05.861 | INFO | } | 20260506 14:28:05.861 | INFO | Perf report for the last 5.13908 seconds: | 20260506 14:28:05.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:28:05.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:05.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:05.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:05.861 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 14:28:05.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:05.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:05.861 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 14:28:05.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:05.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:05.861 | INFO | Uploaded 36GiB to GPU in 2.08355s (17.2782GiB/s) | 20260506 14:28:05.861 | INFO | Downloaded 34GiB to GPU in 1.97404s (17.2235GiB/s) | 20260506 14:28:11.001 | INFO | Analyzing chunk frames = [2071552,2072064) | 20260506 14:28:11.001 | INFO | MemInfo: { | 20260506 14:28:11.001 | INFO | "gpu" : " 17 GB", | 20260506 14:28:11.001 | INFO | "huge" : "", | 20260506 14:28:11.001 | INFO | "huge_shared" : " 27 GB", | 20260506 14:28:11.001 | INFO | "malloc" : " 14 GB", | 20260506 14:28:11.001 | INFO | "pinned" : " 29 GB" | 20260506 14:28:11.001 | INFO | } | 20260506 14:28:11.001 | INFO | Perf report for the last 5.14029 seconds: | 20260506 14:28:11.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:28:11.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:11.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:11.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:11.001 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:28:11.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:11.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:11.001 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:28:11.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:11.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:11.002 | INFO | Uploaded 36GiB to GPU in 2.01427s (17.8725GiB/s) | 20260506 14:28:11.002 | INFO | Downloaded 34GiB to GPU in 1.97896s (17.1807GiB/s) | 20260506 14:28:16.139 | INFO | Analyzing chunk frames = [2072064,2072576) | 20260506 14:28:16.140 | INFO | MemInfo: { | 20260506 14:28:16.140 | INFO | "gpu" : " 17 GB", | 20260506 14:28:16.140 | INFO | "huge" : "", | 20260506 14:28:16.140 | INFO | "huge_shared" : " 27 GB", | 20260506 14:28:16.140 | INFO | "malloc" : " 14 GB", | 20260506 14:28:16.140 | INFO | "pinned" : " 29 GB" | 20260506 14:28:16.140 | INFO | } | 20260506 14:28:16.140 | INFO | Perf report for the last 5.13847 seconds: | 20260506 14:28:16.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:28:16.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:16.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:16.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:16.140 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:28:16.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:16.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:16.140 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:28:16.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:16.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:16.140 | INFO | Uploaded 36GiB to GPU in 2.02937s (17.7395GiB/s) | 20260506 14:28:16.140 | INFO | Downloaded 34GiB to GPU in 1.97131s (17.2474GiB/s) | 20260506 14:28:16.140 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:28:16.140 | INFO | Total mem usage: 17.4 GiB | 20260506 14:28:16.140 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:28:16.140 | INFO | Recent High Water Marks Sum: 17 GiB (Savings: -2.7%) | 20260506 14:28:16.140 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:28:16.140 | INFO | High Water Breakdown: | 20260506 14:28:16.140 | INFO | 62.4% : 10.6 GiB ( 96.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:28:16.140 | INFO | 37.5% : 6.35 GiB ( 94.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:28:16.140 | INFO | 0.117% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:28:16.140 | INFO | ------------------------------------------ | 20260506 14:28:16.140 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:28:16.140 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:28:16.140 | INFO | Total mem usage: 26.5 GiB | 20260506 14:28:16.140 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:28:16.140 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:28:16.140 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:28:16.140 | INFO | High Water Breakdown: | 20260506 14:28:16.140 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:28:16.140 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:28:16.140 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:28:16.140 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:28:16.140 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:28:16.140 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:28:16.140 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:28:16.140 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:28:16.140 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:28:16.140 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:28:16.140 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:28:16.140 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:28:16.140 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:28:16.140 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:28:16.140 | INFO | ------------------------------------------ | 20260506 14:28:16.140 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:28:16.140 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:28:16.140 | INFO | Total mem usage: 25.6 GiB | 20260506 14:28:16.140 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:28:16.140 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:28:16.141 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:28:16.141 | INFO | High Water Breakdown: | 20260506 14:28:16.141 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:28:16.141 | INFO | ------------------------------------------ | 20260506 14:28:16.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:28:16.141 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:28:16.141 | INFO | Total mem usage: 15.7 GiB | 20260506 14:28:16.141 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:28:16.141 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:28:16.141 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 14:28:16.141 | INFO | High Water Breakdown: | 20260506 14:28:16.141 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:28:16.141 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:28:16.141 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:28:16.141 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:28:16.141 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:28:16.141 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:28:16.141 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:28:16.141 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:28:16.141 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:28:16.141 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:28:16.141 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:28:16.141 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:28:16.141 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:28:16.141 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:28:16.141 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:28:16.141 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:28:16.141 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:28:16.141 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:28:16.141 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:28:16.141 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:28:16.141 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:28:16.141 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:28:16.141 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:28:16.141 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:28:16.141 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:28:16.141 | INFO | ------------------------------------------ | 20260506 14:28:16.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:28:16.141 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:28:16.141 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:28:16.141 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:28:16.141 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.59e+01, 1.35e+01 / 2.17e+00 | 20260506 14:28:16.141 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.02e-01, 1.02e-02 / 8.09e-03 | 20260506 14:28:16.141 | INFO | PerfReport: 11.57%, Baseline -- 3.64e+00 / 4.51e+00, 5.62e-01 / 2.19e-01 | 20260506 14:28:16.141 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.50e+00, 1.87e-01 / 1.05e-01 | 20260506 14:28:16.141 | INFO | PerfReport: 15.93%, DME -- 4.01e+01 / 4.50e+01, 4.17e+00 / 1.99e+00 | 20260506 14:28:16.141 | INFO | PerfReport: 56.61%, FrameLabeling -- 1.78e+01 / 1.99e+01, 1.38e+00 / 1.13e+00 | 20260506 14:28:16.141 | INFO | PerfReport: 7.48%, PulseAccumulating -- 2.36e+00 / 2.96e+00, 4.15e-01 / 2.59e-01 | 20260506 14:28:16.141 | INFO | PerfReport: 3.61%, Metrics -- 1.14e+00 / 1.49e+00, 2.89e-01 / 2.24e-01 | 20260506 14:28:16.141 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.98e-01 / 4.51e-01, 6.26e-02 / 3.19e-02 | 20260506 14:28:16.141 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:28:16.141 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:28:16.141 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:28:16.141 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 14:28:16.141 | INFO | PerfReport: 21.58%, Upload -- 2.11e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260506 14:28:16.141 | INFO | PerfReport: 21.13%, Download -- 2.07e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 14:28:16.141 | INFO | PerfReport: 32.19%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 14:28:16.141 | INFO | PerfReport: 1.70%, LockUpload -- 1.66e+00 / 0.00e+00, 6.23e+00 / 0.00e+00 | 20260506 14:28:16.141 | INFO | PerfReport: 1.03%, LockDownload -- 1.01e+00 / 0.00e+00, 2.54e+00 / 0.00e+00 | 20260506 14:28:16.141 | INFO | PerfReport: 22.36%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:28:16.141 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260506 14:28:16.257 | INFO | Starting a round of Baz IO | 20260506 14:28:19.411 | INFO | Populating RTMetrics report | 20260506 14:28:19.411 | INFO | RTMetrics report populated | 20260506 14:28:19.411 | INFO | Opening tmp RTMetrics json file | 20260506 14:28:19.511 | INFO | Serializing RTMetrics to json | 20260506 14:28:19.516 | INFO | Serialization done, swapping tmp file now... | 20260506 14:28:19.517 | INFO | RTMetrics done | 20260506 14:28:19.844 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.586 seconds | 20260506 14:28:19.844 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.27% / 0.01% / 61.71% | 20260506 14:28:19.844 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.54806% / 1.19209e-05% / 6.35783e-05% | 20260506 14:28:19.844 | INFO | Baz IO overall: | 20260506 14:28:19.844 | INFO | BytesWritten : 1069598270418 | 20260506 14:28:19.844 | INFO | NumEvents : 517570182598 | 20260506 14:28:21.280 | INFO | Analyzing chunk frames = [2072576,2073088) | 20260506 14:28:21.280 | INFO | MemInfo: { | 20260506 14:28:21.280 | INFO | "gpu" : " 17 GB", | 20260506 14:28:21.280 | INFO | "huge" : "", | 20260506 14:28:21.280 | INFO | "huge_shared" : " 27 GB", | 20260506 14:28:21.280 | INFO | "malloc" : " 14 GB", | 20260506 14:28:21.280 | INFO | "pinned" : " 29 GB" | 20260506 14:28:21.280 | INFO | } | 20260506 14:28:21.281 | INFO | Perf report for the last 5.14078 seconds: | 20260506 14:28:21.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:28:21.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:21.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:21.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:21.281 | INFO | ANALYSIS : 63.0%, 2.9, 95.4ms, 0 | 20260506 14:28:21.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:21.281 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 14:28:21.281 | INFO | PRE_HQ : 60.9%, 1.9, 59.5ms, 0 | 20260506 14:28:21.281 | INFO | BAZWRITER : 69.1%, 4.8, 169.8ms, 0 | 20260506 14:28:21.281 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 14:28:21.281 | INFO | Uploaded 36GiB to GPU in 2.06057s (17.4709GiB/s) | 20260506 14:28:21.281 | INFO | Downloaded 38GiB to GPU in 2.2335s (17.0136GiB/s) | 20260506 14:28:26.419 | INFO | Analyzing chunk frames = [2073088,2073600) | 20260506 14:28:26.419 | INFO | MemInfo: { | 20260506 14:28:26.419 | INFO | "gpu" : " 17 GB", | 20260506 14:28:26.419 | INFO | "huge" : "", | 20260506 14:28:26.419 | INFO | "huge_shared" : " 27 GB", | 20260506 14:28:26.419 | INFO | "malloc" : " 14 GB", | 20260506 14:28:26.419 | INFO | "pinned" : " 29 GB" | 20260506 14:28:26.419 | INFO | } | 20260506 14:28:26.419 | INFO | Perf report for the last 5.13878 seconds: | 20260506 14:28:26.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:28:26.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:26.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:26.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:26.419 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:28:26.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:26.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:26.419 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:28:26.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:26.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:26.420 | INFO | Uploaded 36GiB to GPU in 2.00122s (17.989GiB/s) | 20260506 14:28:26.420 | INFO | Downloaded 34GiB to GPU in 1.93608s (17.5612GiB/s) | 20260506 14:28:31.557 | INFO | Analyzing chunk frames = [2073600,2074112) | 20260506 14:28:31.557 | INFO | MemInfo: { | 20260506 14:28:31.557 | INFO | "gpu" : " 17 GB", | 20260506 14:28:31.557 | INFO | "huge" : "", | 20260506 14:28:31.557 | INFO | "huge_shared" : " 27 GB", | 20260506 14:28:31.557 | INFO | "malloc" : " 14 GB", | 20260506 14:28:31.557 | INFO | "pinned" : " 29 GB" | 20260506 14:28:31.557 | INFO | } | 20260506 14:28:31.557 | INFO | Perf report for the last 5.13812 seconds: | 20260506 14:28:31.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:28:31.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:31.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:31.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:31.558 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 14:28:31.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:31.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:31.558 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260506 14:28:31.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:31.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:31.558 | INFO | Uploaded 36GiB to GPU in 1.99797s (18.0183GiB/s) | 20260506 14:28:31.558 | INFO | Downloaded 34GiB to GPU in 1.9372s (17.5511GiB/s) | 20260506 14:28:36.697 | INFO | Analyzing chunk frames = [2074112,2074624) | 20260506 14:28:36.697 | INFO | MemInfo: { | 20260506 14:28:36.697 | INFO | "gpu" : " 17 GB", | 20260506 14:28:36.697 | INFO | "huge" : "", | 20260506 14:28:36.698 | INFO | "huge_shared" : " 27 GB", | 20260506 14:28:36.698 | INFO | "malloc" : " 14 GB", | 20260506 14:28:36.698 | INFO | "pinned" : " 29 GB" | 20260506 14:28:36.698 | INFO | } | 20260506 14:28:36.698 | INFO | Perf report for the last 5.14027 seconds: | 20260506 14:28:36.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:28:36.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:36.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:36.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:36.698 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:28:36.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:36.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:36.698 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:28:36.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:36.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:36.698 | INFO | Uploaded 36GiB to GPU in 1.97048s (18.2697GiB/s) | 20260506 14:28:36.698 | INFO | Downloaded 34GiB to GPU in 1.91151s (17.787GiB/s) | 20260506 14:28:41.836 | INFO | Analyzing chunk frames = [2074624,2075136) | 20260506 14:28:41.836 | INFO | MemInfo: { | 20260506 14:28:41.836 | INFO | "gpu" : " 17 GB", | 20260506 14:28:41.836 | INFO | "huge" : "", | 20260506 14:28:41.837 | INFO | "huge_shared" : " 27 GB", | 20260506 14:28:41.837 | INFO | "malloc" : " 14 GB", | 20260506 14:28:41.837 | INFO | "pinned" : " 29 GB" | 20260506 14:28:41.837 | INFO | } | 20260506 14:28:41.837 | INFO | Perf report for the last 5.139 seconds: | 20260506 14:28:41.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:28:41.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:41.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:41.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:41.837 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:28:41.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:41.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:41.837 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:28:41.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:41.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:41.837 | INFO | Uploaded 36GiB to GPU in 1.99622s (18.0341GiB/s) | 20260506 14:28:41.837 | INFO | Downloaded 34GiB to GPU in 1.9343s (17.5774GiB/s) | 20260506 14:28:46.976 | INFO | Analyzing chunk frames = [2075136,2075648) | 20260506 14:28:46.976 | INFO | MemInfo: { | 20260506 14:28:46.976 | INFO | "gpu" : " 17 GB", | 20260506 14:28:46.976 | INFO | "huge" : "", | 20260506 14:28:46.976 | INFO | "huge_shared" : " 27 GB", | 20260506 14:28:46.976 | INFO | "malloc" : " 14 GB", | 20260506 14:28:46.976 | INFO | "pinned" : " 29 GB" | 20260506 14:28:46.976 | INFO | } | 20260506 14:28:46.976 | INFO | Perf report for the last 5.1395 seconds: | 20260506 14:28:46.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:28:46.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:46.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:46.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:46.976 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 14:28:46.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:46.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:46.976 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 14:28:46.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:46.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:46.977 | INFO | Uploaded 36GiB to GPU in 2.0557s (17.5123GiB/s) | 20260506 14:28:46.977 | INFO | Downloaded 34GiB to GPU in 1.92754s (17.6391GiB/s) | 20260506 14:28:52.116 | INFO | Analyzing chunk frames = [2075648,2076160) | 20260506 14:28:52.116 | INFO | MemInfo: { | 20260506 14:28:52.116 | INFO | "gpu" : " 17 GB", | 20260506 14:28:52.116 | INFO | "huge" : "", | 20260506 14:28:52.116 | INFO | "huge_shared" : " 27 GB", | 20260506 14:28:52.116 | INFO | "malloc" : " 14 GB", | 20260506 14:28:52.116 | INFO | "pinned" : " 29 GB" | 20260506 14:28:52.116 | INFO | } | 20260506 14:28:52.116 | INFO | Perf report for the last 5.14001 seconds: | 20260506 14:28:52.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:28:52.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:52.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:52.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:52.116 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 14:28:52.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:52.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:52.116 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:28:52.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:52.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:52.117 | INFO | Uploaded 36GiB to GPU in 2.01625s (17.855GiB/s) | 20260506 14:28:52.117 | INFO | Downloaded 34GiB to GPU in 2.0051s (16.9568GiB/s) | 20260506 14:28:57.255 | INFO | Analyzing chunk frames = [2076160,2076672) | 20260506 14:28:57.255 | INFO | MemInfo: { | 20260506 14:28:57.255 | INFO | "gpu" : " 17 GB", | 20260506 14:28:57.255 | INFO | "huge" : "", | 20260506 14:28:57.255 | INFO | "huge_shared" : " 27 GB", | 20260506 14:28:57.255 | INFO | "malloc" : " 14 GB", | 20260506 14:28:57.255 | INFO | "pinned" : " 29 GB" | 20260506 14:28:57.255 | INFO | } | 20260506 14:28:57.255 | INFO | Perf report for the last 5.13874 seconds: | 20260506 14:28:57.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:28:57.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:57.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:57.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:57.255 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:28:57.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:57.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:57.255 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260506 14:28:57.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:57.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:28:57.255 | INFO | Uploaded 36GiB to GPU in 2.00942s (17.9156GiB/s) | 20260506 14:28:57.255 | INFO | Downloaded 34GiB to GPU in 1.97458s (17.2188GiB/s) | 20260506 14:28:57.374 | INFO | Starting a round of Baz IO | 20260506 14:29:00.524 | INFO | Populating RTMetrics report | 20260506 14:29:00.524 | INFO | RTMetrics report populated | 20260506 14:29:00.524 | INFO | Opening tmp RTMetrics json file | 20260506 14:29:00.765 | INFO | Serializing RTMetrics to json | 20260506 14:29:00.830 | INFO | Serialization done, swapping tmp file now... | 20260506 14:29:00.845 | INFO | RTMetrics done | 20260506 14:29:01.153 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 3.778 seconds | 20260506 14:29:01.153 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.26% / 0.01% / 61.73% | 20260506 14:29:01.153 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.53697% / 1.19209e-05% / 6.35783e-05% | 20260506 14:29:01.153 | INFO | Baz IO overall: | 20260506 14:29:01.153 | INFO | BytesWritten : 1071867393470 | 20260506 14:29:01.153 | INFO | NumEvents : 518669718675 | 20260506 14:29:02.394 | INFO | Analyzing chunk frames = [2076672,2077184) | 20260506 14:29:02.394 | INFO | MemInfo: { | 20260506 14:29:02.394 | INFO | "gpu" : " 17 GB", | 20260506 14:29:02.394 | INFO | "huge" : "", | 20260506 14:29:02.394 | INFO | "huge_shared" : " 27 GB", | 20260506 14:29:02.394 | INFO | "malloc" : " 14 GB", | 20260506 14:29:02.394 | INFO | "pinned" : " 29 GB" | 20260506 14:29:02.394 | INFO | } | 20260506 14:29:02.395 | INFO | Perf report for the last 5.13945 seconds: | 20260506 14:29:02.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:29:02.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:02.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:02.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:02.395 | INFO | ANALYSIS : 63.0%, 2.9, 95.4ms, 0 | 20260506 14:29:02.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:02.395 | INFO | RT_METRICS : 50.3%, 1.0, 26.4ms, 0 | 20260506 14:29:02.395 | INFO | PRE_HQ : 60.9%, 1.9, 59.5ms, 0 | 20260506 14:29:02.395 | INFO | BAZWRITER : 72.9%, 4.8, 180.8ms, 0 | 20260506 14:29:02.395 | INFO | BAZ_SAMPLING : 6.1%, 1.0, 3.2ms, 0 | 20260506 14:29:02.395 | INFO | Uploaded 36GiB to GPU in 2.04631s (17.5926GiB/s) | 20260506 14:29:02.395 | INFO | Downloaded 38GiB to GPU in 2.23621s (16.993GiB/s) | 20260506 14:29:07.534 | INFO | Analyzing chunk frames = [2077184,2077696) | 20260506 14:29:07.534 | INFO | MemInfo: { | 20260506 14:29:07.534 | INFO | "gpu" : " 17 GB", | 20260506 14:29:07.534 | INFO | "huge" : "", | 20260506 14:29:07.534 | INFO | "huge_shared" : " 27 GB", | 20260506 14:29:07.534 | INFO | "malloc" : " 14 GB", | 20260506 14:29:07.534 | INFO | "pinned" : " 29 GB" | 20260506 14:29:07.534 | INFO | } | 20260506 14:29:07.534 | INFO | Perf report for the last 5.13969 seconds: | 20260506 14:29:07.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:29:07.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:07.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:07.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:07.534 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 14:29:07.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:07.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:07.534 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:29:07.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:07.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:07.534 | INFO | Uploaded 36GiB to GPU in 1.98734s (18.1146GiB/s) | 20260506 14:29:07.534 | INFO | Downloaded 34GiB to GPU in 1.94279s (17.5006GiB/s) | 20260506 14:29:12.673 | INFO | Analyzing chunk frames = [2077696,2078208) | 20260506 14:29:12.673 | INFO | MemInfo: { | 20260506 14:29:12.673 | INFO | "gpu" : " 17 GB", | 20260506 14:29:12.673 | INFO | "huge" : "", | 20260506 14:29:12.673 | INFO | "huge_shared" : " 27 GB", | 20260506 14:29:12.673 | INFO | "malloc" : " 14 GB", | 20260506 14:29:12.673 | INFO | "pinned" : " 29 GB" | 20260506 14:29:12.673 | INFO | } | 20260506 14:29:12.673 | INFO | Perf report for the last 5.13857 seconds: | 20260506 14:29:12.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:29:12.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:12.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:12.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:12.673 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 14:29:12.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:12.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:12.673 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:29:12.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:12.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:12.673 | INFO | Uploaded 36GiB to GPU in 1.96843s (18.2887GiB/s) | 20260506 14:29:12.673 | INFO | Downloaded 34GiB to GPU in 1.94451s (17.4851GiB/s) | 20260506 14:29:17.811 | INFO | Analyzing chunk frames = [2078208,2078720) | 20260506 14:29:17.812 | INFO | MemInfo: { | 20260506 14:29:17.812 | INFO | "gpu" : " 17 GB", | 20260506 14:29:17.812 | INFO | "huge" : "", | 20260506 14:29:17.812 | INFO | "huge_shared" : " 27 GB", | 20260506 14:29:17.812 | INFO | "malloc" : " 14 GB", | 20260506 14:29:17.812 | INFO | "pinned" : " 29 GB" | 20260506 14:29:17.812 | INFO | } | 20260506 14:29:17.812 | INFO | Perf report for the last 5.13895 seconds: | 20260506 14:29:17.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:29:17.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:17.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:17.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:17.812 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:29:17.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:17.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:17.812 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:29:17.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:17.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:17.812 | INFO | Uploaded 36GiB to GPU in 1.97816s (18.1988GiB/s) | 20260506 14:29:17.812 | INFO | Downloaded 34GiB to GPU in 1.91588s (17.7464GiB/s) | 20260506 14:29:22.951 | INFO | Analyzing chunk frames = [2078720,2079232) | 20260506 14:29:22.951 | INFO | MemInfo: { | 20260506 14:29:22.951 | INFO | "gpu" : " 17 GB", | 20260506 14:29:22.951 | INFO | "huge" : "", | 20260506 14:29:22.951 | INFO | "huge_shared" : " 27 GB", | 20260506 14:29:22.951 | INFO | "malloc" : " 14 GB", | 20260506 14:29:22.951 | INFO | "pinned" : " 29 GB" | 20260506 14:29:22.951 | INFO | } | 20260506 14:29:22.951 | INFO | Perf report for the last 5.13918 seconds: | 20260506 14:29:22.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:29:22.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:22.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:22.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:22.951 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 14:29:22.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:22.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:22.951 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:29:22.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:22.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:22.951 | INFO | Uploaded 36GiB to GPU in 2.01203s (17.8924GiB/s) | 20260506 14:29:22.951 | INFO | Downloaded 34GiB to GPU in 1.94275s (17.501GiB/s) | 20260506 14:29:28.091 | INFO | Analyzing chunk frames = [2079232,2079744) | 20260506 14:29:28.091 | INFO | MemInfo: { | 20260506 14:29:28.091 | INFO | "gpu" : " 17 GB", | 20260506 14:29:28.091 | INFO | "huge" : "", | 20260506 14:29:28.091 | INFO | "huge_shared" : " 27 GB", | 20260506 14:29:28.091 | INFO | "malloc" : " 14 GB", | 20260506 14:29:28.091 | INFO | "pinned" : " 29 GB" | 20260506 14:29:28.091 | INFO | } | 20260506 14:29:28.091 | INFO | Perf report for the last 5.14022 seconds: | 20260506 14:29:28.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:29:28.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:28.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:28.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:28.091 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 14:29:28.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:28.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:28.091 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:29:28.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:28.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:28.091 | INFO | Uploaded 36GiB to GPU in 2.02749s (17.756GiB/s) | 20260506 14:29:28.091 | INFO | Downloaded 34GiB to GPU in 1.91213s (17.7812GiB/s) | 20260506 14:29:33.231 | INFO | Analyzing chunk frames = [2079744,2080256) | 20260506 14:29:33.231 | INFO | MemInfo: { | 20260506 14:29:33.231 | INFO | "gpu" : " 17 GB", | 20260506 14:29:33.231 | INFO | "huge" : "", | 20260506 14:29:33.231 | INFO | "huge_shared" : " 27 GB", | 20260506 14:29:33.231 | INFO | "malloc" : " 14 GB", | 20260506 14:29:33.231 | INFO | "pinned" : " 29 GB" | 20260506 14:29:33.231 | INFO | } | 20260506 14:29:33.231 | INFO | Perf report for the last 5.13962 seconds: | 20260506 14:29:33.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:29:33.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:33.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:33.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:33.231 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:29:33.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:33.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:33.231 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 14:29:33.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:33.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:33.231 | INFO | Uploaded 36GiB to GPU in 2.00006s (17.9994GiB/s) | 20260506 14:29:33.231 | INFO | Downloaded 34GiB to GPU in 1.99826s (17.0148GiB/s) | 20260506 14:29:38.370 | INFO | Analyzing chunk frames = [2080256,2080768) | 20260506 14:29:38.370 | INFO | MemInfo: { | 20260506 14:29:38.370 | INFO | "gpu" : " 17 GB", | 20260506 14:29:38.370 | INFO | "huge" : "", | 20260506 14:29:38.370 | INFO | "huge_shared" : " 27 GB", | 20260506 14:29:38.370 | INFO | "malloc" : " 14 GB", | 20260506 14:29:38.370 | INFO | "pinned" : " 29 GB" | 20260506 14:29:38.370 | INFO | } | 20260506 14:29:38.370 | INFO | Perf report for the last 5.13954 seconds: | 20260506 14:29:38.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:29:38.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:38.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:38.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:38.370 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 14:29:38.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:38.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:38.370 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:29:38.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:38.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:38.371 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:29:38.371 | INFO | Total mem usage: 17.4 GiB | 20260506 14:29:38.371 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:29:38.371 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -12.2%) | 20260506 14:29:38.371 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:29:38.371 | INFO | High Water Breakdown: | 20260506 14:29:38.371 | INFO | Uploaded 36GiB to GPU in 2.01941s (17.827GiB/s) | 20260506 14:29:38.371 | INFO | 61.1% : 9.47 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:29:38.371 | INFO | 38.8% : 6.02 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:29:38.371 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:29:38.371 | INFO | ------------------------------------------ | 20260506 14:29:38.371 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:29:38.371 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:29:38.371 | INFO | Total mem usage: 26.5 GiB | 20260506 14:29:38.371 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:29:38.371 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:29:38.371 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:29:38.371 | INFO | High Water Breakdown: | 20260506 14:29:38.371 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:29:38.371 | INFO | Downloaded 34GiB to GPU in 1.97752s (17.1933GiB/s) | 20260506 14:29:38.371 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:29:38.371 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:29:38.371 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:29:38.371 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:29:38.371 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:29:38.371 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:29:38.371 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:29:38.371 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:29:38.371 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:29:38.371 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:29:38.371 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:29:38.371 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:29:38.371 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:29:38.371 | INFO | ------------------------------------------ | 20260506 14:29:38.371 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:29:38.371 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:29:38.371 | INFO | Total mem usage: 25.6 GiB | 20260506 14:29:38.371 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:29:38.371 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:29:38.371 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:29:38.371 | INFO | High Water Breakdown: | 20260506 14:29:38.371 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:29:38.371 | INFO | ------------------------------------------ | 20260506 14:29:38.371 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:29:38.371 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:29:38.371 | INFO | Total mem usage: 15.7 GiB | 20260506 14:29:38.371 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:29:38.371 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:29:38.371 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 14:29:38.371 | INFO | High Water Breakdown: | 20260506 14:29:38.372 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:29:38.372 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:29:38.372 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:29:38.372 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:29:38.372 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:29:38.372 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:29:38.372 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:29:38.372 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:29:38.372 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:29:38.372 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:29:38.372 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:29:38.372 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:29:38.372 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:29:38.372 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:29:38.372 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:29:38.372 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:29:38.372 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:29:38.372 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:29:38.372 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:29:38.372 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:29:38.372 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:29:38.372 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:29:38.372 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:29:38.372 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:29:38.372 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:29:38.372 | INFO | ------------------------------------------ | 20260506 14:29:38.372 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:29:38.372 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:29:38.372 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:29:38.372 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:29:38.372 | INFO | PerfReport: Full Execution -- 3.16e+01 / 7.54e+01, 1.34e+01 / 2.03e+00 | 20260506 14:29:38.372 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.01e-01, 1.04e-02 / 6.27e-03 | 20260506 14:29:38.372 | INFO | PerfReport: 11.58%, Baseline -- 3.66e+00 / 4.41e+00, 5.77e-01 / 3.95e-01 | 20260506 14:29:38.372 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.47e+00, 1.89e-01 / 1.45e-01 | 20260506 14:29:38.372 | INFO | PerfReport: 15.81%, DME -- 3.99e+01 / 4.39e+01, 3.82e+00 / 2.26e+00 | 20260506 14:29:38.372 | INFO | PerfReport: 56.73%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.48e+00 / 1.82e+00 | 20260506 14:29:38.372 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.35e+00 / 2.91e+00, 4.13e-01 / 2.68e-01 | 20260506 14:29:38.372 | INFO | PerfReport: 3.61%, Metrics -- 1.14e+00 / 1.54e+00, 2.95e-01 / 2.24e-01 | 20260506 14:29:38.372 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.98e-01 / 4.50e-01, 6.29e-02 / 2.58e-02 | 20260506 14:29:38.372 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:29:38.372 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:29:38.372 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:29:38.372 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 14:29:38.372 | INFO | PerfReport: 21.42%, Upload -- 2.09e+01 / 0.00e+00, 6.91e+00 / 0.00e+00 | 20260506 14:29:38.372 | INFO | PerfReport: 21.14%, Download -- 2.07e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 14:29:38.372 | INFO | PerfReport: 32.30%, Compute -- 3.16e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 14:29:38.372 | INFO | PerfReport: 1.46%, LockUpload -- 1.43e+00 / 0.00e+00, 6.20e+00 / 0.00e+00 | 20260506 14:29:38.372 | INFO | PerfReport: 1.21%, LockDownload -- 1.18e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 14:29:38.372 | INFO | PerfReport: 22.44%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:29:38.372 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260506 14:29:38.484 | INFO | Starting a round of Baz IO | 20260506 14:29:41.642 | INFO | Populating RTMetrics report | 20260506 14:29:41.642 | INFO | RTMetrics report populated | 20260506 14:29:41.642 | INFO | Opening tmp RTMetrics json file | 20260506 14:29:41.674 | INFO | Serializing RTMetrics to json | 20260506 14:29:41.717 | INFO | Serialization done, swapping tmp file now... | 20260506 14:29:41.718 | INFO | RTMetrics done | 20260506 14:29:41.860 | INFO | Finished a round of Baz IO: Wrote 2.115 GiB in 3.376 seconds | 20260506 14:29:41.860 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.25% / 0.01% / 61.74% | 20260506 14:29:41.860 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.54082% / 2.38419e-05% / 5.5631e-05% | 20260506 14:29:41.860 | INFO | Baz IO overall: | 20260506 14:29:41.860 | INFO | BytesWritten : 1074138748826 | 20260506 14:29:41.860 | INFO | NumEvents : 519771017521 | 20260506 14:29:43.509 | INFO | Analyzing chunk frames = [2080768,2081280) | 20260506 14:29:43.509 | INFO | MemInfo: { | 20260506 14:29:43.509 | INFO | "gpu" : " 17 GB", | 20260506 14:29:43.509 | INFO | "huge" : "", | 20260506 14:29:43.509 | INFO | "huge_shared" : " 27 GB", | 20260506 14:29:43.509 | INFO | "malloc" : " 14 GB", | 20260506 14:29:43.509 | INFO | "pinned" : " 29 GB" | 20260506 14:29:43.509 | INFO | } | 20260506 14:29:43.509 | INFO | Perf report for the last 5.13874 seconds: | 20260506 14:29:43.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:29:43.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:43.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:43.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:43.509 | INFO | ANALYSIS : 63.1%, 2.9, 95.5ms, 0 | 20260506 14:29:43.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:43.509 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 14:29:43.509 | INFO | PRE_HQ : 61.1%, 1.9, 59.4ms, 0 | 20260506 14:29:43.509 | INFO | BAZWRITER : 65.0%, 4.8, 160.9ms, 0 | 20260506 14:29:43.509 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 14:29:43.509 | INFO | Uploaded 36GiB to GPU in 2.07335s (17.3632GiB/s) | 20260506 14:29:43.509 | INFO | Downloaded 38GiB to GPU in 2.23223s (17.0233GiB/s) | 20260506 14:29:48.648 | INFO | Analyzing chunk frames = [2081280,2081792) | 20260506 14:29:48.648 | INFO | MemInfo: { | 20260506 14:29:48.648 | INFO | "gpu" : " 17 GB", | 20260506 14:29:48.648 | INFO | "huge" : "", | 20260506 14:29:48.648 | INFO | "huge_shared" : " 27 GB", | 20260506 14:29:48.648 | INFO | "malloc" : " 14 GB", | 20260506 14:29:48.648 | INFO | "pinned" : " 29 GB" | 20260506 14:29:48.648 | INFO | } | 20260506 14:29:48.648 | INFO | Perf report for the last 5.13901 seconds: | 20260506 14:29:48.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:29:48.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:48.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:48.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:48.648 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:29:48.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:48.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:48.648 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:29:48.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:48.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:48.648 | INFO | Uploaded 36GiB to GPU in 2.00734s (17.9342GiB/s) | 20260506 14:29:48.648 | INFO | Downloaded 34GiB to GPU in 1.94412s (17.4887GiB/s) | 20260506 14:29:53.787 | INFO | Analyzing chunk frames = [2081792,2082304) | 20260506 14:29:53.787 | INFO | MemInfo: { | 20260506 14:29:53.787 | INFO | "gpu" : " 17 GB", | 20260506 14:29:53.787 | INFO | "huge" : "", | 20260506 14:29:53.787 | INFO | "huge_shared" : " 27 GB", | 20260506 14:29:53.787 | INFO | "malloc" : " 14 GB", | 20260506 14:29:53.787 | INFO | "pinned" : " 29 GB" | 20260506 14:29:53.787 | INFO | } | 20260506 14:29:53.787 | INFO | Perf report for the last 5.13914 seconds: | 20260506 14:29:53.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:29:53.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:53.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:53.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:53.787 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 14:29:53.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:53.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:53.787 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:29:53.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:53.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:53.788 | INFO | Uploaded 36GiB to GPU in 1.9666s (18.3057GiB/s) | 20260506 14:29:53.788 | INFO | Downloaded 34GiB to GPU in 1.93327s (17.5867GiB/s) | 20260506 14:29:58.926 | INFO | Analyzing chunk frames = [2082304,2082816) | 20260506 14:29:58.926 | INFO | MemInfo: { | 20260506 14:29:58.926 | INFO | "gpu" : " 17 GB", | 20260506 14:29:58.926 | INFO | "huge" : "", | 20260506 14:29:58.926 | INFO | "huge_shared" : " 27 GB", | 20260506 14:29:58.926 | INFO | "malloc" : " 14 GB", | 20260506 14:29:58.926 | INFO | "pinned" : " 29 GB" | 20260506 14:29:58.926 | INFO | } | 20260506 14:29:58.926 | INFO | Perf report for the last 5.13897 seconds: | 20260506 14:29:58.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:29:58.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:58.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:58.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:58.926 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 14:29:58.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:58.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:58.926 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:29:58.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:58.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:29:58.926 | INFO | Uploaded 36GiB to GPU in 2.02444s (17.7827GiB/s) | 20260506 14:29:58.926 | INFO | Downloaded 34GiB to GPU in 1.92128s (17.6965GiB/s) | 20260506 14:30:04.066 | INFO | Analyzing chunk frames = [2082816,2083328) | 20260506 14:30:04.066 | INFO | MemInfo: { | 20260506 14:30:04.066 | INFO | "gpu" : " 17 GB", | 20260506 14:30:04.066 | INFO | "huge" : "", | 20260506 14:30:04.066 | INFO | "huge_shared" : " 27 GB", | 20260506 14:30:04.066 | INFO | "malloc" : " 14 GB", | 20260506 14:30:04.066 | INFO | "pinned" : " 29 GB" | 20260506 14:30:04.066 | INFO | } | 20260506 14:30:04.067 | INFO | Perf report for the last 5.14026 seconds: | 20260506 14:30:04.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:30:04.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:04.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:04.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:04.067 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 14:30:04.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:04.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:04.067 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:30:04.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:04.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:04.067 | INFO | Uploaded 36GiB to GPU in 1.99386s (18.0554GiB/s) | 20260506 14:30:04.067 | INFO | Downloaded 34GiB to GPU in 1.93885s (17.5362GiB/s) | 20260506 14:30:09.206 | INFO | Analyzing chunk frames = [2083328,2083840) | 20260506 14:30:09.206 | INFO | MemInfo: { | 20260506 14:30:09.206 | INFO | "gpu" : " 17 GB", | 20260506 14:30:09.206 | INFO | "huge" : "", | 20260506 14:30:09.206 | INFO | "huge_shared" : " 27 GB", | 20260506 14:30:09.207 | INFO | "malloc" : " 14 GB", | 20260506 14:30:09.207 | INFO | "pinned" : " 29 GB" | 20260506 14:30:09.207 | INFO | } | 20260506 14:30:09.207 | INFO | Perf report for the last 5.14013 seconds: | 20260506 14:30:09.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:30:09.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:09.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:09.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:09.207 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 14:30:09.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:09.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:09.207 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 14:30:09.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:09.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:09.207 | INFO | Uploaded 36GiB to GPU in 2.03521s (17.6886GiB/s) | 20260506 14:30:09.207 | INFO | Downloaded 34GiB to GPU in 1.94456s (17.4846GiB/s) | 20260506 14:30:14.345 | INFO | Analyzing chunk frames = [2083840,2084352) | 20260506 14:30:14.345 | INFO | MemInfo: { | 20260506 14:30:14.345 | INFO | "gpu" : " 17 GB", | 20260506 14:30:14.345 | INFO | "huge" : "", | 20260506 14:30:14.345 | INFO | "huge_shared" : " 27 GB", | 20260506 14:30:14.345 | INFO | "malloc" : " 14 GB", | 20260506 14:30:14.345 | INFO | "pinned" : " 29 GB" | 20260506 14:30:14.345 | INFO | } | 20260506 14:30:14.346 | INFO | Perf report for the last 5.13901 seconds: | 20260506 14:30:14.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:30:14.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:14.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:14.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:14.346 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 14:30:14.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:14.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:14.346 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 14:30:14.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:14.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:14.346 | INFO | Uploaded 36GiB to GPU in 1.95316s (18.4316GiB/s) | 20260506 14:30:14.346 | INFO | Downloaded 34GiB to GPU in 1.98837s (17.0994GiB/s) | 20260506 14:30:19.484 | INFO | Analyzing chunk frames = [2084352,2084864) | 20260506 14:30:19.484 | INFO | MemInfo: { | 20260506 14:30:19.484 | INFO | "gpu" : " 17 GB", | 20260506 14:30:19.484 | INFO | "huge" : "", | 20260506 14:30:19.484 | INFO | "huge_shared" : " 27 GB", | 20260506 14:30:19.484 | INFO | "malloc" : " 14 GB", | 20260506 14:30:19.484 | INFO | "pinned" : " 29 GB" | 20260506 14:30:19.484 | INFO | } | 20260506 14:30:19.485 | INFO | Perf report for the last 5.13884 seconds: | 20260506 14:30:19.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:30:19.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:19.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:19.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:19.485 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 14:30:19.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:19.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:19.485 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:30:19.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:19.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:19.485 | INFO | Uploaded 36GiB to GPU in 2.01823s (17.8374GiB/s) | 20260506 14:30:19.485 | INFO | Downloaded 34GiB to GPU in 1.98468s (17.1312GiB/s) | 20260506 14:30:19.606 | INFO | Starting a round of Baz IO | 20260506 14:30:22.754 | INFO | Populating RTMetrics report | 20260506 14:30:22.755 | INFO | RTMetrics report populated | 20260506 14:30:22.755 | INFO | Opening tmp RTMetrics json file | 20260506 14:30:22.782 | INFO | Serializing RTMetrics to json | 20260506 14:30:22.883 | INFO | Serialization done, swapping tmp file now... | 20260506 14:30:22.885 | INFO | RTMetrics done | 20260506 14:30:23.112 | INFO | Finished a round of Baz IO: Wrote 2.116 GiB in 3.505 seconds | 20260506 14:30:23.112 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.23% / 0.01% / 61.75% | 20260506 14:30:23.112 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.53117% / 1.58946e-05% / 5.5631e-05% | 20260506 14:30:23.112 | INFO | Baz IO overall: | 20260506 14:30:23.112 | INFO | BytesWritten : 1076410258165 | 20260506 14:30:23.112 | INFO | NumEvents : 520872427006 | 20260506 14:30:24.624 | INFO | Analyzing chunk frames = [2084864,2085376) | 20260506 14:30:24.624 | INFO | MemInfo: { | 20260506 14:30:24.624 | INFO | "gpu" : " 17 GB", | 20260506 14:30:24.624 | INFO | "huge" : "", | 20260506 14:30:24.624 | INFO | "huge_shared" : " 27 GB", | 20260506 14:30:24.624 | INFO | "malloc" : " 14 GB", | 20260506 14:30:24.624 | INFO | "pinned" : " 29 GB" | 20260506 14:30:24.624 | INFO | } | 20260506 14:30:24.624 | INFO | Perf report for the last 5.13917 seconds: | 20260506 14:30:24.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:30:24.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:24.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:24.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:24.624 | INFO | ANALYSIS : 63.0%, 2.9, 95.4ms, 0 | 20260506 14:30:24.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:24.624 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260506 14:30:24.624 | INFO | PRE_HQ : 60.9%, 1.9, 59.3ms, 0 | 20260506 14:30:24.624 | INFO | BAZWRITER : 67.6%, 4.8, 167.9ms, 0 | 20260506 14:30:24.624 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.4ms, 0 | 20260506 14:30:24.624 | INFO | Uploaded 36GiB to GPU in 2.04656s (17.5905GiB/s) | 20260506 14:30:24.624 | INFO | Downloaded 38GiB to GPU in 2.23858s (16.975GiB/s) | 20260506 14:30:29.763 | INFO | Analyzing chunk frames = [2085376,2085888) | 20260506 14:30:29.763 | INFO | MemInfo: { | 20260506 14:30:29.763 | INFO | "gpu" : " 17 GB", | 20260506 14:30:29.763 | INFO | "huge" : "", | 20260506 14:30:29.763 | INFO | "huge_shared" : " 27 GB", | 20260506 14:30:29.763 | INFO | "malloc" : " 14 GB", | 20260506 14:30:29.763 | INFO | "pinned" : " 29 GB" | 20260506 14:30:29.763 | INFO | } | 20260506 14:30:29.763 | INFO | Perf report for the last 5.13933 seconds: | 20260506 14:30:29.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:30:29.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:29.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:29.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:29.763 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 14:30:29.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:29.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:29.763 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:30:29.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:29.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:29.763 | INFO | Uploaded 36GiB to GPU in 1.97839s (18.1966GiB/s) | 20260506 14:30:29.763 | INFO | Downloaded 34GiB to GPU in 1.93604s (17.5616GiB/s) | 20260506 14:30:34.902 | INFO | Analyzing chunk frames = [2085888,2086400) | 20260506 14:30:34.902 | INFO | MemInfo: { | 20260506 14:30:34.902 | INFO | "gpu" : " 17 GB", | 20260506 14:30:34.902 | INFO | "huge" : "", | 20260506 14:30:34.902 | INFO | "huge_shared" : " 27 GB", | 20260506 14:30:34.902 | INFO | "malloc" : " 14 GB", | 20260506 14:30:34.902 | INFO | "pinned" : " 29 GB" | 20260506 14:30:34.902 | INFO | } | 20260506 14:30:34.902 | INFO | Perf report for the last 5.13915 seconds: | 20260506 14:30:34.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:30:34.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:34.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:34.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:34.902 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 14:30:34.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:34.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:34.902 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:30:34.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:34.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:34.903 | INFO | Uploaded 36GiB to GPU in 1.97846s (18.196GiB/s) | 20260506 14:30:34.903 | INFO | Downloaded 34GiB to GPU in 1.94524s (17.4786GiB/s) | 20260506 14:30:40.042 | INFO | Analyzing chunk frames = [2086400,2086912) | 20260506 14:30:40.042 | INFO | MemInfo: { | 20260506 14:30:40.042 | INFO | "gpu" : " 17 GB", | 20260506 14:30:40.042 | INFO | "huge" : "", | 20260506 14:30:40.042 | INFO | "huge_shared" : " 27 GB", | 20260506 14:30:40.042 | INFO | "malloc" : " 14 GB", | 20260506 14:30:40.042 | INFO | "pinned" : " 29 GB" | 20260506 14:30:40.042 | INFO | } | 20260506 14:30:40.042 | INFO | Perf report for the last 5.13958 seconds: | 20260506 14:30:40.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:30:40.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:40.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:40.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:40.042 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 14:30:40.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:40.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:40.042 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:30:40.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:40.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:40.042 | INFO | Uploaded 36GiB to GPU in 2.0161s (17.8562GiB/s) | 20260506 14:30:40.042 | INFO | Downloaded 34GiB to GPU in 1.91824s (17.7246GiB/s) | 20260506 14:30:45.182 | INFO | Analyzing chunk frames = [2086912,2087424) | 20260506 14:30:45.183 | INFO | MemInfo: { | 20260506 14:30:45.183 | INFO | "gpu" : " 17 GB", | 20260506 14:30:45.183 | INFO | "huge" : "", | 20260506 14:30:45.183 | INFO | "huge_shared" : " 27 GB", | 20260506 14:30:45.183 | INFO | "malloc" : " 14 GB", | 20260506 14:30:45.183 | INFO | "pinned" : " 29 GB" | 20260506 14:30:45.183 | INFO | } | 20260506 14:30:45.183 | INFO | Perf report for the last 5.14101 seconds: | 20260506 14:30:45.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:30:45.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:45.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:45.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:45.183 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 14:30:45.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:45.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:45.183 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:30:45.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:45.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:45.183 | INFO | Uploaded 36GiB to GPU in 1.98771s (18.1113GiB/s) | 20260506 14:30:45.183 | INFO | Downloaded 34GiB to GPU in 1.93581s (17.5637GiB/s) | 20260506 14:30:50.321 | INFO | Analyzing chunk frames = [2087424,2087936) | 20260506 14:30:50.321 | INFO | MemInfo: { | 20260506 14:30:50.321 | INFO | "gpu" : " 17 GB", | 20260506 14:30:50.321 | INFO | "huge" : "", | 20260506 14:30:50.321 | INFO | "huge_shared" : " 27 GB", | 20260506 14:30:50.321 | INFO | "malloc" : " 14 GB", | 20260506 14:30:50.321 | INFO | "pinned" : " 29 GB" | 20260506 14:30:50.321 | INFO | } | 20260506 14:30:50.321 | INFO | Perf report for the last 5.13853 seconds: | 20260506 14:30:50.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:30:50.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:50.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:50.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:50.322 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 14:30:50.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:50.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:50.322 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260506 14:30:50.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:50.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:50.322 | INFO | Uploaded 36GiB to GPU in 2.05343s (17.5317GiB/s) | 20260506 14:30:50.322 | INFO | Downloaded 34GiB to GPU in 1.97792s (17.1898GiB/s) | 20260506 14:30:55.461 | INFO | Analyzing chunk frames = [2087936,2088448) | 20260506 14:30:55.462 | INFO | MemInfo: { | 20260506 14:30:55.462 | INFO | "gpu" : " 17 GB", | 20260506 14:30:55.462 | INFO | "huge" : "", | 20260506 14:30:55.462 | INFO | "huge_shared" : " 27 GB", | 20260506 14:30:55.462 | INFO | "malloc" : " 14 GB", | 20260506 14:30:55.462 | INFO | "pinned" : " 29 GB" | 20260506 14:30:55.462 | INFO | } | 20260506 14:30:55.462 | INFO | Perf report for the last 5.14036 seconds: | 20260506 14:30:55.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:30:55.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:55.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:55.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:55.462 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:30:55.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:55.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:55.462 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:30:55.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:55.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:30:55.462 | INFO | Uploaded 36GiB to GPU in 2.01611s (17.8562GiB/s) | 20260506 14:30:55.462 | INFO | Downloaded 34GiB to GPU in 2.00589s (16.9501GiB/s) | 20260506 14:31:00.599 | INFO | Analyzing chunk frames = [2088448,2088960) | 20260506 14:31:00.599 | INFO | MemInfo: { | 20260506 14:31:00.599 | INFO | "gpu" : " 17 GB", | 20260506 14:31:00.599 | INFO | "huge" : "", | 20260506 14:31:00.599 | INFO | "huge_shared" : " 27 GB", | 20260506 14:31:00.599 | INFO | "malloc" : " 14 GB", | 20260506 14:31:00.599 | INFO | "pinned" : " 29 GB" | 20260506 14:31:00.599 | INFO | } | 20260506 14:31:00.599 | INFO | Perf report for the last 5.13711 seconds: | 20260506 14:31:00.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:31:00.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:00.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:00.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:00.599 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 14:31:00.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:00.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:00.599 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:31:00.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:00.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:00.599 | INFO | Uploaded 36GiB to GPU in 2.04101s (17.6384GiB/s) | 20260506 14:31:00.599 | INFO | Downloaded 34GiB to GPU in 1.96912s (17.2666GiB/s) | 20260506 14:31:00.599 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:31:00.599 | INFO | Total mem usage: 17.4 GiB | 20260506 14:31:00.599 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:31:00.599 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -9.78%) | 20260506 14:31:00.599 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:31:00.599 | INFO | High Water Breakdown: | 20260506 14:31:00.599 | INFO | 61.4% : 9.74 GiB ( 89.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:31:00.599 | INFO | 38.5% : 6.1 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:31:00.599 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:31:00.599 | INFO | ------------------------------------------ | 20260506 14:31:00.599 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:31:00.599 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:31:00.599 | INFO | Total mem usage: 26.5 GiB | 20260506 14:31:00.599 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:31:00.599 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:31:00.600 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:31:00.600 | INFO | High Water Breakdown: | 20260506 14:31:00.600 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:31:00.600 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:31:00.600 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:31:00.600 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:31:00.600 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:31:00.600 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:31:00.600 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:31:00.600 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:31:00.600 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:31:00.600 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:31:00.600 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:31:00.600 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:31:00.600 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:31:00.600 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:31:00.600 | INFO | ------------------------------------------ | 20260506 14:31:00.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:31:00.600 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:31:00.600 | INFO | Total mem usage: 25.6 GiB | 20260506 14:31:00.600 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:31:00.600 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:31:00.600 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:31:00.600 | INFO | High Water Breakdown: | 20260506 14:31:00.600 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:31:00.600 | INFO | ------------------------------------------ | 20260506 14:31:00.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:31:00.600 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:31:00.600 | INFO | Total mem usage: 15.7 GiB | 20260506 14:31:00.600 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:31:00.600 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:31:00.600 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 14:31:00.600 | INFO | High Water Breakdown: | 20260506 14:31:00.600 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:31:00.600 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:31:00.600 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:31:00.600 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:31:00.600 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:31:00.600 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:31:00.600 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:31:00.600 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:31:00.600 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:31:00.600 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:31:00.600 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:31:00.600 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:31:00.600 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:31:00.600 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:31:00.600 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:31:00.600 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:31:00.600 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:31:00.600 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:31:00.600 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:31:00.600 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:31:00.600 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:31:00.600 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:31:00.600 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:31:00.600 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:31:00.600 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:31:00.600 | INFO | ------------------------------------------ | 20260506 14:31:00.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:31:00.600 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:31:00.600 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:31:00.600 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:31:00.600 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.61e+01, 1.36e+01 / 2.10e+00 | 20260506 14:31:00.600 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.01e-01, 1.02e-02 / 5.42e-03 | 20260506 14:31:00.600 | INFO | PerfReport: 11.56%, Baseline -- 3.65e+00 / 4.40e+00, 5.69e-01 / 3.83e-01 | 20260506 14:31:00.600 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.49e+00, 1.90e-01 / 1.40e-01 | 20260506 14:31:00.600 | INFO | PerfReport: 15.88%, DME -- 4.01e+01 / 4.46e+01, 4.15e+00 / 2.31e+00 | 20260506 14:31:00.600 | INFO | PerfReport: 56.71%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.48e+00 / 1.98e+00 | 20260506 14:31:00.600 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.35e+00 / 2.93e+00, 4.08e-01 / 2.45e-01 | 20260506 14:31:00.600 | INFO | PerfReport: 3.61%, Metrics -- 1.14e+00 / 1.49e+00, 2.94e-01 / 2.20e-01 | 20260506 14:31:00.600 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.98e-01 / 4.55e-01, 6.36e-02 / 3.00e-02 | 20260506 14:31:00.600 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:31:00.600 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:31:00.600 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:31:00.600 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 14:31:00.600 | INFO | PerfReport: 21.44%, Upload -- 2.10e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260506 14:31:00.600 | INFO | PerfReport: 21.19%, Download -- 2.07e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 14:31:00.600 | INFO | PerfReport: 32.28%, Compute -- 3.16e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 14:31:00.600 | INFO | PerfReport: 1.46%, LockUpload -- 1.43e+00 / 0.00e+00, 5.84e+00 / 0.00e+00 | 20260506 14:31:00.600 | INFO | PerfReport: 1.18%, LockDownload -- 1.15e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 14:31:00.600 | INFO | PerfReport: 22.45%, LockCompute -- 2.19e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 14:31:00.600 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.53e-02 / 0.00e+00 | 20260506 14:31:00.719 | INFO | Starting a round of Baz IO | 20260506 14:31:03.879 | INFO | Populating RTMetrics report | 20260506 14:31:03.880 | INFO | RTMetrics report populated | 20260506 14:31:03.880 | INFO | Opening tmp RTMetrics json file | 20260506 14:31:04.017 | INFO | Serializing RTMetrics to json | 20260506 14:31:04.189 | INFO | Serialization done, swapping tmp file now... | 20260506 14:31:04.193 | INFO | RTMetrics done | 20260506 14:31:04.390 | INFO | Finished a round of Baz IO: Wrote 2.115 GiB in 3.671 seconds | 20260506 14:31:04.390 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.22% / 0.01% / 61.77% | 20260506 14:31:04.390 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.52574% / 1.19209e-05% / 4.76837e-05% | 20260506 14:31:04.390 | INFO | Baz IO overall: | 20260506 14:31:04.390 | INFO | BytesWritten : 1078681346963 | 20260506 14:31:04.390 | INFO | NumEvents : 521973438681 | 20260506 14:31:05.738 | INFO | Analyzing chunk frames = [2088960,2089472) | 20260506 14:31:05.738 | INFO | MemInfo: { | 20260506 14:31:05.738 | INFO | "gpu" : " 17 GB", | 20260506 14:31:05.738 | INFO | "huge" : "", | 20260506 14:31:05.738 | INFO | "huge_shared" : " 27 GB", | 20260506 14:31:05.738 | INFO | "malloc" : " 14 GB", | 20260506 14:31:05.738 | INFO | "pinned" : " 29 GB" | 20260506 14:31:05.738 | INFO | } | 20260506 14:31:05.738 | INFO | Perf report for the last 5.13908 seconds: | 20260506 14:31:05.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:31:05.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:05.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:05.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:05.738 | INFO | ANALYSIS : 63.3%, 2.9, 95.9ms, 0 | 20260506 14:31:05.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:05.738 | INFO | RT_METRICS : 50.0%, 1.0, 26.2ms, 0 | 20260506 14:31:05.738 | INFO | PRE_HQ : 60.9%, 1.9, 59.2ms, 0 | 20260506 14:31:05.738 | INFO | BAZWRITER : 70.8%, 4.8, 174.4ms, 0 | 20260506 14:31:05.738 | INFO | BAZ_SAMPLING : 4.6%, 1.0, 2.4ms, 0 | 20260506 14:31:05.738 | INFO | Uploaded 36GiB to GPU in 2.09487s (17.1849GiB/s) | 20260506 14:31:05.738 | INFO | Downloaded 38GiB to GPU in 2.23916s (16.9707GiB/s) | 20260506 14:31:10.878 | INFO | Analyzing chunk frames = [2089472,2089984) | 20260506 14:31:10.878 | INFO | MemInfo: { | 20260506 14:31:10.878 | INFO | "gpu" : " 17 GB", | 20260506 14:31:10.878 | INFO | "huge" : "", | 20260506 14:31:10.878 | INFO | "huge_shared" : " 27 GB", | 20260506 14:31:10.878 | INFO | "malloc" : " 14 GB", | 20260506 14:31:10.878 | INFO | "pinned" : " 29 GB" | 20260506 14:31:10.878 | INFO | } | 20260506 14:31:10.878 | INFO | Perf report for the last 5.14031 seconds: | 20260506 14:31:10.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:31:10.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:10.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:10.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:10.878 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 14:31:10.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:10.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:10.878 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 14:31:10.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:10.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:10.879 | INFO | Uploaded 36GiB to GPU in 2.02663s (17.7634GiB/s) | 20260506 14:31:10.879 | INFO | Downloaded 34GiB to GPU in 1.94096s (17.5171GiB/s) | 20260506 14:31:16.018 | INFO | Analyzing chunk frames = [2089984,2090496) | 20260506 14:31:16.018 | INFO | MemInfo: { | 20260506 14:31:16.018 | INFO | "gpu" : " 17 GB", | 20260506 14:31:16.018 | INFO | "huge" : "", | 20260506 14:31:16.018 | INFO | "huge_shared" : " 27 GB", | 20260506 14:31:16.019 | INFO | "malloc" : " 14 GB", | 20260506 14:31:16.019 | INFO | "pinned" : " 29 GB" | 20260506 14:31:16.019 | INFO | } | 20260506 14:31:16.019 | INFO | Perf report for the last 5.14026 seconds: | 20260506 14:31:16.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:31:16.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:16.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:16.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:16.019 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 14:31:16.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:16.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:16.019 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:31:16.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:16.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:16.019 | INFO | Uploaded 36GiB to GPU in 2.00876s (17.9215GiB/s) | 20260506 14:31:16.019 | INFO | Downloaded 34GiB to GPU in 1.93395s (17.5806GiB/s) | 20260506 14:31:21.156 | INFO | Analyzing chunk frames = [2090496,2091008) | 20260506 14:31:21.156 | INFO | MemInfo: { | 20260506 14:31:21.156 | INFO | "gpu" : " 17 GB", | 20260506 14:31:21.156 | INFO | "huge" : "", | 20260506 14:31:21.156 | INFO | "huge_shared" : " 27 GB", | 20260506 14:31:21.156 | INFO | "malloc" : " 14 GB", | 20260506 14:31:21.156 | INFO | "pinned" : " 29 GB" | 20260506 14:31:21.156 | INFO | } | 20260506 14:31:21.156 | INFO | Perf report for the last 5.13779 seconds: | 20260506 14:31:21.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:31:21.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:21.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:21.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:21.156 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 14:31:21.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:21.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:21.156 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:31:21.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:21.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:21.157 | INFO | Uploaded 36GiB to GPU in 1.97824s (18.198GiB/s) | 20260506 14:31:21.157 | INFO | Downloaded 34GiB to GPU in 1.91232s (17.7795GiB/s) | 20260506 14:31:26.295 | INFO | Analyzing chunk frames = [2091008,2091520) | 20260506 14:31:26.295 | INFO | MemInfo: { | 20260506 14:31:26.295 | INFO | "gpu" : " 17 GB", | 20260506 14:31:26.295 | INFO | "huge" : "", | 20260506 14:31:26.295 | INFO | "huge_shared" : " 27 GB", | 20260506 14:31:26.295 | INFO | "malloc" : " 14 GB", | 20260506 14:31:26.296 | INFO | "pinned" : " 29 GB" | 20260506 14:31:26.296 | INFO | } | 20260506 14:31:26.296 | INFO | Perf report for the last 5.13928 seconds: | 20260506 14:31:26.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:31:26.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:26.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:26.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:26.296 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 14:31:26.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:26.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:26.296 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:31:26.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:26.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:26.296 | INFO | Uploaded 36GiB to GPU in 2.00623s (17.9441GiB/s) | 20260506 14:31:26.296 | INFO | Downloaded 34GiB to GPU in 1.93424s (17.5779GiB/s) | 20260506 14:31:31.435 | INFO | Analyzing chunk frames = [2091520,2092032) | 20260506 14:31:31.435 | INFO | MemInfo: { | 20260506 14:31:31.435 | INFO | "gpu" : " 17 GB", | 20260506 14:31:31.435 | INFO | "huge" : "", | 20260506 14:31:31.435 | INFO | "huge_shared" : " 27 GB", | 20260506 14:31:31.435 | INFO | "malloc" : " 14 GB", | 20260506 14:31:31.435 | INFO | "pinned" : " 29 GB" | 20260506 14:31:31.435 | INFO | } | 20260506 14:31:31.435 | INFO | Perf report for the last 5.13962 seconds: | 20260506 14:31:31.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:31:31.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:31.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:31.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:31.435 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:31:31.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:31.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:31.435 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260506 14:31:31.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:31.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:31.436 | INFO | Uploaded 36GiB to GPU in 2.08406s (17.274GiB/s) | 20260506 14:31:31.436 | INFO | Downloaded 34GiB to GPU in 1.9764s (17.203GiB/s) | 20260506 14:31:36.576 | INFO | Analyzing chunk frames = [2092032,2092544) | 20260506 14:31:36.576 | INFO | MemInfo: { | 20260506 14:31:36.576 | INFO | "gpu" : " 17 GB", | 20260506 14:31:36.576 | INFO | "huge" : "", | 20260506 14:31:36.576 | INFO | "huge_shared" : " 27 GB", | 20260506 14:31:36.576 | INFO | "malloc" : " 14 GB", | 20260506 14:31:36.576 | INFO | "pinned" : " 29 GB" | 20260506 14:31:36.576 | INFO | } | 20260506 14:31:36.576 | INFO | Perf report for the last 5.14061 seconds: | 20260506 14:31:36.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:31:36.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:36.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:36.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:36.576 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:31:36.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:36.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:36.576 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:31:36.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:36.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:36.576 | INFO | Uploaded 36GiB to GPU in 2.06738s (17.4134GiB/s) | 20260506 14:31:36.576 | INFO | Downloaded 34GiB to GPU in 2.01682s (16.8582GiB/s) | 20260506 14:31:41.714 | INFO | Analyzing chunk frames = [2092544,2093056) | 20260506 14:31:41.714 | INFO | MemInfo: { | 20260506 14:31:41.714 | INFO | "gpu" : " 17 GB", | 20260506 14:31:41.714 | INFO | "huge" : "", | 20260506 14:31:41.714 | INFO | "huge_shared" : " 27 GB", | 20260506 14:31:41.714 | INFO | "malloc" : " 14 GB", | 20260506 14:31:41.715 | INFO | "pinned" : " 29 GB" | 20260506 14:31:41.715 | INFO | } | 20260506 14:31:41.715 | INFO | Perf report for the last 5.1387 seconds: | 20260506 14:31:41.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:31:41.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:41.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:41.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:41.715 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:31:41.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:41.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:41.715 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:31:41.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:41.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:41.715 | INFO | Uploaded 36GiB to GPU in 2.02798s (17.7517GiB/s) | 20260506 14:31:41.715 | INFO | Downloaded 34GiB to GPU in 1.97019s (17.2572GiB/s) | 20260506 14:31:41.829 | INFO | Starting a round of Baz IO | 20260506 14:31:44.979 | INFO | Populating RTMetrics report | 20260506 14:31:44.979 | INFO | RTMetrics report populated | 20260506 14:31:44.979 | INFO | Opening tmp RTMetrics json file | 20260506 14:31:45.038 | INFO | Serializing RTMetrics to json | 20260506 14:31:45.146 | INFO | Serialization done, swapping tmp file now... | 20260506 14:31:45.149 | INFO | RTMetrics done | 20260506 14:31:45.558 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.728 seconds | 20260506 14:31:45.558 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.20% / 0.01% / 61.78% | 20260506 14:31:45.558 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.51472% / 3.57628e-05% / 4.76837e-05% | 20260506 14:31:45.558 | INFO | Baz IO overall: | 20260506 14:31:45.558 | INFO | BytesWritten : 1080951010312 | 20260506 14:31:45.558 | INFO | NumEvents : 523073348588 | 20260506 14:31:46.853 | INFO | Analyzing chunk frames = [2093056,2093568) | 20260506 14:31:46.853 | INFO | MemInfo: { | 20260506 14:31:46.853 | INFO | "gpu" : " 17 GB", | 20260506 14:31:46.854 | INFO | "huge" : "", | 20260506 14:31:46.854 | INFO | "huge_shared" : " 27 GB", | 20260506 14:31:46.854 | INFO | "malloc" : " 14 GB", | 20260506 14:31:46.854 | INFO | "pinned" : " 29 GB" | 20260506 14:31:46.854 | INFO | } | 20260506 14:31:46.854 | INFO | Perf report for the last 5.1391 seconds: | 20260506 14:31:46.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:31:46.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:46.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:46.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:46.854 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 14:31:46.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:46.854 | INFO | RT_METRICS : 47.2%, 1.0, 24.8ms, 0 | 20260506 14:31:46.854 | INFO | PRE_HQ : 61.0%, 1.9, 59.6ms, 0 | 20260506 14:31:46.854 | INFO | BAZWRITER : 71.9%, 4.8, 177.2ms, 0 | 20260506 14:31:46.854 | INFO | BAZ_SAMPLING : 3.2%, 1.0, 1.7ms, 0 | 20260506 14:31:46.854 | INFO | Uploaded 36GiB to GPU in 2.08485s (17.2675GiB/s) | 20260506 14:31:46.854 | INFO | Downloaded 38GiB to GPU in 2.23022s (17.0387GiB/s) | 20260506 14:31:51.992 | INFO | Analyzing chunk frames = [2093568,2094080) | 20260506 14:31:51.992 | INFO | MemInfo: { | 20260506 14:31:51.992 | INFO | "gpu" : " 17 GB", | 20260506 14:31:51.992 | INFO | "huge" : "", | 20260506 14:31:51.992 | INFO | "huge_shared" : " 27 GB", | 20260506 14:31:51.992 | INFO | "malloc" : " 14 GB", | 20260506 14:31:51.992 | INFO | "pinned" : " 29 GB" | 20260506 14:31:51.992 | INFO | } | 20260506 14:31:51.992 | INFO | Perf report for the last 5.13839 seconds: | 20260506 14:31:51.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:31:51.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:51.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:51.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:51.992 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 14:31:51.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:51.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:51.992 | INFO | PRE_HQ : 27.9%, 1.0, 14.6ms, 0 | 20260506 14:31:51.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:51.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:51.992 | INFO | Uploaded 36GiB to GPU in 2.0266s (17.7637GiB/s) | 20260506 14:31:51.992 | INFO | Downloaded 34GiB to GPU in 1.94478s (17.4827GiB/s) | 20260506 14:31:57.131 | INFO | Analyzing chunk frames = [2094080,2094592) | 20260506 14:31:57.131 | INFO | MemInfo: { | 20260506 14:31:57.131 | INFO | "gpu" : " 17 GB", | 20260506 14:31:57.131 | INFO | "huge" : "", | 20260506 14:31:57.132 | INFO | "huge_shared" : " 27 GB", | 20260506 14:31:57.132 | INFO | "malloc" : " 14 GB", | 20260506 14:31:57.132 | INFO | "pinned" : " 29 GB" | 20260506 14:31:57.132 | INFO | } | 20260506 14:31:57.132 | INFO | Perf report for the last 5.13964 seconds: | 20260506 14:31:57.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:31:57.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:57.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:57.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:57.132 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:31:57.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:57.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:57.132 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:31:57.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:57.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:31:57.132 | INFO | Uploaded 36GiB to GPU in 2.00949s (17.915GiB/s) | 20260506 14:31:57.132 | INFO | Downloaded 34GiB to GPU in 1.93273s (17.5917GiB/s) | 20260506 14:32:02.273 | INFO | Analyzing chunk frames = [2094592,2095104) | 20260506 14:32:02.273 | INFO | MemInfo: { | 20260506 14:32:02.273 | INFO | "gpu" : " 17 GB", | 20260506 14:32:02.273 | INFO | "huge" : "", | 20260506 14:32:02.273 | INFO | "huge_shared" : " 27 GB", | 20260506 14:32:02.273 | INFO | "malloc" : " 14 GB", | 20260506 14:32:02.273 | INFO | "pinned" : " 29 GB" | 20260506 14:32:02.273 | INFO | } | 20260506 14:32:02.274 | INFO | Perf report for the last 5.1417 seconds: | 20260506 14:32:02.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:32:02.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:02.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:02.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:02.274 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260506 14:32:02.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:02.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:02.274 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 14:32:02.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:02.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:02.274 | INFO | Uploaded 36GiB to GPU in 1.98864s (18.1028GiB/s) | 20260506 14:32:02.274 | INFO | Downloaded 34GiB to GPU in 1.92483s (17.6639GiB/s) | 20260506 14:32:07.411 | INFO | Analyzing chunk frames = [2095104,2095616) | 20260506 14:32:07.411 | INFO | MemInfo: { | 20260506 14:32:07.411 | INFO | "gpu" : " 17 GB", | 20260506 14:32:07.411 | INFO | "huge" : "", | 20260506 14:32:07.411 | INFO | "huge_shared" : " 27 GB", | 20260506 14:32:07.411 | INFO | "malloc" : " 14 GB", | 20260506 14:32:07.411 | INFO | "pinned" : " 29 GB" | 20260506 14:32:07.411 | INFO | } | 20260506 14:32:07.411 | INFO | Perf report for the last 5.13745 seconds: | 20260506 14:32:07.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:32:07.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:07.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:07.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:07.411 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 14:32:07.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:07.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:07.411 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:32:07.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:07.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:07.411 | INFO | Uploaded 36GiB to GPU in 1.98701s (18.1176GiB/s) | 20260506 14:32:07.411 | INFO | Downloaded 34GiB to GPU in 1.93111s (17.6065GiB/s) | 20260506 14:32:12.549 | INFO | Analyzing chunk frames = [2095616,2096128) | 20260506 14:32:12.549 | INFO | MemInfo: { | 20260506 14:32:12.549 | INFO | "gpu" : " 17 GB", | 20260506 14:32:12.549 | INFO | "huge" : "", | 20260506 14:32:12.550 | INFO | "huge_shared" : " 27 GB", | 20260506 14:32:12.550 | INFO | "malloc" : " 14 GB", | 20260506 14:32:12.550 | INFO | "pinned" : " 29 GB" | 20260506 14:32:12.550 | INFO | } | 20260506 14:32:12.550 | INFO | Perf report for the last 5.13879 seconds: | 20260506 14:32:12.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:32:12.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:12.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:12.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:12.550 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 14:32:12.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:12.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:12.550 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 14:32:12.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:12.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:12.550 | INFO | Uploaded 36GiB to GPU in 2.05096s (17.5528GiB/s) | 20260506 14:32:12.550 | INFO | Downloaded 34GiB to GPU in 1.93189s (17.5994GiB/s) | 20260506 14:32:17.691 | INFO | Analyzing chunk frames = [2096128,2096640) | 20260506 14:32:17.691 | INFO | MemInfo: { | 20260506 14:32:17.691 | INFO | "gpu" : " 17 GB", | 20260506 14:32:17.691 | INFO | "huge" : "", | 20260506 14:32:17.691 | INFO | "huge_shared" : " 27 GB", | 20260506 14:32:17.691 | INFO | "malloc" : " 14 GB", | 20260506 14:32:17.691 | INFO | "pinned" : " 29 GB" | 20260506 14:32:17.691 | INFO | } | 20260506 14:32:17.691 | INFO | Perf report for the last 5.14117 seconds: | 20260506 14:32:17.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:32:17.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:17.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:17.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:17.691 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 14:32:17.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:17.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:17.691 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 14:32:17.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:17.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:17.691 | INFO | Uploaded 36GiB to GPU in 2.02383s (17.788GiB/s) | 20260506 14:32:17.691 | INFO | Downloaded 34GiB to GPU in 1.99426s (17.0489GiB/s) | 20260506 14:32:22.829 | INFO | Analyzing chunk frames = [2096640,2097152) | 20260506 14:32:22.829 | INFO | MemInfo: { | 20260506 14:32:22.829 | INFO | "gpu" : " 17 GB", | 20260506 14:32:22.829 | INFO | "huge" : "", | 20260506 14:32:22.829 | INFO | "huge_shared" : " 27 GB", | 20260506 14:32:22.829 | INFO | "malloc" : " 14 GB", | 20260506 14:32:22.829 | INFO | "pinned" : " 29 GB" | 20260506 14:32:22.829 | INFO | } | 20260506 14:32:22.829 | INFO | Perf report for the last 5.13814 seconds: | 20260506 14:32:22.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:32:22.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:22.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:22.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:22.829 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 14:32:22.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:22.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:22.829 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260506 14:32:22.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:22.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:22.830 | INFO | Uploaded 36GiB to GPU in 2.03245s (17.7126GiB/s) | 20260506 14:32:22.830 | INFO | Downloaded 34GiB to GPU in 1.97994s (17.1723GiB/s) | 20260506 14:32:22.830 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:32:22.830 | INFO | Total mem usage: 17.4 GiB | 20260506 14:32:22.830 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:32:22.830 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -9.08%) | 20260506 14:32:22.830 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:32:22.830 | INFO | High Water Breakdown: | 20260506 14:32:22.830 | INFO | 61.5% : 9.82 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:32:22.830 | INFO | 38.4% : 6.12 GiB ( 91.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:32:22.830 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:32:22.830 | INFO | ------------------------------------------ | 20260506 14:32:22.830 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:32:22.830 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:32:22.830 | INFO | Total mem usage: 26.5 GiB | 20260506 14:32:22.830 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:32:22.830 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:32:22.830 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:32:22.830 | INFO | High Water Breakdown: | 20260506 14:32:22.830 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:32:22.830 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:32:22.830 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:32:22.830 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:32:22.830 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:32:22.830 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:32:22.830 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:32:22.830 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:32:22.830 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:32:22.830 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:32:22.830 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:32:22.830 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:32:22.830 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:32:22.830 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:32:22.830 | INFO | ------------------------------------------ | 20260506 14:32:22.830 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:32:22.830 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:32:22.830 | INFO | Total mem usage: 25.6 GiB | 20260506 14:32:22.830 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:32:22.830 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:32:22.830 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:32:22.830 | INFO | High Water Breakdown: | 20260506 14:32:22.831 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:32:22.831 | INFO | ------------------------------------------ | 20260506 14:32:22.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:32:22.831 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:32:22.831 | INFO | Total mem usage: 15.7 GiB | 20260506 14:32:22.831 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:32:22.831 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:32:22.831 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 14:32:22.831 | INFO | High Water Breakdown: | 20260506 14:32:22.831 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:32:22.831 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:32:22.831 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:32:22.831 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:32:22.831 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:32:22.831 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:32:22.831 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:32:22.831 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:32:22.831 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:32:22.831 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:32:22.831 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:32:22.831 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:32:22.831 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:32:22.831 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:32:22.831 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:32:22.831 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:32:22.831 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:32:22.831 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:32:22.831 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:32:22.831 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:32:22.831 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:32:22.831 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:32:22.831 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:32:22.831 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:32:22.831 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:32:22.831 | INFO | ------------------------------------------ | 20260506 14:32:22.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:32:22.831 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:32:22.831 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:32:22.831 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:32:22.831 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.56e+01, 1.35e+01 / 1.98e+00 | 20260506 14:32:22.831 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.01e-01, 1.11e-02 / 5.43e-03 | 20260506 14:32:22.831 | INFO | PerfReport: 11.55%, Baseline -- 3.64e+00 / 4.44e+00, 5.58e-01 / 2.96e-01 | 20260506 14:32:22.831 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.45e+00, 1.86e-01 / 1.23e-01 | 20260506 14:32:22.831 | INFO | PerfReport: 15.85%, DME -- 4.00e+01 / 4.42e+01, 3.87e+00 / 2.19e+00 | 20260506 14:32:22.831 | INFO | PerfReport: 56.74%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.48e+00 / 2.01e+00 | 20260506 14:32:22.831 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.35e+00 / 2.96e+00, 4.14e-01 / 2.40e-01 | 20260506 14:32:22.831 | INFO | PerfReport: 3.60%, Metrics -- 1.14e+00 / 1.51e+00, 2.91e-01 / 2.19e-01 | 20260506 14:32:22.831 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.96e-01 / 4.46e-01, 6.18e-02 / 3.17e-02 | 20260506 14:32:22.831 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:32:22.831 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:32:22.831 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:32:22.831 | INFO | PerfReport: Full Execution -- 9.79e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 14:32:22.831 | INFO | PerfReport: 21.61%, Upload -- 2.12e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260506 14:32:22.831 | INFO | PerfReport: 21.14%, Download -- 2.07e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 14:32:22.831 | INFO | PerfReport: 32.20%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 14:32:22.831 | INFO | PerfReport: 1.63%, LockUpload -- 1.59e+00 / 0.00e+00, 6.21e+00 / 0.00e+00 | 20260506 14:32:22.831 | INFO | PerfReport: 1.03%, LockDownload -- 1.00e+00 / 0.00e+00, 2.57e+00 / 0.00e+00 | 20260506 14:32:22.831 | INFO | PerfReport: 22.38%, LockCompute -- 2.19e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 14:32:22.831 | INFO | PerfReport: 0.02%, Misc -- 1.54e-02 / 0.00e+00, 2.58e-02 / 0.00e+00 | 20260506 14:32:22.957 | INFO | Starting a round of Baz IO | 20260506 14:32:26.101 | INFO | Populating RTMetrics report | 20260506 14:32:26.102 | INFO | RTMetrics report populated | 20260506 14:32:26.102 | INFO | Opening tmp RTMetrics json file | 20260506 14:32:26.181 | INFO | Serializing RTMetrics to json | 20260506 14:32:26.247 | INFO | Serialization done, swapping tmp file now... | 20260506 14:32:26.249 | INFO | RTMetrics done | 20260506 14:32:26.485 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 3.527 seconds | 20260506 14:32:26.485 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.19% / 0.01% / 61.80% | 20260506 14:32:26.485 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.50411% / 1.58946e-05% / 6.35783e-05% | 20260506 14:32:26.485 | INFO | Baz IO overall: | 20260506 14:32:26.485 | INFO | BytesWritten : 1083219418911 | 20260506 14:32:26.485 | INFO | NumEvents : 524172305102 | 20260506 14:32:27.968 | INFO | Analyzing chunk frames = [2097152,2097664) | 20260506 14:32:27.968 | INFO | MemInfo: { | 20260506 14:32:27.968 | INFO | "gpu" : " 17 GB", | 20260506 14:32:27.968 | INFO | "huge" : "", | 20260506 14:32:27.968 | INFO | "huge_shared" : " 27 GB", | 20260506 14:32:27.968 | INFO | "malloc" : " 14 GB", | 20260506 14:32:27.968 | INFO | "pinned" : " 29 GB" | 20260506 14:32:27.968 | INFO | } | 20260506 14:32:27.968 | INFO | Perf report for the last 5.13889 seconds: | 20260506 14:32:27.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:32:27.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:27.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:27.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:27.968 | INFO | ANALYSIS : 63.0%, 2.9, 95.5ms, 0 | 20260506 14:32:27.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:27.968 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260506 14:32:27.968 | INFO | PRE_HQ : 60.8%, 1.9, 59.3ms, 0 | 20260506 14:32:27.968 | INFO | BAZWRITER : 68.0%, 4.8, 167.7ms, 0 | 20260506 14:32:27.968 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 14:32:27.968 | INFO | Uploaded 36GiB to GPU in 2.06689s (17.4175GiB/s) | 20260506 14:32:27.968 | INFO | Downloaded 38GiB to GPU in 2.22145s (17.1059GiB/s) | 20260506 14:32:33.107 | INFO | Analyzing chunk frames = [2097664,2098176) | 20260506 14:32:33.107 | INFO | MemInfo: { | 20260506 14:32:33.107 | INFO | "gpu" : " 17 GB", | 20260506 14:32:33.107 | INFO | "huge" : "", | 20260506 14:32:33.107 | INFO | "huge_shared" : " 27 GB", | 20260506 14:32:33.107 | INFO | "malloc" : " 14 GB", | 20260506 14:32:33.107 | INFO | "pinned" : " 29 GB" | 20260506 14:32:33.107 | INFO | } | 20260506 14:32:33.107 | INFO | Perf report for the last 5.1394 seconds: | 20260506 14:32:33.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:32:33.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:33.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:33.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:33.107 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 14:32:33.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:33.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:33.108 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:32:33.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:33.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:33.108 | INFO | Uploaded 36GiB to GPU in 2.00704s (17.9368GiB/s) | 20260506 14:32:33.108 | INFO | Downloaded 34GiB to GPU in 1.94157s (17.5116GiB/s) | 20260506 14:32:38.246 | INFO | Analyzing chunk frames = [2098176,2098688) | 20260506 14:32:38.246 | INFO | MemInfo: { | 20260506 14:32:38.246 | INFO | "gpu" : " 17 GB", | 20260506 14:32:38.246 | INFO | "huge" : "", | 20260506 14:32:38.246 | INFO | "huge_shared" : " 27 GB", | 20260506 14:32:38.246 | INFO | "malloc" : " 14 GB", | 20260506 14:32:38.246 | INFO | "pinned" : " 29 GB" | 20260506 14:32:38.246 | INFO | } | 20260506 14:32:38.246 | INFO | Perf report for the last 5.13856 seconds: | 20260506 14:32:38.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:32:38.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:38.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:38.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:38.246 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 14:32:38.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:38.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:38.246 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:32:38.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:38.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:38.246 | INFO | Uploaded 36GiB to GPU in 2.00173s (17.9844GiB/s) | 20260506 14:32:38.246 | INFO | Downloaded 34GiB to GPU in 1.93361s (17.5837GiB/s) | 20260506 14:32:43.386 | INFO | Analyzing chunk frames = [2098688,2099200) | 20260506 14:32:43.386 | INFO | MemInfo: { | 20260506 14:32:43.386 | INFO | "gpu" : " 17 GB", | 20260506 14:32:43.386 | INFO | "huge" : "", | 20260506 14:32:43.386 | INFO | "huge_shared" : " 27 GB", | 20260506 14:32:43.386 | INFO | "malloc" : " 14 GB", | 20260506 14:32:43.386 | INFO | "pinned" : " 29 GB" | 20260506 14:32:43.386 | INFO | } | 20260506 14:32:43.386 | INFO | Perf report for the last 5.14017 seconds: | 20260506 14:32:43.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:32:43.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:43.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:43.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:43.386 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260506 14:32:43.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:43.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:43.386 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:32:43.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:43.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:43.386 | INFO | Uploaded 36GiB to GPU in 1.99495s (18.0455GiB/s) | 20260506 14:32:43.386 | INFO | Downloaded 34GiB to GPU in 1.90847s (17.8153GiB/s) | 20260506 14:32:48.527 | INFO | Analyzing chunk frames = [2099200,2099712) | 20260506 14:32:48.527 | INFO | MemInfo: { | 20260506 14:32:48.527 | INFO | "gpu" : " 17 GB", | 20260506 14:32:48.527 | INFO | "huge" : "", | 20260506 14:32:48.527 | INFO | "huge_shared" : " 27 GB", | 20260506 14:32:48.527 | INFO | "malloc" : " 14 GB", | 20260506 14:32:48.527 | INFO | "pinned" : " 29 GB" | 20260506 14:32:48.527 | INFO | } | 20260506 14:32:48.527 | INFO | Perf report for the last 5.1406 seconds: | 20260506 14:32:48.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:32:48.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:48.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:48.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:48.527 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:32:48.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:48.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:48.527 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:32:48.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:48.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:48.527 | INFO | Uploaded 36GiB to GPU in 2.00585s (17.9475GiB/s) | 20260506 14:32:48.527 | INFO | Downloaded 34GiB to GPU in 1.92979s (17.6185GiB/s) | 20260506 14:32:53.667 | INFO | Analyzing chunk frames = [2099712,2100224) | 20260506 14:32:53.667 | INFO | MemInfo: { | 20260506 14:32:53.667 | INFO | "gpu" : " 17 GB", | 20260506 14:32:53.667 | INFO | "huge" : "", | 20260506 14:32:53.667 | INFO | "huge_shared" : " 27 GB", | 20260506 14:32:53.667 | INFO | "malloc" : " 14 GB", | 20260506 14:32:53.667 | INFO | "pinned" : " 29 GB" | 20260506 14:32:53.667 | INFO | } | 20260506 14:32:53.667 | INFO | Perf report for the last 5.14066 seconds: | 20260506 14:32:53.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:32:53.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:53.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:53.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:53.668 | INFO | ANALYSIS : 61.7%, 3.0, 96.4ms, 0 | 20260506 14:32:53.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:53.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:53.668 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:32:53.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:53.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:53.668 | INFO | Uploaded 36GiB to GPU in 2.05877s (17.4861GiB/s) | 20260506 14:32:53.668 | INFO | Downloaded 34GiB to GPU in 1.91355s (17.768GiB/s) | 20260506 14:32:58.804 | INFO | Analyzing chunk frames = [2100224,2100736) | 20260506 14:32:58.804 | INFO | MemInfo: { | 20260506 14:32:58.804 | INFO | "gpu" : " 17 GB", | 20260506 14:32:58.804 | INFO | "huge" : "", | 20260506 14:32:58.804 | INFO | "huge_shared" : " 27 GB", | 20260506 14:32:58.804 | INFO | "malloc" : " 14 GB", | 20260506 14:32:58.804 | INFO | "pinned" : " 29 GB" | 20260506 14:32:58.804 | INFO | } | 20260506 14:32:58.804 | INFO | Perf report for the last 5.13695 seconds: | 20260506 14:32:58.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:32:58.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:58.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:58.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:58.805 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 14:32:58.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:58.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:58.805 | INFO | PRE_HQ : 28.9%, 1.0, 15.1ms, 0 | 20260506 14:32:58.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:58.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:32:58.805 | INFO | Uploaded 36GiB to GPU in 2.02278s (17.7973GiB/s) | 20260506 14:32:58.805 | INFO | Downloaded 34GiB to GPU in 1.99921s (17.0068GiB/s) | 20260506 14:33:03.944 | INFO | Analyzing chunk frames = [2100736,2101248) | 20260506 14:33:03.944 | INFO | MemInfo: { | 20260506 14:33:03.944 | INFO | "gpu" : " 17 GB", | 20260506 14:33:03.944 | INFO | "huge" : "", | 20260506 14:33:03.944 | INFO | "huge_shared" : " 27 GB", | 20260506 14:33:03.944 | INFO | "malloc" : " 14 GB", | 20260506 14:33:03.944 | INFO | "pinned" : " 29 GB" | 20260506 14:33:03.944 | INFO | } | 20260506 14:33:03.944 | INFO | Perf report for the last 5.13941 seconds: | 20260506 14:33:03.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:33:03.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:03.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:03.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:03.944 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:33:03.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:03.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:03.944 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:33:03.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:03.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:03.944 | INFO | Uploaded 36GiB to GPU in 2.019s (17.8306GiB/s) | 20260506 14:33:03.944 | INFO | Downloaded 34GiB to GPU in 1.97909s (17.1796GiB/s) | 20260506 14:33:04.059 | INFO | Starting a round of Baz IO | 20260506 14:33:07.207 | INFO | Populating RTMetrics report | 20260506 14:33:07.207 | INFO | RTMetrics report populated | 20260506 14:33:07.207 | INFO | Opening tmp RTMetrics json file | 20260506 14:33:07.258 | INFO | Serializing RTMetrics to json | 20260506 14:33:07.261 | INFO | Serialization done, swapping tmp file now... | 20260506 14:33:07.265 | INFO | RTMetrics done | 20260506 14:33:07.577 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.517 seconds | 20260506 14:33:07.577 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.18% / 0.01% / 61.81% | 20260506 14:33:07.577 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.50535% / 1.58946e-05% / 6.35783e-05% | 20260506 14:33:07.577 | INFO | Baz IO overall: | 20260506 14:33:07.577 | INFO | BytesWritten : 1085489264355 | 20260506 14:33:07.577 | INFO | NumEvents : 525272249976 | 20260506 14:33:09.083 | INFO | Analyzing chunk frames = [2101248,2101760) | 20260506 14:33:09.083 | INFO | MemInfo: { | 20260506 14:33:09.083 | INFO | "gpu" : " 17 GB", | 20260506 14:33:09.083 | INFO | "huge" : "", | 20260506 14:33:09.083 | INFO | "huge_shared" : " 27 GB", | 20260506 14:33:09.083 | INFO | "malloc" : " 14 GB", | 20260506 14:33:09.083 | INFO | "pinned" : " 29 GB" | 20260506 14:33:09.083 | INFO | } | 20260506 14:33:09.083 | INFO | Perf report for the last 5.1389 seconds: | 20260506 14:33:09.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:33:09.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:09.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:09.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:09.083 | INFO | ANALYSIS : 62.9%, 2.9, 95.1ms, 0 | 20260506 14:33:09.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:09.083 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260506 14:33:09.083 | INFO | PRE_HQ : 61.0%, 1.9, 59.8ms, 0 | 20260506 14:33:09.083 | INFO | BAZWRITER : 67.8%, 4.8, 167.4ms, 0 | 20260506 14:33:09.083 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 14:33:09.083 | INFO | Uploaded 36GiB to GPU in 2.07844s (17.3207GiB/s) | 20260506 14:33:09.083 | INFO | Downloaded 38GiB to GPU in 2.2239s (17.0871GiB/s) | 20260506 14:33:14.222 | INFO | Analyzing chunk frames = [2101760,2102272) | 20260506 14:33:14.222 | INFO | MemInfo: { | 20260506 14:33:14.222 | INFO | "gpu" : " 17 GB", | 20260506 14:33:14.222 | INFO | "huge" : "", | 20260506 14:33:14.222 | INFO | "huge_shared" : " 27 GB", | 20260506 14:33:14.222 | INFO | "malloc" : " 14 GB", | 20260506 14:33:14.222 | INFO | "pinned" : " 29 GB" | 20260506 14:33:14.222 | INFO | } | 20260506 14:33:14.222 | INFO | Perf report for the last 5.13968 seconds: | 20260506 14:33:14.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:33:14.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:14.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:14.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:14.223 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:33:14.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:14.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:14.223 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:33:14.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:14.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:14.223 | INFO | Uploaded 36GiB to GPU in 2.02982s (17.7356GiB/s) | 20260506 14:33:14.223 | INFO | Downloaded 34GiB to GPU in 1.94435s (17.4866GiB/s) | 20260506 14:33:19.363 | INFO | Analyzing chunk frames = [2102272,2102784) | 20260506 14:33:19.363 | INFO | MemInfo: { | 20260506 14:33:19.363 | INFO | "gpu" : " 17 GB", | 20260506 14:33:19.363 | INFO | "huge" : "", | 20260506 14:33:19.363 | INFO | "huge_shared" : " 27 GB", | 20260506 14:33:19.363 | INFO | "malloc" : " 14 GB", | 20260506 14:33:19.363 | INFO | "pinned" : " 29 GB" | 20260506 14:33:19.363 | INFO | } | 20260506 14:33:19.363 | INFO | Perf report for the last 5.14053 seconds: | 20260506 14:33:19.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:33:19.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:19.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:19.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:19.363 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 14:33:19.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:19.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:19.363 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:33:19.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:19.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:19.363 | INFO | Uploaded 36GiB to GPU in 2.00371s (17.9667GiB/s) | 20260506 14:33:19.363 | INFO | Downloaded 34GiB to GPU in 1.93238s (17.5949GiB/s) | 20260506 14:33:24.501 | INFO | Analyzing chunk frames = [2102784,2103296) | 20260506 14:33:24.501 | INFO | MemInfo: { | 20260506 14:33:24.501 | INFO | "gpu" : " 17 GB", | 20260506 14:33:24.501 | INFO | "huge" : "", | 20260506 14:33:24.501 | INFO | "huge_shared" : " 27 GB", | 20260506 14:33:24.501 | INFO | "malloc" : " 14 GB", | 20260506 14:33:24.501 | INFO | "pinned" : " 29 GB" | 20260506 14:33:24.501 | INFO | } | 20260506 14:33:24.501 | INFO | Perf report for the last 5.13802 seconds: | 20260506 14:33:24.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:33:24.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:24.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:24.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:24.501 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 14:33:24.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:24.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:24.501 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:33:24.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:24.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:24.501 | INFO | Uploaded 36GiB to GPU in 1.9916s (18.0759GiB/s) | 20260506 14:33:24.501 | INFO | Downloaded 34GiB to GPU in 1.92486s (17.6636GiB/s) | 20260506 14:33:29.639 | INFO | Analyzing chunk frames = [2103296,2103808) | 20260506 14:33:29.639 | INFO | MemInfo: { | 20260506 14:33:29.639 | INFO | "gpu" : " 17 GB", | 20260506 14:33:29.639 | INFO | "huge" : "", | 20260506 14:33:29.639 | INFO | "huge_shared" : " 27 GB", | 20260506 14:33:29.640 | INFO | "malloc" : " 14 GB", | 20260506 14:33:29.640 | INFO | "pinned" : " 29 GB" | 20260506 14:33:29.640 | INFO | } | 20260506 14:33:29.640 | INFO | Perf report for the last 5.13871 seconds: | 20260506 14:33:29.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:33:29.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:29.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:29.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:29.640 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:33:29.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:29.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:29.640 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:33:29.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:29.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:29.640 | INFO | Uploaded 36GiB to GPU in 2.00447s (17.9598GiB/s) | 20260506 14:33:29.640 | INFO | Downloaded 34GiB to GPU in 1.92687s (17.6452GiB/s) | 20260506 14:33:34.780 | INFO | Analyzing chunk frames = [2103808,2104320) | 20260506 14:33:34.780 | INFO | MemInfo: { | 20260506 14:33:34.780 | INFO | "gpu" : " 17 GB", | 20260506 14:33:34.780 | INFO | "huge" : "", | 20260506 14:33:34.780 | INFO | "huge_shared" : " 27 GB", | 20260506 14:33:34.780 | INFO | "malloc" : " 14 GB", | 20260506 14:33:34.780 | INFO | "pinned" : " 29 GB" | 20260506 14:33:34.780 | INFO | } | 20260506 14:33:34.780 | INFO | Perf report for the last 5.14058 seconds: | 20260506 14:33:34.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:33:34.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:34.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:34.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:34.780 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 14:33:34.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:34.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:34.780 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 14:33:34.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:34.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:34.781 | INFO | Uploaded 36GiB to GPU in 2.05131s (17.5497GiB/s) | 20260506 14:33:34.781 | INFO | Downloaded 34GiB to GPU in 1.88875s (18.0014GiB/s) | 20260506 14:33:39.920 | INFO | Analyzing chunk frames = [2104320,2104832) | 20260506 14:33:39.920 | INFO | MemInfo: { | 20260506 14:33:39.920 | INFO | "gpu" : " 17 GB", | 20260506 14:33:39.920 | INFO | "huge" : "", | 20260506 14:33:39.920 | INFO | "huge_shared" : " 27 GB", | 20260506 14:33:39.920 | INFO | "malloc" : " 14 GB", | 20260506 14:33:39.920 | INFO | "pinned" : " 29 GB" | 20260506 14:33:39.920 | INFO | } | 20260506 14:33:39.920 | INFO | Perf report for the last 5.13948 seconds: | 20260506 14:33:39.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:33:39.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:39.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:39.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:39.920 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 14:33:39.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:39.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:39.920 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 14:33:39.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:39.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:39.920 | INFO | Uploaded 36GiB to GPU in 2.01504s (17.8657GiB/s) | 20260506 14:33:39.920 | INFO | Downloaded 34GiB to GPU in 1.99909s (17.0077GiB/s) | 20260506 14:33:45.058 | INFO | Analyzing chunk frames = [2104832,2105344) | 20260506 14:33:45.058 | INFO | MemInfo: { | 20260506 14:33:45.058 | INFO | "gpu" : " 17 GB", | 20260506 14:33:45.058 | INFO | "huge" : "", | 20260506 14:33:45.058 | INFO | "huge_shared" : " 27 GB", | 20260506 14:33:45.058 | INFO | "malloc" : " 14 GB", | 20260506 14:33:45.059 | INFO | "pinned" : " 29 GB" | 20260506 14:33:45.059 | INFO | } | 20260506 14:33:45.059 | INFO | Perf report for the last 5.13892 seconds: | 20260506 14:33:45.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:33:45.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:45.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:45.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:45.059 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:33:45.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:45.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:45.059 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:33:45.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:45.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:45.059 | INFO | Uploaded 36GiB to GPU in 2.03025s (17.7318GiB/s) | 20260506 14:33:45.059 | INFO | Downloaded 34GiB to GPU in 1.97346s (17.2287GiB/s) | 20260506 14:33:45.059 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:33:45.059 | INFO | Total mem usage: 17.4 GiB | 20260506 14:33:45.059 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:33:45.059 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -7.27%) | 20260506 14:33:45.059 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:33:45.059 | INFO | High Water Breakdown: | 20260506 14:33:45.059 | INFO | 61.8% : 10 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:33:45.059 | INFO | 38.1% : 6.18 GiB ( 92.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:33:45.059 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:33:45.059 | INFO | ------------------------------------------ | 20260506 14:33:45.059 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:33:45.059 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:33:45.059 | INFO | Total mem usage: 26.5 GiB | 20260506 14:33:45.059 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:33:45.059 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:33:45.059 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:33:45.059 | INFO | High Water Breakdown: | 20260506 14:33:45.059 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:33:45.059 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:33:45.059 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:33:45.059 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:33:45.059 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:33:45.059 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:33:45.059 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:33:45.059 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:33:45.060 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:33:45.060 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:33:45.060 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:33:45.060 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:33:45.060 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:33:45.060 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:33:45.060 | INFO | ------------------------------------------ | 20260506 14:33:45.060 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:33:45.060 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:33:45.060 | INFO | Total mem usage: 25.6 GiB | 20260506 14:33:45.060 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:33:45.060 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:33:45.060 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:33:45.060 | INFO | High Water Breakdown: | 20260506 14:33:45.060 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:33:45.060 | INFO | ------------------------------------------ | 20260506 14:33:45.060 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:33:45.060 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:33:45.060 | INFO | Total mem usage: 15.7 GiB | 20260506 14:33:45.060 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:33:45.060 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:33:45.060 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 14:33:45.060 | INFO | High Water Breakdown: | 20260506 14:33:45.060 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:33:45.060 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:33:45.060 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:33:45.060 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:33:45.060 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:33:45.060 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:33:45.060 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:33:45.060 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:33:45.060 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:33:45.060 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:33:45.060 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:33:45.060 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:33:45.060 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:33:45.060 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:33:45.060 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:33:45.060 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:33:45.060 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:33:45.060 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:33:45.060 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:33:45.060 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:33:45.060 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:33:45.060 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:33:45.060 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:33:45.060 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:33:45.060 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:33:45.060 | INFO | ------------------------------------------ | 20260506 14:33:45.060 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:33:45.060 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:33:45.060 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:33:45.060 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:33:45.060 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.55e+01, 1.33e+01 / 2.41e+00 | 20260506 14:33:45.060 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.01e-01, 1.06e-02 / 5.57e-03 | 20260506 14:33:45.060 | INFO | PerfReport: 11.58%, Baseline -- 3.65e+00 / 4.46e+00, 5.90e-01 / 2.99e-01 | 20260506 14:33:45.060 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.49e+00, 1.87e-01 / 1.41e-01 | 20260506 14:33:45.060 | INFO | PerfReport: 15.84%, DME -- 4.00e+01 / 4.42e+01, 3.86e+00 / 2.46e+00 | 20260506 14:33:45.060 | INFO | PerfReport: 56.70%, FrameLabeling -- 1.79e+01 / 2.03e+01, 1.46e+00 / 1.72e+00 | 20260506 14:33:45.060 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.35e+00 / 2.98e+00, 4.15e-01 / 2.15e-01 | 20260506 14:33:45.060 | INFO | PerfReport: 3.60%, Metrics -- 1.14e+00 / 1.50e+00, 2.90e-01 / 1.79e-01 | 20260506 14:33:45.060 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.97e-01 / 4.47e-01, 6.16e-02 / 2.55e-02 | 20260506 14:33:45.060 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:33:45.060 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:33:45.060 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:33:45.060 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 14:33:45.060 | INFO | PerfReport: 21.55%, Upload -- 2.11e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260506 14:33:45.060 | INFO | PerfReport: 21.06%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 14:33:45.060 | INFO | PerfReport: 32.24%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 14:33:45.060 | INFO | PerfReport: 1.62%, LockUpload -- 1.59e+00 / 0.00e+00, 6.34e+00 / 0.00e+00 | 20260506 14:33:45.060 | INFO | PerfReport: 1.07%, LockDownload -- 1.05e+00 / 0.00e+00, 2.61e+00 / 0.00e+00 | 20260506 14:33:45.060 | INFO | PerfReport: 22.43%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:33:45.060 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260506 14:33:45.173 | INFO | Starting a round of Baz IO | 20260506 14:33:48.330 | INFO | Populating RTMetrics report | 20260506 14:33:48.331 | INFO | RTMetrics report populated | 20260506 14:33:48.331 | INFO | Opening tmp RTMetrics json file | 20260506 14:33:48.349 | INFO | Serializing RTMetrics to json | 20260506 14:33:48.350 | INFO | Serialization done, swapping tmp file now... | 20260506 14:33:48.351 | INFO | RTMetrics done | 20260506 14:33:48.770 | INFO | Finished a round of Baz IO: Wrote 2.115 GiB in 3.597 seconds | 20260506 14:33:48.770 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.16% / 0.01% / 61.82% | 20260506 14:33:48.770 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.5039% / 7.94729e-06% / 6.35783e-05% | 20260506 14:33:48.770 | INFO | Baz IO overall: | 20260506 14:33:48.770 | INFO | BytesWritten : 1087760761792 | 20260506 14:33:48.770 | INFO | NumEvents : 526373770317 | 20260506 14:33:50.197 | INFO | Analyzing chunk frames = [2105344,2105856) | 20260506 14:33:50.197 | INFO | MemInfo: { | 20260506 14:33:50.197 | INFO | "gpu" : " 17 GB", | 20260506 14:33:50.197 | INFO | "huge" : "", | 20260506 14:33:50.197 | INFO | "huge_shared" : " 27 GB", | 20260506 14:33:50.197 | INFO | "malloc" : " 14 GB", | 20260506 14:33:50.197 | INFO | "pinned" : " 29 GB" | 20260506 14:33:50.197 | INFO | } | 20260506 14:33:50.197 | INFO | Perf report for the last 5.13868 seconds: | 20260506 14:33:50.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:33:50.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:50.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:50.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:50.198 | INFO | ANALYSIS : 63.1%, 2.9, 95.6ms, 0 | 20260506 14:33:50.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:50.198 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 14:33:50.198 | INFO | PRE_HQ : 60.9%, 1.9, 59.2ms, 0 | 20260506 14:33:50.198 | INFO | BAZWRITER : 69.4%, 4.8, 171.8ms, 0 | 20260506 14:33:50.198 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 14:33:50.198 | INFO | Uploaded 36GiB to GPU in 2.07269s (17.3688GiB/s) | 20260506 14:33:50.198 | INFO | Downloaded 38GiB to GPU in 2.22794s (17.0561GiB/s) | 20260506 14:33:55.337 | INFO | Analyzing chunk frames = [2105856,2106368) | 20260506 14:33:55.337 | INFO | MemInfo: { | 20260506 14:33:55.337 | INFO | "gpu" : " 17 GB", | 20260506 14:33:55.337 | INFO | "huge" : "", | 20260506 14:33:55.337 | INFO | "huge_shared" : " 27 GB", | 20260506 14:33:55.337 | INFO | "malloc" : " 14 GB", | 20260506 14:33:55.337 | INFO | "pinned" : " 29 GB" | 20260506 14:33:55.337 | INFO | } | 20260506 14:33:55.337 | INFO | Perf report for the last 5.1397 seconds: | 20260506 14:33:55.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:33:55.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:55.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:55.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:55.337 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 14:33:55.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:55.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:55.337 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:33:55.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:55.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:33:55.337 | INFO | Uploaded 36GiB to GPU in 2.00079s (17.9929GiB/s) | 20260506 14:33:55.337 | INFO | Downloaded 34GiB to GPU in 1.94272s (17.5012GiB/s) | 20260506 14:34:00.476 | INFO | Analyzing chunk frames = [2106368,2106880) | 20260506 14:34:00.476 | INFO | MemInfo: { | 20260506 14:34:00.476 | INFO | "gpu" : " 17 GB", | 20260506 14:34:00.476 | INFO | "huge" : "", | 20260506 14:34:00.476 | INFO | "huge_shared" : " 27 GB", | 20260506 14:34:00.476 | INFO | "malloc" : " 14 GB", | 20260506 14:34:00.476 | INFO | "pinned" : " 29 GB" | 20260506 14:34:00.477 | INFO | } | 20260506 14:34:00.477 | INFO | Perf report for the last 5.13954 seconds: | 20260506 14:34:00.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:34:00.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:00.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:00.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:00.477 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:34:00.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:00.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:00.477 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:34:00.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:00.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:00.477 | INFO | Uploaded 36GiB to GPU in 2.00609s (17.9454GiB/s) | 20260506 14:34:00.477 | INFO | Downloaded 34GiB to GPU in 1.94192s (17.5085GiB/s) | 20260506 14:34:05.616 | INFO | Analyzing chunk frames = [2106880,2107392) | 20260506 14:34:05.616 | INFO | MemInfo: { | 20260506 14:34:05.616 | INFO | "gpu" : " 17 GB", | 20260506 14:34:05.616 | INFO | "huge" : "", | 20260506 14:34:05.616 | INFO | "huge_shared" : " 27 GB", | 20260506 14:34:05.616 | INFO | "malloc" : " 14 GB", | 20260506 14:34:05.616 | INFO | "pinned" : " 29 GB" | 20260506 14:34:05.617 | INFO | } | 20260506 14:34:05.617 | INFO | Perf report for the last 5.13992 seconds: | 20260506 14:34:05.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:34:05.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:05.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:05.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:05.617 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260506 14:34:05.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:05.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:05.617 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:34:05.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:05.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:05.617 | INFO | Uploaded 36GiB to GPU in 1.9822s (18.1617GiB/s) | 20260506 14:34:05.617 | INFO | Downloaded 34GiB to GPU in 1.91136s (17.7884GiB/s) | 20260506 14:34:10.755 | INFO | Analyzing chunk frames = [2107392,2107904) | 20260506 14:34:10.755 | INFO | MemInfo: { | 20260506 14:34:10.755 | INFO | "gpu" : " 17 GB", | 20260506 14:34:10.755 | INFO | "huge" : "", | 20260506 14:34:10.755 | INFO | "huge_shared" : " 27 GB", | 20260506 14:34:10.755 | INFO | "malloc" : " 14 GB", | 20260506 14:34:10.755 | INFO | "pinned" : " 29 GB" | 20260506 14:34:10.755 | INFO | } | 20260506 14:34:10.756 | INFO | Perf report for the last 5.13888 seconds: | 20260506 14:34:10.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:34:10.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:10.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:10.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:10.756 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 14:34:10.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:10.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:10.756 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:34:10.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:10.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:10.756 | INFO | Uploaded 36GiB to GPU in 2.03895s (17.6561GiB/s) | 20260506 14:34:10.756 | INFO | Downloaded 34GiB to GPU in 1.93628s (17.5594GiB/s) | 20260506 14:34:15.894 | INFO | Analyzing chunk frames = [2107904,2108416) | 20260506 14:34:15.894 | INFO | MemInfo: { | 20260506 14:34:15.894 | INFO | "gpu" : " 17 GB", | 20260506 14:34:15.894 | INFO | "huge" : "", | 20260506 14:34:15.894 | INFO | "huge_shared" : " 27 GB", | 20260506 14:34:15.894 | INFO | "malloc" : " 14 GB", | 20260506 14:34:15.894 | INFO | "pinned" : " 29 GB" | 20260506 14:34:15.894 | INFO | } | 20260506 14:34:15.894 | INFO | Perf report for the last 5.13887 seconds: | 20260506 14:34:15.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:34:15.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:15.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:15.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:15.895 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 14:34:15.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:15.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:15.895 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260506 14:34:15.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:15.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:15.895 | INFO | Uploaded 36GiB to GPU in 2.03405s (17.6987GiB/s) | 20260506 14:34:15.895 | INFO | Downloaded 34GiB to GPU in 1.89775s (17.9159GiB/s) | 20260506 14:34:21.034 | INFO | Analyzing chunk frames = [2108416,2108928) | 20260506 14:34:21.034 | INFO | MemInfo: { | 20260506 14:34:21.034 | INFO | "gpu" : " 17 GB", | 20260506 14:34:21.034 | INFO | "huge" : "", | 20260506 14:34:21.034 | INFO | "huge_shared" : " 27 GB", | 20260506 14:34:21.034 | INFO | "malloc" : " 14 GB", | 20260506 14:34:21.034 | INFO | "pinned" : " 29 GB" | 20260506 14:34:21.035 | INFO | } | 20260506 14:34:21.035 | INFO | Perf report for the last 5.14016 seconds: | 20260506 14:34:21.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:34:21.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:21.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:21.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:21.035 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:34:21.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:21.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:21.035 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:34:21.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:21.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:21.035 | INFO | Uploaded 36GiB to GPU in 2.00074s (17.9934GiB/s) | 20260506 14:34:21.035 | INFO | Downloaded 34GiB to GPU in 1.99471s (17.0451GiB/s) | 20260506 14:34:26.173 | INFO | Analyzing chunk frames = [2108928,2109440) | 20260506 14:34:26.173 | INFO | MemInfo: { | 20260506 14:34:26.173 | INFO | "gpu" : " 17 GB", | 20260506 14:34:26.173 | INFO | "huge" : "", | 20260506 14:34:26.173 | INFO | "huge_shared" : " 27 GB", | 20260506 14:34:26.173 | INFO | "malloc" : " 14 GB", | 20260506 14:34:26.173 | INFO | "pinned" : " 29 GB" | 20260506 14:34:26.173 | INFO | } | 20260506 14:34:26.173 | INFO | Perf report for the last 5.13846 seconds: | 20260506 14:34:26.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:34:26.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:26.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:26.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:26.173 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:34:26.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:26.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:26.173 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:34:26.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:26.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:26.173 | INFO | Uploaded 36GiB to GPU in 2.0183s (17.8368GiB/s) | 20260506 14:34:26.173 | INFO | Downloaded 34GiB to GPU in 1.97502s (17.215GiB/s) | 20260506 14:34:26.284 | INFO | Starting a round of Baz IO | 20260506 14:34:29.439 | INFO | Populating RTMetrics report | 20260506 14:34:29.440 | INFO | RTMetrics report populated | 20260506 14:34:29.440 | INFO | Opening tmp RTMetrics json file | 20260506 14:34:29.447 | INFO | Serializing RTMetrics to json | 20260506 14:34:29.449 | INFO | Serialization done, swapping tmp file now... | 20260506 14:34:29.452 | INFO | RTMetrics done | 20260506 14:34:29.865 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.581 seconds | 20260506 14:34:29.865 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.15% / 0.01% / 61.84% | 20260506 14:34:29.865 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.4967% / 2.78155e-05% / 6.75519e-05% | 20260506 14:34:29.865 | INFO | Baz IO overall: | 20260506 14:34:29.865 | INFO | BytesWritten : 1090030231077 | 20260506 14:34:29.865 | INFO | NumEvents : 527473928451 | 20260506 14:34:31.312 | INFO | Analyzing chunk frames = [2109440,2109952) | 20260506 14:34:31.312 | INFO | MemInfo: { | 20260506 14:34:31.312 | INFO | "gpu" : " 17 GB", | 20260506 14:34:31.312 | INFO | "huge" : "", | 20260506 14:34:31.312 | INFO | "huge_shared" : " 27 GB", | 20260506 14:34:31.313 | INFO | "malloc" : " 14 GB", | 20260506 14:34:31.313 | INFO | "pinned" : " 29 GB" | 20260506 14:34:31.313 | INFO | } | 20260506 14:34:31.313 | INFO | Perf report for the last 5.13961 seconds: | 20260506 14:34:31.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:34:31.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:31.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:31.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:31.313 | INFO | ANALYSIS : 63.0%, 2.9, 95.5ms, 0 | 20260506 14:34:31.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:31.313 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 14:34:31.313 | INFO | PRE_HQ : 60.8%, 1.9, 59.4ms, 0 | 20260506 14:34:31.313 | INFO | BAZWRITER : 68.9%, 4.8, 171.4ms, 0 | 20260506 14:34:31.313 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260506 14:34:31.313 | INFO | Uploaded 36GiB to GPU in 2.08052s (17.3034GiB/s) | 20260506 14:34:31.313 | INFO | Downloaded 38GiB to GPU in 2.22989s (17.0412GiB/s) | 20260506 14:34:36.452 | INFO | Analyzing chunk frames = [2109952,2110464) | 20260506 14:34:36.452 | INFO | MemInfo: { | 20260506 14:34:36.452 | INFO | "gpu" : " 17 GB", | 20260506 14:34:36.452 | INFO | "huge" : "", | 20260506 14:34:36.452 | INFO | "huge_shared" : " 27 GB", | 20260506 14:34:36.452 | INFO | "malloc" : " 14 GB", | 20260506 14:34:36.452 | INFO | "pinned" : " 29 GB" | 20260506 14:34:36.452 | INFO | } | 20260506 14:34:36.452 | INFO | Perf report for the last 5.13935 seconds: | 20260506 14:34:36.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:34:36.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:36.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:36.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:36.452 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 14:34:36.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:36.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:36.452 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:34:36.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:36.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:36.452 | INFO | Uploaded 36GiB to GPU in 2.00889s (17.9204GiB/s) | 20260506 14:34:36.452 | INFO | Downloaded 34GiB to GPU in 1.94282s (17.5003GiB/s) | 20260506 14:34:41.591 | INFO | Analyzing chunk frames = [2110464,2110976) | 20260506 14:34:41.591 | INFO | MemInfo: { | 20260506 14:34:41.591 | INFO | "gpu" : " 17 GB", | 20260506 14:34:41.591 | INFO | "huge" : "", | 20260506 14:34:41.591 | INFO | "huge_shared" : " 27 GB", | 20260506 14:34:41.591 | INFO | "malloc" : " 14 GB", | 20260506 14:34:41.591 | INFO | "pinned" : " 29 GB" | 20260506 14:34:41.591 | INFO | } | 20260506 14:34:41.591 | INFO | Perf report for the last 5.13913 seconds: | 20260506 14:34:41.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:34:41.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:41.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:41.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:41.591 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 14:34:41.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:41.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:41.591 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:34:41.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:41.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:41.591 | INFO | Uploaded 36GiB to GPU in 1.98538s (18.1326GiB/s) | 20260506 14:34:41.591 | INFO | Downloaded 34GiB to GPU in 1.93328s (17.5867GiB/s) | 20260506 14:34:46.730 | INFO | Analyzing chunk frames = [2110976,2111488) | 20260506 14:34:46.730 | INFO | MemInfo: { | 20260506 14:34:46.730 | INFO | "gpu" : " 17 GB", | 20260506 14:34:46.730 | INFO | "huge" : "", | 20260506 14:34:46.730 | INFO | "huge_shared" : " 27 GB", | 20260506 14:34:46.730 | INFO | "malloc" : " 14 GB", | 20260506 14:34:46.730 | INFO | "pinned" : " 29 GB" | 20260506 14:34:46.730 | INFO | } | 20260506 14:34:46.730 | INFO | Perf report for the last 5.13913 seconds: | 20260506 14:34:46.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:34:46.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:46.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:46.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:46.730 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 14:34:46.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:46.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:46.730 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:34:46.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:46.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:46.731 | INFO | Uploaded 36GiB to GPU in 2.01659s (17.8519GiB/s) | 20260506 14:34:46.731 | INFO | Downloaded 34GiB to GPU in 1.91557s (17.7492GiB/s) | 20260506 14:34:51.870 | INFO | Analyzing chunk frames = [2111488,2112000) | 20260506 14:34:51.870 | INFO | MemInfo: { | 20260506 14:34:51.870 | INFO | "gpu" : " 17 GB", | 20260506 14:34:51.870 | INFO | "huge" : "", | 20260506 14:34:51.870 | INFO | "huge_shared" : " 27 GB", | 20260506 14:34:51.870 | INFO | "malloc" : " 14 GB", | 20260506 14:34:51.870 | INFO | "pinned" : " 29 GB" | 20260506 14:34:51.870 | INFO | } | 20260506 14:34:51.870 | INFO | Perf report for the last 5.13985 seconds: | 20260506 14:34:51.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:34:51.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:51.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:51.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:51.870 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 14:34:51.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:51.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:51.870 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:34:51.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:51.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:51.871 | INFO | Uploaded 36GiB to GPU in 1.99131s (18.0786GiB/s) | 20260506 14:34:51.871 | INFO | Downloaded 34GiB to GPU in 1.92677s (17.6461GiB/s) | 20260506 14:34:57.009 | INFO | Analyzing chunk frames = [2112000,2112512) | 20260506 14:34:57.009 | INFO | MemInfo: { | 20260506 14:34:57.009 | INFO | "gpu" : " 17 GB", | 20260506 14:34:57.009 | INFO | "huge" : "", | 20260506 14:34:57.009 | INFO | "huge_shared" : " 27 GB", | 20260506 14:34:57.009 | INFO | "malloc" : " 14 GB", | 20260506 14:34:57.009 | INFO | "pinned" : " 29 GB" | 20260506 14:34:57.009 | INFO | } | 20260506 14:34:57.009 | INFO | Perf report for the last 5.139 seconds: | 20260506 14:34:57.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:34:57.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:57.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:57.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:57.009 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:34:57.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:57.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:57.009 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 14:34:57.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:57.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:34:57.009 | INFO | Uploaded 36GiB to GPU in 2.08482s (17.2677GiB/s) | 20260506 14:34:57.009 | INFO | Downloaded 34GiB to GPU in 1.97477s (17.2172GiB/s) | 20260506 14:35:02.149 | INFO | Analyzing chunk frames = [2112512,2113024) | 20260506 14:35:02.149 | INFO | MemInfo: { | 20260506 14:35:02.149 | INFO | "gpu" : " 17 GB", | 20260506 14:35:02.149 | INFO | "huge" : "", | 20260506 14:35:02.149 | INFO | "huge_shared" : " 27 GB", | 20260506 14:35:02.149 | INFO | "malloc" : " 14 GB", | 20260506 14:35:02.149 | INFO | "pinned" : " 29 GB" | 20260506 14:35:02.149 | INFO | } | 20260506 14:35:02.150 | INFO | Perf report for the last 5.14034 seconds: | 20260506 14:35:02.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:35:02.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:02.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:02.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:02.150 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 14:35:02.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:02.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:02.150 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 14:35:02.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:02.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:02.150 | INFO | Uploaded 36GiB to GPU in 2.0008s (17.9928GiB/s) | 20260506 14:35:02.150 | INFO | Downloaded 34GiB to GPU in 2.01151s (16.9027GiB/s) | 20260506 14:35:07.287 | INFO | Analyzing chunk frames = [2113024,2113536) | 20260506 14:35:07.287 | INFO | MemInfo: { | 20260506 14:35:07.287 | INFO | "gpu" : " 17 GB", | 20260506 14:35:07.287 | INFO | "huge" : "", | 20260506 14:35:07.288 | INFO | "huge_shared" : " 27 GB", | 20260506 14:35:07.288 | INFO | "malloc" : " 14 GB", | 20260506 14:35:07.288 | INFO | "pinned" : " 29 GB" | 20260506 14:35:07.288 | INFO | } | 20260506 14:35:07.288 | INFO | Perf report for the last 5.13804 seconds: | 20260506 14:35:07.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:35:07.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:07.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:07.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:07.288 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:35:07.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:07.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:07.288 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260506 14:35:07.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:07.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:07.288 | INFO | Uploaded 36GiB to GPU in 2.02034s (17.8188GiB/s) | 20260506 14:35:07.288 | INFO | Downloaded 34GiB to GPU in 1.9828s (17.1475GiB/s) | 20260506 14:35:07.288 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:35:07.288 | INFO | Total mem usage: 17.4 GiB | 20260506 14:35:07.288 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:35:07.288 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -6.12%) | 20260506 14:35:07.288 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:35:07.288 | INFO | High Water Breakdown: | 20260506 14:35:07.288 | INFO | 61.9% : 10.2 GiB ( 93.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:35:07.288 | INFO | 37.9% : 6.22 GiB ( 93% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:35:07.289 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:35:07.289 | INFO | ------------------------------------------ | 20260506 14:35:07.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:35:07.289 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:35:07.289 | INFO | Total mem usage: 26.5 GiB | 20260506 14:35:07.289 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:35:07.289 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:35:07.289 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:35:07.289 | INFO | High Water Breakdown: | 20260506 14:35:07.289 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:35:07.289 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:35:07.289 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:35:07.289 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:35:07.289 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:35:07.289 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:35:07.289 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:35:07.289 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:35:07.289 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:35:07.289 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:35:07.289 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:35:07.289 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:35:07.289 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:35:07.289 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:35:07.289 | INFO | ------------------------------------------ | 20260506 14:35:07.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:35:07.289 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:35:07.289 | INFO | Total mem usage: 25.6 GiB | 20260506 14:35:07.289 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:35:07.289 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:35:07.289 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:35:07.289 | INFO | High Water Breakdown: | 20260506 14:35:07.289 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:35:07.289 | INFO | ------------------------------------------ | 20260506 14:35:07.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:35:07.289 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:35:07.289 | INFO | Total mem usage: 15.7 GiB | 20260506 14:35:07.289 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:35:07.289 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:35:07.289 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 14:35:07.289 | INFO | High Water Breakdown: | 20260506 14:35:07.289 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:35:07.289 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:35:07.289 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:35:07.289 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:35:07.289 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:35:07.289 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:35:07.289 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:35:07.289 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:35:07.289 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:35:07.289 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:35:07.289 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:35:07.289 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:35:07.289 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:35:07.289 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:35:07.289 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:35:07.289 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:35:07.289 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:35:07.289 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:35:07.289 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:35:07.289 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:35:07.289 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:35:07.289 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:35:07.289 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:35:07.289 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:35:07.289 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:35:07.289 | INFO | ------------------------------------------ | 20260506 14:35:07.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:35:07.289 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:35:07.289 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:35:07.289 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:35:07.289 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.56e+01, 1.36e+01 / 2.12e+00 | 20260506 14:35:07.289 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.01e-01, 9.95e-03 / 5.78e-03 | 20260506 14:35:07.289 | INFO | PerfReport: 11.59%, Baseline -- 3.65e+00 / 4.46e+00, 5.68e-01 / 3.15e-01 | 20260506 14:35:07.289 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.48e+00, 1.91e-01 / 1.32e-01 | 20260506 14:35:07.290 | INFO | PerfReport: 15.87%, DME -- 4.00e+01 / 4.43e+01, 3.93e+00 / 1.83e+00 | 20260506 14:35:07.290 | INFO | PerfReport: 56.65%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.41e+00 / 1.10e+00 | 20260506 14:35:07.290 | INFO | PerfReport: 7.47%, PulseAccumulating -- 2.36e+00 / 2.95e+00, 4.13e-01 / 2.42e-01 | 20260506 14:35:07.290 | INFO | PerfReport: 3.61%, Metrics -- 1.14e+00 / 1.49e+00, 2.87e-01 / 1.99e-01 | 20260506 14:35:07.290 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.97e-01 / 4.47e-01, 6.18e-02 / 3.11e-02 | 20260506 14:35:07.290 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:35:07.290 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:35:07.290 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:35:07.290 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 14:35:07.290 | INFO | PerfReport: 21.53%, Upload -- 2.11e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 14:35:07.290 | INFO | PerfReport: 21.13%, Download -- 2.07e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 14:35:07.290 | INFO | PerfReport: 32.22%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 14:35:07.290 | INFO | PerfReport: 1.62%, LockUpload -- 1.58e+00 / 0.00e+00, 6.08e+00 / 0.00e+00 | 20260506 14:35:07.290 | INFO | PerfReport: 1.05%, LockDownload -- 1.03e+00 / 0.00e+00, 2.61e+00 / 0.00e+00 | 20260506 14:35:07.290 | INFO | PerfReport: 22.42%, LockCompute -- 2.19e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 14:35:07.290 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.46e-02 / 0.00e+00 | 20260506 14:35:07.405 | INFO | Starting a round of Baz IO | 20260506 14:35:10.593 | INFO | Populating RTMetrics report | 20260506 14:35:10.594 | INFO | RTMetrics report populated | 20260506 14:35:10.594 | INFO | Opening tmp RTMetrics json file | 20260506 14:35:10.625 | INFO | Serializing RTMetrics to json | 20260506 14:35:10.630 | INFO | Serialization done, swapping tmp file now... | 20260506 14:35:10.633 | INFO | RTMetrics done | 20260506 14:35:11.067 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.661 seconds | 20260506 14:35:11.067 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.14% / 0.01% / 61.85% | 20260506 14:35:11.067 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.49089% / 1.98682e-05% / 6.35783e-05% | 20260506 14:35:11.067 | INFO | Baz IO overall: | 20260506 14:35:11.067 | INFO | BytesWritten : 1092299699144 | 20260506 14:35:11.067 | INFO | NumEvents : 528574380937 | 20260506 14:35:12.427 | INFO | Analyzing chunk frames = [2113536,2114048) | 20260506 14:35:12.427 | INFO | MemInfo: { | 20260506 14:35:12.427 | INFO | "gpu" : " 17 GB", | 20260506 14:35:12.427 | INFO | "huge" : "", | 20260506 14:35:12.427 | INFO | "huge_shared" : " 27 GB", | 20260506 14:35:12.427 | INFO | "malloc" : " 14 GB", | 20260506 14:35:12.427 | INFO | "pinned" : " 29 GB" | 20260506 14:35:12.427 | INFO | } | 20260506 14:35:12.427 | INFO | Perf report for the last 5.13929 seconds: | 20260506 14:35:12.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:35:12.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:12.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:12.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:12.427 | INFO | ANALYSIS : 63.4%, 2.9, 94.9ms, 0 | 20260506 14:35:12.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:12.427 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260506 14:35:12.427 | INFO | PRE_HQ : 61.3%, 1.9, 61.9ms, 0 | 20260506 14:35:12.427 | INFO | BAZWRITER : 70.6%, 4.8, 175.3ms, 0 | 20260506 14:35:12.427 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 14:35:12.427 | INFO | Uploaded 36GiB to GPU in 2.06039s (17.4724GiB/s) | 20260506 14:35:12.427 | INFO | Downloaded 38GiB to GPU in 2.20481s (17.2351GiB/s) | 20260506 14:35:17.566 | INFO | Analyzing chunk frames = [2114048,2114560) | 20260506 14:35:17.566 | INFO | MemInfo: { | 20260506 14:35:17.566 | INFO | "gpu" : " 17 GB", | 20260506 14:35:17.566 | INFO | "huge" : "", | 20260506 14:35:17.566 | INFO | "huge_shared" : " 27 GB", | 20260506 14:35:17.566 | INFO | "malloc" : " 14 GB", | 20260506 14:35:17.566 | INFO | "pinned" : " 29 GB" | 20260506 14:35:17.566 | INFO | } | 20260506 14:35:17.566 | INFO | Perf report for the last 5.13916 seconds: | 20260506 14:35:17.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:35:17.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:17.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:17.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:17.567 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:35:17.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:17.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:17.567 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:35:17.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:17.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:17.567 | INFO | Uploaded 36GiB to GPU in 1.9979s (18.0189GiB/s) | 20260506 14:35:17.567 | INFO | Downloaded 34GiB to GPU in 1.94076s (17.5189GiB/s) | 20260506 14:35:22.707 | INFO | Analyzing chunk frames = [2114560,2115072) | 20260506 14:35:22.707 | INFO | MemInfo: { | 20260506 14:35:22.707 | INFO | "gpu" : " 17 GB", | 20260506 14:35:22.707 | INFO | "huge" : "", | 20260506 14:35:22.707 | INFO | "huge_shared" : " 27 GB", | 20260506 14:35:22.707 | INFO | "malloc" : " 14 GB", | 20260506 14:35:22.707 | INFO | "pinned" : " 29 GB" | 20260506 14:35:22.707 | INFO | } | 20260506 14:35:22.707 | INFO | Perf report for the last 5.14127 seconds: | 20260506 14:35:22.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:35:22.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:22.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:22.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:22.707 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 14:35:22.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:22.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:22.707 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 14:35:22.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:22.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:22.708 | INFO | Uploaded 36GiB to GPU in 1.98708s (18.1171GiB/s) | 20260506 14:35:22.708 | INFO | Downloaded 34GiB to GPU in 1.9377s (17.5466GiB/s) | 20260506 14:35:27.846 | INFO | Analyzing chunk frames = [2115072,2115584) | 20260506 14:35:27.846 | INFO | MemInfo: { | 20260506 14:35:27.846 | INFO | "gpu" : " 17 GB", | 20260506 14:35:27.846 | INFO | "huge" : "", | 20260506 14:35:27.846 | INFO | "huge_shared" : " 27 GB", | 20260506 14:35:27.846 | INFO | "malloc" : " 14 GB", | 20260506 14:35:27.846 | INFO | "pinned" : " 29 GB" | 20260506 14:35:27.846 | INFO | } | 20260506 14:35:27.846 | INFO | Perf report for the last 5.13873 seconds: | 20260506 14:35:27.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:35:27.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:27.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:27.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:27.846 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 14:35:27.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:27.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:27.846 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 14:35:27.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:27.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:27.847 | INFO | Uploaded 36GiB to GPU in 1.9944s (18.0505GiB/s) | 20260506 14:35:27.847 | INFO | Downloaded 34GiB to GPU in 1.9103s (17.7982GiB/s) | 20260506 14:35:32.985 | INFO | Analyzing chunk frames = [2115584,2116096) | 20260506 14:35:32.985 | INFO | MemInfo: { | 20260506 14:35:32.985 | INFO | "gpu" : " 17 GB", | 20260506 14:35:32.985 | INFO | "huge" : "", | 20260506 14:35:32.985 | INFO | "huge_shared" : " 27 GB", | 20260506 14:35:32.985 | INFO | "malloc" : " 14 GB", | 20260506 14:35:32.985 | INFO | "pinned" : " 29 GB" | 20260506 14:35:32.985 | INFO | } | 20260506 14:35:32.985 | INFO | Perf report for the last 5.13877 seconds: | 20260506 14:35:32.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:35:32.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:32.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:32.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:32.985 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 14:35:32.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:32.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:32.985 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:35:32.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:32.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:32.985 | INFO | Uploaded 36GiB to GPU in 1.99132s (18.0785GiB/s) | 20260506 14:35:32.985 | INFO | Downloaded 34GiB to GPU in 1.93987s (17.5269GiB/s) | 20260506 14:35:38.123 | INFO | Analyzing chunk frames = [2116096,2116608) | 20260506 14:35:38.123 | INFO | MemInfo: { | 20260506 14:35:38.123 | INFO | "gpu" : " 17 GB", | 20260506 14:35:38.123 | INFO | "huge" : "", | 20260506 14:35:38.123 | INFO | "huge_shared" : " 27 GB", | 20260506 14:35:38.123 | INFO | "malloc" : " 14 GB", | 20260506 14:35:38.123 | INFO | "pinned" : " 29 GB" | 20260506 14:35:38.123 | INFO | } | 20260506 14:35:38.123 | INFO | Perf report for the last 5.13847 seconds: | 20260506 14:35:38.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:35:38.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:38.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:38.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:38.123 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:35:38.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:38.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:38.123 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:35:38.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:38.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:38.124 | INFO | Uploaded 36GiB to GPU in 2.08253s (17.2867GiB/s) | 20260506 14:35:38.124 | INFO | Downloaded 34GiB to GPU in 1.97024s (17.2567GiB/s) | 20260506 14:35:43.263 | INFO | Analyzing chunk frames = [2116608,2117120) | 20260506 14:35:43.263 | INFO | MemInfo: { | 20260506 14:35:43.263 | INFO | "gpu" : " 17 GB", | 20260506 14:35:43.263 | INFO | "huge" : "", | 20260506 14:35:43.263 | INFO | "huge_shared" : " 27 GB", | 20260506 14:35:43.263 | INFO | "malloc" : " 14 GB", | 20260506 14:35:43.263 | INFO | "pinned" : " 29 GB" | 20260506 14:35:43.263 | INFO | } | 20260506 14:35:43.263 | INFO | Perf report for the last 5.1401 seconds: | 20260506 14:35:43.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:35:43.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:43.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:43.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:43.264 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:35:43.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:43.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:43.264 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:35:43.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:43.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:43.264 | INFO | Uploaded 36GiB to GPU in 2.0166s (17.8518GiB/s) | 20260506 14:35:43.264 | INFO | Downloaded 34GiB to GPU in 2.02268s (16.8094GiB/s) | 20260506 14:35:48.402 | INFO | Analyzing chunk frames = [2117120,2117632) | 20260506 14:35:48.402 | INFO | MemInfo: { | 20260506 14:35:48.402 | INFO | "gpu" : " 17 GB", | 20260506 14:35:48.402 | INFO | "huge" : "", | 20260506 14:35:48.402 | INFO | "huge_shared" : " 27 GB", | 20260506 14:35:48.402 | INFO | "malloc" : " 14 GB", | 20260506 14:35:48.402 | INFO | "pinned" : " 29 GB" | 20260506 14:35:48.402 | INFO | } | 20260506 14:35:48.403 | INFO | Perf report for the last 5.13905 seconds: | 20260506 14:35:48.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:35:48.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:48.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:48.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:48.403 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:35:48.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:48.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:48.403 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260506 14:35:48.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:48.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:48.403 | INFO | Uploaded 36GiB to GPU in 2.04458s (17.6075GiB/s) | 20260506 14:35:48.403 | INFO | Downloaded 34GiB to GPU in 1.96368s (17.3145GiB/s) | 20260506 14:35:48.526 | INFO | Starting a round of Baz IO | 20260506 14:35:51.676 | INFO | Populating RTMetrics report | 20260506 14:35:51.677 | INFO | RTMetrics report populated | 20260506 14:35:51.677 | INFO | Opening tmp RTMetrics json file | 20260506 14:35:51.678 | INFO | Serializing RTMetrics to json | 20260506 14:35:51.700 | INFO | Serialization done, swapping tmp file now... | 20260506 14:35:51.710 | INFO | RTMetrics done | 20260506 14:35:51.880 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 3.353 seconds | 20260506 14:35:51.880 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.12% / 0.01% / 61.86% | 20260506 14:35:51.880 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.48223% / 2.38419e-05% / 7.15256e-05% | 20260506 14:35:51.880 | INFO | Baz IO overall: | 20260506 14:35:51.880 | INFO | BytesWritten : 1094568858053 | 20260506 14:35:51.880 | INFO | NumEvents : 529674674846 | 20260506 14:35:53.543 | INFO | Analyzing chunk frames = [2117632,2118144) | 20260506 14:35:53.543 | INFO | MemInfo: { | 20260506 14:35:53.543 | INFO | "gpu" : " 17 GB", | 20260506 14:35:53.543 | INFO | "huge" : "", | 20260506 14:35:53.543 | INFO | "huge_shared" : " 27 GB", | 20260506 14:35:53.543 | INFO | "malloc" : " 14 GB", | 20260506 14:35:53.543 | INFO | "pinned" : " 29 GB" | 20260506 14:35:53.543 | INFO | } | 20260506 14:35:53.543 | INFO | Perf report for the last 5.14081 seconds: | 20260506 14:35:53.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:35:53.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:53.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:53.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:53.543 | INFO | ANALYSIS : 63.1%, 2.9, 95.0ms, 0 | 20260506 14:35:53.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:53.543 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 14:35:53.543 | INFO | PRE_HQ : 61.1%, 1.9, 59.8ms, 0 | 20260506 14:35:53.544 | INFO | BAZWRITER : 64.6%, 4.8, 160.3ms, 0 | 20260506 14:35:53.544 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 14:35:53.544 | INFO | Uploaded 36GiB to GPU in 2.06933s (17.3969GiB/s) | 20260506 14:35:53.544 | INFO | Downloaded 38GiB to GPU in 2.24206s (16.9487GiB/s) | 20260506 14:35:58.681 | INFO | Analyzing chunk frames = [2118144,2118656) | 20260506 14:35:58.681 | INFO | MemInfo: { | 20260506 14:35:58.681 | INFO | "gpu" : " 17 GB", | 20260506 14:35:58.681 | INFO | "huge" : "", | 20260506 14:35:58.681 | INFO | "huge_shared" : " 27 GB", | 20260506 14:35:58.681 | INFO | "malloc" : " 14 GB", | 20260506 14:35:58.681 | INFO | "pinned" : " 29 GB" | 20260506 14:35:58.681 | INFO | } | 20260506 14:35:58.681 | INFO | Perf report for the last 5.13784 seconds: | 20260506 14:35:58.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:35:58.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:58.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:58.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:58.681 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:35:58.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:58.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:58.681 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:35:58.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:58.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:35:58.681 | INFO | Uploaded 36GiB to GPU in 1.99858s (18.0128GiB/s) | 20260506 14:35:58.682 | INFO | Downloaded 34GiB to GPU in 1.93824s (17.5417GiB/s) | 20260506 14:36:03.821 | INFO | Analyzing chunk frames = [2118656,2119168) | 20260506 14:36:03.821 | INFO | MemInfo: { | 20260506 14:36:03.821 | INFO | "gpu" : " 17 GB", | 20260506 14:36:03.821 | INFO | "huge" : "", | 20260506 14:36:03.821 | INFO | "huge_shared" : " 27 GB", | 20260506 14:36:03.821 | INFO | "malloc" : " 14 GB", | 20260506 14:36:03.821 | INFO | "pinned" : " 29 GB" | 20260506 14:36:03.821 | INFO | } | 20260506 14:36:03.821 | INFO | Perf report for the last 5.13974 seconds: | 20260506 14:36:03.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:36:03.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:03.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:03.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:03.821 | INFO | ANALYSIS : 61.3%, 3.0, 94.8ms, 0 | 20260506 14:36:03.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:03.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:03.821 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:36:03.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:03.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:03.821 | INFO | Uploaded 36GiB to GPU in 2.00058s (17.9948GiB/s) | 20260506 14:36:03.821 | INFO | Downloaded 34GiB to GPU in 1.93898s (17.535GiB/s) | 20260506 14:36:08.960 | INFO | Analyzing chunk frames = [2119168,2119680) | 20260506 14:36:08.960 | INFO | MemInfo: { | 20260506 14:36:08.960 | INFO | "gpu" : " 17 GB", | 20260506 14:36:08.960 | INFO | "huge" : "", | 20260506 14:36:08.960 | INFO | "huge_shared" : " 27 GB", | 20260506 14:36:08.960 | INFO | "malloc" : " 14 GB", | 20260506 14:36:08.960 | INFO | "pinned" : " 29 GB" | 20260506 14:36:08.960 | INFO | } | 20260506 14:36:08.960 | INFO | Perf report for the last 5.13939 seconds: | 20260506 14:36:08.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:36:08.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:08.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:08.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:08.960 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 14:36:08.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:08.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:08.960 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:36:08.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:08.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:08.961 | INFO | Uploaded 36GiB to GPU in 1.9866s (18.1214GiB/s) | 20260506 14:36:08.961 | INFO | Downloaded 34GiB to GPU in 1.91206s (17.7819GiB/s) | 20260506 14:36:14.100 | INFO | Analyzing chunk frames = [2119680,2120192) | 20260506 14:36:14.100 | INFO | MemInfo: { | 20260506 14:36:14.100 | INFO | "gpu" : " 17 GB", | 20260506 14:36:14.100 | INFO | "huge" : "", | 20260506 14:36:14.100 | INFO | "huge_shared" : " 27 GB", | 20260506 14:36:14.100 | INFO | "malloc" : " 14 GB", | 20260506 14:36:14.100 | INFO | "pinned" : " 29 GB" | 20260506 14:36:14.100 | INFO | } | 20260506 14:36:14.100 | INFO | Perf report for the last 5.13968 seconds: | 20260506 14:36:14.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:36:14.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:14.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:14.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:14.100 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 14:36:14.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:14.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:14.100 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:36:14.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:14.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:14.100 | INFO | Uploaded 36GiB to GPU in 2.0219s (17.805GiB/s) | 20260506 14:36:14.100 | INFO | Downloaded 34GiB to GPU in 1.92408s (17.6708GiB/s) | 20260506 14:36:19.239 | INFO | Analyzing chunk frames = [2120192,2120704) | 20260506 14:36:19.239 | INFO | MemInfo: { | 20260506 14:36:19.239 | INFO | "gpu" : " 17 GB", | 20260506 14:36:19.239 | INFO | "huge" : "", | 20260506 14:36:19.239 | INFO | "huge_shared" : " 27 GB", | 20260506 14:36:19.239 | INFO | "malloc" : " 14 GB", | 20260506 14:36:19.239 | INFO | "pinned" : " 29 GB" | 20260506 14:36:19.239 | INFO | } | 20260506 14:36:19.239 | INFO | Perf report for the last 5.13876 seconds: | 20260506 14:36:19.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:36:19.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:19.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:19.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:19.239 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 14:36:19.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:19.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:19.239 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:36:19.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:19.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:19.239 | INFO | Uploaded 36GiB to GPU in 2.05066s (17.5553GiB/s) | 20260506 14:36:19.239 | INFO | Downloaded 34GiB to GPU in 1.92378s (17.6735GiB/s) | 20260506 14:36:24.378 | INFO | Analyzing chunk frames = [2120704,2121216) | 20260506 14:36:24.378 | INFO | MemInfo: { | 20260506 14:36:24.378 | INFO | "gpu" : " 17 GB", | 20260506 14:36:24.378 | INFO | "huge" : "", | 20260506 14:36:24.378 | INFO | "huge_shared" : " 27 GB", | 20260506 14:36:24.378 | INFO | "malloc" : " 14 GB", | 20260506 14:36:24.378 | INFO | "pinned" : " 29 GB" | 20260506 14:36:24.378 | INFO | } | 20260506 14:36:24.378 | INFO | Perf report for the last 5.13902 seconds: | 20260506 14:36:24.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:36:24.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:24.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:24.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:24.378 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 14:36:24.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:24.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:24.378 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 14:36:24.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:24.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:24.378 | INFO | Uploaded 36GiB to GPU in 1.96369s (18.3329GiB/s) | 20260506 14:36:24.378 | INFO | Downloaded 34GiB to GPU in 1.99098s (17.077GiB/s) | 20260506 14:36:29.517 | INFO | Analyzing chunk frames = [2121216,2121728) | 20260506 14:36:29.517 | INFO | MemInfo: { | 20260506 14:36:29.517 | INFO | "gpu" : " 17 GB", | 20260506 14:36:29.517 | INFO | "huge" : "", | 20260506 14:36:29.517 | INFO | "huge_shared" : " 27 GB", | 20260506 14:36:29.517 | INFO | "malloc" : " 14 GB", | 20260506 14:36:29.517 | INFO | "pinned" : " 29 GB" | 20260506 14:36:29.517 | INFO | } | 20260506 14:36:29.518 | INFO | Perf report for the last 5.13969 seconds: | 20260506 14:36:29.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:36:29.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:29.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:29.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:29.518 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 14:36:29.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:29.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:29.518 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:36:29.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:29.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:29.518 | INFO | Uploaded 36GiB to GPU in 1.97841s (18.1964GiB/s) | 20260506 14:36:29.518 | INFO | Downloaded 34GiB to GPU in 1.97365s (17.227GiB/s) | 20260506 14:36:29.518 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:36:29.518 | INFO | Total mem usage: 17.4 GiB | 20260506 14:36:29.518 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:36:29.518 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -7.61%) | 20260506 14:36:29.518 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:36:29.518 | INFO | High Water Breakdown: | 20260506 14:36:29.518 | INFO | 61.7% : 9.99 GiB ( 91.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:36:29.518 | INFO | 38.2% : 6.17 GiB ( 92.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:36:29.518 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:36:29.518 | INFO | ------------------------------------------ | 20260506 14:36:29.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:36:29.518 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:36:29.518 | INFO | Total mem usage: 26.5 GiB | 20260506 14:36:29.518 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:36:29.518 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 14:36:29.518 | INFO | Unused Cache: 1.67 GiB (Waste: 6.3%) | 20260506 14:36:29.518 | INFO | High Water Breakdown: | 20260506 14:36:29.518 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:36:29.518 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:36:29.518 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:36:29.518 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:36:29.518 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:36:29.518 | INFO | 0.698% : 182 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:36:29.518 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:36:29.519 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:36:29.519 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:36:29.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:36:29.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:36:29.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:36:29.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:36:29.519 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:36:29.519 | INFO | ------------------------------------------ | 20260506 14:36:29.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:36:29.519 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:36:29.519 | INFO | Total mem usage: 25.6 GiB | 20260506 14:36:29.519 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:36:29.519 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:36:29.519 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:36:29.519 | INFO | High Water Breakdown: | 20260506 14:36:29.519 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:36:29.519 | INFO | ------------------------------------------ | 20260506 14:36:29.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:36:29.519 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:36:29.519 | INFO | Total mem usage: 15.7 GiB | 20260506 14:36:29.519 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:36:29.519 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:36:29.519 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 14:36:29.519 | INFO | High Water Breakdown: | 20260506 14:36:29.519 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:36:29.519 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:36:29.519 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:36:29.519 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:36:29.519 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:36:29.519 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:36:29.519 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:36:29.519 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:36:29.519 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:36:29.519 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:36:29.519 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:36:29.519 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:36:29.519 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:36:29.519 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:36:29.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:36:29.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:36:29.519 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:36:29.519 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:36:29.519 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:36:29.519 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:36:29.519 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:36:29.519 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:36:29.519 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:36:29.519 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:36:29.519 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:36:29.519 | INFO | ------------------------------------------ | 20260506 14:36:29.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:36:29.519 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:36:29.519 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:36:29.519 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:36:29.519 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.60e+01, 1.35e+01 / 2.73e+00 | 20260506 14:36:29.519 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.02e-01, 1.07e-02 / 1.24e-02 | 20260506 14:36:29.519 | INFO | PerfReport: 11.58%, Baseline -- 3.65e+00 / 4.40e+00, 5.70e-01 / 3.70e-01 | 20260506 14:36:29.519 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.46e+00, 1.90e-01 / 1.44e-01 | 20260506 14:36:29.519 | INFO | PerfReport: 15.88%, DME -- 4.01e+01 / 4.48e+01, 4.18e+00 / 2.88e+00 | 20260506 14:36:29.519 | INFO | PerfReport: 56.68%, FrameLabeling -- 1.79e+01 / 2.03e+01, 1.45e+00 / 1.81e+00 | 20260506 14:36:29.519 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.35e+00 / 2.93e+00, 4.10e-01 / 2.59e-01 | 20260506 14:36:29.519 | INFO | PerfReport: 3.61%, Metrics -- 1.14e+00 / 1.53e+00, 2.95e-01 / 1.77e-01 | 20260506 14:36:29.519 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.97e-01 / 4.49e-01, 6.24e-02 / 2.28e-02 | 20260506 14:36:29.519 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:36:29.519 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:36:29.519 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:36:29.519 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 14:36:29.519 | INFO | PerfReport: 21.49%, Upload -- 2.10e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260506 14:36:29.519 | INFO | PerfReport: 21.15%, Download -- 2.07e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 14:36:29.519 | INFO | PerfReport: 32.29%, Compute -- 3.16e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 14:36:29.519 | INFO | PerfReport: 1.61%, LockUpload -- 1.57e+00 / 0.00e+00, 5.97e+00 / 0.00e+00 | 20260506 14:36:29.519 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.69e+00 / 0.00e+00 | 20260506 14:36:29.519 | INFO | PerfReport: 22.36%, LockCompute -- 2.18e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:36:29.519 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260506 14:36:29.637 | INFO | Starting a round of Baz IO | 20260506 14:36:32.781 | INFO | Populating RTMetrics report | 20260506 14:36:32.782 | INFO | RTMetrics report populated | 20260506 14:36:32.782 | INFO | Opening tmp RTMetrics json file | 20260506 14:36:32.878 | INFO | Serializing RTMetrics to json | 20260506 14:36:32.882 | INFO | Serialization done, swapping tmp file now... | 20260506 14:36:32.889 | INFO | RTMetrics done | 20260506 14:36:33.012 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.374 seconds | 20260506 14:36:33.012 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.11% / 0.01% / 61.88% | 20260506 14:36:33.012 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.47927% / 1.58946e-05% / 7.54992e-05% | 20260506 14:36:33.012 | INFO | Baz IO overall: | 20260506 14:36:33.012 | INFO | BytesWritten : 1096839176350 | 20260506 14:36:33.012 | INFO | NumEvents : 530775986388 | 20260506 14:36:34.657 | INFO | Analyzing chunk frames = [2121728,2122240) | 20260506 14:36:34.657 | INFO | MemInfo: { | 20260506 14:36:34.657 | INFO | "gpu" : " 17 GB", | 20260506 14:36:34.657 | INFO | "huge" : "", | 20260506 14:36:34.657 | INFO | "huge_shared" : " 27 GB", | 20260506 14:36:34.657 | INFO | "malloc" : " 14 GB", | 20260506 14:36:34.657 | INFO | "pinned" : " 29 GB" | 20260506 14:36:34.657 | INFO | } | 20260506 14:36:34.657 | INFO | Perf report for the last 5.13961 seconds: | 20260506 14:36:34.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:36:34.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:34.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:34.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:34.657 | INFO | ANALYSIS : 62.9%, 2.9, 95.1ms, 0 | 20260506 14:36:34.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:34.657 | INFO | RT_METRICS : 46.2%, 1.0, 24.3ms, 0 | 20260506 14:36:34.657 | INFO | PRE_HQ : 61.3%, 1.9, 60.3ms, 0 | 20260506 14:36:34.657 | INFO | BAZWRITER : 65.2%, 4.7, 158.8ms, 0 | 20260506 14:36:34.657 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.5ms, 0 | 20260506 14:36:34.658 | INFO | Uploaded 36GiB to GPU in 2.07587s (17.3421GiB/s) | 20260506 14:36:34.658 | INFO | Downloaded 38GiB to GPU in 2.21631s (17.1456GiB/s) | 20260506 14:36:39.797 | INFO | Analyzing chunk frames = [2122240,2122752) | 20260506 14:36:39.797 | INFO | MemInfo: { | 20260506 14:36:39.797 | INFO | "gpu" : " 17 GB", | 20260506 14:36:39.797 | INFO | "huge" : "", | 20260506 14:36:39.797 | INFO | "huge_shared" : " 27 GB", | 20260506 14:36:39.797 | INFO | "malloc" : " 14 GB", | 20260506 14:36:39.797 | INFO | "pinned" : " 29 GB" | 20260506 14:36:39.797 | INFO | } | 20260506 14:36:39.797 | INFO | Perf report for the last 5.1399 seconds: | 20260506 14:36:39.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:36:39.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:39.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:39.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:39.797 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 14:36:39.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:39.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:39.797 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:36:39.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:39.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:39.797 | INFO | Uploaded 36GiB to GPU in 2.02421s (17.7847GiB/s) | 20260506 14:36:39.797 | INFO | Downloaded 34GiB to GPU in 1.9424s (17.5042GiB/s) | 20260506 14:36:44.935 | INFO | Analyzing chunk frames = [2122752,2123264) | 20260506 14:36:44.935 | INFO | MemInfo: { | 20260506 14:36:44.935 | INFO | "gpu" : " 17 GB", | 20260506 14:36:44.935 | INFO | "huge" : "", | 20260506 14:36:44.935 | INFO | "huge_shared" : " 27 GB", | 20260506 14:36:44.935 | INFO | "malloc" : " 14 GB", | 20260506 14:36:44.935 | INFO | "pinned" : " 29 GB" | 20260506 14:36:44.935 | INFO | } | 20260506 14:36:44.935 | INFO | Perf report for the last 5.1383 seconds: | 20260506 14:36:44.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:36:44.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:44.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:44.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:44.935 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 14:36:44.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:44.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:44.935 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:36:44.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:44.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:44.936 | INFO | Uploaded 36GiB to GPU in 2.00322s (17.9711GiB/s) | 20260506 14:36:44.936 | INFO | Downloaded 34GiB to GPU in 1.92955s (17.6207GiB/s) | 20260506 14:36:50.074 | INFO | Analyzing chunk frames = [2123264,2123776) | 20260506 14:36:50.074 | INFO | MemInfo: { | 20260506 14:36:50.074 | INFO | "gpu" : " 17 GB", | 20260506 14:36:50.074 | INFO | "huge" : "", | 20260506 14:36:50.074 | INFO | "huge_shared" : " 27 GB", | 20260506 14:36:50.075 | INFO | "malloc" : " 14 GB", | 20260506 14:36:50.075 | INFO | "pinned" : " 29 GB" | 20260506 14:36:50.075 | INFO | } | 20260506 14:36:50.075 | INFO | Perf report for the last 5.13926 seconds: | 20260506 14:36:50.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:36:50.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:50.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:50.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:50.075 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 14:36:50.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:50.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:50.075 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:36:50.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:50.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:50.075 | INFO | Uploaded 36GiB to GPU in 1.99549s (18.0407GiB/s) | 20260506 14:36:50.075 | INFO | Downloaded 34GiB to GPU in 1.90374s (17.8596GiB/s) | 20260506 14:36:55.214 | INFO | Analyzing chunk frames = [2123776,2124288) | 20260506 14:36:55.214 | INFO | MemInfo: { | 20260506 14:36:55.214 | INFO | "gpu" : " 17 GB", | 20260506 14:36:55.214 | INFO | "huge" : "", | 20260506 14:36:55.214 | INFO | "huge_shared" : " 27 GB", | 20260506 14:36:55.214 | INFO | "malloc" : " 14 GB", | 20260506 14:36:55.214 | INFO | "pinned" : " 29 GB" | 20260506 14:36:55.214 | INFO | } | 20260506 14:36:55.214 | INFO | Perf report for the last 5.13926 seconds: | 20260506 14:36:55.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:36:55.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:55.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:55.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:55.214 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 14:36:55.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:55.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:55.214 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:36:55.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:55.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:36:55.214 | INFO | Uploaded 36GiB to GPU in 2.00527s (17.9527GiB/s) | 20260506 14:36:55.214 | INFO | Downloaded 34GiB to GPU in 1.93303s (17.589GiB/s) | 20260506 14:37:00.353 | INFO | Analyzing chunk frames = [2124288,2124800) | 20260506 14:37:00.353 | INFO | MemInfo: { | 20260506 14:37:00.353 | INFO | "gpu" : " 17 GB", | 20260506 14:37:00.353 | INFO | "huge" : "", | 20260506 14:37:00.354 | INFO | "huge_shared" : " 27 GB", | 20260506 14:37:00.354 | INFO | "malloc" : " 14 GB", | 20260506 14:37:00.354 | INFO | "pinned" : " 29 GB" | 20260506 14:37:00.354 | INFO | } | 20260506 14:37:00.354 | INFO | Perf report for the last 5.13977 seconds: | 20260506 14:37:00.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:37:00.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:00.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:00.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:00.354 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:37:00.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:00.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:00.354 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 14:37:00.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:00.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:00.354 | INFO | Uploaded 36GiB to GPU in 2.03796s (17.6647GiB/s) | 20260506 14:37:00.354 | INFO | Downloaded 34GiB to GPU in 1.92834s (17.6318GiB/s) | 20260506 14:37:05.494 | INFO | Analyzing chunk frames = [2124800,2125312) | 20260506 14:37:05.494 | INFO | MemInfo: { | 20260506 14:37:05.494 | INFO | "gpu" : " 17 GB", | 20260506 14:37:05.494 | INFO | "huge" : "", | 20260506 14:37:05.494 | INFO | "huge_shared" : " 27 GB", | 20260506 14:37:05.494 | INFO | "malloc" : " 14 GB", | 20260506 14:37:05.494 | INFO | "pinned" : " 29 GB" | 20260506 14:37:05.494 | INFO | } | 20260506 14:37:05.494 | INFO | Perf report for the last 5.14021 seconds: | 20260506 14:37:05.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:37:05.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:05.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:05.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:05.494 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:37:05.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:05.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:05.494 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 14:37:05.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:05.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:05.494 | INFO | Uploaded 36GiB to GPU in 1.97649s (18.2141GiB/s) | 20260506 14:37:05.494 | INFO | Downloaded 34GiB to GPU in 1.99159s (17.0718GiB/s) | 20260506 14:37:10.632 | INFO | Analyzing chunk frames = [2125312,2125824) | 20260506 14:37:10.632 | INFO | MemInfo: { | 20260506 14:37:10.632 | INFO | "gpu" : " 17 GB", | 20260506 14:37:10.632 | INFO | "huge" : "", | 20260506 14:37:10.632 | INFO | "huge_shared" : " 27 GB", | 20260506 14:37:10.632 | INFO | "malloc" : " 14 GB", | 20260506 14:37:10.632 | INFO | "pinned" : " 29 GB" | 20260506 14:37:10.632 | INFO | } | 20260506 14:37:10.632 | INFO | Perf report for the last 5.13813 seconds: | 20260506 14:37:10.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:37:10.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:10.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:10.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:10.632 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 14:37:10.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:10.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:10.632 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:37:10.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:10.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:10.632 | INFO | Uploaded 36GiB to GPU in 2.00454s (17.9592GiB/s) | 20260506 14:37:10.632 | INFO | Downloaded 34GiB to GPU in 1.98244s (17.1506GiB/s) | 20260506 14:37:10.742 | INFO | Starting a round of Baz IO | 20260506 14:37:13.897 | INFO | Populating RTMetrics report | 20260506 14:37:13.898 | INFO | RTMetrics report populated | 20260506 14:37:13.898 | INFO | Opening tmp RTMetrics json file | 20260506 14:37:13.899 | INFO | Serializing RTMetrics to json | 20260506 14:37:13.923 | INFO | Serialization done, swapping tmp file now... | 20260506 14:37:13.941 | INFO | RTMetrics done | 20260506 14:37:14.267 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.524 seconds | 20260506 14:37:14.267 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.09% / 0.01% / 61.89% | 20260506 14:37:14.267 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.46464% / 1.98682e-05% / 6.35783e-05% | 20260506 14:37:14.267 | INFO | Baz IO overall: | 20260506 14:37:14.267 | INFO | BytesWritten : 1099106504377 | 20260506 14:37:14.267 | INFO | NumEvents : 531875044239 | 20260506 14:37:15.771 | INFO | Analyzing chunk frames = [2125824,2126336) | 20260506 14:37:15.771 | INFO | MemInfo: { | 20260506 14:37:15.771 | INFO | "gpu" : " 17 GB", | 20260506 14:37:15.771 | INFO | "huge" : "", | 20260506 14:37:15.771 | INFO | "huge_shared" : " 27 GB", | 20260506 14:37:15.771 | INFO | "malloc" : " 14 GB", | 20260506 14:37:15.771 | INFO | "pinned" : " 29 GB" | 20260506 14:37:15.771 | INFO | } | 20260506 14:37:15.771 | INFO | Perf report for the last 5.1392 seconds: | 20260506 14:37:15.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:37:15.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:15.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:15.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:15.771 | INFO | ANALYSIS : 62.9%, 2.9, 95.4ms, 0 | 20260506 14:37:15.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:15.771 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 14:37:15.771 | INFO | PRE_HQ : 60.9%, 1.9, 59.4ms, 0 | 20260506 14:37:15.771 | INFO | BAZWRITER : 68.0%, 4.8, 167.5ms, 0 | 20260506 14:37:15.771 | INFO | BAZ_SAMPLING : 3.0%, 1.0, 1.6ms, 0 | 20260506 14:37:15.772 | INFO | Uploaded 36GiB to GPU in 2.07418s (17.3563GiB/s) | 20260506 14:37:15.772 | INFO | Downloaded 38GiB to GPU in 2.22782s (17.057GiB/s) | 20260506 14:37:20.911 | INFO | Analyzing chunk frames = [2126336,2126848) | 20260506 14:37:20.911 | INFO | MemInfo: { | 20260506 14:37:20.911 | INFO | "gpu" : " 17 GB", | 20260506 14:37:20.911 | INFO | "huge" : "", | 20260506 14:37:20.911 | INFO | "huge_shared" : " 27 GB", | 20260506 14:37:20.911 | INFO | "malloc" : " 14 GB", | 20260506 14:37:20.911 | INFO | "pinned" : " 29 GB" | 20260506 14:37:20.911 | INFO | } | 20260506 14:37:20.911 | INFO | Perf report for the last 5.13996 seconds: | 20260506 14:37:20.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:37:20.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:20.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:20.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:20.911 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:37:20.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:20.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:20.911 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:37:20.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:20.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:20.912 | INFO | Uploaded 36GiB to GPU in 2.01254s (17.8878GiB/s) | 20260506 14:37:20.912 | INFO | Downloaded 34GiB to GPU in 1.94508s (17.48GiB/s) | 20260506 14:37:26.050 | INFO | Analyzing chunk frames = [2126848,2127360) | 20260506 14:37:26.050 | INFO | MemInfo: { | 20260506 14:37:26.050 | INFO | "gpu" : " 17 GB", | 20260506 14:37:26.050 | INFO | "huge" : "", | 20260506 14:37:26.050 | INFO | "huge_shared" : " 27 GB", | 20260506 14:37:26.050 | INFO | "malloc" : " 14 GB", | 20260506 14:37:26.050 | INFO | "pinned" : " 29 GB" | 20260506 14:37:26.050 | INFO | } | 20260506 14:37:26.050 | INFO | Perf report for the last 5.13901 seconds: | 20260506 14:37:26.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:37:26.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:26.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:26.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:26.050 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:37:26.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:26.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:26.050 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:37:26.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:26.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:26.051 | INFO | Uploaded 36GiB to GPU in 2.01021s (17.9085GiB/s) | 20260506 14:37:26.051 | INFO | Downloaded 34GiB to GPU in 1.94218s (17.5061GiB/s) | 20260506 14:37:31.189 | INFO | Analyzing chunk frames = [2127360,2127872) | 20260506 14:37:31.189 | INFO | MemInfo: { | 20260506 14:37:31.189 | INFO | "gpu" : " 17 GB", | 20260506 14:37:31.189 | INFO | "huge" : "", | 20260506 14:37:31.189 | INFO | "huge_shared" : " 27 GB", | 20260506 14:37:31.189 | INFO | "malloc" : " 14 GB", | 20260506 14:37:31.189 | INFO | "pinned" : " 29 GB" | 20260506 14:37:31.189 | INFO | } | 20260506 14:37:31.189 | INFO | Perf report for the last 5.13917 seconds: | 20260506 14:37:31.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:37:31.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:31.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:31.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:31.190 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 14:37:31.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:31.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:31.190 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:37:31.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:31.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:31.190 | INFO | Uploaded 36GiB to GPU in 1.97578s (18.2207GiB/s) | 20260506 14:37:31.190 | INFO | Downloaded 34GiB to GPU in 1.91422s (17.7618GiB/s) | 20260506 14:37:36.329 | INFO | Analyzing chunk frames = [2127872,2128384) | 20260506 14:37:36.329 | INFO | MemInfo: { | 20260506 14:37:36.329 | INFO | "gpu" : " 17 GB", | 20260506 14:37:36.329 | INFO | "huge" : "", | 20260506 14:37:36.329 | INFO | "huge_shared" : " 27 GB", | 20260506 14:37:36.329 | INFO | "malloc" : " 14 GB", | 20260506 14:37:36.329 | INFO | "pinned" : " 29 GB" | 20260506 14:37:36.329 | INFO | } | 20260506 14:37:36.329 | INFO | Perf report for the last 5.13941 seconds: | 20260506 14:37:36.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:37:36.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:36.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:36.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:36.329 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 14:37:36.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:36.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:36.329 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:37:36.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:36.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:36.329 | INFO | Uploaded 36GiB to GPU in 2s (18GiB/s) | 20260506 14:37:36.329 | INFO | Downloaded 34GiB to GPU in 1.93024s (17.6144GiB/s) | 20260506 14:37:41.468 | INFO | Analyzing chunk frames = [2128384,2128896) | 20260506 14:37:41.468 | INFO | MemInfo: { | 20260506 14:37:41.468 | INFO | "gpu" : " 17 GB", | 20260506 14:37:41.468 | INFO | "huge" : "", | 20260506 14:37:41.468 | INFO | "huge_shared" : " 27 GB", | 20260506 14:37:41.468 | INFO | "malloc" : " 14 GB", | 20260506 14:37:41.468 | INFO | "pinned" : " 29 GB" | 20260506 14:37:41.468 | INFO | } | 20260506 14:37:41.468 | INFO | Perf report for the last 5.13947 seconds: | 20260506 14:37:41.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:37:41.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:41.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:41.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:41.468 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:37:41.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:41.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:41.468 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260506 14:37:41.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:41.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:41.469 | INFO | Uploaded 36GiB to GPU in 2.03817s (17.6629GiB/s) | 20260506 14:37:41.469 | INFO | Downloaded 34GiB to GPU in 1.91701s (17.7359GiB/s) | 20260506 14:37:46.610 | INFO | Analyzing chunk frames = [2128896,2129408) | 20260506 14:37:46.610 | INFO | MemInfo: { | 20260506 14:37:46.610 | INFO | "gpu" : " 17 GB", | 20260506 14:37:46.610 | INFO | "huge" : "", | 20260506 14:37:46.610 | INFO | "huge_shared" : " 27 GB", | 20260506 14:37:46.610 | INFO | "malloc" : " 14 GB", | 20260506 14:37:46.610 | INFO | "pinned" : " 29 GB" | 20260506 14:37:46.610 | INFO | } | 20260506 14:37:46.610 | INFO | Perf report for the last 5.14136 seconds: | 20260506 14:37:46.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:37:46.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:46.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:46.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:46.610 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 14:37:46.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:46.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:46.610 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 14:37:46.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:46.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:46.610 | INFO | Uploaded 36GiB to GPU in 2.01349s (17.8794GiB/s) | 20260506 14:37:46.610 | INFO | Downloaded 34GiB to GPU in 2.00445s (16.9623GiB/s) | 20260506 14:37:51.747 | INFO | Analyzing chunk frames = [2129408,2129920) | 20260506 14:37:51.747 | INFO | MemInfo: { | 20260506 14:37:51.747 | INFO | "gpu" : " 17 GB", | 20260506 14:37:51.747 | INFO | "huge" : "", | 20260506 14:37:51.747 | INFO | "huge_shared" : " 27 GB", | 20260506 14:37:51.747 | INFO | "malloc" : " 14 GB", | 20260506 14:37:51.747 | INFO | "pinned" : " 29 GB" | 20260506 14:37:51.747 | INFO | } | 20260506 14:37:51.747 | INFO | Perf report for the last 5.13719 seconds: | 20260506 14:37:51.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:37:51.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:51.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:51.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:51.747 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:37:51.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:51.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:51.747 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:37:51.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:51.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:51.747 | INFO | Uploaded 36GiB to GPU in 2.02696s (17.7606GiB/s) | 20260506 14:37:51.747 | INFO | Downloaded 34GiB to GPU in 1.97878s (17.1823GiB/s) | 20260506 14:37:51.747 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:37:51.747 | INFO | Total mem usage: 17.4 GiB | 20260506 14:37:51.747 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:37:51.747 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -15.4%) | 20260506 14:37:51.747 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:37:51.747 | INFO | High Water Breakdown: | 20260506 14:37:51.747 | INFO | 60.6% : 9.15 GiB ( 83.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:37:51.747 | INFO | 39.2% : 5.92 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:37:51.747 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:37:51.747 | INFO | ------------------------------------------ | 20260506 14:37:51.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:37:51.748 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:37:51.748 | INFO | Total mem usage: 26.5 GiB | 20260506 14:37:51.748 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:37:51.748 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:37:51.748 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:37:51.748 | INFO | High Water Breakdown: | 20260506 14:37:51.748 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:37:51.748 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:37:51.748 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:37:51.748 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:37:51.748 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:37:51.748 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:37:51.748 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:37:51.748 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:37:51.748 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:37:51.748 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:37:51.748 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:37:51.748 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:37:51.748 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:37:51.748 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:37:51.748 | INFO | ------------------------------------------ | 20260506 14:37:51.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:37:51.748 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:37:51.748 | INFO | Total mem usage: 25.6 GiB | 20260506 14:37:51.748 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:37:51.748 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:37:51.748 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:37:51.748 | INFO | High Water Breakdown: | 20260506 14:37:51.748 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:37:51.748 | INFO | ------------------------------------------ | 20260506 14:37:51.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:37:51.748 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:37:51.748 | INFO | Total mem usage: 15.7 GiB | 20260506 14:37:51.748 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:37:51.748 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:37:51.748 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 14:37:51.748 | INFO | High Water Breakdown: | 20260506 14:37:51.748 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:37:51.748 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:37:51.748 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:37:51.748 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:37:51.748 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:37:51.748 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:37:51.748 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:37:51.748 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:37:51.748 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:37:51.748 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:37:51.748 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:37:51.748 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:37:51.748 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:37:51.748 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:37:51.748 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:37:51.749 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:37:51.749 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:37:51.749 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:37:51.749 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:37:51.749 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:37:51.749 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:37:51.749 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:37:51.749 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:37:51.749 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:37:51.749 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:37:51.749 | INFO | ------------------------------------------ | 20260506 14:37:51.749 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:37:51.749 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:37:51.749 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:37:51.749 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:37:51.749 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.60e+01, 1.35e+01 / 2.00e+00 | 20260506 14:37:51.749 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.02e-01, 1.02e-02 / 4.89e-03 | 20260506 14:37:51.749 | INFO | PerfReport: 11.56%, Baseline -- 3.65e+00 / 4.47e+00, 5.67e-01 / 2.57e-01 | 20260506 14:37:51.749 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.49e+00, 1.90e-01 / 1.18e-01 | 20260506 14:37:51.749 | INFO | PerfReport: 15.85%, DME -- 4.00e+01 / 4.46e+01, 3.86e+00 / 2.23e+00 | 20260506 14:37:51.749 | INFO | PerfReport: 56.74%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.49e+00 / 1.55e+00 | 20260506 14:37:51.749 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.35e+00 / 2.98e+00, 4.13e-01 / 2.33e-01 | 20260506 14:37:51.749 | INFO | PerfReport: 3.60%, Metrics -- 1.13e+00 / 1.52e+00, 2.92e-01 / 1.88e-01 | 20260506 14:37:51.749 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.96e-01 / 4.51e-01, 6.15e-02 / 2.73e-02 | 20260506 14:37:51.749 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:37:51.749 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:37:51.749 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:37:51.749 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 14:37:51.749 | INFO | PerfReport: 21.49%, Upload -- 2.10e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260506 14:37:51.749 | INFO | PerfReport: 21.10%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 14:37:51.749 | INFO | PerfReport: 32.26%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 14:37:51.749 | INFO | PerfReport: 1.57%, LockUpload -- 1.54e+00 / 0.00e+00, 6.06e+00 / 0.00e+00 | 20260506 14:37:51.749 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 14:37:51.749 | INFO | PerfReport: 22.47%, LockCompute -- 2.20e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:37:51.749 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260506 14:37:51.863 | INFO | Starting a round of Baz IO | 20260506 14:37:55.021 | INFO | Populating RTMetrics report | 20260506 14:37:55.022 | INFO | RTMetrics report populated | 20260506 14:37:55.022 | INFO | Opening tmp RTMetrics json file | 20260506 14:37:55.140 | INFO | Serializing RTMetrics to json | 20260506 14:37:55.144 | INFO | Serialization done, swapping tmp file now... | 20260506 14:37:55.145 | INFO | RTMetrics done | 20260506 14:37:55.458 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.595 seconds | 20260506 14:37:55.458 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.08% / 0.01% / 61.91% | 20260506 14:37:55.458 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.4564% / 7.94729e-06% / 3.97364e-05% | 20260506 14:37:55.458 | INFO | Baz IO overall: | 20260506 14:37:55.458 | INFO | BytesWritten : 1101372561710 | 20260506 14:37:55.458 | INFO | NumEvents : 532973007206 | 20260506 14:37:56.886 | INFO | Analyzing chunk frames = [2129920,2130432) | 20260506 14:37:56.886 | INFO | MemInfo: { | 20260506 14:37:56.886 | INFO | "gpu" : " 17 GB", | 20260506 14:37:56.886 | INFO | "huge" : "", | 20260506 14:37:56.886 | INFO | "huge_shared" : " 27 GB", | 20260506 14:37:56.886 | INFO | "malloc" : " 14 GB", | 20260506 14:37:56.886 | INFO | "pinned" : " 29 GB" | 20260506 14:37:56.886 | INFO | } | 20260506 14:37:56.886 | INFO | Perf report for the last 5.13875 seconds: | 20260506 14:37:56.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:37:56.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:56.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:56.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:56.886 | INFO | ANALYSIS : 63.1%, 2.9, 95.7ms, 0 | 20260506 14:37:56.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:37:56.886 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260506 14:37:56.886 | INFO | PRE_HQ : 61.2%, 1.9, 59.7ms, 0 | 20260506 14:37:56.886 | INFO | BAZWRITER : 69.3%, 4.8, 170.3ms, 0 | 20260506 14:37:56.886 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 14:37:56.886 | INFO | Uploaded 36GiB to GPU in 2.07184s (17.3758GiB/s) | 20260506 14:37:56.886 | INFO | Downloaded 38GiB to GPU in 2.23929s (16.9697GiB/s) | 20260506 14:38:02.026 | INFO | Analyzing chunk frames = [2130432,2130944) | 20260506 14:38:02.026 | INFO | MemInfo: { | 20260506 14:38:02.026 | INFO | "gpu" : " 17 GB", | 20260506 14:38:02.026 | INFO | "huge" : "", | 20260506 14:38:02.026 | INFO | "huge_shared" : " 27 GB", | 20260506 14:38:02.026 | INFO | "malloc" : " 14 GB", | 20260506 14:38:02.026 | INFO | "pinned" : " 29 GB" | 20260506 14:38:02.026 | INFO | } | 20260506 14:38:02.026 | INFO | Perf report for the last 5.14036 seconds: | 20260506 14:38:02.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:38:02.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:02.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:02.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:02.026 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 14:38:02.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:02.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:02.026 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:38:02.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:02.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:02.026 | INFO | Uploaded 36GiB to GPU in 1.9998s (18.0018GiB/s) | 20260506 14:38:02.026 | INFO | Downloaded 34GiB to GPU in 1.93724s (17.5507GiB/s) | 20260506 14:38:07.165 | INFO | Analyzing chunk frames = [2130944,2131456) | 20260506 14:38:07.165 | INFO | MemInfo: { | 20260506 14:38:07.165 | INFO | "gpu" : " 17 GB", | 20260506 14:38:07.165 | INFO | "huge" : "", | 20260506 14:38:07.165 | INFO | "huge_shared" : " 27 GB", | 20260506 14:38:07.165 | INFO | "malloc" : " 14 GB", | 20260506 14:38:07.165 | INFO | "pinned" : " 29 GB" | 20260506 14:38:07.165 | INFO | } | 20260506 14:38:07.165 | INFO | Perf report for the last 5.13917 seconds: | 20260506 14:38:07.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:38:07.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:07.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:07.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:07.165 | INFO | ANALYSIS : 61.3%, 3.0, 94.8ms, 0 | 20260506 14:38:07.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:07.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:07.165 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:38:07.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:07.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:07.166 | INFO | Uploaded 36GiB to GPU in 2.00226s (17.9796GiB/s) | 20260506 14:38:07.166 | INFO | Downloaded 34GiB to GPU in 1.94171s (17.5103GiB/s) | 20260506 14:38:12.305 | INFO | Analyzing chunk frames = [2131456,2131968) | 20260506 14:38:12.305 | INFO | MemInfo: { | 20260506 14:38:12.305 | INFO | "gpu" : " 17 GB", | 20260506 14:38:12.305 | INFO | "huge" : "", | 20260506 14:38:12.305 | INFO | "huge_shared" : " 27 GB", | 20260506 14:38:12.305 | INFO | "malloc" : " 14 GB", | 20260506 14:38:12.305 | INFO | "pinned" : " 29 GB" | 20260506 14:38:12.305 | INFO | } | 20260506 14:38:12.305 | INFO | Perf report for the last 5.13998 seconds: | 20260506 14:38:12.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:38:12.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:12.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:12.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:12.305 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260506 14:38:12.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:12.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:12.305 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 14:38:12.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:12.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:12.306 | INFO | Uploaded 36GiB to GPU in 1.99599s (18.0362GiB/s) | 20260506 14:38:12.306 | INFO | Downloaded 34GiB to GPU in 1.91213s (17.7812GiB/s) | 20260506 14:38:17.444 | INFO | Analyzing chunk frames = [2131968,2132480) | 20260506 14:38:17.444 | INFO | MemInfo: { | 20260506 14:38:17.444 | INFO | "gpu" : " 17 GB", | 20260506 14:38:17.444 | INFO | "huge" : "", | 20260506 14:38:17.444 | INFO | "huge_shared" : " 27 GB", | 20260506 14:38:17.444 | INFO | "malloc" : " 14 GB", | 20260506 14:38:17.444 | INFO | "pinned" : " 29 GB" | 20260506 14:38:17.444 | INFO | } | 20260506 14:38:17.444 | INFO | Perf report for the last 5.1392 seconds: | 20260506 14:38:17.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:38:17.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:17.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:17.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:17.445 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 14:38:17.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:17.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:17.445 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:38:17.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:17.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:17.445 | INFO | Uploaded 36GiB to GPU in 2.02881s (17.7444GiB/s) | 20260506 14:38:17.445 | INFO | Downloaded 34GiB to GPU in 1.93084s (17.6089GiB/s) | 20260506 14:38:22.582 | INFO | Analyzing chunk frames = [2132480,2132992) | 20260506 14:38:22.582 | INFO | MemInfo: { | 20260506 14:38:22.582 | INFO | "gpu" : " 17 GB", | 20260506 14:38:22.582 | INFO | "huge" : "", | 20260506 14:38:22.582 | INFO | "huge_shared" : " 27 GB", | 20260506 14:38:22.582 | INFO | "malloc" : " 14 GB", | 20260506 14:38:22.582 | INFO | "pinned" : " 29 GB" | 20260506 14:38:22.582 | INFO | } | 20260506 14:38:22.582 | INFO | Perf report for the last 5.13798 seconds: | 20260506 14:38:22.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:38:22.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:22.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:22.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:22.583 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:38:22.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:22.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:22.583 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 14:38:22.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:22.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:22.583 | INFO | Uploaded 36GiB to GPU in 2.08271s (17.2851GiB/s) | 20260506 14:38:22.583 | INFO | Downloaded 34GiB to GPU in 1.97522s (17.2132GiB/s) | 20260506 14:38:27.723 | INFO | Analyzing chunk frames = [2132992,2133504) | 20260506 14:38:27.723 | INFO | MemInfo: { | 20260506 14:38:27.723 | INFO | "gpu" : " 17 GB", | 20260506 14:38:27.723 | INFO | "huge" : "", | 20260506 14:38:27.723 | INFO | "huge_shared" : " 27 GB", | 20260506 14:38:27.723 | INFO | "malloc" : " 14 GB", | 20260506 14:38:27.723 | INFO | "pinned" : " 29 GB" | 20260506 14:38:27.723 | INFO | } | 20260506 14:38:27.723 | INFO | Perf report for the last 5.14029 seconds: | 20260506 14:38:27.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:38:27.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:27.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:27.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:27.723 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:38:27.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:27.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:27.723 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 14:38:27.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:27.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:27.723 | INFO | Uploaded 36GiB to GPU in 2.04231s (17.6271GiB/s) | 20260506 14:38:27.723 | INFO | Downloaded 34GiB to GPU in 2.02029s (16.8293GiB/s) | 20260506 14:38:32.863 | INFO | Analyzing chunk frames = [2133504,2134016) | 20260506 14:38:32.863 | INFO | MemInfo: { | 20260506 14:38:32.863 | INFO | "gpu" : " 17 GB", | 20260506 14:38:32.863 | INFO | "huge" : "", | 20260506 14:38:32.863 | INFO | "huge_shared" : " 27 GB", | 20260506 14:38:32.863 | INFO | "malloc" : " 14 GB", | 20260506 14:38:32.863 | INFO | "pinned" : " 29 GB" | 20260506 14:38:32.863 | INFO | } | 20260506 14:38:32.863 | INFO | Perf report for the last 5.14003 seconds: | 20260506 14:38:32.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:38:32.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:32.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:32.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:32.863 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 14:38:32.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:32.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:32.863 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:38:32.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:32.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:32.863 | INFO | Uploaded 36GiB to GPU in 2.02635s (17.7659GiB/s) | 20260506 14:38:32.863 | INFO | Downloaded 34GiB to GPU in 1.9705s (17.2545GiB/s) | 20260506 14:38:32.975 | INFO | Starting a round of Baz IO | 20260506 14:38:36.136 | INFO | Populating RTMetrics report | 20260506 14:38:36.136 | INFO | RTMetrics report populated | 20260506 14:38:36.136 | INFO | Opening tmp RTMetrics json file | 20260506 14:38:36.139 | INFO | Serializing RTMetrics to json | 20260506 14:38:36.142 | INFO | Serialization done, swapping tmp file now... | 20260506 14:38:36.144 | INFO | RTMetrics done | 20260506 14:38:36.302 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.327 seconds | 20260506 14:38:36.302 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.07% / 0.01% / 61.92% | 20260506 14:38:36.302 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.44509% / 1.58946e-05% / 4.37101e-05% | 20260506 14:38:36.302 | INFO | Baz IO overall: | 20260506 14:38:36.303 | INFO | BytesWritten : 1103637234124 | 20260506 14:38:36.303 | INFO | NumEvents : 534069607821 | 20260506 14:38:38.002 | INFO | Analyzing chunk frames = [2134016,2134528) | 20260506 14:38:38.002 | INFO | MemInfo: { | 20260506 14:38:38.002 | INFO | "gpu" : " 17 GB", | 20260506 14:38:38.002 | INFO | "huge" : "", | 20260506 14:38:38.002 | INFO | "huge_shared" : " 27 GB", | 20260506 14:38:38.002 | INFO | "malloc" : " 14 GB", | 20260506 14:38:38.002 | INFO | "pinned" : " 29 GB" | 20260506 14:38:38.002 | INFO | } | 20260506 14:38:38.002 | INFO | Perf report for the last 5.13898 seconds: | 20260506 14:38:38.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:38:38.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:38.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:38.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:38.002 | INFO | ANALYSIS : 62.9%, 2.9, 95.4ms, 0 | 20260506 14:38:38.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:38.002 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 14:38:38.002 | INFO | PRE_HQ : 60.9%, 1.9, 59.5ms, 0 | 20260506 14:38:38.002 | INFO | BAZWRITER : 64.1%, 4.8, 157.0ms, 0 | 20260506 14:38:38.002 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 14:38:38.002 | INFO | Uploaded 36GiB to GPU in 2.05653s (17.5052GiB/s) | 20260506 14:38:38.002 | INFO | Downloaded 38GiB to GPU in 2.22525s (17.0767GiB/s) | 20260506 14:38:43.140 | INFO | Analyzing chunk frames = [2134528,2135040) | 20260506 14:38:43.140 | INFO | MemInfo: { | 20260506 14:38:43.140 | INFO | "gpu" : " 17 GB", | 20260506 14:38:43.140 | INFO | "huge" : "", | 20260506 14:38:43.140 | INFO | "huge_shared" : " 27 GB", | 20260506 14:38:43.140 | INFO | "malloc" : " 14 GB", | 20260506 14:38:43.140 | INFO | "pinned" : " 29 GB" | 20260506 14:38:43.140 | INFO | } | 20260506 14:38:43.141 | INFO | Perf report for the last 5.1387 seconds: | 20260506 14:38:43.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:38:43.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:43.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:43.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:43.141 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 14:38:43.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:43.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:43.141 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:38:43.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:43.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:43.141 | INFO | Uploaded 36GiB to GPU in 1.98779s (18.1106GiB/s) | 20260506 14:38:43.141 | INFO | Downloaded 34GiB to GPU in 1.92383s (17.6731GiB/s) | 20260506 14:38:48.279 | INFO | Analyzing chunk frames = [2135040,2135552) | 20260506 14:38:48.279 | INFO | MemInfo: { | 20260506 14:38:48.279 | INFO | "gpu" : " 17 GB", | 20260506 14:38:48.279 | INFO | "huge" : "", | 20260506 14:38:48.279 | INFO | "huge_shared" : " 27 GB", | 20260506 14:38:48.279 | INFO | "malloc" : " 14 GB", | 20260506 14:38:48.279 | INFO | "pinned" : " 29 GB" | 20260506 14:38:48.279 | INFO | } | 20260506 14:38:48.279 | INFO | Perf report for the last 5.13884 seconds: | 20260506 14:38:48.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:38:48.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:48.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:48.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:48.279 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 14:38:48.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:48.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:48.279 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:38:48.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:48.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:48.280 | INFO | Uploaded 36GiB to GPU in 1.98739s (18.1142GiB/s) | 20260506 14:38:48.280 | INFO | Downloaded 34GiB to GPU in 1.93904s (17.5345GiB/s) | 20260506 14:38:53.419 | INFO | Analyzing chunk frames = [2135552,2136064) | 20260506 14:38:53.419 | INFO | MemInfo: { | 20260506 14:38:53.419 | INFO | "gpu" : " 17 GB", | 20260506 14:38:53.419 | INFO | "huge" : "", | 20260506 14:38:53.419 | INFO | "huge_shared" : " 27 GB", | 20260506 14:38:53.419 | INFO | "malloc" : " 14 GB", | 20260506 14:38:53.419 | INFO | "pinned" : " 29 GB" | 20260506 14:38:53.419 | INFO | } | 20260506 14:38:53.419 | INFO | Perf report for the last 5.13977 seconds: | 20260506 14:38:53.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:38:53.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:53.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:53.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:53.419 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260506 14:38:53.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:53.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:53.419 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:38:53.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:53.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:53.419 | INFO | Uploaded 36GiB to GPU in 2.02041s (17.8182GiB/s) | 20260506 14:38:53.420 | INFO | Downloaded 34GiB to GPU in 1.91346s (17.7689GiB/s) | 20260506 14:38:58.558 | INFO | Analyzing chunk frames = [2136064,2136576) | 20260506 14:38:58.558 | INFO | MemInfo: { | 20260506 14:38:58.558 | INFO | "gpu" : " 17 GB", | 20260506 14:38:58.558 | INFO | "huge" : "", | 20260506 14:38:58.559 | INFO | "huge_shared" : " 27 GB", | 20260506 14:38:58.559 | INFO | "malloc" : " 14 GB", | 20260506 14:38:58.559 | INFO | "pinned" : " 29 GB" | 20260506 14:38:58.559 | INFO | } | 20260506 14:38:58.559 | INFO | Perf report for the last 5.13951 seconds: | 20260506 14:38:58.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:38:58.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:58.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:58.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:58.559 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 14:38:58.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:58.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:58.559 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:38:58.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:58.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:38:58.559 | INFO | Uploaded 36GiB to GPU in 1.99596s (18.0364GiB/s) | 20260506 14:38:58.559 | INFO | Downloaded 34GiB to GPU in 1.93444s (17.5761GiB/s) | 20260506 14:39:03.698 | INFO | Analyzing chunk frames = [2136576,2137088) | 20260506 14:39:03.698 | INFO | MemInfo: { | 20260506 14:39:03.698 | INFO | "gpu" : " 17 GB", | 20260506 14:39:03.698 | INFO | "huge" : "", | 20260506 14:39:03.698 | INFO | "huge_shared" : " 27 GB", | 20260506 14:39:03.698 | INFO | "malloc" : " 14 GB", | 20260506 14:39:03.698 | INFO | "pinned" : " 29 GB" | 20260506 14:39:03.698 | INFO | } | 20260506 14:39:03.698 | INFO | Perf report for the last 5.13971 seconds: | 20260506 14:39:03.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:39:03.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:03.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:03.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:03.698 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 14:39:03.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:03.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:03.698 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 14:39:03.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:03.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:03.699 | INFO | Uploaded 36GiB to GPU in 2.07845s (17.3206GiB/s) | 20260506 14:39:03.699 | INFO | Downloaded 34GiB to GPU in 1.9712s (17.2484GiB/s) | 20260506 14:39:08.837 | INFO | Analyzing chunk frames = [2137088,2137600) | 20260506 14:39:08.837 | INFO | MemInfo: { | 20260506 14:39:08.837 | INFO | "gpu" : " 17 GB", | 20260506 14:39:08.837 | INFO | "huge" : "", | 20260506 14:39:08.837 | INFO | "huge_shared" : " 27 GB", | 20260506 14:39:08.837 | INFO | "malloc" : " 14 GB", | 20260506 14:39:08.837 | INFO | "pinned" : " 29 GB" | 20260506 14:39:08.837 | INFO | } | 20260506 14:39:08.837 | INFO | Perf report for the last 5.13907 seconds: | 20260506 14:39:08.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:39:08.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:08.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:08.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:08.838 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:39:08.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:08.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:08.838 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 14:39:08.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:08.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:08.838 | INFO | Uploaded 36GiB to GPU in 2.05111s (17.5515GiB/s) | 20260506 14:39:08.838 | INFO | Downloaded 34GiB to GPU in 2.0023s (16.9805GiB/s) | 20260506 14:39:13.977 | INFO | Analyzing chunk frames = [2137600,2138112) | 20260506 14:39:13.977 | INFO | MemInfo: { | 20260506 14:39:13.977 | INFO | "gpu" : " 17 GB", | 20260506 14:39:13.977 | INFO | "huge" : "", | 20260506 14:39:13.977 | INFO | "huge_shared" : " 27 GB", | 20260506 14:39:13.977 | INFO | "malloc" : " 14 GB", | 20260506 14:39:13.977 | INFO | "pinned" : " 29 GB" | 20260506 14:39:13.977 | INFO | } | 20260506 14:39:13.977 | INFO | Perf report for the last 5.13995 seconds: | 20260506 14:39:13.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:39:13.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:13.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:13.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:13.978 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 14:39:13.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:13.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:13.978 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260506 14:39:13.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:13.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:13.978 | INFO | Uploaded 36GiB to GPU in 2.04626s (17.5931GiB/s) | 20260506 14:39:13.978 | INFO | Downloaded 34GiB to GPU in 1.96439s (17.3082GiB/s) | 20260506 14:39:13.978 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:39:13.978 | INFO | Total mem usage: 17.4 GiB | 20260506 14:39:13.978 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:39:13.978 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -14.9%) | 20260506 14:39:13.978 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:39:13.978 | INFO | High Water Breakdown: | 20260506 14:39:13.978 | INFO | 60.7% : 9.2 GiB ( 84.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:39:13.978 | INFO | 39.2% : 5.94 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:39:13.978 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:39:13.978 | INFO | ------------------------------------------ | 20260506 14:39:13.978 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:39:13.978 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:39:13.978 | INFO | Total mem usage: 26.5 GiB | 20260506 14:39:13.978 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:39:13.978 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:39:13.978 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:39:13.978 | INFO | High Water Breakdown: | 20260506 14:39:13.978 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:39:13.978 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:39:13.978 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:39:13.978 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:39:13.978 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:39:13.978 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:39:13.978 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:39:13.978 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:39:13.978 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:39:13.978 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:39:13.978 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:39:13.978 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:39:13.978 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:39:13.978 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:39:13.978 | INFO | ------------------------------------------ | 20260506 14:39:13.978 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:39:13.978 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:39:13.978 | INFO | Total mem usage: 25.6 GiB | 20260506 14:39:13.978 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:39:13.978 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:39:13.978 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:39:13.978 | INFO | High Water Breakdown: | 20260506 14:39:13.978 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:39:13.978 | INFO | ------------------------------------------ | 20260506 14:39:13.978 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:39:13.978 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:39:13.978 | INFO | Total mem usage: 15.7 GiB | 20260506 14:39:13.978 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:39:13.978 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:39:13.978 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 14:39:13.978 | INFO | High Water Breakdown: | 20260506 14:39:13.978 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:39:13.978 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:39:13.978 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:39:13.978 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:39:13.978 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:39:13.978 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:39:13.978 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:39:13.978 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:39:13.978 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:39:13.978 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:39:13.978 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:39:13.978 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:39:13.978 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:39:13.978 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:39:13.979 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:39:13.979 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:39:13.979 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:39:13.979 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:39:13.979 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:39:13.979 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:39:13.979 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:39:13.979 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:39:13.979 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:39:13.979 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:39:13.979 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:39:13.979 | INFO | ------------------------------------------ | 20260506 14:39:13.979 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:39:13.979 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:39:13.979 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:39:13.979 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:39:13.979 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.60e+01, 1.34e+01 / 2.71e+00 | 20260506 14:39:13.979 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 9.98e-02, 1.01e-02 / 6.50e-03 | 20260506 14:39:13.979 | INFO | PerfReport: 11.57%, Baseline -- 3.65e+00 / 4.43e+00, 5.64e-01 / 3.71e-01 | 20260506 14:39:13.979 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.48e+00, 1.91e-01 / 1.28e-01 | 20260506 14:39:13.979 | INFO | PerfReport: 15.94%, DME -- 4.02e+01 / 4.47e+01, 4.13e+00 / 2.69e+00 | 20260506 14:39:13.979 | INFO | PerfReport: 56.64%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.41e+00 / 1.43e+00 | 20260506 14:39:13.979 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.35e+00 / 2.94e+00, 4.10e-01 / 2.24e-01 | 20260506 14:39:13.979 | INFO | PerfReport: 3.60%, Metrics -- 1.13e+00 / 1.50e+00, 2.97e-01 / 2.06e-01 | 20260506 14:39:13.979 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.96e-01 / 4.41e-01, 6.22e-02 / 2.80e-02 | 20260506 14:39:13.979 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:39:13.979 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:39:13.979 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:39:13.979 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 14:39:13.979 | INFO | PerfReport: 21.62%, Upload -- 2.11e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260506 14:39:13.979 | INFO | PerfReport: 21.17%, Download -- 2.07e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 14:39:13.979 | INFO | PerfReport: 32.23%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 14:39:13.979 | INFO | PerfReport: 1.60%, LockUpload -- 1.57e+00 / 0.00e+00, 5.94e+00 / 0.00e+00 | 20260506 14:39:13.979 | INFO | PerfReport: 1.02%, LockDownload -- 1.00e+00 / 0.00e+00, 2.55e+00 / 0.00e+00 | 20260506 14:39:13.979 | INFO | PerfReport: 22.34%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 14:39:13.979 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260506 14:39:14.100 | INFO | Starting a round of Baz IO | 20260506 14:39:17.300 | INFO | Populating RTMetrics report | 20260506 14:39:17.301 | INFO | RTMetrics report populated | 20260506 14:39:17.301 | INFO | Opening tmp RTMetrics json file | 20260506 14:39:17.388 | INFO | Serializing RTMetrics to json | 20260506 14:39:17.440 | INFO | Serialization done, swapping tmp file now... | 20260506 14:39:17.443 | INFO | RTMetrics done | 20260506 14:39:17.876 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.776 seconds | 20260506 14:39:17.876 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.06% / 0.01% / 61.93% | 20260506 14:39:17.876 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.43979% / 1.58946e-05% / 4.76837e-05% | 20260506 14:39:17.876 | INFO | Baz IO overall: | 20260506 14:39:17.876 | INFO | BytesWritten : 1105902509409 | 20260506 14:39:17.876 | INFO | NumEvents : 535166691739 | 20260506 14:39:19.116 | INFO | Analyzing chunk frames = [2138112,2138624) | 20260506 14:39:19.116 | INFO | MemInfo: { | 20260506 14:39:19.116 | INFO | "gpu" : " 17 GB", | 20260506 14:39:19.116 | INFO | "huge" : "", | 20260506 14:39:19.116 | INFO | "huge_shared" : " 27 GB", | 20260506 14:39:19.116 | INFO | "malloc" : " 14 GB", | 20260506 14:39:19.116 | INFO | "pinned" : " 29 GB" | 20260506 14:39:19.116 | INFO | } | 20260506 14:39:19.116 | INFO | Perf report for the last 5.13896 seconds: | 20260506 14:39:19.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:39:19.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:19.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:19.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:19.117 | INFO | ANALYSIS : 63.6%, 2.8, 94.1ms, 0 | 20260506 14:39:19.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:19.117 | INFO | RT_METRICS : 50.6%, 1.0, 26.5ms, 0 | 20260506 14:39:19.117 | INFO | PRE_HQ : 62.6%, 2.0, 65.0ms, 0 | 20260506 14:39:19.117 | INFO | BAZWRITER : 72.8%, 4.8, 180.8ms, 0 | 20260506 14:39:19.117 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 14:39:19.117 | INFO | Uploaded 36GiB to GPU in 2.0929s (17.201GiB/s) | 20260506 14:39:19.117 | INFO | Downloaded 38GiB to GPU in 2.20972s (17.1967GiB/s) | 20260506 14:39:24.255 | INFO | Analyzing chunk frames = [2138624,2139136) | 20260506 14:39:24.255 | INFO | MemInfo: { | 20260506 14:39:24.255 | INFO | "gpu" : " 17 GB", | 20260506 14:39:24.255 | INFO | "huge" : "", | 20260506 14:39:24.255 | INFO | "huge_shared" : " 27 GB", | 20260506 14:39:24.255 | INFO | "malloc" : " 14 GB", | 20260506 14:39:24.255 | INFO | "pinned" : " 29 GB" | 20260506 14:39:24.255 | INFO | } | 20260506 14:39:24.255 | INFO | Perf report for the last 5.13849 seconds: | 20260506 14:39:24.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:39:24.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:24.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:24.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:24.255 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:39:24.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:24.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:24.255 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 14:39:24.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:24.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:24.255 | INFO | Uploaded 36GiB to GPU in 2.04929s (17.5671GiB/s) | 20260506 14:39:24.255 | INFO | Downloaded 34GiB to GPU in 1.94378s (17.4917GiB/s) | 20260506 14:39:29.394 | INFO | Analyzing chunk frames = [2139136,2139648) | 20260506 14:39:29.394 | INFO | MemInfo: { | 20260506 14:39:29.394 | INFO | "gpu" : " 17 GB", | 20260506 14:39:29.394 | INFO | "huge" : "", | 20260506 14:39:29.394 | INFO | "huge_shared" : " 27 GB", | 20260506 14:39:29.394 | INFO | "malloc" : " 14 GB", | 20260506 14:39:29.394 | INFO | "pinned" : " 29 GB" | 20260506 14:39:29.394 | INFO | } | 20260506 14:39:29.394 | INFO | Perf report for the last 5.13913 seconds: | 20260506 14:39:29.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:39:29.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:29.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:29.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:29.394 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 14:39:29.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:29.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:29.394 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:39:29.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:29.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:29.394 | INFO | Uploaded 36GiB to GPU in 2.00854s (17.9235GiB/s) | 20260506 14:39:29.394 | INFO | Downloaded 34GiB to GPU in 1.935s (17.5711GiB/s) | 20260506 14:39:34.533 | INFO | Analyzing chunk frames = [2139648,2140160) | 20260506 14:39:34.533 | INFO | MemInfo: { | 20260506 14:39:34.533 | INFO | "gpu" : " 17 GB", | 20260506 14:39:34.533 | INFO | "huge" : "", | 20260506 14:39:34.533 | INFO | "huge_shared" : " 27 GB", | 20260506 14:39:34.533 | INFO | "malloc" : " 14 GB", | 20260506 14:39:34.533 | INFO | "pinned" : " 29 GB" | 20260506 14:39:34.533 | INFO | } | 20260506 14:39:34.533 | INFO | Perf report for the last 5.13922 seconds: | 20260506 14:39:34.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:39:34.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:34.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:34.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:34.533 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 14:39:34.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:34.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:34.533 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 14:39:34.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:34.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:34.534 | INFO | Uploaded 36GiB to GPU in 1.99194s (18.0728GiB/s) | 20260506 14:39:34.534 | INFO | Downloaded 34GiB to GPU in 1.92501s (17.6622GiB/s) | 20260506 14:39:39.673 | INFO | Analyzing chunk frames = [2140160,2140672) | 20260506 14:39:39.673 | INFO | MemInfo: { | 20260506 14:39:39.673 | INFO | "gpu" : " 17 GB", | 20260506 14:39:39.673 | INFO | "huge" : "", | 20260506 14:39:39.673 | INFO | "huge_shared" : " 27 GB", | 20260506 14:39:39.673 | INFO | "malloc" : " 14 GB", | 20260506 14:39:39.673 | INFO | "pinned" : " 29 GB" | 20260506 14:39:39.673 | INFO | } | 20260506 14:39:39.673 | INFO | Perf report for the last 5.14008 seconds: | 20260506 14:39:39.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:39:39.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:39.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:39.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:39.673 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 14:39:39.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:39.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:39.674 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 14:39:39.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:39.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:39.674 | INFO | Uploaded 36GiB to GPU in 2.03016s (17.7326GiB/s) | 20260506 14:39:39.674 | INFO | Downloaded 34GiB to GPU in 1.93915s (17.5335GiB/s) | 20260506 14:39:44.813 | INFO | Analyzing chunk frames = [2140672,2141184) | 20260506 14:39:44.813 | INFO | MemInfo: { | 20260506 14:39:44.813 | INFO | "gpu" : " 17 GB", | 20260506 14:39:44.813 | INFO | "huge" : "", | 20260506 14:39:44.813 | INFO | "huge_shared" : " 27 GB", | 20260506 14:39:44.813 | INFO | "malloc" : " 14 GB", | 20260506 14:39:44.813 | INFO | "pinned" : " 29 GB" | 20260506 14:39:44.813 | INFO | } | 20260506 14:39:44.813 | INFO | Perf report for the last 5.13981 seconds: | 20260506 14:39:44.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:39:44.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:44.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:44.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:44.813 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:39:44.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:44.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:44.813 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:39:44.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:44.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:44.814 | INFO | Uploaded 36GiB to GPU in 2.04486s (17.6051GiB/s) | 20260506 14:39:44.814 | INFO | Downloaded 34GiB to GPU in 1.92592s (17.6539GiB/s) | 20260506 14:39:49.954 | INFO | Analyzing chunk frames = [2141184,2141696) | 20260506 14:39:49.954 | INFO | MemInfo: { | 20260506 14:39:49.954 | INFO | "gpu" : " 17 GB", | 20260506 14:39:49.954 | INFO | "huge" : "", | 20260506 14:39:49.954 | INFO | "huge_shared" : " 27 GB", | 20260506 14:39:49.954 | INFO | "malloc" : " 14 GB", | 20260506 14:39:49.954 | INFO | "pinned" : " 29 GB" | 20260506 14:39:49.954 | INFO | } | 20260506 14:39:49.954 | INFO | Perf report for the last 5.14115 seconds: | 20260506 14:39:49.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:39:49.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:49.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:49.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:49.954 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:39:49.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:49.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:49.954 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 14:39:49.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:49.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:49.955 | INFO | Uploaded 36GiB to GPU in 1.98749s (18.1133GiB/s) | 20260506 14:39:49.955 | INFO | Downloaded 34GiB to GPU in 2.00435s (16.9631GiB/s) | 20260506 14:39:55.091 | INFO | Analyzing chunk frames = [2141696,2142208) | 20260506 14:39:55.091 | INFO | MemInfo: { | 20260506 14:39:55.091 | INFO | "gpu" : " 17 GB", | 20260506 14:39:55.091 | INFO | "huge" : "", | 20260506 14:39:55.091 | INFO | "huge_shared" : " 27 GB", | 20260506 14:39:55.091 | INFO | "malloc" : " 14 GB", | 20260506 14:39:55.091 | INFO | "pinned" : " 29 GB" | 20260506 14:39:55.091 | INFO | } | 20260506 14:39:55.091 | INFO | Perf report for the last 5.13653 seconds: | 20260506 14:39:55.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:39:55.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:55.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:55.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:55.091 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 14:39:55.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:55.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:55.091 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 14:39:55.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:55.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:39:55.091 | INFO | Uploaded 36GiB to GPU in 2.02123s (17.811GiB/s) | 20260506 14:39:55.091 | INFO | Downloaded 34GiB to GPU in 1.97914s (17.1791GiB/s) | 20260506 14:39:55.201 | INFO | Starting a round of Baz IO | 20260506 14:39:58.356 | INFO | Populating RTMetrics report | 20260506 14:39:58.356 | INFO | RTMetrics report populated | 20260506 14:39:58.357 | INFO | Opening tmp RTMetrics json file | 20260506 14:39:58.377 | INFO | Serializing RTMetrics to json | 20260506 14:39:58.401 | INFO | Serialization done, swapping tmp file now... | 20260506 14:39:58.404 | INFO | RTMetrics done | 20260506 14:39:58.786 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.585 seconds | 20260506 14:39:58.786 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.04% / 0.01% / 61.94% | 20260506 14:39:58.786 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.43803% / 1.58946e-05% / 5.16574e-05% | 20260506 14:39:58.786 | INFO | Baz IO overall: | 20260506 14:39:58.786 | INFO | BytesWritten : 1108168543984 | 20260506 14:39:58.786 | INFO | NumEvents : 536264427910 | 20260506 14:40:00.230 | INFO | Analyzing chunk frames = [2142208,2142720) | 20260506 14:40:00.230 | INFO | MemInfo: { | 20260506 14:40:00.230 | INFO | "gpu" : " 17 GB", | 20260506 14:40:00.230 | INFO | "huge" : "", | 20260506 14:40:00.230 | INFO | "huge_shared" : " 27 GB", | 20260506 14:40:00.230 | INFO | "malloc" : " 14 GB", | 20260506 14:40:00.230 | INFO | "pinned" : " 29 GB" | 20260506 14:40:00.230 | INFO | } | 20260506 14:40:00.230 | INFO | Perf report for the last 5.1393 seconds: | 20260506 14:40:00.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:40:00.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:00.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:00.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:00.230 | INFO | ANALYSIS : 63.0%, 2.9, 95.0ms, 0 | 20260506 14:40:00.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:00.230 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 14:40:00.230 | INFO | PRE_HQ : 61.2%, 1.9, 59.9ms, 0 | 20260506 14:40:00.230 | INFO | BAZWRITER : 69.2%, 4.8, 172.3ms, 0 | 20260506 14:40:00.230 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 14:40:00.230 | INFO | Uploaded 36GiB to GPU in 2.08709s (17.2489GiB/s) | 20260506 14:40:00.230 | INFO | Downloaded 38GiB to GPU in 2.23076s (17.0345GiB/s) | 20260506 14:40:05.370 | INFO | Analyzing chunk frames = [2142720,2143232) | 20260506 14:40:05.370 | INFO | MemInfo: { | 20260506 14:40:05.370 | INFO | "gpu" : " 17 GB", | 20260506 14:40:05.370 | INFO | "huge" : "", | 20260506 14:40:05.371 | INFO | "huge_shared" : " 27 GB", | 20260506 14:40:05.371 | INFO | "malloc" : " 14 GB", | 20260506 14:40:05.371 | INFO | "pinned" : " 29 GB" | 20260506 14:40:05.371 | INFO | } | 20260506 14:40:05.371 | INFO | Perf report for the last 5.14037 seconds: | 20260506 14:40:05.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:40:05.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:05.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:05.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:05.371 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 14:40:05.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:05.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:05.371 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:40:05.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:05.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:05.371 | INFO | Uploaded 36GiB to GPU in 2.03362s (17.7024GiB/s) | 20260506 14:40:05.371 | INFO | Downloaded 34GiB to GPU in 1.94689s (17.4637GiB/s) | 20260506 14:40:10.509 | INFO | Analyzing chunk frames = [2143232,2143744) | 20260506 14:40:10.509 | INFO | MemInfo: { | 20260506 14:40:10.509 | INFO | "gpu" : " 17 GB", | 20260506 14:40:10.509 | INFO | "huge" : "", | 20260506 14:40:10.509 | INFO | "huge_shared" : " 27 GB", | 20260506 14:40:10.509 | INFO | "malloc" : " 14 GB", | 20260506 14:40:10.509 | INFO | "pinned" : " 29 GB" | 20260506 14:40:10.509 | INFO | } | 20260506 14:40:10.509 | INFO | Perf report for the last 5.1388 seconds: | 20260506 14:40:10.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:40:10.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:10.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:10.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:10.509 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:40:10.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:10.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:10.510 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:40:10.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:10.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:10.510 | INFO | Uploaded 36GiB to GPU in 2.00385s (17.9654GiB/s) | 20260506 14:40:10.510 | INFO | Downloaded 34GiB to GPU in 1.92869s (17.6286GiB/s) | 20260506 14:40:15.648 | INFO | Analyzing chunk frames = [2143744,2144256) | 20260506 14:40:15.648 | INFO | MemInfo: { | 20260506 14:40:15.648 | INFO | "gpu" : " 17 GB", | 20260506 14:40:15.648 | INFO | "huge" : "", | 20260506 14:40:15.648 | INFO | "huge_shared" : " 27 GB", | 20260506 14:40:15.649 | INFO | "malloc" : " 14 GB", | 20260506 14:40:15.649 | INFO | "pinned" : " 29 GB" | 20260506 14:40:15.649 | INFO | } | 20260506 14:40:15.649 | INFO | Perf report for the last 5.13915 seconds: | 20260506 14:40:15.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:40:15.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:15.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:15.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:15.649 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260506 14:40:15.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:15.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:15.649 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:40:15.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:15.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:15.649 | INFO | Uploaded 36GiB to GPU in 1.97295s (18.2468GiB/s) | 20260506 14:40:15.649 | INFO | Downloaded 34GiB to GPU in 1.91112s (17.7906GiB/s) | 20260506 14:40:20.788 | INFO | Analyzing chunk frames = [2144256,2144768) | 20260506 14:40:20.788 | INFO | MemInfo: { | 20260506 14:40:20.788 | INFO | "gpu" : " 17 GB", | 20260506 14:40:20.789 | INFO | "huge" : "", | 20260506 14:40:20.789 | INFO | "huge_shared" : " 27 GB", | 20260506 14:40:20.789 | INFO | "malloc" : " 14 GB", | 20260506 14:40:20.789 | INFO | "pinned" : " 29 GB" | 20260506 14:40:20.789 | INFO | } | 20260506 14:40:20.789 | INFO | Perf report for the last 5.14043 seconds: | 20260506 14:40:20.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:40:20.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:20.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:20.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:20.789 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 14:40:20.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:20.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:20.789 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:40:20.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:20.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:20.789 | INFO | Uploaded 36GiB to GPU in 2.0277s (17.7541GiB/s) | 20260506 14:40:20.790 | INFO | Downloaded 34GiB to GPU in 1.94178s (17.5097GiB/s) | 20260506 14:40:25.928 | INFO | Analyzing chunk frames = [2144768,2145280) | 20260506 14:40:25.928 | INFO | MemInfo: { | 20260506 14:40:25.928 | INFO | "gpu" : " 17 GB", | 20260506 14:40:25.928 | INFO | "huge" : "", | 20260506 14:40:25.928 | INFO | "huge_shared" : " 27 GB", | 20260506 14:40:25.928 | INFO | "malloc" : " 14 GB", | 20260506 14:40:25.928 | INFO | "pinned" : " 29 GB" | 20260506 14:40:25.928 | INFO | } | 20260506 14:40:25.928 | INFO | Perf report for the last 5.13879 seconds: | 20260506 14:40:25.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:40:25.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:25.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:25.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:25.928 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 14:40:25.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:25.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:25.928 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 14:40:25.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:25.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:25.928 | INFO | Uploaded 36GiB to GPU in 2.05467s (17.5211GiB/s) | 20260506 14:40:25.928 | INFO | Downloaded 34GiB to GPU in 1.89683s (17.9246GiB/s) | 20260506 14:40:31.067 | INFO | Analyzing chunk frames = [2145280,2145792) | 20260506 14:40:31.067 | INFO | MemInfo: { | 20260506 14:40:31.067 | INFO | "gpu" : " 17 GB", | 20260506 14:40:31.067 | INFO | "huge" : "", | 20260506 14:40:31.067 | INFO | "huge_shared" : " 27 GB", | 20260506 14:40:31.067 | INFO | "malloc" : " 14 GB", | 20260506 14:40:31.067 | INFO | "pinned" : " 29 GB" | 20260506 14:40:31.067 | INFO | } | 20260506 14:40:31.067 | INFO | Perf report for the last 5.13945 seconds: | 20260506 14:40:31.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:40:31.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:31.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:31.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:31.067 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:40:31.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:31.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:31.067 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 14:40:31.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:31.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:31.068 | INFO | Uploaded 36GiB to GPU in 1.99401s (18.0541GiB/s) | 20260506 14:40:31.068 | INFO | Downloaded 34GiB to GPU in 1.99626s (17.0319GiB/s) | 20260506 14:40:36.206 | INFO | Analyzing chunk frames = [2145792,2146304) | 20260506 14:40:36.206 | INFO | MemInfo: { | 20260506 14:40:36.206 | INFO | "gpu" : " 17 GB", | 20260506 14:40:36.206 | INFO | "huge" : "", | 20260506 14:40:36.206 | INFO | "huge_shared" : " 27 GB", | 20260506 14:40:36.206 | INFO | "malloc" : " 14 GB", | 20260506 14:40:36.207 | INFO | "pinned" : " 29 GB" | 20260506 14:40:36.207 | INFO | } | 20260506 14:40:36.207 | INFO | Perf report for the last 5.13928 seconds: | 20260506 14:40:36.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:40:36.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:36.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:36.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:36.207 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:40:36.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:36.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:36.207 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:40:36.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:36.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:36.207 | INFO | Uploaded 36GiB to GPU in 2.01645s (17.8532GiB/s) | 20260506 14:40:36.207 | INFO | Downloaded 34GiB to GPU in 1.9832s (17.144GiB/s) | 20260506 14:40:36.207 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:40:36.207 | INFO | Total mem usage: 17.4 GiB | 20260506 14:40:36.207 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:40:36.207 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: -5.51%) | 20260506 14:40:36.207 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:40:36.207 | INFO | High Water Breakdown: | 20260506 14:40:36.207 | INFO | 62% : 10.2 GiB ( 93.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:40:36.207 | INFO | 37.9% : 6.25 GiB ( 93.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:40:36.207 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:40:36.207 | INFO | ------------------------------------------ | 20260506 14:40:36.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:40:36.207 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:40:36.207 | INFO | Total mem usage: 26.5 GiB | 20260506 14:40:36.207 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:40:36.207 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:40:36.207 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:40:36.207 | INFO | High Water Breakdown: | 20260506 14:40:36.207 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:40:36.207 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:40:36.207 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:40:36.207 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:40:36.207 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:40:36.207 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:40:36.207 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:40:36.207 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:40:36.207 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:40:36.207 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:40:36.207 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:40:36.208 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:40:36.208 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:40:36.208 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:40:36.208 | INFO | ------------------------------------------ | 20260506 14:40:36.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:40:36.208 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:40:36.208 | INFO | Total mem usage: 25.6 GiB | 20260506 14:40:36.208 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:40:36.208 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:40:36.208 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:40:36.208 | INFO | High Water Breakdown: | 20260506 14:40:36.208 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:40:36.208 | INFO | ------------------------------------------ | 20260506 14:40:36.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:40:36.208 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:40:36.208 | INFO | Total mem usage: 15.7 GiB | 20260506 14:40:36.208 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:40:36.208 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:40:36.208 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 14:40:36.208 | INFO | High Water Breakdown: | 20260506 14:40:36.208 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:40:36.208 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:40:36.208 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:40:36.208 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:40:36.208 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:40:36.208 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:40:36.208 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:40:36.208 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:40:36.208 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:40:36.208 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:40:36.208 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:40:36.208 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:40:36.208 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:40:36.208 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:40:36.208 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:40:36.208 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:40:36.208 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:40:36.208 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:40:36.208 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:40:36.208 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:40:36.208 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:40:36.208 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:40:36.208 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:40:36.208 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:40:36.208 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:40:36.208 | INFO | ------------------------------------------ | 20260506 14:40:36.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:40:36.208 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:40:36.208 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:40:36.208 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:40:36.208 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.54e+01, 1.35e+01 / 1.54e+00 | 20260506 14:40:36.208 | INFO | PerfReport: 0.29%, AggregateStats -- 9.25e-02 / 1.04e-01, 1.75e-02 / 7.69e-03 | 20260506 14:40:36.208 | INFO | PerfReport: 11.58%, Baseline -- 3.65e+00 / 4.49e+00, 5.67e-01 / 3.07e-01 | 20260506 14:40:36.208 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.48e+00, 1.87e-01 / 1.39e-01 | 20260506 14:40:36.208 | INFO | PerfReport: 15.78%, DME -- 3.98e+01 / 4.38e+01, 3.82e+00 / 1.81e+00 | 20260506 14:40:36.208 | INFO | PerfReport: 56.79%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.48e+00 / 1.79e+00 | 20260506 14:40:36.208 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.35e+00 / 2.92e+00, 4.13e-01 / 2.71e-01 | 20260506 14:40:36.208 | INFO | PerfReport: 3.60%, Metrics -- 1.13e+00 / 1.48e+00, 2.87e-01 / 1.41e-01 | 20260506 14:40:36.208 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.96e-01 / 4.48e-01, 6.18e-02 / 2.19e-02 | 20260506 14:40:36.208 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:40:36.208 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:40:36.208 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:40:36.208 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 14:40:36.208 | INFO | PerfReport: 21.61%, Upload -- 2.11e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260506 14:40:36.208 | INFO | PerfReport: 21.13%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 14:40:36.208 | INFO | PerfReport: 32.26%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 14:40:36.208 | INFO | PerfReport: 1.63%, LockUpload -- 1.59e+00 / 0.00e+00, 6.39e+00 / 0.00e+00 | 20260506 14:40:36.208 | INFO | PerfReport: 1.11%, LockDownload -- 1.08e+00 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260506 14:40:36.208 | INFO | PerfReport: 22.24%, LockCompute -- 2.17e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 14:40:36.208 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260506 14:40:36.318 | INFO | Starting a round of Baz IO | 20260506 14:40:39.478 | INFO | Populating RTMetrics report | 20260506 14:40:39.479 | INFO | RTMetrics report populated | 20260506 14:40:39.479 | INFO | Opening tmp RTMetrics json file | 20260506 14:40:39.493 | INFO | Serializing RTMetrics to json | 20260506 14:40:39.571 | INFO | Serialization done, swapping tmp file now... | 20260506 14:40:39.629 | INFO | RTMetrics done | 20260506 14:40:39.852 | INFO | Finished a round of Baz IO: Wrote 2.113 GiB in 3.534 seconds | 20260506 14:40:39.852 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.03% / 0.01% / 61.96% | 20260506 14:40:39.852 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.44037% / 3.97364e-06% / 5.96046e-05% | 20260506 14:40:39.852 | INFO | Baz IO overall: | 20260506 14:40:39.852 | INFO | BytesWritten : 1110437418409 | 20260506 14:40:39.853 | INFO | NumEvents : 537364418336 | 20260506 14:40:41.345 | INFO | Analyzing chunk frames = [2146304,2146816) | 20260506 14:40:41.345 | INFO | MemInfo: { | 20260506 14:40:41.345 | INFO | "gpu" : " 17 GB", | 20260506 14:40:41.345 | INFO | "huge" : "", | 20260506 14:40:41.345 | INFO | "huge_shared" : " 27 GB", | 20260506 14:40:41.345 | INFO | "malloc" : " 14 GB", | 20260506 14:40:41.345 | INFO | "pinned" : " 29 GB" | 20260506 14:40:41.345 | INFO | } | 20260506 14:40:41.345 | INFO | Perf report for the last 5.1386 seconds: | 20260506 14:40:41.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:40:41.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:41.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:41.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:41.345 | INFO | ANALYSIS : 63.1%, 2.9, 95.3ms, 0 | 20260506 14:40:41.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:41.345 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260506 14:40:41.345 | INFO | PRE_HQ : 61.1%, 1.9, 59.5ms, 0 | 20260506 14:40:41.345 | INFO | BAZWRITER : 68.1%, 4.8, 169.2ms, 0 | 20260506 14:40:41.345 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.4ms, 0 | 20260506 14:40:41.346 | INFO | Uploaded 36GiB to GPU in 2.05698s (17.5014GiB/s) | 20260506 14:40:41.346 | INFO | Downloaded 38GiB to GPU in 2.21795s (17.133GiB/s) | 20260506 14:40:46.485 | INFO | Analyzing chunk frames = [2146816,2147328) | 20260506 14:40:46.485 | INFO | MemInfo: { | 20260506 14:40:46.485 | INFO | "gpu" : " 17 GB", | 20260506 14:40:46.485 | INFO | "huge" : "", | 20260506 14:40:46.485 | INFO | "huge_shared" : " 27 GB", | 20260506 14:40:46.485 | INFO | "malloc" : " 14 GB", | 20260506 14:40:46.485 | INFO | "pinned" : " 29 GB" | 20260506 14:40:46.485 | INFO | } | 20260506 14:40:46.485 | INFO | Perf report for the last 5.13962 seconds: | 20260506 14:40:46.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:40:46.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:46.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:46.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:46.485 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:40:46.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:46.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:46.485 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:40:46.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:46.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:46.485 | INFO | Uploaded 36GiB to GPU in 1.99508s (18.0444GiB/s) | 20260506 14:40:46.485 | INFO | Downloaded 34GiB to GPU in 1.93336s (17.586GiB/s) | 20260506 14:40:51.625 | INFO | Analyzing chunk frames = [2147328,2147840) | 20260506 14:40:51.625 | INFO | MemInfo: { | 20260506 14:40:51.625 | INFO | "gpu" : " 17 GB", | 20260506 14:40:51.625 | INFO | "huge" : "", | 20260506 14:40:51.625 | INFO | "huge_shared" : " 27 GB", | 20260506 14:40:51.625 | INFO | "malloc" : " 14 GB", | 20260506 14:40:51.625 | INFO | "pinned" : " 29 GB" | 20260506 14:40:51.625 | INFO | } | 20260506 14:40:51.625 | INFO | Perf report for the last 5.13995 seconds: | 20260506 14:40:51.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:40:51.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:51.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:51.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:51.625 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 14:40:51.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:51.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:51.625 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:40:51.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:51.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:51.625 | INFO | Uploaded 36GiB to GPU in 1.99023s (18.0883GiB/s) | 20260506 14:40:51.625 | INFO | Downloaded 34GiB to GPU in 1.93715s (17.5515GiB/s) | 20260506 14:40:56.764 | INFO | Analyzing chunk frames = [2147840,2148352) | 20260506 14:40:56.764 | INFO | MemInfo: { | 20260506 14:40:56.764 | INFO | "gpu" : " 17 GB", | 20260506 14:40:56.764 | INFO | "huge" : "", | 20260506 14:40:56.765 | INFO | "huge_shared" : " 27 GB", | 20260506 14:40:56.765 | INFO | "malloc" : " 14 GB", | 20260506 14:40:56.765 | INFO | "pinned" : " 29 GB" | 20260506 14:40:56.765 | INFO | } | 20260506 14:40:56.765 | INFO | Perf report for the last 5.13983 seconds: | 20260506 14:40:56.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:40:56.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:56.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:56.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:56.765 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:40:56.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:56.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:56.765 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:40:56.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:56.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:40:56.765 | INFO | Uploaded 36GiB to GPU in 1.97373s (18.2396GiB/s) | 20260506 14:40:56.765 | INFO | Downloaded 34GiB to GPU in 1.90973s (17.8035GiB/s) | 20260506 14:41:01.902 | INFO | Analyzing chunk frames = [2148352,2148864) | 20260506 14:41:01.902 | INFO | MemInfo: { | 20260506 14:41:01.902 | INFO | "gpu" : " 17 GB", | 20260506 14:41:01.902 | INFO | "huge" : "", | 20260506 14:41:01.903 | INFO | "huge_shared" : " 27 GB", | 20260506 14:41:01.903 | INFO | "malloc" : " 14 GB", | 20260506 14:41:01.903 | INFO | "pinned" : " 29 GB" | 20260506 14:41:01.903 | INFO | } | 20260506 14:41:01.903 | INFO | Perf report for the last 5.13799 seconds: | 20260506 14:41:01.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:41:01.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:01.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:01.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:01.903 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 14:41:01.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:01.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:01.903 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:41:01.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:01.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:01.903 | INFO | Uploaded 36GiB to GPU in 2.02398s (17.7868GiB/s) | 20260506 14:41:01.903 | INFO | Downloaded 34GiB to GPU in 1.93293s (17.5899GiB/s) | 20260506 14:41:07.042 | INFO | Analyzing chunk frames = [2148864,2149376) | 20260506 14:41:07.042 | INFO | MemInfo: { | 20260506 14:41:07.042 | INFO | "gpu" : " 17 GB", | 20260506 14:41:07.042 | INFO | "huge" : "", | 20260506 14:41:07.042 | INFO | "huge_shared" : " 27 GB", | 20260506 14:41:07.042 | INFO | "malloc" : " 14 GB", | 20260506 14:41:07.042 | INFO | "pinned" : " 29 GB" | 20260506 14:41:07.042 | INFO | } | 20260506 14:41:07.042 | INFO | Perf report for the last 5.13965 seconds: | 20260506 14:41:07.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:41:07.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:07.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:07.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:07.042 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 14:41:07.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:07.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:07.042 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 14:41:07.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:07.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:07.043 | INFO | Uploaded 36GiB to GPU in 2.04483s (17.6054GiB/s) | 20260506 14:41:07.043 | INFO | Downloaded 34GiB to GPU in 1.91631s (17.7424GiB/s) | 20260506 14:41:12.182 | INFO | Analyzing chunk frames = [2149376,2149888) | 20260506 14:41:12.182 | INFO | MemInfo: { | 20260506 14:41:12.182 | INFO | "gpu" : " 17 GB", | 20260506 14:41:12.182 | INFO | "huge" : "", | 20260506 14:41:12.182 | INFO | "huge_shared" : " 27 GB", | 20260506 14:41:12.182 | INFO | "malloc" : " 14 GB", | 20260506 14:41:12.182 | INFO | "pinned" : " 29 GB" | 20260506 14:41:12.182 | INFO | } | 20260506 14:41:12.182 | INFO | Perf report for the last 5.13964 seconds: | 20260506 14:41:12.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:41:12.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:12.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:12.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:12.182 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:41:12.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:12.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:12.182 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:41:12.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:12.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:12.182 | INFO | Uploaded 36GiB to GPU in 2.00592s (17.9468GiB/s) | 20260506 14:41:12.182 | INFO | Downloaded 34GiB to GPU in 1.99694s (17.026GiB/s) | 20260506 14:41:17.321 | INFO | Analyzing chunk frames = [2149888,2150400) | 20260506 14:41:17.321 | INFO | MemInfo: { | 20260506 14:41:17.321 | INFO | "gpu" : " 17 GB", | 20260506 14:41:17.321 | INFO | "huge" : "", | 20260506 14:41:17.321 | INFO | "huge_shared" : " 27 GB", | 20260506 14:41:17.321 | INFO | "malloc" : " 14 GB", | 20260506 14:41:17.321 | INFO | "pinned" : " 29 GB" | 20260506 14:41:17.321 | INFO | } | 20260506 14:41:17.321 | INFO | Perf report for the last 5.13907 seconds: | 20260506 14:41:17.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:41:17.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:17.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:17.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:17.321 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 14:41:17.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:17.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:17.321 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:41:17.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:17.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:17.321 | INFO | Uploaded 36GiB to GPU in 2.00129s (17.9884GiB/s) | 20260506 14:41:17.321 | INFO | Downloaded 34GiB to GPU in 1.98153s (17.1585GiB/s) | 20260506 14:41:17.434 | INFO | Starting a round of Baz IO | 20260506 14:41:20.594 | INFO | Populating RTMetrics report | 20260506 14:41:20.594 | INFO | RTMetrics report populated | 20260506 14:41:20.594 | INFO | Opening tmp RTMetrics json file | 20260506 14:41:20.608 | INFO | Serializing RTMetrics to json | 20260506 14:41:20.634 | INFO | Serialization done, swapping tmp file now... | 20260506 14:41:20.636 | INFO | RTMetrics done | 20260506 14:41:21.037 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.603 seconds | 20260506 14:41:21.037 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.01% / 0.01% / 61.97% | 20260506 14:41:21.037 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.42592% / 1.19209e-05% / 5.16574e-05% | 20260506 14:41:21.037 | INFO | Baz IO overall: | 20260506 14:41:21.037 | INFO | BytesWritten : 1112702790346 | 20260506 14:41:21.037 | INFO | NumEvents : 538461690399 | 20260506 14:41:22.461 | INFO | Analyzing chunk frames = [2150400,2150912) | 20260506 14:41:22.461 | INFO | MemInfo: { | 20260506 14:41:22.461 | INFO | "gpu" : " 17 GB", | 20260506 14:41:22.461 | INFO | "huge" : "", | 20260506 14:41:22.461 | INFO | "huge_shared" : " 27 GB", | 20260506 14:41:22.461 | INFO | "malloc" : " 14 GB", | 20260506 14:41:22.461 | INFO | "pinned" : " 29 GB" | 20260506 14:41:22.461 | INFO | } | 20260506 14:41:22.461 | INFO | Perf report for the last 5.13995 seconds: | 20260506 14:41:22.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:41:22.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:22.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:22.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:22.461 | INFO | ANALYSIS : 63.1%, 2.9, 95.4ms, 0 | 20260506 14:41:22.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:22.461 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 14:41:22.461 | INFO | PRE_HQ : 61.0%, 1.9, 59.4ms, 0 | 20260506 14:41:22.461 | INFO | BAZWRITER : 69.4%, 4.8, 173.0ms, 0 | 20260506 14:41:22.461 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 14:41:22.461 | INFO | Uploaded 36GiB to GPU in 2.05415s (17.5255GiB/s) | 20260506 14:41:22.461 | INFO | Downloaded 38GiB to GPU in 2.23088s (17.0337GiB/s) | 20260506 14:41:27.599 | INFO | Analyzing chunk frames = [2150912,2151424) | 20260506 14:41:27.599 | INFO | MemInfo: { | 20260506 14:41:27.599 | INFO | "gpu" : " 17 GB", | 20260506 14:41:27.599 | INFO | "huge" : "", | 20260506 14:41:27.599 | INFO | "huge_shared" : " 27 GB", | 20260506 14:41:27.599 | INFO | "malloc" : " 14 GB", | 20260506 14:41:27.599 | INFO | "pinned" : " 29 GB" | 20260506 14:41:27.599 | INFO | } | 20260506 14:41:27.599 | INFO | Perf report for the last 5.13841 seconds: | 20260506 14:41:27.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:41:27.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:27.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:27.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:27.600 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 14:41:27.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:27.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:27.600 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:41:27.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:27.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:27.600 | INFO | Uploaded 36GiB to GPU in 1.97484s (18.2294GiB/s) | 20260506 14:41:27.600 | INFO | Downloaded 34GiB to GPU in 1.94084s (17.5182GiB/s) | 20260506 14:41:32.738 | INFO | Analyzing chunk frames = [2151424,2151936) | 20260506 14:41:32.738 | INFO | MemInfo: { | 20260506 14:41:32.738 | INFO | "gpu" : " 17 GB", | 20260506 14:41:32.738 | INFO | "huge" : "", | 20260506 14:41:32.739 | INFO | "huge_shared" : " 27 GB", | 20260506 14:41:32.739 | INFO | "malloc" : " 14 GB", | 20260506 14:41:32.739 | INFO | "pinned" : " 29 GB" | 20260506 14:41:32.739 | INFO | } | 20260506 14:41:32.739 | INFO | Perf report for the last 5.13919 seconds: | 20260506 14:41:32.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:41:32.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:32.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:32.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:32.739 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 14:41:32.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:32.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:32.739 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:41:32.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:32.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:32.739 | INFO | Uploaded 36GiB to GPU in 1.95557s (18.409GiB/s) | 20260506 14:41:32.739 | INFO | Downloaded 34GiB to GPU in 1.94476s (17.4828GiB/s) | 20260506 14:41:37.880 | INFO | Analyzing chunk frames = [2151936,2152448) | 20260506 14:41:37.880 | INFO | MemInfo: { | 20260506 14:41:37.880 | INFO | "gpu" : " 17 GB", | 20260506 14:41:37.880 | INFO | "huge" : "", | 20260506 14:41:37.880 | INFO | "huge_shared" : " 27 GB", | 20260506 14:41:37.880 | INFO | "malloc" : " 14 GB", | 20260506 14:41:37.880 | INFO | "pinned" : " 29 GB" | 20260506 14:41:37.880 | INFO | } | 20260506 14:41:37.880 | INFO | Perf report for the last 5.14132 seconds: | 20260506 14:41:37.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:41:37.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:37.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:37.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:37.880 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 14:41:37.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:37.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:37.880 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:41:37.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:37.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:37.880 | INFO | Uploaded 36GiB to GPU in 1.9473s (18.4872GiB/s) | 20260506 14:41:37.880 | INFO | Downloaded 34GiB to GPU in 1.89289s (17.962GiB/s) | 20260506 14:41:43.018 | INFO | Analyzing chunk frames = [2152448,2152960) | 20260506 14:41:43.018 | INFO | MemInfo: { | 20260506 14:41:43.018 | INFO | "gpu" : " 17 GB", | 20260506 14:41:43.018 | INFO | "huge" : "", | 20260506 14:41:43.018 | INFO | "huge_shared" : " 27 GB", | 20260506 14:41:43.018 | INFO | "malloc" : " 14 GB", | 20260506 14:41:43.018 | INFO | "pinned" : " 29 GB" | 20260506 14:41:43.018 | INFO | } | 20260506 14:41:43.018 | INFO | Perf report for the last 5.13819 seconds: | 20260506 14:41:43.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:41:43.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:43.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:43.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:43.018 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 14:41:43.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:43.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:43.018 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:41:43.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:43.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:43.018 | INFO | Uploaded 36GiB to GPU in 1.98243s (18.1595GiB/s) | 20260506 14:41:43.018 | INFO | Downloaded 34GiB to GPU in 1.91935s (17.7143GiB/s) | 20260506 14:41:48.157 | INFO | Analyzing chunk frames = [2152960,2153472) | 20260506 14:41:48.157 | INFO | MemInfo: { | 20260506 14:41:48.157 | INFO | "gpu" : " 17 GB", | 20260506 14:41:48.157 | INFO | "huge" : "", | 20260506 14:41:48.157 | INFO | "huge_shared" : " 27 GB", | 20260506 14:41:48.157 | INFO | "malloc" : " 14 GB", | 20260506 14:41:48.157 | INFO | "pinned" : " 29 GB" | 20260506 14:41:48.157 | INFO | } | 20260506 14:41:48.157 | INFO | Perf report for the last 5.13897 seconds: | 20260506 14:41:48.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:41:48.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:48.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:48.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:48.157 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 14:41:48.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:48.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:48.157 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:41:48.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:48.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:48.158 | INFO | Uploaded 36GiB to GPU in 2.05504s (17.5179GiB/s) | 20260506 14:41:48.158 | INFO | Downloaded 34GiB to GPU in 1.96759s (17.28GiB/s) | 20260506 14:41:53.297 | INFO | Analyzing chunk frames = [2153472,2153984) | 20260506 14:41:53.297 | INFO | MemInfo: { | 20260506 14:41:53.297 | INFO | "gpu" : " 17 GB", | 20260506 14:41:53.298 | INFO | "huge" : "", | 20260506 14:41:53.298 | INFO | "huge_shared" : " 27 GB", | 20260506 14:41:53.298 | INFO | "malloc" : " 14 GB", | 20260506 14:41:53.298 | INFO | "pinned" : " 29 GB" | 20260506 14:41:53.298 | INFO | } | 20260506 14:41:53.298 | INFO | Perf report for the last 5.14047 seconds: | 20260506 14:41:53.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:41:53.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:53.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:53.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:53.298 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 14:41:53.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:53.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:53.298 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 14:41:53.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:53.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:53.298 | INFO | Uploaded 36GiB to GPU in 2.04434s (17.6096GiB/s) | 20260506 14:41:53.298 | INFO | Downloaded 34GiB to GPU in 2.00982s (16.9169GiB/s) | 20260506 14:41:58.435 | INFO | Analyzing chunk frames = [2153984,2154496) | 20260506 14:41:58.435 | INFO | MemInfo: { | 20260506 14:41:58.435 | INFO | "gpu" : " 17 GB", | 20260506 14:41:58.435 | INFO | "huge" : "", | 20260506 14:41:58.436 | INFO | "huge_shared" : " 27 GB", | 20260506 14:41:58.436 | INFO | "malloc" : " 14 GB", | 20260506 14:41:58.436 | INFO | "pinned" : " 29 GB" | 20260506 14:41:58.436 | INFO | } | 20260506 14:41:58.436 | INFO | Perf report for the last 5.13807 seconds: | 20260506 14:41:58.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:41:58.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:58.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:58.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:58.436 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:41:58.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:58.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:58.436 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:41:58.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:58.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:41:58.436 | INFO | Uploaded 36GiB to GPU in 1.97643s (18.2147GiB/s) | 20260506 14:41:58.436 | INFO | Downloaded 34GiB to GPU in 1.97358s (17.2276GiB/s) | 20260506 14:41:58.436 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:41:58.436 | INFO | Total mem usage: 17.4 GiB | 20260506 14:41:58.436 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:41:58.436 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -7.65%) | 20260506 14:41:58.436 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:41:58.436 | INFO | High Water Breakdown: | 20260506 14:41:58.436 | INFO | 61.7% : 9.98 GiB ( 91.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:41:58.436 | INFO | 38.2% : 6.17 GiB ( 92.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:41:58.436 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:41:58.436 | INFO | ------------------------------------------ | 20260506 14:41:58.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:41:58.436 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:41:58.436 | INFO | Total mem usage: 26.5 GiB | 20260506 14:41:58.437 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:41:58.437 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:41:58.437 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:41:58.437 | INFO | High Water Breakdown: | 20260506 14:41:58.437 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:41:58.437 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:41:58.437 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:41:58.437 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:41:58.437 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:41:58.437 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:41:58.437 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:41:58.437 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:41:58.437 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:41:58.437 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:41:58.437 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:41:58.437 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:41:58.437 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:41:58.437 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:41:58.437 | INFO | ------------------------------------------ | 20260506 14:41:58.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:41:58.437 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:41:58.437 | INFO | Total mem usage: 25.6 GiB | 20260506 14:41:58.437 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:41:58.437 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:41:58.437 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:41:58.437 | INFO | High Water Breakdown: | 20260506 14:41:58.437 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:41:58.437 | INFO | ------------------------------------------ | 20260506 14:41:58.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:41:58.437 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:41:58.437 | INFO | Total mem usage: 15.7 GiB | 20260506 14:41:58.437 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:41:58.437 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:41:58.437 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 14:41:58.437 | INFO | High Water Breakdown: | 20260506 14:41:58.437 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:41:58.437 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:41:58.437 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:41:58.437 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:41:58.437 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:41:58.437 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:41:58.437 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:41:58.437 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:41:58.437 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:41:58.437 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:41:58.437 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:41:58.437 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:41:58.437 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:41:58.437 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:41:58.437 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:41:58.437 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:41:58.437 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:41:58.437 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:41:58.437 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:41:58.437 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:41:58.437 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:41:58.437 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:41:58.437 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:41:58.437 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:41:58.437 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:41:58.437 | INFO | ------------------------------------------ | 20260506 14:41:58.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:41:58.437 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:41:58.437 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:41:58.437 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:41:58.437 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.56e+01, 1.36e+01 / 1.72e+00 | 20260506 14:41:58.437 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.00e-01, 1.01e-02 / 6.45e-03 | 20260506 14:41:58.437 | INFO | PerfReport: 11.58%, Baseline -- 3.65e+00 / 4.47e+00, 5.70e-01 / 3.16e-01 | 20260506 14:41:58.437 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.47e+00, 1.89e-01 / 1.50e-01 | 20260506 14:41:58.437 | INFO | PerfReport: 15.83%, DME -- 3.99e+01 / 4.41e+01, 3.73e+00 / 2.07e+00 | 20260506 14:41:58.437 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.49e+00 / 1.83e+00 | 20260506 14:41:58.437 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.34e+00 / 2.95e+00, 4.10e-01 / 2.39e-01 | 20260506 14:41:58.437 | INFO | PerfReport: 3.59%, Metrics -- 1.13e+00 / 1.52e+00, 2.90e-01 / 1.85e-01 | 20260506 14:41:58.437 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.95e-01 / 4.49e-01, 6.14e-02 / 2.22e-02 | 20260506 14:41:58.437 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:41:58.437 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:41:58.437 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:41:58.437 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 14:41:58.437 | INFO | PerfReport: 21.41%, Upload -- 2.09e+01 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260506 14:41:58.437 | INFO | PerfReport: 21.16%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 14:41:58.437 | INFO | PerfReport: 32.31%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 14:41:58.437 | INFO | PerfReport: 1.44%, LockUpload -- 1.40e+00 / 0.00e+00, 5.77e+00 / 0.00e+00 | 20260506 14:41:58.437 | INFO | PerfReport: 1.15%, LockDownload -- 1.12e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260506 14:41:58.437 | INFO | PerfReport: 22.52%, LockCompute -- 2.20e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:41:58.437 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260506 14:41:58.555 | INFO | Starting a round of Baz IO | 20260506 14:42:01.699 | INFO | Populating RTMetrics report | 20260506 14:42:01.700 | INFO | RTMetrics report populated | 20260506 14:42:01.700 | INFO | Opening tmp RTMetrics json file | 20260506 14:42:01.701 | INFO | Serializing RTMetrics to json | 20260506 14:42:01.711 | INFO | Serialization done, swapping tmp file now... | 20260506 14:42:01.715 | INFO | RTMetrics done | 20260506 14:42:02.045 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.49 seconds | 20260506 14:42:02.045 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 38.00% / 0.01% / 61.99% | 20260506 14:42:02.045 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.42672% / 1.58946e-05% / 4.76837e-05% | 20260506 14:42:02.045 | INFO | Baz IO overall: | 20260506 14:42:02.045 | INFO | BytesWritten : 1114970769871 | 20260506 14:42:02.045 | INFO | NumEvents : 539560978166 | 20260506 14:42:03.575 | INFO | Analyzing chunk frames = [2154496,2155008) | 20260506 14:42:03.575 | INFO | MemInfo: { | 20260506 14:42:03.575 | INFO | "gpu" : " 17 GB", | 20260506 14:42:03.575 | INFO | "huge" : "", | 20260506 14:42:03.575 | INFO | "huge_shared" : " 27 GB", | 20260506 14:42:03.575 | INFO | "malloc" : " 14 GB", | 20260506 14:42:03.575 | INFO | "pinned" : " 29 GB" | 20260506 14:42:03.575 | INFO | } | 20260506 14:42:03.575 | INFO | Perf report for the last 5.13952 seconds: | 20260506 14:42:03.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:42:03.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:03.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:03.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:03.575 | INFO | ANALYSIS : 62.9%, 2.9, 95.2ms, 0 | 20260506 14:42:03.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:03.575 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 14:42:03.575 | INFO | PRE_HQ : 60.9%, 1.9, 59.6ms, 0 | 20260506 14:42:03.575 | INFO | BAZWRITER : 67.5%, 4.8, 167.6ms, 0 | 20260506 14:42:03.576 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 14:42:03.576 | INFO | Uploaded 36GiB to GPU in 2.08396s (17.2748GiB/s) | 20260506 14:42:03.576 | INFO | Downloaded 38GiB to GPU in 2.22155s (17.1052GiB/s) | 20260506 14:42:08.715 | INFO | Analyzing chunk frames = [2155008,2155520) | 20260506 14:42:08.715 | INFO | MemInfo: { | 20260506 14:42:08.715 | INFO | "gpu" : " 17 GB", | 20260506 14:42:08.715 | INFO | "huge" : "", | 20260506 14:42:08.715 | INFO | "huge_shared" : " 27 GB", | 20260506 14:42:08.715 | INFO | "malloc" : " 14 GB", | 20260506 14:42:08.715 | INFO | "pinned" : " 29 GB" | 20260506 14:42:08.715 | INFO | } | 20260506 14:42:08.715 | INFO | Perf report for the last 5.1398 seconds: | 20260506 14:42:08.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:42:08.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:08.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:08.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:08.715 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:42:08.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:08.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:08.715 | INFO | PRE_HQ : 28.2%, 1.0, 14.8ms, 0 | 20260506 14:42:08.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:08.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:08.715 | INFO | Uploaded 36GiB to GPU in 2.01418s (17.8733GiB/s) | 20260506 14:42:08.715 | INFO | Downloaded 34GiB to GPU in 1.94674s (17.4651GiB/s) | 20260506 14:42:13.854 | INFO | Analyzing chunk frames = [2155520,2156032) | 20260506 14:42:13.854 | INFO | MemInfo: { | 20260506 14:42:13.854 | INFO | "gpu" : " 17 GB", | 20260506 14:42:13.854 | INFO | "huge" : "", | 20260506 14:42:13.854 | INFO | "huge_shared" : " 27 GB", | 20260506 14:42:13.854 | INFO | "malloc" : " 14 GB", | 20260506 14:42:13.854 | INFO | "pinned" : " 29 GB" | 20260506 14:42:13.854 | INFO | } | 20260506 14:42:13.854 | INFO | Perf report for the last 5.13932 seconds: | 20260506 14:42:13.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:42:13.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:13.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:13.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:13.854 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:42:13.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:13.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:13.854 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:42:13.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:13.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:13.855 | INFO | Uploaded 36GiB to GPU in 1.99981s (18.0017GiB/s) | 20260506 14:42:13.855 | INFO | Downloaded 34GiB to GPU in 1.93421s (17.5783GiB/s) | 20260506 14:42:18.993 | INFO | Analyzing chunk frames = [2156032,2156544) | 20260506 14:42:18.993 | INFO | MemInfo: { | 20260506 14:42:18.993 | INFO | "gpu" : " 17 GB", | 20260506 14:42:18.993 | INFO | "huge" : "", | 20260506 14:42:18.993 | INFO | "huge_shared" : " 27 GB", | 20260506 14:42:18.993 | INFO | "malloc" : " 14 GB", | 20260506 14:42:18.993 | INFO | "pinned" : " 29 GB" | 20260506 14:42:18.993 | INFO | } | 20260506 14:42:18.993 | INFO | Perf report for the last 5.13824 seconds: | 20260506 14:42:18.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:42:18.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:18.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:18.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:18.993 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 14:42:18.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:18.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:18.993 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:42:18.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:18.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:18.993 | INFO | Uploaded 36GiB to GPU in 1.97112s (18.2638GiB/s) | 20260506 14:42:18.993 | INFO | Downloaded 34GiB to GPU in 1.91215s (17.781GiB/s) | 20260506 14:42:24.133 | INFO | Analyzing chunk frames = [2156544,2157056) | 20260506 14:42:24.133 | INFO | MemInfo: { | 20260506 14:42:24.133 | INFO | "gpu" : " 17 GB", | 20260506 14:42:24.133 | INFO | "huge" : "", | 20260506 14:42:24.133 | INFO | "huge_shared" : " 27 GB", | 20260506 14:42:24.133 | INFO | "malloc" : " 14 GB", | 20260506 14:42:24.133 | INFO | "pinned" : " 29 GB" | 20260506 14:42:24.133 | INFO | } | 20260506 14:42:24.133 | INFO | Perf report for the last 5.1403 seconds: | 20260506 14:42:24.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:42:24.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:24.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:24.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:24.133 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 14:42:24.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:24.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:24.133 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:42:24.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:24.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:24.133 | INFO | Uploaded 36GiB to GPU in 2.00796s (17.9286GiB/s) | 20260506 14:42:24.133 | INFO | Downloaded 34GiB to GPU in 1.93537s (17.5677GiB/s) | 20260506 14:42:29.272 | INFO | Analyzing chunk frames = [2157056,2157568) | 20260506 14:42:29.272 | INFO | MemInfo: { | 20260506 14:42:29.272 | INFO | "gpu" : " 17 GB", | 20260506 14:42:29.272 | INFO | "huge" : "", | 20260506 14:42:29.272 | INFO | "huge_shared" : " 27 GB", | 20260506 14:42:29.273 | INFO | "malloc" : " 14 GB", | 20260506 14:42:29.273 | INFO | "pinned" : " 29 GB" | 20260506 14:42:29.273 | INFO | } | 20260506 14:42:29.273 | INFO | Perf report for the last 5.13966 seconds: | 20260506 14:42:29.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:42:29.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:29.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:29.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:29.273 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:42:29.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:29.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:29.273 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260506 14:42:29.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:29.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:29.273 | INFO | Uploaded 36GiB to GPU in 2.04089s (17.6393GiB/s) | 20260506 14:42:29.273 | INFO | Downloaded 34GiB to GPU in 1.92897s (17.626GiB/s) | 20260506 14:42:34.411 | INFO | Analyzing chunk frames = [2157568,2158080) | 20260506 14:42:34.411 | INFO | MemInfo: { | 20260506 14:42:34.411 | INFO | "gpu" : " 17 GB", | 20260506 14:42:34.411 | INFO | "huge" : "", | 20260506 14:42:34.411 | INFO | "huge_shared" : " 27 GB", | 20260506 14:42:34.411 | INFO | "malloc" : " 14 GB", | 20260506 14:42:34.411 | INFO | "pinned" : " 29 GB" | 20260506 14:42:34.411 | INFO | } | 20260506 14:42:34.411 | INFO | Perf report for the last 5.13845 seconds: | 20260506 14:42:34.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:42:34.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:34.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:34.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:34.411 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:42:34.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:34.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:34.411 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:42:34.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:34.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:34.411 | INFO | Uploaded 36GiB to GPU in 2.00597s (17.9465GiB/s) | 20260506 14:42:34.411 | INFO | Downloaded 34GiB to GPU in 2.0055s (16.9534GiB/s) | 20260506 14:42:39.550 | INFO | Analyzing chunk frames = [2158080,2158592) | 20260506 14:42:39.550 | INFO | MemInfo: { | 20260506 14:42:39.550 | INFO | "gpu" : " 17 GB", | 20260506 14:42:39.550 | INFO | "huge" : "", | 20260506 14:42:39.550 | INFO | "huge_shared" : " 27 GB", | 20260506 14:42:39.550 | INFO | "malloc" : " 14 GB", | 20260506 14:42:39.550 | INFO | "pinned" : " 29 GB" | 20260506 14:42:39.550 | INFO | } | 20260506 14:42:39.550 | INFO | Perf report for the last 5.1389 seconds: | 20260506 14:42:39.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:42:39.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:39.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:39.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:39.550 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:42:39.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:39.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:39.550 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:42:39.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:39.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:39.550 | INFO | Uploaded 36GiB to GPU in 2.00262s (17.9765GiB/s) | 20260506 14:42:39.550 | INFO | Downloaded 34GiB to GPU in 1.98104s (17.1627GiB/s) | 20260506 14:42:39.659 | INFO | Starting a round of Baz IO | 20260506 14:42:42.816 | INFO | Populating RTMetrics report | 20260506 14:42:42.816 | INFO | RTMetrics report populated | 20260506 14:42:42.816 | INFO | Opening tmp RTMetrics json file | 20260506 14:42:42.828 | INFO | Serializing RTMetrics to json | 20260506 14:42:42.840 | INFO | Serialization done, swapping tmp file now... | 20260506 14:42:42.856 | INFO | RTMetrics done | 20260506 14:42:43.189 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.53 seconds | 20260506 14:42:43.190 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.99% / 0.01% / 62.00% | 20260506 14:42:43.190 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.41277% / 2.78155e-05% / 3.97364e-05% | 20260506 14:42:43.190 | INFO | Baz IO overall: | 20260506 14:42:43.190 | INFO | BytesWritten : 1117236643601 | 20260506 14:42:43.190 | INFO | NumEvents : 540658516864 | 20260506 14:42:44.689 | INFO | Analyzing chunk frames = [2158592,2159104) | 20260506 14:42:44.689 | INFO | MemInfo: { | 20260506 14:42:44.689 | INFO | "gpu" : " 17 GB", | 20260506 14:42:44.689 | INFO | "huge" : "", | 20260506 14:42:44.689 | INFO | "huge_shared" : " 27 GB", | 20260506 14:42:44.689 | INFO | "malloc" : " 14 GB", | 20260506 14:42:44.689 | INFO | "pinned" : " 29 GB" | 20260506 14:42:44.689 | INFO | } | 20260506 14:42:44.689 | INFO | Perf report for the last 5.1395 seconds: | 20260506 14:42:44.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:42:44.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:44.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:44.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:44.690 | INFO | ANALYSIS : 63.0%, 2.9, 95.5ms, 0 | 20260506 14:42:44.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:44.690 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 14:42:44.690 | INFO | PRE_HQ : 60.8%, 1.9, 59.1ms, 0 | 20260506 14:42:44.690 | INFO | BAZWRITER : 68.0%, 4.8, 167.3ms, 0 | 20260506 14:42:44.690 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 14:42:44.690 | INFO | Uploaded 36GiB to GPU in 2.06952s (17.3953GiB/s) | 20260506 14:42:44.690 | INFO | Downloaded 38GiB to GPU in 2.23084s (17.034GiB/s) | 20260506 14:42:49.829 | INFO | Analyzing chunk frames = [2159104,2159616) | 20260506 14:42:49.829 | INFO | MemInfo: { | 20260506 14:42:49.829 | INFO | "gpu" : " 17 GB", | 20260506 14:42:49.829 | INFO | "huge" : "", | 20260506 14:42:49.830 | INFO | "huge_shared" : " 27 GB", | 20260506 14:42:49.830 | INFO | "malloc" : " 14 GB", | 20260506 14:42:49.830 | INFO | "pinned" : " 29 GB" | 20260506 14:42:49.830 | INFO | } | 20260506 14:42:49.830 | INFO | Perf report for the last 5.14011 seconds: | 20260506 14:42:49.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:42:49.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:49.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:49.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:49.830 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:42:49.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:49.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:49.830 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:42:49.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:49.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:49.830 | INFO | Uploaded 36GiB to GPU in 2.01737s (17.8451GiB/s) | 20260506 14:42:49.830 | INFO | Downloaded 34GiB to GPU in 1.94551s (17.4761GiB/s) | 20260506 14:42:54.969 | INFO | Analyzing chunk frames = [2159616,2160128) | 20260506 14:42:54.969 | INFO | MemInfo: { | 20260506 14:42:54.969 | INFO | "gpu" : " 17 GB", | 20260506 14:42:54.969 | INFO | "huge" : "", | 20260506 14:42:54.969 | INFO | "huge_shared" : " 27 GB", | 20260506 14:42:54.969 | INFO | "malloc" : " 14 GB", | 20260506 14:42:54.969 | INFO | "pinned" : " 29 GB" | 20260506 14:42:54.969 | INFO | } | 20260506 14:42:54.969 | INFO | Perf report for the last 5.13988 seconds: | 20260506 14:42:54.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:42:54.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:54.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:54.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:54.970 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 14:42:54.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:54.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:54.970 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:42:54.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:54.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:42:54.970 | INFO | Uploaded 36GiB to GPU in 1.98586s (18.1281GiB/s) | 20260506 14:42:54.970 | INFO | Downloaded 34GiB to GPU in 1.93981s (17.5275GiB/s) | 20260506 14:43:00.108 | INFO | Analyzing chunk frames = [2160128,2160640) | 20260506 14:43:00.108 | INFO | MemInfo: { | 20260506 14:43:00.108 | INFO | "gpu" : " 17 GB", | 20260506 14:43:00.108 | INFO | "huge" : "", | 20260506 14:43:00.109 | INFO | "huge_shared" : " 27 GB", | 20260506 14:43:00.109 | INFO | "malloc" : " 14 GB", | 20260506 14:43:00.109 | INFO | "pinned" : " 29 GB" | 20260506 14:43:00.109 | INFO | } | 20260506 14:43:00.109 | INFO | Perf report for the last 5.13917 seconds: | 20260506 14:43:00.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:43:00.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:00.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:00.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:00.109 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:43:00.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:00.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:00.109 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:43:00.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:00.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:00.109 | INFO | Uploaded 36GiB to GPU in 1.97805s (18.1998GiB/s) | 20260506 14:43:00.109 | INFO | Downloaded 34GiB to GPU in 1.91308s (17.7724GiB/s) | 20260506 14:43:05.247 | INFO | Analyzing chunk frames = [2160640,2161152) | 20260506 14:43:05.247 | INFO | MemInfo: { | 20260506 14:43:05.247 | INFO | "gpu" : " 17 GB", | 20260506 14:43:05.247 | INFO | "huge" : "", | 20260506 14:43:05.247 | INFO | "huge_shared" : " 27 GB", | 20260506 14:43:05.247 | INFO | "malloc" : " 14 GB", | 20260506 14:43:05.247 | INFO | "pinned" : " 29 GB" | 20260506 14:43:05.248 | INFO | } | 20260506 14:43:05.248 | INFO | Perf report for the last 5.13893 seconds: | 20260506 14:43:05.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:43:05.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:05.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:05.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:05.248 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 14:43:05.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:05.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:05.248 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:43:05.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:05.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:05.248 | INFO | Uploaded 36GiB to GPU in 2.00133s (17.988GiB/s) | 20260506 14:43:05.248 | INFO | Downloaded 34GiB to GPU in 1.94387s (17.4909GiB/s) | 20260506 14:43:10.386 | INFO | Analyzing chunk frames = [2161152,2161664) | 20260506 14:43:10.386 | INFO | MemInfo: { | 20260506 14:43:10.386 | INFO | "gpu" : " 17 GB", | 20260506 14:43:10.386 | INFO | "huge" : "", | 20260506 14:43:10.386 | INFO | "huge_shared" : " 27 GB", | 20260506 14:43:10.386 | INFO | "malloc" : " 14 GB", | 20260506 14:43:10.386 | INFO | "pinned" : " 29 GB" | 20260506 14:43:10.386 | INFO | } | 20260506 14:43:10.386 | INFO | Perf report for the last 5.13874 seconds: | 20260506 14:43:10.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:43:10.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:10.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:10.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:10.386 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:43:10.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:10.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:10.386 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 14:43:10.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:10.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:10.387 | INFO | Uploaded 36GiB to GPU in 2.07171s (17.3769GiB/s) | 20260506 14:43:10.387 | INFO | Downloaded 34GiB to GPU in 1.98455s (17.1324GiB/s) | 20260506 14:43:15.526 | INFO | Analyzing chunk frames = [2161664,2162176) | 20260506 14:43:15.526 | INFO | MemInfo: { | 20260506 14:43:15.526 | INFO | "gpu" : " 17 GB", | 20260506 14:43:15.526 | INFO | "huge" : "", | 20260506 14:43:15.526 | INFO | "huge_shared" : " 27 GB", | 20260506 14:43:15.526 | INFO | "malloc" : " 14 GB", | 20260506 14:43:15.526 | INFO | "pinned" : " 29 GB" | 20260506 14:43:15.526 | INFO | } | 20260506 14:43:15.526 | INFO | Perf report for the last 5.13958 seconds: | 20260506 14:43:15.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:43:15.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:15.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:15.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:15.526 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 14:43:15.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:15.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:15.526 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 14:43:15.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:15.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:15.526 | INFO | Uploaded 36GiB to GPU in 2.02541s (17.7742GiB/s) | 20260506 14:43:15.526 | INFO | Downloaded 34GiB to GPU in 1.9997s (17.0025GiB/s) | 20260506 14:43:20.665 | INFO | Analyzing chunk frames = [2162176,2162688) | 20260506 14:43:20.665 | INFO | MemInfo: { | 20260506 14:43:20.665 | INFO | "gpu" : " 17 GB", | 20260506 14:43:20.665 | INFO | "huge" : "", | 20260506 14:43:20.665 | INFO | "huge_shared" : " 27 GB", | 20260506 14:43:20.665 | INFO | "malloc" : " 14 GB", | 20260506 14:43:20.666 | INFO | "pinned" : " 29 GB" | 20260506 14:43:20.666 | INFO | } | 20260506 14:43:20.666 | INFO | Perf report for the last 5.13968 seconds: | 20260506 14:43:20.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:43:20.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:20.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:20.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:20.666 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 14:43:20.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:20.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:20.666 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:43:20.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:20.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:20.666 | INFO | Uploaded 36GiB to GPU in 2.02284s (17.7967GiB/s) | 20260506 14:43:20.666 | INFO | Downloaded 34GiB to GPU in 1.97861s (17.1837GiB/s) | 20260506 14:43:20.666 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:43:20.666 | INFO | Total mem usage: 17.4 GiB | 20260506 14:43:20.666 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:43:20.666 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -7.61%) | 20260506 14:43:20.666 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:43:20.666 | INFO | High Water Breakdown: | 20260506 14:43:20.666 | INFO | 61.7% : 9.99 GiB ( 91.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:43:20.666 | INFO | 38.1% : 6.17 GiB ( 92.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:43:20.666 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:43:20.666 | INFO | ------------------------------------------ | 20260506 14:43:20.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:43:20.666 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:43:20.666 | INFO | Total mem usage: 26.5 GiB | 20260506 14:43:20.666 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:43:20.666 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:43:20.666 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:43:20.666 | INFO | High Water Breakdown: | 20260506 14:43:20.666 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:43:20.666 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:43:20.666 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:43:20.666 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:43:20.666 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:43:20.666 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:43:20.666 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:43:20.666 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:43:20.666 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:43:20.666 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:43:20.667 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:43:20.667 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:43:20.667 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:43:20.667 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:43:20.667 | INFO | ------------------------------------------ | 20260506 14:43:20.667 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:43:20.667 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:43:20.667 | INFO | Total mem usage: 25.6 GiB | 20260506 14:43:20.667 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:43:20.667 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:43:20.667 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:43:20.667 | INFO | High Water Breakdown: | 20260506 14:43:20.667 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:43:20.667 | INFO | ------------------------------------------ | 20260506 14:43:20.667 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:43:20.667 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:43:20.667 | INFO | Total mem usage: 15.7 GiB | 20260506 14:43:20.667 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:43:20.667 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:43:20.667 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 14:43:20.667 | INFO | High Water Breakdown: | 20260506 14:43:20.667 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:43:20.667 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:43:20.667 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:43:20.667 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:43:20.667 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:43:20.667 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:43:20.667 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:43:20.667 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:43:20.667 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:43:20.667 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:43:20.667 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:43:20.667 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:43:20.667 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:43:20.667 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:43:20.667 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:43:20.667 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:43:20.667 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:43:20.667 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:43:20.667 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:43:20.667 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:43:20.667 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:43:20.667 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:43:20.667 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:43:20.667 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:43:20.667 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:43:20.667 | INFO | ------------------------------------------ | 20260506 14:43:20.667 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:43:20.667 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:43:20.667 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:43:20.667 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:43:20.667 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.59e+01, 1.35e+01 / 2.19e+00 | 20260506 14:43:20.667 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.01e-01, 9.80e-03 / 5.09e-03 | 20260506 14:43:20.667 | INFO | PerfReport: 11.58%, Baseline -- 3.64e+00 / 4.44e+00, 5.64e-01 / 2.98e-01 | 20260506 14:43:20.667 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.47e+00, 1.83e-01 / 1.20e-01 | 20260506 14:43:20.667 | INFO | PerfReport: 15.89%, DME -- 4.00e+01 / 4.46e+01, 4.17e+00 / 2.47e+00 | 20260506 14:43:20.667 | INFO | PerfReport: 56.70%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.41e+00 / 1.67e+00 | 20260506 14:43:20.667 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.34e+00 / 2.99e+00, 4.13e-01 / 2.25e-01 | 20260506 14:43:20.667 | INFO | PerfReport: 3.59%, Metrics -- 1.13e+00 / 1.47e+00, 2.87e-01 / 1.44e-01 | 20260506 14:43:20.667 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.95e-01 / 4.46e-01, 6.18e-02 / 3.40e-02 | 20260506 14:43:20.667 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:43:20.667 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:43:20.667 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:43:20.667 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 14:43:20.667 | INFO | PerfReport: 21.51%, Upload -- 2.10e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 14:43:20.667 | INFO | PerfReport: 21.18%, Download -- 2.07e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 14:43:20.667 | INFO | PerfReport: 32.21%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 14:43:20.667 | INFO | PerfReport: 1.52%, LockUpload -- 1.48e+00 / 0.00e+00, 6.05e+00 / 0.00e+00 | 20260506 14:43:20.667 | INFO | PerfReport: 1.18%, LockDownload -- 1.15e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 14:43:20.667 | INFO | PerfReport: 22.39%, LockCompute -- 2.19e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 14:43:20.667 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260506 14:43:20.775 | INFO | Starting a round of Baz IO | 20260506 14:43:23.935 | INFO | Populating RTMetrics report | 20260506 14:43:23.935 | INFO | RTMetrics report populated | 20260506 14:43:23.935 | INFO | Opening tmp RTMetrics json file | 20260506 14:43:24.127 | INFO | Serializing RTMetrics to json | 20260506 14:43:24.146 | INFO | Serialization done, swapping tmp file now... | 20260506 14:43:24.190 | INFO | RTMetrics done | 20260506 14:43:24.377 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.602 seconds | 20260506 14:43:24.377 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.98% / 0.01% / 62.01% | 20260506 14:43:24.377 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.40888% / 2.38419e-05% / 3.97364e-05% | 20260506 14:43:24.377 | INFO | Baz IO overall: | 20260506 14:43:24.377 | INFO | BytesWritten : 1119502658622 | 20260506 14:43:24.377 | INFO | NumEvents : 541756258086 | 20260506 14:43:25.805 | INFO | Analyzing chunk frames = [2162688,2163200) | 20260506 14:43:25.805 | INFO | MemInfo: { | 20260506 14:43:25.805 | INFO | "gpu" : " 17 GB", | 20260506 14:43:25.805 | INFO | "huge" : "", | 20260506 14:43:25.805 | INFO | "huge_shared" : " 27 GB", | 20260506 14:43:25.805 | INFO | "malloc" : " 14 GB", | 20260506 14:43:25.805 | INFO | "pinned" : " 29 GB" | 20260506 14:43:25.805 | INFO | } | 20260506 14:43:25.805 | INFO | Perf report for the last 5.13924 seconds: | 20260506 14:43:25.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:43:25.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:25.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:25.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:25.805 | INFO | ANALYSIS : 63.0%, 2.9, 95.3ms, 0 | 20260506 14:43:25.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:25.805 | INFO | RT_METRICS : 48.8%, 1.0, 25.6ms, 0 | 20260506 14:43:25.805 | INFO | PRE_HQ : 60.9%, 1.9, 59.5ms, 0 | 20260506 14:43:25.805 | INFO | BAZWRITER : 69.4%, 4.8, 170.7ms, 0 | 20260506 14:43:25.805 | INFO | BAZ_SAMPLING : 2.9%, 1.0, 1.5ms, 0 | 20260506 14:43:25.805 | INFO | Uploaded 36GiB to GPU in 2.05676s (17.5033GiB/s) | 20260506 14:43:25.805 | INFO | Downloaded 38GiB to GPU in 2.23774s (16.9815GiB/s) | 20260506 14:43:30.945 | INFO | Analyzing chunk frames = [2163200,2163712) | 20260506 14:43:30.945 | INFO | MemInfo: { | 20260506 14:43:30.945 | INFO | "gpu" : " 17 GB", | 20260506 14:43:30.945 | INFO | "huge" : "", | 20260506 14:43:30.945 | INFO | "huge_shared" : " 27 GB", | 20260506 14:43:30.945 | INFO | "malloc" : " 14 GB", | 20260506 14:43:30.945 | INFO | "pinned" : " 29 GB" | 20260506 14:43:30.945 | INFO | } | 20260506 14:43:30.945 | INFO | Perf report for the last 5.13992 seconds: | 20260506 14:43:30.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:43:30.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:30.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:30.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:30.945 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 14:43:30.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:30.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:30.945 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 14:43:30.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:30.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:30.945 | INFO | Uploaded 36GiB to GPU in 2.00032s (17.9972GiB/s) | 20260506 14:43:30.945 | INFO | Downloaded 34GiB to GPU in 1.94136s (17.5135GiB/s) | 20260506 14:43:36.083 | INFO | Analyzing chunk frames = [2163712,2164224) | 20260506 14:43:36.083 | INFO | MemInfo: { | 20260506 14:43:36.083 | INFO | "gpu" : " 17 GB", | 20260506 14:43:36.083 | INFO | "huge" : "", | 20260506 14:43:36.083 | INFO | "huge_shared" : " 27 GB", | 20260506 14:43:36.083 | INFO | "malloc" : " 14 GB", | 20260506 14:43:36.083 | INFO | "pinned" : " 29 GB" | 20260506 14:43:36.083 | INFO | } | 20260506 14:43:36.083 | INFO | Perf report for the last 5.1385 seconds: | 20260506 14:43:36.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:43:36.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:36.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:36.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:36.083 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 14:43:36.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:36.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:36.083 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:43:36.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:36.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:36.084 | INFO | Uploaded 36GiB to GPU in 1.96164s (18.352GiB/s) | 20260506 14:43:36.084 | INFO | Downloaded 34GiB to GPU in 1.94244s (17.5037GiB/s) | 20260506 14:43:41.222 | INFO | Analyzing chunk frames = [2164224,2164736) | 20260506 14:43:41.222 | INFO | MemInfo: { | 20260506 14:43:41.222 | INFO | "gpu" : " 17 GB", | 20260506 14:43:41.222 | INFO | "huge" : "", | 20260506 14:43:41.222 | INFO | "huge_shared" : " 27 GB", | 20260506 14:43:41.222 | INFO | "malloc" : " 14 GB", | 20260506 14:43:41.222 | INFO | "pinned" : " 29 GB" | 20260506 14:43:41.222 | INFO | } | 20260506 14:43:41.222 | INFO | Perf report for the last 5.13891 seconds: | 20260506 14:43:41.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:43:41.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:41.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:41.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:41.222 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:43:41.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:41.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:41.222 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:43:41.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:41.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:41.223 | INFO | Uploaded 36GiB to GPU in 2.00409s (17.9632GiB/s) | 20260506 14:43:41.223 | INFO | Downloaded 34GiB to GPU in 1.91624s (17.743GiB/s) | 20260506 14:43:46.361 | INFO | Analyzing chunk frames = [2164736,2165248) | 20260506 14:43:46.361 | INFO | MemInfo: { | 20260506 14:43:46.361 | INFO | "gpu" : " 17 GB", | 20260506 14:43:46.361 | INFO | "huge" : "", | 20260506 14:43:46.361 | INFO | "huge_shared" : " 27 GB", | 20260506 14:43:46.361 | INFO | "malloc" : " 14 GB", | 20260506 14:43:46.361 | INFO | "pinned" : " 29 GB" | 20260506 14:43:46.361 | INFO | } | 20260506 14:43:46.361 | INFO | Perf report for the last 5.13913 seconds: | 20260506 14:43:46.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:43:46.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:46.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:46.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:46.362 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 14:43:46.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:46.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:46.362 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:43:46.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:46.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:46.362 | INFO | Uploaded 36GiB to GPU in 1.99312s (18.0621GiB/s) | 20260506 14:43:46.362 | INFO | Downloaded 34GiB to GPU in 1.9439s (17.4906GiB/s) | 20260506 14:43:51.501 | INFO | Analyzing chunk frames = [2165248,2165760) | 20260506 14:43:51.501 | INFO | MemInfo: { | 20260506 14:43:51.501 | INFO | "gpu" : " 17 GB", | 20260506 14:43:51.501 | INFO | "huge" : "", | 20260506 14:43:51.501 | INFO | "huge_shared" : " 27 GB", | 20260506 14:43:51.501 | INFO | "malloc" : " 14 GB", | 20260506 14:43:51.501 | INFO | "pinned" : " 29 GB" | 20260506 14:43:51.501 | INFO | } | 20260506 14:43:51.501 | INFO | Perf report for the last 5.13959 seconds: | 20260506 14:43:51.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:43:51.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:51.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:51.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:51.501 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 14:43:51.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:51.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:51.501 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 14:43:51.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:51.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:51.501 | INFO | Uploaded 36GiB to GPU in 2.01911s (17.8296GiB/s) | 20260506 14:43:51.502 | INFO | Downloaded 34GiB to GPU in 1.91423s (17.7617GiB/s) | 20260506 14:43:56.641 | INFO | Analyzing chunk frames = [2165760,2166272) | 20260506 14:43:56.641 | INFO | MemInfo: { | 20260506 14:43:56.641 | INFO | "gpu" : " 17 GB", | 20260506 14:43:56.641 | INFO | "huge" : "", | 20260506 14:43:56.641 | INFO | "huge_shared" : " 27 GB", | 20260506 14:43:56.641 | INFO | "malloc" : " 14 GB", | 20260506 14:43:56.641 | INFO | "pinned" : " 29 GB" | 20260506 14:43:56.641 | INFO | } | 20260506 14:43:56.641 | INFO | Perf report for the last 5.13982 seconds: | 20260506 14:43:56.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:43:56.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:56.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:56.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:56.641 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:43:56.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:56.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:56.641 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 14:43:56.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:56.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:43:56.641 | INFO | Uploaded 36GiB to GPU in 1.99665s (18.0302GiB/s) | 20260506 14:43:56.641 | INFO | Downloaded 34GiB to GPU in 1.98995s (17.0859GiB/s) | 20260506 14:44:01.780 | INFO | Analyzing chunk frames = [2166272,2166784) | 20260506 14:44:01.780 | INFO | MemInfo: { | 20260506 14:44:01.780 | INFO | "gpu" : " 17 GB", | 20260506 14:44:01.780 | INFO | "huge" : "", | 20260506 14:44:01.780 | INFO | "huge_shared" : " 27 GB", | 20260506 14:44:01.780 | INFO | "malloc" : " 14 GB", | 20260506 14:44:01.780 | INFO | "pinned" : " 29 GB" | 20260506 14:44:01.780 | INFO | } | 20260506 14:44:01.780 | INFO | Perf report for the last 5.13925 seconds: | 20260506 14:44:01.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:44:01.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:01.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:01.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:01.780 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:44:01.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:01.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:01.780 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:44:01.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:01.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:01.781 | INFO | Uploaded 36GiB to GPU in 1.99894s (18.0096GiB/s) | 20260506 14:44:01.781 | INFO | Downloaded 34GiB to GPU in 1.98441s (17.1336GiB/s) | 20260506 14:44:01.894 | INFO | Starting a round of Baz IO | 20260506 14:44:05.053 | INFO | Populating RTMetrics report | 20260506 14:44:05.053 | INFO | RTMetrics report populated | 20260506 14:44:05.053 | INFO | Opening tmp RTMetrics json file | 20260506 14:44:05.070 | INFO | Serializing RTMetrics to json | 20260506 14:44:05.073 | INFO | Serialization done, swapping tmp file now... | 20260506 14:44:05.076 | INFO | RTMetrics done | 20260506 14:44:05.277 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.383 seconds | 20260506 14:44:05.277 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.96% / 0.01% / 62.02% | 20260506 14:44:05.277 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.40283% / 2.38419e-05% / 6.75519e-05% | 20260506 14:44:05.277 | INFO | Baz IO overall: | 20260506 14:44:05.277 | INFO | BytesWritten : 1121768791432 | 20260506 14:44:05.277 | INFO | NumEvents : 542854127973 | 20260506 14:44:06.919 | INFO | Analyzing chunk frames = [2166784,2167296) | 20260506 14:44:06.919 | INFO | MemInfo: { | 20260506 14:44:06.919 | INFO | "gpu" : " 17 GB", | 20260506 14:44:06.919 | INFO | "huge" : "", | 20260506 14:44:06.919 | INFO | "huge_shared" : " 27 GB", | 20260506 14:44:06.919 | INFO | "malloc" : " 14 GB", | 20260506 14:44:06.919 | INFO | "pinned" : " 29 GB" | 20260506 14:44:06.919 | INFO | } | 20260506 14:44:06.919 | INFO | Perf report for the last 5.1385 seconds: | 20260506 14:44:06.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:44:06.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:06.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:06.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:06.919 | INFO | ANALYSIS : 63.1%, 2.9, 95.4ms, 0 | 20260506 14:44:06.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:06.919 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 14:44:06.919 | INFO | PRE_HQ : 61.0%, 1.9, 59.5ms, 0 | 20260506 14:44:06.919 | INFO | BAZWRITER : 65.2%, 4.7, 158.2ms, 0 | 20260506 14:44:06.919 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 14:44:06.919 | INFO | Uploaded 36GiB to GPU in 2.04636s (17.5922GiB/s) | 20260506 14:44:06.919 | INFO | Downloaded 38GiB to GPU in 2.22224s (17.0998GiB/s) | 20260506 14:44:12.058 | INFO | Analyzing chunk frames = [2167296,2167808) | 20260506 14:44:12.058 | INFO | MemInfo: { | 20260506 14:44:12.058 | INFO | "gpu" : " 17 GB", | 20260506 14:44:12.058 | INFO | "huge" : "", | 20260506 14:44:12.058 | INFO | "huge_shared" : " 27 GB", | 20260506 14:44:12.058 | INFO | "malloc" : " 14 GB", | 20260506 14:44:12.059 | INFO | "pinned" : " 29 GB" | 20260506 14:44:12.059 | INFO | } | 20260506 14:44:12.059 | INFO | Perf report for the last 5.13994 seconds: | 20260506 14:44:12.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:44:12.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:12.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:12.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:12.059 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 14:44:12.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:12.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:12.059 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 14:44:12.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:12.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:12.059 | INFO | Uploaded 36GiB to GPU in 1.97972s (18.1844GiB/s) | 20260506 14:44:12.059 | INFO | Downloaded 34GiB to GPU in 1.95359s (17.4039GiB/s) | 20260506 14:44:17.198 | INFO | Analyzing chunk frames = [2167808,2168320) | 20260506 14:44:17.198 | INFO | MemInfo: { | 20260506 14:44:17.198 | INFO | "gpu" : " 17 GB", | 20260506 14:44:17.198 | INFO | "huge" : "", | 20260506 14:44:17.198 | INFO | "huge_shared" : " 27 GB", | 20260506 14:44:17.198 | INFO | "malloc" : " 14 GB", | 20260506 14:44:17.198 | INFO | "pinned" : " 29 GB" | 20260506 14:44:17.198 | INFO | } | 20260506 14:44:17.198 | INFO | Perf report for the last 5.13979 seconds: | 20260506 14:44:17.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:44:17.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:17.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:17.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:17.198 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 14:44:17.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:17.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:17.198 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:44:17.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:17.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:17.199 | INFO | Uploaded 36GiB to GPU in 1.98095s (18.1731GiB/s) | 20260506 14:44:17.199 | INFO | Downloaded 34GiB to GPU in 1.94675s (17.465GiB/s) | 20260506 14:44:22.337 | INFO | Analyzing chunk frames = [2168320,2168832) | 20260506 14:44:22.337 | INFO | MemInfo: { | 20260506 14:44:22.337 | INFO | "gpu" : " 17 GB", | 20260506 14:44:22.337 | INFO | "huge" : "", | 20260506 14:44:22.337 | INFO | "huge_shared" : " 27 GB", | 20260506 14:44:22.337 | INFO | "malloc" : " 14 GB", | 20260506 14:44:22.337 | INFO | "pinned" : " 29 GB" | 20260506 14:44:22.337 | INFO | } | 20260506 14:44:22.337 | INFO | Perf report for the last 5.13887 seconds: | 20260506 14:44:22.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:44:22.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:22.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:22.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:22.337 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 14:44:22.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:22.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:22.337 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:44:22.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:22.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:22.338 | INFO | Uploaded 36GiB to GPU in 1.99251s (18.0677GiB/s) | 20260506 14:44:22.338 | INFO | Downloaded 34GiB to GPU in 1.91667s (17.7391GiB/s) | 20260506 14:44:27.476 | INFO | Analyzing chunk frames = [2168832,2169344) | 20260506 14:44:27.476 | INFO | MemInfo: { | 20260506 14:44:27.476 | INFO | "gpu" : " 17 GB", | 20260506 14:44:27.476 | INFO | "huge" : "", | 20260506 14:44:27.476 | INFO | "huge_shared" : " 27 GB", | 20260506 14:44:27.476 | INFO | "malloc" : " 14 GB", | 20260506 14:44:27.476 | INFO | "pinned" : " 29 GB" | 20260506 14:44:27.476 | INFO | } | 20260506 14:44:27.476 | INFO | Perf report for the last 5.13907 seconds: | 20260506 14:44:27.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:44:27.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:27.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:27.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:27.476 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 14:44:27.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:27.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:27.476 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 14:44:27.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:27.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:27.477 | INFO | Uploaded 36GiB to GPU in 1.97571s (18.2213GiB/s) | 20260506 14:44:27.477 | INFO | Downloaded 34GiB to GPU in 1.93029s (17.614GiB/s) | 20260506 14:44:32.616 | INFO | Analyzing chunk frames = [2169344,2169856) | 20260506 14:44:32.616 | INFO | MemInfo: { | 20260506 14:44:32.616 | INFO | "gpu" : " 17 GB", | 20260506 14:44:32.616 | INFO | "huge" : "", | 20260506 14:44:32.616 | INFO | "huge_shared" : " 27 GB", | 20260506 14:44:32.616 | INFO | "malloc" : " 14 GB", | 20260506 14:44:32.616 | INFO | "pinned" : " 29 GB" | 20260506 14:44:32.616 | INFO | } | 20260506 14:44:32.616 | INFO | Perf report for the last 5.13955 seconds: | 20260506 14:44:32.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:44:32.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:32.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:32.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:32.616 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:44:32.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:32.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:32.616 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 14:44:32.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:32.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:32.616 | INFO | Uploaded 36GiB to GPU in 2.07801s (17.3243GiB/s) | 20260506 14:44:32.616 | INFO | Downloaded 34GiB to GPU in 1.98234s (17.1514GiB/s) | 20260506 14:44:37.757 | INFO | Analyzing chunk frames = [2169856,2170368) | 20260506 14:44:37.757 | INFO | MemInfo: { | 20260506 14:44:37.757 | INFO | "gpu" : " 17 GB", | 20260506 14:44:37.757 | INFO | "huge" : "", | 20260506 14:44:37.757 | INFO | "huge_shared" : " 27 GB", | 20260506 14:44:37.757 | INFO | "malloc" : " 14 GB", | 20260506 14:44:37.757 | INFO | "pinned" : " 29 GB" | 20260506 14:44:37.757 | INFO | } | 20260506 14:44:37.757 | INFO | Perf report for the last 5.14126 seconds: | 20260506 14:44:37.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:44:37.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:37.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:37.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:37.757 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:44:37.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:37.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:37.757 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 14:44:37.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:37.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:37.757 | INFO | Uploaded 36GiB to GPU in 2.02055s (17.8169GiB/s) | 20260506 14:44:37.758 | INFO | Downloaded 34GiB to GPU in 2.04256s (16.6458GiB/s) | 20260506 14:44:42.895 | INFO | Analyzing chunk frames = [2170368,2170880) | 20260506 14:44:42.895 | INFO | MemInfo: { | 20260506 14:44:42.895 | INFO | "gpu" : " 17 GB", | 20260506 14:44:42.895 | INFO | "huge" : "", | 20260506 14:44:42.895 | INFO | "huge_shared" : " 27 GB", | 20260506 14:44:42.895 | INFO | "malloc" : " 14 GB", | 20260506 14:44:42.895 | INFO | "pinned" : " 29 GB" | 20260506 14:44:42.895 | INFO | } | 20260506 14:44:42.895 | INFO | Perf report for the last 5.13827 seconds: | 20260506 14:44:42.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:44:42.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:42.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:42.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:42.896 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 14:44:42.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:42.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:42.896 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:44:42.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:42.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:42.896 | INFO | Uploaded 36GiB to GPU in 2.06113s (17.4662GiB/s) | 20260506 14:44:42.896 | INFO | Downloaded 34GiB to GPU in 1.98059s (17.1666GiB/s) | 20260506 14:44:42.896 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:44:42.896 | INFO | Total mem usage: 17.4 GiB | 20260506 14:44:42.896 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:44:42.896 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -11.4%) | 20260506 14:44:42.896 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:44:42.896 | INFO | High Water Breakdown: | 20260506 14:44:42.896 | INFO | 61.2% : 9.57 GiB ( 87.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:44:42.896 | INFO | 38.7% : 6.04 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:44:42.896 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:44:42.896 | INFO | ------------------------------------------ | 20260506 14:44:42.896 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:44:42.896 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:44:42.896 | INFO | Total mem usage: 26.5 GiB | 20260506 14:44:42.896 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:44:42.896 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:44:42.896 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:44:42.896 | INFO | High Water Breakdown: | 20260506 14:44:42.896 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:44:42.896 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:44:42.896 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:44:42.896 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:44:42.896 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:44:42.896 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:44:42.896 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:44:42.896 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:44:42.896 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:44:42.896 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:44:42.896 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:44:42.896 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:44:42.896 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:44:42.896 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:44:42.896 | INFO | ------------------------------------------ | 20260506 14:44:42.896 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:44:42.896 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:44:42.896 | INFO | Total mem usage: 25.6 GiB | 20260506 14:44:42.896 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:44:42.896 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:44:42.896 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:44:42.896 | INFO | High Water Breakdown: | 20260506 14:44:42.896 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:44:42.896 | INFO | ------------------------------------------ | 20260506 14:44:42.896 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:44:42.897 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:44:42.897 | INFO | Total mem usage: 15.7 GiB | 20260506 14:44:42.897 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:44:42.897 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:44:42.897 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 14:44:42.897 | INFO | High Water Breakdown: | 20260506 14:44:42.897 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:44:42.897 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:44:42.897 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:44:42.897 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:44:42.897 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:44:42.897 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:44:42.897 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:44:42.897 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:44:42.897 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:44:42.897 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:44:42.897 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:44:42.897 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:44:42.897 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:44:42.897 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:44:42.897 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:44:42.897 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:44:42.897 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:44:42.897 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:44:42.897 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:44:42.897 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:44:42.897 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:44:42.897 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:44:42.897 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:44:42.897 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:44:42.897 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:44:42.897 | INFO | ------------------------------------------ | 20260506 14:44:42.897 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:44:42.897 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:44:42.897 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:44:42.897 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:44:42.897 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.58e+01, 1.36e+01 / 2.04e+00 | 20260506 14:44:42.897 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.02e-01, 1.03e-02 / 6.59e-03 | 20260506 14:44:42.897 | INFO | PerfReport: 11.58%, Baseline -- 3.65e+00 / 4.40e+00, 5.73e-01 / 4.22e-01 | 20260506 14:44:42.897 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.45e+00, 1.87e-01 / 1.54e-01 | 20260506 14:44:42.897 | INFO | PerfReport: 15.81%, DME -- 3.99e+01 / 4.38e+01, 3.80e+00 / 2.02e+00 | 20260506 14:44:42.897 | INFO | PerfReport: 56.80%, FrameLabeling -- 1.79e+01 / 2.11e+01, 1.50e+00 / 1.68e+00 | 20260506 14:44:42.897 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.34e+00 / 2.90e+00, 4.03e-01 / 2.72e-01 | 20260506 14:44:42.897 | INFO | PerfReport: 3.59%, Metrics -- 1.13e+00 / 1.49e+00, 2.91e-01 / 2.06e-01 | 20260506 14:44:42.897 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.95e-01 / 4.46e-01, 6.15e-02 / 2.85e-02 | 20260506 14:44:42.897 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:44:42.897 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:44:42.897 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:44:42.897 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 14:44:42.897 | INFO | PerfReport: 21.45%, Upload -- 2.09e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 14:44:42.897 | INFO | PerfReport: 21.24%, Download -- 2.07e+01 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260506 14:44:42.897 | INFO | PerfReport: 32.28%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 14:44:42.897 | INFO | PerfReport: 1.41%, LockUpload -- 1.38e+00 / 0.00e+00, 5.79e+00 / 0.00e+00 | 20260506 14:44:42.897 | INFO | PerfReport: 1.20%, LockDownload -- 1.17e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260506 14:44:42.897 | INFO | PerfReport: 22.40%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:44:42.897 | INFO | PerfReport: 0.02%, Misc -- 1.56e-02 / 0.00e+00, 3.01e-02 / 0.00e+00 | 20260506 14:44:43.018 | INFO | Starting a round of Baz IO | 20260506 14:44:46.169 | INFO | Populating RTMetrics report | 20260506 14:44:46.169 | INFO | RTMetrics report populated | 20260506 14:44:46.169 | INFO | Opening tmp RTMetrics json file | 20260506 14:44:46.192 | INFO | Serializing RTMetrics to json | 20260506 14:44:46.195 | INFO | Serialization done, swapping tmp file now... | 20260506 14:44:46.196 | INFO | RTMetrics done | 20260506 14:44:46.691 | INFO | Finished a round of Baz IO: Wrote 2.111 GiB in 3.673 seconds | 20260506 14:44:46.691 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.95% / 0.01% / 62.04% | 20260506 14:44:46.692 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.39798% / 1.19209e-05% / 5.96046e-05% | 20260506 14:44:46.692 | INFO | Baz IO overall: | 20260506 14:44:46.692 | INFO | BytesWritten : 1124035484642 | 20260506 14:44:46.692 | INFO | NumEvents : 543952547602 | 20260506 14:44:48.034 | INFO | Analyzing chunk frames = [2170880,2171392) | 20260506 14:44:48.034 | INFO | MemInfo: { | 20260506 14:44:48.034 | INFO | "gpu" : " 17 GB", | 20260506 14:44:48.034 | INFO | "huge" : "", | 20260506 14:44:48.034 | INFO | "huge_shared" : " 27 GB", | 20260506 14:44:48.034 | INFO | "malloc" : " 14 GB", | 20260506 14:44:48.034 | INFO | "pinned" : " 29 GB" | 20260506 14:44:48.034 | INFO | } | 20260506 14:44:48.034 | INFO | Perf report for the last 5.13829 seconds: | 20260506 14:44:48.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:44:48.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:48.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:48.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:48.034 | INFO | ANALYSIS : 63.1%, 2.9, 95.2ms, 0 | 20260506 14:44:48.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:48.034 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 14:44:48.034 | INFO | PRE_HQ : 61.0%, 1.9, 59.8ms, 0 | 20260506 14:44:48.034 | INFO | BAZWRITER : 70.9%, 4.8, 175.9ms, 0 | 20260506 14:44:48.034 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 14:44:48.034 | INFO | Uploaded 36GiB to GPU in 2.06024s (17.4737GiB/s) | 20260506 14:44:48.034 | INFO | Downloaded 38GiB to GPU in 2.23004s (17.04GiB/s) | 20260506 14:44:53.172 | INFO | Analyzing chunk frames = [2171392,2171904) | 20260506 14:44:53.172 | INFO | MemInfo: { | 20260506 14:44:53.172 | INFO | "gpu" : " 17 GB", | 20260506 14:44:53.172 | INFO | "huge" : "", | 20260506 14:44:53.173 | INFO | "huge_shared" : " 27 GB", | 20260506 14:44:53.173 | INFO | "malloc" : " 14 GB", | 20260506 14:44:53.173 | INFO | "pinned" : " 29 GB" | 20260506 14:44:53.173 | INFO | } | 20260506 14:44:53.173 | INFO | Perf report for the last 5.13893 seconds: | 20260506 14:44:53.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:44:53.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:53.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:53.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:53.173 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 14:44:53.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:53.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:53.173 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:44:53.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:53.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:53.173 | INFO | Uploaded 36GiB to GPU in 2.00192s (17.9828GiB/s) | 20260506 14:44:53.173 | INFO | Downloaded 34GiB to GPU in 1.94412s (17.4887GiB/s) | 20260506 14:44:58.312 | INFO | Analyzing chunk frames = [2171904,2172416) | 20260506 14:44:58.312 | INFO | MemInfo: { | 20260506 14:44:58.312 | INFO | "gpu" : " 17 GB", | 20260506 14:44:58.312 | INFO | "huge" : "", | 20260506 14:44:58.312 | INFO | "huge_shared" : " 27 GB", | 20260506 14:44:58.312 | INFO | "malloc" : " 14 GB", | 20260506 14:44:58.312 | INFO | "pinned" : " 29 GB" | 20260506 14:44:58.312 | INFO | } | 20260506 14:44:58.312 | INFO | Perf report for the last 5.13964 seconds: | 20260506 14:44:58.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:44:58.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:58.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:58.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:58.312 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 14:44:58.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:58.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:58.312 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:44:58.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:58.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:44:58.313 | INFO | Uploaded 36GiB to GPU in 2.00617s (17.9447GiB/s) | 20260506 14:44:58.313 | INFO | Downloaded 34GiB to GPU in 1.93032s (17.6137GiB/s) | 20260506 14:45:03.452 | INFO | Analyzing chunk frames = [2172416,2172928) | 20260506 14:45:03.452 | INFO | MemInfo: { | 20260506 14:45:03.452 | INFO | "gpu" : " 17 GB", | 20260506 14:45:03.452 | INFO | "huge" : "", | 20260506 14:45:03.452 | INFO | "huge_shared" : " 27 GB", | 20260506 14:45:03.452 | INFO | "malloc" : " 14 GB", | 20260506 14:45:03.453 | INFO | "pinned" : " 29 GB" | 20260506 14:45:03.453 | INFO | } | 20260506 14:45:03.453 | INFO | Perf report for the last 5.14019 seconds: | 20260506 14:45:03.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:45:03.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:03.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:03.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:03.453 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:45:03.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:03.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:03.453 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:45:03.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:03.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:03.453 | INFO | Uploaded 36GiB to GPU in 1.98492s (18.1368GiB/s) | 20260506 14:45:03.453 | INFO | Downloaded 34GiB to GPU in 1.89905s (17.9037GiB/s) | 20260506 14:45:08.592 | INFO | Analyzing chunk frames = [2172928,2173440) | 20260506 14:45:08.592 | INFO | MemInfo: { | 20260506 14:45:08.592 | INFO | "gpu" : " 17 GB", | 20260506 14:45:08.592 | INFO | "huge" : "", | 20260506 14:45:08.592 | INFO | "huge_shared" : " 27 GB", | 20260506 14:45:08.592 | INFO | "malloc" : " 14 GB", | 20260506 14:45:08.592 | INFO | "pinned" : " 29 GB" | 20260506 14:45:08.592 | INFO | } | 20260506 14:45:08.592 | INFO | Perf report for the last 5.13953 seconds: | 20260506 14:45:08.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:45:08.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:08.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:08.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:08.592 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 14:45:08.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:08.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:08.592 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:45:08.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:08.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:08.592 | INFO | Uploaded 36GiB to GPU in 2.03973s (17.6494GiB/s) | 20260506 14:45:08.592 | INFO | Downloaded 34GiB to GPU in 1.93003s (17.6163GiB/s) | 20260506 14:45:13.730 | INFO | Analyzing chunk frames = [2173440,2173952) | 20260506 14:45:13.730 | INFO | MemInfo: { | 20260506 14:45:13.730 | INFO | "gpu" : " 17 GB", | 20260506 14:45:13.730 | INFO | "huge" : "", | 20260506 14:45:13.730 | INFO | "huge_shared" : " 27 GB", | 20260506 14:45:13.730 | INFO | "malloc" : " 14 GB", | 20260506 14:45:13.730 | INFO | "pinned" : " 29 GB" | 20260506 14:45:13.730 | INFO | } | 20260506 14:45:13.730 | INFO | Perf report for the last 5.13804 seconds: | 20260506 14:45:13.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:45:13.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:13.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:13.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:13.730 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:45:13.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:13.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:13.730 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 14:45:13.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:13.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:13.730 | INFO | Uploaded 36GiB to GPU in 2.08175s (17.2931GiB/s) | 20260506 14:45:13.730 | INFO | Downloaded 34GiB to GPU in 1.97937s (17.1772GiB/s) | 20260506 14:45:18.870 | INFO | Analyzing chunk frames = [2173952,2174464) | 20260506 14:45:18.870 | INFO | MemInfo: { | 20260506 14:45:18.870 | INFO | "gpu" : " 17 GB", | 20260506 14:45:18.870 | INFO | "huge" : "", | 20260506 14:45:18.870 | INFO | "huge_shared" : " 27 GB", | 20260506 14:45:18.870 | INFO | "malloc" : " 14 GB", | 20260506 14:45:18.870 | INFO | "pinned" : " 29 GB" | 20260506 14:45:18.870 | INFO | } | 20260506 14:45:18.870 | INFO | Perf report for the last 5.14009 seconds: | 20260506 14:45:18.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:45:18.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:18.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:18.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:18.870 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:45:18.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:18.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:18.870 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 14:45:18.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:18.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:18.871 | INFO | Uploaded 36GiB to GPU in 2.01141s (17.8979GiB/s) | 20260506 14:45:18.871 | INFO | Downloaded 34GiB to GPU in 1.97715s (17.1965GiB/s) | 20260506 14:45:24.009 | INFO | Analyzing chunk frames = [2174464,2174976) | 20260506 14:45:24.010 | INFO | MemInfo: { | 20260506 14:45:24.010 | INFO | "gpu" : " 17 GB", | 20260506 14:45:24.010 | INFO | "huge" : "", | 20260506 14:45:24.010 | INFO | "huge_shared" : " 27 GB", | 20260506 14:45:24.010 | INFO | "malloc" : " 14 GB", | 20260506 14:45:24.010 | INFO | "pinned" : " 29 GB" | 20260506 14:45:24.010 | INFO | } | 20260506 14:45:24.010 | INFO | Perf report for the last 5.1395 seconds: | 20260506 14:45:24.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:45:24.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:24.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:24.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:24.010 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:45:24.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:24.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:24.010 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260506 14:45:24.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:24.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:24.010 | INFO | Uploaded 36GiB to GPU in 2.03726s (17.6708GiB/s) | 20260506 14:45:24.010 | INFO | Downloaded 34GiB to GPU in 1.96962s (17.2622GiB/s) | 20260506 14:45:24.128 | INFO | Starting a round of Baz IO | 20260506 14:45:27.291 | INFO | Populating RTMetrics report | 20260506 14:45:27.291 | INFO | RTMetrics report populated | 20260506 14:45:27.291 | INFO | Opening tmp RTMetrics json file | 20260506 14:45:27.292 | INFO | Serializing RTMetrics to json | 20260506 14:45:27.293 | INFO | Serialization done, swapping tmp file now... | 20260506 14:45:27.294 | INFO | RTMetrics done | 20260506 14:45:27.422 | INFO | Finished a round of Baz IO: Wrote 2.114 GiB in 3.293 seconds | 20260506 14:45:27.422 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.94% / 0.01% / 62.05% | 20260506 14:45:27.422 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.40232% / 1.58946e-05% / 7.15256e-05% | 20260506 14:45:27.422 | INFO | Baz IO overall: | 20260506 14:45:27.422 | INFO | BytesWritten : 1126305042105 | 20260506 14:45:27.422 | INFO | NumEvents : 545053350013 | 20260506 14:45:29.149 | INFO | Analyzing chunk frames = [2174976,2175488) | 20260506 14:45:29.149 | INFO | MemInfo: { | 20260506 14:45:29.149 | INFO | "gpu" : " 17 GB", | 20260506 14:45:29.149 | INFO | "huge" : "", | 20260506 14:45:29.149 | INFO | "huge_shared" : " 27 GB", | 20260506 14:45:29.149 | INFO | "malloc" : " 14 GB", | 20260506 14:45:29.149 | INFO | "pinned" : " 29 GB" | 20260506 14:45:29.149 | INFO | } | 20260506 14:45:29.149 | INFO | Perf report for the last 5.13943 seconds: | 20260506 14:45:29.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:45:29.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:29.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:29.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:29.149 | INFO | ANALYSIS : 63.1%, 2.9, 95.5ms, 0 | 20260506 14:45:29.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:29.149 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 14:45:29.149 | INFO | PRE_HQ : 60.8%, 1.9, 59.5ms, 0 | 20260506 14:45:29.149 | INFO | BAZWRITER : 63.5%, 4.6, 151.6ms, 0 | 20260506 14:45:29.149 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 14:45:29.149 | INFO | Uploaded 36GiB to GPU in 2.05871s (17.4866GiB/s) | 20260506 14:45:29.149 | INFO | Downloaded 38GiB to GPU in 2.23012s (17.0394GiB/s) | 20260506 14:45:34.289 | INFO | Analyzing chunk frames = [2175488,2176000) | 20260506 14:45:34.289 | INFO | MemInfo: { | 20260506 14:45:34.289 | INFO | "gpu" : " 17 GB", | 20260506 14:45:34.289 | INFO | "huge" : "", | 20260506 14:45:34.289 | INFO | "huge_shared" : " 27 GB", | 20260506 14:45:34.289 | INFO | "malloc" : " 14 GB", | 20260506 14:45:34.289 | INFO | "pinned" : " 29 GB" | 20260506 14:45:34.289 | INFO | } | 20260506 14:45:34.289 | INFO | Perf report for the last 5.13969 seconds: | 20260506 14:45:34.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:45:34.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:34.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:34.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:34.289 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 14:45:34.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:34.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:34.289 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 14:45:34.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:34.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:34.289 | INFO | Uploaded 36GiB to GPU in 1.99278s (18.0652GiB/s) | 20260506 14:45:34.289 | INFO | Downloaded 34GiB to GPU in 1.93361s (17.5837GiB/s) | 20260506 14:45:39.427 | INFO | Analyzing chunk frames = [2176000,2176512) | 20260506 14:45:39.427 | INFO | MemInfo: { | 20260506 14:45:39.427 | INFO | "gpu" : " 17 GB", | 20260506 14:45:39.427 | INFO | "huge" : "", | 20260506 14:45:39.427 | INFO | "huge_shared" : " 27 GB", | 20260506 14:45:39.427 | INFO | "malloc" : " 14 GB", | 20260506 14:45:39.427 | INFO | "pinned" : " 29 GB" | 20260506 14:45:39.427 | INFO | } | 20260506 14:45:39.427 | INFO | Perf report for the last 5.13852 seconds: | 20260506 14:45:39.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:45:39.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:39.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:39.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:39.427 | INFO | ANALYSIS : 61.3%, 3.0, 94.8ms, 0 | 20260506 14:45:39.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:39.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:39.427 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:45:39.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:39.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:39.428 | INFO | Uploaded 36GiB to GPU in 2.00768s (17.9312GiB/s) | 20260506 14:45:39.428 | INFO | Downloaded 34GiB to GPU in 1.94617s (17.4702GiB/s) | 20260506 14:45:44.566 | INFO | Analyzing chunk frames = [2176512,2177024) | 20260506 14:45:44.566 | INFO | MemInfo: { | 20260506 14:45:44.566 | INFO | "gpu" : " 17 GB", | 20260506 14:45:44.566 | INFO | "huge" : "", | 20260506 14:45:44.566 | INFO | "huge_shared" : " 27 GB", | 20260506 14:45:44.566 | INFO | "malloc" : " 14 GB", | 20260506 14:45:44.566 | INFO | "pinned" : " 29 GB" | 20260506 14:45:44.566 | INFO | } | 20260506 14:45:44.566 | INFO | Perf report for the last 5.13837 seconds: | 20260506 14:45:44.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:45:44.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:44.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:44.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:44.566 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:45:44.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:44.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:44.566 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:45:44.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:44.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:44.566 | INFO | Uploaded 36GiB to GPU in 1.9705s (18.2695GiB/s) | 20260506 14:45:44.566 | INFO | Downloaded 34GiB to GPU in 1.91083s (17.7933GiB/s) | 20260506 14:45:49.706 | INFO | Analyzing chunk frames = [2177024,2177536) | 20260506 14:45:49.706 | INFO | MemInfo: { | 20260506 14:45:49.706 | INFO | "gpu" : " 17 GB", | 20260506 14:45:49.706 | INFO | "huge" : "", | 20260506 14:45:49.706 | INFO | "huge_shared" : " 27 GB", | 20260506 14:45:49.706 | INFO | "malloc" : " 14 GB", | 20260506 14:45:49.706 | INFO | "pinned" : " 29 GB" | 20260506 14:45:49.706 | INFO | } | 20260506 14:45:49.706 | INFO | Perf report for the last 5.14016 seconds: | 20260506 14:45:49.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:45:49.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:49.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:49.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:49.706 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 14:45:49.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:49.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:49.706 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:45:49.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:49.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:49.706 | INFO | Uploaded 36GiB to GPU in 2.0045s (17.9596GiB/s) | 20260506 14:45:49.706 | INFO | Downloaded 34GiB to GPU in 1.93335s (17.5861GiB/s) | 20260506 14:45:54.846 | INFO | Analyzing chunk frames = [2177536,2178048) | 20260506 14:45:54.846 | INFO | MemInfo: { | 20260506 14:45:54.846 | INFO | "gpu" : " 17 GB", | 20260506 14:45:54.846 | INFO | "huge" : "", | 20260506 14:45:54.846 | INFO | "huge_shared" : " 27 GB", | 20260506 14:45:54.846 | INFO | "malloc" : " 14 GB", | 20260506 14:45:54.846 | INFO | "pinned" : " 29 GB" | 20260506 14:45:54.846 | INFO | } | 20260506 14:45:54.846 | INFO | Perf report for the last 5.13997 seconds: | 20260506 14:45:54.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:45:54.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:54.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:54.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:54.846 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 14:45:54.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:54.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:54.846 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:45:54.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:54.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:54.846 | INFO | Uploaded 36GiB to GPU in 2.07654s (17.3366GiB/s) | 20260506 14:45:54.846 | INFO | Downloaded 34GiB to GPU in 1.96572s (17.2964GiB/s) | 20260506 14:45:59.985 | INFO | Analyzing chunk frames = [2178048,2178560) | 20260506 14:45:59.985 | INFO | MemInfo: { | 20260506 14:45:59.985 | INFO | "gpu" : " 17 GB", | 20260506 14:45:59.985 | INFO | "huge" : "", | 20260506 14:45:59.985 | INFO | "huge_shared" : " 27 GB", | 20260506 14:45:59.985 | INFO | "malloc" : " 14 GB", | 20260506 14:45:59.985 | INFO | "pinned" : " 29 GB" | 20260506 14:45:59.985 | INFO | } | 20260506 14:45:59.985 | INFO | Perf report for the last 5.1393 seconds: | 20260506 14:45:59.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:45:59.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:59.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:59.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:59.985 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 14:45:59.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:59.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:59.985 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 14:45:59.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:59.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:45:59.986 | INFO | Uploaded 36GiB to GPU in 2.03252s (17.712GiB/s) | 20260506 14:45:59.986 | INFO | Downloaded 34GiB to GPU in 2.00202s (16.9829GiB/s) | 20260506 14:46:05.124 | INFO | Analyzing chunk frames = [2178560,2179072) | 20260506 14:46:05.124 | INFO | MemInfo: { | 20260506 14:46:05.124 | INFO | "gpu" : " 17 GB", | 20260506 14:46:05.124 | INFO | "huge" : "", | 20260506 14:46:05.124 | INFO | "huge_shared" : " 27 GB", | 20260506 14:46:05.124 | INFO | "malloc" : " 14 GB", | 20260506 14:46:05.124 | INFO | "pinned" : " 29 GB" | 20260506 14:46:05.124 | INFO | } | 20260506 14:46:05.124 | INFO | Perf report for the last 5.13883 seconds: | 20260506 14:46:05.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:46:05.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:05.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:05.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:05.124 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 14:46:05.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:05.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:05.124 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:46:05.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:05.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:05.124 | INFO | Uploaded 36GiB to GPU in 2.03516s (17.689GiB/s) | 20260506 14:46:05.124 | INFO | Downloaded 34GiB to GPU in 1.97067s (17.253GiB/s) | 20260506 14:46:05.125 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:46:05.125 | INFO | Total mem usage: 17.4 GiB | 20260506 14:46:05.125 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:46:05.125 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -8.37%) | 20260506 14:46:05.125 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:46:05.125 | INFO | High Water Breakdown: | 20260506 14:46:05.125 | INFO | 61.6% : 9.9 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:46:05.125 | INFO | 38.2% : 6.14 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:46:05.125 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:46:05.125 | INFO | ------------------------------------------ | 20260506 14:46:05.125 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:46:05.125 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:46:05.125 | INFO | Total mem usage: 26.5 GiB | 20260506 14:46:05.125 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:46:05.125 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:46:05.125 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:46:05.125 | INFO | High Water Breakdown: | 20260506 14:46:05.125 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:46:05.125 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:46:05.125 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:46:05.125 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:46:05.125 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:46:05.125 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:46:05.125 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:46:05.125 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:46:05.125 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:46:05.125 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:46:05.125 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:46:05.125 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:46:05.125 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:46:05.125 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:46:05.125 | INFO | ------------------------------------------ | 20260506 14:46:05.125 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:46:05.125 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:46:05.125 | INFO | Total mem usage: 25.6 GiB | 20260506 14:46:05.125 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:46:05.125 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:46:05.125 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:46:05.125 | INFO | High Water Breakdown: | 20260506 14:46:05.125 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:46:05.125 | INFO | ------------------------------------------ | 20260506 14:46:05.125 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:46:05.125 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:46:05.125 | INFO | Total mem usage: 15.7 GiB | 20260506 14:46:05.125 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:46:05.125 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:46:05.125 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 14:46:05.125 | INFO | High Water Breakdown: | 20260506 14:46:05.125 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:46:05.125 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:46:05.125 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:46:05.125 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:46:05.125 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:46:05.125 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:46:05.125 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:46:05.125 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:46:05.125 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:46:05.125 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:46:05.125 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:46:05.125 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:46:05.125 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:46:05.125 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:46:05.125 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:46:05.125 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:46:05.125 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:46:05.125 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:46:05.125 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:46:05.125 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:46:05.125 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:46:05.125 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:46:05.125 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:46:05.125 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:46:05.125 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:46:05.125 | INFO | ------------------------------------------ | 20260506 14:46:05.125 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:46:05.125 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:46:05.125 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:46:05.125 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:46:05.125 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.58e+01, 1.34e+01 / 2.66e+00 | 20260506 14:46:05.125 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 9.99e-02, 9.98e-03 / 6.22e-03 | 20260506 14:46:05.125 | INFO | PerfReport: 11.58%, Baseline -- 3.65e+00 / 4.43e+00, 5.67e-01 / 3.80e-01 | 20260506 14:46:05.125 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.46e+00, 1.88e-01 / 1.39e-01 | 20260506 14:46:05.125 | INFO | PerfReport: 15.91%, DME -- 4.01e+01 / 4.45e+01, 4.08e+00 / 2.89e+00 | 20260506 14:46:05.125 | INFO | PerfReport: 56.68%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.41e+00 / 1.81e+00 | 20260506 14:46:05.126 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.34e+00 / 2.89e+00, 4.10e-01 / 2.87e-01 | 20260506 14:46:05.126 | INFO | PerfReport: 3.59%, Metrics -- 1.13e+00 / 1.47e+00, 2.92e-01 / 2.12e-01 | 20260506 14:46:05.126 | INFO | PerfReport: 1.26%, PulseSerializing -- 3.96e-01 / 4.46e-01, 6.25e-02 / 3.27e-02 | 20260506 14:46:05.126 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:46:05.126 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:46:05.126 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:46:05.126 | INFO | PerfReport: Full Execution -- 9.78e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 14:46:05.126 | INFO | PerfReport: 21.57%, Upload -- 2.11e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260506 14:46:05.126 | INFO | PerfReport: 21.14%, Download -- 2.07e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 14:46:05.126 | INFO | PerfReport: 32.22%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 14:46:05.126 | INFO | PerfReport: 1.67%, LockUpload -- 1.64e+00 / 0.00e+00, 6.11e+00 / 0.00e+00 | 20260506 14:46:05.126 | INFO | PerfReport: 1.05%, LockDownload -- 1.03e+00 / 0.00e+00, 2.67e+00 / 0.00e+00 | 20260506 14:46:05.126 | INFO | PerfReport: 22.33%, LockCompute -- 2.18e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:46:05.126 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260506 14:46:05.243 | INFO | Starting a round of Baz IO | 20260506 14:46:08.397 | INFO | Populating RTMetrics report | 20260506 14:46:08.397 | INFO | RTMetrics report populated | 20260506 14:46:08.397 | INFO | Opening tmp RTMetrics json file | 20260506 14:46:08.534 | INFO | Serializing RTMetrics to json | 20260506 14:46:08.610 | INFO | Serialization done, swapping tmp file now... | 20260506 14:46:08.648 | INFO | RTMetrics done | 20260506 14:46:08.709 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.466 seconds | 20260506 14:46:08.709 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.92% / 0.01% / 62.06% | 20260506 14:46:08.710 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.38534% / 2.38419e-05% / 5.16574e-05% | 20260506 14:46:08.710 | INFO | Baz IO overall: | 20260506 14:46:08.710 | INFO | BytesWritten : 1128569848786 | 20260506 14:46:08.710 | INFO | NumEvents : 546150425531 | 20260506 14:46:10.264 | INFO | Analyzing chunk frames = [2179072,2179584) | 20260506 14:46:10.264 | INFO | MemInfo: { | 20260506 14:46:10.264 | INFO | "gpu" : " 17 GB", | 20260506 14:46:10.264 | INFO | "huge" : "", | 20260506 14:46:10.264 | INFO | "huge_shared" : " 27 GB", | 20260506 14:46:10.264 | INFO | "malloc" : " 14 GB", | 20260506 14:46:10.264 | INFO | "pinned" : " 29 GB" | 20260506 14:46:10.264 | INFO | } | 20260506 14:46:10.264 | INFO | Perf report for the last 5.1398 seconds: | 20260506 14:46:10.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:46:10.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:10.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:10.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:10.264 | INFO | ANALYSIS : 63.1%, 2.9, 95.3ms, 0 | 20260506 14:46:10.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:10.264 | INFO | RT_METRICS : 49.0%, 1.0, 25.7ms, 0 | 20260506 14:46:10.264 | INFO | PRE_HQ : 61.0%, 1.9, 59.6ms, 0 | 20260506 14:46:10.264 | INFO | BAZWRITER : 66.7%, 4.8, 165.4ms, 0 | 20260506 14:46:10.264 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.5ms, 0 | 20260506 14:46:10.264 | INFO | Uploaded 36GiB to GPU in 2.04959s (17.5645GiB/s) | 20260506 14:46:10.264 | INFO | Downloaded 38GiB to GPU in 2.23441s (17.0067GiB/s) | 20260506 14:46:15.403 | INFO | Analyzing chunk frames = [2179584,2180096) | 20260506 14:46:15.403 | INFO | MemInfo: { | 20260506 14:46:15.403 | INFO | "gpu" : " 17 GB", | 20260506 14:46:15.403 | INFO | "huge" : "", | 20260506 14:46:15.403 | INFO | "huge_shared" : " 27 GB", | 20260506 14:46:15.403 | INFO | "malloc" : " 14 GB", | 20260506 14:46:15.403 | INFO | "pinned" : " 29 GB" | 20260506 14:46:15.403 | INFO | } | 20260506 14:46:15.403 | INFO | Perf report for the last 5.13909 seconds: | 20260506 14:46:15.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:46:15.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:15.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:15.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:15.403 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 14:46:15.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:15.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:15.403 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:46:15.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:15.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:15.404 | INFO | Uploaded 36GiB to GPU in 1.99714s (18.0258GiB/s) | 20260506 14:46:15.404 | INFO | Downloaded 34GiB to GPU in 1.94236s (17.5045GiB/s) | 20260506 14:46:20.542 | INFO | Analyzing chunk frames = [2180096,2180608) | 20260506 14:46:20.542 | INFO | MemInfo: { | 20260506 14:46:20.542 | INFO | "gpu" : " 17 GB", | 20260506 14:46:20.542 | INFO | "huge" : "", | 20260506 14:46:20.542 | INFO | "huge_shared" : " 27 GB", | 20260506 14:46:20.542 | INFO | "malloc" : " 14 GB", | 20260506 14:46:20.542 | INFO | "pinned" : " 29 GB" | 20260506 14:46:20.543 | INFO | } | 20260506 14:46:20.543 | INFO | Perf report for the last 5.1399 seconds: | 20260506 14:46:20.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:46:20.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:20.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:20.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:20.543 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 14:46:20.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:20.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:20.543 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:46:20.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:20.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:20.543 | INFO | Uploaded 36GiB to GPU in 1.96648s (18.3068GiB/s) | 20260506 14:46:20.543 | INFO | Downloaded 34GiB to GPU in 1.94331s (17.4959GiB/s) | 20260506 14:46:25.681 | INFO | Analyzing chunk frames = [2180608,2181120) | 20260506 14:46:25.681 | INFO | MemInfo: { | 20260506 14:46:25.681 | INFO | "gpu" : " 17 GB", | 20260506 14:46:25.681 | INFO | "huge" : "", | 20260506 14:46:25.681 | INFO | "huge_shared" : " 27 GB", | 20260506 14:46:25.681 | INFO | "malloc" : " 14 GB", | 20260506 14:46:25.681 | INFO | "pinned" : " 29 GB" | 20260506 14:46:25.681 | INFO | } | 20260506 14:46:25.681 | INFO | Perf report for the last 5.13795 seconds: | 20260506 14:46:25.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:46:25.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:25.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:25.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:25.681 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:46:25.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:25.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:25.681 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:46:25.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:25.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:25.681 | INFO | Uploaded 36GiB to GPU in 2.01924s (17.8285GiB/s) | 20260506 14:46:25.681 | INFO | Downloaded 34GiB to GPU in 1.92182s (17.6916GiB/s) | 20260506 14:46:30.820 | INFO | Analyzing chunk frames = [2181120,2181632) | 20260506 14:46:30.820 | INFO | MemInfo: { | 20260506 14:46:30.820 | INFO | "gpu" : " 17 GB", | 20260506 14:46:30.820 | INFO | "huge" : "", | 20260506 14:46:30.820 | INFO | "huge_shared" : " 27 GB", | 20260506 14:46:30.820 | INFO | "malloc" : " 14 GB", | 20260506 14:46:30.820 | INFO | "pinned" : " 29 GB" | 20260506 14:46:30.820 | INFO | } | 20260506 14:46:30.820 | INFO | Perf report for the last 5.13905 seconds: | 20260506 14:46:30.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:46:30.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:30.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:30.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:30.820 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 14:46:30.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:30.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:30.820 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:46:30.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:30.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:30.820 | INFO | Uploaded 36GiB to GPU in 2.04106s (17.6379GiB/s) | 20260506 14:46:30.820 | INFO | Downloaded 34GiB to GPU in 1.94829s (17.4512GiB/s) | 20260506 14:46:35.960 | INFO | Analyzing chunk frames = [2181632,2182144) | 20260506 14:46:35.960 | INFO | MemInfo: { | 20260506 14:46:35.960 | INFO | "gpu" : " 17 GB", | 20260506 14:46:35.960 | INFO | "huge" : "", | 20260506 14:46:35.960 | INFO | "huge_shared" : " 27 GB", | 20260506 14:46:35.960 | INFO | "malloc" : " 14 GB", | 20260506 14:46:35.960 | INFO | "pinned" : " 29 GB" | 20260506 14:46:35.960 | INFO | } | 20260506 14:46:35.960 | INFO | Perf report for the last 5.14016 seconds: | 20260506 14:46:35.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:46:35.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:35.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:35.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:35.960 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 14:46:35.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:35.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:35.960 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 14:46:35.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:35.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:35.960 | INFO | Uploaded 36GiB to GPU in 2.03994s (17.6476GiB/s) | 20260506 14:46:35.960 | INFO | Downloaded 34GiB to GPU in 1.92054s (17.7034GiB/s) | 20260506 14:46:41.101 | INFO | Analyzing chunk frames = [2182144,2182656) | 20260506 14:46:41.101 | INFO | MemInfo: { | 20260506 14:46:41.101 | INFO | "gpu" : " 17 GB", | 20260506 14:46:41.101 | INFO | "huge" : "", | 20260506 14:46:41.101 | INFO | "huge_shared" : " 27 GB", | 20260506 14:46:41.101 | INFO | "malloc" : " 14 GB", | 20260506 14:46:41.101 | INFO | "pinned" : " 29 GB" | 20260506 14:46:41.101 | INFO | } | 20260506 14:46:41.101 | INFO | Perf report for the last 5.14116 seconds: | 20260506 14:46:41.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:46:41.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:41.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:41.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:41.101 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:46:41.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:41.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:41.101 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:46:41.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:41.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:41.101 | INFO | Uploaded 36GiB to GPU in 2.00282s (17.9747GiB/s) | 20260506 14:46:41.102 | INFO | Downloaded 34GiB to GPU in 2.00117s (16.9901GiB/s) | 20260506 14:46:46.239 | INFO | Analyzing chunk frames = [2182656,2183168) | 20260506 14:46:46.239 | INFO | MemInfo: { | 20260506 14:46:46.239 | INFO | "gpu" : " 17 GB", | 20260506 14:46:46.239 | INFO | "huge" : "", | 20260506 14:46:46.240 | INFO | "huge_shared" : " 27 GB", | 20260506 14:46:46.240 | INFO | "malloc" : " 14 GB", | 20260506 14:46:46.240 | INFO | "pinned" : " 29 GB" | 20260506 14:46:46.240 | INFO | } | 20260506 14:46:46.240 | INFO | Perf report for the last 5.13827 seconds: | 20260506 14:46:46.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:46:46.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:46.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:46.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:46.240 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 14:46:46.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:46.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:46.240 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:46:46.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:46.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:46.240 | INFO | Uploaded 36GiB to GPU in 2.01925s (17.8284GiB/s) | 20260506 14:46:46.240 | INFO | Downloaded 34GiB to GPU in 1.97665s (17.2008GiB/s) | 20260506 14:46:46.354 | INFO | Starting a round of Baz IO | 20260506 14:46:49.508 | INFO | Populating RTMetrics report | 20260506 14:46:49.508 | INFO | RTMetrics report populated | 20260506 14:46:49.508 | INFO | Opening tmp RTMetrics json file | 20260506 14:46:49.512 | INFO | Serializing RTMetrics to json | 20260506 14:46:49.514 | INFO | Serialization done, swapping tmp file now... | 20260506 14:46:49.515 | INFO | RTMetrics done | 20260506 14:46:49.932 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.578 seconds | 20260506 14:46:49.932 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.91% / 0.01% / 62.08% | 20260506 14:46:49.932 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.37905% / 1.58946e-05% / 5.5631e-05% | 20260506 14:46:49.932 | INFO | Baz IO overall: | 20260506 14:46:49.932 | INFO | BytesWritten : 1130834540272 | 20260506 14:46:49.932 | INFO | NumEvents : 547247517456 | 20260506 14:46:51.378 | INFO | Analyzing chunk frames = [2183168,2183680) | 20260506 14:46:51.378 | INFO | MemInfo: { | 20260506 14:46:51.378 | INFO | "gpu" : " 17 GB", | 20260506 14:46:51.378 | INFO | "huge" : "", | 20260506 14:46:51.378 | INFO | "huge_shared" : " 27 GB", | 20260506 14:46:51.378 | INFO | "malloc" : " 14 GB", | 20260506 14:46:51.378 | INFO | "pinned" : " 29 GB" | 20260506 14:46:51.378 | INFO | } | 20260506 14:46:51.378 | INFO | Perf report for the last 5.13853 seconds: | 20260506 14:46:51.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:46:51.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:51.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:51.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:51.378 | INFO | ANALYSIS : 63.0%, 2.9, 95.5ms, 0 | 20260506 14:46:51.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:51.378 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 14:46:51.378 | INFO | PRE_HQ : 60.9%, 1.9, 59.4ms, 0 | 20260506 14:46:51.378 | INFO | BAZWRITER : 69.0%, 4.8, 169.9ms, 0 | 20260506 14:46:51.378 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 14:46:51.378 | INFO | Uploaded 36GiB to GPU in 2.05829s (17.4902GiB/s) | 20260506 14:46:51.378 | INFO | Downloaded 38GiB to GPU in 2.23379s (17.0114GiB/s) | 20260506 14:46:56.517 | INFO | Analyzing chunk frames = [2183680,2184192) | 20260506 14:46:56.517 | INFO | MemInfo: { | 20260506 14:46:56.517 | INFO | "gpu" : " 17 GB", | 20260506 14:46:56.517 | INFO | "huge" : "", | 20260506 14:46:56.517 | INFO | "huge_shared" : " 27 GB", | 20260506 14:46:56.517 | INFO | "malloc" : " 14 GB", | 20260506 14:46:56.517 | INFO | "pinned" : " 29 GB" | 20260506 14:46:56.517 | INFO | } | 20260506 14:46:56.517 | INFO | Perf report for the last 5.13886 seconds: | 20260506 14:46:56.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:46:56.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:56.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:56.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:56.517 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:46:56.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:56.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:56.517 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 14:46:56.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:56.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:46:56.517 | INFO | Uploaded 36GiB to GPU in 2.00429s (17.9615GiB/s) | 20260506 14:46:56.518 | INFO | Downloaded 34GiB to GPU in 1.94273s (17.5011GiB/s) | 20260506 14:47:01.656 | INFO | Analyzing chunk frames = [2184192,2184704) | 20260506 14:47:01.656 | INFO | MemInfo: { | 20260506 14:47:01.656 | INFO | "gpu" : " 17 GB", | 20260506 14:47:01.656 | INFO | "huge" : "", | 20260506 14:47:01.656 | INFO | "huge_shared" : " 27 GB", | 20260506 14:47:01.656 | INFO | "malloc" : " 14 GB", | 20260506 14:47:01.656 | INFO | "pinned" : " 29 GB" | 20260506 14:47:01.656 | INFO | } | 20260506 14:47:01.657 | INFO | Perf report for the last 5.13949 seconds: | 20260506 14:47:01.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:47:01.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:01.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:01.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:01.657 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 14:47:01.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:01.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:01.657 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:47:01.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:01.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:01.657 | INFO | Uploaded 36GiB to GPU in 1.99926s (18.0067GiB/s) | 20260506 14:47:01.657 | INFO | Downloaded 34GiB to GPU in 1.94529s (17.4782GiB/s) | 20260506 14:47:06.796 | INFO | Analyzing chunk frames = [2184704,2185216) | 20260506 14:47:06.796 | INFO | MemInfo: { | 20260506 14:47:06.796 | INFO | "gpu" : " 17 GB", | 20260506 14:47:06.796 | INFO | "huge" : "", | 20260506 14:47:06.796 | INFO | "huge_shared" : " 27 GB", | 20260506 14:47:06.796 | INFO | "malloc" : " 14 GB", | 20260506 14:47:06.796 | INFO | "pinned" : " 29 GB" | 20260506 14:47:06.796 | INFO | } | 20260506 14:47:06.796 | INFO | Perf report for the last 5.13979 seconds: | 20260506 14:47:06.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:47:06.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:06.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:06.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:06.796 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 14:47:06.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:06.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:06.796 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:47:06.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:06.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:06.797 | INFO | Uploaded 36GiB to GPU in 1.99623s (18.034GiB/s) | 20260506 14:47:06.797 | INFO | Downloaded 34GiB to GPU in 1.914s (17.7639GiB/s) | 20260506 14:47:11.936 | INFO | Analyzing chunk frames = [2185216,2185728) | 20260506 14:47:11.936 | INFO | MemInfo: { | 20260506 14:47:11.936 | INFO | "gpu" : " 17 GB", | 20260506 14:47:11.936 | INFO | "huge" : "", | 20260506 14:47:11.936 | INFO | "huge_shared" : " 27 GB", | 20260506 14:47:11.936 | INFO | "malloc" : " 14 GB", | 20260506 14:47:11.936 | INFO | "pinned" : " 29 GB" | 20260506 14:47:11.936 | INFO | } | 20260506 14:47:11.936 | INFO | Perf report for the last 5.13964 seconds: | 20260506 14:47:11.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:47:11.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:11.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:11.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:11.936 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 14:47:11.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:11.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:11.936 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:47:11.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:11.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:11.936 | INFO | Uploaded 36GiB to GPU in 2.01467s (17.8689GiB/s) | 20260506 14:47:11.936 | INFO | Downloaded 34GiB to GPU in 1.94171s (17.5103GiB/s) | 20260506 14:47:17.075 | INFO | Analyzing chunk frames = [2185728,2186240) | 20260506 14:47:17.075 | INFO | MemInfo: { | 20260506 14:47:17.075 | INFO | "gpu" : " 17 GB", | 20260506 14:47:17.075 | INFO | "huge" : "", | 20260506 14:47:17.075 | INFO | "huge_shared" : " 27 GB", | 20260506 14:47:17.075 | INFO | "malloc" : " 14 GB", | 20260506 14:47:17.075 | INFO | "pinned" : " 29 GB" | 20260506 14:47:17.075 | INFO | } | 20260506 14:47:17.075 | INFO | Perf report for the last 5.13939 seconds: | 20260506 14:47:17.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:47:17.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:17.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:17.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:17.075 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260506 14:47:17.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:17.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:17.075 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 14:47:17.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:17.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:17.076 | INFO | Uploaded 36GiB to GPU in 2.01306s (17.8832GiB/s) | 20260506 14:47:17.076 | INFO | Downloaded 34GiB to GPU in 1.92276s (17.6829GiB/s) | 20260506 14:47:22.215 | INFO | Analyzing chunk frames = [2186240,2186752) | 20260506 14:47:22.215 | INFO | MemInfo: { | 20260506 14:47:22.215 | INFO | "gpu" : " 17 GB", | 20260506 14:47:22.215 | INFO | "huge" : "", | 20260506 14:47:22.215 | INFO | "huge_shared" : " 27 GB", | 20260506 14:47:22.215 | INFO | "malloc" : " 14 GB", | 20260506 14:47:22.215 | INFO | "pinned" : " 29 GB" | 20260506 14:47:22.215 | INFO | } | 20260506 14:47:22.215 | INFO | Perf report for the last 5.13993 seconds: | 20260506 14:47:22.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:47:22.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:22.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:22.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:22.215 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:47:22.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:22.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:22.215 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 14:47:22.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:22.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:22.216 | INFO | Uploaded 36GiB to GPU in 1.9964s (18.0324GiB/s) | 20260506 14:47:22.216 | INFO | Downloaded 34GiB to GPU in 1.9982s (17.0153GiB/s) | 20260506 14:47:27.354 | INFO | Analyzing chunk frames = [2186752,2187264) | 20260506 14:47:27.354 | INFO | MemInfo: { | 20260506 14:47:27.354 | INFO | "gpu" : " 17 GB", | 20260506 14:47:27.354 | INFO | "huge" : "", | 20260506 14:47:27.354 | INFO | "huge_shared" : " 27 GB", | 20260506 14:47:27.354 | INFO | "malloc" : " 14 GB", | 20260506 14:47:27.354 | INFO | "pinned" : " 29 GB" | 20260506 14:47:27.354 | INFO | } | 20260506 14:47:27.354 | INFO | Perf report for the last 5.13869 seconds: | 20260506 14:47:27.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:47:27.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:27.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:27.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:27.354 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:47:27.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:27.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:27.354 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260506 14:47:27.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:27.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:27.354 | INFO | Uploaded 36GiB to GPU in 2.01948s (17.8264GiB/s) | 20260506 14:47:27.354 | INFO | Downloaded 34GiB to GPU in 1.98106s (17.1626GiB/s) | 20260506 14:47:27.354 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:47:27.354 | INFO | Total mem usage: 17.4 GiB | 20260506 14:47:27.354 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:47:27.354 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -11%) | 20260506 14:47:27.354 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:47:27.354 | INFO | High Water Breakdown: | 20260506 14:47:27.354 | INFO | 61.3% : 9.61 GiB ( 88% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:47:27.355 | INFO | 38.6% : 6.05 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:47:27.355 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:47:27.355 | INFO | ------------------------------------------ | 20260506 14:47:27.355 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:47:27.355 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:47:27.355 | INFO | Total mem usage: 26.5 GiB | 20260506 14:47:27.355 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:47:27.355 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:47:27.355 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:47:27.355 | INFO | High Water Breakdown: | 20260506 14:47:27.355 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:47:27.355 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:47:27.355 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:47:27.355 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:47:27.355 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:47:27.355 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:47:27.355 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:47:27.355 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:47:27.355 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:47:27.355 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:47:27.355 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:47:27.355 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:47:27.355 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:47:27.355 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:47:27.355 | INFO | ------------------------------------------ | 20260506 14:47:27.355 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:47:27.355 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:47:27.355 | INFO | Total mem usage: 25.6 GiB | 20260506 14:47:27.355 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:47:27.355 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:47:27.355 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:47:27.355 | INFO | High Water Breakdown: | 20260506 14:47:27.355 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:47:27.355 | INFO | ------------------------------------------ | 20260506 14:47:27.355 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:47:27.355 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:47:27.355 | INFO | Total mem usage: 15.7 GiB | 20260506 14:47:27.355 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:47:27.355 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:47:27.355 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 14:47:27.355 | INFO | High Water Breakdown: | 20260506 14:47:27.355 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:47:27.355 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:47:27.355 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:47:27.355 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:47:27.355 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:47:27.355 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:47:27.355 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:47:27.355 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:47:27.355 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:47:27.355 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:47:27.355 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:47:27.355 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:47:27.355 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:47:27.355 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:47:27.355 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:47:27.355 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:47:27.355 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:47:27.355 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:47:27.355 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:47:27.355 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:47:27.355 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:47:27.355 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:47:27.355 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:47:27.355 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:47:27.355 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:47:27.355 | INFO | ------------------------------------------ | 20260506 14:47:27.355 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:47:27.355 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:47:27.355 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:47:27.355 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:47:27.355 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.51e+01, 1.33e+01 / 1.86e+00 | 20260506 14:47:27.355 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.01e-01, 1.18e-02 / 6.96e-03 | 20260506 14:47:27.355 | INFO | PerfReport: 11.59%, Baseline -- 3.65e+00 / 4.40e+00, 5.66e-01 / 3.86e-01 | 20260506 14:47:27.355 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.47e+00, 1.86e-01 / 1.57e-01 | 20260506 14:47:27.355 | INFO | PerfReport: 15.78%, DME -- 3.98e+01 / 4.35e+01, 3.55e+00 / 1.56e+00 | 20260506 14:47:27.355 | INFO | PerfReport: 56.81%, FrameLabeling -- 1.79e+01 / 2.09e+01, 1.49e+00 / 2.01e+00 | 20260506 14:47:27.355 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.34e+00 / 2.82e+00, 4.25e-01 / 3.15e-01 | 20260506 14:47:27.355 | INFO | PerfReport: 3.59%, Metrics -- 1.13e+00 / 1.47e+00, 2.90e-01 / 2.03e-01 | 20260506 14:47:27.355 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.95e-01 / 4.51e-01, 6.24e-02 / 3.26e-02 | 20260506 14:47:27.355 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:47:27.355 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:47:27.355 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:47:27.355 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 14:47:27.355 | INFO | PerfReport: 21.50%, Upload -- 2.10e+01 / 0.00e+00, 6.94e+00 / 0.00e+00 | 20260506 14:47:27.355 | INFO | PerfReport: 21.18%, Download -- 2.07e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 14:47:27.355 | INFO | PerfReport: 32.28%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 14:47:27.355 | INFO | PerfReport: 1.44%, LockUpload -- 1.41e+00 / 0.00e+00, 6.12e+00 / 0.00e+00 | 20260506 14:47:27.355 | INFO | PerfReport: 1.18%, LockDownload -- 1.15e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 14:47:27.355 | INFO | PerfReport: 22.40%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 14:47:27.355 | INFO | PerfReport: 0.02%, Misc -- 1.52e-02 / 0.00e+00, 2.58e-02 / 0.00e+00 | 20260506 14:47:27.469 | INFO | Starting a round of Baz IO | 20260506 14:47:30.627 | INFO | Populating RTMetrics report | 20260506 14:47:30.627 | INFO | RTMetrics report populated | 20260506 14:47:30.627 | INFO | Opening tmp RTMetrics json file | 20260506 14:47:30.631 | INFO | Serializing RTMetrics to json | 20260506 14:47:30.632 | INFO | Serialization done, swapping tmp file now... | 20260506 14:47:30.633 | INFO | RTMetrics done | 20260506 14:47:30.939 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.47 seconds | 20260506 14:47:30.940 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.90% / 0.01% / 62.09% | 20260506 14:47:30.940 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.37606% / 2.78155e-05% / 5.5631e-05% | 20260506 14:47:30.940 | INFO | Baz IO overall: | 20260506 14:47:30.940 | INFO | BytesWritten : 1133100167397 | 20260506 14:47:30.940 | INFO | NumEvents : 548345213118 | 20260506 14:47:32.493 | INFO | Analyzing chunk frames = [2187264,2187776) | 20260506 14:47:32.493 | INFO | MemInfo: { | 20260506 14:47:32.493 | INFO | "gpu" : " 17 GB", | 20260506 14:47:32.493 | INFO | "huge" : "", | 20260506 14:47:32.493 | INFO | "huge_shared" : " 27 GB", | 20260506 14:47:32.493 | INFO | "malloc" : " 14 GB", | 20260506 14:47:32.493 | INFO | "pinned" : " 29 GB" | 20260506 14:47:32.493 | INFO | } | 20260506 14:47:32.493 | INFO | Perf report for the last 5.13938 seconds: | 20260506 14:47:32.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:47:32.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:32.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:32.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:32.493 | INFO | ANALYSIS : 63.1%, 2.9, 95.2ms, 0 | 20260506 14:47:32.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:32.493 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 14:47:32.494 | INFO | PRE_HQ : 60.9%, 1.9, 59.5ms, 0 | 20260506 14:47:32.494 | INFO | BAZWRITER : 66.9%, 4.8, 164.4ms, 0 | 20260506 14:47:32.494 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 14:47:32.494 | INFO | Uploaded 36GiB to GPU in 2.05502s (17.5181GiB/s) | 20260506 14:47:32.494 | INFO | Downloaded 38GiB to GPU in 2.2403s (16.962GiB/s) | 20260506 14:47:37.633 | INFO | Analyzing chunk frames = [2187776,2188288) | 20260506 14:47:37.633 | INFO | MemInfo: { | 20260506 14:47:37.633 | INFO | "gpu" : " 17 GB", | 20260506 14:47:37.633 | INFO | "huge" : "", | 20260506 14:47:37.633 | INFO | "huge_shared" : " 27 GB", | 20260506 14:47:37.633 | INFO | "malloc" : " 14 GB", | 20260506 14:47:37.633 | INFO | "pinned" : " 29 GB" | 20260506 14:47:37.633 | INFO | } | 20260506 14:47:37.633 | INFO | Perf report for the last 5.1396 seconds: | 20260506 14:47:37.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:47:37.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:37.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:37.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:37.633 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 14:47:37.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:37.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:37.633 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 14:47:37.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:37.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:37.633 | INFO | Uploaded 36GiB to GPU in 1.9841s (18.1443GiB/s) | 20260506 14:47:37.633 | INFO | Downloaded 34GiB to GPU in 1.9422s (17.5059GiB/s) | 20260506 14:47:42.772 | INFO | Analyzing chunk frames = [2188288,2188800) | 20260506 14:47:42.772 | INFO | MemInfo: { | 20260506 14:47:42.772 | INFO | "gpu" : " 17 GB", | 20260506 14:47:42.772 | INFO | "huge" : "", | 20260506 14:47:42.772 | INFO | "huge_shared" : " 27 GB", | 20260506 14:47:42.772 | INFO | "malloc" : " 14 GB", | 20260506 14:47:42.772 | INFO | "pinned" : " 29 GB" | 20260506 14:47:42.772 | INFO | } | 20260506 14:47:42.772 | INFO | Perf report for the last 5.13909 seconds: | 20260506 14:47:42.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:47:42.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:42.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:42.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:42.772 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 14:47:42.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:42.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:42.772 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:47:42.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:42.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:42.772 | INFO | Uploaded 36GiB to GPU in 1.96208s (18.3479GiB/s) | 20260506 14:47:42.772 | INFO | Downloaded 34GiB to GPU in 1.93915s (17.5335GiB/s) | 20260506 14:47:47.911 | INFO | Analyzing chunk frames = [2188800,2189312) | 20260506 14:47:47.911 | INFO | MemInfo: { | 20260506 14:47:47.911 | INFO | "gpu" : " 17 GB", | 20260506 14:47:47.911 | INFO | "huge" : "", | 20260506 14:47:47.911 | INFO | "huge_shared" : " 27 GB", | 20260506 14:47:47.911 | INFO | "malloc" : " 14 GB", | 20260506 14:47:47.911 | INFO | "pinned" : " 29 GB" | 20260506 14:47:47.911 | INFO | } | 20260506 14:47:47.911 | INFO | Perf report for the last 5.13912 seconds: | 20260506 14:47:47.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:47:47.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:47.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:47.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:47.911 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260506 14:47:47.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:47.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:47.911 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:47:47.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:47.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:47.912 | INFO | Uploaded 36GiB to GPU in 1.95564s (18.4083GiB/s) | 20260506 14:47:47.912 | INFO | Downloaded 34GiB to GPU in 1.89354s (17.9558GiB/s) | 20260506 14:47:53.050 | INFO | Analyzing chunk frames = [2189312,2189824) | 20260506 14:47:53.050 | INFO | MemInfo: { | 20260506 14:47:53.050 | INFO | "gpu" : " 17 GB", | 20260506 14:47:53.050 | INFO | "huge" : "", | 20260506 14:47:53.050 | INFO | "huge_shared" : " 27 GB", | 20260506 14:47:53.050 | INFO | "malloc" : " 14 GB", | 20260506 14:47:53.050 | INFO | "pinned" : " 29 GB" | 20260506 14:47:53.050 | INFO | } | 20260506 14:47:53.050 | INFO | Perf report for the last 5.13915 seconds: | 20260506 14:47:53.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:47:53.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:53.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:53.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:53.050 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 14:47:53.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:53.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:53.050 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:47:53.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:53.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:53.051 | INFO | Uploaded 36GiB to GPU in 1.98071s (18.1753GiB/s) | 20260506 14:47:53.051 | INFO | Downloaded 34GiB to GPU in 1.92816s (17.6334GiB/s) | 20260506 14:47:58.190 | INFO | Analyzing chunk frames = [2189824,2190336) | 20260506 14:47:58.190 | INFO | MemInfo: { | 20260506 14:47:58.190 | INFO | "gpu" : " 17 GB", | 20260506 14:47:58.191 | INFO | "huge" : "", | 20260506 14:47:58.191 | INFO | "huge_shared" : " 27 GB", | 20260506 14:47:58.191 | INFO | "malloc" : " 14 GB", | 20260506 14:47:58.191 | INFO | "pinned" : " 29 GB" | 20260506 14:47:58.191 | INFO | } | 20260506 14:47:58.191 | INFO | Perf report for the last 5.14027 seconds: | 20260506 14:47:58.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:47:58.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:58.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:58.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:58.191 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 14:47:58.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:58.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:58.191 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 14:47:58.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:58.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:47:58.191 | INFO | Uploaded 36GiB to GPU in 2.07463s (17.3525GiB/s) | 20260506 14:47:58.191 | INFO | Downloaded 34GiB to GPU in 1.97896s (17.1808GiB/s) | 20260506 14:48:03.330 | INFO | Analyzing chunk frames = [2190336,2190848) | 20260506 14:48:03.330 | INFO | MemInfo: { | 20260506 14:48:03.330 | INFO | "gpu" : " 17 GB", | 20260506 14:48:03.330 | INFO | "huge" : "", | 20260506 14:48:03.330 | INFO | "huge_shared" : " 27 GB", | 20260506 14:48:03.330 | INFO | "malloc" : " 14 GB", | 20260506 14:48:03.330 | INFO | "pinned" : " 29 GB" | 20260506 14:48:03.330 | INFO | } | 20260506 14:48:03.330 | INFO | Perf report for the last 5.13967 seconds: | 20260506 14:48:03.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:48:03.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:03.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:03.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:03.330 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:48:03.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:03.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:03.330 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 14:48:03.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:03.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:03.331 | INFO | Uploaded 36GiB to GPU in 2.0086s (17.9229GiB/s) | 20260506 14:48:03.331 | INFO | Downloaded 34GiB to GPU in 2.00355s (16.9699GiB/s) | 20260506 14:48:08.468 | INFO | Analyzing chunk frames = [2190848,2191360) | 20260506 14:48:08.468 | INFO | MemInfo: { | 20260506 14:48:08.468 | INFO | "gpu" : " 17 GB", | 20260506 14:48:08.468 | INFO | "huge" : "", | 20260506 14:48:08.469 | INFO | "huge_shared" : " 27 GB", | 20260506 14:48:08.469 | INFO | "malloc" : " 14 GB", | 20260506 14:48:08.469 | INFO | "pinned" : " 29 GB" | 20260506 14:48:08.469 | INFO | } | 20260506 14:48:08.469 | INFO | Perf report for the last 5.13832 seconds: | 20260506 14:48:08.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:48:08.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:08.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:08.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:08.469 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:48:08.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:08.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:08.469 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:48:08.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:08.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:08.469 | INFO | Uploaded 36GiB to GPU in 2.02109s (17.8122GiB/s) | 20260506 14:48:08.469 | INFO | Downloaded 34GiB to GPU in 1.98309s (17.145GiB/s) | 20260506 14:48:08.580 | INFO | Starting a round of Baz IO | 20260506 14:48:11.740 | INFO | Populating RTMetrics report | 20260506 14:48:11.740 | INFO | RTMetrics report populated | 20260506 14:48:11.740 | INFO | Opening tmp RTMetrics json file | 20260506 14:48:11.764 | INFO | Serializing RTMetrics to json | 20260506 14:48:11.819 | INFO | Serialization done, swapping tmp file now... | 20260506 14:48:11.830 | INFO | RTMetrics done | 20260506 14:48:12.038 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.458 seconds | 20260506 14:48:12.038 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.88% / 0.01% / 62.10% | 20260506 14:48:12.038 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.36597% / 1.58946e-05% / 8.74201e-05% | 20260506 14:48:12.038 | INFO | Baz IO overall: | 20260506 14:48:12.038 | INFO | BytesWritten : 1135365253613 | 20260506 14:48:12.038 | INFO | NumEvents : 549442550474 | 20260506 14:48:13.608 | INFO | Analyzing chunk frames = [2191360,2191872) | 20260506 14:48:13.608 | INFO | MemInfo: { | 20260506 14:48:13.608 | INFO | "gpu" : " 17 GB", | 20260506 14:48:13.608 | INFO | "huge" : "", | 20260506 14:48:13.608 | INFO | "huge_shared" : " 27 GB", | 20260506 14:48:13.608 | INFO | "malloc" : " 14 GB", | 20260506 14:48:13.608 | INFO | "pinned" : " 29 GB" | 20260506 14:48:13.608 | INFO | } | 20260506 14:48:13.608 | INFO | Perf report for the last 5.1395 seconds: | 20260506 14:48:13.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:48:13.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:13.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:13.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:13.608 | INFO | ANALYSIS : 63.0%, 2.9, 95.7ms, 0 | 20260506 14:48:13.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:13.608 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260506 14:48:13.608 | INFO | PRE_HQ : 61.1%, 1.9, 59.2ms, 0 | 20260506 14:48:13.608 | INFO | BAZWRITER : 66.7%, 4.8, 165.2ms, 0 | 20260506 14:48:13.608 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 14:48:13.609 | INFO | Uploaded 36GiB to GPU in 2.07199s (17.3746GiB/s) | 20260506 14:48:13.609 | INFO | Downloaded 38GiB to GPU in 2.22577s (17.0728GiB/s) | 20260506 14:48:18.748 | INFO | Analyzing chunk frames = [2191872,2192384) | 20260506 14:48:18.748 | INFO | MemInfo: { | 20260506 14:48:18.748 | INFO | "gpu" : " 17 GB", | 20260506 14:48:18.748 | INFO | "huge" : "", | 20260506 14:48:18.748 | INFO | "huge_shared" : " 27 GB", | 20260506 14:48:18.748 | INFO | "malloc" : " 14 GB", | 20260506 14:48:18.748 | INFO | "pinned" : " 29 GB" | 20260506 14:48:18.748 | INFO | } | 20260506 14:48:18.748 | INFO | Perf report for the last 5.13961 seconds: | 20260506 14:48:18.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:48:18.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:18.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:18.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:18.748 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:48:18.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:18.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:18.748 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:48:18.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:18.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:18.748 | INFO | Uploaded 36GiB to GPU in 1.98872s (18.1021GiB/s) | 20260506 14:48:18.748 | INFO | Downloaded 34GiB to GPU in 1.93944s (17.5309GiB/s) | 20260506 14:48:23.886 | INFO | Analyzing chunk frames = [2192384,2192896) | 20260506 14:48:23.886 | INFO | MemInfo: { | 20260506 14:48:23.886 | INFO | "gpu" : " 17 GB", | 20260506 14:48:23.886 | INFO | "huge" : "", | 20260506 14:48:23.886 | INFO | "huge_shared" : " 27 GB", | 20260506 14:48:23.886 | INFO | "malloc" : " 14 GB", | 20260506 14:48:23.886 | INFO | "pinned" : " 29 GB" | 20260506 14:48:23.886 | INFO | } | 20260506 14:48:23.886 | INFO | Perf report for the last 5.13859 seconds: | 20260506 14:48:23.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:48:23.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:23.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:23.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:23.886 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 14:48:23.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:23.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:23.887 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:48:23.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:23.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:23.887 | INFO | Uploaded 36GiB to GPU in 1.98942s (18.0957GiB/s) | 20260506 14:48:23.887 | INFO | Downloaded 34GiB to GPU in 1.94092s (17.5175GiB/s) | 20260506 14:48:29.026 | INFO | Analyzing chunk frames = [2192896,2193408) | 20260506 14:48:29.026 | INFO | MemInfo: { | 20260506 14:48:29.026 | INFO | "gpu" : " 17 GB", | 20260506 14:48:29.026 | INFO | "huge" : "", | 20260506 14:48:29.026 | INFO | "huge_shared" : " 27 GB", | 20260506 14:48:29.026 | INFO | "malloc" : " 14 GB", | 20260506 14:48:29.026 | INFO | "pinned" : " 29 GB" | 20260506 14:48:29.026 | INFO | } | 20260506 14:48:29.026 | INFO | Perf report for the last 5.13975 seconds: | 20260506 14:48:29.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:48:29.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:29.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:29.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:29.026 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 14:48:29.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:29.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:29.026 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:48:29.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:29.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:29.027 | INFO | Uploaded 36GiB to GPU in 2.00613s (17.945GiB/s) | 20260506 14:48:29.027 | INFO | Downloaded 34GiB to GPU in 1.90693s (17.8297GiB/s) | 20260506 14:48:34.165 | INFO | Analyzing chunk frames = [2193408,2193920) | 20260506 14:48:34.165 | INFO | MemInfo: { | 20260506 14:48:34.165 | INFO | "gpu" : " 17 GB", | 20260506 14:48:34.165 | INFO | "huge" : "", | 20260506 14:48:34.165 | INFO | "huge_shared" : " 27 GB", | 20260506 14:48:34.165 | INFO | "malloc" : " 14 GB", | 20260506 14:48:34.165 | INFO | "pinned" : " 29 GB" | 20260506 14:48:34.165 | INFO | } | 20260506 14:48:34.165 | INFO | Perf report for the last 5.13909 seconds: | 20260506 14:48:34.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:48:34.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:34.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:34.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:34.165 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 14:48:34.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:34.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:34.165 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:48:34.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:34.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:34.166 | INFO | Uploaded 36GiB to GPU in 2.03928s (17.6533GiB/s) | 20260506 14:48:34.166 | INFO | Downloaded 34GiB to GPU in 1.92533s (17.6593GiB/s) | 20260506 14:48:39.304 | INFO | Analyzing chunk frames = [2193920,2194432) | 20260506 14:48:39.304 | INFO | MemInfo: { | 20260506 14:48:39.304 | INFO | "gpu" : " 17 GB", | 20260506 14:48:39.304 | INFO | "huge" : "", | 20260506 14:48:39.304 | INFO | "huge_shared" : " 27 GB", | 20260506 14:48:39.304 | INFO | "malloc" : " 14 GB", | 20260506 14:48:39.305 | INFO | "pinned" : " 29 GB" | 20260506 14:48:39.305 | INFO | } | 20260506 14:48:39.305 | INFO | Perf report for the last 5.13931 seconds: | 20260506 14:48:39.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:48:39.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:39.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:39.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:39.305 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:48:39.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:39.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:39.305 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260506 14:48:39.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:39.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:39.305 | INFO | Uploaded 36GiB to GPU in 2.04842s (17.5745GiB/s) | 20260506 14:48:39.305 | INFO | Downloaded 34GiB to GPU in 1.9135s (17.7684GiB/s) | 20260506 14:48:44.445 | INFO | Analyzing chunk frames = [2194432,2194944) | 20260506 14:48:44.445 | INFO | MemInfo: { | 20260506 14:48:44.445 | INFO | "gpu" : " 17 GB", | 20260506 14:48:44.445 | INFO | "huge" : "", | 20260506 14:48:44.445 | INFO | "huge_shared" : " 27 GB", | 20260506 14:48:44.445 | INFO | "malloc" : " 14 GB", | 20260506 14:48:44.445 | INFO | "pinned" : " 29 GB" | 20260506 14:48:44.445 | INFO | } | 20260506 14:48:44.445 | INFO | Perf report for the last 5.14046 seconds: | 20260506 14:48:44.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:48:44.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:44.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:44.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:44.445 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:48:44.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:44.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:44.445 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 14:48:44.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:44.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:44.445 | INFO | Uploaded 36GiB to GPU in 2.00322s (17.9711GiB/s) | 20260506 14:48:44.445 | INFO | Downloaded 34GiB to GPU in 1.9915s (17.0725GiB/s) | 20260506 14:48:49.584 | INFO | Analyzing chunk frames = [2194944,2195456) | 20260506 14:48:49.584 | INFO | MemInfo: { | 20260506 14:48:49.584 | INFO | "gpu" : " 17 GB", | 20260506 14:48:49.584 | INFO | "huge" : "", | 20260506 14:48:49.584 | INFO | "huge_shared" : " 27 GB", | 20260506 14:48:49.584 | INFO | "malloc" : " 14 GB", | 20260506 14:48:49.584 | INFO | "pinned" : " 29 GB" | 20260506 14:48:49.584 | INFO | } | 20260506 14:48:49.584 | INFO | Perf report for the last 5.1389 seconds: | 20260506 14:48:49.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:48:49.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:49.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:49.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:49.584 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 14:48:49.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:49.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:49.584 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:48:49.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:49.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:49.584 | INFO | Uploaded 36GiB to GPU in 2.01734s (17.8453GiB/s) | 20260506 14:48:49.584 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:48:49.584 | INFO | Downloaded 34GiB to GPU in 1.98299s (17.1459GiB/s) | 20260506 14:48:49.585 | INFO | Total mem usage: 17.4 GiB | 20260506 14:48:49.585 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:48:49.585 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -10.6%) | 20260506 14:48:49.585 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:48:49.585 | INFO | High Water Breakdown: | 20260506 14:48:49.585 | INFO | 61.3% : 9.66 GiB ( 88.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:48:49.585 | INFO | 38.5% : 6.07 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:48:49.585 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:48:49.585 | INFO | ------------------------------------------ | 20260506 14:48:49.585 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:48:49.585 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:48:49.585 | INFO | Total mem usage: 26.5 GiB | 20260506 14:48:49.585 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:48:49.585 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:48:49.585 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:48:49.585 | INFO | High Water Breakdown: | 20260506 14:48:49.585 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:48:49.585 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:48:49.585 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:48:49.585 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:48:49.585 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:48:49.585 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:48:49.585 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:48:49.585 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:48:49.585 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:48:49.585 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:48:49.585 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:48:49.585 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:48:49.585 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:48:49.585 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:48:49.585 | INFO | ------------------------------------------ | 20260506 14:48:49.585 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:48:49.585 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:48:49.585 | INFO | Total mem usage: 25.6 GiB | 20260506 14:48:49.585 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:48:49.585 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:48:49.585 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:48:49.585 | INFO | High Water Breakdown: | 20260506 14:48:49.585 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:48:49.585 | INFO | ------------------------------------------ | 20260506 14:48:49.585 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:48:49.585 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:48:49.585 | INFO | Total mem usage: 15.7 GiB | 20260506 14:48:49.585 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:48:49.585 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:48:49.585 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 14:48:49.585 | INFO | High Water Breakdown: | 20260506 14:48:49.585 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:48:49.585 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:48:49.585 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:48:49.585 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:48:49.585 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:48:49.585 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:48:49.585 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:48:49.585 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:48:49.585 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:48:49.585 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:48:49.585 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:48:49.585 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:48:49.585 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:48:49.585 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:48:49.585 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:48:49.585 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:48:49.585 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:48:49.585 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:48:49.585 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:48:49.586 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:48:49.586 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:48:49.586 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:48:49.586 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:48:49.586 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:48:49.586 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:48:49.586 | INFO | ------------------------------------------ | 20260506 14:48:49.586 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:48:49.586 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:48:49.586 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:48:49.586 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:48:49.586 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.51e+01, 1.33e+01 / 1.66e+00 | 20260506 14:48:49.586 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.00e-01, 1.01e-02 / 7.30e-03 | 20260506 14:48:49.586 | INFO | PerfReport: 11.58%, Baseline -- 3.64e+00 / 4.43e+00, 5.65e-01 / 3.94e-01 | 20260506 14:48:49.586 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.46e+00, 1.88e-01 / 1.57e-01 | 20260506 14:48:49.586 | INFO | PerfReport: 15.84%, DME -- 3.99e+01 / 4.38e+01, 3.57e+00 / 1.98e+00 | 20260506 14:48:49.586 | INFO | PerfReport: 56.74%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.41e+00 / 2.04e+00 | 20260506 14:48:49.586 | INFO | PerfReport: 7.45%, PulseAccumulating -- 2.34e+00 / 2.96e+00, 4.08e-01 / 2.34e-01 | 20260506 14:48:49.586 | INFO | PerfReport: 3.60%, Metrics -- 1.13e+00 / 1.53e+00, 2.93e-01 / 1.89e-01 | 20260506 14:48:49.586 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.94e-01 / 4.48e-01, 6.13e-02 / 2.70e-02 | 20260506 14:48:49.586 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:48:49.586 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:48:49.586 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:48:49.586 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 14:48:49.586 | INFO | PerfReport: 21.49%, Upload -- 2.10e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 14:48:49.586 | INFO | PerfReport: 21.18%, Download -- 2.07e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 14:48:49.586 | INFO | PerfReport: 32.26%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 14:48:49.586 | INFO | PerfReport: 1.51%, LockUpload -- 1.48e+00 / 0.00e+00, 5.93e+00 / 0.00e+00 | 20260506 14:48:49.586 | INFO | PerfReport: 1.15%, LockDownload -- 1.12e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260506 14:48:49.586 | INFO | PerfReport: 22.39%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 14:48:49.586 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.15e-02 / 0.00e+00 | 20260506 14:48:49.697 | INFO | Starting a round of Baz IO | 20260506 14:48:52.849 | INFO | Populating RTMetrics report | 20260506 14:48:52.850 | INFO | RTMetrics report populated | 20260506 14:48:52.850 | INFO | Opening tmp RTMetrics json file | 20260506 14:48:52.851 | INFO | Serializing RTMetrics to json | 20260506 14:48:52.854 | INFO | Serialization done, swapping tmp file now... | 20260506 14:48:52.854 | INFO | RTMetrics done | 20260506 14:48:53.017 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.319 seconds | 20260506 14:48:53.017 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.87% / 0.01% / 62.12% | 20260506 14:48:53.017 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.36357% / 2.38419e-05% / 4.37101e-05% | 20260506 14:48:53.017 | INFO | Baz IO overall: | 20260506 14:48:53.017 | INFO | BytesWritten : 1137631084118 | 20260506 14:48:53.017 | INFO | NumEvents : 550540839038 | 20260506 14:48:54.723 | INFO | Analyzing chunk frames = [2195456,2195968) | 20260506 14:48:54.723 | INFO | MemInfo: { | 20260506 14:48:54.723 | INFO | "gpu" : " 17 GB", | 20260506 14:48:54.723 | INFO | "huge" : "", | 20260506 14:48:54.723 | INFO | "huge_shared" : " 27 GB", | 20260506 14:48:54.723 | INFO | "malloc" : " 14 GB", | 20260506 14:48:54.723 | INFO | "pinned" : " 29 GB" | 20260506 14:48:54.723 | INFO | } | 20260506 14:48:54.723 | INFO | Perf report for the last 5.1388 seconds: | 20260506 14:48:54.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:48:54.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:54.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:54.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:54.723 | INFO | ANALYSIS : 62.9%, 2.9, 95.4ms, 0 | 20260506 14:48:54.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:54.723 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 14:48:54.723 | INFO | PRE_HQ : 60.9%, 1.9, 59.3ms, 0 | 20260506 14:48:54.723 | INFO | BAZWRITER : 64.1%, 4.7, 155.5ms, 0 | 20260506 14:48:54.723 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.3ms, 0 | 20260506 14:48:54.723 | INFO | Uploaded 36GiB to GPU in 2.0509s (17.5532GiB/s) | 20260506 14:48:54.723 | INFO | Downloaded 38GiB to GPU in 2.22894s (17.0485GiB/s) | 20260506 14:48:59.862 | INFO | Analyzing chunk frames = [2195968,2196480) | 20260506 14:48:59.862 | INFO | MemInfo: { | 20260506 14:48:59.862 | INFO | "gpu" : " 17 GB", | 20260506 14:48:59.862 | INFO | "huge" : "", | 20260506 14:48:59.862 | INFO | "huge_shared" : " 27 GB", | 20260506 14:48:59.862 | INFO | "malloc" : " 14 GB", | 20260506 14:48:59.862 | INFO | "pinned" : " 29 GB" | 20260506 14:48:59.862 | INFO | } | 20260506 14:48:59.862 | INFO | Perf report for the last 5.13892 seconds: | 20260506 14:48:59.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:48:59.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:59.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:59.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:59.862 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 14:48:59.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:59.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:59.862 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:48:59.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:59.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:48:59.862 | INFO | Uploaded 36GiB to GPU in 1.97465s (18.2311GiB/s) | 20260506 14:48:59.862 | INFO | Downloaded 34GiB to GPU in 1.93101s (17.6073GiB/s) | 20260506 14:49:05.001 | INFO | Analyzing chunk frames = [2196480,2196992) | 20260506 14:49:05.001 | INFO | MemInfo: { | 20260506 14:49:05.001 | INFO | "gpu" : " 17 GB", | 20260506 14:49:05.001 | INFO | "huge" : "", | 20260506 14:49:05.001 | INFO | "huge_shared" : " 27 GB", | 20260506 14:49:05.001 | INFO | "malloc" : " 14 GB", | 20260506 14:49:05.001 | INFO | "pinned" : " 29 GB" | 20260506 14:49:05.001 | INFO | } | 20260506 14:49:05.001 | INFO | Perf report for the last 5.13932 seconds: | 20260506 14:49:05.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:49:05.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:05.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:05.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:05.001 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 14:49:05.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:05.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:05.001 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:49:05.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:05.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:05.001 | INFO | Uploaded 36GiB to GPU in 1.96105s (18.3575GiB/s) | 20260506 14:49:05.001 | INFO | Downloaded 34GiB to GPU in 1.94478s (17.4827GiB/s) | 20260506 14:49:10.141 | INFO | Analyzing chunk frames = [2196992,2197504) | 20260506 14:49:10.141 | INFO | MemInfo: { | 20260506 14:49:10.141 | INFO | "gpu" : " 17 GB", | 20260506 14:49:10.141 | INFO | "huge" : "", | 20260506 14:49:10.141 | INFO | "huge_shared" : " 27 GB", | 20260506 14:49:10.141 | INFO | "malloc" : " 14 GB", | 20260506 14:49:10.141 | INFO | "pinned" : " 29 GB" | 20260506 14:49:10.141 | INFO | } | 20260506 14:49:10.141 | INFO | Perf report for the last 5.14014 seconds: | 20260506 14:49:10.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:49:10.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:10.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:10.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:10.141 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:49:10.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:10.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:10.141 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:49:10.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:10.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:10.142 | INFO | Uploaded 36GiB to GPU in 2.01836s (17.8363GiB/s) | 20260506 14:49:10.142 | INFO | Downloaded 34GiB to GPU in 1.91997s (17.7086GiB/s) | 20260506 14:49:15.281 | INFO | Analyzing chunk frames = [2197504,2198016) | 20260506 14:49:15.281 | INFO | MemInfo: { | 20260506 14:49:15.281 | INFO | "gpu" : " 17 GB", | 20260506 14:49:15.281 | INFO | "huge" : "", | 20260506 14:49:15.282 | INFO | "huge_shared" : " 27 GB", | 20260506 14:49:15.282 | INFO | "malloc" : " 14 GB", | 20260506 14:49:15.282 | INFO | "pinned" : " 29 GB" | 20260506 14:49:15.282 | INFO | } | 20260506 14:49:15.282 | INFO | Perf report for the last 5.1405 seconds: | 20260506 14:49:15.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:49:15.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:15.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:15.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:15.282 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 14:49:15.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:15.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:15.282 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 14:49:15.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:15.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:15.282 | INFO | Uploaded 36GiB to GPU in 2.01853s (17.8347GiB/s) | 20260506 14:49:15.282 | INFO | Downloaded 34GiB to GPU in 1.92915s (17.6243GiB/s) | 20260506 14:49:20.420 | INFO | Analyzing chunk frames = [2198016,2198528) | 20260506 14:49:20.420 | INFO | MemInfo: { | 20260506 14:49:20.420 | INFO | "gpu" : " 17 GB", | 20260506 14:49:20.420 | INFO | "huge" : "", | 20260506 14:49:20.420 | INFO | "huge_shared" : " 27 GB", | 20260506 14:49:20.420 | INFO | "malloc" : " 14 GB", | 20260506 14:49:20.420 | INFO | "pinned" : " 29 GB" | 20260506 14:49:20.420 | INFO | } | 20260506 14:49:20.420 | INFO | Perf report for the last 5.13844 seconds: | 20260506 14:49:20.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:49:20.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:20.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:20.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:20.420 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 14:49:20.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:20.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:20.420 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260506 14:49:20.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:20.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:20.420 | INFO | Uploaded 36GiB to GPU in 2.01769s (17.8422GiB/s) | 20260506 14:49:20.421 | INFO | Downloaded 34GiB to GPU in 1.92166s (17.693GiB/s) | 20260506 14:49:25.559 | INFO | Analyzing chunk frames = [2198528,2199040) | 20260506 14:49:25.559 | INFO | MemInfo: { | 20260506 14:49:25.559 | INFO | "gpu" : " 17 GB", | 20260506 14:49:25.559 | INFO | "huge" : "", | 20260506 14:49:25.559 | INFO | "huge_shared" : " 27 GB", | 20260506 14:49:25.560 | INFO | "malloc" : " 14 GB", | 20260506 14:49:25.560 | INFO | "pinned" : " 29 GB" | 20260506 14:49:25.560 | INFO | } | 20260506 14:49:25.560 | INFO | Perf report for the last 5.13952 seconds: | 20260506 14:49:25.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:49:25.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:25.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:25.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:25.560 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 14:49:25.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:25.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:25.560 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:49:25.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:25.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:25.560 | INFO | Uploaded 36GiB to GPU in 1.98231s (18.1606GiB/s) | 20260506 14:49:25.560 | INFO | Downloaded 34GiB to GPU in 1.98057s (17.1668GiB/s) | 20260506 14:49:30.698 | INFO | Analyzing chunk frames = [2199040,2199552) | 20260506 14:49:30.698 | INFO | MemInfo: { | 20260506 14:49:30.698 | INFO | "gpu" : " 17 GB", | 20260506 14:49:30.698 | INFO | "huge" : "", | 20260506 14:49:30.698 | INFO | "huge_shared" : " 27 GB", | 20260506 14:49:30.698 | INFO | "malloc" : " 14 GB", | 20260506 14:49:30.698 | INFO | "pinned" : " 29 GB" | 20260506 14:49:30.698 | INFO | } | 20260506 14:49:30.698 | INFO | Perf report for the last 5.13829 seconds: | 20260506 14:49:30.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:49:30.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:30.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:30.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:30.698 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:49:30.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:30.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:30.698 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:49:30.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:30.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:30.698 | INFO | Uploaded 36GiB to GPU in 1.99449s (18.0497GiB/s) | 20260506 14:49:30.698 | INFO | Downloaded 34GiB to GPU in 1.98738s (17.108GiB/s) | 20260506 14:49:30.808 | INFO | Starting a round of Baz IO | 20260506 14:49:33.968 | INFO | Populating RTMetrics report | 20260506 14:49:33.969 | INFO | RTMetrics report populated | 20260506 14:49:33.969 | INFO | Opening tmp RTMetrics json file | 20260506 14:49:33.970 | INFO | Serializing RTMetrics to json | 20260506 14:49:33.971 | INFO | Serialization done, swapping tmp file now... | 20260506 14:49:33.972 | INFO | RTMetrics done | 20260506 14:49:34.137 | INFO | Finished a round of Baz IO: Wrote 2.108 GiB in 3.329 seconds | 20260506 14:49:34.137 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.86% / 0.01% / 62.13% | 20260506 14:49:34.137 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.35165% / 1.58946e-05% / 3.97364e-05% | 20260506 14:49:34.137 | INFO | Baz IO overall: | 20260506 14:49:34.137 | INFO | BytesWritten : 1139894539670 | 20260506 14:49:34.137 | INFO | NumEvents : 551637067684 | 20260506 14:49:35.837 | INFO | Analyzing chunk frames = [2199552,2200064) | 20260506 14:49:35.837 | INFO | MemInfo: { | 20260506 14:49:35.837 | INFO | "gpu" : " 17 GB", | 20260506 14:49:35.837 | INFO | "huge" : "", | 20260506 14:49:35.837 | INFO | "huge_shared" : " 27 GB", | 20260506 14:49:35.837 | INFO | "malloc" : " 14 GB", | 20260506 14:49:35.837 | INFO | "pinned" : " 29 GB" | 20260506 14:49:35.837 | INFO | } | 20260506 14:49:35.837 | INFO | Perf report for the last 5.13933 seconds: | 20260506 14:49:35.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:49:35.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:35.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:35.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:35.837 | INFO | ANALYSIS : 63.0%, 2.9, 95.4ms, 0 | 20260506 14:49:35.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:35.838 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 14:49:35.838 | INFO | PRE_HQ : 61.0%, 1.9, 59.6ms, 0 | 20260506 14:49:35.838 | INFO | BAZWRITER : 64.2%, 4.7, 154.0ms, 0 | 20260506 14:49:35.838 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 14:49:35.838 | INFO | Uploaded 36GiB to GPU in 2.03871s (17.6582GiB/s) | 20260506 14:49:35.838 | INFO | Downloaded 38GiB to GPU in 2.23272s (17.0196GiB/s) | 20260506 14:49:40.976 | INFO | Analyzing chunk frames = [2200064,2200576) | 20260506 14:49:40.976 | INFO | MemInfo: { | 20260506 14:49:40.976 | INFO | "gpu" : " 17 GB", | 20260506 14:49:40.976 | INFO | "huge" : "", | 20260506 14:49:40.977 | INFO | "huge_shared" : " 27 GB", | 20260506 14:49:40.977 | INFO | "malloc" : " 14 GB", | 20260506 14:49:40.977 | INFO | "pinned" : " 29 GB" | 20260506 14:49:40.977 | INFO | } | 20260506 14:49:40.977 | INFO | Perf report for the last 5.13933 seconds: | 20260506 14:49:40.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:49:40.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:40.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:40.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:40.977 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 14:49:40.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:40.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:40.977 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:49:40.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:40.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:40.977 | INFO | Uploaded 36GiB to GPU in 1.98511s (18.135GiB/s) | 20260506 14:49:40.977 | INFO | Downloaded 34GiB to GPU in 1.93923s (17.5327GiB/s) | 20260506 14:49:46.116 | INFO | Analyzing chunk frames = [2200576,2201088) | 20260506 14:49:46.117 | INFO | MemInfo: { | 20260506 14:49:46.117 | INFO | "gpu" : " 17 GB", | 20260506 14:49:46.117 | INFO | "huge" : "", | 20260506 14:49:46.117 | INFO | "huge_shared" : " 27 GB", | 20260506 14:49:46.117 | INFO | "malloc" : " 14 GB", | 20260506 14:49:46.117 | INFO | "pinned" : " 29 GB" | 20260506 14:49:46.117 | INFO | } | 20260506 14:49:46.117 | INFO | Perf report for the last 5.14004 seconds: | 20260506 14:49:46.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:49:46.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:46.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:46.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:46.117 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 14:49:46.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:46.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:46.117 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:49:46.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:46.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:46.117 | INFO | Uploaded 36GiB to GPU in 1.94872s (18.4737GiB/s) | 20260506 14:49:46.117 | INFO | Downloaded 34GiB to GPU in 1.93877s (17.5369GiB/s) | 20260506 14:49:51.255 | INFO | Analyzing chunk frames = [2201088,2201600) | 20260506 14:49:51.255 | INFO | MemInfo: { | 20260506 14:49:51.255 | INFO | "gpu" : " 17 GB", | 20260506 14:49:51.255 | INFO | "huge" : "", | 20260506 14:49:51.255 | INFO | "huge_shared" : " 27 GB", | 20260506 14:49:51.255 | INFO | "malloc" : " 14 GB", | 20260506 14:49:51.255 | INFO | "pinned" : " 29 GB" | 20260506 14:49:51.255 | INFO | } | 20260506 14:49:51.255 | INFO | Perf report for the last 5.13844 seconds: | 20260506 14:49:51.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:49:51.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:51.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:51.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:51.255 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260506 14:49:51.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:51.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:51.255 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:49:51.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:51.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:51.256 | INFO | Uploaded 36GiB to GPU in 1.94675s (18.4924GiB/s) | 20260506 14:49:51.256 | INFO | Downloaded 34GiB to GPU in 1.88468s (18.0402GiB/s) | 20260506 14:49:56.395 | INFO | Analyzing chunk frames = [2201600,2202112) | 20260506 14:49:56.395 | INFO | MemInfo: { | 20260506 14:49:56.395 | INFO | "gpu" : " 17 GB", | 20260506 14:49:56.395 | INFO | "huge" : "", | 20260506 14:49:56.395 | INFO | "huge_shared" : " 27 GB", | 20260506 14:49:56.395 | INFO | "malloc" : " 14 GB", | 20260506 14:49:56.395 | INFO | "pinned" : " 29 GB" | 20260506 14:49:56.395 | INFO | } | 20260506 14:49:56.396 | INFO | Perf report for the last 5.14031 seconds: | 20260506 14:49:56.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:49:56.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:56.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:56.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:56.396 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 14:49:56.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:56.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:56.396 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:49:56.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:56.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:49:56.397 | INFO | Uploaded 36GiB to GPU in 2.00497s (17.9554GiB/s) | 20260506 14:49:56.397 | INFO | Downloaded 34GiB to GPU in 1.92348s (17.6763GiB/s) | 20260506 14:50:01.535 | INFO | Analyzing chunk frames = [2202112,2202624) | 20260506 14:50:01.535 | INFO | MemInfo: { | 20260506 14:50:01.535 | INFO | "gpu" : " 17 GB", | 20260506 14:50:01.535 | INFO | "huge" : "", | 20260506 14:50:01.535 | INFO | "huge_shared" : " 27 GB", | 20260506 14:50:01.535 | INFO | "malloc" : " 14 GB", | 20260506 14:50:01.535 | INFO | "pinned" : " 29 GB" | 20260506 14:50:01.535 | INFO | } | 20260506 14:50:01.535 | INFO | Perf report for the last 5.13962 seconds: | 20260506 14:50:01.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:50:01.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:01.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:01.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:01.535 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 14:50:01.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:01.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:01.535 | INFO | PRE_HQ : 32.3%, 1.0, 16.7ms, 0 | 20260506 14:50:01.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:01.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:01.535 | INFO | Uploaded 36GiB to GPU in 2.0209s (17.8139GiB/s) | 20260506 14:50:01.536 | INFO | Downloaded 34GiB to GPU in 1.92463s (17.6657GiB/s) | 20260506 14:50:06.674 | INFO | Analyzing chunk frames = [2202624,2203136) | 20260506 14:50:06.674 | INFO | MemInfo: { | 20260506 14:50:06.674 | INFO | "gpu" : " 17 GB", | 20260506 14:50:06.674 | INFO | "huge" : "", | 20260506 14:50:06.674 | INFO | "huge_shared" : " 27 GB", | 20260506 14:50:06.674 | INFO | "malloc" : " 14 GB", | 20260506 14:50:06.674 | INFO | "pinned" : " 29 GB" | 20260506 14:50:06.674 | INFO | } | 20260506 14:50:06.674 | INFO | Perf report for the last 5.13909 seconds: | 20260506 14:50:06.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:50:06.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:06.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:06.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:06.675 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 14:50:06.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:06.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:06.675 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:50:06.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:06.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:06.675 | INFO | Uploaded 36GiB to GPU in 1.97913s (18.1898GiB/s) | 20260506 14:50:06.675 | INFO | Downloaded 34GiB to GPU in 1.98285s (17.1471GiB/s) | 20260506 14:50:11.813 | INFO | Analyzing chunk frames = [2203136,2203648) | 20260506 14:50:11.813 | INFO | MemInfo: { | 20260506 14:50:11.813 | INFO | "gpu" : " 17 GB", | 20260506 14:50:11.813 | INFO | "huge" : "", | 20260506 14:50:11.813 | INFO | "huge_shared" : " 27 GB", | 20260506 14:50:11.813 | INFO | "malloc" : " 14 GB", | 20260506 14:50:11.813 | INFO | "pinned" : " 29 GB" | 20260506 14:50:11.813 | INFO | } | 20260506 14:50:11.813 | INFO | Perf report for the last 5.13849 seconds: | 20260506 14:50:11.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:50:11.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:11.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:11.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:11.813 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:50:11.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:11.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:11.813 | INFO | PRE_HQ : 32.1%, 1.0, 16.6ms, 0 | 20260506 14:50:11.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:11.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:11.813 | INFO | Uploaded 36GiB to GPU in 2.01563s (17.8605GiB/s) | 20260506 14:50:11.813 | INFO | Downloaded 34GiB to GPU in 1.98589s (17.1208GiB/s) | 20260506 14:50:11.813 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:50:11.813 | INFO | Total mem usage: 17.4 GiB | 20260506 14:50:11.813 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:50:11.814 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -15.4%) | 20260506 14:50:11.814 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:50:11.814 | INFO | High Water Breakdown: | 20260506 14:50:11.814 | INFO | 60.7% : 9.16 GiB ( 83.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:50:11.814 | INFO | 39.2% : 5.91 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:50:11.814 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:50:11.814 | INFO | ------------------------------------------ | 20260506 14:50:11.814 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:50:11.814 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:50:11.814 | INFO | Total mem usage: 26.5 GiB | 20260506 14:50:11.814 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:50:11.814 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:50:11.814 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:50:11.814 | INFO | High Water Breakdown: | 20260506 14:50:11.814 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:50:11.814 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:50:11.814 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:50:11.814 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:50:11.814 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:50:11.814 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:50:11.814 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:50:11.814 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:50:11.814 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:50:11.814 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:50:11.814 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:50:11.814 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:50:11.814 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:50:11.814 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:50:11.814 | INFO | ------------------------------------------ | 20260506 14:50:11.814 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:50:11.814 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:50:11.814 | INFO | Total mem usage: 25.6 GiB | 20260506 14:50:11.814 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:50:11.814 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:50:11.814 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:50:11.814 | INFO | High Water Breakdown: | 20260506 14:50:11.814 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:50:11.814 | INFO | ------------------------------------------ | 20260506 14:50:11.814 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:50:11.814 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:50:11.814 | INFO | Total mem usage: 15.7 GiB | 20260506 14:50:11.814 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:50:11.814 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:50:11.814 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 14:50:11.814 | INFO | High Water Breakdown: | 20260506 14:50:11.814 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:50:11.814 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:50:11.814 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:50:11.814 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:50:11.814 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:50:11.814 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:50:11.814 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:50:11.814 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:50:11.814 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:50:11.814 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:50:11.814 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:50:11.814 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:50:11.814 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:50:11.814 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:50:11.814 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:50:11.814 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:50:11.814 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:50:11.814 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:50:11.814 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:50:11.814 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:50:11.814 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:50:11.814 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:50:11.814 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:50:11.814 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:50:11.814 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:50:11.814 | INFO | ------------------------------------------ | 20260506 14:50:11.814 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:50:11.814 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:50:11.814 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:50:11.814 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:50:11.814 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.52e+01, 1.33e+01 / 1.83e+00 | 20260506 14:50:11.814 | INFO | PerfReport: 0.29%, AggregateStats -- 9.23e-02 / 1.01e-01, 1.09e-02 / 6.12e-03 | 20260506 14:50:11.814 | INFO | PerfReport: 11.61%, Baseline -- 3.66e+00 / 4.43e+00, 5.77e-01 / 3.85e-01 | 20260506 14:50:11.814 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.45e+00, 1.90e-01 / 1.46e-01 | 20260506 14:50:11.814 | INFO | PerfReport: 15.79%, DME -- 3.98e+01 / 4.37e+01, 3.72e+00 / 1.95e+00 | 20260506 14:50:11.815 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.53e+00 / 1.89e+00 | 20260506 14:50:11.815 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.33e+00 / 2.92e+00, 3.95e-01 / 2.42e-01 | 20260506 14:50:11.815 | INFO | PerfReport: 3.59%, Metrics -- 1.13e+00 / 1.50e+00, 2.95e-01 / 1.97e-01 | 20260506 14:50:11.815 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.94e-01 / 4.46e-01, 6.13e-02 / 2.47e-02 | 20260506 14:50:11.815 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:50:11.815 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:50:11.815 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:50:11.815 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 14:50:11.815 | INFO | PerfReport: 21.33%, Upload -- 2.08e+01 / 0.00e+00, 6.84e+00 / 0.00e+00 | 20260506 14:50:11.815 | INFO | PerfReport: 21.13%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 14:50:11.815 | INFO | PerfReport: 32.33%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 14:50:11.815 | INFO | PerfReport: 1.41%, LockUpload -- 1.37e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260506 14:50:11.815 | INFO | PerfReport: 1.22%, LockDownload -- 1.19e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260506 14:50:11.815 | INFO | PerfReport: 22.56%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 14:50:11.815 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260506 14:50:11.925 | INFO | Starting a round of Baz IO | 20260506 14:50:15.081 | INFO | Populating RTMetrics report | 20260506 14:50:15.081 | INFO | RTMetrics report populated | 20260506 14:50:15.081 | INFO | Opening tmp RTMetrics json file | 20260506 14:50:15.127 | INFO | Serializing RTMetrics to json | 20260506 14:50:15.192 | INFO | Serialization done, swapping tmp file now... | 20260506 14:50:15.228 | INFO | RTMetrics done | 20260506 14:50:15.350 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.425 seconds | 20260506 14:50:15.351 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.84% / 0.01% / 62.14% | 20260506 14:50:15.351 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.34623% / 1.98682e-05% / 5.5631e-05% | 20260506 14:50:15.351 | INFO | Baz IO overall: | 20260506 14:50:15.351 | INFO | BytesWritten : 1142158738318 | 20260506 14:50:15.351 | INFO | NumEvents : 552733690716 | 20260506 14:50:16.952 | INFO | Analyzing chunk frames = [2203648,2204160) | 20260506 14:50:16.952 | INFO | MemInfo: { | 20260506 14:50:16.952 | INFO | "gpu" : " 17 GB", | 20260506 14:50:16.952 | INFO | "huge" : "", | 20260506 14:50:16.952 | INFO | "huge_shared" : " 27 GB", | 20260506 14:50:16.952 | INFO | "malloc" : " 14 GB", | 20260506 14:50:16.952 | INFO | "pinned" : " 29 GB" | 20260506 14:50:16.952 | INFO | } | 20260506 14:50:16.952 | INFO | Perf report for the last 5.13881 seconds: | 20260506 14:50:16.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:50:16.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:16.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:16.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:16.952 | INFO | ANALYSIS : 63.0%, 2.9, 94.7ms, 0 | 20260506 14:50:16.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:16.952 | INFO | RT_METRICS : 48.5%, 1.0, 25.4ms, 0 | 20260506 14:50:16.952 | INFO | PRE_HQ : 61.5%, 1.9, 61.8ms, 0 | 20260506 14:50:16.952 | INFO | BAZWRITER : 66.0%, 4.8, 162.3ms, 0 | 20260506 14:50:16.952 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 14:50:16.952 | INFO | Uploaded 36GiB to GPU in 2.05728s (17.4988GiB/s) | 20260506 14:50:16.952 | INFO | Downloaded 38GiB to GPU in 2.22933s (17.0455GiB/s) | 20260506 14:50:22.092 | INFO | Analyzing chunk frames = [2204160,2204672) | 20260506 14:50:22.092 | INFO | MemInfo: { | 20260506 14:50:22.092 | INFO | "gpu" : " 17 GB", | 20260506 14:50:22.092 | INFO | "huge" : "", | 20260506 14:50:22.092 | INFO | "huge_shared" : " 27 GB", | 20260506 14:50:22.092 | INFO | "malloc" : " 14 GB", | 20260506 14:50:22.092 | INFO | "pinned" : " 29 GB" | 20260506 14:50:22.093 | INFO | } | 20260506 14:50:22.093 | INFO | Perf report for the last 5.14111 seconds: | 20260506 14:50:22.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:50:22.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:22.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:22.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:22.093 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 14:50:22.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:22.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:22.093 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 14:50:22.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:22.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:22.093 | INFO | Uploaded 36GiB to GPU in 1.98429s (18.1425GiB/s) | 20260506 14:50:22.093 | INFO | Downloaded 34GiB to GPU in 1.9362s (17.5602GiB/s) | 20260506 14:50:27.232 | INFO | Analyzing chunk frames = [2204672,2205184) | 20260506 14:50:27.232 | INFO | MemInfo: { | 20260506 14:50:27.232 | INFO | "gpu" : " 17 GB", | 20260506 14:50:27.232 | INFO | "huge" : "", | 20260506 14:50:27.232 | INFO | "huge_shared" : " 27 GB", | 20260506 14:50:27.232 | INFO | "malloc" : " 14 GB", | 20260506 14:50:27.232 | INFO | "pinned" : " 29 GB" | 20260506 14:50:27.232 | INFO | } | 20260506 14:50:27.232 | INFO | Perf report for the last 5.13949 seconds: | 20260506 14:50:27.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:50:27.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:27.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:27.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:27.232 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 14:50:27.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:27.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:27.233 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:50:27.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:27.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:27.233 | INFO | Uploaded 36GiB to GPU in 1.96163s (18.352GiB/s) | 20260506 14:50:27.233 | INFO | Downloaded 34GiB to GPU in 1.94257s (17.5026GiB/s) | 20260506 14:50:32.371 | INFO | Analyzing chunk frames = [2205184,2205696) | 20260506 14:50:32.371 | INFO | MemInfo: { | 20260506 14:50:32.371 | INFO | "gpu" : " 17 GB", | 20260506 14:50:32.371 | INFO | "huge" : "", | 20260506 14:50:32.371 | INFO | "huge_shared" : " 27 GB", | 20260506 14:50:32.371 | INFO | "malloc" : " 14 GB", | 20260506 14:50:32.372 | INFO | "pinned" : " 29 GB" | 20260506 14:50:32.372 | INFO | } | 20260506 14:50:32.372 | INFO | Perf report for the last 5.13941 seconds: | 20260506 14:50:32.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:50:32.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:32.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:32.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:32.372 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:50:32.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:32.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:32.372 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:50:32.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:32.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:32.372 | INFO | Uploaded 36GiB to GPU in 2.01733s (17.8454GiB/s) | 20260506 14:50:32.372 | INFO | Downloaded 34GiB to GPU in 1.91745s (17.7319GiB/s) | 20260506 14:50:37.509 | INFO | Analyzing chunk frames = [2205696,2206208) | 20260506 14:50:37.509 | INFO | MemInfo: { | 20260506 14:50:37.509 | INFO | "gpu" : " 17 GB", | 20260506 14:50:37.509 | INFO | "huge" : "", | 20260506 14:50:37.509 | INFO | "huge_shared" : " 27 GB", | 20260506 14:50:37.509 | INFO | "malloc" : " 14 GB", | 20260506 14:50:37.509 | INFO | "pinned" : " 29 GB" | 20260506 14:50:37.509 | INFO | } | 20260506 14:50:37.509 | INFO | Perf report for the last 5.13744 seconds: | 20260506 14:50:37.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:50:37.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:37.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:37.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:37.509 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 14:50:37.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:37.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:37.509 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:50:37.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:37.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:37.509 | INFO | Uploaded 36GiB to GPU in 2.00125s (17.9887GiB/s) | 20260506 14:50:37.509 | INFO | Downloaded 34GiB to GPU in 1.94535s (17.4776GiB/s) | 20260506 14:50:42.648 | INFO | Analyzing chunk frames = [2206208,2206720) | 20260506 14:50:42.648 | INFO | MemInfo: { | 20260506 14:50:42.648 | INFO | "gpu" : " 17 GB", | 20260506 14:50:42.648 | INFO | "huge" : "", | 20260506 14:50:42.648 | INFO | "huge_shared" : " 27 GB", | 20260506 14:50:42.648 | INFO | "malloc" : " 14 GB", | 20260506 14:50:42.648 | INFO | "pinned" : " 29 GB" | 20260506 14:50:42.648 | INFO | } | 20260506 14:50:42.649 | INFO | Perf report for the last 5.13942 seconds: | 20260506 14:50:42.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:50:42.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:42.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:42.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:42.649 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:50:42.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:42.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:42.649 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:50:42.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:42.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:42.649 | INFO | Uploaded 36GiB to GPU in 2.04978s (17.5629GiB/s) | 20260506 14:50:42.649 | INFO | Downloaded 34GiB to GPU in 1.92068s (17.702GiB/s) | 20260506 14:50:47.789 | INFO | Analyzing chunk frames = [2206720,2207232) | 20260506 14:50:47.789 | INFO | MemInfo: { | 20260506 14:50:47.789 | INFO | "gpu" : " 17 GB", | 20260506 14:50:47.789 | INFO | "huge" : "", | 20260506 14:50:47.789 | INFO | "huge_shared" : " 27 GB", | 20260506 14:50:47.789 | INFO | "malloc" : " 14 GB", | 20260506 14:50:47.789 | INFO | "pinned" : " 29 GB" | 20260506 14:50:47.789 | INFO | } | 20260506 14:50:47.789 | INFO | Perf report for the last 5.14086 seconds: | 20260506 14:50:47.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:50:47.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:47.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:47.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:47.790 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 14:50:47.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:47.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:47.790 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:50:47.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:47.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:47.790 | INFO | Uploaded 36GiB to GPU in 2.01853s (17.8348GiB/s) | 20260506 14:50:47.790 | INFO | Downloaded 34GiB to GPU in 1.99417s (17.0497GiB/s) | 20260506 14:50:52.927 | INFO | Analyzing chunk frames = [2207232,2207744) | 20260506 14:50:52.927 | INFO | MemInfo: { | 20260506 14:50:52.927 | INFO | "gpu" : " 17 GB", | 20260506 14:50:52.927 | INFO | "huge" : "", | 20260506 14:50:52.927 | INFO | "huge_shared" : " 27 GB", | 20260506 14:50:52.927 | INFO | "malloc" : " 14 GB", | 20260506 14:50:52.927 | INFO | "pinned" : " 29 GB" | 20260506 14:50:52.927 | INFO | } | 20260506 14:50:52.927 | INFO | Perf report for the last 5.13805 seconds: | 20260506 14:50:52.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:50:52.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:52.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:52.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:52.928 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:50:52.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:52.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:52.928 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 14:50:52.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:52.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:52.928 | INFO | Uploaded 36GiB to GPU in 2.01423s (17.8729GiB/s) | 20260506 14:50:52.928 | INFO | Downloaded 34GiB to GPU in 1.97659s (17.2013GiB/s) | 20260506 14:50:53.039 | INFO | Starting a round of Baz IO | 20260506 14:50:56.205 | INFO | Populating RTMetrics report | 20260506 14:50:56.206 | INFO | RTMetrics report populated | 20260506 14:50:56.206 | INFO | Opening tmp RTMetrics json file | 20260506 14:50:56.253 | INFO | Serializing RTMetrics to json | 20260506 14:50:56.320 | INFO | Serialization done, swapping tmp file now... | 20260506 14:50:56.350 | INFO | RTMetrics done | 20260506 14:50:56.637 | INFO | Finished a round of Baz IO: Wrote 2.106 GiB in 3.598 seconds | 20260506 14:50:56.637 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.83% / 0.01% / 62.16% | 20260506 14:50:56.637 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.33342% / 4.76837e-05% / 5.16574e-05% | 20260506 14:50:56.637 | INFO | Baz IO overall: | 20260506 14:50:56.637 | INFO | BytesWritten : 1144420164307 | 20260506 14:50:56.637 | INFO | NumEvents : 553828044897 | 20260506 14:50:58.067 | INFO | Analyzing chunk frames = [2207744,2208256) | 20260506 14:50:58.067 | INFO | MemInfo: { | 20260506 14:50:58.067 | INFO | "gpu" : " 17 GB", | 20260506 14:50:58.067 | INFO | "huge" : "", | 20260506 14:50:58.067 | INFO | "huge_shared" : " 27 GB", | 20260506 14:50:58.067 | INFO | "malloc" : " 14 GB", | 20260506 14:50:58.067 | INFO | "pinned" : " 29 GB" | 20260506 14:50:58.067 | INFO | } | 20260506 14:50:58.067 | INFO | Perf report for the last 5.13938 seconds: | 20260506 14:50:58.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:50:58.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:58.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:58.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:58.067 | INFO | ANALYSIS : 63.2%, 2.9, 95.0ms, 0 | 20260506 14:50:58.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:50:58.067 | INFO | RT_METRICS : 47.4%, 1.0, 24.9ms, 0 | 20260506 14:50:58.067 | INFO | PRE_HQ : 61.3%, 1.9, 60.8ms, 0 | 20260506 14:50:58.067 | INFO | BAZWRITER : 69.4%, 4.8, 171.0ms, 0 | 20260506 14:50:58.067 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 14:50:58.067 | INFO | Uploaded 36GiB to GPU in 2.07527s (17.3471GiB/s) | 20260506 14:50:58.067 | INFO | Downloaded 38GiB to GPU in 2.22243s (17.0984GiB/s) | 20260506 14:51:03.206 | INFO | Analyzing chunk frames = [2208256,2208768) | 20260506 14:51:03.206 | INFO | MemInfo: { | 20260506 14:51:03.206 | INFO | "gpu" : " 17 GB", | 20260506 14:51:03.206 | INFO | "huge" : "", | 20260506 14:51:03.206 | INFO | "huge_shared" : " 27 GB", | 20260506 14:51:03.206 | INFO | "malloc" : " 14 GB", | 20260506 14:51:03.206 | INFO | "pinned" : " 29 GB" | 20260506 14:51:03.207 | INFO | } | 20260506 14:51:03.207 | INFO | Perf report for the last 5.13967 seconds: | 20260506 14:51:03.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:51:03.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:03.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:03.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:03.207 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 14:51:03.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:03.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:03.207 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:51:03.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:03.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:03.207 | INFO | Uploaded 36GiB to GPU in 2.00134s (17.988GiB/s) | 20260506 14:51:03.207 | INFO | Downloaded 34GiB to GPU in 1.94174s (17.5101GiB/s) | 20260506 14:51:08.345 | INFO | Analyzing chunk frames = [2208768,2209280) | 20260506 14:51:08.345 | INFO | MemInfo: { | 20260506 14:51:08.345 | INFO | "gpu" : " 17 GB", | 20260506 14:51:08.345 | INFO | "huge" : "", | 20260506 14:51:08.345 | INFO | "huge_shared" : " 27 GB", | 20260506 14:51:08.346 | INFO | "malloc" : " 14 GB", | 20260506 14:51:08.346 | INFO | "pinned" : " 29 GB" | 20260506 14:51:08.346 | INFO | } | 20260506 14:51:08.346 | INFO | Perf report for the last 5.13911 seconds: | 20260506 14:51:08.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:51:08.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:08.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:08.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:08.346 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 14:51:08.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:08.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:08.346 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:51:08.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:08.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:08.346 | INFO | Uploaded 36GiB to GPU in 1.98841s (18.1049GiB/s) | 20260506 14:51:08.346 | INFO | Downloaded 34GiB to GPU in 1.94486s (17.482GiB/s) | 20260506 14:51:13.485 | INFO | Analyzing chunk frames = [2209280,2209792) | 20260506 14:51:13.485 | INFO | MemInfo: { | 20260506 14:51:13.485 | INFO | "gpu" : " 17 GB", | 20260506 14:51:13.485 | INFO | "huge" : "", | 20260506 14:51:13.485 | INFO | "huge_shared" : " 27 GB", | 20260506 14:51:13.485 | INFO | "malloc" : " 14 GB", | 20260506 14:51:13.485 | INFO | "pinned" : " 29 GB" | 20260506 14:51:13.485 | INFO | } | 20260506 14:51:13.485 | INFO | Perf report for the last 5.13931 seconds: | 20260506 14:51:13.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:51:13.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:13.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:13.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:13.485 | INFO | ANALYSIS : 60.6%, 3.0, 94.0ms, 0 | 20260506 14:51:13.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:13.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:13.485 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:51:13.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:13.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:13.485 | INFO | Uploaded 36GiB to GPU in 1.97933s (18.188GiB/s) | 20260506 14:51:13.485 | INFO | Downloaded 34GiB to GPU in 1.91177s (17.7846GiB/s) | 20260506 14:51:18.625 | INFO | Analyzing chunk frames = [2209792,2210304) | 20260506 14:51:18.625 | INFO | MemInfo: { | 20260506 14:51:18.625 | INFO | "gpu" : " 17 GB", | 20260506 14:51:18.625 | INFO | "huge" : "", | 20260506 14:51:18.625 | INFO | "huge_shared" : " 27 GB", | 20260506 14:51:18.626 | INFO | "malloc" : " 14 GB", | 20260506 14:51:18.626 | INFO | "pinned" : " 29 GB" | 20260506 14:51:18.626 | INFO | } | 20260506 14:51:18.626 | INFO | Perf report for the last 5.14061 seconds: | 20260506 14:51:18.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:51:18.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:18.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:18.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:18.626 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 14:51:18.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:18.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:18.626 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:51:18.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:18.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:18.626 | INFO | Uploaded 36GiB to GPU in 2.0061s (17.9453GiB/s) | 20260506 14:51:18.626 | INFO | Downloaded 34GiB to GPU in 1.93846s (17.5397GiB/s) | 20260506 14:51:23.763 | INFO | Analyzing chunk frames = [2210304,2210816) | 20260506 14:51:23.763 | INFO | MemInfo: { | 20260506 14:51:23.763 | INFO | "gpu" : " 17 GB", | 20260506 14:51:23.763 | INFO | "huge" : "", | 20260506 14:51:23.763 | INFO | "huge_shared" : " 27 GB", | 20260506 14:51:23.763 | INFO | "malloc" : " 14 GB", | 20260506 14:51:23.763 | INFO | "pinned" : " 29 GB" | 20260506 14:51:23.763 | INFO | } | 20260506 14:51:23.764 | INFO | Perf report for the last 5.13792 seconds: | 20260506 14:51:23.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:51:23.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:23.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:23.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:23.764 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260506 14:51:23.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:23.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:23.764 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:51:23.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:23.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:23.764 | INFO | Uploaded 36GiB to GPU in 2.01197s (17.8929GiB/s) | 20260506 14:51:23.764 | INFO | Downloaded 34GiB to GPU in 1.92552s (17.6575GiB/s) | 20260506 14:51:28.903 | INFO | Analyzing chunk frames = [2210816,2211328) | 20260506 14:51:28.903 | INFO | MemInfo: { | 20260506 14:51:28.903 | INFO | "gpu" : " 17 GB", | 20260506 14:51:28.903 | INFO | "huge" : "", | 20260506 14:51:28.903 | INFO | "huge_shared" : " 27 GB", | 20260506 14:51:28.903 | INFO | "malloc" : " 14 GB", | 20260506 14:51:28.903 | INFO | "pinned" : " 29 GB" | 20260506 14:51:28.903 | INFO | } | 20260506 14:51:28.904 | INFO | Perf report for the last 5.14006 seconds: | 20260506 14:51:28.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:51:28.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:28.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:28.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:28.904 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:51:28.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:28.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:28.904 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:51:28.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:28.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:28.904 | INFO | Uploaded 36GiB to GPU in 1.98617s (18.1254GiB/s) | 20260506 14:51:28.904 | INFO | Downloaded 34GiB to GPU in 1.97671s (17.2003GiB/s) | 20260506 14:51:34.042 | INFO | Analyzing chunk frames = [2211328,2211840) | 20260506 14:51:34.042 | INFO | MemInfo: { | 20260506 14:51:34.042 | INFO | "gpu" : " 17 GB", | 20260506 14:51:34.042 | INFO | "huge" : "", | 20260506 14:51:34.043 | INFO | "huge_shared" : " 27 GB", | 20260506 14:51:34.043 | INFO | "malloc" : " 14 GB", | 20260506 14:51:34.043 | INFO | "pinned" : " 29 GB" | 20260506 14:51:34.043 | INFO | } | 20260506 14:51:34.043 | INFO | Perf report for the last 5.13902 seconds: | 20260506 14:51:34.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:51:34.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:34.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:34.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:34.043 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 14:51:34.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:34.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:34.043 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:51:34.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:34.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:34.043 | INFO | Uploaded 36GiB to GPU in 2.00415s (17.9627GiB/s) | 20260506 14:51:34.043 | INFO | Downloaded 34GiB to GPU in 1.98253s (17.1498GiB/s) | 20260506 14:51:34.043 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:51:34.043 | INFO | Total mem usage: 17.4 GiB | 20260506 14:51:34.043 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:51:34.043 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -14.8%) | 20260506 14:51:34.043 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:51:34.043 | INFO | High Water Breakdown: | 20260506 14:51:34.043 | INFO | 60.7% : 9.2 GiB ( 84.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:51:34.043 | INFO | 39.2% : 5.94 GiB ( 88.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:51:34.043 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:51:34.043 | INFO | ------------------------------------------ | 20260506 14:51:34.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:51:34.043 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:51:34.043 | INFO | Total mem usage: 26.5 GiB | 20260506 14:51:34.043 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:51:34.043 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:51:34.043 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:51:34.043 | INFO | High Water Breakdown: | 20260506 14:51:34.043 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:51:34.043 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:51:34.043 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:51:34.043 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:51:34.043 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:51:34.043 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:51:34.043 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:51:34.043 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:51:34.043 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:51:34.044 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:51:34.044 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:51:34.044 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:51:34.044 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:51:34.044 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:51:34.044 | INFO | ------------------------------------------ | 20260506 14:51:34.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:51:34.044 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:51:34.044 | INFO | Total mem usage: 25.6 GiB | 20260506 14:51:34.044 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:51:34.044 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:51:34.044 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:51:34.044 | INFO | High Water Breakdown: | 20260506 14:51:34.044 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:51:34.044 | INFO | ------------------------------------------ | 20260506 14:51:34.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:51:34.044 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:51:34.044 | INFO | Total mem usage: 15.7 GiB | 20260506 14:51:34.044 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:51:34.044 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:51:34.044 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 14:51:34.044 | INFO | High Water Breakdown: | 20260506 14:51:34.044 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:51:34.044 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:51:34.044 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:51:34.044 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:51:34.044 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:51:34.044 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:51:34.044 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:51:34.044 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:51:34.044 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:51:34.044 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:51:34.044 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:51:34.044 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:51:34.044 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:51:34.044 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:51:34.044 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:51:34.044 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:51:34.044 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:51:34.044 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:51:34.044 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:51:34.044 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:51:34.044 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:51:34.044 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:51:34.044 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:51:34.044 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:51:34.044 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:51:34.044 | INFO | ------------------------------------------ | 20260506 14:51:34.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:51:34.044 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:51:34.044 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:51:34.044 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:51:34.044 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.52e+01, 1.33e+01 / 1.63e+00 | 20260506 14:51:34.044 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.01e-01, 1.11e-02 / 7.18e-03 | 20260506 14:51:34.044 | INFO | PerfReport: 11.57%, Baseline -- 3.65e+00 / 4.36e+00, 5.72e-01 / 3.85e-01 | 20260506 14:51:34.044 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.46e+00, 1.91e-01 / 1.48e-01 | 20260506 14:51:34.044 | INFO | PerfReport: 15.77%, DME -- 3.98e+01 / 4.34e+01, 3.41e+00 / 1.70e+00 | 20260506 14:51:34.044 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.79e+01 / 2.11e+01, 1.55e+00 / 1.96e+00 | 20260506 14:51:34.044 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.33e+00 / 2.85e+00, 3.97e-01 / 3.07e-01 | 20260506 14:51:34.044 | INFO | PerfReport: 3.58%, Metrics -- 1.13e+00 / 1.47e+00, 2.91e-01 / 2.00e-01 | 20260506 14:51:34.044 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.93e-01 / 4.46e-01, 6.18e-02 / 3.28e-02 | 20260506 14:51:34.044 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:51:34.044 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:51:34.044 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:51:34.044 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 14:51:34.044 | INFO | PerfReport: 21.47%, Upload -- 2.09e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 14:51:34.044 | INFO | PerfReport: 21.17%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 14:51:34.044 | INFO | PerfReport: 32.33%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 14:51:34.044 | INFO | PerfReport: 1.42%, LockUpload -- 1.39e+00 / 0.00e+00, 5.84e+00 / 0.00e+00 | 20260506 14:51:34.044 | INFO | PerfReport: 1.18%, LockDownload -- 1.15e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 14:51:34.044 | INFO | PerfReport: 22.41%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 14:51:34.044 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.56e-02 / 0.00e+00 | 20260506 14:51:34.162 | INFO | Starting a round of Baz IO | 20260506 14:51:37.314 | INFO | Populating RTMetrics report | 20260506 14:51:37.315 | INFO | RTMetrics report populated | 20260506 14:51:37.315 | INFO | Opening tmp RTMetrics json file | 20260506 14:51:37.329 | INFO | Serializing RTMetrics to json | 20260506 14:51:37.333 | INFO | Serialization done, swapping tmp file now... | 20260506 14:51:37.338 | INFO | RTMetrics done | 20260506 14:51:37.526 | INFO | Finished a round of Baz IO: Wrote 2.107 GiB in 3.364 seconds | 20260506 14:51:37.526 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.82% / 0.01% / 62.17% | 20260506 14:51:37.527 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.33106% / 1.19209e-05% / 5.5631e-05% | 20260506 14:51:37.527 | INFO | Baz IO overall: | 20260506 14:51:37.527 | INFO | BytesWritten : 1146682761242 | 20260506 14:51:37.527 | INFO | NumEvents : 554923130587 | 20260506 14:51:39.182 | INFO | Analyzing chunk frames = [2211840,2212352) | 20260506 14:51:39.182 | INFO | MemInfo: { | 20260506 14:51:39.182 | INFO | "gpu" : " 17 GB", | 20260506 14:51:39.182 | INFO | "huge" : "", | 20260506 14:51:39.182 | INFO | "huge_shared" : " 27 GB", | 20260506 14:51:39.182 | INFO | "malloc" : " 14 GB", | 20260506 14:51:39.182 | INFO | "pinned" : " 29 GB" | 20260506 14:51:39.182 | INFO | } | 20260506 14:51:39.182 | INFO | Perf report for the last 5.1393 seconds: | 20260506 14:51:39.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:51:39.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:39.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:39.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:39.182 | INFO | ANALYSIS : 63.1%, 2.9, 95.4ms, 0 | 20260506 14:51:39.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:39.182 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 14:51:39.182 | INFO | PRE_HQ : 60.8%, 1.9, 59.7ms, 0 | 20260506 14:51:39.182 | INFO | BAZWRITER : 65.0%, 4.8, 161.0ms, 0 | 20260506 14:51:39.182 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 14:51:39.182 | INFO | Uploaded 36GiB to GPU in 2.06649s (17.4208GiB/s) | 20260506 14:51:39.182 | INFO | Downloaded 38GiB to GPU in 2.23212s (17.0242GiB/s) | 20260506 14:51:44.321 | INFO | Analyzing chunk frames = [2212352,2212864) | 20260506 14:51:44.321 | INFO | MemInfo: { | 20260506 14:51:44.321 | INFO | "gpu" : " 17 GB", | 20260506 14:51:44.321 | INFO | "huge" : "", | 20260506 14:51:44.321 | INFO | "huge_shared" : " 27 GB", | 20260506 14:51:44.321 | INFO | "malloc" : " 14 GB", | 20260506 14:51:44.321 | INFO | "pinned" : " 29 GB" | 20260506 14:51:44.321 | INFO | } | 20260506 14:51:44.321 | INFO | Perf report for the last 5.13889 seconds: | 20260506 14:51:44.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:51:44.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:44.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:44.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:44.321 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 14:51:44.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:44.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:44.321 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:51:44.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:44.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:44.321 | INFO | Uploaded 36GiB to GPU in 1.98062s (18.1761GiB/s) | 20260506 14:51:44.321 | INFO | Downloaded 34GiB to GPU in 1.9424s (17.5041GiB/s) | 20260506 14:51:49.460 | INFO | Analyzing chunk frames = [2212864,2213376) | 20260506 14:51:49.460 | INFO | MemInfo: { | 20260506 14:51:49.460 | INFO | "gpu" : " 17 GB", | 20260506 14:51:49.460 | INFO | "huge" : "", | 20260506 14:51:49.460 | INFO | "huge_shared" : " 27 GB", | 20260506 14:51:49.460 | INFO | "malloc" : " 14 GB", | 20260506 14:51:49.460 | INFO | "pinned" : " 29 GB" | 20260506 14:51:49.460 | INFO | } | 20260506 14:51:49.460 | INFO | Perf report for the last 5.13936 seconds: | 20260506 14:51:49.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:51:49.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:49.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:49.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:49.460 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 14:51:49.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:49.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:49.460 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:51:49.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:49.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:49.460 | INFO | Uploaded 36GiB to GPU in 1.99777s (18.0201GiB/s) | 20260506 14:51:49.461 | INFO | Downloaded 34GiB to GPU in 1.94638s (17.4683GiB/s) | 20260506 14:51:54.599 | INFO | Analyzing chunk frames = [2213376,2213888) | 20260506 14:51:54.599 | INFO | MemInfo: { | 20260506 14:51:54.599 | INFO | "gpu" : " 17 GB", | 20260506 14:51:54.599 | INFO | "huge" : "", | 20260506 14:51:54.599 | INFO | "huge_shared" : " 27 GB", | 20260506 14:51:54.599 | INFO | "malloc" : " 14 GB", | 20260506 14:51:54.600 | INFO | "pinned" : " 29 GB" | 20260506 14:51:54.600 | INFO | } | 20260506 14:51:54.600 | INFO | Perf report for the last 5.1393 seconds: | 20260506 14:51:54.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:51:54.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:54.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:54.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:54.600 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260506 14:51:54.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:54.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:54.600 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:51:54.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:54.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:54.600 | INFO | Uploaded 36GiB to GPU in 2.00958s (17.9142GiB/s) | 20260506 14:51:54.600 | INFO | Downloaded 34GiB to GPU in 1.91609s (17.7444GiB/s) | 20260506 14:51:59.740 | INFO | Analyzing chunk frames = [2213888,2214400) | 20260506 14:51:59.740 | INFO | MemInfo: { | 20260506 14:51:59.740 | INFO | "gpu" : " 17 GB", | 20260506 14:51:59.740 | INFO | "huge" : "", | 20260506 14:51:59.740 | INFO | "huge_shared" : " 27 GB", | 20260506 14:51:59.740 | INFO | "malloc" : " 14 GB", | 20260506 14:51:59.740 | INFO | "pinned" : " 29 GB" | 20260506 14:51:59.740 | INFO | } | 20260506 14:51:59.740 | INFO | Perf report for the last 5.14011 seconds: | 20260506 14:51:59.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:51:59.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:59.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:59.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:59.740 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 14:51:59.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:59.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:59.740 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:51:59.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:59.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:51:59.740 | INFO | Uploaded 36GiB to GPU in 2.00914s (17.9181GiB/s) | 20260506 14:51:59.740 | INFO | Downloaded 34GiB to GPU in 1.9277s (17.6376GiB/s) | 20260506 14:52:04.879 | INFO | Analyzing chunk frames = [2214400,2214912) | 20260506 14:52:04.879 | INFO | MemInfo: { | 20260506 14:52:04.879 | INFO | "gpu" : " 17 GB", | 20260506 14:52:04.879 | INFO | "huge" : "", | 20260506 14:52:04.879 | INFO | "huge_shared" : " 27 GB", | 20260506 14:52:04.879 | INFO | "malloc" : " 14 GB", | 20260506 14:52:04.879 | INFO | "pinned" : " 29 GB" | 20260506 14:52:04.879 | INFO | } | 20260506 14:52:04.880 | INFO | Perf report for the last 5.13982 seconds: | 20260506 14:52:04.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:52:04.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:04.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:04.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:04.880 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 14:52:04.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:04.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:04.880 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 14:52:04.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:04.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:04.880 | INFO | Uploaded 36GiB to GPU in 2.0676s (17.4115GiB/s) | 20260506 14:52:04.880 | INFO | Downloaded 34GiB to GPU in 1.97084s (17.2515GiB/s) | 20260506 14:52:10.018 | INFO | Analyzing chunk frames = [2214912,2215424) | 20260506 14:52:10.018 | INFO | MemInfo: { | 20260506 14:52:10.018 | INFO | "gpu" : " 17 GB", | 20260506 14:52:10.018 | INFO | "huge" : "", | 20260506 14:52:10.018 | INFO | "huge_shared" : " 27 GB", | 20260506 14:52:10.018 | INFO | "malloc" : " 14 GB", | 20260506 14:52:10.018 | INFO | "pinned" : " 29 GB" | 20260506 14:52:10.018 | INFO | } | 20260506 14:52:10.018 | INFO | Perf report for the last 5.13886 seconds: | 20260506 14:52:10.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:52:10.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:10.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:10.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:10.018 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:52:10.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:10.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:10.019 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 14:52:10.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:10.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:10.019 | INFO | Uploaded 36GiB to GPU in 2.01609s (17.8564GiB/s) | 20260506 14:52:10.019 | INFO | Downloaded 34GiB to GPU in 2.00539s (16.9543GiB/s) | 20260506 14:52:15.156 | INFO | Analyzing chunk frames = [2215424,2215936) | 20260506 14:52:15.156 | INFO | MemInfo: { | 20260506 14:52:15.156 | INFO | "gpu" : " 17 GB", | 20260506 14:52:15.156 | INFO | "huge" : "", | 20260506 14:52:15.156 | INFO | "huge_shared" : " 27 GB", | 20260506 14:52:15.156 | INFO | "malloc" : " 14 GB", | 20260506 14:52:15.156 | INFO | "pinned" : " 29 GB" | 20260506 14:52:15.156 | INFO | } | 20260506 14:52:15.156 | INFO | Perf report for the last 5.13809 seconds: | 20260506 14:52:15.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:52:15.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:15.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:15.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:15.157 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 14:52:15.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:15.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:15.157 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:52:15.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:15.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:15.157 | INFO | Uploaded 36GiB to GPU in 2.01965s (17.8248GiB/s) | 20260506 14:52:15.157 | INFO | Downloaded 34GiB to GPU in 1.97839s (17.1857GiB/s) | 20260506 14:52:15.269 | INFO | Starting a round of Baz IO | 20260506 14:52:18.424 | INFO | Populating RTMetrics report | 20260506 14:52:18.425 | INFO | RTMetrics report populated | 20260506 14:52:18.425 | INFO | Opening tmp RTMetrics json file | 20260506 14:52:18.446 | INFO | Serializing RTMetrics to json | 20260506 14:52:18.473 | INFO | Serialization done, swapping tmp file now... | 20260506 14:52:18.474 | INFO | RTMetrics done | 20260506 14:52:18.617 | INFO | Finished a round of Baz IO: Wrote 2.108 GiB in 3.348 seconds | 20260506 14:52:18.617 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.81% / 0.01% / 62.18% | 20260506 14:52:18.617 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.32812% / 3.17891e-05% / 7.15256e-05% | 20260506 14:52:18.617 | INFO | Baz IO overall: | 20260506 14:52:18.617 | INFO | BytesWritten : 1148946571255 | 20260506 14:52:18.617 | INFO | NumEvents : 556019161295 | 20260506 14:52:20.296 | INFO | Analyzing chunk frames = [2215936,2216448) | 20260506 14:52:20.296 | INFO | MemInfo: { | 20260506 14:52:20.296 | INFO | "gpu" : " 17 GB", | 20260506 14:52:20.296 | INFO | "huge" : "", | 20260506 14:52:20.296 | INFO | "huge_shared" : " 27 GB", | 20260506 14:52:20.296 | INFO | "malloc" : " 14 GB", | 20260506 14:52:20.297 | INFO | "pinned" : " 29 GB" | 20260506 14:52:20.297 | INFO | } | 20260506 14:52:20.297 | INFO | Perf report for the last 5.14006 seconds: | 20260506 14:52:20.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:52:20.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:20.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:20.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:20.297 | INFO | ANALYSIS : 63.0%, 2.9, 95.1ms, 0 | 20260506 14:52:20.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:20.297 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 14:52:20.297 | INFO | PRE_HQ : 61.0%, 1.9, 59.8ms, 0 | 20260506 14:52:20.297 | INFO | BAZWRITER : 64.5%, 4.8, 158.7ms, 0 | 20260506 14:52:20.297 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 14:52:20.297 | INFO | Uploaded 36GiB to GPU in 2.0505s (17.5567GiB/s) | 20260506 14:52:20.297 | INFO | Downloaded 38GiB to GPU in 2.22144s (17.106GiB/s) | 20260506 14:52:25.437 | INFO | Analyzing chunk frames = [2216448,2216960) | 20260506 14:52:25.437 | INFO | MemInfo: { | 20260506 14:52:25.437 | INFO | "gpu" : " 17 GB", | 20260506 14:52:25.437 | INFO | "huge" : "", | 20260506 14:52:25.437 | INFO | "huge_shared" : " 27 GB", | 20260506 14:52:25.437 | INFO | "malloc" : " 14 GB", | 20260506 14:52:25.437 | INFO | "pinned" : " 29 GB" | 20260506 14:52:25.437 | INFO | } | 20260506 14:52:25.437 | INFO | Perf report for the last 5.14044 seconds: | 20260506 14:52:25.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:52:25.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:25.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:25.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:25.437 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 14:52:25.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:25.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:25.437 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 14:52:25.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:25.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:25.437 | INFO | Uploaded 36GiB to GPU in 1.99949s (18.0046GiB/s) | 20260506 14:52:25.437 | INFO | Downloaded 34GiB to GPU in 1.93104s (17.6071GiB/s) | 20260506 14:52:30.578 | INFO | Analyzing chunk frames = [2216960,2217472) | 20260506 14:52:30.578 | INFO | MemInfo: { | 20260506 14:52:30.578 | INFO | "gpu" : " 17 GB", | 20260506 14:52:30.578 | INFO | "huge" : "", | 20260506 14:52:30.578 | INFO | "huge_shared" : " 27 GB", | 20260506 14:52:30.578 | INFO | "malloc" : " 14 GB", | 20260506 14:52:30.578 | INFO | "pinned" : " 29 GB" | 20260506 14:52:30.578 | INFO | } | 20260506 14:52:30.578 | INFO | Perf report for the last 5.14135 seconds: | 20260506 14:52:30.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:52:30.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:30.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:30.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:30.578 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 14:52:30.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:30.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:30.578 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 14:52:30.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:30.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:30.579 | INFO | Uploaded 36GiB to GPU in 1.9999s (18.0009GiB/s) | 20260506 14:52:30.579 | INFO | Downloaded 34GiB to GPU in 1.94346s (17.4945GiB/s) | 20260506 14:52:35.715 | INFO | Analyzing chunk frames = [2217472,2217984) | 20260506 14:52:35.715 | INFO | MemInfo: { | 20260506 14:52:35.715 | INFO | "gpu" : " 17 GB", | 20260506 14:52:35.715 | INFO | "huge" : "", | 20260506 14:52:35.715 | INFO | "huge_shared" : " 27 GB", | 20260506 14:52:35.715 | INFO | "malloc" : " 14 GB", | 20260506 14:52:35.715 | INFO | "pinned" : " 29 GB" | 20260506 14:52:35.715 | INFO | } | 20260506 14:52:35.715 | INFO | Perf report for the last 5.13663 seconds: | 20260506 14:52:35.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:52:35.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:35.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:35.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:35.715 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 14:52:35.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:35.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:35.715 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:52:35.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:35.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:35.715 | INFO | Uploaded 36GiB to GPU in 2.01016s (17.9091GiB/s) | 20260506 14:52:35.715 | INFO | Downloaded 34GiB to GPU in 1.9097s (17.8038GiB/s) | 20260506 14:52:40.853 | INFO | Analyzing chunk frames = [2217984,2218496) | 20260506 14:52:40.853 | INFO | MemInfo: { | 20260506 14:52:40.853 | INFO | "gpu" : " 17 GB", | 20260506 14:52:40.853 | INFO | "huge" : "", | 20260506 14:52:40.853 | INFO | "huge_shared" : " 27 GB", | 20260506 14:52:40.853 | INFO | "malloc" : " 14 GB", | 20260506 14:52:40.853 | INFO | "pinned" : " 29 GB" | 20260506 14:52:40.853 | INFO | } | 20260506 14:52:40.854 | INFO | Perf report for the last 5.13856 seconds: | 20260506 14:52:40.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:52:40.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:40.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:40.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:40.854 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 14:52:40.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:40.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:40.854 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:52:40.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:40.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:40.854 | INFO | Uploaded 36GiB to GPU in 2.00328s (17.9705GiB/s) | 20260506 14:52:40.854 | INFO | Downloaded 34GiB to GPU in 1.92837s (17.6315GiB/s) | 20260506 14:52:45.993 | INFO | Analyzing chunk frames = [2218496,2219008) | 20260506 14:52:45.993 | INFO | MemInfo: { | 20260506 14:52:45.993 | INFO | "gpu" : " 17 GB", | 20260506 14:52:45.993 | INFO | "huge" : "", | 20260506 14:52:45.993 | INFO | "huge_shared" : " 27 GB", | 20260506 14:52:45.993 | INFO | "malloc" : " 14 GB", | 20260506 14:52:45.993 | INFO | "pinned" : " 29 GB" | 20260506 14:52:45.993 | INFO | } | 20260506 14:52:45.993 | INFO | Perf report for the last 5.13985 seconds: | 20260506 14:52:45.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:52:45.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:45.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:45.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:45.994 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:52:45.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:45.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:45.994 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 14:52:45.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:45.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:45.994 | INFO | Uploaded 36GiB to GPU in 2.04236s (17.6267GiB/s) | 20260506 14:52:45.994 | INFO | Downloaded 34GiB to GPU in 1.9261s (17.6522GiB/s) | 20260506 14:52:51.133 | INFO | Analyzing chunk frames = [2219008,2219520) | 20260506 14:52:51.133 | INFO | MemInfo: { | 20260506 14:52:51.133 | INFO | "gpu" : " 17 GB", | 20260506 14:52:51.133 | INFO | "huge" : "", | 20260506 14:52:51.133 | INFO | "huge_shared" : " 27 GB", | 20260506 14:52:51.133 | INFO | "malloc" : " 14 GB", | 20260506 14:52:51.133 | INFO | "pinned" : " 29 GB" | 20260506 14:52:51.133 | INFO | } | 20260506 14:52:51.133 | INFO | Perf report for the last 5.13982 seconds: | 20260506 14:52:51.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:52:51.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:51.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:51.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:51.134 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 14:52:51.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:51.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:51.134 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:52:51.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:51.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:51.134 | INFO | Uploaded 36GiB to GPU in 1.97747s (18.2051GiB/s) | 20260506 14:52:51.134 | INFO | Downloaded 34GiB to GPU in 1.96881s (17.2693GiB/s) | 20260506 14:52:56.271 | INFO | Analyzing chunk frames = [2219520,2220032) | 20260506 14:52:56.271 | INFO | MemInfo: { | 20260506 14:52:56.271 | INFO | "gpu" : " 17 GB", | 20260506 14:52:56.271 | INFO | "huge" : "", | 20260506 14:52:56.271 | INFO | "huge_shared" : " 27 GB", | 20260506 14:52:56.271 | INFO | "malloc" : " 14 GB", | 20260506 14:52:56.271 | INFO | "pinned" : " 29 GB" | 20260506 14:52:56.271 | INFO | } | 20260506 14:52:56.271 | INFO | Perf report for the last 5.1381 seconds: | 20260506 14:52:56.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:52:56.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:56.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:56.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:56.271 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 14:52:56.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:56.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:56.272 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:52:56.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:56.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:52:56.272 | INFO | Uploaded 36GiB to GPU in 2.01038s (17.907GiB/s) | 20260506 14:52:56.272 | INFO | Downloaded 34GiB to GPU in 1.97138s (17.2468GiB/s) | 20260506 14:52:56.272 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:52:56.272 | INFO | Total mem usage: 17.4 GiB | 20260506 14:52:56.272 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:52:56.272 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -10.7%) | 20260506 14:52:56.272 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:52:56.272 | INFO | High Water Breakdown: | 20260506 14:52:56.272 | INFO | 61.3% : 9.65 GiB ( 88.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:52:56.272 | INFO | 38.6% : 6.07 GiB ( 90.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:52:56.272 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:52:56.272 | INFO | ------------------------------------------ | 20260506 14:52:56.272 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:52:56.272 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:52:56.272 | INFO | Total mem usage: 26.5 GiB | 20260506 14:52:56.272 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:52:56.272 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:52:56.272 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:52:56.272 | INFO | High Water Breakdown: | 20260506 14:52:56.272 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:52:56.272 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:52:56.272 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:52:56.272 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:52:56.272 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:52:56.272 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:52:56.272 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:52:56.272 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:52:56.272 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:52:56.272 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:52:56.272 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:52:56.272 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:52:56.272 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:52:56.272 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:52:56.272 | INFO | ------------------------------------------ | 20260506 14:52:56.272 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:52:56.272 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:52:56.272 | INFO | Total mem usage: 25.6 GiB | 20260506 14:52:56.272 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:52:56.272 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:52:56.272 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:52:56.272 | INFO | High Water Breakdown: | 20260506 14:52:56.272 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:52:56.272 | INFO | ------------------------------------------ | 20260506 14:52:56.272 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:52:56.273 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:52:56.273 | INFO | Total mem usage: 15.7 GiB | 20260506 14:52:56.273 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:52:56.273 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:52:56.273 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 14:52:56.273 | INFO | High Water Breakdown: | 20260506 14:52:56.273 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:52:56.273 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:52:56.273 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:52:56.273 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:52:56.273 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:52:56.273 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:52:56.273 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:52:56.273 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:52:56.273 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:52:56.273 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:52:56.273 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:52:56.273 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:52:56.273 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:52:56.273 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:52:56.273 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:52:56.273 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:52:56.273 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:52:56.273 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:52:56.273 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:52:56.273 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:52:56.273 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:52:56.273 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:52:56.273 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:52:56.273 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:52:56.273 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:52:56.273 | INFO | ------------------------------------------ | 20260506 14:52:56.273 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:52:56.273 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:52:56.273 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:52:56.273 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:52:56.273 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.58e+01, 1.34e+01 / 1.91e+00 | 20260506 14:52:56.273 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.02e-01, 1.02e-02 / 6.49e-03 | 20260506 14:52:56.273 | INFO | PerfReport: 11.57%, Baseline -- 3.65e+00 / 4.41e+00, 5.70e-01 / 3.64e-01 | 20260506 14:52:56.273 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.46e+00, 1.88e-01 / 1.51e-01 | 20260506 14:52:56.273 | INFO | PerfReport: 15.86%, DME -- 4.00e+01 / 4.43e+01, 3.93e+00 / 2.52e+00 | 20260506 14:52:56.273 | INFO | PerfReport: 56.79%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.52e+00 / 2.07e+00 | 20260506 14:52:56.273 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.33e+00 / 2.92e+00, 4.07e-01 / 2.61e-01 | 20260506 14:52:56.273 | INFO | PerfReport: 3.58%, Metrics -- 1.13e+00 / 1.50e+00, 2.92e-01 / 1.33e-01 | 20260506 14:52:56.273 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.94e-01 / 4.46e-01, 6.16e-02 / 2.37e-02 | 20260506 14:52:56.273 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:52:56.273 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:52:56.273 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:52:56.273 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 14:52:56.273 | INFO | PerfReport: 21.52%, Upload -- 2.10e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 14:52:56.273 | INFO | PerfReport: 21.16%, Download -- 2.07e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 14:52:56.273 | INFO | PerfReport: 32.29%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 14:52:56.273 | INFO | PerfReport: 1.51%, LockUpload -- 1.47e+00 / 0.00e+00, 5.82e+00 / 0.00e+00 | 20260506 14:52:56.273 | INFO | PerfReport: 1.11%, LockDownload -- 1.09e+00 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260506 14:52:56.273 | INFO | PerfReport: 22.40%, LockCompute -- 2.19e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 14:52:56.273 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260506 14:52:56.390 | INFO | Starting a round of Baz IO | 20260506 14:52:59.531 | INFO | Populating RTMetrics report | 20260506 14:52:59.531 | INFO | RTMetrics report populated | 20260506 14:52:59.531 | INFO | Opening tmp RTMetrics json file | 20260506 14:52:59.583 | INFO | Serializing RTMetrics to json | 20260506 14:52:59.663 | INFO | Serialization done, swapping tmp file now... | 20260506 14:52:59.666 | INFO | RTMetrics done | 20260506 14:52:59.697 | INFO | Finished a round of Baz IO: Wrote 2.106 GiB in 3.306 seconds | 20260506 14:52:59.697 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.79% / 0.01% / 62.19% | 20260506 14:52:59.697 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.31843% / 7.94729e-06% / 5.5631e-05% | 20260506 14:52:59.697 | INFO | Baz IO overall: | 20260506 14:52:59.697 | INFO | BytesWritten : 1151208073039 | 20260506 14:52:59.697 | INFO | NumEvents : 557113337702 | 20260506 14:53:01.411 | INFO | Analyzing chunk frames = [2220032,2220544) | 20260506 14:53:01.411 | INFO | MemInfo: { | 20260506 14:53:01.411 | INFO | "gpu" : " 17 GB", | 20260506 14:53:01.411 | INFO | "huge" : "", | 20260506 14:53:01.411 | INFO | "huge_shared" : " 27 GB", | 20260506 14:53:01.411 | INFO | "malloc" : " 14 GB", | 20260506 14:53:01.411 | INFO | "pinned" : " 29 GB" | 20260506 14:53:01.411 | INFO | } | 20260506 14:53:01.411 | INFO | Perf report for the last 5.13952 seconds: | 20260506 14:53:01.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:53:01.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:01.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:01.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:01.411 | INFO | ANALYSIS : 62.8%, 2.9, 95.3ms, 0 | 20260506 14:53:01.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:01.411 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260506 14:53:01.411 | INFO | PRE_HQ : 60.7%, 1.9, 59.6ms, 0 | 20260506 14:53:01.411 | INFO | BAZWRITER : 64.0%, 4.7, 153.9ms, 0 | 20260506 14:53:01.411 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 14:53:01.411 | INFO | Uploaded 36GiB to GPU in 2.06846s (17.4042GiB/s) | 20260506 14:53:01.411 | INFO | Downloaded 38GiB to GPU in 2.20985s (17.1958GiB/s) | 20260506 14:53:06.551 | INFO | Analyzing chunk frames = [2220544,2221056) | 20260506 14:53:06.551 | INFO | MemInfo: { | 20260506 14:53:06.551 | INFO | "gpu" : " 17 GB", | 20260506 14:53:06.552 | INFO | "huge" : "", | 20260506 14:53:06.552 | INFO | "huge_shared" : " 27 GB", | 20260506 14:53:06.552 | INFO | "malloc" : " 14 GB", | 20260506 14:53:06.552 | INFO | "pinned" : " 29 GB" | 20260506 14:53:06.552 | INFO | } | 20260506 14:53:06.552 | INFO | Perf report for the last 5.14079 seconds: | 20260506 14:53:06.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:53:06.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:06.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:06.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:06.552 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 14:53:06.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:06.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:06.552 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:53:06.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:06.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:06.552 | INFO | Uploaded 36GiB to GPU in 2.00625s (17.9439GiB/s) | 20260506 14:53:06.552 | INFO | Downloaded 34GiB to GPU in 1.93397s (17.5804GiB/s) | 20260506 14:53:11.693 | INFO | Analyzing chunk frames = [2221056,2221568) | 20260506 14:53:11.693 | INFO | MemInfo: { | 20260506 14:53:11.693 | INFO | "gpu" : " 17 GB", | 20260506 14:53:11.693 | INFO | "huge" : "", | 20260506 14:53:11.693 | INFO | "huge_shared" : " 27 GB", | 20260506 14:53:11.693 | INFO | "malloc" : " 14 GB", | 20260506 14:53:11.693 | INFO | "pinned" : " 29 GB" | 20260506 14:53:11.693 | INFO | } | 20260506 14:53:11.693 | INFO | Perf report for the last 5.14129 seconds: | 20260506 14:53:11.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:53:11.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:11.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:11.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:11.693 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 14:53:11.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:11.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:11.693 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:53:11.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:11.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:11.693 | INFO | Uploaded 36GiB to GPU in 2.00159s (17.9857GiB/s) | 20260506 14:53:11.693 | INFO | Downloaded 34GiB to GPU in 1.94137s (17.5134GiB/s) | 20260506 14:53:16.830 | INFO | Analyzing chunk frames = [2221568,2222080) | 20260506 14:53:16.830 | INFO | MemInfo: { | 20260506 14:53:16.830 | INFO | "gpu" : " 17 GB", | 20260506 14:53:16.830 | INFO | "huge" : "", | 20260506 14:53:16.830 | INFO | "huge_shared" : " 27 GB", | 20260506 14:53:16.830 | INFO | "malloc" : " 14 GB", | 20260506 14:53:16.830 | INFO | "pinned" : " 29 GB" | 20260506 14:53:16.830 | INFO | } | 20260506 14:53:16.830 | INFO | Perf report for the last 5.13676 seconds: | 20260506 14:53:16.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:53:16.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:16.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:16.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:16.830 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 14:53:16.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:16.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:16.830 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:53:16.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:16.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:16.830 | INFO | Uploaded 36GiB to GPU in 1.96973s (18.2766GiB/s) | 20260506 14:53:16.830 | INFO | Downloaded 34GiB to GPU in 1.91084s (17.7932GiB/s) | 20260506 14:53:21.969 | INFO | Analyzing chunk frames = [2222080,2222592) | 20260506 14:53:21.969 | INFO | MemInfo: { | 20260506 14:53:21.969 | INFO | "gpu" : " 17 GB", | 20260506 14:53:21.969 | INFO | "huge" : "", | 20260506 14:53:21.969 | INFO | "huge_shared" : " 27 GB", | 20260506 14:53:21.969 | INFO | "malloc" : " 14 GB", | 20260506 14:53:21.969 | INFO | "pinned" : " 29 GB" | 20260506 14:53:21.969 | INFO | } | 20260506 14:53:21.969 | INFO | Perf report for the last 5.1396 seconds: | 20260506 14:53:21.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:53:21.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:21.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:21.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:21.969 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 14:53:21.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:21.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:21.969 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 14:53:21.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:21.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:21.970 | INFO | Uploaded 36GiB to GPU in 1.98949s (18.0951GiB/s) | 20260506 14:53:21.970 | INFO | Downloaded 34GiB to GPU in 1.93298s (17.5894GiB/s) | 20260506 14:53:27.108 | INFO | Analyzing chunk frames = [2222592,2223104) | 20260506 14:53:27.108 | INFO | MemInfo: { | 20260506 14:53:27.108 | INFO | "gpu" : " 17 GB", | 20260506 14:53:27.108 | INFO | "huge" : "", | 20260506 14:53:27.108 | INFO | "huge_shared" : " 27 GB", | 20260506 14:53:27.108 | INFO | "malloc" : " 14 GB", | 20260506 14:53:27.108 | INFO | "pinned" : " 29 GB" | 20260506 14:53:27.108 | INFO | } | 20260506 14:53:27.108 | INFO | Perf report for the last 5.13874 seconds: | 20260506 14:53:27.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:53:27.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:27.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:27.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:27.108 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 14:53:27.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:27.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:27.108 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260506 14:53:27.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:27.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:27.109 | INFO | Uploaded 36GiB to GPU in 2.032s (17.7166GiB/s) | 20260506 14:53:27.109 | INFO | Downloaded 34GiB to GPU in 1.93179s (17.6002GiB/s) | 20260506 14:53:32.248 | INFO | Analyzing chunk frames = [2223104,2223616) | 20260506 14:53:32.248 | INFO | MemInfo: { | 20260506 14:53:32.248 | INFO | "gpu" : " 17 GB", | 20260506 14:53:32.248 | INFO | "huge" : "", | 20260506 14:53:32.248 | INFO | "huge_shared" : " 27 GB", | 20260506 14:53:32.248 | INFO | "malloc" : " 14 GB", | 20260506 14:53:32.248 | INFO | "pinned" : " 29 GB" | 20260506 14:53:32.248 | INFO | } | 20260506 14:53:32.248 | INFO | Perf report for the last 5.13947 seconds: | 20260506 14:53:32.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:53:32.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:32.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:32.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:32.248 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 14:53:32.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:32.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:32.248 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 14:53:32.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:32.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:32.248 | INFO | Uploaded 36GiB to GPU in 1.96889s (18.2844GiB/s) | 20260506 14:53:32.248 | INFO | Downloaded 34GiB to GPU in 2.00318s (16.973GiB/s) | 20260506 14:53:37.387 | INFO | Analyzing chunk frames = [2223616,2224128) | 20260506 14:53:37.387 | INFO | MemInfo: { | 20260506 14:53:37.387 | INFO | "gpu" : " 17 GB", | 20260506 14:53:37.387 | INFO | "huge" : "", | 20260506 14:53:37.387 | INFO | "huge_shared" : " 27 GB", | 20260506 14:53:37.387 | INFO | "malloc" : " 14 GB", | 20260506 14:53:37.387 | INFO | "pinned" : " 29 GB" | 20260506 14:53:37.387 | INFO | } | 20260506 14:53:37.387 | INFO | Perf report for the last 5.13964 seconds: | 20260506 14:53:37.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:53:37.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:37.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:37.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:37.387 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 14:53:37.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:37.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:37.387 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:53:37.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:37.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:37.388 | INFO | Uploaded 36GiB to GPU in 1.96145s (18.3538GiB/s) | 20260506 14:53:37.388 | INFO | Downloaded 34GiB to GPU in 1.97986s (17.173GiB/s) | 20260506 14:53:37.502 | INFO | Starting a round of Baz IO | 20260506 14:53:40.647 | INFO | Populating RTMetrics report | 20260506 14:53:40.647 | INFO | RTMetrics report populated | 20260506 14:53:40.647 | INFO | Opening tmp RTMetrics json file | 20260506 14:53:40.755 | INFO | Serializing RTMetrics to json | 20260506 14:53:40.903 | INFO | Serialization done, swapping tmp file now... | 20260506 14:53:40.945 | INFO | RTMetrics done | 20260506 14:53:41.164 | INFO | Finished a round of Baz IO: Wrote 2.11 GiB in 3.661 seconds | 20260506 14:53:41.164 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.78% / 0.01% / 62.21% | 20260506 14:53:41.164 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.32311% / 3.17891e-05% / 6.35783e-05% | 20260506 14:53:41.164 | INFO | Baz IO overall: | 20260506 14:53:41.164 | INFO | BytesWritten : 1153473215787 | 20260506 14:53:41.164 | INFO | NumEvents : 558210467760 | 20260506 14:53:42.527 | INFO | Analyzing chunk frames = [2224128,2224640) | 20260506 14:53:42.527 | INFO | MemInfo: { | 20260506 14:53:42.527 | INFO | "gpu" : " 17 GB", | 20260506 14:53:42.527 | INFO | "huge" : "", | 20260506 14:53:42.527 | INFO | "huge_shared" : " 27 GB", | 20260506 14:53:42.527 | INFO | "malloc" : " 14 GB", | 20260506 14:53:42.527 | INFO | "pinned" : " 29 GB" | 20260506 14:53:42.527 | INFO | } | 20260506 14:53:42.527 | INFO | Perf report for the last 5.13952 seconds: | 20260506 14:53:42.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:53:42.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:42.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:42.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:42.527 | INFO | ANALYSIS : 62.8%, 2.9, 95.1ms, 0 | 20260506 14:53:42.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:42.527 | INFO | RT_METRICS : 50.0%, 1.0, 26.2ms, 0 | 20260506 14:53:42.527 | INFO | PRE_HQ : 60.6%, 1.9, 59.3ms, 0 | 20260506 14:53:42.527 | INFO | BAZWRITER : 70.9%, 4.8, 174.2ms, 0 | 20260506 14:53:42.527 | INFO | BAZ_SAMPLING : 4.0%, 1.0, 2.1ms, 0 | 20260506 14:53:42.527 | INFO | Uploaded 36GiB to GPU in 2.07562s (17.3442GiB/s) | 20260506 14:53:42.527 | INFO | Downloaded 38GiB to GPU in 2.21709s (17.1396GiB/s) | 20260506 14:53:47.665 | INFO | Analyzing chunk frames = [2224640,2225152) | 20260506 14:53:47.665 | INFO | MemInfo: { | 20260506 14:53:47.665 | INFO | "gpu" : " 17 GB", | 20260506 14:53:47.665 | INFO | "huge" : "", | 20260506 14:53:47.665 | INFO | "huge_shared" : " 27 GB", | 20260506 14:53:47.665 | INFO | "malloc" : " 14 GB", | 20260506 14:53:47.665 | INFO | "pinned" : " 29 GB" | 20260506 14:53:47.668 | INFO | } | 20260506 14:53:47.668 | INFO | Perf report for the last 5.14126 seconds: | 20260506 14:53:47.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:53:47.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:47.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:47.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:47.668 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 14:53:47.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:47.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:47.668 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:53:47.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:47.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:47.668 | INFO | Uploaded 36GiB to GPU in 2.00259s (17.9767GiB/s) | 20260506 14:53:47.668 | INFO | Downloaded 34GiB to GPU in 1.9428s (17.5005GiB/s) | 20260506 14:53:52.804 | INFO | Analyzing chunk frames = [2225152,2225664) | 20260506 14:53:52.804 | INFO | MemInfo: { | 20260506 14:53:52.804 | INFO | "gpu" : " 17 GB", | 20260506 14:53:52.804 | INFO | "huge" : "", | 20260506 14:53:52.804 | INFO | "huge_shared" : " 27 GB", | 20260506 14:53:52.804 | INFO | "malloc" : " 14 GB", | 20260506 14:53:52.805 | INFO | "pinned" : " 29 GB" | 20260506 14:53:52.805 | INFO | } | 20260506 14:53:52.805 | INFO | Perf report for the last 5.13663 seconds: | 20260506 14:53:52.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:53:52.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:52.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:52.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:52.805 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 14:53:52.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:52.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:52.805 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:53:52.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:52.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:52.805 | INFO | Uploaded 36GiB to GPU in 1.99408s (18.0534GiB/s) | 20260506 14:53:52.805 | INFO | Downloaded 34GiB to GPU in 1.943s (17.4987GiB/s) | 20260506 14:53:57.944 | INFO | Analyzing chunk frames = [2225664,2226176) | 20260506 14:53:57.944 | INFO | MemInfo: { | 20260506 14:53:57.944 | INFO | "gpu" : " 17 GB", | 20260506 14:53:57.944 | INFO | "huge" : "", | 20260506 14:53:57.944 | INFO | "huge_shared" : " 27 GB", | 20260506 14:53:57.944 | INFO | "malloc" : " 14 GB", | 20260506 14:53:57.944 | INFO | "pinned" : " 29 GB" | 20260506 14:53:57.944 | INFO | } | 20260506 14:53:57.944 | INFO | Perf report for the last 5.13917 seconds: | 20260506 14:53:57.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:53:57.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:57.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:57.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:57.944 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:53:57.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:57.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:57.944 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:53:57.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:57.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:53:57.944 | INFO | Uploaded 36GiB to GPU in 2.0051s (17.9542GiB/s) | 20260506 14:53:57.944 | INFO | Downloaded 34GiB to GPU in 1.91436s (17.7605GiB/s) | 20260506 14:54:03.082 | INFO | Analyzing chunk frames = [2226176,2226688) | 20260506 14:54:03.082 | INFO | MemInfo: { | 20260506 14:54:03.082 | INFO | "gpu" : " 17 GB", | 20260506 14:54:03.082 | INFO | "huge" : "", | 20260506 14:54:03.083 | INFO | "huge_shared" : " 27 GB", | 20260506 14:54:03.083 | INFO | "malloc" : " 14 GB", | 20260506 14:54:03.083 | INFO | "pinned" : " 29 GB" | 20260506 14:54:03.083 | INFO | } | 20260506 14:54:03.083 | INFO | Perf report for the last 5.13882 seconds: | 20260506 14:54:03.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:54:03.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:03.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:03.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:03.083 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 14:54:03.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:03.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:03.083 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:54:03.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:03.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:03.083 | INFO | Uploaded 36GiB to GPU in 1.9992s (18.0072GiB/s) | 20260506 14:54:03.083 | INFO | Downloaded 34GiB to GPU in 1.92895s (17.6262GiB/s) | 20260506 14:54:08.223 | INFO | Analyzing chunk frames = [2226688,2227200) | 20260506 14:54:08.223 | INFO | MemInfo: { | 20260506 14:54:08.223 | INFO | "gpu" : " 17 GB", | 20260506 14:54:08.223 | INFO | "huge" : "", | 20260506 14:54:08.223 | INFO | "huge_shared" : " 27 GB", | 20260506 14:54:08.223 | INFO | "malloc" : " 14 GB", | 20260506 14:54:08.223 | INFO | "pinned" : " 29 GB" | 20260506 14:54:08.223 | INFO | } | 20260506 14:54:08.223 | INFO | Perf report for the last 5.14016 seconds: | 20260506 14:54:08.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:54:08.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:08.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:08.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:08.223 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 14:54:08.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:08.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:08.223 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 14:54:08.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:08.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:08.223 | INFO | Uploaded 36GiB to GPU in 2.02847s (17.7474GiB/s) | 20260506 14:54:08.223 | INFO | Downloaded 34GiB to GPU in 1.939s (17.5348GiB/s) | 20260506 14:54:13.363 | INFO | Analyzing chunk frames = [2227200,2227712) | 20260506 14:54:13.363 | INFO | MemInfo: { | 20260506 14:54:13.363 | INFO | "gpu" : " 17 GB", | 20260506 14:54:13.363 | INFO | "huge" : "", | 20260506 14:54:13.363 | INFO | "huge_shared" : " 27 GB", | 20260506 14:54:13.363 | INFO | "malloc" : " 14 GB", | 20260506 14:54:13.363 | INFO | "pinned" : " 29 GB" | 20260506 14:54:13.363 | INFO | } | 20260506 14:54:13.363 | INFO | Perf report for the last 5.13997 seconds: | 20260506 14:54:13.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:54:13.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:13.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:13.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:13.363 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:54:13.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:13.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:13.363 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 14:54:13.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:13.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:13.363 | INFO | Uploaded 36GiB to GPU in 1.99345s (18.0591GiB/s) | 20260506 14:54:13.364 | INFO | Downloaded 34GiB to GPU in 1.99616s (17.0327GiB/s) | 20260506 14:54:18.502 | INFO | Analyzing chunk frames = [2227712,2228224) | 20260506 14:54:18.502 | INFO | MemInfo: { | 20260506 14:54:18.502 | INFO | "gpu" : " 17 GB", | 20260506 14:54:18.502 | INFO | "huge" : "", | 20260506 14:54:18.502 | INFO | "huge_shared" : " 27 GB", | 20260506 14:54:18.502 | INFO | "malloc" : " 14 GB", | 20260506 14:54:18.502 | INFO | "pinned" : " 29 GB" | 20260506 14:54:18.502 | INFO | } | 20260506 14:54:18.502 | INFO | Perf report for the last 5.13879 seconds: | 20260506 14:54:18.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:54:18.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:18.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:18.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:18.502 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:54:18.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:18.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:18.502 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 14:54:18.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:18.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:18.502 | INFO | Uploaded 36GiB to GPU in 2.02322s (17.7934GiB/s) | 20260506 14:54:18.502 | INFO | Downloaded 34GiB to GPU in 1.97912s (17.1794GiB/s) | 20260506 14:54:18.502 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:54:18.502 | INFO | Total mem usage: 17.4 GiB | 20260506 14:54:18.502 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:54:18.502 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -12.2%) | 20260506 14:54:18.502 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:54:18.502 | INFO | High Water Breakdown: | 20260506 14:54:18.502 | INFO | 61.1% : 9.48 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:54:18.502 | INFO | 38.8% : 6.02 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:54:18.502 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:54:18.502 | INFO | ------------------------------------------ | 20260506 14:54:18.502 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:54:18.502 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:54:18.502 | INFO | Total mem usage: 26.5 GiB | 20260506 14:54:18.502 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:54:18.502 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:54:18.502 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:54:18.502 | INFO | High Water Breakdown: | 20260506 14:54:18.502 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:54:18.502 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:54:18.503 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:54:18.503 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:54:18.503 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:54:18.503 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:54:18.503 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:54:18.503 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:54:18.503 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:54:18.503 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:54:18.503 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:54:18.503 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:54:18.503 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:54:18.503 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:54:18.503 | INFO | ------------------------------------------ | 20260506 14:54:18.503 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:54:18.503 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:54:18.503 | INFO | Total mem usage: 25.6 GiB | 20260506 14:54:18.503 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:54:18.503 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:54:18.503 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:54:18.503 | INFO | High Water Breakdown: | 20260506 14:54:18.503 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:54:18.503 | INFO | ------------------------------------------ | 20260506 14:54:18.503 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:54:18.503 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:54:18.503 | INFO | Total mem usage: 15.7 GiB | 20260506 14:54:18.503 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:54:18.503 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:54:18.503 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 14:54:18.503 | INFO | High Water Breakdown: | 20260506 14:54:18.503 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:54:18.503 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:54:18.503 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:54:18.503 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:54:18.503 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:54:18.503 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:54:18.503 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:54:18.503 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:54:18.503 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:54:18.503 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:54:18.503 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:54:18.503 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:54:18.503 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:54:18.503 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:54:18.503 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:54:18.503 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:54:18.503 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:54:18.503 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:54:18.503 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:54:18.503 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:54:18.503 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:54:18.503 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:54:18.503 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:54:18.503 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:54:18.503 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:54:18.503 | INFO | ------------------------------------------ | 20260506 14:54:18.503 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:54:18.504 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:54:18.504 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:54:18.504 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:54:18.504 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.58e+01, 1.36e+01 / 1.93e+00 | 20260506 14:54:18.504 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.01e-01, 1.01e-02 / 6.67e-03 | 20260506 14:54:18.504 | INFO | PerfReport: 11.58%, Baseline -- 3.65e+00 / 4.46e+00, 5.70e-01 / 3.35e-01 | 20260506 14:54:18.504 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.47e+00, 1.90e-01 / 1.46e-01 | 20260506 14:54:18.504 | INFO | PerfReport: 15.82%, DME -- 3.99e+01 / 4.41e+01, 3.93e+00 / 1.92e+00 | 20260506 14:54:18.504 | INFO | PerfReport: 56.81%, FrameLabeling -- 1.79e+01 / 2.09e+01, 1.48e+00 / 2.07e+00 | 20260506 14:54:18.504 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.33e+00 / 2.89e+00, 4.01e-01 / 2.41e-01 | 20260506 14:54:18.504 | INFO | PerfReport: 3.58%, Metrics -- 1.13e+00 / 1.50e+00, 2.92e-01 / 1.60e-01 | 20260506 14:54:18.504 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.93e-01 / 4.49e-01, 6.22e-02 / 2.37e-02 | 20260506 14:54:18.504 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:54:18.504 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:54:18.504 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:54:18.504 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 14:54:18.504 | INFO | PerfReport: 21.43%, Upload -- 2.09e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260506 14:54:18.504 | INFO | PerfReport: 21.16%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 14:54:18.504 | INFO | PerfReport: 32.28%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 14:54:18.504 | INFO | PerfReport: 1.44%, LockUpload -- 1.41e+00 / 0.00e+00, 5.84e+00 / 0.00e+00 | 20260506 14:54:18.504 | INFO | PerfReport: 1.15%, LockDownload -- 1.12e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 14:54:18.504 | INFO | PerfReport: 22.52%, LockCompute -- 2.20e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 14:54:18.504 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.39e-02 / 0.00e+00 | 20260506 14:54:18.616 | INFO | Starting a round of Baz IO | 20260506 14:54:21.772 | INFO | Populating RTMetrics report | 20260506 14:54:21.772 | INFO | RTMetrics report populated | 20260506 14:54:21.772 | INFO | Opening tmp RTMetrics json file | 20260506 14:54:21.854 | INFO | Serializing RTMetrics to json | 20260506 14:54:21.859 | INFO | Serialization done, swapping tmp file now... | 20260506 14:54:21.860 | INFO | RTMetrics done | 20260506 14:54:22.065 | INFO | Finished a round of Baz IO: Wrote 2.108 GiB in 3.448 seconds | 20260506 14:54:22.065 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.77% / 0.01% / 62.22% | 20260506 14:54:22.065 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.31383% / 3.17891e-05% / 6.75519e-05% | 20260506 14:54:22.065 | INFO | Baz IO overall: | 20260506 14:54:22.065 | INFO | BytesWritten : 1155736315503 | 20260506 14:54:22.065 | INFO | NumEvents : 559305783635 | 20260506 14:54:23.641 | INFO | Analyzing chunk frames = [2228224,2228736) | 20260506 14:54:23.641 | INFO | MemInfo: { | 20260506 14:54:23.641 | INFO | "gpu" : " 17 GB", | 20260506 14:54:23.641 | INFO | "huge" : "", | 20260506 14:54:23.641 | INFO | "huge_shared" : " 27 GB", | 20260506 14:54:23.641 | INFO | "malloc" : " 14 GB", | 20260506 14:54:23.641 | INFO | "pinned" : " 29 GB" | 20260506 14:54:23.641 | INFO | } | 20260506 14:54:23.641 | INFO | Perf report for the last 5.13956 seconds: | 20260506 14:54:23.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:54:23.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:23.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:23.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:23.641 | INFO | ANALYSIS : 63.1%, 2.9, 95.5ms, 0 | 20260506 14:54:23.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:23.641 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260506 14:54:23.642 | INFO | PRE_HQ : 61.2%, 1.9, 59.6ms, 0 | 20260506 14:54:23.642 | INFO | BAZWRITER : 66.6%, 4.8, 165.8ms, 0 | 20260506 14:54:23.642 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 14:54:23.642 | INFO | Uploaded 36GiB to GPU in 2.09661s (17.1706GiB/s) | 20260506 14:54:23.642 | INFO | Downloaded 38GiB to GPU in 2.23487s (17.0033GiB/s) | 20260506 14:54:28.780 | INFO | Analyzing chunk frames = [2228736,2229248) | 20260506 14:54:28.780 | INFO | MemInfo: { | 20260506 14:54:28.780 | INFO | "gpu" : " 17 GB", | 20260506 14:54:28.780 | INFO | "huge" : "", | 20260506 14:54:28.780 | INFO | "huge_shared" : " 27 GB", | 20260506 14:54:28.780 | INFO | "malloc" : " 14 GB", | 20260506 14:54:28.780 | INFO | "pinned" : " 29 GB" | 20260506 14:54:28.780 | INFO | } | 20260506 14:54:28.780 | INFO | Perf report for the last 5.1387 seconds: | 20260506 14:54:28.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:54:28.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:28.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:28.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:28.780 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:54:28.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:28.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:28.780 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:54:28.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:28.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:28.780 | INFO | Uploaded 36GiB to GPU in 2.01196s (17.893GiB/s) | 20260506 14:54:28.781 | INFO | Downloaded 34GiB to GPU in 1.94575s (17.4739GiB/s) | 20260506 14:54:33.919 | INFO | Analyzing chunk frames = [2229248,2229760) | 20260506 14:54:33.919 | INFO | MemInfo: { | 20260506 14:54:33.919 | INFO | "gpu" : " 17 GB", | 20260506 14:54:33.919 | INFO | "huge" : "", | 20260506 14:54:33.919 | INFO | "huge_shared" : " 27 GB", | 20260506 14:54:33.919 | INFO | "malloc" : " 14 GB", | 20260506 14:54:33.919 | INFO | "pinned" : " 29 GB" | 20260506 14:54:33.919 | INFO | } | 20260506 14:54:33.919 | INFO | Perf report for the last 5.13942 seconds: | 20260506 14:54:33.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:54:33.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:33.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:33.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:33.920 | INFO | ANALYSIS : 61.3%, 3.0, 94.8ms, 0 | 20260506 14:54:33.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:33.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:33.920 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:54:33.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:33.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:33.920 | INFO | Uploaded 36GiB to GPU in 2.01207s (17.892GiB/s) | 20260506 14:54:33.920 | INFO | Downloaded 34GiB to GPU in 1.94528s (17.4782GiB/s) | 20260506 14:54:39.059 | INFO | Analyzing chunk frames = [2229760,2230272) | 20260506 14:54:39.059 | INFO | MemInfo: { | 20260506 14:54:39.059 | INFO | "gpu" : " 17 GB", | 20260506 14:54:39.059 | INFO | "huge" : "", | 20260506 14:54:39.059 | INFO | "huge_shared" : " 27 GB", | 20260506 14:54:39.059 | INFO | "malloc" : " 14 GB", | 20260506 14:54:39.059 | INFO | "pinned" : " 29 GB" | 20260506 14:54:39.059 | INFO | } | 20260506 14:54:39.060 | INFO | Perf report for the last 5.14011 seconds: | 20260506 14:54:39.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:54:39.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:39.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:39.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:39.060 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260506 14:54:39.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:39.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:39.060 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 14:54:39.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:39.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:39.060 | INFO | Uploaded 36GiB to GPU in 1.99962s (18.0035GiB/s) | 20260506 14:54:39.060 | INFO | Downloaded 34GiB to GPU in 1.91666s (17.7392GiB/s) | 20260506 14:54:44.198 | INFO | Analyzing chunk frames = [2230272,2230784) | 20260506 14:54:44.198 | INFO | MemInfo: { | 20260506 14:54:44.198 | INFO | "gpu" : " 17 GB", | 20260506 14:54:44.199 | INFO | "huge" : "", | 20260506 14:54:44.199 | INFO | "huge_shared" : " 27 GB", | 20260506 14:54:44.199 | INFO | "malloc" : " 14 GB", | 20260506 14:54:44.199 | INFO | "pinned" : " 29 GB" | 20260506 14:54:44.199 | INFO | } | 20260506 14:54:44.199 | INFO | Perf report for the last 5.13916 seconds: | 20260506 14:54:44.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:54:44.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:44.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:44.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:44.199 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 14:54:44.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:44.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:44.199 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:54:44.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:44.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:44.199 | INFO | Uploaded 36GiB to GPU in 1.99263s (18.0665GiB/s) | 20260506 14:54:44.199 | INFO | Downloaded 34GiB to GPU in 1.92688s (17.6451GiB/s) | 20260506 14:54:49.337 | INFO | Analyzing chunk frames = [2230784,2231296) | 20260506 14:54:49.337 | INFO | MemInfo: { | 20260506 14:54:49.337 | INFO | "gpu" : " 17 GB", | 20260506 14:54:49.337 | INFO | "huge" : "", | 20260506 14:54:49.337 | INFO | "huge_shared" : " 27 GB", | 20260506 14:54:49.337 | INFO | "malloc" : " 14 GB", | 20260506 14:54:49.337 | INFO | "pinned" : " 29 GB" | 20260506 14:54:49.337 | INFO | } | 20260506 14:54:49.337 | INFO | Perf report for the last 5.13837 seconds: | 20260506 14:54:49.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:54:49.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:49.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:49.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:49.337 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 14:54:49.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:49.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:49.337 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:54:49.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:49.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:49.337 | INFO | Uploaded 36GiB to GPU in 2.08105s (17.299GiB/s) | 20260506 14:54:49.337 | INFO | Downloaded 34GiB to GPU in 1.97028s (17.2564GiB/s) | 20260506 14:54:54.477 | INFO | Analyzing chunk frames = [2231296,2231808) | 20260506 14:54:54.477 | INFO | MemInfo: { | 20260506 14:54:54.477 | INFO | "gpu" : " 17 GB", | 20260506 14:54:54.477 | INFO | "huge" : "", | 20260506 14:54:54.478 | INFO | "huge_shared" : " 27 GB", | 20260506 14:54:54.478 | INFO | "malloc" : " 14 GB", | 20260506 14:54:54.478 | INFO | "pinned" : " 29 GB" | 20260506 14:54:54.478 | INFO | } | 20260506 14:54:54.478 | INFO | Perf report for the last 5.14066 seconds: | 20260506 14:54:54.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:54:54.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:54.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:54.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:54.478 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:54:54.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:54.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:54.478 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 14:54:54.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:54.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:54.478 | INFO | Uploaded 36GiB to GPU in 1.99181s (18.0741GiB/s) | 20260506 14:54:54.478 | INFO | Downloaded 34GiB to GPU in 1.9853s (17.1259GiB/s) | 20260506 14:54:59.617 | INFO | Analyzing chunk frames = [2231808,2232320) | 20260506 14:54:59.617 | INFO | MemInfo: { | 20260506 14:54:59.617 | INFO | "gpu" : " 17 GB", | 20260506 14:54:59.619 | INFO | "huge" : "", | 20260506 14:54:59.619 | INFO | "huge_shared" : " 27 GB", | 20260506 14:54:59.619 | INFO | "malloc" : " 14 GB", | 20260506 14:54:59.619 | INFO | "pinned" : " 29 GB" | 20260506 14:54:59.619 | INFO | } | 20260506 14:54:59.619 | INFO | Perf report for the last 5.1413 seconds: | 20260506 14:54:59.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:54:59.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:59.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:59.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:59.619 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:54:59.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:59.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:59.619 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:54:59.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:59.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:54:59.619 | INFO | Uploaded 36GiB to GPU in 2.02223s (17.8021GiB/s) | 20260506 14:54:59.619 | INFO | Downloaded 34GiB to GPU in 1.98018s (17.1701GiB/s) | 20260506 14:54:59.732 | INFO | Starting a round of Baz IO | 20260506 14:55:02.890 | INFO | Populating RTMetrics report | 20260506 14:55:02.890 | INFO | RTMetrics report populated | 20260506 14:55:02.890 | INFO | Opening tmp RTMetrics json file | 20260506 14:55:02.892 | INFO | Serializing RTMetrics to json | 20260506 14:55:02.894 | INFO | Serialization done, swapping tmp file now... | 20260506 14:55:02.895 | INFO | RTMetrics done | 20260506 14:55:03.012 | INFO | Finished a round of Baz IO: Wrote 2.108 GiB in 3.279 seconds | 20260506 14:55:03.012 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.75% / 0.01% / 62.23% | 20260506 14:55:03.012 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.31076% / 1.19209e-05% / 6.75519e-05% | 20260506 14:55:03.012 | INFO | Baz IO overall: | 20260506 14:55:03.012 | INFO | BytesWritten : 1158000204077 | 20260506 14:55:03.012 | INFO | NumEvents : 560401949736 | 20260506 14:55:04.756 | INFO | Analyzing chunk frames = [2232320,2232832) | 20260506 14:55:04.756 | INFO | MemInfo: { | 20260506 14:55:04.756 | INFO | "gpu" : " 17 GB", | 20260506 14:55:04.756 | INFO | "huge" : "", | 20260506 14:55:04.756 | INFO | "huge_shared" : " 27 GB", | 20260506 14:55:04.756 | INFO | "malloc" : " 14 GB", | 20260506 14:55:04.756 | INFO | "pinned" : " 29 GB" | 20260506 14:55:04.756 | INFO | } | 20260506 14:55:04.756 | INFO | Perf report for the last 5.13724 seconds: | 20260506 14:55:04.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:55:04.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:04.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:04.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:04.756 | INFO | ANALYSIS : 63.1%, 2.9, 95.4ms, 0 | 20260506 14:55:04.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:04.756 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 14:55:04.756 | INFO | PRE_HQ : 61.0%, 1.9, 59.7ms, 0 | 20260506 14:55:04.756 | INFO | BAZWRITER : 63.2%, 4.8, 154.6ms, 0 | 20260506 14:55:04.756 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 14:55:04.757 | INFO | Uploaded 36GiB to GPU in 2.10157s (17.1301GiB/s) | 20260506 14:55:04.757 | INFO | Downloaded 38GiB to GPU in 2.24078s (16.9584GiB/s) | 20260506 14:55:09.896 | INFO | Analyzing chunk frames = [2232832,2233344) | 20260506 14:55:09.896 | INFO | MemInfo: { | 20260506 14:55:09.896 | INFO | "gpu" : " 17 GB", | 20260506 14:55:09.896 | INFO | "huge" : "", | 20260506 14:55:09.896 | INFO | "huge_shared" : " 27 GB", | 20260506 14:55:09.896 | INFO | "malloc" : " 14 GB", | 20260506 14:55:09.896 | INFO | "pinned" : " 29 GB" | 20260506 14:55:09.896 | INFO | } | 20260506 14:55:09.896 | INFO | Perf report for the last 5.13956 seconds: | 20260506 14:55:09.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:55:09.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:09.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:09.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:09.896 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 14:55:09.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:09.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:09.896 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:55:09.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:09.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:09.896 | INFO | Uploaded 36GiB to GPU in 2.00056s (17.9949GiB/s) | 20260506 14:55:09.896 | INFO | Downloaded 34GiB to GPU in 1.93897s (17.5351GiB/s) | 20260506 14:55:15.034 | INFO | Analyzing chunk frames = [2233344,2233856) | 20260506 14:55:15.034 | INFO | MemInfo: { | 20260506 14:55:15.034 | INFO | "gpu" : " 17 GB", | 20260506 14:55:15.034 | INFO | "huge" : "", | 20260506 14:55:15.034 | INFO | "huge_shared" : " 27 GB", | 20260506 14:55:15.034 | INFO | "malloc" : " 14 GB", | 20260506 14:55:15.034 | INFO | "pinned" : " 29 GB" | 20260506 14:55:15.034 | INFO | } | 20260506 14:55:15.034 | INFO | Perf report for the last 5.13841 seconds: | 20260506 14:55:15.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:55:15.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:15.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:15.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:15.034 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 14:55:15.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:15.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:15.034 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:55:15.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:15.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:15.035 | INFO | Uploaded 36GiB to GPU in 1.99104s (18.081GiB/s) | 20260506 14:55:15.035 | INFO | Downloaded 34GiB to GPU in 1.94047s (17.5215GiB/s) | 20260506 14:55:20.173 | INFO | Analyzing chunk frames = [2233856,2234368) | 20260506 14:55:20.173 | INFO | MemInfo: { | 20260506 14:55:20.173 | INFO | "gpu" : " 17 GB", | 20260506 14:55:20.173 | INFO | "huge" : "", | 20260506 14:55:20.173 | INFO | "huge_shared" : " 27 GB", | 20260506 14:55:20.173 | INFO | "malloc" : " 14 GB", | 20260506 14:55:20.173 | INFO | "pinned" : " 29 GB" | 20260506 14:55:20.173 | INFO | } | 20260506 14:55:20.173 | INFO | Perf report for the last 5.1389 seconds: | 20260506 14:55:20.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:55:20.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:20.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:20.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:20.173 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 14:55:20.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:20.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:20.173 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:55:20.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:20.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:20.173 | INFO | Uploaded 36GiB to GPU in 2.0307s (17.7279GiB/s) | 20260506 14:55:20.174 | INFO | Downloaded 34GiB to GPU in 1.91251s (17.7776GiB/s) | 20260506 14:55:25.313 | INFO | Analyzing chunk frames = [2234368,2234880) | 20260506 14:55:25.313 | INFO | MemInfo: { | 20260506 14:55:25.313 | INFO | "gpu" : " 17 GB", | 20260506 14:55:25.313 | INFO | "huge" : "", | 20260506 14:55:25.313 | INFO | "huge_shared" : " 27 GB", | 20260506 14:55:25.313 | INFO | "malloc" : " 14 GB", | 20260506 14:55:25.313 | INFO | "pinned" : " 29 GB" | 20260506 14:55:25.313 | INFO | } | 20260506 14:55:25.313 | INFO | Perf report for the last 5.13995 seconds: | 20260506 14:55:25.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:55:25.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:25.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:25.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:25.313 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 14:55:25.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:25.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:25.313 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 14:55:25.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:25.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:25.313 | INFO | Uploaded 36GiB to GPU in 2.01077s (17.9036GiB/s) | 20260506 14:55:25.314 | INFO | Downloaded 34GiB to GPU in 1.93285s (17.5906GiB/s) | 20260506 14:55:30.455 | INFO | Analyzing chunk frames = [2234880,2235392) | 20260506 14:55:30.455 | INFO | MemInfo: { | 20260506 14:55:30.455 | INFO | "gpu" : " 17 GB", | 20260506 14:55:30.455 | INFO | "huge" : "", | 20260506 14:55:30.455 | INFO | "huge_shared" : " 27 GB", | 20260506 14:55:30.455 | INFO | "malloc" : " 14 GB", | 20260506 14:55:30.455 | INFO | "pinned" : " 29 GB" | 20260506 14:55:30.455 | INFO | } | 20260506 14:55:30.455 | INFO | Perf report for the last 5.1421 seconds: | 20260506 14:55:30.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:55:30.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:30.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:30.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:30.455 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:55:30.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:30.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:30.455 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 14:55:30.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:30.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:30.456 | INFO | Uploaded 36GiB to GPU in 2.08489s (17.2671GiB/s) | 20260506 14:55:30.456 | INFO | Downloaded 34GiB to GPU in 1.96049s (17.3426GiB/s) | 20260506 14:55:35.591 | INFO | Analyzing chunk frames = [2235392,2235904) | 20260506 14:55:35.591 | INFO | MemInfo: { | 20260506 14:55:35.591 | INFO | "gpu" : " 17 GB", | 20260506 14:55:35.591 | INFO | "huge" : "", | 20260506 14:55:35.591 | INFO | "huge_shared" : " 27 GB", | 20260506 14:55:35.591 | INFO | "malloc" : " 14 GB", | 20260506 14:55:35.591 | INFO | "pinned" : " 29 GB" | 20260506 14:55:35.591 | INFO | } | 20260506 14:55:35.591 | INFO | Perf report for the last 5.13603 seconds: | 20260506 14:55:35.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:55:35.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:35.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:35.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:35.591 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 14:55:35.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:35.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:35.591 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 14:55:35.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:35.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:35.592 | INFO | Uploaded 36GiB to GPU in 1.99835s (18.0148GiB/s) | 20260506 14:55:35.592 | INFO | Downloaded 34GiB to GPU in 1.97967s (17.1746GiB/s) | 20260506 14:55:40.730 | INFO | Analyzing chunk frames = [2235904,2236416) | 20260506 14:55:40.730 | INFO | MemInfo: { | 20260506 14:55:40.730 | INFO | "gpu" : " 17 GB", | 20260506 14:55:40.730 | INFO | "huge" : "", | 20260506 14:55:40.730 | INFO | "huge_shared" : " 27 GB", | 20260506 14:55:40.730 | INFO | "malloc" : " 14 GB", | 20260506 14:55:40.730 | INFO | "pinned" : " 29 GB" | 20260506 14:55:40.730 | INFO | } | 20260506 14:55:40.730 | INFO | Perf report for the last 5.13894 seconds: | 20260506 14:55:40.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:55:40.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:40.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:40.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:40.730 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:55:40.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:40.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:40.730 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:55:40.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:40.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:40.731 | INFO | Uploaded 36GiB to GPU in 2.01172s (17.8951GiB/s) | 20260506 14:55:40.731 | INFO | Downloaded 34GiB to GPU in 1.98532s (17.1257GiB/s) | 20260506 14:55:40.731 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:55:40.731 | INFO | Total mem usage: 17.4 GiB | 20260506 14:55:40.731 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:55:40.731 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -8.37%) | 20260506 14:55:40.731 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:55:40.731 | INFO | High Water Breakdown: | 20260506 14:55:40.731 | INFO | 61.6% : 9.9 GiB ( 90.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:55:40.731 | INFO | 38.2% : 6.14 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:55:40.731 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:55:40.731 | INFO | ------------------------------------------ | 20260506 14:55:40.731 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:55:40.731 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:55:40.731 | INFO | Total mem usage: 26.5 GiB | 20260506 14:55:40.731 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:55:40.731 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:55:40.731 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:55:40.731 | INFO | High Water Breakdown: | 20260506 14:55:40.731 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:55:40.731 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:55:40.731 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:55:40.731 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:55:40.731 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:55:40.731 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:55:40.731 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:55:40.731 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:55:40.731 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:55:40.731 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:55:40.731 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:55:40.731 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:55:40.731 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:55:40.731 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:55:40.731 | INFO | ------------------------------------------ | 20260506 14:55:40.731 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:55:40.731 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:55:40.731 | INFO | Total mem usage: 25.6 GiB | 20260506 14:55:40.731 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:55:40.731 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:55:40.731 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:55:40.731 | INFO | High Water Breakdown: | 20260506 14:55:40.731 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:55:40.731 | INFO | ------------------------------------------ | 20260506 14:55:40.731 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:55:40.731 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:55:40.731 | INFO | Total mem usage: 15.7 GiB | 20260506 14:55:40.731 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:55:40.731 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:55:40.731 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 14:55:40.731 | INFO | High Water Breakdown: | 20260506 14:55:40.731 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:55:40.731 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:55:40.731 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:55:40.731 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:55:40.731 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:55:40.731 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:55:40.731 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:55:40.731 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:55:40.731 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:55:40.731 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:55:40.731 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:55:40.731 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:55:40.731 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:55:40.731 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:55:40.731 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:55:40.731 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:55:40.731 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:55:40.731 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:55:40.731 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:55:40.731 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:55:40.731 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:55:40.731 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:55:40.731 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:55:40.731 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:55:40.731 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:55:40.731 | INFO | ------------------------------------------ | 20260506 14:55:40.731 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:55:40.731 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:55:40.731 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:55:40.731 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:55:40.731 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.58e+01, 1.34e+01 / 2.10e+00 | 20260506 14:55:40.731 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.01e-01, 1.05e-02 / 6.29e-03 | 20260506 14:55:40.731 | INFO | PerfReport: 11.58%, Baseline -- 3.64e+00 / 4.47e+00, 5.65e-01 / 3.36e-01 | 20260506 14:55:40.731 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.87e-01 / 1.33e-01 | 20260506 14:55:40.732 | INFO | PerfReport: 15.92%, DME -- 4.00e+01 / 4.45e+01, 4.04e+00 / 2.22e+00 | 20260506 14:55:40.732 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.37e+00 / 1.94e+00 | 20260506 14:55:40.732 | INFO | PerfReport: 7.46%, PulseAccumulating -- 2.34e+00 / 2.94e+00, 4.12e-01 / 2.47e-01 | 20260506 14:55:40.732 | INFO | PerfReport: 3.59%, Metrics -- 1.13e+00 / 1.48e+00, 2.88e-01 / 1.70e-01 | 20260506 14:55:40.732 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.94e-01 / 4.45e-01, 6.24e-02 / 2.77e-02 | 20260506 14:55:40.732 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:55:40.732 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:55:40.732 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:55:40.732 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 14:55:40.732 | INFO | PerfReport: 21.61%, Upload -- 2.11e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260506 14:55:40.732 | INFO | PerfReport: 21.18%, Download -- 2.07e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 14:55:40.732 | INFO | PerfReport: 32.18%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 14:55:40.732 | INFO | PerfReport: 1.67%, LockUpload -- 1.63e+00 / 0.00e+00, 5.98e+00 / 0.00e+00 | 20260506 14:55:40.732 | INFO | PerfReport: 1.03%, LockDownload -- 1.00e+00 / 0.00e+00, 2.56e+00 / 0.00e+00 | 20260506 14:55:40.732 | INFO | PerfReport: 22.31%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 14:55:40.732 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260506 14:55:40.842 | INFO | Starting a round of Baz IO | 20260506 14:55:44.006 | INFO | Populating RTMetrics report | 20260506 14:55:44.006 | INFO | RTMetrics report populated | 20260506 14:55:44.006 | INFO | Opening tmp RTMetrics json file | 20260506 14:55:44.014 | INFO | Serializing RTMetrics to json | 20260506 14:55:44.016 | INFO | Serialization done, swapping tmp file now... | 20260506 14:55:44.017 | INFO | RTMetrics done | 20260506 14:55:44.178 | INFO | Finished a round of Baz IO: Wrote 2.108 GiB in 3.335 seconds | 20260506 14:55:44.178 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.74% / 0.01% / 62.25% | 20260506 14:55:44.178 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.30579% / 1.58946e-05% / 5.96046e-05% | 20260506 14:55:44.178 | INFO | Baz IO overall: | 20260506 14:55:44.178 | INFO | BytesWritten : 1160263811686 | 20260506 14:55:44.178 | INFO | NumEvents : 561497945862 | 20260506 14:55:45.871 | INFO | Analyzing chunk frames = [2236416,2236928) | 20260506 14:55:45.871 | INFO | MemInfo: { | 20260506 14:55:45.871 | INFO | "gpu" : " 17 GB", | 20260506 14:55:45.871 | INFO | "huge" : "", | 20260506 14:55:45.871 | INFO | "huge_shared" : " 27 GB", | 20260506 14:55:45.871 | INFO | "malloc" : " 14 GB", | 20260506 14:55:45.871 | INFO | "pinned" : " 29 GB" | 20260506 14:55:45.871 | INFO | } | 20260506 14:55:45.871 | INFO | Perf report for the last 5.14073 seconds: | 20260506 14:55:45.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:55:45.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:45.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:45.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:45.871 | INFO | ANALYSIS : 63.1%, 2.9, 95.5ms, 0 | 20260506 14:55:45.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:45.871 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 14:55:45.871 | INFO | PRE_HQ : 60.9%, 1.9, 59.7ms, 0 | 20260506 14:55:45.871 | INFO | BAZWRITER : 64.3%, 4.7, 155.1ms, 0 | 20260506 14:55:45.871 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.3ms, 0 | 20260506 14:55:45.871 | INFO | Uploaded 36GiB to GPU in 2.04257s (17.6249GiB/s) | 20260506 14:55:45.871 | INFO | Downloaded 38GiB to GPU in 2.23811s (16.9786GiB/s) | 20260506 14:55:51.011 | INFO | Analyzing chunk frames = [2236928,2237440) | 20260506 14:55:51.011 | INFO | MemInfo: { | 20260506 14:55:51.011 | INFO | "gpu" : " 17 GB", | 20260506 14:55:51.011 | INFO | "huge" : "", | 20260506 14:55:51.011 | INFO | "huge_shared" : " 27 GB", | 20260506 14:55:51.011 | INFO | "malloc" : " 14 GB", | 20260506 14:55:51.011 | INFO | "pinned" : " 29 GB" | 20260506 14:55:51.011 | INFO | } | 20260506 14:55:51.011 | INFO | Perf report for the last 5.13989 seconds: | 20260506 14:55:51.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:55:51.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:51.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:51.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:51.011 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 14:55:51.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:51.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:51.011 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 14:55:51.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:51.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:51.011 | INFO | Uploaded 36GiB to GPU in 1.99454s (18.0493GiB/s) | 20260506 14:55:51.011 | INFO | Downloaded 34GiB to GPU in 1.94257s (17.5026GiB/s) | 20260506 14:55:56.149 | INFO | Analyzing chunk frames = [2237440,2237952) | 20260506 14:55:56.149 | INFO | MemInfo: { | 20260506 14:55:56.149 | INFO | "gpu" : " 17 GB", | 20260506 14:55:56.149 | INFO | "huge" : "", | 20260506 14:55:56.149 | INFO | "huge_shared" : " 27 GB", | 20260506 14:55:56.149 | INFO | "malloc" : " 14 GB", | 20260506 14:55:56.149 | INFO | "pinned" : " 29 GB" | 20260506 14:55:56.149 | INFO | } | 20260506 14:55:56.149 | INFO | Perf report for the last 5.13799 seconds: | 20260506 14:55:56.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:55:56.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:56.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:56.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:56.149 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 14:55:56.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:56.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:56.149 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:55:56.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:56.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:55:56.149 | INFO | Uploaded 36GiB to GPU in 1.96583s (18.3129GiB/s) | 20260506 14:55:56.149 | INFO | Downloaded 34GiB to GPU in 1.94602s (17.4716GiB/s) | 20260506 14:56:01.288 | INFO | Analyzing chunk frames = [2237952,2238464) | 20260506 14:56:01.288 | INFO | MemInfo: { | 20260506 14:56:01.288 | INFO | "gpu" : " 17 GB", | 20260506 14:56:01.288 | INFO | "huge" : "", | 20260506 14:56:01.288 | INFO | "huge_shared" : " 27 GB", | 20260506 14:56:01.288 | INFO | "malloc" : " 14 GB", | 20260506 14:56:01.288 | INFO | "pinned" : " 29 GB" | 20260506 14:56:01.288 | INFO | } | 20260506 14:56:01.289 | INFO | Perf report for the last 5.13956 seconds: | 20260506 14:56:01.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:56:01.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:01.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:01.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:01.289 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:56:01.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:01.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:01.289 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:56:01.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:01.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:01.289 | INFO | Uploaded 36GiB to GPU in 2.01841s (17.8358GiB/s) | 20260506 14:56:01.289 | INFO | Downloaded 34GiB to GPU in 1.91846s (17.7226GiB/s) | 20260506 14:56:06.427 | INFO | Analyzing chunk frames = [2238464,2238976) | 20260506 14:56:06.427 | INFO | MemInfo: { | 20260506 14:56:06.427 | INFO | "gpu" : " 17 GB", | 20260506 14:56:06.427 | INFO | "huge" : "", | 20260506 14:56:06.427 | INFO | "huge_shared" : " 27 GB", | 20260506 14:56:06.427 | INFO | "malloc" : " 14 GB", | 20260506 14:56:06.427 | INFO | "pinned" : " 29 GB" | 20260506 14:56:06.427 | INFO | } | 20260506 14:56:06.427 | INFO | Perf report for the last 5.13878 seconds: | 20260506 14:56:06.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:56:06.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:06.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:06.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:06.427 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 14:56:06.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:06.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:06.427 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:56:06.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:06.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:06.428 | INFO | Uploaded 36GiB to GPU in 1.98914s (18.0983GiB/s) | 20260506 14:56:06.428 | INFO | Downloaded 34GiB to GPU in 1.92979s (17.6185GiB/s) | 20260506 14:56:11.568 | INFO | Analyzing chunk frames = [2238976,2239488) | 20260506 14:56:11.568 | INFO | MemInfo: { | 20260506 14:56:11.568 | INFO | "gpu" : " 17 GB", | 20260506 14:56:11.568 | INFO | "huge" : "", | 20260506 14:56:11.568 | INFO | "huge_shared" : " 27 GB", | 20260506 14:56:11.568 | INFO | "malloc" : " 14 GB", | 20260506 14:56:11.568 | INFO | "pinned" : " 29 GB" | 20260506 14:56:11.568 | INFO | } | 20260506 14:56:11.568 | INFO | Perf report for the last 5.14056 seconds: | 20260506 14:56:11.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:56:11.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:11.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:11.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:11.568 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 14:56:11.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:11.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:11.568 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260506 14:56:11.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:11.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:11.568 | INFO | Uploaded 36GiB to GPU in 2.06851s (17.4038GiB/s) | 20260506 14:56:11.568 | INFO | Downloaded 34GiB to GPU in 1.9766s (17.2013GiB/s) | 20260506 14:56:16.709 | INFO | Analyzing chunk frames = [2239488,2240000) | 20260506 14:56:16.709 | INFO | MemInfo: { | 20260506 14:56:16.709 | INFO | "gpu" : " 17 GB", | 20260506 14:56:16.709 | INFO | "huge" : "", | 20260506 14:56:16.709 | INFO | "huge_shared" : " 27 GB", | 20260506 14:56:16.709 | INFO | "malloc" : " 14 GB", | 20260506 14:56:16.709 | INFO | "pinned" : " 29 GB" | 20260506 14:56:16.709 | INFO | } | 20260506 14:56:16.709 | INFO | Perf report for the last 5.14095 seconds: | 20260506 14:56:16.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:56:16.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:16.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:16.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:16.709 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:56:16.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:16.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:16.709 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 14:56:16.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:16.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:16.709 | INFO | Uploaded 36GiB to GPU in 2.0233s (17.7928GiB/s) | 20260506 14:56:16.709 | INFO | Downloaded 34GiB to GPU in 1.9836s (17.1406GiB/s) | 20260506 14:56:21.845 | INFO | Analyzing chunk frames = [2240000,2240512) | 20260506 14:56:21.845 | INFO | MemInfo: { | 20260506 14:56:21.845 | INFO | "gpu" : " 17 GB", | 20260506 14:56:21.845 | INFO | "huge" : "", | 20260506 14:56:21.845 | INFO | "huge_shared" : " 27 GB", | 20260506 14:56:21.845 | INFO | "malloc" : " 14 GB", | 20260506 14:56:21.845 | INFO | "pinned" : " 29 GB" | 20260506 14:56:21.845 | INFO | } | 20260506 14:56:21.845 | INFO | Perf report for the last 5.1365 seconds: | 20260506 14:56:21.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:56:21.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:21.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:21.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:21.845 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:56:21.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:21.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:21.845 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:56:21.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:21.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:21.846 | INFO | Uploaded 36GiB to GPU in 2.02154s (17.8082GiB/s) | 20260506 14:56:21.846 | INFO | Downloaded 34GiB to GPU in 1.98804s (17.1023GiB/s) | 20260506 14:56:21.954 | INFO | Starting a round of Baz IO | 20260506 14:56:25.110 | INFO | Populating RTMetrics report | 20260506 14:56:25.110 | INFO | RTMetrics report populated | 20260506 14:56:25.110 | INFO | Opening tmp RTMetrics json file | 20260506 14:56:25.115 | INFO | Serializing RTMetrics to json | 20260506 14:56:25.175 | INFO | Serialization done, swapping tmp file now... | 20260506 14:56:25.193 | INFO | RTMetrics done | 20260506 14:56:25.431 | INFO | Finished a round of Baz IO: Wrote 2.106 GiB in 3.476 seconds | 20260506 14:56:25.431 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.73% / 0.01% / 62.26% | 20260506 14:56:25.431 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.29795% / 2.38419e-05% / 6.35783e-05% | 20260506 14:56:25.431 | INFO | Baz IO overall: | 20260506 14:56:25.431 | INFO | BytesWritten : 1162525581043 | 20260506 14:56:25.431 | INFO | NumEvents : 562592923673 | 20260506 14:56:26.984 | INFO | Analyzing chunk frames = [2240512,2241024) | 20260506 14:56:26.984 | INFO | MemInfo: { | 20260506 14:56:26.984 | INFO | "gpu" : " 17 GB", | 20260506 14:56:26.984 | INFO | "huge" : "", | 20260506 14:56:26.984 | INFO | "huge_shared" : " 27 GB", | 20260506 14:56:26.984 | INFO | "malloc" : " 14 GB", | 20260506 14:56:26.984 | INFO | "pinned" : " 29 GB" | 20260506 14:56:26.984 | INFO | } | 20260506 14:56:26.984 | INFO | Perf report for the last 5.13881 seconds: | 20260506 14:56:26.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:56:26.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:26.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:26.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:26.984 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 14:56:26.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:26.984 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 14:56:26.984 | INFO | PRE_HQ : 60.8%, 1.9, 59.1ms, 0 | 20260506 14:56:26.984 | INFO | BAZWRITER : 67.0%, 4.8, 166.2ms, 0 | 20260506 14:56:26.984 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260506 14:56:26.984 | INFO | Uploaded 36GiB to GPU in 2.04636s (17.5922GiB/s) | 20260506 14:56:26.984 | INFO | Downloaded 38GiB to GPU in 2.2282s (17.0541GiB/s) | 20260506 14:56:32.124 | INFO | Analyzing chunk frames = [2241024,2241536) | 20260506 14:56:32.124 | INFO | MemInfo: { | 20260506 14:56:32.124 | INFO | "gpu" : " 17 GB", | 20260506 14:56:32.124 | INFO | "huge" : "", | 20260506 14:56:32.124 | INFO | "huge_shared" : " 27 GB", | 20260506 14:56:32.124 | INFO | "malloc" : " 14 GB", | 20260506 14:56:32.124 | INFO | "pinned" : " 29 GB" | 20260506 14:56:32.124 | INFO | } | 20260506 14:56:32.125 | INFO | Perf report for the last 5.14034 seconds: | 20260506 14:56:32.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:56:32.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:32.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:32.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:32.125 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 14:56:32.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:32.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:32.125 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 14:56:32.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:32.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:32.125 | INFO | Uploaded 36GiB to GPU in 1.97687s (18.2106GiB/s) | 20260506 14:56:32.125 | INFO | Downloaded 34GiB to GPU in 1.93265s (17.5924GiB/s) | 20260506 14:56:37.263 | INFO | Analyzing chunk frames = [2241536,2242048) | 20260506 14:56:37.263 | INFO | MemInfo: { | 20260506 14:56:37.263 | INFO | "gpu" : " 17 GB", | 20260506 14:56:37.264 | INFO | "huge" : "", | 20260506 14:56:37.264 | INFO | "huge_shared" : " 27 GB", | 20260506 14:56:37.264 | INFO | "malloc" : " 14 GB", | 20260506 14:56:37.264 | INFO | "pinned" : " 29 GB" | 20260506 14:56:37.264 | INFO | } | 20260506 14:56:37.264 | INFO | Perf report for the last 5.13911 seconds: | 20260506 14:56:37.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:56:37.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:37.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:37.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:37.264 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 14:56:37.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:37.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:37.264 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:56:37.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:37.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:37.264 | INFO | Uploaded 36GiB to GPU in 1.96394s (18.3305GiB/s) | 20260506 14:56:37.264 | INFO | Downloaded 34GiB to GPU in 1.93978s (17.5277GiB/s) | 20260506 14:56:42.403 | INFO | Analyzing chunk frames = [2242048,2242560) | 20260506 14:56:42.403 | INFO | MemInfo: { | 20260506 14:56:42.403 | INFO | "gpu" : " 17 GB", | 20260506 14:56:42.403 | INFO | "huge" : "", | 20260506 14:56:42.403 | INFO | "huge_shared" : " 27 GB", | 20260506 14:56:42.403 | INFO | "malloc" : " 14 GB", | 20260506 14:56:42.403 | INFO | "pinned" : " 29 GB" | 20260506 14:56:42.403 | INFO | } | 20260506 14:56:42.403 | INFO | Perf report for the last 5.13926 seconds: | 20260506 14:56:42.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:56:42.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:42.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:42.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:42.403 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260506 14:56:42.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:42.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:42.403 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:56:42.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:42.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:42.403 | INFO | Uploaded 36GiB to GPU in 2.00582s (17.9478GiB/s) | 20260506 14:56:42.403 | INFO | Downloaded 34GiB to GPU in 1.9132s (17.7713GiB/s) | 20260506 14:56:47.542 | INFO | Analyzing chunk frames = [2242560,2243072) | 20260506 14:56:47.542 | INFO | MemInfo: { | 20260506 14:56:47.542 | INFO | "gpu" : " 17 GB", | 20260506 14:56:47.542 | INFO | "huge" : "", | 20260506 14:56:47.542 | INFO | "huge_shared" : " 27 GB", | 20260506 14:56:47.542 | INFO | "malloc" : " 14 GB", | 20260506 14:56:47.542 | INFO | "pinned" : " 29 GB" | 20260506 14:56:47.542 | INFO | } | 20260506 14:56:47.542 | INFO | Perf report for the last 5.13917 seconds: | 20260506 14:56:47.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:56:47.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:47.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:47.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:47.542 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 14:56:47.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:47.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:47.542 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:56:47.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:47.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:47.542 | INFO | Uploaded 36GiB to GPU in 2.00473s (17.9575GiB/s) | 20260506 14:56:47.542 | INFO | Downloaded 34GiB to GPU in 1.94731s (17.46GiB/s) | 20260506 14:56:52.682 | INFO | Analyzing chunk frames = [2243072,2243584) | 20260506 14:56:52.682 | INFO | MemInfo: { | 20260506 14:56:52.682 | INFO | "gpu" : " 17 GB", | 20260506 14:56:52.682 | INFO | "huge" : "", | 20260506 14:56:52.682 | INFO | "huge_shared" : " 27 GB", | 20260506 14:56:52.682 | INFO | "malloc" : " 14 GB", | 20260506 14:56:52.682 | INFO | "pinned" : " 29 GB" | 20260506 14:56:52.682 | INFO | } | 20260506 14:56:52.682 | INFO | Perf report for the last 5.14001 seconds: | 20260506 14:56:52.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:56:52.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:52.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:52.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:52.682 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 14:56:52.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:52.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:52.682 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 14:56:52.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:52.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:52.683 | INFO | Uploaded 36GiB to GPU in 2.03664s (17.6762GiB/s) | 20260506 14:56:52.683 | INFO | Downloaded 34GiB to GPU in 1.91924s (17.7154GiB/s) | 20260506 14:56:57.822 | INFO | Analyzing chunk frames = [2243584,2244096) | 20260506 14:56:57.822 | INFO | MemInfo: { | 20260506 14:56:57.822 | INFO | "gpu" : " 17 GB", | 20260506 14:56:57.822 | INFO | "huge" : "", | 20260506 14:56:57.822 | INFO | "huge_shared" : " 27 GB", | 20260506 14:56:57.822 | INFO | "malloc" : " 14 GB", | 20260506 14:56:57.822 | INFO | "pinned" : " 29 GB" | 20260506 14:56:57.822 | INFO | } | 20260506 14:56:57.822 | INFO | Perf report for the last 5.13979 seconds: | 20260506 14:56:57.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:56:57.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:57.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:57.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:57.822 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 14:56:57.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:57.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:57.822 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 14:56:57.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:57.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:56:57.822 | INFO | Uploaded 36GiB to GPU in 1.9552s (18.4125GiB/s) | 20260506 14:56:57.822 | INFO | Downloaded 34GiB to GPU in 1.96929s (17.2651GiB/s) | 20260506 14:57:02.961 | INFO | Analyzing chunk frames = [2244096,2244608) | 20260506 14:57:02.961 | INFO | MemInfo: { | 20260506 14:57:02.961 | INFO | "gpu" : " 17 GB", | 20260506 14:57:02.961 | INFO | "huge" : "", | 20260506 14:57:02.961 | INFO | "huge_shared" : " 27 GB", | 20260506 14:57:02.961 | INFO | "malloc" : " 14 GB", | 20260506 14:57:02.961 | INFO | "pinned" : " 29 GB" | 20260506 14:57:02.961 | INFO | } | 20260506 14:57:02.961 | INFO | Perf report for the last 5.1393 seconds: | 20260506 14:57:02.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:57:02.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:02.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:02.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:02.961 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:57:02.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:02.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:02.961 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 14:57:02.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:02.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:02.962 | INFO | Uploaded 36GiB to GPU in 1.97314s (18.2451GiB/s) | 20260506 14:57:02.962 | INFO | Downloaded 34GiB to GPU in 1.98052s (17.1672GiB/s) | 20260506 14:57:02.962 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:57:02.962 | INFO | Total mem usage: 17.4 GiB | 20260506 14:57:02.962 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:57:02.962 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -11%) | 20260506 14:57:02.962 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:57:02.962 | INFO | High Water Breakdown: | 20260506 14:57:02.962 | INFO | 61.3% : 9.62 GiB ( 88.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:57:02.962 | INFO | 38.5% : 6.05 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:57:02.962 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:57:02.962 | INFO | ------------------------------------------ | 20260506 14:57:02.962 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:57:02.962 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:57:02.962 | INFO | Total mem usage: 26.5 GiB | 20260506 14:57:02.962 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:57:02.962 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:57:02.962 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:57:02.962 | INFO | High Water Breakdown: | 20260506 14:57:02.962 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:57:02.962 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:57:02.962 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:57:02.962 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:57:02.962 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:57:02.962 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:57:02.962 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:57:02.962 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:57:02.962 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:57:02.962 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:57:02.962 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:57:02.962 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:57:02.962 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:57:02.962 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:57:02.962 | INFO | ------------------------------------------ | 20260506 14:57:02.962 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:57:02.962 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:57:02.962 | INFO | Total mem usage: 25.6 GiB | 20260506 14:57:02.962 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:57:02.962 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:57:02.962 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:57:02.962 | INFO | High Water Breakdown: | 20260506 14:57:02.962 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:57:02.962 | INFO | ------------------------------------------ | 20260506 14:57:02.962 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:57:02.962 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:57:02.962 | INFO | Total mem usage: 15.7 GiB | 20260506 14:57:02.962 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:57:02.962 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:57:02.962 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 14:57:02.962 | INFO | High Water Breakdown: | 20260506 14:57:02.962 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:57:02.962 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:57:02.962 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:57:02.962 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:57:02.962 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:57:02.962 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:57:02.962 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:57:02.962 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:57:02.962 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:57:02.962 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:57:02.962 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:57:02.962 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:57:02.962 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:57:02.963 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:57:02.963 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:57:02.963 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:57:02.963 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:57:02.963 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:57:02.963 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:57:02.963 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:57:02.963 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:57:02.963 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:57:02.963 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:57:02.963 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:57:02.963 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:57:02.963 | INFO | ------------------------------------------ | 20260506 14:57:02.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:57:02.963 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:57:02.963 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:57:02.963 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:57:02.963 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.58e+01, 1.35e+01 / 1.88e+00 | 20260506 14:57:02.963 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.01e-01, 1.06e-02 / 9.17e-03 | 20260506 14:57:02.963 | INFO | PerfReport: 11.60%, Baseline -- 3.65e+00 / 4.40e+00, 5.78e-01 / 4.24e-01 | 20260506 14:57:02.963 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.46e+00, 1.89e-01 / 1.50e-01 | 20260506 14:57:02.963 | INFO | PerfReport: 15.82%, DME -- 3.99e+01 / 4.41e+01, 3.76e+00 / 1.98e+00 | 20260506 14:57:02.963 | INFO | PerfReport: 56.81%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.51e+00 / 1.96e+00 | 20260506 14:57:02.963 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.33e+00 / 2.94e+00, 4.02e-01 / 2.16e-01 | 20260506 14:57:02.963 | INFO | PerfReport: 3.58%, Metrics -- 1.13e+00 / 1.49e+00, 2.94e-01 / 2.14e-01 | 20260506 14:57:02.963 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.94e-01 / 4.46e-01, 6.22e-02 / 2.20e-02 | 20260506 14:57:02.963 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:57:02.963 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:57:02.963 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:57:02.963 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 14:57:02.963 | INFO | PerfReport: 21.41%, Upload -- 2.09e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260506 14:57:02.963 | INFO | PerfReport: 21.19%, Download -- 2.07e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 14:57:02.963 | INFO | PerfReport: 32.30%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 14:57:02.963 | INFO | PerfReport: 1.43%, LockUpload -- 1.40e+00 / 0.00e+00, 5.62e+00 / 0.00e+00 | 20260506 14:57:02.963 | INFO | PerfReport: 1.16%, LockDownload -- 1.13e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 14:57:02.963 | INFO | PerfReport: 22.50%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 14:57:02.963 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260506 14:57:03.081 | INFO | Starting a round of Baz IO | 20260506 14:57:06.226 | INFO | Populating RTMetrics report | 20260506 14:57:06.226 | INFO | RTMetrics report populated | 20260506 14:57:06.226 | INFO | Opening tmp RTMetrics json file | 20260506 14:57:06.247 | INFO | Serializing RTMetrics to json | 20260506 14:57:06.263 | INFO | Serialization done, swapping tmp file now... | 20260506 14:57:06.274 | INFO | RTMetrics done | 20260506 14:57:06.365 | INFO | Finished a round of Baz IO: Wrote 2.107 GiB in 3.283 seconds | 20260506 14:57:06.365 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.72% / 0.01% / 62.27% | 20260506 14:57:06.365 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.29376% / 1.98682e-05% / 6.75519e-05% | 20260506 14:57:06.365 | INFO | Baz IO overall: | 20260506 14:57:06.365 | INFO | BytesWritten : 1164788421534 | 20260506 14:57:06.365 | INFO | NumEvents : 563688780177 | 20260506 14:57:08.100 | INFO | Analyzing chunk frames = [2244608,2245120) | 20260506 14:57:08.100 | INFO | MemInfo: { | 20260506 14:57:08.100 | INFO | "gpu" : " 17 GB", | 20260506 14:57:08.100 | INFO | "huge" : "", | 20260506 14:57:08.100 | INFO | "huge_shared" : " 27 GB", | 20260506 14:57:08.100 | INFO | "malloc" : " 14 GB", | 20260506 14:57:08.100 | INFO | "pinned" : " 29 GB" | 20260506 14:57:08.100 | INFO | } | 20260506 14:57:08.100 | INFO | Perf report for the last 5.13888 seconds: | 20260506 14:57:08.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:57:08.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:08.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:08.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:08.100 | INFO | ANALYSIS : 62.9%, 2.9, 95.1ms, 0 | 20260506 14:57:08.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:08.100 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 14:57:08.100 | INFO | PRE_HQ : 60.6%, 1.9, 59.6ms, 0 | 20260506 14:57:08.100 | INFO | BAZWRITER : 63.5%, 4.8, 156.5ms, 0 | 20260506 14:57:08.100 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 14:57:08.100 | INFO | Uploaded 36GiB to GPU in 2.05017s (17.5595GiB/s) | 20260506 14:57:08.100 | INFO | Downloaded 38GiB to GPU in 2.2176s (17.1357GiB/s) | 20260506 14:57:13.239 | INFO | Analyzing chunk frames = [2245120,2245632) | 20260506 14:57:13.239 | INFO | MemInfo: { | 20260506 14:57:13.239 | INFO | "gpu" : " 17 GB", | 20260506 14:57:13.239 | INFO | "huge" : "", | 20260506 14:57:13.239 | INFO | "huge_shared" : " 27 GB", | 20260506 14:57:13.239 | INFO | "malloc" : " 14 GB", | 20260506 14:57:13.239 | INFO | "pinned" : " 29 GB" | 20260506 14:57:13.239 | INFO | } | 20260506 14:57:13.239 | INFO | Perf report for the last 5.13876 seconds: | 20260506 14:57:13.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:57:13.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:13.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:13.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:13.239 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 14:57:13.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:13.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:13.239 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:57:13.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:13.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:13.239 | INFO | Uploaded 36GiB to GPU in 1.98946s (18.0953GiB/s) | 20260506 14:57:13.239 | INFO | Downloaded 34GiB to GPU in 1.93639s (17.5584GiB/s) | 20260506 14:57:18.378 | INFO | Analyzing chunk frames = [2245632,2246144) | 20260506 14:57:18.378 | INFO | MemInfo: { | 20260506 14:57:18.378 | INFO | "gpu" : " 17 GB", | 20260506 14:57:18.378 | INFO | "huge" : "", | 20260506 14:57:18.378 | INFO | "huge_shared" : " 27 GB", | 20260506 14:57:18.378 | INFO | "malloc" : " 14 GB", | 20260506 14:57:18.378 | INFO | "pinned" : " 29 GB" | 20260506 14:57:18.378 | INFO | } | 20260506 14:57:18.378 | INFO | Perf report for the last 5.13937 seconds: | 20260506 14:57:18.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:57:18.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:18.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:18.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:18.378 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 14:57:18.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:18.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:18.378 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:57:18.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:18.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:18.379 | INFO | Uploaded 36GiB to GPU in 1.96395s (18.3304GiB/s) | 20260506 14:57:18.379 | INFO | Downloaded 34GiB to GPU in 1.94396s (17.4901GiB/s) | 20260506 14:57:23.518 | INFO | Analyzing chunk frames = [2246144,2246656) | 20260506 14:57:23.518 | INFO | MemInfo: { | 20260506 14:57:23.518 | INFO | "gpu" : " 17 GB", | 20260506 14:57:23.518 | INFO | "huge" : "", | 20260506 14:57:23.518 | INFO | "huge_shared" : " 27 GB", | 20260506 14:57:23.518 | INFO | "malloc" : " 14 GB", | 20260506 14:57:23.518 | INFO | "pinned" : " 29 GB" | 20260506 14:57:23.518 | INFO | } | 20260506 14:57:23.518 | INFO | Perf report for the last 5.13939 seconds: | 20260506 14:57:23.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:57:23.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:23.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:23.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:23.518 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:57:23.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:23.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:23.518 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:57:23.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:23.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:23.518 | INFO | Uploaded 36GiB to GPU in 2.01006s (17.9099GiB/s) | 20260506 14:57:23.518 | INFO | Downloaded 34GiB to GPU in 1.92001s (17.7082GiB/s) | 20260506 14:57:28.657 | INFO | Analyzing chunk frames = [2246656,2247168) | 20260506 14:57:28.657 | INFO | MemInfo: { | 20260506 14:57:28.657 | INFO | "gpu" : " 17 GB", | 20260506 14:57:28.657 | INFO | "huge" : "", | 20260506 14:57:28.657 | INFO | "huge_shared" : " 27 GB", | 20260506 14:57:28.657 | INFO | "malloc" : " 14 GB", | 20260506 14:57:28.657 | INFO | "pinned" : " 29 GB" | 20260506 14:57:28.657 | INFO | } | 20260506 14:57:28.657 | INFO | Perf report for the last 5.13946 seconds: | 20260506 14:57:28.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:57:28.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:28.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:28.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:28.657 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 14:57:28.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:28.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:28.657 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:57:28.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:28.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:28.657 | INFO | Uploaded 36GiB to GPU in 1.99266s (18.0663GiB/s) | 20260506 14:57:28.657 | INFO | Downloaded 34GiB to GPU in 1.93283s (17.5908GiB/s) | 20260506 14:57:33.797 | INFO | Analyzing chunk frames = [2247168,2247680) | 20260506 14:57:33.797 | INFO | MemInfo: { | 20260506 14:57:33.797 | INFO | "gpu" : " 17 GB", | 20260506 14:57:33.797 | INFO | "huge" : "", | 20260506 14:57:33.797 | INFO | "huge_shared" : " 27 GB", | 20260506 14:57:33.797 | INFO | "malloc" : " 14 GB", | 20260506 14:57:33.797 | INFO | "pinned" : " 29 GB" | 20260506 14:57:33.797 | INFO | } | 20260506 14:57:33.797 | INFO | Perf report for the last 5.13955 seconds: | 20260506 14:57:33.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:57:33.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:33.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:33.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:33.797 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 14:57:33.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:33.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:33.797 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260506 14:57:33.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:33.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:33.797 | INFO | Uploaded 36GiB to GPU in 2.03098s (17.7254GiB/s) | 20260506 14:57:33.797 | INFO | Downloaded 34GiB to GPU in 1.90515s (17.8464GiB/s) | 20260506 14:57:38.936 | INFO | Analyzing chunk frames = [2247680,2248192) | 20260506 14:57:38.936 | INFO | MemInfo: { | 20260506 14:57:38.936 | INFO | "gpu" : " 17 GB", | 20260506 14:57:38.936 | INFO | "huge" : "", | 20260506 14:57:38.936 | INFO | "huge_shared" : " 27 GB", | 20260506 14:57:38.936 | INFO | "malloc" : " 14 GB", | 20260506 14:57:38.936 | INFO | "pinned" : " 29 GB" | 20260506 14:57:38.936 | INFO | } | 20260506 14:57:38.936 | INFO | Perf report for the last 5.13954 seconds: | 20260506 14:57:38.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:57:38.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:38.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:38.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:38.936 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:57:38.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:38.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:38.936 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 14:57:38.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:38.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:38.937 | INFO | Uploaded 36GiB to GPU in 2.0067s (17.9399GiB/s) | 20260506 14:57:38.937 | INFO | Downloaded 34GiB to GPU in 1.98304s (17.1454GiB/s) | 20260506 14:57:44.075 | INFO | Analyzing chunk frames = [2248192,2248704) | 20260506 14:57:44.075 | INFO | MemInfo: { | 20260506 14:57:44.075 | INFO | "gpu" : " 17 GB", | 20260506 14:57:44.075 | INFO | "huge" : "", | 20260506 14:57:44.075 | INFO | "huge_shared" : " 27 GB", | 20260506 14:57:44.075 | INFO | "malloc" : " 14 GB", | 20260506 14:57:44.075 | INFO | "pinned" : " 29 GB" | 20260506 14:57:44.075 | INFO | } | 20260506 14:57:44.075 | INFO | Perf report for the last 5.13883 seconds: | 20260506 14:57:44.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:57:44.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:44.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:44.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:44.075 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:57:44.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:44.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:44.075 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:57:44.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:44.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:44.075 | INFO | Uploaded 36GiB to GPU in 2.00875s (17.9215GiB/s) | 20260506 14:57:44.075 | INFO | Downloaded 34GiB to GPU in 1.97319s (17.231GiB/s) | 20260506 14:57:44.186 | INFO | Starting a round of Baz IO | 20260506 14:57:47.344 | INFO | Populating RTMetrics report | 20260506 14:57:47.344 | INFO | RTMetrics report populated | 20260506 14:57:47.344 | INFO | Opening tmp RTMetrics json file | 20260506 14:57:47.398 | INFO | Serializing RTMetrics to json | 20260506 14:57:47.461 | INFO | Serialization done, swapping tmp file now... | 20260506 14:57:47.489 | INFO | RTMetrics done | 20260506 14:57:47.766 | INFO | Finished a round of Baz IO: Wrote 2.106 GiB in 3.579 seconds | 20260506 14:57:47.766 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.70% / 0.01% / 62.28% | 20260506 14:57:47.766 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.28388% / 3.97364e-06% / 0.000103315% | 20260506 14:57:47.766 | INFO | Baz IO overall: | 20260506 14:57:47.766 | INFO | BytesWritten : 1167049279224 | 20260506 14:57:47.766 | INFO | NumEvents : 564783383132 | 20260506 14:57:49.215 | INFO | Analyzing chunk frames = [2248704,2249216) | 20260506 14:57:49.215 | INFO | MemInfo: { | 20260506 14:57:49.215 | INFO | "gpu" : " 17 GB", | 20260506 14:57:49.215 | INFO | "huge" : "", | 20260506 14:57:49.215 | INFO | "huge_shared" : " 27 GB", | 20260506 14:57:49.215 | INFO | "malloc" : " 14 GB", | 20260506 14:57:49.215 | INFO | "pinned" : " 29 GB" | 20260506 14:57:49.215 | INFO | } | 20260506 14:57:49.215 | INFO | Perf report for the last 5.13959 seconds: | 20260506 14:57:49.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:57:49.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:49.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:49.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:49.215 | INFO | ANALYSIS : 63.0%, 2.9, 95.4ms, 0 | 20260506 14:57:49.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:49.215 | INFO | RT_METRICS : 46.6%, 1.0, 24.5ms, 0 | 20260506 14:57:49.215 | INFO | PRE_HQ : 60.9%, 1.9, 59.4ms, 0 | 20260506 14:57:49.215 | INFO | BAZWRITER : 69.0%, 4.8, 170.9ms, 0 | 20260506 14:57:49.215 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 14:57:49.215 | INFO | Uploaded 36GiB to GPU in 2.05759s (17.4962GiB/s) | 20260506 14:57:49.215 | INFO | Downloaded 38GiB to GPU in 2.22849s (17.0519GiB/s) | 20260506 14:57:54.354 | INFO | Analyzing chunk frames = [2249216,2249728) | 20260506 14:57:54.354 | INFO | MemInfo: { | 20260506 14:57:54.354 | INFO | "gpu" : " 17 GB", | 20260506 14:57:54.354 | INFO | "huge" : "", | 20260506 14:57:54.354 | INFO | "huge_shared" : " 27 GB", | 20260506 14:57:54.354 | INFO | "malloc" : " 14 GB", | 20260506 14:57:54.354 | INFO | "pinned" : " 29 GB" | 20260506 14:57:54.354 | INFO | } | 20260506 14:57:54.354 | INFO | Perf report for the last 5.1394 seconds: | 20260506 14:57:54.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:57:54.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:54.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:54.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:54.354 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 14:57:54.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:54.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:54.354 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 14:57:54.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:54.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:54.355 | INFO | Uploaded 36GiB to GPU in 2.00036s (17.9968GiB/s) | 20260506 14:57:54.355 | INFO | Downloaded 34GiB to GPU in 1.93569s (17.5648GiB/s) | 20260506 14:57:59.493 | INFO | Analyzing chunk frames = [2249728,2250240) | 20260506 14:57:59.493 | INFO | MemInfo: { | 20260506 14:57:59.493 | INFO | "gpu" : " 17 GB", | 20260506 14:57:59.493 | INFO | "huge" : "", | 20260506 14:57:59.493 | INFO | "huge_shared" : " 27 GB", | 20260506 14:57:59.493 | INFO | "malloc" : " 14 GB", | 20260506 14:57:59.493 | INFO | "pinned" : " 29 GB" | 20260506 14:57:59.493 | INFO | } | 20260506 14:57:59.493 | INFO | Perf report for the last 5.13888 seconds: | 20260506 14:57:59.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:57:59.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:59.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:59.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:59.493 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 14:57:59.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:59.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:59.493 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:57:59.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:59.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:57:59.493 | INFO | Uploaded 36GiB to GPU in 2.00004s (17.9997GiB/s) | 20260506 14:57:59.493 | INFO | Downloaded 34GiB to GPU in 1.94065s (17.5199GiB/s) | 20260506 14:58:04.632 | INFO | Analyzing chunk frames = [2250240,2250752) | 20260506 14:58:04.632 | INFO | MemInfo: { | 20260506 14:58:04.632 | INFO | "gpu" : " 17 GB", | 20260506 14:58:04.632 | INFO | "huge" : "", | 20260506 14:58:04.632 | INFO | "huge_shared" : " 27 GB", | 20260506 14:58:04.632 | INFO | "malloc" : " 14 GB", | 20260506 14:58:04.632 | INFO | "pinned" : " 29 GB" | 20260506 14:58:04.632 | INFO | } | 20260506 14:58:04.632 | INFO | Perf report for the last 5.13896 seconds: | 20260506 14:58:04.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:58:04.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:04.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:04.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:04.632 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 14:58:04.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:04.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:04.632 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:58:04.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:04.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:04.632 | INFO | Uploaded 36GiB to GPU in 1.99557s (18.04GiB/s) | 20260506 14:58:04.632 | INFO | Downloaded 34GiB to GPU in 1.91199s (17.7825GiB/s) | 20260506 14:58:09.772 | INFO | Analyzing chunk frames = [2250752,2251264) | 20260506 14:58:09.772 | INFO | MemInfo: { | 20260506 14:58:09.772 | INFO | "gpu" : " 17 GB", | 20260506 14:58:09.772 | INFO | "huge" : "", | 20260506 14:58:09.772 | INFO | "huge_shared" : " 27 GB", | 20260506 14:58:09.772 | INFO | "malloc" : " 14 GB", | 20260506 14:58:09.772 | INFO | "pinned" : " 29 GB" | 20260506 14:58:09.772 | INFO | } | 20260506 14:58:09.772 | INFO | Perf report for the last 5.13963 seconds: | 20260506 14:58:09.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:58:09.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:09.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:09.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:09.772 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 14:58:09.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:09.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:09.772 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 14:58:09.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:09.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:09.772 | INFO | Uploaded 36GiB to GPU in 1.9632s (18.3374GiB/s) | 20260506 14:58:09.772 | INFO | Downloaded 34GiB to GPU in 1.92737s (17.6406GiB/s) | 20260506 14:58:14.911 | INFO | Analyzing chunk frames = [2251264,2251776) | 20260506 14:58:14.911 | INFO | MemInfo: { | 20260506 14:58:14.911 | INFO | "gpu" : " 17 GB", | 20260506 14:58:14.911 | INFO | "huge" : "", | 20260506 14:58:14.911 | INFO | "huge_shared" : " 27 GB", | 20260506 14:58:14.911 | INFO | "malloc" : " 14 GB", | 20260506 14:58:14.911 | INFO | "pinned" : " 29 GB" | 20260506 14:58:14.911 | INFO | } | 20260506 14:58:14.911 | INFO | Perf report for the last 5.1397 seconds: | 20260506 14:58:14.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:58:14.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:14.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:14.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:14.911 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 14:58:14.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:14.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:14.912 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260506 14:58:14.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:14.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:14.912 | INFO | Uploaded 36GiB to GPU in 2.08347s (17.2789GiB/s) | 20260506 14:58:14.912 | INFO | Downloaded 34GiB to GPU in 1.96931s (17.2649GiB/s) | 20260506 14:58:20.052 | INFO | Analyzing chunk frames = [2251776,2252288) | 20260506 14:58:20.052 | INFO | MemInfo: { | 20260506 14:58:20.052 | INFO | "gpu" : " 17 GB", | 20260506 14:58:20.052 | INFO | "huge" : "", | 20260506 14:58:20.052 | INFO | "huge_shared" : " 27 GB", | 20260506 14:58:20.052 | INFO | "malloc" : " 14 GB", | 20260506 14:58:20.052 | INFO | "pinned" : " 29 GB" | 20260506 14:58:20.052 | INFO | } | 20260506 14:58:20.052 | INFO | Perf report for the last 5.14084 seconds: | 20260506 14:58:20.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:58:20.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:20.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:20.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:20.052 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 14:58:20.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:20.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:20.053 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 14:58:20.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:20.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:20.053 | INFO | Uploaded 36GiB to GPU in 2.02052s (17.8172GiB/s) | 20260506 14:58:20.053 | INFO | Downloaded 34GiB to GPU in 1.9986s (17.0119GiB/s) | 20260506 14:58:25.191 | INFO | Analyzing chunk frames = [2252288,2252800) | 20260506 14:58:25.191 | INFO | MemInfo: { | 20260506 14:58:25.191 | INFO | "gpu" : " 17 GB", | 20260506 14:58:25.191 | INFO | "huge" : "", | 20260506 14:58:25.191 | INFO | "huge_shared" : " 27 GB", | 20260506 14:58:25.191 | INFO | "malloc" : " 14 GB", | 20260506 14:58:25.191 | INFO | "pinned" : " 29 GB" | 20260506 14:58:25.191 | INFO | } | 20260506 14:58:25.191 | INFO | Perf report for the last 5.13894 seconds: | 20260506 14:58:25.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:58:25.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:25.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:25.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:25.191 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 14:58:25.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:25.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:25.191 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 14:58:25.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:25.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:25.192 | INFO | Uploaded 36GiB to GPU in 2.01032s (17.9076GiB/s) | 20260506 14:58:25.192 | INFO | Downloaded 34GiB to GPU in 1.98016s (17.1704GiB/s) | 20260506 14:58:25.192 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:58:25.192 | INFO | Total mem usage: 17.4 GiB | 20260506 14:58:25.192 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:58:25.192 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -12.2%) | 20260506 14:58:25.192 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:58:25.192 | INFO | High Water Breakdown: | 20260506 14:58:25.192 | INFO | 61.1% : 9.48 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:58:25.192 | INFO | 38.8% : 6.01 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:58:25.192 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:58:25.192 | INFO | ------------------------------------------ | 20260506 14:58:25.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:58:25.192 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:58:25.192 | INFO | Total mem usage: 26.5 GiB | 20260506 14:58:25.192 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:58:25.192 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:58:25.192 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:58:25.192 | INFO | High Water Breakdown: | 20260506 14:58:25.192 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:58:25.192 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:58:25.192 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:58:25.192 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:58:25.192 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:58:25.192 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:58:25.192 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:58:25.192 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:58:25.192 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:58:25.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:58:25.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:58:25.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:58:25.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:58:25.192 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:58:25.192 | INFO | ------------------------------------------ | 20260506 14:58:25.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:58:25.192 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:58:25.192 | INFO | Total mem usage: 25.6 GiB | 20260506 14:58:25.192 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:58:25.192 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:58:25.192 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:58:25.192 | INFO | High Water Breakdown: | 20260506 14:58:25.192 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:58:25.192 | INFO | ------------------------------------------ | 20260506 14:58:25.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:58:25.192 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:58:25.192 | INFO | Total mem usage: 15.7 GiB | 20260506 14:58:25.192 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:58:25.192 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:58:25.192 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 14:58:25.192 | INFO | High Water Breakdown: | 20260506 14:58:25.192 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:58:25.192 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:58:25.192 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:58:25.192 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:58:25.192 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:58:25.192 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:58:25.192 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:58:25.192 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:58:25.192 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:58:25.192 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:58:25.192 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:58:25.192 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:58:25.192 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:58:25.192 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:58:25.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:58:25.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:58:25.192 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:58:25.192 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:58:25.192 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:58:25.192 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:58:25.192 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:58:25.192 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:58:25.193 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:58:25.193 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:58:25.193 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:58:25.193 | INFO | ------------------------------------------ | 20260506 14:58:25.193 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:58:25.193 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:58:25.193 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:58:25.193 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:58:25.193 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.57e+01, 1.35e+01 / 2.04e+00 | 20260506 14:58:25.193 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.00e-01, 9.87e-03 / 7.96e-03 | 20260506 14:58:25.193 | INFO | PerfReport: 11.59%, Baseline -- 3.65e+00 / 4.42e+00, 5.70e-01 / 3.80e-01 | 20260506 14:58:25.193 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.89e-01 / 1.42e-01 | 20260506 14:58:25.193 | INFO | PerfReport: 15.86%, DME -- 3.99e+01 / 4.41e+01, 3.80e+00 / 2.24e+00 | 20260506 14:58:25.193 | INFO | PerfReport: 56.72%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.43e+00 / 2.08e+00 | 20260506 14:58:25.193 | INFO | PerfReport: 7.44%, PulseAccumulating -- 2.34e+00 / 2.92e+00, 4.07e-01 / 2.35e-01 | 20260506 14:58:25.193 | INFO | PerfReport: 3.59%, Metrics -- 1.13e+00 / 1.50e+00, 2.91e-01 / 1.93e-01 | 20260506 14:58:25.193 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.93e-01 / 4.44e-01, 6.19e-02 / 2.40e-02 | 20260506 14:58:25.193 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:58:25.193 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:58:25.193 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:58:25.193 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 14:58:25.193 | INFO | PerfReport: 21.47%, Upload -- 2.10e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260506 14:58:25.193 | INFO | PerfReport: 21.15%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 14:58:25.193 | INFO | PerfReport: 32.23%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 14:58:25.193 | INFO | PerfReport: 1.54%, LockUpload -- 1.50e+00 / 0.00e+00, 5.95e+00 / 0.00e+00 | 20260506 14:58:25.193 | INFO | PerfReport: 1.14%, LockDownload -- 1.12e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260506 14:58:25.193 | INFO | PerfReport: 22.45%, LockCompute -- 2.19e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 14:58:25.193 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260506 14:58:25.304 | INFO | Starting a round of Baz IO | 20260506 14:58:28.459 | INFO | Populating RTMetrics report | 20260506 14:58:28.459 | INFO | RTMetrics report populated | 20260506 14:58:28.459 | INFO | Opening tmp RTMetrics json file | 20260506 14:58:28.470 | INFO | Serializing RTMetrics to json | 20260506 14:58:28.471 | INFO | Serialization done, swapping tmp file now... | 20260506 14:58:28.471 | INFO | RTMetrics done | 20260506 14:58:28.758 | INFO | Finished a round of Baz IO: Wrote 2.108 GiB in 3.454 seconds | 20260506 14:58:28.758 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.69% / 0.01% / 62.30% | 20260506 14:58:28.758 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.28072% / 2.38419e-05% / 5.96046e-05% | 20260506 14:58:28.758 | INFO | Baz IO overall: | 20260506 14:58:28.758 | INFO | BytesWritten : 1169312755959 | 20260506 14:58:28.758 | INFO | NumEvents : 565879946990 | 20260506 14:58:30.329 | INFO | Analyzing chunk frames = [2252800,2253312) | 20260506 14:58:30.329 | INFO | MemInfo: { | 20260506 14:58:30.329 | INFO | "gpu" : " 17 GB", | 20260506 14:58:30.329 | INFO | "huge" : "", | 20260506 14:58:30.329 | INFO | "huge_shared" : " 27 GB", | 20260506 14:58:30.329 | INFO | "malloc" : " 14 GB", | 20260506 14:58:30.329 | INFO | "pinned" : " 29 GB" | 20260506 14:58:30.329 | INFO | } | 20260506 14:58:30.329 | INFO | Perf report for the last 5.13817 seconds: | 20260506 14:58:30.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:58:30.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:30.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:30.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:30.330 | INFO | ANALYSIS : 63.0%, 2.9, 95.2ms, 0 | 20260506 14:58:30.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:30.330 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 14:58:30.330 | INFO | PRE_HQ : 60.9%, 1.9, 59.3ms, 0 | 20260506 14:58:30.330 | INFO | BAZWRITER : 66.6%, 4.8, 164.1ms, 0 | 20260506 14:58:30.330 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 14:58:30.330 | INFO | Uploaded 36GiB to GPU in 2.04451s (17.6081GiB/s) | 20260506 14:58:30.330 | INFO | Downloaded 38GiB to GPU in 2.22823s (17.0539GiB/s) | 20260506 14:58:35.468 | INFO | Analyzing chunk frames = [2253312,2253824) | 20260506 14:58:35.468 | INFO | MemInfo: { | 20260506 14:58:35.468 | INFO | "gpu" : " 17 GB", | 20260506 14:58:35.468 | INFO | "huge" : "", | 20260506 14:58:35.468 | INFO | "huge_shared" : " 27 GB", | 20260506 14:58:35.468 | INFO | "malloc" : " 14 GB", | 20260506 14:58:35.468 | INFO | "pinned" : " 29 GB" | 20260506 14:58:35.468 | INFO | } | 20260506 14:58:35.468 | INFO | Perf report for the last 5.13895 seconds: | 20260506 14:58:35.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:58:35.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:35.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:35.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:35.468 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 14:58:35.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:35.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:35.468 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 14:58:35.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:35.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:35.469 | INFO | Uploaded 36GiB to GPU in 1.97655s (18.2136GiB/s) | 20260506 14:58:35.469 | INFO | Downloaded 34GiB to GPU in 1.93508s (17.5703GiB/s) | 20260506 14:58:40.607 | INFO | Analyzing chunk frames = [2253824,2254336) | 20260506 14:58:40.607 | INFO | MemInfo: { | 20260506 14:58:40.607 | INFO | "gpu" : " 17 GB", | 20260506 14:58:40.607 | INFO | "huge" : "", | 20260506 14:58:40.607 | INFO | "huge_shared" : " 27 GB", | 20260506 14:58:40.607 | INFO | "malloc" : " 14 GB", | 20260506 14:58:40.607 | INFO | "pinned" : " 29 GB" | 20260506 14:58:40.607 | INFO | } | 20260506 14:58:40.607 | INFO | Perf report for the last 5.13907 seconds: | 20260506 14:58:40.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:58:40.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:40.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:40.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:40.608 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 14:58:40.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:40.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:40.608 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 14:58:40.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:40.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:40.608 | INFO | Uploaded 36GiB to GPU in 1.97884s (18.1925GiB/s) | 20260506 14:58:40.608 | INFO | Downloaded 34GiB to GPU in 1.93929s (17.5322GiB/s) | 20260506 14:58:45.747 | INFO | Analyzing chunk frames = [2254336,2254848) | 20260506 14:58:45.747 | INFO | MemInfo: { | 20260506 14:58:45.747 | INFO | "gpu" : " 17 GB", | 20260506 14:58:45.747 | INFO | "huge" : "", | 20260506 14:58:45.747 | INFO | "huge_shared" : " 27 GB", | 20260506 14:58:45.747 | INFO | "malloc" : " 14 GB", | 20260506 14:58:45.747 | INFO | "pinned" : " 29 GB" | 20260506 14:58:45.747 | INFO | } | 20260506 14:58:45.747 | INFO | Perf report for the last 5.13969 seconds: | 20260506 14:58:45.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:58:45.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:45.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:45.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:45.747 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260506 14:58:45.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:45.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:45.747 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:58:45.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:45.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:45.747 | INFO | Uploaded 36GiB to GPU in 2.01868s (17.8335GiB/s) | 20260506 14:58:45.747 | INFO | Downloaded 34GiB to GPU in 1.92067s (17.7022GiB/s) | 20260506 14:58:50.886 | INFO | Analyzing chunk frames = [2254848,2255360) | 20260506 14:58:50.886 | INFO | MemInfo: { | 20260506 14:58:50.886 | INFO | "gpu" : " 17 GB", | 20260506 14:58:50.886 | INFO | "huge" : "", | 20260506 14:58:50.886 | INFO | "huge_shared" : " 27 GB", | 20260506 14:58:50.886 | INFO | "malloc" : " 14 GB", | 20260506 14:58:50.887 | INFO | "pinned" : " 29 GB" | 20260506 14:58:50.887 | INFO | } | 20260506 14:58:50.887 | INFO | Perf report for the last 5.1396 seconds: | 20260506 14:58:50.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:58:50.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:50.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:50.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:50.887 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 14:58:50.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:50.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:50.887 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:58:50.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:50.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:50.887 | INFO | Uploaded 36GiB to GPU in 1.99155s (18.0763GiB/s) | 20260506 14:58:50.887 | INFO | Downloaded 34GiB to GPU in 1.94034s (17.5227GiB/s) | 20260506 14:58:56.026 | INFO | Analyzing chunk frames = [2255360,2255872) | 20260506 14:58:56.026 | INFO | MemInfo: { | 20260506 14:58:56.026 | INFO | "gpu" : " 17 GB", | 20260506 14:58:56.026 | INFO | "huge" : "", | 20260506 14:58:56.026 | INFO | "huge_shared" : " 27 GB", | 20260506 14:58:56.026 | INFO | "malloc" : " 14 GB", | 20260506 14:58:56.026 | INFO | "pinned" : " 29 GB" | 20260506 14:58:56.026 | INFO | } | 20260506 14:58:56.026 | INFO | Perf report for the last 5.1391 seconds: | 20260506 14:58:56.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:58:56.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:56.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:56.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:56.026 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 14:58:56.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:56.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:56.026 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:58:56.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:56.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:58:56.026 | INFO | Uploaded 36GiB to GPU in 2.04951s (17.5652GiB/s) | 20260506 14:58:56.026 | INFO | Downloaded 34GiB to GPU in 1.95993s (17.3476GiB/s) | 20260506 14:59:01.166 | INFO | Analyzing chunk frames = [2255872,2256384) | 20260506 14:59:01.166 | INFO | MemInfo: { | 20260506 14:59:01.166 | INFO | "gpu" : " 17 GB", | 20260506 14:59:01.166 | INFO | "huge" : "", | 20260506 14:59:01.166 | INFO | "huge_shared" : " 27 GB", | 20260506 14:59:01.166 | INFO | "malloc" : " 14 GB", | 20260506 14:59:01.166 | INFO | "pinned" : " 29 GB" | 20260506 14:59:01.166 | INFO | } | 20260506 14:59:01.166 | INFO | Perf report for the last 5.13993 seconds: | 20260506 14:59:01.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:59:01.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:01.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:01.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:01.166 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 14:59:01.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:01.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:01.166 | INFO | PRE_HQ : 29.5%, 1.0, 15.4ms, 0 | 20260506 14:59:01.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:01.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:01.166 | INFO | Uploaded 36GiB to GPU in 1.98729s (18.1152GiB/s) | 20260506 14:59:01.166 | INFO | Downloaded 34GiB to GPU in 1.98299s (17.1458GiB/s) | 20260506 14:59:06.304 | INFO | Analyzing chunk frames = [2256384,2256896) | 20260506 14:59:06.304 | INFO | MemInfo: { | 20260506 14:59:06.304 | INFO | "gpu" : " 17 GB", | 20260506 14:59:06.304 | INFO | "huge" : "", | 20260506 14:59:06.304 | INFO | "huge_shared" : " 27 GB", | 20260506 14:59:06.304 | INFO | "malloc" : " 14 GB", | 20260506 14:59:06.304 | INFO | "pinned" : " 29 GB" | 20260506 14:59:06.304 | INFO | } | 20260506 14:59:06.304 | INFO | Perf report for the last 5.13801 seconds: | 20260506 14:59:06.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:59:06.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:06.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:06.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:06.304 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 14:59:06.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:06.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:06.304 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 14:59:06.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:06.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:06.304 | INFO | Uploaded 36GiB to GPU in 1.98009s (18.181GiB/s) | 20260506 14:59:06.304 | INFO | Downloaded 34GiB to GPU in 1.97792s (17.1898GiB/s) | 20260506 14:59:06.425 | INFO | Starting a round of Baz IO | 20260506 14:59:09.572 | INFO | Populating RTMetrics report | 20260506 14:59:09.572 | INFO | RTMetrics report populated | 20260506 14:59:09.572 | INFO | Opening tmp RTMetrics json file | 20260506 14:59:09.625 | INFO | Serializing RTMetrics to json | 20260506 14:59:09.628 | INFO | Serialization done, swapping tmp file now... | 20260506 14:59:09.632 | INFO | RTMetrics done | 20260506 14:59:09.886 | INFO | Finished a round of Baz IO: Wrote 2.106 GiB in 3.46 seconds | 20260506 14:59:09.886 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.68% / 0.01% / 62.31% | 20260506 14:59:09.886 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.2752% / 1.19209e-05% / 7.54992e-05% | 20260506 14:59:09.886 | INFO | Baz IO overall: | 20260506 14:59:09.886 | INFO | BytesWritten : 1171574058814 | 20260506 14:59:09.886 | INFO | NumEvents : 566974817424 | 20260506 14:59:11.444 | INFO | Analyzing chunk frames = [2256896,2257408) | 20260506 14:59:11.444 | INFO | MemInfo: { | 20260506 14:59:11.444 | INFO | "gpu" : " 17 GB", | 20260506 14:59:11.444 | INFO | "huge" : "", | 20260506 14:59:11.444 | INFO | "huge_shared" : " 27 GB", | 20260506 14:59:11.444 | INFO | "malloc" : " 14 GB", | 20260506 14:59:11.444 | INFO | "pinned" : " 29 GB" | 20260506 14:59:11.444 | INFO | } | 20260506 14:59:11.444 | INFO | Perf report for the last 5.14056 seconds: | 20260506 14:59:11.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:59:11.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:11.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:11.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:11.444 | INFO | ANALYSIS : 63.0%, 2.9, 95.3ms, 0 | 20260506 14:59:11.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:11.444 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260506 14:59:11.444 | INFO | PRE_HQ : 60.8%, 1.9, 59.5ms, 0 | 20260506 14:59:11.444 | INFO | BAZWRITER : 66.9%, 4.8, 165.2ms, 0 | 20260506 14:59:11.445 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 14:59:11.445 | INFO | Uploaded 36GiB to GPU in 2.05977s (17.4777GiB/s) | 20260506 14:59:11.445 | INFO | Downloaded 38GiB to GPU in 2.22288s (17.0949GiB/s) | 20260506 14:59:16.583 | INFO | Analyzing chunk frames = [2257408,2257920) | 20260506 14:59:16.583 | INFO | MemInfo: { | 20260506 14:59:16.583 | INFO | "gpu" : " 17 GB", | 20260506 14:59:16.583 | INFO | "huge" : "", | 20260506 14:59:16.583 | INFO | "huge_shared" : " 27 GB", | 20260506 14:59:16.583 | INFO | "malloc" : " 14 GB", | 20260506 14:59:16.583 | INFO | "pinned" : " 29 GB" | 20260506 14:59:16.583 | INFO | } | 20260506 14:59:16.583 | INFO | Perf report for the last 5.13907 seconds: | 20260506 14:59:16.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:59:16.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:16.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:16.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:16.584 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 14:59:16.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:16.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:16.584 | INFO | PRE_HQ : 27.9%, 1.0, 14.6ms, 0 | 20260506 14:59:16.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:16.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:16.584 | INFO | Uploaded 36GiB to GPU in 1.9957s (18.0388GiB/s) | 20260506 14:59:16.584 | INFO | Downloaded 34GiB to GPU in 1.93643s (17.5581GiB/s) | 20260506 14:59:21.722 | INFO | Analyzing chunk frames = [2257920,2258432) | 20260506 14:59:21.722 | INFO | MemInfo: { | 20260506 14:59:21.722 | INFO | "gpu" : " 17 GB", | 20260506 14:59:21.722 | INFO | "huge" : "", | 20260506 14:59:21.722 | INFO | "huge_shared" : " 27 GB", | 20260506 14:59:21.722 | INFO | "malloc" : " 14 GB", | 20260506 14:59:21.722 | INFO | "pinned" : " 29 GB" | 20260506 14:59:21.722 | INFO | } | 20260506 14:59:21.722 | INFO | Perf report for the last 5.13882 seconds: | 20260506 14:59:21.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:59:21.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:21.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:21.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:21.722 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 14:59:21.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:21.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:21.722 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 14:59:21.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:21.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:21.723 | INFO | Uploaded 36GiB to GPU in 2.00598s (17.9463GiB/s) | 20260506 14:59:21.723 | INFO | Downloaded 34GiB to GPU in 1.93938s (17.5314GiB/s) | 20260506 14:59:26.862 | INFO | Analyzing chunk frames = [2258432,2258944) | 20260506 14:59:26.862 | INFO | MemInfo: { | 20260506 14:59:26.862 | INFO | "gpu" : " 17 GB", | 20260506 14:59:26.862 | INFO | "huge" : "", | 20260506 14:59:26.862 | INFO | "huge_shared" : " 27 GB", | 20260506 14:59:26.862 | INFO | "malloc" : " 14 GB", | 20260506 14:59:26.862 | INFO | "pinned" : " 29 GB" | 20260506 14:59:26.862 | INFO | } | 20260506 14:59:26.862 | INFO | Perf report for the last 5.13956 seconds: | 20260506 14:59:26.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:59:26.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:26.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:26.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:26.862 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 14:59:26.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:26.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:26.862 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 14:59:26.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:26.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:26.862 | INFO | Uploaded 36GiB to GPU in 2.02682s (17.7618GiB/s) | 20260506 14:59:26.862 | INFO | Downloaded 34GiB to GPU in 1.91461s (17.7582GiB/s) | 20260506 14:59:32.001 | INFO | Analyzing chunk frames = [2258944,2259456) | 20260506 14:59:32.001 | INFO | MemInfo: { | 20260506 14:59:32.001 | INFO | "gpu" : " 17 GB", | 20260506 14:59:32.001 | INFO | "huge" : "", | 20260506 14:59:32.001 | INFO | "huge_shared" : " 27 GB", | 20260506 14:59:32.001 | INFO | "malloc" : " 14 GB", | 20260506 14:59:32.001 | INFO | "pinned" : " 29 GB" | 20260506 14:59:32.001 | INFO | } | 20260506 14:59:32.001 | INFO | Perf report for the last 5.13949 seconds: | 20260506 14:59:32.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:59:32.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:32.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:32.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:32.001 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 14:59:32.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:32.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:32.001 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 14:59:32.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:32.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:32.002 | INFO | Uploaded 36GiB to GPU in 2.00531s (17.9524GiB/s) | 20260506 14:59:32.002 | INFO | Downloaded 34GiB to GPU in 1.92962s (17.62GiB/s) | 20260506 14:59:37.141 | INFO | Analyzing chunk frames = [2259456,2259968) | 20260506 14:59:37.141 | INFO | MemInfo: { | 20260506 14:59:37.141 | INFO | "gpu" : " 17 GB", | 20260506 14:59:37.141 | INFO | "huge" : "", | 20260506 14:59:37.141 | INFO | "huge_shared" : " 27 GB", | 20260506 14:59:37.141 | INFO | "malloc" : " 14 GB", | 20260506 14:59:37.141 | INFO | "pinned" : " 29 GB" | 20260506 14:59:37.141 | INFO | } | 20260506 14:59:37.141 | INFO | Perf report for the last 5.13981 seconds: | 20260506 14:59:37.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:59:37.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:37.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:37.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:37.141 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:59:37.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:37.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:37.141 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 14:59:37.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:37.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:37.141 | INFO | Uploaded 36GiB to GPU in 2.08236s (17.2881GiB/s) | 20260506 14:59:37.142 | INFO | Downloaded 34GiB to GPU in 1.97513s (17.214GiB/s) | 20260506 14:59:42.281 | INFO | Analyzing chunk frames = [2259968,2260480) | 20260506 14:59:42.281 | INFO | MemInfo: { | 20260506 14:59:42.281 | INFO | "gpu" : " 17 GB", | 20260506 14:59:42.281 | INFO | "huge" : "", | 20260506 14:59:42.281 | INFO | "huge_shared" : " 27 GB", | 20260506 14:59:42.281 | INFO | "malloc" : " 14 GB", | 20260506 14:59:42.281 | INFO | "pinned" : " 29 GB" | 20260506 14:59:42.281 | INFO | } | 20260506 14:59:42.281 | INFO | Perf report for the last 5.14022 seconds: | 20260506 14:59:42.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:59:42.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:42.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:42.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:42.282 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:59:42.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:42.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:42.282 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 14:59:42.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:42.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:42.282 | INFO | Uploaded 36GiB to GPU in 2.04433s (17.6097GiB/s) | 20260506 14:59:42.282 | INFO | Downloaded 34GiB to GPU in 1.99361s (17.0545GiB/s) | 20260506 14:59:47.419 | INFO | Analyzing chunk frames = [2260480,2260992) | 20260506 14:59:47.419 | INFO | MemInfo: { | 20260506 14:59:47.419 | INFO | "gpu" : " 17 GB", | 20260506 14:59:47.419 | INFO | "huge" : "", | 20260506 14:59:47.419 | INFO | "huge_shared" : " 27 GB", | 20260506 14:59:47.419 | INFO | "malloc" : " 14 GB", | 20260506 14:59:47.419 | INFO | "pinned" : " 29 GB" | 20260506 14:59:47.419 | INFO | } | 20260506 14:59:47.419 | INFO | Perf report for the last 5.13763 seconds: | 20260506 14:59:47.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:59:47.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:47.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:47.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:47.419 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 14:59:47.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:47.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:47.419 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 14:59:47.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:47.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:47.419 | INFO | Uploaded 36GiB to GPU in 2.01398s (17.8751GiB/s) | 20260506 14:59:47.419 | INFO | Downloaded 34GiB to GPU in 1.98398s (17.1373GiB/s) | 20260506 14:59:47.419 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 14:59:47.419 | INFO | Total mem usage: 17.4 GiB | 20260506 14:59:47.419 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 14:59:47.419 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -11.4%) | 20260506 14:59:47.419 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 14:59:47.420 | INFO | High Water Breakdown: | 20260506 14:59:47.420 | INFO | 61.2% : 9.57 GiB ( 87.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 14:59:47.420 | INFO | 38.6% : 6.04 GiB ( 90.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 14:59:47.420 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 14:59:47.420 | INFO | ------------------------------------------ | 20260506 14:59:47.420 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:59:47.420 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 14:59:47.420 | INFO | Total mem usage: 26.5 GiB | 20260506 14:59:47.420 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 14:59:47.420 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 14:59:47.420 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 14:59:47.420 | INFO | High Water Breakdown: | 20260506 14:59:47.420 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:59:47.420 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:59:47.420 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:59:47.420 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:59:47.420 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:59:47.420 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:59:47.420 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 14:59:47.420 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 14:59:47.420 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:59:47.420 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 14:59:47.420 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 14:59:47.420 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 14:59:47.420 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 14:59:47.420 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 14:59:47.420 | INFO | ------------------------------------------ | 20260506 14:59:47.420 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:59:47.420 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 14:59:47.420 | INFO | Total mem usage: 25.6 GiB | 20260506 14:59:47.420 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 14:59:47.420 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 14:59:47.420 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 14:59:47.420 | INFO | High Water Breakdown: | 20260506 14:59:47.420 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 14:59:47.420 | INFO | ------------------------------------------ | 20260506 14:59:47.420 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:59:47.420 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 14:59:47.420 | INFO | Total mem usage: 15.7 GiB | 20260506 14:59:47.420 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 14:59:47.420 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 14:59:47.420 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 14:59:47.420 | INFO | High Water Breakdown: | 20260506 14:59:47.420 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 14:59:47.420 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 14:59:47.420 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 14:59:47.420 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 14:59:47.420 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 14:59:47.420 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 14:59:47.420 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 14:59:47.420 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 14:59:47.420 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 14:59:47.420 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 14:59:47.420 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 14:59:47.420 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 14:59:47.420 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 14:59:47.420 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 14:59:47.420 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 14:59:47.420 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 14:59:47.420 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 14:59:47.420 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 14:59:47.420 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 14:59:47.420 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 14:59:47.420 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 14:59:47.420 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 14:59:47.420 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 14:59:47.420 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 14:59:47.420 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 14:59:47.420 | INFO | ------------------------------------------ | 20260506 14:59:47.420 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 14:59:47.420 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 14:59:47.420 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:59:47.420 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:59:47.420 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.60e+01, 1.36e+01 / 2.51e+00 | 20260506 14:59:47.420 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.01e-01, 1.01e-02 / 9.02e-03 | 20260506 14:59:47.420 | INFO | PerfReport: 11.58%, Baseline -- 3.65e+00 / 4.41e+00, 5.67e-01 / 4.08e-01 | 20260506 14:59:47.420 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.48e+00, 1.89e-01 / 1.52e-01 | 20260506 14:59:47.420 | INFO | PerfReport: 15.92%, DME -- 4.01e+01 / 4.47e+01, 4.12e+00 / 2.75e+00 | 20260506 14:59:47.420 | INFO | PerfReport: 56.73%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.45e+00 / 2.06e+00 | 20260506 14:59:47.420 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.33e+00 / 2.93e+00, 4.04e-01 / 2.46e-01 | 20260506 14:59:47.420 | INFO | PerfReport: 3.58%, Metrics -- 1.13e+00 / 1.47e+00, 2.94e-01 / 2.02e-01 | 20260506 14:59:47.420 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.93e-01 / 4.44e-01, 6.20e-02 / 3.38e-02 | 20260506 14:59:47.420 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 14:59:47.420 | INFO | PerfReport: Gathered 1536 samples. | 20260506 14:59:47.420 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 14:59:47.420 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 14:59:47.421 | INFO | PerfReport: 21.51%, Upload -- 2.10e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 14:59:47.421 | INFO | PerfReport: 21.19%, Download -- 2.07e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 14:59:47.421 | INFO | PerfReport: 32.26%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 14:59:47.421 | INFO | PerfReport: 1.53%, LockUpload -- 1.50e+00 / 0.00e+00, 5.86e+00 / 0.00e+00 | 20260506 14:59:47.421 | INFO | PerfReport: 1.11%, LockDownload -- 1.09e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260506 14:59:47.421 | INFO | PerfReport: 22.39%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 14:59:47.421 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260506 14:59:47.528 | INFO | Starting a round of Baz IO | 20260506 14:59:50.682 | INFO | Populating RTMetrics report | 20260506 14:59:50.682 | INFO | RTMetrics report populated | 20260506 14:59:50.682 | INFO | Opening tmp RTMetrics json file | 20260506 14:59:50.684 | INFO | Serializing RTMetrics to json | 20260506 14:59:50.687 | INFO | Serialization done, swapping tmp file now... | 20260506 14:59:50.688 | INFO | RTMetrics done | 20260506 14:59:50.933 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.404 seconds | 20260506 14:59:50.933 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.67% / 0.01% / 62.32% | 20260506 14:59:50.933 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.27624% / 2.78155e-05% / 7.54992e-05% | 20260506 14:59:50.933 | INFO | Baz IO overall: | 20260506 14:59:50.933 | INFO | BytesWritten : 1173838578851 | 20260506 14:59:50.933 | INFO | NumEvents : 568072090895 | 20260506 14:59:52.558 | INFO | Analyzing chunk frames = [2260992,2261504) | 20260506 14:59:52.558 | INFO | MemInfo: { | 20260506 14:59:52.558 | INFO | "gpu" : " 17 GB", | 20260506 14:59:52.558 | INFO | "huge" : "", | 20260506 14:59:52.558 | INFO | "huge_shared" : " 27 GB", | 20260506 14:59:52.558 | INFO | "malloc" : " 14 GB", | 20260506 14:59:52.558 | INFO | "pinned" : " 29 GB" | 20260506 14:59:52.558 | INFO | } | 20260506 14:59:52.558 | INFO | Perf report for the last 5.13913 seconds: | 20260506 14:59:52.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:59:52.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:52.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:52.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:52.558 | INFO | ANALYSIS : 62.9%, 2.9, 95.3ms, 0 | 20260506 14:59:52.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:52.558 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 14:59:52.558 | INFO | PRE_HQ : 60.9%, 1.9, 59.2ms, 0 | 20260506 14:59:52.558 | INFO | BAZWRITER : 65.6%, 4.8, 161.0ms, 0 | 20260506 14:59:52.558 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 14:59:52.559 | INFO | Uploaded 36GiB to GPU in 2.04107s (17.6378GiB/s) | 20260506 14:59:52.559 | INFO | Downloaded 38GiB to GPU in 2.23077s (17.0345GiB/s) | 20260506 14:59:57.698 | INFO | Analyzing chunk frames = [2261504,2262016) | 20260506 14:59:57.698 | INFO | MemInfo: { | 20260506 14:59:57.698 | INFO | "gpu" : " 17 GB", | 20260506 14:59:57.698 | INFO | "huge" : "", | 20260506 14:59:57.698 | INFO | "huge_shared" : " 27 GB", | 20260506 14:59:57.698 | INFO | "malloc" : " 14 GB", | 20260506 14:59:57.698 | INFO | "pinned" : " 29 GB" | 20260506 14:59:57.698 | INFO | } | 20260506 14:59:57.698 | INFO | Perf report for the last 5.13976 seconds: | 20260506 14:59:57.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 14:59:57.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:57.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:57.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:57.698 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 14:59:57.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:57.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:57.698 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 14:59:57.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:57.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 14:59:57.698 | INFO | Uploaded 36GiB to GPU in 1.98269s (18.1572GiB/s) | 20260506 14:59:57.698 | INFO | Downloaded 34GiB to GPU in 1.93106s (17.6069GiB/s) | 20260506 15:00:02.838 | INFO | Analyzing chunk frames = [2262016,2262528) | 20260506 15:00:02.838 | INFO | MemInfo: { | 20260506 15:00:02.838 | INFO | "gpu" : " 17 GB", | 20260506 15:00:02.838 | INFO | "huge" : "", | 20260506 15:00:02.838 | INFO | "huge_shared" : " 27 GB", | 20260506 15:00:02.838 | INFO | "malloc" : " 14 GB", | 20260506 15:00:02.838 | INFO | "pinned" : " 29 GB" | 20260506 15:00:02.838 | INFO | } | 20260506 15:00:02.838 | INFO | Perf report for the last 5.13998 seconds: | 20260506 15:00:02.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:00:02.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:02.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:02.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:02.838 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 15:00:02.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:02.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:02.838 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:00:02.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:02.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:02.838 | INFO | Uploaded 36GiB to GPU in 1.94455s (18.5133GiB/s) | 20260506 15:00:02.838 | INFO | Downloaded 34GiB to GPU in 1.93457s (17.5749GiB/s) | 20260506 15:00:07.976 | INFO | Analyzing chunk frames = [2262528,2263040) | 20260506 15:00:07.976 | INFO | MemInfo: { | 20260506 15:00:07.976 | INFO | "gpu" : " 17 GB", | 20260506 15:00:07.976 | INFO | "huge" : "", | 20260506 15:00:07.976 | INFO | "huge_shared" : " 27 GB", | 20260506 15:00:07.976 | INFO | "malloc" : " 14 GB", | 20260506 15:00:07.977 | INFO | "pinned" : " 29 GB" | 20260506 15:00:07.977 | INFO | } | 20260506 15:00:07.977 | INFO | Perf report for the last 5.13858 seconds: | 20260506 15:00:07.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:00:07.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:07.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:07.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:07.977 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:00:07.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:07.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:07.977 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:00:07.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:07.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:07.977 | INFO | Uploaded 36GiB to GPU in 2.01729s (17.8457GiB/s) | 20260506 15:00:07.977 | INFO | Downloaded 34GiB to GPU in 1.91545s (17.7504GiB/s) | 20260506 15:00:13.116 | INFO | Analyzing chunk frames = [2263040,2263552) | 20260506 15:00:13.116 | INFO | MemInfo: { | 20260506 15:00:13.116 | INFO | "gpu" : " 17 GB", | 20260506 15:00:13.116 | INFO | "huge" : "", | 20260506 15:00:13.116 | INFO | "huge_shared" : " 27 GB", | 20260506 15:00:13.116 | INFO | "malloc" : " 14 GB", | 20260506 15:00:13.116 | INFO | "pinned" : " 29 GB" | 20260506 15:00:13.116 | INFO | } | 20260506 15:00:13.116 | INFO | Perf report for the last 5.13905 seconds: | 20260506 15:00:13.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:00:13.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:13.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:13.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:13.116 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 15:00:13.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:13.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:13.116 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:00:13.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:13.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:13.116 | INFO | Uploaded 36GiB to GPU in 1.97522s (18.2258GiB/s) | 20260506 15:00:13.116 | INFO | Downloaded 34GiB to GPU in 1.9344s (17.5765GiB/s) | 20260506 15:00:18.255 | INFO | Analyzing chunk frames = [2263552,2264064) | 20260506 15:00:18.255 | INFO | MemInfo: { | 20260506 15:00:18.255 | INFO | "gpu" : " 17 GB", | 20260506 15:00:18.255 | INFO | "huge" : "", | 20260506 15:00:18.255 | INFO | "huge_shared" : " 27 GB", | 20260506 15:00:18.255 | INFO | "malloc" : " 14 GB", | 20260506 15:00:18.255 | INFO | "pinned" : " 29 GB" | 20260506 15:00:18.255 | INFO | } | 20260506 15:00:18.255 | INFO | Perf report for the last 5.13941 seconds: | 20260506 15:00:18.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:00:18.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:18.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:18.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:18.255 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 15:00:18.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:18.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:18.255 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 15:00:18.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:18.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:18.256 | INFO | Uploaded 36GiB to GPU in 2.04272s (17.6236GiB/s) | 20260506 15:00:18.256 | INFO | Downloaded 34GiB to GPU in 1.92647s (17.6489GiB/s) | 20260506 15:00:23.395 | INFO | Analyzing chunk frames = [2264064,2264576) | 20260506 15:00:23.395 | INFO | MemInfo: { | 20260506 15:00:23.395 | INFO | "gpu" : " 17 GB", | 20260506 15:00:23.395 | INFO | "huge" : "", | 20260506 15:00:23.395 | INFO | "huge_shared" : " 27 GB", | 20260506 15:00:23.395 | INFO | "malloc" : " 14 GB", | 20260506 15:00:23.395 | INFO | "pinned" : " 29 GB" | 20260506 15:00:23.395 | INFO | } | 20260506 15:00:23.396 | INFO | Perf report for the last 5.14047 seconds: | 20260506 15:00:23.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:00:23.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:23.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:23.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:23.396 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:00:23.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:23.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:23.396 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 15:00:23.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:23.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:23.396 | INFO | Uploaded 36GiB to GPU in 2.00958s (17.9142GiB/s) | 20260506 15:00:23.396 | INFO | Downloaded 34GiB to GPU in 2.00822s (16.9304GiB/s) | 20260506 15:00:28.535 | INFO | Analyzing chunk frames = [2264576,2265088) | 20260506 15:00:28.535 | INFO | MemInfo: { | 20260506 15:00:28.535 | INFO | "gpu" : " 17 GB", | 20260506 15:00:28.535 | INFO | "huge" : "", | 20260506 15:00:28.535 | INFO | "huge_shared" : " 27 GB", | 20260506 15:00:28.535 | INFO | "malloc" : " 14 GB", | 20260506 15:00:28.535 | INFO | "pinned" : " 29 GB" | 20260506 15:00:28.535 | INFO | } | 20260506 15:00:28.535 | INFO | Perf report for the last 5.13924 seconds: | 20260506 15:00:28.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:00:28.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:28.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:28.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:28.535 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:00:28.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:28.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:28.535 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:00:28.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:28.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:28.535 | INFO | Uploaded 36GiB to GPU in 2.01657s (17.8521GiB/s) | 20260506 15:00:28.535 | INFO | Downloaded 34GiB to GPU in 1.98529s (17.1259GiB/s) | 20260506 15:00:28.655 | INFO | Starting a round of Baz IO | 20260506 15:00:31.801 | INFO | Populating RTMetrics report | 20260506 15:00:31.802 | INFO | RTMetrics report populated | 20260506 15:00:31.802 | INFO | Opening tmp RTMetrics json file | 20260506 15:00:31.810 | INFO | Serializing RTMetrics to json | 20260506 15:00:31.837 | INFO | Serialization done, swapping tmp file now... | 20260506 15:00:31.839 | INFO | RTMetrics done | 20260506 15:00:32.093 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.438 seconds | 20260506 15:00:32.093 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.66% / 0.01% / 62.33% | 20260506 15:00:32.093 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.27273% / 1.98682e-05% / 5.96046e-05% | 20260506 15:00:32.093 | INFO | Baz IO overall: | 20260506 15:00:32.093 | INFO | BytesWritten : 1176103434182 | 20260506 15:00:32.093 | INFO | NumEvents : 569169933393 | 20260506 15:00:33.674 | INFO | Analyzing chunk frames = [2265088,2265600) | 20260506 15:00:33.674 | INFO | MemInfo: { | 20260506 15:00:33.674 | INFO | "gpu" : " 17 GB", | 20260506 15:00:33.674 | INFO | "huge" : "", | 20260506 15:00:33.674 | INFO | "huge_shared" : " 27 GB", | 20260506 15:00:33.674 | INFO | "malloc" : " 14 GB", | 20260506 15:00:33.674 | INFO | "pinned" : " 29 GB" | 20260506 15:00:33.674 | INFO | } | 20260506 15:00:33.674 | INFO | Perf report for the last 5.13963 seconds: | 20260506 15:00:33.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:00:33.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:33.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:33.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:33.674 | INFO | ANALYSIS : 62.9%, 2.9, 94.9ms, 0 | 20260506 15:00:33.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:33.675 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260506 15:00:33.675 | INFO | PRE_HQ : 60.9%, 1.9, 59.9ms, 0 | 20260506 15:00:33.675 | INFO | BAZWRITER : 66.5%, 4.8, 164.4ms, 0 | 20260506 15:00:33.675 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 15:00:33.675 | INFO | Uploaded 36GiB to GPU in 2.04495s (17.6043GiB/s) | 20260506 15:00:33.675 | INFO | Downloaded 38GiB to GPU in 2.21854s (17.1284GiB/s) | 20260506 15:00:38.812 | INFO | Analyzing chunk frames = [2265600,2266112) | 20260506 15:00:38.812 | INFO | MemInfo: { | 20260506 15:00:38.812 | INFO | "gpu" : " 17 GB", | 20260506 15:00:38.812 | INFO | "huge" : "", | 20260506 15:00:38.812 | INFO | "huge_shared" : " 27 GB", | 20260506 15:00:38.812 | INFO | "malloc" : " 14 GB", | 20260506 15:00:38.812 | INFO | "pinned" : " 29 GB" | 20260506 15:00:38.812 | INFO | } | 20260506 15:00:38.812 | INFO | Perf report for the last 5.13768 seconds: | 20260506 15:00:38.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:00:38.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:38.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:38.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:38.812 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 15:00:38.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:38.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:38.812 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:00:38.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:38.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:38.813 | INFO | Uploaded 36GiB to GPU in 1.97075s (18.2672GiB/s) | 20260506 15:00:38.813 | INFO | Downloaded 34GiB to GPU in 1.9341s (17.5792GiB/s) | 20260506 15:00:43.952 | INFO | Analyzing chunk frames = [2266112,2266624) | 20260506 15:00:43.952 | INFO | MemInfo: { | 20260506 15:00:43.952 | INFO | "gpu" : " 17 GB", | 20260506 15:00:43.952 | INFO | "huge" : "", | 20260506 15:00:43.952 | INFO | "huge_shared" : " 27 GB", | 20260506 15:00:43.952 | INFO | "malloc" : " 14 GB", | 20260506 15:00:43.952 | INFO | "pinned" : " 29 GB" | 20260506 15:00:43.952 | INFO | } | 20260506 15:00:43.952 | INFO | Perf report for the last 5.14019 seconds: | 20260506 15:00:43.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:00:43.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:43.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:43.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:43.952 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:00:43.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:43.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:43.952 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:00:43.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:43.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:43.953 | INFO | Uploaded 36GiB to GPU in 1.96746s (18.2977GiB/s) | 20260506 15:00:43.953 | INFO | Downloaded 34GiB to GPU in 1.9319s (17.5993GiB/s) | 20260506 15:00:49.091 | INFO | Analyzing chunk frames = [2266624,2267136) | 20260506 15:00:49.091 | INFO | MemInfo: { | 20260506 15:00:49.091 | INFO | "gpu" : " 17 GB", | 20260506 15:00:49.091 | INFO | "huge" : "", | 20260506 15:00:49.091 | INFO | "huge_shared" : " 27 GB", | 20260506 15:00:49.091 | INFO | "malloc" : " 14 GB", | 20260506 15:00:49.091 | INFO | "pinned" : " 29 GB" | 20260506 15:00:49.091 | INFO | } | 20260506 15:00:49.092 | INFO | Perf report for the last 5.13918 seconds: | 20260506 15:00:49.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:00:49.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:49.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:49.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:49.092 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 15:00:49.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:49.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:49.092 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:00:49.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:49.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:49.092 | INFO | Uploaded 36GiB to GPU in 1.95367s (18.4268GiB/s) | 20260506 15:00:49.092 | INFO | Downloaded 34GiB to GPU in 1.88739s (18.0143GiB/s) | 20260506 15:00:54.231 | INFO | Analyzing chunk frames = [2267136,2267648) | 20260506 15:00:54.231 | INFO | MemInfo: { | 20260506 15:00:54.231 | INFO | "gpu" : " 17 GB", | 20260506 15:00:54.231 | INFO | "huge" : "", | 20260506 15:00:54.231 | INFO | "huge_shared" : " 27 GB", | 20260506 15:00:54.231 | INFO | "malloc" : " 14 GB", | 20260506 15:00:54.232 | INFO | "pinned" : " 29 GB" | 20260506 15:00:54.232 | INFO | } | 20260506 15:00:54.232 | INFO | Perf report for the last 5.14001 seconds: | 20260506 15:00:54.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:00:54.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:54.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:54.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:54.232 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 15:00:54.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:54.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:54.232 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:00:54.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:54.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:54.232 | INFO | Uploaded 36GiB to GPU in 1.97521s (18.2259GiB/s) | 20260506 15:00:54.232 | INFO | Downloaded 34GiB to GPU in 1.92842s (17.631GiB/s) | 20260506 15:00:59.371 | INFO | Analyzing chunk frames = [2267648,2268160) | 20260506 15:00:59.371 | INFO | MemInfo: { | 20260506 15:00:59.371 | INFO | "gpu" : " 17 GB", | 20260506 15:00:59.371 | INFO | "huge" : "", | 20260506 15:00:59.371 | INFO | "huge_shared" : " 27 GB", | 20260506 15:00:59.371 | INFO | "malloc" : " 14 GB", | 20260506 15:00:59.371 | INFO | "pinned" : " 29 GB" | 20260506 15:00:59.371 | INFO | } | 20260506 15:00:59.371 | INFO | Perf report for the last 5.13921 seconds: | 20260506 15:00:59.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:00:59.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:59.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:59.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:59.371 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:00:59.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:59.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:59.371 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 15:00:59.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:59.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:00:59.371 | INFO | Uploaded 36GiB to GPU in 2.07126s (17.3807GiB/s) | 20260506 15:00:59.371 | INFO | Downloaded 34GiB to GPU in 1.97983s (17.1732GiB/s) | 20260506 15:01:04.510 | INFO | Analyzing chunk frames = [2268160,2268672) | 20260506 15:01:04.510 | INFO | MemInfo: { | 20260506 15:01:04.510 | INFO | "gpu" : " 17 GB", | 20260506 15:01:04.510 | INFO | "huge" : "", | 20260506 15:01:04.510 | INFO | "huge_shared" : " 27 GB", | 20260506 15:01:04.510 | INFO | "malloc" : " 14 GB", | 20260506 15:01:04.510 | INFO | "pinned" : " 29 GB" | 20260506 15:01:04.510 | INFO | } | 20260506 15:01:04.510 | INFO | Perf report for the last 5.1393 seconds: | 20260506 15:01:04.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:01:04.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:04.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:04.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:04.510 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:01:04.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:04.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:04.510 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:01:04.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:04.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:04.511 | INFO | Uploaded 36GiB to GPU in 1.99856s (18.0129GiB/s) | 20260506 15:01:04.511 | INFO | Downloaded 34GiB to GPU in 1.99443s (17.0474GiB/s) | 20260506 15:01:09.649 | INFO | Analyzing chunk frames = [2268672,2269184) | 20260506 15:01:09.649 | INFO | MemInfo: { | 20260506 15:01:09.649 | INFO | "gpu" : " 17 GB", | 20260506 15:01:09.649 | INFO | "huge" : "", | 20260506 15:01:09.649 | INFO | "huge_shared" : " 27 GB", | 20260506 15:01:09.649 | INFO | "malloc" : " 14 GB", | 20260506 15:01:09.649 | INFO | "pinned" : " 29 GB" | 20260506 15:01:09.649 | INFO | } | 20260506 15:01:09.649 | INFO | Perf report for the last 5.13932 seconds: | 20260506 15:01:09.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:01:09.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:09.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:09.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:09.649 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:01:09.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:09.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:09.649 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:01:09.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:09.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:09.650 | INFO | Uploaded 36GiB to GPU in 2.02472s (17.7802GiB/s) | 20260506 15:01:09.650 | INFO | Downloaded 34GiB to GPU in 1.98086s (17.1643GiB/s) | 20260506 15:01:09.650 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:01:09.650 | INFO | Total mem usage: 17.4 GiB | 20260506 15:01:09.650 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:01:09.650 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -10.3%) | 20260506 15:01:09.650 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:01:09.650 | INFO | High Water Breakdown: | 20260506 15:01:09.650 | INFO | 61.4% : 9.7 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:01:09.650 | INFO | 38.4% : 6.07 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:01:09.650 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:01:09.650 | INFO | ------------------------------------------ | 20260506 15:01:09.650 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:01:09.650 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:01:09.650 | INFO | Total mem usage: 26.5 GiB | 20260506 15:01:09.650 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:01:09.650 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:01:09.650 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:01:09.650 | INFO | High Water Breakdown: | 20260506 15:01:09.650 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:01:09.650 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:01:09.650 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:01:09.650 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:01:09.650 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:01:09.650 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:01:09.650 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:01:09.650 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:01:09.650 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:01:09.650 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:01:09.650 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:01:09.650 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:01:09.650 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:01:09.650 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:01:09.650 | INFO | ------------------------------------------ | 20260506 15:01:09.650 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:01:09.650 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:01:09.650 | INFO | Total mem usage: 25.6 GiB | 20260506 15:01:09.650 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:01:09.650 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:01:09.650 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:01:09.650 | INFO | High Water Breakdown: | 20260506 15:01:09.650 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:01:09.650 | INFO | ------------------------------------------ | 20260506 15:01:09.650 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:01:09.650 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:01:09.650 | INFO | Total mem usage: 15.7 GiB | 20260506 15:01:09.650 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:01:09.650 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:01:09.651 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 15:01:09.651 | INFO | High Water Breakdown: | 20260506 15:01:09.651 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:01:09.651 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:01:09.651 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:01:09.651 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:01:09.651 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:01:09.651 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:01:09.651 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:01:09.651 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:01:09.651 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:01:09.651 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:01:09.651 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:01:09.651 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:01:09.651 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:01:09.651 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:01:09.651 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:01:09.651 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:01:09.651 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:01:09.651 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:01:09.651 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:01:09.651 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:01:09.651 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:01:09.651 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:01:09.651 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:01:09.651 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:01:09.651 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:01:09.651 | INFO | ------------------------------------------ | 20260506 15:01:09.651 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:01:09.651 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:01:09.651 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:01:09.651 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:01:09.651 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.53e+01, 1.34e+01 / 1.72e+00 | 20260506 15:01:09.651 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 9.99e-02, 1.05e-02 / 7.25e-03 | 20260506 15:01:09.651 | INFO | PerfReport: 11.57%, Baseline -- 3.64e+00 / 4.43e+00, 5.63e-01 / 3.61e-01 | 20260506 15:01:09.651 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.46e+00, 1.88e-01 / 1.52e-01 | 20260506 15:01:09.651 | INFO | PerfReport: 15.80%, DME -- 3.98e+01 / 4.37e+01, 3.68e+00 / 1.72e+00 | 20260506 15:01:09.651 | INFO | PerfReport: 56.85%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.50e+00 / 2.07e+00 | 20260506 15:01:09.651 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.33e+00 / 2.93e+00, 4.04e-01 / 2.57e-01 | 20260506 15:01:09.651 | INFO | PerfReport: 3.57%, Metrics -- 1.12e+00 / 1.51e+00, 2.92e-01 / 2.00e-01 | 20260506 15:01:09.651 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.92e-01 / 4.45e-01, 6.14e-02 / 3.47e-02 | 20260506 15:01:09.651 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:01:09.651 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:01:09.651 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:01:09.651 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 15:01:09.651 | INFO | PerfReport: 21.41%, Upload -- 2.09e+01 / 0.00e+00, 6.94e+00 / 0.00e+00 | 20260506 15:01:09.651 | INFO | PerfReport: 21.20%, Download -- 2.07e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 15:01:09.651 | INFO | PerfReport: 32.29%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 15:01:09.651 | INFO | PerfReport: 1.39%, LockUpload -- 1.35e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260506 15:01:09.651 | INFO | PerfReport: 1.19%, LockDownload -- 1.16e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260506 15:01:09.651 | INFO | PerfReport: 22.50%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 15:01:09.651 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260506 15:01:09.762 | INFO | Starting a round of Baz IO | 20260506 15:01:12.916 | INFO | Populating RTMetrics report | 20260506 15:01:12.916 | INFO | RTMetrics report populated | 20260506 15:01:12.916 | INFO | Opening tmp RTMetrics json file | 20260506 15:01:13.068 | INFO | Serializing RTMetrics to json | 20260506 15:01:13.102 | INFO | Serialization done, swapping tmp file now... | 20260506 15:01:13.105 | INFO | RTMetrics done | 20260506 15:01:13.145 | INFO | Finished a round of Baz IO: Wrote 2.105 GiB in 3.383 seconds | 20260506 15:01:13.145 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.64% / 0.01% / 62.34% | 20260506 15:01:13.145 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.25544% / 1.58946e-05% / 7.94729e-05% | 20260506 15:01:13.145 | INFO | Baz IO overall: | 20260506 15:01:13.145 | INFO | BytesWritten : 1178363461518 | 20260506 15:01:13.145 | INFO | NumEvents : 570264073021 | 20260506 15:01:14.788 | INFO | Analyzing chunk frames = [2269184,2269696) | 20260506 15:01:14.788 | INFO | MemInfo: { | 20260506 15:01:14.788 | INFO | "gpu" : " 17 GB", | 20260506 15:01:14.788 | INFO | "huge" : "", | 20260506 15:01:14.788 | INFO | "huge_shared" : " 27 GB", | 20260506 15:01:14.788 | INFO | "malloc" : " 14 GB", | 20260506 15:01:14.788 | INFO | "pinned" : " 29 GB" | 20260506 15:01:14.788 | INFO | } | 20260506 15:01:14.788 | INFO | Perf report for the last 5.13821 seconds: | 20260506 15:01:14.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:01:14.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:14.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:14.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:14.788 | INFO | ANALYSIS : 63.0%, 2.9, 95.1ms, 0 | 20260506 15:01:14.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:14.788 | INFO | RT_METRICS : 47.6%, 1.0, 25.0ms, 0 | 20260506 15:01:14.788 | INFO | PRE_HQ : 61.1%, 1.9, 59.8ms, 0 | 20260506 15:01:14.788 | INFO | BAZWRITER : 65.2%, 4.8, 160.2ms, 0 | 20260506 15:01:14.788 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.5ms, 0 | 20260506 15:01:14.788 | INFO | Uploaded 36GiB to GPU in 2.05546s (17.5143GiB/s) | 20260506 15:01:14.788 | INFO | Downloaded 38GiB to GPU in 2.22156s (17.1051GiB/s) | 20260506 15:01:19.927 | INFO | Analyzing chunk frames = [2269696,2270208) | 20260506 15:01:19.927 | INFO | MemInfo: { | 20260506 15:01:19.927 | INFO | "gpu" : " 17 GB", | 20260506 15:01:19.927 | INFO | "huge" : "", | 20260506 15:01:19.927 | INFO | "huge_shared" : " 27 GB", | 20260506 15:01:19.927 | INFO | "malloc" : " 14 GB", | 20260506 15:01:19.927 | INFO | "pinned" : " 29 GB" | 20260506 15:01:19.927 | INFO | } | 20260506 15:01:19.927 | INFO | Perf report for the last 5.13958 seconds: | 20260506 15:01:19.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:01:19.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:19.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:19.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:19.927 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 15:01:19.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:19.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:19.927 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:01:19.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:19.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:19.928 | INFO | Uploaded 36GiB to GPU in 1.9936s (18.0578GiB/s) | 20260506 15:01:19.928 | INFO | Downloaded 34GiB to GPU in 1.93769s (17.5467GiB/s) | 20260506 15:01:25.066 | INFO | Analyzing chunk frames = [2270208,2270720) | 20260506 15:01:25.066 | INFO | MemInfo: { | 20260506 15:01:25.066 | INFO | "gpu" : " 17 GB", | 20260506 15:01:25.066 | INFO | "huge" : "", | 20260506 15:01:25.066 | INFO | "huge_shared" : " 27 GB", | 20260506 15:01:25.066 | INFO | "malloc" : " 14 GB", | 20260506 15:01:25.066 | INFO | "pinned" : " 29 GB" | 20260506 15:01:25.066 | INFO | } | 20260506 15:01:25.066 | INFO | Perf report for the last 5.13909 seconds: | 20260506 15:01:25.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:01:25.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:25.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:25.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:25.067 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 15:01:25.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:25.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:25.067 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:01:25.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:25.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:25.067 | INFO | Uploaded 36GiB to GPU in 2.0027s (17.9757GiB/s) | 20260506 15:01:25.067 | INFO | Downloaded 34GiB to GPU in 1.94083s (17.5183GiB/s) | 20260506 15:01:30.206 | INFO | Analyzing chunk frames = [2270720,2271232) | 20260506 15:01:30.206 | INFO | MemInfo: { | 20260506 15:01:30.206 | INFO | "gpu" : " 17 GB", | 20260506 15:01:30.206 | INFO | "huge" : "", | 20260506 15:01:30.206 | INFO | "huge_shared" : " 27 GB", | 20260506 15:01:30.206 | INFO | "malloc" : " 14 GB", | 20260506 15:01:30.206 | INFO | "pinned" : " 29 GB" | 20260506 15:01:30.206 | INFO | } | 20260506 15:01:30.206 | INFO | Perf report for the last 5.14002 seconds: | 20260506 15:01:30.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:01:30.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:30.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:30.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:30.206 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:01:30.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:30.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:30.206 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:01:30.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:30.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:30.207 | INFO | Uploaded 36GiB to GPU in 2.01112s (17.9004GiB/s) | 20260506 15:01:30.207 | INFO | Downloaded 34GiB to GPU in 1.89786s (17.9149GiB/s) | 20260506 15:01:35.346 | INFO | Analyzing chunk frames = [2271232,2271744) | 20260506 15:01:35.346 | INFO | MemInfo: { | 20260506 15:01:35.346 | INFO | "gpu" : " 17 GB", | 20260506 15:01:35.346 | INFO | "huge" : "", | 20260506 15:01:35.346 | INFO | "huge_shared" : " 27 GB", | 20260506 15:01:35.346 | INFO | "malloc" : " 14 GB", | 20260506 15:01:35.346 | INFO | "pinned" : " 29 GB" | 20260506 15:01:35.346 | INFO | } | 20260506 15:01:35.346 | INFO | Perf report for the last 5.13996 seconds: | 20260506 15:01:35.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:01:35.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:35.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:35.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:35.346 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 15:01:35.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:35.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:35.346 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:01:35.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:35.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:35.347 | INFO | Uploaded 36GiB to GPU in 2.00648s (17.9419GiB/s) | 20260506 15:01:35.347 | INFO | Downloaded 34GiB to GPU in 1.93372s (17.5827GiB/s) | 20260506 15:01:40.486 | INFO | Analyzing chunk frames = [2271744,2272256) | 20260506 15:01:40.486 | INFO | MemInfo: { | 20260506 15:01:40.486 | INFO | "gpu" : " 17 GB", | 20260506 15:01:40.486 | INFO | "huge" : "", | 20260506 15:01:40.486 | INFO | "huge_shared" : " 27 GB", | 20260506 15:01:40.486 | INFO | "malloc" : " 14 GB", | 20260506 15:01:40.486 | INFO | "pinned" : " 29 GB" | 20260506 15:01:40.486 | INFO | } | 20260506 15:01:40.486 | INFO | Perf report for the last 5.1397 seconds: | 20260506 15:01:40.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:01:40.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:40.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:40.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:40.486 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 15:01:40.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:40.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:40.486 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260506 15:01:40.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:40.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:40.486 | INFO | Uploaded 36GiB to GPU in 2.04902s (17.5693GiB/s) | 20260506 15:01:40.486 | INFO | Downloaded 34GiB to GPU in 1.92316s (17.6792GiB/s) | 20260506 15:01:45.625 | INFO | Analyzing chunk frames = [2272256,2272768) | 20260506 15:01:45.625 | INFO | MemInfo: { | 20260506 15:01:45.625 | INFO | "gpu" : " 17 GB", | 20260506 15:01:45.625 | INFO | "huge" : "", | 20260506 15:01:45.625 | INFO | "huge_shared" : " 27 GB", | 20260506 15:01:45.625 | INFO | "malloc" : " 14 GB", | 20260506 15:01:45.625 | INFO | "pinned" : " 29 GB" | 20260506 15:01:45.625 | INFO | } | 20260506 15:01:45.625 | INFO | Perf report for the last 5.13855 seconds: | 20260506 15:01:45.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:01:45.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:45.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:45.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:45.625 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:01:45.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:45.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:45.625 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:01:45.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:45.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:45.625 | INFO | Uploaded 36GiB to GPU in 2.00771s (17.9309GiB/s) | 20260506 15:01:45.625 | INFO | Downloaded 34GiB to GPU in 1.99395s (17.0515GiB/s) | 20260506 15:01:50.764 | INFO | Analyzing chunk frames = [2272768,2273280) | 20260506 15:01:50.764 | INFO | MemInfo: { | 20260506 15:01:50.764 | INFO | "gpu" : " 17 GB", | 20260506 15:01:50.764 | INFO | "huge" : "", | 20260506 15:01:50.764 | INFO | "huge_shared" : " 27 GB", | 20260506 15:01:50.764 | INFO | "malloc" : " 14 GB", | 20260506 15:01:50.764 | INFO | "pinned" : " 29 GB" | 20260506 15:01:50.764 | INFO | } | 20260506 15:01:50.764 | INFO | Perf report for the last 5.13951 seconds: | 20260506 15:01:50.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:01:50.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:50.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:50.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:50.764 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 15:01:50.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:50.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:50.764 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:01:50.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:50.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:50.764 | INFO | Uploaded 36GiB to GPU in 2.00436s (17.9608GiB/s) | 20260506 15:01:50.764 | INFO | Downloaded 34GiB to GPU in 1.98139s (17.1597GiB/s) | 20260506 15:01:50.885 | INFO | Starting a round of Baz IO | 20260506 15:01:54.051 | INFO | Populating RTMetrics report | 20260506 15:01:54.052 | INFO | RTMetrics report populated | 20260506 15:01:54.052 | INFO | Opening tmp RTMetrics json file | 20260506 15:01:54.059 | INFO | Serializing RTMetrics to json | 20260506 15:01:54.084 | INFO | Serialization done, swapping tmp file now... | 20260506 15:01:54.108 | INFO | RTMetrics done | 20260506 15:01:54.502 | INFO | Finished a round of Baz IO: Wrote 2.107 GiB in 3.617 seconds | 20260506 15:01:54.502 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.63% / 0.01% / 62.36% | 20260506 15:01:54.502 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.25734% / 3.97364e-05% / 6.35783e-05% | 20260506 15:01:54.502 | INFO | Baz IO overall: | 20260506 15:01:54.502 | INFO | BytesWritten : 1180625626043 | 20260506 15:01:54.502 | INFO | NumEvents : 571359819970 | 20260506 15:01:55.903 | INFO | Analyzing chunk frames = [2273280,2273792) | 20260506 15:01:55.903 | INFO | MemInfo: { | 20260506 15:01:55.903 | INFO | "gpu" : " 17 GB", | 20260506 15:01:55.903 | INFO | "huge" : "", | 20260506 15:01:55.903 | INFO | "huge_shared" : " 27 GB", | 20260506 15:01:55.903 | INFO | "malloc" : " 14 GB", | 20260506 15:01:55.903 | INFO | "pinned" : " 29 GB" | 20260506 15:01:55.903 | INFO | } | 20260506 15:01:55.903 | INFO | Perf report for the last 5.13858 seconds: | 20260506 15:01:55.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:01:55.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:55.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:55.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:55.903 | INFO | ANALYSIS : 63.1%, 2.9, 95.9ms, 0 | 20260506 15:01:55.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:01:55.903 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 15:01:55.903 | INFO | PRE_HQ : 60.8%, 1.9, 59.1ms, 0 | 20260506 15:01:55.903 | INFO | BAZWRITER : 69.7%, 4.8, 170.8ms, 0 | 20260506 15:01:55.903 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 15:01:55.903 | INFO | Uploaded 36GiB to GPU in 2.05188s (17.5449GiB/s) | 20260506 15:01:55.903 | INFO | Downloaded 38GiB to GPU in 2.22165s (17.1044GiB/s) | 20260506 15:02:01.042 | INFO | Analyzing chunk frames = [2273792,2274304) | 20260506 15:02:01.042 | INFO | MemInfo: { | 20260506 15:02:01.042 | INFO | "gpu" : " 17 GB", | 20260506 15:02:01.042 | INFO | "huge" : "", | 20260506 15:02:01.042 | INFO | "huge_shared" : " 27 GB", | 20260506 15:02:01.042 | INFO | "malloc" : " 14 GB", | 20260506 15:02:01.042 | INFO | "pinned" : " 29 GB" | 20260506 15:02:01.042 | INFO | } | 20260506 15:02:01.042 | INFO | Perf report for the last 5.13948 seconds: | 20260506 15:02:01.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:02:01.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:01.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:01.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:01.042 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:02:01.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:01.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:01.042 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:02:01.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:01.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:01.042 | INFO | Uploaded 36GiB to GPU in 1.98324s (18.1521GiB/s) | 20260506 15:02:01.043 | INFO | Downloaded 34GiB to GPU in 1.93728s (17.5504GiB/s) | 20260506 15:02:06.182 | INFO | Analyzing chunk frames = [2274304,2274816) | 20260506 15:02:06.182 | INFO | MemInfo: { | 20260506 15:02:06.182 | INFO | "gpu" : " 17 GB", | 20260506 15:02:06.182 | INFO | "huge" : "", | 20260506 15:02:06.182 | INFO | "huge_shared" : " 27 GB", | 20260506 15:02:06.182 | INFO | "malloc" : " 14 GB", | 20260506 15:02:06.182 | INFO | "pinned" : " 29 GB" | 20260506 15:02:06.182 | INFO | } | 20260506 15:02:06.182 | INFO | Perf report for the last 5.13963 seconds: | 20260506 15:02:06.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:02:06.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:06.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:06.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:06.182 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:02:06.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:06.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:06.182 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:02:06.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:06.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:06.182 | INFO | Uploaded 36GiB to GPU in 1.98973s (18.0929GiB/s) | 20260506 15:02:06.182 | INFO | Downloaded 34GiB to GPU in 1.942s (17.5077GiB/s) | 20260506 15:02:11.322 | INFO | Analyzing chunk frames = [2274816,2275328) | 20260506 15:02:11.322 | INFO | MemInfo: { | 20260506 15:02:11.322 | INFO | "gpu" : " 17 GB", | 20260506 15:02:11.322 | INFO | "huge" : "", | 20260506 15:02:11.322 | INFO | "huge_shared" : " 27 GB", | 20260506 15:02:11.322 | INFO | "malloc" : " 14 GB", | 20260506 15:02:11.322 | INFO | "pinned" : " 29 GB" | 20260506 15:02:11.322 | INFO | } | 20260506 15:02:11.322 | INFO | Perf report for the last 5.14022 seconds: | 20260506 15:02:11.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:02:11.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:11.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:11.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:11.322 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:02:11.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:11.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:11.322 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:02:11.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:11.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:11.322 | INFO | Uploaded 36GiB to GPU in 2.01641s (17.8535GiB/s) | 20260506 15:02:11.322 | INFO | Downloaded 34GiB to GPU in 1.92068s (17.702GiB/s) | 20260506 15:02:16.461 | INFO | Analyzing chunk frames = [2275328,2275840) | 20260506 15:02:16.461 | INFO | MemInfo: { | 20260506 15:02:16.461 | INFO | "gpu" : " 17 GB", | 20260506 15:02:16.461 | INFO | "huge" : "", | 20260506 15:02:16.461 | INFO | "huge_shared" : " 27 GB", | 20260506 15:02:16.461 | INFO | "malloc" : " 14 GB", | 20260506 15:02:16.461 | INFO | "pinned" : " 29 GB" | 20260506 15:02:16.461 | INFO | } | 20260506 15:02:16.461 | INFO | Perf report for the last 5.13853 seconds: | 20260506 15:02:16.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:02:16.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:16.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:16.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:16.461 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 15:02:16.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:16.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:16.461 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:02:16.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:16.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:16.461 | INFO | Uploaded 36GiB to GPU in 1.99015s (18.0891GiB/s) | 20260506 15:02:16.461 | INFO | Downloaded 34GiB to GPU in 1.94657s (17.4666GiB/s) | 20260506 15:02:21.599 | INFO | Analyzing chunk frames = [2275840,2276352) | 20260506 15:02:21.599 | INFO | MemInfo: { | 20260506 15:02:21.599 | INFO | "gpu" : " 17 GB", | 20260506 15:02:21.599 | INFO | "huge" : "", | 20260506 15:02:21.599 | INFO | "huge_shared" : " 27 GB", | 20260506 15:02:21.600 | INFO | "malloc" : " 14 GB", | 20260506 15:02:21.600 | INFO | "pinned" : " 29 GB" | 20260506 15:02:21.600 | INFO | } | 20260506 15:02:21.600 | INFO | Perf report for the last 5.13902 seconds: | 20260506 15:02:21.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:02:21.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:21.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:21.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:21.600 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 15:02:21.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:21.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:21.600 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260506 15:02:21.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:21.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:21.600 | INFO | Uploaded 36GiB to GPU in 2.04919s (17.5679GiB/s) | 20260506 15:02:21.600 | INFO | Downloaded 34GiB to GPU in 1.97118s (17.2485GiB/s) | 20260506 15:02:26.740 | INFO | Analyzing chunk frames = [2276352,2276864) | 20260506 15:02:26.740 | INFO | MemInfo: { | 20260506 15:02:26.740 | INFO | "gpu" : " 17 GB", | 20260506 15:02:26.740 | INFO | "huge" : "", | 20260506 15:02:26.740 | INFO | "huge_shared" : " 27 GB", | 20260506 15:02:26.740 | INFO | "malloc" : " 14 GB", | 20260506 15:02:26.740 | INFO | "pinned" : " 29 GB" | 20260506 15:02:26.740 | INFO | } | 20260506 15:02:26.740 | INFO | Perf report for the last 5.14003 seconds: | 20260506 15:02:26.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:02:26.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:26.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:26.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:26.740 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 15:02:26.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:26.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:26.740 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 15:02:26.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:26.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:26.744 | INFO | Uploaded 36GiB to GPU in 2.05571s (17.5122GiB/s) | 20260506 15:02:26.744 | INFO | Downloaded 34GiB to GPU in 2.0131s (16.8894GiB/s) | 20260506 15:02:31.878 | INFO | Analyzing chunk frames = [2276864,2277376) | 20260506 15:02:31.878 | INFO | MemInfo: { | 20260506 15:02:31.878 | INFO | "gpu" : " 17 GB", | 20260506 15:02:31.878 | INFO | "huge" : "", | 20260506 15:02:31.878 | INFO | "huge_shared" : " 27 GB", | 20260506 15:02:31.878 | INFO | "malloc" : " 14 GB", | 20260506 15:02:31.878 | INFO | "pinned" : " 29 GB" | 20260506 15:02:31.878 | INFO | } | 20260506 15:02:31.878 | INFO | Perf report for the last 5.138 seconds: | 20260506 15:02:31.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:02:31.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:31.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:31.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:31.878 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 15:02:31.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:31.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:31.878 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:02:31.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:31.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:31.878 | INFO | Uploaded 36GiB to GPU in 1.95502s (18.4142GiB/s) | 20260506 15:02:31.878 | INFO | Downloaded 34GiB to GPU in 1.97631s (17.2038GiB/s) | 20260506 15:02:31.878 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:02:31.878 | INFO | Total mem usage: 17.4 GiB | 20260506 15:02:31.878 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:02:31.878 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -12.4%) | 20260506 15:02:31.878 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:02:31.878 | INFO | High Water Breakdown: | 20260506 15:02:31.878 | INFO | 61.1% : 9.46 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:02:31.878 | INFO | 38.8% : 6.01 GiB ( 89.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:02:31.878 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:02:31.878 | INFO | ------------------------------------------ | 20260506 15:02:31.878 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:02:31.878 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:02:31.878 | INFO | Total mem usage: 26.5 GiB | 20260506 15:02:31.878 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:02:31.878 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:02:31.878 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:02:31.878 | INFO | High Water Breakdown: | 20260506 15:02:31.878 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:02:31.878 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:02:31.878 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:02:31.878 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:02:31.878 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:02:31.878 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:02:31.878 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:02:31.878 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:02:31.878 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:02:31.878 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:02:31.878 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:02:31.878 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:02:31.878 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:02:31.878 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:02:31.878 | INFO | ------------------------------------------ | 20260506 15:02:31.878 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:02:31.878 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:02:31.878 | INFO | Total mem usage: 25.6 GiB | 20260506 15:02:31.878 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:02:31.879 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:02:31.879 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:02:31.879 | INFO | High Water Breakdown: | 20260506 15:02:31.879 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:02:31.879 | INFO | ------------------------------------------ | 20260506 15:02:31.879 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:02:31.879 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:02:31.879 | INFO | Total mem usage: 15.7 GiB | 20260506 15:02:31.879 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:02:31.879 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:02:31.879 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 15:02:31.879 | INFO | High Water Breakdown: | 20260506 15:02:31.879 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:02:31.879 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:02:31.879 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:02:31.879 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:02:31.879 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:02:31.879 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:02:31.879 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:02:31.879 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:02:31.879 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:02:31.879 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:02:31.879 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:02:31.879 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:02:31.879 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:02:31.879 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:02:31.879 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:02:31.879 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:02:31.879 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:02:31.879 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:02:31.879 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:02:31.879 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:02:31.879 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:02:31.879 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:02:31.879 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:02:31.879 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:02:31.879 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:02:31.879 | INFO | ------------------------------------------ | 20260506 15:02:31.879 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:02:31.879 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:02:31.879 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:02:31.879 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:02:31.879 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.59e+01, 1.35e+01 / 2.21e+00 | 20260506 15:02:31.879 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.01e-01, 1.01e-02 / 6.65e-03 | 20260506 15:02:31.879 | INFO | PerfReport: 11.57%, Baseline -- 3.65e+00 / 4.43e+00, 5.68e-01 / 3.59e-01 | 20260506 15:02:31.879 | INFO | PerfReport: 3.25%, Histogram -- 1.03e+00 / 1.49e+00, 1.90e-01 / 1.37e-01 | 20260506 15:02:31.879 | INFO | PerfReport: 15.87%, DME -- 4.00e+01 / 4.45e+01, 3.97e+00 / 2.67e+00 | 20260506 15:02:31.879 | INFO | PerfReport: 56.85%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.54e+00 / 1.78e+00 | 20260506 15:02:31.879 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.32e+00 / 2.90e+00, 3.92e-01 / 2.34e-01 | 20260506 15:02:31.879 | INFO | PerfReport: 3.56%, Metrics -- 1.12e+00 / 1.48e+00, 2.89e-01 / 2.15e-01 | 20260506 15:02:31.879 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.91e-01 / 4.45e-01, 6.06e-02 / 2.10e-02 | 20260506 15:02:31.879 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:02:31.879 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:02:31.879 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:02:31.879 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 15:02:31.879 | INFO | PerfReport: 21.49%, Upload -- 2.10e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 15:02:31.879 | INFO | PerfReport: 21.18%, Download -- 2.07e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 15:02:31.879 | INFO | PerfReport: 32.28%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 15:02:31.879 | INFO | PerfReport: 1.48%, LockUpload -- 1.45e+00 / 0.00e+00, 5.74e+00 / 0.00e+00 | 20260506 15:02:31.879 | INFO | PerfReport: 1.10%, LockDownload -- 1.08e+00 / 0.00e+00, 2.69e+00 / 0.00e+00 | 20260506 15:02:31.879 | INFO | PerfReport: 22.46%, LockCompute -- 2.19e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 15:02:31.879 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260506 15:02:31.993 | INFO | Starting a round of Baz IO | 20260506 15:02:35.144 | INFO | Populating RTMetrics report | 20260506 15:02:35.145 | INFO | RTMetrics report populated | 20260506 15:02:35.145 | INFO | Opening tmp RTMetrics json file | 20260506 15:02:35.146 | INFO | Serializing RTMetrics to json | 20260506 15:02:35.148 | INFO | Serialization done, swapping tmp file now... | 20260506 15:02:35.148 | INFO | RTMetrics done | 20260506 15:02:35.579 | INFO | Finished a round of Baz IO: Wrote 2.106 GiB in 3.585 seconds | 20260506 15:02:35.579 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.62% / 0.01% / 62.37% | 20260506 15:02:35.579 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.24689% / 1.58946e-05% / 7.54992e-05% | 20260506 15:02:35.579 | INFO | Baz IO overall: | 20260506 15:02:35.579 | INFO | BytesWritten : 1182886450197 | 20260506 15:02:35.579 | INFO | NumEvents : 572454483964 | 20260506 15:02:37.018 | INFO | Analyzing chunk frames = [2277376,2277888) | 20260506 15:02:37.018 | INFO | MemInfo: { | 20260506 15:02:37.018 | INFO | "gpu" : " 17 GB", | 20260506 15:02:37.018 | INFO | "huge" : "", | 20260506 15:02:37.018 | INFO | "huge_shared" : " 27 GB", | 20260506 15:02:37.018 | INFO | "malloc" : " 14 GB", | 20260506 15:02:37.018 | INFO | "pinned" : " 29 GB" | 20260506 15:02:37.018 | INFO | } | 20260506 15:02:37.018 | INFO | Perf report for the last 5.14017 seconds: | 20260506 15:02:37.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:02:37.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:37.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:37.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:37.018 | INFO | ANALYSIS : 62.9%, 2.9, 95.7ms, 0 | 20260506 15:02:37.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:37.018 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 15:02:37.018 | INFO | PRE_HQ : 60.5%, 1.9, 58.8ms, 0 | 20260506 15:02:37.018 | INFO | BAZWRITER : 69.4%, 4.8, 172.2ms, 0 | 20260506 15:02:37.018 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 15:02:37.018 | INFO | Uploaded 36GiB to GPU in 2.04013s (17.6459GiB/s) | 20260506 15:02:37.018 | INFO | Downloaded 38GiB to GPU in 2.21461s (17.1588GiB/s) | 20260506 15:02:42.157 | INFO | Analyzing chunk frames = [2277888,2278400) | 20260506 15:02:42.157 | INFO | MemInfo: { | 20260506 15:02:42.157 | INFO | "gpu" : " 17 GB", | 20260506 15:02:42.157 | INFO | "huge" : "", | 20260506 15:02:42.157 | INFO | "huge_shared" : " 27 GB", | 20260506 15:02:42.157 | INFO | "malloc" : " 14 GB", | 20260506 15:02:42.157 | INFO | "pinned" : " 29 GB" | 20260506 15:02:42.157 | INFO | } | 20260506 15:02:42.157 | INFO | Perf report for the last 5.13926 seconds: | 20260506 15:02:42.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:02:42.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:42.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:42.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:42.157 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:02:42.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:42.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:42.157 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:02:42.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:42.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:42.158 | INFO | Uploaded 36GiB to GPU in 1.97365s (18.2403GiB/s) | 20260506 15:02:42.158 | INFO | Downloaded 34GiB to GPU in 1.94083s (17.5183GiB/s) | 20260506 15:02:47.297 | INFO | Analyzing chunk frames = [2278400,2278912) | 20260506 15:02:47.297 | INFO | MemInfo: { | 20260506 15:02:47.297 | INFO | "gpu" : " 17 GB", | 20260506 15:02:47.297 | INFO | "huge" : "", | 20260506 15:02:47.297 | INFO | "huge_shared" : " 27 GB", | 20260506 15:02:47.297 | INFO | "malloc" : " 14 GB", | 20260506 15:02:47.297 | INFO | "pinned" : " 29 GB" | 20260506 15:02:47.297 | INFO | } | 20260506 15:02:47.297 | INFO | Perf report for the last 5.13959 seconds: | 20260506 15:02:47.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:02:47.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:47.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:47.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:47.297 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:02:47.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:47.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:47.297 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:02:47.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:47.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:47.297 | INFO | Uploaded 36GiB to GPU in 1.98408s (18.1444GiB/s) | 20260506 15:02:47.297 | INFO | Downloaded 34GiB to GPU in 1.94249s (17.5033GiB/s) | 20260506 15:02:52.437 | INFO | Analyzing chunk frames = [2278912,2279424) | 20260506 15:02:52.437 | INFO | MemInfo: { | 20260506 15:02:52.437 | INFO | "gpu" : " 17 GB", | 20260506 15:02:52.437 | INFO | "huge" : "", | 20260506 15:02:52.437 | INFO | "huge_shared" : " 27 GB", | 20260506 15:02:52.437 | INFO | "malloc" : " 14 GB", | 20260506 15:02:52.437 | INFO | "pinned" : " 29 GB" | 20260506 15:02:52.437 | INFO | } | 20260506 15:02:52.437 | INFO | Perf report for the last 5.14014 seconds: | 20260506 15:02:52.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:02:52.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:52.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:52.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:52.437 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 15:02:52.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:52.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:52.437 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:02:52.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:52.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:52.437 | INFO | Uploaded 36GiB to GPU in 2.01383s (17.8764GiB/s) | 20260506 15:02:52.437 | INFO | Downloaded 34GiB to GPU in 1.92405s (17.6711GiB/s) | 20260506 15:02:57.576 | INFO | Analyzing chunk frames = [2279424,2279936) | 20260506 15:02:57.576 | INFO | MemInfo: { | 20260506 15:02:57.576 | INFO | "gpu" : " 17 GB", | 20260506 15:02:57.576 | INFO | "huge" : "", | 20260506 15:02:57.576 | INFO | "huge_shared" : " 27 GB", | 20260506 15:02:57.576 | INFO | "malloc" : " 14 GB", | 20260506 15:02:57.576 | INFO | "pinned" : " 29 GB" | 20260506 15:02:57.576 | INFO | } | 20260506 15:02:57.577 | INFO | Perf report for the last 5.13964 seconds: | 20260506 15:02:57.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:02:57.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:57.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:57.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:57.577 | INFO | ANALYSIS : 60.9%, 3.0, 95.1ms, 0 | 20260506 15:02:57.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:57.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:57.577 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:02:57.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:57.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:02:57.577 | INFO | Uploaded 36GiB to GPU in 1.98038s (18.1783GiB/s) | 20260506 15:02:57.577 | INFO | Downloaded 34GiB to GPU in 1.9127s (17.7759GiB/s) | 20260506 15:03:02.715 | INFO | Analyzing chunk frames = [2279936,2280448) | 20260506 15:03:02.715 | INFO | MemInfo: { | 20260506 15:03:02.715 | INFO | "gpu" : " 17 GB", | 20260506 15:03:02.715 | INFO | "huge" : "", | 20260506 15:03:02.715 | INFO | "huge_shared" : " 27 GB", | 20260506 15:03:02.715 | INFO | "malloc" : " 14 GB", | 20260506 15:03:02.715 | INFO | "pinned" : " 29 GB" | 20260506 15:03:02.715 | INFO | } | 20260506 15:03:02.715 | INFO | Perf report for the last 5.1389 seconds: | 20260506 15:03:02.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:03:02.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:02.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:02.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:02.716 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 15:03:02.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:02.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:02.716 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 15:03:02.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:02.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:02.716 | INFO | Uploaded 36GiB to GPU in 2.04934s (17.5666GiB/s) | 20260506 15:03:02.716 | INFO | Downloaded 34GiB to GPU in 1.97624s (17.2043GiB/s) | 20260506 15:03:07.854 | INFO | Analyzing chunk frames = [2280448,2280960) | 20260506 15:03:07.854 | INFO | MemInfo: { | 20260506 15:03:07.854 | INFO | "gpu" : " 17 GB", | 20260506 15:03:07.854 | INFO | "huge" : "", | 20260506 15:03:07.854 | INFO | "huge_shared" : " 27 GB", | 20260506 15:03:07.854 | INFO | "malloc" : " 14 GB", | 20260506 15:03:07.854 | INFO | "pinned" : " 29 GB" | 20260506 15:03:07.854 | INFO | } | 20260506 15:03:07.854 | INFO | Perf report for the last 5.13888 seconds: | 20260506 15:03:07.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:03:07.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:07.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:07.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:07.854 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 15:03:07.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:07.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:07.855 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 15:03:07.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:07.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:07.855 | INFO | Uploaded 36GiB to GPU in 2.02962s (17.7373GiB/s) | 20260506 15:03:07.855 | INFO | Downloaded 34GiB to GPU in 2.01551s (16.8692GiB/s) | 20260506 15:03:12.993 | INFO | Analyzing chunk frames = [2280960,2281472) | 20260506 15:03:12.993 | INFO | MemInfo: { | 20260506 15:03:12.993 | INFO | "gpu" : " 17 GB", | 20260506 15:03:12.993 | INFO | "huge" : "", | 20260506 15:03:12.993 | INFO | "huge_shared" : " 27 GB", | 20260506 15:03:12.993 | INFO | "malloc" : " 14 GB", | 20260506 15:03:12.993 | INFO | "pinned" : " 29 GB" | 20260506 15:03:12.993 | INFO | } | 20260506 15:03:12.994 | INFO | Perf report for the last 5.13914 seconds: | 20260506 15:03:12.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:03:12.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:12.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:12.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:12.994 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:03:12.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:12.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:12.994 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 15:03:12.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:12.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:12.994 | INFO | Uploaded 36GiB to GPU in 2.02233s (17.8013GiB/s) | 20260506 15:03:12.994 | INFO | Downloaded 34GiB to GPU in 1.97944s (17.1766GiB/s) | 20260506 15:03:13.103 | INFO | Starting a round of Baz IO | 20260506 15:03:16.263 | INFO | Populating RTMetrics report | 20260506 15:03:16.263 | INFO | RTMetrics report populated | 20260506 15:03:16.263 | INFO | Opening tmp RTMetrics json file | 20260506 15:03:16.324 | INFO | Serializing RTMetrics to json | 20260506 15:03:16.491 | INFO | Serialization done, swapping tmp file now... | 20260506 15:03:16.565 | INFO | RTMetrics done | 20260506 15:03:16.819 | INFO | Finished a round of Baz IO: Wrote 2.104 GiB in 3.716 seconds | 20260506 15:03:16.819 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.61% / 0.01% / 62.38% | 20260506 15:03:16.819 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.23489% / 7.94729e-06% / 6.35783e-05% | 20260506 15:03:16.819 | INFO | Baz IO overall: | 20260506 15:03:16.819 | INFO | BytesWritten : 1185145774528 | 20260506 15:03:16.819 | INFO | NumEvents : 573547812289 | 20260506 15:03:18.132 | INFO | Analyzing chunk frames = [2281472,2281984) | 20260506 15:03:18.132 | INFO | MemInfo: { | 20260506 15:03:18.132 | INFO | "gpu" : " 17 GB", | 20260506 15:03:18.132 | INFO | "huge" : "", | 20260506 15:03:18.132 | INFO | "huge_shared" : " 27 GB", | 20260506 15:03:18.132 | INFO | "malloc" : " 14 GB", | 20260506 15:03:18.132 | INFO | "pinned" : " 29 GB" | 20260506 15:03:18.133 | INFO | } | 20260506 15:03:18.133 | INFO | Perf report for the last 5.13904 seconds: | 20260506 15:03:18.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:03:18.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:18.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:18.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:18.133 | INFO | ANALYSIS : 63.0%, 2.9, 95.4ms, 0 | 20260506 15:03:18.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:18.133 | INFO | RT_METRICS : 49.7%, 1.0, 26.1ms, 0 | 20260506 15:03:18.133 | INFO | PRE_HQ : 61.2%, 1.9, 59.8ms, 0 | 20260506 15:03:18.133 | INFO | BAZWRITER : 71.7%, 4.8, 177.3ms, 0 | 20260506 15:03:18.133 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 15:03:18.133 | INFO | Uploaded 36GiB to GPU in 2.08998s (17.2251GiB/s) | 20260506 15:03:18.133 | INFO | Downloaded 38GiB to GPU in 2.22624s (17.0691GiB/s) | 20260506 15:03:23.272 | INFO | Analyzing chunk frames = [2281984,2282496) | 20260506 15:03:23.272 | INFO | MemInfo: { | 20260506 15:03:23.272 | INFO | "gpu" : " 17 GB", | 20260506 15:03:23.272 | INFO | "huge" : "", | 20260506 15:03:23.272 | INFO | "huge_shared" : " 27 GB", | 20260506 15:03:23.272 | INFO | "malloc" : " 14 GB", | 20260506 15:03:23.272 | INFO | "pinned" : " 29 GB" | 20260506 15:03:23.272 | INFO | } | 20260506 15:03:23.272 | INFO | Perf report for the last 5.13937 seconds: | 20260506 15:03:23.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:03:23.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:23.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:23.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:23.272 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 15:03:23.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:23.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:23.272 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:03:23.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:23.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:23.272 | INFO | Uploaded 36GiB to GPU in 1.99477s (18.0472GiB/s) | 20260506 15:03:23.272 | INFO | Downloaded 34GiB to GPU in 1.94256s (17.5027GiB/s) | 20260506 15:03:28.411 | INFO | Analyzing chunk frames = [2282496,2283008) | 20260506 15:03:28.411 | INFO | MemInfo: { | 20260506 15:03:28.411 | INFO | "gpu" : " 17 GB", | 20260506 15:03:28.411 | INFO | "huge" : "", | 20260506 15:03:28.411 | INFO | "huge_shared" : " 27 GB", | 20260506 15:03:28.411 | INFO | "malloc" : " 14 GB", | 20260506 15:03:28.411 | INFO | "pinned" : " 29 GB" | 20260506 15:03:28.411 | INFO | } | 20260506 15:03:28.411 | INFO | Perf report for the last 5.13896 seconds: | 20260506 15:03:28.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:03:28.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:28.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:28.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:28.411 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:03:28.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:28.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:28.411 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:03:28.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:28.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:28.411 | INFO | Uploaded 36GiB to GPU in 1.98799s (18.1088GiB/s) | 20260506 15:03:28.411 | INFO | Downloaded 34GiB to GPU in 1.93556s (17.566GiB/s) | 20260506 15:03:33.550 | INFO | Analyzing chunk frames = [2283008,2283520) | 20260506 15:03:33.550 | INFO | MemInfo: { | 20260506 15:03:33.550 | INFO | "gpu" : " 17 GB", | 20260506 15:03:33.550 | INFO | "huge" : "", | 20260506 15:03:33.550 | INFO | "huge_shared" : " 27 GB", | 20260506 15:03:33.550 | INFO | "malloc" : " 14 GB", | 20260506 15:03:33.550 | INFO | "pinned" : " 29 GB" | 20260506 15:03:33.550 | INFO | } | 20260506 15:03:33.550 | INFO | Perf report for the last 5.13894 seconds: | 20260506 15:03:33.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:03:33.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:33.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:33.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:33.550 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 15:03:33.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:33.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:33.550 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:03:33.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:33.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:33.550 | INFO | Uploaded 36GiB to GPU in 2.0235s (17.7909GiB/s) | 20260506 15:03:33.550 | INFO | Downloaded 34GiB to GPU in 1.91729s (17.7333GiB/s) | 20260506 15:03:38.689 | INFO | Analyzing chunk frames = [2283520,2284032) | 20260506 15:03:38.689 | INFO | MemInfo: { | 20260506 15:03:38.689 | INFO | "gpu" : " 17 GB", | 20260506 15:03:38.689 | INFO | "huge" : "", | 20260506 15:03:38.689 | INFO | "huge_shared" : " 27 GB", | 20260506 15:03:38.689 | INFO | "malloc" : " 14 GB", | 20260506 15:03:38.689 | INFO | "pinned" : " 29 GB" | 20260506 15:03:38.689 | INFO | } | 20260506 15:03:38.690 | INFO | Perf report for the last 5.13963 seconds: | 20260506 15:03:38.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:03:38.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:38.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:38.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:38.690 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 15:03:38.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:38.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:38.690 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:03:38.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:38.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:38.690 | INFO | Uploaded 36GiB to GPU in 2.0221s (17.8033GiB/s) | 20260506 15:03:38.690 | INFO | Downloaded 34GiB to GPU in 1.92644s (17.6492GiB/s) | 20260506 15:03:43.830 | INFO | Analyzing chunk frames = [2284032,2284544) | 20260506 15:03:43.830 | INFO | MemInfo: { | 20260506 15:03:43.830 | INFO | "gpu" : " 17 GB", | 20260506 15:03:43.830 | INFO | "huge" : "", | 20260506 15:03:43.830 | INFO | "huge_shared" : " 27 GB", | 20260506 15:03:43.830 | INFO | "malloc" : " 14 GB", | 20260506 15:03:43.830 | INFO | "pinned" : " 29 GB" | 20260506 15:03:43.830 | INFO | } | 20260506 15:03:43.830 | INFO | Perf report for the last 5.14054 seconds: | 20260506 15:03:43.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:03:43.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:43.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:43.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:43.830 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 15:03:43.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:43.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:43.830 | INFO | PRE_HQ : 32.1%, 1.0, 16.6ms, 0 | 20260506 15:03:43.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:43.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:43.830 | INFO | Uploaded 36GiB to GPU in 2.06198s (17.459GiB/s) | 20260506 15:03:43.830 | INFO | Downloaded 34GiB to GPU in 1.90402s (17.857GiB/s) | 20260506 15:03:48.970 | INFO | Analyzing chunk frames = [2284544,2285056) | 20260506 15:03:48.970 | INFO | MemInfo: { | 20260506 15:03:48.970 | INFO | "gpu" : " 17 GB", | 20260506 15:03:48.970 | INFO | "huge" : "", | 20260506 15:03:48.970 | INFO | "huge_shared" : " 27 GB", | 20260506 15:03:48.970 | INFO | "malloc" : " 14 GB", | 20260506 15:03:48.970 | INFO | "pinned" : " 29 GB" | 20260506 15:03:48.970 | INFO | } | 20260506 15:03:48.970 | INFO | Perf report for the last 5.14008 seconds: | 20260506 15:03:48.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:03:48.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:48.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:48.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:48.970 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 15:03:48.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:48.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:48.970 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 15:03:48.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:48.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:48.971 | INFO | Uploaded 36GiB to GPU in 2.0085s (17.9238GiB/s) | 20260506 15:03:48.971 | INFO | Downloaded 34GiB to GPU in 2.00645s (16.9454GiB/s) | 20260506 15:03:54.108 | INFO | Analyzing chunk frames = [2285056,2285568) | 20260506 15:03:54.108 | INFO | MemInfo: { | 20260506 15:03:54.108 | INFO | "gpu" : " 17 GB", | 20260506 15:03:54.108 | INFO | "huge" : "", | 20260506 15:03:54.108 | INFO | "huge_shared" : " 27 GB", | 20260506 15:03:54.108 | INFO | "malloc" : " 14 GB", | 20260506 15:03:54.108 | INFO | "pinned" : " 29 GB" | 20260506 15:03:54.108 | INFO | } | 20260506 15:03:54.108 | INFO | Perf report for the last 5.13798 seconds: | 20260506 15:03:54.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:03:54.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:54.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:54.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:54.108 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 15:03:54.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:54.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:54.108 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:03:54.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:54.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:54.108 | INFO | Uploaded 36GiB to GPU in 2.02031s (17.819GiB/s) | 20260506 15:03:54.109 | INFO | Downloaded 34GiB to GPU in 1.98276s (17.1478GiB/s) | 20260506 15:03:54.109 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:03:54.109 | INFO | Total mem usage: 17.4 GiB | 20260506 15:03:54.109 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:03:54.109 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -9.1%) | 20260506 15:03:54.109 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:03:54.109 | INFO | High Water Breakdown: | 20260506 15:03:54.109 | INFO | 61.6% : 9.82 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:03:54.109 | INFO | 38.3% : 6.11 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:03:54.109 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:03:54.109 | INFO | ------------------------------------------ | 20260506 15:03:54.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:03:54.109 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:03:54.109 | INFO | Total mem usage: 26.5 GiB | 20260506 15:03:54.109 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:03:54.109 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:03:54.109 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:03:54.109 | INFO | High Water Breakdown: | 20260506 15:03:54.109 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:03:54.109 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:03:54.109 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:03:54.109 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:03:54.109 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:03:54.109 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:03:54.109 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:03:54.109 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:03:54.109 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:03:54.109 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:03:54.109 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:03:54.109 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:03:54.109 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:03:54.109 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:03:54.109 | INFO | ------------------------------------------ | 20260506 15:03:54.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:03:54.109 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:03:54.109 | INFO | Total mem usage: 25.6 GiB | 20260506 15:03:54.109 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:03:54.109 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:03:54.109 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:03:54.109 | INFO | High Water Breakdown: | 20260506 15:03:54.109 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:03:54.109 | INFO | ------------------------------------------ | 20260506 15:03:54.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:03:54.109 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:03:54.109 | INFO | Total mem usage: 15.7 GiB | 20260506 15:03:54.109 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:03:54.109 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:03:54.109 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 15:03:54.109 | INFO | High Water Breakdown: | 20260506 15:03:54.109 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:03:54.109 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:03:54.109 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:03:54.109 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:03:54.109 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:03:54.109 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:03:54.110 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:03:54.110 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:03:54.110 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:03:54.110 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:03:54.110 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:03:54.110 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:03:54.110 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:03:54.110 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:03:54.110 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:03:54.110 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:03:54.110 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:03:54.110 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:03:54.110 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:03:54.110 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:03:54.110 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:03:54.110 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:03:54.110 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:03:54.110 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:03:54.110 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:03:54.110 | INFO | ------------------------------------------ | 20260506 15:03:54.110 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:03:54.110 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:03:54.110 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:03:54.110 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:03:54.110 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.57e+01, 1.35e+01 / 2.08e+00 | 20260506 15:03:54.110 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.01e-01, 1.11e-02 / 6.28e-03 | 20260506 15:03:54.110 | INFO | PerfReport: 11.58%, Baseline -- 3.65e+00 / 4.46e+00, 5.69e-01 / 3.34e-01 | 20260506 15:03:54.110 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.48e+00, 1.89e-01 / 1.38e-01 | 20260506 15:03:54.110 | INFO | PerfReport: 15.87%, DME -- 4.00e+01 / 4.42e+01, 3.88e+00 / 2.43e+00 | 20260506 15:03:54.110 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.48e+00 / 2.23e+00 | 20260506 15:03:54.110 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.33e+00 / 2.94e+00, 4.05e-01 / 2.65e-01 | 20260506 15:03:54.110 | INFO | PerfReport: 3.57%, Metrics -- 1.12e+00 / 1.50e+00, 2.89e-01 / 1.70e-01 | 20260506 15:03:54.110 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.92e-01 / 4.45e-01, 6.17e-02 / 2.78e-02 | 20260506 15:03:54.110 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:03:54.110 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:03:54.110 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:03:54.110 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 15:03:54.110 | INFO | PerfReport: 21.53%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 15:03:54.110 | INFO | PerfReport: 21.16%, Download -- 2.07e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 15:03:54.110 | INFO | PerfReport: 32.23%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 15:03:54.110 | INFO | PerfReport: 1.50%, LockUpload -- 1.47e+00 / 0.00e+00, 5.85e+00 / 0.00e+00 | 20260506 15:03:54.110 | INFO | PerfReport: 1.13%, LockDownload -- 1.11e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260506 15:03:54.110 | INFO | PerfReport: 22.42%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 15:03:54.110 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.12e-02 / 0.00e+00 | 20260506 15:03:54.227 | INFO | Starting a round of Baz IO | 20260506 15:03:57.377 | INFO | Populating RTMetrics report | 20260506 15:03:57.377 | INFO | RTMetrics report populated | 20260506 15:03:57.377 | INFO | Opening tmp RTMetrics json file | 20260506 15:03:57.432 | INFO | Serializing RTMetrics to json | 20260506 15:03:57.436 | INFO | Serialization done, swapping tmp file now... | 20260506 15:03:57.440 | INFO | RTMetrics done | 20260506 15:03:57.599 | INFO | Finished a round of Baz IO: Wrote 2.104 GiB in 3.371 seconds | 20260506 15:03:57.599 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.60% / 0.01% / 62.39% | 20260506 15:03:57.599 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.23143% / 1.19209e-05% / 5.96046e-05% | 20260506 15:03:57.599 | INFO | Baz IO overall: | 20260506 15:03:57.599 | INFO | BytesWritten : 1187405433164 | 20260506 15:03:57.599 | INFO | NumEvents : 574641434665 | 20260506 15:03:59.247 | INFO | Analyzing chunk frames = [2285568,2286080) | 20260506 15:03:59.247 | INFO | MemInfo: { | 20260506 15:03:59.247 | INFO | "gpu" : " 17 GB", | 20260506 15:03:59.247 | INFO | "huge" : "", | 20260506 15:03:59.247 | INFO | "huge_shared" : " 27 GB", | 20260506 15:03:59.247 | INFO | "malloc" : " 14 GB", | 20260506 15:03:59.247 | INFO | "pinned" : " 29 GB" | 20260506 15:03:59.247 | INFO | } | 20260506 15:03:59.247 | INFO | Perf report for the last 5.13866 seconds: | 20260506 15:03:59.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:03:59.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:59.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:59.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:59.247 | INFO | ANALYSIS : 63.0%, 2.9, 95.1ms, 0 | 20260506 15:03:59.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:03:59.247 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 15:03:59.247 | INFO | PRE_HQ : 60.8%, 1.9, 59.9ms, 0 | 20260506 15:03:59.247 | INFO | BAZWRITER : 65.2%, 4.8, 159.6ms, 0 | 20260506 15:03:59.247 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 15:03:59.247 | INFO | Uploaded 36GiB to GPU in 2.07878s (17.3179GiB/s) | 20260506 15:03:59.247 | INFO | Downloaded 38GiB to GPU in 2.23198s (17.0252GiB/s) | 20260506 15:04:04.386 | INFO | Analyzing chunk frames = [2286080,2286592) | 20260506 15:04:04.386 | INFO | MemInfo: { | 20260506 15:04:04.386 | INFO | "gpu" : " 17 GB", | 20260506 15:04:04.386 | INFO | "huge" : "", | 20260506 15:04:04.386 | INFO | "huge_shared" : " 27 GB", | 20260506 15:04:04.386 | INFO | "malloc" : " 14 GB", | 20260506 15:04:04.386 | INFO | "pinned" : " 29 GB" | 20260506 15:04:04.386 | INFO | } | 20260506 15:04:04.386 | INFO | Perf report for the last 5.1396 seconds: | 20260506 15:04:04.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:04:04.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:04.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:04.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:04.386 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 15:04:04.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:04.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:04.387 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:04:04.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:04.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:04.387 | INFO | Uploaded 36GiB to GPU in 1.99444s (18.0502GiB/s) | 20260506 15:04:04.387 | INFO | Downloaded 34GiB to GPU in 1.94291s (17.4995GiB/s) | 20260506 15:04:09.528 | INFO | Analyzing chunk frames = [2286592,2287104) | 20260506 15:04:09.528 | INFO | MemInfo: { | 20260506 15:04:09.528 | INFO | "gpu" : " 17 GB", | 20260506 15:04:09.528 | INFO | "huge" : "", | 20260506 15:04:09.528 | INFO | "huge_shared" : " 27 GB", | 20260506 15:04:09.528 | INFO | "malloc" : " 14 GB", | 20260506 15:04:09.528 | INFO | "pinned" : " 29 GB" | 20260506 15:04:09.528 | INFO | } | 20260506 15:04:09.528 | INFO | Perf report for the last 5.14139 seconds: | 20260506 15:04:09.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:04:09.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:09.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:09.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:09.528 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 15:04:09.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:09.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:09.528 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:04:09.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:09.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:09.528 | INFO | Uploaded 36GiB to GPU in 1.99917s (18.0075GiB/s) | 20260506 15:04:09.528 | INFO | Downloaded 34GiB to GPU in 1.93977s (17.5278GiB/s) | 20260506 15:04:14.666 | INFO | Analyzing chunk frames = [2287104,2287616) | 20260506 15:04:14.666 | INFO | MemInfo: { | 20260506 15:04:14.666 | INFO | "gpu" : " 17 GB", | 20260506 15:04:14.666 | INFO | "huge" : "", | 20260506 15:04:14.666 | INFO | "huge_shared" : " 27 GB", | 20260506 15:04:14.666 | INFO | "malloc" : " 14 GB", | 20260506 15:04:14.666 | INFO | "pinned" : " 29 GB" | 20260506 15:04:14.666 | INFO | } | 20260506 15:04:14.666 | INFO | Perf report for the last 5.13817 seconds: | 20260506 15:04:14.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:04:14.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:14.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:14.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:14.666 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 15:04:14.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:14.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:14.666 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:04:14.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:14.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:14.666 | INFO | Uploaded 36GiB to GPU in 2.02417s (17.785GiB/s) | 20260506 15:04:14.666 | INFO | Downloaded 34GiB to GPU in 1.91323s (17.771GiB/s) | 20260506 15:04:19.806 | INFO | Analyzing chunk frames = [2287616,2288128) | 20260506 15:04:19.806 | INFO | MemInfo: { | 20260506 15:04:19.806 | INFO | "gpu" : " 17 GB", | 20260506 15:04:19.806 | INFO | "huge" : "", | 20260506 15:04:19.806 | INFO | "huge_shared" : " 27 GB", | 20260506 15:04:19.806 | INFO | "malloc" : " 14 GB", | 20260506 15:04:19.806 | INFO | "pinned" : " 29 GB" | 20260506 15:04:19.806 | INFO | } | 20260506 15:04:19.806 | INFO | Perf report for the last 5.1402 seconds: | 20260506 15:04:19.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:04:19.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:19.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:19.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:19.806 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 15:04:19.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:19.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:19.806 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:04:19.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:19.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:19.807 | INFO | Uploaded 36GiB to GPU in 1.9994s (18.0054GiB/s) | 20260506 15:04:19.807 | INFO | Downloaded 34GiB to GPU in 1.93137s (17.604GiB/s) | 20260506 15:04:24.944 | INFO | Analyzing chunk frames = [2288128,2288640) | 20260506 15:04:24.944 | INFO | MemInfo: { | 20260506 15:04:24.944 | INFO | "gpu" : " 17 GB", | 20260506 15:04:24.944 | INFO | "huge" : "", | 20260506 15:04:24.944 | INFO | "huge_shared" : " 27 GB", | 20260506 15:04:24.944 | INFO | "malloc" : " 14 GB", | 20260506 15:04:24.944 | INFO | "pinned" : " 29 GB" | 20260506 15:04:24.944 | INFO | } | 20260506 15:04:24.944 | INFO | Perf report for the last 5.13764 seconds: | 20260506 15:04:24.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:04:24.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:24.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:24.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:24.944 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 15:04:24.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:24.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:24.944 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 15:04:24.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:24.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:24.944 | INFO | Uploaded 36GiB to GPU in 2.04716s (17.5853GiB/s) | 20260506 15:04:24.944 | INFO | Downloaded 34GiB to GPU in 1.93033s (17.6136GiB/s) | 20260506 15:04:30.083 | INFO | Analyzing chunk frames = [2288640,2289152) | 20260506 15:04:30.083 | INFO | MemInfo: { | 20260506 15:04:30.083 | INFO | "gpu" : " 17 GB", | 20260506 15:04:30.083 | INFO | "huge" : "", | 20260506 15:04:30.083 | INFO | "huge_shared" : " 27 GB", | 20260506 15:04:30.083 | INFO | "malloc" : " 14 GB", | 20260506 15:04:30.083 | INFO | "pinned" : " 29 GB" | 20260506 15:04:30.083 | INFO | } | 20260506 15:04:30.084 | INFO | Perf report for the last 5.13972 seconds: | 20260506 15:04:30.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:04:30.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:30.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:30.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:30.084 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 15:04:30.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:30.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:30.084 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 15:04:30.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:30.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:30.084 | INFO | Uploaded 36GiB to GPU in 2.01282s (17.8854GiB/s) | 20260506 15:04:30.084 | INFO | Downloaded 34GiB to GPU in 1.99456s (17.0464GiB/s) | 20260506 15:04:35.222 | INFO | Analyzing chunk frames = [2289152,2289664) | 20260506 15:04:35.222 | INFO | MemInfo: { | 20260506 15:04:35.222 | INFO | "gpu" : " 17 GB", | 20260506 15:04:35.222 | INFO | "huge" : "", | 20260506 15:04:35.222 | INFO | "huge_shared" : " 27 GB", | 20260506 15:04:35.222 | INFO | "malloc" : " 14 GB", | 20260506 15:04:35.222 | INFO | "pinned" : " 29 GB" | 20260506 15:04:35.222 | INFO | } | 20260506 15:04:35.222 | INFO | Perf report for the last 5.13851 seconds: | 20260506 15:04:35.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:04:35.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:35.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:35.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:35.222 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 15:04:35.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:35.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:35.222 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 15:04:35.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:35.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:35.223 | INFO | Uploaded 36GiB to GPU in 2.01234s (17.8897GiB/s) | 20260506 15:04:35.223 | INFO | Downloaded 34GiB to GPU in 1.98496s (17.1288GiB/s) | 20260506 15:04:35.343 | INFO | Starting a round of Baz IO | 20260506 15:04:38.495 | INFO | Populating RTMetrics report | 20260506 15:04:38.496 | INFO | RTMetrics report populated | 20260506 15:04:38.496 | INFO | Opening tmp RTMetrics json file | 20260506 15:04:38.497 | INFO | Serializing RTMetrics to json | 20260506 15:04:38.499 | INFO | Serialization done, swapping tmp file now... | 20260506 15:04:38.500 | INFO | RTMetrics done | 20260506 15:04:38.867 | INFO | Finished a round of Baz IO: Wrote 2.106 GiB in 3.523 seconds | 20260506 15:04:38.867 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.58% / 0.01% / 62.40% | 20260506 15:04:38.867 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.2301% / 1.19209e-05% / 7.54992e-05% | 20260506 15:04:38.867 | INFO | Baz IO overall: | 20260506 15:04:38.867 | INFO | BytesWritten : 1189666859994 | 20260506 15:04:38.867 | INFO | NumEvents : 575736213871 | 20260506 15:04:40.361 | INFO | Analyzing chunk frames = [2289664,2290176) | 20260506 15:04:40.361 | INFO | MemInfo: { | 20260506 15:04:40.362 | INFO | "gpu" : " 17 GB", | 20260506 15:04:40.362 | INFO | "huge" : "", | 20260506 15:04:40.362 | INFO | "huge_shared" : " 27 GB", | 20260506 15:04:40.362 | INFO | "malloc" : " 14 GB", | 20260506 15:04:40.362 | INFO | "pinned" : " 29 GB" | 20260506 15:04:40.362 | INFO | } | 20260506 15:04:40.362 | INFO | Perf report for the last 5.1395 seconds: | 20260506 15:04:40.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:04:40.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:40.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:40.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:40.362 | INFO | ANALYSIS : 63.1%, 2.9, 95.7ms, 0 | 20260506 15:04:40.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:40.362 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 15:04:40.362 | INFO | PRE_HQ : 61.0%, 1.9, 59.4ms, 0 | 20260506 15:04:40.362 | INFO | BAZWRITER : 68.6%, 4.8, 167.5ms, 0 | 20260506 15:04:40.362 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 15:04:40.362 | INFO | Uploaded 36GiB to GPU in 2.06596s (17.4253GiB/s) | 20260506 15:04:40.362 | INFO | Downloaded 38GiB to GPU in 2.23313s (17.0165GiB/s) | 20260506 15:04:45.501 | INFO | Analyzing chunk frames = [2290176,2290688) | 20260506 15:04:45.501 | INFO | MemInfo: { | 20260506 15:04:45.501 | INFO | "gpu" : " 17 GB", | 20260506 15:04:45.501 | INFO | "huge" : "", | 20260506 15:04:45.501 | INFO | "huge_shared" : " 27 GB", | 20260506 15:04:45.501 | INFO | "malloc" : " 14 GB", | 20260506 15:04:45.501 | INFO | "pinned" : " 29 GB" | 20260506 15:04:45.501 | INFO | } | 20260506 15:04:45.501 | INFO | Perf report for the last 5.13947 seconds: | 20260506 15:04:45.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:04:45.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:45.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:45.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:45.501 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:04:45.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:45.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:45.501 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 15:04:45.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:45.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:45.501 | INFO | Uploaded 36GiB to GPU in 1.9892s (18.0977GiB/s) | 20260506 15:04:45.502 | INFO | Downloaded 34GiB to GPU in 1.95099s (17.4271GiB/s) | 20260506 15:04:50.641 | INFO | Analyzing chunk frames = [2290688,2291200) | 20260506 15:04:50.641 | INFO | MemInfo: { | 20260506 15:04:50.641 | INFO | "gpu" : " 17 GB", | 20260506 15:04:50.641 | INFO | "huge" : "", | 20260506 15:04:50.641 | INFO | "huge_shared" : " 27 GB", | 20260506 15:04:50.641 | INFO | "malloc" : " 14 GB", | 20260506 15:04:50.641 | INFO | "pinned" : " 29 GB" | 20260506 15:04:50.641 | INFO | } | 20260506 15:04:50.641 | INFO | Perf report for the last 5.13977 seconds: | 20260506 15:04:50.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:04:50.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:50.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:50.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:50.641 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 15:04:50.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:50.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:50.641 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 15:04:50.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:50.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:50.641 | INFO | Uploaded 36GiB to GPU in 1.99048s (18.086GiB/s) | 20260506 15:04:50.641 | INFO | Downloaded 34GiB to GPU in 1.944s (17.4898GiB/s) | 20260506 15:04:55.781 | INFO | Analyzing chunk frames = [2291200,2291712) | 20260506 15:04:55.781 | INFO | MemInfo: { | 20260506 15:04:55.781 | INFO | "gpu" : " 17 GB", | 20260506 15:04:55.781 | INFO | "huge" : "", | 20260506 15:04:55.781 | INFO | "huge_shared" : " 27 GB", | 20260506 15:04:55.781 | INFO | "malloc" : " 14 GB", | 20260506 15:04:55.781 | INFO | "pinned" : " 29 GB" | 20260506 15:04:55.781 | INFO | } | 20260506 15:04:55.781 | INFO | Perf report for the last 5.13967 seconds: | 20260506 15:04:55.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:04:55.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:55.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:55.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:55.781 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:04:55.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:55.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:55.781 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:04:55.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:55.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:04:55.781 | INFO | Uploaded 36GiB to GPU in 2.02267s (17.7982GiB/s) | 20260506 15:04:55.781 | INFO | Downloaded 34GiB to GPU in 1.92081s (17.7009GiB/s) | 20260506 15:05:00.920 | INFO | Analyzing chunk frames = [2291712,2292224) | 20260506 15:05:00.920 | INFO | MemInfo: { | 20260506 15:05:00.920 | INFO | "gpu" : " 17 GB", | 20260506 15:05:00.920 | INFO | "huge" : "", | 20260506 15:05:00.920 | INFO | "huge_shared" : " 27 GB", | 20260506 15:05:00.920 | INFO | "malloc" : " 14 GB", | 20260506 15:05:00.920 | INFO | "pinned" : " 29 GB" | 20260506 15:05:00.920 | INFO | } | 20260506 15:05:00.920 | INFO | Perf report for the last 5.13937 seconds: | 20260506 15:05:00.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:05:00.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:00.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:00.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:00.920 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 15:05:00.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:00.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:00.920 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:05:00.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:00.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:00.920 | INFO | Uploaded 36GiB to GPU in 1.96637s (18.3078GiB/s) | 20260506 15:05:00.920 | INFO | Downloaded 34GiB to GPU in 1.93767s (17.5468GiB/s) | 20260506 15:05:06.058 | INFO | Analyzing chunk frames = [2292224,2292736) | 20260506 15:05:06.058 | INFO | MemInfo: { | 20260506 15:05:06.058 | INFO | "gpu" : " 17 GB", | 20260506 15:05:06.058 | INFO | "huge" : "", | 20260506 15:05:06.058 | INFO | "huge_shared" : " 27 GB", | 20260506 15:05:06.058 | INFO | "malloc" : " 14 GB", | 20260506 15:05:06.058 | INFO | "pinned" : " 29 GB" | 20260506 15:05:06.058 | INFO | } | 20260506 15:05:06.058 | INFO | Perf report for the last 5.1384 seconds: | 20260506 15:05:06.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:05:06.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:06.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:06.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:06.059 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 15:05:06.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:06.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:06.059 | INFO | PRE_HQ : 32.6%, 1.0, 16.9ms, 0 | 20260506 15:05:06.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:06.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:06.059 | INFO | Uploaded 36GiB to GPU in 2.02756s (17.7554GiB/s) | 20260506 15:05:06.059 | INFO | Downloaded 34GiB to GPU in 1.93801s (17.5438GiB/s) | 20260506 15:05:11.199 | INFO | Analyzing chunk frames = [2292736,2293248) | 20260506 15:05:11.199 | INFO | MemInfo: { | 20260506 15:05:11.199 | INFO | "gpu" : " 17 GB", | 20260506 15:05:11.199 | INFO | "huge" : "", | 20260506 15:05:11.199 | INFO | "huge_shared" : " 27 GB", | 20260506 15:05:11.199 | INFO | "malloc" : " 14 GB", | 20260506 15:05:11.199 | INFO | "pinned" : " 29 GB" | 20260506 15:05:11.199 | INFO | } | 20260506 15:05:11.199 | INFO | Perf report for the last 5.14021 seconds: | 20260506 15:05:11.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:05:11.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:11.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:11.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:11.199 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:05:11.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:11.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:11.199 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 15:05:11.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:11.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:11.199 | INFO | Uploaded 36GiB to GPU in 1.99449s (18.0498GiB/s) | 20260506 15:05:11.199 | INFO | Downloaded 34GiB to GPU in 1.99147s (17.0728GiB/s) | 20260506 15:05:16.337 | INFO | Analyzing chunk frames = [2293248,2293760) | 20260506 15:05:16.337 | INFO | MemInfo: { | 20260506 15:05:16.337 | INFO | "gpu" : " 17 GB", | 20260506 15:05:16.337 | INFO | "huge" : "", | 20260506 15:05:16.337 | INFO | "huge_shared" : " 27 GB", | 20260506 15:05:16.337 | INFO | "malloc" : " 14 GB", | 20260506 15:05:16.337 | INFO | "pinned" : " 29 GB" | 20260506 15:05:16.337 | INFO | } | 20260506 15:05:16.337 | INFO | Perf report for the last 5.13878 seconds: | 20260506 15:05:16.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:05:16.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:16.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:16.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:16.338 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:05:16.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:16.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:16.338 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 15:05:16.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:16.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:16.338 | INFO | Uploaded 36GiB to GPU in 2.00128s (17.9884GiB/s) | 20260506 15:05:16.338 | INFO | Downloaded 34GiB to GPU in 1.98368s (17.1399GiB/s) | 20260506 15:05:16.338 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:05:16.338 | INFO | Total mem usage: 17.4 GiB | 20260506 15:05:16.338 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:05:16.338 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -15.4%) | 20260506 15:05:16.338 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:05:16.338 | INFO | High Water Breakdown: | 20260506 15:05:16.338 | INFO | 60.7% : 9.16 GiB ( 83.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:05:16.338 | INFO | 39.2% : 5.91 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:05:16.338 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:05:16.338 | INFO | ------------------------------------------ | 20260506 15:05:16.338 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:05:16.338 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:05:16.338 | INFO | Total mem usage: 26.5 GiB | 20260506 15:05:16.338 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:05:16.338 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:05:16.338 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:05:16.338 | INFO | High Water Breakdown: | 20260506 15:05:16.338 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:05:16.338 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:05:16.338 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:05:16.338 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:05:16.338 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:05:16.338 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:05:16.338 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:05:16.338 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:05:16.338 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:05:16.338 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:05:16.338 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:05:16.338 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:05:16.338 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:05:16.338 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:05:16.338 | INFO | ------------------------------------------ | 20260506 15:05:16.338 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:05:16.338 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:05:16.338 | INFO | Total mem usage: 25.6 GiB | 20260506 15:05:16.338 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:05:16.339 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:05:16.339 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:05:16.339 | INFO | High Water Breakdown: | 20260506 15:05:16.339 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:05:16.339 | INFO | ------------------------------------------ | 20260506 15:05:16.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:05:16.339 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:05:16.339 | INFO | Total mem usage: 15.7 GiB | 20260506 15:05:16.339 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:05:16.339 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:05:16.339 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 15:05:16.339 | INFO | High Water Breakdown: | 20260506 15:05:16.339 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:05:16.339 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:05:16.339 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:05:16.339 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:05:16.339 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:05:16.339 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:05:16.339 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:05:16.339 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:05:16.339 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:05:16.339 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:05:16.339 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:05:16.339 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:05:16.339 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:05:16.339 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:05:16.339 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:05:16.339 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:05:16.339 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:05:16.339 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:05:16.339 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:05:16.339 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:05:16.339 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:05:16.339 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:05:16.339 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:05:16.339 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:05:16.339 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:05:16.339 | INFO | ------------------------------------------ | 20260506 15:05:16.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:05:16.339 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:05:16.339 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:05:16.339 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:05:16.339 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.60e+01, 1.36e+01 / 2.15e+00 | 20260506 15:05:16.339 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.03e-01, 1.09e-02 / 1.04e-02 | 20260506 15:05:16.339 | INFO | PerfReport: 11.59%, Baseline -- 3.65e+00 / 4.42e+00, 5.70e-01 / 4.27e-01 | 20260506 15:05:16.339 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.87e-01 / 1.45e-01 | 20260506 15:05:16.339 | INFO | PerfReport: 15.84%, DME -- 3.99e+01 / 4.44e+01, 4.10e+00 / 2.39e+00 | 20260506 15:05:16.339 | INFO | PerfReport: 56.85%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.52e+00 / 1.84e+00 | 20260506 15:05:16.339 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.32e+00 / 2.89e+00, 3.96e-01 / 2.64e-01 | 20260506 15:05:16.339 | INFO | PerfReport: 3.56%, Metrics -- 1.12e+00 / 1.50e+00, 2.92e-01 / 1.85e-01 | 20260506 15:05:16.339 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.92e-01 / 4.45e-01, 6.17e-02 / 2.88e-02 | 20260506 15:05:16.339 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:05:16.339 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:05:16.339 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:05:16.339 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 15:05:16.339 | INFO | PerfReport: 21.48%, Upload -- 2.10e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 15:05:16.339 | INFO | PerfReport: 21.18%, Download -- 2.07e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 15:05:16.339 | INFO | PerfReport: 32.28%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 15:05:16.339 | INFO | PerfReport: 1.51%, LockUpload -- 1.47e+00 / 0.00e+00, 5.82e+00 / 0.00e+00 | 20260506 15:05:16.339 | INFO | PerfReport: 1.14%, LockDownload -- 1.11e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260506 15:05:16.339 | INFO | PerfReport: 22.40%, LockCompute -- 2.19e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260506 15:05:16.339 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260506 15:05:16.450 | INFO | Starting a round of Baz IO | 20260506 15:05:19.606 | INFO | Populating RTMetrics report | 20260506 15:05:19.607 | INFO | RTMetrics report populated | 20260506 15:05:19.607 | INFO | Opening tmp RTMetrics json file | 20260506 15:05:19.629 | INFO | Serializing RTMetrics to json | 20260506 15:05:19.633 | INFO | Serialization done, swapping tmp file now... | 20260506 15:05:19.634 | INFO | RTMetrics done | 20260506 15:05:19.801 | INFO | Finished a round of Baz IO: Wrote 2.107 GiB in 3.35 seconds | 20260506 15:05:19.801 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.57% / 0.01% / 62.42% | 20260506 15:05:19.801 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.2286% / 2.38419e-05% / 6.35783e-05% | 20260506 15:05:19.801 | INFO | Baz IO overall: | 20260506 15:05:19.801 | INFO | BytesWritten : 1191928957784 | 20260506 15:05:19.801 | INFO | NumEvents : 576831522937 | 20260506 15:05:21.478 | INFO | Analyzing chunk frames = [2293760,2294272) | 20260506 15:05:21.478 | INFO | MemInfo: { | 20260506 15:05:21.478 | INFO | "gpu" : " 17 GB", | 20260506 15:05:21.478 | INFO | "huge" : "", | 20260506 15:05:21.478 | INFO | "huge_shared" : " 27 GB", | 20260506 15:05:21.478 | INFO | "malloc" : " 14 GB", | 20260506 15:05:21.478 | INFO | "pinned" : " 29 GB" | 20260506 15:05:21.478 | INFO | } | 20260506 15:05:21.478 | INFO | Perf report for the last 5.14041 seconds: | 20260506 15:05:21.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:05:21.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:21.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:21.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:21.478 | INFO | ANALYSIS : 63.0%, 2.9, 95.5ms, 0 | 20260506 15:05:21.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:21.478 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 15:05:21.478 | INFO | PRE_HQ : 61.0%, 1.9, 59.2ms, 0 | 20260506 15:05:21.478 | INFO | BAZWRITER : 64.7%, 4.8, 160.4ms, 0 | 20260506 15:05:21.478 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 15:05:21.478 | INFO | Uploaded 36GiB to GPU in 2.04257s (17.6249GiB/s) | 20260506 15:05:21.478 | INFO | Downloaded 38GiB to GPU in 2.22002s (17.117GiB/s) | 20260506 15:05:26.618 | INFO | Analyzing chunk frames = [2294272,2294784) | 20260506 15:05:26.618 | INFO | MemInfo: { | 20260506 15:05:26.618 | INFO | "gpu" : " 17 GB", | 20260506 15:05:26.618 | INFO | "huge" : "", | 20260506 15:05:26.618 | INFO | "huge_shared" : " 27 GB", | 20260506 15:05:26.618 | INFO | "malloc" : " 14 GB", | 20260506 15:05:26.618 | INFO | "pinned" : " 29 GB" | 20260506 15:05:26.618 | INFO | } | 20260506 15:05:26.618 | INFO | Perf report for the last 5.1399 seconds: | 20260506 15:05:26.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:05:26.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:26.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:26.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:26.618 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:05:26.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:26.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:26.618 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:05:26.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:26.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:26.618 | INFO | Uploaded 36GiB to GPU in 1.98908s (18.0988GiB/s) | 20260506 15:05:26.618 | INFO | Downloaded 34GiB to GPU in 1.93362s (17.5836GiB/s) | 20260506 15:05:31.756 | INFO | Analyzing chunk frames = [2294784,2295296) | 20260506 15:05:31.756 | INFO | MemInfo: { | 20260506 15:05:31.756 | INFO | "gpu" : " 17 GB", | 20260506 15:05:31.756 | INFO | "huge" : "", | 20260506 15:05:31.756 | INFO | "huge_shared" : " 27 GB", | 20260506 15:05:31.756 | INFO | "malloc" : " 14 GB", | 20260506 15:05:31.756 | INFO | "pinned" : " 29 GB" | 20260506 15:05:31.756 | INFO | } | 20260506 15:05:31.756 | INFO | Perf report for the last 5.13805 seconds: | 20260506 15:05:31.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:05:31.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:31.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:31.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:31.756 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:05:31.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:31.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:31.756 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:05:31.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:31.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:31.756 | INFO | Uploaded 36GiB to GPU in 1.97976s (18.184GiB/s) | 20260506 15:05:31.756 | INFO | Downloaded 34GiB to GPU in 1.94653s (17.467GiB/s) | 20260506 15:05:36.894 | INFO | Analyzing chunk frames = [2295296,2295808) | 20260506 15:05:36.894 | INFO | MemInfo: { | 20260506 15:05:36.894 | INFO | "gpu" : " 17 GB", | 20260506 15:05:36.894 | INFO | "huge" : "", | 20260506 15:05:36.894 | INFO | "huge_shared" : " 27 GB", | 20260506 15:05:36.894 | INFO | "malloc" : " 14 GB", | 20260506 15:05:36.894 | INFO | "pinned" : " 29 GB" | 20260506 15:05:36.894 | INFO | } | 20260506 15:05:36.894 | INFO | Perf report for the last 5.13859 seconds: | 20260506 15:05:36.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:05:36.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:36.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:36.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:36.894 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:05:36.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:36.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:36.895 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:05:36.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:36.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:36.895 | INFO | Uploaded 36GiB to GPU in 2.01737s (17.845GiB/s) | 20260506 15:05:36.895 | INFO | Downloaded 34GiB to GPU in 1.91702s (17.7359GiB/s) | 20260506 15:05:42.034 | INFO | Analyzing chunk frames = [2295808,2296320) | 20260506 15:05:42.034 | INFO | MemInfo: { | 20260506 15:05:42.034 | INFO | "gpu" : " 17 GB", | 20260506 15:05:42.034 | INFO | "huge" : "", | 20260506 15:05:42.034 | INFO | "huge_shared" : " 27 GB", | 20260506 15:05:42.034 | INFO | "malloc" : " 14 GB", | 20260506 15:05:42.034 | INFO | "pinned" : " 29 GB" | 20260506 15:05:42.034 | INFO | } | 20260506 15:05:42.034 | INFO | Perf report for the last 5.13961 seconds: | 20260506 15:05:42.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:05:42.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:42.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:42.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:42.034 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 15:05:42.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:42.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:42.034 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:05:42.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:42.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:42.034 | INFO | Uploaded 36GiB to GPU in 2.00196s (17.9824GiB/s) | 20260506 15:05:42.034 | INFO | Downloaded 34GiB to GPU in 1.93875s (17.5371GiB/s) | 20260506 15:05:47.173 | INFO | Analyzing chunk frames = [2296320,2296832) | 20260506 15:05:47.173 | INFO | MemInfo: { | 20260506 15:05:47.173 | INFO | "gpu" : " 17 GB", | 20260506 15:05:47.173 | INFO | "huge" : "", | 20260506 15:05:47.173 | INFO | "huge_shared" : " 27 GB", | 20260506 15:05:47.173 | INFO | "malloc" : " 14 GB", | 20260506 15:05:47.173 | INFO | "pinned" : " 29 GB" | 20260506 15:05:47.173 | INFO | } | 20260506 15:05:47.173 | INFO | Perf report for the last 5.1394 seconds: | 20260506 15:05:47.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:05:47.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:47.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:47.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:47.174 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:05:47.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:47.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:47.174 | INFO | PRE_HQ : 32.5%, 1.0, 16.9ms, 0 | 20260506 15:05:47.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:47.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:47.174 | INFO | Uploaded 36GiB to GPU in 2.06598s (17.4251GiB/s) | 20260506 15:05:47.174 | INFO | Downloaded 34GiB to GPU in 1.98239s (17.151GiB/s) | 20260506 15:05:52.314 | INFO | Analyzing chunk frames = [2296832,2297344) | 20260506 15:05:52.314 | INFO | MemInfo: { | 20260506 15:05:52.314 | INFO | "gpu" : " 17 GB", | 20260506 15:05:52.314 | INFO | "huge" : "", | 20260506 15:05:52.314 | INFO | "huge_shared" : " 27 GB", | 20260506 15:05:52.314 | INFO | "malloc" : " 14 GB", | 20260506 15:05:52.314 | INFO | "pinned" : " 29 GB" | 20260506 15:05:52.314 | INFO | } | 20260506 15:05:52.314 | INFO | Perf report for the last 5.14031 seconds: | 20260506 15:05:52.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:05:52.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:52.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:52.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:52.314 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:05:52.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:52.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:52.314 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 15:05:52.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:52.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:52.314 | INFO | Uploaded 36GiB to GPU in 2.01144s (17.8977GiB/s) | 20260506 15:05:52.314 | INFO | Downloaded 34GiB to GPU in 2.00456s (16.9613GiB/s) | 20260506 15:05:57.452 | INFO | Analyzing chunk frames = [2297344,2297856) | 20260506 15:05:57.452 | INFO | MemInfo: { | 20260506 15:05:57.452 | INFO | "gpu" : " 17 GB", | 20260506 15:05:57.452 | INFO | "huge" : "", | 20260506 15:05:57.452 | INFO | "huge_shared" : " 27 GB", | 20260506 15:05:57.452 | INFO | "malloc" : " 14 GB", | 20260506 15:05:57.452 | INFO | "pinned" : " 29 GB" | 20260506 15:05:57.452 | INFO | } | 20260506 15:05:57.452 | INFO | Perf report for the last 5.13859 seconds: | 20260506 15:05:57.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:05:57.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:57.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:57.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:57.452 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:05:57.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:57.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:57.453 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 15:05:57.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:57.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:05:57.453 | INFO | Uploaded 36GiB to GPU in 2.01664s (17.8515GiB/s) | 20260506 15:05:57.453 | INFO | Downloaded 34GiB to GPU in 1.98139s (17.1597GiB/s) | 20260506 15:05:57.565 | INFO | Starting a round of Baz IO | 20260506 15:06:00.724 | INFO | Populating RTMetrics report | 20260506 15:06:00.724 | INFO | RTMetrics report populated | 20260506 15:06:00.724 | INFO | Opening tmp RTMetrics json file | 20260506 15:06:00.776 | INFO | Serializing RTMetrics to json | 20260506 15:06:00.780 | INFO | Serialization done, swapping tmp file now... | 20260506 15:06:00.784 | INFO | RTMetrics done | 20260506 15:06:01.155 | INFO | Finished a round of Baz IO: Wrote 2.106 GiB in 3.589 seconds | 20260506 15:06:01.155 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.56% / 0.01% / 62.43% | 20260506 15:06:01.155 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.22152% / 3.57628e-05% / 4.37101e-05% | 20260506 15:06:01.155 | INFO | Baz IO overall: | 20260506 15:06:01.155 | INFO | BytesWritten : 1194189804678 | 20260506 15:06:01.155 | INFO | NumEvents : 577925757432 | 20260506 15:06:02.591 | INFO | Analyzing chunk frames = [2297856,2298368) | 20260506 15:06:02.591 | INFO | MemInfo: { | 20260506 15:06:02.591 | INFO | "gpu" : " 17 GB", | 20260506 15:06:02.591 | INFO | "huge" : "", | 20260506 15:06:02.591 | INFO | "huge_shared" : " 27 GB", | 20260506 15:06:02.591 | INFO | "malloc" : " 14 GB", | 20260506 15:06:02.591 | INFO | "pinned" : " 29 GB" | 20260506 15:06:02.591 | INFO | } | 20260506 15:06:02.591 | INFO | Perf report for the last 5.13868 seconds: | 20260506 15:06:02.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:06:02.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:02.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:02.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:02.591 | INFO | ANALYSIS : 63.0%, 2.9, 95.5ms, 0 | 20260506 15:06:02.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:02.591 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260506 15:06:02.591 | INFO | PRE_HQ : 61.1%, 1.9, 59.5ms, 0 | 20260506 15:06:02.591 | INFO | BAZWRITER : 69.3%, 4.8, 170.8ms, 0 | 20260506 15:06:02.591 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 15:06:02.591 | INFO | Uploaded 36GiB to GPU in 2.06739s (17.4132GiB/s) | 20260506 15:06:02.591 | INFO | Downloaded 38GiB to GPU in 2.22347s (17.0904GiB/s) | 20260506 15:06:07.730 | INFO | Analyzing chunk frames = [2298368,2298880) | 20260506 15:06:07.730 | INFO | MemInfo: { | 20260506 15:06:07.730 | INFO | "gpu" : " 17 GB", | 20260506 15:06:07.730 | INFO | "huge" : "", | 20260506 15:06:07.730 | INFO | "huge_shared" : " 27 GB", | 20260506 15:06:07.730 | INFO | "malloc" : " 14 GB", | 20260506 15:06:07.731 | INFO | "pinned" : " 29 GB" | 20260506 15:06:07.731 | INFO | } | 20260506 15:06:07.731 | INFO | Perf report for the last 5.13948 seconds: | 20260506 15:06:07.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:06:07.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:07.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:07.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:07.731 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 15:06:07.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:07.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:07.731 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:06:07.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:07.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:07.731 | INFO | Uploaded 36GiB to GPU in 2.00062s (17.9944GiB/s) | 20260506 15:06:07.731 | INFO | Downloaded 34GiB to GPU in 1.93681s (17.5546GiB/s) | 20260506 15:06:12.870 | INFO | Analyzing chunk frames = [2298880,2299392) | 20260506 15:06:12.870 | INFO | MemInfo: { | 20260506 15:06:12.870 | INFO | "gpu" : " 17 GB", | 20260506 15:06:12.870 | INFO | "huge" : "", | 20260506 15:06:12.870 | INFO | "huge_shared" : " 27 GB", | 20260506 15:06:12.870 | INFO | "malloc" : " 14 GB", | 20260506 15:06:12.871 | INFO | "pinned" : " 29 GB" | 20260506 15:06:12.871 | INFO | } | 20260506 15:06:12.871 | INFO | Perf report for the last 5.14012 seconds: | 20260506 15:06:12.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:06:12.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:12.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:12.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:12.871 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 15:06:12.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:12.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:12.871 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:06:12.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:12.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:12.871 | INFO | Uploaded 36GiB to GPU in 2.00232s (17.9791GiB/s) | 20260506 15:06:12.871 | INFO | Downloaded 34GiB to GPU in 1.94095s (17.5172GiB/s) | 20260506 15:06:18.010 | INFO | Analyzing chunk frames = [2299392,2299904) | 20260506 15:06:18.010 | INFO | MemInfo: { | 20260506 15:06:18.010 | INFO | "gpu" : " 17 GB", | 20260506 15:06:18.010 | INFO | "huge" : "", | 20260506 15:06:18.010 | INFO | "huge_shared" : " 27 GB", | 20260506 15:06:18.010 | INFO | "malloc" : " 14 GB", | 20260506 15:06:18.010 | INFO | "pinned" : " 29 GB" | 20260506 15:06:18.010 | INFO | } | 20260506 15:06:18.010 | INFO | Perf report for the last 5.1392 seconds: | 20260506 15:06:18.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:06:18.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:18.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:18.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:18.010 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:06:18.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:18.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:18.010 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:06:18.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:18.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:18.010 | INFO | Uploaded 36GiB to GPU in 2.02345s (17.7914GiB/s) | 20260506 15:06:18.010 | INFO | Downloaded 34GiB to GPU in 1.91536s (17.7512GiB/s) | 20260506 15:06:23.149 | INFO | Analyzing chunk frames = [2299904,2300416) | 20260506 15:06:23.149 | INFO | MemInfo: { | 20260506 15:06:23.149 | INFO | "gpu" : " 17 GB", | 20260506 15:06:23.149 | INFO | "huge" : "", | 20260506 15:06:23.149 | INFO | "huge_shared" : " 27 GB", | 20260506 15:06:23.149 | INFO | "malloc" : " 14 GB", | 20260506 15:06:23.149 | INFO | "pinned" : " 29 GB" | 20260506 15:06:23.149 | INFO | } | 20260506 15:06:23.150 | INFO | Perf report for the last 5.13956 seconds: | 20260506 15:06:23.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:06:23.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:23.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:23.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:23.150 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 15:06:23.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:23.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:23.150 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:06:23.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:23.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:23.150 | INFO | Uploaded 36GiB to GPU in 2.05207s (17.5433GiB/s) | 20260506 15:06:23.150 | INFO | Downloaded 34GiB to GPU in 1.92965s (17.6198GiB/s) | 20260506 15:06:28.288 | INFO | Analyzing chunk frames = [2300416,2300928) | 20260506 15:06:28.288 | INFO | MemInfo: { | 20260506 15:06:28.288 | INFO | "gpu" : " 17 GB", | 20260506 15:06:28.288 | INFO | "huge" : "", | 20260506 15:06:28.288 | INFO | "huge_shared" : " 27 GB", | 20260506 15:06:28.288 | INFO | "malloc" : " 14 GB", | 20260506 15:06:28.288 | INFO | "pinned" : " 29 GB" | 20260506 15:06:28.288 | INFO | } | 20260506 15:06:28.289 | INFO | Perf report for the last 5.139 seconds: | 20260506 15:06:28.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:06:28.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:28.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:28.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:28.289 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 15:06:28.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:28.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:28.289 | INFO | PRE_HQ : 30.0%, 1.0, 15.5ms, 0 | 20260506 15:06:28.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:28.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:28.289 | INFO | Uploaded 36GiB to GPU in 2.04851s (17.5737GiB/s) | 20260506 15:06:28.289 | INFO | Downloaded 34GiB to GPU in 1.92483s (17.6639GiB/s) | 20260506 15:06:33.428 | INFO | Analyzing chunk frames = [2300928,2301440) | 20260506 15:06:33.428 | INFO | MemInfo: { | 20260506 15:06:33.428 | INFO | "gpu" : " 17 GB", | 20260506 15:06:33.428 | INFO | "huge" : "", | 20260506 15:06:33.428 | INFO | "huge_shared" : " 27 GB", | 20260506 15:06:33.428 | INFO | "malloc" : " 14 GB", | 20260506 15:06:33.428 | INFO | "pinned" : " 29 GB" | 20260506 15:06:33.428 | INFO | } | 20260506 15:06:33.428 | INFO | Perf report for the last 5.13974 seconds: | 20260506 15:06:33.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:06:33.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:33.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:33.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:33.428 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:06:33.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:33.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:33.428 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 15:06:33.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:33.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:33.429 | INFO | Uploaded 36GiB to GPU in 2.00451s (17.9595GiB/s) | 20260506 15:06:33.429 | INFO | Downloaded 34GiB to GPU in 1.99597s (17.0343GiB/s) | 20260506 15:06:38.568 | INFO | Analyzing chunk frames = [2301440,2301952) | 20260506 15:06:38.568 | INFO | MemInfo: { | 20260506 15:06:38.568 | INFO | "gpu" : " 17 GB", | 20260506 15:06:38.568 | INFO | "huge" : "", | 20260506 15:06:38.568 | INFO | "huge_shared" : " 27 GB", | 20260506 15:06:38.568 | INFO | "malloc" : " 14 GB", | 20260506 15:06:38.568 | INFO | "pinned" : " 29 GB" | 20260506 15:06:38.568 | INFO | } | 20260506 15:06:38.568 | INFO | Perf report for the last 5.13984 seconds: | 20260506 15:06:38.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:06:38.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:38.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:38.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:38.568 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 15:06:38.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:38.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:38.568 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260506 15:06:38.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:38.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:38.568 | INFO | Uploaded 36GiB to GPU in 2.03197s (17.7168GiB/s) | 20260506 15:06:38.568 | INFO | Downloaded 34GiB to GPU in 1.97395s (17.2244GiB/s) | 20260506 15:06:38.568 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:06:38.569 | INFO | Total mem usage: 17.4 GiB | 20260506 15:06:38.569 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:06:38.569 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -11.3%) | 20260506 15:06:38.569 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:06:38.569 | INFO | High Water Breakdown: | 20260506 15:06:38.569 | INFO | 61.2% : 9.58 GiB ( 87.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:06:38.569 | INFO | 38.6% : 6.04 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:06:38.569 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:06:38.569 | INFO | ------------------------------------------ | 20260506 15:06:38.569 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:06:38.569 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:06:38.569 | INFO | Total mem usage: 26.5 GiB | 20260506 15:06:38.569 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:06:38.569 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:06:38.569 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:06:38.569 | INFO | High Water Breakdown: | 20260506 15:06:38.569 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:06:38.569 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:06:38.569 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:06:38.569 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:06:38.569 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:06:38.569 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:06:38.569 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:06:38.569 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:06:38.569 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:06:38.569 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:06:38.569 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:06:38.569 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:06:38.569 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:06:38.569 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:06:38.569 | INFO | ------------------------------------------ | 20260506 15:06:38.569 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:06:38.569 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:06:38.569 | INFO | Total mem usage: 25.6 GiB | 20260506 15:06:38.569 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:06:38.569 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:06:38.569 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:06:38.569 | INFO | High Water Breakdown: | 20260506 15:06:38.569 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:06:38.569 | INFO | ------------------------------------------ | 20260506 15:06:38.569 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:06:38.569 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:06:38.569 | INFO | Total mem usage: 15.7 GiB | 20260506 15:06:38.569 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:06:38.569 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:06:38.569 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 15:06:38.569 | INFO | High Water Breakdown: | 20260506 15:06:38.569 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:06:38.569 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:06:38.570 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:06:38.570 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:06:38.570 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:06:38.570 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:06:38.570 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:06:38.570 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:06:38.570 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:06:38.570 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:06:38.570 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:06:38.570 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:06:38.570 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:06:38.570 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:06:38.570 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:06:38.570 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:06:38.570 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:06:38.570 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:06:38.570 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:06:38.570 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:06:38.570 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:06:38.570 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:06:38.570 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:06:38.570 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:06:38.570 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:06:38.570 | INFO | ------------------------------------------ | 20260506 15:06:38.570 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:06:38.570 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:06:38.570 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:06:38.570 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:06:38.570 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.55e+01, 1.35e+01 / 2.01e+00 | 20260506 15:06:38.570 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.01e-01, 1.13e-02 / 6.62e-03 | 20260506 15:06:38.570 | INFO | PerfReport: 11.60%, Baseline -- 3.65e+00 / 4.44e+00, 5.67e-01 / 3.92e-01 | 20260506 15:06:38.570 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.48e+00, 1.88e-01 / 1.51e-01 | 20260506 15:06:38.570 | INFO | PerfReport: 15.82%, DME -- 3.98e+01 / 4.39e+01, 3.81e+00 / 2.11e+00 | 20260506 15:06:38.570 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.50e+00 / 2.31e+00 | 20260506 15:06:38.570 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.33e+00 / 2.92e+00, 4.02e-01 / 2.31e-01 | 20260506 15:06:38.570 | INFO | PerfReport: 3.57%, Metrics -- 1.12e+00 / 1.48e+00, 2.88e-01 / 2.22e-01 | 20260506 15:06:38.570 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.92e-01 / 4.41e-01, 6.12e-02 / 2.81e-02 | 20260506 15:06:38.570 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:06:38.570 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:06:38.570 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:06:38.570 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 15:06:38.570 | INFO | PerfReport: 21.57%, Upload -- 2.11e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260506 15:06:38.570 | INFO | PerfReport: 21.18%, Download -- 2.07e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 15:06:38.570 | INFO | PerfReport: 32.23%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 15:06:38.570 | INFO | PerfReport: 1.54%, LockUpload -- 1.51e+00 / 0.00e+00, 5.89e+00 / 0.00e+00 | 20260506 15:06:38.570 | INFO | PerfReport: 1.08%, LockDownload -- 1.06e+00 / 0.00e+00, 2.67e+00 / 0.00e+00 | 20260506 15:06:38.570 | INFO | PerfReport: 22.38%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 15:06:38.570 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260506 15:06:38.686 | INFO | Starting a round of Baz IO | 20260506 15:06:41.897 | INFO | Populating RTMetrics report | 20260506 15:06:41.897 | INFO | RTMetrics report populated | 20260506 15:06:41.897 | INFO | Opening tmp RTMetrics json file | 20260506 15:06:41.898 | INFO | Serializing RTMetrics to json | 20260506 15:06:41.900 | INFO | Serialization done, swapping tmp file now... | 20260506 15:06:41.901 | INFO | RTMetrics done | 20260506 15:06:42.473 | INFO | Finished a round of Baz IO: Wrote 2.107 GiB in 3.786 seconds | 20260506 15:06:42.473 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.55% / 0.01% / 62.44% | 20260506 15:06:42.473 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.22179% / 3.97364e-06% / 5.96046e-05% | 20260506 15:06:42.473 | INFO | Baz IO overall: | 20260506 15:06:42.473 | INFO | BytesWritten : 1196452346063 | 20260506 15:06:42.473 | INFO | NumEvents : 579021498798 | 20260506 15:06:43.706 | INFO | Analyzing chunk frames = [2301952,2302464) | 20260506 15:06:43.707 | INFO | MemInfo: { | 20260506 15:06:43.707 | INFO | "gpu" : " 17 GB", | 20260506 15:06:43.707 | INFO | "huge" : "", | 20260506 15:06:43.707 | INFO | "huge_shared" : " 27 GB", | 20260506 15:06:43.707 | INFO | "malloc" : " 14 GB", | 20260506 15:06:43.707 | INFO | "pinned" : " 29 GB" | 20260506 15:06:43.707 | INFO | } | 20260506 15:06:43.707 | INFO | Perf report for the last 5.13842 seconds: | 20260506 15:06:43.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:06:43.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:43.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:43.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:43.707 | INFO | ANALYSIS : 63.2%, 2.8, 93.7ms, 0 | 20260506 15:06:43.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:43.707 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260506 15:06:43.707 | INFO | PRE_HQ : 61.7%, 2.0, 64.8ms, 0 | 20260506 15:06:43.707 | INFO | BAZWRITER : 73.2%, 4.8, 180.7ms, 0 | 20260506 15:06:43.707 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 15:06:43.707 | INFO | Uploaded 36GiB to GPU in 2.03073s (17.7277GiB/s) | 20260506 15:06:43.707 | INFO | Downloaded 38GiB to GPU in 2.20166s (17.2597GiB/s) | 20260506 15:06:48.845 | INFO | Analyzing chunk frames = [2302464,2302976) | 20260506 15:06:48.845 | INFO | MemInfo: { | 20260506 15:06:48.845 | INFO | "gpu" : " 17 GB", | 20260506 15:06:48.845 | INFO | "huge" : "", | 20260506 15:06:48.845 | INFO | "huge_shared" : " 27 GB", | 20260506 15:06:48.845 | INFO | "malloc" : " 14 GB", | 20260506 15:06:48.845 | INFO | "pinned" : " 29 GB" | 20260506 15:06:48.845 | INFO | } | 20260506 15:06:48.845 | INFO | Perf report for the last 5.13875 seconds: | 20260506 15:06:48.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:06:48.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:48.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:48.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:48.845 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:06:48.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:48.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:48.845 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 15:06:48.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:48.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:48.846 | INFO | Uploaded 36GiB to GPU in 1.9949s (18.046GiB/s) | 20260506 15:06:48.846 | INFO | Downloaded 34GiB to GPU in 1.92282s (17.6824GiB/s) | 20260506 15:06:53.985 | INFO | Analyzing chunk frames = [2302976,2303488) | 20260506 15:06:53.985 | INFO | MemInfo: { | 20260506 15:06:53.985 | INFO | "gpu" : " 17 GB", | 20260506 15:06:53.985 | INFO | "huge" : "", | 20260506 15:06:53.985 | INFO | "huge_shared" : " 27 GB", | 20260506 15:06:53.985 | INFO | "malloc" : " 14 GB", | 20260506 15:06:53.985 | INFO | "pinned" : " 29 GB" | 20260506 15:06:53.985 | INFO | } | 20260506 15:06:53.985 | INFO | Perf report for the last 5.13957 seconds: | 20260506 15:06:53.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:06:53.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:53.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:53.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:53.985 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 15:06:53.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:53.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:53.985 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:06:53.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:53.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:53.985 | INFO | Uploaded 36GiB to GPU in 2.00167s (17.985GiB/s) | 20260506 15:06:53.985 | INFO | Downloaded 34GiB to GPU in 1.93995s (17.5262GiB/s) | 20260506 15:06:59.124 | INFO | Analyzing chunk frames = [2303488,2304000) | 20260506 15:06:59.124 | INFO | MemInfo: { | 20260506 15:06:59.124 | INFO | "gpu" : " 17 GB", | 20260506 15:06:59.124 | INFO | "huge" : "", | 20260506 15:06:59.124 | INFO | "huge_shared" : " 27 GB", | 20260506 15:06:59.125 | INFO | "malloc" : " 14 GB", | 20260506 15:06:59.125 | INFO | "pinned" : " 29 GB" | 20260506 15:06:59.125 | INFO | } | 20260506 15:06:59.125 | INFO | Perf report for the last 5.13976 seconds: | 20260506 15:06:59.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:06:59.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:59.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:59.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:59.125 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 15:06:59.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:59.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:59.125 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:06:59.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:59.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:06:59.125 | INFO | Uploaded 36GiB to GPU in 2.02004s (17.8214GiB/s) | 20260506 15:06:59.125 | INFO | Downloaded 34GiB to GPU in 1.91246s (17.7782GiB/s) | 20260506 15:07:04.263 | INFO | Analyzing chunk frames = [2304000,2304512) | 20260506 15:07:04.263 | INFO | MemInfo: { | 20260506 15:07:04.263 | INFO | "gpu" : " 17 GB", | 20260506 15:07:04.263 | INFO | "huge" : "", | 20260506 15:07:04.263 | INFO | "huge_shared" : " 27 GB", | 20260506 15:07:04.263 | INFO | "malloc" : " 14 GB", | 20260506 15:07:04.263 | INFO | "pinned" : " 29 GB" | 20260506 15:07:04.263 | INFO | } | 20260506 15:07:04.263 | INFO | Perf report for the last 5.13835 seconds: | 20260506 15:07:04.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:07:04.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:04.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:04.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:04.263 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 15:07:04.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:04.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:04.263 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:07:04.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:04.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:04.263 | INFO | Uploaded 36GiB to GPU in 2.02288s (17.7964GiB/s) | 20260506 15:07:04.263 | INFO | Downloaded 34GiB to GPU in 1.92584s (17.6547GiB/s) | 20260506 15:07:09.402 | INFO | Analyzing chunk frames = [2304512,2305024) | 20260506 15:07:09.402 | INFO | MemInfo: { | 20260506 15:07:09.402 | INFO | "gpu" : " 17 GB", | 20260506 15:07:09.402 | INFO | "huge" : "", | 20260506 15:07:09.402 | INFO | "huge_shared" : " 27 GB", | 20260506 15:07:09.402 | INFO | "malloc" : " 14 GB", | 20260506 15:07:09.402 | INFO | "pinned" : " 29 GB" | 20260506 15:07:09.402 | INFO | } | 20260506 15:07:09.402 | INFO | Perf report for the last 5.13938 seconds: | 20260506 15:07:09.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:07:09.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:09.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:09.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:09.402 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:07:09.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:09.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:09.403 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:07:09.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:09.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:09.403 | INFO | Uploaded 36GiB to GPU in 2.07562s (17.3442GiB/s) | 20260506 15:07:09.403 | INFO | Downloaded 34GiB to GPU in 1.96466s (17.3058GiB/s) | 20260506 15:07:14.542 | INFO | Analyzing chunk frames = [2305024,2305536) | 20260506 15:07:14.542 | INFO | MemInfo: { | 20260506 15:07:14.542 | INFO | "gpu" : " 17 GB", | 20260506 15:07:14.542 | INFO | "huge" : "", | 20260506 15:07:14.542 | INFO | "huge_shared" : " 27 GB", | 20260506 15:07:14.542 | INFO | "malloc" : " 14 GB", | 20260506 15:07:14.543 | INFO | "pinned" : " 29 GB" | 20260506 15:07:14.543 | INFO | } | 20260506 15:07:14.543 | INFO | Perf report for the last 5.14027 seconds: | 20260506 15:07:14.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:07:14.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:14.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:14.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:14.543 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 15:07:14.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:14.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:14.543 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:07:14.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:14.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:14.543 | INFO | Uploaded 36GiB to GPU in 2.02091s (17.8138GiB/s) | 20260506 15:07:14.543 | INFO | Downloaded 34GiB to GPU in 1.97806s (17.1886GiB/s) | 20260506 15:07:19.681 | INFO | Analyzing chunk frames = [2305536,2306048) | 20260506 15:07:19.681 | INFO | MemInfo: { | 20260506 15:07:19.681 | INFO | "gpu" : " 17 GB", | 20260506 15:07:19.681 | INFO | "huge" : "", | 20260506 15:07:19.681 | INFO | "huge_shared" : " 27 GB", | 20260506 15:07:19.681 | INFO | "malloc" : " 14 GB", | 20260506 15:07:19.682 | INFO | "pinned" : " 29 GB" | 20260506 15:07:19.682 | INFO | } | 20260506 15:07:19.682 | INFO | Perf report for the last 5.13883 seconds: | 20260506 15:07:19.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:07:19.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:19.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:19.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:19.682 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:07:19.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:19.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:19.682 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:07:19.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:19.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:19.682 | INFO | Uploaded 36GiB to GPU in 2.02106s (17.8124GiB/s) | 20260506 15:07:19.682 | INFO | Downloaded 34GiB to GPU in 1.98367s (17.1399GiB/s) | 20260506 15:07:19.791 | INFO | Starting a round of Baz IO | 20260506 15:07:22.948 | INFO | Populating RTMetrics report | 20260506 15:07:22.948 | INFO | RTMetrics report populated | 20260506 15:07:22.948 | INFO | Opening tmp RTMetrics json file | 20260506 15:07:22.953 | INFO | Serializing RTMetrics to json | 20260506 15:07:22.995 | INFO | Serialization done, swapping tmp file now... | 20260506 15:07:23.003 | INFO | RTMetrics done | 20260506 15:07:23.328 | INFO | Finished a round of Baz IO: Wrote 2.107 GiB in 3.536 seconds | 20260506 15:07:23.328 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.53% / 0.01% / 62.45% | 20260506 15:07:23.328 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.21776% / 2.78155e-05% / 5.5631e-05% | 20260506 15:07:23.328 | INFO | Baz IO overall: | 20260506 15:07:23.328 | INFO | BytesWritten : 1198714825295 | 20260506 15:07:23.328 | INFO | NumEvents : 580117731867 | 20260506 15:07:24.821 | INFO | Analyzing chunk frames = [2306048,2306560) | 20260506 15:07:24.821 | INFO | MemInfo: { | 20260506 15:07:24.821 | INFO | "gpu" : " 17 GB", | 20260506 15:07:24.821 | INFO | "huge" : "", | 20260506 15:07:24.821 | INFO | "huge_shared" : " 27 GB", | 20260506 15:07:24.821 | INFO | "malloc" : " 14 GB", | 20260506 15:07:24.821 | INFO | "pinned" : " 29 GB" | 20260506 15:07:24.821 | INFO | } | 20260506 15:07:24.821 | INFO | Perf report for the last 5.13928 seconds: | 20260506 15:07:24.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:07:24.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:24.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:24.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:24.821 | INFO | ANALYSIS : 63.0%, 2.9, 95.4ms, 0 | 20260506 15:07:24.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:24.821 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260506 15:07:24.821 | INFO | PRE_HQ : 60.8%, 1.9, 59.3ms, 0 | 20260506 15:07:24.821 | INFO | BAZWRITER : 68.5%, 4.8, 168.4ms, 0 | 20260506 15:07:24.821 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 15:07:24.821 | INFO | Uploaded 36GiB to GPU in 2.04072s (17.6408GiB/s) | 20260506 15:07:24.821 | INFO | Downloaded 38GiB to GPU in 2.22571s (17.0732GiB/s) | 20260506 15:07:29.961 | INFO | Analyzing chunk frames = [2306560,2307072) | 20260506 15:07:29.961 | INFO | MemInfo: { | 20260506 15:07:29.961 | INFO | "gpu" : " 17 GB", | 20260506 15:07:29.961 | INFO | "huge" : "", | 20260506 15:07:29.961 | INFO | "huge_shared" : " 27 GB", | 20260506 15:07:29.961 | INFO | "malloc" : " 14 GB", | 20260506 15:07:29.961 | INFO | "pinned" : " 29 GB" | 20260506 15:07:29.961 | INFO | } | 20260506 15:07:29.961 | INFO | Perf report for the last 5.1402 seconds: | 20260506 15:07:29.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:07:29.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:29.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:29.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:29.961 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:07:29.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:29.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:29.961 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:07:29.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:29.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:29.961 | INFO | Uploaded 36GiB to GPU in 1.98578s (18.1289GiB/s) | 20260506 15:07:29.961 | INFO | Downloaded 34GiB to GPU in 1.93284s (17.5907GiB/s) | 20260506 15:07:35.100 | INFO | Analyzing chunk frames = [2307072,2307584) | 20260506 15:07:35.100 | INFO | MemInfo: { | 20260506 15:07:35.100 | INFO | "gpu" : " 17 GB", | 20260506 15:07:35.100 | INFO | "huge" : "", | 20260506 15:07:35.100 | INFO | "huge_shared" : " 27 GB", | 20260506 15:07:35.100 | INFO | "malloc" : " 14 GB", | 20260506 15:07:35.100 | INFO | "pinned" : " 29 GB" | 20260506 15:07:35.100 | INFO | } | 20260506 15:07:35.100 | INFO | Perf report for the last 5.13867 seconds: | 20260506 15:07:35.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:07:35.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:35.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:35.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:35.100 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 15:07:35.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:35.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:35.100 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:07:35.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:35.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:35.100 | INFO | Uploaded 36GiB to GPU in 1.96286s (18.3406GiB/s) | 20260506 15:07:35.100 | INFO | Downloaded 34GiB to GPU in 1.9401s (17.5249GiB/s) | 20260506 15:07:40.239 | INFO | Analyzing chunk frames = [2307584,2308096) | 20260506 15:07:40.239 | INFO | MemInfo: { | 20260506 15:07:40.239 | INFO | "gpu" : " 17 GB", | 20260506 15:07:40.239 | INFO | "huge" : "", | 20260506 15:07:40.239 | INFO | "huge_shared" : " 27 GB", | 20260506 15:07:40.239 | INFO | "malloc" : " 14 GB", | 20260506 15:07:40.239 | INFO | "pinned" : " 29 GB" | 20260506 15:07:40.239 | INFO | } | 20260506 15:07:40.239 | INFO | Perf report for the last 5.13916 seconds: | 20260506 15:07:40.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:07:40.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:40.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:40.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:40.239 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 15:07:40.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:40.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:40.239 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:07:40.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:40.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:40.239 | INFO | Uploaded 36GiB to GPU in 2.01612s (17.8561GiB/s) | 20260506 15:07:40.239 | INFO | Downloaded 34GiB to GPU in 1.91856s (17.7216GiB/s) | 20260506 15:07:45.377 | INFO | Analyzing chunk frames = [2308096,2308608) | 20260506 15:07:45.377 | INFO | MemInfo: { | 20260506 15:07:45.377 | INFO | "gpu" : " 17 GB", | 20260506 15:07:45.377 | INFO | "huge" : "", | 20260506 15:07:45.377 | INFO | "huge_shared" : " 27 GB", | 20260506 15:07:45.377 | INFO | "malloc" : " 14 GB", | 20260506 15:07:45.377 | INFO | "pinned" : " 29 GB" | 20260506 15:07:45.377 | INFO | } | 20260506 15:07:45.378 | INFO | Perf report for the last 5.1386 seconds: | 20260506 15:07:45.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:07:45.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:45.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:45.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:45.378 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 15:07:45.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:45.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:45.378 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:07:45.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:45.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:45.378 | INFO | Uploaded 36GiB to GPU in 2.0025s (17.9776GiB/s) | 20260506 15:07:45.378 | INFO | Downloaded 34GiB to GPU in 1.9382s (17.542GiB/s) | 20260506 15:07:50.517 | INFO | Analyzing chunk frames = [2308608,2309120) | 20260506 15:07:50.517 | INFO | MemInfo: { | 20260506 15:07:50.517 | INFO | "gpu" : " 17 GB", | 20260506 15:07:50.517 | INFO | "huge" : "", | 20260506 15:07:50.517 | INFO | "huge_shared" : " 27 GB", | 20260506 15:07:50.517 | INFO | "malloc" : " 14 GB", | 20260506 15:07:50.517 | INFO | "pinned" : " 29 GB" | 20260506 15:07:50.517 | INFO | } | 20260506 15:07:50.517 | INFO | Perf report for the last 5.13963 seconds: | 20260506 15:07:50.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:07:50.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:50.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:50.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:50.517 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 15:07:50.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:50.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:50.517 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 15:07:50.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:50.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:50.517 | INFO | Uploaded 36GiB to GPU in 2.0309s (17.7261GiB/s) | 20260506 15:07:50.517 | INFO | Downloaded 34GiB to GPU in 1.91814s (17.7255GiB/s) | 20260506 15:07:55.658 | INFO | Analyzing chunk frames = [2309120,2309632) | 20260506 15:07:55.658 | INFO | MemInfo: { | 20260506 15:07:55.658 | INFO | "gpu" : " 17 GB", | 20260506 15:07:55.658 | INFO | "huge" : "", | 20260506 15:07:55.658 | INFO | "huge_shared" : " 27 GB", | 20260506 15:07:55.658 | INFO | "malloc" : " 14 GB", | 20260506 15:07:55.658 | INFO | "pinned" : " 29 GB" | 20260506 15:07:55.658 | INFO | } | 20260506 15:07:55.658 | INFO | Perf report for the last 5.1409 seconds: | 20260506 15:07:55.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:07:55.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:55.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:55.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:55.658 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 15:07:55.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:55.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:55.658 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 15:07:55.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:55.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:07:55.658 | INFO | Uploaded 36GiB to GPU in 1.99053s (18.0857GiB/s) | 20260506 15:07:55.658 | INFO | Downloaded 34GiB to GPU in 1.97197s (17.2417GiB/s) | 20260506 15:08:00.797 | INFO | Analyzing chunk frames = [2309632,2310144) | 20260506 15:08:00.797 | INFO | MemInfo: { | 20260506 15:08:00.797 | INFO | "gpu" : " 17 GB", | 20260506 15:08:00.797 | INFO | "huge" : "", | 20260506 15:08:00.797 | INFO | "huge_shared" : " 27 GB", | 20260506 15:08:00.797 | INFO | "malloc" : " 14 GB", | 20260506 15:08:00.797 | INFO | "pinned" : " 29 GB" | 20260506 15:08:00.797 | INFO | } | 20260506 15:08:00.797 | INFO | Perf report for the last 5.13898 seconds: | 20260506 15:08:00.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:08:00.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:00.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:00.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:00.797 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:08:00.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:00.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:00.797 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:08:00.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:00.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:00.798 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:08:00.798 | INFO | Total mem usage: 17.4 GiB | 20260506 15:08:00.798 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:08:00.798 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -10.8%) | 20260506 15:08:00.798 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:08:00.798 | INFO | High Water Breakdown: | 20260506 15:08:00.798 | INFO | 61.3% : 9.63 GiB ( 88.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:08:00.798 | INFO | 38.6% : 6.06 GiB ( 90.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:08:00.798 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:08:00.798 | INFO | ------------------------------------------ | 20260506 15:08:00.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:08:00.798 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:08:00.798 | INFO | Total mem usage: 26.5 GiB | 20260506 15:08:00.798 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:08:00.798 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:08:00.798 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:08:00.798 | INFO | High Water Breakdown: | 20260506 15:08:00.798 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:08:00.798 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:08:00.798 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:08:00.798 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:08:00.798 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:08:00.798 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:08:00.798 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:08:00.798 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:08:00.798 | INFO | Uploaded 36GiB to GPU in 2.01905s (17.8302GiB/s) | 20260506 15:08:00.798 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:08:00.798 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:08:00.798 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:08:00.798 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:08:00.798 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:08:00.798 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:08:00.798 | INFO | ------------------------------------------ | 20260506 15:08:00.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:08:00.798 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:08:00.798 | INFO | Total mem usage: 25.6 GiB | 20260506 15:08:00.798 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:08:00.798 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:08:00.798 | INFO | Downloaded 34GiB to GPU in 1.98418s (17.1355GiB/s) | 20260506 15:08:00.798 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:08:00.798 | INFO | High Water Breakdown: | 20260506 15:08:00.798 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:08:00.798 | INFO | ------------------------------------------ | 20260506 15:08:00.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:08:00.798 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:08:00.798 | INFO | Total mem usage: 15.7 GiB | 20260506 15:08:00.798 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:08:00.798 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:08:00.798 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 15:08:00.798 | INFO | High Water Breakdown: | 20260506 15:08:00.798 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:08:00.798 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:08:00.798 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:08:00.798 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:08:00.798 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:08:00.798 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:08:00.798 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:08:00.798 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:08:00.798 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:08:00.798 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:08:00.798 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:08:00.798 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:08:00.798 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:08:00.798 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:08:00.798 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:08:00.798 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:08:00.798 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:08:00.798 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:08:00.798 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:08:00.798 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:08:00.798 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:08:00.798 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:08:00.799 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:08:00.799 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:08:00.799 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:08:00.799 | INFO | ------------------------------------------ | 20260506 15:08:00.799 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:08:00.799 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:08:00.799 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:08:00.799 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:08:00.799 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.57e+01, 1.33e+01 / 1.92e+00 | 20260506 15:08:00.799 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.00e-01, 1.16e-02 / 6.12e-03 | 20260506 15:08:00.799 | INFO | PerfReport: 11.59%, Baseline -- 3.65e+00 / 4.43e+00, 5.76e-01 / 4.34e-01 | 20260506 15:08:00.799 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.45e+00, 1.87e-01 / 1.56e-01 | 20260506 15:08:00.799 | INFO | PerfReport: 15.80%, DME -- 3.98e+01 / 4.39e+01, 3.79e+00 / 2.02e+00 | 20260506 15:08:00.799 | INFO | PerfReport: 56.89%, FrameLabeling -- 1.79e+01 / 2.10e+01, 1.54e+00 / 2.09e+00 | 20260506 15:08:00.799 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.32e+00 / 2.88e+00, 3.93e-01 / 2.90e-01 | 20260506 15:08:00.799 | INFO | PerfReport: 3.57%, Metrics -- 1.12e+00 / 1.50e+00, 2.90e-01 / 2.07e-01 | 20260506 15:08:00.799 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.91e-01 / 4.40e-01, 6.13e-02 / 2.53e-02 | 20260506 15:08:00.799 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:08:00.799 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:08:00.799 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:08:00.799 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 15:08:00.799 | INFO | PerfReport: 21.51%, Upload -- 2.10e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260506 15:08:00.799 | INFO | PerfReport: 21.13%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 15:08:00.799 | INFO | PerfReport: 32.31%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 15:08:00.799 | INFO | PerfReport: 1.55%, LockUpload -- 1.52e+00 / 0.00e+00, 6.01e+00 / 0.00e+00 | 20260506 15:08:00.799 | INFO | PerfReport: 1.08%, LockDownload -- 1.06e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 15:08:00.799 | INFO | PerfReport: 22.39%, LockCompute -- 2.18e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 15:08:00.799 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260506 15:08:00.910 | INFO | Starting a round of Baz IO | 20260506 15:08:04.066 | INFO | Populating RTMetrics report | 20260506 15:08:04.067 | INFO | RTMetrics report populated | 20260506 15:08:04.067 | INFO | Opening tmp RTMetrics json file | 20260506 15:08:04.102 | INFO | Serializing RTMetrics to json | 20260506 15:08:04.121 | INFO | Serialization done, swapping tmp file now... | 20260506 15:08:04.129 | INFO | RTMetrics done | 20260506 15:08:04.442 | INFO | Finished a round of Baz IO: Wrote 2.104 GiB in 3.532 seconds | 20260506 15:08:04.442 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.52% / 0.01% / 62.47% | 20260506 15:08:04.442 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.20671% / 7.94729e-06% / 5.16574e-05% | 20260506 15:08:04.442 | INFO | Baz IO overall: | 20260506 15:08:04.442 | INFO | BytesWritten : 1200974467700 | 20260506 15:08:04.442 | INFO | NumEvents : 581211880144 | 20260506 15:08:05.935 | INFO | Analyzing chunk frames = [2310144,2310656) | 20260506 15:08:05.935 | INFO | MemInfo: { | 20260506 15:08:05.935 | INFO | "gpu" : " 17 GB", | 20260506 15:08:05.935 | INFO | "huge" : "", | 20260506 15:08:05.935 | INFO | "huge_shared" : " 27 GB", | 20260506 15:08:05.936 | INFO | "malloc" : " 14 GB", | 20260506 15:08:05.936 | INFO | "pinned" : " 29 GB" | 20260506 15:08:05.936 | INFO | } | 20260506 15:08:05.936 | INFO | Perf report for the last 5.13849 seconds: | 20260506 15:08:05.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:08:05.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:05.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:05.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:05.936 | INFO | ANALYSIS : 63.0%, 2.9, 95.4ms, 0 | 20260506 15:08:05.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:05.936 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260506 15:08:05.936 | INFO | PRE_HQ : 60.9%, 1.9, 59.2ms, 0 | 20260506 15:08:05.936 | INFO | BAZWRITER : 68.2%, 4.8, 167.8ms, 0 | 20260506 15:08:05.936 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 15:08:05.936 | INFO | Uploaded 36GiB to GPU in 2.05588s (17.5108GiB/s) | 20260506 15:08:05.936 | INFO | Downloaded 38GiB to GPU in 2.22628s (17.0689GiB/s) | 20260506 15:08:11.075 | INFO | Analyzing chunk frames = [2310656,2311168) | 20260506 15:08:11.075 | INFO | MemInfo: { | 20260506 15:08:11.075 | INFO | "gpu" : " 17 GB", | 20260506 15:08:11.075 | INFO | "huge" : "", | 20260506 15:08:11.075 | INFO | "huge_shared" : " 27 GB", | 20260506 15:08:11.075 | INFO | "malloc" : " 14 GB", | 20260506 15:08:11.075 | INFO | "pinned" : " 29 GB" | 20260506 15:08:11.075 | INFO | } | 20260506 15:08:11.075 | INFO | Perf report for the last 5.13926 seconds: | 20260506 15:08:11.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:08:11.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:11.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:11.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:11.075 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:08:11.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:11.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:11.075 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:08:11.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:11.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:11.075 | INFO | Uploaded 36GiB to GPU in 1.99604s (18.0357GiB/s) | 20260506 15:08:11.075 | INFO | Downloaded 34GiB to GPU in 1.94274s (17.5011GiB/s) | 20260506 15:08:16.213 | INFO | Analyzing chunk frames = [2311168,2311680) | 20260506 15:08:16.213 | INFO | MemInfo: { | 20260506 15:08:16.213 | INFO | "gpu" : " 17 GB", | 20260506 15:08:16.213 | INFO | "huge" : "", | 20260506 15:08:16.213 | INFO | "huge_shared" : " 27 GB", | 20260506 15:08:16.213 | INFO | "malloc" : " 14 GB", | 20260506 15:08:16.213 | INFO | "pinned" : " 29 GB" | 20260506 15:08:16.213 | INFO | } | 20260506 15:08:16.213 | INFO | Perf report for the last 5.13849 seconds: | 20260506 15:08:16.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:08:16.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:16.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:16.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:16.214 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:08:16.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:16.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:16.214 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:08:16.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:16.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:16.214 | INFO | Uploaded 36GiB to GPU in 1.99579s (18.038GiB/s) | 20260506 15:08:16.214 | INFO | Downloaded 34GiB to GPU in 1.93795s (17.5443GiB/s) | 20260506 15:08:21.353 | INFO | Analyzing chunk frames = [2311680,2312192) | 20260506 15:08:21.353 | INFO | MemInfo: { | 20260506 15:08:21.353 | INFO | "gpu" : " 17 GB", | 20260506 15:08:21.353 | INFO | "huge" : "", | 20260506 15:08:21.353 | INFO | "huge_shared" : " 27 GB", | 20260506 15:08:21.353 | INFO | "malloc" : " 14 GB", | 20260506 15:08:21.353 | INFO | "pinned" : " 29 GB" | 20260506 15:08:21.353 | INFO | } | 20260506 15:08:21.353 | INFO | Perf report for the last 5.13979 seconds: | 20260506 15:08:21.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:08:21.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:21.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:21.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:21.353 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:08:21.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:21.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:21.353 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:08:21.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:21.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:21.354 | INFO | Uploaded 36GiB to GPU in 2.00772s (17.9308GiB/s) | 20260506 15:08:21.354 | INFO | Downloaded 34GiB to GPU in 1.91658s (17.7399GiB/s) | 20260506 15:08:26.493 | INFO | Analyzing chunk frames = [2312192,2312704) | 20260506 15:08:26.493 | INFO | MemInfo: { | 20260506 15:08:26.493 | INFO | "gpu" : " 17 GB", | 20260506 15:08:26.493 | INFO | "huge" : "", | 20260506 15:08:26.493 | INFO | "huge_shared" : " 27 GB", | 20260506 15:08:26.493 | INFO | "malloc" : " 14 GB", | 20260506 15:08:26.493 | INFO | "pinned" : " 29 GB" | 20260506 15:08:26.493 | INFO | } | 20260506 15:08:26.493 | INFO | Perf report for the last 5.13999 seconds: | 20260506 15:08:26.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:08:26.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:26.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:26.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:26.493 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 15:08:26.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:26.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:26.493 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 15:08:26.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:26.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:26.493 | INFO | Uploaded 36GiB to GPU in 1.99803s (18.0178GiB/s) | 20260506 15:08:26.493 | INFO | Downloaded 34GiB to GPU in 1.94293s (17.4994GiB/s) | 20260506 15:08:31.632 | INFO | Analyzing chunk frames = [2312704,2313216) | 20260506 15:08:31.632 | INFO | MemInfo: { | 20260506 15:08:31.632 | INFO | "gpu" : " 17 GB", | 20260506 15:08:31.632 | INFO | "huge" : "", | 20260506 15:08:31.632 | INFO | "huge_shared" : " 27 GB", | 20260506 15:08:31.632 | INFO | "malloc" : " 14 GB", | 20260506 15:08:31.633 | INFO | "pinned" : " 29 GB" | 20260506 15:08:31.633 | INFO | } | 20260506 15:08:31.633 | INFO | Perf report for the last 5.13941 seconds: | 20260506 15:08:31.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:08:31.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:31.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:31.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:31.633 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 15:08:31.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:31.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:31.633 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260506 15:08:31.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:31.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:31.633 | INFO | Uploaded 36GiB to GPU in 2.06705s (17.4161GiB/s) | 20260506 15:08:31.633 | INFO | Downloaded 34GiB to GPU in 1.95609s (17.3816GiB/s) | 20260506 15:08:36.773 | INFO | Analyzing chunk frames = [2313216,2313728) | 20260506 15:08:36.773 | INFO | MemInfo: { | 20260506 15:08:36.773 | INFO | "gpu" : " 17 GB", | 20260506 15:08:36.773 | INFO | "huge" : "", | 20260506 15:08:36.773 | INFO | "huge_shared" : " 27 GB", | 20260506 15:08:36.773 | INFO | "malloc" : " 14 GB", | 20260506 15:08:36.773 | INFO | "pinned" : " 29 GB" | 20260506 15:08:36.773 | INFO | } | 20260506 15:08:36.773 | INFO | Perf report for the last 5.14063 seconds: | 20260506 15:08:36.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:08:36.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:36.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:36.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:36.773 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:08:36.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:36.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:36.773 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 15:08:36.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:36.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:36.773 | INFO | Uploaded 36GiB to GPU in 2.01713s (17.8471GiB/s) | 20260506 15:08:36.774 | INFO | Downloaded 34GiB to GPU in 1.99417s (17.0497GiB/s) | 20260506 15:08:41.912 | INFO | Analyzing chunk frames = [2313728,2314240) | 20260506 15:08:41.912 | INFO | MemInfo: { | 20260506 15:08:41.912 | INFO | "gpu" : " 17 GB", | 20260506 15:08:41.912 | INFO | "huge" : "", | 20260506 15:08:41.912 | INFO | "huge_shared" : " 27 GB", | 20260506 15:08:41.912 | INFO | "malloc" : " 14 GB", | 20260506 15:08:41.912 | INFO | "pinned" : " 29 GB" | 20260506 15:08:41.912 | INFO | } | 20260506 15:08:41.912 | INFO | Perf report for the last 5.1386 seconds: | 20260506 15:08:41.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:08:41.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:41.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:41.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:41.912 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:08:41.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:41.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:41.912 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:08:41.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:41.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:41.912 | INFO | Uploaded 36GiB to GPU in 2.02977s (17.736GiB/s) | 20260506 15:08:41.912 | INFO | Downloaded 34GiB to GPU in 1.98303s (17.1455GiB/s) | 20260506 15:08:42.024 | INFO | Starting a round of Baz IO | 20260506 15:08:45.178 | INFO | Populating RTMetrics report | 20260506 15:08:45.178 | INFO | RTMetrics report populated | 20260506 15:08:45.178 | INFO | Opening tmp RTMetrics json file | 20260506 15:08:45.208 | INFO | Serializing RTMetrics to json | 20260506 15:08:45.232 | INFO | Serialization done, swapping tmp file now... | 20260506 15:08:45.233 | INFO | RTMetrics done | 20260506 15:08:45.401 | INFO | Finished a round of Baz IO: Wrote 2.103 GiB in 3.377 seconds | 20260506 15:08:45.401 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.51% / 0.01% / 62.48% | 20260506 15:08:45.401 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.19641% / 1.58946e-05% / 2.78155e-05% | 20260506 15:08:45.401 | INFO | Baz IO overall: | 20260506 15:08:45.402 | INFO | BytesWritten : 1203233048396 | 20260506 15:08:45.402 | INFO | NumEvents : 582305199529 | 20260506 15:08:47.052 | INFO | Analyzing chunk frames = [2314240,2314752) | 20260506 15:08:47.052 | INFO | MemInfo: { | 20260506 15:08:47.052 | INFO | "gpu" : " 17 GB", | 20260506 15:08:47.052 | INFO | "huge" : "", | 20260506 15:08:47.052 | INFO | "huge_shared" : " 27 GB", | 20260506 15:08:47.052 | INFO | "malloc" : " 14 GB", | 20260506 15:08:47.052 | INFO | "pinned" : " 29 GB" | 20260506 15:08:47.052 | INFO | } | 20260506 15:08:47.052 | INFO | Perf report for the last 5.13985 seconds: | 20260506 15:08:47.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:08:47.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:47.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:47.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:47.052 | INFO | ANALYSIS : 62.9%, 2.9, 94.9ms, 0 | 20260506 15:08:47.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:47.052 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260506 15:08:47.052 | INFO | PRE_HQ : 61.1%, 1.9, 60.0ms, 0 | 20260506 15:08:47.052 | INFO | BAZWRITER : 65.1%, 4.8, 160.8ms, 0 | 20260506 15:08:47.052 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 15:08:47.052 | INFO | Uploaded 36GiB to GPU in 2.0574s (17.4978GiB/s) | 20260506 15:08:47.052 | INFO | Downloaded 38GiB to GPU in 2.2217s (17.104GiB/s) | 20260506 15:08:52.189 | INFO | Analyzing chunk frames = [2314752,2315264) | 20260506 15:08:52.189 | INFO | MemInfo: { | 20260506 15:08:52.189 | INFO | "gpu" : " 17 GB", | 20260506 15:08:52.189 | INFO | "huge" : "", | 20260506 15:08:52.189 | INFO | "huge_shared" : " 27 GB", | 20260506 15:08:52.189 | INFO | "malloc" : " 14 GB", | 20260506 15:08:52.189 | INFO | "pinned" : " 29 GB" | 20260506 15:08:52.189 | INFO | } | 20260506 15:08:52.189 | INFO | Perf report for the last 5.13769 seconds: | 20260506 15:08:52.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:08:52.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:52.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:52.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:52.189 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 15:08:52.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:52.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:52.189 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:08:52.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:52.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:52.190 | INFO | Uploaded 36GiB to GPU in 1.99961s (18.0035GiB/s) | 20260506 15:08:52.190 | INFO | Downloaded 34GiB to GPU in 1.93669s (17.5557GiB/s) | 20260506 15:08:57.328 | INFO | Analyzing chunk frames = [2315264,2315776) | 20260506 15:08:57.328 | INFO | MemInfo: { | 20260506 15:08:57.328 | INFO | "gpu" : " 17 GB", | 20260506 15:08:57.328 | INFO | "huge" : "", | 20260506 15:08:57.329 | INFO | "huge_shared" : " 27 GB", | 20260506 15:08:57.329 | INFO | "malloc" : " 14 GB", | 20260506 15:08:57.329 | INFO | "pinned" : " 29 GB" | 20260506 15:08:57.329 | INFO | } | 20260506 15:08:57.329 | INFO | Perf report for the last 5.13934 seconds: | 20260506 15:08:57.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:08:57.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:57.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:57.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:57.329 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:08:57.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:57.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:57.329 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:08:57.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:57.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:08:57.329 | INFO | Uploaded 36GiB to GPU in 1.9776s (18.2039GiB/s) | 20260506 15:08:57.329 | INFO | Downloaded 34GiB to GPU in 1.93331s (17.5864GiB/s) | 20260506 15:09:02.469 | INFO | Analyzing chunk frames = [2315776,2316288) | 20260506 15:09:02.469 | INFO | MemInfo: { | 20260506 15:09:02.469 | INFO | "gpu" : " 17 GB", | 20260506 15:09:02.469 | INFO | "huge" : "", | 20260506 15:09:02.469 | INFO | "huge_shared" : " 27 GB", | 20260506 15:09:02.469 | INFO | "malloc" : " 14 GB", | 20260506 15:09:02.469 | INFO | "pinned" : " 29 GB" | 20260506 15:09:02.469 | INFO | } | 20260506 15:09:02.469 | INFO | Perf report for the last 5.14021 seconds: | 20260506 15:09:02.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:09:02.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:02.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:02.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:02.469 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:09:02.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:02.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:02.469 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:09:02.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:02.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:02.469 | INFO | Uploaded 36GiB to GPU in 2.02091s (17.8138GiB/s) | 20260506 15:09:02.469 | INFO | Downloaded 34GiB to GPU in 1.91685s (17.7375GiB/s) | 20260506 15:09:07.609 | INFO | Analyzing chunk frames = [2316288,2316800) | 20260506 15:09:07.609 | INFO | MemInfo: { | 20260506 15:09:07.609 | INFO | "gpu" : " 17 GB", | 20260506 15:09:07.609 | INFO | "huge" : "", | 20260506 15:09:07.609 | INFO | "huge_shared" : " 27 GB", | 20260506 15:09:07.609 | INFO | "malloc" : " 14 GB", | 20260506 15:09:07.609 | INFO | "pinned" : " 29 GB" | 20260506 15:09:07.609 | INFO | } | 20260506 15:09:07.609 | INFO | Perf report for the last 5.14046 seconds: | 20260506 15:09:07.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:09:07.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:07.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:07.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:07.610 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 15:09:07.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:07.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:07.610 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:09:07.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:07.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:07.610 | INFO | Uploaded 36GiB to GPU in 2.00038s (17.9966GiB/s) | 20260506 15:09:07.610 | INFO | Downloaded 34GiB to GPU in 1.92816s (17.6334GiB/s) | 20260506 15:09:12.747 | INFO | Analyzing chunk frames = [2316800,2317312) | 20260506 15:09:12.747 | INFO | MemInfo: { | 20260506 15:09:12.747 | INFO | "gpu" : " 17 GB", | 20260506 15:09:12.747 | INFO | "huge" : "", | 20260506 15:09:12.747 | INFO | "huge_shared" : " 27 GB", | 20260506 15:09:12.747 | INFO | "malloc" : " 14 GB", | 20260506 15:09:12.747 | INFO | "pinned" : " 29 GB" | 20260506 15:09:12.747 | INFO | } | 20260506 15:09:12.747 | INFO | Perf report for the last 5.13778 seconds: | 20260506 15:09:12.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:09:12.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:12.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:12.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:12.747 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:09:12.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:12.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:12.747 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:09:12.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:12.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:12.747 | INFO | Uploaded 36GiB to GPU in 2.07651s (17.3368GiB/s) | 20260506 15:09:12.748 | INFO | Downloaded 34GiB to GPU in 1.96535s (17.2997GiB/s) | 20260506 15:09:17.887 | INFO | Analyzing chunk frames = [2317312,2317824) | 20260506 15:09:17.887 | INFO | MemInfo: { | 20260506 15:09:17.887 | INFO | "gpu" : " 17 GB", | 20260506 15:09:17.887 | INFO | "huge" : "", | 20260506 15:09:17.887 | INFO | "huge_shared" : " 27 GB", | 20260506 15:09:17.887 | INFO | "malloc" : " 14 GB", | 20260506 15:09:17.887 | INFO | "pinned" : " 29 GB" | 20260506 15:09:17.887 | INFO | } | 20260506 15:09:17.888 | INFO | Perf report for the last 5.14026 seconds: | 20260506 15:09:17.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:09:17.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:17.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:17.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:17.888 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:09:17.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:17.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:17.888 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 15:09:17.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:17.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:17.888 | INFO | Uploaded 36GiB to GPU in 2.0188s (17.8324GiB/s) | 20260506 15:09:17.888 | INFO | Downloaded 34GiB to GPU in 2.00598s (16.9493GiB/s) | 20260506 15:09:23.026 | INFO | Analyzing chunk frames = [2317824,2318336) | 20260506 15:09:23.026 | INFO | MemInfo: { | 20260506 15:09:23.026 | INFO | "gpu" : " 17 GB", | 20260506 15:09:23.026 | INFO | "huge" : "", | 20260506 15:09:23.026 | INFO | "huge_shared" : " 27 GB", | 20260506 15:09:23.026 | INFO | "malloc" : " 14 GB", | 20260506 15:09:23.026 | INFO | "pinned" : " 29 GB" | 20260506 15:09:23.026 | INFO | } | 20260506 15:09:23.026 | INFO | Perf report for the last 5.1381 seconds: | 20260506 15:09:23.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:09:23.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:23.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:23.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:23.026 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:09:23.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:23.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:23.026 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:09:23.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:23.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:23.026 | INFO | Uploaded 36GiB to GPU in 2.02952s (17.7382GiB/s) | 20260506 15:09:23.026 | INFO | Downloaded 34GiB to GPU in 1.96405s (17.3112GiB/s) | 20260506 15:09:23.026 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:09:23.026 | INFO | Total mem usage: 17.4 GiB | 20260506 15:09:23.026 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:09:23.026 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -8.02%) | 20260506 15:09:23.026 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:09:23.026 | INFO | High Water Breakdown: | 20260506 15:09:23.026 | INFO | 61.8% : 9.95 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:09:23.026 | INFO | 38.1% : 6.14 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:09:23.026 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:09:23.026 | INFO | ------------------------------------------ | 20260506 15:09:23.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:09:23.026 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:09:23.026 | INFO | Total mem usage: 26.5 GiB | 20260506 15:09:23.026 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:09:23.026 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:09:23.026 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:09:23.026 | INFO | High Water Breakdown: | 20260506 15:09:23.026 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:09:23.026 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:09:23.026 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:09:23.026 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:09:23.027 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:09:23.027 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:09:23.027 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:09:23.027 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:09:23.027 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:09:23.027 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:09:23.027 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:09:23.027 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:09:23.027 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:09:23.027 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:09:23.027 | INFO | ------------------------------------------ | 20260506 15:09:23.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:09:23.027 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:09:23.027 | INFO | Total mem usage: 25.6 GiB | 20260506 15:09:23.027 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:09:23.027 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:09:23.027 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:09:23.027 | INFO | High Water Breakdown: | 20260506 15:09:23.027 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:09:23.027 | INFO | ------------------------------------------ | 20260506 15:09:23.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:09:23.027 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:09:23.027 | INFO | Total mem usage: 15.7 GiB | 20260506 15:09:23.027 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:09:23.027 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:09:23.027 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 15:09:23.027 | INFO | High Water Breakdown: | 20260506 15:09:23.027 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:09:23.027 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:09:23.027 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:09:23.027 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:09:23.027 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:09:23.027 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:09:23.027 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:09:23.027 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:09:23.027 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:09:23.027 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:09:23.027 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:09:23.027 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:09:23.027 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:09:23.027 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:09:23.027 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:09:23.027 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:09:23.027 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:09:23.027 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:09:23.027 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:09:23.027 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:09:23.027 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:09:23.027 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:09:23.027 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:09:23.027 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:09:23.027 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:09:23.027 | INFO | ------------------------------------------ | 20260506 15:09:23.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:09:23.027 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:09:23.027 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:09:23.027 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:09:23.027 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.58e+01, 1.35e+01 / 1.94e+00 | 20260506 15:09:23.027 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.00e-01, 1.02e-02 / 5.77e-03 | 20260506 15:09:23.027 | INFO | PerfReport: 11.57%, Baseline -- 3.64e+00 / 4.42e+00, 5.56e-01 / 3.67e-01 | 20260506 15:09:23.027 | INFO | PerfReport: 3.23%, Histogram -- 1.01e+00 / 1.49e+00, 1.81e-01 / 1.27e-01 | 20260506 15:09:23.027 | INFO | PerfReport: 15.93%, DME -- 4.00e+01 / 4.46e+01, 4.04e+00 / 2.16e+00 | 20260506 15:09:23.027 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.41e+00 / 1.92e+00 | 20260506 15:09:23.027 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.33e+00 / 2.91e+00, 4.06e-01 / 2.60e-01 | 20260506 15:09:23.027 | INFO | PerfReport: 3.57%, Metrics -- 1.12e+00 / 1.49e+00, 2.88e-01 / 1.98e-01 | 20260506 15:09:23.027 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.91e-01 / 4.42e-01, 6.12e-02 / 2.71e-02 | 20260506 15:09:23.027 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:09:23.027 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:09:23.027 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:09:23.027 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 15:09:23.027 | INFO | PerfReport: 21.57%, Upload -- 2.11e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260506 15:09:23.027 | INFO | PerfReport: 21.19%, Download -- 2.07e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 15:09:23.027 | INFO | PerfReport: 32.19%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 15:09:23.027 | INFO | PerfReport: 1.55%, LockUpload -- 1.52e+00 / 0.00e+00, 5.82e+00 / 0.00e+00 | 20260506 15:09:23.027 | INFO | PerfReport: 1.11%, LockDownload -- 1.08e+00 / 0.00e+00, 2.68e+00 / 0.00e+00 | 20260506 15:09:23.027 | INFO | PerfReport: 22.37%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 15:09:23.027 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.84e-02 / 0.00e+00 | 20260506 15:09:23.136 | INFO | Starting a round of Baz IO | 20260506 15:09:26.291 | INFO | Populating RTMetrics report | 20260506 15:09:26.291 | INFO | RTMetrics report populated | 20260506 15:09:26.291 | INFO | Opening tmp RTMetrics json file | 20260506 15:09:26.294 | INFO | Serializing RTMetrics to json | 20260506 15:09:26.298 | INFO | Serialization done, swapping tmp file now... | 20260506 15:09:26.299 | INFO | RTMetrics done | 20260506 15:09:26.972 | INFO | Finished a round of Baz IO: Wrote 2.105 GiB in 3.836 seconds | 20260506 15:09:26.972 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.50% / 0.01% / 62.49% | 20260506 15:09:26.972 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.19539% / 2.78155e-05% / 6.35783e-05% | 20260506 15:09:26.972 | INFO | Baz IO overall: | 20260506 15:09:26.972 | INFO | BytesWritten : 1205493287313 | 20260506 15:09:26.972 | INFO | NumEvents : 583399876420 | 20260506 15:09:28.165 | INFO | Analyzing chunk frames = [2318336,2318848) | 20260506 15:09:28.165 | INFO | MemInfo: { | 20260506 15:09:28.165 | INFO | "gpu" : " 17 GB", | 20260506 15:09:28.165 | INFO | "huge" : "", | 20260506 15:09:28.165 | INFO | "huge_shared" : " 27 GB", | 20260506 15:09:28.165 | INFO | "malloc" : " 14 GB", | 20260506 15:09:28.165 | INFO | "pinned" : " 29 GB" | 20260506 15:09:28.165 | INFO | } | 20260506 15:09:28.165 | INFO | Perf report for the last 5.13939 seconds: | 20260506 15:09:28.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:09:28.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:28.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:28.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:28.165 | INFO | ANALYSIS : 62.9%, 2.9, 95.4ms, 0 | 20260506 15:09:28.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:28.165 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 15:09:28.165 | INFO | PRE_HQ : 61.0%, 1.9, 59.4ms, 0 | 20260506 15:09:28.165 | INFO | BAZWRITER : 74.0%, 4.8, 182.6ms, 0 | 20260506 15:09:28.165 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 15:09:28.166 | INFO | Uploaded 36GiB to GPU in 2.0653s (17.4309GiB/s) | 20260506 15:09:28.166 | INFO | Downloaded 38GiB to GPU in 2.22157s (17.105GiB/s) | 20260506 15:09:33.304 | INFO | Analyzing chunk frames = [2318848,2319360) | 20260506 15:09:33.304 | INFO | MemInfo: { | 20260506 15:09:33.304 | INFO | "gpu" : " 17 GB", | 20260506 15:09:33.304 | INFO | "huge" : "", | 20260506 15:09:33.304 | INFO | "huge_shared" : " 27 GB", | 20260506 15:09:33.304 | INFO | "malloc" : " 14 GB", | 20260506 15:09:33.304 | INFO | "pinned" : " 29 GB" | 20260506 15:09:33.304 | INFO | } | 20260506 15:09:33.305 | INFO | Perf report for the last 5.13966 seconds: | 20260506 15:09:33.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:09:33.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:33.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:33.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:33.305 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:09:33.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:33.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:33.305 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:09:33.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:33.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:33.305 | INFO | Uploaded 36GiB to GPU in 1.99437s (18.0508GiB/s) | 20260506 15:09:33.305 | INFO | Downloaded 34GiB to GPU in 1.93303s (17.589GiB/s) | 20260506 15:09:38.444 | INFO | Analyzing chunk frames = [2319360,2319872) | 20260506 15:09:38.444 | INFO | MemInfo: { | 20260506 15:09:38.444 | INFO | "gpu" : " 17 GB", | 20260506 15:09:38.444 | INFO | "huge" : "", | 20260506 15:09:38.444 | INFO | "huge_shared" : " 27 GB", | 20260506 15:09:38.444 | INFO | "malloc" : " 14 GB", | 20260506 15:09:38.444 | INFO | "pinned" : " 29 GB" | 20260506 15:09:38.444 | INFO | } | 20260506 15:09:38.445 | INFO | Perf report for the last 5.13983 seconds: | 20260506 15:09:38.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:09:38.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:38.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:38.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:38.445 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:09:38.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:38.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:38.445 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:09:38.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:38.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:38.445 | INFO | Uploaded 36GiB to GPU in 1.99663s (18.0304GiB/s) | 20260506 15:09:38.445 | INFO | Downloaded 34GiB to GPU in 1.94049s (17.5213GiB/s) | 20260506 15:09:43.583 | INFO | Analyzing chunk frames = [2319872,2320384) | 20260506 15:09:43.583 | INFO | MemInfo: { | 20260506 15:09:43.583 | INFO | "gpu" : " 17 GB", | 20260506 15:09:43.583 | INFO | "huge" : "", | 20260506 15:09:43.583 | INFO | "huge_shared" : " 27 GB", | 20260506 15:09:43.583 | INFO | "malloc" : " 14 GB", | 20260506 15:09:43.583 | INFO | "pinned" : " 29 GB" | 20260506 15:09:43.583 | INFO | } | 20260506 15:09:43.583 | INFO | Perf report for the last 5.13889 seconds: | 20260506 15:09:43.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:09:43.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:43.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:43.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:43.584 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 15:09:43.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:43.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:43.584 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:09:43.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:43.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:43.584 | INFO | Uploaded 36GiB to GPU in 2.01053s (17.9058GiB/s) | 20260506 15:09:43.584 | INFO | Downloaded 34GiB to GPU in 1.9121s (17.7815GiB/s) | 20260506 15:09:48.723 | INFO | Analyzing chunk frames = [2320384,2320896) | 20260506 15:09:48.723 | INFO | MemInfo: { | 20260506 15:09:48.723 | INFO | "gpu" : " 17 GB", | 20260506 15:09:48.723 | INFO | "huge" : "", | 20260506 15:09:48.723 | INFO | "huge_shared" : " 27 GB", | 20260506 15:09:48.723 | INFO | "malloc" : " 14 GB", | 20260506 15:09:48.723 | INFO | "pinned" : " 29 GB" | 20260506 15:09:48.723 | INFO | } | 20260506 15:09:48.723 | INFO | Perf report for the last 5.13965 seconds: | 20260506 15:09:48.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:09:48.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:48.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:48.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:48.723 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 15:09:48.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:48.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:48.723 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:09:48.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:48.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:48.723 | INFO | Uploaded 36GiB to GPU in 1.99763s (18.0214GiB/s) | 20260506 15:09:48.723 | INFO | Downloaded 34GiB to GPU in 1.93836s (17.5406GiB/s) | 20260506 15:09:53.863 | INFO | Analyzing chunk frames = [2320896,2321408) | 20260506 15:09:53.863 | INFO | MemInfo: { | 20260506 15:09:53.863 | INFO | "gpu" : " 17 GB", | 20260506 15:09:53.863 | INFO | "huge" : "", | 20260506 15:09:53.864 | INFO | "huge_shared" : " 27 GB", | 20260506 15:09:53.864 | INFO | "malloc" : " 14 GB", | 20260506 15:09:53.864 | INFO | "pinned" : " 29 GB" | 20260506 15:09:53.864 | INFO | } | 20260506 15:09:53.864 | INFO | Perf report for the last 5.14063 seconds: | 20260506 15:09:53.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:09:53.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:53.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:53.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:53.864 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:09:53.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:53.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:53.864 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 15:09:53.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:53.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:53.864 | INFO | Uploaded 36GiB to GPU in 2.07664s (17.3357GiB/s) | 20260506 15:09:53.864 | INFO | Downloaded 34GiB to GPU in 1.96694s (17.2857GiB/s) | 20260506 15:09:59.002 | INFO | Analyzing chunk frames = [2321408,2321920) | 20260506 15:09:59.002 | INFO | MemInfo: { | 20260506 15:09:59.002 | INFO | "gpu" : " 17 GB", | 20260506 15:09:59.002 | INFO | "huge" : "", | 20260506 15:09:59.002 | INFO | "huge_shared" : " 27 GB", | 20260506 15:09:59.002 | INFO | "malloc" : " 14 GB", | 20260506 15:09:59.002 | INFO | "pinned" : " 29 GB" | 20260506 15:09:59.002 | INFO | } | 20260506 15:09:59.002 | INFO | Perf report for the last 5.1383 seconds: | 20260506 15:09:59.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:09:59.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:59.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:59.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:59.002 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:09:59.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:59.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:59.002 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:09:59.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:59.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:09:59.002 | INFO | Uploaded 36GiB to GPU in 2.00572s (17.9487GiB/s) | 20260506 15:09:59.003 | INFO | Downloaded 34GiB to GPU in 2.01297s (16.8905GiB/s) | 20260506 15:10:04.141 | INFO | Analyzing chunk frames = [2321920,2322432) | 20260506 15:10:04.141 | INFO | MemInfo: { | 20260506 15:10:04.141 | INFO | "gpu" : " 17 GB", | 20260506 15:10:04.141 | INFO | "huge" : "", | 20260506 15:10:04.141 | INFO | "huge_shared" : " 27 GB", | 20260506 15:10:04.141 | INFO | "malloc" : " 14 GB", | 20260506 15:10:04.141 | INFO | "pinned" : " 29 GB" | 20260506 15:10:04.141 | INFO | } | 20260506 15:10:04.141 | INFO | Perf report for the last 5.13915 seconds: | 20260506 15:10:04.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:10:04.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:04.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:04.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:04.141 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 15:10:04.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:04.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:04.141 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:10:04.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:04.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:04.142 | INFO | Uploaded 36GiB to GPU in 2.03197s (17.7168GiB/s) | 20260506 15:10:04.142 | INFO | Downloaded 34GiB to GPU in 1.98357s (17.1408GiB/s) | 20260506 15:10:04.253 | INFO | Starting a round of Baz IO | 20260506 15:10:07.414 | INFO | Populating RTMetrics report | 20260506 15:10:07.414 | INFO | RTMetrics report populated | 20260506 15:10:07.414 | INFO | Opening tmp RTMetrics json file | 20260506 15:10:07.438 | INFO | Serializing RTMetrics to json | 20260506 15:10:07.447 | INFO | Serialization done, swapping tmp file now... | 20260506 15:10:07.466 | INFO | RTMetrics done | 20260506 15:10:07.644 | INFO | Finished a round of Baz IO: Wrote 2.102 GiB in 3.391 seconds | 20260506 15:10:07.645 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.49% / 0.01% / 62.50% | 20260506 15:10:07.645 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.18215% / 3.97364e-06% / 6.35783e-05% | 20260506 15:10:07.645 | INFO | Baz IO overall: | 20260506 15:10:07.645 | INFO | BytesWritten : 1207750423995 | 20260506 15:10:07.645 | INFO | NumEvents : 584492112102 | 20260506 15:10:09.280 | INFO | Analyzing chunk frames = [2322432,2322944) | 20260506 15:10:09.280 | INFO | MemInfo: { | 20260506 15:10:09.280 | INFO | "gpu" : " 17 GB", | 20260506 15:10:09.280 | INFO | "huge" : "", | 20260506 15:10:09.280 | INFO | "huge_shared" : " 27 GB", | 20260506 15:10:09.280 | INFO | "malloc" : " 14 GB", | 20260506 15:10:09.280 | INFO | "pinned" : " 29 GB" | 20260506 15:10:09.280 | INFO | } | 20260506 15:10:09.280 | INFO | Perf report for the last 5.139 seconds: | 20260506 15:10:09.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:10:09.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:09.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:09.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:09.280 | INFO | ANALYSIS : 62.9%, 2.9, 95.4ms, 0 | 20260506 15:10:09.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:09.280 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260506 15:10:09.280 | INFO | PRE_HQ : 60.8%, 1.9, 59.5ms, 0 | 20260506 15:10:09.280 | INFO | BAZWRITER : 65.1%, 4.8, 161.7ms, 0 | 20260506 15:10:09.280 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 15:10:09.280 | INFO | Uploaded 36GiB to GPU in 2.07275s (17.3683GiB/s) | 20260506 15:10:09.281 | INFO | Downloaded 38GiB to GPU in 2.23616s (16.9934GiB/s) | 20260506 15:10:14.419 | INFO | Analyzing chunk frames = [2322944,2323456) | 20260506 15:10:14.419 | INFO | MemInfo: { | 20260506 15:10:14.419 | INFO | "gpu" : " 17 GB", | 20260506 15:10:14.419 | INFO | "huge" : "", | 20260506 15:10:14.419 | INFO | "huge_shared" : " 27 GB", | 20260506 15:10:14.419 | INFO | "malloc" : " 14 GB", | 20260506 15:10:14.419 | INFO | "pinned" : " 29 GB" | 20260506 15:10:14.419 | INFO | } | 20260506 15:10:14.419 | INFO | Perf report for the last 5.13911 seconds: | 20260506 15:10:14.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:10:14.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:14.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:14.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:14.419 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:10:14.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:14.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:14.419 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:10:14.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:14.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:14.420 | INFO | Uploaded 36GiB to GPU in 1.99754s (18.0221GiB/s) | 20260506 15:10:14.420 | INFO | Downloaded 34GiB to GPU in 1.92445s (17.6674GiB/s) | 20260506 15:10:19.558 | INFO | Analyzing chunk frames = [2323456,2323968) | 20260506 15:10:19.558 | INFO | MemInfo: { | 20260506 15:10:19.558 | INFO | "gpu" : " 17 GB", | 20260506 15:10:19.558 | INFO | "huge" : "", | 20260506 15:10:19.558 | INFO | "huge_shared" : " 27 GB", | 20260506 15:10:19.558 | INFO | "malloc" : " 14 GB", | 20260506 15:10:19.558 | INFO | "pinned" : " 29 GB" | 20260506 15:10:19.558 | INFO | } | 20260506 15:10:19.558 | INFO | Perf report for the last 5.13883 seconds: | 20260506 15:10:19.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:10:19.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:19.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:19.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:19.558 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 15:10:19.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:19.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:19.558 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:10:19.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:19.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:19.558 | INFO | Uploaded 36GiB to GPU in 1.99707s (18.0264GiB/s) | 20260506 15:10:19.559 | INFO | Downloaded 34GiB to GPU in 1.93422s (17.5782GiB/s) | 20260506 15:10:24.699 | INFO | Analyzing chunk frames = [2323968,2324480) | 20260506 15:10:24.699 | INFO | MemInfo: { | 20260506 15:10:24.699 | INFO | "gpu" : " 17 GB", | 20260506 15:10:24.699 | INFO | "huge" : "", | 20260506 15:10:24.699 | INFO | "huge_shared" : " 27 GB", | 20260506 15:10:24.699 | INFO | "malloc" : " 14 GB", | 20260506 15:10:24.699 | INFO | "pinned" : " 29 GB" | 20260506 15:10:24.699 | INFO | } | 20260506 15:10:24.699 | INFO | Perf report for the last 5.14109 seconds: | 20260506 15:10:24.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:10:24.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:24.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:24.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:24.699 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260506 15:10:24.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:24.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:24.699 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:10:24.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:24.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:24.700 | INFO | Uploaded 36GiB to GPU in 1.99962s (18.0034GiB/s) | 20260506 15:10:24.700 | INFO | Downloaded 34GiB to GPU in 1.91353s (17.7682GiB/s) | 20260506 15:10:29.837 | INFO | Analyzing chunk frames = [2324480,2324992) | 20260506 15:10:29.837 | INFO | MemInfo: { | 20260506 15:10:29.837 | INFO | "gpu" : " 17 GB", | 20260506 15:10:29.837 | INFO | "huge" : "", | 20260506 15:10:29.837 | INFO | "huge_shared" : " 27 GB", | 20260506 15:10:29.837 | INFO | "malloc" : " 14 GB", | 20260506 15:10:29.837 | INFO | "pinned" : " 29 GB" | 20260506 15:10:29.837 | INFO | } | 20260506 15:10:29.837 | INFO | Perf report for the last 5.13797 seconds: | 20260506 15:10:29.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:10:29.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:29.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:29.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:29.837 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 15:10:29.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:29.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:29.837 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:10:29.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:29.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:29.838 | INFO | Uploaded 36GiB to GPU in 2.01928s (17.8281GiB/s) | 20260506 15:10:29.838 | INFO | Downloaded 34GiB to GPU in 1.92355s (17.6757GiB/s) | 20260506 15:10:34.977 | INFO | Analyzing chunk frames = [2324992,2325504) | 20260506 15:10:34.977 | INFO | MemInfo: { | 20260506 15:10:34.977 | INFO | "gpu" : " 17 GB", | 20260506 15:10:34.977 | INFO | "huge" : "", | 20260506 15:10:34.977 | INFO | "huge_shared" : " 27 GB", | 20260506 15:10:34.977 | INFO | "malloc" : " 14 GB", | 20260506 15:10:34.977 | INFO | "pinned" : " 29 GB" | 20260506 15:10:34.977 | INFO | } | 20260506 15:10:34.977 | INFO | Perf report for the last 5.13952 seconds: | 20260506 15:10:34.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:10:34.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:34.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:34.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:34.977 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 15:10:34.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:34.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:34.977 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 15:10:34.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:34.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:34.977 | INFO | Uploaded 36GiB to GPU in 2.04379s (17.6144GiB/s) | 20260506 15:10:34.977 | INFO | Downloaded 34GiB to GPU in 1.92402s (17.6713GiB/s) | 20260506 15:10:40.118 | INFO | Analyzing chunk frames = [2325504,2326016) | 20260506 15:10:40.118 | INFO | MemInfo: { | 20260506 15:10:40.118 | INFO | "gpu" : " 17 GB", | 20260506 15:10:40.118 | INFO | "huge" : "", | 20260506 15:10:40.118 | INFO | "huge_shared" : " 27 GB", | 20260506 15:10:40.118 | INFO | "malloc" : " 14 GB", | 20260506 15:10:40.118 | INFO | "pinned" : " 29 GB" | 20260506 15:10:40.118 | INFO | } | 20260506 15:10:40.118 | INFO | Perf report for the last 5.14104 seconds: | 20260506 15:10:40.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:10:40.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:40.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:40.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:40.118 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:10:40.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:40.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:40.118 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:10:40.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:40.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:40.118 | INFO | Uploaded 36GiB to GPU in 2.00126s (17.9887GiB/s) | 20260506 15:10:40.118 | INFO | Downloaded 34GiB to GPU in 1.99182s (17.0698GiB/s) | 20260506 15:10:45.256 | INFO | Analyzing chunk frames = [2326016,2326528) | 20260506 15:10:45.256 | INFO | MemInfo: { | 20260506 15:10:45.256 | INFO | "gpu" : " 17 GB", | 20260506 15:10:45.256 | INFO | "huge" : "", | 20260506 15:10:45.256 | INFO | "huge_shared" : " 27 GB", | 20260506 15:10:45.256 | INFO | "malloc" : " 14 GB", | 20260506 15:10:45.256 | INFO | "pinned" : " 29 GB" | 20260506 15:10:45.256 | INFO | } | 20260506 15:10:45.256 | INFO | Perf report for the last 5.13844 seconds: | 20260506 15:10:45.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:10:45.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:45.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:45.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:45.256 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 15:10:45.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:45.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:45.256 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:10:45.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:45.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:45.257 | INFO | Uploaded 36GiB to GPU in 2.02896s (17.7431GiB/s) | 20260506 15:10:45.257 | INFO | Downloaded 34GiB to GPU in 1.97855s (17.1843GiB/s) | 20260506 15:10:45.257 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:10:45.257 | INFO | Total mem usage: 17.4 GiB | 20260506 15:10:45.257 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:10:45.257 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -7.72%) | 20260506 15:10:45.257 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:10:45.257 | INFO | High Water Breakdown: | 20260506 15:10:45.257 | INFO | 61.7% : 9.98 GiB ( 91.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:10:45.257 | INFO | 38.1% : 6.16 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:10:45.257 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:10:45.257 | INFO | ------------------------------------------ | 20260506 15:10:45.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:10:45.257 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:10:45.257 | INFO | Total mem usage: 26.5 GiB | 20260506 15:10:45.257 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:10:45.257 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:10:45.257 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:10:45.257 | INFO | High Water Breakdown: | 20260506 15:10:45.257 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:10:45.257 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:10:45.257 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:10:45.257 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:10:45.257 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:10:45.257 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:10:45.257 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:10:45.257 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:10:45.257 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:10:45.257 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:10:45.257 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:10:45.257 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:10:45.257 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:10:45.257 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:10:45.257 | INFO | ------------------------------------------ | 20260506 15:10:45.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:10:45.257 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:10:45.257 | INFO | Total mem usage: 25.6 GiB | 20260506 15:10:45.257 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:10:45.257 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:10:45.257 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:10:45.257 | INFO | High Water Breakdown: | 20260506 15:10:45.257 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:10:45.257 | INFO | ------------------------------------------ | 20260506 15:10:45.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:10:45.257 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:10:45.257 | INFO | Total mem usage: 15.7 GiB | 20260506 15:10:45.257 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:10:45.257 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:10:45.257 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 15:10:45.257 | INFO | High Water Breakdown: | 20260506 15:10:45.257 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:10:45.257 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:10:45.257 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:10:45.257 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:10:45.257 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:10:45.257 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:10:45.257 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:10:45.257 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:10:45.257 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:10:45.257 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:10:45.258 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:10:45.258 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:10:45.258 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:10:45.258 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:10:45.258 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:10:45.258 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:10:45.258 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:10:45.258 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:10:45.258 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:10:45.258 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:10:45.258 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:10:45.258 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:10:45.258 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:10:45.258 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:10:45.258 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:10:45.258 | INFO | ------------------------------------------ | 20260506 15:10:45.258 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:10:45.258 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:10:45.258 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:10:45.258 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:10:45.258 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.56e+01, 1.35e+01 / 1.70e+00 | 20260506 15:10:45.258 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.00e-01, 9.77e-03 / 5.74e-03 | 20260506 15:10:45.258 | INFO | PerfReport: 11.61%, Baseline -- 3.65e+00 / 4.44e+00, 5.65e-01 / 3.49e-01 | 20260506 15:10:45.258 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.48e+00, 1.88e-01 / 1.23e-01 | 20260506 15:10:45.258 | INFO | PerfReport: 15.89%, DME -- 4.00e+01 / 4.43e+01, 3.92e+00 / 1.65e+00 | 20260506 15:10:45.258 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.42e+00 / 1.63e+00 | 20260506 15:10:45.258 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.32e+00 / 2.89e+00, 3.96e-01 / 2.16e-01 | 20260506 15:10:45.258 | INFO | PerfReport: 3.57%, Metrics -- 1.12e+00 / 1.50e+00, 2.94e-01 / 1.87e-01 | 20260506 15:10:45.258 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.90e-01 / 4.47e-01, 6.08e-02 / 2.51e-02 | 20260506 15:10:45.258 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:10:45.258 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:10:45.258 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:10:45.258 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 15:10:45.258 | INFO | PerfReport: 21.56%, Upload -- 2.11e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 15:10:45.258 | INFO | PerfReport: 21.16%, Download -- 2.07e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 15:10:45.258 | INFO | PerfReport: 32.21%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 15:10:45.258 | INFO | PerfReport: 1.57%, LockUpload -- 1.54e+00 / 0.00e+00, 5.91e+00 / 0.00e+00 | 20260506 15:10:45.258 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.70e+00 / 0.00e+00 | 20260506 15:10:45.258 | INFO | PerfReport: 22.38%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 15:10:45.258 | INFO | PerfReport: 0.02%, Misc -- 1.51e-02 / 0.00e+00, 2.48e-02 / 0.00e+00 | 20260506 15:10:45.367 | INFO | Starting a round of Baz IO | 20260506 15:10:48.520 | INFO | Populating RTMetrics report | 20260506 15:10:48.520 | INFO | RTMetrics report populated | 20260506 15:10:48.520 | INFO | Opening tmp RTMetrics json file | 20260506 15:10:48.654 | INFO | Serializing RTMetrics to json | 20260506 15:10:48.711 | INFO | Serialization done, swapping tmp file now... | 20260506 15:10:48.716 | INFO | RTMetrics done | 20260506 15:10:49.059 | INFO | Finished a round of Baz IO: Wrote 2.104 GiB in 3.692 seconds | 20260506 15:10:49.060 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.48% / 0.01% / 62.51% | 20260506 15:10:49.060 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.18203% / 1.98682e-05% / 4.76837e-05% | 20260506 15:10:49.060 | INFO | Baz IO overall: | 20260506 15:10:49.060 | INFO | BytesWritten : 1210009318718 | 20260506 15:10:49.060 | INFO | NumEvents : 585585478707 | 20260506 15:10:50.395 | INFO | Analyzing chunk frames = [2326528,2327040) | 20260506 15:10:50.395 | INFO | MemInfo: { | 20260506 15:10:50.395 | INFO | "gpu" : " 17 GB", | 20260506 15:10:50.395 | INFO | "huge" : "", | 20260506 15:10:50.395 | INFO | "huge_shared" : " 27 GB", | 20260506 15:10:50.395 | INFO | "malloc" : " 14 GB", | 20260506 15:10:50.395 | INFO | "pinned" : " 29 GB" | 20260506 15:10:50.395 | INFO | } | 20260506 15:10:50.395 | INFO | Perf report for the last 5.13862 seconds: | 20260506 15:10:50.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:10:50.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:50.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:50.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:50.395 | INFO | ANALYSIS : 62.9%, 2.9, 95.3ms, 0 | 20260506 15:10:50.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:50.395 | INFO | RT_METRICS : 47.6%, 1.0, 24.9ms, 0 | 20260506 15:10:50.395 | INFO | PRE_HQ : 61.0%, 1.9, 59.3ms, 0 | 20260506 15:10:50.395 | INFO | BAZWRITER : 71.3%, 4.8, 174.7ms, 0 | 20260506 15:10:50.395 | INFO | BAZ_SAMPLING : 2.9%, 1.0, 1.5ms, 0 | 20260506 15:10:50.395 | INFO | Uploaded 36GiB to GPU in 2.07103s (17.3827GiB/s) | 20260506 15:10:50.395 | INFO | Downloaded 38GiB to GPU in 2.22844s (17.0523GiB/s) | 20260506 15:10:55.534 | INFO | Analyzing chunk frames = [2327040,2327552) | 20260506 15:10:55.534 | INFO | MemInfo: { | 20260506 15:10:55.534 | INFO | "gpu" : " 17 GB", | 20260506 15:10:55.534 | INFO | "huge" : "", | 20260506 15:10:55.534 | INFO | "huge_shared" : " 27 GB", | 20260506 15:10:55.534 | INFO | "malloc" : " 14 GB", | 20260506 15:10:55.534 | INFO | "pinned" : " 29 GB" | 20260506 15:10:55.534 | INFO | } | 20260506 15:10:55.535 | INFO | Perf report for the last 5.13953 seconds: | 20260506 15:10:55.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:10:55.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:55.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:55.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:55.535 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:10:55.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:55.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:55.535 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:10:55.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:55.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:10:55.535 | INFO | Uploaded 36GiB to GPU in 1.98954s (18.0946GiB/s) | 20260506 15:10:55.535 | INFO | Downloaded 34GiB to GPU in 1.954s (17.4002GiB/s) | 20260506 15:11:00.673 | INFO | Analyzing chunk frames = [2327552,2328064) | 20260506 15:11:00.673 | INFO | MemInfo: { | 20260506 15:11:00.673 | INFO | "gpu" : " 17 GB", | 20260506 15:11:00.673 | INFO | "huge" : "", | 20260506 15:11:00.673 | INFO | "huge_shared" : " 27 GB", | 20260506 15:11:00.673 | INFO | "malloc" : " 14 GB", | 20260506 15:11:00.673 | INFO | "pinned" : " 29 GB" | 20260506 15:11:00.673 | INFO | } | 20260506 15:11:00.673 | INFO | Perf report for the last 5.13859 seconds: | 20260506 15:11:00.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:11:00.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:00.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:00.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:00.673 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:11:00.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:00.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:00.673 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:11:00.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:00.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:00.673 | INFO | Uploaded 36GiB to GPU in 1.97177s (18.2577GiB/s) | 20260506 15:11:00.674 | INFO | Downloaded 34GiB to GPU in 1.9371s (17.552GiB/s) | 20260506 15:11:05.813 | INFO | Analyzing chunk frames = [2328064,2328576) | 20260506 15:11:05.813 | INFO | MemInfo: { | 20260506 15:11:05.813 | INFO | "gpu" : " 17 GB", | 20260506 15:11:05.813 | INFO | "huge" : "", | 20260506 15:11:05.813 | INFO | "huge_shared" : " 27 GB", | 20260506 15:11:05.813 | INFO | "malloc" : " 14 GB", | 20260506 15:11:05.813 | INFO | "pinned" : " 29 GB" | 20260506 15:11:05.813 | INFO | } | 20260506 15:11:05.813 | INFO | Perf report for the last 5.14022 seconds: | 20260506 15:11:05.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:11:05.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:05.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:05.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:05.813 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 15:11:05.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:05.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:05.813 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:11:05.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:05.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:05.814 | INFO | Uploaded 36GiB to GPU in 1.99527s (18.0427GiB/s) | 20260506 15:11:05.814 | INFO | Downloaded 34GiB to GPU in 1.91549s (17.7501GiB/s) | 20260506 15:11:10.953 | INFO | Analyzing chunk frames = [2328576,2329088) | 20260506 15:11:10.953 | INFO | MemInfo: { | 20260506 15:11:10.953 | INFO | "gpu" : " 17 GB", | 20260506 15:11:10.953 | INFO | "huge" : "", | 20260506 15:11:10.953 | INFO | "huge_shared" : " 27 GB", | 20260506 15:11:10.954 | INFO | "malloc" : " 14 GB", | 20260506 15:11:10.954 | INFO | "pinned" : " 29 GB" | 20260506 15:11:10.954 | INFO | } | 20260506 15:11:10.954 | INFO | Perf report for the last 5.14032 seconds: | 20260506 15:11:10.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:11:10.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:10.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:10.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:10.954 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 15:11:10.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:10.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:10.954 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:11:10.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:10.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:10.954 | INFO | Uploaded 36GiB to GPU in 1.99986s (18.0012GiB/s) | 20260506 15:11:10.954 | INFO | Downloaded 34GiB to GPU in 1.93325s (17.587GiB/s) | 20260506 15:11:16.092 | INFO | Analyzing chunk frames = [2329088,2329600) | 20260506 15:11:16.092 | INFO | MemInfo: { | 20260506 15:11:16.092 | INFO | "gpu" : " 17 GB", | 20260506 15:11:16.092 | INFO | "huge" : "", | 20260506 15:11:16.092 | INFO | "huge_shared" : " 27 GB", | 20260506 15:11:16.092 | INFO | "malloc" : " 14 GB", | 20260506 15:11:16.092 | INFO | "pinned" : " 29 GB" | 20260506 15:11:16.092 | INFO | } | 20260506 15:11:16.093 | INFO | Perf report for the last 5.13891 seconds: | 20260506 15:11:16.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:11:16.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:16.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:16.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:16.093 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260506 15:11:16.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:16.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:16.093 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 15:11:16.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:16.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:16.093 | INFO | Uploaded 36GiB to GPU in 2.03013s (17.7328GiB/s) | 20260506 15:11:16.093 | INFO | Downloaded 34GiB to GPU in 1.92903s (17.6254GiB/s) | 20260506 15:11:21.231 | INFO | Analyzing chunk frames = [2329600,2330112) | 20260506 15:11:21.231 | INFO | MemInfo: { | 20260506 15:11:21.231 | INFO | "gpu" : " 17 GB", | 20260506 15:11:21.231 | INFO | "huge" : "", | 20260506 15:11:21.231 | INFO | "huge_shared" : " 27 GB", | 20260506 15:11:21.231 | INFO | "malloc" : " 14 GB", | 20260506 15:11:21.231 | INFO | "pinned" : " 29 GB" | 20260506 15:11:21.231 | INFO | } | 20260506 15:11:21.231 | INFO | Perf report for the last 5.13823 seconds: | 20260506 15:11:21.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:11:21.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:21.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:21.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:21.231 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:11:21.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:21.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:21.231 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 15:11:21.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:21.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:21.231 | INFO | Uploaded 36GiB to GPU in 2.00208s (17.9813GiB/s) | 20260506 15:11:21.231 | INFO | Downloaded 34GiB to GPU in 1.99371s (17.0536GiB/s) | 20260506 15:11:26.370 | INFO | Analyzing chunk frames = [2330112,2330624) | 20260506 15:11:26.370 | INFO | MemInfo: { | 20260506 15:11:26.370 | INFO | "gpu" : " 17 GB", | 20260506 15:11:26.370 | INFO | "huge" : "", | 20260506 15:11:26.370 | INFO | "huge_shared" : " 27 GB", | 20260506 15:11:26.370 | INFO | "malloc" : " 14 GB", | 20260506 15:11:26.370 | INFO | "pinned" : " 29 GB" | 20260506 15:11:26.370 | INFO | } | 20260506 15:11:26.370 | INFO | Perf report for the last 5.13939 seconds: | 20260506 15:11:26.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:11:26.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:26.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:26.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:26.370 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:11:26.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:26.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:26.370 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:11:26.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:26.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:26.370 | INFO | Uploaded 36GiB to GPU in 2.02047s (17.8177GiB/s) | 20260506 15:11:26.370 | INFO | Downloaded 34GiB to GPU in 1.98632s (17.1171GiB/s) | 20260506 15:11:26.483 | INFO | Starting a round of Baz IO | 20260506 15:11:29.639 | INFO | Populating RTMetrics report | 20260506 15:11:29.639 | INFO | RTMetrics report populated | 20260506 15:11:29.639 | INFO | Opening tmp RTMetrics json file | 20260506 15:11:29.654 | INFO | Serializing RTMetrics to json | 20260506 15:11:29.665 | INFO | Serialization done, swapping tmp file now... | 20260506 15:11:29.687 | INFO | RTMetrics done | 20260506 15:11:30.062 | INFO | Finished a round of Baz IO: Wrote 2.103 GiB in 3.578 seconds | 20260506 15:11:30.062 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.46% / 0.01% / 62.52% | 20260506 15:11:30.062 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.17644% / 1.19209e-05% / 5.16574e-05% | 20260506 15:11:30.062 | INFO | Baz IO overall: | 20260506 15:11:30.062 | INFO | BytesWritten : 1212267715305 | 20260506 15:11:30.062 | INFO | NumEvents : 586678748454 | 20260506 15:11:31.509 | INFO | Analyzing chunk frames = [2330624,2331136) | 20260506 15:11:31.509 | INFO | MemInfo: { | 20260506 15:11:31.509 | INFO | "gpu" : " 17 GB", | 20260506 15:11:31.509 | INFO | "huge" : "", | 20260506 15:11:31.509 | INFO | "huge_shared" : " 27 GB", | 20260506 15:11:31.509 | INFO | "malloc" : " 14 GB", | 20260506 15:11:31.509 | INFO | "pinned" : " 29 GB" | 20260506 15:11:31.509 | INFO | } | 20260506 15:11:31.510 | INFO | Perf report for the last 5.13927 seconds: | 20260506 15:11:31.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:11:31.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:31.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:31.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:31.510 | INFO | ANALYSIS : 63.0%, 2.9, 95.6ms, 0 | 20260506 15:11:31.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:31.510 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260506 15:11:31.510 | INFO | PRE_HQ : 60.7%, 1.9, 59.3ms, 0 | 20260506 15:11:31.510 | INFO | BAZWRITER : 69.1%, 4.8, 170.7ms, 0 | 20260506 15:11:31.510 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 15:11:31.510 | INFO | Uploaded 36GiB to GPU in 2.05375s (17.5289GiB/s) | 20260506 15:11:31.510 | INFO | Downloaded 38GiB to GPU in 2.23269s (17.0199GiB/s) | 20260506 15:11:36.649 | INFO | Analyzing chunk frames = [2331136,2331648) | 20260506 15:11:36.649 | INFO | MemInfo: { | 20260506 15:11:36.649 | INFO | "gpu" : " 17 GB", | 20260506 15:11:36.649 | INFO | "huge" : "", | 20260506 15:11:36.649 | INFO | "huge_shared" : " 27 GB", | 20260506 15:11:36.649 | INFO | "malloc" : " 14 GB", | 20260506 15:11:36.649 | INFO | "pinned" : " 29 GB" | 20260506 15:11:36.649 | INFO | } | 20260506 15:11:36.649 | INFO | Perf report for the last 5.13934 seconds: | 20260506 15:11:36.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:11:36.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:36.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:36.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:36.649 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:11:36.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:36.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:36.649 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:11:36.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:36.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:36.649 | INFO | Uploaded 36GiB to GPU in 1.98583s (18.1284GiB/s) | 20260506 15:11:36.649 | INFO | Downloaded 34GiB to GPU in 1.94577s (17.4738GiB/s) | 20260506 15:11:41.788 | INFO | Analyzing chunk frames = [2331648,2332160) | 20260506 15:11:41.788 | INFO | MemInfo: { | 20260506 15:11:41.788 | INFO | "gpu" : " 17 GB", | 20260506 15:11:41.788 | INFO | "huge" : "", | 20260506 15:11:41.788 | INFO | "huge_shared" : " 27 GB", | 20260506 15:11:41.788 | INFO | "malloc" : " 14 GB", | 20260506 15:11:41.788 | INFO | "pinned" : " 29 GB" | 20260506 15:11:41.788 | INFO | } | 20260506 15:11:41.788 | INFO | Perf report for the last 5.13951 seconds: | 20260506 15:11:41.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:11:41.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:41.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:41.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:41.788 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:11:41.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:41.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:41.788 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:11:41.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:41.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:41.789 | INFO | Uploaded 36GiB to GPU in 1.95992s (18.3681GiB/s) | 20260506 15:11:41.789 | INFO | Downloaded 34GiB to GPU in 1.94086s (17.518GiB/s) | 20260506 15:11:46.928 | INFO | Analyzing chunk frames = [2332160,2332672) | 20260506 15:11:46.928 | INFO | MemInfo: { | 20260506 15:11:46.928 | INFO | "gpu" : " 17 GB", | 20260506 15:11:46.928 | INFO | "huge" : "", | 20260506 15:11:46.928 | INFO | "huge_shared" : " 27 GB", | 20260506 15:11:46.928 | INFO | "malloc" : " 14 GB", | 20260506 15:11:46.928 | INFO | "pinned" : " 29 GB" | 20260506 15:11:46.928 | INFO | } | 20260506 15:11:46.928 | INFO | Perf report for the last 5.13942 seconds: | 20260506 15:11:46.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:11:46.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:46.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:46.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:46.928 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 15:11:46.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:46.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:46.928 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:11:46.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:46.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:46.928 | INFO | Uploaded 36GiB to GPU in 2.01484s (17.8674GiB/s) | 20260506 15:11:46.928 | INFO | Downloaded 34GiB to GPU in 1.91871s (17.7202GiB/s) | 20260506 15:11:52.067 | INFO | Analyzing chunk frames = [2332672,2333184) | 20260506 15:11:52.067 | INFO | MemInfo: { | 20260506 15:11:52.067 | INFO | "gpu" : " 17 GB", | 20260506 15:11:52.067 | INFO | "huge" : "", | 20260506 15:11:52.067 | INFO | "huge_shared" : " 27 GB", | 20260506 15:11:52.067 | INFO | "malloc" : " 14 GB", | 20260506 15:11:52.067 | INFO | "pinned" : " 29 GB" | 20260506 15:11:52.067 | INFO | } | 20260506 15:11:52.067 | INFO | Perf report for the last 5.13957 seconds: | 20260506 15:11:52.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:11:52.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:52.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:52.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:52.067 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 15:11:52.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:52.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:52.068 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:11:52.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:52.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:52.068 | INFO | Uploaded 36GiB to GPU in 2.01967s (17.8247GiB/s) | 20260506 15:11:52.068 | INFO | Downloaded 34GiB to GPU in 1.94656s (17.4667GiB/s) | 20260506 15:11:57.206 | INFO | Analyzing chunk frames = [2333184,2333696) | 20260506 15:11:57.206 | INFO | MemInfo: { | 20260506 15:11:57.206 | INFO | "gpu" : " 17 GB", | 20260506 15:11:57.206 | INFO | "huge" : "", | 20260506 15:11:57.206 | INFO | "huge_shared" : " 27 GB", | 20260506 15:11:57.207 | INFO | "malloc" : " 14 GB", | 20260506 15:11:57.207 | INFO | "pinned" : " 29 GB" | 20260506 15:11:57.207 | INFO | } | 20260506 15:11:57.207 | INFO | Perf report for the last 5.13921 seconds: | 20260506 15:11:57.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:11:57.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:57.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:57.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:57.207 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 15:11:57.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:57.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:57.207 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 15:11:57.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:57.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:11:57.207 | INFO | Uploaded 36GiB to GPU in 2.02334s (17.7924GiB/s) | 20260506 15:11:57.207 | INFO | Downloaded 34GiB to GPU in 1.90839s (17.8161GiB/s) | 20260506 15:12:02.346 | INFO | Analyzing chunk frames = [2333696,2334208) | 20260506 15:12:02.346 | INFO | MemInfo: { | 20260506 15:12:02.346 | INFO | "gpu" : " 17 GB", | 20260506 15:12:02.346 | INFO | "huge" : "", | 20260506 15:12:02.346 | INFO | "huge_shared" : " 27 GB", | 20260506 15:12:02.346 | INFO | "malloc" : " 14 GB", | 20260506 15:12:02.346 | INFO | "pinned" : " 29 GB" | 20260506 15:12:02.346 | INFO | } | 20260506 15:12:02.346 | INFO | Perf report for the last 5.1395 seconds: | 20260506 15:12:02.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:12:02.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:02.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:02.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:02.346 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:12:02.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:02.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:02.346 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 15:12:02.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:02.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:02.347 | INFO | Uploaded 36GiB to GPU in 1.97005s (18.2736GiB/s) | 20260506 15:12:02.347 | INFO | Downloaded 34GiB to GPU in 1.99146s (17.0729GiB/s) | 20260506 15:12:07.484 | INFO | Analyzing chunk frames = [2334208,2334720) | 20260506 15:12:07.484 | INFO | MemInfo: { | 20260506 15:12:07.484 | INFO | "gpu" : " 17 GB", | 20260506 15:12:07.484 | INFO | "huge" : "", | 20260506 15:12:07.484 | INFO | "huge_shared" : " 27 GB", | 20260506 15:12:07.484 | INFO | "malloc" : " 14 GB", | 20260506 15:12:07.484 | INFO | "pinned" : " 29 GB" | 20260506 15:12:07.484 | INFO | } | 20260506 15:12:07.484 | INFO | Perf report for the last 5.1383 seconds: | 20260506 15:12:07.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:12:07.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:07.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:07.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:07.484 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 15:12:07.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:07.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:07.484 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:12:07.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:07.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:07.485 | INFO | Uploaded 36GiB to GPU in 1.96864s (18.2867GiB/s) | 20260506 15:12:07.485 | INFO | Downloaded 34GiB to GPU in 1.97758s (17.1927GiB/s) | 20260506 15:12:07.485 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:12:07.485 | INFO | Total mem usage: 17.4 GiB | 20260506 15:12:07.485 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:12:07.485 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -6.24%) | 20260506 15:12:07.485 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:12:07.485 | INFO | High Water Breakdown: | 20260506 15:12:07.485 | INFO | 62% : 10.2 GiB ( 93% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:12:07.485 | INFO | 37.9% : 6.21 GiB ( 92.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:12:07.485 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:12:07.485 | INFO | ------------------------------------------ | 20260506 15:12:07.485 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:12:07.485 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:12:07.485 | INFO | Total mem usage: 26.5 GiB | 20260506 15:12:07.485 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:12:07.485 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:12:07.485 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:12:07.485 | INFO | High Water Breakdown: | 20260506 15:12:07.485 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:12:07.485 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:12:07.485 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:12:07.485 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:12:07.485 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:12:07.485 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:12:07.485 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:12:07.485 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:12:07.485 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:12:07.485 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:12:07.485 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:12:07.485 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:12:07.485 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:12:07.485 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:12:07.485 | INFO | ------------------------------------------ | 20260506 15:12:07.485 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:12:07.485 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:12:07.485 | INFO | Total mem usage: 25.6 GiB | 20260506 15:12:07.485 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:12:07.485 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:12:07.485 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:12:07.485 | INFO | High Water Breakdown: | 20260506 15:12:07.485 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:12:07.485 | INFO | ------------------------------------------ | 20260506 15:12:07.485 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:12:07.485 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:12:07.485 | INFO | Total mem usage: 15.7 GiB | 20260506 15:12:07.485 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:12:07.485 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:12:07.485 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 15:12:07.485 | INFO | High Water Breakdown: | 20260506 15:12:07.485 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:12:07.485 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:12:07.485 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:12:07.485 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:12:07.485 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:12:07.485 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:12:07.485 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:12:07.485 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:12:07.485 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:12:07.485 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:12:07.485 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:12:07.485 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:12:07.485 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:12:07.485 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:12:07.485 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:12:07.485 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:12:07.485 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:12:07.485 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:12:07.485 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:12:07.485 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:12:07.486 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:12:07.486 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:12:07.486 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:12:07.486 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:12:07.486 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:12:07.486 | INFO | ------------------------------------------ | 20260506 15:12:07.486 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:12:07.486 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:12:07.486 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:12:07.486 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:12:07.486 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.51e+01, 1.31e+01 / 1.64e+00 | 20260506 15:12:07.486 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.00e-01, 9.99e-03 / 7.17e-03 | 20260506 15:12:07.486 | INFO | PerfReport: 11.57%, Baseline -- 3.64e+00 / 4.38e+00, 5.60e-01 / 4.22e-01 | 20260506 15:12:07.486 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.87e-01 / 1.78e-01 | 20260506 15:12:07.486 | INFO | PerfReport: 15.76%, DME -- 3.97e+01 / 4.33e+01, 3.54e+00 / 1.70e+00 | 20260506 15:12:07.486 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.79e+01 / 2.11e+01, 1.54e+00 / 1.96e+00 | 20260506 15:12:07.486 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.32e+00 / 2.87e+00, 3.92e-01 / 2.47e-01 | 20260506 15:12:07.486 | INFO | PerfReport: 3.56%, Metrics -- 1.12e+00 / 1.47e+00, 2.89e-01 / 1.75e-01 | 20260506 15:12:07.486 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.90e-01 / 4.43e-01, 6.00e-02 / 2.24e-02 | 20260506 15:12:07.486 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:12:07.486 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:12:07.486 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:12:07.486 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 15:12:07.486 | INFO | PerfReport: 21.43%, Upload -- 2.09e+01 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260506 15:12:07.486 | INFO | PerfReport: 21.20%, Download -- 2.07e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 15:12:07.486 | INFO | PerfReport: 32.29%, Compute -- 3.15e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 15:12:07.486 | INFO | PerfReport: 1.36%, LockUpload -- 1.32e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260506 15:12:07.486 | INFO | PerfReport: 1.23%, LockDownload -- 1.20e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 15:12:07.486 | INFO | PerfReport: 22.47%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 15:12:07.486 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260506 15:12:07.598 | INFO | Starting a round of Baz IO | 20260506 15:12:10.746 | INFO | Populating RTMetrics report | 20260506 15:12:10.747 | INFO | RTMetrics report populated | 20260506 15:12:10.747 | INFO | Opening tmp RTMetrics json file | 20260506 15:12:10.903 | INFO | Serializing RTMetrics to json | 20260506 15:12:10.904 | INFO | Serialization done, swapping tmp file now... | 20260506 15:12:10.905 | INFO | RTMetrics done | 20260506 15:12:11.112 | INFO | Finished a round of Baz IO: Wrote 2.104 GiB in 3.514 seconds | 20260506 15:12:11.112 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.45% / 0.01% / 62.54% | 20260506 15:12:11.112 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.17009% / 1.58946e-05% / 6.75519e-05% | 20260506 15:12:11.112 | INFO | Baz IO overall: | 20260506 15:12:11.112 | INFO | BytesWritten : 1214526415253 | 20260506 15:12:11.112 | INFO | NumEvents : 587771878243 | 20260506 15:12:12.624 | INFO | Analyzing chunk frames = [2334720,2335232) | 20260506 15:12:12.624 | INFO | MemInfo: { | 20260506 15:12:12.624 | INFO | "gpu" : " 17 GB", | 20260506 15:12:12.624 | INFO | "huge" : "", | 20260506 15:12:12.624 | INFO | "huge_shared" : " 27 GB", | 20260506 15:12:12.624 | INFO | "malloc" : " 14 GB", | 20260506 15:12:12.624 | INFO | "pinned" : " 29 GB" | 20260506 15:12:12.624 | INFO | } | 20260506 15:12:12.624 | INFO | Perf report for the last 5.13967 seconds: | 20260506 15:12:12.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:12:12.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:12.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:12.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:12.624 | INFO | ANALYSIS : 62.9%, 2.9, 95.3ms, 0 | 20260506 15:12:12.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:12.624 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260506 15:12:12.624 | INFO | PRE_HQ : 60.7%, 1.9, 59.4ms, 0 | 20260506 15:12:12.624 | INFO | BAZWRITER : 67.9%, 4.8, 167.4ms, 0 | 20260506 15:12:12.624 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.4ms, 0 | 20260506 15:12:12.625 | INFO | Uploaded 36GiB to GPU in 2.06726s (17.4144GiB/s) | 20260506 15:12:12.625 | INFO | Downloaded 38GiB to GPU in 2.20905s (17.202GiB/s) | 20260506 15:12:17.764 | INFO | Analyzing chunk frames = [2335232,2335744) | 20260506 15:12:17.764 | INFO | MemInfo: { | 20260506 15:12:17.764 | INFO | "gpu" : " 17 GB", | 20260506 15:12:17.764 | INFO | "huge" : "", | 20260506 15:12:17.764 | INFO | "huge_shared" : " 27 GB", | 20260506 15:12:17.764 | INFO | "malloc" : " 14 GB", | 20260506 15:12:17.764 | INFO | "pinned" : " 29 GB" | 20260506 15:12:17.764 | INFO | } | 20260506 15:12:17.764 | INFO | Perf report for the last 5.14018 seconds: | 20260506 15:12:17.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:12:17.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:17.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:17.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:17.764 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:12:17.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:17.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:17.764 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:12:17.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:17.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:17.765 | INFO | Uploaded 36GiB to GPU in 1.98705s (18.1174GiB/s) | 20260506 15:12:17.765 | INFO | Downloaded 34GiB to GPU in 1.94117s (17.5152GiB/s) | 20260506 15:12:22.904 | INFO | Analyzing chunk frames = [2335744,2336256) | 20260506 15:12:22.904 | INFO | MemInfo: { | 20260506 15:12:22.904 | INFO | "gpu" : " 17 GB", | 20260506 15:12:22.904 | INFO | "huge" : "", | 20260506 15:12:22.904 | INFO | "huge_shared" : " 27 GB", | 20260506 15:12:22.904 | INFO | "malloc" : " 14 GB", | 20260506 15:12:22.904 | INFO | "pinned" : " 29 GB" | 20260506 15:12:22.904 | INFO | } | 20260506 15:12:22.904 | INFO | Perf report for the last 5.1401 seconds: | 20260506 15:12:22.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:12:22.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:22.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:22.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:22.905 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:12:22.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:22.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:22.905 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:12:22.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:22.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:22.905 | INFO | Uploaded 36GiB to GPU in 1.9942s (18.0523GiB/s) | 20260506 15:12:22.905 | INFO | Downloaded 34GiB to GPU in 1.94606s (17.4712GiB/s) | 20260506 15:12:28.043 | INFO | Analyzing chunk frames = [2336256,2336768) | 20260506 15:12:28.043 | INFO | MemInfo: { | 20260506 15:12:28.043 | INFO | "gpu" : " 17 GB", | 20260506 15:12:28.043 | INFO | "huge" : "", | 20260506 15:12:28.043 | INFO | "huge_shared" : " 27 GB", | 20260506 15:12:28.043 | INFO | "malloc" : " 14 GB", | 20260506 15:12:28.043 | INFO | "pinned" : " 29 GB" | 20260506 15:12:28.043 | INFO | } | 20260506 15:12:28.043 | INFO | Perf report for the last 5.13867 seconds: | 20260506 15:12:28.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:12:28.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:28.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:28.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:28.043 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 15:12:28.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:28.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:28.043 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:12:28.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:28.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:28.043 | INFO | Uploaded 36GiB to GPU in 2.01408s (17.8742GiB/s) | 20260506 15:12:28.043 | INFO | Downloaded 34GiB to GPU in 1.91631s (17.7425GiB/s) | 20260506 15:12:33.181 | INFO | Analyzing chunk frames = [2336768,2337280) | 20260506 15:12:33.181 | INFO | MemInfo: { | 20260506 15:12:33.181 | INFO | "gpu" : " 17 GB", | 20260506 15:12:33.181 | INFO | "huge" : "", | 20260506 15:12:33.181 | INFO | "huge_shared" : " 27 GB", | 20260506 15:12:33.181 | INFO | "malloc" : " 14 GB", | 20260506 15:12:33.181 | INFO | "pinned" : " 29 GB" | 20260506 15:12:33.181 | INFO | } | 20260506 15:12:33.182 | INFO | Perf report for the last 5.13841 seconds: | 20260506 15:12:33.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:12:33.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:33.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:33.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:33.182 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 15:12:33.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:33.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:33.182 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:12:33.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:33.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:33.182 | INFO | Uploaded 36GiB to GPU in 1.98267s (18.1573GiB/s) | 20260506 15:12:33.182 | INFO | Downloaded 34GiB to GPU in 1.93297s (17.5895GiB/s) | 20260506 15:12:38.321 | INFO | Analyzing chunk frames = [2337280,2337792) | 20260506 15:12:38.321 | INFO | MemInfo: { | 20260506 15:12:38.321 | INFO | "gpu" : " 17 GB", | 20260506 15:12:38.321 | INFO | "huge" : "", | 20260506 15:12:38.321 | INFO | "huge_shared" : " 27 GB", | 20260506 15:12:38.321 | INFO | "malloc" : " 14 GB", | 20260506 15:12:38.321 | INFO | "pinned" : " 29 GB" | 20260506 15:12:38.321 | INFO | } | 20260506 15:12:38.321 | INFO | Perf report for the last 5.13969 seconds: | 20260506 15:12:38.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:12:38.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:38.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:38.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:38.321 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:12:38.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:38.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:38.321 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 15:12:38.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:38.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:38.322 | INFO | Uploaded 36GiB to GPU in 2.07059s (17.3863GiB/s) | 20260506 15:12:38.322 | INFO | Downloaded 34GiB to GPU in 1.97664s (17.2009GiB/s) | 20260506 15:12:43.461 | INFO | Analyzing chunk frames = [2337792,2338304) | 20260506 15:12:43.461 | INFO | MemInfo: { | 20260506 15:12:43.461 | INFO | "gpu" : " 17 GB", | 20260506 15:12:43.461 | INFO | "huge" : "", | 20260506 15:12:43.461 | INFO | "huge_shared" : " 27 GB", | 20260506 15:12:43.461 | INFO | "malloc" : " 14 GB", | 20260506 15:12:43.461 | INFO | "pinned" : " 29 GB" | 20260506 15:12:43.461 | INFO | } | 20260506 15:12:43.461 | INFO | Perf report for the last 5.13997 seconds: | 20260506 15:12:43.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:12:43.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:43.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:43.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:43.461 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:12:43.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:43.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:43.461 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:12:43.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:43.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:43.462 | INFO | Uploaded 36GiB to GPU in 2.00951s (17.9148GiB/s) | 20260506 15:12:43.462 | INFO | Downloaded 34GiB to GPU in 2.00312s (16.9735GiB/s) | 20260506 15:12:48.600 | INFO | Analyzing chunk frames = [2338304,2338816) | 20260506 15:12:48.600 | INFO | MemInfo: { | 20260506 15:12:48.600 | INFO | "gpu" : " 17 GB", | 20260506 15:12:48.600 | INFO | "huge" : "", | 20260506 15:12:48.600 | INFO | "huge_shared" : " 27 GB", | 20260506 15:12:48.600 | INFO | "malloc" : " 14 GB", | 20260506 15:12:48.600 | INFO | "pinned" : " 29 GB" | 20260506 15:12:48.600 | INFO | } | 20260506 15:12:48.600 | INFO | Perf report for the last 5.13886 seconds: | 20260506 15:12:48.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:12:48.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:48.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:48.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:48.600 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:12:48.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:48.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:48.600 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:12:48.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:48.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:48.600 | INFO | Uploaded 36GiB to GPU in 2.00768s (17.9311GiB/s) | 20260506 15:12:48.600 | INFO | Downloaded 34GiB to GPU in 1.98273s (17.1481GiB/s) | 20260506 15:12:48.713 | INFO | Starting a round of Baz IO | 20260506 15:12:51.867 | INFO | Populating RTMetrics report | 20260506 15:12:51.867 | INFO | RTMetrics report populated | 20260506 15:12:51.867 | INFO | Opening tmp RTMetrics json file | 20260506 15:12:51.870 | INFO | Serializing RTMetrics to json | 20260506 15:12:51.877 | INFO | Serialization done, swapping tmp file now... | 20260506 15:12:51.882 | INFO | RTMetrics done | 20260506 15:12:52.316 | INFO | Finished a round of Baz IO: Wrote 2.115 GiB in 3.603 seconds | 20260506 15:12:52.317 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.44% / 0.01% / 62.55% | 20260506 15:12:52.317 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.17519% / 1.19209e-05% / 8.74201e-05% | 20260506 15:12:52.317 | INFO | Baz IO overall: | 20260506 15:12:52.317 | INFO | BytesWritten : 1216797683378 | 20260506 15:12:52.317 | INFO | NumEvents : 588869570346 | 20260506 15:12:53.740 | INFO | Analyzing chunk frames = [2338816,2339328) | 20260506 15:12:53.740 | INFO | MemInfo: { | 20260506 15:12:53.740 | INFO | "gpu" : " 17 GB", | 20260506 15:12:53.740 | INFO | "huge" : "", | 20260506 15:12:53.740 | INFO | "huge_shared" : " 27 GB", | 20260506 15:12:53.740 | INFO | "malloc" : " 14 GB", | 20260506 15:12:53.740 | INFO | "pinned" : " 29 GB" | 20260506 15:12:53.740 | INFO | } | 20260506 15:12:53.740 | INFO | Perf report for the last 5.13953 seconds: | 20260506 15:12:53.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:12:53.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:53.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:53.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:53.740 | INFO | ANALYSIS : 63.0%, 2.9, 95.6ms, 0 | 20260506 15:12:53.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:53.740 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 15:12:53.740 | INFO | PRE_HQ : 61.0%, 1.9, 59.0ms, 0 | 20260506 15:12:53.740 | INFO | BAZWRITER : 69.6%, 4.8, 172.4ms, 0 | 20260506 15:12:53.740 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 15:12:53.740 | INFO | Uploaded 36GiB to GPU in 2.06439s (17.4386GiB/s) | 20260506 15:12:53.740 | INFO | Downloaded 38GiB to GPU in 2.22485s (17.0798GiB/s) | 20260506 15:12:58.878 | INFO | Analyzing chunk frames = [2339328,2339840) | 20260506 15:12:58.878 | INFO | MemInfo: { | 20260506 15:12:58.878 | INFO | "gpu" : " 17 GB", | 20260506 15:12:58.878 | INFO | "huge" : "", | 20260506 15:12:58.878 | INFO | "huge_shared" : " 27 GB", | 20260506 15:12:58.878 | INFO | "malloc" : " 14 GB", | 20260506 15:12:58.878 | INFO | "pinned" : " 29 GB" | 20260506 15:12:58.878 | INFO | } | 20260506 15:12:58.878 | INFO | Perf report for the last 5.13857 seconds: | 20260506 15:12:58.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:12:58.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:58.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:58.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:58.878 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:12:58.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:58.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:58.878 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260506 15:12:58.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:58.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:12:58.878 | INFO | Uploaded 36GiB to GPU in 1.99507s (18.0445GiB/s) | 20260506 15:12:58.879 | INFO | Downloaded 34GiB to GPU in 1.92938s (17.6222GiB/s) | 20260506 15:13:04.018 | INFO | Analyzing chunk frames = [2339840,2340352) | 20260506 15:13:04.018 | INFO | MemInfo: { | 20260506 15:13:04.018 | INFO | "gpu" : " 17 GB", | 20260506 15:13:04.018 | INFO | "huge" : "", | 20260506 15:13:04.018 | INFO | "huge_shared" : " 27 GB", | 20260506 15:13:04.018 | INFO | "malloc" : " 14 GB", | 20260506 15:13:04.018 | INFO | "pinned" : " 29 GB" | 20260506 15:13:04.018 | INFO | } | 20260506 15:13:04.019 | INFO | Perf report for the last 5.14026 seconds: | 20260506 15:13:04.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:13:04.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:04.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:04.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:04.019 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 15:13:04.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:04.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:04.019 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:13:04.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:04.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:04.019 | INFO | Uploaded 36GiB to GPU in 2.00383s (17.9656GiB/s) | 20260506 15:13:04.019 | INFO | Downloaded 34GiB to GPU in 1.94498s (17.4809GiB/s) | 20260506 15:13:09.157 | INFO | Analyzing chunk frames = [2340352,2340864) | 20260506 15:13:09.157 | INFO | MemInfo: { | 20260506 15:13:09.157 | INFO | "gpu" : " 17 GB", | 20260506 15:13:09.157 | INFO | "huge" : "", | 20260506 15:13:09.157 | INFO | "huge_shared" : " 27 GB", | 20260506 15:13:09.157 | INFO | "malloc" : " 14 GB", | 20260506 15:13:09.157 | INFO | "pinned" : " 29 GB" | 20260506 15:13:09.157 | INFO | } | 20260506 15:13:09.157 | INFO | Perf report for the last 5.1389 seconds: | 20260506 15:13:09.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:13:09.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:09.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:09.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:09.158 | INFO | ANALYSIS : 60.6%, 3.0, 94.0ms, 0 | 20260506 15:13:09.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:09.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:09.158 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:13:09.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:09.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:09.158 | INFO | Uploaded 36GiB to GPU in 1.96447s (18.3255GiB/s) | 20260506 15:13:09.158 | INFO | Downloaded 34GiB to GPU in 1.91288s (17.7743GiB/s) | 20260506 15:13:14.297 | INFO | Analyzing chunk frames = [2340864,2341376) | 20260506 15:13:14.297 | INFO | MemInfo: { | 20260506 15:13:14.297 | INFO | "gpu" : " 17 GB", | 20260506 15:13:14.297 | INFO | "huge" : "", | 20260506 15:13:14.297 | INFO | "huge_shared" : " 27 GB", | 20260506 15:13:14.297 | INFO | "malloc" : " 14 GB", | 20260506 15:13:14.297 | INFO | "pinned" : " 29 GB" | 20260506 15:13:14.297 | INFO | } | 20260506 15:13:14.297 | INFO | Perf report for the last 5.13938 seconds: | 20260506 15:13:14.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:13:14.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:14.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:14.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:14.297 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 15:13:14.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:14.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:14.297 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:13:14.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:14.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:14.297 | INFO | Uploaded 36GiB to GPU in 2.00238s (17.9786GiB/s) | 20260506 15:13:14.297 | INFO | Downloaded 34GiB to GPU in 1.93559s (17.5657GiB/s) | 20260506 15:13:19.436 | INFO | Analyzing chunk frames = [2341376,2341888) | 20260506 15:13:19.436 | INFO | MemInfo: { | 20260506 15:13:19.436 | INFO | "gpu" : " 17 GB", | 20260506 15:13:19.436 | INFO | "huge" : "", | 20260506 15:13:19.436 | INFO | "huge_shared" : " 27 GB", | 20260506 15:13:19.436 | INFO | "malloc" : " 14 GB", | 20260506 15:13:19.436 | INFO | "pinned" : " 29 GB" | 20260506 15:13:19.436 | INFO | } | 20260506 15:13:19.437 | INFO | Perf report for the last 5.13975 seconds: | 20260506 15:13:19.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:13:19.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:19.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:19.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:19.437 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 15:13:19.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:19.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:19.437 | INFO | PRE_HQ : 31.4%, 1.0, 16.4ms, 0 | 20260506 15:13:19.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:19.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:19.437 | INFO | Uploaded 36GiB to GPU in 2.0561s (17.5089GiB/s) | 20260506 15:13:19.437 | INFO | Downloaded 34GiB to GPU in 1.94205s (17.5073GiB/s) | 20260506 15:13:24.576 | INFO | Analyzing chunk frames = [2341888,2342400) | 20260506 15:13:24.576 | INFO | MemInfo: { | 20260506 15:13:24.576 | INFO | "gpu" : " 17 GB", | 20260506 15:13:24.576 | INFO | "huge" : "", | 20260506 15:13:24.576 | INFO | "huge_shared" : " 27 GB", | 20260506 15:13:24.576 | INFO | "malloc" : " 14 GB", | 20260506 15:13:24.576 | INFO | "pinned" : " 29 GB" | 20260506 15:13:24.576 | INFO | } | 20260506 15:13:24.576 | INFO | Perf report for the last 5.13938 seconds: | 20260506 15:13:24.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:13:24.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:24.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:24.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:24.576 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:13:24.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:24.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:24.576 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:13:24.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:24.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:24.576 | INFO | Uploaded 36GiB to GPU in 2.0039s (17.965GiB/s) | 20260506 15:13:24.576 | INFO | Downloaded 34GiB to GPU in 2.00334s (16.9716GiB/s) | 20260506 15:13:29.715 | INFO | Analyzing chunk frames = [2342400,2342912) | 20260506 15:13:29.715 | INFO | MemInfo: { | 20260506 15:13:29.715 | INFO | "gpu" : " 17 GB", | 20260506 15:13:29.715 | INFO | "huge" : "", | 20260506 15:13:29.715 | INFO | "huge_shared" : " 27 GB", | 20260506 15:13:29.715 | INFO | "malloc" : " 14 GB", | 20260506 15:13:29.715 | INFO | "pinned" : " 29 GB" | 20260506 15:13:29.715 | INFO | } | 20260506 15:13:29.715 | INFO | Perf report for the last 5.13902 seconds: | 20260506 15:13:29.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:13:29.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:29.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:29.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:29.715 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 15:13:29.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:29.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:29.715 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:13:29.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:29.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:29.715 | INFO | Uploaded 36GiB to GPU in 2.02858s (17.7464GiB/s) | 20260506 15:13:29.715 | INFO | Downloaded 34GiB to GPU in 1.98408s (17.1364GiB/s) | 20260506 15:13:29.716 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:13:29.716 | INFO | Total mem usage: 17.4 GiB | 20260506 15:13:29.716 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:13:29.716 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -7.69%) | 20260506 15:13:29.716 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:13:29.716 | INFO | High Water Breakdown: | 20260506 15:13:29.716 | INFO | 61.8% : 9.99 GiB ( 91.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:13:29.716 | INFO | 38.1% : 6.16 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:13:29.716 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:13:29.716 | INFO | ------------------------------------------ | 20260506 15:13:29.716 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:13:29.716 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:13:29.716 | INFO | Total mem usage: 26.5 GiB | 20260506 15:13:29.716 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:13:29.716 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:13:29.716 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:13:29.716 | INFO | High Water Breakdown: | 20260506 15:13:29.716 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:13:29.716 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:13:29.716 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:13:29.716 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:13:29.716 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:13:29.716 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:13:29.716 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:13:29.716 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:13:29.716 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:13:29.716 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:13:29.716 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:13:29.716 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:13:29.716 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:13:29.716 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:13:29.716 | INFO | ------------------------------------------ | 20260506 15:13:29.716 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:13:29.716 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:13:29.716 | INFO | Total mem usage: 25.6 GiB | 20260506 15:13:29.716 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:13:29.716 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:13:29.716 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:13:29.716 | INFO | High Water Breakdown: | 20260506 15:13:29.716 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:13:29.716 | INFO | ------------------------------------------ | 20260506 15:13:29.716 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:13:29.716 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:13:29.716 | INFO | Total mem usage: 15.7 GiB | 20260506 15:13:29.716 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:13:29.716 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:13:29.716 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 15:13:29.716 | INFO | High Water Breakdown: | 20260506 15:13:29.716 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:13:29.717 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:13:29.717 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:13:29.717 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:13:29.717 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:13:29.717 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:13:29.717 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:13:29.717 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:13:29.717 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:13:29.717 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:13:29.717 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:13:29.717 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:13:29.718 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:13:29.718 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:13:29.718 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:13:29.718 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:13:29.718 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:13:29.718 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:13:29.718 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:13:29.718 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:13:29.718 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:13:29.718 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:13:29.718 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:13:29.718 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:13:29.718 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:13:29.718 | INFO | ------------------------------------------ | 20260506 15:13:29.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:13:29.719 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:13:29.719 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:13:29.719 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:13:29.719 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.56e+01, 1.36e+01 / 2.15e+00 | 20260506 15:13:29.719 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.00e-01, 1.02e-02 / 6.33e-03 | 20260506 15:13:29.719 | INFO | PerfReport: 11.58%, Baseline -- 3.64e+00 / 4.41e+00, 5.62e-01 / 3.73e-01 | 20260506 15:13:29.719 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.48e+00, 1.87e-01 / 1.35e-01 | 20260506 15:13:29.719 | INFO | PerfReport: 15.88%, DME -- 3.99e+01 / 4.43e+01, 3.95e+00 / 2.47e+00 | 20260506 15:13:29.719 | INFO | PerfReport: 56.75%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.41e+00 / 1.79e+00 | 20260506 15:13:29.719 | INFO | PerfReport: 7.43%, PulseAccumulating -- 2.34e+00 / 2.94e+00, 4.05e-01 / 2.38e-01 | 20260506 15:13:29.719 | INFO | PerfReport: 3.57%, Metrics -- 1.12e+00 / 1.49e+00, 2.90e-01 / 2.01e-01 | 20260506 15:13:29.719 | INFO | PerfReport: 1.25%, PulseSerializing -- 3.92e-01 / 4.46e-01, 6.05e-02 / 2.45e-02 | 20260506 15:13:29.719 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:13:29.719 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:13:29.719 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:13:29.719 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 15:13:29.719 | INFO | PerfReport: 21.51%, Upload -- 2.10e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 15:13:29.719 | INFO | PerfReport: 21.20%, Download -- 2.07e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 15:13:29.719 | INFO | PerfReport: 32.21%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 15:13:29.719 | INFO | PerfReport: 1.50%, LockUpload -- 1.46e+00 / 0.00e+00, 5.79e+00 / 0.00e+00 | 20260506 15:13:29.719 | INFO | PerfReport: 1.17%, LockDownload -- 1.14e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260506 15:13:29.719 | INFO | PerfReport: 22.40%, LockCompute -- 2.19e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260506 15:13:29.719 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.32e-02 / 0.00e+00 | 20260506 15:13:29.825 | INFO | Starting a round of Baz IO | 20260506 15:13:32.980 | INFO | Populating RTMetrics report | 20260506 15:13:32.981 | INFO | RTMetrics report populated | 20260506 15:13:32.981 | INFO | Opening tmp RTMetrics json file | 20260506 15:13:32.982 | INFO | Serializing RTMetrics to json | 20260506 15:13:32.983 | INFO | Serialization done, swapping tmp file now... | 20260506 15:13:32.984 | INFO | RTMetrics done | 20260506 15:13:33.126 | INFO | Finished a round of Baz IO: Wrote 2.112 GiB in 3.3 seconds | 20260506 15:13:33.126 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.43% / 0.01% / 62.56% | 20260506 15:13:33.126 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.16583% / 1.58946e-05% / 5.16574e-05% | 20260506 15:13:33.126 | INFO | Baz IO overall: | 20260506 15:13:33.126 | INFO | BytesWritten : 1219065291716 | 20260506 15:13:33.126 | INFO | NumEvents : 589964704666 | 20260506 15:13:34.854 | INFO | Analyzing chunk frames = [2342912,2343424) | 20260506 15:13:34.854 | INFO | MemInfo: { | 20260506 15:13:34.854 | INFO | "gpu" : " 17 GB", | 20260506 15:13:34.854 | INFO | "huge" : "", | 20260506 15:13:34.854 | INFO | "huge_shared" : " 27 GB", | 20260506 15:13:34.854 | INFO | "malloc" : " 14 GB", | 20260506 15:13:34.854 | INFO | "pinned" : " 29 GB" | 20260506 15:13:34.854 | INFO | } | 20260506 15:13:34.854 | INFO | Perf report for the last 5.13875 seconds: | 20260506 15:13:34.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:13:34.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:34.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:34.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:34.854 | INFO | ANALYSIS : 63.0%, 2.9, 95.2ms, 0 | 20260506 15:13:34.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:34.854 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 15:13:34.854 | INFO | PRE_HQ : 61.0%, 1.9, 59.7ms, 0 | 20260506 15:13:34.854 | INFO | BAZWRITER : 63.6%, 4.6, 151.8ms, 0 | 20260506 15:13:34.854 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 15:13:34.854 | INFO | Uploaded 36GiB to GPU in 2.08001s (17.3076GiB/s) | 20260506 15:13:34.854 | INFO | Downloaded 38GiB to GPU in 2.2282s (17.0541GiB/s) | 20260506 15:13:39.994 | INFO | Analyzing chunk frames = [2343424,2343936) | 20260506 15:13:39.994 | INFO | MemInfo: { | 20260506 15:13:39.994 | INFO | "gpu" : " 17 GB", | 20260506 15:13:39.994 | INFO | "huge" : "", | 20260506 15:13:39.994 | INFO | "huge_shared" : " 27 GB", | 20260506 15:13:39.994 | INFO | "malloc" : " 14 GB", | 20260506 15:13:39.994 | INFO | "pinned" : " 29 GB" | 20260506 15:13:39.994 | INFO | } | 20260506 15:13:39.994 | INFO | Perf report for the last 5.1402 seconds: | 20260506 15:13:39.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:13:39.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:39.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:39.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:39.994 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 15:13:39.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:39.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:39.994 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:13:39.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:39.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:39.994 | INFO | Uploaded 36GiB to GPU in 1.99409s (18.0534GiB/s) | 20260506 15:13:39.994 | INFO | Downloaded 34GiB to GPU in 1.94999s (17.436GiB/s) | 20260506 15:13:45.133 | INFO | Analyzing chunk frames = [2343936,2344448) | 20260506 15:13:45.133 | INFO | MemInfo: { | 20260506 15:13:45.133 | INFO | "gpu" : " 17 GB", | 20260506 15:13:45.133 | INFO | "huge" : "", | 20260506 15:13:45.133 | INFO | "huge_shared" : " 27 GB", | 20260506 15:13:45.133 | INFO | "malloc" : " 14 GB", | 20260506 15:13:45.133 | INFO | "pinned" : " 29 GB" | 20260506 15:13:45.133 | INFO | } | 20260506 15:13:45.133 | INFO | Perf report for the last 5.13923 seconds: | 20260506 15:13:45.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:13:45.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:45.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:45.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:45.133 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:13:45.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:45.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:45.133 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:13:45.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:45.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:45.133 | INFO | Uploaded 36GiB to GPU in 1.9869s (18.1186GiB/s) | 20260506 15:13:45.133 | INFO | Downloaded 34GiB to GPU in 1.94034s (17.5227GiB/s) | 20260506 15:13:50.272 | INFO | Analyzing chunk frames = [2344448,2344960) | 20260506 15:13:50.272 | INFO | MemInfo: { | 20260506 15:13:50.272 | INFO | "gpu" : " 17 GB", | 20260506 15:13:50.272 | INFO | "huge" : "", | 20260506 15:13:50.272 | INFO | "huge_shared" : " 27 GB", | 20260506 15:13:50.272 | INFO | "malloc" : " 14 GB", | 20260506 15:13:50.272 | INFO | "pinned" : " 29 GB" | 20260506 15:13:50.272 | INFO | } | 20260506 15:13:50.273 | INFO | Perf report for the last 5.13928 seconds: | 20260506 15:13:50.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:13:50.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:50.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:50.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:50.273 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:13:50.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:50.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:50.273 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:13:50.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:50.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:50.273 | INFO | Uploaded 36GiB to GPU in 2.01995s (17.8222GiB/s) | 20260506 15:13:50.273 | INFO | Downloaded 34GiB to GPU in 1.91542s (17.7506GiB/s) | 20260506 15:13:55.412 | INFO | Analyzing chunk frames = [2344960,2345472) | 20260506 15:13:55.412 | INFO | MemInfo: { | 20260506 15:13:55.412 | INFO | "gpu" : " 17 GB", | 20260506 15:13:55.412 | INFO | "huge" : "", | 20260506 15:13:55.412 | INFO | "huge_shared" : " 27 GB", | 20260506 15:13:55.412 | INFO | "malloc" : " 14 GB", | 20260506 15:13:55.412 | INFO | "pinned" : " 29 GB" | 20260506 15:13:55.412 | INFO | } | 20260506 15:13:55.412 | INFO | Perf report for the last 5.13973 seconds: | 20260506 15:13:55.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:13:55.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:55.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:55.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:55.412 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 15:13:55.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:55.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:55.412 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:13:55.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:55.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:13:55.413 | INFO | Uploaded 36GiB to GPU in 2.00845s (17.9243GiB/s) | 20260506 15:13:55.413 | INFO | Downloaded 34GiB to GPU in 1.93197s (17.5986GiB/s) | 20260506 15:14:00.550 | INFO | Analyzing chunk frames = [2345472,2345984) | 20260506 15:14:00.550 | INFO | MemInfo: { | 20260506 15:14:00.550 | INFO | "gpu" : " 17 GB", | 20260506 15:14:00.550 | INFO | "huge" : "", | 20260506 15:14:00.550 | INFO | "huge_shared" : " 27 GB", | 20260506 15:14:00.550 | INFO | "malloc" : " 14 GB", | 20260506 15:14:00.550 | INFO | "pinned" : " 29 GB" | 20260506 15:14:00.550 | INFO | } | 20260506 15:14:00.550 | INFO | Perf report for the last 5.13813 seconds: | 20260506 15:14:00.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:14:00.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:00.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:00.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:00.550 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 15:14:00.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:00.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:00.551 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260506 15:14:00.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:00.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:00.551 | INFO | Uploaded 36GiB to GPU in 2.0455s (17.5996GiB/s) | 20260506 15:14:00.551 | INFO | Downloaded 34GiB to GPU in 1.92035s (17.7051GiB/s) | 20260506 15:14:05.690 | INFO | Analyzing chunk frames = [2345984,2346496) | 20260506 15:14:05.690 | INFO | MemInfo: { | 20260506 15:14:05.690 | INFO | "gpu" : " 17 GB", | 20260506 15:14:05.690 | INFO | "huge" : "", | 20260506 15:14:05.690 | INFO | "huge_shared" : " 27 GB", | 20260506 15:14:05.690 | INFO | "malloc" : " 14 GB", | 20260506 15:14:05.690 | INFO | "pinned" : " 29 GB" | 20260506 15:14:05.690 | INFO | } | 20260506 15:14:05.690 | INFO | Perf report for the last 5.13993 seconds: | 20260506 15:14:05.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:14:05.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:05.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:05.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:05.690 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:14:05.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:05.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:05.690 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 15:14:05.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:05.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:05.691 | INFO | Uploaded 36GiB to GPU in 1.9829s (18.1552GiB/s) | 20260506 15:14:05.691 | INFO | Downloaded 34GiB to GPU in 1.99065s (17.0799GiB/s) | 20260506 15:14:10.829 | INFO | Analyzing chunk frames = [2346496,2347008) | 20260506 15:14:10.829 | INFO | MemInfo: { | 20260506 15:14:10.829 | INFO | "gpu" : " 17 GB", | 20260506 15:14:10.829 | INFO | "huge" : "", | 20260506 15:14:10.829 | INFO | "huge_shared" : " 27 GB", | 20260506 15:14:10.829 | INFO | "malloc" : " 14 GB", | 20260506 15:14:10.829 | INFO | "pinned" : " 29 GB" | 20260506 15:14:10.829 | INFO | } | 20260506 15:14:10.829 | INFO | Perf report for the last 5.13912 seconds: | 20260506 15:14:10.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:14:10.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:10.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:10.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:10.830 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:14:10.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:10.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:10.830 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:14:10.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:10.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:10.830 | INFO | Uploaded 36GiB to GPU in 2.02378s (17.7885GiB/s) | 20260506 15:14:10.830 | INFO | Downloaded 34GiB to GPU in 1.96771s (17.279GiB/s) | 20260506 15:14:10.940 | INFO | Starting a round of Baz IO | 20260506 15:14:14.092 | INFO | Populating RTMetrics report | 20260506 15:14:14.092 | INFO | RTMetrics report populated | 20260506 15:14:14.092 | INFO | Opening tmp RTMetrics json file | 20260506 15:14:14.116 | INFO | Serializing RTMetrics to json | 20260506 15:14:14.165 | INFO | Serialization done, swapping tmp file now... | 20260506 15:14:14.168 | INFO | RTMetrics done | 20260506 15:14:14.398 | INFO | Finished a round of Baz IO: Wrote 2.109 GiB in 3.457 seconds | 20260506 15:14:14.398 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.42% / 0.01% / 62.57% | 20260506 15:14:14.398 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.16134% / 3.97364e-05% / 6.75519e-05% | 20260506 15:14:14.398 | INFO | Baz IO overall: | 20260506 15:14:14.398 | INFO | BytesWritten : 1221329828347 | 20260506 15:14:14.398 | INFO | NumEvents : 591059320483 | 20260506 15:14:15.969 | INFO | Analyzing chunk frames = [2347008,2347520) | 20260506 15:14:15.969 | INFO | MemInfo: { | 20260506 15:14:15.969 | INFO | "gpu" : " 17 GB", | 20260506 15:14:15.969 | INFO | "huge" : "", | 20260506 15:14:15.969 | INFO | "huge_shared" : " 27 GB", | 20260506 15:14:15.969 | INFO | "malloc" : " 14 GB", | 20260506 15:14:15.969 | INFO | "pinned" : " 29 GB" | 20260506 15:14:15.969 | INFO | } | 20260506 15:14:15.969 | INFO | Perf report for the last 5.13953 seconds: | 20260506 15:14:15.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:14:15.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:15.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:15.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:15.969 | INFO | ANALYSIS : 62.8%, 2.9, 95.2ms, 0 | 20260506 15:14:15.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:15.969 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260506 15:14:15.969 | INFO | PRE_HQ : 60.9%, 1.9, 59.4ms, 0 | 20260506 15:14:15.969 | INFO | BAZWRITER : 66.7%, 4.8, 164.6ms, 0 | 20260506 15:14:15.969 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 15:14:15.969 | INFO | Uploaded 36GiB to GPU in 2.08972s (17.2272GiB/s) | 20260506 15:14:15.969 | INFO | Downloaded 38GiB to GPU in 2.23797s (16.9797GiB/s) | 20260506 15:14:21.109 | INFO | Analyzing chunk frames = [2347520,2348032) | 20260506 15:14:21.109 | INFO | MemInfo: { | 20260506 15:14:21.109 | INFO | "gpu" : " 17 GB", | 20260506 15:14:21.109 | INFO | "huge" : "", | 20260506 15:14:21.109 | INFO | "huge_shared" : " 27 GB", | 20260506 15:14:21.109 | INFO | "malloc" : " 14 GB", | 20260506 15:14:21.109 | INFO | "pinned" : " 29 GB" | 20260506 15:14:21.109 | INFO | } | 20260506 15:14:21.109 | INFO | Perf report for the last 5.13996 seconds: | 20260506 15:14:21.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:14:21.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:21.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:21.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:21.109 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 15:14:21.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:21.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:21.109 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:14:21.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:21.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:21.109 | INFO | Uploaded 36GiB to GPU in 2.0208s (17.8147GiB/s) | 20260506 15:14:21.109 | INFO | Downloaded 34GiB to GPU in 1.95145s (17.423GiB/s) | 20260506 15:14:26.249 | INFO | Analyzing chunk frames = [2348032,2348544) | 20260506 15:14:26.249 | INFO | MemInfo: { | 20260506 15:14:26.249 | INFO | "gpu" : " 17 GB", | 20260506 15:14:26.249 | INFO | "huge" : "", | 20260506 15:14:26.249 | INFO | "huge_shared" : " 27 GB", | 20260506 15:14:26.249 | INFO | "malloc" : " 14 GB", | 20260506 15:14:26.249 | INFO | "pinned" : " 29 GB" | 20260506 15:14:26.249 | INFO | } | 20260506 15:14:26.249 | INFO | Perf report for the last 5.14026 seconds: | 20260506 15:14:26.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:14:26.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:26.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:26.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:26.249 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 15:14:26.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:26.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:26.249 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:14:26.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:26.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:26.250 | INFO | Uploaded 36GiB to GPU in 2.00448s (17.9597GiB/s) | 20260506 15:14:26.250 | INFO | Downloaded 34GiB to GPU in 1.94337s (17.4954GiB/s) | 20260506 15:14:31.388 | INFO | Analyzing chunk frames = [2348544,2349056) | 20260506 15:14:31.388 | INFO | MemInfo: { | 20260506 15:14:31.388 | INFO | "gpu" : " 17 GB", | 20260506 15:14:31.388 | INFO | "huge" : "", | 20260506 15:14:31.388 | INFO | "huge_shared" : " 27 GB", | 20260506 15:14:31.388 | INFO | "malloc" : " 14 GB", | 20260506 15:14:31.388 | INFO | "pinned" : " 29 GB" | 20260506 15:14:31.388 | INFO | } | 20260506 15:14:31.388 | INFO | Perf report for the last 5.13853 seconds: | 20260506 15:14:31.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:14:31.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:31.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:31.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:31.388 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 15:14:31.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:31.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:31.388 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:14:31.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:31.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:31.388 | INFO | Uploaded 36GiB to GPU in 1.98632s (18.1239GiB/s) | 20260506 15:14:31.388 | INFO | Downloaded 34GiB to GPU in 1.91375s (17.7661GiB/s) | 20260506 15:14:36.525 | INFO | Analyzing chunk frames = [2349056,2349568) | 20260506 15:14:36.525 | INFO | MemInfo: { | 20260506 15:14:36.525 | INFO | "gpu" : " 17 GB", | 20260506 15:14:36.525 | INFO | "huge" : "", | 20260506 15:14:36.526 | INFO | "huge_shared" : " 27 GB", | 20260506 15:14:36.526 | INFO | "malloc" : " 14 GB", | 20260506 15:14:36.526 | INFO | "pinned" : " 29 GB" | 20260506 15:14:36.526 | INFO | } | 20260506 15:14:36.526 | INFO | Perf report for the last 5.13795 seconds: | 20260506 15:14:36.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:14:36.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:36.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:36.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:36.526 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 15:14:36.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:36.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:36.526 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:14:36.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:36.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:36.526 | INFO | Uploaded 36GiB to GPU in 2.02382s (17.7881GiB/s) | 20260506 15:14:36.526 | INFO | Downloaded 34GiB to GPU in 1.94787s (17.455GiB/s) | 20260506 15:14:41.665 | INFO | Analyzing chunk frames = [2349568,2350080) | 20260506 15:14:41.665 | INFO | MemInfo: { | 20260506 15:14:41.665 | INFO | "gpu" : " 17 GB", | 20260506 15:14:41.665 | INFO | "huge" : "", | 20260506 15:14:41.665 | INFO | "huge_shared" : " 27 GB", | 20260506 15:14:41.665 | INFO | "malloc" : " 14 GB", | 20260506 15:14:41.665 | INFO | "pinned" : " 29 GB" | 20260506 15:14:41.665 | INFO | } | 20260506 15:14:41.665 | INFO | Perf report for the last 5.13941 seconds: | 20260506 15:14:41.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:14:41.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:41.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:41.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:41.665 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 15:14:41.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:41.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:41.665 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 15:14:41.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:41.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:41.665 | INFO | Uploaded 36GiB to GPU in 2.05157s (17.5476GiB/s) | 20260506 15:14:41.665 | INFO | Downloaded 34GiB to GPU in 1.93927s (17.5324GiB/s) | 20260506 15:14:46.805 | INFO | Analyzing chunk frames = [2350080,2350592) | 20260506 15:14:46.805 | INFO | MemInfo: { | 20260506 15:14:46.805 | INFO | "gpu" : " 17 GB", | 20260506 15:14:46.805 | INFO | "huge" : "", | 20260506 15:14:46.805 | INFO | "huge_shared" : " 27 GB", | 20260506 15:14:46.805 | INFO | "malloc" : " 14 GB", | 20260506 15:14:46.805 | INFO | "pinned" : " 29 GB" | 20260506 15:14:46.806 | INFO | } | 20260506 15:14:46.806 | INFO | Perf report for the last 5.14042 seconds: | 20260506 15:14:46.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:14:46.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:46.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:46.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:46.806 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:14:46.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:46.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:46.806 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:14:46.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:46.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:46.806 | INFO | Uploaded 36GiB to GPU in 2.00971s (17.9131GiB/s) | 20260506 15:14:46.806 | INFO | Downloaded 34GiB to GPU in 2.00133s (16.9887GiB/s) | 20260506 15:14:51.945 | INFO | Analyzing chunk frames = [2350592,2351104) | 20260506 15:14:51.945 | INFO | MemInfo: { | 20260506 15:14:51.945 | INFO | "gpu" : " 17 GB", | 20260506 15:14:51.945 | INFO | "huge" : "", | 20260506 15:14:51.945 | INFO | "huge_shared" : " 27 GB", | 20260506 15:14:51.945 | INFO | "malloc" : " 14 GB", | 20260506 15:14:51.945 | INFO | "pinned" : " 29 GB" | 20260506 15:14:51.945 | INFO | } | 20260506 15:14:51.945 | INFO | Perf report for the last 5.1393 seconds: | 20260506 15:14:51.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:14:51.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:51.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:51.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:51.945 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:14:51.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:51.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:51.945 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:14:51.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:51.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:51.945 | INFO | Uploaded 36GiB to GPU in 1.99716s (18.0256GiB/s) | 20260506 15:14:51.945 | INFO | Downloaded 34GiB to GPU in 1.98228s (17.152GiB/s) | 20260506 15:14:51.945 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:14:51.945 | INFO | Total mem usage: 17.4 GiB | 20260506 15:14:51.945 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:14:51.945 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -13.5%) | 20260506 15:14:51.945 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:14:51.945 | INFO | High Water Breakdown: | 20260506 15:14:51.945 | INFO | 61% : 9.36 GiB ( 85.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:14:51.945 | INFO | 38.9% : 5.97 GiB ( 89.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:14:51.945 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:14:51.945 | INFO | ------------------------------------------ | 20260506 15:14:51.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:14:51.945 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:14:51.945 | INFO | Total mem usage: 26.5 GiB | 20260506 15:14:51.945 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:14:51.946 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:14:51.946 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:14:51.946 | INFO | High Water Breakdown: | 20260506 15:14:51.946 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:14:51.946 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:14:51.946 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:14:51.946 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:14:51.946 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:14:51.946 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:14:51.946 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:14:51.946 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:14:51.946 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:14:51.946 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:14:51.946 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:14:51.946 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:14:51.946 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:14:51.946 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:14:51.946 | INFO | ------------------------------------------ | 20260506 15:14:51.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:14:51.946 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:14:51.946 | INFO | Total mem usage: 25.6 GiB | 20260506 15:14:51.946 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:14:51.946 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:14:51.946 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:14:51.946 | INFO | High Water Breakdown: | 20260506 15:14:51.946 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:14:51.946 | INFO | ------------------------------------------ | 20260506 15:14:51.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:14:51.946 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:14:51.946 | INFO | Total mem usage: 15.7 GiB | 20260506 15:14:51.946 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:14:51.946 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:14:51.946 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 15:14:51.946 | INFO | High Water Breakdown: | 20260506 15:14:51.946 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:14:51.946 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:14:51.946 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:14:51.946 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:14:51.946 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:14:51.946 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:14:51.946 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:14:51.946 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:14:51.946 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:14:51.946 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:14:51.946 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:14:51.946 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:14:51.946 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:14:51.946 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:14:51.946 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:14:51.946 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:14:51.946 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:14:51.946 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:14:51.946 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:14:51.946 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:14:51.946 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:14:51.946 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:14:51.946 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:14:51.946 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:14:51.946 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:14:51.946 | INFO | ------------------------------------------ | 20260506 15:14:51.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:14:51.946 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:14:51.946 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:14:51.946 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:14:51.946 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.58e+01, 1.34e+01 / 1.89e+00 | 20260506 15:14:51.946 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 9.98e-02, 9.85e-03 / 5.30e-03 | 20260506 15:14:51.946 | INFO | PerfReport: 11.58%, Baseline -- 3.65e+00 / 4.46e+00, 5.60e-01 / 3.33e-01 | 20260506 15:14:51.946 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.44e+00, 1.85e-01 / 1.45e-01 | 20260506 15:14:51.946 | INFO | PerfReport: 15.84%, DME -- 3.99e+01 / 4.44e+01, 4.05e+00 / 1.94e+00 | 20260506 15:14:51.946 | INFO | PerfReport: 56.83%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.47e+00 / 1.82e+00 | 20260506 15:14:51.946 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.33e+00 / 2.90e+00, 3.95e-01 / 2.52e-01 | 20260506 15:14:51.946 | INFO | PerfReport: 3.56%, Metrics -- 1.12e+00 / 1.46e+00, 2.84e-01 / 1.68e-01 | 20260506 15:14:51.946 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.92e-01 / 4.51e-01, 6.06e-02 / 3.00e-02 | 20260506 15:14:51.946 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:14:51.946 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:14:51.946 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:14:51.946 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 15:14:51.947 | INFO | PerfReport: 21.55%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 15:14:51.947 | INFO | PerfReport: 21.18%, Download -- 2.07e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 15:14:51.947 | INFO | PerfReport: 32.24%, Compute -- 3.15e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 15:14:51.947 | INFO | PerfReport: 1.60%, LockUpload -- 1.56e+00 / 0.00e+00, 5.96e+00 / 0.00e+00 | 20260506 15:14:51.947 | INFO | PerfReport: 1.07%, LockDownload -- 1.05e+00 / 0.00e+00, 2.65e+00 / 0.00e+00 | 20260506 15:14:51.947 | INFO | PerfReport: 22.34%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 15:14:51.947 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260506 15:14:52.064 | INFO | Starting a round of Baz IO | 20260506 15:14:55.210 | INFO | Populating RTMetrics report | 20260506 15:14:55.211 | INFO | RTMetrics report populated | 20260506 15:14:55.211 | INFO | Opening tmp RTMetrics json file | 20260506 15:14:55.248 | INFO | Serializing RTMetrics to json | 20260506 15:14:55.292 | INFO | Serialization done, swapping tmp file now... | 20260506 15:14:55.298 | INFO | RTMetrics done | 20260506 15:14:55.657 | INFO | Finished a round of Baz IO: Wrote 2.108 GiB in 3.592 seconds | 20260506 15:14:55.657 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.41% / 0.01% / 62.58% | 20260506 15:14:55.657 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.15836% / 2.38419e-05% / 5.5631e-05% | 20260506 15:14:55.657 | INFO | Baz IO overall: | 20260506 15:14:55.657 | INFO | BytesWritten : 1223593406761 | 20260506 15:14:55.657 | INFO | NumEvents : 592154146799 | 20260506 15:14:57.084 | INFO | Analyzing chunk frames = [2351104,2351616) | 20260506 15:14:57.084 | INFO | MemInfo: { | 20260506 15:14:57.084 | INFO | "gpu" : " 17 GB", | 20260506 15:14:57.084 | INFO | "huge" : "", | 20260506 15:14:57.084 | INFO | "huge_shared" : " 27 GB", | 20260506 15:14:57.084 | INFO | "malloc" : " 14 GB", | 20260506 15:14:57.084 | INFO | "pinned" : " 29 GB" | 20260506 15:14:57.084 | INFO | } | 20260506 15:14:57.084 | INFO | Perf report for the last 5.13873 seconds: | 20260506 15:14:57.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:14:57.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:57.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:57.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:57.084 | INFO | ANALYSIS : 62.9%, 2.9, 95.4ms, 0 | 20260506 15:14:57.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:14:57.084 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 15:14:57.084 | INFO | PRE_HQ : 60.5%, 1.9, 59.1ms, 0 | 20260506 15:14:57.084 | INFO | BAZWRITER : 69.5%, 4.8, 171.2ms, 0 | 20260506 15:14:57.084 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 15:14:57.084 | INFO | Uploaded 36GiB to GPU in 2.05089s (17.5534GiB/s) | 20260506 15:14:57.084 | INFO | Downloaded 38GiB to GPU in 2.2208s (17.1109GiB/s) | 20260506 15:15:02.223 | INFO | Analyzing chunk frames = [2351616,2352128) | 20260506 15:15:02.223 | INFO | MemInfo: { | 20260506 15:15:02.223 | INFO | "gpu" : " 17 GB", | 20260506 15:15:02.223 | INFO | "huge" : "", | 20260506 15:15:02.223 | INFO | "huge_shared" : " 27 GB", | 20260506 15:15:02.223 | INFO | "malloc" : " 14 GB", | 20260506 15:15:02.223 | INFO | "pinned" : " 29 GB" | 20260506 15:15:02.223 | INFO | } | 20260506 15:15:02.223 | INFO | Perf report for the last 5.13932 seconds: | 20260506 15:15:02.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:15:02.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:02.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:02.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:02.223 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 15:15:02.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:02.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:02.223 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:15:02.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:02.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:02.224 | INFO | Uploaded 36GiB to GPU in 2.00138s (17.9876GiB/s) | 20260506 15:15:02.224 | INFO | Downloaded 34GiB to GPU in 1.93754s (17.548GiB/s) | 20260506 15:15:07.362 | INFO | Analyzing chunk frames = [2352128,2352640) | 20260506 15:15:07.362 | INFO | MemInfo: { | 20260506 15:15:07.362 | INFO | "gpu" : " 17 GB", | 20260506 15:15:07.362 | INFO | "huge" : "", | 20260506 15:15:07.362 | INFO | "huge_shared" : " 27 GB", | 20260506 15:15:07.362 | INFO | "malloc" : " 14 GB", | 20260506 15:15:07.362 | INFO | "pinned" : " 29 GB" | 20260506 15:15:07.362 | INFO | } | 20260506 15:15:07.362 | INFO | Perf report for the last 5.13919 seconds: | 20260506 15:15:07.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:15:07.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:07.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:07.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:07.362 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:15:07.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:07.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:07.362 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:15:07.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:07.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:07.362 | INFO | Uploaded 36GiB to GPU in 1.98882s (18.1012GiB/s) | 20260506 15:15:07.362 | INFO | Downloaded 34GiB to GPU in 1.94109s (17.5159GiB/s) | 20260506 15:15:12.501 | INFO | Analyzing chunk frames = [2352640,2353152) | 20260506 15:15:12.501 | INFO | MemInfo: { | 20260506 15:15:12.501 | INFO | "gpu" : " 17 GB", | 20260506 15:15:12.501 | INFO | "huge" : "", | 20260506 15:15:12.501 | INFO | "huge_shared" : " 27 GB", | 20260506 15:15:12.501 | INFO | "malloc" : " 14 GB", | 20260506 15:15:12.501 | INFO | "pinned" : " 29 GB" | 20260506 15:15:12.501 | INFO | } | 20260506 15:15:12.502 | INFO | Perf report for the last 5.13932 seconds: | 20260506 15:15:12.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:15:12.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:12.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:12.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:12.502 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 15:15:12.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:12.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:12.502 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:15:12.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:12.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:12.502 | INFO | Uploaded 36GiB to GPU in 2.00706s (17.9367GiB/s) | 20260506 15:15:12.502 | INFO | Downloaded 34GiB to GPU in 1.91581s (17.7471GiB/s) | 20260506 15:15:17.641 | INFO | Analyzing chunk frames = [2353152,2353664) | 20260506 15:15:17.641 | INFO | MemInfo: { | 20260506 15:15:17.641 | INFO | "gpu" : " 17 GB", | 20260506 15:15:17.641 | INFO | "huge" : "", | 20260506 15:15:17.641 | INFO | "huge_shared" : " 27 GB", | 20260506 15:15:17.641 | INFO | "malloc" : " 14 GB", | 20260506 15:15:17.641 | INFO | "pinned" : " 29 GB" | 20260506 15:15:17.641 | INFO | } | 20260506 15:15:17.641 | INFO | Perf report for the last 5.1394 seconds: | 20260506 15:15:17.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:15:17.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:17.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:17.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:17.641 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 15:15:17.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:17.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:17.641 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:15:17.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:17.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:17.641 | INFO | Uploaded 36GiB to GPU in 2.04333s (17.6183GiB/s) | 20260506 15:15:17.641 | INFO | Downloaded 34GiB to GPU in 1.93743s (17.549GiB/s) | 20260506 15:15:22.781 | INFO | Analyzing chunk frames = [2353664,2354176) | 20260506 15:15:22.781 | INFO | MemInfo: { | 20260506 15:15:22.781 | INFO | "gpu" : " 17 GB", | 20260506 15:15:22.781 | INFO | "huge" : "", | 20260506 15:15:22.781 | INFO | "huge_shared" : " 27 GB", | 20260506 15:15:22.781 | INFO | "malloc" : " 14 GB", | 20260506 15:15:22.781 | INFO | "pinned" : " 29 GB" | 20260506 15:15:22.781 | INFO | } | 20260506 15:15:22.781 | INFO | Perf report for the last 5.14012 seconds: | 20260506 15:15:22.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:15:22.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:22.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:22.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:22.781 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 15:15:22.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:22.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:22.781 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 15:15:22.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:22.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:22.781 | INFO | Uploaded 36GiB to GPU in 2.03769s (17.6671GiB/s) | 20260506 15:15:22.781 | INFO | Downloaded 34GiB to GPU in 1.89802s (17.9134GiB/s) | 20260506 15:15:27.920 | INFO | Analyzing chunk frames = [2354176,2354688) | 20260506 15:15:27.920 | INFO | MemInfo: { | 20260506 15:15:27.920 | INFO | "gpu" : " 17 GB", | 20260506 15:15:27.921 | INFO | "huge" : "", | 20260506 15:15:27.921 | INFO | "huge_shared" : " 27 GB", | 20260506 15:15:27.921 | INFO | "malloc" : " 14 GB", | 20260506 15:15:27.921 | INFO | "pinned" : " 29 GB" | 20260506 15:15:27.921 | INFO | } | 20260506 15:15:27.921 | INFO | Perf report for the last 5.13971 seconds: | 20260506 15:15:27.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:15:27.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:27.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:27.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:27.921 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:15:27.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:27.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:27.921 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:15:27.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:27.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:27.921 | INFO | Uploaded 36GiB to GPU in 1.982s (18.1635GiB/s) | 20260506 15:15:27.921 | INFO | Downloaded 34GiB to GPU in 1.96238s (17.3259GiB/s) | 20260506 15:15:33.059 | INFO | Analyzing chunk frames = [2354688,2355200) | 20260506 15:15:33.059 | INFO | MemInfo: { | 20260506 15:15:33.059 | INFO | "gpu" : " 17 GB", | 20260506 15:15:33.059 | INFO | "huge" : "", | 20260506 15:15:33.059 | INFO | "huge_shared" : " 27 GB", | 20260506 15:15:33.059 | INFO | "malloc" : " 14 GB", | 20260506 15:15:33.059 | INFO | "pinned" : " 29 GB" | 20260506 15:15:33.059 | INFO | } | 20260506 15:15:33.059 | INFO | Perf report for the last 5.1384 seconds: | 20260506 15:15:33.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:15:33.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:33.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:33.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:33.059 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:15:33.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:33.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:33.059 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 15:15:33.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:33.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:33.060 | INFO | Uploaded 36GiB to GPU in 2.02098s (17.8131GiB/s) | 20260506 15:15:33.060 | INFO | Downloaded 34GiB to GPU in 1.98621s (17.1181GiB/s) | 20260506 15:15:33.181 | INFO | Starting a round of Baz IO | 20260506 15:15:36.326 | INFO | Populating RTMetrics report | 20260506 15:15:36.326 | INFO | RTMetrics report populated | 20260506 15:15:36.326 | INFO | Opening tmp RTMetrics json file | 20260506 15:15:36.373 | INFO | Serializing RTMetrics to json | 20260506 15:15:36.433 | INFO | Serialization done, swapping tmp file now... | 20260506 15:15:36.440 | INFO | RTMetrics done | 20260506 15:15:36.832 | INFO | Finished a round of Baz IO: Wrote 2.108 GiB in 3.651 seconds | 20260506 15:15:36.832 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.39% / 0.01% / 62.59% | 20260506 15:15:36.832 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.15462% / 1.98682e-05% / 4.37101e-05% | 20260506 15:15:36.832 | INFO | Baz IO overall: | 20260506 15:15:36.832 | INFO | BytesWritten : 1225856587583 | 20260506 15:15:36.832 | INFO | NumEvents : 593249222544 | 20260506 15:15:38.199 | INFO | Analyzing chunk frames = [2355200,2355712) | 20260506 15:15:38.199 | INFO | MemInfo: { | 20260506 15:15:38.199 | INFO | "gpu" : " 17 GB", | 20260506 15:15:38.199 | INFO | "huge" : "", | 20260506 15:15:38.199 | INFO | "huge_shared" : " 27 GB", | 20260506 15:15:38.199 | INFO | "malloc" : " 14 GB", | 20260506 15:15:38.199 | INFO | "pinned" : " 29 GB" | 20260506 15:15:38.199 | INFO | } | 20260506 15:15:38.199 | INFO | Perf report for the last 5.13999 seconds: | 20260506 15:15:38.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:15:38.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:38.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:38.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:38.199 | INFO | ANALYSIS : 63.0%, 2.9, 95.2ms, 0 | 20260506 15:15:38.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:38.199 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260506 15:15:38.199 | INFO | PRE_HQ : 60.7%, 1.9, 59.4ms, 0 | 20260506 15:15:38.199 | INFO | BAZWRITER : 70.5%, 4.8, 175.5ms, 0 | 20260506 15:15:38.199 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 15:15:38.200 | INFO | Uploaded 36GiB to GPU in 2.07548s (17.3454GiB/s) | 20260506 15:15:38.200 | INFO | Downloaded 38GiB to GPU in 2.21978s (17.1189GiB/s) | 20260506 15:15:43.338 | INFO | Analyzing chunk frames = [2355712,2356224) | 20260506 15:15:43.338 | INFO | MemInfo: { | 20260506 15:15:43.338 | INFO | "gpu" : " 17 GB", | 20260506 15:15:43.338 | INFO | "huge" : "", | 20260506 15:15:43.338 | INFO | "huge_shared" : " 27 GB", | 20260506 15:15:43.338 | INFO | "malloc" : " 14 GB", | 20260506 15:15:43.338 | INFO | "pinned" : " 29 GB" | 20260506 15:15:43.338 | INFO | } | 20260506 15:15:43.339 | INFO | Perf report for the last 5.13928 seconds: | 20260506 15:15:43.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:15:43.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:43.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:43.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:43.339 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 15:15:43.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:43.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:43.339 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:15:43.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:43.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:43.339 | INFO | Uploaded 36GiB to GPU in 1.99412s (18.0531GiB/s) | 20260506 15:15:43.339 | INFO | Downloaded 34GiB to GPU in 1.93206s (17.5978GiB/s) | 20260506 15:15:48.477 | INFO | Analyzing chunk frames = [2356224,2356736) | 20260506 15:15:48.477 | INFO | MemInfo: { | 20260506 15:15:48.477 | INFO | "gpu" : " 17 GB", | 20260506 15:15:48.478 | INFO | "huge" : "", | 20260506 15:15:48.478 | INFO | "huge_shared" : " 27 GB", | 20260506 15:15:48.478 | INFO | "malloc" : " 14 GB", | 20260506 15:15:48.478 | INFO | "pinned" : " 29 GB" | 20260506 15:15:48.478 | INFO | } | 20260506 15:15:48.478 | INFO | Perf report for the last 5.13927 seconds: | 20260506 15:15:48.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:15:48.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:48.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:48.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:48.478 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:15:48.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:48.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:48.478 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:15:48.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:48.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:48.478 | INFO | Uploaded 36GiB to GPU in 1.98739s (18.1142GiB/s) | 20260506 15:15:48.478 | INFO | Downloaded 34GiB to GPU in 1.94384s (17.4912GiB/s) | 20260506 15:15:53.616 | INFO | Analyzing chunk frames = [2356736,2357248) | 20260506 15:15:53.616 | INFO | MemInfo: { | 20260506 15:15:53.616 | INFO | "gpu" : " 17 GB", | 20260506 15:15:53.616 | INFO | "huge" : "", | 20260506 15:15:53.616 | INFO | "huge_shared" : " 27 GB", | 20260506 15:15:53.616 | INFO | "malloc" : " 14 GB", | 20260506 15:15:53.616 | INFO | "pinned" : " 29 GB" | 20260506 15:15:53.616 | INFO | } | 20260506 15:15:53.617 | INFO | Perf report for the last 5.13869 seconds: | 20260506 15:15:53.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:15:53.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:53.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:53.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:53.617 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 15:15:53.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:53.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:53.617 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:15:53.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:53.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:53.617 | INFO | Uploaded 36GiB to GPU in 1.99897s (18.0093GiB/s) | 20260506 15:15:53.617 | INFO | Downloaded 34GiB to GPU in 1.91266s (17.7763GiB/s) | 20260506 15:15:58.755 | INFO | Analyzing chunk frames = [2357248,2357760) | 20260506 15:15:58.755 | INFO | MemInfo: { | 20260506 15:15:58.755 | INFO | "gpu" : " 17 GB", | 20260506 15:15:58.755 | INFO | "huge" : "", | 20260506 15:15:58.755 | INFO | "huge_shared" : " 27 GB", | 20260506 15:15:58.755 | INFO | "malloc" : " 14 GB", | 20260506 15:15:58.755 | INFO | "pinned" : " 29 GB" | 20260506 15:15:58.755 | INFO | } | 20260506 15:15:58.755 | INFO | Perf report for the last 5.13894 seconds: | 20260506 15:15:58.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:15:58.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:58.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:58.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:58.756 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 15:15:58.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:58.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:58.756 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:15:58.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:58.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:15:58.756 | INFO | Uploaded 36GiB to GPU in 1.97579s (18.2205GiB/s) | 20260506 15:15:58.756 | INFO | Downloaded 34GiB to GPU in 1.92681s (17.6457GiB/s) | 20260506 15:16:03.895 | INFO | Analyzing chunk frames = [2357760,2358272) | 20260506 15:16:03.895 | INFO | MemInfo: { | 20260506 15:16:03.895 | INFO | "gpu" : " 17 GB", | 20260506 15:16:03.895 | INFO | "huge" : "", | 20260506 15:16:03.895 | INFO | "huge_shared" : " 27 GB", | 20260506 15:16:03.895 | INFO | "malloc" : " 14 GB", | 20260506 15:16:03.895 | INFO | "pinned" : " 29 GB" | 20260506 15:16:03.895 | INFO | } | 20260506 15:16:03.895 | INFO | Perf report for the last 5.13993 seconds: | 20260506 15:16:03.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:16:03.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:03.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:03.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:03.895 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 15:16:03.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:03.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:03.895 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 15:16:03.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:03.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:03.896 | INFO | Uploaded 36GiB to GPU in 2.07949s (17.312GiB/s) | 20260506 15:16:03.896 | INFO | Downloaded 34GiB to GPU in 1.97343s (17.2288GiB/s) | 20260506 15:16:09.035 | INFO | Analyzing chunk frames = [2358272,2358784) | 20260506 15:16:09.035 | INFO | MemInfo: { | 20260506 15:16:09.035 | INFO | "gpu" : " 17 GB", | 20260506 15:16:09.035 | INFO | "huge" : "", | 20260506 15:16:09.035 | INFO | "huge_shared" : " 27 GB", | 20260506 15:16:09.035 | INFO | "malloc" : " 14 GB", | 20260506 15:16:09.035 | INFO | "pinned" : " 29 GB" | 20260506 15:16:09.035 | INFO | } | 20260506 15:16:09.035 | INFO | Perf report for the last 5.1399 seconds: | 20260506 15:16:09.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:16:09.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:09.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:09.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:09.035 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:16:09.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:09.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:09.035 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 15:16:09.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:09.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:09.036 | INFO | Uploaded 36GiB to GPU in 2.0073s (17.9345GiB/s) | 20260506 15:16:09.036 | INFO | Downloaded 34GiB to GPU in 2.01063s (16.9102GiB/s) | 20260506 15:16:14.173 | INFO | Analyzing chunk frames = [2358784,2359296) | 20260506 15:16:14.173 | INFO | MemInfo: { | 20260506 15:16:14.173 | INFO | "gpu" : " 17 GB", | 20260506 15:16:14.173 | INFO | "huge" : "", | 20260506 15:16:14.173 | INFO | "huge_shared" : " 27 GB", | 20260506 15:16:14.173 | INFO | "malloc" : " 14 GB", | 20260506 15:16:14.173 | INFO | "pinned" : " 29 GB" | 20260506 15:16:14.173 | INFO | } | 20260506 15:16:14.173 | INFO | Perf report for the last 5.13814 seconds: | 20260506 15:16:14.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:16:14.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:14.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:14.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:14.174 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 15:16:14.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:14.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:14.174 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:16:14.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:14.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:14.174 | INFO | Uploaded 36GiB to GPU in 2.03591s (17.6825GiB/s) | 20260506 15:16:14.174 | INFO | Downloaded 34GiB to GPU in 1.98247s (17.1503GiB/s) | 20260506 15:16:14.174 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:16:14.174 | INFO | Total mem usage: 17.4 GiB | 20260506 15:16:14.174 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:16:14.174 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -9.92%) | 20260506 15:16:14.174 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:16:14.174 | INFO | High Water Breakdown: | 20260506 15:16:14.174 | INFO | 61.5% : 9.73 GiB ( 89.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:16:14.174 | INFO | 38.4% : 6.09 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:16:14.174 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:16:14.174 | INFO | ------------------------------------------ | 20260506 15:16:14.174 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:16:14.174 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:16:14.174 | INFO | Total mem usage: 26.5 GiB | 20260506 15:16:14.174 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:16:14.174 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:16:14.174 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:16:14.174 | INFO | High Water Breakdown: | 20260506 15:16:14.174 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:16:14.174 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:16:14.174 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:16:14.174 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:16:14.174 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:16:14.174 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:16:14.174 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:16:14.174 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:16:14.174 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:16:14.174 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:16:14.174 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:16:14.174 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:16:14.174 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:16:14.174 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:16:14.174 | INFO | ------------------------------------------ | 20260506 15:16:14.174 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:16:14.174 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:16:14.174 | INFO | Total mem usage: 25.6 GiB | 20260506 15:16:14.174 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:16:14.174 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:16:14.174 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:16:14.174 | INFO | High Water Breakdown: | 20260506 15:16:14.174 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:16:14.174 | INFO | ------------------------------------------ | 20260506 15:16:14.174 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:16:14.174 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:16:14.174 | INFO | Total mem usage: 15.7 GiB | 20260506 15:16:14.174 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:16:14.174 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:16:14.174 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 15:16:14.174 | INFO | High Water Breakdown: | 20260506 15:16:14.174 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:16:14.174 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:16:14.174 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:16:14.174 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:16:14.174 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:16:14.174 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:16:14.174 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:16:14.174 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:16:14.175 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:16:14.175 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:16:14.175 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:16:14.175 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:16:14.175 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:16:14.175 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:16:14.175 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:16:14.175 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:16:14.175 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:16:14.175 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:16:14.175 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:16:14.175 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:16:14.175 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:16:14.175 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:16:14.175 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:16:14.175 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:16:14.175 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:16:14.175 | INFO | ------------------------------------------ | 20260506 15:16:14.175 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:16:14.175 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:16:14.175 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:16:14.175 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:16:14.175 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.50e+01, 1.30e+01 / 1.74e+00 | 20260506 15:16:14.175 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.00e-01, 1.03e-02 / 9.22e-03 | 20260506 15:16:14.175 | INFO | PerfReport: 11.58%, Baseline -- 3.64e+00 / 4.35e+00, 5.59e-01 / 4.28e-01 | 20260506 15:16:14.175 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.44e+00, 1.85e-01 / 1.77e-01 | 20260506 15:16:14.175 | INFO | PerfReport: 15.81%, DME -- 3.97e+01 / 4.33e+01, 3.54e+00 / 2.06e+00 | 20260506 15:16:14.175 | INFO | PerfReport: 56.85%, FrameLabeling -- 1.79e+01 / 2.09e+01, 1.46e+00 / 2.28e+00 | 20260506 15:16:14.175 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.33e+00 / 2.85e+00, 3.99e-01 / 3.17e-01 | 20260506 15:16:14.175 | INFO | PerfReport: 3.57%, Metrics -- 1.12e+00 / 1.51e+00, 2.86e-01 / 1.87e-01 | 20260506 15:16:14.175 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.90e-01 / 4.38e-01, 6.01e-02 / 2.79e-02 | 20260506 15:16:14.175 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:16:14.175 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:16:14.175 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:16:14.175 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 15:16:14.175 | INFO | PerfReport: 21.56%, Upload -- 2.10e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260506 15:16:14.175 | INFO | PerfReport: 21.17%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 15:16:14.175 | INFO | PerfReport: 32.23%, Compute -- 3.14e+01 / 0.00e+00, 1.30e+01 / 0.00e+00 | 20260506 15:16:14.175 | INFO | PerfReport: 1.54%, LockUpload -- 1.50e+00 / 0.00e+00, 5.70e+00 / 0.00e+00 | 20260506 15:16:14.175 | INFO | PerfReport: 1.10%, LockDownload -- 1.08e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 15:16:14.175 | INFO | PerfReport: 22.38%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 15:16:14.175 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.46e-02 / 0.00e+00 | 20260506 15:16:14.292 | INFO | Starting a round of Baz IO | 20260506 15:16:17.439 | INFO | Populating RTMetrics report | 20260506 15:16:17.440 | INFO | RTMetrics report populated | 20260506 15:16:17.440 | INFO | Opening tmp RTMetrics json file | 20260506 15:16:17.547 | INFO | Serializing RTMetrics to json | 20260506 15:16:17.552 | INFO | Serialization done, swapping tmp file now... | 20260506 15:16:17.554 | INFO | RTMetrics done | 20260506 15:16:17.651 | INFO | Finished a round of Baz IO: Wrote 2.105 GiB in 3.358 seconds | 20260506 15:16:17.651 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.38% / 0.01% / 62.61% | 20260506 15:16:17.651 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.14426% / 2.38419e-05% / 7.54992e-05% | 20260506 15:16:17.651 | INFO | Baz IO overall: | 20260506 15:16:17.651 | INFO | BytesWritten : 1228116991656 | 20260506 15:16:17.651 | INFO | NumEvents : 594342424809 | 20260506 15:16:19.312 | INFO | Analyzing chunk frames = [2359296,2359808) | 20260506 15:16:19.312 | INFO | MemInfo: { | 20260506 15:16:19.312 | INFO | "gpu" : " 17 GB", | 20260506 15:16:19.312 | INFO | "huge" : "", | 20260506 15:16:19.312 | INFO | "huge_shared" : " 27 GB", | 20260506 15:16:19.312 | INFO | "malloc" : " 14 GB", | 20260506 15:16:19.312 | INFO | "pinned" : " 29 GB" | 20260506 15:16:19.312 | INFO | } | 20260506 15:16:19.312 | INFO | Perf report for the last 5.13898 seconds: | 20260506 15:16:19.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:16:19.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:19.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:19.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:19.313 | INFO | ANALYSIS : 63.0%, 2.9, 95.3ms, 0 | 20260506 15:16:19.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:19.313 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 15:16:19.313 | INFO | PRE_HQ : 60.8%, 1.9, 59.3ms, 0 | 20260506 15:16:19.313 | INFO | BAZWRITER : 64.7%, 4.8, 159.6ms, 0 | 20260506 15:16:19.313 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 15:16:19.313 | INFO | Uploaded 36GiB to GPU in 2.0693s (17.3972GiB/s) | 20260506 15:16:19.313 | INFO | Downloaded 38GiB to GPU in 2.24379s (16.9357GiB/s) | 20260506 15:16:24.453 | INFO | Analyzing chunk frames = [2359808,2360320) | 20260506 15:16:24.453 | INFO | MemInfo: { | 20260506 15:16:24.453 | INFO | "gpu" : " 17 GB", | 20260506 15:16:24.453 | INFO | "huge" : "", | 20260506 15:16:24.453 | INFO | "huge_shared" : " 27 GB", | 20260506 15:16:24.453 | INFO | "malloc" : " 14 GB", | 20260506 15:16:24.453 | INFO | "pinned" : " 29 GB" | 20260506 15:16:24.453 | INFO | } | 20260506 15:16:24.453 | INFO | Perf report for the last 5.14047 seconds: | 20260506 15:16:24.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:16:24.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:24.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:24.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:24.453 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:16:24.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:24.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:24.453 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:16:24.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:24.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:24.453 | INFO | Uploaded 36GiB to GPU in 1.99481s (18.0469GiB/s) | 20260506 15:16:24.453 | INFO | Downloaded 34GiB to GPU in 1.93297s (17.5895GiB/s) | 20260506 15:16:29.592 | INFO | Analyzing chunk frames = [2360320,2360832) | 20260506 15:16:29.592 | INFO | MemInfo: { | 20260506 15:16:29.592 | INFO | "gpu" : " 17 GB", | 20260506 15:16:29.592 | INFO | "huge" : "", | 20260506 15:16:29.592 | INFO | "huge_shared" : " 27 GB", | 20260506 15:16:29.593 | INFO | "malloc" : " 14 GB", | 20260506 15:16:29.593 | INFO | "pinned" : " 29 GB" | 20260506 15:16:29.593 | INFO | } | 20260506 15:16:29.593 | INFO | Perf report for the last 5.13977 seconds: | 20260506 15:16:29.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:16:29.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:29.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:29.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:29.593 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:16:29.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:29.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:29.593 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:16:29.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:29.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:29.593 | INFO | Uploaded 36GiB to GPU in 1.99077s (18.0835GiB/s) | 20260506 15:16:29.593 | INFO | Downloaded 34GiB to GPU in 1.94351s (17.4941GiB/s) | 20260506 15:16:34.731 | INFO | Analyzing chunk frames = [2360832,2361344) | 20260506 15:16:34.731 | INFO | MemInfo: { | 20260506 15:16:34.731 | INFO | "gpu" : " 17 GB", | 20260506 15:16:34.731 | INFO | "huge" : "", | 20260506 15:16:34.731 | INFO | "huge_shared" : " 27 GB", | 20260506 15:16:34.731 | INFO | "malloc" : " 14 GB", | 20260506 15:16:34.731 | INFO | "pinned" : " 29 GB" | 20260506 15:16:34.731 | INFO | } | 20260506 15:16:34.731 | INFO | Perf report for the last 5.13846 seconds: | 20260506 15:16:34.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:16:34.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:34.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:34.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:34.731 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 15:16:34.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:34.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:34.731 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:16:34.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:34.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:34.732 | INFO | Uploaded 36GiB to GPU in 2.02369s (17.7893GiB/s) | 20260506 15:16:34.732 | INFO | Downloaded 34GiB to GPU in 1.91249s (17.7778GiB/s) | 20260506 15:16:39.870 | INFO | Analyzing chunk frames = [2361344,2361856) | 20260506 15:16:39.870 | INFO | MemInfo: { | 20260506 15:16:39.870 | INFO | "gpu" : " 17 GB", | 20260506 15:16:39.870 | INFO | "huge" : "", | 20260506 15:16:39.870 | INFO | "huge_shared" : " 27 GB", | 20260506 15:16:39.870 | INFO | "malloc" : " 14 GB", | 20260506 15:16:39.870 | INFO | "pinned" : " 29 GB" | 20260506 15:16:39.870 | INFO | } | 20260506 15:16:39.870 | INFO | Perf report for the last 5.13929 seconds: | 20260506 15:16:39.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:16:39.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:39.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:39.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:39.871 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 15:16:39.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:39.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:39.871 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:16:39.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:39.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:39.871 | INFO | Uploaded 36GiB to GPU in 2.03149s (17.721GiB/s) | 20260506 15:16:39.871 | INFO | Downloaded 34GiB to GPU in 1.92515s (17.661GiB/s) | 20260506 15:16:45.009 | INFO | Analyzing chunk frames = [2361856,2362368) | 20260506 15:16:45.009 | INFO | MemInfo: { | 20260506 15:16:45.009 | INFO | "gpu" : " 17 GB", | 20260506 15:16:45.009 | INFO | "huge" : "", | 20260506 15:16:45.009 | INFO | "huge_shared" : " 27 GB", | 20260506 15:16:45.009 | INFO | "malloc" : " 14 GB", | 20260506 15:16:45.009 | INFO | "pinned" : " 29 GB" | 20260506 15:16:45.009 | INFO | } | 20260506 15:16:45.009 | INFO | Perf report for the last 5.1385 seconds: | 20260506 15:16:45.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:16:45.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:45.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:45.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:45.009 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 15:16:45.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:45.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:45.009 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 15:16:45.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:45.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:45.009 | INFO | Uploaded 36GiB to GPU in 2.08229s (17.2887GiB/s) | 20260506 15:16:45.009 | INFO | Downloaded 34GiB to GPU in 1.97315s (17.2313GiB/s) | 20260506 15:16:50.150 | INFO | Analyzing chunk frames = [2362368,2362880) | 20260506 15:16:50.150 | INFO | MemInfo: { | 20260506 15:16:50.150 | INFO | "gpu" : " 17 GB", | 20260506 15:16:50.150 | INFO | "huge" : "", | 20260506 15:16:50.150 | INFO | "huge_shared" : " 27 GB", | 20260506 15:16:50.150 | INFO | "malloc" : " 14 GB", | 20260506 15:16:50.150 | INFO | "pinned" : " 29 GB" | 20260506 15:16:50.150 | INFO | } | 20260506 15:16:50.150 | INFO | Perf report for the last 5.14119 seconds: | 20260506 15:16:50.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:16:50.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:50.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:50.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:50.150 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:16:50.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:50.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:50.150 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 15:16:50.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:50.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:50.151 | INFO | Uploaded 36GiB to GPU in 2.0236s (17.7901GiB/s) | 20260506 15:16:50.151 | INFO | Downloaded 34GiB to GPU in 2.01694s (16.8572GiB/s) | 20260506 15:16:55.289 | INFO | Analyzing chunk frames = [2362880,2363392) | 20260506 15:16:55.289 | INFO | MemInfo: { | 20260506 15:16:55.289 | INFO | "gpu" : " 17 GB", | 20260506 15:16:55.289 | INFO | "huge" : "", | 20260506 15:16:55.289 | INFO | "huge_shared" : " 27 GB", | 20260506 15:16:55.289 | INFO | "malloc" : " 14 GB", | 20260506 15:16:55.289 | INFO | "pinned" : " 29 GB" | 20260506 15:16:55.289 | INFO | } | 20260506 15:16:55.289 | INFO | Perf report for the last 5.13917 seconds: | 20260506 15:16:55.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:16:55.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:55.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:55.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:55.289 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:16:55.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:55.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:55.289 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:16:55.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:55.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:16:55.290 | INFO | Uploaded 36GiB to GPU in 2.00827s (17.9259GiB/s) | 20260506 15:16:55.290 | INFO | Downloaded 34GiB to GPU in 1.97945s (17.1765GiB/s) | 20260506 15:16:55.409 | INFO | Starting a round of Baz IO | 20260506 15:16:58.562 | INFO | Populating RTMetrics report | 20260506 15:16:58.562 | INFO | RTMetrics report populated | 20260506 15:16:58.562 | INFO | Opening tmp RTMetrics json file | 20260506 15:16:58.603 | INFO | Serializing RTMetrics to json | 20260506 15:16:58.675 | INFO | Serialization done, swapping tmp file now... | 20260506 15:16:58.706 | INFO | RTMetrics done | 20260506 15:16:59.019 | INFO | Finished a round of Baz IO: Wrote 2.105 GiB in 3.609 seconds | 20260506 15:16:59.019 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.37% / 0.01% / 62.62% | 20260506 15:16:59.019 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.13986% / 7.94729e-06% / 4.76837e-05% | 20260506 15:16:59.020 | INFO | Baz IO overall: | 20260506 15:16:59.020 | INFO | BytesWritten : 1230377077332 | 20260506 15:16:59.020 | INFO | NumEvents : 595435501154 | 20260506 15:17:00.428 | INFO | Analyzing chunk frames = [2363392,2363904) | 20260506 15:17:00.428 | INFO | MemInfo: { | 20260506 15:17:00.428 | INFO | "gpu" : " 17 GB", | 20260506 15:17:00.428 | INFO | "huge" : "", | 20260506 15:17:00.428 | INFO | "huge_shared" : " 27 GB", | 20260506 15:17:00.428 | INFO | "malloc" : " 14 GB", | 20260506 15:17:00.428 | INFO | "pinned" : " 29 GB" | 20260506 15:17:00.428 | INFO | } | 20260506 15:17:00.428 | INFO | Perf report for the last 5.13832 seconds: | 20260506 15:17:00.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:17:00.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:00.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:00.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:00.428 | INFO | ANALYSIS : 62.9%, 2.9, 95.6ms, 0 | 20260506 15:17:00.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:00.428 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260506 15:17:00.428 | INFO | PRE_HQ : 60.5%, 1.9, 59.3ms, 0 | 20260506 15:17:00.428 | INFO | BAZWRITER : 69.5%, 4.8, 171.1ms, 0 | 20260506 15:17:00.428 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 15:17:00.428 | INFO | Uploaded 36GiB to GPU in 2.05259s (17.5388GiB/s) | 20260506 15:17:00.428 | INFO | Downloaded 38GiB to GPU in 2.22688s (17.0642GiB/s) | 20260506 15:17:05.567 | INFO | Analyzing chunk frames = [2363904,2364416) | 20260506 15:17:05.567 | INFO | MemInfo: { | 20260506 15:17:05.567 | INFO | "gpu" : " 17 GB", | 20260506 15:17:05.567 | INFO | "huge" : "", | 20260506 15:17:05.567 | INFO | "huge_shared" : " 27 GB", | 20260506 15:17:05.567 | INFO | "malloc" : " 14 GB", | 20260506 15:17:05.567 | INFO | "pinned" : " 29 GB" | 20260506 15:17:05.567 | INFO | } | 20260506 15:17:05.567 | INFO | Perf report for the last 5.13958 seconds: | 20260506 15:17:05.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:17:05.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:05.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:05.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:05.567 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:17:05.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:05.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:05.567 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:17:05.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:05.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:05.568 | INFO | Uploaded 36GiB to GPU in 1.981s (18.1726GiB/s) | 20260506 15:17:05.568 | INFO | Downloaded 34GiB to GPU in 1.93738s (17.5495GiB/s) | 20260506 15:17:10.706 | INFO | Analyzing chunk frames = [2364416,2364928) | 20260506 15:17:10.706 | INFO | MemInfo: { | 20260506 15:17:10.706 | INFO | "gpu" : " 17 GB", | 20260506 15:17:10.706 | INFO | "huge" : "", | 20260506 15:17:10.706 | INFO | "huge_shared" : " 27 GB", | 20260506 15:17:10.706 | INFO | "malloc" : " 14 GB", | 20260506 15:17:10.706 | INFO | "pinned" : " 29 GB" | 20260506 15:17:10.706 | INFO | } | 20260506 15:17:10.706 | INFO | Perf report for the last 5.13886 seconds: | 20260506 15:17:10.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:17:10.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:10.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:10.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:10.706 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:17:10.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:10.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:10.706 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:17:10.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:10.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:10.707 | INFO | Uploaded 36GiB to GPU in 1.96435s (18.3267GiB/s) | 20260506 15:17:10.707 | INFO | Downloaded 34GiB to GPU in 1.94s (17.5258GiB/s) | 20260506 15:17:15.846 | INFO | Analyzing chunk frames = [2364928,2365440) | 20260506 15:17:15.846 | INFO | MemInfo: { | 20260506 15:17:15.846 | INFO | "gpu" : " 17 GB", | 20260506 15:17:15.846 | INFO | "huge" : "", | 20260506 15:17:15.846 | INFO | "huge_shared" : " 27 GB", | 20260506 15:17:15.846 | INFO | "malloc" : " 14 GB", | 20260506 15:17:15.846 | INFO | "pinned" : " 29 GB" | 20260506 15:17:15.846 | INFO | } | 20260506 15:17:15.846 | INFO | Perf report for the last 5.13964 seconds: | 20260506 15:17:15.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:17:15.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:15.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:15.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:15.846 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:17:15.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:15.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:15.846 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:17:15.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:15.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:15.846 | INFO | Uploaded 36GiB to GPU in 2.01771s (17.842GiB/s) | 20260506 15:17:15.846 | INFO | Downloaded 34GiB to GPU in 1.91675s (17.7383GiB/s) | 20260506 15:17:20.985 | INFO | Analyzing chunk frames = [2365440,2365952) | 20260506 15:17:20.985 | INFO | MemInfo: { | 20260506 15:17:20.985 | INFO | "gpu" : " 17 GB", | 20260506 15:17:20.985 | INFO | "huge" : "", | 20260506 15:17:20.985 | INFO | "huge_shared" : " 27 GB", | 20260506 15:17:20.985 | INFO | "malloc" : " 14 GB", | 20260506 15:17:20.985 | INFO | "pinned" : " 29 GB" | 20260506 15:17:20.985 | INFO | } | 20260506 15:17:20.985 | INFO | Perf report for the last 5.13934 seconds: | 20260506 15:17:20.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:17:20.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:20.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:20.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:20.985 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 15:17:20.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:20.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:20.985 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:17:20.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:20.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:20.986 | INFO | Uploaded 36GiB to GPU in 1.99242s (18.0685GiB/s) | 20260506 15:17:20.986 | INFO | Downloaded 34GiB to GPU in 1.93328s (17.5867GiB/s) | 20260506 15:17:26.124 | INFO | Analyzing chunk frames = [2365952,2366464) | 20260506 15:17:26.124 | INFO | MemInfo: { | 20260506 15:17:26.124 | INFO | "gpu" : " 17 GB", | 20260506 15:17:26.124 | INFO | "huge" : "", | 20260506 15:17:26.124 | INFO | "huge_shared" : " 27 GB", | 20260506 15:17:26.124 | INFO | "malloc" : " 14 GB", | 20260506 15:17:26.124 | INFO | "pinned" : " 29 GB" | 20260506 15:17:26.124 | INFO | } | 20260506 15:17:26.124 | INFO | Perf report for the last 5.13913 seconds: | 20260506 15:17:26.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:17:26.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:26.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:26.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:26.124 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:17:26.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:26.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:26.124 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 15:17:26.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:26.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:26.125 | INFO | Uploaded 36GiB to GPU in 2.0731s (17.3653GiB/s) | 20260506 15:17:26.125 | INFO | Downloaded 34GiB to GPU in 1.97356s (17.2277GiB/s) | 20260506 15:17:31.264 | INFO | Analyzing chunk frames = [2366464,2366976) | 20260506 15:17:31.264 | INFO | MemInfo: { | 20260506 15:17:31.264 | INFO | "gpu" : " 17 GB", | 20260506 15:17:31.264 | INFO | "huge" : "", | 20260506 15:17:31.264 | INFO | "huge_shared" : " 27 GB", | 20260506 15:17:31.264 | INFO | "malloc" : " 14 GB", | 20260506 15:17:31.264 | INFO | "pinned" : " 29 GB" | 20260506 15:17:31.264 | INFO | } | 20260506 15:17:31.264 | INFO | Perf report for the last 5.13974 seconds: | 20260506 15:17:31.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:17:31.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:31.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:31.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:31.264 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:17:31.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:31.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:31.264 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 15:17:31.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:31.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:31.264 | INFO | Uploaded 36GiB to GPU in 2.01079s (17.9034GiB/s) | 20260506 15:17:31.264 | INFO | Downloaded 34GiB to GPU in 2.03474s (16.7098GiB/s) | 20260506 15:17:36.404 | INFO | Analyzing chunk frames = [2366976,2367488) | 20260506 15:17:36.404 | INFO | MemInfo: { | 20260506 15:17:36.404 | INFO | "gpu" : " 17 GB", | 20260506 15:17:36.404 | INFO | "huge" : "", | 20260506 15:17:36.404 | INFO | "huge_shared" : " 27 GB", | 20260506 15:17:36.404 | INFO | "malloc" : " 14 GB", | 20260506 15:17:36.404 | INFO | "pinned" : " 29 GB" | 20260506 15:17:36.404 | INFO | } | 20260506 15:17:36.404 | INFO | Perf report for the last 5.1402 seconds: | 20260506 15:17:36.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:17:36.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:36.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:36.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:36.404 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 15:17:36.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:36.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:36.404 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:17:36.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:36.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:36.405 | INFO | Uploaded 36GiB to GPU in 2.0348s (17.6921GiB/s) | 20260506 15:17:36.405 | INFO | Downloaded 34GiB to GPU in 1.98323s (17.1438GiB/s) | 20260506 15:17:36.405 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:17:36.405 | INFO | Total mem usage: 17.4 GiB | 20260506 15:17:36.405 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:17:36.405 | INFO | Recent High Water Marks Sum: 16.3 GiB (Savings: -6.99%) | 20260506 15:17:36.405 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:17:36.405 | INFO | High Water Breakdown: | 20260506 15:17:36.405 | INFO | 61.9% : 10.1 GiB ( 92.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:17:36.405 | INFO | 38% : 6.19 GiB ( 92.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:17:36.405 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:17:36.405 | INFO | ------------------------------------------ | 20260506 15:17:36.405 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:17:36.405 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:17:36.405 | INFO | Total mem usage: 26.5 GiB | 20260506 15:17:36.405 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:17:36.405 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:17:36.405 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:17:36.405 | INFO | High Water Breakdown: | 20260506 15:17:36.405 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:17:36.405 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:17:36.405 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:17:36.405 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:17:36.405 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:17:36.405 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:17:36.405 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:17:36.405 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:17:36.405 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:17:36.405 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:17:36.405 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:17:36.405 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:17:36.405 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:17:36.405 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:17:36.405 | INFO | ------------------------------------------ | 20260506 15:17:36.405 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:17:36.405 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:17:36.405 | INFO | Total mem usage: 25.6 GiB | 20260506 15:17:36.405 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:17:36.405 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:17:36.405 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:17:36.405 | INFO | High Water Breakdown: | 20260506 15:17:36.405 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:17:36.405 | INFO | ------------------------------------------ | 20260506 15:17:36.405 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:17:36.405 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:17:36.405 | INFO | Total mem usage: 15.7 GiB | 20260506 15:17:36.405 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:17:36.405 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:17:36.405 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 15:17:36.405 | INFO | High Water Breakdown: | 20260506 15:17:36.405 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:17:36.405 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:17:36.405 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:17:36.405 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:17:36.405 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:17:36.405 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:17:36.405 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:17:36.405 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:17:36.405 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:17:36.405 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:17:36.405 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:17:36.405 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:17:36.405 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:17:36.405 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:17:36.405 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:17:36.405 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:17:36.405 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:17:36.405 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:17:36.405 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:17:36.406 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:17:36.406 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:17:36.406 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:17:36.406 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:17:36.406 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:17:36.406 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:17:36.406 | INFO | ------------------------------------------ | 20260506 15:17:36.406 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:17:36.406 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:17:36.406 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:17:36.406 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:17:36.406 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.59e+01, 1.36e+01 / 2.54e+00 | 20260506 15:17:36.406 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 9.87e-02, 1.04e-02 / 6.81e-03 | 20260506 15:17:36.406 | INFO | PerfReport: 11.57%, Baseline -- 3.64e+00 / 4.37e+00, 5.61e-01 / 3.98e-01 | 20260506 15:17:36.406 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.45e+00, 1.86e-01 / 1.44e-01 | 20260506 15:17:36.406 | INFO | PerfReport: 16.00%, DME -- 4.03e+01 / 4.48e+01, 4.27e+00 / 2.83e+00 | 20260506 15:17:36.406 | INFO | PerfReport: 56.66%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.34e+00 / 1.53e+00 | 20260506 15:17:36.406 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.33e+00 / 2.96e+00, 4.07e-01 / 2.23e-01 | 20260506 15:17:36.406 | INFO | PerfReport: 3.57%, Metrics -- 1.12e+00 / 1.47e+00, 2.93e-01 / 1.91e-01 | 20260506 15:17:36.406 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.91e-01 / 4.39e-01, 6.18e-02 / 2.78e-02 | 20260506 15:17:36.406 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:17:36.406 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:17:36.406 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:17:36.406 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 15:17:36.406 | INFO | PerfReport: 21.57%, Upload -- 2.11e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 15:17:36.406 | INFO | PerfReport: 21.25%, Download -- 2.08e+01 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260506 15:17:36.406 | INFO | PerfReport: 32.21%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 15:17:36.406 | INFO | PerfReport: 1.58%, LockUpload -- 1.54e+00 / 0.00e+00, 5.94e+00 / 0.00e+00 | 20260506 15:17:36.406 | INFO | PerfReport: 1.16%, LockDownload -- 1.14e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260506 15:17:36.406 | INFO | PerfReport: 22.22%, LockCompute -- 2.17e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 15:17:36.406 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260506 15:17:36.525 | INFO | Starting a round of Baz IO | 20260506 15:17:39.661 | INFO | Populating RTMetrics report | 20260506 15:17:39.661 | INFO | RTMetrics report populated | 20260506 15:17:39.661 | INFO | Opening tmp RTMetrics json file | 20260506 15:17:39.666 | INFO | Serializing RTMetrics to json | 20260506 15:17:39.668 | INFO | Serialization done, swapping tmp file now... | 20260506 15:17:39.669 | INFO | RTMetrics done | 20260506 15:17:39.805 | INFO | Finished a round of Baz IO: Wrote 2.106 GiB in 3.28 seconds | 20260506 15:17:39.805 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.36% / 0.01% / 62.63% | 20260506 15:17:39.805 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.13753% / 1.98682e-05% / 4.76837e-05% | 20260506 15:17:39.805 | INFO | Baz IO overall: | 20260506 15:17:39.806 | INFO | BytesWritten : 1232637882966 | 20260506 15:17:39.806 | INFO | NumEvents : 596529666101 | 20260506 15:17:41.543 | INFO | Analyzing chunk frames = [2367488,2368000) | 20260506 15:17:41.543 | INFO | MemInfo: { | 20260506 15:17:41.543 | INFO | "gpu" : " 17 GB", | 20260506 15:17:41.543 | INFO | "huge" : "", | 20260506 15:17:41.543 | INFO | "huge_shared" : " 27 GB", | 20260506 15:17:41.543 | INFO | "malloc" : " 14 GB", | 20260506 15:17:41.543 | INFO | "pinned" : " 29 GB" | 20260506 15:17:41.543 | INFO | } | 20260506 15:17:41.543 | INFO | Perf report for the last 5.13878 seconds: | 20260506 15:17:41.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:17:41.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:41.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:41.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:41.543 | INFO | ANALYSIS : 62.8%, 2.9, 95.1ms, 0 | 20260506 15:17:41.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:41.543 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 15:17:41.543 | INFO | PRE_HQ : 60.6%, 1.9, 59.2ms, 0 | 20260506 15:17:41.543 | INFO | BAZWRITER : 63.4%, 4.8, 155.1ms, 0 | 20260506 15:17:41.543 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 15:17:41.544 | INFO | Uploaded 36GiB to GPU in 2.05727s (17.499GiB/s) | 20260506 15:17:41.544 | INFO | Downloaded 38GiB to GPU in 2.23449s (17.0061GiB/s) | 20260506 15:17:46.682 | INFO | Analyzing chunk frames = [2368000,2368512) | 20260506 15:17:46.682 | INFO | MemInfo: { | 20260506 15:17:46.682 | INFO | "gpu" : " 17 GB", | 20260506 15:17:46.682 | INFO | "huge" : "", | 20260506 15:17:46.682 | INFO | "huge_shared" : " 27 GB", | 20260506 15:17:46.682 | INFO | "malloc" : " 14 GB", | 20260506 15:17:46.682 | INFO | "pinned" : " 29 GB" | 20260506 15:17:46.682 | INFO | } | 20260506 15:17:46.682 | INFO | Perf report for the last 5.13891 seconds: | 20260506 15:17:46.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:17:46.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:46.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:46.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:46.682 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 15:17:46.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:46.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:46.682 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:17:46.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:46.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:46.682 | INFO | Uploaded 36GiB to GPU in 1.9812s (18.1708GiB/s) | 20260506 15:17:46.682 | INFO | Downloaded 34GiB to GPU in 1.93511s (17.5701GiB/s) | 20260506 15:17:51.820 | INFO | Analyzing chunk frames = [2368512,2369024) | 20260506 15:17:51.820 | INFO | MemInfo: { | 20260506 15:17:51.820 | INFO | "gpu" : " 17 GB", | 20260506 15:17:51.820 | INFO | "huge" : "", | 20260506 15:17:51.820 | INFO | "huge_shared" : " 27 GB", | 20260506 15:17:51.820 | INFO | "malloc" : " 14 GB", | 20260506 15:17:51.820 | INFO | "pinned" : " 29 GB" | 20260506 15:17:51.820 | INFO | } | 20260506 15:17:51.820 | INFO | Perf report for the last 5.13849 seconds: | 20260506 15:17:51.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:17:51.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:51.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:51.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:51.821 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 15:17:51.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:51.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:51.821 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:17:51.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:51.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:51.821 | INFO | Uploaded 36GiB to GPU in 1.96853s (18.2878GiB/s) | 20260506 15:17:51.821 | INFO | Downloaded 34GiB to GPU in 1.94572s (17.4743GiB/s) | 20260506 15:17:56.960 | INFO | Analyzing chunk frames = [2369024,2369536) | 20260506 15:17:56.960 | INFO | MemInfo: { | 20260506 15:17:56.960 | INFO | "gpu" : " 17 GB", | 20260506 15:17:56.960 | INFO | "huge" : "", | 20260506 15:17:56.960 | INFO | "huge_shared" : " 27 GB", | 20260506 15:17:56.960 | INFO | "malloc" : " 14 GB", | 20260506 15:17:56.960 | INFO | "pinned" : " 29 GB" | 20260506 15:17:56.960 | INFO | } | 20260506 15:17:56.960 | INFO | Perf report for the last 5.1399 seconds: | 20260506 15:17:56.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:17:56.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:56.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:56.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:56.961 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 15:17:56.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:56.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:56.961 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:17:56.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:56.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:17:56.961 | INFO | Uploaded 36GiB to GPU in 1.96642s (18.3074GiB/s) | 20260506 15:17:56.961 | INFO | Downloaded 34GiB to GPU in 1.88208s (18.0651GiB/s) | 20260506 15:18:02.099 | INFO | Analyzing chunk frames = [2369536,2370048) | 20260506 15:18:02.099 | INFO | MemInfo: { | 20260506 15:18:02.099 | INFO | "gpu" : " 17 GB", | 20260506 15:18:02.099 | INFO | "huge" : "", | 20260506 15:18:02.099 | INFO | "huge_shared" : " 27 GB", | 20260506 15:18:02.099 | INFO | "malloc" : " 14 GB", | 20260506 15:18:02.099 | INFO | "pinned" : " 29 GB" | 20260506 15:18:02.099 | INFO | } | 20260506 15:18:02.099 | INFO | Perf report for the last 5.13897 seconds: | 20260506 15:18:02.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:18:02.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:02.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:02.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:02.099 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 15:18:02.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:02.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:02.100 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:18:02.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:02.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:02.100 | INFO | Uploaded 36GiB to GPU in 1.99575s (18.0383GiB/s) | 20260506 15:18:02.100 | INFO | Downloaded 34GiB to GPU in 1.92296s (17.6811GiB/s) | 20260506 15:18:07.240 | INFO | Analyzing chunk frames = [2370048,2370560) | 20260506 15:18:07.240 | INFO | MemInfo: { | 20260506 15:18:07.240 | INFO | "gpu" : " 17 GB", | 20260506 15:18:07.240 | INFO | "huge" : "", | 20260506 15:18:07.240 | INFO | "huge_shared" : " 27 GB", | 20260506 15:18:07.240 | INFO | "malloc" : " 14 GB", | 20260506 15:18:07.240 | INFO | "pinned" : " 29 GB" | 20260506 15:18:07.240 | INFO | } | 20260506 15:18:07.240 | INFO | Perf report for the last 5.14082 seconds: | 20260506 15:18:07.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:18:07.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:07.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:07.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:07.240 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 15:18:07.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:07.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:07.240 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 15:18:07.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:07.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:07.240 | INFO | Uploaded 36GiB to GPU in 2.0391s (17.6548GiB/s) | 20260506 15:18:07.240 | INFO | Downloaded 34GiB to GPU in 1.94597s (17.472GiB/s) | 20260506 15:18:12.380 | INFO | Analyzing chunk frames = [2370560,2371072) | 20260506 15:18:12.380 | INFO | MemInfo: { | 20260506 15:18:12.380 | INFO | "gpu" : " 17 GB", | 20260506 15:18:12.380 | INFO | "huge" : "", | 20260506 15:18:12.380 | INFO | "huge_shared" : " 27 GB", | 20260506 15:18:12.380 | INFO | "malloc" : " 14 GB", | 20260506 15:18:12.380 | INFO | "pinned" : " 29 GB" | 20260506 15:18:12.380 | INFO | } | 20260506 15:18:12.380 | INFO | Perf report for the last 5.13972 seconds: | 20260506 15:18:12.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:18:12.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:12.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:12.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:12.380 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:18:12.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:12.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:12.380 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:18:12.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:12.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:12.380 | INFO | Uploaded 36GiB to GPU in 2.01178s (17.8946GiB/s) | 20260506 15:18:12.380 | INFO | Downloaded 34GiB to GPU in 1.9963s (17.0315GiB/s) | 20260506 15:18:17.517 | INFO | Analyzing chunk frames = [2371072,2371584) | 20260506 15:18:17.517 | INFO | MemInfo: { | 20260506 15:18:17.517 | INFO | "gpu" : " 17 GB", | 20260506 15:18:17.517 | INFO | "huge" : "", | 20260506 15:18:17.517 | INFO | "huge_shared" : " 27 GB", | 20260506 15:18:17.517 | INFO | "malloc" : " 14 GB", | 20260506 15:18:17.517 | INFO | "pinned" : " 29 GB" | 20260506 15:18:17.517 | INFO | } | 20260506 15:18:17.517 | INFO | Perf report for the last 5.13731 seconds: | 20260506 15:18:17.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:18:17.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:17.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:17.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:17.517 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:18:17.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:17.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:17.517 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 15:18:17.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:17.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:17.518 | INFO | Uploaded 36GiB to GPU in 2.02523s (17.7758GiB/s) | 20260506 15:18:17.518 | INFO | Downloaded 34GiB to GPU in 1.9886s (17.0975GiB/s) | 20260506 15:18:17.631 | INFO | Starting a round of Baz IO | 20260506 15:18:20.778 | INFO | Populating RTMetrics report | 20260506 15:18:20.779 | INFO | RTMetrics report populated | 20260506 15:18:20.779 | INFO | Opening tmp RTMetrics json file | 20260506 15:18:20.785 | INFO | Serializing RTMetrics to json | 20260506 15:18:20.791 | INFO | Serialization done, swapping tmp file now... | 20260506 15:18:20.795 | INFO | RTMetrics done | 20260506 15:18:21.120 | INFO | Finished a round of Baz IO: Wrote 2.103 GiB in 3.489 seconds | 20260506 15:18:21.120 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.35% / 0.01% / 62.64% | 20260506 15:18:21.120 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.12603% / 1.58946e-05% / 5.16574e-05% | 20260506 15:18:21.120 | INFO | Baz IO overall: | 20260506 15:18:21.120 | INFO | BytesWritten : 1234896357896 | 20260506 15:18:21.120 | INFO | NumEvents : 597622171093 | 20260506 15:18:22.657 | INFO | Analyzing chunk frames = [2371584,2372096) | 20260506 15:18:22.657 | INFO | MemInfo: { | 20260506 15:18:22.657 | INFO | "gpu" : " 17 GB", | 20260506 15:18:22.657 | INFO | "huge" : "", | 20260506 15:18:22.657 | INFO | "huge_shared" : " 27 GB", | 20260506 15:18:22.657 | INFO | "malloc" : " 14 GB", | 20260506 15:18:22.657 | INFO | "pinned" : " 29 GB" | 20260506 15:18:22.657 | INFO | } | 20260506 15:18:22.657 | INFO | Perf report for the last 5.13977 seconds: | 20260506 15:18:22.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:18:22.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:22.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:22.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:22.657 | INFO | ANALYSIS : 62.9%, 2.9, 95.1ms, 0 | 20260506 15:18:22.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:22.657 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 15:18:22.657 | INFO | PRE_HQ : 60.8%, 1.9, 59.5ms, 0 | 20260506 15:18:22.657 | INFO | BAZWRITER : 67.3%, 4.6, 158.8ms, 0 | 20260506 15:18:22.657 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 15:18:22.657 | INFO | Uploaded 36GiB to GPU in 2.0838s (17.2761GiB/s) | 20260506 15:18:22.657 | INFO | Downloaded 38GiB to GPU in 2.23372s (17.012GiB/s) | 20260506 15:18:27.796 | INFO | Analyzing chunk frames = [2372096,2372608) | 20260506 15:18:27.797 | INFO | MemInfo: { | 20260506 15:18:27.797 | INFO | "gpu" : " 17 GB", | 20260506 15:18:27.797 | INFO | "huge" : "", | 20260506 15:18:27.797 | INFO | "huge_shared" : " 27 GB", | 20260506 15:18:27.797 | INFO | "malloc" : " 14 GB", | 20260506 15:18:27.797 | INFO | "pinned" : " 29 GB" | 20260506 15:18:27.797 | INFO | } | 20260506 15:18:27.797 | INFO | Perf report for the last 5.13962 seconds: | 20260506 15:18:27.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:18:27.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:27.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:27.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:27.797 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 15:18:27.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:27.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:27.797 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 15:18:27.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:27.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:27.797 | INFO | Uploaded 36GiB to GPU in 2.00242s (17.9782GiB/s) | 20260506 15:18:27.797 | INFO | Downloaded 34GiB to GPU in 1.9486s (17.4484GiB/s) | 20260506 15:18:32.935 | INFO | Analyzing chunk frames = [2372608,2373120) | 20260506 15:18:32.935 | INFO | MemInfo: { | 20260506 15:18:32.935 | INFO | "gpu" : " 17 GB", | 20260506 15:18:32.935 | INFO | "huge" : "", | 20260506 15:18:32.935 | INFO | "huge_shared" : " 27 GB", | 20260506 15:18:32.935 | INFO | "malloc" : " 14 GB", | 20260506 15:18:32.935 | INFO | "pinned" : " 29 GB" | 20260506 15:18:32.935 | INFO | } | 20260506 15:18:32.935 | INFO | Perf report for the last 5.13876 seconds: | 20260506 15:18:32.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:18:32.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:32.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:32.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:32.936 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:18:32.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:32.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:32.936 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:18:32.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:32.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:32.936 | INFO | Uploaded 36GiB to GPU in 1.95078s (18.4542GiB/s) | 20260506 15:18:32.936 | INFO | Downloaded 34GiB to GPU in 1.94331s (17.4959GiB/s) | 20260506 15:18:38.075 | INFO | Analyzing chunk frames = [2373120,2373632) | 20260506 15:18:38.075 | INFO | MemInfo: { | 20260506 15:18:38.075 | INFO | "gpu" : " 17 GB", | 20260506 15:18:38.075 | INFO | "huge" : "", | 20260506 15:18:38.075 | INFO | "huge_shared" : " 27 GB", | 20260506 15:18:38.076 | INFO | "malloc" : " 14 GB", | 20260506 15:18:38.076 | INFO | "pinned" : " 29 GB" | 20260506 15:18:38.076 | INFO | } | 20260506 15:18:38.076 | INFO | Perf report for the last 5.14007 seconds: | 20260506 15:18:38.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:18:38.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:38.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:38.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:38.076 | INFO | ANALYSIS : 60.6%, 3.0, 94.0ms, 0 | 20260506 15:18:38.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:38.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:38.076 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:18:38.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:38.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:38.076 | INFO | Uploaded 36GiB to GPU in 1.99045s (18.0864GiB/s) | 20260506 15:18:38.076 | INFO | Downloaded 34GiB to GPU in 1.91439s (17.7602GiB/s) | 20260506 15:18:43.214 | INFO | Analyzing chunk frames = [2373632,2374144) | 20260506 15:18:43.214 | INFO | MemInfo: { | 20260506 15:18:43.214 | INFO | "gpu" : " 17 GB", | 20260506 15:18:43.214 | INFO | "huge" : "", | 20260506 15:18:43.214 | INFO | "huge_shared" : " 27 GB", | 20260506 15:18:43.214 | INFO | "malloc" : " 14 GB", | 20260506 15:18:43.214 | INFO | "pinned" : " 29 GB" | 20260506 15:18:43.214 | INFO | } | 20260506 15:18:43.214 | INFO | Perf report for the last 5.13891 seconds: | 20260506 15:18:43.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:18:43.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:43.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:43.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:43.215 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 15:18:43.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:43.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:43.215 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 15:18:43.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:43.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:43.215 | INFO | Uploaded 36GiB to GPU in 1.98582s (18.1285GiB/s) | 20260506 15:18:43.215 | INFO | Downloaded 34GiB to GPU in 1.94807s (17.4532GiB/s) | 20260506 15:18:48.354 | INFO | Analyzing chunk frames = [2374144,2374656) | 20260506 15:18:48.354 | INFO | MemInfo: { | 20260506 15:18:48.354 | INFO | "gpu" : " 17 GB", | 20260506 15:18:48.354 | INFO | "huge" : "", | 20260506 15:18:48.354 | INFO | "huge_shared" : " 27 GB", | 20260506 15:18:48.354 | INFO | "malloc" : " 14 GB", | 20260506 15:18:48.354 | INFO | "pinned" : " 29 GB" | 20260506 15:18:48.354 | INFO | } | 20260506 15:18:48.354 | INFO | Perf report for the last 5.1396 seconds: | 20260506 15:18:48.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:18:48.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:48.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:48.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:48.354 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 15:18:48.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:48.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:48.354 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:18:48.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:48.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:48.354 | INFO | Uploaded 36GiB to GPU in 2.07093s (17.3835GiB/s) | 20260506 15:18:48.354 | INFO | Downloaded 34GiB to GPU in 1.97563s (17.2097GiB/s) | 20260506 15:18:53.494 | INFO | Analyzing chunk frames = [2374656,2375168) | 20260506 15:18:53.494 | INFO | MemInfo: { | 20260506 15:18:53.494 | INFO | "gpu" : " 17 GB", | 20260506 15:18:53.494 | INFO | "huge" : "", | 20260506 15:18:53.494 | INFO | "huge_shared" : " 27 GB", | 20260506 15:18:53.494 | INFO | "malloc" : " 14 GB", | 20260506 15:18:53.494 | INFO | "pinned" : " 29 GB" | 20260506 15:18:53.494 | INFO | } | 20260506 15:18:53.494 | INFO | Perf report for the last 5.13996 seconds: | 20260506 15:18:53.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:18:53.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:53.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:53.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:53.494 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:18:53.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:53.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:53.494 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 15:18:53.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:53.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:53.494 | INFO | Uploaded 36GiB to GPU in 2.021s (17.8129GiB/s) | 20260506 15:18:53.494 | INFO | Downloaded 34GiB to GPU in 1.99972s (17.0024GiB/s) | 20260506 15:18:58.633 | INFO | Analyzing chunk frames = [2375168,2375680) | 20260506 15:18:58.633 | INFO | MemInfo: { | 20260506 15:18:58.633 | INFO | "gpu" : " 17 GB", | 20260506 15:18:58.633 | INFO | "huge" : "", | 20260506 15:18:58.633 | INFO | "huge_shared" : " 27 GB", | 20260506 15:18:58.633 | INFO | "malloc" : " 14 GB", | 20260506 15:18:58.633 | INFO | "pinned" : " 29 GB" | 20260506 15:18:58.633 | INFO | } | 20260506 15:18:58.633 | INFO | Perf report for the last 5.13911 seconds: | 20260506 15:18:58.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:18:58.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:58.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:58.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:58.633 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 15:18:58.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:58.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:58.633 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 15:18:58.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:58.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:18:58.633 | INFO | Uploaded 36GiB to GPU in 1.99053s (18.0857GiB/s) | 20260506 15:18:58.633 | INFO | Downloaded 34GiB to GPU in 1.98521s (17.1267GiB/s) | 20260506 15:18:58.633 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:18:58.633 | INFO | Total mem usage: 17.4 GiB | 20260506 15:18:58.633 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:18:58.634 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -14.7%) | 20260506 15:18:58.634 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:18:58.634 | INFO | High Water Breakdown: | 20260506 15:18:58.634 | INFO | 60.8% : 9.23 GiB ( 84.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:18:58.634 | INFO | 39.1% : 5.93 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:18:58.634 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:18:58.634 | INFO | ------------------------------------------ | 20260506 15:18:58.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:18:58.634 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:18:58.634 | INFO | Total mem usage: 26.5 GiB | 20260506 15:18:58.634 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:18:58.634 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:18:58.634 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:18:58.634 | INFO | High Water Breakdown: | 20260506 15:18:58.634 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:18:58.634 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:18:58.634 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:18:58.634 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:18:58.634 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:18:58.634 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:18:58.634 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:18:58.634 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:18:58.634 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:18:58.634 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:18:58.634 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:18:58.634 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:18:58.634 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:18:58.634 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:18:58.634 | INFO | ------------------------------------------ | 20260506 15:18:58.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:18:58.634 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:18:58.634 | INFO | Total mem usage: 25.6 GiB | 20260506 15:18:58.634 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:18:58.634 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:18:58.634 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:18:58.634 | INFO | High Water Breakdown: | 20260506 15:18:58.634 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:18:58.634 | INFO | ------------------------------------------ | 20260506 15:18:58.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:18:58.634 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:18:58.634 | INFO | Total mem usage: 15.7 GiB | 20260506 15:18:58.634 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:18:58.634 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:18:58.634 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 15:18:58.634 | INFO | High Water Breakdown: | 20260506 15:18:58.634 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:18:58.634 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:18:58.634 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:18:58.634 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:18:58.634 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:18:58.634 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:18:58.634 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:18:58.634 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:18:58.634 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:18:58.634 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:18:58.634 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:18:58.634 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:18:58.634 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:18:58.634 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:18:58.634 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:18:58.634 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:18:58.634 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:18:58.634 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:18:58.634 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:18:58.634 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:18:58.634 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:18:58.634 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:18:58.634 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:18:58.634 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:18:58.634 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:18:58.634 | INFO | ------------------------------------------ | 20260506 15:18:58.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:18:58.634 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:18:58.634 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:18:58.634 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:18:58.634 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.50e+01, 1.34e+01 / 1.62e+00 | 20260506 15:18:58.634 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.02e-01, 1.06e-02 / 6.34e-03 | 20260506 15:18:58.634 | INFO | PerfReport: 11.58%, Baseline -- 3.64e+00 / 4.39e+00, 5.56e-01 / 2.78e-01 | 20260506 15:18:58.634 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.48e+00, 1.85e-01 / 1.29e-01 | 20260506 15:18:58.634 | INFO | PerfReport: 15.79%, DME -- 3.97e+01 / 4.34e+01, 3.55e+00 / 1.86e+00 | 20260506 15:18:58.634 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.45e+00 / 1.97e+00 | 20260506 15:18:58.634 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.33e+00 / 2.90e+00, 3.96e-01 / 2.51e-01 | 20260506 15:18:58.634 | INFO | PerfReport: 3.55%, Metrics -- 1.12e+00 / 1.47e+00, 2.85e-01 / 1.75e-01 | 20260506 15:18:58.634 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.89e-01 / 4.40e-01, 5.91e-02 / 2.51e-02 | 20260506 15:18:58.634 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:18:58.634 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:18:58.634 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:18:58.634 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 15:18:58.634 | INFO | PerfReport: 21.49%, Upload -- 2.09e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260506 15:18:58.634 | INFO | PerfReport: 21.26%, Download -- 2.07e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 15:18:58.634 | INFO | PerfReport: 32.26%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 15:18:58.634 | INFO | PerfReport: 1.37%, LockUpload -- 1.34e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260506 15:18:58.634 | INFO | PerfReport: 1.20%, LockDownload -- 1.17e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 15:18:58.634 | INFO | PerfReport: 22.41%, LockCompute -- 2.18e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 15:18:58.634 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260506 15:18:58.752 | INFO | Starting a round of Baz IO | 20260506 15:19:01.896 | INFO | Populating RTMetrics report | 20260506 15:19:01.896 | INFO | RTMetrics report populated | 20260506 15:19:01.896 | INFO | Opening tmp RTMetrics json file | 20260506 15:19:01.961 | INFO | Serializing RTMetrics to json | 20260506 15:19:02.000 | INFO | Serialization done, swapping tmp file now... | 20260506 15:19:02.013 | INFO | RTMetrics done | 20260506 15:19:02.126 | INFO | Finished a round of Baz IO: Wrote 2.102 GiB in 3.373 seconds | 20260506 15:19:02.126 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.34% / 0.01% / 62.65% | 20260506 15:19:02.126 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.11821% / 2.78155e-05% / 7.94729e-05% | 20260506 15:19:02.126 | INFO | Baz IO overall: | 20260506 15:19:02.126 | INFO | BytesWritten : 1237153723396 | 20260506 15:19:02.126 | INFO | NumEvents : 598714203784 | 20260506 15:19:03.773 | INFO | Analyzing chunk frames = [2375680,2376192) | 20260506 15:19:03.773 | INFO | MemInfo: { | 20260506 15:19:03.773 | INFO | "gpu" : " 17 GB", | 20260506 15:19:03.773 | INFO | "huge" : "", | 20260506 15:19:03.773 | INFO | "huge_shared" : " 27 GB", | 20260506 15:19:03.773 | INFO | "malloc" : " 14 GB", | 20260506 15:19:03.773 | INFO | "pinned" : " 29 GB" | 20260506 15:19:03.773 | INFO | } | 20260506 15:19:03.773 | INFO | Perf report for the last 5.13973 seconds: | 20260506 15:19:03.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:19:03.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:03.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:03.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:03.773 | INFO | ANALYSIS : 62.9%, 2.9, 95.1ms, 0 | 20260506 15:19:03.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:03.773 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 15:19:03.773 | INFO | PRE_HQ : 60.9%, 1.9, 59.7ms, 0 | 20260506 15:19:03.773 | INFO | BAZWRITER : 65.3%, 4.8, 162.0ms, 0 | 20260506 15:19:03.773 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 15:19:03.773 | INFO | Uploaded 36GiB to GPU in 2.07028s (17.3889GiB/s) | 20260506 15:19:03.773 | INFO | Downloaded 38GiB to GPU in 2.21695s (17.1407GiB/s) | 20260506 15:19:08.911 | INFO | Analyzing chunk frames = [2376192,2376704) | 20260506 15:19:08.911 | INFO | MemInfo: { | 20260506 15:19:08.911 | INFO | "gpu" : " 17 GB", | 20260506 15:19:08.911 | INFO | "huge" : "", | 20260506 15:19:08.911 | INFO | "huge_shared" : " 27 GB", | 20260506 15:19:08.911 | INFO | "malloc" : " 14 GB", | 20260506 15:19:08.911 | INFO | "pinned" : " 29 GB" | 20260506 15:19:08.911 | INFO | } | 20260506 15:19:08.911 | INFO | Perf report for the last 5.13852 seconds: | 20260506 15:19:08.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:19:08.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:08.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:08.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:08.912 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:19:08.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:08.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:08.912 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:19:08.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:08.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:08.912 | INFO | Uploaded 36GiB to GPU in 1.9869s (18.1187GiB/s) | 20260506 15:19:08.912 | INFO | Downloaded 34GiB to GPU in 1.94231s (17.5049GiB/s) | 20260506 15:19:14.051 | INFO | Analyzing chunk frames = [2376704,2377216) | 20260506 15:19:14.051 | INFO | MemInfo: { | 20260506 15:19:14.051 | INFO | "gpu" : " 17 GB", | 20260506 15:19:14.051 | INFO | "huge" : "", | 20260506 15:19:14.051 | INFO | "huge_shared" : " 27 GB", | 20260506 15:19:14.051 | INFO | "malloc" : " 14 GB", | 20260506 15:19:14.051 | INFO | "pinned" : " 29 GB" | 20260506 15:19:14.051 | INFO | } | 20260506 15:19:14.051 | INFO | Perf report for the last 5.13991 seconds: | 20260506 15:19:14.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:19:14.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:14.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:14.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:14.051 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:19:14.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:14.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:14.051 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:19:14.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:14.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:14.052 | INFO | Uploaded 36GiB to GPU in 1.96683s (18.3036GiB/s) | 20260506 15:19:14.052 | INFO | Downloaded 34GiB to GPU in 1.93954s (17.53GiB/s) | 20260506 15:19:19.190 | INFO | Analyzing chunk frames = [2377216,2377728) | 20260506 15:19:19.190 | INFO | MemInfo: { | 20260506 15:19:19.190 | INFO | "gpu" : " 17 GB", | 20260506 15:19:19.190 | INFO | "huge" : "", | 20260506 15:19:19.190 | INFO | "huge_shared" : " 27 GB", | 20260506 15:19:19.190 | INFO | "malloc" : " 14 GB", | 20260506 15:19:19.190 | INFO | "pinned" : " 29 GB" | 20260506 15:19:19.190 | INFO | } | 20260506 15:19:19.190 | INFO | Perf report for the last 5.1388 seconds: | 20260506 15:19:19.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:19:19.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:19.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:19.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:19.190 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 15:19:19.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:19.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:19.190 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:19:19.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:19.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:19.190 | INFO | Uploaded 36GiB to GPU in 2.00952s (17.9147GiB/s) | 20260506 15:19:19.190 | INFO | Downloaded 34GiB to GPU in 1.91854s (17.7218GiB/s) | 20260506 15:19:24.330 | INFO | Analyzing chunk frames = [2377728,2378240) | 20260506 15:19:24.330 | INFO | MemInfo: { | 20260506 15:19:24.330 | INFO | "gpu" : " 17 GB", | 20260506 15:19:24.330 | INFO | "huge" : "", | 20260506 15:19:24.330 | INFO | "huge_shared" : " 27 GB", | 20260506 15:19:24.330 | INFO | "malloc" : " 14 GB", | 20260506 15:19:24.330 | INFO | "pinned" : " 29 GB" | 20260506 15:19:24.330 | INFO | } | 20260506 15:19:24.330 | INFO | Perf report for the last 5.14017 seconds: | 20260506 15:19:24.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:19:24.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:24.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:24.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:24.330 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 15:19:24.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:24.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:24.330 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:19:24.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:24.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:24.331 | INFO | Uploaded 36GiB to GPU in 2.01336s (17.8806GiB/s) | 20260506 15:19:24.331 | INFO | Downloaded 34GiB to GPU in 1.94565s (17.4749GiB/s) | 20260506 15:19:29.469 | INFO | Analyzing chunk frames = [2378240,2378752) | 20260506 15:19:29.469 | INFO | MemInfo: { | 20260506 15:19:29.469 | INFO | "gpu" : " 17 GB", | 20260506 15:19:29.469 | INFO | "huge" : "", | 20260506 15:19:29.469 | INFO | "huge_shared" : " 27 GB", | 20260506 15:19:29.469 | INFO | "malloc" : " 14 GB", | 20260506 15:19:29.469 | INFO | "pinned" : " 29 GB" | 20260506 15:19:29.469 | INFO | } | 20260506 15:19:29.469 | INFO | Perf report for the last 5.13862 seconds: | 20260506 15:19:29.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:19:29.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:29.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:29.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:29.469 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 15:19:29.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:29.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:29.469 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 15:19:29.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:29.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:29.469 | INFO | Uploaded 36GiB to GPU in 2.04266s (17.6241GiB/s) | 20260506 15:19:29.469 | INFO | Downloaded 34GiB to GPU in 1.92448s (17.6671GiB/s) | 20260506 15:19:34.608 | INFO | Analyzing chunk frames = [2378752,2379264) | 20260506 15:19:34.608 | INFO | MemInfo: { | 20260506 15:19:34.608 | INFO | "gpu" : " 17 GB", | 20260506 15:19:34.608 | INFO | "huge" : "", | 20260506 15:19:34.608 | INFO | "huge_shared" : " 27 GB", | 20260506 15:19:34.608 | INFO | "malloc" : " 14 GB", | 20260506 15:19:34.608 | INFO | "pinned" : " 29 GB" | 20260506 15:19:34.608 | INFO | } | 20260506 15:19:34.609 | INFO | Perf report for the last 5.1395 seconds: | 20260506 15:19:34.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:19:34.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:34.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:34.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:34.609 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:19:34.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:34.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:34.609 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 15:19:34.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:34.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:34.609 | INFO | Uploaded 36GiB to GPU in 2.01042s (17.9067GiB/s) | 20260506 15:19:34.609 | INFO | Downloaded 34GiB to GPU in 1.99157s (17.072GiB/s) | 20260506 15:19:39.748 | INFO | Analyzing chunk frames = [2379264,2379776) | 20260506 15:19:39.748 | INFO | MemInfo: { | 20260506 15:19:39.748 | INFO | "gpu" : " 17 GB", | 20260506 15:19:39.748 | INFO | "huge" : "", | 20260506 15:19:39.748 | INFO | "huge_shared" : " 27 GB", | 20260506 15:19:39.748 | INFO | "malloc" : " 14 GB", | 20260506 15:19:39.748 | INFO | "pinned" : " 29 GB" | 20260506 15:19:39.748 | INFO | } | 20260506 15:19:39.748 | INFO | Perf report for the last 5.13924 seconds: | 20260506 15:19:39.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:19:39.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:39.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:39.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:39.748 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 15:19:39.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:39.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:39.748 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260506 15:19:39.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:39.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:39.748 | INFO | Uploaded 36GiB to GPU in 2.06645s (17.4212GiB/s) | 20260506 15:19:39.748 | INFO | Downloaded 34GiB to GPU in 1.98793s (17.1032GiB/s) | 20260506 15:19:39.876 | INFO | Starting a round of Baz IO | 20260506 15:19:43.021 | INFO | Populating RTMetrics report | 20260506 15:19:43.021 | INFO | RTMetrics report populated | 20260506 15:19:43.021 | INFO | Opening tmp RTMetrics json file | 20260506 15:19:43.039 | INFO | Serializing RTMetrics to json | 20260506 15:19:43.044 | INFO | Serialization done, swapping tmp file now... | 20260506 15:19:43.048 | INFO | RTMetrics done | 20260506 15:19:43.369 | INFO | Finished a round of Baz IO: Wrote 2.105 GiB in 3.493 seconds | 20260506 15:19:43.369 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.32% / 0.01% / 62.66% | 20260506 15:19:43.369 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.12394% / 7.94729e-06% / 6.35783e-05% | 20260506 15:19:43.370 | INFO | Baz IO overall: | 20260506 15:19:43.370 | INFO | BytesWritten : 1239414068606 | 20260506 15:19:43.370 | INFO | NumEvents : 599808855256 | 20260506 15:19:44.887 | INFO | Analyzing chunk frames = [2379776,2380288) | 20260506 15:19:44.887 | INFO | MemInfo: { | 20260506 15:19:44.887 | INFO | "gpu" : " 17 GB", | 20260506 15:19:44.887 | INFO | "huge" : "", | 20260506 15:19:44.887 | INFO | "huge_shared" : " 27 GB", | 20260506 15:19:44.887 | INFO | "malloc" : " 14 GB", | 20260506 15:19:44.887 | INFO | "pinned" : " 29 GB" | 20260506 15:19:44.887 | INFO | } | 20260506 15:19:44.887 | INFO | Perf report for the last 5.13892 seconds: | 20260506 15:19:44.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:19:44.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:44.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:44.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:44.887 | INFO | ANALYSIS : 63.1%, 2.9, 95.2ms, 0 | 20260506 15:19:44.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:44.887 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 15:19:44.887 | INFO | PRE_HQ : 61.2%, 1.9, 60.5ms, 0 | 20260506 15:19:44.887 | INFO | BAZWRITER : 67.4%, 4.7, 164.5ms, 0 | 20260506 15:19:44.887 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 15:19:44.887 | INFO | Uploaded 36GiB to GPU in 2.08689s (17.2506GiB/s) | 20260506 15:19:44.887 | INFO | Downloaded 38GiB to GPU in 2.23242s (17.0219GiB/s) | 20260506 15:19:50.026 | INFO | Analyzing chunk frames = [2380288,2380800) | 20260506 15:19:50.026 | INFO | MemInfo: { | 20260506 15:19:50.026 | INFO | "gpu" : " 17 GB", | 20260506 15:19:50.026 | INFO | "huge" : "", | 20260506 15:19:50.026 | INFO | "huge_shared" : " 27 GB", | 20260506 15:19:50.026 | INFO | "malloc" : " 14 GB", | 20260506 15:19:50.026 | INFO | "pinned" : " 29 GB" | 20260506 15:19:50.026 | INFO | } | 20260506 15:19:50.026 | INFO | Perf report for the last 5.13894 seconds: | 20260506 15:19:50.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:19:50.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:50.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:50.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:50.026 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:19:50.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:50.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:50.026 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 15:19:50.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:50.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:50.026 | INFO | Uploaded 36GiB to GPU in 2.0005s (17.9955GiB/s) | 20260506 15:19:50.026 | INFO | Downloaded 34GiB to GPU in 1.93426s (17.5778GiB/s) | 20260506 15:19:55.165 | INFO | Analyzing chunk frames = [2380800,2381312) | 20260506 15:19:55.165 | INFO | MemInfo: { | 20260506 15:19:55.165 | INFO | "gpu" : " 17 GB", | 20260506 15:19:55.165 | INFO | "huge" : "", | 20260506 15:19:55.165 | INFO | "huge_shared" : " 27 GB", | 20260506 15:19:55.165 | INFO | "malloc" : " 14 GB", | 20260506 15:19:55.165 | INFO | "pinned" : " 29 GB" | 20260506 15:19:55.165 | INFO | } | 20260506 15:19:55.165 | INFO | Perf report for the last 5.13975 seconds: | 20260506 15:19:55.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:19:55.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:55.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:55.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:55.165 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:19:55.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:55.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:55.165 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:19:55.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:55.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:19:55.166 | INFO | Uploaded 36GiB to GPU in 1.98333s (18.1513GiB/s) | 20260506 15:19:55.166 | INFO | Downloaded 34GiB to GPU in 1.94293s (17.4994GiB/s) | 20260506 15:20:00.305 | INFO | Analyzing chunk frames = [2381312,2381824) | 20260506 15:20:00.305 | INFO | MemInfo: { | 20260506 15:20:00.305 | INFO | "gpu" : " 17 GB", | 20260506 15:20:00.305 | INFO | "huge" : "", | 20260506 15:20:00.305 | INFO | "huge_shared" : " 27 GB", | 20260506 15:20:00.305 | INFO | "malloc" : " 14 GB", | 20260506 15:20:00.305 | INFO | "pinned" : " 29 GB" | 20260506 15:20:00.305 | INFO | } | 20260506 15:20:00.305 | INFO | Perf report for the last 5.13926 seconds: | 20260506 15:20:00.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:20:00.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:00.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:00.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:00.305 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 15:20:00.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:00.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:00.305 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:20:00.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:00.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:00.305 | INFO | Uploaded 36GiB to GPU in 2.02365s (17.7897GiB/s) | 20260506 15:20:00.305 | INFO | Downloaded 34GiB to GPU in 1.91408s (17.7631GiB/s) | 20260506 15:20:05.445 | INFO | Analyzing chunk frames = [2381824,2382336) | 20260506 15:20:05.445 | INFO | MemInfo: { | 20260506 15:20:05.445 | INFO | "gpu" : " 17 GB", | 20260506 15:20:05.445 | INFO | "huge" : "", | 20260506 15:20:05.445 | INFO | "huge_shared" : " 27 GB", | 20260506 15:20:05.445 | INFO | "malloc" : " 14 GB", | 20260506 15:20:05.445 | INFO | "pinned" : " 29 GB" | 20260506 15:20:05.445 | INFO | } | 20260506 15:20:05.445 | INFO | Perf report for the last 5.1401 seconds: | 20260506 15:20:05.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:20:05.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:05.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:05.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:05.445 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 15:20:05.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:05.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:05.445 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:20:05.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:05.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:05.445 | INFO | Uploaded 36GiB to GPU in 2.03685s (17.6744GiB/s) | 20260506 15:20:05.445 | INFO | Downloaded 34GiB to GPU in 1.92703s (17.6437GiB/s) | 20260506 15:20:10.584 | INFO | Analyzing chunk frames = [2382336,2382848) | 20260506 15:20:10.584 | INFO | MemInfo: { | 20260506 15:20:10.584 | INFO | "gpu" : " 17 GB", | 20260506 15:20:10.584 | INFO | "huge" : "", | 20260506 15:20:10.584 | INFO | "huge_shared" : " 27 GB", | 20260506 15:20:10.584 | INFO | "malloc" : " 14 GB", | 20260506 15:20:10.584 | INFO | "pinned" : " 29 GB" | 20260506 15:20:10.584 | INFO | } | 20260506 15:20:10.585 | INFO | Perf report for the last 5.13972 seconds: | 20260506 15:20:10.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:20:10.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:10.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:10.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:10.585 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 15:20:10.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:10.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:10.585 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:20:10.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:10.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:10.585 | INFO | Uploaded 36GiB to GPU in 2.07528s (17.3471GiB/s) | 20260506 15:20:10.585 | INFO | Downloaded 34GiB to GPU in 1.96455s (17.3068GiB/s) | 20260506 15:20:15.724 | INFO | Analyzing chunk frames = [2382848,2383360) | 20260506 15:20:15.724 | INFO | MemInfo: { | 20260506 15:20:15.724 | INFO | "gpu" : " 17 GB", | 20260506 15:20:15.724 | INFO | "huge" : "", | 20260506 15:20:15.724 | INFO | "huge_shared" : " 27 GB", | 20260506 15:20:15.724 | INFO | "malloc" : " 14 GB", | 20260506 15:20:15.724 | INFO | "pinned" : " 29 GB" | 20260506 15:20:15.724 | INFO | } | 20260506 15:20:15.724 | INFO | Perf report for the last 5.13942 seconds: | 20260506 15:20:15.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:20:15.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:15.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:15.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:15.724 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:20:15.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:15.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:15.724 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:20:15.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:15.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:15.724 | INFO | Uploaded 36GiB to GPU in 2.04325s (17.619GiB/s) | 20260506 15:20:15.724 | INFO | Downloaded 34GiB to GPU in 2.00224s (16.981GiB/s) | 20260506 15:20:20.862 | INFO | Analyzing chunk frames = [2383360,2383872) | 20260506 15:20:20.862 | INFO | MemInfo: { | 20260506 15:20:20.862 | INFO | "gpu" : " 17 GB", | 20260506 15:20:20.862 | INFO | "huge" : "", | 20260506 15:20:20.862 | INFO | "huge_shared" : " 27 GB", | 20260506 15:20:20.862 | INFO | "malloc" : " 14 GB", | 20260506 15:20:20.862 | INFO | "pinned" : " 29 GB" | 20260506 15:20:20.862 | INFO | } | 20260506 15:20:20.862 | INFO | Perf report for the last 5.13807 seconds: | 20260506 15:20:20.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:20:20.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:20.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:20.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:20.862 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:20:20.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:20.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:20.862 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:20:20.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:20.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:20.862 | INFO | Uploaded 36GiB to GPU in 1.99789s (18.019GiB/s) | 20260506 15:20:20.862 | INFO | Downloaded 34GiB to GPU in 1.98315s (17.1444GiB/s) | 20260506 15:20:20.863 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:20:20.863 | INFO | Total mem usage: 17.4 GiB | 20260506 15:20:20.863 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:20:20.863 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -9.13%) | 20260506 15:20:20.863 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:20:20.863 | INFO | High Water Breakdown: | 20260506 15:20:20.863 | INFO | 61.6% : 9.82 GiB ( 90% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:20:20.863 | INFO | 38.3% : 6.11 GiB ( 91.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:20:20.863 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:20:20.863 | INFO | ------------------------------------------ | 20260506 15:20:20.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:20:20.863 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:20:20.863 | INFO | Total mem usage: 26.5 GiB | 20260506 15:20:20.863 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:20:20.863 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:20:20.863 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:20:20.863 | INFO | High Water Breakdown: | 20260506 15:20:20.863 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:20:20.863 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:20:20.863 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:20:20.863 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:20:20.863 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:20:20.863 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:20:20.863 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:20:20.863 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:20:20.863 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:20:20.863 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:20:20.863 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:20:20.863 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:20:20.863 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:20:20.863 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:20:20.863 | INFO | ------------------------------------------ | 20260506 15:20:20.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:20:20.863 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:20:20.863 | INFO | Total mem usage: 25.6 GiB | 20260506 15:20:20.863 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:20:20.863 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:20:20.863 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:20:20.863 | INFO | High Water Breakdown: | 20260506 15:20:20.863 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:20:20.863 | INFO | ------------------------------------------ | 20260506 15:20:20.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:20:20.863 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:20:20.863 | INFO | Total mem usage: 15.7 GiB | 20260506 15:20:20.863 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:20:20.863 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:20:20.863 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 15:20:20.863 | INFO | High Water Breakdown: | 20260506 15:20:20.863 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:20:20.863 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:20:20.863 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:20:20.863 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:20:20.863 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:20:20.863 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:20:20.863 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:20:20.863 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:20:20.863 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:20:20.863 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:20:20.863 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:20:20.863 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:20:20.863 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:20:20.863 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:20:20.863 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:20:20.863 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:20:20.863 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:20:20.863 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:20:20.863 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:20:20.863 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:20:20.863 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:20:20.863 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:20:20.863 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:20:20.863 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:20:20.863 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:20:20.863 | INFO | ------------------------------------------ | 20260506 15:20:20.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:20:20.864 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:20:20.864 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:20:20.864 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:20:20.864 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.53e+01, 1.35e+01 / 1.77e+00 | 20260506 15:20:20.864 | INFO | PerfReport: 0.29%, AggregateStats -- 9.26e-02 / 1.00e-01, 3.21e-02 / 6.93e-03 | 20260506 15:20:20.864 | INFO | PerfReport: 11.55%, Baseline -- 3.63e+00 / 4.37e+00, 5.54e-01 / 4.22e-01 | 20260506 15:20:20.864 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.46e+00, 1.86e-01 / 1.56e-01 | 20260506 15:20:20.864 | INFO | PerfReport: 15.82%, DME -- 3.98e+01 / 4.38e+01, 3.80e+00 / 1.95e+00 | 20260506 15:20:20.864 | INFO | PerfReport: 56.91%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.51e+00 / 1.98e+00 | 20260506 15:20:20.864 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.32e+00 / 2.92e+00, 3.96e-01 / 2.29e-01 | 20260506 15:20:20.864 | INFO | PerfReport: 3.56%, Metrics -- 1.12e+00 / 1.47e+00, 2.90e-01 / 1.68e-01 | 20260506 15:20:20.864 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.90e-01 / 4.43e-01, 6.08e-02 / 2.26e-02 | 20260506 15:20:20.864 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:20:20.864 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:20:20.864 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:20:20.864 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 15:20:20.864 | INFO | PerfReport: 21.63%, Upload -- 2.11e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260506 15:20:20.864 | INFO | PerfReport: 21.20%, Download -- 2.07e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 15:20:20.864 | INFO | PerfReport: 32.24%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 15:20:20.864 | INFO | PerfReport: 1.56%, LockUpload -- 1.52e+00 / 0.00e+00, 6.01e+00 / 0.00e+00 | 20260506 15:20:20.864 | INFO | PerfReport: 1.06%, LockDownload -- 1.04e+00 / 0.00e+00, 2.60e+00 / 0.00e+00 | 20260506 15:20:20.864 | INFO | PerfReport: 22.30%, LockCompute -- 2.18e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 15:20:20.864 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260506 15:20:20.982 | INFO | Starting a round of Baz IO | 20260506 15:20:24.124 | INFO | Populating RTMetrics report | 20260506 15:20:24.124 | INFO | RTMetrics report populated | 20260506 15:20:24.124 | INFO | Opening tmp RTMetrics json file | 20260506 15:20:24.137 | INFO | Serializing RTMetrics to json | 20260506 15:20:24.213 | INFO | Serialization done, swapping tmp file now... | 20260506 15:20:24.254 | INFO | RTMetrics done | 20260506 15:20:24.579 | INFO | Finished a round of Baz IO: Wrote 2.102 GiB in 3.597 seconds | 20260506 15:20:24.579 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.31% / 0.01% / 62.68% | 20260506 15:20:24.579 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.10845% / 1.19209e-05% / 6.75519e-05% | 20260506 15:20:24.579 | INFO | Baz IO overall: | 20260506 15:20:24.579 | INFO | BytesWritten : 1241671114999 | 20260506 15:20:24.579 | INFO | NumEvents : 600900852496 | 20260506 15:20:26.001 | INFO | Analyzing chunk frames = [2383872,2384384) | 20260506 15:20:26.001 | INFO | MemInfo: { | 20260506 15:20:26.001 | INFO | "gpu" : " 17 GB", | 20260506 15:20:26.001 | INFO | "huge" : "", | 20260506 15:20:26.001 | INFO | "huge_shared" : " 27 GB", | 20260506 15:20:26.001 | INFO | "malloc" : " 14 GB", | 20260506 15:20:26.001 | INFO | "pinned" : " 29 GB" | 20260506 15:20:26.001 | INFO | } | 20260506 15:20:26.001 | INFO | Perf report for the last 5.13925 seconds: | 20260506 15:20:26.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:20:26.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:26.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:26.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:26.001 | INFO | ANALYSIS : 62.9%, 2.9, 95.2ms, 0 | 20260506 15:20:26.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:26.001 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260506 15:20:26.001 | INFO | PRE_HQ : 60.9%, 1.9, 59.5ms, 0 | 20260506 15:20:26.001 | INFO | BAZWRITER : 69.6%, 4.8, 171.6ms, 0 | 20260506 15:20:26.001 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 15:20:26.002 | INFO | Uploaded 36GiB to GPU in 2.08488s (17.2672GiB/s) | 20260506 15:20:26.002 | INFO | Downloaded 38GiB to GPU in 2.22335s (17.0913GiB/s) | 20260506 15:20:31.141 | INFO | Analyzing chunk frames = [2384384,2384896) | 20260506 15:20:31.141 | INFO | MemInfo: { | 20260506 15:20:31.141 | INFO | "gpu" : " 17 GB", | 20260506 15:20:31.141 | INFO | "huge" : "", | 20260506 15:20:31.141 | INFO | "huge_shared" : " 27 GB", | 20260506 15:20:31.141 | INFO | "malloc" : " 14 GB", | 20260506 15:20:31.141 | INFO | "pinned" : " 29 GB" | 20260506 15:20:31.141 | INFO | } | 20260506 15:20:31.141 | INFO | Perf report for the last 5.14014 seconds: | 20260506 15:20:31.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:20:31.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:31.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:31.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:31.141 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 15:20:31.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:31.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:31.142 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:20:31.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:31.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:31.142 | INFO | Uploaded 36GiB to GPU in 2.01119s (17.8998GiB/s) | 20260506 15:20:31.142 | INFO | Downloaded 34GiB to GPU in 1.94858s (17.4486GiB/s) | 20260506 15:20:36.281 | INFO | Analyzing chunk frames = [2384896,2385408) | 20260506 15:20:36.281 | INFO | MemInfo: { | 20260506 15:20:36.281 | INFO | "gpu" : " 17 GB", | 20260506 15:20:36.281 | INFO | "huge" : "", | 20260506 15:20:36.281 | INFO | "huge_shared" : " 27 GB", | 20260506 15:20:36.281 | INFO | "malloc" : " 14 GB", | 20260506 15:20:36.281 | INFO | "pinned" : " 29 GB" | 20260506 15:20:36.281 | INFO | } | 20260506 15:20:36.281 | INFO | Perf report for the last 5.13969 seconds: | 20260506 15:20:36.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:20:36.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:36.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:36.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:36.281 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:20:36.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:36.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:36.281 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:20:36.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:36.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:36.281 | INFO | Uploaded 36GiB to GPU in 2.00226s (17.9796GiB/s) | 20260506 15:20:36.282 | INFO | Downloaded 34GiB to GPU in 1.94153s (17.5119GiB/s) | 20260506 15:20:41.419 | INFO | Analyzing chunk frames = [2385408,2385920) | 20260506 15:20:41.419 | INFO | MemInfo: { | 20260506 15:20:41.419 | INFO | "gpu" : " 17 GB", | 20260506 15:20:41.419 | INFO | "huge" : "", | 20260506 15:20:41.419 | INFO | "huge_shared" : " 27 GB", | 20260506 15:20:41.419 | INFO | "malloc" : " 14 GB", | 20260506 15:20:41.419 | INFO | "pinned" : " 29 GB" | 20260506 15:20:41.419 | INFO | } | 20260506 15:20:41.419 | INFO | Perf report for the last 5.13819 seconds: | 20260506 15:20:41.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:20:41.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:41.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:41.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:41.419 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:20:41.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:41.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:41.419 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:20:41.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:41.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:41.420 | INFO | Uploaded 36GiB to GPU in 2.02316s (17.794GiB/s) | 20260506 15:20:41.420 | INFO | Downloaded 34GiB to GPU in 1.91657s (17.7401GiB/s) | 20260506 15:20:46.559 | INFO | Analyzing chunk frames = [2385920,2386432) | 20260506 15:20:46.559 | INFO | MemInfo: { | 20260506 15:20:46.559 | INFO | "gpu" : " 17 GB", | 20260506 15:20:46.559 | INFO | "huge" : "", | 20260506 15:20:46.559 | INFO | "huge_shared" : " 27 GB", | 20260506 15:20:46.559 | INFO | "malloc" : " 14 GB", | 20260506 15:20:46.559 | INFO | "pinned" : " 29 GB" | 20260506 15:20:46.559 | INFO | } | 20260506 15:20:46.559 | INFO | Perf report for the last 5.13996 seconds: | 20260506 15:20:46.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:20:46.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:46.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:46.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:46.559 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 15:20:46.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:46.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:46.559 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:20:46.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:46.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:46.560 | INFO | Uploaded 36GiB to GPU in 2.00117s (17.9894GiB/s) | 20260506 15:20:46.560 | INFO | Downloaded 34GiB to GPU in 1.93247s (17.594GiB/s) | 20260506 15:20:51.699 | INFO | Analyzing chunk frames = [2386432,2386944) | 20260506 15:20:51.699 | INFO | MemInfo: { | 20260506 15:20:51.699 | INFO | "gpu" : " 17 GB", | 20260506 15:20:51.699 | INFO | "huge" : "", | 20260506 15:20:51.699 | INFO | "huge_shared" : " 27 GB", | 20260506 15:20:51.699 | INFO | "malloc" : " 14 GB", | 20260506 15:20:51.699 | INFO | "pinned" : " 29 GB" | 20260506 15:20:51.699 | INFO | } | 20260506 15:20:51.699 | INFO | Perf report for the last 5.13979 seconds: | 20260506 15:20:51.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:20:51.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:51.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:51.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:51.699 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:20:51.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:51.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:51.699 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 15:20:51.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:51.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:51.699 | INFO | Uploaded 36GiB to GPU in 2.07857s (17.3196GiB/s) | 20260506 15:20:51.699 | INFO | Downloaded 34GiB to GPU in 1.96803s (17.2762GiB/s) | 20260506 15:20:56.837 | INFO | Analyzing chunk frames = [2386944,2387456) | 20260506 15:20:56.837 | INFO | MemInfo: { | 20260506 15:20:56.837 | INFO | "gpu" : " 17 GB", | 20260506 15:20:56.837 | INFO | "huge" : "", | 20260506 15:20:56.838 | INFO | "huge_shared" : " 27 GB", | 20260506 15:20:56.838 | INFO | "malloc" : " 14 GB", | 20260506 15:20:56.838 | INFO | "pinned" : " 29 GB" | 20260506 15:20:56.838 | INFO | } | 20260506 15:20:56.838 | INFO | Perf report for the last 5.13843 seconds: | 20260506 15:20:56.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:20:56.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:56.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:56.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:56.838 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:20:56.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:56.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:56.838 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:20:56.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:56.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:20:56.838 | INFO | Uploaded 36GiB to GPU in 2.03434s (17.6962GiB/s) | 20260506 15:20:56.838 | INFO | Downloaded 34GiB to GPU in 1.99497s (17.0428GiB/s) | 20260506 15:21:01.976 | INFO | Analyzing chunk frames = [2387456,2387968) | 20260506 15:21:01.976 | INFO | MemInfo: { | 20260506 15:21:01.976 | INFO | "gpu" : " 17 GB", | 20260506 15:21:01.976 | INFO | "huge" : "", | 20260506 15:21:01.976 | INFO | "huge_shared" : " 27 GB", | 20260506 15:21:01.976 | INFO | "malloc" : " 14 GB", | 20260506 15:21:01.977 | INFO | "pinned" : " 29 GB" | 20260506 15:21:01.977 | INFO | } | 20260506 15:21:01.977 | INFO | Perf report for the last 5.13896 seconds: | 20260506 15:21:01.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:21:01.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:01.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:01.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:01.977 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:21:01.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:01.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:01.977 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:21:01.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:01.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:01.977 | INFO | Uploaded 36GiB to GPU in 2.04413s (17.6114GiB/s) | 20260506 15:21:01.977 | INFO | Downloaded 34GiB to GPU in 1.9881s (17.1017GiB/s) | 20260506 15:21:02.086 | INFO | Starting a round of Baz IO | 20260506 15:21:05.241 | INFO | Populating RTMetrics report | 20260506 15:21:05.241 | INFO | RTMetrics report populated | 20260506 15:21:05.241 | INFO | Opening tmp RTMetrics json file | 20260506 15:21:05.267 | INFO | Serializing RTMetrics to json | 20260506 15:21:05.296 | INFO | Serialization done, swapping tmp file now... | 20260506 15:21:05.297 | INFO | RTMetrics done | 20260506 15:21:05.847 | INFO | Finished a round of Baz IO: Wrote 2.102 GiB in 3.76 seconds | 20260506 15:21:05.847 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.30% / 0.01% / 62.69% | 20260506 15:21:05.847 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.10501% / 1.58946e-05% / 5.5631e-05% | 20260506 15:21:05.847 | INFO | Baz IO overall: | 20260506 15:21:05.847 | INFO | BytesWritten : 1243928068455 | 20260506 15:21:05.847 | INFO | NumEvents : 601992774135 | 20260506 15:21:07.116 | INFO | Analyzing chunk frames = [2387968,2388480) | 20260506 15:21:07.116 | INFO | MemInfo: { | 20260506 15:21:07.116 | INFO | "gpu" : " 17 GB", | 20260506 15:21:07.116 | INFO | "huge" : "", | 20260506 15:21:07.116 | INFO | "huge_shared" : " 27 GB", | 20260506 15:21:07.116 | INFO | "malloc" : " 14 GB", | 20260506 15:21:07.116 | INFO | "pinned" : " 29 GB" | 20260506 15:21:07.116 | INFO | } | 20260506 15:21:07.116 | INFO | Perf report for the last 5.13954 seconds: | 20260506 15:21:07.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:21:07.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:07.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:07.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:07.116 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 15:21:07.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:07.116 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260506 15:21:07.116 | INFO | PRE_HQ : 60.8%, 1.9, 59.0ms, 0 | 20260506 15:21:07.116 | INFO | BAZWRITER : 72.6%, 4.8, 179.4ms, 0 | 20260506 15:21:07.116 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 15:21:07.116 | INFO | Uploaded 36GiB to GPU in 2.04302s (17.621GiB/s) | 20260506 15:21:07.116 | INFO | Downloaded 38GiB to GPU in 2.22374s (17.0883GiB/s) | 20260506 15:21:12.256 | INFO | Analyzing chunk frames = [2388480,2388992) | 20260506 15:21:12.256 | INFO | MemInfo: { | 20260506 15:21:12.256 | INFO | "gpu" : " 17 GB", | 20260506 15:21:12.256 | INFO | "huge" : "", | 20260506 15:21:12.256 | INFO | "huge_shared" : " 27 GB", | 20260506 15:21:12.256 | INFO | "malloc" : " 14 GB", | 20260506 15:21:12.256 | INFO | "pinned" : " 29 GB" | 20260506 15:21:12.256 | INFO | } | 20260506 15:21:12.256 | INFO | Perf report for the last 5.13977 seconds: | 20260506 15:21:12.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:21:12.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:12.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:12.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:12.256 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 15:21:12.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:12.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:12.256 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:21:12.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:12.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:12.256 | INFO | Uploaded 36GiB to GPU in 1.97565s (18.2219GiB/s) | 20260506 15:21:12.256 | INFO | Downloaded 34GiB to GPU in 1.93968s (17.5287GiB/s) | 20260506 15:21:17.396 | INFO | Analyzing chunk frames = [2388992,2389504) | 20260506 15:21:17.396 | INFO | MemInfo: { | 20260506 15:21:17.396 | INFO | "gpu" : " 17 GB", | 20260506 15:21:17.396 | INFO | "huge" : "", | 20260506 15:21:17.396 | INFO | "huge_shared" : " 27 GB", | 20260506 15:21:17.396 | INFO | "malloc" : " 14 GB", | 20260506 15:21:17.396 | INFO | "pinned" : " 29 GB" | 20260506 15:21:17.396 | INFO | } | 20260506 15:21:17.396 | INFO | Perf report for the last 5.13973 seconds: | 20260506 15:21:17.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:21:17.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:17.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:17.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:17.396 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 15:21:17.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:17.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:17.396 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:21:17.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:17.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:17.396 | INFO | Uploaded 36GiB to GPU in 1.96758s (18.2966GiB/s) | 20260506 15:21:17.396 | INFO | Downloaded 34GiB to GPU in 1.94424s (17.4876GiB/s) | 20260506 15:21:22.536 | INFO | Analyzing chunk frames = [2389504,2390016) | 20260506 15:21:22.536 | INFO | MemInfo: { | 20260506 15:21:22.536 | INFO | "gpu" : " 17 GB", | 20260506 15:21:22.536 | INFO | "huge" : "", | 20260506 15:21:22.536 | INFO | "huge_shared" : " 27 GB", | 20260506 15:21:22.536 | INFO | "malloc" : " 14 GB", | 20260506 15:21:22.536 | INFO | "pinned" : " 29 GB" | 20260506 15:21:22.536 | INFO | } | 20260506 15:21:22.537 | INFO | Perf report for the last 5.14085 seconds: | 20260506 15:21:22.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:21:22.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:22.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:22.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:22.537 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 15:21:22.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:22.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:22.537 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:21:22.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:22.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:22.537 | INFO | Uploaded 36GiB to GPU in 2.01356s (17.8787GiB/s) | 20260506 15:21:22.537 | INFO | Downloaded 34GiB to GPU in 1.91498s (17.7548GiB/s) | 20260506 15:21:27.674 | INFO | Analyzing chunk frames = [2390016,2390528) | 20260506 15:21:27.674 | INFO | MemInfo: { | 20260506 15:21:27.674 | INFO | "gpu" : " 17 GB", | 20260506 15:21:27.674 | INFO | "huge" : "", | 20260506 15:21:27.674 | INFO | "huge_shared" : " 27 GB", | 20260506 15:21:27.674 | INFO | "malloc" : " 14 GB", | 20260506 15:21:27.674 | INFO | "pinned" : " 29 GB" | 20260506 15:21:27.674 | INFO | } | 20260506 15:21:27.674 | INFO | Perf report for the last 5.13765 seconds: | 20260506 15:21:27.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:21:27.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:27.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:27.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:27.674 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 15:21:27.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:27.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:27.674 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:21:27.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:27.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:27.674 | INFO | Uploaded 36GiB to GPU in 1.98408s (18.1444GiB/s) | 20260506 15:21:27.675 | INFO | Downloaded 34GiB to GPU in 1.93409s (17.5793GiB/s) | 20260506 15:21:32.813 | INFO | Analyzing chunk frames = [2390528,2391040) | 20260506 15:21:32.813 | INFO | MemInfo: { | 20260506 15:21:32.813 | INFO | "gpu" : " 17 GB", | 20260506 15:21:32.813 | INFO | "huge" : "", | 20260506 15:21:32.813 | INFO | "huge_shared" : " 27 GB", | 20260506 15:21:32.813 | INFO | "malloc" : " 14 GB", | 20260506 15:21:32.813 | INFO | "pinned" : " 29 GB" | 20260506 15:21:32.813 | INFO | } | 20260506 15:21:32.813 | INFO | Perf report for the last 5.13872 seconds: | 20260506 15:21:32.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:21:32.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:32.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:32.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:32.813 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 15:21:32.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:32.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:32.813 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260506 15:21:32.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:32.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:32.813 | INFO | Uploaded 36GiB to GPU in 2.06176s (17.4608GiB/s) | 20260506 15:21:32.813 | INFO | Downloaded 34GiB to GPU in 1.92583s (17.6547GiB/s) | 20260506 15:21:37.953 | INFO | Analyzing chunk frames = [2391040,2391552) | 20260506 15:21:37.953 | INFO | MemInfo: { | 20260506 15:21:37.953 | INFO | "gpu" : " 17 GB", | 20260506 15:21:37.953 | INFO | "huge" : "", | 20260506 15:21:37.953 | INFO | "huge_shared" : " 27 GB", | 20260506 15:21:37.953 | INFO | "malloc" : " 14 GB", | 20260506 15:21:37.953 | INFO | "pinned" : " 29 GB" | 20260506 15:21:37.953 | INFO | } | 20260506 15:21:37.953 | INFO | Perf report for the last 5.14027 seconds: | 20260506 15:21:37.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:21:37.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:37.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:37.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:37.953 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:21:37.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:37.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:37.953 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:21:37.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:37.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:37.953 | INFO | Uploaded 36GiB to GPU in 1.99487s (18.0463GiB/s) | 20260506 15:21:37.953 | INFO | Downloaded 34GiB to GPU in 1.97623s (17.2045GiB/s) | 20260506 15:21:43.092 | INFO | Analyzing chunk frames = [2391552,2392064) | 20260506 15:21:43.092 | INFO | MemInfo: { | 20260506 15:21:43.092 | INFO | "gpu" : " 17 GB", | 20260506 15:21:43.092 | INFO | "huge" : "", | 20260506 15:21:43.092 | INFO | "huge_shared" : " 27 GB", | 20260506 15:21:43.092 | INFO | "malloc" : " 14 GB", | 20260506 15:21:43.092 | INFO | "pinned" : " 29 GB" | 20260506 15:21:43.092 | INFO | } | 20260506 15:21:43.092 | INFO | Perf report for the last 5.13847 seconds: | 20260506 15:21:43.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:21:43.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:43.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:43.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:43.092 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 15:21:43.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:43.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:43.092 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:21:43.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:43.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:43.092 | INFO | Uploaded 36GiB to GPU in 2.02802s (17.7513GiB/s) | 20260506 15:21:43.092 | INFO | Downloaded 34GiB to GPU in 1.97595s (17.2069GiB/s) | 20260506 15:21:43.092 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:21:43.092 | INFO | Total mem usage: 17.4 GiB | 20260506 15:21:43.092 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:21:43.092 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -10.9%) | 20260506 15:21:43.092 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:21:43.092 | INFO | High Water Breakdown: | 20260506 15:21:43.092 | INFO | 61.3% : 9.63 GiB ( 88.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:21:43.092 | INFO | 38.6% : 6.06 GiB ( 90.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:21:43.092 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:21:43.092 | INFO | ------------------------------------------ | 20260506 15:21:43.092 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:21:43.092 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:21:43.092 | INFO | Total mem usage: 26.5 GiB | 20260506 15:21:43.092 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:21:43.092 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:21:43.092 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:21:43.092 | INFO | High Water Breakdown: | 20260506 15:21:43.092 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:21:43.092 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:21:43.092 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:21:43.092 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:21:43.093 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:21:43.093 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:21:43.093 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:21:43.093 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:21:43.093 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:21:43.093 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:21:43.093 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:21:43.093 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:21:43.093 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:21:43.093 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:21:43.093 | INFO | ------------------------------------------ | 20260506 15:21:43.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:21:43.093 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:21:43.093 | INFO | Total mem usage: 25.6 GiB | 20260506 15:21:43.093 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:21:43.093 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:21:43.093 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:21:43.093 | INFO | High Water Breakdown: | 20260506 15:21:43.093 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:21:43.093 | INFO | ------------------------------------------ | 20260506 15:21:43.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:21:43.093 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:21:43.093 | INFO | Total mem usage: 15.7 GiB | 20260506 15:21:43.093 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:21:43.093 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:21:43.093 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 15:21:43.093 | INFO | High Water Breakdown: | 20260506 15:21:43.093 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:21:43.093 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:21:43.093 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:21:43.093 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:21:43.093 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:21:43.093 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:21:43.093 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:21:43.093 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:21:43.093 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:21:43.093 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:21:43.093 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:21:43.093 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:21:43.093 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:21:43.093 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:21:43.093 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:21:43.093 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:21:43.093 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:21:43.093 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:21:43.093 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:21:43.093 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:21:43.093 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:21:43.093 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:21:43.093 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:21:43.093 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:21:43.093 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:21:43.093 | INFO | ------------------------------------------ | 20260506 15:21:43.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:21:43.093 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:21:43.093 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:21:43.093 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:21:43.093 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.53e+01, 1.34e+01 / 1.99e+00 | 20260506 15:21:43.093 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.02e-01, 1.05e-02 / 7.11e-03 | 20260506 15:21:43.093 | INFO | PerfReport: 11.58%, Baseline -- 3.64e+00 / 4.37e+00, 5.53e-01 / 4.15e-01 | 20260506 15:21:43.093 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.85e-01 / 1.55e-01 | 20260506 15:21:43.093 | INFO | PerfReport: 15.84%, DME -- 3.98e+01 / 4.38e+01, 3.84e+00 / 2.02e+00 | 20260506 15:21:43.093 | INFO | PerfReport: 56.85%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.44e+00 / 2.03e+00 | 20260506 15:21:43.093 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.32e+00 / 2.91e+00, 3.96e-01 / 2.36e-01 | 20260506 15:21:43.093 | INFO | PerfReport: 3.56%, Metrics -- 1.12e+00 / 1.46e+00, 2.87e-01 / 1.94e-01 | 20260506 15:21:43.093 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.89e-01 / 4.39e-01, 5.97e-02 / 2.64e-02 | 20260506 15:21:43.093 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:21:43.093 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:21:43.093 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:21:43.093 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 15:21:43.093 | INFO | PerfReport: 21.58%, Upload -- 2.11e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260506 15:21:43.093 | INFO | PerfReport: 21.18%, Download -- 2.07e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 15:21:43.093 | INFO | PerfReport: 32.20%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 15:21:43.093 | INFO | PerfReport: 1.54%, LockUpload -- 1.50e+00 / 0.00e+00, 5.84e+00 / 0.00e+00 | 20260506 15:21:43.093 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.70e+00 / 0.00e+00 | 20260506 15:21:43.093 | INFO | PerfReport: 22.38%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 15:21:43.093 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260506 15:21:43.203 | INFO | Starting a round of Baz IO | 20260506 15:21:46.358 | INFO | Populating RTMetrics report | 20260506 15:21:46.358 | INFO | RTMetrics report populated | 20260506 15:21:46.358 | INFO | Opening tmp RTMetrics json file | 20260506 15:21:46.376 | INFO | Serializing RTMetrics to json | 20260506 15:21:46.378 | INFO | Serialization done, swapping tmp file now... | 20260506 15:21:46.378 | INFO | RTMetrics done | 20260506 15:21:46.507 | INFO | Finished a round of Baz IO: Wrote 2.101 GiB in 3.304 seconds | 20260506 15:21:46.507 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.29% / 0.01% / 62.70% | 20260506 15:21:46.507 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.09846% / 2.78155e-05% / 5.5631e-05% | 20260506 15:21:46.507 | INFO | Baz IO overall: | 20260506 15:21:46.507 | INFO | BytesWritten : 1246184487786 | 20260506 15:21:46.507 | INFO | NumEvents : 603084571279 | 20260506 15:21:48.231 | INFO | Analyzing chunk frames = [2392064,2392576) | 20260506 15:21:48.231 | INFO | MemInfo: { | 20260506 15:21:48.231 | INFO | "gpu" : " 17 GB", | 20260506 15:21:48.231 | INFO | "huge" : "", | 20260506 15:21:48.231 | INFO | "huge_shared" : " 27 GB", | 20260506 15:21:48.231 | INFO | "malloc" : " 14 GB", | 20260506 15:21:48.231 | INFO | "pinned" : " 29 GB" | 20260506 15:21:48.231 | INFO | } | 20260506 15:21:48.231 | INFO | Perf report for the last 5.13942 seconds: | 20260506 15:21:48.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:21:48.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:48.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:48.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:48.231 | INFO | ANALYSIS : 63.0%, 2.9, 95.1ms, 0 | 20260506 15:21:48.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:48.231 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 15:21:48.231 | INFO | PRE_HQ : 61.1%, 1.9, 59.5ms, 0 | 20260506 15:21:48.231 | INFO | BAZWRITER : 63.7%, 4.8, 157.0ms, 0 | 20260506 15:21:48.231 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 15:21:48.231 | INFO | Uploaded 36GiB to GPU in 2.09156s (17.212GiB/s) | 20260506 15:21:48.231 | INFO | Downloaded 38GiB to GPU in 2.22847s (17.052GiB/s) | 20260506 15:21:53.370 | INFO | Analyzing chunk frames = [2392576,2393088) | 20260506 15:21:53.371 | INFO | MemInfo: { | 20260506 15:21:53.371 | INFO | "gpu" : " 17 GB", | 20260506 15:21:53.371 | INFO | "huge" : "", | 20260506 15:21:53.371 | INFO | "huge_shared" : " 27 GB", | 20260506 15:21:53.371 | INFO | "malloc" : " 14 GB", | 20260506 15:21:53.371 | INFO | "pinned" : " 29 GB" | 20260506 15:21:53.371 | INFO | } | 20260506 15:21:53.371 | INFO | Perf report for the last 5.13954 seconds: | 20260506 15:21:53.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:21:53.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:53.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:53.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:53.371 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 15:21:53.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:53.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:53.371 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:21:53.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:53.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:53.371 | INFO | Uploaded 36GiB to GPU in 2.01751s (17.8437GiB/s) | 20260506 15:21:53.371 | INFO | Downloaded 34GiB to GPU in 1.94363s (17.493GiB/s) | 20260506 15:21:58.510 | INFO | Analyzing chunk frames = [2393088,2393600) | 20260506 15:21:58.510 | INFO | MemInfo: { | 20260506 15:21:58.510 | INFO | "gpu" : " 17 GB", | 20260506 15:21:58.510 | INFO | "huge" : "", | 20260506 15:21:58.510 | INFO | "huge_shared" : " 27 GB", | 20260506 15:21:58.510 | INFO | "malloc" : " 14 GB", | 20260506 15:21:58.510 | INFO | "pinned" : " 29 GB" | 20260506 15:21:58.510 | INFO | } | 20260506 15:21:58.510 | INFO | Perf report for the last 5.13929 seconds: | 20260506 15:21:58.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:21:58.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:58.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:58.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:58.510 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 15:21:58.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:58.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:58.510 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:21:58.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:58.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:21:58.510 | INFO | Uploaded 36GiB to GPU in 2.0052s (17.9533GiB/s) | 20260506 15:21:58.510 | INFO | Downloaded 34GiB to GPU in 1.93827s (17.5414GiB/s) | 20260506 15:22:03.649 | INFO | Analyzing chunk frames = [2393600,2394112) | 20260506 15:22:03.649 | INFO | MemInfo: { | 20260506 15:22:03.649 | INFO | "gpu" : " 17 GB", | 20260506 15:22:03.649 | INFO | "huge" : "", | 20260506 15:22:03.649 | INFO | "huge_shared" : " 27 GB", | 20260506 15:22:03.649 | INFO | "malloc" : " 14 GB", | 20260506 15:22:03.649 | INFO | "pinned" : " 29 GB" | 20260506 15:22:03.649 | INFO | } | 20260506 15:22:03.649 | INFO | Perf report for the last 5.13888 seconds: | 20260506 15:22:03.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:22:03.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:03.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:03.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:03.649 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:22:03.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:03.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:03.649 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:22:03.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:03.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:03.649 | INFO | Uploaded 36GiB to GPU in 1.99827s (18.0156GiB/s) | 20260506 15:22:03.649 | INFO | Downloaded 34GiB to GPU in 1.91855s (17.7217GiB/s) | 20260506 15:22:08.788 | INFO | Analyzing chunk frames = [2394112,2394624) | 20260506 15:22:08.788 | INFO | MemInfo: { | 20260506 15:22:08.788 | INFO | "gpu" : " 17 GB", | 20260506 15:22:08.788 | INFO | "huge" : "", | 20260506 15:22:08.788 | INFO | "huge_shared" : " 27 GB", | 20260506 15:22:08.788 | INFO | "malloc" : " 14 GB", | 20260506 15:22:08.788 | INFO | "pinned" : " 29 GB" | 20260506 15:22:08.788 | INFO | } | 20260506 15:22:08.788 | INFO | Perf report for the last 5.13931 seconds: | 20260506 15:22:08.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:22:08.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:08.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:08.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:08.788 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 15:22:08.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:08.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:08.788 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 15:22:08.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:08.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:08.789 | INFO | Uploaded 36GiB to GPU in 2.04167s (17.6326GiB/s) | 20260506 15:22:08.789 | INFO | Downloaded 34GiB to GPU in 1.93691s (17.5538GiB/s) | 20260506 15:22:13.928 | INFO | Analyzing chunk frames = [2394624,2395136) | 20260506 15:22:13.928 | INFO | MemInfo: { | 20260506 15:22:13.928 | INFO | "gpu" : " 17 GB", | 20260506 15:22:13.928 | INFO | "huge" : "", | 20260506 15:22:13.928 | INFO | "huge_shared" : " 27 GB", | 20260506 15:22:13.928 | INFO | "malloc" : " 14 GB", | 20260506 15:22:13.928 | INFO | "pinned" : " 29 GB" | 20260506 15:22:13.928 | INFO | } | 20260506 15:22:13.928 | INFO | Perf report for the last 5.14009 seconds: | 20260506 15:22:13.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:22:13.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:13.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:13.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:13.928 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260506 15:22:13.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:13.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:13.928 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:22:13.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:13.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:13.928 | INFO | Uploaded 36GiB to GPU in 2.05866s (17.4871GiB/s) | 20260506 15:22:13.928 | INFO | Downloaded 34GiB to GPU in 1.92157s (17.6939GiB/s) | 20260506 15:22:19.067 | INFO | Analyzing chunk frames = [2395136,2395648) | 20260506 15:22:19.067 | INFO | MemInfo: { | 20260506 15:22:19.067 | INFO | "gpu" : " 17 GB", | 20260506 15:22:19.067 | INFO | "huge" : "", | 20260506 15:22:19.068 | INFO | "huge_shared" : " 27 GB", | 20260506 15:22:19.068 | INFO | "malloc" : " 14 GB", | 20260506 15:22:19.068 | INFO | "pinned" : " 29 GB" | 20260506 15:22:19.068 | INFO | } | 20260506 15:22:19.068 | INFO | Perf report for the last 5.13932 seconds: | 20260506 15:22:19.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:22:19.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:19.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:19.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:19.068 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:22:19.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:19.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:19.068 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:22:19.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:19.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:19.068 | INFO | Uploaded 36GiB to GPU in 2.00893s (17.92GiB/s) | 20260506 15:22:19.068 | INFO | Downloaded 34GiB to GPU in 1.98255s (17.1496GiB/s) | 20260506 15:22:24.207 | INFO | Analyzing chunk frames = [2395648,2396160) | 20260506 15:22:24.207 | INFO | MemInfo: { | 20260506 15:22:24.207 | INFO | "gpu" : " 17 GB", | 20260506 15:22:24.207 | INFO | "huge" : "", | 20260506 15:22:24.207 | INFO | "huge_shared" : " 27 GB", | 20260506 15:22:24.207 | INFO | "malloc" : " 14 GB", | 20260506 15:22:24.207 | INFO | "pinned" : " 29 GB" | 20260506 15:22:24.207 | INFO | } | 20260506 15:22:24.207 | INFO | Perf report for the last 5.13946 seconds: | 20260506 15:22:24.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:22:24.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:24.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:24.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:24.207 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 15:22:24.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:24.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:24.207 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:22:24.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:24.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:24.207 | INFO | Uploaded 36GiB to GPU in 2.0387s (17.6583GiB/s) | 20260506 15:22:24.207 | INFO | Downloaded 34GiB to GPU in 1.98374s (17.1393GiB/s) | 20260506 15:22:24.320 | INFO | Starting a round of Baz IO | 20260506 15:22:27.475 | INFO | Populating RTMetrics report | 20260506 15:22:27.476 | INFO | RTMetrics report populated | 20260506 15:22:27.476 | INFO | Opening tmp RTMetrics json file | 20260506 15:22:27.523 | INFO | Serializing RTMetrics to json | 20260506 15:22:27.553 | INFO | Serialization done, swapping tmp file now... | 20260506 15:22:27.560 | INFO | RTMetrics done | 20260506 15:22:27.801 | INFO | Finished a round of Baz IO: Wrote 2.102 GiB in 3.481 seconds | 20260506 15:22:27.801 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.28% / 0.01% / 62.71% | 20260506 15:22:27.801 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.0951% / 1.58946e-05% / 6.35783e-05% | 20260506 15:22:27.801 | INFO | Baz IO overall: | 20260506 15:22:27.801 | INFO | BytesWritten : 1248441455521 | 20260506 15:22:27.802 | INFO | NumEvents : 604176781873 | 20260506 15:22:29.346 | INFO | Analyzing chunk frames = [2396160,2396672) | 20260506 15:22:29.346 | INFO | MemInfo: { | 20260506 15:22:29.346 | INFO | "gpu" : " 17 GB", | 20260506 15:22:29.346 | INFO | "huge" : "", | 20260506 15:22:29.346 | INFO | "huge_shared" : " 27 GB", | 20260506 15:22:29.346 | INFO | "malloc" : " 14 GB", | 20260506 15:22:29.346 | INFO | "pinned" : " 29 GB" | 20260506 15:22:29.346 | INFO | } | 20260506 15:22:29.346 | INFO | Perf report for the last 5.1392 seconds: | 20260506 15:22:29.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:22:29.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:29.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:29.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:29.346 | INFO | ANALYSIS : 63.0%, 2.9, 95.5ms, 0 | 20260506 15:22:29.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:29.346 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 15:22:29.346 | INFO | PRE_HQ : 61.0%, 1.9, 59.4ms, 0 | 20260506 15:22:29.346 | INFO | BAZWRITER : 67.2%, 4.8, 166.4ms, 0 | 20260506 15:22:29.346 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 15:22:29.346 | INFO | Uploaded 36GiB to GPU in 2.07737s (17.3296GiB/s) | 20260506 15:22:29.346 | INFO | Downloaded 38GiB to GPU in 2.22941s (17.0449GiB/s) | 20260506 15:22:34.485 | INFO | Analyzing chunk frames = [2396672,2397184) | 20260506 15:22:34.485 | INFO | MemInfo: { | 20260506 15:22:34.485 | INFO | "gpu" : " 17 GB", | 20260506 15:22:34.485 | INFO | "huge" : "", | 20260506 15:22:34.485 | INFO | "huge_shared" : " 27 GB", | 20260506 15:22:34.485 | INFO | "malloc" : " 14 GB", | 20260506 15:22:34.485 | INFO | "pinned" : " 29 GB" | 20260506 15:22:34.485 | INFO | } | 20260506 15:22:34.486 | INFO | Perf report for the last 5.13923 seconds: | 20260506 15:22:34.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:22:34.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:34.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:34.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:34.486 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:22:34.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:34.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:34.486 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:22:34.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:34.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:34.486 | INFO | Uploaded 36GiB to GPU in 1.99225s (18.07GiB/s) | 20260506 15:22:34.486 | INFO | Downloaded 34GiB to GPU in 1.9483s (17.4511GiB/s) | 20260506 15:22:39.625 | INFO | Analyzing chunk frames = [2397184,2397696) | 20260506 15:22:39.625 | INFO | MemInfo: { | 20260506 15:22:39.625 | INFO | "gpu" : " 17 GB", | 20260506 15:22:39.626 | INFO | "huge" : "", | 20260506 15:22:39.626 | INFO | "huge_shared" : " 27 GB", | 20260506 15:22:39.626 | INFO | "malloc" : " 14 GB", | 20260506 15:22:39.626 | INFO | "pinned" : " 29 GB" | 20260506 15:22:39.626 | INFO | } | 20260506 15:22:39.626 | INFO | Perf report for the last 5.14008 seconds: | 20260506 15:22:39.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:22:39.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:39.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:39.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:39.626 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:22:39.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:39.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:39.626 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:22:39.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:39.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:39.626 | INFO | Uploaded 36GiB to GPU in 1.98624s (18.1247GiB/s) | 20260506 15:22:39.626 | INFO | Downloaded 34GiB to GPU in 1.93993s (17.5264GiB/s) | 20260506 15:22:44.763 | INFO | Analyzing chunk frames = [2397696,2398208) | 20260506 15:22:44.763 | INFO | MemInfo: { | 20260506 15:22:44.763 | INFO | "gpu" : " 17 GB", | 20260506 15:22:44.763 | INFO | "huge" : "", | 20260506 15:22:44.763 | INFO | "huge_shared" : " 27 GB", | 20260506 15:22:44.763 | INFO | "malloc" : " 14 GB", | 20260506 15:22:44.763 | INFO | "pinned" : " 29 GB" | 20260506 15:22:44.763 | INFO | } | 20260506 15:22:44.763 | INFO | Perf report for the last 5.13768 seconds: | 20260506 15:22:44.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:22:44.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:44.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:44.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:44.763 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 15:22:44.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:44.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:44.763 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:22:44.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:44.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:44.764 | INFO | Uploaded 36GiB to GPU in 1.98421s (18.1432GiB/s) | 20260506 15:22:44.764 | INFO | Downloaded 34GiB to GPU in 1.88471s (18.0399GiB/s) | 20260506 15:22:49.902 | INFO | Analyzing chunk frames = [2398208,2398720) | 20260506 15:22:49.903 | INFO | MemInfo: { | 20260506 15:22:49.903 | INFO | "gpu" : " 17 GB", | 20260506 15:22:49.903 | INFO | "huge" : "", | 20260506 15:22:49.903 | INFO | "huge_shared" : " 27 GB", | 20260506 15:22:49.903 | INFO | "malloc" : " 14 GB", | 20260506 15:22:49.903 | INFO | "pinned" : " 29 GB" | 20260506 15:22:49.903 | INFO | } | 20260506 15:22:49.903 | INFO | Perf report for the last 5.13932 seconds: | 20260506 15:22:49.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:22:49.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:49.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:49.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:49.903 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 15:22:49.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:49.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:49.903 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:22:49.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:49.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:49.903 | INFO | Uploaded 36GiB to GPU in 2.00635s (17.943GiB/s) | 20260506 15:22:49.903 | INFO | Downloaded 34GiB to GPU in 1.93885s (17.5362GiB/s) | 20260506 15:22:55.043 | INFO | Analyzing chunk frames = [2398720,2399232) | 20260506 15:22:55.043 | INFO | MemInfo: { | 20260506 15:22:55.043 | INFO | "gpu" : " 17 GB", | 20260506 15:22:55.043 | INFO | "huge" : "", | 20260506 15:22:55.043 | INFO | "huge_shared" : " 27 GB", | 20260506 15:22:55.043 | INFO | "malloc" : " 14 GB", | 20260506 15:22:55.043 | INFO | "pinned" : " 29 GB" | 20260506 15:22:55.043 | INFO | } | 20260506 15:22:55.043 | INFO | Perf report for the last 5.1401 seconds: | 20260506 15:22:55.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:22:55.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:55.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:55.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:55.043 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:22:55.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:55.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:55.043 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260506 15:22:55.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:55.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:22:55.043 | INFO | Uploaded 36GiB to GPU in 2.07983s (17.3091GiB/s) | 20260506 15:22:55.043 | INFO | Downloaded 34GiB to GPU in 1.97317s (17.2312GiB/s) | 20260506 15:23:00.183 | INFO | Analyzing chunk frames = [2399232,2399744) | 20260506 15:23:00.183 | INFO | MemInfo: { | 20260506 15:23:00.183 | INFO | "gpu" : " 17 GB", | 20260506 15:23:00.183 | INFO | "huge" : "", | 20260506 15:23:00.183 | INFO | "huge_shared" : " 27 GB", | 20260506 15:23:00.183 | INFO | "malloc" : " 14 GB", | 20260506 15:23:00.183 | INFO | "pinned" : " 29 GB" | 20260506 15:23:00.183 | INFO | } | 20260506 15:23:00.183 | INFO | Perf report for the last 5.14033 seconds: | 20260506 15:23:00.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:23:00.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:00.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:00.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:00.183 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:23:00.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:00.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:00.183 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:23:00.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:00.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:00.183 | INFO | Uploaded 36GiB to GPU in 2.0112s (17.8997GiB/s) | 20260506 15:23:00.184 | INFO | Downloaded 34GiB to GPU in 1.98941s (17.0905GiB/s) | 20260506 15:23:05.321 | INFO | Analyzing chunk frames = [2399744,2400256) | 20260506 15:23:05.321 | INFO | MemInfo: { | 20260506 15:23:05.321 | INFO | "gpu" : " 17 GB", | 20260506 15:23:05.321 | INFO | "huge" : "", | 20260506 15:23:05.321 | INFO | "huge_shared" : " 27 GB", | 20260506 15:23:05.321 | INFO | "malloc" : " 14 GB", | 20260506 15:23:05.321 | INFO | "pinned" : " 29 GB" | 20260506 15:23:05.321 | INFO | } | 20260506 15:23:05.321 | INFO | Perf report for the last 5.13807 seconds: | 20260506 15:23:05.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:23:05.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:05.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:05.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:05.321 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 15:23:05.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:05.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:05.321 | INFO | PRE_HQ : 32.9%, 1.0, 17.1ms, 0 | 20260506 15:23:05.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:05.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:05.321 | INFO | Uploaded 36GiB to GPU in 2.017s (17.8483GiB/s) | 20260506 15:23:05.322 | INFO | Downloaded 34GiB to GPU in 1.98112s (17.162GiB/s) | 20260506 15:23:05.322 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:23:05.322 | INFO | Total mem usage: 17.4 GiB | 20260506 15:23:05.322 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:23:05.322 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -11.4%) | 20260506 15:23:05.322 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:23:05.322 | INFO | High Water Breakdown: | 20260506 15:23:05.322 | INFO | 61.3% : 9.57 GiB ( 87.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:23:05.322 | INFO | 38.6% : 6.03 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:23:05.322 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:23:05.322 | INFO | ------------------------------------------ | 20260506 15:23:05.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:23:05.322 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:23:05.322 | INFO | Total mem usage: 26.5 GiB | 20260506 15:23:05.322 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:23:05.322 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:23:05.322 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:23:05.322 | INFO | High Water Breakdown: | 20260506 15:23:05.322 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:23:05.322 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:23:05.322 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:23:05.322 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:23:05.322 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:23:05.322 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:23:05.322 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:23:05.322 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:23:05.322 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:23:05.322 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:23:05.322 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:23:05.322 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:23:05.322 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:23:05.322 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:23:05.322 | INFO | ------------------------------------------ | 20260506 15:23:05.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:23:05.322 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:23:05.322 | INFO | Total mem usage: 25.6 GiB | 20260506 15:23:05.322 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:23:05.322 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:23:05.322 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:23:05.322 | INFO | High Water Breakdown: | 20260506 15:23:05.322 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:23:05.322 | INFO | ------------------------------------------ | 20260506 15:23:05.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:23:05.322 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:23:05.322 | INFO | Total mem usage: 15.7 GiB | 20260506 15:23:05.322 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:23:05.322 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:23:05.322 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 15:23:05.322 | INFO | High Water Breakdown: | 20260506 15:23:05.322 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:23:05.322 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:23:05.322 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:23:05.322 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:23:05.322 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:23:05.322 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:23:05.322 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:23:05.322 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:23:05.322 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:23:05.322 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:23:05.322 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:23:05.322 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:23:05.322 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:23:05.322 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:23:05.322 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:23:05.322 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:23:05.322 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:23:05.322 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:23:05.322 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:23:05.323 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:23:05.323 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:23:05.323 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:23:05.323 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:23:05.323 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:23:05.323 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:23:05.323 | INFO | ------------------------------------------ | 20260506 15:23:05.323 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:23:05.323 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:23:05.323 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:23:05.323 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:23:05.323 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.52e+01, 1.34e+01 / 1.94e+00 | 20260506 15:23:05.323 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.00e-01, 1.02e-02 / 5.47e-03 | 20260506 15:23:05.323 | INFO | PerfReport: 11.60%, Baseline -- 3.65e+00 / 4.49e+00, 5.62e-01 / 3.26e-01 | 20260506 15:23:05.323 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.84e-01 / 1.31e-01 | 20260506 15:23:05.323 | INFO | PerfReport: 15.92%, DME -- 4.00e+01 / 4.44e+01, 3.92e+00 / 2.04e+00 | 20260506 15:23:05.323 | INFO | PerfReport: 56.74%, FrameLabeling -- 1.78e+01 / 2.00e+01, 1.39e+00 / 1.44e+00 | 20260506 15:23:05.323 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.33e+00 / 2.94e+00, 4.03e-01 / 2.31e-01 | 20260506 15:23:05.323 | INFO | PerfReport: 3.55%, Metrics -- 1.11e+00 / 1.46e+00, 2.82e-01 / 1.80e-01 | 20260506 15:23:05.323 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.89e-01 / 4.38e-01, 6.04e-02 / 1.94e-02 | 20260506 15:23:05.323 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:23:05.323 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:23:05.323 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:23:05.323 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 15:23:05.323 | INFO | PerfReport: 21.62%, Upload -- 2.11e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260506 15:23:05.323 | INFO | PerfReport: 21.17%, Download -- 2.07e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 15:23:05.323 | INFO | PerfReport: 32.20%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 15:23:05.323 | INFO | PerfReport: 1.64%, LockUpload -- 1.60e+00 / 0.00e+00, 6.08e+00 / 0.00e+00 | 20260506 15:23:05.323 | INFO | PerfReport: 1.05%, LockDownload -- 1.02e+00 / 0.00e+00, 2.57e+00 / 0.00e+00 | 20260506 15:23:05.323 | INFO | PerfReport: 22.30%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 15:23:05.323 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260506 15:23:05.435 | INFO | Starting a round of Baz IO | 20260506 15:23:08.589 | INFO | Populating RTMetrics report | 20260506 15:23:08.589 | INFO | RTMetrics report populated | 20260506 15:23:08.589 | INFO | Opening tmp RTMetrics json file | 20260506 15:23:08.614 | INFO | Serializing RTMetrics to json | 20260506 15:23:08.619 | INFO | Serialization done, swapping tmp file now... | 20260506 15:23:08.624 | INFO | RTMetrics done | 20260506 15:23:08.964 | INFO | Finished a round of Baz IO: Wrote 2.102 GiB in 3.529 seconds | 20260506 15:23:08.964 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.27% / 0.01% / 62.72% | 20260506 15:23:08.964 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.09084% / 1.98682e-05% / 3.97364e-05% | 20260506 15:23:08.964 | INFO | Baz IO overall: | 20260506 15:23:08.964 | INFO | BytesWritten : 1250698914707 | 20260506 15:23:08.964 | INFO | NumEvents : 605269423834 | 20260506 15:23:10.460 | INFO | Analyzing chunk frames = [2400256,2400768) | 20260506 15:23:10.460 | INFO | MemInfo: { | 20260506 15:23:10.460 | INFO | "gpu" : " 17 GB", | 20260506 15:23:10.460 | INFO | "huge" : "", | 20260506 15:23:10.460 | INFO | "huge_shared" : " 27 GB", | 20260506 15:23:10.460 | INFO | "malloc" : " 14 GB", | 20260506 15:23:10.460 | INFO | "pinned" : " 29 GB" | 20260506 15:23:10.460 | INFO | } | 20260506 15:23:10.461 | INFO | Perf report for the last 5.13932 seconds: | 20260506 15:23:10.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:23:10.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:10.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:10.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:10.461 | INFO | ANALYSIS : 63.0%, 2.9, 95.4ms, 0 | 20260506 15:23:10.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:10.461 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 15:23:10.461 | INFO | PRE_HQ : 60.8%, 1.9, 59.5ms, 0 | 20260506 15:23:10.461 | INFO | BAZWRITER : 68.0%, 4.8, 168.4ms, 0 | 20260506 15:23:10.461 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 15:23:10.461 | INFO | Uploaded 36GiB to GPU in 2.03403s (17.6988GiB/s) | 20260506 15:23:10.461 | INFO | Downloaded 38GiB to GPU in 2.23078s (17.0344GiB/s) | 20260506 15:23:15.599 | INFO | Analyzing chunk frames = [2400768,2401280) | 20260506 15:23:15.600 | INFO | MemInfo: { | 20260506 15:23:15.600 | INFO | "gpu" : " 17 GB", | 20260506 15:23:15.600 | INFO | "huge" : "", | 20260506 15:23:15.600 | INFO | "huge_shared" : " 27 GB", | 20260506 15:23:15.600 | INFO | "malloc" : " 14 GB", | 20260506 15:23:15.600 | INFO | "pinned" : " 29 GB" | 20260506 15:23:15.600 | INFO | } | 20260506 15:23:15.600 | INFO | Perf report for the last 5.13912 seconds: | 20260506 15:23:15.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:23:15.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:15.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:15.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:15.600 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:23:15.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:15.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:15.600 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:23:15.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:15.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:15.600 | INFO | Uploaded 36GiB to GPU in 1.99321s (18.0613GiB/s) | 20260506 15:23:15.600 | INFO | Downloaded 34GiB to GPU in 1.94319s (17.497GiB/s) | 20260506 15:23:20.739 | INFO | Analyzing chunk frames = [2401280,2401792) | 20260506 15:23:20.739 | INFO | MemInfo: { | 20260506 15:23:20.739 | INFO | "gpu" : " 17 GB", | 20260506 15:23:20.739 | INFO | "huge" : "", | 20260506 15:23:20.739 | INFO | "huge_shared" : " 27 GB", | 20260506 15:23:20.739 | INFO | "malloc" : " 14 GB", | 20260506 15:23:20.739 | INFO | "pinned" : " 29 GB" | 20260506 15:23:20.739 | INFO | } | 20260506 15:23:20.739 | INFO | Perf report for the last 5.13955 seconds: | 20260506 15:23:20.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:23:20.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:20.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:20.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:20.739 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:23:20.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:20.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:20.739 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:23:20.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:20.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:20.739 | INFO | Uploaded 36GiB to GPU in 1.97208s (18.2548GiB/s) | 20260506 15:23:20.739 | INFO | Downloaded 34GiB to GPU in 1.9478s (17.4556GiB/s) | 20260506 15:23:25.879 | INFO | Analyzing chunk frames = [2401792,2402304) | 20260506 15:23:25.879 | INFO | MemInfo: { | 20260506 15:23:25.879 | INFO | "gpu" : " 17 GB", | 20260506 15:23:25.879 | INFO | "huge" : "", | 20260506 15:23:25.879 | INFO | "huge_shared" : " 27 GB", | 20260506 15:23:25.879 | INFO | "malloc" : " 14 GB", | 20260506 15:23:25.879 | INFO | "pinned" : " 29 GB" | 20260506 15:23:25.879 | INFO | } | 20260506 15:23:25.879 | INFO | Perf report for the last 5.14006 seconds: | 20260506 15:23:25.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:23:25.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:25.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:25.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:25.879 | INFO | ANALYSIS : 60.5%, 3.0, 93.9ms, 0 | 20260506 15:23:25.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:25.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:25.879 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:23:25.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:25.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:25.879 | INFO | Uploaded 36GiB to GPU in 1.99696s (18.0274GiB/s) | 20260506 15:23:25.880 | INFO | Downloaded 34GiB to GPU in 1.91389s (17.7649GiB/s) | 20260506 15:23:31.018 | INFO | Analyzing chunk frames = [2402304,2402816) | 20260506 15:23:31.018 | INFO | MemInfo: { | 20260506 15:23:31.018 | INFO | "gpu" : " 17 GB", | 20260506 15:23:31.018 | INFO | "huge" : "", | 20260506 15:23:31.018 | INFO | "huge_shared" : " 27 GB", | 20260506 15:23:31.018 | INFO | "malloc" : " 14 GB", | 20260506 15:23:31.018 | INFO | "pinned" : " 29 GB" | 20260506 15:23:31.018 | INFO | } | 20260506 15:23:31.018 | INFO | Perf report for the last 5.13897 seconds: | 20260506 15:23:31.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:23:31.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:31.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:31.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:31.018 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 15:23:31.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:31.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:31.018 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:23:31.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:31.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:31.018 | INFO | Uploaded 36GiB to GPU in 1.99193s (18.0729GiB/s) | 20260506 15:23:31.019 | INFO | Downloaded 34GiB to GPU in 1.92646s (17.6489GiB/s) | 20260506 15:23:36.157 | INFO | Analyzing chunk frames = [2402816,2403328) | 20260506 15:23:36.157 | INFO | MemInfo: { | 20260506 15:23:36.157 | INFO | "gpu" : " 17 GB", | 20260506 15:23:36.157 | INFO | "huge" : "", | 20260506 15:23:36.157 | INFO | "huge_shared" : " 27 GB", | 20260506 15:23:36.157 | INFO | "malloc" : " 14 GB", | 20260506 15:23:36.157 | INFO | "pinned" : " 29 GB" | 20260506 15:23:36.157 | INFO | } | 20260506 15:23:36.157 | INFO | Perf report for the last 5.13898 seconds: | 20260506 15:23:36.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:23:36.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:36.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:36.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:36.157 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 15:23:36.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:36.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:36.157 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:23:36.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:36.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:36.157 | INFO | Uploaded 36GiB to GPU in 2.06861s (17.403GiB/s) | 20260506 15:23:36.158 | INFO | Downloaded 34GiB to GPU in 1.96856s (17.2715GiB/s) | 20260506 15:23:41.297 | INFO | Analyzing chunk frames = [2403328,2403840) | 20260506 15:23:41.297 | INFO | MemInfo: { | 20260506 15:23:41.297 | INFO | "gpu" : " 17 GB", | 20260506 15:23:41.297 | INFO | "huge" : "", | 20260506 15:23:41.297 | INFO | "huge_shared" : " 27 GB", | 20260506 15:23:41.297 | INFO | "malloc" : " 14 GB", | 20260506 15:23:41.297 | INFO | "pinned" : " 29 GB" | 20260506 15:23:41.297 | INFO | } | 20260506 15:23:41.297 | INFO | Perf report for the last 5.13986 seconds: | 20260506 15:23:41.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:23:41.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:41.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:41.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:41.297 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:23:41.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:41.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:41.297 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:23:41.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:41.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:41.297 | INFO | Uploaded 36GiB to GPU in 1.99146s (18.0772GiB/s) | 20260506 15:23:41.297 | INFO | Downloaded 34GiB to GPU in 2.00675s (16.9428GiB/s) | 20260506 15:23:46.436 | INFO | Analyzing chunk frames = [2403840,2404352) | 20260506 15:23:46.436 | INFO | MemInfo: { | 20260506 15:23:46.436 | INFO | "gpu" : " 17 GB", | 20260506 15:23:46.436 | INFO | "huge" : "", | 20260506 15:23:46.436 | INFO | "huge_shared" : " 27 GB", | 20260506 15:23:46.436 | INFO | "malloc" : " 14 GB", | 20260506 15:23:46.436 | INFO | "pinned" : " 29 GB" | 20260506 15:23:46.436 | INFO | } | 20260506 15:23:46.436 | INFO | Perf report for the last 5.13886 seconds: | 20260506 15:23:46.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:23:46.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:46.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:46.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:46.436 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 15:23:46.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:46.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:46.436 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:23:46.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:46.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:46.436 | INFO | Uploaded 36GiB to GPU in 1.96067s (18.3611GiB/s) | 20260506 15:23:46.436 | INFO | Downloaded 34GiB to GPU in 1.98725s (17.1091GiB/s) | 20260506 15:23:46.555 | INFO | Starting a round of Baz IO | 20260506 15:23:49.695 | INFO | Populating RTMetrics report | 20260506 15:23:49.696 | INFO | RTMetrics report populated | 20260506 15:23:49.696 | INFO | Opening tmp RTMetrics json file | 20260506 15:23:49.696 | INFO | Serializing RTMetrics to json | 20260506 15:23:49.698 | INFO | Serialization done, swapping tmp file now... | 20260506 15:23:49.698 | INFO | RTMetrics done | 20260506 15:23:50.039 | INFO | Finished a round of Baz IO: Wrote 2.105 GiB in 3.484 seconds | 20260506 15:23:50.039 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.26% / 0.01% / 62.73% | 20260506 15:23:50.039 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.09485% / 1.58946e-05% / 5.5631e-05% | 20260506 15:23:50.039 | INFO | Baz IO overall: | 20260506 15:23:50.039 | INFO | BytesWritten : 1252958846558 | 20260506 15:23:50.039 | INFO | NumEvents : 606363790364 | 20260506 15:23:51.576 | INFO | Analyzing chunk frames = [2404352,2404864) | 20260506 15:23:51.576 | INFO | MemInfo: { | 20260506 15:23:51.576 | INFO | "gpu" : " 17 GB", | 20260506 15:23:51.576 | INFO | "huge" : "", | 20260506 15:23:51.576 | INFO | "huge_shared" : " 27 GB", | 20260506 15:23:51.576 | INFO | "malloc" : " 14 GB", | 20260506 15:23:51.576 | INFO | "pinned" : " 29 GB" | 20260506 15:23:51.576 | INFO | } | 20260506 15:23:51.576 | INFO | Perf report for the last 5.13991 seconds: | 20260506 15:23:51.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:23:51.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:51.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:51.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:51.576 | INFO | ANALYSIS : 62.8%, 2.9, 95.2ms, 0 | 20260506 15:23:51.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:51.576 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 15:23:51.576 | INFO | PRE_HQ : 60.8%, 1.9, 59.6ms, 0 | 20260506 15:23:51.576 | INFO | BAZWRITER : 67.4%, 4.8, 165.7ms, 0 | 20260506 15:23:51.576 | INFO | BAZ_SAMPLING : 2.9%, 1.0, 1.5ms, 0 | 20260506 15:23:51.576 | INFO | Uploaded 36GiB to GPU in 2.05985s (17.477GiB/s) | 20260506 15:23:51.576 | INFO | Downloaded 38GiB to GPU in 2.22005s (17.1167GiB/s) | 20260506 15:23:56.715 | INFO | Analyzing chunk frames = [2404864,2405376) | 20260506 15:23:56.715 | INFO | MemInfo: { | 20260506 15:23:56.715 | INFO | "gpu" : " 17 GB", | 20260506 15:23:56.715 | INFO | "huge" : "", | 20260506 15:23:56.715 | INFO | "huge_shared" : " 27 GB", | 20260506 15:23:56.715 | INFO | "malloc" : " 14 GB", | 20260506 15:23:56.715 | INFO | "pinned" : " 29 GB" | 20260506 15:23:56.715 | INFO | } | 20260506 15:23:56.715 | INFO | Perf report for the last 5.13931 seconds: | 20260506 15:23:56.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:23:56.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:56.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:56.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:56.715 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:23:56.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:56.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:56.715 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:23:56.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:56.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:23:56.715 | INFO | Uploaded 36GiB to GPU in 1.99436s (18.0509GiB/s) | 20260506 15:23:56.715 | INFO | Downloaded 34GiB to GPU in 1.95626s (17.3801GiB/s) | 20260506 15:24:01.854 | INFO | Analyzing chunk frames = [2405376,2405888) | 20260506 15:24:01.854 | INFO | MemInfo: { | 20260506 15:24:01.854 | INFO | "gpu" : " 17 GB", | 20260506 15:24:01.854 | INFO | "huge" : "", | 20260506 15:24:01.854 | INFO | "huge_shared" : " 27 GB", | 20260506 15:24:01.854 | INFO | "malloc" : " 14 GB", | 20260506 15:24:01.854 | INFO | "pinned" : " 29 GB" | 20260506 15:24:01.854 | INFO | } | 20260506 15:24:01.854 | INFO | Perf report for the last 5.13856 seconds: | 20260506 15:24:01.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:24:01.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:01.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:01.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:01.854 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:24:01.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:01.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:01.854 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:24:01.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:01.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:01.854 | INFO | Uploaded 36GiB to GPU in 1.96944s (18.2793GiB/s) | 20260506 15:24:01.854 | INFO | Downloaded 34GiB to GPU in 1.94485s (17.4821GiB/s) | 20260506 15:24:06.993 | INFO | Analyzing chunk frames = [2405888,2406400) | 20260506 15:24:06.993 | INFO | MemInfo: { | 20260506 15:24:06.993 | INFO | "gpu" : " 17 GB", | 20260506 15:24:06.993 | INFO | "huge" : "", | 20260506 15:24:06.993 | INFO | "huge_shared" : " 27 GB", | 20260506 15:24:06.993 | INFO | "malloc" : " 14 GB", | 20260506 15:24:06.993 | INFO | "pinned" : " 29 GB" | 20260506 15:24:06.993 | INFO | } | 20260506 15:24:06.993 | INFO | Perf report for the last 5.1392 seconds: | 20260506 15:24:06.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:24:06.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:06.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:06.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:06.993 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 15:24:06.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:06.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:06.993 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:24:06.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:06.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:06.993 | INFO | Uploaded 36GiB to GPU in 2.01209s (17.8919GiB/s) | 20260506 15:24:06.993 | INFO | Downloaded 34GiB to GPU in 1.91358s (17.7677GiB/s) | 20260506 15:24:12.132 | INFO | Analyzing chunk frames = [2406400,2406912) | 20260506 15:24:12.132 | INFO | MemInfo: { | 20260506 15:24:12.132 | INFO | "gpu" : " 17 GB", | 20260506 15:24:12.132 | INFO | "huge" : "", | 20260506 15:24:12.132 | INFO | "huge_shared" : " 27 GB", | 20260506 15:24:12.132 | INFO | "malloc" : " 14 GB", | 20260506 15:24:12.132 | INFO | "pinned" : " 29 GB" | 20260506 15:24:12.132 | INFO | } | 20260506 15:24:12.133 | INFO | Perf report for the last 5.13946 seconds: | 20260506 15:24:12.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:24:12.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:12.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:12.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:12.133 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 15:24:12.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:12.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:12.133 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:24:12.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:12.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:12.133 | INFO | Uploaded 36GiB to GPU in 2.01332s (17.8809GiB/s) | 20260506 15:24:12.133 | INFO | Downloaded 34GiB to GPU in 1.94615s (17.4704GiB/s) | 20260506 15:24:17.272 | INFO | Analyzing chunk frames = [2406912,2407424) | 20260506 15:24:17.272 | INFO | MemInfo: { | 20260506 15:24:17.272 | INFO | "gpu" : " 17 GB", | 20260506 15:24:17.272 | INFO | "huge" : "", | 20260506 15:24:17.272 | INFO | "huge_shared" : " 27 GB", | 20260506 15:24:17.272 | INFO | "malloc" : " 14 GB", | 20260506 15:24:17.272 | INFO | "pinned" : " 29 GB" | 20260506 15:24:17.272 | INFO | } | 20260506 15:24:17.272 | INFO | Perf report for the last 5.13961 seconds: | 20260506 15:24:17.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:24:17.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:17.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:17.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:17.272 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 15:24:17.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:17.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:17.272 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 15:24:17.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:17.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:17.272 | INFO | Uploaded 36GiB to GPU in 2.04241s (17.6263GiB/s) | 20260506 15:24:17.272 | INFO | Downloaded 34GiB to GPU in 1.93118s (17.6058GiB/s) | 20260506 15:24:22.412 | INFO | Analyzing chunk frames = [2407424,2407936) | 20260506 15:24:22.412 | INFO | MemInfo: { | 20260506 15:24:22.412 | INFO | "gpu" : " 17 GB", | 20260506 15:24:22.412 | INFO | "huge" : "", | 20260506 15:24:22.412 | INFO | "huge_shared" : " 27 GB", | 20260506 15:24:22.412 | INFO | "malloc" : " 14 GB", | 20260506 15:24:22.412 | INFO | "pinned" : " 29 GB" | 20260506 15:24:22.412 | INFO | } | 20260506 15:24:22.412 | INFO | Perf report for the last 5.14016 seconds: | 20260506 15:24:22.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:24:22.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:22.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:22.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:22.412 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:24:22.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:22.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:22.412 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:24:22.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:22.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:22.412 | INFO | Uploaded 36GiB to GPU in 1.9895s (18.095GiB/s) | 20260506 15:24:22.413 | INFO | Downloaded 34GiB to GPU in 1.98402s (17.137GiB/s) | 20260506 15:24:27.550 | INFO | Analyzing chunk frames = [2407936,2408448) | 20260506 15:24:27.550 | INFO | MemInfo: { | 20260506 15:24:27.550 | INFO | "gpu" : " 17 GB", | 20260506 15:24:27.550 | INFO | "huge" : "", | 20260506 15:24:27.550 | INFO | "huge_shared" : " 27 GB", | 20260506 15:24:27.550 | INFO | "malloc" : " 14 GB", | 20260506 15:24:27.550 | INFO | "pinned" : " 29 GB" | 20260506 15:24:27.550 | INFO | } | 20260506 15:24:27.550 | INFO | Perf report for the last 5.1381 seconds: | 20260506 15:24:27.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:24:27.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:27.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:27.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:27.550 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:24:27.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:27.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:27.550 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:24:27.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:27.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:27.551 | INFO | Uploaded 36GiB to GPU in 2.01297s (17.884GiB/s) | 20260506 15:24:27.551 | INFO | Downloaded 34GiB to GPU in 1.98615s (17.1185GiB/s) | 20260506 15:24:27.551 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:24:27.551 | INFO | Total mem usage: 17.4 GiB | 20260506 15:24:27.551 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:24:27.551 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -13%) | 20260506 15:24:27.551 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:24:27.551 | INFO | High Water Breakdown: | 20260506 15:24:27.551 | INFO | 61% : 9.4 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:24:27.551 | INFO | 38.8% : 5.98 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:24:27.551 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:24:27.551 | INFO | ------------------------------------------ | 20260506 15:24:27.551 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:24:27.551 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:24:27.551 | INFO | Total mem usage: 26.5 GiB | 20260506 15:24:27.551 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:24:27.551 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:24:27.551 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:24:27.551 | INFO | High Water Breakdown: | 20260506 15:24:27.551 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:24:27.551 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:24:27.551 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:24:27.551 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:24:27.551 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:24:27.551 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:24:27.551 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:24:27.551 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:24:27.551 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:24:27.551 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:24:27.551 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:24:27.551 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:24:27.551 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:24:27.551 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:24:27.551 | INFO | ------------------------------------------ | 20260506 15:24:27.551 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:24:27.551 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:24:27.551 | INFO | Total mem usage: 25.6 GiB | 20260506 15:24:27.551 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:24:27.551 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:24:27.551 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:24:27.551 | INFO | High Water Breakdown: | 20260506 15:24:27.551 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:24:27.551 | INFO | ------------------------------------------ | 20260506 15:24:27.551 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:24:27.551 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:24:27.551 | INFO | Total mem usage: 15.7 GiB | 20260506 15:24:27.551 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:24:27.551 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:24:27.551 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 15:24:27.551 | INFO | High Water Breakdown: | 20260506 15:24:27.551 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:24:27.551 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:24:27.551 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:24:27.551 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:24:27.552 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:24:27.552 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:24:27.552 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:24:27.552 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:24:27.552 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:24:27.552 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:24:27.552 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:24:27.552 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:24:27.552 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:24:27.552 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:24:27.552 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:24:27.552 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:24:27.552 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:24:27.552 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:24:27.552 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:24:27.552 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:24:27.552 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:24:27.552 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:24:27.552 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:24:27.552 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:24:27.552 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:24:27.552 | INFO | ------------------------------------------ | 20260506 15:24:27.552 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:24:27.552 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:24:27.552 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:24:27.552 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:24:27.552 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.51e+01, 1.33e+01 / 1.67e+00 | 20260506 15:24:27.552 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.02e-01, 1.05e-02 / 7.20e-03 | 20260506 15:24:27.552 | INFO | PerfReport: 11.59%, Baseline -- 3.64e+00 / 4.35e+00, 5.60e-01 / 3.78e-01 | 20260506 15:24:27.552 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.46e+00, 1.83e-01 / 1.31e-01 | 20260506 15:24:27.552 | INFO | PerfReport: 15.80%, DME -- 3.97e+01 / 4.34e+01, 3.65e+00 / 1.62e+00 | 20260506 15:24:27.552 | INFO | PerfReport: 56.89%, FrameLabeling -- 1.79e+01 / 2.10e+01, 1.48e+00 / 2.01e+00 | 20260506 15:24:27.552 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.32e+00 / 2.89e+00, 4.01e-01 / 2.59e-01 | 20260506 15:24:27.552 | INFO | PerfReport: 3.56%, Metrics -- 1.12e+00 / 1.49e+00, 2.87e-01 / 1.78e-01 | 20260506 15:24:27.552 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.89e-01 / 4.46e-01, 6.06e-02 / 2.50e-02 | 20260506 15:24:27.552 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:24:27.552 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:24:27.552 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:24:27.552 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 15:24:27.552 | INFO | PerfReport: 21.45%, Upload -- 2.09e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260506 15:24:27.552 | INFO | PerfReport: 21.25%, Download -- 2.07e+01 / 0.00e+00, 7.52e+00 / 0.00e+00 | 20260506 15:24:27.552 | INFO | PerfReport: 32.25%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 15:24:27.552 | INFO | PerfReport: 1.39%, LockUpload -- 1.35e+00 / 0.00e+00, 5.61e+00 / 0.00e+00 | 20260506 15:24:27.552 | INFO | PerfReport: 1.25%, LockDownload -- 1.22e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260506 15:24:27.552 | INFO | PerfReport: 22.40%, LockCompute -- 2.18e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 15:24:27.552 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260506 15:24:27.670 | INFO | Starting a round of Baz IO | 20260506 15:24:30.820 | INFO | Populating RTMetrics report | 20260506 15:24:30.820 | INFO | RTMetrics report populated | 20260506 15:24:30.820 | INFO | Opening tmp RTMetrics json file | 20260506 15:24:30.948 | INFO | Serializing RTMetrics to json | 20260506 15:24:30.960 | INFO | Serialization done, swapping tmp file now... | 20260506 15:24:30.977 | INFO | RTMetrics done | 20260506 15:24:31.231 | INFO | Finished a round of Baz IO: Wrote 2.103 GiB in 3.561 seconds | 20260506 15:24:31.231 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.24% / 0.01% / 62.74% | 20260506 15:24:31.232 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.08538% / 1.19209e-05% / 6.75519e-05% | 20260506 15:24:31.232 | INFO | Baz IO overall: | 20260506 15:24:31.232 | INFO | BytesWritten : 1255217232978 | 20260506 15:24:31.232 | INFO | NumEvents : 607456720064 | 20260506 15:24:32.691 | INFO | Analyzing chunk frames = [2408448,2408960) | 20260506 15:24:32.691 | INFO | MemInfo: { | 20260506 15:24:32.691 | INFO | "gpu" : " 17 GB", | 20260506 15:24:32.691 | INFO | "huge" : "", | 20260506 15:24:32.691 | INFO | "huge_shared" : " 27 GB", | 20260506 15:24:32.691 | INFO | "malloc" : " 14 GB", | 20260506 15:24:32.691 | INFO | "pinned" : " 29 GB" | 20260506 15:24:32.691 | INFO | } | 20260506 15:24:32.691 | INFO | Perf report for the last 5.1408 seconds: | 20260506 15:24:32.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:24:32.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:32.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:32.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:32.691 | INFO | ANALYSIS : 63.0%, 2.9, 95.4ms, 0 | 20260506 15:24:32.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:32.691 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260506 15:24:32.691 | INFO | PRE_HQ : 60.6%, 1.9, 59.4ms, 0 | 20260506 15:24:32.691 | INFO | BAZWRITER : 68.8%, 4.8, 169.7ms, 0 | 20260506 15:24:32.691 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.2ms, 0 | 20260506 15:24:32.691 | INFO | Uploaded 36GiB to GPU in 2.06929s (17.3973GiB/s) | 20260506 15:24:32.691 | INFO | Downloaded 38GiB to GPU in 2.21809s (17.1319GiB/s) | 20260506 15:24:37.830 | INFO | Analyzing chunk frames = [2408960,2409472) | 20260506 15:24:37.830 | INFO | MemInfo: { | 20260506 15:24:37.830 | INFO | "gpu" : " 17 GB", | 20260506 15:24:37.830 | INFO | "huge" : "", | 20260506 15:24:37.830 | INFO | "huge_shared" : " 27 GB", | 20260506 15:24:37.830 | INFO | "malloc" : " 14 GB", | 20260506 15:24:37.830 | INFO | "pinned" : " 29 GB" | 20260506 15:24:37.830 | INFO | } | 20260506 15:24:37.830 | INFO | Perf report for the last 5.13884 seconds: | 20260506 15:24:37.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:24:37.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:37.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:37.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:37.830 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 15:24:37.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:37.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:37.830 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:24:37.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:37.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:37.830 | INFO | Uploaded 36GiB to GPU in 2.01171s (17.8952GiB/s) | 20260506 15:24:37.830 | INFO | Downloaded 34GiB to GPU in 1.94569s (17.4745GiB/s) | 20260506 15:24:42.969 | INFO | Analyzing chunk frames = [2409472,2409984) | 20260506 15:24:42.969 | INFO | MemInfo: { | 20260506 15:24:42.969 | INFO | "gpu" : " 17 GB", | 20260506 15:24:42.969 | INFO | "huge" : "", | 20260506 15:24:42.969 | INFO | "huge_shared" : " 27 GB", | 20260506 15:24:42.969 | INFO | "malloc" : " 14 GB", | 20260506 15:24:42.969 | INFO | "pinned" : " 29 GB" | 20260506 15:24:42.969 | INFO | } | 20260506 15:24:42.969 | INFO | Perf report for the last 5.13882 seconds: | 20260506 15:24:42.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:24:42.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:42.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:42.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:42.969 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:24:42.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:42.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:42.969 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 15:24:42.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:42.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:42.969 | INFO | Uploaded 36GiB to GPU in 1.99085s (18.0827GiB/s) | 20260506 15:24:42.969 | INFO | Downloaded 34GiB to GPU in 1.94434s (17.4866GiB/s) | 20260506 15:24:48.108 | INFO | Analyzing chunk frames = [2409984,2410496) | 20260506 15:24:48.108 | INFO | MemInfo: { | 20260506 15:24:48.108 | INFO | "gpu" : " 17 GB", | 20260506 15:24:48.108 | INFO | "huge" : "", | 20260506 15:24:48.108 | INFO | "huge_shared" : " 27 GB", | 20260506 15:24:48.108 | INFO | "malloc" : " 14 GB", | 20260506 15:24:48.108 | INFO | "pinned" : " 29 GB" | 20260506 15:24:48.108 | INFO | } | 20260506 15:24:48.108 | INFO | Perf report for the last 5.13919 seconds: | 20260506 15:24:48.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:24:48.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:48.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:48.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:48.108 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 15:24:48.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:48.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:48.108 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:24:48.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:48.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:48.108 | INFO | Uploaded 36GiB to GPU in 2.02263s (17.7986GiB/s) | 20260506 15:24:48.108 | INFO | Downloaded 34GiB to GPU in 1.91828s (17.7242GiB/s) | 20260506 15:24:53.247 | INFO | Analyzing chunk frames = [2410496,2411008) | 20260506 15:24:53.247 | INFO | MemInfo: { | 20260506 15:24:53.247 | INFO | "gpu" : " 17 GB", | 20260506 15:24:53.247 | INFO | "huge" : "", | 20260506 15:24:53.247 | INFO | "huge_shared" : " 27 GB", | 20260506 15:24:53.247 | INFO | "malloc" : " 14 GB", | 20260506 15:24:53.247 | INFO | "pinned" : " 29 GB" | 20260506 15:24:53.247 | INFO | } | 20260506 15:24:53.247 | INFO | Perf report for the last 5.13891 seconds: | 20260506 15:24:53.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:24:53.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:53.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:53.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:53.247 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 15:24:53.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:53.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:53.247 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:24:53.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:53.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:53.247 | INFO | Uploaded 36GiB to GPU in 2.03062s (17.7286GiB/s) | 20260506 15:24:53.247 | INFO | Downloaded 34GiB to GPU in 1.93602s (17.5618GiB/s) | 20260506 15:24:58.387 | INFO | Analyzing chunk frames = [2411008,2411520) | 20260506 15:24:58.387 | INFO | MemInfo: { | 20260506 15:24:58.387 | INFO | "gpu" : " 17 GB", | 20260506 15:24:58.387 | INFO | "huge" : "", | 20260506 15:24:58.387 | INFO | "huge_shared" : " 27 GB", | 20260506 15:24:58.387 | INFO | "malloc" : " 14 GB", | 20260506 15:24:58.387 | INFO | "pinned" : " 29 GB" | 20260506 15:24:58.387 | INFO | } | 20260506 15:24:58.387 | INFO | Perf report for the last 5.14036 seconds: | 20260506 15:24:58.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:24:58.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:58.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:58.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:58.387 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 15:24:58.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:58.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:58.387 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260506 15:24:58.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:58.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:24:58.388 | INFO | Uploaded 36GiB to GPU in 2.07234s (17.3717GiB/s) | 20260506 15:24:58.388 | INFO | Downloaded 34GiB to GPU in 1.95037s (17.4326GiB/s) | 20260506 15:25:03.527 | INFO | Analyzing chunk frames = [2411520,2412032) | 20260506 15:25:03.527 | INFO | MemInfo: { | 20260506 15:25:03.527 | INFO | "gpu" : " 17 GB", | 20260506 15:25:03.527 | INFO | "huge" : "", | 20260506 15:25:03.527 | INFO | "huge_shared" : " 27 GB", | 20260506 15:25:03.527 | INFO | "malloc" : " 14 GB", | 20260506 15:25:03.527 | INFO | "pinned" : " 29 GB" | 20260506 15:25:03.527 | INFO | } | 20260506 15:25:03.527 | INFO | Perf report for the last 5.1399 seconds: | 20260506 15:25:03.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:25:03.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:03.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:03.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:03.527 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:25:03.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:03.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:03.527 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 15:25:03.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:03.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:03.528 | INFO | Uploaded 36GiB to GPU in 2.02157s (17.8079GiB/s) | 20260506 15:25:03.528 | INFO | Downloaded 34GiB to GPU in 1.99158s (17.0719GiB/s) | 20260506 15:25:08.667 | INFO | Analyzing chunk frames = [2412032,2412544) | 20260506 15:25:08.667 | INFO | MemInfo: { | 20260506 15:25:08.667 | INFO | "gpu" : " 17 GB", | 20260506 15:25:08.667 | INFO | "huge" : "", | 20260506 15:25:08.667 | INFO | "huge_shared" : " 27 GB", | 20260506 15:25:08.667 | INFO | "malloc" : " 14 GB", | 20260506 15:25:08.667 | INFO | "pinned" : " 29 GB" | 20260506 15:25:08.667 | INFO | } | 20260506 15:25:08.667 | INFO | Perf report for the last 5.1396 seconds: | 20260506 15:25:08.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:25:08.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:08.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:08.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:08.667 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 15:25:08.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:08.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:08.667 | INFO | PRE_HQ : 32.4%, 1.0, 16.8ms, 0 | 20260506 15:25:08.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:08.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:08.667 | INFO | Uploaded 36GiB to GPU in 2.02866s (17.7457GiB/s) | 20260506 15:25:08.667 | INFO | Downloaded 34GiB to GPU in 1.97556s (17.2103GiB/s) | 20260506 15:25:08.797 | INFO | Starting a round of Baz IO | 20260506 15:25:11.959 | INFO | Populating RTMetrics report | 20260506 15:25:11.959 | INFO | RTMetrics report populated | 20260506 15:25:11.959 | INFO | Opening tmp RTMetrics json file | 20260506 15:25:11.960 | INFO | Serializing RTMetrics to json | 20260506 15:25:11.962 | INFO | Serialization done, swapping tmp file now... | 20260506 15:25:11.963 | INFO | RTMetrics done | 20260506 15:25:12.104 | INFO | Finished a round of Baz IO: Wrote 2.102 GiB in 3.306 seconds | 20260506 15:25:12.104 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.23% / 0.01% / 62.76% | 20260506 15:25:12.104 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.07729% / 1.98682e-05% / 7.15256e-05% | 20260506 15:25:12.104 | INFO | Baz IO overall: | 20260506 15:25:12.104 | INFO | BytesWritten : 1257474205228 | 20260506 15:25:12.104 | INFO | NumEvents : 608548630984 | 20260506 15:25:13.805 | INFO | Analyzing chunk frames = [2412544,2413056) | 20260506 15:25:13.805 | INFO | MemInfo: { | 20260506 15:25:13.805 | INFO | "gpu" : " 17 GB", | 20260506 15:25:13.805 | INFO | "huge" : "", | 20260506 15:25:13.805 | INFO | "huge_shared" : " 27 GB", | 20260506 15:25:13.805 | INFO | "malloc" : " 14 GB", | 20260506 15:25:13.805 | INFO | "pinned" : " 29 GB" | 20260506 15:25:13.805 | INFO | } | 20260506 15:25:13.805 | INFO | Perf report for the last 5.1379 seconds: | 20260506 15:25:13.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:25:13.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:13.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:13.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:13.805 | INFO | ANALYSIS : 63.2%, 2.8, 93.7ms, 0 | 20260506 15:25:13.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:13.805 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260506 15:25:13.805 | INFO | PRE_HQ : 61.2%, 2.0, 63.7ms, 0 | 20260506 15:25:13.805 | INFO | BAZWRITER : 63.9%, 4.8, 156.0ms, 0 | 20260506 15:25:13.805 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 15:25:13.805 | INFO | Uploaded 36GiB to GPU in 2.03935s (17.6526GiB/s) | 20260506 15:25:13.805 | INFO | Downloaded 38GiB to GPU in 2.19234s (17.3331GiB/s) | 20260506 15:25:18.944 | INFO | Analyzing chunk frames = [2413056,2413568) | 20260506 15:25:18.944 | INFO | MemInfo: { | 20260506 15:25:18.944 | INFO | "gpu" : " 17 GB", | 20260506 15:25:18.944 | INFO | "huge" : "", | 20260506 15:25:18.944 | INFO | "huge_shared" : " 27 GB", | 20260506 15:25:18.944 | INFO | "malloc" : " 14 GB", | 20260506 15:25:18.944 | INFO | "pinned" : " 29 GB" | 20260506 15:25:18.944 | INFO | } | 20260506 15:25:18.944 | INFO | Perf report for the last 5.13933 seconds: | 20260506 15:25:18.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:25:18.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:18.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:18.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:18.944 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:25:18.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:18.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:18.944 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:25:18.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:18.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:18.944 | INFO | Uploaded 36GiB to GPU in 1.97752s (18.2047GiB/s) | 20260506 15:25:18.945 | INFO | Downloaded 34GiB to GPU in 1.93896s (17.5352GiB/s) | 20260506 15:25:24.083 | INFO | Analyzing chunk frames = [2413568,2414080) | 20260506 15:25:24.083 | INFO | MemInfo: { | 20260506 15:25:24.083 | INFO | "gpu" : " 17 GB", | 20260506 15:25:24.083 | INFO | "huge" : "", | 20260506 15:25:24.083 | INFO | "huge_shared" : " 27 GB", | 20260506 15:25:24.083 | INFO | "malloc" : " 14 GB", | 20260506 15:25:24.083 | INFO | "pinned" : " 29 GB" | 20260506 15:25:24.083 | INFO | } | 20260506 15:25:24.084 | INFO | Perf report for the last 5.1393 seconds: | 20260506 15:25:24.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:25:24.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:24.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:24.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:24.084 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:25:24.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:24.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:24.084 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:25:24.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:24.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:24.084 | INFO | Uploaded 36GiB to GPU in 1.97466s (18.231GiB/s) | 20260506 15:25:24.084 | INFO | Downloaded 34GiB to GPU in 1.94371s (17.4923GiB/s) | 20260506 15:25:29.223 | INFO | Analyzing chunk frames = [2414080,2414592) | 20260506 15:25:29.223 | INFO | MemInfo: { | 20260506 15:25:29.223 | INFO | "gpu" : " 17 GB", | 20260506 15:25:29.223 | INFO | "huge" : "", | 20260506 15:25:29.223 | INFO | "huge_shared" : " 27 GB", | 20260506 15:25:29.223 | INFO | "malloc" : " 14 GB", | 20260506 15:25:29.223 | INFO | "pinned" : " 29 GB" | 20260506 15:25:29.223 | INFO | } | 20260506 15:25:29.223 | INFO | Perf report for the last 5.13944 seconds: | 20260506 15:25:29.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:25:29.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:29.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:29.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:29.223 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 15:25:29.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:29.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:29.223 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:25:29.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:29.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:29.223 | INFO | Uploaded 36GiB to GPU in 2.01199s (17.8927GiB/s) | 20260506 15:25:29.223 | INFO | Downloaded 34GiB to GPU in 1.91546s (17.7503GiB/s) | 20260506 15:25:34.362 | INFO | Analyzing chunk frames = [2414592,2415104) | 20260506 15:25:34.362 | INFO | MemInfo: { | 20260506 15:25:34.362 | INFO | "gpu" : " 17 GB", | 20260506 15:25:34.362 | INFO | "huge" : "", | 20260506 15:25:34.362 | INFO | "huge_shared" : " 27 GB", | 20260506 15:25:34.362 | INFO | "malloc" : " 14 GB", | 20260506 15:25:34.362 | INFO | "pinned" : " 29 GB" | 20260506 15:25:34.362 | INFO | } | 20260506 15:25:34.362 | INFO | Perf report for the last 5.13917 seconds: | 20260506 15:25:34.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:25:34.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:34.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:34.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:34.362 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 15:25:34.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:34.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:34.362 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:25:34.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:34.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:34.362 | INFO | Uploaded 36GiB to GPU in 1.99055s (18.0854GiB/s) | 20260506 15:25:34.362 | INFO | Downloaded 34GiB to GPU in 1.93446s (17.576GiB/s) | 20260506 15:25:39.502 | INFO | Analyzing chunk frames = [2415104,2415616) | 20260506 15:25:39.502 | INFO | MemInfo: { | 20260506 15:25:39.502 | INFO | "gpu" : " 17 GB", | 20260506 15:25:39.502 | INFO | "huge" : "", | 20260506 15:25:39.502 | INFO | "huge_shared" : " 27 GB", | 20260506 15:25:39.502 | INFO | "malloc" : " 14 GB", | 20260506 15:25:39.502 | INFO | "pinned" : " 29 GB" | 20260506 15:25:39.502 | INFO | } | 20260506 15:25:39.502 | INFO | Perf report for the last 5.14013 seconds: | 20260506 15:25:39.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:25:39.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:39.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:39.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:39.502 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 15:25:39.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:39.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:39.502 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:25:39.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:39.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:39.503 | INFO | Uploaded 36GiB to GPU in 2.03614s (17.6806GiB/s) | 20260506 15:25:39.503 | INFO | Downloaded 34GiB to GPU in 1.9448s (17.4825GiB/s) | 20260506 15:25:44.641 | INFO | Analyzing chunk frames = [2415616,2416128) | 20260506 15:25:44.641 | INFO | MemInfo: { | 20260506 15:25:44.641 | INFO | "gpu" : " 17 GB", | 20260506 15:25:44.641 | INFO | "huge" : "", | 20260506 15:25:44.641 | INFO | "huge_shared" : " 27 GB", | 20260506 15:25:44.641 | INFO | "malloc" : " 14 GB", | 20260506 15:25:44.641 | INFO | "pinned" : " 29 GB" | 20260506 15:25:44.641 | INFO | } | 20260506 15:25:44.641 | INFO | Perf report for the last 5.13891 seconds: | 20260506 15:25:44.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:25:44.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:44.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:44.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:44.641 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 15:25:44.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:44.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:44.641 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 15:25:44.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:44.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:44.641 | INFO | Uploaded 36GiB to GPU in 2.00014s (17.9988GiB/s) | 20260506 15:25:44.641 | INFO | Downloaded 34GiB to GPU in 1.98807s (17.102GiB/s) | 20260506 15:25:49.781 | INFO | Analyzing chunk frames = [2416128,2416640) | 20260506 15:25:49.781 | INFO | MemInfo: { | 20260506 15:25:49.781 | INFO | "gpu" : " 17 GB", | 20260506 15:25:49.781 | INFO | "huge" : "", | 20260506 15:25:49.781 | INFO | "huge_shared" : " 27 GB", | 20260506 15:25:49.781 | INFO | "malloc" : " 14 GB", | 20260506 15:25:49.781 | INFO | "pinned" : " 29 GB" | 20260506 15:25:49.781 | INFO | } | 20260506 15:25:49.781 | INFO | Perf report for the last 5.13978 seconds: | 20260506 15:25:49.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:25:49.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:49.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:49.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:49.781 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:25:49.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:49.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:49.781 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 15:25:49.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:49.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:49.781 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:25:49.781 | INFO | Total mem usage: 17.4 GiB | 20260506 15:25:49.781 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:25:49.781 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -13.1%) | 20260506 15:25:49.781 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:25:49.781 | INFO | High Water Breakdown: | 20260506 15:25:49.781 | INFO | 61% : 9.39 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:25:49.781 | INFO | 38.9% : 5.98 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:25:49.781 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:25:49.781 | INFO | ------------------------------------------ | 20260506 15:25:49.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:25:49.781 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:25:49.781 | INFO | Total mem usage: 26.5 GiB | 20260506 15:25:49.781 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:25:49.781 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:25:49.781 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:25:49.781 | INFO | High Water Breakdown: | 20260506 15:25:49.781 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:25:49.781 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:25:49.781 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:25:49.781 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:25:49.781 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:25:49.781 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:25:49.781 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:25:49.781 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:25:49.781 | INFO | Uploaded 36GiB to GPU in 2.04106s (17.6379GiB/s) | 20260506 15:25:49.782 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:25:49.782 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:25:49.782 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:25:49.782 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:25:49.782 | INFO | Downloaded 34GiB to GPU in 1.9812s (17.1613GiB/s) | 20260506 15:25:49.782 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:25:49.782 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:25:49.782 | INFO | ------------------------------------------ | 20260506 15:25:49.782 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:25:49.782 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:25:49.782 | INFO | Total mem usage: 25.6 GiB | 20260506 15:25:49.782 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:25:49.782 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:25:49.782 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:25:49.782 | INFO | High Water Breakdown: | 20260506 15:25:49.782 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:25:49.782 | INFO | ------------------------------------------ | 20260506 15:25:49.782 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:25:49.782 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:25:49.782 | INFO | Total mem usage: 15.7 GiB | 20260506 15:25:49.782 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:25:49.782 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:25:49.782 | INFO | Unused Cache: 818 MiB (Waste: 5.09%) | 20260506 15:25:49.782 | INFO | High Water Breakdown: | 20260506 15:25:49.782 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:25:49.782 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:25:49.782 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:25:49.782 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:25:49.782 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:25:49.782 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:25:49.782 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:25:49.782 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:25:49.782 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:25:49.782 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:25:49.782 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:25:49.782 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:25:49.782 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:25:49.782 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:25:49.782 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:25:49.782 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:25:49.782 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:25:49.782 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:25:49.782 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:25:49.782 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:25:49.782 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:25:49.782 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:25:49.782 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:25:49.782 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:25:49.782 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:25:49.782 | INFO | ------------------------------------------ | 20260506 15:25:49.782 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:25:49.782 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:25:49.782 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:25:49.782 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:25:49.782 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.56e+01, 1.35e+01 / 1.86e+00 | 20260506 15:25:49.782 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.01e-01, 1.07e-02 / 6.95e-03 | 20260506 15:25:49.782 | INFO | PerfReport: 11.57%, Baseline -- 3.64e+00 / 4.38e+00, 5.63e-01 / 4.24e-01 | 20260506 15:25:49.782 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.45e+00, 1.86e-01 / 1.48e-01 | 20260506 15:25:49.782 | INFO | PerfReport: 15.81%, DME -- 3.98e+01 / 4.38e+01, 3.82e+00 / 1.70e+00 | 20260506 15:25:49.782 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.79e+01 / 2.12e+01, 1.52e+00 / 1.72e+00 | 20260506 15:25:49.782 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.31e+00 / 2.85e+00, 3.89e-01 / 2.63e-01 | 20260506 15:25:49.783 | INFO | PerfReport: 3.54%, Metrics -- 1.11e+00 / 1.48e+00, 2.88e-01 / 2.22e-01 | 20260506 15:25:49.783 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.89e-01 / 4.41e-01, 6.06e-02 / 3.06e-02 | 20260506 15:25:49.783 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:25:49.783 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:25:49.783 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:25:49.783 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 15:25:49.783 | INFO | PerfReport: 21.58%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 15:25:49.783 | INFO | PerfReport: 21.18%, Download -- 2.07e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 15:25:49.783 | INFO | PerfReport: 32.28%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 15:25:49.783 | INFO | PerfReport: 1.57%, LockUpload -- 1.53e+00 / 0.00e+00, 6.01e+00 / 0.00e+00 | 20260506 15:25:49.783 | INFO | PerfReport: 1.07%, LockDownload -- 1.05e+00 / 0.00e+00, 2.67e+00 / 0.00e+00 | 20260506 15:25:49.783 | INFO | PerfReport: 22.30%, LockCompute -- 2.17e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260506 15:25:49.783 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260506 15:25:49.902 | INFO | Starting a round of Baz IO | 20260506 15:25:53.053 | INFO | Populating RTMetrics report | 20260506 15:25:53.053 | INFO | RTMetrics report populated | 20260506 15:25:53.053 | INFO | Opening tmp RTMetrics json file | 20260506 15:25:53.055 | INFO | Serializing RTMetrics to json | 20260506 15:25:53.100 | INFO | Serialization done, swapping tmp file now... | 20260506 15:25:53.174 | INFO | RTMetrics done | 20260506 15:25:53.523 | INFO | Finished a round of Baz IO: Wrote 2.101 GiB in 3.621 seconds | 20260506 15:25:53.523 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.22% / 0.01% / 62.77% | 20260506 15:25:53.523 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.07207% / 1.19209e-05% / 7.54992e-05% | 20260506 15:25:53.523 | INFO | Baz IO overall: | 20260506 15:25:53.523 | INFO | BytesWritten : 1259730520349 | 20260506 15:25:53.523 | INFO | NumEvents : 609640075043 | 20260506 15:25:54.921 | INFO | Analyzing chunk frames = [2416640,2417152) | 20260506 15:25:54.921 | INFO | MemInfo: { | 20260506 15:25:54.921 | INFO | "gpu" : " 17 GB", | 20260506 15:25:54.921 | INFO | "huge" : "", | 20260506 15:25:54.921 | INFO | "huge_shared" : " 27 GB", | 20260506 15:25:54.921 | INFO | "malloc" : " 14 GB", | 20260506 15:25:54.921 | INFO | "pinned" : " 29 GB" | 20260506 15:25:54.921 | INFO | } | 20260506 15:25:54.921 | INFO | Perf report for the last 5.1402 seconds: | 20260506 15:25:54.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:25:54.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:54.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:54.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:54.921 | INFO | ANALYSIS : 63.1%, 2.9, 95.8ms, 0 | 20260506 15:25:54.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:25:54.921 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 15:25:54.921 | INFO | PRE_HQ : 60.8%, 1.9, 59.3ms, 0 | 20260506 15:25:54.921 | INFO | BAZWRITER : 69.9%, 4.8, 171.2ms, 0 | 20260506 15:25:54.921 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 15:25:54.921 | INFO | Uploaded 36GiB to GPU in 2.07125s (17.3808GiB/s) | 20260506 15:25:54.922 | INFO | Downloaded 38GiB to GPU in 2.22271s (17.0963GiB/s) | 20260506 15:26:00.060 | INFO | Analyzing chunk frames = [2417152,2417664) | 20260506 15:26:00.060 | INFO | MemInfo: { | 20260506 15:26:00.060 | INFO | "gpu" : " 17 GB", | 20260506 15:26:00.060 | INFO | "huge" : "", | 20260506 15:26:00.060 | INFO | "huge_shared" : " 27 GB", | 20260506 15:26:00.060 | INFO | "malloc" : " 14 GB", | 20260506 15:26:00.060 | INFO | "pinned" : " 29 GB" | 20260506 15:26:00.060 | INFO | } | 20260506 15:26:00.060 | INFO | Perf report for the last 5.13903 seconds: | 20260506 15:26:00.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:26:00.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:00.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:00.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:00.060 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:26:00.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:00.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:00.060 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:26:00.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:00.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:00.060 | INFO | Uploaded 36GiB to GPU in 2.00242s (17.9782GiB/s) | 20260506 15:26:00.061 | INFO | Downloaded 34GiB to GPU in 1.9398s (17.5275GiB/s) | 20260506 15:26:05.198 | INFO | Analyzing chunk frames = [2417664,2418176) | 20260506 15:26:05.198 | INFO | MemInfo: { | 20260506 15:26:05.198 | INFO | "gpu" : " 17 GB", | 20260506 15:26:05.198 | INFO | "huge" : "", | 20260506 15:26:05.198 | INFO | "huge_shared" : " 27 GB", | 20260506 15:26:05.198 | INFO | "malloc" : " 14 GB", | 20260506 15:26:05.198 | INFO | "pinned" : " 29 GB" | 20260506 15:26:05.198 | INFO | } | 20260506 15:26:05.198 | INFO | Perf report for the last 5.13816 seconds: | 20260506 15:26:05.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:26:05.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:05.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:05.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:05.199 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 15:26:05.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:05.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:05.199 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:26:05.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:05.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:05.199 | INFO | Uploaded 36GiB to GPU in 2.00257s (17.9769GiB/s) | 20260506 15:26:05.199 | INFO | Downloaded 34GiB to GPU in 1.93892s (17.5355GiB/s) | 20260506 15:26:10.338 | INFO | Analyzing chunk frames = [2418176,2418688) | 20260506 15:26:10.338 | INFO | MemInfo: { | 20260506 15:26:10.338 | INFO | "gpu" : " 17 GB", | 20260506 15:26:10.338 | INFO | "huge" : "", | 20260506 15:26:10.338 | INFO | "huge_shared" : " 27 GB", | 20260506 15:26:10.338 | INFO | "malloc" : " 14 GB", | 20260506 15:26:10.338 | INFO | "pinned" : " 29 GB" | 20260506 15:26:10.338 | INFO | } | 20260506 15:26:10.338 | INFO | Perf report for the last 5.13955 seconds: | 20260506 15:26:10.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:26:10.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:10.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:10.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:10.338 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 15:26:10.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:10.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:10.338 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:26:10.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:10.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:10.338 | INFO | Uploaded 36GiB to GPU in 2.02683s (17.7617GiB/s) | 20260506 15:26:10.338 | INFO | Downloaded 34GiB to GPU in 1.91669s (17.7389GiB/s) | 20260506 15:26:15.478 | INFO | Analyzing chunk frames = [2418688,2419200) | 20260506 15:26:15.478 | INFO | MemInfo: { | 20260506 15:26:15.478 | INFO | "gpu" : " 17 GB", | 20260506 15:26:15.478 | INFO | "huge" : "", | 20260506 15:26:15.478 | INFO | "huge_shared" : " 27 GB", | 20260506 15:26:15.478 | INFO | "malloc" : " 14 GB", | 20260506 15:26:15.478 | INFO | "pinned" : " 29 GB" | 20260506 15:26:15.478 | INFO | } | 20260506 15:26:15.478 | INFO | Perf report for the last 5.13989 seconds: | 20260506 15:26:15.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:26:15.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:15.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:15.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:15.478 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 15:26:15.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:15.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:15.478 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:26:15.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:15.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:15.478 | INFO | Uploaded 36GiB to GPU in 2.02243s (17.8003GiB/s) | 20260506 15:26:15.478 | INFO | Downloaded 34GiB to GPU in 1.94062s (17.5202GiB/s) | 20260506 15:26:20.618 | INFO | Analyzing chunk frames = [2419200,2419712) | 20260506 15:26:20.618 | INFO | MemInfo: { | 20260506 15:26:20.618 | INFO | "gpu" : " 17 GB", | 20260506 15:26:20.618 | INFO | "huge" : "", | 20260506 15:26:20.618 | INFO | "huge_shared" : " 27 GB", | 20260506 15:26:20.618 | INFO | "malloc" : " 14 GB", | 20260506 15:26:20.618 | INFO | "pinned" : " 29 GB" | 20260506 15:26:20.618 | INFO | } | 20260506 15:26:20.618 | INFO | Perf report for the last 5.1402 seconds: | 20260506 15:26:20.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:26:20.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:20.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:20.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:20.618 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:26:20.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:20.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:20.618 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 15:26:20.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:20.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:20.618 | INFO | Uploaded 36GiB to GPU in 2.0786s (17.3194GiB/s) | 20260506 15:26:20.618 | INFO | Downloaded 34GiB to GPU in 1.97002s (17.2587GiB/s) | 20260506 15:26:25.756 | INFO | Analyzing chunk frames = [2419712,2420224) | 20260506 15:26:25.756 | INFO | MemInfo: { | 20260506 15:26:25.756 | INFO | "gpu" : " 17 GB", | 20260506 15:26:25.756 | INFO | "huge" : "", | 20260506 15:26:25.756 | INFO | "huge_shared" : " 27 GB", | 20260506 15:26:25.756 | INFO | "malloc" : " 14 GB", | 20260506 15:26:25.756 | INFO | "pinned" : " 29 GB" | 20260506 15:26:25.756 | INFO | } | 20260506 15:26:25.757 | INFO | Perf report for the last 5.13839 seconds: | 20260506 15:26:25.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:26:25.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:25.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:25.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:25.757 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 15:26:25.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:25.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:25.757 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:26:25.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:25.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:25.757 | INFO | Uploaded 36GiB to GPU in 2.04278s (17.623GiB/s) | 20260506 15:26:25.757 | INFO | Downloaded 34GiB to GPU in 1.99861s (17.0118GiB/s) | 20260506 15:26:30.895 | INFO | Analyzing chunk frames = [2420224,2420736) | 20260506 15:26:30.895 | INFO | MemInfo: { | 20260506 15:26:30.895 | INFO | "gpu" : " 17 GB", | 20260506 15:26:30.896 | INFO | "huge" : "", | 20260506 15:26:30.896 | INFO | "huge_shared" : " 27 GB", | 20260506 15:26:30.896 | INFO | "malloc" : " 14 GB", | 20260506 15:26:30.896 | INFO | "pinned" : " 29 GB" | 20260506 15:26:30.896 | INFO | } | 20260506 15:26:30.896 | INFO | Perf report for the last 5.13905 seconds: | 20260506 15:26:30.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:26:30.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:30.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:30.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:30.896 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 15:26:30.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:30.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:30.896 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260506 15:26:30.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:30.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:30.896 | INFO | Uploaded 36GiB to GPU in 2.03891s (17.6565GiB/s) | 20260506 15:26:30.896 | INFO | Downloaded 34GiB to GPU in 1.98988s (17.0865GiB/s) | 20260506 15:26:31.010 | INFO | Starting a round of Baz IO | 20260506 15:26:34.156 | INFO | Populating RTMetrics report | 20260506 15:26:34.157 | INFO | RTMetrics report populated | 20260506 15:26:34.157 | INFO | Opening tmp RTMetrics json file | 20260506 15:26:34.157 | INFO | Serializing RTMetrics to json | 20260506 15:26:34.159 | INFO | Serialization done, swapping tmp file now... | 20260506 15:26:34.159 | INFO | RTMetrics done | 20260506 15:26:34.422 | INFO | Finished a round of Baz IO: Wrote 2.103 GiB in 3.412 seconds | 20260506 15:26:34.422 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.21% / 0.01% / 62.78% | 20260506 15:26:34.422 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.07079% / 1.58946e-05% / 7.54992e-05% | 20260506 15:26:34.422 | INFO | Baz IO overall: | 20260506 15:26:34.422 | INFO | BytesWritten : 1261989062158 | 20260506 15:26:34.422 | INFO | NumEvents : 610733066125 | 20260506 15:26:36.034 | INFO | Analyzing chunk frames = [2420736,2421248) | 20260506 15:26:36.034 | INFO | MemInfo: { | 20260506 15:26:36.034 | INFO | "gpu" : " 17 GB", | 20260506 15:26:36.034 | INFO | "huge" : "", | 20260506 15:26:36.034 | INFO | "huge_shared" : " 27 GB", | 20260506 15:26:36.034 | INFO | "malloc" : " 14 GB", | 20260506 15:26:36.034 | INFO | "pinned" : " 29 GB" | 20260506 15:26:36.034 | INFO | } | 20260506 15:26:36.034 | INFO | Perf report for the last 5.13865 seconds: | 20260506 15:26:36.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:26:36.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:36.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:36.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:36.034 | INFO | ANALYSIS : 62.9%, 2.9, 94.8ms, 0 | 20260506 15:26:36.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:36.034 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 15:26:36.034 | INFO | PRE_HQ : 61.0%, 1.9, 59.7ms, 0 | 20260506 15:26:36.034 | INFO | BAZWRITER : 65.9%, 4.8, 162.0ms, 0 | 20260506 15:26:36.034 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 15:26:36.034 | INFO | Uploaded 36GiB to GPU in 2.0913s (17.2142GiB/s) | 20260506 15:26:36.035 | INFO | Downloaded 38GiB to GPU in 2.22198s (17.1019GiB/s) | 20260506 15:26:41.175 | INFO | Analyzing chunk frames = [2421248,2421760) | 20260506 15:26:41.175 | INFO | MemInfo: { | 20260506 15:26:41.175 | INFO | "gpu" : " 17 GB", | 20260506 15:26:41.175 | INFO | "huge" : "", | 20260506 15:26:41.175 | INFO | "huge_shared" : " 27 GB", | 20260506 15:26:41.175 | INFO | "malloc" : " 14 GB", | 20260506 15:26:41.175 | INFO | "pinned" : " 29 GB" | 20260506 15:26:41.175 | INFO | } | 20260506 15:26:41.175 | INFO | Perf report for the last 5.14043 seconds: | 20260506 15:26:41.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:26:41.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:41.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:41.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:41.175 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:26:41.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:41.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:41.175 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:26:41.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:41.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:41.175 | INFO | Uploaded 36GiB to GPU in 1.99733s (18.0241GiB/s) | 20260506 15:26:41.175 | INFO | Downloaded 34GiB to GPU in 1.94507s (17.4801GiB/s) | 20260506 15:26:46.313 | INFO | Analyzing chunk frames = [2421760,2422272) | 20260506 15:26:46.313 | INFO | MemInfo: { | 20260506 15:26:46.313 | INFO | "gpu" : " 17 GB", | 20260506 15:26:46.313 | INFO | "huge" : "", | 20260506 15:26:46.313 | INFO | "huge_shared" : " 27 GB", | 20260506 15:26:46.313 | INFO | "malloc" : " 14 GB", | 20260506 15:26:46.313 | INFO | "pinned" : " 29 GB" | 20260506 15:26:46.313 | INFO | } | 20260506 15:26:46.313 | INFO | Perf report for the last 5.1384 seconds: | 20260506 15:26:46.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:26:46.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:46.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:46.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:46.313 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:26:46.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:46.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:46.313 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:26:46.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:46.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:46.313 | INFO | Uploaded 36GiB to GPU in 1.98891s (18.1003GiB/s) | 20260506 15:26:46.313 | INFO | Downloaded 34GiB to GPU in 1.93739s (17.5494GiB/s) | 20260506 15:26:51.453 | INFO | Analyzing chunk frames = [2422272,2422784) | 20260506 15:26:51.453 | INFO | MemInfo: { | 20260506 15:26:51.453 | INFO | "gpu" : " 17 GB", | 20260506 15:26:51.453 | INFO | "huge" : "", | 20260506 15:26:51.453 | INFO | "huge_shared" : " 27 GB", | 20260506 15:26:51.453 | INFO | "malloc" : " 14 GB", | 20260506 15:26:51.453 | INFO | "pinned" : " 29 GB" | 20260506 15:26:51.453 | INFO | } | 20260506 15:26:51.453 | INFO | Perf report for the last 5.13973 seconds: | 20260506 15:26:51.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:26:51.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:51.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:51.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:51.453 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:26:51.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:51.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:51.453 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:26:51.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:51.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:51.453 | INFO | Uploaded 36GiB to GPU in 2.02135s (17.8098GiB/s) | 20260506 15:26:51.453 | INFO | Downloaded 34GiB to GPU in 1.91465s (17.7578GiB/s) | 20260506 15:26:56.592 | INFO | Analyzing chunk frames = [2422784,2423296) | 20260506 15:26:56.592 | INFO | MemInfo: { | 20260506 15:26:56.592 | INFO | "gpu" : " 17 GB", | 20260506 15:26:56.592 | INFO | "huge" : "", | 20260506 15:26:56.592 | INFO | "huge_shared" : " 27 GB", | 20260506 15:26:56.592 | INFO | "malloc" : " 14 GB", | 20260506 15:26:56.592 | INFO | "pinned" : " 29 GB" | 20260506 15:26:56.592 | INFO | } | 20260506 15:26:56.592 | INFO | Perf report for the last 5.13912 seconds: | 20260506 15:26:56.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:26:56.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:56.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:56.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:56.592 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 15:26:56.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:56.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:56.592 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:26:56.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:56.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:26:56.592 | INFO | Uploaded 36GiB to GPU in 2.0025s (17.9775GiB/s) | 20260506 15:26:56.592 | INFO | Downloaded 34GiB to GPU in 1.93113s (17.6063GiB/s) | 20260506 15:27:01.731 | INFO | Analyzing chunk frames = [2423296,2423808) | 20260506 15:27:01.731 | INFO | MemInfo: { | 20260506 15:27:01.731 | INFO | "gpu" : " 17 GB", | 20260506 15:27:01.731 | INFO | "huge" : "", | 20260506 15:27:01.731 | INFO | "huge_shared" : " 27 GB", | 20260506 15:27:01.731 | INFO | "malloc" : " 14 GB", | 20260506 15:27:01.731 | INFO | "pinned" : " 29 GB" | 20260506 15:27:01.731 | INFO | } | 20260506 15:27:01.731 | INFO | Perf report for the last 5.13951 seconds: | 20260506 15:27:01.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:27:01.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:01.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:01.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:01.732 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 15:27:01.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:01.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:01.732 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 15:27:01.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:01.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:01.732 | INFO | Uploaded 36GiB to GPU in 2.07473s (17.3516GiB/s) | 20260506 15:27:01.732 | INFO | Downloaded 34GiB to GPU in 1.96263s (17.3237GiB/s) | 20260506 15:27:06.872 | INFO | Analyzing chunk frames = [2423808,2424320) | 20260506 15:27:06.872 | INFO | MemInfo: { | 20260506 15:27:06.872 | INFO | "gpu" : " 17 GB", | 20260506 15:27:06.872 | INFO | "huge" : "", | 20260506 15:27:06.872 | INFO | "huge_shared" : " 27 GB", | 20260506 15:27:06.872 | INFO | "malloc" : " 14 GB", | 20260506 15:27:06.872 | INFO | "pinned" : " 29 GB" | 20260506 15:27:06.872 | INFO | } | 20260506 15:27:06.872 | INFO | Perf report for the last 5.14058 seconds: | 20260506 15:27:06.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:27:06.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:06.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:06.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:06.872 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:27:06.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:06.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:06.872 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 15:27:06.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:06.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:06.872 | INFO | Uploaded 36GiB to GPU in 2.02137s (17.8097GiB/s) | 20260506 15:27:06.872 | INFO | Downloaded 34GiB to GPU in 1.99697s (17.0258GiB/s) | 20260506 15:27:12.011 | INFO | Analyzing chunk frames = [2424320,2424832) | 20260506 15:27:12.011 | INFO | MemInfo: { | 20260506 15:27:12.011 | INFO | "gpu" : " 17 GB", | 20260506 15:27:12.011 | INFO | "huge" : "", | 20260506 15:27:12.011 | INFO | "huge_shared" : " 27 GB", | 20260506 15:27:12.011 | INFO | "malloc" : " 14 GB", | 20260506 15:27:12.011 | INFO | "pinned" : " 29 GB" | 20260506 15:27:12.011 | INFO | } | 20260506 15:27:12.011 | INFO | Perf report for the last 5.13856 seconds: | 20260506 15:27:12.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:27:12.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:12.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:12.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:12.011 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 15:27:12.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:12.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:12.011 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260506 15:27:12.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:12.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:12.011 | INFO | Uploaded 36GiB to GPU in 2.03757s (17.6681GiB/s) | 20260506 15:27:12.011 | INFO | Downloaded 34GiB to GPU in 1.97665s (17.2008GiB/s) | 20260506 15:27:12.011 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:27:12.011 | INFO | Total mem usage: 17.4 GiB | 20260506 15:27:12.011 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:27:12.011 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -12.2%) | 20260506 15:27:12.011 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:27:12.011 | INFO | High Water Breakdown: | 20260506 15:27:12.011 | INFO | 61.1% : 9.48 GiB ( 86.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:27:12.011 | INFO | 38.7% : 6.01 GiB ( 89.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:27:12.011 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:27:12.011 | INFO | ------------------------------------------ | 20260506 15:27:12.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:27:12.011 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:27:12.011 | INFO | Total mem usage: 26.5 GiB | 20260506 15:27:12.011 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:27:12.011 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:27:12.011 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:27:12.011 | INFO | High Water Breakdown: | 20260506 15:27:12.011 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:27:12.011 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:27:12.011 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:27:12.011 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:27:12.011 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:27:12.011 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:27:12.011 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:27:12.011 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:27:12.011 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:27:12.011 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:27:12.011 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:27:12.011 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:27:12.011 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:27:12.011 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:27:12.011 | INFO | ------------------------------------------ | 20260506 15:27:12.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:27:12.011 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:27:12.011 | INFO | Total mem usage: 25.6 GiB | 20260506 15:27:12.011 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:27:12.011 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:27:12.011 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:27:12.011 | INFO | High Water Breakdown: | 20260506 15:27:12.011 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:27:12.011 | INFO | ------------------------------------------ | 20260506 15:27:12.011 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:27:12.011 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:27:12.011 | INFO | Total mem usage: 15.7 GiB | 20260506 15:27:12.011 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:27:12.011 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:27:12.011 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 15:27:12.011 | INFO | High Water Breakdown: | 20260506 15:27:12.011 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:27:12.011 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:27:12.011 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:27:12.011 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:27:12.011 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:27:12.011 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:27:12.011 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:27:12.011 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:27:12.011 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:27:12.012 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:27:12.012 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:27:12.012 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:27:12.012 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:27:12.012 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:27:12.012 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:27:12.012 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:27:12.012 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:27:12.012 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:27:12.012 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:27:12.012 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:27:12.012 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:27:12.012 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:27:12.012 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:27:12.012 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:27:12.012 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:27:12.012 | INFO | ------------------------------------------ | 20260506 15:27:12.012 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:27:12.012 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:27:12.012 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:27:12.012 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:27:12.012 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.57e+01, 1.37e+01 / 2.06e+00 | 20260506 15:27:12.012 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.00e-01, 1.03e-02 / 7.59e-03 | 20260506 15:27:12.012 | INFO | PerfReport: 11.56%, Baseline -- 3.64e+00 / 4.42e+00, 5.56e-01 / 3.74e-01 | 20260506 15:27:12.012 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.45e+00, 1.84e-01 / 1.48e-01 | 20260506 15:27:12.012 | INFO | PerfReport: 15.91%, DME -- 4.00e+01 / 4.46e+01, 4.17e+00 / 2.28e+00 | 20260506 15:27:12.012 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.79e+01 / 2.03e+01, 1.46e+00 / 1.90e+00 | 20260506 15:27:12.012 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.33e+00 / 2.94e+00, 4.06e-01 / 2.20e-01 | 20260506 15:27:12.012 | INFO | PerfReport: 3.55%, Metrics -- 1.12e+00 / 1.49e+00, 2.86e-01 / 1.60e-01 | 20260506 15:27:12.012 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.90e-01 / 4.42e-01, 6.07e-02 / 2.95e-02 | 20260506 15:27:12.012 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:27:12.012 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:27:12.012 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:27:12.012 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260506 15:27:12.012 | INFO | PerfReport: 21.67%, Upload -- 2.12e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260506 15:27:12.012 | INFO | PerfReport: 21.19%, Download -- 2.07e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 15:27:12.012 | INFO | PerfReport: 32.20%, Compute -- 3.15e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 15:27:12.012 | INFO | PerfReport: 1.67%, LockUpload -- 1.63e+00 / 0.00e+00, 6.12e+00 / 0.00e+00 | 20260506 15:27:12.012 | INFO | PerfReport: 1.02%, LockDownload -- 9.97e-01 / 0.00e+00, 2.53e+00 / 0.00e+00 | 20260506 15:27:12.012 | INFO | PerfReport: 22.23%, LockCompute -- 2.17e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 15:27:12.012 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.72e-02 / 0.00e+00 | 20260506 15:27:12.129 | INFO | Starting a round of Baz IO | 20260506 15:27:15.274 | INFO | Populating RTMetrics report | 20260506 15:27:15.274 | INFO | RTMetrics report populated | 20260506 15:27:15.274 | INFO | Opening tmp RTMetrics json file | 20260506 15:27:15.308 | INFO | Serializing RTMetrics to json | 20260506 15:27:15.311 | INFO | Serialization done, swapping tmp file now... | 20260506 15:27:15.315 | INFO | RTMetrics done | 20260506 15:27:15.509 | INFO | Finished a round of Baz IO: Wrote 2.102 GiB in 3.379 seconds | 20260506 15:27:15.509 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.20% / 0.01% / 62.79% | 20260506 15:27:15.509 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.06085% / 7.94729e-06% / 7.54992e-05% | 20260506 15:27:15.509 | INFO | Baz IO overall: | 20260506 15:27:15.509 | INFO | BytesWritten : 1264245557818 | 20260506 15:27:15.509 | INFO | NumEvents : 611824641264 | 20260506 15:27:17.149 | INFO | Analyzing chunk frames = [2424832,2425344) | 20260506 15:27:17.149 | INFO | MemInfo: { | 20260506 15:27:17.149 | INFO | "gpu" : " 17 GB", | 20260506 15:27:17.149 | INFO | "huge" : "", | 20260506 15:27:17.149 | INFO | "huge_shared" : " 27 GB", | 20260506 15:27:17.149 | INFO | "malloc" : " 14 GB", | 20260506 15:27:17.149 | INFO | "pinned" : " 29 GB" | 20260506 15:27:17.149 | INFO | } | 20260506 15:27:17.149 | INFO | Perf report for the last 5.13877 seconds: | 20260506 15:27:17.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:27:17.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:17.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:17.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:17.150 | INFO | ANALYSIS : 62.9%, 2.9, 95.2ms, 0 | 20260506 15:27:17.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:17.150 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 15:27:17.150 | INFO | PRE_HQ : 60.8%, 1.9, 59.3ms, 0 | 20260506 15:27:17.150 | INFO | BAZWRITER : 65.2%, 4.8, 161.2ms, 0 | 20260506 15:27:17.150 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 15:27:17.150 | INFO | Uploaded 36GiB to GPU in 2.04267s (17.624GiB/s) | 20260506 15:27:17.150 | INFO | Downloaded 38GiB to GPU in 2.23663s (16.9898GiB/s) | 20260506 15:27:22.288 | INFO | Analyzing chunk frames = [2425344,2425856) | 20260506 15:27:22.288 | INFO | MemInfo: { | 20260506 15:27:22.288 | INFO | "gpu" : " 17 GB", | 20260506 15:27:22.288 | INFO | "huge" : "", | 20260506 15:27:22.288 | INFO | "huge_shared" : " 27 GB", | 20260506 15:27:22.288 | INFO | "malloc" : " 14 GB", | 20260506 15:27:22.288 | INFO | "pinned" : " 29 GB" | 20260506 15:27:22.288 | INFO | } | 20260506 15:27:22.288 | INFO | Perf report for the last 5.13889 seconds: | 20260506 15:27:22.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:27:22.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:22.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:22.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:22.288 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:27:22.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:22.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:22.288 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:27:22.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:22.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:22.289 | INFO | Uploaded 36GiB to GPU in 2.00089s (17.992GiB/s) | 20260506 15:27:22.289 | INFO | Downloaded 34GiB to GPU in 1.94242s (17.504GiB/s) | 20260506 15:27:27.428 | INFO | Analyzing chunk frames = [2425856,2426368) | 20260506 15:27:27.428 | INFO | MemInfo: { | 20260506 15:27:27.428 | INFO | "gpu" : " 17 GB", | 20260506 15:27:27.428 | INFO | "huge" : "", | 20260506 15:27:27.428 | INFO | "huge_shared" : " 27 GB", | 20260506 15:27:27.428 | INFO | "malloc" : " 14 GB", | 20260506 15:27:27.428 | INFO | "pinned" : " 29 GB" | 20260506 15:27:27.428 | INFO | } | 20260506 15:27:27.428 | INFO | Perf report for the last 5.13945 seconds: | 20260506 15:27:27.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:27:27.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:27.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:27.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:27.428 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:27:27.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:27.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:27.428 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:27:27.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:27.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:27.428 | INFO | Uploaded 36GiB to GPU in 1.98168s (18.1664GiB/s) | 20260506 15:27:27.428 | INFO | Downloaded 34GiB to GPU in 1.94235s (17.5046GiB/s) | 20260506 15:27:32.569 | INFO | Analyzing chunk frames = [2426368,2426880) | 20260506 15:27:32.569 | INFO | MemInfo: { | 20260506 15:27:32.569 | INFO | "gpu" : " 17 GB", | 20260506 15:27:32.569 | INFO | "huge" : "", | 20260506 15:27:32.569 | INFO | "huge_shared" : " 27 GB", | 20260506 15:27:32.569 | INFO | "malloc" : " 14 GB", | 20260506 15:27:32.569 | INFO | "pinned" : " 29 GB" | 20260506 15:27:32.569 | INFO | } | 20260506 15:27:32.569 | INFO | Perf report for the last 5.14087 seconds: | 20260506 15:27:32.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:27:32.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:32.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:32.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:32.569 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 15:27:32.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:32.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:32.569 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:27:32.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:32.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:32.569 | INFO | Uploaded 36GiB to GPU in 2.0153s (17.8634GiB/s) | 20260506 15:27:32.569 | INFO | Downloaded 34GiB to GPU in 1.91957s (17.7123GiB/s) | 20260506 15:27:37.707 | INFO | Analyzing chunk frames = [2426880,2427392) | 20260506 15:27:37.707 | INFO | MemInfo: { | 20260506 15:27:37.707 | INFO | "gpu" : " 17 GB", | 20260506 15:27:37.707 | INFO | "huge" : "", | 20260506 15:27:37.707 | INFO | "huge_shared" : " 27 GB", | 20260506 15:27:37.707 | INFO | "malloc" : " 14 GB", | 20260506 15:27:37.707 | INFO | "pinned" : " 29 GB" | 20260506 15:27:37.707 | INFO | } | 20260506 15:27:37.707 | INFO | Perf report for the last 5.13835 seconds: | 20260506 15:27:37.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:27:37.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:37.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:37.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:37.707 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 15:27:37.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:37.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:37.707 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:27:37.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:37.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:37.707 | INFO | Uploaded 36GiB to GPU in 1.99648s (18.0317GiB/s) | 20260506 15:27:37.707 | INFO | Downloaded 34GiB to GPU in 1.93762s (17.5473GiB/s) | 20260506 15:27:42.846 | INFO | Analyzing chunk frames = [2427392,2427904) | 20260506 15:27:42.846 | INFO | MemInfo: { | 20260506 15:27:42.846 | INFO | "gpu" : " 17 GB", | 20260506 15:27:42.846 | INFO | "huge" : "", | 20260506 15:27:42.846 | INFO | "huge_shared" : " 27 GB", | 20260506 15:27:42.846 | INFO | "malloc" : " 14 GB", | 20260506 15:27:42.846 | INFO | "pinned" : " 29 GB" | 20260506 15:27:42.846 | INFO | } | 20260506 15:27:42.847 | INFO | Perf report for the last 5.13946 seconds: | 20260506 15:27:42.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:27:42.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:42.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:42.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:42.847 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 15:27:42.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:42.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:42.847 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:27:42.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:42.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:42.847 | INFO | Uploaded 36GiB to GPU in 2.05818s (17.4912GiB/s) | 20260506 15:27:42.847 | INFO | Downloaded 34GiB to GPU in 1.96773s (17.2788GiB/s) | 20260506 15:27:47.986 | INFO | Analyzing chunk frames = [2427904,2428416) | 20260506 15:27:47.986 | INFO | MemInfo: { | 20260506 15:27:47.986 | INFO | "gpu" : " 17 GB", | 20260506 15:27:47.986 | INFO | "huge" : "", | 20260506 15:27:47.986 | INFO | "huge_shared" : " 27 GB", | 20260506 15:27:47.986 | INFO | "malloc" : " 14 GB", | 20260506 15:27:47.986 | INFO | "pinned" : " 29 GB" | 20260506 15:27:47.986 | INFO | } | 20260506 15:27:47.986 | INFO | Perf report for the last 5.13987 seconds: | 20260506 15:27:47.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:27:47.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:47.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:47.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:47.986 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 15:27:47.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:47.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:47.986 | INFO | PRE_HQ : 29.1%, 1.0, 15.2ms, 0 | 20260506 15:27:47.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:47.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:47.987 | INFO | Uploaded 36GiB to GPU in 2.01027s (17.908GiB/s) | 20260506 15:27:47.987 | INFO | Downloaded 34GiB to GPU in 1.99747s (17.0215GiB/s) | 20260506 15:27:53.124 | INFO | Analyzing chunk frames = [2428416,2428928) | 20260506 15:27:53.124 | INFO | MemInfo: { | 20260506 15:27:53.124 | INFO | "gpu" : " 17 GB", | 20260506 15:27:53.124 | INFO | "huge" : "", | 20260506 15:27:53.125 | INFO | "huge_shared" : " 27 GB", | 20260506 15:27:53.125 | INFO | "malloc" : " 14 GB", | 20260506 15:27:53.125 | INFO | "pinned" : " 29 GB" | 20260506 15:27:53.125 | INFO | } | 20260506 15:27:53.125 | INFO | Perf report for the last 5.13816 seconds: | 20260506 15:27:53.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:27:53.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:53.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:53.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:53.125 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 15:27:53.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:53.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:53.125 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:27:53.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:53.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:53.125 | INFO | Uploaded 36GiB to GPU in 2.04035s (17.6441GiB/s) | 20260506 15:27:53.125 | INFO | Downloaded 34GiB to GPU in 1.9806s (17.1665GiB/s) | 20260506 15:27:53.236 | INFO | Starting a round of Baz IO | 20260506 15:27:56.388 | INFO | Populating RTMetrics report | 20260506 15:27:56.389 | INFO | RTMetrics report populated | 20260506 15:27:56.389 | INFO | Opening tmp RTMetrics json file | 20260506 15:27:56.414 | INFO | Serializing RTMetrics to json | 20260506 15:27:56.419 | INFO | Serialization done, swapping tmp file now... | 20260506 15:27:56.423 | INFO | RTMetrics done | 20260506 15:27:56.902 | INFO | Finished a round of Baz IO: Wrote 2.101 GiB in 3.665 seconds | 20260506 15:27:56.902 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.19% / 0.01% / 62.80% | 20260506 15:27:56.902 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.05704% / 1.98682e-05% / 6.35783e-05% | 20260506 15:27:56.902 | INFO | Baz IO overall: | 20260506 15:27:56.902 | INFO | BytesWritten : 1266501253435 | 20260506 15:27:56.902 | INFO | NumEvents : 612915459717 | 20260506 15:27:58.264 | INFO | Analyzing chunk frames = [2428928,2429440) | 20260506 15:27:58.264 | INFO | MemInfo: { | 20260506 15:27:58.264 | INFO | "gpu" : " 17 GB", | 20260506 15:27:58.264 | INFO | "huge" : "", | 20260506 15:27:58.264 | INFO | "huge_shared" : " 27 GB", | 20260506 15:27:58.264 | INFO | "malloc" : " 14 GB", | 20260506 15:27:58.264 | INFO | "pinned" : " 29 GB" | 20260506 15:27:58.264 | INFO | } | 20260506 15:27:58.264 | INFO | Perf report for the last 5.13937 seconds: | 20260506 15:27:58.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:27:58.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:58.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:58.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:58.264 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 15:27:58.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:27:58.264 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 15:27:58.264 | INFO | PRE_HQ : 60.8%, 1.9, 58.8ms, 0 | 20260506 15:27:58.264 | INFO | BAZWRITER : 70.7%, 4.8, 174.3ms, 0 | 20260506 15:27:58.264 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 15:27:58.264 | INFO | Uploaded 36GiB to GPU in 2.07629s (17.3386GiB/s) | 20260506 15:27:58.264 | INFO | Downloaded 38GiB to GPU in 2.23506s (17.0018GiB/s) | 20260506 15:28:03.403 | INFO | Analyzing chunk frames = [2429440,2429952) | 20260506 15:28:03.403 | INFO | MemInfo: { | 20260506 15:28:03.403 | INFO | "gpu" : " 17 GB", | 20260506 15:28:03.403 | INFO | "huge" : "", | 20260506 15:28:03.403 | INFO | "huge_shared" : " 27 GB", | 20260506 15:28:03.403 | INFO | "malloc" : " 14 GB", | 20260506 15:28:03.403 | INFO | "pinned" : " 29 GB" | 20260506 15:28:03.403 | INFO | } | 20260506 15:28:03.403 | INFO | Perf report for the last 5.13916 seconds: | 20260506 15:28:03.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:28:03.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:03.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:03.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:03.403 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:28:03.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:03.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:03.403 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:28:03.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:03.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:03.403 | INFO | Uploaded 36GiB to GPU in 1.99595s (18.0366GiB/s) | 20260506 15:28:03.403 | INFO | Downloaded 34GiB to GPU in 1.92825s (17.6326GiB/s) | 20260506 15:28:08.543 | INFO | Analyzing chunk frames = [2429952,2430464) | 20260506 15:28:08.543 | INFO | MemInfo: { | 20260506 15:28:08.543 | INFO | "gpu" : " 17 GB", | 20260506 15:28:08.543 | INFO | "huge" : "", | 20260506 15:28:08.543 | INFO | "huge_shared" : " 27 GB", | 20260506 15:28:08.543 | INFO | "malloc" : " 14 GB", | 20260506 15:28:08.543 | INFO | "pinned" : " 29 GB" | 20260506 15:28:08.543 | INFO | } | 20260506 15:28:08.543 | INFO | Perf report for the last 5.13958 seconds: | 20260506 15:28:08.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:28:08.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:08.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:08.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:08.543 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:28:08.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:08.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:08.543 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 15:28:08.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:08.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:08.543 | INFO | Uploaded 36GiB to GPU in 1.98153s (18.1678GiB/s) | 20260506 15:28:08.543 | INFO | Downloaded 34GiB to GPU in 1.93053s (17.6117GiB/s) | 20260506 15:28:13.682 | INFO | Analyzing chunk frames = [2430464,2430976) | 20260506 15:28:13.682 | INFO | MemInfo: { | 20260506 15:28:13.682 | INFO | "gpu" : " 17 GB", | 20260506 15:28:13.682 | INFO | "huge" : "", | 20260506 15:28:13.682 | INFO | "huge_shared" : " 27 GB", | 20260506 15:28:13.682 | INFO | "malloc" : " 14 GB", | 20260506 15:28:13.682 | INFO | "pinned" : " 29 GB" | 20260506 15:28:13.682 | INFO | } | 20260506 15:28:13.682 | INFO | Perf report for the last 5.1392 seconds: | 20260506 15:28:13.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:28:13.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:13.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:13.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:13.682 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:28:13.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:13.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:13.682 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:28:13.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:13.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:13.682 | INFO | Uploaded 36GiB to GPU in 2.02001s (17.8217GiB/s) | 20260506 15:28:13.682 | INFO | Downloaded 34GiB to GPU in 1.91362s (17.7674GiB/s) | 20260506 15:28:18.822 | INFO | Analyzing chunk frames = [2430976,2431488) | 20260506 15:28:18.822 | INFO | MemInfo: { | 20260506 15:28:18.822 | INFO | "gpu" : " 17 GB", | 20260506 15:28:18.822 | INFO | "huge" : "", | 20260506 15:28:18.822 | INFO | "huge_shared" : " 27 GB", | 20260506 15:28:18.822 | INFO | "malloc" : " 14 GB", | 20260506 15:28:18.822 | INFO | "pinned" : " 29 GB" | 20260506 15:28:18.822 | INFO | } | 20260506 15:28:18.823 | INFO | Perf report for the last 5.14057 seconds: | 20260506 15:28:18.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:28:18.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:18.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:18.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:18.823 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 15:28:18.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:18.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:18.823 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:28:18.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:18.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:18.823 | INFO | Uploaded 36GiB to GPU in 2.01235s (17.8895GiB/s) | 20260506 15:28:18.823 | INFO | Downloaded 34GiB to GPU in 1.93989s (17.5267GiB/s) | 20260506 15:28:23.961 | INFO | Analyzing chunk frames = [2431488,2432000) | 20260506 15:28:23.961 | INFO | MemInfo: { | 20260506 15:28:23.961 | INFO | "gpu" : " 17 GB", | 20260506 15:28:23.961 | INFO | "huge" : "", | 20260506 15:28:23.961 | INFO | "huge_shared" : " 27 GB", | 20260506 15:28:23.961 | INFO | "malloc" : " 14 GB", | 20260506 15:28:23.961 | INFO | "pinned" : " 29 GB" | 20260506 15:28:23.961 | INFO | } | 20260506 15:28:23.961 | INFO | Perf report for the last 5.13886 seconds: | 20260506 15:28:23.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:28:23.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:23.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:23.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:23.961 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:28:23.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:23.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:23.962 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 15:28:23.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:23.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:23.962 | INFO | Uploaded 36GiB to GPU in 2.07368s (17.3604GiB/s) | 20260506 15:28:23.962 | INFO | Downloaded 34GiB to GPU in 1.97118s (17.2486GiB/s) | 20260506 15:28:29.101 | INFO | Analyzing chunk frames = [2432000,2432512) | 20260506 15:28:29.101 | INFO | MemInfo: { | 20260506 15:28:29.101 | INFO | "gpu" : " 17 GB", | 20260506 15:28:29.101 | INFO | "huge" : "", | 20260506 15:28:29.101 | INFO | "huge_shared" : " 27 GB", | 20260506 15:28:29.101 | INFO | "malloc" : " 14 GB", | 20260506 15:28:29.101 | INFO | "pinned" : " 29 GB" | 20260506 15:28:29.101 | INFO | } | 20260506 15:28:29.101 | INFO | Perf report for the last 5.13937 seconds: | 20260506 15:28:29.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:28:29.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:29.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:29.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:29.101 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:28:29.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:29.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:29.101 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 15:28:29.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:29.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:29.101 | INFO | Uploaded 36GiB to GPU in 2.02074s (17.8153GiB/s) | 20260506 15:28:29.101 | INFO | Downloaded 34GiB to GPU in 2.00443s (16.9624GiB/s) | 20260506 15:28:34.239 | INFO | Analyzing chunk frames = [2432512,2433024) | 20260506 15:28:34.239 | INFO | MemInfo: { | 20260506 15:28:34.239 | INFO | "gpu" : " 17 GB", | 20260506 15:28:34.239 | INFO | "huge" : "", | 20260506 15:28:34.239 | INFO | "huge_shared" : " 27 GB", | 20260506 15:28:34.239 | INFO | "malloc" : " 14 GB", | 20260506 15:28:34.239 | INFO | "pinned" : " 29 GB" | 20260506 15:28:34.239 | INFO | } | 20260506 15:28:34.239 | INFO | Perf report for the last 5.13833 seconds: | 20260506 15:28:34.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:28:34.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:34.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:34.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:34.239 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 15:28:34.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:34.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:34.239 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260506 15:28:34.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:34.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:34.239 | INFO | Uploaded 36GiB to GPU in 2.0383s (17.6618GiB/s) | 20260506 15:28:34.240 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:28:34.240 | INFO | Total mem usage: 17.4 GiB | 20260506 15:28:34.240 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:28:34.240 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -7.69%) | 20260506 15:28:34.240 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:28:34.240 | INFO | High Water Breakdown: | 20260506 15:28:34.240 | INFO | 61.8% : 9.99 GiB ( 91.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:28:34.240 | INFO | 38.1% : 6.16 GiB ( 92% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:28:34.240 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:28:34.240 | INFO | ------------------------------------------ | 20260506 15:28:34.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:28:34.240 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:28:34.240 | INFO | Total mem usage: 26.5 GiB | 20260506 15:28:34.240 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:28:34.240 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:28:34.240 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:28:34.240 | INFO | High Water Breakdown: | 20260506 15:28:34.240 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:28:34.240 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:28:34.240 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:28:34.240 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:28:34.240 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:28:34.240 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:28:34.240 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:28:34.240 | INFO | Downloaded 34GiB to GPU in 1.97322s (17.2307GiB/s) | 20260506 15:28:34.240 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:28:34.240 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:28:34.240 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:28:34.240 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:28:34.240 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:28:34.240 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:28:34.240 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:28:34.240 | INFO | ------------------------------------------ | 20260506 15:28:34.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:28:34.240 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:28:34.240 | INFO | Total mem usage: 25.6 GiB | 20260506 15:28:34.240 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:28:34.240 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:28:34.240 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:28:34.240 | INFO | High Water Breakdown: | 20260506 15:28:34.240 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:28:34.240 | INFO | ------------------------------------------ | 20260506 15:28:34.240 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:28:34.240 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:28:34.240 | INFO | Total mem usage: 15.7 GiB | 20260506 15:28:34.240 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:28:34.240 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:28:34.240 | INFO | Unused Cache: 833 MiB (Waste: 5.19%) | 20260506 15:28:34.240 | INFO | High Water Breakdown: | 20260506 15:28:34.240 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:28:34.240 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:28:34.240 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:28:34.240 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:28:34.240 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:28:34.240 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:28:34.240 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:28:34.240 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:28:34.240 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:28:34.240 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:28:34.240 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:28:34.240 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:28:34.240 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:28:34.240 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:28:34.240 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:28:34.240 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:28:34.240 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:28:34.241 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:28:34.241 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:28:34.241 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:28:34.241 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:28:34.241 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:28:34.241 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:28:34.241 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:28:34.241 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:28:34.241 | INFO | ------------------------------------------ | 20260506 15:28:34.241 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:28:34.241 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:28:34.241 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:28:34.241 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:28:34.241 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.55e+01, 1.35e+01 / 1.71e+00 | 20260506 15:28:34.241 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 9.94e-02, 1.04e-02 / 7.56e-03 | 20260506 15:28:34.241 | INFO | PerfReport: 11.59%, Baseline -- 3.64e+00 / 4.39e+00, 5.60e-01 / 4.36e-01 | 20260506 15:28:34.241 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.43e+00, 1.84e-01 / 1.71e-01 | 20260506 15:28:34.241 | INFO | PerfReport: 15.88%, DME -- 3.99e+01 / 4.40e+01, 3.86e+00 / 1.92e+00 | 20260506 15:28:34.241 | INFO | PerfReport: 56.80%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.42e+00 / 1.81e+00 | 20260506 15:28:34.241 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.32e+00 / 2.90e+00, 3.94e-01 / 2.43e-01 | 20260506 15:28:34.241 | INFO | PerfReport: 3.56%, Metrics -- 1.12e+00 / 1.48e+00, 2.88e-01 / 2.03e-01 | 20260506 15:28:34.241 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.90e-01 / 4.41e-01, 8.75e-02 / 2.66e-02 | 20260506 15:28:34.241 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:28:34.241 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:28:34.241 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:28:34.241 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 15:28:34.241 | INFO | PerfReport: 21.60%, Upload -- 2.11e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260506 15:28:34.241 | INFO | PerfReport: 21.23%, Download -- 2.07e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 15:28:34.241 | INFO | PerfReport: 32.21%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 15:28:34.241 | INFO | PerfReport: 1.53%, LockUpload -- 1.49e+00 / 0.00e+00, 5.87e+00 / 0.00e+00 | 20260506 15:28:34.241 | INFO | PerfReport: 1.11%, LockDownload -- 1.08e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260506 15:28:34.241 | INFO | PerfReport: 22.30%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 15:28:34.241 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260506 15:28:34.367 | INFO | Starting a round of Baz IO | 20260506 15:28:37.512 | INFO | Populating RTMetrics report | 20260506 15:28:37.512 | INFO | RTMetrics report populated | 20260506 15:28:37.512 | INFO | Opening tmp RTMetrics json file | 20260506 15:28:37.513 | INFO | Serializing RTMetrics to json | 20260506 15:28:37.515 | INFO | Serialization done, swapping tmp file now... | 20260506 15:28:37.516 | INFO | RTMetrics done | 20260506 15:28:37.810 | INFO | Finished a round of Baz IO: Wrote 2.101 GiB in 3.443 seconds | 20260506 15:28:37.810 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.18% / 0.01% / 62.81% | 20260506 15:28:37.810 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.05286% / 1.98682e-05% / 6.75519e-05% | 20260506 15:28:37.810 | INFO | Baz IO overall: | 20260506 15:28:37.810 | INFO | BytesWritten : 1268756682793 | 20260506 15:28:37.810 | INFO | NumEvents : 614006718828 | 20260506 15:28:39.378 | INFO | Analyzing chunk frames = [2433024,2433536) | 20260506 15:28:39.378 | INFO | MemInfo: { | 20260506 15:28:39.378 | INFO | "gpu" : " 17 GB", | 20260506 15:28:39.378 | INFO | "huge" : "", | 20260506 15:28:39.378 | INFO | "huge_shared" : " 27 GB", | 20260506 15:28:39.378 | INFO | "malloc" : " 14 GB", | 20260506 15:28:39.378 | INFO | "pinned" : " 29 GB" | 20260506 15:28:39.378 | INFO | } | 20260506 15:28:39.378 | INFO | Perf report for the last 5.13919 seconds: | 20260506 15:28:39.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:28:39.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:39.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:39.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:39.378 | INFO | ANALYSIS : 63.1%, 2.9, 95.5ms, 0 | 20260506 15:28:39.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:39.378 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 15:28:39.378 | INFO | PRE_HQ : 61.1%, 1.9, 59.8ms, 0 | 20260506 15:28:39.378 | INFO | BAZWRITER : 66.3%, 4.8, 163.3ms, 0 | 20260506 15:28:39.378 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 15:28:39.379 | INFO | Uploaded 36GiB to GPU in 2.07393s (17.3583GiB/s) | 20260506 15:28:39.379 | INFO | Downloaded 38GiB to GPU in 2.23971s (16.9665GiB/s) | 20260506 15:28:44.518 | INFO | Analyzing chunk frames = [2433536,2434048) | 20260506 15:28:44.518 | INFO | MemInfo: { | 20260506 15:28:44.518 | INFO | "gpu" : " 17 GB", | 20260506 15:28:44.518 | INFO | "huge" : "", | 20260506 15:28:44.518 | INFO | "huge_shared" : " 27 GB", | 20260506 15:28:44.518 | INFO | "malloc" : " 14 GB", | 20260506 15:28:44.518 | INFO | "pinned" : " 29 GB" | 20260506 15:28:44.518 | INFO | } | 20260506 15:28:44.518 | INFO | Perf report for the last 5.13951 seconds: | 20260506 15:28:44.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:28:44.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:44.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:44.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:44.518 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 15:28:44.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:44.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:44.518 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:28:44.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:44.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:44.518 | INFO | Uploaded 36GiB to GPU in 1.99105s (18.0809GiB/s) | 20260506 15:28:44.518 | INFO | Downloaded 34GiB to GPU in 1.93739s (17.5494GiB/s) | 20260506 15:28:49.658 | INFO | Analyzing chunk frames = [2434048,2434560) | 20260506 15:28:49.658 | INFO | MemInfo: { | 20260506 15:28:49.658 | INFO | "gpu" : " 17 GB", | 20260506 15:28:49.658 | INFO | "huge" : "", | 20260506 15:28:49.658 | INFO | "huge_shared" : " 27 GB", | 20260506 15:28:49.658 | INFO | "malloc" : " 14 GB", | 20260506 15:28:49.658 | INFO | "pinned" : " 29 GB" | 20260506 15:28:49.658 | INFO | } | 20260506 15:28:49.659 | INFO | Perf report for the last 5.14063 seconds: | 20260506 15:28:49.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:28:49.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:49.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:49.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:49.659 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:28:49.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:49.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:49.659 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:28:49.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:49.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:49.659 | INFO | Uploaded 36GiB to GPU in 1.97773s (18.2027GiB/s) | 20260506 15:28:49.659 | INFO | Downloaded 34GiB to GPU in 1.93699s (17.553GiB/s) | 20260506 15:28:54.797 | INFO | Analyzing chunk frames = [2434560,2435072) | 20260506 15:28:54.797 | INFO | MemInfo: { | 20260506 15:28:54.797 | INFO | "gpu" : " 17 GB", | 20260506 15:28:54.797 | INFO | "huge" : "", | 20260506 15:28:54.797 | INFO | "huge_shared" : " 27 GB", | 20260506 15:28:54.797 | INFO | "malloc" : " 14 GB", | 20260506 15:28:54.797 | INFO | "pinned" : " 29 GB" | 20260506 15:28:54.797 | INFO | } | 20260506 15:28:54.797 | INFO | Perf report for the last 5.1385 seconds: | 20260506 15:28:54.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:28:54.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:54.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:54.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:54.797 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 15:28:54.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:54.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:54.797 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:28:54.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:54.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:54.797 | INFO | Uploaded 36GiB to GPU in 2.0228s (17.7971GiB/s) | 20260506 15:28:54.797 | INFO | Downloaded 34GiB to GPU in 1.91317s (17.7716GiB/s) | 20260506 15:28:59.935 | INFO | Analyzing chunk frames = [2435072,2435584) | 20260506 15:28:59.935 | INFO | MemInfo: { | 20260506 15:28:59.935 | INFO | "gpu" : " 17 GB", | 20260506 15:28:59.935 | INFO | "huge" : "", | 20260506 15:28:59.935 | INFO | "huge_shared" : " 27 GB", | 20260506 15:28:59.935 | INFO | "malloc" : " 14 GB", | 20260506 15:28:59.936 | INFO | "pinned" : " 29 GB" | 20260506 15:28:59.936 | INFO | } | 20260506 15:28:59.936 | INFO | Perf report for the last 5.13856 seconds: | 20260506 15:28:59.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:28:59.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:59.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:59.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:59.936 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 15:28:59.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:59.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:59.936 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:28:59.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:59.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:28:59.936 | INFO | Uploaded 36GiB to GPU in 2.01062s (17.9049GiB/s) | 20260506 15:28:59.936 | INFO | Downloaded 34GiB to GPU in 1.93806s (17.5433GiB/s) | 20260506 15:29:05.076 | INFO | Analyzing chunk frames = [2435584,2436096) | 20260506 15:29:05.076 | INFO | MemInfo: { | 20260506 15:29:05.076 | INFO | "gpu" : " 17 GB", | 20260506 15:29:05.076 | INFO | "huge" : "", | 20260506 15:29:05.076 | INFO | "huge_shared" : " 27 GB", | 20260506 15:29:05.076 | INFO | "malloc" : " 14 GB", | 20260506 15:29:05.076 | INFO | "pinned" : " 29 GB" | 20260506 15:29:05.076 | INFO | } | 20260506 15:29:05.076 | INFO | Perf report for the last 5.14014 seconds: | 20260506 15:29:05.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:29:05.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:05.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:05.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:05.076 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:29:05.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:05.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:05.076 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:29:05.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:05.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:05.076 | INFO | Uploaded 36GiB to GPU in 2.07792s (17.325GiB/s) | 20260506 15:29:05.076 | INFO | Downloaded 34GiB to GPU in 1.97497s (17.2155GiB/s) | 20260506 15:29:10.215 | INFO | Analyzing chunk frames = [2436096,2436608) | 20260506 15:29:10.215 | INFO | MemInfo: { | 20260506 15:29:10.215 | INFO | "gpu" : " 17 GB", | 20260506 15:29:10.215 | INFO | "huge" : "", | 20260506 15:29:10.216 | INFO | "huge_shared" : " 27 GB", | 20260506 15:29:10.216 | INFO | "malloc" : " 14 GB", | 20260506 15:29:10.216 | INFO | "pinned" : " 29 GB" | 20260506 15:29:10.216 | INFO | } | 20260506 15:29:10.216 | INFO | Perf report for the last 5.13987 seconds: | 20260506 15:29:10.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:29:10.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:10.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:10.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:10.216 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:29:10.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:10.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:10.216 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 15:29:10.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:10.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:10.216 | INFO | Uploaded 36GiB to GPU in 2.03228s (17.7141GiB/s) | 20260506 15:29:10.216 | INFO | Downloaded 34GiB to GPU in 1.98708s (17.1105GiB/s) | 20260506 15:29:15.354 | INFO | Analyzing chunk frames = [2436608,2437120) | 20260506 15:29:15.354 | INFO | MemInfo: { | 20260506 15:29:15.354 | INFO | "gpu" : " 17 GB", | 20260506 15:29:15.354 | INFO | "huge" : "", | 20260506 15:29:15.354 | INFO | "huge_shared" : " 27 GB", | 20260506 15:29:15.354 | INFO | "malloc" : " 14 GB", | 20260506 15:29:15.354 | INFO | "pinned" : " 29 GB" | 20260506 15:29:15.354 | INFO | } | 20260506 15:29:15.354 | INFO | Perf report for the last 5.13853 seconds: | 20260506 15:29:15.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:29:15.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:15.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:15.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:15.354 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 15:29:15.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:15.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:15.354 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 15:29:15.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:15.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:15.354 | INFO | Uploaded 36GiB to GPU in 2.05533s (17.5155GiB/s) | 20260506 15:29:15.354 | INFO | Downloaded 34GiB to GPU in 1.99242s (17.0647GiB/s) | 20260506 15:29:15.469 | INFO | Starting a round of Baz IO | 20260506 15:29:18.621 | INFO | Populating RTMetrics report | 20260506 15:29:18.622 | INFO | RTMetrics report populated | 20260506 15:29:18.622 | INFO | Opening tmp RTMetrics json file | 20260506 15:29:18.651 | INFO | Serializing RTMetrics to json | 20260506 15:29:18.657 | INFO | Serialization done, swapping tmp file now... | 20260506 15:29:18.662 | INFO | RTMetrics done | 20260506 15:29:19.108 | INFO | Finished a round of Baz IO: Wrote 2.102 GiB in 3.638 seconds | 20260506 15:29:19.108 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.17% / 0.01% / 62.82% | 20260506 15:29:19.108 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.05179% / 1.58946e-05% / 6.75519e-05% | 20260506 15:29:19.108 | INFO | Baz IO overall: | 20260506 15:29:19.108 | INFO | BytesWritten : 1271014066691 | 20260506 15:29:19.108 | INFO | NumEvents : 615099363037 | 20260506 15:29:20.494 | INFO | Analyzing chunk frames = [2437120,2437632) | 20260506 15:29:20.494 | INFO | MemInfo: { | 20260506 15:29:20.494 | INFO | "gpu" : " 17 GB", | 20260506 15:29:20.494 | INFO | "huge" : "", | 20260506 15:29:20.494 | INFO | "huge_shared" : " 27 GB", | 20260506 15:29:20.494 | INFO | "malloc" : " 14 GB", | 20260506 15:29:20.494 | INFO | "pinned" : " 29 GB" | 20260506 15:29:20.494 | INFO | } | 20260506 15:29:20.494 | INFO | Perf report for the last 5.13977 seconds: | 20260506 15:29:20.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:29:20.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:20.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:20.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:20.494 | INFO | ANALYSIS : 62.9%, 2.9, 95.4ms, 0 | 20260506 15:29:20.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:20.494 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 15:29:20.494 | INFO | PRE_HQ : 60.8%, 1.9, 59.2ms, 0 | 20260506 15:29:20.494 | INFO | BAZWRITER : 70.3%, 4.8, 173.1ms, 0 | 20260506 15:29:20.494 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 15:29:20.494 | INFO | Uploaded 36GiB to GPU in 2.08111s (17.2985GiB/s) | 20260506 15:29:20.494 | INFO | Downloaded 38GiB to GPU in 2.23672s (16.9892GiB/s) | 20260506 15:29:25.633 | INFO | Analyzing chunk frames = [2437632,2438144) | 20260506 15:29:25.633 | INFO | MemInfo: { | 20260506 15:29:25.633 | INFO | "gpu" : " 17 GB", | 20260506 15:29:25.633 | INFO | "huge" : "", | 20260506 15:29:25.633 | INFO | "huge_shared" : " 27 GB", | 20260506 15:29:25.633 | INFO | "malloc" : " 14 GB", | 20260506 15:29:25.633 | INFO | "pinned" : " 29 GB" | 20260506 15:29:25.633 | INFO | } | 20260506 15:29:25.633 | INFO | Perf report for the last 5.13915 seconds: | 20260506 15:29:25.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:29:25.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:25.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:25.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:25.633 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:29:25.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:25.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:25.633 | INFO | PRE_HQ : 28.0%, 1.0, 14.7ms, 0 | 20260506 15:29:25.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:25.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:25.633 | INFO | Uploaded 36GiB to GPU in 1.99877s (18.0111GiB/s) | 20260506 15:29:25.633 | INFO | Downloaded 34GiB to GPU in 1.93053s (17.6118GiB/s) | 20260506 15:29:30.771 | INFO | Analyzing chunk frames = [2438144,2438656) | 20260506 15:29:30.771 | INFO | MemInfo: { | 20260506 15:29:30.771 | INFO | "gpu" : " 17 GB", | 20260506 15:29:30.772 | INFO | "huge" : "", | 20260506 15:29:30.772 | INFO | "huge_shared" : " 27 GB", | 20260506 15:29:30.772 | INFO | "malloc" : " 14 GB", | 20260506 15:29:30.772 | INFO | "pinned" : " 29 GB" | 20260506 15:29:30.772 | INFO | } | 20260506 15:29:30.772 | INFO | Perf report for the last 5.13847 seconds: | 20260506 15:29:30.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:29:30.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:30.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:30.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:30.772 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:29:30.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:30.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:30.772 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:29:30.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:30.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:30.772 | INFO | Uploaded 36GiB to GPU in 1.97632s (18.2156GiB/s) | 20260506 15:29:30.772 | INFO | Downloaded 34GiB to GPU in 1.93595s (17.5624GiB/s) | 20260506 15:29:35.912 | INFO | Analyzing chunk frames = [2438656,2439168) | 20260506 15:29:35.912 | INFO | MemInfo: { | 20260506 15:29:35.912 | INFO | "gpu" : " 17 GB", | 20260506 15:29:35.912 | INFO | "huge" : "", | 20260506 15:29:35.912 | INFO | "huge_shared" : " 27 GB", | 20260506 15:29:35.912 | INFO | "malloc" : " 14 GB", | 20260506 15:29:35.912 | INFO | "pinned" : " 29 GB" | 20260506 15:29:35.912 | INFO | } | 20260506 15:29:35.912 | INFO | Perf report for the last 5.14055 seconds: | 20260506 15:29:35.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:29:35.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:35.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:35.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:35.912 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:29:35.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:35.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:35.912 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:29:35.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:35.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:35.912 | INFO | Uploaded 36GiB to GPU in 2.02195s (17.8046GiB/s) | 20260506 15:29:35.913 | INFO | Downloaded 34GiB to GPU in 1.91858s (17.7214GiB/s) | 20260506 15:29:41.051 | INFO | Analyzing chunk frames = [2439168,2439680) | 20260506 15:29:41.051 | INFO | MemInfo: { | 20260506 15:29:41.051 | INFO | "gpu" : " 17 GB", | 20260506 15:29:41.051 | INFO | "huge" : "", | 20260506 15:29:41.051 | INFO | "huge_shared" : " 27 GB", | 20260506 15:29:41.051 | INFO | "malloc" : " 14 GB", | 20260506 15:29:41.051 | INFO | "pinned" : " 29 GB" | 20260506 15:29:41.051 | INFO | } | 20260506 15:29:41.051 | INFO | Perf report for the last 5.13907 seconds: | 20260506 15:29:41.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:29:41.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:41.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:41.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:41.051 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 15:29:41.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:41.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:41.051 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:29:41.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:41.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:41.051 | INFO | Uploaded 36GiB to GPU in 2.02467s (17.7807GiB/s) | 20260506 15:29:41.052 | INFO | Downloaded 34GiB to GPU in 1.93859s (17.5385GiB/s) | 20260506 15:29:46.190 | INFO | Analyzing chunk frames = [2439680,2440192) | 20260506 15:29:46.190 | INFO | MemInfo: { | 20260506 15:29:46.190 | INFO | "gpu" : " 17 GB", | 20260506 15:29:46.190 | INFO | "huge" : "", | 20260506 15:29:46.190 | INFO | "huge_shared" : " 27 GB", | 20260506 15:29:46.190 | INFO | "malloc" : " 14 GB", | 20260506 15:29:46.190 | INFO | "pinned" : " 29 GB" | 20260506 15:29:46.190 | INFO | } | 20260506 15:29:46.190 | INFO | Perf report for the last 5.13855 seconds: | 20260506 15:29:46.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:29:46.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:46.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:46.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:46.190 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:29:46.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:46.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:46.190 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:29:46.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:46.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:46.190 | INFO | Uploaded 36GiB to GPU in 2.08109s (17.2986GiB/s) | 20260506 15:29:46.190 | INFO | Downloaded 34GiB to GPU in 1.9743s (17.2213GiB/s) | 20260506 15:29:51.330 | INFO | Analyzing chunk frames = [2440192,2440704) | 20260506 15:29:51.330 | INFO | MemInfo: { | 20260506 15:29:51.330 | INFO | "gpu" : " 17 GB", | 20260506 15:29:51.330 | INFO | "huge" : "", | 20260506 15:29:51.330 | INFO | "huge_shared" : " 27 GB", | 20260506 15:29:51.330 | INFO | "malloc" : " 14 GB", | 20260506 15:29:51.330 | INFO | "pinned" : " 29 GB" | 20260506 15:29:51.330 | INFO | } | 20260506 15:29:51.330 | INFO | Perf report for the last 5.14005 seconds: | 20260506 15:29:51.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:29:51.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:51.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:51.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:51.330 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:29:51.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:51.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:51.330 | INFO | PRE_HQ : 29.6%, 1.0, 15.5ms, 0 | 20260506 15:29:51.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:51.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:51.330 | INFO | Uploaded 36GiB to GPU in 1.99961s (18.0035GiB/s) | 20260506 15:29:51.330 | INFO | Downloaded 34GiB to GPU in 1.99655s (17.0293GiB/s) | 20260506 15:29:56.469 | INFO | Analyzing chunk frames = [2440704,2441216) | 20260506 15:29:56.469 | INFO | MemInfo: { | 20260506 15:29:56.469 | INFO | "gpu" : " 17 GB", | 20260506 15:29:56.469 | INFO | "huge" : "", | 20260506 15:29:56.469 | INFO | "huge_shared" : " 27 GB", | 20260506 15:29:56.469 | INFO | "malloc" : " 14 GB", | 20260506 15:29:56.469 | INFO | "pinned" : " 29 GB" | 20260506 15:29:56.469 | INFO | } | 20260506 15:29:56.469 | INFO | Perf report for the last 5.13933 seconds: | 20260506 15:29:56.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:29:56.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:56.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:56.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:56.469 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:29:56.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:56.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:56.469 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:29:56.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:56.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:29:56.469 | INFO | Uploaded 36GiB to GPU in 2.03117s (17.7238GiB/s) | 20260506 15:29:56.469 | INFO | Downloaded 34GiB to GPU in 1.98133s (17.1602GiB/s) | 20260506 15:29:56.470 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:29:56.470 | INFO | Total mem usage: 17.4 GiB | 20260506 15:29:56.470 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:29:56.470 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -7.3%) | 20260506 15:29:56.470 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:29:56.470 | INFO | High Water Breakdown: | 20260506 15:29:56.470 | INFO | 61.9% : 10 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:29:56.470 | INFO | 38% : 6.16 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:29:56.470 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:29:56.470 | INFO | ------------------------------------------ | 20260506 15:29:56.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:29:56.470 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:29:56.470 | INFO | Total mem usage: 26.5 GiB | 20260506 15:29:56.470 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:29:56.470 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:29:56.470 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:29:56.470 | INFO | High Water Breakdown: | 20260506 15:29:56.470 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:29:56.470 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:29:56.470 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:29:56.470 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:29:56.470 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:29:56.470 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:29:56.470 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:29:56.470 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:29:56.470 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:29:56.470 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:29:56.470 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:29:56.470 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:29:56.470 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:29:56.470 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:29:56.470 | INFO | ------------------------------------------ | 20260506 15:29:56.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:29:56.470 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:29:56.470 | INFO | Total mem usage: 25.6 GiB | 20260506 15:29:56.470 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:29:56.470 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:29:56.470 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:29:56.470 | INFO | High Water Breakdown: | 20260506 15:29:56.470 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:29:56.470 | INFO | ------------------------------------------ | 20260506 15:29:56.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:29:56.470 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:29:56.470 | INFO | Total mem usage: 15.7 GiB | 20260506 15:29:56.470 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:29:56.470 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:29:56.470 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 15:29:56.470 | INFO | High Water Breakdown: | 20260506 15:29:56.470 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:29:56.470 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:29:56.470 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:29:56.470 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:29:56.470 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:29:56.470 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:29:56.470 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:29:56.470 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:29:56.470 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:29:56.470 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:29:56.470 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:29:56.471 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:29:56.471 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:29:56.471 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:29:56.471 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:29:56.471 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:29:56.471 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:29:56.471 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:29:56.471 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:29:56.471 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:29:56.471 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:29:56.471 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:29:56.471 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:29:56.471 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:29:56.471 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:29:56.471 | INFO | ------------------------------------------ | 20260506 15:29:56.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:29:56.471 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:29:56.471 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:29:56.471 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:29:56.471 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.55e+01, 1.31e+01 / 2.27e+00 | 20260506 15:29:56.471 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.01e-01, 1.04e-02 / 7.59e-03 | 20260506 15:29:56.471 | INFO | PerfReport: 11.55%, Baseline -- 3.63e+00 / 4.32e+00, 5.52e-01 / 4.45e-01 | 20260506 15:29:56.471 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.46e+00, 1.86e-01 / 1.63e-01 | 20260506 15:29:56.471 | INFO | PerfReport: 15.92%, DME -- 4.00e+01 / 4.42e+01, 3.94e+00 / 2.80e+00 | 20260506 15:29:56.471 | INFO | PerfReport: 56.82%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.45e+00 / 1.84e+00 | 20260506 15:29:56.471 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.32e+00 / 2.96e+00, 3.99e-01 / 1.99e-01 | 20260506 15:29:56.471 | INFO | PerfReport: 3.55%, Metrics -- 1.12e+00 / 1.47e+00, 2.88e-01 / 1.85e-01 | 20260506 15:29:56.471 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.89e-01 / 4.41e-01, 6.15e-02 / 2.83e-02 | 20260506 15:29:56.471 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:29:56.471 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:29:56.471 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:29:56.471 | INFO | PerfReport: Full Execution -- 9.77e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 15:29:56.471 | INFO | PerfReport: 21.63%, Upload -- 2.11e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260506 15:29:56.471 | INFO | PerfReport: 21.21%, Download -- 2.07e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 15:29:56.471 | INFO | PerfReport: 32.19%, Compute -- 3.15e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 15:29:56.471 | INFO | PerfReport: 1.65%, LockUpload -- 1.61e+00 / 0.00e+00, 5.99e+00 / 0.00e+00 | 20260506 15:29:56.471 | INFO | PerfReport: 1.05%, LockDownload -- 1.03e+00 / 0.00e+00, 2.69e+00 / 0.00e+00 | 20260506 15:29:56.471 | INFO | PerfReport: 22.26%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 15:29:56.471 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.35e-02 / 0.00e+00 | 20260506 15:29:56.582 | INFO | Starting a round of Baz IO | 20260506 15:29:59.739 | INFO | Populating RTMetrics report | 20260506 15:29:59.739 | INFO | RTMetrics report populated | 20260506 15:29:59.739 | INFO | Opening tmp RTMetrics json file | 20260506 15:29:59.759 | INFO | Serializing RTMetrics to json | 20260506 15:29:59.763 | INFO | Serialization done, swapping tmp file now... | 20260506 15:29:59.767 | INFO | RTMetrics done | 20260506 15:29:59.962 | INFO | Finished a round of Baz IO: Wrote 2.101 GiB in 3.38 seconds | 20260506 15:29:59.962 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.16% / 0.01% / 62.83% | 20260506 15:29:59.962 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.0444% / 1.98682e-05% / 5.5631e-05% | 20260506 15:29:59.962 | INFO | Baz IO overall: | 20260506 15:29:59.962 | INFO | BytesWritten : 1273270268626 | 20260506 15:29:59.962 | INFO | NumEvents : 616191157992 | 20260506 15:30:01.608 | INFO | Analyzing chunk frames = [2441216,2441728) | 20260506 15:30:01.608 | INFO | MemInfo: { | 20260506 15:30:01.608 | INFO | "gpu" : " 17 GB", | 20260506 15:30:01.608 | INFO | "huge" : "", | 20260506 15:30:01.608 | INFO | "huge_shared" : " 27 GB", | 20260506 15:30:01.608 | INFO | "malloc" : " 14 GB", | 20260506 15:30:01.608 | INFO | "pinned" : " 29 GB" | 20260506 15:30:01.608 | INFO | } | 20260506 15:30:01.608 | INFO | Perf report for the last 5.13925 seconds: | 20260506 15:30:01.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:30:01.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:01.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:01.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:01.609 | INFO | ANALYSIS : 63.0%, 2.9, 95.6ms, 0 | 20260506 15:30:01.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:01.609 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 15:30:01.609 | INFO | PRE_HQ : 60.9%, 1.9, 59.2ms, 0 | 20260506 15:30:01.609 | INFO | BAZWRITER : 65.2%, 4.8, 161.1ms, 0 | 20260506 15:30:01.609 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 15:30:01.609 | INFO | Uploaded 36GiB to GPU in 2.05483s (17.5197GiB/s) | 20260506 15:30:01.609 | INFO | Downloaded 38GiB to GPU in 2.22304s (17.0937GiB/s) | 20260506 15:30:06.748 | INFO | Analyzing chunk frames = [2441728,2442240) | 20260506 15:30:06.748 | INFO | MemInfo: { | 20260506 15:30:06.748 | INFO | "gpu" : " 17 GB", | 20260506 15:30:06.748 | INFO | "huge" : "", | 20260506 15:30:06.748 | INFO | "huge_shared" : " 27 GB", | 20260506 15:30:06.748 | INFO | "malloc" : " 14 GB", | 20260506 15:30:06.748 | INFO | "pinned" : " 29 GB" | 20260506 15:30:06.748 | INFO | } | 20260506 15:30:06.748 | INFO | Perf report for the last 5.13925 seconds: | 20260506 15:30:06.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:30:06.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:06.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:06.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:06.748 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:30:06.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:06.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:06.748 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:30:06.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:06.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:06.748 | INFO | Uploaded 36GiB to GPU in 1.98798s (18.1088GiB/s) | 20260506 15:30:06.748 | INFO | Downloaded 34GiB to GPU in 1.92804s (17.6345GiB/s) | 20260506 15:30:11.886 | INFO | Analyzing chunk frames = [2442240,2442752) | 20260506 15:30:11.886 | INFO | MemInfo: { | 20260506 15:30:11.886 | INFO | "gpu" : " 17 GB", | 20260506 15:30:11.886 | INFO | "huge" : "", | 20260506 15:30:11.886 | INFO | "huge_shared" : " 27 GB", | 20260506 15:30:11.886 | INFO | "malloc" : " 14 GB", | 20260506 15:30:11.886 | INFO | "pinned" : " 29 GB" | 20260506 15:30:11.886 | INFO | } | 20260506 15:30:11.886 | INFO | Perf report for the last 5.13839 seconds: | 20260506 15:30:11.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:30:11.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:11.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:11.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:11.886 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:30:11.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:11.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:11.886 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:30:11.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:11.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:11.886 | INFO | Uploaded 36GiB to GPU in 1.97655s (18.2136GiB/s) | 20260506 15:30:11.886 | INFO | Downloaded 34GiB to GPU in 1.93679s (17.5548GiB/s) | 20260506 15:30:17.026 | INFO | Analyzing chunk frames = [2442752,2443264) | 20260506 15:30:17.026 | INFO | MemInfo: { | 20260506 15:30:17.026 | INFO | "gpu" : " 17 GB", | 20260506 15:30:17.027 | INFO | "huge" : "", | 20260506 15:30:17.027 | INFO | "huge_shared" : " 27 GB", | 20260506 15:30:17.027 | INFO | "malloc" : " 14 GB", | 20260506 15:30:17.027 | INFO | "pinned" : " 29 GB" | 20260506 15:30:17.027 | INFO | } | 20260506 15:30:17.027 | INFO | Perf report for the last 5.1405 seconds: | 20260506 15:30:17.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:30:17.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:17.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:17.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:17.027 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:30:17.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:17.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:17.027 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:30:17.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:17.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:17.027 | INFO | Uploaded 36GiB to GPU in 2.02331s (17.7926GiB/s) | 20260506 15:30:17.027 | INFO | Downloaded 34GiB to GPU in 1.91524s (17.7523GiB/s) | 20260506 15:30:22.166 | INFO | Analyzing chunk frames = [2443264,2443776) | 20260506 15:30:22.166 | INFO | MemInfo: { | 20260506 15:30:22.166 | INFO | "gpu" : " 17 GB", | 20260506 15:30:22.166 | INFO | "huge" : "", | 20260506 15:30:22.166 | INFO | "huge_shared" : " 27 GB", | 20260506 15:30:22.166 | INFO | "malloc" : " 14 GB", | 20260506 15:30:22.166 | INFO | "pinned" : " 29 GB" | 20260506 15:30:22.166 | INFO | } | 20260506 15:30:22.166 | INFO | Perf report for the last 5.13948 seconds: | 20260506 15:30:22.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:30:22.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:22.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:22.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:22.166 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 15:30:22.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:22.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:22.166 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:30:22.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:22.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:22.166 | INFO | Uploaded 36GiB to GPU in 2.02787s (17.7526GiB/s) | 20260506 15:30:22.166 | INFO | Downloaded 34GiB to GPU in 1.92254s (17.6849GiB/s) | 20260506 15:30:27.306 | INFO | Analyzing chunk frames = [2443776,2444288) | 20260506 15:30:27.306 | INFO | MemInfo: { | 20260506 15:30:27.306 | INFO | "gpu" : " 17 GB", | 20260506 15:30:27.306 | INFO | "huge" : "", | 20260506 15:30:27.306 | INFO | "huge_shared" : " 27 GB", | 20260506 15:30:27.306 | INFO | "malloc" : " 14 GB", | 20260506 15:30:27.306 | INFO | "pinned" : " 29 GB" | 20260506 15:30:27.306 | INFO | } | 20260506 15:30:27.306 | INFO | Perf report for the last 5.13954 seconds: | 20260506 15:30:27.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:30:27.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:27.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:27.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:27.306 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260506 15:30:27.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:27.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:27.306 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 15:30:27.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:27.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:27.306 | INFO | Uploaded 36GiB to GPU in 2.01395s (17.8753GiB/s) | 20260506 15:30:27.306 | INFO | Downloaded 34GiB to GPU in 1.93272s (17.5918GiB/s) | 20260506 15:30:32.444 | INFO | Analyzing chunk frames = [2444288,2444800) | 20260506 15:30:32.444 | INFO | MemInfo: { | 20260506 15:30:32.444 | INFO | "gpu" : " 17 GB", | 20260506 15:30:32.444 | INFO | "huge" : "", | 20260506 15:30:32.444 | INFO | "huge_shared" : " 27 GB", | 20260506 15:30:32.444 | INFO | "malloc" : " 14 GB", | 20260506 15:30:32.444 | INFO | "pinned" : " 29 GB" | 20260506 15:30:32.444 | INFO | } | 20260506 15:30:32.445 | INFO | Perf report for the last 5.13883 seconds: | 20260506 15:30:32.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:30:32.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:32.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:32.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:32.445 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:30:32.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:32.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:32.445 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:30:32.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:32.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:32.445 | INFO | Uploaded 36GiB to GPU in 1.98711s (18.1168GiB/s) | 20260506 15:30:32.445 | INFO | Downloaded 34GiB to GPU in 1.99729s (17.023GiB/s) | 20260506 15:30:37.583 | INFO | Analyzing chunk frames = [2444800,2445312) | 20260506 15:30:37.583 | INFO | MemInfo: { | 20260506 15:30:37.583 | INFO | "gpu" : " 17 GB", | 20260506 15:30:37.583 | INFO | "huge" : "", | 20260506 15:30:37.583 | INFO | "huge_shared" : " 27 GB", | 20260506 15:30:37.583 | INFO | "malloc" : " 14 GB", | 20260506 15:30:37.583 | INFO | "pinned" : " 29 GB" | 20260506 15:30:37.583 | INFO | } | 20260506 15:30:37.583 | INFO | Perf report for the last 5.13855 seconds: | 20260506 15:30:37.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:30:37.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:37.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:37.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:37.583 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 15:30:37.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:37.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:37.583 | INFO | PRE_HQ : 32.8%, 1.0, 17.0ms, 0 | 20260506 15:30:37.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:37.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:37.583 | INFO | Uploaded 36GiB to GPU in 2.04s (17.6471GiB/s) | 20260506 15:30:37.583 | INFO | Downloaded 34GiB to GPU in 2.0348s (16.7092GiB/s) | 20260506 15:30:37.692 | INFO | Starting a round of Baz IO | 20260506 15:30:40.859 | INFO | Populating RTMetrics report | 20260506 15:30:40.859 | INFO | RTMetrics report populated | 20260506 15:30:40.859 | INFO | Opening tmp RTMetrics json file | 20260506 15:30:40.870 | INFO | Serializing RTMetrics to json | 20260506 15:30:40.876 | INFO | Serialization done, swapping tmp file now... | 20260506 15:30:40.890 | INFO | RTMetrics done | 20260506 15:30:41.269 | INFO | Finished a round of Baz IO: Wrote 2.101 GiB in 3.576 seconds | 20260506 15:30:41.269 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.14% / 0.01% / 62.84% | 20260506 15:30:41.269 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.03882% / 7.94729e-06% / 5.5631e-05% | 20260506 15:30:41.269 | INFO | Baz IO overall: | 20260506 15:30:41.269 | INFO | BytesWritten : 1275525820248 | 20260506 15:30:41.269 | INFO | NumEvents : 617282355083 | 20260506 15:30:42.723 | INFO | Analyzing chunk frames = [2445312,2445824) | 20260506 15:30:42.723 | INFO | MemInfo: { | 20260506 15:30:42.723 | INFO | "gpu" : " 17 GB", | 20260506 15:30:42.723 | INFO | "huge" : "", | 20260506 15:30:42.723 | INFO | "huge_shared" : " 27 GB", | 20260506 15:30:42.723 | INFO | "malloc" : " 14 GB", | 20260506 15:30:42.723 | INFO | "pinned" : " 29 GB" | 20260506 15:30:42.723 | INFO | } | 20260506 15:30:42.723 | INFO | Perf report for the last 5.14016 seconds: | 20260506 15:30:42.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:30:42.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:42.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:42.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:42.723 | INFO | ANALYSIS : 63.1%, 2.8, 94.1ms, 0 | 20260506 15:30:42.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:42.723 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260506 15:30:42.723 | INFO | PRE_HQ : 62.0%, 2.0, 63.7ms, 0 | 20260506 15:30:42.723 | INFO | BAZWRITER : 69.0%, 4.8, 170.0ms, 0 | 20260506 15:30:42.723 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 15:30:42.723 | INFO | Uploaded 36GiB to GPU in 2.08594s (17.2584GiB/s) | 20260506 15:30:42.723 | INFO | Downloaded 38GiB to GPU in 2.21902s (17.1247GiB/s) | 20260506 15:30:47.862 | INFO | Analyzing chunk frames = [2445824,2446336) | 20260506 15:30:47.862 | INFO | MemInfo: { | 20260506 15:30:47.862 | INFO | "gpu" : " 17 GB", | 20260506 15:30:47.862 | INFO | "huge" : "", | 20260506 15:30:47.862 | INFO | "huge_shared" : " 27 GB", | 20260506 15:30:47.862 | INFO | "malloc" : " 14 GB", | 20260506 15:30:47.862 | INFO | "pinned" : " 29 GB" | 20260506 15:30:47.862 | INFO | } | 20260506 15:30:47.862 | INFO | Perf report for the last 5.13871 seconds: | 20260506 15:30:47.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:30:47.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:47.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:47.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:47.862 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 15:30:47.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:47.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:47.862 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:30:47.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:47.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:47.862 | INFO | Uploaded 36GiB to GPU in 1.99969s (18.0028GiB/s) | 20260506 15:30:47.862 | INFO | Downloaded 34GiB to GPU in 1.94077s (17.5188GiB/s) | 20260506 15:30:53.003 | INFO | Analyzing chunk frames = [2446336,2446848) | 20260506 15:30:53.003 | INFO | MemInfo: { | 20260506 15:30:53.003 | INFO | "gpu" : " 17 GB", | 20260506 15:30:53.003 | INFO | "huge" : "", | 20260506 15:30:53.003 | INFO | "huge_shared" : " 27 GB", | 20260506 15:30:53.003 | INFO | "malloc" : " 14 GB", | 20260506 15:30:53.003 | INFO | "pinned" : " 29 GB" | 20260506 15:30:53.003 | INFO | } | 20260506 15:30:53.003 | INFO | Perf report for the last 5.14084 seconds: | 20260506 15:30:53.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:30:53.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:53.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:53.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:53.003 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:30:53.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:53.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:53.003 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:30:53.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:53.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:53.003 | INFO | Uploaded 36GiB to GPU in 2.0011s (17.9901GiB/s) | 20260506 15:30:53.003 | INFO | Downloaded 34GiB to GPU in 1.94245s (17.5037GiB/s) | 20260506 15:30:58.141 | INFO | Analyzing chunk frames = [2446848,2447360) | 20260506 15:30:58.141 | INFO | MemInfo: { | 20260506 15:30:58.141 | INFO | "gpu" : " 17 GB", | 20260506 15:30:58.141 | INFO | "huge" : "", | 20260506 15:30:58.141 | INFO | "huge_shared" : " 27 GB", | 20260506 15:30:58.141 | INFO | "malloc" : " 14 GB", | 20260506 15:30:58.141 | INFO | "pinned" : " 29 GB" | 20260506 15:30:58.141 | INFO | } | 20260506 15:30:58.141 | INFO | Perf report for the last 5.13841 seconds: | 20260506 15:30:58.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:30:58.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:58.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:58.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:58.141 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:30:58.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:58.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:58.141 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:30:58.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:58.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:30:58.141 | INFO | Uploaded 36GiB to GPU in 2.02374s (17.7888GiB/s) | 20260506 15:30:58.141 | INFO | Downloaded 34GiB to GPU in 1.91539s (17.751GiB/s) | 20260506 15:31:03.280 | INFO | Analyzing chunk frames = [2447360,2447872) | 20260506 15:31:03.280 | INFO | MemInfo: { | 20260506 15:31:03.280 | INFO | "gpu" : " 17 GB", | 20260506 15:31:03.280 | INFO | "huge" : "", | 20260506 15:31:03.280 | INFO | "huge_shared" : " 27 GB", | 20260506 15:31:03.280 | INFO | "malloc" : " 14 GB", | 20260506 15:31:03.280 | INFO | "pinned" : " 29 GB" | 20260506 15:31:03.280 | INFO | } | 20260506 15:31:03.280 | INFO | Perf report for the last 5.139 seconds: | 20260506 15:31:03.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:31:03.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:03.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:03.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:03.280 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 15:31:03.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:03.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:03.280 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:31:03.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:03.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:03.280 | INFO | Uploaded 36GiB to GPU in 2.03456s (17.6943GiB/s) | 20260506 15:31:03.280 | INFO | Downloaded 34GiB to GPU in 1.92634s (17.65GiB/s) | 20260506 15:31:08.420 | INFO | Analyzing chunk frames = [2447872,2448384) | 20260506 15:31:08.420 | INFO | MemInfo: { | 20260506 15:31:08.420 | INFO | "gpu" : " 17 GB", | 20260506 15:31:08.420 | INFO | "huge" : "", | 20260506 15:31:08.420 | INFO | "huge_shared" : " 27 GB", | 20260506 15:31:08.420 | INFO | "malloc" : " 14 GB", | 20260506 15:31:08.420 | INFO | "pinned" : " 29 GB" | 20260506 15:31:08.420 | INFO | } | 20260506 15:31:08.420 | INFO | Perf report for the last 5.13985 seconds: | 20260506 15:31:08.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:31:08.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:08.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:08.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:08.420 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:31:08.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:08.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:08.420 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:31:08.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:08.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:08.420 | INFO | Uploaded 36GiB to GPU in 2.07943s (17.3124GiB/s) | 20260506 15:31:08.420 | INFO | Downloaded 34GiB to GPU in 1.97653s (17.2018GiB/s) | 20260506 15:31:13.559 | INFO | Analyzing chunk frames = [2448384,2448896) | 20260506 15:31:13.559 | INFO | MemInfo: { | 20260506 15:31:13.559 | INFO | "gpu" : " 17 GB", | 20260506 15:31:13.559 | INFO | "huge" : "", | 20260506 15:31:13.559 | INFO | "huge_shared" : " 27 GB", | 20260506 15:31:13.559 | INFO | "malloc" : " 14 GB", | 20260506 15:31:13.559 | INFO | "pinned" : " 29 GB" | 20260506 15:31:13.559 | INFO | } | 20260506 15:31:13.559 | INFO | Perf report for the last 5.13907 seconds: | 20260506 15:31:13.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:31:13.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:13.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:13.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:13.559 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:31:13.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:13.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:13.559 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:31:13.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:13.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:13.559 | INFO | Uploaded 36GiB to GPU in 2.00674s (17.9396GiB/s) | 20260506 15:31:13.559 | INFO | Downloaded 34GiB to GPU in 2.00605s (16.9487GiB/s) | 20260506 15:31:18.698 | INFO | Analyzing chunk frames = [2448896,2449408) | 20260506 15:31:18.698 | INFO | MemInfo: { | 20260506 15:31:18.698 | INFO | "gpu" : " 17 GB", | 20260506 15:31:18.698 | INFO | "huge" : "", | 20260506 15:31:18.698 | INFO | "huge_shared" : " 27 GB", | 20260506 15:31:18.698 | INFO | "malloc" : " 14 GB", | 20260506 15:31:18.698 | INFO | "pinned" : " 29 GB" | 20260506 15:31:18.698 | INFO | } | 20260506 15:31:18.698 | INFO | Perf report for the last 5.13848 seconds: | 20260506 15:31:18.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:31:18.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:18.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:18.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:18.698 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 15:31:18.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:18.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:18.698 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 15:31:18.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:18.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:18.698 | INFO | Uploaded 36GiB to GPU in 2.03943s (17.652GiB/s) | 20260506 15:31:18.698 | INFO | Downloaded 34GiB to GPU in 1.98258s (17.1494GiB/s) | 20260506 15:31:18.698 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:31:18.698 | INFO | Total mem usage: 17.4 GiB | 20260506 15:31:18.698 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:31:18.698 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: -5.49%) | 20260506 15:31:18.698 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:31:18.698 | INFO | High Water Breakdown: | 20260506 15:31:18.698 | INFO | 62.1% : 10.3 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:31:18.698 | INFO | 37.8% : 6.23 GiB ( 93.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:31:18.698 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:31:18.698 | INFO | ------------------------------------------ | 20260506 15:31:18.698 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:31:18.698 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:31:18.698 | INFO | Total mem usage: 26.5 GiB | 20260506 15:31:18.698 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.54%) | 20260506 15:31:18.698 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:31:18.698 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:31:18.698 | INFO | High Water Breakdown: | 20260506 15:31:18.698 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:31:18.698 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:31:18.698 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:31:18.698 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:31:18.698 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:31:18.698 | INFO | 0.559% : 146 MiB ( 80% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:31:18.698 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:31:18.699 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:31:18.699 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:31:18.699 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:31:18.699 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:31:18.699 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:31:18.699 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:31:18.699 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:31:18.699 | INFO | ------------------------------------------ | 20260506 15:31:18.699 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:31:18.699 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:31:18.699 | INFO | Total mem usage: 25.6 GiB | 20260506 15:31:18.699 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:31:18.699 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:31:18.699 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:31:18.699 | INFO | High Water Breakdown: | 20260506 15:31:18.699 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:31:18.699 | INFO | ------------------------------------------ | 20260506 15:31:18.699 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:31:18.699 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:31:18.699 | INFO | Total mem usage: 15.7 GiB | 20260506 15:31:18.699 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 10.9%) | 20260506 15:31:18.699 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:31:18.699 | INFO | Unused Cache: 822 MiB (Waste: 5.12%) | 20260506 15:31:18.699 | INFO | High Water Breakdown: | 20260506 15:31:18.699 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:31:18.699 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:31:18.699 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:31:18.699 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:31:18.699 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:31:18.699 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:31:18.699 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:31:18.699 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:31:18.699 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:31:18.699 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:31:18.699 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:31:18.699 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:31:18.699 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:31:18.699 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:31:18.699 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:31:18.699 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:31:18.699 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:31:18.699 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:31:18.699 | INFO | 0.216% : 36.5 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:31:18.699 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:31:18.699 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:31:18.699 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:31:18.699 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:31:18.699 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:31:18.699 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:31:18.699 | INFO | ------------------------------------------ | 20260506 15:31:18.699 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:31:18.699 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:31:18.699 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:31:18.699 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:31:18.699 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.55e+01, 1.35e+01 / 1.95e+00 | 20260506 15:31:18.699 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.01e-01, 1.03e-02 / 6.79e-03 | 20260506 15:31:18.699 | INFO | PerfReport: 11.59%, Baseline -- 3.65e+00 / 4.41e+00, 5.62e-01 / 3.68e-01 | 20260506 15:31:18.699 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.86e-01 / 1.44e-01 | 20260506 15:31:18.699 | INFO | PerfReport: 15.91%, DME -- 4.00e+01 / 4.43e+01, 3.94e+00 / 2.01e+00 | 20260506 15:31:18.699 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.44e+00 / 1.64e+00 | 20260506 15:31:18.699 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.33e+00 / 2.92e+00, 4.04e-01 / 2.41e-01 | 20260506 15:31:18.699 | INFO | PerfReport: 3.56%, Metrics -- 1.12e+00 / 1.45e+00, 2.92e-01 / 1.58e-01 | 20260506 15:31:18.699 | INFO | PerfReport: 1.24%, PulseSerializing -- 3.89e-01 / 4.39e-01, 6.12e-02 / 3.24e-02 | 20260506 15:31:18.699 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:31:18.699 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:31:18.699 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:31:18.699 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260506 15:31:18.699 | INFO | PerfReport: 21.60%, Upload -- 2.11e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260506 15:31:18.699 | INFO | PerfReport: 21.22%, Download -- 2.07e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 15:31:18.699 | INFO | PerfReport: 32.24%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 15:31:18.699 | INFO | PerfReport: 1.62%, LockUpload -- 1.58e+00 / 0.00e+00, 5.85e+00 / 0.00e+00 | 20260506 15:31:18.699 | INFO | PerfReport: 1.08%, LockDownload -- 1.05e+00 / 0.00e+00, 2.65e+00 / 0.00e+00 | 20260506 15:31:18.699 | INFO | PerfReport: 22.23%, LockCompute -- 2.17e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 15:31:18.699 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260506 15:31:18.810 | INFO | Starting a round of Baz IO | 20260506 15:31:22.042 | INFO | Populating RTMetrics report | 20260506 15:31:22.042 | INFO | RTMetrics report populated | 20260506 15:31:22.042 | INFO | Opening tmp RTMetrics json file | 20260506 15:31:22.099 | INFO | Serializing RTMetrics to json | 20260506 15:31:22.172 | INFO | Serialization done, swapping tmp file now... | 20260506 15:31:22.203 | INFO | RTMetrics done | 20260506 15:31:22.612 | INFO | Finished a round of Baz IO: Wrote 2.103 GiB in 3.802 seconds | 20260506 15:31:22.612 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.13% / 0.01% / 62.86% | 20260506 15:31:22.612 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.04027% / 1.58946e-05% / 7.54992e-05% | 20260506 15:31:22.612 | INFO | Baz IO overall: | 20260506 15:31:22.612 | INFO | BytesWritten : 1277783454644 | 20260506 15:31:22.612 | INFO | NumEvents : 618375331834 | 20260506 15:31:23.838 | INFO | Analyzing chunk frames = [2449408,2449920) | 20260506 15:31:23.838 | INFO | MemInfo: { | 20260506 15:31:23.838 | INFO | "gpu" : " 17 GB", | 20260506 15:31:23.838 | INFO | "huge" : "", | 20260506 15:31:23.838 | INFO | "huge_shared" : " 27 GB", | 20260506 15:31:23.838 | INFO | "malloc" : " 14 GB", | 20260506 15:31:23.838 | INFO | "pinned" : " 29 GB" | 20260506 15:31:23.838 | INFO | } | 20260506 15:31:23.838 | INFO | Perf report for the last 5.14019 seconds: | 20260506 15:31:23.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:31:23.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:23.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:23.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:23.838 | INFO | ANALYSIS : 64.3%, 2.8, 95.0ms, 0 | 20260506 15:31:23.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:23.838 | INFO | RT_METRICS : 51.4%, 1.0, 27.0ms, 0 | 20260506 15:31:23.838 | INFO | PRE_HQ : 62.9%, 2.0, 65.6ms, 0 | 20260506 15:31:23.838 | INFO | BAZWRITER : 73.4%, 4.8, 182.2ms, 0 | 20260506 15:31:23.838 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 15:31:23.838 | INFO | Uploaded 36GiB to GPU in 2.10312s (17.1174GiB/s) | 20260506 15:31:23.838 | INFO | Downloaded 38GiB to GPU in 2.24064s (16.9594GiB/s) | 20260506 15:31:28.977 | INFO | Analyzing chunk frames = [2449920,2450432) | 20260506 15:31:28.977 | INFO | MemInfo: { | 20260506 15:31:28.977 | INFO | "gpu" : " 17 GB", | 20260506 15:31:28.977 | INFO | "huge" : "", | 20260506 15:31:28.977 | INFO | "huge_shared" : " 27 GB", | 20260506 15:31:28.977 | INFO | "malloc" : " 14 GB", | 20260506 15:31:28.977 | INFO | "pinned" : " 29 GB" | 20260506 15:31:28.977 | INFO | } | 20260506 15:31:28.977 | INFO | Perf report for the last 5.13954 seconds: | 20260506 15:31:28.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:31:28.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:28.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:28.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:28.977 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:31:28.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:28.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:28.978 | INFO | PRE_HQ : 30.2%, 1.0, 16.0ms, 0 | 20260506 15:31:28.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:28.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:28.978 | INFO | Uploaded 36GiB to GPU in 2.02073s (17.8154GiB/s) | 20260506 15:31:28.978 | INFO | Downloaded 34GiB to GPU in 1.95404s (17.3998GiB/s) | 20260506 15:31:34.117 | INFO | Analyzing chunk frames = [2450432,2450944) | 20260506 15:31:34.118 | INFO | MemInfo: { | 20260506 15:31:34.118 | INFO | "gpu" : " 17 GB", | 20260506 15:31:34.118 | INFO | "huge" : "", | 20260506 15:31:34.118 | INFO | "huge_shared" : " 27 GB", | 20260506 15:31:34.118 | INFO | "malloc" : " 14 GB", | 20260506 15:31:34.118 | INFO | "pinned" : " 29 GB" | 20260506 15:31:34.118 | INFO | } | 20260506 15:31:34.118 | INFO | Perf report for the last 5.14021 seconds: | 20260506 15:31:34.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:31:34.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:34.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:34.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:34.118 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:31:34.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:34.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:34.118 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:31:34.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:34.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:34.118 | INFO | Uploaded 36GiB to GPU in 2.00127s (17.9885GiB/s) | 20260506 15:31:34.118 | INFO | Downloaded 34GiB to GPU in 1.94257s (17.5025GiB/s) | 20260506 15:31:39.257 | INFO | Analyzing chunk frames = [2450944,2451456) | 20260506 15:31:39.257 | INFO | MemInfo: { | 20260506 15:31:39.257 | INFO | "gpu" : " 17 GB", | 20260506 15:31:39.257 | INFO | "huge" : "", | 20260506 15:31:39.257 | INFO | "huge_shared" : " 27 GB", | 20260506 15:31:39.257 | INFO | "malloc" : " 14 GB", | 20260506 15:31:39.257 | INFO | "pinned" : " 29 GB" | 20260506 15:31:39.257 | INFO | } | 20260506 15:31:39.257 | INFO | Perf report for the last 5.13935 seconds: | 20260506 15:31:39.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:31:39.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:39.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:39.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:39.257 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 15:31:39.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:39.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:39.257 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:31:39.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:39.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:39.257 | INFO | Uploaded 36GiB to GPU in 2.02231s (17.8015GiB/s) | 20260506 15:31:39.257 | INFO | Downloaded 34GiB to GPU in 1.91611s (17.7442GiB/s) | 20260506 15:31:44.395 | INFO | Analyzing chunk frames = [2451456,2451968) | 20260506 15:31:44.395 | INFO | MemInfo: { | 20260506 15:31:44.395 | INFO | "gpu" : " 17 GB", | 20260506 15:31:44.395 | INFO | "huge" : "", | 20260506 15:31:44.395 | INFO | "huge_shared" : " 27 GB", | 20260506 15:31:44.395 | INFO | "malloc" : " 14 GB", | 20260506 15:31:44.395 | INFO | "pinned" : " 29 GB" | 20260506 15:31:44.395 | INFO | } | 20260506 15:31:44.396 | INFO | Perf report for the last 5.13843 seconds: | 20260506 15:31:44.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:31:44.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:44.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:44.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:44.396 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 15:31:44.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:44.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:44.396 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260506 15:31:44.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:44.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:44.396 | INFO | Uploaded 36GiB to GPU in 1.99985s (18.0013GiB/s) | 20260506 15:31:44.396 | INFO | Downloaded 34GiB to GPU in 1.93381s (17.5819GiB/s) | 20260506 15:31:49.534 | INFO | Analyzing chunk frames = [2451968,2452480) | 20260506 15:31:49.534 | INFO | MemInfo: { | 20260506 15:31:49.534 | INFO | "gpu" : " 17 GB", | 20260506 15:31:49.534 | INFO | "huge" : "", | 20260506 15:31:49.534 | INFO | "huge_shared" : " 27 GB", | 20260506 15:31:49.534 | INFO | "malloc" : " 14 GB", | 20260506 15:31:49.534 | INFO | "pinned" : " 29 GB" | 20260506 15:31:49.534 | INFO | } | 20260506 15:31:49.534 | INFO | Perf report for the last 5.13884 seconds: | 20260506 15:31:49.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:31:49.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:49.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:49.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:49.534 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:31:49.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:49.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:49.534 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:31:49.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:49.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:49.535 | INFO | Uploaded 36GiB to GPU in 2.07667s (17.3354GiB/s) | 20260506 15:31:49.535 | INFO | Downloaded 34GiB to GPU in 1.96638s (17.2907GiB/s) | 20260506 15:31:54.674 | INFO | Analyzing chunk frames = [2452480,2452992) | 20260506 15:31:54.674 | INFO | MemInfo: { | 20260506 15:31:54.674 | INFO | "gpu" : " 17 GB", | 20260506 15:31:54.674 | INFO | "huge" : "", | 20260506 15:31:54.674 | INFO | "huge_shared" : " 27 GB", | 20260506 15:31:54.674 | INFO | "malloc" : " 14 GB", | 20260506 15:31:54.674 | INFO | "pinned" : " 29 GB" | 20260506 15:31:54.674 | INFO | } | 20260506 15:31:54.674 | INFO | Perf report for the last 5.13995 seconds: | 20260506 15:31:54.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:31:54.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:54.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:54.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:54.674 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:31:54.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:54.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:54.674 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:31:54.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:54.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:54.675 | INFO | Uploaded 36GiB to GPU in 2.00257s (17.9769GiB/s) | 20260506 15:31:54.675 | INFO | Downloaded 34GiB to GPU in 2.00141s (16.988GiB/s) | 20260506 15:31:59.814 | INFO | Analyzing chunk frames = [2452992,2453504) | 20260506 15:31:59.814 | INFO | MemInfo: { | 20260506 15:31:59.814 | INFO | "gpu" : " 17 GB", | 20260506 15:31:59.814 | INFO | "huge" : "", | 20260506 15:31:59.814 | INFO | "huge_shared" : " 27 GB", | 20260506 15:31:59.814 | INFO | "malloc" : " 14 GB", | 20260506 15:31:59.814 | INFO | "pinned" : " 29 GB" | 20260506 15:31:59.814 | INFO | } | 20260506 15:31:59.814 | INFO | Perf report for the last 5.14007 seconds: | 20260506 15:31:59.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:31:59.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:59.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:59.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:59.814 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:31:59.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:59.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:59.814 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 15:31:59.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:59.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:31:59.815 | INFO | Uploaded 36GiB to GPU in 2.03966s (17.65GiB/s) | 20260506 15:31:59.815 | INFO | Downloaded 34GiB to GPU in 1.98566s (17.1228GiB/s) | 20260506 15:31:59.926 | INFO | Starting a round of Baz IO | 20260506 15:32:03.076 | INFO | Populating RTMetrics report | 20260506 15:32:03.076 | INFO | RTMetrics report populated | 20260506 15:32:03.076 | INFO | Opening tmp RTMetrics json file | 20260506 15:32:03.132 | INFO | Serializing RTMetrics to json | 20260506 15:32:03.194 | INFO | Serialization done, swapping tmp file now... | 20260506 15:32:03.195 | INFO | RTMetrics done | 20260506 15:32:03.464 | INFO | Finished a round of Baz IO: Wrote 2.102 GiB in 3.537 seconds | 20260506 15:32:03.464 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.12% / 0.01% / 62.87% | 20260506 15:32:03.464 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.03354% / 1.19209e-05% / 5.5631e-05% | 20260506 15:32:03.464 | INFO | Baz IO overall: | 20260506 15:32:03.464 | INFO | BytesWritten : 1280040085448 | 20260506 15:32:03.464 | INFO | NumEvents : 619467604322 | 20260506 15:32:04.954 | INFO | Analyzing chunk frames = [2453504,2454016) | 20260506 15:32:04.954 | INFO | MemInfo: { | 20260506 15:32:04.954 | INFO | "gpu" : " 17 GB", | 20260506 15:32:04.954 | INFO | "huge" : "", | 20260506 15:32:04.954 | INFO | "huge_shared" : " 27 GB", | 20260506 15:32:04.954 | INFO | "malloc" : " 14 GB", | 20260506 15:32:04.954 | INFO | "pinned" : " 29 GB" | 20260506 15:32:04.954 | INFO | } | 20260506 15:32:04.954 | INFO | Perf report for the last 5.13977 seconds: | 20260506 15:32:04.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:32:04.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:04.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:04.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:04.954 | INFO | ANALYSIS : 62.9%, 2.9, 95.3ms, 0 | 20260506 15:32:04.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:04.954 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 15:32:04.954 | INFO | PRE_HQ : 61.0%, 1.9, 59.4ms, 0 | 20260506 15:32:04.954 | INFO | BAZWRITER : 68.2%, 4.8, 168.5ms, 0 | 20260506 15:32:04.954 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260506 15:32:04.955 | INFO | Uploaded 36GiB to GPU in 2.08688s (17.2506GiB/s) | 20260506 15:32:04.955 | INFO | Downloaded 38GiB to GPU in 2.22373s (17.0884GiB/s) | 20260506 15:32:10.091 | INFO | Analyzing chunk frames = [2454016,2454528) | 20260506 15:32:10.091 | INFO | MemInfo: { | 20260506 15:32:10.091 | INFO | "gpu" : " 17 GB", | 20260506 15:32:10.091 | INFO | "huge" : "", | 20260506 15:32:10.091 | INFO | "huge_shared" : " 27 GB", | 20260506 15:32:10.091 | INFO | "malloc" : " 14 GB", | 20260506 15:32:10.091 | INFO | "pinned" : " 29 GB" | 20260506 15:32:10.091 | INFO | } | 20260506 15:32:10.091 | INFO | Perf report for the last 5.13723 seconds: | 20260506 15:32:10.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:32:10.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:10.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:10.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:10.091 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:32:10.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:10.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:10.091 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:32:10.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:10.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:10.092 | INFO | Uploaded 36GiB to GPU in 2.02637s (17.7658GiB/s) | 20260506 15:32:10.092 | INFO | Downloaded 34GiB to GPU in 1.95659s (17.3771GiB/s) | 20260506 15:32:15.231 | INFO | Analyzing chunk frames = [2454528,2455040) | 20260506 15:32:15.231 | INFO | MemInfo: { | 20260506 15:32:15.231 | INFO | "gpu" : " 17 GB", | 20260506 15:32:15.231 | INFO | "huge" : "", | 20260506 15:32:15.231 | INFO | "huge_shared" : " 27 GB", | 20260506 15:32:15.231 | INFO | "malloc" : " 14 GB", | 20260506 15:32:15.231 | INFO | "pinned" : " 29 GB" | 20260506 15:32:15.231 | INFO | } | 20260506 15:32:15.232 | INFO | Perf report for the last 5.14011 seconds: | 20260506 15:32:15.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:32:15.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:15.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:15.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:15.232 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 15:32:15.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:15.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:15.232 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:32:15.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:15.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:15.232 | INFO | Uploaded 36GiB to GPU in 2.00098s (17.9912GiB/s) | 20260506 15:32:15.232 | INFO | Downloaded 34GiB to GPU in 1.94149s (17.5123GiB/s) | 20260506 15:32:20.370 | INFO | Analyzing chunk frames = [2455040,2455552) | 20260506 15:32:20.370 | INFO | MemInfo: { | 20260506 15:32:20.370 | INFO | "gpu" : " 17 GB", | 20260506 15:32:20.370 | INFO | "huge" : "", | 20260506 15:32:20.370 | INFO | "huge_shared" : " 27 GB", | 20260506 15:32:20.370 | INFO | "malloc" : " 14 GB", | 20260506 15:32:20.370 | INFO | "pinned" : " 29 GB" | 20260506 15:32:20.371 | INFO | } | 20260506 15:32:20.371 | INFO | Perf report for the last 5.13901 seconds: | 20260506 15:32:20.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:32:20.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:20.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:20.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:20.371 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:32:20.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:20.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:20.371 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:32:20.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:20.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:20.371 | INFO | Uploaded 36GiB to GPU in 2.01643s (17.8533GiB/s) | 20260506 15:32:20.371 | INFO | Downloaded 34GiB to GPU in 1.91865s (17.7208GiB/s) | 20260506 15:32:25.509 | INFO | Analyzing chunk frames = [2455552,2456064) | 20260506 15:32:25.509 | INFO | MemInfo: { | 20260506 15:32:25.509 | INFO | "gpu" : " 17 GB", | 20260506 15:32:25.509 | INFO | "huge" : "", | 20260506 15:32:25.509 | INFO | "huge_shared" : " 27 GB", | 20260506 15:32:25.509 | INFO | "malloc" : " 14 GB", | 20260506 15:32:25.509 | INFO | "pinned" : " 29 GB" | 20260506 15:32:25.509 | INFO | } | 20260506 15:32:25.509 | INFO | Perf report for the last 5.13892 seconds: | 20260506 15:32:25.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:32:25.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:25.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:25.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:25.510 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 15:32:25.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:25.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:25.510 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:32:25.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:25.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:25.510 | INFO | Uploaded 36GiB to GPU in 2.0148s (17.8677GiB/s) | 20260506 15:32:25.510 | INFO | Downloaded 34GiB to GPU in 1.93237s (17.595GiB/s) | 20260506 15:32:30.649 | INFO | Analyzing chunk frames = [2456064,2456576) | 20260506 15:32:30.649 | INFO | MemInfo: { | 20260506 15:32:30.649 | INFO | "gpu" : " 17 GB", | 20260506 15:32:30.649 | INFO | "huge" : "", | 20260506 15:32:30.649 | INFO | "huge_shared" : " 27 GB", | 20260506 15:32:30.649 | INFO | "malloc" : " 14 GB", | 20260506 15:32:30.649 | INFO | "pinned" : " 29 GB" | 20260506 15:32:30.649 | INFO | } | 20260506 15:32:30.649 | INFO | Perf report for the last 5.13969 seconds: | 20260506 15:32:30.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:32:30.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:30.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:30.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:30.649 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 15:32:30.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:30.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:30.649 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260506 15:32:30.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:30.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:30.649 | INFO | Uploaded 36GiB to GPU in 2.02066s (17.816GiB/s) | 20260506 15:32:30.649 | INFO | Downloaded 34GiB to GPU in 1.92436s (17.6682GiB/s) | 20260506 15:32:35.789 | INFO | Analyzing chunk frames = [2456576,2457088) | 20260506 15:32:35.789 | INFO | MemInfo: { | 20260506 15:32:35.789 | INFO | "gpu" : " 17 GB", | 20260506 15:32:35.789 | INFO | "huge" : "", | 20260506 15:32:35.789 | INFO | "huge_shared" : " 27 GB", | 20260506 15:32:35.789 | INFO | "malloc" : " 14 GB", | 20260506 15:32:35.789 | INFO | "pinned" : " 29 GB" | 20260506 15:32:35.789 | INFO | } | 20260506 15:32:35.789 | INFO | Perf report for the last 5.13994 seconds: | 20260506 15:32:35.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:32:35.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:35.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:35.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:35.789 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:32:35.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:35.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:35.789 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 15:32:35.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:35.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:35.789 | INFO | Uploaded 36GiB to GPU in 2.00112s (17.9899GiB/s) | 20260506 15:32:35.789 | INFO | Downloaded 34GiB to GPU in 1.95486s (17.3925GiB/s) | 20260506 15:32:40.928 | INFO | Analyzing chunk frames = [2457088,2457600) | 20260506 15:32:40.928 | INFO | MemInfo: { | 20260506 15:32:40.928 | INFO | "gpu" : " 17 GB", | 20260506 15:32:40.928 | INFO | "huge" : "", | 20260506 15:32:40.928 | INFO | "huge_shared" : " 27 GB", | 20260506 15:32:40.928 | INFO | "malloc" : " 14 GB", | 20260506 15:32:40.928 | INFO | "pinned" : " 29 GB" | 20260506 15:32:40.928 | INFO | } | 20260506 15:32:40.928 | INFO | Perf report for the last 5.13857 seconds: | 20260506 15:32:40.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:32:40.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:40.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:40.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:40.928 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:32:40.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:40.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:40.928 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 15:32:40.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:40.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:40.928 | INFO | Uploaded 36GiB to GPU in 2.02268s (17.7981GiB/s) | 20260506 15:32:40.928 | INFO | Downloaded 34GiB to GPU in 1.99059s (17.0803GiB/s) | 20260506 15:32:40.928 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:32:40.928 | INFO | Total mem usage: 17.4 GiB | 20260506 15:32:40.928 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:32:40.928 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -7.7%) | 20260506 15:32:40.928 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:32:40.928 | INFO | High Water Breakdown: | 20260506 15:32:40.928 | INFO | 61.8% : 9.99 GiB ( 91.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:32:40.928 | INFO | 38.1% : 6.16 GiB ( 92% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:32:40.928 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:32:40.928 | INFO | ------------------------------------------ | 20260506 15:32:40.928 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:32:40.928 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:32:40.928 | INFO | Total mem usage: 26.5 GiB | 20260506 15:32:40.928 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:32:40.928 | INFO | Recent High Water Marks Sum: 25.7 GiB (Savings: -3.29%) | 20260506 15:32:40.928 | INFO | Unused Cache: 1.57 GiB (Waste: 5.91%) | 20260506 15:32:40.928 | INFO | High Water Breakdown: | 20260506 15:32:40.928 | INFO | 55.9% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:32:40.929 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:32:40.929 | INFO | 3.65% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:32:40.929 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:32:40.929 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:32:40.929 | INFO | 0.833% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:32:40.929 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:32:40.929 | INFO | 9.33e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:32:40.929 | INFO | 8.71e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:32:40.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:32:40.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:32:40.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:32:40.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:32:40.929 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:32:40.929 | INFO | ------------------------------------------ | 20260506 15:32:40.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:32:40.929 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:32:40.929 | INFO | Total mem usage: 25.6 GiB | 20260506 15:32:40.929 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:32:40.929 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:32:40.929 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:32:40.929 | INFO | High Water Breakdown: | 20260506 15:32:40.929 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:32:40.929 | INFO | ------------------------------------------ | 20260506 15:32:40.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:32:40.929 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:32:40.929 | INFO | Total mem usage: 15.7 GiB | 20260506 15:32:40.929 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:32:40.929 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 5.01%) | 20260506 15:32:40.929 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 15:32:40.929 | INFO | High Water Breakdown: | 20260506 15:32:40.929 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:32:40.929 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:32:40.929 | INFO | 8.52% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:32:40.929 | INFO | 8.4% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:32:40.929 | INFO | 6.25% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:32:40.929 | INFO | 6.11% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:32:40.929 | INFO | 4.83% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:32:40.929 | INFO | 3.41% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:32:40.929 | INFO | 3.06% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:32:40.929 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:32:40.929 | INFO | 2.27% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:32:40.929 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:32:40.929 | INFO | 1.65% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:32:40.929 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:32:40.929 | INFO | 0.758% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:32:40.929 | INFO | 0.758% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:32:40.929 | INFO | 0.621% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:32:40.929 | INFO | 0.568% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:32:40.929 | INFO | 0.432% : 73 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:32:40.929 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:32:40.929 | INFO | 0.071% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:32:40.929 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:32:40.929 | INFO | 0.00592% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:32:40.929 | INFO | 1.35e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:32:40.929 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:32:40.929 | INFO | ------------------------------------------ | 20260506 15:32:40.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:32:40.929 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:32:40.929 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:32:40.929 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:32:40.929 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.57e+01, 1.34e+01 / 1.83e+00 | 20260506 15:32:40.929 | INFO | PerfReport: 0.30%, AggregateStats -- 9.36e-02 / 1.01e-01, 6.47e-02 / 7.04e-03 | 20260506 15:32:40.929 | INFO | PerfReport: 11.58%, Baseline -- 3.64e+00 / 4.43e+00, 5.52e-01 / 3.25e-01 | 20260506 15:32:40.929 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.47e+00, 1.82e-01 / 1.31e-01 | 20260506 15:32:40.929 | INFO | PerfReport: 15.93%, DME -- 4.01e+01 / 4.45e+01, 3.93e+00 / 2.14e+00 | 20260506 15:32:40.929 | INFO | PerfReport: 56.74%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.41e+00 / 1.59e+00 | 20260506 15:32:40.929 | INFO | PerfReport: 7.42%, PulseAccumulating -- 2.33e+00 / 2.91e+00, 6.03e-01 / 2.06e-01 | 20260506 15:32:40.929 | INFO | PerfReport: 3.55%, Metrics -- 1.12e+00 / 1.45e+00, 2.89e-01 / 1.36e-01 | 20260506 15:32:40.929 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.88e-01 / 4.42e-01, 6.07e-02 / 3.25e-02 | 20260506 15:32:40.929 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:32:40.929 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:32:40.929 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:32:40.929 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 15:32:40.929 | INFO | PerfReport: 21.65%, Upload -- 2.11e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260506 15:32:40.929 | INFO | PerfReport: 21.20%, Download -- 2.07e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 15:32:40.929 | INFO | PerfReport: 32.21%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 15:32:40.929 | INFO | PerfReport: 1.67%, LockUpload -- 1.63e+00 / 0.00e+00, 5.94e+00 / 0.00e+00 | 20260506 15:32:40.929 | INFO | PerfReport: 1.04%, LockDownload -- 1.02e+00 / 0.00e+00, 2.56e+00 / 0.00e+00 | 20260506 15:32:40.929 | INFO | PerfReport: 22.21%, LockCompute -- 2.17e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 15:32:40.929 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.43e-02 / 0.00e+00 | 20260506 15:32:41.048 | INFO | Starting a round of Baz IO | 20260506 15:32:44.195 | INFO | Populating RTMetrics report | 20260506 15:32:44.195 | INFO | RTMetrics report populated | 20260506 15:32:44.195 | INFO | Opening tmp RTMetrics json file | 20260506 15:32:44.198 | INFO | Serializing RTMetrics to json | 20260506 15:32:44.202 | INFO | Serialization done, swapping tmp file now... | 20260506 15:32:44.206 | INFO | RTMetrics done | 20260506 15:32:44.506 | INFO | Finished a round of Baz IO: Wrote 2.1 GiB in 3.457 seconds | 20260506 15:32:44.506 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.11% / 0.01% / 62.88% | 20260506 15:32:44.506 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.02416% / 1.58946e-05% / 5.5631e-05% | 20260506 15:32:44.506 | INFO | Baz IO overall: | 20260506 15:32:44.506 | INFO | BytesWritten : 1282294992039 | 20260506 15:32:44.506 | INFO | NumEvents : 620558520284 | 20260506 15:32:46.067 | INFO | Analyzing chunk frames = [2457600,2458112) | 20260506 15:32:46.068 | INFO | MemInfo: { | 20260506 15:32:46.068 | INFO | "gpu" : " 17 GB", | 20260506 15:32:46.068 | INFO | "huge" : "", | 20260506 15:32:46.068 | INFO | "huge_shared" : " 27 GB", | 20260506 15:32:46.068 | INFO | "malloc" : " 14 GB", | 20260506 15:32:46.068 | INFO | "pinned" : " 29 GB" | 20260506 15:32:46.068 | INFO | } | 20260506 15:32:46.068 | INFO | Perf report for the last 5.1399 seconds: | 20260506 15:32:46.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:32:46.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:46.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:46.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:46.068 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 15:32:46.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:46.068 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 15:32:46.068 | INFO | PRE_HQ : 60.8%, 1.9, 59.4ms, 0 | 20260506 15:32:46.068 | INFO | BAZWRITER : 66.8%, 4.8, 165.6ms, 0 | 20260506 15:32:46.068 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 15:32:46.068 | INFO | Uploaded 36GiB to GPU in 2.05011s (17.56GiB/s) | 20260506 15:32:46.068 | INFO | Downloaded 38GiB to GPU in 2.22673s (17.0654GiB/s) | 20260506 15:32:51.208 | INFO | Analyzing chunk frames = [2458112,2458624) | 20260506 15:32:51.208 | INFO | MemInfo: { | 20260506 15:32:51.208 | INFO | "gpu" : " 17 GB", | 20260506 15:32:51.208 | INFO | "huge" : "", | 20260506 15:32:51.208 | INFO | "huge_shared" : " 27 GB", | 20260506 15:32:51.208 | INFO | "malloc" : " 14 GB", | 20260506 15:32:51.208 | INFO | "pinned" : " 29 GB" | 20260506 15:32:51.208 | INFO | } | 20260506 15:32:51.208 | INFO | Perf report for the last 5.14011 seconds: | 20260506 15:32:51.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:32:51.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:51.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:51.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:51.208 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 15:32:51.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:51.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:51.208 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:32:51.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:51.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:51.208 | INFO | Uploaded 36GiB to GPU in 1.98624s (18.1247GiB/s) | 20260506 15:32:51.208 | INFO | Downloaded 34GiB to GPU in 1.93657s (17.5568GiB/s) | 20260506 15:32:56.346 | INFO | Analyzing chunk frames = [2458624,2459136) | 20260506 15:32:56.346 | INFO | MemInfo: { | 20260506 15:32:56.346 | INFO | "gpu" : " 17 GB", | 20260506 15:32:56.346 | INFO | "huge" : "", | 20260506 15:32:56.346 | INFO | "huge_shared" : " 27 GB", | 20260506 15:32:56.346 | INFO | "malloc" : " 14 GB", | 20260506 15:32:56.346 | INFO | "pinned" : " 29 GB" | 20260506 15:32:56.346 | INFO | } | 20260506 15:32:56.346 | INFO | Perf report for the last 5.13833 seconds: | 20260506 15:32:56.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:32:56.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:56.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:56.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:56.346 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:32:56.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:56.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:56.346 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:32:56.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:56.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:32:56.346 | INFO | Uploaded 36GiB to GPU in 1.98449s (18.1406GiB/s) | 20260506 15:32:56.346 | INFO | Downloaded 34GiB to GPU in 1.93744s (17.5489GiB/s) | 20260506 15:33:01.485 | INFO | Analyzing chunk frames = [2459136,2459648) | 20260506 15:33:01.485 | INFO | MemInfo: { | 20260506 15:33:01.485 | INFO | "gpu" : " 17 GB", | 20260506 15:33:01.485 | INFO | "huge" : "", | 20260506 15:33:01.485 | INFO | "huge_shared" : " 27 GB", | 20260506 15:33:01.485 | INFO | "malloc" : " 14 GB", | 20260506 15:33:01.485 | INFO | "pinned" : " 29 GB" | 20260506 15:33:01.485 | INFO | } | 20260506 15:33:01.485 | INFO | Perf report for the last 5.13939 seconds: | 20260506 15:33:01.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:33:01.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:01.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:01.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:01.486 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 15:33:01.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:01.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:01.486 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:33:01.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:01.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:01.486 | INFO | Uploaded 36GiB to GPU in 1.95585s (18.4063GiB/s) | 20260506 15:33:01.486 | INFO | Downloaded 34GiB to GPU in 1.88627s (18.025GiB/s) | 20260506 15:33:06.625 | INFO | Analyzing chunk frames = [2459648,2460160) | 20260506 15:33:06.625 | INFO | MemInfo: { | 20260506 15:33:06.625 | INFO | "gpu" : " 17 GB", | 20260506 15:33:06.625 | INFO | "huge" : "", | 20260506 15:33:06.625 | INFO | "huge_shared" : " 27 GB", | 20260506 15:33:06.625 | INFO | "malloc" : " 14 GB", | 20260506 15:33:06.625 | INFO | "pinned" : " 29 GB" | 20260506 15:33:06.625 | INFO | } | 20260506 15:33:06.625 | INFO | Perf report for the last 5.13975 seconds: | 20260506 15:33:06.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:33:06.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:06.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:06.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:06.625 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 15:33:06.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:06.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:06.625 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:33:06.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:06.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:06.625 | INFO | Uploaded 36GiB to GPU in 1.97992s (18.1825GiB/s) | 20260506 15:33:06.626 | INFO | Downloaded 34GiB to GPU in 1.93454s (17.5753GiB/s) | 20260506 15:33:11.764 | INFO | Analyzing chunk frames = [2460160,2460672) | 20260506 15:33:11.764 | INFO | MemInfo: { | 20260506 15:33:11.764 | INFO | "gpu" : " 17 GB", | 20260506 15:33:11.764 | INFO | "huge" : "", | 20260506 15:33:11.764 | INFO | "huge_shared" : " 27 GB", | 20260506 15:33:11.764 | INFO | "malloc" : " 14 GB", | 20260506 15:33:11.764 | INFO | "pinned" : " 29 GB" | 20260506 15:33:11.764 | INFO | } | 20260506 15:33:11.764 | INFO | Perf report for the last 5.13871 seconds: | 20260506 15:33:11.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:33:11.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:11.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:11.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:11.764 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 15:33:11.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:11.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:11.764 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260506 15:33:11.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:11.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:11.764 | INFO | Uploaded 36GiB to GPU in 2.07073s (17.3852GiB/s) | 20260506 15:33:11.764 | INFO | Downloaded 34GiB to GPU in 1.97094s (17.2507GiB/s) | 20260506 15:33:16.903 | INFO | Analyzing chunk frames = [2460672,2461184) | 20260506 15:33:16.903 | INFO | MemInfo: { | 20260506 15:33:16.903 | INFO | "gpu" : " 17 GB", | 20260506 15:33:16.903 | INFO | "huge" : "", | 20260506 15:33:16.903 | INFO | "huge_shared" : " 27 GB", | 20260506 15:33:16.903 | INFO | "malloc" : " 14 GB", | 20260506 15:33:16.903 | INFO | "pinned" : " 29 GB" | 20260506 15:33:16.903 | INFO | } | 20260506 15:33:16.903 | INFO | Perf report for the last 5.13931 seconds: | 20260506 15:33:16.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:33:16.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:16.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:16.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:16.903 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 15:33:16.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:16.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:16.903 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 15:33:16.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:16.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:16.904 | INFO | Uploaded 36GiB to GPU in 2.02793s (17.7521GiB/s) | 20260506 15:33:16.904 | INFO | Downloaded 34GiB to GPU in 2.01152s (16.9026GiB/s) | 20260506 15:33:22.043 | INFO | Analyzing chunk frames = [2461184,2461696) | 20260506 15:33:22.043 | INFO | MemInfo: { | 20260506 15:33:22.043 | INFO | "gpu" : " 17 GB", | 20260506 15:33:22.043 | INFO | "huge" : "", | 20260506 15:33:22.043 | INFO | "huge_shared" : " 27 GB", | 20260506 15:33:22.043 | INFO | "malloc" : " 14 GB", | 20260506 15:33:22.043 | INFO | "pinned" : " 29 GB" | 20260506 15:33:22.043 | INFO | } | 20260506 15:33:22.043 | INFO | Perf report for the last 5.13956 seconds: | 20260506 15:33:22.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:33:22.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:22.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:22.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:22.043 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 15:33:22.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:22.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:22.043 | INFO | PRE_HQ : 32.6%, 1.0, 17.0ms, 0 | 20260506 15:33:22.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:22.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:22.043 | INFO | Uploaded 36GiB to GPU in 2.02843s (17.7477GiB/s) | 20260506 15:33:22.043 | INFO | Downloaded 34GiB to GPU in 1.97592s (17.2072GiB/s) | 20260506 15:33:22.153 | INFO | Starting a round of Baz IO | 20260506 15:33:25.311 | INFO | Populating RTMetrics report | 20260506 15:33:25.311 | INFO | RTMetrics report populated | 20260506 15:33:25.311 | INFO | Opening tmp RTMetrics json file | 20260506 15:33:25.327 | INFO | Serializing RTMetrics to json | 20260506 15:33:25.387 | INFO | Serialization done, swapping tmp file now... | 20260506 15:33:25.408 | INFO | RTMetrics done | 20260506 15:33:25.433 | INFO | Finished a round of Baz IO: Wrote 2.099 GiB in 3.279 seconds | 20260506 15:33:25.433 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.10% / 0.01% / 62.89% | 20260506 15:33:25.433 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.01718% / 1.98682e-05% / 7.94729e-05% | 20260506 15:33:25.433 | INFO | Baz IO overall: | 20260506 15:33:25.433 | INFO | BytesWritten : 1284548438652 | 20260506 15:33:25.433 | INFO | NumEvents : 621648142331 | 20260506 15:33:27.182 | INFO | Analyzing chunk frames = [2461696,2462208) | 20260506 15:33:27.182 | INFO | MemInfo: { | 20260506 15:33:27.182 | INFO | "gpu" : " 17 GB", | 20260506 15:33:27.182 | INFO | "huge" : "", | 20260506 15:33:27.182 | INFO | "huge_shared" : " 27 GB", | 20260506 15:33:27.182 | INFO | "malloc" : " 14 GB", | 20260506 15:33:27.182 | INFO | "pinned" : " 29 GB" | 20260506 15:33:27.182 | INFO | } | 20260506 15:33:27.182 | INFO | Perf report for the last 5.13928 seconds: | 20260506 15:33:27.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:33:27.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:27.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:27.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:27.182 | INFO | ANALYSIS : 63.0%, 2.9, 95.4ms, 0 | 20260506 15:33:27.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:27.182 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260506 15:33:27.182 | INFO | PRE_HQ : 61.0%, 1.9, 59.5ms, 0 | 20260506 15:33:27.182 | INFO | BAZWRITER : 63.2%, 4.8, 155.2ms, 0 | 20260506 15:33:27.182 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 15:33:27.182 | INFO | Uploaded 36GiB to GPU in 2.05244s (17.5401GiB/s) | 20260506 15:33:27.182 | INFO | Downloaded 38GiB to GPU in 2.23415s (17.0087GiB/s) | 20260506 15:33:32.322 | INFO | Analyzing chunk frames = [2462208,2462720) | 20260506 15:33:32.322 | INFO | MemInfo: { | 20260506 15:33:32.322 | INFO | "gpu" : " 17 GB", | 20260506 15:33:32.322 | INFO | "huge" : "", | 20260506 15:33:32.322 | INFO | "huge_shared" : " 27 GB", | 20260506 15:33:32.322 | INFO | "malloc" : " 14 GB", | 20260506 15:33:32.322 | INFO | "pinned" : " 29 GB" | 20260506 15:33:32.322 | INFO | } | 20260506 15:33:32.322 | INFO | Perf report for the last 5.1399 seconds: | 20260506 15:33:32.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:33:32.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:32.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:32.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:32.322 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:33:32.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:32.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:32.322 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:33:32.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:32.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:32.322 | INFO | Uploaded 36GiB to GPU in 1.97764s (18.2036GiB/s) | 20260506 15:33:32.322 | INFO | Downloaded 34GiB to GPU in 1.94316s (17.4973GiB/s) | 20260506 15:33:37.462 | INFO | Analyzing chunk frames = [2462720,2463232) | 20260506 15:33:37.462 | INFO | MemInfo: { | 20260506 15:33:37.462 | INFO | "gpu" : " 17 GB", | 20260506 15:33:37.462 | INFO | "huge" : "", | 20260506 15:33:37.462 | INFO | "huge_shared" : " 27 GB", | 20260506 15:33:37.462 | INFO | "malloc" : " 14 GB", | 20260506 15:33:37.462 | INFO | "pinned" : " 29 GB" | 20260506 15:33:37.462 | INFO | } | 20260506 15:33:37.462 | INFO | Perf report for the last 5.13958 seconds: | 20260506 15:33:37.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:33:37.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:37.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:37.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:37.462 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:33:37.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:37.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:37.462 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:33:37.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:37.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:37.462 | INFO | Uploaded 36GiB to GPU in 1.99067s (18.0844GiB/s) | 20260506 15:33:37.462 | INFO | Downloaded 34GiB to GPU in 1.94101s (17.5167GiB/s) | 20260506 15:33:42.600 | INFO | Analyzing chunk frames = [2463232,2463744) | 20260506 15:33:42.600 | INFO | MemInfo: { | 20260506 15:33:42.600 | INFO | "gpu" : " 17 GB", | 20260506 15:33:42.600 | INFO | "huge" : "", | 20260506 15:33:42.601 | INFO | "huge_shared" : " 27 GB", | 20260506 15:33:42.601 | INFO | "malloc" : " 14 GB", | 20260506 15:33:42.601 | INFO | "pinned" : " 29 GB" | 20260506 15:33:42.601 | INFO | } | 20260506 15:33:42.601 | INFO | Perf report for the last 5.1389 seconds: | 20260506 15:33:42.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:33:42.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:42.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:42.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:42.601 | INFO | ANALYSIS : 60.5%, 3.0, 93.9ms, 0 | 20260506 15:33:42.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:42.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:42.601 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:33:42.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:42.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:42.601 | INFO | Uploaded 36GiB to GPU in 1.99903s (18.0087GiB/s) | 20260506 15:33:42.601 | INFO | Downloaded 34GiB to GPU in 1.9092s (17.8085GiB/s) | 20260506 15:33:47.739 | INFO | Analyzing chunk frames = [2463744,2464256) | 20260506 15:33:47.739 | INFO | MemInfo: { | 20260506 15:33:47.739 | INFO | "gpu" : " 17 GB", | 20260506 15:33:47.739 | INFO | "huge" : "", | 20260506 15:33:47.739 | INFO | "huge_shared" : " 27 GB", | 20260506 15:33:47.739 | INFO | "malloc" : " 14 GB", | 20260506 15:33:47.739 | INFO | "pinned" : " 29 GB" | 20260506 15:33:47.739 | INFO | } | 20260506 15:33:47.739 | INFO | Perf report for the last 5.13875 seconds: | 20260506 15:33:47.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:33:47.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:47.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:47.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:47.739 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 15:33:47.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:47.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:47.739 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:33:47.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:47.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:47.740 | INFO | Uploaded 36GiB to GPU in 1.98927s (18.0971GiB/s) | 20260506 15:33:47.740 | INFO | Downloaded 34GiB to GPU in 1.93788s (17.545GiB/s) | 20260506 15:33:52.879 | INFO | Analyzing chunk frames = [2464256,2464768) | 20260506 15:33:52.879 | INFO | MemInfo: { | 20260506 15:33:52.879 | INFO | "gpu" : " 17 GB", | 20260506 15:33:52.879 | INFO | "huge" : "", | 20260506 15:33:52.879 | INFO | "huge_shared" : " 27 GB", | 20260506 15:33:52.879 | INFO | "malloc" : " 14 GB", | 20260506 15:33:52.879 | INFO | "pinned" : " 29 GB" | 20260506 15:33:52.879 | INFO | } | 20260506 15:33:52.879 | INFO | Perf report for the last 5.13942 seconds: | 20260506 15:33:52.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:33:52.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:52.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:52.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:52.879 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:33:52.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:52.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:52.879 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:33:52.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:52.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:52.879 | INFO | Uploaded 36GiB to GPU in 2.07241s (17.3711GiB/s) | 20260506 15:33:52.879 | INFO | Downloaded 34GiB to GPU in 1.97602s (17.2063GiB/s) | 20260506 15:33:58.018 | INFO | Analyzing chunk frames = [2464768,2465280) | 20260506 15:33:58.018 | INFO | MemInfo: { | 20260506 15:33:58.018 | INFO | "gpu" : " 17 GB", | 20260506 15:33:58.018 | INFO | "huge" : "", | 20260506 15:33:58.019 | INFO | "huge_shared" : " 27 GB", | 20260506 15:33:58.019 | INFO | "malloc" : " 14 GB", | 20260506 15:33:58.019 | INFO | "pinned" : " 29 GB" | 20260506 15:33:58.019 | INFO | } | 20260506 15:33:58.019 | INFO | Perf report for the last 5.14005 seconds: | 20260506 15:33:58.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:33:58.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:58.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:58.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:58.019 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 15:33:58.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:58.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:58.019 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:33:58.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:58.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:33:58.019 | INFO | Uploaded 36GiB to GPU in 2.01843s (17.8356GiB/s) | 20260506 15:33:58.019 | INFO | Downloaded 34GiB to GPU in 2.01428s (16.8795GiB/s) | 20260506 15:34:03.158 | INFO | Analyzing chunk frames = [2465280,2465792) | 20260506 15:34:03.158 | INFO | MemInfo: { | 20260506 15:34:03.158 | INFO | "gpu" : " 17 GB", | 20260506 15:34:03.158 | INFO | "huge" : "", | 20260506 15:34:03.158 | INFO | "huge_shared" : " 27 GB", | 20260506 15:34:03.158 | INFO | "malloc" : " 14 GB", | 20260506 15:34:03.158 | INFO | "pinned" : " 29 GB" | 20260506 15:34:03.158 | INFO | } | 20260506 15:34:03.158 | INFO | Perf report for the last 5.13899 seconds: | 20260506 15:34:03.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:34:03.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:03.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:03.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:03.158 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 15:34:03.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:03.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:03.158 | INFO | PRE_HQ : 31.4%, 1.0, 16.4ms, 0 | 20260506 15:34:03.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:03.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:03.158 | INFO | Uploaded 36GiB to GPU in 2.0605s (17.4715GiB/s) | 20260506 15:34:03.158 | INFO | Downloaded 34GiB to GPU in 1.98453s (17.1325GiB/s) | 20260506 15:34:03.158 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:34:03.158 | INFO | Total mem usage: 17.4 GiB | 20260506 15:34:03.158 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:34:03.158 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -11.5%) | 20260506 15:34:03.158 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:34:03.158 | INFO | High Water Breakdown: | 20260506 15:34:03.158 | INFO | 61.2% : 9.56 GiB ( 87.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:34:03.158 | INFO | 38.6% : 6.03 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:34:03.158 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:34:03.158 | INFO | ------------------------------------------ | 20260506 15:34:03.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:34:03.158 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:34:03.158 | INFO | Total mem usage: 26.5 GiB | 20260506 15:34:03.158 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:34:03.158 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:34:03.158 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 15:34:03.158 | INFO | High Water Breakdown: | 20260506 15:34:03.158 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:34:03.158 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:34:03.158 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:34:03.158 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:34:03.158 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:34:03.158 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:34:03.158 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:34:03.158 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:34:03.158 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:34:03.158 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:34:03.158 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:34:03.158 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:34:03.158 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:34:03.158 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:34:03.158 | INFO | ------------------------------------------ | 20260506 15:34:03.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:34:03.158 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:34:03.158 | INFO | Total mem usage: 25.6 GiB | 20260506 15:34:03.158 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:34:03.158 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:34:03.159 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:34:03.159 | INFO | High Water Breakdown: | 20260506 15:34:03.159 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:34:03.159 | INFO | ------------------------------------------ | 20260506 15:34:03.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:34:03.159 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:34:03.159 | INFO | Total mem usage: 15.7 GiB | 20260506 15:34:03.159 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:34:03.159 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:34:03.159 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 15:34:03.159 | INFO | High Water Breakdown: | 20260506 15:34:03.159 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:34:03.159 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:34:03.159 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:34:03.159 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:34:03.159 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:34:03.159 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:34:03.159 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:34:03.159 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:34:03.159 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:34:03.159 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:34:03.159 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:34:03.159 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:34:03.159 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:34:03.159 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:34:03.159 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:34:03.159 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:34:03.159 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:34:03.159 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:34:03.159 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:34:03.159 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:34:03.159 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:34:03.159 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:34:03.159 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:34:03.159 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:34:03.159 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:34:03.159 | INFO | ------------------------------------------ | 20260506 15:34:03.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:34:03.159 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:34:03.159 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:34:03.159 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:34:03.159 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.52e+01, 1.36e+01 / 1.81e+00 | 20260506 15:34:03.159 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.00e-01, 1.00e-02 / 6.14e-03 | 20260506 15:34:03.159 | INFO | PerfReport: 11.56%, Baseline -- 3.64e+00 / 4.40e+00, 5.59e-01 / 4.15e-01 | 20260506 15:34:03.159 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.87e-01 / 1.51e-01 | 20260506 15:34:03.159 | INFO | PerfReport: 15.85%, DME -- 3.99e+01 / 4.37e+01, 3.52e+00 / 1.77e+00 | 20260506 15:34:03.159 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.48e+00 / 1.80e+00 | 20260506 15:34:03.159 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.32e+00 / 2.88e+00, 3.99e-01 / 2.48e-01 | 20260506 15:34:03.159 | INFO | PerfReport: 3.55%, Metrics -- 1.12e+00 / 1.50e+00, 2.88e-01 / 1.55e-01 | 20260506 15:34:03.159 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.88e-01 / 4.42e-01, 6.06e-02 / 2.39e-02 | 20260506 15:34:03.159 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:34:03.159 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:34:03.159 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:34:03.159 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 15:34:03.159 | INFO | PerfReport: 21.53%, Upload -- 2.10e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260506 15:34:03.159 | INFO | PerfReport: 21.24%, Download -- 2.07e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 15:34:03.159 | INFO | PerfReport: 32.25%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 15:34:03.159 | INFO | PerfReport: 1.43%, LockUpload -- 1.40e+00 / 0.00e+00, 5.74e+00 / 0.00e+00 | 20260506 15:34:03.159 | INFO | PerfReport: 1.13%, LockDownload -- 1.11e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260506 15:34:03.159 | INFO | PerfReport: 22.40%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 15:34:03.159 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260506 15:34:03.284 | INFO | Starting a round of Baz IO | 20260506 15:34:06.426 | INFO | Populating RTMetrics report | 20260506 15:34:06.426 | INFO | RTMetrics report populated | 20260506 15:34:06.426 | INFO | Opening tmp RTMetrics json file | 20260506 15:34:06.688 | INFO | Serializing RTMetrics to json | 20260506 15:34:06.715 | INFO | Serialization done, swapping tmp file now... | 20260506 15:34:06.718 | INFO | RTMetrics done | 20260506 15:34:06.809 | INFO | Finished a round of Baz IO: Wrote 2.1 GiB in 3.524 seconds | 20260506 15:34:06.809 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.09% / 0.01% / 62.90% | 20260506 15:34:06.809 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.01358% / 1.19209e-05% / 7.54992e-05% | 20260506 15:34:06.809 | INFO | Baz IO overall: | 20260506 15:34:06.809 | INFO | BytesWritten : 1286802822638 | 20260506 15:34:06.809 | INFO | NumEvents : 622738502861 | 20260506 15:34:08.298 | INFO | Analyzing chunk frames = [2465792,2466304) | 20260506 15:34:08.298 | INFO | MemInfo: { | 20260506 15:34:08.298 | INFO | "gpu" : " 17 GB", | 20260506 15:34:08.299 | INFO | "huge" : "", | 20260506 15:34:08.299 | INFO | "huge_shared" : " 27 GB", | 20260506 15:34:08.299 | INFO | "malloc" : " 14 GB", | 20260506 15:34:08.299 | INFO | "pinned" : " 29 GB" | 20260506 15:34:08.299 | INFO | } | 20260506 15:34:08.299 | INFO | Perf report for the last 5.14077 seconds: | 20260506 15:34:08.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:34:08.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:08.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:08.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:08.299 | INFO | ANALYSIS : 63.0%, 2.9, 95.6ms, 0 | 20260506 15:34:08.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:08.299 | INFO | RT_METRICS : 49.4%, 1.0, 25.9ms, 0 | 20260506 15:34:08.299 | INFO | PRE_HQ : 60.7%, 1.9, 58.9ms, 0 | 20260506 15:34:08.299 | INFO | BAZWRITER : 67.9%, 4.8, 166.5ms, 0 | 20260506 15:34:08.299 | INFO | BAZ_SAMPLING : 3.9%, 1.0, 2.0ms, 0 | 20260506 15:34:08.299 | INFO | Uploaded 36GiB to GPU in 2.08053s (17.3033GiB/s) | 20260506 15:34:08.299 | INFO | Downloaded 38GiB to GPU in 2.23673s (16.9891GiB/s) | 20260506 15:34:13.437 | INFO | Analyzing chunk frames = [2466304,2466816) | 20260506 15:34:13.437 | INFO | MemInfo: { | 20260506 15:34:13.437 | INFO | "gpu" : " 17 GB", | 20260506 15:34:13.437 | INFO | "huge" : "", | 20260506 15:34:13.437 | INFO | "huge_shared" : " 27 GB", | 20260506 15:34:13.437 | INFO | "malloc" : " 14 GB", | 20260506 15:34:13.437 | INFO | "pinned" : " 29 GB" | 20260506 15:34:13.437 | INFO | } | 20260506 15:34:13.438 | INFO | Perf report for the last 5.1389 seconds: | 20260506 15:34:13.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:34:13.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:13.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:13.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:13.438 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 15:34:13.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:13.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:13.438 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 15:34:13.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:13.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:13.438 | INFO | Uploaded 36GiB to GPU in 1.99613s (18.0349GiB/s) | 20260506 15:34:13.438 | INFO | Downloaded 34GiB to GPU in 1.93097s (17.6077GiB/s) | 20260506 15:34:18.576 | INFO | Analyzing chunk frames = [2466816,2467328) | 20260506 15:34:18.576 | INFO | MemInfo: { | 20260506 15:34:18.576 | INFO | "gpu" : " 17 GB", | 20260506 15:34:18.576 | INFO | "huge" : "", | 20260506 15:34:18.576 | INFO | "huge_shared" : " 27 GB", | 20260506 15:34:18.576 | INFO | "malloc" : " 14 GB", | 20260506 15:34:18.576 | INFO | "pinned" : " 29 GB" | 20260506 15:34:18.576 | INFO | } | 20260506 15:34:18.576 | INFO | Perf report for the last 5.13832 seconds: | 20260506 15:34:18.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:34:18.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:18.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:18.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:18.576 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:34:18.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:18.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:18.576 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:34:18.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:18.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:18.576 | INFO | Uploaded 36GiB to GPU in 1.99423s (18.0521GiB/s) | 20260506 15:34:18.576 | INFO | Downloaded 34GiB to GPU in 1.94475s (17.483GiB/s) | 20260506 15:34:23.715 | INFO | Analyzing chunk frames = [2467328,2467840) | 20260506 15:34:23.715 | INFO | MemInfo: { | 20260506 15:34:23.715 | INFO | "gpu" : " 17 GB", | 20260506 15:34:23.715 | INFO | "huge" : "", | 20260506 15:34:23.715 | INFO | "huge_shared" : " 27 GB", | 20260506 15:34:23.715 | INFO | "malloc" : " 14 GB", | 20260506 15:34:23.715 | INFO | "pinned" : " 29 GB" | 20260506 15:34:23.715 | INFO | } | 20260506 15:34:23.715 | INFO | Perf report for the last 5.13894 seconds: | 20260506 15:34:23.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:34:23.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:23.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:23.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:23.715 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 15:34:23.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:23.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:23.715 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:34:23.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:23.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:23.715 | INFO | Uploaded 36GiB to GPU in 2.02307s (17.7947GiB/s) | 20260506 15:34:23.715 | INFO | Downloaded 34GiB to GPU in 1.91018s (17.7994GiB/s) | 20260506 15:34:28.854 | INFO | Analyzing chunk frames = [2467840,2468352) | 20260506 15:34:28.854 | INFO | MemInfo: { | 20260506 15:34:28.854 | INFO | "gpu" : " 17 GB", | 20260506 15:34:28.854 | INFO | "huge" : "", | 20260506 15:34:28.854 | INFO | "huge_shared" : " 27 GB", | 20260506 15:34:28.854 | INFO | "malloc" : " 14 GB", | 20260506 15:34:28.854 | INFO | "pinned" : " 29 GB" | 20260506 15:34:28.854 | INFO | } | 20260506 15:34:28.854 | INFO | Perf report for the last 5.13948 seconds: | 20260506 15:34:28.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:34:28.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:28.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:28.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:28.854 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 15:34:28.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:28.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:28.854 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:34:28.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:28.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:28.855 | INFO | Uploaded 36GiB to GPU in 2.03304s (17.7075GiB/s) | 20260506 15:34:28.855 | INFO | Downloaded 34GiB to GPU in 1.93083s (17.609GiB/s) | 20260506 15:34:33.993 | INFO | Analyzing chunk frames = [2468352,2468864) | 20260506 15:34:33.993 | INFO | MemInfo: { | 20260506 15:34:33.993 | INFO | "gpu" : " 17 GB", | 20260506 15:34:33.993 | INFO | "huge" : "", | 20260506 15:34:33.994 | INFO | "huge_shared" : " 27 GB", | 20260506 15:34:33.994 | INFO | "malloc" : " 14 GB", | 20260506 15:34:33.994 | INFO | "pinned" : " 29 GB" | 20260506 15:34:33.994 | INFO | } | 20260506 15:34:33.994 | INFO | Perf report for the last 5.13927 seconds: | 20260506 15:34:33.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:34:33.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:33.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:33.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:33.994 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:34:33.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:33.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:33.994 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:34:33.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:33.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:33.994 | INFO | Uploaded 36GiB to GPU in 2.07945s (17.3123GiB/s) | 20260506 15:34:33.994 | INFO | Downloaded 34GiB to GPU in 1.97152s (17.2455GiB/s) | 20260506 15:34:39.133 | INFO | Analyzing chunk frames = [2468864,2469376) | 20260506 15:34:39.133 | INFO | MemInfo: { | 20260506 15:34:39.133 | INFO | "gpu" : " 17 GB", | 20260506 15:34:39.133 | INFO | "huge" : "", | 20260506 15:34:39.133 | INFO | "huge_shared" : " 27 GB", | 20260506 15:34:39.133 | INFO | "malloc" : " 14 GB", | 20260506 15:34:39.133 | INFO | "pinned" : " 29 GB" | 20260506 15:34:39.133 | INFO | } | 20260506 15:34:39.133 | INFO | Perf report for the last 5.13976 seconds: | 20260506 15:34:39.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:34:39.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:39.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:39.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:39.133 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:34:39.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:39.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:39.133 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 15:34:39.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:39.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:39.134 | INFO | Uploaded 36GiB to GPU in 2.02644s (17.7651GiB/s) | 20260506 15:34:39.134 | INFO | Downloaded 34GiB to GPU in 2.00941s (16.9204GiB/s) | 20260506 15:34:44.273 | INFO | Analyzing chunk frames = [2469376,2469888) | 20260506 15:34:44.273 | INFO | MemInfo: { | 20260506 15:34:44.273 | INFO | "gpu" : " 17 GB", | 20260506 15:34:44.273 | INFO | "huge" : "", | 20260506 15:34:44.273 | INFO | "huge_shared" : " 27 GB", | 20260506 15:34:44.273 | INFO | "malloc" : " 14 GB", | 20260506 15:34:44.273 | INFO | "pinned" : " 29 GB" | 20260506 15:34:44.273 | INFO | } | 20260506 15:34:44.273 | INFO | Perf report for the last 5.13959 seconds: | 20260506 15:34:44.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:34:44.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:44.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:44.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:44.273 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:34:44.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:44.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:44.273 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:34:44.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:44.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:44.273 | INFO | Uploaded 36GiB to GPU in 2.02861s (17.7462GiB/s) | 20260506 15:34:44.273 | INFO | Downloaded 34GiB to GPU in 1.98629s (17.1174GiB/s) | 20260506 15:34:44.386 | INFO | Starting a round of Baz IO | 20260506 15:34:47.538 | INFO | Populating RTMetrics report | 20260506 15:34:47.539 | INFO | RTMetrics report populated | 20260506 15:34:47.539 | INFO | Opening tmp RTMetrics json file | 20260506 15:34:47.588 | INFO | Serializing RTMetrics to json | 20260506 15:34:47.669 | INFO | Serialization done, swapping tmp file now... | 20260506 15:34:47.670 | INFO | RTMetrics done | 20260506 15:34:47.905 | INFO | Finished a round of Baz IO: Wrote 2.1 GiB in 3.519 seconds | 20260506 15:34:47.906 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.08% / 0.01% / 62.91% | 20260506 15:34:47.906 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.01056% / 2.78155e-05% / 6.35783e-05% | 20260506 15:34:47.906 | INFO | Baz IO overall: | 20260506 15:34:47.906 | INFO | BytesWritten : 1289057580690 | 20260506 15:34:47.906 | INFO | NumEvents : 623829099211 | 20260506 15:34:49.411 | INFO | Analyzing chunk frames = [2469888,2470400) | 20260506 15:34:49.411 | INFO | MemInfo: { | 20260506 15:34:49.411 | INFO | "gpu" : " 17 GB", | 20260506 15:34:49.411 | INFO | "huge" : "", | 20260506 15:34:49.411 | INFO | "huge_shared" : " 27 GB", | 20260506 15:34:49.411 | INFO | "malloc" : " 14 GB", | 20260506 15:34:49.411 | INFO | "pinned" : " 29 GB" | 20260506 15:34:49.411 | INFO | } | 20260506 15:34:49.411 | INFO | Perf report for the last 5.13845 seconds: | 20260506 15:34:49.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:34:49.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:49.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:49.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:49.411 | INFO | ANALYSIS : 63.0%, 2.9, 95.4ms, 0 | 20260506 15:34:49.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:49.412 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260506 15:34:49.412 | INFO | PRE_HQ : 60.8%, 1.9, 59.1ms, 0 | 20260506 15:34:49.412 | INFO | BAZWRITER : 67.9%, 4.8, 168.3ms, 0 | 20260506 15:34:49.412 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.2ms, 0 | 20260506 15:34:49.412 | INFO | Uploaded 36GiB to GPU in 2.05563s (17.5129GiB/s) | 20260506 15:34:49.412 | INFO | Downloaded 38GiB to GPU in 2.22923s (17.0463GiB/s) | 20260506 15:34:54.551 | INFO | Analyzing chunk frames = [2470400,2470912) | 20260506 15:34:54.551 | INFO | MemInfo: { | 20260506 15:34:54.551 | INFO | "gpu" : " 17 GB", | 20260506 15:34:54.551 | INFO | "huge" : "", | 20260506 15:34:54.551 | INFO | "huge_shared" : " 27 GB", | 20260506 15:34:54.551 | INFO | "malloc" : " 14 GB", | 20260506 15:34:54.551 | INFO | "pinned" : " 29 GB" | 20260506 15:34:54.551 | INFO | } | 20260506 15:34:54.551 | INFO | Perf report for the last 5.13951 seconds: | 20260506 15:34:54.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:34:54.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:54.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:54.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:54.551 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 15:34:54.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:54.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:54.551 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:34:54.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:54.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:54.551 | INFO | Uploaded 36GiB to GPU in 1.97649s (18.2141GiB/s) | 20260506 15:34:54.551 | INFO | Downloaded 34GiB to GPU in 1.944s (17.4897GiB/s) | 20260506 15:34:59.691 | INFO | Analyzing chunk frames = [2470912,2471424) | 20260506 15:34:59.691 | INFO | MemInfo: { | 20260506 15:34:59.691 | INFO | "gpu" : " 17 GB", | 20260506 15:34:59.691 | INFO | "huge" : "", | 20260506 15:34:59.691 | INFO | "huge_shared" : " 27 GB", | 20260506 15:34:59.691 | INFO | "malloc" : " 14 GB", | 20260506 15:34:59.691 | INFO | "pinned" : " 29 GB" | 20260506 15:34:59.691 | INFO | } | 20260506 15:34:59.691 | INFO | Perf report for the last 5.14034 seconds: | 20260506 15:34:59.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:34:59.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:59.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:59.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:59.691 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 15:34:59.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:59.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:59.691 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:34:59.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:59.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:34:59.692 | INFO | Uploaded 36GiB to GPU in 1.97552s (18.223GiB/s) | 20260506 15:34:59.692 | INFO | Downloaded 34GiB to GPU in 1.94528s (17.4782GiB/s) | 20260506 15:35:04.830 | INFO | Analyzing chunk frames = [2471424,2471936) | 20260506 15:35:04.830 | INFO | MemInfo: { | 20260506 15:35:04.830 | INFO | "gpu" : " 17 GB", | 20260506 15:35:04.830 | INFO | "huge" : "", | 20260506 15:35:04.830 | INFO | "huge_shared" : " 27 GB", | 20260506 15:35:04.830 | INFO | "malloc" : " 14 GB", | 20260506 15:35:04.830 | INFO | "pinned" : " 29 GB" | 20260506 15:35:04.830 | INFO | } | 20260506 15:35:04.830 | INFO | Perf report for the last 5.13896 seconds: | 20260506 15:35:04.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:35:04.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:04.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:04.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:04.830 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 15:35:04.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:04.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:04.830 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:35:04.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:04.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:04.831 | INFO | Uploaded 36GiB to GPU in 2.01698s (17.8485GiB/s) | 20260506 15:35:04.831 | INFO | Downloaded 34GiB to GPU in 1.91392s (17.7646GiB/s) | 20260506 15:35:09.969 | INFO | Analyzing chunk frames = [2471936,2472448) | 20260506 15:35:09.969 | INFO | MemInfo: { | 20260506 15:35:09.969 | INFO | "gpu" : " 17 GB", | 20260506 15:35:09.969 | INFO | "huge" : "", | 20260506 15:35:09.969 | INFO | "huge_shared" : " 27 GB", | 20260506 15:35:09.969 | INFO | "malloc" : " 14 GB", | 20260506 15:35:09.969 | INFO | "pinned" : " 29 GB" | 20260506 15:35:09.969 | INFO | } | 20260506 15:35:09.969 | INFO | Perf report for the last 5.1388 seconds: | 20260506 15:35:09.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:35:09.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:09.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:09.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:09.969 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 15:35:09.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:09.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:09.969 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:35:09.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:09.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:09.969 | INFO | Uploaded 36GiB to GPU in 1.97062s (18.2684GiB/s) | 20260506 15:35:09.969 | INFO | Downloaded 34GiB to GPU in 1.94185s (17.5091GiB/s) | 20260506 15:35:15.108 | INFO | Analyzing chunk frames = [2472448,2472960) | 20260506 15:35:15.108 | INFO | MemInfo: { | 20260506 15:35:15.108 | INFO | "gpu" : " 17 GB", | 20260506 15:35:15.108 | INFO | "huge" : "", | 20260506 15:35:15.108 | INFO | "huge_shared" : " 27 GB", | 20260506 15:35:15.108 | INFO | "malloc" : " 14 GB", | 20260506 15:35:15.108 | INFO | "pinned" : " 29 GB" | 20260506 15:35:15.108 | INFO | } | 20260506 15:35:15.108 | INFO | Perf report for the last 5.13922 seconds: | 20260506 15:35:15.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:35:15.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:15.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:15.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:15.108 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:35:15.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:15.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:15.108 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 15:35:15.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:15.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:15.109 | INFO | Uploaded 36GiB to GPU in 2.06894s (17.4002GiB/s) | 20260506 15:35:15.109 | INFO | Downloaded 34GiB to GPU in 1.97487s (17.2164GiB/s) | 20260506 15:35:20.248 | INFO | Analyzing chunk frames = [2472960,2473472) | 20260506 15:35:20.248 | INFO | MemInfo: { | 20260506 15:35:20.248 | INFO | "gpu" : " 17 GB", | 20260506 15:35:20.248 | INFO | "huge" : "", | 20260506 15:35:20.248 | INFO | "huge_shared" : " 27 GB", | 20260506 15:35:20.248 | INFO | "malloc" : " 14 GB", | 20260506 15:35:20.248 | INFO | "pinned" : " 29 GB" | 20260506 15:35:20.248 | INFO | } | 20260506 15:35:20.248 | INFO | Perf report for the last 5.1398 seconds: | 20260506 15:35:20.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:35:20.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:20.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:20.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:20.248 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:35:20.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:20.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:20.248 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 15:35:20.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:20.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:20.248 | INFO | Uploaded 36GiB to GPU in 2.01244s (17.8887GiB/s) | 20260506 15:35:20.249 | INFO | Downloaded 34GiB to GPU in 2.00974s (16.9176GiB/s) | 20260506 15:35:25.389 | INFO | Analyzing chunk frames = [2473472,2473984) | 20260506 15:35:25.389 | INFO | MemInfo: { | 20260506 15:35:25.389 | INFO | "gpu" : " 17 GB", | 20260506 15:35:25.389 | INFO | "huge" : "", | 20260506 15:35:25.389 | INFO | "huge_shared" : " 27 GB", | 20260506 15:35:25.389 | INFO | "malloc" : " 14 GB", | 20260506 15:35:25.389 | INFO | "pinned" : " 29 GB" | 20260506 15:35:25.389 | INFO | } | 20260506 15:35:25.389 | INFO | Perf report for the last 5.14081 seconds: | 20260506 15:35:25.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:35:25.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:25.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:25.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:25.389 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:35:25.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:25.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:25.389 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 15:35:25.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:25.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:25.389 | INFO | Uploaded 36GiB to GPU in 2.03635s (17.6787GiB/s) | 20260506 15:35:25.389 | INFO | Downloaded 34GiB to GPU in 1.98574s (17.1221GiB/s) | 20260506 15:35:25.389 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:35:25.389 | INFO | Total mem usage: 17.4 GiB | 20260506 15:35:25.389 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:35:25.389 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -15.5%) | 20260506 15:35:25.389 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:35:25.389 | INFO | High Water Breakdown: | 20260506 15:35:25.389 | INFO | 60.7% : 9.15 GiB ( 83.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:35:25.389 | INFO | 39.2% : 5.9 GiB ( 88.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:35:25.389 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:35:25.389 | INFO | ------------------------------------------ | 20260506 15:35:25.389 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:35:25.389 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:35:25.389 | INFO | Total mem usage: 26.5 GiB | 20260506 15:35:25.389 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:35:25.389 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:35:25.389 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 15:35:25.389 | INFO | High Water Breakdown: | 20260506 15:35:25.389 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:35:25.389 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:35:25.389 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:35:25.389 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:35:25.389 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:35:25.390 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:35:25.390 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:35:25.390 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:35:25.390 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:35:25.390 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:35:25.390 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:35:25.390 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:35:25.390 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:35:25.390 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:35:25.390 | INFO | ------------------------------------------ | 20260506 15:35:25.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:35:25.390 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:35:25.390 | INFO | Total mem usage: 25.6 GiB | 20260506 15:35:25.390 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:35:25.390 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:35:25.390 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:35:25.390 | INFO | High Water Breakdown: | 20260506 15:35:25.390 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:35:25.390 | INFO | ------------------------------------------ | 20260506 15:35:25.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:35:25.390 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:35:25.390 | INFO | Total mem usage: 15.7 GiB | 20260506 15:35:25.390 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:35:25.390 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:35:25.390 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 15:35:25.390 | INFO | High Water Breakdown: | 20260506 15:35:25.390 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:35:25.390 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:35:25.390 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:35:25.390 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:35:25.390 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:35:25.390 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:35:25.390 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:35:25.390 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:35:25.390 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:35:25.390 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:35:25.390 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:35:25.390 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:35:25.390 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:35:25.390 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:35:25.390 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:35:25.390 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:35:25.390 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:35:25.390 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:35:25.390 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:35:25.390 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:35:25.390 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:35:25.390 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:35:25.390 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:35:25.390 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:35:25.390 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:35:25.390 | INFO | ------------------------------------------ | 20260506 15:35:25.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:35:25.390 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:35:25.390 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:35:25.390 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:35:25.390 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.58e+01, 1.36e+01 / 2.19e+00 | 20260506 15:35:25.390 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 9.98e-02, 1.01e-02 / 6.00e-03 | 20260506 15:35:25.390 | INFO | PerfReport: 11.57%, Baseline -- 3.63e+00 / 4.39e+00, 5.50e-01 / 3.99e-01 | 20260506 15:35:25.390 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.47e+00, 1.84e-01 / 1.48e-01 | 20260506 15:35:25.390 | INFO | PerfReport: 15.94%, DME -- 4.01e+01 / 4.45e+01, 3.94e+00 / 2.15e+00 | 20260506 15:35:25.390 | INFO | PerfReport: 56.77%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.38e+00 / 1.54e+00 | 20260506 15:35:25.390 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.33e+00 / 2.98e+00, 4.04e-01 / 1.99e-01 | 20260506 15:35:25.390 | INFO | PerfReport: 3.54%, Metrics -- 1.11e+00 / 1.47e+00, 2.84e-01 / 1.97e-01 | 20260506 15:35:25.390 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.88e-01 / 4.39e-01, 6.02e-02 / 3.02e-02 | 20260506 15:35:25.390 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:35:25.390 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:35:25.390 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:35:25.390 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 15:35:25.390 | INFO | PerfReport: 21.59%, Upload -- 2.11e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260506 15:35:25.390 | INFO | PerfReport: 21.25%, Download -- 2.07e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 15:35:25.390 | INFO | PerfReport: 32.18%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 15:35:25.390 | INFO | PerfReport: 1.53%, LockUpload -- 1.50e+00 / 0.00e+00, 5.90e+00 / 0.00e+00 | 20260506 15:35:25.390 | INFO | PerfReport: 1.15%, LockDownload -- 1.12e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260506 15:35:25.390 | INFO | PerfReport: 22.27%, LockCompute -- 2.17e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 15:35:25.390 | INFO | PerfReport: 0.02%, Misc -- 1.55e-02 / 0.00e+00, 2.62e-02 / 0.00e+00 | 20260506 15:35:25.509 | INFO | Starting a round of Baz IO | 20260506 15:35:28.653 | INFO | Populating RTMetrics report | 20260506 15:35:28.654 | INFO | RTMetrics report populated | 20260506 15:35:28.654 | INFO | Opening tmp RTMetrics json file | 20260506 15:35:28.865 | INFO | Serializing RTMetrics to json | 20260506 15:35:28.937 | INFO | Serialization done, swapping tmp file now... | 20260506 15:35:28.966 | INFO | RTMetrics done | 20260506 15:35:29.200 | INFO | Finished a round of Baz IO: Wrote 2.099 GiB in 3.691 seconds | 20260506 15:35:29.200 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.07% / 0.01% / 62.92% | 20260506 15:35:29.200 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.00529% / 1.19209e-05% / 5.96046e-05% | 20260506 15:35:29.202 | INFO | Baz IO overall: | 20260506 15:35:29.202 | INFO | BytesWritten : 1291311574446 | 20260506 15:35:29.202 | INFO | NumEvents : 624919183523 | 20260506 15:35:30.527 | INFO | Analyzing chunk frames = [2473984,2474496) | 20260506 15:35:30.527 | INFO | MemInfo: { | 20260506 15:35:30.527 | INFO | "gpu" : " 17 GB", | 20260506 15:35:30.527 | INFO | "huge" : "", | 20260506 15:35:30.527 | INFO | "huge_shared" : " 27 GB", | 20260506 15:35:30.527 | INFO | "malloc" : " 14 GB", | 20260506 15:35:30.527 | INFO | "pinned" : " 29 GB" | 20260506 15:35:30.527 | INFO | } | 20260506 15:35:30.527 | INFO | Perf report for the last 5.13792 seconds: | 20260506 15:35:30.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:35:30.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:30.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:30.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:30.527 | INFO | ANALYSIS : 63.0%, 2.9, 95.3ms, 0 | 20260506 15:35:30.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:30.527 | INFO | RT_METRICS : 49.8%, 1.0, 26.1ms, 0 | 20260506 15:35:30.527 | INFO | PRE_HQ : 60.8%, 1.9, 59.2ms, 0 | 20260506 15:35:30.527 | INFO | BAZWRITER : 71.4%, 4.8, 175.4ms, 0 | 20260506 15:35:30.527 | INFO | BAZ_SAMPLING : 4.0%, 1.0, 2.1ms, 0 | 20260506 15:35:30.527 | INFO | Uploaded 36GiB to GPU in 2.08918s (17.2316GiB/s) | 20260506 15:35:30.527 | INFO | Downloaded 38GiB to GPU in 2.22999s (17.0405GiB/s) | 20260506 15:35:35.666 | INFO | Analyzing chunk frames = [2474496,2475008) | 20260506 15:35:35.666 | INFO | MemInfo: { | 20260506 15:35:35.666 | INFO | "gpu" : " 17 GB", | 20260506 15:35:35.666 | INFO | "huge" : "", | 20260506 15:35:35.666 | INFO | "huge_shared" : " 27 GB", | 20260506 15:35:35.666 | INFO | "malloc" : " 14 GB", | 20260506 15:35:35.666 | INFO | "pinned" : " 29 GB" | 20260506 15:35:35.666 | INFO | } | 20260506 15:35:35.666 | INFO | Perf report for the last 5.13911 seconds: | 20260506 15:35:35.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:35:35.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:35.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:35.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:35.666 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 15:35:35.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:35.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:35.666 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:35:35.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:35.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:35.666 | INFO | Uploaded 36GiB to GPU in 2.01422s (17.873GiB/s) | 20260506 15:35:35.666 | INFO | Downloaded 34GiB to GPU in 1.94512s (17.4797GiB/s) | 20260506 15:35:40.805 | INFO | Analyzing chunk frames = [2475008,2475520) | 20260506 15:35:40.805 | INFO | MemInfo: { | 20260506 15:35:40.805 | INFO | "gpu" : " 17 GB", | 20260506 15:35:40.805 | INFO | "huge" : "", | 20260506 15:35:40.805 | INFO | "huge_shared" : " 27 GB", | 20260506 15:35:40.805 | INFO | "malloc" : " 14 GB", | 20260506 15:35:40.805 | INFO | "pinned" : " 29 GB" | 20260506 15:35:40.805 | INFO | } | 20260506 15:35:40.805 | INFO | Perf report for the last 5.13908 seconds: | 20260506 15:35:40.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:35:40.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:40.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:40.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:40.805 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:35:40.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:40.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:40.805 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:35:40.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:40.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:40.805 | INFO | Uploaded 36GiB to GPU in 1.96554s (18.3156GiB/s) | 20260506 15:35:40.806 | INFO | Downloaded 34GiB to GPU in 1.93949s (17.5304GiB/s) | 20260506 15:35:45.945 | INFO | Analyzing chunk frames = [2475520,2476032) | 20260506 15:35:45.945 | INFO | MemInfo: { | 20260506 15:35:45.945 | INFO | "gpu" : " 17 GB", | 20260506 15:35:45.945 | INFO | "huge" : "", | 20260506 15:35:45.945 | INFO | "huge_shared" : " 27 GB", | 20260506 15:35:45.945 | INFO | "malloc" : " 14 GB", | 20260506 15:35:45.945 | INFO | "pinned" : " 29 GB" | 20260506 15:35:45.945 | INFO | } | 20260506 15:35:45.945 | INFO | Perf report for the last 5.13977 seconds: | 20260506 15:35:45.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:35:45.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:45.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:45.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:45.945 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 15:35:45.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:45.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:45.945 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:35:45.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:45.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:45.945 | INFO | Uploaded 36GiB to GPU in 2.02315s (17.794GiB/s) | 20260506 15:35:45.945 | INFO | Downloaded 34GiB to GPU in 1.90945s (17.8062GiB/s) | 20260506 15:35:51.084 | INFO | Analyzing chunk frames = [2476032,2476544) | 20260506 15:35:51.084 | INFO | MemInfo: { | 20260506 15:35:51.084 | INFO | "gpu" : " 17 GB", | 20260506 15:35:51.084 | INFO | "huge" : "", | 20260506 15:35:51.084 | INFO | "huge_shared" : " 27 GB", | 20260506 15:35:51.084 | INFO | "malloc" : " 14 GB", | 20260506 15:35:51.084 | INFO | "pinned" : " 29 GB" | 20260506 15:35:51.084 | INFO | } | 20260506 15:35:51.084 | INFO | Perf report for the last 5.13881 seconds: | 20260506 15:35:51.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:35:51.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:51.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:51.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:51.084 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 15:35:51.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:51.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:51.084 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:35:51.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:51.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:51.084 | INFO | Uploaded 36GiB to GPU in 2.01098s (17.9017GiB/s) | 20260506 15:35:51.084 | INFO | Downloaded 34GiB to GPU in 1.93029s (17.6139GiB/s) | 20260506 15:35:56.222 | INFO | Analyzing chunk frames = [2476544,2477056) | 20260506 15:35:56.222 | INFO | MemInfo: { | 20260506 15:35:56.222 | INFO | "gpu" : " 17 GB", | 20260506 15:35:56.222 | INFO | "huge" : "", | 20260506 15:35:56.222 | INFO | "huge_shared" : " 27 GB", | 20260506 15:35:56.222 | INFO | "malloc" : " 14 GB", | 20260506 15:35:56.222 | INFO | "pinned" : " 29 GB" | 20260506 15:35:56.222 | INFO | } | 20260506 15:35:56.222 | INFO | Perf report for the last 5.13866 seconds: | 20260506 15:35:56.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:35:56.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:56.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:56.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:56.223 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:35:56.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:56.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:56.223 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 15:35:56.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:56.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:35:56.223 | INFO | Uploaded 36GiB to GPU in 2.08197s (17.2913GiB/s) | 20260506 15:35:56.223 | INFO | Downloaded 34GiB to GPU in 1.97568s (17.2093GiB/s) | 20260506 15:36:01.363 | INFO | Analyzing chunk frames = [2477056,2477568) | 20260506 15:36:01.363 | INFO | MemInfo: { | 20260506 15:36:01.363 | INFO | "gpu" : " 17 GB", | 20260506 15:36:01.363 | INFO | "huge" : "", | 20260506 15:36:01.363 | INFO | "huge_shared" : " 27 GB", | 20260506 15:36:01.363 | INFO | "malloc" : " 14 GB", | 20260506 15:36:01.363 | INFO | "pinned" : " 29 GB" | 20260506 15:36:01.363 | INFO | } | 20260506 15:36:01.363 | INFO | Perf report for the last 5.141 seconds: | 20260506 15:36:01.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:36:01.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:01.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:01.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:01.364 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:36:01.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:01.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:01.364 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 15:36:01.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:01.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:01.364 | INFO | Uploaded 36GiB to GPU in 2.02058s (17.8167GiB/s) | 20260506 15:36:01.364 | INFO | Downloaded 34GiB to GPU in 2.00065s (16.9944GiB/s) | 20260506 15:36:06.502 | INFO | Analyzing chunk frames = [2477568,2478080) | 20260506 15:36:06.502 | INFO | MemInfo: { | 20260506 15:36:06.502 | INFO | "gpu" : " 17 GB", | 20260506 15:36:06.502 | INFO | "huge" : "", | 20260506 15:36:06.502 | INFO | "huge_shared" : " 27 GB", | 20260506 15:36:06.502 | INFO | "malloc" : " 14 GB", | 20260506 15:36:06.502 | INFO | "pinned" : " 29 GB" | 20260506 15:36:06.502 | INFO | } | 20260506 15:36:06.502 | INFO | Perf report for the last 5.13862 seconds: | 20260506 15:36:06.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:36:06.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:06.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:06.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:06.502 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:36:06.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:06.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:06.502 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:36:06.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:06.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:06.502 | INFO | Uploaded 36GiB to GPU in 2.02251s (17.7997GiB/s) | 20260506 15:36:06.502 | INFO | Downloaded 34GiB to GPU in 1.9867s (17.1138GiB/s) | 20260506 15:36:06.622 | INFO | Starting a round of Baz IO | 20260506 15:36:09.765 | INFO | Populating RTMetrics report | 20260506 15:36:09.765 | INFO | RTMetrics report populated | 20260506 15:36:09.765 | INFO | Opening tmp RTMetrics json file | 20260506 15:36:09.826 | INFO | Serializing RTMetrics to json | 20260506 15:36:09.861 | INFO | Serialization done, swapping tmp file now... | 20260506 15:36:09.877 | INFO | RTMetrics done | 20260506 15:36:10.136 | INFO | Finished a round of Baz IO: Wrote 2.1 GiB in 3.514 seconds | 20260506 15:36:10.136 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.06% / 0.01% / 62.93% | 20260506 15:36:10.136 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.00228% / 7.94729e-06% / 6.75519e-05% | 20260506 15:36:10.136 | INFO | Baz IO overall: | 20260506 15:36:10.136 | INFO | BytesWritten : 1293566289591 | 20260506 15:36:10.136 | INFO | NumEvents : 626009596954 | 20260506 15:36:11.641 | INFO | Analyzing chunk frames = [2478080,2478592) | 20260506 15:36:11.641 | INFO | MemInfo: { | 20260506 15:36:11.641 | INFO | "gpu" : " 17 GB", | 20260506 15:36:11.641 | INFO | "huge" : "", | 20260506 15:36:11.641 | INFO | "huge_shared" : " 27 GB", | 20260506 15:36:11.642 | INFO | "malloc" : " 14 GB", | 20260506 15:36:11.642 | INFO | "pinned" : " 29 GB" | 20260506 15:36:11.642 | INFO | } | 20260506 15:36:11.642 | INFO | Perf report for the last 5.13955 seconds: | 20260506 15:36:11.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:36:11.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:11.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:11.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:11.642 | INFO | ANALYSIS : 62.9%, 2.9, 95.7ms, 0 | 20260506 15:36:11.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:11.642 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260506 15:36:11.642 | INFO | PRE_HQ : 60.7%, 1.9, 58.9ms, 0 | 20260506 15:36:11.642 | INFO | BAZWRITER : 67.9%, 4.8, 166.1ms, 0 | 20260506 15:36:11.642 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 15:36:11.642 | INFO | Uploaded 36GiB to GPU in 2.05329s (17.5329GiB/s) | 20260506 15:36:11.642 | INFO | Downloaded 38GiB to GPU in 2.2294s (17.045GiB/s) | 20260506 15:36:16.781 | INFO | Analyzing chunk frames = [2478592,2479104) | 20260506 15:36:16.781 | INFO | MemInfo: { | 20260506 15:36:16.781 | INFO | "gpu" : " 17 GB", | 20260506 15:36:16.781 | INFO | "huge" : "", | 20260506 15:36:16.781 | INFO | "huge_shared" : " 27 GB", | 20260506 15:36:16.781 | INFO | "malloc" : " 14 GB", | 20260506 15:36:16.781 | INFO | "pinned" : " 29 GB" | 20260506 15:36:16.781 | INFO | } | 20260506 15:36:16.781 | INFO | Perf report for the last 5.13964 seconds: | 20260506 15:36:16.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:36:16.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:16.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:16.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:16.781 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:36:16.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:16.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:16.782 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:36:16.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:16.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:16.782 | INFO | Uploaded 36GiB to GPU in 1.98209s (18.1627GiB/s) | 20260506 15:36:16.782 | INFO | Downloaded 34GiB to GPU in 1.94119s (17.515GiB/s) | 20260506 15:36:21.920 | INFO | Analyzing chunk frames = [2479104,2479616) | 20260506 15:36:21.920 | INFO | MemInfo: { | 20260506 15:36:21.920 | INFO | "gpu" : " 17 GB", | 20260506 15:36:21.920 | INFO | "huge" : "", | 20260506 15:36:21.920 | INFO | "huge_shared" : " 27 GB", | 20260506 15:36:21.920 | INFO | "malloc" : " 14 GB", | 20260506 15:36:21.920 | INFO | "pinned" : " 29 GB" | 20260506 15:36:21.920 | INFO | } | 20260506 15:36:21.920 | INFO | Perf report for the last 5.13851 seconds: | 20260506 15:36:21.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:36:21.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:21.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:21.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:21.920 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 15:36:21.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:21.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:21.920 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:36:21.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:21.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:21.920 | INFO | Uploaded 36GiB to GPU in 1.95799s (18.3862GiB/s) | 20260506 15:36:21.920 | INFO | Downloaded 34GiB to GPU in 1.9343s (17.5774GiB/s) | 20260506 15:36:27.059 | INFO | Analyzing chunk frames = [2479616,2480128) | 20260506 15:36:27.059 | INFO | MemInfo: { | 20260506 15:36:27.059 | INFO | "gpu" : " 17 GB", | 20260506 15:36:27.059 | INFO | "huge" : "", | 20260506 15:36:27.059 | INFO | "huge_shared" : " 27 GB", | 20260506 15:36:27.059 | INFO | "malloc" : " 14 GB", | 20260506 15:36:27.059 | INFO | "pinned" : " 29 GB" | 20260506 15:36:27.059 | INFO | } | 20260506 15:36:27.059 | INFO | Perf report for the last 5.13898 seconds: | 20260506 15:36:27.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:36:27.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:27.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:27.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:27.059 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 15:36:27.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:27.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:27.059 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:36:27.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:27.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:27.059 | INFO | Uploaded 36GiB to GPU in 1.96799s (18.2927GiB/s) | 20260506 15:36:27.059 | INFO | Downloaded 34GiB to GPU in 1.88082s (18.0772GiB/s) | 20260506 15:36:32.198 | INFO | Analyzing chunk frames = [2480128,2480640) | 20260506 15:36:32.198 | INFO | MemInfo: { | 20260506 15:36:32.198 | INFO | "gpu" : " 17 GB", | 20260506 15:36:32.198 | INFO | "huge" : "", | 20260506 15:36:32.198 | INFO | "huge_shared" : " 27 GB", | 20260506 15:36:32.198 | INFO | "malloc" : " 14 GB", | 20260506 15:36:32.198 | INFO | "pinned" : " 29 GB" | 20260506 15:36:32.198 | INFO | } | 20260506 15:36:32.199 | INFO | Perf report for the last 5.13973 seconds: | 20260506 15:36:32.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:36:32.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:32.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:32.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:32.199 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 15:36:32.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:32.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:32.199 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:36:32.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:32.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:32.199 | INFO | Uploaded 36GiB to GPU in 1.9912s (18.0795GiB/s) | 20260506 15:36:32.199 | INFO | Downloaded 34GiB to GPU in 1.93401s (17.58GiB/s) | 20260506 15:36:37.338 | INFO | Analyzing chunk frames = [2480640,2481152) | 20260506 15:36:37.338 | INFO | MemInfo: { | 20260506 15:36:37.338 | INFO | "gpu" : " 17 GB", | 20260506 15:36:37.338 | INFO | "huge" : "", | 20260506 15:36:37.338 | INFO | "huge_shared" : " 27 GB", | 20260506 15:36:37.338 | INFO | "malloc" : " 14 GB", | 20260506 15:36:37.338 | INFO | "pinned" : " 29 GB" | 20260506 15:36:37.339 | INFO | } | 20260506 15:36:37.339 | INFO | Perf report for the last 5.14032 seconds: | 20260506 15:36:37.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:36:37.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:37.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:37.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:37.339 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 15:36:37.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:37.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:37.339 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 15:36:37.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:37.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:37.339 | INFO | Uploaded 36GiB to GPU in 2.06067s (17.4701GiB/s) | 20260506 15:36:37.339 | INFO | Downloaded 34GiB to GPU in 1.97052s (17.2543GiB/s) | 20260506 15:36:42.477 | INFO | Analyzing chunk frames = [2481152,2481664) | 20260506 15:36:42.477 | INFO | MemInfo: { | 20260506 15:36:42.477 | INFO | "gpu" : " 17 GB", | 20260506 15:36:42.477 | INFO | "huge" : "", | 20260506 15:36:42.477 | INFO | "huge_shared" : " 27 GB", | 20260506 15:36:42.478 | INFO | "malloc" : " 14 GB", | 20260506 15:36:42.478 | INFO | "pinned" : " 29 GB" | 20260506 15:36:42.478 | INFO | } | 20260506 15:36:42.478 | INFO | Perf report for the last 5.13875 seconds: | 20260506 15:36:42.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:36:42.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:42.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:42.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:42.478 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:36:42.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:42.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:42.478 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:36:42.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:42.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:42.478 | INFO | Uploaded 36GiB to GPU in 2.02547s (17.7736GiB/s) | 20260506 15:36:42.478 | INFO | Downloaded 34GiB to GPU in 1.988s (17.1026GiB/s) | 20260506 15:36:47.617 | INFO | Analyzing chunk frames = [2481664,2482176) | 20260506 15:36:47.617 | INFO | MemInfo: { | 20260506 15:36:47.617 | INFO | "gpu" : " 17 GB", | 20260506 15:36:47.617 | INFO | "huge" : "", | 20260506 15:36:47.617 | INFO | "huge_shared" : " 27 GB", | 20260506 15:36:47.617 | INFO | "malloc" : " 14 GB", | 20260506 15:36:47.617 | INFO | "pinned" : " 29 GB" | 20260506 15:36:47.617 | INFO | } | 20260506 15:36:47.617 | INFO | Perf report for the last 5.13988 seconds: | 20260506 15:36:47.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:36:47.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:47.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:47.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:47.618 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:36:47.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:47.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:47.618 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:36:47.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:47.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:47.618 | INFO | Uploaded 36GiB to GPU in 1.98535s (18.1328GiB/s) | 20260506 15:36:47.618 | INFO | Downloaded 34GiB to GPU in 1.97136s (17.247GiB/s) | 20260506 15:36:47.618 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:36:47.618 | INFO | Total mem usage: 17.4 GiB | 20260506 15:36:47.618 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:36:47.618 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -13.1%) | 20260506 15:36:47.618 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:36:47.618 | INFO | High Water Breakdown: | 20260506 15:36:47.618 | INFO | 61% : 9.4 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:36:47.618 | INFO | 38.8% : 5.98 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:36:47.618 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:36:47.618 | INFO | ------------------------------------------ | 20260506 15:36:47.618 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:36:47.618 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:36:47.618 | INFO | Total mem usage: 26.5 GiB | 20260506 15:36:47.618 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:36:47.618 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:36:47.618 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 15:36:47.618 | INFO | High Water Breakdown: | 20260506 15:36:47.618 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:36:47.618 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:36:47.618 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:36:47.618 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:36:47.618 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:36:47.618 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:36:47.618 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:36:47.618 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:36:47.618 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:36:47.618 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:36:47.618 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:36:47.618 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:36:47.618 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:36:47.618 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:36:47.618 | INFO | ------------------------------------------ | 20260506 15:36:47.618 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:36:47.619 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:36:47.619 | INFO | Total mem usage: 25.6 GiB | 20260506 15:36:47.619 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:36:47.619 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:36:47.619 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:36:47.619 | INFO | High Water Breakdown: | 20260506 15:36:47.619 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:36:47.619 | INFO | ------------------------------------------ | 20260506 15:36:47.619 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:36:47.619 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:36:47.619 | INFO | Total mem usage: 15.7 GiB | 20260506 15:36:47.619 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:36:47.619 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:36:47.619 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 15:36:47.619 | INFO | High Water Breakdown: | 20260506 15:36:47.619 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:36:47.619 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:36:47.619 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:36:47.619 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:36:47.619 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:36:47.619 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:36:47.619 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:36:47.619 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:36:47.619 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:36:47.619 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:36:47.619 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:36:47.619 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:36:47.619 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:36:47.619 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:36:47.619 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:36:47.619 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:36:47.619 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:36:47.619 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:36:47.619 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:36:47.619 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:36:47.619 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:36:47.619 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:36:47.619 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:36:47.619 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:36:47.619 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:36:47.619 | INFO | ------------------------------------------ | 20260506 15:36:47.619 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:36:47.619 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:36:47.619 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:36:47.619 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:36:47.619 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.53e+01, 1.34e+01 / 1.76e+00 | 20260506 15:36:47.619 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 9.99e-02, 1.01e-02 / 7.19e-03 | 20260506 15:36:47.619 | INFO | PerfReport: 11.57%, Baseline -- 3.63e+00 / 4.40e+00, 5.51e-01 / 3.43e-01 | 20260506 15:36:47.619 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.47e+00, 1.82e-01 / 1.29e-01 | 20260506 15:36:47.619 | INFO | PerfReport: 15.88%, DME -- 3.99e+01 / 4.42e+01, 3.81e+00 / 1.97e+00 | 20260506 15:36:47.619 | INFO | PerfReport: 56.83%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.41e+00 / 2.04e+00 | 20260506 15:36:47.619 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.33e+00 / 3.00e+00, 4.06e-01 / 1.83e-01 | 20260506 15:36:47.619 | INFO | PerfReport: 3.55%, Metrics -- 1.12e+00 / 1.49e+00, 2.88e-01 / 1.42e-01 | 20260506 15:36:47.619 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.87e-01 / 4.38e-01, 6.01e-02 / 2.45e-02 | 20260506 15:36:47.619 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:36:47.619 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:36:47.619 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:36:47.619 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 15:36:47.619 | INFO | PerfReport: 21.54%, Upload -- 2.10e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260506 15:36:47.619 | INFO | PerfReport: 21.22%, Download -- 2.07e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 15:36:47.619 | INFO | PerfReport: 32.21%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 15:36:47.619 | INFO | PerfReport: 1.50%, LockUpload -- 1.46e+00 / 0.00e+00, 5.78e+00 / 0.00e+00 | 20260506 15:36:47.619 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260506 15:36:47.619 | INFO | PerfReport: 22.43%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 15:36:47.619 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.35e-02 / 0.00e+00 | 20260506 15:36:47.732 | INFO | Starting a round of Baz IO | 20260506 15:36:50.882 | INFO | Populating RTMetrics report | 20260506 15:36:50.882 | INFO | RTMetrics report populated | 20260506 15:36:50.882 | INFO | Opening tmp RTMetrics json file | 20260506 15:36:50.889 | INFO | Serializing RTMetrics to json | 20260506 15:36:50.912 | INFO | Serialization done, swapping tmp file now... | 20260506 15:36:50.914 | INFO | RTMetrics done | 20260506 15:36:51.001 | INFO | Finished a round of Baz IO: Wrote 2.101 GiB in 3.268 seconds | 20260506 15:36:51.001 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.05% / 0.01% / 62.94% | 20260506 15:36:51.001 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.0006% / 1.58946e-05% / 6.35783e-05% | 20260506 15:36:51.001 | INFO | Baz IO overall: | 20260506 15:36:51.001 | INFO | BytesWritten : 1295822103123 | 20260506 15:36:51.001 | INFO | NumEvents : 627101020533 | 20260506 15:36:52.756 | INFO | Analyzing chunk frames = [2482176,2482688) | 20260506 15:36:52.756 | INFO | MemInfo: { | 20260506 15:36:52.756 | INFO | "gpu" : " 17 GB", | 20260506 15:36:52.756 | INFO | "huge" : "", | 20260506 15:36:52.756 | INFO | "huge_shared" : " 27 GB", | 20260506 15:36:52.756 | INFO | "malloc" : " 14 GB", | 20260506 15:36:52.756 | INFO | "pinned" : " 29 GB" | 20260506 15:36:52.756 | INFO | } | 20260506 15:36:52.756 | INFO | Perf report for the last 5.13886 seconds: | 20260506 15:36:52.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:36:52.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:52.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:52.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:52.757 | INFO | ANALYSIS : 62.9%, 2.9, 95.4ms, 0 | 20260506 15:36:52.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:52.757 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 15:36:52.757 | INFO | PRE_HQ : 60.7%, 1.9, 58.9ms, 0 | 20260506 15:36:52.757 | INFO | BAZWRITER : 63.2%, 4.8, 155.8ms, 0 | 20260506 15:36:52.757 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 15:36:52.757 | INFO | Uploaded 36GiB to GPU in 2.09214s (17.2073GiB/s) | 20260506 15:36:52.757 | INFO | Downloaded 38GiB to GPU in 2.21973s (17.1192GiB/s) | 20260506 15:36:57.895 | INFO | Analyzing chunk frames = [2482688,2483200) | 20260506 15:36:57.895 | INFO | MemInfo: { | 20260506 15:36:57.895 | INFO | "gpu" : " 17 GB", | 20260506 15:36:57.895 | INFO | "huge" : "", | 20260506 15:36:57.895 | INFO | "huge_shared" : " 27 GB", | 20260506 15:36:57.895 | INFO | "malloc" : " 14 GB", | 20260506 15:36:57.895 | INFO | "pinned" : " 29 GB" | 20260506 15:36:57.895 | INFO | } | 20260506 15:36:57.895 | INFO | Perf report for the last 5.13865 seconds: | 20260506 15:36:57.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:36:57.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:57.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:57.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:57.895 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 15:36:57.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:57.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:57.895 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:36:57.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:57.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:36:57.895 | INFO | Uploaded 36GiB to GPU in 2.00701s (17.9371GiB/s) | 20260506 15:36:57.895 | INFO | Downloaded 34GiB to GPU in 1.94337s (17.4954GiB/s) | 20260506 15:37:03.034 | INFO | Analyzing chunk frames = [2483200,2483712) | 20260506 15:37:03.034 | INFO | MemInfo: { | 20260506 15:37:03.034 | INFO | "gpu" : " 17 GB", | 20260506 15:37:03.034 | INFO | "huge" : "", | 20260506 15:37:03.034 | INFO | "huge_shared" : " 27 GB", | 20260506 15:37:03.034 | INFO | "malloc" : " 14 GB", | 20260506 15:37:03.034 | INFO | "pinned" : " 29 GB" | 20260506 15:37:03.034 | INFO | } | 20260506 15:37:03.034 | INFO | Perf report for the last 5.1394 seconds: | 20260506 15:37:03.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:37:03.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:03.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:03.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:03.035 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 15:37:03.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:03.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:03.035 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:37:03.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:03.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:03.035 | INFO | Uploaded 36GiB to GPU in 2.00445s (17.96GiB/s) | 20260506 15:37:03.035 | INFO | Downloaded 34GiB to GPU in 1.94718s (17.4612GiB/s) | 20260506 15:37:08.173 | INFO | Analyzing chunk frames = [2483712,2484224) | 20260506 15:37:08.173 | INFO | MemInfo: { | 20260506 15:37:08.173 | INFO | "gpu" : " 17 GB", | 20260506 15:37:08.173 | INFO | "huge" : "", | 20260506 15:37:08.173 | INFO | "huge_shared" : " 27 GB", | 20260506 15:37:08.173 | INFO | "malloc" : " 14 GB", | 20260506 15:37:08.173 | INFO | "pinned" : " 29 GB" | 20260506 15:37:08.173 | INFO | } | 20260506 15:37:08.173 | INFO | Perf report for the last 5.13863 seconds: | 20260506 15:37:08.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:37:08.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:08.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:08.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:08.173 | INFO | ANALYSIS : 60.6%, 3.0, 94.0ms, 0 | 20260506 15:37:08.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:08.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:08.173 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:37:08.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:08.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:08.174 | INFO | Uploaded 36GiB to GPU in 1.99849s (18.0136GiB/s) | 20260506 15:37:08.174 | INFO | Downloaded 34GiB to GPU in 1.91191s (17.7833GiB/s) | 20260506 15:37:13.313 | INFO | Analyzing chunk frames = [2484224,2484736) | 20260506 15:37:13.313 | INFO | MemInfo: { | 20260506 15:37:13.313 | INFO | "gpu" : " 17 GB", | 20260506 15:37:13.313 | INFO | "huge" : "", | 20260506 15:37:13.313 | INFO | "huge_shared" : " 27 GB", | 20260506 15:37:13.313 | INFO | "malloc" : " 14 GB", | 20260506 15:37:13.313 | INFO | "pinned" : " 29 GB" | 20260506 15:37:13.313 | INFO | } | 20260506 15:37:13.313 | INFO | Perf report for the last 5.13956 seconds: | 20260506 15:37:13.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:37:13.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:13.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:13.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:13.313 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 15:37:13.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:13.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:13.313 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 15:37:13.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:13.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:13.313 | INFO | Uploaded 36GiB to GPU in 2.00172s (17.9846GiB/s) | 20260506 15:37:13.313 | INFO | Downloaded 34GiB to GPU in 1.92646s (17.649GiB/s) | 20260506 15:37:18.454 | INFO | Analyzing chunk frames = [2484736,2485248) | 20260506 15:37:18.454 | INFO | MemInfo: { | 20260506 15:37:18.454 | INFO | "gpu" : " 17 GB", | 20260506 15:37:18.454 | INFO | "huge" : "", | 20260506 15:37:18.454 | INFO | "huge_shared" : " 27 GB", | 20260506 15:37:18.454 | INFO | "malloc" : " 14 GB", | 20260506 15:37:18.454 | INFO | "pinned" : " 29 GB" | 20260506 15:37:18.454 | INFO | } | 20260506 15:37:18.454 | INFO | Perf report for the last 5.14153 seconds: | 20260506 15:37:18.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:37:18.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:18.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:18.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:18.454 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:37:18.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:18.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:18.454 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260506 15:37:18.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:18.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:18.455 | INFO | Uploaded 36GiB to GPU in 2.08029s (17.3053GiB/s) | 20260506 15:37:18.455 | INFO | Downloaded 34GiB to GPU in 1.97303s (17.2324GiB/s) | 20260506 15:37:23.592 | INFO | Analyzing chunk frames = [2485248,2485760) | 20260506 15:37:23.592 | INFO | MemInfo: { | 20260506 15:37:23.592 | INFO | "gpu" : " 17 GB", | 20260506 15:37:23.592 | INFO | "huge" : "", | 20260506 15:37:23.592 | INFO | "huge_shared" : " 27 GB", | 20260506 15:37:23.592 | INFO | "malloc" : " 14 GB", | 20260506 15:37:23.592 | INFO | "pinned" : " 29 GB" | 20260506 15:37:23.592 | INFO | } | 20260506 15:37:23.592 | INFO | Perf report for the last 5.13807 seconds: | 20260506 15:37:23.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:37:23.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:23.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:23.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:23.592 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 15:37:23.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:23.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:23.593 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:37:23.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:23.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:23.593 | INFO | Uploaded 36GiB to GPU in 2.00771s (17.9308GiB/s) | 20260506 15:37:23.593 | INFO | Downloaded 34GiB to GPU in 1.99059s (17.0804GiB/s) | 20260506 15:37:28.731 | INFO | Analyzing chunk frames = [2485760,2486272) | 20260506 15:37:28.731 | INFO | MemInfo: { | 20260506 15:37:28.731 | INFO | "gpu" : " 17 GB", | 20260506 15:37:28.731 | INFO | "huge" : "", | 20260506 15:37:28.731 | INFO | "huge_shared" : " 27 GB", | 20260506 15:37:28.731 | INFO | "malloc" : " 14 GB", | 20260506 15:37:28.731 | INFO | "pinned" : " 29 GB" | 20260506 15:37:28.731 | INFO | } | 20260506 15:37:28.731 | INFO | Perf report for the last 5.1385 seconds: | 20260506 15:37:28.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:37:28.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:28.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:28.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:28.731 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:37:28.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:28.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:28.731 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 15:37:28.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:28.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:28.731 | INFO | Uploaded 36GiB to GPU in 2.02396s (17.7869GiB/s) | 20260506 15:37:28.731 | INFO | Downloaded 34GiB to GPU in 1.97804s (17.1887GiB/s) | 20260506 15:37:28.846 | INFO | Starting a round of Baz IO | 20260506 15:37:32.003 | INFO | Populating RTMetrics report | 20260506 15:37:32.003 | INFO | RTMetrics report populated | 20260506 15:37:32.003 | INFO | Opening tmp RTMetrics json file | 20260506 15:37:32.039 | INFO | Serializing RTMetrics to json | 20260506 15:37:32.257 | INFO | Serialization done, swapping tmp file now... | 20260506 15:37:32.305 | INFO | RTMetrics done | 20260506 15:37:32.334 | INFO | Finished a round of Baz IO: Wrote 2.099 GiB in 3.488 seconds | 20260506 15:37:32.335 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.04% / 0.01% / 62.95% | 20260506 15:37:32.335 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.98962% / 1.58946e-05% / 5.16574e-05% | 20260506 15:37:32.335 | INFO | Baz IO overall: | 20260506 15:37:32.335 | INFO | BytesWritten : 1298075678392 | 20260506 15:37:32.335 | INFO | NumEvents : 628190642051 | 20260506 15:37:33.871 | INFO | Analyzing chunk frames = [2486272,2486784) | 20260506 15:37:33.871 | INFO | MemInfo: { | 20260506 15:37:33.871 | INFO | "gpu" : " 17 GB", | 20260506 15:37:33.871 | INFO | "huge" : "", | 20260506 15:37:33.871 | INFO | "huge_shared" : " 27 GB", | 20260506 15:37:33.871 | INFO | "malloc" : " 14 GB", | 20260506 15:37:33.871 | INFO | "pinned" : " 29 GB" | 20260506 15:37:33.871 | INFO | } | 20260506 15:37:33.871 | INFO | Perf report for the last 5.14039 seconds: | 20260506 15:37:33.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:37:33.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:33.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:33.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:33.871 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 15:37:33.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:33.871 | INFO | RT_METRICS : 49.8%, 1.0, 26.1ms, 0 | 20260506 15:37:33.871 | INFO | PRE_HQ : 60.7%, 1.9, 59.5ms, 0 | 20260506 15:37:33.871 | INFO | BAZWRITER : 67.2%, 4.8, 165.5ms, 0 | 20260506 15:37:33.871 | INFO | BAZ_SAMPLING : 5.2%, 1.0, 2.7ms, 0 | 20260506 15:37:33.871 | INFO | Uploaded 36GiB to GPU in 2.06195s (17.4592GiB/s) | 20260506 15:37:33.872 | INFO | Downloaded 38GiB to GPU in 2.23117s (17.0314GiB/s) | 20260506 15:37:39.010 | INFO | Analyzing chunk frames = [2486784,2487296) | 20260506 15:37:39.010 | INFO | MemInfo: { | 20260506 15:37:39.010 | INFO | "gpu" : " 17 GB", | 20260506 15:37:39.010 | INFO | "huge" : "", | 20260506 15:37:39.010 | INFO | "huge_shared" : " 27 GB", | 20260506 15:37:39.010 | INFO | "malloc" : " 14 GB", | 20260506 15:37:39.010 | INFO | "pinned" : " 29 GB" | 20260506 15:37:39.010 | INFO | } | 20260506 15:37:39.010 | INFO | Perf report for the last 5.13853 seconds: | 20260506 15:37:39.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:37:39.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:39.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:39.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:39.010 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 15:37:39.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:39.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:39.010 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:37:39.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:39.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:39.010 | INFO | Uploaded 36GiB to GPU in 1.9965s (18.0316GiB/s) | 20260506 15:37:39.010 | INFO | Downloaded 34GiB to GPU in 1.933s (17.5893GiB/s) | 20260506 15:37:44.149 | INFO | Analyzing chunk frames = [2487296,2487808) | 20260506 15:37:44.149 | INFO | MemInfo: { | 20260506 15:37:44.149 | INFO | "gpu" : " 17 GB", | 20260506 15:37:44.149 | INFO | "huge" : "", | 20260506 15:37:44.149 | INFO | "huge_shared" : " 27 GB", | 20260506 15:37:44.149 | INFO | "malloc" : " 14 GB", | 20260506 15:37:44.149 | INFO | "pinned" : " 29 GB" | 20260506 15:37:44.149 | INFO | } | 20260506 15:37:44.150 | INFO | Perf report for the last 5.1398 seconds: | 20260506 15:37:44.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:37:44.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:44.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:44.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:44.150 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 15:37:44.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:44.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:44.150 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:37:44.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:44.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:44.150 | INFO | Uploaded 36GiB to GPU in 1.98093s (18.1733GiB/s) | 20260506 15:37:44.150 | INFO | Downloaded 34GiB to GPU in 1.93697s (17.5532GiB/s) | 20260506 15:37:49.288 | INFO | Analyzing chunk frames = [2487808,2488320) | 20260506 15:37:49.288 | INFO | MemInfo: { | 20260506 15:37:49.288 | INFO | "gpu" : " 17 GB", | 20260506 15:37:49.288 | INFO | "huge" : "", | 20260506 15:37:49.288 | INFO | "huge_shared" : " 27 GB", | 20260506 15:37:49.288 | INFO | "malloc" : " 14 GB", | 20260506 15:37:49.288 | INFO | "pinned" : " 29 GB" | 20260506 15:37:49.288 | INFO | } | 20260506 15:37:49.288 | INFO | Perf report for the last 5.13862 seconds: | 20260506 15:37:49.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:37:49.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:49.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:49.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:49.288 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 15:37:49.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:49.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:49.288 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:37:49.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:49.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:49.289 | INFO | Uploaded 36GiB to GPU in 2.02112s (17.8119GiB/s) | 20260506 15:37:49.289 | INFO | Downloaded 34GiB to GPU in 1.91402s (17.7637GiB/s) | 20260506 15:37:54.428 | INFO | Analyzing chunk frames = [2488320,2488832) | 20260506 15:37:54.428 | INFO | MemInfo: { | 20260506 15:37:54.428 | INFO | "gpu" : " 17 GB", | 20260506 15:37:54.428 | INFO | "huge" : "", | 20260506 15:37:54.428 | INFO | "huge_shared" : " 27 GB", | 20260506 15:37:54.428 | INFO | "malloc" : " 14 GB", | 20260506 15:37:54.428 | INFO | "pinned" : " 29 GB" | 20260506 15:37:54.428 | INFO | } | 20260506 15:37:54.428 | INFO | Perf report for the last 5.14008 seconds: | 20260506 15:37:54.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:37:54.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:54.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:54.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:54.428 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 15:37:54.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:54.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:54.428 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:37:54.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:54.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:54.429 | INFO | Uploaded 36GiB to GPU in 2.02812s (17.7505GiB/s) | 20260506 15:37:54.429 | INFO | Downloaded 34GiB to GPU in 1.92992s (17.6173GiB/s) | 20260506 15:37:59.568 | INFO | Analyzing chunk frames = [2488832,2489344) | 20260506 15:37:59.568 | INFO | MemInfo: { | 20260506 15:37:59.568 | INFO | "gpu" : " 17 GB", | 20260506 15:37:59.568 | INFO | "huge" : "", | 20260506 15:37:59.568 | INFO | "huge_shared" : " 27 GB", | 20260506 15:37:59.568 | INFO | "malloc" : " 14 GB", | 20260506 15:37:59.568 | INFO | "pinned" : " 29 GB" | 20260506 15:37:59.568 | INFO | } | 20260506 15:37:59.568 | INFO | Perf report for the last 5.13933 seconds: | 20260506 15:37:59.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:37:59.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:59.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:59.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:59.568 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 15:37:59.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:59.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:59.568 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260506 15:37:59.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:59.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:37:59.568 | INFO | Uploaded 36GiB to GPU in 2.0463s (17.5927GiB/s) | 20260506 15:37:59.568 | INFO | Downloaded 34GiB to GPU in 1.92439s (17.6679GiB/s) | 20260506 15:38:04.707 | INFO | Analyzing chunk frames = [2489344,2489856) | 20260506 15:38:04.707 | INFO | MemInfo: { | 20260506 15:38:04.707 | INFO | "gpu" : " 17 GB", | 20260506 15:38:04.707 | INFO | "huge" : "", | 20260506 15:38:04.707 | INFO | "huge_shared" : " 27 GB", | 20260506 15:38:04.707 | INFO | "malloc" : " 14 GB", | 20260506 15:38:04.707 | INFO | "pinned" : " 29 GB" | 20260506 15:38:04.707 | INFO | } | 20260506 15:38:04.707 | INFO | Perf report for the last 5.1396 seconds: | 20260506 15:38:04.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:38:04.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:04.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:04.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:04.708 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:38:04.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:04.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:04.708 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:38:04.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:04.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:04.708 | INFO | Uploaded 36GiB to GPU in 2.00831s (17.9255GiB/s) | 20260506 15:38:04.708 | INFO | Downloaded 34GiB to GPU in 1.98234s (17.1515GiB/s) | 20260506 15:38:09.846 | INFO | Analyzing chunk frames = [2489856,2490368) | 20260506 15:38:09.846 | INFO | MemInfo: { | 20260506 15:38:09.846 | INFO | "gpu" : " 17 GB", | 20260506 15:38:09.846 | INFO | "huge" : "", | 20260506 15:38:09.846 | INFO | "huge_shared" : " 27 GB", | 20260506 15:38:09.846 | INFO | "malloc" : " 14 GB", | 20260506 15:38:09.846 | INFO | "pinned" : " 29 GB" | 20260506 15:38:09.847 | INFO | } | 20260506 15:38:09.847 | INFO | Perf report for the last 5.13943 seconds: | 20260506 15:38:09.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:38:09.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:09.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:09.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:09.847 | INFO | ANALYSIS : 62.3%, 3.0, 97.2ms, 0 | 20260506 15:38:09.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:09.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:09.847 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:38:09.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:09.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:09.847 | INFO | Uploaded 36GiB to GPU in 2.04498s (17.6041GiB/s) | 20260506 15:38:09.847 | INFO | Downloaded 34GiB to GPU in 1.98944s (17.0902GiB/s) | 20260506 15:38:09.847 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:38:09.847 | INFO | Total mem usage: 17.4 GiB | 20260506 15:38:09.847 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:38:09.847 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -13.7%) | 20260506 15:38:09.847 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:38:09.847 | INFO | High Water Breakdown: | 20260506 15:38:09.847 | INFO | 61% : 9.33 GiB ( 85.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:38:09.847 | INFO | 38.9% : 5.95 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:38:09.847 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:38:09.847 | INFO | ------------------------------------------ | 20260506 15:38:09.847 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:38:09.847 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:38:09.847 | INFO | Total mem usage: 26.5 GiB | 20260506 15:38:09.847 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:38:09.847 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:38:09.847 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 15:38:09.847 | INFO | High Water Breakdown: | 20260506 15:38:09.847 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:38:09.847 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:38:09.847 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:38:09.847 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:38:09.847 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:38:09.847 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:38:09.847 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:38:09.847 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:38:09.847 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:38:09.847 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:38:09.847 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:38:09.847 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:38:09.847 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:38:09.847 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:38:09.848 | INFO | ------------------------------------------ | 20260506 15:38:09.848 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:38:09.848 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:38:09.848 | INFO | Total mem usage: 25.6 GiB | 20260506 15:38:09.848 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:38:09.848 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:38:09.848 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:38:09.848 | INFO | High Water Breakdown: | 20260506 15:38:09.848 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:38:09.848 | INFO | ------------------------------------------ | 20260506 15:38:09.848 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:38:09.848 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:38:09.848 | INFO | Total mem usage: 15.7 GiB | 20260506 15:38:09.848 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:38:09.848 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:38:09.848 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 15:38:09.848 | INFO | High Water Breakdown: | 20260506 15:38:09.848 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:38:09.848 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:38:09.848 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:38:09.848 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:38:09.848 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:38:09.848 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:38:09.848 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:38:09.848 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:38:09.848 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:38:09.848 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:38:09.848 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:38:09.848 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:38:09.848 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:38:09.848 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:38:09.848 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:38:09.848 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:38:09.848 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:38:09.848 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:38:09.848 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:38:09.848 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:38:09.848 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:38:09.848 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:38:09.848 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:38:09.848 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:38:09.848 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:38:09.848 | INFO | ------------------------------------------ | 20260506 15:38:09.848 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:38:09.848 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:38:09.848 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:38:09.848 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:38:09.848 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.55e+01, 1.33e+01 / 1.79e+00 | 20260506 15:38:09.848 | INFO | PerfReport: 0.29%, AggregateStats -- 9.22e-02 / 1.03e-01, 1.06e-02 / 1.13e-02 | 20260506 15:38:09.848 | INFO | PerfReport: 11.58%, Baseline -- 3.64e+00 / 4.37e+00, 5.61e-01 / 4.36e-01 | 20260506 15:38:09.848 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.87e-01 / 1.59e-01 | 20260506 15:38:09.848 | INFO | PerfReport: 15.84%, DME -- 3.99e+01 / 4.39e+01, 3.76e+00 / 2.16e+00 | 20260506 15:38:09.848 | INFO | PerfReport: 56.91%, FrameLabeling -- 1.79e+01 / 2.09e+01, 1.51e+00 / 2.02e+00 | 20260506 15:38:09.848 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.32e+00 / 2.89e+00, 4.01e-01 / 2.14e-01 | 20260506 15:38:09.848 | INFO | PerfReport: 3.53%, Metrics -- 1.11e+00 / 1.46e+00, 2.86e-01 / 1.61e-01 | 20260506 15:38:09.848 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.88e-01 / 4.45e-01, 6.04e-02 / 2.84e-02 | 20260506 15:38:09.848 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:38:09.848 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:38:09.848 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:38:09.848 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 15:38:09.848 | INFO | PerfReport: 21.61%, Upload -- 2.11e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260506 15:38:09.848 | INFO | PerfReport: 21.17%, Download -- 2.07e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 15:38:09.848 | INFO | PerfReport: 32.23%, Compute -- 3.15e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 15:38:09.848 | INFO | PerfReport: 1.62%, LockUpload -- 1.58e+00 / 0.00e+00, 5.90e+00 / 0.00e+00 | 20260506 15:38:09.848 | INFO | PerfReport: 1.04%, LockDownload -- 1.02e+00 / 0.00e+00, 2.63e+00 / 0.00e+00 | 20260506 15:38:09.848 | INFO | PerfReport: 22.31%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 15:38:09.848 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260506 15:38:09.966 | INFO | Starting a round of Baz IO | 20260506 15:38:13.106 | INFO | Populating RTMetrics report | 20260506 15:38:13.106 | INFO | RTMetrics report populated | 20260506 15:38:13.106 | INFO | Opening tmp RTMetrics json file | 20260506 15:38:13.161 | INFO | Serializing RTMetrics to json | 20260506 15:38:13.164 | INFO | Serialization done, swapping tmp file now... | 20260506 15:38:13.169 | INFO | RTMetrics done | 20260506 15:38:13.557 | INFO | Finished a round of Baz IO: Wrote 2.099 GiB in 3.591 seconds | 20260506 15:38:13.558 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.03% / 0.01% / 62.96% | 20260506 15:38:13.558 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.98511% / 3.17891e-05% / 5.5631e-05% | 20260506 15:38:13.558 | INFO | Baz IO overall: | 20260506 15:38:13.558 | INFO | BytesWritten : 1300329471776 | 20260506 15:38:13.558 | INFO | NumEvents : 629280548316 | 20260506 15:38:14.984 | INFO | Analyzing chunk frames = [2490368,2490880) | 20260506 15:38:14.984 | INFO | MemInfo: { | 20260506 15:38:14.984 | INFO | "gpu" : " 17 GB", | 20260506 15:38:14.984 | INFO | "huge" : "", | 20260506 15:38:14.984 | INFO | "huge_shared" : " 27 GB", | 20260506 15:38:14.984 | INFO | "malloc" : " 14 GB", | 20260506 15:38:14.984 | INFO | "pinned" : " 29 GB" | 20260506 15:38:14.984 | INFO | } | 20260506 15:38:14.985 | INFO | Perf report for the last 5.1378 seconds: | 20260506 15:38:14.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:38:14.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:14.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:14.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:14.985 | INFO | ANALYSIS : 62.8%, 2.9, 95.3ms, 0 | 20260506 15:38:14.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:14.985 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 15:38:14.985 | INFO | PRE_HQ : 60.3%, 1.9, 59.0ms, 0 | 20260506 15:38:14.985 | INFO | BAZWRITER : 69.4%, 4.8, 171.4ms, 0 | 20260506 15:38:14.985 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 15:38:14.985 | INFO | Uploaded 36GiB to GPU in 2.03802s (17.6642GiB/s) | 20260506 15:38:14.985 | INFO | Downloaded 38GiB to GPU in 2.22705s (17.0629GiB/s) | 20260506 15:38:20.124 | INFO | Analyzing chunk frames = [2490880,2491392) | 20260506 15:38:20.124 | INFO | MemInfo: { | 20260506 15:38:20.124 | INFO | "gpu" : " 17 GB", | 20260506 15:38:20.124 | INFO | "huge" : "", | 20260506 15:38:20.124 | INFO | "huge_shared" : " 27 GB", | 20260506 15:38:20.124 | INFO | "malloc" : " 14 GB", | 20260506 15:38:20.124 | INFO | "pinned" : " 29 GB" | 20260506 15:38:20.124 | INFO | } | 20260506 15:38:20.124 | INFO | Perf report for the last 5.13957 seconds: | 20260506 15:38:20.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:38:20.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:20.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:20.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:20.124 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 15:38:20.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:20.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:20.124 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:38:20.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:20.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:20.124 | INFO | Uploaded 36GiB to GPU in 1.97787s (18.2014GiB/s) | 20260506 15:38:20.125 | INFO | Downloaded 34GiB to GPU in 1.92913s (17.6245GiB/s) | 20260506 15:38:25.264 | INFO | Analyzing chunk frames = [2491392,2491904) | 20260506 15:38:25.264 | INFO | MemInfo: { | 20260506 15:38:25.264 | INFO | "gpu" : " 17 GB", | 20260506 15:38:25.264 | INFO | "huge" : "", | 20260506 15:38:25.265 | INFO | "huge_shared" : " 27 GB", | 20260506 15:38:25.265 | INFO | "malloc" : " 14 GB", | 20260506 15:38:25.265 | INFO | "pinned" : " 29 GB" | 20260506 15:38:25.265 | INFO | } | 20260506 15:38:25.265 | INFO | Perf report for the last 5.14049 seconds: | 20260506 15:38:25.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:38:25.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:25.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:25.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:25.265 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:38:25.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:25.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:25.265 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:38:25.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:25.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:25.265 | INFO | Uploaded 36GiB to GPU in 1.96366s (18.3331GiB/s) | 20260506 15:38:25.265 | INFO | Downloaded 34GiB to GPU in 1.93021s (17.6146GiB/s) | 20260506 15:38:30.404 | INFO | Analyzing chunk frames = [2491904,2492416) | 20260506 15:38:30.404 | INFO | MemInfo: { | 20260506 15:38:30.404 | INFO | "gpu" : " 17 GB", | 20260506 15:38:30.404 | INFO | "huge" : "", | 20260506 15:38:30.404 | INFO | "huge_shared" : " 27 GB", | 20260506 15:38:30.404 | INFO | "malloc" : " 14 GB", | 20260506 15:38:30.404 | INFO | "pinned" : " 29 GB" | 20260506 15:38:30.404 | INFO | } | 20260506 15:38:30.404 | INFO | Perf report for the last 5.13917 seconds: | 20260506 15:38:30.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:38:30.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:30.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:30.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:30.404 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 15:38:30.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:30.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:30.404 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:38:30.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:30.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:30.404 | INFO | Uploaded 36GiB to GPU in 2.00587s (17.9473GiB/s) | 20260506 15:38:30.404 | INFO | Downloaded 34GiB to GPU in 1.91277s (17.7753GiB/s) | 20260506 15:38:35.542 | INFO | Analyzing chunk frames = [2492416,2492928) | 20260506 15:38:35.542 | INFO | MemInfo: { | 20260506 15:38:35.542 | INFO | "gpu" : " 17 GB", | 20260506 15:38:35.542 | INFO | "huge" : "", | 20260506 15:38:35.542 | INFO | "huge_shared" : " 27 GB", | 20260506 15:38:35.542 | INFO | "malloc" : " 14 GB", | 20260506 15:38:35.542 | INFO | "pinned" : " 29 GB" | 20260506 15:38:35.542 | INFO | } | 20260506 15:38:35.542 | INFO | Perf report for the last 5.13856 seconds: | 20260506 15:38:35.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:38:35.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:35.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:35.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:35.542 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 15:38:35.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:35.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:35.542 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:38:35.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:35.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:35.543 | INFO | Uploaded 36GiB to GPU in 1.98222s (18.1614GiB/s) | 20260506 15:38:35.543 | INFO | Downloaded 34GiB to GPU in 1.92518s (17.6607GiB/s) | 20260506 15:38:40.681 | INFO | Analyzing chunk frames = [2492928,2493440) | 20260506 15:38:40.681 | INFO | MemInfo: { | 20260506 15:38:40.681 | INFO | "gpu" : " 17 GB", | 20260506 15:38:40.681 | INFO | "huge" : "", | 20260506 15:38:40.681 | INFO | "huge_shared" : " 27 GB", | 20260506 15:38:40.681 | INFO | "malloc" : " 14 GB", | 20260506 15:38:40.681 | INFO | "pinned" : " 29 GB" | 20260506 15:38:40.681 | INFO | } | 20260506 15:38:40.681 | INFO | Perf report for the last 5.139 seconds: | 20260506 15:38:40.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:38:40.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:40.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:40.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:40.681 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:38:40.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:40.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:40.681 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:38:40.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:40.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:40.682 | INFO | Uploaded 36GiB to GPU in 2.06761s (17.4114GiB/s) | 20260506 15:38:40.682 | INFO | Downloaded 34GiB to GPU in 1.97129s (17.2476GiB/s) | 20260506 15:38:45.822 | INFO | Analyzing chunk frames = [2493440,2493952) | 20260506 15:38:45.822 | INFO | MemInfo: { | 20260506 15:38:45.822 | INFO | "gpu" : " 17 GB", | 20260506 15:38:45.822 | INFO | "huge" : "", | 20260506 15:38:45.822 | INFO | "huge_shared" : " 27 GB", | 20260506 15:38:45.822 | INFO | "malloc" : " 14 GB", | 20260506 15:38:45.822 | INFO | "pinned" : " 29 GB" | 20260506 15:38:45.822 | INFO | } | 20260506 15:38:45.822 | INFO | Perf report for the last 5.14074 seconds: | 20260506 15:38:45.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:38:45.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:45.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:45.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:45.822 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:38:45.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:45.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:45.822 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:38:45.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:45.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:45.822 | INFO | Uploaded 36GiB to GPU in 2.01883s (17.8321GiB/s) | 20260506 15:38:45.822 | INFO | Downloaded 34GiB to GPU in 2.01236s (16.8955GiB/s) | 20260506 15:38:50.961 | INFO | Analyzing chunk frames = [2493952,2494464) | 20260506 15:38:50.961 | INFO | MemInfo: { | 20260506 15:38:50.961 | INFO | "gpu" : " 17 GB", | 20260506 15:38:50.961 | INFO | "huge" : "", | 20260506 15:38:50.961 | INFO | "huge_shared" : " 27 GB", | 20260506 15:38:50.961 | INFO | "malloc" : " 14 GB", | 20260506 15:38:50.962 | INFO | "pinned" : " 29 GB" | 20260506 15:38:50.962 | INFO | } | 20260506 15:38:50.962 | INFO | Perf report for the last 5.1395 seconds: | 20260506 15:38:50.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:38:50.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:50.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:50.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:50.962 | INFO | ANALYSIS : 62.2%, 3.0, 97.1ms, 0 | 20260506 15:38:50.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:50.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:50.962 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 15:38:50.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:50.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:50.962 | INFO | Uploaded 36GiB to GPU in 2.0363s (17.6791GiB/s) | 20260506 15:38:50.962 | INFO | Downloaded 34GiB to GPU in 2.00023s (16.998GiB/s) | 20260506 15:38:51.080 | INFO | Starting a round of Baz IO | 20260506 15:38:54.239 | INFO | Populating RTMetrics report | 20260506 15:38:54.240 | INFO | RTMetrics report populated | 20260506 15:38:54.240 | INFO | Opening tmp RTMetrics json file | 20260506 15:38:54.259 | INFO | Serializing RTMetrics to json | 20260506 15:38:54.321 | INFO | Serialization done, swapping tmp file now... | 20260506 15:38:54.348 | INFO | RTMetrics done | 20260506 15:38:54.869 | INFO | Finished a round of Baz IO: Wrote 2.099 GiB in 3.788 seconds | 20260506 15:38:54.869 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.02% / 0.01% / 62.97% | 20260506 15:38:54.869 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.98173% / 1.19209e-05% / 5.5631e-05% | 20260506 15:38:54.869 | INFO | Baz IO overall: | 20260506 15:38:54.869 | INFO | BytesWritten : 1302583774857 | 20260506 15:38:54.869 | INFO | NumEvents : 630370881995 | 20260506 15:38:56.100 | INFO | Analyzing chunk frames = [2494464,2494976) | 20260506 15:38:56.100 | INFO | MemInfo: { | 20260506 15:38:56.100 | INFO | "gpu" : " 17 GB", | 20260506 15:38:56.101 | INFO | "huge" : "", | 20260506 15:38:56.101 | INFO | "huge_shared" : " 27 GB", | 20260506 15:38:56.101 | INFO | "malloc" : " 14 GB", | 20260506 15:38:56.101 | INFO | "pinned" : " 29 GB" | 20260506 15:38:56.101 | INFO | } | 20260506 15:38:56.101 | INFO | Perf report for the last 5.13906 seconds: | 20260506 15:38:56.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:38:56.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:56.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:56.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:56.101 | INFO | ANALYSIS : 62.9%, 2.9, 94.1ms, 0 | 20260506 15:38:56.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:38:56.101 | INFO | RT_METRICS : 47.8%, 1.0, 25.1ms, 0 | 20260506 15:38:56.101 | INFO | PRE_HQ : 61.4%, 1.9, 61.8ms, 0 | 20260506 15:38:56.101 | INFO | BAZWRITER : 73.2%, 4.8, 180.9ms, 0 | 20260506 15:38:56.101 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 15:38:56.101 | INFO | Uploaded 36GiB to GPU in 2.06007s (17.4752GiB/s) | 20260506 15:38:56.101 | INFO | Downloaded 38GiB to GPU in 2.21084s (17.188GiB/s) | 20260506 15:39:01.239 | INFO | Analyzing chunk frames = [2494976,2495488) | 20260506 15:39:01.239 | INFO | MemInfo: { | 20260506 15:39:01.239 | INFO | "gpu" : " 17 GB", | 20260506 15:39:01.239 | INFO | "huge" : "", | 20260506 15:39:01.239 | INFO | "huge_shared" : " 27 GB", | 20260506 15:39:01.240 | INFO | "malloc" : " 14 GB", | 20260506 15:39:01.240 | INFO | "pinned" : " 29 GB" | 20260506 15:39:01.240 | INFO | } | 20260506 15:39:01.240 | INFO | Perf report for the last 5.13886 seconds: | 20260506 15:39:01.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:39:01.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:01.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:01.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:01.240 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:39:01.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:01.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:01.240 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:39:01.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:01.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:01.240 | INFO | Uploaded 36GiB to GPU in 1.9772s (18.2075GiB/s) | 20260506 15:39:01.240 | INFO | Downloaded 34GiB to GPU in 1.93635s (17.5588GiB/s) | 20260506 15:39:06.378 | INFO | Analyzing chunk frames = [2495488,2496000) | 20260506 15:39:06.378 | INFO | MemInfo: { | 20260506 15:39:06.378 | INFO | "gpu" : " 17 GB", | 20260506 15:39:06.378 | INFO | "huge" : "", | 20260506 15:39:06.378 | INFO | "huge_shared" : " 27 GB", | 20260506 15:39:06.378 | INFO | "malloc" : " 14 GB", | 20260506 15:39:06.378 | INFO | "pinned" : " 29 GB" | 20260506 15:39:06.378 | INFO | } | 20260506 15:39:06.378 | INFO | Perf report for the last 5.13842 seconds: | 20260506 15:39:06.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:39:06.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:06.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:06.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:06.378 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:39:06.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:06.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:06.378 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:39:06.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:06.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:06.378 | INFO | Uploaded 36GiB to GPU in 1.98198s (18.1636GiB/s) | 20260506 15:39:06.378 | INFO | Downloaded 34GiB to GPU in 1.94011s (17.5248GiB/s) | 20260506 15:39:11.518 | INFO | Analyzing chunk frames = [2496000,2496512) | 20260506 15:39:11.518 | INFO | MemInfo: { | 20260506 15:39:11.518 | INFO | "gpu" : " 17 GB", | 20260506 15:39:11.518 | INFO | "huge" : "", | 20260506 15:39:11.518 | INFO | "huge_shared" : " 27 GB", | 20260506 15:39:11.518 | INFO | "malloc" : " 14 GB", | 20260506 15:39:11.518 | INFO | "pinned" : " 29 GB" | 20260506 15:39:11.518 | INFO | } | 20260506 15:39:11.518 | INFO | Perf report for the last 5.14026 seconds: | 20260506 15:39:11.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:39:11.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:11.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:11.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:11.518 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 15:39:11.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:11.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:11.518 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:39:11.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:11.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:11.519 | INFO | Uploaded 36GiB to GPU in 2.01739s (17.8449GiB/s) | 20260506 15:39:11.519 | INFO | Downloaded 34GiB to GPU in 1.91973s (17.7108GiB/s) | 20260506 15:39:16.658 | INFO | Analyzing chunk frames = [2496512,2497024) | 20260506 15:39:16.658 | INFO | MemInfo: { | 20260506 15:39:16.658 | INFO | "gpu" : " 17 GB", | 20260506 15:39:16.658 | INFO | "huge" : "", | 20260506 15:39:16.658 | INFO | "huge_shared" : " 27 GB", | 20260506 15:39:16.659 | INFO | "malloc" : " 14 GB", | 20260506 15:39:16.659 | INFO | "pinned" : " 29 GB" | 20260506 15:39:16.659 | INFO | } | 20260506 15:39:16.659 | INFO | Perf report for the last 5.14031 seconds: | 20260506 15:39:16.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:39:16.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:16.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:16.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:16.659 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 15:39:16.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:16.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:16.659 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:39:16.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:16.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:16.659 | INFO | Uploaded 36GiB to GPU in 1.99486s (18.0464GiB/s) | 20260506 15:39:16.659 | INFO | Downloaded 34GiB to GPU in 1.93784s (17.5453GiB/s) | 20260506 15:39:21.796 | INFO | Analyzing chunk frames = [2497024,2497536) | 20260506 15:39:21.796 | INFO | MemInfo: { | 20260506 15:39:21.796 | INFO | "gpu" : " 17 GB", | 20260506 15:39:21.796 | INFO | "huge" : "", | 20260506 15:39:21.796 | INFO | "huge_shared" : " 27 GB", | 20260506 15:39:21.796 | INFO | "malloc" : " 14 GB", | 20260506 15:39:21.796 | INFO | "pinned" : " 29 GB" | 20260506 15:39:21.796 | INFO | } | 20260506 15:39:21.796 | INFO | Perf report for the last 5.13789 seconds: | 20260506 15:39:21.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:39:21.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:21.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:21.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:21.797 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 15:39:21.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:21.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:21.797 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:39:21.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:21.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:21.797 | INFO | Uploaded 36GiB to GPU in 2.0683s (17.4056GiB/s) | 20260506 15:39:21.797 | INFO | Downloaded 34GiB to GPU in 1.96861s (17.2711GiB/s) | 20260506 15:39:26.937 | INFO | Analyzing chunk frames = [2497536,2498048) | 20260506 15:39:26.937 | INFO | MemInfo: { | 20260506 15:39:26.937 | INFO | "gpu" : " 17 GB", | 20260506 15:39:26.937 | INFO | "huge" : "", | 20260506 15:39:26.937 | INFO | "huge_shared" : " 27 GB", | 20260506 15:39:26.937 | INFO | "malloc" : " 14 GB", | 20260506 15:39:26.937 | INFO | "pinned" : " 29 GB" | 20260506 15:39:26.937 | INFO | } | 20260506 15:39:26.937 | INFO | Perf report for the last 5.1405 seconds: | 20260506 15:39:26.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:39:26.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:26.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:26.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:26.937 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:39:26.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:26.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:26.937 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:39:26.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:26.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:26.937 | INFO | Uploaded 36GiB to GPU in 1.98233s (18.1604GiB/s) | 20260506 15:39:26.937 | INFO | Downloaded 34GiB to GPU in 1.99831s (17.0144GiB/s) | 20260506 15:39:32.076 | INFO | Analyzing chunk frames = [2498048,2498560) | 20260506 15:39:32.076 | INFO | MemInfo: { | 20260506 15:39:32.076 | INFO | "gpu" : " 17 GB", | 20260506 15:39:32.076 | INFO | "huge" : "", | 20260506 15:39:32.076 | INFO | "huge_shared" : " 27 GB", | 20260506 15:39:32.076 | INFO | "malloc" : " 14 GB", | 20260506 15:39:32.076 | INFO | "pinned" : " 29 GB" | 20260506 15:39:32.076 | INFO | } | 20260506 15:39:32.076 | INFO | Perf report for the last 5.13911 seconds: | 20260506 15:39:32.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:39:32.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:32.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:32.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:32.076 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 15:39:32.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:32.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:32.076 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 15:39:32.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:32.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:32.076 | INFO | Uploaded 36GiB to GPU in 1.99849s (18.0136GiB/s) | 20260506 15:39:32.076 | INFO | Downloaded 34GiB to GPU in 1.98726s (17.109GiB/s) | 20260506 15:39:32.077 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:39:32.077 | INFO | Total mem usage: 17.4 GiB | 20260506 15:39:32.077 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:39:32.077 | INFO | Recent High Water Marks Sum: 16.3 GiB (Savings: -6.92%) | 20260506 15:39:32.077 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:39:32.077 | INFO | High Water Breakdown: | 20260506 15:39:32.077 | INFO | 61.9% : 10.1 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:39:32.077 | INFO | 38% : 6.18 GiB ( 92.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:39:32.077 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:39:32.077 | INFO | ------------------------------------------ | 20260506 15:39:32.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:39:32.077 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:39:32.077 | INFO | Total mem usage: 26.5 GiB | 20260506 15:39:32.077 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:39:32.077 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:39:32.077 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 15:39:32.077 | INFO | High Water Breakdown: | 20260506 15:39:32.077 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:39:32.077 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:39:32.077 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:39:32.077 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:39:32.077 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:39:32.077 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:39:32.077 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:39:32.077 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:39:32.077 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:39:32.077 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:39:32.077 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:39:32.077 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:39:32.077 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:39:32.077 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:39:32.077 | INFO | ------------------------------------------ | 20260506 15:39:32.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:39:32.077 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:39:32.077 | INFO | Total mem usage: 25.6 GiB | 20260506 15:39:32.077 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:39:32.077 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:39:32.077 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:39:32.077 | INFO | High Water Breakdown: | 20260506 15:39:32.077 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:39:32.077 | INFO | ------------------------------------------ | 20260506 15:39:32.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:39:32.077 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:39:32.077 | INFO | Total mem usage: 15.7 GiB | 20260506 15:39:32.077 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:39:32.077 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:39:32.077 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 15:39:32.077 | INFO | High Water Breakdown: | 20260506 15:39:32.077 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:39:32.077 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:39:32.077 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:39:32.077 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:39:32.077 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:39:32.077 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:39:32.077 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:39:32.077 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:39:32.077 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:39:32.077 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:39:32.077 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:39:32.077 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:39:32.077 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:39:32.077 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:39:32.077 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:39:32.077 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:39:32.077 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:39:32.077 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:39:32.077 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:39:32.078 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:39:32.078 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:39:32.078 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:39:32.078 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:39:32.078 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:39:32.078 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:39:32.078 | INFO | ------------------------------------------ | 20260506 15:39:32.078 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:39:32.078 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:39:32.078 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:39:32.078 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:39:32.078 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.53e+01, 1.36e+01 / 1.81e+00 | 20260506 15:39:32.078 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 9.88e-02, 1.02e-02 / 7.34e-03 | 20260506 15:39:32.078 | INFO | PerfReport: 11.58%, Baseline -- 3.64e+00 / 4.36e+00, 5.67e-01 / 4.37e-01 | 20260506 15:39:32.078 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.43e+00, 1.87e-01 / 1.65e-01 | 20260506 15:39:32.078 | INFO | PerfReport: 15.82%, DME -- 3.98e+01 / 4.35e+01, 3.68e+00 / 1.80e+00 | 20260506 15:39:32.078 | INFO | PerfReport: 56.89%, FrameLabeling -- 1.79e+01 / 2.11e+01, 1.47e+00 / 1.97e+00 | 20260506 15:39:32.078 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.32e+00 / 2.87e+00, 4.01e-01 / 2.59e-01 | 20260506 15:39:32.078 | INFO | PerfReport: 3.55%, Metrics -- 1.12e+00 / 1.46e+00, 2.92e-01 / 1.99e-01 | 20260506 15:39:32.078 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.87e-01 / 4.39e-01, 6.03e-02 / 3.20e-02 | 20260506 15:39:32.078 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:39:32.078 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:39:32.078 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:39:32.078 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 15:39:32.078 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 15:39:32.078 | INFO | PerfReport: 21.26%, Download -- 2.07e+01 / 0.00e+00, 7.53e+00 / 0.00e+00 | 20260506 15:39:32.078 | INFO | PerfReport: 32.28%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 15:39:32.078 | INFO | PerfReport: 1.44%, LockUpload -- 1.40e+00 / 0.00e+00, 5.77e+00 / 0.00e+00 | 20260506 15:39:32.078 | INFO | PerfReport: 1.19%, LockDownload -- 1.16e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260506 15:39:32.078 | INFO | PerfReport: 22.32%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 15:39:32.078 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260506 15:39:32.196 | INFO | Starting a round of Baz IO | 20260506 15:39:35.341 | INFO | Populating RTMetrics report | 20260506 15:39:35.341 | INFO | RTMetrics report populated | 20260506 15:39:35.341 | INFO | Opening tmp RTMetrics json file | 20260506 15:39:35.393 | INFO | Serializing RTMetrics to json | 20260506 15:39:35.397 | INFO | Serialization done, swapping tmp file now... | 20260506 15:39:35.402 | INFO | RTMetrics done | 20260506 15:39:36.083 | INFO | Finished a round of Baz IO: Wrote 2.098 GiB in 3.887 seconds | 20260506 15:39:36.083 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 37.00% / 0.01% / 62.98% | 20260506 15:39:36.083 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.97247% / 7.94729e-06% / 7.15256e-05% | 20260506 15:39:36.083 | INFO | Baz IO overall: | 20260506 15:39:36.083 | INFO | BytesWritten : 1304836001417 | 20260506 15:39:36.083 | INFO | NumEvents : 631459972634 | 20260506 15:39:37.214 | INFO | Analyzing chunk frames = [2498560,2499072) | 20260506 15:39:37.214 | INFO | MemInfo: { | 20260506 15:39:37.214 | INFO | "gpu" : " 17 GB", | 20260506 15:39:37.215 | INFO | "huge" : "", | 20260506 15:39:37.215 | INFO | "huge_shared" : " 27 GB", | 20260506 15:39:37.215 | INFO | "malloc" : " 14 GB", | 20260506 15:39:37.215 | INFO | "pinned" : " 29 GB" | 20260506 15:39:37.215 | INFO | } | 20260506 15:39:37.215 | INFO | Perf report for the last 5.13849 seconds: | 20260506 15:39:37.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:39:37.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:37.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:37.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:37.215 | INFO | ANALYSIS : 63.0%, 2.9, 95.3ms, 0 | 20260506 15:39:37.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:37.215 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 15:39:37.215 | INFO | PRE_HQ : 60.8%, 1.9, 59.1ms, 0 | 20260506 15:39:37.215 | INFO | BAZWRITER : 75.3%, 4.9, 187.7ms, 0 | 20260506 15:39:37.215 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 15:39:37.215 | INFO | Uploaded 36GiB to GPU in 2.08238s (17.2879GiB/s) | 20260506 15:39:37.215 | INFO | Downloaded 38GiB to GPU in 2.22915s (17.0469GiB/s) | 20260506 15:39:42.354 | INFO | Analyzing chunk frames = [2499072,2499584) | 20260506 15:39:42.354 | INFO | MemInfo: { | 20260506 15:39:42.354 | INFO | "gpu" : " 17 GB", | 20260506 15:39:42.354 | INFO | "huge" : "", | 20260506 15:39:42.354 | INFO | "huge_shared" : " 27 GB", | 20260506 15:39:42.354 | INFO | "malloc" : " 14 GB", | 20260506 15:39:42.354 | INFO | "pinned" : " 29 GB" | 20260506 15:39:42.354 | INFO | } | 20260506 15:39:42.354 | INFO | Perf report for the last 5.13921 seconds: | 20260506 15:39:42.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:39:42.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:42.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:42.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:42.354 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:39:42.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:42.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:42.354 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:39:42.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:42.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:42.354 | INFO | Uploaded 36GiB to GPU in 1.98726s (18.1154GiB/s) | 20260506 15:39:42.354 | INFO | Downloaded 34GiB to GPU in 1.94476s (17.4829GiB/s) | 20260506 15:39:47.493 | INFO | Analyzing chunk frames = [2499584,2500096) | 20260506 15:39:47.493 | INFO | MemInfo: { | 20260506 15:39:47.493 | INFO | "gpu" : " 17 GB", | 20260506 15:39:47.493 | INFO | "huge" : "", | 20260506 15:39:47.493 | INFO | "huge_shared" : " 27 GB", | 20260506 15:39:47.493 | INFO | "malloc" : " 14 GB", | 20260506 15:39:47.493 | INFO | "pinned" : " 29 GB" | 20260506 15:39:47.493 | INFO | } | 20260506 15:39:47.493 | INFO | Perf report for the last 5.13873 seconds: | 20260506 15:39:47.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:39:47.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:47.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:47.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:47.493 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:39:47.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:47.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:47.493 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:39:47.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:47.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:47.493 | INFO | Uploaded 36GiB to GPU in 1.96057s (18.362GiB/s) | 20260506 15:39:47.493 | INFO | Downloaded 34GiB to GPU in 1.93896s (17.5352GiB/s) | 20260506 15:39:52.632 | INFO | Analyzing chunk frames = [2500096,2500608) | 20260506 15:39:52.632 | INFO | MemInfo: { | 20260506 15:39:52.632 | INFO | "gpu" : " 17 GB", | 20260506 15:39:52.632 | INFO | "huge" : "", | 20260506 15:39:52.632 | INFO | "huge_shared" : " 27 GB", | 20260506 15:39:52.632 | INFO | "malloc" : " 14 GB", | 20260506 15:39:52.632 | INFO | "pinned" : " 29 GB" | 20260506 15:39:52.632 | INFO | } | 20260506 15:39:52.632 | INFO | Perf report for the last 5.13968 seconds: | 20260506 15:39:52.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:39:52.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:52.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:52.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:52.633 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 15:39:52.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:52.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:52.633 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:39:52.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:52.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:52.633 | INFO | Uploaded 36GiB to GPU in 2.01603s (17.8569GiB/s) | 20260506 15:39:52.633 | INFO | Downloaded 34GiB to GPU in 1.91673s (17.7385GiB/s) | 20260506 15:39:57.772 | INFO | Analyzing chunk frames = [2500608,2501120) | 20260506 15:39:57.772 | INFO | MemInfo: { | 20260506 15:39:57.772 | INFO | "gpu" : " 17 GB", | 20260506 15:39:57.772 | INFO | "huge" : "", | 20260506 15:39:57.772 | INFO | "huge_shared" : " 27 GB", | 20260506 15:39:57.772 | INFO | "malloc" : " 14 GB", | 20260506 15:39:57.772 | INFO | "pinned" : " 29 GB" | 20260506 15:39:57.772 | INFO | } | 20260506 15:39:57.772 | INFO | Perf report for the last 5.14011 seconds: | 20260506 15:39:57.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:39:57.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:57.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:57.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:57.773 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 15:39:57.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:57.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:57.773 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:39:57.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:57.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:39:57.773 | INFO | Uploaded 36GiB to GPU in 1.98384s (18.1467GiB/s) | 20260506 15:39:57.773 | INFO | Downloaded 34GiB to GPU in 1.94139s (17.5132GiB/s) | 20260506 15:40:02.912 | INFO | Analyzing chunk frames = [2501120,2501632) | 20260506 15:40:02.912 | INFO | MemInfo: { | 20260506 15:40:02.912 | INFO | "gpu" : " 17 GB", | 20260506 15:40:02.912 | INFO | "huge" : "", | 20260506 15:40:02.912 | INFO | "huge_shared" : " 27 GB", | 20260506 15:40:02.912 | INFO | "malloc" : " 14 GB", | 20260506 15:40:02.912 | INFO | "pinned" : " 29 GB" | 20260506 15:40:02.912 | INFO | } | 20260506 15:40:02.913 | INFO | Perf report for the last 5.1401 seconds: | 20260506 15:40:02.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:40:02.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:02.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:02.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:02.913 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 15:40:02.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:02.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:02.913 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:40:02.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:02.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:02.913 | INFO | Uploaded 36GiB to GPU in 2.07041s (17.3879GiB/s) | 20260506 15:40:02.913 | INFO | Downloaded 34GiB to GPU in 1.96981s (17.2605GiB/s) | 20260506 15:40:08.052 | INFO | Analyzing chunk frames = [2501632,2502144) | 20260506 15:40:08.052 | INFO | MemInfo: { | 20260506 15:40:08.052 | INFO | "gpu" : " 17 GB", | 20260506 15:40:08.052 | INFO | "huge" : "", | 20260506 15:40:08.052 | INFO | "huge_shared" : " 27 GB", | 20260506 15:40:08.052 | INFO | "malloc" : " 14 GB", | 20260506 15:40:08.052 | INFO | "pinned" : " 29 GB" | 20260506 15:40:08.052 | INFO | } | 20260506 15:40:08.052 | INFO | Perf report for the last 5.13914 seconds: | 20260506 15:40:08.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:40:08.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:08.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:08.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:08.052 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 15:40:08.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:08.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:08.052 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:40:08.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:08.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:08.052 | INFO | Uploaded 36GiB to GPU in 2.04628s (17.5929GiB/s) | 20260506 15:40:08.052 | INFO | Downloaded 34GiB to GPU in 2.01328s (16.8879GiB/s) | 20260506 15:40:13.190 | INFO | Analyzing chunk frames = [2502144,2502656) | 20260506 15:40:13.190 | INFO | MemInfo: { | 20260506 15:40:13.190 | INFO | "gpu" : " 17 GB", | 20260506 15:40:13.190 | INFO | "huge" : "", | 20260506 15:40:13.190 | INFO | "huge_shared" : " 27 GB", | 20260506 15:40:13.190 | INFO | "malloc" : " 14 GB", | 20260506 15:40:13.190 | INFO | "pinned" : " 29 GB" | 20260506 15:40:13.190 | INFO | } | 20260506 15:40:13.190 | INFO | Perf report for the last 5.13832 seconds: | 20260506 15:40:13.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:40:13.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:13.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:13.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:13.190 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:40:13.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:13.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:13.190 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:40:13.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:13.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:13.190 | INFO | Uploaded 36GiB to GPU in 2.03329s (17.7053GiB/s) | 20260506 15:40:13.190 | INFO | Downloaded 34GiB to GPU in 1.97658s (17.2014GiB/s) | 20260506 15:40:13.301 | INFO | Starting a round of Baz IO | 20260506 15:40:16.451 | INFO | Populating RTMetrics report | 20260506 15:40:16.452 | INFO | RTMetrics report populated | 20260506 15:40:16.452 | INFO | Opening tmp RTMetrics json file | 20260506 15:40:16.506 | INFO | Serializing RTMetrics to json | 20260506 15:40:16.585 | INFO | Serialization done, swapping tmp file now... | 20260506 15:40:16.667 | INFO | RTMetrics done | 20260506 15:40:17.025 | INFO | Finished a round of Baz IO: Wrote 2.098 GiB in 3.723 seconds | 20260506 15:40:17.025 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.99% / 0.01% / 63.00% | 20260506 15:40:17.025 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.96977% / 2.38419e-05% / 5.5631e-05% | 20260506 15:40:17.025 | INFO | Baz IO overall: | 20260506 15:40:17.025 | INFO | BytesWritten : 1307089131962 | 20260506 15:40:17.025 | INFO | NumEvents : 632549961235 | 20260506 15:40:18.329 | INFO | Analyzing chunk frames = [2502656,2503168) | 20260506 15:40:18.329 | INFO | MemInfo: { | 20260506 15:40:18.329 | INFO | "gpu" : " 17 GB", | 20260506 15:40:18.329 | INFO | "huge" : "", | 20260506 15:40:18.329 | INFO | "huge_shared" : " 27 GB", | 20260506 15:40:18.329 | INFO | "malloc" : " 14 GB", | 20260506 15:40:18.329 | INFO | "pinned" : " 29 GB" | 20260506 15:40:18.329 | INFO | } | 20260506 15:40:18.329 | INFO | Perf report for the last 5.13892 seconds: | 20260506 15:40:18.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:40:18.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:18.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:18.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:18.329 | INFO | ANALYSIS : 62.9%, 2.9, 95.3ms, 0 | 20260506 15:40:18.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:18.329 | INFO | RT_METRICS : 47.9%, 1.0, 25.1ms, 0 | 20260506 15:40:18.329 | INFO | PRE_HQ : 60.8%, 1.9, 59.1ms, 0 | 20260506 15:40:18.329 | INFO | BAZWRITER : 71.8%, 4.8, 178.6ms, 0 | 20260506 15:40:18.329 | INFO | BAZ_SAMPLING : 3.3%, 1.0, 1.7ms, 0 | 20260506 15:40:18.329 | INFO | Uploaded 36GiB to GPU in 2.07698s (17.3328GiB/s) | 20260506 15:40:18.329 | INFO | Downloaded 38GiB to GPU in 2.24763s (16.9067GiB/s) | 20260506 15:40:23.468 | INFO | Analyzing chunk frames = [2503168,2503680) | 20260506 15:40:23.469 | INFO | MemInfo: { | 20260506 15:40:23.469 | INFO | "gpu" : " 17 GB", | 20260506 15:40:23.469 | INFO | "huge" : "", | 20260506 15:40:23.469 | INFO | "huge_shared" : " 27 GB", | 20260506 15:40:23.469 | INFO | "malloc" : " 14 GB", | 20260506 15:40:23.469 | INFO | "pinned" : " 29 GB" | 20260506 15:40:23.469 | INFO | } | 20260506 15:40:23.469 | INFO | Perf report for the last 5.13974 seconds: | 20260506 15:40:23.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:40:23.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:23.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:23.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:23.469 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:40:23.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:23.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:23.469 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:40:23.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:23.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:23.469 | INFO | Uploaded 36GiB to GPU in 1.99194s (18.0728GiB/s) | 20260506 15:40:23.469 | INFO | Downloaded 34GiB to GPU in 1.93117s (17.6059GiB/s) | 20260506 15:40:28.608 | INFO | Analyzing chunk frames = [2503680,2504192) | 20260506 15:40:28.608 | INFO | MemInfo: { | 20260506 15:40:28.608 | INFO | "gpu" : " 17 GB", | 20260506 15:40:28.609 | INFO | "huge" : "", | 20260506 15:40:28.609 | INFO | "huge_shared" : " 27 GB", | 20260506 15:40:28.609 | INFO | "malloc" : " 14 GB", | 20260506 15:40:28.609 | INFO | "pinned" : " 29 GB" | 20260506 15:40:28.609 | INFO | } | 20260506 15:40:28.609 | INFO | Perf report for the last 5.14 seconds: | 20260506 15:40:28.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:40:28.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:28.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:28.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:28.609 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:40:28.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:28.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:28.609 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:40:28.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:28.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:28.609 | INFO | Uploaded 36GiB to GPU in 1.99736s (18.0238GiB/s) | 20260506 15:40:28.609 | INFO | Downloaded 34GiB to GPU in 1.94011s (17.5248GiB/s) | 20260506 15:40:33.749 | INFO | Analyzing chunk frames = [2504192,2504704) | 20260506 15:40:33.749 | INFO | MemInfo: { | 20260506 15:40:33.749 | INFO | "gpu" : " 17 GB", | 20260506 15:40:33.749 | INFO | "huge" : "", | 20260506 15:40:33.749 | INFO | "huge_shared" : " 27 GB", | 20260506 15:40:33.749 | INFO | "malloc" : " 14 GB", | 20260506 15:40:33.749 | INFO | "pinned" : " 29 GB" | 20260506 15:40:33.749 | INFO | } | 20260506 15:40:33.749 | INFO | Perf report for the last 5.1399 seconds: | 20260506 15:40:33.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:40:33.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:33.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:33.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:33.749 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 15:40:33.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:33.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:33.749 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:40:33.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:33.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:33.749 | INFO | Uploaded 36GiB to GPU in 1.97706s (18.2089GiB/s) | 20260506 15:40:33.749 | INFO | Downloaded 34GiB to GPU in 1.88172s (18.0686GiB/s) | 20260506 15:40:38.887 | INFO | Analyzing chunk frames = [2504704,2505216) | 20260506 15:40:38.887 | INFO | MemInfo: { | 20260506 15:40:38.887 | INFO | "gpu" : " 17 GB", | 20260506 15:40:38.887 | INFO | "huge" : "", | 20260506 15:40:38.887 | INFO | "huge_shared" : " 27 GB", | 20260506 15:40:38.887 | INFO | "malloc" : " 14 GB", | 20260506 15:40:38.887 | INFO | "pinned" : " 29 GB" | 20260506 15:40:38.887 | INFO | } | 20260506 15:40:38.887 | INFO | Perf report for the last 5.13827 seconds: | 20260506 15:40:38.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:40:38.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:38.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:38.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:38.887 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 15:40:38.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:38.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:38.887 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 15:40:38.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:38.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:38.887 | INFO | Uploaded 36GiB to GPU in 1.98475s (18.1383GiB/s) | 20260506 15:40:38.887 | INFO | Downloaded 34GiB to GPU in 1.92957s (17.6205GiB/s) | 20260506 15:40:44.027 | INFO | Analyzing chunk frames = [2505216,2505728) | 20260506 15:40:44.027 | INFO | MemInfo: { | 20260506 15:40:44.027 | INFO | "gpu" : " 17 GB", | 20260506 15:40:44.027 | INFO | "huge" : "", | 20260506 15:40:44.027 | INFO | "huge_shared" : " 27 GB", | 20260506 15:40:44.027 | INFO | "malloc" : " 14 GB", | 20260506 15:40:44.027 | INFO | "pinned" : " 29 GB" | 20260506 15:40:44.027 | INFO | } | 20260506 15:40:44.027 | INFO | Perf report for the last 5.14012 seconds: | 20260506 15:40:44.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:40:44.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:44.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:44.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:44.027 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 15:40:44.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:44.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:44.027 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:40:44.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:44.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:44.027 | INFO | Uploaded 36GiB to GPU in 2.07662s (17.3359GiB/s) | 20260506 15:40:44.027 | INFO | Downloaded 34GiB to GPU in 1.96218s (17.3276GiB/s) | 20260506 15:40:49.166 | INFO | Analyzing chunk frames = [2505728,2506240) | 20260506 15:40:49.166 | INFO | MemInfo: { | 20260506 15:40:49.166 | INFO | "gpu" : " 17 GB", | 20260506 15:40:49.166 | INFO | "huge" : "", | 20260506 15:40:49.166 | INFO | "huge_shared" : " 27 GB", | 20260506 15:40:49.166 | INFO | "malloc" : " 14 GB", | 20260506 15:40:49.166 | INFO | "pinned" : " 29 GB" | 20260506 15:40:49.166 | INFO | } | 20260506 15:40:49.166 | INFO | Perf report for the last 5.13894 seconds: | 20260506 15:40:49.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:40:49.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:49.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:49.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:49.166 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 15:40:49.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:49.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:49.166 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:40:49.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:49.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:49.166 | INFO | Uploaded 36GiB to GPU in 2.03563s (17.6849GiB/s) | 20260506 15:40:49.166 | INFO | Downloaded 34GiB to GPU in 1.98757s (17.1063GiB/s) | 20260506 15:40:54.304 | INFO | Analyzing chunk frames = [2506240,2506752) | 20260506 15:40:54.304 | INFO | MemInfo: { | 20260506 15:40:54.304 | INFO | "gpu" : " 17 GB", | 20260506 15:40:54.304 | INFO | "huge" : "", | 20260506 15:40:54.304 | INFO | "huge_shared" : " 27 GB", | 20260506 15:40:54.304 | INFO | "malloc" : " 14 GB", | 20260506 15:40:54.304 | INFO | "pinned" : " 29 GB" | 20260506 15:40:54.304 | INFO | } | 20260506 15:40:54.304 | INFO | Perf report for the last 5.13822 seconds: | 20260506 15:40:54.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:40:54.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:54.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:54.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:54.304 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 15:40:54.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:54.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:54.304 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:40:54.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:54.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:54.304 | INFO | Uploaded 36GiB to GPU in 1.99143s (18.0775GiB/s) | 20260506 15:40:54.304 | INFO | Downloaded 34GiB to GPU in 1.97915s (17.1791GiB/s) | 20260506 15:40:54.305 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:40:54.305 | INFO | Total mem usage: 17.4 GiB | 20260506 15:40:54.305 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:40:54.305 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: -5.49%) | 20260506 15:40:54.305 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:40:54.305 | INFO | High Water Breakdown: | 20260506 15:40:54.305 | INFO | 62.1% : 10.3 GiB ( 93.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:40:54.305 | INFO | 37.8% : 6.23 GiB ( 93.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:40:54.305 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:40:54.305 | INFO | ------------------------------------------ | 20260506 15:40:54.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:40:54.305 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:40:54.305 | INFO | Total mem usage: 26.5 GiB | 20260506 15:40:54.305 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:40:54.305 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:40:54.305 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 15:40:54.305 | INFO | High Water Breakdown: | 20260506 15:40:54.305 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:40:54.305 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:40:54.305 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:40:54.305 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:40:54.305 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:40:54.305 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:40:54.305 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:40:54.305 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:40:54.305 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:40:54.305 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:40:54.305 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:40:54.305 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:40:54.305 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:40:54.305 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:40:54.305 | INFO | ------------------------------------------ | 20260506 15:40:54.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:40:54.305 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:40:54.305 | INFO | Total mem usage: 25.6 GiB | 20260506 15:40:54.305 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:40:54.305 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:40:54.305 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:40:54.305 | INFO | High Water Breakdown: | 20260506 15:40:54.305 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:40:54.305 | INFO | ------------------------------------------ | 20260506 15:40:54.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:40:54.305 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:40:54.305 | INFO | Total mem usage: 15.7 GiB | 20260506 15:40:54.305 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:40:54.305 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:40:54.305 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 15:40:54.305 | INFO | High Water Breakdown: | 20260506 15:40:54.305 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:40:54.305 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:40:54.305 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:40:54.305 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:40:54.305 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:40:54.305 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:40:54.305 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:40:54.305 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:40:54.306 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:40:54.306 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:40:54.306 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:40:54.306 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:40:54.306 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:40:54.306 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:40:54.306 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:40:54.306 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:40:54.306 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:40:54.306 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:40:54.306 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:40:54.306 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:40:54.306 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:40:54.306 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:40:54.306 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:40:54.306 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:40:54.306 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:40:54.306 | INFO | ------------------------------------------ | 20260506 15:40:54.306 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:40:54.306 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:40:54.306 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:40:54.306 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:40:54.306 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.52e+01, 1.35e+01 / 2.16e+00 | 20260506 15:40:54.306 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 1.01e-02 / 7.44e-03 | 20260506 15:40:54.306 | INFO | PerfReport: 11.56%, Baseline -- 3.63e+00 / 4.36e+00, 5.52e-01 / 4.23e-01 | 20260506 15:40:54.306 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.44e+00, 1.82e-01 / 1.68e-01 | 20260506 15:40:54.306 | INFO | PerfReport: 15.88%, DME -- 3.99e+01 / 4.39e+01, 3.81e+00 / 2.49e+00 | 20260506 15:40:54.306 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.42e+00 / 2.10e+00 | 20260506 15:40:54.306 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.33e+00 / 2.92e+00, 4.08e-01 / 2.58e-01 | 20260506 15:40:54.306 | INFO | PerfReport: 3.55%, Metrics -- 1.11e+00 / 1.50e+00, 2.88e-01 / 1.88e-01 | 20260506 15:40:54.306 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.87e-01 / 4.48e-01, 6.08e-02 / 4.25e-02 | 20260506 15:40:54.306 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:40:54.306 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:40:54.306 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:40:54.306 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 15:40:54.306 | INFO | PerfReport: 21.58%, Upload -- 2.10e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260506 15:40:54.306 | INFO | PerfReport: 21.23%, Download -- 2.07e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 15:40:54.306 | INFO | PerfReport: 32.22%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 15:40:54.306 | INFO | PerfReport: 1.47%, LockUpload -- 1.43e+00 / 0.00e+00, 5.74e+00 / 0.00e+00 | 20260506 15:40:54.306 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.70e+00 / 0.00e+00 | 20260506 15:40:54.306 | INFO | PerfReport: 22.38%, LockCompute -- 2.18e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 15:40:54.306 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260506 15:40:54.420 | INFO | Starting a round of Baz IO | 20260506 15:40:57.561 | INFO | Populating RTMetrics report | 20260506 15:40:57.562 | INFO | RTMetrics report populated | 20260506 15:40:57.562 | INFO | Opening tmp RTMetrics json file | 20260506 15:40:57.617 | INFO | Serializing RTMetrics to json | 20260506 15:40:57.636 | INFO | Serialization done, swapping tmp file now... | 20260506 15:40:57.651 | INFO | RTMetrics done | 20260506 15:40:58.014 | INFO | Finished a round of Baz IO: Wrote 2.097 GiB in 3.593 seconds | 20260506 15:40:58.014 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.98% / 0.01% / 63.01% | 20260506 15:40:58.014 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.967% / 3.17891e-05% / 6.75519e-05% | 20260506 15:40:58.014 | INFO | Baz IO overall: | 20260506 15:40:58.014 | INFO | BytesWritten : 1309340790855 | 20260506 15:40:58.014 | INFO | NumEvents : 633639057184 | 20260506 15:40:59.444 | INFO | Analyzing chunk frames = [2506752,2507264) | 20260506 15:40:59.444 | INFO | MemInfo: { | 20260506 15:40:59.444 | INFO | "gpu" : " 17 GB", | 20260506 15:40:59.444 | INFO | "huge" : "", | 20260506 15:40:59.444 | INFO | "huge_shared" : " 27 GB", | 20260506 15:40:59.444 | INFO | "malloc" : " 14 GB", | 20260506 15:40:59.444 | INFO | "pinned" : " 29 GB" | 20260506 15:40:59.444 | INFO | } | 20260506 15:40:59.444 | INFO | Perf report for the last 5.13977 seconds: | 20260506 15:40:59.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:40:59.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:59.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:59.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:59.444 | INFO | ANALYSIS : 62.8%, 2.9, 95.7ms, 0 | 20260506 15:40:59.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:40:59.444 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 15:40:59.444 | INFO | PRE_HQ : 60.4%, 1.9, 59.1ms, 0 | 20260506 15:40:59.444 | INFO | BAZWRITER : 69.5%, 4.8, 170.4ms, 0 | 20260506 15:40:59.444 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 15:40:59.444 | INFO | Uploaded 36GiB to GPU in 2.06902s (17.3995GiB/s) | 20260506 15:40:59.445 | INFO | Downloaded 38GiB to GPU in 2.21283s (17.1726GiB/s) | 20260506 15:41:04.584 | INFO | Analyzing chunk frames = [2507264,2507776) | 20260506 15:41:04.584 | INFO | MemInfo: { | 20260506 15:41:04.584 | INFO | "gpu" : " 17 GB", | 20260506 15:41:04.584 | INFO | "huge" : "", | 20260506 15:41:04.584 | INFO | "huge_shared" : " 27 GB", | 20260506 15:41:04.584 | INFO | "malloc" : " 14 GB", | 20260506 15:41:04.584 | INFO | "pinned" : " 29 GB" | 20260506 15:41:04.584 | INFO | } | 20260506 15:41:04.584 | INFO | Perf report for the last 5.13975 seconds: | 20260506 15:41:04.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:41:04.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:04.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:04.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:04.584 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:41:04.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:04.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:04.584 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:41:04.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:04.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:04.584 | INFO | Uploaded 36GiB to GPU in 1.99617s (18.0346GiB/s) | 20260506 15:41:04.584 | INFO | Downloaded 34GiB to GPU in 1.9415s (17.5123GiB/s) | 20260506 15:41:09.723 | INFO | Analyzing chunk frames = [2507776,2508288) | 20260506 15:41:09.723 | INFO | MemInfo: { | 20260506 15:41:09.723 | INFO | "gpu" : " 17 GB", | 20260506 15:41:09.723 | INFO | "huge" : "", | 20260506 15:41:09.723 | INFO | "huge_shared" : " 27 GB", | 20260506 15:41:09.723 | INFO | "malloc" : " 14 GB", | 20260506 15:41:09.723 | INFO | "pinned" : " 29 GB" | 20260506 15:41:09.723 | INFO | } | 20260506 15:41:09.723 | INFO | Perf report for the last 5.13952 seconds: | 20260506 15:41:09.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:41:09.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:09.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:09.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:09.723 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:41:09.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:09.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:09.723 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:41:09.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:09.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:09.723 | INFO | Uploaded 36GiB to GPU in 1.99036s (18.0872GiB/s) | 20260506 15:41:09.723 | INFO | Downloaded 34GiB to GPU in 1.93675s (17.5551GiB/s) | 20260506 15:41:14.862 | INFO | Analyzing chunk frames = [2508288,2508800) | 20260506 15:41:14.862 | INFO | MemInfo: { | 20260506 15:41:14.862 | INFO | "gpu" : " 17 GB", | 20260506 15:41:14.862 | INFO | "huge" : "", | 20260506 15:41:14.862 | INFO | "huge_shared" : " 27 GB", | 20260506 15:41:14.862 | INFO | "malloc" : " 14 GB", | 20260506 15:41:14.862 | INFO | "pinned" : " 29 GB" | 20260506 15:41:14.862 | INFO | } | 20260506 15:41:14.862 | INFO | Perf report for the last 5.13888 seconds: | 20260506 15:41:14.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:41:14.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:14.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:14.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:14.862 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 15:41:14.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:14.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:14.862 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:41:14.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:14.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:14.862 | INFO | Uploaded 36GiB to GPU in 1.96803s (18.2924GiB/s) | 20260506 15:41:14.862 | INFO | Downloaded 34GiB to GPU in 1.88086s (18.0769GiB/s) | 20260506 15:41:20.002 | INFO | Analyzing chunk frames = [2508800,2509312) | 20260506 15:41:20.002 | INFO | MemInfo: { | 20260506 15:41:20.002 | INFO | "gpu" : " 17 GB", | 20260506 15:41:20.002 | INFO | "huge" : "", | 20260506 15:41:20.002 | INFO | "huge_shared" : " 27 GB", | 20260506 15:41:20.002 | INFO | "malloc" : " 14 GB", | 20260506 15:41:20.002 | INFO | "pinned" : " 29 GB" | 20260506 15:41:20.002 | INFO | } | 20260506 15:41:20.002 | INFO | Perf report for the last 5.13964 seconds: | 20260506 15:41:20.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:41:20.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:20.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:20.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:20.002 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 15:41:20.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:20.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:20.002 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:41:20.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:20.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:20.002 | INFO | Uploaded 36GiB to GPU in 2.00411s (17.9631GiB/s) | 20260506 15:41:20.002 | INFO | Downloaded 34GiB to GPU in 1.92586s (17.6544GiB/s) | 20260506 15:41:25.141 | INFO | Analyzing chunk frames = [2509312,2509824) | 20260506 15:41:25.141 | INFO | MemInfo: { | 20260506 15:41:25.141 | INFO | "gpu" : " 17 GB", | 20260506 15:41:25.141 | INFO | "huge" : "", | 20260506 15:41:25.141 | INFO | "huge_shared" : " 27 GB", | 20260506 15:41:25.141 | INFO | "malloc" : " 14 GB", | 20260506 15:41:25.141 | INFO | "pinned" : " 29 GB" | 20260506 15:41:25.141 | INFO | } | 20260506 15:41:25.141 | INFO | Perf report for the last 5.13918 seconds: | 20260506 15:41:25.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:41:25.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:25.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:25.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:25.141 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 15:41:25.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:25.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:25.141 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260506 15:41:25.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:25.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:25.141 | INFO | Uploaded 36GiB to GPU in 2.04906s (17.5691GiB/s) | 20260506 15:41:25.141 | INFO | Downloaded 34GiB to GPU in 1.92231s (17.687GiB/s) | 20260506 15:41:30.281 | INFO | Analyzing chunk frames = [2509824,2510336) | 20260506 15:41:30.281 | INFO | MemInfo: { | 20260506 15:41:30.281 | INFO | "gpu" : " 17 GB", | 20260506 15:41:30.281 | INFO | "huge" : "", | 20260506 15:41:30.281 | INFO | "huge_shared" : " 27 GB", | 20260506 15:41:30.281 | INFO | "malloc" : " 14 GB", | 20260506 15:41:30.281 | INFO | "pinned" : " 29 GB" | 20260506 15:41:30.281 | INFO | } | 20260506 15:41:30.281 | INFO | Perf report for the last 5.14024 seconds: | 20260506 15:41:30.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:41:30.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:30.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:30.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:30.281 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 15:41:30.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:30.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:30.281 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 15:41:30.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:30.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:30.282 | INFO | Uploaded 36GiB to GPU in 1.99079s (18.0833GiB/s) | 20260506 15:41:30.282 | INFO | Downloaded 34GiB to GPU in 1.98666s (17.1142GiB/s) | 20260506 15:41:35.420 | INFO | Analyzing chunk frames = [2510336,2510848) | 20260506 15:41:35.420 | INFO | MemInfo: { | 20260506 15:41:35.420 | INFO | "gpu" : " 17 GB", | 20260506 15:41:35.420 | INFO | "huge" : "", | 20260506 15:41:35.420 | INFO | "huge_shared" : " 27 GB", | 20260506 15:41:35.420 | INFO | "malloc" : " 14 GB", | 20260506 15:41:35.420 | INFO | "pinned" : " 29 GB" | 20260506 15:41:35.420 | INFO | } | 20260506 15:41:35.421 | INFO | Perf report for the last 5.13931 seconds: | 20260506 15:41:35.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:41:35.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:35.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:35.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:35.421 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:41:35.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:35.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:35.421 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:41:35.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:35.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:35.421 | INFO | Uploaded 36GiB to GPU in 2.01258s (17.8875GiB/s) | 20260506 15:41:35.421 | INFO | Downloaded 34GiB to GPU in 1.97882s (17.182GiB/s) | 20260506 15:41:35.541 | INFO | Starting a round of Baz IO | 20260506 15:41:38.684 | INFO | Populating RTMetrics report | 20260506 15:41:38.684 | INFO | RTMetrics report populated | 20260506 15:41:38.684 | INFO | Opening tmp RTMetrics json file | 20260506 15:41:38.719 | INFO | Serializing RTMetrics to json | 20260506 15:41:38.723 | INFO | Serialization done, swapping tmp file now... | 20260506 15:41:38.727 | INFO | RTMetrics done | 20260506 15:41:38.934 | INFO | Finished a round of Baz IO: Wrote 2.1 GiB in 3.392 seconds | 20260506 15:41:38.934 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.97% / 0.01% / 63.02% | 20260506 15:41:38.934 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.9662% / 2.38419e-05% / 6.35783e-05% | 20260506 15:41:38.935 | INFO | Baz IO overall: | 20260506 15:41:38.935 | INFO | BytesWritten : 1311595346511 | 20260506 15:41:38.935 | INFO | NumEvents : 634730333276 | 20260506 15:41:40.559 | INFO | Analyzing chunk frames = [2510848,2511360) | 20260506 15:41:40.559 | INFO | MemInfo: { | 20260506 15:41:40.559 | INFO | "gpu" : " 17 GB", | 20260506 15:41:40.559 | INFO | "huge" : "", | 20260506 15:41:40.559 | INFO | "huge_shared" : " 27 GB", | 20260506 15:41:40.559 | INFO | "malloc" : " 14 GB", | 20260506 15:41:40.559 | INFO | "pinned" : " 29 GB" | 20260506 15:41:40.559 | INFO | } | 20260506 15:41:40.559 | INFO | Perf report for the last 5.13844 seconds: | 20260506 15:41:40.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:41:40.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:40.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:40.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:40.559 | INFO | ANALYSIS : 62.9%, 2.9, 95.2ms, 0 | 20260506 15:41:40.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:40.559 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260506 15:41:40.559 | INFO | PRE_HQ : 60.8%, 1.9, 59.6ms, 0 | 20260506 15:41:40.559 | INFO | BAZWRITER : 65.6%, 4.8, 162.4ms, 0 | 20260506 15:41:40.559 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260506 15:41:40.559 | INFO | Uploaded 36GiB to GPU in 2.06763s (17.4112GiB/s) | 20260506 15:41:40.559 | INFO | Downloaded 38GiB to GPU in 2.21732s (17.1378GiB/s) | 20260506 15:41:45.698 | INFO | Analyzing chunk frames = [2511360,2511872) | 20260506 15:41:45.698 | INFO | MemInfo: { | 20260506 15:41:45.698 | INFO | "gpu" : " 17 GB", | 20260506 15:41:45.698 | INFO | "huge" : "", | 20260506 15:41:45.698 | INFO | "huge_shared" : " 27 GB", | 20260506 15:41:45.698 | INFO | "malloc" : " 14 GB", | 20260506 15:41:45.698 | INFO | "pinned" : " 29 GB" | 20260506 15:41:45.698 | INFO | } | 20260506 15:41:45.698 | INFO | Perf report for the last 5.13918 seconds: | 20260506 15:41:45.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:41:45.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:45.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:45.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:45.698 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 15:41:45.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:45.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:45.698 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:41:45.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:45.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:45.699 | INFO | Uploaded 36GiB to GPU in 2.003s (17.9731GiB/s) | 20260506 15:41:45.699 | INFO | Downloaded 34GiB to GPU in 1.93615s (17.5606GiB/s) | 20260506 15:41:50.839 | INFO | Analyzing chunk frames = [2511872,2512384) | 20260506 15:41:50.839 | INFO | MemInfo: { | 20260506 15:41:50.839 | INFO | "gpu" : " 17 GB", | 20260506 15:41:50.839 | INFO | "huge" : "", | 20260506 15:41:50.839 | INFO | "huge_shared" : " 27 GB", | 20260506 15:41:50.839 | INFO | "malloc" : " 14 GB", | 20260506 15:41:50.839 | INFO | "pinned" : " 29 GB" | 20260506 15:41:50.839 | INFO | } | 20260506 15:41:50.839 | INFO | Perf report for the last 5.14068 seconds: | 20260506 15:41:50.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:41:50.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:50.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:50.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:50.839 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:41:50.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:50.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:50.839 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:41:50.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:50.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:50.839 | INFO | Uploaded 36GiB to GPU in 1.98518s (18.1344GiB/s) | 20260506 15:41:50.839 | INFO | Downloaded 34GiB to GPU in 1.93418s (17.5785GiB/s) | 20260506 15:41:55.978 | INFO | Analyzing chunk frames = [2512384,2512896) | 20260506 15:41:55.978 | INFO | MemInfo: { | 20260506 15:41:55.978 | INFO | "gpu" : " 17 GB", | 20260506 15:41:55.978 | INFO | "huge" : "", | 20260506 15:41:55.978 | INFO | "huge_shared" : " 27 GB", | 20260506 15:41:55.978 | INFO | "malloc" : " 14 GB", | 20260506 15:41:55.978 | INFO | "pinned" : " 29 GB" | 20260506 15:41:55.978 | INFO | } | 20260506 15:41:55.978 | INFO | Perf report for the last 5.1388 seconds: | 20260506 15:41:55.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:41:55.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:55.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:55.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:55.978 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260506 15:41:55.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:55.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:55.978 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:41:55.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:55.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:41:55.978 | INFO | Uploaded 36GiB to GPU in 1.98504s (18.1357GiB/s) | 20260506 15:41:55.978 | INFO | Downloaded 34GiB to GPU in 1.88597s (18.0278GiB/s) | 20260506 15:42:01.117 | INFO | Analyzing chunk frames = [2512896,2513408) | 20260506 15:42:01.117 | INFO | MemInfo: { | 20260506 15:42:01.117 | INFO | "gpu" : " 17 GB", | 20260506 15:42:01.117 | INFO | "huge" : "", | 20260506 15:42:01.117 | INFO | "huge_shared" : " 27 GB", | 20260506 15:42:01.117 | INFO | "malloc" : " 14 GB", | 20260506 15:42:01.117 | INFO | "pinned" : " 29 GB" | 20260506 15:42:01.117 | INFO | } | 20260506 15:42:01.117 | INFO | Perf report for the last 5.13943 seconds: | 20260506 15:42:01.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:42:01.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:01.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:01.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:01.117 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 15:42:01.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:01.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:01.117 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:42:01.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:01.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:01.118 | INFO | Uploaded 36GiB to GPU in 2.04699s (17.5868GiB/s) | 20260506 15:42:01.118 | INFO | Downloaded 34GiB to GPU in 1.94088s (17.5178GiB/s) | 20260506 15:42:06.256 | INFO | Analyzing chunk frames = [2513408,2513920) | 20260506 15:42:06.256 | INFO | MemInfo: { | 20260506 15:42:06.256 | INFO | "gpu" : " 17 GB", | 20260506 15:42:06.256 | INFO | "huge" : "", | 20260506 15:42:06.256 | INFO | "huge_shared" : " 27 GB", | 20260506 15:42:06.256 | INFO | "malloc" : " 14 GB", | 20260506 15:42:06.256 | INFO | "pinned" : " 29 GB" | 20260506 15:42:06.256 | INFO | } | 20260506 15:42:06.256 | INFO | Perf report for the last 5.13891 seconds: | 20260506 15:42:06.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:42:06.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:06.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:06.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:06.256 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 15:42:06.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:06.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:06.256 | INFO | PRE_HQ : 32.2%, 1.0, 16.7ms, 0 | 20260506 15:42:06.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:06.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:06.256 | INFO | Uploaded 36GiB to GPU in 2.04663s (17.5899GiB/s) | 20260506 15:42:06.256 | INFO | Downloaded 34GiB to GPU in 1.91137s (17.7883GiB/s) | 20260506 15:42:11.396 | INFO | Analyzing chunk frames = [2513920,2514432) | 20260506 15:42:11.396 | INFO | MemInfo: { | 20260506 15:42:11.396 | INFO | "gpu" : " 17 GB", | 20260506 15:42:11.396 | INFO | "huge" : "", | 20260506 15:42:11.396 | INFO | "huge_shared" : " 27 GB", | 20260506 15:42:11.396 | INFO | "malloc" : " 14 GB", | 20260506 15:42:11.396 | INFO | "pinned" : " 29 GB" | 20260506 15:42:11.396 | INFO | } | 20260506 15:42:11.396 | INFO | Perf report for the last 5.13996 seconds: | 20260506 15:42:11.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:42:11.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:11.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:11.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:11.396 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:42:11.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:11.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:11.396 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:42:11.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:11.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:11.397 | INFO | Uploaded 36GiB to GPU in 2.00603s (17.9459GiB/s) | 20260506 15:42:11.397 | INFO | Downloaded 34GiB to GPU in 1.95663s (17.3768GiB/s) | 20260506 15:42:16.535 | INFO | Analyzing chunk frames = [2514432,2514944) | 20260506 15:42:16.535 | INFO | MemInfo: { | 20260506 15:42:16.535 | INFO | "gpu" : " 17 GB", | 20260506 15:42:16.535 | INFO | "huge" : "", | 20260506 15:42:16.535 | INFO | "huge_shared" : " 27 GB", | 20260506 15:42:16.535 | INFO | "malloc" : " 14 GB", | 20260506 15:42:16.535 | INFO | "pinned" : " 29 GB" | 20260506 15:42:16.535 | INFO | } | 20260506 15:42:16.535 | INFO | Perf report for the last 5.13874 seconds: | 20260506 15:42:16.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:42:16.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:16.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:16.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:16.535 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 15:42:16.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:16.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:16.535 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 15:42:16.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:16.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:16.535 | INFO | Uploaded 36GiB to GPU in 2.03066s (17.7283GiB/s) | 20260506 15:42:16.535 | INFO | Downloaded 34GiB to GPU in 1.99926s (17.0063GiB/s) | 20260506 15:42:16.535 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:42:16.535 | INFO | Total mem usage: 17.4 GiB | 20260506 15:42:16.535 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:42:16.535 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -9.23%) | 20260506 15:42:16.535 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:42:16.535 | INFO | High Water Breakdown: | 20260506 15:42:16.535 | INFO | 61.6% : 9.82 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:42:16.535 | INFO | 38.3% : 6.1 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:42:16.535 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:42:16.535 | INFO | ------------------------------------------ | 20260506 15:42:16.535 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:42:16.535 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:42:16.535 | INFO | Total mem usage: 26.5 GiB | 20260506 15:42:16.535 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:42:16.535 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:42:16.535 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 15:42:16.535 | INFO | High Water Breakdown: | 20260506 15:42:16.535 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:42:16.536 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:42:16.536 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:42:16.536 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:42:16.536 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:42:16.536 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:42:16.536 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:42:16.536 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:42:16.536 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:42:16.536 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:42:16.536 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:42:16.536 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:42:16.536 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:42:16.536 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:42:16.536 | INFO | ------------------------------------------ | 20260506 15:42:16.536 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:42:16.536 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:42:16.536 | INFO | Total mem usage: 25.6 GiB | 20260506 15:42:16.536 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:42:16.536 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:42:16.536 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:42:16.536 | INFO | High Water Breakdown: | 20260506 15:42:16.536 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:42:16.536 | INFO | ------------------------------------------ | 20260506 15:42:16.536 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:42:16.536 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:42:16.536 | INFO | Total mem usage: 15.7 GiB | 20260506 15:42:16.536 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:42:16.536 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:42:16.536 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 15:42:16.536 | INFO | High Water Breakdown: | 20260506 15:42:16.536 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:42:16.536 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:42:16.536 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:42:16.536 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:42:16.536 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:42:16.536 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:42:16.536 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:42:16.536 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:42:16.536 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:42:16.536 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:42:16.536 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:42:16.536 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:42:16.536 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:42:16.536 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:42:16.536 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:42:16.536 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:42:16.536 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:42:16.536 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:42:16.536 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:42:16.536 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:42:16.536 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:42:16.536 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:42:16.536 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:42:16.536 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:42:16.536 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:42:16.536 | INFO | ------------------------------------------ | 20260506 15:42:16.536 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:42:16.536 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:42:16.536 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:42:16.536 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:42:16.536 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.47e+01, 1.32e+01 / 1.61e+00 | 20260506 15:42:16.536 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.02e-01, 9.88e-03 / 6.66e-03 | 20260506 15:42:16.536 | INFO | PerfReport: 11.61%, Baseline -- 3.65e+00 / 4.38e+00, 5.65e-01 / 3.76e-01 | 20260506 15:42:16.536 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.46e+00, 1.88e-01 / 1.37e-01 | 20260506 15:42:16.536 | INFO | PerfReport: 15.81%, DME -- 3.97e+01 / 4.34e+01, 3.48e+00 / 1.81e+00 | 20260506 15:42:16.536 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.46e+00 / 1.58e+00 | 20260506 15:42:16.536 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.31e+00 / 2.90e+00, 3.96e-01 / 2.42e-01 | 20260506 15:42:16.536 | INFO | PerfReport: 3.54%, Metrics -- 1.11e+00 / 1.45e+00, 2.86e-01 / 1.62e-01 | 20260506 15:42:16.536 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.87e-01 / 4.34e-01, 5.99e-02 / 2.26e-02 | 20260506 15:42:16.536 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:42:16.536 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:42:16.536 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:42:16.536 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 15:42:16.536 | INFO | PerfReport: 21.56%, Upload -- 2.10e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260506 15:42:16.536 | INFO | PerfReport: 21.10%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 15:42:16.536 | INFO | PerfReport: 32.26%, Compute -- 3.14e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 15:42:16.536 | INFO | PerfReport: 1.51%, LockUpload -- 1.47e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260506 15:42:16.536 | INFO | PerfReport: 1.05%, LockDownload -- 1.03e+00 / 0.00e+00, 2.66e+00 / 0.00e+00 | 20260506 15:42:16.536 | INFO | PerfReport: 22.51%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 15:42:16.536 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260506 15:42:16.643 | INFO | Starting a round of Baz IO | 20260506 15:42:19.802 | INFO | Populating RTMetrics report | 20260506 15:42:19.802 | INFO | RTMetrics report populated | 20260506 15:42:19.802 | INFO | Opening tmp RTMetrics json file | 20260506 15:42:19.803 | INFO | Serializing RTMetrics to json | 20260506 15:42:19.804 | INFO | Serialization done, swapping tmp file now... | 20260506 15:42:19.805 | INFO | RTMetrics done | 20260506 15:42:20.028 | INFO | Finished a round of Baz IO: Wrote 2.099 GiB in 3.384 seconds | 20260506 15:42:20.029 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.96% / 0.01% / 63.03% | 20260506 15:42:20.029 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.9622% / 3.17891e-05% / 6.35783e-05% | 20260506 15:42:20.029 | INFO | Baz IO overall: | 20260506 15:42:20.029 | INFO | BytesWritten : 1313848940564 | 20260506 15:42:20.029 | INFO | NumEvents : 635820977243 | 20260506 15:42:21.674 | INFO | Analyzing chunk frames = [2514944,2515456) | 20260506 15:42:21.674 | INFO | MemInfo: { | 20260506 15:42:21.674 | INFO | "gpu" : " 17 GB", | 20260506 15:42:21.674 | INFO | "huge" : "", | 20260506 15:42:21.674 | INFO | "huge_shared" : " 27 GB", | 20260506 15:42:21.674 | INFO | "malloc" : " 14 GB", | 20260506 15:42:21.674 | INFO | "pinned" : " 29 GB" | 20260506 15:42:21.674 | INFO | } | 20260506 15:42:21.674 | INFO | Perf report for the last 5.13888 seconds: | 20260506 15:42:21.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:42:21.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:21.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:21.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:21.674 | INFO | ANALYSIS : 63.0%, 2.9, 95.2ms, 0 | 20260506 15:42:21.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:21.674 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 15:42:21.674 | INFO | PRE_HQ : 60.8%, 1.9, 59.4ms, 0 | 20260506 15:42:21.674 | INFO | BAZWRITER : 65.2%, 4.8, 161.2ms, 0 | 20260506 15:42:21.674 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 15:42:21.674 | INFO | Uploaded 36GiB to GPU in 2.0417s (17.6324GiB/s) | 20260506 15:42:21.674 | INFO | Downloaded 38GiB to GPU in 2.23468s (17.0047GiB/s) | 20260506 15:42:26.813 | INFO | Analyzing chunk frames = [2515456,2515968) | 20260506 15:42:26.813 | INFO | MemInfo: { | 20260506 15:42:26.813 | INFO | "gpu" : " 17 GB", | 20260506 15:42:26.813 | INFO | "huge" : "", | 20260506 15:42:26.813 | INFO | "huge_shared" : " 27 GB", | 20260506 15:42:26.813 | INFO | "malloc" : " 14 GB", | 20260506 15:42:26.813 | INFO | "pinned" : " 29 GB" | 20260506 15:42:26.813 | INFO | } | 20260506 15:42:26.813 | INFO | Perf report for the last 5.13918 seconds: | 20260506 15:42:26.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:42:26.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:26.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:26.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:26.813 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:42:26.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:26.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:26.813 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:42:26.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:26.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:26.813 | INFO | Uploaded 36GiB to GPU in 1.98657s (18.1217GiB/s) | 20260506 15:42:26.813 | INFO | Downloaded 34GiB to GPU in 1.93868s (17.5377GiB/s) | 20260506 15:42:31.952 | INFO | Analyzing chunk frames = [2515968,2516480) | 20260506 15:42:31.952 | INFO | MemInfo: { | 20260506 15:42:31.952 | INFO | "gpu" : " 17 GB", | 20260506 15:42:31.952 | INFO | "huge" : "", | 20260506 15:42:31.952 | INFO | "huge_shared" : " 27 GB", | 20260506 15:42:31.952 | INFO | "malloc" : " 14 GB", | 20260506 15:42:31.952 | INFO | "pinned" : " 29 GB" | 20260506 15:42:31.952 | INFO | } | 20260506 15:42:31.952 | INFO | Perf report for the last 5.13925 seconds: | 20260506 15:42:31.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:42:31.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:31.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:31.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:31.952 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:42:31.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:31.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:31.952 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:42:31.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:31.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:31.952 | INFO | Uploaded 36GiB to GPU in 1.98371s (18.1478GiB/s) | 20260506 15:42:31.953 | INFO | Downloaded 34GiB to GPU in 1.93608s (17.5612GiB/s) | 20260506 15:42:37.092 | INFO | Analyzing chunk frames = [2516480,2516992) | 20260506 15:42:37.092 | INFO | MemInfo: { | 20260506 15:42:37.092 | INFO | "gpu" : " 17 GB", | 20260506 15:42:37.092 | INFO | "huge" : "", | 20260506 15:42:37.092 | INFO | "huge_shared" : " 27 GB", | 20260506 15:42:37.092 | INFO | "malloc" : " 14 GB", | 20260506 15:42:37.092 | INFO | "pinned" : " 29 GB" | 20260506 15:42:37.092 | INFO | } | 20260506 15:42:37.092 | INFO | Perf report for the last 5.14018 seconds: | 20260506 15:42:37.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:42:37.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:37.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:37.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:37.093 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 15:42:37.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:37.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:37.093 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:42:37.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:37.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:37.093 | INFO | Uploaded 36GiB to GPU in 2.00748s (17.9329GiB/s) | 20260506 15:42:37.093 | INFO | Downloaded 34GiB to GPU in 1.91895s (17.7181GiB/s) | 20260506 15:42:42.233 | INFO | Analyzing chunk frames = [2516992,2517504) | 20260506 15:42:42.233 | INFO | MemInfo: { | 20260506 15:42:42.233 | INFO | "gpu" : " 17 GB", | 20260506 15:42:42.233 | INFO | "huge" : "", | 20260506 15:42:42.233 | INFO | "huge_shared" : " 27 GB", | 20260506 15:42:42.233 | INFO | "malloc" : " 14 GB", | 20260506 15:42:42.233 | INFO | "pinned" : " 29 GB" | 20260506 15:42:42.233 | INFO | } | 20260506 15:42:42.234 | INFO | Perf report for the last 5.14106 seconds: | 20260506 15:42:42.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:42:42.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:42.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:42.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:42.234 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 15:42:42.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:42.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:42.234 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:42:42.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:42.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:42.234 | INFO | Uploaded 36GiB to GPU in 1.98748s (18.1134GiB/s) | 20260506 15:42:42.234 | INFO | Downloaded 34GiB to GPU in 1.93447s (17.5759GiB/s) | 20260506 15:42:47.371 | INFO | Analyzing chunk frames = [2517504,2518016) | 20260506 15:42:47.371 | INFO | MemInfo: { | 20260506 15:42:47.371 | INFO | "gpu" : " 17 GB", | 20260506 15:42:47.371 | INFO | "huge" : "", | 20260506 15:42:47.371 | INFO | "huge_shared" : " 27 GB", | 20260506 15:42:47.371 | INFO | "malloc" : " 14 GB", | 20260506 15:42:47.371 | INFO | "pinned" : " 29 GB" | 20260506 15:42:47.371 | INFO | } | 20260506 15:42:47.371 | INFO | Perf report for the last 5.13796 seconds: | 20260506 15:42:47.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:42:47.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:47.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:47.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:47.372 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:42:47.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:47.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:47.372 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 15:42:47.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:47.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:47.372 | INFO | Uploaded 36GiB to GPU in 2.03245s (17.7126GiB/s) | 20260506 15:42:47.372 | INFO | Downloaded 34GiB to GPU in 1.93107s (17.6068GiB/s) | 20260506 15:42:52.510 | INFO | Analyzing chunk frames = [2518016,2518528) | 20260506 15:42:52.510 | INFO | MemInfo: { | 20260506 15:42:52.510 | INFO | "gpu" : " 17 GB", | 20260506 15:42:52.510 | INFO | "huge" : "", | 20260506 15:42:52.510 | INFO | "huge_shared" : " 27 GB", | 20260506 15:42:52.510 | INFO | "malloc" : " 14 GB", | 20260506 15:42:52.510 | INFO | "pinned" : " 29 GB" | 20260506 15:42:52.510 | INFO | } | 20260506 15:42:52.510 | INFO | Perf report for the last 5.13865 seconds: | 20260506 15:42:52.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:42:52.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:52.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:52.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:52.510 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:42:52.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:52.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:52.510 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 15:42:52.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:52.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:52.511 | INFO | Uploaded 36GiB to GPU in 1.99781s (18.0197GiB/s) | 20260506 15:42:52.511 | INFO | Downloaded 34GiB to GPU in 1.95733s (17.3706GiB/s) | 20260506 15:42:57.649 | INFO | Analyzing chunk frames = [2518528,2519040) | 20260506 15:42:57.649 | INFO | MemInfo: { | 20260506 15:42:57.649 | INFO | "gpu" : " 17 GB", | 20260506 15:42:57.649 | INFO | "huge" : "", | 20260506 15:42:57.649 | INFO | "huge_shared" : " 27 GB", | 20260506 15:42:57.649 | INFO | "malloc" : " 14 GB", | 20260506 15:42:57.649 | INFO | "pinned" : " 29 GB" | 20260506 15:42:57.649 | INFO | } | 20260506 15:42:57.649 | INFO | Perf report for the last 5.13894 seconds: | 20260506 15:42:57.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:42:57.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:57.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:57.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:57.649 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:42:57.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:57.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:57.649 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:42:57.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:57.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:42:57.649 | INFO | Uploaded 36GiB to GPU in 2.02891s (17.7436GiB/s) | 20260506 15:42:57.649 | INFO | Downloaded 34GiB to GPU in 1.98449s (17.1329GiB/s) | 20260506 15:42:57.768 | INFO | Starting a round of Baz IO | 20260506 15:43:00.915 | INFO | Populating RTMetrics report | 20260506 15:43:00.916 | INFO | RTMetrics report populated | 20260506 15:43:00.916 | INFO | Opening tmp RTMetrics json file | 20260506 15:43:00.917 | INFO | Serializing RTMetrics to json | 20260506 15:43:00.918 | INFO | Serialization done, swapping tmp file now... | 20260506 15:43:00.918 | INFO | RTMetrics done | 20260506 15:43:01.033 | INFO | Finished a round of Baz IO: Wrote 2.098 GiB in 3.264 seconds | 20260506 15:43:01.033 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.95% / 0.01% / 63.04% | 20260506 15:43:01.033 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.9534% / 1.98682e-05% / 7.54992e-05% | 20260506 15:43:01.033 | INFO | Baz IO overall: | 20260506 15:43:01.033 | INFO | BytesWritten : 1316101258239 | 20260506 15:43:01.033 | INFO | NumEvents : 636910833719 | 20260506 15:43:02.789 | INFO | Analyzing chunk frames = [2519040,2519552) | 20260506 15:43:02.789 | INFO | MemInfo: { | 20260506 15:43:02.789 | INFO | "gpu" : " 17 GB", | 20260506 15:43:02.789 | INFO | "huge" : "", | 20260506 15:43:02.789 | INFO | "huge_shared" : " 27 GB", | 20260506 15:43:02.789 | INFO | "malloc" : " 14 GB", | 20260506 15:43:02.789 | INFO | "pinned" : " 29 GB" | 20260506 15:43:02.789 | INFO | } | 20260506 15:43:02.789 | INFO | Perf report for the last 5.13989 seconds: | 20260506 15:43:02.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:43:02.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:02.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:02.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:02.789 | INFO | ANALYSIS : 63.0%, 2.9, 95.3ms, 0 | 20260506 15:43:02.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:02.789 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 15:43:02.789 | INFO | PRE_HQ : 60.5%, 1.9, 59.0ms, 0 | 20260506 15:43:02.789 | INFO | BAZWRITER : 63.1%, 4.7, 152.6ms, 0 | 20260506 15:43:02.789 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 15:43:02.789 | INFO | Uploaded 36GiB to GPU in 2.06268s (17.453GiB/s) | 20260506 15:43:02.789 | INFO | Downloaded 38GiB to GPU in 2.23205s (17.0247GiB/s) | 20260506 15:43:07.929 | INFO | Analyzing chunk frames = [2519552,2520064) | 20260506 15:43:07.929 | INFO | MemInfo: { | 20260506 15:43:07.929 | INFO | "gpu" : " 17 GB", | 20260506 15:43:07.929 | INFO | "huge" : "", | 20260506 15:43:07.929 | INFO | "huge_shared" : " 27 GB", | 20260506 15:43:07.929 | INFO | "malloc" : " 14 GB", | 20260506 15:43:07.929 | INFO | "pinned" : " 29 GB" | 20260506 15:43:07.929 | INFO | } | 20260506 15:43:07.929 | INFO | Perf report for the last 5.14021 seconds: | 20260506 15:43:07.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:43:07.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:07.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:07.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:07.929 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 15:43:07.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:07.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:07.929 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:43:07.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:07.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:07.930 | INFO | Uploaded 36GiB to GPU in 1.97184s (18.2571GiB/s) | 20260506 15:43:07.930 | INFO | Downloaded 34GiB to GPU in 1.93668s (17.5558GiB/s) | 20260506 15:43:13.067 | INFO | Analyzing chunk frames = [2520064,2520576) | 20260506 15:43:13.067 | INFO | MemInfo: { | 20260506 15:43:13.067 | INFO | "gpu" : " 17 GB", | 20260506 15:43:13.067 | INFO | "huge" : "", | 20260506 15:43:13.067 | INFO | "huge_shared" : " 27 GB", | 20260506 15:43:13.067 | INFO | "malloc" : " 14 GB", | 20260506 15:43:13.068 | INFO | "pinned" : " 29 GB" | 20260506 15:43:13.068 | INFO | } | 20260506 15:43:13.068 | INFO | Perf report for the last 5.13853 seconds: | 20260506 15:43:13.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:43:13.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:13.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:13.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:13.068 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 15:43:13.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:13.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:13.068 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:43:13.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:13.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:13.068 | INFO | Uploaded 36GiB to GPU in 1.96239s (18.345GiB/s) | 20260506 15:43:13.068 | INFO | Downloaded 34GiB to GPU in 1.93534s (17.568GiB/s) | 20260506 15:43:18.207 | INFO | Analyzing chunk frames = [2520576,2521088) | 20260506 15:43:18.207 | INFO | MemInfo: { | 20260506 15:43:18.207 | INFO | "gpu" : " 17 GB", | 20260506 15:43:18.207 | INFO | "huge" : "", | 20260506 15:43:18.207 | INFO | "huge_shared" : " 27 GB", | 20260506 15:43:18.207 | INFO | "malloc" : " 14 GB", | 20260506 15:43:18.207 | INFO | "pinned" : " 29 GB" | 20260506 15:43:18.207 | INFO | } | 20260506 15:43:18.207 | INFO | Perf report for the last 5.13886 seconds: | 20260506 15:43:18.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:43:18.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:18.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:18.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:18.207 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260506 15:43:18.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:18.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:18.207 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:43:18.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:18.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:18.207 | INFO | Uploaded 36GiB to GPU in 2.00683s (17.9387GiB/s) | 20260506 15:43:18.207 | INFO | Downloaded 34GiB to GPU in 1.91639s (17.7417GiB/s) | 20260506 15:43:23.346 | INFO | Analyzing chunk frames = [2521088,2521600) | 20260506 15:43:23.346 | INFO | MemInfo: { | 20260506 15:43:23.346 | INFO | "gpu" : " 17 GB", | 20260506 15:43:23.346 | INFO | "huge" : "", | 20260506 15:43:23.346 | INFO | "huge_shared" : " 27 GB", | 20260506 15:43:23.346 | INFO | "malloc" : " 14 GB", | 20260506 15:43:23.346 | INFO | "pinned" : " 29 GB" | 20260506 15:43:23.346 | INFO | } | 20260506 15:43:23.346 | INFO | Perf report for the last 5.13957 seconds: | 20260506 15:43:23.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:43:23.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:23.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:23.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:23.346 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 15:43:23.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:23.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:23.346 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:43:23.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:23.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:23.346 | INFO | Uploaded 36GiB to GPU in 1.96171s (18.3513GiB/s) | 20260506 15:43:23.346 | INFO | Downloaded 34GiB to GPU in 1.92758s (17.6387GiB/s) | 20260506 15:43:28.487 | INFO | Analyzing chunk frames = [2521600,2522112) | 20260506 15:43:28.487 | INFO | MemInfo: { | 20260506 15:43:28.487 | INFO | "gpu" : " 17 GB", | 20260506 15:43:28.487 | INFO | "huge" : "", | 20260506 15:43:28.487 | INFO | "huge_shared" : " 27 GB", | 20260506 15:43:28.487 | INFO | "malloc" : " 14 GB", | 20260506 15:43:28.487 | INFO | "pinned" : " 29 GB" | 20260506 15:43:28.487 | INFO | } | 20260506 15:43:28.487 | INFO | Perf report for the last 5.14074 seconds: | 20260506 15:43:28.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:43:28.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:28.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:28.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:28.487 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 15:43:28.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:28.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:28.487 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 15:43:28.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:28.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:28.487 | INFO | Uploaded 36GiB to GPU in 2.06548s (17.4294GiB/s) | 20260506 15:43:28.487 | INFO | Downloaded 34GiB to GPU in 1.97277s (17.2346GiB/s) | 20260506 15:43:33.625 | INFO | Analyzing chunk frames = [2522112,2522624) | 20260506 15:43:33.625 | INFO | MemInfo: { | 20260506 15:43:33.625 | INFO | "gpu" : " 17 GB", | 20260506 15:43:33.625 | INFO | "huge" : "", | 20260506 15:43:33.625 | INFO | "huge_shared" : " 27 GB", | 20260506 15:43:33.625 | INFO | "malloc" : " 14 GB", | 20260506 15:43:33.625 | INFO | "pinned" : " 29 GB" | 20260506 15:43:33.625 | INFO | } | 20260506 15:43:33.625 | INFO | Perf report for the last 5.13835 seconds: | 20260506 15:43:33.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:43:33.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:33.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:33.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:33.625 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 15:43:33.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:33.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:33.626 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 15:43:33.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:33.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:33.626 | INFO | Uploaded 36GiB to GPU in 2.02586s (17.7702GiB/s) | 20260506 15:43:33.626 | INFO | Downloaded 34GiB to GPU in 2.00969s (16.918GiB/s) | 20260506 15:43:38.764 | INFO | Analyzing chunk frames = [2522624,2523136) | 20260506 15:43:38.764 | INFO | MemInfo: { | 20260506 15:43:38.764 | INFO | "gpu" : " 17 GB", | 20260506 15:43:38.764 | INFO | "huge" : "", | 20260506 15:43:38.764 | INFO | "huge_shared" : " 27 GB", | 20260506 15:43:38.764 | INFO | "malloc" : " 14 GB", | 20260506 15:43:38.764 | INFO | "pinned" : " 29 GB" | 20260506 15:43:38.764 | INFO | } | 20260506 15:43:38.764 | INFO | Perf report for the last 5.13882 seconds: | 20260506 15:43:38.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:43:38.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:38.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:38.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:38.764 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:43:38.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:38.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:38.764 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:43:38.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:38.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:38.764 | INFO | Uploaded 36GiB to GPU in 2.01166s (17.8957GiB/s) | 20260506 15:43:38.764 | INFO | Downloaded 34GiB to GPU in 1.98651s (17.1154GiB/s) | 20260506 15:43:38.765 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:43:38.765 | INFO | Total mem usage: 17.4 GiB | 20260506 15:43:38.765 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:43:38.765 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -12.2%) | 20260506 15:43:38.765 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:43:38.765 | INFO | High Water Breakdown: | 20260506 15:43:38.765 | INFO | 61.2% : 9.5 GiB ( 87% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:43:38.765 | INFO | 38.7% : 6 GiB ( 89.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:43:38.765 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:43:38.765 | INFO | ------------------------------------------ | 20260506 15:43:38.765 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:43:38.765 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:43:38.765 | INFO | Total mem usage: 26.5 GiB | 20260506 15:43:38.765 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:43:38.765 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:43:38.765 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 15:43:38.765 | INFO | High Water Breakdown: | 20260506 15:43:38.765 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:43:38.765 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:43:38.765 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:43:38.765 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:43:38.765 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:43:38.765 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:43:38.765 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:43:38.765 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:43:38.765 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:43:38.765 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:43:38.765 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:43:38.765 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:43:38.765 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:43:38.765 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:43:38.765 | INFO | ------------------------------------------ | 20260506 15:43:38.765 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:43:38.765 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:43:38.765 | INFO | Total mem usage: 25.6 GiB | 20260506 15:43:38.765 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:43:38.765 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:43:38.765 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:43:38.765 | INFO | High Water Breakdown: | 20260506 15:43:38.765 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:43:38.765 | INFO | ------------------------------------------ | 20260506 15:43:38.765 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:43:38.765 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:43:38.765 | INFO | Total mem usage: 15.7 GiB | 20260506 15:43:38.765 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:43:38.765 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:43:38.765 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 15:43:38.765 | INFO | High Water Breakdown: | 20260506 15:43:38.765 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:43:38.765 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:43:38.765 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:43:38.765 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:43:38.765 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:43:38.765 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:43:38.765 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:43:38.765 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:43:38.765 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:43:38.765 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:43:38.765 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:43:38.765 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:43:38.765 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:43:38.765 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:43:38.765 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:43:38.765 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:43:38.765 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:43:38.765 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:43:38.765 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:43:38.765 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:43:38.765 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:43:38.766 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:43:38.766 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:43:38.766 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:43:38.766 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:43:38.766 | INFO | ------------------------------------------ | 20260506 15:43:38.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:43:38.766 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:43:38.766 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:43:38.766 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:43:38.766 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.54e+01, 1.35e+01 / 1.80e+00 | 20260506 15:43:38.766 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.00e-01, 1.01e-02 / 6.08e-03 | 20260506 15:43:38.766 | INFO | PerfReport: 11.57%, Baseline -- 3.64e+00 / 4.39e+00, 5.55e-01 / 3.84e-01 | 20260506 15:43:38.766 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.45e+00, 1.85e-01 / 1.38e-01 | 20260506 15:43:38.766 | INFO | PerfReport: 15.80%, DME -- 3.97e+01 / 4.37e+01, 3.73e+00 / 2.01e+00 | 20260506 15:43:38.766 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.79e+01 / 2.11e+01, 1.52e+00 / 1.96e+00 | 20260506 15:43:38.766 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.32e+00 / 2.87e+00, 3.97e-01 / 2.59e-01 | 20260506 15:43:38.766 | INFO | PerfReport: 3.54%, Metrics -- 1.11e+00 / 1.47e+00, 2.87e-01 / 2.12e-01 | 20260506 15:43:38.766 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.87e-01 / 4.39e-01, 6.09e-02 / 2.71e-02 | 20260506 15:43:38.766 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:43:38.766 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:43:38.766 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:43:38.766 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 15:43:38.766 | INFO | PerfReport: 21.47%, Upload -- 2.09e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 15:43:38.766 | INFO | PerfReport: 21.22%, Download -- 2.07e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 15:43:38.766 | INFO | PerfReport: 32.26%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 15:43:38.766 | INFO | PerfReport: 1.40%, LockUpload -- 1.37e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260506 15:43:38.766 | INFO | PerfReport: 1.19%, LockDownload -- 1.16e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260506 15:43:38.766 | INFO | PerfReport: 22.44%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 15:43:38.766 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260506 15:43:38.884 | INFO | Starting a round of Baz IO | 20260506 15:43:42.023 | INFO | Populating RTMetrics report | 20260506 15:43:42.023 | INFO | RTMetrics report populated | 20260506 15:43:42.023 | INFO | Opening tmp RTMetrics json file | 20260506 15:43:42.086 | INFO | Serializing RTMetrics to json | 20260506 15:43:42.091 | INFO | Serialization done, swapping tmp file now... | 20260506 15:43:42.095 | INFO | RTMetrics done | 20260506 15:43:42.181 | INFO | Finished a round of Baz IO: Wrote 2.096 GiB in 3.296 seconds | 20260506 15:43:42.181 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.94% / 0.01% / 63.05% | 20260506 15:43:42.181 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.94699% / 7.94729e-06% / 4.76837e-05% | 20260506 15:43:42.181 | INFO | Baz IO overall: | 20260506 15:43:42.182 | INFO | BytesWritten : 1318351497525 | 20260506 15:43:42.182 | INFO | NumEvents : 637998973789 | 20260506 15:43:43.904 | INFO | Analyzing chunk frames = [2523136,2523648) | 20260506 15:43:43.904 | INFO | MemInfo: { | 20260506 15:43:43.904 | INFO | "gpu" : " 17 GB", | 20260506 15:43:43.904 | INFO | "huge" : "", | 20260506 15:43:43.904 | INFO | "huge_shared" : " 27 GB", | 20260506 15:43:43.904 | INFO | "malloc" : " 14 GB", | 20260506 15:43:43.904 | INFO | "pinned" : " 29 GB" | 20260506 15:43:43.904 | INFO | } | 20260506 15:43:43.904 | INFO | Perf report for the last 5.13967 seconds: | 20260506 15:43:43.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:43:43.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:43.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:43.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:43.904 | INFO | ANALYSIS : 62.8%, 2.9, 95.1ms, 0 | 20260506 15:43:43.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:43.904 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260506 15:43:43.904 | INFO | PRE_HQ : 60.6%, 1.9, 59.1ms, 0 | 20260506 15:43:43.904 | INFO | BAZWRITER : 63.7%, 4.8, 157.4ms, 0 | 20260506 15:43:43.904 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 15:43:43.904 | INFO | Uploaded 36GiB to GPU in 2.09437s (17.189GiB/s) | 20260506 15:43:43.904 | INFO | Downloaded 38GiB to GPU in 2.22029s (17.1149GiB/s) | 20260506 15:43:49.044 | INFO | Analyzing chunk frames = [2523648,2524160) | 20260506 15:43:49.044 | INFO | MemInfo: { | 20260506 15:43:49.044 | INFO | "gpu" : " 17 GB", | 20260506 15:43:49.044 | INFO | "huge" : "", | 20260506 15:43:49.044 | INFO | "huge_shared" : " 27 GB", | 20260506 15:43:49.044 | INFO | "malloc" : " 14 GB", | 20260506 15:43:49.044 | INFO | "pinned" : " 29 GB" | 20260506 15:43:49.044 | INFO | } | 20260506 15:43:49.044 | INFO | Perf report for the last 5.14007 seconds: | 20260506 15:43:49.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:43:49.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:49.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:49.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:49.044 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:43:49.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:49.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:49.044 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 15:43:49.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:49.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:49.044 | INFO | Uploaded 36GiB to GPU in 2.00944s (17.9154GiB/s) | 20260506 15:43:49.044 | INFO | Downloaded 34GiB to GPU in 1.94971s (17.4385GiB/s) | 20260506 15:43:54.184 | INFO | Analyzing chunk frames = [2524160,2524672) | 20260506 15:43:54.184 | INFO | MemInfo: { | 20260506 15:43:54.184 | INFO | "gpu" : " 17 GB", | 20260506 15:43:54.184 | INFO | "huge" : "", | 20260506 15:43:54.184 | INFO | "huge_shared" : " 27 GB", | 20260506 15:43:54.184 | INFO | "malloc" : " 14 GB", | 20260506 15:43:54.184 | INFO | "pinned" : " 29 GB" | 20260506 15:43:54.184 | INFO | } | 20260506 15:43:54.184 | INFO | Perf report for the last 5.14011 seconds: | 20260506 15:43:54.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:43:54.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:54.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:54.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:54.184 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:43:54.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:54.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:54.184 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:43:54.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:54.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:54.184 | INFO | Uploaded 36GiB to GPU in 1.98697s (18.1181GiB/s) | 20260506 15:43:54.185 | INFO | Downloaded 34GiB to GPU in 1.94179s (17.5096GiB/s) | 20260506 15:43:59.321 | INFO | Analyzing chunk frames = [2524672,2525184) | 20260506 15:43:59.321 | INFO | MemInfo: { | 20260506 15:43:59.321 | INFO | "gpu" : " 17 GB", | 20260506 15:43:59.321 | INFO | "huge" : "", | 20260506 15:43:59.321 | INFO | "huge_shared" : " 27 GB", | 20260506 15:43:59.322 | INFO | "malloc" : " 14 GB", | 20260506 15:43:59.322 | INFO | "pinned" : " 29 GB" | 20260506 15:43:59.322 | INFO | } | 20260506 15:43:59.322 | INFO | Perf report for the last 5.13766 seconds: | 20260506 15:43:59.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:43:59.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:59.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:59.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:59.322 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 15:43:59.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:59.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:59.322 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:43:59.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:59.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:43:59.322 | INFO | Uploaded 36GiB to GPU in 2.02178s (17.8061GiB/s) | 20260506 15:43:59.322 | INFO | Downloaded 34GiB to GPU in 1.90949s (17.8058GiB/s) | 20260506 15:44:04.460 | INFO | Analyzing chunk frames = [2525184,2525696) | 20260506 15:44:04.460 | INFO | MemInfo: { | 20260506 15:44:04.460 | INFO | "gpu" : " 17 GB", | 20260506 15:44:04.460 | INFO | "huge" : "", | 20260506 15:44:04.460 | INFO | "huge_shared" : " 27 GB", | 20260506 15:44:04.460 | INFO | "malloc" : " 14 GB", | 20260506 15:44:04.460 | INFO | "pinned" : " 29 GB" | 20260506 15:44:04.460 | INFO | } | 20260506 15:44:04.460 | INFO | Perf report for the last 5.13868 seconds: | 20260506 15:44:04.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:44:04.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:04.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:04.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:04.461 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 15:44:04.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:04.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:04.461 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:44:04.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:04.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:04.461 | INFO | Uploaded 36GiB to GPU in 2.02495s (17.7782GiB/s) | 20260506 15:44:04.461 | INFO | Downloaded 34GiB to GPU in 1.92535s (17.6592GiB/s) | 20260506 15:44:09.600 | INFO | Analyzing chunk frames = [2525696,2526208) | 20260506 15:44:09.600 | INFO | MemInfo: { | 20260506 15:44:09.600 | INFO | "gpu" : " 17 GB", | 20260506 15:44:09.600 | INFO | "huge" : "", | 20260506 15:44:09.600 | INFO | "huge_shared" : " 27 GB", | 20260506 15:44:09.600 | INFO | "malloc" : " 14 GB", | 20260506 15:44:09.600 | INFO | "pinned" : " 29 GB" | 20260506 15:44:09.600 | INFO | } | 20260506 15:44:09.600 | INFO | Perf report for the last 5.13964 seconds: | 20260506 15:44:09.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:44:09.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:09.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:09.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:09.600 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:44:09.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:09.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:09.600 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260506 15:44:09.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:09.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:09.600 | INFO | Uploaded 36GiB to GPU in 2.07763s (17.3274GiB/s) | 20260506 15:44:09.600 | INFO | Downloaded 34GiB to GPU in 1.96915s (17.2663GiB/s) | 20260506 15:44:14.740 | INFO | Analyzing chunk frames = [2526208,2526720) | 20260506 15:44:14.740 | INFO | MemInfo: { | 20260506 15:44:14.740 | INFO | "gpu" : " 17 GB", | 20260506 15:44:14.740 | INFO | "huge" : "", | 20260506 15:44:14.740 | INFO | "huge_shared" : " 27 GB", | 20260506 15:44:14.740 | INFO | "malloc" : " 14 GB", | 20260506 15:44:14.740 | INFO | "pinned" : " 29 GB" | 20260506 15:44:14.740 | INFO | } | 20260506 15:44:14.740 | INFO | Perf report for the last 5.14045 seconds: | 20260506 15:44:14.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:44:14.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:14.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:14.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:14.741 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:44:14.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:14.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:14.741 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:44:14.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:14.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:14.741 | INFO | Uploaded 36GiB to GPU in 2.02421s (17.7848GiB/s) | 20260506 15:44:14.741 | INFO | Downloaded 34GiB to GPU in 2.00782s (16.9338GiB/s) | 20260506 15:44:19.879 | INFO | Analyzing chunk frames = [2526720,2527232) | 20260506 15:44:19.879 | INFO | MemInfo: { | 20260506 15:44:19.879 | INFO | "gpu" : " 17 GB", | 20260506 15:44:19.879 | INFO | "huge" : "", | 20260506 15:44:19.879 | INFO | "huge_shared" : " 27 GB", | 20260506 15:44:19.879 | INFO | "malloc" : " 14 GB", | 20260506 15:44:19.879 | INFO | "pinned" : " 29 GB" | 20260506 15:44:19.879 | INFO | } | 20260506 15:44:19.879 | INFO | Perf report for the last 5.13877 seconds: | 20260506 15:44:19.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:44:19.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:19.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:19.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:19.880 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:44:19.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:19.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:19.880 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:44:19.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:19.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:19.880 | INFO | Uploaded 36GiB to GPU in 2.01712s (17.8472GiB/s) | 20260506 15:44:19.880 | INFO | Downloaded 34GiB to GPU in 1.98261s (17.1491GiB/s) | 20260506 15:44:19.999 | INFO | Starting a round of Baz IO | 20260506 15:44:23.139 | INFO | Populating RTMetrics report | 20260506 15:44:23.140 | INFO | RTMetrics report populated | 20260506 15:44:23.140 | INFO | Opening tmp RTMetrics json file | 20260506 15:44:23.141 | INFO | Serializing RTMetrics to json | 20260506 15:44:23.144 | INFO | Serialization done, swapping tmp file now... | 20260506 15:44:23.148 | INFO | RTMetrics done | 20260506 15:44:23.292 | INFO | Finished a round of Baz IO: Wrote 2.096 GiB in 3.293 seconds | 20260506 15:44:23.292 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.93% / 0.01% / 63.06% | 20260506 15:44:23.292 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.94115% / 1.58946e-05% / 6.35783e-05% | 20260506 15:44:23.292 | INFO | Baz IO overall: | 20260506 15:44:23.292 | INFO | BytesWritten : 1320601980652 | 20260506 15:44:23.292 | INFO | NumEvents : 639087245988 | 20260506 15:44:25.020 | INFO | Analyzing chunk frames = [2527232,2527744) | 20260506 15:44:25.020 | INFO | MemInfo: { | 20260506 15:44:25.020 | INFO | "gpu" : " 17 GB", | 20260506 15:44:25.020 | INFO | "huge" : "", | 20260506 15:44:25.020 | INFO | "huge_shared" : " 27 GB", | 20260506 15:44:25.020 | INFO | "malloc" : " 14 GB", | 20260506 15:44:25.020 | INFO | "pinned" : " 29 GB" | 20260506 15:44:25.020 | INFO | } | 20260506 15:44:25.020 | INFO | Perf report for the last 5.14032 seconds: | 20260506 15:44:25.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:44:25.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:25.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:25.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:25.020 | INFO | ANALYSIS : 62.8%, 2.9, 95.3ms, 0 | 20260506 15:44:25.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:25.020 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 15:44:25.020 | INFO | PRE_HQ : 60.4%, 1.9, 58.9ms, 0 | 20260506 15:44:25.020 | INFO | BAZWRITER : 63.6%, 4.5, 146.9ms, 0 | 20260506 15:44:25.020 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 15:44:25.020 | INFO | Uploaded 36GiB to GPU in 2.05074s (17.5547GiB/s) | 20260506 15:44:25.020 | INFO | Downloaded 38GiB to GPU in 2.23412s (17.0089GiB/s) | 20260506 15:44:30.157 | INFO | Analyzing chunk frames = [2527744,2528256) | 20260506 15:44:30.157 | INFO | MemInfo: { | 20260506 15:44:30.157 | INFO | "gpu" : " 17 GB", | 20260506 15:44:30.157 | INFO | "huge" : "", | 20260506 15:44:30.157 | INFO | "huge_shared" : " 27 GB", | 20260506 15:44:30.157 | INFO | "malloc" : " 14 GB", | 20260506 15:44:30.157 | INFO | "pinned" : " 29 GB" | 20260506 15:44:30.157 | INFO | } | 20260506 15:44:30.157 | INFO | Perf report for the last 5.13761 seconds: | 20260506 15:44:30.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:44:30.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:30.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:30.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:30.157 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 15:44:30.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:30.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:30.158 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:44:30.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:30.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:30.158 | INFO | Uploaded 36GiB to GPU in 1.98663s (18.1211GiB/s) | 20260506 15:44:30.158 | INFO | Downloaded 34GiB to GPU in 1.93466s (17.5741GiB/s) | 20260506 15:44:35.297 | INFO | Analyzing chunk frames = [2528256,2528768) | 20260506 15:44:35.297 | INFO | MemInfo: { | 20260506 15:44:35.297 | INFO | "gpu" : " 17 GB", | 20260506 15:44:35.297 | INFO | "huge" : "", | 20260506 15:44:35.297 | INFO | "huge_shared" : " 27 GB", | 20260506 15:44:35.297 | INFO | "malloc" : " 14 GB", | 20260506 15:44:35.297 | INFO | "pinned" : " 29 GB" | 20260506 15:44:35.297 | INFO | } | 20260506 15:44:35.297 | INFO | Perf report for the last 5.14013 seconds: | 20260506 15:44:35.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:44:35.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:35.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:35.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:35.298 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 15:44:35.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:35.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:35.298 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:44:35.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:35.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:35.298 | INFO | Uploaded 36GiB to GPU in 1.964s (18.3299GiB/s) | 20260506 15:44:35.298 | INFO | Downloaded 34GiB to GPU in 1.93786s (17.5451GiB/s) | 20260506 15:44:40.437 | INFO | Analyzing chunk frames = [2528768,2529280) | 20260506 15:44:40.437 | INFO | MemInfo: { | 20260506 15:44:40.437 | INFO | "gpu" : " 17 GB", | 20260506 15:44:40.437 | INFO | "huge" : "", | 20260506 15:44:40.437 | INFO | "huge_shared" : " 27 GB", | 20260506 15:44:40.437 | INFO | "malloc" : " 14 GB", | 20260506 15:44:40.437 | INFO | "pinned" : " 29 GB" | 20260506 15:44:40.437 | INFO | } | 20260506 15:44:40.437 | INFO | Perf report for the last 5.14005 seconds: | 20260506 15:44:40.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:44:40.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:40.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:40.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:40.438 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260506 15:44:40.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:40.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:40.438 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:44:40.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:40.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:40.438 | INFO | Uploaded 36GiB to GPU in 2.01671s (17.8509GiB/s) | 20260506 15:44:40.438 | INFO | Downloaded 34GiB to GPU in 1.91345s (17.769GiB/s) | 20260506 15:44:45.576 | INFO | Analyzing chunk frames = [2529280,2529792) | 20260506 15:44:45.576 | INFO | MemInfo: { | 20260506 15:44:45.576 | INFO | "gpu" : " 17 GB", | 20260506 15:44:45.576 | INFO | "huge" : "", | 20260506 15:44:45.576 | INFO | "huge_shared" : " 27 GB", | 20260506 15:44:45.576 | INFO | "malloc" : " 14 GB", | 20260506 15:44:45.576 | INFO | "pinned" : " 29 GB" | 20260506 15:44:45.576 | INFO | } | 20260506 15:44:45.576 | INFO | Perf report for the last 5.13892 seconds: | 20260506 15:44:45.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:44:45.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:45.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:45.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:45.576 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 15:44:45.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:45.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:45.576 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:44:45.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:45.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:45.577 | INFO | Uploaded 36GiB to GPU in 2.00923s (17.9173GiB/s) | 20260506 15:44:45.577 | INFO | Downloaded 34GiB to GPU in 1.94028s (17.5233GiB/s) | 20260506 15:44:50.715 | INFO | Analyzing chunk frames = [2529792,2530304) | 20260506 15:44:50.715 | INFO | MemInfo: { | 20260506 15:44:50.715 | INFO | "gpu" : " 17 GB", | 20260506 15:44:50.715 | INFO | "huge" : "", | 20260506 15:44:50.715 | INFO | "huge_shared" : " 27 GB", | 20260506 15:44:50.715 | INFO | "malloc" : " 14 GB", | 20260506 15:44:50.715 | INFO | "pinned" : " 29 GB" | 20260506 15:44:50.715 | INFO | } | 20260506 15:44:50.715 | INFO | Perf report for the last 5.13867 seconds: | 20260506 15:44:50.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:44:50.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:50.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:50.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:50.715 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 15:44:50.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:50.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:50.715 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 15:44:50.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:50.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:50.715 | INFO | Uploaded 36GiB to GPU in 2.06622s (17.4231GiB/s) | 20260506 15:44:50.715 | INFO | Downloaded 34GiB to GPU in 1.97454s (17.2192GiB/s) | 20260506 15:44:55.855 | INFO | Analyzing chunk frames = [2530304,2530816) | 20260506 15:44:55.855 | INFO | MemInfo: { | 20260506 15:44:55.855 | INFO | "gpu" : " 17 GB", | 20260506 15:44:55.855 | INFO | "huge" : "", | 20260506 15:44:55.855 | INFO | "huge_shared" : " 27 GB", | 20260506 15:44:55.855 | INFO | "malloc" : " 14 GB", | 20260506 15:44:55.855 | INFO | "pinned" : " 29 GB" | 20260506 15:44:55.855 | INFO | } | 20260506 15:44:55.855 | INFO | Perf report for the last 5.13993 seconds: | 20260506 15:44:55.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:44:55.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:55.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:55.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:55.855 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:44:55.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:55.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:55.855 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 15:44:55.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:55.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:44:55.855 | INFO | Uploaded 36GiB to GPU in 2.00357s (17.9679GiB/s) | 20260506 15:44:55.856 | INFO | Downloaded 34GiB to GPU in 1.99305s (17.0593GiB/s) | 20260506 15:45:00.994 | INFO | Analyzing chunk frames = [2530816,2531328) | 20260506 15:45:00.994 | INFO | MemInfo: { | 20260506 15:45:00.994 | INFO | "gpu" : " 17 GB", | 20260506 15:45:00.994 | INFO | "huge" : "", | 20260506 15:45:00.994 | INFO | "huge_shared" : " 27 GB", | 20260506 15:45:00.994 | INFO | "malloc" : " 14 GB", | 20260506 15:45:00.994 | INFO | "pinned" : " 29 GB" | 20260506 15:45:00.994 | INFO | } | 20260506 15:45:00.994 | INFO | Perf report for the last 5.13905 seconds: | 20260506 15:45:00.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:45:00.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:00.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:00.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:00.994 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:45:00.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:00.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:00.994 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:45:00.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:00.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:00.994 | INFO | Uploaded 36GiB to GPU in 2.02721s (17.7584GiB/s) | 20260506 15:45:00.994 | INFO | Downloaded 34GiB to GPU in 1.98721s (17.1094GiB/s) | 20260506 15:45:00.995 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:45:00.995 | INFO | Total mem usage: 17.4 GiB | 20260506 15:45:00.995 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:45:00.995 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -12.3%) | 20260506 15:45:00.995 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:45:00.995 | INFO | High Water Breakdown: | 20260506 15:45:00.995 | INFO | 61.2% : 9.49 GiB ( 86.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:45:00.995 | INFO | 38.7% : 6 GiB ( 89.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:45:00.995 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:45:00.995 | INFO | ------------------------------------------ | 20260506 15:45:00.995 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:45:00.995 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:45:00.995 | INFO | Total mem usage: 26.5 GiB | 20260506 15:45:00.995 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:45:00.995 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:45:00.995 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 15:45:00.995 | INFO | High Water Breakdown: | 20260506 15:45:00.995 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:45:00.995 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:45:00.995 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:45:00.995 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:45:00.995 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:45:00.995 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:45:00.995 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:45:00.995 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:45:00.995 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:45:00.995 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:45:00.995 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:45:00.995 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:45:00.995 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:45:00.995 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:45:00.995 | INFO | ------------------------------------------ | 20260506 15:45:00.995 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:45:00.995 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:45:00.995 | INFO | Total mem usage: 25.6 GiB | 20260506 15:45:00.995 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:45:00.995 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:45:00.995 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:45:00.995 | INFO | High Water Breakdown: | 20260506 15:45:00.995 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:45:00.995 | INFO | ------------------------------------------ | 20260506 15:45:00.995 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:45:00.995 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:45:00.995 | INFO | Total mem usage: 15.7 GiB | 20260506 15:45:00.995 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:45:00.995 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:45:00.995 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 15:45:00.995 | INFO | High Water Breakdown: | 20260506 15:45:00.995 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:45:00.995 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:45:00.995 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:45:00.995 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:45:00.995 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:45:00.995 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:45:00.995 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:45:00.995 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:45:00.995 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:45:00.995 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:45:00.995 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:45:00.995 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:45:00.995 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:45:00.995 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:45:00.996 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:45:00.996 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:45:00.996 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:45:00.996 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:45:00.996 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:45:00.996 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:45:00.996 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:45:00.996 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:45:00.996 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:45:00.996 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:45:00.996 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:45:00.996 | INFO | ------------------------------------------ | 20260506 15:45:00.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:45:00.996 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:45:00.996 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:45:00.996 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:45:00.996 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.55e+01, 1.36e+01 / 2.01e+00 | 20260506 15:45:00.996 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.01e-01, 1.04e-02 / 1.06e-02 | 20260506 15:45:00.996 | INFO | PerfReport: 11.58%, Baseline -- 3.63e+00 / 4.42e+00, 5.54e-01 / 4.06e-01 | 20260506 15:45:00.996 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.47e+00, 1.83e-01 / 1.48e-01 | 20260506 15:45:00.996 | INFO | PerfReport: 15.94%, DME -- 4.00e+01 / 4.43e+01, 3.85e+00 / 2.18e+00 | 20260506 15:45:00.996 | INFO | PerfReport: 56.76%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.37e+00 / 1.79e+00 | 20260506 15:45:00.996 | INFO | PerfReport: 7.40%, PulseAccumulating -- 2.32e+00 / 2.95e+00, 4.07e-01 / 2.23e-01 | 20260506 15:45:00.996 | INFO | PerfReport: 3.54%, Metrics -- 1.11e+00 / 1.45e+00, 2.86e-01 / 1.74e-01 | 20260506 15:45:00.996 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.87e-01 / 4.37e-01, 6.08e-02 / 2.92e-02 | 20260506 15:45:00.996 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:45:00.996 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:45:00.996 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:45:00.996 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 15:45:00.996 | INFO | PerfReport: 21.61%, Upload -- 2.11e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260506 15:45:00.996 | INFO | PerfReport: 21.24%, Download -- 2.07e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 15:45:00.996 | INFO | PerfReport: 32.17%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 15:45:00.996 | INFO | PerfReport: 1.55%, LockUpload -- 1.51e+00 / 0.00e+00, 5.83e+00 / 0.00e+00 | 20260506 15:45:00.996 | INFO | PerfReport: 1.15%, LockDownload -- 1.12e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260506 15:45:00.996 | INFO | PerfReport: 22.26%, LockCompute -- 2.17e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 15:45:00.996 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260506 15:45:01.108 | INFO | Starting a round of Baz IO | 20260506 15:45:04.255 | INFO | Populating RTMetrics report | 20260506 15:45:04.255 | INFO | RTMetrics report populated | 20260506 15:45:04.255 | INFO | Opening tmp RTMetrics json file | 20260506 15:45:04.373 | INFO | Serializing RTMetrics to json | 20260506 15:45:04.427 | INFO | Serialization done, swapping tmp file now... | 20260506 15:45:04.450 | INFO | RTMetrics done | 20260506 15:45:04.475 | INFO | Finished a round of Baz IO: Wrote 2.098 GiB in 3.367 seconds | 20260506 15:45:04.475 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.92% / 0.01% / 63.07% | 20260506 15:45:04.475 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.94028% / 3.97364e-06% / 5.96046e-05% | 20260506 15:45:04.475 | INFO | Baz IO overall: | 20260506 15:45:04.475 | INFO | BytesWritten : 1322854298539 | 20260506 15:45:04.475 | INFO | NumEvents : 640177022524 | 20260506 15:45:06.134 | INFO | Analyzing chunk frames = [2531328,2531840) | 20260506 15:45:06.134 | INFO | MemInfo: { | 20260506 15:45:06.134 | INFO | "gpu" : " 17 GB", | 20260506 15:45:06.134 | INFO | "huge" : "", | 20260506 15:45:06.134 | INFO | "huge_shared" : " 27 GB", | 20260506 15:45:06.134 | INFO | "malloc" : " 14 GB", | 20260506 15:45:06.134 | INFO | "pinned" : " 29 GB" | 20260506 15:45:06.134 | INFO | } | 20260506 15:45:06.134 | INFO | Perf report for the last 5.13993 seconds: | 20260506 15:45:06.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:45:06.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:06.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:06.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:06.134 | INFO | ANALYSIS : 62.8%, 2.9, 95.1ms, 0 | 20260506 15:45:06.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:06.134 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260506 15:45:06.134 | INFO | PRE_HQ : 60.6%, 1.9, 59.2ms, 0 | 20260506 15:45:06.134 | INFO | BAZWRITER : 64.7%, 4.8, 160.9ms, 0 | 20260506 15:45:06.135 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 15:45:06.135 | INFO | Uploaded 36GiB to GPU in 2.05505s (17.5178GiB/s) | 20260506 15:45:06.135 | INFO | Downloaded 38GiB to GPU in 2.23486s (17.0033GiB/s) | 20260506 15:45:11.273 | INFO | Analyzing chunk frames = [2531840,2532352) | 20260506 15:45:11.273 | INFO | MemInfo: { | 20260506 15:45:11.273 | INFO | "gpu" : " 17 GB", | 20260506 15:45:11.273 | INFO | "huge" : "", | 20260506 15:45:11.273 | INFO | "huge_shared" : " 27 GB", | 20260506 15:45:11.273 | INFO | "malloc" : " 14 GB", | 20260506 15:45:11.273 | INFO | "pinned" : " 29 GB" | 20260506 15:45:11.273 | INFO | } | 20260506 15:45:11.274 | INFO | Perf report for the last 5.13957 seconds: | 20260506 15:45:11.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:45:11.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:11.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:11.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:11.274 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 15:45:11.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:11.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:11.274 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 15:45:11.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:11.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:11.274 | INFO | Uploaded 36GiB to GPU in 1.9788s (18.1929GiB/s) | 20260506 15:45:11.274 | INFO | Downloaded 34GiB to GPU in 1.93371s (17.5828GiB/s) | 20260506 15:45:16.412 | INFO | Analyzing chunk frames = [2532352,2532864) | 20260506 15:45:16.412 | INFO | MemInfo: { | 20260506 15:45:16.412 | INFO | "gpu" : " 17 GB", | 20260506 15:45:16.412 | INFO | "huge" : "", | 20260506 15:45:16.412 | INFO | "huge_shared" : " 27 GB", | 20260506 15:45:16.412 | INFO | "malloc" : " 14 GB", | 20260506 15:45:16.412 | INFO | "pinned" : " 29 GB" | 20260506 15:45:16.412 | INFO | } | 20260506 15:45:16.412 | INFO | Perf report for the last 5.13825 seconds: | 20260506 15:45:16.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:45:16.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:16.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:16.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:16.412 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 15:45:16.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:16.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:16.412 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:45:16.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:16.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:16.412 | INFO | Uploaded 36GiB to GPU in 1.96525s (18.3183GiB/s) | 20260506 15:45:16.412 | INFO | Downloaded 34GiB to GPU in 1.93117s (17.6059GiB/s) | 20260506 15:45:21.552 | INFO | Analyzing chunk frames = [2532864,2533376) | 20260506 15:45:21.552 | INFO | MemInfo: { | 20260506 15:45:21.552 | INFO | "gpu" : " 17 GB", | 20260506 15:45:21.552 | INFO | "huge" : "", | 20260506 15:45:21.552 | INFO | "huge_shared" : " 27 GB", | 20260506 15:45:21.552 | INFO | "malloc" : " 14 GB", | 20260506 15:45:21.552 | INFO | "pinned" : " 29 GB" | 20260506 15:45:21.552 | INFO | } | 20260506 15:45:21.552 | INFO | Perf report for the last 5.13993 seconds: | 20260506 15:45:21.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:45:21.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:21.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:21.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:21.552 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 15:45:21.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:21.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:21.552 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:45:21.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:21.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:21.552 | INFO | Uploaded 36GiB to GPU in 1.96636s (18.3079GiB/s) | 20260506 15:45:21.552 | INFO | Downloaded 34GiB to GPU in 1.87966s (18.0884GiB/s) | 20260506 15:45:26.690 | INFO | Analyzing chunk frames = [2533376,2533888) | 20260506 15:45:26.690 | INFO | MemInfo: { | 20260506 15:45:26.690 | INFO | "gpu" : " 17 GB", | 20260506 15:45:26.690 | INFO | "huge" : "", | 20260506 15:45:26.690 | INFO | "huge_shared" : " 27 GB", | 20260506 15:45:26.691 | INFO | "malloc" : " 14 GB", | 20260506 15:45:26.691 | INFO | "pinned" : " 29 GB" | 20260506 15:45:26.691 | INFO | } | 20260506 15:45:26.691 | INFO | Perf report for the last 5.13882 seconds: | 20260506 15:45:26.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:45:26.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:26.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:26.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:26.691 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 15:45:26.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:26.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:26.691 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:45:26.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:26.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:26.691 | INFO | Uploaded 36GiB to GPU in 1.97471s (18.2305GiB/s) | 20260506 15:45:26.691 | INFO | Downloaded 34GiB to GPU in 1.92549s (17.6579GiB/s) | 20260506 15:45:31.829 | INFO | Analyzing chunk frames = [2533888,2534400) | 20260506 15:45:31.829 | INFO | MemInfo: { | 20260506 15:45:31.829 | INFO | "gpu" : " 17 GB", | 20260506 15:45:31.829 | INFO | "huge" : "", | 20260506 15:45:31.829 | INFO | "huge_shared" : " 27 GB", | 20260506 15:45:31.829 | INFO | "malloc" : " 14 GB", | 20260506 15:45:31.829 | INFO | "pinned" : " 29 GB" | 20260506 15:45:31.829 | INFO | } | 20260506 15:45:31.829 | INFO | Perf report for the last 5.13879 seconds: | 20260506 15:45:31.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:45:31.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:31.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:31.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:31.829 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:45:31.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:31.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:31.829 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:45:31.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:31.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:31.830 | INFO | Uploaded 36GiB to GPU in 2.06788s (17.4091GiB/s) | 20260506 15:45:31.830 | INFO | Downloaded 34GiB to GPU in 1.97234s (17.2384GiB/s) | 20260506 15:45:36.969 | INFO | Analyzing chunk frames = [2534400,2534912) | 20260506 15:45:36.969 | INFO | MemInfo: { | 20260506 15:45:36.969 | INFO | "gpu" : " 17 GB", | 20260506 15:45:36.969 | INFO | "huge" : "", | 20260506 15:45:36.969 | INFO | "huge_shared" : " 27 GB", | 20260506 15:45:36.969 | INFO | "malloc" : " 14 GB", | 20260506 15:45:36.969 | INFO | "pinned" : " 29 GB" | 20260506 15:45:36.969 | INFO | } | 20260506 15:45:36.969 | INFO | Perf report for the last 5.14006 seconds: | 20260506 15:45:36.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:45:36.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:36.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:36.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:36.970 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:45:36.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:36.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:36.970 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:45:36.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:36.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:36.970 | INFO | Uploaded 36GiB to GPU in 2.00903s (17.9191GiB/s) | 20260506 15:45:36.970 | INFO | Downloaded 34GiB to GPU in 2.01243s (16.895GiB/s) | 20260506 15:45:42.110 | INFO | Analyzing chunk frames = [2534912,2535424) | 20260506 15:45:42.110 | INFO | MemInfo: { | 20260506 15:45:42.110 | INFO | "gpu" : " 17 GB", | 20260506 15:45:42.110 | INFO | "huge" : "", | 20260506 15:45:42.110 | INFO | "huge_shared" : " 27 GB", | 20260506 15:45:42.110 | INFO | "malloc" : " 14 GB", | 20260506 15:45:42.110 | INFO | "pinned" : " 29 GB" | 20260506 15:45:42.110 | INFO | } | 20260506 15:45:42.110 | INFO | Perf report for the last 5.14067 seconds: | 20260506 15:45:42.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:45:42.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:42.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:42.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:42.110 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:45:42.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:42.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:42.110 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260506 15:45:42.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:42.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:42.110 | INFO | Uploaded 36GiB to GPU in 2.04566s (17.5982GiB/s) | 20260506 15:45:42.110 | INFO | Downloaded 34GiB to GPU in 1.9831s (17.1449GiB/s) | 20260506 15:45:42.223 | INFO | Starting a round of Baz IO | 20260506 15:45:45.383 | INFO | Populating RTMetrics report | 20260506 15:45:45.384 | INFO | RTMetrics report populated | 20260506 15:45:45.384 | INFO | Opening tmp RTMetrics json file | 20260506 15:45:45.422 | INFO | Serializing RTMetrics to json | 20260506 15:45:45.424 | INFO | Serialization done, swapping tmp file now... | 20260506 15:45:45.425 | INFO | RTMetrics done | 20260506 15:45:45.728 | INFO | Finished a round of Baz IO: Wrote 2.099 GiB in 3.505 seconds | 20260506 15:45:45.728 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.91% / 0.01% / 63.08% | 20260506 15:45:45.728 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.93679% / 7.94729e-06% / 6.75519e-05% | 20260506 15:45:45.728 | INFO | Baz IO overall: | 20260506 15:45:45.728 | INFO | BytesWritten : 1325107607647 | 20260506 15:45:45.728 | INFO | NumEvents : 641267729186 | 20260506 15:45:47.247 | INFO | Analyzing chunk frames = [2535424,2535936) | 20260506 15:45:47.247 | INFO | MemInfo: { | 20260506 15:45:47.247 | INFO | "gpu" : " 17 GB", | 20260506 15:45:47.247 | INFO | "huge" : "", | 20260506 15:45:47.247 | INFO | "huge_shared" : " 27 GB", | 20260506 15:45:47.247 | INFO | "malloc" : " 14 GB", | 20260506 15:45:47.247 | INFO | "pinned" : " 29 GB" | 20260506 15:45:47.247 | INFO | } | 20260506 15:45:47.247 | INFO | Perf report for the last 5.13699 seconds: | 20260506 15:45:47.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:45:47.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:47.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:47.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:47.247 | INFO | ANALYSIS : 63.0%, 2.9, 94.7ms, 0 | 20260506 15:45:47.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:47.247 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260506 15:45:47.247 | INFO | PRE_HQ : 61.6%, 2.0, 62.6ms, 0 | 20260506 15:45:47.247 | INFO | BAZWRITER : 67.5%, 4.8, 165.4ms, 0 | 20260506 15:45:47.247 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 15:45:47.248 | INFO | Uploaded 36GiB to GPU in 2.05869s (17.4869GiB/s) | 20260506 15:45:47.248 | INFO | Downloaded 38GiB to GPU in 2.21885s (17.126GiB/s) | 20260506 15:45:52.388 | INFO | Analyzing chunk frames = [2535936,2536448) | 20260506 15:45:52.388 | INFO | MemInfo: { | 20260506 15:45:52.388 | INFO | "gpu" : " 17 GB", | 20260506 15:45:52.388 | INFO | "huge" : "", | 20260506 15:45:52.388 | INFO | "huge_shared" : " 27 GB", | 20260506 15:45:52.388 | INFO | "malloc" : " 14 GB", | 20260506 15:45:52.388 | INFO | "pinned" : " 29 GB" | 20260506 15:45:52.388 | INFO | } | 20260506 15:45:52.388 | INFO | Perf report for the last 5.14067 seconds: | 20260506 15:45:52.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:45:52.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:52.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:52.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:52.388 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 15:45:52.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:52.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:52.388 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:45:52.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:52.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:52.388 | INFO | Uploaded 36GiB to GPU in 2.01173s (17.8951GiB/s) | 20260506 15:45:52.388 | INFO | Downloaded 34GiB to GPU in 1.94732s (17.4599GiB/s) | 20260506 15:45:57.527 | INFO | Analyzing chunk frames = [2536448,2536960) | 20260506 15:45:57.527 | INFO | MemInfo: { | 20260506 15:45:57.527 | INFO | "gpu" : " 17 GB", | 20260506 15:45:57.527 | INFO | "huge" : "", | 20260506 15:45:57.527 | INFO | "huge_shared" : " 27 GB", | 20260506 15:45:57.527 | INFO | "malloc" : " 14 GB", | 20260506 15:45:57.527 | INFO | "pinned" : " 29 GB" | 20260506 15:45:57.527 | INFO | } | 20260506 15:45:57.527 | INFO | Perf report for the last 5.1389 seconds: | 20260506 15:45:57.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:45:57.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:57.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:57.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:57.527 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:45:57.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:57.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:57.527 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:45:57.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:57.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:45:57.527 | INFO | Uploaded 36GiB to GPU in 1.98445s (18.141GiB/s) | 20260506 15:45:57.527 | INFO | Downloaded 34GiB to GPU in 1.93895s (17.5353GiB/s) | 20260506 15:46:02.666 | INFO | Analyzing chunk frames = [2536960,2537472) | 20260506 15:46:02.666 | INFO | MemInfo: { | 20260506 15:46:02.666 | INFO | "gpu" : " 17 GB", | 20260506 15:46:02.666 | INFO | "huge" : "", | 20260506 15:46:02.666 | INFO | "huge_shared" : " 27 GB", | 20260506 15:46:02.666 | INFO | "malloc" : " 14 GB", | 20260506 15:46:02.666 | INFO | "pinned" : " 29 GB" | 20260506 15:46:02.666 | INFO | } | 20260506 15:46:02.666 | INFO | Perf report for the last 5.13908 seconds: | 20260506 15:46:02.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:46:02.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:02.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:02.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:02.666 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 15:46:02.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:02.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:02.666 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:46:02.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:02.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:02.666 | INFO | Uploaded 36GiB to GPU in 2.02073s (17.8153GiB/s) | 20260506 15:46:02.666 | INFO | Downloaded 34GiB to GPU in 1.91437s (17.7604GiB/s) | 20260506 15:46:07.805 | INFO | Analyzing chunk frames = [2537472,2537984) | 20260506 15:46:07.805 | INFO | MemInfo: { | 20260506 15:46:07.805 | INFO | "gpu" : " 17 GB", | 20260506 15:46:07.805 | INFO | "huge" : "", | 20260506 15:46:07.805 | INFO | "huge_shared" : " 27 GB", | 20260506 15:46:07.805 | INFO | "malloc" : " 14 GB", | 20260506 15:46:07.805 | INFO | "pinned" : " 29 GB" | 20260506 15:46:07.805 | INFO | } | 20260506 15:46:07.805 | INFO | Perf report for the last 5.13902 seconds: | 20260506 15:46:07.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:46:07.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:07.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:07.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:07.805 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 15:46:07.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:07.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:07.805 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:46:07.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:07.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:07.805 | INFO | Uploaded 36GiB to GPU in 1.98683s (18.1193GiB/s) | 20260506 15:46:07.805 | INFO | Downloaded 34GiB to GPU in 1.93174s (17.6007GiB/s) | 20260506 15:46:12.944 | INFO | Analyzing chunk frames = [2537984,2538496) | 20260506 15:46:12.944 | INFO | MemInfo: { | 20260506 15:46:12.944 | INFO | "gpu" : " 17 GB", | 20260506 15:46:12.944 | INFO | "huge" : "", | 20260506 15:46:12.944 | INFO | "huge_shared" : " 27 GB", | 20260506 15:46:12.944 | INFO | "malloc" : " 14 GB", | 20260506 15:46:12.944 | INFO | "pinned" : " 29 GB" | 20260506 15:46:12.945 | INFO | } | 20260506 15:46:12.945 | INFO | Perf report for the last 5.13973 seconds: | 20260506 15:46:12.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:46:12.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:12.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:12.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:12.945 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:46:12.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:12.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:12.945 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 15:46:12.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:12.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:12.945 | INFO | Uploaded 36GiB to GPU in 2.0769s (17.3336GiB/s) | 20260506 15:46:12.945 | INFO | Downloaded 34GiB to GPU in 1.97073s (17.2525GiB/s) | 20260506 15:46:18.084 | INFO | Analyzing chunk frames = [2538496,2539008) | 20260506 15:46:18.084 | INFO | MemInfo: { | 20260506 15:46:18.084 | INFO | "gpu" : " 17 GB", | 20260506 15:46:18.084 | INFO | "huge" : "", | 20260506 15:46:18.084 | INFO | "huge_shared" : " 27 GB", | 20260506 15:46:18.084 | INFO | "malloc" : " 14 GB", | 20260506 15:46:18.084 | INFO | "pinned" : " 29 GB" | 20260506 15:46:18.084 | INFO | } | 20260506 15:46:18.084 | INFO | Perf report for the last 5.13933 seconds: | 20260506 15:46:18.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:46:18.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:18.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:18.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:18.084 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:46:18.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:18.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:18.084 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 15:46:18.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:18.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:18.084 | INFO | Uploaded 36GiB to GPU in 1.99149s (18.0769GiB/s) | 20260506 15:46:18.084 | INFO | Downloaded 34GiB to GPU in 1.98112s (17.162GiB/s) | 20260506 15:46:23.224 | INFO | Analyzing chunk frames = [2539008,2539520) | 20260506 15:46:23.224 | INFO | MemInfo: { | 20260506 15:46:23.224 | INFO | "gpu" : " 17 GB", | 20260506 15:46:23.224 | INFO | "huge" : "", | 20260506 15:46:23.224 | INFO | "huge_shared" : " 27 GB", | 20260506 15:46:23.224 | INFO | "malloc" : " 14 GB", | 20260506 15:46:23.224 | INFO | "pinned" : " 29 GB" | 20260506 15:46:23.224 | INFO | } | 20260506 15:46:23.224 | INFO | Perf report for the last 5.13996 seconds: | 20260506 15:46:23.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:46:23.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:23.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:23.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:23.224 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 15:46:23.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:23.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:23.224 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 15:46:23.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:23.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:23.224 | INFO | Uploaded 36GiB to GPU in 1.99538s (18.0417GiB/s) | 20260506 15:46:23.224 | INFO | Downloaded 34GiB to GPU in 1.98385s (17.1384GiB/s) | 20260506 15:46:23.224 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:46:23.224 | INFO | Total mem usage: 17.4 GiB | 20260506 15:46:23.224 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:46:23.224 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -14.6%) | 20260506 15:46:23.224 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:46:23.224 | INFO | High Water Breakdown: | 20260506 15:46:23.225 | INFO | 60.9% : 9.25 GiB ( 84.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:46:23.225 | INFO | 39% : 5.93 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:46:23.225 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:46:23.225 | INFO | ------------------------------------------ | 20260506 15:46:23.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:46:23.225 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:46:23.225 | INFO | Total mem usage: 26.5 GiB | 20260506 15:46:23.225 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:46:23.225 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 15:46:23.225 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:46:23.225 | INFO | High Water Breakdown: | 20260506 15:46:23.225 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:46:23.225 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:46:23.225 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:46:23.225 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:46:23.225 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:46:23.225 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:46:23.225 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:46:23.225 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:46:23.225 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:46:23.225 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:46:23.225 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:46:23.225 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:46:23.225 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:46:23.225 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:46:23.225 | INFO | ------------------------------------------ | 20260506 15:46:23.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:46:23.225 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:46:23.225 | INFO | Total mem usage: 25.6 GiB | 20260506 15:46:23.225 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:46:23.225 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:46:23.225 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:46:23.225 | INFO | High Water Breakdown: | 20260506 15:46:23.225 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:46:23.225 | INFO | ------------------------------------------ | 20260506 15:46:23.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:46:23.225 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:46:23.225 | INFO | Total mem usage: 15.7 GiB | 20260506 15:46:23.225 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:46:23.225 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:46:23.225 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 15:46:23.225 | INFO | High Water Breakdown: | 20260506 15:46:23.225 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:46:23.225 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:46:23.225 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:46:23.225 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:46:23.225 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:46:23.225 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:46:23.225 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:46:23.225 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:46:23.225 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:46:23.225 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:46:23.225 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:46:23.225 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:46:23.225 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:46:23.225 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:46:23.225 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:46:23.225 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:46:23.225 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:46:23.225 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:46:23.225 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:46:23.225 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:46:23.225 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:46:23.225 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:46:23.225 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:46:23.225 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:46:23.225 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:46:23.226 | INFO | ------------------------------------------ | 20260506 15:46:23.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:46:23.226 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:46:23.226 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:46:23.226 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:46:23.226 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.51e+01, 1.35e+01 / 1.99e+00 | 20260506 15:46:23.226 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 1.07e-02 / 8.37e-03 | 20260506 15:46:23.226 | INFO | PerfReport: 11.58%, Baseline -- 3.63e+00 / 4.40e+00, 5.53e-01 / 4.15e-01 | 20260506 15:46:23.226 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.46e+00, 1.84e-01 / 1.52e-01 | 20260506 15:46:23.226 | INFO | PerfReport: 15.90%, DME -- 3.99e+01 / 4.38e+01, 3.70e+00 / 2.26e+00 | 20260506 15:46:23.226 | INFO | PerfReport: 56.80%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.37e+00 / 1.95e+00 | 20260506 15:46:23.226 | INFO | PerfReport: 7.41%, PulseAccumulating -- 2.33e+00 / 2.93e+00, 4.06e-01 / 2.34e-01 | 20260506 15:46:23.226 | INFO | PerfReport: 3.55%, Metrics -- 1.11e+00 / 1.48e+00, 2.89e-01 / 1.70e-01 | 20260506 15:46:23.226 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.87e-01 / 4.44e-01, 6.07e-02 / 2.98e-02 | 20260506 15:46:23.226 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:46:23.226 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:46:23.226 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:46:23.226 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 15:46:23.226 | INFO | PerfReport: 21.52%, Upload -- 2.10e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 15:46:23.226 | INFO | PerfReport: 21.23%, Download -- 2.07e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 15:46:23.226 | INFO | PerfReport: 32.23%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 15:46:23.226 | INFO | PerfReport: 1.49%, LockUpload -- 1.45e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260506 15:46:23.226 | INFO | PerfReport: 1.15%, LockDownload -- 1.12e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 15:46:23.226 | INFO | PerfReport: 22.37%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 15:46:23.226 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.28e-02 / 0.00e+00 | 20260506 15:46:23.342 | INFO | Starting a round of Baz IO | 20260506 15:46:26.487 | INFO | Populating RTMetrics report | 20260506 15:46:26.487 | INFO | RTMetrics report populated | 20260506 15:46:26.487 | INFO | Opening tmp RTMetrics json file | 20260506 15:46:26.616 | INFO | Serializing RTMetrics to json | 20260506 15:46:26.677 | INFO | Serialization done, swapping tmp file now... | 20260506 15:46:26.700 | INFO | RTMetrics done | 20260506 15:46:26.950 | INFO | Finished a round of Baz IO: Wrote 2.098 GiB in 3.607 seconds | 20260506 15:46:26.950 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.90% / 0.01% / 63.09% | 20260506 15:46:26.950 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.93211% / 1.98682e-05% / 5.96046e-05% | 20260506 15:46:26.950 | INFO | Baz IO overall: | 20260506 15:46:26.950 | INFO | BytesWritten : 1327360660538 | 20260506 15:46:26.950 | INFO | NumEvents : 642358299378 | 20260506 15:46:28.364 | INFO | Analyzing chunk frames = [2539520,2540032) | 20260506 15:46:28.364 | INFO | MemInfo: { | 20260506 15:46:28.364 | INFO | "gpu" : " 17 GB", | 20260506 15:46:28.364 | INFO | "huge" : "", | 20260506 15:46:28.364 | INFO | "huge_shared" : " 27 GB", | 20260506 15:46:28.364 | INFO | "malloc" : " 14 GB", | 20260506 15:46:28.364 | INFO | "pinned" : " 29 GB" | 20260506 15:46:28.364 | INFO | } | 20260506 15:46:28.364 | INFO | Perf report for the last 5.14033 seconds: | 20260506 15:46:28.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:46:28.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:28.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:28.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:28.365 | INFO | ANALYSIS : 62.9%, 2.9, 95.6ms, 0 | 20260506 15:46:28.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:28.365 | INFO | RT_METRICS : 47.9%, 1.0, 25.1ms, 0 | 20260506 15:46:28.365 | INFO | PRE_HQ : 60.5%, 1.9, 58.8ms, 0 | 20260506 15:46:28.365 | INFO | BAZWRITER : 69.8%, 4.8, 172.9ms, 0 | 20260506 15:46:28.365 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 15:46:28.365 | INFO | Uploaded 36GiB to GPU in 2.0782s (17.3227GiB/s) | 20260506 15:46:28.365 | INFO | Downloaded 38GiB to GPU in 2.22338s (17.0911GiB/s) | 20260506 15:46:33.502 | INFO | Analyzing chunk frames = [2540032,2540544) | 20260506 15:46:33.502 | INFO | MemInfo: { | 20260506 15:46:33.502 | INFO | "gpu" : " 17 GB", | 20260506 15:46:33.502 | INFO | "huge" : "", | 20260506 15:46:33.502 | INFO | "huge_shared" : " 27 GB", | 20260506 15:46:33.502 | INFO | "malloc" : " 14 GB", | 20260506 15:46:33.502 | INFO | "pinned" : " 29 GB" | 20260506 15:46:33.502 | INFO | } | 20260506 15:46:33.502 | INFO | Perf report for the last 5.13752 seconds: | 20260506 15:46:33.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:46:33.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:33.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:33.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:33.502 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 15:46:33.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:33.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:33.502 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:46:33.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:33.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:33.502 | INFO | Uploaded 36GiB to GPU in 2.01117s (17.9GiB/s) | 20260506 15:46:33.502 | INFO | Downloaded 34GiB to GPU in 1.94342s (17.4949GiB/s) | 20260506 15:46:38.641 | INFO | Analyzing chunk frames = [2540544,2541056) | 20260506 15:46:38.641 | INFO | MemInfo: { | 20260506 15:46:38.641 | INFO | "gpu" : " 17 GB", | 20260506 15:46:38.641 | INFO | "huge" : "", | 20260506 15:46:38.641 | INFO | "huge_shared" : " 27 GB", | 20260506 15:46:38.641 | INFO | "malloc" : " 14 GB", | 20260506 15:46:38.641 | INFO | "pinned" : " 29 GB" | 20260506 15:46:38.641 | INFO | } | 20260506 15:46:38.641 | INFO | Perf report for the last 5.13933 seconds: | 20260506 15:46:38.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:46:38.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:38.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:38.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:38.641 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 15:46:38.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:38.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:38.641 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:46:38.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:38.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:38.642 | INFO | Uploaded 36GiB to GPU in 1.99985s (18.0013GiB/s) | 20260506 15:46:38.642 | INFO | Downloaded 34GiB to GPU in 1.93954s (17.53GiB/s) | 20260506 15:46:43.780 | INFO | Analyzing chunk frames = [2541056,2541568) | 20260506 15:46:43.780 | INFO | MemInfo: { | 20260506 15:46:43.780 | INFO | "gpu" : " 17 GB", | 20260506 15:46:43.780 | INFO | "huge" : "", | 20260506 15:46:43.780 | INFO | "huge_shared" : " 27 GB", | 20260506 15:46:43.780 | INFO | "malloc" : " 14 GB", | 20260506 15:46:43.780 | INFO | "pinned" : " 29 GB" | 20260506 15:46:43.780 | INFO | } | 20260506 15:46:43.780 | INFO | Perf report for the last 5.13906 seconds: | 20260506 15:46:43.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:46:43.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:43.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:43.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:43.780 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 15:46:43.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:43.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:43.780 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:46:43.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:43.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:43.780 | INFO | Uploaded 36GiB to GPU in 2.01901s (17.8305GiB/s) | 20260506 15:46:43.781 | INFO | Downloaded 34GiB to GPU in 1.90856s (17.8145GiB/s) | 20260506 15:46:48.921 | INFO | Analyzing chunk frames = [2541568,2542080) | 20260506 15:46:48.921 | INFO | MemInfo: { | 20260506 15:46:48.921 | INFO | "gpu" : " 17 GB", | 20260506 15:46:48.921 | INFO | "huge" : "", | 20260506 15:46:48.921 | INFO | "huge_shared" : " 27 GB", | 20260506 15:46:48.921 | INFO | "malloc" : " 14 GB", | 20260506 15:46:48.921 | INFO | "pinned" : " 29 GB" | 20260506 15:46:48.921 | INFO | } | 20260506 15:46:48.921 | INFO | Perf report for the last 5.1407 seconds: | 20260506 15:46:48.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:46:48.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:48.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:48.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:48.921 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 15:46:48.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:48.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:48.921 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:46:48.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:48.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:48.921 | INFO | Uploaded 36GiB to GPU in 1.96916s (18.2819GiB/s) | 20260506 15:46:48.921 | INFO | Downloaded 34GiB to GPU in 1.93443s (17.5762GiB/s) | 20260506 15:46:54.059 | INFO | Analyzing chunk frames = [2542080,2542592) | 20260506 15:46:54.059 | INFO | MemInfo: { | 20260506 15:46:54.059 | INFO | "gpu" : " 17 GB", | 20260506 15:46:54.059 | INFO | "huge" : "", | 20260506 15:46:54.059 | INFO | "huge_shared" : " 27 GB", | 20260506 15:46:54.059 | INFO | "malloc" : " 14 GB", | 20260506 15:46:54.059 | INFO | "pinned" : " 29 GB" | 20260506 15:46:54.059 | INFO | } | 20260506 15:46:54.059 | INFO | Perf report for the last 5.13831 seconds: | 20260506 15:46:54.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:46:54.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:54.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:54.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:54.059 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 15:46:54.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:54.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:54.059 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 15:46:54.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:54.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:54.060 | INFO | Uploaded 36GiB to GPU in 2.02008s (17.8211GiB/s) | 20260506 15:46:54.060 | INFO | Downloaded 34GiB to GPU in 1.92954s (17.6208GiB/s) | 20260506 15:46:59.199 | INFO | Analyzing chunk frames = [2542592,2543104) | 20260506 15:46:59.199 | INFO | MemInfo: { | 20260506 15:46:59.199 | INFO | "gpu" : " 17 GB", | 20260506 15:46:59.199 | INFO | "huge" : "", | 20260506 15:46:59.200 | INFO | "huge_shared" : " 27 GB", | 20260506 15:46:59.200 | INFO | "malloc" : " 14 GB", | 20260506 15:46:59.200 | INFO | "pinned" : " 29 GB" | 20260506 15:46:59.200 | INFO | } | 20260506 15:46:59.200 | INFO | Perf report for the last 5.14037 seconds: | 20260506 15:46:59.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:46:59.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:59.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:59.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:59.200 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:46:59.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:59.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:59.200 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:46:59.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:59.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:46:59.200 | INFO | Uploaded 36GiB to GPU in 1.98898s (18.0997GiB/s) | 20260506 15:46:59.200 | INFO | Downloaded 34GiB to GPU in 1.97861s (17.1838GiB/s) | 20260506 15:47:04.337 | INFO | Analyzing chunk frames = [2543104,2543616) | 20260506 15:47:04.337 | INFO | MemInfo: { | 20260506 15:47:04.337 | INFO | "gpu" : " 17 GB", | 20260506 15:47:04.337 | INFO | "huge" : "", | 20260506 15:47:04.337 | INFO | "huge_shared" : " 27 GB", | 20260506 15:47:04.337 | INFO | "malloc" : " 14 GB", | 20260506 15:47:04.337 | INFO | "pinned" : " 29 GB" | 20260506 15:47:04.337 | INFO | } | 20260506 15:47:04.337 | INFO | Perf report for the last 5.13772 seconds: | 20260506 15:47:04.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:47:04.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:04.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:04.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:04.337 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 15:47:04.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:04.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:04.337 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260506 15:47:04.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:04.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:04.338 | INFO | Uploaded 36GiB to GPU in 2.03599s (17.6818GiB/s) | 20260506 15:47:04.338 | INFO | Downloaded 34GiB to GPU in 1.98889s (17.0949GiB/s) | 20260506 15:47:04.452 | INFO | Starting a round of Baz IO | 20260506 15:47:07.600 | INFO | Populating RTMetrics report | 20260506 15:47:07.600 | INFO | RTMetrics report populated | 20260506 15:47:07.600 | INFO | Opening tmp RTMetrics json file | 20260506 15:47:07.623 | INFO | Serializing RTMetrics to json | 20260506 15:47:07.670 | INFO | Serialization done, swapping tmp file now... | 20260506 15:47:07.679 | INFO | RTMetrics done | 20260506 15:47:07.912 | INFO | Finished a round of Baz IO: Wrote 2.097 GiB in 3.46 seconds | 20260506 15:47:07.912 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.89% / 0.01% / 63.10% | 20260506 15:47:07.912 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.9287% / 2.38419e-05% / 6.35783e-05% | 20260506 15:47:07.913 | INFO | Baz IO overall: | 20260506 15:47:07.913 | INFO | BytesWritten : 1329612719788 | 20260506 15:47:07.913 | INFO | NumEvents : 643447852633 | 20260506 15:47:09.477 | INFO | Analyzing chunk frames = [2543616,2544128) | 20260506 15:47:09.477 | INFO | MemInfo: { | 20260506 15:47:09.477 | INFO | "gpu" : " 17 GB", | 20260506 15:47:09.477 | INFO | "huge" : "", | 20260506 15:47:09.477 | INFO | "huge_shared" : " 27 GB", | 20260506 15:47:09.477 | INFO | "malloc" : " 14 GB", | 20260506 15:47:09.477 | INFO | "pinned" : " 29 GB" | 20260506 15:47:09.477 | INFO | } | 20260506 15:47:09.477 | INFO | Perf report for the last 5.13995 seconds: | 20260506 15:47:09.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:47:09.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:09.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:09.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:09.477 | INFO | ANALYSIS : 62.9%, 2.9, 95.3ms, 0 | 20260506 15:47:09.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:09.477 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 15:47:09.477 | INFO | PRE_HQ : 60.8%, 1.9, 59.1ms, 0 | 20260506 15:47:09.477 | INFO | BAZWRITER : 66.7%, 4.8, 165.8ms, 0 | 20260506 15:47:09.477 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 15:47:09.478 | INFO | Uploaded 36GiB to GPU in 2.05438s (17.5236GiB/s) | 20260506 15:47:09.478 | INFO | Downloaded 38GiB to GPU in 2.23261s (17.0205GiB/s) | 20260506 15:47:14.618 | INFO | Analyzing chunk frames = [2544128,2544640) | 20260506 15:47:14.618 | INFO | MemInfo: { | 20260506 15:47:14.618 | INFO | "gpu" : " 17 GB", | 20260506 15:47:14.618 | INFO | "huge" : "", | 20260506 15:47:14.618 | INFO | "huge_shared" : " 27 GB", | 20260506 15:47:14.618 | INFO | "malloc" : " 14 GB", | 20260506 15:47:14.618 | INFO | "pinned" : " 29 GB" | 20260506 15:47:14.618 | INFO | } | 20260506 15:47:14.618 | INFO | Perf report for the last 5.14048 seconds: | 20260506 15:47:14.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:47:14.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:14.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:14.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:14.618 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:47:14.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:14.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:14.618 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:47:14.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:14.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:14.618 | INFO | Uploaded 36GiB to GPU in 1.99598s (18.0363GiB/s) | 20260506 15:47:14.618 | INFO | Downloaded 34GiB to GPU in 1.92708s (17.6433GiB/s) | 20260506 15:47:19.756 | INFO | Analyzing chunk frames = [2544640,2545152) | 20260506 15:47:19.756 | INFO | MemInfo: { | 20260506 15:47:19.756 | INFO | "gpu" : " 17 GB", | 20260506 15:47:19.756 | INFO | "huge" : "", | 20260506 15:47:19.756 | INFO | "huge_shared" : " 27 GB", | 20260506 15:47:19.756 | INFO | "malloc" : " 14 GB", | 20260506 15:47:19.756 | INFO | "pinned" : " 29 GB" | 20260506 15:47:19.756 | INFO | } | 20260506 15:47:19.756 | INFO | Perf report for the last 5.13854 seconds: | 20260506 15:47:19.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:47:19.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:19.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:19.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:19.756 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:47:19.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:19.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:19.756 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:47:19.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:19.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:19.757 | INFO | Uploaded 36GiB to GPU in 1.99946s (18.0049GiB/s) | 20260506 15:47:19.757 | INFO | Downloaded 34GiB to GPU in 1.9398s (17.5276GiB/s) | 20260506 15:47:24.895 | INFO | Analyzing chunk frames = [2545152,2545664) | 20260506 15:47:24.895 | INFO | MemInfo: { | 20260506 15:47:24.895 | INFO | "gpu" : " 17 GB", | 20260506 15:47:24.895 | INFO | "huge" : "", | 20260506 15:47:24.895 | INFO | "huge_shared" : " 27 GB", | 20260506 15:47:24.895 | INFO | "malloc" : " 14 GB", | 20260506 15:47:24.895 | INFO | "pinned" : " 29 GB" | 20260506 15:47:24.895 | INFO | } | 20260506 15:47:24.895 | INFO | Perf report for the last 5.139 seconds: | 20260506 15:47:24.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:47:24.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:24.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:24.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:24.895 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260506 15:47:24.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:24.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:24.896 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:47:24.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:24.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:24.896 | INFO | Uploaded 36GiB to GPU in 2.01589s (17.8581GiB/s) | 20260506 15:47:24.896 | INFO | Downloaded 34GiB to GPU in 1.91165s (17.7857GiB/s) | 20260506 15:47:30.034 | INFO | Analyzing chunk frames = [2545664,2546176) | 20260506 15:47:30.034 | INFO | MemInfo: { | 20260506 15:47:30.034 | INFO | "gpu" : " 17 GB", | 20260506 15:47:30.034 | INFO | "huge" : "", | 20260506 15:47:30.035 | INFO | "huge_shared" : " 27 GB", | 20260506 15:47:30.035 | INFO | "malloc" : " 14 GB", | 20260506 15:47:30.035 | INFO | "pinned" : " 29 GB" | 20260506 15:47:30.035 | INFO | } | 20260506 15:47:30.035 | INFO | Perf report for the last 5.13921 seconds: | 20260506 15:47:30.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:47:30.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:30.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:30.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:30.035 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 15:47:30.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:30.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:30.035 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:47:30.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:30.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:30.035 | INFO | Uploaded 36GiB to GPU in 2.02583s (17.7705GiB/s) | 20260506 15:47:30.035 | INFO | Downloaded 34GiB to GPU in 1.9368s (17.5548GiB/s) | 20260506 15:47:35.174 | INFO | Analyzing chunk frames = [2546176,2546688) | 20260506 15:47:35.174 | INFO | MemInfo: { | 20260506 15:47:35.174 | INFO | "gpu" : " 17 GB", | 20260506 15:47:35.174 | INFO | "huge" : "", | 20260506 15:47:35.174 | INFO | "huge_shared" : " 27 GB", | 20260506 15:47:35.174 | INFO | "malloc" : " 14 GB", | 20260506 15:47:35.174 | INFO | "pinned" : " 29 GB" | 20260506 15:47:35.174 | INFO | } | 20260506 15:47:35.174 | INFO | Perf report for the last 5.1395 seconds: | 20260506 15:47:35.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:47:35.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:35.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:35.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:35.174 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 15:47:35.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:35.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:35.174 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:47:35.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:35.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:35.174 | INFO | Uploaded 36GiB to GPU in 2.07194s (17.375GiB/s) | 20260506 15:47:35.174 | INFO | Downloaded 34GiB to GPU in 1.96362s (17.315GiB/s) | 20260506 15:47:40.314 | INFO | Analyzing chunk frames = [2546688,2547200) | 20260506 15:47:40.314 | INFO | MemInfo: { | 20260506 15:47:40.314 | INFO | "gpu" : " 17 GB", | 20260506 15:47:40.314 | INFO | "huge" : "", | 20260506 15:47:40.314 | INFO | "huge_shared" : " 27 GB", | 20260506 15:47:40.314 | INFO | "malloc" : " 14 GB", | 20260506 15:47:40.314 | INFO | "pinned" : " 29 GB" | 20260506 15:47:40.314 | INFO | } | 20260506 15:47:40.314 | INFO | Perf report for the last 5.14006 seconds: | 20260506 15:47:40.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:47:40.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:40.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:40.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:40.314 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:47:40.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:40.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:40.314 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 15:47:40.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:40.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:40.315 | INFO | Uploaded 36GiB to GPU in 2.00904s (17.919GiB/s) | 20260506 15:47:40.315 | INFO | Downloaded 34GiB to GPU in 2.00458s (16.9612GiB/s) | 20260506 15:47:45.453 | INFO | Analyzing chunk frames = [2547200,2547712) | 20260506 15:47:45.453 | INFO | MemInfo: { | 20260506 15:47:45.453 | INFO | "gpu" : " 17 GB", | 20260506 15:47:45.453 | INFO | "huge" : "", | 20260506 15:47:45.453 | INFO | "huge_shared" : " 27 GB", | 20260506 15:47:45.453 | INFO | "malloc" : " 14 GB", | 20260506 15:47:45.453 | INFO | "pinned" : " 29 GB" | 20260506 15:47:45.453 | INFO | } | 20260506 15:47:45.453 | INFO | Perf report for the last 5.13891 seconds: | 20260506 15:47:45.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:47:45.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:45.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:45.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:45.453 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:47:45.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:45.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:45.453 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 15:47:45.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:45.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:45.453 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:47:45.454 | INFO | Uploaded 36GiB to GPU in 2.02356s (17.7904GiB/s) | 20260506 15:47:45.454 | INFO | Total mem usage: 17.4 GiB | 20260506 15:47:45.454 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:47:45.454 | INFO | Downloaded 34GiB to GPU in 1.98119s (17.1614GiB/s) | 20260506 15:47:45.454 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -13.1%) | 20260506 15:47:45.454 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:47:45.454 | INFO | High Water Breakdown: | 20260506 15:47:45.454 | INFO | 61.1% : 9.4 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:47:45.454 | INFO | 38.8% : 5.98 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:47:45.454 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:47:45.454 | INFO | ------------------------------------------ | 20260506 15:47:45.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:47:45.454 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:47:45.454 | INFO | Total mem usage: 26.5 GiB | 20260506 15:47:45.454 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:47:45.454 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:47:45.454 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 15:47:45.454 | INFO | High Water Breakdown: | 20260506 15:47:45.454 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:47:45.454 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:47:45.454 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:47:45.454 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:47:45.454 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:47:45.454 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:47:45.454 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:47:45.454 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:47:45.454 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:47:45.454 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:47:45.454 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:47:45.454 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:47:45.454 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:47:45.454 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:47:45.454 | INFO | ------------------------------------------ | 20260506 15:47:45.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:47:45.454 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:47:45.454 | INFO | Total mem usage: 25.6 GiB | 20260506 15:47:45.454 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:47:45.454 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:47:45.454 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:47:45.454 | INFO | High Water Breakdown: | 20260506 15:47:45.454 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:47:45.454 | INFO | ------------------------------------------ | 20260506 15:47:45.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:47:45.454 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:47:45.454 | INFO | Total mem usage: 15.7 GiB | 20260506 15:47:45.454 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:47:45.454 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:47:45.454 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 15:47:45.454 | INFO | High Water Breakdown: | 20260506 15:47:45.454 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:47:45.454 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:47:45.454 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:47:45.454 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:47:45.454 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:47:45.454 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:47:45.454 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:47:45.454 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:47:45.454 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:47:45.454 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:47:45.454 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:47:45.454 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:47:45.454 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:47:45.454 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:47:45.454 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:47:45.454 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:47:45.454 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:47:45.454 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:47:45.455 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:47:45.455 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:47:45.455 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:47:45.455 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:47:45.455 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:47:45.455 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:47:45.455 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:47:45.455 | INFO | ------------------------------------------ | 20260506 15:47:45.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:47:45.455 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:47:45.455 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:47:45.455 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:47:45.455 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.55e+01, 1.32e+01 / 1.87e+00 | 20260506 15:47:45.455 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.00e-01, 1.08e-02 / 6.05e-03 | 20260506 15:47:45.455 | INFO | PerfReport: 11.61%, Baseline -- 3.65e+00 / 4.41e+00, 5.71e-01 / 3.93e-01 | 20260506 15:47:45.455 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.46e+00, 1.84e-01 / 1.38e-01 | 20260506 15:47:45.455 | INFO | PerfReport: 15.83%, DME -- 3.98e+01 / 4.39e+01, 3.70e+00 / 1.67e+00 | 20260506 15:47:45.455 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.46e+00 / 1.95e+00 | 20260506 15:47:45.455 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.31e+00 / 2.88e+00, 3.95e-01 / 2.67e-01 | 20260506 15:47:45.455 | INFO | PerfReport: 3.53%, Metrics -- 1.11e+00 / 1.51e+00, 2.86e-01 / 1.95e-01 | 20260506 15:47:45.455 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.86e-01 / 4.46e-01, 6.02e-02 / 2.47e-02 | 20260506 15:47:45.455 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:47:45.455 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:47:45.455 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:47:45.455 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 15:47:45.455 | INFO | PerfReport: 21.57%, Upload -- 2.10e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260506 15:47:45.455 | INFO | PerfReport: 21.19%, Download -- 2.07e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 15:47:45.455 | INFO | PerfReport: 32.21%, Compute -- 3.14e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 15:47:45.455 | INFO | PerfReport: 1.53%, LockUpload -- 1.50e+00 / 0.00e+00, 5.77e+00 / 0.00e+00 | 20260506 15:47:45.455 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260506 15:47:45.455 | INFO | PerfReport: 22.40%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 15:47:45.455 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260506 15:47:45.565 | INFO | Starting a round of Baz IO | 20260506 15:47:48.734 | INFO | Populating RTMetrics report | 20260506 15:47:48.734 | INFO | RTMetrics report populated | 20260506 15:47:48.734 | INFO | Opening tmp RTMetrics json file | 20260506 15:47:48.828 | INFO | Serializing RTMetrics to json | 20260506 15:47:48.896 | INFO | Serialization done, swapping tmp file now... | 20260506 15:47:48.910 | INFO | RTMetrics done | 20260506 15:47:49.062 | INFO | Finished a round of Baz IO: Wrote 2.095 GiB in 3.497 seconds | 20260506 15:47:49.062 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.88% / 0.01% / 63.11% | 20260506 15:47:49.062 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.91617% / 1.98682e-05% / 7.15256e-05% | 20260506 15:47:49.062 | INFO | Baz IO overall: | 20260506 15:47:49.062 | INFO | BytesWritten : 1331862298108 | 20260506 15:47:49.062 | INFO | NumEvents : 644535396083 | 20260506 15:47:50.591 | INFO | Analyzing chunk frames = [2547712,2548224) | 20260506 15:47:50.591 | INFO | MemInfo: { | 20260506 15:47:50.591 | INFO | "gpu" : " 17 GB", | 20260506 15:47:50.591 | INFO | "huge" : "", | 20260506 15:47:50.591 | INFO | "huge_shared" : " 27 GB", | 20260506 15:47:50.591 | INFO | "malloc" : " 14 GB", | 20260506 15:47:50.591 | INFO | "pinned" : " 29 GB" | 20260506 15:47:50.591 | INFO | } | 20260506 15:47:50.591 | INFO | Perf report for the last 5.13834 seconds: | 20260506 15:47:50.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:47:50.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:50.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:50.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:50.592 | INFO | ANALYSIS : 63.2%, 2.9, 95.1ms, 0 | 20260506 15:47:50.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:50.592 | INFO | RT_METRICS : 48.0%, 1.0, 25.2ms, 0 | 20260506 15:47:50.592 | INFO | PRE_HQ : 61.1%, 1.9, 61.0ms, 0 | 20260506 15:47:50.592 | INFO | BAZWRITER : 67.5%, 4.8, 165.7ms, 0 | 20260506 15:47:50.592 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 15:47:50.592 | INFO | Uploaded 36GiB to GPU in 2.0895s (17.229GiB/s) | 20260506 15:47:50.592 | INFO | Downloaded 38GiB to GPU in 2.21195s (17.1794GiB/s) | 20260506 15:47:55.731 | INFO | Analyzing chunk frames = [2548224,2548736) | 20260506 15:47:55.731 | INFO | MemInfo: { | 20260506 15:47:55.731 | INFO | "gpu" : " 17 GB", | 20260506 15:47:55.731 | INFO | "huge" : "", | 20260506 15:47:55.731 | INFO | "huge_shared" : " 27 GB", | 20260506 15:47:55.731 | INFO | "malloc" : " 14 GB", | 20260506 15:47:55.731 | INFO | "pinned" : " 29 GB" | 20260506 15:47:55.731 | INFO | } | 20260506 15:47:55.731 | INFO | Perf report for the last 5.1398 seconds: | 20260506 15:47:55.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:47:55.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:55.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:55.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:55.731 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:47:55.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:55.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:55.731 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 15:47:55.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:55.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:47:55.731 | INFO | Uploaded 36GiB to GPU in 1.99518s (18.0435GiB/s) | 20260506 15:47:55.732 | INFO | Downloaded 34GiB to GPU in 1.94541s (17.477GiB/s) | 20260506 15:48:00.871 | INFO | Analyzing chunk frames = [2548736,2549248) | 20260506 15:48:00.871 | INFO | MemInfo: { | 20260506 15:48:00.872 | INFO | "gpu" : " 17 GB", | 20260506 15:48:00.872 | INFO | "huge" : "", | 20260506 15:48:00.872 | INFO | "huge_shared" : " 27 GB", | 20260506 15:48:00.872 | INFO | "malloc" : " 14 GB", | 20260506 15:48:00.872 | INFO | "pinned" : " 29 GB" | 20260506 15:48:00.872 | INFO | } | 20260506 15:48:00.872 | INFO | Perf report for the last 5.14045 seconds: | 20260506 15:48:00.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:48:00.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:00.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:00.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:00.872 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 15:48:00.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:00.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:00.872 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:48:00.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:00.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:00.872 | INFO | Uploaded 36GiB to GPU in 1.98712s (18.1167GiB/s) | 20260506 15:48:00.872 | INFO | Downloaded 34GiB to GPU in 1.94168s (17.5106GiB/s) | 20260506 15:48:06.010 | INFO | Analyzing chunk frames = [2549248,2549760) | 20260506 15:48:06.010 | INFO | MemInfo: { | 20260506 15:48:06.010 | INFO | "gpu" : " 17 GB", | 20260506 15:48:06.010 | INFO | "huge" : "", | 20260506 15:48:06.010 | INFO | "huge_shared" : " 27 GB", | 20260506 15:48:06.010 | INFO | "malloc" : " 14 GB", | 20260506 15:48:06.010 | INFO | "pinned" : " 29 GB" | 20260506 15:48:06.010 | INFO | } | 20260506 15:48:06.010 | INFO | Perf report for the last 5.13848 seconds: | 20260506 15:48:06.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:48:06.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:06.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:06.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:06.010 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 15:48:06.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:06.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:06.010 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:48:06.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:06.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:06.011 | INFO | Uploaded 36GiB to GPU in 2.01946s (17.8265GiB/s) | 20260506 15:48:06.011 | INFO | Downloaded 34GiB to GPU in 1.90883s (17.8119GiB/s) | 20260506 15:48:11.149 | INFO | Analyzing chunk frames = [2549760,2550272) | 20260506 15:48:11.149 | INFO | MemInfo: { | 20260506 15:48:11.149 | INFO | "gpu" : " 17 GB", | 20260506 15:48:11.149 | INFO | "huge" : "", | 20260506 15:48:11.149 | INFO | "huge_shared" : " 27 GB", | 20260506 15:48:11.149 | INFO | "malloc" : " 14 GB", | 20260506 15:48:11.149 | INFO | "pinned" : " 29 GB" | 20260506 15:48:11.149 | INFO | } | 20260506 15:48:11.149 | INFO | Perf report for the last 5.13879 seconds: | 20260506 15:48:11.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:48:11.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:11.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:11.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:11.149 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 15:48:11.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:11.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:11.149 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:48:11.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:11.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:11.149 | INFO | Uploaded 36GiB to GPU in 2.04287s (17.6223GiB/s) | 20260506 15:48:11.149 | INFO | Downloaded 34GiB to GPU in 1.94156s (17.5117GiB/s) | 20260506 15:48:16.288 | INFO | Analyzing chunk frames = [2550272,2550784) | 20260506 15:48:16.288 | INFO | MemInfo: { | 20260506 15:48:16.288 | INFO | "gpu" : " 17 GB", | 20260506 15:48:16.288 | INFO | "huge" : "", | 20260506 15:48:16.288 | INFO | "huge_shared" : " 27 GB", | 20260506 15:48:16.288 | INFO | "malloc" : " 14 GB", | 20260506 15:48:16.288 | INFO | "pinned" : " 29 GB" | 20260506 15:48:16.288 | INFO | } | 20260506 15:48:16.288 | INFO | Perf report for the last 5.13919 seconds: | 20260506 15:48:16.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:48:16.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:16.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:16.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:16.288 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 15:48:16.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:16.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:16.288 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:48:16.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:16.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:16.289 | INFO | Uploaded 36GiB to GPU in 2.08111s (17.2985GiB/s) | 20260506 15:48:16.289 | INFO | Downloaded 34GiB to GPU in 1.97641s (17.2029GiB/s) | 20260506 15:48:21.428 | INFO | Analyzing chunk frames = [2550784,2551296) | 20260506 15:48:21.428 | INFO | MemInfo: { | 20260506 15:48:21.428 | INFO | "gpu" : " 17 GB", | 20260506 15:48:21.428 | INFO | "huge" : "", | 20260506 15:48:21.428 | INFO | "huge_shared" : " 27 GB", | 20260506 15:48:21.429 | INFO | "malloc" : " 14 GB", | 20260506 15:48:21.429 | INFO | "pinned" : " 29 GB" | 20260506 15:48:21.429 | INFO | } | 20260506 15:48:21.429 | INFO | Perf report for the last 5.14041 seconds: | 20260506 15:48:21.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:48:21.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:21.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:21.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:21.429 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:48:21.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:21.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:21.429 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:48:21.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:21.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:21.429 | INFO | Uploaded 36GiB to GPU in 1.98428s (18.1426GiB/s) | 20260506 15:48:21.429 | INFO | Downloaded 34GiB to GPU in 1.98644s (17.116GiB/s) | 20260506 15:48:26.568 | INFO | Analyzing chunk frames = [2551296,2551808) | 20260506 15:48:26.569 | INFO | MemInfo: { | 20260506 15:48:26.569 | INFO | "gpu" : " 17 GB", | 20260506 15:48:26.569 | INFO | "huge" : "", | 20260506 15:48:26.569 | INFO | "huge_shared" : " 27 GB", | 20260506 15:48:26.569 | INFO | "malloc" : " 14 GB", | 20260506 15:48:26.569 | INFO | "pinned" : " 29 GB" | 20260506 15:48:26.569 | INFO | } | 20260506 15:48:26.569 | INFO | Perf report for the last 5.14008 seconds: | 20260506 15:48:26.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:48:26.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:26.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:26.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:26.569 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:48:26.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:26.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:26.569 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:48:26.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:26.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:26.569 | INFO | Uploaded 36GiB to GPU in 1.97908s (18.1902GiB/s) | 20260506 15:48:26.569 | INFO | Downloaded 34GiB to GPU in 1.96883s (17.2691GiB/s) | 20260506 15:48:26.682 | INFO | Starting a round of Baz IO | 20260506 15:48:29.826 | INFO | Populating RTMetrics report | 20260506 15:48:29.827 | INFO | RTMetrics report populated | 20260506 15:48:29.827 | INFO | Opening tmp RTMetrics json file | 20260506 15:48:29.855 | INFO | Serializing RTMetrics to json | 20260506 15:48:29.906 | INFO | Serialization done, swapping tmp file now... | 20260506 15:48:29.907 | INFO | RTMetrics done | 20260506 15:48:30.007 | INFO | Finished a round of Baz IO: Wrote 2.096 GiB in 3.324 seconds | 20260506 15:48:30.007 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.87% / 0.01% / 63.12% | 20260506 15:48:30.007 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.91659% / 1.98682e-05% / 6.35783e-05% | 20260506 15:48:30.007 | INFO | Baz IO overall: | 20260506 15:48:30.007 | INFO | BytesWritten : 1334113373462 | 20260506 15:48:30.007 | INFO | NumEvents : 645624280382 | 20260506 15:48:31.708 | INFO | Analyzing chunk frames = [2551808,2552320) | 20260506 15:48:31.708 | INFO | MemInfo: { | 20260506 15:48:31.708 | INFO | "gpu" : " 17 GB", | 20260506 15:48:31.708 | INFO | "huge" : "", | 20260506 15:48:31.708 | INFO | "huge_shared" : " 27 GB", | 20260506 15:48:31.708 | INFO | "malloc" : " 14 GB", | 20260506 15:48:31.708 | INFO | "pinned" : " 29 GB" | 20260506 15:48:31.708 | INFO | } | 20260506 15:48:31.708 | INFO | Perf report for the last 5.13942 seconds: | 20260506 15:48:31.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:48:31.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:31.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:31.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:31.708 | INFO | ANALYSIS : 62.8%, 2.9, 95.2ms, 0 | 20260506 15:48:31.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:31.708 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260506 15:48:31.708 | INFO | PRE_HQ : 60.5%, 1.9, 59.2ms, 0 | 20260506 15:48:31.708 | INFO | BAZWRITER : 64.3%, 4.8, 158.8ms, 0 | 20260506 15:48:31.708 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 15:48:31.709 | INFO | Uploaded 36GiB to GPU in 2.08012s (17.3067GiB/s) | 20260506 15:48:31.709 | INFO | Downloaded 38GiB to GPU in 2.22145s (17.106GiB/s) | 20260506 15:48:36.849 | INFO | Analyzing chunk frames = [2552320,2552832) | 20260506 15:48:36.849 | INFO | MemInfo: { | 20260506 15:48:36.849 | INFO | "gpu" : " 17 GB", | 20260506 15:48:36.849 | INFO | "huge" : "", | 20260506 15:48:36.849 | INFO | "huge_shared" : " 27 GB", | 20260506 15:48:36.849 | INFO | "malloc" : " 14 GB", | 20260506 15:48:36.849 | INFO | "pinned" : " 29 GB" | 20260506 15:48:36.849 | INFO | } | 20260506 15:48:36.849 | INFO | Perf report for the last 5.14088 seconds: | 20260506 15:48:36.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:48:36.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:36.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:36.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:36.849 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:48:36.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:36.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:36.849 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 15:48:36.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:36.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:36.849 | INFO | Uploaded 36GiB to GPU in 1.99764s (18.0213GiB/s) | 20260506 15:48:36.849 | INFO | Downloaded 34GiB to GPU in 1.93288s (17.5904GiB/s) | 20260506 15:48:41.985 | INFO | Analyzing chunk frames = [2552832,2553344) | 20260506 15:48:41.985 | INFO | MemInfo: { | 20260506 15:48:41.985 | INFO | "gpu" : " 17 GB", | 20260506 15:48:41.985 | INFO | "huge" : "", | 20260506 15:48:41.985 | INFO | "huge_shared" : " 27 GB", | 20260506 15:48:41.985 | INFO | "malloc" : " 14 GB", | 20260506 15:48:41.985 | INFO | "pinned" : " 29 GB" | 20260506 15:48:41.985 | INFO | } | 20260506 15:48:41.985 | INFO | Perf report for the last 5.13613 seconds: | 20260506 15:48:41.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:48:41.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:41.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:41.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:41.985 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:48:41.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:41.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:41.985 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 15:48:41.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:41.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:41.986 | INFO | Uploaded 36GiB to GPU in 1.98282s (18.156GiB/s) | 20260506 15:48:41.986 | INFO | Downloaded 34GiB to GPU in 1.9317s (17.6011GiB/s) | 20260506 15:48:47.124 | INFO | Analyzing chunk frames = [2553344,2553856) | 20260506 15:48:47.124 | INFO | MemInfo: { | 20260506 15:48:47.124 | INFO | "gpu" : " 17 GB", | 20260506 15:48:47.124 | INFO | "huge" : "", | 20260506 15:48:47.124 | INFO | "huge_shared" : " 27 GB", | 20260506 15:48:47.124 | INFO | "malloc" : " 14 GB", | 20260506 15:48:47.125 | INFO | "pinned" : " 29 GB" | 20260506 15:48:47.125 | INFO | } | 20260506 15:48:47.125 | INFO | Perf report for the last 5.13933 seconds: | 20260506 15:48:47.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:48:47.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:47.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:47.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:47.125 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 15:48:47.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:47.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:47.125 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 15:48:47.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:47.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:47.125 | INFO | Uploaded 36GiB to GPU in 1.97883s (18.1926GiB/s) | 20260506 15:48:47.125 | INFO | Downloaded 34GiB to GPU in 1.88486s (18.0385GiB/s) | 20260506 15:48:52.264 | INFO | Analyzing chunk frames = [2553856,2554368) | 20260506 15:48:52.264 | INFO | MemInfo: { | 20260506 15:48:52.264 | INFO | "gpu" : " 17 GB", | 20260506 15:48:52.264 | INFO | "huge" : "", | 20260506 15:48:52.265 | INFO | "huge_shared" : " 27 GB", | 20260506 15:48:52.265 | INFO | "malloc" : " 14 GB", | 20260506 15:48:52.265 | INFO | "pinned" : " 29 GB" | 20260506 15:48:52.265 | INFO | } | 20260506 15:48:52.265 | INFO | Perf report for the last 5.14018 seconds: | 20260506 15:48:52.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:48:52.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:52.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:52.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:52.265 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 15:48:52.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:52.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:52.265 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:48:52.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:52.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:52.265 | INFO | Uploaded 36GiB to GPU in 2.02382s (17.7881GiB/s) | 20260506 15:48:52.265 | INFO | Downloaded 34GiB to GPU in 1.9374s (17.5493GiB/s) | 20260506 15:48:57.403 | INFO | Analyzing chunk frames = [2554368,2554880) | 20260506 15:48:57.403 | INFO | MemInfo: { | 20260506 15:48:57.403 | INFO | "gpu" : " 17 GB", | 20260506 15:48:57.403 | INFO | "huge" : "", | 20260506 15:48:57.403 | INFO | "huge_shared" : " 27 GB", | 20260506 15:48:57.403 | INFO | "malloc" : " 14 GB", | 20260506 15:48:57.403 | INFO | "pinned" : " 29 GB" | 20260506 15:48:57.403 | INFO | } | 20260506 15:48:57.403 | INFO | Perf report for the last 5.13817 seconds: | 20260506 15:48:57.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:48:57.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:57.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:57.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:57.403 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 15:48:57.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:57.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:57.403 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 15:48:57.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:57.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:48:57.403 | INFO | Uploaded 36GiB to GPU in 2.05688s (17.5023GiB/s) | 20260506 15:48:57.403 | INFO | Downloaded 34GiB to GPU in 1.91843s (17.7228GiB/s) | 20260506 15:49:02.543 | INFO | Analyzing chunk frames = [2554880,2555392) | 20260506 15:49:02.543 | INFO | MemInfo: { | 20260506 15:49:02.543 | INFO | "gpu" : " 17 GB", | 20260506 15:49:02.543 | INFO | "huge" : "", | 20260506 15:49:02.543 | INFO | "huge_shared" : " 27 GB", | 20260506 15:49:02.543 | INFO | "malloc" : " 14 GB", | 20260506 15:49:02.543 | INFO | "pinned" : " 29 GB" | 20260506 15:49:02.543 | INFO | } | 20260506 15:49:02.543 | INFO | Perf report for the last 5.14048 seconds: | 20260506 15:49:02.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:49:02.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:02.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:02.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:02.543 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:49:02.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:02.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:02.544 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:49:02.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:02.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:02.544 | INFO | Uploaded 36GiB to GPU in 1.97685s (18.2108GiB/s) | 20260506 15:49:02.544 | INFO | Downloaded 34GiB to GPU in 1.97644s (17.2027GiB/s) | 20260506 15:49:07.682 | INFO | Analyzing chunk frames = [2555392,2555904) | 20260506 15:49:07.682 | INFO | MemInfo: { | 20260506 15:49:07.682 | INFO | "gpu" : " 17 GB", | 20260506 15:49:07.682 | INFO | "huge" : "", | 20260506 15:49:07.682 | INFO | "huge_shared" : " 27 GB", | 20260506 15:49:07.682 | INFO | "malloc" : " 14 GB", | 20260506 15:49:07.682 | INFO | "pinned" : " 29 GB" | 20260506 15:49:07.682 | INFO | } | 20260506 15:49:07.682 | INFO | Perf report for the last 5.13899 seconds: | 20260506 15:49:07.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:49:07.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:07.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:07.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:07.682 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 15:49:07.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:07.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:07.683 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:49:07.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:07.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:07.683 | INFO | Uploaded 36GiB to GPU in 2.02156s (17.808GiB/s) | 20260506 15:49:07.683 | INFO | Downloaded 34GiB to GPU in 1.99228s (17.0659GiB/s) | 20260506 15:49:07.683 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:49:07.683 | INFO | Total mem usage: 17.4 GiB | 20260506 15:49:07.683 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:49:07.683 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -8.43%) | 20260506 15:49:07.683 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:49:07.683 | INFO | High Water Breakdown: | 20260506 15:49:07.683 | INFO | 61.7% : 9.91 GiB ( 90.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:49:07.683 | INFO | 38.2% : 6.13 GiB ( 91.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:49:07.683 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:49:07.683 | INFO | ------------------------------------------ | 20260506 15:49:07.683 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:49:07.683 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:49:07.683 | INFO | Total mem usage: 26.5 GiB | 20260506 15:49:07.683 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:49:07.683 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 15:49:07.683 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 15:49:07.683 | INFO | High Water Breakdown: | 20260506 15:49:07.683 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:49:07.683 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:49:07.683 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:49:07.683 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:49:07.683 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:49:07.683 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:49:07.683 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:49:07.683 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:49:07.683 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:49:07.683 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:49:07.683 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:49:07.683 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:49:07.683 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:49:07.683 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:49:07.683 | INFO | ------------------------------------------ | 20260506 15:49:07.683 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:49:07.683 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:49:07.683 | INFO | Total mem usage: 25.6 GiB | 20260506 15:49:07.683 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:49:07.683 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:49:07.683 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:49:07.683 | INFO | High Water Breakdown: | 20260506 15:49:07.683 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:49:07.683 | INFO | ------------------------------------------ | 20260506 15:49:07.683 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:49:07.683 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:49:07.683 | INFO | Total mem usage: 15.7 GiB | 20260506 15:49:07.683 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:49:07.683 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:49:07.683 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 15:49:07.683 | INFO | High Water Breakdown: | 20260506 15:49:07.683 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:49:07.683 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:49:07.683 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:49:07.683 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:49:07.683 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:49:07.683 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:49:07.683 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:49:07.683 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:49:07.683 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:49:07.683 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:49:07.683 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:49:07.683 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:49:07.683 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:49:07.683 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:49:07.683 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:49:07.684 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:49:07.684 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:49:07.684 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:49:07.684 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:49:07.684 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:49:07.684 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:49:07.684 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:49:07.684 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:49:07.684 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:49:07.684 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:49:07.684 | INFO | ------------------------------------------ | 20260506 15:49:07.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:49:07.684 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:49:07.684 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:49:07.684 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:49:07.684 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.49e+01, 1.35e+01 / 1.84e+00 | 20260506 15:49:07.684 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.00e-01, 9.93e-03 / 6.99e-03 | 20260506 15:49:07.684 | INFO | PerfReport: 11.61%, Baseline -- 3.65e+00 / 4.42e+00, 5.63e-01 / 3.67e-01 | 20260506 15:49:07.684 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.46e+00, 1.84e-01 / 1.35e-01 | 20260506 15:49:07.684 | INFO | PerfReport: 15.83%, DME -- 3.98e+01 / 4.35e+01, 3.47e+00 / 1.82e+00 | 20260506 15:49:07.684 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.45e+00 / 1.84e+00 | 20260506 15:49:07.684 | INFO | PerfReport: 7.38%, PulseAccumulating -- 2.32e+00 / 2.85e+00, 3.98e-01 / 2.83e-01 | 20260506 15:49:07.684 | INFO | PerfReport: 3.53%, Metrics -- 1.11e+00 / 1.42e+00, 2.82e-01 / 1.67e-01 | 20260506 15:49:07.684 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.85e-01 / 4.29e-01, 5.92e-02 / 2.90e-02 | 20260506 15:49:07.684 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:49:07.684 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:49:07.684 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:49:07.684 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 15:49:07.684 | INFO | PerfReport: 21.59%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 15:49:07.684 | INFO | PerfReport: 21.18%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 15:49:07.684 | INFO | PerfReport: 32.25%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 15:49:07.684 | INFO | PerfReport: 1.48%, LockUpload -- 1.44e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260506 15:49:07.684 | INFO | PerfReport: 1.05%, LockDownload -- 1.03e+00 / 0.00e+00, 2.61e+00 / 0.00e+00 | 20260506 15:49:07.684 | INFO | PerfReport: 22.43%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 15:49:07.684 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260506 15:49:07.795 | INFO | Starting a round of Baz IO | 20260506 15:49:10.947 | INFO | Populating RTMetrics report | 20260506 15:49:10.948 | INFO | RTMetrics report populated | 20260506 15:49:10.948 | INFO | Opening tmp RTMetrics json file | 20260506 15:49:11.098 | INFO | Serializing RTMetrics to json | 20260506 15:49:11.104 | INFO | Serialization done, swapping tmp file now... | 20260506 15:49:11.105 | INFO | RTMetrics done | 20260506 15:49:11.267 | INFO | Finished a round of Baz IO: Wrote 2.096 GiB in 3.472 seconds | 20260506 15:49:11.267 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.86% / 0.01% / 63.13% | 20260506 15:49:11.267 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.911% / 7.94729e-06% / 3.57628e-05% | 20260506 15:49:11.267 | INFO | Baz IO overall: | 20260506 15:49:11.267 | INFO | BytesWritten : 1336363471089 | 20260506 15:49:11.267 | INFO | NumEvents : 646712255330 | 20260506 15:49:12.821 | INFO | Analyzing chunk frames = [2555904,2556416) | 20260506 15:49:12.821 | INFO | MemInfo: { | 20260506 15:49:12.821 | INFO | "gpu" : " 17 GB", | 20260506 15:49:12.821 | INFO | "huge" : "", | 20260506 15:49:12.821 | INFO | "huge_shared" : " 27 GB", | 20260506 15:49:12.821 | INFO | "malloc" : " 14 GB", | 20260506 15:49:12.821 | INFO | "pinned" : " 29 GB" | 20260506 15:49:12.821 | INFO | } | 20260506 15:49:12.821 | INFO | Perf report for the last 5.13863 seconds: | 20260506 15:49:12.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:49:12.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:12.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:12.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:12.821 | INFO | ANALYSIS : 62.9%, 2.9, 95.8ms, 0 | 20260506 15:49:12.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:12.821 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260506 15:49:12.821 | INFO | PRE_HQ : 60.9%, 1.9, 59.2ms, 0 | 20260506 15:49:12.821 | INFO | BAZWRITER : 66.8%, 4.7, 162.6ms, 0 | 20260506 15:49:12.821 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.5ms, 0 | 20260506 15:49:12.821 | INFO | Uploaded 36GiB to GPU in 2.05729s (17.4988GiB/s) | 20260506 15:49:12.821 | INFO | Downloaded 38GiB to GPU in 2.20712s (17.217GiB/s) | 20260506 15:49:17.961 | INFO | Analyzing chunk frames = [2556416,2556928) | 20260506 15:49:17.961 | INFO | MemInfo: { | 20260506 15:49:17.961 | INFO | "gpu" : " 17 GB", | 20260506 15:49:17.961 | INFO | "huge" : "", | 20260506 15:49:17.961 | INFO | "huge_shared" : " 27 GB", | 20260506 15:49:17.961 | INFO | "malloc" : " 14 GB", | 20260506 15:49:17.961 | INFO | "pinned" : " 29 GB" | 20260506 15:49:17.961 | INFO | } | 20260506 15:49:17.961 | INFO | Perf report for the last 5.13977 seconds: | 20260506 15:49:17.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:49:17.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:17.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:17.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:17.961 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:49:17.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:17.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:17.961 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 15:49:17.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:17.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:17.961 | INFO | Uploaded 36GiB to GPU in 1.99291s (18.0641GiB/s) | 20260506 15:49:17.961 | INFO | Downloaded 34GiB to GPU in 1.93123s (17.6053GiB/s) | 20260506 15:49:23.101 | INFO | Analyzing chunk frames = [2556928,2557440) | 20260506 15:49:23.101 | INFO | MemInfo: { | 20260506 15:49:23.101 | INFO | "gpu" : " 17 GB", | 20260506 15:49:23.101 | INFO | "huge" : "", | 20260506 15:49:23.101 | INFO | "huge_shared" : " 27 GB", | 20260506 15:49:23.101 | INFO | "malloc" : " 14 GB", | 20260506 15:49:23.101 | INFO | "pinned" : " 29 GB" | 20260506 15:49:23.101 | INFO | } | 20260506 15:49:23.101 | INFO | Perf report for the last 5.1405 seconds: | 20260506 15:49:23.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:49:23.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:23.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:23.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:23.101 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 15:49:23.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:23.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:23.101 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:49:23.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:23.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:23.102 | INFO | Uploaded 36GiB to GPU in 2.00145s (17.9869GiB/s) | 20260506 15:49:23.102 | INFO | Downloaded 34GiB to GPU in 1.93671s (17.5555GiB/s) | 20260506 15:49:28.240 | INFO | Analyzing chunk frames = [2557440,2557952) | 20260506 15:49:28.240 | INFO | MemInfo: { | 20260506 15:49:28.240 | INFO | "gpu" : " 17 GB", | 20260506 15:49:28.240 | INFO | "huge" : "", | 20260506 15:49:28.240 | INFO | "huge_shared" : " 27 GB", | 20260506 15:49:28.240 | INFO | "malloc" : " 14 GB", | 20260506 15:49:28.240 | INFO | "pinned" : " 29 GB" | 20260506 15:49:28.240 | INFO | } | 20260506 15:49:28.240 | INFO | Perf report for the last 5.13908 seconds: | 20260506 15:49:28.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:49:28.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:28.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:28.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:28.240 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 15:49:28.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:28.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:28.241 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:49:28.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:28.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:28.241 | INFO | Uploaded 36GiB to GPU in 2.0176s (17.8429GiB/s) | 20260506 15:49:28.241 | INFO | Downloaded 34GiB to GPU in 1.91201s (17.7823GiB/s) | 20260506 15:49:33.379 | INFO | Analyzing chunk frames = [2557952,2558464) | 20260506 15:49:33.379 | INFO | MemInfo: { | 20260506 15:49:33.379 | INFO | "gpu" : " 17 GB", | 20260506 15:49:33.379 | INFO | "huge" : "", | 20260506 15:49:33.379 | INFO | "huge_shared" : " 27 GB", | 20260506 15:49:33.379 | INFO | "malloc" : " 14 GB", | 20260506 15:49:33.379 | INFO | "pinned" : " 29 GB" | 20260506 15:49:33.379 | INFO | } | 20260506 15:49:33.379 | INFO | Perf report for the last 5.13834 seconds: | 20260506 15:49:33.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:49:33.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:33.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:33.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:33.379 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 15:49:33.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:33.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:33.379 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:49:33.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:33.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:33.379 | INFO | Uploaded 36GiB to GPU in 2.03473s (17.6928GiB/s) | 20260506 15:49:33.379 | INFO | Downloaded 34GiB to GPU in 1.94384s (17.4911GiB/s) | 20260506 15:49:38.518 | INFO | Analyzing chunk frames = [2558464,2558976) | 20260506 15:49:38.518 | INFO | MemInfo: { | 20260506 15:49:38.518 | INFO | "gpu" : " 17 GB", | 20260506 15:49:38.518 | INFO | "huge" : "", | 20260506 15:49:38.518 | INFO | "huge_shared" : " 27 GB", | 20260506 15:49:38.518 | INFO | "malloc" : " 14 GB", | 20260506 15:49:38.518 | INFO | "pinned" : " 29 GB" | 20260506 15:49:38.518 | INFO | } | 20260506 15:49:38.518 | INFO | Perf report for the last 5.13944 seconds: | 20260506 15:49:38.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:49:38.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:38.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:38.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:38.518 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 15:49:38.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:38.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:38.519 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 15:49:38.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:38.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:38.519 | INFO | Uploaded 36GiB to GPU in 2.04316s (17.6197GiB/s) | 20260506 15:49:38.519 | INFO | Downloaded 34GiB to GPU in 1.90238s (17.8723GiB/s) | 20260506 15:49:43.657 | INFO | Analyzing chunk frames = [2558976,2559488) | 20260506 15:49:43.657 | INFO | MemInfo: { | 20260506 15:49:43.657 | INFO | "gpu" : " 17 GB", | 20260506 15:49:43.657 | INFO | "huge" : "", | 20260506 15:49:43.657 | INFO | "huge_shared" : " 27 GB", | 20260506 15:49:43.657 | INFO | "malloc" : " 14 GB", | 20260506 15:49:43.657 | INFO | "pinned" : " 29 GB" | 20260506 15:49:43.658 | INFO | } | 20260506 15:49:43.658 | INFO | Perf report for the last 5.13928 seconds: | 20260506 15:49:43.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:49:43.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:43.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:43.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:43.658 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:49:43.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:43.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:43.658 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:49:43.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:43.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:43.658 | INFO | Uploaded 36GiB to GPU in 1.99554s (18.0403GiB/s) | 20260506 15:49:43.658 | INFO | Downloaded 34GiB to GPU in 1.99631s (17.0314GiB/s) | 20260506 15:49:48.797 | INFO | Analyzing chunk frames = [2559488,2560000) | 20260506 15:49:48.797 | INFO | MemInfo: { | 20260506 15:49:48.797 | INFO | "gpu" : " 17 GB", | 20260506 15:49:48.797 | INFO | "huge" : "", | 20260506 15:49:48.797 | INFO | "huge_shared" : " 27 GB", | 20260506 15:49:48.797 | INFO | "malloc" : " 14 GB", | 20260506 15:49:48.797 | INFO | "pinned" : " 29 GB" | 20260506 15:49:48.797 | INFO | } | 20260506 15:49:48.797 | INFO | Perf report for the last 5.13937 seconds: | 20260506 15:49:48.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:49:48.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:48.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:48.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:48.797 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 15:49:48.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:48.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:48.797 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:49:48.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:48.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:48.797 | INFO | Uploaded 36GiB to GPU in 2.03602s (17.6815GiB/s) | 20260506 15:49:48.797 | INFO | Downloaded 34GiB to GPU in 1.97458s (17.2188GiB/s) | 20260506 15:49:48.909 | INFO | Starting a round of Baz IO | 20260506 15:49:52.057 | INFO | Populating RTMetrics report | 20260506 15:49:52.058 | INFO | RTMetrics report populated | 20260506 15:49:52.058 | INFO | Opening tmp RTMetrics json file | 20260506 15:49:52.109 | INFO | Serializing RTMetrics to json | 20260506 15:49:52.176 | INFO | Serialization done, swapping tmp file now... | 20260506 15:49:52.181 | INFO | RTMetrics done | 20260506 15:49:52.546 | INFO | Finished a round of Baz IO: Wrote 2.096 GiB in 3.636 seconds | 20260506 15:49:52.546 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.85% / 0.01% / 63.14% | 20260506 15:49:52.546 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.90699% / 7.94729e-06% / 5.16574e-05% | 20260506 15:49:52.546 | INFO | Baz IO overall: | 20260506 15:49:52.546 | INFO | BytesWritten : 1338613850371 | 20260506 15:49:52.546 | INFO | NumEvents : 647800580373 | 20260506 15:49:53.936 | INFO | Analyzing chunk frames = [2560000,2560512) | 20260506 15:49:53.936 | INFO | MemInfo: { | 20260506 15:49:53.936 | INFO | "gpu" : " 17 GB", | 20260506 15:49:53.936 | INFO | "huge" : "", | 20260506 15:49:53.936 | INFO | "huge_shared" : " 27 GB", | 20260506 15:49:53.936 | INFO | "malloc" : " 14 GB", | 20260506 15:49:53.936 | INFO | "pinned" : " 29 GB" | 20260506 15:49:53.936 | INFO | } | 20260506 15:49:53.936 | INFO | Perf report for the last 5.13937 seconds: | 20260506 15:49:53.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:49:53.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:53.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:53.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:53.936 | INFO | ANALYSIS : 62.8%, 2.9, 95.3ms, 0 | 20260506 15:49:53.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:53.936 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 15:49:53.936 | INFO | PRE_HQ : 60.8%, 1.9, 59.1ms, 0 | 20260506 15:49:53.936 | INFO | BAZWRITER : 70.0%, 4.8, 173.1ms, 0 | 20260506 15:49:53.936 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.2ms, 0 | 20260506 15:49:53.937 | INFO | Uploaded 36GiB to GPU in 2.05081s (17.5541GiB/s) | 20260506 15:49:53.937 | INFO | Downloaded 38GiB to GPU in 2.22656s (17.0667GiB/s) | 20260506 15:49:59.075 | INFO | Analyzing chunk frames = [2560512,2561024) | 20260506 15:49:59.075 | INFO | MemInfo: { | 20260506 15:49:59.075 | INFO | "gpu" : " 17 GB", | 20260506 15:49:59.075 | INFO | "huge" : "", | 20260506 15:49:59.075 | INFO | "huge_shared" : " 27 GB", | 20260506 15:49:59.075 | INFO | "malloc" : " 14 GB", | 20260506 15:49:59.075 | INFO | "pinned" : " 29 GB" | 20260506 15:49:59.075 | INFO | } | 20260506 15:49:59.076 | INFO | Perf report for the last 5.13918 seconds: | 20260506 15:49:59.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:49:59.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:59.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:59.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:59.076 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 15:49:59.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:59.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:59.076 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 15:49:59.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:59.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:49:59.076 | INFO | Uploaded 36GiB to GPU in 1.98721s (18.1159GiB/s) | 20260506 15:49:59.076 | INFO | Downloaded 34GiB to GPU in 1.93221s (17.5964GiB/s) | 20260506 15:50:04.214 | INFO | Analyzing chunk frames = [2561024,2561536) | 20260506 15:50:04.215 | INFO | MemInfo: { | 20260506 15:50:04.215 | INFO | "gpu" : " 17 GB", | 20260506 15:50:04.215 | INFO | "huge" : "", | 20260506 15:50:04.215 | INFO | "huge_shared" : " 27 GB", | 20260506 15:50:04.215 | INFO | "malloc" : " 14 GB", | 20260506 15:50:04.215 | INFO | "pinned" : " 29 GB" | 20260506 15:50:04.215 | INFO | } | 20260506 15:50:04.215 | INFO | Perf report for the last 5.13917 seconds: | 20260506 15:50:04.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:50:04.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:04.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:04.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:04.215 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:50:04.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:04.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:04.215 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:50:04.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:04.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:04.215 | INFO | Uploaded 36GiB to GPU in 1.97872s (18.1936GiB/s) | 20260506 15:50:04.215 | INFO | Downloaded 34GiB to GPU in 1.93683s (17.5545GiB/s) | 20260506 15:50:09.354 | INFO | Analyzing chunk frames = [2561536,2562048) | 20260506 15:50:09.354 | INFO | MemInfo: { | 20260506 15:50:09.354 | INFO | "gpu" : " 17 GB", | 20260506 15:50:09.354 | INFO | "huge" : "", | 20260506 15:50:09.354 | INFO | "huge_shared" : " 27 GB", | 20260506 15:50:09.355 | INFO | "malloc" : " 14 GB", | 20260506 15:50:09.355 | INFO | "pinned" : " 29 GB" | 20260506 15:50:09.355 | INFO | } | 20260506 15:50:09.355 | INFO | Perf report for the last 5.13996 seconds: | 20260506 15:50:09.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:50:09.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:09.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:09.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:09.355 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260506 15:50:09.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:09.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:09.355 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:50:09.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:09.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:09.355 | INFO | Uploaded 36GiB to GPU in 2.00658s (17.9409GiB/s) | 20260506 15:50:09.355 | INFO | Downloaded 34GiB to GPU in 1.91373s (17.7664GiB/s) | 20260506 15:50:14.493 | INFO | Analyzing chunk frames = [2562048,2562560) | 20260506 15:50:14.493 | INFO | MemInfo: { | 20260506 15:50:14.493 | INFO | "gpu" : " 17 GB", | 20260506 15:50:14.493 | INFO | "huge" : "", | 20260506 15:50:14.493 | INFO | "huge_shared" : " 27 GB", | 20260506 15:50:14.493 | INFO | "malloc" : " 14 GB", | 20260506 15:50:14.493 | INFO | "pinned" : " 29 GB" | 20260506 15:50:14.493 | INFO | } | 20260506 15:50:14.493 | INFO | Perf report for the last 5.13845 seconds: | 20260506 15:50:14.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:50:14.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:14.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:14.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:14.493 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 15:50:14.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:14.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:14.493 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:50:14.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:14.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:14.493 | INFO | Uploaded 36GiB to GPU in 1.9737s (18.2398GiB/s) | 20260506 15:50:14.493 | INFO | Downloaded 34GiB to GPU in 1.94162s (17.5111GiB/s) | 20260506 15:50:19.632 | INFO | Analyzing chunk frames = [2562560,2563072) | 20260506 15:50:19.632 | INFO | MemInfo: { | 20260506 15:50:19.632 | INFO | "gpu" : " 17 GB", | 20260506 15:50:19.632 | INFO | "huge" : "", | 20260506 15:50:19.633 | INFO | "huge_shared" : " 27 GB", | 20260506 15:50:19.633 | INFO | "malloc" : " 14 GB", | 20260506 15:50:19.633 | INFO | "pinned" : " 29 GB" | 20260506 15:50:19.633 | INFO | } | 20260506 15:50:19.633 | INFO | Perf report for the last 5.1396 seconds: | 20260506 15:50:19.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:50:19.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:19.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:19.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:19.633 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 15:50:19.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:19.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:19.633 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 15:50:19.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:19.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:19.633 | INFO | Uploaded 36GiB to GPU in 2.0305s (17.7296GiB/s) | 20260506 15:50:19.633 | INFO | Downloaded 34GiB to GPU in 1.92764s (17.6382GiB/s) | 20260506 15:50:24.772 | INFO | Analyzing chunk frames = [2563072,2563584) | 20260506 15:50:24.772 | INFO | MemInfo: { | 20260506 15:50:24.772 | INFO | "gpu" : " 17 GB", | 20260506 15:50:24.772 | INFO | "huge" : "", | 20260506 15:50:24.772 | INFO | "huge_shared" : " 27 GB", | 20260506 15:50:24.772 | INFO | "malloc" : " 14 GB", | 20260506 15:50:24.772 | INFO | "pinned" : " 29 GB" | 20260506 15:50:24.772 | INFO | } | 20260506 15:50:24.772 | INFO | Perf report for the last 5.13962 seconds: | 20260506 15:50:24.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:50:24.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:24.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:24.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:24.773 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 15:50:24.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:24.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:24.773 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:50:24.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:24.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:24.773 | INFO | Uploaded 36GiB to GPU in 1.97644s (18.2146GiB/s) | 20260506 15:50:24.773 | INFO | Downloaded 34GiB to GPU in 1.97233s (17.2385GiB/s) | 20260506 15:50:29.911 | INFO | Analyzing chunk frames = [2563584,2564096) | 20260506 15:50:29.911 | INFO | MemInfo: { | 20260506 15:50:29.911 | INFO | "gpu" : " 17 GB", | 20260506 15:50:29.911 | INFO | "huge" : "", | 20260506 15:50:29.911 | INFO | "huge_shared" : " 27 GB", | 20260506 15:50:29.911 | INFO | "malloc" : " 14 GB", | 20260506 15:50:29.911 | INFO | "pinned" : " 29 GB" | 20260506 15:50:29.911 | INFO | } | 20260506 15:50:29.911 | INFO | Perf report for the last 5.13853 seconds: | 20260506 15:50:29.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:50:29.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:29.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:29.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:29.911 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:50:29.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:29.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:29.911 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:50:29.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:29.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:29.911 | INFO | Uploaded 36GiB to GPU in 1.97203s (18.2553GiB/s) | 20260506 15:50:29.911 | INFO | Downloaded 34GiB to GPU in 1.98656s (17.115GiB/s) | 20260506 15:50:29.911 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:50:29.911 | INFO | Total mem usage: 17.4 GiB | 20260506 15:50:29.911 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:50:29.911 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -7.35%) | 20260506 15:50:29.911 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:50:29.911 | INFO | High Water Breakdown: | 20260506 15:50:29.911 | INFO | 61.9% : 10 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:50:29.911 | INFO | 38% : 6.16 GiB ( 92% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:50:29.911 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:50:29.911 | INFO | ------------------------------------------ | 20260506 15:50:29.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:50:29.911 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:50:29.911 | INFO | Total mem usage: 26.5 GiB | 20260506 15:50:29.911 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:50:29.911 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:50:29.911 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 15:50:29.912 | INFO | High Water Breakdown: | 20260506 15:50:29.912 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:50:29.912 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:50:29.912 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:50:29.912 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:50:29.912 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:50:29.912 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:50:29.912 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:50:29.912 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:50:29.912 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:50:29.912 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:50:29.912 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:50:29.912 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:50:29.912 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:50:29.912 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:50:29.912 | INFO | ------------------------------------------ | 20260506 15:50:29.912 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:50:29.912 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:50:29.912 | INFO | Total mem usage: 25.6 GiB | 20260506 15:50:29.912 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:50:29.912 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:50:29.912 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:50:29.912 | INFO | High Water Breakdown: | 20260506 15:50:29.912 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:50:29.912 | INFO | ------------------------------------------ | 20260506 15:50:29.912 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:50:29.912 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:50:29.912 | INFO | Total mem usage: 15.7 GiB | 20260506 15:50:29.912 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:50:29.912 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:50:29.912 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 15:50:29.912 | INFO | High Water Breakdown: | 20260506 15:50:29.912 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:50:29.912 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:50:29.912 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:50:29.912 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:50:29.912 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:50:29.912 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:50:29.912 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:50:29.912 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:50:29.912 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:50:29.912 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:50:29.912 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:50:29.912 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:50:29.912 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:50:29.912 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:50:29.912 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:50:29.912 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:50:29.912 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:50:29.912 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:50:29.912 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:50:29.912 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:50:29.912 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:50:29.912 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:50:29.912 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:50:29.912 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:50:29.912 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:50:29.912 | INFO | ------------------------------------------ | 20260506 15:50:29.912 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:50:29.912 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:50:29.912 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:50:29.912 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:50:29.912 | INFO | PerfReport: Full Execution -- 3.15e+01 / 7.56e+01, 1.36e+01 / 2.29e+00 | 20260506 15:50:29.912 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.02e-01, 9.72e-03 / 8.50e-03 | 20260506 15:50:29.912 | INFO | PerfReport: 11.59%, Baseline -- 3.65e+00 / 4.36e+00, 5.66e-01 / 4.20e-01 | 20260506 15:50:29.912 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.48e+00, 1.88e-01 / 1.39e-01 | 20260506 15:50:29.912 | INFO | PerfReport: 15.86%, DME -- 3.99e+01 / 4.40e+01, 3.72e+00 / 2.31e+00 | 20260506 15:50:29.912 | INFO | PerfReport: 56.91%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.52e+00 / 1.70e+00 | 20260506 15:50:29.912 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.31e+00 / 2.87e+00, 3.97e-01 / 2.14e-01 | 20260506 15:50:29.912 | INFO | PerfReport: 3.53%, Metrics -- 1.11e+00 / 1.48e+00, 2.90e-01 / 1.76e-01 | 20260506 15:50:29.912 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.85e-01 / 4.33e-01, 5.96e-02 / 2.24e-02 | 20260506 15:50:29.912 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:50:29.912 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:50:29.912 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:50:29.912 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 15:50:29.912 | INFO | PerfReport: 21.48%, Upload -- 2.09e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260506 15:50:29.912 | INFO | PerfReport: 21.13%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 15:50:29.912 | INFO | PerfReport: 32.29%, Compute -- 3.15e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 15:50:29.912 | INFO | PerfReport: 1.43%, LockUpload -- 1.40e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260506 15:50:29.912 | INFO | PerfReport: 1.15%, LockDownload -- 1.12e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260506 15:50:29.912 | INFO | PerfReport: 22.51%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 15:50:29.912 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260506 15:50:30.023 | INFO | Starting a round of Baz IO | 20260506 15:50:33.174 | INFO | Populating RTMetrics report | 20260506 15:50:33.175 | INFO | RTMetrics report populated | 20260506 15:50:33.175 | INFO | Opening tmp RTMetrics json file | 20260506 15:50:33.281 | INFO | Serializing RTMetrics to json | 20260506 15:50:33.354 | INFO | Serialization done, swapping tmp file now... | 20260506 15:50:33.387 | INFO | RTMetrics done | 20260506 15:50:33.587 | INFO | Finished a round of Baz IO: Wrote 2.095 GiB in 3.563 seconds | 20260506 15:50:33.587 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.84% / 0.01% / 63.15% | 20260506 15:50:33.587 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.90392% / 1.98682e-05% / 5.5631e-05% | 20260506 15:50:33.587 | INFO | Baz IO overall: | 20260506 15:50:33.587 | INFO | BytesWritten : 1340863231017 | 20260506 15:50:33.587 | INFO | NumEvents : 648888413196 | 20260506 15:50:35.050 | INFO | Analyzing chunk frames = [2564096,2564608) | 20260506 15:50:35.050 | INFO | MemInfo: { | 20260506 15:50:35.050 | INFO | "gpu" : " 17 GB", | 20260506 15:50:35.050 | INFO | "huge" : "", | 20260506 15:50:35.050 | INFO | "huge_shared" : " 27 GB", | 20260506 15:50:35.050 | INFO | "malloc" : " 14 GB", | 20260506 15:50:35.050 | INFO | "pinned" : " 29 GB" | 20260506 15:50:35.050 | INFO | } | 20260506 15:50:35.050 | INFO | Perf report for the last 5.13942 seconds: | 20260506 15:50:35.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:50:35.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:35.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:35.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:35.051 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 15:50:35.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:35.051 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260506 15:50:35.051 | INFO | PRE_HQ : 60.7%, 1.9, 58.8ms, 0 | 20260506 15:50:35.051 | INFO | BAZWRITER : 68.9%, 4.8, 170.5ms, 0 | 20260506 15:50:35.051 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.4ms, 0 | 20260506 15:50:35.051 | INFO | Uploaded 36GiB to GPU in 2.07613s (17.34GiB/s) | 20260506 15:50:35.051 | INFO | Downloaded 38GiB to GPU in 2.2229s (17.0948GiB/s) | 20260506 15:50:40.192 | INFO | Analyzing chunk frames = [2564608,2565120) | 20260506 15:50:40.192 | INFO | MemInfo: { | 20260506 15:50:40.192 | INFO | "gpu" : " 17 GB", | 20260506 15:50:40.192 | INFO | "huge" : "", | 20260506 15:50:40.192 | INFO | "huge_shared" : " 27 GB", | 20260506 15:50:40.192 | INFO | "malloc" : " 14 GB", | 20260506 15:50:40.192 | INFO | "pinned" : " 29 GB" | 20260506 15:50:40.192 | INFO | } | 20260506 15:50:40.192 | INFO | Perf report for the last 5.14137 seconds: | 20260506 15:50:40.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:50:40.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:40.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:40.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:40.192 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:50:40.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:40.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:40.192 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 15:50:40.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:40.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:40.192 | INFO | Uploaded 36GiB to GPU in 2.00244s (17.9781GiB/s) | 20260506 15:50:40.192 | INFO | Downloaded 34GiB to GPU in 1.95052s (17.4313GiB/s) | 20260506 15:50:45.331 | INFO | Analyzing chunk frames = [2565120,2565632) | 20260506 15:50:45.331 | INFO | MemInfo: { | 20260506 15:50:45.331 | INFO | "gpu" : " 17 GB", | 20260506 15:50:45.331 | INFO | "huge" : "", | 20260506 15:50:45.331 | INFO | "huge_shared" : " 27 GB", | 20260506 15:50:45.331 | INFO | "malloc" : " 14 GB", | 20260506 15:50:45.331 | INFO | "pinned" : " 29 GB" | 20260506 15:50:45.331 | INFO | } | 20260506 15:50:45.331 | INFO | Perf report for the last 5.139 seconds: | 20260506 15:50:45.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:50:45.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:45.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:45.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:45.331 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 15:50:45.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:45.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:45.331 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:50:45.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:45.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:45.331 | INFO | Uploaded 36GiB to GPU in 1.95921s (18.3747GiB/s) | 20260506 15:50:45.331 | INFO | Downloaded 34GiB to GPU in 1.94465s (17.4838GiB/s) | 20260506 15:50:50.468 | INFO | Analyzing chunk frames = [2565632,2566144) | 20260506 15:50:50.468 | INFO | MemInfo: { | 20260506 15:50:50.468 | INFO | "gpu" : " 17 GB", | 20260506 15:50:50.468 | INFO | "huge" : "", | 20260506 15:50:50.468 | INFO | "huge_shared" : " 27 GB", | 20260506 15:50:50.468 | INFO | "malloc" : " 14 GB", | 20260506 15:50:50.469 | INFO | "pinned" : " 29 GB" | 20260506 15:50:50.469 | INFO | } | 20260506 15:50:50.469 | INFO | Perf report for the last 5.13782 seconds: | 20260506 15:50:50.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:50:50.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:50.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:50.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:50.469 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 15:50:50.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:50.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:50.469 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:50:50.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:50.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:50.469 | INFO | Uploaded 36GiB to GPU in 1.94063s (18.5507GiB/s) | 20260506 15:50:50.469 | INFO | Downloaded 34GiB to GPU in 1.88236s (18.0625GiB/s) | 20260506 15:50:55.607 | INFO | Analyzing chunk frames = [2566144,2566656) | 20260506 15:50:55.607 | INFO | MemInfo: { | 20260506 15:50:55.607 | INFO | "gpu" : " 17 GB", | 20260506 15:50:55.607 | INFO | "huge" : "", | 20260506 15:50:55.607 | INFO | "huge_shared" : " 27 GB", | 20260506 15:50:55.607 | INFO | "malloc" : " 14 GB", | 20260506 15:50:55.607 | INFO | "pinned" : " 29 GB" | 20260506 15:50:55.607 | INFO | } | 20260506 15:50:55.607 | INFO | Perf report for the last 5.13886 seconds: | 20260506 15:50:55.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:50:55.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:55.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:55.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:55.608 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 15:50:55.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:55.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:55.608 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:50:55.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:55.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:50:55.608 | INFO | Uploaded 36GiB to GPU in 1.97707s (18.2088GiB/s) | 20260506 15:50:55.608 | INFO | Downloaded 34GiB to GPU in 1.92227s (17.6874GiB/s) | 20260506 15:51:00.747 | INFO | Analyzing chunk frames = [2566656,2567168) | 20260506 15:51:00.747 | INFO | MemInfo: { | 20260506 15:51:00.747 | INFO | "gpu" : " 17 GB", | 20260506 15:51:00.747 | INFO | "huge" : "", | 20260506 15:51:00.747 | INFO | "huge_shared" : " 27 GB", | 20260506 15:51:00.747 | INFO | "malloc" : " 14 GB", | 20260506 15:51:00.747 | INFO | "pinned" : " 29 GB" | 20260506 15:51:00.748 | INFO | } | 20260506 15:51:00.748 | INFO | Perf report for the last 5.14007 seconds: | 20260506 15:51:00.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:51:00.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:00.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:00.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:00.748 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:51:00.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:00.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:00.748 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 15:51:00.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:00.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:00.748 | INFO | Uploaded 36GiB to GPU in 2.05981s (17.4773GiB/s) | 20260506 15:51:00.748 | INFO | Downloaded 34GiB to GPU in 1.97293s (17.2332GiB/s) | 20260506 15:51:05.887 | INFO | Analyzing chunk frames = [2567168,2567680) | 20260506 15:51:05.887 | INFO | MemInfo: { | 20260506 15:51:05.887 | INFO | "gpu" : " 17 GB", | 20260506 15:51:05.887 | INFO | "huge" : "", | 20260506 15:51:05.887 | INFO | "huge_shared" : " 27 GB", | 20260506 15:51:05.887 | INFO | "malloc" : " 14 GB", | 20260506 15:51:05.887 | INFO | "pinned" : " 29 GB" | 20260506 15:51:05.887 | INFO | } | 20260506 15:51:05.887 | INFO | Perf report for the last 5.13982 seconds: | 20260506 15:51:05.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:51:05.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:05.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:05.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:05.887 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:51:05.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:05.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:05.887 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:51:05.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:05.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:05.888 | INFO | Uploaded 36GiB to GPU in 2.01516s (17.8646GiB/s) | 20260506 15:51:05.888 | INFO | Downloaded 34GiB to GPU in 1.99978s (17.0019GiB/s) | 20260506 15:51:11.026 | INFO | Analyzing chunk frames = [2567680,2568192) | 20260506 15:51:11.026 | INFO | MemInfo: { | 20260506 15:51:11.026 | INFO | "gpu" : " 17 GB", | 20260506 15:51:11.026 | INFO | "huge" : "", | 20260506 15:51:11.026 | INFO | "huge_shared" : " 27 GB", | 20260506 15:51:11.026 | INFO | "malloc" : " 14 GB", | 20260506 15:51:11.026 | INFO | "pinned" : " 29 GB" | 20260506 15:51:11.026 | INFO | } | 20260506 15:51:11.027 | INFO | Perf report for the last 5.13916 seconds: | 20260506 15:51:11.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:51:11.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:11.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:11.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:11.027 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:51:11.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:11.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:11.027 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:51:11.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:11.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:11.027 | INFO | Uploaded 36GiB to GPU in 2.04204s (17.6294GiB/s) | 20260506 15:51:11.027 | INFO | Downloaded 34GiB to GPU in 2.00033s (16.9972GiB/s) | 20260506 15:51:11.149 | INFO | Starting a round of Baz IO | 20260506 15:51:14.301 | INFO | Populating RTMetrics report | 20260506 15:51:14.301 | INFO | RTMetrics report populated | 20260506 15:51:14.301 | INFO | Opening tmp RTMetrics json file | 20260506 15:51:14.303 | INFO | Serializing RTMetrics to json | 20260506 15:51:14.306 | INFO | Serialization done, swapping tmp file now... | 20260506 15:51:14.306 | INFO | RTMetrics done | 20260506 15:51:14.519 | INFO | Finished a round of Baz IO: Wrote 2.095 GiB in 3.37 seconds | 20260506 15:51:14.519 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.83% / 0.01% / 63.16% | 20260506 15:51:14.519 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.89764% / 1.58946e-05% / 6.35783e-05% | 20260506 15:51:14.519 | INFO | Baz IO overall: | 20260506 15:51:14.519 | INFO | BytesWritten : 1343112701115 | 20260506 15:51:14.519 | INFO | NumEvents : 649976410931 | 20260506 15:51:16.165 | INFO | Analyzing chunk frames = [2568192,2568704) | 20260506 15:51:16.165 | INFO | MemInfo: { | 20260506 15:51:16.165 | INFO | "gpu" : " 17 GB", | 20260506 15:51:16.165 | INFO | "huge" : "", | 20260506 15:51:16.165 | INFO | "huge_shared" : " 27 GB", | 20260506 15:51:16.165 | INFO | "malloc" : " 14 GB", | 20260506 15:51:16.165 | INFO | "pinned" : " 29 GB" | 20260506 15:51:16.165 | INFO | } | 20260506 15:51:16.165 | INFO | Perf report for the last 5.13876 seconds: | 20260506 15:51:16.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:51:16.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:16.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:16.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:16.165 | INFO | ANALYSIS : 63.0%, 2.9, 95.7ms, 0 | 20260506 15:51:16.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:16.165 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 15:51:16.165 | INFO | PRE_HQ : 60.6%, 1.9, 58.9ms, 0 | 20260506 15:51:16.165 | INFO | BAZWRITER : 65.0%, 4.8, 161.4ms, 0 | 20260506 15:51:16.165 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 15:51:16.166 | INFO | Uploaded 36GiB to GPU in 2.0469s (17.5876GiB/s) | 20260506 15:51:16.166 | INFO | Downloaded 38GiB to GPU in 2.23674s (16.989GiB/s) | 20260506 15:51:21.304 | INFO | Analyzing chunk frames = [2568704,2569216) | 20260506 15:51:21.304 | INFO | MemInfo: { | 20260506 15:51:21.304 | INFO | "gpu" : " 17 GB", | 20260506 15:51:21.304 | INFO | "huge" : "", | 20260506 15:51:21.304 | INFO | "huge_shared" : " 27 GB", | 20260506 15:51:21.304 | INFO | "malloc" : " 14 GB", | 20260506 15:51:21.304 | INFO | "pinned" : " 29 GB" | 20260506 15:51:21.304 | INFO | } | 20260506 15:51:21.304 | INFO | Perf report for the last 5.13884 seconds: | 20260506 15:51:21.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:51:21.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:21.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:21.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:21.304 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:51:21.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:21.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:21.304 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 15:51:21.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:21.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:21.304 | INFO | Uploaded 36GiB to GPU in 1.99126s (18.079GiB/s) | 20260506 15:51:21.304 | INFO | Downloaded 34GiB to GPU in 1.93192s (17.5991GiB/s) | 20260506 15:51:26.443 | INFO | Analyzing chunk frames = [2569216,2569728) | 20260506 15:51:26.443 | INFO | MemInfo: { | 20260506 15:51:26.443 | INFO | "gpu" : " 17 GB", | 20260506 15:51:26.443 | INFO | "huge" : "", | 20260506 15:51:26.444 | INFO | "huge_shared" : " 27 GB", | 20260506 15:51:26.444 | INFO | "malloc" : " 14 GB", | 20260506 15:51:26.444 | INFO | "pinned" : " 29 GB" | 20260506 15:51:26.444 | INFO | } | 20260506 15:51:26.444 | INFO | Perf report for the last 5.13949 seconds: | 20260506 15:51:26.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:51:26.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:26.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:26.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:26.444 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 15:51:26.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:26.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:26.444 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:51:26.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:26.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:26.444 | INFO | Uploaded 36GiB to GPU in 1.98972s (18.093GiB/s) | 20260506 15:51:26.444 | INFO | Downloaded 34GiB to GPU in 1.93651s (17.5574GiB/s) | 20260506 15:51:31.585 | INFO | Analyzing chunk frames = [2569728,2570240) | 20260506 15:51:31.585 | INFO | MemInfo: { | 20260506 15:51:31.585 | INFO | "gpu" : " 17 GB", | 20260506 15:51:31.585 | INFO | "huge" : "", | 20260506 15:51:31.585 | INFO | "huge_shared" : " 27 GB", | 20260506 15:51:31.585 | INFO | "malloc" : " 14 GB", | 20260506 15:51:31.585 | INFO | "pinned" : " 29 GB" | 20260506 15:51:31.585 | INFO | } | 20260506 15:51:31.585 | INFO | Perf report for the last 5.14157 seconds: | 20260506 15:51:31.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:51:31.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:31.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:31.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:31.585 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 15:51:31.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:31.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:31.585 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:51:31.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:31.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:31.585 | INFO | Uploaded 36GiB to GPU in 2.02213s (17.803GiB/s) | 20260506 15:51:31.586 | INFO | Downloaded 34GiB to GPU in 1.91344s (17.769GiB/s) | 20260506 15:51:36.723 | INFO | Analyzing chunk frames = [2570240,2570752) | 20260506 15:51:36.723 | INFO | MemInfo: { | 20260506 15:51:36.723 | INFO | "gpu" : " 17 GB", | 20260506 15:51:36.723 | INFO | "huge" : "", | 20260506 15:51:36.723 | INFO | "huge_shared" : " 27 GB", | 20260506 15:51:36.723 | INFO | "malloc" : " 14 GB", | 20260506 15:51:36.723 | INFO | "pinned" : " 29 GB" | 20260506 15:51:36.723 | INFO | } | 20260506 15:51:36.723 | INFO | Perf report for the last 5.13819 seconds: | 20260506 15:51:36.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:51:36.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:36.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:36.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:36.724 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 15:51:36.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:36.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:36.724 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:51:36.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:36.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:36.724 | INFO | Uploaded 36GiB to GPU in 2.02222s (17.8022GiB/s) | 20260506 15:51:36.724 | INFO | Downloaded 34GiB to GPU in 1.9283s (17.6321GiB/s) | 20260506 15:51:41.862 | INFO | Analyzing chunk frames = [2570752,2571264) | 20260506 15:51:41.862 | INFO | MemInfo: { | 20260506 15:51:41.862 | INFO | "gpu" : " 17 GB", | 20260506 15:51:41.862 | INFO | "huge" : "", | 20260506 15:51:41.862 | INFO | "huge_shared" : " 27 GB", | 20260506 15:51:41.862 | INFO | "malloc" : " 14 GB", | 20260506 15:51:41.862 | INFO | "pinned" : " 29 GB" | 20260506 15:51:41.862 | INFO | } | 20260506 15:51:41.862 | INFO | Perf report for the last 5.13878 seconds: | 20260506 15:51:41.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:51:41.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:41.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:41.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:41.862 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 15:51:41.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:41.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:41.862 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 15:51:41.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:41.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:41.863 | INFO | Uploaded 36GiB to GPU in 2.0452s (17.6022GiB/s) | 20260506 15:51:41.863 | INFO | Downloaded 34GiB to GPU in 1.92688s (17.6451GiB/s) | 20260506 15:51:47.002 | INFO | Analyzing chunk frames = [2571264,2571776) | 20260506 15:51:47.002 | INFO | MemInfo: { | 20260506 15:51:47.002 | INFO | "gpu" : " 17 GB", | 20260506 15:51:47.002 | INFO | "huge" : "", | 20260506 15:51:47.002 | INFO | "huge_shared" : " 27 GB", | 20260506 15:51:47.002 | INFO | "malloc" : " 14 GB", | 20260506 15:51:47.002 | INFO | "pinned" : " 29 GB" | 20260506 15:51:47.002 | INFO | } | 20260506 15:51:47.002 | INFO | Perf report for the last 5.13987 seconds: | 20260506 15:51:47.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:51:47.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:47.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:47.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:47.002 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 15:51:47.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:47.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:47.002 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:51:47.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:47.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:47.002 | INFO | Uploaded 36GiB to GPU in 1.98463s (18.1394GiB/s) | 20260506 15:51:47.002 | INFO | Downloaded 34GiB to GPU in 1.97405s (17.2234GiB/s) | 20260506 15:51:52.140 | INFO | Analyzing chunk frames = [2571776,2572288) | 20260506 15:51:52.140 | INFO | MemInfo: { | 20260506 15:51:52.140 | INFO | "gpu" : " 17 GB", | 20260506 15:51:52.140 | INFO | "huge" : "", | 20260506 15:51:52.140 | INFO | "huge_shared" : " 27 GB", | 20260506 15:51:52.140 | INFO | "malloc" : " 14 GB", | 20260506 15:51:52.140 | INFO | "pinned" : " 29 GB" | 20260506 15:51:52.140 | INFO | } | 20260506 15:51:52.140 | INFO | Perf report for the last 5.13793 seconds: | 20260506 15:51:52.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:51:52.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:52.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:52.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:52.140 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:51:52.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:52.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:52.140 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:51:52.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:52.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:52.141 | INFO | Uploaded 36GiB to GPU in 1.99974s (18.0023GiB/s) | 20260506 15:51:52.141 | INFO | Downloaded 34GiB to GPU in 1.99324s (17.0577GiB/s) | 20260506 15:51:52.141 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:51:52.141 | INFO | Total mem usage: 17.4 GiB | 20260506 15:51:52.141 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:51:52.141 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -11.5%) | 20260506 15:51:52.141 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:51:52.141 | INFO | High Water Breakdown: | 20260506 15:51:52.141 | INFO | 61.3% : 9.58 GiB ( 87.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:51:52.141 | INFO | 38.6% : 6.02 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:51:52.141 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:51:52.141 | INFO | ------------------------------------------ | 20260506 15:51:52.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:51:52.141 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:51:52.141 | INFO | Total mem usage: 26.5 GiB | 20260506 15:51:52.141 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:51:52.141 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:51:52.141 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 15:51:52.141 | INFO | High Water Breakdown: | 20260506 15:51:52.141 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:51:52.141 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:51:52.141 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:51:52.141 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:51:52.141 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:51:52.141 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:51:52.141 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:51:52.141 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:51:52.141 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:51:52.141 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:51:52.141 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:51:52.141 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:51:52.141 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:51:52.141 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:51:52.141 | INFO | ------------------------------------------ | 20260506 15:51:52.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:51:52.141 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:51:52.141 | INFO | Total mem usage: 25.6 GiB | 20260506 15:51:52.141 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:51:52.141 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:51:52.141 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:51:52.141 | INFO | High Water Breakdown: | 20260506 15:51:52.141 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:51:52.141 | INFO | ------------------------------------------ | 20260506 15:51:52.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:51:52.141 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:51:52.141 | INFO | Total mem usage: 15.7 GiB | 20260506 15:51:52.141 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:51:52.141 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:51:52.141 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 15:51:52.141 | INFO | High Water Breakdown: | 20260506 15:51:52.141 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:51:52.141 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:51:52.141 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:51:52.141 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:51:52.141 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:51:52.141 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:51:52.141 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:51:52.141 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:51:52.141 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:51:52.141 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:51:52.141 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:51:52.141 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:51:52.141 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:51:52.141 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:51:52.141 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:51:52.141 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:51:52.141 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:51:52.141 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:51:52.141 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:51:52.141 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:51:52.141 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:51:52.141 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:51:52.141 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:51:52.141 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:51:52.141 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:51:52.141 | INFO | ------------------------------------------ | 20260506 15:51:52.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:51:52.141 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:51:52.141 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:51:52.141 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:51:52.141 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.52e+01, 1.35e+01 / 1.82e+00 | 20260506 15:51:52.141 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.02e-01, 1.08e-02 / 1.61e-02 | 20260506 15:51:52.141 | INFO | PerfReport: 11.59%, Baseline -- 3.64e+00 / 4.42e+00, 5.52e-01 / 3.64e-01 | 20260506 15:51:52.141 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.44e+00, 1.81e-01 / 1.49e-01 | 20260506 15:51:52.141 | INFO | PerfReport: 15.86%, DME -- 3.98e+01 / 4.39e+01, 3.76e+00 / 2.20e+00 | 20260506 15:51:52.141 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.44e+00 / 2.02e+00 | 20260506 15:51:52.141 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.31e+00 / 2.91e+00, 3.94e-01 / 2.23e-01 | 20260506 15:51:52.141 | INFO | PerfReport: 3.53%, Metrics -- 1.11e+00 / 1.45e+00, 2.88e-01 / 1.53e-01 | 20260506 15:51:52.141 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.86e-01 / 4.34e-01, 6.05e-02 / 2.78e-02 | 20260506 15:51:52.141 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:51:52.141 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:51:52.141 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:51:52.141 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 15:51:52.141 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 15:51:52.141 | INFO | PerfReport: 21.21%, Download -- 2.07e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 15:51:52.141 | INFO | PerfReport: 32.24%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 15:51:52.142 | INFO | PerfReport: 1.41%, LockUpload -- 1.38e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260506 15:51:52.142 | INFO | PerfReport: 1.17%, LockDownload -- 1.14e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 15:51:52.142 | INFO | PerfReport: 22.46%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 15:51:52.142 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260506 15:51:52.260 | INFO | Starting a round of Baz IO | 20260506 15:51:55.406 | INFO | Populating RTMetrics report | 20260506 15:51:55.406 | INFO | RTMetrics report populated | 20260506 15:51:55.406 | INFO | Opening tmp RTMetrics json file | 20260506 15:51:55.407 | INFO | Serializing RTMetrics to json | 20260506 15:51:55.409 | INFO | Serialization done, swapping tmp file now... | 20260506 15:51:55.409 | INFO | RTMetrics done | 20260506 15:51:55.531 | INFO | Finished a round of Baz IO: Wrote 2.095 GiB in 3.27 seconds | 20260506 15:51:55.531 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.82% / 0.01% / 63.17% | 20260506 15:51:55.531 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.89624% / 2.78155e-05% / 6.75519e-05% | 20260506 15:51:55.531 | INFO | Baz IO overall: | 20260506 15:51:55.531 | INFO | BytesWritten : 1345362245385 | 20260506 15:51:55.531 | INFO | NumEvents : 651064930658 | 20260506 15:51:57.280 | INFO | Analyzing chunk frames = [2572288,2572800) | 20260506 15:51:57.280 | INFO | MemInfo: { | 20260506 15:51:57.280 | INFO | "gpu" : " 17 GB", | 20260506 15:51:57.280 | INFO | "huge" : "", | 20260506 15:51:57.280 | INFO | "huge_shared" : " 27 GB", | 20260506 15:51:57.280 | INFO | "malloc" : " 14 GB", | 20260506 15:51:57.280 | INFO | "pinned" : " 29 GB" | 20260506 15:51:57.280 | INFO | } | 20260506 15:51:57.280 | INFO | Perf report for the last 5.14017 seconds: | 20260506 15:51:57.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:51:57.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:57.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:57.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:57.280 | INFO | ANALYSIS : 62.9%, 2.9, 95.3ms, 0 | 20260506 15:51:57.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:51:57.280 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 15:51:57.280 | INFO | PRE_HQ : 60.7%, 1.9, 59.3ms, 0 | 20260506 15:51:57.280 | INFO | BAZWRITER : 63.0%, 4.7, 151.7ms, 0 | 20260506 15:51:57.280 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 15:51:57.281 | INFO | Uploaded 36GiB to GPU in 2.04454s (17.6079GiB/s) | 20260506 15:51:57.281 | INFO | Downloaded 38GiB to GPU in 2.21991s (17.1178GiB/s) | 20260506 15:52:02.422 | INFO | Analyzing chunk frames = [2572800,2573312) | 20260506 15:52:02.422 | INFO | MemInfo: { | 20260506 15:52:02.422 | INFO | "gpu" : " 17 GB", | 20260506 15:52:02.422 | INFO | "huge" : "", | 20260506 15:52:02.422 | INFO | "huge_shared" : " 27 GB", | 20260506 15:52:02.422 | INFO | "malloc" : " 14 GB", | 20260506 15:52:02.422 | INFO | "pinned" : " 29 GB" | 20260506 15:52:02.422 | INFO | } | 20260506 15:52:02.422 | INFO | Perf report for the last 5.14155 seconds: | 20260506 15:52:02.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:52:02.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:02.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:02.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:02.422 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 15:52:02.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:02.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:02.422 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:52:02.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:02.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:02.422 | INFO | Uploaded 36GiB to GPU in 1.98348s (18.1499GiB/s) | 20260506 15:52:02.422 | INFO | Downloaded 34GiB to GPU in 1.93303s (17.589GiB/s) | 20260506 15:52:07.558 | INFO | Analyzing chunk frames = [2573312,2573824) | 20260506 15:52:07.558 | INFO | MemInfo: { | 20260506 15:52:07.558 | INFO | "gpu" : " 17 GB", | 20260506 15:52:07.558 | INFO | "huge" : "", | 20260506 15:52:07.558 | INFO | "huge_shared" : " 27 GB", | 20260506 15:52:07.558 | INFO | "malloc" : " 14 GB", | 20260506 15:52:07.558 | INFO | "pinned" : " 29 GB" | 20260506 15:52:07.558 | INFO | } | 20260506 15:52:07.559 | INFO | Perf report for the last 5.13668 seconds: | 20260506 15:52:07.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:52:07.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:07.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:07.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:07.559 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 15:52:07.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:07.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:07.559 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:52:07.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:07.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:07.559 | INFO | Uploaded 36GiB to GPU in 1.966s (18.3113GiB/s) | 20260506 15:52:07.559 | INFO | Downloaded 34GiB to GPU in 1.93598s (17.5622GiB/s) | 20260506 15:52:12.698 | INFO | Analyzing chunk frames = [2573824,2574336) | 20260506 15:52:12.698 | INFO | MemInfo: { | 20260506 15:52:12.698 | INFO | "gpu" : " 17 GB", | 20260506 15:52:12.698 | INFO | "huge" : "", | 20260506 15:52:12.698 | INFO | "huge_shared" : " 27 GB", | 20260506 15:52:12.698 | INFO | "malloc" : " 14 GB", | 20260506 15:52:12.698 | INFO | "pinned" : " 29 GB" | 20260506 15:52:12.698 | INFO | } | 20260506 15:52:12.698 | INFO | Perf report for the last 5.13972 seconds: | 20260506 15:52:12.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:52:12.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:12.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:12.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:12.698 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260506 15:52:12.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:12.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:12.698 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:52:12.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:12.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:12.699 | INFO | Uploaded 36GiB to GPU in 2.00719s (17.9355GiB/s) | 20260506 15:52:12.699 | INFO | Downloaded 34GiB to GPU in 1.91437s (17.7604GiB/s) | 20260506 15:52:17.838 | INFO | Analyzing chunk frames = [2574336,2574848) | 20260506 15:52:17.838 | INFO | MemInfo: { | 20260506 15:52:17.838 | INFO | "gpu" : " 17 GB", | 20260506 15:52:17.838 | INFO | "huge" : "", | 20260506 15:52:17.838 | INFO | "huge_shared" : " 27 GB", | 20260506 15:52:17.838 | INFO | "malloc" : " 14 GB", | 20260506 15:52:17.838 | INFO | "pinned" : " 29 GB" | 20260506 15:52:17.838 | INFO | } | 20260506 15:52:17.838 | INFO | Perf report for the last 5.14 seconds: | 20260506 15:52:17.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:52:17.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:17.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:17.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:17.838 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 15:52:17.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:17.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:17.838 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:52:17.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:17.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:17.838 | INFO | Uploaded 36GiB to GPU in 1.99245s (18.0682GiB/s) | 20260506 15:52:17.839 | INFO | Downloaded 34GiB to GPU in 1.92234s (17.6867GiB/s) | 20260506 15:52:22.977 | INFO | Analyzing chunk frames = [2574848,2575360) | 20260506 15:52:22.977 | INFO | MemInfo: { | 20260506 15:52:22.977 | INFO | "gpu" : " 17 GB", | 20260506 15:52:22.977 | INFO | "huge" : "", | 20260506 15:52:22.977 | INFO | "huge_shared" : " 27 GB", | 20260506 15:52:22.977 | INFO | "malloc" : " 14 GB", | 20260506 15:52:22.977 | INFO | "pinned" : " 29 GB" | 20260506 15:52:22.977 | INFO | } | 20260506 15:52:22.977 | INFO | Perf report for the last 5.13904 seconds: | 20260506 15:52:22.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:52:22.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:22.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:22.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:22.977 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 15:52:22.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:22.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:22.977 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260506 15:52:22.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:22.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:22.978 | INFO | Uploaded 36GiB to GPU in 2.01611s (17.8562GiB/s) | 20260506 15:52:22.978 | INFO | Downloaded 34GiB to GPU in 1.9419s (17.5086GiB/s) | 20260506 15:52:28.116 | INFO | Analyzing chunk frames = [2575360,2575872) | 20260506 15:52:28.116 | INFO | MemInfo: { | 20260506 15:52:28.116 | INFO | "gpu" : " 17 GB", | 20260506 15:52:28.116 | INFO | "huge" : "", | 20260506 15:52:28.116 | INFO | "huge_shared" : " 27 GB", | 20260506 15:52:28.116 | INFO | "malloc" : " 14 GB", | 20260506 15:52:28.116 | INFO | "pinned" : " 29 GB" | 20260506 15:52:28.116 | INFO | } | 20260506 15:52:28.116 | INFO | Perf report for the last 5.13871 seconds: | 20260506 15:52:28.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:52:28.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:28.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:28.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:28.116 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 15:52:28.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:28.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:28.116 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:52:28.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:28.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:28.116 | INFO | Uploaded 36GiB to GPU in 1.98287s (18.1555GiB/s) | 20260506 15:52:28.116 | INFO | Downloaded 34GiB to GPU in 1.96549s (17.2985GiB/s) | 20260506 15:52:33.255 | INFO | Analyzing chunk frames = [2575872,2576384) | 20260506 15:52:33.255 | INFO | MemInfo: { | 20260506 15:52:33.255 | INFO | "gpu" : " 17 GB", | 20260506 15:52:33.255 | INFO | "huge" : "", | 20260506 15:52:33.255 | INFO | "huge_shared" : " 27 GB", | 20260506 15:52:33.255 | INFO | "malloc" : " 14 GB", | 20260506 15:52:33.255 | INFO | "pinned" : " 29 GB" | 20260506 15:52:33.255 | INFO | } | 20260506 15:52:33.255 | INFO | Perf report for the last 5.13904 seconds: | 20260506 15:52:33.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:52:33.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:33.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:33.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:33.255 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:52:33.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:33.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:33.255 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:52:33.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:33.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:33.255 | INFO | Uploaded 36GiB to GPU in 2.02164s (17.8074GiB/s) | 20260506 15:52:33.255 | INFO | Downloaded 34GiB to GPU in 1.98816s (17.1013GiB/s) | 20260506 15:52:33.368 | INFO | Starting a round of Baz IO | 20260506 15:52:36.521 | INFO | Populating RTMetrics report | 20260506 15:52:36.522 | INFO | RTMetrics report populated | 20260506 15:52:36.522 | INFO | Opening tmp RTMetrics json file | 20260506 15:52:36.522 | INFO | Serializing RTMetrics to json | 20260506 15:52:36.524 | INFO | Serialization done, swapping tmp file now... | 20260506 15:52:36.524 | INFO | RTMetrics done | 20260506 15:52:36.637 | INFO | Finished a round of Baz IO: Wrote 2.093 GiB in 3.268 seconds | 20260506 15:52:36.637 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.81% / 0.01% / 63.18% | 20260506 15:52:36.637 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.88649% / 7.94729e-06% / 6.35783e-05% | 20260506 15:52:36.637 | INFO | Baz IO overall: | 20260506 15:52:36.637 | INFO | BytesWritten : 1347609548203 | 20260506 15:52:36.637 | INFO | NumEvents : 652151932091 | 20260506 15:52:38.394 | INFO | Analyzing chunk frames = [2576384,2576896) | 20260506 15:52:38.394 | INFO | MemInfo: { | 20260506 15:52:38.394 | INFO | "gpu" : " 17 GB", | 20260506 15:52:38.394 | INFO | "huge" : "", | 20260506 15:52:38.394 | INFO | "huge_shared" : " 27 GB", | 20260506 15:52:38.395 | INFO | "malloc" : " 14 GB", | 20260506 15:52:38.395 | INFO | "pinned" : " 29 GB" | 20260506 15:52:38.395 | INFO | } | 20260506 15:52:38.395 | INFO | Perf report for the last 5.13948 seconds: | 20260506 15:52:38.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:52:38.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:38.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:38.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:38.395 | INFO | ANALYSIS : 62.9%, 2.9, 95.3ms, 0 | 20260506 15:52:38.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:38.395 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 15:52:38.395 | INFO | PRE_HQ : 60.7%, 1.9, 59.0ms, 0 | 20260506 15:52:38.395 | INFO | BAZWRITER : 63.0%, 4.7, 153.5ms, 0 | 20260506 15:52:38.395 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 15:52:38.395 | INFO | Uploaded 36GiB to GPU in 2.05562s (17.513GiB/s) | 20260506 15:52:38.395 | INFO | Downloaded 38GiB to GPU in 2.23233s (17.0226GiB/s) | 20260506 15:52:43.534 | INFO | Analyzing chunk frames = [2576896,2577408) | 20260506 15:52:43.534 | INFO | MemInfo: { | 20260506 15:52:43.534 | INFO | "gpu" : " 17 GB", | 20260506 15:52:43.534 | INFO | "huge" : "", | 20260506 15:52:43.534 | INFO | "huge_shared" : " 27 GB", | 20260506 15:52:43.534 | INFO | "malloc" : " 14 GB", | 20260506 15:52:43.534 | INFO | "pinned" : " 29 GB" | 20260506 15:52:43.534 | INFO | } | 20260506 15:52:43.534 | INFO | Perf report for the last 5.13951 seconds: | 20260506 15:52:43.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:52:43.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:43.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:43.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:43.534 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:52:43.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:43.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:43.534 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:52:43.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:43.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:43.534 | INFO | Uploaded 36GiB to GPU in 2.002s (17.982GiB/s) | 20260506 15:52:43.534 | INFO | Downloaded 34GiB to GPU in 1.9489s (17.4458GiB/s) | 20260506 15:52:48.674 | INFO | Analyzing chunk frames = [2577408,2577920) | 20260506 15:52:48.674 | INFO | MemInfo: { | 20260506 15:52:48.674 | INFO | "gpu" : " 17 GB", | 20260506 15:52:48.674 | INFO | "huge" : "", | 20260506 15:52:48.674 | INFO | "huge_shared" : " 27 GB", | 20260506 15:52:48.674 | INFO | "malloc" : " 14 GB", | 20260506 15:52:48.674 | INFO | "pinned" : " 29 GB" | 20260506 15:52:48.674 | INFO | } | 20260506 15:52:48.675 | INFO | Perf report for the last 5.14037 seconds: | 20260506 15:52:48.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:52:48.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:48.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:48.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:48.675 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:52:48.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:48.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:48.675 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:52:48.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:48.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:48.675 | INFO | Uploaded 36GiB to GPU in 1.98309s (18.1535GiB/s) | 20260506 15:52:48.675 | INFO | Downloaded 34GiB to GPU in 1.93557s (17.5659GiB/s) | 20260506 15:52:53.814 | INFO | Analyzing chunk frames = [2577920,2578432) | 20260506 15:52:53.814 | INFO | MemInfo: { | 20260506 15:52:53.814 | INFO | "gpu" : " 17 GB", | 20260506 15:52:53.814 | INFO | "huge" : "", | 20260506 15:52:53.814 | INFO | "huge_shared" : " 27 GB", | 20260506 15:52:53.814 | INFO | "malloc" : " 14 GB", | 20260506 15:52:53.814 | INFO | "pinned" : " 29 GB" | 20260506 15:52:53.814 | INFO | } | 20260506 15:52:53.814 | INFO | Perf report for the last 5.13931 seconds: | 20260506 15:52:53.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:52:53.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:53.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:53.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:53.814 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 15:52:53.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:53.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:53.814 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:52:53.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:53.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:53.814 | INFO | Uploaded 36GiB to GPU in 2.01853s (17.8348GiB/s) | 20260506 15:52:53.814 | INFO | Downloaded 34GiB to GPU in 1.90782s (17.8214GiB/s) | 20260506 15:52:58.953 | INFO | Analyzing chunk frames = [2578432,2578944) | 20260506 15:52:58.953 | INFO | MemInfo: { | 20260506 15:52:58.953 | INFO | "gpu" : " 17 GB", | 20260506 15:52:58.953 | INFO | "huge" : "", | 20260506 15:52:58.953 | INFO | "huge_shared" : " 27 GB", | 20260506 15:52:58.953 | INFO | "malloc" : " 14 GB", | 20260506 15:52:58.953 | INFO | "pinned" : " 29 GB" | 20260506 15:52:58.953 | INFO | } | 20260506 15:52:58.953 | INFO | Perf report for the last 5.13888 seconds: | 20260506 15:52:58.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:52:58.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:58.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:58.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:58.953 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 15:52:58.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:58.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:58.953 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:52:58.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:58.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:52:58.953 | INFO | Uploaded 36GiB to GPU in 2.00497s (17.9554GiB/s) | 20260506 15:52:58.953 | INFO | Downloaded 34GiB to GPU in 1.9347s (17.5738GiB/s) | 20260506 15:53:04.091 | INFO | Analyzing chunk frames = [2578944,2579456) | 20260506 15:53:04.091 | INFO | MemInfo: { | 20260506 15:53:04.091 | INFO | "gpu" : " 17 GB", | 20260506 15:53:04.091 | INFO | "huge" : "", | 20260506 15:53:04.091 | INFO | "huge_shared" : " 27 GB", | 20260506 15:53:04.091 | INFO | "malloc" : " 14 GB", | 20260506 15:53:04.091 | INFO | "pinned" : " 29 GB" | 20260506 15:53:04.091 | INFO | } | 20260506 15:53:04.091 | INFO | Perf report for the last 5.13875 seconds: | 20260506 15:53:04.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:53:04.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:04.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:04.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:04.092 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 15:53:04.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:04.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:04.092 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 15:53:04.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:04.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:04.092 | INFO | Uploaded 36GiB to GPU in 2.03711s (17.6721GiB/s) | 20260506 15:53:04.092 | INFO | Downloaded 34GiB to GPU in 1.9407s (17.5195GiB/s) | 20260506 15:53:09.230 | INFO | Analyzing chunk frames = [2579456,2579968) | 20260506 15:53:09.230 | INFO | MemInfo: { | 20260506 15:53:09.231 | INFO | "gpu" : " 17 GB", | 20260506 15:53:09.231 | INFO | "huge" : "", | 20260506 15:53:09.231 | INFO | "huge_shared" : " 27 GB", | 20260506 15:53:09.231 | INFO | "malloc" : " 14 GB", | 20260506 15:53:09.231 | INFO | "pinned" : " 29 GB" | 20260506 15:53:09.231 | INFO | } | 20260506 15:53:09.231 | INFO | Perf report for the last 5.13932 seconds: | 20260506 15:53:09.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:53:09.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:09.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:09.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:09.231 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 15:53:09.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:09.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:09.231 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:53:09.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:09.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:09.231 | INFO | Uploaded 36GiB to GPU in 1.99505s (18.0446GiB/s) | 20260506 15:53:09.231 | INFO | Downloaded 34GiB to GPU in 1.98019s (17.17GiB/s) | 20260506 15:53:14.370 | INFO | Analyzing chunk frames = [2579968,2580480) | 20260506 15:53:14.370 | INFO | MemInfo: { | 20260506 15:53:14.370 | INFO | "gpu" : " 17 GB", | 20260506 15:53:14.370 | INFO | "huge" : "", | 20260506 15:53:14.370 | INFO | "huge_shared" : " 27 GB", | 20260506 15:53:14.370 | INFO | "malloc" : " 14 GB", | 20260506 15:53:14.370 | INFO | "pinned" : " 29 GB" | 20260506 15:53:14.370 | INFO | } | 20260506 15:53:14.370 | INFO | Perf report for the last 5.13943 seconds: | 20260506 15:53:14.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:53:14.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:14.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:14.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:14.370 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:53:14.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:14.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:14.370 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:53:14.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:14.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:14.371 | INFO | Uploaded 36GiB to GPU in 2.02445s (17.7826GiB/s) | 20260506 15:53:14.371 | INFO | Downloaded 34GiB to GPU in 1.97727s (17.1954GiB/s) | 20260506 15:53:14.371 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:53:14.371 | INFO | Total mem usage: 17.4 GiB | 20260506 15:53:14.371 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:53:14.371 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -13.1%) | 20260506 15:53:14.371 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:53:14.371 | INFO | High Water Breakdown: | 20260506 15:53:14.371 | INFO | 61.1% : 9.4 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:53:14.371 | INFO | 38.8% : 5.97 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:53:14.371 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:53:14.371 | INFO | ------------------------------------------ | 20260506 15:53:14.371 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:53:14.371 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:53:14.371 | INFO | Total mem usage: 26.5 GiB | 20260506 15:53:14.371 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:53:14.371 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:53:14.371 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 15:53:14.371 | INFO | High Water Breakdown: | 20260506 15:53:14.371 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:53:14.371 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:53:14.371 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:53:14.371 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:53:14.371 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:53:14.371 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:53:14.371 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:53:14.371 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:53:14.371 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:53:14.371 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:53:14.371 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:53:14.371 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:53:14.371 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:53:14.371 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:53:14.371 | INFO | ------------------------------------------ | 20260506 15:53:14.371 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:53:14.371 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:53:14.371 | INFO | Total mem usage: 25.6 GiB | 20260506 15:53:14.371 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:53:14.371 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:53:14.371 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:53:14.371 | INFO | High Water Breakdown: | 20260506 15:53:14.371 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:53:14.371 | INFO | ------------------------------------------ | 20260506 15:53:14.371 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:53:14.371 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:53:14.371 | INFO | Total mem usage: 15.7 GiB | 20260506 15:53:14.371 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:53:14.371 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:53:14.371 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 15:53:14.371 | INFO | High Water Breakdown: | 20260506 15:53:14.371 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:53:14.371 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:53:14.371 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:53:14.371 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:53:14.371 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:53:14.371 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:53:14.371 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:53:14.371 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:53:14.371 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:53:14.371 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:53:14.371 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:53:14.371 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:53:14.371 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:53:14.371 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:53:14.371 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:53:14.371 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:53:14.371 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:53:14.371 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:53:14.371 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:53:14.371 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:53:14.371 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:53:14.371 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:53:14.371 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:53:14.371 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:53:14.371 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:53:14.371 | INFO | ------------------------------------------ | 20260506 15:53:14.371 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:53:14.371 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:53:14.371 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:53:14.371 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:53:14.371 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.54e+01, 1.35e+01 / 1.71e+00 | 20260506 15:53:14.371 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.00e-01, 1.11e-02 / 6.67e-03 | 20260506 15:53:14.371 | INFO | PerfReport: 11.59%, Baseline -- 3.64e+00 / 4.36e+00, 5.65e-01 / 4.01e-01 | 20260506 15:53:14.371 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.87e-01 / 1.45e-01 | 20260506 15:53:14.371 | INFO | PerfReport: 15.82%, DME -- 3.98e+01 / 4.36e+01, 3.74e+00 / 1.55e+00 | 20260506 15:53:14.371 | INFO | PerfReport: 56.97%, FrameLabeling -- 1.79e+01 / 2.11e+01, 1.52e+00 / 1.54e+00 | 20260506 15:53:14.372 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.31e+00 / 2.84e+00, 3.89e-01 / 2.76e-01 | 20260506 15:53:14.372 | INFO | PerfReport: 3.52%, Metrics -- 1.11e+00 / 1.46e+00, 2.91e-01 / 1.76e-01 | 20260506 15:53:14.372 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.85e-01 / 4.33e-01, 6.01e-02 / 2.62e-02 | 20260506 15:53:14.372 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:53:14.372 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:53:14.372 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:53:14.372 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 15:53:14.372 | INFO | PerfReport: 21.48%, Upload -- 2.09e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260506 15:53:14.372 | INFO | PerfReport: 21.17%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 15:53:14.372 | INFO | PerfReport: 32.29%, Compute -- 3.15e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 15:53:14.372 | INFO | PerfReport: 1.42%, LockUpload -- 1.38e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260506 15:53:14.372 | INFO | PerfReport: 1.13%, LockDownload -- 1.10e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260506 15:53:14.372 | INFO | PerfReport: 22.50%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 15:53:14.372 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.37e-02 / 0.00e+00 | 20260506 15:53:14.484 | INFO | Starting a round of Baz IO | 20260506 15:53:17.635 | INFO | Populating RTMetrics report | 20260506 15:53:17.635 | INFO | RTMetrics report populated | 20260506 15:53:17.635 | INFO | Opening tmp RTMetrics json file | 20260506 15:53:17.643 | INFO | Serializing RTMetrics to json | 20260506 15:53:17.646 | INFO | Serialization done, swapping tmp file now... | 20260506 15:53:17.651 | INFO | RTMetrics done | 20260506 15:53:18.093 | INFO | Finished a round of Baz IO: Wrote 2.093 GiB in 3.609 seconds | 20260506 15:53:18.094 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.80% / 0.01% / 63.19% | 20260506 15:53:18.094 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.88192% / 1.19209e-05% / 6.75519e-05% | 20260506 15:53:18.094 | INFO | Baz IO overall: | 20260506 15:53:18.094 | INFO | BytesWritten : 1349857384115 | 20260506 15:53:18.094 | INFO | NumEvents : 653239292652 | 20260506 15:53:19.512 | INFO | Analyzing chunk frames = [2580480,2580992) | 20260506 15:53:19.512 | INFO | MemInfo: { | 20260506 15:53:19.512 | INFO | "gpu" : " 17 GB", | 20260506 15:53:19.512 | INFO | "huge" : "", | 20260506 15:53:19.512 | INFO | "huge_shared" : " 27 GB", | 20260506 15:53:19.512 | INFO | "malloc" : " 14 GB", | 20260506 15:53:19.512 | INFO | "pinned" : " 29 GB" | 20260506 15:53:19.512 | INFO | } | 20260506 15:53:19.512 | INFO | Perf report for the last 5.14207 seconds: | 20260506 15:53:19.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:53:19.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:19.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:19.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:19.513 | INFO | ANALYSIS : 62.9%, 2.9, 95.7ms, 0 | 20260506 15:53:19.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:19.513 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 15:53:19.513 | INFO | PRE_HQ : 60.8%, 1.9, 58.5ms, 0 | 20260506 15:53:19.513 | INFO | BAZWRITER : 69.6%, 4.8, 172.0ms, 0 | 20260506 15:53:19.513 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 15:53:19.513 | INFO | Uploaded 36GiB to GPU in 2.05781s (17.4944GiB/s) | 20260506 15:53:19.513 | INFO | Downloaded 38GiB to GPU in 2.231s (17.0327GiB/s) | 20260506 15:53:24.649 | INFO | Analyzing chunk frames = [2580992,2581504) | 20260506 15:53:24.649 | INFO | MemInfo: { | 20260506 15:53:24.649 | INFO | "gpu" : " 17 GB", | 20260506 15:53:24.649 | INFO | "huge" : "", | 20260506 15:53:24.649 | INFO | "huge_shared" : " 27 GB", | 20260506 15:53:24.649 | INFO | "malloc" : " 14 GB", | 20260506 15:53:24.649 | INFO | "pinned" : " 29 GB" | 20260506 15:53:24.649 | INFO | } | 20260506 15:53:24.649 | INFO | Perf report for the last 5.13678 seconds: | 20260506 15:53:24.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:53:24.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:24.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:24.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:24.649 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 15:53:24.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:24.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:24.649 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 15:53:24.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:24.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:24.649 | INFO | Uploaded 36GiB to GPU in 1.98256s (18.1583GiB/s) | 20260506 15:53:24.649 | INFO | Downloaded 34GiB to GPU in 1.93958s (17.5296GiB/s) | 20260506 15:53:29.788 | INFO | Analyzing chunk frames = [2581504,2582016) | 20260506 15:53:29.788 | INFO | MemInfo: { | 20260506 15:53:29.788 | INFO | "gpu" : " 17 GB", | 20260506 15:53:29.788 | INFO | "huge" : "", | 20260506 15:53:29.788 | INFO | "huge_shared" : " 27 GB", | 20260506 15:53:29.788 | INFO | "malloc" : " 14 GB", | 20260506 15:53:29.788 | INFO | "pinned" : " 29 GB" | 20260506 15:53:29.788 | INFO | } | 20260506 15:53:29.788 | INFO | Perf report for the last 5.13879 seconds: | 20260506 15:53:29.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:53:29.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:29.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:29.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:29.788 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 15:53:29.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:29.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:29.788 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:53:29.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:29.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:29.789 | INFO | Uploaded 36GiB to GPU in 1.97184s (18.257GiB/s) | 20260506 15:53:29.789 | INFO | Downloaded 34GiB to GPU in 1.93562s (17.5654GiB/s) | 20260506 15:53:34.928 | INFO | Analyzing chunk frames = [2582016,2582528) | 20260506 15:53:34.928 | INFO | MemInfo: { | 20260506 15:53:34.928 | INFO | "gpu" : " 17 GB", | 20260506 15:53:34.928 | INFO | "huge" : "", | 20260506 15:53:34.928 | INFO | "huge_shared" : " 27 GB", | 20260506 15:53:34.928 | INFO | "malloc" : " 14 GB", | 20260506 15:53:34.928 | INFO | "pinned" : " 29 GB" | 20260506 15:53:34.928 | INFO | } | 20260506 15:53:34.928 | INFO | Perf report for the last 5.13965 seconds: | 20260506 15:53:34.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:53:34.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:34.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:34.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:34.928 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260506 15:53:34.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:34.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:34.928 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:53:34.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:34.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:34.928 | INFO | Uploaded 36GiB to GPU in 2.00614s (17.9449GiB/s) | 20260506 15:53:34.928 | INFO | Downloaded 34GiB to GPU in 1.91088s (17.7929GiB/s) | 20260506 15:53:40.067 | INFO | Analyzing chunk frames = [2582528,2583040) | 20260506 15:53:40.067 | INFO | MemInfo: { | 20260506 15:53:40.067 | INFO | "gpu" : " 17 GB", | 20260506 15:53:40.067 | INFO | "huge" : "", | 20260506 15:53:40.067 | INFO | "huge_shared" : " 27 GB", | 20260506 15:53:40.067 | INFO | "malloc" : " 14 GB", | 20260506 15:53:40.067 | INFO | "pinned" : " 29 GB" | 20260506 15:53:40.067 | INFO | } | 20260506 15:53:40.068 | INFO | Perf report for the last 5.1399 seconds: | 20260506 15:53:40.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:53:40.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:40.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:40.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:40.068 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 15:53:40.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:40.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:40.068 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:53:40.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:40.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:40.068 | INFO | Uploaded 36GiB to GPU in 2.00942s (17.9156GiB/s) | 20260506 15:53:40.068 | INFO | Downloaded 34GiB to GPU in 1.93892s (17.5355GiB/s) | 20260506 15:53:45.207 | INFO | Analyzing chunk frames = [2583040,2583552) | 20260506 15:53:45.207 | INFO | MemInfo: { | 20260506 15:53:45.207 | INFO | "gpu" : " 17 GB", | 20260506 15:53:45.207 | INFO | "huge" : "", | 20260506 15:53:45.208 | INFO | "huge_shared" : " 27 GB", | 20260506 15:53:45.208 | INFO | "malloc" : " 14 GB", | 20260506 15:53:45.208 | INFO | "pinned" : " 29 GB" | 20260506 15:53:45.208 | INFO | } | 20260506 15:53:45.208 | INFO | Perf report for the last 5.14026 seconds: | 20260506 15:53:45.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:53:45.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:45.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:45.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:45.208 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:53:45.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:45.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:45.208 | INFO | PRE_HQ : 29.5%, 1.0, 15.4ms, 0 | 20260506 15:53:45.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:45.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:45.208 | INFO | Uploaded 36GiB to GPU in 2.06405s (17.4414GiB/s) | 20260506 15:53:45.208 | INFO | Downloaded 34GiB to GPU in 1.98057s (17.1668GiB/s) | 20260506 15:53:50.347 | INFO | Analyzing chunk frames = [2583552,2584064) | 20260506 15:53:50.347 | INFO | MemInfo: { | 20260506 15:53:50.347 | INFO | "gpu" : " 17 GB", | 20260506 15:53:50.347 | INFO | "huge" : "", | 20260506 15:53:50.347 | INFO | "huge_shared" : " 27 GB", | 20260506 15:53:50.347 | INFO | "malloc" : " 14 GB", | 20260506 15:53:50.347 | INFO | "pinned" : " 29 GB" | 20260506 15:53:50.347 | INFO | } | 20260506 15:53:50.347 | INFO | Perf report for the last 5.13939 seconds: | 20260506 15:53:50.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:53:50.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:50.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:50.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:50.347 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 15:53:50.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:50.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:50.347 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 15:53:50.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:50.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:50.347 | INFO | Uploaded 36GiB to GPU in 1.97279s (18.2482GiB/s) | 20260506 15:53:50.348 | INFO | Downloaded 34GiB to GPU in 2.00076s (16.9935GiB/s) | 20260506 15:53:55.485 | INFO | Analyzing chunk frames = [2584064,2584576) | 20260506 15:53:55.485 | INFO | MemInfo: { | 20260506 15:53:55.485 | INFO | "gpu" : " 17 GB", | 20260506 15:53:55.485 | INFO | "huge" : "", | 20260506 15:53:55.485 | INFO | "huge_shared" : " 27 GB", | 20260506 15:53:55.485 | INFO | "malloc" : " 14 GB", | 20260506 15:53:55.485 | INFO | "pinned" : " 29 GB" | 20260506 15:53:55.485 | INFO | } | 20260506 15:53:55.485 | INFO | Perf report for the last 5.13782 seconds: | 20260506 15:53:55.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:53:55.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:55.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:55.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:55.485 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 15:53:55.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:55.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:55.485 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:53:55.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:55.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:53:55.485 | INFO | Uploaded 36GiB to GPU in 1.99551s (18.0405GiB/s) | 20260506 15:53:55.485 | INFO | Downloaded 34GiB to GPU in 1.97369s (17.2266GiB/s) | 20260506 15:53:55.600 | INFO | Starting a round of Baz IO | 20260506 15:53:58.746 | INFO | Populating RTMetrics report | 20260506 15:53:58.747 | INFO | RTMetrics report populated | 20260506 15:53:58.747 | INFO | Opening tmp RTMetrics json file | 20260506 15:53:58.763 | INFO | Serializing RTMetrics to json | 20260506 15:53:58.795 | INFO | Serialization done, swapping tmp file now... | 20260506 15:53:58.796 | INFO | RTMetrics done | 20260506 15:53:58.885 | INFO | Finished a round of Baz IO: Wrote 2.095 GiB in 3.285 seconds | 20260506 15:53:58.885 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.79% / 0.01% / 63.20% | 20260506 15:53:58.885 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.88038% / 1.98682e-05% / 5.5631e-05% | 20260506 15:53:58.885 | INFO | Baz IO overall: | 20260506 15:53:58.885 | INFO | BytesWritten : 1352107221235 | 20260506 15:53:58.885 | INFO | NumEvents : 654328343994 | 20260506 15:54:00.624 | INFO | Analyzing chunk frames = [2584576,2585088) | 20260506 15:54:00.624 | INFO | MemInfo: { | 20260506 15:54:00.624 | INFO | "gpu" : " 17 GB", | 20260506 15:54:00.624 | INFO | "huge" : "", | 20260506 15:54:00.625 | INFO | "huge_shared" : " 27 GB", | 20260506 15:54:00.625 | INFO | "malloc" : " 14 GB", | 20260506 15:54:00.625 | INFO | "pinned" : " 29 GB" | 20260506 15:54:00.625 | INFO | } | 20260506 15:54:00.625 | INFO | Perf report for the last 5.13961 seconds: | 20260506 15:54:00.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:54:00.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:00.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:00.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:00.625 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 15:54:00.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:00.625 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 15:54:00.625 | INFO | PRE_HQ : 60.6%, 1.9, 58.7ms, 0 | 20260506 15:54:00.625 | INFO | BAZWRITER : 63.6%, 4.8, 156.4ms, 0 | 20260506 15:54:00.625 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 15:54:00.625 | INFO | Uploaded 36GiB to GPU in 2.05673s (17.5035GiB/s) | 20260506 15:54:00.625 | INFO | Downloaded 38GiB to GPU in 2.21739s (17.1372GiB/s) | 20260506 15:54:05.764 | INFO | Analyzing chunk frames = [2585088,2585600) | 20260506 15:54:05.764 | INFO | MemInfo: { | 20260506 15:54:05.764 | INFO | "gpu" : " 17 GB", | 20260506 15:54:05.764 | INFO | "huge" : "", | 20260506 15:54:05.764 | INFO | "huge_shared" : " 27 GB", | 20260506 15:54:05.764 | INFO | "malloc" : " 14 GB", | 20260506 15:54:05.764 | INFO | "pinned" : " 29 GB" | 20260506 15:54:05.764 | INFO | } | 20260506 15:54:05.764 | INFO | Perf report for the last 5.13975 seconds: | 20260506 15:54:05.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:54:05.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:05.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:05.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:05.764 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:54:05.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:05.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:05.765 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:54:05.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:05.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:05.765 | INFO | Uploaded 36GiB to GPU in 1.99691s (18.0279GiB/s) | 20260506 15:54:05.765 | INFO | Downloaded 34GiB to GPU in 1.93351s (17.5846GiB/s) | 20260506 15:54:10.903 | INFO | Analyzing chunk frames = [2585600,2586112) | 20260506 15:54:10.903 | INFO | MemInfo: { | 20260506 15:54:10.903 | INFO | "gpu" : " 17 GB", | 20260506 15:54:10.904 | INFO | "huge" : "", | 20260506 15:54:10.904 | INFO | "huge_shared" : " 27 GB", | 20260506 15:54:10.904 | INFO | "malloc" : " 14 GB", | 20260506 15:54:10.904 | INFO | "pinned" : " 29 GB" | 20260506 15:54:10.904 | INFO | } | 20260506 15:54:10.904 | INFO | Perf report for the last 5.13929 seconds: | 20260506 15:54:10.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:54:10.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:10.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:10.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:10.904 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:54:10.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:10.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:10.904 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:54:10.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:10.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:10.904 | INFO | Uploaded 36GiB to GPU in 1.99854s (18.0131GiB/s) | 20260506 15:54:10.904 | INFO | Downloaded 34GiB to GPU in 1.93535s (17.5678GiB/s) | 20260506 15:54:16.042 | INFO | Analyzing chunk frames = [2586112,2586624) | 20260506 15:54:16.042 | INFO | MemInfo: { | 20260506 15:54:16.042 | INFO | "gpu" : " 17 GB", | 20260506 15:54:16.042 | INFO | "huge" : "", | 20260506 15:54:16.042 | INFO | "huge_shared" : " 27 GB", | 20260506 15:54:16.043 | INFO | "malloc" : " 14 GB", | 20260506 15:54:16.043 | INFO | "pinned" : " 29 GB" | 20260506 15:54:16.043 | INFO | } | 20260506 15:54:16.043 | INFO | Perf report for the last 5.13888 seconds: | 20260506 15:54:16.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:54:16.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:16.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:16.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:16.043 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 15:54:16.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:16.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:16.043 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:54:16.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:16.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:16.043 | INFO | Uploaded 36GiB to GPU in 2.02095s (17.8134GiB/s) | 20260506 15:54:16.043 | INFO | Downloaded 34GiB to GPU in 1.91812s (17.7257GiB/s) | 20260506 15:54:21.182 | INFO | Analyzing chunk frames = [2586624,2587136) | 20260506 15:54:21.182 | INFO | MemInfo: { | 20260506 15:54:21.182 | INFO | "gpu" : " 17 GB", | 20260506 15:54:21.182 | INFO | "huge" : "", | 20260506 15:54:21.182 | INFO | "huge_shared" : " 27 GB", | 20260506 15:54:21.182 | INFO | "malloc" : " 14 GB", | 20260506 15:54:21.182 | INFO | "pinned" : " 29 GB" | 20260506 15:54:21.182 | INFO | } | 20260506 15:54:21.182 | INFO | Perf report for the last 5.13947 seconds: | 20260506 15:54:21.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:54:21.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:21.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:21.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:21.182 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 15:54:21.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:21.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:21.182 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:54:21.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:21.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:21.182 | INFO | Uploaded 36GiB to GPU in 2.04217s (17.6283GiB/s) | 20260506 15:54:21.182 | INFO | Downloaded 34GiB to GPU in 1.92194s (17.6905GiB/s) | 20260506 15:54:26.321 | INFO | Analyzing chunk frames = [2587136,2587648) | 20260506 15:54:26.321 | INFO | MemInfo: { | 20260506 15:54:26.321 | INFO | "gpu" : " 17 GB", | 20260506 15:54:26.321 | INFO | "huge" : "", | 20260506 15:54:26.321 | INFO | "huge_shared" : " 27 GB", | 20260506 15:54:26.321 | INFO | "malloc" : " 14 GB", | 20260506 15:54:26.321 | INFO | "pinned" : " 29 GB" | 20260506 15:54:26.321 | INFO | } | 20260506 15:54:26.322 | INFO | Perf report for the last 5.13944 seconds: | 20260506 15:54:26.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:54:26.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:26.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:26.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:26.322 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 15:54:26.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:26.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:26.322 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260506 15:54:26.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:26.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:26.322 | INFO | Uploaded 36GiB to GPU in 2.07134s (17.3801GiB/s) | 20260506 15:54:26.322 | INFO | Downloaded 34GiB to GPU in 1.97284s (17.234GiB/s) | 20260506 15:54:31.462 | INFO | Analyzing chunk frames = [2587648,2588160) | 20260506 15:54:31.462 | INFO | MemInfo: { | 20260506 15:54:31.462 | INFO | "gpu" : " 17 GB", | 20260506 15:54:31.462 | INFO | "huge" : "", | 20260506 15:54:31.462 | INFO | "huge_shared" : " 27 GB", | 20260506 15:54:31.462 | INFO | "malloc" : " 14 GB", | 20260506 15:54:31.462 | INFO | "pinned" : " 29 GB" | 20260506 15:54:31.462 | INFO | } | 20260506 15:54:31.462 | INFO | Perf report for the last 5.1403 seconds: | 20260506 15:54:31.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:54:31.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:31.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:31.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:31.462 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 15:54:31.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:31.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:31.462 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 15:54:31.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:31.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:31.462 | INFO | Uploaded 36GiB to GPU in 2.04983s (17.5625GiB/s) | 20260506 15:54:31.462 | INFO | Downloaded 34GiB to GPU in 1.99285s (17.061GiB/s) | 20260506 15:54:36.599 | INFO | Analyzing chunk frames = [2588160,2588672) | 20260506 15:54:36.599 | INFO | MemInfo: { | 20260506 15:54:36.599 | INFO | "gpu" : " 17 GB", | 20260506 15:54:36.599 | INFO | "huge" : "", | 20260506 15:54:36.599 | INFO | "huge_shared" : " 27 GB", | 20260506 15:54:36.599 | INFO | "malloc" : " 14 GB", | 20260506 15:54:36.599 | INFO | "pinned" : " 29 GB" | 20260506 15:54:36.599 | INFO | } | 20260506 15:54:36.599 | INFO | Perf report for the last 5.13752 seconds: | 20260506 15:54:36.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:54:36.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:36.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:36.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:36.599 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 15:54:36.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:36.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:36.600 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:54:36.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:36.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:36.600 | INFO | Uploaded 36GiB to GPU in 1.98034s (18.1787GiB/s) | 20260506 15:54:36.600 | INFO | Downloaded 34GiB to GPU in 1.97395s (17.2243GiB/s) | 20260506 15:54:36.600 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:54:36.600 | INFO | Total mem usage: 17.4 GiB | 20260506 15:54:36.600 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:54:36.600 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -9.98%) | 20260506 15:54:36.600 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:54:36.600 | INFO | High Water Breakdown: | 20260506 15:54:36.600 | INFO | 61.5% : 9.73 GiB ( 89.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:54:36.600 | INFO | 38.4% : 6.08 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:54:36.600 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:54:36.600 | INFO | ------------------------------------------ | 20260506 15:54:36.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:54:36.600 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:54:36.600 | INFO | Total mem usage: 26.5 GiB | 20260506 15:54:36.600 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:54:36.600 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:54:36.600 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 15:54:36.600 | INFO | High Water Breakdown: | 20260506 15:54:36.600 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:54:36.600 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:54:36.600 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:54:36.600 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:54:36.600 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:54:36.600 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:54:36.600 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:54:36.600 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:54:36.600 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:54:36.600 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:54:36.600 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:54:36.600 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:54:36.600 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:54:36.600 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:54:36.600 | INFO | ------------------------------------------ | 20260506 15:54:36.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:54:36.600 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:54:36.600 | INFO | Total mem usage: 25.6 GiB | 20260506 15:54:36.600 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:54:36.600 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:54:36.600 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:54:36.600 | INFO | High Water Breakdown: | 20260506 15:54:36.600 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:54:36.600 | INFO | ------------------------------------------ | 20260506 15:54:36.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:54:36.600 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:54:36.600 | INFO | Total mem usage: 15.7 GiB | 20260506 15:54:36.600 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:54:36.600 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:54:36.600 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 15:54:36.600 | INFO | High Water Breakdown: | 20260506 15:54:36.600 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:54:36.600 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:54:36.600 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:54:36.600 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:54:36.600 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:54:36.600 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:54:36.600 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:54:36.600 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:54:36.600 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:54:36.600 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:54:36.600 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:54:36.600 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:54:36.600 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:54:36.600 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:54:36.600 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:54:36.600 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:54:36.600 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:54:36.600 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:54:36.600 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:54:36.600 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:54:36.600 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:54:36.600 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:54:36.600 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:54:36.600 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:54:36.600 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:54:36.601 | INFO | ------------------------------------------ | 20260506 15:54:36.601 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:54:36.601 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:54:36.601 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:54:36.601 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:54:36.601 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.56e+01, 1.35e+01 / 1.74e+00 | 20260506 15:54:36.601 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.04e-01, 1.15e-02 / 2.08e-02 | 20260506 15:54:36.601 | INFO | PerfReport: 11.57%, Baseline -- 3.64e+00 / 4.39e+00, 5.58e-01 / 3.97e-01 | 20260506 15:54:36.601 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.46e+00, 1.87e-01 / 1.60e-01 | 20260506 15:54:36.601 | INFO | PerfReport: 15.88%, DME -- 3.99e+01 / 4.41e+01, 3.70e+00 / 1.92e+00 | 20260506 15:54:36.601 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.49e+00 / 1.71e+00 | 20260506 15:54:36.601 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.31e+00 / 2.92e+00, 3.96e-01 / 2.29e-01 | 20260506 15:54:36.601 | INFO | PerfReport: 3.53%, Metrics -- 1.11e+00 / 1.48e+00, 2.86e-01 / 1.64e-01 | 20260506 15:54:36.601 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.86e-01 / 4.34e-01, 6.05e-02 / 2.23e-02 | 20260506 15:54:36.601 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:54:36.601 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:54:36.601 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:54:36.601 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 15:54:36.601 | INFO | PerfReport: 21.55%, Upload -- 2.10e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260506 15:54:36.601 | INFO | PerfReport: 21.22%, Download -- 2.07e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 15:54:36.601 | INFO | PerfReport: 32.24%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 15:54:36.601 | INFO | PerfReport: 1.45%, LockUpload -- 1.41e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260506 15:54:36.601 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.68e+00 / 0.00e+00 | 20260506 15:54:36.601 | INFO | PerfReport: 22.44%, LockCompute -- 2.19e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 15:54:36.601 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260506 15:54:36.713 | INFO | Starting a round of Baz IO | 20260506 15:54:39.862 | INFO | Populating RTMetrics report | 20260506 15:54:39.862 | INFO | RTMetrics report populated | 20260506 15:54:39.862 | INFO | Opening tmp RTMetrics json file | 20260506 15:54:39.937 | INFO | Serializing RTMetrics to json | 20260506 15:54:39.984 | INFO | Serialization done, swapping tmp file now... | 20260506 15:54:39.988 | INFO | RTMetrics done | 20260506 15:54:40.225 | INFO | Finished a round of Baz IO: Wrote 2.095 GiB in 3.512 seconds | 20260506 15:54:40.225 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.78% / 0.01% / 63.21% | 20260506 15:54:40.225 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.87312% / 1.98682e-05% / 4.76837e-05% | 20260506 15:54:40.225 | INFO | Baz IO overall: | 20260506 15:54:40.225 | INFO | BytesWritten : 1354356196169 | 20260506 15:54:40.225 | INFO | NumEvents : 655416843863 | 20260506 15:54:41.739 | INFO | Analyzing chunk frames = [2588672,2589184) | 20260506 15:54:41.739 | INFO | MemInfo: { | 20260506 15:54:41.739 | INFO | "gpu" : " 17 GB", | 20260506 15:54:41.739 | INFO | "huge" : "", | 20260506 15:54:41.739 | INFO | "huge_shared" : " 27 GB", | 20260506 15:54:41.739 | INFO | "malloc" : " 14 GB", | 20260506 15:54:41.739 | INFO | "pinned" : " 29 GB" | 20260506 15:54:41.739 | INFO | } | 20260506 15:54:41.739 | INFO | Perf report for the last 5.13975 seconds: | 20260506 15:54:41.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:54:41.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:41.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:41.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:41.739 | INFO | ANALYSIS : 62.9%, 2.9, 95.6ms, 0 | 20260506 15:54:41.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:41.739 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260506 15:54:41.739 | INFO | PRE_HQ : 60.6%, 1.9, 58.7ms, 0 | 20260506 15:54:41.739 | INFO | BAZWRITER : 68.0%, 4.8, 166.8ms, 0 | 20260506 15:54:41.739 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.9ms, 0 | 20260506 15:54:41.740 | INFO | Uploaded 36GiB to GPU in 2.05338s (17.532GiB/s) | 20260506 15:54:41.740 | INFO | Downloaded 38GiB to GPU in 2.22712s (17.0624GiB/s) | 20260506 15:54:46.879 | INFO | Analyzing chunk frames = [2589184,2589696) | 20260506 15:54:46.879 | INFO | MemInfo: { | 20260506 15:54:46.879 | INFO | "gpu" : " 17 GB", | 20260506 15:54:46.879 | INFO | "huge" : "", | 20260506 15:54:46.879 | INFO | "huge_shared" : " 27 GB", | 20260506 15:54:46.879 | INFO | "malloc" : " 14 GB", | 20260506 15:54:46.879 | INFO | "pinned" : " 29 GB" | 20260506 15:54:46.879 | INFO | } | 20260506 15:54:46.879 | INFO | Perf report for the last 5.13968 seconds: | 20260506 15:54:46.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:54:46.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:46.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:46.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:46.879 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 15:54:46.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:46.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:46.879 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 15:54:46.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:46.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:46.879 | INFO | Uploaded 36GiB to GPU in 1.99159s (18.076GiB/s) | 20260506 15:54:46.879 | INFO | Downloaded 34GiB to GPU in 1.9384s (17.5402GiB/s) | 20260506 15:54:52.018 | INFO | Analyzing chunk frames = [2589696,2590208) | 20260506 15:54:52.018 | INFO | MemInfo: { | 20260506 15:54:52.018 | INFO | "gpu" : " 17 GB", | 20260506 15:54:52.018 | INFO | "huge" : "", | 20260506 15:54:52.018 | INFO | "huge_shared" : " 27 GB", | 20260506 15:54:52.018 | INFO | "malloc" : " 14 GB", | 20260506 15:54:52.018 | INFO | "pinned" : " 29 GB" | 20260506 15:54:52.018 | INFO | } | 20260506 15:54:52.018 | INFO | Perf report for the last 5.13942 seconds: | 20260506 15:54:52.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:54:52.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:52.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:52.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:52.018 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 15:54:52.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:52.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:52.018 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:54:52.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:52.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:52.018 | INFO | Uploaded 36GiB to GPU in 1.96771s (18.2954GiB/s) | 20260506 15:54:52.019 | INFO | Downloaded 34GiB to GPU in 1.93346s (17.5851GiB/s) | 20260506 15:54:57.158 | INFO | Analyzing chunk frames = [2590208,2590720) | 20260506 15:54:57.158 | INFO | MemInfo: { | 20260506 15:54:57.158 | INFO | "gpu" : " 17 GB", | 20260506 15:54:57.158 | INFO | "huge" : "", | 20260506 15:54:57.158 | INFO | "huge_shared" : " 27 GB", | 20260506 15:54:57.158 | INFO | "malloc" : " 14 GB", | 20260506 15:54:57.158 | INFO | "pinned" : " 29 GB" | 20260506 15:54:57.158 | INFO | } | 20260506 15:54:57.158 | INFO | Perf report for the last 5.13997 seconds: | 20260506 15:54:57.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:54:57.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:57.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:57.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:57.158 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 15:54:57.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:57.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:57.158 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:54:57.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:57.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:54:57.158 | INFO | Uploaded 36GiB to GPU in 1.9358s (18.597GiB/s) | 20260506 15:54:57.159 | INFO | Downloaded 34GiB to GPU in 1.88168s (18.069GiB/s) | 20260506 15:55:02.296 | INFO | Analyzing chunk frames = [2590720,2591232) | 20260506 15:55:02.296 | INFO | MemInfo: { | 20260506 15:55:02.296 | INFO | "gpu" : " 17 GB", | 20260506 15:55:02.296 | INFO | "huge" : "", | 20260506 15:55:02.297 | INFO | "huge_shared" : " 27 GB", | 20260506 15:55:02.297 | INFO | "malloc" : " 14 GB", | 20260506 15:55:02.297 | INFO | "pinned" : " 29 GB" | 20260506 15:55:02.297 | INFO | } | 20260506 15:55:02.297 | INFO | Perf report for the last 5.13843 seconds: | 20260506 15:55:02.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:55:02.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:02.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:02.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:02.297 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 15:55:02.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:02.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:02.297 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 15:55:02.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:02.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:02.297 | INFO | Uploaded 36GiB to GPU in 1.98134s (18.1695GiB/s) | 20260506 15:55:02.297 | INFO | Downloaded 34GiB to GPU in 1.92503s (17.662GiB/s) | 20260506 15:55:07.435 | INFO | Analyzing chunk frames = [2591232,2591744) | 20260506 15:55:07.435 | INFO | MemInfo: { | 20260506 15:55:07.435 | INFO | "gpu" : " 17 GB", | 20260506 15:55:07.435 | INFO | "huge" : "", | 20260506 15:55:07.435 | INFO | "huge_shared" : " 27 GB", | 20260506 15:55:07.435 | INFO | "malloc" : " 14 GB", | 20260506 15:55:07.435 | INFO | "pinned" : " 29 GB" | 20260506 15:55:07.435 | INFO | } | 20260506 15:55:07.436 | INFO | Perf report for the last 5.13886 seconds: | 20260506 15:55:07.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:55:07.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:07.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:07.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:07.436 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:55:07.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:07.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:07.436 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 15:55:07.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:07.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:07.436 | INFO | Uploaded 36GiB to GPU in 2.06423s (17.4399GiB/s) | 20260506 15:55:07.436 | INFO | Downloaded 34GiB to GPU in 1.97685s (17.1991GiB/s) | 20260506 15:55:12.576 | INFO | Analyzing chunk frames = [2591744,2592256) | 20260506 15:55:12.576 | INFO | MemInfo: { | 20260506 15:55:12.576 | INFO | "gpu" : " 17 GB", | 20260506 15:55:12.576 | INFO | "huge" : "", | 20260506 15:55:12.576 | INFO | "huge_shared" : " 27 GB", | 20260506 15:55:12.576 | INFO | "malloc" : " 14 GB", | 20260506 15:55:12.576 | INFO | "pinned" : " 29 GB" | 20260506 15:55:12.576 | INFO | } | 20260506 15:55:12.576 | INFO | Perf report for the last 5.14006 seconds: | 20260506 15:55:12.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:55:12.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:12.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:12.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:12.576 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 15:55:12.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:12.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:12.576 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 15:55:12.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:12.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:12.576 | INFO | Uploaded 36GiB to GPU in 1.99881s (18.0107GiB/s) | 20260506 15:55:12.576 | INFO | Downloaded 34GiB to GPU in 1.99559s (17.0376GiB/s) | 20260506 15:55:17.714 | INFO | Analyzing chunk frames = [2592256,2592768) | 20260506 15:55:17.714 | INFO | MemInfo: { | 20260506 15:55:17.714 | INFO | "gpu" : " 17 GB", | 20260506 15:55:17.714 | INFO | "huge" : "", | 20260506 15:55:17.714 | INFO | "huge_shared" : " 27 GB", | 20260506 15:55:17.714 | INFO | "malloc" : " 14 GB", | 20260506 15:55:17.714 | INFO | "pinned" : " 29 GB" | 20260506 15:55:17.714 | INFO | } | 20260506 15:55:17.715 | INFO | Perf report for the last 5.13887 seconds: | 20260506 15:55:17.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:55:17.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:17.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:17.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:17.715 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 15:55:17.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:17.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:17.715 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:55:17.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:17.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:17.715 | INFO | Uploaded 36GiB to GPU in 1.98401s (18.1451GiB/s) | 20260506 15:55:17.715 | INFO | Downloaded 34GiB to GPU in 1.97576s (17.2086GiB/s) | 20260506 15:55:17.831 | INFO | Starting a round of Baz IO | 20260506 15:55:20.975 | INFO | Populating RTMetrics report | 20260506 15:55:20.975 | INFO | RTMetrics report populated | 20260506 15:55:20.975 | INFO | Opening tmp RTMetrics json file | 20260506 15:55:20.976 | INFO | Serializing RTMetrics to json | 20260506 15:55:20.978 | INFO | Serialization done, swapping tmp file now... | 20260506 15:55:20.978 | INFO | RTMetrics done | 20260506 15:55:21.110 | INFO | Finished a round of Baz IO: Wrote 2.094 GiB in 3.279 seconds | 20260506 15:55:21.110 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.77% / 0.01% / 63.22% | 20260506 15:55:21.110 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.86693% / 1.19209e-05% / 7.15256e-05% | 20260506 15:55:21.110 | INFO | Baz IO overall: | 20260506 15:55:21.110 | INFO | BytesWritten : 1356604150886 | 20260506 15:55:21.110 | INFO | NumEvents : 656504176620 | 20260506 15:55:22.854 | INFO | Analyzing chunk frames = [2592768,2593280) | 20260506 15:55:22.854 | INFO | MemInfo: { | 20260506 15:55:22.854 | INFO | "gpu" : " 17 GB", | 20260506 15:55:22.854 | INFO | "huge" : "", | 20260506 15:55:22.854 | INFO | "huge_shared" : " 27 GB", | 20260506 15:55:22.854 | INFO | "malloc" : " 14 GB", | 20260506 15:55:22.854 | INFO | "pinned" : " 29 GB" | 20260506 15:55:22.854 | INFO | } | 20260506 15:55:22.854 | INFO | Perf report for the last 5.13927 seconds: | 20260506 15:55:22.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:55:22.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:22.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:22.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:22.854 | INFO | ANALYSIS : 62.9%, 2.9, 95.3ms, 0 | 20260506 15:55:22.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:22.854 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 15:55:22.854 | INFO | PRE_HQ : 60.5%, 1.9, 58.8ms, 0 | 20260506 15:55:22.854 | INFO | BAZWRITER : 63.5%, 4.8, 155.2ms, 0 | 20260506 15:55:22.854 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 15:55:22.854 | INFO | Uploaded 36GiB to GPU in 2.09115s (17.2154GiB/s) | 20260506 15:55:22.854 | INFO | Downloaded 38GiB to GPU in 2.20968s (17.1971GiB/s) | 20260506 15:55:27.994 | INFO | Analyzing chunk frames = [2593280,2593792) | 20260506 15:55:27.994 | INFO | MemInfo: { | 20260506 15:55:27.994 | INFO | "gpu" : " 17 GB", | 20260506 15:55:27.994 | INFO | "huge" : "", | 20260506 15:55:27.994 | INFO | "huge_shared" : " 27 GB", | 20260506 15:55:27.994 | INFO | "malloc" : " 14 GB", | 20260506 15:55:27.994 | INFO | "pinned" : " 29 GB" | 20260506 15:55:27.994 | INFO | } | 20260506 15:55:27.994 | INFO | Perf report for the last 5.14039 seconds: | 20260506 15:55:27.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:55:27.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:27.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:27.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:27.994 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 15:55:27.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:27.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:27.994 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:55:27.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:27.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:27.995 | INFO | Uploaded 36GiB to GPU in 2.00321s (17.9711GiB/s) | 20260506 15:55:27.995 | INFO | Downloaded 34GiB to GPU in 1.94868s (17.4477GiB/s) | 20260506 15:55:33.133 | INFO | Analyzing chunk frames = [2593792,2594304) | 20260506 15:55:33.133 | INFO | MemInfo: { | 20260506 15:55:33.133 | INFO | "gpu" : " 17 GB", | 20260506 15:55:33.133 | INFO | "huge" : "", | 20260506 15:55:33.134 | INFO | "huge_shared" : " 27 GB", | 20260506 15:55:33.134 | INFO | "malloc" : " 14 GB", | 20260506 15:55:33.134 | INFO | "pinned" : " 29 GB" | 20260506 15:55:33.134 | INFO | } | 20260506 15:55:33.134 | INFO | Perf report for the last 5.13941 seconds: | 20260506 15:55:33.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:55:33.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:33.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:33.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:33.134 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 15:55:33.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:33.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:33.134 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:55:33.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:33.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:33.134 | INFO | Uploaded 36GiB to GPU in 1.98735s (18.1146GiB/s) | 20260506 15:55:33.134 | INFO | Downloaded 34GiB to GPU in 1.93072s (17.61GiB/s) | 20260506 15:55:38.273 | INFO | Analyzing chunk frames = [2594304,2594816) | 20260506 15:55:38.273 | INFO | MemInfo: { | 20260506 15:55:38.273 | INFO | "gpu" : " 17 GB", | 20260506 15:55:38.273 | INFO | "huge" : "", | 20260506 15:55:38.273 | INFO | "huge_shared" : " 27 GB", | 20260506 15:55:38.273 | INFO | "malloc" : " 14 GB", | 20260506 15:55:38.273 | INFO | "pinned" : " 29 GB" | 20260506 15:55:38.273 | INFO | } | 20260506 15:55:38.273 | INFO | Perf report for the last 5.1392 seconds: | 20260506 15:55:38.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:55:38.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:38.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:38.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:38.273 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 15:55:38.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:38.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:38.273 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 15:55:38.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:38.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:38.273 | INFO | Uploaded 36GiB to GPU in 2.02049s (17.8175GiB/s) | 20260506 15:55:38.273 | INFO | Downloaded 34GiB to GPU in 1.90835s (17.8165GiB/s) | 20260506 15:55:43.411 | INFO | Analyzing chunk frames = [2594816,2595328) | 20260506 15:55:43.411 | INFO | MemInfo: { | 20260506 15:55:43.411 | INFO | "gpu" : " 17 GB", | 20260506 15:55:43.411 | INFO | "huge" : "", | 20260506 15:55:43.412 | INFO | "huge_shared" : " 27 GB", | 20260506 15:55:43.412 | INFO | "malloc" : " 14 GB", | 20260506 15:55:43.412 | INFO | "pinned" : " 29 GB" | 20260506 15:55:43.412 | INFO | } | 20260506 15:55:43.412 | INFO | Perf report for the last 5.13898 seconds: | 20260506 15:55:43.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:55:43.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:43.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:43.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:43.412 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 15:55:43.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:43.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:43.412 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:55:43.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:43.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:43.412 | INFO | Uploaded 36GiB to GPU in 2.01559s (17.8608GiB/s) | 20260506 15:55:43.412 | INFO | Downloaded 34GiB to GPU in 1.92681s (17.6457GiB/s) | 20260506 15:55:48.551 | INFO | Analyzing chunk frames = [2595328,2595840) | 20260506 15:55:48.551 | INFO | MemInfo: { | 20260506 15:55:48.551 | INFO | "gpu" : " 17 GB", | 20260506 15:55:48.551 | INFO | "huge" : "", | 20260506 15:55:48.551 | INFO | "huge_shared" : " 27 GB", | 20260506 15:55:48.551 | INFO | "malloc" : " 14 GB", | 20260506 15:55:48.551 | INFO | "pinned" : " 29 GB" | 20260506 15:55:48.551 | INFO | } | 20260506 15:55:48.551 | INFO | Perf report for the last 5.13897 seconds: | 20260506 15:55:48.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:55:48.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:48.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:48.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:48.551 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 15:55:48.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:48.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:48.551 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260506 15:55:48.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:48.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:48.551 | INFO | Uploaded 36GiB to GPU in 2.04266s (17.6241GiB/s) | 20260506 15:55:48.551 | INFO | Downloaded 34GiB to GPU in 1.915s (17.7546GiB/s) | 20260506 15:55:53.690 | INFO | Analyzing chunk frames = [2595840,2596352) | 20260506 15:55:53.690 | INFO | MemInfo: { | 20260506 15:55:53.690 | INFO | "gpu" : " 17 GB", | 20260506 15:55:53.690 | INFO | "huge" : "", | 20260506 15:55:53.690 | INFO | "huge_shared" : " 27 GB", | 20260506 15:55:53.690 | INFO | "malloc" : " 14 GB", | 20260506 15:55:53.690 | INFO | "pinned" : " 29 GB" | 20260506 15:55:53.690 | INFO | } | 20260506 15:55:53.690 | INFO | Perf report for the last 5.13961 seconds: | 20260506 15:55:53.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:55:53.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:53.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:53.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:53.691 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:55:53.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:53.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:53.691 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 15:55:53.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:53.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:53.691 | INFO | Uploaded 36GiB to GPU in 1.99483s (18.0466GiB/s) | 20260506 15:55:53.691 | INFO | Downloaded 34GiB to GPU in 1.96295s (17.3209GiB/s) | 20260506 15:55:58.829 | INFO | Analyzing chunk frames = [2596352,2596864) | 20260506 15:55:58.829 | INFO | MemInfo: { | 20260506 15:55:58.829 | INFO | "gpu" : " 17 GB", | 20260506 15:55:58.829 | INFO | "huge" : "", | 20260506 15:55:58.829 | INFO | "huge_shared" : " 27 GB", | 20260506 15:55:58.829 | INFO | "malloc" : " 14 GB", | 20260506 15:55:58.829 | INFO | "pinned" : " 29 GB" | 20260506 15:55:58.829 | INFO | } | 20260506 15:55:58.830 | INFO | Perf report for the last 5.13914 seconds: | 20260506 15:55:58.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:55:58.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:58.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:58.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:58.830 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:55:58.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:58.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:58.830 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:55:58.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:58.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:55:58.830 | INFO | Uploaded 36GiB to GPU in 2.01715s (17.847GiB/s) | 20260506 15:55:58.830 | INFO | Downloaded 34GiB to GPU in 1.98677s (17.1132GiB/s) | 20260506 15:55:58.830 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:55:58.830 | INFO | Total mem usage: 17.4 GiB | 20260506 15:55:58.830 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:55:58.830 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -10.8%) | 20260506 15:55:58.830 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:55:58.830 | INFO | High Water Breakdown: | 20260506 15:55:58.830 | INFO | 61.4% : 9.65 GiB ( 88.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:55:58.830 | INFO | 38.5% : 6.05 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:55:58.830 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:55:58.830 | INFO | ------------------------------------------ | 20260506 15:55:58.830 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:55:58.830 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:55:58.831 | INFO | Total mem usage: 26.5 GiB | 20260506 15:55:58.831 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:55:58.831 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:55:58.831 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 15:55:58.831 | INFO | High Water Breakdown: | 20260506 15:55:58.831 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:55:58.831 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:55:58.831 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:55:58.831 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:55:58.831 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:55:58.831 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:55:58.831 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:55:58.831 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:55:58.831 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:55:58.831 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:55:58.831 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:55:58.831 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:55:58.831 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:55:58.831 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:55:58.831 | INFO | ------------------------------------------ | 20260506 15:55:58.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:55:58.831 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:55:58.831 | INFO | Total mem usage: 25.6 GiB | 20260506 15:55:58.831 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:55:58.831 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:55:58.831 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:55:58.832 | INFO | High Water Breakdown: | 20260506 15:55:58.832 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:55:58.832 | INFO | ------------------------------------------ | 20260506 15:55:58.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:55:58.832 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:55:58.832 | INFO | Total mem usage: 15.7 GiB | 20260506 15:55:58.832 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:55:58.832 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:55:58.832 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 15:55:58.832 | INFO | High Water Breakdown: | 20260506 15:55:58.832 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:55:58.832 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:55:58.832 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:55:58.832 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:55:58.832 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:55:58.832 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:55:58.832 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:55:58.832 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:55:58.832 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:55:58.832 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:55:58.832 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:55:58.832 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:55:58.832 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:55:58.832 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:55:58.832 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:55:58.832 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:55:58.832 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:55:58.832 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:55:58.832 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:55:58.832 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:55:58.832 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:55:58.832 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:55:58.832 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:55:58.832 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:55:58.832 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:55:58.832 | INFO | ------------------------------------------ | 20260506 15:55:58.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:55:58.832 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:55:58.832 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:55:58.832 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:55:58.832 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.50e+01, 1.34e+01 / 1.69e+00 | 20260506 15:55:58.832 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.01e-01, 9.84e-03 / 6.35e-03 | 20260506 15:55:58.832 | INFO | PerfReport: 11.59%, Baseline -- 3.63e+00 / 4.38e+00, 5.56e-01 / 3.71e-01 | 20260506 15:55:58.832 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.85e-01 / 1.42e-01 | 20260506 15:55:58.832 | INFO | PerfReport: 15.83%, DME -- 3.97e+01 / 4.36e+01, 3.61e+00 / 1.62e+00 | 20260506 15:55:58.832 | INFO | PerfReport: 56.89%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.43e+00 / 1.47e+00 | 20260506 15:55:58.832 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.32e+00 / 2.91e+00, 4.01e-01 / 2.25e-01 | 20260506 15:55:58.832 | INFO | PerfReport: 3.53%, Metrics -- 1.11e+00 / 1.46e+00, 2.87e-01 / 1.71e-01 | 20260506 15:55:58.832 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.85e-01 / 4.31e-01, 5.94e-02 / 2.34e-02 | 20260506 15:55:58.832 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:55:58.832 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:55:58.832 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:55:58.832 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 15:55:58.832 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 7.04e+00 / 0.00e+00 | 20260506 15:55:58.832 | INFO | PerfReport: 21.16%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 15:55:58.832 | INFO | PerfReport: 32.23%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 15:55:58.832 | INFO | PerfReport: 1.44%, LockUpload -- 1.41e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260506 15:55:58.832 | INFO | PerfReport: 1.15%, LockDownload -- 1.12e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 15:55:58.832 | INFO | PerfReport: 22.50%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 15:55:58.832 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.50e-02 / 0.00e+00 | 20260506 15:55:58.942 | INFO | Starting a round of Baz IO | 20260506 15:56:02.086 | INFO | Populating RTMetrics report | 20260506 15:56:02.086 | INFO | RTMetrics report populated | 20260506 15:56:02.086 | INFO | Opening tmp RTMetrics json file | 20260506 15:56:02.116 | INFO | Serializing RTMetrics to json | 20260506 15:56:02.152 | INFO | Serialization done, swapping tmp file now... | 20260506 15:56:02.191 | INFO | RTMetrics done | 20260506 15:56:02.311 | INFO | Finished a round of Baz IO: Wrote 2.094 GiB in 3.368 seconds | 20260506 15:56:02.311 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.76% / 0.01% / 63.23% | 20260506 15:56:02.311 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.8633% / 2.38419e-05% / 6.35783e-05% | 20260506 15:56:02.311 | INFO | Baz IO overall: | 20260506 15:56:02.311 | INFO | BytesWritten : 1358852349264 | 20260506 15:56:02.311 | INFO | NumEvents : 657591640300 | 20260506 15:56:03.970 | INFO | Analyzing chunk frames = [2596864,2597376) | 20260506 15:56:03.970 | INFO | MemInfo: { | 20260506 15:56:03.970 | INFO | "gpu" : " 17 GB", | 20260506 15:56:03.970 | INFO | "huge" : "", | 20260506 15:56:03.970 | INFO | "huge_shared" : " 27 GB", | 20260506 15:56:03.970 | INFO | "malloc" : " 14 GB", | 20260506 15:56:03.970 | INFO | "pinned" : " 29 GB" | 20260506 15:56:03.970 | INFO | } | 20260506 15:56:03.970 | INFO | Perf report for the last 5.14013 seconds: | 20260506 15:56:03.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:56:03.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:03.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:03.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:03.970 | INFO | ANALYSIS : 62.7%, 2.9, 94.9ms, 0 | 20260506 15:56:03.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:03.970 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260506 15:56:03.970 | INFO | PRE_HQ : 60.6%, 1.9, 59.2ms, 0 | 20260506 15:56:03.970 | INFO | BAZWRITER : 65.0%, 4.8, 161.1ms, 0 | 20260506 15:56:03.970 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 15:56:03.970 | INFO | Uploaded 36GiB to GPU in 2.04413s (17.6114GiB/s) | 20260506 15:56:03.970 | INFO | Downloaded 38GiB to GPU in 2.23514s (17.0012GiB/s) | 20260506 15:56:09.108 | INFO | Analyzing chunk frames = [2597376,2597888) | 20260506 15:56:09.108 | INFO | MemInfo: { | 20260506 15:56:09.108 | INFO | "gpu" : " 17 GB", | 20260506 15:56:09.108 | INFO | "huge" : "", | 20260506 15:56:09.108 | INFO | "huge_shared" : " 27 GB", | 20260506 15:56:09.108 | INFO | "malloc" : " 14 GB", | 20260506 15:56:09.108 | INFO | "pinned" : " 29 GB" | 20260506 15:56:09.108 | INFO | } | 20260506 15:56:09.108 | INFO | Perf report for the last 5.13854 seconds: | 20260506 15:56:09.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:56:09.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:09.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:09.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:09.108 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:56:09.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:09.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:09.108 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 15:56:09.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:09.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:09.109 | INFO | Uploaded 36GiB to GPU in 1.98398s (18.1453GiB/s) | 20260506 15:56:09.109 | INFO | Downloaded 34GiB to GPU in 1.93616s (17.5606GiB/s) | 20260506 15:56:14.247 | INFO | Analyzing chunk frames = [2597888,2598400) | 20260506 15:56:14.247 | INFO | MemInfo: { | 20260506 15:56:14.247 | INFO | "gpu" : " 17 GB", | 20260506 15:56:14.247 | INFO | "huge" : "", | 20260506 15:56:14.247 | INFO | "huge_shared" : " 27 GB", | 20260506 15:56:14.247 | INFO | "malloc" : " 14 GB", | 20260506 15:56:14.247 | INFO | "pinned" : " 29 GB" | 20260506 15:56:14.247 | INFO | } | 20260506 15:56:14.247 | INFO | Perf report for the last 5.13887 seconds: | 20260506 15:56:14.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:56:14.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:14.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:14.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:14.247 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 15:56:14.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:14.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:14.247 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:56:14.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:14.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:14.248 | INFO | Uploaded 36GiB to GPU in 1.9617s (18.3514GiB/s) | 20260506 15:56:14.248 | INFO | Downloaded 34GiB to GPU in 1.9346s (17.5747GiB/s) | 20260506 15:56:19.387 | INFO | Analyzing chunk frames = [2598400,2598912) | 20260506 15:56:19.387 | INFO | MemInfo: { | 20260506 15:56:19.387 | INFO | "gpu" : " 17 GB", | 20260506 15:56:19.387 | INFO | "huge" : "", | 20260506 15:56:19.387 | INFO | "huge_shared" : " 27 GB", | 20260506 15:56:19.387 | INFO | "malloc" : " 14 GB", | 20260506 15:56:19.387 | INFO | "pinned" : " 29 GB" | 20260506 15:56:19.387 | INFO | } | 20260506 15:56:19.387 | INFO | Perf report for the last 5.13957 seconds: | 20260506 15:56:19.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:56:19.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:19.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:19.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:19.387 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 15:56:19.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:19.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:19.387 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:56:19.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:19.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:19.387 | INFO | Uploaded 36GiB to GPU in 1.96222s (18.3465GiB/s) | 20260506 15:56:19.387 | INFO | Downloaded 34GiB to GPU in 1.88063s (18.079GiB/s) | 20260506 15:56:24.526 | INFO | Analyzing chunk frames = [2598912,2599424) | 20260506 15:56:24.526 | INFO | MemInfo: { | 20260506 15:56:24.526 | INFO | "gpu" : " 17 GB", | 20260506 15:56:24.526 | INFO | "huge" : "", | 20260506 15:56:24.527 | INFO | "huge_shared" : " 27 GB", | 20260506 15:56:24.527 | INFO | "malloc" : " 14 GB", | 20260506 15:56:24.527 | INFO | "pinned" : " 29 GB" | 20260506 15:56:24.527 | INFO | } | 20260506 15:56:24.527 | INFO | Perf report for the last 5.13995 seconds: | 20260506 15:56:24.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:56:24.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:24.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:24.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:24.527 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 15:56:24.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:24.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:24.527 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:56:24.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:24.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:24.527 | INFO | Uploaded 36GiB to GPU in 1.9881s (18.1077GiB/s) | 20260506 15:56:24.527 | INFO | Downloaded 34GiB to GPU in 1.92298s (17.6809GiB/s) | 20260506 15:56:29.666 | INFO | Analyzing chunk frames = [2599424,2599936) | 20260506 15:56:29.666 | INFO | MemInfo: { | 20260506 15:56:29.666 | INFO | "gpu" : " 17 GB", | 20260506 15:56:29.666 | INFO | "huge" : "", | 20260506 15:56:29.666 | INFO | "huge_shared" : " 27 GB", | 20260506 15:56:29.666 | INFO | "malloc" : " 14 GB", | 20260506 15:56:29.666 | INFO | "pinned" : " 29 GB" | 20260506 15:56:29.666 | INFO | } | 20260506 15:56:29.666 | INFO | Perf report for the last 5.13975 seconds: | 20260506 15:56:29.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:56:29.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:29.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:29.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:29.667 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 15:56:29.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:29.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:29.667 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 15:56:29.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:29.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:29.667 | INFO | Uploaded 36GiB to GPU in 2.06311s (17.4494GiB/s) | 20260506 15:56:29.667 | INFO | Downloaded 34GiB to GPU in 1.97643s (17.2028GiB/s) | 20260506 15:56:34.806 | INFO | Analyzing chunk frames = [2599936,2600448) | 20260506 15:56:34.806 | INFO | MemInfo: { | 20260506 15:56:34.806 | INFO | "gpu" : " 17 GB", | 20260506 15:56:34.806 | INFO | "huge" : "", | 20260506 15:56:34.806 | INFO | "huge_shared" : " 27 GB", | 20260506 15:56:34.806 | INFO | "malloc" : " 14 GB", | 20260506 15:56:34.806 | INFO | "pinned" : " 29 GB" | 20260506 15:56:34.806 | INFO | } | 20260506 15:56:34.806 | INFO | Perf report for the last 5.13923 seconds: | 20260506 15:56:34.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:56:34.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:34.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:34.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:34.806 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:56:34.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:34.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:34.806 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:56:34.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:34.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:34.806 | INFO | Uploaded 36GiB to GPU in 1.9965s (18.0316GiB/s) | 20260506 15:56:34.806 | INFO | Downloaded 34GiB to GPU in 1.98286s (17.1469GiB/s) | 20260506 15:56:39.943 | INFO | Analyzing chunk frames = [2600448,2600960) | 20260506 15:56:39.943 | INFO | MemInfo: { | 20260506 15:56:39.943 | INFO | "gpu" : " 17 GB", | 20260506 15:56:39.943 | INFO | "huge" : "", | 20260506 15:56:39.943 | INFO | "huge_shared" : " 27 GB", | 20260506 15:56:39.943 | INFO | "malloc" : " 14 GB", | 20260506 15:56:39.943 | INFO | "pinned" : " 29 GB" | 20260506 15:56:39.943 | INFO | } | 20260506 15:56:39.943 | INFO | Perf report for the last 5.13767 seconds: | 20260506 15:56:39.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:56:39.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:39.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:39.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:39.944 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 15:56:39.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:39.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:39.944 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:56:39.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:39.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:39.944 | INFO | Uploaded 36GiB to GPU in 2.03325s (17.7057GiB/s) | 20260506 15:56:39.944 | INFO | Downloaded 34GiB to GPU in 1.98723s (17.1092GiB/s) | 20260506 15:56:40.053 | INFO | Starting a round of Baz IO | 20260506 15:56:43.206 | INFO | Populating RTMetrics report | 20260506 15:56:43.206 | INFO | RTMetrics report populated | 20260506 15:56:43.206 | INFO | Opening tmp RTMetrics json file | 20260506 15:56:43.221 | INFO | Serializing RTMetrics to json | 20260506 15:56:43.250 | INFO | Serialization done, swapping tmp file now... | 20260506 15:56:43.257 | INFO | RTMetrics done | 20260506 15:56:43.418 | INFO | Finished a round of Baz IO: Wrote 2.093 GiB in 3.364 seconds | 20260506 15:56:43.418 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.75% / 0.01% / 63.24% | 20260506 15:56:43.418 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.85853% / 2.78155e-05% / 3.97364e-05% | 20260506 15:56:43.418 | INFO | Baz IO overall: | 20260506 15:56:43.418 | INFO | BytesWritten : 1361099876191 | 20260506 15:56:43.418 | INFO | NumEvents : 658678644104 | 20260506 15:56:45.083 | INFO | Analyzing chunk frames = [2600960,2601472) | 20260506 15:56:45.083 | INFO | MemInfo: { | 20260506 15:56:45.083 | INFO | "gpu" : " 17 GB", | 20260506 15:56:45.083 | INFO | "huge" : "", | 20260506 15:56:45.083 | INFO | "huge_shared" : " 27 GB", | 20260506 15:56:45.083 | INFO | "malloc" : " 14 GB", | 20260506 15:56:45.083 | INFO | "pinned" : " 29 GB" | 20260506 15:56:45.083 | INFO | } | 20260506 15:56:45.083 | INFO | Perf report for the last 5.13983 seconds: | 20260506 15:56:45.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:56:45.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:45.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:45.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:45.083 | INFO | ANALYSIS : 62.9%, 2.9, 95.6ms, 0 | 20260506 15:56:45.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:45.083 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 15:56:45.083 | INFO | PRE_HQ : 60.7%, 1.9, 58.6ms, 0 | 20260506 15:56:45.083 | INFO | BAZWRITER : 64.8%, 4.8, 160.1ms, 0 | 20260506 15:56:45.083 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 15:56:45.083 | INFO | Uploaded 36GiB to GPU in 2.08685s (17.2509GiB/s) | 20260506 15:56:45.083 | INFO | Downloaded 38GiB to GPU in 2.22457s (17.082GiB/s) | 20260506 15:56:50.223 | INFO | Analyzing chunk frames = [2601472,2601984) | 20260506 15:56:50.223 | INFO | MemInfo: { | 20260506 15:56:50.223 | INFO | "gpu" : " 17 GB", | 20260506 15:56:50.223 | INFO | "huge" : "", | 20260506 15:56:50.223 | INFO | "huge_shared" : " 27 GB", | 20260506 15:56:50.223 | INFO | "malloc" : " 14 GB", | 20260506 15:56:50.223 | INFO | "pinned" : " 29 GB" | 20260506 15:56:50.223 | INFO | } | 20260506 15:56:50.223 | INFO | Perf report for the last 5.13974 seconds: | 20260506 15:56:50.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:56:50.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:50.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:50.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:50.223 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 15:56:50.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:50.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:50.223 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:56:50.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:50.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:50.223 | INFO | Uploaded 36GiB to GPU in 1.98169s (18.1663GiB/s) | 20260506 15:56:50.223 | INFO | Downloaded 34GiB to GPU in 1.93727s (17.5505GiB/s) | 20260506 15:56:55.362 | INFO | Analyzing chunk frames = [2601984,2602496) | 20260506 15:56:55.362 | INFO | MemInfo: { | 20260506 15:56:55.362 | INFO | "gpu" : " 17 GB", | 20260506 15:56:55.362 | INFO | "huge" : "", | 20260506 15:56:55.362 | INFO | "huge_shared" : " 27 GB", | 20260506 15:56:55.362 | INFO | "malloc" : " 14 GB", | 20260506 15:56:55.362 | INFO | "pinned" : " 29 GB" | 20260506 15:56:55.362 | INFO | } | 20260506 15:56:55.362 | INFO | Perf report for the last 5.13908 seconds: | 20260506 15:56:55.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:56:55.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:55.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:55.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:55.362 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 15:56:55.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:55.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:55.362 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:56:55.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:55.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:56:55.362 | INFO | Uploaded 36GiB to GPU in 1.98552s (18.1312GiB/s) | 20260506 15:56:55.363 | INFO | Downloaded 34GiB to GPU in 1.92916s (17.6242GiB/s) | 20260506 15:57:00.502 | INFO | Analyzing chunk frames = [2602496,2603008) | 20260506 15:57:00.502 | INFO | MemInfo: { | 20260506 15:57:00.502 | INFO | "gpu" : " 17 GB", | 20260506 15:57:00.502 | INFO | "huge" : "", | 20260506 15:57:00.502 | INFO | "huge_shared" : " 27 GB", | 20260506 15:57:00.502 | INFO | "malloc" : " 14 GB", | 20260506 15:57:00.502 | INFO | "pinned" : " 29 GB" | 20260506 15:57:00.502 | INFO | } | 20260506 15:57:00.502 | INFO | Perf report for the last 5.13981 seconds: | 20260506 15:57:00.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:57:00.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:00.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:00.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:00.502 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 15:57:00.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:00.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:00.502 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:57:00.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:00.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:00.502 | INFO | Uploaded 36GiB to GPU in 2.02013s (17.8206GiB/s) | 20260506 15:57:00.502 | INFO | Downloaded 34GiB to GPU in 1.91084s (17.7932GiB/s) | 20260506 15:57:05.641 | INFO | Analyzing chunk frames = [2603008,2603520) | 20260506 15:57:05.641 | INFO | MemInfo: { | 20260506 15:57:05.641 | INFO | "gpu" : " 17 GB", | 20260506 15:57:05.641 | INFO | "huge" : "", | 20260506 15:57:05.641 | INFO | "huge_shared" : " 27 GB", | 20260506 15:57:05.641 | INFO | "malloc" : " 14 GB", | 20260506 15:57:05.641 | INFO | "pinned" : " 29 GB" | 20260506 15:57:05.641 | INFO | } | 20260506 15:57:05.641 | INFO | Perf report for the last 5.13916 seconds: | 20260506 15:57:05.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:57:05.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:05.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:05.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:05.641 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 15:57:05.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:05.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:05.641 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:57:05.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:05.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:05.641 | INFO | Uploaded 36GiB to GPU in 2.0283s (17.7489GiB/s) | 20260506 15:57:05.641 | INFO | Downloaded 34GiB to GPU in 1.9413s (17.514GiB/s) | 20260506 15:57:10.780 | INFO | Analyzing chunk frames = [2603520,2604032) | 20260506 15:57:10.780 | INFO | MemInfo: { | 20260506 15:57:10.780 | INFO | "gpu" : " 17 GB", | 20260506 15:57:10.780 | INFO | "huge" : "", | 20260506 15:57:10.780 | INFO | "huge_shared" : " 27 GB", | 20260506 15:57:10.780 | INFO | "malloc" : " 14 GB", | 20260506 15:57:10.780 | INFO | "pinned" : " 29 GB" | 20260506 15:57:10.780 | INFO | } | 20260506 15:57:10.780 | INFO | Perf report for the last 5.13902 seconds: | 20260506 15:57:10.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:57:10.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:10.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:10.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:10.780 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 15:57:10.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:10.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:10.780 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260506 15:57:10.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:10.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:10.781 | INFO | Uploaded 36GiB to GPU in 2.02853s (17.7468GiB/s) | 20260506 15:57:10.781 | INFO | Downloaded 34GiB to GPU in 1.91547s (17.7502GiB/s) | 20260506 15:57:15.920 | INFO | Analyzing chunk frames = [2604032,2604544) | 20260506 15:57:15.920 | INFO | MemInfo: { | 20260506 15:57:15.920 | INFO | "gpu" : " 17 GB", | 20260506 15:57:15.920 | INFO | "huge" : "", | 20260506 15:57:15.920 | INFO | "huge_shared" : " 27 GB", | 20260506 15:57:15.921 | INFO | "malloc" : " 14 GB", | 20260506 15:57:15.921 | INFO | "pinned" : " 29 GB" | 20260506 15:57:15.921 | INFO | } | 20260506 15:57:15.921 | INFO | Perf report for the last 5.14044 seconds: | 20260506 15:57:15.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:57:15.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:15.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:15.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:15.921 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:57:15.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:15.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:15.921 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:57:15.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:15.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:15.921 | INFO | Uploaded 36GiB to GPU in 1.98921s (18.0976GiB/s) | 20260506 15:57:15.921 | INFO | Downloaded 34GiB to GPU in 1.96209s (17.3284GiB/s) | 20260506 15:57:21.059 | INFO | Analyzing chunk frames = [2604544,2605056) | 20260506 15:57:21.059 | INFO | MemInfo: { | 20260506 15:57:21.059 | INFO | "gpu" : " 17 GB", | 20260506 15:57:21.059 | INFO | "huge" : "", | 20260506 15:57:21.059 | INFO | "huge_shared" : " 27 GB", | 20260506 15:57:21.059 | INFO | "malloc" : " 14 GB", | 20260506 15:57:21.059 | INFO | "pinned" : " 29 GB" | 20260506 15:57:21.059 | INFO | } | 20260506 15:57:21.059 | INFO | Perf report for the last 5.13837 seconds: | 20260506 15:57:21.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:57:21.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:21.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:21.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:21.059 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:57:21.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:21.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:21.059 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:57:21.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:21.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:21.059 | INFO | Uploaded 36GiB to GPU in 2.01053s (17.9057GiB/s) | 20260506 15:57:21.059 | INFO | Downloaded 34GiB to GPU in 1.98982s (17.087GiB/s) | 20260506 15:57:21.060 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:57:21.060 | INFO | Total mem usage: 17.4 GiB | 20260506 15:57:21.060 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:57:21.060 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -10.7%) | 20260506 15:57:21.060 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:57:21.060 | INFO | High Water Breakdown: | 20260506 15:57:21.060 | INFO | 61.4% : 9.65 GiB ( 88.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:57:21.060 | INFO | 38.5% : 6.05 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:57:21.060 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:57:21.060 | INFO | ------------------------------------------ | 20260506 15:57:21.060 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:57:21.060 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:57:21.060 | INFO | Total mem usage: 26.5 GiB | 20260506 15:57:21.060 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:57:21.060 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:57:21.060 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 15:57:21.060 | INFO | High Water Breakdown: | 20260506 15:57:21.060 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:57:21.060 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:57:21.060 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:57:21.060 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:57:21.060 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:57:21.060 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:57:21.060 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:57:21.060 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:57:21.060 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:57:21.060 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:57:21.060 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:57:21.060 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:57:21.060 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:57:21.060 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:57:21.060 | INFO | ------------------------------------------ | 20260506 15:57:21.060 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:57:21.060 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:57:21.060 | INFO | Total mem usage: 25.6 GiB | 20260506 15:57:21.060 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:57:21.060 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:57:21.060 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:57:21.060 | INFO | High Water Breakdown: | 20260506 15:57:21.060 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:57:21.060 | INFO | ------------------------------------------ | 20260506 15:57:21.060 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:57:21.060 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:57:21.060 | INFO | Total mem usage: 15.7 GiB | 20260506 15:57:21.060 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:57:21.060 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:57:21.060 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 15:57:21.060 | INFO | High Water Breakdown: | 20260506 15:57:21.060 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:57:21.060 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:57:21.060 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:57:21.060 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:57:21.060 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:57:21.060 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:57:21.060 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:57:21.060 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:57:21.060 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:57:21.060 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:57:21.060 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:57:21.060 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:57:21.060 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:57:21.060 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:57:21.060 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:57:21.060 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:57:21.060 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:57:21.060 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:57:21.060 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:57:21.060 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:57:21.060 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:57:21.060 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:57:21.060 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:57:21.060 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:57:21.060 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:57:21.060 | INFO | ------------------------------------------ | 20260506 15:57:21.060 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:57:21.060 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:57:21.060 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:57:21.060 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:57:21.060 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.49e+01, 1.32e+01 / 1.71e+00 | 20260506 15:57:21.060 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 9.95e-03 / 5.44e-03 | 20260506 15:57:21.060 | INFO | PerfReport: 11.60%, Baseline -- 3.64e+00 / 4.37e+00, 5.63e-01 / 3.91e-01 | 20260506 15:57:21.060 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.84e-01 / 1.52e-01 | 20260506 15:57:21.060 | INFO | PerfReport: 15.84%, DME -- 3.98e+01 / 4.35e+01, 3.47e+00 / 1.86e+00 | 20260506 15:57:21.060 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.44e+00 / 2.08e+00 | 20260506 15:57:21.060 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.31e+00 / 2.88e+00, 3.97e-01 / 2.59e-01 | 20260506 15:57:21.060 | INFO | PerfReport: 3.53%, Metrics -- 1.11e+00 / 1.46e+00, 2.89e-01 / 1.80e-01 | 20260506 15:57:21.060 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.85e-01 / 4.32e-01, 6.02e-02 / 2.74e-02 | 20260506 15:57:21.061 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:57:21.061 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:57:21.061 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:57:21.061 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 15:57:21.061 | INFO | PerfReport: 21.51%, Upload -- 2.09e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260506 15:57:21.061 | INFO | PerfReport: 21.18%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 15:57:21.061 | INFO | PerfReport: 32.25%, Compute -- 3.14e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 15:57:21.061 | INFO | PerfReport: 1.41%, LockUpload -- 1.37e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260506 15:57:21.061 | INFO | PerfReport: 1.16%, LockDownload -- 1.13e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 15:57:21.061 | INFO | PerfReport: 22.49%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 15:57:21.061 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260506 15:57:21.172 | INFO | Starting a round of Baz IO | 20260506 15:57:24.324 | INFO | Populating RTMetrics report | 20260506 15:57:24.324 | INFO | RTMetrics report populated | 20260506 15:57:24.324 | INFO | Opening tmp RTMetrics json file | 20260506 15:57:24.545 | INFO | Serializing RTMetrics to json | 20260506 15:57:24.556 | INFO | Serialization done, swapping tmp file now... | 20260506 15:57:24.560 | INFO | RTMetrics done | 20260506 15:57:24.593 | INFO | Finished a round of Baz IO: Wrote 2.093 GiB in 3.421 seconds | 20260506 15:57:24.593 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.74% / 0.01% / 63.25% | 20260506 15:57:24.593 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.85013% / 0% / 7.15256e-05% | 20260506 15:57:24.594 | INFO | Baz IO overall: | 20260506 15:57:24.594 | INFO | BytesWritten : 1363347042391 | 20260506 15:57:24.594 | INFO | NumEvents : 659765433874 | 20260506 15:57:26.198 | INFO | Analyzing chunk frames = [2605056,2605568) | 20260506 15:57:26.198 | INFO | MemInfo: { | 20260506 15:57:26.198 | INFO | "gpu" : " 17 GB", | 20260506 15:57:26.198 | INFO | "huge" : "", | 20260506 15:57:26.198 | INFO | "huge_shared" : " 27 GB", | 20260506 15:57:26.198 | INFO | "malloc" : " 14 GB", | 20260506 15:57:26.198 | INFO | "pinned" : " 29 GB" | 20260506 15:57:26.198 | INFO | } | 20260506 15:57:26.198 | INFO | Perf report for the last 5.13873 seconds: | 20260506 15:57:26.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:57:26.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:26.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:26.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:26.198 | INFO | ANALYSIS : 62.9%, 2.9, 95.4ms, 0 | 20260506 15:57:26.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:26.198 | INFO | RT_METRICS : 48.4%, 1.0, 25.4ms, 0 | 20260506 15:57:26.198 | INFO | PRE_HQ : 60.9%, 1.9, 59.1ms, 0 | 20260506 15:57:26.198 | INFO | BAZWRITER : 66.0%, 4.6, 154.6ms, 0 | 20260506 15:57:26.198 | INFO | BAZ_SAMPLING : 4.2%, 1.0, 2.2ms, 0 | 20260506 15:57:26.198 | INFO | Uploaded 36GiB to GPU in 2.04187s (17.6309GiB/s) | 20260506 15:57:26.198 | INFO | Downloaded 38GiB to GPU in 2.22709s (17.0626GiB/s) | 20260506 15:57:31.338 | INFO | Analyzing chunk frames = [2605568,2606080) | 20260506 15:57:31.338 | INFO | MemInfo: { | 20260506 15:57:31.338 | INFO | "gpu" : " 17 GB", | 20260506 15:57:31.338 | INFO | "huge" : "", | 20260506 15:57:31.338 | INFO | "huge_shared" : " 27 GB", | 20260506 15:57:31.338 | INFO | "malloc" : " 14 GB", | 20260506 15:57:31.338 | INFO | "pinned" : " 29 GB" | 20260506 15:57:31.338 | INFO | } | 20260506 15:57:31.338 | INFO | Perf report for the last 5.1403 seconds: | 20260506 15:57:31.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:57:31.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:31.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:31.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:31.338 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:57:31.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:31.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:31.338 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:57:31.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:31.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:31.338 | INFO | Uploaded 36GiB to GPU in 1.99094s (18.0819GiB/s) | 20260506 15:57:31.338 | INFO | Downloaded 34GiB to GPU in 1.94246s (17.5036GiB/s) | 20260506 15:57:36.478 | INFO | Analyzing chunk frames = [2606080,2606592) | 20260506 15:57:36.478 | INFO | MemInfo: { | 20260506 15:57:36.478 | INFO | "gpu" : " 17 GB", | 20260506 15:57:36.478 | INFO | "huge" : "", | 20260506 15:57:36.478 | INFO | "huge_shared" : " 27 GB", | 20260506 15:57:36.478 | INFO | "malloc" : " 14 GB", | 20260506 15:57:36.478 | INFO | "pinned" : " 29 GB" | 20260506 15:57:36.478 | INFO | } | 20260506 15:57:36.478 | INFO | Perf report for the last 5.1404 seconds: | 20260506 15:57:36.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:57:36.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:36.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:36.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:36.478 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 15:57:36.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:36.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:36.479 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:57:36.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:36.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:36.479 | INFO | Uploaded 36GiB to GPU in 1.97255s (18.2505GiB/s) | 20260506 15:57:36.479 | INFO | Downloaded 34GiB to GPU in 1.93733s (17.55GiB/s) | 20260506 15:57:41.617 | INFO | Analyzing chunk frames = [2606592,2607104) | 20260506 15:57:41.617 | INFO | MemInfo: { | 20260506 15:57:41.617 | INFO | "gpu" : " 17 GB", | 20260506 15:57:41.617 | INFO | "huge" : "", | 20260506 15:57:41.617 | INFO | "huge_shared" : " 27 GB", | 20260506 15:57:41.617 | INFO | "malloc" : " 14 GB", | 20260506 15:57:41.617 | INFO | "pinned" : " 29 GB" | 20260506 15:57:41.617 | INFO | } | 20260506 15:57:41.617 | INFO | Perf report for the last 5.13879 seconds: | 20260506 15:57:41.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:57:41.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:41.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:41.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:41.617 | INFO | ANALYSIS : 60.6%, 3.0, 94.0ms, 0 | 20260506 15:57:41.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:41.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:41.617 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:57:41.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:41.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:41.618 | INFO | Uploaded 36GiB to GPU in 2.01224s (17.8905GiB/s) | 20260506 15:57:41.618 | INFO | Downloaded 34GiB to GPU in 1.9107s (17.7945GiB/s) | 20260506 15:57:46.756 | INFO | Analyzing chunk frames = [2607104,2607616) | 20260506 15:57:46.756 | INFO | MemInfo: { | 20260506 15:57:46.756 | INFO | "gpu" : " 17 GB", | 20260506 15:57:46.756 | INFO | "huge" : "", | 20260506 15:57:46.756 | INFO | "huge_shared" : " 27 GB", | 20260506 15:57:46.756 | INFO | "malloc" : " 14 GB", | 20260506 15:57:46.756 | INFO | "pinned" : " 29 GB" | 20260506 15:57:46.756 | INFO | } | 20260506 15:57:46.756 | INFO | Perf report for the last 5.13914 seconds: | 20260506 15:57:46.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:57:46.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:46.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:46.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:46.756 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 15:57:46.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:46.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:46.757 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 15:57:46.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:46.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:46.757 | INFO | Uploaded 36GiB to GPU in 2.02951s (17.7383GiB/s) | 20260506 15:57:46.757 | INFO | Downloaded 34GiB to GPU in 1.94445s (17.4857GiB/s) | 20260506 15:57:51.895 | INFO | Analyzing chunk frames = [2607616,2608128) | 20260506 15:57:51.895 | INFO | MemInfo: { | 20260506 15:57:51.895 | INFO | "gpu" : " 17 GB", | 20260506 15:57:51.895 | INFO | "huge" : "", | 20260506 15:57:51.895 | INFO | "huge_shared" : " 27 GB", | 20260506 15:57:51.895 | INFO | "malloc" : " 14 GB", | 20260506 15:57:51.895 | INFO | "pinned" : " 29 GB" | 20260506 15:57:51.895 | INFO | } | 20260506 15:57:51.895 | INFO | Perf report for the last 5.13865 seconds: | 20260506 15:57:51.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:57:51.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:51.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:51.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:51.895 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:57:51.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:51.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:51.895 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 15:57:51.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:51.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:51.895 | INFO | Uploaded 36GiB to GPU in 2.02318s (17.7938GiB/s) | 20260506 15:57:51.896 | INFO | Downloaded 34GiB to GPU in 1.92798s (17.6351GiB/s) | 20260506 15:57:57.035 | INFO | Analyzing chunk frames = [2608128,2608640) | 20260506 15:57:57.035 | INFO | MemInfo: { | 20260506 15:57:57.035 | INFO | "gpu" : " 17 GB", | 20260506 15:57:57.035 | INFO | "huge" : "", | 20260506 15:57:57.035 | INFO | "huge_shared" : " 27 GB", | 20260506 15:57:57.035 | INFO | "malloc" : " 14 GB", | 20260506 15:57:57.035 | INFO | "pinned" : " 29 GB" | 20260506 15:57:57.035 | INFO | } | 20260506 15:57:57.035 | INFO | Perf report for the last 5.13996 seconds: | 20260506 15:57:57.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:57:57.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:57.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:57.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:57.035 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:57:57.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:57.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:57.035 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 15:57:57.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:57.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:57:57.035 | INFO | Uploaded 36GiB to GPU in 1.98918s (18.0979GiB/s) | 20260506 15:57:57.035 | INFO | Downloaded 34GiB to GPU in 1.96608s (17.2933GiB/s) | 20260506 15:58:02.174 | INFO | Analyzing chunk frames = [2608640,2609152) | 20260506 15:58:02.174 | INFO | MemInfo: { | 20260506 15:58:02.174 | INFO | "gpu" : " 17 GB", | 20260506 15:58:02.174 | INFO | "huge" : "", | 20260506 15:58:02.174 | INFO | "huge_shared" : " 27 GB", | 20260506 15:58:02.174 | INFO | "malloc" : " 14 GB", | 20260506 15:58:02.174 | INFO | "pinned" : " 29 GB" | 20260506 15:58:02.174 | INFO | } | 20260506 15:58:02.174 | INFO | Perf report for the last 5.13907 seconds: | 20260506 15:58:02.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:58:02.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:02.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:02.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:02.174 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 15:58:02.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:02.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:02.174 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:58:02.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:02.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:02.174 | INFO | Uploaded 36GiB to GPU in 2.01873s (17.833GiB/s) | 20260506 15:58:02.174 | INFO | Downloaded 34GiB to GPU in 1.97835s (17.1861GiB/s) | 20260506 15:58:02.294 | INFO | Starting a round of Baz IO | 20260506 15:58:05.433 | INFO | Populating RTMetrics report | 20260506 15:58:05.434 | INFO | RTMetrics report populated | 20260506 15:58:05.434 | INFO | Opening tmp RTMetrics json file | 20260506 15:58:05.438 | INFO | Serializing RTMetrics to json | 20260506 15:58:05.439 | INFO | Serialization done, swapping tmp file now... | 20260506 15:58:05.440 | INFO | RTMetrics done | 20260506 15:58:05.550 | INFO | Finished a round of Baz IO: Wrote 2.092 GiB in 3.256 seconds | 20260506 15:58:05.550 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.73% / 0.01% / 63.26% | 20260506 15:58:05.550 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.84506% / 3.97364e-05% / 7.54992e-05% | 20260506 15:58:05.550 | INFO | Baz IO overall: | 20260506 15:58:05.550 | INFO | BytesWritten : 1365593170720 | 20260506 15:58:05.550 | INFO | NumEvents : 660851391295 | 20260506 15:58:07.314 | INFO | Analyzing chunk frames = [2609152,2609664) | 20260506 15:58:07.314 | INFO | MemInfo: { | 20260506 15:58:07.314 | INFO | "gpu" : " 17 GB", | 20260506 15:58:07.314 | INFO | "huge" : "", | 20260506 15:58:07.314 | INFO | "huge_shared" : " 27 GB", | 20260506 15:58:07.315 | INFO | "malloc" : " 14 GB", | 20260506 15:58:07.315 | INFO | "pinned" : " 29 GB" | 20260506 15:58:07.315 | INFO | } | 20260506 15:58:07.315 | INFO | Perf report for the last 5.14055 seconds: | 20260506 15:58:07.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:58:07.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:07.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:07.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:07.315 | INFO | ANALYSIS : 62.8%, 2.9, 95.3ms, 0 | 20260506 15:58:07.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:07.315 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 15:58:07.315 | INFO | PRE_HQ : 60.6%, 1.9, 58.8ms, 0 | 20260506 15:58:07.315 | INFO | BAZWRITER : 62.8%, 4.7, 151.7ms, 0 | 20260506 15:58:07.315 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 15:58:07.315 | INFO | Uploaded 36GiB to GPU in 2.0662s (17.4233GiB/s) | 20260506 15:58:07.315 | INFO | Downloaded 38GiB to GPU in 2.2374s (16.984GiB/s) | 20260506 15:58:12.452 | INFO | Analyzing chunk frames = [2609664,2610176) | 20260506 15:58:12.453 | INFO | MemInfo: { | 20260506 15:58:12.453 | INFO | "gpu" : " 17 GB", | 20260506 15:58:12.453 | INFO | "huge" : "", | 20260506 15:58:12.453 | INFO | "huge_shared" : " 27 GB", | 20260506 15:58:12.453 | INFO | "malloc" : " 14 GB", | 20260506 15:58:12.453 | INFO | "pinned" : " 29 GB" | 20260506 15:58:12.453 | INFO | } | 20260506 15:58:12.453 | INFO | Perf report for the last 5.13806 seconds: | 20260506 15:58:12.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:58:12.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:12.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:12.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:12.453 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 15:58:12.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:12.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:12.453 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 15:58:12.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:12.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:12.453 | INFO | Uploaded 36GiB to GPU in 2.00792s (17.929GiB/s) | 20260506 15:58:12.453 | INFO | Downloaded 34GiB to GPU in 1.92398s (17.6717GiB/s) | 20260506 15:58:17.592 | INFO | Analyzing chunk frames = [2610176,2610688) | 20260506 15:58:17.592 | INFO | MemInfo: { | 20260506 15:58:17.592 | INFO | "gpu" : " 17 GB", | 20260506 15:58:17.592 | INFO | "huge" : "", | 20260506 15:58:17.592 | INFO | "huge_shared" : " 27 GB", | 20260506 15:58:17.592 | INFO | "malloc" : " 14 GB", | 20260506 15:58:17.592 | INFO | "pinned" : " 29 GB" | 20260506 15:58:17.592 | INFO | } | 20260506 15:58:17.592 | INFO | Perf report for the last 5.13929 seconds: | 20260506 15:58:17.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:58:17.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:17.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:17.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:17.592 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 15:58:17.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:17.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:17.592 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:58:17.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:17.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:17.592 | INFO | Uploaded 36GiB to GPU in 2.00182s (17.9836GiB/s) | 20260506 15:58:17.592 | INFO | Downloaded 34GiB to GPU in 1.93844s (17.5399GiB/s) | 20260506 15:58:22.731 | INFO | Analyzing chunk frames = [2610688,2611200) | 20260506 15:58:22.731 | INFO | MemInfo: { | 20260506 15:58:22.731 | INFO | "gpu" : " 17 GB", | 20260506 15:58:22.731 | INFO | "huge" : "", | 20260506 15:58:22.731 | INFO | "huge_shared" : " 27 GB", | 20260506 15:58:22.731 | INFO | "malloc" : " 14 GB", | 20260506 15:58:22.731 | INFO | "pinned" : " 29 GB" | 20260506 15:58:22.731 | INFO | } | 20260506 15:58:22.731 | INFO | Perf report for the last 5.1393 seconds: | 20260506 15:58:22.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:58:22.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:22.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:22.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:22.731 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 15:58:22.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:22.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:22.731 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:58:22.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:22.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:22.732 | INFO | Uploaded 36GiB to GPU in 2.02059s (17.8166GiB/s) | 20260506 15:58:22.732 | INFO | Downloaded 34GiB to GPU in 1.91407s (17.7632GiB/s) | 20260506 15:58:27.870 | INFO | Analyzing chunk frames = [2611200,2611712) | 20260506 15:58:27.870 | INFO | MemInfo: { | 20260506 15:58:27.870 | INFO | "gpu" : " 17 GB", | 20260506 15:58:27.870 | INFO | "huge" : "", | 20260506 15:58:27.870 | INFO | "huge_shared" : " 27 GB", | 20260506 15:58:27.870 | INFO | "malloc" : " 14 GB", | 20260506 15:58:27.871 | INFO | "pinned" : " 29 GB" | 20260506 15:58:27.871 | INFO | } | 20260506 15:58:27.871 | INFO | Perf report for the last 5.13934 seconds: | 20260506 15:58:27.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:58:27.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:27.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:27.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:27.871 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 15:58:27.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:27.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:27.871 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:58:27.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:27.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:27.871 | INFO | Uploaded 36GiB to GPU in 1.99226s (18.0699GiB/s) | 20260506 15:58:27.871 | INFO | Downloaded 34GiB to GPU in 1.93404s (17.5798GiB/s) | 20260506 15:58:33.009 | INFO | Analyzing chunk frames = [2611712,2612224) | 20260506 15:58:33.009 | INFO | MemInfo: { | 20260506 15:58:33.009 | INFO | "gpu" : " 17 GB", | 20260506 15:58:33.009 | INFO | "huge" : "", | 20260506 15:58:33.010 | INFO | "huge_shared" : " 27 GB", | 20260506 15:58:33.010 | INFO | "malloc" : " 14 GB", | 20260506 15:58:33.010 | INFO | "pinned" : " 29 GB" | 20260506 15:58:33.010 | INFO | } | 20260506 15:58:33.010 | INFO | Perf report for the last 5.13895 seconds: | 20260506 15:58:33.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:58:33.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:33.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:33.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:33.010 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 15:58:33.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:33.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:33.010 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260506 15:58:33.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:33.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:33.010 | INFO | Uploaded 36GiB to GPU in 2.07158s (17.378GiB/s) | 20260506 15:58:33.010 | INFO | Downloaded 34GiB to GPU in 1.96752s (17.2806GiB/s) | 20260506 15:58:38.149 | INFO | Analyzing chunk frames = [2612224,2612736) | 20260506 15:58:38.149 | INFO | MemInfo: { | 20260506 15:58:38.149 | INFO | "gpu" : " 17 GB", | 20260506 15:58:38.149 | INFO | "huge" : "", | 20260506 15:58:38.150 | INFO | "huge_shared" : " 27 GB", | 20260506 15:58:38.150 | INFO | "malloc" : " 14 GB", | 20260506 15:58:38.150 | INFO | "pinned" : " 29 GB" | 20260506 15:58:38.150 | INFO | } | 20260506 15:58:38.150 | INFO | Perf report for the last 5.13996 seconds: | 20260506 15:58:38.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:58:38.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:38.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:38.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:38.150 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 15:58:38.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:38.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:38.150 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 15:58:38.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:38.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:38.150 | INFO | Uploaded 36GiB to GPU in 2.01699s (17.8484GiB/s) | 20260506 15:58:38.150 | INFO | Downloaded 34GiB to GPU in 2.01693s (16.8573GiB/s) | 20260506 15:58:43.288 | INFO | Analyzing chunk frames = [2612736,2613248) | 20260506 15:58:43.288 | INFO | MemInfo: { | 20260506 15:58:43.288 | INFO | "gpu" : " 17 GB", | 20260506 15:58:43.288 | INFO | "huge" : "", | 20260506 15:58:43.288 | INFO | "huge_shared" : " 27 GB", | 20260506 15:58:43.288 | INFO | "malloc" : " 14 GB", | 20260506 15:58:43.288 | INFO | "pinned" : " 29 GB" | 20260506 15:58:43.288 | INFO | } | 20260506 15:58:43.289 | INFO | Perf report for the last 5.13886 seconds: | 20260506 15:58:43.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:58:43.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:43.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:43.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:43.289 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:58:43.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:43.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:43.289 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 15:58:43.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:43.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:43.289 | INFO | Uploaded 36GiB to GPU in 2.00541s (17.9514GiB/s) | 20260506 15:58:43.289 | INFO | Downloaded 34GiB to GPU in 1.98635s (17.1168GiB/s) | 20260506 15:58:43.289 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 15:58:43.289 | INFO | Total mem usage: 17.4 GiB | 20260506 15:58:43.289 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 15:58:43.289 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -15.5%) | 20260506 15:58:43.289 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 15:58:43.289 | INFO | High Water Breakdown: | 20260506 15:58:43.289 | INFO | 60.7% : 9.15 GiB ( 83.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 15:58:43.289 | INFO | 39.1% : 5.9 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 15:58:43.289 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 15:58:43.289 | INFO | ------------------------------------------ | 20260506 15:58:43.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:58:43.289 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 15:58:43.289 | INFO | Total mem usage: 26.5 GiB | 20260506 15:58:43.289 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 15:58:43.289 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 15:58:43.289 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 15:58:43.289 | INFO | High Water Breakdown: | 20260506 15:58:43.289 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:58:43.289 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:58:43.289 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:58:43.289 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:58:43.289 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:58:43.289 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:58:43.290 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 15:58:43.290 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 15:58:43.290 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:58:43.290 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 15:58:43.290 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 15:58:43.290 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 15:58:43.290 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 15:58:43.290 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 15:58:43.290 | INFO | ------------------------------------------ | 20260506 15:58:43.290 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:58:43.290 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 15:58:43.290 | INFO | Total mem usage: 25.6 GiB | 20260506 15:58:43.290 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 15:58:43.290 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 15:58:43.290 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 15:58:43.290 | INFO | High Water Breakdown: | 20260506 15:58:43.290 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 15:58:43.290 | INFO | ------------------------------------------ | 20260506 15:58:43.290 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:58:43.290 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 15:58:43.290 | INFO | Total mem usage: 15.7 GiB | 20260506 15:58:43.290 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 15:58:43.290 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 15:58:43.290 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 15:58:43.290 | INFO | High Water Breakdown: | 20260506 15:58:43.290 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 15:58:43.290 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 15:58:43.290 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 15:58:43.290 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 15:58:43.290 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 15:58:43.290 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 15:58:43.290 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 15:58:43.290 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 15:58:43.290 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 15:58:43.290 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 15:58:43.290 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 15:58:43.290 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 15:58:43.290 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 15:58:43.290 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 15:58:43.290 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 15:58:43.290 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 15:58:43.290 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 15:58:43.290 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 15:58:43.290 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 15:58:43.290 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 15:58:43.290 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 15:58:43.290 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 15:58:43.290 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 15:58:43.290 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 15:58:43.290 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 15:58:43.290 | INFO | ------------------------------------------ | 20260506 15:58:43.290 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 15:58:43.290 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 15:58:43.290 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:58:43.290 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:58:43.290 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.55e+01, 1.35e+01 / 1.85e+00 | 20260506 15:58:43.290 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 9.95e-02, 9.93e-03 / 6.45e-03 | 20260506 15:58:43.290 | INFO | PerfReport: 11.57%, Baseline -- 3.64e+00 / 4.37e+00, 5.58e-01 / 4.33e-01 | 20260506 15:58:43.290 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.46e+00, 1.87e-01 / 1.48e-01 | 20260506 15:58:43.290 | INFO | PerfReport: 15.86%, DME -- 3.99e+01 / 4.40e+01, 3.78e+00 / 1.91e+00 | 20260506 15:58:43.290 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.53e+00 / 2.04e+00 | 20260506 15:58:43.290 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.30e+00 / 2.88e+00, 3.91e-01 / 2.52e-01 | 20260506 15:58:43.290 | INFO | PerfReport: 3.52%, Metrics -- 1.11e+00 / 1.48e+00, 2.90e-01 / 2.25e-01 | 20260506 15:58:43.290 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.86e-01 / 4.37e-01, 6.19e-02 / 2.55e-02 | 20260506 15:58:43.290 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 15:58:43.290 | INFO | PerfReport: Gathered 1536 samples. | 20260506 15:58:43.290 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 15:58:43.290 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 15:58:43.290 | INFO | PerfReport: 21.54%, Upload -- 2.10e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 15:58:43.290 | INFO | PerfReport: 21.21%, Download -- 2.07e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 15:58:43.290 | INFO | PerfReport: 32.25%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 15:58:43.290 | INFO | PerfReport: 1.46%, LockUpload -- 1.43e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260506 15:58:43.290 | INFO | PerfReport: 1.15%, LockDownload -- 1.12e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260506 15:58:43.290 | INFO | PerfReport: 22.37%, LockCompute -- 2.18e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 15:58:43.290 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260506 15:58:43.409 | INFO | Starting a round of Baz IO | 20260506 15:58:46.559 | INFO | Populating RTMetrics report | 20260506 15:58:46.559 | INFO | RTMetrics report populated | 20260506 15:58:46.559 | INFO | Opening tmp RTMetrics json file | 20260506 15:58:46.561 | INFO | Serializing RTMetrics to json | 20260506 15:58:46.563 | INFO | Serialization done, swapping tmp file now... | 20260506 15:58:46.564 | INFO | RTMetrics done | 20260506 15:58:46.748 | INFO | Finished a round of Baz IO: Wrote 2.094 GiB in 3.339 seconds | 20260506 15:58:46.748 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.72% / 0.01% / 63.27% | 20260506 15:58:46.748 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.84735% / 7.94729e-06% / 8.34465e-05% | 20260506 15:58:46.748 | INFO | Baz IO overall: | 20260506 15:58:46.748 | INFO | BytesWritten : 1367841158048 | 20260506 15:58:46.748 | INFO | NumEvents : 661938597126 | 20260506 15:58:48.428 | INFO | Analyzing chunk frames = [2613248,2613760) | 20260506 15:58:48.428 | INFO | MemInfo: { | 20260506 15:58:48.428 | INFO | "gpu" : " 17 GB", | 20260506 15:58:48.428 | INFO | "huge" : "", | 20260506 15:58:48.428 | INFO | "huge_shared" : " 27 GB", | 20260506 15:58:48.428 | INFO | "malloc" : " 14 GB", | 20260506 15:58:48.428 | INFO | "pinned" : " 29 GB" | 20260506 15:58:48.428 | INFO | } | 20260506 15:58:48.429 | INFO | Perf report for the last 5.13999 seconds: | 20260506 15:58:48.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:58:48.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:48.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:48.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:48.429 | INFO | ANALYSIS : 62.9%, 2.8, 93.9ms, 0 | 20260506 15:58:48.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:48.429 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 15:58:48.429 | INFO | PRE_HQ : 61.4%, 2.0, 62.3ms, 0 | 20260506 15:58:48.429 | INFO | BAZWRITER : 64.5%, 4.6, 151.3ms, 0 | 20260506 15:58:48.429 | INFO | BAZ_SAMPLING : 3.1%, 1.0, 1.6ms, 0 | 20260506 15:58:48.429 | INFO | Uploaded 36GiB to GPU in 2.03388s (17.7002GiB/s) | 20260506 15:58:48.429 | INFO | Downloaded 38GiB to GPU in 2.22351s (17.0901GiB/s) | 20260506 15:58:53.568 | INFO | Analyzing chunk frames = [2613760,2614272) | 20260506 15:58:53.568 | INFO | MemInfo: { | 20260506 15:58:53.568 | INFO | "gpu" : " 17 GB", | 20260506 15:58:53.568 | INFO | "huge" : "", | 20260506 15:58:53.568 | INFO | "huge_shared" : " 27 GB", | 20260506 15:58:53.568 | INFO | "malloc" : " 14 GB", | 20260506 15:58:53.568 | INFO | "pinned" : " 29 GB" | 20260506 15:58:53.568 | INFO | } | 20260506 15:58:53.568 | INFO | Perf report for the last 5.13927 seconds: | 20260506 15:58:53.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:58:53.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:53.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:53.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:53.568 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 15:58:53.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:53.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:53.568 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 15:58:53.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:53.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:53.569 | INFO | Uploaded 36GiB to GPU in 1.98959s (18.0941GiB/s) | 20260506 15:58:53.569 | INFO | Downloaded 34GiB to GPU in 1.93842s (17.5401GiB/s) | 20260506 15:58:58.707 | INFO | Analyzing chunk frames = [2614272,2614784) | 20260506 15:58:58.707 | INFO | MemInfo: { | 20260506 15:58:58.707 | INFO | "gpu" : " 17 GB", | 20260506 15:58:58.707 | INFO | "huge" : "", | 20260506 15:58:58.707 | INFO | "huge_shared" : " 27 GB", | 20260506 15:58:58.707 | INFO | "malloc" : " 14 GB", | 20260506 15:58:58.707 | INFO | "pinned" : " 29 GB" | 20260506 15:58:58.707 | INFO | } | 20260506 15:58:58.707 | INFO | Perf report for the last 5.13911 seconds: | 20260506 15:58:58.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:58:58.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:58.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:58.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:58.707 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 15:58:58.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:58.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:58.707 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 15:58:58.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:58.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:58:58.707 | INFO | Uploaded 36GiB to GPU in 1.96352s (18.3344GiB/s) | 20260506 15:58:58.707 | INFO | Downloaded 34GiB to GPU in 1.93574s (17.5644GiB/s) | 20260506 15:59:03.846 | INFO | Analyzing chunk frames = [2614784,2615296) | 20260506 15:59:03.846 | INFO | MemInfo: { | 20260506 15:59:03.846 | INFO | "gpu" : " 17 GB", | 20260506 15:59:03.846 | INFO | "huge" : "", | 20260506 15:59:03.846 | INFO | "huge_shared" : " 27 GB", | 20260506 15:59:03.846 | INFO | "malloc" : " 14 GB", | 20260506 15:59:03.846 | INFO | "pinned" : " 29 GB" | 20260506 15:59:03.846 | INFO | } | 20260506 15:59:03.846 | INFO | Perf report for the last 5.13939 seconds: | 20260506 15:59:03.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:59:03.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:03.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:03.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:03.846 | INFO | ANALYSIS : 60.6%, 3.0, 94.0ms, 0 | 20260506 15:59:03.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:03.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:03.846 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:59:03.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:03.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:03.847 | INFO | Uploaded 36GiB to GPU in 2.01021s (17.9086GiB/s) | 20260506 15:59:03.847 | INFO | Downloaded 34GiB to GPU in 1.90868s (17.8134GiB/s) | 20260506 15:59:08.986 | INFO | Analyzing chunk frames = [2615296,2615808) | 20260506 15:59:08.986 | INFO | MemInfo: { | 20260506 15:59:08.986 | INFO | "gpu" : " 17 GB", | 20260506 15:59:08.986 | INFO | "huge" : "", | 20260506 15:59:08.986 | INFO | "huge_shared" : " 27 GB", | 20260506 15:59:08.986 | INFO | "malloc" : " 14 GB", | 20260506 15:59:08.986 | INFO | "pinned" : " 29 GB" | 20260506 15:59:08.986 | INFO | } | 20260506 15:59:08.986 | INFO | Perf report for the last 5.13961 seconds: | 20260506 15:59:08.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:59:08.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:08.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:08.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:08.986 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 15:59:08.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:08.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:08.986 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 15:59:08.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:08.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:08.986 | INFO | Uploaded 36GiB to GPU in 1.9864s (18.1232GiB/s) | 20260506 15:59:08.986 | INFO | Downloaded 34GiB to GPU in 1.9247s (17.6651GiB/s) | 20260506 15:59:14.125 | INFO | Analyzing chunk frames = [2615808,2616320) | 20260506 15:59:14.125 | INFO | MemInfo: { | 20260506 15:59:14.125 | INFO | "gpu" : " 17 GB", | 20260506 15:59:14.125 | INFO | "huge" : "", | 20260506 15:59:14.125 | INFO | "huge_shared" : " 27 GB", | 20260506 15:59:14.125 | INFO | "malloc" : " 14 GB", | 20260506 15:59:14.125 | INFO | "pinned" : " 29 GB" | 20260506 15:59:14.125 | INFO | } | 20260506 15:59:14.125 | INFO | Perf report for the last 5.13904 seconds: | 20260506 15:59:14.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:59:14.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:14.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:14.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:14.125 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 15:59:14.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:14.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:14.125 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 15:59:14.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:14.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:14.125 | INFO | Uploaded 36GiB to GPU in 2.06226s (17.4566GiB/s) | 20260506 15:59:14.125 | INFO | Downloaded 34GiB to GPU in 1.97981s (17.1734GiB/s) | 20260506 15:59:19.264 | INFO | Analyzing chunk frames = [2616320,2616832) | 20260506 15:59:19.264 | INFO | MemInfo: { | 20260506 15:59:19.264 | INFO | "gpu" : " 17 GB", | 20260506 15:59:19.264 | INFO | "huge" : "", | 20260506 15:59:19.264 | INFO | "huge_shared" : " 27 GB", | 20260506 15:59:19.264 | INFO | "malloc" : " 14 GB", | 20260506 15:59:19.264 | INFO | "pinned" : " 29 GB" | 20260506 15:59:19.264 | INFO | } | 20260506 15:59:19.264 | INFO | Perf report for the last 5.13948 seconds: | 20260506 15:59:19.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:59:19.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:19.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:19.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:19.265 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:59:19.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:19.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:19.265 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 15:59:19.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:19.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:19.265 | INFO | Uploaded 36GiB to GPU in 2.00312s (17.9719GiB/s) | 20260506 15:59:19.265 | INFO | Downloaded 34GiB to GPU in 1.98883s (17.0955GiB/s) | 20260506 15:59:24.403 | INFO | Analyzing chunk frames = [2616832,2617344) | 20260506 15:59:24.403 | INFO | MemInfo: { | 20260506 15:59:24.403 | INFO | "gpu" : " 17 GB", | 20260506 15:59:24.403 | INFO | "huge" : "", | 20260506 15:59:24.403 | INFO | "huge_shared" : " 27 GB", | 20260506 15:59:24.404 | INFO | "malloc" : " 14 GB", | 20260506 15:59:24.404 | INFO | "pinned" : " 29 GB" | 20260506 15:59:24.404 | INFO | } | 20260506 15:59:24.404 | INFO | Perf report for the last 5.13922 seconds: | 20260506 15:59:24.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:59:24.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:24.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:24.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:24.404 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 15:59:24.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:24.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:24.404 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 15:59:24.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:24.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:24.404 | INFO | Uploaded 36GiB to GPU in 2.02671s (17.7628GiB/s) | 20260506 15:59:24.404 | INFO | Downloaded 34GiB to GPU in 1.9837s (17.1397GiB/s) | 20260506 15:59:24.536 | INFO | Starting a round of Baz IO | 20260506 15:59:27.669 | INFO | Populating RTMetrics report | 20260506 15:59:27.670 | INFO | RTMetrics report populated | 20260506 15:59:27.670 | INFO | Opening tmp RTMetrics json file | 20260506 15:59:27.677 | INFO | Serializing RTMetrics to json | 20260506 15:59:27.728 | INFO | Serialization done, swapping tmp file now... | 20260506 15:59:27.761 | INFO | RTMetrics done | 20260506 15:59:28.265 | INFO | Finished a round of Baz IO: Wrote 2.095 GiB in 3.728 seconds | 20260506 15:59:28.265 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.71% / 0.01% / 63.28% | 20260506 15:59:28.265 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.8471% / 1.58946e-05% / 5.5631e-05% | 20260506 15:59:28.265 | INFO | Baz IO overall: | 20260506 15:59:28.265 | INFO | BytesWritten : 1370090154089 | 20260506 15:59:28.265 | INFO | NumEvents : 663026557923 | 20260506 15:59:29.543 | INFO | Analyzing chunk frames = [2617344,2617856) | 20260506 15:59:29.543 | INFO | MemInfo: { | 20260506 15:59:29.543 | INFO | "gpu" : " 17 GB", | 20260506 15:59:29.543 | INFO | "huge" : "", | 20260506 15:59:29.543 | INFO | "huge_shared" : " 27 GB", | 20260506 15:59:29.543 | INFO | "malloc" : " 14 GB", | 20260506 15:59:29.543 | INFO | "pinned" : " 29 GB" | 20260506 15:59:29.543 | INFO | } | 20260506 15:59:29.543 | INFO | Perf report for the last 5.139 seconds: | 20260506 15:59:29.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:59:29.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:29.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:29.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:29.543 | INFO | ANALYSIS : 63.0%, 2.9, 95.8ms, 0 | 20260506 15:59:29.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:29.543 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 15:59:29.543 | INFO | PRE_HQ : 60.5%, 1.9, 58.4ms, 0 | 20260506 15:59:29.543 | INFO | BAZWRITER : 72.6%, 4.8, 178.9ms, 0 | 20260506 15:59:29.543 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 15:59:29.543 | INFO | Uploaded 36GiB to GPU in 2.07801s (17.3242GiB/s) | 20260506 15:59:29.543 | INFO | Downloaded 38GiB to GPU in 2.22332s (17.0915GiB/s) | 20260506 15:59:34.682 | INFO | Analyzing chunk frames = [2617856,2618368) | 20260506 15:59:34.682 | INFO | MemInfo: { | 20260506 15:59:34.682 | INFO | "gpu" : " 17 GB", | 20260506 15:59:34.682 | INFO | "huge" : "", | 20260506 15:59:34.682 | INFO | "huge_shared" : " 27 GB", | 20260506 15:59:34.682 | INFO | "malloc" : " 14 GB", | 20260506 15:59:34.682 | INFO | "pinned" : " 29 GB" | 20260506 15:59:34.682 | INFO | } | 20260506 15:59:34.683 | INFO | Perf report for the last 5.14001 seconds: | 20260506 15:59:34.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:59:34.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:34.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:34.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:34.683 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 15:59:34.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:34.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:34.683 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 15:59:34.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:34.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:34.683 | INFO | Uploaded 36GiB to GPU in 2.0033s (17.9704GiB/s) | 20260506 15:59:34.683 | INFO | Downloaded 34GiB to GPU in 1.94652s (17.467GiB/s) | 20260506 15:59:39.821 | INFO | Analyzing chunk frames = [2618368,2618880) | 20260506 15:59:39.821 | INFO | MemInfo: { | 20260506 15:59:39.821 | INFO | "gpu" : " 17 GB", | 20260506 15:59:39.821 | INFO | "huge" : "", | 20260506 15:59:39.821 | INFO | "huge_shared" : " 27 GB", | 20260506 15:59:39.821 | INFO | "malloc" : " 14 GB", | 20260506 15:59:39.821 | INFO | "pinned" : " 29 GB" | 20260506 15:59:39.821 | INFO | } | 20260506 15:59:39.821 | INFO | Perf report for the last 5.13816 seconds: | 20260506 15:59:39.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:59:39.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:39.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:39.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:39.821 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 15:59:39.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:39.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:39.821 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 15:59:39.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:39.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:39.821 | INFO | Uploaded 36GiB to GPU in 1.98829s (18.106GiB/s) | 20260506 15:59:39.821 | INFO | Downloaded 34GiB to GPU in 1.93756s (17.5479GiB/s) | 20260506 15:59:44.961 | INFO | Analyzing chunk frames = [2618880,2619392) | 20260506 15:59:44.961 | INFO | MemInfo: { | 20260506 15:59:44.961 | INFO | "gpu" : " 17 GB", | 20260506 15:59:44.961 | INFO | "huge" : "", | 20260506 15:59:44.961 | INFO | "huge_shared" : " 27 GB", | 20260506 15:59:44.961 | INFO | "malloc" : " 14 GB", | 20260506 15:59:44.961 | INFO | "pinned" : " 29 GB" | 20260506 15:59:44.961 | INFO | } | 20260506 15:59:44.961 | INFO | Perf report for the last 5.13994 seconds: | 20260506 15:59:44.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:59:44.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:44.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:44.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:44.961 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 15:59:44.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:44.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:44.961 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:59:44.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:44.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:44.961 | INFO | Uploaded 36GiB to GPU in 1.97885s (18.1924GiB/s) | 20260506 15:59:44.961 | INFO | Downloaded 34GiB to GPU in 1.88202s (18.0657GiB/s) | 20260506 15:59:50.100 | INFO | Analyzing chunk frames = [2619392,2619904) | 20260506 15:59:50.100 | INFO | MemInfo: { | 20260506 15:59:50.100 | INFO | "gpu" : " 17 GB", | 20260506 15:59:50.100 | INFO | "huge" : "", | 20260506 15:59:50.100 | INFO | "huge_shared" : " 27 GB", | 20260506 15:59:50.100 | INFO | "malloc" : " 14 GB", | 20260506 15:59:50.100 | INFO | "pinned" : " 29 GB" | 20260506 15:59:50.100 | INFO | } | 20260506 15:59:50.100 | INFO | Perf report for the last 5.13955 seconds: | 20260506 15:59:50.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:59:50.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:50.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:50.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:50.101 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 15:59:50.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:50.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:50.101 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 15:59:50.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:50.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:50.101 | INFO | Uploaded 36GiB to GPU in 2.00007s (17.9994GiB/s) | 20260506 15:59:50.101 | INFO | Downloaded 34GiB to GPU in 1.92141s (17.6953GiB/s) | 20260506 15:59:55.239 | INFO | Analyzing chunk frames = [2619904,2620416) | 20260506 15:59:55.239 | INFO | MemInfo: { | 20260506 15:59:55.239 | INFO | "gpu" : " 17 GB", | 20260506 15:59:55.239 | INFO | "huge" : "", | 20260506 15:59:55.239 | INFO | "huge_shared" : " 27 GB", | 20260506 15:59:55.239 | INFO | "malloc" : " 14 GB", | 20260506 15:59:55.239 | INFO | "pinned" : " 29 GB" | 20260506 15:59:55.239 | INFO | } | 20260506 15:59:55.239 | INFO | Perf report for the last 5.1389 seconds: | 20260506 15:59:55.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 15:59:55.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:55.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:55.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:55.239 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 15:59:55.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:55.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:55.239 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 15:59:55.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:55.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 15:59:55.240 | INFO | Uploaded 36GiB to GPU in 2.07674s (17.3349GiB/s) | 20260506 15:59:55.240 | INFO | Downloaded 34GiB to GPU in 1.98152s (17.1586GiB/s) | 20260506 16:00:00.379 | INFO | Analyzing chunk frames = [2620416,2620928) | 20260506 16:00:00.379 | INFO | MemInfo: { | 20260506 16:00:00.379 | INFO | "gpu" : " 17 GB", | 20260506 16:00:00.379 | INFO | "huge" : "", | 20260506 16:00:00.379 | INFO | "huge_shared" : " 27 GB", | 20260506 16:00:00.379 | INFO | "malloc" : " 14 GB", | 20260506 16:00:00.379 | INFO | "pinned" : " 29 GB" | 20260506 16:00:00.379 | INFO | } | 20260506 16:00:00.379 | INFO | Perf report for the last 5.13988 seconds: | 20260506 16:00:00.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:00:00.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:00.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:00.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:00.379 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:00:00.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:00.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:00.380 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 16:00:00.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:00.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:00.380 | INFO | Uploaded 36GiB to GPU in 2.00293s (17.9737GiB/s) | 20260506 16:00:00.380 | INFO | Downloaded 34GiB to GPU in 1.98416s (17.1358GiB/s) | 20260506 16:00:05.517 | INFO | Analyzing chunk frames = [2620928,2621440) | 20260506 16:00:05.517 | INFO | MemInfo: { | 20260506 16:00:05.517 | INFO | "gpu" : " 17 GB", | 20260506 16:00:05.517 | INFO | "huge" : "", | 20260506 16:00:05.517 | INFO | "huge_shared" : " 27 GB", | 20260506 16:00:05.518 | INFO | "malloc" : " 14 GB", | 20260506 16:00:05.518 | INFO | "pinned" : " 29 GB" | 20260506 16:00:05.518 | INFO | } | 20260506 16:00:05.518 | INFO | Perf report for the last 5.13837 seconds: | 20260506 16:00:05.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:00:05.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:05.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:05.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:05.518 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 16:00:05.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:05.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:05.518 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:00:05.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:05.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:05.518 | INFO | Uploaded 36GiB to GPU in 2.03062s (17.7286GiB/s) | 20260506 16:00:05.518 | INFO | Downloaded 34GiB to GPU in 1.98949s (17.0898GiB/s) | 20260506 16:00:05.518 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:00:05.518 | INFO | Total mem usage: 17.4 GiB | 20260506 16:00:05.518 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 16:00:05.518 | INFO | Recent High Water Marks Sum: 16.6 GiB (Savings: -4.89%) | 20260506 16:00:05.518 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:00:05.518 | INFO | High Water Breakdown: | 20260506 16:00:05.518 | INFO | 62.2% : 10.3 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:00:05.518 | INFO | 37.7% : 6.25 GiB ( 93.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:00:05.518 | INFO | 0.12% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:00:05.518 | INFO | ------------------------------------------ | 20260506 16:00:05.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:00:05.518 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:00:05.518 | INFO | Total mem usage: 26.5 GiB | 20260506 16:00:05.518 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:00:05.518 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:00:05.518 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:00:05.518 | INFO | High Water Breakdown: | 20260506 16:00:05.518 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:00:05.518 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:00:05.518 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:00:05.518 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:00:05.518 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:00:05.518 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:00:05.518 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:00:05.518 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:00:05.518 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:00:05.518 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:00:05.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:00:05.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:00:05.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:00:05.519 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:00:05.519 | INFO | ------------------------------------------ | 20260506 16:00:05.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:00:05.519 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:00:05.519 | INFO | Total mem usage: 25.6 GiB | 20260506 16:00:05.519 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:00:05.519 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:00:05.519 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:00:05.519 | INFO | High Water Breakdown: | 20260506 16:00:05.519 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:00:05.519 | INFO | ------------------------------------------ | 20260506 16:00:05.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:00:05.519 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:00:05.519 | INFO | Total mem usage: 15.7 GiB | 20260506 16:00:05.519 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:00:05.519 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:00:05.519 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:00:05.519 | INFO | High Water Breakdown: | 20260506 16:00:05.519 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:00:05.519 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:00:05.519 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:00:05.519 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:00:05.519 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:00:05.519 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:00:05.519 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:00:05.519 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:00:05.519 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:00:05.519 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:00:05.519 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:00:05.519 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:00:05.519 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:00:05.519 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:00:05.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:00:05.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:00:05.519 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:00:05.519 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:00:05.519 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:00:05.519 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:00:05.519 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:00:05.519 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:00:05.519 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:00:05.519 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:00:05.519 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:00:05.519 | INFO | ------------------------------------------ | 20260506 16:00:05.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:00:05.519 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:00:05.519 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:00:05.519 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:00:05.519 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.49e+01, 1.35e+01 / 1.73e+00 | 20260506 16:00:05.519 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 1.08e-02 / 8.72e-03 | 20260506 16:00:05.519 | INFO | PerfReport: 11.58%, Baseline -- 3.63e+00 / 4.43e+00, 5.54e-01 / 3.57e-01 | 20260506 16:00:05.519 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.45e+00, 1.83e-01 / 1.41e-01 | 20260506 16:00:05.519 | INFO | PerfReport: 15.84%, DME -- 3.98e+01 / 4.35e+01, 3.56e+00 / 1.65e+00 | 20260506 16:00:05.519 | INFO | PerfReport: 56.91%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.43e+00 / 1.68e+00 | 20260506 16:00:05.519 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.32e+00 / 2.89e+00, 4.03e-01 / 2.44e-01 | 20260506 16:00:05.519 | INFO | PerfReport: 3.52%, Metrics -- 1.10e+00 / 1.43e+00, 2.83e-01 / 1.84e-01 | 20260506 16:00:05.519 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.84e-01 / 4.34e-01, 5.99e-02 / 2.93e-02 | 20260506 16:00:05.519 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:00:05.519 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:00:05.519 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:00:05.519 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 16:00:05.519 | INFO | PerfReport: 21.56%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 16:00:05.519 | INFO | PerfReport: 21.24%, Download -- 2.07e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 16:00:05.519 | INFO | PerfReport: 32.23%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 16:00:05.519 | INFO | PerfReport: 1.43%, LockUpload -- 1.39e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260506 16:00:05.519 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260506 16:00:05.519 | INFO | PerfReport: 22.42%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 16:00:05.519 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260506 16:00:05.630 | INFO | Starting a round of Baz IO | 20260506 16:00:08.786 | INFO | Populating RTMetrics report | 20260506 16:00:08.786 | INFO | RTMetrics report populated | 20260506 16:00:08.786 | INFO | Opening tmp RTMetrics json file | 20260506 16:00:08.787 | INFO | Serializing RTMetrics to json | 20260506 16:00:08.788 | INFO | Serialization done, swapping tmp file now... | 20260506 16:00:08.789 | INFO | RTMetrics done | 20260506 16:00:08.984 | INFO | Finished a round of Baz IO: Wrote 2.091 GiB in 3.354 seconds | 20260506 16:00:08.984 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.70% / 0.01% / 63.29% | 20260506 16:00:08.984 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.83465% / 1.98682e-05% / 5.96046e-05% | 20260506 16:00:08.984 | INFO | Baz IO overall: | 20260506 16:00:08.984 | INFO | BytesWritten : 1372335500254 | 20260506 16:00:08.984 | INFO | NumEvents : 664111875912 | 20260506 16:00:10.657 | INFO | Analyzing chunk frames = [2621440,2621952) | 20260506 16:00:10.657 | INFO | MemInfo: { | 20260506 16:00:10.657 | INFO | "gpu" : " 17 GB", | 20260506 16:00:10.657 | INFO | "huge" : "", | 20260506 16:00:10.657 | INFO | "huge_shared" : " 27 GB", | 20260506 16:00:10.657 | INFO | "malloc" : " 14 GB", | 20260506 16:00:10.657 | INFO | "pinned" : " 29 GB" | 20260506 16:00:10.657 | INFO | } | 20260506 16:00:10.657 | INFO | Perf report for the last 5.1394 seconds: | 20260506 16:00:10.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:00:10.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:10.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:10.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:10.657 | INFO | ANALYSIS : 63.0%, 2.9, 95.4ms, 0 | 20260506 16:00:10.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:10.657 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 16:00:10.657 | INFO | PRE_HQ : 60.7%, 1.9, 59.2ms, 0 | 20260506 16:00:10.657 | INFO | BAZWRITER : 64.7%, 4.8, 160.2ms, 0 | 20260506 16:00:10.657 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 16:00:10.657 | INFO | Uploaded 36GiB to GPU in 2.0741s (17.3569GiB/s) | 20260506 16:00:10.657 | INFO | Downloaded 38GiB to GPU in 2.23221s (17.0235GiB/s) | 20260506 16:00:15.797 | INFO | Analyzing chunk frames = [2621952,2622464) | 20260506 16:00:15.797 | INFO | MemInfo: { | 20260506 16:00:15.797 | INFO | "gpu" : " 17 GB", | 20260506 16:00:15.797 | INFO | "huge" : "", | 20260506 16:00:15.797 | INFO | "huge_shared" : " 27 GB", | 20260506 16:00:15.797 | INFO | "malloc" : " 14 GB", | 20260506 16:00:15.797 | INFO | "pinned" : " 29 GB" | 20260506 16:00:15.797 | INFO | } | 20260506 16:00:15.797 | INFO | Perf report for the last 5.14012 seconds: | 20260506 16:00:15.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:00:15.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:15.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:15.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:15.797 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:00:15.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:15.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:15.797 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:00:15.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:15.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:15.798 | INFO | Uploaded 36GiB to GPU in 2.02044s (17.8179GiB/s) | 20260506 16:00:15.798 | INFO | Downloaded 34GiB to GPU in 1.95312s (17.408GiB/s) | 20260506 16:00:20.936 | INFO | Analyzing chunk frames = [2622464,2622976) | 20260506 16:00:20.936 | INFO | MemInfo: { | 20260506 16:00:20.936 | INFO | "gpu" : " 17 GB", | 20260506 16:00:20.936 | INFO | "huge" : "", | 20260506 16:00:20.936 | INFO | "huge_shared" : " 27 GB", | 20260506 16:00:20.936 | INFO | "malloc" : " 14 GB", | 20260506 16:00:20.936 | INFO | "pinned" : " 29 GB" | 20260506 16:00:20.936 | INFO | } | 20260506 16:00:20.936 | INFO | Perf report for the last 5.13921 seconds: | 20260506 16:00:20.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:00:20.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:20.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:20.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:20.936 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 16:00:20.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:20.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:20.937 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:00:20.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:20.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:20.937 | INFO | Uploaded 36GiB to GPU in 1.98743s (18.1139GiB/s) | 20260506 16:00:20.937 | INFO | Downloaded 34GiB to GPU in 1.94231s (17.505GiB/s) | 20260506 16:00:26.075 | INFO | Analyzing chunk frames = [2622976,2623488) | 20260506 16:00:26.075 | INFO | MemInfo: { | 20260506 16:00:26.075 | INFO | "gpu" : " 17 GB", | 20260506 16:00:26.075 | INFO | "huge" : "", | 20260506 16:00:26.075 | INFO | "huge_shared" : " 27 GB", | 20260506 16:00:26.075 | INFO | "malloc" : " 14 GB", | 20260506 16:00:26.075 | INFO | "pinned" : " 29 GB" | 20260506 16:00:26.075 | INFO | } | 20260506 16:00:26.076 | INFO | Perf report for the last 5.1391 seconds: | 20260506 16:00:26.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:00:26.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:26.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:26.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:26.076 | INFO | ANALYSIS : 60.6%, 3.0, 94.0ms, 0 | 20260506 16:00:26.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:26.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:26.076 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:00:26.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:26.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:26.076 | INFO | Uploaded 36GiB to GPU in 2.01277s (17.8858GiB/s) | 20260506 16:00:26.076 | INFO | Downloaded 34GiB to GPU in 1.91076s (17.7939GiB/s) | 20260506 16:00:31.214 | INFO | Analyzing chunk frames = [2623488,2624000) | 20260506 16:00:31.214 | INFO | MemInfo: { | 20260506 16:00:31.214 | INFO | "gpu" : " 17 GB", | 20260506 16:00:31.214 | INFO | "huge" : "", | 20260506 16:00:31.214 | INFO | "huge_shared" : " 27 GB", | 20260506 16:00:31.214 | INFO | "malloc" : " 14 GB", | 20260506 16:00:31.214 | INFO | "pinned" : " 29 GB" | 20260506 16:00:31.214 | INFO | } | 20260506 16:00:31.214 | INFO | Perf report for the last 5.13854 seconds: | 20260506 16:00:31.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:00:31.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:31.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:31.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:31.214 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 16:00:31.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:31.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:31.214 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:00:31.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:31.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:31.215 | INFO | Uploaded 36GiB to GPU in 2.00885s (17.9207GiB/s) | 20260506 16:00:31.215 | INFO | Downloaded 34GiB to GPU in 1.93061s (17.611GiB/s) | 20260506 16:00:36.353 | INFO | Analyzing chunk frames = [2624000,2624512) | 20260506 16:00:36.353 | INFO | MemInfo: { | 20260506 16:00:36.353 | INFO | "gpu" : " 17 GB", | 20260506 16:00:36.353 | INFO | "huge" : "", | 20260506 16:00:36.353 | INFO | "huge_shared" : " 27 GB", | 20260506 16:00:36.353 | INFO | "malloc" : " 14 GB", | 20260506 16:00:36.353 | INFO | "pinned" : " 29 GB" | 20260506 16:00:36.353 | INFO | } | 20260506 16:00:36.353 | INFO | Perf report for the last 5.13944 seconds: | 20260506 16:00:36.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:00:36.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:36.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:36.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:36.354 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 16:00:36.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:36.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:36.354 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260506 16:00:36.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:36.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:36.354 | INFO | Uploaded 36GiB to GPU in 2.03097s (17.7255GiB/s) | 20260506 16:00:36.354 | INFO | Downloaded 34GiB to GPU in 1.92355s (17.6757GiB/s) | 20260506 16:00:41.494 | INFO | Analyzing chunk frames = [2624512,2625024) | 20260506 16:00:41.494 | INFO | MemInfo: { | 20260506 16:00:41.494 | INFO | "gpu" : " 17 GB", | 20260506 16:00:41.494 | INFO | "huge" : "", | 20260506 16:00:41.494 | INFO | "huge_shared" : " 27 GB", | 20260506 16:00:41.494 | INFO | "malloc" : " 14 GB", | 20260506 16:00:41.494 | INFO | "pinned" : " 29 GB" | 20260506 16:00:41.494 | INFO | } | 20260506 16:00:41.494 | INFO | Perf report for the last 5.14069 seconds: | 20260506 16:00:41.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:00:41.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:41.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:41.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:41.494 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:00:41.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:41.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:41.494 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:00:41.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:41.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:41.495 | INFO | Uploaded 36GiB to GPU in 2.01495s (17.8665GiB/s) | 20260506 16:00:41.495 | INFO | Downloaded 34GiB to GPU in 1.987s (17.1112GiB/s) | 20260506 16:00:46.633 | INFO | Analyzing chunk frames = [2625024,2625536) | 20260506 16:00:46.633 | INFO | MemInfo: { | 20260506 16:00:46.633 | INFO | "gpu" : " 17 GB", | 20260506 16:00:46.633 | INFO | "huge" : "", | 20260506 16:00:46.633 | INFO | "huge_shared" : " 27 GB", | 20260506 16:00:46.633 | INFO | "malloc" : " 14 GB", | 20260506 16:00:46.633 | INFO | "pinned" : " 29 GB" | 20260506 16:00:46.633 | INFO | } | 20260506 16:00:46.633 | INFO | Perf report for the last 5.13878 seconds: | 20260506 16:00:46.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:00:46.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:46.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:46.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:46.633 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 16:00:46.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:46.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:46.633 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:00:46.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:46.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:46.633 | INFO | Uploaded 36GiB to GPU in 2.00408s (17.9633GiB/s) | 20260506 16:00:46.633 | INFO | Downloaded 34GiB to GPU in 1.98545s (17.1245GiB/s) | 20260506 16:00:46.750 | INFO | Starting a round of Baz IO | 20260506 16:00:49.891 | INFO | Populating RTMetrics report | 20260506 16:00:49.892 | INFO | RTMetrics report populated | 20260506 16:00:49.892 | INFO | Opening tmp RTMetrics json file | 20260506 16:00:49.893 | INFO | Serializing RTMetrics to json | 20260506 16:00:49.894 | INFO | Serialization done, swapping tmp file now... | 20260506 16:00:49.894 | INFO | RTMetrics done | 20260506 16:00:50.014 | INFO | Finished a round of Baz IO: Wrote 2.093 GiB in 3.263 seconds | 20260506 16:00:50.014 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.69% / 0.01% / 63.30% | 20260506 16:00:50.014 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.83331% / 3.97364e-06% / 5.5631e-05% | 20260506 16:00:50.014 | INFO | Baz IO overall: | 20260506 16:00:50.014 | INFO | BytesWritten : 1374582352592 | 20260506 16:00:50.014 | INFO | NumEvents : 665198648724 | 20260506 16:00:51.773 | INFO | Analyzing chunk frames = [2625536,2626048) | 20260506 16:00:51.773 | INFO | MemInfo: { | 20260506 16:00:51.773 | INFO | "gpu" : " 17 GB", | 20260506 16:00:51.773 | INFO | "huge" : "", | 20260506 16:00:51.773 | INFO | "huge_shared" : " 27 GB", | 20260506 16:00:51.773 | INFO | "malloc" : " 14 GB", | 20260506 16:00:51.773 | INFO | "pinned" : " 29 GB" | 20260506 16:00:51.773 | INFO | } | 20260506 16:00:51.773 | INFO | Perf report for the last 5.14 seconds: | 20260506 16:00:51.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:00:51.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:51.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:51.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:51.773 | INFO | ANALYSIS : 62.8%, 2.9, 95.4ms, 0 | 20260506 16:00:51.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:51.773 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 16:00:51.773 | INFO | PRE_HQ : 60.6%, 1.9, 58.8ms, 0 | 20260506 16:00:51.773 | INFO | BAZWRITER : 63.1%, 4.7, 153.7ms, 0 | 20260506 16:00:51.773 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 16:00:51.773 | INFO | Uploaded 36GiB to GPU in 2.05084s (17.5538GiB/s) | 20260506 16:00:51.773 | INFO | Downloaded 38GiB to GPU in 2.22341s (17.0909GiB/s) | 20260506 16:00:56.911 | INFO | Analyzing chunk frames = [2626048,2626560) | 20260506 16:00:56.911 | INFO | MemInfo: { | 20260506 16:00:56.911 | INFO | "gpu" : " 17 GB", | 20260506 16:00:56.911 | INFO | "huge" : "", | 20260506 16:00:56.911 | INFO | "huge_shared" : " 27 GB", | 20260506 16:00:56.911 | INFO | "malloc" : " 14 GB", | 20260506 16:00:56.911 | INFO | "pinned" : " 29 GB" | 20260506 16:00:56.912 | INFO | } | 20260506 16:00:56.912 | INFO | Perf report for the last 5.13855 seconds: | 20260506 16:00:56.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:00:56.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:56.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:56.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:56.912 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:00:56.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:56.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:56.912 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 16:00:56.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:56.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:00:56.912 | INFO | Uploaded 36GiB to GPU in 1.98044s (18.1777GiB/s) | 20260506 16:00:56.912 | INFO | Downloaded 34GiB to GPU in 1.93969s (17.5286GiB/s) | 20260506 16:01:02.051 | INFO | Analyzing chunk frames = [2626560,2627072) | 20260506 16:01:02.051 | INFO | MemInfo: { | 20260506 16:01:02.051 | INFO | "gpu" : " 17 GB", | 20260506 16:01:02.051 | INFO | "huge" : "", | 20260506 16:01:02.051 | INFO | "huge_shared" : " 27 GB", | 20260506 16:01:02.051 | INFO | "malloc" : " 14 GB", | 20260506 16:01:02.051 | INFO | "pinned" : " 29 GB" | 20260506 16:01:02.051 | INFO | } | 20260506 16:01:02.051 | INFO | Perf report for the last 5.1397 seconds: | 20260506 16:01:02.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:01:02.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:02.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:02.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:02.051 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 16:01:02.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:02.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:02.051 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 16:01:02.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:02.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:02.052 | INFO | Uploaded 36GiB to GPU in 1.96953s (18.2784GiB/s) | 20260506 16:01:02.052 | INFO | Downloaded 34GiB to GPU in 1.93391s (17.581GiB/s) | 20260506 16:01:07.190 | INFO | Analyzing chunk frames = [2627072,2627584) | 20260506 16:01:07.190 | INFO | MemInfo: { | 20260506 16:01:07.190 | INFO | "gpu" : " 17 GB", | 20260506 16:01:07.190 | INFO | "huge" : "", | 20260506 16:01:07.190 | INFO | "huge_shared" : " 27 GB", | 20260506 16:01:07.190 | INFO | "malloc" : " 14 GB", | 20260506 16:01:07.190 | INFO | "pinned" : " 29 GB" | 20260506 16:01:07.190 | INFO | } | 20260506 16:01:07.190 | INFO | Perf report for the last 5.13882 seconds: | 20260506 16:01:07.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:01:07.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:07.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:07.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:07.190 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 16:01:07.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:07.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:07.190 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:01:07.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:07.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:07.190 | INFO | Uploaded 36GiB to GPU in 1.95414s (18.4224GiB/s) | 20260506 16:01:07.190 | INFO | Downloaded 34GiB to GPU in 1.88083s (18.0771GiB/s) | 20260506 16:01:12.329 | INFO | Analyzing chunk frames = [2627584,2628096) | 20260506 16:01:12.329 | INFO | MemInfo: { | 20260506 16:01:12.329 | INFO | "gpu" : " 17 GB", | 20260506 16:01:12.329 | INFO | "huge" : "", | 20260506 16:01:12.329 | INFO | "huge_shared" : " 27 GB", | 20260506 16:01:12.329 | INFO | "malloc" : " 14 GB", | 20260506 16:01:12.329 | INFO | "pinned" : " 29 GB" | 20260506 16:01:12.329 | INFO | } | 20260506 16:01:12.329 | INFO | Perf report for the last 5.13923 seconds: | 20260506 16:01:12.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:01:12.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:12.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:12.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:12.329 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 16:01:12.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:12.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:12.329 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:01:12.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:12.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:12.330 | INFO | Uploaded 36GiB to GPU in 1.98947s (18.0953GiB/s) | 20260506 16:01:12.330 | INFO | Downloaded 34GiB to GPU in 1.93592s (17.5627GiB/s) | 20260506 16:01:17.468 | INFO | Analyzing chunk frames = [2628096,2628608) | 20260506 16:01:17.468 | INFO | MemInfo: { | 20260506 16:01:17.468 | INFO | "gpu" : " 17 GB", | 20260506 16:01:17.468 | INFO | "huge" : "", | 20260506 16:01:17.468 | INFO | "huge_shared" : " 27 GB", | 20260506 16:01:17.469 | INFO | "malloc" : " 14 GB", | 20260506 16:01:17.469 | INFO | "pinned" : " 29 GB" | 20260506 16:01:17.469 | INFO | } | 20260506 16:01:17.469 | INFO | Perf report for the last 5.13923 seconds: | 20260506 16:01:17.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:01:17.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:17.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:17.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:17.469 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:01:17.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:17.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:17.469 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260506 16:01:17.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:17.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:17.469 | INFO | Uploaded 36GiB to GPU in 2.06476s (17.4355GiB/s) | 20260506 16:01:17.469 | INFO | Downloaded 34GiB to GPU in 1.98205s (17.1539GiB/s) | 20260506 16:01:22.609 | INFO | Analyzing chunk frames = [2628608,2629120) | 20260506 16:01:22.609 | INFO | MemInfo: { | 20260506 16:01:22.609 | INFO | "gpu" : " 17 GB", | 20260506 16:01:22.609 | INFO | "huge" : "", | 20260506 16:01:22.609 | INFO | "huge_shared" : " 27 GB", | 20260506 16:01:22.609 | INFO | "malloc" : " 14 GB", | 20260506 16:01:22.609 | INFO | "pinned" : " 29 GB" | 20260506 16:01:22.609 | INFO | } | 20260506 16:01:22.609 | INFO | Perf report for the last 5.14083 seconds: | 20260506 16:01:22.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:01:22.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:22.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:22.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:22.610 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:01:22.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:22.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:22.610 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 16:01:22.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:22.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:22.610 | INFO | Uploaded 36GiB to GPU in 2.01129s (17.899GiB/s) | 20260506 16:01:22.610 | INFO | Downloaded 34GiB to GPU in 1.99045s (17.0815GiB/s) | 20260506 16:01:27.747 | INFO | Analyzing chunk frames = [2629120,2629632) | 20260506 16:01:27.747 | INFO | MemInfo: { | 20260506 16:01:27.747 | INFO | "gpu" : " 17 GB", | 20260506 16:01:27.747 | INFO | "huge" : "", | 20260506 16:01:27.747 | INFO | "huge_shared" : " 27 GB", | 20260506 16:01:27.747 | INFO | "malloc" : " 14 GB", | 20260506 16:01:27.747 | INFO | "pinned" : " 29 GB" | 20260506 16:01:27.747 | INFO | } | 20260506 16:01:27.748 | INFO | Perf report for the last 5.13804 seconds: | 20260506 16:01:27.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:01:27.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:27.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:27.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:27.748 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 16:01:27.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:27.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:27.748 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:01:27.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:27.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:27.748 | INFO | Uploaded 36GiB to GPU in 2.02527s (17.7754GiB/s) | 20260506 16:01:27.748 | INFO | Downloaded 34GiB to GPU in 1.98891s (17.0948GiB/s) | 20260506 16:01:27.748 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:01:27.748 | INFO | Total mem usage: 17.4 GiB | 20260506 16:01:27.748 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 16:01:27.748 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -13.1%) | 20260506 16:01:27.748 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:01:27.748 | INFO | High Water Breakdown: | 20260506 16:01:27.748 | INFO | 61.1% : 9.4 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:01:27.748 | INFO | 38.8% : 5.97 GiB ( 89.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:01:27.748 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:01:27.748 | INFO | ------------------------------------------ | 20260506 16:01:27.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:01:27.748 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:01:27.748 | INFO | Total mem usage: 26.5 GiB | 20260506 16:01:27.748 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:01:27.748 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:01:27.748 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:01:27.748 | INFO | High Water Breakdown: | 20260506 16:01:27.748 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:01:27.748 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:01:27.748 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:01:27.748 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:01:27.748 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:01:27.748 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:01:27.748 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:01:27.748 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:01:27.748 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:01:27.748 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:01:27.748 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:01:27.748 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:01:27.748 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:01:27.748 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:01:27.748 | INFO | ------------------------------------------ | 20260506 16:01:27.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:01:27.748 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:01:27.748 | INFO | Total mem usage: 25.6 GiB | 20260506 16:01:27.748 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:01:27.748 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:01:27.748 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:01:27.748 | INFO | High Water Breakdown: | 20260506 16:01:27.748 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:01:27.748 | INFO | ------------------------------------------ | 20260506 16:01:27.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:01:27.748 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:01:27.749 | INFO | Total mem usage: 15.7 GiB | 20260506 16:01:27.749 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:01:27.749 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:01:27.749 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:01:27.749 | INFO | High Water Breakdown: | 20260506 16:01:27.749 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:01:27.749 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:01:27.749 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:01:27.749 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:01:27.749 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:01:27.749 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:01:27.749 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:01:27.749 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:01:27.749 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:01:27.749 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:01:27.749 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:01:27.749 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:01:27.749 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:01:27.749 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:01:27.749 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:01:27.749 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:01:27.749 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:01:27.749 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:01:27.749 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:01:27.749 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:01:27.749 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:01:27.749 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:01:27.749 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:01:27.749 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:01:27.749 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:01:27.749 | INFO | ------------------------------------------ | 20260506 16:01:27.749 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:01:27.749 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:01:27.749 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:01:27.749 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:01:27.749 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.53e+01, 1.32e+01 / 1.88e+00 | 20260506 16:01:27.749 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.03e-01, 1.03e-02 / 1.17e-02 | 20260506 16:01:27.749 | INFO | PerfReport: 11.59%, Baseline -- 3.64e+00 / 4.42e+00, 5.57e-01 / 3.39e-01 | 20260506 16:01:27.749 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.47e+00, 1.83e-01 / 1.39e-01 | 20260506 16:01:27.749 | INFO | PerfReport: 15.85%, DME -- 3.98e+01 / 4.39e+01, 3.71e+00 / 2.02e+00 | 20260506 16:01:27.749 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.45e+00 / 2.11e+00 | 20260506 16:01:27.749 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.31e+00 / 2.90e+00, 3.97e-01 / 2.50e-01 | 20260506 16:01:27.749 | INFO | PerfReport: 3.52%, Metrics -- 1.10e+00 / 1.47e+00, 2.85e-01 / 1.94e-01 | 20260506 16:01:27.749 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.84e-01 / 4.39e-01, 6.03e-02 / 2.61e-02 | 20260506 16:01:27.749 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:01:27.749 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:01:27.749 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:01:27.749 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 16:01:27.749 | INFO | PerfReport: 21.53%, Upload -- 2.10e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 16:01:27.749 | INFO | PerfReport: 21.22%, Download -- 2.07e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 16:01:27.749 | INFO | PerfReport: 32.22%, Compute -- 3.14e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 16:01:27.749 | INFO | PerfReport: 1.42%, LockUpload -- 1.38e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260506 16:01:27.749 | INFO | PerfReport: 1.15%, LockDownload -- 1.12e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260506 16:01:27.749 | INFO | PerfReport: 22.45%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 16:01:27.749 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.22e-02 / 0.00e+00 | 20260506 16:01:27.859 | INFO | Starting a round of Baz IO | 20260506 16:01:31.014 | INFO | Populating RTMetrics report | 20260506 16:01:31.014 | INFO | RTMetrics report populated | 20260506 16:01:31.014 | INFO | Opening tmp RTMetrics json file | 20260506 16:01:31.050 | INFO | Serializing RTMetrics to json | 20260506 16:01:31.056 | INFO | Serialization done, swapping tmp file now... | 20260506 16:01:31.064 | INFO | RTMetrics done | 20260506 16:01:31.340 | INFO | Finished a round of Baz IO: Wrote 2.094 GiB in 3.481 seconds | 20260506 16:01:31.340 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.68% / 0.01% / 63.31% | 20260506 16:01:31.340 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.83196% / 7.94729e-06% / 6.35783e-05% | 20260506 16:01:31.340 | INFO | Baz IO overall: | 20260506 16:01:31.340 | INFO | BytesWritten : 1376831083337 | 20260506 16:01:31.340 | INFO | NumEvents : 666286962042 | 20260506 16:01:32.887 | INFO | Analyzing chunk frames = [2629632,2630144) | 20260506 16:01:32.887 | INFO | MemInfo: { | 20260506 16:01:32.887 | INFO | "gpu" : " 17 GB", | 20260506 16:01:32.887 | INFO | "huge" : "", | 20260506 16:01:32.887 | INFO | "huge_shared" : " 27 GB", | 20260506 16:01:32.887 | INFO | "malloc" : " 14 GB", | 20260506 16:01:32.887 | INFO | "pinned" : " 29 GB" | 20260506 16:01:32.887 | INFO | } | 20260506 16:01:32.887 | INFO | Perf report for the last 5.13997 seconds: | 20260506 16:01:32.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:01:32.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:32.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:32.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:32.888 | INFO | ANALYSIS : 62.9%, 2.9, 95.7ms, 0 | 20260506 16:01:32.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:32.888 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 16:01:32.888 | INFO | PRE_HQ : 60.9%, 1.9, 59.0ms, 0 | 20260506 16:01:32.888 | INFO | BAZWRITER : 67.1%, 4.8, 166.2ms, 0 | 20260506 16:01:32.888 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.5ms, 0 | 20260506 16:01:32.888 | INFO | Uploaded 36GiB to GPU in 2.05906s (17.4837GiB/s) | 20260506 16:01:32.888 | INFO | Downloaded 38GiB to GPU in 2.22263s (17.0969GiB/s) | 20260506 16:01:38.026 | INFO | Analyzing chunk frames = [2630144,2630656) | 20260506 16:01:38.026 | INFO | MemInfo: { | 20260506 16:01:38.026 | INFO | "gpu" : " 17 GB", | 20260506 16:01:38.026 | INFO | "huge" : "", | 20260506 16:01:38.026 | INFO | "huge_shared" : " 27 GB", | 20260506 16:01:38.026 | INFO | "malloc" : " 14 GB", | 20260506 16:01:38.026 | INFO | "pinned" : " 29 GB" | 20260506 16:01:38.026 | INFO | } | 20260506 16:01:38.026 | INFO | Perf report for the last 5.13868 seconds: | 20260506 16:01:38.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:01:38.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:38.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:38.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:38.026 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:01:38.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:38.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:38.026 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 16:01:38.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:38.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:38.026 | INFO | Uploaded 36GiB to GPU in 2.0024s (17.9784GiB/s) | 20260506 16:01:38.026 | INFO | Downloaded 34GiB to GPU in 1.95058s (17.4307GiB/s) | 20260506 16:01:43.166 | INFO | Analyzing chunk frames = [2630656,2631168) | 20260506 16:01:43.166 | INFO | MemInfo: { | 20260506 16:01:43.166 | INFO | "gpu" : " 17 GB", | 20260506 16:01:43.166 | INFO | "huge" : "", | 20260506 16:01:43.166 | INFO | "huge_shared" : " 27 GB", | 20260506 16:01:43.166 | INFO | "malloc" : " 14 GB", | 20260506 16:01:43.166 | INFO | "pinned" : " 29 GB" | 20260506 16:01:43.166 | INFO | } | 20260506 16:01:43.166 | INFO | Perf report for the last 5.13961 seconds: | 20260506 16:01:43.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:01:43.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:43.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:43.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:43.166 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 16:01:43.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:43.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:43.166 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:01:43.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:43.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:43.166 | INFO | Uploaded 36GiB to GPU in 1.97527s (18.2253GiB/s) | 20260506 16:01:43.166 | INFO | Downloaded 34GiB to GPU in 1.94057s (17.5207GiB/s) | 20260506 16:01:48.306 | INFO | Analyzing chunk frames = [2631168,2631680) | 20260506 16:01:48.306 | INFO | MemInfo: { | 20260506 16:01:48.306 | INFO | "gpu" : " 17 GB", | 20260506 16:01:48.306 | INFO | "huge" : "", | 20260506 16:01:48.306 | INFO | "huge_shared" : " 27 GB", | 20260506 16:01:48.306 | INFO | "malloc" : " 14 GB", | 20260506 16:01:48.306 | INFO | "pinned" : " 29 GB" | 20260506 16:01:48.306 | INFO | } | 20260506 16:01:48.306 | INFO | Perf report for the last 5.14029 seconds: | 20260506 16:01:48.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:01:48.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:48.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:48.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:48.306 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 16:01:48.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:48.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:48.306 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:01:48.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:48.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:48.306 | INFO | Uploaded 36GiB to GPU in 1.97232s (18.2526GiB/s) | 20260506 16:01:48.306 | INFO | Downloaded 34GiB to GPU in 1.87554s (18.1281GiB/s) | 20260506 16:01:53.444 | INFO | Analyzing chunk frames = [2631680,2632192) | 20260506 16:01:53.444 | INFO | MemInfo: { | 20260506 16:01:53.444 | INFO | "gpu" : " 17 GB", | 20260506 16:01:53.444 | INFO | "huge" : "", | 20260506 16:01:53.445 | INFO | "huge_shared" : " 27 GB", | 20260506 16:01:53.445 | INFO | "malloc" : " 14 GB", | 20260506 16:01:53.445 | INFO | "pinned" : " 29 GB" | 20260506 16:01:53.445 | INFO | } | 20260506 16:01:53.445 | INFO | Perf report for the last 5.13861 seconds: | 20260506 16:01:53.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:01:53.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:53.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:53.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:53.445 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 16:01:53.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:53.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:53.445 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:01:53.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:53.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:53.445 | INFO | Uploaded 36GiB to GPU in 1.97053s (18.2692GiB/s) | 20260506 16:01:53.445 | INFO | Downloaded 34GiB to GPU in 1.92122s (17.6971GiB/s) | 20260506 16:01:58.583 | INFO | Analyzing chunk frames = [2632192,2632704) | 20260506 16:01:58.583 | INFO | MemInfo: { | 20260506 16:01:58.583 | INFO | "gpu" : " 17 GB", | 20260506 16:01:58.583 | INFO | "huge" : "", | 20260506 16:01:58.584 | INFO | "huge_shared" : " 27 GB", | 20260506 16:01:58.584 | INFO | "malloc" : " 14 GB", | 20260506 16:01:58.584 | INFO | "pinned" : " 29 GB" | 20260506 16:01:58.584 | INFO | } | 20260506 16:01:58.584 | INFO | Perf report for the last 5.13894 seconds: | 20260506 16:01:58.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:01:58.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:58.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:58.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:58.584 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:01:58.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:58.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:58.584 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 16:01:58.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:58.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:01:58.584 | INFO | Uploaded 36GiB to GPU in 2.07211s (17.3736GiB/s) | 20260506 16:01:58.584 | INFO | Downloaded 34GiB to GPU in 1.96973s (17.2612GiB/s) | 20260506 16:02:03.724 | INFO | Analyzing chunk frames = [2632704,2633216) | 20260506 16:02:03.724 | INFO | MemInfo: { | 20260506 16:02:03.724 | INFO | "gpu" : " 17 GB", | 20260506 16:02:03.724 | INFO | "huge" : "", | 20260506 16:02:03.724 | INFO | "huge_shared" : " 27 GB", | 20260506 16:02:03.724 | INFO | "malloc" : " 14 GB", | 20260506 16:02:03.724 | INFO | "pinned" : " 29 GB" | 20260506 16:02:03.724 | INFO | } | 20260506 16:02:03.724 | INFO | Perf report for the last 5.14034 seconds: | 20260506 16:02:03.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:02:03.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:03.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:03.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:03.724 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:02:03.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:03.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:03.724 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 16:02:03.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:03.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:03.724 | INFO | Uploaded 36GiB to GPU in 2.02619s (17.7673GiB/s) | 20260506 16:02:03.724 | INFO | Downloaded 34GiB to GPU in 1.9937s (17.0537GiB/s) | 20260506 16:02:08.863 | INFO | Analyzing chunk frames = [2633216,2633728) | 20260506 16:02:08.863 | INFO | MemInfo: { | 20260506 16:02:08.863 | INFO | "gpu" : " 17 GB", | 20260506 16:02:08.863 | INFO | "huge" : "", | 20260506 16:02:08.864 | INFO | "huge_shared" : " 27 GB", | 20260506 16:02:08.864 | INFO | "malloc" : " 14 GB", | 20260506 16:02:08.864 | INFO | "pinned" : " 29 GB" | 20260506 16:02:08.864 | INFO | } | 20260506 16:02:08.864 | INFO | Perf report for the last 5.13962 seconds: | 20260506 16:02:08.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:02:08.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:08.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:08.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:08.864 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:02:08.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:08.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:08.864 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:02:08.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:08.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:08.864 | INFO | Uploaded 36GiB to GPU in 2.01389s (17.8758GiB/s) | 20260506 16:02:08.864 | INFO | Downloaded 34GiB to GPU in 1.99372s (17.0536GiB/s) | 20260506 16:02:08.983 | INFO | Starting a round of Baz IO | 20260506 16:02:12.122 | INFO | Populating RTMetrics report | 20260506 16:02:12.122 | INFO | RTMetrics report populated | 20260506 16:02:12.122 | INFO | Opening tmp RTMetrics json file | 20260506 16:02:12.125 | INFO | Serializing RTMetrics to json | 20260506 16:02:12.135 | INFO | Serialization done, swapping tmp file now... | 20260506 16:02:12.159 | INFO | RTMetrics done | 20260506 16:02:12.330 | INFO | Finished a round of Baz IO: Wrote 2.09 GiB in 3.347 seconds | 20260506 16:02:12.330 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.67% / 0.01% / 63.32% | 20260506 16:02:12.330 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.82114% / 1.58946e-05% / 4.76837e-05% | 20260506 16:02:12.330 | INFO | Baz IO overall: | 20260506 16:02:12.330 | INFO | BytesWritten : 1379075049578 | 20260506 16:02:12.330 | INFO | NumEvents : 667372122594 | 20260506 16:02:14.002 | INFO | Analyzing chunk frames = [2633728,2634240) | 20260506 16:02:14.002 | INFO | MemInfo: { | 20260506 16:02:14.002 | INFO | "gpu" : " 17 GB", | 20260506 16:02:14.002 | INFO | "huge" : "", | 20260506 16:02:14.002 | INFO | "huge_shared" : " 27 GB", | 20260506 16:02:14.002 | INFO | "malloc" : " 14 GB", | 20260506 16:02:14.002 | INFO | "pinned" : " 29 GB" | 20260506 16:02:14.002 | INFO | } | 20260506 16:02:14.002 | INFO | Perf report for the last 5.1384 seconds: | 20260506 16:02:14.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:02:14.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:14.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:14.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:14.002 | INFO | ANALYSIS : 62.8%, 2.9, 95.4ms, 0 | 20260506 16:02:14.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:14.002 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 16:02:14.002 | INFO | PRE_HQ : 60.6%, 1.9, 58.6ms, 0 | 20260506 16:02:14.002 | INFO | BAZWRITER : 64.7%, 4.8, 160.6ms, 0 | 20260506 16:02:14.002 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 16:02:14.002 | INFO | Uploaded 36GiB to GPU in 2.05177s (17.5458GiB/s) | 20260506 16:02:14.002 | INFO | Downloaded 38GiB to GPU in 2.23528s (17.0001GiB/s) | 20260506 16:02:19.141 | INFO | Analyzing chunk frames = [2634240,2634752) | 20260506 16:02:19.141 | INFO | MemInfo: { | 20260506 16:02:19.141 | INFO | "gpu" : " 17 GB", | 20260506 16:02:19.141 | INFO | "huge" : "", | 20260506 16:02:19.141 | INFO | "huge_shared" : " 27 GB", | 20260506 16:02:19.141 | INFO | "malloc" : " 14 GB", | 20260506 16:02:19.141 | INFO | "pinned" : " 29 GB" | 20260506 16:02:19.141 | INFO | } | 20260506 16:02:19.141 | INFO | Perf report for the last 5.13934 seconds: | 20260506 16:02:19.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:02:19.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:19.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:19.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:19.141 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:02:19.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:19.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:19.142 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:02:19.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:19.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:19.142 | INFO | Uploaded 36GiB to GPU in 1.98316s (18.1528GiB/s) | 20260506 16:02:19.142 | INFO | Downloaded 34GiB to GPU in 1.93129s (17.6048GiB/s) | 20260506 16:02:24.280 | INFO | Analyzing chunk frames = [2634752,2635264) | 20260506 16:02:24.280 | INFO | MemInfo: { | 20260506 16:02:24.280 | INFO | "gpu" : " 17 GB", | 20260506 16:02:24.280 | INFO | "huge" : "", | 20260506 16:02:24.280 | INFO | "huge_shared" : " 27 GB", | 20260506 16:02:24.280 | INFO | "malloc" : " 14 GB", | 20260506 16:02:24.280 | INFO | "pinned" : " 29 GB" | 20260506 16:02:24.280 | INFO | } | 20260506 16:02:24.280 | INFO | Perf report for the last 5.1389 seconds: | 20260506 16:02:24.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:02:24.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:24.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:24.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:24.280 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 16:02:24.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:24.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:24.280 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:02:24.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:24.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:24.281 | INFO | Uploaded 36GiB to GPU in 1.96327s (18.3368GiB/s) | 20260506 16:02:24.281 | INFO | Downloaded 34GiB to GPU in 1.93673s (17.5554GiB/s) | 20260506 16:02:29.420 | INFO | Analyzing chunk frames = [2635264,2635776) | 20260506 16:02:29.420 | INFO | MemInfo: { | 20260506 16:02:29.420 | INFO | "gpu" : " 17 GB", | 20260506 16:02:29.420 | INFO | "huge" : "", | 20260506 16:02:29.420 | INFO | "huge_shared" : " 27 GB", | 20260506 16:02:29.420 | INFO | "malloc" : " 14 GB", | 20260506 16:02:29.420 | INFO | "pinned" : " 29 GB" | 20260506 16:02:29.420 | INFO | } | 20260506 16:02:29.420 | INFO | Perf report for the last 5.13957 seconds: | 20260506 16:02:29.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:02:29.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:29.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:29.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:29.420 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 16:02:29.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:29.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:29.420 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:02:29.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:29.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:29.420 | INFO | Uploaded 36GiB to GPU in 1.95646s (18.4006GiB/s) | 20260506 16:02:29.420 | INFO | Downloaded 34GiB to GPU in 1.883s (18.0563GiB/s) | 20260506 16:02:34.559 | INFO | Analyzing chunk frames = [2635776,2636288) | 20260506 16:02:34.559 | INFO | MemInfo: { | 20260506 16:02:34.559 | INFO | "gpu" : " 17 GB", | 20260506 16:02:34.559 | INFO | "huge" : "", | 20260506 16:02:34.559 | INFO | "huge_shared" : " 27 GB", | 20260506 16:02:34.559 | INFO | "malloc" : " 14 GB", | 20260506 16:02:34.559 | INFO | "pinned" : " 29 GB" | 20260506 16:02:34.559 | INFO | } | 20260506 16:02:34.559 | INFO | Perf report for the last 5.13921 seconds: | 20260506 16:02:34.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:02:34.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:34.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:34.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:34.559 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 16:02:34.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:34.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:34.559 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:02:34.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:34.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:34.559 | INFO | Uploaded 36GiB to GPU in 1.9916s (18.0759GiB/s) | 20260506 16:02:34.560 | INFO | Downloaded 34GiB to GPU in 1.92462s (17.6658GiB/s) | 20260506 16:02:39.697 | INFO | Analyzing chunk frames = [2636288,2636800) | 20260506 16:02:39.698 | INFO | MemInfo: { | 20260506 16:02:39.698 | INFO | "gpu" : " 17 GB", | 20260506 16:02:39.698 | INFO | "huge" : "", | 20260506 16:02:39.698 | INFO | "huge_shared" : " 27 GB", | 20260506 16:02:39.698 | INFO | "malloc" : " 14 GB", | 20260506 16:02:39.698 | INFO | "pinned" : " 29 GB" | 20260506 16:02:39.698 | INFO | } | 20260506 16:02:39.698 | INFO | Perf report for the last 5.13863 seconds: | 20260506 16:02:39.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:02:39.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:39.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:39.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:39.698 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 16:02:39.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:39.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:39.698 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 16:02:39.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:39.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:39.698 | INFO | Uploaded 36GiB to GPU in 2.06211s (17.4579GiB/s) | 20260506 16:02:39.698 | INFO | Downloaded 34GiB to GPU in 1.97607s (17.2059GiB/s) | 20260506 16:02:44.839 | INFO | Analyzing chunk frames = [2636800,2637312) | 20260506 16:02:44.839 | INFO | MemInfo: { | 20260506 16:02:44.839 | INFO | "gpu" : " 17 GB", | 20260506 16:02:44.839 | INFO | "huge" : "", | 20260506 16:02:44.839 | INFO | "huge_shared" : " 27 GB", | 20260506 16:02:44.839 | INFO | "malloc" : " 14 GB", | 20260506 16:02:44.839 | INFO | "pinned" : " 29 GB" | 20260506 16:02:44.839 | INFO | } | 20260506 16:02:44.839 | INFO | Perf report for the last 5.14108 seconds: | 20260506 16:02:44.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:02:44.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:44.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:44.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:44.839 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:02:44.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:44.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:44.839 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:02:44.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:44.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:44.839 | INFO | Uploaded 36GiB to GPU in 2.00184s (17.9835GiB/s) | 20260506 16:02:44.839 | INFO | Downloaded 34GiB to GPU in 2.00795s (16.9327GiB/s) | 20260506 16:02:49.977 | INFO | Analyzing chunk frames = [2637312,2637824) | 20260506 16:02:49.977 | INFO | MemInfo: { | 20260506 16:02:49.977 | INFO | "gpu" : " 17 GB", | 20260506 16:02:49.977 | INFO | "huge" : "", | 20260506 16:02:49.977 | INFO | "huge_shared" : " 27 GB", | 20260506 16:02:49.977 | INFO | "malloc" : " 14 GB", | 20260506 16:02:49.977 | INFO | "pinned" : " 29 GB" | 20260506 16:02:49.977 | INFO | } | 20260506 16:02:49.978 | INFO | Perf report for the last 5.13866 seconds: | 20260506 16:02:49.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:02:49.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:49.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:49.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:49.978 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 16:02:49.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:49.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:49.978 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:02:49.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:49.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:49.978 | INFO | Uploaded 36GiB to GPU in 2.0259s (17.7699GiB/s) | 20260506 16:02:49.978 | INFO | Downloaded 34GiB to GPU in 1.99326s (17.0575GiB/s) | 20260506 16:02:49.978 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:02:49.978 | INFO | Total mem usage: 17.4 GiB | 20260506 16:02:49.978 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 16:02:49.978 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -9.96%) | 20260506 16:02:49.978 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:02:49.978 | INFO | High Water Breakdown: | 20260506 16:02:49.978 | INFO | 61.5% : 9.74 GiB ( 89.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:02:49.978 | INFO | 38.4% : 6.07 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:02:49.978 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:02:49.978 | INFO | ------------------------------------------ | 20260506 16:02:49.978 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:02:49.978 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:02:49.978 | INFO | Total mem usage: 26.5 GiB | 20260506 16:02:49.978 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:02:49.978 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:02:49.978 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:02:49.978 | INFO | High Water Breakdown: | 20260506 16:02:49.978 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:02:49.978 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:02:49.978 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:02:49.978 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:02:49.978 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:02:49.978 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:02:49.978 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:02:49.978 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:02:49.978 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:02:49.978 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:02:49.978 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:02:49.978 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:02:49.978 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:02:49.978 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:02:49.979 | INFO | ------------------------------------------ | 20260506 16:02:49.979 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:02:49.979 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:02:49.979 | INFO | Total mem usage: 25.6 GiB | 20260506 16:02:49.979 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:02:49.979 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:02:49.979 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:02:49.979 | INFO | High Water Breakdown: | 20260506 16:02:49.979 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:02:49.979 | INFO | ------------------------------------------ | 20260506 16:02:49.979 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:02:49.979 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:02:49.979 | INFO | Total mem usage: 15.7 GiB | 20260506 16:02:49.979 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:02:49.979 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:02:49.979 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:02:49.979 | INFO | High Water Breakdown: | 20260506 16:02:49.979 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:02:49.979 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:02:49.979 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:02:49.979 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:02:49.979 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:02:49.979 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:02:49.979 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:02:49.979 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:02:49.979 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:02:49.979 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:02:49.979 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:02:49.979 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:02:49.979 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:02:49.979 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:02:49.979 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:02:49.979 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:02:49.979 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:02:49.979 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:02:49.979 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:02:49.979 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:02:49.979 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:02:49.979 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:02:49.979 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:02:49.979 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:02:49.979 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:02:49.979 | INFO | ------------------------------------------ | 20260506 16:02:49.979 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:02:49.979 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:02:49.979 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:02:49.979 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:02:49.979 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.46e+01, 1.34e+01 / 1.82e+00 | 20260506 16:02:49.979 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 1.06e-02 / 5.86e-03 | 20260506 16:02:49.979 | INFO | PerfReport: 11.60%, Baseline -- 3.63e+00 / 4.40e+00, 5.50e-01 / 3.18e-01 | 20260506 16:02:49.979 | INFO | PerfReport: 3.23%, Histogram -- 1.01e+00 / 1.45e+00, 1.80e-01 / 1.41e-01 | 20260506 16:02:49.979 | INFO | PerfReport: 15.87%, DME -- 3.98e+01 / 4.36e+01, 3.60e+00 / 1.84e+00 | 20260506 16:02:49.979 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.38e+00 / 1.83e+00 | 20260506 16:02:49.979 | INFO | PerfReport: 7.39%, PulseAccumulating -- 2.32e+00 / 2.85e+00, 4.05e-01 / 2.74e-01 | 20260506 16:02:49.979 | INFO | PerfReport: 3.52%, Metrics -- 1.10e+00 / 1.45e+00, 2.87e-01 / 2.21e-01 | 20260506 16:02:49.979 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.84e-01 / 4.36e-01, 6.02e-02 / 3.42e-02 | 20260506 16:02:49.979 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:02:49.979 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:02:49.979 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:02:49.979 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 16:02:49.979 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 16:02:49.979 | INFO | PerfReport: 21.25%, Download -- 2.07e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 16:02:49.979 | INFO | PerfReport: 32.21%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 16:02:49.979 | INFO | PerfReport: 1.36%, LockUpload -- 1.33e+00 / 0.00e+00, 5.49e+00 / 0.00e+00 | 20260506 16:02:49.979 | INFO | PerfReport: 1.20%, LockDownload -- 1.17e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 16:02:49.979 | INFO | PerfReport: 22.46%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 16:02:49.979 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260506 16:02:50.089 | INFO | Starting a round of Baz IO | 20260506 16:02:53.239 | INFO | Populating RTMetrics report | 20260506 16:02:53.239 | INFO | RTMetrics report populated | 20260506 16:02:53.239 | INFO | Opening tmp RTMetrics json file | 20260506 16:02:53.288 | INFO | Serializing RTMetrics to json | 20260506 16:02:53.315 | INFO | Serialization done, swapping tmp file now... | 20260506 16:02:53.316 | INFO | RTMetrics done | 20260506 16:02:53.583 | INFO | Finished a round of Baz IO: Wrote 2.092 GiB in 3.493 seconds | 20260506 16:02:53.583 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.66% / 0.01% / 63.33% | 20260506 16:02:53.583 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.82029% / 1.58946e-05% / 5.96046e-05% | 20260506 16:02:53.583 | INFO | Baz IO overall: | 20260506 16:02:53.583 | INFO | BytesWritten : 1381321064290 | 20260506 16:02:53.583 | INFO | NumEvents : 668459103785 | 20260506 16:02:55.118 | INFO | Analyzing chunk frames = [2637824,2638336) | 20260506 16:02:55.118 | INFO | MemInfo: { | 20260506 16:02:55.118 | INFO | "gpu" : " 17 GB", | 20260506 16:02:55.118 | INFO | "huge" : "", | 20260506 16:02:55.118 | INFO | "huge_shared" : " 27 GB", | 20260506 16:02:55.118 | INFO | "malloc" : " 14 GB", | 20260506 16:02:55.118 | INFO | "pinned" : " 29 GB" | 20260506 16:02:55.118 | INFO | } | 20260506 16:02:55.118 | INFO | Perf report for the last 5.14075 seconds: | 20260506 16:02:55.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:02:55.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:55.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:55.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:55.118 | INFO | ANALYSIS : 62.9%, 2.9, 95.3ms, 0 | 20260506 16:02:55.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:02:55.118 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 16:02:55.118 | INFO | PRE_HQ : 60.6%, 1.9, 58.7ms, 0 | 20260506 16:02:55.118 | INFO | BAZWRITER : 67.4%, 4.8, 166.0ms, 0 | 20260506 16:02:55.118 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 16:02:55.119 | INFO | Uploaded 36GiB to GPU in 2.05314s (17.5341GiB/s) | 20260506 16:02:55.119 | INFO | Downloaded 38GiB to GPU in 2.22526s (17.0766GiB/s) | 20260506 16:03:00.256 | INFO | Analyzing chunk frames = [2638336,2638848) | 20260506 16:03:00.256 | INFO | MemInfo: { | 20260506 16:03:00.256 | INFO | "gpu" : " 17 GB", | 20260506 16:03:00.256 | INFO | "huge" : "", | 20260506 16:03:00.256 | INFO | "huge_shared" : " 27 GB", | 20260506 16:03:00.256 | INFO | "malloc" : " 14 GB", | 20260506 16:03:00.256 | INFO | "pinned" : " 29 GB" | 20260506 16:03:00.256 | INFO | } | 20260506 16:03:00.256 | INFO | Perf report for the last 5.13801 seconds: | 20260506 16:03:00.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:03:00.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:00.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:00.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:00.256 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 16:03:00.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:00.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:00.256 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:03:00.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:00.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:00.257 | INFO | Uploaded 36GiB to GPU in 1.98229s (18.1608GiB/s) | 20260506 16:03:00.257 | INFO | Downloaded 34GiB to GPU in 1.93303s (17.589GiB/s) | 20260506 16:03:05.395 | INFO | Analyzing chunk frames = [2638848,2639360) | 20260506 16:03:05.395 | INFO | MemInfo: { | 20260506 16:03:05.395 | INFO | "gpu" : " 17 GB", | 20260506 16:03:05.395 | INFO | "huge" : "", | 20260506 16:03:05.395 | INFO | "huge_shared" : " 27 GB", | 20260506 16:03:05.395 | INFO | "malloc" : " 14 GB", | 20260506 16:03:05.395 | INFO | "pinned" : " 29 GB" | 20260506 16:03:05.395 | INFO | } | 20260506 16:03:05.395 | INFO | Perf report for the last 5.13908 seconds: | 20260506 16:03:05.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:03:05.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:05.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:05.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:05.396 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 16:03:05.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:05.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:05.396 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:03:05.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:05.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:05.396 | INFO | Uploaded 36GiB to GPU in 1.95542s (18.4104GiB/s) | 20260506 16:03:05.396 | INFO | Downloaded 34GiB to GPU in 1.92698s (17.6442GiB/s) | 20260506 16:03:10.534 | INFO | Analyzing chunk frames = [2639360,2639872) | 20260506 16:03:10.534 | INFO | MemInfo: { | 20260506 16:03:10.534 | INFO | "gpu" : " 17 GB", | 20260506 16:03:10.534 | INFO | "huge" : "", | 20260506 16:03:10.534 | INFO | "huge_shared" : " 27 GB", | 20260506 16:03:10.534 | INFO | "malloc" : " 14 GB", | 20260506 16:03:10.534 | INFO | "pinned" : " 29 GB" | 20260506 16:03:10.534 | INFO | } | 20260506 16:03:10.534 | INFO | Perf report for the last 5.13866 seconds: | 20260506 16:03:10.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:03:10.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:10.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:10.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:10.534 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 16:03:10.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:10.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:10.534 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 16:03:10.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:10.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:10.534 | INFO | Uploaded 36GiB to GPU in 1.93568s (18.5981GiB/s) | 20260506 16:03:10.534 | INFO | Downloaded 34GiB to GPU in 1.88017s (18.0834GiB/s) | 20260506 16:03:15.673 | INFO | Analyzing chunk frames = [2639872,2640384) | 20260506 16:03:15.673 | INFO | MemInfo: { | 20260506 16:03:15.673 | INFO | "gpu" : " 17 GB", | 20260506 16:03:15.673 | INFO | "huge" : "", | 20260506 16:03:15.674 | INFO | "huge_shared" : " 27 GB", | 20260506 16:03:15.674 | INFO | "malloc" : " 14 GB", | 20260506 16:03:15.674 | INFO | "pinned" : " 29 GB" | 20260506 16:03:15.674 | INFO | } | 20260506 16:03:15.674 | INFO | Perf report for the last 5.13964 seconds: | 20260506 16:03:15.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:03:15.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:15.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:15.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:15.674 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 16:03:15.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:15.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:15.674 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:03:15.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:15.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:15.674 | INFO | Uploaded 36GiB to GPU in 1.98568s (18.1298GiB/s) | 20260506 16:03:15.674 | INFO | Downloaded 34GiB to GPU in 1.92106s (17.6985GiB/s) | 20260506 16:03:20.814 | INFO | Analyzing chunk frames = [2640384,2640896) | 20260506 16:03:20.814 | INFO | MemInfo: { | 20260506 16:03:20.814 | INFO | "gpu" : " 17 GB", | 20260506 16:03:20.814 | INFO | "huge" : "", | 20260506 16:03:20.814 | INFO | "huge_shared" : " 27 GB", | 20260506 16:03:20.814 | INFO | "malloc" : " 14 GB", | 20260506 16:03:20.814 | INFO | "pinned" : " 29 GB" | 20260506 16:03:20.814 | INFO | } | 20260506 16:03:20.814 | INFO | Perf report for the last 5.14063 seconds: | 20260506 16:03:20.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:03:20.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:20.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:20.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:20.814 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 16:03:20.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:20.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:20.815 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 16:03:20.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:20.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:20.815 | INFO | Uploaded 36GiB to GPU in 2.06267s (17.4531GiB/s) | 20260506 16:03:20.815 | INFO | Downloaded 34GiB to GPU in 1.97058s (17.2538GiB/s) | 20260506 16:03:25.954 | INFO | Analyzing chunk frames = [2640896,2641408) | 20260506 16:03:25.954 | INFO | MemInfo: { | 20260506 16:03:25.954 | INFO | "gpu" : " 17 GB", | 20260506 16:03:25.954 | INFO | "huge" : "", | 20260506 16:03:25.954 | INFO | "huge_shared" : " 27 GB", | 20260506 16:03:25.954 | INFO | "malloc" : " 14 GB", | 20260506 16:03:25.954 | INFO | "pinned" : " 29 GB" | 20260506 16:03:25.954 | INFO | } | 20260506 16:03:25.954 | INFO | Perf report for the last 5.13959 seconds: | 20260506 16:03:25.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:03:25.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:25.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:25.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:25.954 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 16:03:25.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:25.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:25.954 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:03:25.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:25.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:25.954 | INFO | Uploaded 36GiB to GPU in 1.99575s (18.0383GiB/s) | 20260506 16:03:25.954 | INFO | Downloaded 34GiB to GPU in 1.99793s (17.0176GiB/s) | 20260506 16:03:31.092 | INFO | Analyzing chunk frames = [2641408,2641920) | 20260506 16:03:31.092 | INFO | MemInfo: { | 20260506 16:03:31.092 | INFO | "gpu" : " 17 GB", | 20260506 16:03:31.092 | INFO | "huge" : "", | 20260506 16:03:31.092 | INFO | "huge_shared" : " 27 GB", | 20260506 16:03:31.092 | INFO | "malloc" : " 14 GB", | 20260506 16:03:31.092 | INFO | "pinned" : " 29 GB" | 20260506 16:03:31.092 | INFO | } | 20260506 16:03:31.092 | INFO | Perf report for the last 5.13843 seconds: | 20260506 16:03:31.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:03:31.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:31.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:31.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:31.092 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 16:03:31.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:31.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:31.093 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:03:31.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:31.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:31.093 | INFO | Uploaded 36GiB to GPU in 1.97835s (18.1969GiB/s) | 20260506 16:03:31.093 | INFO | Downloaded 34GiB to GPU in 1.97521s (17.2134GiB/s) | 20260506 16:03:31.207 | INFO | Starting a round of Baz IO | 20260506 16:03:34.349 | INFO | Populating RTMetrics report | 20260506 16:03:34.349 | INFO | RTMetrics report populated | 20260506 16:03:34.349 | INFO | Opening tmp RTMetrics json file | 20260506 16:03:34.432 | INFO | Serializing RTMetrics to json | 20260506 16:03:34.617 | INFO | Serialization done, swapping tmp file now... | 20260506 16:03:34.621 | INFO | RTMetrics done | 20260506 16:03:34.642 | INFO | Finished a round of Baz IO: Wrote 2.089 GiB in 3.435 seconds | 20260506 16:03:34.642 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.65% / 0.01% / 63.34% | 20260506 16:03:34.642 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.80967% / 1.98682e-05% / 6.75519e-05% | 20260506 16:03:34.642 | INFO | Baz IO overall: | 20260506 16:03:34.642 | INFO | BytesWritten : 1383563935233 | 20260506 16:03:34.642 | INFO | NumEvents : 669543547172 | 20260506 16:03:36.231 | INFO | Analyzing chunk frames = [2641920,2642432) | 20260506 16:03:36.231 | INFO | MemInfo: { | 20260506 16:03:36.231 | INFO | "gpu" : " 17 GB", | 20260506 16:03:36.231 | INFO | "huge" : "", | 20260506 16:03:36.231 | INFO | "huge_shared" : " 27 GB", | 20260506 16:03:36.231 | INFO | "malloc" : " 14 GB", | 20260506 16:03:36.231 | INFO | "pinned" : " 29 GB" | 20260506 16:03:36.231 | INFO | } | 20260506 16:03:36.231 | INFO | Perf report for the last 5.13905 seconds: | 20260506 16:03:36.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:03:36.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:36.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:36.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:36.232 | INFO | ANALYSIS : 62.8%, 2.9, 95.3ms, 0 | 20260506 16:03:36.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:36.232 | INFO | RT_METRICS : 49.0%, 1.0, 25.7ms, 0 | 20260506 16:03:36.232 | INFO | PRE_HQ : 60.3%, 1.9, 58.7ms, 0 | 20260506 16:03:36.232 | INFO | BAZWRITER : 66.5%, 4.8, 164.0ms, 0 | 20260506 16:03:36.232 | INFO | BAZ_SAMPLING : 4.2%, 1.0, 2.2ms, 0 | 20260506 16:03:36.232 | INFO | Uploaded 36GiB to GPU in 2.04663s (17.5899GiB/s) | 20260506 16:03:36.232 | INFO | Downloaded 38GiB to GPU in 2.22081s (17.1109GiB/s) | 20260506 16:03:41.370 | INFO | Analyzing chunk frames = [2642432,2642944) | 20260506 16:03:41.370 | INFO | MemInfo: { | 20260506 16:03:41.370 | INFO | "gpu" : " 17 GB", | 20260506 16:03:41.370 | INFO | "huge" : "", | 20260506 16:03:41.370 | INFO | "huge_shared" : " 27 GB", | 20260506 16:03:41.370 | INFO | "malloc" : " 14 GB", | 20260506 16:03:41.370 | INFO | "pinned" : " 29 GB" | 20260506 16:03:41.371 | INFO | } | 20260506 16:03:41.371 | INFO | Perf report for the last 5.13908 seconds: | 20260506 16:03:41.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:03:41.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:41.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:41.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:41.371 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:03:41.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:41.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:41.371 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:03:41.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:41.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:41.371 | INFO | Uploaded 36GiB to GPU in 2.00538s (17.9517GiB/s) | 20260506 16:03:41.371 | INFO | Downloaded 34GiB to GPU in 1.94513s (17.4796GiB/s) | 20260506 16:03:46.510 | INFO | Analyzing chunk frames = [2642944,2643456) | 20260506 16:03:46.510 | INFO | MemInfo: { | 20260506 16:03:46.510 | INFO | "gpu" : " 17 GB", | 20260506 16:03:46.510 | INFO | "huge" : "", | 20260506 16:03:46.510 | INFO | "huge_shared" : " 27 GB", | 20260506 16:03:46.510 | INFO | "malloc" : " 14 GB", | 20260506 16:03:46.510 | INFO | "pinned" : " 29 GB" | 20260506 16:03:46.511 | INFO | } | 20260506 16:03:46.511 | INFO | Perf report for the last 5.14008 seconds: | 20260506 16:03:46.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:03:46.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:46.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:46.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:46.511 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 16:03:46.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:46.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:46.511 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:03:46.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:46.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:46.511 | INFO | Uploaded 36GiB to GPU in 1.96708s (18.3013GiB/s) | 20260506 16:03:46.511 | INFO | Downloaded 34GiB to GPU in 1.93765s (17.547GiB/s) | 20260506 16:03:51.649 | INFO | Analyzing chunk frames = [2643456,2643968) | 20260506 16:03:51.649 | INFO | MemInfo: { | 20260506 16:03:51.649 | INFO | "gpu" : " 17 GB", | 20260506 16:03:51.649 | INFO | "huge" : "", | 20260506 16:03:51.649 | INFO | "huge_shared" : " 27 GB", | 20260506 16:03:51.649 | INFO | "malloc" : " 14 GB", | 20260506 16:03:51.649 | INFO | "pinned" : " 29 GB" | 20260506 16:03:51.649 | INFO | } | 20260506 16:03:51.649 | INFO | Perf report for the last 5.13861 seconds: | 20260506 16:03:51.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:03:51.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:51.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:51.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:51.649 | INFO | ANALYSIS : 60.6%, 3.0, 94.0ms, 0 | 20260506 16:03:51.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:51.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:51.649 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:03:51.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:51.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:51.650 | INFO | Uploaded 36GiB to GPU in 2.0125s (17.8882GiB/s) | 20260506 16:03:51.650 | INFO | Downloaded 34GiB to GPU in 1.91376s (17.7661GiB/s) | 20260506 16:03:56.788 | INFO | Analyzing chunk frames = [2643968,2644480) | 20260506 16:03:56.788 | INFO | MemInfo: { | 20260506 16:03:56.788 | INFO | "gpu" : " 17 GB", | 20260506 16:03:56.788 | INFO | "huge" : "", | 20260506 16:03:56.789 | INFO | "huge_shared" : " 27 GB", | 20260506 16:03:56.789 | INFO | "malloc" : " 14 GB", | 20260506 16:03:56.789 | INFO | "pinned" : " 29 GB" | 20260506 16:03:56.789 | INFO | } | 20260506 16:03:56.789 | INFO | Perf report for the last 5.13944 seconds: | 20260506 16:03:56.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:03:56.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:56.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:56.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:56.789 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 16:03:56.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:56.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:56.789 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:03:56.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:56.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:03:56.790 | INFO | Uploaded 36GiB to GPU in 1.96858s (18.2873GiB/s) | 20260506 16:03:56.790 | INFO | Downloaded 34GiB to GPU in 1.92541s (17.6586GiB/s) | 20260506 16:04:01.929 | INFO | Analyzing chunk frames = [2644480,2644992) | 20260506 16:04:01.929 | INFO | MemInfo: { | 20260506 16:04:01.929 | INFO | "gpu" : " 17 GB", | 20260506 16:04:01.929 | INFO | "huge" : "", | 20260506 16:04:01.930 | INFO | "huge_shared" : " 27 GB", | 20260506 16:04:01.930 | INFO | "malloc" : " 14 GB", | 20260506 16:04:01.930 | INFO | "pinned" : " 29 GB" | 20260506 16:04:01.930 | INFO | } | 20260506 16:04:01.930 | INFO | Perf report for the last 5.14097 seconds: | 20260506 16:04:01.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:04:01.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:01.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:01.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:01.930 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:04:01.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:01.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:01.930 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 16:04:01.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:01.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:01.930 | INFO | Uploaded 36GiB to GPU in 2.06245s (17.455GiB/s) | 20260506 16:04:01.930 | INFO | Downloaded 34GiB to GPU in 1.97642s (17.2029GiB/s) | 20260506 16:04:07.069 | INFO | Analyzing chunk frames = [2644992,2645504) | 20260506 16:04:07.069 | INFO | MemInfo: { | 20260506 16:04:07.069 | INFO | "gpu" : " 17 GB", | 20260506 16:04:07.069 | INFO | "huge" : "", | 20260506 16:04:07.069 | INFO | "huge_shared" : " 27 GB", | 20260506 16:04:07.069 | INFO | "malloc" : " 14 GB", | 20260506 16:04:07.069 | INFO | "pinned" : " 29 GB" | 20260506 16:04:07.069 | INFO | } | 20260506 16:04:07.069 | INFO | Perf report for the last 5.13965 seconds: | 20260506 16:04:07.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:04:07.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:07.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:07.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:07.069 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 16:04:07.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:07.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:07.069 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:04:07.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:07.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:07.070 | INFO | Uploaded 36GiB to GPU in 2.0232s (17.7936GiB/s) | 20260506 16:04:07.070 | INFO | Downloaded 34GiB to GPU in 2.01092s (16.9077GiB/s) | 20260506 16:04:12.207 | INFO | Analyzing chunk frames = [2645504,2646016) | 20260506 16:04:12.207 | INFO | MemInfo: { | 20260506 16:04:12.207 | INFO | "gpu" : " 17 GB", | 20260506 16:04:12.207 | INFO | "huge" : "", | 20260506 16:04:12.207 | INFO | "huge_shared" : " 27 GB", | 20260506 16:04:12.207 | INFO | "malloc" : " 14 GB", | 20260506 16:04:12.207 | INFO | "pinned" : " 29 GB" | 20260506 16:04:12.207 | INFO | } | 20260506 16:04:12.207 | INFO | Perf report for the last 5.13806 seconds: | 20260506 16:04:12.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:04:12.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:12.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:12.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:12.208 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:04:12.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:12.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:12.208 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:04:12.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:12.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:12.208 | INFO | Uploaded 36GiB to GPU in 2.02743s (17.7564GiB/s) | 20260506 16:04:12.208 | INFO | Downloaded 34GiB to GPU in 1.98343s (17.142GiB/s) | 20260506 16:04:12.208 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:04:12.208 | INFO | Total mem usage: 17.4 GiB | 20260506 16:04:12.208 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 16:04:12.208 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -11.6%) | 20260506 16:04:12.208 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:04:12.208 | INFO | High Water Breakdown: | 20260506 16:04:12.208 | INFO | 61.3% : 9.56 GiB ( 87.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:04:12.208 | INFO | 38.6% : 6.02 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:04:12.208 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:04:12.208 | INFO | ------------------------------------------ | 20260506 16:04:12.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:04:12.208 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:04:12.208 | INFO | Total mem usage: 26.5 GiB | 20260506 16:04:12.208 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:04:12.208 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:04:12.208 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:04:12.208 | INFO | High Water Breakdown: | 20260506 16:04:12.208 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:04:12.208 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:04:12.208 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:04:12.208 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:04:12.208 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:04:12.208 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:04:12.208 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:04:12.208 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:04:12.208 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:04:12.208 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:04:12.208 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:04:12.208 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:04:12.208 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:04:12.208 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:04:12.208 | INFO | ------------------------------------------ | 20260506 16:04:12.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:04:12.208 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:04:12.208 | INFO | Total mem usage: 25.6 GiB | 20260506 16:04:12.208 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:04:12.208 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:04:12.208 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:04:12.208 | INFO | High Water Breakdown: | 20260506 16:04:12.208 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:04:12.208 | INFO | ------------------------------------------ | 20260506 16:04:12.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:04:12.208 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:04:12.208 | INFO | Total mem usage: 15.7 GiB | 20260506 16:04:12.208 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:04:12.208 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:04:12.208 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 16:04:12.208 | INFO | High Water Breakdown: | 20260506 16:04:12.208 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:04:12.208 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:04:12.208 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:04:12.208 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:04:12.208 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:04:12.208 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:04:12.208 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:04:12.208 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:04:12.208 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:04:12.208 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:04:12.208 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:04:12.208 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:04:12.208 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:04:12.208 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:04:12.208 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:04:12.208 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:04:12.208 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:04:12.208 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:04:12.209 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:04:12.209 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:04:12.209 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:04:12.209 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:04:12.209 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:04:12.209 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:04:12.209 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:04:12.209 | INFO | ------------------------------------------ | 20260506 16:04:12.209 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:04:12.209 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:04:12.209 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:04:12.209 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:04:12.209 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.53e+01, 1.35e+01 / 1.88e+00 | 20260506 16:04:12.209 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 9.97e-02, 9.82e-03 / 7.43e-03 | 20260506 16:04:12.209 | INFO | PerfReport: 11.58%, Baseline -- 3.63e+00 / 4.44e+00, 5.53e-01 / 3.61e-01 | 20260506 16:04:12.209 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.45e+00, 1.84e-01 / 1.46e-01 | 20260506 16:04:12.209 | INFO | PerfReport: 15.82%, DME -- 3.97e+01 / 4.35e+01, 3.52e+00 / 1.69e+00 | 20260506 16:04:12.209 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.79e+01 / 2.10e+01, 1.48e+00 / 1.93e+00 | 20260506 16:04:12.209 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.31e+00 / 2.90e+00, 3.95e-01 / 2.30e-01 | 20260506 16:04:12.209 | INFO | PerfReport: 3.52%, Metrics -- 1.10e+00 / 1.48e+00, 2.86e-01 / 1.95e-01 | 20260506 16:04:12.209 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.83e-01 / 4.34e-01, 6.07e-02 / 2.29e-02 | 20260506 16:04:12.209 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:04:12.209 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:04:12.209 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:04:12.209 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 16:04:12.209 | INFO | PerfReport: 21.46%, Upload -- 2.09e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260506 16:04:12.209 | INFO | PerfReport: 21.25%, Download -- 2.07e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 16:04:12.209 | INFO | PerfReport: 32.26%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 16:04:12.209 | INFO | PerfReport: 1.34%, LockUpload -- 1.30e+00 / 0.00e+00, 5.49e+00 / 0.00e+00 | 20260506 16:04:12.209 | INFO | PerfReport: 1.20%, LockDownload -- 1.16e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 16:04:12.209 | INFO | PerfReport: 22.49%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 16:04:12.209 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260506 16:04:12.327 | INFO | Starting a round of Baz IO | 20260506 16:04:15.472 | INFO | Populating RTMetrics report | 20260506 16:04:15.473 | INFO | RTMetrics report populated | 20260506 16:04:15.473 | INFO | Opening tmp RTMetrics json file | 20260506 16:04:15.576 | INFO | Serializing RTMetrics to json | 20260506 16:04:15.601 | INFO | Serialization done, swapping tmp file now... | 20260506 16:04:15.609 | INFO | RTMetrics done | 20260506 16:04:15.928 | INFO | Finished a round of Baz IO: Wrote 2.09 GiB in 3.6 seconds | 20260506 16:04:15.928 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.64% / 0.01% / 63.35% | 20260506 16:04:15.928 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.8078% / 1.98682e-05% / 5.16574e-05% | 20260506 16:04:15.928 | INFO | Baz IO overall: | 20260506 16:04:15.928 | INFO | BytesWritten : 1385808265394 | 20260506 16:04:15.928 | INFO | NumEvents : 670629042448 | 20260506 16:04:17.346 | INFO | Analyzing chunk frames = [2646016,2646528) | 20260506 16:04:17.346 | INFO | MemInfo: { | 20260506 16:04:17.346 | INFO | "gpu" : " 17 GB", | 20260506 16:04:17.346 | INFO | "huge" : "", | 20260506 16:04:17.346 | INFO | "huge_shared" : " 27 GB", | 20260506 16:04:17.346 | INFO | "malloc" : " 14 GB", | 20260506 16:04:17.346 | INFO | "pinned" : " 29 GB" | 20260506 16:04:17.346 | INFO | } | 20260506 16:04:17.346 | INFO | Perf report for the last 5.13837 seconds: | 20260506 16:04:17.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:04:17.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:17.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:17.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:17.346 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 16:04:17.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:17.346 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260506 16:04:17.346 | INFO | PRE_HQ : 60.6%, 1.9, 58.9ms, 0 | 20260506 16:04:17.346 | INFO | BAZWRITER : 69.7%, 4.8, 172.4ms, 0 | 20260506 16:04:17.346 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 16:04:17.346 | INFO | Uploaded 36GiB to GPU in 2.05751s (17.4968GiB/s) | 20260506 16:04:17.346 | INFO | Downloaded 38GiB to GPU in 2.21655s (17.1438GiB/s) | 20260506 16:04:22.485 | INFO | Analyzing chunk frames = [2646528,2647040) | 20260506 16:04:22.485 | INFO | MemInfo: { | 20260506 16:04:22.485 | INFO | "gpu" : " 17 GB", | 20260506 16:04:22.485 | INFO | "huge" : "", | 20260506 16:04:22.485 | INFO | "huge_shared" : " 27 GB", | 20260506 16:04:22.485 | INFO | "malloc" : " 14 GB", | 20260506 16:04:22.485 | INFO | "pinned" : " 29 GB" | 20260506 16:04:22.485 | INFO | } | 20260506 16:04:22.485 | INFO | Perf report for the last 5.13918 seconds: | 20260506 16:04:22.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:04:22.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:22.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:22.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:22.485 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:04:22.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:22.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:22.485 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:04:22.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:22.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:22.485 | INFO | Uploaded 36GiB to GPU in 1.99737s (18.0237GiB/s) | 20260506 16:04:22.485 | INFO | Downloaded 34GiB to GPU in 1.93764s (17.5471GiB/s) | 20260506 16:04:27.624 | INFO | Analyzing chunk frames = [2647040,2647552) | 20260506 16:04:27.624 | INFO | MemInfo: { | 20260506 16:04:27.624 | INFO | "gpu" : " 17 GB", | 20260506 16:04:27.624 | INFO | "huge" : "", | 20260506 16:04:27.625 | INFO | "huge_shared" : " 27 GB", | 20260506 16:04:27.625 | INFO | "malloc" : " 14 GB", | 20260506 16:04:27.625 | INFO | "pinned" : " 29 GB" | 20260506 16:04:27.625 | INFO | } | 20260506 16:04:27.625 | INFO | Perf report for the last 5.13967 seconds: | 20260506 16:04:27.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:04:27.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:27.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:27.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:27.625 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 16:04:27.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:27.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:27.625 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:04:27.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:27.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:27.625 | INFO | Uploaded 36GiB to GPU in 1.99141s (18.0777GiB/s) | 20260506 16:04:27.625 | INFO | Downloaded 34GiB to GPU in 1.93508s (17.5703GiB/s) | 20260506 16:04:32.764 | INFO | Analyzing chunk frames = [2647552,2648064) | 20260506 16:04:32.764 | INFO | MemInfo: { | 20260506 16:04:32.764 | INFO | "gpu" : " 17 GB", | 20260506 16:04:32.764 | INFO | "huge" : "", | 20260506 16:04:32.765 | INFO | "huge_shared" : " 27 GB", | 20260506 16:04:32.765 | INFO | "malloc" : " 14 GB", | 20260506 16:04:32.765 | INFO | "pinned" : " 29 GB" | 20260506 16:04:32.765 | INFO | } | 20260506 16:04:32.765 | INFO | Perf report for the last 5.1399 seconds: | 20260506 16:04:32.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:04:32.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:32.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:32.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:32.765 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 16:04:32.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:32.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:32.765 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:04:32.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:32.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:32.765 | INFO | Uploaded 36GiB to GPU in 1.97107s (18.2642GiB/s) | 20260506 16:04:32.765 | INFO | Downloaded 34GiB to GPU in 1.88321s (18.0543GiB/s) | 20260506 16:04:37.904 | INFO | Analyzing chunk frames = [2648064,2648576) | 20260506 16:04:37.904 | INFO | MemInfo: { | 20260506 16:04:37.904 | INFO | "gpu" : " 17 GB", | 20260506 16:04:37.904 | INFO | "huge" : "", | 20260506 16:04:37.904 | INFO | "huge_shared" : " 27 GB", | 20260506 16:04:37.904 | INFO | "malloc" : " 14 GB", | 20260506 16:04:37.904 | INFO | "pinned" : " 29 GB" | 20260506 16:04:37.904 | INFO | } | 20260506 16:04:37.904 | INFO | Perf report for the last 5.13919 seconds: | 20260506 16:04:37.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:04:37.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:37.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:37.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:37.904 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 16:04:37.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:37.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:37.904 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:04:37.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:37.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:37.904 | INFO | Uploaded 36GiB to GPU in 2.01772s (17.8419GiB/s) | 20260506 16:04:37.904 | INFO | Downloaded 34GiB to GPU in 1.93454s (17.5753GiB/s) | 20260506 16:04:43.042 | INFO | Analyzing chunk frames = [2648576,2649088) | 20260506 16:04:43.042 | INFO | MemInfo: { | 20260506 16:04:43.042 | INFO | "gpu" : " 17 GB", | 20260506 16:04:43.042 | INFO | "huge" : "", | 20260506 16:04:43.042 | INFO | "huge_shared" : " 27 GB", | 20260506 16:04:43.042 | INFO | "malloc" : " 14 GB", | 20260506 16:04:43.042 | INFO | "pinned" : " 29 GB" | 20260506 16:04:43.042 | INFO | } | 20260506 16:04:43.042 | INFO | Perf report for the last 5.13846 seconds: | 20260506 16:04:43.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:04:43.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:43.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:43.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:43.042 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:04:43.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:43.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:43.042 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 16:04:43.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:43.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:43.043 | INFO | Uploaded 36GiB to GPU in 2.07309s (17.3654GiB/s) | 20260506 16:04:43.043 | INFO | Downloaded 34GiB to GPU in 1.96721s (17.2834GiB/s) | 20260506 16:04:48.183 | INFO | Analyzing chunk frames = [2649088,2649600) | 20260506 16:04:48.183 | INFO | MemInfo: { | 20260506 16:04:48.183 | INFO | "gpu" : " 17 GB", | 20260506 16:04:48.183 | INFO | "huge" : "", | 20260506 16:04:48.183 | INFO | "huge_shared" : " 27 GB", | 20260506 16:04:48.183 | INFO | "malloc" : " 14 GB", | 20260506 16:04:48.183 | INFO | "pinned" : " 29 GB" | 20260506 16:04:48.183 | INFO | } | 20260506 16:04:48.183 | INFO | Perf report for the last 5.14106 seconds: | 20260506 16:04:48.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:04:48.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:48.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:48.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:48.183 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:04:48.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:48.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:48.184 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 16:04:48.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:48.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:48.184 | INFO | Uploaded 36GiB to GPU in 2.01684s (17.8497GiB/s) | 20260506 16:04:48.184 | INFO | Downloaded 34GiB to GPU in 2.01304s (16.8899GiB/s) | 20260506 16:04:53.322 | INFO | Analyzing chunk frames = [2649600,2650112) | 20260506 16:04:53.322 | INFO | MemInfo: { | 20260506 16:04:53.322 | INFO | "gpu" : " 17 GB", | 20260506 16:04:53.322 | INFO | "huge" : "", | 20260506 16:04:53.322 | INFO | "huge_shared" : " 27 GB", | 20260506 16:04:53.323 | INFO | "malloc" : " 14 GB", | 20260506 16:04:53.323 | INFO | "pinned" : " 29 GB" | 20260506 16:04:53.323 | INFO | } | 20260506 16:04:53.323 | INFO | Perf report for the last 5.13933 seconds: | 20260506 16:04:53.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:04:53.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:53.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:53.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:53.323 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:04:53.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:53.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:53.323 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:04:53.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:53.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:53.323 | INFO | Uploaded 36GiB to GPU in 2.03238s (17.7133GiB/s) | 20260506 16:04:53.323 | INFO | Downloaded 34GiB to GPU in 1.98257s (17.1495GiB/s) | 20260506 16:04:53.435 | INFO | Starting a round of Baz IO | 20260506 16:04:56.586 | INFO | Populating RTMetrics report | 20260506 16:04:56.586 | INFO | RTMetrics report populated | 20260506 16:04:56.586 | INFO | Opening tmp RTMetrics json file | 20260506 16:04:56.598 | INFO | Serializing RTMetrics to json | 20260506 16:04:56.600 | INFO | Serialization done, swapping tmp file now... | 20260506 16:04:56.601 | INFO | RTMetrics done | 20260506 16:04:56.961 | INFO | Finished a round of Baz IO: Wrote 2.091 GiB in 3.525 seconds | 20260506 16:04:56.961 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.63% / 0.01% / 63.36% | 20260506 16:04:56.961 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.80699% / 1.98682e-05% / 5.96046e-05% | 20260506 16:04:56.961 | INFO | Baz IO overall: | 20260506 16:04:56.961 | INFO | BytesWritten : 1388053673660 | 20260506 16:04:56.961 | INFO | NumEvents : 671715347988 | 20260506 16:04:58.462 | INFO | Analyzing chunk frames = [2650112,2650624) | 20260506 16:04:58.462 | INFO | MemInfo: { | 20260506 16:04:58.462 | INFO | "gpu" : " 17 GB", | 20260506 16:04:58.462 | INFO | "huge" : "", | 20260506 16:04:58.462 | INFO | "huge_shared" : " 27 GB", | 20260506 16:04:58.462 | INFO | "malloc" : " 14 GB", | 20260506 16:04:58.462 | INFO | "pinned" : " 29 GB" | 20260506 16:04:58.462 | INFO | } | 20260506 16:04:58.462 | INFO | Perf report for the last 5.13916 seconds: | 20260506 16:04:58.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:04:58.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:58.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:58.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:58.462 | INFO | ANALYSIS : 62.9%, 2.9, 95.2ms, 0 | 20260506 16:04:58.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:04:58.462 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 16:04:58.462 | INFO | PRE_HQ : 60.7%, 1.9, 59.3ms, 0 | 20260506 16:04:58.462 | INFO | BAZWRITER : 68.0%, 4.8, 167.5ms, 0 | 20260506 16:04:58.462 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260506 16:04:58.462 | INFO | Uploaded 36GiB to GPU in 2.05741s (17.4977GiB/s) | 20260506 16:04:58.462 | INFO | Downloaded 38GiB to GPU in 2.23421s (17.0083GiB/s) | 20260506 16:05:03.600 | INFO | Analyzing chunk frames = [2650624,2651136) | 20260506 16:05:03.600 | INFO | MemInfo: { | 20260506 16:05:03.600 | INFO | "gpu" : " 17 GB", | 20260506 16:05:03.600 | INFO | "huge" : "", | 20260506 16:05:03.600 | INFO | "huge_shared" : " 27 GB", | 20260506 16:05:03.600 | INFO | "malloc" : " 14 GB", | 20260506 16:05:03.600 | INFO | "pinned" : " 29 GB" | 20260506 16:05:03.600 | INFO | } | 20260506 16:05:03.601 | INFO | Perf report for the last 5.13876 seconds: | 20260506 16:05:03.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:05:03.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:03.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:03.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:03.601 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 16:05:03.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:03.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:03.601 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:05:03.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:03.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:03.601 | INFO | Uploaded 36GiB to GPU in 1.99898s (18.0092GiB/s) | 20260506 16:05:03.601 | INFO | Downloaded 34GiB to GPU in 1.9329s (17.5902GiB/s) | 20260506 16:05:08.739 | INFO | Analyzing chunk frames = [2651136,2651648) | 20260506 16:05:08.739 | INFO | MemInfo: { | 20260506 16:05:08.739 | INFO | "gpu" : " 17 GB", | 20260506 16:05:08.739 | INFO | "huge" : "", | 20260506 16:05:08.739 | INFO | "huge_shared" : " 27 GB", | 20260506 16:05:08.739 | INFO | "malloc" : " 14 GB", | 20260506 16:05:08.740 | INFO | "pinned" : " 29 GB" | 20260506 16:05:08.740 | INFO | } | 20260506 16:05:08.740 | INFO | Perf report for the last 5.13907 seconds: | 20260506 16:05:08.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:05:08.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:08.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:08.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:08.740 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 16:05:08.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:08.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:08.740 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 16:05:08.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:08.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:08.740 | INFO | Uploaded 36GiB to GPU in 1.99992s (18.0007GiB/s) | 20260506 16:05:08.740 | INFO | Downloaded 34GiB to GPU in 1.93216s (17.5969GiB/s) | 20260506 16:05:13.879 | INFO | Analyzing chunk frames = [2651648,2652160) | 20260506 16:05:13.879 | INFO | MemInfo: { | 20260506 16:05:13.879 | INFO | "gpu" : " 17 GB", | 20260506 16:05:13.879 | INFO | "huge" : "", | 20260506 16:05:13.879 | INFO | "huge_shared" : " 27 GB", | 20260506 16:05:13.879 | INFO | "malloc" : " 14 GB", | 20260506 16:05:13.879 | INFO | "pinned" : " 29 GB" | 20260506 16:05:13.879 | INFO | } | 20260506 16:05:13.879 | INFO | Perf report for the last 5.1393 seconds: | 20260506 16:05:13.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:05:13.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:13.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:13.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:13.879 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 16:05:13.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:13.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:13.879 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:05:13.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:13.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:13.879 | INFO | Uploaded 36GiB to GPU in 1.95252s (18.4377GiB/s) | 20260506 16:05:13.879 | INFO | Downloaded 34GiB to GPU in 1.87507s (18.1327GiB/s) | 20260506 16:05:19.020 | INFO | Analyzing chunk frames = [2652160,2652672) | 20260506 16:05:19.020 | INFO | MemInfo: { | 20260506 16:05:19.020 | INFO | "gpu" : " 17 GB", | 20260506 16:05:19.020 | INFO | "huge" : "", | 20260506 16:05:19.020 | INFO | "huge_shared" : " 27 GB", | 20260506 16:05:19.020 | INFO | "malloc" : " 14 GB", | 20260506 16:05:19.020 | INFO | "pinned" : " 29 GB" | 20260506 16:05:19.021 | INFO | } | 20260506 16:05:19.021 | INFO | Perf report for the last 5.14153 seconds: | 20260506 16:05:19.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:05:19.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:19.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:19.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:19.021 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260506 16:05:19.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:19.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:19.021 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:05:19.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:19.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:19.021 | INFO | Uploaded 36GiB to GPU in 2.00165s (17.9852GiB/s) | 20260506 16:05:19.021 | INFO | Downloaded 34GiB to GPU in 1.92646s (17.6489GiB/s) | 20260506 16:05:24.157 | INFO | Analyzing chunk frames = [2652672,2653184) | 20260506 16:05:24.157 | INFO | MemInfo: { | 20260506 16:05:24.157 | INFO | "gpu" : " 17 GB", | 20260506 16:05:24.157 | INFO | "huge" : "", | 20260506 16:05:24.157 | INFO | "huge_shared" : " 27 GB", | 20260506 16:05:24.158 | INFO | "malloc" : " 14 GB", | 20260506 16:05:24.158 | INFO | "pinned" : " 29 GB" | 20260506 16:05:24.158 | INFO | } | 20260506 16:05:24.158 | INFO | Perf report for the last 5.13713 seconds: | 20260506 16:05:24.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:05:24.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:24.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:24.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:24.158 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:05:24.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:24.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:24.158 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 16:05:24.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:24.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:24.158 | INFO | Uploaded 36GiB to GPU in 2.07433s (17.355GiB/s) | 20260506 16:05:24.158 | INFO | Downloaded 34GiB to GPU in 1.97163s (17.2447GiB/s) | 20260506 16:05:29.298 | INFO | Analyzing chunk frames = [2653184,2653696) | 20260506 16:05:29.298 | INFO | MemInfo: { | 20260506 16:05:29.298 | INFO | "gpu" : " 17 GB", | 20260506 16:05:29.298 | INFO | "huge" : "", | 20260506 16:05:29.298 | INFO | "huge_shared" : " 27 GB", | 20260506 16:05:29.298 | INFO | "malloc" : " 14 GB", | 20260506 16:05:29.298 | INFO | "pinned" : " 29 GB" | 20260506 16:05:29.298 | INFO | } | 20260506 16:05:29.298 | INFO | Perf report for the last 5.14005 seconds: | 20260506 16:05:29.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:05:29.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:29.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:29.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:29.298 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 16:05:29.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:29.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:29.298 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 16:05:29.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:29.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:29.298 | INFO | Uploaded 36GiB to GPU in 2.03733s (17.6702GiB/s) | 20260506 16:05:29.298 | INFO | Downloaded 34GiB to GPU in 1.98616s (17.1184GiB/s) | 20260506 16:05:34.436 | INFO | Analyzing chunk frames = [2653696,2654208) | 20260506 16:05:34.436 | INFO | MemInfo: { | 20260506 16:05:34.436 | INFO | "gpu" : " 17 GB", | 20260506 16:05:34.436 | INFO | "huge" : "", | 20260506 16:05:34.436 | INFO | "huge_shared" : " 27 GB", | 20260506 16:05:34.437 | INFO | "malloc" : " 14 GB", | 20260506 16:05:34.437 | INFO | "pinned" : " 29 GB" | 20260506 16:05:34.437 | INFO | } | 20260506 16:05:34.437 | INFO | Perf report for the last 5.13883 seconds: | 20260506 16:05:34.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:05:34.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:34.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:34.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:34.437 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:05:34.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:34.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:34.437 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:05:34.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:34.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:34.437 | INFO | Uploaded 36GiB to GPU in 1.95616s (18.4034GiB/s) | 20260506 16:05:34.437 | INFO | Downloaded 34GiB to GPU in 1.99107s (17.0762GiB/s) | 20260506 16:05:34.437 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:05:34.437 | INFO | Total mem usage: 17.4 GiB | 20260506 16:05:34.437 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 16:05:34.437 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -11.7%) | 20260506 16:05:34.437 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:05:34.437 | INFO | High Water Breakdown: | 20260506 16:05:34.437 | INFO | 61.3% : 9.55 GiB ( 87.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:05:34.437 | INFO | 38.6% : 6.02 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:05:34.437 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:05:34.437 | INFO | ------------------------------------------ | 20260506 16:05:34.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:05:34.437 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:05:34.437 | INFO | Total mem usage: 26.5 GiB | 20260506 16:05:34.437 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:05:34.437 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:05:34.437 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:05:34.437 | INFO | High Water Breakdown: | 20260506 16:05:34.437 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:05:34.437 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:05:34.437 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:05:34.437 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:05:34.437 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:05:34.437 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:05:34.437 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:05:34.437 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:05:34.437 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:05:34.437 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:05:34.437 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:05:34.437 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:05:34.437 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:05:34.437 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:05:34.437 | INFO | ------------------------------------------ | 20260506 16:05:34.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:05:34.437 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:05:34.437 | INFO | Total mem usage: 25.6 GiB | 20260506 16:05:34.437 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:05:34.438 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:05:34.438 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:05:34.438 | INFO | High Water Breakdown: | 20260506 16:05:34.438 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:05:34.438 | INFO | ------------------------------------------ | 20260506 16:05:34.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:05:34.438 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:05:34.438 | INFO | Total mem usage: 15.7 GiB | 20260506 16:05:34.438 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:05:34.438 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:05:34.438 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:05:34.438 | INFO | High Water Breakdown: | 20260506 16:05:34.438 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:05:34.438 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:05:34.438 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:05:34.438 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:05:34.438 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:05:34.438 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:05:34.438 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:05:34.438 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:05:34.438 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:05:34.438 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:05:34.438 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:05:34.438 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:05:34.438 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:05:34.438 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:05:34.438 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:05:34.438 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:05:34.438 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:05:34.438 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:05:34.438 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:05:34.438 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:05:34.438 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:05:34.438 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:05:34.438 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:05:34.438 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:05:34.438 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:05:34.438 | INFO | ------------------------------------------ | 20260506 16:05:34.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:05:34.438 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:05:34.438 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:05:34.438 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:05:34.438 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.47e+01, 1.35e+01 / 1.74e+00 | 20260506 16:05:34.438 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.00e-01, 1.02e-02 / 6.44e-03 | 20260506 16:05:34.438 | INFO | PerfReport: 11.59%, Baseline -- 3.63e+00 / 4.39e+00, 5.51e-01 / 3.73e-01 | 20260506 16:05:34.438 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.44e+00, 1.82e-01 / 1.54e-01 | 20260506 16:05:34.438 | INFO | PerfReport: 15.86%, DME -- 3.98e+01 / 4.36e+01, 3.58e+00 / 1.58e+00 | 20260506 16:05:34.438 | INFO | PerfReport: 56.91%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.42e+00 / 1.92e+00 | 20260506 16:05:34.438 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.31e+00 / 2.86e+00, 3.95e-01 / 2.68e-01 | 20260506 16:05:34.438 | INFO | PerfReport: 3.52%, Metrics -- 1.10e+00 / 1.49e+00, 2.82e-01 / 1.95e-01 | 20260506 16:05:34.438 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.83e-01 / 4.39e-01, 6.00e-02 / 2.71e-02 | 20260506 16:05:34.438 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:05:34.438 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:05:34.438 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:05:34.438 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 16:05:34.438 | INFO | PerfReport: 21.57%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 16:05:34.438 | INFO | PerfReport: 21.22%, Download -- 2.07e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 16:05:34.438 | INFO | PerfReport: 32.21%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 16:05:34.438 | INFO | PerfReport: 1.45%, LockUpload -- 1.42e+00 / 0.00e+00, 5.39e+00 / 0.00e+00 | 20260506 16:05:34.438 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260506 16:05:34.438 | INFO | PerfReport: 22.44%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 16:05:34.438 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260506 16:05:34.551 | INFO | Starting a round of Baz IO | 20260506 16:05:37.693 | INFO | Populating RTMetrics report | 20260506 16:05:37.693 | INFO | RTMetrics report populated | 20260506 16:05:37.693 | INFO | Opening tmp RTMetrics json file | 20260506 16:05:37.834 | INFO | Serializing RTMetrics to json | 20260506 16:05:37.836 | INFO | Serialization done, swapping tmp file now... | 20260506 16:05:37.836 | INFO | RTMetrics done | 20260506 16:05:38.121 | INFO | Finished a round of Baz IO: Wrote 2.089 GiB in 3.57 seconds | 20260506 16:05:38.121 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.62% / 0.01% / 63.37% | 20260506 16:05:38.121 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.79916% / 2.78155e-05% / 5.96046e-05% | 20260506 16:05:38.121 | INFO | Baz IO overall: | 20260506 16:05:38.121 | INFO | BytesWritten : 1390297005609 | 20260506 16:05:38.121 | INFO | NumEvents : 672800038914 | 20260506 16:05:39.577 | INFO | Analyzing chunk frames = [2654208,2654720) | 20260506 16:05:39.577 | INFO | MemInfo: { | 20260506 16:05:39.577 | INFO | "gpu" : " 17 GB", | 20260506 16:05:39.577 | INFO | "huge" : "", | 20260506 16:05:39.577 | INFO | "huge_shared" : " 27 GB", | 20260506 16:05:39.577 | INFO | "malloc" : " 14 GB", | 20260506 16:05:39.577 | INFO | "pinned" : " 29 GB" | 20260506 16:05:39.577 | INFO | } | 20260506 16:05:39.577 | INFO | Perf report for the last 5.14022 seconds: | 20260506 16:05:39.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:05:39.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:39.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:39.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:39.577 | INFO | ANALYSIS : 62.8%, 2.9, 95.5ms, 0 | 20260506 16:05:39.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:39.577 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260506 16:05:39.577 | INFO | PRE_HQ : 60.4%, 1.9, 58.7ms, 0 | 20260506 16:05:39.577 | INFO | BAZWRITER : 69.1%, 4.7, 168.5ms, 0 | 20260506 16:05:39.577 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.3ms, 0 | 20260506 16:05:39.577 | INFO | Uploaded 36GiB to GPU in 2.04179s (17.6316GiB/s) | 20260506 16:05:39.577 | INFO | Downloaded 38GiB to GPU in 2.21964s (17.1199GiB/s) | 20260506 16:05:44.716 | INFO | Analyzing chunk frames = [2654720,2655232) | 20260506 16:05:44.716 | INFO | MemInfo: { | 20260506 16:05:44.716 | INFO | "gpu" : " 17 GB", | 20260506 16:05:44.716 | INFO | "huge" : "", | 20260506 16:05:44.716 | INFO | "huge_shared" : " 27 GB", | 20260506 16:05:44.716 | INFO | "malloc" : " 14 GB", | 20260506 16:05:44.716 | INFO | "pinned" : " 29 GB" | 20260506 16:05:44.716 | INFO | } | 20260506 16:05:44.716 | INFO | Perf report for the last 5.13887 seconds: | 20260506 16:05:44.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:05:44.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:44.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:44.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:44.716 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 16:05:44.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:44.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:44.716 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:05:44.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:44.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:44.716 | INFO | Uploaded 36GiB to GPU in 2.00177s (17.9841GiB/s) | 20260506 16:05:44.716 | INFO | Downloaded 34GiB to GPU in 1.94226s (17.5054GiB/s) | 20260506 16:05:49.854 | INFO | Analyzing chunk frames = [2655232,2655744) | 20260506 16:05:49.854 | INFO | MemInfo: { | 20260506 16:05:49.854 | INFO | "gpu" : " 17 GB", | 20260506 16:05:49.854 | INFO | "huge" : "", | 20260506 16:05:49.854 | INFO | "huge_shared" : " 27 GB", | 20260506 16:05:49.854 | INFO | "malloc" : " 14 GB", | 20260506 16:05:49.854 | INFO | "pinned" : " 29 GB" | 20260506 16:05:49.854 | INFO | } | 20260506 16:05:49.854 | INFO | Perf report for the last 5.13849 seconds: | 20260506 16:05:49.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:05:49.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:49.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:49.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:49.854 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 16:05:49.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:49.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:49.854 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:05:49.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:49.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:49.855 | INFO | Uploaded 36GiB to GPU in 1.96862s (18.2869GiB/s) | 20260506 16:05:49.855 | INFO | Downloaded 34GiB to GPU in 1.93577s (17.5641GiB/s) | 20260506 16:05:54.994 | INFO | Analyzing chunk frames = [2655744,2656256) | 20260506 16:05:54.994 | INFO | MemInfo: { | 20260506 16:05:54.994 | INFO | "gpu" : " 17 GB", | 20260506 16:05:54.994 | INFO | "huge" : "", | 20260506 16:05:54.994 | INFO | "huge_shared" : " 27 GB", | 20260506 16:05:54.994 | INFO | "malloc" : " 14 GB", | 20260506 16:05:54.994 | INFO | "pinned" : " 29 GB" | 20260506 16:05:54.994 | INFO | } | 20260506 16:05:54.994 | INFO | Perf report for the last 5.1397 seconds: | 20260506 16:05:54.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:05:54.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:54.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:54.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:54.994 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 16:05:54.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:54.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:54.994 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260506 16:05:54.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:54.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:05:54.994 | INFO | Uploaded 36GiB to GPU in 1.96123s (18.3558GiB/s) | 20260506 16:05:54.994 | INFO | Downloaded 34GiB to GPU in 1.87844s (18.1002GiB/s) | 20260506 16:06:00.133 | INFO | Analyzing chunk frames = [2656256,2656768) | 20260506 16:06:00.133 | INFO | MemInfo: { | 20260506 16:06:00.133 | INFO | "gpu" : " 17 GB", | 20260506 16:06:00.133 | INFO | "huge" : "", | 20260506 16:06:00.133 | INFO | "huge_shared" : " 27 GB", | 20260506 16:06:00.133 | INFO | "malloc" : " 14 GB", | 20260506 16:06:00.133 | INFO | "pinned" : " 29 GB" | 20260506 16:06:00.133 | INFO | } | 20260506 16:06:00.133 | INFO | Perf report for the last 5.13898 seconds: | 20260506 16:06:00.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:06:00.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:00.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:00.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:00.133 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 16:06:00.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:00.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:00.133 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:06:00.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:00.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:00.133 | INFO | Uploaded 36GiB to GPU in 1.985s (18.136GiB/s) | 20260506 16:06:00.133 | INFO | Downloaded 34GiB to GPU in 1.92452s (17.6667GiB/s) | 20260506 16:06:05.274 | INFO | Analyzing chunk frames = [2656768,2657280) | 20260506 16:06:05.274 | INFO | MemInfo: { | 20260506 16:06:05.274 | INFO | "gpu" : " 17 GB", | 20260506 16:06:05.274 | INFO | "huge" : "", | 20260506 16:06:05.274 | INFO | "huge_shared" : " 27 GB", | 20260506 16:06:05.274 | INFO | "malloc" : " 14 GB", | 20260506 16:06:05.274 | INFO | "pinned" : " 29 GB" | 20260506 16:06:05.274 | INFO | } | 20260506 16:06:05.274 | INFO | Perf report for the last 5.14103 seconds: | 20260506 16:06:05.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:06:05.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:05.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:05.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:05.274 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:06:05.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:05.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:05.274 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 16:06:05.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:05.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:05.274 | INFO | Uploaded 36GiB to GPU in 2.05859s (17.4877GiB/s) | 20260506 16:06:05.274 | INFO | Downloaded 34GiB to GPU in 1.97814s (17.1879GiB/s) | 20260506 16:06:10.414 | INFO | Analyzing chunk frames = [2657280,2657792) | 20260506 16:06:10.414 | INFO | MemInfo: { | 20260506 16:06:10.414 | INFO | "gpu" : " 17 GB", | 20260506 16:06:10.414 | INFO | "huge" : "", | 20260506 16:06:10.414 | INFO | "huge_shared" : " 27 GB", | 20260506 16:06:10.414 | INFO | "malloc" : " 14 GB", | 20260506 16:06:10.414 | INFO | "pinned" : " 29 GB" | 20260506 16:06:10.415 | INFO | } | 20260506 16:06:10.415 | INFO | Perf report for the last 5.14057 seconds: | 20260506 16:06:10.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:06:10.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:10.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:10.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:10.415 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 16:06:10.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:10.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:10.415 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 16:06:10.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:10.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:10.415 | INFO | Uploaded 36GiB to GPU in 2.04637s (17.5921GiB/s) | 20260506 16:06:10.415 | INFO | Downloaded 34GiB to GPU in 2.01391s (16.8826GiB/s) | 20260506 16:06:15.552 | INFO | Analyzing chunk frames = [2657792,2658304) | 20260506 16:06:15.552 | INFO | MemInfo: { | 20260506 16:06:15.552 | INFO | "gpu" : " 17 GB", | 20260506 16:06:15.552 | INFO | "huge" : "", | 20260506 16:06:15.552 | INFO | "huge_shared" : " 27 GB", | 20260506 16:06:15.552 | INFO | "malloc" : " 14 GB", | 20260506 16:06:15.552 | INFO | "pinned" : " 29 GB" | 20260506 16:06:15.552 | INFO | } | 20260506 16:06:15.552 | INFO | Perf report for the last 5.1374 seconds: | 20260506 16:06:15.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:06:15.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:15.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:15.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:15.552 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:06:15.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:15.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:15.552 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:06:15.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:15.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:15.552 | INFO | Uploaded 36GiB to GPU in 1.95914s (18.3754GiB/s) | 20260506 16:06:15.552 | INFO | Downloaded 34GiB to GPU in 1.98067s (17.1659GiB/s) | 20260506 16:06:15.666 | INFO | Starting a round of Baz IO | 20260506 16:06:18.814 | INFO | Populating RTMetrics report | 20260506 16:06:18.815 | INFO | RTMetrics report populated | 20260506 16:06:18.815 | INFO | Opening tmp RTMetrics json file | 20260506 16:06:18.886 | INFO | Serializing RTMetrics to json | 20260506 16:06:18.959 | INFO | Serialization done, swapping tmp file now... | 20260506 16:06:18.963 | INFO | RTMetrics done | 20260506 16:06:19.451 | INFO | Finished a round of Baz IO: Wrote 2.088 GiB in 3.784 seconds | 20260506 16:06:19.451 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.62% / 0.01% / 63.38% | 20260506 16:06:19.451 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.79182% / 1.19209e-05% / 6.75519e-05% | 20260506 16:06:19.451 | INFO | Baz IO overall: | 20260506 16:06:19.451 | INFO | BytesWritten : 1392538481725 | 20260506 16:06:19.451 | INFO | NumEvents : 673883689050 | 20260506 16:06:20.690 | INFO | Analyzing chunk frames = [2658304,2658816) | 20260506 16:06:20.690 | INFO | MemInfo: { | 20260506 16:06:20.690 | INFO | "gpu" : " 17 GB", | 20260506 16:06:20.690 | INFO | "huge" : "", | 20260506 16:06:20.690 | INFO | "huge_shared" : " 27 GB", | 20260506 16:06:20.690 | INFO | "malloc" : " 14 GB", | 20260506 16:06:20.690 | INFO | "pinned" : " 29 GB" | 20260506 16:06:20.690 | INFO | } | 20260506 16:06:20.690 | INFO | Perf report for the last 5.13829 seconds: | 20260506 16:06:20.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:06:20.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:20.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:20.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:20.690 | INFO | ANALYSIS : 62.9%, 2.9, 95.6ms, 0 | 20260506 16:06:20.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:20.690 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260506 16:06:20.690 | INFO | PRE_HQ : 60.6%, 1.9, 58.7ms, 0 | 20260506 16:06:20.690 | INFO | BAZWRITER : 73.3%, 4.8, 180.8ms, 0 | 20260506 16:06:20.690 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 16:06:20.691 | INFO | Uploaded 36GiB to GPU in 2.06596s (17.4254GiB/s) | 20260506 16:06:20.691 | INFO | Downloaded 38GiB to GPU in 2.2131s (17.1705GiB/s) | 20260506 16:06:25.830 | INFO | Analyzing chunk frames = [2658816,2659328) | 20260506 16:06:25.830 | INFO | MemInfo: { | 20260506 16:06:25.830 | INFO | "gpu" : " 17 GB", | 20260506 16:06:25.830 | INFO | "huge" : "", | 20260506 16:06:25.830 | INFO | "huge_shared" : " 27 GB", | 20260506 16:06:25.830 | INFO | "malloc" : " 14 GB", | 20260506 16:06:25.830 | INFO | "pinned" : " 29 GB" | 20260506 16:06:25.830 | INFO | } | 20260506 16:06:25.830 | INFO | Perf report for the last 5.13941 seconds: | 20260506 16:06:25.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:06:25.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:25.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:25.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:25.830 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:06:25.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:25.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:25.830 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:06:25.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:25.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:25.830 | INFO | Uploaded 36GiB to GPU in 1.99906s (18.0084GiB/s) | 20260506 16:06:25.830 | INFO | Downloaded 34GiB to GPU in 1.95424s (17.3981GiB/s) | 20260506 16:06:30.969 | INFO | Analyzing chunk frames = [2659328,2659840) | 20260506 16:06:30.969 | INFO | MemInfo: { | 20260506 16:06:30.969 | INFO | "gpu" : " 17 GB", | 20260506 16:06:30.969 | INFO | "huge" : "", | 20260506 16:06:30.969 | INFO | "huge_shared" : " 27 GB", | 20260506 16:06:30.969 | INFO | "malloc" : " 14 GB", | 20260506 16:06:30.969 | INFO | "pinned" : " 29 GB" | 20260506 16:06:30.970 | INFO | } | 20260506 16:06:30.970 | INFO | Perf report for the last 5.14001 seconds: | 20260506 16:06:30.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:06:30.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:30.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:30.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:30.970 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 16:06:30.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:30.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:30.970 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:06:30.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:30.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:30.970 | INFO | Uploaded 36GiB to GPU in 1.99299s (18.0633GiB/s) | 20260506 16:06:30.970 | INFO | Downloaded 34GiB to GPU in 1.9407s (17.5195GiB/s) | 20260506 16:06:36.110 | INFO | Analyzing chunk frames = [2659840,2660352) | 20260506 16:06:36.110 | INFO | MemInfo: { | 20260506 16:06:36.110 | INFO | "gpu" : " 17 GB", | 20260506 16:06:36.111 | INFO | "huge" : "", | 20260506 16:06:36.111 | INFO | "huge_shared" : " 27 GB", | 20260506 16:06:36.111 | INFO | "malloc" : " 14 GB", | 20260506 16:06:36.111 | INFO | "pinned" : " 29 GB" | 20260506 16:06:36.111 | INFO | } | 20260506 16:06:36.111 | INFO | Perf report for the last 5.141 seconds: | 20260506 16:06:36.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:06:36.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:36.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:36.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:36.111 | INFO | ANALYSIS : 60.6%, 3.0, 94.0ms, 0 | 20260506 16:06:36.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:36.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:36.111 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:06:36.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:36.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:36.111 | INFO | Uploaded 36GiB to GPU in 2.00859s (17.9231GiB/s) | 20260506 16:06:36.111 | INFO | Downloaded 34GiB to GPU in 1.91224s (17.7802GiB/s) | 20260506 16:06:41.248 | INFO | Analyzing chunk frames = [2660352,2660864) | 20260506 16:06:41.248 | INFO | MemInfo: { | 20260506 16:06:41.248 | INFO | "gpu" : " 17 GB", | 20260506 16:06:41.248 | INFO | "huge" : "", | 20260506 16:06:41.248 | INFO | "huge_shared" : " 27 GB", | 20260506 16:06:41.248 | INFO | "malloc" : " 14 GB", | 20260506 16:06:41.248 | INFO | "pinned" : " 29 GB" | 20260506 16:06:41.248 | INFO | } | 20260506 16:06:41.248 | INFO | Perf report for the last 5.13754 seconds: | 20260506 16:06:41.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:06:41.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:41.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:41.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:41.248 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 16:06:41.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:41.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:41.248 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:06:41.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:41.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:41.249 | INFO | Uploaded 36GiB to GPU in 1.98967s (18.0935GiB/s) | 20260506 16:06:41.249 | INFO | Downloaded 34GiB to GPU in 1.92331s (17.6779GiB/s) | 20260506 16:06:46.387 | INFO | Analyzing chunk frames = [2660864,2661376) | 20260506 16:06:46.387 | INFO | MemInfo: { | 20260506 16:06:46.387 | INFO | "gpu" : " 17 GB", | 20260506 16:06:46.387 | INFO | "huge" : "", | 20260506 16:06:46.387 | INFO | "huge_shared" : " 27 GB", | 20260506 16:06:46.387 | INFO | "malloc" : " 14 GB", | 20260506 16:06:46.387 | INFO | "pinned" : " 29 GB" | 20260506 16:06:46.387 | INFO | } | 20260506 16:06:46.387 | INFO | Perf report for the last 5.13896 seconds: | 20260506 16:06:46.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:06:46.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:46.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:46.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:46.387 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:06:46.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:46.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:46.387 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 16:06:46.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:46.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:46.388 | INFO | Uploaded 36GiB to GPU in 2.06416s (17.4405GiB/s) | 20260506 16:06:46.388 | INFO | Downloaded 34GiB to GPU in 1.98237s (17.1512GiB/s) | 20260506 16:06:51.528 | INFO | Analyzing chunk frames = [2661376,2661888) | 20260506 16:06:51.528 | INFO | MemInfo: { | 20260506 16:06:51.528 | INFO | "gpu" : " 17 GB", | 20260506 16:06:51.528 | INFO | "huge" : "", | 20260506 16:06:51.528 | INFO | "huge_shared" : " 27 GB", | 20260506 16:06:51.528 | INFO | "malloc" : " 14 GB", | 20260506 16:06:51.528 | INFO | "pinned" : " 29 GB" | 20260506 16:06:51.528 | INFO | } | 20260506 16:06:51.528 | INFO | Perf report for the last 5.1409 seconds: | 20260506 16:06:51.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:06:51.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:51.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:51.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:51.528 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:06:51.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:51.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:51.528 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 16:06:51.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:51.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:51.528 | INFO | Uploaded 36GiB to GPU in 2.00244s (17.9781GiB/s) | 20260506 16:06:51.529 | INFO | Downloaded 34GiB to GPU in 2.00181s (16.9846GiB/s) | 20260506 16:06:56.667 | INFO | Analyzing chunk frames = [2661888,2662400) | 20260506 16:06:56.667 | INFO | MemInfo: { | 20260506 16:06:56.667 | INFO | "gpu" : " 17 GB", | 20260506 16:06:56.667 | INFO | "huge" : "", | 20260506 16:06:56.668 | INFO | "huge_shared" : " 27 GB", | 20260506 16:06:56.668 | INFO | "malloc" : " 14 GB", | 20260506 16:06:56.668 | INFO | "pinned" : " 29 GB" | 20260506 16:06:56.668 | INFO | } | 20260506 16:06:56.668 | INFO | Perf report for the last 5.13955 seconds: | 20260506 16:06:56.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:06:56.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:56.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:56.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:56.668 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 16:06:56.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:56.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:56.668 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:06:56.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:56.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:06:56.668 | INFO | Uploaded 36GiB to GPU in 2.01943s (17.8268GiB/s) | 20260506 16:06:56.668 | INFO | Downloaded 34GiB to GPU in 1.98543s (17.1247GiB/s) | 20260506 16:06:56.668 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:06:56.668 | INFO | Total mem usage: 17.4 GiB | 20260506 16:06:56.668 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 16:06:56.668 | INFO | Recent High Water Marks Sum: 16.3 GiB (Savings: -7.03%) | 20260506 16:06:56.668 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:06:56.668 | INFO | High Water Breakdown: | 20260506 16:06:56.668 | INFO | 61.9% : 10.1 GiB ( 92.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:06:56.668 | INFO | 38% : 6.17 GiB ( 92.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:06:56.668 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:06:56.668 | INFO | ------------------------------------------ | 20260506 16:06:56.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:06:56.668 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:06:56.668 | INFO | Total mem usage: 26.5 GiB | 20260506 16:06:56.668 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:06:56.668 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:06:56.668 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:06:56.668 | INFO | High Water Breakdown: | 20260506 16:06:56.668 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:06:56.668 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:06:56.668 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:06:56.669 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:06:56.669 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:06:56.669 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:06:56.669 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:06:56.669 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:06:56.669 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:06:56.669 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:06:56.669 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:06:56.669 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:06:56.669 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:06:56.669 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:06:56.669 | INFO | ------------------------------------------ | 20260506 16:06:56.669 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:06:56.669 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:06:56.669 | INFO | Total mem usage: 25.6 GiB | 20260506 16:06:56.669 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:06:56.669 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:06:56.669 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:06:56.669 | INFO | High Water Breakdown: | 20260506 16:06:56.669 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:06:56.669 | INFO | ------------------------------------------ | 20260506 16:06:56.669 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:06:56.669 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:06:56.669 | INFO | Total mem usage: 15.7 GiB | 20260506 16:06:56.669 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:06:56.669 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:06:56.669 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:06:56.669 | INFO | High Water Breakdown: | 20260506 16:06:56.669 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:06:56.669 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:06:56.669 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:06:56.669 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:06:56.669 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:06:56.669 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:06:56.669 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:06:56.669 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:06:56.669 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:06:56.669 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:06:56.669 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:06:56.669 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:06:56.669 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:06:56.669 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:06:56.669 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:06:56.669 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:06:56.669 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:06:56.669 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:06:56.669 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:06:56.669 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:06:56.669 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:06:56.669 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:06:56.669 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:06:56.669 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:06:56.669 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:06:56.669 | INFO | ------------------------------------------ | 20260506 16:06:56.669 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:06:56.669 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:06:56.669 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:06:56.669 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:06:56.669 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.50e+01, 1.34e+01 / 1.60e+00 | 20260506 16:06:56.669 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.04e-01, 1.08e-02 / 9.57e-03 | 20260506 16:06:56.669 | INFO | PerfReport: 11.59%, Baseline -- 3.63e+00 / 4.42e+00, 5.56e-01 / 3.53e-01 | 20260506 16:06:56.669 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.84e-01 / 1.47e-01 | 20260506 16:06:56.669 | INFO | PerfReport: 15.83%, DME -- 3.97e+01 / 4.34e+01, 3.58e+00 / 1.85e+00 | 20260506 16:06:56.669 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.45e+00 / 2.07e+00 | 20260506 16:06:56.669 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.31e+00 / 2.88e+00, 3.97e-01 / 2.53e-01 | 20260506 16:06:56.669 | INFO | PerfReport: 3.51%, Metrics -- 1.10e+00 / 1.43e+00, 2.84e-01 / 1.63e-01 | 20260506 16:06:56.669 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.82e-01 / 4.28e-01, 5.91e-02 / 2.36e-02 | 20260506 16:06:56.669 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:06:56.669 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:06:56.669 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:06:56.669 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 16:06:56.669 | INFO | PerfReport: 21.51%, Upload -- 2.09e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260506 16:06:56.669 | INFO | PerfReport: 21.26%, Download -- 2.07e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 16:06:56.669 | INFO | PerfReport: 32.21%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 16:06:56.669 | INFO | PerfReport: 1.31%, LockUpload -- 1.28e+00 / 0.00e+00, 5.36e+00 / 0.00e+00 | 20260506 16:06:56.669 | INFO | PerfReport: 1.19%, LockDownload -- 1.16e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 16:06:56.669 | INFO | PerfReport: 22.50%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 16:06:56.669 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260506 16:06:56.782 | INFO | Starting a round of Baz IO | 20260506 16:06:59.928 | INFO | Populating RTMetrics report | 20260506 16:06:59.929 | INFO | RTMetrics report populated | 20260506 16:06:59.929 | INFO | Opening tmp RTMetrics json file | 20260506 16:06:59.966 | INFO | Serializing RTMetrics to json | 20260506 16:06:59.991 | INFO | Serialization done, swapping tmp file now... | 20260506 16:07:00.016 | INFO | RTMetrics done | 20260506 16:07:00.545 | INFO | Finished a round of Baz IO: Wrote 2.087 GiB in 3.763 seconds | 20260506 16:07:00.545 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.61% / 0.01% / 63.38% | 20260506 16:07:00.545 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.78348% / 3.97364e-06% / 5.5631e-05% | 20260506 16:07:00.545 | INFO | Baz IO overall: | 20260506 16:07:00.546 | INFO | BytesWritten : 1394779101952 | 20260506 16:07:00.546 | INFO | NumEvents : 674967083583 | 20260506 16:07:01.806 | INFO | Analyzing chunk frames = [2662400,2662912) | 20260506 16:07:01.806 | INFO | MemInfo: { | 20260506 16:07:01.806 | INFO | "gpu" : " 17 GB", | 20260506 16:07:01.806 | INFO | "huge" : "", | 20260506 16:07:01.806 | INFO | "huge_shared" : " 27 GB", | 20260506 16:07:01.806 | INFO | "malloc" : " 14 GB", | 20260506 16:07:01.806 | INFO | "pinned" : " 29 GB" | 20260506 16:07:01.806 | INFO | } | 20260506 16:07:01.806 | INFO | Perf report for the last 5.1383 seconds: | 20260506 16:07:01.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:07:01.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:01.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:01.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:01.806 | INFO | ANALYSIS : 62.9%, 2.9, 95.3ms, 0 | 20260506 16:07:01.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:01.806 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 16:07:01.806 | INFO | PRE_HQ : 60.6%, 1.9, 58.8ms, 0 | 20260506 16:07:01.806 | INFO | BAZWRITER : 72.6%, 4.9, 181.1ms, 0 | 20260506 16:07:01.806 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 16:07:01.806 | INFO | Uploaded 36GiB to GPU in 2.05899s (17.4843GiB/s) | 20260506 16:07:01.806 | INFO | Downloaded 38GiB to GPU in 2.2423s (16.9469GiB/s) | 20260506 16:07:06.945 | INFO | Analyzing chunk frames = [2662912,2663424) | 20260506 16:07:06.945 | INFO | MemInfo: { | 20260506 16:07:06.945 | INFO | "gpu" : " 17 GB", | 20260506 16:07:06.945 | INFO | "huge" : "", | 20260506 16:07:06.945 | INFO | "huge_shared" : " 27 GB", | 20260506 16:07:06.945 | INFO | "malloc" : " 14 GB", | 20260506 16:07:06.945 | INFO | "pinned" : " 29 GB" | 20260506 16:07:06.945 | INFO | } | 20260506 16:07:06.945 | INFO | Perf report for the last 5.13915 seconds: | 20260506 16:07:06.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:07:06.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:06.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:06.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:06.945 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 16:07:06.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:06.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:06.945 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:07:06.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:06.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:06.945 | INFO | Uploaded 36GiB to GPU in 1.97726s (18.2071GiB/s) | 20260506 16:07:06.945 | INFO | Downloaded 34GiB to GPU in 1.93246s (17.5941GiB/s) | 20260506 16:07:12.085 | INFO | Analyzing chunk frames = [2663424,2663936) | 20260506 16:07:12.085 | INFO | MemInfo: { | 20260506 16:07:12.085 | INFO | "gpu" : " 17 GB", | 20260506 16:07:12.085 | INFO | "huge" : "", | 20260506 16:07:12.085 | INFO | "huge_shared" : " 27 GB", | 20260506 16:07:12.085 | INFO | "malloc" : " 14 GB", | 20260506 16:07:12.085 | INFO | "pinned" : " 29 GB" | 20260506 16:07:12.085 | INFO | } | 20260506 16:07:12.085 | INFO | Perf report for the last 5.13978 seconds: | 20260506 16:07:12.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:07:12.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:12.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:12.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:12.085 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 16:07:12.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:12.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:12.085 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:07:12.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:12.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:12.085 | INFO | Uploaded 36GiB to GPU in 1.94743s (18.4859GiB/s) | 20260506 16:07:12.085 | INFO | Downloaded 34GiB to GPU in 1.92767s (17.6379GiB/s) | 20260506 16:07:17.224 | INFO | Analyzing chunk frames = [2663936,2664448) | 20260506 16:07:17.224 | INFO | MemInfo: { | 20260506 16:07:17.224 | INFO | "gpu" : " 17 GB", | 20260506 16:07:17.224 | INFO | "huge" : "", | 20260506 16:07:17.224 | INFO | "huge_shared" : " 27 GB", | 20260506 16:07:17.224 | INFO | "malloc" : " 14 GB", | 20260506 16:07:17.224 | INFO | "pinned" : " 29 GB" | 20260506 16:07:17.224 | INFO | } | 20260506 16:07:17.224 | INFO | Perf report for the last 5.13927 seconds: | 20260506 16:07:17.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:07:17.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:17.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:17.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:17.224 | INFO | ANALYSIS : 60.6%, 3.0, 94.0ms, 0 | 20260506 16:07:17.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:17.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:17.224 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:07:17.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:17.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:17.225 | INFO | Uploaded 36GiB to GPU in 2.01485s (17.8674GiB/s) | 20260506 16:07:17.225 | INFO | Downloaded 34GiB to GPU in 1.90797s (17.82GiB/s) | 20260506 16:07:22.363 | INFO | Analyzing chunk frames = [2664448,2664960) | 20260506 16:07:22.363 | INFO | MemInfo: { | 20260506 16:07:22.363 | INFO | "gpu" : " 17 GB", | 20260506 16:07:22.363 | INFO | "huge" : "", | 20260506 16:07:22.363 | INFO | "huge_shared" : " 27 GB", | 20260506 16:07:22.363 | INFO | "malloc" : " 14 GB", | 20260506 16:07:22.363 | INFO | "pinned" : " 29 GB" | 20260506 16:07:22.363 | INFO | } | 20260506 16:07:22.363 | INFO | Perf report for the last 5.13901 seconds: | 20260506 16:07:22.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:07:22.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:22.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:22.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:22.363 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 16:07:22.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:22.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:22.363 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:07:22.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:22.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:22.364 | INFO | Uploaded 36GiB to GPU in 1.97922s (18.189GiB/s) | 20260506 16:07:22.364 | INFO | Downloaded 34GiB to GPU in 1.92669s (17.6469GiB/s) | 20260506 16:07:27.502 | INFO | Analyzing chunk frames = [2664960,2665472) | 20260506 16:07:27.502 | INFO | MemInfo: { | 20260506 16:07:27.502 | INFO | "gpu" : " 17 GB", | 20260506 16:07:27.502 | INFO | "huge" : "", | 20260506 16:07:27.502 | INFO | "huge_shared" : " 27 GB", | 20260506 16:07:27.502 | INFO | "malloc" : " 14 GB", | 20260506 16:07:27.502 | INFO | "pinned" : " 29 GB" | 20260506 16:07:27.502 | INFO | } | 20260506 16:07:27.502 | INFO | Perf report for the last 5.13871 seconds: | 20260506 16:07:27.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:07:27.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:27.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:27.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:27.502 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260506 16:07:27.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:27.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:27.502 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260506 16:07:27.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:27.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:27.502 | INFO | Uploaded 36GiB to GPU in 2.02857s (17.7465GiB/s) | 20260506 16:07:27.502 | INFO | Downloaded 34GiB to GPU in 1.94327s (17.4963GiB/s) | 20260506 16:07:32.643 | INFO | Analyzing chunk frames = [2665472,2665984) | 20260506 16:07:32.643 | INFO | MemInfo: { | 20260506 16:07:32.643 | INFO | "gpu" : " 17 GB", | 20260506 16:07:32.643 | INFO | "huge" : "", | 20260506 16:07:32.643 | INFO | "huge_shared" : " 27 GB", | 20260506 16:07:32.643 | INFO | "malloc" : " 14 GB", | 20260506 16:07:32.643 | INFO | "pinned" : " 29 GB" | 20260506 16:07:32.643 | INFO | } | 20260506 16:07:32.643 | INFO | Perf report for the last 5.14084 seconds: | 20260506 16:07:32.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:07:32.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:32.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:32.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:32.643 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:07:32.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:32.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:32.643 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:07:32.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:32.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:32.643 | INFO | Uploaded 36GiB to GPU in 1.99973s (18.0024GiB/s) | 20260506 16:07:32.643 | INFO | Downloaded 34GiB to GPU in 1.97534s (17.2123GiB/s) | 20260506 16:07:37.783 | INFO | Analyzing chunk frames = [2665984,2666496) | 20260506 16:07:37.783 | INFO | MemInfo: { | 20260506 16:07:37.783 | INFO | "gpu" : " 17 GB", | 20260506 16:07:37.783 | INFO | "huge" : "", | 20260506 16:07:37.783 | INFO | "huge_shared" : " 27 GB", | 20260506 16:07:37.783 | INFO | "malloc" : " 14 GB", | 20260506 16:07:37.783 | INFO | "pinned" : " 29 GB" | 20260506 16:07:37.783 | INFO | } | 20260506 16:07:37.783 | INFO | Perf report for the last 5.13988 seconds: | 20260506 16:07:37.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:07:37.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:37.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:37.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:37.783 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:07:37.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:37.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:37.783 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:07:37.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:37.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:37.783 | INFO | Uploaded 36GiB to GPU in 2.02676s (17.7624GiB/s) | 20260506 16:07:37.783 | INFO | Downloaded 34GiB to GPU in 1.98694s (17.1118GiB/s) | 20260506 16:07:37.902 | INFO | Starting a round of Baz IO | 20260506 16:07:41.047 | INFO | Populating RTMetrics report | 20260506 16:07:41.047 | INFO | RTMetrics report populated | 20260506 16:07:41.047 | INFO | Opening tmp RTMetrics json file | 20260506 16:07:41.060 | INFO | Serializing RTMetrics to json | 20260506 16:07:41.065 | INFO | Serialization done, swapping tmp file now... | 20260506 16:07:41.071 | INFO | RTMetrics done | 20260506 16:07:41.381 | INFO | Finished a round of Baz IO: Wrote 2.087 GiB in 3.478 seconds | 20260506 16:07:41.381 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.60% / 0.01% / 63.39% | 20260506 16:07:41.381 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.77869% / 1.58946e-05% / 6.35783e-05% | 20260506 16:07:41.381 | INFO | Baz IO overall: | 20260506 16:07:41.381 | INFO | BytesWritten : 1397020161045 | 20260506 16:07:41.381 | INFO | NumEvents : 676050606908 | 20260506 16:07:42.920 | INFO | Analyzing chunk frames = [2666496,2667008) | 20260506 16:07:42.920 | INFO | MemInfo: { | 20260506 16:07:42.920 | INFO | "gpu" : " 17 GB", | 20260506 16:07:42.920 | INFO | "huge" : "", | 20260506 16:07:42.920 | INFO | "huge_shared" : " 27 GB", | 20260506 16:07:42.920 | INFO | "malloc" : " 14 GB", | 20260506 16:07:42.920 | INFO | "pinned" : " 29 GB" | 20260506 16:07:42.920 | INFO | } | 20260506 16:07:42.920 | INFO | Perf report for the last 5.13757 seconds: | 20260506 16:07:42.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:07:42.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:42.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:42.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:42.920 | INFO | ANALYSIS : 62.9%, 2.9, 95.7ms, 0 | 20260506 16:07:42.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:42.921 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 16:07:42.921 | INFO | PRE_HQ : 60.5%, 1.9, 58.5ms, 0 | 20260506 16:07:42.921 | INFO | BAZWRITER : 67.3%, 4.8, 166.8ms, 0 | 20260506 16:07:42.921 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 16:07:42.921 | INFO | Uploaded 36GiB to GPU in 2.05282s (17.5369GiB/s) | 20260506 16:07:42.921 | INFO | Downloaded 38GiB to GPU in 2.22434s (17.0837GiB/s) | 20260506 16:07:48.060 | INFO | Analyzing chunk frames = [2667008,2667520) | 20260506 16:07:48.060 | INFO | MemInfo: { | 20260506 16:07:48.060 | INFO | "gpu" : " 17 GB", | 20260506 16:07:48.060 | INFO | "huge" : "", | 20260506 16:07:48.060 | INFO | "huge_shared" : " 27 GB", | 20260506 16:07:48.060 | INFO | "malloc" : " 14 GB", | 20260506 16:07:48.060 | INFO | "pinned" : " 29 GB" | 20260506 16:07:48.060 | INFO | } | 20260506 16:07:48.060 | INFO | Perf report for the last 5.13977 seconds: | 20260506 16:07:48.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:07:48.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:48.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:48.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:48.060 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:07:48.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:48.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:48.060 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:07:48.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:48.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:48.060 | INFO | Uploaded 36GiB to GPU in 1.99235s (18.0691GiB/s) | 20260506 16:07:48.060 | INFO | Downloaded 34GiB to GPU in 1.93379s (17.582GiB/s) | 20260506 16:07:53.199 | INFO | Analyzing chunk frames = [2667520,2668032) | 20260506 16:07:53.199 | INFO | MemInfo: { | 20260506 16:07:53.199 | INFO | "gpu" : " 17 GB", | 20260506 16:07:53.199 | INFO | "huge" : "", | 20260506 16:07:53.199 | INFO | "huge_shared" : " 27 GB", | 20260506 16:07:53.199 | INFO | "malloc" : " 14 GB", | 20260506 16:07:53.199 | INFO | "pinned" : " 29 GB" | 20260506 16:07:53.199 | INFO | } | 20260506 16:07:53.199 | INFO | Perf report for the last 5.13867 seconds: | 20260506 16:07:53.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:07:53.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:53.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:53.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:53.199 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 16:07:53.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:53.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:53.199 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:07:53.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:53.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:53.199 | INFO | Uploaded 36GiB to GPU in 1.99175s (18.0746GiB/s) | 20260506 16:07:53.199 | INFO | Downloaded 34GiB to GPU in 1.92486s (17.6637GiB/s) | 20260506 16:07:58.340 | INFO | Analyzing chunk frames = [2668032,2668544) | 20260506 16:07:58.340 | INFO | MemInfo: { | 20260506 16:07:58.340 | INFO | "gpu" : " 17 GB", | 20260506 16:07:58.340 | INFO | "huge" : "", | 20260506 16:07:58.340 | INFO | "huge_shared" : " 27 GB", | 20260506 16:07:58.340 | INFO | "malloc" : " 14 GB", | 20260506 16:07:58.340 | INFO | "pinned" : " 29 GB" | 20260506 16:07:58.340 | INFO | } | 20260506 16:07:58.340 | INFO | Perf report for the last 5.14153 seconds: | 20260506 16:07:58.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:07:58.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:58.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:58.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:58.340 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 16:07:58.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:58.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:58.340 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:07:58.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:58.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:07:58.341 | INFO | Uploaded 36GiB to GPU in 2.02006s (17.8212GiB/s) | 20260506 16:07:58.341 | INFO | Downloaded 34GiB to GPU in 1.9112s (17.7899GiB/s) | 20260506 16:08:03.478 | INFO | Analyzing chunk frames = [2668544,2669056) | 20260506 16:08:03.478 | INFO | MemInfo: { | 20260506 16:08:03.478 | INFO | "gpu" : " 17 GB", | 20260506 16:08:03.478 | INFO | "huge" : "", | 20260506 16:08:03.478 | INFO | "huge_shared" : " 27 GB", | 20260506 16:08:03.478 | INFO | "malloc" : " 14 GB", | 20260506 16:08:03.478 | INFO | "pinned" : " 29 GB" | 20260506 16:08:03.478 | INFO | } | 20260506 16:08:03.478 | INFO | Perf report for the last 5.13761 seconds: | 20260506 16:08:03.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:08:03.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:03.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:03.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:03.478 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 16:08:03.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:03.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:03.478 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 16:08:03.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:03.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:03.478 | INFO | Uploaded 36GiB to GPU in 1.9999s (18.0009GiB/s) | 20260506 16:08:03.478 | INFO | Downloaded 34GiB to GPU in 1.9394s (17.5312GiB/s) | 20260506 16:08:08.617 | INFO | Analyzing chunk frames = [2669056,2669568) | 20260506 16:08:08.617 | INFO | MemInfo: { | 20260506 16:08:08.617 | INFO | "gpu" : " 17 GB", | 20260506 16:08:08.617 | INFO | "huge" : "", | 20260506 16:08:08.617 | INFO | "huge_shared" : " 27 GB", | 20260506 16:08:08.617 | INFO | "malloc" : " 14 GB", | 20260506 16:08:08.617 | INFO | "pinned" : " 29 GB" | 20260506 16:08:08.617 | INFO | } | 20260506 16:08:08.617 | INFO | Perf report for the last 5.13907 seconds: | 20260506 16:08:08.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:08:08.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:08.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:08.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:08.617 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:08:08.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:08.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:08.617 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 16:08:08.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:08.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:08.617 | INFO | Uploaded 36GiB to GPU in 2.07086s (17.3841GiB/s) | 20260506 16:08:08.617 | INFO | Downloaded 34GiB to GPU in 1.97354s (17.2279GiB/s) | 20260506 16:08:13.757 | INFO | Analyzing chunk frames = [2669568,2670080) | 20260506 16:08:13.758 | INFO | MemInfo: { | 20260506 16:08:13.758 | INFO | "gpu" : " 17 GB", | 20260506 16:08:13.758 | INFO | "huge" : "", | 20260506 16:08:13.758 | INFO | "huge_shared" : " 27 GB", | 20260506 16:08:13.758 | INFO | "malloc" : " 14 GB", | 20260506 16:08:13.758 | INFO | "pinned" : " 29 GB" | 20260506 16:08:13.758 | INFO | } | 20260506 16:08:13.758 | INFO | Perf report for the last 5.14068 seconds: | 20260506 16:08:13.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:08:13.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:13.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:13.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:13.758 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 16:08:13.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:13.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:13.758 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:08:13.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:13.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:13.758 | INFO | Uploaded 36GiB to GPU in 2.0272s (17.7585GiB/s) | 20260506 16:08:13.758 | INFO | Downloaded 34GiB to GPU in 1.99448s (17.0471GiB/s) | 20260506 16:08:18.896 | INFO | Analyzing chunk frames = [2670080,2670592) | 20260506 16:08:18.896 | INFO | MemInfo: { | 20260506 16:08:18.896 | INFO | "gpu" : " 17 GB", | 20260506 16:08:18.896 | INFO | "huge" : "", | 20260506 16:08:18.896 | INFO | "huge_shared" : " 27 GB", | 20260506 16:08:18.896 | INFO | "malloc" : " 14 GB", | 20260506 16:08:18.896 | INFO | "pinned" : " 29 GB" | 20260506 16:08:18.896 | INFO | } | 20260506 16:08:18.896 | INFO | Perf report for the last 5.13827 seconds: | 20260506 16:08:18.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:08:18.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:18.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:18.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:18.896 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 16:08:18.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:18.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:18.896 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 16:08:18.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:18.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:18.896 | INFO | Uploaded 36GiB to GPU in 2.04185s (17.6311GiB/s) | 20260506 16:08:18.897 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:08:18.897 | INFO | Total mem usage: 17.4 GiB | 20260506 16:08:18.897 | INFO | Downloaded 34GiB to GPU in 2.00099s (16.9916GiB/s) | 20260506 16:08:18.897 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 16:08:18.897 | INFO | Recent High Water Marks Sum: 16.3 GiB (Savings: -7.07%) | 20260506 16:08:18.897 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:08:18.897 | INFO | High Water Breakdown: | 20260506 16:08:18.897 | INFO | 61.9% : 10.1 GiB ( 92.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:08:18.897 | INFO | 38% : 6.17 GiB ( 92.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:08:18.897 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:08:18.897 | INFO | ------------------------------------------ | 20260506 16:08:18.897 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:08:18.897 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:08:18.897 | INFO | Total mem usage: 26.5 GiB | 20260506 16:08:18.897 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:08:18.897 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:08:18.897 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:08:18.897 | INFO | High Water Breakdown: | 20260506 16:08:18.897 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:08:18.897 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:08:18.897 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:08:18.897 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:08:18.897 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:08:18.897 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:08:18.897 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:08:18.897 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:08:18.897 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:08:18.897 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:08:18.897 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:08:18.897 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:08:18.897 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:08:18.897 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:08:18.897 | INFO | ------------------------------------------ | 20260506 16:08:18.897 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:08:18.897 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:08:18.897 | INFO | Total mem usage: 25.6 GiB | 20260506 16:08:18.897 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:08:18.897 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:08:18.897 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:08:18.897 | INFO | High Water Breakdown: | 20260506 16:08:18.897 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:08:18.897 | INFO | ------------------------------------------ | 20260506 16:08:18.897 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:08:18.897 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:08:18.897 | INFO | Total mem usage: 15.7 GiB | 20260506 16:08:18.897 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:08:18.897 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:08:18.897 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 16:08:18.897 | INFO | High Water Breakdown: | 20260506 16:08:18.897 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:08:18.897 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:08:18.897 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:08:18.897 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:08:18.897 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:08:18.897 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:08:18.897 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:08:18.897 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:08:18.897 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:08:18.897 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:08:18.897 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:08:18.897 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:08:18.897 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:08:18.897 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:08:18.897 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:08:18.897 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:08:18.897 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:08:18.897 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:08:18.897 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:08:18.897 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:08:18.897 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:08:18.898 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:08:18.898 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:08:18.898 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:08:18.898 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:08:18.898 | INFO | ------------------------------------------ | 20260506 16:08:18.898 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:08:18.898 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:08:18.898 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:08:18.898 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:08:18.898 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.55e+01, 1.35e+01 / 1.94e+00 | 20260506 16:08:18.898 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.01e-01, 1.11e-02 / 7.39e-03 | 20260506 16:08:18.898 | INFO | PerfReport: 11.57%, Baseline -- 3.63e+00 / 4.36e+00, 5.54e-01 / 3.68e-01 | 20260506 16:08:18.898 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.43e+00, 1.82e-01 / 1.64e-01 | 20260506 16:08:18.898 | INFO | PerfReport: 15.87%, DME -- 3.99e+01 / 4.39e+01, 3.73e+00 / 2.09e+00 | 20260506 16:08:18.898 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.79e+01 / 2.09e+01, 1.50e+00 / 1.52e+00 | 20260506 16:08:18.898 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.30e+00 / 2.90e+00, 3.88e-01 / 2.02e-01 | 20260506 16:08:18.898 | INFO | PerfReport: 3.50%, Metrics -- 1.10e+00 / 1.47e+00, 2.86e-01 / 1.88e-01 | 20260506 16:08:18.898 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.83e-01 / 4.37e-01, 6.13e-02 / 3.30e-02 | 20260506 16:08:18.898 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:08:18.898 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:08:18.898 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:08:18.898 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 16:08:18.898 | INFO | PerfReport: 21.54%, Upload -- 2.10e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260506 16:08:18.898 | INFO | PerfReport: 21.22%, Download -- 2.07e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 16:08:18.898 | INFO | PerfReport: 32.23%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 16:08:18.898 | INFO | PerfReport: 1.44%, LockUpload -- 1.41e+00 / 0.00e+00, 5.38e+00 / 0.00e+00 | 20260506 16:08:18.898 | INFO | PerfReport: 1.12%, LockDownload -- 1.09e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260506 16:08:18.898 | INFO | PerfReport: 22.43%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 16:08:18.898 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260506 16:08:19.017 | INFO | Starting a round of Baz IO | 20260506 16:08:22.162 | INFO | Populating RTMetrics report | 20260506 16:08:22.162 | INFO | RTMetrics report populated | 20260506 16:08:22.162 | INFO | Opening tmp RTMetrics json file | 20260506 16:08:22.190 | INFO | Serializing RTMetrics to json | 20260506 16:08:22.194 | INFO | Serialization done, swapping tmp file now... | 20260506 16:08:22.198 | INFO | RTMetrics done | 20260506 16:08:22.594 | INFO | Finished a round of Baz IO: Wrote 2.086 GiB in 3.577 seconds | 20260506 16:08:22.595 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.59% / 0.01% / 63.40% | 20260506 16:08:22.595 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.77308% / 1.98682e-05% / 7.94729e-05% | 20260506 16:08:22.595 | INFO | Baz IO overall: | 20260506 16:08:22.595 | INFO | BytesWritten : 1399260396726 | 20260506 16:08:22.595 | INFO | NumEvents : 677133484514 | 20260506 16:08:24.035 | INFO | Analyzing chunk frames = [2670592,2671104) | 20260506 16:08:24.035 | INFO | MemInfo: { | 20260506 16:08:24.035 | INFO | "gpu" : " 17 GB", | 20260506 16:08:24.035 | INFO | "huge" : "", | 20260506 16:08:24.035 | INFO | "huge_shared" : " 27 GB", | 20260506 16:08:24.035 | INFO | "malloc" : " 14 GB", | 20260506 16:08:24.035 | INFO | "pinned" : " 29 GB" | 20260506 16:08:24.035 | INFO | } | 20260506 16:08:24.035 | INFO | Perf report for the last 5.13921 seconds: | 20260506 16:08:24.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:08:24.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:24.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:24.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:24.035 | INFO | ANALYSIS : 62.9%, 2.9, 95.6ms, 0 | 20260506 16:08:24.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:24.035 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 16:08:24.035 | INFO | PRE_HQ : 60.9%, 1.9, 58.9ms, 0 | 20260506 16:08:24.035 | INFO | BAZWRITER : 69.0%, 4.8, 171.0ms, 0 | 20260506 16:08:24.036 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 16:08:24.036 | INFO | Uploaded 36GiB to GPU in 2.05555s (17.5136GiB/s) | 20260506 16:08:24.036 | INFO | Downloaded 38GiB to GPU in 2.24223s (16.9475GiB/s) | 20260506 16:08:29.176 | INFO | Analyzing chunk frames = [2671104,2671616) | 20260506 16:08:29.176 | INFO | MemInfo: { | 20260506 16:08:29.176 | INFO | "gpu" : " 17 GB", | 20260506 16:08:29.176 | INFO | "huge" : "", | 20260506 16:08:29.176 | INFO | "huge_shared" : " 27 GB", | 20260506 16:08:29.176 | INFO | "malloc" : " 14 GB", | 20260506 16:08:29.176 | INFO | "pinned" : " 29 GB" | 20260506 16:08:29.176 | INFO | } | 20260506 16:08:29.176 | INFO | Perf report for the last 5.14111 seconds: | 20260506 16:08:29.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:08:29.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:29.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:29.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:29.176 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:08:29.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:29.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:29.177 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:08:29.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:29.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:29.177 | INFO | Uploaded 36GiB to GPU in 2.0231s (17.7944GiB/s) | 20260506 16:08:29.177 | INFO | Downloaded 34GiB to GPU in 1.93292s (17.5899GiB/s) | 20260506 16:08:34.314 | INFO | Analyzing chunk frames = [2671616,2672128) | 20260506 16:08:34.315 | INFO | MemInfo: { | 20260506 16:08:34.315 | INFO | "gpu" : " 17 GB", | 20260506 16:08:34.315 | INFO | "huge" : "", | 20260506 16:08:34.315 | INFO | "huge_shared" : " 27 GB", | 20260506 16:08:34.315 | INFO | "malloc" : " 14 GB", | 20260506 16:08:34.315 | INFO | "pinned" : " 29 GB" | 20260506 16:08:34.315 | INFO | } | 20260506 16:08:34.315 | INFO | Perf report for the last 5.13836 seconds: | 20260506 16:08:34.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:08:34.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:34.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:34.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:34.315 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 16:08:34.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:34.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:34.315 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:08:34.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:34.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:34.315 | INFO | Uploaded 36GiB to GPU in 1.98889s (18.1005GiB/s) | 20260506 16:08:34.315 | INFO | Downloaded 34GiB to GPU in 1.93623s (17.5599GiB/s) | 20260506 16:08:39.453 | INFO | Analyzing chunk frames = [2672128,2672640) | 20260506 16:08:39.453 | INFO | MemInfo: { | 20260506 16:08:39.453 | INFO | "gpu" : " 17 GB", | 20260506 16:08:39.453 | INFO | "huge" : "", | 20260506 16:08:39.453 | INFO | "huge_shared" : " 27 GB", | 20260506 16:08:39.453 | INFO | "malloc" : " 14 GB", | 20260506 16:08:39.453 | INFO | "pinned" : " 29 GB" | 20260506 16:08:39.453 | INFO | } | 20260506 16:08:39.453 | INFO | Perf report for the last 5.13853 seconds: | 20260506 16:08:39.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:08:39.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:39.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:39.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:39.453 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:08:39.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:39.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:39.453 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:08:39.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:39.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:39.454 | INFO | Uploaded 36GiB to GPU in 1.97369s (18.2399GiB/s) | 20260506 16:08:39.454 | INFO | Downloaded 34GiB to GPU in 1.88453s (18.0416GiB/s) | 20260506 16:08:44.593 | INFO | Analyzing chunk frames = [2672640,2673152) | 20260506 16:08:44.593 | INFO | MemInfo: { | 20260506 16:08:44.593 | INFO | "gpu" : " 17 GB", | 20260506 16:08:44.593 | INFO | "huge" : "", | 20260506 16:08:44.593 | INFO | "huge_shared" : " 27 GB", | 20260506 16:08:44.593 | INFO | "malloc" : " 14 GB", | 20260506 16:08:44.593 | INFO | "pinned" : " 29 GB" | 20260506 16:08:44.593 | INFO | } | 20260506 16:08:44.593 | INFO | Perf report for the last 5.13983 seconds: | 20260506 16:08:44.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:08:44.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:44.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:44.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:44.593 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 16:08:44.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:44.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:44.593 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:08:44.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:44.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:44.594 | INFO | Uploaded 36GiB to GPU in 1.96953s (18.2785GiB/s) | 20260506 16:08:44.594 | INFO | Downloaded 34GiB to GPU in 1.92026s (17.706GiB/s) | 20260506 16:08:49.733 | INFO | Analyzing chunk frames = [2673152,2673664) | 20260506 16:08:49.733 | INFO | MemInfo: { | 20260506 16:08:49.733 | INFO | "gpu" : " 17 GB", | 20260506 16:08:49.733 | INFO | "huge" : "", | 20260506 16:08:49.733 | INFO | "huge_shared" : " 27 GB", | 20260506 16:08:49.733 | INFO | "malloc" : " 14 GB", | 20260506 16:08:49.733 | INFO | "pinned" : " 29 GB" | 20260506 16:08:49.733 | INFO | } | 20260506 16:08:49.733 | INFO | Perf report for the last 5.13981 seconds: | 20260506 16:08:49.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:08:49.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:49.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:49.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:49.733 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:08:49.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:49.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:49.733 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260506 16:08:49.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:49.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:49.733 | INFO | Uploaded 36GiB to GPU in 2.06824s (17.4061GiB/s) | 20260506 16:08:49.733 | INFO | Downloaded 34GiB to GPU in 1.97068s (17.2529GiB/s) | 20260506 16:08:54.873 | INFO | Analyzing chunk frames = [2673664,2674176) | 20260506 16:08:54.873 | INFO | MemInfo: { | 20260506 16:08:54.873 | INFO | "gpu" : " 17 GB", | 20260506 16:08:54.873 | INFO | "huge" : "", | 20260506 16:08:54.873 | INFO | "huge_shared" : " 27 GB", | 20260506 16:08:54.873 | INFO | "malloc" : " 14 GB", | 20260506 16:08:54.873 | INFO | "pinned" : " 29 GB" | 20260506 16:08:54.873 | INFO | } | 20260506 16:08:54.873 | INFO | Perf report for the last 5.13975 seconds: | 20260506 16:08:54.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:08:54.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:54.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:54.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:54.873 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:08:54.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:54.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:54.873 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 16:08:54.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:54.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:08:54.873 | INFO | Uploaded 36GiB to GPU in 2.02906s (17.7422GiB/s) | 20260506 16:08:54.873 | INFO | Downloaded 34GiB to GPU in 2.00046s (16.9961GiB/s) | 20260506 16:09:00.011 | INFO | Analyzing chunk frames = [2674176,2674688) | 20260506 16:09:00.011 | INFO | MemInfo: { | 20260506 16:09:00.011 | INFO | "gpu" : " 17 GB", | 20260506 16:09:00.011 | INFO | "huge" : "", | 20260506 16:09:00.011 | INFO | "huge_shared" : " 27 GB", | 20260506 16:09:00.011 | INFO | "malloc" : " 14 GB", | 20260506 16:09:00.011 | INFO | "pinned" : " 29 GB" | 20260506 16:09:00.011 | INFO | } | 20260506 16:09:00.011 | INFO | Perf report for the last 5.13864 seconds: | 20260506 16:09:00.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:09:00.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:00.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:00.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:00.012 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 16:09:00.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:00.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:00.012 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:09:00.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:00.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:00.012 | INFO | Uploaded 36GiB to GPU in 1.97566s (18.2217GiB/s) | 20260506 16:09:00.012 | INFO | Downloaded 34GiB to GPU in 1.97534s (17.2122GiB/s) | 20260506 16:09:00.127 | INFO | Starting a round of Baz IO | 20260506 16:09:03.264 | INFO | Populating RTMetrics report | 20260506 16:09:03.265 | INFO | RTMetrics report populated | 20260506 16:09:03.265 | INFO | Opening tmp RTMetrics json file | 20260506 16:09:03.271 | INFO | Serializing RTMetrics to json | 20260506 16:09:03.273 | INFO | Serialization done, swapping tmp file now... | 20260506 16:09:03.273 | INFO | RTMetrics done | 20260506 16:09:03.860 | INFO | Finished a round of Baz IO: Wrote 2.088 GiB in 3.733 seconds | 20260506 16:09:03.860 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.58% / 0.01% / 63.41% | 20260506 16:09:03.860 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.77395% / 1.98682e-05% / 7.15256e-05% | 20260506 16:09:03.860 | INFO | Baz IO overall: | 20260506 16:09:03.860 | INFO | BytesWritten : 1401502237591 | 20260506 16:09:03.860 | INFO | NumEvents : 678217339108 | 20260506 16:09:05.150 | INFO | Analyzing chunk frames = [2674688,2675200) | 20260506 16:09:05.150 | INFO | MemInfo: { | 20260506 16:09:05.150 | INFO | "gpu" : " 17 GB", | 20260506 16:09:05.150 | INFO | "huge" : "", | 20260506 16:09:05.150 | INFO | "huge_shared" : " 27 GB", | 20260506 16:09:05.150 | INFO | "malloc" : " 14 GB", | 20260506 16:09:05.150 | INFO | "pinned" : " 29 GB" | 20260506 16:09:05.150 | INFO | } | 20260506 16:09:05.150 | INFO | Perf report for the last 5.1387 seconds: | 20260506 16:09:05.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:09:05.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:05.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:05.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:05.150 | INFO | ANALYSIS : 62.7%, 2.9, 95.3ms, 0 | 20260506 16:09:05.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:05.150 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 16:09:05.150 | INFO | PRE_HQ : 60.3%, 1.9, 58.5ms, 0 | 20260506 16:09:05.150 | INFO | BAZWRITER : 72.4%, 4.8, 178.2ms, 0 | 20260506 16:09:05.150 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 16:09:05.150 | INFO | Uploaded 36GiB to GPU in 2.07592s (17.3417GiB/s) | 20260506 16:09:05.150 | INFO | Downloaded 38GiB to GPU in 2.21106s (17.1863GiB/s) | 20260506 16:09:10.290 | INFO | Analyzing chunk frames = [2675200,2675712) | 20260506 16:09:10.290 | INFO | MemInfo: { | 20260506 16:09:10.290 | INFO | "gpu" : " 17 GB", | 20260506 16:09:10.290 | INFO | "huge" : "", | 20260506 16:09:10.290 | INFO | "huge_shared" : " 27 GB", | 20260506 16:09:10.290 | INFO | "malloc" : " 14 GB", | 20260506 16:09:10.290 | INFO | "pinned" : " 29 GB" | 20260506 16:09:10.290 | INFO | } | 20260506 16:09:10.290 | INFO | Perf report for the last 5.13999 seconds: | 20260506 16:09:10.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:09:10.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:10.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:10.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:10.290 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:09:10.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:10.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:10.290 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:09:10.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:10.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:10.290 | INFO | Uploaded 36GiB to GPU in 1.99349s (18.0588GiB/s) | 20260506 16:09:10.290 | INFO | Downloaded 34GiB to GPU in 1.93632s (17.5591GiB/s) | 20260506 16:09:15.429 | INFO | Analyzing chunk frames = [2675712,2676224) | 20260506 16:09:15.429 | INFO | MemInfo: { | 20260506 16:09:15.429 | INFO | "gpu" : " 17 GB", | 20260506 16:09:15.429 | INFO | "huge" : "", | 20260506 16:09:15.429 | INFO | "huge_shared" : " 27 GB", | 20260506 16:09:15.429 | INFO | "malloc" : " 14 GB", | 20260506 16:09:15.429 | INFO | "pinned" : " 29 GB" | 20260506 16:09:15.429 | INFO | } | 20260506 16:09:15.429 | INFO | Perf report for the last 5.13928 seconds: | 20260506 16:09:15.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:09:15.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:15.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:15.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:15.429 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 16:09:15.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:15.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:15.430 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:09:15.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:15.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:15.430 | INFO | Uploaded 36GiB to GPU in 1.98199s (18.1636GiB/s) | 20260506 16:09:15.430 | INFO | Downloaded 34GiB to GPU in 1.93092s (17.6081GiB/s) | 20260506 16:09:20.570 | INFO | Analyzing chunk frames = [2676224,2676736) | 20260506 16:09:20.570 | INFO | MemInfo: { | 20260506 16:09:20.570 | INFO | "gpu" : " 17 GB", | 20260506 16:09:20.570 | INFO | "huge" : "", | 20260506 16:09:20.570 | INFO | "huge_shared" : " 27 GB", | 20260506 16:09:20.570 | INFO | "malloc" : " 14 GB", | 20260506 16:09:20.570 | INFO | "pinned" : " 29 GB" | 20260506 16:09:20.570 | INFO | } | 20260506 16:09:20.570 | INFO | Perf report for the last 5.14044 seconds: | 20260506 16:09:20.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:09:20.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:20.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:20.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:20.570 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:09:20.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:20.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:20.570 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:09:20.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:20.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:20.570 | INFO | Uploaded 36GiB to GPU in 1.97292s (18.247GiB/s) | 20260506 16:09:20.570 | INFO | Downloaded 34GiB to GPU in 1.87797s (18.1047GiB/s) | 20260506 16:09:25.708 | INFO | Analyzing chunk frames = [2676736,2677248) | 20260506 16:09:25.708 | INFO | MemInfo: { | 20260506 16:09:25.708 | INFO | "gpu" : " 17 GB", | 20260506 16:09:25.708 | INFO | "huge" : "", | 20260506 16:09:25.708 | INFO | "huge_shared" : " 27 GB", | 20260506 16:09:25.708 | INFO | "malloc" : " 14 GB", | 20260506 16:09:25.709 | INFO | "pinned" : " 29 GB" | 20260506 16:09:25.709 | INFO | } | 20260506 16:09:25.709 | INFO | Perf report for the last 5.13878 seconds: | 20260506 16:09:25.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:09:25.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:25.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:25.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:25.709 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 16:09:25.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:25.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:25.709 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:09:25.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:25.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:25.709 | INFO | Uploaded 36GiB to GPU in 1.96905s (18.2829GiB/s) | 20260506 16:09:25.709 | INFO | Downloaded 34GiB to GPU in 1.92176s (17.6921GiB/s) | 20260506 16:09:30.848 | INFO | Analyzing chunk frames = [2677248,2677760) | 20260506 16:09:30.848 | INFO | MemInfo: { | 20260506 16:09:30.848 | INFO | "gpu" : " 17 GB", | 20260506 16:09:30.848 | INFO | "huge" : "", | 20260506 16:09:30.848 | INFO | "huge_shared" : " 27 GB", | 20260506 16:09:30.848 | INFO | "malloc" : " 14 GB", | 20260506 16:09:30.848 | INFO | "pinned" : " 29 GB" | 20260506 16:09:30.848 | INFO | } | 20260506 16:09:30.848 | INFO | Perf report for the last 5.13939 seconds: | 20260506 16:09:30.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:09:30.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:30.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:30.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:30.848 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 16:09:30.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:30.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:30.848 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260506 16:09:30.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:30.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:30.848 | INFO | Uploaded 36GiB to GPU in 2.01948s (17.8264GiB/s) | 20260506 16:09:30.848 | INFO | Downloaded 34GiB to GPU in 1.92572s (17.6557GiB/s) | 20260506 16:09:35.987 | INFO | Analyzing chunk frames = [2677760,2678272) | 20260506 16:09:35.987 | INFO | MemInfo: { | 20260506 16:09:35.987 | INFO | "gpu" : " 17 GB", | 20260506 16:09:35.987 | INFO | "huge" : "", | 20260506 16:09:35.987 | INFO | "huge_shared" : " 27 GB", | 20260506 16:09:35.987 | INFO | "malloc" : " 14 GB", | 20260506 16:09:35.987 | INFO | "pinned" : " 29 GB" | 20260506 16:09:35.987 | INFO | } | 20260506 16:09:35.987 | INFO | Perf report for the last 5.13912 seconds: | 20260506 16:09:35.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:09:35.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:35.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:35.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:35.987 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 16:09:35.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:35.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:35.987 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:09:35.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:35.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:35.988 | INFO | Uploaded 36GiB to GPU in 1.97424s (18.2348GiB/s) | 20260506 16:09:35.988 | INFO | Downloaded 34GiB to GPU in 1.95992s (17.3477GiB/s) | 20260506 16:09:41.125 | INFO | Analyzing chunk frames = [2678272,2678784) | 20260506 16:09:41.125 | INFO | MemInfo: { | 20260506 16:09:41.125 | INFO | "gpu" : " 17 GB", | 20260506 16:09:41.125 | INFO | "huge" : "", | 20260506 16:09:41.125 | INFO | "huge_shared" : " 27 GB", | 20260506 16:09:41.125 | INFO | "malloc" : " 14 GB", | 20260506 16:09:41.125 | INFO | "pinned" : " 29 GB" | 20260506 16:09:41.125 | INFO | } | 20260506 16:09:41.126 | INFO | Perf report for the last 5.13835 seconds: | 20260506 16:09:41.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:09:41.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:41.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:41.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:41.126 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:09:41.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:41.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:41.126 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:09:41.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:41.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:41.126 | INFO | Uploaded 36GiB to GPU in 2.03233s (17.7136GiB/s) | 20260506 16:09:41.126 | INFO | Downloaded 34GiB to GPU in 1.97973s (17.1741GiB/s) | 20260506 16:09:41.126 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:09:41.126 | INFO | Total mem usage: 17.4 GiB | 20260506 16:09:41.126 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 16:09:41.126 | INFO | Recent High Water Marks Sum: 16.6 GiB (Savings: -4.9%) | 20260506 16:09:41.126 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:09:41.126 | INFO | High Water Breakdown: | 20260506 16:09:41.126 | INFO | 62.2% : 10.3 GiB ( 94.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:09:41.126 | INFO | 37.7% : 6.25 GiB ( 93.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:09:41.126 | INFO | 0.12% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:09:41.126 | INFO | ------------------------------------------ | 20260506 16:09:41.126 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:09:41.126 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:09:41.126 | INFO | Total mem usage: 26.5 GiB | 20260506 16:09:41.126 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:09:41.126 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:09:41.126 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:09:41.126 | INFO | High Water Breakdown: | 20260506 16:09:41.126 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:09:41.126 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:09:41.126 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:09:41.126 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:09:41.127 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:09:41.127 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:09:41.127 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:09:41.127 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:09:41.127 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:09:41.127 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:09:41.127 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:09:41.127 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:09:41.127 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:09:41.127 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:09:41.127 | INFO | ------------------------------------------ | 20260506 16:09:41.127 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:09:41.127 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:09:41.127 | INFO | Total mem usage: 25.6 GiB | 20260506 16:09:41.127 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:09:41.127 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:09:41.127 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:09:41.127 | INFO | High Water Breakdown: | 20260506 16:09:41.127 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:09:41.127 | INFO | ------------------------------------------ | 20260506 16:09:41.127 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:09:41.127 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:09:41.127 | INFO | Total mem usage: 15.7 GiB | 20260506 16:09:41.127 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:09:41.127 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:09:41.127 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 16:09:41.127 | INFO | High Water Breakdown: | 20260506 16:09:41.127 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:09:41.127 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:09:41.127 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:09:41.127 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:09:41.127 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:09:41.127 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:09:41.127 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:09:41.127 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:09:41.127 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:09:41.127 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:09:41.127 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:09:41.127 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:09:41.127 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:09:41.127 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:09:41.127 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:09:41.127 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:09:41.127 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:09:41.127 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:09:41.127 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:09:41.127 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:09:41.127 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:09:41.127 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:09:41.127 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:09:41.127 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:09:41.127 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:09:41.127 | INFO | ------------------------------------------ | 20260506 16:09:41.127 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:09:41.127 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:09:41.127 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:09:41.127 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:09:41.127 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.49e+01, 1.33e+01 / 1.50e+00 | 20260506 16:09:41.127 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 9.85e-02, 9.78e-03 / 6.42e-03 | 20260506 16:09:41.127 | INFO | PerfReport: 11.61%, Baseline -- 3.64e+00 / 4.43e+00, 5.63e-01 / 3.49e-01 | 20260506 16:09:41.127 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.44e+00, 1.87e-01 / 1.44e-01 | 20260506 16:09:41.127 | INFO | PerfReport: 15.83%, DME -- 3.97e+01 / 4.36e+01, 3.60e+00 / 1.55e+00 | 20260506 16:09:41.127 | INFO | PerfReport: 56.91%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.42e+00 / 1.21e+00 | 20260506 16:09:41.127 | INFO | PerfReport: 7.36%, PulseAccumulating -- 2.31e+00 / 2.92e+00, 4.01e-01 / 2.25e-01 | 20260506 16:09:41.127 | INFO | PerfReport: 3.51%, Metrics -- 1.10e+00 / 1.49e+00, 2.85e-01 / 1.64e-01 | 20260506 16:09:41.127 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.83e-01 / 4.34e-01, 6.03e-02 / 2.32e-02 | 20260506 16:09:41.127 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:09:41.127 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:09:41.127 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:09:41.127 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 16:09:41.127 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260506 16:09:41.127 | INFO | PerfReport: 21.16%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 16:09:41.127 | INFO | PerfReport: 32.27%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 16:09:41.127 | INFO | PerfReport: 1.41%, LockUpload -- 1.37e+00 / 0.00e+00, 5.43e+00 / 0.00e+00 | 20260506 16:09:41.127 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260506 16:09:41.128 | INFO | PerfReport: 22.54%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 16:09:41.128 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260506 16:09:41.246 | INFO | Starting a round of Baz IO | 20260506 16:09:44.389 | INFO | Populating RTMetrics report | 20260506 16:09:44.389 | INFO | RTMetrics report populated | 20260506 16:09:44.389 | INFO | Opening tmp RTMetrics json file | 20260506 16:09:44.464 | INFO | Serializing RTMetrics to json | 20260506 16:09:44.468 | INFO | Serialization done, swapping tmp file now... | 20260506 16:09:44.468 | INFO | RTMetrics done | 20260506 16:09:44.748 | INFO | Finished a round of Baz IO: Wrote 2.088 GiB in 3.501 seconds | 20260506 16:09:44.748 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.57% / 0.01% / 63.42% | 20260506 16:09:44.748 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.77159% / 1.58946e-05% / 8.34465e-05% | 20260506 16:09:44.748 | INFO | Baz IO overall: | 20260506 16:09:44.748 | INFO | BytesWritten : 1403743899119 | 20260506 16:09:44.748 | INFO | NumEvents : 679300937483 | 20260506 16:09:46.265 | INFO | Analyzing chunk frames = [2678784,2679296) | 20260506 16:09:46.265 | INFO | MemInfo: { | 20260506 16:09:46.265 | INFO | "gpu" : " 17 GB", | 20260506 16:09:46.265 | INFO | "huge" : "", | 20260506 16:09:46.265 | INFO | "huge_shared" : " 27 GB", | 20260506 16:09:46.265 | INFO | "malloc" : " 14 GB", | 20260506 16:09:46.265 | INFO | "pinned" : " 29 GB" | 20260506 16:09:46.265 | INFO | } | 20260506 16:09:46.265 | INFO | Perf report for the last 5.13916 seconds: | 20260506 16:09:46.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:09:46.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:46.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:46.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:46.265 | INFO | ANALYSIS : 62.9%, 2.9, 95.4ms, 0 | 20260506 16:09:46.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:46.265 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 16:09:46.265 | INFO | PRE_HQ : 60.7%, 1.9, 58.8ms, 0 | 20260506 16:09:46.265 | INFO | BAZWRITER : 67.7%, 4.8, 167.7ms, 0 | 20260506 16:09:46.265 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 16:09:46.265 | INFO | Uploaded 36GiB to GPU in 2.0589s (17.4851GiB/s) | 20260506 16:09:46.265 | INFO | Downloaded 38GiB to GPU in 2.21839s (17.1295GiB/s) | 20260506 16:09:51.404 | INFO | Analyzing chunk frames = [2679296,2679808) | 20260506 16:09:51.404 | INFO | MemInfo: { | 20260506 16:09:51.404 | INFO | "gpu" : " 17 GB", | 20260506 16:09:51.404 | INFO | "huge" : "", | 20260506 16:09:51.404 | INFO | "huge_shared" : " 27 GB", | 20260506 16:09:51.404 | INFO | "malloc" : " 14 GB", | 20260506 16:09:51.404 | INFO | "pinned" : " 29 GB" | 20260506 16:09:51.404 | INFO | } | 20260506 16:09:51.405 | INFO | Perf report for the last 5.13981 seconds: | 20260506 16:09:51.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:09:51.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:51.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:51.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:51.405 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 16:09:51.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:51.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:51.405 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:09:51.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:51.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:51.405 | INFO | Uploaded 36GiB to GPU in 1.99586s (18.0373GiB/s) | 20260506 16:09:51.405 | INFO | Downloaded 34GiB to GPU in 1.92763s (17.6383GiB/s) | 20260506 16:09:56.544 | INFO | Analyzing chunk frames = [2679808,2680320) | 20260506 16:09:56.544 | INFO | MemInfo: { | 20260506 16:09:56.544 | INFO | "gpu" : " 17 GB", | 20260506 16:09:56.544 | INFO | "huge" : "", | 20260506 16:09:56.544 | INFO | "huge_shared" : " 27 GB", | 20260506 16:09:56.544 | INFO | "malloc" : " 14 GB", | 20260506 16:09:56.544 | INFO | "pinned" : " 29 GB" | 20260506 16:09:56.544 | INFO | } | 20260506 16:09:56.544 | INFO | Perf report for the last 5.13963 seconds: | 20260506 16:09:56.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:09:56.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:56.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:56.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:56.544 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 16:09:56.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:56.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:56.544 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:09:56.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:56.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:09:56.545 | INFO | Uploaded 36GiB to GPU in 1.99855s (18.013GiB/s) | 20260506 16:09:56.545 | INFO | Downloaded 34GiB to GPU in 1.92999s (17.6167GiB/s) | 20260506 16:10:01.683 | INFO | Analyzing chunk frames = [2680320,2680832) | 20260506 16:10:01.683 | INFO | MemInfo: { | 20260506 16:10:01.683 | INFO | "gpu" : " 17 GB", | 20260506 16:10:01.683 | INFO | "huge" : "", | 20260506 16:10:01.683 | INFO | "huge_shared" : " 27 GB", | 20260506 16:10:01.683 | INFO | "malloc" : " 14 GB", | 20260506 16:10:01.683 | INFO | "pinned" : " 29 GB" | 20260506 16:10:01.683 | INFO | } | 20260506 16:10:01.683 | INFO | Perf report for the last 5.13891 seconds: | 20260506 16:10:01.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:10:01.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:01.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:01.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:01.683 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 16:10:01.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:01.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:01.683 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:10:01.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:01.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:01.683 | INFO | Uploaded 36GiB to GPU in 2.02112s (17.8119GiB/s) | 20260506 16:10:01.683 | INFO | Downloaded 34GiB to GPU in 1.91055s (17.796GiB/s) | 20260506 16:10:06.822 | INFO | Analyzing chunk frames = [2680832,2681344) | 20260506 16:10:06.822 | INFO | MemInfo: { | 20260506 16:10:06.822 | INFO | "gpu" : " 17 GB", | 20260506 16:10:06.822 | INFO | "huge" : "", | 20260506 16:10:06.822 | INFO | "huge_shared" : " 27 GB", | 20260506 16:10:06.822 | INFO | "malloc" : " 14 GB", | 20260506 16:10:06.822 | INFO | "pinned" : " 29 GB" | 20260506 16:10:06.822 | INFO | } | 20260506 16:10:06.823 | INFO | Perf report for the last 5.13942 seconds: | 20260506 16:10:06.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:10:06.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:06.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:06.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:06.823 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 16:10:06.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:06.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:06.823 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:10:06.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:06.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:06.823 | INFO | Uploaded 36GiB to GPU in 2.01311s (17.8827GiB/s) | 20260506 16:10:06.823 | INFO | Downloaded 34GiB to GPU in 1.9289s (17.6266GiB/s) | 20260506 16:10:11.962 | INFO | Analyzing chunk frames = [2681344,2681856) | 20260506 16:10:11.962 | INFO | MemInfo: { | 20260506 16:10:11.962 | INFO | "gpu" : " 17 GB", | 20260506 16:10:11.962 | INFO | "huge" : "", | 20260506 16:10:11.962 | INFO | "huge_shared" : " 27 GB", | 20260506 16:10:11.962 | INFO | "malloc" : " 14 GB", | 20260506 16:10:11.962 | INFO | "pinned" : " 29 GB" | 20260506 16:10:11.962 | INFO | } | 20260506 16:10:11.962 | INFO | Perf report for the last 5.1396 seconds: | 20260506 16:10:11.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:10:11.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:11.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:11.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:11.962 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:10:11.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:11.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:11.962 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 16:10:11.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:11.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:11.962 | INFO | Uploaded 36GiB to GPU in 2.03057s (17.729GiB/s) | 20260506 16:10:11.963 | INFO | Downloaded 34GiB to GPU in 1.92463s (17.6657GiB/s) | 20260506 16:10:17.103 | INFO | Analyzing chunk frames = [2681856,2682368) | 20260506 16:10:17.103 | INFO | MemInfo: { | 20260506 16:10:17.103 | INFO | "gpu" : " 17 GB", | 20260506 16:10:17.103 | INFO | "huge" : "", | 20260506 16:10:17.103 | INFO | "huge_shared" : " 27 GB", | 20260506 16:10:17.103 | INFO | "malloc" : " 14 GB", | 20260506 16:10:17.103 | INFO | "pinned" : " 29 GB" | 20260506 16:10:17.103 | INFO | } | 20260506 16:10:17.103 | INFO | Perf report for the last 5.14066 seconds: | 20260506 16:10:17.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:10:17.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:17.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:17.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:17.103 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:10:17.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:17.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:17.103 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 16:10:17.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:17.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:17.103 | INFO | Uploaded 36GiB to GPU in 1.98621s (18.125GiB/s) | 20260506 16:10:17.103 | INFO | Downloaded 34GiB to GPU in 1.98088s (17.1641GiB/s) | 20260506 16:10:22.241 | INFO | Analyzing chunk frames = [2682368,2682880) | 20260506 16:10:22.241 | INFO | MemInfo: { | 20260506 16:10:22.241 | INFO | "gpu" : " 17 GB", | 20260506 16:10:22.241 | INFO | "huge" : "", | 20260506 16:10:22.241 | INFO | "huge_shared" : " 27 GB", | 20260506 16:10:22.241 | INFO | "malloc" : " 14 GB", | 20260506 16:10:22.241 | INFO | "pinned" : " 29 GB" | 20260506 16:10:22.241 | INFO | } | 20260506 16:10:22.241 | INFO | Perf report for the last 5.13842 seconds: | 20260506 16:10:22.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:10:22.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:22.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:22.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:22.241 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:10:22.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:22.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:22.241 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:10:22.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:22.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:22.242 | INFO | Uploaded 36GiB to GPU in 2.02376s (17.7887GiB/s) | 20260506 16:10:22.242 | INFO | Downloaded 34GiB to GPU in 1.98443s (17.1334GiB/s) | 20260506 16:10:22.360 | INFO | Starting a round of Baz IO | 20260506 16:10:25.502 | INFO | Populating RTMetrics report | 20260506 16:10:25.502 | INFO | RTMetrics report populated | 20260506 16:10:25.502 | INFO | Opening tmp RTMetrics json file | 20260506 16:10:25.532 | INFO | Serializing RTMetrics to json | 20260506 16:10:25.534 | INFO | Serialization done, swapping tmp file now... | 20260506 16:10:25.536 | INFO | RTMetrics done | 20260506 16:10:26.023 | INFO | Finished a round of Baz IO: Wrote 2.088 GiB in 3.662 seconds | 20260506 16:10:26.023 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.56% / 0.01% / 63.43% | 20260506 16:10:26.023 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.76804% / 1.98682e-05% / 7.15256e-05% | 20260506 16:10:26.023 | INFO | Baz IO overall: | 20260506 16:10:26.023 | INFO | BytesWritten : 1405986091989 | 20260506 16:10:26.023 | INFO | NumEvents : 680384832948 | 20260506 16:10:27.380 | INFO | Analyzing chunk frames = [2682880,2683392) | 20260506 16:10:27.380 | INFO | MemInfo: { | 20260506 16:10:27.380 | INFO | "gpu" : " 17 GB", | 20260506 16:10:27.380 | INFO | "huge" : "", | 20260506 16:10:27.380 | INFO | "huge_shared" : " 27 GB", | 20260506 16:10:27.380 | INFO | "malloc" : " 14 GB", | 20260506 16:10:27.380 | INFO | "pinned" : " 29 GB" | 20260506 16:10:27.380 | INFO | } | 20260506 16:10:27.380 | INFO | Perf report for the last 5.1392 seconds: | 20260506 16:10:27.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:10:27.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:27.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:27.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:27.381 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 16:10:27.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:27.381 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 16:10:27.381 | INFO | PRE_HQ : 60.3%, 1.9, 58.5ms, 0 | 20260506 16:10:27.381 | INFO | BAZWRITER : 70.9%, 4.8, 176.2ms, 0 | 20260506 16:10:27.381 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 16:10:27.381 | INFO | Uploaded 36GiB to GPU in 2.04691s (17.5875GiB/s) | 20260506 16:10:27.381 | INFO | Downloaded 38GiB to GPU in 2.22006s (17.1167GiB/s) | 20260506 16:10:32.519 | INFO | Analyzing chunk frames = [2683392,2683904) | 20260506 16:10:32.519 | INFO | MemInfo: { | 20260506 16:10:32.519 | INFO | "gpu" : " 17 GB", | 20260506 16:10:32.519 | INFO | "huge" : "", | 20260506 16:10:32.519 | INFO | "huge_shared" : " 27 GB", | 20260506 16:10:32.519 | INFO | "malloc" : " 14 GB", | 20260506 16:10:32.519 | INFO | "pinned" : " 29 GB" | 20260506 16:10:32.519 | INFO | } | 20260506 16:10:32.519 | INFO | Perf report for the last 5.13841 seconds: | 20260506 16:10:32.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:10:32.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:32.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:32.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:32.519 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 16:10:32.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:32.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:32.519 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 16:10:32.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:32.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:32.519 | INFO | Uploaded 36GiB to GPU in 1.98673s (18.1202GiB/s) | 20260506 16:10:32.519 | INFO | Downloaded 34GiB to GPU in 1.92201s (17.6898GiB/s) | 20260506 16:10:37.660 | INFO | Analyzing chunk frames = [2683904,2684416) | 20260506 16:10:37.660 | INFO | MemInfo: { | 20260506 16:10:37.660 | INFO | "gpu" : " 17 GB", | 20260506 16:10:37.660 | INFO | "huge" : "", | 20260506 16:10:37.660 | INFO | "huge_shared" : " 27 GB", | 20260506 16:10:37.660 | INFO | "malloc" : " 14 GB", | 20260506 16:10:37.660 | INFO | "pinned" : " 29 GB" | 20260506 16:10:37.660 | INFO | } | 20260506 16:10:37.661 | INFO | Perf report for the last 5.14163 seconds: | 20260506 16:10:37.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:10:37.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:37.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:37.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:37.661 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 16:10:37.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:37.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:37.661 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:10:37.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:37.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:37.661 | INFO | Uploaded 36GiB to GPU in 1.97398s (18.2372GiB/s) | 20260506 16:10:37.661 | INFO | Downloaded 34GiB to GPU in 1.92646s (17.649GiB/s) | 20260506 16:10:42.798 | INFO | Analyzing chunk frames = [2684416,2684928) | 20260506 16:10:42.798 | INFO | MemInfo: { | 20260506 16:10:42.798 | INFO | "gpu" : " 17 GB", | 20260506 16:10:42.798 | INFO | "huge" : "", | 20260506 16:10:42.798 | INFO | "huge_shared" : " 27 GB", | 20260506 16:10:42.798 | INFO | "malloc" : " 14 GB", | 20260506 16:10:42.798 | INFO | "pinned" : " 29 GB" | 20260506 16:10:42.798 | INFO | } | 20260506 16:10:42.798 | INFO | Perf report for the last 5.13735 seconds: | 20260506 16:10:42.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:10:42.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:42.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:42.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:42.798 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 16:10:42.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:42.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:42.798 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:10:42.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:42.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:42.798 | INFO | Uploaded 36GiB to GPU in 2.01904s (17.8302GiB/s) | 20260506 16:10:42.798 | INFO | Downloaded 34GiB to GPU in 1.90586s (17.8397GiB/s) | 20260506 16:10:47.938 | INFO | Analyzing chunk frames = [2684928,2685440) | 20260506 16:10:47.938 | INFO | MemInfo: { | 20260506 16:10:47.938 | INFO | "gpu" : " 17 GB", | 20260506 16:10:47.938 | INFO | "huge" : "", | 20260506 16:10:47.938 | INFO | "huge_shared" : " 27 GB", | 20260506 16:10:47.938 | INFO | "malloc" : " 14 GB", | 20260506 16:10:47.938 | INFO | "pinned" : " 29 GB" | 20260506 16:10:47.938 | INFO | } | 20260506 16:10:47.938 | INFO | Perf report for the last 5.14018 seconds: | 20260506 16:10:47.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:10:47.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:47.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:47.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:47.938 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 16:10:47.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:47.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:47.938 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:10:47.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:47.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:47.939 | INFO | Uploaded 36GiB to GPU in 2.01456s (17.8699GiB/s) | 20260506 16:10:47.939 | INFO | Downloaded 34GiB to GPU in 1.93749s (17.5485GiB/s) | 20260506 16:10:53.076 | INFO | Analyzing chunk frames = [2685440,2685952) | 20260506 16:10:53.076 | INFO | MemInfo: { | 20260506 16:10:53.076 | INFO | "gpu" : " 17 GB", | 20260506 16:10:53.076 | INFO | "huge" : "", | 20260506 16:10:53.077 | INFO | "huge_shared" : " 27 GB", | 20260506 16:10:53.077 | INFO | "malloc" : " 14 GB", | 20260506 16:10:53.077 | INFO | "pinned" : " 29 GB" | 20260506 16:10:53.077 | INFO | } | 20260506 16:10:53.077 | INFO | Perf report for the last 5.13851 seconds: | 20260506 16:10:53.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:10:53.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:53.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:53.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:53.077 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 16:10:53.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:53.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:53.077 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 16:10:53.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:53.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:53.077 | INFO | Uploaded 36GiB to GPU in 2.04204s (17.6294GiB/s) | 20260506 16:10:53.077 | INFO | Downloaded 34GiB to GPU in 1.91922s (17.7155GiB/s) | 20260506 16:10:58.217 | INFO | Analyzing chunk frames = [2685952,2686464) | 20260506 16:10:58.217 | INFO | MemInfo: { | 20260506 16:10:58.217 | INFO | "gpu" : " 17 GB", | 20260506 16:10:58.217 | INFO | "huge" : "", | 20260506 16:10:58.217 | INFO | "huge_shared" : " 27 GB", | 20260506 16:10:58.217 | INFO | "malloc" : " 14 GB", | 20260506 16:10:58.217 | INFO | "pinned" : " 29 GB" | 20260506 16:10:58.217 | INFO | } | 20260506 16:10:58.217 | INFO | Perf report for the last 5.1407 seconds: | 20260506 16:10:58.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:10:58.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:58.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:58.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:58.217 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:10:58.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:58.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:58.217 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 16:10:58.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:58.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:10:58.218 | INFO | Uploaded 36GiB to GPU in 2.01139s (17.898GiB/s) | 20260506 16:10:58.218 | INFO | Downloaded 34GiB to GPU in 1.9791s (17.1795GiB/s) | 20260506 16:11:03.356 | INFO | Analyzing chunk frames = [2686464,2686976) | 20260506 16:11:03.356 | INFO | MemInfo: { | 20260506 16:11:03.356 | INFO | "gpu" : " 17 GB", | 20260506 16:11:03.356 | INFO | "huge" : "", | 20260506 16:11:03.356 | INFO | "huge_shared" : " 27 GB", | 20260506 16:11:03.356 | INFO | "malloc" : " 14 GB", | 20260506 16:11:03.356 | INFO | "pinned" : " 29 GB" | 20260506 16:11:03.356 | INFO | } | 20260506 16:11:03.356 | INFO | Perf report for the last 5.13847 seconds: | 20260506 16:11:03.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:11:03.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:03.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:03.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:03.356 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 16:11:03.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:03.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:03.356 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 16:11:03.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:03.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:03.356 | INFO | Uploaded 36GiB to GPU in 2.02533s (17.7749GiB/s) | 20260506 16:11:03.356 | INFO | Downloaded 34GiB to GPU in 1.9823s (17.1518GiB/s) | 20260506 16:11:03.356 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:11:03.356 | INFO | Total mem usage: 17.4 GiB | 20260506 16:11:03.356 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 1.25%) | 20260506 16:11:03.356 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: -5.61%) | 20260506 16:11:03.356 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:11:03.356 | INFO | High Water Breakdown: | 20260506 16:11:03.356 | INFO | 62.1% : 10.2 GiB ( 93.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:11:03.356 | INFO | 37.8% : 6.22 GiB ( 93% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:11:03.356 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:11:03.356 | INFO | ------------------------------------------ | 20260506 16:11:03.356 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:11:03.356 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:11:03.356 | INFO | Total mem usage: 26.5 GiB | 20260506 16:11:03.357 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:11:03.357 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:11:03.357 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:11:03.357 | INFO | High Water Breakdown: | 20260506 16:11:03.357 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:11:03.357 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:11:03.357 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:11:03.357 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:11:03.357 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:11:03.357 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:11:03.357 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:11:03.357 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:11:03.357 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:11:03.357 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:11:03.357 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:11:03.357 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:11:03.357 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:11:03.357 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:11:03.357 | INFO | ------------------------------------------ | 20260506 16:11:03.357 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:11:03.357 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:11:03.357 | INFO | Total mem usage: 25.6 GiB | 20260506 16:11:03.357 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:11:03.357 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:11:03.357 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:11:03.357 | INFO | High Water Breakdown: | 20260506 16:11:03.357 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:11:03.357 | INFO | ------------------------------------------ | 20260506 16:11:03.357 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:11:03.357 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:11:03.357 | INFO | Total mem usage: 15.7 GiB | 20260506 16:11:03.357 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:11:03.357 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:11:03.357 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 16:11:03.357 | INFO | High Water Breakdown: | 20260506 16:11:03.357 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:11:03.357 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:11:03.357 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:11:03.357 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:11:03.357 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:11:03.357 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:11:03.357 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:11:03.357 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:11:03.357 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:11:03.357 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:11:03.357 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:11:03.357 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:11:03.357 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:11:03.357 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:11:03.357 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:11:03.357 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:11:03.357 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:11:03.357 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:11:03.357 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:11:03.357 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:11:03.357 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:11:03.357 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:11:03.357 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:11:03.357 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:11:03.357 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:11:03.357 | INFO | ------------------------------------------ | 20260506 16:11:03.357 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:11:03.357 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:11:03.357 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:11:03.357 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:11:03.357 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.53e+01, 1.35e+01 / 1.92e+00 | 20260506 16:11:03.357 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 9.89e-02, 1.07e-02 / 6.95e-03 | 20260506 16:11:03.357 | INFO | PerfReport: 11.60%, Baseline -- 3.65e+00 / 4.37e+00, 5.68e-01 / 4.21e-01 | 20260506 16:11:03.357 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.87e-01 / 1.56e-01 | 20260506 16:11:03.357 | INFO | PerfReport: 15.82%, DME -- 3.98e+01 / 4.37e+01, 3.63e+00 / 2.01e+00 | 20260506 16:11:03.357 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.79e+01 / 2.09e+01, 1.55e+00 / 2.02e+00 | 20260506 16:11:03.357 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.29e+00 / 2.84e+00, 3.78e-01 / 2.65e-01 | 20260506 16:11:03.357 | INFO | PerfReport: 3.51%, Metrics -- 1.10e+00 / 1.45e+00, 2.87e-01 / 1.98e-01 | 20260506 16:11:03.357 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.83e-01 / 4.35e-01, 6.02e-02 / 3.19e-02 | 20260506 16:11:03.357 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:11:03.357 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:11:03.357 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:11:03.357 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 16:11:03.357 | INFO | PerfReport: 21.56%, Upload -- 2.10e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260506 16:11:03.357 | INFO | PerfReport: 21.12%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 16:11:03.357 | INFO | PerfReport: 32.28%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 16:11:03.357 | INFO | PerfReport: 1.45%, LockUpload -- 1.42e+00 / 0.00e+00, 5.38e+00 / 0.00e+00 | 20260506 16:11:03.357 | INFO | PerfReport: 1.05%, LockDownload -- 1.02e+00 / 0.00e+00, 2.68e+00 / 0.00e+00 | 20260506 16:11:03.357 | INFO | PerfReport: 22.52%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 16:11:03.357 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260506 16:11:03.470 | INFO | Starting a round of Baz IO | 20260506 16:11:06.616 | INFO | Populating RTMetrics report | 20260506 16:11:06.617 | INFO | RTMetrics report populated | 20260506 16:11:06.617 | INFO | Opening tmp RTMetrics json file | 20260506 16:11:06.661 | INFO | Serializing RTMetrics to json | 20260506 16:11:06.687 | INFO | Serialization done, swapping tmp file now... | 20260506 16:11:06.691 | INFO | RTMetrics done | 20260506 16:11:07.843 | INFO | Finished a round of Baz IO: Wrote 2.09 GiB in 4.372 seconds | 20260506 16:11:07.843 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.55% / 0.01% / 63.44% | 20260506 16:11:07.843 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.77295% / 7.94729e-06% / 5.96046e-05% | 20260506 16:11:07.843 | INFO | Baz IO overall: | 20260506 16:11:07.843 | INFO | BytesWritten : 1408230490048 | 20260506 16:11:07.843 | INFO | NumEvents : 681470556307 | 20260506 16:11:08.495 | INFO | Analyzing chunk frames = [2686976,2687488) | 20260506 16:11:08.495 | INFO | MemInfo: { | 20260506 16:11:08.495 | INFO | "gpu" : " 17 GB", | 20260506 16:11:08.495 | INFO | "huge" : "", | 20260506 16:11:08.495 | INFO | "huge_shared" : " 27 GB", | 20260506 16:11:08.495 | INFO | "malloc" : " 14 GB", | 20260506 16:11:08.495 | INFO | "pinned" : " 29 GB" | 20260506 16:11:08.495 | INFO | } | 20260506 16:11:08.495 | INFO | Perf report for the last 5.13895 seconds: | 20260506 16:11:08.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:11:08.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:08.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:08.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:08.495 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 16:11:08.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:08.495 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260506 16:11:08.495 | INFO | PRE_HQ : 60.7%, 1.9, 58.7ms, 0 | 20260506 16:11:08.495 | INFO | BAZWRITER : 84.5%, 4.9, 211.0ms, 0 | 20260506 16:11:08.495 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 16:11:08.495 | INFO | Uploaded 36GiB to GPU in 2.05454s (17.5221GiB/s) | 20260506 16:11:08.495 | INFO | Downloaded 38GiB to GPU in 2.22041s (17.114GiB/s) | 20260506 16:11:13.634 | INFO | Analyzing chunk frames = [2687488,2688000) | 20260506 16:11:13.634 | INFO | MemInfo: { | 20260506 16:11:13.634 | INFO | "gpu" : " 17 GB", | 20260506 16:11:13.634 | INFO | "huge" : "", | 20260506 16:11:13.634 | INFO | "huge_shared" : " 27 GB", | 20260506 16:11:13.634 | INFO | "malloc" : " 14 GB", | 20260506 16:11:13.634 | INFO | "pinned" : " 29 GB" | 20260506 16:11:13.634 | INFO | } | 20260506 16:11:13.634 | INFO | Perf report for the last 5.13964 seconds: | 20260506 16:11:13.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:11:13.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:13.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:13.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:13.634 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:11:13.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:13.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:13.635 | INFO | PRE_HQ : 27.8%, 1.0, 14.6ms, 0 | 20260506 16:11:13.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:13.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:13.635 | INFO | Uploaded 36GiB to GPU in 1.99959s (18.0037GiB/s) | 20260506 16:11:13.635 | INFO | Downloaded 34GiB to GPU in 1.93378s (17.5822GiB/s) | 20260506 16:11:18.773 | INFO | Analyzing chunk frames = [2688000,2688512) | 20260506 16:11:18.773 | INFO | MemInfo: { | 20260506 16:11:18.773 | INFO | "gpu" : " 17 GB", | 20260506 16:11:18.773 | INFO | "huge" : "", | 20260506 16:11:18.773 | INFO | "huge_shared" : " 27 GB", | 20260506 16:11:18.773 | INFO | "malloc" : " 14 GB", | 20260506 16:11:18.773 | INFO | "pinned" : " 29 GB" | 20260506 16:11:18.773 | INFO | } | 20260506 16:11:18.773 | INFO | Perf report for the last 5.13884 seconds: | 20260506 16:11:18.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:11:18.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:18.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:18.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:18.773 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 16:11:18.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:18.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:18.773 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:11:18.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:18.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:18.773 | INFO | Uploaded 36GiB to GPU in 1.99468s (18.048GiB/s) | 20260506 16:11:18.774 | INFO | Downloaded 34GiB to GPU in 1.93187s (17.5995GiB/s) | 20260506 16:11:23.913 | INFO | Analyzing chunk frames = [2688512,2689024) | 20260506 16:11:23.913 | INFO | MemInfo: { | 20260506 16:11:23.913 | INFO | "gpu" : " 17 GB", | 20260506 16:11:23.913 | INFO | "huge" : "", | 20260506 16:11:23.913 | INFO | "huge_shared" : " 27 GB", | 20260506 16:11:23.913 | INFO | "malloc" : " 14 GB", | 20260506 16:11:23.913 | INFO | "pinned" : " 29 GB" | 20260506 16:11:23.913 | INFO | } | 20260506 16:11:23.913 | INFO | Perf report for the last 5.13988 seconds: | 20260506 16:11:23.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:11:23.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:23.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:23.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:23.913 | INFO | ANALYSIS : 60.6%, 3.0, 94.0ms, 0 | 20260506 16:11:23.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:23.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:23.913 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:11:23.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:23.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:23.913 | INFO | Uploaded 36GiB to GPU in 2.01445s (17.8708GiB/s) | 20260506 16:11:23.913 | INFO | Downloaded 34GiB to GPU in 1.90968s (17.8041GiB/s) | 20260506 16:11:29.054 | INFO | Analyzing chunk frames = [2689024,2689536) | 20260506 16:11:29.054 | INFO | MemInfo: { | 20260506 16:11:29.054 | INFO | "gpu" : " 17 GB", | 20260506 16:11:29.054 | INFO | "huge" : "", | 20260506 16:11:29.054 | INFO | "huge_shared" : " 27 GB", | 20260506 16:11:29.054 | INFO | "malloc" : " 14 GB", | 20260506 16:11:29.054 | INFO | "pinned" : " 29 GB" | 20260506 16:11:29.054 | INFO | } | 20260506 16:11:29.054 | INFO | Perf report for the last 5.14101 seconds: | 20260506 16:11:29.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:11:29.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:29.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:29.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:29.054 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 16:11:29.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:29.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:29.054 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:11:29.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:29.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:29.054 | INFO | Uploaded 36GiB to GPU in 2.00228s (17.9795GiB/s) | 20260506 16:11:29.055 | INFO | Downloaded 34GiB to GPU in 1.9282s (17.633GiB/s) | 20260506 16:11:34.192 | INFO | Analyzing chunk frames = [2689536,2690048) | 20260506 16:11:34.192 | INFO | MemInfo: { | 20260506 16:11:34.192 | INFO | "gpu" : " 17 GB", | 20260506 16:11:34.192 | INFO | "huge" : "", | 20260506 16:11:34.192 | INFO | "huge_shared" : " 27 GB", | 20260506 16:11:34.192 | INFO | "malloc" : " 14 GB", | 20260506 16:11:34.192 | INFO | "pinned" : " 29 GB" | 20260506 16:11:34.192 | INFO | } | 20260506 16:11:34.192 | INFO | Perf report for the last 5.13783 seconds: | 20260506 16:11:34.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:11:34.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:34.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:34.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:34.192 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:11:34.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:34.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:34.192 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 16:11:34.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:34.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:34.192 | INFO | Uploaded 36GiB to GPU in 2.07185s (17.3758GiB/s) | 20260506 16:11:34.193 | INFO | Downloaded 34GiB to GPU in 1.97366s (17.2268GiB/s) | 20260506 16:11:39.331 | INFO | Analyzing chunk frames = [2690048,2690560) | 20260506 16:11:39.331 | INFO | MemInfo: { | 20260506 16:11:39.331 | INFO | "gpu" : " 17 GB", | 20260506 16:11:39.331 | INFO | "huge" : "", | 20260506 16:11:39.331 | INFO | "huge_shared" : " 27 GB", | 20260506 16:11:39.331 | INFO | "malloc" : " 14 GB", | 20260506 16:11:39.331 | INFO | "pinned" : " 29 GB" | 20260506 16:11:39.331 | INFO | } | 20260506 16:11:39.331 | INFO | Perf report for the last 5.13924 seconds: | 20260506 16:11:39.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:11:39.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:39.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:39.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:39.331 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 16:11:39.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:39.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:39.331 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:11:39.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:39.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:39.332 | INFO | Uploaded 36GiB to GPU in 2.07874s (17.3182GiB/s) | 20260506 16:11:39.332 | INFO | Downloaded 34GiB to GPU in 2.01746s (16.8529GiB/s) | 20260506 16:11:44.470 | INFO | Analyzing chunk frames = [2690560,2691072) | 20260506 16:11:44.470 | INFO | MemInfo: { | 20260506 16:11:44.470 | INFO | "gpu" : " 17 GB", | 20260506 16:11:44.470 | INFO | "huge" : "", | 20260506 16:11:44.470 | INFO | "huge_shared" : " 27 GB", | 20260506 16:11:44.470 | INFO | "malloc" : " 14 GB", | 20260506 16:11:44.470 | INFO | "pinned" : " 29 GB" | 20260506 16:11:44.470 | INFO | } | 20260506 16:11:44.470 | INFO | Perf report for the last 5.13883 seconds: | 20260506 16:11:44.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:11:44.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:44.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:44.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:44.470 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:11:44.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:44.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:44.470 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:11:44.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:44.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:44.470 | INFO | Uploaded 36GiB to GPU in 1.98914s (18.0983GiB/s) | 20260506 16:11:44.470 | INFO | Downloaded 34GiB to GPU in 1.97776s (17.1912GiB/s) | 20260506 16:11:44.584 | INFO | Starting a round of Baz IO | 20260506 16:11:47.723 | INFO | Populating RTMetrics report | 20260506 16:11:47.724 | INFO | RTMetrics report populated | 20260506 16:11:47.724 | INFO | Opening tmp RTMetrics json file | 20260506 16:11:47.813 | INFO | Serializing RTMetrics to json | 20260506 16:11:47.965 | INFO | Serialization done, swapping tmp file now... | 20260506 16:11:47.967 | INFO | RTMetrics done | 20260506 16:11:48.515 | INFO | Finished a round of Baz IO: Wrote 2.088 GiB in 3.931 seconds | 20260506 16:11:48.515 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.54% / 0.01% / 63.45% | 20260506 16:11:48.515 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.76348% / 3.97364e-06% / 5.5631e-05% | 20260506 16:11:48.515 | INFO | Baz IO overall: | 20260506 16:11:48.515 | INFO | BytesWritten : 1410472924485 | 20260506 16:11:48.515 | INFO | NumEvents : 682554901086 | 20260506 16:11:49.610 | INFO | Analyzing chunk frames = [2691072,2691584) | 20260506 16:11:49.610 | INFO | MemInfo: { | 20260506 16:11:49.610 | INFO | "gpu" : " 17 GB", | 20260506 16:11:49.610 | INFO | "huge" : "", | 20260506 16:11:49.610 | INFO | "huge_shared" : " 27 GB", | 20260506 16:11:49.610 | INFO | "malloc" : " 14 GB", | 20260506 16:11:49.610 | INFO | "pinned" : " 29 GB" | 20260506 16:11:49.610 | INFO | } | 20260506 16:11:49.610 | INFO | Perf report for the last 5.13983 seconds: | 20260506 16:11:49.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:11:49.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:49.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:49.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:49.610 | INFO | ANALYSIS : 62.7%, 2.9, 95.1ms, 0 | 20260506 16:11:49.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:49.610 | INFO | RT_METRICS : 48.4%, 1.0, 25.4ms, 0 | 20260506 16:11:49.610 | INFO | PRE_HQ : 60.3%, 1.9, 58.7ms, 0 | 20260506 16:11:49.610 | INFO | BAZWRITER : 76.1%, 4.8, 188.4ms, 0 | 20260506 16:11:49.610 | INFO | BAZ_SAMPLING : 4.0%, 1.0, 2.1ms, 0 | 20260506 16:11:49.610 | INFO | Uploaded 36GiB to GPU in 2.04169s (17.6325GiB/s) | 20260506 16:11:49.610 | INFO | Downloaded 38GiB to GPU in 2.20709s (17.2172GiB/s) | 20260506 16:11:54.751 | INFO | Analyzing chunk frames = [2691584,2692096) | 20260506 16:11:54.751 | INFO | MemInfo: { | 20260506 16:11:54.751 | INFO | "gpu" : " 17 GB", | 20260506 16:11:54.751 | INFO | "huge" : "", | 20260506 16:11:54.751 | INFO | "huge_shared" : " 27 GB", | 20260506 16:11:54.751 | INFO | "malloc" : " 14 GB", | 20260506 16:11:54.751 | INFO | "pinned" : " 29 GB" | 20260506 16:11:54.751 | INFO | } | 20260506 16:11:54.751 | INFO | Perf report for the last 5.14117 seconds: | 20260506 16:11:54.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:11:54.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:54.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:54.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:54.751 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:11:54.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:54.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:54.751 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:11:54.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:54.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:54.752 | INFO | Uploaded 36GiB to GPU in 2.00085s (17.9924GiB/s) | 20260506 16:11:54.752 | INFO | Downloaded 34GiB to GPU in 1.92509s (17.6615GiB/s) | 20260506 16:11:59.888 | INFO | Analyzing chunk frames = [2692096,2692608) | 20260506 16:11:59.888 | INFO | MemInfo: { | 20260506 16:11:59.888 | INFO | "gpu" : " 17 GB", | 20260506 16:11:59.888 | INFO | "huge" : "", | 20260506 16:11:59.888 | INFO | "huge_shared" : " 27 GB", | 20260506 16:11:59.888 | INFO | "malloc" : " 14 GB", | 20260506 16:11:59.888 | INFO | "pinned" : " 29 GB" | 20260506 16:11:59.888 | INFO | } | 20260506 16:11:59.888 | INFO | Perf report for the last 5.1369 seconds: | 20260506 16:11:59.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:11:59.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:59.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:59.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:59.888 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 16:11:59.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:59.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:59.888 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:11:59.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:59.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:11:59.888 | INFO | Uploaded 36GiB to GPU in 1.98434s (18.1421GiB/s) | 20260506 16:11:59.888 | INFO | Downloaded 34GiB to GPU in 1.93037s (17.6132GiB/s) | 20260506 16:12:05.028 | INFO | Analyzing chunk frames = [2692608,2693120) | 20260506 16:12:05.028 | INFO | MemInfo: { | 20260506 16:12:05.028 | INFO | "gpu" : " 17 GB", | 20260506 16:12:05.028 | INFO | "huge" : "", | 20260506 16:12:05.028 | INFO | "huge_shared" : " 27 GB", | 20260506 16:12:05.028 | INFO | "malloc" : " 14 GB", | 20260506 16:12:05.028 | INFO | "pinned" : " 29 GB" | 20260506 16:12:05.028 | INFO | } | 20260506 16:12:05.028 | INFO | Perf report for the last 5.14007 seconds: | 20260506 16:12:05.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:12:05.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:05.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:05.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:05.028 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:12:05.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:05.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:05.028 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:12:05.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:05.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:05.028 | INFO | Uploaded 36GiB to GPU in 1.98209s (18.1627GiB/s) | 20260506 16:12:05.028 | INFO | Downloaded 34GiB to GPU in 1.88166s (18.0691GiB/s) | 20260506 16:12:10.166 | INFO | Analyzing chunk frames = [2693120,2693632) | 20260506 16:12:10.166 | INFO | MemInfo: { | 20260506 16:12:10.166 | INFO | "gpu" : " 17 GB", | 20260506 16:12:10.166 | INFO | "huge" : "", | 20260506 16:12:10.166 | INFO | "huge_shared" : " 27 GB", | 20260506 16:12:10.166 | INFO | "malloc" : " 14 GB", | 20260506 16:12:10.166 | INFO | "pinned" : " 29 GB" | 20260506 16:12:10.166 | INFO | } | 20260506 16:12:10.167 | INFO | Perf report for the last 5.13846 seconds: | 20260506 16:12:10.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:12:10.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:10.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:10.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:10.167 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 16:12:10.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:10.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:10.167 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:12:10.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:10.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:10.167 | INFO | Uploaded 36GiB to GPU in 1.9755s (18.2232GiB/s) | 20260506 16:12:10.167 | INFO | Downloaded 34GiB to GPU in 1.92692s (17.6448GiB/s) | 20260506 16:12:15.306 | INFO | Analyzing chunk frames = [2693632,2694144) | 20260506 16:12:15.306 | INFO | MemInfo: { | 20260506 16:12:15.306 | INFO | "gpu" : " 17 GB", | 20260506 16:12:15.306 | INFO | "huge" : "", | 20260506 16:12:15.306 | INFO | "huge_shared" : " 27 GB", | 20260506 16:12:15.306 | INFO | "malloc" : " 14 GB", | 20260506 16:12:15.306 | INFO | "pinned" : " 29 GB" | 20260506 16:12:15.306 | INFO | } | 20260506 16:12:15.307 | INFO | Perf report for the last 5.13995 seconds: | 20260506 16:12:15.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:12:15.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:15.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:15.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:15.307 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:12:15.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:15.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:15.307 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:12:15.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:15.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:15.307 | INFO | Uploaded 36GiB to GPU in 2.06969s (17.3939GiB/s) | 20260506 16:12:15.307 | INFO | Downloaded 34GiB to GPU in 1.96239s (17.3258GiB/s) | 20260506 16:12:20.446 | INFO | Analyzing chunk frames = [2694144,2694656) | 20260506 16:12:20.446 | INFO | MemInfo: { | 20260506 16:12:20.446 | INFO | "gpu" : " 17 GB", | 20260506 16:12:20.446 | INFO | "huge" : "", | 20260506 16:12:20.446 | INFO | "huge_shared" : " 27 GB", | 20260506 16:12:20.446 | INFO | "malloc" : " 14 GB", | 20260506 16:12:20.446 | INFO | "pinned" : " 29 GB" | 20260506 16:12:20.446 | INFO | } | 20260506 16:12:20.446 | INFO | Perf report for the last 5.13954 seconds: | 20260506 16:12:20.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:12:20.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:20.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:20.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:20.446 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 16:12:20.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:20.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:20.446 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:12:20.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:20.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:20.446 | INFO | Uploaded 36GiB to GPU in 2.0704s (17.3879GiB/s) | 20260506 16:12:20.446 | INFO | Downloaded 34GiB to GPU in 2.00786s (16.9335GiB/s) | 20260506 16:12:25.584 | INFO | Analyzing chunk frames = [2694656,2695168) | 20260506 16:12:25.584 | INFO | MemInfo: { | 20260506 16:12:25.584 | INFO | "gpu" : " 17 GB", | 20260506 16:12:25.584 | INFO | "huge" : "", | 20260506 16:12:25.584 | INFO | "huge_shared" : " 27 GB", | 20260506 16:12:25.584 | INFO | "malloc" : " 14 GB", | 20260506 16:12:25.584 | INFO | "pinned" : " 29 GB" | 20260506 16:12:25.584 | INFO | } | 20260506 16:12:25.584 | INFO | Perf report for the last 5.1383 seconds: | 20260506 16:12:25.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:12:25.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:25.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:25.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:25.584 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:12:25.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:25.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:25.585 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:12:25.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:25.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:25.585 | INFO | Uploaded 36GiB to GPU in 1.97419s (18.2353GiB/s) | 20260506 16:12:25.585 | INFO | Downloaded 34GiB to GPU in 1.9742s (17.2222GiB/s) | 20260506 16:12:25.585 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:12:25.585 | INFO | Total mem usage: 18 GiB | 20260506 16:12:25.585 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:12:25.585 | INFO | Recent High Water Marks Sum: 18 GiB (Savings: 0.0427%) | 20260506 16:12:25.585 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:12:25.585 | INFO | High Water Breakdown: | 20260506 16:12:25.585 | INFO | 63.4% : 11.4 GiB ( 100% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:12:25.585 | INFO | 36.5% : 6.58 GiB ( 98.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:12:25.585 | INFO | 0.11% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:12:25.585 | INFO | ------------------------------------------ | 20260506 16:12:25.585 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:12:25.585 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:12:25.585 | INFO | Total mem usage: 26.5 GiB | 20260506 16:12:25.585 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:12:25.585 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:12:25.585 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:12:25.585 | INFO | High Water Breakdown: | 20260506 16:12:25.585 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:12:25.585 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:12:25.585 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:12:25.585 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:12:25.585 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:12:25.585 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:12:25.585 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:12:25.585 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:12:25.585 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:12:25.585 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:12:25.585 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:12:25.585 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:12:25.585 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:12:25.585 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:12:25.585 | INFO | ------------------------------------------ | 20260506 16:12:25.585 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:12:25.585 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:12:25.585 | INFO | Total mem usage: 25.6 GiB | 20260506 16:12:25.585 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:12:25.585 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:12:25.585 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:12:25.585 | INFO | High Water Breakdown: | 20260506 16:12:25.585 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:12:25.585 | INFO | ------------------------------------------ | 20260506 16:12:25.585 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:12:25.586 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:12:25.586 | INFO | Total mem usage: 15.7 GiB | 20260506 16:12:25.586 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:12:25.586 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:12:25.586 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 16:12:25.586 | INFO | High Water Breakdown: | 20260506 16:12:25.586 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:12:25.586 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:12:25.586 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:12:25.586 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:12:25.586 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:12:25.586 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:12:25.586 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:12:25.586 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:12:25.586 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:12:25.586 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:12:25.586 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:12:25.586 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:12:25.586 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:12:25.586 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:12:25.586 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:12:25.586 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:12:25.586 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:12:25.586 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:12:25.586 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:12:25.586 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:12:25.586 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:12:25.586 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:12:25.586 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:12:25.586 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:12:25.586 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:12:25.586 | INFO | ------------------------------------------ | 20260506 16:12:25.586 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:12:25.586 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:12:25.586 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:12:25.586 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:12:25.586 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.52e+01, 1.33e+01 / 1.67e+00 | 20260506 16:12:25.586 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.02e-01, 1.11e-02 / 9.29e-03 | 20260506 16:12:25.586 | INFO | PerfReport: 11.57%, Baseline -- 3.63e+00 / 4.37e+00, 5.54e-01 / 3.97e-01 | 20260506 16:12:25.586 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.43e+00, 1.83e-01 / 1.34e-01 | 20260506 16:12:25.586 | INFO | PerfReport: 15.80%, DME -- 3.97e+01 / 4.34e+01, 3.49e+00 / 1.35e+00 | 20260506 16:12:25.586 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.79e+01 / 2.10e+01, 1.51e+00 / 1.66e+00 | 20260506 16:12:25.586 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.30e+00 / 2.90e+00, 3.95e-01 / 2.07e-01 | 20260506 16:12:25.586 | INFO | PerfReport: 3.52%, Metrics -- 1.10e+00 / 1.51e+00, 2.86e-01 / 1.49e-01 | 20260506 16:12:25.586 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.83e-01 / 4.35e-01, 6.02e-02 / 2.14e-02 | 20260506 16:12:25.586 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:12:25.586 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:12:25.586 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:12:25.586 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 16:12:25.586 | INFO | PerfReport: 21.60%, Upload -- 2.10e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260506 16:12:25.586 | INFO | PerfReport: 21.20%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 16:12:25.586 | INFO | PerfReport: 32.24%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 16:12:25.586 | INFO | PerfReport: 1.47%, LockUpload -- 1.43e+00 / 0.00e+00, 5.43e+00 / 0.00e+00 | 20260506 16:12:25.586 | INFO | PerfReport: 0.98%, LockDownload -- 9.57e-01 / 0.00e+00, 2.52e+00 / 0.00e+00 | 20260506 16:12:25.586 | INFO | PerfReport: 22.49%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 16:12:25.586 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260506 16:12:25.699 | INFO | Starting a round of Baz IO | 20260506 16:12:28.883 | INFO | Populating RTMetrics report | 20260506 16:12:28.884 | INFO | RTMetrics report populated | 20260506 16:12:28.884 | INFO | Opening tmp RTMetrics json file | 20260506 16:12:28.889 | INFO | Serializing RTMetrics to json | 20260506 16:12:28.892 | INFO | Serialization done, swapping tmp file now... | 20260506 16:12:28.892 | INFO | RTMetrics done | 20260506 16:12:29.052 | INFO | Finished a round of Baz IO: Wrote 2.097 GiB in 3.352 seconds | 20260506 16:12:29.052 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.53% / 0.01% / 63.46% | 20260506 16:12:29.052 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.76864% / 7.94729e-06% / 5.16574e-05% | 20260506 16:12:29.052 | INFO | Baz IO overall: | 20260506 16:12:29.052 | INFO | BytesWritten : 1412724478697 | 20260506 16:12:29.052 | INFO | NumEvents : 683641872221 | 20260506 16:12:30.725 | INFO | Analyzing chunk frames = [2695168,2695680) | 20260506 16:12:30.725 | INFO | MemInfo: { | 20260506 16:12:30.725 | INFO | "gpu" : " 17 GB", | 20260506 16:12:30.725 | INFO | "huge" : "", | 20260506 16:12:30.725 | INFO | "huge_shared" : " 27 GB", | 20260506 16:12:30.725 | INFO | "malloc" : " 14 GB", | 20260506 16:12:30.725 | INFO | "pinned" : " 29 GB" | 20260506 16:12:30.725 | INFO | } | 20260506 16:12:30.725 | INFO | Perf report for the last 5.14029 seconds: | 20260506 16:12:30.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:12:30.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:30.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:30.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:30.725 | INFO | ANALYSIS : 63.1%, 2.8, 94.0ms, 0 | 20260506 16:12:30.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:30.725 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 16:12:30.725 | INFO | PRE_HQ : 61.5%, 2.0, 62.6ms, 0 | 20260506 16:12:30.725 | INFO | BAZWRITER : 64.9%, 4.5, 150.6ms, 0 | 20260506 16:12:30.725 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 16:12:30.725 | INFO | Uploaded 36GiB to GPU in 2.04387s (17.6136GiB/s) | 20260506 16:12:30.725 | INFO | Downloaded 38GiB to GPU in 2.18905s (17.3591GiB/s) | 20260506 16:12:35.864 | INFO | Analyzing chunk frames = [2695680,2696192) | 20260506 16:12:35.864 | INFO | MemInfo: { | 20260506 16:12:35.865 | INFO | "gpu" : " 17 GB", | 20260506 16:12:35.865 | INFO | "huge" : "", | 20260506 16:12:35.865 | INFO | "huge_shared" : " 27 GB", | 20260506 16:12:35.865 | INFO | "malloc" : " 14 GB", | 20260506 16:12:35.865 | INFO | "pinned" : " 29 GB" | 20260506 16:12:35.865 | INFO | } | 20260506 16:12:35.865 | INFO | Perf report for the last 5.14001 seconds: | 20260506 16:12:35.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:12:35.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:35.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:35.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:35.865 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:12:35.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:35.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:35.865 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 16:12:35.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:35.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:35.865 | INFO | Uploaded 36GiB to GPU in 2.02256s (17.7992GiB/s) | 20260506 16:12:35.865 | INFO | Downloaded 34GiB to GPU in 1.94867s (17.4478GiB/s) | 20260506 16:12:41.005 | INFO | Analyzing chunk frames = [2696192,2696704) | 20260506 16:12:41.005 | INFO | MemInfo: { | 20260506 16:12:41.005 | INFO | "gpu" : " 17 GB", | 20260506 16:12:41.005 | INFO | "huge" : "", | 20260506 16:12:41.005 | INFO | "huge_shared" : " 27 GB", | 20260506 16:12:41.005 | INFO | "malloc" : " 14 GB", | 20260506 16:12:41.005 | INFO | "pinned" : " 29 GB" | 20260506 16:12:41.005 | INFO | } | 20260506 16:12:41.005 | INFO | Perf report for the last 5.14034 seconds: | 20260506 16:12:41.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:12:41.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:41.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:41.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:41.005 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 16:12:41.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:41.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:41.005 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:12:41.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:41.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:41.005 | INFO | Uploaded 36GiB to GPU in 2.00326s (17.9707GiB/s) | 20260506 16:12:41.005 | INFO | Downloaded 34GiB to GPU in 1.94341s (17.495GiB/s) | 20260506 16:12:46.142 | INFO | Analyzing chunk frames = [2696704,2697216) | 20260506 16:12:46.142 | INFO | MemInfo: { | 20260506 16:12:46.142 | INFO | "gpu" : " 17 GB", | 20260506 16:12:46.142 | INFO | "huge" : "", | 20260506 16:12:46.142 | INFO | "huge_shared" : " 27 GB", | 20260506 16:12:46.142 | INFO | "malloc" : " 14 GB", | 20260506 16:12:46.142 | INFO | "pinned" : " 29 GB" | 20260506 16:12:46.142 | INFO | } | 20260506 16:12:46.142 | INFO | Perf report for the last 5.13706 seconds: | 20260506 16:12:46.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:12:46.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:46.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:46.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:46.142 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 16:12:46.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:46.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:46.142 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:12:46.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:46.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:46.142 | INFO | Uploaded 36GiB to GPU in 2.01703s (17.848GiB/s) | 20260506 16:12:46.143 | INFO | Downloaded 34GiB to GPU in 1.91514s (17.7533GiB/s) | 20260506 16:12:51.281 | INFO | Analyzing chunk frames = [2697216,2697728) | 20260506 16:12:51.281 | INFO | MemInfo: { | 20260506 16:12:51.281 | INFO | "gpu" : " 17 GB", | 20260506 16:12:51.281 | INFO | "huge" : "", | 20260506 16:12:51.281 | INFO | "huge_shared" : " 27 GB", | 20260506 16:12:51.281 | INFO | "malloc" : " 14 GB", | 20260506 16:12:51.281 | INFO | "pinned" : " 29 GB" | 20260506 16:12:51.281 | INFO | } | 20260506 16:12:51.281 | INFO | Perf report for the last 5.13895 seconds: | 20260506 16:12:51.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:12:51.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:51.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:51.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:51.281 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 16:12:51.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:51.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:51.281 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:12:51.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:51.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:51.281 | INFO | Uploaded 36GiB to GPU in 1.99574s (18.0384GiB/s) | 20260506 16:12:51.281 | INFO | Downloaded 34GiB to GPU in 1.94502s (17.4806GiB/s) | 20260506 16:12:56.421 | INFO | Analyzing chunk frames = [2697728,2698240) | 20260506 16:12:56.421 | INFO | MemInfo: { | 20260506 16:12:56.421 | INFO | "gpu" : " 17 GB", | 20260506 16:12:56.421 | INFO | "huge" : "", | 20260506 16:12:56.421 | INFO | "huge_shared" : " 27 GB", | 20260506 16:12:56.421 | INFO | "malloc" : " 14 GB", | 20260506 16:12:56.421 | INFO | "pinned" : " 29 GB" | 20260506 16:12:56.421 | INFO | } | 20260506 16:12:56.421 | INFO | Perf report for the last 5.13968 seconds: | 20260506 16:12:56.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:12:56.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:56.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:56.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:56.421 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:12:56.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:56.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:56.421 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:12:56.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:56.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:12:56.421 | INFO | Uploaded 36GiB to GPU in 2.07312s (17.3651GiB/s) | 20260506 16:12:56.421 | INFO | Downloaded 34GiB to GPU in 1.96707s (17.2846GiB/s) | 20260506 16:13:01.560 | INFO | Analyzing chunk frames = [2698240,2698752) | 20260506 16:13:01.561 | INFO | MemInfo: { | 20260506 16:13:01.561 | INFO | "gpu" : " 17 GB", | 20260506 16:13:01.561 | INFO | "huge" : "", | 20260506 16:13:01.561 | INFO | "huge_shared" : " 27 GB", | 20260506 16:13:01.561 | INFO | "malloc" : " 14 GB", | 20260506 16:13:01.561 | INFO | "pinned" : " 29 GB" | 20260506 16:13:01.561 | INFO | } | 20260506 16:13:01.561 | INFO | Perf report for the last 5.13992 seconds: | 20260506 16:13:01.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:13:01.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:01.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:01.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:01.561 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 16:13:01.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:01.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:01.561 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 16:13:01.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:01.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:01.561 | INFO | Uploaded 36GiB to GPU in 2.04532s (17.6012GiB/s) | 20260506 16:13:01.561 | INFO | Downloaded 34GiB to GPU in 1.9744s (17.2204GiB/s) | 20260506 16:13:06.701 | INFO | Analyzing chunk frames = [2698752,2699264) | 20260506 16:13:06.701 | INFO | MemInfo: { | 20260506 16:13:06.701 | INFO | "gpu" : " 17 GB", | 20260506 16:13:06.701 | INFO | "huge" : "", | 20260506 16:13:06.701 | INFO | "huge_shared" : " 27 GB", | 20260506 16:13:06.701 | INFO | "malloc" : " 14 GB", | 20260506 16:13:06.701 | INFO | "pinned" : " 29 GB" | 20260506 16:13:06.701 | INFO | } | 20260506 16:13:06.701 | INFO | Perf report for the last 5.1402 seconds: | 20260506 16:13:06.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:13:06.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:06.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:06.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:06.701 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 16:13:06.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:06.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:06.701 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:13:06.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:06.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:06.701 | INFO | Uploaded 36GiB to GPU in 2.06226s (17.4566GiB/s) | 20260506 16:13:06.701 | INFO | Downloaded 34GiB to GPU in 1.98529s (17.126GiB/s) | 20260506 16:13:06.814 | INFO | Starting a round of Baz IO | 20260506 16:13:09.963 | INFO | Populating RTMetrics report | 20260506 16:13:09.963 | INFO | RTMetrics report populated | 20260506 16:13:09.964 | INFO | Opening tmp RTMetrics json file | 20260506 16:13:10.041 | INFO | Serializing RTMetrics to json | 20260506 16:13:10.267 | INFO | Serialization done, swapping tmp file now... | 20260506 16:13:10.318 | INFO | RTMetrics done | 20260506 16:13:10.457 | INFO | Finished a round of Baz IO: Wrote 2.107 GiB in 3.643 seconds | 20260506 16:13:10.457 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.52% / 0.01% / 63.47% | 20260506 16:13:10.457 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.78659% / 1.98682e-05% / 5.16574e-05% | 20260506 16:13:10.457 | INFO | Baz IO overall: | 20260506 16:13:10.457 | INFO | BytesWritten : 1414986726238 | 20260506 16:13:10.457 | INFO | NumEvents : 684733578209 | 20260506 16:13:11.838 | INFO | Analyzing chunk frames = [2699264,2699776) | 20260506 16:13:11.838 | INFO | MemInfo: { | 20260506 16:13:11.838 | INFO | "gpu" : " 17 GB", | 20260506 16:13:11.838 | INFO | "huge" : "", | 20260506 16:13:11.838 | INFO | "huge_shared" : " 27 GB", | 20260506 16:13:11.838 | INFO | "malloc" : " 14 GB", | 20260506 16:13:11.838 | INFO | "pinned" : " 29 GB" | 20260506 16:13:11.839 | INFO | } | 20260506 16:13:11.839 | INFO | Perf report for the last 5.13757 seconds: | 20260506 16:13:11.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:13:11.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:11.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:11.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:11.839 | INFO | ANALYSIS : 62.9%, 2.9, 95.2ms, 0 | 20260506 16:13:11.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:11.839 | INFO | RT_METRICS : 50.7%, 1.0, 26.6ms, 0 | 20260506 16:13:11.839 | INFO | PRE_HQ : 60.9%, 1.9, 59.4ms, 0 | 20260506 16:13:11.839 | INFO | BAZWRITER : 70.4%, 4.8, 174.0ms, 0 | 20260506 16:13:11.839 | INFO | BAZ_SAMPLING : 5.7%, 1.0, 3.0ms, 0 | 20260506 16:13:11.839 | INFO | Uploaded 36GiB to GPU in 2.054s (17.5267GiB/s) | 20260506 16:13:11.839 | INFO | Downloaded 38GiB to GPU in 2.21434s (17.1608GiB/s) | 20260506 16:13:16.978 | INFO | Analyzing chunk frames = [2699776,2700288) | 20260506 16:13:16.978 | INFO | MemInfo: { | 20260506 16:13:16.978 | INFO | "gpu" : " 17 GB", | 20260506 16:13:16.978 | INFO | "huge" : "", | 20260506 16:13:16.978 | INFO | "huge_shared" : " 27 GB", | 20260506 16:13:16.978 | INFO | "malloc" : " 14 GB", | 20260506 16:13:16.978 | INFO | "pinned" : " 29 GB" | 20260506 16:13:16.978 | INFO | } | 20260506 16:13:16.978 | INFO | Perf report for the last 5.13956 seconds: | 20260506 16:13:16.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:13:16.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:16.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:16.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:16.978 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:13:16.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:16.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:16.978 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:13:16.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:16.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:16.978 | INFO | Uploaded 36GiB to GPU in 1.99795s (18.0185GiB/s) | 20260506 16:13:16.978 | INFO | Downloaded 34GiB to GPU in 1.93872s (17.5373GiB/s) | 20260506 16:13:22.117 | INFO | Analyzing chunk frames = [2700288,2700800) | 20260506 16:13:22.117 | INFO | MemInfo: { | 20260506 16:13:22.117 | INFO | "gpu" : " 17 GB", | 20260506 16:13:22.117 | INFO | "huge" : "", | 20260506 16:13:22.117 | INFO | "huge_shared" : " 27 GB", | 20260506 16:13:22.117 | INFO | "malloc" : " 14 GB", | 20260506 16:13:22.117 | INFO | "pinned" : " 29 GB" | 20260506 16:13:22.117 | INFO | } | 20260506 16:13:22.117 | INFO | Perf report for the last 5.1392 seconds: | 20260506 16:13:22.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:13:22.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:22.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:22.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:22.117 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 16:13:22.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:22.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:22.117 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:13:22.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:22.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:22.118 | INFO | Uploaded 36GiB to GPU in 1.99314s (18.0619GiB/s) | 20260506 16:13:22.118 | INFO | Downloaded 34GiB to GPU in 1.93766s (17.5469GiB/s) | 20260506 16:13:27.257 | INFO | Analyzing chunk frames = [2700800,2701312) | 20260506 16:13:27.257 | INFO | MemInfo: { | 20260506 16:13:27.257 | INFO | "gpu" : " 17 GB", | 20260506 16:13:27.257 | INFO | "huge" : "", | 20260506 16:13:27.257 | INFO | "huge_shared" : " 27 GB", | 20260506 16:13:27.257 | INFO | "malloc" : " 14 GB", | 20260506 16:13:27.257 | INFO | "pinned" : " 29 GB" | 20260506 16:13:27.257 | INFO | } | 20260506 16:13:27.257 | INFO | Perf report for the last 5.13993 seconds: | 20260506 16:13:27.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:13:27.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:27.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:27.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:27.257 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 16:13:27.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:27.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:27.257 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:13:27.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:27.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:27.258 | INFO | Uploaded 36GiB to GPU in 2.01695s (17.8487GiB/s) | 20260506 16:13:27.258 | INFO | Downloaded 34GiB to GPU in 1.91062s (17.7952GiB/s) | 20260506 16:13:32.396 | INFO | Analyzing chunk frames = [2701312,2701824) | 20260506 16:13:32.396 | INFO | MemInfo: { | 20260506 16:13:32.396 | INFO | "gpu" : " 17 GB", | 20260506 16:13:32.396 | INFO | "huge" : "", | 20260506 16:13:32.396 | INFO | "huge_shared" : " 27 GB", | 20260506 16:13:32.396 | INFO | "malloc" : " 14 GB", | 20260506 16:13:32.396 | INFO | "pinned" : " 29 GB" | 20260506 16:13:32.396 | INFO | } | 20260506 16:13:32.396 | INFO | Perf report for the last 5.13895 seconds: | 20260506 16:13:32.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:13:32.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:32.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:32.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:32.396 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 16:13:32.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:32.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:32.396 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:13:32.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:32.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:32.397 | INFO | Uploaded 36GiB to GPU in 2.02255s (17.7993GiB/s) | 20260506 16:13:32.397 | INFO | Downloaded 34GiB to GPU in 1.93857s (17.5387GiB/s) | 20260506 16:13:37.535 | INFO | Analyzing chunk frames = [2701824,2702336) | 20260506 16:13:37.535 | INFO | MemInfo: { | 20260506 16:13:37.535 | INFO | "gpu" : " 17 GB", | 20260506 16:13:37.535 | INFO | "huge" : "", | 20260506 16:13:37.535 | INFO | "huge_shared" : " 27 GB", | 20260506 16:13:37.535 | INFO | "malloc" : " 14 GB", | 20260506 16:13:37.535 | INFO | "pinned" : " 29 GB" | 20260506 16:13:37.535 | INFO | } | 20260506 16:13:37.535 | INFO | Perf report for the last 5.1391 seconds: | 20260506 16:13:37.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:13:37.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:37.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:37.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:37.536 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 16:13:37.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:37.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:37.536 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 16:13:37.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:37.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:37.536 | INFO | Uploaded 36GiB to GPU in 2.03723s (17.671GiB/s) | 20260506 16:13:37.536 | INFO | Downloaded 34GiB to GPU in 1.92459s (17.6661GiB/s) | 20260506 16:13:42.675 | INFO | Analyzing chunk frames = [2702336,2702848) | 20260506 16:13:42.675 | INFO | MemInfo: { | 20260506 16:13:42.675 | INFO | "gpu" : " 17 GB", | 20260506 16:13:42.675 | INFO | "huge" : "", | 20260506 16:13:42.675 | INFO | "huge_shared" : " 27 GB", | 20260506 16:13:42.676 | INFO | "malloc" : " 14 GB", | 20260506 16:13:42.676 | INFO | "pinned" : " 29 GB" | 20260506 16:13:42.676 | INFO | } | 20260506 16:13:42.676 | INFO | Perf report for the last 5.14032 seconds: | 20260506 16:13:42.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:13:42.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:42.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:42.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:42.676 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:13:42.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:42.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:42.676 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:13:42.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:42.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:42.676 | INFO | Uploaded 36GiB to GPU in 2.00303s (17.9727GiB/s) | 20260506 16:13:42.676 | INFO | Downloaded 34GiB to GPU in 1.96038s (17.3436GiB/s) | 20260506 16:13:47.815 | INFO | Analyzing chunk frames = [2702848,2703360) | 20260506 16:13:47.815 | INFO | MemInfo: { | 20260506 16:13:47.815 | INFO | "gpu" : " 17 GB", | 20260506 16:13:47.815 | INFO | "huge" : "", | 20260506 16:13:47.815 | INFO | "huge_shared" : " 27 GB", | 20260506 16:13:47.815 | INFO | "malloc" : " 14 GB", | 20260506 16:13:47.815 | INFO | "pinned" : " 29 GB" | 20260506 16:13:47.815 | INFO | } | 20260506 16:13:47.815 | INFO | Perf report for the last 5.1396 seconds: | 20260506 16:13:47.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:13:47.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:47.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:47.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:47.815 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 16:13:47.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:47.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:47.815 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:13:47.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:47.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:47.816 | INFO | Uploaded 36GiB to GPU in 2.03679s (17.6748GiB/s) | 20260506 16:13:47.816 | INFO | Downloaded 34GiB to GPU in 1.99055s (17.0807GiB/s) | 20260506 16:13:47.816 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:13:47.816 | INFO | Total mem usage: 18 GiB | 20260506 16:13:47.816 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:13:47.816 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -13.8%) | 20260506 16:13:47.816 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:13:47.816 | INFO | High Water Breakdown: | 20260506 16:13:47.816 | INFO | 61.5% : 9.74 GiB ( 85.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:13:47.816 | INFO | 38.4% : 6.07 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:13:47.816 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:13:47.816 | INFO | ------------------------------------------ | 20260506 16:13:47.816 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:13:47.816 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:13:47.816 | INFO | Total mem usage: 26.5 GiB | 20260506 16:13:47.816 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:13:47.816 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 16:13:47.816 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260506 16:13:47.816 | INFO | High Water Breakdown: | 20260506 16:13:47.816 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:13:47.816 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:13:47.816 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:13:47.816 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:13:47.816 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:13:47.816 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:13:47.816 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:13:47.816 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:13:47.816 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:13:47.816 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:13:47.816 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:13:47.816 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:13:47.816 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:13:47.816 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:13:47.816 | INFO | ------------------------------------------ | 20260506 16:13:47.816 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:13:47.816 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:13:47.816 | INFO | Total mem usage: 25.6 GiB | 20260506 16:13:47.816 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:13:47.816 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:13:47.816 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:13:47.816 | INFO | High Water Breakdown: | 20260506 16:13:47.816 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:13:47.816 | INFO | ------------------------------------------ | 20260506 16:13:47.816 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:13:47.816 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:13:47.816 | INFO | Total mem usage: 15.7 GiB | 20260506 16:13:47.816 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:13:47.816 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:13:47.816 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 16:13:47.817 | INFO | High Water Breakdown: | 20260506 16:13:47.817 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:13:47.817 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:13:47.817 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:13:47.817 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:13:47.817 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:13:47.817 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:13:47.817 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:13:47.817 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:13:47.817 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:13:47.817 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:13:47.817 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:13:47.817 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:13:47.817 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:13:47.817 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:13:47.817 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:13:47.817 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:13:47.817 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:13:47.817 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:13:47.817 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:13:47.817 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:13:47.817 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:13:47.817 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:13:47.817 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:13:47.817 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:13:47.817 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:13:47.817 | INFO | ------------------------------------------ | 20260506 16:13:47.817 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:13:47.817 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:13:47.817 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:13:47.817 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:13:47.817 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.54e+01, 1.35e+01 / 1.66e+00 | 20260506 16:13:47.817 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.03e-01, 1.02e-02 / 1.22e-02 | 20260506 16:13:47.817 | INFO | PerfReport: 11.58%, Baseline -- 3.64e+00 / 4.36e+00, 5.52e-01 / 3.87e-01 | 20260506 16:13:47.817 | INFO | PerfReport: 3.24%, Histogram -- 1.02e+00 / 1.45e+00, 1.82e-01 / 1.53e-01 | 20260506 16:13:47.817 | INFO | PerfReport: 15.82%, DME -- 3.98e+01 / 4.38e+01, 3.65e+00 / 2.03e+00 | 20260506 16:13:47.817 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.79e+01 / 2.09e+01, 1.51e+00 / 1.64e+00 | 20260506 16:13:47.817 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.31e+00 / 2.87e+00, 3.95e-01 / 2.75e-01 | 20260506 16:13:47.817 | INFO | PerfReport: 3.53%, Metrics -- 1.11e+00 / 1.46e+00, 2.84e-01 / 1.76e-01 | 20260506 16:13:47.817 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.87e-01 / 4.40e-01, 6.10e-02 / 3.05e-02 | 20260506 16:13:47.817 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:13:47.817 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:13:47.817 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:13:47.817 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 16:13:47.817 | INFO | PerfReport: 21.67%, Upload -- 2.11e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260506 16:13:47.817 | INFO | PerfReport: 21.17%, Download -- 2.06e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 16:13:47.817 | INFO | PerfReport: 32.26%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 16:13:47.817 | INFO | PerfReport: 1.63%, LockUpload -- 1.59e+00 / 0.00e+00, 5.89e+00 / 0.00e+00 | 20260506 16:13:47.817 | INFO | PerfReport: 0.98%, LockDownload -- 9.56e-01 / 0.00e+00, 2.49e+00 / 0.00e+00 | 20260506 16:13:47.817 | INFO | PerfReport: 22.28%, LockCompute -- 2.17e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 16:13:47.817 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260506 16:13:47.929 | INFO | Starting a round of Baz IO | 20260506 16:13:51.077 | INFO | Populating RTMetrics report | 20260506 16:13:51.078 | INFO | RTMetrics report populated | 20260506 16:13:51.078 | INFO | Opening tmp RTMetrics json file | 20260506 16:13:51.118 | INFO | Serializing RTMetrics to json | 20260506 16:13:51.164 | INFO | Serialization done, swapping tmp file now... | 20260506 16:13:51.165 | INFO | RTMetrics done | 20260506 16:13:51.228 | INFO | Finished a round of Baz IO: Wrote 2.101 GiB in 3.299 seconds | 20260506 16:13:51.228 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.51% / 0.01% / 63.48% | 20260506 16:13:51.228 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.77715% / 1.58946e-05% / 7.94729e-05% | 20260506 16:13:51.228 | INFO | Baz IO overall: | 20260506 16:13:51.229 | INFO | BytesWritten : 1417242538729 | 20260506 16:13:51.229 | INFO | NumEvents : 685822456211 | 20260506 16:13:52.954 | INFO | Analyzing chunk frames = [2703360,2703872) | 20260506 16:13:52.954 | INFO | MemInfo: { | 20260506 16:13:52.954 | INFO | "gpu" : " 17 GB", | 20260506 16:13:52.954 | INFO | "huge" : "", | 20260506 16:13:52.954 | INFO | "huge_shared" : " 27 GB", | 20260506 16:13:52.954 | INFO | "malloc" : " 14 GB", | 20260506 16:13:52.954 | INFO | "pinned" : " 29 GB" | 20260506 16:13:52.954 | INFO | } | 20260506 16:13:52.954 | INFO | Perf report for the last 5.1387 seconds: | 20260506 16:13:52.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:13:52.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:52.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:52.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:52.954 | INFO | ANALYSIS : 62.9%, 2.9, 95.3ms, 0 | 20260506 16:13:52.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:52.954 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 16:13:52.954 | INFO | PRE_HQ : 60.8%, 1.9, 59.2ms, 0 | 20260506 16:13:52.954 | INFO | BAZWRITER : 63.7%, 4.8, 157.0ms, 0 | 20260506 16:13:52.954 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 16:13:52.954 | INFO | Uploaded 36GiB to GPU in 2.07715s (17.3314GiB/s) | 20260506 16:13:52.954 | INFO | Downloaded 38GiB to GPU in 2.23073s (17.0348GiB/s) | 20260506 16:13:58.094 | INFO | Analyzing chunk frames = [2703872,2704384) | 20260506 16:13:58.094 | INFO | MemInfo: { | 20260506 16:13:58.094 | INFO | "gpu" : " 17 GB", | 20260506 16:13:58.094 | INFO | "huge" : "", | 20260506 16:13:58.094 | INFO | "huge_shared" : " 27 GB", | 20260506 16:13:58.094 | INFO | "malloc" : " 14 GB", | 20260506 16:13:58.094 | INFO | "pinned" : " 29 GB" | 20260506 16:13:58.094 | INFO | } | 20260506 16:13:58.094 | INFO | Perf report for the last 5.13975 seconds: | 20260506 16:13:58.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:13:58.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:58.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:58.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:58.094 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 16:13:58.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:58.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:58.094 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:13:58.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:58.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:13:58.094 | INFO | Uploaded 36GiB to GPU in 2.00145s (17.9869GiB/s) | 20260506 16:13:58.094 | INFO | Downloaded 34GiB to GPU in 1.92863s (17.6291GiB/s) | 20260506 16:14:03.231 | INFO | Analyzing chunk frames = [2704384,2704896) | 20260506 16:14:03.231 | INFO | MemInfo: { | 20260506 16:14:03.231 | INFO | "gpu" : " 17 GB", | 20260506 16:14:03.231 | INFO | "huge" : "", | 20260506 16:14:03.231 | INFO | "huge_shared" : " 27 GB", | 20260506 16:14:03.231 | INFO | "malloc" : " 14 GB", | 20260506 16:14:03.231 | INFO | "pinned" : " 29 GB" | 20260506 16:14:03.231 | INFO | } | 20260506 16:14:03.231 | INFO | Perf report for the last 5.13743 seconds: | 20260506 16:14:03.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:14:03.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:03.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:03.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:03.231 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 16:14:03.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:03.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:03.231 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:14:03.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:03.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:03.232 | INFO | Uploaded 36GiB to GPU in 1.98672s (18.1203GiB/s) | 20260506 16:14:03.232 | INFO | Downloaded 34GiB to GPU in 1.9306s (17.6111GiB/s) | 20260506 16:14:08.371 | INFO | Analyzing chunk frames = [2704896,2705408) | 20260506 16:14:08.371 | INFO | MemInfo: { | 20260506 16:14:08.371 | INFO | "gpu" : " 17 GB", | 20260506 16:14:08.371 | INFO | "huge" : "", | 20260506 16:14:08.371 | INFO | "huge_shared" : " 27 GB", | 20260506 16:14:08.371 | INFO | "malloc" : " 14 GB", | 20260506 16:14:08.372 | INFO | "pinned" : " 29 GB" | 20260506 16:14:08.372 | INFO | } | 20260506 16:14:08.372 | INFO | Perf report for the last 5.14039 seconds: | 20260506 16:14:08.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:14:08.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:08.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:08.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:08.372 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 16:14:08.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:08.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:08.372 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:14:08.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:08.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:08.372 | INFO | Uploaded 36GiB to GPU in 2.01665s (17.8514GiB/s) | 20260506 16:14:08.372 | INFO | Downloaded 34GiB to GPU in 1.91279s (17.7751GiB/s) | 20260506 16:14:13.513 | INFO | Analyzing chunk frames = [2705408,2705920) | 20260506 16:14:13.513 | INFO | MemInfo: { | 20260506 16:14:13.513 | INFO | "gpu" : " 17 GB", | 20260506 16:14:13.513 | INFO | "huge" : "", | 20260506 16:14:13.514 | INFO | "huge_shared" : " 27 GB", | 20260506 16:14:13.514 | INFO | "malloc" : " 14 GB", | 20260506 16:14:13.514 | INFO | "pinned" : " 29 GB" | 20260506 16:14:13.514 | INFO | } | 20260506 16:14:13.514 | INFO | Perf report for the last 5.142 seconds: | 20260506 16:14:13.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:14:13.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:13.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:13.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:13.514 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 16:14:13.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:13.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:13.514 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:14:13.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:13.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:13.514 | INFO | Uploaded 36GiB to GPU in 1.99501s (18.0451GiB/s) | 20260506 16:14:13.514 | INFO | Downloaded 34GiB to GPU in 1.93096s (17.6078GiB/s) | 20260506 16:14:18.650 | INFO | Analyzing chunk frames = [2705920,2706432) | 20260506 16:14:18.650 | INFO | MemInfo: { | 20260506 16:14:18.650 | INFO | "gpu" : " 17 GB", | 20260506 16:14:18.650 | INFO | "huge" : "", | 20260506 16:14:18.650 | INFO | "huge_shared" : " 27 GB", | 20260506 16:14:18.650 | INFO | "malloc" : " 14 GB", | 20260506 16:14:18.650 | INFO | "pinned" : " 29 GB" | 20260506 16:14:18.650 | INFO | } | 20260506 16:14:18.650 | INFO | Perf report for the last 5.1363 seconds: | 20260506 16:14:18.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:14:18.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:18.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:18.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:18.650 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:14:18.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:18.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:18.650 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260506 16:14:18.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:18.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:18.650 | INFO | Uploaded 36GiB to GPU in 2.04091s (17.6392GiB/s) | 20260506 16:14:18.650 | INFO | Downloaded 34GiB to GPU in 1.92707s (17.6433GiB/s) | 20260506 16:14:23.789 | INFO | Analyzing chunk frames = [2706432,2706944) | 20260506 16:14:23.789 | INFO | MemInfo: { | 20260506 16:14:23.789 | INFO | "gpu" : " 17 GB", | 20260506 16:14:23.789 | INFO | "huge" : "", | 20260506 16:14:23.789 | INFO | "huge_shared" : " 27 GB", | 20260506 16:14:23.789 | INFO | "malloc" : " 14 GB", | 20260506 16:14:23.789 | INFO | "pinned" : " 29 GB" | 20260506 16:14:23.789 | INFO | } | 20260506 16:14:23.789 | INFO | Perf report for the last 5.13907 seconds: | 20260506 16:14:23.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:14:23.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:23.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:23.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:23.789 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 16:14:23.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:23.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:23.789 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 16:14:23.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:23.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:23.789 | INFO | Uploaded 36GiB to GPU in 1.99385s (18.0555GiB/s) | 20260506 16:14:23.790 | INFO | Downloaded 34GiB to GPU in 1.96641s (17.2904GiB/s) | 20260506 16:14:28.928 | INFO | Analyzing chunk frames = [2706944,2707456) | 20260506 16:14:28.929 | INFO | MemInfo: { | 20260506 16:14:28.929 | INFO | "gpu" : " 17 GB", | 20260506 16:14:28.929 | INFO | "huge" : "", | 20260506 16:14:28.929 | INFO | "huge_shared" : " 27 GB", | 20260506 16:14:28.929 | INFO | "malloc" : " 14 GB", | 20260506 16:14:28.929 | INFO | "pinned" : " 29 GB" | 20260506 16:14:28.929 | INFO | } | 20260506 16:14:28.929 | INFO | Perf report for the last 5.13964 seconds: | 20260506 16:14:28.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:14:28.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:28.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:28.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:28.929 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 16:14:28.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:28.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:28.929 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:14:28.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:28.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:28.929 | INFO | Uploaded 36GiB to GPU in 2.04765s (17.5811GiB/s) | 20260506 16:14:28.929 | INFO | Downloaded 34GiB to GPU in 1.97028s (17.2564GiB/s) | 20260506 16:14:29.041 | INFO | Starting a round of Baz IO | 20260506 16:14:32.189 | INFO | Populating RTMetrics report | 20260506 16:14:32.189 | INFO | RTMetrics report populated | 20260506 16:14:32.189 | INFO | Opening tmp RTMetrics json file | 20260506 16:14:32.216 | INFO | Serializing RTMetrics to json | 20260506 16:14:32.223 | INFO | Serialization done, swapping tmp file now... | 20260506 16:14:32.227 | INFO | RTMetrics done | 20260506 16:14:32.794 | INFO | Finished a round of Baz IO: Wrote 2.101 GiB in 3.752 seconds | 20260506 16:14:32.794 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.50% / 0.01% / 63.49% | 20260506 16:14:32.794 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.77779% / 7.94729e-06% / 6.75519e-05% | 20260506 16:14:32.794 | INFO | Baz IO overall: | 20260506 16:14:32.794 | INFO | BytesWritten : 1419497995241 | 20260506 16:14:32.794 | INFO | NumEvents : 686912254181 | 20260506 16:14:34.068 | INFO | Analyzing chunk frames = [2707456,2707968) | 20260506 16:14:34.068 | INFO | MemInfo: { | 20260506 16:14:34.068 | INFO | "gpu" : " 17 GB", | 20260506 16:14:34.068 | INFO | "huge" : "", | 20260506 16:14:34.068 | INFO | "huge_shared" : " 27 GB", | 20260506 16:14:34.068 | INFO | "malloc" : " 14 GB", | 20260506 16:14:34.068 | INFO | "pinned" : " 29 GB" | 20260506 16:14:34.068 | INFO | } | 20260506 16:14:34.068 | INFO | Perf report for the last 5.13942 seconds: | 20260506 16:14:34.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:14:34.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:34.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:34.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:34.068 | INFO | ANALYSIS : 62.8%, 2.9, 95.2ms, 0 | 20260506 16:14:34.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:34.068 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 16:14:34.068 | INFO | PRE_HQ : 60.8%, 1.9, 59.4ms, 0 | 20260506 16:14:34.068 | INFO | BAZWRITER : 72.5%, 4.8, 179.8ms, 0 | 20260506 16:14:34.068 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 16:14:34.068 | INFO | Uploaded 36GiB to GPU in 2.06617s (17.4235GiB/s) | 20260506 16:14:34.069 | INFO | Downloaded 38GiB to GPU in 2.23212s (17.0242GiB/s) | 20260506 16:14:39.208 | INFO | Analyzing chunk frames = [2707968,2708480) | 20260506 16:14:39.208 | INFO | MemInfo: { | 20260506 16:14:39.208 | INFO | "gpu" : " 17 GB", | 20260506 16:14:39.208 | INFO | "huge" : "", | 20260506 16:14:39.208 | INFO | "huge_shared" : " 27 GB", | 20260506 16:14:39.208 | INFO | "malloc" : " 14 GB", | 20260506 16:14:39.208 | INFO | "pinned" : " 29 GB" | 20260506 16:14:39.208 | INFO | } | 20260506 16:14:39.208 | INFO | Perf report for the last 5.1399 seconds: | 20260506 16:14:39.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:14:39.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:39.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:39.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:39.208 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:14:39.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:39.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:39.208 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:14:39.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:39.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:39.208 | INFO | Uploaded 36GiB to GPU in 1.99864s (18.0122GiB/s) | 20260506 16:14:39.208 | INFO | Downloaded 34GiB to GPU in 1.93693s (17.5536GiB/s) | 20260506 16:14:44.347 | INFO | Analyzing chunk frames = [2708480,2708992) | 20260506 16:14:44.347 | INFO | MemInfo: { | 20260506 16:14:44.347 | INFO | "gpu" : " 17 GB", | 20260506 16:14:44.347 | INFO | "huge" : "", | 20260506 16:14:44.347 | INFO | "huge_shared" : " 27 GB", | 20260506 16:14:44.347 | INFO | "malloc" : " 14 GB", | 20260506 16:14:44.347 | INFO | "pinned" : " 29 GB" | 20260506 16:14:44.347 | INFO | } | 20260506 16:14:44.347 | INFO | Perf report for the last 5.13908 seconds: | 20260506 16:14:44.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:14:44.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:44.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:44.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:44.347 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 16:14:44.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:44.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:44.347 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:14:44.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:44.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:44.347 | INFO | Uploaded 36GiB to GPU in 1.99226s (18.0699GiB/s) | 20260506 16:14:44.347 | INFO | Downloaded 34GiB to GPU in 1.93716s (17.5515GiB/s) | 20260506 16:14:49.485 | INFO | Analyzing chunk frames = [2708992,2709504) | 20260506 16:14:49.485 | INFO | MemInfo: { | 20260506 16:14:49.485 | INFO | "gpu" : " 17 GB", | 20260506 16:14:49.485 | INFO | "huge" : "", | 20260506 16:14:49.486 | INFO | "huge_shared" : " 27 GB", | 20260506 16:14:49.486 | INFO | "malloc" : " 14 GB", | 20260506 16:14:49.486 | INFO | "pinned" : " 29 GB" | 20260506 16:14:49.486 | INFO | } | 20260506 16:14:49.486 | INFO | Perf report for the last 5.13856 seconds: | 20260506 16:14:49.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:14:49.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:49.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:49.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:49.486 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 16:14:49.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:49.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:49.486 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:14:49.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:49.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:49.486 | INFO | Uploaded 36GiB to GPU in 1.9736s (18.2408GiB/s) | 20260506 16:14:49.486 | INFO | Downloaded 34GiB to GPU in 1.87886s (18.096GiB/s) | 20260506 16:14:54.625 | INFO | Analyzing chunk frames = [2709504,2710016) | 20260506 16:14:54.625 | INFO | MemInfo: { | 20260506 16:14:54.625 | INFO | "gpu" : " 17 GB", | 20260506 16:14:54.625 | INFO | "huge" : "", | 20260506 16:14:54.625 | INFO | "huge_shared" : " 27 GB", | 20260506 16:14:54.625 | INFO | "malloc" : " 14 GB", | 20260506 16:14:54.625 | INFO | "pinned" : " 29 GB" | 20260506 16:14:54.625 | INFO | } | 20260506 16:14:54.626 | INFO | Perf report for the last 5.13982 seconds: | 20260506 16:14:54.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:14:54.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:54.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:54.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:54.626 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 16:14:54.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:54.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:54.626 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:14:54.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:54.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:54.626 | INFO | Uploaded 36GiB to GPU in 2.00017s (17.9984GiB/s) | 20260506 16:14:54.626 | INFO | Downloaded 34GiB to GPU in 1.92198s (17.6901GiB/s) | 20260506 16:14:59.765 | INFO | Analyzing chunk frames = [2710016,2710528) | 20260506 16:14:59.765 | INFO | MemInfo: { | 20260506 16:14:59.765 | INFO | "gpu" : " 17 GB", | 20260506 16:14:59.765 | INFO | "huge" : "", | 20260506 16:14:59.765 | INFO | "huge_shared" : " 27 GB", | 20260506 16:14:59.765 | INFO | "malloc" : " 14 GB", | 20260506 16:14:59.766 | INFO | "pinned" : " 29 GB" | 20260506 16:14:59.766 | INFO | } | 20260506 16:14:59.766 | INFO | Perf report for the last 5.14016 seconds: | 20260506 16:14:59.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:14:59.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:59.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:59.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:59.766 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:14:59.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:59.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:59.766 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:14:59.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:59.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:14:59.766 | INFO | Uploaded 36GiB to GPU in 2.07239s (17.3713GiB/s) | 20260506 16:14:59.766 | INFO | Downloaded 34GiB to GPU in 1.97412s (17.2228GiB/s) | 20260506 16:15:04.904 | INFO | Analyzing chunk frames = [2710528,2711040) | 20260506 16:15:04.904 | INFO | MemInfo: { | 20260506 16:15:04.904 | INFO | "gpu" : " 17 GB", | 20260506 16:15:04.904 | INFO | "huge" : "", | 20260506 16:15:04.904 | INFO | "huge_shared" : " 27 GB", | 20260506 16:15:04.904 | INFO | "malloc" : " 14 GB", | 20260506 16:15:04.904 | INFO | "pinned" : " 29 GB" | 20260506 16:15:04.905 | INFO | } | 20260506 16:15:04.905 | INFO | Perf report for the last 5.13882 seconds: | 20260506 16:15:04.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:15:04.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:04.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:04.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:04.905 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 16:15:04.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:04.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:04.905 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 16:15:04.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:04.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:04.905 | INFO | Uploaded 36GiB to GPU in 2.05563s (17.5129GiB/s) | 20260506 16:15:04.905 | INFO | Downloaded 34GiB to GPU in 2.00903s (16.9236GiB/s) | 20260506 16:15:10.043 | INFO | Analyzing chunk frames = [2711040,2711552) | 20260506 16:15:10.043 | INFO | MemInfo: { | 20260506 16:15:10.043 | INFO | "gpu" : " 17 GB", | 20260506 16:15:10.043 | INFO | "huge" : "", | 20260506 16:15:10.044 | INFO | "huge_shared" : " 27 GB", | 20260506 16:15:10.044 | INFO | "malloc" : " 14 GB", | 20260506 16:15:10.044 | INFO | "pinned" : " 29 GB" | 20260506 16:15:10.044 | INFO | } | 20260506 16:15:10.044 | INFO | Perf report for the last 5.13917 seconds: | 20260506 16:15:10.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:15:10.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:10.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:10.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:10.044 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 16:15:10.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:10.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:10.044 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:15:10.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:10.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:10.044 | INFO | Uploaded 36GiB to GPU in 2.0303s (17.7314GiB/s) | 20260506 16:15:10.044 | INFO | Downloaded 34GiB to GPU in 1.99436s (17.048GiB/s) | 20260506 16:15:10.044 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:15:10.044 | INFO | Total mem usage: 18 GiB | 20260506 16:15:10.044 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:15:10.044 | INFO | Recent High Water Marks Sum: 16.7 GiB (Savings: -7.8%) | 20260506 16:15:10.044 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:15:10.044 | INFO | High Water Breakdown: | 20260506 16:15:10.044 | INFO | 62.3% : 10.4 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:15:10.044 | INFO | 37.6% : 6.27 GiB ( 93.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:15:10.044 | INFO | 0.119% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:15:10.045 | INFO | ------------------------------------------ | 20260506 16:15:10.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:15:10.045 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:15:10.045 | INFO | Total mem usage: 26.5 GiB | 20260506 16:15:10.045 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:15:10.045 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:15:10.045 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:15:10.045 | INFO | High Water Breakdown: | 20260506 16:15:10.045 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:15:10.045 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:15:10.045 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:15:10.045 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:15:10.045 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:15:10.045 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:15:10.045 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:15:10.045 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:15:10.045 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:15:10.045 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:15:10.045 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:15:10.045 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:15:10.045 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:15:10.045 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:15:10.045 | INFO | ------------------------------------------ | 20260506 16:15:10.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:15:10.045 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:15:10.045 | INFO | Total mem usage: 25.6 GiB | 20260506 16:15:10.045 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:15:10.045 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:15:10.045 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:15:10.045 | INFO | High Water Breakdown: | 20260506 16:15:10.045 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:15:10.045 | INFO | ------------------------------------------ | 20260506 16:15:10.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:15:10.045 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:15:10.045 | INFO | Total mem usage: 15.7 GiB | 20260506 16:15:10.045 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:15:10.045 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:15:10.045 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:15:10.045 | INFO | High Water Breakdown: | 20260506 16:15:10.045 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:15:10.045 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:15:10.045 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:15:10.045 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:15:10.045 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:15:10.045 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:15:10.045 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:15:10.045 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:15:10.045 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:15:10.045 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:15:10.045 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:15:10.045 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:15:10.045 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:15:10.045 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:15:10.045 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:15:10.045 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:15:10.045 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:15:10.045 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:15:10.045 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:15:10.045 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:15:10.045 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:15:10.045 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:15:10.045 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:15:10.045 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:15:10.045 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:15:10.045 | INFO | ------------------------------------------ | 20260506 16:15:10.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:15:10.045 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:15:10.045 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:15:10.045 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:15:10.045 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.50e+01, 1.34e+01 / 1.64e+00 | 20260506 16:15:10.045 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.02e-01, 1.05e-02 / 7.85e-03 | 20260506 16:15:10.045 | INFO | PerfReport: 11.59%, Baseline -- 3.64e+00 / 4.34e+00, 5.58e-01 / 3.92e-01 | 20260506 16:15:10.045 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.45e+00, 1.85e-01 / 1.45e-01 | 20260506 16:15:10.045 | INFO | PerfReport: 15.80%, DME -- 3.97e+01 / 4.36e+01, 3.62e+00 / 1.33e+00 | 20260506 16:15:10.045 | INFO | PerfReport: 56.97%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.48e+00 / 1.76e+00 | 20260506 16:15:10.045 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.31e+00 / 2.84e+00, 3.92e-01 / 2.68e-01 | 20260506 16:15:10.045 | INFO | PerfReport: 3.53%, Metrics -- 1.11e+00 / 1.47e+00, 2.88e-01 / 1.88e-01 | 20260506 16:15:10.045 | INFO | PerfReport: 1.23%, PulseSerializing -- 3.85e-01 / 4.39e-01, 6.10e-02 / 3.15e-02 | 20260506 16:15:10.045 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:15:10.045 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:15:10.045 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:15:10.045 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 16:15:10.045 | INFO | PerfReport: 21.63%, Upload -- 2.11e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260506 16:15:10.045 | INFO | PerfReport: 21.19%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 16:15:10.045 | INFO | PerfReport: 32.24%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 16:15:10.046 | INFO | PerfReport: 1.49%, LockUpload -- 1.45e+00 / 0.00e+00, 5.70e+00 / 0.00e+00 | 20260506 16:15:10.046 | INFO | PerfReport: 1.05%, LockDownload -- 1.02e+00 / 0.00e+00, 2.62e+00 / 0.00e+00 | 20260506 16:15:10.046 | INFO | PerfReport: 22.38%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 16:15:10.046 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260506 16:15:10.156 | INFO | Starting a round of Baz IO | 20260506 16:15:13.307 | INFO | Populating RTMetrics report | 20260506 16:15:13.307 | INFO | RTMetrics report populated | 20260506 16:15:13.307 | INFO | Opening tmp RTMetrics json file | 20260506 16:15:13.318 | INFO | Serializing RTMetrics to json | 20260506 16:15:13.345 | INFO | Serialization done, swapping tmp file now... | 20260506 16:15:13.347 | INFO | RTMetrics done | 20260506 16:15:13.864 | INFO | Finished a round of Baz IO: Wrote 2.099 GiB in 3.707 seconds | 20260506 16:15:13.864 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.49% / 0.01% / 63.50% | 20260506 16:15:13.864 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.77049% / 1.58946e-05% / 5.5631e-05% | 20260506 16:15:13.864 | INFO | Baz IO overall: | 20260506 16:15:13.864 | INFO | BytesWritten : 1421751364515 | 20260506 16:15:13.864 | INFO | NumEvents : 688001602838 | 20260506 16:15:15.183 | INFO | Analyzing chunk frames = [2711552,2712064) | 20260506 16:15:15.183 | INFO | MemInfo: { | 20260506 16:15:15.183 | INFO | "gpu" : " 17 GB", | 20260506 16:15:15.183 | INFO | "huge" : "", | 20260506 16:15:15.183 | INFO | "huge_shared" : " 27 GB", | 20260506 16:15:15.183 | INFO | "malloc" : " 14 GB", | 20260506 16:15:15.183 | INFO | "pinned" : " 29 GB" | 20260506 16:15:15.183 | INFO | } | 20260506 16:15:15.184 | INFO | Perf report for the last 5.13977 seconds: | 20260506 16:15:15.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:15:15.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:15.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:15.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:15.184 | INFO | ANALYSIS : 62.9%, 2.9, 95.8ms, 0 | 20260506 16:15:15.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:15.184 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 16:15:15.184 | INFO | PRE_HQ : 60.5%, 1.9, 58.3ms, 0 | 20260506 16:15:15.184 | INFO | BAZWRITER : 71.5%, 4.8, 176.6ms, 0 | 20260506 16:15:15.184 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 16:15:15.184 | INFO | Uploaded 36GiB to GPU in 2.0703s (17.3888GiB/s) | 20260506 16:15:15.184 | INFO | Downloaded 38GiB to GPU in 2.23585s (16.9958GiB/s) | 20260506 16:15:20.322 | INFO | Analyzing chunk frames = [2712064,2712576) | 20260506 16:15:20.322 | INFO | MemInfo: { | 20260506 16:15:20.322 | INFO | "gpu" : " 17 GB", | 20260506 16:15:20.322 | INFO | "huge" : "", | 20260506 16:15:20.322 | INFO | "huge_shared" : " 27 GB", | 20260506 16:15:20.322 | INFO | "malloc" : " 14 GB", | 20260506 16:15:20.322 | INFO | "pinned" : " 29 GB" | 20260506 16:15:20.322 | INFO | } | 20260506 16:15:20.322 | INFO | Perf report for the last 5.13868 seconds: | 20260506 16:15:20.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:15:20.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:20.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:20.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:20.322 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:15:20.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:20.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:20.322 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 16:15:20.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:20.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:20.323 | INFO | Uploaded 36GiB to GPU in 2.00364s (17.9673GiB/s) | 20260506 16:15:20.323 | INFO | Downloaded 34GiB to GPU in 1.93456s (17.575GiB/s) | 20260506 16:15:25.461 | INFO | Analyzing chunk frames = [2712576,2713088) | 20260506 16:15:25.461 | INFO | MemInfo: { | 20260506 16:15:25.461 | INFO | "gpu" : " 17 GB", | 20260506 16:15:25.461 | INFO | "huge" : "", | 20260506 16:15:25.461 | INFO | "huge_shared" : " 27 GB", | 20260506 16:15:25.461 | INFO | "malloc" : " 14 GB", | 20260506 16:15:25.461 | INFO | "pinned" : " 29 GB" | 20260506 16:15:25.461 | INFO | } | 20260506 16:15:25.461 | INFO | Perf report for the last 5.13892 seconds: | 20260506 16:15:25.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:15:25.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:25.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:25.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:25.461 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 16:15:25.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:25.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:25.461 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:15:25.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:25.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:25.461 | INFO | Uploaded 36GiB to GPU in 2.00086s (17.9922GiB/s) | 20260506 16:15:25.461 | INFO | Downloaded 34GiB to GPU in 1.93709s (17.5521GiB/s) | 20260506 16:15:30.600 | INFO | Analyzing chunk frames = [2713088,2713600) | 20260506 16:15:30.600 | INFO | MemInfo: { | 20260506 16:15:30.600 | INFO | "gpu" : " 17 GB", | 20260506 16:15:30.600 | INFO | "huge" : "", | 20260506 16:15:30.600 | INFO | "huge_shared" : " 27 GB", | 20260506 16:15:30.600 | INFO | "malloc" : " 14 GB", | 20260506 16:15:30.600 | INFO | "pinned" : " 29 GB" | 20260506 16:15:30.600 | INFO | } | 20260506 16:15:30.600 | INFO | Perf report for the last 5.13931 seconds: | 20260506 16:15:30.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:15:30.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:30.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:30.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:30.601 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 16:15:30.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:30.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:30.601 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:15:30.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:30.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:30.601 | INFO | Uploaded 36GiB to GPU in 1.97446s (18.2329GiB/s) | 20260506 16:15:30.601 | INFO | Downloaded 34GiB to GPU in 1.87977s (18.0873GiB/s) | 20260506 16:15:35.739 | INFO | Analyzing chunk frames = [2713600,2714112) | 20260506 16:15:35.739 | INFO | MemInfo: { | 20260506 16:15:35.739 | INFO | "gpu" : " 17 GB", | 20260506 16:15:35.739 | INFO | "huge" : "", | 20260506 16:15:35.739 | INFO | "huge_shared" : " 27 GB", | 20260506 16:15:35.739 | INFO | "malloc" : " 14 GB", | 20260506 16:15:35.740 | INFO | "pinned" : " 29 GB" | 20260506 16:15:35.740 | INFO | } | 20260506 16:15:35.740 | INFO | Perf report for the last 5.13911 seconds: | 20260506 16:15:35.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:15:35.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:35.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:35.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:35.740 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 16:15:35.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:35.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:35.740 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:15:35.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:35.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:35.740 | INFO | Uploaded 36GiB to GPU in 1.97274s (18.2487GiB/s) | 20260506 16:15:35.740 | INFO | Downloaded 34GiB to GPU in 1.92347s (17.6764GiB/s) | 20260506 16:15:40.879 | INFO | Analyzing chunk frames = [2714112,2714624) | 20260506 16:15:40.879 | INFO | MemInfo: { | 20260506 16:15:40.879 | INFO | "gpu" : " 17 GB", | 20260506 16:15:40.879 | INFO | "huge" : "", | 20260506 16:15:40.879 | INFO | "huge_shared" : " 27 GB", | 20260506 16:15:40.879 | INFO | "malloc" : " 14 GB", | 20260506 16:15:40.879 | INFO | "pinned" : " 29 GB" | 20260506 16:15:40.879 | INFO | } | 20260506 16:15:40.879 | INFO | Perf report for the last 5.13962 seconds: | 20260506 16:15:40.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:15:40.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:40.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:40.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:40.879 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:15:40.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:40.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:40.879 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:15:40.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:40.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:40.880 | INFO | Uploaded 36GiB to GPU in 2.06898s (17.3999GiB/s) | 20260506 16:15:40.880 | INFO | Downloaded 34GiB to GPU in 1.96903s (17.2673GiB/s) | 20260506 16:15:46.020 | INFO | Analyzing chunk frames = [2714624,2715136) | 20260506 16:15:46.020 | INFO | MemInfo: { | 20260506 16:15:46.020 | INFO | "gpu" : " 17 GB", | 20260506 16:15:46.020 | INFO | "huge" : "", | 20260506 16:15:46.020 | INFO | "huge_shared" : " 27 GB", | 20260506 16:15:46.020 | INFO | "malloc" : " 14 GB", | 20260506 16:15:46.020 | INFO | "pinned" : " 29 GB" | 20260506 16:15:46.020 | INFO | } | 20260506 16:15:46.020 | INFO | Perf report for the last 5.14096 seconds: | 20260506 16:15:46.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:15:46.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:46.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:46.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:46.020 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 16:15:46.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:46.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:46.020 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 16:15:46.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:46.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:46.020 | INFO | Uploaded 36GiB to GPU in 2.04552s (17.5994GiB/s) | 20260506 16:15:46.020 | INFO | Downloaded 34GiB to GPU in 2.00115s (16.9902GiB/s) | 20260506 16:15:51.159 | INFO | Analyzing chunk frames = [2715136,2715648) | 20260506 16:15:51.159 | INFO | MemInfo: { | 20260506 16:15:51.159 | INFO | "gpu" : " 17 GB", | 20260506 16:15:51.159 | INFO | "huge" : "", | 20260506 16:15:51.159 | INFO | "huge_shared" : " 27 GB", | 20260506 16:15:51.160 | INFO | "malloc" : " 14 GB", | 20260506 16:15:51.160 | INFO | "pinned" : " 29 GB" | 20260506 16:15:51.160 | INFO | } | 20260506 16:15:51.160 | INFO | Perf report for the last 5.13942 seconds: | 20260506 16:15:51.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:15:51.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:51.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:51.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:51.160 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:15:51.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:51.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:51.160 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:15:51.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:51.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:51.160 | INFO | Uploaded 36GiB to GPU in 2.00321s (17.9711GiB/s) | 20260506 16:15:51.160 | INFO | Downloaded 34GiB to GPU in 1.97653s (17.2018GiB/s) | 20260506 16:15:51.275 | INFO | Starting a round of Baz IO | 20260506 16:15:54.422 | INFO | Populating RTMetrics report | 20260506 16:15:54.423 | INFO | RTMetrics report populated | 20260506 16:15:54.423 | INFO | Opening tmp RTMetrics json file | 20260506 16:15:54.434 | INFO | Serializing RTMetrics to json | 20260506 16:15:54.458 | INFO | Serialization done, swapping tmp file now... | 20260506 16:15:54.461 | INFO | RTMetrics done | 20260506 16:15:54.600 | INFO | Finished a round of Baz IO: Wrote 2.096 GiB in 3.325 seconds | 20260506 16:15:54.600 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.48% / 0.01% / 63.51% | 20260506 16:15:54.600 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.76393% / 1.58946e-05% / 3.17891e-05% | 20260506 16:15:54.600 | INFO | Baz IO overall: | 20260506 16:15:54.600 | INFO | BytesWritten : 1424001700846 | 20260506 16:15:54.600 | INFO | NumEvents : 689089632389 | 20260506 16:15:56.297 | INFO | Analyzing chunk frames = [2715648,2716160) | 20260506 16:15:56.297 | INFO | MemInfo: { | 20260506 16:15:56.297 | INFO | "gpu" : " 17 GB", | 20260506 16:15:56.297 | INFO | "huge" : "", | 20260506 16:15:56.298 | INFO | "huge_shared" : " 27 GB", | 20260506 16:15:56.298 | INFO | "malloc" : " 14 GB", | 20260506 16:15:56.298 | INFO | "pinned" : " 29 GB" | 20260506 16:15:56.298 | INFO | } | 20260506 16:15:56.298 | INFO | Perf report for the last 5.138 seconds: | 20260506 16:15:56.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:15:56.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:56.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:56.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:56.298 | INFO | ANALYSIS : 62.9%, 2.9, 95.2ms, 0 | 20260506 16:15:56.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:15:56.298 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 16:15:56.298 | INFO | PRE_HQ : 60.6%, 1.9, 59.0ms, 0 | 20260506 16:15:56.298 | INFO | BAZWRITER : 64.4%, 4.8, 158.0ms, 0 | 20260506 16:15:56.298 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 16:15:56.298 | INFO | Uploaded 36GiB to GPU in 2.07544s (17.3457GiB/s) | 20260506 16:15:56.298 | INFO | Downloaded 38GiB to GPU in 2.21494s (17.1562GiB/s) | 20260506 16:16:01.436 | INFO | Analyzing chunk frames = [2716160,2716672) | 20260506 16:16:01.436 | INFO | MemInfo: { | 20260506 16:16:01.436 | INFO | "gpu" : " 17 GB", | 20260506 16:16:01.436 | INFO | "huge" : "", | 20260506 16:16:01.436 | INFO | "huge_shared" : " 27 GB", | 20260506 16:16:01.436 | INFO | "malloc" : " 14 GB", | 20260506 16:16:01.437 | INFO | "pinned" : " 29 GB" | 20260506 16:16:01.437 | INFO | } | 20260506 16:16:01.437 | INFO | Perf report for the last 5.13897 seconds: | 20260506 16:16:01.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:16:01.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:01.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:01.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:01.437 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 16:16:01.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:01.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:01.437 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:16:01.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:01.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:01.437 | INFO | Uploaded 36GiB to GPU in 2.00826s (17.9259GiB/s) | 20260506 16:16:01.437 | INFO | Downloaded 34GiB to GPU in 1.9408s (17.5185GiB/s) | 20260506 16:16:06.591 | INFO | Analyzing chunk frames = [2716672,2717184) | 20260506 16:16:06.591 | INFO | MemInfo: { | 20260506 16:16:06.591 | INFO | "gpu" : " 17 GB", | 20260506 16:16:06.591 | INFO | "huge" : "", | 20260506 16:16:06.591 | INFO | "huge_shared" : " 27 GB", | 20260506 16:16:06.591 | INFO | "malloc" : " 14 GB", | 20260506 16:16:06.591 | INFO | "pinned" : " 29 GB" | 20260506 16:16:06.591 | INFO | } | 20260506 16:16:06.591 | INFO | Perf report for the last 5.15465 seconds: | 20260506 16:16:06.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:16:06.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:06.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:06.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:06.592 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260506 16:16:06.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:06.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:06.592 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:16:06.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:06.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:06.592 | INFO | Uploaded 36GiB to GPU in 1.99863s (18.0123GiB/s) | 20260506 16:16:06.592 | INFO | Downloaded 34GiB to GPU in 1.94231s (17.5049GiB/s) | 20260506 16:16:11.727 | INFO | Analyzing chunk frames = [2717184,2717696) | 20260506 16:16:11.727 | INFO | MemInfo: { | 20260506 16:16:11.727 | INFO | "gpu" : " 17 GB", | 20260506 16:16:11.727 | INFO | "huge" : "", | 20260506 16:16:11.727 | INFO | "huge_shared" : " 27 GB", | 20260506 16:16:11.727 | INFO | "malloc" : " 14 GB", | 20260506 16:16:11.727 | INFO | "pinned" : " 29 GB" | 20260506 16:16:11.727 | INFO | } | 20260506 16:16:11.727 | INFO | Perf report for the last 5.13575 seconds: | 20260506 16:16:11.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:16:11.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:11.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:11.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:11.727 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 16:16:11.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:11.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:11.727 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:16:11.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:11.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:11.727 | INFO | Uploaded 36GiB to GPU in 1.9287s (18.6654GiB/s) | 20260506 16:16:11.727 | INFO | Downloaded 34GiB to GPU in 1.87176s (18.1647GiB/s) | 20260506 16:16:16.856 | INFO | Analyzing chunk frames = [2717696,2718208) | 20260506 16:16:16.856 | INFO | MemInfo: { | 20260506 16:16:16.856 | INFO | "gpu" : " 17 GB", | 20260506 16:16:16.856 | INFO | "huge" : "", | 20260506 16:16:16.856 | INFO | "huge_shared" : " 27 GB", | 20260506 16:16:16.856 | INFO | "malloc" : " 14 GB", | 20260506 16:16:16.856 | INFO | "pinned" : " 29 GB" | 20260506 16:16:16.856 | INFO | } | 20260506 16:16:16.856 | INFO | Perf report for the last 5.12921 seconds: | 20260506 16:16:16.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:16:16.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:16.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:16.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:16.856 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 16:16:16.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:16.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:16.856 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 16:16:16.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:16.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:16.857 | INFO | Uploaded 36GiB to GPU in 2.0304s (17.7305GiB/s) | 20260506 16:16:16.857 | INFO | Downloaded 34GiB to GPU in 1.9424s (17.5041GiB/s) | 20260506 16:16:21.994 | INFO | Analyzing chunk frames = [2718208,2718720) | 20260506 16:16:21.994 | INFO | MemInfo: { | 20260506 16:16:21.994 | INFO | "gpu" : " 17 GB", | 20260506 16:16:21.994 | INFO | "huge" : "", | 20260506 16:16:21.994 | INFO | "huge_shared" : " 27 GB", | 20260506 16:16:21.994 | INFO | "malloc" : " 14 GB", | 20260506 16:16:21.994 | INFO | "pinned" : " 29 GB" | 20260506 16:16:21.994 | INFO | } | 20260506 16:16:21.994 | INFO | Perf report for the last 5.13757 seconds: | 20260506 16:16:21.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:16:21.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:21.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:21.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:21.994 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:16:21.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:21.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:21.994 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 16:16:21.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:21.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:21.994 | INFO | Uploaded 36GiB to GPU in 2.03362s (17.7024GiB/s) | 20260506 16:16:21.994 | INFO | Downloaded 34GiB to GPU in 1.92254s (17.685GiB/s) | 20260506 16:16:27.134 | INFO | Analyzing chunk frames = [2718720,2719232) | 20260506 16:16:27.134 | INFO | MemInfo: { | 20260506 16:16:27.134 | INFO | "gpu" : " 17 GB", | 20260506 16:16:27.134 | INFO | "huge" : "", | 20260506 16:16:27.134 | INFO | "huge_shared" : " 27 GB", | 20260506 16:16:27.134 | INFO | "malloc" : " 14 GB", | 20260506 16:16:27.134 | INFO | "pinned" : " 29 GB" | 20260506 16:16:27.134 | INFO | } | 20260506 16:16:27.134 | INFO | Perf report for the last 5.14039 seconds: | 20260506 16:16:27.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:16:27.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:27.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:27.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:27.134 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:16:27.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:27.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:27.134 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:16:27.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:27.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:27.135 | INFO | Uploaded 36GiB to GPU in 1.99347s (18.059GiB/s) | 20260506 16:16:27.135 | INFO | Downloaded 34GiB to GPU in 1.95801s (17.3646GiB/s) | 20260506 16:16:32.273 | INFO | Analyzing chunk frames = [2719232,2719744) | 20260506 16:16:32.273 | INFO | MemInfo: { | 20260506 16:16:32.273 | INFO | "gpu" : " 17 GB", | 20260506 16:16:32.273 | INFO | "huge" : "", | 20260506 16:16:32.273 | INFO | "huge_shared" : " 27 GB", | 20260506 16:16:32.273 | INFO | "malloc" : " 14 GB", | 20260506 16:16:32.273 | INFO | "pinned" : " 29 GB" | 20260506 16:16:32.273 | INFO | } | 20260506 16:16:32.273 | INFO | Perf report for the last 5.13908 seconds: | 20260506 16:16:32.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:16:32.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:32.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:32.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:32.273 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:16:32.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:32.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:32.273 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 16:16:32.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:32.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:32.274 | INFO | Uploaded 36GiB to GPU in 2.03246s (17.7126GiB/s) | 20260506 16:16:32.274 | INFO | Downloaded 34GiB to GPU in 1.98893s (17.0946GiB/s) | 20260506 16:16:32.274 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:16:32.274 | INFO | Total mem usage: 18 GiB | 20260506 16:16:32.274 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:16:32.274 | INFO | Recent High Water Marks Sum: 16.9 GiB (Savings: -6.27%) | 20260506 16:16:32.274 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:16:32.274 | INFO | High Water Breakdown: | 20260506 16:16:32.274 | INFO | 62.6% : 10.6 GiB ( 92.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:16:32.274 | INFO | 37.3% : 6.32 GiB ( 94.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:16:32.274 | INFO | 0.117% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:16:32.274 | INFO | ------------------------------------------ | 20260506 16:16:32.274 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:16:32.274 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:16:32.274 | INFO | Total mem usage: 26.5 GiB | 20260506 16:16:32.274 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:16:32.274 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:16:32.274 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:16:32.274 | INFO | High Water Breakdown: | 20260506 16:16:32.274 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:16:32.274 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:16:32.274 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:16:32.274 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:16:32.274 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:16:32.274 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:16:32.274 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:16:32.274 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:16:32.274 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:16:32.274 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:16:32.274 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:16:32.274 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:16:32.274 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:16:32.274 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:16:32.274 | INFO | ------------------------------------------ | 20260506 16:16:32.274 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:16:32.274 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:16:32.274 | INFO | Total mem usage: 25.6 GiB | 20260506 16:16:32.274 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:16:32.274 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:16:32.274 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:16:32.274 | INFO | High Water Breakdown: | 20260506 16:16:32.274 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:16:32.274 | INFO | ------------------------------------------ | 20260506 16:16:32.274 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:16:32.274 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:16:32.274 | INFO | Total mem usage: 15.7 GiB | 20260506 16:16:32.274 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:16:32.275 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:16:32.275 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 16:16:32.275 | INFO | High Water Breakdown: | 20260506 16:16:32.275 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:16:32.275 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:16:32.275 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:16:32.275 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:16:32.275 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:16:32.275 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:16:32.275 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:16:32.275 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:16:32.275 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:16:32.275 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:16:32.275 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:16:32.275 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:16:32.275 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:16:32.275 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:16:32.275 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:16:32.275 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:16:32.275 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:16:32.275 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:16:32.275 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:16:32.275 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:16:32.275 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:16:32.275 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:16:32.275 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:16:32.275 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:16:32.275 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:16:32.275 | INFO | ------------------------------------------ | 20260506 16:16:32.275 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:16:32.275 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:16:32.275 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:16:32.275 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:16:32.275 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.48e+01, 1.34e+01 / 1.61e+00 | 20260506 16:16:32.275 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.01e-01, 1.30e-02 / 6.51e-03 | 20260506 16:16:32.275 | INFO | PerfReport: 11.59%, Baseline -- 3.63e+00 / 4.37e+00, 5.55e-01 / 3.80e-01 | 20260506 16:16:32.275 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.84e-01 / 1.36e-01 | 20260506 16:16:32.275 | INFO | PerfReport: 15.86%, DME -- 3.98e+01 / 4.39e+01, 3.63e+00 / 1.84e+00 | 20260506 16:16:32.275 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.78e+01 / 2.01e+01, 1.44e+00 / 1.55e+00 | 20260506 16:16:32.275 | INFO | PerfReport: 7.37%, PulseAccumulating -- 2.31e+00 / 2.94e+00, 3.99e-01 / 1.94e-01 | 20260506 16:16:32.275 | INFO | PerfReport: 3.52%, Metrics -- 1.10e+00 / 1.46e+00, 2.84e-01 / 1.76e-01 | 20260506 16:16:32.275 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.84e-01 / 4.38e-01, 6.05e-02 / 1.85e-02 | 20260506 16:16:32.275 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:16:32.275 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:16:32.275 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:16:32.275 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 16:16:32.275 | INFO | PerfReport: 21.56%, Upload -- 2.10e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260506 16:16:32.275 | INFO | PerfReport: 21.16%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 16:16:32.275 | INFO | PerfReport: 32.23%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 16:16:32.275 | INFO | PerfReport: 1.50%, LockUpload -- 1.46e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260506 16:16:32.275 | INFO | PerfReport: 1.06%, LockDownload -- 1.03e+00 / 0.00e+00, 2.62e+00 / 0.00e+00 | 20260506 16:16:32.275 | INFO | PerfReport: 22.47%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 16:16:32.275 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260506 16:16:32.393 | INFO | Starting a round of Baz IO | 20260506 16:16:35.535 | INFO | Populating RTMetrics report | 20260506 16:16:35.535 | INFO | RTMetrics report populated | 20260506 16:16:35.535 | INFO | Opening tmp RTMetrics json file | 20260506 16:16:35.616 | INFO | Serializing RTMetrics to json | 20260506 16:16:35.638 | INFO | Serialization done, swapping tmp file now... | 20260506 16:16:35.639 | INFO | RTMetrics done | 20260506 16:16:35.954 | INFO | Finished a round of Baz IO: Wrote 2.095 GiB in 3.56 seconds | 20260506 16:16:35.954 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.48% / 0.01% / 63.52% | 20260506 16:16:35.955 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.75666% / 2.78155e-05% / 4.76837e-05% | 20260506 16:16:35.955 | INFO | Baz IO overall: | 20260506 16:16:35.955 | INFO | BytesWritten : 1426251319626 | 20260506 16:16:35.955 | INFO | NumEvents : 690177516276 | 20260506 16:16:37.413 | INFO | Analyzing chunk frames = [2719744,2720256) | 20260506 16:16:37.413 | INFO | MemInfo: { | 20260506 16:16:37.413 | INFO | "gpu" : " 17 GB", | 20260506 16:16:37.413 | INFO | "huge" : "", | 20260506 16:16:37.413 | INFO | "huge_shared" : " 27 GB", | 20260506 16:16:37.413 | INFO | "malloc" : " 14 GB", | 20260506 16:16:37.413 | INFO | "pinned" : " 29 GB" | 20260506 16:16:37.413 | INFO | } | 20260506 16:16:37.413 | INFO | Perf report for the last 5.13967 seconds: | 20260506 16:16:37.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:16:37.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:37.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:37.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:37.413 | INFO | ANALYSIS : 62.8%, 2.9, 95.3ms, 0 | 20260506 16:16:37.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:37.413 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 16:16:37.413 | INFO | PRE_HQ : 60.5%, 1.9, 58.7ms, 0 | 20260506 16:16:37.413 | INFO | BAZWRITER : 68.8%, 4.8, 170.1ms, 0 | 20260506 16:16:37.413 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 16:16:37.413 | INFO | Uploaded 36GiB to GPU in 2.07302s (17.3659GiB/s) | 20260506 16:16:37.413 | INFO | Downloaded 38GiB to GPU in 2.23179s (17.0267GiB/s) | 20260506 16:16:42.551 | INFO | Analyzing chunk frames = [2720256,2720768) | 20260506 16:16:42.551 | INFO | MemInfo: { | 20260506 16:16:42.551 | INFO | "gpu" : " 17 GB", | 20260506 16:16:42.551 | INFO | "huge" : "", | 20260506 16:16:42.551 | INFO | "huge_shared" : " 27 GB", | 20260506 16:16:42.551 | INFO | "malloc" : " 14 GB", | 20260506 16:16:42.551 | INFO | "pinned" : " 29 GB" | 20260506 16:16:42.551 | INFO | } | 20260506 16:16:42.552 | INFO | Perf report for the last 5.13849 seconds: | 20260506 16:16:42.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:16:42.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:42.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:42.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:42.552 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 16:16:42.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:42.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:42.552 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:16:42.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:42.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:42.552 | INFO | Uploaded 36GiB to GPU in 1.99547s (18.0409GiB/s) | 20260506 16:16:42.552 | INFO | Downloaded 34GiB to GPU in 1.92855s (17.6299GiB/s) | 20260506 16:16:47.691 | INFO | Analyzing chunk frames = [2720768,2721280) | 20260506 16:16:47.691 | INFO | MemInfo: { | 20260506 16:16:47.691 | INFO | "gpu" : " 17 GB", | 20260506 16:16:47.691 | INFO | "huge" : "", | 20260506 16:16:47.691 | INFO | "huge_shared" : " 27 GB", | 20260506 16:16:47.691 | INFO | "malloc" : " 14 GB", | 20260506 16:16:47.691 | INFO | "pinned" : " 29 GB" | 20260506 16:16:47.691 | INFO | } | 20260506 16:16:47.691 | INFO | Perf report for the last 5.13949 seconds: | 20260506 16:16:47.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:16:47.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:47.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:47.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:47.691 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 16:16:47.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:47.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:47.691 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:16:47.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:47.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:47.691 | INFO | Uploaded 36GiB to GPU in 1.98141s (18.1689GiB/s) | 20260506 16:16:47.691 | INFO | Downloaded 34GiB to GPU in 1.93567s (17.565GiB/s) | 20260506 16:16:52.830 | INFO | Analyzing chunk frames = [2721280,2721792) | 20260506 16:16:52.831 | INFO | MemInfo: { | 20260506 16:16:52.831 | INFO | "gpu" : " 17 GB", | 20260506 16:16:52.831 | INFO | "huge" : "", | 20260506 16:16:52.831 | INFO | "huge_shared" : " 27 GB", | 20260506 16:16:52.831 | INFO | "malloc" : " 14 GB", | 20260506 16:16:52.831 | INFO | "pinned" : " 29 GB" | 20260506 16:16:52.831 | INFO | } | 20260506 16:16:52.831 | INFO | Perf report for the last 5.13977 seconds: | 20260506 16:16:52.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:16:52.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:52.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:52.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:52.831 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:16:52.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:52.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:52.831 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:16:52.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:52.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:52.831 | INFO | Uploaded 36GiB to GPU in 1.9681s (18.2918GiB/s) | 20260506 16:16:52.831 | INFO | Downloaded 34GiB to GPU in 1.87769s (18.1074GiB/s) | 20260506 16:16:57.970 | INFO | Analyzing chunk frames = [2721792,2722304) | 20260506 16:16:57.970 | INFO | MemInfo: { | 20260506 16:16:57.970 | INFO | "gpu" : " 17 GB", | 20260506 16:16:57.970 | INFO | "huge" : "", | 20260506 16:16:57.970 | INFO | "huge_shared" : " 27 GB", | 20260506 16:16:57.970 | INFO | "malloc" : " 14 GB", | 20260506 16:16:57.970 | INFO | "pinned" : " 29 GB" | 20260506 16:16:57.970 | INFO | } | 20260506 16:16:57.970 | INFO | Perf report for the last 5.13949 seconds: | 20260506 16:16:57.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:16:57.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:57.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:57.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:57.970 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 16:16:57.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:57.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:57.970 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:16:57.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:57.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:16:57.971 | INFO | Uploaded 36GiB to GPU in 1.99684s (18.0285GiB/s) | 20260506 16:16:57.971 | INFO | Downloaded 34GiB to GPU in 1.92155s (17.6941GiB/s) | 20260506 16:17:03.109 | INFO | Analyzing chunk frames = [2722304,2722816) | 20260506 16:17:03.109 | INFO | MemInfo: { | 20260506 16:17:03.109 | INFO | "gpu" : " 17 GB", | 20260506 16:17:03.109 | INFO | "huge" : "", | 20260506 16:17:03.109 | INFO | "huge_shared" : " 27 GB", | 20260506 16:17:03.109 | INFO | "malloc" : " 14 GB", | 20260506 16:17:03.109 | INFO | "pinned" : " 29 GB" | 20260506 16:17:03.109 | INFO | } | 20260506 16:17:03.109 | INFO | Perf report for the last 5.13913 seconds: | 20260506 16:17:03.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:17:03.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:03.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:03.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:03.110 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 16:17:03.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:03.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:03.110 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 16:17:03.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:03.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:03.110 | INFO | Uploaded 36GiB to GPU in 2.03886s (17.6569GiB/s) | 20260506 16:17:03.110 | INFO | Downloaded 34GiB to GPU in 1.92374s (17.6739GiB/s) | 20260506 16:17:08.250 | INFO | Analyzing chunk frames = [2722816,2723328) | 20260506 16:17:08.250 | INFO | MemInfo: { | 20260506 16:17:08.250 | INFO | "gpu" : " 17 GB", | 20260506 16:17:08.250 | INFO | "huge" : "", | 20260506 16:17:08.250 | INFO | "huge_shared" : " 27 GB", | 20260506 16:17:08.251 | INFO | "malloc" : " 14 GB", | 20260506 16:17:08.251 | INFO | "pinned" : " 29 GB" | 20260506 16:17:08.251 | INFO | } | 20260506 16:17:08.251 | INFO | Perf report for the last 5.14124 seconds: | 20260506 16:17:08.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:17:08.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:08.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:08.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:08.251 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:17:08.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:08.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:08.251 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 16:17:08.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:08.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:08.251 | INFO | Uploaded 36GiB to GPU in 1.99672s (18.0296GiB/s) | 20260506 16:17:08.251 | INFO | Downloaded 34GiB to GPU in 1.9895s (17.0898GiB/s) | 20260506 16:17:13.388 | INFO | Analyzing chunk frames = [2723328,2723840) | 20260506 16:17:13.388 | INFO | MemInfo: { | 20260506 16:17:13.388 | INFO | "gpu" : " 17 GB", | 20260506 16:17:13.388 | INFO | "huge" : "", | 20260506 16:17:13.388 | INFO | "huge_shared" : " 27 GB", | 20260506 16:17:13.388 | INFO | "malloc" : " 14 GB", | 20260506 16:17:13.388 | INFO | "pinned" : " 29 GB" | 20260506 16:17:13.388 | INFO | } | 20260506 16:17:13.388 | INFO | Perf report for the last 5.13763 seconds: | 20260506 16:17:13.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:17:13.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:13.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:13.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:13.388 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:17:13.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:13.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:13.388 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:17:13.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:13.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:13.389 | INFO | Uploaded 36GiB to GPU in 2.01526s (17.8637GiB/s) | 20260506 16:17:13.389 | INFO | Downloaded 34GiB to GPU in 1.99129s (17.0744GiB/s) | 20260506 16:17:13.509 | INFO | Starting a round of Baz IO | 20260506 16:17:16.649 | INFO | Populating RTMetrics report | 20260506 16:17:16.650 | INFO | RTMetrics report populated | 20260506 16:17:16.650 | INFO | Opening tmp RTMetrics json file | 20260506 16:17:16.650 | INFO | Serializing RTMetrics to json | 20260506 16:17:16.652 | INFO | Serialization done, swapping tmp file now... | 20260506 16:17:16.652 | INFO | RTMetrics done | 20260506 16:17:17.007 | INFO | Finished a round of Baz IO: Wrote 2.092 GiB in 3.498 seconds | 20260506 16:17:17.007 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.47% / 0.01% / 63.52% | 20260506 16:17:17.007 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.74897% / 1.58946e-05% / 6.35783e-05% | 20260506 16:17:17.008 | INFO | Baz IO overall: | 20260506 16:17:17.008 | INFO | BytesWritten : 1428497671902 | 20260506 16:17:17.008 | INFO | NumEvents : 691263296179 | 20260506 16:17:18.527 | INFO | Analyzing chunk frames = [2723840,2724352) | 20260506 16:17:18.527 | INFO | MemInfo: { | 20260506 16:17:18.527 | INFO | "gpu" : " 17 GB", | 20260506 16:17:18.527 | INFO | "huge" : "", | 20260506 16:17:18.527 | INFO | "huge_shared" : " 27 GB", | 20260506 16:17:18.527 | INFO | "malloc" : " 14 GB", | 20260506 16:17:18.527 | INFO | "pinned" : " 29 GB" | 20260506 16:17:18.527 | INFO | } | 20260506 16:17:18.527 | INFO | Perf report for the last 5.13898 seconds: | 20260506 16:17:18.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:17:18.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:18.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:18.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:18.527 | INFO | ANALYSIS : 62.8%, 2.9, 95.2ms, 0 | 20260506 16:17:18.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:18.527 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 16:17:18.527 | INFO | PRE_HQ : 60.5%, 1.9, 58.8ms, 0 | 20260506 16:17:18.528 | INFO | BAZWRITER : 67.6%, 4.8, 166.0ms, 0 | 20260506 16:17:18.528 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 16:17:18.528 | INFO | Uploaded 36GiB to GPU in 2.07821s (17.3226GiB/s) | 20260506 16:17:18.528 | INFO | Downloaded 38GiB to GPU in 2.23874s (16.9738GiB/s) | 20260506 16:17:23.666 | INFO | Analyzing chunk frames = [2724352,2724864) | 20260506 16:17:23.666 | INFO | MemInfo: { | 20260506 16:17:23.666 | INFO | "gpu" : " 17 GB", | 20260506 16:17:23.666 | INFO | "huge" : "", | 20260506 16:17:23.666 | INFO | "huge_shared" : " 27 GB", | 20260506 16:17:23.666 | INFO | "malloc" : " 14 GB", | 20260506 16:17:23.666 | INFO | "pinned" : " 29 GB" | 20260506 16:17:23.666 | INFO | } | 20260506 16:17:23.666 | INFO | Perf report for the last 5.13897 seconds: | 20260506 16:17:23.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:17:23.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:23.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:23.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:23.666 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 16:17:23.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:23.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:23.666 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:17:23.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:23.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:23.667 | INFO | Uploaded 36GiB to GPU in 1.99416s (18.0527GiB/s) | 20260506 16:17:23.667 | INFO | Downloaded 34GiB to GPU in 1.92691s (17.6448GiB/s) | 20260506 16:17:28.805 | INFO | Analyzing chunk frames = [2724864,2725376) | 20260506 16:17:28.805 | INFO | MemInfo: { | 20260506 16:17:28.805 | INFO | "gpu" : " 17 GB", | 20260506 16:17:28.805 | INFO | "huge" : "", | 20260506 16:17:28.805 | INFO | "huge_shared" : " 27 GB", | 20260506 16:17:28.805 | INFO | "malloc" : " 14 GB", | 20260506 16:17:28.805 | INFO | "pinned" : " 29 GB" | 20260506 16:17:28.805 | INFO | } | 20260506 16:17:28.805 | INFO | Perf report for the last 5.13887 seconds: | 20260506 16:17:28.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:17:28.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:28.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:28.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:28.805 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 16:17:28.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:28.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:28.805 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:17:28.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:28.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:28.806 | INFO | Uploaded 36GiB to GPU in 1.95915s (18.3753GiB/s) | 20260506 16:17:28.806 | INFO | Downloaded 34GiB to GPU in 1.92615s (17.6518GiB/s) | 20260506 16:17:33.946 | INFO | Analyzing chunk frames = [2725376,2725888) | 20260506 16:17:33.946 | INFO | MemInfo: { | 20260506 16:17:33.946 | INFO | "gpu" : " 17 GB", | 20260506 16:17:33.946 | INFO | "huge" : "", | 20260506 16:17:33.946 | INFO | "huge_shared" : " 27 GB", | 20260506 16:17:33.946 | INFO | "malloc" : " 14 GB", | 20260506 16:17:33.946 | INFO | "pinned" : " 29 GB" | 20260506 16:17:33.946 | INFO | } | 20260506 16:17:33.946 | INFO | Perf report for the last 5.14069 seconds: | 20260506 16:17:33.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:17:33.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:33.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:33.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:33.946 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260506 16:17:33.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:33.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:33.946 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:17:33.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:33.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:33.946 | INFO | Uploaded 36GiB to GPU in 2.01908s (17.8299GiB/s) | 20260506 16:17:33.946 | INFO | Downloaded 34GiB to GPU in 1.90828s (17.8171GiB/s) | 20260506 16:17:39.105 | INFO | Analyzing chunk frames = [2725888,2726400) | 20260506 16:17:39.105 | INFO | MemInfo: { | 20260506 16:17:39.105 | INFO | "gpu" : " 17 GB", | 20260506 16:17:39.105 | INFO | "huge" : "", | 20260506 16:17:39.105 | INFO | "huge_shared" : " 27 GB", | 20260506 16:17:39.105 | INFO | "malloc" : " 14 GB", | 20260506 16:17:39.105 | INFO | "pinned" : " 29 GB" | 20260506 16:17:39.105 | INFO | } | 20260506 16:17:39.105 | INFO | Perf report for the last 5.15905 seconds: | 20260506 16:17:39.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:17:39.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:39.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:39.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:39.105 | INFO | ANALYSIS : 60.3%, 3.0, 94.3ms, 0 | 20260506 16:17:39.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:39.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:39.105 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260506 16:17:39.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:39.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:39.105 | INFO | Uploaded 36GiB to GPU in 1.96703s (18.3017GiB/s) | 20260506 16:17:39.105 | INFO | Downloaded 34GiB to GPU in 1.92903s (17.6254GiB/s) | 20260506 16:17:44.241 | INFO | Analyzing chunk frames = [2726400,2726912) | 20260506 16:17:44.241 | INFO | MemInfo: { | 20260506 16:17:44.241 | INFO | "gpu" : " 17 GB", | 20260506 16:17:44.241 | INFO | "huge" : "", | 20260506 16:17:44.241 | INFO | "huge_shared" : " 27 GB", | 20260506 16:17:44.241 | INFO | "malloc" : " 14 GB", | 20260506 16:17:44.241 | INFO | "pinned" : " 29 GB" | 20260506 16:17:44.241 | INFO | } | 20260506 16:17:44.241 | INFO | Perf report for the last 5.13635 seconds: | 20260506 16:17:44.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:17:44.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:44.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:44.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:44.241 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 16:17:44.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:44.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:44.241 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 16:17:44.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:44.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:44.242 | INFO | Uploaded 36GiB to GPU in 1.97242s (18.2517GiB/s) | 20260506 16:17:44.242 | INFO | Downloaded 34GiB to GPU in 1.93616s (17.5605GiB/s) | 20260506 16:17:49.383 | INFO | Analyzing chunk frames = [2726912,2727424) | 20260506 16:17:49.383 | INFO | MemInfo: { | 20260506 16:17:49.383 | INFO | "gpu" : " 17 GB", | 20260506 16:17:49.383 | INFO | "huge" : "", | 20260506 16:17:49.383 | INFO | "huge_shared" : " 27 GB", | 20260506 16:17:49.383 | INFO | "malloc" : " 14 GB", | 20260506 16:17:49.383 | INFO | "pinned" : " 29 GB" | 20260506 16:17:49.383 | INFO | } | 20260506 16:17:49.383 | INFO | Perf report for the last 5.14146 seconds: | 20260506 16:17:49.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:17:49.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:49.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:49.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:49.383 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 16:17:49.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:49.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:49.383 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 16:17:49.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:49.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:49.383 | INFO | Uploaded 36GiB to GPU in 2.03453s (17.6945GiB/s) | 20260506 16:17:49.383 | INFO | Downloaded 34GiB to GPU in 2.00172s (16.9854GiB/s) | 20260506 16:17:54.518 | INFO | Analyzing chunk frames = [2727424,2727936) | 20260506 16:17:54.518 | INFO | MemInfo: { | 20260506 16:17:54.518 | INFO | "gpu" : " 17 GB", | 20260506 16:17:54.518 | INFO | "huge" : "", | 20260506 16:17:54.518 | INFO | "huge_shared" : " 27 GB", | 20260506 16:17:54.518 | INFO | "malloc" : " 14 GB", | 20260506 16:17:54.518 | INFO | "pinned" : " 29 GB" | 20260506 16:17:54.518 | INFO | } | 20260506 16:17:54.519 | INFO | Perf report for the last 5.13567 seconds: | 20260506 16:17:54.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:17:54.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:54.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:54.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:54.519 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 16:17:54.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:54.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:54.519 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:17:54.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:54.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:54.519 | INFO | Uploaded 36GiB to GPU in 1.99319s (18.0615GiB/s) | 20260506 16:17:54.519 | INFO | Downloaded 34GiB to GPU in 1.96114s (17.3368GiB/s) | 20260506 16:17:54.519 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:17:54.519 | INFO | Total mem usage: 18 GiB | 20260506 16:17:54.519 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:17:54.519 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.9%) | 20260506 16:17:54.519 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:17:54.519 | INFO | High Water Breakdown: | 20260506 16:17:54.519 | INFO | 61.7% : 9.84 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:17:54.519 | INFO | 38.2% : 6.1 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:17:54.519 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:17:54.519 | INFO | ------------------------------------------ | 20260506 16:17:54.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:17:54.519 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:17:54.519 | INFO | Total mem usage: 26.5 GiB | 20260506 16:17:54.519 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:17:54.519 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:17:54.519 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:17:54.519 | INFO | High Water Breakdown: | 20260506 16:17:54.519 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:17:54.519 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:17:54.519 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:17:54.519 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:17:54.519 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:17:54.519 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:17:54.519 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:17:54.519 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:17:54.519 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:17:54.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:17:54.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:17:54.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:17:54.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:17:54.519 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:17:54.519 | INFO | ------------------------------------------ | 20260506 16:17:54.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:17:54.519 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:17:54.519 | INFO | Total mem usage: 25.6 GiB | 20260506 16:17:54.519 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:17:54.519 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:17:54.519 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:17:54.519 | INFO | High Water Breakdown: | 20260506 16:17:54.519 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:17:54.519 | INFO | ------------------------------------------ | 20260506 16:17:54.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:17:54.519 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:17:54.519 | INFO | Total mem usage: 15.7 GiB | 20260506 16:17:54.519 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:17:54.519 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:17:54.519 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 16:17:54.520 | INFO | High Water Breakdown: | 20260506 16:17:54.520 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:17:54.520 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:17:54.520 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:17:54.520 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:17:54.520 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:17:54.520 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:17:54.520 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:17:54.520 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:17:54.520 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:17:54.520 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:17:54.520 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:17:54.520 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:17:54.520 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:17:54.520 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:17:54.520 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:17:54.520 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:17:54.520 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:17:54.520 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:17:54.520 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:17:54.520 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:17:54.520 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:17:54.520 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:17:54.520 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:17:54.520 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:17:54.520 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:17:54.520 | INFO | ------------------------------------------ | 20260506 16:17:54.520 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:17:54.520 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:17:54.520 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:17:54.520 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:17:54.520 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.50e+01, 1.33e+01 / 2.05e+00 | 20260506 16:17:54.520 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.00e-01, 1.06e-02 / 7.04e-03 | 20260506 16:17:54.520 | INFO | PerfReport: 11.60%, Baseline -- 3.64e+00 / 4.35e+00, 5.69e-01 / 4.49e-01 | 20260506 16:17:54.520 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.88e-01 / 1.45e-01 | 20260506 16:17:54.520 | INFO | PerfReport: 15.79%, DME -- 3.96e+01 / 4.31e+01, 3.48e+00 / 1.90e+00 | 20260506 16:17:54.520 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.79e+01 / 2.12e+01, 1.48e+00 / 2.13e+00 | 20260506 16:17:54.520 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.30e+00 / 2.88e+00, 3.92e-01 / 2.47e-01 | 20260506 16:17:54.520 | INFO | PerfReport: 3.52%, Metrics -- 1.10e+00 / 1.47e+00, 2.88e-01 / 1.83e-01 | 20260506 16:17:54.520 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.83e-01 / 4.35e-01, 6.03e-02 / 2.55e-02 | 20260506 16:17:54.520 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:17:54.520 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:17:54.520 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:17:54.520 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 16:17:54.520 | INFO | PerfReport: 21.49%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 16:17:54.520 | INFO | PerfReport: 21.18%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 16:17:54.520 | INFO | PerfReport: 32.30%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 16:17:54.520 | INFO | PerfReport: 1.37%, LockUpload -- 1.33e+00 / 0.00e+00, 5.42e+00 / 0.00e+00 | 20260506 16:17:54.520 | INFO | PerfReport: 1.14%, LockDownload -- 1.11e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 16:17:54.520 | INFO | PerfReport: 22.50%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 16:17:54.520 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260506 16:17:54.639 | INFO | Starting a round of Baz IO | 20260506 16:17:57.775 | INFO | Populating RTMetrics report | 20260506 16:17:57.775 | INFO | RTMetrics report populated | 20260506 16:17:57.775 | INFO | Opening tmp RTMetrics json file | 20260506 16:17:57.781 | INFO | Serializing RTMetrics to json | 20260506 16:17:57.787 | INFO | Serialization done, swapping tmp file now... | 20260506 16:17:57.791 | INFO | RTMetrics done | 20260506 16:17:58.429 | INFO | Finished a round of Baz IO: Wrote 2.093 GiB in 3.79 seconds | 20260506 16:17:58.429 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.46% / 0.01% / 63.53% | 20260506 16:17:58.429 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.74359% / 1.19209e-05% / 5.5631e-05% | 20260506 16:17:58.429 | INFO | Baz IO overall: | 20260506 16:17:58.429 | INFO | BytesWritten : 1430745042218 | 20260506 16:17:58.429 | INFO | NumEvents : 692349962760 | 20260506 16:17:59.658 | INFO | Analyzing chunk frames = [2727936,2728448) | 20260506 16:17:59.658 | INFO | MemInfo: { | 20260506 16:17:59.658 | INFO | "gpu" : " 17 GB", | 20260506 16:17:59.658 | INFO | "huge" : "", | 20260506 16:17:59.658 | INFO | "huge_shared" : " 27 GB", | 20260506 16:17:59.658 | INFO | "malloc" : " 14 GB", | 20260506 16:17:59.658 | INFO | "pinned" : " 29 GB" | 20260506 16:17:59.658 | INFO | } | 20260506 16:17:59.658 | INFO | Perf report for the last 5.13973 seconds: | 20260506 16:17:59.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:17:59.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:59.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:59.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:59.658 | INFO | ANALYSIS : 62.8%, 2.9, 95.3ms, 0 | 20260506 16:17:59.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:17:59.658 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 16:17:59.658 | INFO | PRE_HQ : 60.4%, 1.9, 58.4ms, 0 | 20260506 16:17:59.658 | INFO | BAZWRITER : 73.4%, 4.8, 181.3ms, 0 | 20260506 16:17:59.659 | INFO | BAZ_SAMPLING : 4.6%, 1.0, 2.4ms, 0 | 20260506 16:17:59.659 | INFO | Uploaded 36GiB to GPU in 2.05438s (17.5236GiB/s) | 20260506 16:17:59.659 | INFO | Downloaded 38GiB to GPU in 2.21446s (17.1599GiB/s) | 20260506 16:18:04.793 | INFO | Analyzing chunk frames = [2728448,2728960) | 20260506 16:18:04.793 | INFO | MemInfo: { | 20260506 16:18:04.793 | INFO | "gpu" : " 17 GB", | 20260506 16:18:04.793 | INFO | "huge" : "", | 20260506 16:18:04.793 | INFO | "huge_shared" : " 27 GB", | 20260506 16:18:04.793 | INFO | "malloc" : " 14 GB", | 20260506 16:18:04.793 | INFO | "pinned" : " 29 GB" | 20260506 16:18:04.793 | INFO | } | 20260506 16:18:04.793 | INFO | Perf report for the last 5.13502 seconds: | 20260506 16:18:04.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:18:04.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:04.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:04.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:04.793 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:18:04.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:04.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:04.793 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:18:04.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:04.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:04.794 | INFO | Uploaded 36GiB to GPU in 1.99319s (18.0615GiB/s) | 20260506 16:18:04.794 | INFO | Downloaded 34GiB to GPU in 1.9415s (17.5123GiB/s) | 20260506 16:18:09.934 | INFO | Analyzing chunk frames = [2728960,2729472) | 20260506 16:18:09.934 | INFO | MemInfo: { | 20260506 16:18:09.934 | INFO | "gpu" : " 17 GB", | 20260506 16:18:09.934 | INFO | "huge" : "", | 20260506 16:18:09.934 | INFO | "huge_shared" : " 27 GB", | 20260506 16:18:09.934 | INFO | "malloc" : " 14 GB", | 20260506 16:18:09.934 | INFO | "pinned" : " 29 GB" | 20260506 16:18:09.934 | INFO | } | 20260506 16:18:09.934 | INFO | Perf report for the last 5.14098 seconds: | 20260506 16:18:09.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:18:09.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:09.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:09.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:09.935 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 16:18:09.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:09.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:09.935 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:18:09.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:09.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:09.935 | INFO | Uploaded 36GiB to GPU in 1.9786s (18.1947GiB/s) | 20260506 16:18:09.935 | INFO | Downloaded 34GiB to GPU in 1.9295s (17.6212GiB/s) | 20260506 16:18:15.078 | INFO | Analyzing chunk frames = [2729472,2729984) | 20260506 16:18:15.079 | INFO | MemInfo: { | 20260506 16:18:15.079 | INFO | "gpu" : " 17 GB", | 20260506 16:18:15.079 | INFO | "huge" : "", | 20260506 16:18:15.079 | INFO | "huge_shared" : " 27 GB", | 20260506 16:18:15.079 | INFO | "malloc" : " 14 GB", | 20260506 16:18:15.079 | INFO | "pinned" : " 29 GB" | 20260506 16:18:15.079 | INFO | } | 20260506 16:18:15.079 | INFO | Perf report for the last 5.14451 seconds: | 20260506 16:18:15.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:18:15.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:15.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:15.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:15.079 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260506 16:18:15.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:15.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:15.079 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260506 16:18:15.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:15.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:15.079 | INFO | Uploaded 36GiB to GPU in 1.92712s (18.6808GiB/s) | 20260506 16:18:15.079 | INFO | Downloaded 34GiB to GPU in 1.87646s (18.1192GiB/s) | 20260506 16:18:20.222 | INFO | Analyzing chunk frames = [2729984,2730496) | 20260506 16:18:20.222 | INFO | MemInfo: { | 20260506 16:18:20.222 | INFO | "gpu" : " 17 GB", | 20260506 16:18:20.222 | INFO | "huge" : "", | 20260506 16:18:20.222 | INFO | "huge_shared" : " 27 GB", | 20260506 16:18:20.223 | INFO | "malloc" : " 14 GB", | 20260506 16:18:20.223 | INFO | "pinned" : " 29 GB" | 20260506 16:18:20.223 | INFO | } | 20260506 16:18:20.223 | INFO | Perf report for the last 5.14383 seconds: | 20260506 16:18:20.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:18:20.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:20.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:20.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:20.223 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 16:18:20.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:20.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:20.223 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:18:20.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:20.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:20.223 | INFO | Uploaded 36GiB to GPU in 2.01402s (17.8747GiB/s) | 20260506 16:18:20.223 | INFO | Downloaded 34GiB to GPU in 1.90542s (17.8439GiB/s) | 20260506 16:18:25.364 | INFO | Analyzing chunk frames = [2730496,2731008) | 20260506 16:18:25.364 | INFO | MemInfo: { | 20260506 16:18:25.364 | INFO | "gpu" : " 17 GB", | 20260506 16:18:25.364 | INFO | "huge" : "", | 20260506 16:18:25.364 | INFO | "huge_shared" : " 27 GB", | 20260506 16:18:25.364 | INFO | "malloc" : " 14 GB", | 20260506 16:18:25.364 | INFO | "pinned" : " 29 GB" | 20260506 16:18:25.364 | INFO | } | 20260506 16:18:25.364 | INFO | Perf report for the last 5.14168 seconds: | 20260506 16:18:25.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:18:25.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:25.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:25.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:25.364 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 16:18:25.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:25.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:25.364 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 16:18:25.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:25.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:25.365 | INFO | Uploaded 36GiB to GPU in 2.07433s (17.355GiB/s) | 20260506 16:18:25.365 | INFO | Downloaded 34GiB to GPU in 1.96105s (17.3376GiB/s) | 20260506 16:18:30.495 | INFO | Analyzing chunk frames = [2731008,2731520) | 20260506 16:18:30.495 | INFO | MemInfo: { | 20260506 16:18:30.495 | INFO | "gpu" : " 17 GB", | 20260506 16:18:30.495 | INFO | "huge" : "", | 20260506 16:18:30.495 | INFO | "huge_shared" : " 27 GB", | 20260506 16:18:30.495 | INFO | "malloc" : " 14 GB", | 20260506 16:18:30.495 | INFO | "pinned" : " 29 GB" | 20260506 16:18:30.495 | INFO | } | 20260506 16:18:30.495 | INFO | Perf report for the last 5.13068 seconds: | 20260506 16:18:30.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:18:30.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:30.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:30.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:30.495 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 16:18:30.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:30.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:30.495 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 16:18:30.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:30.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:30.495 | INFO | Uploaded 36GiB to GPU in 1.99811s (18.017GiB/s) | 20260506 16:18:30.495 | INFO | Downloaded 34GiB to GPU in 1.99023s (17.0834GiB/s) | 20260506 16:18:35.630 | INFO | Analyzing chunk frames = [2731520,2732032) | 20260506 16:18:35.630 | INFO | MemInfo: { | 20260506 16:18:35.630 | INFO | "gpu" : " 17 GB", | 20260506 16:18:35.630 | INFO | "huge" : "", | 20260506 16:18:35.631 | INFO | "huge_shared" : " 27 GB", | 20260506 16:18:35.631 | INFO | "malloc" : " 14 GB", | 20260506 16:18:35.631 | INFO | "pinned" : " 29 GB" | 20260506 16:18:35.631 | INFO | } | 20260506 16:18:35.631 | INFO | Perf report for the last 5.13556 seconds: | 20260506 16:18:35.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:18:35.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:35.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:35.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:35.631 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:18:35.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:35.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:35.631 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 16:18:35.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:35.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:35.631 | INFO | Uploaded 36GiB to GPU in 1.99009s (18.0896GiB/s) | 20260506 16:18:35.631 | INFO | Downloaded 34GiB to GPU in 1.97296s (17.233GiB/s) | 20260506 16:18:35.743 | INFO | Starting a round of Baz IO | 20260506 16:18:38.893 | INFO | Populating RTMetrics report | 20260506 16:18:38.893 | INFO | RTMetrics report populated | 20260506 16:18:38.893 | INFO | Opening tmp RTMetrics json file | 20260506 16:18:38.895 | INFO | Serializing RTMetrics to json | 20260506 16:18:38.896 | INFO | Serialization done, swapping tmp file now... | 20260506 16:18:38.900 | INFO | RTMetrics done | 20260506 16:18:39.087 | INFO | Finished a round of Baz IO: Wrote 2.091 GiB in 3.344 seconds | 20260506 16:18:39.087 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.45% / 0.01% / 63.54% | 20260506 16:18:39.087 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.73775% / 1.58946e-05% / 5.16574e-05% | 20260506 16:18:39.087 | INFO | Baz IO overall: | 20260506 16:18:39.087 | INFO | BytesWritten : 1432990144984 | 20260506 16:18:39.087 | INFO | NumEvents : 693434923172 | 20260506 16:18:40.770 | INFO | Analyzing chunk frames = [2732032,2732544) | 20260506 16:18:40.770 | INFO | MemInfo: { | 20260506 16:18:40.770 | INFO | "gpu" : " 17 GB", | 20260506 16:18:40.770 | INFO | "huge" : "", | 20260506 16:18:40.770 | INFO | "huge_shared" : " 27 GB", | 20260506 16:18:40.770 | INFO | "malloc" : " 14 GB", | 20260506 16:18:40.770 | INFO | "pinned" : " 29 GB" | 20260506 16:18:40.770 | INFO | } | 20260506 16:18:40.770 | INFO | Perf report for the last 5.13962 seconds: | 20260506 16:18:40.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:18:40.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:40.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:40.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:40.770 | INFO | ANALYSIS : 62.9%, 2.9, 95.2ms, 0 | 20260506 16:18:40.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:40.770 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 16:18:40.770 | INFO | PRE_HQ : 60.7%, 1.9, 59.3ms, 0 | 20260506 16:18:40.770 | INFO | BAZWRITER : 64.5%, 4.7, 154.8ms, 0 | 20260506 16:18:40.770 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.5ms, 0 | 20260506 16:18:40.771 | INFO | Uploaded 36GiB to GPU in 2.06715s (17.4153GiB/s) | 20260506 16:18:40.771 | INFO | Downloaded 38GiB to GPU in 2.22083s (17.1107GiB/s) | 20260506 16:18:45.909 | INFO | Analyzing chunk frames = [2732544,2733056) | 20260506 16:18:45.909 | INFO | MemInfo: { | 20260506 16:18:45.909 | INFO | "gpu" : " 17 GB", | 20260506 16:18:45.909 | INFO | "huge" : "", | 20260506 16:18:45.909 | INFO | "huge_shared" : " 27 GB", | 20260506 16:18:45.909 | INFO | "malloc" : " 14 GB", | 20260506 16:18:45.909 | INFO | "pinned" : " 29 GB" | 20260506 16:18:45.909 | INFO | } | 20260506 16:18:45.909 | INFO | Perf report for the last 5.13878 seconds: | 20260506 16:18:45.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:18:45.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:45.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:45.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:45.909 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 16:18:45.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:45.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:45.909 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:18:45.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:45.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:45.909 | INFO | Uploaded 36GiB to GPU in 2.02081s (17.8147GiB/s) | 20260506 16:18:45.909 | INFO | Downloaded 34GiB to GPU in 1.98813s (17.1015GiB/s) | 20260506 16:18:51.049 | INFO | Analyzing chunk frames = [2733056,2733568) | 20260506 16:18:51.049 | INFO | MemInfo: { | 20260506 16:18:51.049 | INFO | "gpu" : " 17 GB", | 20260506 16:18:51.049 | INFO | "huge" : "", | 20260506 16:18:51.049 | INFO | "huge_shared" : " 27 GB", | 20260506 16:18:51.049 | INFO | "malloc" : " 14 GB", | 20260506 16:18:51.049 | INFO | "pinned" : " 29 GB" | 20260506 16:18:51.049 | INFO | } | 20260506 16:18:51.049 | INFO | Perf report for the last 5.14034 seconds: | 20260506 16:18:51.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:18:51.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:51.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:51.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:51.049 | INFO | ANALYSIS : 61.4%, 3.0, 95.1ms, 0 | 20260506 16:18:51.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:51.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:51.049 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:18:51.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:51.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:51.050 | INFO | Uploaded 36GiB to GPU in 2.01114s (17.9003GiB/s) | 20260506 16:18:51.050 | INFO | Downloaded 34GiB to GPU in 1.93283s (17.5908GiB/s) | 20260506 16:18:56.193 | INFO | Analyzing chunk frames = [2733568,2734080) | 20260506 16:18:56.193 | INFO | MemInfo: { | 20260506 16:18:56.193 | INFO | "gpu" : " 17 GB", | 20260506 16:18:56.193 | INFO | "huge" : "", | 20260506 16:18:56.193 | INFO | "huge_shared" : " 27 GB", | 20260506 16:18:56.193 | INFO | "malloc" : " 14 GB", | 20260506 16:18:56.193 | INFO | "pinned" : " 29 GB" | 20260506 16:18:56.193 | INFO | } | 20260506 16:18:56.193 | INFO | Perf report for the last 5.14327 seconds: | 20260506 16:18:56.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:18:56.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:56.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:56.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:56.193 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260506 16:18:56.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:56.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:56.193 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:18:56.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:56.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:18:56.193 | INFO | Uploaded 36GiB to GPU in 1.94654s (18.4944GiB/s) | 20260506 16:18:56.193 | INFO | Downloaded 34GiB to GPU in 1.88201s (18.0658GiB/s) | 20260506 16:19:01.331 | INFO | Analyzing chunk frames = [2734080,2734592) | 20260506 16:19:01.331 | INFO | MemInfo: { | 20260506 16:19:01.331 | INFO | "gpu" : " 17 GB", | 20260506 16:19:01.331 | INFO | "huge" : "", | 20260506 16:19:01.331 | INFO | "huge_shared" : " 27 GB", | 20260506 16:19:01.331 | INFO | "malloc" : " 14 GB", | 20260506 16:19:01.331 | INFO | "pinned" : " 29 GB" | 20260506 16:19:01.331 | INFO | } | 20260506 16:19:01.331 | INFO | Perf report for the last 5.13872 seconds: | 20260506 16:19:01.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:19:01.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:01.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:01.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:01.331 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 16:19:01.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:01.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:01.331 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:19:01.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:01.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:01.332 | INFO | Uploaded 36GiB to GPU in 2.01267s (17.8867GiB/s) | 20260506 16:19:01.332 | INFO | Downloaded 34GiB to GPU in 1.91084s (17.7933GiB/s) | 20260506 16:19:06.470 | INFO | Analyzing chunk frames = [2734592,2735104) | 20260506 16:19:06.470 | INFO | MemInfo: { | 20260506 16:19:06.470 | INFO | "gpu" : " 17 GB", | 20260506 16:19:06.470 | INFO | "huge" : "", | 20260506 16:19:06.470 | INFO | "huge_shared" : " 27 GB", | 20260506 16:19:06.470 | INFO | "malloc" : " 14 GB", | 20260506 16:19:06.470 | INFO | "pinned" : " 29 GB" | 20260506 16:19:06.470 | INFO | } | 20260506 16:19:06.470 | INFO | Perf report for the last 5.13867 seconds: | 20260506 16:19:06.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:19:06.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:06.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:06.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:06.470 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 16:19:06.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:06.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:06.470 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:19:06.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:06.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:06.470 | INFO | Uploaded 36GiB to GPU in 2.02342s (17.7916GiB/s) | 20260506 16:19:06.470 | INFO | Downloaded 34GiB to GPU in 1.94596s (17.4721GiB/s) | 20260506 16:19:11.611 | INFO | Analyzing chunk frames = [2735104,2735616) | 20260506 16:19:11.611 | INFO | MemInfo: { | 20260506 16:19:11.611 | INFO | "gpu" : " 17 GB", | 20260506 16:19:11.611 | INFO | "huge" : "", | 20260506 16:19:11.611 | INFO | "huge_shared" : " 27 GB", | 20260506 16:19:11.611 | INFO | "malloc" : " 14 GB", | 20260506 16:19:11.611 | INFO | "pinned" : " 29 GB" | 20260506 16:19:11.611 | INFO | } | 20260506 16:19:11.611 | INFO | Perf report for the last 5.14116 seconds: | 20260506 16:19:11.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:19:11.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:11.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:11.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:11.611 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 16:19:11.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:11.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:11.611 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:19:11.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:11.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:11.612 | INFO | Uploaded 36GiB to GPU in 2.04011s (17.6461GiB/s) | 20260506 16:19:11.612 | INFO | Downloaded 34GiB to GPU in 1.99566s (17.037GiB/s) | 20260506 16:19:16.751 | INFO | Analyzing chunk frames = [2735616,2736128) | 20260506 16:19:16.751 | INFO | MemInfo: { | 20260506 16:19:16.751 | INFO | "gpu" : " 17 GB", | 20260506 16:19:16.751 | INFO | "huge" : "", | 20260506 16:19:16.751 | INFO | "huge_shared" : " 27 GB", | 20260506 16:19:16.751 | INFO | "malloc" : " 14 GB", | 20260506 16:19:16.751 | INFO | "pinned" : " 29 GB" | 20260506 16:19:16.751 | INFO | } | 20260506 16:19:16.752 | INFO | Perf report for the last 5.14029 seconds: | 20260506 16:19:16.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:19:16.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:16.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:16.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:16.752 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 16:19:16.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:16.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:16.752 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260506 16:19:16.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:16.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:16.752 | INFO | Uploaded 36GiB to GPU in 2.00031s (17.9972GiB/s) | 20260506 16:19:16.752 | INFO | Downloaded 34GiB to GPU in 1.9681s (17.2756GiB/s) | 20260506 16:19:16.752 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:19:16.752 | INFO | Total mem usage: 18 GiB | 20260506 16:19:16.752 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:19:16.752 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.6%) | 20260506 16:19:16.752 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:19:16.752 | INFO | High Water Breakdown: | 20260506 16:19:16.752 | INFO | 61.2% : 9.54 GiB ( 83.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:19:16.752 | INFO | 38.6% : 6.02 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:19:16.752 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:19:16.752 | INFO | ------------------------------------------ | 20260506 16:19:16.752 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:19:16.752 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:19:16.752 | INFO | Total mem usage: 26.5 GiB | 20260506 16:19:16.752 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:19:16.752 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:19:16.752 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:19:16.752 | INFO | High Water Breakdown: | 20260506 16:19:16.752 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:19:16.752 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:19:16.752 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:19:16.752 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:19:16.752 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:19:16.752 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:19:16.752 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:19:16.752 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:19:16.752 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:19:16.752 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:19:16.752 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:19:16.752 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:19:16.752 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:19:16.752 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:19:16.752 | INFO | ------------------------------------------ | 20260506 16:19:16.753 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:19:16.753 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:19:16.753 | INFO | Total mem usage: 25.6 GiB | 20260506 16:19:16.753 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:19:16.753 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:19:16.753 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:19:16.753 | INFO | High Water Breakdown: | 20260506 16:19:16.753 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:19:16.753 | INFO | ------------------------------------------ | 20260506 16:19:16.753 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:19:16.753 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:19:16.753 | INFO | Total mem usage: 15.7 GiB | 20260506 16:19:16.753 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:19:16.753 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:19:16.753 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 16:19:16.753 | INFO | High Water Breakdown: | 20260506 16:19:16.753 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:19:16.753 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:19:16.753 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:19:16.753 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:19:16.753 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:19:16.753 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:19:16.753 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:19:16.753 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:19:16.753 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:19:16.753 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:19:16.753 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:19:16.753 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:19:16.753 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:19:16.753 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:19:16.753 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:19:16.753 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:19:16.753 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:19:16.753 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:19:16.753 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:19:16.753 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:19:16.753 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:19:16.753 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:19:16.753 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:19:16.753 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:19:16.753 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:19:16.753 | INFO | ------------------------------------------ | 20260506 16:19:16.753 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:19:16.753 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:19:16.753 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:19:16.753 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:19:16.753 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.56e+01, 1.33e+01 / 2.02e+00 | 20260506 16:19:16.753 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.01e-01, 9.66e-03 / 4.66e-03 | 20260506 16:19:16.753 | INFO | PerfReport: 11.60%, Baseline -- 3.64e+00 / 4.47e+00, 5.58e-01 / 2.69e-01 | 20260506 16:19:16.753 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.50e+00, 1.86e-01 / 9.60e-02 | 20260506 16:19:16.753 | INFO | PerfReport: 15.91%, DME -- 3.99e+01 / 4.44e+01, 3.80e+00 / 1.94e+00 | 20260506 16:19:16.753 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.38e+00 / 1.45e+00 | 20260506 16:19:16.753 | INFO | PerfReport: 7.35%, PulseAccumulating -- 2.31e+00 / 2.89e+00, 3.88e-01 / 1.85e-01 | 20260506 16:19:16.753 | INFO | PerfReport: 3.52%, Metrics -- 1.10e+00 / 1.46e+00, 2.85e-01 / 1.49e-01 | 20260506 16:19:16.753 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.83e-01 / 4.34e-01, 5.95e-02 / 2.13e-02 | 20260506 16:19:16.753 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:19:16.753 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:19:16.753 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:19:16.753 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 16:19:16.753 | INFO | PerfReport: 21.48%, Upload -- 2.09e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 16:19:16.753 | INFO | PerfReport: 21.14%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 16:19:16.753 | INFO | PerfReport: 32.20%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 16:19:16.753 | INFO | PerfReport: 1.53%, LockUpload -- 1.49e+00 / 0.00e+00, 6.02e+00 / 0.00e+00 | 20260506 16:19:16.753 | INFO | PerfReport: 1.04%, LockDownload -- 1.02e+00 / 0.00e+00, 2.58e+00 / 0.00e+00 | 20260506 16:19:16.753 | INFO | PerfReport: 22.58%, LockCompute -- 2.20e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 16:19:16.753 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260506 16:19:16.871 | INFO | Starting a round of Baz IO | 20260506 16:19:20.012 | INFO | Populating RTMetrics report | 20260506 16:19:20.012 | INFO | RTMetrics report populated | 20260506 16:19:20.012 | INFO | Opening tmp RTMetrics json file | 20260506 16:19:20.264 | INFO | Serializing RTMetrics to json | 20260506 16:19:20.273 | INFO | Serialization done, swapping tmp file now... | 20260506 16:19:20.278 | INFO | RTMetrics done | 20260506 16:19:20.532 | INFO | Finished a round of Baz IO: Wrote 2.091 GiB in 3.661 seconds | 20260506 16:19:20.532 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.44% / 0.01% / 63.55% | 20260506 16:19:20.532 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.73483% / 1.58946e-05% / 5.5631e-05% | 20260506 16:19:20.533 | INFO | Baz IO overall: | 20260506 16:19:20.533 | INFO | BytesWritten : 1435235669867 | 20260506 16:19:20.533 | INFO | NumEvents : 694520275244 | 20260506 16:19:21.885 | INFO | Analyzing chunk frames = [2736128,2736640) | 20260506 16:19:21.885 | INFO | MemInfo: { | 20260506 16:19:21.885 | INFO | "gpu" : " 17 GB", | 20260506 16:19:21.885 | INFO | "huge" : "", | 20260506 16:19:21.885 | INFO | "huge_shared" : " 27 GB", | 20260506 16:19:21.885 | INFO | "malloc" : " 14 GB", | 20260506 16:19:21.885 | INFO | "pinned" : " 29 GB" | 20260506 16:19:21.885 | INFO | } | 20260506 16:19:21.885 | INFO | Perf report for the last 5.13368 seconds: | 20260506 16:19:21.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:19:21.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:21.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:21.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:21.885 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 16:19:21.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:21.885 | INFO | RT_METRICS : 48.8%, 1.0, 25.6ms, 0 | 20260506 16:19:21.885 | INFO | PRE_HQ : 60.6%, 1.9, 58.8ms, 0 | 20260506 16:19:21.885 | INFO | BAZWRITER : 70.9%, 4.8, 175.6ms, 0 | 20260506 16:19:21.886 | INFO | BAZ_SAMPLING : 4.7%, 1.0, 2.4ms, 0 | 20260506 16:19:21.886 | INFO | Uploaded 36GiB to GPU in 2.07436s (17.3548GiB/s) | 20260506 16:19:21.886 | INFO | Downloaded 38GiB to GPU in 2.21735s (17.1376GiB/s) | 20260506 16:19:27.025 | INFO | Analyzing chunk frames = [2736640,2737152) | 20260506 16:19:27.025 | INFO | MemInfo: { | 20260506 16:19:27.025 | INFO | "gpu" : " 17 GB", | 20260506 16:19:27.025 | INFO | "huge" : "", | 20260506 16:19:27.025 | INFO | "huge_shared" : " 27 GB", | 20260506 16:19:27.025 | INFO | "malloc" : " 14 GB", | 20260506 16:19:27.025 | INFO | "pinned" : " 29 GB" | 20260506 16:19:27.025 | INFO | } | 20260506 16:19:27.025 | INFO | Perf report for the last 5.13975 seconds: | 20260506 16:19:27.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:19:27.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:27.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:27.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:27.025 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:19:27.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:27.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:27.025 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 16:19:27.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:27.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:27.025 | INFO | Uploaded 36GiB to GPU in 2.00236s (17.9788GiB/s) | 20260506 16:19:27.025 | INFO | Downloaded 34GiB to GPU in 1.94588s (17.4728GiB/s) | 20260506 16:19:32.169 | INFO | Analyzing chunk frames = [2737152,2737664) | 20260506 16:19:32.169 | INFO | MemInfo: { | 20260506 16:19:32.169 | INFO | "gpu" : " 17 GB", | 20260506 16:19:32.169 | INFO | "huge" : "", | 20260506 16:19:32.169 | INFO | "huge_shared" : " 27 GB", | 20260506 16:19:32.169 | INFO | "malloc" : " 14 GB", | 20260506 16:19:32.169 | INFO | "pinned" : " 29 GB" | 20260506 16:19:32.169 | INFO | } | 20260506 16:19:32.169 | INFO | Perf report for the last 5.14443 seconds: | 20260506 16:19:32.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:19:32.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:32.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:32.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:32.170 | INFO | ANALYSIS : 60.4%, 3.0, 93.8ms, 0 | 20260506 16:19:32.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:32.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:32.170 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:19:32.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:32.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:32.170 | INFO | Uploaded 36GiB to GPU in 1.94436s (18.5151GiB/s) | 20260506 16:19:32.170 | INFO | Downloaded 34GiB to GPU in 1.88953s (17.9939GiB/s) | 20260506 16:19:37.309 | INFO | Analyzing chunk frames = [2737664,2738176) | 20260506 16:19:37.309 | INFO | MemInfo: { | 20260506 16:19:37.309 | INFO | "gpu" : " 17 GB", | 20260506 16:19:37.309 | INFO | "huge" : "", | 20260506 16:19:37.309 | INFO | "huge_shared" : " 27 GB", | 20260506 16:19:37.309 | INFO | "malloc" : " 14 GB", | 20260506 16:19:37.309 | INFO | "pinned" : " 29 GB" | 20260506 16:19:37.310 | INFO | } | 20260506 16:19:37.310 | INFO | Perf report for the last 5.14026 seconds: | 20260506 16:19:37.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:19:37.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:37.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:37.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:37.310 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 16:19:37.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:37.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:37.310 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:19:37.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:37.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:37.310 | INFO | Uploaded 36GiB to GPU in 1.93666s (18.5887GiB/s) | 20260506 16:19:37.310 | INFO | Downloaded 34GiB to GPU in 1.8767s (18.1169GiB/s) | 20260506 16:19:42.453 | INFO | Analyzing chunk frames = [2738176,2738688) | 20260506 16:19:42.453 | INFO | MemInfo: { | 20260506 16:19:42.453 | INFO | "gpu" : " 17 GB", | 20260506 16:19:42.453 | INFO | "huge" : "", | 20260506 16:19:42.453 | INFO | "huge_shared" : " 27 GB", | 20260506 16:19:42.453 | INFO | "malloc" : " 14 GB", | 20260506 16:19:42.453 | INFO | "pinned" : " 29 GB" | 20260506 16:19:42.453 | INFO | } | 20260506 16:19:42.453 | INFO | Perf report for the last 5.14358 seconds: | 20260506 16:19:42.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:19:42.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:42.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:42.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:42.454 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 16:19:42.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:42.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:42.454 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:19:42.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:42.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:42.454 | INFO | Uploaded 36GiB to GPU in 2.02115s (17.8116GiB/s) | 20260506 16:19:42.454 | INFO | Downloaded 34GiB to GPU in 1.91226s (17.78GiB/s) | 20260506 16:19:47.585 | INFO | Analyzing chunk frames = [2738688,2739200) | 20260506 16:19:47.585 | INFO | MemInfo: { | 20260506 16:19:47.585 | INFO | "gpu" : " 17 GB", | 20260506 16:19:47.585 | INFO | "huge" : "", | 20260506 16:19:47.585 | INFO | "huge_shared" : " 27 GB", | 20260506 16:19:47.585 | INFO | "malloc" : " 14 GB", | 20260506 16:19:47.585 | INFO | "pinned" : " 29 GB" | 20260506 16:19:47.585 | INFO | } | 20260506 16:19:47.585 | INFO | Perf report for the last 5.13167 seconds: | 20260506 16:19:47.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:19:47.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:47.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:47.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:47.585 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 16:19:47.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:47.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:47.585 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:19:47.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:47.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:47.585 | INFO | Uploaded 36GiB to GPU in 2.06735s (17.4136GiB/s) | 20260506 16:19:47.585 | INFO | Downloaded 34GiB to GPU in 1.95931s (17.3531GiB/s) | 20260506 16:19:52.723 | INFO | Analyzing chunk frames = [2739200,2739712) | 20260506 16:19:52.723 | INFO | MemInfo: { | 20260506 16:19:52.723 | INFO | "gpu" : " 17 GB", | 20260506 16:19:52.723 | INFO | "huge" : "", | 20260506 16:19:52.723 | INFO | "huge_shared" : " 27 GB", | 20260506 16:19:52.723 | INFO | "malloc" : " 14 GB", | 20260506 16:19:52.723 | INFO | "pinned" : " 29 GB" | 20260506 16:19:52.723 | INFO | } | 20260506 16:19:52.723 | INFO | Perf report for the last 5.13808 seconds: | 20260506 16:19:52.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:19:52.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:52.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:52.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:52.723 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 16:19:52.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:52.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:52.723 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 16:19:52.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:52.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:52.724 | INFO | Uploaded 36GiB to GPU in 2.02928s (17.7403GiB/s) | 20260506 16:19:52.724 | INFO | Downloaded 34GiB to GPU in 1.9961s (17.0332GiB/s) | 20260506 16:19:57.865 | INFO | Analyzing chunk frames = [2739712,2740224) | 20260506 16:19:57.865 | INFO | MemInfo: { | 20260506 16:19:57.865 | INFO | "gpu" : " 17 GB", | 20260506 16:19:57.865 | INFO | "huge" : "", | 20260506 16:19:57.865 | INFO | "huge_shared" : " 27 GB", | 20260506 16:19:57.865 | INFO | "malloc" : " 14 GB", | 20260506 16:19:57.866 | INFO | "pinned" : " 29 GB" | 20260506 16:19:57.866 | INFO | } | 20260506 16:19:57.866 | INFO | Perf report for the last 5.1426 seconds: | 20260506 16:19:57.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:19:57.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:57.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:57.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:57.866 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 16:19:57.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:57.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:57.866 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:19:57.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:57.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:19:57.866 | INFO | Uploaded 36GiB to GPU in 2.0646s (17.4368GiB/s) | 20260506 16:19:57.866 | INFO | Downloaded 34GiB to GPU in 1.97627s (17.2042GiB/s) | 20260506 16:19:57.977 | INFO | Starting a round of Baz IO | 20260506 16:20:01.131 | INFO | Populating RTMetrics report | 20260506 16:20:01.131 | INFO | RTMetrics report populated | 20260506 16:20:01.131 | INFO | Opening tmp RTMetrics json file | 20260506 16:20:01.137 | INFO | Serializing RTMetrics to json | 20260506 16:20:01.140 | INFO | Serialization done, swapping tmp file now... | 20260506 16:20:01.146 | INFO | RTMetrics done | 20260506 16:20:01.541 | INFO | Finished a round of Baz IO: Wrote 2.092 GiB in 3.564 seconds | 20260506 16:20:01.542 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.43% / 0.01% / 63.56% | 20260506 16:20:01.542 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.73131% / 1.19209e-05% / 5.16574e-05% | 20260506 16:20:01.542 | INFO | Baz IO overall: | 20260506 16:20:01.542 | INFO | BytesWritten : 1437481935963 | 20260506 16:20:01.542 | INFO | NumEvents : 695606240601 | 20260506 16:20:03.000 | INFO | Analyzing chunk frames = [2740224,2740736) | 20260506 16:20:03.000 | INFO | MemInfo: { | 20260506 16:20:03.000 | INFO | "gpu" : " 17 GB", | 20260506 16:20:03.000 | INFO | "huge" : "", | 20260506 16:20:03.000 | INFO | "huge_shared" : " 27 GB", | 20260506 16:20:03.000 | INFO | "malloc" : " 14 GB", | 20260506 16:20:03.000 | INFO | "pinned" : " 29 GB" | 20260506 16:20:03.000 | INFO | } | 20260506 16:20:03.000 | INFO | Perf report for the last 5.13465 seconds: | 20260506 16:20:03.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:20:03.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:03.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:03.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:03.000 | INFO | ANALYSIS : 63.0%, 2.9, 95.0ms, 0 | 20260506 16:20:03.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:03.001 | INFO | RT_METRICS : 44.2%, 1.0, 23.1ms, 0 | 20260506 16:20:03.001 | INFO | PRE_HQ : 60.6%, 1.9, 59.1ms, 0 | 20260506 16:20:03.001 | INFO | BAZWRITER : 68.7%, 4.8, 170.2ms, 0 | 20260506 16:20:03.001 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 16:20:03.001 | INFO | Uploaded 36GiB to GPU in 2.04045s (17.6431GiB/s) | 20260506 16:20:03.001 | INFO | Downloaded 38GiB to GPU in 2.23004s (17.0401GiB/s) | 20260506 16:20:08.140 | INFO | Analyzing chunk frames = [2740736,2741248) | 20260506 16:20:08.140 | INFO | MemInfo: { | 20260506 16:20:08.140 | INFO | "gpu" : " 17 GB", | 20260506 16:20:08.140 | INFO | "huge" : "", | 20260506 16:20:08.140 | INFO | "huge_shared" : " 27 GB", | 20260506 16:20:08.140 | INFO | "malloc" : " 14 GB", | 20260506 16:20:08.140 | INFO | "pinned" : " 29 GB" | 20260506 16:20:08.140 | INFO | } | 20260506 16:20:08.140 | INFO | Perf report for the last 5.13963 seconds: | 20260506 16:20:08.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:20:08.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:08.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:08.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:08.140 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:20:08.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:08.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:08.140 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:20:08.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:08.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:08.140 | INFO | Uploaded 36GiB to GPU in 1.98966s (18.0936GiB/s) | 20260506 16:20:08.140 | INFO | Downloaded 34GiB to GPU in 1.94025s (17.5235GiB/s) | 20260506 16:20:13.280 | INFO | Analyzing chunk frames = [2741248,2741760) | 20260506 16:20:13.280 | INFO | MemInfo: { | 20260506 16:20:13.280 | INFO | "gpu" : " 17 GB", | 20260506 16:20:13.280 | INFO | "huge" : "", | 20260506 16:20:13.280 | INFO | "huge_shared" : " 27 GB", | 20260506 16:20:13.280 | INFO | "malloc" : " 14 GB", | 20260506 16:20:13.280 | INFO | "pinned" : " 29 GB" | 20260506 16:20:13.280 | INFO | } | 20260506 16:20:13.280 | INFO | Perf report for the last 5.13997 seconds: | 20260506 16:20:13.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:20:13.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:13.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:13.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:13.280 | INFO | ANALYSIS : 60.4%, 3.0, 93.8ms, 0 | 20260506 16:20:13.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:13.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:13.280 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:20:13.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:13.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:13.280 | INFO | Uploaded 36GiB to GPU in 1.94479s (18.511GiB/s) | 20260506 16:20:13.280 | INFO | Downloaded 34GiB to GPU in 1.88886s (18.0003GiB/s) | 20260506 16:20:18.423 | INFO | Analyzing chunk frames = [2741760,2742272) | 20260506 16:20:18.423 | INFO | MemInfo: { | 20260506 16:20:18.423 | INFO | "gpu" : " 17 GB", | 20260506 16:20:18.423 | INFO | "huge" : "", | 20260506 16:20:18.423 | INFO | "huge_shared" : " 27 GB", | 20260506 16:20:18.423 | INFO | "malloc" : " 14 GB", | 20260506 16:20:18.423 | INFO | "pinned" : " 29 GB" | 20260506 16:20:18.423 | INFO | } | 20260506 16:20:18.423 | INFO | Perf report for the last 5.14285 seconds: | 20260506 16:20:18.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:20:18.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:18.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:18.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:18.423 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260506 16:20:18.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:18.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:18.423 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:20:18.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:18.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:18.423 | INFO | Uploaded 36GiB to GPU in 1.93961s (18.5605GiB/s) | 20260506 16:20:18.423 | INFO | Downloaded 34GiB to GPU in 1.87869s (18.0977GiB/s) | 20260506 16:20:23.560 | INFO | Analyzing chunk frames = [2742272,2742784) | 20260506 16:20:23.560 | INFO | MemInfo: { | 20260506 16:20:23.560 | INFO | "gpu" : " 17 GB", | 20260506 16:20:23.560 | INFO | "huge" : "", | 20260506 16:20:23.560 | INFO | "huge_shared" : " 27 GB", | 20260506 16:20:23.560 | INFO | "malloc" : " 14 GB", | 20260506 16:20:23.561 | INFO | "pinned" : " 29 GB" | 20260506 16:20:23.561 | INFO | } | 20260506 16:20:23.561 | INFO | Perf report for the last 5.13773 seconds: | 20260506 16:20:23.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:20:23.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:23.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:23.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:23.561 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 16:20:23.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:23.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:23.561 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 16:20:23.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:23.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:23.561 | INFO | Uploaded 36GiB to GPU in 2.01728s (17.8458GiB/s) | 20260506 16:20:23.561 | INFO | Downloaded 34GiB to GPU in 1.90418s (17.8555GiB/s) | 20260506 16:20:28.702 | INFO | Analyzing chunk frames = [2742784,2743296) | 20260506 16:20:28.702 | INFO | MemInfo: { | 20260506 16:20:28.702 | INFO | "gpu" : " 17 GB", | 20260506 16:20:28.702 | INFO | "huge" : "", | 20260506 16:20:28.702 | INFO | "huge_shared" : " 27 GB", | 20260506 16:20:28.702 | INFO | "malloc" : " 14 GB", | 20260506 16:20:28.702 | INFO | "pinned" : " 29 GB" | 20260506 16:20:28.702 | INFO | } | 20260506 16:20:28.702 | INFO | Perf report for the last 5.14178 seconds: | 20260506 16:20:28.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:20:28.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:28.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:28.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:28.702 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:20:28.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:28.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:28.702 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 16:20:28.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:28.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:28.703 | INFO | Uploaded 36GiB to GPU in 2.0736s (17.3611GiB/s) | 20260506 16:20:28.703 | INFO | Downloaded 34GiB to GPU in 1.96501s (17.3027GiB/s) | 20260506 16:20:33.842 | INFO | Analyzing chunk frames = [2743296,2743808) | 20260506 16:20:33.842 | INFO | MemInfo: { | 20260506 16:20:33.842 | INFO | "gpu" : " 17 GB", | 20260506 16:20:33.842 | INFO | "huge" : "", | 20260506 16:20:33.842 | INFO | "huge_shared" : " 27 GB", | 20260506 16:20:33.842 | INFO | "malloc" : " 14 GB", | 20260506 16:20:33.842 | INFO | "pinned" : " 29 GB" | 20260506 16:20:33.842 | INFO | } | 20260506 16:20:33.842 | INFO | Perf report for the last 5.13968 seconds: | 20260506 16:20:33.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:20:33.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:33.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:33.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:33.842 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:20:33.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:33.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:33.842 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 16:20:33.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:33.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:33.843 | INFO | Uploaded 36GiB to GPU in 1.99798s (18.0182GiB/s) | 20260506 16:20:33.843 | INFO | Downloaded 34GiB to GPU in 1.99157s (17.072GiB/s) | 20260506 16:20:38.978 | INFO | Analyzing chunk frames = [2743808,2744320) | 20260506 16:20:38.978 | INFO | MemInfo: { | 20260506 16:20:38.978 | INFO | "gpu" : " 17 GB", | 20260506 16:20:38.978 | INFO | "huge" : "", | 20260506 16:20:38.978 | INFO | "huge_shared" : " 27 GB", | 20260506 16:20:38.978 | INFO | "malloc" : " 14 GB", | 20260506 16:20:38.978 | INFO | "pinned" : " 29 GB" | 20260506 16:20:38.978 | INFO | } | 20260506 16:20:38.978 | INFO | Perf report for the last 5.13602 seconds: | 20260506 16:20:38.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:20:38.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:38.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:38.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:38.978 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 16:20:38.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:38.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:38.978 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260506 16:20:38.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:38.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:38.979 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:20:38.979 | INFO | Total mem usage: 18 GiB | 20260506 16:20:38.979 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:20:38.979 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.5%) | 20260506 16:20:38.979 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:20:38.979 | INFO | High Water Breakdown: | 20260506 16:20:38.979 | INFO | 61.3% : 9.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:20:38.979 | INFO | 38.6% : 6.02 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:20:38.979 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:20:38.978 | INFO | Uploaded 36GiB to GPU in 2.06618s (17.4235GiB/s) | 20260506 16:20:38.979 | INFO | Downloaded 34GiB to GPU in 1.97604s (17.2061GiB/s) | 20260506 16:20:38.979 | INFO | ------------------------------------------ | 20260506 16:20:38.979 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:20:38.979 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:20:38.979 | INFO | Total mem usage: 26.5 GiB | 20260506 16:20:38.979 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:20:38.979 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:20:38.979 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:20:38.979 | INFO | High Water Breakdown: | 20260506 16:20:38.979 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:20:38.979 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:20:38.979 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:20:38.979 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:20:38.979 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:20:38.979 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:20:38.979 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:20:38.979 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:20:38.979 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:20:38.979 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:20:38.979 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:20:38.979 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:20:38.979 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:20:38.979 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:20:38.979 | INFO | ------------------------------------------ | 20260506 16:20:38.979 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:20:38.979 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:20:38.979 | INFO | Total mem usage: 25.6 GiB | 20260506 16:20:38.979 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:20:38.979 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:20:38.979 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:20:38.979 | INFO | High Water Breakdown: | 20260506 16:20:38.979 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:20:38.979 | INFO | ------------------------------------------ | 20260506 16:20:38.979 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:20:38.979 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:20:38.979 | INFO | Total mem usage: 15.7 GiB | 20260506 16:20:38.979 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:20:38.979 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:20:38.979 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:20:38.979 | INFO | High Water Breakdown: | 20260506 16:20:38.979 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:20:38.979 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:20:38.979 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:20:38.979 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:20:38.979 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:20:38.979 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:20:38.979 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:20:38.979 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:20:38.979 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:20:38.979 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:20:38.979 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:20:38.980 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:20:38.980 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:20:38.980 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:20:38.980 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:20:38.980 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:20:38.980 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:20:38.980 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:20:38.980 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:20:38.980 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:20:38.980 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:20:38.980 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:20:38.980 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:20:38.980 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:20:38.980 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:20:38.980 | INFO | ------------------------------------------ | 20260506 16:20:38.980 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:20:38.980 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:20:38.980 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:20:38.980 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:20:38.980 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.54e+01, 1.33e+01 / 1.97e+00 | 20260506 16:20:38.980 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.01e-01, 1.05e-02 / 5.41e-03 | 20260506 16:20:38.980 | INFO | PerfReport: 11.63%, Baseline -- 3.65e+00 / 4.49e+00, 5.68e-01 / 2.50e-01 | 20260506 16:20:38.980 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.50e+00, 1.85e-01 / 1.19e-01 | 20260506 16:20:38.980 | INFO | PerfReport: 15.90%, DME -- 3.99e+01 / 4.46e+01, 3.78e+00 / 1.66e+00 | 20260506 16:20:38.980 | INFO | PerfReport: 56.84%, FrameLabeling -- 1.78e+01 / 1.99e+01, 1.41e+00 / 1.91e+00 | 20260506 16:20:38.980 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.30e+00 / 2.89e+00, 3.92e-01 / 2.28e-01 | 20260506 16:20:38.980 | INFO | PerfReport: 3.51%, Metrics -- 1.10e+00 / 1.49e+00, 2.84e-01 / 1.48e-01 | 20260506 16:20:38.980 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.83e-01 / 4.35e-01, 5.91e-02 / 1.70e-02 | 20260506 16:20:38.980 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:20:38.980 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:20:38.980 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:20:38.980 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 16:20:38.980 | INFO | PerfReport: 21.54%, Upload -- 2.10e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260506 16:20:38.980 | INFO | PerfReport: 21.10%, Download -- 2.05e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 16:20:38.980 | INFO | PerfReport: 32.24%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 16:20:38.980 | INFO | PerfReport: 1.52%, LockUpload -- 1.48e+00 / 0.00e+00, 5.98e+00 / 0.00e+00 | 20260506 16:20:38.980 | INFO | PerfReport: 1.02%, LockDownload -- 9.89e-01 / 0.00e+00, 2.57e+00 / 0.00e+00 | 20260506 16:20:38.980 | INFO | PerfReport: 22.58%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 16:20:38.980 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.33e-02 / 0.00e+00 | 20260506 16:20:39.099 | INFO | Starting a round of Baz IO | 20260506 16:20:42.282 | INFO | Populating RTMetrics report | 20260506 16:20:42.282 | INFO | RTMetrics report populated | 20260506 16:20:42.282 | INFO | Opening tmp RTMetrics json file | 20260506 16:20:42.397 | INFO | Serializing RTMetrics to json | 20260506 16:20:42.403 | INFO | Serialization done, swapping tmp file now... | 20260506 16:20:42.408 | INFO | RTMetrics done | 20260506 16:20:42.941 | INFO | Finished a round of Baz IO: Wrote 2.092 GiB in 3.841 seconds | 20260506 16:20:42.941 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.42% / 0.01% / 63.57% | 20260506 16:20:42.941 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.72664% / 3.17891e-05% / 6.35783e-05% | 20260506 16:20:42.941 | INFO | Baz IO overall: | 20260506 16:20:42.941 | INFO | BytesWritten : 1439727873391 | 20260506 16:20:42.941 | INFO | NumEvents : 696692045333 | 20260506 16:20:44.117 | INFO | Analyzing chunk frames = [2744320,2744832) | 20260506 16:20:44.117 | INFO | MemInfo: { | 20260506 16:20:44.117 | INFO | "gpu" : " 17 GB", | 20260506 16:20:44.117 | INFO | "huge" : "", | 20260506 16:20:44.117 | INFO | "huge_shared" : " 27 GB", | 20260506 16:20:44.117 | INFO | "malloc" : " 14 GB", | 20260506 16:20:44.117 | INFO | "pinned" : " 29 GB" | 20260506 16:20:44.117 | INFO | } | 20260506 16:20:44.117 | INFO | Perf report for the last 5.13929 seconds: | 20260506 16:20:44.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:20:44.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:44.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:44.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:44.118 | INFO | ANALYSIS : 63.4%, 2.8, 94.7ms, 0 | 20260506 16:20:44.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:44.118 | INFO | RT_METRICS : 48.4%, 1.0, 25.4ms, 0 | 20260506 16:20:44.118 | INFO | PRE_HQ : 61.2%, 2.0, 62.2ms, 0 | 20260506 16:20:44.118 | INFO | BAZWRITER : 74.1%, 4.8, 183.9ms, 0 | 20260506 16:20:44.118 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 16:20:44.118 | INFO | Uploaded 36GiB to GPU in 2.08157s (17.2947GiB/s) | 20260506 16:20:44.118 | INFO | Downloaded 38GiB to GPU in 2.20431s (17.2389GiB/s) | 20260506 16:20:49.253 | INFO | Analyzing chunk frames = [2744832,2745344) | 20260506 16:20:49.253 | INFO | MemInfo: { | 20260506 16:20:49.253 | INFO | "gpu" : " 17 GB", | 20260506 16:20:49.253 | INFO | "huge" : "", | 20260506 16:20:49.253 | INFO | "huge_shared" : " 27 GB", | 20260506 16:20:49.253 | INFO | "malloc" : " 14 GB", | 20260506 16:20:49.253 | INFO | "pinned" : " 29 GB" | 20260506 16:20:49.253 | INFO | } | 20260506 16:20:49.253 | INFO | Perf report for the last 5.13585 seconds: | 20260506 16:20:49.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:20:49.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:49.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:49.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:49.253 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 16:20:49.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:49.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:49.253 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 16:20:49.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:49.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:49.254 | INFO | Uploaded 36GiB to GPU in 1.98407s (18.1446GiB/s) | 20260506 16:20:49.254 | INFO | Downloaded 34GiB to GPU in 1.94225s (17.5054GiB/s) | 20260506 16:20:54.395 | INFO | Analyzing chunk frames = [2745344,2745856) | 20260506 16:20:54.395 | INFO | MemInfo: { | 20260506 16:20:54.395 | INFO | "gpu" : " 17 GB", | 20260506 16:20:54.395 | INFO | "huge" : "", | 20260506 16:20:54.395 | INFO | "huge_shared" : " 27 GB", | 20260506 16:20:54.395 | INFO | "malloc" : " 14 GB", | 20260506 16:20:54.395 | INFO | "pinned" : " 29 GB" | 20260506 16:20:54.395 | INFO | } | 20260506 16:20:54.395 | INFO | Perf report for the last 5.14197 seconds: | 20260506 16:20:54.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:20:54.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:54.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:54.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:54.395 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 16:20:54.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:54.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:54.395 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:20:54.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:54.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:54.396 | INFO | Uploaded 36GiB to GPU in 1.9706s (18.2686GiB/s) | 20260506 16:20:54.396 | INFO | Downloaded 34GiB to GPU in 1.93012s (17.6155GiB/s) | 20260506 16:20:59.539 | INFO | Analyzing chunk frames = [2745856,2746368) | 20260506 16:20:59.539 | INFO | MemInfo: { | 20260506 16:20:59.539 | INFO | "gpu" : " 17 GB", | 20260506 16:20:59.539 | INFO | "huge" : "", | 20260506 16:20:59.539 | INFO | "huge_shared" : " 27 GB", | 20260506 16:20:59.539 | INFO | "malloc" : " 14 GB", | 20260506 16:20:59.539 | INFO | "pinned" : " 29 GB" | 20260506 16:20:59.539 | INFO | } | 20260506 16:20:59.539 | INFO | Perf report for the last 5.14407 seconds: | 20260506 16:20:59.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:20:59.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:59.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:59.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:59.539 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260506 16:20:59.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:59.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:59.539 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:20:59.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:59.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:20:59.540 | INFO | Uploaded 36GiB to GPU in 1.93144s (18.6389GiB/s) | 20260506 16:20:59.540 | INFO | Downloaded 34GiB to GPU in 1.87174s (18.1649GiB/s) | 20260506 16:21:04.675 | INFO | Analyzing chunk frames = [2746368,2746880) | 20260506 16:21:04.675 | INFO | MemInfo: { | 20260506 16:21:04.675 | INFO | "gpu" : " 17 GB", | 20260506 16:21:04.675 | INFO | "huge" : "", | 20260506 16:21:04.675 | INFO | "huge_shared" : " 27 GB", | 20260506 16:21:04.676 | INFO | "malloc" : " 14 GB", | 20260506 16:21:04.676 | INFO | "pinned" : " 29 GB" | 20260506 16:21:04.676 | INFO | } | 20260506 16:21:04.676 | INFO | Perf report for the last 5.13631 seconds: | 20260506 16:21:04.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:21:04.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:04.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:04.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:04.676 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 16:21:04.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:04.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:04.676 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:21:04.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:04.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:04.676 | INFO | Uploaded 36GiB to GPU in 2.00454s (17.9593GiB/s) | 20260506 16:21:04.676 | INFO | Downloaded 34GiB to GPU in 1.90687s (17.8303GiB/s) | 20260506 16:21:09.816 | INFO | Analyzing chunk frames = [2746880,2747392) | 20260506 16:21:09.816 | INFO | MemInfo: { | 20260506 16:21:09.816 | INFO | "gpu" : " 17 GB", | 20260506 16:21:09.816 | INFO | "huge" : "", | 20260506 16:21:09.816 | INFO | "huge_shared" : " 27 GB", | 20260506 16:21:09.816 | INFO | "malloc" : " 14 GB", | 20260506 16:21:09.816 | INFO | "pinned" : " 29 GB" | 20260506 16:21:09.816 | INFO | } | 20260506 16:21:09.816 | INFO | Perf report for the last 5.1402 seconds: | 20260506 16:21:09.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:21:09.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:09.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:09.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:09.816 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 16:21:09.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:09.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:09.816 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:21:09.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:09.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:09.816 | INFO | Uploaded 36GiB to GPU in 2.04706s (17.5862GiB/s) | 20260506 16:21:09.816 | INFO | Downloaded 34GiB to GPU in 1.94821s (17.4519GiB/s) | 20260506 16:21:14.953 | INFO | Analyzing chunk frames = [2747392,2747904) | 20260506 16:21:14.953 | INFO | MemInfo: { | 20260506 16:21:14.953 | INFO | "gpu" : " 17 GB", | 20260506 16:21:14.953 | INFO | "huge" : "", | 20260506 16:21:14.953 | INFO | "huge_shared" : " 27 GB", | 20260506 16:21:14.953 | INFO | "malloc" : " 14 GB", | 20260506 16:21:14.953 | INFO | "pinned" : " 29 GB" | 20260506 16:21:14.953 | INFO | } | 20260506 16:21:14.953 | INFO | Perf report for the last 5.13721 seconds: | 20260506 16:21:14.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:21:14.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:14.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:14.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:14.953 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 16:21:14.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:14.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:14.953 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:21:14.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:14.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:14.953 | INFO | Uploaded 36GiB to GPU in 2.03128s (17.7228GiB/s) | 20260506 16:21:14.953 | INFO | Downloaded 34GiB to GPU in 2.01824s (16.8464GiB/s) | 20260506 16:21:20.092 | INFO | Analyzing chunk frames = [2747904,2748416) | 20260506 16:21:20.092 | INFO | MemInfo: { | 20260506 16:21:20.092 | INFO | "gpu" : " 17 GB", | 20260506 16:21:20.092 | INFO | "huge" : "", | 20260506 16:21:20.092 | INFO | "huge_shared" : " 27 GB", | 20260506 16:21:20.092 | INFO | "malloc" : " 14 GB", | 20260506 16:21:20.092 | INFO | "pinned" : " 29 GB" | 20260506 16:21:20.092 | INFO | } | 20260506 16:21:20.092 | INFO | Perf report for the last 5.13891 seconds: | 20260506 16:21:20.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:21:20.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:20.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:20.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:20.092 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:21:20.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:20.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:20.092 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:21:20.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:20.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:20.092 | INFO | Uploaded 36GiB to GPU in 2.00511s (17.9542GiB/s) | 20260506 16:21:20.092 | INFO | Downloaded 34GiB to GPU in 1.96103s (17.3378GiB/s) | 20260506 16:21:20.212 | INFO | Starting a round of Baz IO | 20260506 16:21:23.349 | INFO | Populating RTMetrics report | 20260506 16:21:23.349 | INFO | RTMetrics report populated | 20260506 16:21:23.349 | INFO | Opening tmp RTMetrics json file | 20260506 16:21:23.368 | INFO | Serializing RTMetrics to json | 20260506 16:21:23.381 | INFO | Serialization done, swapping tmp file now... | 20260506 16:21:23.410 | INFO | RTMetrics done | 20260506 16:21:23.908 | INFO | Finished a round of Baz IO: Wrote 2.09 GiB in 3.695 seconds | 20260506 16:21:23.908 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.41% / 0.01% / 63.58% | 20260506 16:21:23.908 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.72131% / 3.97364e-06% / 7.94729e-05% | 20260506 16:21:23.908 | INFO | Baz IO overall: | 20260506 16:21:23.908 | INFO | BytesWritten : 1441972352801 | 20260506 16:21:23.908 | INFO | NumEvents : 697776833488 | 20260506 16:21:25.228 | INFO | Analyzing chunk frames = [2748416,2748928) | 20260506 16:21:25.228 | INFO | MemInfo: { | 20260506 16:21:25.228 | INFO | "gpu" : " 17 GB", | 20260506 16:21:25.228 | INFO | "huge" : "", | 20260506 16:21:25.228 | INFO | "huge_shared" : " 27 GB", | 20260506 16:21:25.228 | INFO | "malloc" : " 14 GB", | 20260506 16:21:25.228 | INFO | "pinned" : " 29 GB" | 20260506 16:21:25.229 | INFO | } | 20260506 16:21:25.229 | INFO | Perf report for the last 5.13656 seconds: | 20260506 16:21:25.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:21:25.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:25.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:25.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:25.229 | INFO | ANALYSIS : 62.8%, 2.9, 95.2ms, 0 | 20260506 16:21:25.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:25.229 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260506 16:21:25.229 | INFO | PRE_HQ : 60.5%, 1.9, 58.9ms, 0 | 20260506 16:21:25.229 | INFO | BAZWRITER : 71.5%, 4.8, 175.8ms, 0 | 20260506 16:21:25.229 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260506 16:21:25.229 | INFO | Uploaded 36GiB to GPU in 2.05336s (17.5323GiB/s) | 20260506 16:21:25.229 | INFO | Downloaded 38GiB to GPU in 2.20559s (17.2289GiB/s) | 20260506 16:21:30.371 | INFO | Analyzing chunk frames = [2748928,2749440) | 20260506 16:21:30.371 | INFO | MemInfo: { | 20260506 16:21:30.371 | INFO | "gpu" : " 17 GB", | 20260506 16:21:30.371 | INFO | "huge" : "", | 20260506 16:21:30.371 | INFO | "huge_shared" : " 27 GB", | 20260506 16:21:30.371 | INFO | "malloc" : " 14 GB", | 20260506 16:21:30.371 | INFO | "pinned" : " 29 GB" | 20260506 16:21:30.371 | INFO | } | 20260506 16:21:30.371 | INFO | Perf report for the last 5.14243 seconds: | 20260506 16:21:30.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:21:30.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:30.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:30.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:30.371 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 16:21:30.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:30.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:30.371 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:21:30.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:30.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:30.371 | INFO | Uploaded 36GiB to GPU in 1.99246s (18.0681GiB/s) | 20260506 16:21:30.371 | INFO | Downloaded 34GiB to GPU in 1.95317s (17.4076GiB/s) | 20260506 16:21:35.517 | INFO | Analyzing chunk frames = [2749440,2749952) | 20260506 16:21:35.517 | INFO | MemInfo: { | 20260506 16:21:35.517 | INFO | "gpu" : " 17 GB", | 20260506 16:21:35.517 | INFO | "huge" : "", | 20260506 16:21:35.517 | INFO | "huge_shared" : " 27 GB", | 20260506 16:21:35.517 | INFO | "malloc" : " 14 GB", | 20260506 16:21:35.517 | INFO | "pinned" : " 29 GB" | 20260506 16:21:35.517 | INFO | } | 20260506 16:21:35.517 | INFO | Perf report for the last 5.14582 seconds: | 20260506 16:21:35.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:21:35.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:35.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:35.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:35.517 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 16:21:35.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:35.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:35.517 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:21:35.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:35.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:35.517 | INFO | Uploaded 36GiB to GPU in 1.98023s (18.1797GiB/s) | 20260506 16:21:35.517 | INFO | Downloaded 34GiB to GPU in 1.93069s (17.6102GiB/s) | 20260506 16:21:40.653 | INFO | Analyzing chunk frames = [2749952,2750464) | 20260506 16:21:40.653 | INFO | MemInfo: { | 20260506 16:21:40.653 | INFO | "gpu" : " 17 GB", | 20260506 16:21:40.653 | INFO | "huge" : "", | 20260506 16:21:40.653 | INFO | "huge_shared" : " 27 GB", | 20260506 16:21:40.653 | INFO | "malloc" : " 14 GB", | 20260506 16:21:40.653 | INFO | "pinned" : " 29 GB" | 20260506 16:21:40.653 | INFO | } | 20260506 16:21:40.653 | INFO | Perf report for the last 5.13636 seconds: | 20260506 16:21:40.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:21:40.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:40.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:40.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:40.653 | INFO | ANALYSIS : 60.1%, 3.0, 93.0ms, 0 | 20260506 16:21:40.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:40.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:40.653 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:21:40.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:40.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:40.653 | INFO | Uploaded 36GiB to GPU in 1.9515s (18.4473GiB/s) | 20260506 16:21:40.654 | INFO | Downloaded 34GiB to GPU in 1.87353s (18.1476GiB/s) | 20260506 16:21:45.793 | INFO | Analyzing chunk frames = [2750464,2750976) | 20260506 16:21:45.793 | INFO | MemInfo: { | 20260506 16:21:45.793 | INFO | "gpu" : " 17 GB", | 20260506 16:21:45.793 | INFO | "huge" : "", | 20260506 16:21:45.793 | INFO | "huge_shared" : " 27 GB", | 20260506 16:21:45.793 | INFO | "malloc" : " 14 GB", | 20260506 16:21:45.793 | INFO | "pinned" : " 29 GB" | 20260506 16:21:45.794 | INFO | } | 20260506 16:21:45.794 | INFO | Perf report for the last 5.14042 seconds: | 20260506 16:21:45.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:21:45.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:45.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:45.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:45.794 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 16:21:45.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:45.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:45.794 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:21:45.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:45.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:45.794 | INFO | Uploaded 36GiB to GPU in 2.0069s (17.9381GiB/s) | 20260506 16:21:45.794 | INFO | Downloaded 34GiB to GPU in 1.90688s (17.8302GiB/s) | 20260506 16:21:50.928 | INFO | Analyzing chunk frames = [2750976,2751488) | 20260506 16:21:50.928 | INFO | MemInfo: { | 20260506 16:21:50.928 | INFO | "gpu" : " 17 GB", | 20260506 16:21:50.928 | INFO | "huge" : "", | 20260506 16:21:50.928 | INFO | "huge_shared" : " 27 GB", | 20260506 16:21:50.928 | INFO | "malloc" : " 14 GB", | 20260506 16:21:50.928 | INFO | "pinned" : " 29 GB" | 20260506 16:21:50.928 | INFO | } | 20260506 16:21:50.928 | INFO | Perf report for the last 5.13429 seconds: | 20260506 16:21:50.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:21:50.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:50.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:50.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:50.928 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 16:21:50.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:50.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:50.928 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 16:21:50.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:50.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:50.928 | INFO | Uploaded 36GiB to GPU in 2.03784s (17.6657GiB/s) | 20260506 16:21:50.928 | INFO | Downloaded 34GiB to GPU in 1.93715s (17.5515GiB/s) | 20260506 16:21:56.068 | INFO | Analyzing chunk frames = [2751488,2752000) | 20260506 16:21:56.068 | INFO | MemInfo: { | 20260506 16:21:56.068 | INFO | "gpu" : " 17 GB", | 20260506 16:21:56.068 | INFO | "huge" : "", | 20260506 16:21:56.068 | INFO | "huge_shared" : " 27 GB", | 20260506 16:21:56.068 | INFO | "malloc" : " 14 GB", | 20260506 16:21:56.068 | INFO | "pinned" : " 29 GB" | 20260506 16:21:56.068 | INFO | } | 20260506 16:21:56.068 | INFO | Perf report for the last 5.13992 seconds: | 20260506 16:21:56.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:21:56.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:56.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:56.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:56.068 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:21:56.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:56.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:56.068 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:21:56.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:56.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:21:56.068 | INFO | Uploaded 36GiB to GPU in 1.99559s (18.0398GiB/s) | 20260506 16:21:56.068 | INFO | Downloaded 34GiB to GPU in 2.01107s (16.9065GiB/s) | 20260506 16:22:01.205 | INFO | Analyzing chunk frames = [2752000,2752512) | 20260506 16:22:01.205 | INFO | MemInfo: { | 20260506 16:22:01.205 | INFO | "gpu" : " 17 GB", | 20260506 16:22:01.205 | INFO | "huge" : "", | 20260506 16:22:01.205 | INFO | "huge_shared" : " 27 GB", | 20260506 16:22:01.206 | INFO | "malloc" : " 14 GB", | 20260506 16:22:01.206 | INFO | "pinned" : " 29 GB" | 20260506 16:22:01.206 | INFO | } | 20260506 16:22:01.206 | INFO | Perf report for the last 5.13772 seconds: | 20260506 16:22:01.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:22:01.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:01.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:01.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:01.206 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 16:22:01.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:01.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:01.206 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 16:22:01.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:01.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:01.206 | INFO | Uploaded 36GiB to GPU in 2.04996s (17.5613GiB/s) | 20260506 16:22:01.206 | INFO | Downloaded 34GiB to GPU in 1.98796s (17.103GiB/s) | 20260506 16:22:01.206 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:22:01.206 | INFO | Total mem usage: 18 GiB | 20260506 16:22:01.206 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:22:01.206 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -13.9%) | 20260506 16:22:01.206 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:22:01.206 | INFO | High Water Breakdown: | 20260506 16:22:01.206 | INFO | 61.5% : 9.71 GiB ( 85.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:22:01.206 | INFO | 38.4% : 6.07 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:22:01.206 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:22:01.206 | INFO | ------------------------------------------ | 20260506 16:22:01.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:22:01.206 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:22:01.206 | INFO | Total mem usage: 26.5 GiB | 20260506 16:22:01.206 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:22:01.206 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 16:22:01.206 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 16:22:01.206 | INFO | High Water Breakdown: | 20260506 16:22:01.206 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:22:01.206 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:22:01.207 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:22:01.207 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:22:01.207 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:22:01.207 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:22:01.207 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:22:01.207 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:22:01.207 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:22:01.207 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:22:01.207 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:22:01.207 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:22:01.207 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:22:01.207 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:22:01.207 | INFO | ------------------------------------------ | 20260506 16:22:01.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:22:01.207 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:22:01.207 | INFO | Total mem usage: 25.6 GiB | 20260506 16:22:01.207 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:22:01.207 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:22:01.207 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:22:01.207 | INFO | High Water Breakdown: | 20260506 16:22:01.207 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:22:01.207 | INFO | ------------------------------------------ | 20260506 16:22:01.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:22:01.207 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:22:01.207 | INFO | Total mem usage: 15.7 GiB | 20260506 16:22:01.207 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:22:01.207 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:22:01.207 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:22:01.207 | INFO | High Water Breakdown: | 20260506 16:22:01.207 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:22:01.207 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:22:01.207 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:22:01.207 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:22:01.207 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:22:01.207 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:22:01.207 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:22:01.207 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:22:01.207 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:22:01.207 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:22:01.207 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:22:01.207 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:22:01.207 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:22:01.207 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:22:01.207 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:22:01.207 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:22:01.207 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:22:01.207 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:22:01.207 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:22:01.207 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:22:01.207 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:22:01.207 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:22:01.207 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:22:01.207 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:22:01.207 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:22:01.207 | INFO | ------------------------------------------ | 20260506 16:22:01.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:22:01.207 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:22:01.207 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:22:01.207 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:22:01.207 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.55e+01, 1.33e+01 / 2.75e+00 | 20260506 16:22:01.207 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.01e-01, 1.05e-02 / 6.22e-03 | 20260506 16:22:01.207 | INFO | PerfReport: 11.61%, Baseline -- 3.64e+00 / 4.39e+00, 5.68e-01 / 3.80e-01 | 20260506 16:22:01.207 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.47e+00, 1.89e-01 / 1.43e-01 | 20260506 16:22:01.208 | INFO | PerfReport: 15.91%, DME -- 3.99e+01 / 4.44e+01, 3.95e+00 / 2.72e+00 | 20260506 16:22:01.208 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.42e+00 / 2.06e+00 | 20260506 16:22:01.208 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.30e+00 / 2.81e+00, 3.87e-01 / 2.87e-01 | 20260506 16:22:01.208 | INFO | PerfReport: 3.51%, Metrics -- 1.10e+00 / 1.47e+00, 2.85e-01 / 1.91e-01 | 20260506 16:22:01.208 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.82e-01 / 4.32e-01, 5.95e-02 / 2.51e-02 | 20260506 16:22:01.208 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:22:01.208 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:22:01.208 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:22:01.208 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 16:22:01.208 | INFO | PerfReport: 21.49%, Upload -- 2.09e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 16:22:01.208 | INFO | PerfReport: 21.13%, Download -- 2.06e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 16:22:01.208 | INFO | PerfReport: 32.27%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 16:22:01.208 | INFO | PerfReport: 1.47%, LockUpload -- 1.43e+00 / 0.00e+00, 5.86e+00 / 0.00e+00 | 20260506 16:22:01.208 | INFO | PerfReport: 1.07%, LockDownload -- 1.04e+00 / 0.00e+00, 2.65e+00 / 0.00e+00 | 20260506 16:22:01.208 | INFO | PerfReport: 22.56%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 16:22:01.208 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260506 16:22:01.319 | INFO | Starting a round of Baz IO | 20260506 16:22:04.473 | INFO | Populating RTMetrics report | 20260506 16:22:04.473 | INFO | RTMetrics report populated | 20260506 16:22:04.473 | INFO | Opening tmp RTMetrics json file | 20260506 16:22:04.481 | INFO | Serializing RTMetrics to json | 20260506 16:22:04.484 | INFO | Serialization done, swapping tmp file now... | 20260506 16:22:04.486 | INFO | RTMetrics done | 20260506 16:22:04.913 | INFO | Finished a round of Baz IO: Wrote 2.091 GiB in 3.593 seconds | 20260506 16:22:04.913 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.40% / 0.01% / 63.59% | 20260506 16:22:04.913 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.719% / 1.19209e-05% / 8.34465e-05% | 20260506 16:22:04.913 | INFO | Baz IO overall: | 20260506 16:22:04.913 | INFO | BytesWritten : 1444217263973 | 20260506 16:22:04.913 | INFO | NumEvents : 698861938662 | 20260506 16:22:06.350 | INFO | Analyzing chunk frames = [2752512,2753024) | 20260506 16:22:06.350 | INFO | MemInfo: { | 20260506 16:22:06.351 | INFO | "gpu" : " 17 GB", | 20260506 16:22:06.351 | INFO | "huge" : "", | 20260506 16:22:06.351 | INFO | "huge_shared" : " 27 GB", | 20260506 16:22:06.351 | INFO | "malloc" : " 14 GB", | 20260506 16:22:06.351 | INFO | "pinned" : " 29 GB" | 20260506 16:22:06.351 | INFO | } | 20260506 16:22:06.351 | INFO | Perf report for the last 5.14505 seconds: | 20260506 16:22:06.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:22:06.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:06.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:06.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:06.351 | INFO | ANALYSIS : 62.9%, 2.9, 95.4ms, 0 | 20260506 16:22:06.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:06.351 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 16:22:06.351 | INFO | PRE_HQ : 60.6%, 1.9, 58.9ms, 0 | 20260506 16:22:06.351 | INFO | BAZWRITER : 69.2%, 4.8, 171.9ms, 0 | 20260506 16:22:06.351 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 16:22:06.351 | INFO | Uploaded 36GiB to GPU in 2.03021s (17.7321GiB/s) | 20260506 16:22:06.351 | INFO | Downloaded 38GiB to GPU in 2.22452s (17.0823GiB/s) | 20260506 16:22:11.489 | INFO | Analyzing chunk frames = [2753024,2753536) | 20260506 16:22:11.489 | INFO | MemInfo: { | 20260506 16:22:11.489 | INFO | "gpu" : " 17 GB", | 20260506 16:22:11.489 | INFO | "huge" : "", | 20260506 16:22:11.489 | INFO | "huge_shared" : " 27 GB", | 20260506 16:22:11.489 | INFO | "malloc" : " 14 GB", | 20260506 16:22:11.489 | INFO | "pinned" : " 29 GB" | 20260506 16:22:11.489 | INFO | } | 20260506 16:22:11.489 | INFO | Perf report for the last 5.13809 seconds: | 20260506 16:22:11.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:22:11.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:11.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:11.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:11.489 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:22:11.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:11.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:11.489 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:22:11.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:11.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:11.489 | INFO | Uploaded 36GiB to GPU in 1.99s (18.0905GiB/s) | 20260506 16:22:11.489 | INFO | Downloaded 34GiB to GPU in 1.92785s (17.6362GiB/s) | 20260506 16:22:16.628 | INFO | Analyzing chunk frames = [2753536,2754048) | 20260506 16:22:16.628 | INFO | MemInfo: { | 20260506 16:22:16.628 | INFO | "gpu" : " 17 GB", | 20260506 16:22:16.628 | INFO | "huge" : "", | 20260506 16:22:16.628 | INFO | "huge_shared" : " 27 GB", | 20260506 16:22:16.628 | INFO | "malloc" : " 14 GB", | 20260506 16:22:16.628 | INFO | "pinned" : " 29 GB" | 20260506 16:22:16.628 | INFO | } | 20260506 16:22:16.628 | INFO | Perf report for the last 5.13899 seconds: | 20260506 16:22:16.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:22:16.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:16.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:16.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:16.628 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 16:22:16.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:16.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:16.628 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:22:16.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:16.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:16.628 | INFO | Uploaded 36GiB to GPU in 1.99137s (18.078GiB/s) | 20260506 16:22:16.628 | INFO | Downloaded 34GiB to GPU in 1.93311s (17.5882GiB/s) | 20260506 16:22:21.767 | INFO | Analyzing chunk frames = [2754048,2754560) | 20260506 16:22:21.767 | INFO | MemInfo: { | 20260506 16:22:21.767 | INFO | "gpu" : " 17 GB", | 20260506 16:22:21.767 | INFO | "huge" : "", | 20260506 16:22:21.767 | INFO | "huge_shared" : " 27 GB", | 20260506 16:22:21.767 | INFO | "malloc" : " 14 GB", | 20260506 16:22:21.767 | INFO | "pinned" : " 29 GB" | 20260506 16:22:21.767 | INFO | } | 20260506 16:22:21.767 | INFO | Perf report for the last 5.13918 seconds: | 20260506 16:22:21.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:22:21.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:21.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:21.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:21.767 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:22:21.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:21.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:21.767 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:22:21.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:21.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:21.767 | INFO | Uploaded 36GiB to GPU in 1.93235s (18.6301GiB/s) | 20260506 16:22:21.767 | INFO | Downloaded 34GiB to GPU in 1.87419s (18.1412GiB/s) | 20260506 16:22:26.906 | INFO | Analyzing chunk frames = [2754560,2755072) | 20260506 16:22:26.906 | INFO | MemInfo: { | 20260506 16:22:26.906 | INFO | "gpu" : " 17 GB", | 20260506 16:22:26.906 | INFO | "huge" : "", | 20260506 16:22:26.906 | INFO | "huge_shared" : " 27 GB", | 20260506 16:22:26.906 | INFO | "malloc" : " 14 GB", | 20260506 16:22:26.906 | INFO | "pinned" : " 29 GB" | 20260506 16:22:26.906 | INFO | } | 20260506 16:22:26.906 | INFO | Perf report for the last 5.13919 seconds: | 20260506 16:22:26.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:22:26.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:26.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:26.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:26.906 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 16:22:26.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:26.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:26.906 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:22:26.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:26.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:26.906 | INFO | Uploaded 36GiB to GPU in 1.97664s (18.2128GiB/s) | 20260506 16:22:26.907 | INFO | Downloaded 34GiB to GPU in 1.92179s (17.6919GiB/s) | 20260506 16:22:32.041 | INFO | Analyzing chunk frames = [2755072,2755584) | 20260506 16:22:32.041 | INFO | MemInfo: { | 20260506 16:22:32.041 | INFO | "gpu" : " 17 GB", | 20260506 16:22:32.041 | INFO | "huge" : "", | 20260506 16:22:32.041 | INFO | "huge_shared" : " 27 GB", | 20260506 16:22:32.041 | INFO | "malloc" : " 14 GB", | 20260506 16:22:32.041 | INFO | "pinned" : " 29 GB" | 20260506 16:22:32.041 | INFO | } | 20260506 16:22:32.041 | INFO | Perf report for the last 5.1352 seconds: | 20260506 16:22:32.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:22:32.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:32.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:32.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:32.041 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 16:22:32.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:32.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:32.042 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:22:32.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:32.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:32.042 | INFO | Uploaded 36GiB to GPU in 1.98784s (18.1101GiB/s) | 20260506 16:22:32.042 | INFO | Downloaded 34GiB to GPU in 1.93539s (17.5675GiB/s) | 20260506 16:22:37.179 | INFO | Analyzing chunk frames = [2755584,2756096) | 20260506 16:22:37.179 | INFO | MemInfo: { | 20260506 16:22:37.179 | INFO | "gpu" : " 17 GB", | 20260506 16:22:37.179 | INFO | "huge" : "", | 20260506 16:22:37.179 | INFO | "huge_shared" : " 27 GB", | 20260506 16:22:37.179 | INFO | "malloc" : " 14 GB", | 20260506 16:22:37.179 | INFO | "pinned" : " 29 GB" | 20260506 16:22:37.179 | INFO | } | 20260506 16:22:37.179 | INFO | Perf report for the last 5.13756 seconds: | 20260506 16:22:37.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:22:37.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:37.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:37.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:37.179 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 16:22:37.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:37.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:37.179 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 16:22:37.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:37.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:37.179 | INFO | Uploaded 36GiB to GPU in 2.045s (17.6039GiB/s) | 20260506 16:22:37.179 | INFO | Downloaded 34GiB to GPU in 2.00098s (16.9917GiB/s) | 20260506 16:22:42.319 | INFO | Analyzing chunk frames = [2756096,2756608) | 20260506 16:22:42.319 | INFO | MemInfo: { | 20260506 16:22:42.319 | INFO | "gpu" : " 17 GB", | 20260506 16:22:42.319 | INFO | "huge" : "", | 20260506 16:22:42.319 | INFO | "huge_shared" : " 27 GB", | 20260506 16:22:42.319 | INFO | "malloc" : " 14 GB", | 20260506 16:22:42.319 | INFO | "pinned" : " 29 GB" | 20260506 16:22:42.319 | INFO | } | 20260506 16:22:42.319 | INFO | Perf report for the last 5.1402 seconds: | 20260506 16:22:42.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:22:42.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:42.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:42.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:42.319 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 16:22:42.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:42.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:42.319 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 16:22:42.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:42.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:42.320 | INFO | Uploaded 36GiB to GPU in 2.07627s (17.3388GiB/s) | 20260506 16:22:42.320 | INFO | Downloaded 34GiB to GPU in 1.97099s (17.2502GiB/s) | 20260506 16:22:42.438 | INFO | Starting a round of Baz IO | 20260506 16:22:45.607 | INFO | Populating RTMetrics report | 20260506 16:22:45.608 | INFO | RTMetrics report populated | 20260506 16:22:45.608 | INFO | Opening tmp RTMetrics json file | 20260506 16:22:45.676 | INFO | Serializing RTMetrics to json | 20260506 16:22:45.737 | INFO | Serialization done, swapping tmp file now... | 20260506 16:22:45.763 | INFO | RTMetrics done | 20260506 16:22:46.083 | INFO | Finished a round of Baz IO: Wrote 2.089 GiB in 3.645 seconds | 20260506 16:22:46.083 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.39% / 0.01% / 63.60% | 20260506 16:22:46.083 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.7115% / 1.19209e-05% / 7.15256e-05% | 20260506 16:22:46.083 | INFO | Baz IO overall: | 20260506 16:22:46.083 | INFO | BytesWritten : 1446460219936 | 20260506 16:22:46.083 | INFO | NumEvents : 699945732845 | 20260506 16:22:47.463 | INFO | Analyzing chunk frames = [2756608,2757120) | 20260506 16:22:47.463 | INFO | MemInfo: { | 20260506 16:22:47.463 | INFO | "gpu" : " 17 GB", | 20260506 16:22:47.463 | INFO | "huge" : "", | 20260506 16:22:47.463 | INFO | "huge_shared" : " 27 GB", | 20260506 16:22:47.463 | INFO | "malloc" : " 14 GB", | 20260506 16:22:47.463 | INFO | "pinned" : " 29 GB" | 20260506 16:22:47.463 | INFO | } | 20260506 16:22:47.463 | INFO | Perf report for the last 5.1435 seconds: | 20260506 16:22:47.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:22:47.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:47.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:47.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:47.463 | INFO | ANALYSIS : 63.0%, 2.9, 95.0ms, 0 | 20260506 16:22:47.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:47.463 | INFO | RT_METRICS : 49.1%, 1.0, 25.8ms, 0 | 20260506 16:22:47.463 | INFO | PRE_HQ : 61.2%, 2.0, 62.7ms, 0 | 20260506 16:22:47.463 | INFO | BAZWRITER : 70.3%, 4.8, 173.3ms, 0 | 20260506 16:22:47.463 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 16:22:47.463 | INFO | Uploaded 36GiB to GPU in 2.07599s (17.3411GiB/s) | 20260506 16:22:47.463 | INFO | Downloaded 38GiB to GPU in 2.20456s (17.237GiB/s) | 20260506 16:22:52.613 | INFO | Analyzing chunk frames = [2757120,2757632) | 20260506 16:22:52.613 | INFO | MemInfo: { | 20260506 16:22:52.613 | INFO | "gpu" : " 17 GB", | 20260506 16:22:52.613 | INFO | "huge" : "", | 20260506 16:22:52.613 | INFO | "huge_shared" : " 27 GB", | 20260506 16:22:52.613 | INFO | "malloc" : " 14 GB", | 20260506 16:22:52.613 | INFO | "pinned" : " 29 GB" | 20260506 16:22:52.613 | INFO | } | 20260506 16:22:52.613 | INFO | Perf report for the last 5.15061 seconds: | 20260506 16:22:52.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:22:52.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:52.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:52.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:52.613 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 16:22:52.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:52.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:52.613 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 16:22:52.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:52.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:52.614 | INFO | Uploaded 36GiB to GPU in 2.00198s (17.9822GiB/s) | 20260506 16:22:52.614 | INFO | Downloaded 34GiB to GPU in 1.94418s (17.4881GiB/s) | 20260506 16:22:57.746 | INFO | Analyzing chunk frames = [2757632,2758144) | 20260506 16:22:57.746 | INFO | MemInfo: { | 20260506 16:22:57.746 | INFO | "gpu" : " 17 GB", | 20260506 16:22:57.746 | INFO | "huge" : "", | 20260506 16:22:57.746 | INFO | "huge_shared" : " 27 GB", | 20260506 16:22:57.746 | INFO | "malloc" : " 14 GB", | 20260506 16:22:57.746 | INFO | "pinned" : " 29 GB" | 20260506 16:22:57.746 | INFO | } | 20260506 16:22:57.746 | INFO | Perf report for the last 5.13279 seconds: | 20260506 16:22:57.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:22:57.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:57.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:57.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:57.746 | INFO | ANALYSIS : 61.0%, 3.0, 94.3ms, 0 | 20260506 16:22:57.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:57.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:57.746 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:22:57.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:57.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:22:57.746 | INFO | Uploaded 36GiB to GPU in 1.97801s (18.2001GiB/s) | 20260506 16:22:57.747 | INFO | Downloaded 34GiB to GPU in 1.92457s (17.6663GiB/s) | 20260506 16:23:02.881 | INFO | Analyzing chunk frames = [2758144,2758656) | 20260506 16:23:02.881 | INFO | MemInfo: { | 20260506 16:23:02.881 | INFO | "gpu" : " 17 GB", | 20260506 16:23:02.881 | INFO | "huge" : "", | 20260506 16:23:02.881 | INFO | "huge_shared" : " 27 GB", | 20260506 16:23:02.881 | INFO | "malloc" : " 14 GB", | 20260506 16:23:02.881 | INFO | "pinned" : " 29 GB" | 20260506 16:23:02.881 | INFO | } | 20260506 16:23:02.881 | INFO | Perf report for the last 5.13532 seconds: | 20260506 16:23:02.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:23:02.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:02.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:02.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:02.881 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 16:23:02.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:02.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:02.882 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:23:02.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:02.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:02.882 | INFO | Uploaded 36GiB to GPU in 1.93597s (18.5953GiB/s) | 20260506 16:23:02.882 | INFO | Downloaded 34GiB to GPU in 1.87401s (18.1429GiB/s) | 20260506 16:23:08.019 | INFO | Analyzing chunk frames = [2758656,2759168) | 20260506 16:23:08.019 | INFO | MemInfo: { | 20260506 16:23:08.019 | INFO | "gpu" : " 17 GB", | 20260506 16:23:08.019 | INFO | "huge" : "", | 20260506 16:23:08.019 | INFO | "huge_shared" : " 27 GB", | 20260506 16:23:08.019 | INFO | "malloc" : " 14 GB", | 20260506 16:23:08.019 | INFO | "pinned" : " 29 GB" | 20260506 16:23:08.019 | INFO | } | 20260506 16:23:08.019 | INFO | Perf report for the last 5.13751 seconds: | 20260506 16:23:08.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:23:08.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:08.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:08.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:08.019 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 16:23:08.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:08.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:08.019 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:23:08.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:08.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:08.019 | INFO | Uploaded 36GiB to GPU in 2.01787s (17.8406GiB/s) | 20260506 16:23:08.019 | INFO | Downloaded 34GiB to GPU in 1.91175s (17.7847GiB/s) | 20260506 16:23:13.155 | INFO | Analyzing chunk frames = [2759168,2759680) | 20260506 16:23:13.155 | INFO | MemInfo: { | 20260506 16:23:13.155 | INFO | "gpu" : " 17 GB", | 20260506 16:23:13.155 | INFO | "huge" : "", | 20260506 16:23:13.155 | INFO | "huge_shared" : " 27 GB", | 20260506 16:23:13.155 | INFO | "malloc" : " 14 GB", | 20260506 16:23:13.155 | INFO | "pinned" : " 29 GB" | 20260506 16:23:13.155 | INFO | } | 20260506 16:23:13.155 | INFO | Perf report for the last 5.13586 seconds: | 20260506 16:23:13.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:23:13.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:13.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:13.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:13.155 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 16:23:13.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:13.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:13.155 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:23:13.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:13.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:13.155 | INFO | Uploaded 36GiB to GPU in 2.01697s (17.8486GiB/s) | 20260506 16:23:13.155 | INFO | Downloaded 34GiB to GPU in 1.92734s (17.6409GiB/s) | 20260506 16:23:18.296 | INFO | Analyzing chunk frames = [2759680,2760192) | 20260506 16:23:18.296 | INFO | MemInfo: { | 20260506 16:23:18.296 | INFO | "gpu" : " 17 GB", | 20260506 16:23:18.296 | INFO | "huge" : "", | 20260506 16:23:18.296 | INFO | "huge_shared" : " 27 GB", | 20260506 16:23:18.296 | INFO | "malloc" : " 14 GB", | 20260506 16:23:18.296 | INFO | "pinned" : " 29 GB" | 20260506 16:23:18.296 | INFO | } | 20260506 16:23:18.296 | INFO | Perf report for the last 5.14105 seconds: | 20260506 16:23:18.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:23:18.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:18.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:18.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:18.296 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 16:23:18.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:18.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:18.296 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:23:18.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:18.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:18.296 | INFO | Uploaded 36GiB to GPU in 1.99883s (18.0105GiB/s) | 20260506 16:23:18.296 | INFO | Downloaded 34GiB to GPU in 1.96334s (17.3174GiB/s) | 20260506 16:23:23.442 | INFO | Analyzing chunk frames = [2760192,2760704) | 20260506 16:23:23.442 | INFO | MemInfo: { | 20260506 16:23:23.442 | INFO | "gpu" : " 17 GB", | 20260506 16:23:23.442 | INFO | "huge" : "", | 20260506 16:23:23.442 | INFO | "huge_shared" : " 27 GB", | 20260506 16:23:23.442 | INFO | "malloc" : " 14 GB", | 20260506 16:23:23.442 | INFO | "pinned" : " 29 GB" | 20260506 16:23:23.442 | INFO | } | 20260506 16:23:23.442 | INFO | Perf report for the last 5.14625 seconds: | 20260506 16:23:23.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:23:23.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:23.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:23.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:23.442 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 16:23:23.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:23.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:23.442 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 16:23:23.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:23.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:23.442 | INFO | Uploaded 36GiB to GPU in 2.00422s (17.9621GiB/s) | 20260506 16:23:23.443 | INFO | Downloaded 34GiB to GPU in 1.96719s (17.2836GiB/s) | 20260506 16:23:23.443 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:23:23.443 | INFO | Total mem usage: 18 GiB | 20260506 16:23:23.443 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:23:23.443 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.4%) | 20260506 16:23:23.443 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:23:23.443 | INFO | High Water Breakdown: | 20260506 16:23:23.443 | INFO | 61.1% : 9.45 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:23:23.443 | INFO | 38.8% : 5.99 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:23:23.443 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:23:23.443 | INFO | ------------------------------------------ | 20260506 16:23:23.443 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:23:23.443 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:23:23.443 | INFO | Total mem usage: 26.5 GiB | 20260506 16:23:23.443 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:23:23.443 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:23:23.443 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:23:23.443 | INFO | High Water Breakdown: | 20260506 16:23:23.443 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:23:23.443 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:23:23.443 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:23:23.443 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:23:23.443 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:23:23.443 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:23:23.443 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:23:23.443 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:23:23.443 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:23:23.443 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:23:23.443 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:23:23.443 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:23:23.443 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:23:23.443 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:23:23.443 | INFO | ------------------------------------------ | 20260506 16:23:23.443 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:23:23.443 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:23:23.443 | INFO | Total mem usage: 25.6 GiB | 20260506 16:23:23.443 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:23:23.443 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:23:23.443 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:23:23.443 | INFO | High Water Breakdown: | 20260506 16:23:23.443 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:23:23.443 | INFO | ------------------------------------------ | 20260506 16:23:23.443 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:23:23.443 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:23:23.443 | INFO | Total mem usage: 15.7 GiB | 20260506 16:23:23.443 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:23:23.443 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:23:23.443 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 16:23:23.443 | INFO | High Water Breakdown: | 20260506 16:23:23.443 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:23:23.443 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:23:23.443 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:23:23.443 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:23:23.443 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:23:23.443 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:23:23.443 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:23:23.443 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:23:23.443 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:23:23.443 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:23:23.443 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:23:23.443 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:23:23.443 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:23:23.443 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:23:23.443 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:23:23.443 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:23:23.443 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:23:23.443 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:23:23.443 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:23:23.443 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:23:23.443 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:23:23.443 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:23:23.443 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:23:23.443 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:23:23.443 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:23:23.443 | INFO | ------------------------------------------ | 20260506 16:23:23.444 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:23:23.444 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:23:23.444 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:23:23.444 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:23:23.444 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.59e+01, 1.33e+01 / 2.13e+00 | 20260506 16:23:23.444 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.02e-01, 1.07e-02 / 7.37e-03 | 20260506 16:23:23.444 | INFO | PerfReport: 11.61%, Baseline -- 3.65e+00 / 4.43e+00, 5.84e-01 / 3.41e-01 | 20260506 16:23:23.444 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.47e+00, 1.89e-01 / 1.26e-01 | 20260506 16:23:23.444 | INFO | PerfReport: 15.92%, DME -- 4.00e+01 / 4.46e+01, 3.99e+00 / 2.45e+00 | 20260506 16:23:23.444 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.50e+00 / 1.96e+00 | 20260506 16:23:23.444 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.29e+00 / 2.90e+00, 3.84e-01 / 2.27e-01 | 20260506 16:23:23.444 | INFO | PerfReport: 3.50%, Metrics -- 1.10e+00 / 1.48e+00, 2.89e-01 / 2.01e-01 | 20260506 16:23:23.444 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.83e-01 / 4.35e-01, 6.07e-02 / 2.48e-02 | 20260506 16:23:23.444 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:23:23.444 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:23:23.444 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:23:23.444 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 16:23:23.444 | INFO | PerfReport: 21.44%, Upload -- 2.09e+01 / 0.00e+00, 7.04e+00 / 0.00e+00 | 20260506 16:23:23.444 | INFO | PerfReport: 21.07%, Download -- 2.05e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 16:23:23.444 | INFO | PerfReport: 32.30%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 16:23:23.444 | INFO | PerfReport: 1.45%, LockUpload -- 1.41e+00 / 0.00e+00, 5.86e+00 / 0.00e+00 | 20260506 16:23:23.444 | INFO | PerfReport: 1.08%, LockDownload -- 1.06e+00 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260506 16:23:23.444 | INFO | PerfReport: 22.63%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 16:23:23.444 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260506 16:23:23.556 | INFO | Starting a round of Baz IO | 20260506 16:23:26.702 | INFO | Populating RTMetrics report | 20260506 16:23:26.703 | INFO | RTMetrics report populated | 20260506 16:23:26.703 | INFO | Opening tmp RTMetrics json file | 20260506 16:23:26.711 | INFO | Serializing RTMetrics to json | 20260506 16:23:26.799 | INFO | Serialization done, swapping tmp file now... | 20260506 16:23:26.803 | INFO | RTMetrics done | 20260506 16:23:27.130 | INFO | Finished a round of Baz IO: Wrote 2.09 GiB in 3.573 seconds | 20260506 16:23:27.130 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.38% / 0.01% / 63.61% | 20260506 16:23:27.130 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.70803% / 7.94729e-06% / 7.94729e-05% | 20260506 16:23:27.130 | INFO | Baz IO overall: | 20260506 16:23:27.130 | INFO | BytesWritten : 1448704654426 | 20260506 16:23:27.130 | INFO | NumEvents : 701030675623 | 20260506 16:23:28.576 | INFO | Analyzing chunk frames = [2760704,2761216) | 20260506 16:23:28.576 | INFO | MemInfo: { | 20260506 16:23:28.576 | INFO | "gpu" : " 17 GB", | 20260506 16:23:28.576 | INFO | "huge" : "", | 20260506 16:23:28.576 | INFO | "huge_shared" : " 27 GB", | 20260506 16:23:28.576 | INFO | "malloc" : " 14 GB", | 20260506 16:23:28.576 | INFO | "pinned" : " 29 GB" | 20260506 16:23:28.576 | INFO | } | 20260506 16:23:28.576 | INFO | Perf report for the last 5.13407 seconds: | 20260506 16:23:28.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:23:28.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:28.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:28.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:28.576 | INFO | ANALYSIS : 62.9%, 2.9, 95.4ms, 0 | 20260506 16:23:28.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:28.576 | INFO | RT_METRICS : 45.7%, 1.0, 23.9ms, 0 | 20260506 16:23:28.576 | INFO | PRE_HQ : 60.7%, 1.9, 58.9ms, 0 | 20260506 16:23:28.576 | INFO | BAZWRITER : 69.2%, 4.8, 170.9ms, 0 | 20260506 16:23:28.576 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 16:23:28.577 | INFO | Uploaded 36GiB to GPU in 2.05292s (17.536GiB/s) | 20260506 16:23:28.577 | INFO | Downloaded 38GiB to GPU in 2.21553s (17.1517GiB/s) | 20260506 16:23:33.718 | INFO | Analyzing chunk frames = [2761216,2761728) | 20260506 16:23:33.718 | INFO | MemInfo: { | 20260506 16:23:33.718 | INFO | "gpu" : " 17 GB", | 20260506 16:23:33.718 | INFO | "huge" : "", | 20260506 16:23:33.718 | INFO | "huge_shared" : " 27 GB", | 20260506 16:23:33.718 | INFO | "malloc" : " 14 GB", | 20260506 16:23:33.718 | INFO | "pinned" : " 29 GB" | 20260506 16:23:33.718 | INFO | } | 20260506 16:23:33.718 | INFO | Perf report for the last 5.14161 seconds: | 20260506 16:23:33.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:23:33.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:33.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:33.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:33.718 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:23:33.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:33.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:33.718 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:23:33.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:33.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:33.718 | INFO | Uploaded 36GiB to GPU in 1.9951s (18.0442GiB/s) | 20260506 16:23:33.718 | INFO | Downloaded 34GiB to GPU in 1.93152s (17.6027GiB/s) | 20260506 16:23:38.856 | INFO | Analyzing chunk frames = [2761728,2762240) | 20260506 16:23:38.856 | INFO | MemInfo: { | 20260506 16:23:38.856 | INFO | "gpu" : " 17 GB", | 20260506 16:23:38.856 | INFO | "huge" : "", | 20260506 16:23:38.856 | INFO | "huge_shared" : " 27 GB", | 20260506 16:23:38.856 | INFO | "malloc" : " 14 GB", | 20260506 16:23:38.856 | INFO | "pinned" : " 29 GB" | 20260506 16:23:38.856 | INFO | } | 20260506 16:23:38.856 | INFO | Perf report for the last 5.13842 seconds: | 20260506 16:23:38.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:23:38.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:38.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:38.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:38.856 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 16:23:38.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:38.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:38.856 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:23:38.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:38.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:38.857 | INFO | Uploaded 36GiB to GPU in 1.98763s (18.112GiB/s) | 20260506 16:23:38.857 | INFO | Downloaded 34GiB to GPU in 1.9275s (17.6394GiB/s) | 20260506 16:23:43.994 | INFO | Analyzing chunk frames = [2762240,2762752) | 20260506 16:23:43.994 | INFO | MemInfo: { | 20260506 16:23:43.994 | INFO | "gpu" : " 17 GB", | 20260506 16:23:43.994 | INFO | "huge" : "", | 20260506 16:23:43.994 | INFO | "huge_shared" : " 27 GB", | 20260506 16:23:43.994 | INFO | "malloc" : " 14 GB", | 20260506 16:23:43.994 | INFO | "pinned" : " 29 GB" | 20260506 16:23:43.994 | INFO | } | 20260506 16:23:43.994 | INFO | Perf report for the last 5.13749 seconds: | 20260506 16:23:43.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:23:43.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:43.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:43.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:43.994 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260506 16:23:43.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:43.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:43.994 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:23:43.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:43.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:43.994 | INFO | Uploaded 36GiB to GPU in 1.94158s (18.5416GiB/s) | 20260506 16:23:43.994 | INFO | Downloaded 34GiB to GPU in 1.88196s (18.0663GiB/s) | 20260506 16:23:49.130 | INFO | Analyzing chunk frames = [2762752,2763264) | 20260506 16:23:49.130 | INFO | MemInfo: { | 20260506 16:23:49.130 | INFO | "gpu" : " 17 GB", | 20260506 16:23:49.130 | INFO | "huge" : "", | 20260506 16:23:49.130 | INFO | "huge_shared" : " 27 GB", | 20260506 16:23:49.130 | INFO | "malloc" : " 14 GB", | 20260506 16:23:49.130 | INFO | "pinned" : " 29 GB" | 20260506 16:23:49.130 | INFO | } | 20260506 16:23:49.131 | INFO | Perf report for the last 5.13666 seconds: | 20260506 16:23:49.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:23:49.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:49.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:49.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:49.131 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 16:23:49.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:49.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:49.131 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:23:49.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:49.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:49.131 | INFO | Uploaded 36GiB to GPU in 2.01991s (17.8226GiB/s) | 20260506 16:23:49.131 | INFO | Downloaded 34GiB to GPU in 1.91155s (17.7866GiB/s) | 20260506 16:23:54.270 | INFO | Analyzing chunk frames = [2763264,2763776) | 20260506 16:23:54.270 | INFO | MemInfo: { | 20260506 16:23:54.270 | INFO | "gpu" : " 17 GB", | 20260506 16:23:54.270 | INFO | "huge" : "", | 20260506 16:23:54.270 | INFO | "huge_shared" : " 27 GB", | 20260506 16:23:54.270 | INFO | "malloc" : " 14 GB", | 20260506 16:23:54.270 | INFO | "pinned" : " 29 GB" | 20260506 16:23:54.270 | INFO | } | 20260506 16:23:54.270 | INFO | Perf report for the last 5.13974 seconds: | 20260506 16:23:54.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:23:54.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:54.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:54.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:54.270 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:23:54.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:54.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:54.270 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 16:23:54.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:54.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:54.271 | INFO | Uploaded 36GiB to GPU in 2.08371s (17.2769GiB/s) | 20260506 16:23:54.271 | INFO | Downloaded 34GiB to GPU in 1.96336s (17.3173GiB/s) | 20260506 16:23:59.417 | INFO | Analyzing chunk frames = [2763776,2764288) | 20260506 16:23:59.417 | INFO | MemInfo: { | 20260506 16:23:59.417 | INFO | "gpu" : " 17 GB", | 20260506 16:23:59.417 | INFO | "huge" : "", | 20260506 16:23:59.417 | INFO | "huge_shared" : " 27 GB", | 20260506 16:23:59.417 | INFO | "malloc" : " 14 GB", | 20260506 16:23:59.417 | INFO | "pinned" : " 29 GB" | 20260506 16:23:59.417 | INFO | } | 20260506 16:23:59.417 | INFO | Perf report for the last 5.14677 seconds: | 20260506 16:23:59.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:23:59.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:59.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:59.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:59.417 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 16:23:59.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:59.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:59.417 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:23:59.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:59.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:23:59.417 | INFO | Uploaded 36GiB to GPU in 2.03814s (17.6632GiB/s) | 20260506 16:23:59.417 | INFO | Downloaded 34GiB to GPU in 2.0268s (16.7752GiB/s) | 20260506 16:24:04.557 | INFO | Analyzing chunk frames = [2764288,2764800) | 20260506 16:24:04.557 | INFO | MemInfo: { | 20260506 16:24:04.557 | INFO | "gpu" : " 17 GB", | 20260506 16:24:04.557 | INFO | "huge" : "", | 20260506 16:24:04.557 | INFO | "huge_shared" : " 27 GB", | 20260506 16:24:04.557 | INFO | "malloc" : " 14 GB", | 20260506 16:24:04.557 | INFO | "pinned" : " 29 GB" | 20260506 16:24:04.557 | INFO | } | 20260506 16:24:04.557 | INFO | Perf report for the last 5.13981 seconds: | 20260506 16:24:04.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:24:04.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:04.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:04.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:04.557 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:24:04.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:04.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:04.557 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:24:04.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:04.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:04.557 | INFO | Uploaded 36GiB to GPU in 2.01422s (17.8729GiB/s) | 20260506 16:24:04.557 | INFO | Downloaded 34GiB to GPU in 1.97646s (17.2025GiB/s) | 20260506 16:24:04.676 | INFO | Starting a round of Baz IO | 20260506 16:24:07.816 | INFO | Populating RTMetrics report | 20260506 16:24:07.816 | INFO | RTMetrics report populated | 20260506 16:24:07.816 | INFO | Opening tmp RTMetrics json file | 20260506 16:24:07.831 | INFO | Serializing RTMetrics to json | 20260506 16:24:07.920 | INFO | Serialization done, swapping tmp file now... | 20260506 16:24:07.952 | INFO | RTMetrics done | 20260506 16:24:08.419 | INFO | Finished a round of Baz IO: Wrote 2.088 GiB in 3.742 seconds | 20260506 16:24:08.419 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.38% / 0.01% / 63.62% | 20260506 16:24:08.419 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.7027% / 1.58946e-05% / 7.54992e-05% | 20260506 16:24:08.419 | INFO | Baz IO overall: | 20260506 16:24:08.419 | INFO | BytesWritten : 1450946529532 | 20260506 16:24:08.419 | INFO | NumEvents : 702114326174 | 20260506 16:24:09.702 | INFO | Analyzing chunk frames = [2764800,2765312) | 20260506 16:24:09.702 | INFO | MemInfo: { | 20260506 16:24:09.702 | INFO | "gpu" : " 17 GB", | 20260506 16:24:09.702 | INFO | "huge" : "", | 20260506 16:24:09.702 | INFO | "huge_shared" : " 27 GB", | 20260506 16:24:09.702 | INFO | "malloc" : " 14 GB", | 20260506 16:24:09.702 | INFO | "pinned" : " 29 GB" | 20260506 16:24:09.702 | INFO | } | 20260506 16:24:09.702 | INFO | Perf report for the last 5.14492 seconds: | 20260506 16:24:09.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:24:09.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:09.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:09.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:09.702 | INFO | ANALYSIS : 62.8%, 2.9, 95.4ms, 0 | 20260506 16:24:09.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:09.702 | INFO | RT_METRICS : 46.2%, 1.0, 24.3ms, 0 | 20260506 16:24:09.702 | INFO | PRE_HQ : 60.3%, 1.9, 58.6ms, 0 | 20260506 16:24:09.702 | INFO | BAZWRITER : 72.3%, 4.8, 178.2ms, 0 | 20260506 16:24:09.702 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.2ms, 0 | 20260506 16:24:09.702 | INFO | Uploaded 36GiB to GPU in 2.07788s (17.3254GiB/s) | 20260506 16:24:09.702 | INFO | Downloaded 38GiB to GPU in 2.21978s (17.1188GiB/s) | 20260506 16:24:14.830 | INFO | Analyzing chunk frames = [2765312,2765824) | 20260506 16:24:14.831 | INFO | MemInfo: { | 20260506 16:24:14.831 | INFO | "gpu" : " 17 GB", | 20260506 16:24:14.831 | INFO | "huge" : "", | 20260506 16:24:14.831 | INFO | "huge_shared" : " 27 GB", | 20260506 16:24:14.831 | INFO | "malloc" : " 14 GB", | 20260506 16:24:14.831 | INFO | "pinned" : " 29 GB" | 20260506 16:24:14.831 | INFO | } | 20260506 16:24:14.831 | INFO | Perf report for the last 5.129 seconds: | 20260506 16:24:14.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:24:14.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:14.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:14.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:14.831 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 16:24:14.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:14.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:14.831 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 16:24:14.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:14.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:14.831 | INFO | Uploaded 36GiB to GPU in 1.99615s (18.0348GiB/s) | 20260506 16:24:14.831 | INFO | Downloaded 34GiB to GPU in 1.93881s (17.5365GiB/s) | 20260506 16:24:19.967 | INFO | Analyzing chunk frames = [2765824,2766336) | 20260506 16:24:19.967 | INFO | MemInfo: { | 20260506 16:24:19.967 | INFO | "gpu" : " 17 GB", | 20260506 16:24:19.967 | INFO | "huge" : "", | 20260506 16:24:19.967 | INFO | "huge_shared" : " 27 GB", | 20260506 16:24:19.967 | INFO | "malloc" : " 14 GB", | 20260506 16:24:19.967 | INFO | "pinned" : " 29 GB" | 20260506 16:24:19.967 | INFO | } | 20260506 16:24:19.967 | INFO | Perf report for the last 5.13647 seconds: | 20260506 16:24:19.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:24:19.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:19.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:19.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:19.967 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 16:24:19.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:19.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:19.967 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:24:19.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:19.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:19.968 | INFO | Uploaded 36GiB to GPU in 1.97966s (18.1849GiB/s) | 20260506 16:24:19.968 | INFO | Downloaded 34GiB to GPU in 1.93016s (17.6151GiB/s) | 20260506 16:24:25.107 | INFO | Analyzing chunk frames = [2766336,2766848) | 20260506 16:24:25.107 | INFO | MemInfo: { | 20260506 16:24:25.107 | INFO | "gpu" : " 17 GB", | 20260506 16:24:25.107 | INFO | "huge" : "", | 20260506 16:24:25.107 | INFO | "huge_shared" : " 27 GB", | 20260506 16:24:25.107 | INFO | "malloc" : " 14 GB", | 20260506 16:24:25.107 | INFO | "pinned" : " 29 GB" | 20260506 16:24:25.107 | INFO | } | 20260506 16:24:25.107 | INFO | Perf report for the last 5.13974 seconds: | 20260506 16:24:25.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:24:25.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:25.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:25.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:25.107 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 16:24:25.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:25.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:25.107 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:24:25.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:25.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:25.107 | INFO | Uploaded 36GiB to GPU in 1.94015s (18.5553GiB/s) | 20260506 16:24:25.107 | INFO | Downloaded 34GiB to GPU in 1.87645s (18.1193GiB/s) | 20260506 16:24:30.244 | INFO | Analyzing chunk frames = [2766848,2767360) | 20260506 16:24:30.244 | INFO | MemInfo: { | 20260506 16:24:30.244 | INFO | "gpu" : " 17 GB", | 20260506 16:24:30.244 | INFO | "huge" : "", | 20260506 16:24:30.244 | INFO | "huge_shared" : " 27 GB", | 20260506 16:24:30.244 | INFO | "malloc" : " 14 GB", | 20260506 16:24:30.244 | INFO | "pinned" : " 29 GB" | 20260506 16:24:30.244 | INFO | } | 20260506 16:24:30.244 | INFO | Perf report for the last 5.13709 seconds: | 20260506 16:24:30.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:24:30.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:30.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:30.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:30.244 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 16:24:30.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:30.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:30.244 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:24:30.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:30.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:30.244 | INFO | Uploaded 36GiB to GPU in 2.02136s (17.8098GiB/s) | 20260506 16:24:30.245 | INFO | Downloaded 34GiB to GPU in 1.91351s (17.7684GiB/s) | 20260506 16:24:35.385 | INFO | Analyzing chunk frames = [2767360,2767872) | 20260506 16:24:35.385 | INFO | MemInfo: { | 20260506 16:24:35.385 | INFO | "gpu" : " 17 GB", | 20260506 16:24:35.385 | INFO | "huge" : "", | 20260506 16:24:35.385 | INFO | "huge_shared" : " 27 GB", | 20260506 16:24:35.385 | INFO | "malloc" : " 14 GB", | 20260506 16:24:35.385 | INFO | "pinned" : " 29 GB" | 20260506 16:24:35.385 | INFO | } | 20260506 16:24:35.386 | INFO | Perf report for the last 5.14149 seconds: | 20260506 16:24:35.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:24:35.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:35.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:35.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:35.386 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:24:35.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:35.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:35.386 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260506 16:24:35.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:35.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:35.386 | INFO | Uploaded 36GiB to GPU in 2.0722s (17.3729GiB/s) | 20260506 16:24:35.386 | INFO | Downloaded 34GiB to GPU in 1.96258s (17.3241GiB/s) | 20260506 16:24:40.530 | INFO | Analyzing chunk frames = [2767872,2768384) | 20260506 16:24:40.530 | INFO | MemInfo: { | 20260506 16:24:40.530 | INFO | "gpu" : " 17 GB", | 20260506 16:24:40.530 | INFO | "huge" : "", | 20260506 16:24:40.530 | INFO | "huge_shared" : " 27 GB", | 20260506 16:24:40.530 | INFO | "malloc" : " 14 GB", | 20260506 16:24:40.530 | INFO | "pinned" : " 29 GB" | 20260506 16:24:40.530 | INFO | } | 20260506 16:24:40.530 | INFO | Perf report for the last 5.14404 seconds: | 20260506 16:24:40.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:24:40.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:40.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:40.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:40.530 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 16:24:40.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:40.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:40.530 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 16:24:40.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:40.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:40.530 | INFO | Uploaded 36GiB to GPU in 2.02683s (17.7618GiB/s) | 20260506 16:24:40.530 | INFO | Downloaded 34GiB to GPU in 2.01714s (16.8555GiB/s) | 20260506 16:24:45.666 | INFO | Analyzing chunk frames = [2768384,2768896) | 20260506 16:24:45.666 | INFO | MemInfo: { | 20260506 16:24:45.666 | INFO | "gpu" : " 17 GB", | 20260506 16:24:45.666 | INFO | "huge" : "", | 20260506 16:24:45.666 | INFO | "huge_shared" : " 27 GB", | 20260506 16:24:45.666 | INFO | "malloc" : " 14 GB", | 20260506 16:24:45.666 | INFO | "pinned" : " 29 GB" | 20260506 16:24:45.666 | INFO | } | 20260506 16:24:45.666 | INFO | Perf report for the last 5.13615 seconds: | 20260506 16:24:45.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:24:45.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:45.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:45.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:45.666 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:24:45.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:45.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:45.666 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 16:24:45.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:45.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:45.666 | INFO | Uploaded 36GiB to GPU in 2.01089s (17.9025GiB/s) | 20260506 16:24:45.666 | INFO | Downloaded 34GiB to GPU in 1.97305s (17.2322GiB/s) | 20260506 16:24:45.666 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:24:45.666 | INFO | Total mem usage: 18 GiB | 20260506 16:24:45.666 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:24:45.666 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.3%) | 20260506 16:24:45.666 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:24:45.666 | INFO | High Water Breakdown: | 20260506 16:24:45.666 | INFO | 61.7% : 9.9 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:24:45.666 | INFO | 38.2% : 6.12 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:24:45.666 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:24:45.666 | INFO | ------------------------------------------ | 20260506 16:24:45.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:24:45.666 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:24:45.666 | INFO | Total mem usage: 26.5 GiB | 20260506 16:24:45.666 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:24:45.666 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:24:45.666 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:24:45.666 | INFO | High Water Breakdown: | 20260506 16:24:45.666 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:24:45.667 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:24:45.667 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:24:45.667 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:24:45.667 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:24:45.667 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:24:45.667 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:24:45.667 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:24:45.667 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:24:45.667 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:24:45.667 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:24:45.667 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:24:45.667 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:24:45.667 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:24:45.667 | INFO | ------------------------------------------ | 20260506 16:24:45.667 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:24:45.667 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:24:45.667 | INFO | Total mem usage: 25.6 GiB | 20260506 16:24:45.667 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:24:45.667 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:24:45.667 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:24:45.667 | INFO | High Water Breakdown: | 20260506 16:24:45.667 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:24:45.667 | INFO | ------------------------------------------ | 20260506 16:24:45.667 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:24:45.667 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:24:45.667 | INFO | Total mem usage: 15.7 GiB | 20260506 16:24:45.667 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:24:45.667 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:24:45.667 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:24:45.667 | INFO | High Water Breakdown: | 20260506 16:24:45.667 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:24:45.667 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:24:45.667 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:24:45.667 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:24:45.667 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:24:45.667 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:24:45.667 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:24:45.667 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:24:45.667 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:24:45.667 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:24:45.667 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:24:45.667 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:24:45.667 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:24:45.667 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:24:45.667 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:24:45.667 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:24:45.667 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:24:45.667 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:24:45.667 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:24:45.667 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:24:45.667 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:24:45.667 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:24:45.667 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:24:45.667 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:24:45.667 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:24:45.667 | INFO | ------------------------------------------ | 20260506 16:24:45.667 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:24:45.667 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:24:45.667 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:24:45.667 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:24:45.667 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.60e+01, 1.37e+01 / 2.51e+00 | 20260506 16:24:45.667 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.01e-01, 1.05e-02 / 6.02e-03 | 20260506 16:24:45.667 | INFO | PerfReport: 11.58%, Baseline -- 3.64e+00 / 4.43e+00, 5.61e-01 / 3.46e-01 | 20260506 16:24:45.667 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.88e-01 / 1.36e-01 | 20260506 16:24:45.667 | INFO | PerfReport: 15.98%, DME -- 4.01e+01 / 4.48e+01, 4.06e+00 / 2.49e+00 | 20260506 16:24:45.667 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.44e+00 / 1.28e+00 | 20260506 16:24:45.667 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.29e+00 / 2.86e+00, 3.83e-01 / 2.29e-01 | 20260506 16:24:45.667 | INFO | PerfReport: 3.50%, Metrics -- 1.10e+00 / 1.41e+00, 2.83e-01 / 1.75e-01 | 20260506 16:24:45.667 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.82e-01 / 4.29e-01, 5.97e-02 / 2.43e-02 | 20260506 16:24:45.667 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:24:45.667 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:24:45.667 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:24:45.667 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 16:24:45.667 | INFO | PerfReport: 21.55%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 16:24:45.667 | INFO | PerfReport: 21.15%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 16:24:45.667 | INFO | PerfReport: 32.22%, Compute -- 3.14e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 16:24:45.667 | INFO | PerfReport: 1.45%, LockUpload -- 1.42e+00 / 0.00e+00, 5.96e+00 / 0.00e+00 | 20260506 16:24:45.667 | INFO | PerfReport: 1.02%, LockDownload -- 9.91e-01 / 0.00e+00, 2.54e+00 / 0.00e+00 | 20260506 16:24:45.667 | INFO | PerfReport: 22.59%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 16:24:45.667 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260506 16:24:45.785 | INFO | Starting a round of Baz IO | 20260506 16:24:48.921 | INFO | Populating RTMetrics report | 20260506 16:24:48.922 | INFO | RTMetrics report populated | 20260506 16:24:48.922 | INFO | Opening tmp RTMetrics json file | 20260506 16:24:49.038 | INFO | Serializing RTMetrics to json | 20260506 16:24:49.103 | INFO | Serialization done, swapping tmp file now... | 20260506 16:24:49.134 | INFO | RTMetrics done | 20260506 16:24:49.163 | INFO | Finished a round of Baz IO: Wrote 2.088 GiB in 3.377 seconds | 20260506 16:24:49.163 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.37% / 0.01% / 63.62% | 20260506 16:24:49.163 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.70114% / 3.97364e-06% / 6.35783e-05% | 20260506 16:24:49.163 | INFO | Baz IO overall: | 20260506 16:24:49.163 | INFO | BytesWritten : 1453189001084 | 20260506 16:24:49.163 | INFO | NumEvents : 703198507559 | 20260506 16:24:50.806 | INFO | Analyzing chunk frames = [2768896,2769408) | 20260506 16:24:50.806 | INFO | MemInfo: { | 20260506 16:24:50.806 | INFO | "gpu" : " 17 GB", | 20260506 16:24:50.806 | INFO | "huge" : "", | 20260506 16:24:50.806 | INFO | "huge_shared" : " 27 GB", | 20260506 16:24:50.806 | INFO | "malloc" : " 14 GB", | 20260506 16:24:50.806 | INFO | "pinned" : " 29 GB" | 20260506 16:24:50.806 | INFO | } | 20260506 16:24:50.806 | INFO | Perf report for the last 5.13992 seconds: | 20260506 16:24:50.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:24:50.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:50.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:50.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:50.806 | INFO | ANALYSIS : 62.8%, 2.9, 95.4ms, 0 | 20260506 16:24:50.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:50.806 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260506 16:24:50.806 | INFO | PRE_HQ : 60.5%, 1.9, 59.1ms, 0 | 20260506 16:24:50.806 | INFO | BAZWRITER : 65.1%, 4.8, 160.8ms, 0 | 20260506 16:24:50.806 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 16:24:50.806 | INFO | Uploaded 36GiB to GPU in 2.04621s (17.5935GiB/s) | 20260506 16:24:50.806 | INFO | Downloaded 38GiB to GPU in 2.21369s (17.1659GiB/s) | 20260506 16:24:55.948 | INFO | Analyzing chunk frames = [2769408,2769920) | 20260506 16:24:55.948 | INFO | MemInfo: { | 20260506 16:24:55.948 | INFO | "gpu" : " 17 GB", | 20260506 16:24:55.948 | INFO | "huge" : "", | 20260506 16:24:55.948 | INFO | "huge_shared" : " 27 GB", | 20260506 16:24:55.948 | INFO | "malloc" : " 14 GB", | 20260506 16:24:55.948 | INFO | "pinned" : " 29 GB" | 20260506 16:24:55.948 | INFO | } | 20260506 16:24:55.948 | INFO | Perf report for the last 5.14235 seconds: | 20260506 16:24:55.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:24:55.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:55.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:55.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:55.948 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:24:55.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:55.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:55.948 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:24:55.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:55.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:24:55.948 | INFO | Uploaded 36GiB to GPU in 2.01031s (17.9077GiB/s) | 20260506 16:24:55.948 | INFO | Downloaded 34GiB to GPU in 1.94105s (17.5163GiB/s) | 20260506 16:25:01.086 | INFO | Analyzing chunk frames = [2769920,2770432) | 20260506 16:25:01.086 | INFO | MemInfo: { | 20260506 16:25:01.086 | INFO | "gpu" : " 17 GB", | 20260506 16:25:01.086 | INFO | "huge" : "", | 20260506 16:25:01.086 | INFO | "huge_shared" : " 27 GB", | 20260506 16:25:01.086 | INFO | "malloc" : " 14 GB", | 20260506 16:25:01.086 | INFO | "pinned" : " 29 GB" | 20260506 16:25:01.086 | INFO | } | 20260506 16:25:01.086 | INFO | Perf report for the last 5.13782 seconds: | 20260506 16:25:01.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:25:01.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:01.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:01.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:01.086 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 16:25:01.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:01.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:01.086 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:25:01.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:01.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:01.086 | INFO | Uploaded 36GiB to GPU in 1.97821s (18.1982GiB/s) | 20260506 16:25:01.087 | INFO | Downloaded 34GiB to GPU in 1.93152s (17.6027GiB/s) | 20260506 16:25:06.221 | INFO | Analyzing chunk frames = [2770432,2770944) | 20260506 16:25:06.221 | INFO | MemInfo: { | 20260506 16:25:06.221 | INFO | "gpu" : " 17 GB", | 20260506 16:25:06.221 | INFO | "huge" : "", | 20260506 16:25:06.221 | INFO | "huge_shared" : " 27 GB", | 20260506 16:25:06.221 | INFO | "malloc" : " 14 GB", | 20260506 16:25:06.221 | INFO | "pinned" : " 29 GB" | 20260506 16:25:06.221 | INFO | } | 20260506 16:25:06.221 | INFO | Perf report for the last 5.13514 seconds: | 20260506 16:25:06.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:25:06.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:06.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:06.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:06.221 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260506 16:25:06.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:06.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:06.221 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 16:25:06.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:06.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:06.221 | INFO | Uploaded 36GiB to GPU in 1.94895s (18.4715GiB/s) | 20260506 16:25:06.221 | INFO | Downloaded 34GiB to GPU in 1.88083s (18.0772GiB/s) | 20260506 16:25:11.359 | INFO | Analyzing chunk frames = [2770944,2771456) | 20260506 16:25:11.359 | INFO | MemInfo: { | 20260506 16:25:11.359 | INFO | "gpu" : " 17 GB", | 20260506 16:25:11.359 | INFO | "huge" : "", | 20260506 16:25:11.359 | INFO | "huge_shared" : " 27 GB", | 20260506 16:25:11.359 | INFO | "malloc" : " 14 GB", | 20260506 16:25:11.359 | INFO | "pinned" : " 29 GB" | 20260506 16:25:11.359 | INFO | } | 20260506 16:25:11.360 | INFO | Perf report for the last 5.1384 seconds: | 20260506 16:25:11.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:25:11.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:11.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:11.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:11.360 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 16:25:11.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:11.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:11.360 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:25:11.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:11.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:11.360 | INFO | Uploaded 36GiB to GPU in 2.02396s (17.7869GiB/s) | 20260506 16:25:11.360 | INFO | Downloaded 34GiB to GPU in 1.91255s (17.7773GiB/s) | 20260506 16:25:16.499 | INFO | Analyzing chunk frames = [2771456,2771968) | 20260506 16:25:16.499 | INFO | MemInfo: { | 20260506 16:25:16.499 | INFO | "gpu" : " 17 GB", | 20260506 16:25:16.499 | INFO | "huge" : "", | 20260506 16:25:16.499 | INFO | "huge_shared" : " 27 GB", | 20260506 16:25:16.499 | INFO | "malloc" : " 14 GB", | 20260506 16:25:16.500 | INFO | "pinned" : " 29 GB" | 20260506 16:25:16.500 | INFO | } | 20260506 16:25:16.500 | INFO | Perf report for the last 5.14011 seconds: | 20260506 16:25:16.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:25:16.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:16.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:16.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:16.500 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:25:16.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:16.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:16.500 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260506 16:25:16.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:16.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:16.500 | INFO | Uploaded 36GiB to GPU in 2.03764s (17.6675GiB/s) | 20260506 16:25:16.500 | INFO | Downloaded 34GiB to GPU in 1.94189s (17.5087GiB/s) | 20260506 16:25:21.643 | INFO | Analyzing chunk frames = [2771968,2772480) | 20260506 16:25:21.643 | INFO | MemInfo: { | 20260506 16:25:21.643 | INFO | "gpu" : " 17 GB", | 20260506 16:25:21.643 | INFO | "huge" : "", | 20260506 16:25:21.643 | INFO | "huge_shared" : " 27 GB", | 20260506 16:25:21.643 | INFO | "malloc" : " 14 GB", | 20260506 16:25:21.643 | INFO | "pinned" : " 29 GB" | 20260506 16:25:21.643 | INFO | } | 20260506 16:25:21.643 | INFO | Perf report for the last 5.14297 seconds: | 20260506 16:25:21.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:25:21.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:21.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:21.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:21.643 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 16:25:21.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:21.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:21.643 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 16:25:21.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:21.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:21.643 | INFO | Uploaded 36GiB to GPU in 2.03104s (17.7249GiB/s) | 20260506 16:25:21.643 | INFO | Downloaded 34GiB to GPU in 2.01532s (16.8708GiB/s) | 20260506 16:25:26.784 | INFO | Analyzing chunk frames = [2772480,2772992) | 20260506 16:25:26.784 | INFO | MemInfo: { | 20260506 16:25:26.784 | INFO | "gpu" : " 17 GB", | 20260506 16:25:26.784 | INFO | "huge" : "", | 20260506 16:25:26.784 | INFO | "huge_shared" : " 27 GB", | 20260506 16:25:26.784 | INFO | "malloc" : " 14 GB", | 20260506 16:25:26.784 | INFO | "pinned" : " 29 GB" | 20260506 16:25:26.784 | INFO | } | 20260506 16:25:26.785 | INFO | Perf report for the last 5.14186 seconds: | 20260506 16:25:26.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:25:26.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:26.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:26.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:26.785 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 16:25:26.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:26.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:26.785 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:25:26.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:26.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:26.785 | INFO | Uploaded 36GiB to GPU in 2.07678s (17.3345GiB/s) | 20260506 16:25:26.785 | INFO | Downloaded 34GiB to GPU in 1.97375s (17.2261GiB/s) | 20260506 16:25:26.900 | INFO | Starting a round of Baz IO | 20260506 16:25:30.056 | INFO | Populating RTMetrics report | 20260506 16:25:30.057 | INFO | RTMetrics report populated | 20260506 16:25:30.057 | INFO | Opening tmp RTMetrics json file | 20260506 16:25:30.072 | INFO | Serializing RTMetrics to json | 20260506 16:25:30.077 | INFO | Serialization done, swapping tmp file now... | 20260506 16:25:30.082 | INFO | RTMetrics done | 20260506 16:25:30.535 | INFO | Finished a round of Baz IO: Wrote 2.088 GiB in 3.635 seconds | 20260506 16:25:30.535 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.36% / 0.01% / 63.63% | 20260506 16:25:30.535 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.69798% / 2.78155e-05% / 7.94729e-05% | 20260506 16:25:30.535 | INFO | Baz IO overall: | 20260506 16:25:30.535 | INFO | BytesWritten : 1455430846883 | 20260506 16:25:30.535 | INFO | NumEvents : 704282367768 | 20260506 16:25:31.919 | INFO | Analyzing chunk frames = [2772992,2773504) | 20260506 16:25:31.919 | INFO | MemInfo: { | 20260506 16:25:31.919 | INFO | "gpu" : " 17 GB", | 20260506 16:25:31.919 | INFO | "huge" : "", | 20260506 16:25:31.919 | INFO | "huge_shared" : " 27 GB", | 20260506 16:25:31.919 | INFO | "malloc" : " 14 GB", | 20260506 16:25:31.919 | INFO | "pinned" : " 29 GB" | 20260506 16:25:31.919 | INFO | } | 20260506 16:25:31.919 | INFO | Perf report for the last 5.13468 seconds: | 20260506 16:25:31.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:25:31.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:31.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:31.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:31.919 | INFO | ANALYSIS : 63.1%, 2.9, 95.6ms, 0 | 20260506 16:25:31.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:31.919 | INFO | RT_METRICS : 44.2%, 1.0, 23.1ms, 0 | 20260506 16:25:31.919 | INFO | PRE_HQ : 61.1%, 1.9, 59.1ms, 0 | 20260506 16:25:31.919 | INFO | BAZWRITER : 70.2%, 4.8, 173.1ms, 0 | 20260506 16:25:31.919 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 16:25:31.920 | INFO | Uploaded 36GiB to GPU in 2.06245s (17.4549GiB/s) | 20260506 16:25:31.920 | INFO | Downloaded 38GiB to GPU in 2.23479s (17.0038GiB/s) | 20260506 16:25:37.057 | INFO | Analyzing chunk frames = [2773504,2774016) | 20260506 16:25:37.057 | INFO | MemInfo: { | 20260506 16:25:37.057 | INFO | "gpu" : " 17 GB", | 20260506 16:25:37.057 | INFO | "huge" : "", | 20260506 16:25:37.057 | INFO | "huge_shared" : " 27 GB", | 20260506 16:25:37.057 | INFO | "malloc" : " 14 GB", | 20260506 16:25:37.057 | INFO | "pinned" : " 29 GB" | 20260506 16:25:37.057 | INFO | } | 20260506 16:25:37.057 | INFO | Perf report for the last 5.1381 seconds: | 20260506 16:25:37.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:25:37.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:37.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:37.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:37.057 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 16:25:37.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:37.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:37.057 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:25:37.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:37.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:37.058 | INFO | Uploaded 36GiB to GPU in 1.9935s (18.0587GiB/s) | 20260506 16:25:37.058 | INFO | Downloaded 34GiB to GPU in 1.9378s (17.5456GiB/s) | 20260506 16:25:42.198 | INFO | Analyzing chunk frames = [2774016,2774528) | 20260506 16:25:42.198 | INFO | MemInfo: { | 20260506 16:25:42.198 | INFO | "gpu" : " 17 GB", | 20260506 16:25:42.198 | INFO | "huge" : "", | 20260506 16:25:42.199 | INFO | "huge_shared" : " 27 GB", | 20260506 16:25:42.199 | INFO | "malloc" : " 14 GB", | 20260506 16:25:42.199 | INFO | "pinned" : " 29 GB" | 20260506 16:25:42.199 | INFO | } | 20260506 16:25:42.199 | INFO | Perf report for the last 5.14136 seconds: | 20260506 16:25:42.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:25:42.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:42.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:42.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:42.199 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 16:25:42.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:42.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:42.199 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:25:42.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:42.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:42.199 | INFO | Uploaded 36GiB to GPU in 1.97899s (18.1911GiB/s) | 20260506 16:25:42.199 | INFO | Downloaded 34GiB to GPU in 1.93132s (17.6045GiB/s) | 20260506 16:25:47.336 | INFO | Analyzing chunk frames = [2774528,2775040) | 20260506 16:25:47.336 | INFO | MemInfo: { | 20260506 16:25:47.336 | INFO | "gpu" : " 17 GB", | 20260506 16:25:47.336 | INFO | "huge" : "", | 20260506 16:25:47.336 | INFO | "huge_shared" : " 27 GB", | 20260506 16:25:47.337 | INFO | "malloc" : " 14 GB", | 20260506 16:25:47.337 | INFO | "pinned" : " 29 GB" | 20260506 16:25:47.337 | INFO | } | 20260506 16:25:47.337 | INFO | Perf report for the last 5.13797 seconds: | 20260506 16:25:47.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:25:47.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:47.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:47.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:47.337 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260506 16:25:47.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:47.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:47.337 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:25:47.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:47.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:47.337 | INFO | Uploaded 36GiB to GPU in 1.93718s (18.5837GiB/s) | 20260506 16:25:47.337 | INFO | Downloaded 34GiB to GPU in 1.87414s (18.1416GiB/s) | 20260506 16:25:52.472 | INFO | Analyzing chunk frames = [2775040,2775552) | 20260506 16:25:52.472 | INFO | MemInfo: { | 20260506 16:25:52.472 | INFO | "gpu" : " 17 GB", | 20260506 16:25:52.472 | INFO | "huge" : "", | 20260506 16:25:52.472 | INFO | "huge_shared" : " 27 GB", | 20260506 16:25:52.472 | INFO | "malloc" : " 14 GB", | 20260506 16:25:52.472 | INFO | "pinned" : " 29 GB" | 20260506 16:25:52.472 | INFO | } | 20260506 16:25:52.472 | INFO | Perf report for the last 5.13579 seconds: | 20260506 16:25:52.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:25:52.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:52.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:52.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:52.473 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 16:25:52.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:52.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:52.473 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:25:52.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:52.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:52.473 | INFO | Uploaded 36GiB to GPU in 2.02089s (17.814GiB/s) | 20260506 16:25:52.473 | INFO | Downloaded 34GiB to GPU in 1.91174s (17.7848GiB/s) | 20260506 16:25:57.612 | INFO | Analyzing chunk frames = [2775552,2776064) | 20260506 16:25:57.612 | INFO | MemInfo: { | 20260506 16:25:57.612 | INFO | "gpu" : " 17 GB", | 20260506 16:25:57.612 | INFO | "huge" : "", | 20260506 16:25:57.612 | INFO | "huge_shared" : " 27 GB", | 20260506 16:25:57.612 | INFO | "malloc" : " 14 GB", | 20260506 16:25:57.613 | INFO | "pinned" : " 29 GB" | 20260506 16:25:57.613 | INFO | } | 20260506 16:25:57.613 | INFO | Perf report for the last 5.14018 seconds: | 20260506 16:25:57.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:25:57.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:57.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:57.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:57.613 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:25:57.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:57.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:57.613 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 16:25:57.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:57.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:25:57.613 | INFO | Uploaded 36GiB to GPU in 2.05381s (17.5284GiB/s) | 20260506 16:25:57.613 | INFO | Downloaded 34GiB to GPU in 1.94982s (17.4375GiB/s) | 20260506 16:26:02.757 | INFO | Analyzing chunk frames = [2776064,2776576) | 20260506 16:26:02.757 | INFO | MemInfo: { | 20260506 16:26:02.757 | INFO | "gpu" : " 17 GB", | 20260506 16:26:02.757 | INFO | "huge" : "", | 20260506 16:26:02.757 | INFO | "huge_shared" : " 27 GB", | 20260506 16:26:02.757 | INFO | "malloc" : " 14 GB", | 20260506 16:26:02.757 | INFO | "pinned" : " 29 GB" | 20260506 16:26:02.757 | INFO | } | 20260506 16:26:02.757 | INFO | Perf report for the last 5.14468 seconds: | 20260506 16:26:02.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:26:02.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:02.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:02.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:02.758 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 16:26:02.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:02.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:02.758 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 16:26:02.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:02.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:02.758 | INFO | Uploaded 36GiB to GPU in 2.02402s (17.7864GiB/s) | 20260506 16:26:02.758 | INFO | Downloaded 34GiB to GPU in 2.02476s (16.7921GiB/s) | 20260506 16:26:07.898 | INFO | Analyzing chunk frames = [2776576,2777088) | 20260506 16:26:07.898 | INFO | MemInfo: { | 20260506 16:26:07.898 | INFO | "gpu" : " 17 GB", | 20260506 16:26:07.898 | INFO | "huge" : "", | 20260506 16:26:07.898 | INFO | "huge_shared" : " 27 GB", | 20260506 16:26:07.898 | INFO | "malloc" : " 14 GB", | 20260506 16:26:07.898 | INFO | "pinned" : " 29 GB" | 20260506 16:26:07.898 | INFO | } | 20260506 16:26:07.898 | INFO | Perf report for the last 5.14063 seconds: | 20260506 16:26:07.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:26:07.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:07.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:07.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:07.898 | INFO | ANALYSIS : 62.6%, 3.0, 97.5ms, 0 | 20260506 16:26:07.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:07.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:07.898 | INFO | PRE_HQ : 32.6%, 1.0, 17.0ms, 0 | 20260506 16:26:07.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:07.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:07.898 | INFO | Uploaded 36GiB to GPU in 2.09008s (17.2242GiB/s) | 20260506 16:26:07.898 | INFO | Downloaded 34GiB to GPU in 1.99736s (17.0224GiB/s) | 20260506 16:26:07.899 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:26:07.899 | INFO | Total mem usage: 18 GiB | 20260506 16:26:07.899 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:26:07.899 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -18.8%) | 20260506 16:26:07.899 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:26:07.899 | INFO | High Water Breakdown: | 20260506 16:26:07.899 | INFO | 60.8% : 9.22 GiB ( 80.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:26:07.899 | INFO | 39% : 5.92 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:26:07.899 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:26:07.899 | INFO | ------------------------------------------ | 20260506 16:26:07.899 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:26:07.899 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:26:07.899 | INFO | Total mem usage: 26.5 GiB | 20260506 16:26:07.899 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:26:07.899 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:26:07.899 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:26:07.899 | INFO | High Water Breakdown: | 20260506 16:26:07.899 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:26:07.899 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:26:07.899 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:26:07.899 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:26:07.899 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:26:07.899 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:26:07.899 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:26:07.899 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:26:07.899 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:26:07.899 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:26:07.899 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:26:07.899 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:26:07.899 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:26:07.899 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:26:07.899 | INFO | ------------------------------------------ | 20260506 16:26:07.899 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:26:07.899 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:26:07.899 | INFO | Total mem usage: 25.6 GiB | 20260506 16:26:07.899 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:26:07.899 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:26:07.899 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:26:07.899 | INFO | High Water Breakdown: | 20260506 16:26:07.899 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:26:07.899 | INFO | ------------------------------------------ | 20260506 16:26:07.899 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:26:07.899 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:26:07.899 | INFO | Total mem usage: 15.7 GiB | 20260506 16:26:07.899 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:26:07.899 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:26:07.899 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 16:26:07.899 | INFO | High Water Breakdown: | 20260506 16:26:07.899 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:26:07.899 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:26:07.899 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:26:07.899 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:26:07.899 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:26:07.899 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:26:07.899 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:26:07.899 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:26:07.899 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:26:07.899 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:26:07.899 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:26:07.899 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:26:07.899 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:26:07.899 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:26:07.899 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:26:07.899 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:26:07.899 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:26:07.899 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:26:07.899 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:26:07.899 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:26:07.899 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:26:07.899 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:26:07.899 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:26:07.899 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:26:07.899 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:26:07.900 | INFO | ------------------------------------------ | 20260506 16:26:07.900 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:26:07.900 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:26:07.900 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:26:07.900 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:26:07.900 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.63e+01, 1.37e+01 / 2.45e+00 | 20260506 16:26:07.900 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.02e-01, 9.96e-03 / 6.09e-03 | 20260506 16:26:07.900 | INFO | PerfReport: 11.59%, Baseline -- 3.64e+00 / 4.44e+00, 5.66e-01 / 3.11e-01 | 20260506 16:26:07.900 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.48e+00, 1.90e-01 / 1.22e-01 | 20260506 16:26:07.900 | INFO | PerfReport: 15.99%, DME -- 4.02e+01 / 4.50e+01, 4.12e+00 / 3.04e+00 | 20260506 16:26:07.900 | INFO | PerfReport: 56.83%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.44e+00 / 1.81e+00 | 20260506 16:26:07.900 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.30e+00 / 2.88e+00, 3.85e-01 / 2.28e-01 | 20260506 16:26:07.900 | INFO | PerfReport: 3.51%, Metrics -- 1.10e+00 / 1.46e+00, 2.88e-01 / 1.87e-01 | 20260506 16:26:07.900 | INFO | PerfReport: 1.22%, PulseSerializing -- 3.82e-01 / 4.33e-01, 5.99e-02 / 2.37e-02 | 20260506 16:26:07.900 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:26:07.900 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:26:07.900 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:26:07.900 | INFO | PerfReport: Full Execution -- 9.76e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 16:26:07.900 | INFO | PerfReport: 21.56%, Upload -- 2.10e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260506 16:26:07.900 | INFO | PerfReport: 21.13%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 16:26:07.900 | INFO | PerfReport: 32.23%, Compute -- 3.15e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 16:26:07.900 | INFO | PerfReport: 1.49%, LockUpload -- 1.46e+00 / 0.00e+00, 6.04e+00 / 0.00e+00 | 20260506 16:26:07.900 | INFO | PerfReport: 1.04%, LockDownload -- 1.01e+00 / 0.00e+00, 2.64e+00 / 0.00e+00 | 20260506 16:26:07.900 | INFO | PerfReport: 22.53%, LockCompute -- 2.20e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 16:26:07.900 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260506 16:26:08.033 | INFO | Starting a round of Baz IO | 20260506 16:26:11.183 | INFO | Populating RTMetrics report | 20260506 16:26:11.184 | INFO | RTMetrics report populated | 20260506 16:26:11.184 | INFO | Opening tmp RTMetrics json file | 20260506 16:26:11.262 | INFO | Serializing RTMetrics to json | 20260506 16:26:11.288 | INFO | Serialization done, swapping tmp file now... | 20260506 16:26:11.292 | INFO | RTMetrics done | 20260506 16:26:11.473 | INFO | Finished a round of Baz IO: Wrote 2.088 GiB in 3.439 seconds | 20260506 16:26:11.473 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.35% / 0.01% / 63.64% | 20260506 16:26:11.473 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.6941% / 1.19209e-05% / 5.5631e-05% | 20260506 16:26:11.473 | INFO | Baz IO overall: | 20260506 16:26:11.473 | INFO | BytesWritten : 1457672566830 | 20260506 16:26:11.473 | INFO | NumEvents : 705366325955 | 20260506 16:26:13.040 | INFO | Analyzing chunk frames = [2777088,2777600) | 20260506 16:26:13.040 | INFO | MemInfo: { | 20260506 16:26:13.040 | INFO | "gpu" : " 17 GB", | 20260506 16:26:13.040 | INFO | "huge" : "", | 20260506 16:26:13.040 | INFO | "huge_shared" : " 27 GB", | 20260506 16:26:13.040 | INFO | "malloc" : " 14 GB", | 20260506 16:26:13.040 | INFO | "pinned" : " 29 GB" | 20260506 16:26:13.040 | INFO | } | 20260506 16:26:13.040 | INFO | Perf report for the last 5.14185 seconds: | 20260506 16:26:13.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:26:13.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:13.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:13.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:13.040 | INFO | ANALYSIS : 63.3%, 2.9, 95.0ms, 0 | 20260506 16:26:13.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:13.040 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260506 16:26:13.040 | INFO | PRE_HQ : 61.2%, 1.9, 61.9ms, 0 | 20260506 16:26:13.040 | INFO | BAZWRITER : 66.3%, 4.8, 162.5ms, 0 | 20260506 16:26:13.040 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 16:26:13.040 | INFO | Uploaded 36GiB to GPU in 2.09788s (17.1602GiB/s) | 20260506 16:26:13.040 | INFO | Downloaded 38GiB to GPU in 2.23049s (17.0366GiB/s) | 20260506 16:26:18.174 | INFO | Analyzing chunk frames = [2777600,2778112) | 20260506 16:26:18.174 | INFO | MemInfo: { | 20260506 16:26:18.174 | INFO | "gpu" : " 17 GB", | 20260506 16:26:18.174 | INFO | "huge" : "", | 20260506 16:26:18.174 | INFO | "huge_shared" : " 27 GB", | 20260506 16:26:18.174 | INFO | "malloc" : " 14 GB", | 20260506 16:26:18.174 | INFO | "pinned" : " 29 GB" | 20260506 16:26:18.174 | INFO | } | 20260506 16:26:18.175 | INFO | Perf report for the last 5.13464 seconds: | 20260506 16:26:18.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:26:18.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:18.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:18.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:18.175 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:26:18.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:18.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:18.175 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:26:18.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:18.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:18.175 | INFO | Uploaded 36GiB to GPU in 1.99447s (18.0499GiB/s) | 20260506 16:26:18.175 | INFO | Downloaded 34GiB to GPU in 1.94524s (17.4786GiB/s) | 20260506 16:26:23.312 | INFO | Analyzing chunk frames = [2778112,2778624) | 20260506 16:26:23.312 | INFO | MemInfo: { | 20260506 16:26:23.312 | INFO | "gpu" : " 17 GB", | 20260506 16:26:23.312 | INFO | "huge" : "", | 20260506 16:26:23.312 | INFO | "huge_shared" : " 27 GB", | 20260506 16:26:23.312 | INFO | "malloc" : " 14 GB", | 20260506 16:26:23.312 | INFO | "pinned" : " 29 GB" | 20260506 16:26:23.312 | INFO | } | 20260506 16:26:23.312 | INFO | Perf report for the last 5.13795 seconds: | 20260506 16:26:23.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:26:23.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:23.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:23.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:23.313 | INFO | ANALYSIS : 61.2%, 3.0, 94.6ms, 0 | 20260506 16:26:23.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:23.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:23.313 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:26:23.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:23.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:23.313 | INFO | Uploaded 36GiB to GPU in 1.98098s (18.1728GiB/s) | 20260506 16:26:23.313 | INFO | Downloaded 34GiB to GPU in 1.93237s (17.595GiB/s) | 20260506 16:26:28.450 | INFO | Analyzing chunk frames = [2778624,2779136) | 20260506 16:26:28.450 | INFO | MemInfo: { | 20260506 16:26:28.450 | INFO | "gpu" : " 17 GB", | 20260506 16:26:28.450 | INFO | "huge" : "", | 20260506 16:26:28.450 | INFO | "huge_shared" : " 27 GB", | 20260506 16:26:28.451 | INFO | "malloc" : " 14 GB", | 20260506 16:26:28.451 | INFO | "pinned" : " 29 GB" | 20260506 16:26:28.451 | INFO | } | 20260506 16:26:28.451 | INFO | Perf report for the last 5.13808 seconds: | 20260506 16:26:28.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:26:28.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:28.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:28.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:28.451 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 16:26:28.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:28.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:28.451 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:26:28.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:28.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:28.451 | INFO | Uploaded 36GiB to GPU in 1.93247s (18.629GiB/s) | 20260506 16:26:28.451 | INFO | Downloaded 34GiB to GPU in 1.87743s (18.1099GiB/s) | 20260506 16:26:33.589 | INFO | Analyzing chunk frames = [2779136,2779648) | 20260506 16:26:33.589 | INFO | MemInfo: { | 20260506 16:26:33.589 | INFO | "gpu" : " 17 GB", | 20260506 16:26:33.589 | INFO | "huge" : "", | 20260506 16:26:33.589 | INFO | "huge_shared" : " 27 GB", | 20260506 16:26:33.589 | INFO | "malloc" : " 14 GB", | 20260506 16:26:33.589 | INFO | "pinned" : " 29 GB" | 20260506 16:26:33.589 | INFO | } | 20260506 16:26:33.589 | INFO | Perf report for the last 5.13844 seconds: | 20260506 16:26:33.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:26:33.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:33.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:33.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:33.589 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 16:26:33.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:33.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:33.589 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:26:33.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:33.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:33.589 | INFO | Uploaded 36GiB to GPU in 2.02324s (17.7932GiB/s) | 20260506 16:26:33.589 | INFO | Downloaded 34GiB to GPU in 1.90876s (17.8126GiB/s) | 20260506 16:26:38.730 | INFO | Analyzing chunk frames = [2779648,2780160) | 20260506 16:26:38.730 | INFO | MemInfo: { | 20260506 16:26:38.730 | INFO | "gpu" : " 17 GB", | 20260506 16:26:38.730 | INFO | "huge" : "", | 20260506 16:26:38.730 | INFO | "huge_shared" : " 27 GB", | 20260506 16:26:38.730 | INFO | "malloc" : " 14 GB", | 20260506 16:26:38.730 | INFO | "pinned" : " 29 GB" | 20260506 16:26:38.730 | INFO | } | 20260506 16:26:38.730 | INFO | Perf report for the last 5.14125 seconds: | 20260506 16:26:38.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:26:38.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:38.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:38.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:38.730 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:26:38.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:38.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:38.730 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:26:38.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:38.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:38.731 | INFO | Uploaded 36GiB to GPU in 2.0516s (17.5473GiB/s) | 20260506 16:26:38.731 | INFO | Downloaded 34GiB to GPU in 1.95026s (17.4336GiB/s) | 20260506 16:26:43.875 | INFO | Analyzing chunk frames = [2780160,2780672) | 20260506 16:26:43.875 | INFO | MemInfo: { | 20260506 16:26:43.875 | INFO | "gpu" : " 17 GB", | 20260506 16:26:43.875 | INFO | "huge" : "", | 20260506 16:26:43.875 | INFO | "huge_shared" : " 27 GB", | 20260506 16:26:43.875 | INFO | "malloc" : " 14 GB", | 20260506 16:26:43.875 | INFO | "pinned" : " 29 GB" | 20260506 16:26:43.875 | INFO | } | 20260506 16:26:43.875 | INFO | Perf report for the last 5.14464 seconds: | 20260506 16:26:43.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:26:43.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:43.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:43.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:43.875 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 16:26:43.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:43.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:43.875 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:26:43.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:43.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:43.875 | INFO | Uploaded 36GiB to GPU in 2.03057s (17.729GiB/s) | 20260506 16:26:43.875 | INFO | Downloaded 34GiB to GPU in 2.00216s (16.9817GiB/s) | 20260506 16:26:49.012 | INFO | Analyzing chunk frames = [2780672,2781184) | 20260506 16:26:49.012 | INFO | MemInfo: { | 20260506 16:26:49.012 | INFO | "gpu" : " 17 GB", | 20260506 16:26:49.012 | INFO | "huge" : "", | 20260506 16:26:49.012 | INFO | "huge_shared" : " 27 GB", | 20260506 16:26:49.012 | INFO | "malloc" : " 14 GB", | 20260506 16:26:49.012 | INFO | "pinned" : " 29 GB" | 20260506 16:26:49.012 | INFO | } | 20260506 16:26:49.012 | INFO | Perf report for the last 5.13703 seconds: | 20260506 16:26:49.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:26:49.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:49.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:49.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:49.012 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:26:49.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:49.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:49.012 | INFO | PRE_HQ : 33.0%, 1.0, 17.1ms, 0 | 20260506 16:26:49.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:49.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:49.012 | INFO | Uploaded 36GiB to GPU in 1.99147s (18.0771GiB/s) | 20260506 16:26:49.012 | INFO | Downloaded 34GiB to GPU in 1.95846s (17.3605GiB/s) | 20260506 16:26:49.131 | INFO | Starting a round of Baz IO | 20260506 16:26:52.273 | INFO | Populating RTMetrics report | 20260506 16:26:52.273 | INFO | RTMetrics report populated | 20260506 16:26:52.273 | INFO | Opening tmp RTMetrics json file | 20260506 16:26:52.377 | INFO | Serializing RTMetrics to json | 20260506 16:26:52.437 | INFO | Serialization done, swapping tmp file now... | 20260506 16:26:52.461 | INFO | RTMetrics done | 20260506 16:26:52.694 | INFO | Finished a round of Baz IO: Wrote 2.088 GiB in 3.562 seconds | 20260506 16:26:52.694 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.34% / 0.01% / 63.65% | 20260506 16:26:52.694 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.68914% / 3.97364e-06% / 5.16574e-05% | 20260506 16:26:52.694 | INFO | Baz IO overall: | 20260506 16:26:52.694 | INFO | BytesWritten : 1459914409764 | 20260506 16:26:52.694 | INFO | NumEvents : 706450272961 | 20260506 16:26:54.151 | INFO | Analyzing chunk frames = [2781184,2781696) | 20260506 16:26:54.151 | INFO | MemInfo: { | 20260506 16:26:54.151 | INFO | "gpu" : " 17 GB", | 20260506 16:26:54.151 | INFO | "huge" : "", | 20260506 16:26:54.151 | INFO | "huge_shared" : " 27 GB", | 20260506 16:26:54.151 | INFO | "malloc" : " 14 GB", | 20260506 16:26:54.151 | INFO | "pinned" : " 29 GB" | 20260506 16:26:54.151 | INFO | } | 20260506 16:26:54.151 | INFO | Perf report for the last 5.13873 seconds: | 20260506 16:26:54.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:26:54.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:54.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:54.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:54.151 | INFO | ANALYSIS : 62.8%, 2.9, 94.3ms, 0 | 20260506 16:26:54.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:54.151 | INFO | RT_METRICS : 48.8%, 1.0, 25.6ms, 0 | 20260506 16:26:54.151 | INFO | PRE_HQ : 60.8%, 1.9, 61.2ms, 0 | 20260506 16:26:54.151 | INFO | BAZWRITER : 68.9%, 4.8, 170.7ms, 0 | 20260506 16:26:54.151 | INFO | BAZ_SAMPLING : 3.1%, 1.0, 1.6ms, 0 | 20260506 16:26:54.151 | INFO | Uploaded 36GiB to GPU in 2.05508s (17.5176GiB/s) | 20260506 16:26:54.151 | INFO | Downloaded 38GiB to GPU in 2.21655s (17.1437GiB/s) | 20260506 16:26:59.288 | INFO | Analyzing chunk frames = [2781696,2782208) | 20260506 16:26:59.288 | INFO | MemInfo: { | 20260506 16:26:59.288 | INFO | "gpu" : " 17 GB", | 20260506 16:26:59.288 | INFO | "huge" : "", | 20260506 16:26:59.288 | INFO | "huge_shared" : " 27 GB", | 20260506 16:26:59.288 | INFO | "malloc" : " 14 GB", | 20260506 16:26:59.288 | INFO | "pinned" : " 29 GB" | 20260506 16:26:59.288 | INFO | } | 20260506 16:26:59.288 | INFO | Perf report for the last 5.13757 seconds: | 20260506 16:26:59.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:26:59.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:59.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:59.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:59.288 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:26:59.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:59.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:59.288 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 16:26:59.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:59.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:26:59.289 | INFO | Uploaded 36GiB to GPU in 2.01879s (17.8325GiB/s) | 20260506 16:26:59.289 | INFO | Downloaded 34GiB to GPU in 1.94883s (17.4463GiB/s) | 20260506 16:27:04.426 | INFO | Analyzing chunk frames = [2782208,2782720) | 20260506 16:27:04.426 | INFO | MemInfo: { | 20260506 16:27:04.426 | INFO | "gpu" : " 17 GB", | 20260506 16:27:04.426 | INFO | "huge" : "", | 20260506 16:27:04.426 | INFO | "huge_shared" : " 27 GB", | 20260506 16:27:04.426 | INFO | "malloc" : " 14 GB", | 20260506 16:27:04.426 | INFO | "pinned" : " 29 GB" | 20260506 16:27:04.426 | INFO | } | 20260506 16:27:04.426 | INFO | Perf report for the last 5.13742 seconds: | 20260506 16:27:04.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:27:04.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:04.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:04.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:04.426 | INFO | ANALYSIS : 61.2%, 3.0, 94.6ms, 0 | 20260506 16:27:04.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:04.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:04.426 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:27:04.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:04.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:04.426 | INFO | Uploaded 36GiB to GPU in 1.97454s (18.2321GiB/s) | 20260506 16:27:04.426 | INFO | Downloaded 34GiB to GPU in 1.93197s (17.5986GiB/s) | 20260506 16:27:09.563 | INFO | Analyzing chunk frames = [2782720,2783232) | 20260506 16:27:09.563 | INFO | MemInfo: { | 20260506 16:27:09.563 | INFO | "gpu" : " 17 GB", | 20260506 16:27:09.563 | INFO | "huge" : "", | 20260506 16:27:09.563 | INFO | "huge_shared" : " 27 GB", | 20260506 16:27:09.563 | INFO | "malloc" : " 14 GB", | 20260506 16:27:09.563 | INFO | "pinned" : " 29 GB" | 20260506 16:27:09.563 | INFO | } | 20260506 16:27:09.563 | INFO | Perf report for the last 5.13709 seconds: | 20260506 16:27:09.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:27:09.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:09.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:09.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:09.563 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260506 16:27:09.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:09.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:09.563 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:27:09.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:09.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:09.563 | INFO | Uploaded 36GiB to GPU in 1.92597s (18.6919GiB/s) | 20260506 16:27:09.563 | INFO | Downloaded 34GiB to GPU in 1.87641s (18.1197GiB/s) | 20260506 16:27:14.703 | INFO | Analyzing chunk frames = [2783232,2783744) | 20260506 16:27:14.703 | INFO | MemInfo: { | 20260506 16:27:14.703 | INFO | "gpu" : " 17 GB", | 20260506 16:27:14.703 | INFO | "huge" : "", | 20260506 16:27:14.703 | INFO | "huge_shared" : " 27 GB", | 20260506 16:27:14.703 | INFO | "malloc" : " 14 GB", | 20260506 16:27:14.703 | INFO | "pinned" : " 29 GB" | 20260506 16:27:14.703 | INFO | } | 20260506 16:27:14.703 | INFO | Perf report for the last 5.14064 seconds: | 20260506 16:27:14.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:27:14.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:14.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:14.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:14.704 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 16:27:14.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:14.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:14.704 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:27:14.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:14.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:14.704 | INFO | Uploaded 36GiB to GPU in 1.99818s (18.0164GiB/s) | 20260506 16:27:14.704 | INFO | Downloaded 34GiB to GPU in 1.91974s (17.7108GiB/s) | 20260506 16:27:19.852 | INFO | Analyzing chunk frames = [2783744,2784256) | 20260506 16:27:19.852 | INFO | MemInfo: { | 20260506 16:27:19.852 | INFO | "gpu" : " 17 GB", | 20260506 16:27:19.852 | INFO | "huge" : "", | 20260506 16:27:19.852 | INFO | "huge_shared" : " 27 GB", | 20260506 16:27:19.852 | INFO | "malloc" : " 14 GB", | 20260506 16:27:19.852 | INFO | "pinned" : " 29 GB" | 20260506 16:27:19.852 | INFO | } | 20260506 16:27:19.852 | INFO | Perf report for the last 5.14867 seconds: | 20260506 16:27:19.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:27:19.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:19.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:19.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:19.852 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 16:27:19.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:19.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:19.852 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:27:19.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:19.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:19.852 | INFO | Uploaded 36GiB to GPU in 2.03273s (17.7101GiB/s) | 20260506 16:27:19.852 | INFO | Downloaded 34GiB to GPU in 1.93497s (17.5713GiB/s) | 20260506 16:27:24.988 | INFO | Analyzing chunk frames = [2784256,2784768) | 20260506 16:27:24.988 | INFO | MemInfo: { | 20260506 16:27:24.988 | INFO | "gpu" : " 17 GB", | 20260506 16:27:24.988 | INFO | "huge" : "", | 20260506 16:27:24.988 | INFO | "huge_shared" : " 27 GB", | 20260506 16:27:24.988 | INFO | "malloc" : " 14 GB", | 20260506 16:27:24.988 | INFO | "pinned" : " 29 GB" | 20260506 16:27:24.988 | INFO | } | 20260506 16:27:24.988 | INFO | Perf report for the last 5.13587 seconds: | 20260506 16:27:24.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:27:24.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:24.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:24.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:24.989 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:27:24.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:24.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:24.989 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 16:27:24.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:24.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:24.989 | INFO | Uploaded 36GiB to GPU in 2.00953s (17.9146GiB/s) | 20260506 16:27:24.989 | INFO | Downloaded 34GiB to GPU in 1.99389s (17.0521GiB/s) | 20260506 16:27:30.131 | INFO | Analyzing chunk frames = [2784768,2785280) | 20260506 16:27:30.131 | INFO | MemInfo: { | 20260506 16:27:30.131 | INFO | "gpu" : " 17 GB", | 20260506 16:27:30.131 | INFO | "huge" : "", | 20260506 16:27:30.132 | INFO | "huge_shared" : " 27 GB", | 20260506 16:27:30.132 | INFO | "malloc" : " 14 GB", | 20260506 16:27:30.132 | INFO | "pinned" : " 29 GB" | 20260506 16:27:30.132 | INFO | } | 20260506 16:27:30.132 | INFO | Perf report for the last 5.14362 seconds: | 20260506 16:27:30.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:27:30.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:30.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:30.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:30.132 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 16:27:30.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:30.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:30.132 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 16:27:30.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:30.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:30.132 | INFO | Uploaded 36GiB to GPU in 2.04348s (17.617GiB/s) | 20260506 16:27:30.132 | INFO | Downloaded 34GiB to GPU in 1.98095s (17.1635GiB/s) | 20260506 16:27:30.132 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:27:30.132 | INFO | Total mem usage: 18 GiB | 20260506 16:27:30.132 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:27:30.132 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.2%) | 20260506 16:27:30.132 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:27:30.132 | INFO | High Water Breakdown: | 20260506 16:27:30.132 | INFO | 61.2% : 9.48 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:27:30.132 | INFO | 38.7% : 5.99 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:27:30.132 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:27:30.132 | INFO | ------------------------------------------ | 20260506 16:27:30.132 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:27:30.132 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:27:30.132 | INFO | Total mem usage: 26.5 GiB | 20260506 16:27:30.132 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:27:30.132 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 16:27:30.132 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 16:27:30.132 | INFO | High Water Breakdown: | 20260506 16:27:30.132 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:27:30.132 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:27:30.132 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:27:30.132 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:27:30.132 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:27:30.133 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:27:30.133 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:27:30.133 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:27:30.133 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:27:30.133 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:27:30.133 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:27:30.133 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:27:30.133 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:27:30.133 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:27:30.133 | INFO | ------------------------------------------ | 20260506 16:27:30.133 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:27:30.133 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:27:30.133 | INFO | Total mem usage: 25.6 GiB | 20260506 16:27:30.133 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:27:30.133 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:27:30.133 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:27:30.133 | INFO | High Water Breakdown: | 20260506 16:27:30.133 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:27:30.133 | INFO | ------------------------------------------ | 20260506 16:27:30.133 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:27:30.133 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:27:30.133 | INFO | Total mem usage: 15.7 GiB | 20260506 16:27:30.133 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:27:30.133 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:27:30.133 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 16:27:30.133 | INFO | High Water Breakdown: | 20260506 16:27:30.133 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:27:30.133 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:27:30.133 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:27:30.133 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:27:30.133 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:27:30.133 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:27:30.133 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:27:30.133 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:27:30.133 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:27:30.133 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:27:30.133 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:27:30.133 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:27:30.133 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:27:30.133 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:27:30.133 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:27:30.133 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:27:30.133 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:27:30.133 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:27:30.133 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:27:30.133 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:27:30.133 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:27:30.133 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:27:30.133 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:27:30.133 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:27:30.133 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:27:30.133 | INFO | ------------------------------------------ | 20260506 16:27:30.133 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:27:30.133 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:27:30.133 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:27:30.133 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:27:30.133 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.58e+01, 1.35e+01 / 2.42e+00 | 20260506 16:27:30.133 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.00e-01, 1.00e-02 / 5.31e-03 | 20260506 16:27:30.133 | INFO | PerfReport: 11.61%, Baseline -- 3.65e+00 / 4.48e+00, 5.65e-01 / 3.01e-01 | 20260506 16:27:30.133 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.47e+00, 1.87e-01 / 1.34e-01 | 20260506 16:27:30.133 | INFO | PerfReport: 15.95%, DME -- 4.01e+01 / 4.45e+01, 3.98e+00 / 2.66e+00 | 20260506 16:27:30.133 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.45e+00 / 1.75e+00 | 20260506 16:27:30.133 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.30e+00 / 2.86e+00, 3.84e-01 / 2.42e-01 | 20260506 16:27:30.133 | INFO | PerfReport: 3.50%, Metrics -- 1.10e+00 / 1.46e+00, 2.84e-01 / 1.80e-01 | 20260506 16:27:30.133 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.82e-01 / 4.31e-01, 5.95e-02 / 2.28e-02 | 20260506 16:27:30.133 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:27:30.133 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:27:30.133 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:27:30.133 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 16:27:30.133 | INFO | PerfReport: 21.51%, Upload -- 2.09e+01 / 0.00e+00, 7.04e+00 / 0.00e+00 | 20260506 16:27:30.133 | INFO | PerfReport: 21.14%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 16:27:30.133 | INFO | PerfReport: 32.29%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 16:27:30.133 | INFO | PerfReport: 1.49%, LockUpload -- 1.45e+00 / 0.00e+00, 5.90e+00 / 0.00e+00 | 20260506 16:27:30.133 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 16:27:30.133 | INFO | PerfReport: 22.46%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 16:27:30.133 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260506 16:27:30.245 | INFO | Starting a round of Baz IO | 20260506 16:27:33.397 | INFO | Populating RTMetrics report | 20260506 16:27:33.397 | INFO | RTMetrics report populated | 20260506 16:27:33.397 | INFO | Opening tmp RTMetrics json file | 20260506 16:27:33.407 | INFO | Serializing RTMetrics to json | 20260506 16:27:33.411 | INFO | Serialization done, swapping tmp file now... | 20260506 16:27:33.412 | INFO | RTMetrics done | 20260506 16:27:33.903 | INFO | Finished a round of Baz IO: Wrote 2.087 GiB in 3.658 seconds | 20260506 16:27:33.903 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.33% / 0.01% / 63.66% | 20260506 16:27:33.903 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.68304% / 3.17891e-05% / 5.16574e-05% | 20260506 16:27:33.903 | INFO | Baz IO overall: | 20260506 16:27:33.904 | INFO | BytesWritten : 1462154951544 | 20260506 16:27:33.904 | INFO | NumEvents : 707533314988 | 20260506 16:27:35.265 | INFO | Analyzing chunk frames = [2785280,2785792) | 20260506 16:27:35.265 | INFO | MemInfo: { | 20260506 16:27:35.265 | INFO | "gpu" : " 17 GB", | 20260506 16:27:35.265 | INFO | "huge" : "", | 20260506 16:27:35.265 | INFO | "huge_shared" : " 27 GB", | 20260506 16:27:35.265 | INFO | "malloc" : " 14 GB", | 20260506 16:27:35.265 | INFO | "pinned" : " 29 GB" | 20260506 16:27:35.265 | INFO | } | 20260506 16:27:35.265 | INFO | Perf report for the last 5.13349 seconds: | 20260506 16:27:35.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:27:35.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:35.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:35.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:35.265 | INFO | ANALYSIS : 63.0%, 2.9, 95.5ms, 0 | 20260506 16:27:35.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:35.265 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260506 16:27:35.265 | INFO | PRE_HQ : 60.7%, 1.9, 58.6ms, 0 | 20260506 16:27:35.265 | INFO | BAZWRITER : 70.6%, 4.8, 175.6ms, 0 | 20260506 16:27:35.265 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 16:27:35.265 | INFO | Uploaded 36GiB to GPU in 2.05057s (17.5561GiB/s) | 20260506 16:27:35.265 | INFO | Downloaded 38GiB to GPU in 2.21637s (17.1452GiB/s) | 20260506 16:27:40.401 | INFO | Analyzing chunk frames = [2785792,2786304) | 20260506 16:27:40.401 | INFO | MemInfo: { | 20260506 16:27:40.401 | INFO | "gpu" : " 17 GB", | 20260506 16:27:40.401 | INFO | "huge" : "", | 20260506 16:27:40.401 | INFO | "huge_shared" : " 27 GB", | 20260506 16:27:40.401 | INFO | "malloc" : " 14 GB", | 20260506 16:27:40.401 | INFO | "pinned" : " 29 GB" | 20260506 16:27:40.401 | INFO | } | 20260506 16:27:40.401 | INFO | Perf report for the last 5.13621 seconds: | 20260506 16:27:40.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:27:40.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:40.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:40.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:40.401 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:27:40.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:40.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:40.401 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:27:40.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:40.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:40.402 | INFO | Uploaded 36GiB to GPU in 1.98941s (18.0959GiB/s) | 20260506 16:27:40.402 | INFO | Downloaded 34GiB to GPU in 1.93096s (17.6079GiB/s) | 20260506 16:27:45.537 | INFO | Analyzing chunk frames = [2786304,2786816) | 20260506 16:27:45.537 | INFO | MemInfo: { | 20260506 16:27:45.537 | INFO | "gpu" : " 17 GB", | 20260506 16:27:45.538 | INFO | "huge" : "", | 20260506 16:27:45.538 | INFO | "huge_shared" : " 27 GB", | 20260506 16:27:45.538 | INFO | "malloc" : " 14 GB", | 20260506 16:27:45.538 | INFO | "pinned" : " 29 GB" | 20260506 16:27:45.538 | INFO | } | 20260506 16:27:45.538 | INFO | Perf report for the last 5.13621 seconds: | 20260506 16:27:45.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:27:45.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:45.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:45.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:45.538 | INFO | ANALYSIS : 61.2%, 3.0, 94.6ms, 0 | 20260506 16:27:45.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:45.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:45.538 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:27:45.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:45.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:45.538 | INFO | Uploaded 36GiB to GPU in 1.98071s (18.1753GiB/s) | 20260506 16:27:45.538 | INFO | Downloaded 34GiB to GPU in 1.93124s (17.6053GiB/s) | 20260506 16:27:50.686 | INFO | Analyzing chunk frames = [2786816,2787328) | 20260506 16:27:50.686 | INFO | MemInfo: { | 20260506 16:27:50.686 | INFO | "gpu" : " 17 GB", | 20260506 16:27:50.686 | INFO | "huge" : "", | 20260506 16:27:50.686 | INFO | "huge_shared" : " 27 GB", | 20260506 16:27:50.686 | INFO | "malloc" : " 14 GB", | 20260506 16:27:50.686 | INFO | "pinned" : " 29 GB" | 20260506 16:27:50.686 | INFO | } | 20260506 16:27:50.686 | INFO | Perf report for the last 5.14856 seconds: | 20260506 16:27:50.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:27:50.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:50.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:50.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:50.686 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260506 16:27:50.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:50.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:50.686 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:27:50.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:50.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:50.686 | INFO | Uploaded 36GiB to GPU in 1.94026s (18.5542GiB/s) | 20260506 16:27:50.686 | INFO | Downloaded 34GiB to GPU in 1.88073s (18.0781GiB/s) | 20260506 16:27:55.824 | INFO | Analyzing chunk frames = [2787328,2787840) | 20260506 16:27:55.824 | INFO | MemInfo: { | 20260506 16:27:55.824 | INFO | "gpu" : " 17 GB", | 20260506 16:27:55.824 | INFO | "huge" : "", | 20260506 16:27:55.824 | INFO | "huge_shared" : " 27 GB", | 20260506 16:27:55.824 | INFO | "malloc" : " 14 GB", | 20260506 16:27:55.824 | INFO | "pinned" : " 29 GB" | 20260506 16:27:55.824 | INFO | } | 20260506 16:27:55.824 | INFO | Perf report for the last 5.13753 seconds: | 20260506 16:27:55.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:27:55.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:55.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:55.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:55.824 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 16:27:55.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:55.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:55.824 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:27:55.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:55.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:27:55.824 | INFO | Uploaded 36GiB to GPU in 2.02063s (17.8162GiB/s) | 20260506 16:27:55.824 | INFO | Downloaded 34GiB to GPU in 1.9078s (17.8216GiB/s) | 20260506 16:28:00.963 | INFO | Analyzing chunk frames = [2787840,2788352) | 20260506 16:28:00.963 | INFO | MemInfo: { | 20260506 16:28:00.963 | INFO | "gpu" : " 17 GB", | 20260506 16:28:00.963 | INFO | "huge" : "", | 20260506 16:28:00.963 | INFO | "huge_shared" : " 27 GB", | 20260506 16:28:00.963 | INFO | "malloc" : " 14 GB", | 20260506 16:28:00.963 | INFO | "pinned" : " 29 GB" | 20260506 16:28:00.963 | INFO | } | 20260506 16:28:00.963 | INFO | Perf report for the last 5.1395 seconds: | 20260506 16:28:00.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:28:00.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:00.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:00.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:00.963 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:28:00.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:00.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:00.963 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 16:28:00.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:00.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:00.964 | INFO | Uploaded 36GiB to GPU in 2.08508s (17.2655GiB/s) | 20260506 16:28:00.964 | INFO | Downloaded 34GiB to GPU in 1.96919s (17.266GiB/s) | 20260506 16:28:06.104 | INFO | Analyzing chunk frames = [2788352,2788864) | 20260506 16:28:06.104 | INFO | MemInfo: { | 20260506 16:28:06.104 | INFO | "gpu" : " 17 GB", | 20260506 16:28:06.104 | INFO | "huge" : "", | 20260506 16:28:06.104 | INFO | "huge_shared" : " 27 GB", | 20260506 16:28:06.104 | INFO | "malloc" : " 14 GB", | 20260506 16:28:06.104 | INFO | "pinned" : " 29 GB" | 20260506 16:28:06.104 | INFO | } | 20260506 16:28:06.104 | INFO | Perf report for the last 5.14088 seconds: | 20260506 16:28:06.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:28:06.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:06.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:06.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:06.104 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 16:28:06.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:06.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:06.104 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 16:28:06.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:06.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:06.104 | INFO | Uploaded 36GiB to GPU in 2.01827s (17.837GiB/s) | 20260506 16:28:06.105 | INFO | Downloaded 34GiB to GPU in 2.03329s (16.7217GiB/s) | 20260506 16:28:11.242 | INFO | Analyzing chunk frames = [2788864,2789376) | 20260506 16:28:11.242 | INFO | MemInfo: { | 20260506 16:28:11.242 | INFO | "gpu" : " 17 GB", | 20260506 16:28:11.242 | INFO | "huge" : "", | 20260506 16:28:11.242 | INFO | "huge_shared" : " 27 GB", | 20260506 16:28:11.242 | INFO | "malloc" : " 14 GB", | 20260506 16:28:11.242 | INFO | "pinned" : " 29 GB" | 20260506 16:28:11.242 | INFO | } | 20260506 16:28:11.242 | INFO | Perf report for the last 5.138 seconds: | 20260506 16:28:11.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:28:11.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:11.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:11.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:11.242 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 16:28:11.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:11.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:11.242 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260506 16:28:11.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:11.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:11.242 | INFO | Uploaded 36GiB to GPU in 2.05001s (17.5609GiB/s) | 20260506 16:28:11.242 | INFO | Downloaded 34GiB to GPU in 1.97785s (17.1904GiB/s) | 20260506 16:28:11.366 | INFO | Starting a round of Baz IO | 20260506 16:28:14.551 | INFO | Populating RTMetrics report | 20260506 16:28:14.551 | INFO | RTMetrics report populated | 20260506 16:28:14.551 | INFO | Opening tmp RTMetrics json file | 20260506 16:28:14.616 | INFO | Serializing RTMetrics to json | 20260506 16:28:14.678 | INFO | Serialization done, swapping tmp file now... | 20260506 16:28:14.703 | INFO | RTMetrics done | 20260506 16:28:15.135 | INFO | Finished a round of Baz IO: Wrote 2.087 GiB in 3.768 seconds | 20260506 16:28:15.135 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.32% / 0.01% / 63.67% | 20260506 16:28:15.135 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.68151% / 1.19209e-05% / 5.16574e-05% | 20260506 16:28:15.135 | INFO | Baz IO overall: | 20260506 16:28:15.135 | INFO | BytesWritten : 1464395862857 | 20260506 16:28:15.135 | INFO | NumEvents : 708616597984 | 20260506 16:28:16.376 | INFO | Analyzing chunk frames = [2789376,2789888) | 20260506 16:28:16.376 | INFO | MemInfo: { | 20260506 16:28:16.376 | INFO | "gpu" : " 17 GB", | 20260506 16:28:16.376 | INFO | "huge" : "", | 20260506 16:28:16.376 | INFO | "huge_shared" : " 27 GB", | 20260506 16:28:16.376 | INFO | "malloc" : " 14 GB", | 20260506 16:28:16.376 | INFO | "pinned" : " 29 GB" | 20260506 16:28:16.376 | INFO | } | 20260506 16:28:16.376 | INFO | Perf report for the last 5.13361 seconds: | 20260506 16:28:16.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:28:16.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:16.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:16.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:16.376 | INFO | ANALYSIS : 63.6%, 2.8, 94.7ms, 0 | 20260506 16:28:16.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:16.376 | INFO | RT_METRICS : 49.7%, 1.0, 26.0ms, 0 | 20260506 16:28:16.376 | INFO | PRE_HQ : 61.3%, 2.0, 63.4ms, 0 | 20260506 16:28:16.376 | INFO | BAZWRITER : 72.8%, 4.8, 179.8ms, 0 | 20260506 16:28:16.376 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 16:28:16.376 | INFO | Uploaded 36GiB to GPU in 2.07476s (17.3514GiB/s) | 20260506 16:28:16.376 | INFO | Downloaded 38GiB to GPU in 2.21148s (17.183GiB/s) | 20260506 16:28:21.517 | INFO | Analyzing chunk frames = [2789888,2790400) | 20260506 16:28:21.517 | INFO | MemInfo: { | 20260506 16:28:21.517 | INFO | "gpu" : " 17 GB", | 20260506 16:28:21.517 | INFO | "huge" : "", | 20260506 16:28:21.517 | INFO | "huge_shared" : " 27 GB", | 20260506 16:28:21.517 | INFO | "malloc" : " 14 GB", | 20260506 16:28:21.517 | INFO | "pinned" : " 29 GB" | 20260506 16:28:21.517 | INFO | } | 20260506 16:28:21.517 | INFO | Perf report for the last 5.14134 seconds: | 20260506 16:28:21.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:28:21.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:21.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:21.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:21.517 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 16:28:21.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:21.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:21.517 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:28:21.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:21.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:21.517 | INFO | Uploaded 36GiB to GPU in 1.9828s (18.1562GiB/s) | 20260506 16:28:21.517 | INFO | Downloaded 34GiB to GPU in 1.94333s (17.4958GiB/s) | 20260506 16:28:26.656 | INFO | Analyzing chunk frames = [2790400,2790912) | 20260506 16:28:26.656 | INFO | MemInfo: { | 20260506 16:28:26.656 | INFO | "gpu" : " 17 GB", | 20260506 16:28:26.656 | INFO | "huge" : "", | 20260506 16:28:26.656 | INFO | "huge_shared" : " 27 GB", | 20260506 16:28:26.656 | INFO | "malloc" : " 14 GB", | 20260506 16:28:26.656 | INFO | "pinned" : " 29 GB" | 20260506 16:28:26.656 | INFO | } | 20260506 16:28:26.656 | INFO | Perf report for the last 5.13868 seconds: | 20260506 16:28:26.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:28:26.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:26.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:26.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:26.656 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 16:28:26.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:26.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:26.656 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:28:26.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:26.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:26.656 | INFO | Uploaded 36GiB to GPU in 1.95846s (18.3818GiB/s) | 20260506 16:28:26.656 | INFO | Downloaded 34GiB to GPU in 1.92968s (17.6195GiB/s) | 20260506 16:28:31.792 | INFO | Analyzing chunk frames = [2790912,2791424) | 20260506 16:28:31.792 | INFO | MemInfo: { | 20260506 16:28:31.792 | INFO | "gpu" : " 17 GB", | 20260506 16:28:31.792 | INFO | "huge" : "", | 20260506 16:28:31.792 | INFO | "huge_shared" : " 27 GB", | 20260506 16:28:31.792 | INFO | "malloc" : " 14 GB", | 20260506 16:28:31.792 | INFO | "pinned" : " 29 GB" | 20260506 16:28:31.792 | INFO | } | 20260506 16:28:31.792 | INFO | Perf report for the last 5.13599 seconds: | 20260506 16:28:31.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:28:31.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:31.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:31.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:31.792 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:28:31.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:31.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:31.792 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:28:31.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:31.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:31.792 | INFO | Uploaded 36GiB to GPU in 1.92297s (18.721GiB/s) | 20260506 16:28:31.792 | INFO | Downloaded 34GiB to GPU in 1.87851s (18.0995GiB/s) | 20260506 16:28:36.945 | INFO | Analyzing chunk frames = [2791424,2791936) | 20260506 16:28:36.945 | INFO | MemInfo: { | 20260506 16:28:36.945 | INFO | "gpu" : " 17 GB", | 20260506 16:28:36.945 | INFO | "huge" : "", | 20260506 16:28:36.945 | INFO | "huge_shared" : " 27 GB", | 20260506 16:28:36.945 | INFO | "malloc" : " 14 GB", | 20260506 16:28:36.945 | INFO | "pinned" : " 29 GB" | 20260506 16:28:36.945 | INFO | } | 20260506 16:28:36.945 | INFO | Perf report for the last 5.15322 seconds: | 20260506 16:28:36.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:28:36.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:36.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:36.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:36.945 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260506 16:28:36.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:36.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:36.945 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260506 16:28:36.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:36.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:36.945 | INFO | Uploaded 36GiB to GPU in 1.94872s (18.4737GiB/s) | 20260506 16:28:36.945 | INFO | Downloaded 34GiB to GPU in 1.88407s (18.046GiB/s) | 20260506 16:28:42.078 | INFO | Analyzing chunk frames = [2791936,2792448) | 20260506 16:28:42.078 | INFO | MemInfo: { | 20260506 16:28:42.078 | INFO | "gpu" : " 17 GB", | 20260506 16:28:42.078 | INFO | "huge" : "", | 20260506 16:28:42.078 | INFO | "huge_shared" : " 27 GB", | 20260506 16:28:42.078 | INFO | "malloc" : " 14 GB", | 20260506 16:28:42.078 | INFO | "pinned" : " 29 GB" | 20260506 16:28:42.078 | INFO | } | 20260506 16:28:42.078 | INFO | Perf report for the last 5.13314 seconds: | 20260506 16:28:42.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:28:42.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:42.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:42.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:42.078 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 16:28:42.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:42.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:42.078 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:28:42.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:42.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:42.079 | INFO | Uploaded 36GiB to GPU in 2.00987s (17.9116GiB/s) | 20260506 16:28:42.079 | INFO | Downloaded 34GiB to GPU in 1.96269s (17.3231GiB/s) | 20260506 16:28:47.218 | INFO | Analyzing chunk frames = [2792448,2792960) | 20260506 16:28:47.218 | INFO | MemInfo: { | 20260506 16:28:47.218 | INFO | "gpu" : " 17 GB", | 20260506 16:28:47.218 | INFO | "huge" : "", | 20260506 16:28:47.218 | INFO | "huge_shared" : " 27 GB", | 20260506 16:28:47.218 | INFO | "malloc" : " 14 GB", | 20260506 16:28:47.218 | INFO | "pinned" : " 29 GB" | 20260506 16:28:47.218 | INFO | } | 20260506 16:28:47.218 | INFO | Perf report for the last 5.1397 seconds: | 20260506 16:28:47.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:28:47.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:47.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:47.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:47.218 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 16:28:47.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:47.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:47.218 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:28:47.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:47.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:47.218 | INFO | Uploaded 36GiB to GPU in 2.01393s (17.8755GiB/s) | 20260506 16:28:47.218 | INFO | Downloaded 34GiB to GPU in 2.01603s (16.8648GiB/s) | 20260506 16:28:52.356 | INFO | Analyzing chunk frames = [2792960,2793472) | 20260506 16:28:52.356 | INFO | MemInfo: { | 20260506 16:28:52.356 | INFO | "gpu" : " 17 GB", | 20260506 16:28:52.356 | INFO | "huge" : "", | 20260506 16:28:52.356 | INFO | "huge_shared" : " 27 GB", | 20260506 16:28:52.356 | INFO | "malloc" : " 14 GB", | 20260506 16:28:52.356 | INFO | "pinned" : " 29 GB" | 20260506 16:28:52.356 | INFO | } | 20260506 16:28:52.356 | INFO | Perf report for the last 5.13787 seconds: | 20260506 16:28:52.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:28:52.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:52.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:52.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:52.356 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 16:28:52.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:52.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:52.356 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:28:52.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:52.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:52.356 | INFO | Uploaded 36GiB to GPU in 2.00579s (17.948GiB/s) | 20260506 16:28:52.356 | INFO | Downloaded 34GiB to GPU in 1.97082s (17.2517GiB/s) | 20260506 16:28:52.356 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:28:52.356 | INFO | Total mem usage: 18 GiB | 20260506 16:28:52.356 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:28:52.356 | INFO | Recent High Water Marks Sum: 16.6 GiB (Savings: -8.54%) | 20260506 16:28:52.356 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:28:52.356 | INFO | High Water Breakdown: | 20260506 16:28:52.356 | INFO | 62.2% : 10.3 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:28:52.356 | INFO | 37.7% : 6.25 GiB ( 93.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:28:52.356 | INFO | 0.12% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:28:52.356 | INFO | ------------------------------------------ | 20260506 16:28:52.356 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:28:52.356 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:28:52.356 | INFO | Total mem usage: 26.5 GiB | 20260506 16:28:52.356 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:28:52.356 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 16:28:52.356 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260506 16:28:52.356 | INFO | High Water Breakdown: | 20260506 16:28:52.357 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:28:52.357 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:28:52.357 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:28:52.357 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:28:52.357 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:28:52.357 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:28:52.357 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:28:52.357 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:28:52.357 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:28:52.357 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:28:52.357 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:28:52.357 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:28:52.357 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:28:52.357 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:28:52.357 | INFO | ------------------------------------------ | 20260506 16:28:52.357 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:28:52.357 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:28:52.357 | INFO | Total mem usage: 25.6 GiB | 20260506 16:28:52.357 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:28:52.357 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:28:52.357 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:28:52.357 | INFO | High Water Breakdown: | 20260506 16:28:52.357 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:28:52.357 | INFO | ------------------------------------------ | 20260506 16:28:52.357 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:28:52.357 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:28:52.357 | INFO | Total mem usage: 15.7 GiB | 20260506 16:28:52.357 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:28:52.357 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:28:52.357 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 16:28:52.357 | INFO | High Water Breakdown: | 20260506 16:28:52.357 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:28:52.357 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:28:52.357 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:28:52.357 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:28:52.357 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:28:52.357 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:28:52.357 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:28:52.357 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:28:52.357 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:28:52.357 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:28:52.357 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:28:52.357 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:28:52.357 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:28:52.357 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:28:52.357 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:28:52.357 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:28:52.357 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:28:52.357 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:28:52.357 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:28:52.357 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:28:52.357 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:28:52.357 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:28:52.357 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:28:52.357 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:28:52.357 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:28:52.357 | INFO | ------------------------------------------ | 20260506 16:28:52.357 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:28:52.357 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:28:52.357 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:28:52.357 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:28:52.357 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.61e+01, 1.34e+01 / 2.85e+00 | 20260506 16:28:52.357 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.02e-01, 1.09e-02 / 5.79e-03 | 20260506 16:28:52.357 | INFO | PerfReport: 11.56%, Baseline -- 3.64e+00 / 4.40e+00, 5.58e-01 / 3.42e-01 | 20260506 16:28:52.357 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.48e+00, 1.85e-01 / 1.24e-01 | 20260506 16:28:52.357 | INFO | PerfReport: 16.00%, DME -- 4.02e+01 / 4.50e+01, 4.18e+00 / 2.92e+00 | 20260506 16:28:52.357 | INFO | PerfReport: 56.91%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.49e+00 / 1.63e+00 | 20260506 16:28:52.357 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.29e+00 / 2.83e+00, 3.85e-01 / 2.48e-01 | 20260506 16:28:52.357 | INFO | PerfReport: 3.48%, Metrics -- 1.10e+00 / 1.43e+00, 2.84e-01 / 1.82e-01 | 20260506 16:28:52.357 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.82e-01 / 4.30e-01, 6.01e-02 / 3.12e-02 | 20260506 16:28:52.358 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:28:52.358 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:28:52.358 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:28:52.358 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 16:28:52.358 | INFO | PerfReport: 21.43%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 16:28:52.358 | INFO | PerfReport: 21.16%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 16:28:52.358 | INFO | PerfReport: 32.30%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 16:28:52.358 | INFO | PerfReport: 1.39%, LockUpload -- 1.35e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260506 16:28:52.358 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 16:28:52.358 | INFO | PerfReport: 22.60%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 16:28:52.358 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260506 16:28:52.476 | INFO | Starting a round of Baz IO | 20260506 16:28:55.621 | INFO | Populating RTMetrics report | 20260506 16:28:55.621 | INFO | RTMetrics report populated | 20260506 16:28:55.621 | INFO | Opening tmp RTMetrics json file | 20260506 16:28:55.824 | INFO | Serializing RTMetrics to json | 20260506 16:28:55.982 | INFO | Serialization done, swapping tmp file now... | 20260506 16:28:56.057 | INFO | RTMetrics done | 20260506 16:28:56.078 | INFO | Finished a round of Baz IO: Wrote 2.086 GiB in 3.602 seconds | 20260506 16:28:56.078 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.31% / 0.01% / 63.68% | 20260506 16:28:56.078 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.67487% / 3.97364e-06% / 5.96046e-05% | 20260506 16:28:56.078 | INFO | Baz IO overall: | 20260506 16:28:56.078 | INFO | BytesWritten : 1466636181298 | 20260506 16:28:56.078 | INFO | NumEvents : 709699177254 | 20260506 16:28:57.490 | INFO | Analyzing chunk frames = [2793472,2793984) | 20260506 16:28:57.490 | INFO | MemInfo: { | 20260506 16:28:57.490 | INFO | "gpu" : " 17 GB", | 20260506 16:28:57.490 | INFO | "huge" : "", | 20260506 16:28:57.490 | INFO | "huge_shared" : " 27 GB", | 20260506 16:28:57.490 | INFO | "malloc" : " 14 GB", | 20260506 16:28:57.490 | INFO | "pinned" : " 29 GB" | 20260506 16:28:57.490 | INFO | } | 20260506 16:28:57.490 | INFO | Perf report for the last 5.13427 seconds: | 20260506 16:28:57.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:28:57.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:57.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:57.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:57.490 | INFO | ANALYSIS : 63.0%, 2.9, 95.4ms, 0 | 20260506 16:28:57.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:28:57.490 | INFO | RT_METRICS : 52.2%, 1.0, 27.4ms, 0 | 20260506 16:28:57.490 | INFO | PRE_HQ : 60.4%, 1.9, 58.6ms, 0 | 20260506 16:28:57.490 | INFO | BAZWRITER : 69.7%, 4.8, 172.4ms, 0 | 20260506 16:28:57.490 | INFO | BAZ_SAMPLING : 3.8%, 1.0, 2.0ms, 0 | 20260506 16:28:57.491 | INFO | Uploaded 36GiB to GPU in 2.06966s (17.3942GiB/s) | 20260506 16:28:57.491 | INFO | Downloaded 38GiB to GPU in 2.21719s (17.1388GiB/s) | 20260506 16:29:02.631 | INFO | Analyzing chunk frames = [2793984,2794496) | 20260506 16:29:02.631 | INFO | MemInfo: { | 20260506 16:29:02.631 | INFO | "gpu" : " 17 GB", | 20260506 16:29:02.631 | INFO | "huge" : "", | 20260506 16:29:02.631 | INFO | "huge_shared" : " 27 GB", | 20260506 16:29:02.631 | INFO | "malloc" : " 14 GB", | 20260506 16:29:02.631 | INFO | "pinned" : " 29 GB" | 20260506 16:29:02.631 | INFO | } | 20260506 16:29:02.631 | INFO | Perf report for the last 5.1412 seconds: | 20260506 16:29:02.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:29:02.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:02.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:02.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:02.631 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 16:29:02.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:02.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:02.631 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:29:02.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:02.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:02.632 | INFO | Uploaded 36GiB to GPU in 1.99263s (18.0665GiB/s) | 20260506 16:29:02.632 | INFO | Downloaded 34GiB to GPU in 1.93738s (17.5494GiB/s) | 20260506 16:29:07.779 | INFO | Analyzing chunk frames = [2794496,2795008) | 20260506 16:29:07.779 | INFO | MemInfo: { | 20260506 16:29:07.779 | INFO | "gpu" : " 17 GB", | 20260506 16:29:07.779 | INFO | "huge" : "", | 20260506 16:29:07.779 | INFO | "huge_shared" : " 27 GB", | 20260506 16:29:07.779 | INFO | "malloc" : " 14 GB", | 20260506 16:29:07.779 | INFO | "pinned" : " 29 GB" | 20260506 16:29:07.779 | INFO | } | 20260506 16:29:07.779 | INFO | Perf report for the last 5.1476 seconds: | 20260506 16:29:07.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:29:07.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:07.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:07.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:07.779 | INFO | ANALYSIS : 61.1%, 3.0, 94.8ms, 0 | 20260506 16:29:07.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:07.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:07.779 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:29:07.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:07.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:07.779 | INFO | Uploaded 36GiB to GPU in 1.99805s (18.0176GiB/s) | 20260506 16:29:07.779 | INFO | Downloaded 34GiB to GPU in 1.93189s (17.5993GiB/s) | 20260506 16:29:12.908 | INFO | Analyzing chunk frames = [2795008,2795520) | 20260506 16:29:12.908 | INFO | MemInfo: { | 20260506 16:29:12.908 | INFO | "gpu" : " 17 GB", | 20260506 16:29:12.908 | INFO | "huge" : "", | 20260506 16:29:12.908 | INFO | "huge_shared" : " 27 GB", | 20260506 16:29:12.908 | INFO | "malloc" : " 14 GB", | 20260506 16:29:12.908 | INFO | "pinned" : " 29 GB" | 20260506 16:29:12.908 | INFO | } | 20260506 16:29:12.908 | INFO | Perf report for the last 5.12904 seconds: | 20260506 16:29:12.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:29:12.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:12.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:12.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:12.908 | INFO | ANALYSIS : 60.2%, 3.0, 93.1ms, 0 | 20260506 16:29:12.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:12.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:12.908 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 16:29:12.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:12.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:12.908 | INFO | Uploaded 36GiB to GPU in 1.94324s (18.5258GiB/s) | 20260506 16:29:12.908 | INFO | Downloaded 34GiB to GPU in 1.87537s (18.1297GiB/s) | 20260506 16:29:18.049 | INFO | Analyzing chunk frames = [2795520,2796032) | 20260506 16:29:18.049 | INFO | MemInfo: { | 20260506 16:29:18.049 | INFO | "gpu" : " 17 GB", | 20260506 16:29:18.049 | INFO | "huge" : "", | 20260506 16:29:18.049 | INFO | "huge_shared" : " 27 GB", | 20260506 16:29:18.049 | INFO | "malloc" : " 14 GB", | 20260506 16:29:18.049 | INFO | "pinned" : " 29 GB" | 20260506 16:29:18.049 | INFO | } | 20260506 16:29:18.049 | INFO | Perf report for the last 5.14126 seconds: | 20260506 16:29:18.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:29:18.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:18.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:18.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:18.049 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 16:29:18.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:18.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:18.049 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:29:18.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:18.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:18.050 | INFO | Uploaded 36GiB to GPU in 2.02765s (17.7546GiB/s) | 20260506 16:29:18.050 | INFO | Downloaded 34GiB to GPU in 1.90751s (17.8243GiB/s) | 20260506 16:29:23.193 | INFO | Analyzing chunk frames = [2796032,2796544) | 20260506 16:29:23.193 | INFO | MemInfo: { | 20260506 16:29:23.193 | INFO | "gpu" : " 17 GB", | 20260506 16:29:23.193 | INFO | "huge" : "", | 20260506 16:29:23.193 | INFO | "huge_shared" : " 27 GB", | 20260506 16:29:23.193 | INFO | "malloc" : " 14 GB", | 20260506 16:29:23.193 | INFO | "pinned" : " 29 GB" | 20260506 16:29:23.193 | INFO | } | 20260506 16:29:23.193 | INFO | Perf report for the last 5.14385 seconds: | 20260506 16:29:23.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:29:23.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:23.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:23.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:23.193 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:29:23.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:23.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:23.193 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260506 16:29:23.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:23.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:23.193 | INFO | Uploaded 36GiB to GPU in 2.0252s (17.776GiB/s) | 20260506 16:29:23.193 | INFO | Downloaded 34GiB to GPU in 1.92594s (17.6537GiB/s) | 20260506 16:29:28.335 | INFO | Analyzing chunk frames = [2796544,2797056) | 20260506 16:29:28.335 | INFO | MemInfo: { | 20260506 16:29:28.335 | INFO | "gpu" : " 17 GB", | 20260506 16:29:28.335 | INFO | "huge" : "", | 20260506 16:29:28.335 | INFO | "huge_shared" : " 27 GB", | 20260506 16:29:28.335 | INFO | "malloc" : " 14 GB", | 20260506 16:29:28.335 | INFO | "pinned" : " 29 GB" | 20260506 16:29:28.335 | INFO | } | 20260506 16:29:28.335 | INFO | Perf report for the last 5.1417 seconds: | 20260506 16:29:28.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:29:28.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:28.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:28.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:28.335 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 16:29:28.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:28.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:28.335 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:29:28.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:28.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:28.335 | INFO | Uploaded 36GiB to GPU in 1.99716s (18.0256GiB/s) | 20260506 16:29:28.335 | INFO | Downloaded 34GiB to GPU in 1.96889s (17.2686GiB/s) | 20260506 16:29:33.470 | INFO | Analyzing chunk frames = [2797056,2797568) | 20260506 16:29:33.470 | INFO | MemInfo: { | 20260506 16:29:33.470 | INFO | "gpu" : " 17 GB", | 20260506 16:29:33.470 | INFO | "huge" : "", | 20260506 16:29:33.470 | INFO | "huge_shared" : " 27 GB", | 20260506 16:29:33.470 | INFO | "malloc" : " 14 GB", | 20260506 16:29:33.470 | INFO | "pinned" : " 29 GB" | 20260506 16:29:33.470 | INFO | } | 20260506 16:29:33.470 | INFO | Perf report for the last 5.13536 seconds: | 20260506 16:29:33.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:29:33.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:33.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:33.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:33.470 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:29:33.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:33.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:33.470 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 16:29:33.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:33.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:33.470 | INFO | Uploaded 36GiB to GPU in 2.00877s (17.9214GiB/s) | 20260506 16:29:33.470 | INFO | Downloaded 34GiB to GPU in 1.99037s (17.0823GiB/s) | 20260506 16:29:33.590 | INFO | Starting a round of Baz IO | 20260506 16:29:36.727 | INFO | Populating RTMetrics report | 20260506 16:29:36.727 | INFO | RTMetrics report populated | 20260506 16:29:36.727 | INFO | Opening tmp RTMetrics json file | 20260506 16:29:36.755 | INFO | Serializing RTMetrics to json | 20260506 16:29:36.757 | INFO | Serialization done, swapping tmp file now... | 20260506 16:29:36.761 | INFO | RTMetrics done | 20260506 16:29:36.956 | INFO | Finished a round of Baz IO: Wrote 2.087 GiB in 3.365 seconds | 20260506 16:29:36.956 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.30% / 0.01% / 63.69% | 20260506 16:29:36.956 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.67176% / 7.94729e-06% / 6.75519e-05% | 20260506 16:29:36.956 | INFO | Baz IO overall: | 20260506 16:29:36.956 | INFO | BytesWritten : 1468876865637 | 20260506 16:29:36.956 | INFO | NumEvents : 710782445580 | 20260506 16:29:38.609 | INFO | Analyzing chunk frames = [2797568,2798080) | 20260506 16:29:38.609 | INFO | MemInfo: { | 20260506 16:29:38.609 | INFO | "gpu" : " 17 GB", | 20260506 16:29:38.609 | INFO | "huge" : "", | 20260506 16:29:38.609 | INFO | "huge_shared" : " 27 GB", | 20260506 16:29:38.609 | INFO | "malloc" : " 14 GB", | 20260506 16:29:38.609 | INFO | "pinned" : " 29 GB" | 20260506 16:29:38.609 | INFO | } | 20260506 16:29:38.609 | INFO | Perf report for the last 5.13871 seconds: | 20260506 16:29:38.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:29:38.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:38.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:38.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:38.609 | INFO | ANALYSIS : 62.8%, 2.9, 95.0ms, 0 | 20260506 16:29:38.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:38.609 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 16:29:38.609 | INFO | PRE_HQ : 60.5%, 1.9, 59.0ms, 0 | 20260506 16:29:38.609 | INFO | BAZWRITER : 65.0%, 4.8, 161.2ms, 0 | 20260506 16:29:38.609 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 16:29:38.609 | INFO | Uploaded 36GiB to GPU in 2.05085s (17.5537GiB/s) | 20260506 16:29:38.609 | INFO | Downloaded 38GiB to GPU in 2.21978s (17.1188GiB/s) | 20260506 16:29:43.745 | INFO | Analyzing chunk frames = [2798080,2798592) | 20260506 16:29:43.745 | INFO | MemInfo: { | 20260506 16:29:43.745 | INFO | "gpu" : " 17 GB", | 20260506 16:29:43.745 | INFO | "huge" : "", | 20260506 16:29:43.745 | INFO | "huge_shared" : " 27 GB", | 20260506 16:29:43.745 | INFO | "malloc" : " 14 GB", | 20260506 16:29:43.745 | INFO | "pinned" : " 29 GB" | 20260506 16:29:43.745 | INFO | } | 20260506 16:29:43.745 | INFO | Perf report for the last 5.13617 seconds: | 20260506 16:29:43.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:29:43.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:43.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:43.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:43.745 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:29:43.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:43.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:43.745 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:29:43.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:43.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:43.745 | INFO | Uploaded 36GiB to GPU in 1.97714s (18.2082GiB/s) | 20260506 16:29:43.745 | INFO | Downloaded 34GiB to GPU in 1.95408s (17.3995GiB/s) | 20260506 16:29:48.886 | INFO | Analyzing chunk frames = [2798592,2799104) | 20260506 16:29:48.886 | INFO | MemInfo: { | 20260506 16:29:48.886 | INFO | "gpu" : " 17 GB", | 20260506 16:29:48.886 | INFO | "huge" : "", | 20260506 16:29:48.886 | INFO | "huge_shared" : " 27 GB", | 20260506 16:29:48.886 | INFO | "malloc" : " 14 GB", | 20260506 16:29:48.886 | INFO | "pinned" : " 29 GB" | 20260506 16:29:48.886 | INFO | } | 20260506 16:29:48.886 | INFO | Perf report for the last 5.14094 seconds: | 20260506 16:29:48.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:29:48.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:48.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:48.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:48.886 | INFO | ANALYSIS : 60.3%, 3.0, 93.7ms, 0 | 20260506 16:29:48.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:48.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:48.886 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:29:48.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:48.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:48.886 | INFO | Uploaded 36GiB to GPU in 1.93311s (18.6228GiB/s) | 20260506 16:29:48.886 | INFO | Downloaded 34GiB to GPU in 1.89404s (17.9511GiB/s) | 20260506 16:29:54.029 | INFO | Analyzing chunk frames = [2799104,2799616) | 20260506 16:29:54.029 | INFO | MemInfo: { | 20260506 16:29:54.029 | INFO | "gpu" : " 17 GB", | 20260506 16:29:54.029 | INFO | "huge" : "", | 20260506 16:29:54.029 | INFO | "huge_shared" : " 27 GB", | 20260506 16:29:54.030 | INFO | "malloc" : " 14 GB", | 20260506 16:29:54.030 | INFO | "pinned" : " 29 GB" | 20260506 16:29:54.030 | INFO | } | 20260506 16:29:54.030 | INFO | Perf report for the last 5.14352 seconds: | 20260506 16:29:54.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:29:54.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:54.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:54.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:54.030 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 16:29:54.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:54.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:54.030 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:29:54.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:54.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:54.030 | INFO | Uploaded 36GiB to GPU in 1.93028s (18.6502GiB/s) | 20260506 16:29:54.030 | INFO | Downloaded 34GiB to GPU in 1.87959s (18.0891GiB/s) | 20260506 16:29:59.164 | INFO | Analyzing chunk frames = [2799616,2800128) | 20260506 16:29:59.164 | INFO | MemInfo: { | 20260506 16:29:59.164 | INFO | "gpu" : " 17 GB", | 20260506 16:29:59.164 | INFO | "huge" : "", | 20260506 16:29:59.164 | INFO | "huge_shared" : " 27 GB", | 20260506 16:29:59.164 | INFO | "malloc" : " 14 GB", | 20260506 16:29:59.164 | INFO | "pinned" : " 29 GB" | 20260506 16:29:59.164 | INFO | } | 20260506 16:29:59.164 | INFO | Perf report for the last 5.13481 seconds: | 20260506 16:29:59.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:29:59.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:59.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:59.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:59.164 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 16:29:59.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:59.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:59.165 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:29:59.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:59.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:29:59.165 | INFO | Uploaded 36GiB to GPU in 2.00013s (17.9988GiB/s) | 20260506 16:29:59.165 | INFO | Downloaded 34GiB to GPU in 1.90966s (17.8042GiB/s) | 20260506 16:30:04.305 | INFO | Analyzing chunk frames = [2800128,2800640) | 20260506 16:30:04.305 | INFO | MemInfo: { | 20260506 16:30:04.305 | INFO | "gpu" : " 17 GB", | 20260506 16:30:04.305 | INFO | "huge" : "", | 20260506 16:30:04.305 | INFO | "huge_shared" : " 27 GB", | 20260506 16:30:04.305 | INFO | "malloc" : " 14 GB", | 20260506 16:30:04.305 | INFO | "pinned" : " 29 GB" | 20260506 16:30:04.305 | INFO | } | 20260506 16:30:04.305 | INFO | Perf report for the last 5.14099 seconds: | 20260506 16:30:04.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:30:04.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:04.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:04.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:04.305 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 16:30:04.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:04.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:04.305 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 16:30:04.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:04.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:04.306 | INFO | Uploaded 36GiB to GPU in 2.0073s (17.9345GiB/s) | 20260506 16:30:04.306 | INFO | Downloaded 34GiB to GPU in 1.94492s (17.4815GiB/s) | 20260506 16:30:09.447 | INFO | Analyzing chunk frames = [2800640,2801152) | 20260506 16:30:09.447 | INFO | MemInfo: { | 20260506 16:30:09.447 | INFO | "gpu" : " 17 GB", | 20260506 16:30:09.447 | INFO | "huge" : "", | 20260506 16:30:09.447 | INFO | "huge_shared" : " 27 GB", | 20260506 16:30:09.447 | INFO | "malloc" : " 14 GB", | 20260506 16:30:09.447 | INFO | "pinned" : " 29 GB" | 20260506 16:30:09.447 | INFO | } | 20260506 16:30:09.447 | INFO | Perf report for the last 5.14174 seconds: | 20260506 16:30:09.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:30:09.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:09.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:09.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:09.447 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 16:30:09.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:09.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:09.447 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 16:30:09.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:09.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:09.447 | INFO | Uploaded 36GiB to GPU in 2.0007s (17.9937GiB/s) | 20260506 16:30:09.448 | INFO | Downloaded 34GiB to GPU in 2.0133s (16.8877GiB/s) | 20260506 16:30:14.585 | INFO | Analyzing chunk frames = [2801152,2801664) | 20260506 16:30:14.585 | INFO | MemInfo: { | 20260506 16:30:14.585 | INFO | "gpu" : " 17 GB", | 20260506 16:30:14.585 | INFO | "huge" : "", | 20260506 16:30:14.585 | INFO | "huge_shared" : " 27 GB", | 20260506 16:30:14.585 | INFO | "malloc" : " 14 GB", | 20260506 16:30:14.585 | INFO | "pinned" : " 29 GB" | 20260506 16:30:14.585 | INFO | } | 20260506 16:30:14.585 | INFO | Perf report for the last 5.13793 seconds: | 20260506 16:30:14.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:30:14.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:14.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:14.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:14.585 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:30:14.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:14.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:14.585 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:30:14.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:14.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:14.585 | INFO | Uploaded 36GiB to GPU in 2.00539s (17.9516GiB/s) | 20260506 16:30:14.585 | INFO | Downloaded 34GiB to GPU in 1.97437s (17.2207GiB/s) | 20260506 16:30:14.586 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:30:14.586 | INFO | Total mem usage: 18 GiB | 20260506 16:30:14.586 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:30:14.586 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.3%) | 20260506 16:30:14.586 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:30:14.586 | INFO | High Water Breakdown: | 20260506 16:30:14.586 | INFO | 61.2% : 9.47 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:30:14.586 | INFO | 38.7% : 5.99 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:30:14.586 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:30:14.586 | INFO | ------------------------------------------ | 20260506 16:30:14.586 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:30:14.586 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:30:14.586 | INFO | Total mem usage: 26.5 GiB | 20260506 16:30:14.586 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:30:14.586 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:30:14.586 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:30:14.586 | INFO | High Water Breakdown: | 20260506 16:30:14.586 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:30:14.586 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:30:14.586 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:30:14.586 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:30:14.586 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:30:14.586 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:30:14.586 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:30:14.586 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:30:14.586 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:30:14.586 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:30:14.587 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:30:14.587 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:30:14.587 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:30:14.587 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:30:14.587 | INFO | ------------------------------------------ | 20260506 16:30:14.587 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:30:14.587 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:30:14.587 | INFO | Total mem usage: 25.6 GiB | 20260506 16:30:14.587 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:30:14.587 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:30:14.587 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:30:14.587 | INFO | High Water Breakdown: | 20260506 16:30:14.587 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:30:14.587 | INFO | ------------------------------------------ | 20260506 16:30:14.587 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:30:14.587 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:30:14.587 | INFO | Total mem usage: 15.7 GiB | 20260506 16:30:14.587 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:30:14.587 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:30:14.587 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 16:30:14.587 | INFO | High Water Breakdown: | 20260506 16:30:14.587 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:30:14.587 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:30:14.587 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:30:14.587 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:30:14.587 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:30:14.587 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:30:14.587 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:30:14.587 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:30:14.587 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:30:14.587 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:30:14.587 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:30:14.587 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:30:14.587 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:30:14.587 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:30:14.587 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:30:14.587 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:30:14.587 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:30:14.587 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:30:14.587 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:30:14.587 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:30:14.587 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:30:14.587 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:30:14.587 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:30:14.587 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:30:14.587 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:30:14.587 | INFO | ------------------------------------------ | 20260506 16:30:14.587 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:30:14.587 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:30:14.587 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:30:14.587 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:30:14.587 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.60e+01, 1.34e+01 / 2.56e+00 | 20260506 16:30:14.587 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 9.71e-03 / 4.21e-03 | 20260506 16:30:14.587 | INFO | PerfReport: 11.62%, Baseline -- 3.64e+00 / 4.42e+00, 5.61e-01 / 3.07e-01 | 20260506 16:30:14.587 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.49e+00, 1.86e-01 / 1.10e-01 | 20260506 16:30:14.587 | INFO | PerfReport: 15.94%, DME -- 4.00e+01 / 4.50e+01, 4.05e+00 / 2.64e+00 | 20260506 16:30:14.587 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.39e+00 / 1.94e+00 | 20260506 16:30:14.587 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.30e+00 / 2.87e+00, 3.82e-01 / 2.14e-01 | 20260506 16:30:14.587 | INFO | PerfReport: 3.50%, Metrics -- 1.10e+00 / 1.48e+00, 2.77e-01 / 1.22e-01 | 20260506 16:30:14.587 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.81e-01 / 4.38e-01, 5.92e-02 / 2.01e-02 | 20260506 16:30:14.587 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:30:14.587 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:30:14.587 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:30:14.587 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 16:30:14.587 | INFO | PerfReport: 21.39%, Upload -- 2.08e+01 / 0.00e+00, 6.89e+00 / 0.00e+00 | 20260506 16:30:14.587 | INFO | PerfReport: 21.11%, Download -- 2.05e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260506 16:30:14.587 | INFO | PerfReport: 32.25%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 16:30:14.587 | INFO | PerfReport: 1.42%, LockUpload -- 1.38e+00 / 0.00e+00, 5.85e+00 / 0.00e+00 | 20260506 16:30:14.587 | INFO | PerfReport: 1.13%, LockDownload -- 1.10e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260506 16:30:14.587 | INFO | PerfReport: 22.68%, LockCompute -- 2.21e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 16:30:14.587 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260506 16:30:14.705 | INFO | Starting a round of Baz IO | 20260506 16:30:17.843 | INFO | Populating RTMetrics report | 20260506 16:30:17.843 | INFO | RTMetrics report populated | 20260506 16:30:17.843 | INFO | Opening tmp RTMetrics json file | 20260506 16:30:17.857 | INFO | Serializing RTMetrics to json | 20260506 16:30:17.862 | INFO | Serialization done, swapping tmp file now... | 20260506 16:30:17.862 | INFO | RTMetrics done | 20260506 16:30:18.270 | INFO | Finished a round of Baz IO: Wrote 2.087 GiB in 3.565 seconds | 20260506 16:30:18.270 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.30% / 0.01% / 63.70% | 20260506 16:30:18.270 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.66849% / 1.19209e-05% / 5.96046e-05% | 20260506 16:30:18.270 | INFO | Baz IO overall: | 20260506 16:30:18.270 | INFO | BytesWritten : 1471117742055 | 20260506 16:30:18.270 | INFO | NumEvents : 711865535814 | 20260506 16:30:19.723 | INFO | Analyzing chunk frames = [2801664,2802176) | 20260506 16:30:19.723 | INFO | MemInfo: { | 20260506 16:30:19.723 | INFO | "gpu" : " 17 GB", | 20260506 16:30:19.723 | INFO | "huge" : "", | 20260506 16:30:19.723 | INFO | "huge_shared" : " 27 GB", | 20260506 16:30:19.723 | INFO | "malloc" : " 14 GB", | 20260506 16:30:19.723 | INFO | "pinned" : " 29 GB" | 20260506 16:30:19.723 | INFO | } | 20260506 16:30:19.723 | INFO | Perf report for the last 5.13794 seconds: | 20260506 16:30:19.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:30:19.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:19.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:19.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:19.723 | INFO | ANALYSIS : 62.8%, 2.9, 95.5ms, 0 | 20260506 16:30:19.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:19.723 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 16:30:19.723 | INFO | PRE_HQ : 60.3%, 1.9, 58.3ms, 0 | 20260506 16:30:19.723 | INFO | BAZWRITER : 69.0%, 4.8, 170.7ms, 0 | 20260506 16:30:19.723 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 16:30:19.723 | INFO | Uploaded 36GiB to GPU in 2.06345s (17.4465GiB/s) | 20260506 16:30:19.723 | INFO | Downloaded 38GiB to GPU in 2.21683s (17.1416GiB/s) | 20260506 16:30:24.859 | INFO | Analyzing chunk frames = [2802176,2802688) | 20260506 16:30:24.859 | INFO | MemInfo: { | 20260506 16:30:24.859 | INFO | "gpu" : " 17 GB", | 20260506 16:30:24.859 | INFO | "huge" : "", | 20260506 16:30:24.859 | INFO | "huge_shared" : " 27 GB", | 20260506 16:30:24.859 | INFO | "malloc" : " 14 GB", | 20260506 16:30:24.859 | INFO | "pinned" : " 29 GB" | 20260506 16:30:24.859 | INFO | } | 20260506 16:30:24.859 | INFO | Perf report for the last 5.13639 seconds: | 20260506 16:30:24.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:30:24.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:24.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:24.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:24.859 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:30:24.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:24.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:24.859 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:30:24.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:24.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:24.860 | INFO | Uploaded 36GiB to GPU in 2.01512s (17.8649GiB/s) | 20260506 16:30:24.860 | INFO | Downloaded 34GiB to GPU in 1.94703s (17.4625GiB/s) | 20260506 16:30:30.001 | INFO | Analyzing chunk frames = [2802688,2803200) | 20260506 16:30:30.001 | INFO | MemInfo: { | 20260506 16:30:30.001 | INFO | "gpu" : " 17 GB", | 20260506 16:30:30.001 | INFO | "huge" : "", | 20260506 16:30:30.001 | INFO | "huge_shared" : " 27 GB", | 20260506 16:30:30.001 | INFO | "malloc" : " 14 GB", | 20260506 16:30:30.001 | INFO | "pinned" : " 29 GB" | 20260506 16:30:30.001 | INFO | } | 20260506 16:30:30.001 | INFO | Perf report for the last 5.14161 seconds: | 20260506 16:30:30.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:30:30.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:30.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:30.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:30.001 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 16:30:30.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:30.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:30.001 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:30:30.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:30.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:30.001 | INFO | Uploaded 36GiB to GPU in 1.99168s (18.0752GiB/s) | 20260506 16:30:30.001 | INFO | Downloaded 34GiB to GPU in 1.92936s (17.6225GiB/s) | 20260506 16:30:35.142 | INFO | Analyzing chunk frames = [2803200,2803712) | 20260506 16:30:35.142 | INFO | MemInfo: { | 20260506 16:30:35.142 | INFO | "gpu" : " 17 GB", | 20260506 16:30:35.142 | INFO | "huge" : "", | 20260506 16:30:35.142 | INFO | "huge_shared" : " 27 GB", | 20260506 16:30:35.142 | INFO | "malloc" : " 14 GB", | 20260506 16:30:35.142 | INFO | "pinned" : " 29 GB" | 20260506 16:30:35.142 | INFO | } | 20260506 16:30:35.142 | INFO | Perf report for the last 5.1411 seconds: | 20260506 16:30:35.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:30:35.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:35.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:35.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:35.142 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 16:30:35.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:35.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:35.142 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:30:35.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:35.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:35.142 | INFO | Uploaded 36GiB to GPU in 1.9303s (18.6499GiB/s) | 20260506 16:30:35.143 | INFO | Downloaded 34GiB to GPU in 1.8779s (18.1054GiB/s) | 20260506 16:30:40.284 | INFO | Analyzing chunk frames = [2803712,2804224) | 20260506 16:30:40.284 | INFO | MemInfo: { | 20260506 16:30:40.284 | INFO | "gpu" : " 17 GB", | 20260506 16:30:40.284 | INFO | "huge" : "", | 20260506 16:30:40.284 | INFO | "huge_shared" : " 27 GB", | 20260506 16:30:40.284 | INFO | "malloc" : " 14 GB", | 20260506 16:30:40.284 | INFO | "pinned" : " 29 GB" | 20260506 16:30:40.284 | INFO | } | 20260506 16:30:40.284 | INFO | Perf report for the last 5.14206 seconds: | 20260506 16:30:40.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:30:40.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:40.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:40.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:40.284 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 16:30:40.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:40.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:40.284 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:30:40.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:40.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:40.285 | INFO | Uploaded 36GiB to GPU in 2.01789s (17.8404GiB/s) | 20260506 16:30:40.285 | INFO | Downloaded 34GiB to GPU in 1.90582s (17.8401GiB/s) | 20260506 16:30:45.423 | INFO | Analyzing chunk frames = [2804224,2804736) | 20260506 16:30:45.423 | INFO | MemInfo: { | 20260506 16:30:45.423 | INFO | "gpu" : " 17 GB", | 20260506 16:30:45.424 | INFO | "huge" : "", | 20260506 16:30:45.424 | INFO | "huge_shared" : " 27 GB", | 20260506 16:30:45.424 | INFO | "malloc" : " 14 GB", | 20260506 16:30:45.424 | INFO | "pinned" : " 29 GB" | 20260506 16:30:45.424 | INFO | } | 20260506 16:30:45.424 | INFO | Perf report for the last 5.13948 seconds: | 20260506 16:30:45.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:30:45.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:45.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:45.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:45.424 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 16:30:45.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:45.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:45.424 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:30:45.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:45.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:45.424 | INFO | Uploaded 36GiB to GPU in 2.03542s (17.6868GiB/s) | 20260506 16:30:45.424 | INFO | Downloaded 34GiB to GPU in 1.94658s (17.4665GiB/s) | 20260506 16:30:50.559 | INFO | Analyzing chunk frames = [2804736,2805248) | 20260506 16:30:50.559 | INFO | MemInfo: { | 20260506 16:30:50.559 | INFO | "gpu" : " 17 GB", | 20260506 16:30:50.559 | INFO | "huge" : "", | 20260506 16:30:50.559 | INFO | "huge_shared" : " 27 GB", | 20260506 16:30:50.559 | INFO | "malloc" : " 14 GB", | 20260506 16:30:50.559 | INFO | "pinned" : " 29 GB" | 20260506 16:30:50.559 | INFO | } | 20260506 16:30:50.559 | INFO | Perf report for the last 5.13542 seconds: | 20260506 16:30:50.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:30:50.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:50.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:50.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:50.559 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 16:30:50.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:50.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:50.559 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:30:50.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:50.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:50.560 | INFO | Uploaded 36GiB to GPU in 1.98355s (18.1493GiB/s) | 20260506 16:30:50.560 | INFO | Downloaded 34GiB to GPU in 1.97716s (17.1964GiB/s) | 20260506 16:30:55.701 | INFO | Analyzing chunk frames = [2805248,2805760) | 20260506 16:30:55.701 | INFO | MemInfo: { | 20260506 16:30:55.701 | INFO | "gpu" : " 17 GB", | 20260506 16:30:55.701 | INFO | "huge" : "", | 20260506 16:30:55.701 | INFO | "huge_shared" : " 27 GB", | 20260506 16:30:55.701 | INFO | "malloc" : " 14 GB", | 20260506 16:30:55.701 | INFO | "pinned" : " 29 GB" | 20260506 16:30:55.701 | INFO | } | 20260506 16:30:55.701 | INFO | Perf report for the last 5.14202 seconds: | 20260506 16:30:55.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:30:55.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:55.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:55.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:55.701 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 16:30:55.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:55.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:55.701 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:30:55.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:55.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:30:55.701 | INFO | Uploaded 36GiB to GPU in 1.99012s (18.0893GiB/s) | 20260506 16:30:55.702 | INFO | Downloaded 34GiB to GPU in 1.97418s (17.2224GiB/s) | 20260506 16:30:55.816 | INFO | Starting a round of Baz IO | 20260506 16:30:58.956 | INFO | Populating RTMetrics report | 20260506 16:30:58.956 | INFO | RTMetrics report populated | 20260506 16:30:58.956 | INFO | Opening tmp RTMetrics json file | 20260506 16:30:58.971 | INFO | Serializing RTMetrics to json | 20260506 16:30:58.972 | INFO | Serialization done, swapping tmp file now... | 20260506 16:30:58.973 | INFO | RTMetrics done | 20260506 16:30:59.222 | INFO | Finished a round of Baz IO: Wrote 2.084 GiB in 3.406 seconds | 20260506 16:30:59.222 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.29% / 0.01% / 63.70% | 20260506 16:30:59.222 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.65952% / 0% / 6.35783e-05% | 20260506 16:30:59.222 | INFO | Baz IO overall: | 20260506 16:30:59.222 | INFO | BytesWritten : 1473355708562 | 20260506 16:30:59.222 | INFO | NumEvents : 712946499529 | 20260506 16:31:00.833 | INFO | Analyzing chunk frames = [2805760,2806272) | 20260506 16:31:00.834 | INFO | MemInfo: { | 20260506 16:31:00.834 | INFO | "gpu" : " 17 GB", | 20260506 16:31:00.834 | INFO | "huge" : "", | 20260506 16:31:00.834 | INFO | "huge_shared" : " 27 GB", | 20260506 16:31:00.834 | INFO | "malloc" : " 14 GB", | 20260506 16:31:00.834 | INFO | "pinned" : " 29 GB" | 20260506 16:31:00.834 | INFO | } | 20260506 16:31:00.834 | INFO | Perf report for the last 5.13252 seconds: | 20260506 16:31:00.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:31:00.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:00.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:00.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:00.834 | INFO | ANALYSIS : 62.8%, 2.9, 95.2ms, 0 | 20260506 16:31:00.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:00.834 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260506 16:31:00.834 | INFO | PRE_HQ : 60.6%, 1.9, 58.7ms, 0 | 20260506 16:31:00.834 | INFO | BAZWRITER : 66.0%, 4.8, 162.1ms, 0 | 20260506 16:31:00.834 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 16:31:00.834 | INFO | Uploaded 36GiB to GPU in 2.06061s (17.4705GiB/s) | 20260506 16:31:00.834 | INFO | Downloaded 38GiB to GPU in 2.20536s (17.2307GiB/s) | 20260506 16:31:05.972 | INFO | Analyzing chunk frames = [2806272,2806784) | 20260506 16:31:05.972 | INFO | MemInfo: { | 20260506 16:31:05.972 | INFO | "gpu" : " 17 GB", | 20260506 16:31:05.972 | INFO | "huge" : "", | 20260506 16:31:05.972 | INFO | "huge_shared" : " 27 GB", | 20260506 16:31:05.972 | INFO | "malloc" : " 14 GB", | 20260506 16:31:05.972 | INFO | "pinned" : " 29 GB" | 20260506 16:31:05.972 | INFO | } | 20260506 16:31:05.972 | INFO | Perf report for the last 5.13829 seconds: | 20260506 16:31:05.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:31:05.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:05.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:05.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:05.972 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:31:05.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:05.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:05.972 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:31:05.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:05.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:05.972 | INFO | Uploaded 36GiB to GPU in 2.02109s (17.8121GiB/s) | 20260506 16:31:05.972 | INFO | Downloaded 34GiB to GPU in 1.98798s (17.1028GiB/s) | 20260506 16:31:11.120 | INFO | Analyzing chunk frames = [2806784,2807296) | 20260506 16:31:11.120 | INFO | MemInfo: { | 20260506 16:31:11.120 | INFO | "gpu" : " 17 GB", | 20260506 16:31:11.120 | INFO | "huge" : "", | 20260506 16:31:11.120 | INFO | "huge_shared" : " 27 GB", | 20260506 16:31:11.120 | INFO | "malloc" : " 14 GB", | 20260506 16:31:11.120 | INFO | "pinned" : " 29 GB" | 20260506 16:31:11.121 | INFO | } | 20260506 16:31:11.121 | INFO | Perf report for the last 5.14848 seconds: | 20260506 16:31:11.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:31:11.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:11.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:11.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:11.121 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 16:31:11.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:11.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:11.121 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:31:11.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:11.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:11.121 | INFO | Uploaded 36GiB to GPU in 1.99952s (18.0043GiB/s) | 20260506 16:31:11.121 | INFO | Downloaded 34GiB to GPU in 1.9319s (17.5992GiB/s) | 20260506 16:31:16.258 | INFO | Analyzing chunk frames = [2807296,2807808) | 20260506 16:31:16.258 | INFO | MemInfo: { | 20260506 16:31:16.258 | INFO | "gpu" : " 17 GB", | 20260506 16:31:16.258 | INFO | "huge" : "", | 20260506 16:31:16.258 | INFO | "huge_shared" : " 27 GB", | 20260506 16:31:16.258 | INFO | "malloc" : " 14 GB", | 20260506 16:31:16.258 | INFO | "pinned" : " 29 GB" | 20260506 16:31:16.258 | INFO | } | 20260506 16:31:16.258 | INFO | Perf report for the last 5.13777 seconds: | 20260506 16:31:16.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:31:16.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:16.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:16.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:16.258 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 16:31:16.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:16.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:16.258 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:31:16.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:16.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:16.259 | INFO | Uploaded 36GiB to GPU in 1.94021s (18.5547GiB/s) | 20260506 16:31:16.259 | INFO | Downloaded 34GiB to GPU in 1.88062s (18.0792GiB/s) | 20260506 16:31:21.399 | INFO | Analyzing chunk frames = [2807808,2808320) | 20260506 16:31:21.399 | INFO | MemInfo: { | 20260506 16:31:21.399 | INFO | "gpu" : " 17 GB", | 20260506 16:31:21.399 | INFO | "huge" : "", | 20260506 16:31:21.399 | INFO | "huge_shared" : " 27 GB", | 20260506 16:31:21.399 | INFO | "malloc" : " 14 GB", | 20260506 16:31:21.399 | INFO | "pinned" : " 29 GB" | 20260506 16:31:21.399 | INFO | } | 20260506 16:31:21.399 | INFO | Perf report for the last 5.14063 seconds: | 20260506 16:31:21.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:31:21.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:21.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:21.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:21.399 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 16:31:21.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:21.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:21.399 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:31:21.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:21.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:21.399 | INFO | Uploaded 36GiB to GPU in 2.01918s (17.829GiB/s) | 20260506 16:31:21.399 | INFO | Downloaded 34GiB to GPU in 1.90816s (17.8182GiB/s) | 20260506 16:31:26.539 | INFO | Analyzing chunk frames = [2808320,2808832) | 20260506 16:31:26.539 | INFO | MemInfo: { | 20260506 16:31:26.539 | INFO | "gpu" : " 17 GB", | 20260506 16:31:26.539 | INFO | "huge" : "", | 20260506 16:31:26.539 | INFO | "huge_shared" : " 27 GB", | 20260506 16:31:26.539 | INFO | "malloc" : " 14 GB", | 20260506 16:31:26.539 | INFO | "pinned" : " 29 GB" | 20260506 16:31:26.539 | INFO | } | 20260506 16:31:26.539 | INFO | Perf report for the last 5.14022 seconds: | 20260506 16:31:26.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:31:26.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:26.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:26.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:26.539 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:31:26.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:26.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:26.539 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 16:31:26.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:26.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:26.539 | INFO | Uploaded 36GiB to GPU in 2.07231s (17.3719GiB/s) | 20260506 16:31:26.540 | INFO | Downloaded 34GiB to GPU in 1.96322s (17.3185GiB/s) | 20260506 16:31:31.673 | INFO | Analyzing chunk frames = [2808832,2809344) | 20260506 16:31:31.673 | INFO | MemInfo: { | 20260506 16:31:31.673 | INFO | "gpu" : " 17 GB", | 20260506 16:31:31.673 | INFO | "huge" : "", | 20260506 16:31:31.674 | INFO | "huge_shared" : " 27 GB", | 20260506 16:31:31.674 | INFO | "malloc" : " 14 GB", | 20260506 16:31:31.674 | INFO | "pinned" : " 29 GB" | 20260506 16:31:31.674 | INFO | } | 20260506 16:31:31.674 | INFO | Perf report for the last 5.13441 seconds: | 20260506 16:31:31.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:31:31.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:31.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:31.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:31.674 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 16:31:31.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:31.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:31.674 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:31:31.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:31.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:31.674 | INFO | Uploaded 36GiB to GPU in 2.00988s (17.9115GiB/s) | 20260506 16:31:31.674 | INFO | Downloaded 34GiB to GPU in 1.97485s (17.2165GiB/s) | 20260506 16:31:36.810 | INFO | Analyzing chunk frames = [2809344,2809856) | 20260506 16:31:36.810 | INFO | MemInfo: { | 20260506 16:31:36.810 | INFO | "gpu" : " 17 GB", | 20260506 16:31:36.810 | INFO | "huge" : "", | 20260506 16:31:36.810 | INFO | "huge_shared" : " 27 GB", | 20260506 16:31:36.810 | INFO | "malloc" : " 14 GB", | 20260506 16:31:36.810 | INFO | "pinned" : " 29 GB" | 20260506 16:31:36.810 | INFO | } | 20260506 16:31:36.810 | INFO | Perf report for the last 5.1365 seconds: | 20260506 16:31:36.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:31:36.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:36.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:36.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:36.810 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 16:31:36.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:36.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:36.810 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:31:36.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:36.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:36.810 | INFO | Uploaded 36GiB to GPU in 2.00775s (17.9305GiB/s) | 20260506 16:31:36.811 | INFO | Downloaded 34GiB to GPU in 1.96563s (17.2973GiB/s) | 20260506 16:31:36.811 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:31:36.811 | INFO | Total mem usage: 18 GiB | 20260506 16:31:36.811 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:31:36.811 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -9.97%) | 20260506 16:31:36.811 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:31:36.811 | INFO | High Water Breakdown: | 20260506 16:31:36.811 | INFO | 62% : 10.2 GiB ( 89% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:31:36.811 | INFO | 37.8% : 6.19 GiB ( 92.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:31:36.811 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:31:36.811 | INFO | ------------------------------------------ | 20260506 16:31:36.811 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:31:36.811 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:31:36.811 | INFO | Total mem usage: 26.5 GiB | 20260506 16:31:36.811 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:31:36.811 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:31:36.811 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:31:36.811 | INFO | High Water Breakdown: | 20260506 16:31:36.811 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:31:36.811 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:31:36.811 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:31:36.811 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:31:36.811 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:31:36.811 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:31:36.811 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:31:36.811 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:31:36.811 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:31:36.811 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:31:36.811 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:31:36.811 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:31:36.811 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:31:36.811 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:31:36.811 | INFO | ------------------------------------------ | 20260506 16:31:36.811 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:31:36.811 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:31:36.811 | INFO | Total mem usage: 25.6 GiB | 20260506 16:31:36.811 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:31:36.811 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:31:36.811 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:31:36.811 | INFO | High Water Breakdown: | 20260506 16:31:36.811 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:31:36.811 | INFO | ------------------------------------------ | 20260506 16:31:36.811 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:31:36.811 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:31:36.811 | INFO | Total mem usage: 15.7 GiB | 20260506 16:31:36.811 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:31:36.811 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:31:36.811 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 16:31:36.811 | INFO | High Water Breakdown: | 20260506 16:31:36.811 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:31:36.811 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:31:36.811 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:31:36.811 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:31:36.811 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:31:36.811 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:31:36.811 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:31:36.811 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:31:36.811 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:31:36.811 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:31:36.811 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:31:36.811 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:31:36.811 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:31:36.811 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:31:36.811 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:31:36.811 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:31:36.811 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:31:36.811 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:31:36.811 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:31:36.811 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:31:36.811 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:31:36.811 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:31:36.811 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:31:36.811 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:31:36.812 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:31:36.812 | INFO | ------------------------------------------ | 20260506 16:31:36.812 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:31:36.812 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:31:36.812 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:31:36.812 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:31:36.812 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.60e+01, 1.34e+01 / 1.91e+00 | 20260506 16:31:36.812 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 1.10e-02 / 5.83e-03 | 20260506 16:31:36.812 | INFO | PerfReport: 11.57%, Baseline -- 3.63e+00 / 4.47e+00, 5.59e-01 / 2.93e-01 | 20260506 16:31:36.812 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.50e+00, 1.89e-01 / 1.15e-01 | 20260506 16:31:36.812 | INFO | PerfReport: 15.92%, DME -- 4.00e+01 / 4.46e+01, 3.86e+00 / 1.85e+00 | 20260506 16:31:36.812 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.45e+00 / 1.68e+00 | 20260506 16:31:36.812 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.29e+00 / 2.92e+00, 3.85e-01 / 1.88e-01 | 20260506 16:31:36.812 | INFO | PerfReport: 3.50%, Metrics -- 1.10e+00 / 1.45e+00, 2.87e-01 / 1.23e-01 | 20260506 16:31:36.812 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.80e-01 / 4.33e-01, 5.96e-02 / 2.38e-02 | 20260506 16:31:36.812 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:31:36.812 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:31:36.812 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:31:36.812 | INFO | PerfReport: Full Execution -- 9.75e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 16:31:36.812 | INFO | PerfReport: 21.48%, Upload -- 2.09e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 16:31:36.812 | INFO | PerfReport: 21.10%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 16:31:36.812 | INFO | PerfReport: 32.22%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 16:31:36.812 | INFO | PerfReport: 1.53%, LockUpload -- 1.49e+00 / 0.00e+00, 5.91e+00 / 0.00e+00 | 20260506 16:31:36.812 | INFO | PerfReport: 1.02%, LockDownload -- 9.97e-01 / 0.00e+00, 2.60e+00 / 0.00e+00 | 20260506 16:31:36.812 | INFO | PerfReport: 22.63%, LockCompute -- 2.21e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 16:31:36.812 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.83e-02 / 0.00e+00 | 20260506 16:31:36.930 | INFO | Starting a round of Baz IO | 20260506 16:31:40.076 | INFO | Populating RTMetrics report | 20260506 16:31:40.076 | INFO | RTMetrics report populated | 20260506 16:31:40.076 | INFO | Opening tmp RTMetrics json file | 20260506 16:31:40.147 | INFO | Serializing RTMetrics to json | 20260506 16:31:40.150 | INFO | Serialization done, swapping tmp file now... | 20260506 16:31:40.168 | INFO | RTMetrics done | 20260506 16:31:40.339 | INFO | Finished a round of Baz IO: Wrote 2.085 GiB in 3.409 seconds | 20260506 16:31:40.339 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.28% / 0.01% / 63.71% | 20260506 16:31:40.339 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.6556% / 1.19209e-05% / 5.96046e-05% | 20260506 16:31:40.339 | INFO | Baz IO overall: | 20260506 16:31:40.339 | INFO | BytesWritten : 1475594193174 | 20260506 16:31:40.339 | INFO | NumEvents : 714027963075 | 20260506 16:31:41.947 | INFO | Analyzing chunk frames = [2809856,2810368) | 20260506 16:31:41.947 | INFO | MemInfo: { | 20260506 16:31:41.947 | INFO | "gpu" : " 17 GB", | 20260506 16:31:41.947 | INFO | "huge" : "", | 20260506 16:31:41.947 | INFO | "huge_shared" : " 27 GB", | 20260506 16:31:41.947 | INFO | "malloc" : " 14 GB", | 20260506 16:31:41.947 | INFO | "pinned" : " 29 GB" | 20260506 16:31:41.947 | INFO | } | 20260506 16:31:41.947 | INFO | Perf report for the last 5.13707 seconds: | 20260506 16:31:41.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:31:41.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:41.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:41.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:41.947 | INFO | ANALYSIS : 63.0%, 2.9, 95.5ms, 0 | 20260506 16:31:41.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:41.947 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260506 16:31:41.947 | INFO | PRE_HQ : 60.7%, 1.9, 59.0ms, 0 | 20260506 16:31:41.947 | INFO | BAZWRITER : 65.9%, 4.8, 162.8ms, 0 | 20260506 16:31:41.947 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 16:31:41.948 | INFO | Uploaded 36GiB to GPU in 2.04796s (17.5785GiB/s) | 20260506 16:31:41.948 | INFO | Downloaded 38GiB to GPU in 2.22186s (17.1028GiB/s) | 20260506 16:31:47.093 | INFO | Analyzing chunk frames = [2810368,2810880) | 20260506 16:31:47.093 | INFO | MemInfo: { | 20260506 16:31:47.093 | INFO | "gpu" : " 17 GB", | 20260506 16:31:47.093 | INFO | "huge" : "", | 20260506 16:31:47.093 | INFO | "huge_shared" : " 27 GB", | 20260506 16:31:47.093 | INFO | "malloc" : " 14 GB", | 20260506 16:31:47.093 | INFO | "pinned" : " 29 GB" | 20260506 16:31:47.093 | INFO | } | 20260506 16:31:47.094 | INFO | Perf report for the last 5.14631 seconds: | 20260506 16:31:47.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:31:47.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:47.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:47.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:47.094 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 16:31:47.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:47.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:47.094 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 16:31:47.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:47.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:47.094 | INFO | Uploaded 36GiB to GPU in 2.0226s (17.7989GiB/s) | 20260506 16:31:47.094 | INFO | Downloaded 34GiB to GPU in 1.97927s (17.178GiB/s) | 20260506 16:31:52.233 | INFO | Analyzing chunk frames = [2810880,2811392) | 20260506 16:31:52.233 | INFO | MemInfo: { | 20260506 16:31:52.233 | INFO | "gpu" : " 17 GB", | 20260506 16:31:52.233 | INFO | "huge" : "", | 20260506 16:31:52.233 | INFO | "huge_shared" : " 27 GB", | 20260506 16:31:52.233 | INFO | "malloc" : " 14 GB", | 20260506 16:31:52.233 | INFO | "pinned" : " 29 GB" | 20260506 16:31:52.233 | INFO | } | 20260506 16:31:52.234 | INFO | Perf report for the last 5.13996 seconds: | 20260506 16:31:52.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:31:52.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:52.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:52.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:52.234 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 16:31:52.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:52.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:52.234 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:31:52.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:52.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:52.234 | INFO | Uploaded 36GiB to GPU in 1.9917s (18.075GiB/s) | 20260506 16:31:52.234 | INFO | Downloaded 34GiB to GPU in 1.92983s (17.6181GiB/s) | 20260506 16:31:57.376 | INFO | Analyzing chunk frames = [2811392,2811904) | 20260506 16:31:57.376 | INFO | MemInfo: { | 20260506 16:31:57.376 | INFO | "gpu" : " 17 GB", | 20260506 16:31:57.376 | INFO | "huge" : "", | 20260506 16:31:57.376 | INFO | "huge_shared" : " 27 GB", | 20260506 16:31:57.376 | INFO | "malloc" : " 14 GB", | 20260506 16:31:57.376 | INFO | "pinned" : " 29 GB" | 20260506 16:31:57.376 | INFO | } | 20260506 16:31:57.376 | INFO | Perf report for the last 5.14272 seconds: | 20260506 16:31:57.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:31:57.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:57.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:57.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:57.376 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 16:31:57.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:57.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:57.377 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:31:57.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:57.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:31:57.377 | INFO | Uploaded 36GiB to GPU in 1.94535s (18.5057GiB/s) | 20260506 16:31:57.377 | INFO | Downloaded 34GiB to GPU in 1.88219s (18.0641GiB/s) | 20260506 16:32:02.513 | INFO | Analyzing chunk frames = [2811904,2812416) | 20260506 16:32:02.513 | INFO | MemInfo: { | 20260506 16:32:02.513 | INFO | "gpu" : " 17 GB", | 20260506 16:32:02.513 | INFO | "huge" : "", | 20260506 16:32:02.513 | INFO | "huge_shared" : " 27 GB", | 20260506 16:32:02.513 | INFO | "malloc" : " 14 GB", | 20260506 16:32:02.513 | INFO | "pinned" : " 29 GB" | 20260506 16:32:02.513 | INFO | } | 20260506 16:32:02.513 | INFO | Perf report for the last 5.1368 seconds: | 20260506 16:32:02.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:32:02.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:02.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:02.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:02.513 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 16:32:02.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:02.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:02.513 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:32:02.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:02.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:02.513 | INFO | Uploaded 36GiB to GPU in 2.01s (17.9105GiB/s) | 20260506 16:32:02.513 | INFO | Downloaded 34GiB to GPU in 1.91023s (17.7989GiB/s) | 20260506 16:32:07.652 | INFO | Analyzing chunk frames = [2812416,2812928) | 20260506 16:32:07.652 | INFO | MemInfo: { | 20260506 16:32:07.652 | INFO | "gpu" : " 17 GB", | 20260506 16:32:07.652 | INFO | "huge" : "", | 20260506 16:32:07.652 | INFO | "huge_shared" : " 27 GB", | 20260506 16:32:07.652 | INFO | "malloc" : " 14 GB", | 20260506 16:32:07.652 | INFO | "pinned" : " 29 GB" | 20260506 16:32:07.652 | INFO | } | 20260506 16:32:07.652 | INFO | Perf report for the last 5.13916 seconds: | 20260506 16:32:07.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:32:07.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:07.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:07.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:07.652 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 16:32:07.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:07.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:07.652 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:32:07.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:07.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:07.652 | INFO | Uploaded 36GiB to GPU in 2.03905s (17.6553GiB/s) | 20260506 16:32:07.653 | INFO | Downloaded 34GiB to GPU in 1.95038s (17.4325GiB/s) | 20260506 16:32:12.790 | INFO | Analyzing chunk frames = [2812928,2813440) | 20260506 16:32:12.790 | INFO | MemInfo: { | 20260506 16:32:12.790 | INFO | "gpu" : " 17 GB", | 20260506 16:32:12.790 | INFO | "huge" : "", | 20260506 16:32:12.790 | INFO | "huge_shared" : " 27 GB", | 20260506 16:32:12.791 | INFO | "malloc" : " 14 GB", | 20260506 16:32:12.791 | INFO | "pinned" : " 29 GB" | 20260506 16:32:12.791 | INFO | } | 20260506 16:32:12.791 | INFO | Perf report for the last 5.13843 seconds: | 20260506 16:32:12.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:32:12.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:12.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:12.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:12.791 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 16:32:12.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:12.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:12.791 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 16:32:12.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:12.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:12.791 | INFO | Uploaded 36GiB to GPU in 2.01633s (17.8543GiB/s) | 20260506 16:32:12.791 | INFO | Downloaded 34GiB to GPU in 1.98912s (17.093GiB/s) | 20260506 16:32:17.926 | INFO | Analyzing chunk frames = [2813440,2813952) | 20260506 16:32:17.926 | INFO | MemInfo: { | 20260506 16:32:17.926 | INFO | "gpu" : " 17 GB", | 20260506 16:32:17.926 | INFO | "huge" : "", | 20260506 16:32:17.926 | INFO | "huge_shared" : " 27 GB", | 20260506 16:32:17.926 | INFO | "malloc" : " 14 GB", | 20260506 16:32:17.926 | INFO | "pinned" : " 29 GB" | 20260506 16:32:17.926 | INFO | } | 20260506 16:32:17.926 | INFO | Perf report for the last 5.13539 seconds: | 20260506 16:32:17.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:32:17.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:17.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:17.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:17.926 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:32:17.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:17.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:17.926 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 16:32:17.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:17.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:17.926 | INFO | Uploaded 36GiB to GPU in 2.00099s (17.9911GiB/s) | 20260506 16:32:17.926 | INFO | Downloaded 34GiB to GPU in 1.97768s (17.1919GiB/s) | 20260506 16:32:18.059 | INFO | Starting a round of Baz IO | 20260506 16:32:21.227 | INFO | Populating RTMetrics report | 20260506 16:32:21.227 | INFO | RTMetrics report populated | 20260506 16:32:21.227 | INFO | Opening tmp RTMetrics json file | 20260506 16:32:21.228 | INFO | Serializing RTMetrics to json | 20260506 16:32:21.241 | INFO | Serialization done, swapping tmp file now... | 20260506 16:32:21.256 | INFO | RTMetrics done | 20260506 16:32:21.470 | INFO | Finished a round of Baz IO: Wrote 2.086 GiB in 3.411 seconds | 20260506 16:32:21.470 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.27% / 0.01% / 63.72% | 20260506 16:32:21.470 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.6566% / 3.17891e-05% / 8.74201e-05% | 20260506 16:32:21.470 | INFO | Baz IO overall: | 20260506 16:32:21.470 | INFO | BytesWritten : 1477834302423 | 20260506 16:32:21.470 | INFO | NumEvents : 715110472147 | 20260506 16:32:23.065 | INFO | Analyzing chunk frames = [2813952,2814464) | 20260506 16:32:23.065 | INFO | MemInfo: { | 20260506 16:32:23.065 | INFO | "gpu" : " 17 GB", | 20260506 16:32:23.065 | INFO | "huge" : "", | 20260506 16:32:23.065 | INFO | "huge_shared" : " 27 GB", | 20260506 16:32:23.065 | INFO | "malloc" : " 14 GB", | 20260506 16:32:23.065 | INFO | "pinned" : " 29 GB" | 20260506 16:32:23.065 | INFO | } | 20260506 16:32:23.065 | INFO | Perf report for the last 5.13898 seconds: | 20260506 16:32:23.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:32:23.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:23.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:23.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:23.065 | INFO | ANALYSIS : 63.3%, 2.8, 94.4ms, 0 | 20260506 16:32:23.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:23.065 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260506 16:32:23.065 | INFO | PRE_HQ : 61.3%, 2.0, 62.7ms, 0 | 20260506 16:32:23.065 | INFO | BAZWRITER : 65.9%, 4.8, 162.8ms, 0 | 20260506 16:32:23.065 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 16:32:23.065 | INFO | Uploaded 36GiB to GPU in 2.0578s (17.4944GiB/s) | 20260506 16:32:23.065 | INFO | Downloaded 38GiB to GPU in 2.1929s (17.3287GiB/s) | 20260506 16:32:28.211 | INFO | Analyzing chunk frames = [2814464,2814976) | 20260506 16:32:28.211 | INFO | MemInfo: { | 20260506 16:32:28.211 | INFO | "gpu" : " 17 GB", | 20260506 16:32:28.211 | INFO | "huge" : "", | 20260506 16:32:28.211 | INFO | "huge_shared" : " 27 GB", | 20260506 16:32:28.211 | INFO | "malloc" : " 14 GB", | 20260506 16:32:28.211 | INFO | "pinned" : " 29 GB" | 20260506 16:32:28.211 | INFO | } | 20260506 16:32:28.211 | INFO | Perf report for the last 5.14598 seconds: | 20260506 16:32:28.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:32:28.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:28.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:28.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:28.211 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 16:32:28.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:28.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:28.211 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 16:32:28.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:28.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:28.211 | INFO | Uploaded 36GiB to GPU in 1.98902s (18.0994GiB/s) | 20260506 16:32:28.211 | INFO | Downloaded 34GiB to GPU in 1.94315s (17.4973GiB/s) | 20260506 16:32:33.347 | INFO | Analyzing chunk frames = [2814976,2815488) | 20260506 16:32:33.347 | INFO | MemInfo: { | 20260506 16:32:33.347 | INFO | "gpu" : " 17 GB", | 20260506 16:32:33.347 | INFO | "huge" : "", | 20260506 16:32:33.348 | INFO | "huge_shared" : " 27 GB", | 20260506 16:32:33.348 | INFO | "malloc" : " 14 GB", | 20260506 16:32:33.348 | INFO | "pinned" : " 29 GB" | 20260506 16:32:33.348 | INFO | } | 20260506 16:32:33.348 | INFO | Perf report for the last 5.13655 seconds: | 20260506 16:32:33.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:32:33.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:33.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:33.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:33.348 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 16:32:33.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:33.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:33.348 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 16:32:33.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:33.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:33.348 | INFO | Uploaded 36GiB to GPU in 1.97593s (18.2193GiB/s) | 20260506 16:32:33.348 | INFO | Downloaded 34GiB to GPU in 1.92581s (17.6549GiB/s) | 20260506 16:32:38.488 | INFO | Analyzing chunk frames = [2815488,2816000) | 20260506 16:32:38.488 | INFO | MemInfo: { | 20260506 16:32:38.488 | INFO | "gpu" : " 17 GB", | 20260506 16:32:38.488 | INFO | "huge" : "", | 20260506 16:32:38.488 | INFO | "huge_shared" : " 27 GB", | 20260506 16:32:38.488 | INFO | "malloc" : " 14 GB", | 20260506 16:32:38.488 | INFO | "pinned" : " 29 GB" | 20260506 16:32:38.488 | INFO | } | 20260506 16:32:38.488 | INFO | Perf report for the last 5.14056 seconds: | 20260506 16:32:38.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:32:38.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:38.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:38.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:38.488 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 16:32:38.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:38.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:38.488 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:32:38.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:38.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:38.488 | INFO | Uploaded 36GiB to GPU in 1.93171s (18.6363GiB/s) | 20260506 16:32:38.489 | INFO | Downloaded 34GiB to GPU in 1.87187s (18.1636GiB/s) | 20260506 16:32:43.625 | INFO | Analyzing chunk frames = [2816000,2816512) | 20260506 16:32:43.625 | INFO | MemInfo: { | 20260506 16:32:43.625 | INFO | "gpu" : " 17 GB", | 20260506 16:32:43.625 | INFO | "huge" : "", | 20260506 16:32:43.625 | INFO | "huge_shared" : " 27 GB", | 20260506 16:32:43.625 | INFO | "malloc" : " 14 GB", | 20260506 16:32:43.625 | INFO | "pinned" : " 29 GB" | 20260506 16:32:43.625 | INFO | } | 20260506 16:32:43.625 | INFO | Perf report for the last 5.13714 seconds: | 20260506 16:32:43.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:32:43.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:43.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:43.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:43.625 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 16:32:43.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:43.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:43.625 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:32:43.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:43.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:43.626 | INFO | Uploaded 36GiB to GPU in 2.01991s (17.8226GiB/s) | 20260506 16:32:43.626 | INFO | Downloaded 34GiB to GPU in 1.90553s (17.8428GiB/s) | 20260506 16:32:48.763 | INFO | Analyzing chunk frames = [2816512,2817024) | 20260506 16:32:48.763 | INFO | MemInfo: { | 20260506 16:32:48.763 | INFO | "gpu" : " 17 GB", | 20260506 16:32:48.763 | INFO | "huge" : "", | 20260506 16:32:48.763 | INFO | "huge_shared" : " 27 GB", | 20260506 16:32:48.763 | INFO | "malloc" : " 14 GB", | 20260506 16:32:48.763 | INFO | "pinned" : " 29 GB" | 20260506 16:32:48.763 | INFO | } | 20260506 16:32:48.763 | INFO | Perf report for the last 5.13743 seconds: | 20260506 16:32:48.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:32:48.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:48.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:48.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:48.763 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:32:48.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:48.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:48.763 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:32:48.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:48.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:48.763 | INFO | Uploaded 36GiB to GPU in 2.04938s (17.5663GiB/s) | 20260506 16:32:48.763 | INFO | Downloaded 34GiB to GPU in 1.94995s (17.4363GiB/s) | 20260506 16:32:53.899 | INFO | Analyzing chunk frames = [2817024,2817536) | 20260506 16:32:53.899 | INFO | MemInfo: { | 20260506 16:32:53.899 | INFO | "gpu" : " 17 GB", | 20260506 16:32:53.899 | INFO | "huge" : "", | 20260506 16:32:53.899 | INFO | "huge_shared" : " 27 GB", | 20260506 16:32:53.899 | INFO | "malloc" : " 14 GB", | 20260506 16:32:53.899 | INFO | "pinned" : " 29 GB" | 20260506 16:32:53.899 | INFO | } | 20260506 16:32:53.899 | INFO | Perf report for the last 5.13641 seconds: | 20260506 16:32:53.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:32:53.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:53.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:53.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:53.899 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 16:32:53.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:53.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:53.899 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 16:32:53.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:53.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:53.899 | INFO | Uploaded 36GiB to GPU in 2.00223s (17.9799GiB/s) | 20260506 16:32:53.899 | INFO | Downloaded 34GiB to GPU in 1.98253s (17.1498GiB/s) | 20260506 16:32:59.039 | INFO | Analyzing chunk frames = [2817536,2818048) | 20260506 16:32:59.039 | INFO | MemInfo: { | 20260506 16:32:59.039 | INFO | "gpu" : " 17 GB", | 20260506 16:32:59.039 | INFO | "huge" : "", | 20260506 16:32:59.039 | INFO | "huge_shared" : " 27 GB", | 20260506 16:32:59.039 | INFO | "malloc" : " 14 GB", | 20260506 16:32:59.039 | INFO | "pinned" : " 29 GB" | 20260506 16:32:59.039 | INFO | } | 20260506 16:32:59.039 | INFO | Perf report for the last 5.13999 seconds: | 20260506 16:32:59.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:32:59.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:59.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:59.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:59.039 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:32:59.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:59.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:59.039 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:32:59.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:59.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:32:59.040 | INFO | Uploaded 36GiB to GPU in 1.98028s (18.1792GiB/s) | 20260506 16:32:59.040 | INFO | Downloaded 34GiB to GPU in 1.97075s (17.2523GiB/s) | 20260506 16:32:59.040 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:32:59.040 | INFO | Total mem usage: 18 GiB | 20260506 16:32:59.040 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:32:59.040 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.6%) | 20260506 16:32:59.040 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:32:59.040 | INFO | High Water Breakdown: | 20260506 16:32:59.040 | INFO | 61.7% : 9.88 GiB ( 86.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:32:59.040 | INFO | 38.1% : 6.1 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:32:59.040 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:32:59.040 | INFO | ------------------------------------------ | 20260506 16:32:59.040 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:32:59.040 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:32:59.040 | INFO | Total mem usage: 26.5 GiB | 20260506 16:32:59.040 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:32:59.040 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 16:32:59.040 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260506 16:32:59.040 | INFO | High Water Breakdown: | 20260506 16:32:59.040 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:32:59.040 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:32:59.040 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:32:59.040 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:32:59.040 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:32:59.040 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:32:59.040 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:32:59.040 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:32:59.040 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:32:59.040 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:32:59.040 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:32:59.040 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:32:59.040 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:32:59.040 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:32:59.040 | INFO | ------------------------------------------ | 20260506 16:32:59.040 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:32:59.040 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:32:59.040 | INFO | Total mem usage: 25.6 GiB | 20260506 16:32:59.040 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:32:59.040 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:32:59.040 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:32:59.040 | INFO | High Water Breakdown: | 20260506 16:32:59.040 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:32:59.040 | INFO | ------------------------------------------ | 20260506 16:32:59.040 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:32:59.040 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:32:59.040 | INFO | Total mem usage: 15.7 GiB | 20260506 16:32:59.040 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:32:59.040 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:32:59.040 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:32:59.040 | INFO | High Water Breakdown: | 20260506 16:32:59.040 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:32:59.040 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:32:59.040 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:32:59.040 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:32:59.040 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:32:59.040 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:32:59.040 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:32:59.040 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:32:59.040 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:32:59.040 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:32:59.040 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:32:59.042 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:32:59.042 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:32:59.042 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:32:59.042 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:32:59.042 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:32:59.042 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:32:59.042 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:32:59.042 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:32:59.042 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:32:59.042 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:32:59.042 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:32:59.042 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:32:59.042 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:32:59.042 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:32:59.042 | INFO | ------------------------------------------ | 20260506 16:32:59.042 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:32:59.042 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:32:59.042 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:32:59.042 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:32:59.042 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.58e+01, 1.34e+01 / 2.00e+00 | 20260506 16:32:59.042 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.01e-01, 1.03e-02 / 6.85e-03 | 20260506 16:32:59.042 | INFO | PerfReport: 11.59%, Baseline -- 3.64e+00 / 4.42e+00, 5.65e-01 / 3.52e-01 | 20260506 16:32:59.042 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.47e+00, 1.89e-01 / 1.29e-01 | 20260506 16:32:59.042 | INFO | PerfReport: 15.92%, DME -- 4.00e+01 / 4.45e+01, 3.83e+00 / 2.15e+00 | 20260506 16:32:59.042 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.46e+00 / 1.84e+00 | 20260506 16:32:59.042 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.29e+00 / 2.86e+00, 3.84e-01 / 2.44e-01 | 20260506 16:32:59.042 | INFO | PerfReport: 3.49%, Metrics -- 1.10e+00 / 1.47e+00, 2.80e-01 / 1.69e-01 | 20260506 16:32:59.042 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.81e-01 / 4.34e-01, 5.92e-02 / 2.52e-02 | 20260506 16:32:59.042 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:32:59.042 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:32:59.042 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:32:59.042 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 16:32:59.042 | INFO | PerfReport: 21.45%, Upload -- 2.09e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260506 16:32:59.042 | INFO | PerfReport: 21.12%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 16:32:59.042 | INFO | PerfReport: 32.27%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 16:32:59.042 | INFO | PerfReport: 1.51%, LockUpload -- 1.47e+00 / 0.00e+00, 5.94e+00 / 0.00e+00 | 20260506 16:32:59.042 | INFO | PerfReport: 1.07%, LockDownload -- 1.04e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260506 16:32:59.042 | INFO | PerfReport: 22.56%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 16:32:59.042 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260506 16:32:59.158 | INFO | Starting a round of Baz IO | 20260506 16:33:02.304 | INFO | Populating RTMetrics report | 20260506 16:33:02.305 | INFO | RTMetrics report populated | 20260506 16:33:02.305 | INFO | Opening tmp RTMetrics json file | 20260506 16:33:02.336 | INFO | Serializing RTMetrics to json | 20260506 16:33:02.340 | INFO | Serialization done, swapping tmp file now... | 20260506 16:33:02.345 | INFO | RTMetrics done | 20260506 16:33:02.841 | INFO | Finished a round of Baz IO: Wrote 2.085 GiB in 3.682 seconds | 20260506 16:33:02.841 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.26% / 0.01% / 63.73% | 20260506 16:33:02.841 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.65055% / 7.94729e-06% / 6.75519e-05% | 20260506 16:33:02.841 | INFO | Baz IO overall: | 20260506 16:33:02.841 | INFO | BytesWritten : 1480073468293 | 20260506 16:33:02.842 | INFO | NumEvents : 716192690881 | 20260506 16:33:04.184 | INFO | Analyzing chunk frames = [2818048,2818560) | 20260506 16:33:04.184 | INFO | MemInfo: { | 20260506 16:33:04.184 | INFO | "gpu" : " 17 GB", | 20260506 16:33:04.184 | INFO | "huge" : "", | 20260506 16:33:04.184 | INFO | "huge_shared" : " 27 GB", | 20260506 16:33:04.184 | INFO | "malloc" : " 14 GB", | 20260506 16:33:04.184 | INFO | "pinned" : " 29 GB" | 20260506 16:33:04.184 | INFO | } | 20260506 16:33:04.184 | INFO | Perf report for the last 5.1451 seconds: | 20260506 16:33:04.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:33:04.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:04.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:04.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:04.184 | INFO | ANALYSIS : 62.9%, 2.9, 95.7ms, 0 | 20260506 16:33:04.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:04.184 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260506 16:33:04.184 | INFO | PRE_HQ : 60.5%, 1.9, 58.4ms, 0 | 20260506 16:33:04.184 | INFO | BAZWRITER : 71.2%, 4.8, 176.3ms, 0 | 20260506 16:33:04.184 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260506 16:33:04.185 | INFO | Uploaded 36GiB to GPU in 2.0386s (17.6592GiB/s) | 20260506 16:33:04.185 | INFO | Downloaded 38GiB to GPU in 2.22012s (17.1162GiB/s) | 20260506 16:33:09.324 | INFO | Analyzing chunk frames = [2818560,2819072) | 20260506 16:33:09.324 | INFO | MemInfo: { | 20260506 16:33:09.324 | INFO | "gpu" : " 17 GB", | 20260506 16:33:09.324 | INFO | "huge" : "", | 20260506 16:33:09.324 | INFO | "huge_shared" : " 27 GB", | 20260506 16:33:09.324 | INFO | "malloc" : " 14 GB", | 20260506 16:33:09.324 | INFO | "pinned" : " 29 GB" | 20260506 16:33:09.325 | INFO | } | 20260506 16:33:09.325 | INFO | Perf report for the last 5.14026 seconds: | 20260506 16:33:09.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:33:09.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:09.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:09.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:09.325 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 16:33:09.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:09.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:09.325 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:33:09.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:09.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:09.325 | INFO | Uploaded 36GiB to GPU in 1.97568s (18.2216GiB/s) | 20260506 16:33:09.325 | INFO | Downloaded 34GiB to GPU in 1.93489s (17.5721GiB/s) | 20260506 16:33:14.469 | INFO | Analyzing chunk frames = [2819072,2819584) | 20260506 16:33:14.469 | INFO | MemInfo: { | 20260506 16:33:14.469 | INFO | "gpu" : " 17 GB", | 20260506 16:33:14.469 | INFO | "huge" : "", | 20260506 16:33:14.469 | INFO | "huge_shared" : " 27 GB", | 20260506 16:33:14.469 | INFO | "malloc" : " 14 GB", | 20260506 16:33:14.469 | INFO | "pinned" : " 29 GB" | 20260506 16:33:14.469 | INFO | } | 20260506 16:33:14.469 | INFO | Perf report for the last 5.14461 seconds: | 20260506 16:33:14.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:33:14.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:14.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:14.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:14.469 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 16:33:14.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:14.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:14.469 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:33:14.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:14.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:14.469 | INFO | Uploaded 36GiB to GPU in 1.96639s (18.3077GiB/s) | 20260506 16:33:14.469 | INFO | Downloaded 34GiB to GPU in 1.92379s (17.6734GiB/s) | 20260506 16:33:19.602 | INFO | Analyzing chunk frames = [2819584,2820096) | 20260506 16:33:19.602 | INFO | MemInfo: { | 20260506 16:33:19.602 | INFO | "gpu" : " 17 GB", | 20260506 16:33:19.602 | INFO | "huge" : "", | 20260506 16:33:19.602 | INFO | "huge_shared" : " 27 GB", | 20260506 16:33:19.602 | INFO | "malloc" : " 14 GB", | 20260506 16:33:19.602 | INFO | "pinned" : " 29 GB" | 20260506 16:33:19.602 | INFO | } | 20260506 16:33:19.602 | INFO | Perf report for the last 5.13315 seconds: | 20260506 16:33:19.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:33:19.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:19.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:19.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:19.602 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 16:33:19.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:19.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:19.603 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:33:19.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:19.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:19.603 | INFO | Uploaded 36GiB to GPU in 1.92184s (18.7321GiB/s) | 20260506 16:33:19.603 | INFO | Downloaded 34GiB to GPU in 1.87474s (18.1359GiB/s) | 20260506 16:33:24.737 | INFO | Analyzing chunk frames = [2820096,2820608) | 20260506 16:33:24.737 | INFO | MemInfo: { | 20260506 16:33:24.737 | INFO | "gpu" : " 17 GB", | 20260506 16:33:24.737 | INFO | "huge" : "", | 20260506 16:33:24.737 | INFO | "huge_shared" : " 27 GB", | 20260506 16:33:24.738 | INFO | "malloc" : " 14 GB", | 20260506 16:33:24.738 | INFO | "pinned" : " 29 GB" | 20260506 16:33:24.738 | INFO | } | 20260506 16:33:24.738 | INFO | Perf report for the last 5.13523 seconds: | 20260506 16:33:24.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:33:24.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:24.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:24.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:24.738 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 16:33:24.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:24.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:24.738 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:33:24.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:24.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:24.738 | INFO | Uploaded 36GiB to GPU in 2.01362s (17.8783GiB/s) | 20260506 16:33:24.738 | INFO | Downloaded 34GiB to GPU in 1.90812s (17.8186GiB/s) | 20260506 16:33:29.878 | INFO | Analyzing chunk frames = [2820608,2821120) | 20260506 16:33:29.878 | INFO | MemInfo: { | 20260506 16:33:29.878 | INFO | "gpu" : " 17 GB", | 20260506 16:33:29.879 | INFO | "huge" : "", | 20260506 16:33:29.879 | INFO | "huge_shared" : " 27 GB", | 20260506 16:33:29.879 | INFO | "malloc" : " 14 GB", | 20260506 16:33:29.879 | INFO | "pinned" : " 29 GB" | 20260506 16:33:29.879 | INFO | } | 20260506 16:33:29.879 | INFO | Perf report for the last 5.14099 seconds: | 20260506 16:33:29.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:33:29.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:29.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:29.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:29.879 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 16:33:29.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:29.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:29.879 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:33:29.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:29.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:29.879 | INFO | Uploaded 36GiB to GPU in 2.01702s (17.8481GiB/s) | 20260506 16:33:29.879 | INFO | Downloaded 34GiB to GPU in 1.95289s (17.4101GiB/s) | 20260506 16:33:35.015 | INFO | Analyzing chunk frames = [2821120,2821632) | 20260506 16:33:35.015 | INFO | MemInfo: { | 20260506 16:33:35.015 | INFO | "gpu" : " 17 GB", | 20260506 16:33:35.015 | INFO | "huge" : "", | 20260506 16:33:35.015 | INFO | "huge_shared" : " 27 GB", | 20260506 16:33:35.015 | INFO | "malloc" : " 14 GB", | 20260506 16:33:35.015 | INFO | "pinned" : " 29 GB" | 20260506 16:33:35.015 | INFO | } | 20260506 16:33:35.015 | INFO | Perf report for the last 5.1366 seconds: | 20260506 16:33:35.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:33:35.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:35.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:35.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:35.015 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 16:33:35.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:35.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:35.015 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:33:35.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:35.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:35.016 | INFO | Uploaded 36GiB to GPU in 2.00395s (17.9645GiB/s) | 20260506 16:33:35.016 | INFO | Downloaded 34GiB to GPU in 2.01992s (16.8324GiB/s) | 20260506 16:33:40.158 | INFO | Analyzing chunk frames = [2821632,2822144) | 20260506 16:33:40.158 | INFO | MemInfo: { | 20260506 16:33:40.158 | INFO | "gpu" : " 17 GB", | 20260506 16:33:40.158 | INFO | "huge" : "", | 20260506 16:33:40.159 | INFO | "huge_shared" : " 27 GB", | 20260506 16:33:40.159 | INFO | "malloc" : " 14 GB", | 20260506 16:33:40.159 | INFO | "pinned" : " 29 GB" | 20260506 16:33:40.159 | INFO | } | 20260506 16:33:40.159 | INFO | Perf report for the last 5.1434 seconds: | 20260506 16:33:40.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:33:40.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:40.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:40.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:40.159 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 16:33:40.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:40.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:40.159 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 16:33:40.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:40.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:40.159 | INFO | Uploaded 36GiB to GPU in 1.99795s (18.0184GiB/s) | 20260506 16:33:40.159 | INFO | Downloaded 34GiB to GPU in 1.96863s (17.2709GiB/s) | 20260506 16:33:40.278 | INFO | Starting a round of Baz IO | 20260506 16:33:43.426 | INFO | Populating RTMetrics report | 20260506 16:33:43.427 | INFO | RTMetrics report populated | 20260506 16:33:43.427 | INFO | Opening tmp RTMetrics json file | 20260506 16:33:43.471 | INFO | Serializing RTMetrics to json | 20260506 16:33:43.475 | INFO | Serialization done, swapping tmp file now... | 20260506 16:33:43.478 | INFO | RTMetrics done | 20260506 16:33:44.079 | INFO | Finished a round of Baz IO: Wrote 2.083 GiB in 3.801 seconds | 20260506 16:33:44.079 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.25% / 0.01% / 63.74% | 20260506 16:33:44.080 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.64465% / 3.97364e-05% / 5.5631e-05% | 20260506 16:33:44.080 | INFO | Baz IO overall: | 20260506 16:33:44.080 | INFO | BytesWritten : 1482310473465 | 20260506 16:33:44.080 | INFO | NumEvents : 717273266694 | 20260506 16:33:45.302 | INFO | Analyzing chunk frames = [2822144,2822656) | 20260506 16:33:45.302 | INFO | MemInfo: { | 20260506 16:33:45.302 | INFO | "gpu" : " 17 GB", | 20260506 16:33:45.302 | INFO | "huge" : "", | 20260506 16:33:45.303 | INFO | "huge_shared" : " 27 GB", | 20260506 16:33:45.303 | INFO | "malloc" : " 14 GB", | 20260506 16:33:45.303 | INFO | "pinned" : " 29 GB" | 20260506 16:33:45.303 | INFO | } | 20260506 16:33:45.303 | INFO | Perf report for the last 5.14397 seconds: | 20260506 16:33:45.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:33:45.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:45.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:45.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:45.303 | INFO | ANALYSIS : 63.0%, 2.9, 95.3ms, 0 | 20260506 16:33:45.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:45.303 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 16:33:45.303 | INFO | PRE_HQ : 60.6%, 1.9, 59.2ms, 0 | 20260506 16:33:45.303 | INFO | BAZWRITER : 73.5%, 4.8, 183.2ms, 0 | 20260506 16:33:45.303 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 16:33:45.303 | INFO | Uploaded 36GiB to GPU in 2.059s (17.4842GiB/s) | 20260506 16:33:45.303 | INFO | Downloaded 38GiB to GPU in 2.22222s (17.1GiB/s) | 20260506 16:33:50.437 | INFO | Analyzing chunk frames = [2822656,2823168) | 20260506 16:33:50.437 | INFO | MemInfo: { | 20260506 16:33:50.437 | INFO | "gpu" : " 17 GB", | 20260506 16:33:50.438 | INFO | "huge" : "", | 20260506 16:33:50.438 | INFO | "huge_shared" : " 27 GB", | 20260506 16:33:50.438 | INFO | "malloc" : " 14 GB", | 20260506 16:33:50.438 | INFO | "pinned" : " 29 GB" | 20260506 16:33:50.438 | INFO | } | 20260506 16:33:50.438 | INFO | Perf report for the last 5.13508 seconds: | 20260506 16:33:50.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:33:50.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:50.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:50.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:50.438 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:33:50.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:50.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:50.438 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:33:50.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:50.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:50.438 | INFO | Uploaded 36GiB to GPU in 2.0149s (17.8669GiB/s) | 20260506 16:33:50.438 | INFO | Downloaded 34GiB to GPU in 1.9412s (17.515GiB/s) | 20260506 16:33:55.577 | INFO | Analyzing chunk frames = [2823168,2823680) | 20260506 16:33:55.577 | INFO | MemInfo: { | 20260506 16:33:55.577 | INFO | "gpu" : " 17 GB", | 20260506 16:33:55.577 | INFO | "huge" : "", | 20260506 16:33:55.577 | INFO | "huge_shared" : " 27 GB", | 20260506 16:33:55.577 | INFO | "malloc" : " 14 GB", | 20260506 16:33:55.577 | INFO | "pinned" : " 29 GB" | 20260506 16:33:55.577 | INFO | } | 20260506 16:33:55.577 | INFO | Perf report for the last 5.13935 seconds: | 20260506 16:33:55.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:33:55.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:55.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:55.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:55.577 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 16:33:55.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:55.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:55.577 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:33:55.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:55.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:33:55.577 | INFO | Uploaded 36GiB to GPU in 1.9953s (18.0424GiB/s) | 20260506 16:33:55.577 | INFO | Downloaded 34GiB to GPU in 1.93396s (17.5805GiB/s) | 20260506 16:34:00.715 | INFO | Analyzing chunk frames = [2823680,2824192) | 20260506 16:34:00.715 | INFO | MemInfo: { | 20260506 16:34:00.715 | INFO | "gpu" : " 17 GB", | 20260506 16:34:00.715 | INFO | "huge" : "", | 20260506 16:34:00.715 | INFO | "huge_shared" : " 27 GB", | 20260506 16:34:00.715 | INFO | "malloc" : " 14 GB", | 20260506 16:34:00.715 | INFO | "pinned" : " 29 GB" | 20260506 16:34:00.715 | INFO | } | 20260506 16:34:00.715 | INFO | Perf report for the last 5.13799 seconds: | 20260506 16:34:00.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:34:00.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:00.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:00.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:00.715 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:34:00.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:00.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:00.715 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:34:00.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:00.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:00.715 | INFO | Uploaded 36GiB to GPU in 1.93611s (18.594GiB/s) | 20260506 16:34:00.715 | INFO | Downloaded 34GiB to GPU in 1.87815s (18.1029GiB/s) | 20260506 16:34:05.856 | INFO | Analyzing chunk frames = [2824192,2824704) | 20260506 16:34:05.856 | INFO | MemInfo: { | 20260506 16:34:05.856 | INFO | "gpu" : " 17 GB", | 20260506 16:34:05.856 | INFO | "huge" : "", | 20260506 16:34:05.856 | INFO | "huge_shared" : " 27 GB", | 20260506 16:34:05.856 | INFO | "malloc" : " 14 GB", | 20260506 16:34:05.856 | INFO | "pinned" : " 29 GB" | 20260506 16:34:05.856 | INFO | } | 20260506 16:34:05.856 | INFO | Perf report for the last 5.14112 seconds: | 20260506 16:34:05.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:34:05.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:05.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:05.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:05.856 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 16:34:05.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:05.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:05.856 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:34:05.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:05.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:05.857 | INFO | Uploaded 36GiB to GPU in 2.01799s (17.8396GiB/s) | 20260506 16:34:05.857 | INFO | Downloaded 34GiB to GPU in 1.91397s (17.7641GiB/s) | 20260506 16:34:10.992 | INFO | Analyzing chunk frames = [2824704,2825216) | 20260506 16:34:10.992 | INFO | MemInfo: { | 20260506 16:34:10.992 | INFO | "gpu" : " 17 GB", | 20260506 16:34:10.992 | INFO | "huge" : "", | 20260506 16:34:10.993 | INFO | "huge_shared" : " 27 GB", | 20260506 16:34:10.993 | INFO | "malloc" : " 14 GB", | 20260506 16:34:10.993 | INFO | "pinned" : " 29 GB" | 20260506 16:34:10.993 | INFO | } | 20260506 16:34:10.993 | INFO | Perf report for the last 5.13647 seconds: | 20260506 16:34:10.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:34:10.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:10.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:10.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:10.993 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 16:34:10.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:10.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:10.993 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:34:10.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:10.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:10.993 | INFO | Uploaded 36GiB to GPU in 2.07935s (17.3131GiB/s) | 20260506 16:34:10.993 | INFO | Downloaded 34GiB to GPU in 1.96747s (17.2811GiB/s) | 20260506 16:34:16.133 | INFO | Analyzing chunk frames = [2825216,2825728) | 20260506 16:34:16.133 | INFO | MemInfo: { | 20260506 16:34:16.133 | INFO | "gpu" : " 17 GB", | 20260506 16:34:16.133 | INFO | "huge" : "", | 20260506 16:34:16.133 | INFO | "huge_shared" : " 27 GB", | 20260506 16:34:16.133 | INFO | "malloc" : " 14 GB", | 20260506 16:34:16.133 | INFO | "pinned" : " 29 GB" | 20260506 16:34:16.133 | INFO | } | 20260506 16:34:16.133 | INFO | Perf report for the last 5.14039 seconds: | 20260506 16:34:16.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:34:16.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:16.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:16.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:16.133 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 16:34:16.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:16.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:16.133 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 16:34:16.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:16.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:16.133 | INFO | Uploaded 36GiB to GPU in 2.02939s (17.7393GiB/s) | 20260506 16:34:16.133 | INFO | Downloaded 34GiB to GPU in 2.01537s (16.8703GiB/s) | 20260506 16:34:21.277 | INFO | Analyzing chunk frames = [2825728,2826240) | 20260506 16:34:21.277 | INFO | MemInfo: { | 20260506 16:34:21.277 | INFO | "gpu" : " 17 GB", | 20260506 16:34:21.277 | INFO | "huge" : "", | 20260506 16:34:21.277 | INFO | "huge_shared" : " 27 GB", | 20260506 16:34:21.277 | INFO | "malloc" : " 14 GB", | 20260506 16:34:21.277 | INFO | "pinned" : " 29 GB" | 20260506 16:34:21.277 | INFO | } | 20260506 16:34:21.277 | INFO | Perf report for the last 5.14358 seconds: | 20260506 16:34:21.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:34:21.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:21.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:21.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:21.277 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 16:34:21.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:21.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:21.277 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:34:21.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:21.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:21.277 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:34:21.277 | INFO | Total mem usage: 18 GiB | 20260506 16:34:21.277 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:34:21.277 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -11.5%) | 20260506 16:34:21.277 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:34:21.277 | INFO | High Water Breakdown: | 20260506 16:34:21.277 | INFO | 61.8% : 9.98 GiB ( 87.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:34:21.277 | INFO | 38.1% : 6.14 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:34:21.277 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:34:21.277 | INFO | ------------------------------------------ | 20260506 16:34:21.277 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:34:21.277 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:34:21.277 | INFO | Total mem usage: 26.5 GiB | 20260506 16:34:21.277 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:34:21.277 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:34:21.277 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:34:21.277 | INFO | High Water Breakdown: | 20260506 16:34:21.277 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:34:21.277 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:34:21.277 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:34:21.277 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:34:21.277 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:34:21.277 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:34:21.277 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:34:21.277 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:34:21.277 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:34:21.277 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:34:21.277 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:34:21.277 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:34:21.277 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:34:21.277 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:34:21.277 | INFO | ------------------------------------------ | 20260506 16:34:21.277 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:34:21.277 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:34:21.277 | INFO | Total mem usage: 25.6 GiB | 20260506 16:34:21.277 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:34:21.277 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:34:21.277 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:34:21.277 | INFO | High Water Breakdown: | 20260506 16:34:21.277 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:34:21.277 | INFO | ------------------------------------------ | 20260506 16:34:21.277 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:34:21.277 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:34:21.277 | INFO | Total mem usage: 15.7 GiB | 20260506 16:34:21.277 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:34:21.277 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:34:21.277 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:34:21.277 | INFO | High Water Breakdown: | 20260506 16:34:21.277 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:34:21.277 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:34:21.277 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:34:21.277 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:34:21.277 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:34:21.277 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:34:21.277 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:34:21.277 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:34:21.277 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:34:21.277 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:34:21.277 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:34:21.277 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:34:21.277 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:34:21.277 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:34:21.277 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:34:21.277 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:34:21.277 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:34:21.277 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:34:21.277 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:34:21.277 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:34:21.277 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:34:21.277 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:34:21.277 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:34:21.277 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:34:21.277 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:34:21.277 | INFO | ------------------------------------------ | 20260506 16:34:21.277 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:34:21.277 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:34:21.277 | INFO | Uploaded 36GiB to GPU in 2.04099s (17.6385GiB/s) | 20260506 16:34:21.278 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:34:21.278 | INFO | Downloaded 34GiB to GPU in 1.97824s (17.187GiB/s) | 20260506 16:34:21.278 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:34:21.278 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.61e+01, 1.34e+01 / 2.59e+00 | 20260506 16:34:21.278 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.02e-01, 1.09e-02 / 6.35e-03 | 20260506 16:34:21.278 | INFO | PerfReport: 11.58%, Baseline -- 3.64e+00 / 4.46e+00, 5.63e-01 / 3.27e-01 | 20260506 16:34:21.278 | INFO | PerfReport: 3.26%, Histogram -- 1.03e+00 / 1.47e+00, 1.88e-01 / 1.38e-01 | 20260506 16:34:21.278 | INFO | PerfReport: 15.93%, DME -- 4.00e+01 / 4.47e+01, 4.10e+00 / 2.24e+00 | 20260506 16:34:21.278 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.49e+00 / 1.75e+00 | 20260506 16:34:21.278 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.29e+00 / 2.82e+00, 3.80e-01 / 2.43e-01 | 20260506 16:34:21.278 | INFO | PerfReport: 3.48%, Metrics -- 1.09e+00 / 1.44e+00, 2.81e-01 / 1.82e-01 | 20260506 16:34:21.278 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.81e-01 / 4.30e-01, 5.99e-02 / 2.36e-02 | 20260506 16:34:21.278 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:34:21.278 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:34:21.278 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:34:21.278 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 16:34:21.278 | INFO | PerfReport: 21.46%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 16:34:21.278 | INFO | PerfReport: 21.16%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 16:34:21.278 | INFO | PerfReport: 32.26%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 16:34:21.278 | INFO | PerfReport: 1.41%, LockUpload -- 1.37e+00 / 0.00e+00, 5.76e+00 / 0.00e+00 | 20260506 16:34:21.279 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260506 16:34:21.279 | INFO | PerfReport: 22.61%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 16:34:21.279 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.41e-02 / 0.00e+00 | 20260506 16:34:21.397 | INFO | Starting a round of Baz IO | 20260506 16:34:24.537 | INFO | Populating RTMetrics report | 20260506 16:34:24.538 | INFO | RTMetrics report populated | 20260506 16:34:24.538 | INFO | Opening tmp RTMetrics json file | 20260506 16:34:24.641 | INFO | Serializing RTMetrics to json | 20260506 16:34:24.649 | INFO | Serialization done, swapping tmp file now... | 20260506 16:34:24.650 | INFO | RTMetrics done | 20260506 16:34:25.201 | INFO | Finished a round of Baz IO: Wrote 2.085 GiB in 3.803 seconds | 20260506 16:34:25.201 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.24% / 0.01% / 63.75% | 20260506 16:34:25.201 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.64232% / 1.58946e-05% / 5.5631e-05% | 20260506 16:34:25.201 | INFO | Baz IO overall: | 20260506 16:34:25.201 | INFO | BytesWritten : 1484549098419 | 20260506 16:34:25.201 | INFO | NumEvents : 718354995233 | 20260506 16:34:26.418 | INFO | Analyzing chunk frames = [2826240,2826752) | 20260506 16:34:26.418 | INFO | MemInfo: { | 20260506 16:34:26.418 | INFO | "gpu" : " 17 GB", | 20260506 16:34:26.418 | INFO | "huge" : "", | 20260506 16:34:26.418 | INFO | "huge_shared" : " 27 GB", | 20260506 16:34:26.418 | INFO | "malloc" : " 14 GB", | 20260506 16:34:26.418 | INFO | "pinned" : " 29 GB" | 20260506 16:34:26.418 | INFO | } | 20260506 16:34:26.418 | INFO | Perf report for the last 5.14125 seconds: | 20260506 16:34:26.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:34:26.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:26.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:26.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:26.418 | INFO | ANALYSIS : 62.8%, 2.9, 95.3ms, 0 | 20260506 16:34:26.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:26.418 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260506 16:34:26.418 | INFO | PRE_HQ : 60.3%, 1.9, 58.7ms, 0 | 20260506 16:34:26.418 | INFO | BAZWRITER : 73.5%, 4.8, 182.0ms, 0 | 20260506 16:34:26.418 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 16:34:26.418 | INFO | Uploaded 36GiB to GPU in 2.05126s (17.5502GiB/s) | 20260506 16:34:26.418 | INFO | Downloaded 38GiB to GPU in 2.20961s (17.1976GiB/s) | 20260506 16:34:31.557 | INFO | Analyzing chunk frames = [2826752,2827264) | 20260506 16:34:31.557 | INFO | MemInfo: { | 20260506 16:34:31.557 | INFO | "gpu" : " 17 GB", | 20260506 16:34:31.557 | INFO | "huge" : "", | 20260506 16:34:31.557 | INFO | "huge_shared" : " 27 GB", | 20260506 16:34:31.557 | INFO | "malloc" : " 14 GB", | 20260506 16:34:31.557 | INFO | "pinned" : " 29 GB" | 20260506 16:34:31.557 | INFO | } | 20260506 16:34:31.557 | INFO | Perf report for the last 5.13911 seconds: | 20260506 16:34:31.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:34:31.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:31.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:31.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:31.557 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 16:34:31.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:31.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:31.557 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 16:34:31.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:31.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:31.557 | INFO | Uploaded 36GiB to GPU in 1.99042s (18.0866GiB/s) | 20260506 16:34:31.557 | INFO | Downloaded 34GiB to GPU in 1.95779s (17.3665GiB/s) | 20260506 16:34:36.690 | INFO | Analyzing chunk frames = [2827264,2827776) | 20260506 16:34:36.690 | INFO | MemInfo: { | 20260506 16:34:36.690 | INFO | "gpu" : " 17 GB", | 20260506 16:34:36.690 | INFO | "huge" : "", | 20260506 16:34:36.691 | INFO | "huge_shared" : " 27 GB", | 20260506 16:34:36.691 | INFO | "malloc" : " 14 GB", | 20260506 16:34:36.691 | INFO | "pinned" : " 29 GB" | 20260506 16:34:36.691 | INFO | } | 20260506 16:34:36.691 | INFO | Perf report for the last 5.13352 seconds: | 20260506 16:34:36.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:34:36.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:36.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:36.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:36.691 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 16:34:36.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:36.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:36.691 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:34:36.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:36.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:36.691 | INFO | Uploaded 36GiB to GPU in 1.98064s (18.176GiB/s) | 20260506 16:34:36.691 | INFO | Downloaded 34GiB to GPU in 1.93669s (17.5558GiB/s) | 20260506 16:34:41.827 | INFO | Analyzing chunk frames = [2827776,2828288) | 20260506 16:34:41.827 | INFO | MemInfo: { | 20260506 16:34:41.827 | INFO | "gpu" : " 17 GB", | 20260506 16:34:41.827 | INFO | "huge" : "", | 20260506 16:34:41.827 | INFO | "huge_shared" : " 27 GB", | 20260506 16:34:41.827 | INFO | "malloc" : " 14 GB", | 20260506 16:34:41.827 | INFO | "pinned" : " 29 GB" | 20260506 16:34:41.827 | INFO | } | 20260506 16:34:41.827 | INFO | Perf report for the last 5.13661 seconds: | 20260506 16:34:41.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:34:41.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:41.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:41.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:41.827 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 16:34:41.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:41.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:41.827 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:34:41.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:41.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:41.828 | INFO | Uploaded 36GiB to GPU in 1.94038s (18.5531GiB/s) | 20260506 16:34:41.828 | INFO | Downloaded 34GiB to GPU in 1.88106s (18.0749GiB/s) | 20260506 16:34:46.970 | INFO | Analyzing chunk frames = [2828288,2828800) | 20260506 16:34:46.970 | INFO | MemInfo: { | 20260506 16:34:46.970 | INFO | "gpu" : " 17 GB", | 20260506 16:34:46.970 | INFO | "huge" : "", | 20260506 16:34:46.971 | INFO | "huge_shared" : " 27 GB", | 20260506 16:34:46.971 | INFO | "malloc" : " 14 GB", | 20260506 16:34:46.971 | INFO | "pinned" : " 29 GB" | 20260506 16:34:46.971 | INFO | } | 20260506 16:34:46.971 | INFO | Perf report for the last 5.14341 seconds: | 20260506 16:34:46.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:34:46.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:46.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:46.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:46.971 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 16:34:46.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:46.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:46.971 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:34:46.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:46.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:46.971 | INFO | Uploaded 36GiB to GPU in 1.96954s (18.2784GiB/s) | 20260506 16:34:46.971 | INFO | Downloaded 34GiB to GPU in 1.88394s (18.0473GiB/s) | 20260506 16:34:52.107 | INFO | Analyzing chunk frames = [2828800,2829312) | 20260506 16:34:52.107 | INFO | MemInfo: { | 20260506 16:34:52.107 | INFO | "gpu" : " 17 GB", | 20260506 16:34:52.107 | INFO | "huge" : "", | 20260506 16:34:52.107 | INFO | "huge_shared" : " 27 GB", | 20260506 16:34:52.107 | INFO | "malloc" : " 14 GB", | 20260506 16:34:52.107 | INFO | "pinned" : " 29 GB" | 20260506 16:34:52.107 | INFO | } | 20260506 16:34:52.107 | INFO | Perf report for the last 5.13659 seconds: | 20260506 16:34:52.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:34:52.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:52.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:52.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:52.107 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 16:34:52.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:52.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:52.107 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:34:52.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:52.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:52.108 | INFO | Uploaded 36GiB to GPU in 2.00716s (17.9358GiB/s) | 20260506 16:34:52.108 | INFO | Downloaded 34GiB to GPU in 1.94921s (17.4429GiB/s) | 20260506 16:34:57.248 | INFO | Analyzing chunk frames = [2829312,2829824) | 20260506 16:34:57.248 | INFO | MemInfo: { | 20260506 16:34:57.248 | INFO | "gpu" : " 17 GB", | 20260506 16:34:57.248 | INFO | "huge" : "", | 20260506 16:34:57.248 | INFO | "huge_shared" : " 27 GB", | 20260506 16:34:57.248 | INFO | "malloc" : " 14 GB", | 20260506 16:34:57.248 | INFO | "pinned" : " 29 GB" | 20260506 16:34:57.248 | INFO | } | 20260506 16:34:57.248 | INFO | Perf report for the last 5.14071 seconds: | 20260506 16:34:57.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:34:57.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:57.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:57.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:57.248 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 16:34:57.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:57.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:57.248 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 16:34:57.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:57.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:34:57.248 | INFO | Uploaded 36GiB to GPU in 1.98243s (18.1595GiB/s) | 20260506 16:34:57.248 | INFO | Downloaded 34GiB to GPU in 2.01448s (16.8778GiB/s) | 20260506 16:35:02.386 | INFO | Analyzing chunk frames = [2829824,2830336) | 20260506 16:35:02.386 | INFO | MemInfo: { | 20260506 16:35:02.386 | INFO | "gpu" : " 17 GB", | 20260506 16:35:02.386 | INFO | "huge" : "", | 20260506 16:35:02.386 | INFO | "huge_shared" : " 27 GB", | 20260506 16:35:02.386 | INFO | "malloc" : " 14 GB", | 20260506 16:35:02.386 | INFO | "pinned" : " 29 GB" | 20260506 16:35:02.386 | INFO | } | 20260506 16:35:02.386 | INFO | Perf report for the last 5.13814 seconds: | 20260506 16:35:02.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:35:02.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:02.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:02.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:02.386 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:35:02.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:02.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:02.386 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 16:35:02.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:02.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:02.386 | INFO | Uploaded 36GiB to GPU in 2.02526s (17.7755GiB/s) | 20260506 16:35:02.386 | INFO | Downloaded 34GiB to GPU in 1.98298s (17.1459GiB/s) | 20260506 16:35:02.506 | INFO | Starting a round of Baz IO | 20260506 16:35:05.648 | INFO | Populating RTMetrics report | 20260506 16:35:05.649 | INFO | RTMetrics report populated | 20260506 16:35:05.649 | INFO | Opening tmp RTMetrics json file | 20260506 16:35:05.680 | INFO | Serializing RTMetrics to json | 20260506 16:35:05.682 | INFO | Serialization done, swapping tmp file now... | 20260506 16:35:05.682 | INFO | RTMetrics done | 20260506 16:35:06.287 | INFO | Finished a round of Baz IO: Wrote 2.084 GiB in 3.781 seconds | 20260506 16:35:06.287 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.24% / 0.01% / 63.76% | 20260506 16:35:06.287 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.63864% / 1.58946e-05% / 5.16574e-05% | 20260506 16:35:06.287 | INFO | Baz IO overall: | 20260506 16:35:06.287 | INFO | BytesWritten : 1486786884755 | 20260506 16:35:06.287 | INFO | NumEvents : 719436364149 | 20260506 16:35:07.531 | INFO | Analyzing chunk frames = [2830336,2830848) | 20260506 16:35:07.531 | INFO | MemInfo: { | 20260506 16:35:07.531 | INFO | "gpu" : " 17 GB", | 20260506 16:35:07.531 | INFO | "huge" : "", | 20260506 16:35:07.531 | INFO | "huge_shared" : " 27 GB", | 20260506 16:35:07.531 | INFO | "malloc" : " 14 GB", | 20260506 16:35:07.531 | INFO | "pinned" : " 29 GB" | 20260506 16:35:07.531 | INFO | } | 20260506 16:35:07.531 | INFO | Perf report for the last 5.14482 seconds: | 20260506 16:35:07.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:35:07.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:07.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:07.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:07.531 | INFO | ANALYSIS : 62.8%, 2.9, 95.4ms, 0 | 20260506 16:35:07.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:07.531 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 16:35:07.531 | INFO | PRE_HQ : 60.7%, 1.9, 58.8ms, 0 | 20260506 16:35:07.531 | INFO | BAZWRITER : 72.9%, 4.8, 180.9ms, 0 | 20260506 16:35:07.531 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 16:35:07.531 | INFO | Uploaded 36GiB to GPU in 2.04742s (17.5831GiB/s) | 20260506 16:35:07.531 | INFO | Downloaded 38GiB to GPU in 2.21424s (17.1616GiB/s) | 20260506 16:35:12.668 | INFO | Analyzing chunk frames = [2830848,2831360) | 20260506 16:35:12.668 | INFO | MemInfo: { | 20260506 16:35:12.668 | INFO | "gpu" : " 17 GB", | 20260506 16:35:12.668 | INFO | "huge" : "", | 20260506 16:35:12.668 | INFO | "huge_shared" : " 27 GB", | 20260506 16:35:12.668 | INFO | "malloc" : " 14 GB", | 20260506 16:35:12.668 | INFO | "pinned" : " 29 GB" | 20260506 16:35:12.668 | INFO | } | 20260506 16:35:12.668 | INFO | Perf report for the last 5.13741 seconds: | 20260506 16:35:12.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:35:12.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:12.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:12.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:12.669 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:35:12.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:12.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:12.669 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 16:35:12.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:12.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:12.669 | INFO | Uploaded 36GiB to GPU in 1.99674s (18.0294GiB/s) | 20260506 16:35:12.669 | INFO | Downloaded 34GiB to GPU in 1.9249s (17.6633GiB/s) | 20260506 16:35:17.807 | INFO | Analyzing chunk frames = [2831360,2831872) | 20260506 16:35:17.807 | INFO | MemInfo: { | 20260506 16:35:17.807 | INFO | "gpu" : " 17 GB", | 20260506 16:35:17.807 | INFO | "huge" : "", | 20260506 16:35:17.807 | INFO | "huge_shared" : " 27 GB", | 20260506 16:35:17.808 | INFO | "malloc" : " 14 GB", | 20260506 16:35:17.808 | INFO | "pinned" : " 29 GB" | 20260506 16:35:17.808 | INFO | } | 20260506 16:35:17.808 | INFO | Perf report for the last 5.13912 seconds: | 20260506 16:35:17.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:35:17.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:17.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:17.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:17.808 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 16:35:17.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:17.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:17.808 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:35:17.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:17.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:17.808 | INFO | Uploaded 36GiB to GPU in 1.98774s (18.111GiB/s) | 20260506 16:35:17.808 | INFO | Downloaded 34GiB to GPU in 1.93145s (17.6033GiB/s) | 20260506 16:35:22.945 | INFO | Analyzing chunk frames = [2831872,2832384) | 20260506 16:35:22.945 | INFO | MemInfo: { | 20260506 16:35:22.945 | INFO | "gpu" : " 17 GB", | 20260506 16:35:22.945 | INFO | "huge" : "", | 20260506 16:35:22.945 | INFO | "huge_shared" : " 27 GB", | 20260506 16:35:22.945 | INFO | "malloc" : " 14 GB", | 20260506 16:35:22.945 | INFO | "pinned" : " 29 GB" | 20260506 16:35:22.945 | INFO | } | 20260506 16:35:22.945 | INFO | Perf report for the last 5.13763 seconds: | 20260506 16:35:22.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:35:22.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:22.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:22.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:22.945 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:35:22.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:22.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:22.945 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:35:22.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:22.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:22.946 | INFO | Uploaded 36GiB to GPU in 1.93631s (18.5921GiB/s) | 20260506 16:35:22.946 | INFO | Downloaded 34GiB to GPU in 1.87409s (18.1421GiB/s) | 20260506 16:35:28.082 | INFO | Analyzing chunk frames = [2832384,2832896) | 20260506 16:35:28.082 | INFO | MemInfo: { | 20260506 16:35:28.082 | INFO | "gpu" : " 17 GB", | 20260506 16:35:28.082 | INFO | "huge" : "", | 20260506 16:35:28.082 | INFO | "huge_shared" : " 27 GB", | 20260506 16:35:28.082 | INFO | "malloc" : " 14 GB", | 20260506 16:35:28.082 | INFO | "pinned" : " 29 GB" | 20260506 16:35:28.082 | INFO | } | 20260506 16:35:28.082 | INFO | Perf report for the last 5.13709 seconds: | 20260506 16:35:28.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:35:28.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:28.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:28.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:28.082 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 16:35:28.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:28.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:28.082 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:35:28.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:28.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:28.083 | INFO | Uploaded 36GiB to GPU in 2.00998s (17.9107GiB/s) | 20260506 16:35:28.083 | INFO | Downloaded 34GiB to GPU in 1.90605s (17.8379GiB/s) | 20260506 16:35:33.225 | INFO | Analyzing chunk frames = [2832896,2833408) | 20260506 16:35:33.225 | INFO | MemInfo: { | 20260506 16:35:33.225 | INFO | "gpu" : " 17 GB", | 20260506 16:35:33.225 | INFO | "huge" : "", | 20260506 16:35:33.225 | INFO | "huge_shared" : " 27 GB", | 20260506 16:35:33.225 | INFO | "malloc" : " 14 GB", | 20260506 16:35:33.225 | INFO | "pinned" : " 29 GB" | 20260506 16:35:33.225 | INFO | } | 20260506 16:35:33.225 | INFO | Perf report for the last 5.14253 seconds: | 20260506 16:35:33.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:35:33.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:33.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:33.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:33.225 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 16:35:33.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:33.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:33.225 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:35:33.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:33.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:33.225 | INFO | Uploaded 36GiB to GPU in 2.00938s (17.9159GiB/s) | 20260506 16:35:33.225 | INFO | Downloaded 34GiB to GPU in 1.94601s (17.4717GiB/s) | 20260506 16:35:38.363 | INFO | Analyzing chunk frames = [2833408,2833920) | 20260506 16:35:38.363 | INFO | MemInfo: { | 20260506 16:35:38.363 | INFO | "gpu" : " 17 GB", | 20260506 16:35:38.363 | INFO | "huge" : "", | 20260506 16:35:38.363 | INFO | "huge_shared" : " 27 GB", | 20260506 16:35:38.363 | INFO | "malloc" : " 14 GB", | 20260506 16:35:38.363 | INFO | "pinned" : " 29 GB" | 20260506 16:35:38.363 | INFO | } | 20260506 16:35:38.363 | INFO | Perf report for the last 5.13806 seconds: | 20260506 16:35:38.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:35:38.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:38.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:38.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:38.363 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 16:35:38.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:38.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:38.363 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:35:38.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:38.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:38.363 | INFO | Uploaded 36GiB to GPU in 2.00574s (17.9485GiB/s) | 20260506 16:35:38.363 | INFO | Downloaded 34GiB to GPU in 2.00478s (16.9594GiB/s) | 20260506 16:35:43.505 | INFO | Analyzing chunk frames = [2833920,2834432) | 20260506 16:35:43.505 | INFO | MemInfo: { | 20260506 16:35:43.505 | INFO | "gpu" : " 17 GB", | 20260506 16:35:43.505 | INFO | "huge" : "", | 20260506 16:35:43.505 | INFO | "huge_shared" : " 27 GB", | 20260506 16:35:43.505 | INFO | "malloc" : " 14 GB", | 20260506 16:35:43.505 | INFO | "pinned" : " 29 GB" | 20260506 16:35:43.505 | INFO | } | 20260506 16:35:43.505 | INFO | Perf report for the last 5.14188 seconds: | 20260506 16:35:43.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:35:43.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:43.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:43.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:43.505 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:35:43.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:43.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:43.505 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 16:35:43.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:43.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:43.505 | INFO | Uploaded 36GiB to GPU in 1.9848s (18.1378GiB/s) | 20260506 16:35:43.505 | INFO | Downloaded 34GiB to GPU in 1.96951s (17.2632GiB/s) | 20260506 16:35:43.505 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:35:43.505 | INFO | Total mem usage: 18 GiB | 20260506 16:35:43.505 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:35:43.505 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -13.8%) | 20260506 16:35:43.505 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:35:43.505 | INFO | High Water Breakdown: | 20260506 16:35:43.505 | INFO | 61.5% : 9.74 GiB ( 85.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:35:43.505 | INFO | 38.3% : 6.07 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:35:43.505 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:35:43.505 | INFO | ------------------------------------------ | 20260506 16:35:43.506 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:35:43.506 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:35:43.506 | INFO | Total mem usage: 26.5 GiB | 20260506 16:35:43.506 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:35:43.506 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:35:43.506 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:35:43.506 | INFO | High Water Breakdown: | 20260506 16:35:43.506 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:35:43.506 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:35:43.506 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:35:43.506 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:35:43.506 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:35:43.506 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:35:43.506 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:35:43.506 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:35:43.506 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:35:43.506 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:35:43.506 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:35:43.506 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:35:43.506 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:35:43.506 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:35:43.506 | INFO | ------------------------------------------ | 20260506 16:35:43.506 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:35:43.506 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:35:43.506 | INFO | Total mem usage: 25.6 GiB | 20260506 16:35:43.506 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:35:43.506 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:35:43.506 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:35:43.506 | INFO | High Water Breakdown: | 20260506 16:35:43.506 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:35:43.506 | INFO | ------------------------------------------ | 20260506 16:35:43.506 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:35:43.506 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:35:43.506 | INFO | Total mem usage: 15.7 GiB | 20260506 16:35:43.506 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:35:43.506 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:35:43.506 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:35:43.506 | INFO | High Water Breakdown: | 20260506 16:35:43.506 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:35:43.506 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:35:43.506 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:35:43.506 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:35:43.506 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:35:43.506 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:35:43.506 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:35:43.506 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:35:43.506 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:35:43.506 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:35:43.506 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:35:43.506 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:35:43.506 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:35:43.506 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:35:43.506 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:35:43.506 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:35:43.506 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:35:43.506 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:35:43.506 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:35:43.506 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:35:43.506 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:35:43.506 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:35:43.506 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:35:43.506 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:35:43.506 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:35:43.506 | INFO | ------------------------------------------ | 20260506 16:35:43.506 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:35:43.506 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:35:43.506 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:35:43.506 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:35:43.506 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.61e+01, 1.35e+01 / 2.30e+00 | 20260506 16:35:43.506 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.03e-01, 1.02e-02 / 8.09e-03 | 20260506 16:35:43.506 | INFO | PerfReport: 11.60%, Baseline -- 3.64e+00 / 4.40e+00, 5.65e-01 / 3.45e-01 | 20260506 16:35:43.506 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.48e+00, 1.87e-01 / 1.24e-01 | 20260506 16:35:43.506 | INFO | PerfReport: 15.96%, DME -- 4.01e+01 / 4.50e+01, 4.13e+00 / 2.58e+00 | 20260506 16:35:43.506 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.40e+00 / 1.88e+00 | 20260506 16:35:43.506 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.29e+00 / 2.86e+00, 3.89e-01 / 2.71e-01 | 20260506 16:35:43.506 | INFO | PerfReport: 3.49%, Metrics -- 1.09e+00 / 1.48e+00, 2.83e-01 / 1.75e-01 | 20260506 16:35:43.506 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.79e-01 / 4.30e-01, 5.89e-02 / 2.10e-02 | 20260506 16:35:43.506 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:35:43.506 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:35:43.506 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:35:43.507 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 16:35:43.507 | INFO | PerfReport: 21.38%, Upload -- 2.08e+01 / 0.00e+00, 6.91e+00 / 0.00e+00 | 20260506 16:35:43.507 | INFO | PerfReport: 21.15%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 16:35:43.507 | INFO | PerfReport: 32.27%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 16:35:43.507 | INFO | PerfReport: 1.36%, LockUpload -- 1.32e+00 / 0.00e+00, 5.61e+00 / 0.00e+00 | 20260506 16:35:43.507 | INFO | PerfReport: 1.18%, LockDownload -- 1.15e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 16:35:43.507 | INFO | PerfReport: 22.65%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 16:35:43.507 | INFO | PerfReport: 0.02%, Misc -- 1.53e-02 / 0.00e+00, 4.41e-02 / 0.00e+00 | 20260506 16:35:43.619 | INFO | Starting a round of Baz IO | 20260506 16:35:46.764 | INFO | Populating RTMetrics report | 20260506 16:35:46.764 | INFO | RTMetrics report populated | 20260506 16:35:46.764 | INFO | Opening tmp RTMetrics json file | 20260506 16:35:46.821 | INFO | Serializing RTMetrics to json | 20260506 16:35:46.827 | INFO | Serialization done, swapping tmp file now... | 20260506 16:35:46.832 | INFO | RTMetrics done | 20260506 16:35:47.137 | INFO | Finished a round of Baz IO: Wrote 2.083 GiB in 3.517 seconds | 20260506 16:35:47.137 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.23% / 0.01% / 63.76% | 20260506 16:35:47.137 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.63548% / 3.17891e-05% / 5.96046e-05% | 20260506 16:35:47.137 | INFO | Baz IO overall: | 20260506 16:35:47.137 | INFO | BytesWritten : 1489023765060 | 20260506 16:35:47.137 | INFO | NumEvents : 720517347406 | 20260506 16:35:48.645 | INFO | Analyzing chunk frames = [2834432,2834944) | 20260506 16:35:48.645 | INFO | MemInfo: { | 20260506 16:35:48.645 | INFO | "gpu" : " 17 GB", | 20260506 16:35:48.645 | INFO | "huge" : "", | 20260506 16:35:48.645 | INFO | "huge_shared" : " 27 GB", | 20260506 16:35:48.645 | INFO | "malloc" : " 14 GB", | 20260506 16:35:48.645 | INFO | "pinned" : " 29 GB" | 20260506 16:35:48.645 | INFO | } | 20260506 16:35:48.645 | INFO | Perf report for the last 5.14024 seconds: | 20260506 16:35:48.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:35:48.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:48.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:48.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:48.645 | INFO | ANALYSIS : 62.8%, 2.9, 95.4ms, 0 | 20260506 16:35:48.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:48.645 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 16:35:48.645 | INFO | PRE_HQ : 60.6%, 1.9, 58.8ms, 0 | 20260506 16:35:48.645 | INFO | BAZWRITER : 68.1%, 4.8, 168.3ms, 0 | 20260506 16:35:48.645 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 16:35:48.645 | INFO | Uploaded 36GiB to GPU in 2.01725s (17.8461GiB/s) | 20260506 16:35:48.646 | INFO | Downloaded 38GiB to GPU in 2.21175s (17.1809GiB/s) | 20260506 16:35:53.780 | INFO | Analyzing chunk frames = [2834944,2835456) | 20260506 16:35:53.780 | INFO | MemInfo: { | 20260506 16:35:53.780 | INFO | "gpu" : " 17 GB", | 20260506 16:35:53.780 | INFO | "huge" : "", | 20260506 16:35:53.780 | INFO | "huge_shared" : " 27 GB", | 20260506 16:35:53.780 | INFO | "malloc" : " 14 GB", | 20260506 16:35:53.780 | INFO | "pinned" : " 29 GB" | 20260506 16:35:53.780 | INFO | } | 20260506 16:35:53.780 | INFO | Perf report for the last 5.13533 seconds: | 20260506 16:35:53.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:35:53.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:53.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:53.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:53.781 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 16:35:53.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:53.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:53.781 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:35:53.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:53.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:53.781 | INFO | Uploaded 36GiB to GPU in 1.98432s (18.1422GiB/s) | 20260506 16:35:53.781 | INFO | Downloaded 34GiB to GPU in 1.93627s (17.5595GiB/s) | 20260506 16:35:58.919 | INFO | Analyzing chunk frames = [2835456,2835968) | 20260506 16:35:58.919 | INFO | MemInfo: { | 20260506 16:35:58.919 | INFO | "gpu" : " 17 GB", | 20260506 16:35:58.919 | INFO | "huge" : "", | 20260506 16:35:58.920 | INFO | "huge_shared" : " 27 GB", | 20260506 16:35:58.920 | INFO | "malloc" : " 14 GB", | 20260506 16:35:58.920 | INFO | "pinned" : " 29 GB" | 20260506 16:35:58.920 | INFO | } | 20260506 16:35:58.920 | INFO | Perf report for the last 5.13916 seconds: | 20260506 16:35:58.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:35:58.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:58.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:58.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:58.920 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 16:35:58.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:58.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:58.920 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:35:58.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:58.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:35:58.920 | INFO | Uploaded 36GiB to GPU in 1.96976s (18.2763GiB/s) | 20260506 16:35:58.920 | INFO | Downloaded 34GiB to GPU in 1.92956s (17.6206GiB/s) | 20260506 16:36:04.058 | INFO | Analyzing chunk frames = [2835968,2836480) | 20260506 16:36:04.058 | INFO | MemInfo: { | 20260506 16:36:04.058 | INFO | "gpu" : " 17 GB", | 20260506 16:36:04.058 | INFO | "huge" : "", | 20260506 16:36:04.058 | INFO | "huge_shared" : " 27 GB", | 20260506 16:36:04.058 | INFO | "malloc" : " 14 GB", | 20260506 16:36:04.058 | INFO | "pinned" : " 29 GB" | 20260506 16:36:04.058 | INFO | } | 20260506 16:36:04.058 | INFO | Perf report for the last 5.13877 seconds: | 20260506 16:36:04.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:36:04.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:04.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:04.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:04.058 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 16:36:04.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:04.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:04.059 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:36:04.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:04.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:04.059 | INFO | Uploaded 36GiB to GPU in 1.91516s (18.7973GiB/s) | 20260506 16:36:04.059 | INFO | Downloaded 34GiB to GPU in 1.87574s (18.1261GiB/s) | 20260506 16:36:09.196 | INFO | Analyzing chunk frames = [2836480,2836992) | 20260506 16:36:09.196 | INFO | MemInfo: { | 20260506 16:36:09.196 | INFO | "gpu" : " 17 GB", | 20260506 16:36:09.196 | INFO | "huge" : "", | 20260506 16:36:09.196 | INFO | "huge_shared" : " 27 GB", | 20260506 16:36:09.196 | INFO | "malloc" : " 14 GB", | 20260506 16:36:09.196 | INFO | "pinned" : " 29 GB" | 20260506 16:36:09.196 | INFO | } | 20260506 16:36:09.196 | INFO | Perf report for the last 5.13747 seconds: | 20260506 16:36:09.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:36:09.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:09.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:09.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:09.196 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 16:36:09.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:09.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:09.196 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 16:36:09.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:09.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:09.196 | INFO | Uploaded 36GiB to GPU in 1.94843s (18.4764GiB/s) | 20260506 16:36:09.196 | INFO | Downloaded 34GiB to GPU in 1.88454s (18.0416GiB/s) | 20260506 16:36:14.335 | INFO | Analyzing chunk frames = [2836992,2837504) | 20260506 16:36:14.335 | INFO | MemInfo: { | 20260506 16:36:14.335 | INFO | "gpu" : " 17 GB", | 20260506 16:36:14.335 | INFO | "huge" : "", | 20260506 16:36:14.335 | INFO | "huge_shared" : " 27 GB", | 20260506 16:36:14.335 | INFO | "malloc" : " 14 GB", | 20260506 16:36:14.335 | INFO | "pinned" : " 29 GB" | 20260506 16:36:14.335 | INFO | } | 20260506 16:36:14.336 | INFO | Perf report for the last 5.13966 seconds: | 20260506 16:36:14.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:36:14.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:14.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:14.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:14.336 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 16:36:14.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:14.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:14.336 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:36:14.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:14.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:14.336 | INFO | Uploaded 36GiB to GPU in 1.99212s (18.0712GiB/s) | 20260506 16:36:14.336 | INFO | Downloaded 34GiB to GPU in 1.95274s (17.4114GiB/s) | 20260506 16:36:19.478 | INFO | Analyzing chunk frames = [2837504,2838016) | 20260506 16:36:19.478 | INFO | MemInfo: { | 20260506 16:36:19.478 | INFO | "gpu" : " 17 GB", | 20260506 16:36:19.478 | INFO | "huge" : "", | 20260506 16:36:19.478 | INFO | "huge_shared" : " 27 GB", | 20260506 16:36:19.478 | INFO | "malloc" : " 14 GB", | 20260506 16:36:19.478 | INFO | "pinned" : " 29 GB" | 20260506 16:36:19.478 | INFO | } | 20260506 16:36:19.478 | INFO | Perf report for the last 5.1429 seconds: | 20260506 16:36:19.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:36:19.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:19.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:19.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:19.479 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:36:19.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:19.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:19.479 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:36:19.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:19.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:19.479 | INFO | Uploaded 36GiB to GPU in 1.99011s (18.0895GiB/s) | 20260506 16:36:19.479 | INFO | Downloaded 34GiB to GPU in 2.01542s (16.87GiB/s) | 20260506 16:36:24.620 | INFO | Analyzing chunk frames = [2838016,2838528) | 20260506 16:36:24.620 | INFO | MemInfo: { | 20260506 16:36:24.620 | INFO | "gpu" : " 17 GB", | 20260506 16:36:24.621 | INFO | "huge" : "", | 20260506 16:36:24.621 | INFO | "huge_shared" : " 27 GB", | 20260506 16:36:24.621 | INFO | "malloc" : " 14 GB", | 20260506 16:36:24.621 | INFO | "pinned" : " 29 GB" | 20260506 16:36:24.621 | INFO | } | 20260506 16:36:24.621 | INFO | Perf report for the last 5.14253 seconds: | 20260506 16:36:24.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:36:24.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:24.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:24.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:24.621 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:36:24.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:24.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:24.621 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:36:24.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:24.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:24.621 | INFO | Uploaded 36GiB to GPU in 2.00261s (17.9765GiB/s) | 20260506 16:36:24.621 | INFO | Downloaded 34GiB to GPU in 1.96967s (17.2617GiB/s) | 20260506 16:36:24.741 | INFO | Starting a round of Baz IO | 20260506 16:36:27.876 | INFO | Populating RTMetrics report | 20260506 16:36:27.876 | INFO | RTMetrics report populated | 20260506 16:36:27.876 | INFO | Opening tmp RTMetrics json file | 20260506 16:36:27.877 | INFO | Serializing RTMetrics to json | 20260506 16:36:27.880 | INFO | Serialization done, swapping tmp file now... | 20260506 16:36:27.881 | INFO | RTMetrics done | 20260506 16:36:28.035 | INFO | Finished a round of Baz IO: Wrote 2.082 GiB in 3.294 seconds | 20260506 16:36:28.035 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.22% / 0.01% / 63.77% | 20260506 16:36:28.035 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.62942% / 3.97364e-06% / 5.5631e-05% | 20260506 16:36:28.035 | INFO | Baz IO overall: | 20260506 16:36:28.035 | INFO | BytesWritten : 1491259525479 | 20260506 16:36:28.035 | INFO | NumEvents : 721597613667 | 20260506 16:36:29.761 | INFO | Analyzing chunk frames = [2838528,2839040) | 20260506 16:36:29.761 | INFO | MemInfo: { | 20260506 16:36:29.761 | INFO | "gpu" : " 17 GB", | 20260506 16:36:29.761 | INFO | "huge" : "", | 20260506 16:36:29.761 | INFO | "huge_shared" : " 27 GB", | 20260506 16:36:29.761 | INFO | "malloc" : " 14 GB", | 20260506 16:36:29.761 | INFO | "pinned" : " 29 GB" | 20260506 16:36:29.761 | INFO | } | 20260506 16:36:29.761 | INFO | Perf report for the last 5.13984 seconds: | 20260506 16:36:29.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:36:29.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:29.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:29.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:29.761 | INFO | ANALYSIS : 62.7%, 2.9, 95.4ms, 0 | 20260506 16:36:29.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:29.761 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 16:36:29.761 | INFO | PRE_HQ : 60.2%, 1.9, 58.5ms, 0 | 20260506 16:36:29.761 | INFO | BAZWRITER : 63.7%, 4.5, 148.2ms, 0 | 20260506 16:36:29.761 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.2ms, 0 | 20260506 16:36:29.761 | INFO | Uploaded 36GiB to GPU in 2.05003s (17.5607GiB/s) | 20260506 16:36:29.761 | INFO | Downloaded 38GiB to GPU in 2.21728s (17.1382GiB/s) | 20260506 16:36:34.898 | INFO | Analyzing chunk frames = [2839040,2839552) | 20260506 16:36:34.898 | INFO | MemInfo: { | 20260506 16:36:34.898 | INFO | "gpu" : " 17 GB", | 20260506 16:36:34.898 | INFO | "huge" : "", | 20260506 16:36:34.898 | INFO | "huge_shared" : " 27 GB", | 20260506 16:36:34.898 | INFO | "malloc" : " 14 GB", | 20260506 16:36:34.898 | INFO | "pinned" : " 29 GB" | 20260506 16:36:34.898 | INFO | } | 20260506 16:36:34.898 | INFO | Perf report for the last 5.13721 seconds: | 20260506 16:36:34.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:36:34.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:34.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:34.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:34.898 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 16:36:34.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:34.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:34.898 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:36:34.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:34.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:34.898 | INFO | Uploaded 36GiB to GPU in 1.98717s (18.1162GiB/s) | 20260506 16:36:34.899 | INFO | Downloaded 34GiB to GPU in 1.94521s (17.4789GiB/s) | 20260506 16:36:40.034 | INFO | Analyzing chunk frames = [2839552,2840064) | 20260506 16:36:40.034 | INFO | MemInfo: { | 20260506 16:36:40.034 | INFO | "gpu" : " 17 GB", | 20260506 16:36:40.034 | INFO | "huge" : "", | 20260506 16:36:40.034 | INFO | "huge_shared" : " 27 GB", | 20260506 16:36:40.034 | INFO | "malloc" : " 14 GB", | 20260506 16:36:40.034 | INFO | "pinned" : " 29 GB" | 20260506 16:36:40.034 | INFO | } | 20260506 16:36:40.034 | INFO | Perf report for the last 5.13609 seconds: | 20260506 16:36:40.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:36:40.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:40.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:40.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:40.034 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 16:36:40.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:40.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:40.034 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:36:40.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:40.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:40.034 | INFO | Uploaded 36GiB to GPU in 1.96119s (18.3562GiB/s) | 20260506 16:36:40.035 | INFO | Downloaded 34GiB to GPU in 1.92984s (17.6181GiB/s) | 20260506 16:36:45.171 | INFO | Analyzing chunk frames = [2840064,2840576) | 20260506 16:36:45.171 | INFO | MemInfo: { | 20260506 16:36:45.171 | INFO | "gpu" : " 17 GB", | 20260506 16:36:45.171 | INFO | "huge" : "", | 20260506 16:36:45.171 | INFO | "huge_shared" : " 27 GB", | 20260506 16:36:45.171 | INFO | "malloc" : " 14 GB", | 20260506 16:36:45.171 | INFO | "pinned" : " 29 GB" | 20260506 16:36:45.171 | INFO | } | 20260506 16:36:45.171 | INFO | Perf report for the last 5.13665 seconds: | 20260506 16:36:45.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:36:45.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:45.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:45.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:45.171 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:36:45.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:45.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:45.171 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:36:45.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:45.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:45.171 | INFO | Uploaded 36GiB to GPU in 1.92727s (18.6793GiB/s) | 20260506 16:36:45.171 | INFO | Downloaded 34GiB to GPU in 1.87936s (18.0912GiB/s) | 20260506 16:36:50.310 | INFO | Analyzing chunk frames = [2840576,2841088) | 20260506 16:36:50.310 | INFO | MemInfo: { | 20260506 16:36:50.310 | INFO | "gpu" : " 17 GB", | 20260506 16:36:50.310 | INFO | "huge" : "", | 20260506 16:36:50.310 | INFO | "huge_shared" : " 27 GB", | 20260506 16:36:50.310 | INFO | "malloc" : " 14 GB", | 20260506 16:36:50.310 | INFO | "pinned" : " 29 GB" | 20260506 16:36:50.310 | INFO | } | 20260506 16:36:50.311 | INFO | Perf report for the last 5.13965 seconds: | 20260506 16:36:50.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:36:50.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:50.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:50.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:50.311 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 16:36:50.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:50.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:50.311 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:36:50.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:50.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:50.311 | INFO | Uploaded 36GiB to GPU in 1.95008s (18.4608GiB/s) | 20260506 16:36:50.311 | INFO | Downloaded 34GiB to GPU in 1.88598s (18.0277GiB/s) | 20260506 16:36:55.451 | INFO | Analyzing chunk frames = [2841088,2841600) | 20260506 16:36:55.451 | INFO | MemInfo: { | 20260506 16:36:55.451 | INFO | "gpu" : " 17 GB", | 20260506 16:36:55.451 | INFO | "huge" : "", | 20260506 16:36:55.451 | INFO | "huge_shared" : " 27 GB", | 20260506 16:36:55.451 | INFO | "malloc" : " 14 GB", | 20260506 16:36:55.451 | INFO | "pinned" : " 29 GB" | 20260506 16:36:55.451 | INFO | } | 20260506 16:36:55.451 | INFO | Perf report for the last 5.14086 seconds: | 20260506 16:36:55.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:36:55.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:55.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:55.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:55.451 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 16:36:55.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:55.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:55.452 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:36:55.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:55.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:36:55.452 | INFO | Uploaded 36GiB to GPU in 2.02193s (17.8047GiB/s) | 20260506 16:36:55.452 | INFO | Downloaded 34GiB to GPU in 1.96171s (17.3318GiB/s) | 20260506 16:37:00.594 | INFO | Analyzing chunk frames = [2841600,2842112) | 20260506 16:37:00.594 | INFO | MemInfo: { | 20260506 16:37:00.594 | INFO | "gpu" : " 17 GB", | 20260506 16:37:00.594 | INFO | "huge" : "", | 20260506 16:37:00.594 | INFO | "huge_shared" : " 27 GB", | 20260506 16:37:00.594 | INFO | "malloc" : " 14 GB", | 20260506 16:37:00.594 | INFO | "pinned" : " 29 GB" | 20260506 16:37:00.594 | INFO | } | 20260506 16:37:00.594 | INFO | Perf report for the last 5.14278 seconds: | 20260506 16:37:00.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:37:00.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:00.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:00.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:00.594 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:37:00.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:00.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:00.594 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 16:37:00.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:00.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:00.595 | INFO | Uploaded 36GiB to GPU in 1.99405s (18.0537GiB/s) | 20260506 16:37:00.595 | INFO | Downloaded 34GiB to GPU in 2.02s (16.8317GiB/s) | 20260506 16:37:05.736 | INFO | Analyzing chunk frames = [2842112,2842624) | 20260506 16:37:05.736 | INFO | MemInfo: { | 20260506 16:37:05.736 | INFO | "gpu" : " 17 GB", | 20260506 16:37:05.736 | INFO | "huge" : "", | 20260506 16:37:05.736 | INFO | "huge_shared" : " 27 GB", | 20260506 16:37:05.736 | INFO | "malloc" : " 14 GB", | 20260506 16:37:05.736 | INFO | "pinned" : " 29 GB" | 20260506 16:37:05.736 | INFO | } | 20260506 16:37:05.736 | INFO | Perf report for the last 5.14163 seconds: | 20260506 16:37:05.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:37:05.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:05.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:05.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:05.736 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 16:37:05.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:05.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:05.736 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 16:37:05.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:05.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:05.736 | INFO | Uploaded 36GiB to GPU in 2.04362s (17.6158GiB/s) | 20260506 16:37:05.736 | INFO | Downloaded 34GiB to GPU in 1.97907s (17.1798GiB/s) | 20260506 16:37:05.736 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:37:05.736 | INFO | Total mem usage: 18 GiB | 20260506 16:37:05.736 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:37:05.736 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -13.7%) | 20260506 16:37:05.737 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:37:05.737 | INFO | High Water Breakdown: | 20260506 16:37:05.737 | INFO | 61.5% : 9.74 GiB ( 85.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:37:05.737 | INFO | 38.3% : 6.07 GiB ( 90.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:37:05.737 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:37:05.737 | INFO | ------------------------------------------ | 20260506 16:37:05.737 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:37:05.737 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:37:05.737 | INFO | Total mem usage: 26.5 GiB | 20260506 16:37:05.737 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:37:05.737 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:37:05.737 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:37:05.737 | INFO | High Water Breakdown: | 20260506 16:37:05.737 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:37:05.737 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:37:05.737 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:37:05.737 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:37:05.737 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:37:05.737 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:37:05.737 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:37:05.737 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:37:05.737 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:37:05.737 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:37:05.737 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:37:05.737 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:37:05.737 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:37:05.737 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:37:05.737 | INFO | ------------------------------------------ | 20260506 16:37:05.737 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:37:05.737 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:37:05.737 | INFO | Total mem usage: 25.6 GiB | 20260506 16:37:05.737 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:37:05.737 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:37:05.737 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:37:05.737 | INFO | High Water Breakdown: | 20260506 16:37:05.737 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:37:05.737 | INFO | ------------------------------------------ | 20260506 16:37:05.737 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:37:05.737 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:37:05.737 | INFO | Total mem usage: 15.7 GiB | 20260506 16:37:05.737 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:37:05.737 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:37:05.737 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 16:37:05.737 | INFO | High Water Breakdown: | 20260506 16:37:05.737 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:37:05.737 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:37:05.737 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:37:05.737 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:37:05.737 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:37:05.737 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:37:05.737 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:37:05.737 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:37:05.737 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:37:05.737 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:37:05.737 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:37:05.737 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:37:05.737 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:37:05.737 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:37:05.737 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:37:05.737 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:37:05.737 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:37:05.737 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:37:05.737 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:37:05.737 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:37:05.737 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:37:05.737 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:37:05.737 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:37:05.737 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:37:05.737 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:37:05.737 | INFO | ------------------------------------------ | 20260506 16:37:05.737 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:37:05.737 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:37:05.737 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:37:05.737 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:37:05.737 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.52e+01, 1.32e+01 / 2.28e+00 | 20260506 16:37:05.738 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 9.93e-03 / 5.85e-03 | 20260506 16:37:05.738 | INFO | PerfReport: 11.59%, Baseline -- 3.63e+00 / 4.36e+00, 5.59e-01 / 3.92e-01 | 20260506 16:37:05.738 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.84e-01 / 1.31e-01 | 20260506 16:37:05.738 | INFO | PerfReport: 15.90%, DME -- 3.99e+01 / 4.41e+01, 3.96e+00 / 2.30e+00 | 20260506 16:37:05.738 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.41e+00 / 2.00e+00 | 20260506 16:37:05.738 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.30e+00 / 2.89e+00, 3.91e-01 / 2.29e-01 | 20260506 16:37:05.738 | INFO | PerfReport: 3.49%, Metrics -- 1.09e+00 / 1.46e+00, 2.83e-01 / 1.89e-01 | 20260506 16:37:05.738 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.79e-01 / 4.26e-01, 5.92e-02 / 2.29e-02 | 20260506 16:37:05.738 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:37:05.738 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:37:05.738 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:37:05.738 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.28e+01 / 0.00e+00 | 20260506 16:37:05.738 | INFO | PerfReport: 21.28%, Upload -- 2.07e+01 / 0.00e+00, 6.76e+00 / 0.00e+00 | 20260506 16:37:05.738 | INFO | PerfReport: 21.18%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 16:37:05.738 | INFO | PerfReport: 32.28%, Compute -- 3.14e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 16:37:05.738 | INFO | PerfReport: 1.26%, LockUpload -- 1.22e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260506 16:37:05.738 | INFO | PerfReport: 1.23%, LockDownload -- 1.19e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260506 16:37:05.738 | INFO | PerfReport: 22.76%, LockCompute -- 2.21e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 16:37:05.738 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260506 16:37:05.846 | INFO | Starting a round of Baz IO | 20260506 16:37:09.028 | INFO | Populating RTMetrics report | 20260506 16:37:09.028 | INFO | RTMetrics report populated | 20260506 16:37:09.028 | INFO | Opening tmp RTMetrics json file | 20260506 16:37:09.208 | INFO | Serializing RTMetrics to json | 20260506 16:37:09.218 | INFO | Serialization done, swapping tmp file now... | 20260506 16:37:09.220 | INFO | RTMetrics done | 20260506 16:37:09.237 | INFO | Finished a round of Baz IO: Wrote 2.081 GiB in 3.39 seconds | 20260506 16:37:09.237 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.21% / 0.01% / 63.78% | 20260506 16:37:09.237 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.62337% / 3.97364e-06% / 6.75519e-05% | 20260506 16:37:09.237 | INFO | Baz IO overall: | 20260506 16:37:09.237 | INFO | BytesWritten : 1493493833581 | 20260506 16:37:09.237 | INFO | NumEvents : 722676791966 | 20260506 16:37:10.872 | INFO | Analyzing chunk frames = [2842624,2843136) | 20260506 16:37:10.872 | INFO | MemInfo: { | 20260506 16:37:10.872 | INFO | "gpu" : " 17 GB", | 20260506 16:37:10.872 | INFO | "huge" : "", | 20260506 16:37:10.872 | INFO | "huge_shared" : " 27 GB", | 20260506 16:37:10.872 | INFO | "malloc" : " 14 GB", | 20260506 16:37:10.872 | INFO | "pinned" : " 29 GB" | 20260506 16:37:10.872 | INFO | } | 20260506 16:37:10.872 | INFO | Perf report for the last 5.13621 seconds: | 20260506 16:37:10.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:37:10.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:10.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:10.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:10.872 | INFO | ANALYSIS : 62.9%, 2.9, 95.2ms, 0 | 20260506 16:37:10.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:10.872 | INFO | RT_METRICS : 48.3%, 1.0, 25.3ms, 0 | 20260506 16:37:10.872 | INFO | PRE_HQ : 61.0%, 1.9, 60.0ms, 0 | 20260506 16:37:10.872 | INFO | BAZWRITER : 65.4%, 4.8, 161.1ms, 0 | 20260506 16:37:10.872 | INFO | BAZ_SAMPLING : 3.7%, 1.0, 1.9ms, 0 | 20260506 16:37:10.872 | INFO | Uploaded 36GiB to GPU in 2.03867s (17.6585GiB/s) | 20260506 16:37:10.872 | INFO | Downloaded 38GiB to GPU in 2.22518s (17.0773GiB/s) | 20260506 16:37:16.009 | INFO | Analyzing chunk frames = [2843136,2843648) | 20260506 16:37:16.009 | INFO | MemInfo: { | 20260506 16:37:16.009 | INFO | "gpu" : " 17 GB", | 20260506 16:37:16.009 | INFO | "huge" : "", | 20260506 16:37:16.009 | INFO | "huge_shared" : " 27 GB", | 20260506 16:37:16.009 | INFO | "malloc" : " 14 GB", | 20260506 16:37:16.009 | INFO | "pinned" : " 29 GB" | 20260506 16:37:16.010 | INFO | } | 20260506 16:37:16.010 | INFO | Perf report for the last 5.1375 seconds: | 20260506 16:37:16.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:37:16.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:16.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:16.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:16.010 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:37:16.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:16.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:16.010 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:37:16.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:16.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:16.010 | INFO | Uploaded 36GiB to GPU in 1.98635s (18.1237GiB/s) | 20260506 16:37:16.010 | INFO | Downloaded 34GiB to GPU in 1.93993s (17.5264GiB/s) | 20260506 16:37:21.148 | INFO | Analyzing chunk frames = [2843648,2844160) | 20260506 16:37:21.148 | INFO | MemInfo: { | 20260506 16:37:21.148 | INFO | "gpu" : " 17 GB", | 20260506 16:37:21.148 | INFO | "huge" : "", | 20260506 16:37:21.148 | INFO | "huge_shared" : " 27 GB", | 20260506 16:37:21.148 | INFO | "malloc" : " 14 GB", | 20260506 16:37:21.148 | INFO | "pinned" : " 29 GB" | 20260506 16:37:21.148 | INFO | } | 20260506 16:37:21.148 | INFO | Perf report for the last 5.13864 seconds: | 20260506 16:37:21.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:37:21.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:21.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:21.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:21.148 | INFO | ANALYSIS : 60.2%, 3.0, 93.5ms, 0 | 20260506 16:37:21.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:21.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:21.148 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:37:21.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:21.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:21.149 | INFO | Uploaded 36GiB to GPU in 1.91703s (18.7791GiB/s) | 20260506 16:37:21.149 | INFO | Downloaded 34GiB to GPU in 1.89248s (17.9659GiB/s) | 20260506 16:37:26.288 | INFO | Analyzing chunk frames = [2844160,2844672) | 20260506 16:37:26.288 | INFO | MemInfo: { | 20260506 16:37:26.288 | INFO | "gpu" : " 17 GB", | 20260506 16:37:26.288 | INFO | "huge" : "", | 20260506 16:37:26.288 | INFO | "huge_shared" : " 27 GB", | 20260506 16:37:26.288 | INFO | "malloc" : " 14 GB", | 20260506 16:37:26.288 | INFO | "pinned" : " 29 GB" | 20260506 16:37:26.288 | INFO | } | 20260506 16:37:26.288 | INFO | Perf report for the last 5.13986 seconds: | 20260506 16:37:26.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:37:26.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:26.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:26.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:26.288 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 16:37:26.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:26.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:26.288 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:37:26.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:26.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:26.288 | INFO | Uploaded 36GiB to GPU in 1.91918s (18.758GiB/s) | 20260506 16:37:26.289 | INFO | Downloaded 34GiB to GPU in 1.87745s (18.1097GiB/s) | 20260506 16:37:31.427 | INFO | Analyzing chunk frames = [2844672,2845184) | 20260506 16:37:31.427 | INFO | MemInfo: { | 20260506 16:37:31.427 | INFO | "gpu" : " 17 GB", | 20260506 16:37:31.427 | INFO | "huge" : "", | 20260506 16:37:31.427 | INFO | "huge_shared" : " 27 GB", | 20260506 16:37:31.427 | INFO | "malloc" : " 14 GB", | 20260506 16:37:31.427 | INFO | "pinned" : " 29 GB" | 20260506 16:37:31.427 | INFO | } | 20260506 16:37:31.427 | INFO | Perf report for the last 5.13932 seconds: | 20260506 16:37:31.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:37:31.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:31.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:31.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:31.427 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 16:37:31.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:31.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:31.427 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:37:31.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:31.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:31.428 | INFO | Uploaded 36GiB to GPU in 1.96335s (18.336GiB/s) | 20260506 16:37:31.428 | INFO | Downloaded 34GiB to GPU in 1.88261s (18.06GiB/s) | 20260506 16:37:36.571 | INFO | Analyzing chunk frames = [2845184,2845696) | 20260506 16:37:36.571 | INFO | MemInfo: { | 20260506 16:37:36.571 | INFO | "gpu" : " 17 GB", | 20260506 16:37:36.571 | INFO | "huge" : "", | 20260506 16:37:36.572 | INFO | "huge_shared" : " 27 GB", | 20260506 16:37:36.572 | INFO | "malloc" : " 14 GB", | 20260506 16:37:36.572 | INFO | "pinned" : " 29 GB" | 20260506 16:37:36.572 | INFO | } | 20260506 16:37:36.572 | INFO | Perf report for the last 5.14481 seconds: | 20260506 16:37:36.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:37:36.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:36.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:36.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:36.572 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 16:37:36.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:36.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:36.572 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:37:36.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:36.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:36.572 | INFO | Uploaded 36GiB to GPU in 2.00062s (17.9944GiB/s) | 20260506 16:37:36.573 | INFO | Downloaded 34GiB to GPU in 1.93631s (17.5591GiB/s) | 20260506 16:37:41.711 | INFO | Analyzing chunk frames = [2845696,2846208) | 20260506 16:37:41.712 | INFO | MemInfo: { | 20260506 16:37:41.712 | INFO | "gpu" : " 17 GB", | 20260506 16:37:41.712 | INFO | "huge" : "", | 20260506 16:37:41.712 | INFO | "huge_shared" : " 27 GB", | 20260506 16:37:41.712 | INFO | "malloc" : " 14 GB", | 20260506 16:37:41.712 | INFO | "pinned" : " 29 GB" | 20260506 16:37:41.712 | INFO | } | 20260506 16:37:41.712 | INFO | Perf report for the last 5.13952 seconds: | 20260506 16:37:41.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:37:41.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:41.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:41.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:41.712 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:37:41.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:41.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:41.712 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:37:41.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:41.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:41.712 | INFO | Uploaded 36GiB to GPU in 1.99566s (18.0391GiB/s) | 20260506 16:37:41.712 | INFO | Downloaded 34GiB to GPU in 2.01688s (16.8577GiB/s) | 20260506 16:37:46.849 | INFO | Analyzing chunk frames = [2846208,2846720) | 20260506 16:37:46.849 | INFO | MemInfo: { | 20260506 16:37:46.849 | INFO | "gpu" : " 17 GB", | 20260506 16:37:46.849 | INFO | "huge" : "", | 20260506 16:37:46.849 | INFO | "huge_shared" : " 27 GB", | 20260506 16:37:46.849 | INFO | "malloc" : " 14 GB", | 20260506 16:37:46.849 | INFO | "pinned" : " 29 GB" | 20260506 16:37:46.849 | INFO | } | 20260506 16:37:46.849 | INFO | Perf report for the last 5.13736 seconds: | 20260506 16:37:46.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:37:46.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:46.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:46.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:46.849 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:37:46.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:46.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:46.849 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:37:46.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:46.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:46.849 | INFO | Uploaded 36GiB to GPU in 2.02238s (17.8008GiB/s) | 20260506 16:37:46.849 | INFO | Downloaded 34GiB to GPU in 1.98396s (17.1374GiB/s) | 20260506 16:37:46.969 | INFO | Starting a round of Baz IO | 20260506 16:37:50.101 | INFO | Populating RTMetrics report | 20260506 16:37:50.102 | INFO | RTMetrics report populated | 20260506 16:37:50.102 | INFO | Opening tmp RTMetrics json file | 20260506 16:37:50.110 | INFO | Serializing RTMetrics to json | 20260506 16:37:50.161 | INFO | Serialization done, swapping tmp file now... | 20260506 16:37:50.203 | INFO | RTMetrics done | 20260506 16:37:50.520 | INFO | Finished a round of Baz IO: Wrote 2.082 GiB in 3.551 seconds | 20260506 16:37:50.520 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.20% / 0.01% / 63.79% | 20260506 16:37:50.520 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.62263% / 7.94729e-06% / 5.96046e-05% | 20260506 16:37:50.521 | INFO | Baz IO overall: | 20260506 16:37:50.521 | INFO | BytesWritten : 1495729152717 | 20260506 16:37:50.521 | INFO | NumEvents : 723756796030 | 20260506 16:37:51.989 | INFO | Analyzing chunk frames = [2846720,2847232) | 20260506 16:37:51.989 | INFO | MemInfo: { | 20260506 16:37:51.989 | INFO | "gpu" : " 17 GB", | 20260506 16:37:51.989 | INFO | "huge" : "", | 20260506 16:37:51.989 | INFO | "huge_shared" : " 27 GB", | 20260506 16:37:51.989 | INFO | "malloc" : " 14 GB", | 20260506 16:37:51.989 | INFO | "pinned" : " 29 GB" | 20260506 16:37:51.989 | INFO | } | 20260506 16:37:51.989 | INFO | Perf report for the last 5.13997 seconds: | 20260506 16:37:51.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:37:51.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:51.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:51.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:51.989 | INFO | ANALYSIS : 62.7%, 2.9, 95.2ms, 0 | 20260506 16:37:51.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:51.989 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 16:37:51.989 | INFO | PRE_HQ : 60.2%, 1.9, 58.6ms, 0 | 20260506 16:37:51.989 | INFO | BAZWRITER : 68.7%, 4.8, 169.6ms, 0 | 20260506 16:37:51.989 | INFO | BAZ_SAMPLING : 3.3%, 1.0, 1.7ms, 0 | 20260506 16:37:51.989 | INFO | Uploaded 36GiB to GPU in 2.06637s (17.4219GiB/s) | 20260506 16:37:51.989 | INFO | Downloaded 38GiB to GPU in 2.22173s (17.1038GiB/s) | 20260506 16:37:57.120 | INFO | Analyzing chunk frames = [2847232,2847744) | 20260506 16:37:57.120 | INFO | MemInfo: { | 20260506 16:37:57.120 | INFO | "gpu" : " 17 GB", | 20260506 16:37:57.120 | INFO | "huge" : "", | 20260506 16:37:57.120 | INFO | "huge_shared" : " 27 GB", | 20260506 16:37:57.121 | INFO | "malloc" : " 14 GB", | 20260506 16:37:57.121 | INFO | "pinned" : " 29 GB" | 20260506 16:37:57.121 | INFO | } | 20260506 16:37:57.121 | INFO | Perf report for the last 5.13157 seconds: | 20260506 16:37:57.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:37:57.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:57.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:57.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:57.121 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 16:37:57.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:57.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:57.121 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 16:37:57.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:57.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:37:57.121 | INFO | Uploaded 36GiB to GPU in 2.0055s (17.9506GiB/s) | 20260506 16:37:57.121 | INFO | Downloaded 34GiB to GPU in 1.93861s (17.5383GiB/s) | 20260506 16:38:02.258 | INFO | Analyzing chunk frames = [2847744,2848256) | 20260506 16:38:02.258 | INFO | MemInfo: { | 20260506 16:38:02.258 | INFO | "gpu" : " 17 GB", | 20260506 16:38:02.258 | INFO | "huge" : "", | 20260506 16:38:02.258 | INFO | "huge_shared" : " 27 GB", | 20260506 16:38:02.258 | INFO | "malloc" : " 14 GB", | 20260506 16:38:02.258 | INFO | "pinned" : " 29 GB" | 20260506 16:38:02.258 | INFO | } | 20260506 16:38:02.258 | INFO | Perf report for the last 5.13745 seconds: | 20260506 16:38:02.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:38:02.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:02.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:02.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:02.258 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 16:38:02.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:02.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:02.258 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:38:02.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:02.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:02.258 | INFO | Uploaded 36GiB to GPU in 1.96397s (18.3302GiB/s) | 20260506 16:38:02.259 | INFO | Downloaded 34GiB to GPU in 1.93339s (17.5857GiB/s) | 20260506 16:38:07.405 | INFO | Analyzing chunk frames = [2848256,2848768) | 20260506 16:38:07.405 | INFO | MemInfo: { | 20260506 16:38:07.405 | INFO | "gpu" : " 17 GB", | 20260506 16:38:07.405 | INFO | "huge" : "", | 20260506 16:38:07.405 | INFO | "huge_shared" : " 27 GB", | 20260506 16:38:07.405 | INFO | "malloc" : " 14 GB", | 20260506 16:38:07.405 | INFO | "pinned" : " 29 GB" | 20260506 16:38:07.405 | INFO | } | 20260506 16:38:07.405 | INFO | Perf report for the last 5.14661 seconds: | 20260506 16:38:07.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:38:07.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:07.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:07.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:07.405 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 16:38:07.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:07.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:07.405 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:38:07.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:07.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:07.405 | INFO | Uploaded 36GiB to GPU in 1.93604s (18.5947GiB/s) | 20260506 16:38:07.405 | INFO | Downloaded 34GiB to GPU in 1.87932s (18.0916GiB/s) | 20260506 16:38:12.546 | INFO | Analyzing chunk frames = [2848768,2849280) | 20260506 16:38:12.546 | INFO | MemInfo: { | 20260506 16:38:12.546 | INFO | "gpu" : " 17 GB", | 20260506 16:38:12.546 | INFO | "huge" : "", | 20260506 16:38:12.546 | INFO | "huge_shared" : " 27 GB", | 20260506 16:38:12.546 | INFO | "malloc" : " 14 GB", | 20260506 16:38:12.546 | INFO | "pinned" : " 29 GB" | 20260506 16:38:12.546 | INFO | } | 20260506 16:38:12.546 | INFO | Perf report for the last 5.14101 seconds: | 20260506 16:38:12.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:38:12.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:12.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:12.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:12.546 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 16:38:12.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:12.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:12.546 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:38:12.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:12.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:12.547 | INFO | Uploaded 36GiB to GPU in 1.98534s (18.1329GiB/s) | 20260506 16:38:12.547 | INFO | Downloaded 34GiB to GPU in 1.92278s (17.6827GiB/s) | 20260506 16:38:17.686 | INFO | Analyzing chunk frames = [2849280,2849792) | 20260506 16:38:17.686 | INFO | MemInfo: { | 20260506 16:38:17.686 | INFO | "gpu" : " 17 GB", | 20260506 16:38:17.686 | INFO | "huge" : "", | 20260506 16:38:17.686 | INFO | "huge_shared" : " 27 GB", | 20260506 16:38:17.686 | INFO | "malloc" : " 14 GB", | 20260506 16:38:17.686 | INFO | "pinned" : " 29 GB" | 20260506 16:38:17.686 | INFO | } | 20260506 16:38:17.686 | INFO | Perf report for the last 5.14045 seconds: | 20260506 16:38:17.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:38:17.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:17.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:17.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:17.686 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 16:38:17.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:17.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:17.687 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:38:17.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:17.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:17.687 | INFO | Uploaded 36GiB to GPU in 2.02416s (17.7852GiB/s) | 20260506 16:38:17.687 | INFO | Downloaded 34GiB to GPU in 1.9411s (17.5158GiB/s) | 20260506 16:38:22.827 | INFO | Analyzing chunk frames = [2849792,2850304) | 20260506 16:38:22.827 | INFO | MemInfo: { | 20260506 16:38:22.827 | INFO | "gpu" : " 17 GB", | 20260506 16:38:22.827 | INFO | "huge" : "", | 20260506 16:38:22.827 | INFO | "huge_shared" : " 27 GB", | 20260506 16:38:22.827 | INFO | "malloc" : " 14 GB", | 20260506 16:38:22.827 | INFO | "pinned" : " 29 GB" | 20260506 16:38:22.827 | INFO | } | 20260506 16:38:22.827 | INFO | Perf report for the last 5.14093 seconds: | 20260506 16:38:22.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:38:22.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:22.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:22.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:22.827 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:38:22.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:22.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:22.827 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:38:22.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:22.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:22.828 | INFO | Uploaded 36GiB to GPU in 2.01497s (17.8662GiB/s) | 20260506 16:38:22.828 | INFO | Downloaded 34GiB to GPU in 1.99823s (17.0151GiB/s) | 20260506 16:38:27.962 | INFO | Analyzing chunk frames = [2850304,2850816) | 20260506 16:38:27.962 | INFO | MemInfo: { | 20260506 16:38:27.962 | INFO | "gpu" : " 17 GB", | 20260506 16:38:27.962 | INFO | "huge" : "", | 20260506 16:38:27.962 | INFO | "huge_shared" : " 27 GB", | 20260506 16:38:27.962 | INFO | "malloc" : " 14 GB", | 20260506 16:38:27.962 | INFO | "pinned" : " 29 GB" | 20260506 16:38:27.962 | INFO | } | 20260506 16:38:27.962 | INFO | Perf report for the last 5.13501 seconds: | 20260506 16:38:27.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:38:27.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:27.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:27.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:27.962 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 16:38:27.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:27.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:27.962 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:38:27.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:27.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:27.962 | INFO | Uploaded 36GiB to GPU in 1.99692s (18.0277GiB/s) | 20260506 16:38:27.963 | INFO | Downloaded 34GiB to GPU in 1.97878s (17.1823GiB/s) | 20260506 16:38:27.963 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:38:27.963 | INFO | Total mem usage: 18 GiB | 20260506 16:38:27.963 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:38:27.963 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13%) | 20260506 16:38:27.963 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:38:27.963 | INFO | High Water Breakdown: | 20260506 16:38:27.963 | INFO | 61.6% : 9.82 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:38:27.963 | INFO | 38.2% : 6.09 GiB ( 91% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:38:27.963 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:38:27.963 | INFO | ------------------------------------------ | 20260506 16:38:27.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:38:27.963 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:38:27.963 | INFO | Total mem usage: 26.5 GiB | 20260506 16:38:27.963 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:38:27.963 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:38:27.963 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:38:27.963 | INFO | High Water Breakdown: | 20260506 16:38:27.963 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:38:27.963 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:38:27.963 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:38:27.963 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:38:27.963 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:38:27.963 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:38:27.963 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:38:27.963 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:38:27.963 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:38:27.963 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:38:27.963 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:38:27.963 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:38:27.963 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:38:27.963 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:38:27.963 | INFO | ------------------------------------------ | 20260506 16:38:27.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:38:27.963 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:38:27.963 | INFO | Total mem usage: 25.6 GiB | 20260506 16:38:27.963 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:38:27.963 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:38:27.963 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:38:27.963 | INFO | High Water Breakdown: | 20260506 16:38:27.963 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:38:27.963 | INFO | ------------------------------------------ | 20260506 16:38:27.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:38:27.963 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:38:27.963 | INFO | Total mem usage: 15.7 GiB | 20260506 16:38:27.963 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:38:27.963 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:38:27.963 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 16:38:27.963 | INFO | High Water Breakdown: | 20260506 16:38:27.963 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:38:27.963 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:38:27.963 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:38:27.963 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:38:27.963 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:38:27.963 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:38:27.963 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:38:27.963 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:38:27.963 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:38:27.963 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:38:27.963 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:38:27.963 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:38:27.963 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:38:27.963 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:38:27.963 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:38:27.963 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:38:27.963 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:38:27.963 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:38:27.963 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:38:27.963 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:38:27.963 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:38:27.963 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:38:27.963 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:38:27.963 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:38:27.963 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:38:27.963 | INFO | ------------------------------------------ | 20260506 16:38:27.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:38:27.964 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:38:27.964 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:38:27.964 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:38:27.964 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.58e+01, 1.35e+01 / 2.56e+00 | 20260506 16:38:27.964 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.00e-01, 1.02e-02 / 7.17e-03 | 20260506 16:38:27.964 | INFO | PerfReport: 11.60%, Baseline -- 3.64e+00 / 4.41e+00, 5.62e-01 / 3.00e-01 | 20260506 16:38:27.964 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.44e+00, 1.84e-01 / 1.37e-01 | 20260506 16:38:27.964 | INFO | PerfReport: 15.95%, DME -- 4.00e+01 / 4.52e+01, 4.21e+00 / 2.76e+00 | 20260506 16:38:27.964 | INFO | PerfReport: 56.89%, FrameLabeling -- 1.78e+01 / 1.99e+01, 1.41e+00 / 1.76e+00 | 20260506 16:38:27.964 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.30e+00 / 2.90e+00, 3.93e-01 / 2.39e-01 | 20260506 16:38:27.964 | INFO | PerfReport: 3.48%, Metrics -- 1.09e+00 / 1.49e+00, 2.81e-01 / 1.15e-01 | 20260506 16:38:27.964 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.79e-01 / 4.33e-01, 5.90e-02 / 1.73e-02 | 20260506 16:38:27.964 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:38:27.964 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:38:27.964 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:38:27.964 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 16:38:27.964 | INFO | PerfReport: 21.34%, Upload -- 2.07e+01 / 0.00e+00, 6.85e+00 / 0.00e+00 | 20260506 16:38:27.964 | INFO | PerfReport: 21.16%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 16:38:27.964 | INFO | PerfReport: 32.28%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 16:38:27.964 | INFO | PerfReport: 1.32%, LockUpload -- 1.28e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260506 16:38:27.964 | INFO | PerfReport: 1.18%, LockDownload -- 1.15e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 16:38:27.964 | INFO | PerfReport: 22.71%, LockCompute -- 2.21e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 16:38:27.964 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.12e-02 / 0.00e+00 | 20260506 16:38:28.079 | INFO | Starting a round of Baz IO | 20260506 16:38:31.221 | INFO | Populating RTMetrics report | 20260506 16:38:31.221 | INFO | RTMetrics report populated | 20260506 16:38:31.221 | INFO | Opening tmp RTMetrics json file | 20260506 16:38:31.230 | INFO | Serializing RTMetrics to json | 20260506 16:38:31.315 | INFO | Serialization done, swapping tmp file now... | 20260506 16:38:31.363 | INFO | RTMetrics done | 20260506 16:38:31.401 | INFO | Finished a round of Baz IO: Wrote 2.083 GiB in 3.321 seconds | 20260506 16:38:31.401 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.19% / 0.01% / 63.80% | 20260506 16:38:31.401 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.62253% / 1.98682e-05% / 3.97364e-05% | 20260506 16:38:31.401 | INFO | Baz IO overall: | 20260506 16:38:31.401 | INFO | BytesWritten : 1497966209080 | 20260506 16:38:31.401 | INFO | NumEvents : 724838032938 | 20260506 16:38:33.097 | INFO | Analyzing chunk frames = [2850816,2851328) | 20260506 16:38:33.097 | INFO | MemInfo: { | 20260506 16:38:33.097 | INFO | "gpu" : " 17 GB", | 20260506 16:38:33.097 | INFO | "huge" : "", | 20260506 16:38:33.097 | INFO | "huge_shared" : " 27 GB", | 20260506 16:38:33.097 | INFO | "malloc" : " 14 GB", | 20260506 16:38:33.097 | INFO | "pinned" : " 29 GB" | 20260506 16:38:33.097 | INFO | } | 20260506 16:38:33.097 | INFO | Perf report for the last 5.13521 seconds: | 20260506 16:38:33.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:38:33.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:33.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:33.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:33.098 | INFO | ANALYSIS : 62.9%, 2.9, 95.3ms, 0 | 20260506 16:38:33.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:33.098 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260506 16:38:33.098 | INFO | PRE_HQ : 60.7%, 1.9, 58.7ms, 0 | 20260506 16:38:33.098 | INFO | BAZWRITER : 64.2%, 4.7, 156.4ms, 0 | 20260506 16:38:33.098 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.4ms, 0 | 20260506 16:38:33.098 | INFO | Uploaded 36GiB to GPU in 2.01522s (17.8641GiB/s) | 20260506 16:38:33.098 | INFO | Downloaded 38GiB to GPU in 2.21119s (17.1853GiB/s) | 20260506 16:38:38.238 | INFO | Analyzing chunk frames = [2851328,2851840) | 20260506 16:38:38.238 | INFO | MemInfo: { | 20260506 16:38:38.238 | INFO | "gpu" : " 17 GB", | 20260506 16:38:38.238 | INFO | "huge" : "", | 20260506 16:38:38.238 | INFO | "huge_shared" : " 27 GB", | 20260506 16:38:38.238 | INFO | "malloc" : " 14 GB", | 20260506 16:38:38.238 | INFO | "pinned" : " 29 GB" | 20260506 16:38:38.238 | INFO | } | 20260506 16:38:38.238 | INFO | Perf report for the last 5.14048 seconds: | 20260506 16:38:38.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:38:38.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:38.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:38.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:38.238 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:38:38.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:38.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:38.238 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:38:38.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:38.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:38.238 | INFO | Uploaded 36GiB to GPU in 2.01461s (17.8695GiB/s) | 20260506 16:38:38.238 | INFO | Downloaded 34GiB to GPU in 1.9766s (17.2013GiB/s) | 20260506 16:38:43.378 | INFO | Analyzing chunk frames = [2851840,2852352) | 20260506 16:38:43.378 | INFO | MemInfo: { | 20260506 16:38:43.378 | INFO | "gpu" : " 17 GB", | 20260506 16:38:43.379 | INFO | "huge" : "", | 20260506 16:38:43.379 | INFO | "huge_shared" : " 27 GB", | 20260506 16:38:43.379 | INFO | "malloc" : " 14 GB", | 20260506 16:38:43.379 | INFO | "pinned" : " 29 GB" | 20260506 16:38:43.379 | INFO | } | 20260506 16:38:43.379 | INFO | Perf report for the last 5.1408 seconds: | 20260506 16:38:43.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:38:43.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:43.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:43.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:43.379 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 16:38:43.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:43.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:43.379 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:38:43.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:43.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:43.379 | INFO | Uploaded 36GiB to GPU in 1.96283s (18.3408GiB/s) | 20260506 16:38:43.379 | INFO | Downloaded 34GiB to GPU in 1.92517s (17.6608GiB/s) | 20260506 16:38:48.520 | INFO | Analyzing chunk frames = [2852352,2852864) | 20260506 16:38:48.520 | INFO | MemInfo: { | 20260506 16:38:48.520 | INFO | "gpu" : " 17 GB", | 20260506 16:38:48.520 | INFO | "huge" : "", | 20260506 16:38:48.520 | INFO | "huge_shared" : " 27 GB", | 20260506 16:38:48.520 | INFO | "malloc" : " 14 GB", | 20260506 16:38:48.520 | INFO | "pinned" : " 29 GB" | 20260506 16:38:48.520 | INFO | } | 20260506 16:38:48.520 | INFO | Perf report for the last 5.14148 seconds: | 20260506 16:38:48.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:38:48.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:48.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:48.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:48.520 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:38:48.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:48.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:48.520 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:38:48.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:48.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:48.521 | INFO | Uploaded 36GiB to GPU in 1.92748s (18.6773GiB/s) | 20260506 16:38:48.521 | INFO | Downloaded 34GiB to GPU in 1.87633s (18.1205GiB/s) | 20260506 16:38:53.664 | INFO | Analyzing chunk frames = [2852864,2853376) | 20260506 16:38:53.664 | INFO | MemInfo: { | 20260506 16:38:53.664 | INFO | "gpu" : " 17 GB", | 20260506 16:38:53.664 | INFO | "huge" : "", | 20260506 16:38:53.664 | INFO | "huge_shared" : " 27 GB", | 20260506 16:38:53.664 | INFO | "malloc" : " 14 GB", | 20260506 16:38:53.664 | INFO | "pinned" : " 29 GB" | 20260506 16:38:53.664 | INFO | } | 20260506 16:38:53.664 | INFO | Perf report for the last 5.144 seconds: | 20260506 16:38:53.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:38:53.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:53.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:53.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:53.664 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 16:38:53.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:53.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:53.664 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260506 16:38:53.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:53.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:53.665 | INFO | Uploaded 36GiB to GPU in 2.01065s (17.9046GiB/s) | 20260506 16:38:53.665 | INFO | Downloaded 34GiB to GPU in 1.91059s (17.7955GiB/s) | 20260506 16:38:58.802 | INFO | Analyzing chunk frames = [2853376,2853888) | 20260506 16:38:58.802 | INFO | MemInfo: { | 20260506 16:38:58.802 | INFO | "gpu" : " 17 GB", | 20260506 16:38:58.802 | INFO | "huge" : "", | 20260506 16:38:58.802 | INFO | "huge_shared" : " 27 GB", | 20260506 16:38:58.802 | INFO | "malloc" : " 14 GB", | 20260506 16:38:58.802 | INFO | "pinned" : " 29 GB" | 20260506 16:38:58.802 | INFO | } | 20260506 16:38:58.802 | INFO | Perf report for the last 5.13757 seconds: | 20260506 16:38:58.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:38:58.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:58.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:58.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:58.802 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 16:38:58.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:58.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:58.802 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:38:58.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:58.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:38:58.802 | INFO | Uploaded 36GiB to GPU in 1.99706s (18.0265GiB/s) | 20260506 16:38:58.802 | INFO | Downloaded 34GiB to GPU in 1.94841s (17.4501GiB/s) | 20260506 16:39:03.939 | INFO | Analyzing chunk frames = [2853888,2854400) | 20260506 16:39:03.939 | INFO | MemInfo: { | 20260506 16:39:03.939 | INFO | "gpu" : " 17 GB", | 20260506 16:39:03.939 | INFO | "huge" : "", | 20260506 16:39:03.939 | INFO | "huge_shared" : " 27 GB", | 20260506 16:39:03.939 | INFO | "malloc" : " 14 GB", | 20260506 16:39:03.939 | INFO | "pinned" : " 29 GB" | 20260506 16:39:03.939 | INFO | } | 20260506 16:39:03.939 | INFO | Perf report for the last 5.13698 seconds: | 20260506 16:39:03.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:39:03.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:03.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:03.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:03.939 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 16:39:03.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:03.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:03.939 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:39:03.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:03.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:03.939 | INFO | Uploaded 36GiB to GPU in 1.96343s (18.3353GiB/s) | 20260506 16:39:03.939 | INFO | Downloaded 34GiB to GPU in 2.00075s (16.9936GiB/s) | 20260506 16:39:09.075 | INFO | Analyzing chunk frames = [2854400,2854912) | 20260506 16:39:09.075 | INFO | MemInfo: { | 20260506 16:39:09.075 | INFO | "gpu" : " 17 GB", | 20260506 16:39:09.075 | INFO | "huge" : "", | 20260506 16:39:09.075 | INFO | "huge_shared" : " 27 GB", | 20260506 16:39:09.075 | INFO | "malloc" : " 14 GB", | 20260506 16:39:09.075 | INFO | "pinned" : " 29 GB" | 20260506 16:39:09.075 | INFO | } | 20260506 16:39:09.076 | INFO | Perf report for the last 5.13665 seconds: | 20260506 16:39:09.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:39:09.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:09.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:09.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:09.076 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:39:09.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:09.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:09.076 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:39:09.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:09.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:09.076 | INFO | Uploaded 36GiB to GPU in 2.00035s (17.9968GiB/s) | 20260506 16:39:09.076 | INFO | Downloaded 34GiB to GPU in 1.96392s (17.3123GiB/s) | 20260506 16:39:09.194 | INFO | Starting a round of Baz IO | 20260506 16:39:12.328 | INFO | Populating RTMetrics report | 20260506 16:39:12.329 | INFO | RTMetrics report populated | 20260506 16:39:12.329 | INFO | Opening tmp RTMetrics json file | 20260506 16:39:12.334 | INFO | Serializing RTMetrics to json | 20260506 16:39:12.375 | INFO | Serialization done, swapping tmp file now... | 20260506 16:39:12.404 | INFO | RTMetrics done | 20260506 16:39:12.859 | INFO | Finished a round of Baz IO: Wrote 2.083 GiB in 3.664 seconds | 20260506 16:39:12.859 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.18% / 0.01% / 63.81% | 20260506 16:39:12.859 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.61723% / 2.38419e-05% / 8.34465e-05% | 20260506 16:39:12.859 | INFO | Baz IO overall: | 20260506 16:39:12.859 | INFO | BytesWritten : 1500203136547 | 20260506 16:39:12.859 | INFO | NumEvents : 725919231857 | 20260506 16:39:14.213 | INFO | Analyzing chunk frames = [2854912,2855424) | 20260506 16:39:14.213 | INFO | MemInfo: { | 20260506 16:39:14.213 | INFO | "gpu" : " 17 GB", | 20260506 16:39:14.213 | INFO | "huge" : "", | 20260506 16:39:14.213 | INFO | "huge_shared" : " 27 GB", | 20260506 16:39:14.213 | INFO | "malloc" : " 14 GB", | 20260506 16:39:14.213 | INFO | "pinned" : " 29 GB" | 20260506 16:39:14.213 | INFO | } | 20260506 16:39:14.213 | INFO | Perf report for the last 5.13751 seconds: | 20260506 16:39:14.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:39:14.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:14.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:14.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:14.213 | INFO | ANALYSIS : 62.7%, 2.9, 95.1ms, 0 | 20260506 16:39:14.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:14.213 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260506 16:39:14.213 | INFO | PRE_HQ : 60.5%, 1.9, 58.6ms, 0 | 20260506 16:39:14.213 | INFO | BAZWRITER : 71.0%, 4.8, 175.3ms, 0 | 20260506 16:39:14.213 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 16:39:14.213 | INFO | Uploaded 36GiB to GPU in 2.04406s (17.612GiB/s) | 20260506 16:39:14.213 | INFO | Downloaded 38GiB to GPU in 2.21366s (17.1661GiB/s) | 20260506 16:39:19.353 | INFO | Analyzing chunk frames = [2855424,2855936) | 20260506 16:39:19.353 | INFO | MemInfo: { | 20260506 16:39:19.353 | INFO | "gpu" : " 17 GB", | 20260506 16:39:19.353 | INFO | "huge" : "", | 20260506 16:39:19.353 | INFO | "huge_shared" : " 27 GB", | 20260506 16:39:19.353 | INFO | "malloc" : " 14 GB", | 20260506 16:39:19.353 | INFO | "pinned" : " 29 GB" | 20260506 16:39:19.353 | INFO | } | 20260506 16:39:19.354 | INFO | Perf report for the last 5.14047 seconds: | 20260506 16:39:19.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:39:19.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:19.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:19.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:19.354 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 16:39:19.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:19.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:19.354 | INFO | PRE_HQ : 27.1%, 1.0, 14.2ms, 0 | 20260506 16:39:19.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:19.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:19.354 | INFO | Uploaded 36GiB to GPU in 1.99542s (18.0413GiB/s) | 20260506 16:39:19.354 | INFO | Downloaded 34GiB to GPU in 1.9422s (17.506GiB/s) | 20260506 16:39:24.499 | INFO | Analyzing chunk frames = [2855936,2856448) | 20260506 16:39:24.499 | INFO | MemInfo: { | 20260506 16:39:24.499 | INFO | "gpu" : " 17 GB", | 20260506 16:39:24.499 | INFO | "huge" : "", | 20260506 16:39:24.499 | INFO | "huge_shared" : " 27 GB", | 20260506 16:39:24.500 | INFO | "malloc" : " 14 GB", | 20260506 16:39:24.500 | INFO | "pinned" : " 29 GB" | 20260506 16:39:24.500 | INFO | } | 20260506 16:39:24.500 | INFO | Perf report for the last 5.14605 seconds: | 20260506 16:39:24.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:39:24.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:24.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:24.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:24.500 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 16:39:24.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:24.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:24.500 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:39:24.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:24.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:24.500 | INFO | Uploaded 36GiB to GPU in 1.9694s (18.2797GiB/s) | 20260506 16:39:24.500 | INFO | Downloaded 34GiB to GPU in 1.92633s (17.6501GiB/s) | 20260506 16:39:29.633 | INFO | Analyzing chunk frames = [2856448,2856960) | 20260506 16:39:29.633 | INFO | MemInfo: { | 20260506 16:39:29.633 | INFO | "gpu" : " 17 GB", | 20260506 16:39:29.633 | INFO | "huge" : "", | 20260506 16:39:29.633 | INFO | "huge_shared" : " 27 GB", | 20260506 16:39:29.633 | INFO | "malloc" : " 14 GB", | 20260506 16:39:29.633 | INFO | "pinned" : " 29 GB" | 20260506 16:39:29.633 | INFO | } | 20260506 16:39:29.633 | INFO | Perf report for the last 5.1332 seconds: | 20260506 16:39:29.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:39:29.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:29.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:29.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:29.633 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:39:29.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:29.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:29.633 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:39:29.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:29.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:29.633 | INFO | Uploaded 36GiB to GPU in 1.93715s (18.584GiB/s) | 20260506 16:39:29.633 | INFO | Downloaded 34GiB to GPU in 1.86901s (18.1914GiB/s) | 20260506 16:39:34.775 | INFO | Analyzing chunk frames = [2856960,2857472) | 20260506 16:39:34.775 | INFO | MemInfo: { | 20260506 16:39:34.775 | INFO | "gpu" : " 17 GB", | 20260506 16:39:34.775 | INFO | "huge" : "", | 20260506 16:39:34.775 | INFO | "huge_shared" : " 27 GB", | 20260506 16:39:34.775 | INFO | "malloc" : " 14 GB", | 20260506 16:39:34.775 | INFO | "pinned" : " 29 GB" | 20260506 16:39:34.775 | INFO | } | 20260506 16:39:34.775 | INFO | Perf report for the last 5.14227 seconds: | 20260506 16:39:34.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:39:34.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:34.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:34.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:34.775 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 16:39:34.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:34.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:34.775 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:39:34.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:34.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:34.776 | INFO | Uploaded 36GiB to GPU in 2.02235s (17.8011GiB/s) | 20260506 16:39:34.776 | INFO | Downloaded 34GiB to GPU in 1.91246s (17.7782GiB/s) | 20260506 16:39:39.911 | INFO | Analyzing chunk frames = [2857472,2857984) | 20260506 16:39:39.911 | INFO | MemInfo: { | 20260506 16:39:39.911 | INFO | "gpu" : " 17 GB", | 20260506 16:39:39.911 | INFO | "huge" : "", | 20260506 16:39:39.911 | INFO | "huge_shared" : " 27 GB", | 20260506 16:39:39.911 | INFO | "malloc" : " 14 GB", | 20260506 16:39:39.911 | INFO | "pinned" : " 29 GB" | 20260506 16:39:39.911 | INFO | } | 20260506 16:39:39.911 | INFO | Perf report for the last 5.13602 seconds: | 20260506 16:39:39.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:39:39.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:39.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:39.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:39.911 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 16:39:39.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:39.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:39.911 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:39:39.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:39.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:39.911 | INFO | Uploaded 36GiB to GPU in 2.05231s (17.5412GiB/s) | 20260506 16:39:39.911 | INFO | Downloaded 34GiB to GPU in 1.95788s (17.3657GiB/s) | 20260506 16:39:45.053 | INFO | Analyzing chunk frames = [2857984,2858496) | 20260506 16:39:45.053 | INFO | MemInfo: { | 20260506 16:39:45.053 | INFO | "gpu" : " 17 GB", | 20260506 16:39:45.053 | INFO | "huge" : "", | 20260506 16:39:45.053 | INFO | "huge_shared" : " 27 GB", | 20260506 16:39:45.053 | INFO | "malloc" : " 14 GB", | 20260506 16:39:45.053 | INFO | "pinned" : " 29 GB" | 20260506 16:39:45.053 | INFO | } | 20260506 16:39:45.053 | INFO | Perf report for the last 5.14172 seconds: | 20260506 16:39:45.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:39:45.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:45.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:45.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:45.053 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 16:39:45.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:45.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:45.053 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 16:39:45.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:45.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:45.053 | INFO | Uploaded 36GiB to GPU in 2.01951s (17.8261GiB/s) | 20260506 16:39:45.053 | INFO | Downloaded 34GiB to GPU in 1.97543s (17.2114GiB/s) | 20260506 16:39:50.191 | INFO | Analyzing chunk frames = [2858496,2859008) | 20260506 16:39:50.191 | INFO | MemInfo: { | 20260506 16:39:50.191 | INFO | "gpu" : " 17 GB", | 20260506 16:39:50.191 | INFO | "huge" : "", | 20260506 16:39:50.191 | INFO | "huge_shared" : " 27 GB", | 20260506 16:39:50.191 | INFO | "malloc" : " 14 GB", | 20260506 16:39:50.191 | INFO | "pinned" : " 29 GB" | 20260506 16:39:50.191 | INFO | } | 20260506 16:39:50.191 | INFO | Perf report for the last 5.13815 seconds: | 20260506 16:39:50.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:39:50.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:50.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:50.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:50.191 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 16:39:50.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:50.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:50.191 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:39:50.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:50.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:50.191 | INFO | Uploaded 36GiB to GPU in 2.06241s (17.4553GiB/s) | 20260506 16:39:50.191 | INFO | Downloaded 34GiB to GPU in 1.99557s (17.0378GiB/s) | 20260506 16:39:50.192 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:39:50.192 | INFO | Total mem usage: 18 GiB | 20260506 16:39:50.192 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:39:50.192 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: -9.29%) | 20260506 16:39:50.192 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:39:50.192 | INFO | High Water Breakdown: | 20260506 16:39:50.192 | INFO | 62.1% : 10.2 GiB ( 89.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:39:50.192 | INFO | 37.7% : 6.22 GiB ( 92.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:39:50.192 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:39:50.192 | INFO | ------------------------------------------ | 20260506 16:39:50.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:39:50.192 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:39:50.192 | INFO | Total mem usage: 26.5 GiB | 20260506 16:39:50.192 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:39:50.192 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:39:50.192 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:39:50.192 | INFO | High Water Breakdown: | 20260506 16:39:50.192 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:39:50.192 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:39:50.192 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:39:50.192 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:39:50.192 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:39:50.192 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:39:50.192 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:39:50.192 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:39:50.192 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:39:50.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:39:50.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:39:50.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:39:50.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:39:50.192 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:39:50.192 | INFO | ------------------------------------------ | 20260506 16:39:50.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:39:50.192 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:39:50.192 | INFO | Total mem usage: 25.6 GiB | 20260506 16:39:50.192 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:39:50.192 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:39:50.192 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:39:50.192 | INFO | High Water Breakdown: | 20260506 16:39:50.192 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:39:50.192 | INFO | ------------------------------------------ | 20260506 16:39:50.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:39:50.192 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:39:50.192 | INFO | Total mem usage: 15.7 GiB | 20260506 16:39:50.192 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:39:50.192 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:39:50.192 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 16:39:50.192 | INFO | High Water Breakdown: | 20260506 16:39:50.192 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:39:50.192 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:39:50.192 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:39:50.192 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:39:50.192 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:39:50.192 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:39:50.192 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:39:50.192 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:39:50.192 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:39:50.192 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:39:50.192 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:39:50.192 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:39:50.192 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:39:50.192 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:39:50.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:39:50.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:39:50.192 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:39:50.192 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:39:50.192 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:39:50.192 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:39:50.192 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:39:50.192 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:39:50.193 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:39:50.193 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:39:50.193 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:39:50.193 | INFO | ------------------------------------------ | 20260506 16:39:50.193 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:39:50.193 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:39:50.193 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:39:50.193 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:39:50.193 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.58e+01, 1.34e+01 / 2.24e+00 | 20260506 16:39:50.193 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.02e-01, 9.81e-03 / 7.61e-03 | 20260506 16:39:50.193 | INFO | PerfReport: 11.60%, Baseline -- 3.64e+00 / 4.44e+00, 5.65e-01 / 2.85e-01 | 20260506 16:39:50.193 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.86e-01 / 1.13e-01 | 20260506 16:39:50.193 | INFO | PerfReport: 15.93%, DME -- 4.00e+01 / 4.46e+01, 4.05e+00 / 2.65e+00 | 20260506 16:39:50.193 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.43e+00 / 1.95e+00 | 20260506 16:39:50.193 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.29e+00 / 2.85e+00, 3.84e-01 / 2.42e-01 | 20260506 16:39:50.193 | INFO | PerfReport: 3.49%, Metrics -- 1.09e+00 / 1.44e+00, 2.86e-01 / 1.97e-01 | 20260506 16:39:50.193 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.79e-01 / 4.26e-01, 5.90e-02 / 2.33e-02 | 20260506 16:39:50.193 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:39:50.193 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:39:50.193 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:39:50.193 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 16:39:50.193 | INFO | PerfReport: 21.40%, Upload -- 2.08e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260506 16:39:50.193 | INFO | PerfReport: 21.14%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 16:39:50.193 | INFO | PerfReport: 32.24%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 16:39:50.193 | INFO | PerfReport: 1.41%, LockUpload -- 1.37e+00 / 0.00e+00, 5.76e+00 / 0.00e+00 | 20260506 16:39:50.193 | INFO | PerfReport: 1.14%, LockDownload -- 1.11e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 16:39:50.193 | INFO | PerfReport: 22.64%, LockCompute -- 2.20e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 16:39:50.193 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260506 16:39:50.300 | INFO | Starting a round of Baz IO | 20260506 16:39:53.453 | INFO | Populating RTMetrics report | 20260506 16:39:53.454 | INFO | RTMetrics report populated | 20260506 16:39:53.454 | INFO | Opening tmp RTMetrics json file | 20260506 16:39:53.500 | INFO | Serializing RTMetrics to json | 20260506 16:39:53.503 | INFO | Serialization done, swapping tmp file now... | 20260506 16:39:53.506 | INFO | RTMetrics done | 20260506 16:39:53.853 | INFO | Finished a round of Baz IO: Wrote 2.084 GiB in 3.553 seconds | 20260506 16:39:53.853 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.18% / 0.01% / 63.82% | 20260506 16:39:53.853 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.61486% / 3.97364e-06% / 5.96046e-05% | 20260506 16:39:53.853 | INFO | Baz IO overall: | 20260506 16:39:53.853 | INFO | BytesWritten : 1502440375081 | 20260506 16:39:53.853 | INFO | NumEvents : 727000592366 | 20260506 16:39:55.327 | INFO | Analyzing chunk frames = [2859008,2859520) | 20260506 16:39:55.327 | INFO | MemInfo: { | 20260506 16:39:55.327 | INFO | "gpu" : " 17 GB", | 20260506 16:39:55.327 | INFO | "huge" : "", | 20260506 16:39:55.327 | INFO | "huge_shared" : " 27 GB", | 20260506 16:39:55.327 | INFO | "malloc" : " 14 GB", | 20260506 16:39:55.327 | INFO | "pinned" : " 29 GB" | 20260506 16:39:55.327 | INFO | } | 20260506 16:39:55.328 | INFO | Perf report for the last 5.13648 seconds: | 20260506 16:39:55.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:39:55.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:55.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:55.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:55.328 | INFO | ANALYSIS : 62.9%, 2.9, 95.8ms, 0 | 20260506 16:39:55.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:39:55.328 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 16:39:55.328 | INFO | PRE_HQ : 60.6%, 1.9, 58.3ms, 0 | 20260506 16:39:55.328 | INFO | BAZWRITER : 68.5%, 4.8, 169.3ms, 0 | 20260506 16:39:55.328 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 16:39:55.328 | INFO | Uploaded 36GiB to GPU in 2.02281s (17.797GiB/s) | 20260506 16:39:55.328 | INFO | Downloaded 38GiB to GPU in 2.22257s (17.0973GiB/s) | 20260506 16:40:00.470 | INFO | Analyzing chunk frames = [2859520,2860032) | 20260506 16:40:00.470 | INFO | MemInfo: { | 20260506 16:40:00.470 | INFO | "gpu" : " 17 GB", | 20260506 16:40:00.470 | INFO | "huge" : "", | 20260506 16:40:00.470 | INFO | "huge_shared" : " 27 GB", | 20260506 16:40:00.470 | INFO | "malloc" : " 14 GB", | 20260506 16:40:00.470 | INFO | "pinned" : " 29 GB" | 20260506 16:40:00.470 | INFO | } | 20260506 16:40:00.470 | INFO | Perf report for the last 5.14285 seconds: | 20260506 16:40:00.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:40:00.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:00.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:00.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:00.470 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 16:40:00.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:00.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:00.471 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 16:40:00.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:00.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:00.471 | INFO | Uploaded 36GiB to GPU in 2.01394s (17.8754GiB/s) | 20260506 16:40:00.471 | INFO | Downloaded 34GiB to GPU in 1.97387s (17.2251GiB/s) | 20260506 16:40:05.611 | INFO | Analyzing chunk frames = [2860032,2860544) | 20260506 16:40:05.611 | INFO | MemInfo: { | 20260506 16:40:05.611 | INFO | "gpu" : " 17 GB", | 20260506 16:40:05.611 | INFO | "huge" : "", | 20260506 16:40:05.611 | INFO | "huge_shared" : " 27 GB", | 20260506 16:40:05.611 | INFO | "malloc" : " 14 GB", | 20260506 16:40:05.611 | INFO | "pinned" : " 29 GB" | 20260506 16:40:05.611 | INFO | } | 20260506 16:40:05.611 | INFO | Perf report for the last 5.14106 seconds: | 20260506 16:40:05.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:40:05.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:05.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:05.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:05.612 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 16:40:05.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:05.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:05.612 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:40:05.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:05.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:05.612 | INFO | Uploaded 36GiB to GPU in 1.96439s (18.3263GiB/s) | 20260506 16:40:05.612 | INFO | Downloaded 34GiB to GPU in 1.92017s (17.7068GiB/s) | 20260506 16:40:10.751 | INFO | Analyzing chunk frames = [2860544,2861056) | 20260506 16:40:10.751 | INFO | MemInfo: { | 20260506 16:40:10.751 | INFO | "gpu" : " 17 GB", | 20260506 16:40:10.751 | INFO | "huge" : "", | 20260506 16:40:10.751 | INFO | "huge_shared" : " 27 GB", | 20260506 16:40:10.751 | INFO | "malloc" : " 14 GB", | 20260506 16:40:10.751 | INFO | "pinned" : " 29 GB" | 20260506 16:40:10.751 | INFO | } | 20260506 16:40:10.751 | INFO | Perf report for the last 5.13957 seconds: | 20260506 16:40:10.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:40:10.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:10.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:10.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:10.751 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 16:40:10.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:10.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:10.751 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:40:10.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:10.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:10.751 | INFO | Uploaded 36GiB to GPU in 1.91587s (18.7904GiB/s) | 20260506 16:40:10.751 | INFO | Downloaded 34GiB to GPU in 1.88069s (18.0784GiB/s) | 20260506 16:40:15.889 | INFO | Analyzing chunk frames = [2861056,2861568) | 20260506 16:40:15.889 | INFO | MemInfo: { | 20260506 16:40:15.889 | INFO | "gpu" : " 17 GB", | 20260506 16:40:15.890 | INFO | "huge" : "", | 20260506 16:40:15.890 | INFO | "huge_shared" : " 27 GB", | 20260506 16:40:15.890 | INFO | "malloc" : " 14 GB", | 20260506 16:40:15.890 | INFO | "pinned" : " 29 GB" | 20260506 16:40:15.890 | INFO | } | 20260506 16:40:15.890 | INFO | Perf report for the last 5.1387 seconds: | 20260506 16:40:15.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:40:15.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:15.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:15.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:15.890 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 16:40:15.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:15.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:15.890 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:40:15.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:15.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:15.890 | INFO | Uploaded 36GiB to GPU in 1.97284s (18.2478GiB/s) | 20260506 16:40:15.890 | INFO | Downloaded 34GiB to GPU in 1.9229s (17.6817GiB/s) | 20260506 16:40:21.026 | INFO | Analyzing chunk frames = [2861568,2862080) | 20260506 16:40:21.026 | INFO | MemInfo: { | 20260506 16:40:21.026 | INFO | "gpu" : " 17 GB", | 20260506 16:40:21.026 | INFO | "huge" : "", | 20260506 16:40:21.026 | INFO | "huge_shared" : " 27 GB", | 20260506 16:40:21.026 | INFO | "malloc" : " 14 GB", | 20260506 16:40:21.026 | INFO | "pinned" : " 29 GB" | 20260506 16:40:21.026 | INFO | } | 20260506 16:40:21.027 | INFO | Perf report for the last 5.13678 seconds: | 20260506 16:40:21.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:40:21.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:21.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:21.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:21.027 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 16:40:21.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:21.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:21.027 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:40:21.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:21.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:21.027 | INFO | Uploaded 36GiB to GPU in 1.98841s (18.1049GiB/s) | 20260506 16:40:21.027 | INFO | Downloaded 34GiB to GPU in 1.93965s (17.529GiB/s) | 20260506 16:40:26.168 | INFO | Analyzing chunk frames = [2862080,2862592) | 20260506 16:40:26.168 | INFO | MemInfo: { | 20260506 16:40:26.168 | INFO | "gpu" : " 17 GB", | 20260506 16:40:26.168 | INFO | "huge" : "", | 20260506 16:40:26.168 | INFO | "huge_shared" : " 27 GB", | 20260506 16:40:26.168 | INFO | "malloc" : " 14 GB", | 20260506 16:40:26.168 | INFO | "pinned" : " 29 GB" | 20260506 16:40:26.168 | INFO | } | 20260506 16:40:26.168 | INFO | Perf report for the last 5.14148 seconds: | 20260506 16:40:26.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:40:26.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:26.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:26.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:26.168 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:40:26.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:26.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:26.168 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:40:26.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:26.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:26.168 | INFO | Uploaded 36GiB to GPU in 1.99763s (18.0214GiB/s) | 20260506 16:40:26.168 | INFO | Downloaded 34GiB to GPU in 2.01529s (16.871GiB/s) | 20260506 16:40:31.307 | INFO | Analyzing chunk frames = [2862592,2863104) | 20260506 16:40:31.307 | INFO | MemInfo: { | 20260506 16:40:31.307 | INFO | "gpu" : " 17 GB", | 20260506 16:40:31.307 | INFO | "huge" : "", | 20260506 16:40:31.307 | INFO | "huge_shared" : " 27 GB", | 20260506 16:40:31.307 | INFO | "malloc" : " 14 GB", | 20260506 16:40:31.307 | INFO | "pinned" : " 29 GB" | 20260506 16:40:31.307 | INFO | } | 20260506 16:40:31.308 | INFO | Perf report for the last 5.13945 seconds: | 20260506 16:40:31.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:40:31.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:31.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:31.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:31.308 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:40:31.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:31.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:31.308 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 16:40:31.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:31.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:31.308 | INFO | Uploaded 36GiB to GPU in 2.01219s (17.8909GiB/s) | 20260506 16:40:31.308 | INFO | Downloaded 34GiB to GPU in 1.98358s (17.1407GiB/s) | 20260506 16:40:31.427 | INFO | Starting a round of Baz IO | 20260506 16:40:34.563 | INFO | Populating RTMetrics report | 20260506 16:40:34.563 | INFO | RTMetrics report populated | 20260506 16:40:34.563 | INFO | Opening tmp RTMetrics json file | 20260506 16:40:34.656 | INFO | Serializing RTMetrics to json | 20260506 16:40:34.774 | INFO | Serialization done, swapping tmp file now... | 20260506 16:40:34.777 | INFO | RTMetrics done | 20260506 16:40:35.367 | INFO | Finished a round of Baz IO: Wrote 2.083 GiB in 3.94 seconds | 20260506 16:40:35.367 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.17% / 0.01% / 63.82% | 20260506 16:40:35.367 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.60905% / 1.98682e-05% / 5.16574e-05% | 20260506 16:40:35.367 | INFO | Baz IO overall: | 20260506 16:40:35.367 | INFO | BytesWritten : 1504676535218 | 20260506 16:40:35.367 | INFO | NumEvents : 728081216254 | 20260506 16:40:36.444 | INFO | Analyzing chunk frames = [2863104,2863616) | 20260506 16:40:36.444 | INFO | MemInfo: { | 20260506 16:40:36.444 | INFO | "gpu" : " 17 GB", | 20260506 16:40:36.444 | INFO | "huge" : "", | 20260506 16:40:36.444 | INFO | "huge_shared" : " 27 GB", | 20260506 16:40:36.444 | INFO | "malloc" : " 14 GB", | 20260506 16:40:36.444 | INFO | "pinned" : " 29 GB" | 20260506 16:40:36.444 | INFO | } | 20260506 16:40:36.444 | INFO | Perf report for the last 5.13683 seconds: | 20260506 16:40:36.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:40:36.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:36.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:36.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:36.444 | INFO | ANALYSIS : 62.8%, 2.9, 95.2ms, 0 | 20260506 16:40:36.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:36.444 | INFO | RT_METRICS : 48.3%, 1.0, 25.3ms, 0 | 20260506 16:40:36.444 | INFO | PRE_HQ : 60.2%, 1.9, 58.4ms, 0 | 20260506 16:40:36.445 | INFO | BAZWRITER : 76.3%, 4.8, 189.0ms, 0 | 20260506 16:40:36.445 | INFO | BAZ_SAMPLING : 3.1%, 1.0, 1.6ms, 0 | 20260506 16:40:36.445 | INFO | Uploaded 36GiB to GPU in 2.06189s (17.4597GiB/s) | 20260506 16:40:36.445 | INFO | Downloaded 38GiB to GPU in 2.21754s (17.1361GiB/s) | 20260506 16:40:41.579 | INFO | Analyzing chunk frames = [2863616,2864128) | 20260506 16:40:41.579 | INFO | MemInfo: { | 20260506 16:40:41.579 | INFO | "gpu" : " 17 GB", | 20260506 16:40:41.579 | INFO | "huge" : "", | 20260506 16:40:41.579 | INFO | "huge_shared" : " 27 GB", | 20260506 16:40:41.579 | INFO | "malloc" : " 14 GB", | 20260506 16:40:41.579 | INFO | "pinned" : " 29 GB" | 20260506 16:40:41.579 | INFO | } | 20260506 16:40:41.579 | INFO | Perf report for the last 5.13502 seconds: | 20260506 16:40:41.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:40:41.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:41.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:41.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:41.579 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:40:41.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:41.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:41.580 | INFO | PRE_HQ : 27.7%, 1.0, 14.5ms, 0 | 20260506 16:40:41.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:41.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:41.580 | INFO | Uploaded 36GiB to GPU in 2.01984s (17.8232GiB/s) | 20260506 16:40:41.580 | INFO | Downloaded 34GiB to GPU in 1.94701s (17.4627GiB/s) | 20260506 16:40:46.720 | INFO | Analyzing chunk frames = [2864128,2864640) | 20260506 16:40:46.720 | INFO | MemInfo: { | 20260506 16:40:46.720 | INFO | "gpu" : " 17 GB", | 20260506 16:40:46.721 | INFO | "huge" : "", | 20260506 16:40:46.721 | INFO | "huge_shared" : " 27 GB", | 20260506 16:40:46.721 | INFO | "malloc" : " 14 GB", | 20260506 16:40:46.721 | INFO | "pinned" : " 29 GB" | 20260506 16:40:46.721 | INFO | } | 20260506 16:40:46.721 | INFO | Perf report for the last 5.14137 seconds: | 20260506 16:40:46.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:40:46.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:46.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:46.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:46.721 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 16:40:46.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:46.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:46.721 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:40:46.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:46.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:46.721 | INFO | Uploaded 36GiB to GPU in 1.94263s (18.5316GiB/s) | 20260506 16:40:46.721 | INFO | Downloaded 34GiB to GPU in 1.8847s (18.04GiB/s) | 20260506 16:40:51.867 | INFO | Analyzing chunk frames = [2864640,2865152) | 20260506 16:40:51.867 | INFO | MemInfo: { | 20260506 16:40:51.867 | INFO | "gpu" : " 17 GB", | 20260506 16:40:51.867 | INFO | "huge" : "", | 20260506 16:40:51.867 | INFO | "huge_shared" : " 27 GB", | 20260506 16:40:51.867 | INFO | "malloc" : " 14 GB", | 20260506 16:40:51.867 | INFO | "pinned" : " 29 GB" | 20260506 16:40:51.867 | INFO | } | 20260506 16:40:51.867 | INFO | Perf report for the last 5.14657 seconds: | 20260506 16:40:51.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:40:51.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:51.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:51.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:51.867 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260506 16:40:51.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:51.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:51.867 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:40:51.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:51.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:51.868 | INFO | Uploaded 36GiB to GPU in 1.9354s (18.6008GiB/s) | 20260506 16:40:51.868 | INFO | Downloaded 34GiB to GPU in 1.88084s (18.0771GiB/s) | 20260506 16:40:57.007 | INFO | Analyzing chunk frames = [2865152,2865664) | 20260506 16:40:57.007 | INFO | MemInfo: { | 20260506 16:40:57.007 | INFO | "gpu" : " 17 GB", | 20260506 16:40:57.007 | INFO | "huge" : "", | 20260506 16:40:57.007 | INFO | "huge_shared" : " 27 GB", | 20260506 16:40:57.007 | INFO | "malloc" : " 14 GB", | 20260506 16:40:57.007 | INFO | "pinned" : " 29 GB" | 20260506 16:40:57.007 | INFO | } | 20260506 16:40:57.007 | INFO | Perf report for the last 5.13991 seconds: | 20260506 16:40:57.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:40:57.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:57.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:57.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:57.007 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 16:40:57.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:57.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:57.007 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:40:57.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:57.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:40:57.008 | INFO | Uploaded 36GiB to GPU in 2.0088s (17.9211GiB/s) | 20260506 16:40:57.008 | INFO | Downloaded 34GiB to GPU in 1.91547s (17.7502GiB/s) | 20260506 16:41:02.147 | INFO | Analyzing chunk frames = [2865664,2866176) | 20260506 16:41:02.147 | INFO | MemInfo: { | 20260506 16:41:02.147 | INFO | "gpu" : " 17 GB", | 20260506 16:41:02.147 | INFO | "huge" : "", | 20260506 16:41:02.147 | INFO | "huge_shared" : " 27 GB", | 20260506 16:41:02.147 | INFO | "malloc" : " 14 GB", | 20260506 16:41:02.147 | INFO | "pinned" : " 29 GB" | 20260506 16:41:02.147 | INFO | } | 20260506 16:41:02.147 | INFO | Perf report for the last 5.1399 seconds: | 20260506 16:41:02.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:41:02.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:02.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:02.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:02.147 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 16:41:02.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:02.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:02.147 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:41:02.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:02.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:02.147 | INFO | Uploaded 36GiB to GPU in 2.06956s (17.395GiB/s) | 20260506 16:41:02.147 | INFO | Downloaded 34GiB to GPU in 1.96194s (17.3298GiB/s) | 20260506 16:41:07.281 | INFO | Analyzing chunk frames = [2866176,2866688) | 20260506 16:41:07.281 | INFO | MemInfo: { | 20260506 16:41:07.281 | INFO | "gpu" : " 17 GB", | 20260506 16:41:07.281 | INFO | "huge" : "", | 20260506 16:41:07.281 | INFO | "huge_shared" : " 27 GB", | 20260506 16:41:07.281 | INFO | "malloc" : " 14 GB", | 20260506 16:41:07.281 | INFO | "pinned" : " 29 GB" | 20260506 16:41:07.281 | INFO | } | 20260506 16:41:07.281 | INFO | Perf report for the last 5.13383 seconds: | 20260506 16:41:07.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:41:07.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:07.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:07.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:07.281 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260506 16:41:07.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:07.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:07.281 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:41:07.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:07.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:07.281 | INFO | Uploaded 36GiB to GPU in 2.00066s (17.994GiB/s) | 20260506 16:41:07.281 | INFO | Downloaded 34GiB to GPU in 1.97238s (17.2381GiB/s) | 20260506 16:41:12.419 | INFO | Analyzing chunk frames = [2866688,2867200) | 20260506 16:41:12.419 | INFO | MemInfo: { | 20260506 16:41:12.419 | INFO | "gpu" : " 17 GB", | 20260506 16:41:12.419 | INFO | "huge" : "", | 20260506 16:41:12.419 | INFO | "huge_shared" : " 27 GB", | 20260506 16:41:12.419 | INFO | "malloc" : " 14 GB", | 20260506 16:41:12.419 | INFO | "pinned" : " 29 GB" | 20260506 16:41:12.419 | INFO | } | 20260506 16:41:12.419 | INFO | Perf report for the last 5.13791 seconds: | 20260506 16:41:12.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:41:12.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:12.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:12.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:12.419 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:41:12.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:12.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:12.419 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 16:41:12.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:12.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:12.419 | INFO | Uploaded 36GiB to GPU in 2.00888s (17.9205GiB/s) | 20260506 16:41:12.419 | INFO | Downloaded 34GiB to GPU in 1.98504s (17.1281GiB/s) | 20260506 16:41:12.419 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:41:12.419 | INFO | Total mem usage: 18 GiB | 20260506 16:41:12.419 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:41:12.419 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -11.2%) | 20260506 16:41:12.419 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:41:12.419 | INFO | High Water Breakdown: | 20260506 16:41:12.419 | INFO | 61.9% : 10 GiB ( 87.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:41:12.419 | INFO | 38% : 6.15 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:41:12.419 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:41:12.419 | INFO | ------------------------------------------ | 20260506 16:41:12.419 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:41:12.419 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:41:12.420 | INFO | Total mem usage: 26.5 GiB | 20260506 16:41:12.420 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:41:12.420 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:41:12.420 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:41:12.420 | INFO | High Water Breakdown: | 20260506 16:41:12.420 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:41:12.420 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:41:12.420 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:41:12.420 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:41:12.420 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:41:12.420 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:41:12.420 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:41:12.420 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:41:12.420 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:41:12.420 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:41:12.420 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:41:12.420 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:41:12.420 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:41:12.420 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:41:12.420 | INFO | ------------------------------------------ | 20260506 16:41:12.420 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:41:12.420 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:41:12.420 | INFO | Total mem usage: 25.6 GiB | 20260506 16:41:12.420 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:41:12.420 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:41:12.420 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:41:12.420 | INFO | High Water Breakdown: | 20260506 16:41:12.420 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:41:12.420 | INFO | ------------------------------------------ | 20260506 16:41:12.420 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:41:12.420 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:41:12.420 | INFO | Total mem usage: 15.7 GiB | 20260506 16:41:12.420 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:41:12.420 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:41:12.420 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 16:41:12.420 | INFO | High Water Breakdown: | 20260506 16:41:12.420 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:41:12.420 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:41:12.420 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:41:12.420 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:41:12.420 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:41:12.420 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:41:12.420 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:41:12.420 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:41:12.420 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:41:12.420 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:41:12.420 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:41:12.420 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:41:12.420 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:41:12.420 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:41:12.420 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:41:12.420 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:41:12.420 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:41:12.420 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:41:12.420 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:41:12.420 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:41:12.420 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:41:12.420 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:41:12.420 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:41:12.420 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:41:12.420 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:41:12.420 | INFO | ------------------------------------------ | 20260506 16:41:12.420 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:41:12.420 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:41:12.420 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:41:12.420 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:41:12.420 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.59e+01, 1.32e+01 / 1.95e+00 | 20260506 16:41:12.420 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.00e-01, 9.80e-03 / 4.34e-03 | 20260506 16:41:12.420 | INFO | PerfReport: 11.60%, Baseline -- 3.64e+00 / 4.52e+00, 5.58e-01 / 2.56e-01 | 20260506 16:41:12.420 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.82e-01 / 1.11e-01 | 20260506 16:41:12.420 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.49e+01, 4.04e+00 / 2.53e+00 | 20260506 16:41:12.420 | INFO | PerfReport: 56.85%, FrameLabeling -- 1.78e+01 / 2.01e+01, 1.35e+00 / 1.77e+00 | 20260506 16:41:12.420 | INFO | PerfReport: 7.33%, PulseAccumulating -- 2.30e+00 / 2.90e+00, 3.91e-01 / 2.42e-01 | 20260506 16:41:12.420 | INFO | PerfReport: 3.49%, Metrics -- 1.09e+00 / 1.49e+00, 2.84e-01 / 1.62e-01 | 20260506 16:41:12.420 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.79e-01 / 4.37e-01, 5.95e-02 / 2.22e-02 | 20260506 16:41:12.420 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:41:12.420 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:41:12.420 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:41:12.420 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 16:41:12.420 | INFO | PerfReport: 21.37%, Upload -- 2.08e+01 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260506 16:41:12.420 | INFO | PerfReport: 21.16%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 16:41:12.420 | INFO | PerfReport: 32.21%, Compute -- 3.13e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 16:41:12.420 | INFO | PerfReport: 1.41%, LockUpload -- 1.37e+00 / 0.00e+00, 5.73e+00 / 0.00e+00 | 20260506 16:41:12.420 | INFO | PerfReport: 1.16%, LockDownload -- 1.13e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 16:41:12.420 | INFO | PerfReport: 22.67%, LockCompute -- 2.21e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 16:41:12.420 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.34e-02 / 0.00e+00 | 20260506 16:41:12.537 | INFO | Starting a round of Baz IO | 20260506 16:41:15.689 | INFO | Populating RTMetrics report | 20260506 16:41:15.690 | INFO | RTMetrics report populated | 20260506 16:41:15.690 | INFO | Opening tmp RTMetrics json file | 20260506 16:41:15.958 | INFO | Serializing RTMetrics to json | 20260506 16:41:16.035 | INFO | Serialization done, swapping tmp file now... | 20260506 16:41:16.039 | INFO | RTMetrics done | 20260506 16:41:16.232 | INFO | Finished a round of Baz IO: Wrote 2.081 GiB in 3.694 seconds | 20260506 16:41:16.232 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.16% / 0.01% / 63.83% | 20260506 16:41:16.232 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.60315% / 1.19209e-05% / 6.75519e-05% | 20260506 16:41:16.232 | INFO | Baz IO overall: | 20260506 16:41:16.232 | INFO | BytesWritten : 1506910823859 | 20260506 16:41:16.232 | INFO | NumEvents : 729160406712 | 20260506 16:41:17.559 | INFO | Analyzing chunk frames = [2867200,2867712) | 20260506 16:41:17.559 | INFO | MemInfo: { | 20260506 16:41:17.559 | INFO | "gpu" : " 17 GB", | 20260506 16:41:17.559 | INFO | "huge" : "", | 20260506 16:41:17.559 | INFO | "huge_shared" : " 27 GB", | 20260506 16:41:17.559 | INFO | "malloc" : " 14 GB", | 20260506 16:41:17.559 | INFO | "pinned" : " 29 GB" | 20260506 16:41:17.559 | INFO | } | 20260506 16:41:17.559 | INFO | Perf report for the last 5.14031 seconds: | 20260506 16:41:17.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:41:17.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:17.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:17.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:17.559 | INFO | ANALYSIS : 63.1%, 2.9, 95.8ms, 0 | 20260506 16:41:17.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:17.559 | INFO | RT_METRICS : 50.6%, 1.0, 26.5ms, 0 | 20260506 16:41:17.559 | INFO | PRE_HQ : 60.7%, 1.9, 58.9ms, 0 | 20260506 16:41:17.559 | INFO | BAZWRITER : 71.4%, 4.8, 176.6ms, 0 | 20260506 16:41:17.560 | INFO | BAZ_SAMPLING : 5.6%, 1.0, 2.9ms, 0 | 20260506 16:41:17.560 | INFO | Uploaded 36GiB to GPU in 2.06817s (17.4067GiB/s) | 20260506 16:41:17.560 | INFO | Downloaded 38GiB to GPU in 2.22509s (17.078GiB/s) | 20260506 16:41:22.692 | INFO | Analyzing chunk frames = [2867712,2868224) | 20260506 16:41:22.692 | INFO | MemInfo: { | 20260506 16:41:22.692 | INFO | "gpu" : " 17 GB", | 20260506 16:41:22.692 | INFO | "huge" : "", | 20260506 16:41:22.692 | INFO | "huge_shared" : " 27 GB", | 20260506 16:41:22.692 | INFO | "malloc" : " 14 GB", | 20260506 16:41:22.692 | INFO | "pinned" : " 29 GB" | 20260506 16:41:22.692 | INFO | } | 20260506 16:41:22.692 | INFO | Perf report for the last 5.13261 seconds: | 20260506 16:41:22.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:41:22.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:22.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:22.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:22.692 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260506 16:41:22.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:22.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:22.692 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 16:41:22.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:22.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:22.692 | INFO | Uploaded 36GiB to GPU in 1.99213s (18.0711GiB/s) | 20260506 16:41:22.692 | INFO | Downloaded 34GiB to GPU in 1.94916s (17.4434GiB/s) | 20260506 16:41:27.837 | INFO | Analyzing chunk frames = [2868224,2868736) | 20260506 16:41:27.837 | INFO | MemInfo: { | 20260506 16:41:27.837 | INFO | "gpu" : " 17 GB", | 20260506 16:41:27.837 | INFO | "huge" : "", | 20260506 16:41:27.837 | INFO | "huge_shared" : " 27 GB", | 20260506 16:41:27.838 | INFO | "malloc" : " 14 GB", | 20260506 16:41:27.838 | INFO | "pinned" : " 29 GB" | 20260506 16:41:27.838 | INFO | } | 20260506 16:41:27.838 | INFO | Perf report for the last 5.14571 seconds: | 20260506 16:41:27.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:41:27.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:27.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:27.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:27.838 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 16:41:27.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:27.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:27.838 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:41:27.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:27.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:27.838 | INFO | Uploaded 36GiB to GPU in 1.94398s (18.5187GiB/s) | 20260506 16:41:27.838 | INFO | Downloaded 34GiB to GPU in 1.88921s (17.9969GiB/s) | 20260506 16:41:32.979 | INFO | Analyzing chunk frames = [2868736,2869248) | 20260506 16:41:32.979 | INFO | MemInfo: { | 20260506 16:41:32.979 | INFO | "gpu" : " 17 GB", | 20260506 16:41:32.979 | INFO | "huge" : "", | 20260506 16:41:32.979 | INFO | "huge_shared" : " 27 GB", | 20260506 16:41:32.979 | INFO | "malloc" : " 14 GB", | 20260506 16:41:32.979 | INFO | "pinned" : " 29 GB" | 20260506 16:41:32.979 | INFO | } | 20260506 16:41:32.979 | INFO | Perf report for the last 5.14162 seconds: | 20260506 16:41:32.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:41:32.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:32.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:32.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:32.979 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 16:41:32.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:32.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:32.979 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:41:32.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:32.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:32.979 | INFO | Uploaded 36GiB to GPU in 1.93261s (18.6277GiB/s) | 20260506 16:41:32.979 | INFO | Downloaded 34GiB to GPU in 1.8766s (18.1179GiB/s) | 20260506 16:41:38.118 | INFO | Analyzing chunk frames = [2869248,2869760) | 20260506 16:41:38.118 | INFO | MemInfo: { | 20260506 16:41:38.118 | INFO | "gpu" : " 17 GB", | 20260506 16:41:38.118 | INFO | "huge" : "", | 20260506 16:41:38.118 | INFO | "huge_shared" : " 27 GB", | 20260506 16:41:38.118 | INFO | "malloc" : " 14 GB", | 20260506 16:41:38.118 | INFO | "pinned" : " 29 GB" | 20260506 16:41:38.118 | INFO | } | 20260506 16:41:38.118 | INFO | Perf report for the last 5.13886 seconds: | 20260506 16:41:38.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:41:38.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:38.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:38.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:38.118 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 16:41:38.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:38.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:38.118 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:41:38.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:38.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:38.118 | INFO | Uploaded 36GiB to GPU in 2.01843s (17.8356GiB/s) | 20260506 16:41:38.118 | INFO | Downloaded 34GiB to GPU in 1.91206s (17.7819GiB/s) | 20260506 16:41:43.258 | INFO | Analyzing chunk frames = [2869760,2870272) | 20260506 16:41:43.258 | INFO | MemInfo: { | 20260506 16:41:43.258 | INFO | "gpu" : " 17 GB", | 20260506 16:41:43.258 | INFO | "huge" : "", | 20260506 16:41:43.258 | INFO | "huge_shared" : " 27 GB", | 20260506 16:41:43.258 | INFO | "malloc" : " 14 GB", | 20260506 16:41:43.258 | INFO | "pinned" : " 29 GB" | 20260506 16:41:43.259 | INFO | } | 20260506 16:41:43.259 | INFO | Perf report for the last 5.14047 seconds: | 20260506 16:41:43.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:41:43.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:43.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:43.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:43.259 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:41:43.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:43.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:43.259 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:41:43.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:43.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:43.259 | INFO | Uploaded 36GiB to GPU in 2.07955s (17.3114GiB/s) | 20260506 16:41:43.259 | INFO | Downloaded 34GiB to GPU in 1.96593s (17.2946GiB/s) | 20260506 16:41:48.394 | INFO | Analyzing chunk frames = [2870272,2870784) | 20260506 16:41:48.394 | INFO | MemInfo: { | 20260506 16:41:48.394 | INFO | "gpu" : " 17 GB", | 20260506 16:41:48.394 | INFO | "huge" : "", | 20260506 16:41:48.394 | INFO | "huge_shared" : " 27 GB", | 20260506 16:41:48.394 | INFO | "malloc" : " 14 GB", | 20260506 16:41:48.394 | INFO | "pinned" : " 29 GB" | 20260506 16:41:48.394 | INFO | } | 20260506 16:41:48.394 | INFO | Perf report for the last 5.13555 seconds: | 20260506 16:41:48.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:41:48.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:48.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:48.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:48.394 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:41:48.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:48.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:48.394 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:41:48.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:48.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:48.395 | INFO | Uploaded 36GiB to GPU in 1.9883s (18.1059GiB/s) | 20260506 16:41:48.395 | INFO | Downloaded 34GiB to GPU in 1.96493s (17.3035GiB/s) | 20260506 16:41:53.533 | INFO | Analyzing chunk frames = [2870784,2871296) | 20260506 16:41:53.533 | INFO | MemInfo: { | 20260506 16:41:53.533 | INFO | "gpu" : " 17 GB", | 20260506 16:41:53.534 | INFO | "huge" : "", | 20260506 16:41:53.534 | INFO | "huge_shared" : " 27 GB", | 20260506 16:41:53.534 | INFO | "malloc" : " 14 GB", | 20260506 16:41:53.534 | INFO | "pinned" : " 29 GB" | 20260506 16:41:53.534 | INFO | } | 20260506 16:41:53.534 | INFO | Perf report for the last 5.13949 seconds: | 20260506 16:41:53.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:41:53.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:53.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:53.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:53.534 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 16:41:53.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:53.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:53.534 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:41:53.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:53.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:53.534 | INFO | Uploaded 36GiB to GPU in 2.03091s (17.7261GiB/s) | 20260506 16:41:53.534 | INFO | Downloaded 34GiB to GPU in 1.98192s (17.1551GiB/s) | 20260506 16:41:53.673 | INFO | Starting a round of Baz IO | 20260506 16:41:56.794 | INFO | Populating RTMetrics report | 20260506 16:41:56.794 | INFO | RTMetrics report populated | 20260506 16:41:56.794 | INFO | Opening tmp RTMetrics json file | 20260506 16:41:56.838 | INFO | Serializing RTMetrics to json | 20260506 16:41:57.028 | INFO | Serialization done, swapping tmp file now... | 20260506 16:41:57.038 | INFO | RTMetrics done | 20260506 16:41:57.059 | INFO | Finished a round of Baz IO: Wrote 2.082 GiB in 3.385 seconds | 20260506 16:41:57.059 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.15% / 0.01% / 63.84% | 20260506 16:41:57.059 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.60048% / 1.19209e-05% / 7.15256e-05% | 20260506 16:41:57.059 | INFO | Baz IO overall: | 20260506 16:41:57.059 | INFO | BytesWritten : 1509146244398 | 20260506 16:41:57.059 | INFO | NumEvents : 730240077000 | 20260506 16:41:58.669 | INFO | Analyzing chunk frames = [2871296,2871808) | 20260506 16:41:58.669 | INFO | MemInfo: { | 20260506 16:41:58.669 | INFO | "gpu" : " 17 GB", | 20260506 16:41:58.669 | INFO | "huge" : "", | 20260506 16:41:58.669 | INFO | "huge_shared" : " 27 GB", | 20260506 16:41:58.669 | INFO | "malloc" : " 14 GB", | 20260506 16:41:58.669 | INFO | "pinned" : " 29 GB" | 20260506 16:41:58.669 | INFO | } | 20260506 16:41:58.669 | INFO | Perf report for the last 5.13557 seconds: | 20260506 16:41:58.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:41:58.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:58.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:58.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:58.669 | INFO | ANALYSIS : 62.9%, 2.9, 95.3ms, 0 | 20260506 16:41:58.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:41:58.669 | INFO | RT_METRICS : 48.3%, 1.0, 25.3ms, 0 | 20260506 16:41:58.669 | INFO | PRE_HQ : 60.3%, 1.9, 58.7ms, 0 | 20260506 16:41:58.669 | INFO | BAZWRITER : 65.9%, 4.8, 163.2ms, 0 | 20260506 16:41:58.669 | INFO | BAZ_SAMPLING : 4.4%, 1.0, 2.3ms, 0 | 20260506 16:41:58.670 | INFO | Uploaded 36GiB to GPU in 2.04131s (17.6358GiB/s) | 20260506 16:41:58.670 | INFO | Downloaded 38GiB to GPU in 2.21459s (17.1589GiB/s) | 20260506 16:42:03.815 | INFO | Analyzing chunk frames = [2871808,2872320) | 20260506 16:42:03.815 | INFO | MemInfo: { | 20260506 16:42:03.815 | INFO | "gpu" : " 17 GB", | 20260506 16:42:03.815 | INFO | "huge" : "", | 20260506 16:42:03.815 | INFO | "huge_shared" : " 27 GB", | 20260506 16:42:03.815 | INFO | "malloc" : " 14 GB", | 20260506 16:42:03.815 | INFO | "pinned" : " 29 GB" | 20260506 16:42:03.815 | INFO | } | 20260506 16:42:03.815 | INFO | Perf report for the last 5.14599 seconds: | 20260506 16:42:03.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:42:03.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:03.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:03.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:03.815 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 16:42:03.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:03.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:03.815 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:42:03.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:03.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:03.815 | INFO | Uploaded 36GiB to GPU in 2.0195s (17.8262GiB/s) | 20260506 16:42:03.816 | INFO | Downloaded 34GiB to GPU in 1.97847s (17.185GiB/s) | 20260506 16:42:08.957 | INFO | Analyzing chunk frames = [2872320,2872832) | 20260506 16:42:08.957 | INFO | MemInfo: { | 20260506 16:42:08.957 | INFO | "gpu" : " 17 GB", | 20260506 16:42:08.957 | INFO | "huge" : "", | 20260506 16:42:08.957 | INFO | "huge_shared" : " 27 GB", | 20260506 16:42:08.958 | INFO | "malloc" : " 14 GB", | 20260506 16:42:08.958 | INFO | "pinned" : " 29 GB" | 20260506 16:42:08.958 | INFO | } | 20260506 16:42:08.958 | INFO | Perf report for the last 5.14235 seconds: | 20260506 16:42:08.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:42:08.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:08.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:08.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:08.958 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 16:42:08.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:08.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:08.958 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:42:08.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:08.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:08.958 | INFO | Uploaded 36GiB to GPU in 1.98478s (18.138GiB/s) | 20260506 16:42:08.958 | INFO | Downloaded 34GiB to GPU in 1.92604s (17.6528GiB/s) | 20260506 16:42:14.096 | INFO | Analyzing chunk frames = [2872832,2873344) | 20260506 16:42:14.096 | INFO | MemInfo: { | 20260506 16:42:14.096 | INFO | "gpu" : " 17 GB", | 20260506 16:42:14.097 | INFO | "huge" : "", | 20260506 16:42:14.097 | INFO | "huge_shared" : " 27 GB", | 20260506 16:42:14.097 | INFO | "malloc" : " 14 GB", | 20260506 16:42:14.097 | INFO | "pinned" : " 29 GB" | 20260506 16:42:14.097 | INFO | } | 20260506 16:42:14.097 | INFO | Perf report for the last 5.13899 seconds: | 20260506 16:42:14.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:42:14.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:14.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:14.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:14.097 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:42:14.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:14.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:14.097 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:42:14.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:14.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:14.097 | INFO | Uploaded 36GiB to GPU in 1.95926s (18.3743GiB/s) | 20260506 16:42:14.097 | INFO | Downloaded 34GiB to GPU in 1.88133s (18.0723GiB/s) | 20260506 16:42:19.239 | INFO | Analyzing chunk frames = [2873344,2873856) | 20260506 16:42:19.239 | INFO | MemInfo: { | 20260506 16:42:19.239 | INFO | "gpu" : " 17 GB", | 20260506 16:42:19.239 | INFO | "huge" : "", | 20260506 16:42:19.240 | INFO | "huge_shared" : " 27 GB", | 20260506 16:42:19.240 | INFO | "malloc" : " 14 GB", | 20260506 16:42:19.240 | INFO | "pinned" : " 29 GB" | 20260506 16:42:19.240 | INFO | } | 20260506 16:42:19.240 | INFO | Perf report for the last 5.14297 seconds: | 20260506 16:42:19.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:42:19.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:19.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:19.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:19.240 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 16:42:19.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:19.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:19.240 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:42:19.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:19.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:19.240 | INFO | Uploaded 36GiB to GPU in 1.98101s (18.1726GiB/s) | 20260506 16:42:19.240 | INFO | Downloaded 34GiB to GPU in 1.91664s (17.7394GiB/s) | 20260506 16:42:24.372 | INFO | Analyzing chunk frames = [2873856,2874368) | 20260506 16:42:24.372 | INFO | MemInfo: { | 20260506 16:42:24.372 | INFO | "gpu" : " 17 GB", | 20260506 16:42:24.372 | INFO | "huge" : "", | 20260506 16:42:24.372 | INFO | "huge_shared" : " 27 GB", | 20260506 16:42:24.372 | INFO | "malloc" : " 14 GB", | 20260506 16:42:24.372 | INFO | "pinned" : " 29 GB" | 20260506 16:42:24.372 | INFO | } | 20260506 16:42:24.372 | INFO | Perf report for the last 5.13254 seconds: | 20260506 16:42:24.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:42:24.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:24.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:24.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:24.372 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 16:42:24.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:24.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:24.372 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 16:42:24.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:24.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:24.372 | INFO | Uploaded 36GiB to GPU in 2.00183s (17.9835GiB/s) | 20260506 16:42:24.372 | INFO | Downloaded 34GiB to GPU in 1.93802s (17.5437GiB/s) | 20260506 16:42:29.509 | INFO | Analyzing chunk frames = [2874368,2874880) | 20260506 16:42:29.509 | INFO | MemInfo: { | 20260506 16:42:29.509 | INFO | "gpu" : " 17 GB", | 20260506 16:42:29.509 | INFO | "huge" : "", | 20260506 16:42:29.509 | INFO | "huge_shared" : " 27 GB", | 20260506 16:42:29.509 | INFO | "malloc" : " 14 GB", | 20260506 16:42:29.509 | INFO | "pinned" : " 29 GB" | 20260506 16:42:29.509 | INFO | } | 20260506 16:42:29.509 | INFO | Perf report for the last 5.13707 seconds: | 20260506 16:42:29.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:42:29.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:29.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:29.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:29.509 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 16:42:29.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:29.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:29.509 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:42:29.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:29.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:29.509 | INFO | Uploaded 36GiB to GPU in 2.03063s (17.7285GiB/s) | 20260506 16:42:29.509 | INFO | Downloaded 34GiB to GPU in 2.01156s (16.9023GiB/s) | 20260506 16:42:34.649 | INFO | Analyzing chunk frames = [2874880,2875392) | 20260506 16:42:34.649 | INFO | MemInfo: { | 20260506 16:42:34.649 | INFO | "gpu" : " 17 GB", | 20260506 16:42:34.649 | INFO | "huge" : "", | 20260506 16:42:34.649 | INFO | "huge_shared" : " 27 GB", | 20260506 16:42:34.649 | INFO | "malloc" : " 14 GB", | 20260506 16:42:34.649 | INFO | "pinned" : " 29 GB" | 20260506 16:42:34.649 | INFO | } | 20260506 16:42:34.649 | INFO | Perf report for the last 5.13962 seconds: | 20260506 16:42:34.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:42:34.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:34.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:34.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:34.649 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:42:34.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:34.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:34.649 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260506 16:42:34.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:34.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:34.649 | INFO | Uploaded 36GiB to GPU in 2.0066s (17.9408GiB/s) | 20260506 16:42:34.649 | INFO | Downloaded 34GiB to GPU in 1.97126s (17.2479GiB/s) | 20260506 16:42:34.649 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:42:34.649 | INFO | Total mem usage: 18 GiB | 20260506 16:42:34.649 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:42:34.649 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -14.6%) | 20260506 16:42:34.649 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:42:34.649 | INFO | High Water Breakdown: | 20260506 16:42:34.649 | INFO | 61.4% : 9.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:42:34.649 | INFO | 38.5% : 6.04 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:42:34.649 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:42:34.649 | INFO | ------------------------------------------ | 20260506 16:42:34.649 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:42:34.649 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:42:34.649 | INFO | Total mem usage: 26.5 GiB | 20260506 16:42:34.649 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:42:34.649 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:42:34.649 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:42:34.649 | INFO | High Water Breakdown: | 20260506 16:42:34.649 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:42:34.649 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:42:34.649 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:42:34.649 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:42:34.649 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:42:34.649 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:42:34.650 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:42:34.650 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:42:34.650 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:42:34.650 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:42:34.650 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:42:34.650 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:42:34.650 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:42:34.650 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:42:34.650 | INFO | ------------------------------------------ | 20260506 16:42:34.650 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:42:34.650 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:42:34.650 | INFO | Total mem usage: 25.6 GiB | 20260506 16:42:34.650 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:42:34.650 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:42:34.650 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:42:34.650 | INFO | High Water Breakdown: | 20260506 16:42:34.650 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:42:34.650 | INFO | ------------------------------------------ | 20260506 16:42:34.650 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:42:34.650 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:42:34.650 | INFO | Total mem usage: 15.7 GiB | 20260506 16:42:34.650 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:42:34.650 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:42:34.650 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:42:34.650 | INFO | High Water Breakdown: | 20260506 16:42:34.650 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:42:34.650 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:42:34.650 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:42:34.650 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:42:34.650 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:42:34.650 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:42:34.650 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:42:34.650 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:42:34.650 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:42:34.650 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:42:34.650 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:42:34.650 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:42:34.650 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:42:34.650 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:42:34.650 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:42:34.650 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:42:34.650 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:42:34.650 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:42:34.650 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:42:34.650 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:42:34.650 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:42:34.650 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:42:34.650 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:42:34.650 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:42:34.650 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:42:34.650 | INFO | ------------------------------------------ | 20260506 16:42:34.650 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:42:34.650 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:42:34.650 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:42:34.650 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:42:34.650 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.59e+01, 1.33e+01 / 2.11e+00 | 20260506 16:42:34.650 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.00e-01, 9.82e-03 / 5.43e-03 | 20260506 16:42:34.650 | INFO | PerfReport: 11.57%, Baseline -- 3.63e+00 / 4.46e+00, 5.50e-01 / 2.70e-01 | 20260506 16:42:34.650 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.46e+00, 1.82e-01 / 1.29e-01 | 20260506 16:42:34.650 | INFO | PerfReport: 15.96%, DME -- 4.01e+01 / 4.50e+01, 4.00e+00 / 2.22e+00 | 20260506 16:42:34.650 | INFO | PerfReport: 56.91%, FrameLabeling -- 1.79e+01 / 2.01e+01, 1.42e+00 / 1.66e+00 | 20260506 16:42:34.650 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.30e+00 / 2.91e+00, 3.92e-01 / 2.11e-01 | 20260506 16:42:34.650 | INFO | PerfReport: 3.49%, Metrics -- 1.09e+00 / 1.43e+00, 2.85e-01 / 1.83e-01 | 20260506 16:42:34.650 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.80e-01 / 4.29e-01, 6.04e-02 / 2.50e-02 | 20260506 16:42:34.650 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:42:34.650 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:42:34.650 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:42:34.650 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 16:42:34.650 | INFO | PerfReport: 21.45%, Upload -- 2.09e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260506 16:42:34.650 | INFO | PerfReport: 21.13%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 16:42:34.650 | INFO | PerfReport: 32.23%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 16:42:34.650 | INFO | PerfReport: 1.52%, LockUpload -- 1.48e+00 / 0.00e+00, 5.86e+00 / 0.00e+00 | 20260506 16:42:34.650 | INFO | PerfReport: 1.03%, LockDownload -- 1.01e+00 / 0.00e+00, 2.61e+00 / 0.00e+00 | 20260506 16:42:34.650 | INFO | PerfReport: 22.62%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 16:42:34.650 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260506 16:42:34.768 | INFO | Starting a round of Baz IO | 20260506 16:42:37.914 | INFO | Populating RTMetrics report | 20260506 16:42:37.914 | INFO | RTMetrics report populated | 20260506 16:42:37.914 | INFO | Opening tmp RTMetrics json file | 20260506 16:42:37.915 | INFO | Serializing RTMetrics to json | 20260506 16:42:37.916 | INFO | Serialization done, swapping tmp file now... | 20260506 16:42:37.917 | INFO | RTMetrics done | 20260506 16:42:38.028 | INFO | Finished a round of Baz IO: Wrote 2.082 GiB in 3.259 seconds | 20260506 16:42:38.028 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.14% / 0.01% / 63.85% | 20260506 16:42:38.028 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.5975% / 1.58946e-05% / 7.54992e-05% | 20260506 16:42:38.028 | INFO | Baz IO overall: | 20260506 16:42:38.028 | INFO | BytesWritten : 1511382288534 | 20260506 16:42:38.028 | INFO | NumEvents : 731320405186 | 20260506 16:42:39.785 | INFO | Analyzing chunk frames = [2875392,2875904) | 20260506 16:42:39.785 | INFO | MemInfo: { | 20260506 16:42:39.785 | INFO | "gpu" : " 17 GB", | 20260506 16:42:39.785 | INFO | "huge" : "", | 20260506 16:42:39.785 | INFO | "huge_shared" : " 27 GB", | 20260506 16:42:39.785 | INFO | "malloc" : " 14 GB", | 20260506 16:42:39.785 | INFO | "pinned" : " 29 GB" | 20260506 16:42:39.785 | INFO | } | 20260506 16:42:39.785 | INFO | Perf report for the last 5.13579 seconds: | 20260506 16:42:39.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:42:39.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:39.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:39.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:39.785 | INFO | ANALYSIS : 63.0%, 2.9, 95.8ms, 0 | 20260506 16:42:39.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:39.785 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 16:42:39.785 | INFO | PRE_HQ : 60.6%, 1.9, 58.4ms, 0 | 20260506 16:42:39.785 | INFO | BAZWRITER : 63.1%, 4.6, 149.5ms, 0 | 20260506 16:42:39.785 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 16:42:39.785 | INFO | Uploaded 36GiB to GPU in 2.05191s (17.5446GiB/s) | 20260506 16:42:39.785 | INFO | Downloaded 38GiB to GPU in 2.22411s (17.0855GiB/s) | 20260506 16:42:44.933 | INFO | Analyzing chunk frames = [2875904,2876416) | 20260506 16:42:44.933 | INFO | MemInfo: { | 20260506 16:42:44.933 | INFO | "gpu" : " 17 GB", | 20260506 16:42:44.933 | INFO | "huge" : "", | 20260506 16:42:44.933 | INFO | "huge_shared" : " 27 GB", | 20260506 16:42:44.933 | INFO | "malloc" : " 14 GB", | 20260506 16:42:44.933 | INFO | "pinned" : " 29 GB" | 20260506 16:42:44.933 | INFO | } | 20260506 16:42:44.933 | INFO | Perf report for the last 5.14829 seconds: | 20260506 16:42:44.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:42:44.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:44.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:44.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:44.933 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 16:42:44.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:44.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:44.933 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 16:42:44.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:44.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:44.933 | INFO | Uploaded 36GiB to GPU in 1.99902s (18.0088GiB/s) | 20260506 16:42:44.933 | INFO | Downloaded 34GiB to GPU in 1.93923s (17.5327GiB/s) | 20260506 16:42:50.071 | INFO | Analyzing chunk frames = [2876416,2876928) | 20260506 16:42:50.071 | INFO | MemInfo: { | 20260506 16:42:50.071 | INFO | "gpu" : " 17 GB", | 20260506 16:42:50.071 | INFO | "huge" : "", | 20260506 16:42:50.071 | INFO | "huge_shared" : " 27 GB", | 20260506 16:42:50.071 | INFO | "malloc" : " 14 GB", | 20260506 16:42:50.071 | INFO | "pinned" : " 29 GB" | 20260506 16:42:50.071 | INFO | } | 20260506 16:42:50.071 | INFO | Perf report for the last 5.13802 seconds: | 20260506 16:42:50.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:42:50.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:50.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:50.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:50.071 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 16:42:50.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:50.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:50.071 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:42:50.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:50.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:50.071 | INFO | Uploaded 36GiB to GPU in 1.94936s (18.4676GiB/s) | 20260506 16:42:50.071 | INFO | Downloaded 34GiB to GPU in 1.92081s (17.7009GiB/s) | 20260506 16:42:55.210 | INFO | Analyzing chunk frames = [2876928,2877440) | 20260506 16:42:55.210 | INFO | MemInfo: { | 20260506 16:42:55.210 | INFO | "gpu" : " 17 GB", | 20260506 16:42:55.210 | INFO | "huge" : "", | 20260506 16:42:55.210 | INFO | "huge_shared" : " 27 GB", | 20260506 16:42:55.210 | INFO | "malloc" : " 14 GB", | 20260506 16:42:55.210 | INFO | "pinned" : " 29 GB" | 20260506 16:42:55.210 | INFO | } | 20260506 16:42:55.210 | INFO | Perf report for the last 5.13946 seconds: | 20260506 16:42:55.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:42:55.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:55.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:55.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:55.210 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 16:42:55.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:55.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:55.211 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:42:55.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:55.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:42:55.211 | INFO | Uploaded 36GiB to GPU in 1.93488s (18.6058GiB/s) | 20260506 16:42:55.211 | INFO | Downloaded 34GiB to GPU in 1.8711s (18.1711GiB/s) | 20260506 16:43:00.346 | INFO | Analyzing chunk frames = [2877440,2877952) | 20260506 16:43:00.346 | INFO | MemInfo: { | 20260506 16:43:00.346 | INFO | "gpu" : " 17 GB", | 20260506 16:43:00.346 | INFO | "huge" : "", | 20260506 16:43:00.346 | INFO | "huge_shared" : " 27 GB", | 20260506 16:43:00.346 | INFO | "malloc" : " 14 GB", | 20260506 16:43:00.346 | INFO | "pinned" : " 29 GB" | 20260506 16:43:00.346 | INFO | } | 20260506 16:43:00.346 | INFO | Perf report for the last 5.1358 seconds: | 20260506 16:43:00.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:43:00.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:00.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:00.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:00.346 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 16:43:00.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:00.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:00.346 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:43:00.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:00.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:00.347 | INFO | Uploaded 36GiB to GPU in 2.0166s (17.8518GiB/s) | 20260506 16:43:00.347 | INFO | Downloaded 34GiB to GPU in 1.91005s (17.8005GiB/s) | 20260506 16:43:05.485 | INFO | Analyzing chunk frames = [2877952,2878464) | 20260506 16:43:05.485 | INFO | MemInfo: { | 20260506 16:43:05.485 | INFO | "gpu" : " 17 GB", | 20260506 16:43:05.485 | INFO | "huge" : "", | 20260506 16:43:05.485 | INFO | "huge_shared" : " 27 GB", | 20260506 16:43:05.485 | INFO | "malloc" : " 14 GB", | 20260506 16:43:05.485 | INFO | "pinned" : " 29 GB" | 20260506 16:43:05.485 | INFO | } | 20260506 16:43:05.485 | INFO | Perf report for the last 5.13912 seconds: | 20260506 16:43:05.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:43:05.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:05.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:05.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:05.486 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 16:43:05.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:05.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:05.486 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:43:05.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:05.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:05.486 | INFO | Uploaded 36GiB to GPU in 2.01069s (17.9043GiB/s) | 20260506 16:43:05.486 | INFO | Downloaded 34GiB to GPU in 1.94187s (17.5089GiB/s) | 20260506 16:43:10.625 | INFO | Analyzing chunk frames = [2878464,2878976) | 20260506 16:43:10.625 | INFO | MemInfo: { | 20260506 16:43:10.625 | INFO | "gpu" : " 17 GB", | 20260506 16:43:10.625 | INFO | "huge" : "", | 20260506 16:43:10.625 | INFO | "huge_shared" : " 27 GB", | 20260506 16:43:10.625 | INFO | "malloc" : " 14 GB", | 20260506 16:43:10.625 | INFO | "pinned" : " 29 GB" | 20260506 16:43:10.625 | INFO | } | 20260506 16:43:10.625 | INFO | Perf report for the last 5.14004 seconds: | 20260506 16:43:10.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:43:10.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:10.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:10.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:10.626 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 16:43:10.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:10.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:10.626 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:43:10.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:10.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:10.626 | INFO | Uploaded 36GiB to GPU in 2.02346s (17.7913GiB/s) | 20260506 16:43:10.626 | INFO | Downloaded 34GiB to GPU in 2.01306s (16.8897GiB/s) | 20260506 16:43:15.761 | INFO | Analyzing chunk frames = [2878976,2879488) | 20260506 16:43:15.761 | INFO | MemInfo: { | 20260506 16:43:15.761 | INFO | "gpu" : " 17 GB", | 20260506 16:43:15.761 | INFO | "huge" : "", | 20260506 16:43:15.761 | INFO | "huge_shared" : " 27 GB", | 20260506 16:43:15.761 | INFO | "malloc" : " 14 GB", | 20260506 16:43:15.761 | INFO | "pinned" : " 29 GB" | 20260506 16:43:15.761 | INFO | } | 20260506 16:43:15.761 | INFO | Perf report for the last 5.1353 seconds: | 20260506 16:43:15.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:43:15.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:15.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:15.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:15.761 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:43:15.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:15.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:15.761 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260506 16:43:15.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:15.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:15.761 | INFO | Uploaded 36GiB to GPU in 2.01335s (17.8807GiB/s) | 20260506 16:43:15.761 | INFO | Downloaded 34GiB to GPU in 1.97938s (17.1771GiB/s) | 20260506 16:43:15.881 | INFO | Starting a round of Baz IO | 20260506 16:43:19.019 | INFO | Populating RTMetrics report | 20260506 16:43:19.020 | INFO | RTMetrics report populated | 20260506 16:43:19.020 | INFO | Opening tmp RTMetrics json file | 20260506 16:43:19.055 | INFO | Serializing RTMetrics to json | 20260506 16:43:19.082 | INFO | Serialization done, swapping tmp file now... | 20260506 16:43:19.090 | INFO | RTMetrics done | 20260506 16:43:19.436 | INFO | Finished a round of Baz IO: Wrote 2.082 GiB in 3.555 seconds | 20260506 16:43:19.436 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.13% / 0.01% / 63.86% | 20260506 16:43:19.436 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.59079% / 7.94729e-06% / 5.96046e-05% | 20260506 16:43:19.436 | INFO | Baz IO overall: | 20260506 16:43:19.436 | INFO | BytesWritten : 1513617666276 | 20260506 16:43:19.436 | INFO | NumEvents : 732400253370 | 20260506 16:43:20.906 | INFO | Analyzing chunk frames = [2879488,2880000) | 20260506 16:43:20.906 | INFO | MemInfo: { | 20260506 16:43:20.906 | INFO | "gpu" : " 17 GB", | 20260506 16:43:20.906 | INFO | "huge" : "", | 20260506 16:43:20.906 | INFO | "huge_shared" : " 27 GB", | 20260506 16:43:20.906 | INFO | "malloc" : " 14 GB", | 20260506 16:43:20.906 | INFO | "pinned" : " 29 GB" | 20260506 16:43:20.906 | INFO | } | 20260506 16:43:20.906 | INFO | Perf report for the last 5.14554 seconds: | 20260506 16:43:20.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:43:20.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:20.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:20.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:20.906 | INFO | ANALYSIS : 62.7%, 2.9, 95.4ms, 0 | 20260506 16:43:20.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:20.906 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 16:43:20.906 | INFO | PRE_HQ : 60.4%, 1.9, 58.5ms, 0 | 20260506 16:43:20.906 | INFO | BAZWRITER : 68.6%, 4.8, 170.9ms, 0 | 20260506 16:43:20.906 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 16:43:20.907 | INFO | Uploaded 36GiB to GPU in 2.06432s (17.4392GiB/s) | 20260506 16:43:20.907 | INFO | Downloaded 38GiB to GPU in 2.21849s (17.1288GiB/s) | 20260506 16:43:26.049 | INFO | Analyzing chunk frames = [2880000,2880512) | 20260506 16:43:26.049 | INFO | MemInfo: { | 20260506 16:43:26.049 | INFO | "gpu" : " 17 GB", | 20260506 16:43:26.049 | INFO | "huge" : "", | 20260506 16:43:26.049 | INFO | "huge_shared" : " 27 GB", | 20260506 16:43:26.049 | INFO | "malloc" : " 14 GB", | 20260506 16:43:26.049 | INFO | "pinned" : " 29 GB" | 20260506 16:43:26.049 | INFO | } | 20260506 16:43:26.049 | INFO | Perf report for the last 5.14275 seconds: | 20260506 16:43:26.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:43:26.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:26.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:26.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:26.049 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 16:43:26.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:26.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:26.049 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 16:43:26.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:26.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:26.049 | INFO | Uploaded 36GiB to GPU in 2.01488s (17.867GiB/s) | 20260506 16:43:26.049 | INFO | Downloaded 34GiB to GPU in 1.94805s (17.4533GiB/s) | 20260506 16:43:31.186 | INFO | Analyzing chunk frames = [2880512,2881024) | 20260506 16:43:31.186 | INFO | MemInfo: { | 20260506 16:43:31.186 | INFO | "gpu" : " 17 GB", | 20260506 16:43:31.186 | INFO | "huge" : "", | 20260506 16:43:31.186 | INFO | "huge_shared" : " 27 GB", | 20260506 16:43:31.186 | INFO | "malloc" : " 14 GB", | 20260506 16:43:31.186 | INFO | "pinned" : " 29 GB" | 20260506 16:43:31.186 | INFO | } | 20260506 16:43:31.186 | INFO | Perf report for the last 5.13728 seconds: | 20260506 16:43:31.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:43:31.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:31.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:31.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:31.187 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 16:43:31.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:31.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:31.187 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:43:31.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:31.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:31.187 | INFO | Uploaded 36GiB to GPU in 1.98134s (18.1696GiB/s) | 20260506 16:43:31.187 | INFO | Downloaded 34GiB to GPU in 1.92748s (17.6396GiB/s) | 20260506 16:43:36.327 | INFO | Analyzing chunk frames = [2881024,2881536) | 20260506 16:43:36.328 | INFO | MemInfo: { | 20260506 16:43:36.328 | INFO | "gpu" : " 17 GB", | 20260506 16:43:36.328 | INFO | "huge" : "", | 20260506 16:43:36.328 | INFO | "huge_shared" : " 27 GB", | 20260506 16:43:36.328 | INFO | "malloc" : " 14 GB", | 20260506 16:43:36.328 | INFO | "pinned" : " 29 GB" | 20260506 16:43:36.328 | INFO | } | 20260506 16:43:36.328 | INFO | Perf report for the last 5.14136 seconds: | 20260506 16:43:36.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:43:36.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:36.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:36.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:36.328 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 16:43:36.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:36.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:36.328 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:43:36.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:36.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:36.328 | INFO | Uploaded 36GiB to GPU in 1.96246s (18.3443GiB/s) | 20260506 16:43:36.328 | INFO | Downloaded 34GiB to GPU in 1.88243s (18.0618GiB/s) | 20260506 16:43:41.460 | INFO | Analyzing chunk frames = [2881536,2882048) | 20260506 16:43:41.460 | INFO | MemInfo: { | 20260506 16:43:41.460 | INFO | "gpu" : " 17 GB", | 20260506 16:43:41.460 | INFO | "huge" : "", | 20260506 16:43:41.460 | INFO | "huge_shared" : " 27 GB", | 20260506 16:43:41.460 | INFO | "malloc" : " 14 GB", | 20260506 16:43:41.460 | INFO | "pinned" : " 29 GB" | 20260506 16:43:41.460 | INFO | } | 20260506 16:43:41.460 | INFO | Perf report for the last 5.13251 seconds: | 20260506 16:43:41.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:43:41.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:41.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:41.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:41.460 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260506 16:43:41.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:41.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:41.460 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:43:41.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:41.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:41.461 | INFO | Uploaded 36GiB to GPU in 2.00945s (17.9153GiB/s) | 20260506 16:43:41.461 | INFO | Downloaded 34GiB to GPU in 1.91349s (17.7686GiB/s) | 20260506 16:43:46.600 | INFO | Analyzing chunk frames = [2882048,2882560) | 20260506 16:43:46.600 | INFO | MemInfo: { | 20260506 16:43:46.600 | INFO | "gpu" : " 17 GB", | 20260506 16:43:46.600 | INFO | "huge" : "", | 20260506 16:43:46.600 | INFO | "huge_shared" : " 27 GB", | 20260506 16:43:46.600 | INFO | "malloc" : " 14 GB", | 20260506 16:43:46.600 | INFO | "pinned" : " 29 GB" | 20260506 16:43:46.600 | INFO | } | 20260506 16:43:46.600 | INFO | Perf report for the last 5.14018 seconds: | 20260506 16:43:46.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:43:46.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:46.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:46.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:46.600 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 16:43:46.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:46.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:46.601 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:43:46.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:46.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:46.601 | INFO | Uploaded 36GiB to GPU in 2.02731s (17.7575GiB/s) | 20260506 16:43:46.601 | INFO | Downloaded 34GiB to GPU in 1.94827s (17.4514GiB/s) | 20260506 16:43:51.742 | INFO | Analyzing chunk frames = [2882560,2883072) | 20260506 16:43:51.742 | INFO | MemInfo: { | 20260506 16:43:51.742 | INFO | "gpu" : " 17 GB", | 20260506 16:43:51.742 | INFO | "huge" : "", | 20260506 16:43:51.742 | INFO | "huge_shared" : " 27 GB", | 20260506 16:43:51.742 | INFO | "malloc" : " 14 GB", | 20260506 16:43:51.742 | INFO | "pinned" : " 29 GB" | 20260506 16:43:51.742 | INFO | } | 20260506 16:43:51.742 | INFO | Perf report for the last 5.1416 seconds: | 20260506 16:43:51.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:43:51.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:51.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:51.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:51.742 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:43:51.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:51.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:51.742 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:43:51.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:51.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:51.742 | INFO | Uploaded 36GiB to GPU in 2.02454s (17.7818GiB/s) | 20260506 16:43:51.743 | INFO | Downloaded 34GiB to GPU in 1.99443s (17.0475GiB/s) | 20260506 16:43:56.876 | INFO | Analyzing chunk frames = [2883072,2883584) | 20260506 16:43:56.876 | INFO | MemInfo: { | 20260506 16:43:56.876 | INFO | "gpu" : " 17 GB", | 20260506 16:43:56.876 | INFO | "huge" : "", | 20260506 16:43:56.876 | INFO | "huge_shared" : " 27 GB", | 20260506 16:43:56.876 | INFO | "malloc" : " 14 GB", | 20260506 16:43:56.876 | INFO | "pinned" : " 29 GB" | 20260506 16:43:56.876 | INFO | } | 20260506 16:43:56.876 | INFO | Perf report for the last 5.13406 seconds: | 20260506 16:43:56.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:43:56.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:56.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:56.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:56.876 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 16:43:56.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:56.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:56.876 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260506 16:43:56.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:56.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:43:56.877 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:43:56.877 | INFO | Total mem usage: 18 GiB | 20260506 16:43:56.877 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:43:56.877 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -14.2%) | 20260506 16:43:56.877 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:43:56.877 | INFO | High Water Breakdown: | 20260506 16:43:56.877 | INFO | Uploaded 36GiB to GPU in 2.01426s (17.8725GiB/s) | 20260506 16:43:56.877 | INFO | 61.5% : 9.7 GiB ( 85% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:43:56.877 | INFO | 38.4% : 6.05 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:43:56.877 | INFO | Downloaded 34GiB to GPU in 1.97716s (17.1964GiB/s) | 20260506 16:43:56.877 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:43:56.877 | INFO | ------------------------------------------ | 20260506 16:43:56.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:43:56.877 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:43:56.877 | INFO | Total mem usage: 26.5 GiB | 20260506 16:43:56.877 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:43:56.877 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:43:56.877 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:43:56.877 | INFO | High Water Breakdown: | 20260506 16:43:56.877 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:43:56.877 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:43:56.877 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:43:56.877 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:43:56.877 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:43:56.877 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:43:56.877 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:43:56.877 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:43:56.877 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:43:56.877 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:43:56.877 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:43:56.877 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:43:56.877 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:43:56.877 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:43:56.877 | INFO | ------------------------------------------ | 20260506 16:43:56.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:43:56.877 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:43:56.877 | INFO | Total mem usage: 25.6 GiB | 20260506 16:43:56.877 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:43:56.877 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:43:56.878 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:43:56.878 | INFO | High Water Breakdown: | 20260506 16:43:56.878 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:43:56.878 | INFO | ------------------------------------------ | 20260506 16:43:56.878 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:43:56.878 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:43:56.878 | INFO | Total mem usage: 15.7 GiB | 20260506 16:43:56.878 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:43:56.878 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:43:56.878 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:43:56.878 | INFO | High Water Breakdown: | 20260506 16:43:56.878 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:43:56.878 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:43:56.878 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:43:56.878 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:43:56.878 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:43:56.878 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:43:56.878 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:43:56.878 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:43:56.878 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:43:56.878 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:43:56.878 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:43:56.878 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:43:56.878 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:43:56.878 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:43:56.878 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:43:56.878 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:43:56.878 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:43:56.878 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:43:56.878 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:43:56.878 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:43:56.878 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:43:56.878 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:43:56.878 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:43:56.878 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:43:56.878 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:43:56.878 | INFO | ------------------------------------------ | 20260506 16:43:56.878 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:43:56.878 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:43:56.878 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:43:56.878 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:43:56.878 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.59e+01, 1.34e+01 / 2.33e+00 | 20260506 16:43:56.878 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 9.82e-03 / 8.29e-03 | 20260506 16:43:56.878 | INFO | PerfReport: 11.60%, Baseline -- 3.64e+00 / 4.45e+00, 5.63e-01 / 3.18e-01 | 20260506 16:43:56.878 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.46e+00, 1.87e-01 / 1.41e-01 | 20260506 16:43:56.878 | INFO | PerfReport: 15.93%, DME -- 4.00e+01 / 4.45e+01, 3.99e+00 / 2.53e+00 | 20260506 16:43:56.878 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.44e+00 / 2.06e+00 | 20260506 16:43:56.878 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.29e+00 / 2.88e+00, 3.88e-01 / 2.39e-01 | 20260506 16:43:56.878 | INFO | PerfReport: 3.48%, Metrics -- 1.09e+00 / 1.44e+00, 2.84e-01 / 1.50e-01 | 20260506 16:43:56.878 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.79e-01 / 4.25e-01, 5.97e-02 / 1.72e-02 | 20260506 16:43:56.878 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:43:56.878 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:43:56.878 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:43:56.878 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 16:43:56.878 | INFO | PerfReport: 21.47%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 16:43:56.878 | INFO | PerfReport: 21.14%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 16:43:56.878 | INFO | PerfReport: 32.24%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 16:43:56.878 | INFO | PerfReport: 1.41%, LockUpload -- 1.38e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260506 16:43:56.878 | INFO | PerfReport: 1.09%, LockDownload -- 1.07e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260506 16:43:56.878 | INFO | PerfReport: 22.63%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 16:43:56.878 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260506 16:43:56.990 | INFO | Starting a round of Baz IO | 20260506 16:44:00.139 | INFO | Populating RTMetrics report | 20260506 16:44:00.140 | INFO | RTMetrics report populated | 20260506 16:44:00.140 | INFO | Opening tmp RTMetrics json file | 20260506 16:44:00.155 | INFO | Serializing RTMetrics to json | 20260506 16:44:00.189 | INFO | Serialization done, swapping tmp file now... | 20260506 16:44:00.190 | INFO | RTMetrics done | 20260506 16:44:00.918 | INFO | Finished a round of Baz IO: Wrote 2.08 GiB in 3.927 seconds | 20260506 16:44:00.918 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.13% / 0.01% / 63.87% | 20260506 16:44:00.918 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.58563% / 1.19209e-05% / 7.94729e-05% | 20260506 16:44:00.918 | INFO | Baz IO overall: | 20260506 16:44:00.918 | INFO | BytesWritten : 1515851370390 | 20260506 16:44:00.918 | INFO | NumEvents : 733478898580 | 20260506 16:44:02.018 | INFO | Analyzing chunk frames = [2883584,2884096) | 20260506 16:44:02.018 | INFO | MemInfo: { | 20260506 16:44:02.018 | INFO | "gpu" : " 17 GB", | 20260506 16:44:02.018 | INFO | "huge" : "", | 20260506 16:44:02.018 | INFO | "huge_shared" : " 27 GB", | 20260506 16:44:02.018 | INFO | "malloc" : " 14 GB", | 20260506 16:44:02.018 | INFO | "pinned" : " 29 GB" | 20260506 16:44:02.018 | INFO | } | 20260506 16:44:02.018 | INFO | Perf report for the last 5.142 seconds: | 20260506 16:44:02.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:44:02.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:02.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:02.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:02.018 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 16:44:02.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:02.018 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260506 16:44:02.018 | INFO | PRE_HQ : 60.5%, 1.9, 58.5ms, 0 | 20260506 16:44:02.018 | INFO | BAZWRITER : 75.8%, 4.8, 187.2ms, 0 | 20260506 16:44:02.018 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 16:44:02.018 | INFO | Uploaded 36GiB to GPU in 2.05215s (17.5425GiB/s) | 20260506 16:44:02.018 | INFO | Downloaded 38GiB to GPU in 2.22648s (17.0673GiB/s) | 20260506 16:44:07.159 | INFO | Analyzing chunk frames = [2884096,2884608) | 20260506 16:44:07.159 | INFO | MemInfo: { | 20260506 16:44:07.159 | INFO | "gpu" : " 17 GB", | 20260506 16:44:07.159 | INFO | "huge" : "", | 20260506 16:44:07.159 | INFO | "huge_shared" : " 27 GB", | 20260506 16:44:07.159 | INFO | "malloc" : " 14 GB", | 20260506 16:44:07.159 | INFO | "pinned" : " 29 GB" | 20260506 16:44:07.159 | INFO | } | 20260506 16:44:07.159 | INFO | Perf report for the last 5.14124 seconds: | 20260506 16:44:07.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:44:07.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:07.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:07.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:07.160 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:44:07.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:07.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:07.160 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 16:44:07.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:07.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:07.160 | INFO | Uploaded 36GiB to GPU in 1.98399s (18.1452GiB/s) | 20260506 16:44:07.160 | INFO | Downloaded 34GiB to GPU in 1.93038s (17.6131GiB/s) | 20260506 16:44:12.298 | INFO | Analyzing chunk frames = [2884608,2885120) | 20260506 16:44:12.298 | INFO | MemInfo: { | 20260506 16:44:12.298 | INFO | "gpu" : " 17 GB", | 20260506 16:44:12.298 | INFO | "huge" : "", | 20260506 16:44:12.298 | INFO | "huge_shared" : " 27 GB", | 20260506 16:44:12.298 | INFO | "malloc" : " 14 GB", | 20260506 16:44:12.298 | INFO | "pinned" : " 29 GB" | 20260506 16:44:12.298 | INFO | } | 20260506 16:44:12.298 | INFO | Perf report for the last 5.13892 seconds: | 20260506 16:44:12.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:44:12.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:12.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:12.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:12.299 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 16:44:12.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:12.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:12.299 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:44:12.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:12.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:12.299 | INFO | Uploaded 36GiB to GPU in 1.99332s (18.0604GiB/s) | 20260506 16:44:12.299 | INFO | Downloaded 34GiB to GPU in 1.92931s (17.6229GiB/s) | 20260506 16:44:17.437 | INFO | Analyzing chunk frames = [2885120,2885632) | 20260506 16:44:17.437 | INFO | MemInfo: { | 20260506 16:44:17.437 | INFO | "gpu" : " 17 GB", | 20260506 16:44:17.437 | INFO | "huge" : "", | 20260506 16:44:17.437 | INFO | "huge_shared" : " 27 GB", | 20260506 16:44:17.437 | INFO | "malloc" : " 14 GB", | 20260506 16:44:17.437 | INFO | "pinned" : " 29 GB" | 20260506 16:44:17.437 | INFO | } | 20260506 16:44:17.437 | INFO | Perf report for the last 5.13878 seconds: | 20260506 16:44:17.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:44:17.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:17.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:17.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:17.437 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:44:17.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:17.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:17.437 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:44:17.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:17.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:17.438 | INFO | Uploaded 36GiB to GPU in 1.92441s (18.707GiB/s) | 20260506 16:44:17.438 | INFO | Downloaded 34GiB to GPU in 1.86936s (18.1881GiB/s) | 20260506 16:44:22.573 | INFO | Analyzing chunk frames = [2885632,2886144) | 20260506 16:44:22.573 | INFO | MemInfo: { | 20260506 16:44:22.573 | INFO | "gpu" : " 17 GB", | 20260506 16:44:22.573 | INFO | "huge" : "", | 20260506 16:44:22.573 | INFO | "huge_shared" : " 27 GB", | 20260506 16:44:22.573 | INFO | "malloc" : " 14 GB", | 20260506 16:44:22.573 | INFO | "pinned" : " 29 GB" | 20260506 16:44:22.573 | INFO | } | 20260506 16:44:22.573 | INFO | Perf report for the last 5.13602 seconds: | 20260506 16:44:22.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:44:22.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:22.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:22.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:22.573 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 16:44:22.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:22.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:22.573 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:44:22.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:22.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:22.573 | INFO | Uploaded 36GiB to GPU in 2.0183s (17.8368GiB/s) | 20260506 16:44:22.574 | INFO | Downloaded 34GiB to GPU in 1.9083s (17.8169GiB/s) | 20260506 16:44:27.716 | INFO | Analyzing chunk frames = [2886144,2886656) | 20260506 16:44:27.716 | INFO | MemInfo: { | 20260506 16:44:27.716 | INFO | "gpu" : " 17 GB", | 20260506 16:44:27.716 | INFO | "huge" : "", | 20260506 16:44:27.716 | INFO | "huge_shared" : " 27 GB", | 20260506 16:44:27.716 | INFO | "malloc" : " 14 GB", | 20260506 16:44:27.716 | INFO | "pinned" : " 29 GB" | 20260506 16:44:27.716 | INFO | } | 20260506 16:44:27.716 | INFO | Perf report for the last 5.14317 seconds: | 20260506 16:44:27.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:44:27.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:27.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:27.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:27.716 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:44:27.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:27.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:27.716 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:44:27.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:27.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:27.717 | INFO | Uploaded 36GiB to GPU in 2.03498s (17.6906GiB/s) | 20260506 16:44:27.717 | INFO | Downloaded 34GiB to GPU in 1.9486s (17.4484GiB/s) | 20260506 16:44:32.855 | INFO | Analyzing chunk frames = [2886656,2887168) | 20260506 16:44:32.855 | INFO | MemInfo: { | 20260506 16:44:32.855 | INFO | "gpu" : " 17 GB", | 20260506 16:44:32.855 | INFO | "huge" : "", | 20260506 16:44:32.855 | INFO | "huge_shared" : " 27 GB", | 20260506 16:44:32.855 | INFO | "malloc" : " 14 GB", | 20260506 16:44:32.855 | INFO | "pinned" : " 29 GB" | 20260506 16:44:32.855 | INFO | } | 20260506 16:44:32.855 | INFO | Perf report for the last 5.13866 seconds: | 20260506 16:44:32.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:44:32.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:32.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:32.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:32.855 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:44:32.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:32.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:32.855 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 16:44:32.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:32.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:32.855 | INFO | Uploaded 36GiB to GPU in 2.02145s (17.809GiB/s) | 20260506 16:44:32.855 | INFO | Downloaded 34GiB to GPU in 1.99062s (17.0801GiB/s) | 20260506 16:44:37.996 | INFO | Analyzing chunk frames = [2887168,2887680) | 20260506 16:44:37.996 | INFO | MemInfo: { | 20260506 16:44:37.996 | INFO | "gpu" : " 17 GB", | 20260506 16:44:37.996 | INFO | "huge" : "", | 20260506 16:44:37.996 | INFO | "huge_shared" : " 27 GB", | 20260506 16:44:37.996 | INFO | "malloc" : " 14 GB", | 20260506 16:44:37.996 | INFO | "pinned" : " 29 GB" | 20260506 16:44:37.996 | INFO | } | 20260506 16:44:37.996 | INFO | Perf report for the last 5.14122 seconds: | 20260506 16:44:37.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:44:37.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:37.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:37.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:37.996 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:44:37.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:37.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:37.996 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:44:37.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:37.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:37.997 | INFO | Uploaded 36GiB to GPU in 2.01659s (17.8519GiB/s) | 20260506 16:44:37.997 | INFO | Downloaded 34GiB to GPU in 1.97366s (17.2269GiB/s) | 20260506 16:44:38.111 | INFO | Starting a round of Baz IO | 20260506 16:44:41.257 | INFO | Populating RTMetrics report | 20260506 16:44:41.257 | INFO | RTMetrics report populated | 20260506 16:44:41.257 | INFO | Opening tmp RTMetrics json file | 20260506 16:44:41.266 | INFO | Serializing RTMetrics to json | 20260506 16:44:41.322 | INFO | Serialization done, swapping tmp file now... | 20260506 16:44:41.362 | INFO | RTMetrics done | 20260506 16:44:41.864 | INFO | Finished a round of Baz IO: Wrote 2.08 GiB in 3.752 seconds | 20260506 16:44:41.864 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.12% / 0.01% / 63.87% | 20260506 16:44:41.864 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.58364% / 7.94729e-06% / 8.34465e-05% | 20260506 16:44:41.864 | INFO | Baz IO overall: | 20260506 16:44:41.864 | INFO | BytesWritten : 1518084223004 | 20260506 16:44:41.864 | INFO | NumEvents : 734557169459 | 20260506 16:44:43.133 | INFO | Analyzing chunk frames = [2887680,2888192) | 20260506 16:44:43.133 | INFO | MemInfo: { | 20260506 16:44:43.133 | INFO | "gpu" : " 17 GB", | 20260506 16:44:43.133 | INFO | "huge" : "", | 20260506 16:44:43.133 | INFO | "huge_shared" : " 27 GB", | 20260506 16:44:43.133 | INFO | "malloc" : " 14 GB", | 20260506 16:44:43.133 | INFO | "pinned" : " 29 GB" | 20260506 16:44:43.133 | INFO | } | 20260506 16:44:43.133 | INFO | Perf report for the last 5.13702 seconds: | 20260506 16:44:43.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:44:43.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:43.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:43.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:43.133 | INFO | ANALYSIS : 62.8%, 2.9, 95.6ms, 0 | 20260506 16:44:43.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:43.133 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 16:44:43.133 | INFO | PRE_HQ : 60.5%, 1.9, 58.4ms, 0 | 20260506 16:44:43.133 | INFO | BAZWRITER : 72.7%, 4.8, 179.0ms, 0 | 20260506 16:44:43.133 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260506 16:44:43.134 | INFO | Uploaded 36GiB to GPU in 2.0336s (17.7026GiB/s) | 20260506 16:44:43.134 | INFO | Downloaded 38GiB to GPU in 2.20702s (17.2178GiB/s) | 20260506 16:44:48.275 | INFO | Analyzing chunk frames = [2888192,2888704) | 20260506 16:44:48.275 | INFO | MemInfo: { | 20260506 16:44:48.275 | INFO | "gpu" : " 17 GB", | 20260506 16:44:48.275 | INFO | "huge" : "", | 20260506 16:44:48.275 | INFO | "huge_shared" : " 27 GB", | 20260506 16:44:48.275 | INFO | "malloc" : " 14 GB", | 20260506 16:44:48.275 | INFO | "pinned" : " 29 GB" | 20260506 16:44:48.276 | INFO | } | 20260506 16:44:48.276 | INFO | Perf report for the last 5.14223 seconds: | 20260506 16:44:48.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:44:48.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:48.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:48.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:48.276 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:44:48.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:48.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:48.276 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:44:48.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:48.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:48.276 | INFO | Uploaded 36GiB to GPU in 1.99326s (18.0609GiB/s) | 20260506 16:44:48.276 | INFO | Downloaded 34GiB to GPU in 1.92936s (17.6224GiB/s) | 20260506 16:44:53.415 | INFO | Analyzing chunk frames = [2888704,2889216) | 20260506 16:44:53.415 | INFO | MemInfo: { | 20260506 16:44:53.415 | INFO | "gpu" : " 17 GB", | 20260506 16:44:53.415 | INFO | "huge" : "", | 20260506 16:44:53.415 | INFO | "huge_shared" : " 27 GB", | 20260506 16:44:53.415 | INFO | "malloc" : " 14 GB", | 20260506 16:44:53.415 | INFO | "pinned" : " 29 GB" | 20260506 16:44:53.415 | INFO | } | 20260506 16:44:53.415 | INFO | Perf report for the last 5.13987 seconds: | 20260506 16:44:53.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:44:53.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:53.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:53.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:53.416 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 16:44:53.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:53.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:53.416 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:44:53.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:53.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:53.416 | INFO | Uploaded 36GiB to GPU in 1.97571s (18.2213GiB/s) | 20260506 16:44:53.416 | INFO | Downloaded 34GiB to GPU in 1.92061s (17.7027GiB/s) | 20260506 16:44:58.551 | INFO | Analyzing chunk frames = [2889216,2889728) | 20260506 16:44:58.551 | INFO | MemInfo: { | 20260506 16:44:58.551 | INFO | "gpu" : " 17 GB", | 20260506 16:44:58.551 | INFO | "huge" : "", | 20260506 16:44:58.551 | INFO | "huge_shared" : " 27 GB", | 20260506 16:44:58.551 | INFO | "malloc" : " 14 GB", | 20260506 16:44:58.551 | INFO | "pinned" : " 29 GB" | 20260506 16:44:58.551 | INFO | } | 20260506 16:44:58.551 | INFO | Perf report for the last 5.13552 seconds: | 20260506 16:44:58.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:44:58.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:58.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:58.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:58.551 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260506 16:44:58.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:58.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:58.551 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:44:58.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:58.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:44:58.551 | INFO | Uploaded 36GiB to GPU in 1.95179s (18.4446GiB/s) | 20260506 16:44:58.551 | INFO | Downloaded 34GiB to GPU in 1.87743s (18.1099GiB/s) | 20260506 16:45:03.690 | INFO | Analyzing chunk frames = [2889728,2890240) | 20260506 16:45:03.690 | INFO | MemInfo: { | 20260506 16:45:03.690 | INFO | "gpu" : " 17 GB", | 20260506 16:45:03.690 | INFO | "huge" : "", | 20260506 16:45:03.690 | INFO | "huge_shared" : " 27 GB", | 20260506 16:45:03.690 | INFO | "malloc" : " 14 GB", | 20260506 16:45:03.690 | INFO | "pinned" : " 29 GB" | 20260506 16:45:03.690 | INFO | } | 20260506 16:45:03.690 | INFO | Perf report for the last 5.13908 seconds: | 20260506 16:45:03.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:45:03.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:03.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:03.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:03.690 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 16:45:03.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:03.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:03.690 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:45:03.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:03.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:03.690 | INFO | Uploaded 36GiB to GPU in 2.01889s (17.8316GiB/s) | 20260506 16:45:03.690 | INFO | Downloaded 34GiB to GPU in 1.90688s (17.8302GiB/s) | 20260506 16:45:08.828 | INFO | Analyzing chunk frames = [2890240,2890752) | 20260506 16:45:08.828 | INFO | MemInfo: { | 20260506 16:45:08.828 | INFO | "gpu" : " 17 GB", | 20260506 16:45:08.828 | INFO | "huge" : "", | 20260506 16:45:08.828 | INFO | "huge_shared" : " 27 GB", | 20260506 16:45:08.828 | INFO | "malloc" : " 14 GB", | 20260506 16:45:08.828 | INFO | "pinned" : " 29 GB" | 20260506 16:45:08.828 | INFO | } | 20260506 16:45:08.828 | INFO | Perf report for the last 5.13819 seconds: | 20260506 16:45:08.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:45:08.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:08.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:08.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:08.828 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 16:45:08.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:08.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:08.828 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:45:08.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:08.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:08.829 | INFO | Uploaded 36GiB to GPU in 2.02896s (17.7431GiB/s) | 20260506 16:45:08.829 | INFO | Downloaded 34GiB to GPU in 1.93843s (17.5399GiB/s) | 20260506 16:45:13.965 | INFO | Analyzing chunk frames = [2890752,2891264) | 20260506 16:45:13.965 | INFO | MemInfo: { | 20260506 16:45:13.965 | INFO | "gpu" : " 17 GB", | 20260506 16:45:13.965 | INFO | "huge" : "", | 20260506 16:45:13.965 | INFO | "huge_shared" : " 27 GB", | 20260506 16:45:13.965 | INFO | "malloc" : " 14 GB", | 20260506 16:45:13.965 | INFO | "pinned" : " 29 GB" | 20260506 16:45:13.965 | INFO | } | 20260506 16:45:13.965 | INFO | Perf report for the last 5.13719 seconds: | 20260506 16:45:13.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:45:13.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:13.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:13.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:13.966 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 16:45:13.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:13.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:13.966 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:45:13.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:13.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:13.966 | INFO | Uploaded 36GiB to GPU in 2.03235s (17.7135GiB/s) | 20260506 16:45:13.966 | INFO | Downloaded 34GiB to GPU in 2.01584s (16.8664GiB/s) | 20260506 16:45:19.108 | INFO | Analyzing chunk frames = [2891264,2891776) | 20260506 16:45:19.108 | INFO | MemInfo: { | 20260506 16:45:19.108 | INFO | "gpu" : " 17 GB", | 20260506 16:45:19.108 | INFO | "huge" : "", | 20260506 16:45:19.108 | INFO | "huge_shared" : " 27 GB", | 20260506 16:45:19.108 | INFO | "malloc" : " 14 GB", | 20260506 16:45:19.108 | INFO | "pinned" : " 29 GB" | 20260506 16:45:19.108 | INFO | } | 20260506 16:45:19.108 | INFO | Perf report for the last 5.14268 seconds: | 20260506 16:45:19.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:45:19.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:19.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:19.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:19.108 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 16:45:19.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:19.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:19.108 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260506 16:45:19.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:19.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:19.108 | INFO | Uploaded 36GiB to GPU in 2.05209s (17.5431GiB/s) | 20260506 16:45:19.108 | INFO | Downloaded 34GiB to GPU in 1.9723s (17.2387GiB/s) | 20260506 16:45:19.108 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:45:19.108 | INFO | Total mem usage: 18 GiB | 20260506 16:45:19.108 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:45:19.108 | INFO | Recent High Water Marks Sum: 16.3 GiB (Savings: -10.1%) | 20260506 16:45:19.108 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:45:19.108 | INFO | High Water Breakdown: | 20260506 16:45:19.108 | INFO | 62% : 10.1 GiB ( 88.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:45:19.108 | INFO | 37.9% : 6.19 GiB ( 92.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:45:19.109 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:45:19.109 | INFO | ------------------------------------------ | 20260506 16:45:19.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:45:19.109 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:45:19.109 | INFO | Total mem usage: 26.5 GiB | 20260506 16:45:19.109 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:45:19.109 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:45:19.109 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:45:19.109 | INFO | High Water Breakdown: | 20260506 16:45:19.109 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:45:19.109 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:45:19.109 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:45:19.109 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:45:19.109 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:45:19.109 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:45:19.109 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:45:19.109 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:45:19.109 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:45:19.109 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:45:19.109 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:45:19.109 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:45:19.109 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:45:19.109 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:45:19.109 | INFO | ------------------------------------------ | 20260506 16:45:19.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:45:19.109 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:45:19.109 | INFO | Total mem usage: 25.6 GiB | 20260506 16:45:19.109 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:45:19.109 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:45:19.109 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:45:19.109 | INFO | High Water Breakdown: | 20260506 16:45:19.109 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:45:19.109 | INFO | ------------------------------------------ | 20260506 16:45:19.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:45:19.109 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:45:19.109 | INFO | Total mem usage: 15.7 GiB | 20260506 16:45:19.109 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:45:19.109 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:45:19.109 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:45:19.109 | INFO | High Water Breakdown: | 20260506 16:45:19.109 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:45:19.109 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:45:19.109 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:45:19.109 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:45:19.109 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:45:19.109 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:45:19.109 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:45:19.109 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:45:19.109 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:45:19.109 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:45:19.109 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:45:19.109 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:45:19.109 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:45:19.109 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:45:19.109 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:45:19.109 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:45:19.109 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:45:19.109 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:45:19.109 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:45:19.109 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:45:19.109 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:45:19.109 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:45:19.109 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:45:19.109 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:45:19.109 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:45:19.109 | INFO | ------------------------------------------ | 20260506 16:45:19.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:45:19.109 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:45:19.109 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:45:19.109 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:45:19.109 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.60e+01, 1.35e+01 / 2.76e+00 | 20260506 16:45:19.109 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.02e-01, 9.87e-03 / 6.64e-03 | 20260506 16:45:19.109 | INFO | PerfReport: 11.59%, Baseline -- 3.64e+00 / 4.43e+00, 5.66e-01 / 3.03e-01 | 20260506 16:45:19.109 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.48e+00, 1.91e-01 / 1.35e-01 | 20260506 16:45:19.109 | INFO | PerfReport: 15.95%, DME -- 4.01e+01 / 4.45e+01, 4.08e+00 / 2.93e+00 | 20260506 16:45:19.109 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.47e+00 / 1.95e+00 | 20260506 16:45:19.109 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.28e+00 / 2.83e+00, 3.78e-01 / 2.43e-01 | 20260506 16:45:19.109 | INFO | PerfReport: 3.48%, Metrics -- 1.09e+00 / 1.45e+00, 2.87e-01 / 1.92e-01 | 20260506 16:45:19.109 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.79e-01 / 4.26e-01, 5.94e-02 / 2.21e-02 | 20260506 16:45:19.109 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:45:19.109 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:45:19.109 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:45:19.109 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 16:45:19.109 | INFO | PerfReport: 21.48%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 16:45:19.109 | INFO | PerfReport: 21.09%, Download -- 2.05e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 16:45:19.109 | INFO | PerfReport: 32.25%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 16:45:19.109 | INFO | PerfReport: 1.46%, LockUpload -- 1.42e+00 / 0.00e+00, 5.85e+00 / 0.00e+00 | 20260506 16:45:19.109 | INFO | PerfReport: 1.06%, LockDownload -- 1.03e+00 / 0.00e+00, 2.69e+00 / 0.00e+00 | 20260506 16:45:19.109 | INFO | PerfReport: 22.65%, LockCompute -- 2.21e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 16:45:19.109 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260506 16:45:19.220 | INFO | Starting a round of Baz IO | 20260506 16:45:22.376 | INFO | Populating RTMetrics report | 20260506 16:45:22.377 | INFO | RTMetrics report populated | 20260506 16:45:22.377 | INFO | Opening tmp RTMetrics json file | 20260506 16:45:22.388 | INFO | Serializing RTMetrics to json | 20260506 16:45:22.389 | INFO | Serialization done, swapping tmp file now... | 20260506 16:45:22.391 | INFO | RTMetrics done | 20260506 16:45:22.659 | INFO | Finished a round of Baz IO: Wrote 2.08 GiB in 3.438 seconds | 20260506 16:45:22.659 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.11% / 0.01% / 63.88% | 20260506 16:45:22.659 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.58237% / 1.19209e-05% / 3.57628e-05% | 20260506 16:45:22.659 | INFO | Baz IO overall: | 20260506 16:45:22.659 | INFO | BytesWritten : 1520317446921 | 20260506 16:45:22.659 | INFO | NumEvents : 735635739323 | 20260506 16:45:24.248 | INFO | Analyzing chunk frames = [2891776,2892288) | 20260506 16:45:24.248 | INFO | MemInfo: { | 20260506 16:45:24.248 | INFO | "gpu" : " 17 GB", | 20260506 16:45:24.248 | INFO | "huge" : "", | 20260506 16:45:24.248 | INFO | "huge_shared" : " 27 GB", | 20260506 16:45:24.248 | INFO | "malloc" : " 14 GB", | 20260506 16:45:24.248 | INFO | "pinned" : " 29 GB" | 20260506 16:45:24.248 | INFO | } | 20260506 16:45:24.248 | INFO | Perf report for the last 5.14006 seconds: | 20260506 16:45:24.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:45:24.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:24.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:24.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:24.248 | INFO | ANALYSIS : 63.0%, 2.9, 95.3ms, 0 | 20260506 16:45:24.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:24.248 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260506 16:45:24.248 | INFO | PRE_HQ : 60.9%, 1.9, 59.7ms, 0 | 20260506 16:45:24.248 | INFO | BAZWRITER : 66.2%, 4.8, 162.8ms, 0 | 20260506 16:45:24.248 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 16:45:24.248 | INFO | Uploaded 36GiB to GPU in 2.07822s (17.3225GiB/s) | 20260506 16:45:24.248 | INFO | Downloaded 38GiB to GPU in 2.22368s (17.0888GiB/s) | 20260506 16:45:29.388 | INFO | Analyzing chunk frames = [2892288,2892800) | 20260506 16:45:29.388 | INFO | MemInfo: { | 20260506 16:45:29.388 | INFO | "gpu" : " 17 GB", | 20260506 16:45:29.388 | INFO | "huge" : "", | 20260506 16:45:29.388 | INFO | "huge_shared" : " 27 GB", | 20260506 16:45:29.388 | INFO | "malloc" : " 14 GB", | 20260506 16:45:29.388 | INFO | "pinned" : " 29 GB" | 20260506 16:45:29.388 | INFO | } | 20260506 16:45:29.388 | INFO | Perf report for the last 5.14004 seconds: | 20260506 16:45:29.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:45:29.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:29.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:29.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:29.388 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 16:45:29.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:29.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:29.388 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:45:29.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:29.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:29.389 | INFO | Uploaded 36GiB to GPU in 2.02259s (17.799GiB/s) | 20260506 16:45:29.389 | INFO | Downloaded 34GiB to GPU in 1.95339s (17.4056GiB/s) | 20260506 16:45:34.526 | INFO | Analyzing chunk frames = [2892800,2893312) | 20260506 16:45:34.526 | INFO | MemInfo: { | 20260506 16:45:34.526 | INFO | "gpu" : " 17 GB", | 20260506 16:45:34.526 | INFO | "huge" : "", | 20260506 16:45:34.526 | INFO | "huge_shared" : " 27 GB", | 20260506 16:45:34.526 | INFO | "malloc" : " 14 GB", | 20260506 16:45:34.526 | INFO | "pinned" : " 29 GB" | 20260506 16:45:34.526 | INFO | } | 20260506 16:45:34.526 | INFO | Perf report for the last 5.13776 seconds: | 20260506 16:45:34.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:45:34.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:34.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:34.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:34.526 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 16:45:34.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:34.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:34.526 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:45:34.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:34.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:34.526 | INFO | Uploaded 36GiB to GPU in 1.98346s (18.1501GiB/s) | 20260506 16:45:34.526 | INFO | Downloaded 34GiB to GPU in 1.93115s (17.6061GiB/s) | 20260506 16:45:39.666 | INFO | Analyzing chunk frames = [2893312,2893824) | 20260506 16:45:39.666 | INFO | MemInfo: { | 20260506 16:45:39.666 | INFO | "gpu" : " 17 GB", | 20260506 16:45:39.666 | INFO | "huge" : "", | 20260506 16:45:39.667 | INFO | "huge_shared" : " 27 GB", | 20260506 16:45:39.667 | INFO | "malloc" : " 14 GB", | 20260506 16:45:39.667 | INFO | "pinned" : " 29 GB" | 20260506 16:45:39.667 | INFO | } | 20260506 16:45:39.667 | INFO | Perf report for the last 5.14056 seconds: | 20260506 16:45:39.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:45:39.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:39.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:39.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:39.667 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 16:45:39.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:39.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:39.667 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:45:39.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:39.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:39.667 | INFO | Uploaded 36GiB to GPU in 1.93881s (18.5681GiB/s) | 20260506 16:45:39.667 | INFO | Downloaded 34GiB to GPU in 1.87906s (18.0941GiB/s) | 20260506 16:45:44.807 | INFO | Analyzing chunk frames = [2893824,2894336) | 20260506 16:45:44.807 | INFO | MemInfo: { | 20260506 16:45:44.807 | INFO | "gpu" : " 17 GB", | 20260506 16:45:44.807 | INFO | "huge" : "", | 20260506 16:45:44.807 | INFO | "huge_shared" : " 27 GB", | 20260506 16:45:44.807 | INFO | "malloc" : " 14 GB", | 20260506 16:45:44.807 | INFO | "pinned" : " 29 GB" | 20260506 16:45:44.807 | INFO | } | 20260506 16:45:44.807 | INFO | Perf report for the last 5.14067 seconds: | 20260506 16:45:44.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:45:44.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:44.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:44.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:44.807 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 16:45:44.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:44.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:44.807 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:45:44.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:44.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:44.808 | INFO | Uploaded 36GiB to GPU in 1.9544s (18.42GiB/s) | 20260506 16:45:44.808 | INFO | Downloaded 34GiB to GPU in 1.87657s (18.1182GiB/s) | 20260506 16:45:49.943 | INFO | Analyzing chunk frames = [2894336,2894848) | 20260506 16:45:49.943 | INFO | MemInfo: { | 20260506 16:45:49.943 | INFO | "gpu" : " 17 GB", | 20260506 16:45:49.943 | INFO | "huge" : "", | 20260506 16:45:49.944 | INFO | "huge_shared" : " 27 GB", | 20260506 16:45:49.944 | INFO | "malloc" : " 14 GB", | 20260506 16:45:49.944 | INFO | "pinned" : " 29 GB" | 20260506 16:45:49.944 | INFO | } | 20260506 16:45:49.944 | INFO | Perf report for the last 5.13625 seconds: | 20260506 16:45:49.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:45:49.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:49.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:49.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:49.944 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 16:45:49.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:49.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:49.944 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:45:49.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:49.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:49.944 | INFO | Uploaded 36GiB to GPU in 2.01598s (17.8574GiB/s) | 20260506 16:45:49.944 | INFO | Downloaded 34GiB to GPU in 1.9401s (17.5249GiB/s) | 20260506 16:45:55.082 | INFO | Analyzing chunk frames = [2894848,2895360) | 20260506 16:45:55.082 | INFO | MemInfo: { | 20260506 16:45:55.082 | INFO | "gpu" : " 17 GB", | 20260506 16:45:55.082 | INFO | "huge" : "", | 20260506 16:45:55.082 | INFO | "huge_shared" : " 27 GB", | 20260506 16:45:55.082 | INFO | "malloc" : " 14 GB", | 20260506 16:45:55.082 | INFO | "pinned" : " 29 GB" | 20260506 16:45:55.082 | INFO | } | 20260506 16:45:55.082 | INFO | Perf report for the last 5.13886 seconds: | 20260506 16:45:55.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:45:55.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:55.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:55.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:55.082 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:45:55.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:55.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:55.083 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:45:55.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:55.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:45:55.083 | INFO | Uploaded 36GiB to GPU in 1.99828s (18.0155GiB/s) | 20260506 16:45:55.083 | INFO | Downloaded 34GiB to GPU in 2.01017s (16.914GiB/s) | 20260506 16:46:00.223 | INFO | Analyzing chunk frames = [2895360,2895872) | 20260506 16:46:00.223 | INFO | MemInfo: { | 20260506 16:46:00.223 | INFO | "gpu" : " 17 GB", | 20260506 16:46:00.223 | INFO | "huge" : "", | 20260506 16:46:00.223 | INFO | "huge_shared" : " 27 GB", | 20260506 16:46:00.223 | INFO | "malloc" : " 14 GB", | 20260506 16:46:00.223 | INFO | "pinned" : " 29 GB" | 20260506 16:46:00.223 | INFO | } | 20260506 16:46:00.223 | INFO | Perf report for the last 5.14031 seconds: | 20260506 16:46:00.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:46:00.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:00.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:00.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:00.223 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:46:00.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:00.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:00.223 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 16:46:00.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:00.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:00.223 | INFO | Uploaded 36GiB to GPU in 2.00706s (17.9367GiB/s) | 20260506 16:46:00.223 | INFO | Downloaded 34GiB to GPU in 1.96506s (17.3022GiB/s) | 20260506 16:46:00.342 | INFO | Starting a round of Baz IO | 20260506 16:46:03.483 | INFO | Populating RTMetrics report | 20260506 16:46:03.483 | INFO | RTMetrics report populated | 20260506 16:46:03.483 | INFO | Opening tmp RTMetrics json file | 20260506 16:46:03.488 | INFO | Serializing RTMetrics to json | 20260506 16:46:03.493 | INFO | Serialization done, swapping tmp file now... | 20260506 16:46:03.496 | INFO | RTMetrics done | 20260506 16:46:03.744 | INFO | Finished a round of Baz IO: Wrote 2.079 GiB in 3.401 seconds | 20260506 16:46:03.744 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.10% / 0.01% / 63.89% | 20260506 16:46:03.744 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.57773% / 3.97364e-06% / 7.15256e-05% | 20260506 16:46:03.744 | INFO | Baz IO overall: | 20260506 16:46:03.744 | INFO | BytesWritten : 1522549856542 | 20260506 16:46:03.744 | INFO | NumEvents : 736714207102 | 20260506 16:46:05.365 | INFO | Analyzing chunk frames = [2895872,2896384) | 20260506 16:46:05.365 | INFO | MemInfo: { | 20260506 16:46:05.365 | INFO | "gpu" : " 17 GB", | 20260506 16:46:05.365 | INFO | "huge" : "", | 20260506 16:46:05.365 | INFO | "huge_shared" : " 27 GB", | 20260506 16:46:05.365 | INFO | "malloc" : " 14 GB", | 20260506 16:46:05.365 | INFO | "pinned" : " 29 GB" | 20260506 16:46:05.365 | INFO | } | 20260506 16:46:05.365 | INFO | Perf report for the last 5.14197 seconds: | 20260506 16:46:05.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:46:05.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:05.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:05.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:05.365 | INFO | ANALYSIS : 62.8%, 2.9, 95.6ms, 0 | 20260506 16:46:05.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:05.365 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 16:46:05.365 | INFO | PRE_HQ : 60.3%, 1.9, 58.7ms, 0 | 20260506 16:46:05.365 | INFO | BAZWRITER : 65.7%, 4.8, 161.9ms, 0 | 20260506 16:46:05.365 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 16:46:05.365 | INFO | Uploaded 36GiB to GPU in 2.05264s (17.5384GiB/s) | 20260506 16:46:05.365 | INFO | Downloaded 38GiB to GPU in 2.22477s (17.0804GiB/s) | 20260506 16:46:10.508 | INFO | Analyzing chunk frames = [2896384,2896896) | 20260506 16:46:10.508 | INFO | MemInfo: { | 20260506 16:46:10.508 | INFO | "gpu" : " 17 GB", | 20260506 16:46:10.508 | INFO | "huge" : "", | 20260506 16:46:10.508 | INFO | "huge_shared" : " 27 GB", | 20260506 16:46:10.508 | INFO | "malloc" : " 14 GB", | 20260506 16:46:10.508 | INFO | "pinned" : " 29 GB" | 20260506 16:46:10.508 | INFO | } | 20260506 16:46:10.508 | INFO | Perf report for the last 5.14326 seconds: | 20260506 16:46:10.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:46:10.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:10.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:10.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:10.508 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 16:46:10.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:10.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:10.508 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:46:10.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:10.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:10.508 | INFO | Uploaded 36GiB to GPU in 1.99151s (18.0768GiB/s) | 20260506 16:46:10.508 | INFO | Downloaded 34GiB to GPU in 1.93431s (17.5773GiB/s) | 20260506 16:46:15.643 | INFO | Analyzing chunk frames = [2896896,2897408) | 20260506 16:46:15.643 | INFO | MemInfo: { | 20260506 16:46:15.643 | INFO | "gpu" : " 17 GB", | 20260506 16:46:15.643 | INFO | "huge" : "", | 20260506 16:46:15.643 | INFO | "huge_shared" : " 27 GB", | 20260506 16:46:15.643 | INFO | "malloc" : " 14 GB", | 20260506 16:46:15.643 | INFO | "pinned" : " 29 GB" | 20260506 16:46:15.643 | INFO | } | 20260506 16:46:15.643 | INFO | Perf report for the last 5.13543 seconds: | 20260506 16:46:15.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:46:15.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:15.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:15.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:15.644 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 16:46:15.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:15.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:15.644 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 16:46:15.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:15.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:15.644 | INFO | Uploaded 36GiB to GPU in 1.97179s (18.2575GiB/s) | 20260506 16:46:15.644 | INFO | Downloaded 34GiB to GPU in 1.92188s (17.6911GiB/s) | 20260506 16:46:20.780 | INFO | Analyzing chunk frames = [2897408,2897920) | 20260506 16:46:20.780 | INFO | MemInfo: { | 20260506 16:46:20.780 | INFO | "gpu" : " 17 GB", | 20260506 16:46:20.780 | INFO | "huge" : "", | 20260506 16:46:20.780 | INFO | "huge_shared" : " 27 GB", | 20260506 16:46:20.780 | INFO | "malloc" : " 14 GB", | 20260506 16:46:20.780 | INFO | "pinned" : " 29 GB" | 20260506 16:46:20.780 | INFO | } | 20260506 16:46:20.780 | INFO | Perf report for the last 5.13635 seconds: | 20260506 16:46:20.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:46:20.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:20.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:20.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:20.780 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:46:20.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:20.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:20.780 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:46:20.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:20.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:20.780 | INFO | Uploaded 36GiB to GPU in 1.932s (18.6335GiB/s) | 20260506 16:46:20.780 | INFO | Downloaded 34GiB to GPU in 1.87101s (18.172GiB/s) | 20260506 16:46:25.914 | INFO | Analyzing chunk frames = [2897920,2898432) | 20260506 16:46:25.914 | INFO | MemInfo: { | 20260506 16:46:25.914 | INFO | "gpu" : " 17 GB", | 20260506 16:46:25.914 | INFO | "huge" : "", | 20260506 16:46:25.914 | INFO | "huge_shared" : " 27 GB", | 20260506 16:46:25.914 | INFO | "malloc" : " 14 GB", | 20260506 16:46:25.914 | INFO | "pinned" : " 29 GB" | 20260506 16:46:25.914 | INFO | } | 20260506 16:46:25.914 | INFO | Perf report for the last 5.13447 seconds: | 20260506 16:46:25.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:46:25.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:25.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:25.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:25.914 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 16:46:25.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:25.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:25.914 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:46:25.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:25.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:25.915 | INFO | Uploaded 36GiB to GPU in 2.01714s (17.8471GiB/s) | 20260506 16:46:25.915 | INFO | Downloaded 34GiB to GPU in 1.91155s (17.7866GiB/s) | 20260506 16:46:31.054 | INFO | Analyzing chunk frames = [2898432,2898944) | 20260506 16:46:31.054 | INFO | MemInfo: { | 20260506 16:46:31.054 | INFO | "gpu" : " 17 GB", | 20260506 16:46:31.054 | INFO | "huge" : "", | 20260506 16:46:31.054 | INFO | "huge_shared" : " 27 GB", | 20260506 16:46:31.054 | INFO | "malloc" : " 14 GB", | 20260506 16:46:31.054 | INFO | "pinned" : " 29 GB" | 20260506 16:46:31.054 | INFO | } | 20260506 16:46:31.054 | INFO | Perf report for the last 5.1394 seconds: | 20260506 16:46:31.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:46:31.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:31.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:31.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:31.054 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:46:31.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:31.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:31.054 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:46:31.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:31.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:31.054 | INFO | Uploaded 36GiB to GPU in 2.02462s (17.7811GiB/s) | 20260506 16:46:31.054 | INFO | Downloaded 34GiB to GPU in 1.95985s (17.3483GiB/s) | 20260506 16:46:36.199 | INFO | Analyzing chunk frames = [2898944,2899456) | 20260506 16:46:36.199 | INFO | MemInfo: { | 20260506 16:46:36.199 | INFO | "gpu" : " 17 GB", | 20260506 16:46:36.199 | INFO | "huge" : "", | 20260506 16:46:36.199 | INFO | "huge_shared" : " 27 GB", | 20260506 16:46:36.199 | INFO | "malloc" : " 14 GB", | 20260506 16:46:36.199 | INFO | "pinned" : " 29 GB" | 20260506 16:46:36.199 | INFO | } | 20260506 16:46:36.199 | INFO | Perf report for the last 5.1451 seconds: | 20260506 16:46:36.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:46:36.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:36.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:36.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:36.199 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 16:46:36.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:36.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:36.199 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 16:46:36.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:36.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:36.199 | INFO | Uploaded 36GiB to GPU in 2.02989s (17.735GiB/s) | 20260506 16:46:36.199 | INFO | Downloaded 34GiB to GPU in 2.02592s (16.7825GiB/s) | 20260506 16:46:41.339 | INFO | Analyzing chunk frames = [2899456,2899968) | 20260506 16:46:41.339 | INFO | MemInfo: { | 20260506 16:46:41.339 | INFO | "gpu" : " 17 GB", | 20260506 16:46:41.339 | INFO | "huge" : "", | 20260506 16:46:41.339 | INFO | "huge_shared" : " 27 GB", | 20260506 16:46:41.339 | INFO | "malloc" : " 14 GB", | 20260506 16:46:41.339 | INFO | "pinned" : " 29 GB" | 20260506 16:46:41.339 | INFO | } | 20260506 16:46:41.339 | INFO | Perf report for the last 5.13983 seconds: | 20260506 16:46:41.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:46:41.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:41.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:41.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:41.339 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:46:41.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:41.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:41.339 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:46:41.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:41.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:41.339 | INFO | Uploaded 36GiB to GPU in 1.99188s (18.0733GiB/s) | 20260506 16:46:41.339 | INFO | Downloaded 34GiB to GPU in 1.9664s (17.2905GiB/s) | 20260506 16:46:41.339 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:46:41.339 | INFO | Total mem usage: 18 GiB | 20260506 16:46:41.339 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:46:41.339 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -13.9%) | 20260506 16:46:41.339 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:46:41.339 | INFO | High Water Breakdown: | 20260506 16:46:41.339 | INFO | 61.5% : 9.72 GiB ( 85.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:46:41.339 | INFO | 38.4% : 6.06 GiB ( 90.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:46:41.339 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:46:41.339 | INFO | ------------------------------------------ | 20260506 16:46:41.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:46:41.339 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:46:41.339 | INFO | Total mem usage: 26.5 GiB | 20260506 16:46:41.339 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:46:41.339 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:46:41.339 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:46:41.339 | INFO | High Water Breakdown: | 20260506 16:46:41.339 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:46:41.339 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:46:41.339 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:46:41.339 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:46:41.339 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:46:41.339 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:46:41.339 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:46:41.339 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:46:41.339 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:46:41.339 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:46:41.339 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:46:41.339 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:46:41.339 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:46:41.339 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:46:41.339 | INFO | ------------------------------------------ | 20260506 16:46:41.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:46:41.339 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:46:41.339 | INFO | Total mem usage: 25.6 GiB | 20260506 16:46:41.339 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:46:41.339 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:46:41.339 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:46:41.339 | INFO | High Water Breakdown: | 20260506 16:46:41.339 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:46:41.340 | INFO | ------------------------------------------ | 20260506 16:46:41.340 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:46:41.340 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:46:41.340 | INFO | Total mem usage: 15.7 GiB | 20260506 16:46:41.340 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:46:41.340 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:46:41.340 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:46:41.340 | INFO | High Water Breakdown: | 20260506 16:46:41.340 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:46:41.340 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:46:41.340 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:46:41.340 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:46:41.340 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:46:41.340 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:46:41.340 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:46:41.340 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:46:41.340 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:46:41.340 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:46:41.340 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:46:41.340 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:46:41.340 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:46:41.340 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:46:41.340 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:46:41.340 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:46:41.340 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:46:41.340 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:46:41.340 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:46:41.340 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:46:41.340 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:46:41.340 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:46:41.340 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:46:41.340 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:46:41.340 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:46:41.340 | INFO | ------------------------------------------ | 20260506 16:46:41.340 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:46:41.340 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:46:41.340 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:46:41.340 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:46:41.340 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.57e+01, 1.34e+01 / 2.83e+00 | 20260506 16:46:41.340 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.00e-01, 1.04e-02 / 5.86e-03 | 20260506 16:46:41.340 | INFO | PerfReport: 11.59%, Baseline -- 3.64e+00 / 4.38e+00, 5.64e-01 / 3.78e-01 | 20260506 16:46:41.340 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.45e+00, 1.86e-01 / 1.51e-01 | 20260506 16:46:41.340 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.47e+01, 4.16e+00 / 3.28e+00 | 20260506 16:46:41.340 | INFO | PerfReport: 56.89%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.40e+00 / 1.60e+00 | 20260506 16:46:41.340 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.29e+00 / 2.90e+00, 3.93e-01 / 2.10e-01 | 20260506 16:46:41.340 | INFO | PerfReport: 3.48%, Metrics -- 1.09e+00 / 1.45e+00, 2.82e-01 / 1.92e-01 | 20260506 16:46:41.340 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.78e-01 / 4.27e-01, 5.93e-02 / 2.26e-02 | 20260506 16:46:41.340 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:46:41.340 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:46:41.340 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:46:41.340 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 16:46:41.340 | INFO | PerfReport: 21.42%, Upload -- 2.08e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260506 16:46:41.340 | INFO | PerfReport: 21.15%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 16:46:41.340 | INFO | PerfReport: 32.25%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 16:46:41.340 | INFO | PerfReport: 1.40%, LockUpload -- 1.36e+00 / 0.00e+00, 5.77e+00 / 0.00e+00 | 20260506 16:46:41.340 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260506 16:46:41.340 | INFO | PerfReport: 22.66%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 16:46:41.340 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260506 16:46:41.453 | INFO | Starting a round of Baz IO | 20260506 16:46:44.601 | INFO | Populating RTMetrics report | 20260506 16:46:44.601 | INFO | RTMetrics report populated | 20260506 16:46:44.601 | INFO | Opening tmp RTMetrics json file | 20260506 16:46:44.604 | INFO | Serializing RTMetrics to json | 20260506 16:46:44.636 | INFO | Serialization done, swapping tmp file now... | 20260506 16:46:44.652 | INFO | RTMetrics done | 20260506 16:46:44.767 | INFO | Finished a round of Baz IO: Wrote 2.079 GiB in 3.314 seconds | 20260506 16:46:44.767 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.09% / 0.01% / 63.90% | 20260506 16:46:44.767 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.57259% / 1.19209e-05% / 5.5631e-05% | 20260506 16:46:44.767 | INFO | Baz IO overall: | 20260506 16:46:44.767 | INFO | BytesWritten : 1524781734023 | 20260506 16:46:44.767 | INFO | NumEvents : 737792301567 | 20260506 16:46:46.483 | INFO | Analyzing chunk frames = [2899968,2900480) | 20260506 16:46:46.483 | INFO | MemInfo: { | 20260506 16:46:46.483 | INFO | "gpu" : " 17 GB", | 20260506 16:46:46.483 | INFO | "huge" : "", | 20260506 16:46:46.483 | INFO | "huge_shared" : " 27 GB", | 20260506 16:46:46.483 | INFO | "malloc" : " 14 GB", | 20260506 16:46:46.483 | INFO | "pinned" : " 29 GB" | 20260506 16:46:46.483 | INFO | } | 20260506 16:46:46.483 | INFO | Perf report for the last 5.1446 seconds: | 20260506 16:46:46.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:46:46.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:46.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:46.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:46.483 | INFO | ANALYSIS : 62.8%, 2.9, 95.6ms, 0 | 20260506 16:46:46.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:46.483 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260506 16:46:46.483 | INFO | PRE_HQ : 60.3%, 1.9, 58.6ms, 0 | 20260506 16:46:46.483 | INFO | BAZWRITER : 64.0%, 4.5, 149.5ms, 0 | 20260506 16:46:46.483 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 16:46:46.484 | INFO | Uploaded 36GiB to GPU in 2.0455s (17.5996GiB/s) | 20260506 16:46:46.484 | INFO | Downloaded 38GiB to GPU in 2.21465s (17.1585GiB/s) | 20260506 16:46:51.620 | INFO | Analyzing chunk frames = [2900480,2900992) | 20260506 16:46:51.620 | INFO | MemInfo: { | 20260506 16:46:51.620 | INFO | "gpu" : " 17 GB", | 20260506 16:46:51.620 | INFO | "huge" : "", | 20260506 16:46:51.620 | INFO | "huge_shared" : " 27 GB", | 20260506 16:46:51.620 | INFO | "malloc" : " 14 GB", | 20260506 16:46:51.620 | INFO | "pinned" : " 29 GB" | 20260506 16:46:51.620 | INFO | } | 20260506 16:46:51.620 | INFO | Perf report for the last 5.13676 seconds: | 20260506 16:46:51.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:46:51.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:51.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:51.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:51.620 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 16:46:51.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:51.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:51.620 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 16:46:51.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:51.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:51.620 | INFO | Uploaded 36GiB to GPU in 2.00467s (17.9581GiB/s) | 20260506 16:46:51.620 | INFO | Downloaded 34GiB to GPU in 1.92575s (17.6554GiB/s) | 20260506 16:46:56.757 | INFO | Analyzing chunk frames = [2900992,2901504) | 20260506 16:46:56.757 | INFO | MemInfo: { | 20260506 16:46:56.757 | INFO | "gpu" : " 17 GB", | 20260506 16:46:56.757 | INFO | "huge" : "", | 20260506 16:46:56.757 | INFO | "huge_shared" : " 27 GB", | 20260506 16:46:56.757 | INFO | "malloc" : " 14 GB", | 20260506 16:46:56.757 | INFO | "pinned" : " 29 GB" | 20260506 16:46:56.757 | INFO | } | 20260506 16:46:56.757 | INFO | Perf report for the last 5.13706 seconds: | 20260506 16:46:56.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:46:56.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:56.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:56.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:56.757 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 16:46:56.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:56.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:56.757 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:46:56.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:56.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:46:56.757 | INFO | Uploaded 36GiB to GPU in 1.9591s (18.3758GiB/s) | 20260506 16:46:56.757 | INFO | Downloaded 34GiB to GPU in 1.92442s (17.6677GiB/s) | 20260506 16:47:01.895 | INFO | Analyzing chunk frames = [2901504,2902016) | 20260506 16:47:01.895 | INFO | MemInfo: { | 20260506 16:47:01.895 | INFO | "gpu" : " 17 GB", | 20260506 16:47:01.895 | INFO | "huge" : "", | 20260506 16:47:01.895 | INFO | "huge_shared" : " 27 GB", | 20260506 16:47:01.895 | INFO | "malloc" : " 14 GB", | 20260506 16:47:01.895 | INFO | "pinned" : " 29 GB" | 20260506 16:47:01.896 | INFO | } | 20260506 16:47:01.896 | INFO | Perf report for the last 5.1384 seconds: | 20260506 16:47:01.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:47:01.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:01.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:01.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:01.896 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260506 16:47:01.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:01.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:01.896 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:47:01.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:01.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:01.896 | INFO | Uploaded 36GiB to GPU in 1.93748s (18.5809GiB/s) | 20260506 16:47:01.896 | INFO | Downloaded 34GiB to GPU in 1.87456s (18.1376GiB/s) | 20260506 16:47:07.033 | INFO | Analyzing chunk frames = [2902016,2902528) | 20260506 16:47:07.033 | INFO | MemInfo: { | 20260506 16:47:07.033 | INFO | "gpu" : " 17 GB", | 20260506 16:47:07.033 | INFO | "huge" : "", | 20260506 16:47:07.033 | INFO | "huge_shared" : " 27 GB", | 20260506 16:47:07.033 | INFO | "malloc" : " 14 GB", | 20260506 16:47:07.033 | INFO | "pinned" : " 29 GB" | 20260506 16:47:07.033 | INFO | } | 20260506 16:47:07.033 | INFO | Perf report for the last 5.13722 seconds: | 20260506 16:47:07.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:47:07.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:07.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:07.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:07.033 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 16:47:07.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:07.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:07.033 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:47:07.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:07.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:07.033 | INFO | Uploaded 36GiB to GPU in 1.95561s (18.4086GiB/s) | 20260506 16:47:07.033 | INFO | Downloaded 34GiB to GPU in 1.8819s (18.0668GiB/s) | 20260506 16:47:12.173 | INFO | Analyzing chunk frames = [2902528,2903040) | 20260506 16:47:12.173 | INFO | MemInfo: { | 20260506 16:47:12.173 | INFO | "gpu" : " 17 GB", | 20260506 16:47:12.173 | INFO | "huge" : "", | 20260506 16:47:12.173 | INFO | "huge_shared" : " 27 GB", | 20260506 16:47:12.173 | INFO | "malloc" : " 14 GB", | 20260506 16:47:12.173 | INFO | "pinned" : " 29 GB" | 20260506 16:47:12.173 | INFO | } | 20260506 16:47:12.173 | INFO | Perf report for the last 5.14025 seconds: | 20260506 16:47:12.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:47:12.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:12.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:12.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:12.173 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 16:47:12.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:12.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:12.173 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:47:12.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:12.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:12.173 | INFO | Uploaded 36GiB to GPU in 2.02782s (17.7531GiB/s) | 20260506 16:47:12.173 | INFO | Downloaded 34GiB to GPU in 1.94278s (17.5007GiB/s) | 20260506 16:47:17.318 | INFO | Analyzing chunk frames = [2903040,2903552) | 20260506 16:47:17.318 | INFO | MemInfo: { | 20260506 16:47:17.318 | INFO | "gpu" : " 17 GB", | 20260506 16:47:17.318 | INFO | "huge" : "", | 20260506 16:47:17.319 | INFO | "huge_shared" : " 27 GB", | 20260506 16:47:17.319 | INFO | "malloc" : " 14 GB", | 20260506 16:47:17.319 | INFO | "pinned" : " 29 GB" | 20260506 16:47:17.319 | INFO | } | 20260506 16:47:17.319 | INFO | Perf report for the last 5.14559 seconds: | 20260506 16:47:17.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:47:17.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:17.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:17.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:17.319 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 16:47:17.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:17.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:17.319 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 16:47:17.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:17.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:17.319 | INFO | Uploaded 36GiB to GPU in 2.02525s (17.7756GiB/s) | 20260506 16:47:17.319 | INFO | Downloaded 34GiB to GPU in 1.99798s (17.0172GiB/s) | 20260506 16:47:22.456 | INFO | Analyzing chunk frames = [2903552,2904064) | 20260506 16:47:22.456 | INFO | MemInfo: { | 20260506 16:47:22.456 | INFO | "gpu" : " 17 GB", | 20260506 16:47:22.456 | INFO | "huge" : "", | 20260506 16:47:22.456 | INFO | "huge_shared" : " 27 GB", | 20260506 16:47:22.456 | INFO | "malloc" : " 14 GB", | 20260506 16:47:22.456 | INFO | "pinned" : " 29 GB" | 20260506 16:47:22.456 | INFO | } | 20260506 16:47:22.456 | INFO | Perf report for the last 5.13777 seconds: | 20260506 16:47:22.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:47:22.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:22.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:22.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:22.456 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 16:47:22.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:22.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:22.456 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 16:47:22.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:22.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:22.457 | INFO | Uploaded 36GiB to GPU in 2.01868s (17.8334GiB/s) | 20260506 16:47:22.457 | INFO | Downloaded 34GiB to GPU in 1.98651s (17.1155GiB/s) | 20260506 16:47:22.577 | INFO | Starting a round of Baz IO | 20260506 16:47:25.717 | INFO | Populating RTMetrics report | 20260506 16:47:25.717 | INFO | RTMetrics report populated | 20260506 16:47:25.717 | INFO | Opening tmp RTMetrics json file | 20260506 16:47:25.718 | INFO | Serializing RTMetrics to json | 20260506 16:47:25.722 | INFO | Serialization done, swapping tmp file now... | 20260506 16:47:25.724 | INFO | RTMetrics done | 20260506 16:47:25.867 | INFO | Finished a round of Baz IO: Wrote 2.078 GiB in 3.29 seconds | 20260506 16:47:25.867 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.08% / 0.01% / 63.91% | 20260506 16:47:25.867 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.56914% / 7.94729e-06% / 5.5631e-05% | 20260506 16:47:25.867 | INFO | Baz IO overall: | 20260506 16:47:25.867 | INFO | BytesWritten : 1527012689707 | 20260506 16:47:25.867 | INFO | NumEvents : 738869955509 | 20260506 16:47:27.595 | INFO | Analyzing chunk frames = [2904064,2904576) | 20260506 16:47:27.595 | INFO | MemInfo: { | 20260506 16:47:27.595 | INFO | "gpu" : " 17 GB", | 20260506 16:47:27.595 | INFO | "huge" : "", | 20260506 16:47:27.595 | INFO | "huge_shared" : " 27 GB", | 20260506 16:47:27.595 | INFO | "malloc" : " 14 GB", | 20260506 16:47:27.595 | INFO | "pinned" : " 29 GB" | 20260506 16:47:27.595 | INFO | } | 20260506 16:47:27.595 | INFO | Perf report for the last 5.13875 seconds: | 20260506 16:47:27.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:47:27.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:27.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:27.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:27.595 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 16:47:27.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:27.595 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 16:47:27.595 | INFO | PRE_HQ : 60.3%, 1.9, 58.8ms, 0 | 20260506 16:47:27.595 | INFO | BAZWRITER : 63.6%, 4.5, 147.8ms, 0 | 20260506 16:47:27.595 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260506 16:47:27.595 | INFO | Uploaded 36GiB to GPU in 2.064s (17.4419GiB/s) | 20260506 16:47:27.595 | INFO | Downloaded 38GiB to GPU in 2.22382s (17.0877GiB/s) | 20260506 16:47:32.730 | INFO | Analyzing chunk frames = [2904576,2905088) | 20260506 16:47:32.730 | INFO | MemInfo: { | 20260506 16:47:32.730 | INFO | "gpu" : " 17 GB", | 20260506 16:47:32.730 | INFO | "huge" : "", | 20260506 16:47:32.730 | INFO | "huge_shared" : " 27 GB", | 20260506 16:47:32.730 | INFO | "malloc" : " 14 GB", | 20260506 16:47:32.730 | INFO | "pinned" : " 29 GB" | 20260506 16:47:32.730 | INFO | } | 20260506 16:47:32.730 | INFO | Perf report for the last 5.13473 seconds: | 20260506 16:47:32.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:47:32.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:32.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:32.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:32.730 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 16:47:32.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:32.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:32.730 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:47:32.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:32.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:32.730 | INFO | Uploaded 36GiB to GPU in 1.99167s (18.0753GiB/s) | 20260506 16:47:32.730 | INFO | Downloaded 34GiB to GPU in 1.9372s (17.5511GiB/s) | 20260506 16:47:37.867 | INFO | Analyzing chunk frames = [2905088,2905600) | 20260506 16:47:37.867 | INFO | MemInfo: { | 20260506 16:47:37.867 | INFO | "gpu" : " 17 GB", | 20260506 16:47:37.867 | INFO | "huge" : "", | 20260506 16:47:37.867 | INFO | "huge_shared" : " 27 GB", | 20260506 16:47:37.867 | INFO | "malloc" : " 14 GB", | 20260506 16:47:37.867 | INFO | "pinned" : " 29 GB" | 20260506 16:47:37.867 | INFO | } | 20260506 16:47:37.867 | INFO | Perf report for the last 5.13689 seconds: | 20260506 16:47:37.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:47:37.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:37.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:37.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:37.867 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 16:47:37.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:37.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:37.867 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:47:37.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:37.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:37.867 | INFO | Uploaded 36GiB to GPU in 1.96756s (18.2967GiB/s) | 20260506 16:47:37.867 | INFO | Downloaded 34GiB to GPU in 1.93236s (17.595GiB/s) | 20260506 16:47:43.004 | INFO | Analyzing chunk frames = [2905600,2906112) | 20260506 16:47:43.004 | INFO | MemInfo: { | 20260506 16:47:43.004 | INFO | "gpu" : " 17 GB", | 20260506 16:47:43.004 | INFO | "huge" : "", | 20260506 16:47:43.004 | INFO | "huge_shared" : " 27 GB", | 20260506 16:47:43.004 | INFO | "malloc" : " 14 GB", | 20260506 16:47:43.004 | INFO | "pinned" : " 29 GB" | 20260506 16:47:43.004 | INFO | } | 20260506 16:47:43.004 | INFO | Perf report for the last 5.13758 seconds: | 20260506 16:47:43.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:47:43.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:43.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:43.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:43.004 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 16:47:43.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:43.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:43.004 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:47:43.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:43.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:43.005 | INFO | Uploaded 36GiB to GPU in 1.92999s (18.653GiB/s) | 20260506 16:47:43.005 | INFO | Downloaded 34GiB to GPU in 1.8784s (18.1005GiB/s) | 20260506 16:47:48.149 | INFO | Analyzing chunk frames = [2906112,2906624) | 20260506 16:47:48.149 | INFO | MemInfo: { | 20260506 16:47:48.149 | INFO | "gpu" : " 17 GB", | 20260506 16:47:48.149 | INFO | "huge" : "", | 20260506 16:47:48.149 | INFO | "huge_shared" : " 27 GB", | 20260506 16:47:48.149 | INFO | "malloc" : " 14 GB", | 20260506 16:47:48.149 | INFO | "pinned" : " 29 GB" | 20260506 16:47:48.149 | INFO | } | 20260506 16:47:48.149 | INFO | Perf report for the last 5.14478 seconds: | 20260506 16:47:48.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:47:48.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:48.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:48.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:48.149 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 16:47:48.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:48.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:48.149 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:47:48.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:48.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:48.149 | INFO | Uploaded 36GiB to GPU in 1.95534s (18.4111GiB/s) | 20260506 16:47:48.149 | INFO | Downloaded 34GiB to GPU in 1.87857s (18.0989GiB/s) | 20260506 16:47:53.293 | INFO | Analyzing chunk frames = [2906624,2907136) | 20260506 16:47:53.293 | INFO | MemInfo: { | 20260506 16:47:53.293 | INFO | "gpu" : " 17 GB", | 20260506 16:47:53.293 | INFO | "huge" : "", | 20260506 16:47:53.293 | INFO | "huge_shared" : " 27 GB", | 20260506 16:47:53.293 | INFO | "malloc" : " 14 GB", | 20260506 16:47:53.293 | INFO | "pinned" : " 29 GB" | 20260506 16:47:53.293 | INFO | } | 20260506 16:47:53.293 | INFO | Perf report for the last 5.14386 seconds: | 20260506 16:47:53.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:47:53.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:53.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:53.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:53.293 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 16:47:53.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:53.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:53.293 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:47:53.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:53.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:53.293 | INFO | Uploaded 36GiB to GPU in 2.02242s (17.8005GiB/s) | 20260506 16:47:53.293 | INFO | Downloaded 34GiB to GPU in 1.94051s (17.5211GiB/s) | 20260506 16:47:58.434 | INFO | Analyzing chunk frames = [2907136,2907648) | 20260506 16:47:58.434 | INFO | MemInfo: { | 20260506 16:47:58.434 | INFO | "gpu" : " 17 GB", | 20260506 16:47:58.434 | INFO | "huge" : "", | 20260506 16:47:58.434 | INFO | "huge_shared" : " 27 GB", | 20260506 16:47:58.434 | INFO | "malloc" : " 14 GB", | 20260506 16:47:58.434 | INFO | "pinned" : " 29 GB" | 20260506 16:47:58.434 | INFO | } | 20260506 16:47:58.434 | INFO | Perf report for the last 5.14076 seconds: | 20260506 16:47:58.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:47:58.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:58.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:58.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:58.434 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 16:47:58.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:58.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:58.434 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:47:58.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:58.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:47:58.434 | INFO | Uploaded 36GiB to GPU in 2.01309s (17.8829GiB/s) | 20260506 16:47:58.434 | INFO | Downloaded 34GiB to GPU in 1.99584s (17.0354GiB/s) | 20260506 16:48:03.576 | INFO | Analyzing chunk frames = [2907648,2908160) | 20260506 16:48:03.576 | INFO | MemInfo: { | 20260506 16:48:03.576 | INFO | "gpu" : " 17 GB", | 20260506 16:48:03.576 | INFO | "huge" : "", | 20260506 16:48:03.576 | INFO | "huge_shared" : " 27 GB", | 20260506 16:48:03.576 | INFO | "malloc" : " 14 GB", | 20260506 16:48:03.576 | INFO | "pinned" : " 29 GB" | 20260506 16:48:03.576 | INFO | } | 20260506 16:48:03.576 | INFO | Perf report for the last 5.14256 seconds: | 20260506 16:48:03.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:48:03.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:03.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:03.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:03.576 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:48:03.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:03.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:03.576 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 16:48:03.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:03.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:03.577 | INFO | Uploaded 36GiB to GPU in 1.99869s (18.0118GiB/s) | 20260506 16:48:03.577 | INFO | Downloaded 34GiB to GPU in 1.97528s (17.2127GiB/s) | 20260506 16:48:03.577 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:48:03.577 | INFO | Total mem usage: 18 GiB | 20260506 16:48:03.577 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:48:03.577 | INFO | Recent High Water Marks Sum: 14.6 GiB (Savings: -23.1%) | 20260506 16:48:03.577 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:48:03.577 | INFO | High Water Breakdown: | 20260506 16:48:03.577 | INFO | 60.3% : 8.82 GiB ( 77.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:48:03.577 | INFO | 39.6% : 5.79 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:48:03.577 | INFO | 0.136% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:48:03.577 | INFO | ------------------------------------------ | 20260506 16:48:03.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:48:03.577 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:48:03.577 | INFO | Total mem usage: 26.5 GiB | 20260506 16:48:03.577 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:48:03.577 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:48:03.577 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:48:03.577 | INFO | High Water Breakdown: | 20260506 16:48:03.577 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:48:03.577 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:48:03.577 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:48:03.577 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:48:03.577 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:48:03.577 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:48:03.577 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:48:03.577 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:48:03.577 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:48:03.577 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:48:03.577 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:48:03.577 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:48:03.577 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:48:03.577 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:48:03.577 | INFO | ------------------------------------------ | 20260506 16:48:03.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:48:03.577 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:48:03.577 | INFO | Total mem usage: 25.6 GiB | 20260506 16:48:03.577 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:48:03.577 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:48:03.577 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:48:03.577 | INFO | High Water Breakdown: | 20260506 16:48:03.577 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:48:03.577 | INFO | ------------------------------------------ | 20260506 16:48:03.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:48:03.577 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:48:03.577 | INFO | Total mem usage: 15.7 GiB | 20260506 16:48:03.577 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:48:03.577 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:48:03.577 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:48:03.577 | INFO | High Water Breakdown: | 20260506 16:48:03.577 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:48:03.577 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:48:03.577 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:48:03.577 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:48:03.577 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:48:03.577 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:48:03.577 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:48:03.577 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:48:03.577 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:48:03.577 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:48:03.577 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:48:03.577 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:48:03.577 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:48:03.577 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:48:03.577 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:48:03.577 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:48:03.577 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:48:03.577 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:48:03.577 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:48:03.577 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:48:03.577 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:48:03.577 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:48:03.577 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:48:03.577 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:48:03.577 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:48:03.577 | INFO | ------------------------------------------ | 20260506 16:48:03.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:48:03.578 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:48:03.578 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:48:03.578 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:48:03.578 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.53e+01, 1.34e+01 / 2.49e+00 | 20260506 16:48:03.578 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 9.87e-02, 9.93e-03 / 8.27e-03 | 20260506 16:48:03.578 | INFO | PerfReport: 11.58%, Baseline -- 3.63e+00 / 4.37e+00, 5.53e-01 / 3.97e-01 | 20260506 16:48:03.578 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.40e+00, 1.82e-01 / 1.68e-01 | 20260506 16:48:03.578 | INFO | PerfReport: 15.93%, DME -- 4.00e+01 / 4.44e+01, 3.91e+00 / 2.71e+00 | 20260506 16:48:03.578 | INFO | PerfReport: 56.97%, FrameLabeling -- 1.79e+01 / 2.02e+01, 1.47e+00 / 2.11e+00 | 20260506 16:48:03.578 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.29e+00 / 2.89e+00, 3.90e-01 / 2.17e-01 | 20260506 16:48:03.578 | INFO | PerfReport: 3.47%, Metrics -- 1.09e+00 / 1.47e+00, 2.82e-01 / 2.01e-01 | 20260506 16:48:03.578 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.78e-01 / 4.35e-01, 5.90e-02 / 2.72e-02 | 20260506 16:48:03.578 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:48:03.578 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:48:03.578 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:48:03.578 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 16:48:03.578 | INFO | PerfReport: 21.38%, Upload -- 2.08e+01 / 0.00e+00, 6.93e+00 / 0.00e+00 | 20260506 16:48:03.578 | INFO | PerfReport: 21.11%, Download -- 2.05e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260506 16:48:03.578 | INFO | PerfReport: 32.27%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 16:48:03.578 | INFO | PerfReport: 1.38%, LockUpload -- 1.34e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260506 16:48:03.578 | INFO | PerfReport: 1.11%, LockDownload -- 1.08e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260506 16:48:03.578 | INFO | PerfReport: 22.74%, LockCompute -- 2.21e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 16:48:03.578 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260506 16:48:03.689 | INFO | Starting a round of Baz IO | 20260506 16:48:06.832 | INFO | Populating RTMetrics report | 20260506 16:48:06.832 | INFO | RTMetrics report populated | 20260506 16:48:06.832 | INFO | Opening tmp RTMetrics json file | 20260506 16:48:06.833 | INFO | Serializing RTMetrics to json | 20260506 16:48:06.834 | INFO | Serialization done, swapping tmp file now... | 20260506 16:48:06.835 | INFO | RTMetrics done | 20260506 16:48:06.945 | INFO | Finished a round of Baz IO: Wrote 2.079 GiB in 3.255 seconds | 20260506 16:48:06.945 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.08% / 0.01% / 63.92% | 20260506 16:48:06.945 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.56711% / 1.19209e-05% / 5.5631e-05% | 20260506 16:48:06.945 | INFO | Baz IO overall: | 20260506 16:48:06.945 | INFO | BytesWritten : 1529244932859 | 20260506 16:48:06.945 | INFO | NumEvents : 739948521884 | 20260506 16:48:08.710 | INFO | Analyzing chunk frames = [2908160,2908672) | 20260506 16:48:08.710 | INFO | MemInfo: { | 20260506 16:48:08.710 | INFO | "gpu" : " 17 GB", | 20260506 16:48:08.710 | INFO | "huge" : "", | 20260506 16:48:08.710 | INFO | "huge_shared" : " 27 GB", | 20260506 16:48:08.710 | INFO | "malloc" : " 14 GB", | 20260506 16:48:08.710 | INFO | "pinned" : " 29 GB" | 20260506 16:48:08.710 | INFO | } | 20260506 16:48:08.710 | INFO | Perf report for the last 5.13341 seconds: | 20260506 16:48:08.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:48:08.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:08.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:08.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:08.710 | INFO | ANALYSIS : 62.8%, 2.9, 95.1ms, 0 | 20260506 16:48:08.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:08.710 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260506 16:48:08.710 | INFO | PRE_HQ : 60.7%, 1.9, 58.8ms, 0 | 20260506 16:48:08.710 | INFO | BAZWRITER : 63.1%, 4.8, 154.7ms, 0 | 20260506 16:48:08.710 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 16:48:08.710 | INFO | Uploaded 36GiB to GPU in 2.04679s (17.5886GiB/s) | 20260506 16:48:08.710 | INFO | Downloaded 38GiB to GPU in 2.21209s (17.1783GiB/s) | 20260506 16:48:13.846 | INFO | Analyzing chunk frames = [2908672,2909184) | 20260506 16:48:13.846 | INFO | MemInfo: { | 20260506 16:48:13.846 | INFO | "gpu" : " 17 GB", | 20260506 16:48:13.846 | INFO | "huge" : "", | 20260506 16:48:13.846 | INFO | "huge_shared" : " 27 GB", | 20260506 16:48:13.846 | INFO | "malloc" : " 14 GB", | 20260506 16:48:13.846 | INFO | "pinned" : " 29 GB" | 20260506 16:48:13.846 | INFO | } | 20260506 16:48:13.846 | INFO | Perf report for the last 5.13614 seconds: | 20260506 16:48:13.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:48:13.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:13.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:13.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:13.846 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:48:13.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:13.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:13.846 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:48:13.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:13.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:13.846 | INFO | Uploaded 36GiB to GPU in 2.00441s (17.9604GiB/s) | 20260506 16:48:13.846 | INFO | Downloaded 34GiB to GPU in 1.93498s (17.5712GiB/s) | 20260506 16:48:18.984 | INFO | Analyzing chunk frames = [2909184,2909696) | 20260506 16:48:18.984 | INFO | MemInfo: { | 20260506 16:48:18.984 | INFO | "gpu" : " 17 GB", | 20260506 16:48:18.984 | INFO | "huge" : "", | 20260506 16:48:18.984 | INFO | "huge_shared" : " 27 GB", | 20260506 16:48:18.984 | INFO | "malloc" : " 14 GB", | 20260506 16:48:18.984 | INFO | "pinned" : " 29 GB" | 20260506 16:48:18.984 | INFO | } | 20260506 16:48:18.984 | INFO | Perf report for the last 5.13767 seconds: | 20260506 16:48:18.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:48:18.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:18.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:18.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:18.984 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 16:48:18.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:18.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:18.984 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:48:18.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:18.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:18.984 | INFO | Uploaded 36GiB to GPU in 1.98145s (18.1686GiB/s) | 20260506 16:48:18.984 | INFO | Downloaded 34GiB to GPU in 1.93126s (17.6051GiB/s) | 20260506 16:48:24.121 | INFO | Analyzing chunk frames = [2909696,2910208) | 20260506 16:48:24.121 | INFO | MemInfo: { | 20260506 16:48:24.121 | INFO | "gpu" : " 17 GB", | 20260506 16:48:24.121 | INFO | "huge" : "", | 20260506 16:48:24.121 | INFO | "huge_shared" : " 27 GB", | 20260506 16:48:24.121 | INFO | "malloc" : " 14 GB", | 20260506 16:48:24.121 | INFO | "pinned" : " 29 GB" | 20260506 16:48:24.121 | INFO | } | 20260506 16:48:24.121 | INFO | Perf report for the last 5.13701 seconds: | 20260506 16:48:24.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:48:24.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:24.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:24.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:24.121 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:48:24.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:24.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:24.121 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:48:24.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:24.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:24.121 | INFO | Uploaded 36GiB to GPU in 1.92909s (18.6617GiB/s) | 20260506 16:48:24.121 | INFO | Downloaded 34GiB to GPU in 1.87272s (18.1555GiB/s) | 20260506 16:48:29.268 | INFO | Analyzing chunk frames = [2910208,2910720) | 20260506 16:48:29.268 | INFO | MemInfo: { | 20260506 16:48:29.268 | INFO | "gpu" : " 17 GB", | 20260506 16:48:29.268 | INFO | "huge" : "", | 20260506 16:48:29.268 | INFO | "huge_shared" : " 27 GB", | 20260506 16:48:29.268 | INFO | "malloc" : " 14 GB", | 20260506 16:48:29.268 | INFO | "pinned" : " 29 GB" | 20260506 16:48:29.268 | INFO | } | 20260506 16:48:29.268 | INFO | Perf report for the last 5.14721 seconds: | 20260506 16:48:29.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:48:29.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:29.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:29.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:29.268 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260506 16:48:29.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:29.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:29.268 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:48:29.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:29.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:29.268 | INFO | Uploaded 36GiB to GPU in 1.95585s (18.4063GiB/s) | 20260506 16:48:29.268 | INFO | Downloaded 34GiB to GPU in 1.87503s (18.133GiB/s) | 20260506 16:48:34.409 | INFO | Analyzing chunk frames = [2910720,2911232) | 20260506 16:48:34.409 | INFO | MemInfo: { | 20260506 16:48:34.409 | INFO | "gpu" : " 17 GB", | 20260506 16:48:34.409 | INFO | "huge" : "", | 20260506 16:48:34.409 | INFO | "huge_shared" : " 27 GB", | 20260506 16:48:34.409 | INFO | "malloc" : " 14 GB", | 20260506 16:48:34.409 | INFO | "pinned" : " 29 GB" | 20260506 16:48:34.409 | INFO | } | 20260506 16:48:34.409 | INFO | Perf report for the last 5.14106 seconds: | 20260506 16:48:34.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:48:34.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:34.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:34.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:34.409 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 16:48:34.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:34.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:34.409 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:48:34.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:34.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:34.409 | INFO | Uploaded 36GiB to GPU in 1.99904s (18.0087GiB/s) | 20260506 16:48:34.409 | INFO | Downloaded 34GiB to GPU in 1.93631s (17.5592GiB/s) | 20260506 16:48:39.543 | INFO | Analyzing chunk frames = [2911232,2911744) | 20260506 16:48:39.543 | INFO | MemInfo: { | 20260506 16:48:39.543 | INFO | "gpu" : " 17 GB", | 20260506 16:48:39.543 | INFO | "huge" : "", | 20260506 16:48:39.544 | INFO | "huge_shared" : " 27 GB", | 20260506 16:48:39.544 | INFO | "malloc" : " 14 GB", | 20260506 16:48:39.544 | INFO | "pinned" : " 29 GB" | 20260506 16:48:39.544 | INFO | } | 20260506 16:48:39.544 | INFO | Perf report for the last 5.13457 seconds: | 20260506 16:48:39.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:48:39.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:39.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:39.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:39.544 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 16:48:39.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:39.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:39.544 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 16:48:39.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:39.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:39.544 | INFO | Uploaded 36GiB to GPU in 1.98712s (18.1166GiB/s) | 20260506 16:48:39.544 | INFO | Downloaded 34GiB to GPU in 1.99646s (17.0302GiB/s) | 20260506 16:48:44.680 | INFO | Analyzing chunk frames = [2911744,2912256) | 20260506 16:48:44.680 | INFO | MemInfo: { | 20260506 16:48:44.680 | INFO | "gpu" : " 17 GB", | 20260506 16:48:44.680 | INFO | "huge" : "", | 20260506 16:48:44.680 | INFO | "huge_shared" : " 27 GB", | 20260506 16:48:44.680 | INFO | "malloc" : " 14 GB", | 20260506 16:48:44.680 | INFO | "pinned" : " 29 GB" | 20260506 16:48:44.680 | INFO | } | 20260506 16:48:44.680 | INFO | Perf report for the last 5.13676 seconds: | 20260506 16:48:44.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:48:44.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:44.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:44.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:44.680 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 16:48:44.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:44.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:44.681 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 16:48:44.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:44.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:44.681 | INFO | Uploaded 36GiB to GPU in 2.08656s (17.2533GiB/s) | 20260506 16:48:44.681 | INFO | Downloaded 34GiB to GPU in 1.98416s (17.1357GiB/s) | 20260506 16:48:44.804 | INFO | Starting a round of Baz IO | 20260506 16:48:47.952 | INFO | Populating RTMetrics report | 20260506 16:48:47.953 | INFO | RTMetrics report populated | 20260506 16:48:47.953 | INFO | Opening tmp RTMetrics json file | 20260506 16:48:47.974 | INFO | Serializing RTMetrics to json | 20260506 16:48:48.005 | INFO | Serialization done, swapping tmp file now... | 20260506 16:48:48.008 | INFO | RTMetrics done | 20260506 16:48:48.207 | INFO | Finished a round of Baz IO: Wrote 2.078 GiB in 3.402 seconds | 20260506 16:48:48.207 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.07% / 0.01% / 63.92% | 20260506 16:48:48.207 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.56589% / 1.58946e-05% / 6.75519e-05% | 20260506 16:48:48.207 | INFO | Baz IO overall: | 20260506 16:48:48.207 | INFO | BytesWritten : 1531476060431 | 20260506 16:48:48.207 | INFO | NumEvents : 741026355265 | 20260506 16:48:49.820 | INFO | Analyzing chunk frames = [2912256,2912768) | 20260506 16:48:49.820 | INFO | MemInfo: { | 20260506 16:48:49.820 | INFO | "gpu" : " 17 GB", | 20260506 16:48:49.820 | INFO | "huge" : "", | 20260506 16:48:49.820 | INFO | "huge_shared" : " 27 GB", | 20260506 16:48:49.820 | INFO | "malloc" : " 14 GB", | 20260506 16:48:49.820 | INFO | "pinned" : " 29 GB" | 20260506 16:48:49.820 | INFO | } | 20260506 16:48:49.820 | INFO | Perf report for the last 5.13962 seconds: | 20260506 16:48:49.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:48:49.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:49.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:49.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:49.820 | INFO | ANALYSIS : 63.0%, 2.9, 95.7ms, 0 | 20260506 16:48:49.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:49.820 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260506 16:48:49.820 | INFO | PRE_HQ : 60.8%, 1.9, 58.7ms, 0 | 20260506 16:48:49.820 | INFO | BAZWRITER : 65.8%, 4.8, 162.7ms, 0 | 20260506 16:48:49.820 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 16:48:49.820 | INFO | Uploaded 36GiB to GPU in 2.09094s (17.2171GiB/s) | 20260506 16:48:49.820 | INFO | Downloaded 38GiB to GPU in 2.23593s (16.9951GiB/s) | 20260506 16:48:54.961 | INFO | Analyzing chunk frames = [2912768,2913280) | 20260506 16:48:54.961 | INFO | MemInfo: { | 20260506 16:48:54.961 | INFO | "gpu" : " 17 GB", | 20260506 16:48:54.961 | INFO | "huge" : "", | 20260506 16:48:54.961 | INFO | "huge_shared" : " 27 GB", | 20260506 16:48:54.961 | INFO | "malloc" : " 14 GB", | 20260506 16:48:54.961 | INFO | "pinned" : " 29 GB" | 20260506 16:48:54.961 | INFO | } | 20260506 16:48:54.961 | INFO | Perf report for the last 5.14069 seconds: | 20260506 16:48:54.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:48:54.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:54.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:54.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:54.961 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 16:48:54.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:54.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:54.961 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 16:48:54.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:54.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:48:54.961 | INFO | Uploaded 36GiB to GPU in 2.00936s (17.9162GiB/s) | 20260506 16:48:54.961 | INFO | Downloaded 34GiB to GPU in 1.94663s (17.4661GiB/s) | 20260506 16:49:00.098 | INFO | Analyzing chunk frames = [2913280,2913792) | 20260506 16:49:00.098 | INFO | MemInfo: { | 20260506 16:49:00.098 | INFO | "gpu" : " 17 GB", | 20260506 16:49:00.099 | INFO | "huge" : "", | 20260506 16:49:00.099 | INFO | "huge_shared" : " 27 GB", | 20260506 16:49:00.099 | INFO | "malloc" : " 14 GB", | 20260506 16:49:00.099 | INFO | "pinned" : " 29 GB" | 20260506 16:49:00.099 | INFO | } | 20260506 16:49:00.099 | INFO | Perf report for the last 5.13789 seconds: | 20260506 16:49:00.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:49:00.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:00.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:00.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:00.099 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 16:49:00.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:00.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:00.099 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:49:00.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:00.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:00.099 | INFO | Uploaded 36GiB to GPU in 1.97821s (18.1983GiB/s) | 20260506 16:49:00.099 | INFO | Downloaded 34GiB to GPU in 1.92962s (17.62GiB/s) | 20260506 16:49:05.243 | INFO | Analyzing chunk frames = [2913792,2914304) | 20260506 16:49:05.243 | INFO | MemInfo: { | 20260506 16:49:05.243 | INFO | "gpu" : " 17 GB", | 20260506 16:49:05.244 | INFO | "huge" : "", | 20260506 16:49:05.244 | INFO | "huge_shared" : " 27 GB", | 20260506 16:49:05.244 | INFO | "malloc" : " 14 GB", | 20260506 16:49:05.244 | INFO | "pinned" : " 29 GB" | 20260506 16:49:05.244 | INFO | } | 20260506 16:49:05.244 | INFO | Perf report for the last 5.14502 seconds: | 20260506 16:49:05.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:49:05.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:05.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:05.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:05.244 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 16:49:05.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:05.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:05.244 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:49:05.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:05.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:05.244 | INFO | Uploaded 36GiB to GPU in 1.93409s (18.6134GiB/s) | 20260506 16:49:05.244 | INFO | Downloaded 34GiB to GPU in 1.87376s (18.1454GiB/s) | 20260506 16:49:10.381 | INFO | Analyzing chunk frames = [2914304,2914816) | 20260506 16:49:10.381 | INFO | MemInfo: { | 20260506 16:49:10.381 | INFO | "gpu" : " 17 GB", | 20260506 16:49:10.381 | INFO | "huge" : "", | 20260506 16:49:10.381 | INFO | "huge_shared" : " 27 GB", | 20260506 16:49:10.381 | INFO | "malloc" : " 14 GB", | 20260506 16:49:10.381 | INFO | "pinned" : " 29 GB" | 20260506 16:49:10.381 | INFO | } | 20260506 16:49:10.381 | INFO | Perf report for the last 5.13754 seconds: | 20260506 16:49:10.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:49:10.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:10.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:10.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:10.381 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 16:49:10.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:10.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:10.381 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:49:10.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:10.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:10.382 | INFO | Uploaded 36GiB to GPU in 2.01949s (17.8263GiB/s) | 20260506 16:49:10.382 | INFO | Downloaded 34GiB to GPU in 1.91336s (17.7697GiB/s) | 20260506 16:49:15.522 | INFO | Analyzing chunk frames = [2914816,2915328) | 20260506 16:49:15.522 | INFO | MemInfo: { | 20260506 16:49:15.522 | INFO | "gpu" : " 17 GB", | 20260506 16:49:15.522 | INFO | "huge" : "", | 20260506 16:49:15.522 | INFO | "huge_shared" : " 27 GB", | 20260506 16:49:15.522 | INFO | "malloc" : " 14 GB", | 20260506 16:49:15.522 | INFO | "pinned" : " 29 GB" | 20260506 16:49:15.522 | INFO | } | 20260506 16:49:15.522 | INFO | Perf report for the last 5.1412 seconds: | 20260506 16:49:15.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:49:15.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:15.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:15.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:15.522 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:49:15.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:15.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:15.522 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:49:15.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:15.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:15.523 | INFO | Uploaded 36GiB to GPU in 2.07697s (17.333GiB/s) | 20260506 16:49:15.523 | INFO | Downloaded 34GiB to GPU in 1.96383s (17.3131GiB/s) | 20260506 16:49:20.665 | INFO | Analyzing chunk frames = [2915328,2915840) | 20260506 16:49:20.665 | INFO | MemInfo: { | 20260506 16:49:20.665 | INFO | "gpu" : " 17 GB", | 20260506 16:49:20.665 | INFO | "huge" : "", | 20260506 16:49:20.665 | INFO | "huge_shared" : " 27 GB", | 20260506 16:49:20.665 | INFO | "malloc" : " 14 GB", | 20260506 16:49:20.665 | INFO | "pinned" : " 29 GB" | 20260506 16:49:20.665 | INFO | } | 20260506 16:49:20.665 | INFO | Perf report for the last 5.14282 seconds: | 20260506 16:49:20.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:49:20.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:20.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:20.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:20.665 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:49:20.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:20.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:20.665 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 16:49:20.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:20.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:20.666 | INFO | Uploaded 36GiB to GPU in 2.00801s (17.9282GiB/s) | 20260506 16:49:20.666 | INFO | Downloaded 34GiB to GPU in 1.97698s (17.1979GiB/s) | 20260506 16:49:25.798 | INFO | Analyzing chunk frames = [2915840,2916352) | 20260506 16:49:25.798 | INFO | MemInfo: { | 20260506 16:49:25.798 | INFO | "gpu" : " 17 GB", | 20260506 16:49:25.798 | INFO | "huge" : "", | 20260506 16:49:25.798 | INFO | "huge_shared" : " 27 GB", | 20260506 16:49:25.798 | INFO | "malloc" : " 14 GB", | 20260506 16:49:25.798 | INFO | "pinned" : " 29 GB" | 20260506 16:49:25.798 | INFO | } | 20260506 16:49:25.798 | INFO | Perf report for the last 5.13302 seconds: | 20260506 16:49:25.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:49:25.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:25.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:25.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:25.798 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 16:49:25.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:25.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:25.798 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:49:25.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:25.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:25.798 | INFO | Uploaded 36GiB to GPU in 2.03117s (17.7238GiB/s) | 20260506 16:49:25.799 | INFO | Downloaded 34GiB to GPU in 1.98468s (17.1312GiB/s) | 20260506 16:49:25.799 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:49:25.799 | INFO | Total mem usage: 18 GiB | 20260506 16:49:25.799 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:49:25.799 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.3%) | 20260506 16:49:25.799 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:49:25.799 | INFO | High Water Breakdown: | 20260506 16:49:25.799 | INFO | 61.2% : 9.47 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:49:25.799 | INFO | 38.7% : 5.99 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:49:25.799 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:49:25.799 | INFO | ------------------------------------------ | 20260506 16:49:25.799 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:49:25.799 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:49:25.799 | INFO | Total mem usage: 26.5 GiB | 20260506 16:49:25.799 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:49:25.799 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:49:25.799 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:49:25.799 | INFO | High Water Breakdown: | 20260506 16:49:25.799 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:49:25.799 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:49:25.799 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:49:25.799 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:49:25.799 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:49:25.799 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:49:25.799 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:49:25.799 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:49:25.799 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:49:25.799 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:49:25.799 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:49:25.799 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:49:25.799 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:49:25.799 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:49:25.799 | INFO | ------------------------------------------ | 20260506 16:49:25.799 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:49:25.799 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:49:25.799 | INFO | Total mem usage: 25.6 GiB | 20260506 16:49:25.799 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:49:25.799 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:49:25.799 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:49:25.799 | INFO | High Water Breakdown: | 20260506 16:49:25.799 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:49:25.799 | INFO | ------------------------------------------ | 20260506 16:49:25.799 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:49:25.799 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:49:25.799 | INFO | Total mem usage: 15.7 GiB | 20260506 16:49:25.799 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:49:25.799 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:49:25.799 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:49:25.799 | INFO | High Water Breakdown: | 20260506 16:49:25.799 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:49:25.799 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:49:25.799 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:49:25.799 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:49:25.799 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:49:25.799 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:49:25.799 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:49:25.799 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:49:25.799 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:49:25.799 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:49:25.799 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:49:25.799 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:49:25.799 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:49:25.799 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:49:25.799 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:49:25.799 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:49:25.799 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:49:25.799 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:49:25.799 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:49:25.799 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:49:25.799 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:49:25.799 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:49:25.799 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:49:25.799 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:49:25.799 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:49:25.799 | INFO | ------------------------------------------ | 20260506 16:49:25.799 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:49:25.799 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:49:25.799 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:49:25.799 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:49:25.799 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.53e+01, 1.35e+01 / 2.06e+00 | 20260506 16:49:25.799 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 9.98e-02, 1.02e-02 / 5.69e-03 | 20260506 16:49:25.799 | INFO | PerfReport: 11.61%, Baseline -- 3.64e+00 / 4.40e+00, 5.64e-01 / 3.61e-01 | 20260506 16:49:25.799 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.42e+00, 1.79e-01 / 1.54e-01 | 20260506 16:49:25.799 | INFO | PerfReport: 15.91%, DME -- 3.99e+01 / 4.44e+01, 3.97e+00 / 2.52e+00 | 20260506 16:49:25.800 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.42e+00 / 1.82e+00 | 20260506 16:49:25.800 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.29e+00 / 2.89e+00, 3.89e-01 / 2.36e-01 | 20260506 16:49:25.800 | INFO | PerfReport: 3.48%, Metrics -- 1.09e+00 / 1.48e+00, 2.84e-01 / 1.89e-01 | 20260506 16:49:25.800 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.78e-01 / 4.30e-01, 5.88e-02 / 2.70e-02 | 20260506 16:49:25.800 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:49:25.800 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:49:25.800 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:49:25.800 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 16:49:25.800 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 16:49:25.800 | INFO | PerfReport: 21.12%, Download -- 2.06e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 16:49:25.800 | INFO | PerfReport: 32.21%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 16:49:25.800 | INFO | PerfReport: 1.50%, LockUpload -- 1.46e+00 / 0.00e+00, 5.85e+00 / 0.00e+00 | 20260506 16:49:25.800 | INFO | PerfReport: 1.06%, LockDownload -- 1.03e+00 / 0.00e+00, 2.66e+00 / 0.00e+00 | 20260506 16:49:25.800 | INFO | PerfReport: 22.59%, LockCompute -- 2.20e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 16:49:25.800 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260506 16:49:25.911 | INFO | Starting a round of Baz IO | 20260506 16:49:29.061 | INFO | Populating RTMetrics report | 20260506 16:49:29.061 | INFO | RTMetrics report populated | 20260506 16:49:29.061 | INFO | Opening tmp RTMetrics json file | 20260506 16:49:29.135 | INFO | Serializing RTMetrics to json | 20260506 16:49:29.140 | INFO | Serialization done, swapping tmp file now... | 20260506 16:49:29.140 | INFO | RTMetrics done | 20260506 16:49:29.541 | INFO | Finished a round of Baz IO: Wrote 2.077 GiB in 3.629 seconds | 20260506 16:49:29.541 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.06% / 0.01% / 63.93% | 20260506 16:49:29.541 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.55735% / 1.58946e-05% / 5.96046e-05% | 20260506 16:49:29.541 | INFO | Baz IO overall: | 20260506 16:49:29.541 | INFO | BytesWritten : 1533706433208 | 20260506 16:49:29.541 | INFO | NumEvents : 742103514219 | 20260506 16:49:30.936 | INFO | Analyzing chunk frames = [2916352,2916864) | 20260506 16:49:30.936 | INFO | MemInfo: { | 20260506 16:49:30.936 | INFO | "gpu" : " 17 GB", | 20260506 16:49:30.936 | INFO | "huge" : "", | 20260506 16:49:30.936 | INFO | "huge_shared" : " 27 GB", | 20260506 16:49:30.936 | INFO | "malloc" : " 14 GB", | 20260506 16:49:30.936 | INFO | "pinned" : " 29 GB" | 20260506 16:49:30.936 | INFO | } | 20260506 16:49:30.936 | INFO | Perf report for the last 5.13783 seconds: | 20260506 16:49:30.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:49:30.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:30.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:30.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:30.936 | INFO | ANALYSIS : 62.9%, 2.9, 94.5ms, 0 | 20260506 16:49:30.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:30.936 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260506 16:49:30.936 | INFO | PRE_HQ : 61.0%, 2.0, 61.8ms, 0 | 20260506 16:49:30.936 | INFO | BAZWRITER : 69.9%, 4.8, 173.3ms, 0 | 20260506 16:49:30.936 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 16:49:30.936 | INFO | Uploaded 36GiB to GPU in 2.06405s (17.4414GiB/s) | 20260506 16:49:30.936 | INFO | Downloaded 38GiB to GPU in 2.22776s (17.0575GiB/s) | 20260506 16:49:36.074 | INFO | Analyzing chunk frames = [2916864,2917376) | 20260506 16:49:36.074 | INFO | MemInfo: { | 20260506 16:49:36.074 | INFO | "gpu" : " 17 GB", | 20260506 16:49:36.074 | INFO | "huge" : "", | 20260506 16:49:36.074 | INFO | "huge_shared" : " 27 GB", | 20260506 16:49:36.074 | INFO | "malloc" : " 14 GB", | 20260506 16:49:36.074 | INFO | "pinned" : " 29 GB" | 20260506 16:49:36.074 | INFO | } | 20260506 16:49:36.074 | INFO | Perf report for the last 5.13803 seconds: | 20260506 16:49:36.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:49:36.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:36.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:36.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:36.074 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:49:36.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:36.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:36.074 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 16:49:36.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:36.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:36.074 | INFO | Uploaded 36GiB to GPU in 2.01487s (17.8671GiB/s) | 20260506 16:49:36.074 | INFO | Downloaded 34GiB to GPU in 1.95093s (17.4276GiB/s) | 20260506 16:49:41.211 | INFO | Analyzing chunk frames = [2917376,2917888) | 20260506 16:49:41.211 | INFO | MemInfo: { | 20260506 16:49:41.211 | INFO | "gpu" : " 17 GB", | 20260506 16:49:41.211 | INFO | "huge" : "", | 20260506 16:49:41.211 | INFO | "huge_shared" : " 27 GB", | 20260506 16:49:41.211 | INFO | "malloc" : " 14 GB", | 20260506 16:49:41.211 | INFO | "pinned" : " 29 GB" | 20260506 16:49:41.211 | INFO | } | 20260506 16:49:41.211 | INFO | Perf report for the last 5.13717 seconds: | 20260506 16:49:41.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:49:41.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:41.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:41.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:41.211 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 16:49:41.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:41.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:41.211 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:49:41.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:41.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:41.212 | INFO | Uploaded 36GiB to GPU in 1.99515s (18.0438GiB/s) | 20260506 16:49:41.212 | INFO | Downloaded 34GiB to GPU in 1.93197s (17.5986GiB/s) | 20260506 16:49:46.357 | INFO | Analyzing chunk frames = [2917888,2918400) | 20260506 16:49:46.357 | INFO | MemInfo: { | 20260506 16:49:46.357 | INFO | "gpu" : " 17 GB", | 20260506 16:49:46.357 | INFO | "huge" : "", | 20260506 16:49:46.357 | INFO | "huge_shared" : " 27 GB", | 20260506 16:49:46.357 | INFO | "malloc" : " 14 GB", | 20260506 16:49:46.357 | INFO | "pinned" : " 29 GB" | 20260506 16:49:46.357 | INFO | } | 20260506 16:49:46.357 | INFO | Perf report for the last 5.146 seconds: | 20260506 16:49:46.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:49:46.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:46.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:46.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:46.357 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 16:49:46.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:46.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:46.357 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:49:46.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:46.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:46.358 | INFO | Uploaded 36GiB to GPU in 1.93297s (18.6242GiB/s) | 20260506 16:49:46.358 | INFO | Downloaded 34GiB to GPU in 1.88066s (18.0788GiB/s) | 20260506 16:49:51.494 | INFO | Analyzing chunk frames = [2918400,2918912) | 20260506 16:49:51.494 | INFO | MemInfo: { | 20260506 16:49:51.494 | INFO | "gpu" : " 17 GB", | 20260506 16:49:51.494 | INFO | "huge" : "", | 20260506 16:49:51.494 | INFO | "huge_shared" : " 27 GB", | 20260506 16:49:51.494 | INFO | "malloc" : " 14 GB", | 20260506 16:49:51.494 | INFO | "pinned" : " 29 GB" | 20260506 16:49:51.494 | INFO | } | 20260506 16:49:51.494 | INFO | Perf report for the last 5.13709 seconds: | 20260506 16:49:51.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:49:51.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:51.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:51.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:51.495 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 16:49:51.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:51.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:51.495 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:49:51.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:51.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:51.495 | INFO | Uploaded 36GiB to GPU in 2.01974s (17.8241GiB/s) | 20260506 16:49:51.495 | INFO | Downloaded 34GiB to GPU in 1.91231s (17.7795GiB/s) | 20260506 16:49:56.633 | INFO | Analyzing chunk frames = [2918912,2919424) | 20260506 16:49:56.633 | INFO | MemInfo: { | 20260506 16:49:56.633 | INFO | "gpu" : " 17 GB", | 20260506 16:49:56.633 | INFO | "huge" : "", | 20260506 16:49:56.633 | INFO | "huge_shared" : " 27 GB", | 20260506 16:49:56.633 | INFO | "malloc" : " 14 GB", | 20260506 16:49:56.633 | INFO | "pinned" : " 29 GB" | 20260506 16:49:56.633 | INFO | } | 20260506 16:49:56.633 | INFO | Perf report for the last 5.1389 seconds: | 20260506 16:49:56.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:49:56.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:56.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:56.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:56.633 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 16:49:56.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:56.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:56.633 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:49:56.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:56.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:49:56.634 | INFO | Uploaded 36GiB to GPU in 2.05558s (17.5133GiB/s) | 20260506 16:49:56.634 | INFO | Downloaded 34GiB to GPU in 1.9583s (17.362GiB/s) | 20260506 16:50:01.772 | INFO | Analyzing chunk frames = [2919424,2919936) | 20260506 16:50:01.772 | INFO | MemInfo: { | 20260506 16:50:01.772 | INFO | "gpu" : " 17 GB", | 20260506 16:50:01.772 | INFO | "huge" : "", | 20260506 16:50:01.772 | INFO | "huge_shared" : " 27 GB", | 20260506 16:50:01.772 | INFO | "malloc" : " 14 GB", | 20260506 16:50:01.772 | INFO | "pinned" : " 29 GB" | 20260506 16:50:01.772 | INFO | } | 20260506 16:50:01.772 | INFO | Perf report for the last 5.13848 seconds: | 20260506 16:50:01.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:50:01.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:01.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:01.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:01.772 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 16:50:01.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:01.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:01.772 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 16:50:01.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:01.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:01.772 | INFO | Uploaded 36GiB to GPU in 2.00567s (17.9492GiB/s) | 20260506 16:50:01.772 | INFO | Downloaded 34GiB to GPU in 1.99348s (17.0556GiB/s) | 20260506 16:50:06.914 | INFO | Analyzing chunk frames = [2919936,2920448) | 20260506 16:50:06.914 | INFO | MemInfo: { | 20260506 16:50:06.914 | INFO | "gpu" : " 17 GB", | 20260506 16:50:06.914 | INFO | "huge" : "", | 20260506 16:50:06.914 | INFO | "huge_shared" : " 27 GB", | 20260506 16:50:06.914 | INFO | "malloc" : " 14 GB", | 20260506 16:50:06.914 | INFO | "pinned" : " 29 GB" | 20260506 16:50:06.915 | INFO | } | 20260506 16:50:06.915 | INFO | Perf report for the last 5.14271 seconds: | 20260506 16:50:06.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:50:06.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:06.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:06.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:06.915 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:50:06.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:06.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:06.915 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:50:06.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:06.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:06.915 | INFO | Uploaded 36GiB to GPU in 2.00932s (17.9165GiB/s) | 20260506 16:50:06.915 | INFO | Downloaded 34GiB to GPU in 1.98327s (17.1434GiB/s) | 20260506 16:50:07.032 | INFO | Starting a round of Baz IO | 20260506 16:50:10.192 | INFO | Populating RTMetrics report | 20260506 16:50:10.193 | INFO | RTMetrics report populated | 20260506 16:50:10.193 | INFO | Opening tmp RTMetrics json file | 20260506 16:50:10.244 | INFO | Serializing RTMetrics to json | 20260506 16:50:10.343 | INFO | Serialization done, swapping tmp file now... | 20260506 16:50:10.346 | INFO | RTMetrics done | 20260506 16:50:10.648 | INFO | Finished a round of Baz IO: Wrote 2.077 GiB in 3.615 seconds | 20260506 16:50:10.648 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.05% / 0.01% / 63.94% | 20260506 16:50:10.648 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.55385% / 0% / 7.15256e-05% | 20260506 16:50:10.648 | INFO | Baz IO overall: | 20260506 16:50:10.648 | INFO | BytesWritten : 1535936839001 | 20260506 16:50:10.648 | INFO | NumEvents : 743180654021 | 20260506 16:50:12.054 | INFO | Analyzing chunk frames = [2920448,2920960) | 20260506 16:50:12.054 | INFO | MemInfo: { | 20260506 16:50:12.054 | INFO | "gpu" : " 17 GB", | 20260506 16:50:12.054 | INFO | "huge" : "", | 20260506 16:50:12.054 | INFO | "huge_shared" : " 27 GB", | 20260506 16:50:12.054 | INFO | "malloc" : " 14 GB", | 20260506 16:50:12.054 | INFO | "pinned" : " 29 GB" | 20260506 16:50:12.054 | INFO | } | 20260506 16:50:12.054 | INFO | Perf report for the last 5.13917 seconds: | 20260506 16:50:12.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:50:12.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:12.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:12.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:12.054 | INFO | ANALYSIS : 63.0%, 2.8, 94.1ms, 0 | 20260506 16:50:12.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:12.054 | INFO | RT_METRICS : 49.0%, 1.0, 25.7ms, 0 | 20260506 16:50:12.054 | INFO | PRE_HQ : 61.2%, 2.0, 62.6ms, 0 | 20260506 16:50:12.054 | INFO | BAZWRITER : 70.0%, 4.8, 172.1ms, 0 | 20260506 16:50:12.054 | INFO | BAZ_SAMPLING : 3.3%, 1.0, 1.7ms, 0 | 20260506 16:50:12.054 | INFO | Uploaded 36GiB to GPU in 2.03925s (17.6536GiB/s) | 20260506 16:50:12.054 | INFO | Downloaded 38GiB to GPU in 2.21655s (17.1437GiB/s) | 20260506 16:50:17.193 | INFO | Analyzing chunk frames = [2920960,2921472) | 20260506 16:50:17.193 | INFO | MemInfo: { | 20260506 16:50:17.193 | INFO | "gpu" : " 17 GB", | 20260506 16:50:17.193 | INFO | "huge" : "", | 20260506 16:50:17.193 | INFO | "huge_shared" : " 27 GB", | 20260506 16:50:17.193 | INFO | "malloc" : " 14 GB", | 20260506 16:50:17.193 | INFO | "pinned" : " 29 GB" | 20260506 16:50:17.193 | INFO | } | 20260506 16:50:17.193 | INFO | Perf report for the last 5.13914 seconds: | 20260506 16:50:17.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:50:17.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:17.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:17.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:17.193 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 16:50:17.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:17.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:17.193 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 16:50:17.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:17.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:17.193 | INFO | Uploaded 36GiB to GPU in 2.0235s (17.7909GiB/s) | 20260506 16:50:17.193 | INFO | Downloaded 34GiB to GPU in 1.93528s (17.5685GiB/s) | 20260506 16:50:22.332 | INFO | Analyzing chunk frames = [2921472,2921984) | 20260506 16:50:22.332 | INFO | MemInfo: { | 20260506 16:50:22.332 | INFO | "gpu" : " 17 GB", | 20260506 16:50:22.332 | INFO | "huge" : "", | 20260506 16:50:22.332 | INFO | "huge_shared" : " 27 GB", | 20260506 16:50:22.332 | INFO | "malloc" : " 14 GB", | 20260506 16:50:22.332 | INFO | "pinned" : " 29 GB" | 20260506 16:50:22.332 | INFO | } | 20260506 16:50:22.332 | INFO | Perf report for the last 5.13953 seconds: | 20260506 16:50:22.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:50:22.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:22.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:22.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:22.332 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 16:50:22.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:22.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:22.333 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:50:22.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:22.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:22.333 | INFO | Uploaded 36GiB to GPU in 1.97457s (18.2318GiB/s) | 20260506 16:50:22.333 | INFO | Downloaded 34GiB to GPU in 1.93079s (17.6094GiB/s) | 20260506 16:50:27.468 | INFO | Analyzing chunk frames = [2921984,2922496) | 20260506 16:50:27.468 | INFO | MemInfo: { | 20260506 16:50:27.468 | INFO | "gpu" : " 17 GB", | 20260506 16:50:27.468 | INFO | "huge" : "", | 20260506 16:50:27.468 | INFO | "huge_shared" : " 27 GB", | 20260506 16:50:27.468 | INFO | "malloc" : " 14 GB", | 20260506 16:50:27.468 | INFO | "pinned" : " 29 GB" | 20260506 16:50:27.468 | INFO | } | 20260506 16:50:27.468 | INFO | Perf report for the last 5.1359 seconds: | 20260506 16:50:27.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:50:27.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:27.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:27.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:27.468 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260506 16:50:27.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:27.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:27.468 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:50:27.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:27.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:27.469 | INFO | Uploaded 36GiB to GPU in 1.96559s (18.3151GiB/s) | 20260506 16:50:27.469 | INFO | Downloaded 34GiB to GPU in 1.89068s (17.983GiB/s) | 20260506 16:50:32.609 | INFO | Analyzing chunk frames = [2922496,2923008) | 20260506 16:50:32.609 | INFO | MemInfo: { | 20260506 16:50:32.609 | INFO | "gpu" : " 17 GB", | 20260506 16:50:32.609 | INFO | "huge" : "", | 20260506 16:50:32.609 | INFO | "huge_shared" : " 27 GB", | 20260506 16:50:32.609 | INFO | "malloc" : " 14 GB", | 20260506 16:50:32.609 | INFO | "pinned" : " 29 GB" | 20260506 16:50:32.609 | INFO | } | 20260506 16:50:32.609 | INFO | Perf report for the last 5.1409 seconds: | 20260506 16:50:32.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:50:32.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:32.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:32.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:32.609 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 16:50:32.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:32.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:32.609 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:50:32.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:32.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:32.609 | INFO | Uploaded 36GiB to GPU in 2.02253s (17.7995GiB/s) | 20260506 16:50:32.610 | INFO | Downloaded 34GiB to GPU in 1.91233s (17.7794GiB/s) | 20260506 16:50:37.755 | INFO | Analyzing chunk frames = [2923008,2923520) | 20260506 16:50:37.755 | INFO | MemInfo: { | 20260506 16:50:37.755 | INFO | "gpu" : " 17 GB", | 20260506 16:50:37.755 | INFO | "huge" : "", | 20260506 16:50:37.755 | INFO | "huge_shared" : " 27 GB", | 20260506 16:50:37.755 | INFO | "malloc" : " 14 GB", | 20260506 16:50:37.755 | INFO | "pinned" : " 29 GB" | 20260506 16:50:37.755 | INFO | } | 20260506 16:50:37.755 | INFO | Perf report for the last 5.1456 seconds: | 20260506 16:50:37.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:50:37.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:37.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:37.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:37.755 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 16:50:37.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:37.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:37.755 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:50:37.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:37.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:37.755 | INFO | Uploaded 36GiB to GPU in 2.0329s (17.7087GiB/s) | 20260506 16:50:37.755 | INFO | Downloaded 34GiB to GPU in 1.94766s (17.4568GiB/s) | 20260506 16:50:42.887 | INFO | Analyzing chunk frames = [2923520,2924032) | 20260506 16:50:42.887 | INFO | MemInfo: { | 20260506 16:50:42.887 | INFO | "gpu" : " 17 GB", | 20260506 16:50:42.887 | INFO | "huge" : "", | 20260506 16:50:42.887 | INFO | "huge_shared" : " 27 GB", | 20260506 16:50:42.887 | INFO | "malloc" : " 14 GB", | 20260506 16:50:42.887 | INFO | "pinned" : " 29 GB" | 20260506 16:50:42.887 | INFO | } | 20260506 16:50:42.887 | INFO | Perf report for the last 5.13189 seconds: | 20260506 16:50:42.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:50:42.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:42.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:42.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:42.887 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260506 16:50:42.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:42.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:42.887 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260506 16:50:42.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:42.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:42.887 | INFO | Uploaded 36GiB to GPU in 2.02363s (17.7898GiB/s) | 20260506 16:50:42.887 | INFO | Downloaded 34GiB to GPU in 1.9812s (17.1613GiB/s) | 20260506 16:50:48.026 | INFO | Analyzing chunk frames = [2924032,2924544) | 20260506 16:50:48.026 | INFO | MemInfo: { | 20260506 16:50:48.026 | INFO | "gpu" : " 17 GB", | 20260506 16:50:48.026 | INFO | "huge" : "", | 20260506 16:50:48.026 | INFO | "huge_shared" : " 27 GB", | 20260506 16:50:48.026 | INFO | "malloc" : " 14 GB", | 20260506 16:50:48.026 | INFO | "pinned" : " 29 GB" | 20260506 16:50:48.026 | INFO | } | 20260506 16:50:48.026 | INFO | Perf report for the last 5.13951 seconds: | 20260506 16:50:48.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:50:48.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:48.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:48.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:48.026 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:50:48.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:48.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:48.026 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 16:50:48.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:48.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:48.027 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:50:48.027 | INFO | Uploaded 36GiB to GPU in 2.024s (17.7865GiB/s) | 20260506 16:50:48.027 | INFO | Total mem usage: 18 GiB | 20260506 16:50:48.027 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:50:48.027 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -15.8%) | 20260506 16:50:48.027 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:50:48.027 | INFO | High Water Breakdown: | 20260506 16:50:48.027 | INFO | Downloaded 34GiB to GPU in 1.96913s (17.2665GiB/s) | 20260506 16:50:48.027 | INFO | 61.3% : 9.53 GiB ( 83.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:50:48.027 | INFO | 38.6% : 5.99 GiB ( 89.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:50:48.027 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:50:48.027 | INFO | ------------------------------------------ | 20260506 16:50:48.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:50:48.027 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:50:48.027 | INFO | Total mem usage: 26.5 GiB | 20260506 16:50:48.027 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:50:48.027 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 16:50:48.027 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260506 16:50:48.027 | INFO | High Water Breakdown: | 20260506 16:50:48.027 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:50:48.027 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:50:48.027 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:50:48.027 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:50:48.027 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:50:48.027 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:50:48.027 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:50:48.027 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:50:48.027 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:50:48.027 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:50:48.027 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:50:48.027 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:50:48.027 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:50:48.027 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:50:48.027 | INFO | ------------------------------------------ | 20260506 16:50:48.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:50:48.027 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:50:48.027 | INFO | Total mem usage: 25.6 GiB | 20260506 16:50:48.027 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:50:48.027 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:50:48.027 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:50:48.027 | INFO | High Water Breakdown: | 20260506 16:50:48.027 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:50:48.027 | INFO | ------------------------------------------ | 20260506 16:50:48.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:50:48.027 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:50:48.027 | INFO | Total mem usage: 15.7 GiB | 20260506 16:50:48.027 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:50:48.027 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:50:48.027 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:50:48.027 | INFO | High Water Breakdown: | 20260506 16:50:48.027 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:50:48.027 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:50:48.027 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:50:48.027 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:50:48.027 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:50:48.027 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:50:48.027 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:50:48.027 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:50:48.027 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:50:48.027 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:50:48.027 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:50:48.027 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:50:48.027 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:50:48.027 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:50:48.027 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:50:48.027 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:50:48.027 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:50:48.027 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:50:48.027 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:50:48.027 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:50:48.027 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:50:48.027 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:50:48.027 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:50:48.027 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:50:48.027 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:50:48.027 | INFO | ------------------------------------------ | 20260506 16:50:48.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:50:48.027 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:50:48.027 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:50:48.027 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:50:48.027 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.55e+01, 1.35e+01 / 2.10e+00 | 20260506 16:50:48.028 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 9.95e-02, 1.07e-02 / 5.73e-03 | 20260506 16:50:48.028 | INFO | PerfReport: 11.62%, Baseline -- 3.64e+00 / 4.45e+00, 5.60e-01 / 3.36e-01 | 20260506 16:50:48.028 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.46e+00, 1.86e-01 / 1.34e-01 | 20260506 16:50:48.028 | INFO | PerfReport: 15.93%, DME -- 4.00e+01 / 4.42e+01, 3.77e+00 / 2.06e+00 | 20260506 16:50:48.028 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.45e+00 / 1.31e+00 | 20260506 16:50:48.028 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.29e+00 / 2.81e+00, 3.80e-01 / 2.28e-01 | 20260506 16:50:48.028 | INFO | PerfReport: 3.47%, Metrics -- 1.09e+00 / 1.42e+00, 2.81e-01 / 2.02e-01 | 20260506 16:50:48.028 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.78e-01 / 4.24e-01, 5.89e-02 / 2.15e-02 | 20260506 16:50:48.028 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:50:48.028 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:50:48.028 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:50:48.028 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 16:50:48.028 | INFO | PerfReport: 21.56%, Upload -- 2.10e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 16:50:48.028 | INFO | PerfReport: 21.17%, Download -- 2.06e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 16:50:48.028 | INFO | PerfReport: 32.26%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 16:50:48.028 | INFO | PerfReport: 1.47%, LockUpload -- 1.43e+00 / 0.00e+00, 5.87e+00 / 0.00e+00 | 20260506 16:50:48.028 | INFO | PerfReport: 1.05%, LockDownload -- 1.03e+00 / 0.00e+00, 2.62e+00 / 0.00e+00 | 20260506 16:50:48.028 | INFO | PerfReport: 22.48%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 16:50:48.028 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260506 16:50:48.146 | INFO | Starting a round of Baz IO | 20260506 16:50:51.287 | INFO | Populating RTMetrics report | 20260506 16:50:51.288 | INFO | RTMetrics report populated | 20260506 16:50:51.288 | INFO | Opening tmp RTMetrics json file | 20260506 16:50:51.289 | INFO | Serializing RTMetrics to json | 20260506 16:50:51.291 | INFO | Serialization done, swapping tmp file now... | 20260506 16:50:51.293 | INFO | RTMetrics done | 20260506 16:50:51.420 | INFO | Finished a round of Baz IO: Wrote 2.077 GiB in 3.273 seconds | 20260506 16:50:51.420 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.04% / 0.01% / 63.95% | 20260506 16:50:51.420 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.55241% / 3.17891e-05% / 5.5631e-05% | 20260506 16:50:51.420 | INFO | Baz IO overall: | 20260506 16:50:51.420 | INFO | BytesWritten : 1538167432610 | 20260506 16:50:51.420 | INFO | NumEvents : 744258086152 | 20260506 16:50:53.165 | INFO | Analyzing chunk frames = [2924544,2925056) | 20260506 16:50:53.165 | INFO | MemInfo: { | 20260506 16:50:53.165 | INFO | "gpu" : " 17 GB", | 20260506 16:50:53.165 | INFO | "huge" : "", | 20260506 16:50:53.165 | INFO | "huge_shared" : " 27 GB", | 20260506 16:50:53.165 | INFO | "malloc" : " 14 GB", | 20260506 16:50:53.165 | INFO | "pinned" : " 29 GB" | 20260506 16:50:53.165 | INFO | } | 20260506 16:50:53.166 | INFO | Perf report for the last 5.13917 seconds: | 20260506 16:50:53.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:50:53.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:53.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:53.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:53.166 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 16:50:53.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:53.166 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 16:50:53.166 | INFO | PRE_HQ : 60.5%, 1.9, 58.8ms, 0 | 20260506 16:50:53.166 | INFO | BAZWRITER : 63.3%, 4.8, 154.6ms, 0 | 20260506 16:50:53.166 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 16:50:53.166 | INFO | Uploaded 36GiB to GPU in 2.08086s (17.3006GiB/s) | 20260506 16:50:53.166 | INFO | Downloaded 38GiB to GPU in 2.21929s (17.1226GiB/s) | 20260506 16:50:58.300 | INFO | Analyzing chunk frames = [2925056,2925568) | 20260506 16:50:58.300 | INFO | MemInfo: { | 20260506 16:50:58.300 | INFO | "gpu" : " 17 GB", | 20260506 16:50:58.300 | INFO | "huge" : "", | 20260506 16:50:58.300 | INFO | "huge_shared" : " 27 GB", | 20260506 16:50:58.300 | INFO | "malloc" : " 14 GB", | 20260506 16:50:58.300 | INFO | "pinned" : " 29 GB" | 20260506 16:50:58.300 | INFO | } | 20260506 16:50:58.300 | INFO | Perf report for the last 5.13451 seconds: | 20260506 16:50:58.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:50:58.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:58.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:58.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:58.300 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:50:58.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:58.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:58.300 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:50:58.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:58.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:50:58.300 | INFO | Uploaded 36GiB to GPU in 2.00182s (17.9836GiB/s) | 20260506 16:50:58.300 | INFO | Downloaded 34GiB to GPU in 1.93961s (17.5293GiB/s) | 20260506 16:51:03.441 | INFO | Analyzing chunk frames = [2925568,2926080) | 20260506 16:51:03.441 | INFO | MemInfo: { | 20260506 16:51:03.441 | INFO | "gpu" : " 17 GB", | 20260506 16:51:03.441 | INFO | "huge" : "", | 20260506 16:51:03.441 | INFO | "huge_shared" : " 27 GB", | 20260506 16:51:03.441 | INFO | "malloc" : " 14 GB", | 20260506 16:51:03.441 | INFO | "pinned" : " 29 GB" | 20260506 16:51:03.441 | INFO | } | 20260506 16:51:03.441 | INFO | Perf report for the last 5.14108 seconds: | 20260506 16:51:03.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:51:03.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:03.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:03.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:03.441 | INFO | ANALYSIS : 61.1%, 3.0, 94.7ms, 0 | 20260506 16:51:03.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:03.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:03.441 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:51:03.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:03.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:03.441 | INFO | Uploaded 36GiB to GPU in 1.97613s (18.2174GiB/s) | 20260506 16:51:03.441 | INFO | Downloaded 34GiB to GPU in 1.93243s (17.5945GiB/s) | 20260506 16:51:08.584 | INFO | Analyzing chunk frames = [2926080,2926592) | 20260506 16:51:08.584 | INFO | MemInfo: { | 20260506 16:51:08.584 | INFO | "gpu" : " 17 GB", | 20260506 16:51:08.584 | INFO | "huge" : "", | 20260506 16:51:08.584 | INFO | "huge_shared" : " 27 GB", | 20260506 16:51:08.584 | INFO | "malloc" : " 14 GB", | 20260506 16:51:08.584 | INFO | "pinned" : " 29 GB" | 20260506 16:51:08.584 | INFO | } | 20260506 16:51:08.584 | INFO | Perf report for the last 5.14258 seconds: | 20260506 16:51:08.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:51:08.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:08.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:08.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:08.584 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260506 16:51:08.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:08.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:08.584 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:51:08.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:08.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:08.584 | INFO | Uploaded 36GiB to GPU in 1.93186s (18.6349GiB/s) | 20260506 16:51:08.584 | INFO | Downloaded 34GiB to GPU in 1.87228s (18.1596GiB/s) | 20260506 16:51:13.725 | INFO | Analyzing chunk frames = [2926592,2927104) | 20260506 16:51:13.725 | INFO | MemInfo: { | 20260506 16:51:13.725 | INFO | "gpu" : " 17 GB", | 20260506 16:51:13.725 | INFO | "huge" : "", | 20260506 16:51:13.725 | INFO | "huge_shared" : " 27 GB", | 20260506 16:51:13.725 | INFO | "malloc" : " 14 GB", | 20260506 16:51:13.725 | INFO | "pinned" : " 29 GB" | 20260506 16:51:13.725 | INFO | } | 20260506 16:51:13.725 | INFO | Perf report for the last 5.14146 seconds: | 20260506 16:51:13.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:51:13.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:13.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:13.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:13.725 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 16:51:13.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:13.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:13.725 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:51:13.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:13.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:13.725 | INFO | Uploaded 36GiB to GPU in 2.01893s (17.8313GiB/s) | 20260506 16:51:13.726 | INFO | Downloaded 34GiB to GPU in 1.91095s (17.7922GiB/s) | 20260506 16:51:18.867 | INFO | Analyzing chunk frames = [2927104,2927616) | 20260506 16:51:18.867 | INFO | MemInfo: { | 20260506 16:51:18.867 | INFO | "gpu" : " 17 GB", | 20260506 16:51:18.867 | INFO | "huge" : "", | 20260506 16:51:18.867 | INFO | "huge_shared" : " 27 GB", | 20260506 16:51:18.867 | INFO | "malloc" : " 14 GB", | 20260506 16:51:18.867 | INFO | "pinned" : " 29 GB" | 20260506 16:51:18.867 | INFO | } | 20260506 16:51:18.868 | INFO | Perf report for the last 5.14238 seconds: | 20260506 16:51:18.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:51:18.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:18.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:18.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:18.868 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:51:18.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:18.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:18.868 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 16:51:18.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:18.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:18.868 | INFO | Uploaded 36GiB to GPU in 2.07748s (17.3286GiB/s) | 20260506 16:51:18.868 | INFO | Downloaded 34GiB to GPU in 1.97068s (17.2529GiB/s) | 20260506 16:51:24.011 | INFO | Analyzing chunk frames = [2927616,2928128) | 20260506 16:51:24.011 | INFO | MemInfo: { | 20260506 16:51:24.011 | INFO | "gpu" : " 17 GB", | 20260506 16:51:24.011 | INFO | "huge" : "", | 20260506 16:51:24.011 | INFO | "huge_shared" : " 27 GB", | 20260506 16:51:24.011 | INFO | "malloc" : " 14 GB", | 20260506 16:51:24.011 | INFO | "pinned" : " 29 GB" | 20260506 16:51:24.011 | INFO | } | 20260506 16:51:24.011 | INFO | Perf report for the last 5.14361 seconds: | 20260506 16:51:24.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:51:24.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:24.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:24.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:24.011 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 16:51:24.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:24.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:24.011 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 16:51:24.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:24.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:24.011 | INFO | Uploaded 36GiB to GPU in 2.01858s (17.8343GiB/s) | 20260506 16:51:24.011 | INFO | Downloaded 34GiB to GPU in 1.99882s (17.01GiB/s) | 20260506 16:51:29.146 | INFO | Analyzing chunk frames = [2928128,2928640) | 20260506 16:51:29.146 | INFO | MemInfo: { | 20260506 16:51:29.146 | INFO | "gpu" : " 17 GB", | 20260506 16:51:29.146 | INFO | "huge" : "", | 20260506 16:51:29.146 | INFO | "huge_shared" : " 27 GB", | 20260506 16:51:29.146 | INFO | "malloc" : " 14 GB", | 20260506 16:51:29.146 | INFO | "pinned" : " 29 GB" | 20260506 16:51:29.146 | INFO | } | 20260506 16:51:29.146 | INFO | Perf report for the last 5.13473 seconds: | 20260506 16:51:29.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:51:29.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:29.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:29.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:29.146 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:51:29.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:29.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:29.146 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 16:51:29.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:29.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:29.146 | INFO | Uploaded 36GiB to GPU in 2.01471s (17.8686GiB/s) | 20260506 16:51:29.146 | INFO | Downloaded 34GiB to GPU in 1.99073s (17.0792GiB/s) | 20260506 16:51:29.258 | INFO | Starting a round of Baz IO | 20260506 16:51:32.405 | INFO | Populating RTMetrics report | 20260506 16:51:32.406 | INFO | RTMetrics report populated | 20260506 16:51:32.406 | INFO | Opening tmp RTMetrics json file | 20260506 16:51:32.507 | INFO | Serializing RTMetrics to json | 20260506 16:51:32.679 | INFO | Serialization done, swapping tmp file now... | 20260506 16:51:32.722 | INFO | RTMetrics done | 20260506 16:51:32.949 | INFO | Finished a round of Baz IO: Wrote 2.077 GiB in 3.69 seconds | 20260506 16:51:32.949 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.04% / 0.01% / 63.96% | 20260506 16:51:32.949 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.54674% / 1.58946e-05% / 3.17891e-05% | 20260506 16:51:32.949 | INFO | Baz IO overall: | 20260506 16:51:32.949 | INFO | BytesWritten : 1540397533208 | 20260506 16:51:32.949 | INFO | NumEvents : 745334817310 | 20260506 16:51:34.279 | INFO | Analyzing chunk frames = [2928640,2929152) | 20260506 16:51:34.279 | INFO | MemInfo: { | 20260506 16:51:34.279 | INFO | "gpu" : " 17 GB", | 20260506 16:51:34.279 | INFO | "huge" : "", | 20260506 16:51:34.279 | INFO | "huge_shared" : " 27 GB", | 20260506 16:51:34.279 | INFO | "malloc" : " 14 GB", | 20260506 16:51:34.279 | INFO | "pinned" : " 29 GB" | 20260506 16:51:34.279 | INFO | } | 20260506 16:51:34.279 | INFO | Perf report for the last 5.1333 seconds: | 20260506 16:51:34.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:51:34.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:34.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:34.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:34.279 | INFO | ANALYSIS : 62.9%, 2.9, 95.4ms, 0 | 20260506 16:51:34.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:34.279 | INFO | RT_METRICS : 49.7%, 1.0, 26.1ms, 0 | 20260506 16:51:34.279 | INFO | PRE_HQ : 60.7%, 1.9, 58.6ms, 0 | 20260506 16:51:34.279 | INFO | BAZWRITER : 71.3%, 4.8, 175.8ms, 0 | 20260506 16:51:34.279 | INFO | BAZ_SAMPLING : 4.4%, 1.0, 2.3ms, 0 | 20260506 16:51:34.280 | INFO | Uploaded 36GiB to GPU in 2.03612s (17.6807GiB/s) | 20260506 16:51:34.280 | INFO | Downloaded 38GiB to GPU in 2.22741s (17.0602GiB/s) | 20260506 16:51:39.418 | INFO | Analyzing chunk frames = [2929152,2929664) | 20260506 16:51:39.418 | INFO | MemInfo: { | 20260506 16:51:39.418 | INFO | "gpu" : " 17 GB", | 20260506 16:51:39.418 | INFO | "huge" : "", | 20260506 16:51:39.418 | INFO | "huge_shared" : " 27 GB", | 20260506 16:51:39.418 | INFO | "malloc" : " 14 GB", | 20260506 16:51:39.418 | INFO | "pinned" : " 29 GB" | 20260506 16:51:39.418 | INFO | } | 20260506 16:51:39.418 | INFO | Perf report for the last 5.13883 seconds: | 20260506 16:51:39.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:51:39.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:39.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:39.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:39.418 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:51:39.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:39.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:39.418 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 16:51:39.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:39.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:39.418 | INFO | Uploaded 36GiB to GPU in 1.99323s (18.0611GiB/s) | 20260506 16:51:39.418 | INFO | Downloaded 34GiB to GPU in 1.93439s (17.5766GiB/s) | 20260506 16:51:44.558 | INFO | Analyzing chunk frames = [2929664,2930176) | 20260506 16:51:44.558 | INFO | MemInfo: { | 20260506 16:51:44.558 | INFO | "gpu" : " 17 GB", | 20260506 16:51:44.558 | INFO | "huge" : "", | 20260506 16:51:44.558 | INFO | "huge_shared" : " 27 GB", | 20260506 16:51:44.558 | INFO | "malloc" : " 14 GB", | 20260506 16:51:44.558 | INFO | "pinned" : " 29 GB" | 20260506 16:51:44.558 | INFO | } | 20260506 16:51:44.559 | INFO | Perf report for the last 5.14039 seconds: | 20260506 16:51:44.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:51:44.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:44.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:44.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:44.559 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 16:51:44.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:44.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:44.559 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:51:44.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:44.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:44.559 | INFO | Uploaded 36GiB to GPU in 1.97371s (18.2398GiB/s) | 20260506 16:51:44.559 | INFO | Downloaded 34GiB to GPU in 1.93193s (17.599GiB/s) | 20260506 16:51:49.705 | INFO | Analyzing chunk frames = [2930176,2930688) | 20260506 16:51:49.705 | INFO | MemInfo: { | 20260506 16:51:49.705 | INFO | "gpu" : " 17 GB", | 20260506 16:51:49.705 | INFO | "huge" : "", | 20260506 16:51:49.705 | INFO | "huge_shared" : " 27 GB", | 20260506 16:51:49.705 | INFO | "malloc" : " 14 GB", | 20260506 16:51:49.705 | INFO | "pinned" : " 29 GB" | 20260506 16:51:49.705 | INFO | } | 20260506 16:51:49.705 | INFO | Perf report for the last 5.14634 seconds: | 20260506 16:51:49.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:51:49.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:49.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:49.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:49.705 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 16:51:49.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:49.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:49.705 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260506 16:51:49.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:49.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:49.705 | INFO | Uploaded 36GiB to GPU in 1.93063s (18.6468GiB/s) | 20260506 16:51:49.705 | INFO | Downloaded 34GiB to GPU in 1.87255s (18.1571GiB/s) | 20260506 16:51:54.842 | INFO | Analyzing chunk frames = [2930688,2931200) | 20260506 16:51:54.843 | INFO | MemInfo: { | 20260506 16:51:54.843 | INFO | "gpu" : " 17 GB", | 20260506 16:51:54.843 | INFO | "huge" : "", | 20260506 16:51:54.843 | INFO | "huge_shared" : " 27 GB", | 20260506 16:51:54.843 | INFO | "malloc" : " 14 GB", | 20260506 16:51:54.843 | INFO | "pinned" : " 29 GB" | 20260506 16:51:54.843 | INFO | } | 20260506 16:51:54.843 | INFO | Perf report for the last 5.13783 seconds: | 20260506 16:51:54.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:51:54.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:54.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:54.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:54.843 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 16:51:54.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:54.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:54.843 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:51:54.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:54.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:54.843 | INFO | Uploaded 36GiB to GPU in 1.99288s (18.0643GiB/s) | 20260506 16:51:54.843 | INFO | Downloaded 34GiB to GPU in 1.91961s (17.712GiB/s) | 20260506 16:51:59.979 | INFO | Analyzing chunk frames = [2931200,2931712) | 20260506 16:51:59.979 | INFO | MemInfo: { | 20260506 16:51:59.979 | INFO | "gpu" : " 17 GB", | 20260506 16:51:59.979 | INFO | "huge" : "", | 20260506 16:51:59.979 | INFO | "huge_shared" : " 27 GB", | 20260506 16:51:59.979 | INFO | "malloc" : " 14 GB", | 20260506 16:51:59.979 | INFO | "pinned" : " 29 GB" | 20260506 16:51:59.979 | INFO | } | 20260506 16:51:59.979 | INFO | Perf report for the last 5.13651 seconds: | 20260506 16:51:59.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:51:59.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:59.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:59.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:59.979 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 16:51:59.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:59.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:59.979 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:51:59.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:59.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:51:59.980 | INFO | Uploaded 36GiB to GPU in 2.05878s (17.4861GiB/s) | 20260506 16:51:59.980 | INFO | Downloaded 34GiB to GPU in 1.95998s (17.3471GiB/s) | 20260506 16:52:05.113 | INFO | Analyzing chunk frames = [2931712,2932224) | 20260506 16:52:05.113 | INFO | MemInfo: { | 20260506 16:52:05.113 | INFO | "gpu" : " 17 GB", | 20260506 16:52:05.113 | INFO | "huge" : "", | 20260506 16:52:05.113 | INFO | "huge_shared" : " 27 GB", | 20260506 16:52:05.113 | INFO | "malloc" : " 14 GB", | 20260506 16:52:05.113 | INFO | "pinned" : " 29 GB" | 20260506 16:52:05.113 | INFO | } | 20260506 16:52:05.113 | INFO | Perf report for the last 5.13365 seconds: | 20260506 16:52:05.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:52:05.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:05.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:05.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:05.113 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 16:52:05.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:05.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:05.113 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 16:52:05.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:05.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:05.113 | INFO | Uploaded 36GiB to GPU in 2.04396s (17.6129GiB/s) | 20260506 16:52:05.113 | INFO | Downloaded 34GiB to GPU in 2.00708s (16.9401GiB/s) | 20260506 16:52:10.253 | INFO | Analyzing chunk frames = [2932224,2932736) | 20260506 16:52:10.253 | INFO | MemInfo: { | 20260506 16:52:10.253 | INFO | "gpu" : " 17 GB", | 20260506 16:52:10.253 | INFO | "huge" : "", | 20260506 16:52:10.253 | INFO | "huge_shared" : " 27 GB", | 20260506 16:52:10.253 | INFO | "malloc" : " 14 GB", | 20260506 16:52:10.253 | INFO | "pinned" : " 29 GB" | 20260506 16:52:10.253 | INFO | } | 20260506 16:52:10.253 | INFO | Perf report for the last 5.14044 seconds: | 20260506 16:52:10.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:52:10.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:10.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:10.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:10.253 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:52:10.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:10.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:10.253 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 16:52:10.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:10.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:10.254 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:52:10.254 | INFO | Total mem usage: 18 GiB | 20260506 16:52:10.254 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:52:10.254 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.3%) | 20260506 16:52:10.254 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:52:10.254 | INFO | High Water Breakdown: | 20260506 16:52:10.254 | INFO | 61.7% : 9.9 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:52:10.254 | INFO | 38.1% : 6.11 GiB ( 91.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:52:10.254 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:52:10.254 | INFO | ------------------------------------------ | 20260506 16:52:10.254 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:52:10.254 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:52:10.254 | INFO | Total mem usage: 26.5 GiB | 20260506 16:52:10.254 | INFO | Uploaded 36GiB to GPU in 1.99956s (18.004GiB/s) | 20260506 16:52:10.254 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:52:10.254 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:52:10.254 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:52:10.254 | INFO | Downloaded 34GiB to GPU in 1.96627s (17.2916GiB/s) | 20260506 16:52:10.254 | INFO | High Water Breakdown: | 20260506 16:52:10.254 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:52:10.254 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:52:10.254 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:52:10.254 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:52:10.254 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:52:10.254 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:52:10.254 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:52:10.254 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:52:10.254 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:52:10.254 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:52:10.254 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:52:10.254 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:52:10.254 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:52:10.254 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:52:10.254 | INFO | ------------------------------------------ | 20260506 16:52:10.254 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:52:10.254 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:52:10.254 | INFO | Total mem usage: 25.6 GiB | 20260506 16:52:10.254 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:52:10.254 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:52:10.254 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:52:10.254 | INFO | High Water Breakdown: | 20260506 16:52:10.254 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:52:10.254 | INFO | ------------------------------------------ | 20260506 16:52:10.254 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:52:10.254 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:52:10.254 | INFO | Total mem usage: 15.7 GiB | 20260506 16:52:10.254 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:52:10.254 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:52:10.254 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 16:52:10.254 | INFO | High Water Breakdown: | 20260506 16:52:10.254 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:52:10.254 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:52:10.254 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:52:10.254 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:52:10.254 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:52:10.254 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:52:10.254 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:52:10.254 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:52:10.254 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:52:10.254 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:52:10.254 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:52:10.254 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:52:10.254 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:52:10.254 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:52:10.254 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:52:10.254 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:52:10.254 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:52:10.254 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:52:10.254 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:52:10.254 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:52:10.254 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:52:10.254 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:52:10.255 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:52:10.255 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:52:10.255 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:52:10.255 | INFO | ------------------------------------------ | 20260506 16:52:10.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:52:10.255 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:52:10.255 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:52:10.255 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:52:10.255 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.53e+01, 1.34e+01 / 2.03e+00 | 20260506 16:52:10.255 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.02e-01, 1.12e-02 / 8.38e-03 | 20260506 16:52:10.255 | INFO | PerfReport: 11.59%, Baseline -- 3.64e+00 / 4.43e+00, 5.56e-01 / 3.48e-01 | 20260506 16:52:10.255 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.48e+00, 1.86e-01 / 1.33e-01 | 20260506 16:52:10.255 | INFO | PerfReport: 15.91%, DME -- 3.99e+01 / 4.41e+01, 3.80e+00 / 2.05e+00 | 20260506 16:52:10.255 | INFO | PerfReport: 56.97%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.47e+00 / 1.89e+00 | 20260506 16:52:10.255 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.29e+00 / 2.86e+00, 3.86e-01 / 2.39e-01 | 20260506 16:52:10.255 | INFO | PerfReport: 3.48%, Metrics -- 1.09e+00 / 1.46e+00, 2.82e-01 / 1.75e-01 | 20260506 16:52:10.255 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.79e-01 / 4.30e-01, 5.89e-02 / 2.60e-02 | 20260506 16:52:10.255 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:52:10.255 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:52:10.255 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:52:10.255 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 16:52:10.255 | INFO | PerfReport: 21.49%, Upload -- 2.09e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260506 16:52:10.255 | INFO | PerfReport: 21.16%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 16:52:10.255 | INFO | PerfReport: 32.22%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 16:52:10.255 | INFO | PerfReport: 1.46%, LockUpload -- 1.43e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260506 16:52:10.255 | INFO | PerfReport: 1.03%, LockDownload -- 1.00e+00 / 0.00e+00, 2.61e+00 / 0.00e+00 | 20260506 16:52:10.255 | INFO | PerfReport: 22.61%, LockCompute -- 2.20e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 16:52:10.255 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.48e-02 / 0.00e+00 | 20260506 16:52:10.371 | INFO | Starting a round of Baz IO | 20260506 16:52:13.515 | INFO | Populating RTMetrics report | 20260506 16:52:13.515 | INFO | RTMetrics report populated | 20260506 16:52:13.515 | INFO | Opening tmp RTMetrics json file | 20260506 16:52:13.516 | INFO | Serializing RTMetrics to json | 20260506 16:52:13.518 | INFO | Serialization done, swapping tmp file now... | 20260506 16:52:13.519 | INFO | RTMetrics done | 20260506 16:52:13.939 | INFO | Finished a round of Baz IO: Wrote 2.078 GiB in 3.567 seconds | 20260506 16:52:13.939 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.03% / 0.01% / 63.96% | 20260506 16:52:13.939 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.5434% / 7.94729e-06% / 4.37101e-05% | 20260506 16:52:13.939 | INFO | Baz IO overall: | 20260506 16:52:13.939 | INFO | BytesWritten : 1542628588441 | 20260506 16:52:13.939 | INFO | NumEvents : 746412389775 | 20260506 16:52:15.393 | INFO | Analyzing chunk frames = [2932736,2933248) | 20260506 16:52:15.393 | INFO | MemInfo: { | 20260506 16:52:15.393 | INFO | "gpu" : " 17 GB", | 20260506 16:52:15.393 | INFO | "huge" : "", | 20260506 16:52:15.393 | INFO | "huge_shared" : " 27 GB", | 20260506 16:52:15.393 | INFO | "malloc" : " 14 GB", | 20260506 16:52:15.393 | INFO | "pinned" : " 29 GB" | 20260506 16:52:15.393 | INFO | } | 20260506 16:52:15.393 | INFO | Perf report for the last 5.13952 seconds: | 20260506 16:52:15.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:52:15.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:15.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:15.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:15.393 | INFO | ANALYSIS : 62.9%, 2.9, 95.2ms, 0 | 20260506 16:52:15.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:15.393 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260506 16:52:15.393 | INFO | PRE_HQ : 60.9%, 1.9, 60.1ms, 0 | 20260506 16:52:15.393 | INFO | BAZWRITER : 68.6%, 4.8, 170.0ms, 0 | 20260506 16:52:15.393 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 16:52:15.393 | INFO | Uploaded 36GiB to GPU in 2.04128s (17.636GiB/s) | 20260506 16:52:15.393 | INFO | Downloaded 38GiB to GPU in 2.22566s (17.0736GiB/s) | 20260506 16:52:20.531 | INFO | Analyzing chunk frames = [2933248,2933760) | 20260506 16:52:20.531 | INFO | MemInfo: { | 20260506 16:52:20.531 | INFO | "gpu" : " 17 GB", | 20260506 16:52:20.531 | INFO | "huge" : "", | 20260506 16:52:20.531 | INFO | "huge_shared" : " 27 GB", | 20260506 16:52:20.532 | INFO | "malloc" : " 14 GB", | 20260506 16:52:20.532 | INFO | "pinned" : " 29 GB" | 20260506 16:52:20.532 | INFO | } | 20260506 16:52:20.532 | INFO | Perf report for the last 5.13873 seconds: | 20260506 16:52:20.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:52:20.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:20.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:20.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:20.532 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:52:20.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:20.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:20.532 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 16:52:20.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:20.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:20.532 | INFO | Uploaded 36GiB to GPU in 2.00165s (17.9852GiB/s) | 20260506 16:52:20.532 | INFO | Downloaded 34GiB to GPU in 1.93781s (17.5456GiB/s) | 20260506 16:52:25.675 | INFO | Analyzing chunk frames = [2933760,2934272) | 20260506 16:52:25.675 | INFO | MemInfo: { | 20260506 16:52:25.675 | INFO | "gpu" : " 17 GB", | 20260506 16:52:25.675 | INFO | "huge" : "", | 20260506 16:52:25.675 | INFO | "huge_shared" : " 27 GB", | 20260506 16:52:25.675 | INFO | "malloc" : " 14 GB", | 20260506 16:52:25.675 | INFO | "pinned" : " 29 GB" | 20260506 16:52:25.675 | INFO | } | 20260506 16:52:25.675 | INFO | Perf report for the last 5.1433 seconds: | 20260506 16:52:25.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:52:25.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:25.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:25.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:25.675 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 16:52:25.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:25.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:25.675 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:52:25.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:25.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:25.675 | INFO | Uploaded 36GiB to GPU in 1.94088s (18.5482GiB/s) | 20260506 16:52:25.675 | INFO | Downloaded 34GiB to GPU in 1.88812s (18.0073GiB/s) | 20260506 16:52:30.824 | INFO | Analyzing chunk frames = [2934272,2934784) | 20260506 16:52:30.824 | INFO | MemInfo: { | 20260506 16:52:30.824 | INFO | "gpu" : " 17 GB", | 20260506 16:52:30.824 | INFO | "huge" : "", | 20260506 16:52:30.824 | INFO | "huge_shared" : " 27 GB", | 20260506 16:52:30.824 | INFO | "malloc" : " 14 GB", | 20260506 16:52:30.824 | INFO | "pinned" : " 29 GB" | 20260506 16:52:30.824 | INFO | } | 20260506 16:52:30.824 | INFO | Perf report for the last 5.1491 seconds: | 20260506 16:52:30.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:52:30.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:30.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:30.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:30.824 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 16:52:30.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:30.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:30.824 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260506 16:52:30.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:30.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:30.824 | INFO | Uploaded 36GiB to GPU in 1.92947s (18.658GiB/s) | 20260506 16:52:30.824 | INFO | Downloaded 34GiB to GPU in 1.86954s (18.1863GiB/s) | 20260506 16:52:35.959 | INFO | Analyzing chunk frames = [2934784,2935296) | 20260506 16:52:35.959 | INFO | MemInfo: { | 20260506 16:52:35.959 | INFO | "gpu" : " 17 GB", | 20260506 16:52:35.959 | INFO | "huge" : "", | 20260506 16:52:35.959 | INFO | "huge_shared" : " 27 GB", | 20260506 16:52:35.959 | INFO | "malloc" : " 14 GB", | 20260506 16:52:35.959 | INFO | "pinned" : " 29 GB" | 20260506 16:52:35.959 | INFO | } | 20260506 16:52:35.959 | INFO | Perf report for the last 5.13501 seconds: | 20260506 16:52:35.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:52:35.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:35.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:35.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:35.959 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 16:52:35.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:35.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:35.959 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:52:35.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:35.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:35.959 | INFO | Uploaded 36GiB to GPU in 2.01102s (17.9014GiB/s) | 20260506 16:52:35.959 | INFO | Downloaded 34GiB to GPU in 1.90626s (17.836GiB/s) | 20260506 16:52:41.097 | INFO | Analyzing chunk frames = [2935296,2935808) | 20260506 16:52:41.097 | INFO | MemInfo: { | 20260506 16:52:41.097 | INFO | "gpu" : " 17 GB", | 20260506 16:52:41.097 | INFO | "huge" : "", | 20260506 16:52:41.097 | INFO | "huge_shared" : " 27 GB", | 20260506 16:52:41.097 | INFO | "malloc" : " 14 GB", | 20260506 16:52:41.097 | INFO | "pinned" : " 29 GB" | 20260506 16:52:41.097 | INFO | } | 20260506 16:52:41.097 | INFO | Perf report for the last 5.13801 seconds: | 20260506 16:52:41.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:52:41.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:41.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:41.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:41.097 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 16:52:41.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:41.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:41.097 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:52:41.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:41.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:41.097 | INFO | Uploaded 36GiB to GPU in 2.05256s (17.539GiB/s) | 20260506 16:52:41.097 | INFO | Downloaded 34GiB to GPU in 1.95491s (17.3921GiB/s) | 20260506 16:52:46.230 | INFO | Analyzing chunk frames = [2935808,2936320) | 20260506 16:52:46.230 | INFO | MemInfo: { | 20260506 16:52:46.230 | INFO | "gpu" : " 17 GB", | 20260506 16:52:46.230 | INFO | "huge" : "", | 20260506 16:52:46.230 | INFO | "huge_shared" : " 27 GB", | 20260506 16:52:46.230 | INFO | "malloc" : " 14 GB", | 20260506 16:52:46.230 | INFO | "pinned" : " 29 GB" | 20260506 16:52:46.230 | INFO | } | 20260506 16:52:46.230 | INFO | Perf report for the last 5.13258 seconds: | 20260506 16:52:46.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:52:46.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:46.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:46.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:46.230 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 16:52:46.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:46.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:46.230 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 16:52:46.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:46.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:46.230 | INFO | Uploaded 36GiB to GPU in 2.0245s (17.7822GiB/s) | 20260506 16:52:46.230 | INFO | Downloaded 34GiB to GPU in 2.01657s (16.8603GiB/s) | 20260506 16:52:51.368 | INFO | Analyzing chunk frames = [2936320,2936832) | 20260506 16:52:51.368 | INFO | MemInfo: { | 20260506 16:52:51.368 | INFO | "gpu" : " 17 GB", | 20260506 16:52:51.368 | INFO | "huge" : "", | 20260506 16:52:51.368 | INFO | "huge_shared" : " 27 GB", | 20260506 16:52:51.368 | INFO | "malloc" : " 14 GB", | 20260506 16:52:51.368 | INFO | "pinned" : " 29 GB" | 20260506 16:52:51.368 | INFO | } | 20260506 16:52:51.368 | INFO | Perf report for the last 5.13875 seconds: | 20260506 16:52:51.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:52:51.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:51.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:51.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:51.368 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 16:52:51.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:51.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:51.368 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:52:51.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:51.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:51.369 | INFO | Uploaded 36GiB to GPU in 2.04705s (17.5863GiB/s) | 20260506 16:52:51.369 | INFO | Downloaded 34GiB to GPU in 1.9722s (17.2396GiB/s) | 20260506 16:52:51.480 | INFO | Starting a round of Baz IO | 20260506 16:52:54.625 | INFO | Populating RTMetrics report | 20260506 16:52:54.626 | INFO | RTMetrics report populated | 20260506 16:52:54.626 | INFO | Opening tmp RTMetrics json file | 20260506 16:52:54.642 | INFO | Serializing RTMetrics to json | 20260506 16:52:54.672 | INFO | Serialization done, swapping tmp file now... | 20260506 16:52:54.678 | INFO | RTMetrics done | 20260506 16:52:55.004 | INFO | Finished a round of Baz IO: Wrote 2.077 GiB in 3.523 seconds | 20260506 16:52:55.004 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.02% / 0.01% / 63.97% | 20260506 16:52:55.004 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.54086% / 3.97364e-06% / 5.5631e-05% | 20260506 16:52:55.004 | INFO | Baz IO overall: | 20260506 16:52:55.004 | INFO | BytesWritten : 1544859023902 | 20260506 16:52:55.004 | INFO | NumEvents : 747489770450 | 20260506 16:52:56.510 | INFO | Analyzing chunk frames = [2936832,2937344) | 20260506 16:52:56.510 | INFO | MemInfo: { | 20260506 16:52:56.510 | INFO | "gpu" : " 17 GB", | 20260506 16:52:56.510 | INFO | "huge" : "", | 20260506 16:52:56.510 | INFO | "huge_shared" : " 27 GB", | 20260506 16:52:56.510 | INFO | "malloc" : " 14 GB", | 20260506 16:52:56.510 | INFO | "pinned" : " 29 GB" | 20260506 16:52:56.510 | INFO | } | 20260506 16:52:56.510 | INFO | Perf report for the last 5.14195 seconds: | 20260506 16:52:56.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:52:56.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:56.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:56.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:56.510 | INFO | ANALYSIS : 62.8%, 2.9, 95.4ms, 0 | 20260506 16:52:56.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:52:56.510 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 16:52:56.510 | INFO | PRE_HQ : 60.6%, 1.9, 58.3ms, 0 | 20260506 16:52:56.511 | INFO | BAZWRITER : 67.8%, 4.8, 168.7ms, 0 | 20260506 16:52:56.511 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 16:52:56.511 | INFO | Uploaded 36GiB to GPU in 2.05366s (17.5297GiB/s) | 20260506 16:52:56.511 | INFO | Downloaded 38GiB to GPU in 2.22845s (17.0522GiB/s) | 20260506 16:53:01.649 | INFO | Analyzing chunk frames = [2937344,2937856) | 20260506 16:53:01.649 | INFO | MemInfo: { | 20260506 16:53:01.649 | INFO | "gpu" : " 17 GB", | 20260506 16:53:01.649 | INFO | "huge" : "", | 20260506 16:53:01.649 | INFO | "huge_shared" : " 27 GB", | 20260506 16:53:01.649 | INFO | "malloc" : " 14 GB", | 20260506 16:53:01.649 | INFO | "pinned" : " 29 GB" | 20260506 16:53:01.649 | INFO | } | 20260506 16:53:01.649 | INFO | Perf report for the last 5.13901 seconds: | 20260506 16:53:01.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:53:01.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:01.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:01.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:01.650 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:53:01.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:01.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:01.650 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:53:01.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:01.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:01.650 | INFO | Uploaded 36GiB to GPU in 2.01842s (17.8357GiB/s) | 20260506 16:53:01.650 | INFO | Downloaded 34GiB to GPU in 1.9497s (17.4386GiB/s) | 20260506 16:53:06.793 | INFO | Analyzing chunk frames = [2937856,2938368) | 20260506 16:53:06.793 | INFO | MemInfo: { | 20260506 16:53:06.793 | INFO | "gpu" : " 17 GB", | 20260506 16:53:06.793 | INFO | "huge" : "", | 20260506 16:53:06.793 | INFO | "huge_shared" : " 27 GB", | 20260506 16:53:06.793 | INFO | "malloc" : " 14 GB", | 20260506 16:53:06.793 | INFO | "pinned" : " 29 GB" | 20260506 16:53:06.793 | INFO | } | 20260506 16:53:06.793 | INFO | Perf report for the last 5.14378 seconds: | 20260506 16:53:06.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:53:06.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:06.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:06.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:06.793 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 16:53:06.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:06.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:06.793 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:53:06.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:06.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:06.794 | INFO | Uploaded 36GiB to GPU in 1.97054s (18.2691GiB/s) | 20260506 16:53:06.794 | INFO | Downloaded 34GiB to GPU in 1.9319s (17.5992GiB/s) | 20260506 16:53:11.928 | INFO | Analyzing chunk frames = [2938368,2938880) | 20260506 16:53:11.928 | INFO | MemInfo: { | 20260506 16:53:11.928 | INFO | "gpu" : " 17 GB", | 20260506 16:53:11.928 | INFO | "huge" : "", | 20260506 16:53:11.928 | INFO | "huge_shared" : " 27 GB", | 20260506 16:53:11.928 | INFO | "malloc" : " 14 GB", | 20260506 16:53:11.928 | INFO | "pinned" : " 29 GB" | 20260506 16:53:11.928 | INFO | } | 20260506 16:53:11.928 | INFO | Perf report for the last 5.13524 seconds: | 20260506 16:53:11.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:53:11.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:11.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:11.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:11.928 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 16:53:11.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:11.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:11.929 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:53:11.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:11.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:11.929 | INFO | Uploaded 36GiB to GPU in 1.92038s (18.7463GiB/s) | 20260506 16:53:11.929 | INFO | Downloaded 34GiB to GPU in 1.87158s (18.1665GiB/s) | 20260506 16:53:17.068 | INFO | Analyzing chunk frames = [2938880,2939392) | 20260506 16:53:17.068 | INFO | MemInfo: { | 20260506 16:53:17.068 | INFO | "gpu" : " 17 GB", | 20260506 16:53:17.068 | INFO | "huge" : "", | 20260506 16:53:17.068 | INFO | "huge_shared" : " 27 GB", | 20260506 16:53:17.068 | INFO | "malloc" : " 14 GB", | 20260506 16:53:17.068 | INFO | "pinned" : " 29 GB" | 20260506 16:53:17.068 | INFO | } | 20260506 16:53:17.068 | INFO | Perf report for the last 5.13955 seconds: | 20260506 16:53:17.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:53:17.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:17.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:17.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:17.068 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 16:53:17.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:17.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:17.068 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:53:17.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:17.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:17.068 | INFO | Uploaded 36GiB to GPU in 1.95089s (18.4531GiB/s) | 20260506 16:53:17.068 | INFO | Downloaded 34GiB to GPU in 1.8793s (18.0918GiB/s) | 20260506 16:53:22.205 | INFO | Analyzing chunk frames = [2939392,2939904) | 20260506 16:53:22.205 | INFO | MemInfo: { | 20260506 16:53:22.205 | INFO | "gpu" : " 17 GB", | 20260506 16:53:22.205 | INFO | "huge" : "", | 20260506 16:53:22.205 | INFO | "huge_shared" : " 27 GB", | 20260506 16:53:22.205 | INFO | "malloc" : " 14 GB", | 20260506 16:53:22.205 | INFO | "pinned" : " 29 GB" | 20260506 16:53:22.205 | INFO | } | 20260506 16:53:22.205 | INFO | Perf report for the last 5.13748 seconds: | 20260506 16:53:22.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:53:22.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:22.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:22.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:22.206 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 16:53:22.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:22.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:22.206 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260506 16:53:22.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:22.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:22.206 | INFO | Uploaded 36GiB to GPU in 2.02479s (17.7796GiB/s) | 20260506 16:53:22.206 | INFO | Downloaded 34GiB to GPU in 1.94535s (17.4775GiB/s) | 20260506 16:53:27.346 | INFO | Analyzing chunk frames = [2939904,2940416) | 20260506 16:53:27.346 | INFO | MemInfo: { | 20260506 16:53:27.346 | INFO | "gpu" : " 17 GB", | 20260506 16:53:27.346 | INFO | "huge" : "", | 20260506 16:53:27.346 | INFO | "huge_shared" : " 27 GB", | 20260506 16:53:27.346 | INFO | "malloc" : " 14 GB", | 20260506 16:53:27.346 | INFO | "pinned" : " 29 GB" | 20260506 16:53:27.346 | INFO | } | 20260506 16:53:27.346 | INFO | Perf report for the last 5.14081 seconds: | 20260506 16:53:27.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:53:27.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:27.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:27.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:27.346 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 16:53:27.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:27.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:27.346 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:53:27.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:27.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:27.347 | INFO | Uploaded 36GiB to GPU in 2.02979s (17.7358GiB/s) | 20260506 16:53:27.347 | INFO | Downloaded 34GiB to GPU in 2s (17GiB/s) | 20260506 16:53:32.488 | INFO | Analyzing chunk frames = [2940416,2940928) | 20260506 16:53:32.488 | INFO | MemInfo: { | 20260506 16:53:32.488 | INFO | "gpu" : " 17 GB", | 20260506 16:53:32.488 | INFO | "huge" : "", | 20260506 16:53:32.488 | INFO | "huge_shared" : " 27 GB", | 20260506 16:53:32.488 | INFO | "malloc" : " 14 GB", | 20260506 16:53:32.488 | INFO | "pinned" : " 29 GB" | 20260506 16:53:32.488 | INFO | } | 20260506 16:53:32.488 | INFO | Perf report for the last 5.14151 seconds: | 20260506 16:53:32.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:53:32.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:32.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:32.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:32.488 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 16:53:32.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:32.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:32.488 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260506 16:53:32.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:32.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:32.488 | INFO | Uploaded 36GiB to GPU in 2.00897s (17.9196GiB/s) | 20260506 16:53:32.488 | INFO | Downloaded 34GiB to GPU in 1.9887s (17.0966GiB/s) | 20260506 16:53:32.488 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:53:32.488 | INFO | Total mem usage: 18 GiB | 20260506 16:53:32.488 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:53:32.488 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -10.8%) | 20260506 16:53:32.488 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:53:32.488 | INFO | High Water Breakdown: | 20260506 16:53:32.488 | INFO | 61.9% : 10.1 GiB ( 88.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:53:32.488 | INFO | 37.9% : 6.16 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:53:32.488 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:53:32.488 | INFO | ------------------------------------------ | 20260506 16:53:32.488 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:53:32.488 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:53:32.488 | INFO | Total mem usage: 26.5 GiB | 20260506 16:53:32.488 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:53:32.488 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:53:32.488 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:53:32.488 | INFO | High Water Breakdown: | 20260506 16:53:32.489 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:53:32.489 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:53:32.489 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:53:32.489 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:53:32.489 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:53:32.489 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:53:32.489 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:53:32.489 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:53:32.489 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:53:32.489 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:53:32.489 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:53:32.489 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:53:32.489 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:53:32.489 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:53:32.489 | INFO | ------------------------------------------ | 20260506 16:53:32.489 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:53:32.489 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:53:32.489 | INFO | Total mem usage: 25.6 GiB | 20260506 16:53:32.489 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:53:32.489 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:53:32.489 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:53:32.489 | INFO | High Water Breakdown: | 20260506 16:53:32.489 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:53:32.489 | INFO | ------------------------------------------ | 20260506 16:53:32.489 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:53:32.489 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:53:32.489 | INFO | Total mem usage: 15.7 GiB | 20260506 16:53:32.489 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:53:32.489 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:53:32.489 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:53:32.489 | INFO | High Water Breakdown: | 20260506 16:53:32.489 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:53:32.489 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:53:32.489 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:53:32.489 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:53:32.489 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:53:32.489 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:53:32.489 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:53:32.489 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:53:32.489 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:53:32.489 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:53:32.489 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:53:32.489 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:53:32.489 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:53:32.489 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:53:32.489 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:53:32.489 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:53:32.489 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:53:32.489 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:53:32.489 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:53:32.489 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:53:32.489 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:53:32.489 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:53:32.489 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:53:32.489 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:53:32.489 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:53:32.489 | INFO | ------------------------------------------ | 20260506 16:53:32.489 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:53:32.489 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:53:32.489 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:53:32.489 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:53:32.489 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.56e+01, 1.35e+01 / 2.48e+00 | 20260506 16:53:32.489 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.03e-01, 9.95e-03 / 7.56e-03 | 20260506 16:53:32.489 | INFO | PerfReport: 11.60%, Baseline -- 3.64e+00 / 4.44e+00, 5.54e-01 / 2.94e-01 | 20260506 16:53:32.489 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.84e-01 / 1.23e-01 | 20260506 16:53:32.489 | INFO | PerfReport: 15.92%, DME -- 3.99e+01 / 4.47e+01, 4.12e+00 / 2.66e+00 | 20260506 16:53:32.489 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.78e+01 / 2.01e+01, 1.41e+00 / 1.91e+00 | 20260506 16:53:32.489 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.29e+00 / 2.87e+00, 3.94e-01 / 2.30e-01 | 20260506 16:53:32.489 | INFO | PerfReport: 3.47%, Metrics -- 1.09e+00 / 1.46e+00, 2.80e-01 / 1.73e-01 | 20260506 16:53:32.490 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.78e-01 / 4.28e-01, 5.85e-02 / 2.54e-02 | 20260506 16:53:32.490 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:53:32.490 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:53:32.490 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:53:32.490 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 16:53:32.490 | INFO | PerfReport: 21.44%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 16:53:32.490 | INFO | PerfReport: 21.14%, Download -- 2.06e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 16:53:32.490 | INFO | PerfReport: 32.25%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 16:53:32.490 | INFO | PerfReport: 1.42%, LockUpload -- 1.38e+00 / 0.00e+00, 5.77e+00 / 0.00e+00 | 20260506 16:53:32.490 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260506 16:53:32.490 | INFO | PerfReport: 22.64%, LockCompute -- 2.20e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 16:53:32.490 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260506 16:53:32.606 | INFO | Starting a round of Baz IO | 20260506 16:53:35.747 | INFO | Populating RTMetrics report | 20260506 16:53:35.747 | INFO | RTMetrics report populated | 20260506 16:53:35.747 | INFO | Opening tmp RTMetrics json file | 20260506 16:53:35.901 | INFO | Serializing RTMetrics to json | 20260506 16:53:35.915 | INFO | Serialization done, swapping tmp file now... | 20260506 16:53:35.938 | INFO | RTMetrics done | 20260506 16:53:36.234 | INFO | Finished a round of Baz IO: Wrote 2.077 GiB in 3.627 seconds | 20260506 16:53:36.234 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.01% / 0.01% / 63.98% | 20260506 16:53:36.234 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.5374% / 2.38419e-05% / 4.37101e-05% | 20260506 16:53:36.234 | INFO | Baz IO overall: | 20260506 16:53:36.234 | INFO | BytesWritten : 1547088796686 | 20260506 16:53:36.234 | INFO | NumEvents : 748566540949 | 20260506 16:53:37.623 | INFO | Analyzing chunk frames = [2940928,2941440) | 20260506 16:53:37.623 | INFO | MemInfo: { | 20260506 16:53:37.623 | INFO | "gpu" : " 17 GB", | 20260506 16:53:37.623 | INFO | "huge" : "", | 20260506 16:53:37.623 | INFO | "huge_shared" : " 27 GB", | 20260506 16:53:37.623 | INFO | "malloc" : " 14 GB", | 20260506 16:53:37.623 | INFO | "pinned" : " 29 GB" | 20260506 16:53:37.623 | INFO | } | 20260506 16:53:37.623 | INFO | Perf report for the last 5.13564 seconds: | 20260506 16:53:37.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:53:37.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:37.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:37.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:37.624 | INFO | ANALYSIS : 62.9%, 2.9, 95.6ms, 0 | 20260506 16:53:37.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:37.624 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260506 16:53:37.624 | INFO | PRE_HQ : 60.4%, 1.9, 58.4ms, 0 | 20260506 16:53:37.624 | INFO | BAZWRITER : 70.3%, 4.8, 172.5ms, 0 | 20260506 16:53:37.624 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.4ms, 0 | 20260506 16:53:37.624 | INFO | Uploaded 36GiB to GPU in 2.08436s (17.2715GiB/s) | 20260506 16:53:37.624 | INFO | Downloaded 38GiB to GPU in 2.21902s (17.1246GiB/s) | 20260506 16:53:42.768 | INFO | Analyzing chunk frames = [2941440,2941952) | 20260506 16:53:42.768 | INFO | MemInfo: { | 20260506 16:53:42.768 | INFO | "gpu" : " 17 GB", | 20260506 16:53:42.768 | INFO | "huge" : "", | 20260506 16:53:42.768 | INFO | "huge_shared" : " 27 GB", | 20260506 16:53:42.768 | INFO | "malloc" : " 14 GB", | 20260506 16:53:42.768 | INFO | "pinned" : " 29 GB" | 20260506 16:53:42.768 | INFO | } | 20260506 16:53:42.768 | INFO | Perf report for the last 5.14494 seconds: | 20260506 16:53:42.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:53:42.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:42.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:42.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:42.769 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 16:53:42.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:42.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:42.769 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 16:53:42.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:42.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:42.769 | INFO | Uploaded 36GiB to GPU in 2.00544s (17.9512GiB/s) | 20260506 16:53:42.769 | INFO | Downloaded 34GiB to GPU in 1.95113s (17.4258GiB/s) | 20260506 16:53:47.913 | INFO | Analyzing chunk frames = [2941952,2942464) | 20260506 16:53:47.913 | INFO | MemInfo: { | 20260506 16:53:47.913 | INFO | "gpu" : " 17 GB", | 20260506 16:53:47.913 | INFO | "huge" : "", | 20260506 16:53:47.913 | INFO | "huge_shared" : " 27 GB", | 20260506 16:53:47.913 | INFO | "malloc" : " 14 GB", | 20260506 16:53:47.913 | INFO | "pinned" : " 29 GB" | 20260506 16:53:47.913 | INFO | } | 20260506 16:53:47.913 | INFO | Perf report for the last 5.14459 seconds: | 20260506 16:53:47.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:53:47.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:47.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:47.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:47.913 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 16:53:47.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:47.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:47.913 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:53:47.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:47.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:47.914 | INFO | Uploaded 36GiB to GPU in 1.97818s (18.1985GiB/s) | 20260506 16:53:47.914 | INFO | Downloaded 34GiB to GPU in 1.92231s (17.687GiB/s) | 20260506 16:53:53.047 | INFO | Analyzing chunk frames = [2942464,2942976) | 20260506 16:53:53.047 | INFO | MemInfo: { | 20260506 16:53:53.047 | INFO | "gpu" : " 17 GB", | 20260506 16:53:53.047 | INFO | "huge" : "", | 20260506 16:53:53.047 | INFO | "huge_shared" : " 27 GB", | 20260506 16:53:53.047 | INFO | "malloc" : " 14 GB", | 20260506 16:53:53.047 | INFO | "pinned" : " 29 GB" | 20260506 16:53:53.047 | INFO | } | 20260506 16:53:53.047 | INFO | Perf report for the last 5.13374 seconds: | 20260506 16:53:53.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:53:53.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:53.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:53.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:53.047 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 16:53:53.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:53.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:53.047 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:53:53.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:53.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:53.047 | INFO | Uploaded 36GiB to GPU in 1.94483s (18.5106GiB/s) | 20260506 16:53:53.047 | INFO | Downloaded 34GiB to GPU in 1.87271s (18.1555GiB/s) | 20260506 16:53:58.185 | INFO | Analyzing chunk frames = [2942976,2943488) | 20260506 16:53:58.185 | INFO | MemInfo: { | 20260506 16:53:58.185 | INFO | "gpu" : " 17 GB", | 20260506 16:53:58.185 | INFO | "huge" : "", | 20260506 16:53:58.185 | INFO | "huge_shared" : " 27 GB", | 20260506 16:53:58.185 | INFO | "malloc" : " 14 GB", | 20260506 16:53:58.185 | INFO | "pinned" : " 29 GB" | 20260506 16:53:58.185 | INFO | } | 20260506 16:53:58.185 | INFO | Perf report for the last 5.1385 seconds: | 20260506 16:53:58.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:53:58.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:58.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:58.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:58.185 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 16:53:58.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:58.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:58.185 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:53:58.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:58.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:53:58.186 | INFO | Uploaded 36GiB to GPU in 1.97809s (18.1994GiB/s) | 20260506 16:53:58.186 | INFO | Downloaded 34GiB to GPU in 1.92487s (17.6635GiB/s) | 20260506 16:54:03.317 | INFO | Analyzing chunk frames = [2943488,2944000) | 20260506 16:54:03.318 | INFO | MemInfo: { | 20260506 16:54:03.318 | INFO | "gpu" : " 17 GB", | 20260506 16:54:03.318 | INFO | "huge" : "", | 20260506 16:54:03.318 | INFO | "huge_shared" : " 27 GB", | 20260506 16:54:03.318 | INFO | "malloc" : " 14 GB", | 20260506 16:54:03.318 | INFO | "pinned" : " 29 GB" | 20260506 16:54:03.318 | INFO | } | 20260506 16:54:03.318 | INFO | Perf report for the last 5.13223 seconds: | 20260506 16:54:03.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:54:03.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:03.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:03.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:03.318 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 16:54:03.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:03.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:03.318 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:54:03.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:03.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:03.318 | INFO | Uploaded 36GiB to GPU in 2.02524s (17.7757GiB/s) | 20260506 16:54:03.318 | INFO | Downloaded 34GiB to GPU in 1.93181s (17.6001GiB/s) | 20260506 16:54:08.459 | INFO | Analyzing chunk frames = [2944000,2944512) | 20260506 16:54:08.459 | INFO | MemInfo: { | 20260506 16:54:08.459 | INFO | "gpu" : " 17 GB", | 20260506 16:54:08.459 | INFO | "huge" : "", | 20260506 16:54:08.459 | INFO | "huge_shared" : " 27 GB", | 20260506 16:54:08.459 | INFO | "malloc" : " 14 GB", | 20260506 16:54:08.459 | INFO | "pinned" : " 29 GB" | 20260506 16:54:08.459 | INFO | } | 20260506 16:54:08.459 | INFO | Perf report for the last 5.14138 seconds: | 20260506 16:54:08.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:54:08.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:08.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:08.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:08.459 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 16:54:08.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:08.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:08.459 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 16:54:08.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:08.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:08.459 | INFO | Uploaded 36GiB to GPU in 2.02292s (17.796GiB/s) | 20260506 16:54:08.459 | INFO | Downloaded 34GiB to GPU in 2.01964s (16.8347GiB/s) | 20260506 16:54:13.601 | INFO | Analyzing chunk frames = [2944512,2945024) | 20260506 16:54:13.601 | INFO | MemInfo: { | 20260506 16:54:13.601 | INFO | "gpu" : " 17 GB", | 20260506 16:54:13.601 | INFO | "huge" : "", | 20260506 16:54:13.601 | INFO | "huge_shared" : " 27 GB", | 20260506 16:54:13.601 | INFO | "malloc" : " 14 GB", | 20260506 16:54:13.601 | INFO | "pinned" : " 29 GB" | 20260506 16:54:13.601 | INFO | } | 20260506 16:54:13.601 | INFO | Perf report for the last 5.14193 seconds: | 20260506 16:54:13.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:54:13.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:13.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:13.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:13.601 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:54:13.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:13.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:13.601 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:54:13.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:13.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:13.601 | INFO | Uploaded 36GiB to GPU in 1.99367s (18.0572GiB/s) | 20260506 16:54:13.602 | INFO | Downloaded 34GiB to GPU in 1.96233s (17.3264GiB/s) | 20260506 16:54:13.715 | INFO | Starting a round of Baz IO | 20260506 16:54:16.860 | INFO | Populating RTMetrics report | 20260506 16:54:16.860 | INFO | RTMetrics report populated | 20260506 16:54:16.860 | INFO | Opening tmp RTMetrics json file | 20260506 16:54:16.871 | INFO | Serializing RTMetrics to json | 20260506 16:54:16.899 | INFO | Serialization done, swapping tmp file now... | 20260506 16:54:16.909 | INFO | RTMetrics done | 20260506 16:54:16.998 | INFO | Finished a round of Baz IO: Wrote 2.076 GiB in 3.283 seconds | 20260506 16:54:16.998 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.00% / 0.01% / 63.99% | 20260506 16:54:16.998 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.53219% / 1.58946e-05% / 6.35783e-05% | 20260506 16:54:16.998 | INFO | Baz IO overall: | 20260506 16:54:16.998 | INFO | BytesWritten : 1549318047051 | 20260506 16:54:16.998 | INFO | NumEvents : 749643134597 | 20260506 16:54:18.739 | INFO | Analyzing chunk frames = [2945024,2945536) | 20260506 16:54:18.739 | INFO | MemInfo: { | 20260506 16:54:18.739 | INFO | "gpu" : " 17 GB", | 20260506 16:54:18.739 | INFO | "huge" : "", | 20260506 16:54:18.739 | INFO | "huge_shared" : " 27 GB", | 20260506 16:54:18.739 | INFO | "malloc" : " 14 GB", | 20260506 16:54:18.739 | INFO | "pinned" : " 29 GB" | 20260506 16:54:18.739 | INFO | } | 20260506 16:54:18.740 | INFO | Perf report for the last 5.13861 seconds: | 20260506 16:54:18.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:54:18.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:18.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:18.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:18.740 | INFO | ANALYSIS : 62.8%, 2.9, 95.7ms, 0 | 20260506 16:54:18.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:18.740 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 16:54:18.740 | INFO | PRE_HQ : 60.5%, 1.9, 58.4ms, 0 | 20260506 16:54:18.740 | INFO | BAZWRITER : 63.5%, 4.8, 156.2ms, 0 | 20260506 16:54:18.740 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 16:54:18.740 | INFO | Uploaded 36GiB to GPU in 2.04934s (17.5666GiB/s) | 20260506 16:54:18.740 | INFO | Downloaded 38GiB to GPU in 2.21074s (17.1888GiB/s) | 20260506 16:54:23.878 | INFO | Analyzing chunk frames = [2945536,2946048) | 20260506 16:54:23.878 | INFO | MemInfo: { | 20260506 16:54:23.878 | INFO | "gpu" : " 17 GB", | 20260506 16:54:23.878 | INFO | "huge" : "", | 20260506 16:54:23.878 | INFO | "huge_shared" : " 27 GB", | 20260506 16:54:23.878 | INFO | "malloc" : " 14 GB", | 20260506 16:54:23.878 | INFO | "pinned" : " 29 GB" | 20260506 16:54:23.878 | INFO | } | 20260506 16:54:23.878 | INFO | Perf report for the last 5.13883 seconds: | 20260506 16:54:23.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:54:23.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:23.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:23.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:23.878 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:54:23.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:23.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:23.879 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 16:54:23.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:23.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:23.879 | INFO | Uploaded 36GiB to GPU in 2.02142s (17.8092GiB/s) | 20260506 16:54:23.879 | INFO | Downloaded 34GiB to GPU in 1.97533s (17.2123GiB/s) | 20260506 16:54:29.019 | INFO | Analyzing chunk frames = [2946048,2946560) | 20260506 16:54:29.019 | INFO | MemInfo: { | 20260506 16:54:29.019 | INFO | "gpu" : " 17 GB", | 20260506 16:54:29.019 | INFO | "huge" : "", | 20260506 16:54:29.019 | INFO | "huge_shared" : " 27 GB", | 20260506 16:54:29.019 | INFO | "malloc" : " 14 GB", | 20260506 16:54:29.019 | INFO | "pinned" : " 29 GB" | 20260506 16:54:29.019 | INFO | } | 20260506 16:54:29.019 | INFO | Perf report for the last 5.141 seconds: | 20260506 16:54:29.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:54:29.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:29.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:29.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:29.019 | INFO | ANALYSIS : 61.3%, 3.0, 94.9ms, 0 | 20260506 16:54:29.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:29.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:29.019 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:54:29.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:29.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:29.020 | INFO | Uploaded 36GiB to GPU in 2.00963s (17.9138GiB/s) | 20260506 16:54:29.020 | INFO | Downloaded 34GiB to GPU in 1.93478s (17.573GiB/s) | 20260506 16:54:34.162 | INFO | Analyzing chunk frames = [2946560,2947072) | 20260506 16:54:34.162 | INFO | MemInfo: { | 20260506 16:54:34.162 | INFO | "gpu" : " 17 GB", | 20260506 16:54:34.162 | INFO | "huge" : "", | 20260506 16:54:34.162 | INFO | "huge_shared" : " 27 GB", | 20260506 16:54:34.162 | INFO | "malloc" : " 14 GB", | 20260506 16:54:34.162 | INFO | "pinned" : " 29 GB" | 20260506 16:54:34.162 | INFO | } | 20260506 16:54:34.162 | INFO | Perf report for the last 5.14238 seconds: | 20260506 16:54:34.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:54:34.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:34.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:34.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:34.162 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 16:54:34.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:34.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:34.162 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:54:34.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:34.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:34.162 | INFO | Uploaded 36GiB to GPU in 1.96725s (18.2996GiB/s) | 20260506 16:54:34.162 | INFO | Downloaded 34GiB to GPU in 1.88036s (18.0817GiB/s) | 20260506 16:54:39.300 | INFO | Analyzing chunk frames = [2947072,2947584) | 20260506 16:54:39.300 | INFO | MemInfo: { | 20260506 16:54:39.300 | INFO | "gpu" : " 17 GB", | 20260506 16:54:39.300 | INFO | "huge" : "", | 20260506 16:54:39.300 | INFO | "huge_shared" : " 27 GB", | 20260506 16:54:39.300 | INFO | "malloc" : " 14 GB", | 20260506 16:54:39.300 | INFO | "pinned" : " 29 GB" | 20260506 16:54:39.300 | INFO | } | 20260506 16:54:39.300 | INFO | Perf report for the last 5.13839 seconds: | 20260506 16:54:39.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:54:39.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:39.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:39.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:39.300 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 16:54:39.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:39.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:39.300 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:54:39.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:39.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:39.300 | INFO | Uploaded 36GiB to GPU in 2.00766s (17.9314GiB/s) | 20260506 16:54:39.301 | INFO | Downloaded 34GiB to GPU in 1.91465s (17.7578GiB/s) | 20260506 16:54:44.436 | INFO | Analyzing chunk frames = [2947584,2948096) | 20260506 16:54:44.436 | INFO | MemInfo: { | 20260506 16:54:44.436 | INFO | "gpu" : " 17 GB", | 20260506 16:54:44.436 | INFO | "huge" : "", | 20260506 16:54:44.436 | INFO | "huge_shared" : " 27 GB", | 20260506 16:54:44.436 | INFO | "malloc" : " 14 GB", | 20260506 16:54:44.436 | INFO | "pinned" : " 29 GB" | 20260506 16:54:44.436 | INFO | } | 20260506 16:54:44.436 | INFO | Perf report for the last 5.13586 seconds: | 20260506 16:54:44.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:54:44.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:44.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:44.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:44.436 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 16:54:44.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:44.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:44.436 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 16:54:44.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:44.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:44.436 | INFO | Uploaded 36GiB to GPU in 2.07271s (17.3685GiB/s) | 20260506 16:54:44.436 | INFO | Downloaded 34GiB to GPU in 1.96097s (17.3383GiB/s) | 20260506 16:54:49.577 | INFO | Analyzing chunk frames = [2948096,2948608) | 20260506 16:54:49.577 | INFO | MemInfo: { | 20260506 16:54:49.577 | INFO | "gpu" : " 17 GB", | 20260506 16:54:49.577 | INFO | "huge" : "", | 20260506 16:54:49.577 | INFO | "huge_shared" : " 27 GB", | 20260506 16:54:49.577 | INFO | "malloc" : " 14 GB", | 20260506 16:54:49.577 | INFO | "pinned" : " 29 GB" | 20260506 16:54:49.577 | INFO | } | 20260506 16:54:49.577 | INFO | Perf report for the last 5.14085 seconds: | 20260506 16:54:49.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:54:49.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:49.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:49.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:49.577 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:54:49.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:49.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:49.577 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:54:49.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:49.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:49.577 | INFO | Uploaded 36GiB to GPU in 2.00864s (17.9226GiB/s) | 20260506 16:54:49.577 | INFO | Downloaded 34GiB to GPU in 1.97103s (17.2499GiB/s) | 20260506 16:54:54.714 | INFO | Analyzing chunk frames = [2948608,2949120) | 20260506 16:54:54.714 | INFO | MemInfo: { | 20260506 16:54:54.714 | INFO | "gpu" : " 17 GB", | 20260506 16:54:54.714 | INFO | "huge" : "", | 20260506 16:54:54.714 | INFO | "huge_shared" : " 27 GB", | 20260506 16:54:54.714 | INFO | "malloc" : " 14 GB", | 20260506 16:54:54.714 | INFO | "pinned" : " 29 GB" | 20260506 16:54:54.714 | INFO | } | 20260506 16:54:54.714 | INFO | Perf report for the last 5.13742 seconds: | 20260506 16:54:54.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:54:54.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:54.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:54.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:54.714 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:54:54.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:54.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:54.714 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 16:54:54.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:54.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:54.715 | INFO | Uploaded 36GiB to GPU in 1.99668s (18.0299GiB/s) | 20260506 16:54:54.715 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:54:54.715 | INFO | Total mem usage: 18 GiB | 20260506 16:54:54.715 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:54:54.715 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.1%) | 20260506 16:54:54.715 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:54:54.715 | INFO | High Water Breakdown: | 20260506 16:54:54.715 | INFO | 61.2% : 9.49 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:54:54.715 | INFO | 38.7% : 5.99 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:54:54.715 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:54:54.715 | INFO | ------------------------------------------ | 20260506 16:54:54.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:54:54.715 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:54:54.715 | INFO | Total mem usage: 26.5 GiB | 20260506 16:54:54.715 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:54:54.715 | INFO | Downloaded 34GiB to GPU in 1.97644s (17.2027GiB/s) | 20260506 16:54:54.715 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:54:54.715 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:54:54.715 | INFO | High Water Breakdown: | 20260506 16:54:54.715 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:54:54.715 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:54:54.715 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:54:54.715 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:54:54.715 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:54:54.715 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:54:54.715 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:54:54.715 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:54:54.715 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:54:54.715 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:54:54.715 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:54:54.715 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:54:54.715 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:54:54.715 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:54:54.715 | INFO | ------------------------------------------ | 20260506 16:54:54.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:54:54.715 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:54:54.715 | INFO | Total mem usage: 25.6 GiB | 20260506 16:54:54.715 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:54:54.715 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:54:54.715 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:54:54.715 | INFO | High Water Breakdown: | 20260506 16:54:54.715 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:54:54.715 | INFO | ------------------------------------------ | 20260506 16:54:54.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:54:54.715 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:54:54.715 | INFO | Total mem usage: 15.7 GiB | 20260506 16:54:54.715 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:54:54.715 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:54:54.715 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:54:54.715 | INFO | High Water Breakdown: | 20260506 16:54:54.715 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:54:54.715 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:54:54.715 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:54:54.715 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:54:54.715 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:54:54.715 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:54:54.715 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:54:54.715 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:54:54.715 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:54:54.715 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:54:54.715 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:54:54.715 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:54:54.716 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:54:54.716 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:54:54.716 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:54:54.716 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:54:54.716 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:54:54.716 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:54:54.716 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:54:54.716 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:54:54.716 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:54:54.716 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:54:54.716 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:54:54.716 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:54:54.716 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:54:54.716 | INFO | ------------------------------------------ | 20260506 16:54:54.716 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:54:54.716 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:54:54.716 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:54:54.716 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:54:54.716 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.57e+01, 1.36e+01 / 2.15e+00 | 20260506 16:54:54.716 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 9.94e-02, 1.01e-02 / 5.19e-03 | 20260506 16:54:54.716 | INFO | PerfReport: 11.60%, Baseline -- 3.63e+00 / 4.45e+00, 5.55e-01 / 2.80e-01 | 20260506 16:54:54.716 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.83e-01 / 1.17e-01 | 20260506 16:54:54.716 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.47e+01, 3.95e+00 / 2.30e+00 | 20260506 16:54:54.716 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.35e+00 / 1.37e+00 | 20260506 16:54:54.716 | INFO | PerfReport: 7.34%, PulseAccumulating -- 2.30e+00 / 2.85e+00, 3.95e-01 / 2.29e-01 | 20260506 16:54:54.716 | INFO | PerfReport: 3.48%, Metrics -- 1.09e+00 / 1.45e+00, 2.78e-01 / 1.80e-01 | 20260506 16:54:54.716 | INFO | PerfReport: 1.21%, PulseSerializing -- 3.78e-01 / 4.32e-01, 5.92e-02 / 2.74e-02 | 20260506 16:54:54.716 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:54:54.716 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:54:54.716 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:54:54.716 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 16:54:54.716 | INFO | PerfReport: 21.51%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260506 16:54:54.716 | INFO | PerfReport: 21.15%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 16:54:54.716 | INFO | PerfReport: 32.18%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 16:54:54.716 | INFO | PerfReport: 1.51%, LockUpload -- 1.47e+00 / 0.00e+00, 5.78e+00 / 0.00e+00 | 20260506 16:54:54.716 | INFO | PerfReport: 1.06%, LockDownload -- 1.03e+00 / 0.00e+00, 2.63e+00 / 0.00e+00 | 20260506 16:54:54.716 | INFO | PerfReport: 22.57%, LockCompute -- 2.20e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 16:54:54.716 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.65e-02 / 0.00e+00 | 20260506 16:54:54.835 | INFO | Starting a round of Baz IO | 20260506 16:54:57.976 | INFO | Populating RTMetrics report | 20260506 16:54:57.977 | INFO | RTMetrics report populated | 20260506 16:54:57.978 | INFO | Opening tmp RTMetrics json file | 20260506 16:54:57.979 | INFO | Serializing RTMetrics to json | 20260506 16:54:57.980 | INFO | Serialization done, swapping tmp file now... | 20260506 16:54:57.981 | INFO | RTMetrics done | 20260506 16:54:58.091 | INFO | Finished a round of Baz IO: Wrote 2.076 GiB in 3.256 seconds | 20260506 16:54:58.091 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 36.00% / 0.01% / 64.00% | 20260506 16:54:58.091 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.53025% / 1.58946e-05% / 4.37101e-05% | 20260506 16:54:58.091 | INFO | Baz IO overall: | 20260506 16:54:58.091 | INFO | BytesWritten : 1551547318042 | 20260506 16:54:58.091 | INFO | NumEvents : 750719796568 | 20260506 16:54:59.856 | INFO | Analyzing chunk frames = [2949120,2949632) | 20260506 16:54:59.856 | INFO | MemInfo: { | 20260506 16:54:59.856 | INFO | "gpu" : " 17 GB", | 20260506 16:54:59.856 | INFO | "huge" : "", | 20260506 16:54:59.856 | INFO | "huge_shared" : " 27 GB", | 20260506 16:54:59.856 | INFO | "malloc" : " 14 GB", | 20260506 16:54:59.856 | INFO | "pinned" : " 29 GB" | 20260506 16:54:59.856 | INFO | } | 20260506 16:54:59.856 | INFO | Perf report for the last 5.14148 seconds: | 20260506 16:54:59.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:54:59.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:59.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:59.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:59.856 | INFO | ANALYSIS : 62.9%, 2.9, 95.4ms, 0 | 20260506 16:54:59.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:54:59.856 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260506 16:54:59.856 | INFO | PRE_HQ : 60.5%, 1.9, 58.4ms, 0 | 20260506 16:54:59.856 | INFO | BAZWRITER : 63.0%, 4.5, 145.1ms, 0 | 20260506 16:54:59.856 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 16:54:59.856 | INFO | Uploaded 36GiB to GPU in 2.06002s (17.4756GiB/s) | 20260506 16:54:59.856 | INFO | Downloaded 38GiB to GPU in 2.21439s (17.1605GiB/s) | 20260506 16:55:04.999 | INFO | Analyzing chunk frames = [2949632,2950144) | 20260506 16:55:04.999 | INFO | MemInfo: { | 20260506 16:55:04.999 | INFO | "gpu" : " 17 GB", | 20260506 16:55:04.999 | INFO | "huge" : "", | 20260506 16:55:04.999 | INFO | "huge_shared" : " 27 GB", | 20260506 16:55:04.999 | INFO | "malloc" : " 14 GB", | 20260506 16:55:04.999 | INFO | "pinned" : " 29 GB" | 20260506 16:55:04.999 | INFO | } | 20260506 16:55:04.999 | INFO | Perf report for the last 5.14286 seconds: | 20260506 16:55:04.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:55:04.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:04.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:04.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:04.999 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 16:55:04.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:04.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:04.999 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:55:04.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:04.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:04.999 | INFO | Uploaded 36GiB to GPU in 1.98985s (18.0918GiB/s) | 20260506 16:55:04.999 | INFO | Downloaded 34GiB to GPU in 1.92998s (17.6167GiB/s) | 20260506 16:55:10.133 | INFO | Analyzing chunk frames = [2950144,2950656) | 20260506 16:55:10.133 | INFO | MemInfo: { | 20260506 16:55:10.133 | INFO | "gpu" : " 17 GB", | 20260506 16:55:10.133 | INFO | "huge" : "", | 20260506 16:55:10.133 | INFO | "huge_shared" : " 27 GB", | 20260506 16:55:10.133 | INFO | "malloc" : " 14 GB", | 20260506 16:55:10.133 | INFO | "pinned" : " 29 GB" | 20260506 16:55:10.133 | INFO | } | 20260506 16:55:10.133 | INFO | Perf report for the last 5.1346 seconds: | 20260506 16:55:10.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:55:10.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:10.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:10.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:10.133 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 16:55:10.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:10.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:10.133 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:55:10.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:10.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:10.134 | INFO | Uploaded 36GiB to GPU in 1.98491s (18.1368GiB/s) | 20260506 16:55:10.134 | INFO | Downloaded 34GiB to GPU in 1.92496s (17.6627GiB/s) | 20260506 16:55:15.273 | INFO | Analyzing chunk frames = [2950656,2951168) | 20260506 16:55:15.273 | INFO | MemInfo: { | 20260506 16:55:15.273 | INFO | "gpu" : " 17 GB", | 20260506 16:55:15.273 | INFO | "huge" : "", | 20260506 16:55:15.273 | INFO | "huge_shared" : " 27 GB", | 20260506 16:55:15.273 | INFO | "malloc" : " 14 GB", | 20260506 16:55:15.273 | INFO | "pinned" : " 29 GB" | 20260506 16:55:15.273 | INFO | } | 20260506 16:55:15.273 | INFO | Perf report for the last 5.1401 seconds: | 20260506 16:55:15.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:55:15.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:15.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:15.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:15.274 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 16:55:15.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:15.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:15.274 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:55:15.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:15.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:15.274 | INFO | Uploaded 36GiB to GPU in 1.94934s (18.4678GiB/s) | 20260506 16:55:15.274 | INFO | Downloaded 34GiB to GPU in 1.87793s (18.105GiB/s) | 20260506 16:55:20.414 | INFO | Analyzing chunk frames = [2951168,2951680) | 20260506 16:55:20.414 | INFO | MemInfo: { | 20260506 16:55:20.414 | INFO | "gpu" : " 17 GB", | 20260506 16:55:20.414 | INFO | "huge" : "", | 20260506 16:55:20.414 | INFO | "huge_shared" : " 27 GB", | 20260506 16:55:20.414 | INFO | "malloc" : " 14 GB", | 20260506 16:55:20.414 | INFO | "pinned" : " 29 GB" | 20260506 16:55:20.414 | INFO | } | 20260506 16:55:20.414 | INFO | Perf report for the last 5.14089 seconds: | 20260506 16:55:20.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:55:20.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:20.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:20.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:20.414 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 16:55:20.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:20.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:20.414 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 16:55:20.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:20.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:20.415 | INFO | Uploaded 36GiB to GPU in 2.01794s (17.84GiB/s) | 20260506 16:55:20.415 | INFO | Downloaded 34GiB to GPU in 1.90472s (17.8504GiB/s) | 20260506 16:55:25.551 | INFO | Analyzing chunk frames = [2951680,2952192) | 20260506 16:55:25.551 | INFO | MemInfo: { | 20260506 16:55:25.551 | INFO | "gpu" : " 17 GB", | 20260506 16:55:25.551 | INFO | "huge" : "", | 20260506 16:55:25.551 | INFO | "huge_shared" : " 27 GB", | 20260506 16:55:25.551 | INFO | "malloc" : " 14 GB", | 20260506 16:55:25.551 | INFO | "pinned" : " 29 GB" | 20260506 16:55:25.551 | INFO | } | 20260506 16:55:25.551 | INFO | Perf report for the last 5.13631 seconds: | 20260506 16:55:25.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:55:25.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:25.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:25.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:25.551 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 16:55:25.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:25.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:25.551 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 16:55:25.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:25.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:25.551 | INFO | Uploaded 36GiB to GPU in 2.01498s (17.8661GiB/s) | 20260506 16:55:25.551 | INFO | Downloaded 34GiB to GPU in 1.94458s (17.4845GiB/s) | 20260506 16:55:30.686 | INFO | Analyzing chunk frames = [2952192,2952704) | 20260506 16:55:30.686 | INFO | MemInfo: { | 20260506 16:55:30.686 | INFO | "gpu" : " 17 GB", | 20260506 16:55:30.686 | INFO | "huge" : "", | 20260506 16:55:30.686 | INFO | "huge_shared" : " 27 GB", | 20260506 16:55:30.686 | INFO | "malloc" : " 14 GB", | 20260506 16:55:30.686 | INFO | "pinned" : " 29 GB" | 20260506 16:55:30.686 | INFO | } | 20260506 16:55:30.686 | INFO | Perf report for the last 5.13564 seconds: | 20260506 16:55:30.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:55:30.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:30.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:30.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:30.686 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 16:55:30.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:30.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:30.686 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:55:30.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:30.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:30.687 | INFO | Uploaded 36GiB to GPU in 2.02554s (17.773GiB/s) | 20260506 16:55:30.687 | INFO | Downloaded 34GiB to GPU in 2.02231s (16.8124GiB/s) | 20260506 16:55:35.826 | INFO | Analyzing chunk frames = [2952704,2953216) | 20260506 16:55:35.826 | INFO | MemInfo: { | 20260506 16:55:35.826 | INFO | "gpu" : " 17 GB", | 20260506 16:55:35.826 | INFO | "huge" : "", | 20260506 16:55:35.826 | INFO | "huge_shared" : " 27 GB", | 20260506 16:55:35.826 | INFO | "malloc" : " 14 GB", | 20260506 16:55:35.826 | INFO | "pinned" : " 29 GB" | 20260506 16:55:35.826 | INFO | } | 20260506 16:55:35.826 | INFO | Perf report for the last 5.14004 seconds: | 20260506 16:55:35.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:55:35.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:35.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:35.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:35.826 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:55:35.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:35.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:35.826 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:55:35.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:35.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:35.827 | INFO | Uploaded 36GiB to GPU in 2.00551s (17.9505GiB/s) | 20260506 16:55:35.827 | INFO | Downloaded 34GiB to GPU in 1.96423s (17.3095GiB/s) | 20260506 16:55:35.947 | INFO | Starting a round of Baz IO | 20260506 16:55:39.082 | INFO | Populating RTMetrics report | 20260506 16:55:39.083 | INFO | RTMetrics report populated | 20260506 16:55:39.083 | INFO | Opening tmp RTMetrics json file | 20260506 16:55:39.086 | INFO | Serializing RTMetrics to json | 20260506 16:55:39.088 | INFO | Serialization done, swapping tmp file now... | 20260506 16:55:39.089 | INFO | RTMetrics done | 20260506 16:55:39.203 | INFO | Finished a round of Baz IO: Wrote 2.078 GiB in 3.256 seconds | 20260506 16:55:39.203 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.99% / 0.01% / 64.00% | 20260506 16:55:39.203 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.53056% / 1.19209e-05% / 7.54992e-05% | 20260506 16:55:39.203 | INFO | Baz IO overall: | 20260506 16:55:39.203 | INFO | BytesWritten : 1553778060558 | 20260506 16:55:39.203 | INFO | NumEvents : 751797671063 | 20260506 16:55:40.968 | INFO | Analyzing chunk frames = [2953216,2953728) | 20260506 16:55:40.968 | INFO | MemInfo: { | 20260506 16:55:40.968 | INFO | "gpu" : " 17 GB", | 20260506 16:55:40.968 | INFO | "huge" : "", | 20260506 16:55:40.968 | INFO | "huge_shared" : " 27 GB", | 20260506 16:55:40.968 | INFO | "malloc" : " 14 GB", | 20260506 16:55:40.968 | INFO | "pinned" : " 29 GB" | 20260506 16:55:40.968 | INFO | } | 20260506 16:55:40.968 | INFO | Perf report for the last 5.14174 seconds: | 20260506 16:55:40.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:55:40.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:40.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:40.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:40.968 | INFO | ANALYSIS : 62.7%, 2.9, 95.5ms, 0 | 20260506 16:55:40.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:40.968 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 16:55:40.968 | INFO | PRE_HQ : 60.5%, 1.9, 58.4ms, 0 | 20260506 16:55:40.968 | INFO | BAZWRITER : 62.9%, 4.8, 154.0ms, 0 | 20260506 16:55:40.968 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 16:55:40.968 | INFO | Uploaded 36GiB to GPU in 2.06127s (17.465GiB/s) | 20260506 16:55:40.969 | INFO | Downloaded 38GiB to GPU in 2.21287s (17.1723GiB/s) | 20260506 16:55:46.110 | INFO | Analyzing chunk frames = [2953728,2954240) | 20260506 16:55:46.110 | INFO | MemInfo: { | 20260506 16:55:46.110 | INFO | "gpu" : " 17 GB", | 20260506 16:55:46.110 | INFO | "huge" : "", | 20260506 16:55:46.110 | INFO | "huge_shared" : " 27 GB", | 20260506 16:55:46.110 | INFO | "malloc" : " 14 GB", | 20260506 16:55:46.110 | INFO | "pinned" : " 29 GB" | 20260506 16:55:46.110 | INFO | } | 20260506 16:55:46.110 | INFO | Perf report for the last 5.14192 seconds: | 20260506 16:55:46.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:55:46.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:46.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:46.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:46.110 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 16:55:46.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:46.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:46.110 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:55:46.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:46.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:46.110 | INFO | Uploaded 36GiB to GPU in 2.0035s (17.9686GiB/s) | 20260506 16:55:46.110 | INFO | Downloaded 34GiB to GPU in 1.93733s (17.5499GiB/s) | 20260506 16:55:51.259 | INFO | Analyzing chunk frames = [2954240,2954752) | 20260506 16:55:51.259 | INFO | MemInfo: { | 20260506 16:55:51.259 | INFO | "gpu" : " 17 GB", | 20260506 16:55:51.259 | INFO | "huge" : "", | 20260506 16:55:51.259 | INFO | "huge_shared" : " 27 GB", | 20260506 16:55:51.259 | INFO | "malloc" : " 14 GB", | 20260506 16:55:51.259 | INFO | "pinned" : " 29 GB" | 20260506 16:55:51.259 | INFO | } | 20260506 16:55:51.259 | INFO | Perf report for the last 5.1494 seconds: | 20260506 16:55:51.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:55:51.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:51.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:51.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:51.260 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 16:55:51.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:51.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:51.260 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:55:51.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:51.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:51.260 | INFO | Uploaded 36GiB to GPU in 1.96821s (18.2907GiB/s) | 20260506 16:55:51.260 | INFO | Downloaded 34GiB to GPU in 1.92964s (17.6199GiB/s) | 20260506 16:55:56.391 | INFO | Analyzing chunk frames = [2954752,2955264) | 20260506 16:55:56.391 | INFO | MemInfo: { | 20260506 16:55:56.391 | INFO | "gpu" : " 17 GB", | 20260506 16:55:56.391 | INFO | "huge" : "", | 20260506 16:55:56.391 | INFO | "huge_shared" : " 27 GB", | 20260506 16:55:56.391 | INFO | "malloc" : " 14 GB", | 20260506 16:55:56.391 | INFO | "pinned" : " 29 GB" | 20260506 16:55:56.391 | INFO | } | 20260506 16:55:56.391 | INFO | Perf report for the last 5.1312 seconds: | 20260506 16:55:56.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:55:56.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:56.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:56.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:56.391 | INFO | ANALYSIS : 60.2%, 3.0, 93.0ms, 0 | 20260506 16:55:56.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:56.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:56.391 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 16:55:56.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:56.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:55:56.391 | INFO | Uploaded 36GiB to GPU in 1.97161s (18.2591GiB/s) | 20260506 16:55:56.391 | INFO | Downloaded 34GiB to GPU in 1.87158s (18.1665GiB/s) | 20260506 16:56:01.528 | INFO | Analyzing chunk frames = [2955264,2955776) | 20260506 16:56:01.528 | INFO | MemInfo: { | 20260506 16:56:01.528 | INFO | "gpu" : " 17 GB", | 20260506 16:56:01.528 | INFO | "huge" : "", | 20260506 16:56:01.528 | INFO | "huge_shared" : " 27 GB", | 20260506 16:56:01.528 | INFO | "malloc" : " 14 GB", | 20260506 16:56:01.528 | INFO | "pinned" : " 29 GB" | 20260506 16:56:01.528 | INFO | } | 20260506 16:56:01.528 | INFO | Perf report for the last 5.13769 seconds: | 20260506 16:56:01.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:56:01.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:01.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:01.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:01.529 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 16:56:01.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:01.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:01.529 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:56:01.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:01.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:01.529 | INFO | Uploaded 36GiB to GPU in 2.01102s (17.9014GiB/s) | 20260506 16:56:01.529 | INFO | Downloaded 34GiB to GPU in 1.91368s (17.7668GiB/s) | 20260506 16:56:06.665 | INFO | Analyzing chunk frames = [2955776,2956288) | 20260506 16:56:06.665 | INFO | MemInfo: { | 20260506 16:56:06.665 | INFO | "gpu" : " 17 GB", | 20260506 16:56:06.665 | INFO | "huge" : "", | 20260506 16:56:06.665 | INFO | "huge_shared" : " 27 GB", | 20260506 16:56:06.665 | INFO | "malloc" : " 14 GB", | 20260506 16:56:06.665 | INFO | "pinned" : " 29 GB" | 20260506 16:56:06.665 | INFO | } | 20260506 16:56:06.665 | INFO | Perf report for the last 5.13685 seconds: | 20260506 16:56:06.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:56:06.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:06.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:06.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:06.665 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 16:56:06.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:06.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:06.665 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:56:06.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:06.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:06.665 | INFO | Uploaded 36GiB to GPU in 2.04587s (17.5964GiB/s) | 20260506 16:56:06.665 | INFO | Downloaded 34GiB to GPU in 1.9436s (17.4933GiB/s) | 20260506 16:56:11.800 | INFO | Analyzing chunk frames = [2956288,2956800) | 20260506 16:56:11.800 | INFO | MemInfo: { | 20260506 16:56:11.800 | INFO | "gpu" : " 17 GB", | 20260506 16:56:11.800 | INFO | "huge" : "", | 20260506 16:56:11.800 | INFO | "huge_shared" : " 27 GB", | 20260506 16:56:11.800 | INFO | "malloc" : " 14 GB", | 20260506 16:56:11.800 | INFO | "pinned" : " 29 GB" | 20260506 16:56:11.800 | INFO | } | 20260506 16:56:11.800 | INFO | Perf report for the last 5.13493 seconds: | 20260506 16:56:11.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:56:11.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:11.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:11.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:11.800 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 16:56:11.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:11.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:11.800 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:56:11.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:11.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:11.801 | INFO | Uploaded 36GiB to GPU in 2.01673s (17.8506GiB/s) | 20260506 16:56:11.801 | INFO | Downloaded 34GiB to GPU in 2.023s (16.8067GiB/s) | 20260506 16:56:16.945 | INFO | Analyzing chunk frames = [2956800,2957312) | 20260506 16:56:16.945 | INFO | MemInfo: { | 20260506 16:56:16.945 | INFO | "gpu" : " 17 GB", | 20260506 16:56:16.945 | INFO | "huge" : "", | 20260506 16:56:16.945 | INFO | "huge_shared" : " 27 GB", | 20260506 16:56:16.945 | INFO | "malloc" : " 14 GB", | 20260506 16:56:16.945 | INFO | "pinned" : " 29 GB" | 20260506 16:56:16.945 | INFO | } | 20260506 16:56:16.945 | INFO | Perf report for the last 5.14476 seconds: | 20260506 16:56:16.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:56:16.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:16.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:16.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:16.945 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 16:56:16.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:16.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:16.945 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:56:16.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:16.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:16.945 | INFO | Uploaded 36GiB to GPU in 2.02486s (17.779GiB/s) | 20260506 16:56:16.945 | INFO | Downloaded 34GiB to GPU in 1.96642s (17.2903GiB/s) | 20260506 16:56:16.945 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:56:16.945 | INFO | Total mem usage: 18 GiB | 20260506 16:56:16.945 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:56:16.945 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -15%) | 20260506 16:56:16.945 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:56:16.945 | INFO | High Water Breakdown: | 20260506 16:56:16.945 | INFO | 61.4% : 9.62 GiB ( 84.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:56:16.946 | INFO | 38.4% : 6.02 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:56:16.946 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:56:16.946 | INFO | ------------------------------------------ | 20260506 16:56:16.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:56:16.946 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:56:16.946 | INFO | Total mem usage: 26.5 GiB | 20260506 16:56:16.946 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:56:16.946 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:56:16.946 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:56:16.946 | INFO | High Water Breakdown: | 20260506 16:56:16.946 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:56:16.946 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:56:16.946 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:56:16.946 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:56:16.946 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:56:16.946 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:56:16.946 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:56:16.946 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:56:16.946 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:56:16.946 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:56:16.946 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:56:16.946 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:56:16.946 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:56:16.946 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:56:16.946 | INFO | ------------------------------------------ | 20260506 16:56:16.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:56:16.946 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:56:16.946 | INFO | Total mem usage: 25.6 GiB | 20260506 16:56:16.946 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:56:16.946 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:56:16.946 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:56:16.946 | INFO | High Water Breakdown: | 20260506 16:56:16.946 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:56:16.946 | INFO | ------------------------------------------ | 20260506 16:56:16.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:56:16.946 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:56:16.946 | INFO | Total mem usage: 15.7 GiB | 20260506 16:56:16.946 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:56:16.946 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:56:16.946 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:56:16.946 | INFO | High Water Breakdown: | 20260506 16:56:16.946 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:56:16.946 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:56:16.946 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:56:16.946 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:56:16.946 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:56:16.946 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:56:16.946 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:56:16.946 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:56:16.946 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:56:16.946 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:56:16.946 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:56:16.946 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:56:16.946 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:56:16.946 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:56:16.946 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:56:16.946 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:56:16.946 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:56:16.946 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:56:16.946 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:56:16.946 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:56:16.946 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:56:16.946 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:56:16.946 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:56:16.946 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:56:16.946 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:56:16.946 | INFO | ------------------------------------------ | 20260506 16:56:16.946 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:56:16.947 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:56:16.947 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:56:16.947 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:56:16.947 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.60e+01, 1.35e+01 / 2.80e+00 | 20260506 16:56:16.947 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 9.78e-03 / 7.41e-03 | 20260506 16:56:16.947 | INFO | PerfReport: 11.60%, Baseline -- 3.64e+00 / 4.44e+00, 5.68e-01 / 3.04e-01 | 20260506 16:56:16.947 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.87e-01 / 1.29e-01 | 20260506 16:56:16.947 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.49e+01, 4.14e+00 / 3.01e+00 | 20260506 16:56:16.947 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.41e+00 / 1.96e+00 | 20260506 16:56:16.947 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.28e+00 / 2.79e+00, 3.85e-01 / 2.68e-01 | 20260506 16:56:16.947 | INFO | PerfReport: 3.47%, Metrics -- 1.09e+00 / 1.43e+00, 2.83e-01 / 1.77e-01 | 20260506 16:56:16.947 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.77e-01 / 4.24e-01, 5.92e-02 / 2.40e-02 | 20260506 16:56:16.947 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:56:16.947 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:56:16.947 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:56:16.947 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 16:56:16.947 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 16:56:16.947 | INFO | PerfReport: 21.12%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 16:56:16.947 | INFO | PerfReport: 32.23%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 16:56:16.947 | INFO | PerfReport: 1.43%, LockUpload -- 1.39e+00 / 0.00e+00, 5.78e+00 / 0.00e+00 | 20260506 16:56:16.947 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 16:56:16.947 | INFO | PerfReport: 22.60%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 16:56:16.947 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260506 16:56:17.064 | INFO | Starting a round of Baz IO | 20260506 16:56:20.201 | INFO | Populating RTMetrics report | 20260506 16:56:20.201 | INFO | RTMetrics report populated | 20260506 16:56:20.201 | INFO | Opening tmp RTMetrics json file | 20260506 16:56:20.203 | INFO | Serializing RTMetrics to json | 20260506 16:56:20.204 | INFO | Serialization done, swapping tmp file now... | 20260506 16:56:20.204 | INFO | RTMetrics done | 20260506 16:56:20.326 | INFO | Finished a round of Baz IO: Wrote 2.076 GiB in 3.261 seconds | 20260506 16:56:20.326 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.98% / 0.01% / 64.01% | 20260506 16:56:20.326 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.52294% / 0% / 4.76837e-05% | 20260506 16:56:20.326 | INFO | Baz IO overall: | 20260506 16:56:20.326 | INFO | BytesWritten : 1556007505197 | 20260506 16:56:20.326 | INFO | NumEvents : 752874691964 | 20260506 16:56:22.089 | INFO | Analyzing chunk frames = [2957312,2957824) | 20260506 16:56:22.089 | INFO | MemInfo: { | 20260506 16:56:22.089 | INFO | "gpu" : " 17 GB", | 20260506 16:56:22.089 | INFO | "huge" : "", | 20260506 16:56:22.089 | INFO | "huge_shared" : " 27 GB", | 20260506 16:56:22.089 | INFO | "malloc" : " 14 GB", | 20260506 16:56:22.089 | INFO | "pinned" : " 29 GB" | 20260506 16:56:22.089 | INFO | } | 20260506 16:56:22.089 | INFO | Perf report for the last 5.14434 seconds: | 20260506 16:56:22.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:56:22.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:22.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:22.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:22.089 | INFO | ANALYSIS : 62.7%, 2.9, 95.2ms, 0 | 20260506 16:56:22.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:22.090 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260506 16:56:22.090 | INFO | PRE_HQ : 60.2%, 1.9, 58.6ms, 0 | 20260506 16:56:22.090 | INFO | BAZWRITER : 63.1%, 4.7, 153.6ms, 0 | 20260506 16:56:22.090 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 16:56:22.090 | INFO | Uploaded 36GiB to GPU in 2.06235s (17.4558GiB/s) | 20260506 16:56:22.090 | INFO | Downloaded 38GiB to GPU in 2.21107s (17.1862GiB/s) | 20260506 16:56:27.223 | INFO | Analyzing chunk frames = [2957824,2958336) | 20260506 16:56:27.223 | INFO | MemInfo: { | 20260506 16:56:27.223 | INFO | "gpu" : " 17 GB", | 20260506 16:56:27.223 | INFO | "huge" : "", | 20260506 16:56:27.223 | INFO | "huge_shared" : " 27 GB", | 20260506 16:56:27.223 | INFO | "malloc" : " 14 GB", | 20260506 16:56:27.223 | INFO | "pinned" : " 29 GB" | 20260506 16:56:27.223 | INFO | } | 20260506 16:56:27.223 | INFO | Perf report for the last 5.13353 seconds: | 20260506 16:56:27.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:56:27.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:27.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:27.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:27.223 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:56:27.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:27.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:27.223 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 16:56:27.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:27.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:27.223 | INFO | Uploaded 36GiB to GPU in 2.00006s (17.9995GiB/s) | 20260506 16:56:27.223 | INFO | Downloaded 34GiB to GPU in 1.93977s (17.5278GiB/s) | 20260506 16:56:32.365 | INFO | Analyzing chunk frames = [2958336,2958848) | 20260506 16:56:32.365 | INFO | MemInfo: { | 20260506 16:56:32.365 | INFO | "gpu" : " 17 GB", | 20260506 16:56:32.365 | INFO | "huge" : "", | 20260506 16:56:32.365 | INFO | "huge_shared" : " 27 GB", | 20260506 16:56:32.365 | INFO | "malloc" : " 14 GB", | 20260506 16:56:32.365 | INFO | "pinned" : " 29 GB" | 20260506 16:56:32.365 | INFO | } | 20260506 16:56:32.365 | INFO | Perf report for the last 5.14203 seconds: | 20260506 16:56:32.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:56:32.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:32.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:32.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:32.365 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 16:56:32.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:32.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:32.365 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:56:32.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:32.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:32.365 | INFO | Uploaded 36GiB to GPU in 1.97636s (18.2153GiB/s) | 20260506 16:56:32.365 | INFO | Downloaded 34GiB to GPU in 1.93522s (17.5691GiB/s) | 20260506 16:56:37.504 | INFO | Analyzing chunk frames = [2958848,2959360) | 20260506 16:56:37.504 | INFO | MemInfo: { | 20260506 16:56:37.504 | INFO | "gpu" : " 17 GB", | 20260506 16:56:37.504 | INFO | "huge" : "", | 20260506 16:56:37.504 | INFO | "huge_shared" : " 27 GB", | 20260506 16:56:37.504 | INFO | "malloc" : " 14 GB", | 20260506 16:56:37.504 | INFO | "pinned" : " 29 GB" | 20260506 16:56:37.504 | INFO | } | 20260506 16:56:37.504 | INFO | Perf report for the last 5.13943 seconds: | 20260506 16:56:37.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:56:37.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:37.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:37.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:37.504 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:56:37.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:37.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:37.504 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:56:37.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:37.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:37.505 | INFO | Uploaded 36GiB to GPU in 1.93161s (18.6373GiB/s) | 20260506 16:56:37.505 | INFO | Downloaded 34GiB to GPU in 1.87638s (18.12GiB/s) | 20260506 16:56:42.642 | INFO | Analyzing chunk frames = [2959360,2959872) | 20260506 16:56:42.642 | INFO | MemInfo: { | 20260506 16:56:42.642 | INFO | "gpu" : " 17 GB", | 20260506 16:56:42.642 | INFO | "huge" : "", | 20260506 16:56:42.642 | INFO | "huge_shared" : " 27 GB", | 20260506 16:56:42.642 | INFO | "malloc" : " 14 GB", | 20260506 16:56:42.642 | INFO | "pinned" : " 29 GB" | 20260506 16:56:42.642 | INFO | } | 20260506 16:56:42.642 | INFO | Perf report for the last 5.13726 seconds: | 20260506 16:56:42.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:56:42.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:42.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:42.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:42.642 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 16:56:42.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:42.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:42.642 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:56:42.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:42.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:42.642 | INFO | Uploaded 36GiB to GPU in 2.01784s (17.8409GiB/s) | 20260506 16:56:42.642 | INFO | Downloaded 34GiB to GPU in 1.91897s (17.7179GiB/s) | 20260506 16:56:47.780 | INFO | Analyzing chunk frames = [2959872,2960384) | 20260506 16:56:47.780 | INFO | MemInfo: { | 20260506 16:56:47.780 | INFO | "gpu" : " 17 GB", | 20260506 16:56:47.780 | INFO | "huge" : "", | 20260506 16:56:47.780 | INFO | "huge_shared" : " 27 GB", | 20260506 16:56:47.780 | INFO | "malloc" : " 14 GB", | 20260506 16:56:47.780 | INFO | "pinned" : " 29 GB" | 20260506 16:56:47.780 | INFO | } | 20260506 16:56:47.780 | INFO | Perf report for the last 5.13837 seconds: | 20260506 16:56:47.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:56:47.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:47.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:47.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:47.780 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:56:47.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:47.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:47.780 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 16:56:47.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:47.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:47.780 | INFO | Uploaded 36GiB to GPU in 2.07978s (17.3095GiB/s) | 20260506 16:56:47.780 | INFO | Downloaded 34GiB to GPU in 1.96562s (17.2973GiB/s) | 20260506 16:56:52.918 | INFO | Analyzing chunk frames = [2960384,2960896) | 20260506 16:56:52.918 | INFO | MemInfo: { | 20260506 16:56:52.918 | INFO | "gpu" : " 17 GB", | 20260506 16:56:52.918 | INFO | "huge" : "", | 20260506 16:56:52.918 | INFO | "huge_shared" : " 27 GB", | 20260506 16:56:52.918 | INFO | "malloc" : " 14 GB", | 20260506 16:56:52.918 | INFO | "pinned" : " 29 GB" | 20260506 16:56:52.918 | INFO | } | 20260506 16:56:52.918 | INFO | Perf report for the last 5.13813 seconds: | 20260506 16:56:52.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:56:52.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:52.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:52.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:52.918 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 16:56:52.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:52.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:52.918 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 16:56:52.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:52.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:52.919 | INFO | Uploaded 36GiB to GPU in 1.99882s (18.0107GiB/s) | 20260506 16:56:52.919 | INFO | Downloaded 34GiB to GPU in 1.97798s (17.1892GiB/s) | 20260506 16:56:58.061 | INFO | Analyzing chunk frames = [2960896,2961408) | 20260506 16:56:58.061 | INFO | MemInfo: { | 20260506 16:56:58.061 | INFO | "gpu" : " 17 GB", | 20260506 16:56:58.061 | INFO | "huge" : "", | 20260506 16:56:58.062 | INFO | "huge_shared" : " 27 GB", | 20260506 16:56:58.062 | INFO | "malloc" : " 14 GB", | 20260506 16:56:58.062 | INFO | "pinned" : " 29 GB" | 20260506 16:56:58.062 | INFO | } | 20260506 16:56:58.062 | INFO | Perf report for the last 5.14354 seconds: | 20260506 16:56:58.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:56:58.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:58.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:58.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:58.062 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 16:56:58.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:58.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:58.062 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:56:58.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:58.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:56:58.062 | INFO | Uploaded 36GiB to GPU in 1.99511s (18.0441GiB/s) | 20260506 16:56:58.062 | INFO | Downloaded 34GiB to GPU in 1.97414s (17.2227GiB/s) | 20260506 16:56:58.176 | INFO | Starting a round of Baz IO | 20260506 16:57:01.325 | INFO | Populating RTMetrics report | 20260506 16:57:01.325 | INFO | RTMetrics report populated | 20260506 16:57:01.325 | INFO | Opening tmp RTMetrics json file | 20260506 16:57:01.349 | INFO | Serializing RTMetrics to json | 20260506 16:57:01.374 | INFO | Serialization done, swapping tmp file now... | 20260506 16:57:01.377 | INFO | RTMetrics done | 20260506 16:57:01.569 | INFO | Finished a round of Baz IO: Wrote 2.077 GiB in 3.393 seconds | 20260506 16:57:01.569 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.97% / 0.01% / 64.02% | 20260506 16:57:01.569 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.52182% / 7.94729e-06% / 3.97364e-05% | 20260506 16:57:01.569 | INFO | Baz IO overall: | 20260506 16:57:01.569 | INFO | BytesWritten : 1558238067306 | 20260506 16:57:01.569 | INFO | NumEvents : 753952680196 | 20260506 16:57:03.206 | INFO | Analyzing chunk frames = [2961408,2961920) | 20260506 16:57:03.206 | INFO | MemInfo: { | 20260506 16:57:03.206 | INFO | "gpu" : " 17 GB", | 20260506 16:57:03.206 | INFO | "huge" : "", | 20260506 16:57:03.206 | INFO | "huge_shared" : " 27 GB", | 20260506 16:57:03.206 | INFO | "malloc" : " 14 GB", | 20260506 16:57:03.206 | INFO | "pinned" : " 29 GB" | 20260506 16:57:03.206 | INFO | } | 20260506 16:57:03.206 | INFO | Perf report for the last 5.14432 seconds: | 20260506 16:57:03.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:57:03.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:03.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:03.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:03.206 | INFO | ANALYSIS : 62.9%, 2.9, 95.2ms, 0 | 20260506 16:57:03.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:03.206 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260506 16:57:03.206 | INFO | PRE_HQ : 60.3%, 1.9, 58.9ms, 0 | 20260506 16:57:03.206 | INFO | BAZWRITER : 65.6%, 4.8, 161.4ms, 0 | 20260506 16:57:03.206 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 16:57:03.206 | INFO | Uploaded 36GiB to GPU in 2.06653s (17.4205GiB/s) | 20260506 16:57:03.206 | INFO | Downloaded 38GiB to GPU in 2.21504s (17.1554GiB/s) | 20260506 16:57:08.353 | INFO | Analyzing chunk frames = [2961920,2962432) | 20260506 16:57:08.353 | INFO | MemInfo: { | 20260506 16:57:08.353 | INFO | "gpu" : " 17 GB", | 20260506 16:57:08.353 | INFO | "huge" : "", | 20260506 16:57:08.353 | INFO | "huge_shared" : " 27 GB", | 20260506 16:57:08.353 | INFO | "malloc" : " 14 GB", | 20260506 16:57:08.353 | INFO | "pinned" : " 29 GB" | 20260506 16:57:08.353 | INFO | } | 20260506 16:57:08.353 | INFO | Perf report for the last 5.1468 seconds: | 20260506 16:57:08.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:57:08.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:08.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:08.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:08.353 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 16:57:08.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:08.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:08.353 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 16:57:08.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:08.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:08.353 | INFO | Uploaded 36GiB to GPU in 2.02315s (17.794GiB/s) | 20260506 16:57:08.353 | INFO | Downloaded 34GiB to GPU in 1.93674s (17.5552GiB/s) | 20260506 16:57:13.479 | INFO | Analyzing chunk frames = [2962432,2962944) | 20260506 16:57:13.479 | INFO | MemInfo: { | 20260506 16:57:13.479 | INFO | "gpu" : " 17 GB", | 20260506 16:57:13.479 | INFO | "huge" : "", | 20260506 16:57:13.480 | INFO | "huge_shared" : " 27 GB", | 20260506 16:57:13.480 | INFO | "malloc" : " 14 GB", | 20260506 16:57:13.480 | INFO | "pinned" : " 29 GB" | 20260506 16:57:13.480 | INFO | } | 20260506 16:57:13.480 | INFO | Perf report for the last 5.12685 seconds: | 20260506 16:57:13.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:57:13.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:13.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:13.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:13.480 | INFO | ANALYSIS : 61.0%, 3.0, 94.3ms, 0 | 20260506 16:57:13.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:13.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:13.480 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:57:13.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:13.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:13.480 | INFO | Uploaded 36GiB to GPU in 1.95428s (18.4211GiB/s) | 20260506 16:57:13.480 | INFO | Downloaded 34GiB to GPU in 1.91747s (17.7317GiB/s) | 20260506 16:57:18.614 | INFO | Analyzing chunk frames = [2962944,2963456) | 20260506 16:57:18.614 | INFO | MemInfo: { | 20260506 16:57:18.614 | INFO | "gpu" : " 17 GB", | 20260506 16:57:18.614 | INFO | "huge" : "", | 20260506 16:57:18.614 | INFO | "huge_shared" : " 27 GB", | 20260506 16:57:18.614 | INFO | "malloc" : " 14 GB", | 20260506 16:57:18.614 | INFO | "pinned" : " 29 GB" | 20260506 16:57:18.614 | INFO | } | 20260506 16:57:18.614 | INFO | Perf report for the last 5.13465 seconds: | 20260506 16:57:18.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:57:18.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:18.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:18.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:18.614 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:57:18.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:18.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:18.615 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:57:18.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:18.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:18.615 | INFO | Uploaded 36GiB to GPU in 1.94285s (18.5295GiB/s) | 20260506 16:57:18.615 | INFO | Downloaded 34GiB to GPU in 1.87819s (18.1025GiB/s) | 20260506 16:57:23.753 | INFO | Analyzing chunk frames = [2963456,2963968) | 20260506 16:57:23.754 | INFO | MemInfo: { | 20260506 16:57:23.754 | INFO | "gpu" : " 17 GB", | 20260506 16:57:23.754 | INFO | "huge" : "", | 20260506 16:57:23.754 | INFO | "huge_shared" : " 27 GB", | 20260506 16:57:23.754 | INFO | "malloc" : " 14 GB", | 20260506 16:57:23.754 | INFO | "pinned" : " 29 GB" | 20260506 16:57:23.754 | INFO | } | 20260506 16:57:23.754 | INFO | Perf report for the last 5.13937 seconds: | 20260506 16:57:23.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:57:23.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:23.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:23.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:23.754 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 16:57:23.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:23.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:23.754 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:57:23.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:23.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:23.754 | INFO | Uploaded 36GiB to GPU in 2.01511s (17.865GiB/s) | 20260506 16:57:23.754 | INFO | Downloaded 34GiB to GPU in 1.90846s (17.8154GiB/s) | 20260506 16:57:28.892 | INFO | Analyzing chunk frames = [2963968,2964480) | 20260506 16:57:28.892 | INFO | MemInfo: { | 20260506 16:57:28.892 | INFO | "gpu" : " 17 GB", | 20260506 16:57:28.892 | INFO | "huge" : "", | 20260506 16:57:28.893 | INFO | "huge_shared" : " 27 GB", | 20260506 16:57:28.893 | INFO | "malloc" : " 14 GB", | 20260506 16:57:28.893 | INFO | "pinned" : " 29 GB" | 20260506 16:57:28.893 | INFO | } | 20260506 16:57:28.893 | INFO | Perf report for the last 5.13887 seconds: | 20260506 16:57:28.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:57:28.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:28.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:28.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:28.893 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 16:57:28.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:28.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:28.893 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:57:28.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:28.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:28.893 | INFO | Uploaded 36GiB to GPU in 2.02355s (17.7905GiB/s) | 20260506 16:57:28.893 | INFO | Downloaded 34GiB to GPU in 1.93775s (17.5461GiB/s) | 20260506 16:57:34.035 | INFO | Analyzing chunk frames = [2964480,2964992) | 20260506 16:57:34.035 | INFO | MemInfo: { | 20260506 16:57:34.035 | INFO | "gpu" : " 17 GB", | 20260506 16:57:34.035 | INFO | "huge" : "", | 20260506 16:57:34.035 | INFO | "huge_shared" : " 27 GB", | 20260506 16:57:34.035 | INFO | "malloc" : " 14 GB", | 20260506 16:57:34.035 | INFO | "pinned" : " 29 GB" | 20260506 16:57:34.035 | INFO | } | 20260506 16:57:34.035 | INFO | Perf report for the last 5.14278 seconds: | 20260506 16:57:34.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:57:34.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:34.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:34.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:34.036 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 16:57:34.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:34.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:34.036 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 16:57:34.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:34.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:34.036 | INFO | Uploaded 36GiB to GPU in 2.00931s (17.9166GiB/s) | 20260506 16:57:34.036 | INFO | Downloaded 34GiB to GPU in 2.00573s (16.9515GiB/s) | 20260506 16:57:39.177 | INFO | Analyzing chunk frames = [2964992,2965504) | 20260506 16:57:39.177 | INFO | MemInfo: { | 20260506 16:57:39.177 | INFO | "gpu" : " 17 GB", | 20260506 16:57:39.177 | INFO | "huge" : "", | 20260506 16:57:39.177 | INFO | "huge_shared" : " 27 GB", | 20260506 16:57:39.177 | INFO | "malloc" : " 14 GB", | 20260506 16:57:39.177 | INFO | "pinned" : " 29 GB" | 20260506 16:57:39.177 | INFO | } | 20260506 16:57:39.177 | INFO | Perf report for the last 5.14153 seconds: | 20260506 16:57:39.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:57:39.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:39.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:39.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:39.177 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:57:39.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:39.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:39.177 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 16:57:39.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:39.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:39.177 | INFO | Uploaded 36GiB to GPU in 2.04755s (17.582GiB/s) | 20260506 16:57:39.177 | INFO | Downloaded 34GiB to GPU in 1.98868s (17.0968GiB/s) | 20260506 16:57:39.177 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:57:39.177 | INFO | Total mem usage: 18 GiB | 20260506 16:57:39.177 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:57:39.177 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.4%) | 20260506 16:57:39.177 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:57:39.177 | INFO | High Water Breakdown: | 20260506 16:57:39.177 | INFO | 61.3% : 9.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:57:39.177 | INFO | 38.5% : 6.01 GiB ( 89.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:57:39.178 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:57:39.178 | INFO | ------------------------------------------ | 20260506 16:57:39.178 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:57:39.178 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:57:39.178 | INFO | Total mem usage: 26.5 GiB | 20260506 16:57:39.178 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:57:39.178 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 16:57:39.178 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 16:57:39.178 | INFO | High Water Breakdown: | 20260506 16:57:39.178 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:57:39.178 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:57:39.178 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:57:39.178 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:57:39.178 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:57:39.178 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:57:39.178 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:57:39.178 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:57:39.178 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:57:39.178 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:57:39.178 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:57:39.178 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:57:39.178 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:57:39.178 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:57:39.178 | INFO | ------------------------------------------ | 20260506 16:57:39.178 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:57:39.178 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:57:39.178 | INFO | Total mem usage: 25.6 GiB | 20260506 16:57:39.178 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:57:39.178 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:57:39.178 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:57:39.178 | INFO | High Water Breakdown: | 20260506 16:57:39.178 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:57:39.178 | INFO | ------------------------------------------ | 20260506 16:57:39.178 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:57:39.178 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:57:39.178 | INFO | Total mem usage: 15.7 GiB | 20260506 16:57:39.178 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:57:39.178 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:57:39.178 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:57:39.178 | INFO | High Water Breakdown: | 20260506 16:57:39.178 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:57:39.178 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:57:39.178 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:57:39.178 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:57:39.178 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:57:39.178 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:57:39.178 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:57:39.178 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:57:39.178 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:57:39.178 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:57:39.178 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:57:39.178 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:57:39.178 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:57:39.178 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:57:39.178 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:57:39.178 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:57:39.178 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:57:39.178 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:57:39.178 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:57:39.178 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:57:39.178 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:57:39.178 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:57:39.178 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:57:39.178 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:57:39.178 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:57:39.178 | INFO | ------------------------------------------ | 20260506 16:57:39.178 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:57:39.178 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:57:39.178 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:57:39.178 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:57:39.178 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.56e+01, 1.34e+01 / 2.33e+00 | 20260506 16:57:39.178 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.00e-01, 9.82e-03 / 6.38e-03 | 20260506 16:57:39.178 | INFO | PerfReport: 11.61%, Baseline -- 3.64e+00 / 4.45e+00, 5.61e-01 / 3.53e-01 | 20260506 16:57:39.178 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.46e+00, 1.88e-01 / 1.48e-01 | 20260506 16:57:39.178 | INFO | PerfReport: 15.92%, DME -- 4.00e+01 / 4.41e+01, 3.74e+00 / 2.42e+00 | 20260506 16:57:39.178 | INFO | PerfReport: 56.97%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.47e+00 / 2.04e+00 | 20260506 16:57:39.178 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.28e+00 / 2.89e+00, 3.81e-01 / 2.25e-01 | 20260506 16:57:39.178 | INFO | PerfReport: 3.48%, Metrics -- 1.09e+00 / 1.44e+00, 2.84e-01 / 1.86e-01 | 20260506 16:57:39.178 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.77e-01 / 4.22e-01, 5.86e-02 / 2.37e-02 | 20260506 16:57:39.178 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:57:39.178 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:57:39.178 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:57:39.178 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 16:57:39.178 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260506 16:57:39.179 | INFO | PerfReport: 21.13%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 16:57:39.179 | INFO | PerfReport: 32.24%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 16:57:39.179 | INFO | PerfReport: 1.47%, LockUpload -- 1.43e+00 / 0.00e+00, 5.77e+00 / 0.00e+00 | 20260506 16:57:39.179 | INFO | PerfReport: 1.04%, LockDownload -- 1.01e+00 / 0.00e+00, 2.61e+00 / 0.00e+00 | 20260506 16:57:39.179 | INFO | PerfReport: 22.60%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 16:57:39.179 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.20e-02 / 0.00e+00 | 20260506 16:57:39.297 | INFO | Starting a round of Baz IO | 20260506 16:57:42.427 | INFO | Populating RTMetrics report | 20260506 16:57:42.427 | INFO | RTMetrics report populated | 20260506 16:57:42.427 | INFO | Opening tmp RTMetrics json file | 20260506 16:57:42.469 | INFO | Serializing RTMetrics to json | 20260506 16:57:42.488 | INFO | Serialization done, swapping tmp file now... | 20260506 16:57:42.501 | INFO | RTMetrics done | 20260506 16:57:42.911 | INFO | Finished a round of Baz IO: Wrote 2.075 GiB in 3.614 seconds | 20260506 16:57:42.911 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.96% / 0.01% / 64.03% | 20260506 16:57:42.911 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.5168% / 7.94729e-06% / 3.57628e-05% | 20260506 16:57:42.911 | INFO | Baz IO overall: | 20260506 16:57:42.911 | INFO | BytesWritten : 1560466374359 | 20260506 16:57:42.911 | INFO | NumEvents : 755029118106 | 20260506 16:57:44.316 | INFO | Analyzing chunk frames = [2965504,2966016) | 20260506 16:57:44.316 | INFO | MemInfo: { | 20260506 16:57:44.316 | INFO | "gpu" : " 17 GB", | 20260506 16:57:44.316 | INFO | "huge" : "", | 20260506 16:57:44.316 | INFO | "huge_shared" : " 27 GB", | 20260506 16:57:44.316 | INFO | "malloc" : " 14 GB", | 20260506 16:57:44.316 | INFO | "pinned" : " 29 GB" | 20260506 16:57:44.316 | INFO | } | 20260506 16:57:44.316 | INFO | Perf report for the last 5.13875 seconds: | 20260506 16:57:44.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:57:44.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:44.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:44.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:44.316 | INFO | ANALYSIS : 62.7%, 2.9, 94.2ms, 0 | 20260506 16:57:44.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:44.316 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260506 16:57:44.316 | INFO | PRE_HQ : 60.9%, 2.0, 62.1ms, 0 | 20260506 16:57:44.316 | INFO | BAZWRITER : 69.9%, 4.8, 172.5ms, 0 | 20260506 16:57:44.316 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 16:57:44.316 | INFO | Uploaded 36GiB to GPU in 2.05999s (17.4758GiB/s) | 20260506 16:57:44.316 | INFO | Downloaded 38GiB to GPU in 2.20853s (17.2061GiB/s) | 20260506 16:57:49.454 | INFO | Analyzing chunk frames = [2966016,2966528) | 20260506 16:57:49.454 | INFO | MemInfo: { | 20260506 16:57:49.454 | INFO | "gpu" : " 17 GB", | 20260506 16:57:49.454 | INFO | "huge" : "", | 20260506 16:57:49.454 | INFO | "huge_shared" : " 27 GB", | 20260506 16:57:49.454 | INFO | "malloc" : " 14 GB", | 20260506 16:57:49.454 | INFO | "pinned" : " 29 GB" | 20260506 16:57:49.454 | INFO | } | 20260506 16:57:49.454 | INFO | Perf report for the last 5.13813 seconds: | 20260506 16:57:49.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:57:49.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:49.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:49.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:49.454 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:57:49.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:49.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:49.454 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 16:57:49.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:49.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:49.454 | INFO | Uploaded 36GiB to GPU in 1.99929s (18.0064GiB/s) | 20260506 16:57:49.454 | INFO | Downloaded 34GiB to GPU in 1.93879s (17.5367GiB/s) | 20260506 16:57:54.593 | INFO | Analyzing chunk frames = [2966528,2967040) | 20260506 16:57:54.593 | INFO | MemInfo: { | 20260506 16:57:54.593 | INFO | "gpu" : " 17 GB", | 20260506 16:57:54.593 | INFO | "huge" : "", | 20260506 16:57:54.593 | INFO | "huge_shared" : " 27 GB", | 20260506 16:57:54.593 | INFO | "malloc" : " 14 GB", | 20260506 16:57:54.593 | INFO | "pinned" : " 29 GB" | 20260506 16:57:54.593 | INFO | } | 20260506 16:57:54.593 | INFO | Perf report for the last 5.13903 seconds: | 20260506 16:57:54.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:57:54.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:54.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:54.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:54.593 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 16:57:54.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:54.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:54.593 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:57:54.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:54.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:54.593 | INFO | Uploaded 36GiB to GPU in 1.98133s (18.1696GiB/s) | 20260506 16:57:54.593 | INFO | Downloaded 34GiB to GPU in 1.93399s (17.5803GiB/s) | 20260506 16:57:59.732 | INFO | Analyzing chunk frames = [2967040,2967552) | 20260506 16:57:59.732 | INFO | MemInfo: { | 20260506 16:57:59.732 | INFO | "gpu" : " 17 GB", | 20260506 16:57:59.732 | INFO | "huge" : "", | 20260506 16:57:59.732 | INFO | "huge_shared" : " 27 GB", | 20260506 16:57:59.732 | INFO | "malloc" : " 14 GB", | 20260506 16:57:59.732 | INFO | "pinned" : " 29 GB" | 20260506 16:57:59.732 | INFO | } | 20260506 16:57:59.732 | INFO | Perf report for the last 5.13906 seconds: | 20260506 16:57:59.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:57:59.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:59.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:59.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:59.732 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 16:57:59.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:59.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:59.732 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:57:59.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:59.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:57:59.732 | INFO | Uploaded 36GiB to GPU in 1.9316s (18.6374GiB/s) | 20260506 16:57:59.732 | INFO | Downloaded 34GiB to GPU in 1.8763s (18.1208GiB/s) | 20260506 16:58:04.867 | INFO | Analyzing chunk frames = [2967552,2968064) | 20260506 16:58:04.867 | INFO | MemInfo: { | 20260506 16:58:04.867 | INFO | "gpu" : " 17 GB", | 20260506 16:58:04.867 | INFO | "huge" : "", | 20260506 16:58:04.867 | INFO | "huge_shared" : " 27 GB", | 20260506 16:58:04.867 | INFO | "malloc" : " 14 GB", | 20260506 16:58:04.867 | INFO | "pinned" : " 29 GB" | 20260506 16:58:04.867 | INFO | } | 20260506 16:58:04.867 | INFO | Perf report for the last 5.13479 seconds: | 20260506 16:58:04.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:58:04.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:04.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:04.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:04.867 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 16:58:04.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:04.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:04.867 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 16:58:04.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:04.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:04.867 | INFO | Uploaded 36GiB to GPU in 2.00808s (17.9276GiB/s) | 20260506 16:58:04.867 | INFO | Downloaded 34GiB to GPU in 1.91531s (17.7517GiB/s) | 20260506 16:58:10.010 | INFO | Analyzing chunk frames = [2968064,2968576) | 20260506 16:58:10.010 | INFO | MemInfo: { | 20260506 16:58:10.010 | INFO | "gpu" : " 17 GB", | 20260506 16:58:10.010 | INFO | "huge" : "", | 20260506 16:58:10.010 | INFO | "huge_shared" : " 27 GB", | 20260506 16:58:10.010 | INFO | "malloc" : " 14 GB", | 20260506 16:58:10.010 | INFO | "pinned" : " 29 GB" | 20260506 16:58:10.010 | INFO | } | 20260506 16:58:10.010 | INFO | Perf report for the last 5.1433 seconds: | 20260506 16:58:10.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:58:10.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:10.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:10.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:10.010 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 16:58:10.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:10.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:10.010 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 16:58:10.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:10.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:10.010 | INFO | Uploaded 36GiB to GPU in 2.02315s (17.7941GiB/s) | 20260506 16:58:10.010 | INFO | Downloaded 34GiB to GPU in 1.93803s (17.5436GiB/s) | 20260506 16:58:15.153 | INFO | Analyzing chunk frames = [2968576,2969088) | 20260506 16:58:15.153 | INFO | MemInfo: { | 20260506 16:58:15.153 | INFO | "gpu" : " 17 GB", | 20260506 16:58:15.153 | INFO | "huge" : "", | 20260506 16:58:15.153 | INFO | "huge_shared" : " 27 GB", | 20260506 16:58:15.153 | INFO | "malloc" : " 14 GB", | 20260506 16:58:15.153 | INFO | "pinned" : " 29 GB" | 20260506 16:58:15.153 | INFO | } | 20260506 16:58:15.153 | INFO | Perf report for the last 5.14271 seconds: | 20260506 16:58:15.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:58:15.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:15.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:15.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:15.153 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 16:58:15.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:15.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:15.153 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 16:58:15.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:15.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:15.153 | INFO | Uploaded 36GiB to GPU in 2.01657s (17.8521GiB/s) | 20260506 16:58:15.153 | INFO | Downloaded 34GiB to GPU in 2.0123s (16.8961GiB/s) | 20260506 16:58:20.292 | INFO | Analyzing chunk frames = [2969088,2969600) | 20260506 16:58:20.292 | INFO | MemInfo: { | 20260506 16:58:20.292 | INFO | "gpu" : " 17 GB", | 20260506 16:58:20.292 | INFO | "huge" : "", | 20260506 16:58:20.292 | INFO | "huge_shared" : " 27 GB", | 20260506 16:58:20.292 | INFO | "malloc" : " 14 GB", | 20260506 16:58:20.292 | INFO | "pinned" : " 29 GB" | 20260506 16:58:20.292 | INFO | } | 20260506 16:58:20.292 | INFO | Perf report for the last 5.13932 seconds: | 20260506 16:58:20.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:58:20.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:20.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:20.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:20.292 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:58:20.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:20.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:20.292 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:58:20.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:20.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:20.292 | INFO | Uploaded 36GiB to GPU in 1.98997s (18.0907GiB/s) | 20260506 16:58:20.293 | INFO | Downloaded 34GiB to GPU in 1.97284s (17.234GiB/s) | 20260506 16:58:20.410 | INFO | Starting a round of Baz IO | 20260506 16:58:23.590 | INFO | Populating RTMetrics report | 20260506 16:58:23.591 | INFO | RTMetrics report populated | 20260506 16:58:23.591 | INFO | Opening tmp RTMetrics json file | 20260506 16:58:23.594 | INFO | Serializing RTMetrics to json | 20260506 16:58:23.600 | INFO | Serialization done, swapping tmp file now... | 20260506 16:58:23.602 | INFO | RTMetrics done | 20260506 16:58:24.097 | INFO | Finished a round of Baz IO: Wrote 2.076 GiB in 3.686 seconds | 20260506 16:58:24.097 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.96% / 0.01% / 64.04% | 20260506 16:58:24.097 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.51454% / 7.94729e-06% / 5.16574e-05% | 20260506 16:58:24.097 | INFO | Baz IO overall: | 20260506 16:58:24.097 | INFO | BytesWritten : 1562695348894 | 20260506 16:58:24.097 | INFO | NumEvents : 756105960131 | 20260506 16:58:25.436 | INFO | Analyzing chunk frames = [2969600,2970112) | 20260506 16:58:25.436 | INFO | MemInfo: { | 20260506 16:58:25.436 | INFO | "gpu" : " 17 GB", | 20260506 16:58:25.436 | INFO | "huge" : "", | 20260506 16:58:25.436 | INFO | "huge_shared" : " 27 GB", | 20260506 16:58:25.436 | INFO | "malloc" : " 14 GB", | 20260506 16:58:25.436 | INFO | "pinned" : " 29 GB" | 20260506 16:58:25.436 | INFO | } | 20260506 16:58:25.436 | INFO | Perf report for the last 5.14398 seconds: | 20260506 16:58:25.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:58:25.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:25.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:25.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:25.436 | INFO | ANALYSIS : 63.2%, 2.8, 94.0ms, 0 | 20260506 16:58:25.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:25.436 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260506 16:58:25.436 | INFO | PRE_HQ : 61.1%, 2.0, 63.0ms, 0 | 20260506 16:58:25.436 | INFO | BAZWRITER : 71.3%, 4.8, 175.5ms, 0 | 20260506 16:58:25.436 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 16:58:25.436 | INFO | Uploaded 36GiB to GPU in 2.0798s (17.3093GiB/s) | 20260506 16:58:25.436 | INFO | Downloaded 38GiB to GPU in 2.18615s (17.3821GiB/s) | 20260506 16:58:30.569 | INFO | Analyzing chunk frames = [2970112,2970624) | 20260506 16:58:30.569 | INFO | MemInfo: { | 20260506 16:58:30.569 | INFO | "gpu" : " 17 GB", | 20260506 16:58:30.569 | INFO | "huge" : "", | 20260506 16:58:30.569 | INFO | "huge_shared" : " 27 GB", | 20260506 16:58:30.569 | INFO | "malloc" : " 14 GB", | 20260506 16:58:30.569 | INFO | "pinned" : " 29 GB" | 20260506 16:58:30.569 | INFO | } | 20260506 16:58:30.569 | INFO | Perf report for the last 5.13286 seconds: | 20260506 16:58:30.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:58:30.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:30.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:30.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:30.569 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 16:58:30.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:30.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:30.569 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:58:30.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:30.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:30.569 | INFO | Uploaded 36GiB to GPU in 2.00055s (17.995GiB/s) | 20260506 16:58:30.569 | INFO | Downloaded 34GiB to GPU in 1.93315s (17.5878GiB/s) | 20260506 16:58:35.708 | INFO | Analyzing chunk frames = [2970624,2971136) | 20260506 16:58:35.708 | INFO | MemInfo: { | 20260506 16:58:35.708 | INFO | "gpu" : " 17 GB", | 20260506 16:58:35.708 | INFO | "huge" : "", | 20260506 16:58:35.708 | INFO | "huge_shared" : " 27 GB", | 20260506 16:58:35.708 | INFO | "malloc" : " 14 GB", | 20260506 16:58:35.708 | INFO | "pinned" : " 29 GB" | 20260506 16:58:35.708 | INFO | } | 20260506 16:58:35.708 | INFO | Perf report for the last 5.13871 seconds: | 20260506 16:58:35.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:58:35.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:35.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:35.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:35.708 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 16:58:35.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:35.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:35.708 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:58:35.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:35.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:35.708 | INFO | Uploaded 36GiB to GPU in 1.9761s (18.2177GiB/s) | 20260506 16:58:35.708 | INFO | Downloaded 34GiB to GPU in 1.92974s (17.619GiB/s) | 20260506 16:58:40.849 | INFO | Analyzing chunk frames = [2971136,2971648) | 20260506 16:58:40.849 | INFO | MemInfo: { | 20260506 16:58:40.849 | INFO | "gpu" : " 17 GB", | 20260506 16:58:40.849 | INFO | "huge" : "", | 20260506 16:58:40.850 | INFO | "huge_shared" : " 27 GB", | 20260506 16:58:40.850 | INFO | "malloc" : " 14 GB", | 20260506 16:58:40.850 | INFO | "pinned" : " 29 GB" | 20260506 16:58:40.850 | INFO | } | 20260506 16:58:40.850 | INFO | Perf report for the last 5.14187 seconds: | 20260506 16:58:40.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:58:40.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:40.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:40.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:40.850 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 16:58:40.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:40.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:40.850 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 16:58:40.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:40.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:40.850 | INFO | Uploaded 36GiB to GPU in 1.92924s (18.6602GiB/s) | 20260506 16:58:40.850 | INFO | Downloaded 34GiB to GPU in 1.87109s (18.1712GiB/s) | 20260506 16:58:45.983 | INFO | Analyzing chunk frames = [2971648,2972160) | 20260506 16:58:45.983 | INFO | MemInfo: { | 20260506 16:58:45.983 | INFO | "gpu" : " 17 GB", | 20260506 16:58:45.983 | INFO | "huge" : "", | 20260506 16:58:45.983 | INFO | "huge_shared" : " 27 GB", | 20260506 16:58:45.983 | INFO | "malloc" : " 14 GB", | 20260506 16:58:45.983 | INFO | "pinned" : " 29 GB" | 20260506 16:58:45.983 | INFO | } | 20260506 16:58:45.983 | INFO | Perf report for the last 5.13322 seconds: | 20260506 16:58:45.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:58:45.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:45.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:45.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:45.983 | INFO | ANALYSIS : 60.2%, 3.0, 93.5ms, 0 | 20260506 16:58:45.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:45.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:45.983 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:58:45.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:45.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:45.983 | INFO | Uploaded 36GiB to GPU in 1.96363s (18.3334GiB/s) | 20260506 16:58:45.983 | INFO | Downloaded 34GiB to GPU in 1.87969s (18.0881GiB/s) | 20260506 16:58:51.123 | INFO | Analyzing chunk frames = [2972160,2972672) | 20260506 16:58:51.123 | INFO | MemInfo: { | 20260506 16:58:51.123 | INFO | "gpu" : " 17 GB", | 20260506 16:58:51.123 | INFO | "huge" : "", | 20260506 16:58:51.123 | INFO | "huge_shared" : " 27 GB", | 20260506 16:58:51.123 | INFO | "malloc" : " 14 GB", | 20260506 16:58:51.123 | INFO | "pinned" : " 29 GB" | 20260506 16:58:51.123 | INFO | } | 20260506 16:58:51.123 | INFO | Perf report for the last 5.13984 seconds: | 20260506 16:58:51.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:58:51.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:51.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:51.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:51.123 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 16:58:51.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:51.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:51.123 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:58:51.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:51.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:51.123 | INFO | Uploaded 36GiB to GPU in 2.00671s (17.9398GiB/s) | 20260506 16:58:51.123 | INFO | Downloaded 34GiB to GPU in 1.93941s (17.5311GiB/s) | 20260506 16:58:56.264 | INFO | Analyzing chunk frames = [2972672,2973184) | 20260506 16:58:56.264 | INFO | MemInfo: { | 20260506 16:58:56.264 | INFO | "gpu" : " 17 GB", | 20260506 16:58:56.264 | INFO | "huge" : "", | 20260506 16:58:56.264 | INFO | "huge_shared" : " 27 GB", | 20260506 16:58:56.264 | INFO | "malloc" : " 14 GB", | 20260506 16:58:56.264 | INFO | "pinned" : " 29 GB" | 20260506 16:58:56.264 | INFO | } | 20260506 16:58:56.264 | INFO | Perf report for the last 5.14166 seconds: | 20260506 16:58:56.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:58:56.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:56.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:56.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:56.264 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 16:58:56.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:56.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:56.265 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 16:58:56.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:56.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:58:56.265 | INFO | Uploaded 36GiB to GPU in 1.98894s (18.1001GiB/s) | 20260506 16:58:56.265 | INFO | Downloaded 34GiB to GPU in 2.00093s (16.9921GiB/s) | 20260506 16:59:01.405 | INFO | Analyzing chunk frames = [2973184,2973696) | 20260506 16:59:01.406 | INFO | MemInfo: { | 20260506 16:59:01.406 | INFO | "gpu" : " 17 GB", | 20260506 16:59:01.406 | INFO | "huge" : "", | 20260506 16:59:01.406 | INFO | "huge_shared" : " 27 GB", | 20260506 16:59:01.406 | INFO | "malloc" : " 14 GB", | 20260506 16:59:01.406 | INFO | "pinned" : " 29 GB" | 20260506 16:59:01.406 | INFO | } | 20260506 16:59:01.406 | INFO | Perf report for the last 5.14136 seconds: | 20260506 16:59:01.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:59:01.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:01.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:01.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:01.406 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 16:59:01.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:01.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:01.406 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 16:59:01.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:01.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:01.406 | INFO | Uploaded 36GiB to GPU in 2.0037s (17.9668GiB/s) | 20260506 16:59:01.406 | INFO | Downloaded 34GiB to GPU in 1.95957s (17.3508GiB/s) | 20260506 16:59:01.406 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 16:59:01.406 | INFO | Total mem usage: 18 GiB | 20260506 16:59:01.406 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 16:59:01.406 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -10.1%) | 20260506 16:59:01.406 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 16:59:01.406 | INFO | High Water Breakdown: | 20260506 16:59:01.406 | INFO | 62% : 10.1 GiB ( 88.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 16:59:01.406 | INFO | 37.8% : 6.19 GiB ( 92.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 16:59:01.406 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 16:59:01.406 | INFO | ------------------------------------------ | 20260506 16:59:01.406 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:59:01.406 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 16:59:01.406 | INFO | Total mem usage: 26.5 GiB | 20260506 16:59:01.406 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 16:59:01.406 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260506 16:59:01.406 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260506 16:59:01.406 | INFO | High Water Breakdown: | 20260506 16:59:01.406 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:59:01.406 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:59:01.406 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:59:01.406 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:59:01.406 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:59:01.406 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:59:01.406 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 16:59:01.406 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 16:59:01.406 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:59:01.406 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 16:59:01.406 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 16:59:01.407 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 16:59:01.407 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 16:59:01.407 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 16:59:01.407 | INFO | ------------------------------------------ | 20260506 16:59:01.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:59:01.407 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 16:59:01.407 | INFO | Total mem usage: 25.6 GiB | 20260506 16:59:01.407 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 16:59:01.407 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 16:59:01.407 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 16:59:01.407 | INFO | High Water Breakdown: | 20260506 16:59:01.407 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 16:59:01.407 | INFO | ------------------------------------------ | 20260506 16:59:01.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:59:01.407 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 16:59:01.407 | INFO | Total mem usage: 15.7 GiB | 20260506 16:59:01.407 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 16:59:01.407 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 16:59:01.407 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 16:59:01.407 | INFO | High Water Breakdown: | 20260506 16:59:01.407 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 16:59:01.407 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 16:59:01.407 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 16:59:01.407 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 16:59:01.407 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 16:59:01.407 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 16:59:01.407 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 16:59:01.407 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 16:59:01.407 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 16:59:01.407 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 16:59:01.407 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 16:59:01.407 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 16:59:01.407 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 16:59:01.407 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 16:59:01.407 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 16:59:01.407 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 16:59:01.407 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 16:59:01.407 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 16:59:01.407 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 16:59:01.407 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 16:59:01.407 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 16:59:01.407 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 16:59:01.407 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 16:59:01.407 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 16:59:01.407 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 16:59:01.407 | INFO | ------------------------------------------ | 20260506 16:59:01.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 16:59:01.407 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 16:59:01.407 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:59:01.407 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:59:01.407 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.56e+01, 1.36e+01 / 2.52e+00 | 20260506 16:59:01.407 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 1.01e-02 / 7.46e-03 | 20260506 16:59:01.407 | INFO | PerfReport: 11.63%, Baseline -- 3.65e+00 / 4.36e+00, 5.70e-01 / 3.79e-01 | 20260506 16:59:01.407 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.46e+00, 1.88e-01 / 1.36e-01 | 20260506 16:59:01.407 | INFO | PerfReport: 15.94%, DME -- 4.00e+01 / 4.46e+01, 4.04e+00 / 2.58e+00 | 20260506 16:59:01.407 | INFO | PerfReport: 56.91%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.42e+00 / 1.49e+00 | 20260506 16:59:01.407 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.29e+00 / 2.79e+00, 3.89e-01 / 2.33e-01 | 20260506 16:59:01.407 | INFO | PerfReport: 3.46%, Metrics -- 1.08e+00 / 1.43e+00, 2.77e-01 / 1.77e-01 | 20260506 16:59:01.407 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.77e-01 / 4.23e-01, 5.82e-02 / 2.16e-02 | 20260506 16:59:01.407 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 16:59:01.407 | INFO | PerfReport: Gathered 1536 samples. | 20260506 16:59:01.407 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 16:59:01.407 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 16:59:01.407 | INFO | PerfReport: 21.44%, Upload -- 2.08e+01 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260506 16:59:01.407 | INFO | PerfReport: 21.13%, Download -- 2.05e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260506 16:59:01.407 | INFO | PerfReport: 32.32%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 16:59:01.407 | INFO | PerfReport: 1.42%, LockUpload -- 1.38e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260506 16:59:01.407 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260506 16:59:01.407 | INFO | PerfReport: 22.58%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 16:59:01.407 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260506 16:59:01.521 | INFO | Starting a round of Baz IO | 20260506 16:59:04.667 | INFO | Populating RTMetrics report | 20260506 16:59:04.667 | INFO | RTMetrics report populated | 20260506 16:59:04.667 | INFO | Opening tmp RTMetrics json file | 20260506 16:59:04.914 | INFO | Serializing RTMetrics to json | 20260506 16:59:04.923 | INFO | Serialization done, swapping tmp file now... | 20260506 16:59:04.928 | INFO | RTMetrics done | 20260506 16:59:05.550 | INFO | Finished a round of Baz IO: Wrote 2.074 GiB in 4.029 seconds | 20260506 16:59:05.550 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.95% / 0.01% / 64.04% | 20260506 16:59:05.550 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.50899% / 3.97364e-06% / 5.16574e-05% | 20260506 16:59:05.550 | INFO | Baz IO overall: | 20260506 16:59:05.550 | INFO | BytesWritten : 1564922288345 | 20260506 16:59:05.550 | INFO | NumEvents : 757181200804 | 20260506 16:59:06.544 | INFO | Analyzing chunk frames = [2973696,2974208) | 20260506 16:59:06.544 | INFO | MemInfo: { | 20260506 16:59:06.544 | INFO | "gpu" : " 17 GB", | 20260506 16:59:06.544 | INFO | "huge" : "", | 20260506 16:59:06.544 | INFO | "huge_shared" : " 27 GB", | 20260506 16:59:06.544 | INFO | "malloc" : " 14 GB", | 20260506 16:59:06.544 | INFO | "pinned" : " 29 GB" | 20260506 16:59:06.544 | INFO | } | 20260506 16:59:06.544 | INFO | Perf report for the last 5.13837 seconds: | 20260506 16:59:06.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:59:06.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:06.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:06.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:06.544 | INFO | ANALYSIS : 62.9%, 2.9, 95.6ms, 0 | 20260506 16:59:06.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:06.544 | INFO | RT_METRICS : 48.6%, 1.0, 25.5ms, 0 | 20260506 16:59:06.544 | INFO | PRE_HQ : 60.6%, 1.9, 58.6ms, 0 | 20260506 16:59:06.544 | INFO | BAZWRITER : 77.9%, 4.8, 192.2ms, 0 | 20260506 16:59:06.544 | INFO | BAZ_SAMPLING : 4.6%, 1.0, 2.4ms, 0 | 20260506 16:59:06.545 | INFO | Uploaded 36GiB to GPU in 2.07853s (17.3199GiB/s) | 20260506 16:59:06.545 | INFO | Downloaded 38GiB to GPU in 2.20195s (17.2574GiB/s) | 20260506 16:59:11.683 | INFO | Analyzing chunk frames = [2974208,2974720) | 20260506 16:59:11.683 | INFO | MemInfo: { | 20260506 16:59:11.683 | INFO | "gpu" : " 17 GB", | 20260506 16:59:11.683 | INFO | "huge" : "", | 20260506 16:59:11.683 | INFO | "huge_shared" : " 27 GB", | 20260506 16:59:11.683 | INFO | "malloc" : " 14 GB", | 20260506 16:59:11.683 | INFO | "pinned" : " 29 GB" | 20260506 16:59:11.683 | INFO | } | 20260506 16:59:11.683 | INFO | Perf report for the last 5.13922 seconds: | 20260506 16:59:11.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:59:11.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:11.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:11.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:11.683 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 16:59:11.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:11.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:11.684 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:59:11.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:11.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:11.684 | INFO | Uploaded 36GiB to GPU in 2.0067s (17.9399GiB/s) | 20260506 16:59:11.684 | INFO | Downloaded 34GiB to GPU in 1.94883s (17.4464GiB/s) | 20260506 16:59:16.822 | INFO | Analyzing chunk frames = [2974720,2975232) | 20260506 16:59:16.822 | INFO | MemInfo: { | 20260506 16:59:16.822 | INFO | "gpu" : " 17 GB", | 20260506 16:59:16.822 | INFO | "huge" : "", | 20260506 16:59:16.822 | INFO | "huge_shared" : " 27 GB", | 20260506 16:59:16.822 | INFO | "malloc" : " 14 GB", | 20260506 16:59:16.822 | INFO | "pinned" : " 29 GB" | 20260506 16:59:16.822 | INFO | } | 20260506 16:59:16.822 | INFO | Perf report for the last 5.13881 seconds: | 20260506 16:59:16.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:59:16.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:16.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:16.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:16.822 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 16:59:16.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:16.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:16.822 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:59:16.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:16.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:16.822 | INFO | Uploaded 36GiB to GPU in 1.98332s (18.1514GiB/s) | 20260506 16:59:16.823 | INFO | Downloaded 34GiB to GPU in 1.92825s (17.6326GiB/s) | 20260506 16:59:21.960 | INFO | Analyzing chunk frames = [2975232,2975744) | 20260506 16:59:21.960 | INFO | MemInfo: { | 20260506 16:59:21.960 | INFO | "gpu" : " 17 GB", | 20260506 16:59:21.960 | INFO | "huge" : "", | 20260506 16:59:21.960 | INFO | "huge_shared" : " 27 GB", | 20260506 16:59:21.960 | INFO | "malloc" : " 14 GB", | 20260506 16:59:21.960 | INFO | "pinned" : " 29 GB" | 20260506 16:59:21.960 | INFO | } | 20260506 16:59:21.961 | INFO | Perf report for the last 5.1383 seconds: | 20260506 16:59:21.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:59:21.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:21.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:21.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:21.961 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 16:59:21.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:21.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:21.961 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 16:59:21.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:21.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:21.961 | INFO | Uploaded 36GiB to GPU in 1.95225s (18.4403GiB/s) | 20260506 16:59:21.961 | INFO | Downloaded 34GiB to GPU in 1.87555s (18.128GiB/s) | 20260506 16:59:27.106 | INFO | Analyzing chunk frames = [2975744,2976256) | 20260506 16:59:27.106 | INFO | MemInfo: { | 20260506 16:59:27.106 | INFO | "gpu" : " 17 GB", | 20260506 16:59:27.106 | INFO | "huge" : "", | 20260506 16:59:27.106 | INFO | "huge_shared" : " 27 GB", | 20260506 16:59:27.106 | INFO | "malloc" : " 14 GB", | 20260506 16:59:27.106 | INFO | "pinned" : " 29 GB" | 20260506 16:59:27.106 | INFO | } | 20260506 16:59:27.106 | INFO | Perf report for the last 5.14571 seconds: | 20260506 16:59:27.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:59:27.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:27.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:27.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:27.106 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260506 16:59:27.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:27.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:27.106 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 16:59:27.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:27.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:27.107 | INFO | Uploaded 36GiB to GPU in 1.95611s (18.4038GiB/s) | 20260506 16:59:27.107 | INFO | Downloaded 34GiB to GPU in 1.88179s (18.0679GiB/s) | 20260506 16:59:32.242 | INFO | Analyzing chunk frames = [2976256,2976768) | 20260506 16:59:32.242 | INFO | MemInfo: { | 20260506 16:59:32.242 | INFO | "gpu" : " 17 GB", | 20260506 16:59:32.242 | INFO | "huge" : "", | 20260506 16:59:32.242 | INFO | "huge_shared" : " 27 GB", | 20260506 16:59:32.242 | INFO | "malloc" : " 14 GB", | 20260506 16:59:32.242 | INFO | "pinned" : " 29 GB" | 20260506 16:59:32.242 | INFO | } | 20260506 16:59:32.242 | INFO | Perf report for the last 5.13565 seconds: | 20260506 16:59:32.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:59:32.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:32.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:32.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:32.242 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 16:59:32.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:32.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:32.242 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 16:59:32.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:32.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:32.242 | INFO | Uploaded 36GiB to GPU in 2.01902s (17.8305GiB/s) | 20260506 16:59:32.242 | INFO | Downloaded 34GiB to GPU in 1.93704s (17.5525GiB/s) | 20260506 16:59:37.380 | INFO | Analyzing chunk frames = [2976768,2977280) | 20260506 16:59:37.380 | INFO | MemInfo: { | 20260506 16:59:37.380 | INFO | "gpu" : " 17 GB", | 20260506 16:59:37.380 | INFO | "huge" : "", | 20260506 16:59:37.380 | INFO | "huge_shared" : " 27 GB", | 20260506 16:59:37.380 | INFO | "malloc" : " 14 GB", | 20260506 16:59:37.380 | INFO | "pinned" : " 29 GB" | 20260506 16:59:37.380 | INFO | } | 20260506 16:59:37.380 | INFO | Perf report for the last 5.13823 seconds: | 20260506 16:59:37.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:59:37.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:37.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:37.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:37.380 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 16:59:37.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:37.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:37.380 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 16:59:37.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:37.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:37.380 | INFO | Uploaded 36GiB to GPU in 2.02744s (17.7564GiB/s) | 20260506 16:59:37.380 | INFO | Downloaded 34GiB to GPU in 1.99487s (17.0437GiB/s) | 20260506 16:59:42.521 | INFO | Analyzing chunk frames = [2977280,2977792) | 20260506 16:59:42.521 | INFO | MemInfo: { | 20260506 16:59:42.521 | INFO | "gpu" : " 17 GB", | 20260506 16:59:42.521 | INFO | "huge" : "", | 20260506 16:59:42.521 | INFO | "huge_shared" : " 27 GB", | 20260506 16:59:42.521 | INFO | "malloc" : " 14 GB", | 20260506 16:59:42.521 | INFO | "pinned" : " 29 GB" | 20260506 16:59:42.521 | INFO | } | 20260506 16:59:42.521 | INFO | Perf report for the last 5.14092 seconds: | 20260506 16:59:42.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:59:42.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:42.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:42.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:42.521 | INFO | ANALYSIS : 62.0%, 3.0, 95.7ms, 0 | 20260506 16:59:42.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:42.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:42.521 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260506 16:59:42.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:42.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:42.521 | INFO | Uploaded 36GiB to GPU in 2.05107s (17.5518GiB/s) | 20260506 16:59:42.521 | INFO | Downloaded 34GiB to GPU in 1.98767s (17.1055GiB/s) | 20260506 16:59:42.641 | INFO | Starting a round of Baz IO | 20260506 16:59:45.785 | INFO | Populating RTMetrics report | 20260506 16:59:45.786 | INFO | RTMetrics report populated | 20260506 16:59:45.786 | INFO | Opening tmp RTMetrics json file | 20260506 16:59:45.787 | INFO | Serializing RTMetrics to json | 20260506 16:59:45.788 | INFO | Serialization done, swapping tmp file now... | 20260506 16:59:45.789 | INFO | RTMetrics done | 20260506 16:59:46.468 | INFO | Finished a round of Baz IO: Wrote 2.075 GiB in 3.826 seconds | 20260506 16:59:46.468 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.94% / 0.01% / 64.05% | 20260506 16:59:46.468 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.50551% / 1.58946e-05% / 5.5631e-05% | 20260506 16:59:46.468 | INFO | Baz IO overall: | 20260506 16:59:46.468 | INFO | BytesWritten : 1567150539813 | 20260506 16:59:46.468 | INFO | NumEvents : 758257540496 | 20260506 16:59:47.658 | INFO | Analyzing chunk frames = [2977792,2978304) | 20260506 16:59:47.658 | INFO | MemInfo: { | 20260506 16:59:47.658 | INFO | "gpu" : " 17 GB", | 20260506 16:59:47.658 | INFO | "huge" : "", | 20260506 16:59:47.658 | INFO | "huge_shared" : " 27 GB", | 20260506 16:59:47.658 | INFO | "malloc" : " 14 GB", | 20260506 16:59:47.658 | INFO | "pinned" : " 29 GB" | 20260506 16:59:47.658 | INFO | } | 20260506 16:59:47.658 | INFO | Perf report for the last 5.13676 seconds: | 20260506 16:59:47.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:59:47.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:47.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:47.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:47.658 | INFO | ANALYSIS : 63.0%, 2.9, 95.6ms, 0 | 20260506 16:59:47.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:47.658 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 16:59:47.658 | INFO | PRE_HQ : 60.4%, 1.9, 58.8ms, 0 | 20260506 16:59:47.658 | INFO | BAZWRITER : 74.1%, 4.8, 182.8ms, 0 | 20260506 16:59:47.658 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 16:59:47.658 | INFO | Uploaded 36GiB to GPU in 2.07258s (17.3696GiB/s) | 20260506 16:59:47.658 | INFO | Downloaded 38GiB to GPU in 2.22363s (17.0892GiB/s) | 20260506 16:59:52.798 | INFO | Analyzing chunk frames = [2978304,2978816) | 20260506 16:59:52.798 | INFO | MemInfo: { | 20260506 16:59:52.798 | INFO | "gpu" : " 17 GB", | 20260506 16:59:52.798 | INFO | "huge" : "", | 20260506 16:59:52.798 | INFO | "huge_shared" : " 27 GB", | 20260506 16:59:52.798 | INFO | "malloc" : " 14 GB", | 20260506 16:59:52.798 | INFO | "pinned" : " 29 GB" | 20260506 16:59:52.798 | INFO | } | 20260506 16:59:52.798 | INFO | Perf report for the last 5.14026 seconds: | 20260506 16:59:52.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:59:52.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:52.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:52.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:52.798 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 16:59:52.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:52.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:52.798 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 16:59:52.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:52.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:52.798 | INFO | Uploaded 36GiB to GPU in 2.00165s (17.9852GiB/s) | 20260506 16:59:52.798 | INFO | Downloaded 34GiB to GPU in 1.93618s (17.5604GiB/s) | 20260506 16:59:57.936 | INFO | Analyzing chunk frames = [2978816,2979328) | 20260506 16:59:57.936 | INFO | MemInfo: { | 20260506 16:59:57.936 | INFO | "gpu" : " 17 GB", | 20260506 16:59:57.936 | INFO | "huge" : "", | 20260506 16:59:57.936 | INFO | "huge_shared" : " 27 GB", | 20260506 16:59:57.936 | INFO | "malloc" : " 14 GB", | 20260506 16:59:57.937 | INFO | "pinned" : " 29 GB" | 20260506 16:59:57.937 | INFO | } | 20260506 16:59:57.937 | INFO | Perf report for the last 5.1385 seconds: | 20260506 16:59:57.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 16:59:57.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:57.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:57.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:57.937 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 16:59:57.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:57.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:57.937 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 16:59:57.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:57.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 16:59:57.937 | INFO | Uploaded 36GiB to GPU in 1.98031s (18.179GiB/s) | 20260506 16:59:57.937 | INFO | Downloaded 34GiB to GPU in 1.92505s (17.6619GiB/s) | 20260506 17:00:03.073 | INFO | Analyzing chunk frames = [2979328,2979840) | 20260506 17:00:03.073 | INFO | MemInfo: { | 20260506 17:00:03.073 | INFO | "gpu" : " 17 GB", | 20260506 17:00:03.073 | INFO | "huge" : "", | 20260506 17:00:03.073 | INFO | "huge_shared" : " 27 GB", | 20260506 17:00:03.073 | INFO | "malloc" : " 14 GB", | 20260506 17:00:03.073 | INFO | "pinned" : " 29 GB" | 20260506 17:00:03.073 | INFO | } | 20260506 17:00:03.073 | INFO | Perf report for the last 5.13648 seconds: | 20260506 17:00:03.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:00:03.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:03.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:03.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:03.073 | INFO | ANALYSIS : 60.2%, 3.0, 93.1ms, 0 | 20260506 17:00:03.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:03.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:03.073 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 17:00:03.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:03.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:03.073 | INFO | Uploaded 36GiB to GPU in 1.93308s (18.6231GiB/s) | 20260506 17:00:03.074 | INFO | Downloaded 34GiB to GPU in 1.87547s (18.1288GiB/s) | 20260506 17:00:08.216 | INFO | Analyzing chunk frames = [2979840,2980352) | 20260506 17:00:08.216 | INFO | MemInfo: { | 20260506 17:00:08.216 | INFO | "gpu" : " 17 GB", | 20260506 17:00:08.216 | INFO | "huge" : "", | 20260506 17:00:08.216 | INFO | "huge_shared" : " 27 GB", | 20260506 17:00:08.216 | INFO | "malloc" : " 14 GB", | 20260506 17:00:08.216 | INFO | "pinned" : " 29 GB" | 20260506 17:00:08.216 | INFO | } | 20260506 17:00:08.216 | INFO | Perf report for the last 5.14313 seconds: | 20260506 17:00:08.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:00:08.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:08.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:08.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:08.216 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260506 17:00:08.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:08.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:08.216 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:00:08.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:08.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:08.217 | INFO | Uploaded 36GiB to GPU in 1.95596s (18.4053GiB/s) | 20260506 17:00:08.217 | INFO | Downloaded 34GiB to GPU in 1.88136s (18.0721GiB/s) | 20260506 17:00:13.356 | INFO | Analyzing chunk frames = [2980352,2980864) | 20260506 17:00:13.356 | INFO | MemInfo: { | 20260506 17:00:13.356 | INFO | "gpu" : " 17 GB", | 20260506 17:00:13.356 | INFO | "huge" : "", | 20260506 17:00:13.356 | INFO | "huge_shared" : " 27 GB", | 20260506 17:00:13.356 | INFO | "malloc" : " 14 GB", | 20260506 17:00:13.356 | INFO | "pinned" : " 29 GB" | 20260506 17:00:13.356 | INFO | } | 20260506 17:00:13.357 | INFO | Perf report for the last 5.14031 seconds: | 20260506 17:00:13.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:00:13.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:13.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:13.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:13.357 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 17:00:13.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:13.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:13.357 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:00:13.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:13.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:13.357 | INFO | Uploaded 36GiB to GPU in 2.00641s (17.9425GiB/s) | 20260506 17:00:13.357 | INFO | Downloaded 34GiB to GPU in 1.94609s (17.471GiB/s) | 20260506 17:00:18.499 | INFO | Analyzing chunk frames = [2980864,2981376) | 20260506 17:00:18.499 | INFO | MemInfo: { | 20260506 17:00:18.499 | INFO | "gpu" : " 17 GB", | 20260506 17:00:18.499 | INFO | "huge" : "", | 20260506 17:00:18.499 | INFO | "huge_shared" : " 27 GB", | 20260506 17:00:18.499 | INFO | "malloc" : " 14 GB", | 20260506 17:00:18.499 | INFO | "pinned" : " 29 GB" | 20260506 17:00:18.499 | INFO | } | 20260506 17:00:18.500 | INFO | Perf report for the last 5.14291 seconds: | 20260506 17:00:18.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:00:18.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:18.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:18.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:18.500 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 17:00:18.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:18.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:18.500 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 17:00:18.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:18.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:18.500 | INFO | Uploaded 36GiB to GPU in 2.02811s (17.7505GiB/s) | 20260506 17:00:18.500 | INFO | Downloaded 34GiB to GPU in 1.99734s (17.0227GiB/s) | 20260506 17:00:23.634 | INFO | Analyzing chunk frames = [2981376,2981888) | 20260506 17:00:23.634 | INFO | MemInfo: { | 20260506 17:00:23.634 | INFO | "gpu" : " 17 GB", | 20260506 17:00:23.634 | INFO | "huge" : "", | 20260506 17:00:23.634 | INFO | "huge_shared" : " 27 GB", | 20260506 17:00:23.634 | INFO | "malloc" : " 14 GB", | 20260506 17:00:23.634 | INFO | "pinned" : " 29 GB" | 20260506 17:00:23.634 | INFO | } | 20260506 17:00:23.634 | INFO | Perf report for the last 5.13442 seconds: | 20260506 17:00:23.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:00:23.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:23.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:23.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:23.634 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 17:00:23.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:23.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:23.634 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 17:00:23.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:23.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:23.634 | INFO | Uploaded 36GiB to GPU in 2.04572s (17.5977GiB/s) | 20260506 17:00:23.634 | INFO | Downloaded 34GiB to GPU in 1.98494s (17.129GiB/s) | 20260506 17:00:23.635 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:00:23.635 | INFO | Total mem usage: 18 GiB | 20260506 17:00:23.635 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:00:23.635 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.5%) | 20260506 17:00:23.635 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:00:23.635 | INFO | High Water Breakdown: | 20260506 17:00:23.635 | INFO | 61.7% : 9.88 GiB ( 86.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:00:23.635 | INFO | 38.2% : 6.11 GiB ( 91.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:00:23.635 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:00:23.635 | INFO | ------------------------------------------ | 20260506 17:00:23.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:00:23.635 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:00:23.635 | INFO | Total mem usage: 26.5 GiB | 20260506 17:00:23.635 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:00:23.635 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:00:23.635 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:00:23.635 | INFO | High Water Breakdown: | 20260506 17:00:23.635 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:00:23.635 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:00:23.635 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:00:23.635 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:00:23.635 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:00:23.635 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:00:23.635 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:00:23.635 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:00:23.635 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:00:23.635 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:00:23.635 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:00:23.635 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:00:23.635 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:00:23.635 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:00:23.635 | INFO | ------------------------------------------ | 20260506 17:00:23.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:00:23.635 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:00:23.635 | INFO | Total mem usage: 25.6 GiB | 20260506 17:00:23.635 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:00:23.635 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:00:23.635 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:00:23.635 | INFO | High Water Breakdown: | 20260506 17:00:23.635 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:00:23.635 | INFO | ------------------------------------------ | 20260506 17:00:23.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:00:23.635 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:00:23.635 | INFO | Total mem usage: 15.7 GiB | 20260506 17:00:23.635 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:00:23.635 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:00:23.635 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:00:23.635 | INFO | High Water Breakdown: | 20260506 17:00:23.635 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:00:23.635 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:00:23.635 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:00:23.635 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:00:23.635 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:00:23.635 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:00:23.635 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:00:23.635 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:00:23.635 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:00:23.635 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:00:23.635 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:00:23.635 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:00:23.636 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:00:23.636 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:00:23.636 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:00:23.636 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:00:23.636 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:00:23.636 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:00:23.636 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:00:23.636 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:00:23.636 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:00:23.636 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:00:23.636 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:00:23.636 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:00:23.636 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:00:23.636 | INFO | ------------------------------------------ | 20260506 17:00:23.636 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:00:23.636 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:00:23.636 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:00:23.636 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:00:23.636 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.53e+01, 1.35e+01 / 2.36e+00 | 20260506 17:00:23.636 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 1.02e-02 / 5.50e-03 | 20260506 17:00:23.636 | INFO | PerfReport: 11.59%, Baseline -- 3.63e+00 / 4.40e+00, 5.57e-01 / 3.52e-01 | 20260506 17:00:23.636 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.44e+00, 1.83e-01 / 1.52e-01 | 20260506 17:00:23.636 | INFO | PerfReport: 15.92%, DME -- 3.99e+01 / 4.43e+01, 3.95e+00 / 2.64e+00 | 20260506 17:00:23.636 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.79e+01 / 2.02e+01, 1.49e+00 / 2.01e+00 | 20260506 17:00:23.636 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.28e+00 / 2.87e+00, 3.87e-01 / 2.42e-01 | 20260506 17:00:23.636 | INFO | PerfReport: 3.48%, Metrics -- 1.09e+00 / 1.44e+00, 3.37e-01 / 1.59e-01 | 20260506 17:00:23.636 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.77e-01 / 4.26e-01, 5.91e-02 / 2.00e-02 | 20260506 17:00:23.636 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:00:23.636 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:00:23.636 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:00:23.636 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 17:00:23.636 | INFO | PerfReport: 21.49%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 17:00:23.636 | INFO | PerfReport: 21.11%, Download -- 2.05e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 17:00:23.636 | INFO | PerfReport: 32.27%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 17:00:23.636 | INFO | PerfReport: 1.42%, LockUpload -- 1.38e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260506 17:00:23.636 | INFO | PerfReport: 1.07%, LockDownload -- 1.04e+00 / 0.00e+00, 2.62e+00 / 0.00e+00 | 20260506 17:00:23.637 | INFO | PerfReport: 22.63%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 17:00:23.637 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.12e-02 / 0.00e+00 | 20260506 17:00:23.748 | INFO | Starting a round of Baz IO | 20260506 17:00:26.888 | INFO | Populating RTMetrics report | 20260506 17:00:26.888 | INFO | RTMetrics report populated | 20260506 17:00:26.888 | INFO | Opening tmp RTMetrics json file | 20260506 17:00:27.034 | INFO | Serializing RTMetrics to json | 20260506 17:00:27.098 | INFO | Serialization done, swapping tmp file now... | 20260506 17:00:27.131 | INFO | RTMetrics done | 20260506 17:00:27.480 | INFO | Finished a round of Baz IO: Wrote 2.074 GiB in 3.732 seconds | 20260506 17:00:27.480 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.93% / 0.01% / 64.06% | 20260506 17:00:27.480 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.5022% / 1.19209e-05% / 5.16574e-05% | 20260506 17:00:27.480 | INFO | Baz IO overall: | 20260506 17:00:27.480 | INFO | BytesWritten : 1569377727219 | 20260506 17:00:27.480 | INFO | NumEvents : 759332982214 | 20260506 17:00:28.772 | INFO | Analyzing chunk frames = [2981888,2982400) | 20260506 17:00:28.772 | INFO | MemInfo: { | 20260506 17:00:28.772 | INFO | "gpu" : " 17 GB", | 20260506 17:00:28.772 | INFO | "huge" : "", | 20260506 17:00:28.772 | INFO | "huge_shared" : " 27 GB", | 20260506 17:00:28.772 | INFO | "malloc" : " 14 GB", | 20260506 17:00:28.772 | INFO | "pinned" : " 29 GB" | 20260506 17:00:28.772 | INFO | } | 20260506 17:00:28.772 | INFO | Perf report for the last 5.13813 seconds: | 20260506 17:00:28.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:00:28.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:28.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:28.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:28.772 | INFO | ANALYSIS : 62.8%, 2.9, 95.4ms, 0 | 20260506 17:00:28.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:28.772 | INFO | RT_METRICS : 48.6%, 1.0, 25.5ms, 0 | 20260506 17:00:28.772 | INFO | PRE_HQ : 60.8%, 1.9, 58.8ms, 0 | 20260506 17:00:28.772 | INFO | BAZWRITER : 72.0%, 4.7, 174.8ms, 0 | 20260506 17:00:28.772 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 17:00:28.772 | INFO | Uploaded 36GiB to GPU in 2.07093s (17.3835GiB/s) | 20260506 17:00:28.773 | INFO | Downloaded 38GiB to GPU in 2.22603s (17.0707GiB/s) | 20260506 17:00:33.911 | INFO | Analyzing chunk frames = [2982400,2982912) | 20260506 17:00:33.911 | INFO | MemInfo: { | 20260506 17:00:33.911 | INFO | "gpu" : " 17 GB", | 20260506 17:00:33.911 | INFO | "huge" : "", | 20260506 17:00:33.911 | INFO | "huge_shared" : " 27 GB", | 20260506 17:00:33.911 | INFO | "malloc" : " 14 GB", | 20260506 17:00:33.911 | INFO | "pinned" : " 29 GB" | 20260506 17:00:33.911 | INFO | } | 20260506 17:00:33.911 | INFO | Perf report for the last 5.13918 seconds: | 20260506 17:00:33.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:00:33.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:33.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:33.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:33.911 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:00:33.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:33.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:33.912 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 17:00:33.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:33.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:33.912 | INFO | Uploaded 36GiB to GPU in 1.99847s (18.0138GiB/s) | 20260506 17:00:33.912 | INFO | Downloaded 34GiB to GPU in 1.94375s (17.492GiB/s) | 20260506 17:00:39.047 | INFO | Analyzing chunk frames = [2982912,2983424) | 20260506 17:00:39.047 | INFO | MemInfo: { | 20260506 17:00:39.047 | INFO | "gpu" : " 17 GB", | 20260506 17:00:39.047 | INFO | "huge" : "", | 20260506 17:00:39.047 | INFO | "huge_shared" : " 27 GB", | 20260506 17:00:39.047 | INFO | "malloc" : " 14 GB", | 20260506 17:00:39.048 | INFO | "pinned" : " 29 GB" | 20260506 17:00:39.048 | INFO | } | 20260506 17:00:39.048 | INFO | Perf report for the last 5.13628 seconds: | 20260506 17:00:39.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:00:39.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:39.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:39.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:39.048 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 17:00:39.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:39.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:39.048 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:00:39.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:39.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:39.048 | INFO | Uploaded 36GiB to GPU in 1.97761s (18.2038GiB/s) | 20260506 17:00:39.048 | INFO | Downloaded 34GiB to GPU in 1.92713s (17.6428GiB/s) | 20260506 17:00:44.192 | INFO | Analyzing chunk frames = [2983424,2983936) | 20260506 17:00:44.192 | INFO | MemInfo: { | 20260506 17:00:44.192 | INFO | "gpu" : " 17 GB", | 20260506 17:00:44.192 | INFO | "huge" : "", | 20260506 17:00:44.192 | INFO | "huge_shared" : " 27 GB", | 20260506 17:00:44.192 | INFO | "malloc" : " 14 GB", | 20260506 17:00:44.192 | INFO | "pinned" : " 29 GB" | 20260506 17:00:44.192 | INFO | } | 20260506 17:00:44.192 | INFO | Perf report for the last 5.14433 seconds: | 20260506 17:00:44.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:00:44.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:44.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:44.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:44.192 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 17:00:44.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:44.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:44.192 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:00:44.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:44.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:44.192 | INFO | Uploaded 36GiB to GPU in 1.93924s (18.5639GiB/s) | 20260506 17:00:44.192 | INFO | Downloaded 34GiB to GPU in 1.88127s (18.0729GiB/s) | 20260506 17:00:49.331 | INFO | Analyzing chunk frames = [2983936,2984448) | 20260506 17:00:49.331 | INFO | MemInfo: { | 20260506 17:00:49.331 | INFO | "gpu" : " 17 GB", | 20260506 17:00:49.331 | INFO | "huge" : "", | 20260506 17:00:49.331 | INFO | "huge_shared" : " 27 GB", | 20260506 17:00:49.331 | INFO | "malloc" : " 14 GB", | 20260506 17:00:49.331 | INFO | "pinned" : " 29 GB" | 20260506 17:00:49.331 | INFO | } | 20260506 17:00:49.331 | INFO | Perf report for the last 5.13921 seconds: | 20260506 17:00:49.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:00:49.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:49.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:49.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:49.331 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 17:00:49.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:49.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:49.331 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:00:49.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:49.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:49.332 | INFO | Uploaded 36GiB to GPU in 2.01079s (17.9034GiB/s) | 20260506 17:00:49.332 | INFO | Downloaded 34GiB to GPU in 1.90542s (17.8439GiB/s) | 20260506 17:00:54.470 | INFO | Analyzing chunk frames = [2984448,2984960) | 20260506 17:00:54.470 | INFO | MemInfo: { | 20260506 17:00:54.470 | INFO | "gpu" : " 17 GB", | 20260506 17:00:54.470 | INFO | "huge" : "", | 20260506 17:00:54.470 | INFO | "huge_shared" : " 27 GB", | 20260506 17:00:54.470 | INFO | "malloc" : " 14 GB", | 20260506 17:00:54.470 | INFO | "pinned" : " 29 GB" | 20260506 17:00:54.470 | INFO | } | 20260506 17:00:54.470 | INFO | Perf report for the last 5.1392 seconds: | 20260506 17:00:54.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:00:54.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:54.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:54.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:54.470 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 17:00:54.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:54.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:54.470 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:00:54.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:54.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:54.471 | INFO | Uploaded 36GiB to GPU in 2.03201s (17.7164GiB/s) | 20260506 17:00:54.471 | INFO | Downloaded 34GiB to GPU in 1.93925s (17.5326GiB/s) | 20260506 17:00:59.612 | INFO | Analyzing chunk frames = [2984960,2985472) | 20260506 17:00:59.612 | INFO | MemInfo: { | 20260506 17:00:59.612 | INFO | "gpu" : " 17 GB", | 20260506 17:00:59.612 | INFO | "huge" : "", | 20260506 17:00:59.612 | INFO | "huge_shared" : " 27 GB", | 20260506 17:00:59.612 | INFO | "malloc" : " 14 GB", | 20260506 17:00:59.612 | INFO | "pinned" : " 29 GB" | 20260506 17:00:59.612 | INFO | } | 20260506 17:00:59.612 | INFO | Perf report for the last 5.14197 seconds: | 20260506 17:00:59.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:00:59.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:59.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:59.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:59.612 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 17:00:59.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:59.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:59.612 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 17:00:59.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:59.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:00:59.613 | INFO | Uploaded 36GiB to GPU in 2.03147s (17.7211GiB/s) | 20260506 17:00:59.613 | INFO | Downloaded 34GiB to GPU in 1.99273s (17.062GiB/s) | 20260506 17:01:04.749 | INFO | Analyzing chunk frames = [2985472,2985984) | 20260506 17:01:04.749 | INFO | MemInfo: { | 20260506 17:01:04.749 | INFO | "gpu" : " 17 GB", | 20260506 17:01:04.749 | INFO | "huge" : "", | 20260506 17:01:04.750 | INFO | "huge_shared" : " 27 GB", | 20260506 17:01:04.750 | INFO | "malloc" : " 14 GB", | 20260506 17:01:04.750 | INFO | "pinned" : " 29 GB" | 20260506 17:01:04.750 | INFO | } | 20260506 17:01:04.750 | INFO | Perf report for the last 5.13732 seconds: | 20260506 17:01:04.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:01:04.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:04.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:04.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:04.750 | INFO | ANALYSIS : 62.5%, 3.0, 97.3ms, 0 | 20260506 17:01:04.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:04.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:04.750 | INFO | PRE_HQ : 32.6%, 1.0, 17.1ms, 0 | 20260506 17:01:04.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:04.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:04.750 | INFO | Uploaded 36GiB to GPU in 2.10038s (17.1398GiB/s) | 20260506 17:01:04.750 | INFO | Downloaded 34GiB to GPU in 2.01731s (16.8541GiB/s) | 20260506 17:01:04.881 | INFO | Starting a round of Baz IO | 20260506 17:01:08.021 | INFO | Populating RTMetrics report | 20260506 17:01:08.021 | INFO | RTMetrics report populated | 20260506 17:01:08.022 | INFO | Opening tmp RTMetrics json file | 20260506 17:01:08.025 | INFO | Serializing RTMetrics to json | 20260506 17:01:08.027 | INFO | Serialization done, swapping tmp file now... | 20260506 17:01:08.031 | INFO | RTMetrics done | 20260506 17:01:08.414 | INFO | Finished a round of Baz IO: Wrote 2.074 GiB in 3.533 seconds | 20260506 17:01:08.414 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.92% / 0.01% / 64.07% | 20260506 17:01:08.414 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.49595% / 3.97364e-06% / 5.96046e-05% | 20260506 17:01:08.414 | INFO | Baz IO overall: | 20260506 17:01:08.414 | INFO | BytesWritten : 1571605032945 | 20260506 17:01:08.414 | INFO | NumEvents : 760408517039 | 20260506 17:01:09.889 | INFO | Analyzing chunk frames = [2985984,2986496) | 20260506 17:01:09.889 | INFO | MemInfo: { | 20260506 17:01:09.889 | INFO | "gpu" : " 17 GB", | 20260506 17:01:09.889 | INFO | "huge" : "", | 20260506 17:01:09.889 | INFO | "huge_shared" : " 27 GB", | 20260506 17:01:09.889 | INFO | "malloc" : " 14 GB", | 20260506 17:01:09.889 | INFO | "pinned" : " 29 GB" | 20260506 17:01:09.889 | INFO | } | 20260506 17:01:09.889 | INFO | Perf report for the last 5.1396 seconds: | 20260506 17:01:09.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:01:09.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:09.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:09.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:09.889 | INFO | ANALYSIS : 63.1%, 2.9, 95.8ms, 0 | 20260506 17:01:09.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:09.889 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 17:01:09.889 | INFO | PRE_HQ : 60.7%, 1.9, 59.1ms, 0 | 20260506 17:01:09.889 | INFO | BAZWRITER : 68.1%, 4.8, 168.9ms, 0 | 20260506 17:01:09.889 | INFO | BAZ_SAMPLING : 6.8%, 1.0, 3.6ms, 0 | 20260506 17:01:09.890 | INFO | Uploaded 36GiB to GPU in 2.06904s (17.3994GiB/s) | 20260506 17:01:09.890 | INFO | Downloaded 38GiB to GPU in 2.24077s (16.9585GiB/s) | 20260506 17:01:15.024 | INFO | Analyzing chunk frames = [2986496,2987008) | 20260506 17:01:15.024 | INFO | MemInfo: { | 20260506 17:01:15.024 | INFO | "gpu" : " 17 GB", | 20260506 17:01:15.024 | INFO | "huge" : "", | 20260506 17:01:15.024 | INFO | "huge_shared" : " 27 GB", | 20260506 17:01:15.024 | INFO | "malloc" : " 14 GB", | 20260506 17:01:15.024 | INFO | "pinned" : " 29 GB" | 20260506 17:01:15.024 | INFO | } | 20260506 17:01:15.024 | INFO | Perf report for the last 5.13495 seconds: | 20260506 17:01:15.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:01:15.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:15.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:15.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:15.024 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:01:15.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:15.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:15.024 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 17:01:15.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:15.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:15.024 | INFO | Uploaded 36GiB to GPU in 1.99879s (18.0109GiB/s) | 20260506 17:01:15.025 | INFO | Downloaded 34GiB to GPU in 1.93507s (17.5704GiB/s) | 20260506 17:01:20.161 | INFO | Analyzing chunk frames = [2987008,2987520) | 20260506 17:01:20.161 | INFO | MemInfo: { | 20260506 17:01:20.161 | INFO | "gpu" : " 17 GB", | 20260506 17:01:20.161 | INFO | "huge" : "", | 20260506 17:01:20.161 | INFO | "huge_shared" : " 27 GB", | 20260506 17:01:20.161 | INFO | "malloc" : " 14 GB", | 20260506 17:01:20.161 | INFO | "pinned" : " 29 GB" | 20260506 17:01:20.161 | INFO | } | 20260506 17:01:20.162 | INFO | Perf report for the last 5.1373 seconds: | 20260506 17:01:20.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:01:20.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:20.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:20.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:20.162 | INFO | ANALYSIS : 61.2%, 3.0, 94.8ms, 0 | 20260506 17:01:20.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:20.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:20.162 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:01:20.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:20.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:20.162 | INFO | Uploaded 36GiB to GPU in 1.99627s (18.0336GiB/s) | 20260506 17:01:20.162 | INFO | Downloaded 34GiB to GPU in 1.93489s (17.572GiB/s) | 20260506 17:01:25.307 | INFO | Analyzing chunk frames = [2987520,2988032) | 20260506 17:01:25.307 | INFO | MemInfo: { | 20260506 17:01:25.307 | INFO | "gpu" : " 17 GB", | 20260506 17:01:25.307 | INFO | "huge" : "", | 20260506 17:01:25.307 | INFO | "huge_shared" : " 27 GB", | 20260506 17:01:25.307 | INFO | "malloc" : " 14 GB", | 20260506 17:01:25.307 | INFO | "pinned" : " 29 GB" | 20260506 17:01:25.307 | INFO | } | 20260506 17:01:25.307 | INFO | Perf report for the last 5.1455 seconds: | 20260506 17:01:25.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:01:25.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:25.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:25.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:25.307 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260506 17:01:25.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:25.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:25.307 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:01:25.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:25.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:25.307 | INFO | Uploaded 36GiB to GPU in 1.93435s (18.6109GiB/s) | 20260506 17:01:25.307 | INFO | Downloaded 34GiB to GPU in 1.88155s (18.0702GiB/s) | 20260506 17:01:30.448 | INFO | Analyzing chunk frames = [2988032,2988544) | 20260506 17:01:30.448 | INFO | MemInfo: { | 20260506 17:01:30.448 | INFO | "gpu" : " 17 GB", | 20260506 17:01:30.448 | INFO | "huge" : "", | 20260506 17:01:30.448 | INFO | "huge_shared" : " 27 GB", | 20260506 17:01:30.448 | INFO | "malloc" : " 14 GB", | 20260506 17:01:30.448 | INFO | "pinned" : " 29 GB" | 20260506 17:01:30.448 | INFO | } | 20260506 17:01:30.449 | INFO | Perf report for the last 5.14161 seconds: | 20260506 17:01:30.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:01:30.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:30.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:30.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:30.449 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 17:01:30.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:30.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:30.449 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:01:30.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:30.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:30.449 | INFO | Uploaded 36GiB to GPU in 1.98697s (18.118GiB/s) | 20260506 17:01:30.449 | INFO | Downloaded 34GiB to GPU in 1.91735s (17.7328GiB/s) | 20260506 17:01:35.592 | INFO | Analyzing chunk frames = [2988544,2989056) | 20260506 17:01:35.592 | INFO | MemInfo: { | 20260506 17:01:35.592 | INFO | "gpu" : " 17 GB", | 20260506 17:01:35.592 | INFO | "huge" : "", | 20260506 17:01:35.592 | INFO | "huge_shared" : " 27 GB", | 20260506 17:01:35.592 | INFO | "malloc" : " 14 GB", | 20260506 17:01:35.592 | INFO | "pinned" : " 29 GB" | 20260506 17:01:35.592 | INFO | } | 20260506 17:01:35.593 | INFO | Perf report for the last 5.14382 seconds: | 20260506 17:01:35.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:01:35.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:35.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:35.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:35.593 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 17:01:35.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:35.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:35.593 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:01:35.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:35.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:35.593 | INFO | Uploaded 36GiB to GPU in 2.01919s (17.8289GiB/s) | 20260506 17:01:35.593 | INFO | Downloaded 34GiB to GPU in 1.94388s (17.4908GiB/s) | 20260506 17:01:40.726 | INFO | Analyzing chunk frames = [2989056,2989568) | 20260506 17:01:40.726 | INFO | MemInfo: { | 20260506 17:01:40.726 | INFO | "gpu" : " 17 GB", | 20260506 17:01:40.726 | INFO | "huge" : "", | 20260506 17:01:40.726 | INFO | "huge_shared" : " 27 GB", | 20260506 17:01:40.726 | INFO | "malloc" : " 14 GB", | 20260506 17:01:40.726 | INFO | "pinned" : " 29 GB" | 20260506 17:01:40.726 | INFO | } | 20260506 17:01:40.727 | INFO | Perf report for the last 5.13418 seconds: | 20260506 17:01:40.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:01:40.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:40.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:40.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:40.727 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 17:01:40.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:40.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:40.727 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260506 17:01:40.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:40.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:40.727 | INFO | Uploaded 36GiB to GPU in 2.0104s (17.9069GiB/s) | 20260506 17:01:40.727 | INFO | Downloaded 34GiB to GPU in 1.98557s (17.1236GiB/s) | 20260506 17:01:45.862 | INFO | Analyzing chunk frames = [2989568,2990080) | 20260506 17:01:45.862 | INFO | MemInfo: { | 20260506 17:01:45.862 | INFO | "gpu" : " 17 GB", | 20260506 17:01:45.862 | INFO | "huge" : "", | 20260506 17:01:45.862 | INFO | "huge_shared" : " 27 GB", | 20260506 17:01:45.862 | INFO | "malloc" : " 14 GB", | 20260506 17:01:45.862 | INFO | "pinned" : " 29 GB" | 20260506 17:01:45.863 | INFO | } | 20260506 17:01:45.863 | INFO | Perf report for the last 5.13604 seconds: | 20260506 17:01:45.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:01:45.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:45.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:45.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:45.863 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:01:45.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:45.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:45.863 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 17:01:45.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:45.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:45.863 | INFO | Uploaded 36GiB to GPU in 2.00297s (17.9733GiB/s) | 20260506 17:01:45.863 | INFO | Downloaded 34GiB to GPU in 1.96999s (17.259GiB/s) | 20260506 17:01:45.863 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:01:45.863 | INFO | Total mem usage: 18 GiB | 20260506 17:01:45.863 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:01:45.863 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -20.8%) | 20260506 17:01:45.863 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:01:45.863 | INFO | High Water Breakdown: | 20260506 17:01:45.863 | INFO | 60.6% : 9.02 GiB ( 79.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:01:45.863 | INFO | 39.3% : 5.86 GiB ( 87.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:01:45.863 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:01:45.863 | INFO | ------------------------------------------ | 20260506 17:01:45.863 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:01:45.863 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:01:45.863 | INFO | Total mem usage: 26.5 GiB | 20260506 17:01:45.864 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:01:45.864 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:01:45.864 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:01:45.864 | INFO | High Water Breakdown: | 20260506 17:01:45.864 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:01:45.864 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:01:45.864 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:01:45.864 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:01:45.864 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:01:45.864 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:01:45.864 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:01:45.864 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:01:45.864 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:01:45.864 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:01:45.864 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:01:45.864 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:01:45.864 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:01:45.864 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:01:45.864 | INFO | ------------------------------------------ | 20260506 17:01:45.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:01:45.864 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:01:45.864 | INFO | Total mem usage: 25.6 GiB | 20260506 17:01:45.864 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:01:45.864 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:01:45.864 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:01:45.864 | INFO | High Water Breakdown: | 20260506 17:01:45.864 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:01:45.864 | INFO | ------------------------------------------ | 20260506 17:01:45.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:01:45.864 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:01:45.864 | INFO | Total mem usage: 15.7 GiB | 20260506 17:01:45.864 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:01:45.864 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:01:45.864 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:01:45.864 | INFO | High Water Breakdown: | 20260506 17:01:45.864 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:01:45.864 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:01:45.864 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:01:45.864 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:01:45.864 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:01:45.864 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:01:45.864 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:01:45.864 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:01:45.864 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:01:45.864 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:01:45.864 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:01:45.864 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:01:45.864 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:01:45.864 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:01:45.864 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:01:45.864 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:01:45.864 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:01:45.864 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:01:45.864 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:01:45.864 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:01:45.864 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:01:45.864 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:01:45.864 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:01:45.864 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:01:45.864 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:01:45.864 | INFO | ------------------------------------------ | 20260506 17:01:45.864 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:01:45.864 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:01:45.864 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:01:45.864 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:01:45.864 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.56e+01, 1.35e+01 / 2.22e+00 | 20260506 17:01:45.865 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 1.02e-02 / 7.28e-03 | 20260506 17:01:45.865 | INFO | PerfReport: 11.61%, Baseline -- 3.65e+00 / 4.38e+00, 5.63e-01 / 3.38e-01 | 20260506 17:01:45.865 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.87e-01 / 1.23e-01 | 20260506 17:01:45.865 | INFO | PerfReport: 15.92%, DME -- 4.00e+01 / 4.44e+01, 4.03e+00 / 2.45e+00 | 20260506 17:01:45.865 | INFO | PerfReport: 56.97%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.46e+00 / 1.78e+00 | 20260506 17:01:45.865 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.28e+00 / 2.83e+00, 3.83e-01 / 2.15e-01 | 20260506 17:01:45.865 | INFO | PerfReport: 3.46%, Metrics -- 1.09e+00 / 1.42e+00, 2.81e-01 / 1.78e-01 | 20260506 17:01:45.865 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.77e-01 / 4.24e-01, 5.88e-02 / 2.25e-02 | 20260506 17:01:45.865 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:01:45.865 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:01:45.865 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:01:45.865 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 17:01:45.865 | INFO | PerfReport: 21.52%, Upload -- 2.10e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 17:01:45.865 | INFO | PerfReport: 21.16%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 17:01:45.865 | INFO | PerfReport: 32.24%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 17:01:45.865 | INFO | PerfReport: 1.48%, LockUpload -- 1.44e+00 / 0.00e+00, 5.75e+00 / 0.00e+00 | 20260506 17:01:45.865 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.70e+00 / 0.00e+00 | 20260506 17:01:45.865 | INFO | PerfReport: 22.50%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 17:01:45.865 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260506 17:01:45.977 | INFO | Starting a round of Baz IO | 20260506 17:01:49.114 | INFO | Populating RTMetrics report | 20260506 17:01:49.115 | INFO | RTMetrics report populated | 20260506 17:01:49.115 | INFO | Opening tmp RTMetrics json file | 20260506 17:01:49.118 | INFO | Serializing RTMetrics to json | 20260506 17:01:49.121 | INFO | Serialization done, swapping tmp file now... | 20260506 17:01:49.124 | INFO | RTMetrics done | 20260506 17:01:49.275 | INFO | Finished a round of Baz IO: Wrote 2.073 GiB in 3.297 seconds | 20260506 17:01:49.275 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.92% / 0.01% / 64.08% | 20260506 17:01:49.275 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.49284% / 7.94729e-06% / 5.16574e-05% | 20260506 17:01:49.275 | INFO | Baz IO overall: | 20260506 17:01:49.275 | INFO | BytesWritten : 1573831050083 | 20260506 17:01:49.275 | INFO | NumEvents : 761483326301 | 20260506 17:01:50.999 | INFO | Analyzing chunk frames = [2990080,2990592) | 20260506 17:01:50.999 | INFO | MemInfo: { | 20260506 17:01:50.999 | INFO | "gpu" : " 17 GB", | 20260506 17:01:50.999 | INFO | "huge" : "", | 20260506 17:01:50.999 | INFO | "huge_shared" : " 27 GB", | 20260506 17:01:50.999 | INFO | "malloc" : " 14 GB", | 20260506 17:01:50.999 | INFO | "pinned" : " 29 GB" | 20260506 17:01:50.999 | INFO | } | 20260506 17:01:50.999 | INFO | Perf report for the last 5.13598 seconds: | 20260506 17:01:50.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:01:50.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:50.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:50.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:50.999 | INFO | ANALYSIS : 62.7%, 2.9, 95.1ms, 0 | 20260506 17:01:50.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:50.999 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 17:01:50.999 | INFO | PRE_HQ : 60.4%, 1.9, 58.4ms, 0 | 20260506 17:01:50.999 | INFO | BAZWRITER : 63.8%, 4.8, 157.0ms, 0 | 20260506 17:01:50.999 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 17:01:50.999 | INFO | Uploaded 36GiB to GPU in 2.05284s (17.5367GiB/s) | 20260506 17:01:50.999 | INFO | Downloaded 38GiB to GPU in 2.20881s (17.2039GiB/s) | 20260506 17:01:56.137 | INFO | Analyzing chunk frames = [2990592,2991104) | 20260506 17:01:56.137 | INFO | MemInfo: { | 20260506 17:01:56.137 | INFO | "gpu" : " 17 GB", | 20260506 17:01:56.137 | INFO | "huge" : "", | 20260506 17:01:56.137 | INFO | "huge_shared" : " 27 GB", | 20260506 17:01:56.137 | INFO | "malloc" : " 14 GB", | 20260506 17:01:56.137 | INFO | "pinned" : " 29 GB" | 20260506 17:01:56.137 | INFO | } | 20260506 17:01:56.137 | INFO | Perf report for the last 5.13843 seconds: | 20260506 17:01:56.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:01:56.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:56.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:56.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:56.137 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:01:56.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:56.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:56.137 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:01:56.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:56.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:01:56.137 | INFO | Uploaded 36GiB to GPU in 1.99785s (18.0193GiB/s) | 20260506 17:01:56.138 | INFO | Downloaded 34GiB to GPU in 1.93495s (17.5716GiB/s) | 20260506 17:02:01.280 | INFO | Analyzing chunk frames = [2991104,2991616) | 20260506 17:02:01.280 | INFO | MemInfo: { | 20260506 17:02:01.280 | INFO | "gpu" : " 17 GB", | 20260506 17:02:01.280 | INFO | "huge" : "", | 20260506 17:02:01.280 | INFO | "huge_shared" : " 27 GB", | 20260506 17:02:01.280 | INFO | "malloc" : " 14 GB", | 20260506 17:02:01.280 | INFO | "pinned" : " 29 GB" | 20260506 17:02:01.280 | INFO | } | 20260506 17:02:01.280 | INFO | Perf report for the last 5.14242 seconds: | 20260506 17:02:01.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:02:01.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:01.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:01.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:01.280 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 17:02:01.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:01.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:01.280 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:02:01.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:01.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:01.280 | INFO | Uploaded 36GiB to GPU in 1.94148s (18.5426GiB/s) | 20260506 17:02:01.280 | INFO | Downloaded 34GiB to GPU in 1.88961s (17.9931GiB/s) | 20260506 17:02:06.424 | INFO | Analyzing chunk frames = [2991616,2992128) | 20260506 17:02:06.424 | INFO | MemInfo: { | 20260506 17:02:06.424 | INFO | "gpu" : " 17 GB", | 20260506 17:02:06.424 | INFO | "huge" : "", | 20260506 17:02:06.424 | INFO | "huge_shared" : " 27 GB", | 20260506 17:02:06.424 | INFO | "malloc" : " 14 GB", | 20260506 17:02:06.424 | INFO | "pinned" : " 29 GB" | 20260506 17:02:06.424 | INFO | } | 20260506 17:02:06.424 | INFO | Perf report for the last 5.14476 seconds: | 20260506 17:02:06.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:02:06.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:06.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:06.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:06.425 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 17:02:06.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:06.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:06.425 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:02:06.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:06.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:06.425 | INFO | Uploaded 36GiB to GPU in 1.93788s (18.577GiB/s) | 20260506 17:02:06.425 | INFO | Downloaded 34GiB to GPU in 1.87312s (18.1515GiB/s) | 20260506 17:02:11.565 | INFO | Analyzing chunk frames = [2992128,2992640) | 20260506 17:02:11.565 | INFO | MemInfo: { | 20260506 17:02:11.565 | INFO | "gpu" : " 17 GB", | 20260506 17:02:11.565 | INFO | "huge" : "", | 20260506 17:02:11.565 | INFO | "huge_shared" : " 27 GB", | 20260506 17:02:11.565 | INFO | "malloc" : " 14 GB", | 20260506 17:02:11.565 | INFO | "pinned" : " 29 GB" | 20260506 17:02:11.565 | INFO | } | 20260506 17:02:11.565 | INFO | Perf report for the last 5.14057 seconds: | 20260506 17:02:11.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:02:11.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:11.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:11.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:11.565 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 17:02:11.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:11.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:11.565 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:02:11.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:11.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:11.565 | INFO | Uploaded 36GiB to GPU in 1.97591s (18.2194GiB/s) | 20260506 17:02:11.565 | INFO | Downloaded 34GiB to GPU in 1.92255s (17.6849GiB/s) | 20260506 17:02:16.701 | INFO | Analyzing chunk frames = [2992640,2993152) | 20260506 17:02:16.701 | INFO | MemInfo: { | 20260506 17:02:16.701 | INFO | "gpu" : " 17 GB", | 20260506 17:02:16.701 | INFO | "huge" : "", | 20260506 17:02:16.701 | INFO | "huge_shared" : " 27 GB", | 20260506 17:02:16.701 | INFO | "malloc" : " 14 GB", | 20260506 17:02:16.701 | INFO | "pinned" : " 29 GB" | 20260506 17:02:16.701 | INFO | } | 20260506 17:02:16.701 | INFO | Perf report for the last 5.1365 seconds: | 20260506 17:02:16.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:02:16.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:16.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:16.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:16.702 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 17:02:16.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:16.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:16.702 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 17:02:16.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:16.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:16.702 | INFO | Uploaded 36GiB to GPU in 2.03149s (17.721GiB/s) | 20260506 17:02:16.702 | INFO | Downloaded 34GiB to GPU in 1.94855s (17.4489GiB/s) | 20260506 17:02:21.840 | INFO | Analyzing chunk frames = [2993152,2993664) | 20260506 17:02:21.840 | INFO | MemInfo: { | 20260506 17:02:21.840 | INFO | "gpu" : " 17 GB", | 20260506 17:02:21.840 | INFO | "huge" : "", | 20260506 17:02:21.840 | INFO | "huge_shared" : " 27 GB", | 20260506 17:02:21.840 | INFO | "malloc" : " 14 GB", | 20260506 17:02:21.840 | INFO | "pinned" : " 29 GB" | 20260506 17:02:21.840 | INFO | } | 20260506 17:02:21.840 | INFO | Perf report for the last 5.1384 seconds: | 20260506 17:02:21.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:02:21.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:21.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:21.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:21.840 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 17:02:21.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:21.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:21.840 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:02:21.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:21.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:21.840 | INFO | Uploaded 36GiB to GPU in 2.03569s (17.6844GiB/s) | 20260506 17:02:21.840 | INFO | Downloaded 34GiB to GPU in 2.013s (16.8902GiB/s) | 20260506 17:02:26.976 | INFO | Analyzing chunk frames = [2993664,2994176) | 20260506 17:02:26.976 | INFO | MemInfo: { | 20260506 17:02:26.976 | INFO | "gpu" : " 17 GB", | 20260506 17:02:26.976 | INFO | "huge" : "", | 20260506 17:02:26.976 | INFO | "huge_shared" : " 27 GB", | 20260506 17:02:26.976 | INFO | "malloc" : " 14 GB", | 20260506 17:02:26.976 | INFO | "pinned" : " 29 GB" | 20260506 17:02:26.976 | INFO | } | 20260506 17:02:26.976 | INFO | Perf report for the last 5.136 seconds: | 20260506 17:02:26.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:02:26.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:26.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:26.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:26.976 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 17:02:26.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:26.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:26.976 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260506 17:02:26.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:26.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:26.976 | INFO | Uploaded 36GiB to GPU in 2.04638s (17.592GiB/s) | 20260506 17:02:26.976 | INFO | Downloaded 34GiB to GPU in 1.97793s (17.1897GiB/s) | 20260506 17:02:27.087 | INFO | Starting a round of Baz IO | 20260506 17:02:30.274 | INFO | Populating RTMetrics report | 20260506 17:02:30.274 | INFO | RTMetrics report populated | 20260506 17:02:30.274 | INFO | Opening tmp RTMetrics json file | 20260506 17:02:30.313 | INFO | Serializing RTMetrics to json | 20260506 17:02:30.348 | INFO | Serialization done, swapping tmp file now... | 20260506 17:02:30.355 | INFO | RTMetrics done | 20260506 17:02:30.613 | INFO | Finished a round of Baz IO: Wrote 2.074 GiB in 3.526 seconds | 20260506 17:02:30.613 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.91% / 0.01% / 64.08% | 20260506 17:02:30.613 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.49093% / 7.94729e-06% / 6.35783e-05% | 20260506 17:02:30.614 | INFO | Baz IO overall: | 20260506 17:02:30.614 | INFO | BytesWritten : 1576058346874 | 20260506 17:02:30.614 | INFO | NumEvents : 762559132588 | 20260506 17:02:32.114 | INFO | Analyzing chunk frames = [2994176,2994688) | 20260506 17:02:32.114 | INFO | MemInfo: { | 20260506 17:02:32.114 | INFO | "gpu" : " 17 GB", | 20260506 17:02:32.114 | INFO | "huge" : "", | 20260506 17:02:32.114 | INFO | "huge_shared" : " 27 GB", | 20260506 17:02:32.114 | INFO | "malloc" : " 14 GB", | 20260506 17:02:32.114 | INFO | "pinned" : " 29 GB" | 20260506 17:02:32.114 | INFO | } | 20260506 17:02:32.114 | INFO | Perf report for the last 5.13814 seconds: | 20260506 17:02:32.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:02:32.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:32.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:32.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:32.114 | INFO | ANALYSIS : 63.2%, 2.8, 93.9ms, 0 | 20260506 17:02:32.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:32.114 | INFO | RT_METRICS : 50.1%, 1.0, 26.3ms, 0 | 20260506 17:02:32.114 | INFO | PRE_HQ : 61.8%, 2.0, 65.5ms, 0 | 20260506 17:02:32.114 | INFO | BAZWRITER : 68.0%, 4.8, 168.1ms, 0 | 20260506 17:02:32.114 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 17:02:32.114 | INFO | Uploaded 36GiB to GPU in 2.0254s (17.7743GiB/s) | 20260506 17:02:32.114 | INFO | Downloaded 38GiB to GPU in 2.21697s (17.1405GiB/s) | 20260506 17:02:37.254 | INFO | Analyzing chunk frames = [2994688,2995200) | 20260506 17:02:37.254 | INFO | MemInfo: { | 20260506 17:02:37.254 | INFO | "gpu" : " 17 GB", | 20260506 17:02:37.254 | INFO | "huge" : "", | 20260506 17:02:37.254 | INFO | "huge_shared" : " 27 GB", | 20260506 17:02:37.254 | INFO | "malloc" : " 14 GB", | 20260506 17:02:37.254 | INFO | "pinned" : " 29 GB" | 20260506 17:02:37.254 | INFO | } | 20260506 17:02:37.254 | INFO | Perf report for the last 5.14001 seconds: | 20260506 17:02:37.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:02:37.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:37.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:37.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:37.254 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 17:02:37.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:37.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:37.254 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:02:37.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:37.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:37.254 | INFO | Uploaded 36GiB to GPU in 2.00172s (17.9845GiB/s) | 20260506 17:02:37.254 | INFO | Downloaded 34GiB to GPU in 1.93511s (17.5701GiB/s) | 20260506 17:02:42.394 | INFO | Analyzing chunk frames = [2995200,2995712) | 20260506 17:02:42.394 | INFO | MemInfo: { | 20260506 17:02:42.394 | INFO | "gpu" : " 17 GB", | 20260506 17:02:42.394 | INFO | "huge" : "", | 20260506 17:02:42.394 | INFO | "huge_shared" : " 27 GB", | 20260506 17:02:42.394 | INFO | "malloc" : " 14 GB", | 20260506 17:02:42.394 | INFO | "pinned" : " 29 GB" | 20260506 17:02:42.394 | INFO | } | 20260506 17:02:42.394 | INFO | Perf report for the last 5.14029 seconds: | 20260506 17:02:42.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:02:42.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:42.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:42.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:42.394 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 17:02:42.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:42.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:42.394 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:02:42.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:42.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:42.395 | INFO | Uploaded 36GiB to GPU in 1.98341s (18.1506GiB/s) | 20260506 17:02:42.395 | INFO | Downloaded 34GiB to GPU in 1.92585s (17.6545GiB/s) | 20260506 17:02:47.539 | INFO | Analyzing chunk frames = [2995712,2996224) | 20260506 17:02:47.539 | INFO | MemInfo: { | 20260506 17:02:47.539 | INFO | "gpu" : " 17 GB", | 20260506 17:02:47.539 | INFO | "huge" : "", | 20260506 17:02:47.539 | INFO | "huge_shared" : " 27 GB", | 20260506 17:02:47.539 | INFO | "malloc" : " 14 GB", | 20260506 17:02:47.539 | INFO | "pinned" : " 29 GB" | 20260506 17:02:47.540 | INFO | } | 20260506 17:02:47.540 | INFO | Perf report for the last 5.14524 seconds: | 20260506 17:02:47.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:02:47.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:47.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:47.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:47.540 | INFO | ANALYSIS : 60.0%, 3.0, 93.2ms, 0 | 20260506 17:02:47.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:47.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:47.540 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:02:47.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:47.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:47.540 | INFO | Uploaded 36GiB to GPU in 1.9304s (18.649GiB/s) | 20260506 17:02:47.540 | INFO | Downloaded 34GiB to GPU in 1.87166s (18.1657GiB/s) | 20260506 17:02:52.679 | INFO | Analyzing chunk frames = [2996224,2996736) | 20260506 17:02:52.679 | INFO | MemInfo: { | 20260506 17:02:52.679 | INFO | "gpu" : " 17 GB", | 20260506 17:02:52.679 | INFO | "huge" : "", | 20260506 17:02:52.679 | INFO | "huge_shared" : " 27 GB", | 20260506 17:02:52.679 | INFO | "malloc" : " 14 GB", | 20260506 17:02:52.680 | INFO | "pinned" : " 29 GB" | 20260506 17:02:52.680 | INFO | } | 20260506 17:02:52.680 | INFO | Perf report for the last 5.14004 seconds: | 20260506 17:02:52.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:02:52.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:52.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:52.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:52.680 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 17:02:52.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:52.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:52.680 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:02:52.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:52.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:52.680 | INFO | Uploaded 36GiB to GPU in 2.01623s (17.8551GiB/s) | 20260506 17:02:52.680 | INFO | Downloaded 34GiB to GPU in 1.91053s (17.7961GiB/s) | 20260506 17:02:57.816 | INFO | Analyzing chunk frames = [2996736,2997248) | 20260506 17:02:57.816 | INFO | MemInfo: { | 20260506 17:02:57.816 | INFO | "gpu" : " 17 GB", | 20260506 17:02:57.816 | INFO | "huge" : "", | 20260506 17:02:57.816 | INFO | "huge_shared" : " 27 GB", | 20260506 17:02:57.816 | INFO | "malloc" : " 14 GB", | 20260506 17:02:57.817 | INFO | "pinned" : " 29 GB" | 20260506 17:02:57.817 | INFO | } | 20260506 17:02:57.817 | INFO | Perf report for the last 5.13686 seconds: | 20260506 17:02:57.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:02:57.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:57.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:57.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:57.817 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:02:57.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:57.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:57.817 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:02:57.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:57.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:02:57.817 | INFO | Uploaded 36GiB to GPU in 2.02699s (17.7603GiB/s) | 20260506 17:02:57.817 | INFO | Downloaded 34GiB to GPU in 1.93559s (17.5657GiB/s) | 20260506 17:03:02.952 | INFO | Analyzing chunk frames = [2997248,2997760) | 20260506 17:03:02.952 | INFO | MemInfo: { | 20260506 17:03:02.952 | INFO | "gpu" : " 17 GB", | 20260506 17:03:02.952 | INFO | "huge" : "", | 20260506 17:03:02.952 | INFO | "huge_shared" : " 27 GB", | 20260506 17:03:02.952 | INFO | "malloc" : " 14 GB", | 20260506 17:03:02.952 | INFO | "pinned" : " 29 GB" | 20260506 17:03:02.952 | INFO | } | 20260506 17:03:02.953 | INFO | Perf report for the last 5.13594 seconds: | 20260506 17:03:02.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:03:02.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:02.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:02.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:02.953 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 17:03:02.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:02.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:02.953 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 17:03:02.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:02.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:02.953 | INFO | Uploaded 36GiB to GPU in 2.04027s (17.6448GiB/s) | 20260506 17:03:02.953 | INFO | Downloaded 34GiB to GPU in 1.99573s (17.0364GiB/s) | 20260506 17:03:08.093 | INFO | Analyzing chunk frames = [2997760,2998272) | 20260506 17:03:08.093 | INFO | MemInfo: { | 20260506 17:03:08.093 | INFO | "gpu" : " 17 GB", | 20260506 17:03:08.094 | INFO | "huge" : "", | 20260506 17:03:08.094 | INFO | "huge_shared" : " 27 GB", | 20260506 17:03:08.094 | INFO | "malloc" : " 14 GB", | 20260506 17:03:08.094 | INFO | "pinned" : " 29 GB" | 20260506 17:03:08.094 | INFO | } | 20260506 17:03:08.094 | INFO | Perf report for the last 5.14109 seconds: | 20260506 17:03:08.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:03:08.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:08.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:08.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:08.094 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 17:03:08.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:08.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:08.094 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:03:08.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:08.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:08.094 | INFO | Uploaded 36GiB to GPU in 1.99348s (18.0589GiB/s) | 20260506 17:03:08.094 | INFO | Downloaded 34GiB to GPU in 1.97056s (17.254GiB/s) | 20260506 17:03:08.094 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:03:08.094 | INFO | Total mem usage: 18 GiB | 20260506 17:03:08.094 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:03:08.094 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -17%) | 20260506 17:03:08.094 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:03:08.094 | INFO | High Water Breakdown: | 20260506 17:03:08.094 | INFO | 61.1% : 9.41 GiB ( 82.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:03:08.094 | INFO | 38.7% : 5.96 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:03:08.094 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:03:08.094 | INFO | ------------------------------------------ | 20260506 17:03:08.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:03:08.094 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:03:08.094 | INFO | Total mem usage: 26.5 GiB | 20260506 17:03:08.094 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:03:08.094 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:03:08.094 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:03:08.094 | INFO | High Water Breakdown: | 20260506 17:03:08.094 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:03:08.095 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:03:08.095 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:03:08.095 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:03:08.095 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:03:08.095 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:03:08.095 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:03:08.095 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:03:08.095 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:03:08.095 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:03:08.095 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:03:08.095 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:03:08.095 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:03:08.095 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:03:08.095 | INFO | ------------------------------------------ | 20260506 17:03:08.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:03:08.095 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:03:08.095 | INFO | Total mem usage: 25.6 GiB | 20260506 17:03:08.095 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:03:08.095 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:03:08.095 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:03:08.095 | INFO | High Water Breakdown: | 20260506 17:03:08.095 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:03:08.095 | INFO | ------------------------------------------ | 20260506 17:03:08.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:03:08.095 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:03:08.095 | INFO | Total mem usage: 15.7 GiB | 20260506 17:03:08.095 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:03:08.095 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:03:08.095 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:03:08.095 | INFO | High Water Breakdown: | 20260506 17:03:08.095 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:03:08.095 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:03:08.095 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:03:08.095 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:03:08.095 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:03:08.095 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:03:08.095 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:03:08.095 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:03:08.095 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:03:08.095 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:03:08.095 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:03:08.095 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:03:08.095 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:03:08.095 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:03:08.095 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:03:08.095 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:03:08.095 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:03:08.095 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:03:08.095 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:03:08.095 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:03:08.095 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:03:08.095 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:03:08.095 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:03:08.095 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:03:08.095 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:03:08.095 | INFO | ------------------------------------------ | 20260506 17:03:08.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:03:08.095 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:03:08.095 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:03:08.095 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:03:08.095 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.57e+01, 1.34e+01 / 2.56e+00 | 20260506 17:03:08.095 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.00e-01, 1.04e-02 / 6.79e-03 | 20260506 17:03:08.095 | INFO | PerfReport: 11.60%, Baseline -- 3.64e+00 / 4.40e+00, 5.62e-01 / 3.59e-01 | 20260506 17:03:08.095 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.85e-01 / 1.35e-01 | 20260506 17:03:08.095 | INFO | PerfReport: 15.94%, DME -- 4.00e+01 / 4.48e+01, 4.15e+00 / 2.85e+00 | 20260506 17:03:08.095 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.79e+01 / 2.03e+01, 1.43e+00 / 2.11e+00 | 20260506 17:03:08.095 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.29e+00 / 2.85e+00, 3.88e-01 / 2.58e-01 | 20260506 17:03:08.095 | INFO | PerfReport: 3.46%, Metrics -- 1.09e+00 / 1.45e+00, 2.79e-01 / 1.58e-01 | 20260506 17:03:08.095 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.76e-01 / 4.27e-01, 5.82e-02 / 2.60e-02 | 20260506 17:03:08.096 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:03:08.096 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:03:08.096 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:03:08.096 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 17:03:08.096 | INFO | PerfReport: 21.47%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 17:03:08.096 | INFO | PerfReport: 21.13%, Download -- 2.05e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 17:03:08.096 | INFO | PerfReport: 32.29%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:03:08.096 | INFO | PerfReport: 1.43%, LockUpload -- 1.39e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260506 17:03:08.096 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 17:03:08.096 | INFO | PerfReport: 22.58%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 17:03:08.096 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260506 17:03:08.208 | INFO | Starting a round of Baz IO | 20260506 17:03:11.341 | INFO | Populating RTMetrics report | 20260506 17:03:11.341 | INFO | RTMetrics report populated | 20260506 17:03:11.341 | INFO | Opening tmp RTMetrics json file | 20260506 17:03:11.444 | INFO | Serializing RTMetrics to json | 20260506 17:03:11.453 | INFO | Serialization done, swapping tmp file now... | 20260506 17:03:11.458 | INFO | RTMetrics done | 20260506 17:03:11.659 | INFO | Finished a round of Baz IO: Wrote 2.075 GiB in 3.451 seconds | 20260506 17:03:11.660 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.90% / 0.01% / 64.09% | 20260506 17:03:11.660 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.48747% / 3.97364e-06% / 6.75519e-05% | 20260506 17:03:11.660 | INFO | Baz IO overall: | 20260506 17:03:11.660 | INFO | BytesWritten : 1578286217024 | 20260506 17:03:11.660 | INFO | NumEvents : 763635265268 | 20260506 17:03:13.231 | INFO | Analyzing chunk frames = [2998272,2998784) | 20260506 17:03:13.231 | INFO | MemInfo: { | 20260506 17:03:13.231 | INFO | "gpu" : " 17 GB", | 20260506 17:03:13.231 | INFO | "huge" : "", | 20260506 17:03:13.231 | INFO | "huge_shared" : " 27 GB", | 20260506 17:03:13.231 | INFO | "malloc" : " 14 GB", | 20260506 17:03:13.231 | INFO | "pinned" : " 29 GB" | 20260506 17:03:13.231 | INFO | } | 20260506 17:03:13.231 | INFO | Perf report for the last 5.13697 seconds: | 20260506 17:03:13.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:03:13.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:13.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:13.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:13.231 | INFO | ANALYSIS : 62.6%, 2.9, 95.1ms, 0 | 20260506 17:03:13.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:13.231 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260506 17:03:13.231 | INFO | PRE_HQ : 60.5%, 1.9, 58.6ms, 0 | 20260506 17:03:13.231 | INFO | BAZWRITER : 66.9%, 4.8, 164.6ms, 0 | 20260506 17:03:13.231 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 17:03:13.231 | INFO | Uploaded 36GiB to GPU in 2.04763s (17.5813GiB/s) | 20260506 17:03:13.231 | INFO | Downloaded 38GiB to GPU in 2.20142s (17.2616GiB/s) | 20260506 17:03:18.370 | INFO | Analyzing chunk frames = [2998784,2999296) | 20260506 17:03:18.370 | INFO | MemInfo: { | 20260506 17:03:18.370 | INFO | "gpu" : " 17 GB", | 20260506 17:03:18.370 | INFO | "huge" : "", | 20260506 17:03:18.370 | INFO | "huge_shared" : " 27 GB", | 20260506 17:03:18.370 | INFO | "malloc" : " 14 GB", | 20260506 17:03:18.370 | INFO | "pinned" : " 29 GB" | 20260506 17:03:18.370 | INFO | } | 20260506 17:03:18.370 | INFO | Perf report for the last 5.13978 seconds: | 20260506 17:03:18.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:03:18.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:18.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:18.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:18.371 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 17:03:18.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:18.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:18.371 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:03:18.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:18.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:18.371 | INFO | Uploaded 36GiB to GPU in 1.9916s (18.0759GiB/s) | 20260506 17:03:18.371 | INFO | Downloaded 34GiB to GPU in 1.93438s (17.5767GiB/s) | 20260506 17:03:23.514 | INFO | Analyzing chunk frames = [2999296,2999808) | 20260506 17:03:23.514 | INFO | MemInfo: { | 20260506 17:03:23.514 | INFO | "gpu" : " 17 GB", | 20260506 17:03:23.514 | INFO | "huge" : "", | 20260506 17:03:23.514 | INFO | "huge_shared" : " 27 GB", | 20260506 17:03:23.514 | INFO | "malloc" : " 14 GB", | 20260506 17:03:23.514 | INFO | "pinned" : " 29 GB" | 20260506 17:03:23.514 | INFO | } | 20260506 17:03:23.514 | INFO | Perf report for the last 5.14352 seconds: | 20260506 17:03:23.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:03:23.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:23.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:23.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:23.514 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 17:03:23.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:23.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:23.514 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:03:23.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:23.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:23.514 | INFO | Uploaded 36GiB to GPU in 1.98492s (18.1368GiB/s) | 20260506 17:03:23.514 | INFO | Downloaded 34GiB to GPU in 1.92676s (17.6462GiB/s) | 20260506 17:03:28.655 | INFO | Analyzing chunk frames = [2999808,3000320) | 20260506 17:03:28.655 | INFO | MemInfo: { | 20260506 17:03:28.655 | INFO | "gpu" : " 17 GB", | 20260506 17:03:28.655 | INFO | "huge" : "", | 20260506 17:03:28.655 | INFO | "huge_shared" : " 27 GB", | 20260506 17:03:28.655 | INFO | "malloc" : " 14 GB", | 20260506 17:03:28.655 | INFO | "pinned" : " 29 GB" | 20260506 17:03:28.655 | INFO | } | 20260506 17:03:28.655 | INFO | Perf report for the last 5.1415 seconds: | 20260506 17:03:28.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:03:28.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:28.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:28.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:28.655 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 17:03:28.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:28.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:28.656 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 17:03:28.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:28.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:28.656 | INFO | Uploaded 36GiB to GPU in 1.92519s (18.6995GiB/s) | 20260506 17:03:28.656 | INFO | Downloaded 34GiB to GPU in 1.87272s (18.1555GiB/s) | 20260506 17:03:33.789 | INFO | Analyzing chunk frames = [3000320,3000832) | 20260506 17:03:33.789 | INFO | MemInfo: { | 20260506 17:03:33.789 | INFO | "gpu" : " 17 GB", | 20260506 17:03:33.789 | INFO | "huge" : "", | 20260506 17:03:33.789 | INFO | "huge_shared" : " 27 GB", | 20260506 17:03:33.789 | INFO | "malloc" : " 14 GB", | 20260506 17:03:33.789 | INFO | "pinned" : " 29 GB" | 20260506 17:03:33.789 | INFO | } | 20260506 17:03:33.789 | INFO | Perf report for the last 5.13356 seconds: | 20260506 17:03:33.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:03:33.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:33.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:33.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:33.789 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 17:03:33.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:33.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:33.789 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:03:33.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:33.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:33.789 | INFO | Uploaded 36GiB to GPU in 2.01928s (17.8281GiB/s) | 20260506 17:03:33.790 | INFO | Downloaded 34GiB to GPU in 1.9098s (17.8029GiB/s) | 20260506 17:03:38.929 | INFO | Analyzing chunk frames = [3000832,3001344) | 20260506 17:03:38.929 | INFO | MemInfo: { | 20260506 17:03:38.929 | INFO | "gpu" : " 17 GB", | 20260506 17:03:38.929 | INFO | "huge" : "", | 20260506 17:03:38.929 | INFO | "huge_shared" : " 27 GB", | 20260506 17:03:38.929 | INFO | "malloc" : " 14 GB", | 20260506 17:03:38.929 | INFO | "pinned" : " 29 GB" | 20260506 17:03:38.929 | INFO | } | 20260506 17:03:38.929 | INFO | Perf report for the last 5.14027 seconds: | 20260506 17:03:38.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:03:38.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:38.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:38.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:38.929 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 17:03:38.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:38.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:38.929 | INFO | PRE_HQ : 32.3%, 1.0, 16.8ms, 0 | 20260506 17:03:38.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:38.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:38.929 | INFO | Uploaded 36GiB to GPU in 2.08192s (17.2917GiB/s) | 20260506 17:03:38.929 | INFO | Downloaded 34GiB to GPU in 1.96924s (17.2655GiB/s) | 20260506 17:03:44.066 | INFO | Analyzing chunk frames = [3001344,3001856) | 20260506 17:03:44.066 | INFO | MemInfo: { | 20260506 17:03:44.066 | INFO | "gpu" : " 17 GB", | 20260506 17:03:44.066 | INFO | "huge" : "", | 20260506 17:03:44.066 | INFO | "huge_shared" : " 27 GB", | 20260506 17:03:44.066 | INFO | "malloc" : " 14 GB", | 20260506 17:03:44.067 | INFO | "pinned" : " 29 GB" | 20260506 17:03:44.067 | INFO | } | 20260506 17:03:44.067 | INFO | Perf report for the last 5.13729 seconds: | 20260506 17:03:44.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:03:44.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:44.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:44.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:44.067 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:03:44.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:44.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:44.067 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 17:03:44.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:44.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:44.067 | INFO | Uploaded 36GiB to GPU in 2.00294s (17.9736GiB/s) | 20260506 17:03:44.067 | INFO | Downloaded 34GiB to GPU in 1.96668s (17.288GiB/s) | 20260506 17:03:49.205 | INFO | Analyzing chunk frames = [3001856,3002368) | 20260506 17:03:49.205 | INFO | MemInfo: { | 20260506 17:03:49.205 | INFO | "gpu" : " 17 GB", | 20260506 17:03:49.205 | INFO | "huge" : "", | 20260506 17:03:49.205 | INFO | "huge_shared" : " 27 GB", | 20260506 17:03:49.205 | INFO | "malloc" : " 14 GB", | 20260506 17:03:49.205 | INFO | "pinned" : " 29 GB" | 20260506 17:03:49.205 | INFO | } | 20260506 17:03:49.205 | INFO | Perf report for the last 5.13843 seconds: | 20260506 17:03:49.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:03:49.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:49.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:49.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:49.205 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:03:49.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:49.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:49.205 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:03:49.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:49.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:49.205 | INFO | Uploaded 36GiB to GPU in 2.01431s (17.8721GiB/s) | 20260506 17:03:49.205 | INFO | Downloaded 34GiB to GPU in 1.98826s (17.1004GiB/s) | 20260506 17:03:49.325 | INFO | Starting a round of Baz IO | 20260506 17:03:52.463 | INFO | Populating RTMetrics report | 20260506 17:03:52.463 | INFO | RTMetrics report populated | 20260506 17:03:52.463 | INFO | Opening tmp RTMetrics json file | 20260506 17:03:52.496 | INFO | Serializing RTMetrics to json | 20260506 17:03:52.613 | INFO | Serialization done, swapping tmp file now... | 20260506 17:03:52.672 | INFO | RTMetrics done | 20260506 17:03:52.737 | INFO | Finished a round of Baz IO: Wrote 2.073 GiB in 3.411 seconds | 20260506 17:03:52.737 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.89% / 0.01% / 64.10% | 20260506 17:03:52.737 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.48239% / 1.58946e-05% / 3.57628e-05% | 20260506 17:03:52.737 | INFO | Baz IO overall: | 20260506 17:03:52.737 | INFO | BytesWritten : 1580512449242 | 20260506 17:03:52.737 | INFO | NumEvents : 764710266329 | 20260506 17:03:54.348 | INFO | Analyzing chunk frames = [3002368,3002880) | 20260506 17:03:54.348 | INFO | MemInfo: { | 20260506 17:03:54.348 | INFO | "gpu" : " 17 GB", | 20260506 17:03:54.348 | INFO | "huge" : "", | 20260506 17:03:54.348 | INFO | "huge_shared" : " 27 GB", | 20260506 17:03:54.348 | INFO | "malloc" : " 14 GB", | 20260506 17:03:54.348 | INFO | "pinned" : " 29 GB" | 20260506 17:03:54.348 | INFO | } | 20260506 17:03:54.348 | INFO | Perf report for the last 5.14333 seconds: | 20260506 17:03:54.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:03:54.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:54.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:54.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:54.348 | INFO | ANALYSIS : 62.8%, 2.9, 95.1ms, 0 | 20260506 17:03:54.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:54.349 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260506 17:03:54.349 | INFO | PRE_HQ : 60.3%, 1.9, 59.0ms, 0 | 20260506 17:03:54.349 | INFO | BAZWRITER : 65.9%, 4.8, 163.3ms, 0 | 20260506 17:03:54.349 | INFO | BAZ_SAMPLING : 3.8%, 1.0, 2.0ms, 0 | 20260506 17:03:54.349 | INFO | Uploaded 36GiB to GPU in 2.05516s (17.5169GiB/s) | 20260506 17:03:54.349 | INFO | Downloaded 38GiB to GPU in 2.22121s (17.1078GiB/s) | 20260506 17:03:59.483 | INFO | Analyzing chunk frames = [3002880,3003392) | 20260506 17:03:59.483 | INFO | MemInfo: { | 20260506 17:03:59.483 | INFO | "gpu" : " 17 GB", | 20260506 17:03:59.483 | INFO | "huge" : "", | 20260506 17:03:59.483 | INFO | "huge_shared" : " 27 GB", | 20260506 17:03:59.483 | INFO | "malloc" : " 14 GB", | 20260506 17:03:59.483 | INFO | "pinned" : " 29 GB" | 20260506 17:03:59.483 | INFO | } | 20260506 17:03:59.483 | INFO | Perf report for the last 5.13497 seconds: | 20260506 17:03:59.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:03:59.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:59.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:59.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:59.483 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 17:03:59.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:59.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:59.484 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:03:59.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:59.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:03:59.484 | INFO | Uploaded 36GiB to GPU in 2.01987s (17.823GiB/s) | 20260506 17:03:59.484 | INFO | Downloaded 34GiB to GPU in 1.97295s (17.2331GiB/s) | 20260506 17:04:04.624 | INFO | Analyzing chunk frames = [3003392,3003904) | 20260506 17:04:04.624 | INFO | MemInfo: { | 20260506 17:04:04.624 | INFO | "gpu" : " 17 GB", | 20260506 17:04:04.624 | INFO | "huge" : "", | 20260506 17:04:04.624 | INFO | "huge_shared" : " 27 GB", | 20260506 17:04:04.624 | INFO | "malloc" : " 14 GB", | 20260506 17:04:04.624 | INFO | "pinned" : " 29 GB" | 20260506 17:04:04.624 | INFO | } | 20260506 17:04:04.624 | INFO | Perf report for the last 5.14097 seconds: | 20260506 17:04:04.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:04:04.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:04.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:04.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:04.624 | INFO | ANALYSIS : 61.3%, 3.0, 95.0ms, 0 | 20260506 17:04:04.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:04.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:04.624 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:04:04.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:04.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:04.625 | INFO | Uploaded 36GiB to GPU in 2.00929s (17.9168GiB/s) | 20260506 17:04:04.625 | INFO | Downloaded 34GiB to GPU in 1.93332s (17.5864GiB/s) | 20260506 17:04:09.769 | INFO | Analyzing chunk frames = [3003904,3004416) | 20260506 17:04:09.769 | INFO | MemInfo: { | 20260506 17:04:09.769 | INFO | "gpu" : " 17 GB", | 20260506 17:04:09.769 | INFO | "huge" : "", | 20260506 17:04:09.769 | INFO | "huge_shared" : " 27 GB", | 20260506 17:04:09.769 | INFO | "malloc" : " 14 GB", | 20260506 17:04:09.769 | INFO | "pinned" : " 29 GB" | 20260506 17:04:09.769 | INFO | } | 20260506 17:04:09.769 | INFO | Perf report for the last 5.14493 seconds: | 20260506 17:04:09.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:04:09.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:09.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:09.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:09.769 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 17:04:09.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:09.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:09.769 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:04:09.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:09.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:09.770 | INFO | Uploaded 36GiB to GPU in 1.95153s (18.4471GiB/s) | 20260506 17:04:09.770 | INFO | Downloaded 34GiB to GPU in 1.87983s (18.0868GiB/s) | 20260506 17:04:14.907 | INFO | Analyzing chunk frames = [3004416,3004928) | 20260506 17:04:14.907 | INFO | MemInfo: { | 20260506 17:04:14.907 | INFO | "gpu" : " 17 GB", | 20260506 17:04:14.907 | INFO | "huge" : "", | 20260506 17:04:14.907 | INFO | "huge_shared" : " 27 GB", | 20260506 17:04:14.907 | INFO | "malloc" : " 14 GB", | 20260506 17:04:14.907 | INFO | "pinned" : " 29 GB" | 20260506 17:04:14.908 | INFO | } | 20260506 17:04:14.908 | INFO | Perf report for the last 5.13826 seconds: | 20260506 17:04:14.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:04:14.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:14.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:14.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:14.908 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 17:04:14.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:14.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:14.908 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:04:14.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:14.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:14.908 | INFO | Uploaded 36GiB to GPU in 2.01855s (17.8346GiB/s) | 20260506 17:04:14.908 | INFO | Downloaded 34GiB to GPU in 1.90571s (17.8411GiB/s) | 20260506 17:04:20.045 | INFO | Analyzing chunk frames = [3004928,3005440) | 20260506 17:04:20.045 | INFO | MemInfo: { | 20260506 17:04:20.045 | INFO | "gpu" : " 17 GB", | 20260506 17:04:20.045 | INFO | "huge" : "", | 20260506 17:04:20.045 | INFO | "huge_shared" : " 27 GB", | 20260506 17:04:20.045 | INFO | "malloc" : " 14 GB", | 20260506 17:04:20.045 | INFO | "pinned" : " 29 GB" | 20260506 17:04:20.045 | INFO | } | 20260506 17:04:20.045 | INFO | Perf report for the last 5.1377 seconds: | 20260506 17:04:20.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:04:20.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:20.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:20.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:20.045 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 17:04:20.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:20.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:20.045 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 17:04:20.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:20.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:20.046 | INFO | Uploaded 36GiB to GPU in 2.05774s (17.4949GiB/s) | 20260506 17:04:20.046 | INFO | Downloaded 34GiB to GPU in 1.95932s (17.353GiB/s) | 20260506 17:04:25.185 | INFO | Analyzing chunk frames = [3005440,3005952) | 20260506 17:04:25.185 | INFO | MemInfo: { | 20260506 17:04:25.185 | INFO | "gpu" : " 17 GB", | 20260506 17:04:25.185 | INFO | "huge" : "", | 20260506 17:04:25.185 | INFO | "huge_shared" : " 27 GB", | 20260506 17:04:25.185 | INFO | "malloc" : " 14 GB", | 20260506 17:04:25.185 | INFO | "pinned" : " 29 GB" | 20260506 17:04:25.185 | INFO | } | 20260506 17:04:25.185 | INFO | Perf report for the last 5.13984 seconds: | 20260506 17:04:25.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:04:25.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:25.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:25.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:25.185 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 17:04:25.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:25.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:25.185 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:04:25.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:25.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:25.186 | INFO | Uploaded 36GiB to GPU in 2.01795s (17.8399GiB/s) | 20260506 17:04:25.186 | INFO | Downloaded 34GiB to GPU in 1.99812s (17.016GiB/s) | 20260506 17:04:30.321 | INFO | Analyzing chunk frames = [3005952,3006464) | 20260506 17:04:30.321 | INFO | MemInfo: { | 20260506 17:04:30.321 | INFO | "gpu" : " 17 GB", | 20260506 17:04:30.321 | INFO | "huge" : "", | 20260506 17:04:30.321 | INFO | "huge_shared" : " 27 GB", | 20260506 17:04:30.321 | INFO | "malloc" : " 14 GB", | 20260506 17:04:30.321 | INFO | "pinned" : " 29 GB" | 20260506 17:04:30.321 | INFO | } | 20260506 17:04:30.321 | INFO | Perf report for the last 5.13575 seconds: | 20260506 17:04:30.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:04:30.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:30.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:30.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:30.321 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 17:04:30.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:30.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:30.321 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 17:04:30.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:30.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:30.321 | INFO | Uploaded 36GiB to GPU in 2.05428s (17.5244GiB/s) | 20260506 17:04:30.321 | INFO | Downloaded 34GiB to GPU in 1.97674s (17.2GiB/s) | 20260506 17:04:30.321 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:04:30.321 | INFO | Total mem usage: 18 GiB | 20260506 17:04:30.321 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:04:30.321 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -18.7%) | 20260506 17:04:30.321 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:04:30.321 | INFO | High Water Breakdown: | 20260506 17:04:30.321 | INFO | 60.9% : 9.24 GiB ( 81% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:04:30.321 | INFO | 38.9% : 5.91 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:04:30.321 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:04:30.321 | INFO | ------------------------------------------ | 20260506 17:04:30.321 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:04:30.322 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:04:30.322 | INFO | Total mem usage: 26.5 GiB | 20260506 17:04:30.322 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:04:30.322 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:04:30.322 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:04:30.322 | INFO | High Water Breakdown: | 20260506 17:04:30.322 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:04:30.322 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:04:30.322 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:04:30.322 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:04:30.322 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:04:30.322 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:04:30.322 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:04:30.322 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:04:30.322 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:04:30.322 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:04:30.322 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:04:30.322 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:04:30.322 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:04:30.322 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:04:30.322 | INFO | ------------------------------------------ | 20260506 17:04:30.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:04:30.322 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:04:30.322 | INFO | Total mem usage: 25.6 GiB | 20260506 17:04:30.322 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:04:30.322 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:04:30.322 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:04:30.322 | INFO | High Water Breakdown: | 20260506 17:04:30.322 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:04:30.322 | INFO | ------------------------------------------ | 20260506 17:04:30.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:04:30.322 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:04:30.322 | INFO | Total mem usage: 15.7 GiB | 20260506 17:04:30.322 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:04:30.322 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:04:30.322 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:04:30.322 | INFO | High Water Breakdown: | 20260506 17:04:30.322 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:04:30.322 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:04:30.322 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:04:30.322 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:04:30.322 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:04:30.322 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:04:30.322 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:04:30.322 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:04:30.322 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:04:30.322 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:04:30.322 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:04:30.322 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:04:30.322 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:04:30.322 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:04:30.322 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:04:30.322 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:04:30.322 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:04:30.322 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:04:30.322 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:04:30.322 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:04:30.322 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:04:30.322 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:04:30.322 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:04:30.322 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:04:30.322 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:04:30.322 | INFO | ------------------------------------------ | 20260506 17:04:30.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:04:30.322 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:04:30.322 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:04:30.322 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:04:30.322 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.56e+01, 1.35e+01 / 1.97e+00 | 20260506 17:04:30.322 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 1.03e-02 / 5.39e-03 | 20260506 17:04:30.322 | INFO | PerfReport: 11.59%, Baseline -- 3.63e+00 / 4.46e+00, 5.51e-01 / 2.99e-01 | 20260506 17:04:30.322 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.84e-01 / 1.28e-01 | 20260506 17:04:30.322 | INFO | PerfReport: 15.92%, DME -- 3.99e+01 / 4.43e+01, 3.92e+00 / 2.13e+00 | 20260506 17:04:30.322 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.43e+00 / 1.93e+00 | 20260506 17:04:30.322 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.28e+00 / 2.85e+00, 3.82e-01 / 2.16e-01 | 20260506 17:04:30.322 | INFO | PerfReport: 3.47%, Metrics -- 1.09e+00 / 1.41e+00, 2.82e-01 / 1.50e-01 | 20260506 17:04:30.322 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.77e-01 / 4.31e-01, 5.87e-02 / 3.19e-02 | 20260506 17:04:30.323 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:04:30.323 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:04:30.323 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:04:30.323 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 17:04:30.323 | INFO | PerfReport: 21.56%, Upload -- 2.10e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260506 17:04:30.323 | INFO | PerfReport: 21.14%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 17:04:30.323 | INFO | PerfReport: 32.18%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 17:04:30.323 | INFO | PerfReport: 1.55%, LockUpload -- 1.51e+00 / 0.00e+00, 5.96e+00 / 0.00e+00 | 20260506 17:04:30.323 | INFO | PerfReport: 1.02%, LockDownload -- 9.90e-01 / 0.00e+00, 2.57e+00 / 0.00e+00 | 20260506 17:04:30.323 | INFO | PerfReport: 22.53%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 17:04:30.323 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260506 17:04:30.438 | INFO | Starting a round of Baz IO | 20260506 17:04:33.593 | INFO | Populating RTMetrics report | 20260506 17:04:33.593 | INFO | RTMetrics report populated | 20260506 17:04:33.593 | INFO | Opening tmp RTMetrics json file | 20260506 17:04:33.594 | INFO | Serializing RTMetrics to json | 20260506 17:04:33.596 | INFO | Serialization done, swapping tmp file now... | 20260506 17:04:33.596 | INFO | RTMetrics done | 20260506 17:04:33.802 | INFO | Finished a round of Baz IO: Wrote 2.073 GiB in 3.363 seconds | 20260506 17:04:33.802 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.89% / 0.01% / 64.11% | 20260506 17:04:33.802 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.47775% / 1.98682e-05% / 6.35783e-05% | 20260506 17:04:33.802 | INFO | Baz IO overall: | 20260506 17:04:33.802 | INFO | BytesWritten : 1582737850038 | 20260506 17:04:33.802 | INFO | NumEvents : 765784514890 | 20260506 17:04:35.456 | INFO | Analyzing chunk frames = [3006464,3006976) | 20260506 17:04:35.456 | INFO | MemInfo: { | 20260506 17:04:35.456 | INFO | "gpu" : " 17 GB", | 20260506 17:04:35.456 | INFO | "huge" : "", | 20260506 17:04:35.456 | INFO | "huge_shared" : " 27 GB", | 20260506 17:04:35.456 | INFO | "malloc" : " 14 GB", | 20260506 17:04:35.456 | INFO | "pinned" : " 29 GB" | 20260506 17:04:35.456 | INFO | } | 20260506 17:04:35.456 | INFO | Perf report for the last 5.13497 seconds: | 20260506 17:04:35.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:04:35.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:35.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:35.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:35.456 | INFO | ANALYSIS : 63.1%, 2.9, 96.0ms, 0 | 20260506 17:04:35.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:35.456 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260506 17:04:35.456 | INFO | PRE_HQ : 60.9%, 1.9, 58.6ms, 0 | 20260506 17:04:35.456 | INFO | BAZWRITER : 64.9%, 4.8, 160.2ms, 0 | 20260506 17:04:35.456 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 17:04:35.456 | INFO | Uploaded 36GiB to GPU in 2.04986s (17.5622GiB/s) | 20260506 17:04:35.456 | INFO | Downloaded 38GiB to GPU in 2.23509s (17.0015GiB/s) | 20260506 17:04:40.598 | INFO | Analyzing chunk frames = [3006976,3007488) | 20260506 17:04:40.598 | INFO | MemInfo: { | 20260506 17:04:40.598 | INFO | "gpu" : " 17 GB", | 20260506 17:04:40.598 | INFO | "huge" : "", | 20260506 17:04:40.598 | INFO | "huge_shared" : " 27 GB", | 20260506 17:04:40.598 | INFO | "malloc" : " 14 GB", | 20260506 17:04:40.599 | INFO | "pinned" : " 29 GB" | 20260506 17:04:40.599 | INFO | } | 20260506 17:04:40.599 | INFO | Perf report for the last 5.14274 seconds: | 20260506 17:04:40.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:04:40.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:40.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:40.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:40.599 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 17:04:40.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:40.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:40.599 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:04:40.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:40.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:40.599 | INFO | Uploaded 36GiB to GPU in 2.0169s (17.8491GiB/s) | 20260506 17:04:40.599 | INFO | Downloaded 34GiB to GPU in 1.9807s (17.1656GiB/s) | 20260506 17:04:45.742 | INFO | Analyzing chunk frames = [3007488,3008000) | 20260506 17:04:45.742 | INFO | MemInfo: { | 20260506 17:04:45.742 | INFO | "gpu" : " 17 GB", | 20260506 17:04:45.742 | INFO | "huge" : "", | 20260506 17:04:45.742 | INFO | "huge_shared" : " 27 GB", | 20260506 17:04:45.742 | INFO | "malloc" : " 14 GB", | 20260506 17:04:45.742 | INFO | "pinned" : " 29 GB" | 20260506 17:04:45.742 | INFO | } | 20260506 17:04:45.742 | INFO | Perf report for the last 5.14352 seconds: | 20260506 17:04:45.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:04:45.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:45.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:45.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:45.742 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 17:04:45.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:45.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:45.742 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 17:04:45.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:45.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:45.742 | INFO | Uploaded 36GiB to GPU in 1.9566s (18.3993GiB/s) | 20260506 17:04:45.742 | INFO | Downloaded 34GiB to GPU in 1.91943s (17.7136GiB/s) | 20260506 17:04:50.879 | INFO | Analyzing chunk frames = [3008000,3008512) | 20260506 17:04:50.879 | INFO | MemInfo: { | 20260506 17:04:50.879 | INFO | "gpu" : " 17 GB", | 20260506 17:04:50.879 | INFO | "huge" : "", | 20260506 17:04:50.879 | INFO | "huge_shared" : " 27 GB", | 20260506 17:04:50.879 | INFO | "malloc" : " 14 GB", | 20260506 17:04:50.879 | INFO | "pinned" : " 29 GB" | 20260506 17:04:50.879 | INFO | } | 20260506 17:04:50.879 | INFO | Perf report for the last 5.13675 seconds: | 20260506 17:04:50.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:04:50.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:50.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:50.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:50.879 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 17:04:50.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:50.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:50.879 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 17:04:50.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:50.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:50.879 | INFO | Uploaded 36GiB to GPU in 1.93468s (18.6078GiB/s) | 20260506 17:04:50.879 | INFO | Downloaded 34GiB to GPU in 1.87132s (18.169GiB/s) | 20260506 17:04:56.021 | INFO | Analyzing chunk frames = [3008512,3009024) | 20260506 17:04:56.021 | INFO | MemInfo: { | 20260506 17:04:56.021 | INFO | "gpu" : " 17 GB", | 20260506 17:04:56.021 | INFO | "huge" : "", | 20260506 17:04:56.021 | INFO | "huge_shared" : " 27 GB", | 20260506 17:04:56.021 | INFO | "malloc" : " 14 GB", | 20260506 17:04:56.021 | INFO | "pinned" : " 29 GB" | 20260506 17:04:56.021 | INFO | } | 20260506 17:04:56.021 | INFO | Perf report for the last 5.1421 seconds: | 20260506 17:04:56.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:04:56.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:56.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:56.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:56.021 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 17:04:56.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:56.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:56.021 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:04:56.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:56.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:04:56.021 | INFO | Uploaded 36GiB to GPU in 2.00306s (17.9725GiB/s) | 20260506 17:04:56.022 | INFO | Downloaded 34GiB to GPU in 1.91295s (17.7736GiB/s) | 20260506 17:05:01.160 | INFO | Analyzing chunk frames = [3009024,3009536) | 20260506 17:05:01.160 | INFO | MemInfo: { | 20260506 17:05:01.160 | INFO | "gpu" : " 17 GB", | 20260506 17:05:01.160 | INFO | "huge" : "", | 20260506 17:05:01.160 | INFO | "huge_shared" : " 27 GB", | 20260506 17:05:01.160 | INFO | "malloc" : " 14 GB", | 20260506 17:05:01.160 | INFO | "pinned" : " 29 GB" | 20260506 17:05:01.160 | INFO | } | 20260506 17:05:01.160 | INFO | Perf report for the last 5.13926 seconds: | 20260506 17:05:01.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:05:01.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:01.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:01.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:01.160 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 17:05:01.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:01.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:01.160 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:05:01.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:01.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:01.161 | INFO | Uploaded 36GiB to GPU in 2.03575s (17.6839GiB/s) | 20260506 17:05:01.161 | INFO | Downloaded 34GiB to GPU in 1.94298s (17.4989GiB/s) | 20260506 17:05:06.299 | INFO | Analyzing chunk frames = [3009536,3010048) | 20260506 17:05:06.299 | INFO | MemInfo: { | 20260506 17:05:06.299 | INFO | "gpu" : " 17 GB", | 20260506 17:05:06.299 | INFO | "huge" : "", | 20260506 17:05:06.299 | INFO | "huge_shared" : " 27 GB", | 20260506 17:05:06.299 | INFO | "malloc" : " 14 GB", | 20260506 17:05:06.299 | INFO | "pinned" : " 29 GB" | 20260506 17:05:06.299 | INFO | } | 20260506 17:05:06.299 | INFO | Perf report for the last 5.13853 seconds: | 20260506 17:05:06.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:05:06.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:06.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:06.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:06.299 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 17:05:06.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:06.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:06.299 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 17:05:06.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:06.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:06.299 | INFO | Uploaded 36GiB to GPU in 2.03491s (17.6912GiB/s) | 20260506 17:05:06.299 | INFO | Downloaded 34GiB to GPU in 1.99121s (17.075GiB/s) | 20260506 17:05:11.436 | INFO | Analyzing chunk frames = [3010048,3010560) | 20260506 17:05:11.436 | INFO | MemInfo: { | 20260506 17:05:11.436 | INFO | "gpu" : " 17 GB", | 20260506 17:05:11.436 | INFO | "huge" : "", | 20260506 17:05:11.436 | INFO | "huge_shared" : " 27 GB", | 20260506 17:05:11.436 | INFO | "malloc" : " 14 GB", | 20260506 17:05:11.436 | INFO | "pinned" : " 29 GB" | 20260506 17:05:11.436 | INFO | } | 20260506 17:05:11.436 | INFO | Perf report for the last 5.13755 seconds: | 20260506 17:05:11.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:05:11.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:11.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:11.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:11.436 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:05:11.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:11.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:11.437 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:05:11.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:11.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:11.437 | INFO | Uploaded 36GiB to GPU in 2.00003s (17.9997GiB/s) | 20260506 17:05:11.437 | INFO | Downloaded 34GiB to GPU in 1.96769s (17.2791GiB/s) | 20260506 17:05:11.550 | INFO | Starting a round of Baz IO | 20260506 17:05:14.695 | INFO | Populating RTMetrics report | 20260506 17:05:14.696 | INFO | RTMetrics report populated | 20260506 17:05:14.696 | INFO | Opening tmp RTMetrics json file | 20260506 17:05:14.697 | INFO | Serializing RTMetrics to json | 20260506 17:05:14.699 | INFO | Serialization done, swapping tmp file now... | 20260506 17:05:14.699 | INFO | RTMetrics done | 20260506 17:05:14.941 | INFO | Finished a round of Baz IO: Wrote 2.073 GiB in 3.39 seconds | 20260506 17:05:14.941 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.88% / 0.01% / 64.11% | 20260506 17:05:14.941 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.47507% / 0% / 4.37101e-05% | 20260506 17:05:14.941 | INFO | Baz IO overall: | 20260506 17:05:14.941 | INFO | BytesWritten : 1584964112572 | 20260506 17:05:14.941 | INFO | NumEvents : 766859538314 | 20260506 17:05:16.572 | INFO | Analyzing chunk frames = [3010560,3011072) | 20260506 17:05:16.572 | INFO | MemInfo: { | 20260506 17:05:16.572 | INFO | "gpu" : " 17 GB", | 20260506 17:05:16.572 | INFO | "huge" : "", | 20260506 17:05:16.572 | INFO | "huge_shared" : " 27 GB", | 20260506 17:05:16.572 | INFO | "malloc" : " 14 GB", | 20260506 17:05:16.572 | INFO | "pinned" : " 29 GB" | 20260506 17:05:16.572 | INFO | } | 20260506 17:05:16.572 | INFO | Perf report for the last 5.1353 seconds: | 20260506 17:05:16.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:05:16.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:16.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:16.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:16.572 | INFO | ANALYSIS : 62.9%, 2.9, 95.3ms, 0 | 20260506 17:05:16.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:16.572 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 17:05:16.572 | INFO | PRE_HQ : 60.6%, 1.9, 58.6ms, 0 | 20260506 17:05:16.572 | INFO | BAZWRITER : 65.7%, 4.8, 161.9ms, 0 | 20260506 17:05:16.572 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 17:05:16.572 | INFO | Uploaded 36GiB to GPU in 2.04014s (17.6459GiB/s) | 20260506 17:05:16.572 | INFO | Downloaded 38GiB to GPU in 2.21206s (17.1786GiB/s) | 20260506 17:05:21.717 | INFO | Analyzing chunk frames = [3011072,3011584) | 20260506 17:05:21.717 | INFO | MemInfo: { | 20260506 17:05:21.717 | INFO | "gpu" : " 17 GB", | 20260506 17:05:21.717 | INFO | "huge" : "", | 20260506 17:05:21.717 | INFO | "huge_shared" : " 27 GB", | 20260506 17:05:21.717 | INFO | "malloc" : " 14 GB", | 20260506 17:05:21.717 | INFO | "pinned" : " 29 GB" | 20260506 17:05:21.717 | INFO | } | 20260506 17:05:21.717 | INFO | Perf report for the last 5.1455 seconds: | 20260506 17:05:21.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:05:21.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:21.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:21.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:21.717 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 17:05:21.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:21.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:21.717 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:05:21.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:21.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:21.718 | INFO | Uploaded 36GiB to GPU in 2.02087s (17.8141GiB/s) | 20260506 17:05:21.718 | INFO | Downloaded 34GiB to GPU in 1.97296s (17.233GiB/s) | 20260506 17:05:26.860 | INFO | Analyzing chunk frames = [3011584,3012096) | 20260506 17:05:26.860 | INFO | MemInfo: { | 20260506 17:05:26.860 | INFO | "gpu" : " 17 GB", | 20260506 17:05:26.860 | INFO | "huge" : "", | 20260506 17:05:26.860 | INFO | "huge_shared" : " 27 GB", | 20260506 17:05:26.860 | INFO | "malloc" : " 14 GB", | 20260506 17:05:26.860 | INFO | "pinned" : " 29 GB" | 20260506 17:05:26.860 | INFO | } | 20260506 17:05:26.860 | INFO | Perf report for the last 5.14313 seconds: | 20260506 17:05:26.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:05:26.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:26.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:26.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:26.860 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 17:05:26.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:26.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:26.861 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:05:26.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:26.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:26.861 | INFO | Uploaded 36GiB to GPU in 1.97737s (18.206GiB/s) | 20260506 17:05:26.861 | INFO | Downloaded 34GiB to GPU in 1.92431s (17.6686GiB/s) | 20260506 17:05:32.000 | INFO | Analyzing chunk frames = [3012096,3012608) | 20260506 17:05:32.000 | INFO | MemInfo: { | 20260506 17:05:32.000 | INFO | "gpu" : " 17 GB", | 20260506 17:05:32.000 | INFO | "huge" : "", | 20260506 17:05:32.000 | INFO | "huge_shared" : " 27 GB", | 20260506 17:05:32.000 | INFO | "malloc" : " 14 GB", | 20260506 17:05:32.000 | INFO | "pinned" : " 29 GB" | 20260506 17:05:32.000 | INFO | } | 20260506 17:05:32.000 | INFO | Perf report for the last 5.13951 seconds: | 20260506 17:05:32.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:05:32.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:32.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:32.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:32.000 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 17:05:32.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:32.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:32.000 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:05:32.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:32.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:32.000 | INFO | Uploaded 36GiB to GPU in 1.96299s (18.3394GiB/s) | 20260506 17:05:32.000 | INFO | Downloaded 34GiB to GPU in 1.8768s (18.1159GiB/s) | 20260506 17:05:37.138 | INFO | Analyzing chunk frames = [3012608,3013120) | 20260506 17:05:37.138 | INFO | MemInfo: { | 20260506 17:05:37.138 | INFO | "gpu" : " 17 GB", | 20260506 17:05:37.138 | INFO | "huge" : "", | 20260506 17:05:37.138 | INFO | "huge_shared" : " 27 GB", | 20260506 17:05:37.138 | INFO | "malloc" : " 14 GB", | 20260506 17:05:37.138 | INFO | "pinned" : " 29 GB" | 20260506 17:05:37.138 | INFO | } | 20260506 17:05:37.138 | INFO | Perf report for the last 5.13833 seconds: | 20260506 17:05:37.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:05:37.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:37.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:37.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:37.138 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 17:05:37.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:37.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:37.139 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:05:37.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:37.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:37.139 | INFO | Uploaded 36GiB to GPU in 2.01943s (17.8268GiB/s) | 20260506 17:05:37.139 | INFO | Downloaded 34GiB to GPU in 1.91388s (17.765GiB/s) | 20260506 17:05:42.271 | INFO | Analyzing chunk frames = [3013120,3013632) | 20260506 17:05:42.271 | INFO | MemInfo: { | 20260506 17:05:42.271 | INFO | "gpu" : " 17 GB", | 20260506 17:05:42.272 | INFO | "huge" : "", | 20260506 17:05:42.272 | INFO | "huge_shared" : " 27 GB", | 20260506 17:05:42.272 | INFO | "malloc" : " 14 GB", | 20260506 17:05:42.272 | INFO | "pinned" : " 29 GB" | 20260506 17:05:42.272 | INFO | } | 20260506 17:05:42.272 | INFO | Perf report for the last 5.13349 seconds: | 20260506 17:05:42.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:05:42.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:42.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:42.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:42.272 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 17:05:42.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:42.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:42.272 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 17:05:42.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:42.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:42.272 | INFO | Uploaded 36GiB to GPU in 2.07119s (17.3813GiB/s) | 20260506 17:05:42.272 | INFO | Downloaded 34GiB to GPU in 1.96691s (17.286GiB/s) | 20260506 17:05:47.410 | INFO | Analyzing chunk frames = [3013632,3014144) | 20260506 17:05:47.410 | INFO | MemInfo: { | 20260506 17:05:47.410 | INFO | "gpu" : " 17 GB", | 20260506 17:05:47.410 | INFO | "huge" : "", | 20260506 17:05:47.410 | INFO | "huge_shared" : " 27 GB", | 20260506 17:05:47.410 | INFO | "malloc" : " 14 GB", | 20260506 17:05:47.410 | INFO | "pinned" : " 29 GB" | 20260506 17:05:47.410 | INFO | } | 20260506 17:05:47.410 | INFO | Perf report for the last 5.13849 seconds: | 20260506 17:05:47.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:05:47.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:47.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:47.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:47.410 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 17:05:47.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:47.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:47.410 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:05:47.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:47.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:47.411 | INFO | Uploaded 36GiB to GPU in 2.01931s (17.8279GiB/s) | 20260506 17:05:47.411 | INFO | Downloaded 34GiB to GPU in 2.01125s (16.9049GiB/s) | 20260506 17:05:52.546 | INFO | Analyzing chunk frames = [3014144,3014656) | 20260506 17:05:52.546 | INFO | MemInfo: { | 20260506 17:05:52.546 | INFO | "gpu" : " 17 GB", | 20260506 17:05:52.546 | INFO | "huge" : "", | 20260506 17:05:52.546 | INFO | "huge_shared" : " 27 GB", | 20260506 17:05:52.546 | INFO | "malloc" : " 14 GB", | 20260506 17:05:52.546 | INFO | "pinned" : " 29 GB" | 20260506 17:05:52.546 | INFO | } | 20260506 17:05:52.546 | INFO | Perf report for the last 5.13584 seconds: | 20260506 17:05:52.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:05:52.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:52.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:52.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:52.546 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 17:05:52.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:52.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:52.546 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:05:52.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:52.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:52.546 | INFO | Uploaded 36GiB to GPU in 2.00898s (17.9196GiB/s) | 20260506 17:05:52.546 | INFO | Downloaded 34GiB to GPU in 1.96525s (17.3006GiB/s) | 20260506 17:05:52.547 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:05:52.547 | INFO | Total mem usage: 18 GiB | 20260506 17:05:52.547 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:05:52.547 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -17.1%) | 20260506 17:05:52.547 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:05:52.547 | INFO | High Water Breakdown: | 20260506 17:05:52.547 | INFO | 61.1% : 9.4 GiB ( 82.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:05:52.547 | INFO | 38.8% : 5.96 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:05:52.547 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:05:52.547 | INFO | ------------------------------------------ | 20260506 17:05:52.547 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:05:52.547 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:05:52.547 | INFO | Total mem usage: 26.5 GiB | 20260506 17:05:52.547 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:05:52.547 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:05:52.547 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:05:52.547 | INFO | High Water Breakdown: | 20260506 17:05:52.547 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:05:52.547 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:05:52.547 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:05:52.547 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:05:52.547 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:05:52.547 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:05:52.547 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:05:52.547 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:05:52.547 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:05:52.547 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:05:52.547 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:05:52.547 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:05:52.547 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:05:52.547 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:05:52.547 | INFO | ------------------------------------------ | 20260506 17:05:52.547 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:05:52.547 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:05:52.547 | INFO | Total mem usage: 25.6 GiB | 20260506 17:05:52.547 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:05:52.547 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:05:52.547 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:05:52.547 | INFO | High Water Breakdown: | 20260506 17:05:52.547 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:05:52.547 | INFO | ------------------------------------------ | 20260506 17:05:52.547 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:05:52.547 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:05:52.547 | INFO | Total mem usage: 15.7 GiB | 20260506 17:05:52.547 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:05:52.547 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:05:52.547 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 17:05:52.547 | INFO | High Water Breakdown: | 20260506 17:05:52.547 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:05:52.547 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:05:52.547 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:05:52.547 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:05:52.547 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:05:52.547 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:05:52.547 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:05:52.547 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:05:52.547 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:05:52.547 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:05:52.547 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:05:52.547 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:05:52.547 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:05:52.547 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:05:52.547 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:05:52.547 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:05:52.547 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:05:52.547 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:05:52.547 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:05:52.547 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:05:52.547 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:05:52.547 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:05:52.547 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:05:52.547 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:05:52.547 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:05:52.547 | INFO | ------------------------------------------ | 20260506 17:05:52.547 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:05:52.548 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:05:52.548 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:05:52.548 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:05:52.548 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.62e+01, 1.35e+01 / 2.19e+00 | 20260506 17:05:52.548 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.01e-01, 9.80e-03 / 4.66e-03 | 20260506 17:05:52.548 | INFO | PerfReport: 11.60%, Baseline -- 3.63e+00 / 4.47e+00, 5.52e-01 / 2.27e-01 | 20260506 17:05:52.548 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.49e+00, 1.84e-01 / 9.16e-02 | 20260506 17:05:52.548 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.49e+01, 4.07e+00 / 2.69e+00 | 20260506 17:05:52.548 | INFO | PerfReport: 56.88%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.34e+00 / 1.63e+00 | 20260506 17:05:52.548 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.29e+00 / 2.89e+00, 3.93e-01 / 1.73e-01 | 20260506 17:05:52.548 | INFO | PerfReport: 3.47%, Metrics -- 1.09e+00 / 1.45e+00, 2.83e-01 / 1.73e-01 | 20260506 17:05:52.548 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.75e-01 / 4.29e-01, 5.80e-02 / 1.66e-02 | 20260506 17:05:52.548 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:05:52.548 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:05:52.548 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:05:52.548 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 17:05:52.548 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 17:05:52.548 | INFO | PerfReport: 21.18%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 17:05:52.548 | INFO | PerfReport: 32.16%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 17:05:52.548 | INFO | PerfReport: 1.49%, LockUpload -- 1.45e+00 / 0.00e+00, 5.80e+00 / 0.00e+00 | 20260506 17:05:52.548 | INFO | PerfReport: 1.08%, LockDownload -- 1.05e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260506 17:05:52.548 | INFO | PerfReport: 22.57%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 17:05:52.548 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260506 17:05:52.665 | INFO | Starting a round of Baz IO | 20260506 17:05:55.802 | INFO | Populating RTMetrics report | 20260506 17:05:55.802 | INFO | RTMetrics report populated | 20260506 17:05:55.802 | INFO | Opening tmp RTMetrics json file | 20260506 17:05:55.868 | INFO | Serializing RTMetrics to json | 20260506 17:05:55.939 | INFO | Serialization done, swapping tmp file now... | 20260506 17:05:55.942 | INFO | RTMetrics done | 20260506 17:05:56.434 | INFO | Finished a round of Baz IO: Wrote 2.072 GiB in 3.768 seconds | 20260506 17:05:56.434 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.87% / 0.01% / 64.12% | 20260506 17:05:56.434 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.47175% / 1.19209e-05% / 3.57628e-05% | 20260506 17:05:56.434 | INFO | Baz IO overall: | 20260506 17:05:56.434 | INFO | BytesWritten : 1587189414395 | 20260506 17:05:56.434 | INFO | NumEvents : 767933953008 | 20260506 17:05:57.690 | INFO | Analyzing chunk frames = [3014656,3015168) | 20260506 17:05:57.690 | INFO | MemInfo: { | 20260506 17:05:57.690 | INFO | "gpu" : " 17 GB", | 20260506 17:05:57.690 | INFO | "huge" : "", | 20260506 17:05:57.690 | INFO | "huge_shared" : " 27 GB", | 20260506 17:05:57.690 | INFO | "malloc" : " 14 GB", | 20260506 17:05:57.690 | INFO | "pinned" : " 29 GB" | 20260506 17:05:57.690 | INFO | } | 20260506 17:05:57.690 | INFO | Perf report for the last 5.14401 seconds: | 20260506 17:05:57.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:05:57.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:57.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:57.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:57.690 | INFO | ANALYSIS : 62.7%, 2.9, 95.6ms, 0 | 20260506 17:05:57.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:05:57.690 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260506 17:05:57.690 | INFO | PRE_HQ : 60.4%, 1.8, 57.9ms, 0 | 20260506 17:05:57.690 | INFO | BAZWRITER : 72.9%, 4.8, 181.5ms, 0 | 20260506 17:05:57.690 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 17:05:57.691 | INFO | Uploaded 36GiB to GPU in 2.06109s (17.4665GiB/s) | 20260506 17:05:57.691 | INFO | Downloaded 38GiB to GPU in 2.2169s (17.141GiB/s) | 20260506 17:06:02.833 | INFO | Analyzing chunk frames = [3015168,3015680) | 20260506 17:06:02.833 | INFO | MemInfo: { | 20260506 17:06:02.833 | INFO | "gpu" : " 17 GB", | 20260506 17:06:02.833 | INFO | "huge" : "", | 20260506 17:06:02.833 | INFO | "huge_shared" : " 27 GB", | 20260506 17:06:02.833 | INFO | "malloc" : " 14 GB", | 20260506 17:06:02.833 | INFO | "pinned" : " 29 GB" | 20260506 17:06:02.833 | INFO | } | 20260506 17:06:02.833 | INFO | Perf report for the last 5.14274 seconds: | 20260506 17:06:02.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:06:02.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:02.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:02.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:02.833 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:06:02.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:02.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:02.833 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:06:02.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:02.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:02.833 | INFO | Uploaded 36GiB to GPU in 1.99158s (18.0761GiB/s) | 20260506 17:06:02.833 | INFO | Downloaded 34GiB to GPU in 1.93211s (17.5973GiB/s) | 20260506 17:06:07.972 | INFO | Analyzing chunk frames = [3015680,3016192) | 20260506 17:06:07.972 | INFO | MemInfo: { | 20260506 17:06:07.972 | INFO | "gpu" : " 17 GB", | 20260506 17:06:07.972 | INFO | "huge" : "", | 20260506 17:06:07.972 | INFO | "huge_shared" : " 27 GB", | 20260506 17:06:07.972 | INFO | "malloc" : " 14 GB", | 20260506 17:06:07.972 | INFO | "pinned" : " 29 GB" | 20260506 17:06:07.972 | INFO | } | 20260506 17:06:07.973 | INFO | Perf report for the last 5.13962 seconds: | 20260506 17:06:07.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:06:07.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:07.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:07.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:07.973 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 17:06:07.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:07.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:07.973 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:06:07.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:07.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:07.973 | INFO | Uploaded 36GiB to GPU in 1.99275s (18.0655GiB/s) | 20260506 17:06:07.973 | INFO | Downloaded 34GiB to GPU in 1.93532s (17.5682GiB/s) | 20260506 17:06:13.118 | INFO | Analyzing chunk frames = [3016192,3016704) | 20260506 17:06:13.118 | INFO | MemInfo: { | 20260506 17:06:13.118 | INFO | "gpu" : " 17 GB", | 20260506 17:06:13.118 | INFO | "huge" : "", | 20260506 17:06:13.118 | INFO | "huge_shared" : " 27 GB", | 20260506 17:06:13.118 | INFO | "malloc" : " 14 GB", | 20260506 17:06:13.118 | INFO | "pinned" : " 29 GB" | 20260506 17:06:13.118 | INFO | } | 20260506 17:06:13.118 | INFO | Perf report for the last 5.14587 seconds: | 20260506 17:06:13.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:06:13.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:13.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:13.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:13.119 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260506 17:06:13.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:13.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:13.119 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:06:13.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:13.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:13.119 | INFO | Uploaded 36GiB to GPU in 1.94254s (18.5324GiB/s) | 20260506 17:06:13.119 | INFO | Downloaded 34GiB to GPU in 1.87524s (18.131GiB/s) | 20260506 17:06:18.249 | INFO | Analyzing chunk frames = [3016704,3017216) | 20260506 17:06:18.249 | INFO | MemInfo: { | 20260506 17:06:18.249 | INFO | "gpu" : " 17 GB", | 20260506 17:06:18.249 | INFO | "huge" : "", | 20260506 17:06:18.249 | INFO | "huge_shared" : " 27 GB", | 20260506 17:06:18.249 | INFO | "malloc" : " 14 GB", | 20260506 17:06:18.249 | INFO | "pinned" : " 29 GB" | 20260506 17:06:18.249 | INFO | } | 20260506 17:06:18.250 | INFO | Perf report for the last 5.13107 seconds: | 20260506 17:06:18.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:06:18.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:18.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:18.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:18.250 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260506 17:06:18.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:18.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:18.250 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:06:18.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:18.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:18.250 | INFO | Uploaded 36GiB to GPU in 2.02034s (17.8188GiB/s) | 20260506 17:06:18.250 | INFO | Downloaded 34GiB to GPU in 1.90723s (17.8269GiB/s) | 20260506 17:06:23.386 | INFO | Analyzing chunk frames = [3017216,3017728) | 20260506 17:06:23.386 | INFO | MemInfo: { | 20260506 17:06:23.386 | INFO | "gpu" : " 17 GB", | 20260506 17:06:23.386 | INFO | "huge" : "", | 20260506 17:06:23.386 | INFO | "huge_shared" : " 27 GB", | 20260506 17:06:23.386 | INFO | "malloc" : " 14 GB", | 20260506 17:06:23.386 | INFO | "pinned" : " 29 GB" | 20260506 17:06:23.386 | INFO | } | 20260506 17:06:23.386 | INFO | Perf report for the last 5.13682 seconds: | 20260506 17:06:23.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:06:23.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:23.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:23.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:23.386 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:06:23.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:23.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:23.386 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:06:23.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:23.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:23.387 | INFO | Uploaded 36GiB to GPU in 2.05031s (17.5583GiB/s) | 20260506 17:06:23.387 | INFO | Downloaded 34GiB to GPU in 1.94601s (17.4716GiB/s) | 20260506 17:06:28.526 | INFO | Analyzing chunk frames = [3017728,3018240) | 20260506 17:06:28.526 | INFO | MemInfo: { | 20260506 17:06:28.526 | INFO | "gpu" : " 17 GB", | 20260506 17:06:28.526 | INFO | "huge" : "", | 20260506 17:06:28.526 | INFO | "huge_shared" : " 27 GB", | 20260506 17:06:28.526 | INFO | "malloc" : " 14 GB", | 20260506 17:06:28.526 | INFO | "pinned" : " 29 GB" | 20260506 17:06:28.526 | INFO | } | 20260506 17:06:28.526 | INFO | Perf report for the last 5.13951 seconds: | 20260506 17:06:28.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:06:28.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:28.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:28.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:28.526 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:06:28.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:28.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:28.526 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:06:28.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:28.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:28.526 | INFO | Uploaded 36GiB to GPU in 2.00944s (17.9155GiB/s) | 20260506 17:06:28.526 | INFO | Downloaded 34GiB to GPU in 2.00909s (16.9231GiB/s) | 20260506 17:06:33.666 | INFO | Analyzing chunk frames = [3018240,3018752) | 20260506 17:06:33.666 | INFO | MemInfo: { | 20260506 17:06:33.666 | INFO | "gpu" : " 17 GB", | 20260506 17:06:33.666 | INFO | "huge" : "", | 20260506 17:06:33.666 | INFO | "huge_shared" : " 27 GB", | 20260506 17:06:33.666 | INFO | "malloc" : " 14 GB", | 20260506 17:06:33.666 | INFO | "pinned" : " 29 GB" | 20260506 17:06:33.666 | INFO | } | 20260506 17:06:33.666 | INFO | Perf report for the last 5.13987 seconds: | 20260506 17:06:33.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:06:33.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:33.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:33.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:33.666 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 17:06:33.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:33.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:33.666 | INFO | PRE_HQ : 31.8%, 1.0, 16.5ms, 0 | 20260506 17:06:33.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:33.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:33.666 | INFO | Uploaded 36GiB to GPU in 2.02154s (17.8082GiB/s) | 20260506 17:06:33.666 | INFO | Downloaded 34GiB to GPU in 1.98272s (17.1481GiB/s) | 20260506 17:06:33.786 | INFO | Starting a round of Baz IO | 20260506 17:06:36.943 | INFO | Populating RTMetrics report | 20260506 17:06:36.943 | INFO | RTMetrics report populated | 20260506 17:06:36.943 | INFO | Opening tmp RTMetrics json file | 20260506 17:06:36.952 | INFO | Serializing RTMetrics to json | 20260506 17:06:36.988 | INFO | Serialization done, swapping tmp file now... | 20260506 17:06:36.989 | INFO | RTMetrics done | 20260506 17:06:37.099 | INFO | Finished a round of Baz IO: Wrote 2.072 GiB in 3.312 seconds | 20260506 17:06:37.099 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.86% / 0.01% / 64.13% | 20260506 17:06:37.099 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.46958% / 1.98682e-05% / 4.76837e-05% | 20260506 17:06:37.099 | INFO | Baz IO overall: | 20260506 17:06:37.099 | INFO | BytesWritten : 1589414351419 | 20260506 17:06:37.099 | INFO | NumEvents : 769008215380 | 20260506 17:06:38.809 | INFO | Analyzing chunk frames = [3018752,3019264) | 20260506 17:06:38.809 | INFO | MemInfo: { | 20260506 17:06:38.809 | INFO | "gpu" : " 17 GB", | 20260506 17:06:38.809 | INFO | "huge" : "", | 20260506 17:06:38.809 | INFO | "huge_shared" : " 27 GB", | 20260506 17:06:38.809 | INFO | "malloc" : " 14 GB", | 20260506 17:06:38.809 | INFO | "pinned" : " 29 GB" | 20260506 17:06:38.809 | INFO | } | 20260506 17:06:38.809 | INFO | Perf report for the last 5.14321 seconds: | 20260506 17:06:38.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:06:38.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:38.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:38.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:38.809 | INFO | ANALYSIS : 63.1%, 2.8, 94.2ms, 0 | 20260506 17:06:38.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:38.809 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260506 17:06:38.809 | INFO | PRE_HQ : 61.3%, 1.9, 61.3ms, 0 | 20260506 17:06:38.809 | INFO | BAZWRITER : 64.0%, 4.8, 157.1ms, 0 | 20260506 17:06:38.809 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 17:06:38.809 | INFO | Uploaded 36GiB to GPU in 2.06293s (17.4509GiB/s) | 20260506 17:06:38.809 | INFO | Downloaded 38GiB to GPU in 2.21218s (17.1776GiB/s) | 20260506 17:06:43.950 | INFO | Analyzing chunk frames = [3019264,3019776) | 20260506 17:06:43.950 | INFO | MemInfo: { | 20260506 17:06:43.950 | INFO | "gpu" : " 17 GB", | 20260506 17:06:43.950 | INFO | "huge" : "", | 20260506 17:06:43.950 | INFO | "huge_shared" : " 27 GB", | 20260506 17:06:43.950 | INFO | "malloc" : " 14 GB", | 20260506 17:06:43.950 | INFO | "pinned" : " 29 GB" | 20260506 17:06:43.950 | INFO | } | 20260506 17:06:43.950 | INFO | Perf report for the last 5.1408 seconds: | 20260506 17:06:43.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:06:43.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:43.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:43.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:43.950 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 17:06:43.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:43.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:43.950 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 17:06:43.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:43.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:43.950 | INFO | Uploaded 36GiB to GPU in 2.01865s (17.8337GiB/s) | 20260506 17:06:43.950 | INFO | Downloaded 34GiB to GPU in 1.94448s (17.4854GiB/s) | 20260506 17:06:49.086 | INFO | Analyzing chunk frames = [3019776,3020288) | 20260506 17:06:49.086 | INFO | MemInfo: { | 20260506 17:06:49.086 | INFO | "gpu" : " 17 GB", | 20260506 17:06:49.086 | INFO | "huge" : "", | 20260506 17:06:49.086 | INFO | "huge_shared" : " 27 GB", | 20260506 17:06:49.086 | INFO | "malloc" : " 14 GB", | 20260506 17:06:49.086 | INFO | "pinned" : " 29 GB" | 20260506 17:06:49.086 | INFO | } | 20260506 17:06:49.087 | INFO | Perf report for the last 5.13675 seconds: | 20260506 17:06:49.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:06:49.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:49.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:49.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:49.087 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 17:06:49.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:49.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:49.087 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 17:06:49.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:49.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:49.087 | INFO | Uploaded 36GiB to GPU in 1.98371s (18.1479GiB/s) | 20260506 17:06:49.087 | INFO | Downloaded 34GiB to GPU in 1.9266s (17.6477GiB/s) | 20260506 17:06:54.224 | INFO | Analyzing chunk frames = [3020288,3020800) | 20260506 17:06:54.224 | INFO | MemInfo: { | 20260506 17:06:54.224 | INFO | "gpu" : " 17 GB", | 20260506 17:06:54.224 | INFO | "huge" : "", | 20260506 17:06:54.224 | INFO | "huge_shared" : " 27 GB", | 20260506 17:06:54.224 | INFO | "malloc" : " 14 GB", | 20260506 17:06:54.224 | INFO | "pinned" : " 29 GB" | 20260506 17:06:54.224 | INFO | } | 20260506 17:06:54.224 | INFO | Perf report for the last 5.13787 seconds: | 20260506 17:06:54.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:06:54.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:54.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:54.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:54.224 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 17:06:54.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:54.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:54.225 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:06:54.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:54.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:54.225 | INFO | Uploaded 36GiB to GPU in 1.93432s (18.6112GiB/s) | 20260506 17:06:54.225 | INFO | Downloaded 34GiB to GPU in 1.87535s (18.13GiB/s) | 20260506 17:06:59.363 | INFO | Analyzing chunk frames = [3020800,3021312) | 20260506 17:06:59.363 | INFO | MemInfo: { | 20260506 17:06:59.363 | INFO | "gpu" : " 17 GB", | 20260506 17:06:59.363 | INFO | "huge" : "", | 20260506 17:06:59.363 | INFO | "huge_shared" : " 27 GB", | 20260506 17:06:59.363 | INFO | "malloc" : " 14 GB", | 20260506 17:06:59.363 | INFO | "pinned" : " 29 GB" | 20260506 17:06:59.363 | INFO | } | 20260506 17:06:59.363 | INFO | Perf report for the last 5.1385 seconds: | 20260506 17:06:59.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:06:59.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:59.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:59.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:59.363 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 17:06:59.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:59.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:59.363 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:06:59.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:59.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:06:59.363 | INFO | Uploaded 36GiB to GPU in 1.96565s (18.3145GiB/s) | 20260506 17:06:59.363 | INFO | Downloaded 34GiB to GPU in 1.92517s (17.6607GiB/s) | 20260506 17:07:04.502 | INFO | Analyzing chunk frames = [3021312,3021824) | 20260506 17:07:04.502 | INFO | MemInfo: { | 20260506 17:07:04.502 | INFO | "gpu" : " 17 GB", | 20260506 17:07:04.502 | INFO | "huge" : "", | 20260506 17:07:04.502 | INFO | "huge_shared" : " 27 GB", | 20260506 17:07:04.502 | INFO | "malloc" : " 14 GB", | 20260506 17:07:04.502 | INFO | "pinned" : " 29 GB" | 20260506 17:07:04.502 | INFO | } | 20260506 17:07:04.502 | INFO | Perf report for the last 5.1394 seconds: | 20260506 17:07:04.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:07:04.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:04.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:04.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:04.502 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 17:07:04.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:04.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:04.502 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:07:04.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:04.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:04.503 | INFO | Uploaded 36GiB to GPU in 2.02989s (17.735GiB/s) | 20260506 17:07:04.503 | INFO | Downloaded 34GiB to GPU in 1.9346s (17.5747GiB/s) | 20260506 17:07:09.641 | INFO | Analyzing chunk frames = [3021824,3022336) | 20260506 17:07:09.641 | INFO | MemInfo: { | 20260506 17:07:09.641 | INFO | "gpu" : " 17 GB", | 20260506 17:07:09.641 | INFO | "huge" : "", | 20260506 17:07:09.641 | INFO | "huge_shared" : " 27 GB", | 20260506 17:07:09.641 | INFO | "malloc" : " 14 GB", | 20260506 17:07:09.641 | INFO | "pinned" : " 29 GB" | 20260506 17:07:09.641 | INFO | } | 20260506 17:07:09.641 | INFO | Perf report for the last 5.13885 seconds: | 20260506 17:07:09.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:07:09.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:09.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:09.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:09.641 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 17:07:09.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:09.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:09.641 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 17:07:09.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:09.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:09.642 | INFO | Uploaded 36GiB to GPU in 2.01979s (17.8236GiB/s) | 20260506 17:07:09.642 | INFO | Downloaded 34GiB to GPU in 2.01393s (16.8824GiB/s) | 20260506 17:07:14.780 | INFO | Analyzing chunk frames = [3022336,3022848) | 20260506 17:07:14.780 | INFO | MemInfo: { | 20260506 17:07:14.780 | INFO | "gpu" : " 17 GB", | 20260506 17:07:14.780 | INFO | "huge" : "", | 20260506 17:07:14.780 | INFO | "huge_shared" : " 27 GB", | 20260506 17:07:14.780 | INFO | "malloc" : " 14 GB", | 20260506 17:07:14.780 | INFO | "pinned" : " 29 GB" | 20260506 17:07:14.780 | INFO | } | 20260506 17:07:14.781 | INFO | Perf report for the last 5.13933 seconds: | 20260506 17:07:14.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:07:14.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:14.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:14.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:14.781 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 17:07:14.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:14.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:14.781 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260506 17:07:14.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:14.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:14.781 | INFO | Uploaded 36GiB to GPU in 2.06193s (17.4593GiB/s) | 20260506 17:07:14.781 | INFO | Downloaded 34GiB to GPU in 1.97579s (17.2083GiB/s) | 20260506 17:07:14.781 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:07:14.781 | INFO | Total mem usage: 18 GiB | 20260506 17:07:14.781 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:07:14.781 | INFO | Recent High Water Marks Sum: 16.8 GiB (Savings: -7.11%) | 20260506 17:07:14.781 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:07:14.781 | INFO | High Water Breakdown: | 20260506 17:07:14.781 | INFO | 62.4% : 10.5 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:07:14.781 | INFO | 37.4% : 6.29 GiB ( 94.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:07:14.781 | INFO | 0.118% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:07:14.781 | INFO | ------------------------------------------ | 20260506 17:07:14.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:07:14.781 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:07:14.781 | INFO | Total mem usage: 26.5 GiB | 20260506 17:07:14.781 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:07:14.781 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:07:14.781 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:07:14.781 | INFO | High Water Breakdown: | 20260506 17:07:14.781 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:07:14.781 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:07:14.781 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:07:14.781 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:07:14.781 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:07:14.781 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:07:14.782 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:07:14.782 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:07:14.782 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:07:14.782 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:07:14.782 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:07:14.782 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:07:14.782 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:07:14.782 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:07:14.782 | INFO | ------------------------------------------ | 20260506 17:07:14.782 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:07:14.782 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:07:14.782 | INFO | Total mem usage: 25.6 GiB | 20260506 17:07:14.782 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:07:14.782 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:07:14.782 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:07:14.782 | INFO | High Water Breakdown: | 20260506 17:07:14.782 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:07:14.782 | INFO | ------------------------------------------ | 20260506 17:07:14.782 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:07:14.782 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:07:14.782 | INFO | Total mem usage: 15.7 GiB | 20260506 17:07:14.782 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:07:14.782 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:07:14.782 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:07:14.782 | INFO | High Water Breakdown: | 20260506 17:07:14.782 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:07:14.782 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:07:14.782 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:07:14.782 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:07:14.782 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:07:14.782 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:07:14.782 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:07:14.782 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:07:14.782 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:07:14.782 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:07:14.782 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:07:14.782 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:07:14.782 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:07:14.782 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:07:14.782 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:07:14.782 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:07:14.782 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:07:14.782 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:07:14.782 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:07:14.782 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:07:14.782 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:07:14.782 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:07:14.782 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:07:14.782 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:07:14.782 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:07:14.782 | INFO | ------------------------------------------ | 20260506 17:07:14.782 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:07:14.782 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:07:14.782 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:07:14.782 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:07:14.782 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.57e+01, 1.36e+01 / 2.78e+00 | 20260506 17:07:14.782 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.02e-01, 1.00e-02 / 5.83e-03 | 20260506 17:07:14.782 | INFO | PerfReport: 11.62%, Baseline -- 3.64e+00 / 4.41e+00, 5.57e-01 / 3.48e-01 | 20260506 17:07:14.782 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.47e+00, 1.85e-01 / 1.29e-01 | 20260506 17:07:14.782 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.47e+01, 4.14e+00 / 3.23e+00 | 20260506 17:07:14.782 | INFO | PerfReport: 56.91%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.38e+00 / 1.79e+00 | 20260506 17:07:14.782 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.28e+00 / 2.84e+00, 3.83e-01 / 2.54e-01 | 20260506 17:07:14.782 | INFO | PerfReport: 3.46%, Metrics -- 1.08e+00 / 1.41e+00, 2.76e-01 / 1.68e-01 | 20260506 17:07:14.782 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.76e-01 / 4.27e-01, 5.80e-02 / 3.11e-02 | 20260506 17:07:14.782 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:07:14.782 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:07:14.782 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:07:14.782 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 17:07:14.782 | INFO | PerfReport: 21.53%, Upload -- 2.09e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260506 17:07:14.782 | INFO | PerfReport: 21.16%, Download -- 2.06e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260506 17:07:14.782 | INFO | PerfReport: 32.23%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 17:07:14.782 | INFO | PerfReport: 1.47%, LockUpload -- 1.43e+00 / 0.00e+00, 5.83e+00 / 0.00e+00 | 20260506 17:07:14.782 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.69e+00 / 0.00e+00 | 20260506 17:07:14.782 | INFO | PerfReport: 22.50%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 17:07:14.782 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.61e-02 / 0.00e+00 | 20260506 17:07:14.895 | INFO | Starting a round of Baz IO | 20260506 17:07:18.048 | INFO | Populating RTMetrics report | 20260506 17:07:18.049 | INFO | RTMetrics report populated | 20260506 17:07:18.049 | INFO | Opening tmp RTMetrics json file | 20260506 17:07:18.093 | INFO | Serializing RTMetrics to json | 20260506 17:07:18.245 | INFO | Serialization done, swapping tmp file now... | 20260506 17:07:18.282 | INFO | RTMetrics done | 20260506 17:07:18.311 | INFO | Finished a round of Baz IO: Wrote 2.072 GiB in 3.415 seconds | 20260506 17:07:18.311 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.85% / 0.01% / 64.14% | 20260506 17:07:18.311 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.46531% / 7.94729e-06% / 5.16574e-05% | 20260506 17:07:18.311 | INFO | Baz IO overall: | 20260506 17:07:18.311 | INFO | BytesWritten : 1591639321258 | 20260506 17:07:18.311 | INFO | NumEvents : 770082501554 | 20260506 17:07:19.925 | INFO | Analyzing chunk frames = [3022848,3023360) | 20260506 17:07:19.925 | INFO | MemInfo: { | 20260506 17:07:19.925 | INFO | "gpu" : " 17 GB", | 20260506 17:07:19.925 | INFO | "huge" : "", | 20260506 17:07:19.925 | INFO | "huge_shared" : " 27 GB", | 20260506 17:07:19.925 | INFO | "malloc" : " 14 GB", | 20260506 17:07:19.925 | INFO | "pinned" : " 29 GB" | 20260506 17:07:19.926 | INFO | } | 20260506 17:07:19.926 | INFO | Perf report for the last 5.14499 seconds: | 20260506 17:07:19.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:07:19.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:19.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:19.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:19.926 | INFO | ANALYSIS : 62.9%, 2.9, 95.4ms, 0 | 20260506 17:07:19.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:19.926 | INFO | RT_METRICS : 48.1%, 1.0, 25.2ms, 0 | 20260506 17:07:19.926 | INFO | PRE_HQ : 60.4%, 1.9, 58.6ms, 0 | 20260506 17:07:19.926 | INFO | BAZWRITER : 65.8%, 4.8, 162.7ms, 0 | 20260506 17:07:19.926 | INFO | BAZ_SAMPLING : 3.7%, 1.0, 1.9ms, 0 | 20260506 17:07:19.926 | INFO | Uploaded 36GiB to GPU in 2.06839s (17.4049GiB/s) | 20260506 17:07:19.926 | INFO | Downloaded 38GiB to GPU in 2.2362s (16.9931GiB/s) | 20260506 17:07:25.067 | INFO | Analyzing chunk frames = [3023360,3023872) | 20260506 17:07:25.067 | INFO | MemInfo: { | 20260506 17:07:25.067 | INFO | "gpu" : " 17 GB", | 20260506 17:07:25.067 | INFO | "huge" : "", | 20260506 17:07:25.067 | INFO | "huge_shared" : " 27 GB", | 20260506 17:07:25.067 | INFO | "malloc" : " 14 GB", | 20260506 17:07:25.067 | INFO | "pinned" : " 29 GB" | 20260506 17:07:25.067 | INFO | } | 20260506 17:07:25.068 | INFO | Perf report for the last 5.14196 seconds: | 20260506 17:07:25.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:07:25.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:25.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:25.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:25.068 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 17:07:25.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:25.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:25.068 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:07:25.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:25.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:25.068 | INFO | Uploaded 36GiB to GPU in 2.01072s (17.904GiB/s) | 20260506 17:07:25.068 | INFO | Downloaded 34GiB to GPU in 1.93417s (17.5786GiB/s) | 20260506 17:07:30.206 | INFO | Analyzing chunk frames = [3023872,3024384) | 20260506 17:07:30.206 | INFO | MemInfo: { | 20260506 17:07:30.206 | INFO | "gpu" : " 17 GB", | 20260506 17:07:30.206 | INFO | "huge" : "", | 20260506 17:07:30.206 | INFO | "huge_shared" : " 27 GB", | 20260506 17:07:30.206 | INFO | "malloc" : " 14 GB", | 20260506 17:07:30.206 | INFO | "pinned" : " 29 GB" | 20260506 17:07:30.206 | INFO | } | 20260506 17:07:30.206 | INFO | Perf report for the last 5.1388 seconds: | 20260506 17:07:30.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:07:30.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:30.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:30.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:30.206 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:07:30.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:30.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:30.206 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:07:30.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:30.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:30.207 | INFO | Uploaded 36GiB to GPU in 1.98333s (18.1513GiB/s) | 20260506 17:07:30.207 | INFO | Downloaded 34GiB to GPU in 1.92023s (17.7062GiB/s) | 20260506 17:07:35.337 | INFO | Analyzing chunk frames = [3024384,3024896) | 20260506 17:07:35.337 | INFO | MemInfo: { | 20260506 17:07:35.337 | INFO | "gpu" : " 17 GB", | 20260506 17:07:35.337 | INFO | "huge" : "", | 20260506 17:07:35.337 | INFO | "huge_shared" : " 27 GB", | 20260506 17:07:35.337 | INFO | "malloc" : " 14 GB", | 20260506 17:07:35.337 | INFO | "pinned" : " 29 GB" | 20260506 17:07:35.337 | INFO | } | 20260506 17:07:35.337 | INFO | Perf report for the last 5.13065 seconds: | 20260506 17:07:35.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:07:35.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:35.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:35.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:35.337 | INFO | ANALYSIS : 60.1%, 3.0, 92.9ms, 0 | 20260506 17:07:35.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:35.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:35.337 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 17:07:35.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:35.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:35.337 | INFO | Uploaded 36GiB to GPU in 1.93087s (18.6444GiB/s) | 20260506 17:07:35.337 | INFO | Downloaded 34GiB to GPU in 1.87254s (18.1572GiB/s) | 20260506 17:07:40.475 | INFO | Analyzing chunk frames = [3024896,3025408) | 20260506 17:07:40.475 | INFO | MemInfo: { | 20260506 17:07:40.475 | INFO | "gpu" : " 17 GB", | 20260506 17:07:40.475 | INFO | "huge" : "", | 20260506 17:07:40.475 | INFO | "huge_shared" : " 27 GB", | 20260506 17:07:40.475 | INFO | "malloc" : " 14 GB", | 20260506 17:07:40.475 | INFO | "pinned" : " 29 GB" | 20260506 17:07:40.475 | INFO | } | 20260506 17:07:40.475 | INFO | Perf report for the last 5.13792 seconds: | 20260506 17:07:40.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:07:40.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:40.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:40.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:40.475 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 17:07:40.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:40.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:40.475 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:07:40.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:40.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:40.475 | INFO | Uploaded 36GiB to GPU in 2.01534s (17.863GiB/s) | 20260506 17:07:40.475 | INFO | Downloaded 34GiB to GPU in 1.90675s (17.8314GiB/s) | 20260506 17:07:45.618 | INFO | Analyzing chunk frames = [3025408,3025920) | 20260506 17:07:45.618 | INFO | MemInfo: { | 20260506 17:07:45.618 | INFO | "gpu" : " 17 GB", | 20260506 17:07:45.618 | INFO | "huge" : "", | 20260506 17:07:45.618 | INFO | "huge_shared" : " 27 GB", | 20260506 17:07:45.618 | INFO | "malloc" : " 14 GB", | 20260506 17:07:45.618 | INFO | "pinned" : " 29 GB" | 20260506 17:07:45.618 | INFO | } | 20260506 17:07:45.619 | INFO | Perf report for the last 5.14365 seconds: | 20260506 17:07:45.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:07:45.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:45.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:45.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:45.619 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 17:07:45.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:45.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:45.619 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:07:45.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:45.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:45.619 | INFO | Uploaded 36GiB to GPU in 2.00836s (17.925GiB/s) | 20260506 17:07:45.619 | INFO | Downloaded 34GiB to GPU in 1.95168s (17.4209GiB/s) | 20260506 17:07:50.758 | INFO | Analyzing chunk frames = [3025920,3026432) | 20260506 17:07:50.758 | INFO | MemInfo: { | 20260506 17:07:50.758 | INFO | "gpu" : " 17 GB", | 20260506 17:07:50.758 | INFO | "huge" : "", | 20260506 17:07:50.758 | INFO | "huge_shared" : " 27 GB", | 20260506 17:07:50.758 | INFO | "malloc" : " 14 GB", | 20260506 17:07:50.758 | INFO | "pinned" : " 29 GB" | 20260506 17:07:50.758 | INFO | } | 20260506 17:07:50.758 | INFO | Perf report for the last 5.13912 seconds: | 20260506 17:07:50.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:07:50.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:50.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:50.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:50.758 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:07:50.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:50.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:50.758 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 17:07:50.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:50.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:50.758 | INFO | Uploaded 36GiB to GPU in 2.01768s (17.8423GiB/s) | 20260506 17:07:50.758 | INFO | Downloaded 34GiB to GPU in 2.00301s (16.9744GiB/s) | 20260506 17:07:55.898 | INFO | Analyzing chunk frames = [3026432,3026944) | 20260506 17:07:55.898 | INFO | MemInfo: { | 20260506 17:07:55.898 | INFO | "gpu" : " 17 GB", | 20260506 17:07:55.898 | INFO | "huge" : "", | 20260506 17:07:55.898 | INFO | "huge_shared" : " 27 GB", | 20260506 17:07:55.898 | INFO | "malloc" : " 14 GB", | 20260506 17:07:55.898 | INFO | "pinned" : " 29 GB" | 20260506 17:07:55.898 | INFO | } | 20260506 17:07:55.898 | INFO | Perf report for the last 5.14023 seconds: | 20260506 17:07:55.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:07:55.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:55.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:55.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:55.898 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 17:07:55.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:55.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:55.898 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260506 17:07:55.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:55.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:07:55.898 | INFO | Uploaded 36GiB to GPU in 2.00774s (17.9306GiB/s) | 20260506 17:07:55.898 | INFO | Downloaded 34GiB to GPU in 1.97518s (17.2136GiB/s) | 20260506 17:07:56.012 | INFO | Starting a round of Baz IO | 20260506 17:07:59.156 | INFO | Populating RTMetrics report | 20260506 17:07:59.156 | INFO | RTMetrics report populated | 20260506 17:07:59.156 | INFO | Opening tmp RTMetrics json file | 20260506 17:07:59.157 | INFO | Serializing RTMetrics to json | 20260506 17:07:59.159 | INFO | Serialization done, swapping tmp file now... | 20260506 17:07:59.160 | INFO | RTMetrics done | 20260506 17:07:59.285 | INFO | Finished a round of Baz IO: Wrote 2.07 GiB in 3.272 seconds | 20260506 17:07:59.285 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.85% / 0.01% / 64.15% | 20260506 17:07:59.285 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.46093% / 1.58946e-05% / 4.76837e-05% | 20260506 17:07:59.285 | INFO | Baz IO overall: | 20260506 17:07:59.285 | INFO | BytesWritten : 1593862441174 | 20260506 17:07:59.285 | INFO | NumEvents : 771155881843 | 20260506 17:08:01.037 | INFO | Analyzing chunk frames = [3026944,3027456) | 20260506 17:08:01.037 | INFO | MemInfo: { | 20260506 17:08:01.037 | INFO | "gpu" : " 17 GB", | 20260506 17:08:01.037 | INFO | "huge" : "", | 20260506 17:08:01.037 | INFO | "huge_shared" : " 27 GB", | 20260506 17:08:01.037 | INFO | "malloc" : " 14 GB", | 20260506 17:08:01.037 | INFO | "pinned" : " 29 GB" | 20260506 17:08:01.037 | INFO | } | 20260506 17:08:01.037 | INFO | Perf report for the last 5.13895 seconds: | 20260506 17:08:01.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:08:01.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:01.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:01.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:01.037 | INFO | ANALYSIS : 62.8%, 2.9, 95.3ms, 0 | 20260506 17:08:01.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:01.037 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260506 17:08:01.037 | INFO | PRE_HQ : 60.5%, 1.9, 58.7ms, 0 | 20260506 17:08:01.037 | INFO | BAZWRITER : 63.4%, 4.7, 153.0ms, 0 | 20260506 17:08:01.037 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 17:08:01.037 | INFO | Uploaded 36GiB to GPU in 2.05314s (17.5341GiB/s) | 20260506 17:08:01.037 | INFO | Downloaded 38GiB to GPU in 2.20268s (17.2517GiB/s) | 20260506 17:08:06.176 | INFO | Analyzing chunk frames = [3027456,3027968) | 20260506 17:08:06.176 | INFO | MemInfo: { | 20260506 17:08:06.176 | INFO | "gpu" : " 17 GB", | 20260506 17:08:06.176 | INFO | "huge" : "", | 20260506 17:08:06.176 | INFO | "huge_shared" : " 27 GB", | 20260506 17:08:06.176 | INFO | "malloc" : " 14 GB", | 20260506 17:08:06.176 | INFO | "pinned" : " 29 GB" | 20260506 17:08:06.176 | INFO | } | 20260506 17:08:06.176 | INFO | Perf report for the last 5.13887 seconds: | 20260506 17:08:06.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:08:06.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:06.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:06.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:06.176 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 17:08:06.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:06.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:06.176 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:08:06.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:06.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:06.176 | INFO | Uploaded 36GiB to GPU in 2.02128s (17.8105GiB/s) | 20260506 17:08:06.176 | INFO | Downloaded 34GiB to GPU in 1.97242s (17.2377GiB/s) | 20260506 17:08:11.312 | INFO | Analyzing chunk frames = [3027968,3028480) | 20260506 17:08:11.312 | INFO | MemInfo: { | 20260506 17:08:11.312 | INFO | "gpu" : " 17 GB", | 20260506 17:08:11.312 | INFO | "huge" : "", | 20260506 17:08:11.312 | INFO | "huge_shared" : " 27 GB", | 20260506 17:08:11.312 | INFO | "malloc" : " 14 GB", | 20260506 17:08:11.312 | INFO | "pinned" : " 29 GB" | 20260506 17:08:11.312 | INFO | } | 20260506 17:08:11.312 | INFO | Perf report for the last 5.13627 seconds: | 20260506 17:08:11.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:08:11.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:11.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:11.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:11.312 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 17:08:11.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:11.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:11.312 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:08:11.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:11.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:11.312 | INFO | Uploaded 36GiB to GPU in 1.98614s (18.1256GiB/s) | 20260506 17:08:11.312 | INFO | Downloaded 34GiB to GPU in 1.93229s (17.5957GiB/s) | 20260506 17:08:16.454 | INFO | Analyzing chunk frames = [3028480,3028992) | 20260506 17:08:16.454 | INFO | MemInfo: { | 20260506 17:08:16.454 | INFO | "gpu" : " 17 GB", | 20260506 17:08:16.454 | INFO | "huge" : "", | 20260506 17:08:16.454 | INFO | "huge_shared" : " 27 GB", | 20260506 17:08:16.454 | INFO | "malloc" : " 14 GB", | 20260506 17:08:16.454 | INFO | "pinned" : " 29 GB" | 20260506 17:08:16.455 | INFO | } | 20260506 17:08:16.455 | INFO | Perf report for the last 5.14247 seconds: | 20260506 17:08:16.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:08:16.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:16.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:16.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:16.455 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 17:08:16.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:16.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:16.455 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:08:16.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:16.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:16.455 | INFO | Uploaded 36GiB to GPU in 1.942s (18.5376GiB/s) | 20260506 17:08:16.455 | INFO | Downloaded 34GiB to GPU in 1.88272s (18.0589GiB/s) | 20260506 17:08:21.592 | INFO | Analyzing chunk frames = [3028992,3029504) | 20260506 17:08:21.592 | INFO | MemInfo: { | 20260506 17:08:21.592 | INFO | "gpu" : " 17 GB", | 20260506 17:08:21.592 | INFO | "huge" : "", | 20260506 17:08:21.592 | INFO | "huge_shared" : " 27 GB", | 20260506 17:08:21.592 | INFO | "malloc" : " 14 GB", | 20260506 17:08:21.592 | INFO | "pinned" : " 29 GB" | 20260506 17:08:21.592 | INFO | } | 20260506 17:08:21.592 | INFO | Perf report for the last 5.13717 seconds: | 20260506 17:08:21.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:08:21.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:21.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:21.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:21.592 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 17:08:21.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:21.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:21.592 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:08:21.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:21.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:21.592 | INFO | Uploaded 36GiB to GPU in 2.01612s (17.8561GiB/s) | 20260506 17:08:21.592 | INFO | Downloaded 34GiB to GPU in 1.90569s (17.8413GiB/s) | 20260506 17:08:26.728 | INFO | Analyzing chunk frames = [3029504,3030016) | 20260506 17:08:26.728 | INFO | MemInfo: { | 20260506 17:08:26.728 | INFO | "gpu" : " 17 GB", | 20260506 17:08:26.728 | INFO | "huge" : "", | 20260506 17:08:26.728 | INFO | "huge_shared" : " 27 GB", | 20260506 17:08:26.728 | INFO | "malloc" : " 14 GB", | 20260506 17:08:26.728 | INFO | "pinned" : " 29 GB" | 20260506 17:08:26.729 | INFO | } | 20260506 17:08:26.729 | INFO | Perf report for the last 5.13684 seconds: | 20260506 17:08:26.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:08:26.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:26.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:26.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:26.729 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 17:08:26.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:26.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:26.729 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260506 17:08:26.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:26.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:26.729 | INFO | Uploaded 36GiB to GPU in 2.02857s (17.7465GiB/s) | 20260506 17:08:26.729 | INFO | Downloaded 34GiB to GPU in 1.94462s (17.4841GiB/s) | 20260506 17:08:31.876 | INFO | Analyzing chunk frames = [3030016,3030528) | 20260506 17:08:31.876 | INFO | MemInfo: { | 20260506 17:08:31.876 | INFO | "gpu" : " 17 GB", | 20260506 17:08:31.877 | INFO | "huge" : "", | 20260506 17:08:31.877 | INFO | "huge_shared" : " 27 GB", | 20260506 17:08:31.877 | INFO | "malloc" : " 14 GB", | 20260506 17:08:31.877 | INFO | "pinned" : " 29 GB" | 20260506 17:08:31.877 | INFO | } | 20260506 17:08:31.877 | INFO | Perf report for the last 5.14803 seconds: | 20260506 17:08:31.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:08:31.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:31.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:31.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:31.877 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 17:08:31.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:31.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:31.877 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:08:31.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:31.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:31.877 | INFO | Uploaded 36GiB to GPU in 2.03183s (17.718GiB/s) | 20260506 17:08:31.877 | INFO | Downloaded 34GiB to GPU in 2.01762s (16.8515GiB/s) | 20260506 17:08:37.012 | INFO | Analyzing chunk frames = [3030528,3031040) | 20260506 17:08:37.012 | INFO | MemInfo: { | 20260506 17:08:37.012 | INFO | "gpu" : " 17 GB", | 20260506 17:08:37.013 | INFO | "huge" : "", | 20260506 17:08:37.013 | INFO | "huge_shared" : " 27 GB", | 20260506 17:08:37.013 | INFO | "malloc" : " 14 GB", | 20260506 17:08:37.013 | INFO | "pinned" : " 29 GB" | 20260506 17:08:37.013 | INFO | } | 20260506 17:08:37.013 | INFO | Perf report for the last 5.13607 seconds: | 20260506 17:08:37.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:08:37.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:37.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:37.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:37.013 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 17:08:37.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:37.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:37.013 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:08:37.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:37.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:37.013 | INFO | Uploaded 36GiB to GPU in 2.05844s (17.489GiB/s) | 20260506 17:08:37.013 | INFO | Downloaded 34GiB to GPU in 1.9757s (17.2091GiB/s) | 20260506 17:08:37.013 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:08:37.014 | INFO | Total mem usage: 18 GiB | 20260506 17:08:37.014 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:08:37.014 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -17.9%) | 20260506 17:08:37.014 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:08:37.014 | INFO | High Water Breakdown: | 20260506 17:08:37.014 | INFO | 61% : 9.32 GiB ( 81.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:08:37.014 | INFO | 38.9% : 5.93 GiB ( 88.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:08:37.014 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:08:37.014 | INFO | ------------------------------------------ | 20260506 17:08:37.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:08:37.014 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:08:37.014 | INFO | Total mem usage: 26.5 GiB | 20260506 17:08:37.014 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:08:37.014 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:08:37.014 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:08:37.014 | INFO | High Water Breakdown: | 20260506 17:08:37.014 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:08:37.014 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:08:37.014 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:08:37.014 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:08:37.014 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:08:37.014 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:08:37.014 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:08:37.014 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:08:37.014 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:08:37.014 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:08:37.014 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:08:37.014 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:08:37.014 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:08:37.014 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:08:37.014 | INFO | ------------------------------------------ | 20260506 17:08:37.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:08:37.014 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:08:37.014 | INFO | Total mem usage: 25.6 GiB | 20260506 17:08:37.014 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:08:37.014 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:08:37.014 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:08:37.014 | INFO | High Water Breakdown: | 20260506 17:08:37.014 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:08:37.014 | INFO | ------------------------------------------ | 20260506 17:08:37.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:08:37.014 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:08:37.014 | INFO | Total mem usage: 15.7 GiB | 20260506 17:08:37.014 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:08:37.014 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:08:37.014 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:08:37.014 | INFO | High Water Breakdown: | 20260506 17:08:37.014 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:08:37.014 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:08:37.014 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:08:37.014 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:08:37.014 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:08:37.014 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:08:37.014 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:08:37.014 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:08:37.014 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:08:37.014 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:08:37.014 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:08:37.014 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:08:37.014 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:08:37.014 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:08:37.014 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:08:37.014 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:08:37.014 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:08:37.014 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:08:37.014 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:08:37.014 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:08:37.014 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:08:37.014 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:08:37.014 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:08:37.014 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:08:37.014 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:08:37.014 | INFO | ------------------------------------------ | 20260506 17:08:37.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:08:37.014 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:08:37.014 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:08:37.014 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:08:37.014 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.63e+01, 1.36e+01 / 2.29e+00 | 20260506 17:08:37.014 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.02e-01, 1.03e-02 / 6.58e-03 | 20260506 17:08:37.014 | INFO | PerfReport: 11.61%, Baseline -- 3.64e+00 / 4.44e+00, 5.60e-01 / 2.90e-01 | 20260506 17:08:37.014 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.88e-01 / 1.32e-01 | 20260506 17:08:37.014 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.51e+01, 4.17e+00 / 2.22e+00 | 20260506 17:08:37.014 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.40e+00 / 1.85e+00 | 20260506 17:08:37.015 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.28e+00 / 2.85e+00, 3.83e-01 / 2.32e-01 | 20260506 17:08:37.015 | INFO | PerfReport: 3.46%, Metrics -- 1.08e+00 / 1.45e+00, 2.81e-01 / 1.61e-01 | 20260506 17:08:37.015 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.75e-01 / 4.27e-01, 5.88e-02 / 2.05e-02 | 20260506 17:08:37.015 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:08:37.015 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:08:37.015 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:08:37.015 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 17:08:37.015 | INFO | PerfReport: 21.53%, Upload -- 2.10e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 17:08:37.015 | INFO | PerfReport: 21.16%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 17:08:37.015 | INFO | PerfReport: 32.22%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 17:08:37.015 | INFO | PerfReport: 1.46%, LockUpload -- 1.42e+00 / 0.00e+00, 5.77e+00 / 0.00e+00 | 20260506 17:08:37.015 | INFO | PerfReport: 1.07%, LockDownload -- 1.04e+00 / 0.00e+00, 2.67e+00 / 0.00e+00 | 20260506 17:08:37.015 | INFO | PerfReport: 22.55%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 17:08:37.015 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260506 17:08:37.126 | INFO | Starting a round of Baz IO | 20260506 17:08:40.279 | INFO | Populating RTMetrics report | 20260506 17:08:40.279 | INFO | RTMetrics report populated | 20260506 17:08:40.279 | INFO | Opening tmp RTMetrics json file | 20260506 17:08:40.441 | INFO | Serializing RTMetrics to json | 20260506 17:08:40.630 | INFO | Serialization done, swapping tmp file now... | 20260506 17:08:40.666 | INFO | RTMetrics done | 20260506 17:08:40.686 | INFO | Finished a round of Baz IO: Wrote 2.071 GiB in 3.56 seconds | 20260506 17:08:40.686 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.84% / 0.01% / 64.15% | 20260506 17:08:40.686 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.45904% / 7.94729e-06% / 5.96046e-05% | 20260506 17:08:40.686 | INFO | Baz IO overall: | 20260506 17:08:40.687 | INFO | BytesWritten : 1596085804444 | 20260506 17:08:40.687 | INFO | NumEvents : 772229988073 | 20260506 17:08:42.152 | INFO | Analyzing chunk frames = [3031040,3031552) | 20260506 17:08:42.152 | INFO | MemInfo: { | 20260506 17:08:42.152 | INFO | "gpu" : " 17 GB", | 20260506 17:08:42.152 | INFO | "huge" : "", | 20260506 17:08:42.152 | INFO | "huge_shared" : " 27 GB", | 20260506 17:08:42.152 | INFO | "malloc" : " 14 GB", | 20260506 17:08:42.152 | INFO | "pinned" : " 29 GB" | 20260506 17:08:42.152 | INFO | } | 20260506 17:08:42.152 | INFO | Perf report for the last 5.13886 seconds: | 20260506 17:08:42.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:08:42.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:42.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:42.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:42.152 | INFO | ANALYSIS : 63.0%, 2.9, 95.5ms, 0 | 20260506 17:08:42.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:42.152 | INFO | RT_METRICS : 51.2%, 1.0, 26.9ms, 0 | 20260506 17:08:42.152 | INFO | PRE_HQ : 60.6%, 1.9, 58.6ms, 0 | 20260506 17:08:42.152 | INFO | BAZWRITER : 68.7%, 4.8, 170.1ms, 0 | 20260506 17:08:42.152 | INFO | BAZ_SAMPLING : 3.4%, 1.0, 1.8ms, 0 | 20260506 17:08:42.152 | INFO | Uploaded 36GiB to GPU in 2.04507s (17.6033GiB/s) | 20260506 17:08:42.152 | INFO | Downloaded 38GiB to GPU in 2.22614s (17.0699GiB/s) | 20260506 17:08:47.291 | INFO | Analyzing chunk frames = [3031552,3032064) | 20260506 17:08:47.291 | INFO | MemInfo: { | 20260506 17:08:47.291 | INFO | "gpu" : " 17 GB", | 20260506 17:08:47.291 | INFO | "huge" : "", | 20260506 17:08:47.291 | INFO | "huge_shared" : " 27 GB", | 20260506 17:08:47.291 | INFO | "malloc" : " 14 GB", | 20260506 17:08:47.291 | INFO | "pinned" : " 29 GB" | 20260506 17:08:47.291 | INFO | } | 20260506 17:08:47.291 | INFO | Perf report for the last 5.13984 seconds: | 20260506 17:08:47.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:08:47.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:47.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:47.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:47.292 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 17:08:47.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:47.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:47.292 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:08:47.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:47.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:47.292 | INFO | Uploaded 36GiB to GPU in 1.99102s (18.0812GiB/s) | 20260506 17:08:47.292 | INFO | Downloaded 34GiB to GPU in 1.92913s (17.6245GiB/s) | 20260506 17:08:52.426 | INFO | Analyzing chunk frames = [3032064,3032576) | 20260506 17:08:52.426 | INFO | MemInfo: { | 20260506 17:08:52.426 | INFO | "gpu" : " 17 GB", | 20260506 17:08:52.426 | INFO | "huge" : "", | 20260506 17:08:52.426 | INFO | "huge_shared" : " 27 GB", | 20260506 17:08:52.426 | INFO | "malloc" : " 14 GB", | 20260506 17:08:52.426 | INFO | "pinned" : " 29 GB" | 20260506 17:08:52.426 | INFO | } | 20260506 17:08:52.426 | INFO | Perf report for the last 5.13443 seconds: | 20260506 17:08:52.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:08:52.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:52.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:52.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:52.426 | INFO | ANALYSIS : 61.1%, 3.0, 94.4ms, 0 | 20260506 17:08:52.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:52.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:52.426 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:08:52.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:52.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:52.426 | INFO | Uploaded 36GiB to GPU in 1.97699s (18.2095GiB/s) | 20260506 17:08:52.426 | INFO | Downloaded 34GiB to GPU in 1.92411s (17.6706GiB/s) | 20260506 17:08:57.567 | INFO | Analyzing chunk frames = [3032576,3033088) | 20260506 17:08:57.567 | INFO | MemInfo: { | 20260506 17:08:57.567 | INFO | "gpu" : " 17 GB", | 20260506 17:08:57.567 | INFO | "huge" : "", | 20260506 17:08:57.567 | INFO | "huge_shared" : " 27 GB", | 20260506 17:08:57.567 | INFO | "malloc" : " 14 GB", | 20260506 17:08:57.567 | INFO | "pinned" : " 29 GB" | 20260506 17:08:57.567 | INFO | } | 20260506 17:08:57.568 | INFO | Perf report for the last 5.14164 seconds: | 20260506 17:08:57.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:08:57.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:57.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:57.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:57.568 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 17:08:57.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:57.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:57.568 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:08:57.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:57.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:08:57.568 | INFO | Uploaded 36GiB to GPU in 1.9307s (18.6461GiB/s) | 20260506 17:08:57.568 | INFO | Downloaded 34GiB to GPU in 1.87095s (18.1726GiB/s) | 20260506 17:09:02.709 | INFO | Analyzing chunk frames = [3033088,3033600) | 20260506 17:09:02.709 | INFO | MemInfo: { | 20260506 17:09:02.709 | INFO | "gpu" : " 17 GB", | 20260506 17:09:02.709 | INFO | "huge" : "", | 20260506 17:09:02.709 | INFO | "huge_shared" : " 27 GB", | 20260506 17:09:02.709 | INFO | "malloc" : " 14 GB", | 20260506 17:09:02.709 | INFO | "pinned" : " 29 GB" | 20260506 17:09:02.709 | INFO | } | 20260506 17:09:02.709 | INFO | Perf report for the last 5.14146 seconds: | 20260506 17:09:02.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:09:02.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:02.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:02.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:02.709 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 17:09:02.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:02.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:02.709 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:09:02.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:02.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:02.709 | INFO | Uploaded 36GiB to GPU in 1.95577s (18.4071GiB/s) | 20260506 17:09:02.709 | INFO | Downloaded 34GiB to GPU in 1.88344s (18.0521GiB/s) | 20260506 17:09:07.844 | INFO | Analyzing chunk frames = [3033600,3034112) | 20260506 17:09:07.844 | INFO | MemInfo: { | 20260506 17:09:07.844 | INFO | "gpu" : " 17 GB", | 20260506 17:09:07.844 | INFO | "huge" : "", | 20260506 17:09:07.844 | INFO | "huge_shared" : " 27 GB", | 20260506 17:09:07.844 | INFO | "malloc" : " 14 GB", | 20260506 17:09:07.844 | INFO | "pinned" : " 29 GB" | 20260506 17:09:07.844 | INFO | } | 20260506 17:09:07.844 | INFO | Perf report for the last 5.13525 seconds: | 20260506 17:09:07.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:09:07.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:07.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:07.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:07.844 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 17:09:07.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:07.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:07.844 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:09:07.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:07.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:07.845 | INFO | Uploaded 36GiB to GPU in 2.02411s (17.7856GiB/s) | 20260506 17:09:07.845 | INFO | Downloaded 34GiB to GPU in 1.94308s (17.498GiB/s) | 20260506 17:09:12.986 | INFO | Analyzing chunk frames = [3034112,3034624) | 20260506 17:09:12.986 | INFO | MemInfo: { | 20260506 17:09:12.986 | INFO | "gpu" : " 17 GB", | 20260506 17:09:12.986 | INFO | "huge" : "", | 20260506 17:09:12.986 | INFO | "huge_shared" : " 27 GB", | 20260506 17:09:12.986 | INFO | "malloc" : " 14 GB", | 20260506 17:09:12.986 | INFO | "pinned" : " 29 GB" | 20260506 17:09:12.986 | INFO | } | 20260506 17:09:12.986 | INFO | Perf report for the last 5.14212 seconds: | 20260506 17:09:12.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:09:12.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:12.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:12.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:12.987 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:09:12.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:12.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:12.987 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:09:12.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:12.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:12.987 | INFO | Uploaded 36GiB to GPU in 1.99003s (18.0902GiB/s) | 20260506 17:09:12.987 | INFO | Downloaded 34GiB to GPU in 2.00097s (16.9918GiB/s) | 20260506 17:09:18.128 | INFO | Analyzing chunk frames = [3034624,3035136) | 20260506 17:09:18.128 | INFO | MemInfo: { | 20260506 17:09:18.128 | INFO | "gpu" : " 17 GB", | 20260506 17:09:18.128 | INFO | "huge" : "", | 20260506 17:09:18.128 | INFO | "huge_shared" : " 27 GB", | 20260506 17:09:18.128 | INFO | "malloc" : " 14 GB", | 20260506 17:09:18.128 | INFO | "pinned" : " 29 GB" | 20260506 17:09:18.128 | INFO | } | 20260506 17:09:18.128 | INFO | Perf report for the last 5.1414 seconds: | 20260506 17:09:18.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:09:18.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:18.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:18.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:18.128 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:09:18.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:18.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:18.128 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 17:09:18.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:18.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:18.128 | INFO | Uploaded 36GiB to GPU in 1.99855s (18.0131GiB/s) | 20260506 17:09:18.128 | INFO | Downloaded 34GiB to GPU in 1.96944s (17.2638GiB/s) | 20260506 17:09:18.245 | INFO | Starting a round of Baz IO | 20260506 17:09:21.377 | INFO | Populating RTMetrics report | 20260506 17:09:21.378 | INFO | RTMetrics report populated | 20260506 17:09:21.378 | INFO | Opening tmp RTMetrics json file | 20260506 17:09:21.396 | INFO | Serializing RTMetrics to json | 20260506 17:09:21.399 | INFO | Serialization done, swapping tmp file now... | 20260506 17:09:21.403 | INFO | RTMetrics done | 20260506 17:09:22.041 | INFO | Finished a round of Baz IO: Wrote 2.068 GiB in 3.796 seconds | 20260506 17:09:22.041 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.83% / 0.01% / 64.16% | 20260506 17:09:22.041 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.45085% / 1.98682e-05% / 5.5631e-05% | 20260506 17:09:22.041 | INFO | Baz IO overall: | 20260506 17:09:22.041 | INFO | BytesWritten : 1598306400908 | 20260506 17:09:22.041 | INFO | NumEvents : 773301660985 | 20260506 17:09:23.272 | INFO | Analyzing chunk frames = [3035136,3035648) | 20260506 17:09:23.272 | INFO | MemInfo: { | 20260506 17:09:23.272 | INFO | "gpu" : " 17 GB", | 20260506 17:09:23.272 | INFO | "huge" : "", | 20260506 17:09:23.272 | INFO | "huge_shared" : " 27 GB", | 20260506 17:09:23.272 | INFO | "malloc" : " 14 GB", | 20260506 17:09:23.272 | INFO | "pinned" : " 29 GB" | 20260506 17:09:23.272 | INFO | } | 20260506 17:09:23.272 | INFO | Perf report for the last 5.14452 seconds: | 20260506 17:09:23.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:09:23.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:23.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:23.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:23.273 | INFO | ANALYSIS : 62.6%, 2.9, 95.2ms, 0 | 20260506 17:09:23.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:23.273 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 17:09:23.273 | INFO | PRE_HQ : 60.1%, 1.9, 58.4ms, 0 | 20260506 17:09:23.273 | INFO | BAZWRITER : 73.1%, 4.9, 183.0ms, 0 | 20260506 17:09:23.273 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 17:09:23.273 | INFO | Uploaded 36GiB to GPU in 2.04454s (17.6079GiB/s) | 20260506 17:09:23.273 | INFO | Downloaded 38GiB to GPU in 2.20849s (17.2063GiB/s) | 20260506 17:09:28.404 | INFO | Analyzing chunk frames = [3035648,3036160) | 20260506 17:09:28.404 | INFO | MemInfo: { | 20260506 17:09:28.404 | INFO | "gpu" : " 17 GB", | 20260506 17:09:28.404 | INFO | "huge" : "", | 20260506 17:09:28.404 | INFO | "huge_shared" : " 27 GB", | 20260506 17:09:28.404 | INFO | "malloc" : " 14 GB", | 20260506 17:09:28.404 | INFO | "pinned" : " 29 GB" | 20260506 17:09:28.404 | INFO | } | 20260506 17:09:28.404 | INFO | Perf report for the last 5.13165 seconds: | 20260506 17:09:28.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:09:28.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:28.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:28.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:28.404 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:09:28.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:28.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:28.404 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:09:28.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:28.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:28.404 | INFO | Uploaded 36GiB to GPU in 2.00701s (17.9372GiB/s) | 20260506 17:09:28.404 | INFO | Downloaded 34GiB to GPU in 1.93619s (17.5602GiB/s) | 20260506 17:09:33.545 | INFO | Analyzing chunk frames = [3036160,3036672) | 20260506 17:09:33.545 | INFO | MemInfo: { | 20260506 17:09:33.545 | INFO | "gpu" : " 17 GB", | 20260506 17:09:33.545 | INFO | "huge" : "", | 20260506 17:09:33.545 | INFO | "huge_shared" : " 27 GB", | 20260506 17:09:33.545 | INFO | "malloc" : " 14 GB", | 20260506 17:09:33.545 | INFO | "pinned" : " 29 GB" | 20260506 17:09:33.545 | INFO | } | 20260506 17:09:33.545 | INFO | Perf report for the last 5.14061 seconds: | 20260506 17:09:33.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:09:33.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:33.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:33.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:33.545 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 17:09:33.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:33.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:33.545 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:09:33.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:33.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:33.545 | INFO | Uploaded 36GiB to GPU in 1.97799s (18.2003GiB/s) | 20260506 17:09:33.545 | INFO | Downloaded 34GiB to GPU in 1.92685s (17.6454GiB/s) | 20260506 17:09:38.681 | INFO | Analyzing chunk frames = [3036672,3037184) | 20260506 17:09:38.681 | INFO | MemInfo: { | 20260506 17:09:38.681 | INFO | "gpu" : " 17 GB", | 20260506 17:09:38.681 | INFO | "huge" : "", | 20260506 17:09:38.681 | INFO | "huge_shared" : " 27 GB", | 20260506 17:09:38.682 | INFO | "malloc" : " 14 GB", | 20260506 17:09:38.682 | INFO | "pinned" : " 29 GB" | 20260506 17:09:38.682 | INFO | } | 20260506 17:09:38.682 | INFO | Perf report for the last 5.13711 seconds: | 20260506 17:09:38.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:09:38.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:38.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:38.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:38.682 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260506 17:09:38.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:38.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:38.682 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 17:09:38.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:38.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:38.682 | INFO | Uploaded 36GiB to GPU in 1.93159s (18.6375GiB/s) | 20260506 17:09:38.682 | INFO | Downloaded 34GiB to GPU in 1.8715s (18.1672GiB/s) | 20260506 17:09:43.820 | INFO | Analyzing chunk frames = [3037184,3037696) | 20260506 17:09:43.820 | INFO | MemInfo: { | 20260506 17:09:43.820 | INFO | "gpu" : " 17 GB", | 20260506 17:09:43.820 | INFO | "huge" : "", | 20260506 17:09:43.820 | INFO | "huge_shared" : " 27 GB", | 20260506 17:09:43.820 | INFO | "malloc" : " 14 GB", | 20260506 17:09:43.820 | INFO | "pinned" : " 29 GB" | 20260506 17:09:43.820 | INFO | } | 20260506 17:09:43.820 | INFO | Perf report for the last 5.13798 seconds: | 20260506 17:09:43.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:09:43.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:43.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:43.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:43.820 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 17:09:43.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:43.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:43.820 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:09:43.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:43.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:43.820 | INFO | Uploaded 36GiB to GPU in 1.95936s (18.3734GiB/s) | 20260506 17:09:43.820 | INFO | Downloaded 34GiB to GPU in 1.87583s (18.1253GiB/s) | 20260506 17:09:48.965 | INFO | Analyzing chunk frames = [3037696,3038208) | 20260506 17:09:48.965 | INFO | MemInfo: { | 20260506 17:09:48.965 | INFO | "gpu" : " 17 GB", | 20260506 17:09:48.965 | INFO | "huge" : "", | 20260506 17:09:48.965 | INFO | "huge_shared" : " 27 GB", | 20260506 17:09:48.965 | INFO | "malloc" : " 14 GB", | 20260506 17:09:48.965 | INFO | "pinned" : " 29 GB" | 20260506 17:09:48.965 | INFO | } | 20260506 17:09:48.965 | INFO | Perf report for the last 5.14571 seconds: | 20260506 17:09:48.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:09:48.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:48.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:48.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:48.966 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 17:09:48.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:48.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:48.966 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260506 17:09:48.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:48.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:48.966 | INFO | Uploaded 36GiB to GPU in 2.02559s (17.7726GiB/s) | 20260506 17:09:48.966 | INFO | Downloaded 34GiB to GPU in 1.94099s (17.5168GiB/s) | 20260506 17:09:54.099 | INFO | Analyzing chunk frames = [3038208,3038720) | 20260506 17:09:54.099 | INFO | MemInfo: { | 20260506 17:09:54.099 | INFO | "gpu" : " 17 GB", | 20260506 17:09:54.099 | INFO | "huge" : "", | 20260506 17:09:54.099 | INFO | "huge_shared" : " 27 GB", | 20260506 17:09:54.099 | INFO | "malloc" : " 14 GB", | 20260506 17:09:54.099 | INFO | "pinned" : " 29 GB" | 20260506 17:09:54.099 | INFO | } | 20260506 17:09:54.099 | INFO | Perf report for the last 5.13369 seconds: | 20260506 17:09:54.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:09:54.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:54.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:54.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:54.099 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 17:09:54.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:54.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:54.099 | INFO | PRE_HQ : 27.9%, 1.0, 14.4ms, 0 | 20260506 17:09:54.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:54.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:54.099 | INFO | Uploaded 36GiB to GPU in 2.01811s (17.8385GiB/s) | 20260506 17:09:54.099 | INFO | Downloaded 34GiB to GPU in 1.99409s (17.0504GiB/s) | 20260506 17:09:59.240 | INFO | Analyzing chunk frames = [3038720,3039232) | 20260506 17:09:59.240 | INFO | MemInfo: { | 20260506 17:09:59.240 | INFO | "gpu" : " 17 GB", | 20260506 17:09:59.240 | INFO | "huge" : "", | 20260506 17:09:59.240 | INFO | "huge_shared" : " 27 GB", | 20260506 17:09:59.240 | INFO | "malloc" : " 14 GB", | 20260506 17:09:59.240 | INFO | "pinned" : " 29 GB" | 20260506 17:09:59.240 | INFO | } | 20260506 17:09:59.241 | INFO | Perf report for the last 5.14141 seconds: | 20260506 17:09:59.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:09:59.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:59.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:59.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:59.241 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 17:09:59.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:59.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:59.241 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260506 17:09:59.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:59.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:09:59.241 | INFO | Uploaded 36GiB to GPU in 2.04322s (17.6193GiB/s) | 20260506 17:09:59.241 | INFO | Downloaded 34GiB to GPU in 1.97838s (17.1858GiB/s) | 20260506 17:09:59.241 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:09:59.241 | INFO | Total mem usage: 18 GiB | 20260506 17:09:59.241 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:09:59.241 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: -9.28%) | 20260506 17:09:59.241 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:09:59.241 | INFO | High Water Breakdown: | 20260506 17:09:59.241 | INFO | 62.2% : 10.2 GiB ( 89.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:09:59.241 | INFO | 37.7% : 6.21 GiB ( 92.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:09:59.241 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:09:59.241 | INFO | ------------------------------------------ | 20260506 17:09:59.241 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:09:59.241 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:09:59.241 | INFO | Total mem usage: 26.5 GiB | 20260506 17:09:59.241 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:09:59.241 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:09:59.241 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:09:59.241 | INFO | High Water Breakdown: | 20260506 17:09:59.241 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:09:59.241 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:09:59.241 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:09:59.241 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:09:59.242 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:09:59.242 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:09:59.242 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:09:59.242 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:09:59.242 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:09:59.242 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:09:59.242 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:09:59.242 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:09:59.242 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:09:59.242 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:09:59.242 | INFO | ------------------------------------------ | 20260506 17:09:59.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:09:59.242 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:09:59.242 | INFO | Total mem usage: 25.6 GiB | 20260506 17:09:59.242 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:09:59.242 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:09:59.242 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:09:59.242 | INFO | High Water Breakdown: | 20260506 17:09:59.242 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:09:59.242 | INFO | ------------------------------------------ | 20260506 17:09:59.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:09:59.242 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:09:59.242 | INFO | Total mem usage: 15.7 GiB | 20260506 17:09:59.242 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:09:59.242 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:09:59.242 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 17:09:59.242 | INFO | High Water Breakdown: | 20260506 17:09:59.242 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:09:59.242 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:09:59.242 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:09:59.242 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:09:59.242 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:09:59.242 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:09:59.242 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:09:59.242 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:09:59.242 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:09:59.242 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:09:59.242 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:09:59.242 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:09:59.242 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:09:59.242 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:09:59.242 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:09:59.242 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:09:59.242 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:09:59.242 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:09:59.242 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:09:59.242 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:09:59.242 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:09:59.243 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:09:59.243 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:09:59.243 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:09:59.243 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:09:59.243 | INFO | ------------------------------------------ | 20260506 17:09:59.243 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:09:59.243 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:09:59.243 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:09:59.243 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:09:59.243 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.53e+01, 1.34e+01 / 2.23e+00 | 20260506 17:09:59.243 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 9.97e-02, 1.02e-02 / 6.05e-03 | 20260506 17:09:59.243 | INFO | PerfReport: 11.62%, Baseline -- 3.64e+00 / 4.38e+00, 5.63e-01 / 4.02e-01 | 20260506 17:09:59.243 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.43e+00, 1.82e-01 / 1.64e-01 | 20260506 17:09:59.243 | INFO | PerfReport: 15.94%, DME -- 4.00e+01 / 4.44e+01, 3.95e+00 / 2.46e+00 | 20260506 17:09:59.243 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.42e+00 / 2.06e+00 | 20260506 17:09:59.243 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.28e+00 / 2.90e+00, 3.93e-01 / 2.10e-01 | 20260506 17:09:59.243 | INFO | PerfReport: 3.45%, Metrics -- 1.08e+00 / 1.45e+00, 2.80e-01 / 1.59e-01 | 20260506 17:09:59.243 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.75e-01 / 4.25e-01, 5.82e-02 / 2.27e-02 | 20260506 17:09:59.243 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:09:59.243 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:09:59.243 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:09:59.243 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 17:09:59.243 | INFO | PerfReport: 21.39%, Upload -- 2.08e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260506 17:09:59.243 | INFO | PerfReport: 21.10%, Download -- 2.05e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260506 17:09:59.243 | INFO | PerfReport: 32.25%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:09:59.243 | INFO | PerfReport: 1.41%, LockUpload -- 1.37e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260506 17:09:59.243 | INFO | PerfReport: 1.11%, LockDownload -- 1.07e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260506 17:09:59.243 | INFO | PerfReport: 22.72%, LockCompute -- 2.21e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 17:09:59.243 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260506 17:09:59.354 | INFO | Starting a round of Baz IO | 20260506 17:10:02.497 | INFO | Populating RTMetrics report | 20260506 17:10:02.497 | INFO | RTMetrics report populated | 20260506 17:10:02.497 | INFO | Opening tmp RTMetrics json file | 20260506 17:10:02.691 | INFO | Serializing RTMetrics to json | 20260506 17:10:02.723 | INFO | Serialization done, swapping tmp file now... | 20260506 17:10:02.726 | INFO | RTMetrics done | 20260506 17:10:03.093 | INFO | Finished a round of Baz IO: Wrote 2.068 GiB in 3.739 seconds | 20260506 17:10:03.093 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.82% / 0.01% / 64.17% | 20260506 17:10:03.093 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.44809% / 1.19209e-05% / 8.34465e-05% | 20260506 17:10:03.093 | INFO | Baz IO overall: | 20260506 17:10:03.093 | INFO | BytesWritten : 1600527180926 | 20260506 17:10:03.093 | INFO | NumEvents : 774373691507 | 20260506 17:10:04.381 | INFO | Analyzing chunk frames = [3039232,3039744) | 20260506 17:10:04.381 | INFO | MemInfo: { | 20260506 17:10:04.381 | INFO | "gpu" : " 17 GB", | 20260506 17:10:04.381 | INFO | "huge" : "", | 20260506 17:10:04.381 | INFO | "huge_shared" : " 27 GB", | 20260506 17:10:04.381 | INFO | "malloc" : " 14 GB", | 20260506 17:10:04.381 | INFO | "pinned" : " 29 GB" | 20260506 17:10:04.381 | INFO | } | 20260506 17:10:04.381 | INFO | Perf report for the last 5.14015 seconds: | 20260506 17:10:04.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:10:04.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:04.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:04.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:04.381 | INFO | ANALYSIS : 62.8%, 2.9, 95.2ms, 0 | 20260506 17:10:04.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:04.381 | INFO | RT_METRICS : 48.0%, 1.0, 25.2ms, 0 | 20260506 17:10:04.381 | INFO | PRE_HQ : 60.5%, 1.9, 58.7ms, 0 | 20260506 17:10:04.381 | INFO | BAZWRITER : 72.1%, 4.8, 177.6ms, 0 | 20260506 17:10:04.381 | INFO | BAZ_SAMPLING : 3.6%, 1.0, 1.8ms, 0 | 20260506 17:10:04.381 | INFO | Uploaded 36GiB to GPU in 2.05364s (17.5299GiB/s) | 20260506 17:10:04.381 | INFO | Downloaded 38GiB to GPU in 2.22466s (17.0813GiB/s) | 20260506 17:10:09.520 | INFO | Analyzing chunk frames = [3039744,3040256) | 20260506 17:10:09.520 | INFO | MemInfo: { | 20260506 17:10:09.520 | INFO | "gpu" : " 17 GB", | 20260506 17:10:09.520 | INFO | "huge" : "", | 20260506 17:10:09.520 | INFO | "huge_shared" : " 27 GB", | 20260506 17:10:09.520 | INFO | "malloc" : " 14 GB", | 20260506 17:10:09.520 | INFO | "pinned" : " 29 GB" | 20260506 17:10:09.520 | INFO | } | 20260506 17:10:09.520 | INFO | Perf report for the last 5.1394 seconds: | 20260506 17:10:09.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:10:09.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:09.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:09.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:09.520 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:10:09.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:09.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:09.520 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:10:09.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:09.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:09.520 | INFO | Uploaded 36GiB to GPU in 2.00643s (17.9423GiB/s) | 20260506 17:10:09.520 | INFO | Downloaded 34GiB to GPU in 1.94794s (17.4543GiB/s) | 20260506 17:10:14.657 | INFO | Analyzing chunk frames = [3040256,3040768) | 20260506 17:10:14.657 | INFO | MemInfo: { | 20260506 17:10:14.657 | INFO | "gpu" : " 17 GB", | 20260506 17:10:14.657 | INFO | "huge" : "", | 20260506 17:10:14.657 | INFO | "huge_shared" : " 27 GB", | 20260506 17:10:14.657 | INFO | "malloc" : " 14 GB", | 20260506 17:10:14.657 | INFO | "pinned" : " 29 GB" | 20260506 17:10:14.657 | INFO | } | 20260506 17:10:14.657 | INFO | Perf report for the last 5.13727 seconds: | 20260506 17:10:14.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:10:14.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:14.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:14.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:14.658 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 17:10:14.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:14.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:14.658 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:10:14.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:14.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:14.658 | INFO | Uploaded 36GiB to GPU in 1.97791s (18.2011GiB/s) | 20260506 17:10:14.658 | INFO | Downloaded 34GiB to GPU in 1.92317s (17.6792GiB/s) | 20260506 17:10:19.797 | INFO | Analyzing chunk frames = [3040768,3041280) | 20260506 17:10:19.797 | INFO | MemInfo: { | 20260506 17:10:19.797 | INFO | "gpu" : " 17 GB", | 20260506 17:10:19.797 | INFO | "huge" : "", | 20260506 17:10:19.797 | INFO | "huge_shared" : " 27 GB", | 20260506 17:10:19.797 | INFO | "malloc" : " 14 GB", | 20260506 17:10:19.797 | INFO | "pinned" : " 29 GB" | 20260506 17:10:19.797 | INFO | } | 20260506 17:10:19.797 | INFO | Perf report for the last 5.13972 seconds: | 20260506 17:10:19.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:10:19.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:19.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:19.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:19.797 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 17:10:19.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:19.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:19.797 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:10:19.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:19.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:19.797 | INFO | Uploaded 36GiB to GPU in 1.92741s (18.6779GiB/s) | 20260506 17:10:19.797 | INFO | Downloaded 34GiB to GPU in 1.87332s (18.1496GiB/s) | 20260506 17:10:24.935 | INFO | Analyzing chunk frames = [3041280,3041792) | 20260506 17:10:24.935 | INFO | MemInfo: { | 20260506 17:10:24.935 | INFO | "gpu" : " 17 GB", | 20260506 17:10:24.935 | INFO | "huge" : "", | 20260506 17:10:24.935 | INFO | "huge_shared" : " 27 GB", | 20260506 17:10:24.935 | INFO | "malloc" : " 14 GB", | 20260506 17:10:24.935 | INFO | "pinned" : " 29 GB" | 20260506 17:10:24.935 | INFO | } | 20260506 17:10:24.935 | INFO | Perf report for the last 5.13798 seconds: | 20260506 17:10:24.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:10:24.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:24.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:24.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:24.935 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 17:10:24.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:24.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:24.935 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:10:24.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:24.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:24.935 | INFO | Uploaded 36GiB to GPU in 2.01982s (17.8233GiB/s) | 20260506 17:10:24.936 | INFO | Downloaded 34GiB to GPU in 1.91498s (17.7547GiB/s) | 20260506 17:10:30.076 | INFO | Analyzing chunk frames = [3041792,3042304) | 20260506 17:10:30.076 | INFO | MemInfo: { | 20260506 17:10:30.076 | INFO | "gpu" : " 17 GB", | 20260506 17:10:30.076 | INFO | "huge" : "", | 20260506 17:10:30.076 | INFO | "huge_shared" : " 27 GB", | 20260506 17:10:30.076 | INFO | "malloc" : " 14 GB", | 20260506 17:10:30.076 | INFO | "pinned" : " 29 GB" | 20260506 17:10:30.076 | INFO | } | 20260506 17:10:30.076 | INFO | Perf report for the last 5.14112 seconds: | 20260506 17:10:30.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:10:30.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:30.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:30.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:30.076 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:10:30.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:30.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:30.076 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:10:30.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:30.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:30.077 | INFO | Uploaded 36GiB to GPU in 2.05985s (17.477GiB/s) | 20260506 17:10:30.077 | INFO | Downloaded 34GiB to GPU in 1.95693s (17.3742GiB/s) | 20260506 17:10:35.221 | INFO | Analyzing chunk frames = [3042304,3042816) | 20260506 17:10:35.221 | INFO | MemInfo: { | 20260506 17:10:35.221 | INFO | "gpu" : " 17 GB", | 20260506 17:10:35.221 | INFO | "huge" : "", | 20260506 17:10:35.221 | INFO | "huge_shared" : " 27 GB", | 20260506 17:10:35.221 | INFO | "malloc" : " 14 GB", | 20260506 17:10:35.221 | INFO | "pinned" : " 29 GB" | 20260506 17:10:35.221 | INFO | } | 20260506 17:10:35.222 | INFO | Perf report for the last 5.1452 seconds: | 20260506 17:10:35.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:10:35.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:35.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:35.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:35.222 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 17:10:35.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:35.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:35.222 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 17:10:35.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:35.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:35.222 | INFO | Uploaded 36GiB to GPU in 2.0301s (17.7331GiB/s) | 20260506 17:10:35.222 | INFO | Downloaded 34GiB to GPU in 1.99792s (17.0177GiB/s) | 20260506 17:10:40.361 | INFO | Analyzing chunk frames = [3042816,3043328) | 20260506 17:10:40.361 | INFO | MemInfo: { | 20260506 17:10:40.361 | INFO | "gpu" : " 17 GB", | 20260506 17:10:40.361 | INFO | "huge" : "", | 20260506 17:10:40.361 | INFO | "huge_shared" : " 27 GB", | 20260506 17:10:40.361 | INFO | "malloc" : " 14 GB", | 20260506 17:10:40.361 | INFO | "pinned" : " 29 GB" | 20260506 17:10:40.361 | INFO | } | 20260506 17:10:40.361 | INFO | Perf report for the last 5.1393 seconds: | 20260506 17:10:40.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:10:40.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:40.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:40.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:40.361 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 17:10:40.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:40.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:40.361 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 17:10:40.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:40.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:40.361 | INFO | Uploaded 36GiB to GPU in 2.03797s (17.6646GiB/s) | 20260506 17:10:40.361 | INFO | Downloaded 34GiB to GPU in 1.98633s (17.117GiB/s) | 20260506 17:10:40.481 | INFO | Starting a round of Baz IO | 20260506 17:10:43.621 | INFO | Populating RTMetrics report | 20260506 17:10:43.621 | INFO | RTMetrics report populated | 20260506 17:10:43.621 | INFO | Opening tmp RTMetrics json file | 20260506 17:10:43.625 | INFO | Serializing RTMetrics to json | 20260506 17:10:43.626 | INFO | Serialization done, swapping tmp file now... | 20260506 17:10:43.629 | INFO | RTMetrics done | 20260506 17:10:44.014 | INFO | Finished a round of Baz IO: Wrote 2.068 GiB in 3.532 seconds | 20260506 17:10:44.014 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.82% / 0.01% / 64.18% | 20260506 17:10:44.014 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.44698% / 1.19209e-05% / 7.94729e-05% | 20260506 17:10:44.014 | INFO | Baz IO overall: | 20260506 17:10:44.014 | INFO | BytesWritten : 1602748199720 | 20260506 17:10:44.014 | INFO | NumEvents : 775445884738 | 20260506 17:10:45.495 | INFO | Analyzing chunk frames = [3043328,3043840) | 20260506 17:10:45.495 | INFO | MemInfo: { | 20260506 17:10:45.495 | INFO | "gpu" : " 17 GB", | 20260506 17:10:45.495 | INFO | "huge" : "", | 20260506 17:10:45.495 | INFO | "huge_shared" : " 27 GB", | 20260506 17:10:45.495 | INFO | "malloc" : " 14 GB", | 20260506 17:10:45.495 | INFO | "pinned" : " 29 GB" | 20260506 17:10:45.495 | INFO | } | 20260506 17:10:45.495 | INFO | Perf report for the last 5.13447 seconds: | 20260506 17:10:45.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:10:45.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:45.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:45.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:45.495 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 17:10:45.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:45.495 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 17:10:45.495 | INFO | PRE_HQ : 60.6%, 1.9, 58.2ms, 0 | 20260506 17:10:45.495 | INFO | BAZWRITER : 68.4%, 4.8, 168.6ms, 0 | 20260506 17:10:45.495 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 17:10:45.496 | INFO | Uploaded 36GiB to GPU in 2.05924s (17.4822GiB/s) | 20260506 17:10:45.496 | INFO | Downloaded 38GiB to GPU in 2.22125s (17.1075GiB/s) | 20260506 17:10:50.635 | INFO | Analyzing chunk frames = [3043840,3044352) | 20260506 17:10:50.635 | INFO | MemInfo: { | 20260506 17:10:50.635 | INFO | "gpu" : " 17 GB", | 20260506 17:10:50.635 | INFO | "huge" : "", | 20260506 17:10:50.635 | INFO | "huge_shared" : " 27 GB", | 20260506 17:10:50.635 | INFO | "malloc" : " 14 GB", | 20260506 17:10:50.635 | INFO | "pinned" : " 29 GB" | 20260506 17:10:50.635 | INFO | } | 20260506 17:10:50.635 | INFO | Perf report for the last 5.13986 seconds: | 20260506 17:10:50.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:10:50.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:50.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:50.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:50.635 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 17:10:50.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:50.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:50.635 | INFO | PRE_HQ : 27.4%, 1.0, 14.4ms, 0 | 20260506 17:10:50.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:50.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:50.635 | INFO | Uploaded 36GiB to GPU in 2.01266s (17.8868GiB/s) | 20260506 17:10:50.636 | INFO | Downloaded 34GiB to GPU in 1.93654s (17.5571GiB/s) | 20260506 17:10:55.767 | INFO | Analyzing chunk frames = [3044352,3044864) | 20260506 17:10:55.767 | INFO | MemInfo: { | 20260506 17:10:55.767 | INFO | "gpu" : " 17 GB", | 20260506 17:10:55.767 | INFO | "huge" : "", | 20260506 17:10:55.767 | INFO | "huge_shared" : " 27 GB", | 20260506 17:10:55.768 | INFO | "malloc" : " 14 GB", | 20260506 17:10:55.768 | INFO | "pinned" : " 29 GB" | 20260506 17:10:55.768 | INFO | } | 20260506 17:10:55.768 | INFO | Perf report for the last 5.13243 seconds: | 20260506 17:10:55.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:10:55.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:55.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:55.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:55.768 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 17:10:55.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:55.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:55.768 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:10:55.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:55.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:10:55.768 | INFO | Uploaded 36GiB to GPU in 1.97306s (18.2457GiB/s) | 20260506 17:10:55.768 | INFO | Downloaded 34GiB to GPU in 1.92494s (17.6629GiB/s) | 20260506 17:11:00.908 | INFO | Analyzing chunk frames = [3044864,3045376) | 20260506 17:11:00.908 | INFO | MemInfo: { | 20260506 17:11:00.908 | INFO | "gpu" : " 17 GB", | 20260506 17:11:00.908 | INFO | "huge" : "", | 20260506 17:11:00.908 | INFO | "huge_shared" : " 27 GB", | 20260506 17:11:00.908 | INFO | "malloc" : " 14 GB", | 20260506 17:11:00.908 | INFO | "pinned" : " 29 GB" | 20260506 17:11:00.908 | INFO | } | 20260506 17:11:00.908 | INFO | Perf report for the last 5.14083 seconds: | 20260506 17:11:00.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:11:00.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:00.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:00.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:00.908 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 17:11:00.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:00.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:00.909 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:11:00.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:00.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:00.909 | INFO | Uploaded 36GiB to GPU in 1.93013s (18.6516GiB/s) | 20260506 17:11:00.909 | INFO | Downloaded 34GiB to GPU in 1.88057s (18.0796GiB/s) | 20260506 17:11:06.055 | INFO | Analyzing chunk frames = [3045376,3045888) | 20260506 17:11:06.055 | INFO | MemInfo: { | 20260506 17:11:06.055 | INFO | "gpu" : " 17 GB", | 20260506 17:11:06.055 | INFO | "huge" : "", | 20260506 17:11:06.055 | INFO | "huge_shared" : " 27 GB", | 20260506 17:11:06.055 | INFO | "malloc" : " 14 GB", | 20260506 17:11:06.055 | INFO | "pinned" : " 29 GB" | 20260506 17:11:06.055 | INFO | } | 20260506 17:11:06.055 | INFO | Perf report for the last 5.14643 seconds: | 20260506 17:11:06.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:11:06.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:06.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:06.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:06.055 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 17:11:06.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:06.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:06.055 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:11:06.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:06.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:06.055 | INFO | Uploaded 36GiB to GPU in 2.01807s (17.8388GiB/s) | 20260506 17:11:06.055 | INFO | Downloaded 34GiB to GPU in 1.91406s (17.7633GiB/s) | 20260506 17:11:11.194 | INFO | Analyzing chunk frames = [3045888,3046400) | 20260506 17:11:11.194 | INFO | MemInfo: { | 20260506 17:11:11.194 | INFO | "gpu" : " 17 GB", | 20260506 17:11:11.194 | INFO | "huge" : "", | 20260506 17:11:11.194 | INFO | "huge_shared" : " 27 GB", | 20260506 17:11:11.194 | INFO | "malloc" : " 14 GB", | 20260506 17:11:11.194 | INFO | "pinned" : " 29 GB" | 20260506 17:11:11.194 | INFO | } | 20260506 17:11:11.194 | INFO | Perf report for the last 5.13892 seconds: | 20260506 17:11:11.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:11:11.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:11.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:11.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:11.194 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 17:11:11.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:11.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:11.194 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 17:11:11.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:11.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:11.194 | INFO | Uploaded 36GiB to GPU in 2.04168s (17.6326GiB/s) | 20260506 17:11:11.194 | INFO | Downloaded 34GiB to GPU in 1.95122s (17.425GiB/s) | 20260506 17:11:16.332 | INFO | Analyzing chunk frames = [3046400,3046912) | 20260506 17:11:16.333 | INFO | MemInfo: { | 20260506 17:11:16.333 | INFO | "gpu" : " 17 GB", | 20260506 17:11:16.333 | INFO | "huge" : "", | 20260506 17:11:16.333 | INFO | "huge_shared" : " 27 GB", | 20260506 17:11:16.333 | INFO | "malloc" : " 14 GB", | 20260506 17:11:16.333 | INFO | "pinned" : " 29 GB" | 20260506 17:11:16.333 | INFO | } | 20260506 17:11:16.333 | INFO | Perf report for the last 5.13889 seconds: | 20260506 17:11:16.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:11:16.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:16.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:16.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:16.333 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:11:16.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:16.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:16.333 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 17:11:16.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:16.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:16.333 | INFO | Uploaded 36GiB to GPU in 2.02125s (17.8108GiB/s) | 20260506 17:11:16.333 | INFO | Downloaded 34GiB to GPU in 1.99185s (17.0696GiB/s) | 20260506 17:11:21.476 | INFO | Analyzing chunk frames = [3046912,3047424) | 20260506 17:11:21.476 | INFO | MemInfo: { | 20260506 17:11:21.476 | INFO | "gpu" : " 17 GB", | 20260506 17:11:21.476 | INFO | "huge" : "", | 20260506 17:11:21.476 | INFO | "huge_shared" : " 27 GB", | 20260506 17:11:21.476 | INFO | "malloc" : " 14 GB", | 20260506 17:11:21.476 | INFO | "pinned" : " 29 GB" | 20260506 17:11:21.476 | INFO | } | 20260506 17:11:21.476 | INFO | Perf report for the last 5.14316 seconds: | 20260506 17:11:21.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:11:21.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:21.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:21.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:21.476 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 17:11:21.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:21.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:21.476 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 17:11:21.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:21.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:21.476 | INFO | Uploaded 36GiB to GPU in 2.04876s (17.5716GiB/s) | 20260506 17:11:21.476 | INFO | Downloaded 34GiB to GPU in 1.97812s (17.188GiB/s) | 20260506 17:11:21.476 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:11:21.476 | INFO | Total mem usage: 18 GiB | 20260506 17:11:21.476 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:11:21.476 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -10%) | 20260506 17:11:21.476 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:11:21.476 | INFO | High Water Breakdown: | 20260506 17:11:21.476 | INFO | 62% : 10.2 GiB ( 89% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:11:21.476 | INFO | 37.8% : 6.19 GiB ( 92.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:11:21.476 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:11:21.476 | INFO | ------------------------------------------ | 20260506 17:11:21.476 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:11:21.477 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:11:21.477 | INFO | Total mem usage: 26.5 GiB | 20260506 17:11:21.477 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:11:21.477 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:11:21.477 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:11:21.477 | INFO | High Water Breakdown: | 20260506 17:11:21.477 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:11:21.477 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:11:21.477 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:11:21.477 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:11:21.477 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:11:21.477 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:11:21.477 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:11:21.477 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:11:21.477 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:11:21.477 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:11:21.477 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:11:21.477 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:11:21.477 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:11:21.477 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:11:21.477 | INFO | ------------------------------------------ | 20260506 17:11:21.477 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:11:21.477 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:11:21.477 | INFO | Total mem usage: 25.6 GiB | 20260506 17:11:21.477 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:11:21.477 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:11:21.477 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:11:21.477 | INFO | High Water Breakdown: | 20260506 17:11:21.477 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:11:21.477 | INFO | ------------------------------------------ | 20260506 17:11:21.477 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:11:21.477 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:11:21.477 | INFO | Total mem usage: 15.7 GiB | 20260506 17:11:21.477 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:11:21.477 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:11:21.477 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 17:11:21.477 | INFO | High Water Breakdown: | 20260506 17:11:21.477 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:11:21.477 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:11:21.477 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:11:21.477 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:11:21.477 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:11:21.477 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:11:21.477 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:11:21.477 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:11:21.477 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:11:21.477 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:11:21.477 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:11:21.477 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:11:21.477 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:11:21.477 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:11:21.477 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:11:21.477 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:11:21.477 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:11:21.477 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:11:21.477 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:11:21.477 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:11:21.477 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:11:21.477 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:11:21.477 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:11:21.477 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:11:21.477 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:11:21.477 | INFO | ------------------------------------------ | 20260506 17:11:21.477 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:11:21.477 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:11:21.477 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:11:21.477 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:11:21.477 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.57e+01, 1.33e+01 / 1.98e+00 | 20260506 17:11:21.477 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 9.99e-02, 9.90e-03 / 5.56e-03 | 20260506 17:11:21.477 | INFO | PerfReport: 11.60%, Baseline -- 3.64e+00 / 4.44e+00, 5.56e-01 / 3.15e-01 | 20260506 17:11:21.477 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.85e-01 / 1.43e-01 | 20260506 17:11:21.477 | INFO | PerfReport: 15.94%, DME -- 4.00e+01 / 4.46e+01, 3.88e+00 / 2.01e+00 | 20260506 17:11:21.477 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.49e+00 / 2.12e+00 | 20260506 17:11:21.477 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.27e+00 / 2.87e+00, 3.77e-01 / 2.52e-01 | 20260506 17:11:21.477 | INFO | PerfReport: 3.44%, Metrics -- 1.08e+00 / 1.41e+00, 2.79e-01 / 2.04e-01 | 20260506 17:11:21.477 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.75e-01 / 4.18e-01, 5.82e-02 / 2.45e-02 | 20260506 17:11:21.477 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:11:21.477 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:11:21.477 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:11:21.477 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 17:11:21.477 | INFO | PerfReport: 21.55%, Upload -- 2.10e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 17:11:21.477 | INFO | PerfReport: 21.15%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 17:11:21.477 | INFO | PerfReport: 32.23%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 17:11:21.477 | INFO | PerfReport: 1.46%, LockUpload -- 1.42e+00 / 0.00e+00, 5.73e+00 / 0.00e+00 | 20260506 17:11:21.477 | INFO | PerfReport: 1.02%, LockDownload -- 9.97e-01 / 0.00e+00, 2.59e+00 / 0.00e+00 | 20260506 17:11:21.478 | INFO | PerfReport: 22.58%, LockCompute -- 2.20e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 17:11:21.478 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.13e-02 / 0.00e+00 | 20260506 17:11:21.595 | INFO | Starting a round of Baz IO | 20260506 17:11:24.739 | INFO | Populating RTMetrics report | 20260506 17:11:24.739 | INFO | RTMetrics report populated | 20260506 17:11:24.739 | INFO | Opening tmp RTMetrics json file | 20260506 17:11:24.916 | INFO | Serializing RTMetrics to json | 20260506 17:11:24.919 | INFO | Serialization done, swapping tmp file now... | 20260506 17:11:24.924 | INFO | RTMetrics done | 20260506 17:11:25.411 | INFO | Finished a round of Baz IO: Wrote 2.066 GiB in 3.815 seconds | 20260506 17:11:25.411 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.81% / 0.01% / 64.18% | 20260506 17:11:25.411 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.43974% / 1.58946e-05% / 7.15256e-05% | 20260506 17:11:25.411 | INFO | Baz IO overall: | 20260506 17:11:25.411 | INFO | BytesWritten : 1604966901379 | 20260506 17:11:25.411 | INFO | NumEvents : 776516498757 | 20260506 17:11:26.609 | INFO | Analyzing chunk frames = [3047424,3047936) | 20260506 17:11:26.609 | INFO | MemInfo: { | 20260506 17:11:26.609 | INFO | "gpu" : " 17 GB", | 20260506 17:11:26.609 | INFO | "huge" : "", | 20260506 17:11:26.609 | INFO | "huge_shared" : " 27 GB", | 20260506 17:11:26.609 | INFO | "malloc" : " 14 GB", | 20260506 17:11:26.609 | INFO | "pinned" : " 29 GB" | 20260506 17:11:26.609 | INFO | } | 20260506 17:11:26.609 | INFO | Perf report for the last 5.13316 seconds: | 20260506 17:11:26.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:11:26.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:26.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:26.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:26.609 | INFO | ANALYSIS : 63.0%, 2.9, 95.0ms, 0 | 20260506 17:11:26.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:26.609 | INFO | RT_METRICS : 47.4%, 1.0, 24.8ms, 0 | 20260506 17:11:26.609 | INFO | PRE_HQ : 60.7%, 1.9, 59.1ms, 0 | 20260506 17:11:26.609 | INFO | BAZWRITER : 73.9%, 4.8, 182.8ms, 0 | 20260506 17:11:26.609 | INFO | BAZ_SAMPLING : 3.5%, 1.0, 1.8ms, 0 | 20260506 17:11:26.610 | INFO | Uploaded 36GiB to GPU in 2.05733s (17.4984GiB/s) | 20260506 17:11:26.610 | INFO | Downloaded 38GiB to GPU in 2.21482s (17.1571GiB/s) | 20260506 17:11:31.746 | INFO | Analyzing chunk frames = [3047936,3048448) | 20260506 17:11:31.746 | INFO | MemInfo: { | 20260506 17:11:31.746 | INFO | "gpu" : " 17 GB", | 20260506 17:11:31.746 | INFO | "huge" : "", | 20260506 17:11:31.746 | INFO | "huge_shared" : " 27 GB", | 20260506 17:11:31.746 | INFO | "malloc" : " 14 GB", | 20260506 17:11:31.746 | INFO | "pinned" : " 29 GB" | 20260506 17:11:31.746 | INFO | } | 20260506 17:11:31.746 | INFO | Perf report for the last 5.13688 seconds: | 20260506 17:11:31.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:11:31.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:31.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:31.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:31.746 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:11:31.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:31.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:31.746 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 17:11:31.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:31.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:31.746 | INFO | Uploaded 36GiB to GPU in 1.99468s (18.048GiB/s) | 20260506 17:11:31.747 | INFO | Downloaded 34GiB to GPU in 1.93122s (17.6055GiB/s) | 20260506 17:11:36.885 | INFO | Analyzing chunk frames = [3048448,3048960) | 20260506 17:11:36.885 | INFO | MemInfo: { | 20260506 17:11:36.885 | INFO | "gpu" : " 17 GB", | 20260506 17:11:36.885 | INFO | "huge" : "", | 20260506 17:11:36.885 | INFO | "huge_shared" : " 27 GB", | 20260506 17:11:36.885 | INFO | "malloc" : " 14 GB", | 20260506 17:11:36.885 | INFO | "pinned" : " 29 GB" | 20260506 17:11:36.885 | INFO | } | 20260506 17:11:36.886 | INFO | Perf report for the last 5.13949 seconds: | 20260506 17:11:36.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:11:36.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:36.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:36.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:36.886 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 17:11:36.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:36.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:36.886 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 17:11:36.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:36.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:36.886 | INFO | Uploaded 36GiB to GPU in 1.97479s (18.2298GiB/s) | 20260506 17:11:36.886 | INFO | Downloaded 34GiB to GPU in 1.92728s (17.6415GiB/s) | 20260506 17:11:42.026 | INFO | Analyzing chunk frames = [3048960,3049472) | 20260506 17:11:42.026 | INFO | MemInfo: { | 20260506 17:11:42.026 | INFO | "gpu" : " 17 GB", | 20260506 17:11:42.026 | INFO | "huge" : "", | 20260506 17:11:42.026 | INFO | "huge_shared" : " 27 GB", | 20260506 17:11:42.026 | INFO | "malloc" : " 14 GB", | 20260506 17:11:42.026 | INFO | "pinned" : " 29 GB" | 20260506 17:11:42.026 | INFO | } | 20260506 17:11:42.026 | INFO | Perf report for the last 5.14078 seconds: | 20260506 17:11:42.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:11:42.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:42.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:42.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:42.026 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 17:11:42.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:42.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:42.026 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:11:42.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:42.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:42.027 | INFO | Uploaded 36GiB to GPU in 1.93735s (18.5821GiB/s) | 20260506 17:11:42.027 | INFO | Downloaded 34GiB to GPU in 1.87674s (18.1166GiB/s) | 20260506 17:11:47.169 | INFO | Analyzing chunk frames = [3049472,3049984) | 20260506 17:11:47.169 | INFO | MemInfo: { | 20260506 17:11:47.169 | INFO | "gpu" : " 17 GB", | 20260506 17:11:47.169 | INFO | "huge" : "", | 20260506 17:11:47.169 | INFO | "huge_shared" : " 27 GB", | 20260506 17:11:47.169 | INFO | "malloc" : " 14 GB", | 20260506 17:11:47.169 | INFO | "pinned" : " 29 GB" | 20260506 17:11:47.169 | INFO | } | 20260506 17:11:47.169 | INFO | Perf report for the last 5.1428 seconds: | 20260506 17:11:47.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:11:47.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:47.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:47.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:47.169 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 17:11:47.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:47.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:47.169 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:11:47.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:47.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:47.169 | INFO | Uploaded 36GiB to GPU in 1.98237s (18.1601GiB/s) | 20260506 17:11:47.169 | INFO | Downloaded 34GiB to GPU in 1.91566s (17.7484GiB/s) | 20260506 17:11:52.314 | INFO | Analyzing chunk frames = [3049984,3050496) | 20260506 17:11:52.314 | INFO | MemInfo: { | 20260506 17:11:52.314 | INFO | "gpu" : " 17 GB", | 20260506 17:11:52.314 | INFO | "huge" : "", | 20260506 17:11:52.314 | INFO | "huge_shared" : " 27 GB", | 20260506 17:11:52.314 | INFO | "malloc" : " 14 GB", | 20260506 17:11:52.314 | INFO | "pinned" : " 29 GB" | 20260506 17:11:52.314 | INFO | } | 20260506 17:11:52.314 | INFO | Perf report for the last 5.14502 seconds: | 20260506 17:11:52.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:11:52.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:52.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:52.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:52.314 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 17:11:52.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:52.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:52.314 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:11:52.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:52.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:52.315 | INFO | Uploaded 36GiB to GPU in 2.00261s (17.9766GiB/s) | 20260506 17:11:52.315 | INFO | Downloaded 34GiB to GPU in 1.95553s (17.3866GiB/s) | 20260506 17:11:57.451 | INFO | Analyzing chunk frames = [3050496,3051008) | 20260506 17:11:57.451 | INFO | MemInfo: { | 20260506 17:11:57.451 | INFO | "gpu" : " 17 GB", | 20260506 17:11:57.451 | INFO | "huge" : "", | 20260506 17:11:57.451 | INFO | "huge_shared" : " 27 GB", | 20260506 17:11:57.451 | INFO | "malloc" : " 14 GB", | 20260506 17:11:57.451 | INFO | "pinned" : " 29 GB" | 20260506 17:11:57.451 | INFO | } | 20260506 17:11:57.451 | INFO | Perf report for the last 5.13708 seconds: | 20260506 17:11:57.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:11:57.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:57.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:57.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:57.451 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:11:57.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:57.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:57.451 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:11:57.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:57.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:11:57.452 | INFO | Uploaded 36GiB to GPU in 2.02846s (17.7474GiB/s) | 20260506 17:11:57.452 | INFO | Downloaded 34GiB to GPU in 1.99486s (17.0438GiB/s) | 20260506 17:12:02.588 | INFO | Analyzing chunk frames = [3051008,3051520) | 20260506 17:12:02.588 | INFO | MemInfo: { | 20260506 17:12:02.588 | INFO | "gpu" : " 17 GB", | 20260506 17:12:02.589 | INFO | "huge" : "", | 20260506 17:12:02.589 | INFO | "huge_shared" : " 27 GB", | 20260506 17:12:02.589 | INFO | "malloc" : " 14 GB", | 20260506 17:12:02.589 | INFO | "pinned" : " 29 GB" | 20260506 17:12:02.589 | INFO | } | 20260506 17:12:02.589 | INFO | Perf report for the last 5.13738 seconds: | 20260506 17:12:02.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:12:02.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:02.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:02.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:02.589 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 17:12:02.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:02.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:02.589 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:12:02.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:02.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:02.589 | INFO | Uploaded 36GiB to GPU in 2.04774s (17.5804GiB/s) | 20260506 17:12:02.589 | INFO | Downloaded 34GiB to GPU in 1.97749s (17.1935GiB/s) | 20260506 17:12:02.718 | INFO | Starting a round of Baz IO | 20260506 17:12:05.841 | INFO | Populating RTMetrics report | 20260506 17:12:05.842 | INFO | RTMetrics report populated | 20260506 17:12:05.842 | INFO | Opening tmp RTMetrics json file | 20260506 17:12:05.898 | INFO | Serializing RTMetrics to json | 20260506 17:12:06.134 | INFO | Serialization done, swapping tmp file now... | 20260506 17:12:06.208 | INFO | RTMetrics done | 20260506 17:12:06.753 | INFO | Finished a round of Baz IO: Wrote 2.068 GiB in 4.035 seconds | 20260506 17:12:06.753 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.80% / 0.01% / 64.19% | 20260506 17:12:06.753 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.43878% / 3.97364e-06% / 7.15256e-05% | 20260506 17:12:06.753 | INFO | Baz IO overall: | 20260506 17:12:06.753 | INFO | BytesWritten : 1607187112409 | 20260506 17:12:06.753 | INFO | NumEvents : 777588109609 | 20260506 17:12:07.725 | INFO | Analyzing chunk frames = [3051520,3052032) | 20260506 17:12:07.725 | INFO | MemInfo: { | 20260506 17:12:07.725 | INFO | "gpu" : " 17 GB", | 20260506 17:12:07.725 | INFO | "huge" : "", | 20260506 17:12:07.725 | INFO | "huge_shared" : " 27 GB", | 20260506 17:12:07.725 | INFO | "malloc" : " 14 GB", | 20260506 17:12:07.725 | INFO | "pinned" : " 29 GB" | 20260506 17:12:07.725 | INFO | } | 20260506 17:12:07.725 | INFO | Perf report for the last 5.13652 seconds: | 20260506 17:12:07.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:12:07.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:07.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:07.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:07.725 | INFO | ANALYSIS : 62.7%, 2.9, 95.3ms, 0 | 20260506 17:12:07.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:07.725 | INFO | RT_METRICS : 50.8%, 1.0, 26.6ms, 0 | 20260506 17:12:07.725 | INFO | PRE_HQ : 60.5%, 1.9, 58.3ms, 0 | 20260506 17:12:07.725 | INFO | BAZWRITER : 78.6%, 4.8, 193.0ms, 0 | 20260506 17:12:07.725 | INFO | BAZ_SAMPLING : 5.3%, 1.0, 2.7ms, 0 | 20260506 17:12:07.725 | INFO | Uploaded 36GiB to GPU in 2.06043s (17.4721GiB/s) | 20260506 17:12:07.726 | INFO | Downloaded 38GiB to GPU in 2.22878s (17.0497GiB/s) | 20260506 17:12:12.861 | INFO | Analyzing chunk frames = [3052032,3052544) | 20260506 17:12:12.861 | INFO | MemInfo: { | 20260506 17:12:12.861 | INFO | "gpu" : " 17 GB", | 20260506 17:12:12.861 | INFO | "huge" : "", | 20260506 17:12:12.861 | INFO | "huge_shared" : " 27 GB", | 20260506 17:12:12.861 | INFO | "malloc" : " 14 GB", | 20260506 17:12:12.861 | INFO | "pinned" : " 29 GB" | 20260506 17:12:12.861 | INFO | } | 20260506 17:12:12.861 | INFO | Perf report for the last 5.136 seconds: | 20260506 17:12:12.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:12:12.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:12.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:12.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:12.861 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 17:12:12.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:12.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:12.861 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:12:12.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:12.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:12.861 | INFO | Uploaded 36GiB to GPU in 2.01608s (17.8564GiB/s) | 20260506 17:12:12.862 | INFO | Downloaded 34GiB to GPU in 1.94709s (17.4619GiB/s) | 20260506 17:12:18.000 | INFO | Analyzing chunk frames = [3052544,3053056) | 20260506 17:12:18.000 | INFO | MemInfo: { | 20260506 17:12:18.000 | INFO | "gpu" : " 17 GB", | 20260506 17:12:18.000 | INFO | "huge" : "", | 20260506 17:12:18.000 | INFO | "huge_shared" : " 27 GB", | 20260506 17:12:18.000 | INFO | "malloc" : " 14 GB", | 20260506 17:12:18.000 | INFO | "pinned" : " 29 GB" | 20260506 17:12:18.000 | INFO | } | 20260506 17:12:18.000 | INFO | Perf report for the last 5.1391 seconds: | 20260506 17:12:18.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:12:18.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:18.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:18.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:18.000 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 17:12:18.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:18.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:18.000 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:12:18.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:18.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:18.001 | INFO | Uploaded 36GiB to GPU in 1.98037s (18.1784GiB/s) | 20260506 17:12:18.001 | INFO | Downloaded 34GiB to GPU in 1.93597s (17.5622GiB/s) | 20260506 17:12:23.144 | INFO | Analyzing chunk frames = [3053056,3053568) | 20260506 17:12:23.144 | INFO | MemInfo: { | 20260506 17:12:23.144 | INFO | "gpu" : " 17 GB", | 20260506 17:12:23.144 | INFO | "huge" : "", | 20260506 17:12:23.144 | INFO | "huge_shared" : " 27 GB", | 20260506 17:12:23.144 | INFO | "malloc" : " 14 GB", | 20260506 17:12:23.144 | INFO | "pinned" : " 29 GB" | 20260506 17:12:23.144 | INFO | } | 20260506 17:12:23.144 | INFO | Perf report for the last 5.14404 seconds: | 20260506 17:12:23.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:12:23.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:23.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:23.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:23.144 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260506 17:12:23.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:23.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:23.144 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:12:23.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:23.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:23.145 | INFO | Uploaded 36GiB to GPU in 1.92793s (18.6729GiB/s) | 20260506 17:12:23.145 | INFO | Downloaded 34GiB to GPU in 1.874s (18.143GiB/s) | 20260506 17:12:28.283 | INFO | Analyzing chunk frames = [3053568,3054080) | 20260506 17:12:28.283 | INFO | MemInfo: { | 20260506 17:12:28.283 | INFO | "gpu" : " 17 GB", | 20260506 17:12:28.283 | INFO | "huge" : "", | 20260506 17:12:28.283 | INFO | "huge_shared" : " 27 GB", | 20260506 17:12:28.283 | INFO | "malloc" : " 14 GB", | 20260506 17:12:28.283 | INFO | "pinned" : " 29 GB" | 20260506 17:12:28.283 | INFO | } | 20260506 17:12:28.283 | INFO | Perf report for the last 5.13852 seconds: | 20260506 17:12:28.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:12:28.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:28.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:28.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:28.283 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 17:12:28.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:28.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:28.283 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:12:28.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:28.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:28.283 | INFO | Uploaded 36GiB to GPU in 1.96974s (18.2765GiB/s) | 20260506 17:12:28.283 | INFO | Downloaded 34GiB to GPU in 1.91957s (17.7123GiB/s) | 20260506 17:12:33.423 | INFO | Analyzing chunk frames = [3054080,3054592) | 20260506 17:12:33.423 | INFO | MemInfo: { | 20260506 17:12:33.423 | INFO | "gpu" : " 17 GB", | 20260506 17:12:33.423 | INFO | "huge" : "", | 20260506 17:12:33.423 | INFO | "huge_shared" : " 27 GB", | 20260506 17:12:33.423 | INFO | "malloc" : " 14 GB", | 20260506 17:12:33.423 | INFO | "pinned" : " 29 GB" | 20260506 17:12:33.423 | INFO | } | 20260506 17:12:33.423 | INFO | Perf report for the last 5.14059 seconds: | 20260506 17:12:33.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:12:33.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:33.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:33.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:33.424 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 17:12:33.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:33.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:33.424 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:12:33.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:33.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:33.424 | INFO | Uploaded 36GiB to GPU in 2.0401s (17.6462GiB/s) | 20260506 17:12:33.424 | INFO | Downloaded 34GiB to GPU in 1.94545s (17.4766GiB/s) | 20260506 17:12:38.561 | INFO | Analyzing chunk frames = [3054592,3055104) | 20260506 17:12:38.561 | INFO | MemInfo: { | 20260506 17:12:38.561 | INFO | "gpu" : " 17 GB", | 20260506 17:12:38.561 | INFO | "huge" : "", | 20260506 17:12:38.561 | INFO | "huge_shared" : " 27 GB", | 20260506 17:12:38.561 | INFO | "malloc" : " 14 GB", | 20260506 17:12:38.561 | INFO | "pinned" : " 29 GB" | 20260506 17:12:38.561 | INFO | } | 20260506 17:12:38.561 | INFO | Perf report for the last 5.13773 seconds: | 20260506 17:12:38.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:12:38.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:38.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:38.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:38.561 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 17:12:38.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:38.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:38.562 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 17:12:38.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:38.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:38.562 | INFO | Uploaded 36GiB to GPU in 2.02201s (17.8041GiB/s) | 20260506 17:12:38.562 | INFO | Downloaded 34GiB to GPU in 2.01639s (16.8618GiB/s) | 20260506 17:12:43.701 | INFO | Analyzing chunk frames = [3055104,3055616) | 20260506 17:12:43.701 | INFO | MemInfo: { | 20260506 17:12:43.701 | INFO | "gpu" : " 17 GB", | 20260506 17:12:43.701 | INFO | "huge" : "", | 20260506 17:12:43.701 | INFO | "huge_shared" : " 27 GB", | 20260506 17:12:43.701 | INFO | "malloc" : " 14 GB", | 20260506 17:12:43.701 | INFO | "pinned" : " 29 GB" | 20260506 17:12:43.701 | INFO | } | 20260506 17:12:43.701 | INFO | Perf report for the last 5.13938 seconds: | 20260506 17:12:43.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:12:43.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:43.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:43.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:43.701 | INFO | ANALYSIS : 62.5%, 3.0, 97.4ms, 0 | 20260506 17:12:43.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:43.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:43.701 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:12:43.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:43.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:43.701 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:12:43.701 | INFO | Uploaded 36GiB to GPU in 2.09068s (17.2193GiB/s) | 20260506 17:12:43.701 | INFO | Total mem usage: 18 GiB | 20260506 17:12:43.701 | INFO | Downloaded 34GiB to GPU in 2.00999s (16.9155GiB/s) | 20260506 17:12:43.701 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:12:43.701 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -17.3%) | 20260506 17:12:43.701 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:12:43.701 | INFO | High Water Breakdown: | 20260506 17:12:43.701 | INFO | 61.1% : 9.37 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:12:43.701 | INFO | 38.8% : 5.96 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:12:43.701 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:12:43.701 | INFO | ------------------------------------------ | 20260506 17:12:43.701 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:12:43.701 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:12:43.701 | INFO | Total mem usage: 26.5 GiB | 20260506 17:12:43.701 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:12:43.701 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:12:43.701 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:12:43.701 | INFO | High Water Breakdown: | 20260506 17:12:43.701 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:12:43.701 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:12:43.701 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:12:43.701 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:12:43.701 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:12:43.701 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:12:43.701 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:12:43.701 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:12:43.701 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:12:43.701 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:12:43.701 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:12:43.701 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:12:43.702 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:12:43.702 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:12:43.702 | INFO | ------------------------------------------ | 20260506 17:12:43.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:12:43.702 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:12:43.702 | INFO | Total mem usage: 25.6 GiB | 20260506 17:12:43.702 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:12:43.702 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:12:43.702 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:12:43.702 | INFO | High Water Breakdown: | 20260506 17:12:43.702 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:12:43.702 | INFO | ------------------------------------------ | 20260506 17:12:43.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:12:43.702 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:12:43.702 | INFO | Total mem usage: 15.7 GiB | 20260506 17:12:43.702 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:12:43.702 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:12:43.702 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 17:12:43.702 | INFO | High Water Breakdown: | 20260506 17:12:43.702 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:12:43.702 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:12:43.702 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:12:43.702 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:12:43.702 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:12:43.702 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:12:43.702 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:12:43.702 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:12:43.702 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:12:43.702 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:12:43.702 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:12:43.702 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:12:43.702 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:12:43.702 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:12:43.702 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:12:43.702 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:12:43.702 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:12:43.702 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:12:43.702 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:12:43.702 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:12:43.702 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:12:43.702 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:12:43.702 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:12:43.702 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:12:43.702 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:12:43.702 | INFO | ------------------------------------------ | 20260506 17:12:43.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:12:43.702 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:12:43.702 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:12:43.702 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:12:43.702 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.59e+01, 1.33e+01 / 2.46e+00 | 20260506 17:12:43.702 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 9.98e-03 / 7.13e-03 | 20260506 17:12:43.702 | INFO | PerfReport: 11.60%, Baseline -- 3.64e+00 / 4.38e+00, 5.52e-01 / 3.42e-01 | 20260506 17:12:43.702 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.83e-01 / 1.19e-01 | 20260506 17:12:43.702 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.49e+01, 4.19e+00 / 3.07e+00 | 20260506 17:12:43.702 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.79e+01 / 2.03e+01, 1.43e+00 / 1.84e+00 | 20260506 17:12:43.702 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.28e+00 / 2.88e+00, 3.87e-01 / 2.13e-01 | 20260506 17:12:43.702 | INFO | PerfReport: 3.46%, Metrics -- 1.08e+00 / 1.46e+00, 2.82e-01 / 1.68e-01 | 20260506 17:12:43.702 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.75e-01 / 4.29e-01, 5.82e-02 / 2.60e-02 | 20260506 17:12:43.702 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:12:43.702 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:12:43.702 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:12:43.702 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 17:12:43.702 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 17:12:43.702 | INFO | PerfReport: 21.19%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 17:12:43.702 | INFO | PerfReport: 32.23%, Compute -- 3.14e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 17:12:43.702 | INFO | PerfReport: 1.38%, LockUpload -- 1.34e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260506 17:12:43.702 | INFO | PerfReport: 1.14%, LockDownload -- 1.11e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260506 17:12:43.702 | INFO | PerfReport: 22.55%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 17:12:43.702 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260506 17:12:43.837 | INFO | Starting a round of Baz IO | 20260506 17:12:46.970 | INFO | Populating RTMetrics report | 20260506 17:12:46.971 | INFO | RTMetrics report populated | 20260506 17:12:46.971 | INFO | Opening tmp RTMetrics json file | 20260506 17:12:47.045 | INFO | Serializing RTMetrics to json | 20260506 17:12:47.047 | INFO | Serialization done, swapping tmp file now... | 20260506 17:12:47.047 | INFO | RTMetrics done | 20260506 17:12:47.324 | INFO | Finished a round of Baz IO: Wrote 2.08 GiB in 3.487 seconds | 20260506 17:12:47.324 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.79% / 0.01% / 64.20% | 20260506 17:12:47.324 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.45065% / 1.19209e-05% / 7.15256e-05% | 20260506 17:12:47.324 | INFO | Baz IO overall: | 20260506 17:12:47.324 | INFO | BytesWritten : 1609420619078 | 20260506 17:12:47.324 | INFO | NumEvents : 778665182308 | 20260506 17:12:48.835 | INFO | Analyzing chunk frames = [3055616,3056128) | 20260506 17:12:48.835 | INFO | MemInfo: { | 20260506 17:12:48.835 | INFO | "gpu" : " 17 GB", | 20260506 17:12:48.835 | INFO | "huge" : "", | 20260506 17:12:48.835 | INFO | "huge_shared" : " 27 GB", | 20260506 17:12:48.835 | INFO | "malloc" : " 14 GB", | 20260506 17:12:48.835 | INFO | "pinned" : " 29 GB" | 20260506 17:12:48.835 | INFO | } | 20260506 17:12:48.835 | INFO | Perf report for the last 5.13464 seconds: | 20260506 17:12:48.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:12:48.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:48.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:48.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:48.835 | INFO | ANALYSIS : 63.1%, 2.9, 95.6ms, 0 | 20260506 17:12:48.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:48.835 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 17:12:48.835 | INFO | PRE_HQ : 60.7%, 1.9, 59.2ms, 0 | 20260506 17:12:48.836 | INFO | BAZWRITER : 67.5%, 4.8, 166.9ms, 0 | 20260506 17:12:48.836 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 17:12:48.836 | INFO | Uploaded 36GiB to GPU in 2.07047s (17.3874GiB/s) | 20260506 17:12:48.836 | INFO | Downloaded 38GiB to GPU in 2.22828s (17.0535GiB/s) | 20260506 17:12:53.975 | INFO | Analyzing chunk frames = [3056128,3056640) | 20260506 17:12:53.975 | INFO | MemInfo: { | 20260506 17:12:53.975 | INFO | "gpu" : " 17 GB", | 20260506 17:12:53.975 | INFO | "huge" : "", | 20260506 17:12:53.975 | INFO | "huge_shared" : " 27 GB", | 20260506 17:12:53.975 | INFO | "malloc" : " 14 GB", | 20260506 17:12:53.975 | INFO | "pinned" : " 29 GB" | 20260506 17:12:53.975 | INFO | } | 20260506 17:12:53.975 | INFO | Perf report for the last 5.13987 seconds: | 20260506 17:12:53.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:12:53.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:53.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:53.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:53.975 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 17:12:53.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:53.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:53.975 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:12:53.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:53.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:53.976 | INFO | Uploaded 36GiB to GPU in 2.01979s (17.8236GiB/s) | 20260506 17:12:53.976 | INFO | Downloaded 34GiB to GPU in 1.94372s (17.4923GiB/s) | 20260506 17:12:59.120 | INFO | Analyzing chunk frames = [3056640,3057152) | 20260506 17:12:59.120 | INFO | MemInfo: { | 20260506 17:12:59.120 | INFO | "gpu" : " 17 GB", | 20260506 17:12:59.121 | INFO | "huge" : "", | 20260506 17:12:59.121 | INFO | "huge_shared" : " 27 GB", | 20260506 17:12:59.121 | INFO | "malloc" : " 14 GB", | 20260506 17:12:59.121 | INFO | "pinned" : " 29 GB" | 20260506 17:12:59.121 | INFO | } | 20260506 17:12:59.121 | INFO | Perf report for the last 5.14551 seconds: | 20260506 17:12:59.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:12:59.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:59.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:59.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:59.121 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 17:12:59.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:59.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:59.121 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:12:59.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:59.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:12:59.121 | INFO | Uploaded 36GiB to GPU in 1.93963s (18.5603GiB/s) | 20260506 17:12:59.121 | INFO | Downloaded 34GiB to GPU in 1.89055s (17.9842GiB/s) | 20260506 17:13:04.258 | INFO | Analyzing chunk frames = [3057152,3057664) | 20260506 17:13:04.258 | INFO | MemInfo: { | 20260506 17:13:04.258 | INFO | "gpu" : " 17 GB", | 20260506 17:13:04.258 | INFO | "huge" : "", | 20260506 17:13:04.258 | INFO | "huge_shared" : " 27 GB", | 20260506 17:13:04.258 | INFO | "malloc" : " 14 GB", | 20260506 17:13:04.258 | INFO | "pinned" : " 29 GB" | 20260506 17:13:04.258 | INFO | } | 20260506 17:13:04.258 | INFO | Perf report for the last 5.13772 seconds: | 20260506 17:13:04.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:13:04.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:04.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:04.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:04.259 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:13:04.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:04.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:04.259 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:13:04.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:04.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:04.259 | INFO | Uploaded 36GiB to GPU in 1.94975s (18.4639GiB/s) | 20260506 17:13:04.259 | INFO | Downloaded 34GiB to GPU in 1.87149s (18.1674GiB/s) | 20260506 17:13:09.401 | INFO | Analyzing chunk frames = [3057664,3058176) | 20260506 17:13:09.401 | INFO | MemInfo: { | 20260506 17:13:09.401 | INFO | "gpu" : " 17 GB", | 20260506 17:13:09.401 | INFO | "huge" : "", | 20260506 17:13:09.401 | INFO | "huge_shared" : " 27 GB", | 20260506 17:13:09.401 | INFO | "malloc" : " 14 GB", | 20260506 17:13:09.401 | INFO | "pinned" : " 29 GB" | 20260506 17:13:09.401 | INFO | } | 20260506 17:13:09.401 | INFO | Perf report for the last 5.14279 seconds: | 20260506 17:13:09.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:13:09.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:09.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:09.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:09.402 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 17:13:09.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:09.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:09.402 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:13:09.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:09.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:09.402 | INFO | Uploaded 36GiB to GPU in 1.96185s (18.35GiB/s) | 20260506 17:13:09.402 | INFO | Downloaded 34GiB to GPU in 1.88303s (18.056GiB/s) | 20260506 17:13:14.537 | INFO | Analyzing chunk frames = [3058176,3058688) | 20260506 17:13:14.537 | INFO | MemInfo: { | 20260506 17:13:14.537 | INFO | "gpu" : " 17 GB", | 20260506 17:13:14.537 | INFO | "huge" : "", | 20260506 17:13:14.537 | INFO | "huge_shared" : " 27 GB", | 20260506 17:13:14.537 | INFO | "malloc" : " 14 GB", | 20260506 17:13:14.539 | INFO | "pinned" : " 29 GB" | 20260506 17:13:14.539 | INFO | } | 20260506 17:13:14.539 | INFO | Perf report for the last 5.1375 seconds: | 20260506 17:13:14.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:13:14.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:14.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:14.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:14.539 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 17:13:14.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:14.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:14.539 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:13:14.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:14.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:14.539 | INFO | Uploaded 36GiB to GPU in 2.03698s (17.6732GiB/s) | 20260506 17:13:14.539 | INFO | Downloaded 34GiB to GPU in 1.94509s (17.4799GiB/s) | 20260506 17:13:19.675 | INFO | Analyzing chunk frames = [3058688,3059200) | 20260506 17:13:19.675 | INFO | MemInfo: { | 20260506 17:13:19.675 | INFO | "gpu" : " 17 GB", | 20260506 17:13:19.675 | INFO | "huge" : "", | 20260506 17:13:19.675 | INFO | "huge_shared" : " 27 GB", | 20260506 17:13:19.675 | INFO | "malloc" : " 14 GB", | 20260506 17:13:19.675 | INFO | "pinned" : " 29 GB" | 20260506 17:13:19.675 | INFO | } | 20260506 17:13:19.675 | INFO | Perf report for the last 5.13644 seconds: | 20260506 17:13:19.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:13:19.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:19.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:19.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:19.675 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:13:19.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:19.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:19.675 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 17:13:19.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:19.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:19.676 | INFO | Uploaded 36GiB to GPU in 2.01031s (17.9077GiB/s) | 20260506 17:13:19.676 | INFO | Downloaded 34GiB to GPU in 1.99651s (17.0297GiB/s) | 20260506 17:13:24.816 | INFO | Analyzing chunk frames = [3059200,3059712) | 20260506 17:13:24.816 | INFO | MemInfo: { | 20260506 17:13:24.816 | INFO | "gpu" : " 17 GB", | 20260506 17:13:24.816 | INFO | "huge" : "", | 20260506 17:13:24.816 | INFO | "huge_shared" : " 27 GB", | 20260506 17:13:24.816 | INFO | "malloc" : " 14 GB", | 20260506 17:13:24.816 | INFO | "pinned" : " 29 GB" | 20260506 17:13:24.816 | INFO | } | 20260506 17:13:24.816 | INFO | Perf report for the last 5.14091 seconds: | 20260506 17:13:24.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:13:24.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:24.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:24.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:24.816 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 17:13:24.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:24.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:24.816 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:13:24.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:24.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:24.816 | INFO | Uploaded 36GiB to GPU in 2.01066s (17.9045GiB/s) | 20260506 17:13:24.817 | INFO | Downloaded 34GiB to GPU in 1.98517s (17.127GiB/s) | 20260506 17:13:24.934 | INFO | Starting a round of Baz IO | 20260506 17:13:28.079 | INFO | Populating RTMetrics report | 20260506 17:13:28.080 | INFO | RTMetrics report populated | 20260506 17:13:28.080 | INFO | Opening tmp RTMetrics json file | 20260506 17:13:28.105 | INFO | Serializing RTMetrics to json | 20260506 17:13:28.167 | INFO | Serialization done, swapping tmp file now... | 20260506 17:13:28.193 | INFO | RTMetrics done | 20260506 17:13:28.531 | INFO | Finished a round of Baz IO: Wrote 2.08 GiB in 3.597 seconds | 20260506 17:13:28.531 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.79% / 0.01% / 64.21% | 20260506 17:13:28.531 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.44761% / 7.94729e-06% / 6.75519e-05% | 20260506 17:13:28.531 | INFO | Baz IO overall: | 20260506 17:13:28.531 | INFO | BytesWritten : 1611653724405 | 20260506 17:13:28.531 | INFO | NumEvents : 779741732539 | 20260506 17:13:29.954 | INFO | Analyzing chunk frames = [3059712,3060224) | 20260506 17:13:29.954 | INFO | MemInfo: { | 20260506 17:13:29.954 | INFO | "gpu" : " 17 GB", | 20260506 17:13:29.954 | INFO | "huge" : "", | 20260506 17:13:29.954 | INFO | "huge_shared" : " 27 GB", | 20260506 17:13:29.954 | INFO | "malloc" : " 14 GB", | 20260506 17:13:29.954 | INFO | "pinned" : " 29 GB" | 20260506 17:13:29.954 | INFO | } | 20260506 17:13:29.954 | INFO | Perf report for the last 5.13763 seconds: | 20260506 17:13:29.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:13:29.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:29.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:29.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:29.954 | INFO | ANALYSIS : 62.9%, 2.9, 95.7ms, 0 | 20260506 17:13:29.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:29.954 | INFO | RT_METRICS : 45.9%, 1.0, 24.0ms, 0 | 20260506 17:13:29.954 | INFO | PRE_HQ : 60.2%, 1.9, 58.3ms, 0 | 20260506 17:13:29.954 | INFO | BAZWRITER : 69.6%, 4.8, 171.5ms, 0 | 20260506 17:13:29.954 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 17:13:29.954 | INFO | Uploaded 36GiB to GPU in 2.08017s (17.3063GiB/s) | 20260506 17:13:29.954 | INFO | Downloaded 38GiB to GPU in 2.22072s (17.1116GiB/s) | 20260506 17:13:35.093 | INFO | Analyzing chunk frames = [3060224,3060736) | 20260506 17:13:35.093 | INFO | MemInfo: { | 20260506 17:13:35.093 | INFO | "gpu" : " 17 GB", | 20260506 17:13:35.093 | INFO | "huge" : "", | 20260506 17:13:35.094 | INFO | "huge_shared" : " 27 GB", | 20260506 17:13:35.094 | INFO | "malloc" : " 14 GB", | 20260506 17:13:35.094 | INFO | "pinned" : " 29 GB" | 20260506 17:13:35.094 | INFO | } | 20260506 17:13:35.094 | INFO | Perf report for the last 5.13981 seconds: | 20260506 17:13:35.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:13:35.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:35.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:35.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:35.094 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:13:35.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:35.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:35.094 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:13:35.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:35.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:35.094 | INFO | Uploaded 36GiB to GPU in 1.99937s (18.0056GiB/s) | 20260506 17:13:35.094 | INFO | Downloaded 34GiB to GPU in 1.93854s (17.539GiB/s) | 20260506 17:13:40.228 | INFO | Analyzing chunk frames = [3060736,3061248) | 20260506 17:13:40.228 | INFO | MemInfo: { | 20260506 17:13:40.228 | INFO | "gpu" : " 17 GB", | 20260506 17:13:40.228 | INFO | "huge" : "", | 20260506 17:13:40.228 | INFO | "huge_shared" : " 27 GB", | 20260506 17:13:40.228 | INFO | "malloc" : " 14 GB", | 20260506 17:13:40.228 | INFO | "pinned" : " 29 GB" | 20260506 17:13:40.228 | INFO | } | 20260506 17:13:40.228 | INFO | Perf report for the last 5.13484 seconds: | 20260506 17:13:40.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:13:40.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:40.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:40.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:40.229 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 17:13:40.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:40.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:40.229 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:13:40.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:40.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:40.229 | INFO | Uploaded 36GiB to GPU in 1.96604s (18.311GiB/s) | 20260506 17:13:40.229 | INFO | Downloaded 34GiB to GPU in 1.92696s (17.6444GiB/s) | 20260506 17:13:45.371 | INFO | Analyzing chunk frames = [3061248,3061760) | 20260506 17:13:45.371 | INFO | MemInfo: { | 20260506 17:13:45.371 | INFO | "gpu" : " 17 GB", | 20260506 17:13:45.371 | INFO | "huge" : "", | 20260506 17:13:45.371 | INFO | "huge_shared" : " 27 GB", | 20260506 17:13:45.371 | INFO | "malloc" : " 14 GB", | 20260506 17:13:45.371 | INFO | "pinned" : " 29 GB" | 20260506 17:13:45.371 | INFO | } | 20260506 17:13:45.371 | INFO | Perf report for the last 5.14306 seconds: | 20260506 17:13:45.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:13:45.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:45.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:45.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:45.371 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260506 17:13:45.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:45.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:45.372 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:13:45.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:45.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:45.372 | INFO | Uploaded 36GiB to GPU in 1.9335s (18.6191GiB/s) | 20260506 17:13:45.372 | INFO | Downloaded 34GiB to GPU in 1.86985s (18.1833GiB/s) | 20260506 17:13:50.511 | INFO | Analyzing chunk frames = [3061760,3062272) | 20260506 17:13:50.511 | INFO | MemInfo: { | 20260506 17:13:50.511 | INFO | "gpu" : " 17 GB", | 20260506 17:13:50.511 | INFO | "huge" : "", | 20260506 17:13:50.511 | INFO | "huge_shared" : " 27 GB", | 20260506 17:13:50.511 | INFO | "malloc" : " 14 GB", | 20260506 17:13:50.511 | INFO | "pinned" : " 29 GB" | 20260506 17:13:50.511 | INFO | } | 20260506 17:13:50.511 | INFO | Perf report for the last 5.13953 seconds: | 20260506 17:13:50.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:13:50.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:50.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:50.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:50.511 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 17:13:50.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:50.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:50.511 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:13:50.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:50.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:50.511 | INFO | Uploaded 36GiB to GPU in 2.01963s (17.8251GiB/s) | 20260506 17:13:50.511 | INFO | Downloaded 34GiB to GPU in 1.91224s (17.7802GiB/s) | 20260506 17:13:55.652 | INFO | Analyzing chunk frames = [3062272,3062784) | 20260506 17:13:55.653 | INFO | MemInfo: { | 20260506 17:13:55.653 | INFO | "gpu" : " 17 GB", | 20260506 17:13:55.653 | INFO | "huge" : "", | 20260506 17:13:55.653 | INFO | "huge_shared" : " 27 GB", | 20260506 17:13:55.653 | INFO | "malloc" : " 14 GB", | 20260506 17:13:55.653 | INFO | "pinned" : " 29 GB" | 20260506 17:13:55.653 | INFO | } | 20260506 17:13:55.653 | INFO | Perf report for the last 5.14173 seconds: | 20260506 17:13:55.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:13:55.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:55.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:55.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:55.653 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:13:55.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:55.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:55.653 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:13:55.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:55.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:13:55.653 | INFO | Uploaded 36GiB to GPU in 2.04998s (17.5611GiB/s) | 20260506 17:13:55.653 | INFO | Downloaded 34GiB to GPU in 1.95228s (17.4156GiB/s) | 20260506 17:14:00.791 | INFO | Analyzing chunk frames = [3062784,3063296) | 20260506 17:14:00.791 | INFO | MemInfo: { | 20260506 17:14:00.791 | INFO | "gpu" : " 17 GB", | 20260506 17:14:00.791 | INFO | "huge" : "", | 20260506 17:14:00.791 | INFO | "huge_shared" : " 27 GB", | 20260506 17:14:00.791 | INFO | "malloc" : " 14 GB", | 20260506 17:14:00.791 | INFO | "pinned" : " 29 GB" | 20260506 17:14:00.791 | INFO | } | 20260506 17:14:00.791 | INFO | Perf report for the last 5.13865 seconds: | 20260506 17:14:00.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:14:00.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:00.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:00.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:00.792 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:14:00.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:00.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:00.792 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:14:00.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:00.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:00.792 | INFO | Uploaded 36GiB to GPU in 2.01828s (17.837GiB/s) | 20260506 17:14:00.792 | INFO | Downloaded 34GiB to GPU in 1.9714s (17.2467GiB/s) | 20260506 17:14:05.928 | INFO | Analyzing chunk frames = [3063296,3063808) | 20260506 17:14:05.928 | INFO | MemInfo: { | 20260506 17:14:05.928 | INFO | "gpu" : " 17 GB", | 20260506 17:14:05.928 | INFO | "huge" : "", | 20260506 17:14:05.928 | INFO | "huge_shared" : " 27 GB", | 20260506 17:14:05.928 | INFO | "malloc" : " 14 GB", | 20260506 17:14:05.928 | INFO | "pinned" : " 29 GB" | 20260506 17:14:05.928 | INFO | } | 20260506 17:14:05.928 | INFO | Perf report for the last 5.13637 seconds: | 20260506 17:14:05.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:14:05.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:05.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:05.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:05.928 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 17:14:05.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:05.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:05.928 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:14:05.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:05.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:05.928 | INFO | Uploaded 36GiB to GPU in 2.03703s (17.6727GiB/s) | 20260506 17:14:05.928 | INFO | Downloaded 34GiB to GPU in 1.99231s (17.0656GiB/s) | 20260506 17:14:05.928 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:14:05.928 | INFO | Total mem usage: 18 GiB | 20260506 17:14:05.928 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:14:05.928 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -17.2%) | 20260506 17:14:05.928 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:14:05.928 | INFO | High Water Breakdown: | 20260506 17:14:05.928 | INFO | 61.1% : 9.38 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:14:05.928 | INFO | 38.8% : 5.96 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:14:05.928 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:14:05.928 | INFO | ------------------------------------------ | 20260506 17:14:05.928 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:14:05.928 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:14:05.928 | INFO | Total mem usage: 26.5 GiB | 20260506 17:14:05.928 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:14:05.928 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:14:05.928 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:14:05.928 | INFO | High Water Breakdown: | 20260506 17:14:05.928 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:14:05.928 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:14:05.928 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:14:05.928 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:14:05.928 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:14:05.928 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:14:05.928 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:14:05.928 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:14:05.929 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:14:05.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:14:05.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:14:05.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:14:05.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:14:05.929 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:14:05.929 | INFO | ------------------------------------------ | 20260506 17:14:05.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:14:05.929 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:14:05.929 | INFO | Total mem usage: 25.6 GiB | 20260506 17:14:05.929 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:14:05.929 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:14:05.929 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:14:05.929 | INFO | High Water Breakdown: | 20260506 17:14:05.929 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:14:05.929 | INFO | ------------------------------------------ | 20260506 17:14:05.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:14:05.929 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:14:05.929 | INFO | Total mem usage: 15.7 GiB | 20260506 17:14:05.929 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:14:05.929 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:14:05.929 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:14:05.929 | INFO | High Water Breakdown: | 20260506 17:14:05.929 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:14:05.929 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:14:05.929 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:14:05.929 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:14:05.929 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:14:05.929 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:14:05.929 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:14:05.929 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:14:05.929 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:14:05.929 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:14:05.929 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:14:05.929 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:14:05.929 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:14:05.929 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:14:05.929 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:14:05.929 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:14:05.929 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:14:05.929 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:14:05.929 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:14:05.929 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:14:05.929 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:14:05.929 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:14:05.929 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:14:05.929 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:14:05.929 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:14:05.929 | INFO | ------------------------------------------ | 20260506 17:14:05.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:14:05.929 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:14:05.929 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:14:05.929 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:14:05.929 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.55e+01, 1.35e+01 / 2.27e+00 | 20260506 17:14:05.929 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.00e-01, 1.03e-02 / 5.53e-03 | 20260506 17:14:05.929 | INFO | PerfReport: 11.59%, Baseline -- 3.63e+00 / 4.46e+00, 5.54e-01 / 3.22e-01 | 20260506 17:14:05.929 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.83e-01 / 1.26e-01 | 20260506 17:14:05.929 | INFO | PerfReport: 15.90%, DME -- 3.99e+01 / 4.45e+01, 3.71e+00 / 2.09e+00 | 20260506 17:14:05.929 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.79e+01 / 2.02e+01, 1.47e+00 / 2.02e+00 | 20260506 17:14:05.929 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.28e+00 / 2.90e+00, 3.85e-01 / 2.22e-01 | 20260506 17:14:05.929 | INFO | PerfReport: 3.46%, Metrics -- 1.08e+00 / 1.45e+00, 2.81e-01 / 1.59e-01 | 20260506 17:14:05.929 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.77e-01 / 4.32e-01, 5.99e-02 / 2.51e-02 | 20260506 17:14:05.929 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:14:05.929 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:14:05.929 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:14:05.929 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 17:14:05.929 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260506 17:14:05.929 | INFO | PerfReport: 21.11%, Download -- 2.05e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 17:14:05.929 | INFO | PerfReport: 32.25%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 17:14:05.929 | INFO | PerfReport: 1.44%, LockUpload -- 1.40e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260506 17:14:05.929 | INFO | PerfReport: 1.04%, LockDownload -- 1.01e+00 / 0.00e+00, 2.60e+00 / 0.00e+00 | 20260506 17:14:05.929 | INFO | PerfReport: 22.64%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 17:14:05.929 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.57e-02 / 0.00e+00 | 20260506 17:14:06.050 | INFO | Starting a round of Baz IO | 20260506 17:14:09.189 | INFO | Populating RTMetrics report | 20260506 17:14:09.189 | INFO | RTMetrics report populated | 20260506 17:14:09.189 | INFO | Opening tmp RTMetrics json file | 20260506 17:14:09.298 | INFO | Serializing RTMetrics to json | 20260506 17:14:09.444 | INFO | Serialization done, swapping tmp file now... | 20260506 17:14:09.448 | INFO | RTMetrics done | 20260506 17:14:09.662 | INFO | Finished a round of Baz IO: Wrote 2.075 GiB in 3.611 seconds | 20260506 17:14:09.662 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.78% / 0.01% / 64.21% | 20260506 17:14:09.662 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.44268% / 7.94729e-06% / 5.5631e-05% | 20260506 17:14:09.662 | INFO | Baz IO overall: | 20260506 17:14:09.662 | INFO | BytesWritten : 1613881772660 | 20260506 17:14:09.662 | INFO | NumEvents : 780816423745 | 20260506 17:14:11.070 | INFO | Analyzing chunk frames = [3063808,3064320) | 20260506 17:14:11.070 | INFO | MemInfo: { | 20260506 17:14:11.070 | INFO | "gpu" : " 17 GB", | 20260506 17:14:11.070 | INFO | "huge" : "", | 20260506 17:14:11.070 | INFO | "huge_shared" : " 27 GB", | 20260506 17:14:11.070 | INFO | "malloc" : " 14 GB", | 20260506 17:14:11.070 | INFO | "pinned" : " 29 GB" | 20260506 17:14:11.070 | INFO | } | 20260506 17:14:11.070 | INFO | Perf report for the last 5.14223 seconds: | 20260506 17:14:11.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:14:11.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:11.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:11.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:11.070 | INFO | ANALYSIS : 62.8%, 2.9, 95.7ms, 0 | 20260506 17:14:11.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:11.070 | INFO | RT_METRICS : 48.4%, 1.0, 25.4ms, 0 | 20260506 17:14:11.070 | INFO | PRE_HQ : 60.3%, 1.9, 58.4ms, 0 | 20260506 17:14:11.070 | INFO | BAZWRITER : 69.8%, 4.8, 172.9ms, 0 | 20260506 17:14:11.070 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 17:14:11.070 | INFO | Uploaded 36GiB to GPU in 2.06611s (17.4241GiB/s) | 20260506 17:14:11.071 | INFO | Downloaded 38GiB to GPU in 2.22314s (17.093GiB/s) | 20260506 17:14:16.209 | INFO | Analyzing chunk frames = [3064320,3064832) | 20260506 17:14:16.209 | INFO | MemInfo: { | 20260506 17:14:16.209 | INFO | "gpu" : " 17 GB", | 20260506 17:14:16.209 | INFO | "huge" : "", | 20260506 17:14:16.209 | INFO | "huge_shared" : " 27 GB", | 20260506 17:14:16.209 | INFO | "malloc" : " 14 GB", | 20260506 17:14:16.209 | INFO | "pinned" : " 29 GB" | 20260506 17:14:16.209 | INFO | } | 20260506 17:14:16.209 | INFO | Perf report for the last 5.1387 seconds: | 20260506 17:14:16.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:14:16.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:16.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:16.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:16.209 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:14:16.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:16.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:16.209 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:14:16.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:16.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:16.209 | INFO | Uploaded 36GiB to GPU in 1.99206s (18.0718GiB/s) | 20260506 17:14:16.209 | INFO | Downloaded 34GiB to GPU in 1.92808s (17.6341GiB/s) | 20260506 17:14:21.348 | INFO | Analyzing chunk frames = [3064832,3065344) | 20260506 17:14:21.348 | INFO | MemInfo: { | 20260506 17:14:21.348 | INFO | "gpu" : " 17 GB", | 20260506 17:14:21.348 | INFO | "huge" : "", | 20260506 17:14:21.348 | INFO | "huge_shared" : " 27 GB", | 20260506 17:14:21.348 | INFO | "malloc" : " 14 GB", | 20260506 17:14:21.348 | INFO | "pinned" : " 29 GB" | 20260506 17:14:21.348 | INFO | } | 20260506 17:14:21.348 | INFO | Perf report for the last 5.139 seconds: | 20260506 17:14:21.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:14:21.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:21.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:21.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:21.348 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 17:14:21.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:21.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:21.348 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 17:14:21.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:21.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:21.348 | INFO | Uploaded 36GiB to GPU in 1.97622s (18.2166GiB/s) | 20260506 17:14:21.348 | INFO | Downloaded 34GiB to GPU in 1.9255s (17.6578GiB/s) | 20260506 17:14:26.493 | INFO | Analyzing chunk frames = [3065344,3065856) | 20260506 17:14:26.493 | INFO | MemInfo: { | 20260506 17:14:26.493 | INFO | "gpu" : " 17 GB", | 20260506 17:14:26.493 | INFO | "huge" : "", | 20260506 17:14:26.493 | INFO | "huge_shared" : " 27 GB", | 20260506 17:14:26.493 | INFO | "malloc" : " 14 GB", | 20260506 17:14:26.493 | INFO | "pinned" : " 29 GB" | 20260506 17:14:26.493 | INFO | } | 20260506 17:14:26.494 | INFO | Perf report for the last 5.14601 seconds: | 20260506 17:14:26.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:14:26.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:26.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:26.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:26.494 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260506 17:14:26.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:26.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:26.494 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:14:26.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:26.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:26.494 | INFO | Uploaded 36GiB to GPU in 1.92882s (18.6642GiB/s) | 20260506 17:14:26.494 | INFO | Downloaded 34GiB to GPU in 1.8704s (18.1779GiB/s) | 20260506 17:14:31.627 | INFO | Analyzing chunk frames = [3065856,3066368) | 20260506 17:14:31.627 | INFO | MemInfo: { | 20260506 17:14:31.627 | INFO | "gpu" : " 17 GB", | 20260506 17:14:31.627 | INFO | "huge" : "", | 20260506 17:14:31.627 | INFO | "huge_shared" : " 27 GB", | 20260506 17:14:31.628 | INFO | "malloc" : " 14 GB", | 20260506 17:14:31.628 | INFO | "pinned" : " 29 GB" | 20260506 17:14:31.628 | INFO | } | 20260506 17:14:31.628 | INFO | Perf report for the last 5.13375 seconds: | 20260506 17:14:31.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:14:31.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:31.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:31.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:31.628 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260506 17:14:31.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:31.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:31.628 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:14:31.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:31.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:31.628 | INFO | Uploaded 36GiB to GPU in 2.00624s (17.944GiB/s) | 20260506 17:14:31.628 | INFO | Downloaded 34GiB to GPU in 1.91432s (17.7609GiB/s) | 20260506 17:14:36.764 | INFO | Analyzing chunk frames = [3066368,3066880) | 20260506 17:14:36.764 | INFO | MemInfo: { | 20260506 17:14:36.764 | INFO | "gpu" : " 17 GB", | 20260506 17:14:36.764 | INFO | "huge" : "", | 20260506 17:14:36.764 | INFO | "huge_shared" : " 27 GB", | 20260506 17:14:36.764 | INFO | "malloc" : " 14 GB", | 20260506 17:14:36.764 | INFO | "pinned" : " 29 GB" | 20260506 17:14:36.764 | INFO | } | 20260506 17:14:36.764 | INFO | Perf report for the last 5.13684 seconds: | 20260506 17:14:36.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:14:36.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:36.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:36.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:36.764 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 17:14:36.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:36.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:36.765 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:14:36.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:36.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:36.765 | INFO | Uploaded 36GiB to GPU in 2.0262s (17.7673GiB/s) | 20260506 17:14:36.765 | INFO | Downloaded 34GiB to GPU in 1.94473s (17.4831GiB/s) | 20260506 17:14:41.905 | INFO | Analyzing chunk frames = [3066880,3067392) | 20260506 17:14:41.905 | INFO | MemInfo: { | 20260506 17:14:41.905 | INFO | "gpu" : " 17 GB", | 20260506 17:14:41.905 | INFO | "huge" : "", | 20260506 17:14:41.905 | INFO | "huge_shared" : " 27 GB", | 20260506 17:14:41.905 | INFO | "malloc" : " 14 GB", | 20260506 17:14:41.905 | INFO | "pinned" : " 29 GB" | 20260506 17:14:41.905 | INFO | } | 20260506 17:14:41.906 | INFO | Perf report for the last 5.14109 seconds: | 20260506 17:14:41.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:14:41.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:41.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:41.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:41.906 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:14:41.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:41.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:41.906 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:14:41.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:41.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:41.906 | INFO | Uploaded 36GiB to GPU in 2.02493s (17.7784GiB/s) | 20260506 17:14:41.906 | INFO | Downloaded 34GiB to GPU in 1.99813s (17.0159GiB/s) | 20260506 17:14:47.041 | INFO | Analyzing chunk frames = [3067392,3067904) | 20260506 17:14:47.041 | INFO | MemInfo: { | 20260506 17:14:47.041 | INFO | "gpu" : " 17 GB", | 20260506 17:14:47.041 | INFO | "huge" : "", | 20260506 17:14:47.041 | INFO | "huge_shared" : " 27 GB", | 20260506 17:14:47.042 | INFO | "malloc" : " 14 GB", | 20260506 17:14:47.042 | INFO | "pinned" : " 29 GB" | 20260506 17:14:47.042 | INFO | } | 20260506 17:14:47.042 | INFO | Perf report for the last 5.13617 seconds: | 20260506 17:14:47.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:14:47.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:47.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:47.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:47.042 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 17:14:47.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:47.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:47.042 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260506 17:14:47.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:47.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:47.042 | INFO | Uploaded 36GiB to GPU in 1.9978s (18.0198GiB/s) | 20260506 17:14:47.042 | INFO | Downloaded 34GiB to GPU in 1.96934s (17.2646GiB/s) | 20260506 17:14:47.166 | INFO | Starting a round of Baz IO | 20260506 17:14:50.303 | INFO | Populating RTMetrics report | 20260506 17:14:50.303 | INFO | RTMetrics report populated | 20260506 17:14:50.303 | INFO | Opening tmp RTMetrics json file | 20260506 17:14:50.418 | INFO | Serializing RTMetrics to json | 20260506 17:14:50.481 | INFO | Serialization done, swapping tmp file now... | 20260506 17:14:50.483 | INFO | RTMetrics done | 20260506 17:14:50.808 | INFO | Finished a round of Baz IO: Wrote 2.071 GiB in 3.641 seconds | 20260506 17:14:50.808 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.77% / 0.01% / 64.22% | 20260506 17:14:50.808 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.43224% / 1.58946e-05% / 4.76837e-05% | 20260506 17:14:50.808 | INFO | Baz IO overall: | 20260506 17:14:50.808 | INFO | BytesWritten : 1616105336988 | 20260506 17:14:50.808 | INFO | NumEvents : 781888257370 | 20260506 17:14:52.179 | INFO | Analyzing chunk frames = [3067904,3068416) | 20260506 17:14:52.179 | INFO | MemInfo: { | 20260506 17:14:52.179 | INFO | "gpu" : " 17 GB", | 20260506 17:14:52.179 | INFO | "huge" : "", | 20260506 17:14:52.179 | INFO | "huge_shared" : " 27 GB", | 20260506 17:14:52.179 | INFO | "malloc" : " 14 GB", | 20260506 17:14:52.179 | INFO | "pinned" : " 29 GB" | 20260506 17:14:52.179 | INFO | } | 20260506 17:14:52.179 | INFO | Perf report for the last 5.13754 seconds: | 20260506 17:14:52.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:14:52.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:52.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:52.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:52.179 | INFO | ANALYSIS : 62.9%, 2.9, 95.4ms, 0 | 20260506 17:14:52.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:52.179 | INFO | RT_METRICS : 47.0%, 1.0, 24.7ms, 0 | 20260506 17:14:52.179 | INFO | PRE_HQ : 60.3%, 1.9, 58.3ms, 0 | 20260506 17:14:52.179 | INFO | BAZWRITER : 70.9%, 4.8, 175.3ms, 0 | 20260506 17:14:52.179 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.5ms, 0 | 20260506 17:14:52.180 | INFO | Uploaded 36GiB to GPU in 2.06224s (17.4567GiB/s) | 20260506 17:14:52.180 | INFO | Downloaded 38GiB to GPU in 2.2095s (17.1985GiB/s) | 20260506 17:14:57.318 | INFO | Analyzing chunk frames = [3068416,3068928) | 20260506 17:14:57.318 | INFO | MemInfo: { | 20260506 17:14:57.318 | INFO | "gpu" : " 17 GB", | 20260506 17:14:57.318 | INFO | "huge" : "", | 20260506 17:14:57.318 | INFO | "huge_shared" : " 27 GB", | 20260506 17:14:57.319 | INFO | "malloc" : " 14 GB", | 20260506 17:14:57.319 | INFO | "pinned" : " 29 GB" | 20260506 17:14:57.319 | INFO | } | 20260506 17:14:57.319 | INFO | Perf report for the last 5.13936 seconds: | 20260506 17:14:57.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:14:57.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:57.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:57.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:57.319 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 17:14:57.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:57.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:57.319 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:14:57.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:57.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:14:57.319 | INFO | Uploaded 36GiB to GPU in 2.0188s (17.8323GiB/s) | 20260506 17:14:57.319 | INFO | Downloaded 34GiB to GPU in 1.97863s (17.1836GiB/s) | 20260506 17:15:02.462 | INFO | Analyzing chunk frames = [3068928,3069440) | 20260506 17:15:02.463 | INFO | MemInfo: { | 20260506 17:15:02.463 | INFO | "gpu" : " 17 GB", | 20260506 17:15:02.463 | INFO | "huge" : "", | 20260506 17:15:02.463 | INFO | "huge_shared" : " 27 GB", | 20260506 17:15:02.463 | INFO | "malloc" : " 14 GB", | 20260506 17:15:02.463 | INFO | "pinned" : " 29 GB" | 20260506 17:15:02.463 | INFO | } | 20260506 17:15:02.463 | INFO | Perf report for the last 5.14416 seconds: | 20260506 17:15:02.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:15:02.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:02.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:02.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:02.463 | INFO | ANALYSIS : 61.2%, 3.0, 94.9ms, 0 | 20260506 17:15:02.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:02.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:02.463 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:15:02.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:02.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:02.463 | INFO | Uploaded 36GiB to GPU in 2.00246s (17.9778GiB/s) | 20260506 17:15:02.463 | INFO | Downloaded 34GiB to GPU in 1.93203s (17.5981GiB/s) | 20260506 17:15:07.601 | INFO | Analyzing chunk frames = [3069440,3069952) | 20260506 17:15:07.601 | INFO | MemInfo: { | 20260506 17:15:07.601 | INFO | "gpu" : " 17 GB", | 20260506 17:15:07.601 | INFO | "huge" : "", | 20260506 17:15:07.601 | INFO | "huge_shared" : " 27 GB", | 20260506 17:15:07.601 | INFO | "malloc" : " 14 GB", | 20260506 17:15:07.601 | INFO | "pinned" : " 29 GB" | 20260506 17:15:07.601 | INFO | } | 20260506 17:15:07.601 | INFO | Perf report for the last 5.13821 seconds: | 20260506 17:15:07.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:15:07.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:07.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:07.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:07.601 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 17:15:07.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:07.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:07.601 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:15:07.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:07.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:07.601 | INFO | Uploaded 36GiB to GPU in 1.94387s (18.5197GiB/s) | 20260506 17:15:07.602 | INFO | Downloaded 34GiB to GPU in 1.88321s (18.0543GiB/s) | 20260506 17:15:12.745 | INFO | Analyzing chunk frames = [3069952,3070464) | 20260506 17:15:12.745 | INFO | MemInfo: { | 20260506 17:15:12.745 | INFO | "gpu" : " 17 GB", | 20260506 17:15:12.745 | INFO | "huge" : "", | 20260506 17:15:12.745 | INFO | "huge_shared" : " 27 GB", | 20260506 17:15:12.745 | INFO | "malloc" : " 14 GB", | 20260506 17:15:12.745 | INFO | "pinned" : " 29 GB" | 20260506 17:15:12.746 | INFO | } | 20260506 17:15:12.746 | INFO | Perf report for the last 5.14452 seconds: | 20260506 17:15:12.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:15:12.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:12.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:12.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:12.746 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 17:15:12.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:12.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:12.746 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:15:12.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:12.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:12.746 | INFO | Uploaded 36GiB to GPU in 1.99661s (18.0306GiB/s) | 20260506 17:15:12.746 | INFO | Downloaded 34GiB to GPU in 1.91249s (17.7779GiB/s) | 20260506 17:15:17.882 | INFO | Analyzing chunk frames = [3070464,3070976) | 20260506 17:15:17.882 | INFO | MemInfo: { | 20260506 17:15:17.882 | INFO | "gpu" : " 17 GB", | 20260506 17:15:17.882 | INFO | "huge" : "", | 20260506 17:15:17.882 | INFO | "huge_shared" : " 27 GB", | 20260506 17:15:17.882 | INFO | "malloc" : " 14 GB", | 20260506 17:15:17.882 | INFO | "pinned" : " 29 GB" | 20260506 17:15:17.882 | INFO | } | 20260506 17:15:17.882 | INFO | Perf report for the last 5.13636 seconds: | 20260506 17:15:17.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:15:17.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:17.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:17.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:17.882 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 17:15:17.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:17.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:17.882 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 17:15:17.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:17.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:17.882 | INFO | Uploaded 36GiB to GPU in 2.02213s (17.803GiB/s) | 20260506 17:15:17.883 | INFO | Downloaded 34GiB to GPU in 1.94269s (17.5015GiB/s) | 20260506 17:15:23.020 | INFO | Analyzing chunk frames = [3070976,3071488) | 20260506 17:15:23.020 | INFO | MemInfo: { | 20260506 17:15:23.020 | INFO | "gpu" : " 17 GB", | 20260506 17:15:23.020 | INFO | "huge" : "", | 20260506 17:15:23.020 | INFO | "huge_shared" : " 27 GB", | 20260506 17:15:23.020 | INFO | "malloc" : " 14 GB", | 20260506 17:15:23.020 | INFO | "pinned" : " 29 GB" | 20260506 17:15:23.020 | INFO | } | 20260506 17:15:23.020 | INFO | Perf report for the last 5.13847 seconds: | 20260506 17:15:23.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:15:23.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:23.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:23.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:23.021 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:15:23.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:23.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:23.021 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 17:15:23.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:23.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:23.021 | INFO | Uploaded 36GiB to GPU in 2.02415s (17.7852GiB/s) | 20260506 17:15:23.021 | INFO | Downloaded 34GiB to GPU in 1.99215s (17.067GiB/s) | 20260506 17:15:28.158 | INFO | Analyzing chunk frames = [3071488,3072000) | 20260506 17:15:28.158 | INFO | MemInfo: { | 20260506 17:15:28.158 | INFO | "gpu" : " 17 GB", | 20260506 17:15:28.158 | INFO | "huge" : "", | 20260506 17:15:28.158 | INFO | "huge_shared" : " 27 GB", | 20260506 17:15:28.158 | INFO | "malloc" : " 14 GB", | 20260506 17:15:28.158 | INFO | "pinned" : " 29 GB" | 20260506 17:15:28.158 | INFO | } | 20260506 17:15:28.158 | INFO | Perf report for the last 5.1379 seconds: | 20260506 17:15:28.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:15:28.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:28.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:28.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:28.158 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 17:15:28.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:28.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:28.158 | INFO | PRE_HQ : 31.7%, 1.0, 16.4ms, 0 | 20260506 17:15:28.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:28.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:28.159 | INFO | Uploaded 36GiB to GPU in 2.04472s (17.6063GiB/s) | 20260506 17:15:28.159 | INFO | Downloaded 34GiB to GPU in 1.99186s (17.0695GiB/s) | 20260506 17:15:28.159 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:15:28.159 | INFO | Total mem usage: 18 GiB | 20260506 17:15:28.159 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:15:28.159 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -19.1%) | 20260506 17:15:28.159 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:15:28.159 | INFO | High Water Breakdown: | 20260506 17:15:28.159 | INFO | 60.8% : 9.2 GiB ( 80.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:15:28.159 | INFO | 39.1% : 5.91 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:15:28.159 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:15:28.159 | INFO | ------------------------------------------ | 20260506 17:15:28.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:15:28.159 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:15:28.159 | INFO | Total mem usage: 26.5 GiB | 20260506 17:15:28.159 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:15:28.159 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:15:28.159 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:15:28.159 | INFO | High Water Breakdown: | 20260506 17:15:28.159 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:15:28.159 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:15:28.159 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:15:28.159 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:15:28.159 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:15:28.159 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:15:28.159 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:15:28.159 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:15:28.159 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:15:28.159 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:15:28.159 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:15:28.159 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:15:28.159 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:15:28.159 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:15:28.159 | INFO | ------------------------------------------ | 20260506 17:15:28.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:15:28.159 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:15:28.159 | INFO | Total mem usage: 25.6 GiB | 20260506 17:15:28.159 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:15:28.159 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:15:28.159 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:15:28.159 | INFO | High Water Breakdown: | 20260506 17:15:28.159 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:15:28.159 | INFO | ------------------------------------------ | 20260506 17:15:28.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:15:28.159 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:15:28.159 | INFO | Total mem usage: 15.7 GiB | 20260506 17:15:28.159 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:15:28.159 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:15:28.159 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:15:28.159 | INFO | High Water Breakdown: | 20260506 17:15:28.159 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:15:28.159 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:15:28.159 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:15:28.160 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:15:28.160 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:15:28.160 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:15:28.160 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:15:28.160 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:15:28.160 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:15:28.160 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:15:28.160 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:15:28.160 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:15:28.160 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:15:28.160 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:15:28.160 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:15:28.160 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:15:28.160 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:15:28.160 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:15:28.160 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:15:28.160 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:15:28.160 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:15:28.160 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:15:28.160 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:15:28.160 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:15:28.160 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:15:28.160 | INFO | ------------------------------------------ | 20260506 17:15:28.160 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:15:28.160 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:15:28.160 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:15:28.160 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:15:28.160 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.60e+01, 1.34e+01 / 2.44e+00 | 20260506 17:15:28.160 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 9.99e-02, 1.02e-02 / 5.80e-03 | 20260506 17:15:28.160 | INFO | PerfReport: 11.61%, Baseline -- 3.64e+00 / 4.49e+00, 5.62e-01 / 2.81e-01 | 20260506 17:15:28.160 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.84e-01 / 1.39e-01 | 20260506 17:15:28.160 | INFO | PerfReport: 15.92%, DME -- 3.99e+01 / 4.46e+01, 4.02e+00 / 2.77e+00 | 20260506 17:15:28.160 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.45e+00 / 1.82e+00 | 20260506 17:15:28.160 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.28e+00 / 2.83e+00, 3.90e-01 / 2.28e-01 | 20260506 17:15:28.160 | INFO | PerfReport: 3.45%, Metrics -- 1.08e+00 / 1.44e+00, 2.80e-01 / 1.63e-01 | 20260506 17:15:28.160 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.75e-01 / 4.22e-01, 5.89e-02 / 1.81e-02 | 20260506 17:15:28.160 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:15:28.160 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:15:28.160 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:15:28.160 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 17:15:28.160 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260506 17:15:28.160 | INFO | PerfReport: 21.15%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 17:15:28.160 | INFO | PerfReport: 32.23%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:15:28.160 | INFO | PerfReport: 1.43%, LockUpload -- 1.39e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260506 17:15:28.160 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.66e+00 / 0.00e+00 | 20260506 17:15:28.160 | INFO | PerfReport: 22.59%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 17:15:28.160 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 3.01e-02 / 0.00e+00 | 20260506 17:15:28.271 | INFO | Starting a round of Baz IO | 20260506 17:15:31.418 | INFO | Populating RTMetrics report | 20260506 17:15:31.419 | INFO | RTMetrics report populated | 20260506 17:15:31.419 | INFO | Opening tmp RTMetrics json file | 20260506 17:15:31.627 | INFO | Serializing RTMetrics to json | 20260506 17:15:31.750 | INFO | Serialization done, swapping tmp file now... | 20260506 17:15:31.754 | INFO | RTMetrics done | 20260506 17:15:31.777 | INFO | Finished a round of Baz IO: Wrote 2.07 GiB in 3.506 seconds | 20260506 17:15:31.777 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.76% / 0.01% / 64.23% | 20260506 17:15:31.777 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.42889% / 1.19209e-05% / 6.75519e-05% | 20260506 17:15:31.777 | INFO | Baz IO overall: | 20260506 17:15:31.778 | INFO | BytesWritten : 1618328511052 | 20260506 17:15:31.778 | INFO | NumEvents : 782960537825 | 20260506 17:15:33.295 | INFO | Analyzing chunk frames = [3072000,3072512) | 20260506 17:15:33.295 | INFO | MemInfo: { | 20260506 17:15:33.295 | INFO | "gpu" : " 17 GB", | 20260506 17:15:33.295 | INFO | "huge" : "", | 20260506 17:15:33.295 | INFO | "huge_shared" : " 27 GB", | 20260506 17:15:33.295 | INFO | "malloc" : " 14 GB", | 20260506 17:15:33.295 | INFO | "pinned" : " 29 GB" | 20260506 17:15:33.295 | INFO | } | 20260506 17:15:33.295 | INFO | Perf report for the last 5.13647 seconds: | 20260506 17:15:33.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:15:33.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:33.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:33.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:33.295 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 17:15:33.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:33.295 | INFO | RT_METRICS : 50.1%, 1.0, 26.3ms, 0 | 20260506 17:15:33.295 | INFO | PRE_HQ : 60.8%, 1.9, 58.7ms, 0 | 20260506 17:15:33.295 | INFO | BAZWRITER : 67.5%, 4.8, 165.6ms, 0 | 20260506 17:15:33.295 | INFO | BAZ_SAMPLING : 3.8%, 1.0, 2.0ms, 0 | 20260506 17:15:33.295 | INFO | Uploaded 36GiB to GPU in 2.06317s (17.4489GiB/s) | 20260506 17:15:33.295 | INFO | Downloaded 38GiB to GPU in 2.21868s (17.1273GiB/s) | 20260506 17:15:38.437 | INFO | Analyzing chunk frames = [3072512,3073024) | 20260506 17:15:38.437 | INFO | MemInfo: { | 20260506 17:15:38.437 | INFO | "gpu" : " 17 GB", | 20260506 17:15:38.437 | INFO | "huge" : "", | 20260506 17:15:38.437 | INFO | "huge_shared" : " 27 GB", | 20260506 17:15:38.437 | INFO | "malloc" : " 14 GB", | 20260506 17:15:38.437 | INFO | "pinned" : " 29 GB" | 20260506 17:15:38.437 | INFO | } | 20260506 17:15:38.437 | INFO | Perf report for the last 5.1424 seconds: | 20260506 17:15:38.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:15:38.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:38.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:38.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:38.437 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 17:15:38.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:38.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:38.437 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:15:38.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:38.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:38.438 | INFO | Uploaded 36GiB to GPU in 1.99634s (18.033GiB/s) | 20260506 17:15:38.438 | INFO | Downloaded 34GiB to GPU in 1.94325s (17.4964GiB/s) | 20260506 17:15:43.580 | INFO | Analyzing chunk frames = [3073024,3073536) | 20260506 17:15:43.580 | INFO | MemInfo: { | 20260506 17:15:43.580 | INFO | "gpu" : " 17 GB", | 20260506 17:15:43.580 | INFO | "huge" : "", | 20260506 17:15:43.580 | INFO | "huge_shared" : " 27 GB", | 20260506 17:15:43.580 | INFO | "malloc" : " 14 GB", | 20260506 17:15:43.580 | INFO | "pinned" : " 29 GB" | 20260506 17:15:43.580 | INFO | } | 20260506 17:15:43.581 | INFO | Perf report for the last 5.14329 seconds: | 20260506 17:15:43.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:15:43.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:43.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:43.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:43.581 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 17:15:43.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:43.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:43.581 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:15:43.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:43.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:43.581 | INFO | Uploaded 36GiB to GPU in 1.98143s (18.1687GiB/s) | 20260506 17:15:43.581 | INFO | Downloaded 34GiB to GPU in 1.92448s (17.6671GiB/s) | 20260506 17:15:48.716 | INFO | Analyzing chunk frames = [3073536,3074048) | 20260506 17:15:48.716 | INFO | MemInfo: { | 20260506 17:15:48.716 | INFO | "gpu" : " 17 GB", | 20260506 17:15:48.716 | INFO | "huge" : "", | 20260506 17:15:48.716 | INFO | "huge_shared" : " 27 GB", | 20260506 17:15:48.716 | INFO | "malloc" : " 14 GB", | 20260506 17:15:48.716 | INFO | "pinned" : " 29 GB" | 20260506 17:15:48.716 | INFO | } | 20260506 17:15:48.716 | INFO | Perf report for the last 5.13563 seconds: | 20260506 17:15:48.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:15:48.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:48.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:48.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:48.716 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 17:15:48.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:48.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:48.716 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:15:48.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:48.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:48.717 | INFO | Uploaded 36GiB to GPU in 1.96326s (18.3369GiB/s) | 20260506 17:15:48.717 | INFO | Downloaded 34GiB to GPU in 1.87177s (18.1647GiB/s) | 20260506 17:15:53.857 | INFO | Analyzing chunk frames = [3074048,3074560) | 20260506 17:15:53.857 | INFO | MemInfo: { | 20260506 17:15:53.857 | INFO | "gpu" : " 17 GB", | 20260506 17:15:53.857 | INFO | "huge" : "", | 20260506 17:15:53.857 | INFO | "huge_shared" : " 27 GB", | 20260506 17:15:53.857 | INFO | "malloc" : " 14 GB", | 20260506 17:15:53.857 | INFO | "pinned" : " 29 GB" | 20260506 17:15:53.857 | INFO | } | 20260506 17:15:53.857 | INFO | Perf report for the last 5.14122 seconds: | 20260506 17:15:53.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:15:53.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:53.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:53.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:53.858 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 17:15:53.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:53.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:53.858 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:15:53.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:53.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:53.858 | INFO | Uploaded 36GiB to GPU in 2.01959s (17.8254GiB/s) | 20260506 17:15:53.858 | INFO | Downloaded 34GiB to GPU in 1.90777s (17.8218GiB/s) | 20260506 17:15:58.994 | INFO | Analyzing chunk frames = [3074560,3075072) | 20260506 17:15:58.994 | INFO | MemInfo: { | 20260506 17:15:58.994 | INFO | "gpu" : " 17 GB", | 20260506 17:15:58.994 | INFO | "huge" : "", | 20260506 17:15:58.994 | INFO | "huge_shared" : " 27 GB", | 20260506 17:15:58.994 | INFO | "malloc" : " 14 GB", | 20260506 17:15:58.994 | INFO | "pinned" : " 29 GB" | 20260506 17:15:58.994 | INFO | } | 20260506 17:15:58.994 | INFO | Perf report for the last 5.13666 seconds: | 20260506 17:15:58.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:15:58.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:58.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:58.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:58.994 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:15:58.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:58.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:58.994 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:15:58.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:58.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:15:58.994 | INFO | Uploaded 36GiB to GPU in 2.06069s (17.4699GiB/s) | 20260506 17:15:58.994 | INFO | Downloaded 34GiB to GPU in 1.95777s (17.3667GiB/s) | 20260506 17:16:04.131 | INFO | Analyzing chunk frames = [3075072,3075584) | 20260506 17:16:04.131 | INFO | MemInfo: { | 20260506 17:16:04.131 | INFO | "gpu" : " 17 GB", | 20260506 17:16:04.131 | INFO | "huge" : "", | 20260506 17:16:04.131 | INFO | "huge_shared" : " 27 GB", | 20260506 17:16:04.131 | INFO | "malloc" : " 14 GB", | 20260506 17:16:04.131 | INFO | "pinned" : " 29 GB" | 20260506 17:16:04.131 | INFO | } | 20260506 17:16:04.131 | INFO | Perf report for the last 5.13708 seconds: | 20260506 17:16:04.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:16:04.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:04.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:04.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:04.131 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 17:16:04.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:04.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:04.131 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:16:04.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:04.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:04.132 | INFO | Uploaded 36GiB to GPU in 2.03047s (17.7299GiB/s) | 20260506 17:16:04.132 | INFO | Downloaded 34GiB to GPU in 2.02443s (16.7948GiB/s) | 20260506 17:16:09.269 | INFO | Analyzing chunk frames = [3075584,3076096) | 20260506 17:16:09.269 | INFO | MemInfo: { | 20260506 17:16:09.269 | INFO | "gpu" : " 17 GB", | 20260506 17:16:09.269 | INFO | "huge" : "", | 20260506 17:16:09.269 | INFO | "huge_shared" : " 27 GB", | 20260506 17:16:09.269 | INFO | "malloc" : " 14 GB", | 20260506 17:16:09.269 | INFO | "pinned" : " 29 GB" | 20260506 17:16:09.269 | INFO | } | 20260506 17:16:09.269 | INFO | Perf report for the last 5.13774 seconds: | 20260506 17:16:09.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:16:09.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:09.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:09.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:09.269 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 17:16:09.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:09.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:09.269 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:16:09.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:09.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:09.269 | INFO | Uploaded 36GiB to GPU in 1.99258s (18.067GiB/s) | 20260506 17:16:09.269 | INFO | Downloaded 34GiB to GPU in 1.96719s (17.2835GiB/s) | 20260506 17:16:09.383 | INFO | Starting a round of Baz IO | 20260506 17:16:12.527 | INFO | Populating RTMetrics report | 20260506 17:16:12.527 | INFO | RTMetrics report populated | 20260506 17:16:12.527 | INFO | Opening tmp RTMetrics json file | 20260506 17:16:12.549 | INFO | Serializing RTMetrics to json | 20260506 17:16:12.556 | INFO | Serialization done, swapping tmp file now... | 20260506 17:16:12.562 | INFO | RTMetrics done | 20260506 17:16:12.855 | INFO | Finished a round of Baz IO: Wrote 2.071 GiB in 3.471 seconds | 20260506 17:16:12.855 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.76% / 0.01% / 64.24% | 20260506 17:16:12.855 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.42888% / 1.58946e-05% / 5.5631e-05% | 20260506 17:16:12.855 | INFO | Baz IO overall: | 20260506 17:16:12.855 | INFO | BytesWritten : 1620552021060 | 20260506 17:16:12.855 | INFO | NumEvents : 784033140820 | 20260506 17:16:14.411 | INFO | Analyzing chunk frames = [3076096,3076608) | 20260506 17:16:14.411 | INFO | MemInfo: { | 20260506 17:16:14.411 | INFO | "gpu" : " 17 GB", | 20260506 17:16:14.411 | INFO | "huge" : "", | 20260506 17:16:14.411 | INFO | "huge_shared" : " 27 GB", | 20260506 17:16:14.411 | INFO | "malloc" : " 14 GB", | 20260506 17:16:14.411 | INFO | "pinned" : " 29 GB" | 20260506 17:16:14.411 | INFO | } | 20260506 17:16:14.411 | INFO | Perf report for the last 5.14221 seconds: | 20260506 17:16:14.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:16:14.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:14.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:14.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:14.411 | INFO | ANALYSIS : 62.8%, 2.9, 95.1ms, 0 | 20260506 17:16:14.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:14.411 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260506 17:16:14.411 | INFO | PRE_HQ : 60.6%, 1.9, 58.7ms, 0 | 20260506 17:16:14.411 | INFO | BAZWRITER : 67.1%, 4.8, 165.8ms, 0 | 20260506 17:16:14.411 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 17:16:14.411 | INFO | Uploaded 36GiB to GPU in 2.05416s (17.5254GiB/s) | 20260506 17:16:14.411 | INFO | Downloaded 38GiB to GPU in 2.20761s (17.2132GiB/s) | 20260506 17:16:19.556 | INFO | Analyzing chunk frames = [3076608,3077120) | 20260506 17:16:19.556 | INFO | MemInfo: { | 20260506 17:16:19.556 | INFO | "gpu" : " 17 GB", | 20260506 17:16:19.556 | INFO | "huge" : "", | 20260506 17:16:19.556 | INFO | "huge_shared" : " 27 GB", | 20260506 17:16:19.556 | INFO | "malloc" : " 14 GB", | 20260506 17:16:19.556 | INFO | "pinned" : " 29 GB" | 20260506 17:16:19.556 | INFO | } | 20260506 17:16:19.556 | INFO | Perf report for the last 5.14475 seconds: | 20260506 17:16:19.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:16:19.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:19.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:19.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:19.556 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 17:16:19.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:19.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:19.556 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 17:16:19.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:19.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:19.556 | INFO | Uploaded 36GiB to GPU in 2.00014s (17.9988GiB/s) | 20260506 17:16:19.556 | INFO | Downloaded 34GiB to GPU in 1.93813s (17.5427GiB/s) | 20260506 17:16:24.697 | INFO | Analyzing chunk frames = [3077120,3077632) | 20260506 17:16:24.697 | INFO | MemInfo: { | 20260506 17:16:24.697 | INFO | "gpu" : " 17 GB", | 20260506 17:16:24.697 | INFO | "huge" : "", | 20260506 17:16:24.697 | INFO | "huge_shared" : " 27 GB", | 20260506 17:16:24.697 | INFO | "malloc" : " 14 GB", | 20260506 17:16:24.697 | INFO | "pinned" : " 29 GB" | 20260506 17:16:24.697 | INFO | } | 20260506 17:16:24.697 | INFO | Perf report for the last 5.14139 seconds: | 20260506 17:16:24.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:16:24.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:24.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:24.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:24.697 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 17:16:24.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:24.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:24.698 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:16:24.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:24.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:24.698 | INFO | Uploaded 36GiB to GPU in 1.96684s (18.3035GiB/s) | 20260506 17:16:24.698 | INFO | Downloaded 34GiB to GPU in 1.92316s (17.6792GiB/s) | 20260506 17:16:29.840 | INFO | Analyzing chunk frames = [3077632,3078144) | 20260506 17:16:29.841 | INFO | MemInfo: { | 20260506 17:16:29.841 | INFO | "gpu" : " 17 GB", | 20260506 17:16:29.841 | INFO | "huge" : "", | 20260506 17:16:29.841 | INFO | "huge_shared" : " 27 GB", | 20260506 17:16:29.841 | INFO | "malloc" : " 14 GB", | 20260506 17:16:29.841 | INFO | "pinned" : " 29 GB" | 20260506 17:16:29.841 | INFO | } | 20260506 17:16:29.841 | INFO | Perf report for the last 5.14335 seconds: | 20260506 17:16:29.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:16:29.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:29.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:29.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:29.841 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260506 17:16:29.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:29.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:29.841 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:16:29.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:29.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:29.841 | INFO | Uploaded 36GiB to GPU in 1.94342s (18.5241GiB/s) | 20260506 17:16:29.841 | INFO | Downloaded 34GiB to GPU in 1.86589s (18.2218GiB/s) | 20260506 17:16:34.970 | INFO | Analyzing chunk frames = [3078144,3078656) | 20260506 17:16:34.970 | INFO | MemInfo: { | 20260506 17:16:34.970 | INFO | "gpu" : " 17 GB", | 20260506 17:16:34.970 | INFO | "huge" : "", | 20260506 17:16:34.970 | INFO | "huge_shared" : " 27 GB", | 20260506 17:16:34.970 | INFO | "malloc" : " 14 GB", | 20260506 17:16:34.970 | INFO | "pinned" : " 29 GB" | 20260506 17:16:34.970 | INFO | } | 20260506 17:16:34.970 | INFO | Perf report for the last 5.12935 seconds: | 20260506 17:16:34.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:16:34.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:34.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:34.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:34.970 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260506 17:16:34.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:34.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:34.970 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:16:34.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:34.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:34.970 | INFO | Uploaded 36GiB to GPU in 2.01386s (17.8761GiB/s) | 20260506 17:16:34.970 | INFO | Downloaded 34GiB to GPU in 1.90859s (17.8142GiB/s) | 20260506 17:16:40.107 | INFO | Analyzing chunk frames = [3078656,3079168) | 20260506 17:16:40.107 | INFO | MemInfo: { | 20260506 17:16:40.107 | INFO | "gpu" : " 17 GB", | 20260506 17:16:40.107 | INFO | "huge" : "", | 20260506 17:16:40.107 | INFO | "huge_shared" : " 27 GB", | 20260506 17:16:40.107 | INFO | "malloc" : " 14 GB", | 20260506 17:16:40.107 | INFO | "pinned" : " 29 GB" | 20260506 17:16:40.107 | INFO | } | 20260506 17:16:40.107 | INFO | Perf report for the last 5.13688 seconds: | 20260506 17:16:40.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:16:40.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:40.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:40.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:40.107 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:16:40.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:40.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:40.107 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:16:40.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:40.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:40.107 | INFO | Uploaded 36GiB to GPU in 2.02467s (17.7807GiB/s) | 20260506 17:16:40.107 | INFO | Downloaded 34GiB to GPU in 1.93794s (17.5444GiB/s) | 20260506 17:16:45.245 | INFO | Analyzing chunk frames = [3079168,3079680) | 20260506 17:16:45.245 | INFO | MemInfo: { | 20260506 17:16:45.245 | INFO | "gpu" : " 17 GB", | 20260506 17:16:45.245 | INFO | "huge" : "", | 20260506 17:16:45.245 | INFO | "huge_shared" : " 27 GB", | 20260506 17:16:45.245 | INFO | "malloc" : " 14 GB", | 20260506 17:16:45.245 | INFO | "pinned" : " 29 GB" | 20260506 17:16:45.245 | INFO | } | 20260506 17:16:45.245 | INFO | Perf report for the last 5.13829 seconds: | 20260506 17:16:45.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:16:45.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:45.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:45.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:45.245 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 17:16:45.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:45.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:45.245 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:16:45.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:45.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:45.246 | INFO | Uploaded 36GiB to GPU in 2.01458s (17.8697GiB/s) | 20260506 17:16:45.246 | INFO | Downloaded 34GiB to GPU in 2.01298s (16.8904GiB/s) | 20260506 17:16:50.390 | INFO | Analyzing chunk frames = [3079680,3080192) | 20260506 17:16:50.390 | INFO | MemInfo: { | 20260506 17:16:50.390 | INFO | "gpu" : " 17 GB", | 20260506 17:16:50.390 | INFO | "huge" : "", | 20260506 17:16:50.390 | INFO | "huge_shared" : " 27 GB", | 20260506 17:16:50.390 | INFO | "malloc" : " 14 GB", | 20260506 17:16:50.390 | INFO | "pinned" : " 29 GB" | 20260506 17:16:50.390 | INFO | } | 20260506 17:16:50.390 | INFO | Perf report for the last 5.14475 seconds: | 20260506 17:16:50.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:16:50.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:50.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:50.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:50.390 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 17:16:50.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:50.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:50.390 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 17:16:50.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:50.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:50.390 | INFO | Uploaded 36GiB to GPU in 2.00241s (17.9783GiB/s) | 20260506 17:16:50.390 | INFO | Downloaded 34GiB to GPU in 1.96805s (17.276GiB/s) | 20260506 17:16:50.390 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:16:50.390 | INFO | Total mem usage: 18 GiB | 20260506 17:16:50.390 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:16:50.390 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.4%) | 20260506 17:16:50.390 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:16:50.390 | INFO | High Water Breakdown: | 20260506 17:16:50.390 | INFO | 61.4% : 9.58 GiB ( 83.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:16:50.391 | INFO | 38.5% : 6.01 GiB ( 89.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:16:50.391 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:16:50.391 | INFO | ------------------------------------------ | 20260506 17:16:50.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:16:50.391 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:16:50.391 | INFO | Total mem usage: 26.5 GiB | 20260506 17:16:50.391 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:16:50.391 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:16:50.391 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:16:50.391 | INFO | High Water Breakdown: | 20260506 17:16:50.391 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:16:50.391 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:16:50.391 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:16:50.391 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:16:50.391 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:16:50.391 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:16:50.391 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:16:50.391 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:16:50.391 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:16:50.391 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:16:50.391 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:16:50.391 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:16:50.391 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:16:50.391 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:16:50.391 | INFO | ------------------------------------------ | 20260506 17:16:50.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:16:50.391 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:16:50.391 | INFO | Total mem usage: 25.6 GiB | 20260506 17:16:50.391 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:16:50.391 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:16:50.391 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:16:50.391 | INFO | High Water Breakdown: | 20260506 17:16:50.391 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:16:50.391 | INFO | ------------------------------------------ | 20260506 17:16:50.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:16:50.391 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:16:50.391 | INFO | Total mem usage: 15.7 GiB | 20260506 17:16:50.391 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:16:50.391 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:16:50.391 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:16:50.391 | INFO | High Water Breakdown: | 20260506 17:16:50.391 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:16:50.391 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:16:50.391 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:16:50.391 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:16:50.391 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:16:50.391 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:16:50.391 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:16:50.391 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:16:50.391 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:16:50.391 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:16:50.391 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:16:50.391 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:16:50.391 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:16:50.391 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:16:50.391 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:16:50.391 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:16:50.391 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:16:50.391 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:16:50.391 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:16:50.391 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:16:50.391 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:16:50.391 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:16:50.391 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:16:50.391 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:16:50.391 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:16:50.391 | INFO | ------------------------------------------ | 20260506 17:16:50.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:16:50.391 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:16:50.391 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:16:50.391 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:16:50.391 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.60e+01, 1.30e+01 / 2.60e+00 | 20260506 17:16:50.391 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.02e-01, 9.80e-03 / 6.21e-03 | 20260506 17:16:50.391 | INFO | PerfReport: 11.62%, Baseline -- 3.64e+00 / 4.43e+00, 5.64e-01 / 3.26e-01 | 20260506 17:16:50.391 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.48e+00, 1.89e-01 / 1.31e-01 | 20260506 17:16:50.391 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.47e+01, 4.19e+00 / 2.66e+00 | 20260506 17:16:50.391 | INFO | PerfReport: 56.89%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.39e+00 / 1.88e+00 | 20260506 17:16:50.391 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.29e+00 / 2.86e+00, 3.91e-01 / 2.03e-01 | 20260506 17:16:50.391 | INFO | PerfReport: 3.46%, Metrics -- 1.09e+00 / 1.43e+00, 2.84e-01 / 1.37e-01 | 20260506 17:16:50.391 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.76e-01 / 4.29e-01, 5.90e-02 / 2.85e-02 | 20260506 17:16:50.391 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:16:50.391 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:16:50.391 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:16:50.391 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 17:16:50.391 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 17:16:50.391 | INFO | PerfReport: 21.14%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 17:16:50.391 | INFO | PerfReport: 32.23%, Compute -- 3.14e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 17:16:50.391 | INFO | PerfReport: 1.43%, LockUpload -- 1.39e+00 / 0.00e+00, 5.77e+00 / 0.00e+00 | 20260506 17:16:50.391 | INFO | PerfReport: 1.08%, LockDownload -- 1.05e+00 / 0.00e+00, 2.69e+00 / 0.00e+00 | 20260506 17:16:50.391 | INFO | PerfReport: 22.60%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 17:16:50.392 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260506 17:16:50.504 | INFO | Starting a round of Baz IO | 20260506 17:16:53.637 | INFO | Populating RTMetrics report | 20260506 17:16:53.637 | INFO | RTMetrics report populated | 20260506 17:16:53.637 | INFO | Opening tmp RTMetrics json file | 20260506 17:16:53.639 | INFO | Serializing RTMetrics to json | 20260506 17:16:53.641 | INFO | Serialization done, swapping tmp file now... | 20260506 17:16:53.641 | INFO | RTMetrics done | 20260506 17:16:53.786 | INFO | Finished a round of Baz IO: Wrote 2.072 GiB in 3.282 seconds | 20260506 17:16:53.786 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.75% / 0.01% / 64.24% | 20260506 17:16:53.786 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.4281% / 1.19209e-05% / 7.15256e-05% | 20260506 17:16:53.786 | INFO | Baz IO overall: | 20260506 17:16:53.786 | INFO | BytesWritten : 1622776299269 | 20260506 17:16:53.786 | INFO | NumEvents : 785106571640 | 20260506 17:16:55.532 | INFO | Analyzing chunk frames = [3080192,3080704) | 20260506 17:16:55.532 | INFO | MemInfo: { | 20260506 17:16:55.532 | INFO | "gpu" : " 17 GB", | 20260506 17:16:55.532 | INFO | "huge" : "", | 20260506 17:16:55.532 | INFO | "huge_shared" : " 27 GB", | 20260506 17:16:55.532 | INFO | "malloc" : " 14 GB", | 20260506 17:16:55.532 | INFO | "pinned" : " 29 GB" | 20260506 17:16:55.532 | INFO | } | 20260506 17:16:55.532 | INFO | Perf report for the last 5.14184 seconds: | 20260506 17:16:55.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:16:55.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:55.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:55.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:55.532 | INFO | ANALYSIS : 62.5%, 2.9, 95.1ms, 0 | 20260506 17:16:55.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:16:55.532 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260506 17:16:55.532 | INFO | PRE_HQ : 60.4%, 1.9, 58.5ms, 0 | 20260506 17:16:55.532 | INFO | BAZWRITER : 63.6%, 4.7, 155.2ms, 0 | 20260506 17:16:55.532 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 17:16:55.532 | INFO | Uploaded 36GiB to GPU in 2.04721s (17.5849GiB/s) | 20260506 17:16:55.532 | INFO | Downloaded 38GiB to GPU in 2.20496s (17.2339GiB/s) | 20260506 17:17:00.668 | INFO | Analyzing chunk frames = [3080704,3081216) | 20260506 17:17:00.668 | INFO | MemInfo: { | 20260506 17:17:00.668 | INFO | "gpu" : " 17 GB", | 20260506 17:17:00.668 | INFO | "huge" : "", | 20260506 17:17:00.668 | INFO | "huge_shared" : " 27 GB", | 20260506 17:17:00.668 | INFO | "malloc" : " 14 GB", | 20260506 17:17:00.668 | INFO | "pinned" : " 29 GB" | 20260506 17:17:00.668 | INFO | } | 20260506 17:17:00.668 | INFO | Perf report for the last 5.13656 seconds: | 20260506 17:17:00.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:17:00.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:00.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:00.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:00.669 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:17:00.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:00.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:00.669 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 17:17:00.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:00.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:00.669 | INFO | Uploaded 36GiB to GPU in 2.00378s (17.966GiB/s) | 20260506 17:17:00.669 | INFO | Downloaded 34GiB to GPU in 1.93318s (17.5876GiB/s) | 20260506 17:17:05.808 | INFO | Analyzing chunk frames = [3081216,3081728) | 20260506 17:17:05.808 | INFO | MemInfo: { | 20260506 17:17:05.808 | INFO | "gpu" : " 17 GB", | 20260506 17:17:05.808 | INFO | "huge" : "", | 20260506 17:17:05.808 | INFO | "huge_shared" : " 27 GB", | 20260506 17:17:05.808 | INFO | "malloc" : " 14 GB", | 20260506 17:17:05.808 | INFO | "pinned" : " 29 GB" | 20260506 17:17:05.808 | INFO | } | 20260506 17:17:05.808 | INFO | Perf report for the last 5.13961 seconds: | 20260506 17:17:05.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:17:05.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:05.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:05.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:05.808 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 17:17:05.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:05.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:05.808 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:17:05.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:05.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:05.808 | INFO | Uploaded 36GiB to GPU in 1.97973s (18.1843GiB/s) | 20260506 17:17:05.809 | INFO | Downloaded 34GiB to GPU in 1.92516s (17.6609GiB/s) | 20260506 17:17:10.948 | INFO | Analyzing chunk frames = [3081728,3082240) | 20260506 17:17:10.948 | INFO | MemInfo: { | 20260506 17:17:10.948 | INFO | "gpu" : " 17 GB", | 20260506 17:17:10.948 | INFO | "huge" : "", | 20260506 17:17:10.948 | INFO | "huge_shared" : " 27 GB", | 20260506 17:17:10.948 | INFO | "malloc" : " 14 GB", | 20260506 17:17:10.948 | INFO | "pinned" : " 29 GB" | 20260506 17:17:10.948 | INFO | } | 20260506 17:17:10.948 | INFO | Perf report for the last 5.14001 seconds: | 20260506 17:17:10.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:17:10.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:10.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:10.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:10.948 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 17:17:10.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:10.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:10.948 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:17:10.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:10.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:10.948 | INFO | Uploaded 36GiB to GPU in 1.93898s (18.5665GiB/s) | 20260506 17:17:10.948 | INFO | Downloaded 34GiB to GPU in 1.87313s (18.1515GiB/s) | 20260506 17:17:16.083 | INFO | Analyzing chunk frames = [3082240,3082752) | 20260506 17:17:16.083 | INFO | MemInfo: { | 20260506 17:17:16.083 | INFO | "gpu" : " 17 GB", | 20260506 17:17:16.083 | INFO | "huge" : "", | 20260506 17:17:16.083 | INFO | "huge_shared" : " 27 GB", | 20260506 17:17:16.083 | INFO | "malloc" : " 14 GB", | 20260506 17:17:16.083 | INFO | "pinned" : " 29 GB" | 20260506 17:17:16.083 | INFO | } | 20260506 17:17:16.083 | INFO | Perf report for the last 5.13522 seconds: | 20260506 17:17:16.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:17:16.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:16.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:16.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:16.083 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 17:17:16.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:16.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:16.084 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:17:16.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:16.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:16.084 | INFO | Uploaded 36GiB to GPU in 2.01997s (17.822GiB/s) | 20260506 17:17:16.084 | INFO | Downloaded 34GiB to GPU in 1.91004s (17.8007GiB/s) | 20260506 17:17:21.223 | INFO | Analyzing chunk frames = [3082752,3083264) | 20260506 17:17:21.223 | INFO | MemInfo: { | 20260506 17:17:21.223 | INFO | "gpu" : " 17 GB", | 20260506 17:17:21.223 | INFO | "huge" : "", | 20260506 17:17:21.223 | INFO | "huge_shared" : " 27 GB", | 20260506 17:17:21.223 | INFO | "malloc" : " 14 GB", | 20260506 17:17:21.224 | INFO | "pinned" : " 29 GB" | 20260506 17:17:21.224 | INFO | } | 20260506 17:17:21.224 | INFO | Perf report for the last 5.14025 seconds: | 20260506 17:17:21.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:17:21.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:21.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:21.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:21.224 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 17:17:21.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:21.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:21.224 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:17:21.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:21.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:21.224 | INFO | Uploaded 36GiB to GPU in 2.0692s (17.398GiB/s) | 20260506 17:17:21.224 | INFO | Downloaded 34GiB to GPU in 1.95462s (17.3946GiB/s) | 20260506 17:17:26.358 | INFO | Analyzing chunk frames = [3083264,3083776) | 20260506 17:17:26.358 | INFO | MemInfo: { | 20260506 17:17:26.358 | INFO | "gpu" : " 17 GB", | 20260506 17:17:26.358 | INFO | "huge" : "", | 20260506 17:17:26.358 | INFO | "huge_shared" : " 27 GB", | 20260506 17:17:26.358 | INFO | "malloc" : " 14 GB", | 20260506 17:17:26.358 | INFO | "pinned" : " 29 GB" | 20260506 17:17:26.358 | INFO | } | 20260506 17:17:26.359 | INFO | Perf report for the last 5.1349 seconds: | 20260506 17:17:26.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:17:26.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:26.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:26.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:26.359 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 17:17:26.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:26.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:26.359 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:17:26.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:26.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:26.359 | INFO | Uploaded 36GiB to GPU in 2.01252s (17.888GiB/s) | 20260506 17:17:26.359 | INFO | Downloaded 34GiB to GPU in 1.97995s (17.1721GiB/s) | 20260506 17:17:31.501 | INFO | Analyzing chunk frames = [3083776,3084288) | 20260506 17:17:31.501 | INFO | MemInfo: { | 20260506 17:17:31.501 | INFO | "gpu" : " 17 GB", | 20260506 17:17:31.501 | INFO | "huge" : "", | 20260506 17:17:31.502 | INFO | "huge_shared" : " 27 GB", | 20260506 17:17:31.502 | INFO | "malloc" : " 14 GB", | 20260506 17:17:31.502 | INFO | "pinned" : " 29 GB" | 20260506 17:17:31.502 | INFO | } | 20260506 17:17:31.502 | INFO | Perf report for the last 5.14313 seconds: | 20260506 17:17:31.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:17:31.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:31.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:31.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:31.502 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:17:31.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:31.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:31.502 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:17:31.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:31.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:31.502 | INFO | Uploaded 36GiB to GPU in 2.01826s (17.8372GiB/s) | 20260506 17:17:31.502 | INFO | Downloaded 34GiB to GPU in 1.97463s (17.2184GiB/s) | 20260506 17:17:31.616 | INFO | Starting a round of Baz IO | 20260506 17:17:34.754 | INFO | Populating RTMetrics report | 20260506 17:17:34.754 | INFO | RTMetrics report populated | 20260506 17:17:34.754 | INFO | Opening tmp RTMetrics json file | 20260506 17:17:34.778 | INFO | Serializing RTMetrics to json | 20260506 17:17:34.782 | INFO | Serialization done, swapping tmp file now... | 20260506 17:17:34.785 | INFO | RTMetrics done | 20260506 17:17:34.991 | INFO | Finished a round of Baz IO: Wrote 2.069 GiB in 3.375 seconds | 20260506 17:17:34.991 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.74% / 0.01% / 64.25% | 20260506 17:17:34.991 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.42146% / 1.98682e-05% / 5.96046e-05% | 20260506 17:17:34.991 | INFO | Baz IO overall: | 20260506 17:17:34.991 | INFO | BytesWritten : 1624997982930 | 20260506 17:17:34.991 | INFO | NumEvents : 786178118929 | 20260506 17:17:36.649 | INFO | Analyzing chunk frames = [3084288,3084800) | 20260506 17:17:36.649 | INFO | MemInfo: { | 20260506 17:17:36.649 | INFO | "gpu" : " 17 GB", | 20260506 17:17:36.649 | INFO | "huge" : "", | 20260506 17:17:36.649 | INFO | "huge_shared" : " 27 GB", | 20260506 17:17:36.649 | INFO | "malloc" : " 14 GB", | 20260506 17:17:36.649 | INFO | "pinned" : " 29 GB" | 20260506 17:17:36.649 | INFO | } | 20260506 17:17:36.649 | INFO | Perf report for the last 5.14741 seconds: | 20260506 17:17:36.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:17:36.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:36.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:36.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:36.649 | INFO | ANALYSIS : 62.6%, 2.9, 95.3ms, 0 | 20260506 17:17:36.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:36.649 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 17:17:36.649 | INFO | PRE_HQ : 60.3%, 1.9, 58.3ms, 0 | 20260506 17:17:36.649 | INFO | BAZWRITER : 65.3%, 4.8, 161.2ms, 0 | 20260506 17:17:36.649 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 17:17:36.649 | INFO | Uploaded 36GiB to GPU in 2.06214s (17.4576GiB/s) | 20260506 17:17:36.649 | INFO | Downloaded 38GiB to GPU in 2.20305s (17.2488GiB/s) | 20260506 17:17:41.786 | INFO | Analyzing chunk frames = [3084800,3085312) | 20260506 17:17:41.786 | INFO | MemInfo: { | 20260506 17:17:41.786 | INFO | "gpu" : " 17 GB", | 20260506 17:17:41.786 | INFO | "huge" : "", | 20260506 17:17:41.786 | INFO | "huge_shared" : " 27 GB", | 20260506 17:17:41.786 | INFO | "malloc" : " 14 GB", | 20260506 17:17:41.786 | INFO | "pinned" : " 29 GB" | 20260506 17:17:41.786 | INFO | } | 20260506 17:17:41.786 | INFO | Perf report for the last 5.13685 seconds: | 20260506 17:17:41.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:17:41.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:41.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:41.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:41.786 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:17:41.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:41.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:41.786 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 17:17:41.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:41.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:41.786 | INFO | Uploaded 36GiB to GPU in 2.00381s (17.9658GiB/s) | 20260506 17:17:41.786 | INFO | Downloaded 34GiB to GPU in 1.93278s (17.5912GiB/s) | 20260506 17:17:46.928 | INFO | Analyzing chunk frames = [3085312,3085824) | 20260506 17:17:46.928 | INFO | MemInfo: { | 20260506 17:17:46.928 | INFO | "gpu" : " 17 GB", | 20260506 17:17:46.928 | INFO | "huge" : "", | 20260506 17:17:46.928 | INFO | "huge_shared" : " 27 GB", | 20260506 17:17:46.928 | INFO | "malloc" : " 14 GB", | 20260506 17:17:46.928 | INFO | "pinned" : " 29 GB" | 20260506 17:17:46.928 | INFO | } | 20260506 17:17:46.928 | INFO | Perf report for the last 5.14182 seconds: | 20260506 17:17:46.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:17:46.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:46.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:46.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:46.928 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 17:17:46.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:46.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:46.928 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:17:46.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:46.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:46.928 | INFO | Uploaded 36GiB to GPU in 1.97514s (18.2266GiB/s) | 20260506 17:17:46.928 | INFO | Downloaded 34GiB to GPU in 1.92466s (17.6655GiB/s) | 20260506 17:17:52.059 | INFO | Analyzing chunk frames = [3085824,3086336) | 20260506 17:17:52.059 | INFO | MemInfo: { | 20260506 17:17:52.059 | INFO | "gpu" : " 17 GB", | 20260506 17:17:52.059 | INFO | "huge" : "", | 20260506 17:17:52.059 | INFO | "huge_shared" : " 27 GB", | 20260506 17:17:52.059 | INFO | "malloc" : " 14 GB", | 20260506 17:17:52.059 | INFO | "pinned" : " 29 GB" | 20260506 17:17:52.059 | INFO | } | 20260506 17:17:52.059 | INFO | Perf report for the last 5.13128 seconds: | 20260506 17:17:52.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:17:52.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:52.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:52.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:52.059 | INFO | ANALYSIS : 60.1%, 3.0, 93.0ms, 0 | 20260506 17:17:52.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:52.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:52.059 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:17:52.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:52.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:52.059 | INFO | Uploaded 36GiB to GPU in 1.93781s (18.5777GiB/s) | 20260506 17:17:52.059 | INFO | Downloaded 34GiB to GPU in 1.86838s (18.1976GiB/s) | 20260506 17:17:57.198 | INFO | Analyzing chunk frames = [3086336,3086848) | 20260506 17:17:57.198 | INFO | MemInfo: { | 20260506 17:17:57.198 | INFO | "gpu" : " 17 GB", | 20260506 17:17:57.198 | INFO | "huge" : "", | 20260506 17:17:57.198 | INFO | "huge_shared" : " 27 GB", | 20260506 17:17:57.198 | INFO | "malloc" : " 14 GB", | 20260506 17:17:57.198 | INFO | "pinned" : " 29 GB" | 20260506 17:17:57.198 | INFO | } | 20260506 17:17:57.199 | INFO | Perf report for the last 5.13945 seconds: | 20260506 17:17:57.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:17:57.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:57.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:57.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:57.199 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 17:17:57.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:57.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:57.199 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 17:17:57.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:57.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:17:57.199 | INFO | Uploaded 36GiB to GPU in 2.00972s (17.913GiB/s) | 20260506 17:17:57.199 | INFO | Downloaded 34GiB to GPU in 1.91472s (17.7572GiB/s) | 20260506 17:18:02.342 | INFO | Analyzing chunk frames = [3086848,3087360) | 20260506 17:18:02.342 | INFO | MemInfo: { | 20260506 17:18:02.342 | INFO | "gpu" : " 17 GB", | 20260506 17:18:02.342 | INFO | "huge" : "", | 20260506 17:18:02.342 | INFO | "huge_shared" : " 27 GB", | 20260506 17:18:02.342 | INFO | "malloc" : " 14 GB", | 20260506 17:18:02.342 | INFO | "pinned" : " 29 GB" | 20260506 17:18:02.342 | INFO | } | 20260506 17:18:02.342 | INFO | Perf report for the last 5.14338 seconds: | 20260506 17:18:02.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:18:02.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:02.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:02.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:02.342 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 17:18:02.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:02.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:02.342 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:18:02.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:02.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:02.342 | INFO | Uploaded 36GiB to GPU in 2.02656s (17.7641GiB/s) | 20260506 17:18:02.342 | INFO | Downloaded 34GiB to GPU in 1.94358s (17.4934GiB/s) | 20260506 17:18:07.477 | INFO | Analyzing chunk frames = [3087360,3087872) | 20260506 17:18:07.477 | INFO | MemInfo: { | 20260506 17:18:07.477 | INFO | "gpu" : " 17 GB", | 20260506 17:18:07.477 | INFO | "huge" : "", | 20260506 17:18:07.477 | INFO | "huge_shared" : " 27 GB", | 20260506 17:18:07.477 | INFO | "malloc" : " 14 GB", | 20260506 17:18:07.477 | INFO | "pinned" : " 29 GB" | 20260506 17:18:07.477 | INFO | } | 20260506 17:18:07.477 | INFO | Perf report for the last 5.13555 seconds: | 20260506 17:18:07.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:18:07.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:07.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:07.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:07.478 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:18:07.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:07.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:07.478 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:18:07.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:07.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:07.478 | INFO | Uploaded 36GiB to GPU in 1.98965s (18.0936GiB/s) | 20260506 17:18:07.478 | INFO | Downloaded 34GiB to GPU in 2.00132s (16.9888GiB/s) | 20260506 17:18:12.620 | INFO | Analyzing chunk frames = [3087872,3088384) | 20260506 17:18:12.620 | INFO | MemInfo: { | 20260506 17:18:12.620 | INFO | "gpu" : " 17 GB", | 20260506 17:18:12.620 | INFO | "huge" : "", | 20260506 17:18:12.620 | INFO | "huge_shared" : " 27 GB", | 20260506 17:18:12.620 | INFO | "malloc" : " 14 GB", | 20260506 17:18:12.620 | INFO | "pinned" : " 29 GB" | 20260506 17:18:12.620 | INFO | } | 20260506 17:18:12.620 | INFO | Perf report for the last 5.1425 seconds: | 20260506 17:18:12.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:18:12.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:12.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:12.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:12.620 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:18:12.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:12.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:12.620 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:18:12.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:12.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:12.620 | INFO | Uploaded 36GiB to GPU in 1.996s (18.0361GiB/s) | 20260506 17:18:12.620 | INFO | Downloaded 34GiB to GPU in 1.96024s (17.3448GiB/s) | 20260506 17:18:12.620 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:18:12.620 | INFO | Total mem usage: 18 GiB | 20260506 17:18:12.620 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:18:12.620 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -13.8%) | 20260506 17:18:12.620 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:18:12.620 | INFO | High Water Breakdown: | 20260506 17:18:12.620 | INFO | 61.6% : 9.74 GiB ( 85.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:18:12.620 | INFO | 38.3% : 6.06 GiB ( 90.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:18:12.620 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:18:12.620 | INFO | ------------------------------------------ | 20260506 17:18:12.621 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:18:12.621 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:18:12.621 | INFO | Total mem usage: 26.5 GiB | 20260506 17:18:12.621 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:18:12.621 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:18:12.621 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:18:12.621 | INFO | High Water Breakdown: | 20260506 17:18:12.621 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:18:12.621 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:18:12.621 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:18:12.621 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:18:12.621 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:18:12.621 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:18:12.621 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:18:12.621 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:18:12.621 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:18:12.621 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:18:12.621 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:18:12.621 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:18:12.621 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:18:12.621 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:18:12.621 | INFO | ------------------------------------------ | 20260506 17:18:12.621 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:18:12.621 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:18:12.621 | INFO | Total mem usage: 25.6 GiB | 20260506 17:18:12.621 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:18:12.621 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:18:12.621 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:18:12.621 | INFO | High Water Breakdown: | 20260506 17:18:12.621 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:18:12.621 | INFO | ------------------------------------------ | 20260506 17:18:12.621 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:18:12.621 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:18:12.621 | INFO | Total mem usage: 15.7 GiB | 20260506 17:18:12.621 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:18:12.621 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:18:12.621 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 17:18:12.621 | INFO | High Water Breakdown: | 20260506 17:18:12.621 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:18:12.621 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:18:12.621 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:18:12.621 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:18:12.621 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:18:12.621 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:18:12.621 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:18:12.621 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:18:12.621 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:18:12.621 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:18:12.621 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:18:12.621 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:18:12.621 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:18:12.621 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:18:12.621 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:18:12.621 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:18:12.621 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:18:12.621 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:18:12.621 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:18:12.621 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:18:12.621 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:18:12.621 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:18:12.621 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:18:12.621 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:18:12.621 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:18:12.621 | INFO | ------------------------------------------ | 20260506 17:18:12.621 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:18:12.621 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:18:12.621 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:18:12.621 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:18:12.621 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.56e+01, 1.35e+01 / 2.05e+00 | 20260506 17:18:12.621 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 9.99e-02, 9.64e-03 / 5.09e-03 | 20260506 17:18:12.621 | INFO | PerfReport: 11.62%, Baseline -- 3.64e+00 / 4.43e+00, 5.67e-01 / 3.41e-01 | 20260506 17:18:12.621 | INFO | PerfReport: 3.28%, Histogram -- 1.03e+00 / 1.47e+00, 1.88e-01 / 1.27e-01 | 20260506 17:18:12.621 | INFO | PerfReport: 15.92%, DME -- 3.99e+01 / 4.42e+01, 3.72e+00 / 2.40e+00 | 20260506 17:18:12.621 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.41e+00 / 2.13e+00 | 20260506 17:18:12.621 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.28e+00 / 2.84e+00, 3.87e-01 / 2.62e-01 | 20260506 17:18:12.621 | INFO | PerfReport: 3.47%, Metrics -- 1.09e+00 / 1.45e+00, 2.86e-01 / 1.81e-01 | 20260506 17:18:12.621 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.75e-01 / 4.28e-01, 5.94e-02 / 2.15e-02 | 20260506 17:18:12.621 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:18:12.621 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:18:12.621 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:18:12.622 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 17:18:12.622 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260506 17:18:12.622 | INFO | PerfReport: 21.10%, Download -- 2.05e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260506 17:18:12.622 | INFO | PerfReport: 32.24%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 17:18:12.622 | INFO | PerfReport: 1.43%, LockUpload -- 1.39e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260506 17:18:12.622 | INFO | PerfReport: 1.06%, LockDownload -- 1.03e+00 / 0.00e+00, 2.67e+00 / 0.00e+00 | 20260506 17:18:12.622 | INFO | PerfReport: 22.66%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 17:18:12.622 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260506 17:18:12.734 | INFO | Starting a round of Baz IO | 20260506 17:18:15.878 | INFO | Populating RTMetrics report | 20260506 17:18:15.878 | INFO | RTMetrics report populated | 20260506 17:18:15.878 | INFO | Opening tmp RTMetrics json file | 20260506 17:18:15.879 | INFO | Serializing RTMetrics to json | 20260506 17:18:15.881 | INFO | Serialization done, swapping tmp file now... | 20260506 17:18:15.881 | INFO | RTMetrics done | 20260506 17:18:15.991 | INFO | Finished a round of Baz IO: Wrote 2.067 GiB in 3.256 seconds | 20260506 17:18:15.991 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.74% / 0.01% / 64.26% | 20260506 17:18:15.991 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.41526% / 1.19209e-05% / 4.76837e-05% | 20260506 17:18:15.991 | INFO | Baz IO overall: | 20260506 17:18:15.991 | INFO | BytesWritten : 1627217710049 | 20260506 17:18:15.991 | INFO | NumEvents : 787248688285 | 20260506 17:18:17.759 | INFO | Analyzing chunk frames = [3088384,3088896) | 20260506 17:18:17.759 | INFO | MemInfo: { | 20260506 17:18:17.759 | INFO | "gpu" : " 17 GB", | 20260506 17:18:17.759 | INFO | "huge" : "", | 20260506 17:18:17.759 | INFO | "huge_shared" : " 27 GB", | 20260506 17:18:17.759 | INFO | "malloc" : " 14 GB", | 20260506 17:18:17.759 | INFO | "pinned" : " 29 GB" | 20260506 17:18:17.759 | INFO | } | 20260506 17:18:17.759 | INFO | Perf report for the last 5.13925 seconds: | 20260506 17:18:17.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:18:17.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:17.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:17.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:17.759 | INFO | ANALYSIS : 62.8%, 2.9, 95.3ms, 0 | 20260506 17:18:17.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:17.759 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260506 17:18:17.759 | INFO | PRE_HQ : 60.3%, 1.9, 58.4ms, 0 | 20260506 17:18:17.759 | INFO | BAZWRITER : 63.0%, 4.6, 147.6ms, 0 | 20260506 17:18:17.760 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 17:18:17.760 | INFO | Uploaded 36GiB to GPU in 2.04242s (17.6262GiB/s) | 20260506 17:18:17.760 | INFO | Downloaded 38GiB to GPU in 2.20342s (17.2459GiB/s) | 20260506 17:18:22.898 | INFO | Analyzing chunk frames = [3088896,3089408) | 20260506 17:18:22.898 | INFO | MemInfo: { | 20260506 17:18:22.898 | INFO | "gpu" : " 17 GB", | 20260506 17:18:22.898 | INFO | "huge" : "", | 20260506 17:18:22.898 | INFO | "huge_shared" : " 27 GB", | 20260506 17:18:22.898 | INFO | "malloc" : " 14 GB", | 20260506 17:18:22.898 | INFO | "pinned" : " 29 GB" | 20260506 17:18:22.898 | INFO | } | 20260506 17:18:22.899 | INFO | Perf report for the last 5.13926 seconds: | 20260506 17:18:22.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:18:22.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:22.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:22.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:22.899 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 17:18:22.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:22.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:22.899 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:18:22.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:22.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:22.899 | INFO | Uploaded 36GiB to GPU in 2.02166s (17.8071GiB/s) | 20260506 17:18:22.899 | INFO | Downloaded 34GiB to GPU in 1.97384s (17.2253GiB/s) | 20260506 17:18:28.037 | INFO | Analyzing chunk frames = [3089408,3089920) | 20260506 17:18:28.037 | INFO | MemInfo: { | 20260506 17:18:28.037 | INFO | "gpu" : " 17 GB", | 20260506 17:18:28.037 | INFO | "huge" : "", | 20260506 17:18:28.037 | INFO | "huge_shared" : " 27 GB", | 20260506 17:18:28.037 | INFO | "malloc" : " 14 GB", | 20260506 17:18:28.037 | INFO | "pinned" : " 29 GB" | 20260506 17:18:28.037 | INFO | } | 20260506 17:18:28.037 | INFO | Perf report for the last 5.13854 seconds: | 20260506 17:18:28.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:18:28.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:28.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:28.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:28.037 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 17:18:28.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:28.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:28.037 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:18:28.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:28.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:28.037 | INFO | Uploaded 36GiB to GPU in 1.97856s (18.1951GiB/s) | 20260506 17:18:28.037 | INFO | Downloaded 34GiB to GPU in 1.93021s (17.6146GiB/s) | 20260506 17:18:33.174 | INFO | Analyzing chunk frames = [3089920,3090432) | 20260506 17:18:33.174 | INFO | MemInfo: { | 20260506 17:18:33.174 | INFO | "gpu" : " 17 GB", | 20260506 17:18:33.174 | INFO | "huge" : "", | 20260506 17:18:33.174 | INFO | "huge_shared" : " 27 GB", | 20260506 17:18:33.174 | INFO | "malloc" : " 14 GB", | 20260506 17:18:33.174 | INFO | "pinned" : " 29 GB" | 20260506 17:18:33.174 | INFO | } | 20260506 17:18:33.174 | INFO | Perf report for the last 5.13691 seconds: | 20260506 17:18:33.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:18:33.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:33.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:33.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:33.174 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 17:18:33.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:33.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:33.174 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:18:33.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:33.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:33.174 | INFO | Uploaded 36GiB to GPU in 1.96342s (18.3354GiB/s) | 20260506 17:18:33.174 | INFO | Downloaded 34GiB to GPU in 1.87693s (18.1147GiB/s) | 20260506 17:18:38.313 | INFO | Analyzing chunk frames = [3090432,3090944) | 20260506 17:18:38.313 | INFO | MemInfo: { | 20260506 17:18:38.313 | INFO | "gpu" : " 17 GB", | 20260506 17:18:38.313 | INFO | "huge" : "", | 20260506 17:18:38.313 | INFO | "huge_shared" : " 27 GB", | 20260506 17:18:38.313 | INFO | "malloc" : " 14 GB", | 20260506 17:18:38.313 | INFO | "pinned" : " 29 GB" | 20260506 17:18:38.313 | INFO | } | 20260506 17:18:38.313 | INFO | Perf report for the last 5.13897 seconds: | 20260506 17:18:38.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:18:38.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:38.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:38.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:38.313 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 17:18:38.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:38.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:38.313 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 17:18:38.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:38.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:38.313 | INFO | Uploaded 36GiB to GPU in 2.00733s (17.9343GiB/s) | 20260506 17:18:38.313 | INFO | Downloaded 34GiB to GPU in 1.91056s (17.7959GiB/s) | 20260506 17:18:43.452 | INFO | Analyzing chunk frames = [3090944,3091456) | 20260506 17:18:43.452 | INFO | MemInfo: { | 20260506 17:18:43.452 | INFO | "gpu" : " 17 GB", | 20260506 17:18:43.452 | INFO | "huge" : "", | 20260506 17:18:43.452 | INFO | "huge_shared" : " 27 GB", | 20260506 17:18:43.452 | INFO | "malloc" : " 14 GB", | 20260506 17:18:43.452 | INFO | "pinned" : " 29 GB" | 20260506 17:18:43.452 | INFO | } | 20260506 17:18:43.452 | INFO | Perf report for the last 5.13942 seconds: | 20260506 17:18:43.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:18:43.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:43.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:43.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:43.452 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 17:18:43.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:43.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:43.453 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:18:43.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:43.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:43.453 | INFO | Uploaded 36GiB to GPU in 2.0348s (17.6922GiB/s) | 20260506 17:18:43.453 | INFO | Downloaded 34GiB to GPU in 1.94728s (17.4603GiB/s) | 20260506 17:18:48.595 | INFO | Analyzing chunk frames = [3091456,3091968) | 20260506 17:18:48.595 | INFO | MemInfo: { | 20260506 17:18:48.595 | INFO | "gpu" : " 17 GB", | 20260506 17:18:48.595 | INFO | "huge" : "", | 20260506 17:18:48.595 | INFO | "huge_shared" : " 27 GB", | 20260506 17:18:48.595 | INFO | "malloc" : " 14 GB", | 20260506 17:18:48.595 | INFO | "pinned" : " 29 GB" | 20260506 17:18:48.595 | INFO | } | 20260506 17:18:48.595 | INFO | Perf report for the last 5.14265 seconds: | 20260506 17:18:48.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:18:48.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:48.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:48.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:48.595 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 17:18:48.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:48.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:48.595 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:18:48.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:48.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:48.595 | INFO | Uploaded 36GiB to GPU in 1.98382s (18.1468GiB/s) | 20260506 17:18:48.595 | INFO | Downloaded 34GiB to GPU in 1.99763s (17.0202GiB/s) | 20260506 17:18:53.733 | INFO | Analyzing chunk frames = [3091968,3092480) | 20260506 17:18:53.733 | INFO | MemInfo: { | 20260506 17:18:53.733 | INFO | "gpu" : " 17 GB", | 20260506 17:18:53.733 | INFO | "huge" : "", | 20260506 17:18:53.733 | INFO | "huge_shared" : " 27 GB", | 20260506 17:18:53.733 | INFO | "malloc" : " 14 GB", | 20260506 17:18:53.733 | INFO | "pinned" : " 29 GB" | 20260506 17:18:53.733 | INFO | } | 20260506 17:18:53.733 | INFO | Perf report for the last 5.13825 seconds: | 20260506 17:18:53.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:18:53.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:53.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:53.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:53.733 | INFO | ANALYSIS : 62.6%, 3.0, 97.4ms, 0 | 20260506 17:18:53.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:53.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:53.734 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 17:18:53.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:53.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:53.734 | INFO | Uploaded 36GiB to GPU in 2.09563s (17.1786GiB/s) | 20260506 17:18:53.734 | INFO | Downloaded 34GiB to GPU in 2.00734s (16.9379GiB/s) | 20260506 17:18:53.868 | INFO | Starting a round of Baz IO | 20260506 17:18:57.029 | INFO | Populating RTMetrics report | 20260506 17:18:57.030 | INFO | RTMetrics report populated | 20260506 17:18:57.030 | INFO | Opening tmp RTMetrics json file | 20260506 17:18:57.053 | INFO | Serializing RTMetrics to json | 20260506 17:18:57.105 | INFO | Serialization done, swapping tmp file now... | 20260506 17:18:57.107 | INFO | RTMetrics done | 20260506 17:18:57.222 | INFO | Finished a round of Baz IO: Wrote 2.069 GiB in 3.354 seconds | 20260506 17:18:57.222 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.73% / 0.01% / 64.26% | 20260506 17:18:57.222 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.41565% / 1.19209e-05% / 5.96046e-05% | 20260506 17:18:57.222 | INFO | Baz IO overall: | 20260506 17:18:57.222 | INFO | BytesWritten : 1629439532648 | 20260506 17:18:57.222 | INFO | NumEvents : 788321034169 | 20260506 17:18:58.873 | INFO | Analyzing chunk frames = [3092480,3092992) | 20260506 17:18:58.873 | INFO | MemInfo: { | 20260506 17:18:58.873 | INFO | "gpu" : " 17 GB", | 20260506 17:18:58.873 | INFO | "huge" : "", | 20260506 17:18:58.873 | INFO | "huge_shared" : " 27 GB", | 20260506 17:18:58.873 | INFO | "malloc" : " 14 GB", | 20260506 17:18:58.873 | INFO | "pinned" : " 29 GB" | 20260506 17:18:58.873 | INFO | } | 20260506 17:18:58.873 | INFO | Perf report for the last 5.13944 seconds: | 20260506 17:18:58.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:18:58.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:58.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:58.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:58.873 | INFO | ANALYSIS : 63.2%, 2.9, 95.8ms, 0 | 20260506 17:18:58.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:18:58.873 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260506 17:18:58.873 | INFO | PRE_HQ : 60.5%, 1.9, 59.5ms, 0 | 20260506 17:18:58.873 | INFO | BAZWRITER : 64.6%, 4.8, 158.0ms, 0 | 20260506 17:18:58.873 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 17:18:58.873 | INFO | Uploaded 36GiB to GPU in 2.10641s (17.0907GiB/s) | 20260506 17:18:58.873 | INFO | Downloaded 38GiB to GPU in 2.22176s (17.1036GiB/s) | 20260506 17:19:04.014 | INFO | Analyzing chunk frames = [3092992,3093504) | 20260506 17:19:04.014 | INFO | MemInfo: { | 20260506 17:19:04.014 | INFO | "gpu" : " 17 GB", | 20260506 17:19:04.014 | INFO | "huge" : "", | 20260506 17:19:04.014 | INFO | "huge_shared" : " 27 GB", | 20260506 17:19:04.014 | INFO | "malloc" : " 14 GB", | 20260506 17:19:04.014 | INFO | "pinned" : " 29 GB" | 20260506 17:19:04.014 | INFO | } | 20260506 17:19:04.014 | INFO | Perf report for the last 5.14141 seconds: | 20260506 17:19:04.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:19:04.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:04.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:04.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:04.014 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:19:04.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:04.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:04.014 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 17:19:04.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:04.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:04.015 | INFO | Uploaded 36GiB to GPU in 2.011s (17.9015GiB/s) | 20260506 17:19:04.015 | INFO | Downloaded 34GiB to GPU in 1.93918s (17.5332GiB/s) | 20260506 17:19:09.151 | INFO | Analyzing chunk frames = [3093504,3094016) | 20260506 17:19:09.151 | INFO | MemInfo: { | 20260506 17:19:09.151 | INFO | "gpu" : " 17 GB", | 20260506 17:19:09.151 | INFO | "huge" : "", | 20260506 17:19:09.151 | INFO | "huge_shared" : " 27 GB", | 20260506 17:19:09.151 | INFO | "malloc" : " 14 GB", | 20260506 17:19:09.151 | INFO | "pinned" : " 29 GB" | 20260506 17:19:09.151 | INFO | } | 20260506 17:19:09.151 | INFO | Perf report for the last 5.13658 seconds: | 20260506 17:19:09.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:19:09.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:09.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:09.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:09.151 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:19:09.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:09.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:09.151 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:19:09.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:09.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:09.151 | INFO | Uploaded 36GiB to GPU in 1.97512s (18.2267GiB/s) | 20260506 17:19:09.151 | INFO | Downloaded 34GiB to GPU in 1.92486s (17.6636GiB/s) | 20260506 17:19:14.290 | INFO | Analyzing chunk frames = [3094016,3094528) | 20260506 17:19:14.290 | INFO | MemInfo: { | 20260506 17:19:14.290 | INFO | "gpu" : " 17 GB", | 20260506 17:19:14.290 | INFO | "huge" : "", | 20260506 17:19:14.290 | INFO | "huge_shared" : " 27 GB", | 20260506 17:19:14.290 | INFO | "malloc" : " 14 GB", | 20260506 17:19:14.290 | INFO | "pinned" : " 29 GB" | 20260506 17:19:14.290 | INFO | } | 20260506 17:19:14.290 | INFO | Perf report for the last 5.13906 seconds: | 20260506 17:19:14.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:19:14.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:14.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:14.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:14.290 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:19:14.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:14.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:14.290 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:19:14.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:14.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:14.290 | INFO | Uploaded 36GiB to GPU in 1.93746s (18.581GiB/s) | 20260506 17:19:14.290 | INFO | Downloaded 34GiB to GPU in 1.86968s (18.1849GiB/s) | 20260506 17:19:19.430 | INFO | Analyzing chunk frames = [3094528,3095040) | 20260506 17:19:19.430 | INFO | MemInfo: { | 20260506 17:19:19.430 | INFO | "gpu" : " 17 GB", | 20260506 17:19:19.430 | INFO | "huge" : "", | 20260506 17:19:19.430 | INFO | "huge_shared" : " 27 GB", | 20260506 17:19:19.430 | INFO | "malloc" : " 14 GB", | 20260506 17:19:19.430 | INFO | "pinned" : " 29 GB" | 20260506 17:19:19.431 | INFO | } | 20260506 17:19:19.431 | INFO | Perf report for the last 5.14065 seconds: | 20260506 17:19:19.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:19:19.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:19.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:19.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:19.431 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 17:19:19.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:19.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:19.431 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:19:19.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:19.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:19.431 | INFO | Uploaded 36GiB to GPU in 2.02005s (17.8213GiB/s) | 20260506 17:19:19.431 | INFO | Downloaded 34GiB to GPU in 1.9123s (17.7797GiB/s) | 20260506 17:19:24.566 | INFO | Analyzing chunk frames = [3095040,3095552) | 20260506 17:19:24.566 | INFO | MemInfo: { | 20260506 17:19:24.566 | INFO | "gpu" : " 17 GB", | 20260506 17:19:24.566 | INFO | "huge" : "", | 20260506 17:19:24.566 | INFO | "huge_shared" : " 27 GB", | 20260506 17:19:24.566 | INFO | "malloc" : " 14 GB", | 20260506 17:19:24.566 | INFO | "pinned" : " 29 GB" | 20260506 17:19:24.566 | INFO | } | 20260506 17:19:24.566 | INFO | Perf report for the last 5.13536 seconds: | 20260506 17:19:24.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:19:24.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:24.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:24.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:24.566 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 17:19:24.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:24.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:24.566 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 17:19:24.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:24.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:24.566 | INFO | Uploaded 36GiB to GPU in 2.04027s (17.6447GiB/s) | 20260506 17:19:24.566 | INFO | Downloaded 34GiB to GPU in 1.94723s (17.4607GiB/s) | 20260506 17:19:29.707 | INFO | Analyzing chunk frames = [3095552,3096064) | 20260506 17:19:29.707 | INFO | MemInfo: { | 20260506 17:19:29.707 | INFO | "gpu" : " 17 GB", | 20260506 17:19:29.707 | INFO | "huge" : "", | 20260506 17:19:29.707 | INFO | "huge_shared" : " 27 GB", | 20260506 17:19:29.707 | INFO | "malloc" : " 14 GB", | 20260506 17:19:29.707 | INFO | "pinned" : " 29 GB" | 20260506 17:19:29.707 | INFO | } | 20260506 17:19:29.707 | INFO | Perf report for the last 5.14133 seconds: | 20260506 17:19:29.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:19:29.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:29.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:29.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:29.707 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 17:19:29.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:29.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:29.707 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 17:19:29.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:29.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:29.708 | INFO | Uploaded 36GiB to GPU in 2.02182s (17.8058GiB/s) | 20260506 17:19:29.708 | INFO | Downloaded 34GiB to GPU in 2.01807s (16.8477GiB/s) | 20260506 17:19:34.848 | INFO | Analyzing chunk frames = [3096064,3096576) | 20260506 17:19:34.848 | INFO | MemInfo: { | 20260506 17:19:34.848 | INFO | "gpu" : " 17 GB", | 20260506 17:19:34.848 | INFO | "huge" : "", | 20260506 17:19:34.848 | INFO | "huge_shared" : " 27 GB", | 20260506 17:19:34.848 | INFO | "malloc" : " 14 GB", | 20260506 17:19:34.848 | INFO | "pinned" : " 29 GB" | 20260506 17:19:34.848 | INFO | } | 20260506 17:19:34.848 | INFO | Perf report for the last 5.14092 seconds: | 20260506 17:19:34.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:19:34.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:34.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:34.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:34.848 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 17:19:34.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:34.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:34.848 | INFO | PRE_HQ : 31.7%, 1.0, 16.6ms, 0 | 20260506 17:19:34.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:34.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:34.849 | INFO | Uploaded 36GiB to GPU in 2.04994s (17.5615GiB/s) | 20260506 17:19:34.849 | INFO | Downloaded 34GiB to GPU in 1.99371s (17.0537GiB/s) | 20260506 17:19:34.849 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:19:34.849 | INFO | Total mem usage: 18 GiB | 20260506 17:19:34.849 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:19:34.849 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -17.1%) | 20260506 17:19:34.849 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:19:34.849 | INFO | High Water Breakdown: | 20260506 17:19:34.849 | INFO | 61.1% : 9.4 GiB ( 82.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:19:34.849 | INFO | 38.7% : 5.96 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:19:34.849 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:19:34.849 | INFO | ------------------------------------------ | 20260506 17:19:34.849 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:19:34.849 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:19:34.849 | INFO | Total mem usage: 26.5 GiB | 20260506 17:19:34.849 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:19:34.849 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:19:34.849 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:19:34.849 | INFO | High Water Breakdown: | 20260506 17:19:34.849 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:19:34.849 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:19:34.849 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:19:34.849 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:19:34.849 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:19:34.849 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:19:34.849 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:19:34.849 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:19:34.849 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:19:34.849 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:19:34.849 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:19:34.849 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:19:34.849 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:19:34.849 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:19:34.849 | INFO | ------------------------------------------ | 20260506 17:19:34.849 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:19:34.849 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:19:34.849 | INFO | Total mem usage: 25.6 GiB | 20260506 17:19:34.849 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:19:34.849 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:19:34.849 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:19:34.849 | INFO | High Water Breakdown: | 20260506 17:19:34.849 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:19:34.849 | INFO | ------------------------------------------ | 20260506 17:19:34.849 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:19:34.849 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:19:34.849 | INFO | Total mem usage: 15.7 GiB | 20260506 17:19:34.849 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:19:34.849 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:19:34.849 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:19:34.850 | INFO | High Water Breakdown: | 20260506 17:19:34.850 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:19:34.850 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:19:34.850 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:19:34.850 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:19:34.850 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:19:34.850 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:19:34.850 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:19:34.850 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:19:34.850 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:19:34.850 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:19:34.850 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:19:34.850 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:19:34.850 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:19:34.850 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:19:34.850 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:19:34.850 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:19:34.850 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:19:34.850 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:19:34.850 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:19:34.850 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:19:34.850 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:19:34.850 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:19:34.850 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:19:34.850 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:19:34.850 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:19:34.850 | INFO | ------------------------------------------ | 20260506 17:19:34.850 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:19:34.850 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:19:34.850 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:19:34.850 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:19:34.850 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.60e+01, 1.34e+01 / 2.23e+00 | 20260506 17:19:34.850 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.01e-01, 1.01e-02 / 6.67e-03 | 20260506 17:19:34.850 | INFO | PerfReport: 11.60%, Baseline -- 3.64e+00 / 4.43e+00, 5.57e-01 / 3.51e-01 | 20260506 17:19:34.850 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.85e-01 / 1.40e-01 | 20260506 17:19:34.850 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.46e+01, 4.04e+00 / 2.58e+00 | 20260506 17:19:34.850 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.38e+00 / 2.00e+00 | 20260506 17:19:34.850 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.29e+00 / 2.88e+00, 3.92e-01 / 2.57e-01 | 20260506 17:19:34.850 | INFO | PerfReport: 3.47%, Metrics -- 1.09e+00 / 1.43e+00, 2.86e-01 / 2.00e-01 | 20260506 17:19:34.850 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.76e-01 / 4.23e-01, 5.94e-02 / 2.45e-02 | 20260506 17:19:34.850 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:19:34.850 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:19:34.850 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:19:34.850 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 17:19:34.850 | INFO | PerfReport: 21.59%, Upload -- 2.10e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 17:19:34.850 | INFO | PerfReport: 21.18%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 17:19:34.850 | INFO | PerfReport: 32.19%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:19:34.850 | INFO | PerfReport: 1.54%, LockUpload -- 1.50e+00 / 0.00e+00, 5.95e+00 / 0.00e+00 | 20260506 17:19:34.850 | INFO | PerfReport: 1.06%, LockDownload -- 1.03e+00 / 0.00e+00, 2.63e+00 / 0.00e+00 | 20260506 17:19:34.850 | INFO | PerfReport: 22.42%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 17:19:34.850 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260506 17:19:34.976 | INFO | Starting a round of Baz IO | 20260506 17:19:38.105 | INFO | Populating RTMetrics report | 20260506 17:19:38.105 | INFO | RTMetrics report populated | 20260506 17:19:38.105 | INFO | Opening tmp RTMetrics json file | 20260506 17:19:38.106 | INFO | Serializing RTMetrics to json | 20260506 17:19:38.108 | INFO | Serialization done, swapping tmp file now... | 20260506 17:19:38.108 | INFO | RTMetrics done | 20260506 17:19:38.848 | INFO | Finished a round of Baz IO: Wrote 2.067 GiB in 3.872 seconds | 20260506 17:19:38.848 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.72% / 0.01% / 64.27% | 20260506 17:19:38.848 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.40852% / 1.98682e-05% / 3.97364e-05% | 20260506 17:19:38.848 | INFO | Baz IO overall: | 20260506 17:19:38.848 | INFO | BytesWritten : 1631659165824 | 20260506 17:19:38.848 | INFO | NumEvents : 789392016091 | 20260506 17:19:39.988 | INFO | Analyzing chunk frames = [3096576,3097088) | 20260506 17:19:39.988 | INFO | MemInfo: { | 20260506 17:19:39.988 | INFO | "gpu" : " 17 GB", | 20260506 17:19:39.988 | INFO | "huge" : "", | 20260506 17:19:39.988 | INFO | "huge_shared" : " 27 GB", | 20260506 17:19:39.988 | INFO | "malloc" : " 14 GB", | 20260506 17:19:39.988 | INFO | "pinned" : " 29 GB" | 20260506 17:19:39.988 | INFO | } | 20260506 17:19:39.988 | INFO | Perf report for the last 5.13957 seconds: | 20260506 17:19:39.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:19:39.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:39.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:39.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:39.988 | INFO | ANALYSIS : 62.8%, 2.9, 95.3ms, 0 | 20260506 17:19:39.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:39.988 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 17:19:39.988 | INFO | PRE_HQ : 60.6%, 1.9, 58.9ms, 0 | 20260506 17:19:39.988 | INFO | BAZWRITER : 74.9%, 4.8, 186.1ms, 0 | 20260506 17:19:39.988 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 17:19:39.988 | INFO | Uploaded 36GiB to GPU in 2.0812s (17.2977GiB/s) | 20260506 17:19:39.988 | INFO | Downloaded 38GiB to GPU in 2.2252s (17.0771GiB/s) | 20260506 17:19:45.126 | INFO | Analyzing chunk frames = [3097088,3097600) | 20260506 17:19:45.126 | INFO | MemInfo: { | 20260506 17:19:45.126 | INFO | "gpu" : " 17 GB", | 20260506 17:19:45.126 | INFO | "huge" : "", | 20260506 17:19:45.126 | INFO | "huge_shared" : " 27 GB", | 20260506 17:19:45.126 | INFO | "malloc" : " 14 GB", | 20260506 17:19:45.126 | INFO | "pinned" : " 29 GB" | 20260506 17:19:45.126 | INFO | } | 20260506 17:19:45.126 | INFO | Perf report for the last 5.13868 seconds: | 20260506 17:19:45.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:19:45.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:45.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:45.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:45.127 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 17:19:45.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:45.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:45.127 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:19:45.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:45.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:45.127 | INFO | Uploaded 36GiB to GPU in 1.98662s (18.1213GiB/s) | 20260506 17:19:45.127 | INFO | Downloaded 34GiB to GPU in 1.93908s (17.5341GiB/s) | 20260506 17:19:50.265 | INFO | Analyzing chunk frames = [3097600,3098112) | 20260506 17:19:50.265 | INFO | MemInfo: { | 20260506 17:19:50.265 | INFO | "gpu" : " 17 GB", | 20260506 17:19:50.265 | INFO | "huge" : "", | 20260506 17:19:50.265 | INFO | "huge_shared" : " 27 GB", | 20260506 17:19:50.265 | INFO | "malloc" : " 14 GB", | 20260506 17:19:50.265 | INFO | "pinned" : " 29 GB" | 20260506 17:19:50.265 | INFO | } | 20260506 17:19:50.265 | INFO | Perf report for the last 5.1384 seconds: | 20260506 17:19:50.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:19:50.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:50.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:50.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:50.265 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 17:19:50.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:50.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:50.265 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:19:50.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:50.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:50.265 | INFO | Uploaded 36GiB to GPU in 1.96472s (18.3232GiB/s) | 20260506 17:19:50.265 | INFO | Downloaded 34GiB to GPU in 1.92317s (17.6791GiB/s) | 20260506 17:19:55.405 | INFO | Analyzing chunk frames = [3098112,3098624) | 20260506 17:19:55.405 | INFO | MemInfo: { | 20260506 17:19:55.405 | INFO | "gpu" : " 17 GB", | 20260506 17:19:55.405 | INFO | "huge" : "", | 20260506 17:19:55.405 | INFO | "huge_shared" : " 27 GB", | 20260506 17:19:55.405 | INFO | "malloc" : " 14 GB", | 20260506 17:19:55.405 | INFO | "pinned" : " 29 GB" | 20260506 17:19:55.405 | INFO | } | 20260506 17:19:55.405 | INFO | Perf report for the last 5.14015 seconds: | 20260506 17:19:55.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:19:55.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:55.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:55.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:55.405 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 17:19:55.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:55.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:55.405 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 17:19:55.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:55.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:19:55.405 | INFO | Uploaded 36GiB to GPU in 1.93185s (18.635GiB/s) | 20260506 17:19:55.405 | INFO | Downloaded 34GiB to GPU in 1.86723s (18.2088GiB/s) | 20260506 17:20:00.542 | INFO | Analyzing chunk frames = [3098624,3099136) | 20260506 17:20:00.542 | INFO | MemInfo: { | 20260506 17:20:00.542 | INFO | "gpu" : " 17 GB", | 20260506 17:20:00.542 | INFO | "huge" : "", | 20260506 17:20:00.542 | INFO | "huge_shared" : " 27 GB", | 20260506 17:20:00.542 | INFO | "malloc" : " 14 GB", | 20260506 17:20:00.542 | INFO | "pinned" : " 29 GB" | 20260506 17:20:00.542 | INFO | } | 20260506 17:20:00.542 | INFO | Perf report for the last 5.1368 seconds: | 20260506 17:20:00.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:20:00.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:00.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:00.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:00.542 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 17:20:00.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:00.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:00.542 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:20:00.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:00.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:00.542 | INFO | Uploaded 36GiB to GPU in 2.00196s (17.9824GiB/s) | 20260506 17:20:00.542 | INFO | Downloaded 34GiB to GPU in 1.89522s (17.9399GiB/s) | 20260506 17:20:05.685 | INFO | Analyzing chunk frames = [3099136,3099648) | 20260506 17:20:05.685 | INFO | MemInfo: { | 20260506 17:20:05.685 | INFO | "gpu" : " 17 GB", | 20260506 17:20:05.685 | INFO | "huge" : "", | 20260506 17:20:05.685 | INFO | "huge_shared" : " 27 GB", | 20260506 17:20:05.685 | INFO | "malloc" : " 14 GB", | 20260506 17:20:05.685 | INFO | "pinned" : " 29 GB" | 20260506 17:20:05.685 | INFO | } | 20260506 17:20:05.685 | INFO | Perf report for the last 5.14303 seconds: | 20260506 17:20:05.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:20:05.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:05.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:05.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:05.685 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 17:20:05.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:05.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:05.685 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:20:05.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:05.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:05.685 | INFO | Uploaded 36GiB to GPU in 2.02838s (17.7481GiB/s) | 20260506 17:20:05.685 | INFO | Downloaded 34GiB to GPU in 1.93701s (17.5528GiB/s) | 20260506 17:20:10.826 | INFO | Analyzing chunk frames = [3099648,3100160) | 20260506 17:20:10.826 | INFO | MemInfo: { | 20260506 17:20:10.826 | INFO | "gpu" : " 17 GB", | 20260506 17:20:10.826 | INFO | "huge" : "", | 20260506 17:20:10.826 | INFO | "huge_shared" : " 27 GB", | 20260506 17:20:10.826 | INFO | "malloc" : " 14 GB", | 20260506 17:20:10.826 | INFO | "pinned" : " 29 GB" | 20260506 17:20:10.826 | INFO | } | 20260506 17:20:10.826 | INFO | Perf report for the last 5.1412 seconds: | 20260506 17:20:10.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:20:10.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:10.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:10.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:10.826 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:20:10.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:10.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:10.826 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:20:10.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:10.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:10.826 | INFO | Uploaded 36GiB to GPU in 2.01576s (17.8593GiB/s) | 20260506 17:20:10.826 | INFO | Downloaded 34GiB to GPU in 1.99414s (17.05GiB/s) | 20260506 17:20:15.964 | INFO | Analyzing chunk frames = [3100160,3100672) | 20260506 17:20:15.964 | INFO | MemInfo: { | 20260506 17:20:15.964 | INFO | "gpu" : " 17 GB", | 20260506 17:20:15.964 | INFO | "huge" : "", | 20260506 17:20:15.964 | INFO | "huge_shared" : " 27 GB", | 20260506 17:20:15.964 | INFO | "malloc" : " 14 GB", | 20260506 17:20:15.964 | INFO | "pinned" : " 29 GB" | 20260506 17:20:15.964 | INFO | } | 20260506 17:20:15.965 | INFO | Perf report for the last 5.13837 seconds: | 20260506 17:20:15.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:20:15.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:15.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:15.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:15.965 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 17:20:15.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:15.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:15.965 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:20:15.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:15.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:15.965 | INFO | Uploaded 36GiB to GPU in 2.04113s (17.6373GiB/s) | 20260506 17:20:15.965 | INFO | Downloaded 34GiB to GPU in 1.99267s (17.0626GiB/s) | 20260506 17:20:16.078 | INFO | Starting a round of Baz IO | 20260506 17:20:19.222 | INFO | Populating RTMetrics report | 20260506 17:20:19.222 | INFO | RTMetrics report populated | 20260506 17:20:19.222 | INFO | Opening tmp RTMetrics json file | 20260506 17:20:19.307 | INFO | Serializing RTMetrics to json | 20260506 17:20:19.313 | INFO | Serialization done, swapping tmp file now... | 20260506 17:20:19.317 | INFO | RTMetrics done | 20260506 17:20:19.483 | INFO | Finished a round of Baz IO: Wrote 2.067 GiB in 3.404 seconds | 20260506 17:20:19.483 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.71% / 0.01% / 64.28% | 20260506 17:20:19.483 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.40638% / 1.19209e-05% / 5.96046e-05% | 20260506 17:20:19.483 | INFO | Baz IO overall: | 20260506 17:20:19.483 | INFO | BytesWritten : 1633878154349 | 20260506 17:20:19.483 | INFO | NumEvents : 790462608474 | 20260506 17:20:21.104 | INFO | Analyzing chunk frames = [3100672,3101184) | 20260506 17:20:21.104 | INFO | MemInfo: { | 20260506 17:20:21.104 | INFO | "gpu" : " 17 GB", | 20260506 17:20:21.104 | INFO | "huge" : "", | 20260506 17:20:21.104 | INFO | "huge_shared" : " 27 GB", | 20260506 17:20:21.104 | INFO | "malloc" : " 14 GB", | 20260506 17:20:21.104 | INFO | "pinned" : " 29 GB" | 20260506 17:20:21.104 | INFO | } | 20260506 17:20:21.104 | INFO | Perf report for the last 5.13975 seconds: | 20260506 17:20:21.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:20:21.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:21.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:21.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:21.104 | INFO | ANALYSIS : 62.8%, 2.9, 95.5ms, 0 | 20260506 17:20:21.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:21.104 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260506 17:20:21.104 | INFO | PRE_HQ : 60.3%, 1.9, 58.1ms, 0 | 20260506 17:20:21.104 | INFO | BAZWRITER : 65.6%, 4.8, 161.8ms, 0 | 20260506 17:20:21.104 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 17:20:21.105 | INFO | Uploaded 36GiB to GPU in 2.07739s (17.3294GiB/s) | 20260506 17:20:21.105 | INFO | Downloaded 38GiB to GPU in 2.22411s (17.0855GiB/s) | 20260506 17:20:26.240 | INFO | Analyzing chunk frames = [3101184,3101696) | 20260506 17:20:26.240 | INFO | MemInfo: { | 20260506 17:20:26.240 | INFO | "gpu" : " 17 GB", | 20260506 17:20:26.240 | INFO | "huge" : "", | 20260506 17:20:26.240 | INFO | "huge_shared" : " 27 GB", | 20260506 17:20:26.240 | INFO | "malloc" : " 14 GB", | 20260506 17:20:26.240 | INFO | "pinned" : " 29 GB" | 20260506 17:20:26.240 | INFO | } | 20260506 17:20:26.240 | INFO | Perf report for the last 5.13597 seconds: | 20260506 17:20:26.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:20:26.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:26.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:26.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:26.240 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 17:20:26.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:26.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:26.240 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 17:20:26.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:26.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:26.241 | INFO | Uploaded 36GiB to GPU in 1.99282s (18.0649GiB/s) | 20260506 17:20:26.241 | INFO | Downloaded 34GiB to GPU in 1.94061s (17.5203GiB/s) | 20260506 17:20:31.378 | INFO | Analyzing chunk frames = [3101696,3102208) | 20260506 17:20:31.378 | INFO | MemInfo: { | 20260506 17:20:31.378 | INFO | "gpu" : " 17 GB", | 20260506 17:20:31.378 | INFO | "huge" : "", | 20260506 17:20:31.378 | INFO | "huge_shared" : " 27 GB", | 20260506 17:20:31.378 | INFO | "malloc" : " 14 GB", | 20260506 17:20:31.378 | INFO | "pinned" : " 29 GB" | 20260506 17:20:31.378 | INFO | } | 20260506 17:20:31.378 | INFO | Perf report for the last 5.13822 seconds: | 20260506 17:20:31.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:20:31.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:31.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:31.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:31.379 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 17:20:31.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:31.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:31.379 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:20:31.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:31.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:31.379 | INFO | Uploaded 36GiB to GPU in 1.968s (18.2927GiB/s) | 20260506 17:20:31.379 | INFO | Downloaded 34GiB to GPU in 1.92869s (17.6285GiB/s) | 20260506 17:20:36.516 | INFO | Analyzing chunk frames = [3102208,3102720) | 20260506 17:20:36.516 | INFO | MemInfo: { | 20260506 17:20:36.516 | INFO | "gpu" : " 17 GB", | 20260506 17:20:36.516 | INFO | "huge" : "", | 20260506 17:20:36.516 | INFO | "huge_shared" : " 27 GB", | 20260506 17:20:36.516 | INFO | "malloc" : " 14 GB", | 20260506 17:20:36.516 | INFO | "pinned" : " 29 GB" | 20260506 17:20:36.516 | INFO | } | 20260506 17:20:36.516 | INFO | Perf report for the last 5.13771 seconds: | 20260506 17:20:36.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:20:36.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:36.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:36.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:36.516 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260506 17:20:36.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:36.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:36.516 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:20:36.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:36.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:36.516 | INFO | Uploaded 36GiB to GPU in 1.93369s (18.6172GiB/s) | 20260506 17:20:36.516 | INFO | Downloaded 34GiB to GPU in 1.87668s (18.1171GiB/s) | 20260506 17:20:41.656 | INFO | Analyzing chunk frames = [3102720,3103232) | 20260506 17:20:41.656 | INFO | MemInfo: { | 20260506 17:20:41.656 | INFO | "gpu" : " 17 GB", | 20260506 17:20:41.656 | INFO | "huge" : "", | 20260506 17:20:41.656 | INFO | "huge_shared" : " 27 GB", | 20260506 17:20:41.656 | INFO | "malloc" : " 14 GB", | 20260506 17:20:41.657 | INFO | "pinned" : " 29 GB" | 20260506 17:20:41.657 | INFO | } | 20260506 17:20:41.657 | INFO | Perf report for the last 5.1404 seconds: | 20260506 17:20:41.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:20:41.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:41.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:41.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:41.657 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 17:20:41.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:41.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:41.657 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:20:41.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:41.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:41.657 | INFO | Uploaded 36GiB to GPU in 1.95727s (18.393GiB/s) | 20260506 17:20:41.657 | INFO | Downloaded 34GiB to GPU in 1.87759s (18.1083GiB/s) | 20260506 17:20:46.798 | INFO | Analyzing chunk frames = [3103232,3103744) | 20260506 17:20:46.798 | INFO | MemInfo: { | 20260506 17:20:46.798 | INFO | "gpu" : " 17 GB", | 20260506 17:20:46.798 | INFO | "huge" : "", | 20260506 17:20:46.798 | INFO | "huge_shared" : " 27 GB", | 20260506 17:20:46.798 | INFO | "malloc" : " 14 GB", | 20260506 17:20:46.798 | INFO | "pinned" : " 29 GB" | 20260506 17:20:46.798 | INFO | } | 20260506 17:20:46.798 | INFO | Perf report for the last 5.14155 seconds: | 20260506 17:20:46.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:20:46.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:46.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:46.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:46.798 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:20:46.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:46.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:46.798 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:20:46.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:46.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:46.799 | INFO | Uploaded 36GiB to GPU in 2.02551s (17.7733GiB/s) | 20260506 17:20:46.799 | INFO | Downloaded 34GiB to GPU in 1.95379s (17.4021GiB/s) | 20260506 17:20:51.938 | INFO | Analyzing chunk frames = [3103744,3104256) | 20260506 17:20:51.938 | INFO | MemInfo: { | 20260506 17:20:51.938 | INFO | "gpu" : " 17 GB", | 20260506 17:20:51.938 | INFO | "huge" : "", | 20260506 17:20:51.938 | INFO | "huge_shared" : " 27 GB", | 20260506 17:20:51.938 | INFO | "malloc" : " 14 GB", | 20260506 17:20:51.938 | INFO | "pinned" : " 29 GB" | 20260506 17:20:51.938 | INFO | } | 20260506 17:20:51.938 | INFO | Perf report for the last 5.13966 seconds: | 20260506 17:20:51.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:20:51.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:51.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:51.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:51.938 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 17:20:51.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:51.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:51.938 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 17:20:51.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:51.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:51.938 | INFO | Uploaded 36GiB to GPU in 2.02179s (17.806GiB/s) | 20260506 17:20:51.938 | INFO | Downloaded 34GiB to GPU in 2.01044s (16.9117GiB/s) | 20260506 17:20:57.080 | INFO | Analyzing chunk frames = [3104256,3104768) | 20260506 17:20:57.080 | INFO | MemInfo: { | 20260506 17:20:57.080 | INFO | "gpu" : " 17 GB", | 20260506 17:20:57.080 | INFO | "huge" : "", | 20260506 17:20:57.080 | INFO | "huge_shared" : " 27 GB", | 20260506 17:20:57.081 | INFO | "malloc" : " 14 GB", | 20260506 17:20:57.081 | INFO | "pinned" : " 29 GB" | 20260506 17:20:57.081 | INFO | } | 20260506 17:20:57.081 | INFO | Perf report for the last 5.14274 seconds: | 20260506 17:20:57.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:20:57.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:57.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:57.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:57.081 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 17:20:57.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:57.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:57.081 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 17:20:57.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:57.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:20:57.081 | INFO | Uploaded 36GiB to GPU in 2.00444s (17.9601GiB/s) | 20260506 17:20:57.081 | INFO | Downloaded 34GiB to GPU in 1.9772s (17.196GiB/s) | 20260506 17:20:57.081 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:20:57.081 | INFO | Total mem usage: 18 GiB | 20260506 17:20:57.081 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:20:57.081 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.3%) | 20260506 17:20:57.081 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:20:57.081 | INFO | High Water Breakdown: | 20260506 17:20:57.081 | INFO | 61.8% : 9.9 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:20:57.081 | INFO | 38.1% : 6.11 GiB ( 91.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:20:57.081 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:20:57.081 | INFO | ------------------------------------------ | 20260506 17:20:57.081 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:20:57.081 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:20:57.081 | INFO | Total mem usage: 26.5 GiB | 20260506 17:20:57.081 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:20:57.081 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:20:57.081 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:20:57.081 | INFO | High Water Breakdown: | 20260506 17:20:57.081 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:20:57.081 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:20:57.081 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:20:57.081 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:20:57.081 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:20:57.081 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:20:57.081 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:20:57.082 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:20:57.082 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:20:57.082 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:20:57.082 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:20:57.082 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:20:57.082 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:20:57.082 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:20:57.082 | INFO | ------------------------------------------ | 20260506 17:20:57.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:20:57.082 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:20:57.082 | INFO | Total mem usage: 25.6 GiB | 20260506 17:20:57.082 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:20:57.082 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:20:57.082 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:20:57.082 | INFO | High Water Breakdown: | 20260506 17:20:57.082 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:20:57.082 | INFO | ------------------------------------------ | 20260506 17:20:57.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:20:57.082 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:20:57.082 | INFO | Total mem usage: 15.7 GiB | 20260506 17:20:57.082 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:20:57.082 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:20:57.082 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 17:20:57.082 | INFO | High Water Breakdown: | 20260506 17:20:57.082 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:20:57.082 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:20:57.082 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:20:57.082 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:20:57.082 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:20:57.082 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:20:57.082 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:20:57.082 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:20:57.082 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:20:57.082 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:20:57.082 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:20:57.082 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:20:57.082 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:20:57.082 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:20:57.082 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:20:57.082 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:20:57.082 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:20:57.082 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:20:57.082 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:20:57.082 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:20:57.082 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:20:57.082 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:20:57.082 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:20:57.082 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:20:57.082 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:20:57.082 | INFO | ------------------------------------------ | 20260506 17:20:57.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:20:57.082 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:20:57.082 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:20:57.082 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:20:57.082 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.56e+01, 1.35e+01 / 2.48e+00 | 20260506 17:20:57.082 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 1.09e-02 / 1.34e-02 | 20260506 17:20:57.082 | INFO | PerfReport: 11.59%, Baseline -- 3.63e+00 / 4.36e+00, 5.52e-01 / 3.92e-01 | 20260506 17:20:57.082 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.41e+00, 1.81e-01 / 1.55e-01 | 20260506 17:20:57.082 | INFO | PerfReport: 16.02%, DME -- 4.01e+01 / 4.49e+01, 4.12e+00 / 2.51e+00 | 20260506 17:20:57.082 | INFO | PerfReport: 56.86%, FrameLabeling -- 1.78e+01 / 2.00e+01, 1.35e+00 / 1.78e+00 | 20260506 17:20:57.082 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.29e+00 / 2.94e+00, 3.97e-01 / 1.63e-01 | 20260506 17:20:57.082 | INFO | PerfReport: 3.46%, Metrics -- 1.08e+00 / 1.44e+00, 2.81e-01 / 1.71e-01 | 20260506 17:20:57.082 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.74e-01 / 4.24e-01, 5.83e-02 / 2.23e-02 | 20260506 17:20:57.082 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:20:57.082 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:20:57.082 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:20:57.082 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 17:20:57.083 | INFO | PerfReport: 21.46%, Upload -- 2.09e+01 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260506 17:20:57.083 | INFO | PerfReport: 21.15%, Download -- 2.05e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 17:20:57.083 | INFO | PerfReport: 32.22%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 17:20:57.083 | INFO | PerfReport: 1.38%, LockUpload -- 1.34e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260506 17:20:57.083 | INFO | PerfReport: 1.13%, LockDownload -- 1.10e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 17:20:57.083 | INFO | PerfReport: 22.65%, LockCompute -- 2.20e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 17:20:57.083 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260506 17:20:57.196 | INFO | Starting a round of Baz IO | 20260506 17:21:00.337 | INFO | Populating RTMetrics report | 20260506 17:21:00.338 | INFO | RTMetrics report populated | 20260506 17:21:00.338 | INFO | Opening tmp RTMetrics json file | 20260506 17:21:00.376 | INFO | Serializing RTMetrics to json | 20260506 17:21:00.380 | INFO | Serialization done, swapping tmp file now... | 20260506 17:21:00.382 | INFO | RTMetrics done | 20260506 17:21:00.680 | INFO | Finished a round of Baz IO: Wrote 2.069 GiB in 3.484 seconds | 20260506 17:21:00.681 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.71% / 0.01% / 64.29% | 20260506 17:21:00.681 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.40633% / 1.58946e-05% / 6.75519e-05% | 20260506 17:21:00.681 | INFO | Baz IO overall: | 20260506 17:21:00.681 | INFO | BytesWritten : 1636099264943 | 20260506 17:21:00.681 | INFO | NumEvents : 791534991839 | 20260506 17:21:02.218 | INFO | Analyzing chunk frames = [3104768,3105280) | 20260506 17:21:02.218 | INFO | MemInfo: { | 20260506 17:21:02.218 | INFO | "gpu" : " 17 GB", | 20260506 17:21:02.218 | INFO | "huge" : "", | 20260506 17:21:02.218 | INFO | "huge_shared" : " 27 GB", | 20260506 17:21:02.218 | INFO | "malloc" : " 14 GB", | 20260506 17:21:02.218 | INFO | "pinned" : " 29 GB" | 20260506 17:21:02.218 | INFO | } | 20260506 17:21:02.218 | INFO | Perf report for the last 5.13699 seconds: | 20260506 17:21:02.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:21:02.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:02.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:02.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:02.218 | INFO | ANALYSIS : 62.8%, 2.9, 95.0ms, 0 | 20260506 17:21:02.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:02.218 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 17:21:02.218 | INFO | PRE_HQ : 60.3%, 1.9, 59.4ms, 0 | 20260506 17:21:02.218 | INFO | BAZWRITER : 67.5%, 4.8, 166.6ms, 0 | 20260506 17:21:02.218 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 17:21:02.218 | INFO | Uploaded 36GiB to GPU in 2.06522s (17.4316GiB/s) | 20260506 17:21:02.218 | INFO | Downloaded 38GiB to GPU in 2.21397s (17.1637GiB/s) | 20260506 17:21:07.358 | INFO | Analyzing chunk frames = [3105280,3105792) | 20260506 17:21:07.358 | INFO | MemInfo: { | 20260506 17:21:07.358 | INFO | "gpu" : " 17 GB", | 20260506 17:21:07.358 | INFO | "huge" : "", | 20260506 17:21:07.358 | INFO | "huge_shared" : " 27 GB", | 20260506 17:21:07.358 | INFO | "malloc" : " 14 GB", | 20260506 17:21:07.358 | INFO | "pinned" : " 29 GB" | 20260506 17:21:07.358 | INFO | } | 20260506 17:21:07.358 | INFO | Perf report for the last 5.14073 seconds: | 20260506 17:21:07.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:21:07.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:07.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:07.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:07.358 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:21:07.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:07.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:07.358 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 17:21:07.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:07.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:07.359 | INFO | Uploaded 36GiB to GPU in 2.01069s (17.9043GiB/s) | 20260506 17:21:07.359 | INFO | Downloaded 34GiB to GPU in 1.93876s (17.537GiB/s) | 20260506 17:21:12.491 | INFO | Analyzing chunk frames = [3105792,3106304) | 20260506 17:21:12.491 | INFO | MemInfo: { | 20260506 17:21:12.491 | INFO | "gpu" : " 17 GB", | 20260506 17:21:12.491 | INFO | "huge" : "", | 20260506 17:21:12.491 | INFO | "huge_shared" : " 27 GB", | 20260506 17:21:12.491 | INFO | "malloc" : " 14 GB", | 20260506 17:21:12.491 | INFO | "pinned" : " 29 GB" | 20260506 17:21:12.491 | INFO | } | 20260506 17:21:12.491 | INFO | Perf report for the last 5.13297 seconds: | 20260506 17:21:12.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:21:12.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:12.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:12.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:12.491 | INFO | ANALYSIS : 61.0%, 3.0, 94.3ms, 0 | 20260506 17:21:12.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:12.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:12.491 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:21:12.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:12.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:12.492 | INFO | Uploaded 36GiB to GPU in 1.94553s (18.504GiB/s) | 20260506 17:21:12.492 | INFO | Downloaded 34GiB to GPU in 1.92141s (17.6953GiB/s) | 20260506 17:21:17.628 | INFO | Analyzing chunk frames = [3106304,3106816) | 20260506 17:21:17.628 | INFO | MemInfo: { | 20260506 17:21:17.628 | INFO | "gpu" : " 17 GB", | 20260506 17:21:17.628 | INFO | "huge" : "", | 20260506 17:21:17.628 | INFO | "huge_shared" : " 27 GB", | 20260506 17:21:17.628 | INFO | "malloc" : " 14 GB", | 20260506 17:21:17.628 | INFO | "pinned" : " 29 GB" | 20260506 17:21:17.628 | INFO | } | 20260506 17:21:17.628 | INFO | Perf report for the last 5.13647 seconds: | 20260506 17:21:17.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:21:17.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:17.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:17.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:17.628 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 17:21:17.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:17.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:17.628 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:21:17.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:17.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:17.628 | INFO | Uploaded 36GiB to GPU in 1.92714s (18.6805GiB/s) | 20260506 17:21:17.628 | INFO | Downloaded 34GiB to GPU in 1.86899s (18.1916GiB/s) | 20260506 17:21:22.781 | INFO | Analyzing chunk frames = [3106816,3107328) | 20260506 17:21:22.781 | INFO | MemInfo: { | 20260506 17:21:22.781 | INFO | "gpu" : " 17 GB", | 20260506 17:21:22.781 | INFO | "huge" : "", | 20260506 17:21:22.781 | INFO | "huge_shared" : " 27 GB", | 20260506 17:21:22.781 | INFO | "malloc" : " 14 GB", | 20260506 17:21:22.781 | INFO | "pinned" : " 29 GB" | 20260506 17:21:22.781 | INFO | } | 20260506 17:21:22.781 | INFO | Perf report for the last 5.15344 seconds: | 20260506 17:21:22.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:21:22.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:22.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:22.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:22.781 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260506 17:21:22.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:22.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:22.781 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:21:22.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:22.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:22.782 | INFO | Uploaded 36GiB to GPU in 1.96049s (18.3628GiB/s) | 20260506 17:21:22.782 | INFO | Downloaded 34GiB to GPU in 1.87813s (18.1032GiB/s) | 20260506 17:21:27.917 | INFO | Analyzing chunk frames = [3107328,3107840) | 20260506 17:21:27.917 | INFO | MemInfo: { | 20260506 17:21:27.917 | INFO | "gpu" : " 17 GB", | 20260506 17:21:27.917 | INFO | "huge" : "", | 20260506 17:21:27.917 | INFO | "huge_shared" : " 27 GB", | 20260506 17:21:27.917 | INFO | "malloc" : " 14 GB", | 20260506 17:21:27.917 | INFO | "pinned" : " 29 GB" | 20260506 17:21:27.917 | INFO | } | 20260506 17:21:27.917 | INFO | Perf report for the last 5.13559 seconds: | 20260506 17:21:27.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:21:27.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:27.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:27.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:27.917 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 17:21:27.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:27.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:27.917 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:21:27.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:27.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:27.918 | INFO | Uploaded 36GiB to GPU in 2.00035s (17.9969GiB/s) | 20260506 17:21:27.918 | INFO | Downloaded 34GiB to GPU in 1.93457s (17.575GiB/s) | 20260506 17:21:33.054 | INFO | Analyzing chunk frames = [3107840,3108352) | 20260506 17:21:33.054 | INFO | MemInfo: { | 20260506 17:21:33.054 | INFO | "gpu" : " 17 GB", | 20260506 17:21:33.054 | INFO | "huge" : "", | 20260506 17:21:33.054 | INFO | "huge_shared" : " 27 GB", | 20260506 17:21:33.054 | INFO | "malloc" : " 14 GB", | 20260506 17:21:33.054 | INFO | "pinned" : " 29 GB" | 20260506 17:21:33.054 | INFO | } | 20260506 17:21:33.055 | INFO | Perf report for the last 5.13767 seconds: | 20260506 17:21:33.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:21:33.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:33.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:33.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:33.055 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:21:33.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:33.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:33.055 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:21:33.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:33.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:33.055 | INFO | Uploaded 36GiB to GPU in 2.00799s (17.9284GiB/s) | 20260506 17:21:33.055 | INFO | Downloaded 34GiB to GPU in 1.98799s (17.1027GiB/s) | 20260506 17:21:38.195 | INFO | Analyzing chunk frames = [3108352,3108864) | 20260506 17:21:38.195 | INFO | MemInfo: { | 20260506 17:21:38.195 | INFO | "gpu" : " 17 GB", | 20260506 17:21:38.195 | INFO | "huge" : "", | 20260506 17:21:38.195 | INFO | "huge_shared" : " 27 GB", | 20260506 17:21:38.195 | INFO | "malloc" : " 14 GB", | 20260506 17:21:38.195 | INFO | "pinned" : " 29 GB" | 20260506 17:21:38.195 | INFO | } | 20260506 17:21:38.195 | INFO | Perf report for the last 5.14008 seconds: | 20260506 17:21:38.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:21:38.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:38.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:38.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:38.195 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:21:38.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:38.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:38.195 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:21:38.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:38.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:38.195 | INFO | Uploaded 36GiB to GPU in 2.00197s (17.9823GiB/s) | 20260506 17:21:38.195 | INFO | Downloaded 34GiB to GPU in 1.98283s (17.1472GiB/s) | 20260506 17:21:38.309 | INFO | Starting a round of Baz IO | 20260506 17:21:41.443 | INFO | Populating RTMetrics report | 20260506 17:21:41.444 | INFO | RTMetrics report populated | 20260506 17:21:41.444 | INFO | Opening tmp RTMetrics json file | 20260506 17:21:41.462 | INFO | Serializing RTMetrics to json | 20260506 17:21:41.526 | INFO | Serialization done, swapping tmp file now... | 20260506 17:21:41.553 | INFO | RTMetrics done | 20260506 17:21:41.916 | INFO | Finished a round of Baz IO: Wrote 2.068 GiB in 3.607 seconds | 20260506 17:21:41.917 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.70% / 0.01% / 64.29% | 20260506 17:21:41.917 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.40276% / 1.19209e-05% / 3.17891e-05% | 20260506 17:21:41.917 | INFO | Baz IO overall: | 20260506 17:21:41.917 | INFO | BytesWritten : 1638319895166 | 20260506 17:21:41.917 | INFO | NumEvents : 792606778411 | 20260506 17:21:43.329 | INFO | Analyzing chunk frames = [3108864,3109376) | 20260506 17:21:43.329 | INFO | MemInfo: { | 20260506 17:21:43.329 | INFO | "gpu" : " 17 GB", | 20260506 17:21:43.329 | INFO | "huge" : "", | 20260506 17:21:43.329 | INFO | "huge_shared" : " 27 GB", | 20260506 17:21:43.329 | INFO | "malloc" : " 14 GB", | 20260506 17:21:43.329 | INFO | "pinned" : " 29 GB" | 20260506 17:21:43.330 | INFO | } | 20260506 17:21:43.330 | INFO | Perf report for the last 5.13497 seconds: | 20260506 17:21:43.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:21:43.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:43.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:43.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:43.330 | INFO | ANALYSIS : 62.7%, 2.9, 95.2ms, 0 | 20260506 17:21:43.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:43.330 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260506 17:21:43.330 | INFO | PRE_HQ : 60.2%, 1.9, 58.1ms, 0 | 20260506 17:21:43.330 | INFO | BAZWRITER : 69.9%, 4.8, 172.1ms, 0 | 20260506 17:21:43.330 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 17:21:43.330 | INFO | Uploaded 36GiB to GPU in 2.05042s (17.5574GiB/s) | 20260506 17:21:43.330 | INFO | Downloaded 38GiB to GPU in 2.21263s (17.1741GiB/s) | 20260506 17:21:48.467 | INFO | Analyzing chunk frames = [3109376,3109888) | 20260506 17:21:48.467 | INFO | MemInfo: { | 20260506 17:21:48.467 | INFO | "gpu" : " 17 GB", | 20260506 17:21:48.467 | INFO | "huge" : "", | 20260506 17:21:48.467 | INFO | "huge_shared" : " 27 GB", | 20260506 17:21:48.467 | INFO | "malloc" : " 14 GB", | 20260506 17:21:48.467 | INFO | "pinned" : " 29 GB" | 20260506 17:21:48.467 | INFO | } | 20260506 17:21:48.467 | INFO | Perf report for the last 5.13711 seconds: | 20260506 17:21:48.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:21:48.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:48.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:48.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:48.467 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:21:48.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:48.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:48.467 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 17:21:48.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:48.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:48.467 | INFO | Uploaded 36GiB to GPU in 1.99587s (18.0373GiB/s) | 20260506 17:21:48.467 | INFO | Downloaded 34GiB to GPU in 1.94062s (17.5202GiB/s) | 20260506 17:21:53.607 | INFO | Analyzing chunk frames = [3109888,3110400) | 20260506 17:21:53.607 | INFO | MemInfo: { | 20260506 17:21:53.607 | INFO | "gpu" : " 17 GB", | 20260506 17:21:53.607 | INFO | "huge" : "", | 20260506 17:21:53.607 | INFO | "huge_shared" : " 27 GB", | 20260506 17:21:53.607 | INFO | "malloc" : " 14 GB", | 20260506 17:21:53.607 | INFO | "pinned" : " 29 GB" | 20260506 17:21:53.607 | INFO | } | 20260506 17:21:53.607 | INFO | Perf report for the last 5.14035 seconds: | 20260506 17:21:53.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:21:53.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:53.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:53.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:53.607 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 17:21:53.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:53.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:53.607 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:21:53.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:53.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:53.607 | INFO | Uploaded 36GiB to GPU in 1.99372s (18.0567GiB/s) | 20260506 17:21:53.607 | INFO | Downloaded 34GiB to GPU in 1.93096s (17.6078GiB/s) | 20260506 17:21:58.749 | INFO | Analyzing chunk frames = [3110400,3110912) | 20260506 17:21:58.749 | INFO | MemInfo: { | 20260506 17:21:58.749 | INFO | "gpu" : " 17 GB", | 20260506 17:21:58.749 | INFO | "huge" : "", | 20260506 17:21:58.749 | INFO | "huge_shared" : " 27 GB", | 20260506 17:21:58.749 | INFO | "malloc" : " 14 GB", | 20260506 17:21:58.749 | INFO | "pinned" : " 29 GB" | 20260506 17:21:58.749 | INFO | } | 20260506 17:21:58.750 | INFO | Perf report for the last 5.14237 seconds: | 20260506 17:21:58.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:21:58.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:58.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:58.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:58.750 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 17:21:58.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:58.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:58.750 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:21:58.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:58.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:21:58.750 | INFO | Uploaded 36GiB to GPU in 1.93223s (18.6313GiB/s) | 20260506 17:21:58.750 | INFO | Downloaded 34GiB to GPU in 1.87473s (18.136GiB/s) | 20260506 17:22:03.892 | INFO | Analyzing chunk frames = [3110912,3111424) | 20260506 17:22:03.892 | INFO | MemInfo: { | 20260506 17:22:03.892 | INFO | "gpu" : " 17 GB", | 20260506 17:22:03.892 | INFO | "huge" : "", | 20260506 17:22:03.892 | INFO | "huge_shared" : " 27 GB", | 20260506 17:22:03.892 | INFO | "malloc" : " 14 GB", | 20260506 17:22:03.892 | INFO | "pinned" : " 29 GB" | 20260506 17:22:03.892 | INFO | } | 20260506 17:22:03.892 | INFO | Perf report for the last 5.14215 seconds: | 20260506 17:22:03.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:22:03.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:03.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:03.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:03.892 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 17:22:03.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:03.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:03.892 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:22:03.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:03.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:03.892 | INFO | Uploaded 36GiB to GPU in 1.98732s (18.1149GiB/s) | 20260506 17:22:03.892 | INFO | Downloaded 34GiB to GPU in 1.92083s (17.7007GiB/s) | 20260506 17:22:09.034 | INFO | Analyzing chunk frames = [3111424,3111936) | 20260506 17:22:09.034 | INFO | MemInfo: { | 20260506 17:22:09.034 | INFO | "gpu" : " 17 GB", | 20260506 17:22:09.034 | INFO | "huge" : "", | 20260506 17:22:09.034 | INFO | "huge_shared" : " 27 GB", | 20260506 17:22:09.034 | INFO | "malloc" : " 14 GB", | 20260506 17:22:09.034 | INFO | "pinned" : " 29 GB" | 20260506 17:22:09.034 | INFO | } | 20260506 17:22:09.035 | INFO | Perf report for the last 5.14287 seconds: | 20260506 17:22:09.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:22:09.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:09.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:09.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:09.035 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 17:22:09.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:09.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:09.035 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260506 17:22:09.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:09.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:09.035 | INFO | Uploaded 36GiB to GPU in 2.01779s (17.8413GiB/s) | 20260506 17:22:09.035 | INFO | Downloaded 34GiB to GPU in 1.94438s (17.4863GiB/s) | 20260506 17:22:14.171 | INFO | Analyzing chunk frames = [3111936,3112448) | 20260506 17:22:14.171 | INFO | MemInfo: { | 20260506 17:22:14.171 | INFO | "gpu" : " 17 GB", | 20260506 17:22:14.171 | INFO | "huge" : "", | 20260506 17:22:14.171 | INFO | "huge_shared" : " 27 GB", | 20260506 17:22:14.171 | INFO | "malloc" : " 14 GB", | 20260506 17:22:14.171 | INFO | "pinned" : " 29 GB" | 20260506 17:22:14.171 | INFO | } | 20260506 17:22:14.171 | INFO | Perf report for the last 5.1369 seconds: | 20260506 17:22:14.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:22:14.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:14.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:14.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:14.172 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:22:14.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:14.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:14.172 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 17:22:14.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:14.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:14.172 | INFO | Uploaded 36GiB to GPU in 2.02516s (17.7764GiB/s) | 20260506 17:22:14.172 | INFO | Downloaded 34GiB to GPU in 1.98595s (17.1203GiB/s) | 20260506 17:22:19.307 | INFO | Analyzing chunk frames = [3112448,3112960) | 20260506 17:22:19.307 | INFO | MemInfo: { | 20260506 17:22:19.307 | INFO | "gpu" : " 17 GB", | 20260506 17:22:19.307 | INFO | "huge" : "", | 20260506 17:22:19.307 | INFO | "huge_shared" : " 27 GB", | 20260506 17:22:19.307 | INFO | "malloc" : " 14 GB", | 20260506 17:22:19.307 | INFO | "pinned" : " 29 GB" | 20260506 17:22:19.307 | INFO | } | 20260506 17:22:19.307 | INFO | Perf report for the last 5.13551 seconds: | 20260506 17:22:19.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:22:19.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:19.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:19.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:19.307 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 17:22:19.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:19.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:19.307 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 17:22:19.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:19.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:19.307 | INFO | Uploaded 36GiB to GPU in 2.03937s (17.6525GiB/s) | 20260506 17:22:19.308 | INFO | Downloaded 34GiB to GPU in 1.99424s (17.0491GiB/s) | 20260506 17:22:19.308 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:22:19.308 | INFO | Total mem usage: 18 GiB | 20260506 17:22:19.308 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:22:19.308 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -14.6%) | 20260506 17:22:19.308 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:22:19.308 | INFO | High Water Breakdown: | 20260506 17:22:19.308 | INFO | 61.5% : 9.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:22:19.308 | INFO | 38.4% : 6.03 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:22:19.308 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:22:19.308 | INFO | ------------------------------------------ | 20260506 17:22:19.308 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:22:19.308 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:22:19.308 | INFO | Total mem usage: 26.5 GiB | 20260506 17:22:19.308 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:22:19.308 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:22:19.308 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:22:19.308 | INFO | High Water Breakdown: | 20260506 17:22:19.308 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:22:19.308 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:22:19.308 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:22:19.308 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:22:19.308 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:22:19.308 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:22:19.308 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:22:19.308 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:22:19.308 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:22:19.308 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:22:19.308 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:22:19.308 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:22:19.308 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:22:19.308 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:22:19.308 | INFO | ------------------------------------------ | 20260506 17:22:19.308 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:22:19.308 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:22:19.308 | INFO | Total mem usage: 25.6 GiB | 20260506 17:22:19.308 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:22:19.308 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:22:19.308 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:22:19.308 | INFO | High Water Breakdown: | 20260506 17:22:19.308 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:22:19.308 | INFO | ------------------------------------------ | 20260506 17:22:19.308 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:22:19.308 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:22:19.308 | INFO | Total mem usage: 15.7 GiB | 20260506 17:22:19.308 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:22:19.308 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:22:19.308 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:22:19.308 | INFO | High Water Breakdown: | 20260506 17:22:19.308 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:22:19.308 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:22:19.308 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:22:19.308 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:22:19.308 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:22:19.308 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:22:19.308 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:22:19.308 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:22:19.308 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:22:19.308 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:22:19.308 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:22:19.308 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:22:19.308 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:22:19.308 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:22:19.308 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:22:19.308 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:22:19.308 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:22:19.308 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:22:19.308 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:22:19.308 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:22:19.308 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:22:19.308 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:22:19.308 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:22:19.308 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:22:19.308 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:22:19.308 | INFO | ------------------------------------------ | 20260506 17:22:19.308 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:22:19.309 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:22:19.309 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:22:19.309 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:22:19.309 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.57e+01, 1.35e+01 / 2.22e+00 | 20260506 17:22:19.309 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 9.96e-02, 1.02e-02 / 5.78e-03 | 20260506 17:22:19.309 | INFO | PerfReport: 11.60%, Baseline -- 3.64e+00 / 4.43e+00, 5.59e-01 / 3.31e-01 | 20260506 17:22:19.309 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.43e+00, 1.83e-01 / 1.25e-01 | 20260506 17:22:19.309 | INFO | PerfReport: 15.95%, DME -- 4.00e+01 / 4.47e+01, 3.92e+00 / 2.48e+00 | 20260506 17:22:19.309 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.79e+01 / 2.03e+01, 1.45e+00 / 1.77e+00 | 20260506 17:22:19.309 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.28e+00 / 2.85e+00, 3.89e-01 / 2.29e-01 | 20260506 17:22:19.309 | INFO | PerfReport: 3.45%, Metrics -- 1.08e+00 / 1.44e+00, 2.79e-01 / 1.67e-01 | 20260506 17:22:19.309 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.75e-01 / 4.24e-01, 5.91e-02 / 1.98e-02 | 20260506 17:22:19.309 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:22:19.309 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:22:19.309 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:22:19.309 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 17:22:19.309 | INFO | PerfReport: 21.42%, Upload -- 2.08e+01 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260506 17:22:19.309 | INFO | PerfReport: 21.14%, Download -- 2.05e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260506 17:22:19.309 | INFO | PerfReport: 32.27%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 17:22:19.309 | INFO | PerfReport: 1.36%, LockUpload -- 1.32e+00 / 0.00e+00, 5.39e+00 / 0.00e+00 | 20260506 17:22:19.309 | INFO | PerfReport: 1.11%, LockDownload -- 1.07e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260506 17:22:19.309 | INFO | PerfReport: 22.68%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 17:22:19.309 | INFO | PerfReport: 0.02%, Misc -- 1.74e-02 / 0.00e+00, 4.08e-02 / 0.00e+00 | 20260506 17:22:19.418 | INFO | Starting a round of Baz IO | 20260506 17:22:22.569 | INFO | Populating RTMetrics report | 20260506 17:22:22.570 | INFO | RTMetrics report populated | 20260506 17:22:22.570 | INFO | Opening tmp RTMetrics json file | 20260506 17:22:22.582 | INFO | Serializing RTMetrics to json | 20260506 17:22:22.607 | INFO | Serialization done, swapping tmp file now... | 20260506 17:22:22.608 | INFO | RTMetrics done | 20260506 17:22:22.862 | INFO | Finished a round of Baz IO: Wrote 2.067 GiB in 3.443 seconds | 20260506 17:22:22.862 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.69% / 0.01% / 64.30% | 20260506 17:22:22.862 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.39918% / 1.98682e-05% / 7.94729e-05% | 20260506 17:22:22.862 | INFO | Baz IO overall: | 20260506 17:22:22.862 | INFO | BytesWritten : 1640539033165 | 20260506 17:22:22.862 | INFO | NumEvents : 793677572629 | 20260506 17:22:24.446 | INFO | Analyzing chunk frames = [3112960,3113472) | 20260506 17:22:24.446 | INFO | MemInfo: { | 20260506 17:22:24.446 | INFO | "gpu" : " 17 GB", | 20260506 17:22:24.446 | INFO | "huge" : "", | 20260506 17:22:24.446 | INFO | "huge_shared" : " 27 GB", | 20260506 17:22:24.446 | INFO | "malloc" : " 14 GB", | 20260506 17:22:24.446 | INFO | "pinned" : " 29 GB" | 20260506 17:22:24.446 | INFO | } | 20260506 17:22:24.446 | INFO | Perf report for the last 5.13888 seconds: | 20260506 17:22:24.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:22:24.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:24.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:24.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:24.446 | INFO | ANALYSIS : 62.9%, 2.9, 95.6ms, 0 | 20260506 17:22:24.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:24.446 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 17:22:24.446 | INFO | PRE_HQ : 60.4%, 1.9, 58.1ms, 0 | 20260506 17:22:24.446 | INFO | BAZWRITER : 66.4%, 4.8, 164.5ms, 0 | 20260506 17:22:24.446 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 17:22:24.446 | INFO | Uploaded 36GiB to GPU in 2.06891s (17.4005GiB/s) | 20260506 17:22:24.446 | INFO | Downloaded 38GiB to GPU in 2.23148s (17.0291GiB/s) | 20260506 17:22:29.581 | INFO | Analyzing chunk frames = [3113472,3113984) | 20260506 17:22:29.581 | INFO | MemInfo: { | 20260506 17:22:29.581 | INFO | "gpu" : " 17 GB", | 20260506 17:22:29.581 | INFO | "huge" : "", | 20260506 17:22:29.581 | INFO | "huge_shared" : " 27 GB", | 20260506 17:22:29.581 | INFO | "malloc" : " 14 GB", | 20260506 17:22:29.581 | INFO | "pinned" : " 29 GB" | 20260506 17:22:29.581 | INFO | } | 20260506 17:22:29.581 | INFO | Perf report for the last 5.13511 seconds: | 20260506 17:22:29.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:22:29.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:29.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:29.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:29.581 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:22:29.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:29.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:29.581 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:22:29.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:29.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:29.581 | INFO | Uploaded 36GiB to GPU in 2.00193s (17.9826GiB/s) | 20260506 17:22:29.581 | INFO | Downloaded 34GiB to GPU in 1.93355s (17.5842GiB/s) | 20260506 17:22:34.724 | INFO | Analyzing chunk frames = [3113984,3114496) | 20260506 17:22:34.724 | INFO | MemInfo: { | 20260506 17:22:34.724 | INFO | "gpu" : " 17 GB", | 20260506 17:22:34.724 | INFO | "huge" : "", | 20260506 17:22:34.724 | INFO | "huge_shared" : " 27 GB", | 20260506 17:22:34.724 | INFO | "malloc" : " 14 GB", | 20260506 17:22:34.724 | INFO | "pinned" : " 29 GB" | 20260506 17:22:34.724 | INFO | } | 20260506 17:22:34.724 | INFO | Perf report for the last 5.14337 seconds: | 20260506 17:22:34.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:22:34.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:34.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:34.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:34.724 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 17:22:34.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:34.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:34.724 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:22:34.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:34.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:34.725 | INFO | Uploaded 36GiB to GPU in 1.98155s (18.1676GiB/s) | 20260506 17:22:34.725 | INFO | Downloaded 34GiB to GPU in 1.92906s (17.6252GiB/s) | 20260506 17:22:39.872 | INFO | Analyzing chunk frames = [3114496,3115008) | 20260506 17:22:39.872 | INFO | MemInfo: { | 20260506 17:22:39.872 | INFO | "gpu" : " 17 GB", | 20260506 17:22:39.872 | INFO | "huge" : "", | 20260506 17:22:39.872 | INFO | "huge_shared" : " 27 GB", | 20260506 17:22:39.872 | INFO | "malloc" : " 14 GB", | 20260506 17:22:39.872 | INFO | "pinned" : " 29 GB" | 20260506 17:22:39.872 | INFO | } | 20260506 17:22:39.872 | INFO | Perf report for the last 5.14748 seconds: | 20260506 17:22:39.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:22:39.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:39.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:39.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:39.872 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260506 17:22:39.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:39.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:39.872 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:22:39.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:39.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:39.872 | INFO | Uploaded 36GiB to GPU in 1.94012s (18.5556GiB/s) | 20260506 17:22:39.872 | INFO | Downloaded 34GiB to GPU in 1.87416s (18.1415GiB/s) | 20260506 17:22:45.004 | INFO | Analyzing chunk frames = [3115008,3115520) | 20260506 17:22:45.004 | INFO | MemInfo: { | 20260506 17:22:45.004 | INFO | "gpu" : " 17 GB", | 20260506 17:22:45.004 | INFO | "huge" : "", | 20260506 17:22:45.004 | INFO | "huge_shared" : " 27 GB", | 20260506 17:22:45.004 | INFO | "malloc" : " 14 GB", | 20260506 17:22:45.004 | INFO | "pinned" : " 29 GB" | 20260506 17:22:45.004 | INFO | } | 20260506 17:22:45.004 | INFO | Perf report for the last 5.13217 seconds: | 20260506 17:22:45.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:22:45.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:45.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:45.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:45.004 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 17:22:45.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:45.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:45.004 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:22:45.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:45.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:45.004 | INFO | Uploaded 36GiB to GPU in 2.01464s (17.8692GiB/s) | 20260506 17:22:45.005 | INFO | Downloaded 34GiB to GPU in 1.90637s (17.8349GiB/s) | 20260506 17:22:50.144 | INFO | Analyzing chunk frames = [3115520,3116032) | 20260506 17:22:50.144 | INFO | MemInfo: { | 20260506 17:22:50.144 | INFO | "gpu" : " 17 GB", | 20260506 17:22:50.144 | INFO | "huge" : "", | 20260506 17:22:50.144 | INFO | "huge_shared" : " 27 GB", | 20260506 17:22:50.144 | INFO | "malloc" : " 14 GB", | 20260506 17:22:50.144 | INFO | "pinned" : " 29 GB" | 20260506 17:22:50.144 | INFO | } | 20260506 17:22:50.144 | INFO | Perf report for the last 5.14009 seconds: | 20260506 17:22:50.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:22:50.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:50.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:50.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:50.144 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:22:50.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:50.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:50.144 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 17:22:50.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:50.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:50.144 | INFO | Uploaded 36GiB to GPU in 2.07248s (17.3705GiB/s) | 20260506 17:22:50.144 | INFO | Downloaded 34GiB to GPU in 1.96243s (17.3254GiB/s) | 20260506 17:22:55.283 | INFO | Analyzing chunk frames = [3116032,3116544) | 20260506 17:22:55.283 | INFO | MemInfo: { | 20260506 17:22:55.283 | INFO | "gpu" : " 17 GB", | 20260506 17:22:55.283 | INFO | "huge" : "", | 20260506 17:22:55.283 | INFO | "huge_shared" : " 27 GB", | 20260506 17:22:55.283 | INFO | "malloc" : " 14 GB", | 20260506 17:22:55.283 | INFO | "pinned" : " 29 GB" | 20260506 17:22:55.283 | INFO | } | 20260506 17:22:55.283 | INFO | Perf report for the last 5.13892 seconds: | 20260506 17:22:55.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:22:55.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:55.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:55.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:55.283 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 17:22:55.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:55.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:55.283 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:22:55.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:55.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:22:55.284 | INFO | Uploaded 36GiB to GPU in 1.99443s (18.0503GiB/s) | 20260506 17:22:55.284 | INFO | Downloaded 34GiB to GPU in 1.97735s (17.1947GiB/s) | 20260506 17:23:00.421 | INFO | Analyzing chunk frames = [3116544,3117056) | 20260506 17:23:00.421 | INFO | MemInfo: { | 20260506 17:23:00.421 | INFO | "gpu" : " 17 GB", | 20260506 17:23:00.421 | INFO | "huge" : "", | 20260506 17:23:00.421 | INFO | "huge_shared" : " 27 GB", | 20260506 17:23:00.421 | INFO | "malloc" : " 14 GB", | 20260506 17:23:00.421 | INFO | "pinned" : " 29 GB" | 20260506 17:23:00.421 | INFO | } | 20260506 17:23:00.421 | INFO | Perf report for the last 5.13755 seconds: | 20260506 17:23:00.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:23:00.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:00.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:00.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:00.421 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 17:23:00.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:00.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:00.421 | INFO | PRE_HQ : 32.7%, 1.0, 17.0ms, 0 | 20260506 17:23:00.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:00.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:00.421 | INFO | Uploaded 36GiB to GPU in 1.9934s (18.0596GiB/s) | 20260506 17:23:00.421 | INFO | Downloaded 34GiB to GPU in 1.97119s (17.2485GiB/s) | 20260506 17:23:00.551 | INFO | Starting a round of Baz IO | 20260506 17:23:03.672 | INFO | Populating RTMetrics report | 20260506 17:23:03.673 | INFO | RTMetrics report populated | 20260506 17:23:03.673 | INFO | Opening tmp RTMetrics json file | 20260506 17:23:03.680 | INFO | Serializing RTMetrics to json | 20260506 17:23:03.682 | INFO | Serialization done, swapping tmp file now... | 20260506 17:23:03.682 | INFO | RTMetrics done | 20260506 17:23:03.915 | INFO | Finished a round of Baz IO: Wrote 2.066 GiB in 3.364 seconds | 20260506 17:23:03.915 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.68% / 0.01% / 64.31% | 20260506 17:23:03.915 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.39437% / 3.97364e-06% / 7.94729e-05% | 20260506 17:23:03.915 | INFO | Baz IO overall: | 20260506 17:23:03.915 | INFO | BytesWritten : 1642757402369 | 20260506 17:23:03.915 | INFO | NumEvents : 794747816503 | 20260506 17:23:05.558 | INFO | Analyzing chunk frames = [3117056,3117568) | 20260506 17:23:05.558 | INFO | MemInfo: { | 20260506 17:23:05.558 | INFO | "gpu" : " 17 GB", | 20260506 17:23:05.558 | INFO | "huge" : "", | 20260506 17:23:05.558 | INFO | "huge_shared" : " 27 GB", | 20260506 17:23:05.558 | INFO | "malloc" : " 14 GB", | 20260506 17:23:05.558 | INFO | "pinned" : " 29 GB" | 20260506 17:23:05.558 | INFO | } | 20260506 17:23:05.558 | INFO | Perf report for the last 5.13731 seconds: | 20260506 17:23:05.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:23:05.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:05.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:05.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:05.558 | INFO | ANALYSIS : 62.7%, 2.9, 95.1ms, 0 | 20260506 17:23:05.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:05.558 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 17:23:05.558 | INFO | PRE_HQ : 60.6%, 1.9, 58.7ms, 0 | 20260506 17:23:05.558 | INFO | BAZWRITER : 65.5%, 4.8, 161.3ms, 0 | 20260506 17:23:05.558 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 17:23:05.558 | INFO | Uploaded 36GiB to GPU in 2.04287s (17.6223GiB/s) | 20260506 17:23:05.558 | INFO | Downloaded 38GiB to GPU in 2.21039s (17.1915GiB/s) | 20260506 17:23:10.697 | INFO | Analyzing chunk frames = [3117568,3118080) | 20260506 17:23:10.697 | INFO | MemInfo: { | 20260506 17:23:10.697 | INFO | "gpu" : " 17 GB", | 20260506 17:23:10.697 | INFO | "huge" : "", | 20260506 17:23:10.697 | INFO | "huge_shared" : " 27 GB", | 20260506 17:23:10.697 | INFO | "malloc" : " 14 GB", | 20260506 17:23:10.697 | INFO | "pinned" : " 29 GB" | 20260506 17:23:10.697 | INFO | } | 20260506 17:23:10.697 | INFO | Perf report for the last 5.13898 seconds: | 20260506 17:23:10.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:23:10.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:10.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:10.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:10.697 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:23:10.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:10.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:10.697 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:23:10.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:10.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:10.697 | INFO | Uploaded 36GiB to GPU in 2.00022s (17.998GiB/s) | 20260506 17:23:10.697 | INFO | Downloaded 34GiB to GPU in 1.94052s (17.5211GiB/s) | 20260506 17:23:15.839 | INFO | Analyzing chunk frames = [3118080,3118592) | 20260506 17:23:15.839 | INFO | MemInfo: { | 20260506 17:23:15.839 | INFO | "gpu" : " 17 GB", | 20260506 17:23:15.839 | INFO | "huge" : "", | 20260506 17:23:15.840 | INFO | "huge_shared" : " 27 GB", | 20260506 17:23:15.840 | INFO | "malloc" : " 14 GB", | 20260506 17:23:15.840 | INFO | "pinned" : " 29 GB" | 20260506 17:23:15.840 | INFO | } | 20260506 17:23:15.840 | INFO | Perf report for the last 5.1426 seconds: | 20260506 17:23:15.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:23:15.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:15.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:15.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:15.840 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 17:23:15.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:15.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:15.840 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:23:15.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:15.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:15.840 | INFO | Uploaded 36GiB to GPU in 1.99301s (18.0632GiB/s) | 20260506 17:23:15.840 | INFO | Downloaded 34GiB to GPU in 1.9249s (17.6632GiB/s) | 20260506 17:23:20.983 | INFO | Analyzing chunk frames = [3118592,3119104) | 20260506 17:23:20.983 | INFO | MemInfo: { | 20260506 17:23:20.983 | INFO | "gpu" : " 17 GB", | 20260506 17:23:20.983 | INFO | "huge" : "", | 20260506 17:23:20.983 | INFO | "huge_shared" : " 27 GB", | 20260506 17:23:20.983 | INFO | "malloc" : " 14 GB", | 20260506 17:23:20.983 | INFO | "pinned" : " 29 GB" | 20260506 17:23:20.983 | INFO | } | 20260506 17:23:20.983 | INFO | Perf report for the last 5.14338 seconds: | 20260506 17:23:20.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:23:20.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:20.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:20.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:20.983 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:23:20.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:20.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:20.983 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:23:20.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:20.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:20.983 | INFO | Uploaded 36GiB to GPU in 1.93203s (18.6332GiB/s) | 20260506 17:23:20.983 | INFO | Downloaded 34GiB to GPU in 1.86963s (18.1854GiB/s) | 20260506 17:23:26.125 | INFO | Analyzing chunk frames = [3119104,3119616) | 20260506 17:23:26.125 | INFO | MemInfo: { | 20260506 17:23:26.125 | INFO | "gpu" : " 17 GB", | 20260506 17:23:26.125 | INFO | "huge" : "", | 20260506 17:23:26.125 | INFO | "huge_shared" : " 27 GB", | 20260506 17:23:26.125 | INFO | "malloc" : " 14 GB", | 20260506 17:23:26.125 | INFO | "pinned" : " 29 GB" | 20260506 17:23:26.125 | INFO | } | 20260506 17:23:26.125 | INFO | Perf report for the last 5.14241 seconds: | 20260506 17:23:26.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:23:26.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:26.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:26.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:26.125 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 17:23:26.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:26.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:26.125 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:23:26.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:26.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:26.126 | INFO | Uploaded 36GiB to GPU in 2.01914s (17.8294GiB/s) | 20260506 17:23:26.126 | INFO | Downloaded 34GiB to GPU in 1.91544s (17.7505GiB/s) | 20260506 17:23:31.258 | INFO | Analyzing chunk frames = [3119616,3120128) | 20260506 17:23:31.258 | INFO | MemInfo: { | 20260506 17:23:31.258 | INFO | "gpu" : " 17 GB", | 20260506 17:23:31.258 | INFO | "huge" : "", | 20260506 17:23:31.258 | INFO | "huge_shared" : " 27 GB", | 20260506 17:23:31.258 | INFO | "malloc" : " 14 GB", | 20260506 17:23:31.258 | INFO | "pinned" : " 29 GB" | 20260506 17:23:31.258 | INFO | } | 20260506 17:23:31.258 | INFO | Perf report for the last 5.13249 seconds: | 20260506 17:23:31.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:23:31.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:31.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:31.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:31.258 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 17:23:31.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:31.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:31.258 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:23:31.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:31.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:31.258 | INFO | Uploaded 36GiB to GPU in 2.07064s (17.3859GiB/s) | 20260506 17:23:31.258 | INFO | Downloaded 34GiB to GPU in 1.96114s (17.3369GiB/s) | 20260506 17:23:36.395 | INFO | Analyzing chunk frames = [3120128,3120640) | 20260506 17:23:36.395 | INFO | MemInfo: { | 20260506 17:23:36.395 | INFO | "gpu" : " 17 GB", | 20260506 17:23:36.395 | INFO | "huge" : "", | 20260506 17:23:36.395 | INFO | "huge_shared" : " 27 GB", | 20260506 17:23:36.395 | INFO | "malloc" : " 14 GB", | 20260506 17:23:36.395 | INFO | "pinned" : " 29 GB" | 20260506 17:23:36.395 | INFO | } | 20260506 17:23:36.395 | INFO | Perf report for the last 5.13719 seconds: | 20260506 17:23:36.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:23:36.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:36.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:36.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:36.395 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:23:36.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:36.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:36.395 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 17:23:36.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:36.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:36.395 | INFO | Uploaded 36GiB to GPU in 2.0048s (17.9569GiB/s) | 20260506 17:23:36.395 | INFO | Downloaded 34GiB to GPU in 1.96787s (17.2775GiB/s) | 20260506 17:23:41.536 | INFO | Analyzing chunk frames = [3120640,3121152) | 20260506 17:23:41.536 | INFO | MemInfo: { | 20260506 17:23:41.536 | INFO | "gpu" : " 17 GB", | 20260506 17:23:41.536 | INFO | "huge" : "", | 20260506 17:23:41.536 | INFO | "huge_shared" : " 27 GB", | 20260506 17:23:41.536 | INFO | "malloc" : " 14 GB", | 20260506 17:23:41.536 | INFO | "pinned" : " 29 GB" | 20260506 17:23:41.536 | INFO | } | 20260506 17:23:41.536 | INFO | Perf report for the last 5.14098 seconds: | 20260506 17:23:41.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:23:41.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:41.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:41.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:41.536 | INFO | ANALYSIS : 62.2%, 3.0, 97.0ms, 0 | 20260506 17:23:41.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:41.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:41.536 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:23:41.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:41.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:41.536 | INFO | Uploaded 36GiB to GPU in 2.06171s (17.4612GiB/s) | 20260506 17:23:41.537 | INFO | Downloaded 34GiB to GPU in 1.98256s (17.1496GiB/s) | 20260506 17:23:41.537 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:23:41.537 | INFO | Total mem usage: 18 GiB | 20260506 17:23:41.537 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:23:41.537 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.1%) | 20260506 17:23:41.537 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:23:41.537 | INFO | High Water Breakdown: | 20260506 17:23:41.537 | INFO | 61.7% : 9.81 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:23:41.537 | INFO | 38.2% : 6.08 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:23:41.537 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:23:41.537 | INFO | ------------------------------------------ | 20260506 17:23:41.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:23:41.537 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:23:41.537 | INFO | Total mem usage: 26.5 GiB | 20260506 17:23:41.537 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:23:41.537 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:23:41.537 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:23:41.537 | INFO | High Water Breakdown: | 20260506 17:23:41.537 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:23:41.537 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:23:41.537 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:23:41.537 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:23:41.537 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:23:41.537 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:23:41.537 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:23:41.537 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:23:41.537 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:23:41.537 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:23:41.537 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:23:41.537 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:23:41.537 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:23:41.537 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:23:41.537 | INFO | ------------------------------------------ | 20260506 17:23:41.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:23:41.537 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:23:41.537 | INFO | Total mem usage: 25.6 GiB | 20260506 17:23:41.537 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:23:41.537 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:23:41.537 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:23:41.537 | INFO | High Water Breakdown: | 20260506 17:23:41.537 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:23:41.537 | INFO | ------------------------------------------ | 20260506 17:23:41.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:23:41.537 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:23:41.537 | INFO | Total mem usage: 15.7 GiB | 20260506 17:23:41.537 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:23:41.537 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:23:41.537 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:23:41.537 | INFO | High Water Breakdown: | 20260506 17:23:41.537 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:23:41.537 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:23:41.537 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:23:41.537 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:23:41.537 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:23:41.537 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:23:41.537 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:23:41.537 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:23:41.537 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:23:41.537 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:23:41.537 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:23:41.537 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:23:41.537 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:23:41.537 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:23:41.537 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:23:41.537 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:23:41.537 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:23:41.537 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:23:41.537 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:23:41.537 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:23:41.537 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:23:41.537 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:23:41.537 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:23:41.537 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:23:41.537 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:23:41.537 | INFO | ------------------------------------------ | 20260506 17:23:41.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:23:41.538 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:23:41.538 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:23:41.538 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:23:41.538 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.55e+01, 1.34e+01 / 1.77e+00 | 20260506 17:23:41.538 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.01e-01, 1.04e-02 / 5.09e-03 | 20260506 17:23:41.538 | INFO | PerfReport: 11.62%, Baseline -- 3.65e+00 / 4.42e+00, 5.65e-01 / 3.59e-01 | 20260506 17:23:41.538 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.48e+00, 1.86e-01 / 1.22e-01 | 20260506 17:23:41.538 | INFO | PerfReport: 15.88%, DME -- 3.99e+01 / 4.40e+01, 3.91e+00 / 2.03e+00 | 20260506 17:23:41.538 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.50e+00 / 2.10e+00 | 20260506 17:23:41.538 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.28e+00 / 2.82e+00, 3.81e-01 / 2.43e-01 | 20260506 17:23:41.538 | INFO | PerfReport: 3.46%, Metrics -- 1.08e+00 / 1.44e+00, 2.85e-01 / 1.96e-01 | 20260506 17:23:41.538 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.75e-01 / 4.25e-01, 5.91e-02 / 2.91e-02 | 20260506 17:23:41.538 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:23:41.538 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:23:41.538 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:23:41.538 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 17:23:41.538 | INFO | PerfReport: 21.53%, Upload -- 2.10e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 17:23:41.538 | INFO | PerfReport: 21.11%, Download -- 2.05e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 17:23:41.538 | INFO | PerfReport: 32.23%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:23:41.538 | INFO | PerfReport: 1.52%, LockUpload -- 1.48e+00 / 0.00e+00, 5.83e+00 / 0.00e+00 | 20260506 17:23:41.538 | INFO | PerfReport: 1.00%, LockDownload -- 9.71e-01 / 0.00e+00, 2.56e+00 / 0.00e+00 | 20260506 17:23:41.538 | INFO | PerfReport: 22.59%, LockCompute -- 2.20e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260506 17:23:41.538 | INFO | PerfReport: 0.02%, Misc -- 1.60e-02 / 0.00e+00, 3.47e-02 / 0.00e+00 | 20260506 17:23:41.651 | INFO | Starting a round of Baz IO | 20260506 17:23:44.801 | INFO | Populating RTMetrics report | 20260506 17:23:44.801 | INFO | RTMetrics report populated | 20260506 17:23:44.801 | INFO | Opening tmp RTMetrics json file | 20260506 17:23:44.866 | INFO | Serializing RTMetrics to json | 20260506 17:23:44.868 | INFO | Serialization done, swapping tmp file now... | 20260506 17:23:44.871 | INFO | RTMetrics done | 20260506 17:23:45.293 | INFO | Finished a round of Baz IO: Wrote 2.068 GiB in 3.642 seconds | 20260506 17:23:45.293 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.68% / 0.01% / 64.32% | 20260506 17:23:45.294 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.39349% / 1.19209e-05% / 9.53674e-05% | 20260506 17:23:45.294 | INFO | Baz IO overall: | 20260506 17:23:45.294 | INFO | BytesWritten : 1644977477648 | 20260506 17:23:45.294 | INFO | NumEvents : 795819280715 | 20260506 17:23:46.674 | INFO | Analyzing chunk frames = [3121152,3121664) | 20260506 17:23:46.674 | INFO | MemInfo: { | 20260506 17:23:46.674 | INFO | "gpu" : " 17 GB", | 20260506 17:23:46.674 | INFO | "huge" : "", | 20260506 17:23:46.674 | INFO | "huge_shared" : " 27 GB", | 20260506 17:23:46.674 | INFO | "malloc" : " 14 GB", | 20260506 17:23:46.674 | INFO | "pinned" : " 29 GB" | 20260506 17:23:46.674 | INFO | } | 20260506 17:23:46.674 | INFO | Perf report for the last 5.13802 seconds: | 20260506 17:23:46.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:23:46.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:46.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:46.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:46.674 | INFO | ANALYSIS : 62.9%, 2.9, 95.7ms, 0 | 20260506 17:23:46.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:46.674 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260506 17:23:46.674 | INFO | PRE_HQ : 60.8%, 1.9, 58.5ms, 0 | 20260506 17:23:46.674 | INFO | BAZWRITER : 70.4%, 4.8, 173.7ms, 0 | 20260506 17:23:46.674 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 17:23:46.674 | INFO | Uploaded 36GiB to GPU in 2.06902s (17.3996GiB/s) | 20260506 17:23:46.674 | INFO | Downloaded 38GiB to GPU in 2.23459s (17.0054GiB/s) | 20260506 17:23:51.813 | INFO | Analyzing chunk frames = [3121664,3122176) | 20260506 17:23:51.813 | INFO | MemInfo: { | 20260506 17:23:51.813 | INFO | "gpu" : " 17 GB", | 20260506 17:23:51.813 | INFO | "huge" : "", | 20260506 17:23:51.813 | INFO | "huge_shared" : " 27 GB", | 20260506 17:23:51.813 | INFO | "malloc" : " 14 GB", | 20260506 17:23:51.813 | INFO | "pinned" : " 29 GB" | 20260506 17:23:51.813 | INFO | } | 20260506 17:23:51.813 | INFO | Perf report for the last 5.13916 seconds: | 20260506 17:23:51.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:23:51.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:51.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:51.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:51.813 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 17:23:51.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:51.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:51.813 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:23:51.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:51.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:51.814 | INFO | Uploaded 36GiB to GPU in 1.99983s (18.0015GiB/s) | 20260506 17:23:51.814 | INFO | Downloaded 34GiB to GPU in 1.93654s (17.5571GiB/s) | 20260506 17:23:56.955 | INFO | Analyzing chunk frames = [3122176,3122688) | 20260506 17:23:56.955 | INFO | MemInfo: { | 20260506 17:23:56.955 | INFO | "gpu" : " 17 GB", | 20260506 17:23:56.955 | INFO | "huge" : "", | 20260506 17:23:56.955 | INFO | "huge_shared" : " 27 GB", | 20260506 17:23:56.955 | INFO | "malloc" : " 14 GB", | 20260506 17:23:56.955 | INFO | "pinned" : " 29 GB" | 20260506 17:23:56.955 | INFO | } | 20260506 17:23:56.955 | INFO | Perf report for the last 5.14221 seconds: | 20260506 17:23:56.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:23:56.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:56.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:56.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:56.956 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 17:23:56.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:56.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:56.956 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:23:56.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:56.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:23:56.956 | INFO | Uploaded 36GiB to GPU in 1.95767s (18.3892GiB/s) | 20260506 17:23:56.956 | INFO | Downloaded 34GiB to GPU in 1.92584s (17.6546GiB/s) | 20260506 17:24:02.092 | INFO | Analyzing chunk frames = [3122688,3123200) | 20260506 17:24:02.092 | INFO | MemInfo: { | 20260506 17:24:02.092 | INFO | "gpu" : " 17 GB", | 20260506 17:24:02.092 | INFO | "huge" : "", | 20260506 17:24:02.092 | INFO | "huge_shared" : " 27 GB", | 20260506 17:24:02.092 | INFO | "malloc" : " 14 GB", | 20260506 17:24:02.092 | INFO | "pinned" : " 29 GB" | 20260506 17:24:02.092 | INFO | } | 20260506 17:24:02.092 | INFO | Perf report for the last 5.13613 seconds: | 20260506 17:24:02.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:24:02.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:02.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:02.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:02.092 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 17:24:02.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:02.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:02.092 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:24:02.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:02.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:02.092 | INFO | Uploaded 36GiB to GPU in 1.95343s (18.4291GiB/s) | 20260506 17:24:02.092 | INFO | Downloaded 34GiB to GPU in 1.88746s (18.0136GiB/s) | 20260506 17:24:07.232 | INFO | Analyzing chunk frames = [3123200,3123712) | 20260506 17:24:07.232 | INFO | MemInfo: { | 20260506 17:24:07.232 | INFO | "gpu" : " 17 GB", | 20260506 17:24:07.233 | INFO | "huge" : "", | 20260506 17:24:07.233 | INFO | "huge_shared" : " 27 GB", | 20260506 17:24:07.233 | INFO | "malloc" : " 14 GB", | 20260506 17:24:07.233 | INFO | "pinned" : " 29 GB" | 20260506 17:24:07.233 | INFO | } | 20260506 17:24:07.233 | INFO | Perf report for the last 5.14099 seconds: | 20260506 17:24:07.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:24:07.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:07.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:07.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:07.233 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 17:24:07.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:07.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:07.233 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:24:07.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:07.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:07.233 | INFO | Uploaded 36GiB to GPU in 2.02077s (17.815GiB/s) | 20260506 17:24:07.233 | INFO | Downloaded 34GiB to GPU in 1.91558s (17.7492GiB/s) | 20260506 17:24:12.374 | INFO | Analyzing chunk frames = [3123712,3124224) | 20260506 17:24:12.374 | INFO | MemInfo: { | 20260506 17:24:12.374 | INFO | "gpu" : " 17 GB", | 20260506 17:24:12.374 | INFO | "huge" : "", | 20260506 17:24:12.374 | INFO | "huge_shared" : " 27 GB", | 20260506 17:24:12.374 | INFO | "malloc" : " 14 GB", | 20260506 17:24:12.374 | INFO | "pinned" : " 29 GB" | 20260506 17:24:12.374 | INFO | } | 20260506 17:24:12.374 | INFO | Perf report for the last 5.14135 seconds: | 20260506 17:24:12.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:24:12.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:12.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:12.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:12.374 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:24:12.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:12.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:12.374 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 17:24:12.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:12.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:12.374 | INFO | Uploaded 36GiB to GPU in 2.0754s (17.346GiB/s) | 20260506 17:24:12.374 | INFO | Downloaded 34GiB to GPU in 1.96575s (17.2962GiB/s) | 20260506 17:24:17.513 | INFO | Analyzing chunk frames = [3124224,3124736) | 20260506 17:24:17.513 | INFO | MemInfo: { | 20260506 17:24:17.513 | INFO | "gpu" : " 17 GB", | 20260506 17:24:17.513 | INFO | "huge" : "", | 20260506 17:24:17.513 | INFO | "huge_shared" : " 27 GB", | 20260506 17:24:17.513 | INFO | "malloc" : " 14 GB", | 20260506 17:24:17.513 | INFO | "pinned" : " 29 GB" | 20260506 17:24:17.513 | INFO | } | 20260506 17:24:17.513 | INFO | Perf report for the last 5.13871 seconds: | 20260506 17:24:17.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:24:17.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:17.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:17.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:17.513 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:24:17.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:17.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:17.513 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 17:24:17.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:17.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:17.513 | INFO | Uploaded 36GiB to GPU in 2.00955s (17.9145GiB/s) | 20260506 17:24:17.513 | INFO | Downloaded 34GiB to GPU in 2.00324s (16.9725GiB/s) | 20260506 17:24:22.650 | INFO | Analyzing chunk frames = [3124736,3125248) | 20260506 17:24:22.650 | INFO | MemInfo: { | 20260506 17:24:22.650 | INFO | "gpu" : " 17 GB", | 20260506 17:24:22.650 | INFO | "huge" : "", | 20260506 17:24:22.650 | INFO | "huge_shared" : " 27 GB", | 20260506 17:24:22.650 | INFO | "malloc" : " 14 GB", | 20260506 17:24:22.650 | INFO | "pinned" : " 29 GB" | 20260506 17:24:22.650 | INFO | } | 20260506 17:24:22.650 | INFO | Perf report for the last 5.13706 seconds: | 20260506 17:24:22.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:24:22.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:22.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:22.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:22.650 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:24:22.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:22.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:22.650 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:24:22.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:22.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:22.650 | INFO | Uploaded 36GiB to GPU in 2.04076s (17.6405GiB/s) | 20260506 17:24:22.650 | INFO | Downloaded 34GiB to GPU in 1.99099s (17.0769GiB/s) | 20260506 17:24:22.770 | INFO | Starting a round of Baz IO | 20260506 17:24:25.904 | INFO | Populating RTMetrics report | 20260506 17:24:25.905 | INFO | RTMetrics report populated | 20260506 17:24:25.905 | INFO | Opening tmp RTMetrics json file | 20260506 17:24:25.930 | INFO | Serializing RTMetrics to json | 20260506 17:24:25.991 | INFO | Serialization done, swapping tmp file now... | 20260506 17:24:25.995 | INFO | RTMetrics done | 20260506 17:24:26.300 | INFO | Finished a round of Baz IO: Wrote 2.067 GiB in 3.53 seconds | 20260506 17:24:26.300 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.67% / 0.01% / 64.32% | 20260506 17:24:26.300 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.38968% / 3.97364e-06% / 9.13938e-05% | 20260506 17:24:26.300 | INFO | Baz IO overall: | 20260506 17:24:26.300 | INFO | BytesWritten : 1647196800085 | 20260506 17:24:26.300 | INFO | NumEvents : 796890399682 | 20260506 17:24:27.788 | INFO | Analyzing chunk frames = [3125248,3125760) | 20260506 17:24:27.788 | INFO | MemInfo: { | 20260506 17:24:27.788 | INFO | "gpu" : " 17 GB", | 20260506 17:24:27.788 | INFO | "huge" : "", | 20260506 17:24:27.788 | INFO | "huge_shared" : " 27 GB", | 20260506 17:24:27.788 | INFO | "malloc" : " 14 GB", | 20260506 17:24:27.788 | INFO | "pinned" : " 29 GB" | 20260506 17:24:27.788 | INFO | } | 20260506 17:24:27.788 | INFO | Perf report for the last 5.1381 seconds: | 20260506 17:24:27.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:24:27.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:27.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:27.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:27.788 | INFO | ANALYSIS : 62.7%, 2.9, 95.5ms, 0 | 20260506 17:24:27.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:27.788 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260506 17:24:27.788 | INFO | PRE_HQ : 60.1%, 1.9, 58.1ms, 0 | 20260506 17:24:27.788 | INFO | BAZWRITER : 68.3%, 4.8, 167.9ms, 0 | 20260506 17:24:27.788 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 17:24:27.788 | INFO | Uploaded 36GiB to GPU in 2.08366s (17.2773GiB/s) | 20260506 17:24:27.788 | INFO | Downloaded 38GiB to GPU in 2.21981s (17.1186GiB/s) | 20260506 17:24:32.931 | INFO | Analyzing chunk frames = [3125760,3126272) | 20260506 17:24:32.931 | INFO | MemInfo: { | 20260506 17:24:32.931 | INFO | "gpu" : " 17 GB", | 20260506 17:24:32.931 | INFO | "huge" : "", | 20260506 17:24:32.931 | INFO | "huge_shared" : " 27 GB", | 20260506 17:24:32.931 | INFO | "malloc" : " 14 GB", | 20260506 17:24:32.931 | INFO | "pinned" : " 29 GB" | 20260506 17:24:32.931 | INFO | } | 20260506 17:24:32.931 | INFO | Perf report for the last 5.14342 seconds: | 20260506 17:24:32.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:24:32.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:32.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:32.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:32.931 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:24:32.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:32.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:32.931 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:24:32.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:32.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:32.932 | INFO | Uploaded 36GiB to GPU in 2.01932s (17.8278GiB/s) | 20260506 17:24:32.932 | INFO | Downloaded 34GiB to GPU in 1.98206s (17.1539GiB/s) | 20260506 17:24:38.069 | INFO | Analyzing chunk frames = [3126272,3126784) | 20260506 17:24:38.069 | INFO | MemInfo: { | 20260506 17:24:38.069 | INFO | "gpu" : " 17 GB", | 20260506 17:24:38.069 | INFO | "huge" : "", | 20260506 17:24:38.069 | INFO | "huge_shared" : " 27 GB", | 20260506 17:24:38.069 | INFO | "malloc" : " 14 GB", | 20260506 17:24:38.069 | INFO | "pinned" : " 29 GB" | 20260506 17:24:38.069 | INFO | } | 20260506 17:24:38.069 | INFO | Perf report for the last 5.13776 seconds: | 20260506 17:24:38.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:24:38.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:38.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:38.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:38.069 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 17:24:38.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:38.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:38.069 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:24:38.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:38.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:38.070 | INFO | Uploaded 36GiB to GPU in 1.99256s (18.0672GiB/s) | 20260506 17:24:38.070 | INFO | Downloaded 34GiB to GPU in 1.92589s (17.6542GiB/s) | 20260506 17:24:43.210 | INFO | Analyzing chunk frames = [3126784,3127296) | 20260506 17:24:43.210 | INFO | MemInfo: { | 20260506 17:24:43.210 | INFO | "gpu" : " 17 GB", | 20260506 17:24:43.210 | INFO | "huge" : "", | 20260506 17:24:43.210 | INFO | "huge_shared" : " 27 GB", | 20260506 17:24:43.210 | INFO | "malloc" : " 14 GB", | 20260506 17:24:43.210 | INFO | "pinned" : " 29 GB" | 20260506 17:24:43.210 | INFO | } | 20260506 17:24:43.210 | INFO | Perf report for the last 5.14065 seconds: | 20260506 17:24:43.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:24:43.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:43.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:43.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:43.210 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 17:24:43.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:43.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:43.210 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:24:43.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:43.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:43.210 | INFO | Uploaded 36GiB to GPU in 1.93175s (18.6359GiB/s) | 20260506 17:24:43.210 | INFO | Downloaded 34GiB to GPU in 1.88063s (18.0791GiB/s) | 20260506 17:24:48.349 | INFO | Analyzing chunk frames = [3127296,3127808) | 20260506 17:24:48.349 | INFO | MemInfo: { | 20260506 17:24:48.349 | INFO | "gpu" : " 17 GB", | 20260506 17:24:48.349 | INFO | "huge" : "", | 20260506 17:24:48.349 | INFO | "huge_shared" : " 27 GB", | 20260506 17:24:48.349 | INFO | "malloc" : " 14 GB", | 20260506 17:24:48.349 | INFO | "pinned" : " 29 GB" | 20260506 17:24:48.349 | INFO | } | 20260506 17:24:48.349 | INFO | Perf report for the last 5.13915 seconds: | 20260506 17:24:48.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:24:48.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:48.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:48.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:48.349 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 17:24:48.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:48.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:48.349 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:24:48.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:48.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:48.349 | INFO | Uploaded 36GiB to GPU in 2.00109s (17.9902GiB/s) | 20260506 17:24:48.350 | INFO | Downloaded 34GiB to GPU in 1.91338s (17.7696GiB/s) | 20260506 17:24:53.488 | INFO | Analyzing chunk frames = [3127808,3128320) | 20260506 17:24:53.488 | INFO | MemInfo: { | 20260506 17:24:53.488 | INFO | "gpu" : " 17 GB", | 20260506 17:24:53.488 | INFO | "huge" : "", | 20260506 17:24:53.488 | INFO | "huge_shared" : " 27 GB", | 20260506 17:24:53.488 | INFO | "malloc" : " 14 GB", | 20260506 17:24:53.488 | INFO | "pinned" : " 29 GB" | 20260506 17:24:53.488 | INFO | } | 20260506 17:24:53.488 | INFO | Perf report for the last 5.13912 seconds: | 20260506 17:24:53.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:24:53.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:53.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:53.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:53.488 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:24:53.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:53.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:53.488 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 17:24:53.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:53.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:53.488 | INFO | Uploaded 36GiB to GPU in 2.05923s (17.4822GiB/s) | 20260506 17:24:53.488 | INFO | Downloaded 34GiB to GPU in 1.95615s (17.3811GiB/s) | 20260506 17:24:58.630 | INFO | Analyzing chunk frames = [3128320,3128832) | 20260506 17:24:58.630 | INFO | MemInfo: { | 20260506 17:24:58.630 | INFO | "gpu" : " 17 GB", | 20260506 17:24:58.630 | INFO | "huge" : "", | 20260506 17:24:58.631 | INFO | "huge_shared" : " 27 GB", | 20260506 17:24:58.631 | INFO | "malloc" : " 14 GB", | 20260506 17:24:58.631 | INFO | "pinned" : " 29 GB" | 20260506 17:24:58.631 | INFO | } | 20260506 17:24:58.631 | INFO | Perf report for the last 5.14256 seconds: | 20260506 17:24:58.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:24:58.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:58.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:58.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:58.631 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 17:24:58.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:58.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:58.631 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 17:24:58.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:58.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:24:58.631 | INFO | Uploaded 36GiB to GPU in 2.00174s (17.9843GiB/s) | 20260506 17:24:58.631 | INFO | Downloaded 34GiB to GPU in 1.97333s (17.2297GiB/s) | 20260506 17:25:03.766 | INFO | Analyzing chunk frames = [3128832,3129344) | 20260506 17:25:03.766 | INFO | MemInfo: { | 20260506 17:25:03.766 | INFO | "gpu" : " 17 GB", | 20260506 17:25:03.766 | INFO | "huge" : "", | 20260506 17:25:03.766 | INFO | "huge_shared" : " 27 GB", | 20260506 17:25:03.766 | INFO | "malloc" : " 14 GB", | 20260506 17:25:03.766 | INFO | "pinned" : " 29 GB" | 20260506 17:25:03.766 | INFO | } | 20260506 17:25:03.766 | INFO | Perf report for the last 5.13526 seconds: | 20260506 17:25:03.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:25:03.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:03.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:03.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:03.766 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:25:03.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:03.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:03.766 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 17:25:03.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:03.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:03.766 | INFO | Uploaded 36GiB to GPU in 2.02448s (17.7823GiB/s) | 20260506 17:25:03.766 | INFO | Downloaded 34GiB to GPU in 1.98714s (17.11GiB/s) | 20260506 17:25:03.766 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:25:03.766 | INFO | Total mem usage: 18 GiB | 20260506 17:25:03.766 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:25:03.766 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: -9.2%) | 20260506 17:25:03.766 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:25:03.766 | INFO | High Water Breakdown: | 20260506 17:25:03.766 | INFO | 62.2% : 10.3 GiB ( 89.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:25:03.766 | INFO | 37.7% : 6.21 GiB ( 92.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:25:03.766 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:25:03.766 | INFO | ------------------------------------------ | 20260506 17:25:03.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:25:03.766 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:25:03.767 | INFO | Total mem usage: 26.5 GiB | 20260506 17:25:03.767 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:25:03.767 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:25:03.767 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:25:03.767 | INFO | High Water Breakdown: | 20260506 17:25:03.767 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:25:03.767 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:25:03.767 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:25:03.767 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:25:03.767 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:25:03.767 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:25:03.767 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:25:03.767 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:25:03.767 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:25:03.767 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:25:03.767 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:25:03.767 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:25:03.767 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:25:03.767 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:25:03.767 | INFO | ------------------------------------------ | 20260506 17:25:03.767 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:25:03.767 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:25:03.767 | INFO | Total mem usage: 25.6 GiB | 20260506 17:25:03.767 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:25:03.767 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:25:03.767 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:25:03.767 | INFO | High Water Breakdown: | 20260506 17:25:03.767 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:25:03.767 | INFO | ------------------------------------------ | 20260506 17:25:03.767 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:25:03.767 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:25:03.767 | INFO | Total mem usage: 15.7 GiB | 20260506 17:25:03.767 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:25:03.767 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:25:03.767 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 17:25:03.767 | INFO | High Water Breakdown: | 20260506 17:25:03.767 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:25:03.767 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:25:03.767 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:25:03.767 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:25:03.767 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:25:03.767 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:25:03.767 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:25:03.767 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:25:03.767 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:25:03.767 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:25:03.767 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:25:03.767 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:25:03.767 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:25:03.767 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:25:03.767 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:25:03.767 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:25:03.767 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:25:03.767 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:25:03.767 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:25:03.767 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:25:03.767 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:25:03.767 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:25:03.767 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:25:03.767 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:25:03.767 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:25:03.767 | INFO | ------------------------------------------ | 20260506 17:25:03.767 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:25:03.767 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:25:03.767 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:25:03.767 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:25:03.767 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.53e+01, 1.34e+01 / 1.97e+00 | 20260506 17:25:03.767 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.00e-01, 1.01e-02 / 5.69e-03 | 20260506 17:25:03.767 | INFO | PerfReport: 11.59%, Baseline -- 3.63e+00 / 4.42e+00, 5.50e-01 / 3.38e-01 | 20260506 17:25:03.767 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.46e+00, 1.81e-01 / 1.31e-01 | 20260506 17:25:03.767 | INFO | PerfReport: 15.92%, DME -- 3.99e+01 / 4.42e+01, 3.74e+00 / 1.79e+00 | 20260506 17:25:03.767 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.43e+00 / 1.84e+00 | 20260506 17:25:03.767 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.28e+00 / 2.85e+00, 3.91e-01 / 2.29e-01 | 20260506 17:25:03.767 | INFO | PerfReport: 3.47%, Metrics -- 1.09e+00 / 1.45e+00, 2.84e-01 / 2.14e-01 | 20260506 17:25:03.767 | INFO | PerfReport: 1.20%, PulseSerializing -- 3.75e-01 / 4.22e-01, 5.91e-02 / 2.38e-02 | 20260506 17:25:03.767 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:25:03.767 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:25:03.767 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:25:03.767 | INFO | PerfReport: Full Execution -- 9.74e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 17:25:03.768 | INFO | PerfReport: 21.55%, Upload -- 2.10e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260506 17:25:03.768 | INFO | PerfReport: 21.19%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 17:25:03.768 | INFO | PerfReport: 32.15%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:25:03.768 | INFO | PerfReport: 1.49%, LockUpload -- 1.45e+00 / 0.00e+00, 5.78e+00 / 0.00e+00 | 20260506 17:25:03.768 | INFO | PerfReport: 1.05%, LockDownload -- 1.02e+00 / 0.00e+00, 2.59e+00 / 0.00e+00 | 20260506 17:25:03.768 | INFO | PerfReport: 22.54%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 17:25:03.768 | INFO | PerfReport: 0.01%, Misc -- 1.46e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260506 17:25:03.886 | INFO | Starting a round of Baz IO | 20260506 17:25:07.020 | INFO | Populating RTMetrics report | 20260506 17:25:07.020 | INFO | RTMetrics report populated | 20260506 17:25:07.020 | INFO | Opening tmp RTMetrics json file | 20260506 17:25:07.179 | INFO | Serializing RTMetrics to json | 20260506 17:25:07.239 | INFO | Serialization done, swapping tmp file now... | 20260506 17:25:07.242 | INFO | RTMetrics done | 20260506 17:25:07.368 | INFO | Finished a round of Baz IO: Wrote 2.067 GiB in 3.481 seconds | 20260506 17:25:07.368 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.66% / 0.01% / 64.33% | 20260506 17:25:07.368 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.38537% / 1.19209e-05% / 7.15256e-05% | 20260506 17:25:07.368 | INFO | Baz IO overall: | 20260506 17:25:07.368 | INFO | BytesWritten : 1649415690688 | 20260506 17:25:07.368 | INFO | NumEvents : 797961323398 | 20260506 17:25:08.899 | INFO | Analyzing chunk frames = [3129344,3129856) | 20260506 17:25:08.899 | INFO | MemInfo: { | 20260506 17:25:08.899 | INFO | "gpu" : " 17 GB", | 20260506 17:25:08.899 | INFO | "huge" : "", | 20260506 17:25:08.899 | INFO | "huge_shared" : " 27 GB", | 20260506 17:25:08.899 | INFO | "malloc" : " 14 GB", | 20260506 17:25:08.899 | INFO | "pinned" : " 29 GB" | 20260506 17:25:08.899 | INFO | } | 20260506 17:25:08.899 | INFO | Perf report for the last 5.1335 seconds: | 20260506 17:25:08.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:25:08.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:08.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:08.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:08.900 | INFO | ANALYSIS : 62.8%, 2.9, 95.3ms, 0 | 20260506 17:25:08.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:08.900 | INFO | RT_METRICS : 47.8%, 1.0, 25.0ms, 0 | 20260506 17:25:08.900 | INFO | PRE_HQ : 60.5%, 1.9, 58.6ms, 0 | 20260506 17:25:08.900 | INFO | BAZWRITER : 67.5%, 4.8, 165.0ms, 0 | 20260506 17:25:08.900 | INFO | BAZ_SAMPLING : 2.9%, 1.0, 1.5ms, 0 | 20260506 17:25:08.900 | INFO | Uploaded 36GiB to GPU in 2.06018s (17.4742GiB/s) | 20260506 17:25:08.900 | INFO | Downloaded 38GiB to GPU in 2.21475s (17.1577GiB/s) | 20260506 17:25:14.041 | INFO | Analyzing chunk frames = [3129856,3130368) | 20260506 17:25:14.041 | INFO | MemInfo: { | 20260506 17:25:14.041 | INFO | "gpu" : " 17 GB", | 20260506 17:25:14.041 | INFO | "huge" : "", | 20260506 17:25:14.041 | INFO | "huge_shared" : " 27 GB", | 20260506 17:25:14.041 | INFO | "malloc" : " 14 GB", | 20260506 17:25:14.041 | INFO | "pinned" : " 29 GB" | 20260506 17:25:14.041 | INFO | } | 20260506 17:25:14.041 | INFO | Perf report for the last 5.1416 seconds: | 20260506 17:25:14.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:25:14.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:14.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:14.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:14.041 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 17:25:14.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:14.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:14.041 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:25:14.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:14.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:14.041 | INFO | Uploaded 36GiB to GPU in 2.01922s (17.8287GiB/s) | 20260506 17:25:14.041 | INFO | Downloaded 34GiB to GPU in 1.98999s (17.0856GiB/s) | 20260506 17:25:19.186 | INFO | Analyzing chunk frames = [3130368,3130880) | 20260506 17:25:19.186 | INFO | MemInfo: { | 20260506 17:25:19.186 | INFO | "gpu" : " 17 GB", | 20260506 17:25:19.186 | INFO | "huge" : "", | 20260506 17:25:19.186 | INFO | "huge_shared" : " 27 GB", | 20260506 17:25:19.186 | INFO | "malloc" : " 14 GB", | 20260506 17:25:19.186 | INFO | "pinned" : " 29 GB" | 20260506 17:25:19.186 | INFO | } | 20260506 17:25:19.186 | INFO | Perf report for the last 5.14464 seconds: | 20260506 17:25:19.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:25:19.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:19.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:19.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:19.186 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 17:25:19.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:19.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:19.186 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:25:19.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:19.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:19.186 | INFO | Uploaded 36GiB to GPU in 1.96861s (18.287GiB/s) | 20260506 17:25:19.186 | INFO | Downloaded 34GiB to GPU in 1.92472s (17.6649GiB/s) | 20260506 17:25:24.325 | INFO | Analyzing chunk frames = [3130880,3131392) | 20260506 17:25:24.325 | INFO | MemInfo: { | 20260506 17:25:24.325 | INFO | "gpu" : " 17 GB", | 20260506 17:25:24.325 | INFO | "huge" : "", | 20260506 17:25:24.325 | INFO | "huge_shared" : " 27 GB", | 20260506 17:25:24.325 | INFO | "malloc" : " 14 GB", | 20260506 17:25:24.325 | INFO | "pinned" : " 29 GB" | 20260506 17:25:24.325 | INFO | } | 20260506 17:25:24.325 | INFO | Perf report for the last 5.13949 seconds: | 20260506 17:25:24.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:25:24.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:24.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:24.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:24.325 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 17:25:24.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:24.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:24.325 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:25:24.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:24.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:24.325 | INFO | Uploaded 36GiB to GPU in 1.93764s (18.5793GiB/s) | 20260506 17:25:24.325 | INFO | Downloaded 34GiB to GPU in 1.87414s (18.1417GiB/s) | 20260506 17:25:29.464 | INFO | Analyzing chunk frames = [3131392,3131904) | 20260506 17:25:29.464 | INFO | MemInfo: { | 20260506 17:25:29.464 | INFO | "gpu" : " 17 GB", | 20260506 17:25:29.464 | INFO | "huge" : "", | 20260506 17:25:29.464 | INFO | "huge_shared" : " 27 GB", | 20260506 17:25:29.464 | INFO | "malloc" : " 14 GB", | 20260506 17:25:29.464 | INFO | "pinned" : " 29 GB" | 20260506 17:25:29.464 | INFO | } | 20260506 17:25:29.464 | INFO | Perf report for the last 5.13869 seconds: | 20260506 17:25:29.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:25:29.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:29.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:29.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:29.464 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 17:25:29.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:29.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:29.464 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:25:29.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:29.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:29.464 | INFO | Uploaded 36GiB to GPU in 2.01251s (17.8881GiB/s) | 20260506 17:25:29.464 | INFO | Downloaded 34GiB to GPU in 1.90167s (17.879GiB/s) | 20260506 17:25:34.604 | INFO | Analyzing chunk frames = [3131904,3132416) | 20260506 17:25:34.604 | INFO | MemInfo: { | 20260506 17:25:34.604 | INFO | "gpu" : " 17 GB", | 20260506 17:25:34.604 | INFO | "huge" : "", | 20260506 17:25:34.604 | INFO | "huge_shared" : " 27 GB", | 20260506 17:25:34.604 | INFO | "malloc" : " 14 GB", | 20260506 17:25:34.604 | INFO | "pinned" : " 29 GB" | 20260506 17:25:34.604 | INFO | } | 20260506 17:25:34.605 | INFO | Perf report for the last 5.14057 seconds: | 20260506 17:25:34.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:25:34.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:34.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:34.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:34.605 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 17:25:34.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:34.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:34.605 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 17:25:34.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:34.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:34.605 | INFO | Uploaded 36GiB to GPU in 2.04884s (17.5709GiB/s) | 20260506 17:25:34.605 | INFO | Downloaded 34GiB to GPU in 1.95333s (17.4062GiB/s) | 20260506 17:25:39.741 | INFO | Analyzing chunk frames = [3132416,3132928) | 20260506 17:25:39.741 | INFO | MemInfo: { | 20260506 17:25:39.741 | INFO | "gpu" : " 17 GB", | 20260506 17:25:39.741 | INFO | "huge" : "", | 20260506 17:25:39.742 | INFO | "huge_shared" : " 27 GB", | 20260506 17:25:39.742 | INFO | "malloc" : " 14 GB", | 20260506 17:25:39.742 | INFO | "pinned" : " 29 GB" | 20260506 17:25:39.742 | INFO | } | 20260506 17:25:39.742 | INFO | Perf report for the last 5.1371 seconds: | 20260506 17:25:39.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:25:39.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:39.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:39.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:39.742 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:25:39.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:39.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:39.742 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 17:25:39.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:39.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:39.742 | INFO | Uploaded 36GiB to GPU in 2.02302s (17.7952GiB/s) | 20260506 17:25:39.742 | INFO | Downloaded 34GiB to GPU in 1.98954s (17.0894GiB/s) | 20260506 17:25:44.879 | INFO | Analyzing chunk frames = [3132928,3133440) | 20260506 17:25:44.879 | INFO | MemInfo: { | 20260506 17:25:44.879 | INFO | "gpu" : " 17 GB", | 20260506 17:25:44.879 | INFO | "huge" : "", | 20260506 17:25:44.879 | INFO | "huge_shared" : " 27 GB", | 20260506 17:25:44.879 | INFO | "malloc" : " 14 GB", | 20260506 17:25:44.879 | INFO | "pinned" : " 29 GB" | 20260506 17:25:44.879 | INFO | } | 20260506 17:25:44.879 | INFO | Perf report for the last 5.13778 seconds: | 20260506 17:25:44.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:25:44.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:44.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:44.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:44.879 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 17:25:44.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:44.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:44.879 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:25:44.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:44.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:44.880 | INFO | Uploaded 36GiB to GPU in 2.0505s (17.5567GiB/s) | 20260506 17:25:44.880 | INFO | Downloaded 34GiB to GPU in 1.97309s (17.2319GiB/s) | 20260506 17:25:44.990 | INFO | Starting a round of Baz IO | 20260506 17:25:48.140 | INFO | Populating RTMetrics report | 20260506 17:25:48.140 | INFO | RTMetrics report populated | 20260506 17:25:48.140 | INFO | Opening tmp RTMetrics json file | 20260506 17:25:48.148 | INFO | Serializing RTMetrics to json | 20260506 17:25:48.150 | INFO | Serialization done, swapping tmp file now... | 20260506 17:25:48.154 | INFO | RTMetrics done | 20260506 17:25:48.488 | INFO | Finished a round of Baz IO: Wrote 2.065 GiB in 3.497 seconds | 20260506 17:25:48.488 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.65% / 0.01% / 64.34% | 20260506 17:25:48.488 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.38061% / 1.58946e-05% / 6.35783e-05% | 20260506 17:25:48.488 | INFO | Baz IO overall: | 20260506 17:25:48.488 | INFO | BytesWritten : 1651633217872 | 20260506 17:25:48.488 | INFO | NumEvents : 799031045512 | 20260506 17:25:50.014 | INFO | Analyzing chunk frames = [3133440,3133952) | 20260506 17:25:50.014 | INFO | MemInfo: { | 20260506 17:25:50.014 | INFO | "gpu" : " 17 GB", | 20260506 17:25:50.014 | INFO | "huge" : "", | 20260506 17:25:50.014 | INFO | "huge_shared" : " 27 GB", | 20260506 17:25:50.014 | INFO | "malloc" : " 14 GB", | 20260506 17:25:50.014 | INFO | "pinned" : " 29 GB" | 20260506 17:25:50.014 | INFO | } | 20260506 17:25:50.014 | INFO | Perf report for the last 5.13505 seconds: | 20260506 17:25:50.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:25:50.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:50.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:50.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:50.015 | INFO | ANALYSIS : 62.9%, 2.9, 95.8ms, 0 | 20260506 17:25:50.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:50.015 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 17:25:50.015 | INFO | PRE_HQ : 60.5%, 1.8, 57.9ms, 0 | 20260506 17:25:50.015 | INFO | BAZWRITER : 67.5%, 4.8, 167.4ms, 0 | 20260506 17:25:50.015 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 17:25:50.015 | INFO | Uploaded 36GiB to GPU in 2.04404s (17.6122GiB/s) | 20260506 17:25:50.015 | INFO | Downloaded 38GiB to GPU in 2.22301s (17.094GiB/s) | 20260506 17:25:55.157 | INFO | Analyzing chunk frames = [3133952,3134464) | 20260506 17:25:55.157 | INFO | MemInfo: { | 20260506 17:25:55.157 | INFO | "gpu" : " 17 GB", | 20260506 17:25:55.157 | INFO | "huge" : "", | 20260506 17:25:55.157 | INFO | "huge_shared" : " 27 GB", | 20260506 17:25:55.157 | INFO | "malloc" : " 14 GB", | 20260506 17:25:55.157 | INFO | "pinned" : " 29 GB" | 20260506 17:25:55.157 | INFO | } | 20260506 17:25:55.157 | INFO | Perf report for the last 5.14276 seconds: | 20260506 17:25:55.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:25:55.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:55.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:55.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:55.157 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:25:55.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:55.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:55.157 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:25:55.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:55.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:25:55.157 | INFO | Uploaded 36GiB to GPU in 1.9946s (18.0487GiB/s) | 20260506 17:25:55.158 | INFO | Downloaded 34GiB to GPU in 1.93529s (17.5685GiB/s) | 20260506 17:26:00.302 | INFO | Analyzing chunk frames = [3134464,3134976) | 20260506 17:26:00.302 | INFO | MemInfo: { | 20260506 17:26:00.302 | INFO | "gpu" : " 17 GB", | 20260506 17:26:00.302 | INFO | "huge" : "", | 20260506 17:26:00.302 | INFO | "huge_shared" : " 27 GB", | 20260506 17:26:00.302 | INFO | "malloc" : " 14 GB", | 20260506 17:26:00.302 | INFO | "pinned" : " 29 GB" | 20260506 17:26:00.302 | INFO | } | 20260506 17:26:00.302 | INFO | Perf report for the last 5.14495 seconds: | 20260506 17:26:00.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:26:00.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:00.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:00.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:00.302 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 17:26:00.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:00.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:00.302 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:26:00.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:00.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:00.303 | INFO | Uploaded 36GiB to GPU in 1.96932s (18.2804GiB/s) | 20260506 17:26:00.303 | INFO | Downloaded 34GiB to GPU in 1.92418s (17.6699GiB/s) | 20260506 17:26:05.440 | INFO | Analyzing chunk frames = [3134976,3135488) | 20260506 17:26:05.440 | INFO | MemInfo: { | 20260506 17:26:05.440 | INFO | "gpu" : " 17 GB", | 20260506 17:26:05.440 | INFO | "huge" : "", | 20260506 17:26:05.440 | INFO | "huge_shared" : " 27 GB", | 20260506 17:26:05.440 | INFO | "malloc" : " 14 GB", | 20260506 17:26:05.440 | INFO | "pinned" : " 29 GB" | 20260506 17:26:05.440 | INFO | } | 20260506 17:26:05.440 | INFO | Perf report for the last 5.13781 seconds: | 20260506 17:26:05.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:26:05.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:05.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:05.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:05.440 | INFO | ANALYSIS : 60.1%, 3.0, 93.0ms, 0 | 20260506 17:26:05.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:05.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:05.440 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:26:05.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:05.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:05.440 | INFO | Uploaded 36GiB to GPU in 1.954s (18.4237GiB/s) | 20260506 17:26:05.440 | INFO | Downloaded 34GiB to GPU in 1.87132s (18.169GiB/s) | 20260506 17:26:10.580 | INFO | Analyzing chunk frames = [3135488,3136000) | 20260506 17:26:10.580 | INFO | MemInfo: { | 20260506 17:26:10.580 | INFO | "gpu" : " 17 GB", | 20260506 17:26:10.580 | INFO | "huge" : "", | 20260506 17:26:10.580 | INFO | "huge_shared" : " 27 GB", | 20260506 17:26:10.580 | INFO | "malloc" : " 14 GB", | 20260506 17:26:10.580 | INFO | "pinned" : " 29 GB" | 20260506 17:26:10.580 | INFO | } | 20260506 17:26:10.580 | INFO | Perf report for the last 5.13991 seconds: | 20260506 17:26:10.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:26:10.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:10.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:10.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:10.580 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 17:26:10.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:10.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:10.580 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:26:10.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:10.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:10.580 | INFO | Uploaded 36GiB to GPU in 2.01663s (17.8516GiB/s) | 20260506 17:26:10.580 | INFO | Downloaded 34GiB to GPU in 1.90421s (17.8552GiB/s) | 20260506 17:26:15.714 | INFO | Analyzing chunk frames = [3136000,3136512) | 20260506 17:26:15.714 | INFO | MemInfo: { | 20260506 17:26:15.714 | INFO | "gpu" : " 17 GB", | 20260506 17:26:15.714 | INFO | "huge" : "", | 20260506 17:26:15.714 | INFO | "huge_shared" : " 27 GB", | 20260506 17:26:15.714 | INFO | "malloc" : " 14 GB", | 20260506 17:26:15.714 | INFO | "pinned" : " 29 GB" | 20260506 17:26:15.714 | INFO | } | 20260506 17:26:15.714 | INFO | Perf report for the last 5.13415 seconds: | 20260506 17:26:15.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:26:15.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:15.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:15.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:15.714 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:26:15.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:15.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:15.714 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 17:26:15.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:15.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:15.715 | INFO | Uploaded 36GiB to GPU in 2.06768s (17.4108GiB/s) | 20260506 17:26:15.715 | INFO | Downloaded 34GiB to GPU in 1.94794s (17.4543GiB/s) | 20260506 17:26:20.851 | INFO | Analyzing chunk frames = [3136512,3137024) | 20260506 17:26:20.851 | INFO | MemInfo: { | 20260506 17:26:20.851 | INFO | "gpu" : " 17 GB", | 20260506 17:26:20.851 | INFO | "huge" : "", | 20260506 17:26:20.851 | INFO | "huge_shared" : " 27 GB", | 20260506 17:26:20.851 | INFO | "malloc" : " 14 GB", | 20260506 17:26:20.851 | INFO | "pinned" : " 29 GB" | 20260506 17:26:20.851 | INFO | } | 20260506 17:26:20.851 | INFO | Perf report for the last 5.13726 seconds: | 20260506 17:26:20.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:26:20.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:20.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:20.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:20.851 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 17:26:20.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:20.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:20.851 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 17:26:20.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:20.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:20.852 | INFO | Uploaded 36GiB to GPU in 2.01206s (17.8921GiB/s) | 20260506 17:26:20.852 | INFO | Downloaded 34GiB to GPU in 1.98179s (17.1562GiB/s) | 20260506 17:26:25.990 | INFO | Analyzing chunk frames = [3137024,3137536) | 20260506 17:26:25.990 | INFO | MemInfo: { | 20260506 17:26:25.990 | INFO | "gpu" : " 17 GB", | 20260506 17:26:25.990 | INFO | "huge" : "", | 20260506 17:26:25.990 | INFO | "huge_shared" : " 27 GB", | 20260506 17:26:25.990 | INFO | "malloc" : " 14 GB", | 20260506 17:26:25.990 | INFO | "pinned" : " 29 GB" | 20260506 17:26:25.990 | INFO | } | 20260506 17:26:25.990 | INFO | Perf report for the last 5.13844 seconds: | 20260506 17:26:25.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:26:25.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:25.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:25.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:25.990 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:26:25.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:25.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:25.990 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:26:25.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:25.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:25.990 | INFO | Uploaded 36GiB to GPU in 2.01478s (17.868GiB/s) | 20260506 17:26:25.990 | INFO | Downloaded 34GiB to GPU in 1.97101s (17.25GiB/s) | 20260506 17:26:25.990 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:26:25.990 | INFO | Total mem usage: 18 GiB | 20260506 17:26:25.990 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:26:25.990 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -11.6%) | 20260506 17:26:25.990 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:26:25.990 | INFO | High Water Breakdown: | 20260506 17:26:25.990 | INFO | 61.9% : 9.98 GiB ( 87.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:26:25.990 | INFO | 38% : 6.13 GiB ( 91.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:26:25.990 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:26:25.990 | INFO | ------------------------------------------ | 20260506 17:26:25.990 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:26:25.990 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:26:25.991 | INFO | Total mem usage: 26.5 GiB | 20260506 17:26:25.991 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:26:25.991 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:26:25.991 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:26:25.991 | INFO | High Water Breakdown: | 20260506 17:26:25.991 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:26:25.991 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:26:25.991 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:26:25.991 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:26:25.991 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:26:25.991 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:26:25.991 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:26:25.991 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:26:25.991 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:26:25.991 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:26:25.991 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:26:25.991 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:26:25.991 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:26:25.991 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:26:25.991 | INFO | ------------------------------------------ | 20260506 17:26:25.991 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:26:25.991 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:26:25.991 | INFO | Total mem usage: 25.6 GiB | 20260506 17:26:25.991 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:26:25.991 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:26:25.991 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:26:25.991 | INFO | High Water Breakdown: | 20260506 17:26:25.991 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:26:25.991 | INFO | ------------------------------------------ | 20260506 17:26:25.991 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:26:25.991 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:26:25.991 | INFO | Total mem usage: 15.7 GiB | 20260506 17:26:25.991 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:26:25.991 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:26:25.991 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:26:25.991 | INFO | High Water Breakdown: | 20260506 17:26:25.991 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:26:25.991 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:26:25.991 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:26:25.991 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:26:25.991 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:26:25.991 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:26:25.991 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:26:25.991 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:26:25.991 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:26:25.991 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:26:25.991 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:26:25.991 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:26:25.991 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:26:25.991 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:26:25.991 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:26:25.991 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:26:25.991 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:26:25.991 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:26:25.991 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:26:25.991 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:26:25.991 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:26:25.991 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:26:25.991 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:26:25.991 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:26:25.991 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:26:25.991 | INFO | ------------------------------------------ | 20260506 17:26:25.991 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:26:25.991 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:26:25.991 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:26:25.991 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:26:25.991 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.57e+01, 1.34e+01 / 1.86e+00 | 20260506 17:26:25.991 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 1.01e-02 / 6.31e-03 | 20260506 17:26:25.991 | INFO | PerfReport: 11.61%, Baseline -- 3.64e+00 / 4.47e+00, 5.61e-01 / 2.92e-01 | 20260506 17:26:25.991 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.48e+00, 1.86e-01 / 1.07e-01 | 20260506 17:26:25.991 | INFO | PerfReport: 15.94%, DME -- 4.00e+01 / 4.44e+01, 3.83e+00 / 2.60e+00 | 20260506 17:26:25.991 | INFO | PerfReport: 56.96%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.42e+00 / 2.09e+00 | 20260506 17:26:25.991 | INFO | PerfReport: 7.29%, PulseAccumulating -- 2.28e+00 / 2.85e+00, 3.89e-01 / 2.12e-01 | 20260506 17:26:25.991 | INFO | PerfReport: 3.45%, Metrics -- 1.08e+00 / 1.43e+00, 2.80e-01 / 1.74e-01 | 20260506 17:26:25.991 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.74e-01 / 4.28e-01, 5.84e-02 / 2.84e-02 | 20260506 17:26:25.991 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:26:25.991 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:26:25.991 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:26:25.991 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 17:26:25.991 | INFO | PerfReport: 21.53%, Upload -- 2.10e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260506 17:26:25.991 | INFO | PerfReport: 21.12%, Download -- 2.06e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 17:26:25.991 | INFO | PerfReport: 32.19%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:26:25.991 | INFO | PerfReport: 1.52%, LockUpload -- 1.48e+00 / 0.00e+00, 5.85e+00 / 0.00e+00 | 20260506 17:26:25.991 | INFO | PerfReport: 1.04%, LockDownload -- 1.01e+00 / 0.00e+00, 2.58e+00 / 0.00e+00 | 20260506 17:26:25.991 | INFO | PerfReport: 22.59%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 17:26:25.991 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.28e-02 / 0.00e+00 | 20260506 17:26:26.104 | INFO | Starting a round of Baz IO | 20260506 17:26:29.239 | INFO | Populating RTMetrics report | 20260506 17:26:29.239 | INFO | RTMetrics report populated | 20260506 17:26:29.239 | INFO | Opening tmp RTMetrics json file | 20260506 17:26:29.325 | INFO | Serializing RTMetrics to json | 20260506 17:26:29.333 | INFO | Serialization done, swapping tmp file now... | 20260506 17:26:29.338 | INFO | RTMetrics done | 20260506 17:26:29.504 | INFO | Finished a round of Baz IO: Wrote 2.067 GiB in 3.4 seconds | 20260506 17:26:29.504 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.65% / 0.01% / 64.35% | 20260506 17:26:29.504 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.38067% / 1.19209e-05% / 7.54992e-05% | 20260506 17:26:29.505 | INFO | Baz IO overall: | 20260506 17:26:29.505 | INFO | BytesWritten : 1653852853941 | 20260506 17:26:29.505 | INFO | NumEvents : 800102189616 | 20260506 17:26:31.131 | INFO | Analyzing chunk frames = [3137536,3138048) | 20260506 17:26:31.131 | INFO | MemInfo: { | 20260506 17:26:31.131 | INFO | "gpu" : " 17 GB", | 20260506 17:26:31.131 | INFO | "huge" : "", | 20260506 17:26:31.131 | INFO | "huge_shared" : " 27 GB", | 20260506 17:26:31.131 | INFO | "malloc" : " 14 GB", | 20260506 17:26:31.131 | INFO | "pinned" : " 29 GB" | 20260506 17:26:31.131 | INFO | } | 20260506 17:26:31.131 | INFO | Perf report for the last 5.14137 seconds: | 20260506 17:26:31.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:26:31.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:31.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:31.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:31.131 | INFO | ANALYSIS : 62.6%, 2.9, 95.0ms, 0 | 20260506 17:26:31.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:31.131 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 17:26:31.131 | INFO | PRE_HQ : 60.5%, 1.9, 58.9ms, 0 | 20260506 17:26:31.131 | INFO | BAZWRITER : 65.8%, 4.8, 161.5ms, 0 | 20260506 17:26:31.132 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 17:26:31.132 | INFO | Uploaded 36GiB to GPU in 2.05009s (17.5602GiB/s) | 20260506 17:26:31.132 | INFO | Downloaded 38GiB to GPU in 2.20338s (17.2462GiB/s) | 20260506 17:26:36.277 | INFO | Analyzing chunk frames = [3138048,3138560) | 20260506 17:26:36.277 | INFO | MemInfo: { | 20260506 17:26:36.277 | INFO | "gpu" : " 17 GB", | 20260506 17:26:36.277 | INFO | "huge" : "", | 20260506 17:26:36.277 | INFO | "huge_shared" : " 27 GB", | 20260506 17:26:36.277 | INFO | "malloc" : " 14 GB", | 20260506 17:26:36.277 | INFO | "pinned" : " 29 GB" | 20260506 17:26:36.277 | INFO | } | 20260506 17:26:36.277 | INFO | Perf report for the last 5.146 seconds: | 20260506 17:26:36.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:26:36.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:36.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:36.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:36.277 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 17:26:36.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:36.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:36.277 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260506 17:26:36.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:36.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:36.277 | INFO | Uploaded 36GiB to GPU in 2.00616s (17.9447GiB/s) | 20260506 17:26:36.278 | INFO | Downloaded 34GiB to GPU in 1.93774s (17.5462GiB/s) | 20260506 17:26:41.418 | INFO | Analyzing chunk frames = [3138560,3139072) | 20260506 17:26:41.418 | INFO | MemInfo: { | 20260506 17:26:41.418 | INFO | "gpu" : " 17 GB", | 20260506 17:26:41.418 | INFO | "huge" : "", | 20260506 17:26:41.418 | INFO | "huge_shared" : " 27 GB", | 20260506 17:26:41.418 | INFO | "malloc" : " 14 GB", | 20260506 17:26:41.418 | INFO | "pinned" : " 29 GB" | 20260506 17:26:41.418 | INFO | } | 20260506 17:26:41.418 | INFO | Perf report for the last 5.14109 seconds: | 20260506 17:26:41.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:26:41.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:41.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:41.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:41.418 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 17:26:41.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:41.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:41.418 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:26:41.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:41.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:41.419 | INFO | Uploaded 36GiB to GPU in 1.97097s (18.2651GiB/s) | 20260506 17:26:41.419 | INFO | Downloaded 34GiB to GPU in 1.92161s (17.6935GiB/s) | 20260506 17:26:46.559 | INFO | Analyzing chunk frames = [3139072,3139584) | 20260506 17:26:46.559 | INFO | MemInfo: { | 20260506 17:26:46.559 | INFO | "gpu" : " 17 GB", | 20260506 17:26:46.559 | INFO | "huge" : "", | 20260506 17:26:46.559 | INFO | "huge_shared" : " 27 GB", | 20260506 17:26:46.559 | INFO | "malloc" : " 14 GB", | 20260506 17:26:46.559 | INFO | "pinned" : " 29 GB" | 20260506 17:26:46.559 | INFO | } | 20260506 17:26:46.559 | INFO | Perf report for the last 5.14099 seconds: | 20260506 17:26:46.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:26:46.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:46.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:46.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:46.559 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 17:26:46.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:46.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:46.559 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:26:46.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:46.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:46.560 | INFO | Uploaded 36GiB to GPU in 1.96607s (18.3106GiB/s) | 20260506 17:26:46.560 | INFO | Downloaded 34GiB to GPU in 1.87141s (18.1681GiB/s) | 20260506 17:26:51.695 | INFO | Analyzing chunk frames = [3139584,3140096) | 20260506 17:26:51.695 | INFO | MemInfo: { | 20260506 17:26:51.695 | INFO | "gpu" : " 17 GB", | 20260506 17:26:51.695 | INFO | "huge" : "", | 20260506 17:26:51.695 | INFO | "huge_shared" : " 27 GB", | 20260506 17:26:51.695 | INFO | "malloc" : " 14 GB", | 20260506 17:26:51.695 | INFO | "pinned" : " 29 GB" | 20260506 17:26:51.695 | INFO | } | 20260506 17:26:51.695 | INFO | Perf report for the last 5.13558 seconds: | 20260506 17:26:51.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:26:51.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:51.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:51.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:51.695 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260506 17:26:51.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:51.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:51.695 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:26:51.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:51.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:51.695 | INFO | Uploaded 36GiB to GPU in 2.01102s (17.9013GiB/s) | 20260506 17:26:51.695 | INFO | Downloaded 34GiB to GPU in 1.91327s (17.7706GiB/s) | 20260506 17:26:56.830 | INFO | Analyzing chunk frames = [3140096,3140608) | 20260506 17:26:56.830 | INFO | MemInfo: { | 20260506 17:26:56.830 | INFO | "gpu" : " 17 GB", | 20260506 17:26:56.830 | INFO | "huge" : "", | 20260506 17:26:56.830 | INFO | "huge_shared" : " 27 GB", | 20260506 17:26:56.830 | INFO | "malloc" : " 14 GB", | 20260506 17:26:56.830 | INFO | "pinned" : " 29 GB" | 20260506 17:26:56.830 | INFO | } | 20260506 17:26:56.830 | INFO | Perf report for the last 5.13501 seconds: | 20260506 17:26:56.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:26:56.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:56.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:56.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:56.830 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 17:26:56.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:56.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:56.830 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 17:26:56.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:56.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:26:56.830 | INFO | Uploaded 36GiB to GPU in 2.02862s (17.7461GiB/s) | 20260506 17:26:56.830 | INFO | Downloaded 34GiB to GPU in 1.95437s (17.3969GiB/s) | 20260506 17:27:01.969 | INFO | Analyzing chunk frames = [3140608,3141120) | 20260506 17:27:01.969 | INFO | MemInfo: { | 20260506 17:27:01.969 | INFO | "gpu" : " 17 GB", | 20260506 17:27:01.969 | INFO | "huge" : "", | 20260506 17:27:01.969 | INFO | "huge_shared" : " 27 GB", | 20260506 17:27:01.969 | INFO | "malloc" : " 14 GB", | 20260506 17:27:01.969 | INFO | "pinned" : " 29 GB" | 20260506 17:27:01.969 | INFO | } | 20260506 17:27:01.969 | INFO | Perf report for the last 5.13893 seconds: | 20260506 17:27:01.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:27:01.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:01.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:01.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:01.969 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 17:27:01.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:01.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:01.969 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:27:01.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:01.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:01.969 | INFO | Uploaded 36GiB to GPU in 2.01902s (17.8305GiB/s) | 20260506 17:27:01.969 | INFO | Downloaded 34GiB to GPU in 2.00416s (16.9647GiB/s) | 20260506 17:27:07.110 | INFO | Analyzing chunk frames = [3141120,3141632) | 20260506 17:27:07.110 | INFO | MemInfo: { | 20260506 17:27:07.110 | INFO | "gpu" : " 17 GB", | 20260506 17:27:07.110 | INFO | "huge" : "", | 20260506 17:27:07.110 | INFO | "huge_shared" : " 27 GB", | 20260506 17:27:07.110 | INFO | "malloc" : " 14 GB", | 20260506 17:27:07.110 | INFO | "pinned" : " 29 GB" | 20260506 17:27:07.110 | INFO | } | 20260506 17:27:07.110 | INFO | Perf report for the last 5.14119 seconds: | 20260506 17:27:07.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:27:07.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:07.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:07.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:07.110 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:27:07.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:07.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:07.110 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:27:07.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:07.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:07.110 | INFO | Uploaded 36GiB to GPU in 1.99925s (18.0068GiB/s) | 20260506 17:27:07.110 | INFO | Downloaded 34GiB to GPU in 1.97652s (17.202GiB/s) | 20260506 17:27:07.225 | INFO | Starting a round of Baz IO | 20260506 17:27:10.355 | INFO | Populating RTMetrics report | 20260506 17:27:10.356 | INFO | RTMetrics report populated | 20260506 17:27:10.356 | INFO | Opening tmp RTMetrics json file | 20260506 17:27:10.357 | INFO | Serializing RTMetrics to json | 20260506 17:27:10.359 | INFO | Serialization done, swapping tmp file now... | 20260506 17:27:10.360 | INFO | RTMetrics done | 20260506 17:27:10.739 | INFO | Finished a round of Baz IO: Wrote 2.067 GiB in 3.514 seconds | 20260506 17:27:10.739 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.64% / 0.01% / 64.35% | 20260506 17:27:10.739 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.37788% / 1.58946e-05% / 5.5631e-05% | 20260506 17:27:10.739 | INFO | Baz IO overall: | 20260506 17:27:10.739 | INFO | BytesWritten : 1656071906575 | 20260506 17:27:10.739 | INFO | NumEvents : 801173306494 | 20260506 17:27:12.252 | INFO | Analyzing chunk frames = [3141632,3142144) | 20260506 17:27:12.252 | INFO | MemInfo: { | 20260506 17:27:12.252 | INFO | "gpu" : " 17 GB", | 20260506 17:27:12.252 | INFO | "huge" : "", | 20260506 17:27:12.252 | INFO | "huge_shared" : " 27 GB", | 20260506 17:27:12.252 | INFO | "malloc" : " 14 GB", | 20260506 17:27:12.252 | INFO | "pinned" : " 29 GB" | 20260506 17:27:12.252 | INFO | } | 20260506 17:27:12.252 | INFO | Perf report for the last 5.14212 seconds: | 20260506 17:27:12.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:27:12.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:12.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:12.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:12.252 | INFO | ANALYSIS : 62.5%, 2.9, 95.2ms, 0 | 20260506 17:27:12.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:12.252 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260506 17:27:12.252 | INFO | PRE_HQ : 60.2%, 1.9, 58.1ms, 0 | 20260506 17:27:12.252 | INFO | BAZWRITER : 68.0%, 4.5, 155.8ms, 0 | 20260506 17:27:12.252 | INFO | BAZ_SAMPLING : 6.4%, 1.0, 3.3ms, 0 | 20260506 17:27:12.252 | INFO | Uploaded 36GiB to GPU in 2.03618s (17.6802GiB/s) | 20260506 17:27:12.253 | INFO | Downloaded 38GiB to GPU in 2.20388s (17.2423GiB/s) | 20260506 17:27:17.394 | INFO | Analyzing chunk frames = [3142144,3142656) | 20260506 17:27:17.394 | INFO | MemInfo: { | 20260506 17:27:17.394 | INFO | "gpu" : " 17 GB", | 20260506 17:27:17.394 | INFO | "huge" : "", | 20260506 17:27:17.394 | INFO | "huge_shared" : " 27 GB", | 20260506 17:27:17.394 | INFO | "malloc" : " 14 GB", | 20260506 17:27:17.394 | INFO | "pinned" : " 29 GB" | 20260506 17:27:17.394 | INFO | } | 20260506 17:27:17.394 | INFO | Perf report for the last 5.14175 seconds: | 20260506 17:27:17.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:27:17.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:17.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:17.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:17.394 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 17:27:17.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:17.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:17.394 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:27:17.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:17.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:17.394 | INFO | Uploaded 36GiB to GPU in 1.9991s (18.0081GiB/s) | 20260506 17:27:17.394 | INFO | Downloaded 34GiB to GPU in 1.92684s (17.6455GiB/s) | 20260506 17:27:22.528 | INFO | Analyzing chunk frames = [3142656,3143168) | 20260506 17:27:22.528 | INFO | MemInfo: { | 20260506 17:27:22.528 | INFO | "gpu" : " 17 GB", | 20260506 17:27:22.528 | INFO | "huge" : "", | 20260506 17:27:22.528 | INFO | "huge_shared" : " 27 GB", | 20260506 17:27:22.528 | INFO | "malloc" : " 14 GB", | 20260506 17:27:22.528 | INFO | "pinned" : " 29 GB" | 20260506 17:27:22.528 | INFO | } | 20260506 17:27:22.528 | INFO | Perf report for the last 5.13415 seconds: | 20260506 17:27:22.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:27:22.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:22.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:22.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:22.528 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:27:22.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:22.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:22.528 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:27:22.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:22.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:22.528 | INFO | Uploaded 36GiB to GPU in 1.9851s (18.1351GiB/s) | 20260506 17:27:22.528 | INFO | Downloaded 34GiB to GPU in 1.92s (17.7083GiB/s) | 20260506 17:27:27.668 | INFO | Analyzing chunk frames = [3143168,3143680) | 20260506 17:27:27.668 | INFO | MemInfo: { | 20260506 17:27:27.668 | INFO | "gpu" : " 17 GB", | 20260506 17:27:27.668 | INFO | "huge" : "", | 20260506 17:27:27.668 | INFO | "huge_shared" : " 27 GB", | 20260506 17:27:27.668 | INFO | "malloc" : " 14 GB", | 20260506 17:27:27.668 | INFO | "pinned" : " 29 GB" | 20260506 17:27:27.668 | INFO | } | 20260506 17:27:27.668 | INFO | Perf report for the last 5.13966 seconds: | 20260506 17:27:27.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:27:27.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:27.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:27.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:27.668 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 17:27:27.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:27.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:27.668 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:27:27.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:27.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:27.668 | INFO | Uploaded 36GiB to GPU in 1.92965s (18.6563GiB/s) | 20260506 17:27:27.668 | INFO | Downloaded 34GiB to GPU in 1.87655s (18.1184GiB/s) | 20260506 17:27:32.806 | INFO | Analyzing chunk frames = [3143680,3144192) | 20260506 17:27:32.806 | INFO | MemInfo: { | 20260506 17:27:32.806 | INFO | "gpu" : " 17 GB", | 20260506 17:27:32.806 | INFO | "huge" : "", | 20260506 17:27:32.806 | INFO | "huge_shared" : " 27 GB", | 20260506 17:27:32.806 | INFO | "malloc" : " 14 GB", | 20260506 17:27:32.806 | INFO | "pinned" : " 29 GB" | 20260506 17:27:32.806 | INFO | } | 20260506 17:27:32.806 | INFO | Perf report for the last 5.13796 seconds: | 20260506 17:27:32.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:27:32.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:32.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:32.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:32.806 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260506 17:27:32.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:32.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:32.806 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:27:32.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:32.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:32.806 | INFO | Uploaded 36GiB to GPU in 1.97245s (18.2514GiB/s) | 20260506 17:27:32.806 | INFO | Downloaded 34GiB to GPU in 1.92336s (17.6774GiB/s) | 20260506 17:27:37.945 | INFO | Analyzing chunk frames = [3144192,3144704) | 20260506 17:27:37.945 | INFO | MemInfo: { | 20260506 17:27:37.945 | INFO | "gpu" : " 17 GB", | 20260506 17:27:37.945 | INFO | "huge" : "", | 20260506 17:27:37.945 | INFO | "huge_shared" : " 27 GB", | 20260506 17:27:37.945 | INFO | "malloc" : " 14 GB", | 20260506 17:27:37.945 | INFO | "pinned" : " 29 GB" | 20260506 17:27:37.945 | INFO | } | 20260506 17:27:37.945 | INFO | Perf report for the last 5.13936 seconds: | 20260506 17:27:37.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:27:37.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:37.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:37.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:37.945 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:27:37.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:37.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:37.945 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260506 17:27:37.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:37.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:37.945 | INFO | Uploaded 36GiB to GPU in 2.05286s (17.5365GiB/s) | 20260506 17:27:37.946 | INFO | Downloaded 34GiB to GPU in 1.95994s (17.3475GiB/s) | 20260506 17:27:43.082 | INFO | Analyzing chunk frames = [3144704,3145216) | 20260506 17:27:43.082 | INFO | MemInfo: { | 20260506 17:27:43.082 | INFO | "gpu" : " 17 GB", | 20260506 17:27:43.082 | INFO | "huge" : "", | 20260506 17:27:43.082 | INFO | "huge_shared" : " 27 GB", | 20260506 17:27:43.082 | INFO | "malloc" : " 14 GB", | 20260506 17:27:43.082 | INFO | "pinned" : " 29 GB" | 20260506 17:27:43.082 | INFO | } | 20260506 17:27:43.082 | INFO | Perf report for the last 5.13731 seconds: | 20260506 17:27:43.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:27:43.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:43.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:43.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:43.083 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 17:27:43.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:43.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:43.083 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:27:43.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:43.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:43.083 | INFO | Uploaded 36GiB to GPU in 1.99739s (18.0235GiB/s) | 20260506 17:27:43.083 | INFO | Downloaded 34GiB to GPU in 1.96625s (17.2918GiB/s) | 20260506 17:27:48.222 | INFO | Analyzing chunk frames = [3145216,3145728) | 20260506 17:27:48.222 | INFO | MemInfo: { | 20260506 17:27:48.222 | INFO | "gpu" : " 17 GB", | 20260506 17:27:48.222 | INFO | "huge" : "", | 20260506 17:27:48.222 | INFO | "huge_shared" : " 27 GB", | 20260506 17:27:48.222 | INFO | "malloc" : " 14 GB", | 20260506 17:27:48.222 | INFO | "pinned" : " 29 GB" | 20260506 17:27:48.222 | INFO | } | 20260506 17:27:48.222 | INFO | Perf report for the last 5.13948 seconds: | 20260506 17:27:48.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:27:48.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:48.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:48.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:48.222 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 17:27:48.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:48.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:48.222 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:27:48.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:48.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:48.222 | INFO | Uploaded 36GiB to GPU in 2.02954s (17.738GiB/s) | 20260506 17:27:48.222 | INFO | Downloaded 34GiB to GPU in 1.98702s (17.1111GiB/s) | 20260506 17:27:48.222 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:27:48.222 | INFO | Total mem usage: 18 GiB | 20260506 17:27:48.222 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:27:48.222 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.3%) | 20260506 17:27:48.222 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:27:48.222 | INFO | High Water Breakdown: | 20260506 17:27:48.222 | INFO | 61.8% : 9.9 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:27:48.222 | INFO | 38.1% : 6.11 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:27:48.222 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:27:48.223 | INFO | ------------------------------------------ | 20260506 17:27:48.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:27:48.223 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:27:48.223 | INFO | Total mem usage: 26.5 GiB | 20260506 17:27:48.223 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:27:48.223 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:27:48.223 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:27:48.223 | INFO | High Water Breakdown: | 20260506 17:27:48.223 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:27:48.223 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:27:48.223 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:27:48.223 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:27:48.223 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:27:48.223 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:27:48.223 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:27:48.223 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:27:48.223 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:27:48.223 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:27:48.223 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:27:48.223 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:27:48.223 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:27:48.223 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:27:48.223 | INFO | ------------------------------------------ | 20260506 17:27:48.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:27:48.223 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:27:48.223 | INFO | Total mem usage: 25.6 GiB | 20260506 17:27:48.223 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:27:48.223 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:27:48.223 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:27:48.223 | INFO | High Water Breakdown: | 20260506 17:27:48.223 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:27:48.223 | INFO | ------------------------------------------ | 20260506 17:27:48.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:27:48.223 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:27:48.223 | INFO | Total mem usage: 15.7 GiB | 20260506 17:27:48.223 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:27:48.223 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:27:48.223 | INFO | Unused Cache: 862 MiB (Waste: 5.37%) | 20260506 17:27:48.223 | INFO | High Water Breakdown: | 20260506 17:27:48.223 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:27:48.223 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:27:48.223 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:27:48.223 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:27:48.223 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:27:48.223 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:27:48.223 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:27:48.223 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:27:48.223 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:27:48.223 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:27:48.223 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:27:48.223 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:27:48.223 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:27:48.223 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:27:48.223 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:27:48.223 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:27:48.223 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:27:48.223 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:27:48.223 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:27:48.223 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:27:48.223 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:27:48.223 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:27:48.223 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:27:48.223 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:27:48.223 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:27:48.223 | INFO | ------------------------------------------ | 20260506 17:27:48.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:27:48.224 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:27:48.224 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:27:48.224 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:27:48.224 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.53e+01, 1.34e+01 / 2.04e+00 | 20260506 17:27:48.224 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.00e-01, 1.03e-02 / 5.98e-03 | 20260506 17:27:48.224 | INFO | PerfReport: 11.62%, Baseline -- 3.64e+00 / 4.43e+00, 5.61e-01 / 3.16e-01 | 20260506 17:27:48.224 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.46e+00, 1.86e-01 / 1.30e-01 | 20260506 17:27:48.224 | INFO | PerfReport: 15.94%, DME -- 3.99e+01 / 4.44e+01, 3.92e+00 / 2.36e+00 | 20260506 17:27:48.224 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.41e+00 / 1.96e+00 | 20260506 17:27:48.224 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.28e+00 / 2.84e+00, 3.85e-01 / 2.53e-01 | 20260506 17:27:48.224 | INFO | PerfReport: 3.45%, Metrics -- 1.08e+00 / 1.45e+00, 2.82e-01 / 1.51e-01 | 20260506 17:27:48.224 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.74e-01 / 4.32e-01, 5.89e-02 / 2.60e-02 | 20260506 17:27:48.224 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:27:48.224 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:27:48.224 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:27:48.224 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 17:27:48.224 | INFO | PerfReport: 21.47%, Upload -- 2.09e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260506 17:27:48.224 | INFO | PerfReport: 21.14%, Download -- 2.05e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260506 17:27:48.224 | INFO | PerfReport: 32.24%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:27:48.224 | INFO | PerfReport: 1.41%, LockUpload -- 1.37e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260506 17:27:48.224 | INFO | PerfReport: 1.11%, LockDownload -- 1.08e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260506 17:27:48.224 | INFO | PerfReport: 22.62%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 17:27:48.224 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260506 17:27:48.342 | INFO | Starting a round of Baz IO | 20260506 17:27:51.484 | INFO | Populating RTMetrics report | 20260506 17:27:51.484 | INFO | RTMetrics report populated | 20260506 17:27:51.484 | INFO | Opening tmp RTMetrics json file | 20260506 17:27:51.485 | INFO | Serializing RTMetrics to json | 20260506 17:27:51.487 | INFO | Serialization done, swapping tmp file now... | 20260506 17:27:51.488 | INFO | RTMetrics done | 20260506 17:27:51.667 | INFO | Finished a round of Baz IO: Wrote 2.064 GiB in 3.325 seconds | 20260506 17:27:51.667 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.63% / 0.01% / 64.36% | 20260506 17:27:51.667 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.37079% / 3.57628e-05% / 7.54992e-05% | 20260506 17:27:51.667 | INFO | Baz IO overall: | 20260506 17:27:51.667 | INFO | BytesWritten : 1658288557847 | 20260506 17:27:51.667 | INFO | NumEvents : 802242407177 | 20260506 17:27:53.372 | INFO | Analyzing chunk frames = [3145728,3146240) | 20260506 17:27:53.372 | INFO | MemInfo: { | 20260506 17:27:53.372 | INFO | "gpu" : " 17 GB", | 20260506 17:27:53.372 | INFO | "huge" : "", | 20260506 17:27:53.372 | INFO | "huge_shared" : " 27 GB", | 20260506 17:27:53.372 | INFO | "malloc" : " 14 GB", | 20260506 17:27:53.372 | INFO | "pinned" : " 29 GB" | 20260506 17:27:53.373 | INFO | } | 20260506 17:27:53.373 | INFO | Perf report for the last 5.15056 seconds: | 20260506 17:27:53.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:27:53.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:53.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:53.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:53.373 | INFO | ANALYSIS : 62.8%, 2.9, 95.4ms, 0 | 20260506 17:27:53.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:53.373 | INFO | RT_METRICS : 43.5%, 1.0, 22.9ms, 0 | 20260506 17:27:53.373 | INFO | PRE_HQ : 60.1%, 1.9, 58.5ms, 0 | 20260506 17:27:53.373 | INFO | BAZWRITER : 64.0%, 4.6, 153.1ms, 0 | 20260506 17:27:53.373 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 17:27:53.373 | INFO | Uploaded 36GiB to GPU in 2.0442s (17.6108GiB/s) | 20260506 17:27:53.373 | INFO | Downloaded 38GiB to GPU in 2.22205s (17.1013GiB/s) | 20260506 17:27:58.508 | INFO | Analyzing chunk frames = [3146240,3146752) | 20260506 17:27:58.508 | INFO | MemInfo: { | 20260506 17:27:58.508 | INFO | "gpu" : " 17 GB", | 20260506 17:27:58.508 | INFO | "huge" : "", | 20260506 17:27:58.508 | INFO | "huge_shared" : " 27 GB", | 20260506 17:27:58.508 | INFO | "malloc" : " 14 GB", | 20260506 17:27:58.508 | INFO | "pinned" : " 29 GB" | 20260506 17:27:58.508 | INFO | } | 20260506 17:27:58.508 | INFO | Perf report for the last 5.13554 seconds: | 20260506 17:27:58.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:27:58.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:58.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:58.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:58.508 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:27:58.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:58.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:58.508 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:27:58.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:58.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:27:58.508 | INFO | Uploaded 36GiB to GPU in 2.00815s (17.9269GiB/s) | 20260506 17:27:58.508 | INFO | Downloaded 34GiB to GPU in 1.93047s (17.6123GiB/s) | 20260506 17:28:03.648 | INFO | Analyzing chunk frames = [3146752,3147264) | 20260506 17:28:03.648 | INFO | MemInfo: { | 20260506 17:28:03.648 | INFO | "gpu" : " 17 GB", | 20260506 17:28:03.648 | INFO | "huge" : "", | 20260506 17:28:03.648 | INFO | "huge_shared" : " 27 GB", | 20260506 17:28:03.648 | INFO | "malloc" : " 14 GB", | 20260506 17:28:03.648 | INFO | "pinned" : " 29 GB" | 20260506 17:28:03.648 | INFO | } | 20260506 17:28:03.649 | INFO | Perf report for the last 5.14045 seconds: | 20260506 17:28:03.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:28:03.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:03.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:03.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:03.649 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:28:03.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:03.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:03.649 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:28:03.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:03.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:03.649 | INFO | Uploaded 36GiB to GPU in 1.97806s (18.1996GiB/s) | 20260506 17:28:03.649 | INFO | Downloaded 34GiB to GPU in 1.92269s (17.6835GiB/s) | 20260506 17:28:08.782 | INFO | Analyzing chunk frames = [3147264,3147776) | 20260506 17:28:08.782 | INFO | MemInfo: { | 20260506 17:28:08.782 | INFO | "gpu" : " 17 GB", | 20260506 17:28:08.782 | INFO | "huge" : "", | 20260506 17:28:08.782 | INFO | "huge_shared" : " 27 GB", | 20260506 17:28:08.782 | INFO | "malloc" : " 14 GB", | 20260506 17:28:08.782 | INFO | "pinned" : " 29 GB" | 20260506 17:28:08.782 | INFO | } | 20260506 17:28:08.782 | INFO | Perf report for the last 5.13383 seconds: | 20260506 17:28:08.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:28:08.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:08.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:08.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:08.782 | INFO | ANALYSIS : 60.1%, 3.0, 93.0ms, 0 | 20260506 17:28:08.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:08.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:08.782 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:28:08.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:08.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:08.783 | INFO | Uploaded 36GiB to GPU in 1.95767s (18.3892GiB/s) | 20260506 17:28:08.783 | INFO | Downloaded 34GiB to GPU in 1.88759s (18.0124GiB/s) | 20260506 17:28:13.919 | INFO | Analyzing chunk frames = [3147776,3148288) | 20260506 17:28:13.919 | INFO | MemInfo: { | 20260506 17:28:13.919 | INFO | "gpu" : " 17 GB", | 20260506 17:28:13.919 | INFO | "huge" : "", | 20260506 17:28:13.919 | INFO | "huge_shared" : " 27 GB", | 20260506 17:28:13.919 | INFO | "malloc" : " 14 GB", | 20260506 17:28:13.919 | INFO | "pinned" : " 29 GB" | 20260506 17:28:13.919 | INFO | } | 20260506 17:28:13.919 | INFO | Perf report for the last 5.13655 seconds: | 20260506 17:28:13.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:28:13.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:13.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:13.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:13.919 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 17:28:13.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:13.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:13.919 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:28:13.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:13.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:13.919 | INFO | Uploaded 36GiB to GPU in 2.00569s (17.9489GiB/s) | 20260506 17:28:13.919 | INFO | Downloaded 34GiB to GPU in 1.90286s (17.8678GiB/s) | 20260506 17:28:19.060 | INFO | Analyzing chunk frames = [3148288,3148800) | 20260506 17:28:19.060 | INFO | MemInfo: { | 20260506 17:28:19.060 | INFO | "gpu" : " 17 GB", | 20260506 17:28:19.060 | INFO | "huge" : "", | 20260506 17:28:19.060 | INFO | "huge_shared" : " 27 GB", | 20260506 17:28:19.060 | INFO | "malloc" : " 14 GB", | 20260506 17:28:19.060 | INFO | "pinned" : " 29 GB" | 20260506 17:28:19.060 | INFO | } | 20260506 17:28:19.060 | INFO | Perf report for the last 5.14085 seconds: | 20260506 17:28:19.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:28:19.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:19.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:19.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:19.060 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 17:28:19.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:19.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:19.060 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 17:28:19.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:19.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:19.060 | INFO | Uploaded 36GiB to GPU in 2.03767s (17.6673GiB/s) | 20260506 17:28:19.060 | INFO | Downloaded 34GiB to GPU in 1.9482s (17.452GiB/s) | 20260506 17:28:24.197 | INFO | Analyzing chunk frames = [3148800,3149312) | 20260506 17:28:24.197 | INFO | MemInfo: { | 20260506 17:28:24.197 | INFO | "gpu" : " 17 GB", | 20260506 17:28:24.198 | INFO | "huge" : "", | 20260506 17:28:24.198 | INFO | "huge_shared" : " 27 GB", | 20260506 17:28:24.198 | INFO | "malloc" : " 14 GB", | 20260506 17:28:24.198 | INFO | "pinned" : " 29 GB" | 20260506 17:28:24.198 | INFO | } | 20260506 17:28:24.198 | INFO | Perf report for the last 5.13783 seconds: | 20260506 17:28:24.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:28:24.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:24.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:24.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:24.198 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:28:24.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:24.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:24.198 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:28:24.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:24.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:24.198 | INFO | Uploaded 36GiB to GPU in 2.00062s (17.9944GiB/s) | 20260506 17:28:24.198 | INFO | Downloaded 34GiB to GPU in 2.01718s (16.8552GiB/s) | 20260506 17:28:29.337 | INFO | Analyzing chunk frames = [3149312,3149824) | 20260506 17:28:29.337 | INFO | MemInfo: { | 20260506 17:28:29.337 | INFO | "gpu" : " 17 GB", | 20260506 17:28:29.337 | INFO | "huge" : "", | 20260506 17:28:29.337 | INFO | "huge_shared" : " 27 GB", | 20260506 17:28:29.337 | INFO | "malloc" : " 14 GB", | 20260506 17:28:29.337 | INFO | "pinned" : " 29 GB" | 20260506 17:28:29.337 | INFO | } | 20260506 17:28:29.337 | INFO | Perf report for the last 5.13963 seconds: | 20260506 17:28:29.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:28:29.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:29.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:29.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:29.337 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:28:29.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:29.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:29.337 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 17:28:29.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:29.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:29.338 | INFO | Uploaded 36GiB to GPU in 2.00775s (17.9305GiB/s) | 20260506 17:28:29.338 | INFO | Downloaded 34GiB to GPU in 1.96792s (17.2771GiB/s) | 20260506 17:28:29.451 | INFO | Starting a round of Baz IO | 20260506 17:28:32.583 | INFO | Populating RTMetrics report | 20260506 17:28:32.583 | INFO | RTMetrics report populated | 20260506 17:28:32.583 | INFO | Opening tmp RTMetrics json file | 20260506 17:28:32.585 | INFO | Serializing RTMetrics to json | 20260506 17:28:32.590 | INFO | Serialization done, swapping tmp file now... | 20260506 17:28:32.596 | INFO | RTMetrics done | 20260506 17:28:32.802 | INFO | Finished a round of Baz IO: Wrote 2.064 GiB in 3.351 seconds | 20260506 17:28:32.802 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.63% / 0.01% / 64.37% | 20260506 17:28:32.802 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.36859% / 3.97364e-06% / 7.15256e-05% | 20260506 17:28:32.802 | INFO | Baz IO overall: | 20260506 17:28:32.802 | INFO | BytesWritten : 1660505204136 | 20260506 17:28:32.802 | INFO | NumEvents : 803311646791 | 20260506 17:28:34.476 | INFO | Analyzing chunk frames = [3149824,3150336) | 20260506 17:28:34.476 | INFO | MemInfo: { | 20260506 17:28:34.476 | INFO | "gpu" : " 17 GB", | 20260506 17:28:34.476 | INFO | "huge" : "", | 20260506 17:28:34.476 | INFO | "huge_shared" : " 27 GB", | 20260506 17:28:34.476 | INFO | "malloc" : " 14 GB", | 20260506 17:28:34.476 | INFO | "pinned" : " 29 GB" | 20260506 17:28:34.476 | INFO | } | 20260506 17:28:34.476 | INFO | Perf report for the last 5.13879 seconds: | 20260506 17:28:34.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:28:34.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:34.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:34.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:34.476 | INFO | ANALYSIS : 62.6%, 2.9, 95.2ms, 0 | 20260506 17:28:34.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:34.476 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 17:28:34.476 | INFO | PRE_HQ : 60.4%, 1.9, 58.6ms, 0 | 20260506 17:28:34.476 | INFO | BAZWRITER : 64.9%, 4.5, 150.1ms, 0 | 20260506 17:28:34.476 | INFO | BAZ_SAMPLING : 3.4%, 1.0, 1.8ms, 0 | 20260506 17:28:34.476 | INFO | Uploaded 36GiB to GPU in 2.03894s (17.6562GiB/s) | 20260506 17:28:34.476 | INFO | Downloaded 38GiB to GPU in 2.19879s (17.2822GiB/s) | 20260506 17:28:39.618 | INFO | Analyzing chunk frames = [3150336,3150848) | 20260506 17:28:39.618 | INFO | MemInfo: { | 20260506 17:28:39.618 | INFO | "gpu" : " 17 GB", | 20260506 17:28:39.618 | INFO | "huge" : "", | 20260506 17:28:39.618 | INFO | "huge_shared" : " 27 GB", | 20260506 17:28:39.618 | INFO | "malloc" : " 14 GB", | 20260506 17:28:39.618 | INFO | "pinned" : " 29 GB" | 20260506 17:28:39.618 | INFO | } | 20260506 17:28:39.619 | INFO | Perf report for the last 5.14243 seconds: | 20260506 17:28:39.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:28:39.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:39.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:39.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:39.619 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 17:28:39.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:39.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:39.619 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:28:39.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:39.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:39.619 | INFO | Uploaded 36GiB to GPU in 2.02273s (17.7977GiB/s) | 20260506 17:28:39.619 | INFO | Downloaded 34GiB to GPU in 1.96949s (17.2633GiB/s) | 20260506 17:28:44.757 | INFO | Analyzing chunk frames = [3150848,3151360) | 20260506 17:28:44.757 | INFO | MemInfo: { | 20260506 17:28:44.757 | INFO | "gpu" : " 17 GB", | 20260506 17:28:44.757 | INFO | "huge" : "", | 20260506 17:28:44.757 | INFO | "huge_shared" : " 27 GB", | 20260506 17:28:44.757 | INFO | "malloc" : " 14 GB", | 20260506 17:28:44.757 | INFO | "pinned" : " 29 GB" | 20260506 17:28:44.757 | INFO | } | 20260506 17:28:44.757 | INFO | Perf report for the last 5.13892 seconds: | 20260506 17:28:44.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:28:44.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:44.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:44.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:44.757 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 17:28:44.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:44.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:44.758 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:28:44.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:44.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:44.758 | INFO | Uploaded 36GiB to GPU in 1.98528s (18.1334GiB/s) | 20260506 17:28:44.758 | INFO | Downloaded 34GiB to GPU in 1.9268s (17.6458GiB/s) | 20260506 17:28:49.899 | INFO | Analyzing chunk frames = [3151360,3151872) | 20260506 17:28:49.899 | INFO | MemInfo: { | 20260506 17:28:49.899 | INFO | "gpu" : " 17 GB", | 20260506 17:28:49.899 | INFO | "huge" : "", | 20260506 17:28:49.899 | INFO | "huge_shared" : " 27 GB", | 20260506 17:28:49.899 | INFO | "malloc" : " 14 GB", | 20260506 17:28:49.899 | INFO | "pinned" : " 29 GB" | 20260506 17:28:49.899 | INFO | } | 20260506 17:28:49.899 | INFO | Perf report for the last 5.14201 seconds: | 20260506 17:28:49.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:28:49.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:49.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:49.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:49.900 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:28:49.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:49.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:49.900 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:28:49.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:49.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:49.900 | INFO | Uploaded 36GiB to GPU in 1.94255s (18.5323GiB/s) | 20260506 17:28:49.900 | INFO | Downloaded 34GiB to GPU in 1.87625s (18.1212GiB/s) | 20260506 17:28:55.037 | INFO | Analyzing chunk frames = [3151872,3152384) | 20260506 17:28:55.037 | INFO | MemInfo: { | 20260506 17:28:55.037 | INFO | "gpu" : " 17 GB", | 20260506 17:28:55.037 | INFO | "huge" : "", | 20260506 17:28:55.037 | INFO | "huge_shared" : " 27 GB", | 20260506 17:28:55.037 | INFO | "malloc" : " 14 GB", | 20260506 17:28:55.037 | INFO | "pinned" : " 29 GB" | 20260506 17:28:55.037 | INFO | } | 20260506 17:28:55.037 | INFO | Perf report for the last 5.13789 seconds: | 20260506 17:28:55.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:28:55.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:55.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:55.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:55.037 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 17:28:55.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:55.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:55.037 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:28:55.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:55.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:28:55.038 | INFO | Uploaded 36GiB to GPU in 1.98528s (18.1335GiB/s) | 20260506 17:28:55.038 | INFO | Downloaded 34GiB to GPU in 1.92604s (17.6528GiB/s) | 20260506 17:29:00.173 | INFO | Analyzing chunk frames = [3152384,3152896) | 20260506 17:29:00.173 | INFO | MemInfo: { | 20260506 17:29:00.173 | INFO | "gpu" : " 17 GB", | 20260506 17:29:00.173 | INFO | "huge" : "", | 20260506 17:29:00.173 | INFO | "huge_shared" : " 27 GB", | 20260506 17:29:00.173 | INFO | "malloc" : " 14 GB", | 20260506 17:29:00.173 | INFO | "pinned" : " 29 GB" | 20260506 17:29:00.173 | INFO | } | 20260506 17:29:00.173 | INFO | Perf report for the last 5.13595 seconds: | 20260506 17:29:00.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:29:00.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:00.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:00.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:00.173 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 17:29:00.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:00.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:00.173 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:29:00.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:00.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:00.174 | INFO | Uploaded 36GiB to GPU in 2.01836s (17.8362GiB/s) | 20260506 17:29:00.174 | INFO | Downloaded 34GiB to GPU in 1.93838s (17.5404GiB/s) | 20260506 17:29:05.314 | INFO | Analyzing chunk frames = [3152896,3153408) | 20260506 17:29:05.315 | INFO | MemInfo: { | 20260506 17:29:05.315 | INFO | "gpu" : " 17 GB", | 20260506 17:29:05.315 | INFO | "huge" : "", | 20260506 17:29:05.315 | INFO | "huge_shared" : " 27 GB", | 20260506 17:29:05.315 | INFO | "malloc" : " 14 GB", | 20260506 17:29:05.315 | INFO | "pinned" : " 29 GB" | 20260506 17:29:05.315 | INFO | } | 20260506 17:29:05.315 | INFO | Perf report for the last 5.14134 seconds: | 20260506 17:29:05.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:29:05.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:05.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:05.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:05.315 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 17:29:05.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:05.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:05.315 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 17:29:05.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:05.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:05.315 | INFO | Uploaded 36GiB to GPU in 2.00935s (17.9163GiB/s) | 20260506 17:29:05.315 | INFO | Downloaded 34GiB to GPU in 2.00974s (16.9177GiB/s) | 20260506 17:29:10.455 | INFO | Analyzing chunk frames = [3153408,3153920) | 20260506 17:29:10.455 | INFO | MemInfo: { | 20260506 17:29:10.455 | INFO | "gpu" : " 17 GB", | 20260506 17:29:10.455 | INFO | "huge" : "", | 20260506 17:29:10.455 | INFO | "huge_shared" : " 27 GB", | 20260506 17:29:10.455 | INFO | "malloc" : " 14 GB", | 20260506 17:29:10.456 | INFO | "pinned" : " 29 GB" | 20260506 17:29:10.456 | INFO | } | 20260506 17:29:10.456 | INFO | Perf report for the last 5.141 seconds: | 20260506 17:29:10.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:29:10.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:10.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:10.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:10.456 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:29:10.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:10.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:10.456 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:29:10.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:10.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:10.456 | INFO | Uploaded 36GiB to GPU in 2.00563s (17.9494GiB/s) | 20260506 17:29:10.456 | INFO | Downloaded 34GiB to GPU in 1.98209s (17.1536GiB/s) | 20260506 17:29:10.456 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:29:10.456 | INFO | Total mem usage: 18 GiB | 20260506 17:29:10.456 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:29:10.456 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -17.1%) | 20260506 17:29:10.456 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:29:10.456 | INFO | High Water Breakdown: | 20260506 17:29:10.456 | INFO | 61.1% : 9.4 GiB ( 82.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:29:10.456 | INFO | 38.7% : 5.95 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:29:10.456 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:29:10.456 | INFO | ------------------------------------------ | 20260506 17:29:10.456 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:29:10.456 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:29:10.456 | INFO | Total mem usage: 26.5 GiB | 20260506 17:29:10.456 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:29:10.456 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:29:10.456 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:29:10.456 | INFO | High Water Breakdown: | 20260506 17:29:10.456 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:29:10.456 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:29:10.456 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:29:10.456 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:29:10.456 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:29:10.456 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:29:10.456 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:29:10.456 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:29:10.456 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:29:10.456 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:29:10.456 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:29:10.456 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:29:10.456 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:29:10.456 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:29:10.456 | INFO | ------------------------------------------ | 20260506 17:29:10.456 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:29:10.456 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:29:10.456 | INFO | Total mem usage: 25.6 GiB | 20260506 17:29:10.456 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:29:10.456 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:29:10.456 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:29:10.456 | INFO | High Water Breakdown: | 20260506 17:29:10.456 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:29:10.456 | INFO | ------------------------------------------ | 20260506 17:29:10.456 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:29:10.456 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:29:10.456 | INFO | Total mem usage: 15.7 GiB | 20260506 17:29:10.456 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:29:10.456 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:29:10.456 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:29:10.456 | INFO | High Water Breakdown: | 20260506 17:29:10.456 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:29:10.456 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:29:10.456 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:29:10.456 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:29:10.456 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:29:10.456 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:29:10.456 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:29:10.456 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:29:10.456 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:29:10.457 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:29:10.457 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:29:10.457 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:29:10.457 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:29:10.457 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:29:10.457 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:29:10.457 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:29:10.457 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:29:10.457 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:29:10.457 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:29:10.457 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:29:10.457 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:29:10.457 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:29:10.457 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:29:10.457 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:29:10.457 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:29:10.457 | INFO | ------------------------------------------ | 20260506 17:29:10.457 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:29:10.457 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:29:10.457 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:29:10.457 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:29:10.457 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.61e+01, 1.34e+01 / 2.91e+00 | 20260506 17:29:10.457 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 9.88e-03 / 6.00e-03 | 20260506 17:29:10.457 | INFO | PerfReport: 11.61%, Baseline -- 3.64e+00 / 4.41e+00, 5.55e-01 / 3.39e-01 | 20260506 17:29:10.457 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.83e-01 / 1.38e-01 | 20260506 17:29:10.457 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.49e+01, 4.25e+00 / 3.28e+00 | 20260506 17:29:10.457 | INFO | PerfReport: 56.87%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.32e+00 / 1.68e+00 | 20260506 17:29:10.457 | INFO | PerfReport: 7.31%, PulseAccumulating -- 2.29e+00 / 2.87e+00, 3.93e-01 / 1.99e-01 | 20260506 17:29:10.457 | INFO | PerfReport: 3.45%, Metrics -- 1.08e+00 / 1.45e+00, 2.83e-01 / 1.68e-01 | 20260506 17:29:10.457 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.74e-01 / 4.21e-01, 5.87e-02 / 2.11e-02 | 20260506 17:29:10.457 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:29:10.457 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:29:10.457 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:29:10.457 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 17:29:10.457 | INFO | PerfReport: 21.46%, Upload -- 2.09e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260506 17:29:10.457 | INFO | PerfReport: 21.18%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 17:29:10.457 | INFO | PerfReport: 32.20%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:29:10.457 | INFO | PerfReport: 1.44%, LockUpload -- 1.40e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260506 17:29:10.457 | INFO | PerfReport: 1.14%, LockDownload -- 1.11e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260506 17:29:10.457 | INFO | PerfReport: 22.56%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 17:29:10.457 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260506 17:29:10.575 | INFO | Starting a round of Baz IO | 20260506 17:29:13.721 | INFO | Populating RTMetrics report | 20260506 17:29:13.721 | INFO | RTMetrics report populated | 20260506 17:29:13.721 | INFO | Opening tmp RTMetrics json file | 20260506 17:29:13.892 | INFO | Serializing RTMetrics to json | 20260506 17:29:13.916 | INFO | Serialization done, swapping tmp file now... | 20260506 17:29:13.917 | INFO | RTMetrics done | 20260506 17:29:14.161 | INFO | Finished a round of Baz IO: Wrote 2.064 GiB in 3.585 seconds | 20260506 17:29:14.161 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.62% / 0.01% / 64.37% | 20260506 17:29:14.161 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.36489% / 1.58946e-05% / 5.5631e-05% | 20260506 17:29:14.161 | INFO | Baz IO overall: | 20260506 17:29:14.161 | INFO | BytesWritten : 1662721745893 | 20260506 17:29:14.161 | INFO | NumEvents : 804380966788 | 20260506 17:29:15.595 | INFO | Analyzing chunk frames = [3153920,3154432) | 20260506 17:29:15.595 | INFO | MemInfo: { | 20260506 17:29:15.595 | INFO | "gpu" : " 17 GB", | 20260506 17:29:15.595 | INFO | "huge" : "", | 20260506 17:29:15.595 | INFO | "huge_shared" : " 27 GB", | 20260506 17:29:15.595 | INFO | "malloc" : " 14 GB", | 20260506 17:29:15.595 | INFO | "pinned" : " 29 GB" | 20260506 17:29:15.595 | INFO | } | 20260506 17:29:15.595 | INFO | Perf report for the last 5.13923 seconds: | 20260506 17:29:15.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:29:15.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:15.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:15.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:15.595 | INFO | ANALYSIS : 62.9%, 2.9, 95.7ms, 0 | 20260506 17:29:15.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:15.595 | INFO | RT_METRICS : 47.2%, 1.0, 24.8ms, 0 | 20260506 17:29:15.595 | INFO | PRE_HQ : 60.3%, 1.9, 58.2ms, 0 | 20260506 17:29:15.595 | INFO | BAZWRITER : 69.4%, 4.8, 169.9ms, 0 | 20260506 17:29:15.595 | INFO | BAZ_SAMPLING : 3.2%, 1.0, 1.7ms, 0 | 20260506 17:29:15.595 | INFO | Uploaded 36GiB to GPU in 2.06894s (17.4002GiB/s) | 20260506 17:29:15.595 | INFO | Downloaded 38GiB to GPU in 2.22509s (17.078GiB/s) | 20260506 17:29:20.734 | INFO | Analyzing chunk frames = [3154432,3154944) | 20260506 17:29:20.734 | INFO | MemInfo: { | 20260506 17:29:20.734 | INFO | "gpu" : " 17 GB", | 20260506 17:29:20.735 | INFO | "huge" : "", | 20260506 17:29:20.735 | INFO | "huge_shared" : " 27 GB", | 20260506 17:29:20.735 | INFO | "malloc" : " 14 GB", | 20260506 17:29:20.735 | INFO | "pinned" : " 29 GB" | 20260506 17:29:20.735 | INFO | } | 20260506 17:29:20.735 | INFO | Perf report for the last 5.1397 seconds: | 20260506 17:29:20.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:29:20.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:20.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:20.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:20.735 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:29:20.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:20.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:20.735 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:29:20.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:20.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:20.735 | INFO | Uploaded 36GiB to GPU in 1.99753s (18.0223GiB/s) | 20260506 17:29:20.735 | INFO | Downloaded 34GiB to GPU in 1.92628s (17.6506GiB/s) | 20260506 17:29:25.874 | INFO | Analyzing chunk frames = [3154944,3155456) | 20260506 17:29:25.874 | INFO | MemInfo: { | 20260506 17:29:25.874 | INFO | "gpu" : " 17 GB", | 20260506 17:29:25.874 | INFO | "huge" : "", | 20260506 17:29:25.874 | INFO | "huge_shared" : " 27 GB", | 20260506 17:29:25.874 | INFO | "malloc" : " 14 GB", | 20260506 17:29:25.874 | INFO | "pinned" : " 29 GB" | 20260506 17:29:25.874 | INFO | } | 20260506 17:29:25.874 | INFO | Perf report for the last 5.13908 seconds: | 20260506 17:29:25.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:29:25.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:25.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:25.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:25.874 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 17:29:25.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:25.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:25.874 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:29:25.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:25.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:25.874 | INFO | Uploaded 36GiB to GPU in 1.9576s (18.3899GiB/s) | 20260506 17:29:25.874 | INFO | Downloaded 34GiB to GPU in 1.91942s (17.7137GiB/s) | 20260506 17:29:31.012 | INFO | Analyzing chunk frames = [3155456,3155968) | 20260506 17:29:31.012 | INFO | MemInfo: { | 20260506 17:29:31.012 | INFO | "gpu" : " 17 GB", | 20260506 17:29:31.012 | INFO | "huge" : "", | 20260506 17:29:31.013 | INFO | "huge_shared" : " 27 GB", | 20260506 17:29:31.013 | INFO | "malloc" : " 14 GB", | 20260506 17:29:31.013 | INFO | "pinned" : " 29 GB" | 20260506 17:29:31.013 | INFO | } | 20260506 17:29:31.013 | INFO | Perf report for the last 5.13895 seconds: | 20260506 17:29:31.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:29:31.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:31.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:31.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:31.013 | INFO | ANALYSIS : 60.1%, 3.0, 93.0ms, 0 | 20260506 17:29:31.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:31.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:31.013 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:29:31.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:31.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:31.013 | INFO | Uploaded 36GiB to GPU in 1.93256s (18.6282GiB/s) | 20260506 17:29:31.013 | INFO | Downloaded 34GiB to GPU in 1.86672s (18.2138GiB/s) | 20260506 17:29:36.150 | INFO | Analyzing chunk frames = [3155968,3156480) | 20260506 17:29:36.150 | INFO | MemInfo: { | 20260506 17:29:36.150 | INFO | "gpu" : " 17 GB", | 20260506 17:29:36.150 | INFO | "huge" : "", | 20260506 17:29:36.150 | INFO | "huge_shared" : " 27 GB", | 20260506 17:29:36.150 | INFO | "malloc" : " 14 GB", | 20260506 17:29:36.150 | INFO | "pinned" : " 29 GB" | 20260506 17:29:36.150 | INFO | } | 20260506 17:29:36.150 | INFO | Perf report for the last 5.1377 seconds: | 20260506 17:29:36.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:29:36.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:36.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:36.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:36.150 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 17:29:36.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:36.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:36.151 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:29:36.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:36.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:36.151 | INFO | Uploaded 36GiB to GPU in 2.01079s (17.9034GiB/s) | 20260506 17:29:36.151 | INFO | Downloaded 34GiB to GPU in 1.9052s (17.8459GiB/s) | 20260506 17:29:41.288 | INFO | Analyzing chunk frames = [3156480,3156992) | 20260506 17:29:41.288 | INFO | MemInfo: { | 20260506 17:29:41.288 | INFO | "gpu" : " 17 GB", | 20260506 17:29:41.288 | INFO | "huge" : "", | 20260506 17:29:41.288 | INFO | "huge_shared" : " 27 GB", | 20260506 17:29:41.288 | INFO | "malloc" : " 14 GB", | 20260506 17:29:41.288 | INFO | "pinned" : " 29 GB" | 20260506 17:29:41.288 | INFO | } | 20260506 17:29:41.288 | INFO | Perf report for the last 5.138 seconds: | 20260506 17:29:41.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:29:41.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:41.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:41.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:41.288 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:29:41.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:41.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:41.288 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 17:29:41.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:41.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:41.289 | INFO | Uploaded 36GiB to GPU in 2.04447s (17.6085GiB/s) | 20260506 17:29:41.289 | INFO | Downloaded 34GiB to GPU in 1.94614s (17.4705GiB/s) | 20260506 17:29:46.426 | INFO | Analyzing chunk frames = [3156992,3157504) | 20260506 17:29:46.426 | INFO | MemInfo: { | 20260506 17:29:46.426 | INFO | "gpu" : " 17 GB", | 20260506 17:29:46.426 | INFO | "huge" : "", | 20260506 17:29:46.426 | INFO | "huge_shared" : " 27 GB", | 20260506 17:29:46.426 | INFO | "malloc" : " 14 GB", | 20260506 17:29:46.426 | INFO | "pinned" : " 29 GB" | 20260506 17:29:46.426 | INFO | } | 20260506 17:29:46.426 | INFO | Perf report for the last 5.13789 seconds: | 20260506 17:29:46.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:29:46.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:46.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:46.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:46.426 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:29:46.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:46.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:46.426 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:29:46.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:46.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:46.426 | INFO | Uploaded 36GiB to GPU in 2.00142s (17.9872GiB/s) | 20260506 17:29:46.427 | INFO | Downloaded 34GiB to GPU in 2.01211s (16.8976GiB/s) | 20260506 17:29:51.571 | INFO | Analyzing chunk frames = [3157504,3158016) | 20260506 17:29:51.571 | INFO | MemInfo: { | 20260506 17:29:51.571 | INFO | "gpu" : " 17 GB", | 20260506 17:29:51.571 | INFO | "huge" : "", | 20260506 17:29:51.571 | INFO | "huge_shared" : " 27 GB", | 20260506 17:29:51.571 | INFO | "malloc" : " 14 GB", | 20260506 17:29:51.571 | INFO | "pinned" : " 29 GB" | 20260506 17:29:51.571 | INFO | } | 20260506 17:29:51.571 | INFO | Perf report for the last 5.14434 seconds: | 20260506 17:29:51.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:29:51.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:51.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:51.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:51.571 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:29:51.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:51.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:51.571 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:29:51.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:51.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:51.571 | INFO | Uploaded 36GiB to GPU in 2.01792s (17.8402GiB/s) | 20260506 17:29:51.571 | INFO | Downloaded 34GiB to GPU in 1.97768s (17.1919GiB/s) | 20260506 17:29:51.685 | INFO | Starting a round of Baz IO | 20260506 17:29:54.822 | INFO | Populating RTMetrics report | 20260506 17:29:54.822 | INFO | RTMetrics report populated | 20260506 17:29:54.822 | INFO | Opening tmp RTMetrics json file | 20260506 17:29:54.870 | INFO | Serializing RTMetrics to json | 20260506 17:29:54.930 | INFO | Serialization done, swapping tmp file now... | 20260506 17:29:54.942 | INFO | RTMetrics done | 20260506 17:29:55.791 | INFO | Finished a round of Baz IO: Wrote 2.063 GiB in 4.106 seconds | 20260506 17:29:55.791 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.61% / 0.01% / 64.38% | 20260506 17:29:55.791 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.36149% / 3.97364e-06% / 8.74201e-05% | 20260506 17:29:55.792 | INFO | Baz IO overall: | 20260506 17:29:55.792 | INFO | BytesWritten : 1664936716526 | 20260506 17:29:55.792 | INFO | NumEvents : 805449349039 | 20260506 17:29:56.714 | INFO | Analyzing chunk frames = [3158016,3158528) | 20260506 17:29:56.714 | INFO | MemInfo: { | 20260506 17:29:56.714 | INFO | "gpu" : " 17 GB", | 20260506 17:29:56.714 | INFO | "huge" : "", | 20260506 17:29:56.714 | INFO | "huge_shared" : " 27 GB", | 20260506 17:29:56.714 | INFO | "malloc" : " 14 GB", | 20260506 17:29:56.714 | INFO | "pinned" : " 29 GB" | 20260506 17:29:56.714 | INFO | } | 20260506 17:29:56.714 | INFO | Perf report for the last 5.1436 seconds: | 20260506 17:29:56.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:29:56.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:56.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:56.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:56.714 | INFO | ANALYSIS : 62.6%, 2.9, 95.4ms, 0 | 20260506 17:29:56.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:29:56.714 | INFO | RT_METRICS : 45.6%, 1.0, 24.0ms, 0 | 20260506 17:29:56.714 | INFO | PRE_HQ : 60.2%, 1.9, 58.1ms, 0 | 20260506 17:29:56.714 | INFO | BAZWRITER : 79.5%, 4.8, 197.5ms, 0 | 20260506 17:29:56.714 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 17:29:56.714 | INFO | Uploaded 36GiB to GPU in 2.07217s (17.3731GiB/s) | 20260506 17:29:56.715 | INFO | Downloaded 38GiB to GPU in 2.21553s (17.1516GiB/s) | 20260506 17:30:01.850 | INFO | Analyzing chunk frames = [3158528,3159040) | 20260506 17:30:01.850 | INFO | MemInfo: { | 20260506 17:30:01.850 | INFO | "gpu" : " 17 GB", | 20260506 17:30:01.850 | INFO | "huge" : "", | 20260506 17:30:01.850 | INFO | "huge_shared" : " 27 GB", | 20260506 17:30:01.850 | INFO | "malloc" : " 14 GB", | 20260506 17:30:01.850 | INFO | "pinned" : " 29 GB" | 20260506 17:30:01.850 | INFO | } | 20260506 17:30:01.850 | INFO | Perf report for the last 5.13595 seconds: | 20260506 17:30:01.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:30:01.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:01.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:01.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:01.850 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:30:01.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:01.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:01.850 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 17:30:01.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:01.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:01.851 | INFO | Uploaded 36GiB to GPU in 2.00478s (17.9571GiB/s) | 20260506 17:30:01.851 | INFO | Downloaded 34GiB to GPU in 1.93459s (17.5748GiB/s) | 20260506 17:30:06.988 | INFO | Analyzing chunk frames = [3159040,3159552) | 20260506 17:30:06.988 | INFO | MemInfo: { | 20260506 17:30:06.988 | INFO | "gpu" : " 17 GB", | 20260506 17:30:06.988 | INFO | "huge" : "", | 20260506 17:30:06.988 | INFO | "huge_shared" : " 27 GB", | 20260506 17:30:06.988 | INFO | "malloc" : " 14 GB", | 20260506 17:30:06.988 | INFO | "pinned" : " 29 GB" | 20260506 17:30:06.988 | INFO | } | 20260506 17:30:06.988 | INFO | Perf report for the last 5.13793 seconds: | 20260506 17:30:06.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:30:06.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:06.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:06.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:06.988 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:30:06.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:06.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:06.988 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:30:06.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:06.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:06.989 | INFO | Uploaded 36GiB to GPU in 1.98467s (18.139GiB/s) | 20260506 17:30:06.989 | INFO | Downloaded 34GiB to GPU in 1.9214s (17.6955GiB/s) | 20260506 17:30:12.125 | INFO | Analyzing chunk frames = [3159552,3160064) | 20260506 17:30:12.125 | INFO | MemInfo: { | 20260506 17:30:12.125 | INFO | "gpu" : " 17 GB", | 20260506 17:30:12.125 | INFO | "huge" : "", | 20260506 17:30:12.125 | INFO | "huge_shared" : " 27 GB", | 20260506 17:30:12.125 | INFO | "malloc" : " 14 GB", | 20260506 17:30:12.125 | INFO | "pinned" : " 29 GB" | 20260506 17:30:12.125 | INFO | } | 20260506 17:30:12.125 | INFO | Perf report for the last 5.13694 seconds: | 20260506 17:30:12.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:30:12.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:12.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:12.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:12.125 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:30:12.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:12.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:12.125 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 17:30:12.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:12.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:12.125 | INFO | Uploaded 36GiB to GPU in 1.92725s (18.6795GiB/s) | 20260506 17:30:12.125 | INFO | Downloaded 34GiB to GPU in 1.86352s (18.245GiB/s) | 20260506 17:30:17.261 | INFO | Analyzing chunk frames = [3160064,3160576) | 20260506 17:30:17.261 | INFO | MemInfo: { | 20260506 17:30:17.261 | INFO | "gpu" : " 17 GB", | 20260506 17:30:17.261 | INFO | "huge" : "", | 20260506 17:30:17.261 | INFO | "huge_shared" : " 27 GB", | 20260506 17:30:17.261 | INFO | "malloc" : " 14 GB", | 20260506 17:30:17.261 | INFO | "pinned" : " 29 GB" | 20260506 17:30:17.261 | INFO | } | 20260506 17:30:17.261 | INFO | Perf report for the last 5.13623 seconds: | 20260506 17:30:17.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:30:17.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:17.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:17.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:17.262 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 17:30:17.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:17.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:17.262 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:30:17.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:17.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:17.262 | INFO | Uploaded 36GiB to GPU in 1.95705s (18.395GiB/s) | 20260506 17:30:17.262 | INFO | Downloaded 34GiB to GPU in 1.8796s (18.089GiB/s) | 20260506 17:30:22.401 | INFO | Analyzing chunk frames = [3160576,3161088) | 20260506 17:30:22.401 | INFO | MemInfo: { | 20260506 17:30:22.401 | INFO | "gpu" : " 17 GB", | 20260506 17:30:22.401 | INFO | "huge" : "", | 20260506 17:30:22.401 | INFO | "huge_shared" : " 27 GB", | 20260506 17:30:22.401 | INFO | "malloc" : " 14 GB", | 20260506 17:30:22.401 | INFO | "pinned" : " 29 GB" | 20260506 17:30:22.401 | INFO | } | 20260506 17:30:22.401 | INFO | Perf report for the last 5.1396 seconds: | 20260506 17:30:22.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:30:22.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:22.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:22.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:22.401 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 17:30:22.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:22.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:22.401 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 17:30:22.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:22.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:22.401 | INFO | Uploaded 36GiB to GPU in 1.99743s (18.0231GiB/s) | 20260506 17:30:22.401 | INFO | Downloaded 34GiB to GPU in 1.94982s (17.4375GiB/s) | 20260506 17:30:27.544 | INFO | Analyzing chunk frames = [3161088,3161600) | 20260506 17:30:27.544 | INFO | MemInfo: { | 20260506 17:30:27.544 | INFO | "gpu" : " 17 GB", | 20260506 17:30:27.544 | INFO | "huge" : "", | 20260506 17:30:27.545 | INFO | "huge_shared" : " 27 GB", | 20260506 17:30:27.545 | INFO | "malloc" : " 14 GB", | 20260506 17:30:27.545 | INFO | "pinned" : " 29 GB" | 20260506 17:30:27.545 | INFO | } | 20260506 17:30:27.545 | INFO | Perf report for the last 5.14357 seconds: | 20260506 17:30:27.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:30:27.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:27.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:27.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:27.545 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 17:30:27.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:27.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:27.545 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:30:27.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:27.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:27.545 | INFO | Uploaded 36GiB to GPU in 2.01726s (17.846GiB/s) | 20260506 17:30:27.545 | INFO | Downloaded 34GiB to GPU in 2.00815s (16.931GiB/s) | 20260506 17:30:32.686 | INFO | Analyzing chunk frames = [3161600,3162112) | 20260506 17:30:32.686 | INFO | MemInfo: { | 20260506 17:30:32.686 | INFO | "gpu" : " 17 GB", | 20260506 17:30:32.686 | INFO | "huge" : "", | 20260506 17:30:32.686 | INFO | "huge_shared" : " 27 GB", | 20260506 17:30:32.686 | INFO | "malloc" : " 14 GB", | 20260506 17:30:32.686 | INFO | "pinned" : " 29 GB" | 20260506 17:30:32.686 | INFO | } | 20260506 17:30:32.686 | INFO | Perf report for the last 5.14147 seconds: | 20260506 17:30:32.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:30:32.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:32.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:32.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:32.686 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:30:32.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:32.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:32.686 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 17:30:32.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:32.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:32.686 | INFO | Uploaded 36GiB to GPU in 2.00553s (17.9504GiB/s) | 20260506 17:30:32.686 | INFO | Downloaded 34GiB to GPU in 1.98289s (17.1467GiB/s) | 20260506 17:30:32.686 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:30:32.686 | INFO | Total mem usage: 18 GiB | 20260506 17:30:32.686 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:30:32.686 | INFO | Recent High Water Marks Sum: 17.3 GiB (Savings: -3.87%) | 20260506 17:30:32.686 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:30:32.686 | INFO | High Water Breakdown: | 20260506 17:30:32.686 | INFO | 62.9% : 10.9 GiB ( 95.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:30:32.686 | INFO | 37% : 6.41 GiB ( 95.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:30:32.686 | INFO | 0.115% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:30:32.687 | INFO | ------------------------------------------ | 20260506 17:30:32.687 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:30:32.687 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:30:32.687 | INFO | Total mem usage: 26.5 GiB | 20260506 17:30:32.687 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:30:32.687 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:30:32.687 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:30:32.687 | INFO | High Water Breakdown: | 20260506 17:30:32.687 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:30:32.687 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:30:32.687 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:30:32.687 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:30:32.687 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:30:32.687 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:30:32.687 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:30:32.687 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:30:32.687 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:30:32.687 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:30:32.687 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:30:32.687 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:30:32.687 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:30:32.687 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:30:32.687 | INFO | ------------------------------------------ | 20260506 17:30:32.687 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:30:32.687 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:30:32.687 | INFO | Total mem usage: 25.6 GiB | 20260506 17:30:32.687 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:30:32.687 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:30:32.687 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:30:32.687 | INFO | High Water Breakdown: | 20260506 17:30:32.687 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:30:32.687 | INFO | ------------------------------------------ | 20260506 17:30:32.687 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:30:32.687 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:30:32.687 | INFO | Total mem usage: 15.7 GiB | 20260506 17:30:32.687 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:30:32.687 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:30:32.687 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:30:32.687 | INFO | High Water Breakdown: | 20260506 17:30:32.687 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:30:32.687 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:30:32.687 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:30:32.687 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:30:32.687 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:30:32.687 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:30:32.687 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:30:32.687 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:30:32.687 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:30:32.687 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:30:32.687 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:30:32.687 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:30:32.687 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:30:32.687 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:30:32.687 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:30:32.687 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:30:32.687 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:30:32.687 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:30:32.687 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:30:32.687 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:30:32.687 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:30:32.687 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:30:32.687 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:30:32.687 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:30:32.687 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:30:32.687 | INFO | ------------------------------------------ | 20260506 17:30:32.687 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:30:32.687 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:30:32.687 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:30:32.687 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:30:32.687 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.61e+01, 1.34e+01 / 2.94e+00 | 20260506 17:30:32.687 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.00e-01, 1.02e-02 / 6.92e-03 | 20260506 17:30:32.687 | INFO | PerfReport: 11.62%, Baseline -- 3.64e+00 / 4.38e+00, 5.58e-01 / 3.51e-01 | 20260506 17:30:32.687 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.42e+00, 1.82e-01 / 1.51e-01 | 20260506 17:30:32.687 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.51e+01, 4.34e+00 / 3.27e+00 | 20260506 17:30:32.687 | INFO | PerfReport: 56.91%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.37e+00 / 2.01e+00 | 20260506 17:30:32.687 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.28e+00 / 2.89e+00, 3.85e-01 / 2.24e-01 | 20260506 17:30:32.687 | INFO | PerfReport: 3.44%, Metrics -- 1.08e+00 / 1.47e+00, 2.80e-01 / 1.79e-01 | 20260506 17:30:32.687 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.73e-01 / 4.30e-01, 5.85e-02 / 2.69e-02 | 20260506 17:30:32.687 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:30:32.687 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:30:32.687 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:30:32.687 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 17:30:32.687 | INFO | PerfReport: 21.43%, Upload -- 2.08e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260506 17:30:32.687 | INFO | PerfReport: 21.12%, Download -- 2.05e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260506 17:30:32.687 | INFO | PerfReport: 32.23%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:30:32.687 | INFO | PerfReport: 1.40%, LockUpload -- 1.36e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260506 17:30:32.687 | INFO | PerfReport: 1.12%, LockDownload -- 1.09e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260506 17:30:32.687 | INFO | PerfReport: 22.68%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 17:30:32.687 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260506 17:30:32.799 | INFO | Starting a round of Baz IO | 20260506 17:30:35.937 | INFO | Populating RTMetrics report | 20260506 17:30:35.937 | INFO | RTMetrics report populated | 20260506 17:30:35.937 | INFO | Opening tmp RTMetrics json file | 20260506 17:30:36.017 | INFO | Serializing RTMetrics to json | 20260506 17:30:36.018 | INFO | Serialization done, swapping tmp file now... | 20260506 17:30:36.018 | INFO | RTMetrics done | 20260506 17:30:36.556 | INFO | Finished a round of Baz IO: Wrote 2.062 GiB in 3.756 seconds | 20260506 17:30:36.556 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.60% / 0.01% / 64.39% | 20260506 17:30:36.556 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.35671% / 1.19209e-05% / 5.5631e-05% | 20260506 17:30:36.556 | INFO | Baz IO overall: | 20260506 17:30:36.556 | INFO | BytesWritten : 1667150983771 | 20260506 17:30:36.556 | INFO | NumEvents : 806517415632 | 20260506 17:30:37.826 | INFO | Analyzing chunk frames = [3162112,3162624) | 20260506 17:30:37.826 | INFO | MemInfo: { | 20260506 17:30:37.826 | INFO | "gpu" : " 17 GB", | 20260506 17:30:37.826 | INFO | "huge" : "", | 20260506 17:30:37.826 | INFO | "huge_shared" : " 27 GB", | 20260506 17:30:37.826 | INFO | "malloc" : " 14 GB", | 20260506 17:30:37.826 | INFO | "pinned" : " 29 GB" | 20260506 17:30:37.826 | INFO | } | 20260506 17:30:37.826 | INFO | Perf report for the last 5.14011 seconds: | 20260506 17:30:37.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:30:37.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:37.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:37.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:37.826 | INFO | ANALYSIS : 62.7%, 2.9, 95.3ms, 0 | 20260506 17:30:37.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:37.826 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 17:30:37.826 | INFO | PRE_HQ : 60.3%, 1.9, 58.5ms, 0 | 20260506 17:30:37.826 | INFO | BAZWRITER : 72.7%, 4.8, 179.4ms, 0 | 20260506 17:30:37.826 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 17:30:37.826 | INFO | Uploaded 36GiB to GPU in 2.04838s (17.5748GiB/s) | 20260506 17:30:37.826 | INFO | Downloaded 38GiB to GPU in 2.20614s (17.2247GiB/s) | 20260506 17:30:42.965 | INFO | Analyzing chunk frames = [3162624,3163136) | 20260506 17:30:42.965 | INFO | MemInfo: { | 20260506 17:30:42.965 | INFO | "gpu" : " 17 GB", | 20260506 17:30:42.965 | INFO | "huge" : "", | 20260506 17:30:42.965 | INFO | "huge_shared" : " 27 GB", | 20260506 17:30:42.965 | INFO | "malloc" : " 14 GB", | 20260506 17:30:42.965 | INFO | "pinned" : " 29 GB" | 20260506 17:30:42.965 | INFO | } | 20260506 17:30:42.965 | INFO | Perf report for the last 5.13926 seconds: | 20260506 17:30:42.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:30:42.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:42.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:42.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:42.966 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 17:30:42.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:42.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:42.966 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:30:42.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:42.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:42.966 | INFO | Uploaded 36GiB to GPU in 2.01418s (17.8733GiB/s) | 20260506 17:30:42.966 | INFO | Downloaded 34GiB to GPU in 1.9443s (17.487GiB/s) | 20260506 17:30:48.100 | INFO | Analyzing chunk frames = [3163136,3163648) | 20260506 17:30:48.100 | INFO | MemInfo: { | 20260506 17:30:48.100 | INFO | "gpu" : " 17 GB", | 20260506 17:30:48.100 | INFO | "huge" : "", | 20260506 17:30:48.100 | INFO | "huge_shared" : " 27 GB", | 20260506 17:30:48.100 | INFO | "malloc" : " 14 GB", | 20260506 17:30:48.100 | INFO | "pinned" : " 29 GB" | 20260506 17:30:48.100 | INFO | } | 20260506 17:30:48.100 | INFO | Perf report for the last 5.13477 seconds: | 20260506 17:30:48.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:30:48.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:48.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:48.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:48.100 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:30:48.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:48.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:48.100 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:30:48.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:48.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:48.100 | INFO | Uploaded 36GiB to GPU in 1.97909s (18.1902GiB/s) | 20260506 17:30:48.101 | INFO | Downloaded 34GiB to GPU in 1.92323s (17.6786GiB/s) | 20260506 17:30:53.239 | INFO | Analyzing chunk frames = [3163648,3164160) | 20260506 17:30:53.239 | INFO | MemInfo: { | 20260506 17:30:53.239 | INFO | "gpu" : " 17 GB", | 20260506 17:30:53.239 | INFO | "huge" : "", | 20260506 17:30:53.239 | INFO | "huge_shared" : " 27 GB", | 20260506 17:30:53.239 | INFO | "malloc" : " 14 GB", | 20260506 17:30:53.239 | INFO | "pinned" : " 29 GB" | 20260506 17:30:53.239 | INFO | } | 20260506 17:30:53.239 | INFO | Perf report for the last 5.139 seconds: | 20260506 17:30:53.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:30:53.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:53.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:53.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:53.239 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:30:53.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:53.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:53.239 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:30:53.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:53.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:53.239 | INFO | Uploaded 36GiB to GPU in 1.93358s (18.6183GiB/s) | 20260506 17:30:53.240 | INFO | Downloaded 34GiB to GPU in 1.87545s (18.129GiB/s) | 20260506 17:30:58.376 | INFO | Analyzing chunk frames = [3164160,3164672) | 20260506 17:30:58.376 | INFO | MemInfo: { | 20260506 17:30:58.376 | INFO | "gpu" : " 17 GB", | 20260506 17:30:58.376 | INFO | "huge" : "", | 20260506 17:30:58.376 | INFO | "huge_shared" : " 27 GB", | 20260506 17:30:58.376 | INFO | "malloc" : " 14 GB", | 20260506 17:30:58.376 | INFO | "pinned" : " 29 GB" | 20260506 17:30:58.376 | INFO | } | 20260506 17:30:58.376 | INFO | Perf report for the last 5.1369 seconds: | 20260506 17:30:58.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:30:58.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:58.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:58.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:58.376 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 17:30:58.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:58.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:58.376 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:30:58.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:58.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:30:58.376 | INFO | Uploaded 36GiB to GPU in 1.95964s (18.3708GiB/s) | 20260506 17:30:58.377 | INFO | Downloaded 34GiB to GPU in 1.87933s (18.0915GiB/s) | 20260506 17:31:03.521 | INFO | Analyzing chunk frames = [3164672,3165184) | 20260506 17:31:03.521 | INFO | MemInfo: { | 20260506 17:31:03.521 | INFO | "gpu" : " 17 GB", | 20260506 17:31:03.521 | INFO | "huge" : "", | 20260506 17:31:03.521 | INFO | "huge_shared" : " 27 GB", | 20260506 17:31:03.521 | INFO | "malloc" : " 14 GB", | 20260506 17:31:03.521 | INFO | "pinned" : " 29 GB" | 20260506 17:31:03.521 | INFO | } | 20260506 17:31:03.521 | INFO | Perf report for the last 5.14526 seconds: | 20260506 17:31:03.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:31:03.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:03.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:03.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:03.521 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 17:31:03.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:03.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:03.522 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 17:31:03.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:03.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:03.522 | INFO | Uploaded 36GiB to GPU in 2.01618s (17.8555GiB/s) | 20260506 17:31:03.522 | INFO | Downloaded 34GiB to GPU in 1.94098s (17.5169GiB/s) | 20260506 17:31:08.661 | INFO | Analyzing chunk frames = [3165184,3165696) | 20260506 17:31:08.661 | INFO | MemInfo: { | 20260506 17:31:08.661 | INFO | "gpu" : " 17 GB", | 20260506 17:31:08.661 | INFO | "huge" : "", | 20260506 17:31:08.661 | INFO | "huge_shared" : " 27 GB", | 20260506 17:31:08.661 | INFO | "malloc" : " 14 GB", | 20260506 17:31:08.661 | INFO | "pinned" : " 29 GB" | 20260506 17:31:08.661 | INFO | } | 20260506 17:31:08.661 | INFO | Perf report for the last 5.13958 seconds: | 20260506 17:31:08.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:31:08.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:08.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:08.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:08.661 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:31:08.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:08.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:08.661 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 17:31:08.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:08.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:08.661 | INFO | Uploaded 36GiB to GPU in 2.01467s (17.8689GiB/s) | 20260506 17:31:08.661 | INFO | Downloaded 34GiB to GPU in 1.99988s (17.001GiB/s) | 20260506 17:31:13.805 | INFO | Analyzing chunk frames = [3165696,3166208) | 20260506 17:31:13.805 | INFO | MemInfo: { | 20260506 17:31:13.805 | INFO | "gpu" : " 17 GB", | 20260506 17:31:13.805 | INFO | "huge" : "", | 20260506 17:31:13.805 | INFO | "huge_shared" : " 27 GB", | 20260506 17:31:13.805 | INFO | "malloc" : " 14 GB", | 20260506 17:31:13.805 | INFO | "pinned" : " 29 GB" | 20260506 17:31:13.805 | INFO | } | 20260506 17:31:13.805 | INFO | Perf report for the last 5.1443 seconds: | 20260506 17:31:13.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:31:13.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:13.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:13.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:13.805 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 17:31:13.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:13.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:13.805 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 17:31:13.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:13.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:13.806 | INFO | Uploaded 36GiB to GPU in 2.0035s (17.9685GiB/s) | 20260506 17:31:13.806 | INFO | Downloaded 34GiB to GPU in 1.97599s (17.2066GiB/s) | 20260506 17:31:13.922 | INFO | Starting a round of Baz IO | 20260506 17:31:17.064 | INFO | Populating RTMetrics report | 20260506 17:31:17.065 | INFO | RTMetrics report populated | 20260506 17:31:17.065 | INFO | Opening tmp RTMetrics json file | 20260506 17:31:17.090 | INFO | Serializing RTMetrics to json | 20260506 17:31:17.143 | INFO | Serialization done, swapping tmp file now... | 20260506 17:31:17.213 | INFO | RTMetrics done | 20260506 17:31:17.540 | INFO | Finished a round of Baz IO: Wrote 2.063 GiB in 3.618 seconds | 20260506 17:31:17.540 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.60% / 0.01% / 64.40% | 20260506 17:31:17.540 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.35515% / 7.94729e-06% / 5.5631e-05% | 20260506 17:31:17.540 | INFO | Baz IO overall: | 20260506 17:31:17.540 | INFO | BytesWritten : 1669365629003 | 20260506 17:31:17.540 | INFO | NumEvents : 807586032161 | 20260506 17:31:18.942 | INFO | Analyzing chunk frames = [3166208,3166720) | 20260506 17:31:18.942 | INFO | MemInfo: { | 20260506 17:31:18.942 | INFO | "gpu" : " 17 GB", | 20260506 17:31:18.942 | INFO | "huge" : "", | 20260506 17:31:18.942 | INFO | "huge_shared" : " 27 GB", | 20260506 17:31:18.942 | INFO | "malloc" : " 14 GB", | 20260506 17:31:18.942 | INFO | "pinned" : " 29 GB" | 20260506 17:31:18.942 | INFO | } | 20260506 17:31:18.942 | INFO | Perf report for the last 5.13645 seconds: | 20260506 17:31:18.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:31:18.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:18.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:18.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:18.942 | INFO | ANALYSIS : 62.8%, 2.9, 95.5ms, 0 | 20260506 17:31:18.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:18.942 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260506 17:31:18.942 | INFO | PRE_HQ : 60.4%, 1.9, 58.3ms, 0 | 20260506 17:31:18.942 | INFO | BAZWRITER : 70.2%, 4.8, 173.1ms, 0 | 20260506 17:31:18.942 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260506 17:31:18.942 | INFO | Uploaded 36GiB to GPU in 2.0702s (17.3897GiB/s) | 20260506 17:31:18.942 | INFO | Downloaded 38GiB to GPU in 2.21623s (17.1463GiB/s) | 20260506 17:31:24.085 | INFO | Analyzing chunk frames = [3166720,3167232) | 20260506 17:31:24.085 | INFO | MemInfo: { | 20260506 17:31:24.085 | INFO | "gpu" : " 17 GB", | 20260506 17:31:24.085 | INFO | "huge" : "", | 20260506 17:31:24.085 | INFO | "huge_shared" : " 27 GB", | 20260506 17:31:24.085 | INFO | "malloc" : " 14 GB", | 20260506 17:31:24.085 | INFO | "pinned" : " 29 GB" | 20260506 17:31:24.085 | INFO | } | 20260506 17:31:24.085 | INFO | Perf report for the last 5.14327 seconds: | 20260506 17:31:24.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:31:24.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:24.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:24.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:24.085 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 17:31:24.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:24.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:24.085 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:31:24.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:24.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:24.085 | INFO | Uploaded 36GiB to GPU in 2.00157s (17.9859GiB/s) | 20260506 17:31:24.085 | INFO | Downloaded 34GiB to GPU in 1.9268s (17.6458GiB/s) | 20260506 17:31:29.215 | INFO | Analyzing chunk frames = [3167232,3167744) | 20260506 17:31:29.215 | INFO | MemInfo: { | 20260506 17:31:29.215 | INFO | "gpu" : " 17 GB", | 20260506 17:31:29.215 | INFO | "huge" : "", | 20260506 17:31:29.215 | INFO | "huge_shared" : " 27 GB", | 20260506 17:31:29.215 | INFO | "malloc" : " 14 GB", | 20260506 17:31:29.215 | INFO | "pinned" : " 29 GB" | 20260506 17:31:29.215 | INFO | } | 20260506 17:31:29.215 | INFO | Perf report for the last 5.12993 seconds: | 20260506 17:31:29.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:31:29.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:29.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:29.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:29.215 | INFO | ANALYSIS : 61.1%, 3.0, 94.4ms, 0 | 20260506 17:31:29.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:29.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:29.215 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:31:29.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:29.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:29.215 | INFO | Uploaded 36GiB to GPU in 1.97817s (18.1986GiB/s) | 20260506 17:31:29.215 | INFO | Downloaded 34GiB to GPU in 1.91465s (17.7578GiB/s) | 20260506 17:31:34.353 | INFO | Analyzing chunk frames = [3167744,3168256) | 20260506 17:31:34.353 | INFO | MemInfo: { | 20260506 17:31:34.353 | INFO | "gpu" : " 17 GB", | 20260506 17:31:34.353 | INFO | "huge" : "", | 20260506 17:31:34.353 | INFO | "huge_shared" : " 27 GB", | 20260506 17:31:34.353 | INFO | "malloc" : " 14 GB", | 20260506 17:31:34.353 | INFO | "pinned" : " 29 GB" | 20260506 17:31:34.353 | INFO | } | 20260506 17:31:34.353 | INFO | Perf report for the last 5.13845 seconds: | 20260506 17:31:34.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:31:34.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:34.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:34.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:34.354 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:31:34.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:34.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:34.354 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:31:34.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:34.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:34.354 | INFO | Uploaded 36GiB to GPU in 1.94014s (18.5553GiB/s) | 20260506 17:31:34.354 | INFO | Downloaded 34GiB to GPU in 1.87859s (18.0987GiB/s) | 20260506 17:31:39.496 | INFO | Analyzing chunk frames = [3168256,3168768) | 20260506 17:31:39.496 | INFO | MemInfo: { | 20260506 17:31:39.496 | INFO | "gpu" : " 17 GB", | 20260506 17:31:39.496 | INFO | "huge" : "", | 20260506 17:31:39.496 | INFO | "huge_shared" : " 27 GB", | 20260506 17:31:39.496 | INFO | "malloc" : " 14 GB", | 20260506 17:31:39.496 | INFO | "pinned" : " 29 GB" | 20260506 17:31:39.496 | INFO | } | 20260506 17:31:39.496 | INFO | Perf report for the last 5.14281 seconds: | 20260506 17:31:39.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:31:39.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:39.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:39.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:39.496 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 17:31:39.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:39.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:39.496 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:31:39.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:39.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:39.497 | INFO | Uploaded 36GiB to GPU in 1.98936s (18.0963GiB/s) | 20260506 17:31:39.497 | INFO | Downloaded 34GiB to GPU in 1.89861s (17.9078GiB/s) | 20260506 17:31:44.638 | INFO | Analyzing chunk frames = [3168768,3169280) | 20260506 17:31:44.638 | INFO | MemInfo: { | 20260506 17:31:44.638 | INFO | "gpu" : " 17 GB", | 20260506 17:31:44.638 | INFO | "huge" : "", | 20260506 17:31:44.638 | INFO | "huge_shared" : " 27 GB", | 20260506 17:31:44.638 | INFO | "malloc" : " 14 GB", | 20260506 17:31:44.638 | INFO | "pinned" : " 29 GB" | 20260506 17:31:44.638 | INFO | } | 20260506 17:31:44.638 | INFO | Perf report for the last 5.14137 seconds: | 20260506 17:31:44.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:31:44.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:44.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:44.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:44.638 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 17:31:44.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:44.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:44.638 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 17:31:44.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:44.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:44.638 | INFO | Uploaded 36GiB to GPU in 2.0241s (17.7857GiB/s) | 20260506 17:31:44.638 | INFO | Downloaded 34GiB to GPU in 1.94347s (17.4945GiB/s) | 20260506 17:31:49.779 | INFO | Analyzing chunk frames = [3169280,3169792) | 20260506 17:31:49.779 | INFO | MemInfo: { | 20260506 17:31:49.779 | INFO | "gpu" : " 17 GB", | 20260506 17:31:49.779 | INFO | "huge" : "", | 20260506 17:31:49.780 | INFO | "huge_shared" : " 27 GB", | 20260506 17:31:49.780 | INFO | "malloc" : " 14 GB", | 20260506 17:31:49.780 | INFO | "pinned" : " 29 GB" | 20260506 17:31:49.780 | INFO | } | 20260506 17:31:49.780 | INFO | Perf report for the last 5.14191 seconds: | 20260506 17:31:49.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:31:49.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:49.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:49.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:49.780 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:31:49.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:49.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:49.780 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:31:49.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:49.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:49.780 | INFO | Uploaded 36GiB to GPU in 2.01767s (17.8424GiB/s) | 20260506 17:31:49.780 | INFO | Downloaded 34GiB to GPU in 2.00918s (16.9223GiB/s) | 20260506 17:31:54.914 | INFO | Analyzing chunk frames = [3169792,3170304) | 20260506 17:31:54.914 | INFO | MemInfo: { | 20260506 17:31:54.914 | INFO | "gpu" : " 17 GB", | 20260506 17:31:54.914 | INFO | "huge" : "", | 20260506 17:31:54.914 | INFO | "huge_shared" : " 27 GB", | 20260506 17:31:54.914 | INFO | "malloc" : " 14 GB", | 20260506 17:31:54.914 | INFO | "pinned" : " 29 GB" | 20260506 17:31:54.914 | INFO | } | 20260506 17:31:54.914 | INFO | Perf report for the last 5.13461 seconds: | 20260506 17:31:54.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:31:54.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:54.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:54.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:54.914 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 17:31:54.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:54.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:54.914 | INFO | PRE_HQ : 31.9%, 1.0, 16.6ms, 0 | 20260506 17:31:54.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:54.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:31:54.915 | INFO | Uploaded 36GiB to GPU in 2.05967s (17.4785GiB/s) | 20260506 17:31:54.915 | INFO | Downloaded 34GiB to GPU in 1.98704s (17.1109GiB/s) | 20260506 17:31:54.915 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:31:54.915 | INFO | Total mem usage: 18 GiB | 20260506 17:31:54.915 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:31:54.915 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.3%) | 20260506 17:31:54.915 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:31:54.915 | INFO | High Water Breakdown: | 20260506 17:31:54.915 | INFO | 61.8% : 9.91 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:31:54.915 | INFO | 38.1% : 6.11 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:31:54.915 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:31:54.915 | INFO | ------------------------------------------ | 20260506 17:31:54.915 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:31:54.915 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:31:54.915 | INFO | Total mem usage: 26.5 GiB | 20260506 17:31:54.915 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:31:54.915 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:31:54.915 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:31:54.915 | INFO | High Water Breakdown: | 20260506 17:31:54.915 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:31:54.915 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:31:54.915 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:31:54.915 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:31:54.915 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:31:54.915 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:31:54.915 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:31:54.915 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:31:54.915 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:31:54.915 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:31:54.915 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:31:54.915 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:31:54.915 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:31:54.915 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:31:54.915 | INFO | ------------------------------------------ | 20260506 17:31:54.915 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:31:54.915 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:31:54.915 | INFO | Total mem usage: 25.6 GiB | 20260506 17:31:54.915 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:31:54.915 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:31:54.915 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:31:54.915 | INFO | High Water Breakdown: | 20260506 17:31:54.915 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:31:54.915 | INFO | ------------------------------------------ | 20260506 17:31:54.915 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:31:54.915 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:31:54.915 | INFO | Total mem usage: 15.7 GiB | 20260506 17:31:54.915 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:31:54.915 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:31:54.915 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 17:31:54.915 | INFO | High Water Breakdown: | 20260506 17:31:54.915 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:31:54.915 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:31:54.915 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:31:54.915 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:31:54.915 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:31:54.915 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:31:54.915 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:31:54.915 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:31:54.915 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:31:54.915 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:31:54.915 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:31:54.915 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:31:54.915 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:31:54.915 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:31:54.915 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:31:54.915 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:31:54.916 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:31:54.916 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:31:54.916 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:31:54.916 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:31:54.916 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:31:54.916 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:31:54.916 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:31:54.916 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:31:54.916 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:31:54.916 | INFO | ------------------------------------------ | 20260506 17:31:54.916 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:31:54.916 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:31:54.916 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:31:54.916 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:31:54.916 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.56e+01, 1.32e+01 / 2.75e+00 | 20260506 17:31:54.916 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 9.92e-03 / 5.58e-03 | 20260506 17:31:54.916 | INFO | PerfReport: 11.60%, Baseline -- 3.63e+00 / 4.36e+00, 5.52e-01 / 3.68e-01 | 20260506 17:31:54.916 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.82e-01 / 1.35e-01 | 20260506 17:31:54.916 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.45e+01, 4.07e+00 / 3.06e+00 | 20260506 17:31:54.916 | INFO | PerfReport: 56.95%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.40e+00 / 1.92e+00 | 20260506 17:31:54.916 | INFO | PerfReport: 7.30%, PulseAccumulating -- 2.28e+00 / 2.89e+00, 3.93e-01 / 2.04e-01 | 20260506 17:31:54.916 | INFO | PerfReport: 3.45%, Metrics -- 1.08e+00 / 1.45e+00, 2.82e-01 / 1.74e-01 | 20260506 17:31:54.916 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.73e-01 / 4.21e-01, 5.79e-02 / 1.97e-02 | 20260506 17:31:54.916 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:31:54.916 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:31:54.916 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:31:54.916 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 17:31:54.916 | INFO | PerfReport: 21.48%, Upload -- 2.09e+01 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260506 17:31:54.916 | INFO | PerfReport: 21.13%, Download -- 2.05e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260506 17:31:54.916 | INFO | PerfReport: 32.23%, Compute -- 3.13e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 17:31:54.916 | INFO | PerfReport: 1.42%, LockUpload -- 1.38e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260506 17:31:54.916 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260506 17:31:54.916 | INFO | PerfReport: 22.63%, LockCompute -- 2.20e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 17:31:54.916 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260506 17:31:55.030 | INFO | Starting a round of Baz IO | 20260506 17:31:58.171 | INFO | Populating RTMetrics report | 20260506 17:31:58.171 | INFO | RTMetrics report populated | 20260506 17:31:58.171 | INFO | Opening tmp RTMetrics json file | 20260506 17:31:58.195 | INFO | Serializing RTMetrics to json | 20260506 17:31:58.199 | INFO | Serialization done, swapping tmp file now... | 20260506 17:31:58.203 | INFO | RTMetrics done | 20260506 17:31:58.629 | INFO | Finished a round of Baz IO: Wrote 2.061 GiB in 3.598 seconds | 20260506 17:31:58.629 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.59% / 0.01% / 64.40% | 20260506 17:31:58.629 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.35121% / 1.58946e-05% / 5.5631e-05% | 20260506 17:31:58.629 | INFO | Baz IO overall: | 20260506 17:31:58.629 | INFO | BytesWritten : 1671579126823 | 20260506 17:31:58.629 | INFO | NumEvents : 808653972567 | 20260506 17:32:00.052 | INFO | Analyzing chunk frames = [3170304,3170816) | 20260506 17:32:00.052 | INFO | MemInfo: { | 20260506 17:32:00.052 | INFO | "gpu" : " 17 GB", | 20260506 17:32:00.052 | INFO | "huge" : "", | 20260506 17:32:00.052 | INFO | "huge_shared" : " 27 GB", | 20260506 17:32:00.052 | INFO | "malloc" : " 14 GB", | 20260506 17:32:00.052 | INFO | "pinned" : " 29 GB" | 20260506 17:32:00.052 | INFO | } | 20260506 17:32:00.052 | INFO | Perf report for the last 5.13789 seconds: | 20260506 17:32:00.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:32:00.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:00.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:00.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:00.052 | INFO | ANALYSIS : 62.8%, 2.9, 95.4ms, 0 | 20260506 17:32:00.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:00.052 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 17:32:00.052 | INFO | PRE_HQ : 60.6%, 1.9, 58.5ms, 0 | 20260506 17:32:00.052 | INFO | BAZWRITER : 69.4%, 4.8, 171.7ms, 0 | 20260506 17:32:00.052 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 17:32:00.053 | INFO | Uploaded 36GiB to GPU in 2.0583s (17.4902GiB/s) | 20260506 17:32:00.053 | INFO | Downloaded 38GiB to GPU in 2.23484s (17.0034GiB/s) | 20260506 17:32:05.190 | INFO | Analyzing chunk frames = [3170816,3171328) | 20260506 17:32:05.190 | INFO | MemInfo: { | 20260506 17:32:05.190 | INFO | "gpu" : " 17 GB", | 20260506 17:32:05.190 | INFO | "huge" : "", | 20260506 17:32:05.190 | INFO | "huge_shared" : " 27 GB", | 20260506 17:32:05.190 | INFO | "malloc" : " 14 GB", | 20260506 17:32:05.190 | INFO | "pinned" : " 29 GB" | 20260506 17:32:05.190 | INFO | } | 20260506 17:32:05.190 | INFO | Perf report for the last 5.1379 seconds: | 20260506 17:32:05.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:32:05.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:05.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:05.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:05.190 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:32:05.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:05.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:05.190 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:32:05.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:05.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:05.190 | INFO | Uploaded 36GiB to GPU in 1.98394s (18.1457GiB/s) | 20260506 17:32:05.190 | INFO | Downloaded 34GiB to GPU in 1.93658s (17.5567GiB/s) | 20260506 17:32:10.328 | INFO | Analyzing chunk frames = [3171328,3171840) | 20260506 17:32:10.328 | INFO | MemInfo: { | 20260506 17:32:10.328 | INFO | "gpu" : " 17 GB", | 20260506 17:32:10.328 | INFO | "huge" : "", | 20260506 17:32:10.328 | INFO | "huge_shared" : " 27 GB", | 20260506 17:32:10.328 | INFO | "malloc" : " 14 GB", | 20260506 17:32:10.328 | INFO | "pinned" : " 29 GB" | 20260506 17:32:10.328 | INFO | } | 20260506 17:32:10.328 | INFO | Perf report for the last 5.13805 seconds: | 20260506 17:32:10.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:32:10.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:10.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:10.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:10.328 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 17:32:10.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:10.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:10.328 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 17:32:10.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:10.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:10.328 | INFO | Uploaded 36GiB to GPU in 1.97842s (18.1963GiB/s) | 20260506 17:32:10.329 | INFO | Downloaded 34GiB to GPU in 1.92715s (17.6427GiB/s) | 20260506 17:32:15.468 | INFO | Analyzing chunk frames = [3171840,3172352) | 20260506 17:32:15.468 | INFO | MemInfo: { | 20260506 17:32:15.468 | INFO | "gpu" : " 17 GB", | 20260506 17:32:15.468 | INFO | "huge" : "", | 20260506 17:32:15.468 | INFO | "huge_shared" : " 27 GB", | 20260506 17:32:15.468 | INFO | "malloc" : " 14 GB", | 20260506 17:32:15.468 | INFO | "pinned" : " 29 GB" | 20260506 17:32:15.468 | INFO | } | 20260506 17:32:15.468 | INFO | Perf report for the last 5.13955 seconds: | 20260506 17:32:15.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:32:15.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:15.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:15.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:15.468 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 17:32:15.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:15.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:15.468 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:32:15.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:15.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:15.468 | INFO | Uploaded 36GiB to GPU in 1.93094s (18.6438GiB/s) | 20260506 17:32:15.468 | INFO | Downloaded 34GiB to GPU in 1.8756s (18.1276GiB/s) | 20260506 17:32:20.609 | INFO | Analyzing chunk frames = [3172352,3172864) | 20260506 17:32:20.609 | INFO | MemInfo: { | 20260506 17:32:20.609 | INFO | "gpu" : " 17 GB", | 20260506 17:32:20.609 | INFO | "huge" : "", | 20260506 17:32:20.609 | INFO | "huge_shared" : " 27 GB", | 20260506 17:32:20.609 | INFO | "malloc" : " 14 GB", | 20260506 17:32:20.609 | INFO | "pinned" : " 29 GB" | 20260506 17:32:20.609 | INFO | } | 20260506 17:32:20.609 | INFO | Perf report for the last 5.14091 seconds: | 20260506 17:32:20.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:32:20.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:20.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:20.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:20.609 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 17:32:20.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:20.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:20.609 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:32:20.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:20.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:20.609 | INFO | Uploaded 36GiB to GPU in 2.01905s (17.8301GiB/s) | 20260506 17:32:20.609 | INFO | Downloaded 34GiB to GPU in 1.91217s (17.7809GiB/s) | 20260506 17:32:25.754 | INFO | Analyzing chunk frames = [3172864,3173376) | 20260506 17:32:25.754 | INFO | MemInfo: { | 20260506 17:32:25.754 | INFO | "gpu" : " 17 GB", | 20260506 17:32:25.754 | INFO | "huge" : "", | 20260506 17:32:25.754 | INFO | "huge_shared" : " 27 GB", | 20260506 17:32:25.754 | INFO | "malloc" : " 14 GB", | 20260506 17:32:25.754 | INFO | "pinned" : " 29 GB" | 20260506 17:32:25.754 | INFO | } | 20260506 17:32:25.754 | INFO | Perf report for the last 5.14502 seconds: | 20260506 17:32:25.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:32:25.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:25.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:25.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:25.754 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 17:32:25.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:25.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:25.754 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:32:25.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:25.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:25.754 | INFO | Uploaded 36GiB to GPU in 2.02199s (17.8042GiB/s) | 20260506 17:32:25.754 | INFO | Downloaded 34GiB to GPU in 1.94091s (17.5175GiB/s) | 20260506 17:32:30.891 | INFO | Analyzing chunk frames = [3173376,3173888) | 20260506 17:32:30.891 | INFO | MemInfo: { | 20260506 17:32:30.891 | INFO | "gpu" : " 17 GB", | 20260506 17:32:30.891 | INFO | "huge" : "", | 20260506 17:32:30.891 | INFO | "huge_shared" : " 27 GB", | 20260506 17:32:30.891 | INFO | "malloc" : " 14 GB", | 20260506 17:32:30.891 | INFO | "pinned" : " 29 GB" | 20260506 17:32:30.891 | INFO | } | 20260506 17:32:30.891 | INFO | Perf report for the last 5.13745 seconds: | 20260506 17:32:30.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:32:30.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:30.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:30.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:30.891 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:32:30.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:30.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:30.891 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 17:32:30.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:30.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:30.891 | INFO | Uploaded 36GiB to GPU in 2.0115s (17.8971GiB/s) | 20260506 17:32:30.891 | INFO | Downloaded 34GiB to GPU in 1.99231s (17.0657GiB/s) | 20260506 17:32:36.030 | INFO | Analyzing chunk frames = [3173888,3174400) | 20260506 17:32:36.030 | INFO | MemInfo: { | 20260506 17:32:36.030 | INFO | "gpu" : " 17 GB", | 20260506 17:32:36.030 | INFO | "huge" : "", | 20260506 17:32:36.030 | INFO | "huge_shared" : " 27 GB", | 20260506 17:32:36.030 | INFO | "malloc" : " 14 GB", | 20260506 17:32:36.030 | INFO | "pinned" : " 29 GB" | 20260506 17:32:36.030 | INFO | } | 20260506 17:32:36.030 | INFO | Perf report for the last 5.13877 seconds: | 20260506 17:32:36.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:32:36.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:36.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:36.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:36.030 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:32:36.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:36.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:36.030 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:32:36.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:36.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:36.030 | INFO | Uploaded 36GiB to GPU in 1.9946s (18.0487GiB/s) | 20260506 17:32:36.030 | INFO | Downloaded 34GiB to GPU in 1.9591s (17.3549GiB/s) | 20260506 17:32:36.144 | INFO | Starting a round of Baz IO | 20260506 17:32:39.285 | INFO | Populating RTMetrics report | 20260506 17:32:39.285 | INFO | RTMetrics report populated | 20260506 17:32:39.285 | INFO | Opening tmp RTMetrics json file | 20260506 17:32:39.307 | INFO | Serializing RTMetrics to json | 20260506 17:32:39.311 | INFO | Serialization done, swapping tmp file now... | 20260506 17:32:39.314 | INFO | RTMetrics done | 20260506 17:32:39.618 | INFO | Finished a round of Baz IO: Wrote 2.063 GiB in 3.473 seconds | 20260506 17:32:39.618 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.58% / 0.01% / 64.41% | 20260506 17:32:39.618 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.34934% / 1.19209e-05% / 4.76837e-05% | 20260506 17:32:39.618 | INFO | Baz IO overall: | 20260506 17:32:39.618 | INFO | BytesWritten : 1673793919142 | 20260506 17:32:39.618 | INFO | NumEvents : 809722810092 | 20260506 17:32:41.172 | INFO | Analyzing chunk frames = [3174400,3174912) | 20260506 17:32:41.172 | INFO | MemInfo: { | 20260506 17:32:41.172 | INFO | "gpu" : " 17 GB", | 20260506 17:32:41.172 | INFO | "huge" : "", | 20260506 17:32:41.172 | INFO | "huge_shared" : " 27 GB", | 20260506 17:32:41.172 | INFO | "malloc" : " 14 GB", | 20260506 17:32:41.172 | INFO | "pinned" : " 29 GB" | 20260506 17:32:41.172 | INFO | } | 20260506 17:32:41.172 | INFO | Perf report for the last 5.14221 seconds: | 20260506 17:32:41.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:32:41.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:41.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:41.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:41.172 | INFO | ANALYSIS : 62.7%, 2.9, 95.2ms, 0 | 20260506 17:32:41.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:41.172 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260506 17:32:41.172 | INFO | PRE_HQ : 60.4%, 1.9, 58.7ms, 0 | 20260506 17:32:41.172 | INFO | BAZWRITER : 67.2%, 4.8, 166.3ms, 0 | 20260506 17:32:41.172 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 17:32:41.172 | INFO | Uploaded 36GiB to GPU in 2.02277s (17.7974GiB/s) | 20260506 17:32:41.173 | INFO | Downloaded 38GiB to GPU in 2.21802s (17.1324GiB/s) | 20260506 17:32:46.305 | INFO | Analyzing chunk frames = [3174912,3175424) | 20260506 17:32:46.305 | INFO | MemInfo: { | 20260506 17:32:46.305 | INFO | "gpu" : " 17 GB", | 20260506 17:32:46.305 | INFO | "huge" : "", | 20260506 17:32:46.305 | INFO | "huge_shared" : " 27 GB", | 20260506 17:32:46.305 | INFO | "malloc" : " 14 GB", | 20260506 17:32:46.305 | INFO | "pinned" : " 29 GB" | 20260506 17:32:46.305 | INFO | } | 20260506 17:32:46.305 | INFO | Perf report for the last 5.13268 seconds: | 20260506 17:32:46.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:32:46.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:46.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:46.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:46.305 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:32:46.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:46.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:46.305 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:32:46.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:46.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:46.305 | INFO | Uploaded 36GiB to GPU in 2.00306s (17.9725GiB/s) | 20260506 17:32:46.305 | INFO | Downloaded 34GiB to GPU in 1.92824s (17.6327GiB/s) | 20260506 17:32:51.443 | INFO | Analyzing chunk frames = [3175424,3175936) | 20260506 17:32:51.443 | INFO | MemInfo: { | 20260506 17:32:51.443 | INFO | "gpu" : " 17 GB", | 20260506 17:32:51.444 | INFO | "huge" : "", | 20260506 17:32:51.444 | INFO | "huge_shared" : " 27 GB", | 20260506 17:32:51.444 | INFO | "malloc" : " 14 GB", | 20260506 17:32:51.444 | INFO | "pinned" : " 29 GB" | 20260506 17:32:51.444 | INFO | } | 20260506 17:32:51.444 | INFO | Perf report for the last 5.13872 seconds: | 20260506 17:32:51.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:32:51.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:51.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:51.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:51.444 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 17:32:51.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:51.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:51.444 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:32:51.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:51.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:51.444 | INFO | Uploaded 36GiB to GPU in 1.96987s (18.2753GiB/s) | 20260506 17:32:51.444 | INFO | Downloaded 34GiB to GPU in 1.92828s (17.6323GiB/s) | 20260506 17:32:56.584 | INFO | Analyzing chunk frames = [3175936,3176448) | 20260506 17:32:56.584 | INFO | MemInfo: { | 20260506 17:32:56.584 | INFO | "gpu" : " 17 GB", | 20260506 17:32:56.584 | INFO | "huge" : "", | 20260506 17:32:56.584 | INFO | "huge_shared" : " 27 GB", | 20260506 17:32:56.584 | INFO | "malloc" : " 14 GB", | 20260506 17:32:56.584 | INFO | "pinned" : " 29 GB" | 20260506 17:32:56.584 | INFO | } | 20260506 17:32:56.584 | INFO | Perf report for the last 5.14085 seconds: | 20260506 17:32:56.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:32:56.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:56.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:56.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:56.584 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:32:56.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:56.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:56.584 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:32:56.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:56.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:32:56.585 | INFO | Uploaded 36GiB to GPU in 1.91159s (18.8325GiB/s) | 20260506 17:32:56.585 | INFO | Downloaded 34GiB to GPU in 1.87463s (18.1369GiB/s) | 20260506 17:33:01.720 | INFO | Analyzing chunk frames = [3176448,3176960) | 20260506 17:33:01.720 | INFO | MemInfo: { | 20260506 17:33:01.720 | INFO | "gpu" : " 17 GB", | 20260506 17:33:01.720 | INFO | "huge" : "", | 20260506 17:33:01.720 | INFO | "huge_shared" : " 27 GB", | 20260506 17:33:01.720 | INFO | "malloc" : " 14 GB", | 20260506 17:33:01.720 | INFO | "pinned" : " 29 GB" | 20260506 17:33:01.720 | INFO | } | 20260506 17:33:01.720 | INFO | Perf report for the last 5.1357 seconds: | 20260506 17:33:01.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:33:01.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:01.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:01.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:01.720 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 17:33:01.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:01.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:01.720 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 17:33:01.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:01.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:01.720 | INFO | Uploaded 36GiB to GPU in 1.95249s (18.438GiB/s) | 20260506 17:33:01.720 | INFO | Downloaded 34GiB to GPU in 1.88237s (18.0623GiB/s) | 20260506 17:33:06.865 | INFO | Analyzing chunk frames = [3176960,3177472) | 20260506 17:33:06.865 | INFO | MemInfo: { | 20260506 17:33:06.865 | INFO | "gpu" : " 17 GB", | 20260506 17:33:06.865 | INFO | "huge" : "", | 20260506 17:33:06.865 | INFO | "huge_shared" : " 27 GB", | 20260506 17:33:06.865 | INFO | "malloc" : " 14 GB", | 20260506 17:33:06.865 | INFO | "pinned" : " 29 GB" | 20260506 17:33:06.865 | INFO | } | 20260506 17:33:06.865 | INFO | Perf report for the last 5.14466 seconds: | 20260506 17:33:06.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:33:06.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:06.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:06.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:06.865 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 17:33:06.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:06.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:06.865 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:33:06.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:06.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:06.865 | INFO | Uploaded 36GiB to GPU in 2.01566s (17.8601GiB/s) | 20260506 17:33:06.865 | INFO | Downloaded 34GiB to GPU in 1.96131s (17.3353GiB/s) | 20260506 17:33:12.004 | INFO | Analyzing chunk frames = [3177472,3177984) | 20260506 17:33:12.004 | INFO | MemInfo: { | 20260506 17:33:12.004 | INFO | "gpu" : " 17 GB", | 20260506 17:33:12.005 | INFO | "huge" : "", | 20260506 17:33:12.005 | INFO | "huge_shared" : " 27 GB", | 20260506 17:33:12.005 | INFO | "malloc" : " 14 GB", | 20260506 17:33:12.005 | INFO | "pinned" : " 29 GB" | 20260506 17:33:12.005 | INFO | } | 20260506 17:33:12.005 | INFO | Perf report for the last 5.13984 seconds: | 20260506 17:33:12.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:33:12.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:12.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:12.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:12.005 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:33:12.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:12.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:12.005 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 17:33:12.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:12.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:12.005 | INFO | Uploaded 36GiB to GPU in 1.98644s (18.1229GiB/s) | 20260506 17:33:12.005 | INFO | Downloaded 34GiB to GPU in 2.01694s (16.8572GiB/s) | 20260506 17:33:17.148 | INFO | Analyzing chunk frames = [3177984,3178496) | 20260506 17:33:17.148 | INFO | MemInfo: { | 20260506 17:33:17.148 | INFO | "gpu" : " 17 GB", | 20260506 17:33:17.148 | INFO | "huge" : "", | 20260506 17:33:17.148 | INFO | "huge_shared" : " 27 GB", | 20260506 17:33:17.148 | INFO | "malloc" : " 14 GB", | 20260506 17:33:17.148 | INFO | "pinned" : " 29 GB" | 20260506 17:33:17.148 | INFO | } | 20260506 17:33:17.148 | INFO | Perf report for the last 5.14322 seconds: | 20260506 17:33:17.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:33:17.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:17.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:17.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:17.148 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:33:17.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:17.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:17.148 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 17:33:17.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:17.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:17.148 | INFO | Uploaded 36GiB to GPU in 2.01132s (17.8987GiB/s) | 20260506 17:33:17.148 | INFO | Downloaded 34GiB to GPU in 1.97498s (17.2154GiB/s) | 20260506 17:33:17.148 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:33:17.148 | INFO | Total mem usage: 18 GiB | 20260506 17:33:17.148 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:33:17.148 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.4%) | 20260506 17:33:17.148 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:33:17.148 | INFO | High Water Breakdown: | 20260506 17:33:17.148 | INFO | 61.8% : 9.9 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:33:17.148 | INFO | 38.1% : 6.11 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:33:17.148 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:33:17.148 | INFO | ------------------------------------------ | 20260506 17:33:17.148 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:33:17.148 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:33:17.148 | INFO | Total mem usage: 26.5 GiB | 20260506 17:33:17.148 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:33:17.148 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:33:17.148 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:33:17.148 | INFO | High Water Breakdown: | 20260506 17:33:17.148 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:33:17.148 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:33:17.148 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:33:17.148 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:33:17.148 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:33:17.148 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:33:17.148 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:33:17.148 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:33:17.148 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:33:17.148 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:33:17.148 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:33:17.148 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:33:17.148 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:33:17.149 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:33:17.149 | INFO | ------------------------------------------ | 20260506 17:33:17.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:33:17.149 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:33:17.149 | INFO | Total mem usage: 25.6 GiB | 20260506 17:33:17.149 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:33:17.149 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:33:17.149 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:33:17.149 | INFO | High Water Breakdown: | 20260506 17:33:17.149 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:33:17.149 | INFO | ------------------------------------------ | 20260506 17:33:17.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:33:17.149 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:33:17.149 | INFO | Total mem usage: 15.7 GiB | 20260506 17:33:17.149 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:33:17.149 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:33:17.149 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 17:33:17.149 | INFO | High Water Breakdown: | 20260506 17:33:17.149 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:33:17.149 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:33:17.149 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:33:17.149 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:33:17.149 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:33:17.149 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:33:17.149 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:33:17.149 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:33:17.149 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:33:17.149 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:33:17.149 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:33:17.149 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:33:17.149 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:33:17.149 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:33:17.149 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:33:17.149 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:33:17.149 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:33:17.149 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:33:17.149 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:33:17.149 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:33:17.149 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:33:17.149 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:33:17.149 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:33:17.149 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:33:17.149 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:33:17.149 | INFO | ------------------------------------------ | 20260506 17:33:17.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:33:17.149 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:33:17.149 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:33:17.149 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:33:17.149 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.58e+01, 1.35e+01 / 2.70e+00 | 20260506 17:33:17.149 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 1.03e-02 / 6.49e-03 | 20260506 17:33:17.149 | INFO | PerfReport: 11.60%, Baseline -- 3.64e+00 / 4.40e+00, 5.62e-01 / 3.16e-01 | 20260506 17:33:17.149 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.85e-01 / 1.28e-01 | 20260506 17:33:17.149 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.49e+01, 4.14e+00 / 2.77e+00 | 20260506 17:33:17.149 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.79e+01 / 2.03e+01, 1.46e+00 / 1.82e+00 | 20260506 17:33:17.149 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.27e+00 / 2.85e+00, 3.79e-01 / 2.18e-01 | 20260506 17:33:17.149 | INFO | PerfReport: 3.44%, Metrics -- 1.08e+00 / 1.42e+00, 2.80e-01 / 1.81e-01 | 20260506 17:33:17.149 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.73e-01 / 4.22e-01, 5.83e-02 / 2.24e-02 | 20260506 17:33:17.149 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:33:17.149 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:33:17.149 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:33:17.149 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 17:33:17.149 | INFO | PerfReport: 21.36%, Upload -- 2.08e+01 / 0.00e+00, 6.86e+00 / 0.00e+00 | 20260506 17:33:17.149 | INFO | PerfReport: 21.15%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 17:33:17.149 | INFO | PerfReport: 32.29%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 17:33:17.149 | INFO | PerfReport: 1.30%, LockUpload -- 1.27e+00 / 0.00e+00, 5.30e+00 / 0.00e+00 | 20260506 17:33:17.149 | INFO | PerfReport: 1.18%, LockDownload -- 1.14e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260506 17:33:17.149 | INFO | PerfReport: 22.71%, LockCompute -- 2.21e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 17:33:17.149 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260506 17:33:17.266 | INFO | Starting a round of Baz IO | 20260506 17:33:20.398 | INFO | Populating RTMetrics report | 20260506 17:33:20.398 | INFO | RTMetrics report populated | 20260506 17:33:20.398 | INFO | Opening tmp RTMetrics json file | 20260506 17:33:20.403 | INFO | Serializing RTMetrics to json | 20260506 17:33:20.406 | INFO | Serialization done, swapping tmp file now... | 20260506 17:33:20.408 | INFO | RTMetrics done | 20260506 17:33:20.922 | INFO | Finished a round of Baz IO: Wrote 2.065 GiB in 3.656 seconds | 20260506 17:33:20.922 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.58% / 0.01% / 64.42% | 20260506 17:33:20.922 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.35029% / 3.97364e-06% / 6.35783e-05% | 20260506 17:33:20.922 | INFO | Baz IO overall: | 20260506 17:33:20.922 | INFO | BytesWritten : 1676011417336 | 20260506 17:33:20.922 | INFO | NumEvents : 810793684333 | 20260506 17:33:22.282 | INFO | Analyzing chunk frames = [3178496,3179008) | 20260506 17:33:22.282 | INFO | MemInfo: { | 20260506 17:33:22.282 | INFO | "gpu" : " 17 GB", | 20260506 17:33:22.282 | INFO | "huge" : "", | 20260506 17:33:22.282 | INFO | "huge_shared" : " 27 GB", | 20260506 17:33:22.282 | INFO | "malloc" : " 14 GB", | 20260506 17:33:22.282 | INFO | "pinned" : " 29 GB" | 20260506 17:33:22.282 | INFO | } | 20260506 17:33:22.282 | INFO | Perf report for the last 5.1345 seconds: | 20260506 17:33:22.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:33:22.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:22.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:22.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:22.282 | INFO | ANALYSIS : 62.7%, 2.9, 95.2ms, 0 | 20260506 17:33:22.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:22.282 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 17:33:22.282 | INFO | PRE_HQ : 60.2%, 1.9, 58.5ms, 0 | 20260506 17:33:22.282 | INFO | BAZWRITER : 70.9%, 4.8, 175.6ms, 0 | 20260506 17:33:22.282 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260506 17:33:22.283 | INFO | Uploaded 36GiB to GPU in 2.0675s (17.4123GiB/s) | 20260506 17:33:22.283 | INFO | Downloaded 38GiB to GPU in 2.20769s (17.2126GiB/s) | 20260506 17:33:27.424 | INFO | Analyzing chunk frames = [3179008,3179520) | 20260506 17:33:27.424 | INFO | MemInfo: { | 20260506 17:33:27.424 | INFO | "gpu" : " 17 GB", | 20260506 17:33:27.424 | INFO | "huge" : "", | 20260506 17:33:27.424 | INFO | "huge_shared" : " 27 GB", | 20260506 17:33:27.424 | INFO | "malloc" : " 14 GB", | 20260506 17:33:27.424 | INFO | "pinned" : " 29 GB" | 20260506 17:33:27.424 | INFO | } | 20260506 17:33:27.424 | INFO | Perf report for the last 5.14144 seconds: | 20260506 17:33:27.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:33:27.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:27.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:27.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:27.424 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:33:27.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:27.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:27.424 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:33:27.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:27.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:27.424 | INFO | Uploaded 36GiB to GPU in 2.01372s (17.8774GiB/s) | 20260506 17:33:27.424 | INFO | Downloaded 34GiB to GPU in 1.9463s (17.4691GiB/s) | 20260506 17:33:32.559 | INFO | Analyzing chunk frames = [3179520,3180032) | 20260506 17:33:32.559 | INFO | MemInfo: { | 20260506 17:33:32.559 | INFO | "gpu" : " 17 GB", | 20260506 17:33:32.559 | INFO | "huge" : "", | 20260506 17:33:32.559 | INFO | "huge_shared" : " 27 GB", | 20260506 17:33:32.559 | INFO | "malloc" : " 14 GB", | 20260506 17:33:32.559 | INFO | "pinned" : " 29 GB" | 20260506 17:33:32.559 | INFO | } | 20260506 17:33:32.559 | INFO | Perf report for the last 5.13527 seconds: | 20260506 17:33:32.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:33:32.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:32.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:32.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:32.559 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:33:32.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:32.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:32.559 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 17:33:32.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:32.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:32.559 | INFO | Uploaded 36GiB to GPU in 1.97501s (18.2278GiB/s) | 20260506 17:33:32.559 | INFO | Downloaded 34GiB to GPU in 1.92599s (17.6533GiB/s) | 20260506 17:33:37.698 | INFO | Analyzing chunk frames = [3180032,3180544) | 20260506 17:33:37.698 | INFO | MemInfo: { | 20260506 17:33:37.698 | INFO | "gpu" : " 17 GB", | 20260506 17:33:37.698 | INFO | "huge" : "", | 20260506 17:33:37.698 | INFO | "huge_shared" : " 27 GB", | 20260506 17:33:37.698 | INFO | "malloc" : " 14 GB", | 20260506 17:33:37.698 | INFO | "pinned" : " 29 GB" | 20260506 17:33:37.698 | INFO | } | 20260506 17:33:37.698 | INFO | Perf report for the last 5.13866 seconds: | 20260506 17:33:37.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:33:37.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:37.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:37.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:37.698 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 17:33:37.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:37.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:37.698 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:33:37.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:37.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:37.698 | INFO | Uploaded 36GiB to GPU in 1.93428s (18.6116GiB/s) | 20260506 17:33:37.698 | INFO | Downloaded 34GiB to GPU in 1.8756s (18.1276GiB/s) | 20260506 17:33:42.842 | INFO | Analyzing chunk frames = [3180544,3181056) | 20260506 17:33:42.842 | INFO | MemInfo: { | 20260506 17:33:42.842 | INFO | "gpu" : " 17 GB", | 20260506 17:33:42.842 | INFO | "huge" : "", | 20260506 17:33:42.842 | INFO | "huge_shared" : " 27 GB", | 20260506 17:33:42.842 | INFO | "malloc" : " 14 GB", | 20260506 17:33:42.843 | INFO | "pinned" : " 29 GB" | 20260506 17:33:42.843 | INFO | } | 20260506 17:33:42.843 | INFO | Perf report for the last 5.14473 seconds: | 20260506 17:33:42.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:33:42.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:42.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:42.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:42.843 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 17:33:42.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:42.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:42.843 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:33:42.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:42.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:42.843 | INFO | Uploaded 36GiB to GPU in 1.9596s (18.3711GiB/s) | 20260506 17:33:42.843 | INFO | Downloaded 34GiB to GPU in 1.87773s (18.107GiB/s) | 20260506 17:33:47.980 | INFO | Analyzing chunk frames = [3181056,3181568) | 20260506 17:33:47.980 | INFO | MemInfo: { | 20260506 17:33:47.980 | INFO | "gpu" : " 17 GB", | 20260506 17:33:47.980 | INFO | "huge" : "", | 20260506 17:33:47.980 | INFO | "huge_shared" : " 27 GB", | 20260506 17:33:47.980 | INFO | "malloc" : " 14 GB", | 20260506 17:33:47.980 | INFO | "pinned" : " 29 GB" | 20260506 17:33:47.980 | INFO | } | 20260506 17:33:47.980 | INFO | Perf report for the last 5.13726 seconds: | 20260506 17:33:47.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:33:47.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:47.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:47.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:47.980 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 17:33:47.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:47.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:47.980 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:33:47.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:47.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:47.980 | INFO | Uploaded 36GiB to GPU in 2.02437s (17.7833GiB/s) | 20260506 17:33:47.980 | INFO | Downloaded 34GiB to GPU in 1.94717s (17.4613GiB/s) | 20260506 17:33:53.116 | INFO | Analyzing chunk frames = [3181568,3182080) | 20260506 17:33:53.116 | INFO | MemInfo: { | 20260506 17:33:53.116 | INFO | "gpu" : " 17 GB", | 20260506 17:33:53.116 | INFO | "huge" : "", | 20260506 17:33:53.116 | INFO | "huge_shared" : " 27 GB", | 20260506 17:33:53.116 | INFO | "malloc" : " 14 GB", | 20260506 17:33:53.116 | INFO | "pinned" : " 29 GB" | 20260506 17:33:53.116 | INFO | } | 20260506 17:33:53.116 | INFO | Perf report for the last 5.13644 seconds: | 20260506 17:33:53.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:33:53.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:53.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:53.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:53.116 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:33:53.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:53.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:53.116 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:33:53.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:53.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:53.117 | INFO | Uploaded 36GiB to GPU in 2.01428s (17.8724GiB/s) | 20260506 17:33:53.117 | INFO | Downloaded 34GiB to GPU in 2.01194s (16.8991GiB/s) | 20260506 17:33:58.258 | INFO | Analyzing chunk frames = [3182080,3182592) | 20260506 17:33:58.258 | INFO | MemInfo: { | 20260506 17:33:58.258 | INFO | "gpu" : " 17 GB", | 20260506 17:33:58.258 | INFO | "huge" : "", | 20260506 17:33:58.258 | INFO | "huge_shared" : " 27 GB", | 20260506 17:33:58.258 | INFO | "malloc" : " 14 GB", | 20260506 17:33:58.258 | INFO | "pinned" : " 29 GB" | 20260506 17:33:58.258 | INFO | } | 20260506 17:33:58.258 | INFO | Perf report for the last 5.14162 seconds: | 20260506 17:33:58.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:33:58.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:58.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:58.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:58.258 | INFO | ANALYSIS : 62.4%, 3.0, 97.2ms, 0 | 20260506 17:33:58.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:58.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:58.258 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 17:33:58.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:58.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:33:58.258 | INFO | Uploaded 36GiB to GPU in 2.07674s (17.3348GiB/s) | 20260506 17:33:58.258 | INFO | Downloaded 34GiB to GPU in 2.02103s (16.8231GiB/s) | 20260506 17:33:58.370 | INFO | Starting a round of Baz IO | 20260506 17:34:01.538 | INFO | Populating RTMetrics report | 20260506 17:34:01.538 | INFO | RTMetrics report populated | 20260506 17:34:01.538 | INFO | Opening tmp RTMetrics json file | 20260506 17:34:01.570 | INFO | Serializing RTMetrics to json | 20260506 17:34:01.581 | INFO | Serialization done, swapping tmp file now... | 20260506 17:34:01.605 | INFO | RTMetrics done | 20260506 17:34:02.114 | INFO | Finished a round of Baz IO: Wrote 2.063 GiB in 3.743 seconds | 20260506 17:34:02.114 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.57% / 0.01% / 64.42% | 20260506 17:34:02.114 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.34377% / 0% / 6.75519e-05% | 20260506 17:34:02.114 | INFO | Baz IO overall: | 20260506 17:34:02.114 | INFO | BytesWritten : 1678226894462 | 20260506 17:34:02.114 | INFO | NumEvents : 811862983631 | 20260506 17:34:03.396 | INFO | Analyzing chunk frames = [3182592,3183104) | 20260506 17:34:03.396 | INFO | MemInfo: { | 20260506 17:34:03.396 | INFO | "gpu" : " 17 GB", | 20260506 17:34:03.396 | INFO | "huge" : "", | 20260506 17:34:03.396 | INFO | "huge_shared" : " 27 GB", | 20260506 17:34:03.396 | INFO | "malloc" : " 14 GB", | 20260506 17:34:03.396 | INFO | "pinned" : " 29 GB" | 20260506 17:34:03.396 | INFO | } | 20260506 17:34:03.396 | INFO | Perf report for the last 5.13847 seconds: | 20260506 17:34:03.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:34:03.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:03.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:03.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:03.396 | INFO | ANALYSIS : 63.2%, 2.8, 94.0ms, 0 | 20260506 17:34:03.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:03.397 | INFO | RT_METRICS : 47.2%, 1.0, 24.8ms, 0 | 20260506 17:34:03.397 | INFO | PRE_HQ : 61.8%, 1.9, 62.0ms, 0 | 20260506 17:34:03.397 | INFO | BAZWRITER : 72.2%, 4.8, 178.8ms, 0 | 20260506 17:34:03.397 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 17:34:03.397 | INFO | Uploaded 36GiB to GPU in 2.07156s (17.3782GiB/s) | 20260506 17:34:03.397 | INFO | Downloaded 38GiB to GPU in 2.22087s (17.1104GiB/s) | 20260506 17:34:08.534 | INFO | Analyzing chunk frames = [3183104,3183616) | 20260506 17:34:08.534 | INFO | MemInfo: { | 20260506 17:34:08.534 | INFO | "gpu" : " 17 GB", | 20260506 17:34:08.534 | INFO | "huge" : "", | 20260506 17:34:08.534 | INFO | "huge_shared" : " 27 GB", | 20260506 17:34:08.534 | INFO | "malloc" : " 14 GB", | 20260506 17:34:08.534 | INFO | "pinned" : " 29 GB" | 20260506 17:34:08.534 | INFO | } | 20260506 17:34:08.534 | INFO | Perf report for the last 5.13751 seconds: | 20260506 17:34:08.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:34:08.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:08.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:08.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:08.534 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:34:08.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:08.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:08.534 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 17:34:08.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:08.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:08.534 | INFO | Uploaded 36GiB to GPU in 1.99586s (18.0373GiB/s) | 20260506 17:34:08.534 | INFO | Downloaded 34GiB to GPU in 1.9569s (17.3744GiB/s) | 20260506 17:34:13.672 | INFO | Analyzing chunk frames = [3183616,3184128) | 20260506 17:34:13.672 | INFO | MemInfo: { | 20260506 17:34:13.672 | INFO | "gpu" : " 17 GB", | 20260506 17:34:13.672 | INFO | "huge" : "", | 20260506 17:34:13.672 | INFO | "huge_shared" : " 27 GB", | 20260506 17:34:13.672 | INFO | "malloc" : " 14 GB", | 20260506 17:34:13.672 | INFO | "pinned" : " 29 GB" | 20260506 17:34:13.673 | INFO | } | 20260506 17:34:13.673 | INFO | Perf report for the last 5.13861 seconds: | 20260506 17:34:13.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:34:13.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:13.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:13.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:13.673 | INFO | ANALYSIS : 60.2%, 3.0, 93.5ms, 0 | 20260506 17:34:13.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:13.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:13.673 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:34:13.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:13.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:13.673 | INFO | Uploaded 36GiB to GPU in 1.93998s (18.5569GiB/s) | 20260506 17:34:13.673 | INFO | Downloaded 34GiB to GPU in 1.88663s (18.0215GiB/s) | 20260506 17:34:18.812 | INFO | Analyzing chunk frames = [3184128,3184640) | 20260506 17:34:18.812 | INFO | MemInfo: { | 20260506 17:34:18.812 | INFO | "gpu" : " 17 GB", | 20260506 17:34:18.812 | INFO | "huge" : "", | 20260506 17:34:18.812 | INFO | "huge_shared" : " 27 GB", | 20260506 17:34:18.812 | INFO | "malloc" : " 14 GB", | 20260506 17:34:18.812 | INFO | "pinned" : " 29 GB" | 20260506 17:34:18.812 | INFO | } | 20260506 17:34:18.812 | INFO | Perf report for the last 5.13962 seconds: | 20260506 17:34:18.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:34:18.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:18.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:18.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:18.812 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 17:34:18.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:18.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:18.812 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:34:18.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:18.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:18.812 | INFO | Uploaded 36GiB to GPU in 1.91696s (18.7798GiB/s) | 20260506 17:34:18.812 | INFO | Downloaded 34GiB to GPU in 1.8774s (18.1101GiB/s) | 20260506 17:34:23.954 | INFO | Analyzing chunk frames = [3184640,3185152) | 20260506 17:34:23.954 | INFO | MemInfo: { | 20260506 17:34:23.954 | INFO | "gpu" : " 17 GB", | 20260506 17:34:23.954 | INFO | "huge" : "", | 20260506 17:34:23.954 | INFO | "huge_shared" : " 27 GB", | 20260506 17:34:23.954 | INFO | "malloc" : " 14 GB", | 20260506 17:34:23.954 | INFO | "pinned" : " 29 GB" | 20260506 17:34:23.954 | INFO | } | 20260506 17:34:23.954 | INFO | Perf report for the last 5.14183 seconds: | 20260506 17:34:23.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:34:23.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:23.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:23.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:23.954 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260506 17:34:23.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:23.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:23.954 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:34:23.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:23.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:23.954 | INFO | Uploaded 36GiB to GPU in 1.9492s (18.4692GiB/s) | 20260506 17:34:23.954 | INFO | Downloaded 34GiB to GPU in 1.88674s (18.0205GiB/s) | 20260506 17:34:29.099 | INFO | Analyzing chunk frames = [3185152,3185664) | 20260506 17:34:29.099 | INFO | MemInfo: { | 20260506 17:34:29.099 | INFO | "gpu" : " 17 GB", | 20260506 17:34:29.099 | INFO | "huge" : "", | 20260506 17:34:29.099 | INFO | "huge_shared" : " 27 GB", | 20260506 17:34:29.099 | INFO | "malloc" : " 14 GB", | 20260506 17:34:29.099 | INFO | "pinned" : " 29 GB" | 20260506 17:34:29.099 | INFO | } | 20260506 17:34:29.099 | INFO | Perf report for the last 5.14517 seconds: | 20260506 17:34:29.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:34:29.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:29.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:29.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:29.099 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 17:34:29.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:29.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:29.099 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:34:29.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:29.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:29.099 | INFO | Uploaded 36GiB to GPU in 1.9913s (18.0786GiB/s) | 20260506 17:34:29.100 | INFO | Downloaded 34GiB to GPU in 1.94233s (17.5048GiB/s) | 20260506 17:34:34.238 | INFO | Analyzing chunk frames = [3185664,3186176) | 20260506 17:34:34.238 | INFO | MemInfo: { | 20260506 17:34:34.238 | INFO | "gpu" : " 17 GB", | 20260506 17:34:34.238 | INFO | "huge" : "", | 20260506 17:34:34.238 | INFO | "huge_shared" : " 27 GB", | 20260506 17:34:34.238 | INFO | "malloc" : " 14 GB", | 20260506 17:34:34.238 | INFO | "pinned" : " 29 GB" | 20260506 17:34:34.238 | INFO | } | 20260506 17:34:34.238 | INFO | Perf report for the last 5.13893 seconds: | 20260506 17:34:34.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:34:34.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:34.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:34.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:34.238 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:34:34.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:34.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:34.238 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 17:34:34.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:34.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:34.238 | INFO | Uploaded 36GiB to GPU in 1.96884s (18.2849GiB/s) | 20260506 17:34:34.238 | INFO | Downloaded 34GiB to GPU in 2.00565s (16.9521GiB/s) | 20260506 17:34:39.374 | INFO | Analyzing chunk frames = [3186176,3186688) | 20260506 17:34:39.374 | INFO | MemInfo: { | 20260506 17:34:39.374 | INFO | "gpu" : " 17 GB", | 20260506 17:34:39.374 | INFO | "huge" : "", | 20260506 17:34:39.374 | INFO | "huge_shared" : " 27 GB", | 20260506 17:34:39.374 | INFO | "malloc" : " 14 GB", | 20260506 17:34:39.374 | INFO | "pinned" : " 29 GB" | 20260506 17:34:39.374 | INFO | } | 20260506 17:34:39.374 | INFO | Perf report for the last 5.13588 seconds: | 20260506 17:34:39.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:34:39.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:39.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:39.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:39.374 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:34:39.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:39.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:39.374 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 17:34:39.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:39.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:39.374 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:34:39.374 | INFO | Total mem usage: 18 GiB | 20260506 17:34:39.374 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:34:39.374 | INFO | Uploaded 36GiB to GPU in 2.01443s (17.8711GiB/s) | 20260506 17:34:39.374 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.2%) | 20260506 17:34:39.374 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:34:39.374 | INFO | Downloaded 34GiB to GPU in 1.98359s (17.1407GiB/s) | 20260506 17:34:39.374 | INFO | High Water Breakdown: | 20260506 17:34:39.374 | INFO | 61.7% : 9.81 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:34:39.374 | INFO | 38.2% : 6.08 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:34:39.374 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:34:39.374 | INFO | ------------------------------------------ | 20260506 17:34:39.374 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:34:39.374 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:34:39.374 | INFO | Total mem usage: 26.5 GiB | 20260506 17:34:39.374 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:34:39.374 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:34:39.375 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:34:39.375 | INFO | High Water Breakdown: | 20260506 17:34:39.375 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:34:39.375 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:34:39.375 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:34:39.375 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:34:39.375 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:34:39.375 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:34:39.375 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:34:39.375 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:34:39.375 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:34:39.375 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:34:39.375 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:34:39.375 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:34:39.375 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:34:39.375 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:34:39.375 | INFO | ------------------------------------------ | 20260506 17:34:39.375 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:34:39.375 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:34:39.375 | INFO | Total mem usage: 25.6 GiB | 20260506 17:34:39.375 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:34:39.375 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:34:39.375 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:34:39.375 | INFO | High Water Breakdown: | 20260506 17:34:39.375 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:34:39.375 | INFO | ------------------------------------------ | 20260506 17:34:39.375 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:34:39.375 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:34:39.375 | INFO | Total mem usage: 15.7 GiB | 20260506 17:34:39.375 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:34:39.375 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:34:39.375 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 17:34:39.375 | INFO | High Water Breakdown: | 20260506 17:34:39.375 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:34:39.375 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:34:39.375 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:34:39.375 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:34:39.375 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:34:39.375 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:34:39.375 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:34:39.375 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:34:39.375 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:34:39.375 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:34:39.375 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:34:39.375 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:34:39.375 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:34:39.375 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:34:39.375 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:34:39.375 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:34:39.375 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:34:39.375 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:34:39.375 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:34:39.375 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:34:39.375 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:34:39.375 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:34:39.375 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:34:39.375 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:34:39.375 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:34:39.375 | INFO | ------------------------------------------ | 20260506 17:34:39.375 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:34:39.375 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:34:39.375 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:34:39.375 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:34:39.375 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.55e+01, 1.36e+01 / 2.25e+00 | 20260506 17:34:39.375 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.01e-01, 1.01e-02 / 5.34e-03 | 20260506 17:34:39.375 | INFO | PerfReport: 11.60%, Baseline -- 3.63e+00 / 4.39e+00, 5.56e-01 / 3.44e-01 | 20260506 17:34:39.375 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.82e-01 / 1.34e-01 | 20260506 17:34:39.375 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.52e+01, 4.17e+00 / 2.09e+00 | 20260506 17:34:39.375 | INFO | PerfReport: 56.90%, FrameLabeling -- 1.78e+01 / 1.95e+01, 1.34e+00 / 1.37e+00 | 20260506 17:34:39.375 | INFO | PerfReport: 7.32%, PulseAccumulating -- 2.29e+00 / 2.92e+00, 3.99e-01 / 1.72e-01 | 20260506 17:34:39.375 | INFO | PerfReport: 3.45%, Metrics -- 1.08e+00 / 1.44e+00, 2.79e-01 / 1.66e-01 | 20260506 17:34:39.375 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.74e-01 / 4.24e-01, 5.85e-02 / 2.36e-02 | 20260506 17:34:39.375 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:34:39.375 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:34:39.375 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:34:39.375 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 17:34:39.375 | INFO | PerfReport: 21.42%, Upload -- 2.08e+01 / 0.00e+00, 6.88e+00 / 0.00e+00 | 20260506 17:34:39.375 | INFO | PerfReport: 21.20%, Download -- 2.06e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 17:34:39.375 | INFO | PerfReport: 32.29%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 17:34:39.375 | INFO | PerfReport: 1.33%, LockUpload -- 1.29e+00 / 0.00e+00, 5.38e+00 / 0.00e+00 | 20260506 17:34:39.375 | INFO | PerfReport: 1.17%, LockDownload -- 1.14e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 17:34:39.375 | INFO | PerfReport: 22.57%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 17:34:39.375 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260506 17:34:39.493 | INFO | Starting a round of Baz IO | 20260506 17:34:42.631 | INFO | Populating RTMetrics report | 20260506 17:34:42.631 | INFO | RTMetrics report populated | 20260506 17:34:42.631 | INFO | Opening tmp RTMetrics json file | 20260506 17:34:42.836 | INFO | Serializing RTMetrics to json | 20260506 17:34:42.855 | INFO | Serialization done, swapping tmp file now... | 20260506 17:34:42.860 | INFO | RTMetrics done | 20260506 17:34:42.880 | INFO | Finished a round of Baz IO: Wrote 2.062 GiB in 3.386 seconds | 20260506 17:34:42.880 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.56% / 0.01% / 64.43% | 20260506 17:34:42.880 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.33858% / 1.58946e-05% / 5.96046e-05% | 20260506 17:34:42.880 | INFO | Baz IO overall: | 20260506 17:34:42.880 | INFO | BytesWritten : 1680440633636 | 20260506 17:34:42.880 | INFO | NumEvents : 812930858566 | 20260506 17:34:44.513 | INFO | Analyzing chunk frames = [3186688,3187200) | 20260506 17:34:44.513 | INFO | MemInfo: { | 20260506 17:34:44.513 | INFO | "gpu" : " 17 GB", | 20260506 17:34:44.513 | INFO | "huge" : "", | 20260506 17:34:44.513 | INFO | "huge_shared" : " 27 GB", | 20260506 17:34:44.514 | INFO | "malloc" : " 14 GB", | 20260506 17:34:44.514 | INFO | "pinned" : " 29 GB" | 20260506 17:34:44.514 | INFO | } | 20260506 17:34:44.514 | INFO | Perf report for the last 5.13958 seconds: | 20260506 17:34:44.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:34:44.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:44.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:44.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:44.514 | INFO | ANALYSIS : 62.8%, 2.9, 95.5ms, 0 | 20260506 17:34:44.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:44.514 | INFO | RT_METRICS : 48.0%, 1.0, 25.2ms, 0 | 20260506 17:34:44.514 | INFO | PRE_HQ : 60.4%, 1.9, 58.2ms, 0 | 20260506 17:34:44.514 | INFO | BAZWRITER : 65.4%, 4.8, 161.8ms, 0 | 20260506 17:34:44.514 | INFO | BAZ_SAMPLING : 3.1%, 1.0, 1.6ms, 0 | 20260506 17:34:44.514 | INFO | Uploaded 36GiB to GPU in 2.04909s (17.5687GiB/s) | 20260506 17:34:44.514 | INFO | Downloaded 38GiB to GPU in 2.22447s (17.0827GiB/s) | 20260506 17:34:49.647 | INFO | Analyzing chunk frames = [3187200,3187712) | 20260506 17:34:49.647 | INFO | MemInfo: { | 20260506 17:34:49.647 | INFO | "gpu" : " 17 GB", | 20260506 17:34:49.647 | INFO | "huge" : "", | 20260506 17:34:49.647 | INFO | "huge_shared" : " 27 GB", | 20260506 17:34:49.647 | INFO | "malloc" : " 14 GB", | 20260506 17:34:49.647 | INFO | "pinned" : " 29 GB" | 20260506 17:34:49.647 | INFO | } | 20260506 17:34:49.647 | INFO | Perf report for the last 5.13347 seconds: | 20260506 17:34:49.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:34:49.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:49.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:49.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:49.647 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:34:49.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:49.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:49.647 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 17:34:49.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:49.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:49.647 | INFO | Uploaded 36GiB to GPU in 2.01398s (17.8751GiB/s) | 20260506 17:34:49.647 | INFO | Downloaded 34GiB to GPU in 1.97121s (17.2483GiB/s) | 20260506 17:34:54.786 | INFO | Analyzing chunk frames = [3187712,3188224) | 20260506 17:34:54.786 | INFO | MemInfo: { | 20260506 17:34:54.786 | INFO | "gpu" : " 17 GB", | 20260506 17:34:54.786 | INFO | "huge" : "", | 20260506 17:34:54.786 | INFO | "huge_shared" : " 27 GB", | 20260506 17:34:54.786 | INFO | "malloc" : " 14 GB", | 20260506 17:34:54.786 | INFO | "pinned" : " 29 GB" | 20260506 17:34:54.786 | INFO | } | 20260506 17:34:54.786 | INFO | Perf report for the last 5.13877 seconds: | 20260506 17:34:54.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:34:54.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:54.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:54.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:54.786 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 17:34:54.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:54.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:54.786 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:34:54.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:54.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:54.786 | INFO | Uploaded 36GiB to GPU in 1.95236s (18.4392GiB/s) | 20260506 17:34:54.786 | INFO | Downloaded 34GiB to GPU in 1.92201s (17.6898GiB/s) | 20260506 17:34:59.931 | INFO | Analyzing chunk frames = [3188224,3188736) | 20260506 17:34:59.931 | INFO | MemInfo: { | 20260506 17:34:59.931 | INFO | "gpu" : " 17 GB", | 20260506 17:34:59.931 | INFO | "huge" : "", | 20260506 17:34:59.931 | INFO | "huge_shared" : " 27 GB", | 20260506 17:34:59.931 | INFO | "malloc" : " 14 GB", | 20260506 17:34:59.931 | INFO | "pinned" : " 29 GB" | 20260506 17:34:59.931 | INFO | } | 20260506 17:34:59.931 | INFO | Perf report for the last 5.1455 seconds: | 20260506 17:34:59.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:34:59.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:59.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:59.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:59.931 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:34:59.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:59.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:59.932 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:34:59.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:59.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:34:59.932 | INFO | Uploaded 36GiB to GPU in 1.92644s (18.6874GiB/s) | 20260506 17:34:59.932 | INFO | Downloaded 34GiB to GPU in 1.87709s (18.1131GiB/s) | 20260506 17:35:05.070 | INFO | Analyzing chunk frames = [3188736,3189248) | 20260506 17:35:05.070 | INFO | MemInfo: { | 20260506 17:35:05.070 | INFO | "gpu" : " 17 GB", | 20260506 17:35:05.070 | INFO | "huge" : "", | 20260506 17:35:05.070 | INFO | "huge_shared" : " 27 GB", | 20260506 17:35:05.070 | INFO | "malloc" : " 14 GB", | 20260506 17:35:05.070 | INFO | "pinned" : " 29 GB" | 20260506 17:35:05.070 | INFO | } | 20260506 17:35:05.070 | INFO | Perf report for the last 5.13903 seconds: | 20260506 17:35:05.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:35:05.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:05.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:05.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:05.070 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 17:35:05.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:05.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:05.071 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:35:05.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:05.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:05.071 | INFO | Uploaded 36GiB to GPU in 2.01168s (17.8955GiB/s) | 20260506 17:35:05.071 | INFO | Downloaded 34GiB to GPU in 1.91627s (17.7428GiB/s) | 20260506 17:35:10.212 | INFO | Analyzing chunk frames = [3189248,3189760) | 20260506 17:35:10.212 | INFO | MemInfo: { | 20260506 17:35:10.212 | INFO | "gpu" : " 17 GB", | 20260506 17:35:10.212 | INFO | "huge" : "", | 20260506 17:35:10.212 | INFO | "huge_shared" : " 27 GB", | 20260506 17:35:10.212 | INFO | "malloc" : " 14 GB", | 20260506 17:35:10.212 | INFO | "pinned" : " 29 GB" | 20260506 17:35:10.212 | INFO | } | 20260506 17:35:10.212 | INFO | Perf report for the last 5.14177 seconds: | 20260506 17:35:10.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:35:10.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:10.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:10.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:10.212 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 17:35:10.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:10.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:10.212 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:35:10.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:10.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:10.213 | INFO | Uploaded 36GiB to GPU in 2.02119s (17.8113GiB/s) | 20260506 17:35:10.213 | INFO | Downloaded 34GiB to GPU in 1.95555s (17.3864GiB/s) | 20260506 17:35:15.351 | INFO | Analyzing chunk frames = [3189760,3190272) | 20260506 17:35:15.351 | INFO | MemInfo: { | 20260506 17:35:15.351 | INFO | "gpu" : " 17 GB", | 20260506 17:35:15.351 | INFO | "huge" : "", | 20260506 17:35:15.351 | INFO | "huge_shared" : " 27 GB", | 20260506 17:35:15.351 | INFO | "malloc" : " 14 GB", | 20260506 17:35:15.352 | INFO | "pinned" : " 29 GB" | 20260506 17:35:15.352 | INFO | } | 20260506 17:35:15.352 | INFO | Perf report for the last 5.13937 seconds: | 20260506 17:35:15.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:35:15.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:15.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:15.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:15.352 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 17:35:15.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:15.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:15.352 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 17:35:15.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:15.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:15.352 | INFO | Uploaded 36GiB to GPU in 2.00813s (17.9271GiB/s) | 20260506 17:35:15.352 | INFO | Downloaded 34GiB to GPU in 1.98894s (17.0946GiB/s) | 20260506 17:35:20.486 | INFO | Analyzing chunk frames = [3190272,3190784) | 20260506 17:35:20.486 | INFO | MemInfo: { | 20260506 17:35:20.486 | INFO | "gpu" : " 17 GB", | 20260506 17:35:20.486 | INFO | "huge" : "", | 20260506 17:35:20.486 | INFO | "huge_shared" : " 27 GB", | 20260506 17:35:20.486 | INFO | "malloc" : " 14 GB", | 20260506 17:35:20.486 | INFO | "pinned" : " 29 GB" | 20260506 17:35:20.486 | INFO | } | 20260506 17:35:20.486 | INFO | Perf report for the last 5.13442 seconds: | 20260506 17:35:20.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:35:20.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:20.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:20.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:20.486 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 17:35:20.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:20.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:20.486 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:35:20.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:20.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:20.486 | INFO | Uploaded 36GiB to GPU in 2.01729s (17.8458GiB/s) | 20260506 17:35:20.486 | INFO | Downloaded 34GiB to GPU in 1.9864s (17.1164GiB/s) | 20260506 17:35:20.606 | INFO | Starting a round of Baz IO | 20260506 17:35:23.744 | INFO | Populating RTMetrics report | 20260506 17:35:23.744 | INFO | RTMetrics report populated | 20260506 17:35:23.744 | INFO | Opening tmp RTMetrics json file | 20260506 17:35:23.770 | INFO | Serializing RTMetrics to json | 20260506 17:35:23.912 | INFO | Serialization done, swapping tmp file now... | 20260506 17:35:23.973 | INFO | RTMetrics done | 20260506 17:35:24.303 | INFO | Finished a round of Baz IO: Wrote 2.06 GiB in 3.697 seconds | 20260506 17:35:24.303 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.56% / 0.01% / 64.44% | 20260506 17:35:24.303 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.3326% / 1.98682e-05% / 5.16574e-05% | 20260506 17:35:24.303 | INFO | Baz IO overall: | 20260506 17:35:24.303 | INFO | BytesWritten : 1682652872246 | 20260506 17:35:24.303 | INFO | NumEvents : 813997412896 | 20260506 17:35:25.623 | INFO | Analyzing chunk frames = [3190784,3191296) | 20260506 17:35:25.623 | INFO | MemInfo: { | 20260506 17:35:25.623 | INFO | "gpu" : " 17 GB", | 20260506 17:35:25.623 | INFO | "huge" : "", | 20260506 17:35:25.623 | INFO | "huge_shared" : " 27 GB", | 20260506 17:35:25.623 | INFO | "malloc" : " 14 GB", | 20260506 17:35:25.623 | INFO | "pinned" : " 29 GB" | 20260506 17:35:25.623 | INFO | } | 20260506 17:35:25.623 | INFO | Perf report for the last 5.1366 seconds: | 20260506 17:35:25.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:35:25.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:25.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:25.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:25.623 | INFO | ANALYSIS : 62.8%, 2.9, 95.5ms, 0 | 20260506 17:35:25.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:25.623 | INFO | RT_METRICS : 47.9%, 1.0, 25.1ms, 0 | 20260506 17:35:25.623 | INFO | PRE_HQ : 60.2%, 1.9, 58.2ms, 0 | 20260506 17:35:25.623 | INFO | BAZWRITER : 71.6%, 4.8, 176.8ms, 0 | 20260506 17:35:25.623 | INFO | BAZ_SAMPLING : 3.1%, 1.0, 1.6ms, 0 | 20260506 17:35:25.623 | INFO | Uploaded 36GiB to GPU in 2.05154s (17.5478GiB/s) | 20260506 17:35:25.623 | INFO | Downloaded 38GiB to GPU in 2.2249s (17.0794GiB/s) | 20260506 17:35:30.761 | INFO | Analyzing chunk frames = [3191296,3191808) | 20260506 17:35:30.761 | INFO | MemInfo: { | 20260506 17:35:30.761 | INFO | "gpu" : " 17 GB", | 20260506 17:35:30.761 | INFO | "huge" : "", | 20260506 17:35:30.761 | INFO | "huge_shared" : " 27 GB", | 20260506 17:35:30.761 | INFO | "malloc" : " 14 GB", | 20260506 17:35:30.761 | INFO | "pinned" : " 29 GB" | 20260506 17:35:30.761 | INFO | } | 20260506 17:35:30.761 | INFO | Perf report for the last 5.13861 seconds: | 20260506 17:35:30.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:35:30.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:30.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:30.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:30.761 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 17:35:30.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:30.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:30.761 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:35:30.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:30.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:30.761 | INFO | Uploaded 36GiB to GPU in 2.02087s (17.8141GiB/s) | 20260506 17:35:30.762 | INFO | Downloaded 34GiB to GPU in 1.97672s (17.2002GiB/s) | 20260506 17:35:35.901 | INFO | Analyzing chunk frames = [3191808,3192320) | 20260506 17:35:35.901 | INFO | MemInfo: { | 20260506 17:35:35.901 | INFO | "gpu" : " 17 GB", | 20260506 17:35:35.901 | INFO | "huge" : "", | 20260506 17:35:35.901 | INFO | "huge_shared" : " 27 GB", | 20260506 17:35:35.901 | INFO | "malloc" : " 14 GB", | 20260506 17:35:35.901 | INFO | "pinned" : " 29 GB" | 20260506 17:35:35.901 | INFO | } | 20260506 17:35:35.901 | INFO | Perf report for the last 5.14012 seconds: | 20260506 17:35:35.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:35:35.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:35.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:35.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:35.901 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 17:35:35.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:35.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:35.901 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:35:35.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:35.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:35.902 | INFO | Uploaded 36GiB to GPU in 1.99786s (18.0193GiB/s) | 20260506 17:35:35.902 | INFO | Downloaded 34GiB to GPU in 1.92781s (17.6366GiB/s) | 20260506 17:35:41.045 | INFO | Analyzing chunk frames = [3192320,3192832) | 20260506 17:35:41.045 | INFO | MemInfo: { | 20260506 17:35:41.045 | INFO | "gpu" : " 17 GB", | 20260506 17:35:41.045 | INFO | "huge" : "", | 20260506 17:35:41.045 | INFO | "huge_shared" : " 27 GB", | 20260506 17:35:41.045 | INFO | "malloc" : " 14 GB", | 20260506 17:35:41.045 | INFO | "pinned" : " 29 GB" | 20260506 17:35:41.045 | INFO | } | 20260506 17:35:41.045 | INFO | Perf report for the last 5.14344 seconds: | 20260506 17:35:41.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:35:41.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:41.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:41.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:41.045 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:35:41.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:41.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:41.045 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:35:41.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:41.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:41.045 | INFO | Uploaded 36GiB to GPU in 1.92562s (18.6953GiB/s) | 20260506 17:35:41.045 | INFO | Downloaded 34GiB to GPU in 1.87606s (18.1231GiB/s) | 20260506 17:35:46.193 | INFO | Analyzing chunk frames = [3192832,3193344) | 20260506 17:35:46.193 | INFO | MemInfo: { | 20260506 17:35:46.193 | INFO | "gpu" : " 17 GB", | 20260506 17:35:46.193 | INFO | "huge" : "", | 20260506 17:35:46.193 | INFO | "huge_shared" : " 27 GB", | 20260506 17:35:46.193 | INFO | "malloc" : " 14 GB", | 20260506 17:35:46.193 | INFO | "pinned" : " 29 GB" | 20260506 17:35:46.193 | INFO | } | 20260506 17:35:46.193 | INFO | Perf report for the last 5.14784 seconds: | 20260506 17:35:46.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:35:46.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:46.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:46.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:46.193 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 17:35:46.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:46.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:46.193 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:35:46.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:46.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:46.193 | INFO | Uploaded 36GiB to GPU in 2.02076s (17.815GiB/s) | 20260506 17:35:46.193 | INFO | Downloaded 34GiB to GPU in 1.90874s (17.8128GiB/s) | 20260506 17:35:51.326 | INFO | Analyzing chunk frames = [3193344,3193856) | 20260506 17:35:51.326 | INFO | MemInfo: { | 20260506 17:35:51.326 | INFO | "gpu" : " 17 GB", | 20260506 17:35:51.326 | INFO | "huge" : "", | 20260506 17:35:51.326 | INFO | "huge_shared" : " 27 GB", | 20260506 17:35:51.326 | INFO | "malloc" : " 14 GB", | 20260506 17:35:51.326 | INFO | "pinned" : " 29 GB" | 20260506 17:35:51.326 | INFO | } | 20260506 17:35:51.326 | INFO | Perf report for the last 5.13308 seconds: | 20260506 17:35:51.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:35:51.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:51.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:51.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:51.326 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:35:51.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:51.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:51.326 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:35:51.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:51.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:51.326 | INFO | Uploaded 36GiB to GPU in 2.06588s (17.426GiB/s) | 20260506 17:35:51.326 | INFO | Downloaded 34GiB to GPU in 1.95785s (17.366GiB/s) | 20260506 17:35:56.465 | INFO | Analyzing chunk frames = [3193856,3194368) | 20260506 17:35:56.465 | INFO | MemInfo: { | 20260506 17:35:56.465 | INFO | "gpu" : " 17 GB", | 20260506 17:35:56.465 | INFO | "huge" : "", | 20260506 17:35:56.465 | INFO | "huge_shared" : " 27 GB", | 20260506 17:35:56.465 | INFO | "malloc" : " 14 GB", | 20260506 17:35:56.465 | INFO | "pinned" : " 29 GB" | 20260506 17:35:56.465 | INFO | } | 20260506 17:35:56.465 | INFO | Perf report for the last 5.13935 seconds: | 20260506 17:35:56.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:35:56.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:56.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:56.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:56.465 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 17:35:56.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:56.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:56.465 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:35:56.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:56.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:35:56.465 | INFO | Uploaded 36GiB to GPU in 2.0136s (17.8784GiB/s) | 20260506 17:35:56.465 | INFO | Downloaded 34GiB to GPU in 1.97482s (17.2167GiB/s) | 20260506 17:36:01.603 | INFO | Analyzing chunk frames = [3194368,3194880) | 20260506 17:36:01.603 | INFO | MemInfo: { | 20260506 17:36:01.603 | INFO | "gpu" : " 17 GB", | 20260506 17:36:01.603 | INFO | "huge" : "", | 20260506 17:36:01.603 | INFO | "huge_shared" : " 27 GB", | 20260506 17:36:01.603 | INFO | "malloc" : " 14 GB", | 20260506 17:36:01.603 | INFO | "pinned" : " 29 GB" | 20260506 17:36:01.603 | INFO | } | 20260506 17:36:01.603 | INFO | Perf report for the last 5.13831 seconds: | 20260506 17:36:01.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:36:01.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:01.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:01.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:01.604 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 17:36:01.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:01.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:01.604 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 17:36:01.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:01.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:01.604 | INFO | Uploaded 36GiB to GPU in 2.0224s (17.8006GiB/s) | 20260506 17:36:01.604 | INFO | Downloaded 34GiB to GPU in 1.99503s (17.0424GiB/s) | 20260506 17:36:01.604 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:36:01.604 | INFO | Total mem usage: 18 GiB | 20260506 17:36:01.604 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:36:01.604 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -14.7%) | 20260506 17:36:01.604 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:36:01.604 | INFO | High Water Breakdown: | 20260506 17:36:01.604 | INFO | 61.5% : 9.65 GiB ( 84.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:36:01.604 | INFO | 38.4% : 6.03 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:36:01.604 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:36:01.604 | INFO | ------------------------------------------ | 20260506 17:36:01.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:36:01.604 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:36:01.604 | INFO | Total mem usage: 26.5 GiB | 20260506 17:36:01.604 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:36:01.604 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:36:01.604 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:36:01.604 | INFO | High Water Breakdown: | 20260506 17:36:01.604 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:36:01.604 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:36:01.604 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:36:01.604 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:36:01.604 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:36:01.604 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:36:01.604 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:36:01.604 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:36:01.604 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:36:01.604 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:36:01.604 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:36:01.604 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:36:01.604 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:36:01.604 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:36:01.604 | INFO | ------------------------------------------ | 20260506 17:36:01.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:36:01.604 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:36:01.604 | INFO | Total mem usage: 25.6 GiB | 20260506 17:36:01.604 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:36:01.604 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:36:01.604 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:36:01.604 | INFO | High Water Breakdown: | 20260506 17:36:01.604 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:36:01.604 | INFO | ------------------------------------------ | 20260506 17:36:01.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:36:01.604 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:36:01.604 | INFO | Total mem usage: 15.7 GiB | 20260506 17:36:01.604 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:36:01.604 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:36:01.604 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:36:01.604 | INFO | High Water Breakdown: | 20260506 17:36:01.604 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:36:01.604 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:36:01.604 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:36:01.604 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:36:01.604 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:36:01.604 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:36:01.604 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:36:01.604 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:36:01.604 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:36:01.604 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:36:01.604 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:36:01.604 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:36:01.604 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:36:01.604 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:36:01.604 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:36:01.604 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:36:01.604 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:36:01.604 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:36:01.604 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:36:01.604 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:36:01.604 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:36:01.604 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:36:01.604 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:36:01.604 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:36:01.604 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:36:01.604 | INFO | ------------------------------------------ | 20260506 17:36:01.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:36:01.605 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:36:01.605 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:36:01.605 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:36:01.605 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.58e+01, 1.34e+01 / 1.63e+00 | 20260506 17:36:01.605 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 9.96e-02, 1.02e-02 / 5.15e-03 | 20260506 17:36:01.605 | INFO | PerfReport: 11.57%, Baseline -- 3.62e+00 / 4.49e+00, 5.45e-01 / 2.42e-01 | 20260506 17:36:01.605 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.81e-01 / 1.29e-01 | 20260506 17:36:01.605 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.46e+01, 4.10e+00 / 2.10e+00 | 20260506 17:36:01.605 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.48e+00 / 1.69e+00 | 20260506 17:36:01.605 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.27e+00 / 2.88e+00, 3.84e-01 / 2.37e-01 | 20260506 17:36:01.605 | INFO | PerfReport: 3.43%, Metrics -- 1.08e+00 / 1.40e+00, 2.82e-01 / 1.90e-01 | 20260506 17:36:01.605 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.73e-01 / 4.16e-01, 5.87e-02 / 2.77e-02 | 20260506 17:36:01.605 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:36:01.605 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:36:01.605 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:36:01.605 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 17:36:01.605 | INFO | PerfReport: 21.49%, Upload -- 2.09e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 17:36:01.605 | INFO | PerfReport: 21.20%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 17:36:01.605 | INFO | PerfReport: 32.19%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 17:36:01.605 | INFO | PerfReport: 1.44%, LockUpload -- 1.40e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260506 17:36:01.605 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260506 17:36:01.605 | INFO | PerfReport: 22.56%, LockCompute -- 2.20e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 17:36:01.605 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260506 17:36:01.723 | INFO | Starting a round of Baz IO | 20260506 17:36:04.860 | INFO | Populating RTMetrics report | 20260506 17:36:04.860 | INFO | RTMetrics report populated | 20260506 17:36:04.860 | INFO | Opening tmp RTMetrics json file | 20260506 17:36:05.059 | INFO | Serializing RTMetrics to json | 20260506 17:36:05.093 | INFO | Serialization done, swapping tmp file now... | 20260506 17:36:05.096 | INFO | RTMetrics done | 20260506 17:36:05.212 | INFO | Finished a round of Baz IO: Wrote 2.061 GiB in 3.488 seconds | 20260506 17:36:05.212 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.55% / 0.01% / 64.44% | 20260506 17:36:05.212 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.3333% / 1.19209e-05% / 6.35783e-05% | 20260506 17:36:05.212 | INFO | Baz IO overall: | 20260506 17:36:05.212 | INFO | BytesWritten : 1684866221114 | 20260506 17:36:05.212 | INFO | NumEvents : 815065188204 | 20260506 17:36:06.739 | INFO | Analyzing chunk frames = [3194880,3195392) | 20260506 17:36:06.739 | INFO | MemInfo: { | 20260506 17:36:06.739 | INFO | "gpu" : " 17 GB", | 20260506 17:36:06.739 | INFO | "huge" : "", | 20260506 17:36:06.739 | INFO | "huge_shared" : " 27 GB", | 20260506 17:36:06.739 | INFO | "malloc" : " 14 GB", | 20260506 17:36:06.739 | INFO | "pinned" : " 29 GB" | 20260506 17:36:06.739 | INFO | } | 20260506 17:36:06.739 | INFO | Perf report for the last 5.13553 seconds: | 20260506 17:36:06.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:36:06.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:06.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:06.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:06.739 | INFO | ANALYSIS : 62.8%, 2.9, 95.5ms, 0 | 20260506 17:36:06.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:06.739 | INFO | RT_METRICS : 48.2%, 1.0, 25.2ms, 0 | 20260506 17:36:06.739 | INFO | PRE_HQ : 60.3%, 1.9, 58.4ms, 0 | 20260506 17:36:06.739 | INFO | BAZWRITER : 67.6%, 4.8, 166.4ms, 0 | 20260506 17:36:06.739 | INFO | BAZ_SAMPLING : 3.7%, 1.0, 1.9ms, 0 | 20260506 17:36:06.739 | INFO | Uploaded 36GiB to GPU in 2.04101s (17.6384GiB/s) | 20260506 17:36:06.739 | INFO | Downloaded 38GiB to GPU in 2.23237s (17.0223GiB/s) | 20260506 17:36:11.880 | INFO | Analyzing chunk frames = [3195392,3195904) | 20260506 17:36:11.880 | INFO | MemInfo: { | 20260506 17:36:11.880 | INFO | "gpu" : " 17 GB", | 20260506 17:36:11.880 | INFO | "huge" : "", | 20260506 17:36:11.880 | INFO | "huge_shared" : " 27 GB", | 20260506 17:36:11.880 | INFO | "malloc" : " 14 GB", | 20260506 17:36:11.880 | INFO | "pinned" : " 29 GB" | 20260506 17:36:11.880 | INFO | } | 20260506 17:36:11.880 | INFO | Perf report for the last 5.14128 seconds: | 20260506 17:36:11.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:36:11.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:11.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:11.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:11.880 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 17:36:11.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:11.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:11.880 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:36:11.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:11.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:11.881 | INFO | Uploaded 36GiB to GPU in 1.97998s (18.182GiB/s) | 20260506 17:36:11.881 | INFO | Downloaded 34GiB to GPU in 1.93245s (17.5943GiB/s) | 20260506 17:36:17.020 | INFO | Analyzing chunk frames = [3195904,3196416) | 20260506 17:36:17.020 | INFO | MemInfo: { | 20260506 17:36:17.020 | INFO | "gpu" : " 17 GB", | 20260506 17:36:17.020 | INFO | "huge" : "", | 20260506 17:36:17.020 | INFO | "huge_shared" : " 27 GB", | 20260506 17:36:17.020 | INFO | "malloc" : " 14 GB", | 20260506 17:36:17.020 | INFO | "pinned" : " 29 GB" | 20260506 17:36:17.020 | INFO | } | 20260506 17:36:17.020 | INFO | Perf report for the last 5.14004 seconds: | 20260506 17:36:17.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:36:17.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:17.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:17.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:17.020 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 17:36:17.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:17.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:17.020 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:36:17.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:17.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:17.021 | INFO | Uploaded 36GiB to GPU in 1.96637s (18.3079GiB/s) | 20260506 17:36:17.021 | INFO | Downloaded 34GiB to GPU in 1.92425s (17.6692GiB/s) | 20260506 17:36:22.158 | INFO | Analyzing chunk frames = [3196416,3196928) | 20260506 17:36:22.158 | INFO | MemInfo: { | 20260506 17:36:22.158 | INFO | "gpu" : " 17 GB", | 20260506 17:36:22.158 | INFO | "huge" : "", | 20260506 17:36:22.158 | INFO | "huge_shared" : " 27 GB", | 20260506 17:36:22.158 | INFO | "malloc" : " 14 GB", | 20260506 17:36:22.158 | INFO | "pinned" : " 29 GB" | 20260506 17:36:22.159 | INFO | } | 20260506 17:36:22.159 | INFO | Perf report for the last 5.13823 seconds: | 20260506 17:36:22.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:36:22.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:22.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:22.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:22.159 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 17:36:22.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:22.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:22.159 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:36:22.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:22.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:22.159 | INFO | Uploaded 36GiB to GPU in 1.91803s (18.7693GiB/s) | 20260506 17:36:22.159 | INFO | Downloaded 34GiB to GPU in 1.86638s (18.2171GiB/s) | 20260506 17:36:27.299 | INFO | Analyzing chunk frames = [3196928,3197440) | 20260506 17:36:27.299 | INFO | MemInfo: { | 20260506 17:36:27.299 | INFO | "gpu" : " 17 GB", | 20260506 17:36:27.299 | INFO | "huge" : "", | 20260506 17:36:27.299 | INFO | "huge_shared" : " 27 GB", | 20260506 17:36:27.299 | INFO | "malloc" : " 14 GB", | 20260506 17:36:27.299 | INFO | "pinned" : " 29 GB" | 20260506 17:36:27.299 | INFO | } | 20260506 17:36:27.299 | INFO | Perf report for the last 5.14087 seconds: | 20260506 17:36:27.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:36:27.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:27.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:27.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:27.300 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260506 17:36:27.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:27.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:27.300 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:36:27.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:27.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:27.300 | INFO | Uploaded 36GiB to GPU in 1.9382s (18.5739GiB/s) | 20260506 17:36:27.300 | INFO | Downloaded 34GiB to GPU in 1.88394s (18.0473GiB/s) | 20260506 17:36:32.439 | INFO | Analyzing chunk frames = [3197440,3197952) | 20260506 17:36:32.439 | INFO | MemInfo: { | 20260506 17:36:32.439 | INFO | "gpu" : " 17 GB", | 20260506 17:36:32.439 | INFO | "huge" : "", | 20260506 17:36:32.439 | INFO | "huge_shared" : " 27 GB", | 20260506 17:36:32.439 | INFO | "malloc" : " 14 GB", | 20260506 17:36:32.439 | INFO | "pinned" : " 29 GB" | 20260506 17:36:32.439 | INFO | } | 20260506 17:36:32.439 | INFO | Perf report for the last 5.13918 seconds: | 20260506 17:36:32.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:36:32.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:32.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:32.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:32.439 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 17:36:32.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:32.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:32.439 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:36:32.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:32.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:32.439 | INFO | Uploaded 36GiB to GPU in 1.98607s (18.1262GiB/s) | 20260506 17:36:32.439 | INFO | Downloaded 34GiB to GPU in 1.94391s (17.4905GiB/s) | 20260506 17:36:37.579 | INFO | Analyzing chunk frames = [3197952,3198464) | 20260506 17:36:37.579 | INFO | MemInfo: { | 20260506 17:36:37.579 | INFO | "gpu" : " 17 GB", | 20260506 17:36:37.579 | INFO | "huge" : "", | 20260506 17:36:37.579 | INFO | "huge_shared" : " 27 GB", | 20260506 17:36:37.580 | INFO | "malloc" : " 14 GB", | 20260506 17:36:37.580 | INFO | "pinned" : " 29 GB" | 20260506 17:36:37.580 | INFO | } | 20260506 17:36:37.580 | INFO | Perf report for the last 5.14104 seconds: | 20260506 17:36:37.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:36:37.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:37.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:37.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:37.580 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:36:37.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:37.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:37.580 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:36:37.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:37.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:37.580 | INFO | Uploaded 36GiB to GPU in 2.01009s (17.9097GiB/s) | 20260506 17:36:37.580 | INFO | Downloaded 34GiB to GPU in 2.02001s (16.8316GiB/s) | 20260506 17:36:42.714 | INFO | Analyzing chunk frames = [3198464,3198976) | 20260506 17:36:42.714 | INFO | MemInfo: { | 20260506 17:36:42.714 | INFO | "gpu" : " 17 GB", | 20260506 17:36:42.714 | INFO | "huge" : "", | 20260506 17:36:42.715 | INFO | "huge_shared" : " 27 GB", | 20260506 17:36:42.715 | INFO | "malloc" : " 14 GB", | 20260506 17:36:42.715 | INFO | "pinned" : " 29 GB" | 20260506 17:36:42.715 | INFO | } | 20260506 17:36:42.715 | INFO | Perf report for the last 5.1348 seconds: | 20260506 17:36:42.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:36:42.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:42.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:42.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:42.715 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 17:36:42.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:42.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:42.715 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:36:42.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:42.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:42.715 | INFO | Uploaded 36GiB to GPU in 2.0269s (17.7611GiB/s) | 20260506 17:36:42.715 | INFO | Downloaded 34GiB to GPU in 1.99085s (17.0782GiB/s) | 20260506 17:36:42.833 | INFO | Starting a round of Baz IO | 20260506 17:36:45.969 | INFO | Populating RTMetrics report | 20260506 17:36:45.969 | INFO | RTMetrics report populated | 20260506 17:36:45.969 | INFO | Opening tmp RTMetrics json file | 20260506 17:36:45.986 | INFO | Serializing RTMetrics to json | 20260506 17:36:46.018 | INFO | Serialization done, swapping tmp file now... | 20260506 17:36:46.026 | INFO | RTMetrics done | 20260506 17:36:46.375 | INFO | Finished a round of Baz IO: Wrote 2.061 GiB in 3.541 seconds | 20260506 17:36:46.375 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.54% / 0.01% / 64.45% | 20260506 17:36:46.375 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.32981% / 7.94729e-06% / 6.35783e-05% | 20260506 17:36:46.375 | INFO | Baz IO overall: | 20260506 17:36:46.375 | INFO | BytesWritten : 1687079205245 | 20260506 17:36:46.375 | INFO | NumEvents : 816132861875 | 20260506 17:36:47.856 | INFO | Analyzing chunk frames = [3198976,3199488) | 20260506 17:36:47.856 | INFO | MemInfo: { | 20260506 17:36:47.856 | INFO | "gpu" : " 17 GB", | 20260506 17:36:47.856 | INFO | "huge" : "", | 20260506 17:36:47.856 | INFO | "huge_shared" : " 27 GB", | 20260506 17:36:47.856 | INFO | "malloc" : " 14 GB", | 20260506 17:36:47.856 | INFO | "pinned" : " 29 GB" | 20260506 17:36:47.856 | INFO | } | 20260506 17:36:47.856 | INFO | Perf report for the last 5.14123 seconds: | 20260506 17:36:47.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:36:47.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:47.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:47.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:47.856 | INFO | ANALYSIS : 62.7%, 2.9, 95.2ms, 0 | 20260506 17:36:47.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:47.856 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 17:36:47.856 | INFO | PRE_HQ : 60.1%, 1.9, 58.0ms, 0 | 20260506 17:36:47.856 | INFO | BAZWRITER : 68.5%, 4.8, 169.6ms, 0 | 20260506 17:36:47.856 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 17:36:47.856 | INFO | Uploaded 36GiB to GPU in 2.02057s (17.8167GiB/s) | 20260506 17:36:47.856 | INFO | Downloaded 38GiB to GPU in 2.22602s (17.0708GiB/s) | 20260506 17:36:52.995 | INFO | Analyzing chunk frames = [3199488,3200000) | 20260506 17:36:52.995 | INFO | MemInfo: { | 20260506 17:36:52.995 | INFO | "gpu" : " 17 GB", | 20260506 17:36:52.995 | INFO | "huge" : "", | 20260506 17:36:52.995 | INFO | "huge_shared" : " 27 GB", | 20260506 17:36:52.995 | INFO | "malloc" : " 14 GB", | 20260506 17:36:52.995 | INFO | "pinned" : " 29 GB" | 20260506 17:36:52.995 | INFO | } | 20260506 17:36:52.995 | INFO | Perf report for the last 5.13928 seconds: | 20260506 17:36:52.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:36:52.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:52.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:52.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:52.995 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 17:36:52.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:52.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:52.995 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:36:52.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:52.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:52.995 | INFO | Uploaded 36GiB to GPU in 1.97669s (18.2123GiB/s) | 20260506 17:36:52.995 | INFO | Downloaded 34GiB to GPU in 1.9297s (17.6193GiB/s) | 20260506 17:36:58.134 | INFO | Analyzing chunk frames = [3200000,3200512) | 20260506 17:36:58.134 | INFO | MemInfo: { | 20260506 17:36:58.134 | INFO | "gpu" : " 17 GB", | 20260506 17:36:58.134 | INFO | "huge" : "", | 20260506 17:36:58.134 | INFO | "huge_shared" : " 27 GB", | 20260506 17:36:58.134 | INFO | "malloc" : " 14 GB", | 20260506 17:36:58.134 | INFO | "pinned" : " 29 GB" | 20260506 17:36:58.134 | INFO | } | 20260506 17:36:58.134 | INFO | Perf report for the last 5.13929 seconds: | 20260506 17:36:58.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:36:58.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:58.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:58.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:58.134 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260506 17:36:58.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:58.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:58.135 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:36:58.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:58.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:36:58.135 | INFO | Uploaded 36GiB to GPU in 1.92977s (18.6551GiB/s) | 20260506 17:36:58.135 | INFO | Downloaded 34GiB to GPU in 1.87985s (18.0865GiB/s) | 20260506 17:37:03.278 | INFO | Analyzing chunk frames = [3200512,3201024) | 20260506 17:37:03.278 | INFO | MemInfo: { | 20260506 17:37:03.278 | INFO | "gpu" : " 17 GB", | 20260506 17:37:03.278 | INFO | "huge" : "", | 20260506 17:37:03.278 | INFO | "huge_shared" : " 27 GB", | 20260506 17:37:03.278 | INFO | "malloc" : " 14 GB", | 20260506 17:37:03.278 | INFO | "pinned" : " 29 GB" | 20260506 17:37:03.278 | INFO | } | 20260506 17:37:03.278 | INFO | Perf report for the last 5.14332 seconds: | 20260506 17:37:03.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:37:03.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:03.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:03.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:03.278 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:37:03.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:03.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:03.278 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:37:03.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:03.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:03.278 | INFO | Uploaded 36GiB to GPU in 1.9268s (18.6838GiB/s) | 20260506 17:37:03.278 | INFO | Downloaded 34GiB to GPU in 1.87529s (18.1306GiB/s) | 20260506 17:37:08.413 | INFO | Analyzing chunk frames = [3201024,3201536) | 20260506 17:37:08.413 | INFO | MemInfo: { | 20260506 17:37:08.413 | INFO | "gpu" : " 17 GB", | 20260506 17:37:08.413 | INFO | "huge" : "", | 20260506 17:37:08.413 | INFO | "huge_shared" : " 27 GB", | 20260506 17:37:08.413 | INFO | "malloc" : " 14 GB", | 20260506 17:37:08.413 | INFO | "pinned" : " 29 GB" | 20260506 17:37:08.413 | INFO | } | 20260506 17:37:08.414 | INFO | Perf report for the last 5.13576 seconds: | 20260506 17:37:08.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:37:08.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:08.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:08.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:08.414 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 17:37:08.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:08.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:08.414 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 17:37:08.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:08.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:08.414 | INFO | Uploaded 36GiB to GPU in 2.00536s (17.9519GiB/s) | 20260506 17:37:08.414 | INFO | Downloaded 34GiB to GPU in 1.91352s (17.7683GiB/s) | 20260506 17:37:13.554 | INFO | Analyzing chunk frames = [3201536,3202048) | 20260506 17:37:13.554 | INFO | MemInfo: { | 20260506 17:37:13.554 | INFO | "gpu" : " 17 GB", | 20260506 17:37:13.554 | INFO | "huge" : "", | 20260506 17:37:13.554 | INFO | "huge_shared" : " 27 GB", | 20260506 17:37:13.554 | INFO | "malloc" : " 14 GB", | 20260506 17:37:13.554 | INFO | "pinned" : " 29 GB" | 20260506 17:37:13.554 | INFO | } | 20260506 17:37:13.554 | INFO | Perf report for the last 5.14064 seconds: | 20260506 17:37:13.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:37:13.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:13.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:13.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:13.554 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 17:37:13.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:13.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:13.554 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260506 17:37:13.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:13.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:13.554 | INFO | Uploaded 36GiB to GPU in 2.05482s (17.5198GiB/s) | 20260506 17:37:13.554 | INFO | Downloaded 34GiB to GPU in 1.96762s (17.2797GiB/s) | 20260506 17:37:18.695 | INFO | Analyzing chunk frames = [3202048,3202560) | 20260506 17:37:18.695 | INFO | MemInfo: { | 20260506 17:37:18.695 | INFO | "gpu" : " 17 GB", | 20260506 17:37:18.695 | INFO | "huge" : "", | 20260506 17:37:18.695 | INFO | "huge_shared" : " 27 GB", | 20260506 17:37:18.695 | INFO | "malloc" : " 14 GB", | 20260506 17:37:18.695 | INFO | "pinned" : " 29 GB" | 20260506 17:37:18.695 | INFO | } | 20260506 17:37:18.695 | INFO | Perf report for the last 5.14079 seconds: | 20260506 17:37:18.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:37:18.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:18.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:18.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:18.695 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 17:37:18.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:18.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:18.695 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 17:37:18.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:18.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:18.695 | INFO | Uploaded 36GiB to GPU in 2.02298s (17.7955GiB/s) | 20260506 17:37:18.695 | INFO | Downloaded 34GiB to GPU in 2.00619s (16.9476GiB/s) | 20260506 17:37:23.831 | INFO | Analyzing chunk frames = [3202560,3203072) | 20260506 17:37:23.831 | INFO | MemInfo: { | 20260506 17:37:23.831 | INFO | "gpu" : " 17 GB", | 20260506 17:37:23.831 | INFO | "huge" : "", | 20260506 17:37:23.831 | INFO | "huge_shared" : " 27 GB", | 20260506 17:37:23.831 | INFO | "malloc" : " 14 GB", | 20260506 17:37:23.831 | INFO | "pinned" : " 29 GB" | 20260506 17:37:23.831 | INFO | } | 20260506 17:37:23.831 | INFO | Perf report for the last 5.13592 seconds: | 20260506 17:37:23.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:37:23.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:23.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:23.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:23.831 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 17:37:23.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:23.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:23.831 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:37:23.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:23.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:23.831 | INFO | Uploaded 36GiB to GPU in 2.01413s (17.8737GiB/s) | 20260506 17:37:23.831 | INFO | Downloaded 34GiB to GPU in 1.9846s (17.1319GiB/s) | 20260506 17:37:23.831 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:37:23.831 | INFO | Total mem usage: 18 GiB | 20260506 17:37:23.831 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:37:23.831 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.5%) | 20260506 17:37:23.831 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:37:23.831 | INFO | High Water Breakdown: | 20260506 17:37:23.831 | INFO | 61.8% : 9.9 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:37:23.831 | INFO | 38% : 6.09 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:37:23.831 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:37:23.831 | INFO | ------------------------------------------ | 20260506 17:37:23.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:37:23.831 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:37:23.831 | INFO | Total mem usage: 26.5 GiB | 20260506 17:37:23.831 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:37:23.831 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:37:23.831 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:37:23.831 | INFO | High Water Breakdown: | 20260506 17:37:23.831 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:37:23.831 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:37:23.831 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:37:23.831 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:37:23.831 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:37:23.831 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:37:23.831 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:37:23.831 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:37:23.831 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:37:23.832 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:37:23.832 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:37:23.832 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:37:23.832 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:37:23.832 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:37:23.832 | INFO | ------------------------------------------ | 20260506 17:37:23.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:37:23.832 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:37:23.832 | INFO | Total mem usage: 25.6 GiB | 20260506 17:37:23.832 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:37:23.832 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:37:23.832 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:37:23.832 | INFO | High Water Breakdown: | 20260506 17:37:23.832 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:37:23.832 | INFO | ------------------------------------------ | 20260506 17:37:23.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:37:23.832 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:37:23.832 | INFO | Total mem usage: 15.7 GiB | 20260506 17:37:23.832 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:37:23.832 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:37:23.832 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 17:37:23.832 | INFO | High Water Breakdown: | 20260506 17:37:23.832 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:37:23.832 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:37:23.832 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:37:23.832 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:37:23.832 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:37:23.832 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:37:23.832 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:37:23.832 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:37:23.832 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:37:23.832 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:37:23.832 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:37:23.832 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:37:23.832 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:37:23.832 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:37:23.832 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:37:23.832 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:37:23.832 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:37:23.832 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:37:23.832 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:37:23.832 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:37:23.832 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:37:23.832 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:37:23.832 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:37:23.832 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:37:23.832 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:37:23.832 | INFO | ------------------------------------------ | 20260506 17:37:23.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:37:23.832 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:37:23.832 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:37:23.832 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:37:23.832 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.55e+01, 1.34e+01 / 2.39e+00 | 20260506 17:37:23.832 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 9.96e-02, 9.87e-03 / 5.98e-03 | 20260506 17:37:23.832 | INFO | PerfReport: 11.60%, Baseline -- 3.63e+00 / 4.39e+00, 5.59e-01 / 3.80e-01 | 20260506 17:37:23.832 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.43e+00, 1.81e-01 / 1.29e-01 | 20260506 17:37:23.832 | INFO | PerfReport: 15.90%, DME -- 3.99e+01 / 4.45e+01, 3.91e+00 / 2.40e+00 | 20260506 17:37:23.832 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.79e+01 / 2.03e+01, 1.47e+00 / 1.90e+00 | 20260506 17:37:23.832 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.27e+00 / 2.86e+00, 3.84e-01 / 2.04e-01 | 20260506 17:37:23.832 | INFO | PerfReport: 3.44%, Metrics -- 1.08e+00 / 1.46e+00, 2.81e-01 / 1.76e-01 | 20260506 17:37:23.832 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.72e-01 / 4.20e-01, 5.83e-02 / 1.67e-02 | 20260506 17:37:23.832 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:37:23.832 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:37:23.832 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:37:23.832 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 17:37:23.832 | INFO | PerfReport: 21.35%, Upload -- 2.07e+01 / 0.00e+00, 6.83e+00 / 0.00e+00 | 20260506 17:37:23.832 | INFO | PerfReport: 21.19%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 17:37:23.832 | INFO | PerfReport: 32.29%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:37:23.832 | INFO | PerfReport: 1.24%, LockUpload -- 1.20e+00 / 0.00e+00, 5.43e+00 / 0.00e+00 | 20260506 17:37:23.832 | INFO | PerfReport: 1.21%, LockDownload -- 1.17e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260506 17:37:23.832 | INFO | PerfReport: 22.71%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 17:37:23.832 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260506 17:37:23.970 | INFO | Starting a round of Baz IO | 20260506 17:37:27.081 | INFO | Populating RTMetrics report | 20260506 17:37:27.081 | INFO | RTMetrics report populated | 20260506 17:37:27.081 | INFO | Opening tmp RTMetrics json file | 20260506 17:37:27.087 | INFO | Serializing RTMetrics to json | 20260506 17:37:27.104 | INFO | Serialization done, swapping tmp file now... | 20260506 17:37:27.121 | INFO | RTMetrics done | 20260506 17:37:27.521 | INFO | Finished a round of Baz IO: Wrote 2.06 GiB in 3.551 seconds | 20260506 17:37:27.522 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.53% / 0.01% / 64.46% | 20260506 17:37:27.522 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.32559% / 0% / 6.35783e-05% | 20260506 17:37:27.522 | INFO | Baz IO overall: | 20260506 17:37:27.522 | INFO | BytesWritten : 1689291131231 | 20260506 17:37:27.522 | INFO | NumEvents : 817199701580 | 20260506 17:37:28.970 | INFO | Analyzing chunk frames = [3203072,3203584) | 20260506 17:37:28.970 | INFO | MemInfo: { | 20260506 17:37:28.970 | INFO | "gpu" : " 17 GB", | 20260506 17:37:28.970 | INFO | "huge" : "", | 20260506 17:37:28.970 | INFO | "huge_shared" : " 27 GB", | 20260506 17:37:28.970 | INFO | "malloc" : " 14 GB", | 20260506 17:37:28.970 | INFO | "pinned" : " 29 GB" | 20260506 17:37:28.970 | INFO | } | 20260506 17:37:28.970 | INFO | Perf report for the last 5.13949 seconds: | 20260506 17:37:28.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:37:28.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:28.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:28.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:28.970 | INFO | ANALYSIS : 62.7%, 2.9, 95.4ms, 0 | 20260506 17:37:28.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:28.970 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 17:37:28.970 | INFO | PRE_HQ : 60.1%, 1.9, 58.2ms, 0 | 20260506 17:37:28.970 | INFO | BAZWRITER : 69.1%, 4.8, 169.4ms, 0 | 20260506 17:37:28.970 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 17:37:28.971 | INFO | Uploaded 36GiB to GPU in 2.05808s (17.492GiB/s) | 20260506 17:37:28.971 | INFO | Downloaded 38GiB to GPU in 2.21639s (17.145GiB/s) | 20260506 17:37:34.105 | INFO | Analyzing chunk frames = [3203584,3204096) | 20260506 17:37:34.105 | INFO | MemInfo: { | 20260506 17:37:34.105 | INFO | "gpu" : " 17 GB", | 20260506 17:37:34.105 | INFO | "huge" : "", | 20260506 17:37:34.105 | INFO | "huge_shared" : " 27 GB", | 20260506 17:37:34.105 | INFO | "malloc" : " 14 GB", | 20260506 17:37:34.105 | INFO | "pinned" : " 29 GB" | 20260506 17:37:34.106 | INFO | } | 20260506 17:37:34.106 | INFO | Perf report for the last 5.13516 seconds: | 20260506 17:37:34.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:37:34.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:34.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:34.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:34.106 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:37:34.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:34.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:34.106 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:37:34.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:34.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:34.106 | INFO | Uploaded 36GiB to GPU in 2.00151s (17.9864GiB/s) | 20260506 17:37:34.106 | INFO | Downloaded 34GiB to GPU in 1.93663s (17.5563GiB/s) | 20260506 17:37:39.247 | INFO | Analyzing chunk frames = [3204096,3204608) | 20260506 17:37:39.247 | INFO | MemInfo: { | 20260506 17:37:39.247 | INFO | "gpu" : " 17 GB", | 20260506 17:37:39.247 | INFO | "huge" : "", | 20260506 17:37:39.247 | INFO | "huge_shared" : " 27 GB", | 20260506 17:37:39.247 | INFO | "malloc" : " 14 GB", | 20260506 17:37:39.247 | INFO | "pinned" : " 29 GB" | 20260506 17:37:39.247 | INFO | } | 20260506 17:37:39.247 | INFO | Perf report for the last 5.14117 seconds: | 20260506 17:37:39.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:37:39.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:39.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:39.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:39.247 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 17:37:39.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:39.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:39.247 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:37:39.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:39.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:39.247 | INFO | Uploaded 36GiB to GPU in 1.93852s (18.5709GiB/s) | 20260506 17:37:39.247 | INFO | Downloaded 34GiB to GPU in 1.88322s (18.0542GiB/s) | 20260506 17:37:44.389 | INFO | Analyzing chunk frames = [3204608,3205120) | 20260506 17:37:44.389 | INFO | MemInfo: { | 20260506 17:37:44.389 | INFO | "gpu" : " 17 GB", | 20260506 17:37:44.389 | INFO | "huge" : "", | 20260506 17:37:44.389 | INFO | "huge_shared" : " 27 GB", | 20260506 17:37:44.389 | INFO | "malloc" : " 14 GB", | 20260506 17:37:44.389 | INFO | "pinned" : " 29 GB" | 20260506 17:37:44.389 | INFO | } | 20260506 17:37:44.389 | INFO | Perf report for the last 5.14205 seconds: | 20260506 17:37:44.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:37:44.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:44.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:44.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:44.389 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 17:37:44.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:44.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:44.389 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:37:44.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:44.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:44.389 | INFO | Uploaded 36GiB to GPU in 1.93384s (18.6158GiB/s) | 20260506 17:37:44.389 | INFO | Downloaded 34GiB to GPU in 1.87451s (18.1381GiB/s) | 20260506 17:37:49.529 | INFO | Analyzing chunk frames = [3205120,3205632) | 20260506 17:37:49.529 | INFO | MemInfo: { | 20260506 17:37:49.529 | INFO | "gpu" : " 17 GB", | 20260506 17:37:49.529 | INFO | "huge" : "", | 20260506 17:37:49.529 | INFO | "huge_shared" : " 27 GB", | 20260506 17:37:49.529 | INFO | "malloc" : " 14 GB", | 20260506 17:37:49.529 | INFO | "pinned" : " 29 GB" | 20260506 17:37:49.529 | INFO | } | 20260506 17:37:49.529 | INFO | Perf report for the last 5.14012 seconds: | 20260506 17:37:49.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:37:49.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:49.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:49.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:49.529 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 17:37:49.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:49.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:49.529 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:37:49.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:49.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:49.529 | INFO | Uploaded 36GiB to GPU in 2.02021s (17.8199GiB/s) | 20260506 17:37:49.529 | INFO | Downloaded 34GiB to GPU in 1.91238s (17.7789GiB/s) | 20260506 17:37:54.670 | INFO | Analyzing chunk frames = [3205632,3206144) | 20260506 17:37:54.670 | INFO | MemInfo: { | 20260506 17:37:54.670 | INFO | "gpu" : " 17 GB", | 20260506 17:37:54.670 | INFO | "huge" : "", | 20260506 17:37:54.670 | INFO | "huge_shared" : " 27 GB", | 20260506 17:37:54.670 | INFO | "malloc" : " 14 GB", | 20260506 17:37:54.670 | INFO | "pinned" : " 29 GB" | 20260506 17:37:54.670 | INFO | } | 20260506 17:37:54.670 | INFO | Perf report for the last 5.14153 seconds: | 20260506 17:37:54.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:37:54.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:54.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:54.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:54.671 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:37:54.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:54.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:54.671 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 17:37:54.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:54.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:54.671 | INFO | Uploaded 36GiB to GPU in 2.0743s (17.3553GiB/s) | 20260506 17:37:54.671 | INFO | Downloaded 34GiB to GPU in 1.96214s (17.328GiB/s) | 20260506 17:37:59.808 | INFO | Analyzing chunk frames = [3206144,3206656) | 20260506 17:37:59.808 | INFO | MemInfo: { | 20260506 17:37:59.808 | INFO | "gpu" : " 17 GB", | 20260506 17:37:59.808 | INFO | "huge" : "", | 20260506 17:37:59.808 | INFO | "huge_shared" : " 27 GB", | 20260506 17:37:59.808 | INFO | "malloc" : " 14 GB", | 20260506 17:37:59.808 | INFO | "pinned" : " 29 GB" | 20260506 17:37:59.808 | INFO | } | 20260506 17:37:59.808 | INFO | Perf report for the last 5.1378 seconds: | 20260506 17:37:59.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:37:59.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:59.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:59.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:59.808 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 17:37:59.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:59.808 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:59.808 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 17:37:59.808 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:59.808 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:37:59.809 | INFO | Uploaded 36GiB to GPU in 2.02147s (17.8088GiB/s) | 20260506 17:37:59.809 | INFO | Downloaded 34GiB to GPU in 2.01145s (16.9032GiB/s) | 20260506 17:38:04.952 | INFO | Analyzing chunk frames = [3206656,3207168) | 20260506 17:38:04.952 | INFO | MemInfo: { | 20260506 17:38:04.952 | INFO | "gpu" : " 17 GB", | 20260506 17:38:04.952 | INFO | "huge" : "", | 20260506 17:38:04.952 | INFO | "huge_shared" : " 27 GB", | 20260506 17:38:04.952 | INFO | "malloc" : " 14 GB", | 20260506 17:38:04.952 | INFO | "pinned" : " 29 GB" | 20260506 17:38:04.952 | INFO | } | 20260506 17:38:04.952 | INFO | Perf report for the last 5.14341 seconds: | 20260506 17:38:04.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:38:04.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:04.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:04.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:04.952 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:38:04.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:04.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:04.952 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 17:38:04.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:04.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:04.952 | INFO | Uploaded 36GiB to GPU in 1.99617s (18.0346GiB/s) | 20260506 17:38:04.952 | INFO | Downloaded 34GiB to GPU in 1.96646s (17.29GiB/s) | 20260506 17:38:05.068 | INFO | Starting a round of Baz IO | 20260506 17:38:08.233 | INFO | Populating RTMetrics report | 20260506 17:38:08.234 | INFO | RTMetrics report populated | 20260506 17:38:08.234 | INFO | Opening tmp RTMetrics json file | 20260506 17:38:08.273 | INFO | Serializing RTMetrics to json | 20260506 17:38:08.320 | INFO | Serialization done, swapping tmp file now... | 20260506 17:38:08.322 | INFO | RTMetrics done | 20260506 17:38:08.794 | INFO | Finished a round of Baz IO: Wrote 2.061 GiB in 3.726 seconds | 20260506 17:38:08.794 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.53% / 0.01% / 64.47% | 20260506 17:38:08.794 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.32363% / 1.58946e-05% / 5.16574e-05% | 20260506 17:38:08.795 | INFO | Baz IO overall: | 20260506 17:38:08.795 | INFO | BytesWritten : 1691504245626 | 20260506 17:38:08.795 | INFO | NumEvents : 818267329192 | 20260506 17:38:10.083 | INFO | Analyzing chunk frames = [3207168,3207680) | 20260506 17:38:10.083 | INFO | MemInfo: { | 20260506 17:38:10.083 | INFO | "gpu" : " 17 GB", | 20260506 17:38:10.083 | INFO | "huge" : "", | 20260506 17:38:10.083 | INFO | "huge_shared" : " 27 GB", | 20260506 17:38:10.083 | INFO | "malloc" : " 14 GB", | 20260506 17:38:10.083 | INFO | "pinned" : " 29 GB" | 20260506 17:38:10.083 | INFO | } | 20260506 17:38:10.083 | INFO | Perf report for the last 5.1312 seconds: | 20260506 17:38:10.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:38:10.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:10.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:10.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:10.083 | INFO | ANALYSIS : 62.8%, 2.9, 95.0ms, 0 | 20260506 17:38:10.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:10.083 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260506 17:38:10.083 | INFO | PRE_HQ : 61.0%, 1.9, 60.6ms, 0 | 20260506 17:38:10.083 | INFO | BAZWRITER : 72.2%, 4.8, 178.1ms, 0 | 20260506 17:38:10.083 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260506 17:38:10.083 | INFO | Uploaded 36GiB to GPU in 2.07718s (17.3312GiB/s) | 20260506 17:38:10.083 | INFO | Downloaded 38GiB to GPU in 2.19855s (17.2841GiB/s) | 20260506 17:38:15.221 | INFO | Analyzing chunk frames = [3207680,3208192) | 20260506 17:38:15.221 | INFO | MemInfo: { | 20260506 17:38:15.221 | INFO | "gpu" : " 17 GB", | 20260506 17:38:15.221 | INFO | "huge" : "", | 20260506 17:38:15.221 | INFO | "huge_shared" : " 27 GB", | 20260506 17:38:15.221 | INFO | "malloc" : " 14 GB", | 20260506 17:38:15.221 | INFO | "pinned" : " 29 GB" | 20260506 17:38:15.221 | INFO | } | 20260506 17:38:15.221 | INFO | Perf report for the last 5.13837 seconds: | 20260506 17:38:15.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:38:15.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:15.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:15.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:15.221 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:38:15.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:15.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:15.221 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 17:38:15.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:15.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:15.222 | INFO | Uploaded 36GiB to GPU in 1.99645s (18.032GiB/s) | 20260506 17:38:15.222 | INFO | Downloaded 34GiB to GPU in 1.95032s (17.4331GiB/s) | 20260506 17:38:20.367 | INFO | Analyzing chunk frames = [3208192,3208704) | 20260506 17:38:20.367 | INFO | MemInfo: { | 20260506 17:38:20.367 | INFO | "gpu" : " 17 GB", | 20260506 17:38:20.367 | INFO | "huge" : "", | 20260506 17:38:20.367 | INFO | "huge_shared" : " 27 GB", | 20260506 17:38:20.367 | INFO | "malloc" : " 14 GB", | 20260506 17:38:20.367 | INFO | "pinned" : " 29 GB" | 20260506 17:38:20.367 | INFO | } | 20260506 17:38:20.367 | INFO | Perf report for the last 5.14562 seconds: | 20260506 17:38:20.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:38:20.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:20.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:20.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:20.367 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 17:38:20.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:20.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:20.367 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:38:20.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:20.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:20.367 | INFO | Uploaded 36GiB to GPU in 1.9923s (18.0696GiB/s) | 20260506 17:38:20.367 | INFO | Downloaded 34GiB to GPU in 1.92247s (17.6855GiB/s) | 20260506 17:38:25.506 | INFO | Analyzing chunk frames = [3208704,3209216) | 20260506 17:38:25.506 | INFO | MemInfo: { | 20260506 17:38:25.506 | INFO | "gpu" : " 17 GB", | 20260506 17:38:25.507 | INFO | "huge" : "", | 20260506 17:38:25.507 | INFO | "huge_shared" : " 27 GB", | 20260506 17:38:25.507 | INFO | "malloc" : " 14 GB", | 20260506 17:38:25.507 | INFO | "pinned" : " 29 GB" | 20260506 17:38:25.507 | INFO | } | 20260506 17:38:25.507 | INFO | Perf report for the last 5.13977 seconds: | 20260506 17:38:25.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:38:25.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:25.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:25.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:25.507 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:38:25.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:25.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:25.507 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:38:25.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:25.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:25.507 | INFO | Uploaded 36GiB to GPU in 1.94353s (18.523GiB/s) | 20260506 17:38:25.507 | INFO | Downloaded 34GiB to GPU in 1.87568s (18.1267GiB/s) | 20260506 17:38:30.646 | INFO | Analyzing chunk frames = [3209216,3209728) | 20260506 17:38:30.646 | INFO | MemInfo: { | 20260506 17:38:30.646 | INFO | "gpu" : " 17 GB", | 20260506 17:38:30.646 | INFO | "huge" : "", | 20260506 17:38:30.646 | INFO | "huge_shared" : " 27 GB", | 20260506 17:38:30.646 | INFO | "malloc" : " 14 GB", | 20260506 17:38:30.646 | INFO | "pinned" : " 29 GB" | 20260506 17:38:30.646 | INFO | } | 20260506 17:38:30.646 | INFO | Perf report for the last 5.13953 seconds: | 20260506 17:38:30.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:38:30.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:30.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:30.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:30.646 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 17:38:30.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:30.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:30.646 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:38:30.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:30.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:30.647 | INFO | Uploaded 36GiB to GPU in 2.01044s (17.9065GiB/s) | 20260506 17:38:30.647 | INFO | Downloaded 34GiB to GPU in 1.91071s (17.7945GiB/s) | 20260506 17:38:35.783 | INFO | Analyzing chunk frames = [3209728,3210240) | 20260506 17:38:35.783 | INFO | MemInfo: { | 20260506 17:38:35.783 | INFO | "gpu" : " 17 GB", | 20260506 17:38:35.783 | INFO | "huge" : "", | 20260506 17:38:35.783 | INFO | "huge_shared" : " 27 GB", | 20260506 17:38:35.783 | INFO | "malloc" : " 14 GB", | 20260506 17:38:35.783 | INFO | "pinned" : " 29 GB" | 20260506 17:38:35.783 | INFO | } | 20260506 17:38:35.783 | INFO | Perf report for the last 5.13713 seconds: | 20260506 17:38:35.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:38:35.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:35.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:35.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:35.783 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 17:38:35.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:35.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:35.783 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:38:35.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:35.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:35.784 | INFO | Uploaded 36GiB to GPU in 2.02886s (17.7439GiB/s) | 20260506 17:38:35.784 | INFO | Downloaded 34GiB to GPU in 1.94031s (17.523GiB/s) | 20260506 17:38:40.921 | INFO | Analyzing chunk frames = [3210240,3210752) | 20260506 17:38:40.921 | INFO | MemInfo: { | 20260506 17:38:40.921 | INFO | "gpu" : " 17 GB", | 20260506 17:38:40.921 | INFO | "huge" : "", | 20260506 17:38:40.921 | INFO | "huge_shared" : " 27 GB", | 20260506 17:38:40.921 | INFO | "malloc" : " 14 GB", | 20260506 17:38:40.921 | INFO | "pinned" : " 29 GB" | 20260506 17:38:40.921 | INFO | } | 20260506 17:38:40.921 | INFO | Perf report for the last 5.13728 seconds: | 20260506 17:38:40.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:38:40.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:40.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:40.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:40.921 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:38:40.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:40.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:40.921 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 17:38:40.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:40.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:40.921 | INFO | Uploaded 36GiB to GPU in 2.00825s (17.926GiB/s) | 20260506 17:38:40.921 | INFO | Downloaded 34GiB to GPU in 1.98624s (17.1178GiB/s) | 20260506 17:38:46.059 | INFO | Analyzing chunk frames = [3210752,3211264) | 20260506 17:38:46.060 | INFO | MemInfo: { | 20260506 17:38:46.060 | INFO | "gpu" : " 17 GB", | 20260506 17:38:46.060 | INFO | "huge" : "", | 20260506 17:38:46.060 | INFO | "huge_shared" : " 27 GB", | 20260506 17:38:46.060 | INFO | "malloc" : " 14 GB", | 20260506 17:38:46.060 | INFO | "pinned" : " 29 GB" | 20260506 17:38:46.060 | INFO | } | 20260506 17:38:46.060 | INFO | Perf report for the last 5.13889 seconds: | 20260506 17:38:46.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:38:46.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:46.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:46.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:46.060 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 17:38:46.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:46.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:46.060 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 17:38:46.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:46.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:46.060 | INFO | Uploaded 36GiB to GPU in 2.077s (17.3327GiB/s) | 20260506 17:38:46.060 | INFO | Downloaded 34GiB to GPU in 1.96871s (17.2702GiB/s) | 20260506 17:38:46.060 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:38:46.060 | INFO | Total mem usage: 18 GiB | 20260506 17:38:46.060 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:38:46.060 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -11.6%) | 20260506 17:38:46.060 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:38:46.060 | INFO | High Water Breakdown: | 20260506 17:38:46.060 | INFO | 61.9% : 9.99 GiB ( 87.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:38:46.060 | INFO | 38% : 6.13 GiB ( 91.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:38:46.060 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:38:46.060 | INFO | ------------------------------------------ | 20260506 17:38:46.060 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:38:46.060 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:38:46.060 | INFO | Total mem usage: 26.5 GiB | 20260506 17:38:46.060 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:38:46.060 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:38:46.060 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:38:46.060 | INFO | High Water Breakdown: | 20260506 17:38:46.060 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:38:46.060 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:38:46.060 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:38:46.060 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:38:46.060 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:38:46.060 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:38:46.060 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:38:46.060 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:38:46.060 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:38:46.060 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:38:46.060 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:38:46.060 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:38:46.060 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:38:46.060 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:38:46.060 | INFO | ------------------------------------------ | 20260506 17:38:46.060 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:38:46.060 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:38:46.060 | INFO | Total mem usage: 25.6 GiB | 20260506 17:38:46.060 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:38:46.060 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:38:46.060 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:38:46.060 | INFO | High Water Breakdown: | 20260506 17:38:46.060 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:38:46.060 | INFO | ------------------------------------------ | 20260506 17:38:46.060 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:38:46.060 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:38:46.060 | INFO | Total mem usage: 15.7 GiB | 20260506 17:38:46.060 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:38:46.060 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:38:46.060 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 17:38:46.060 | INFO | High Water Breakdown: | 20260506 17:38:46.060 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:38:46.060 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:38:46.060 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:38:46.060 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:38:46.060 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:38:46.060 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:38:46.060 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:38:46.060 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:38:46.060 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:38:46.060 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:38:46.060 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:38:46.060 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:38:46.060 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:38:46.060 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:38:46.061 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:38:46.061 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:38:46.061 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:38:46.061 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:38:46.061 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:38:46.061 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:38:46.061 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:38:46.061 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:38:46.061 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:38:46.061 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:38:46.061 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:38:46.061 | INFO | ------------------------------------------ | 20260506 17:38:46.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:38:46.061 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:38:46.061 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:38:46.061 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:38:46.061 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.54e+01, 1.35e+01 / 2.20e+00 | 20260506 17:38:46.061 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.01e-01, 1.03e-02 / 5.90e-03 | 20260506 17:38:46.061 | INFO | PerfReport: 11.60%, Baseline -- 3.64e+00 / 4.46e+00, 5.55e-01 / 2.92e-01 | 20260506 17:38:46.061 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.85e-01 / 1.28e-01 | 20260506 17:38:46.061 | INFO | PerfReport: 15.92%, DME -- 3.99e+01 / 4.44e+01, 3.89e+00 / 2.27e+00 | 20260506 17:38:46.061 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.79e+01 / 2.02e+01, 1.48e+00 / 1.65e+00 | 20260506 17:38:46.061 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.27e+00 / 2.86e+00, 3.79e-01 / 1.97e-01 | 20260506 17:38:46.061 | INFO | PerfReport: 3.43%, Metrics -- 1.08e+00 / 1.45e+00, 2.79e-01 / 1.81e-01 | 20260506 17:38:46.061 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.73e-01 / 4.22e-01, 5.83e-02 / 1.90e-02 | 20260506 17:38:46.061 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:38:46.061 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:38:46.061 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:38:46.061 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 17:38:46.061 | INFO | PerfReport: 21.55%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 17:38:46.061 | INFO | PerfReport: 21.10%, Download -- 2.05e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260506 17:38:46.061 | INFO | PerfReport: 32.24%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 17:38:46.061 | INFO | PerfReport: 1.46%, LockUpload -- 1.42e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260506 17:38:46.061 | INFO | PerfReport: 1.03%, LockDownload -- 1.00e+00 / 0.00e+00, 2.60e+00 / 0.00e+00 | 20260506 17:38:46.061 | INFO | PerfReport: 22.61%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 17:38:46.061 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.44e-02 / 0.00e+00 | 20260506 17:38:46.187 | INFO | Starting a round of Baz IO | 20260506 17:38:49.321 | INFO | Populating RTMetrics report | 20260506 17:38:49.322 | INFO | RTMetrics report populated | 20260506 17:38:49.322 | INFO | Opening tmp RTMetrics json file | 20260506 17:38:49.323 | INFO | Serializing RTMetrics to json | 20260506 17:38:49.325 | INFO | Serialization done, swapping tmp file now... | 20260506 17:38:49.329 | INFO | RTMetrics done | 20260506 17:38:49.498 | INFO | Finished a round of Baz IO: Wrote 2.061 GiB in 3.31 seconds | 20260506 17:38:49.498 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.52% / 0.01% / 64.47% | 20260506 17:38:49.498 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.3215% / 0% / 7.15256e-05% | 20260506 17:38:49.498 | INFO | Baz IO overall: | 20260506 17:38:49.498 | INFO | BytesWritten : 1693716814853 | 20260506 17:38:49.498 | INFO | NumEvents : 819334819955 | 20260506 17:38:51.200 | INFO | Analyzing chunk frames = [3211264,3211776) | 20260506 17:38:51.200 | INFO | MemInfo: { | 20260506 17:38:51.200 | INFO | "gpu" : " 17 GB", | 20260506 17:38:51.200 | INFO | "huge" : "", | 20260506 17:38:51.200 | INFO | "huge_shared" : " 27 GB", | 20260506 17:38:51.200 | INFO | "malloc" : " 14 GB", | 20260506 17:38:51.200 | INFO | "pinned" : " 29 GB" | 20260506 17:38:51.200 | INFO | } | 20260506 17:38:51.200 | INFO | Perf report for the last 5.14016 seconds: | 20260506 17:38:51.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:38:51.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:51.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:51.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:51.200 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 17:38:51.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:51.200 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 17:38:51.200 | INFO | PRE_HQ : 60.5%, 1.9, 58.6ms, 0 | 20260506 17:38:51.200 | INFO | BAZWRITER : 64.1%, 4.6, 152.6ms, 0 | 20260506 17:38:51.200 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 17:38:51.200 | INFO | Uploaded 36GiB to GPU in 2.06645s (17.4212GiB/s) | 20260506 17:38:51.200 | INFO | Downloaded 38GiB to GPU in 2.23019s (17.0389GiB/s) | 20260506 17:38:56.339 | INFO | Analyzing chunk frames = [3211776,3212288) | 20260506 17:38:56.339 | INFO | MemInfo: { | 20260506 17:38:56.339 | INFO | "gpu" : " 17 GB", | 20260506 17:38:56.339 | INFO | "huge" : "", | 20260506 17:38:56.339 | INFO | "huge_shared" : " 27 GB", | 20260506 17:38:56.339 | INFO | "malloc" : " 14 GB", | 20260506 17:38:56.339 | INFO | "pinned" : " 29 GB" | 20260506 17:38:56.339 | INFO | } | 20260506 17:38:56.339 | INFO | Perf report for the last 5.13915 seconds: | 20260506 17:38:56.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:38:56.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:56.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:56.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:56.339 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:38:56.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:56.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:56.339 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:38:56.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:56.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:38:56.339 | INFO | Uploaded 36GiB to GPU in 2.0141s (17.874GiB/s) | 20260506 17:38:56.339 | INFO | Downloaded 34GiB to GPU in 1.941s (17.5167GiB/s) | 20260506 17:39:01.477 | INFO | Analyzing chunk frames = [3212288,3212800) | 20260506 17:39:01.477 | INFO | MemInfo: { | 20260506 17:39:01.477 | INFO | "gpu" : " 17 GB", | 20260506 17:39:01.477 | INFO | "huge" : "", | 20260506 17:39:01.477 | INFO | "huge_shared" : " 27 GB", | 20260506 17:39:01.477 | INFO | "malloc" : " 14 GB", | 20260506 17:39:01.477 | INFO | "pinned" : " 29 GB" | 20260506 17:39:01.477 | INFO | } | 20260506 17:39:01.477 | INFO | Perf report for the last 5.13804 seconds: | 20260506 17:39:01.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:39:01.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:01.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:01.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:01.477 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:39:01.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:01.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:01.477 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:39:01.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:01.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:01.477 | INFO | Uploaded 36GiB to GPU in 1.96408s (18.3292GiB/s) | 20260506 17:39:01.477 | INFO | Downloaded 34GiB to GPU in 1.92258s (17.6845GiB/s) | 20260506 17:39:06.625 | INFO | Analyzing chunk frames = [3212800,3213312) | 20260506 17:39:06.625 | INFO | MemInfo: { | 20260506 17:39:06.625 | INFO | "gpu" : " 17 GB", | 20260506 17:39:06.625 | INFO | "huge" : "", | 20260506 17:39:06.625 | INFO | "huge_shared" : " 27 GB", | 20260506 17:39:06.625 | INFO | "malloc" : " 14 GB", | 20260506 17:39:06.625 | INFO | "pinned" : " 29 GB" | 20260506 17:39:06.625 | INFO | } | 20260506 17:39:06.625 | INFO | Perf report for the last 5.1479 seconds: | 20260506 17:39:06.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:39:06.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:06.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:06.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:06.625 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260506 17:39:06.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:06.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:06.625 | INFO | PRE_HQ : 26.6%, 1.0, 13.9ms, 0 | 20260506 17:39:06.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:06.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:06.625 | INFO | Uploaded 36GiB to GPU in 1.9401s (18.5557GiB/s) | 20260506 17:39:06.625 | INFO | Downloaded 34GiB to GPU in 1.87657s (18.1181GiB/s) | 20260506 17:39:11.763 | INFO | Analyzing chunk frames = [3213312,3213824) | 20260506 17:39:11.763 | INFO | MemInfo: { | 20260506 17:39:11.763 | INFO | "gpu" : " 17 GB", | 20260506 17:39:11.763 | INFO | "huge" : "", | 20260506 17:39:11.763 | INFO | "huge_shared" : " 27 GB", | 20260506 17:39:11.763 | INFO | "malloc" : " 14 GB", | 20260506 17:39:11.763 | INFO | "pinned" : " 29 GB" | 20260506 17:39:11.763 | INFO | } | 20260506 17:39:11.763 | INFO | Perf report for the last 5.13817 seconds: | 20260506 17:39:11.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:39:11.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:11.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:11.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:11.763 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 17:39:11.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:11.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:11.763 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 17:39:11.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:11.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:11.763 | INFO | Uploaded 36GiB to GPU in 2.01586s (17.8583GiB/s) | 20260506 17:39:11.763 | INFO | Downloaded 34GiB to GPU in 1.90644s (17.8343GiB/s) | 20260506 17:39:16.898 | INFO | Analyzing chunk frames = [3213824,3214336) | 20260506 17:39:16.898 | INFO | MemInfo: { | 20260506 17:39:16.898 | INFO | "gpu" : " 17 GB", | 20260506 17:39:16.898 | INFO | "huge" : "", | 20260506 17:39:16.898 | INFO | "huge_shared" : " 27 GB", | 20260506 17:39:16.898 | INFO | "malloc" : " 14 GB", | 20260506 17:39:16.898 | INFO | "pinned" : " 29 GB" | 20260506 17:39:16.898 | INFO | } | 20260506 17:39:16.898 | INFO | Perf report for the last 5.13514 seconds: | 20260506 17:39:16.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:39:16.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:16.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:16.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:16.898 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:39:16.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:16.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:16.898 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 17:39:16.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:16.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:16.899 | INFO | Uploaded 36GiB to GPU in 2.07333s (17.3634GiB/s) | 20260506 17:39:16.899 | INFO | Downloaded 34GiB to GPU in 1.9602s (17.3452GiB/s) | 20260506 17:39:22.037 | INFO | Analyzing chunk frames = [3214336,3214848) | 20260506 17:39:22.037 | INFO | MemInfo: { | 20260506 17:39:22.037 | INFO | "gpu" : " 17 GB", | 20260506 17:39:22.037 | INFO | "huge" : "", | 20260506 17:39:22.037 | INFO | "huge_shared" : " 27 GB", | 20260506 17:39:22.037 | INFO | "malloc" : " 14 GB", | 20260506 17:39:22.037 | INFO | "pinned" : " 29 GB" | 20260506 17:39:22.037 | INFO | } | 20260506 17:39:22.037 | INFO | Perf report for the last 5.13898 seconds: | 20260506 17:39:22.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:39:22.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:22.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:22.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:22.037 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:39:22.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:22.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:22.037 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 17:39:22.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:22.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:22.037 | INFO | Uploaded 36GiB to GPU in 1.9752s (18.226GiB/s) | 20260506 17:39:22.038 | INFO | Downloaded 34GiB to GPU in 1.97165s (17.2445GiB/s) | 20260506 17:39:27.172 | INFO | Analyzing chunk frames = [3214848,3215360) | 20260506 17:39:27.172 | INFO | MemInfo: { | 20260506 17:39:27.172 | INFO | "gpu" : " 17 GB", | 20260506 17:39:27.172 | INFO | "huge" : "", | 20260506 17:39:27.172 | INFO | "huge_shared" : " 27 GB", | 20260506 17:39:27.172 | INFO | "malloc" : " 14 GB", | 20260506 17:39:27.172 | INFO | "pinned" : " 29 GB" | 20260506 17:39:27.172 | INFO | } | 20260506 17:39:27.172 | INFO | Perf report for the last 5.13471 seconds: | 20260506 17:39:27.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:39:27.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:27.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:27.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:27.172 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 17:39:27.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:27.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:27.172 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 17:39:27.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:27.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:27.172 | INFO | Uploaded 36GiB to GPU in 2.00266s (17.9761GiB/s) | 20260506 17:39:27.172 | INFO | Downloaded 34GiB to GPU in 1.99767s (17.0199GiB/s) | 20260506 17:39:27.291 | INFO | Starting a round of Baz IO | 20260506 17:39:30.424 | INFO | Populating RTMetrics report | 20260506 17:39:30.425 | INFO | RTMetrics report populated | 20260506 17:39:30.425 | INFO | Opening tmp RTMetrics json file | 20260506 17:39:30.451 | INFO | Serializing RTMetrics to json | 20260506 17:39:30.456 | INFO | Serialization done, swapping tmp file now... | 20260506 17:39:30.461 | INFO | RTMetrics done | 20260506 17:39:30.905 | INFO | Finished a round of Baz IO: Wrote 2.061 GiB in 3.613 seconds | 20260506 17:39:30.905 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.51% / 0.01% / 64.48% | 20260506 17:39:30.905 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.32115% / 3.97364e-06% / 5.5631e-05% | 20260506 17:39:30.905 | INFO | Baz IO overall: | 20260506 17:39:30.905 | INFO | BytesWritten : 1695930171642 | 20260506 17:39:30.905 | INFO | NumEvents : 820402841455 | 20260506 17:39:32.312 | INFO | Analyzing chunk frames = [3215360,3215872) | 20260506 17:39:32.312 | INFO | MemInfo: { | 20260506 17:39:32.312 | INFO | "gpu" : " 17 GB", | 20260506 17:39:32.312 | INFO | "huge" : "", | 20260506 17:39:32.312 | INFO | "huge_shared" : " 27 GB", | 20260506 17:39:32.312 | INFO | "malloc" : " 14 GB", | 20260506 17:39:32.312 | INFO | "pinned" : " 29 GB" | 20260506 17:39:32.312 | INFO | } | 20260506 17:39:32.312 | INFO | Perf report for the last 5.13972 seconds: | 20260506 17:39:32.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:39:32.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:32.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:32.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:32.312 | INFO | ANALYSIS : 62.7%, 2.9, 95.7ms, 0 | 20260506 17:39:32.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:32.312 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 17:39:32.312 | INFO | PRE_HQ : 60.1%, 1.9, 58.0ms, 0 | 20260506 17:39:32.312 | INFO | BAZWRITER : 69.9%, 4.8, 172.3ms, 0 | 20260506 17:39:32.312 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260506 17:39:32.312 | INFO | Uploaded 36GiB to GPU in 2.05141s (17.5489GiB/s) | 20260506 17:39:32.312 | INFO | Downloaded 38GiB to GPU in 2.21454s (17.1593GiB/s) | 20260506 17:39:37.456 | INFO | Analyzing chunk frames = [3215872,3216384) | 20260506 17:39:37.456 | INFO | MemInfo: { | 20260506 17:39:37.456 | INFO | "gpu" : " 17 GB", | 20260506 17:39:37.456 | INFO | "huge" : "", | 20260506 17:39:37.456 | INFO | "huge_shared" : " 27 GB", | 20260506 17:39:37.456 | INFO | "malloc" : " 14 GB", | 20260506 17:39:37.457 | INFO | "pinned" : " 29 GB" | 20260506 17:39:37.457 | INFO | } | 20260506 17:39:37.457 | INFO | Perf report for the last 5.14483 seconds: | 20260506 17:39:37.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:39:37.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:37.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:37.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:37.457 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 17:39:37.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:37.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:37.457 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:39:37.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:37.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:37.457 | INFO | Uploaded 36GiB to GPU in 1.99016s (18.089GiB/s) | 20260506 17:39:37.457 | INFO | Downloaded 34GiB to GPU in 1.94763s (17.4571GiB/s) | 20260506 17:39:42.591 | INFO | Analyzing chunk frames = [3216384,3216896) | 20260506 17:39:42.591 | INFO | MemInfo: { | 20260506 17:39:42.591 | INFO | "gpu" : " 17 GB", | 20260506 17:39:42.591 | INFO | "huge" : "", | 20260506 17:39:42.591 | INFO | "huge_shared" : " 27 GB", | 20260506 17:39:42.591 | INFO | "malloc" : " 14 GB", | 20260506 17:39:42.591 | INFO | "pinned" : " 29 GB" | 20260506 17:39:42.591 | INFO | } | 20260506 17:39:42.591 | INFO | Perf report for the last 5.13493 seconds: | 20260506 17:39:42.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:39:42.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:42.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:42.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:42.591 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:39:42.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:42.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:42.591 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:39:42.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:42.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:42.592 | INFO | Uploaded 36GiB to GPU in 1.96497s (18.3209GiB/s) | 20260506 17:39:42.592 | INFO | Downloaded 34GiB to GPU in 1.92446s (17.6673GiB/s) | 20260506 17:39:47.732 | INFO | Analyzing chunk frames = [3216896,3217408) | 20260506 17:39:47.732 | INFO | MemInfo: { | 20260506 17:39:47.732 | INFO | "gpu" : " 17 GB", | 20260506 17:39:47.732 | INFO | "huge" : "", | 20260506 17:39:47.732 | INFO | "huge_shared" : " 27 GB", | 20260506 17:39:47.732 | INFO | "malloc" : " 14 GB", | 20260506 17:39:47.732 | INFO | "pinned" : " 29 GB" | 20260506 17:39:47.732 | INFO | } | 20260506 17:39:47.732 | INFO | Perf report for the last 5.14071 seconds: | 20260506 17:39:47.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:39:47.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:47.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:47.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:47.732 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:39:47.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:47.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:47.732 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:39:47.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:47.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:47.732 | INFO | Uploaded 36GiB to GPU in 1.92239s (18.7267GiB/s) | 20260506 17:39:47.733 | INFO | Downloaded 34GiB to GPU in 1.87733s (18.1108GiB/s) | 20260506 17:39:52.872 | INFO | Analyzing chunk frames = [3217408,3217920) | 20260506 17:39:52.872 | INFO | MemInfo: { | 20260506 17:39:52.872 | INFO | "gpu" : " 17 GB", | 20260506 17:39:52.872 | INFO | "huge" : "", | 20260506 17:39:52.872 | INFO | "huge_shared" : " 27 GB", | 20260506 17:39:52.872 | INFO | "malloc" : " 14 GB", | 20260506 17:39:52.872 | INFO | "pinned" : " 29 GB" | 20260506 17:39:52.872 | INFO | } | 20260506 17:39:52.872 | INFO | Perf report for the last 5.13984 seconds: | 20260506 17:39:52.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:39:52.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:52.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:52.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:52.872 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260506 17:39:52.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:52.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:52.872 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:39:52.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:52.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:52.872 | INFO | Uploaded 36GiB to GPU in 1.93957s (18.5608GiB/s) | 20260506 17:39:52.872 | INFO | Downloaded 34GiB to GPU in 1.8871s (18.017GiB/s) | 20260506 17:39:58.016 | INFO | Analyzing chunk frames = [3217920,3218432) | 20260506 17:39:58.016 | INFO | MemInfo: { | 20260506 17:39:58.016 | INFO | "gpu" : " 17 GB", | 20260506 17:39:58.016 | INFO | "huge" : "", | 20260506 17:39:58.016 | INFO | "huge_shared" : " 27 GB", | 20260506 17:39:58.016 | INFO | "malloc" : " 14 GB", | 20260506 17:39:58.016 | INFO | "pinned" : " 29 GB" | 20260506 17:39:58.016 | INFO | } | 20260506 17:39:58.016 | INFO | Perf report for the last 5.14422 seconds: | 20260506 17:39:58.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:39:58.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:58.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:58.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:58.016 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 17:39:58.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:58.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:58.016 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 17:39:58.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:58.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:39:58.017 | INFO | Uploaded 36GiB to GPU in 2.00164s (17.9853GiB/s) | 20260506 17:39:58.017 | INFO | Downloaded 34GiB to GPU in 1.96405s (17.3112GiB/s) | 20260506 17:40:03.152 | INFO | Analyzing chunk frames = [3218432,3218944) | 20260506 17:40:03.152 | INFO | MemInfo: { | 20260506 17:40:03.152 | INFO | "gpu" : " 17 GB", | 20260506 17:40:03.152 | INFO | "huge" : "", | 20260506 17:40:03.152 | INFO | "huge_shared" : " 27 GB", | 20260506 17:40:03.152 | INFO | "malloc" : " 14 GB", | 20260506 17:40:03.152 | INFO | "pinned" : " 29 GB" | 20260506 17:40:03.152 | INFO | } | 20260506 17:40:03.152 | INFO | Perf report for the last 5.13606 seconds: | 20260506 17:40:03.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:40:03.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:03.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:03.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:03.152 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 17:40:03.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:03.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:03.152 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:40:03.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:03.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:03.153 | INFO | Uploaded 36GiB to GPU in 2.01026s (17.9082GiB/s) | 20260506 17:40:03.153 | INFO | Downloaded 34GiB to GPU in 2.01222s (16.8968GiB/s) | 20260506 17:40:08.289 | INFO | Analyzing chunk frames = [3218944,3219456) | 20260506 17:40:08.289 | INFO | MemInfo: { | 20260506 17:40:08.289 | INFO | "gpu" : " 17 GB", | 20260506 17:40:08.289 | INFO | "huge" : "", | 20260506 17:40:08.289 | INFO | "huge_shared" : " 27 GB", | 20260506 17:40:08.289 | INFO | "malloc" : " 14 GB", | 20260506 17:40:08.289 | INFO | "pinned" : " 29 GB" | 20260506 17:40:08.289 | INFO | } | 20260506 17:40:08.289 | INFO | Perf report for the last 5.13634 seconds: | 20260506 17:40:08.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:40:08.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:08.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:08.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:08.289 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 17:40:08.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:08.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:08.289 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260506 17:40:08.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:08.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:08.289 | INFO | Uploaded 36GiB to GPU in 2.05568s (17.5124GiB/s) | 20260506 17:40:08.289 | INFO | Downloaded 34GiB to GPU in 1.97902s (17.1802GiB/s) | 20260506 17:40:08.289 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:40:08.289 | INFO | Total mem usage: 18 GiB | 20260506 17:40:08.289 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:40:08.289 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -17.3%) | 20260506 17:40:08.289 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:40:08.289 | INFO | High Water Breakdown: | 20260506 17:40:08.289 | INFO | 61.1% : 9.37 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:40:08.289 | INFO | 38.8% : 5.95 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:40:08.289 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:40:08.289 | INFO | ------------------------------------------ | 20260506 17:40:08.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:40:08.289 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:40:08.289 | INFO | Total mem usage: 26.5 GiB | 20260506 17:40:08.289 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:40:08.289 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:40:08.289 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:40:08.289 | INFO | High Water Breakdown: | 20260506 17:40:08.289 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:40:08.289 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:40:08.289 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:40:08.289 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:40:08.289 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:40:08.289 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:40:08.289 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:40:08.289 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:40:08.289 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:40:08.289 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:40:08.289 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:40:08.289 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:40:08.289 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:40:08.289 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:40:08.289 | INFO | ------------------------------------------ | 20260506 17:40:08.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:40:08.289 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:40:08.289 | INFO | Total mem usage: 25.6 GiB | 20260506 17:40:08.289 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:40:08.289 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:40:08.289 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:40:08.289 | INFO | High Water Breakdown: | 20260506 17:40:08.289 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:40:08.289 | INFO | ------------------------------------------ | 20260506 17:40:08.289 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:40:08.289 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:40:08.289 | INFO | Total mem usage: 15.7 GiB | 20260506 17:40:08.289 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:40:08.289 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:40:08.289 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:40:08.289 | INFO | High Water Breakdown: | 20260506 17:40:08.289 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:40:08.289 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:40:08.289 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:40:08.290 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:40:08.290 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:40:08.290 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:40:08.290 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:40:08.290 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:40:08.290 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:40:08.290 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:40:08.290 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:40:08.290 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:40:08.290 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:40:08.290 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:40:08.290 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:40:08.290 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:40:08.290 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:40:08.290 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:40:08.290 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:40:08.290 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:40:08.290 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:40:08.290 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:40:08.290 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:40:08.290 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:40:08.290 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:40:08.290 | INFO | ------------------------------------------ | 20260506 17:40:08.290 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:40:08.290 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:40:08.290 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:40:08.290 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:40:08.290 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.55e+01, 1.33e+01 / 2.29e+00 | 20260506 17:40:08.290 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.01e-01, 9.96e-03 / 5.64e-03 | 20260506 17:40:08.290 | INFO | PerfReport: 11.61%, Baseline -- 3.64e+00 / 4.42e+00, 5.59e-01 / 3.44e-01 | 20260506 17:40:08.290 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.44e+00, 1.82e-01 / 1.30e-01 | 20260506 17:40:08.290 | INFO | PerfReport: 15.94%, DME -- 4.00e+01 / 4.45e+01, 4.23e+00 / 2.33e+00 | 20260506 17:40:08.290 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.43e+00 / 1.82e+00 | 20260506 17:40:08.290 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.28e+00 / 2.84e+00, 3.88e-01 / 2.16e-01 | 20260506 17:40:08.290 | INFO | PerfReport: 3.44%, Metrics -- 1.08e+00 / 1.39e+00, 2.84e-01 / 1.41e-01 | 20260506 17:40:08.290 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.73e-01 / 4.20e-01, 5.91e-02 / 2.34e-02 | 20260506 17:40:08.290 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:40:08.290 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:40:08.290 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:40:08.290 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 17:40:08.290 | INFO | PerfReport: 21.42%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 17:40:08.290 | INFO | PerfReport: 21.17%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 17:40:08.290 | INFO | PerfReport: 32.24%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 17:40:08.290 | INFO | PerfReport: 1.39%, LockUpload -- 1.35e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260506 17:40:08.290 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260506 17:40:08.290 | INFO | PerfReport: 22.66%, LockCompute -- 2.20e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 17:40:08.290 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.28e-02 / 0.00e+00 | 20260506 17:40:08.403 | INFO | Starting a round of Baz IO | 20260506 17:40:11.552 | INFO | Populating RTMetrics report | 20260506 17:40:11.553 | INFO | RTMetrics report populated | 20260506 17:40:11.553 | INFO | Opening tmp RTMetrics json file | 20260506 17:40:11.689 | INFO | Serializing RTMetrics to json | 20260506 17:40:11.723 | INFO | Serialization done, swapping tmp file now... | 20260506 17:40:11.734 | INFO | RTMetrics done | 20260506 17:40:12.145 | INFO | Finished a round of Baz IO: Wrote 2.06 GiB in 3.741 seconds | 20260506 17:40:12.145 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.51% / 0.01% / 64.49% | 20260506 17:40:12.145 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.31687% / 1.19209e-05% / 4.76837e-05% | 20260506 17:40:12.145 | INFO | Baz IO overall: | 20260506 17:40:12.145 | INFO | BytesWritten : 1698142605446 | 20260506 17:40:12.145 | INFO | NumEvents : 821470385919 | 20260506 17:40:13.429 | INFO | Analyzing chunk frames = [3219456,3219968) | 20260506 17:40:13.429 | INFO | MemInfo: { | 20260506 17:40:13.429 | INFO | "gpu" : " 17 GB", | 20260506 17:40:13.429 | INFO | "huge" : "", | 20260506 17:40:13.429 | INFO | "huge_shared" : " 27 GB", | 20260506 17:40:13.429 | INFO | "malloc" : " 14 GB", | 20260506 17:40:13.429 | INFO | "pinned" : " 29 GB" | 20260506 17:40:13.429 | INFO | } | 20260506 17:40:13.429 | INFO | Perf report for the last 5.14074 seconds: | 20260506 17:40:13.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:40:13.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:13.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:13.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:13.430 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 17:40:13.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:13.430 | INFO | RT_METRICS : 47.0%, 1.0, 24.6ms, 0 | 20260506 17:40:13.430 | INFO | PRE_HQ : 60.8%, 1.9, 58.6ms, 0 | 20260506 17:40:13.430 | INFO | BAZWRITER : 72.2%, 4.9, 179.9ms, 0 | 20260506 17:40:13.430 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 17:40:13.430 | INFO | Uploaded 36GiB to GPU in 2.08543s (17.2626GiB/s) | 20260506 17:40:13.430 | INFO | Downloaded 38GiB to GPU in 2.23367s (17.0123GiB/s) | 20260506 17:40:18.565 | INFO | Analyzing chunk frames = [3219968,3220480) | 20260506 17:40:18.565 | INFO | MemInfo: { | 20260506 17:40:18.565 | INFO | "gpu" : " 17 GB", | 20260506 17:40:18.565 | INFO | "huge" : "", | 20260506 17:40:18.566 | INFO | "huge_shared" : " 27 GB", | 20260506 17:40:18.566 | INFO | "malloc" : " 14 GB", | 20260506 17:40:18.566 | INFO | "pinned" : " 29 GB" | 20260506 17:40:18.566 | INFO | } | 20260506 17:40:18.566 | INFO | Perf report for the last 5.13613 seconds: | 20260506 17:40:18.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:40:18.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:18.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:18.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:18.566 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:40:18.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:18.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:18.566 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 17:40:18.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:18.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:18.566 | INFO | Uploaded 36GiB to GPU in 1.99925s (18.0067GiB/s) | 20260506 17:40:18.566 | INFO | Downloaded 34GiB to GPU in 1.92733s (17.641GiB/s) | 20260506 17:40:23.708 | INFO | Analyzing chunk frames = [3220480,3220992) | 20260506 17:40:23.708 | INFO | MemInfo: { | 20260506 17:40:23.708 | INFO | "gpu" : " 17 GB", | 20260506 17:40:23.708 | INFO | "huge" : "", | 20260506 17:40:23.708 | INFO | "huge_shared" : " 27 GB", | 20260506 17:40:23.708 | INFO | "malloc" : " 14 GB", | 20260506 17:40:23.708 | INFO | "pinned" : " 29 GB" | 20260506 17:40:23.708 | INFO | } | 20260506 17:40:23.709 | INFO | Perf report for the last 5.14288 seconds: | 20260506 17:40:23.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:40:23.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:23.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:23.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:23.709 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 17:40:23.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:23.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:23.709 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:40:23.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:23.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:23.709 | INFO | Uploaded 36GiB to GPU in 1.93956s (18.5609GiB/s) | 20260506 17:40:23.709 | INFO | Downloaded 34GiB to GPU in 1.88556s (18.0318GiB/s) | 20260506 17:40:28.848 | INFO | Analyzing chunk frames = [3220992,3221504) | 20260506 17:40:28.848 | INFO | MemInfo: { | 20260506 17:40:28.848 | INFO | "gpu" : " 17 GB", | 20260506 17:40:28.848 | INFO | "huge" : "", | 20260506 17:40:28.848 | INFO | "huge_shared" : " 27 GB", | 20260506 17:40:28.848 | INFO | "malloc" : " 14 GB", | 20260506 17:40:28.848 | INFO | "pinned" : " 29 GB" | 20260506 17:40:28.848 | INFO | } | 20260506 17:40:28.848 | INFO | Perf report for the last 5.13979 seconds: | 20260506 17:40:28.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:40:28.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:28.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:28.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:28.848 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:40:28.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:28.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:28.848 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 17:40:28.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:28.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:28.849 | INFO | Uploaded 36GiB to GPU in 1.9361s (18.594GiB/s) | 20260506 17:40:28.849 | INFO | Downloaded 34GiB to GPU in 1.87254s (18.1571GiB/s) | 20260506 17:40:33.988 | INFO | Analyzing chunk frames = [3221504,3222016) | 20260506 17:40:33.988 | INFO | MemInfo: { | 20260506 17:40:33.988 | INFO | "gpu" : " 17 GB", | 20260506 17:40:33.988 | INFO | "huge" : "", | 20260506 17:40:33.988 | INFO | "huge_shared" : " 27 GB", | 20260506 17:40:33.988 | INFO | "malloc" : " 14 GB", | 20260506 17:40:33.988 | INFO | "pinned" : " 29 GB" | 20260506 17:40:33.988 | INFO | } | 20260506 17:40:33.988 | INFO | Perf report for the last 5.13947 seconds: | 20260506 17:40:33.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:40:33.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:33.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:33.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:33.988 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 17:40:33.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:33.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:33.988 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:40:33.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:33.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:33.988 | INFO | Uploaded 36GiB to GPU in 2.01754s (17.8435GiB/s) | 20260506 17:40:33.988 | INFO | Downloaded 34GiB to GPU in 1.90825s (17.8173GiB/s) | 20260506 17:40:39.131 | INFO | Analyzing chunk frames = [3222016,3222528) | 20260506 17:40:39.131 | INFO | MemInfo: { | 20260506 17:40:39.131 | INFO | "gpu" : " 17 GB", | 20260506 17:40:39.131 | INFO | "huge" : "", | 20260506 17:40:39.131 | INFO | "huge_shared" : " 27 GB", | 20260506 17:40:39.131 | INFO | "malloc" : " 14 GB", | 20260506 17:40:39.131 | INFO | "pinned" : " 29 GB" | 20260506 17:40:39.131 | INFO | } | 20260506 17:40:39.131 | INFO | Perf report for the last 5.14341 seconds: | 20260506 17:40:39.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:40:39.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:39.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:39.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:39.131 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 17:40:39.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:39.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:39.131 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:40:39.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:39.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:39.131 | INFO | Uploaded 36GiB to GPU in 2.03741s (17.6695GiB/s) | 20260506 17:40:39.132 | INFO | Downloaded 34GiB to GPU in 1.94566s (17.4748GiB/s) | 20260506 17:40:44.268 | INFO | Analyzing chunk frames = [3222528,3223040) | 20260506 17:40:44.268 | INFO | MemInfo: { | 20260506 17:40:44.268 | INFO | "gpu" : " 17 GB", | 20260506 17:40:44.268 | INFO | "huge" : "", | 20260506 17:40:44.268 | INFO | "huge_shared" : " 27 GB", | 20260506 17:40:44.268 | INFO | "malloc" : " 14 GB", | 20260506 17:40:44.268 | INFO | "pinned" : " 29 GB" | 20260506 17:40:44.268 | INFO | } | 20260506 17:40:44.268 | INFO | Perf report for the last 5.13659 seconds: | 20260506 17:40:44.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:40:44.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:44.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:44.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:44.268 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 17:40:44.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:44.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:44.268 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:40:44.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:44.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:44.268 | INFO | Uploaded 36GiB to GPU in 2.03987s (17.6482GiB/s) | 20260506 17:40:44.268 | INFO | Downloaded 34GiB to GPU in 1.99219s (17.0667GiB/s) | 20260506 17:40:49.406 | INFO | Analyzing chunk frames = [3223040,3223552) | 20260506 17:40:49.406 | INFO | MemInfo: { | 20260506 17:40:49.406 | INFO | "gpu" : " 17 GB", | 20260506 17:40:49.406 | INFO | "huge" : "", | 20260506 17:40:49.406 | INFO | "huge_shared" : " 27 GB", | 20260506 17:40:49.406 | INFO | "malloc" : " 14 GB", | 20260506 17:40:49.406 | INFO | "pinned" : " 29 GB" | 20260506 17:40:49.406 | INFO | } | 20260506 17:40:49.406 | INFO | Perf report for the last 5.13811 seconds: | 20260506 17:40:49.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:40:49.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:49.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:49.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:49.406 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 17:40:49.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:49.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:49.406 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:40:49.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:49.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:49.406 | INFO | Uploaded 36GiB to GPU in 2.02217s (17.8027GiB/s) | 20260506 17:40:49.406 | INFO | Downloaded 34GiB to GPU in 1.99568s (17.0368GiB/s) | 20260506 17:40:49.525 | INFO | Starting a round of Baz IO | 20260506 17:40:52.665 | INFO | Populating RTMetrics report | 20260506 17:40:52.666 | INFO | RTMetrics report populated | 20260506 17:40:52.666 | INFO | Opening tmp RTMetrics json file | 20260506 17:40:52.669 | INFO | Serializing RTMetrics to json | 20260506 17:40:52.671 | INFO | Serialization done, swapping tmp file now... | 20260506 17:40:52.673 | INFO | RTMetrics done | 20260506 17:40:52.807 | INFO | Finished a round of Baz IO: Wrote 2.058 GiB in 3.282 seconds | 20260506 17:40:52.807 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.50% / 0.01% / 64.49% | 20260506 17:40:52.807 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.30928% / 3.97364e-06% / 5.96046e-05% | 20260506 17:40:52.807 | INFO | Baz IO overall: | 20260506 17:40:52.807 | INFO | BytesWritten : 1700351916541 | 20260506 17:40:52.807 | INFO | NumEvents : 822535582024 | 20260506 17:40:54.538 | INFO | Analyzing chunk frames = [3223552,3224064) | 20260506 17:40:54.538 | INFO | MemInfo: { | 20260506 17:40:54.538 | INFO | "gpu" : " 17 GB", | 20260506 17:40:54.538 | INFO | "huge" : "", | 20260506 17:40:54.538 | INFO | "huge_shared" : " 27 GB", | 20260506 17:40:54.538 | INFO | "malloc" : " 14 GB", | 20260506 17:40:54.538 | INFO | "pinned" : " 29 GB" | 20260506 17:40:54.538 | INFO | } | 20260506 17:40:54.538 | INFO | Perf report for the last 5.13231 seconds: | 20260506 17:40:54.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:40:54.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:54.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:54.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:54.538 | INFO | ANALYSIS : 62.9%, 2.9, 95.5ms, 0 | 20260506 17:40:54.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:54.538 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 17:40:54.538 | INFO | PRE_HQ : 60.3%, 1.9, 58.2ms, 0 | 20260506 17:40:54.538 | INFO | BAZWRITER : 63.5%, 4.7, 152.2ms, 0 | 20260506 17:40:54.538 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 17:40:54.539 | INFO | Uploaded 36GiB to GPU in 2.04005s (17.6466GiB/s) | 20260506 17:40:54.539 | INFO | Downloaded 38GiB to GPU in 2.22795s (17.056GiB/s) | 20260506 17:40:59.679 | INFO | Analyzing chunk frames = [3224064,3224576) | 20260506 17:40:59.680 | INFO | MemInfo: { | 20260506 17:40:59.680 | INFO | "gpu" : " 17 GB", | 20260506 17:40:59.680 | INFO | "huge" : "", | 20260506 17:40:59.680 | INFO | "huge_shared" : " 27 GB", | 20260506 17:40:59.680 | INFO | "malloc" : " 14 GB", | 20260506 17:40:59.680 | INFO | "pinned" : " 29 GB" | 20260506 17:40:59.680 | INFO | } | 20260506 17:40:59.680 | INFO | Perf report for the last 5.14136 seconds: | 20260506 17:40:59.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:40:59.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:59.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:59.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:59.680 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 17:40:59.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:59.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:59.680 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:40:59.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:59.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:40:59.680 | INFO | Uploaded 36GiB to GPU in 2.01242s (17.8889GiB/s) | 20260506 17:40:59.680 | INFO | Downloaded 34GiB to GPU in 1.9753s (17.2126GiB/s) | 20260506 17:41:04.825 | INFO | Analyzing chunk frames = [3224576,3225088) | 20260506 17:41:04.825 | INFO | MemInfo: { | 20260506 17:41:04.825 | INFO | "gpu" : " 17 GB", | 20260506 17:41:04.825 | INFO | "huge" : "", | 20260506 17:41:04.825 | INFO | "huge_shared" : " 27 GB", | 20260506 17:41:04.825 | INFO | "malloc" : " 14 GB", | 20260506 17:41:04.825 | INFO | "pinned" : " 29 GB" | 20260506 17:41:04.825 | INFO | } | 20260506 17:41:04.825 | INFO | Perf report for the last 5.14541 seconds: | 20260506 17:41:04.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:41:04.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:04.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:04.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:04.825 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 17:41:04.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:04.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:04.825 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:41:04.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:04.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:04.825 | INFO | Uploaded 36GiB to GPU in 1.97881s (18.1927GiB/s) | 20260506 17:41:04.825 | INFO | Downloaded 34GiB to GPU in 1.92938s (17.6223GiB/s) | 20260506 17:41:09.966 | INFO | Analyzing chunk frames = [3225088,3225600) | 20260506 17:41:09.966 | INFO | MemInfo: { | 20260506 17:41:09.966 | INFO | "gpu" : " 17 GB", | 20260506 17:41:09.966 | INFO | "huge" : "", | 20260506 17:41:09.966 | INFO | "huge_shared" : " 27 GB", | 20260506 17:41:09.966 | INFO | "malloc" : " 14 GB", | 20260506 17:41:09.966 | INFO | "pinned" : " 29 GB" | 20260506 17:41:09.966 | INFO | } | 20260506 17:41:09.966 | INFO | Perf report for the last 5.1411 seconds: | 20260506 17:41:09.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:41:09.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:09.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:09.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:09.966 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 17:41:09.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:09.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:09.966 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:41:09.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:09.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:09.966 | INFO | Uploaded 36GiB to GPU in 1.93069s (18.6462GiB/s) | 20260506 17:41:09.966 | INFO | Downloaded 34GiB to GPU in 1.87179s (18.1644GiB/s) | 20260506 17:41:15.108 | INFO | Analyzing chunk frames = [3225600,3226112) | 20260506 17:41:15.108 | INFO | MemInfo: { | 20260506 17:41:15.108 | INFO | "gpu" : " 17 GB", | 20260506 17:41:15.108 | INFO | "huge" : "", | 20260506 17:41:15.108 | INFO | "huge_shared" : " 27 GB", | 20260506 17:41:15.108 | INFO | "malloc" : " 14 GB", | 20260506 17:41:15.108 | INFO | "pinned" : " 29 GB" | 20260506 17:41:15.108 | INFO | } | 20260506 17:41:15.108 | INFO | Perf report for the last 5.1421 seconds: | 20260506 17:41:15.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:41:15.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:15.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:15.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:15.108 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 17:41:15.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:15.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:15.108 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:41:15.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:15.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:15.109 | INFO | Uploaded 36GiB to GPU in 2.00717s (17.9357GiB/s) | 20260506 17:41:15.109 | INFO | Downloaded 34GiB to GPU in 1.91199s (17.7826GiB/s) | 20260506 17:41:20.242 | INFO | Analyzing chunk frames = [3226112,3226624) | 20260506 17:41:20.242 | INFO | MemInfo: { | 20260506 17:41:20.242 | INFO | "gpu" : " 17 GB", | 20260506 17:41:20.242 | INFO | "huge" : "", | 20260506 17:41:20.242 | INFO | "huge_shared" : " 27 GB", | 20260506 17:41:20.242 | INFO | "malloc" : " 14 GB", | 20260506 17:41:20.242 | INFO | "pinned" : " 29 GB" | 20260506 17:41:20.242 | INFO | } | 20260506 17:41:20.242 | INFO | Perf report for the last 5.1341 seconds: | 20260506 17:41:20.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:41:20.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:20.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:20.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:20.242 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:41:20.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:20.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:20.242 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260506 17:41:20.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:20.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:20.243 | INFO | Uploaded 36GiB to GPU in 2.05185s (17.5451GiB/s) | 20260506 17:41:20.243 | INFO | Downloaded 34GiB to GPU in 1.97612s (17.2054GiB/s) | 20260506 17:41:25.379 | INFO | Analyzing chunk frames = [3226624,3227136) | 20260506 17:41:25.379 | INFO | MemInfo: { | 20260506 17:41:25.379 | INFO | "gpu" : " 17 GB", | 20260506 17:41:25.379 | INFO | "huge" : "", | 20260506 17:41:25.379 | INFO | "huge_shared" : " 27 GB", | 20260506 17:41:25.379 | INFO | "malloc" : " 14 GB", | 20260506 17:41:25.379 | INFO | "pinned" : " 29 GB" | 20260506 17:41:25.379 | INFO | } | 20260506 17:41:25.379 | INFO | Perf report for the last 5.13656 seconds: | 20260506 17:41:25.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:41:25.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:25.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:25.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:25.379 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:41:25.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:25.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:25.379 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 17:41:25.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:25.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:25.379 | INFO | Uploaded 36GiB to GPU in 1.9844s (18.1415GiB/s) | 20260506 17:41:25.379 | INFO | Downloaded 34GiB to GPU in 1.97799s (17.1892GiB/s) | 20260506 17:41:30.517 | INFO | Analyzing chunk frames = [3227136,3227648) | 20260506 17:41:30.517 | INFO | MemInfo: { | 20260506 17:41:30.517 | INFO | "gpu" : " 17 GB", | 20260506 17:41:30.517 | INFO | "huge" : "", | 20260506 17:41:30.517 | INFO | "huge_shared" : " 27 GB", | 20260506 17:41:30.517 | INFO | "malloc" : " 14 GB", | 20260506 17:41:30.517 | INFO | "pinned" : " 29 GB" | 20260506 17:41:30.517 | INFO | } | 20260506 17:41:30.517 | INFO | Perf report for the last 5.13771 seconds: | 20260506 17:41:30.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:41:30.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:30.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:30.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:30.517 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 17:41:30.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:30.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:30.517 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260506 17:41:30.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:30.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:30.517 | INFO | Uploaded 36GiB to GPU in 2.04779s (17.5799GiB/s) | 20260506 17:41:30.517 | INFO | Downloaded 34GiB to GPU in 1.98483s (17.13GiB/s) | 20260506 17:41:30.517 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:41:30.517 | INFO | Total mem usage: 18 GiB | 20260506 17:41:30.517 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:41:30.517 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: -8.94%) | 20260506 17:41:30.517 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:41:30.517 | INFO | High Water Breakdown: | 20260506 17:41:30.517 | INFO | 62.3% : 10.3 GiB ( 90.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:41:30.517 | INFO | 37.6% : 6.21 GiB ( 92.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:41:30.517 | INFO | 0.12% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:41:30.517 | INFO | ------------------------------------------ | 20260506 17:41:30.517 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:41:30.517 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:41:30.517 | INFO | Total mem usage: 26.5 GiB | 20260506 17:41:30.517 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:41:30.517 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:41:30.517 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:41:30.517 | INFO | High Water Breakdown: | 20260506 17:41:30.517 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:41:30.517 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:41:30.517 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:41:30.517 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:41:30.517 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:41:30.517 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:41:30.517 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:41:30.517 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:41:30.517 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:41:30.517 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:41:30.517 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:41:30.517 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:41:30.517 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:41:30.517 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:41:30.517 | INFO | ------------------------------------------ | 20260506 17:41:30.517 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:41:30.517 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:41:30.517 | INFO | Total mem usage: 25.6 GiB | 20260506 17:41:30.517 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:41:30.517 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:41:30.517 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:41:30.517 | INFO | High Water Breakdown: | 20260506 17:41:30.517 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:41:30.517 | INFO | ------------------------------------------ | 20260506 17:41:30.517 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:41:30.517 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:41:30.517 | INFO | Total mem usage: 15.7 GiB | 20260506 17:41:30.517 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:41:30.518 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:41:30.518 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:41:30.518 | INFO | High Water Breakdown: | 20260506 17:41:30.518 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:41:30.518 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:41:30.518 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:41:30.518 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:41:30.518 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:41:30.518 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:41:30.518 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:41:30.518 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:41:30.518 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:41:30.518 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:41:30.518 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:41:30.518 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:41:30.518 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:41:30.518 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:41:30.518 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:41:30.518 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:41:30.518 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:41:30.518 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:41:30.518 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:41:30.518 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:41:30.518 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:41:30.518 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:41:30.518 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:41:30.518 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:41:30.518 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:41:30.518 | INFO | ------------------------------------------ | 20260506 17:41:30.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:41:30.518 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:41:30.518 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:41:30.518 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:41:30.518 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.56e+01, 1.34e+01 / 1.83e+00 | 20260506 17:41:30.518 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.02e-01, 1.01e-02 / 5.24e-03 | 20260506 17:41:30.518 | INFO | PerfReport: 11.62%, Baseline -- 3.64e+00 / 4.47e+00, 5.55e-01 / 2.60e-01 | 20260506 17:41:30.518 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.49e+00, 1.83e-01 / 1.14e-01 | 20260506 17:41:30.518 | INFO | PerfReport: 15.93%, DME -- 3.99e+01 / 4.45e+01, 3.90e+00 / 1.62e+00 | 20260506 17:41:30.518 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.79e+01 / 2.03e+01, 1.44e+00 / 1.78e+00 | 20260506 17:41:30.518 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.27e+00 / 2.83e+00, 3.79e-01 / 2.25e-01 | 20260506 17:41:30.518 | INFO | PerfReport: 3.44%, Metrics -- 1.08e+00 / 1.44e+00, 2.83e-01 / 2.02e-01 | 20260506 17:41:30.518 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.73e-01 / 4.22e-01, 5.92e-02 / 2.93e-02 | 20260506 17:41:30.518 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:41:30.518 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:41:30.518 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:41:30.518 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 17:41:30.518 | INFO | PerfReport: 21.51%, Upload -- 2.09e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 17:41:30.518 | INFO | PerfReport: 21.17%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 17:41:30.518 | INFO | PerfReport: 32.22%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:41:30.518 | INFO | PerfReport: 1.42%, LockUpload -- 1.38e+00 / 0.00e+00, 5.63e+00 / 0.00e+00 | 20260506 17:41:30.518 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260506 17:41:30.518 | INFO | PerfReport: 22.56%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 17:41:30.518 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260506 17:41:30.628 | INFO | Starting a round of Baz IO | 20260506 17:41:33.810 | INFO | Populating RTMetrics report | 20260506 17:41:33.810 | INFO | RTMetrics report populated | 20260506 17:41:33.810 | INFO | Opening tmp RTMetrics json file | 20260506 17:41:33.904 | INFO | Serializing RTMetrics to json | 20260506 17:41:33.906 | INFO | Serialization done, swapping tmp file now... | 20260506 17:41:33.906 | INFO | RTMetrics done | 20260506 17:41:34.831 | INFO | Finished a round of Baz IO: Wrote 2.057 GiB in 4.202 seconds | 20260506 17:41:34.831 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.49% / 0.01% / 64.50% | 20260506 17:41:34.831 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.30287% / 0% / 5.16574e-05% | 20260506 17:41:34.831 | INFO | Baz IO overall: | 20260506 17:41:34.831 | INFO | BytesWritten : 1702560343824 | 20260506 17:41:34.831 | INFO | NumEvents : 823600279772 | 20260506 17:41:35.653 | INFO | Analyzing chunk frames = [3227648,3228160) | 20260506 17:41:35.653 | INFO | MemInfo: { | 20260506 17:41:35.653 | INFO | "gpu" : " 17 GB", | 20260506 17:41:35.653 | INFO | "huge" : "", | 20260506 17:41:35.653 | INFO | "huge_shared" : " 27 GB", | 20260506 17:41:35.653 | INFO | "malloc" : " 14 GB", | 20260506 17:41:35.653 | INFO | "pinned" : " 29 GB" | 20260506 17:41:35.653 | INFO | } | 20260506 17:41:35.653 | INFO | Perf report for the last 5.13603 seconds: | 20260506 17:41:35.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:41:35.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:35.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:35.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:35.653 | INFO | ANALYSIS : 63.4%, 2.8, 94.1ms, 0 | 20260506 17:41:35.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:35.653 | INFO | RT_METRICS : 49.2%, 1.0, 25.8ms, 0 | 20260506 17:41:35.653 | INFO | PRE_HQ : 62.1%, 2.0, 64.3ms, 0 | 20260506 17:41:35.653 | INFO | BAZWRITER : 81.2%, 4.8, 201.0ms, 0 | 20260506 17:41:35.653 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 17:41:35.653 | INFO | Uploaded 36GiB to GPU in 2.08101s (17.2993GiB/s) | 20260506 17:41:35.653 | INFO | Downloaded 38GiB to GPU in 2.22881s (17.0495GiB/s) | 20260506 17:41:40.796 | INFO | Analyzing chunk frames = [3228160,3228672) | 20260506 17:41:40.796 | INFO | MemInfo: { | 20260506 17:41:40.796 | INFO | "gpu" : " 17 GB", | 20260506 17:41:40.796 | INFO | "huge" : "", | 20260506 17:41:40.796 | INFO | "huge_shared" : " 27 GB", | 20260506 17:41:40.796 | INFO | "malloc" : " 14 GB", | 20260506 17:41:40.796 | INFO | "pinned" : " 29 GB" | 20260506 17:41:40.796 | INFO | } | 20260506 17:41:40.796 | INFO | Perf report for the last 5.14329 seconds: | 20260506 17:41:40.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:41:40.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:40.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:40.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:40.796 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:41:40.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:40.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:40.796 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 17:41:40.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:40.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:40.796 | INFO | Uploaded 36GiB to GPU in 1.99072s (18.0839GiB/s) | 20260506 17:41:40.796 | INFO | Downloaded 34GiB to GPU in 1.94783s (17.4553GiB/s) | 20260506 17:41:45.941 | INFO | Analyzing chunk frames = [3228672,3229184) | 20260506 17:41:45.941 | INFO | MemInfo: { | 20260506 17:41:45.941 | INFO | "gpu" : " 17 GB", | 20260506 17:41:45.941 | INFO | "huge" : "", | 20260506 17:41:45.941 | INFO | "huge_shared" : " 27 GB", | 20260506 17:41:45.941 | INFO | "malloc" : " 14 GB", | 20260506 17:41:45.941 | INFO | "pinned" : " 29 GB" | 20260506 17:41:45.941 | INFO | } | 20260506 17:41:45.941 | INFO | Perf report for the last 5.14518 seconds: | 20260506 17:41:45.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:41:45.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:45.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:45.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:45.941 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 17:41:45.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:45.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:45.941 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:41:45.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:45.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:45.942 | INFO | Uploaded 36GiB to GPU in 1.98549s (18.1315GiB/s) | 20260506 17:41:45.942 | INFO | Downloaded 34GiB to GPU in 1.92344s (17.6767GiB/s) | 20260506 17:41:51.080 | INFO | Analyzing chunk frames = [3229184,3229696) | 20260506 17:41:51.080 | INFO | MemInfo: { | 20260506 17:41:51.080 | INFO | "gpu" : " 17 GB", | 20260506 17:41:51.080 | INFO | "huge" : "", | 20260506 17:41:51.080 | INFO | "huge_shared" : " 27 GB", | 20260506 17:41:51.080 | INFO | "malloc" : " 14 GB", | 20260506 17:41:51.080 | INFO | "pinned" : " 29 GB" | 20260506 17:41:51.080 | INFO | } | 20260506 17:41:51.080 | INFO | Perf report for the last 5.13886 seconds: | 20260506 17:41:51.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:41:51.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:51.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:51.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:51.080 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 17:41:51.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:51.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:51.080 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:41:51.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:51.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:51.080 | INFO | Uploaded 36GiB to GPU in 1.9345s (18.6094GiB/s) | 20260506 17:41:51.080 | INFO | Downloaded 34GiB to GPU in 1.86873s (18.1942GiB/s) | 20260506 17:41:56.220 | INFO | Analyzing chunk frames = [3229696,3230208) | 20260506 17:41:56.220 | INFO | MemInfo: { | 20260506 17:41:56.220 | INFO | "gpu" : " 17 GB", | 20260506 17:41:56.220 | INFO | "huge" : "", | 20260506 17:41:56.220 | INFO | "huge_shared" : " 27 GB", | 20260506 17:41:56.220 | INFO | "malloc" : " 14 GB", | 20260506 17:41:56.220 | INFO | "pinned" : " 29 GB" | 20260506 17:41:56.220 | INFO | } | 20260506 17:41:56.220 | INFO | Perf report for the last 5.13962 seconds: | 20260506 17:41:56.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:41:56.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:56.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:56.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:56.220 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 17:41:56.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:56.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:56.220 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 17:41:56.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:56.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:41:56.220 | INFO | Uploaded 36GiB to GPU in 2.01324s (17.8816GiB/s) | 20260506 17:41:56.221 | INFO | Downloaded 34GiB to GPU in 1.90329s (17.8638GiB/s) | 20260506 17:42:01.355 | INFO | Analyzing chunk frames = [3230208,3230720) | 20260506 17:42:01.355 | INFO | MemInfo: { | 20260506 17:42:01.355 | INFO | "gpu" : " 17 GB", | 20260506 17:42:01.355 | INFO | "huge" : "", | 20260506 17:42:01.355 | INFO | "huge_shared" : " 27 GB", | 20260506 17:42:01.355 | INFO | "malloc" : " 14 GB", | 20260506 17:42:01.355 | INFO | "pinned" : " 29 GB" | 20260506 17:42:01.355 | INFO | } | 20260506 17:42:01.355 | INFO | Perf report for the last 5.13528 seconds: | 20260506 17:42:01.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:42:01.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:01.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:01.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:01.355 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:42:01.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:01.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:01.355 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:42:01.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:01.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:01.355 | INFO | Uploaded 36GiB to GPU in 2.05927s (17.4819GiB/s) | 20260506 17:42:01.355 | INFO | Downloaded 34GiB to GPU in 1.95543s (17.3875GiB/s) | 20260506 17:42:06.493 | INFO | Analyzing chunk frames = [3230720,3231232) | 20260506 17:42:06.493 | INFO | MemInfo: { | 20260506 17:42:06.493 | INFO | "gpu" : " 17 GB", | 20260506 17:42:06.493 | INFO | "huge" : "", | 20260506 17:42:06.493 | INFO | "huge_shared" : " 27 GB", | 20260506 17:42:06.493 | INFO | "malloc" : " 14 GB", | 20260506 17:42:06.493 | INFO | "pinned" : " 29 GB" | 20260506 17:42:06.493 | INFO | } | 20260506 17:42:06.493 | INFO | Perf report for the last 5.13813 seconds: | 20260506 17:42:06.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:42:06.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:06.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:06.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:06.493 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 17:42:06.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:06.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:06.493 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:42:06.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:06.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:06.493 | INFO | Uploaded 36GiB to GPU in 2.02476s (17.7799GiB/s) | 20260506 17:42:06.494 | INFO | Downloaded 34GiB to GPU in 2.00693s (16.9413GiB/s) | 20260506 17:42:11.631 | INFO | Analyzing chunk frames = [3231232,3231744) | 20260506 17:42:11.631 | INFO | MemInfo: { | 20260506 17:42:11.631 | INFO | "gpu" : " 17 GB", | 20260506 17:42:11.631 | INFO | "huge" : "", | 20260506 17:42:11.632 | INFO | "huge_shared" : " 27 GB", | 20260506 17:42:11.632 | INFO | "malloc" : " 14 GB", | 20260506 17:42:11.632 | INFO | "pinned" : " 29 GB" | 20260506 17:42:11.632 | INFO | } | 20260506 17:42:11.632 | INFO | Perf report for the last 5.13837 seconds: | 20260506 17:42:11.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:42:11.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:11.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:11.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:11.632 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 17:42:11.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:11.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:11.632 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 17:42:11.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:11.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:11.632 | INFO | Uploaded 36GiB to GPU in 1.99631s (18.0333GiB/s) | 20260506 17:42:11.632 | INFO | Downloaded 34GiB to GPU in 1.98844s (17.0989GiB/s) | 20260506 17:42:11.746 | INFO | Starting a round of Baz IO | 20260506 17:42:14.879 | INFO | Populating RTMetrics report | 20260506 17:42:14.879 | INFO | RTMetrics report populated | 20260506 17:42:14.879 | INFO | Opening tmp RTMetrics json file | 20260506 17:42:14.889 | INFO | Serializing RTMetrics to json | 20260506 17:42:14.895 | INFO | Serialization done, swapping tmp file now... | 20260506 17:42:14.898 | INFO | RTMetrics done | 20260506 17:42:15.332 | INFO | Finished a round of Baz IO: Wrote 2.059 GiB in 3.586 seconds | 20260506 17:42:15.332 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.49% / 0.01% / 64.51% | 20260506 17:42:15.332 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.30376% / 1.19209e-05% / 8.34465e-05% | 20260506 17:42:15.332 | INFO | Baz IO overall: | 20260506 17:42:15.332 | INFO | BytesWritten : 1704770695516 | 20260506 17:42:15.332 | INFO | NumEvents : 824666333746 | 20260506 17:42:16.773 | INFO | Analyzing chunk frames = [3231744,3232256) | 20260506 17:42:16.773 | INFO | MemInfo: { | 20260506 17:42:16.773 | INFO | "gpu" : " 17 GB", | 20260506 17:42:16.774 | INFO | "huge" : "", | 20260506 17:42:16.774 | INFO | "huge_shared" : " 27 GB", | 20260506 17:42:16.774 | INFO | "malloc" : " 14 GB", | 20260506 17:42:16.774 | INFO | "pinned" : " 29 GB" | 20260506 17:42:16.774 | INFO | } | 20260506 17:42:16.774 | INFO | Perf report for the last 5.1425 seconds: | 20260506 17:42:16.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:42:16.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:16.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:16.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:16.774 | INFO | ANALYSIS : 62.6%, 2.9, 95.1ms, 0 | 20260506 17:42:16.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:16.774 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 17:42:16.774 | INFO | PRE_HQ : 60.3%, 1.9, 58.3ms, 0 | 20260506 17:42:16.774 | INFO | BAZWRITER : 69.4%, 4.8, 171.7ms, 0 | 20260506 17:42:16.774 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 17:42:16.774 | INFO | Uploaded 36GiB to GPU in 2.03586s (17.6829GiB/s) | 20260506 17:42:16.774 | INFO | Downloaded 38GiB to GPU in 2.20668s (17.2204GiB/s) | 20260506 17:42:21.916 | INFO | Analyzing chunk frames = [3232256,3232768) | 20260506 17:42:21.916 | INFO | MemInfo: { | 20260506 17:42:21.916 | INFO | "gpu" : " 17 GB", | 20260506 17:42:21.916 | INFO | "huge" : "", | 20260506 17:42:21.916 | INFO | "huge_shared" : " 27 GB", | 20260506 17:42:21.916 | INFO | "malloc" : " 14 GB", | 20260506 17:42:21.916 | INFO | "pinned" : " 29 GB" | 20260506 17:42:21.916 | INFO | } | 20260506 17:42:21.916 | INFO | Perf report for the last 5.14169 seconds: | 20260506 17:42:21.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:42:21.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:21.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:21.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:21.916 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:42:21.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:21.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:21.916 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 17:42:21.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:21.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:21.916 | INFO | Uploaded 36GiB to GPU in 1.98944s (18.0956GiB/s) | 20260506 17:42:21.916 | INFO | Downloaded 34GiB to GPU in 1.93946s (17.5307GiB/s) | 20260506 17:42:27.056 | INFO | Analyzing chunk frames = [3232768,3233280) | 20260506 17:42:27.056 | INFO | MemInfo: { | 20260506 17:42:27.056 | INFO | "gpu" : " 17 GB", | 20260506 17:42:27.056 | INFO | "huge" : "", | 20260506 17:42:27.056 | INFO | "huge_shared" : " 27 GB", | 20260506 17:42:27.056 | INFO | "malloc" : " 14 GB", | 20260506 17:42:27.056 | INFO | "pinned" : " 29 GB" | 20260506 17:42:27.056 | INFO | } | 20260506 17:42:27.056 | INFO | Perf report for the last 5.14041 seconds: | 20260506 17:42:27.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:42:27.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:27.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:27.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:27.056 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 17:42:27.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:27.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:27.056 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:42:27.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:27.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:27.056 | INFO | Uploaded 36GiB to GPU in 1.97138s (18.2613GiB/s) | 20260506 17:42:27.056 | INFO | Downloaded 34GiB to GPU in 1.92356s (17.6755GiB/s) | 20260506 17:42:32.198 | INFO | Analyzing chunk frames = [3233280,3233792) | 20260506 17:42:32.198 | INFO | MemInfo: { | 20260506 17:42:32.198 | INFO | "gpu" : " 17 GB", | 20260506 17:42:32.198 | INFO | "huge" : "", | 20260506 17:42:32.198 | INFO | "huge_shared" : " 27 GB", | 20260506 17:42:32.198 | INFO | "malloc" : " 14 GB", | 20260506 17:42:32.198 | INFO | "pinned" : " 29 GB" | 20260506 17:42:32.198 | INFO | } | 20260506 17:42:32.198 | INFO | Perf report for the last 5.1415 seconds: | 20260506 17:42:32.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:42:32.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:32.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:32.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:32.198 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 17:42:32.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:32.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:32.198 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:42:32.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:32.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:32.198 | INFO | Uploaded 36GiB to GPU in 1.92306s (18.7202GiB/s) | 20260506 17:42:32.198 | INFO | Downloaded 34GiB to GPU in 1.87372s (18.1457GiB/s) | 20260506 17:42:37.335 | INFO | Analyzing chunk frames = [3233792,3234304) | 20260506 17:42:37.335 | INFO | MemInfo: { | 20260506 17:42:37.335 | INFO | "gpu" : " 17 GB", | 20260506 17:42:37.335 | INFO | "huge" : "", | 20260506 17:42:37.336 | INFO | "huge_shared" : " 27 GB", | 20260506 17:42:37.336 | INFO | "malloc" : " 14 GB", | 20260506 17:42:37.336 | INFO | "pinned" : " 29 GB" | 20260506 17:42:37.336 | INFO | } | 20260506 17:42:37.336 | INFO | Perf report for the last 5.13786 seconds: | 20260506 17:42:37.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:42:37.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:37.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:37.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:37.336 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 17:42:37.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:37.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:37.336 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:42:37.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:37.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:37.336 | INFO | Uploaded 36GiB to GPU in 2.01204s (17.8922GiB/s) | 20260506 17:42:37.336 | INFO | Downloaded 34GiB to GPU in 1.91441s (17.76GiB/s) | 20260506 17:42:42.471 | INFO | Analyzing chunk frames = [3234304,3234816) | 20260506 17:42:42.471 | INFO | MemInfo: { | 20260506 17:42:42.471 | INFO | "gpu" : " 17 GB", | 20260506 17:42:42.471 | INFO | "huge" : "", | 20260506 17:42:42.471 | INFO | "huge_shared" : " 27 GB", | 20260506 17:42:42.471 | INFO | "malloc" : " 14 GB", | 20260506 17:42:42.471 | INFO | "pinned" : " 29 GB" | 20260506 17:42:42.471 | INFO | } | 20260506 17:42:42.471 | INFO | Perf report for the last 5.13554 seconds: | 20260506 17:42:42.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:42:42.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:42.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:42.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:42.471 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:42:42.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:42.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:42.471 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260506 17:42:42.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:42.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:42.471 | INFO | Uploaded 36GiB to GPU in 2.05474s (17.5205GiB/s) | 20260506 17:42:42.471 | INFO | Downloaded 34GiB to GPU in 1.97014s (17.2577GiB/s) | 20260506 17:42:47.607 | INFO | Analyzing chunk frames = [3234816,3235328) | 20260506 17:42:47.607 | INFO | MemInfo: { | 20260506 17:42:47.607 | INFO | "gpu" : " 17 GB", | 20260506 17:42:47.607 | INFO | "huge" : "", | 20260506 17:42:47.607 | INFO | "huge_shared" : " 27 GB", | 20260506 17:42:47.607 | INFO | "malloc" : " 14 GB", | 20260506 17:42:47.607 | INFO | "pinned" : " 29 GB" | 20260506 17:42:47.607 | INFO | } | 20260506 17:42:47.607 | INFO | Perf report for the last 5.13562 seconds: | 20260506 17:42:47.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:42:47.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:47.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:47.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:47.607 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:42:47.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:47.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:47.607 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:42:47.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:47.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:47.607 | INFO | Uploaded 36GiB to GPU in 2.00693s (17.9379GiB/s) | 20260506 17:42:47.607 | INFO | Downloaded 34GiB to GPU in 1.99637s (17.0309GiB/s) | 20260506 17:42:52.746 | INFO | Analyzing chunk frames = [3235328,3235840) | 20260506 17:42:52.746 | INFO | MemInfo: { | 20260506 17:42:52.746 | INFO | "gpu" : " 17 GB", | 20260506 17:42:52.746 | INFO | "huge" : "", | 20260506 17:42:52.746 | INFO | "huge_shared" : " 27 GB", | 20260506 17:42:52.746 | INFO | "malloc" : " 14 GB", | 20260506 17:42:52.746 | INFO | "pinned" : " 29 GB" | 20260506 17:42:52.746 | INFO | } | 20260506 17:42:52.746 | INFO | Perf report for the last 5.13958 seconds: | 20260506 17:42:52.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:42:52.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:52.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:52.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:52.746 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 17:42:52.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:52.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:52.746 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 17:42:52.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:52.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:52.747 | INFO | Uploaded 36GiB to GPU in 2.03917s (17.6542GiB/s) | 20260506 17:42:52.747 | INFO | Downloaded 34GiB to GPU in 1.98835s (17.0996GiB/s) | 20260506 17:42:52.747 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:42:52.747 | INFO | Total mem usage: 18 GiB | 20260506 17:42:52.747 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:42:52.747 | INFO | Recent High Water Marks Sum: 17.1 GiB (Savings: -5.25%) | 20260506 17:42:52.747 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:42:52.747 | INFO | High Water Breakdown: | 20260506 17:42:52.747 | INFO | 62.7% : 10.7 GiB ( 94% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:42:52.747 | INFO | 37.2% : 6.36 GiB ( 95% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:42:52.747 | INFO | 0.116% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:42:52.747 | INFO | ------------------------------------------ | 20260506 17:42:52.747 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:42:52.747 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:42:52.747 | INFO | Total mem usage: 26.5 GiB | 20260506 17:42:52.747 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:42:52.747 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260506 17:42:52.747 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260506 17:42:52.747 | INFO | High Water Breakdown: | 20260506 17:42:52.747 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:42:52.747 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:42:52.747 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:42:52.747 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:42:52.747 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:42:52.747 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:42:52.747 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:42:52.747 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:42:52.747 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:42:52.747 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:42:52.747 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:42:52.747 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:42:52.747 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:42:52.747 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:42:52.747 | INFO | ------------------------------------------ | 20260506 17:42:52.747 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:42:52.747 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:42:52.747 | INFO | Total mem usage: 25.6 GiB | 20260506 17:42:52.747 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:42:52.747 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:42:52.747 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:42:52.747 | INFO | High Water Breakdown: | 20260506 17:42:52.747 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:42:52.747 | INFO | ------------------------------------------ | 20260506 17:42:52.747 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:42:52.747 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:42:52.747 | INFO | Total mem usage: 15.7 GiB | 20260506 17:42:52.747 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:42:52.747 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:42:52.747 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:42:52.748 | INFO | High Water Breakdown: | 20260506 17:42:52.748 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:42:52.748 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:42:52.748 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:42:52.748 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:42:52.748 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:42:52.748 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:42:52.748 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:42:52.748 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:42:52.748 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:42:52.748 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:42:52.748 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:42:52.748 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:42:52.748 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:42:52.748 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:42:52.748 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:42:52.748 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:42:52.748 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:42:52.748 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:42:52.748 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:42:52.748 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:42:52.748 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:42:52.748 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:42:52.748 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:42:52.748 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:42:52.748 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:42:52.748 | INFO | ------------------------------------------ | 20260506 17:42:52.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:42:52.748 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:42:52.748 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:42:52.748 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:42:52.748 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.55e+01, 1.35e+01 / 2.03e+00 | 20260506 17:42:52.748 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.02e-01, 1.07e-02 / 6.60e-03 | 20260506 17:42:52.748 | INFO | PerfReport: 11.61%, Baseline -- 3.64e+00 / 4.44e+00, 5.54e-01 / 3.44e-01 | 20260506 17:42:52.748 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.47e+00, 1.84e-01 / 1.27e-01 | 20260506 17:42:52.748 | INFO | PerfReport: 15.92%, DME -- 3.99e+01 / 4.42e+01, 3.95e+00 / 2.31e+00 | 20260506 17:42:52.748 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.46e+00 / 1.76e+00 | 20260506 17:42:52.748 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.28e+00 / 2.82e+00, 3.83e-01 / 2.67e-01 | 20260506 17:42:52.748 | INFO | PerfReport: 3.45%, Metrics -- 1.08e+00 / 1.40e+00, 2.84e-01 / 1.57e-01 | 20260506 17:42:52.748 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.73e-01 / 4.21e-01, 5.89e-02 / 2.69e-02 | 20260506 17:42:52.748 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:42:52.748 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:42:52.748 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:42:52.748 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 17:42:52.748 | INFO | PerfReport: 21.52%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260506 17:42:52.748 | INFO | PerfReport: 21.20%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 17:42:52.748 | INFO | PerfReport: 32.25%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:42:52.748 | INFO | PerfReport: 1.40%, LockUpload -- 1.36e+00 / 0.00e+00, 5.72e+00 / 0.00e+00 | 20260506 17:42:52.748 | INFO | PerfReport: 1.12%, LockDownload -- 1.08e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260506 17:42:52.748 | INFO | PerfReport: 22.50%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 17:42:52.748 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260506 17:42:52.859 | INFO | Starting a round of Baz IO | 20260506 17:42:56.006 | INFO | Populating RTMetrics report | 20260506 17:42:56.007 | INFO | RTMetrics report populated | 20260506 17:42:56.007 | INFO | Opening tmp RTMetrics json file | 20260506 17:42:56.054 | INFO | Serializing RTMetrics to json | 20260506 17:42:56.094 | INFO | Serialization done, swapping tmp file now... | 20260506 17:42:56.105 | INFO | RTMetrics done | 20260506 17:42:56.125 | INFO | Finished a round of Baz IO: Wrote 2.059 GiB in 3.266 seconds | 20260506 17:42:56.125 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.48% / 0.01% / 64.51% | 20260506 17:42:56.125 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.30267% / 3.97364e-06% / 5.5631e-05% | 20260506 17:42:56.125 | INFO | Baz IO overall: | 20260506 17:42:56.125 | INFO | BytesWritten : 1706981736258 | 20260506 17:42:56.125 | INFO | NumEvents : 825733078659 | 20260506 17:42:57.888 | INFO | Analyzing chunk frames = [3235840,3236352) | 20260506 17:42:57.888 | INFO | MemInfo: { | 20260506 17:42:57.888 | INFO | "gpu" : " 17 GB", | 20260506 17:42:57.888 | INFO | "huge" : "", | 20260506 17:42:57.888 | INFO | "huge_shared" : " 27 GB", | 20260506 17:42:57.888 | INFO | "malloc" : " 14 GB", | 20260506 17:42:57.888 | INFO | "pinned" : " 29 GB" | 20260506 17:42:57.888 | INFO | } | 20260506 17:42:57.888 | INFO | Perf report for the last 5.14129 seconds: | 20260506 17:42:57.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:42:57.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:57.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:57.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:57.888 | INFO | ANALYSIS : 62.8%, 2.9, 95.3ms, 0 | 20260506 17:42:57.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:42:57.888 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 17:42:57.888 | INFO | PRE_HQ : 60.5%, 1.9, 58.6ms, 0 | 20260506 17:42:57.888 | INFO | BAZWRITER : 62.9%, 4.8, 155.2ms, 0 | 20260506 17:42:57.888 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 17:42:57.888 | INFO | Uploaded 36GiB to GPU in 2.0753s (17.3469GiB/s) | 20260506 17:42:57.888 | INFO | Downloaded 38GiB to GPU in 2.23035s (17.0377GiB/s) | 20260506 17:43:03.029 | INFO | Analyzing chunk frames = [3236352,3236864) | 20260506 17:43:03.029 | INFO | MemInfo: { | 20260506 17:43:03.029 | INFO | "gpu" : " 17 GB", | 20260506 17:43:03.029 | INFO | "huge" : "", | 20260506 17:43:03.029 | INFO | "huge_shared" : " 27 GB", | 20260506 17:43:03.029 | INFO | "malloc" : " 14 GB", | 20260506 17:43:03.029 | INFO | "pinned" : " 29 GB" | 20260506 17:43:03.029 | INFO | } | 20260506 17:43:03.029 | INFO | Perf report for the last 5.14119 seconds: | 20260506 17:43:03.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:43:03.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:03.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:03.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:03.029 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:43:03.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:03.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:03.029 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:43:03.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:03.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:03.029 | INFO | Uploaded 36GiB to GPU in 1.9937s (18.0568GiB/s) | 20260506 17:43:03.029 | INFO | Downloaded 34GiB to GPU in 1.93995s (17.5262GiB/s) | 20260506 17:43:08.169 | INFO | Analyzing chunk frames = [3236864,3237376) | 20260506 17:43:08.169 | INFO | MemInfo: { | 20260506 17:43:08.169 | INFO | "gpu" : " 17 GB", | 20260506 17:43:08.169 | INFO | "huge" : "", | 20260506 17:43:08.169 | INFO | "huge_shared" : " 27 GB", | 20260506 17:43:08.169 | INFO | "malloc" : " 14 GB", | 20260506 17:43:08.170 | INFO | "pinned" : " 29 GB" | 20260506 17:43:08.170 | INFO | } | 20260506 17:43:08.170 | INFO | Perf report for the last 5.14074 seconds: | 20260506 17:43:08.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:43:08.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:08.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:08.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:08.170 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:43:08.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:08.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:08.170 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:43:08.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:08.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:08.170 | INFO | Uploaded 36GiB to GPU in 1.97119s (18.2631GiB/s) | 20260506 17:43:08.170 | INFO | Downloaded 34GiB to GPU in 1.92187s (17.6911GiB/s) | 20260506 17:43:13.305 | INFO | Analyzing chunk frames = [3237376,3237888) | 20260506 17:43:13.305 | INFO | MemInfo: { | 20260506 17:43:13.305 | INFO | "gpu" : " 17 GB", | 20260506 17:43:13.305 | INFO | "huge" : "", | 20260506 17:43:13.305 | INFO | "huge_shared" : " 27 GB", | 20260506 17:43:13.305 | INFO | "malloc" : " 14 GB", | 20260506 17:43:13.305 | INFO | "pinned" : " 29 GB" | 20260506 17:43:13.305 | INFO | } | 20260506 17:43:13.305 | INFO | Perf report for the last 5.13509 seconds: | 20260506 17:43:13.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:43:13.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:13.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:13.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:13.305 | INFO | ANALYSIS : 60.1%, 3.0, 93.0ms, 0 | 20260506 17:43:13.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:13.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:13.305 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:43:13.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:13.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:13.305 | INFO | Uploaded 36GiB to GPU in 1.93431s (18.6113GiB/s) | 20260506 17:43:13.305 | INFO | Downloaded 34GiB to GPU in 1.87017s (18.1801GiB/s) | 20260506 17:43:18.446 | INFO | Analyzing chunk frames = [3237888,3238400) | 20260506 17:43:18.446 | INFO | MemInfo: { | 20260506 17:43:18.446 | INFO | "gpu" : " 17 GB", | 20260506 17:43:18.446 | INFO | "huge" : "", | 20260506 17:43:18.446 | INFO | "huge_shared" : " 27 GB", | 20260506 17:43:18.446 | INFO | "malloc" : " 14 GB", | 20260506 17:43:18.447 | INFO | "pinned" : " 29 GB" | 20260506 17:43:18.447 | INFO | } | 20260506 17:43:18.447 | INFO | Perf report for the last 5.14187 seconds: | 20260506 17:43:18.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:43:18.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:18.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:18.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:18.447 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 17:43:18.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:18.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:18.447 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:43:18.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:18.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:18.447 | INFO | Uploaded 36GiB to GPU in 2.01608s (17.8565GiB/s) | 20260506 17:43:18.447 | INFO | Downloaded 34GiB to GPU in 1.90894s (17.811GiB/s) | 20260506 17:43:23.583 | INFO | Analyzing chunk frames = [3238400,3238912) | 20260506 17:43:23.583 | INFO | MemInfo: { | 20260506 17:43:23.583 | INFO | "gpu" : " 17 GB", | 20260506 17:43:23.583 | INFO | "huge" : "", | 20260506 17:43:23.583 | INFO | "huge_shared" : " 27 GB", | 20260506 17:43:23.583 | INFO | "malloc" : " 14 GB", | 20260506 17:43:23.583 | INFO | "pinned" : " 29 GB" | 20260506 17:43:23.583 | INFO | } | 20260506 17:43:23.583 | INFO | Perf report for the last 5.13685 seconds: | 20260506 17:43:23.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:43:23.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:23.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:23.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:23.584 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:43:23.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:23.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:23.584 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:43:23.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:23.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:23.584 | INFO | Uploaded 36GiB to GPU in 2.06832s (17.4054GiB/s) | 20260506 17:43:23.584 | INFO | Downloaded 34GiB to GPU in 1.96003s (17.3467GiB/s) | 20260506 17:43:28.721 | INFO | Analyzing chunk frames = [3238912,3239424) | 20260506 17:43:28.721 | INFO | MemInfo: { | 20260506 17:43:28.721 | INFO | "gpu" : " 17 GB", | 20260506 17:43:28.721 | INFO | "huge" : "", | 20260506 17:43:28.721 | INFO | "huge_shared" : " 27 GB", | 20260506 17:43:28.721 | INFO | "malloc" : " 14 GB", | 20260506 17:43:28.721 | INFO | "pinned" : " 29 GB" | 20260506 17:43:28.722 | INFO | } | 20260506 17:43:28.722 | INFO | Perf report for the last 5.13821 seconds: | 20260506 17:43:28.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:43:28.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:28.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:28.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:28.722 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:43:28.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:28.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:28.722 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:43:28.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:28.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:28.722 | INFO | Uploaded 36GiB to GPU in 1.99855s (18.0131GiB/s) | 20260506 17:43:28.722 | INFO | Downloaded 34GiB to GPU in 1.97412s (17.2229GiB/s) | 20260506 17:43:33.866 | INFO | Analyzing chunk frames = [3239424,3239936) | 20260506 17:43:33.866 | INFO | MemInfo: { | 20260506 17:43:33.866 | INFO | "gpu" : " 17 GB", | 20260506 17:43:33.866 | INFO | "huge" : "", | 20260506 17:43:33.866 | INFO | "huge_shared" : " 27 GB", | 20260506 17:43:33.866 | INFO | "malloc" : " 14 GB", | 20260506 17:43:33.867 | INFO | "pinned" : " 29 GB" | 20260506 17:43:33.867 | INFO | } | 20260506 17:43:33.867 | INFO | Perf report for the last 5.14502 seconds: | 20260506 17:43:33.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:43:33.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:33.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:33.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:33.867 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:43:33.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:33.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:33.867 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:43:33.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:33.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:33.867 | INFO | Uploaded 36GiB to GPU in 1.99429s (18.0516GiB/s) | 20260506 17:43:33.867 | INFO | Downloaded 34GiB to GPU in 1.99081s (17.0785GiB/s) | 20260506 17:43:33.980 | INFO | Starting a round of Baz IO | 20260506 17:43:37.111 | INFO | Populating RTMetrics report | 20260506 17:43:37.112 | INFO | RTMetrics report populated | 20260506 17:43:37.112 | INFO | Opening tmp RTMetrics json file | 20260506 17:43:37.142 | INFO | Serializing RTMetrics to json | 20260506 17:43:37.244 | INFO | Serialization done, swapping tmp file now... | 20260506 17:43:37.280 | INFO | RTMetrics done | 20260506 17:43:37.565 | INFO | Finished a round of Baz IO: Wrote 2.059 GiB in 3.584 seconds | 20260506 17:43:37.565 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.47% / 0.01% / 64.52% | 20260506 17:43:37.565 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.29939% / 1.58946e-05% / 3.57628e-05% | 20260506 17:43:37.565 | INFO | Baz IO overall: | 20260506 17:43:37.565 | INFO | BytesWritten : 1709192732946 | 20260506 17:43:37.565 | INFO | NumEvents : 826799616519 | 20260506 17:43:39.005 | INFO | Analyzing chunk frames = [3239936,3240448) | 20260506 17:43:39.005 | INFO | MemInfo: { | 20260506 17:43:39.005 | INFO | "gpu" : " 17 GB", | 20260506 17:43:39.005 | INFO | "huge" : "", | 20260506 17:43:39.005 | INFO | "huge_shared" : " 27 GB", | 20260506 17:43:39.005 | INFO | "malloc" : " 14 GB", | 20260506 17:43:39.005 | INFO | "pinned" : " 29 GB" | 20260506 17:43:39.005 | INFO | } | 20260506 17:43:39.005 | INFO | Perf report for the last 5.13847 seconds: | 20260506 17:43:39.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:43:39.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:39.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:39.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:39.005 | INFO | ANALYSIS : 62.6%, 2.9, 95.2ms, 0 | 20260506 17:43:39.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:39.005 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260506 17:43:39.005 | INFO | PRE_HQ : 60.2%, 1.9, 57.9ms, 0 | 20260506 17:43:39.005 | INFO | BAZWRITER : 69.4%, 4.8, 171.9ms, 0 | 20260506 17:43:39.005 | INFO | BAZ_SAMPLING : 3.3%, 1.0, 1.7ms, 0 | 20260506 17:43:39.005 | INFO | Uploaded 36GiB to GPU in 2.03665s (17.6761GiB/s) | 20260506 17:43:39.006 | INFO | Downloaded 38GiB to GPU in 2.21166s (17.1817GiB/s) | 20260506 17:43:44.147 | INFO | Analyzing chunk frames = [3240448,3240960) | 20260506 17:43:44.147 | INFO | MemInfo: { | 20260506 17:43:44.147 | INFO | "gpu" : " 17 GB", | 20260506 17:43:44.147 | INFO | "huge" : "", | 20260506 17:43:44.147 | INFO | "huge_shared" : " 27 GB", | 20260506 17:43:44.147 | INFO | "malloc" : " 14 GB", | 20260506 17:43:44.147 | INFO | "pinned" : " 29 GB" | 20260506 17:43:44.147 | INFO | } | 20260506 17:43:44.147 | INFO | Perf report for the last 5.14167 seconds: | 20260506 17:43:44.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:43:44.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:44.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:44.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:44.147 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 17:43:44.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:44.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:44.147 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:43:44.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:44.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:44.147 | INFO | Uploaded 36GiB to GPU in 1.99193s (18.073GiB/s) | 20260506 17:43:44.147 | INFO | Downloaded 34GiB to GPU in 1.93653s (17.5572GiB/s) | 20260506 17:43:49.284 | INFO | Analyzing chunk frames = [3240960,3241472) | 20260506 17:43:49.284 | INFO | MemInfo: { | 20260506 17:43:49.284 | INFO | "gpu" : " 17 GB", | 20260506 17:43:49.284 | INFO | "huge" : "", | 20260506 17:43:49.284 | INFO | "huge_shared" : " 27 GB", | 20260506 17:43:49.284 | INFO | "malloc" : " 14 GB", | 20260506 17:43:49.284 | INFO | "pinned" : " 29 GB" | 20260506 17:43:49.284 | INFO | } | 20260506 17:43:49.284 | INFO | Perf report for the last 5.13682 seconds: | 20260506 17:43:49.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:43:49.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:49.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:49.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:49.284 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 17:43:49.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:49.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:49.284 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:43:49.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:49.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:49.284 | INFO | Uploaded 36GiB to GPU in 1.96642s (18.3074GiB/s) | 20260506 17:43:49.284 | INFO | Downloaded 34GiB to GPU in 1.92311s (17.6797GiB/s) | 20260506 17:43:54.423 | INFO | Analyzing chunk frames = [3241472,3241984) | 20260506 17:43:54.423 | INFO | MemInfo: { | 20260506 17:43:54.423 | INFO | "gpu" : " 17 GB", | 20260506 17:43:54.423 | INFO | "huge" : "", | 20260506 17:43:54.423 | INFO | "huge_shared" : " 27 GB", | 20260506 17:43:54.423 | INFO | "malloc" : " 14 GB", | 20260506 17:43:54.423 | INFO | "pinned" : " 29 GB" | 20260506 17:43:54.423 | INFO | } | 20260506 17:43:54.423 | INFO | Perf report for the last 5.13935 seconds: | 20260506 17:43:54.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:43:54.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:54.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:54.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:54.423 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 17:43:54.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:54.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:54.423 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 17:43:54.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:54.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:54.423 | INFO | Uploaded 36GiB to GPU in 1.9429s (18.529GiB/s) | 20260506 17:43:54.423 | INFO | Downloaded 34GiB to GPU in 1.88154s (18.0703GiB/s) | 20260506 17:43:59.561 | INFO | Analyzing chunk frames = [3241984,3242496) | 20260506 17:43:59.561 | INFO | MemInfo: { | 20260506 17:43:59.561 | INFO | "gpu" : " 17 GB", | 20260506 17:43:59.561 | INFO | "huge" : "", | 20260506 17:43:59.561 | INFO | "huge_shared" : " 27 GB", | 20260506 17:43:59.561 | INFO | "malloc" : " 14 GB", | 20260506 17:43:59.561 | INFO | "pinned" : " 29 GB" | 20260506 17:43:59.561 | INFO | } | 20260506 17:43:59.561 | INFO | Perf report for the last 5.13784 seconds: | 20260506 17:43:59.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:43:59.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:59.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:59.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:59.561 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 17:43:59.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:59.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:59.561 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:43:59.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:59.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:43:59.561 | INFO | Uploaded 36GiB to GPU in 2.01292s (17.8844GiB/s) | 20260506 17:43:59.561 | INFO | Downloaded 34GiB to GPU in 1.91513s (17.7534GiB/s) | 20260506 17:44:04.701 | INFO | Analyzing chunk frames = [3242496,3243008) | 20260506 17:44:04.701 | INFO | MemInfo: { | 20260506 17:44:04.701 | INFO | "gpu" : " 17 GB", | 20260506 17:44:04.701 | INFO | "huge" : "", | 20260506 17:44:04.701 | INFO | "huge_shared" : " 27 GB", | 20260506 17:44:04.701 | INFO | "malloc" : " 14 GB", | 20260506 17:44:04.701 | INFO | "pinned" : " 29 GB" | 20260506 17:44:04.701 | INFO | } | 20260506 17:44:04.701 | INFO | Perf report for the last 5.14047 seconds: | 20260506 17:44:04.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:44:04.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:04.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:04.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:04.702 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:44:04.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:04.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:04.702 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 17:44:04.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:04.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:04.702 | INFO | Uploaded 36GiB to GPU in 2.0542s (17.5251GiB/s) | 20260506 17:44:04.702 | INFO | Downloaded 34GiB to GPU in 1.96905s (17.2672GiB/s) | 20260506 17:44:09.839 | INFO | Analyzing chunk frames = [3243008,3243520) | 20260506 17:44:09.839 | INFO | MemInfo: { | 20260506 17:44:09.839 | INFO | "gpu" : " 17 GB", | 20260506 17:44:09.839 | INFO | "huge" : "", | 20260506 17:44:09.839 | INFO | "huge_shared" : " 27 GB", | 20260506 17:44:09.839 | INFO | "malloc" : " 14 GB", | 20260506 17:44:09.839 | INFO | "pinned" : " 29 GB" | 20260506 17:44:09.839 | INFO | } | 20260506 17:44:09.839 | INFO | Perf report for the last 5.13795 seconds: | 20260506 17:44:09.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:44:09.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:09.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:09.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:09.839 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:44:09.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:09.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:09.839 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:44:09.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:09.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:09.840 | INFO | Uploaded 36GiB to GPU in 1.99097s (18.0816GiB/s) | 20260506 17:44:09.840 | INFO | Downloaded 34GiB to GPU in 2.02062s (16.8265GiB/s) | 20260506 17:44:14.974 | INFO | Analyzing chunk frames = [3243520,3244032) | 20260506 17:44:14.974 | INFO | MemInfo: { | 20260506 17:44:14.974 | INFO | "gpu" : " 17 GB", | 20260506 17:44:14.974 | INFO | "huge" : "", | 20260506 17:44:14.974 | INFO | "huge_shared" : " 27 GB", | 20260506 17:44:14.974 | INFO | "malloc" : " 14 GB", | 20260506 17:44:14.974 | INFO | "pinned" : " 29 GB" | 20260506 17:44:14.974 | INFO | } | 20260506 17:44:14.974 | INFO | Perf report for the last 5.13428 seconds: | 20260506 17:44:14.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:44:14.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:14.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:14.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:14.974 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 17:44:14.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:14.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:14.974 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 17:44:14.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:14.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:14.974 | INFO | Uploaded 36GiB to GPU in 1.99094s (18.0819GiB/s) | 20260506 17:44:14.974 | INFO | Downloaded 34GiB to GPU in 1.9781s (17.1882GiB/s) | 20260506 17:44:14.974 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:44:14.974 | INFO | Total mem usage: 18 GiB | 20260506 17:44:14.974 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:44:14.974 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.5%) | 20260506 17:44:14.974 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:44:14.974 | INFO | High Water Breakdown: | 20260506 17:44:14.974 | INFO | 61.4% : 9.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:44:14.974 | INFO | 38.5% : 6 GiB ( 89.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:44:14.974 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:44:14.974 | INFO | ------------------------------------------ | 20260506 17:44:14.974 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:44:14.974 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:44:14.974 | INFO | Total mem usage: 26.5 GiB | 20260506 17:44:14.974 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:44:14.974 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:44:14.974 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:44:14.974 | INFO | High Water Breakdown: | 20260506 17:44:14.974 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:44:14.974 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:44:14.974 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:44:14.974 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:44:14.975 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:44:14.975 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:44:14.975 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:44:14.975 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:44:14.975 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:44:14.975 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:44:14.975 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:44:14.975 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:44:14.975 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:44:14.975 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:44:14.975 | INFO | ------------------------------------------ | 20260506 17:44:14.975 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:44:14.975 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:44:14.975 | INFO | Total mem usage: 25.6 GiB | 20260506 17:44:14.975 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:44:14.975 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:44:14.975 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:44:14.975 | INFO | High Water Breakdown: | 20260506 17:44:14.975 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:44:14.975 | INFO | ------------------------------------------ | 20260506 17:44:14.975 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:44:14.975 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:44:14.975 | INFO | Total mem usage: 15.7 GiB | 20260506 17:44:14.975 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:44:14.975 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:44:14.975 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 17:44:14.975 | INFO | High Water Breakdown: | 20260506 17:44:14.975 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:44:14.975 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:44:14.975 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:44:14.975 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:44:14.975 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:44:14.975 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:44:14.975 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:44:14.975 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:44:14.975 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:44:14.975 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:44:14.975 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:44:14.975 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:44:14.975 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:44:14.975 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:44:14.975 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:44:14.975 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:44:14.975 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:44:14.975 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:44:14.975 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:44:14.975 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:44:14.975 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:44:14.975 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:44:14.975 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:44:14.975 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:44:14.975 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:44:14.975 | INFO | ------------------------------------------ | 20260506 17:44:14.975 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:44:14.975 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:44:14.975 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:44:14.975 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:44:14.975 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.57e+01, 1.34e+01 / 2.25e+00 | 20260506 17:44:14.975 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 9.81e-03 / 4.72e-03 | 20260506 17:44:14.975 | INFO | PerfReport: 11.62%, Baseline -- 3.64e+00 / 4.45e+00, 5.61e-01 / 3.36e-01 | 20260506 17:44:14.975 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.85e-01 / 1.31e-01 | 20260506 17:44:14.975 | INFO | PerfReport: 15.95%, DME -- 4.00e+01 / 4.45e+01, 3.98e+00 / 2.22e+00 | 20260506 17:44:14.975 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.41e+00 / 2.11e+00 | 20260506 17:44:14.975 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.27e+00 / 2.83e+00, 3.82e-01 / 2.56e-01 | 20260506 17:44:14.975 | INFO | PerfReport: 3.44%, Metrics -- 1.08e+00 / 1.41e+00, 2.83e-01 / 1.50e-01 | 20260506 17:44:14.975 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.72e-01 / 4.14e-01, 5.74e-02 / 2.05e-02 | 20260506 17:44:14.975 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:44:14.975 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:44:14.975 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:44:14.975 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 17:44:14.975 | INFO | PerfReport: 21.46%, Upload -- 2.09e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 17:44:14.975 | INFO | PerfReport: 21.19%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 17:44:14.975 | INFO | PerfReport: 32.23%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:44:14.975 | INFO | PerfReport: 1.37%, LockUpload -- 1.33e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260506 17:44:14.975 | INFO | PerfReport: 1.14%, LockDownload -- 1.11e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260506 17:44:14.975 | INFO | PerfReport: 22.60%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 17:44:14.975 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260506 17:44:15.098 | INFO | Starting a round of Baz IO | 20260506 17:44:18.236 | INFO | Populating RTMetrics report | 20260506 17:44:18.236 | INFO | RTMetrics report populated | 20260506 17:44:18.236 | INFO | Opening tmp RTMetrics json file | 20260506 17:44:18.354 | INFO | Serializing RTMetrics to json | 20260506 17:44:18.447 | INFO | Serialization done, swapping tmp file now... | 20260506 17:44:18.450 | INFO | RTMetrics done | 20260506 17:44:18.728 | INFO | Finished a round of Baz IO: Wrote 2.057 GiB in 3.63 seconds | 20260506 17:44:18.728 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.47% / 0.01% / 64.53% | 20260506 17:44:18.728 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.29358% / 1.19209e-05% / 5.5631e-05% | 20260506 17:44:18.728 | INFO | Baz IO overall: | 20260506 17:44:18.728 | INFO | BytesWritten : 1711401471516 | 20260506 17:44:18.728 | INFO | NumEvents : 827864639668 | 20260506 17:44:20.115 | INFO | Analyzing chunk frames = [3244032,3244544) | 20260506 17:44:20.115 | INFO | MemInfo: { | 20260506 17:44:20.115 | INFO | "gpu" : " 17 GB", | 20260506 17:44:20.115 | INFO | "huge" : "", | 20260506 17:44:20.115 | INFO | "huge_shared" : " 27 GB", | 20260506 17:44:20.115 | INFO | "malloc" : " 14 GB", | 20260506 17:44:20.115 | INFO | "pinned" : " 29 GB" | 20260506 17:44:20.115 | INFO | } | 20260506 17:44:20.115 | INFO | Perf report for the last 5.14175 seconds: | 20260506 17:44:20.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:44:20.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:20.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:20.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:20.116 | INFO | ANALYSIS : 62.9%, 2.9, 94.9ms, 0 | 20260506 17:44:20.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:20.116 | INFO | RT_METRICS : 48.7%, 1.0, 25.5ms, 0 | 20260506 17:44:20.116 | INFO | PRE_HQ : 60.6%, 1.9, 60.3ms, 0 | 20260506 17:44:20.116 | INFO | BAZWRITER : 70.3%, 4.8, 174.0ms, 0 | 20260506 17:44:20.116 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.4ms, 0 | 20260506 17:44:20.116 | INFO | Uploaded 36GiB to GPU in 2.01607s (17.8565GiB/s) | 20260506 17:44:20.116 | INFO | Downloaded 38GiB to GPU in 2.22524s (17.0768GiB/s) | 20260506 17:44:25.260 | INFO | Analyzing chunk frames = [3244544,3245056) | 20260506 17:44:25.260 | INFO | MemInfo: { | 20260506 17:44:25.260 | INFO | "gpu" : " 17 GB", | 20260506 17:44:25.260 | INFO | "huge" : "", | 20260506 17:44:25.260 | INFO | "huge_shared" : " 27 GB", | 20260506 17:44:25.260 | INFO | "malloc" : " 14 GB", | 20260506 17:44:25.260 | INFO | "pinned" : " 29 GB" | 20260506 17:44:25.260 | INFO | } | 20260506 17:44:25.260 | INFO | Perf report for the last 5.1447 seconds: | 20260506 17:44:25.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:44:25.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:25.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:25.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:25.260 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 17:44:25.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:25.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:25.260 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:44:25.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:25.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:25.260 | INFO | Uploaded 36GiB to GPU in 2.01378s (17.8768GiB/s) | 20260506 17:44:25.260 | INFO | Downloaded 34GiB to GPU in 1.97854s (17.1844GiB/s) | 20260506 17:44:30.398 | INFO | Analyzing chunk frames = [3245056,3245568) | 20260506 17:44:30.398 | INFO | MemInfo: { | 20260506 17:44:30.398 | INFO | "gpu" : " 17 GB", | 20260506 17:44:30.398 | INFO | "huge" : "", | 20260506 17:44:30.398 | INFO | "huge_shared" : " 27 GB", | 20260506 17:44:30.398 | INFO | "malloc" : " 14 GB", | 20260506 17:44:30.398 | INFO | "pinned" : " 29 GB" | 20260506 17:44:30.398 | INFO | } | 20260506 17:44:30.398 | INFO | Perf report for the last 5.13813 seconds: | 20260506 17:44:30.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:44:30.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:30.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:30.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:30.398 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:44:30.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:30.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:30.398 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:44:30.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:30.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:30.399 | INFO | Uploaded 36GiB to GPU in 1.9736s (18.2408GiB/s) | 20260506 17:44:30.399 | INFO | Downloaded 34GiB to GPU in 1.92676s (17.6462GiB/s) | 20260506 17:44:35.542 | INFO | Analyzing chunk frames = [3245568,3246080) | 20260506 17:44:35.542 | INFO | MemInfo: { | 20260506 17:44:35.542 | INFO | "gpu" : " 17 GB", | 20260506 17:44:35.542 | INFO | "huge" : "", | 20260506 17:44:35.542 | INFO | "huge_shared" : " 27 GB", | 20260506 17:44:35.542 | INFO | "malloc" : " 14 GB", | 20260506 17:44:35.542 | INFO | "pinned" : " 29 GB" | 20260506 17:44:35.543 | INFO | } | 20260506 17:44:35.543 | INFO | Perf report for the last 5.14424 seconds: | 20260506 17:44:35.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:44:35.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:35.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:35.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:35.543 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 17:44:35.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:35.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:35.543 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:44:35.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:35.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:35.543 | INFO | Uploaded 36GiB to GPU in 1.92894s (18.6631GiB/s) | 20260506 17:44:35.543 | INFO | Downloaded 34GiB to GPU in 1.87874s (18.0972GiB/s) | 20260506 17:44:40.676 | INFO | Analyzing chunk frames = [3246080,3246592) | 20260506 17:44:40.676 | INFO | MemInfo: { | 20260506 17:44:40.676 | INFO | "gpu" : " 17 GB", | 20260506 17:44:40.676 | INFO | "huge" : "", | 20260506 17:44:40.676 | INFO | "huge_shared" : " 27 GB", | 20260506 17:44:40.676 | INFO | "malloc" : " 14 GB", | 20260506 17:44:40.676 | INFO | "pinned" : " 29 GB" | 20260506 17:44:40.676 | INFO | } | 20260506 17:44:40.676 | INFO | Perf report for the last 5.1337 seconds: | 20260506 17:44:40.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:44:40.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:40.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:40.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:40.676 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 17:44:40.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:40.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:40.676 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:44:40.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:40.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:40.677 | INFO | Uploaded 36GiB to GPU in 1.99189s (18.0733GiB/s) | 20260506 17:44:40.677 | INFO | Downloaded 34GiB to GPU in 1.92045s (17.7041GiB/s) | 20260506 17:44:45.816 | INFO | Analyzing chunk frames = [3246592,3247104) | 20260506 17:44:45.816 | INFO | MemInfo: { | 20260506 17:44:45.816 | INFO | "gpu" : " 17 GB", | 20260506 17:44:45.816 | INFO | "huge" : "", | 20260506 17:44:45.816 | INFO | "huge_shared" : " 27 GB", | 20260506 17:44:45.816 | INFO | "malloc" : " 14 GB", | 20260506 17:44:45.816 | INFO | "pinned" : " 29 GB" | 20260506 17:44:45.816 | INFO | } | 20260506 17:44:45.816 | INFO | Perf report for the last 5.13982 seconds: | 20260506 17:44:45.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:44:45.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:45.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:45.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:45.816 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 17:44:45.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:45.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:45.816 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:44:45.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:45.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:45.816 | INFO | Uploaded 36GiB to GPU in 2.0122s (17.8908GiB/s) | 20260506 17:44:45.816 | INFO | Downloaded 34GiB to GPU in 1.94882s (17.4464GiB/s) | 20260506 17:44:50.952 | INFO | Analyzing chunk frames = [3247104,3247616) | 20260506 17:44:50.952 | INFO | MemInfo: { | 20260506 17:44:50.952 | INFO | "gpu" : " 17 GB", | 20260506 17:44:50.952 | INFO | "huge" : "", | 20260506 17:44:50.953 | INFO | "huge_shared" : " 27 GB", | 20260506 17:44:50.953 | INFO | "malloc" : " 14 GB", | 20260506 17:44:50.953 | INFO | "pinned" : " 29 GB" | 20260506 17:44:50.953 | INFO | } | 20260506 17:44:50.953 | INFO | Perf report for the last 5.13651 seconds: | 20260506 17:44:50.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:44:50.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:50.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:50.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:50.953 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 17:44:50.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:50.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:50.953 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 17:44:50.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:50.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:50.953 | INFO | Uploaded 36GiB to GPU in 1.98568s (18.1298GiB/s) | 20260506 17:44:50.953 | INFO | Downloaded 34GiB to GPU in 2.0089s (16.9247GiB/s) | 20260506 17:44:56.090 | INFO | Analyzing chunk frames = [3247616,3248128) | 20260506 17:44:56.090 | INFO | MemInfo: { | 20260506 17:44:56.090 | INFO | "gpu" : " 17 GB", | 20260506 17:44:56.090 | INFO | "huge" : "", | 20260506 17:44:56.090 | INFO | "huge_shared" : " 27 GB", | 20260506 17:44:56.090 | INFO | "malloc" : " 14 GB", | 20260506 17:44:56.090 | INFO | "pinned" : " 29 GB" | 20260506 17:44:56.090 | INFO | } | 20260506 17:44:56.090 | INFO | Perf report for the last 5.13764 seconds: | 20260506 17:44:56.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:44:56.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:56.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:56.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:56.090 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:44:56.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:56.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:56.090 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:44:56.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:56.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:44:56.091 | INFO | Uploaded 36GiB to GPU in 2.01594s (17.8577GiB/s) | 20260506 17:44:56.091 | INFO | Downloaded 34GiB to GPU in 1.97547s (17.2111GiB/s) | 20260506 17:44:56.207 | INFO | Starting a round of Baz IO | 20260506 17:44:59.339 | INFO | Populating RTMetrics report | 20260506 17:44:59.339 | INFO | RTMetrics report populated | 20260506 17:44:59.339 | INFO | Opening tmp RTMetrics json file | 20260506 17:44:59.350 | INFO | Serializing RTMetrics to json | 20260506 17:44:59.370 | INFO | Serialization done, swapping tmp file now... | 20260506 17:44:59.374 | INFO | RTMetrics done | 20260506 17:44:59.996 | INFO | Finished a round of Baz IO: Wrote 2.057 GiB in 3.788 seconds | 20260506 17:44:59.997 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.46% / 0.01% / 64.53% | 20260506 17:44:59.997 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.29108% / 3.97364e-06% / 4.76837e-05% | 20260506 17:44:59.997 | INFO | Baz IO overall: | 20260506 17:44:59.997 | INFO | BytesWritten : 1713610657882 | 20260506 17:44:59.997 | INFO | NumEvents : 828930134243 | 20260506 17:45:01.235 | INFO | Analyzing chunk frames = [3248128,3248640) | 20260506 17:45:01.235 | INFO | MemInfo: { | 20260506 17:45:01.235 | INFO | "gpu" : " 17 GB", | 20260506 17:45:01.235 | INFO | "huge" : "", | 20260506 17:45:01.235 | INFO | "huge_shared" : " 27 GB", | 20260506 17:45:01.235 | INFO | "malloc" : " 14 GB", | 20260506 17:45:01.235 | INFO | "pinned" : " 29 GB" | 20260506 17:45:01.235 | INFO | } | 20260506 17:45:01.235 | INFO | Perf report for the last 5.14481 seconds: | 20260506 17:45:01.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:45:01.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:01.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:01.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:01.235 | INFO | ANALYSIS : 62.6%, 2.9, 95.5ms, 0 | 20260506 17:45:01.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:01.235 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 17:45:01.235 | INFO | PRE_HQ : 60.3%, 1.8, 57.7ms, 0 | 20260506 17:45:01.235 | INFO | BAZWRITER : 73.3%, 4.8, 181.6ms, 0 | 20260506 17:45:01.235 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260506 17:45:01.235 | INFO | Uploaded 36GiB to GPU in 2.0564s (17.5063GiB/s) | 20260506 17:45:01.236 | INFO | Downloaded 38GiB to GPU in 2.20805s (17.2097GiB/s) | 20260506 17:45:06.375 | INFO | Analyzing chunk frames = [3248640,3249152) | 20260506 17:45:06.375 | INFO | MemInfo: { | 20260506 17:45:06.375 | INFO | "gpu" : " 17 GB", | 20260506 17:45:06.375 | INFO | "huge" : "", | 20260506 17:45:06.375 | INFO | "huge_shared" : " 27 GB", | 20260506 17:45:06.375 | INFO | "malloc" : " 14 GB", | 20260506 17:45:06.375 | INFO | "pinned" : " 29 GB" | 20260506 17:45:06.375 | INFO | } | 20260506 17:45:06.375 | INFO | Perf report for the last 5.13997 seconds: | 20260506 17:45:06.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:45:06.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:06.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:06.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:06.375 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 17:45:06.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:06.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:06.375 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:45:06.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:06.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:06.375 | INFO | Uploaded 36GiB to GPU in 2.01249s (17.8883GiB/s) | 20260506 17:45:06.375 | INFO | Downloaded 34GiB to GPU in 1.9362s (17.5602GiB/s) | 20260506 17:45:11.513 | INFO | Analyzing chunk frames = [3249152,3249664) | 20260506 17:45:11.513 | INFO | MemInfo: { | 20260506 17:45:11.513 | INFO | "gpu" : " 17 GB", | 20260506 17:45:11.513 | INFO | "huge" : "", | 20260506 17:45:11.513 | INFO | "huge_shared" : " 27 GB", | 20260506 17:45:11.513 | INFO | "malloc" : " 14 GB", | 20260506 17:45:11.513 | INFO | "pinned" : " 29 GB" | 20260506 17:45:11.513 | INFO | } | 20260506 17:45:11.513 | INFO | Perf report for the last 5.13777 seconds: | 20260506 17:45:11.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:45:11.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:11.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:11.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:11.513 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:45:11.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:11.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:11.513 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:45:11.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:11.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:11.513 | INFO | Uploaded 36GiB to GPU in 1.97373s (18.2395GiB/s) | 20260506 17:45:11.513 | INFO | Downloaded 34GiB to GPU in 1.92216s (17.6884GiB/s) | 20260506 17:45:16.650 | INFO | Analyzing chunk frames = [3249664,3250176) | 20260506 17:45:16.650 | INFO | MemInfo: { | 20260506 17:45:16.650 | INFO | "gpu" : " 17 GB", | 20260506 17:45:16.650 | INFO | "huge" : "", | 20260506 17:45:16.650 | INFO | "huge_shared" : " 27 GB", | 20260506 17:45:16.650 | INFO | "malloc" : " 14 GB", | 20260506 17:45:16.650 | INFO | "pinned" : " 29 GB" | 20260506 17:45:16.650 | INFO | } | 20260506 17:45:16.651 | INFO | Perf report for the last 5.13771 seconds: | 20260506 17:45:16.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:45:16.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:16.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:16.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:16.651 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:45:16.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:16.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:16.651 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:45:16.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:16.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:16.651 | INFO | Uploaded 36GiB to GPU in 1.94027s (18.5542GiB/s) | 20260506 17:45:16.651 | INFO | Downloaded 34GiB to GPU in 1.87545s (18.129GiB/s) | 20260506 17:45:21.789 | INFO | Analyzing chunk frames = [3250176,3250688) | 20260506 17:45:21.789 | INFO | MemInfo: { | 20260506 17:45:21.789 | INFO | "gpu" : " 17 GB", | 20260506 17:45:21.789 | INFO | "huge" : "", | 20260506 17:45:21.789 | INFO | "huge_shared" : " 27 GB", | 20260506 17:45:21.789 | INFO | "malloc" : " 14 GB", | 20260506 17:45:21.790 | INFO | "pinned" : " 29 GB" | 20260506 17:45:21.790 | INFO | } | 20260506 17:45:21.790 | INFO | Perf report for the last 5.13905 seconds: | 20260506 17:45:21.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:45:21.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:21.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:21.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:21.790 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 17:45:21.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:21.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:21.790 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:45:21.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:21.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:21.790 | INFO | Uploaded 36GiB to GPU in 2.01785s (17.8408GiB/s) | 20260506 17:45:21.790 | INFO | Downloaded 34GiB to GPU in 1.91058s (17.7956GiB/s) | 20260506 17:45:26.926 | INFO | Analyzing chunk frames = [3250688,3251200) | 20260506 17:45:26.926 | INFO | MemInfo: { | 20260506 17:45:26.926 | INFO | "gpu" : " 17 GB", | 20260506 17:45:26.926 | INFO | "huge" : "", | 20260506 17:45:26.926 | INFO | "huge_shared" : " 27 GB", | 20260506 17:45:26.926 | INFO | "malloc" : " 14 GB", | 20260506 17:45:26.926 | INFO | "pinned" : " 29 GB" | 20260506 17:45:26.926 | INFO | } | 20260506 17:45:26.926 | INFO | Perf report for the last 5.13632 seconds: | 20260506 17:45:26.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:45:26.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:26.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:26.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:26.926 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 17:45:26.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:26.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:26.926 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 17:45:26.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:26.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:26.926 | INFO | Uploaded 36GiB to GPU in 2.02577s (17.771GiB/s) | 20260506 17:45:26.926 | INFO | Downloaded 34GiB to GPU in 1.94516s (17.4793GiB/s) | 20260506 17:45:32.065 | INFO | Analyzing chunk frames = [3251200,3251712) | 20260506 17:45:32.065 | INFO | MemInfo: { | 20260506 17:45:32.065 | INFO | "gpu" : " 17 GB", | 20260506 17:45:32.065 | INFO | "huge" : "", | 20260506 17:45:32.065 | INFO | "huge_shared" : " 27 GB", | 20260506 17:45:32.065 | INFO | "malloc" : " 14 GB", | 20260506 17:45:32.065 | INFO | "pinned" : " 29 GB" | 20260506 17:45:32.065 | INFO | } | 20260506 17:45:32.065 | INFO | Perf report for the last 5.13908 seconds: | 20260506 17:45:32.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:45:32.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:32.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:32.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:32.065 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 17:45:32.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:32.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:32.065 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:45:32.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:32.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:32.065 | INFO | Uploaded 36GiB to GPU in 1.99312s (18.0621GiB/s) | 20260506 17:45:32.065 | INFO | Downloaded 34GiB to GPU in 2.00847s (16.9283GiB/s) | 20260506 17:45:37.207 | INFO | Analyzing chunk frames = [3251712,3252224) | 20260506 17:45:37.207 | INFO | MemInfo: { | 20260506 17:45:37.207 | INFO | "gpu" : " 17 GB", | 20260506 17:45:37.207 | INFO | "huge" : "", | 20260506 17:45:37.207 | INFO | "huge_shared" : " 27 GB", | 20260506 17:45:37.207 | INFO | "malloc" : " 14 GB", | 20260506 17:45:37.207 | INFO | "pinned" : " 29 GB" | 20260506 17:45:37.207 | INFO | } | 20260506 17:45:37.207 | INFO | Perf report for the last 5.14193 seconds: | 20260506 17:45:37.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:45:37.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:37.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:37.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:37.207 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:45:37.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:37.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:37.207 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 17:45:37.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:37.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:37.207 | INFO | Uploaded 36GiB to GPU in 1.98492s (18.1367GiB/s) | 20260506 17:45:37.207 | INFO | Downloaded 34GiB to GPU in 1.97407s (17.2233GiB/s) | 20260506 17:45:37.208 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:45:37.208 | INFO | Total mem usage: 18 GiB | 20260506 17:45:37.208 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:45:37.208 | INFO | Recent High Water Marks Sum: 16.9 GiB (Savings: -6.46%) | 20260506 17:45:37.208 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:45:37.208 | INFO | High Water Breakdown: | 20260506 17:45:37.208 | INFO | 62.6% : 10.6 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:45:37.208 | INFO | 37.3% : 6.31 GiB ( 94.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:45:37.208 | INFO | 0.118% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:45:37.208 | INFO | ------------------------------------------ | 20260506 17:45:37.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:45:37.208 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:45:37.208 | INFO | Total mem usage: 26.5 GiB | 20260506 17:45:37.208 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:45:37.208 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 17:45:37.208 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 17:45:37.208 | INFO | High Water Breakdown: | 20260506 17:45:37.208 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:45:37.208 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:45:37.208 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:45:37.208 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:45:37.208 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:45:37.208 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:45:37.208 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:45:37.208 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:45:37.208 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:45:37.208 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:45:37.208 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:45:37.208 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:45:37.208 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:45:37.208 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:45:37.208 | INFO | ------------------------------------------ | 20260506 17:45:37.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:45:37.208 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:45:37.208 | INFO | Total mem usage: 25.6 GiB | 20260506 17:45:37.208 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:45:37.208 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:45:37.208 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:45:37.208 | INFO | High Water Breakdown: | 20260506 17:45:37.208 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:45:37.208 | INFO | ------------------------------------------ | 20260506 17:45:37.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:45:37.208 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:45:37.208 | INFO | Total mem usage: 15.7 GiB | 20260506 17:45:37.208 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:45:37.208 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:45:37.208 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 17:45:37.208 | INFO | High Water Breakdown: | 20260506 17:45:37.208 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:45:37.208 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:45:37.208 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:45:37.208 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:45:37.208 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:45:37.208 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:45:37.208 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:45:37.208 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:45:37.208 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:45:37.208 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:45:37.208 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:45:37.209 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:45:37.209 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:45:37.209 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:45:37.209 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:45:37.209 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:45:37.209 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:45:37.209 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:45:37.209 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:45:37.209 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:45:37.209 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:45:37.209 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:45:37.209 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:45:37.209 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:45:37.209 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:45:37.209 | INFO | ------------------------------------------ | 20260506 17:45:37.209 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:45:37.209 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:45:37.209 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:45:37.209 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:45:37.209 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.64e+01, 1.35e+01 / 2.50e+00 | 20260506 17:45:37.209 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 1.02e-02 / 5.50e-03 | 20260506 17:45:37.209 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.48e+00, 5.56e-01 / 2.38e-01 | 20260506 17:45:37.209 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.48e+00, 1.84e-01 / 1.09e-01 | 20260506 17:45:37.209 | INFO | PerfReport: 16.00%, DME -- 4.01e+01 / 4.50e+01, 4.22e+00 / 2.94e+00 | 20260506 17:45:37.209 | INFO | PerfReport: 56.93%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.35e+00 / 1.79e+00 | 20260506 17:45:37.209 | INFO | PerfReport: 7.28%, PulseAccumulating -- 2.28e+00 / 2.86e+00, 3.86e-01 / 1.97e-01 | 20260506 17:45:37.209 | INFO | PerfReport: 3.44%, Metrics -- 1.08e+00 / 1.42e+00, 2.79e-01 / 1.36e-01 | 20260506 17:45:37.209 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.72e-01 / 4.22e-01, 5.77e-02 / 2.47e-02 | 20260506 17:45:37.209 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:45:37.209 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:45:37.209 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:45:37.209 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 17:45:37.209 | INFO | PerfReport: 21.41%, Upload -- 2.08e+01 / 0.00e+00, 6.85e+00 / 0.00e+00 | 20260506 17:45:37.209 | INFO | PerfReport: 21.21%, Download -- 2.06e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 17:45:37.209 | INFO | PerfReport: 32.22%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 17:45:37.209 | INFO | PerfReport: 1.35%, LockUpload -- 1.32e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260506 17:45:37.209 | INFO | PerfReport: 1.23%, LockDownload -- 1.19e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260506 17:45:37.209 | INFO | PerfReport: 22.56%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 17:45:37.209 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260506 17:45:37.322 | INFO | Starting a round of Baz IO | 20260506 17:45:40.461 | INFO | Populating RTMetrics report | 20260506 17:45:40.461 | INFO | RTMetrics report populated | 20260506 17:45:40.461 | INFO | Opening tmp RTMetrics json file | 20260506 17:45:40.479 | INFO | Serializing RTMetrics to json | 20260506 17:45:40.481 | INFO | Serialization done, swapping tmp file now... | 20260506 17:45:40.481 | INFO | RTMetrics done | 20260506 17:45:40.933 | INFO | Finished a round of Baz IO: Wrote 2.057 GiB in 3.61 seconds | 20260506 17:45:40.933 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.45% / 0.01% / 64.54% | 20260506 17:45:40.933 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.28875% / 3.97364e-06% / 5.96046e-05% | 20260506 17:45:40.933 | INFO | Baz IO overall: | 20260506 17:45:40.933 | INFO | BytesWritten : 1715819756939 | 20260506 17:45:40.933 | INFO | NumEvents : 829995390648 | 20260506 17:45:42.350 | INFO | Analyzing chunk frames = [3252224,3252736) | 20260506 17:45:42.350 | INFO | MemInfo: { | 20260506 17:45:42.350 | INFO | "gpu" : " 17 GB", | 20260506 17:45:42.350 | INFO | "huge" : "", | 20260506 17:45:42.350 | INFO | "huge_shared" : " 27 GB", | 20260506 17:45:42.350 | INFO | "malloc" : " 14 GB", | 20260506 17:45:42.350 | INFO | "pinned" : " 29 GB" | 20260506 17:45:42.350 | INFO | } | 20260506 17:45:42.350 | INFO | Perf report for the last 5.14265 seconds: | 20260506 17:45:42.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:45:42.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:42.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:42.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:42.350 | INFO | ANALYSIS : 62.7%, 2.9, 95.4ms, 0 | 20260506 17:45:42.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:42.350 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 17:45:42.350 | INFO | PRE_HQ : 60.1%, 1.9, 58.4ms, 0 | 20260506 17:45:42.350 | INFO | BAZWRITER : 69.9%, 4.8, 170.7ms, 0 | 20260506 17:45:42.350 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 17:45:42.350 | INFO | Uploaded 36GiB to GPU in 2.01044s (17.9066GiB/s) | 20260506 17:45:42.350 | INFO | Downloaded 38GiB to GPU in 2.20946s (17.1988GiB/s) | 20260506 17:45:47.494 | INFO | Analyzing chunk frames = [3252736,3253248) | 20260506 17:45:47.494 | INFO | MemInfo: { | 20260506 17:45:47.494 | INFO | "gpu" : " 17 GB", | 20260506 17:45:47.494 | INFO | "huge" : "", | 20260506 17:45:47.494 | INFO | "huge_shared" : " 27 GB", | 20260506 17:45:47.494 | INFO | "malloc" : " 14 GB", | 20260506 17:45:47.495 | INFO | "pinned" : " 29 GB" | 20260506 17:45:47.495 | INFO | } | 20260506 17:45:47.495 | INFO | Perf report for the last 5.14492 seconds: | 20260506 17:45:47.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:45:47.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:47.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:47.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:47.495 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 17:45:47.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:47.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:47.495 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:45:47.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:47.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:47.495 | INFO | Uploaded 36GiB to GPU in 1.98454s (18.1402GiB/s) | 20260506 17:45:47.495 | INFO | Downloaded 34GiB to GPU in 1.92644s (17.6491GiB/s) | 20260506 17:45:52.630 | INFO | Analyzing chunk frames = [3253248,3253760) | 20260506 17:45:52.630 | INFO | MemInfo: { | 20260506 17:45:52.630 | INFO | "gpu" : " 17 GB", | 20260506 17:45:52.630 | INFO | "huge" : "", | 20260506 17:45:52.630 | INFO | "huge_shared" : " 27 GB", | 20260506 17:45:52.630 | INFO | "malloc" : " 14 GB", | 20260506 17:45:52.630 | INFO | "pinned" : " 29 GB" | 20260506 17:45:52.630 | INFO | } | 20260506 17:45:52.630 | INFO | Perf report for the last 5.13538 seconds: | 20260506 17:45:52.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:45:52.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:52.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:52.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:52.630 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 17:45:52.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:52.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:52.630 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:45:52.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:52.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:52.630 | INFO | Uploaded 36GiB to GPU in 1.97941s (18.1873GiB/s) | 20260506 17:45:52.631 | INFO | Downloaded 34GiB to GPU in 1.92076s (17.7013GiB/s) | 20260506 17:45:57.768 | INFO | Analyzing chunk frames = [3253760,3254272) | 20260506 17:45:57.768 | INFO | MemInfo: { | 20260506 17:45:57.768 | INFO | "gpu" : " 17 GB", | 20260506 17:45:57.768 | INFO | "huge" : "", | 20260506 17:45:57.768 | INFO | "huge_shared" : " 27 GB", | 20260506 17:45:57.768 | INFO | "malloc" : " 14 GB", | 20260506 17:45:57.768 | INFO | "pinned" : " 29 GB" | 20260506 17:45:57.768 | INFO | } | 20260506 17:45:57.768 | INFO | Perf report for the last 5.13818 seconds: | 20260506 17:45:57.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:45:57.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:57.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:57.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:57.768 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 17:45:57.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:57.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:57.768 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:45:57.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:57.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:45:57.768 | INFO | Uploaded 36GiB to GPU in 1.91569s (18.7921GiB/s) | 20260506 17:45:57.769 | INFO | Downloaded 34GiB to GPU in 1.87225s (18.16GiB/s) | 20260506 17:46:02.905 | INFO | Analyzing chunk frames = [3254272,3254784) | 20260506 17:46:02.905 | INFO | MemInfo: { | 20260506 17:46:02.905 | INFO | "gpu" : " 17 GB", | 20260506 17:46:02.905 | INFO | "huge" : "", | 20260506 17:46:02.905 | INFO | "huge_shared" : " 27 GB", | 20260506 17:46:02.905 | INFO | "malloc" : " 14 GB", | 20260506 17:46:02.905 | INFO | "pinned" : " 29 GB" | 20260506 17:46:02.905 | INFO | } | 20260506 17:46:02.905 | INFO | Perf report for the last 5.13697 seconds: | 20260506 17:46:02.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:46:02.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:02.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:02.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:02.905 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 17:46:02.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:02.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:02.905 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 17:46:02.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:02.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:02.905 | INFO | Uploaded 36GiB to GPU in 2.01018s (17.9089GiB/s) | 20260506 17:46:02.905 | INFO | Downloaded 34GiB to GPU in 1.9104s (17.7973GiB/s) | 20260506 17:46:08.042 | INFO | Analyzing chunk frames = [3254784,3255296) | 20260506 17:46:08.042 | INFO | MemInfo: { | 20260506 17:46:08.042 | INFO | "gpu" : " 17 GB", | 20260506 17:46:08.042 | INFO | "huge" : "", | 20260506 17:46:08.042 | INFO | "huge_shared" : " 27 GB", | 20260506 17:46:08.042 | INFO | "malloc" : " 14 GB", | 20260506 17:46:08.042 | INFO | "pinned" : " 29 GB" | 20260506 17:46:08.042 | INFO | } | 20260506 17:46:08.042 | INFO | Perf report for the last 5.13688 seconds: | 20260506 17:46:08.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:46:08.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:08.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:08.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:08.042 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:46:08.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:08.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:08.042 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 17:46:08.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:08.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:08.042 | INFO | Uploaded 36GiB to GPU in 2.02666s (17.7632GiB/s) | 20260506 17:46:08.042 | INFO | Downloaded 34GiB to GPU in 1.94689s (17.4638GiB/s) | 20260506 17:46:13.183 | INFO | Analyzing chunk frames = [3255296,3255808) | 20260506 17:46:13.183 | INFO | MemInfo: { | 20260506 17:46:13.183 | INFO | "gpu" : " 17 GB", | 20260506 17:46:13.183 | INFO | "huge" : "", | 20260506 17:46:13.183 | INFO | "huge_shared" : " 27 GB", | 20260506 17:46:13.183 | INFO | "malloc" : " 14 GB", | 20260506 17:46:13.183 | INFO | "pinned" : " 29 GB" | 20260506 17:46:13.183 | INFO | } | 20260506 17:46:13.183 | INFO | Perf report for the last 5.14076 seconds: | 20260506 17:46:13.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:46:13.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:13.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:13.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:13.183 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 17:46:13.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:13.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:13.183 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 17:46:13.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:13.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:13.183 | INFO | Uploaded 36GiB to GPU in 2.00465s (17.9583GiB/s) | 20260506 17:46:13.183 | INFO | Downloaded 34GiB to GPU in 2.00115s (16.9902GiB/s) | 20260506 17:46:18.330 | INFO | Analyzing chunk frames = [3255808,3256320) | 20260506 17:46:18.330 | INFO | MemInfo: { | 20260506 17:46:18.330 | INFO | "gpu" : " 17 GB", | 20260506 17:46:18.330 | INFO | "huge" : "", | 20260506 17:46:18.330 | INFO | "huge_shared" : " 27 GB", | 20260506 17:46:18.330 | INFO | "malloc" : " 14 GB", | 20260506 17:46:18.330 | INFO | "pinned" : " 29 GB" | 20260506 17:46:18.330 | INFO | } | 20260506 17:46:18.330 | INFO | Perf report for the last 5.14702 seconds: | 20260506 17:46:18.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:46:18.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:18.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:18.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:18.330 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 17:46:18.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:18.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:18.330 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 17:46:18.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:18.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:18.330 | INFO | Uploaded 36GiB to GPU in 2.02855s (17.7467GiB/s) | 20260506 17:46:18.330 | INFO | Downloaded 34GiB to GPU in 1.98954s (17.0894GiB/s) | 20260506 17:46:18.449 | INFO | Starting a round of Baz IO | 20260506 17:46:21.583 | INFO | Populating RTMetrics report | 20260506 17:46:21.583 | INFO | RTMetrics report populated | 20260506 17:46:21.583 | INFO | Opening tmp RTMetrics json file | 20260506 17:46:21.633 | INFO | Serializing RTMetrics to json | 20260506 17:46:21.766 | INFO | Serialization done, swapping tmp file now... | 20260506 17:46:21.803 | INFO | RTMetrics done | 20260506 17:46:22.186 | INFO | Finished a round of Baz IO: Wrote 2.057 GiB in 3.736 seconds | 20260506 17:46:22.186 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.45% / 0.01% / 64.55% | 20260506 17:46:22.186 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.28471% / 7.94729e-06% / 9.53674e-05% | 20260506 17:46:22.186 | INFO | Baz IO overall: | 20260506 17:46:22.186 | INFO | BytesWritten : 1718028228282 | 20260506 17:46:22.186 | INFO | NumEvents : 831060311579 | 20260506 17:46:23.464 | INFO | Analyzing chunk frames = [3256320,3256832) | 20260506 17:46:23.464 | INFO | MemInfo: { | 20260506 17:46:23.464 | INFO | "gpu" : " 17 GB", | 20260506 17:46:23.464 | INFO | "huge" : "", | 20260506 17:46:23.464 | INFO | "huge_shared" : " 27 GB", | 20260506 17:46:23.464 | INFO | "malloc" : " 14 GB", | 20260506 17:46:23.464 | INFO | "pinned" : " 29 GB" | 20260506 17:46:23.464 | INFO | } | 20260506 17:46:23.465 | INFO | Perf report for the last 5.13465 seconds: | 20260506 17:46:23.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:46:23.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:23.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:23.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:23.465 | INFO | ANALYSIS : 62.8%, 2.9, 95.5ms, 0 | 20260506 17:46:23.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:23.465 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260506 17:46:23.465 | INFO | PRE_HQ : 60.2%, 1.9, 58.2ms, 0 | 20260506 17:46:23.465 | INFO | BAZWRITER : 72.4%, 4.8, 178.8ms, 0 | 20260506 17:46:23.465 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.5ms, 0 | 20260506 17:46:23.465 | INFO | Uploaded 36GiB to GPU in 2.03639s (17.6784GiB/s) | 20260506 17:46:23.465 | INFO | Downloaded 38GiB to GPU in 2.22043s (17.1138GiB/s) | 20260506 17:46:28.608 | INFO | Analyzing chunk frames = [3256832,3257344) | 20260506 17:46:28.608 | INFO | MemInfo: { | 20260506 17:46:28.608 | INFO | "gpu" : " 17 GB", | 20260506 17:46:28.608 | INFO | "huge" : "", | 20260506 17:46:28.608 | INFO | "huge_shared" : " 27 GB", | 20260506 17:46:28.608 | INFO | "malloc" : " 14 GB", | 20260506 17:46:28.608 | INFO | "pinned" : " 29 GB" | 20260506 17:46:28.608 | INFO | } | 20260506 17:46:28.609 | INFO | Perf report for the last 5.14397 seconds: | 20260506 17:46:28.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:46:28.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:28.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:28.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:28.609 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 17:46:28.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:28.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:28.609 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 17:46:28.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:28.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:28.609 | INFO | Uploaded 36GiB to GPU in 1.98925s (18.0973GiB/s) | 20260506 17:46:28.609 | INFO | Downloaded 34GiB to GPU in 1.92921s (17.6238GiB/s) | 20260506 17:46:33.741 | INFO | Analyzing chunk frames = [3257344,3257856) | 20260506 17:46:33.741 | INFO | MemInfo: { | 20260506 17:46:33.741 | INFO | "gpu" : " 17 GB", | 20260506 17:46:33.741 | INFO | "huge" : "", | 20260506 17:46:33.741 | INFO | "huge_shared" : " 27 GB", | 20260506 17:46:33.741 | INFO | "malloc" : " 14 GB", | 20260506 17:46:33.741 | INFO | "pinned" : " 29 GB" | 20260506 17:46:33.741 | INFO | } | 20260506 17:46:33.741 | INFO | Perf report for the last 5.13286 seconds: | 20260506 17:46:33.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:46:33.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:33.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:33.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:33.741 | INFO | ANALYSIS : 61.0%, 3.0, 94.2ms, 0 | 20260506 17:46:33.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:33.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:33.741 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:46:33.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:33.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:33.742 | INFO | Uploaded 36GiB to GPU in 1.95572s (18.4075GiB/s) | 20260506 17:46:33.742 | INFO | Downloaded 34GiB to GPU in 1.92094s (17.6997GiB/s) | 20260506 17:46:38.878 | INFO | Analyzing chunk frames = [3257856,3258368) | 20260506 17:46:38.878 | INFO | MemInfo: { | 20260506 17:46:38.878 | INFO | "gpu" : " 17 GB", | 20260506 17:46:38.878 | INFO | "huge" : "", | 20260506 17:46:38.879 | INFO | "huge_shared" : " 27 GB", | 20260506 17:46:38.879 | INFO | "malloc" : " 14 GB", | 20260506 17:46:38.879 | INFO | "pinned" : " 29 GB" | 20260506 17:46:38.879 | INFO | } | 20260506 17:46:38.879 | INFO | Perf report for the last 5.13729 seconds: | 20260506 17:46:38.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:46:38.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:38.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:38.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:38.879 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:46:38.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:38.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:38.879 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:46:38.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:38.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:38.879 | INFO | Uploaded 36GiB to GPU in 1.92241s (18.7265GiB/s) | 20260506 17:46:38.879 | INFO | Downloaded 34GiB to GPU in 1.87178s (18.1645GiB/s) | 20260506 17:46:44.016 | INFO | Analyzing chunk frames = [3258368,3258880) | 20260506 17:46:44.016 | INFO | MemInfo: { | 20260506 17:46:44.016 | INFO | "gpu" : " 17 GB", | 20260506 17:46:44.016 | INFO | "huge" : "", | 20260506 17:46:44.016 | INFO | "huge_shared" : " 27 GB", | 20260506 17:46:44.016 | INFO | "malloc" : " 14 GB", | 20260506 17:46:44.016 | INFO | "pinned" : " 29 GB" | 20260506 17:46:44.016 | INFO | } | 20260506 17:46:44.016 | INFO | Perf report for the last 5.13759 seconds: | 20260506 17:46:44.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:46:44.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:44.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:44.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:44.016 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 17:46:44.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:44.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:44.016 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:46:44.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:44.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:44.017 | INFO | Uploaded 36GiB to GPU in 2.0149s (17.8669GiB/s) | 20260506 17:46:44.017 | INFO | Downloaded 34GiB to GPU in 1.91433s (17.7608GiB/s) | 20260506 17:46:49.154 | INFO | Analyzing chunk frames = [3258880,3259392) | 20260506 17:46:49.154 | INFO | MemInfo: { | 20260506 17:46:49.154 | INFO | "gpu" : " 17 GB", | 20260506 17:46:49.154 | INFO | "huge" : "", | 20260506 17:46:49.154 | INFO | "huge_shared" : " 27 GB", | 20260506 17:46:49.154 | INFO | "malloc" : " 14 GB", | 20260506 17:46:49.154 | INFO | "pinned" : " 29 GB" | 20260506 17:46:49.155 | INFO | } | 20260506 17:46:49.155 | INFO | Perf report for the last 5.13829 seconds: | 20260506 17:46:49.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:46:49.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:49.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:49.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:49.155 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 17:46:49.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:49.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:49.155 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:46:49.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:49.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:49.155 | INFO | Uploaded 36GiB to GPU in 2.01792s (17.8402GiB/s) | 20260506 17:46:49.155 | INFO | Downloaded 34GiB to GPU in 1.95351s (17.4046GiB/s) | 20260506 17:46:54.301 | INFO | Analyzing chunk frames = [3259392,3259904) | 20260506 17:46:54.301 | INFO | MemInfo: { | 20260506 17:46:54.301 | INFO | "gpu" : " 17 GB", | 20260506 17:46:54.302 | INFO | "huge" : "", | 20260506 17:46:54.302 | INFO | "huge_shared" : " 27 GB", | 20260506 17:46:54.302 | INFO | "malloc" : " 14 GB", | 20260506 17:46:54.302 | INFO | "pinned" : " 29 GB" | 20260506 17:46:54.302 | INFO | } | 20260506 17:46:54.302 | INFO | Perf report for the last 5.14715 seconds: | 20260506 17:46:54.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:46:54.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:54.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:54.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:54.302 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 17:46:54.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:54.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:54.302 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 17:46:54.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:54.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:54.302 | INFO | Uploaded 36GiB to GPU in 2.00317s (17.9715GiB/s) | 20260506 17:46:54.302 | INFO | Downloaded 34GiB to GPU in 2.01537s (16.8704GiB/s) | 20260506 17:46:59.442 | INFO | Analyzing chunk frames = [3259904,3260416) | 20260506 17:46:59.442 | INFO | MemInfo: { | 20260506 17:46:59.442 | INFO | "gpu" : " 17 GB", | 20260506 17:46:59.442 | INFO | "huge" : "", | 20260506 17:46:59.442 | INFO | "huge_shared" : " 27 GB", | 20260506 17:46:59.442 | INFO | "malloc" : " 14 GB", | 20260506 17:46:59.442 | INFO | "pinned" : " 29 GB" | 20260506 17:46:59.442 | INFO | } | 20260506 17:46:59.442 | INFO | Perf report for the last 5.14039 seconds: | 20260506 17:46:59.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:46:59.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:59.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:59.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:59.442 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 17:46:59.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:59.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:59.442 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 17:46:59.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:59.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:46:59.443 | INFO | Uploaded 36GiB to GPU in 2.01814s (17.8382GiB/s) | 20260506 17:46:59.443 | INFO | Downloaded 34GiB to GPU in 1.98922s (17.0921GiB/s) | 20260506 17:46:59.443 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:46:59.443 | INFO | Total mem usage: 18 GiB | 20260506 17:46:59.443 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:46:59.443 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: -9.35%) | 20260506 17:46:59.443 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:46:59.443 | INFO | High Water Breakdown: | 20260506 17:46:59.443 | INFO | 62.2% : 10.2 GiB ( 89.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:46:59.443 | INFO | 37.7% : 6.21 GiB ( 92.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:46:59.443 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:46:59.443 | INFO | ------------------------------------------ | 20260506 17:46:59.443 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:46:59.443 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:46:59.443 | INFO | Total mem usage: 26.5 GiB | 20260506 17:46:59.443 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:46:59.443 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:46:59.443 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:46:59.443 | INFO | High Water Breakdown: | 20260506 17:46:59.443 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:46:59.443 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:46:59.443 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:46:59.443 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:46:59.443 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:46:59.443 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:46:59.443 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:46:59.443 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:46:59.443 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:46:59.443 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:46:59.443 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:46:59.443 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:46:59.443 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:46:59.443 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:46:59.443 | INFO | ------------------------------------------ | 20260506 17:46:59.443 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:46:59.443 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:46:59.443 | INFO | Total mem usage: 25.6 GiB | 20260506 17:46:59.443 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:46:59.443 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:46:59.443 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:46:59.443 | INFO | High Water Breakdown: | 20260506 17:46:59.443 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:46:59.443 | INFO | ------------------------------------------ | 20260506 17:46:59.443 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:46:59.443 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:46:59.443 | INFO | Total mem usage: 15.7 GiB | 20260506 17:46:59.443 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:46:59.443 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:46:59.443 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 17:46:59.443 | INFO | High Water Breakdown: | 20260506 17:46:59.443 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:46:59.443 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:46:59.443 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:46:59.443 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:46:59.443 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:46:59.443 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:46:59.443 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:46:59.443 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:46:59.443 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:46:59.443 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:46:59.443 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:46:59.443 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:46:59.444 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:46:59.444 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:46:59.444 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:46:59.444 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:46:59.444 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:46:59.444 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:46:59.444 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:46:59.444 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:46:59.444 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:46:59.444 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:46:59.444 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:46:59.444 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:46:59.444 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:46:59.444 | INFO | ------------------------------------------ | 20260506 17:46:59.444 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:46:59.444 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:46:59.444 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:46:59.444 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:46:59.444 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.61e+01, 1.34e+01 / 2.56e+00 | 20260506 17:46:59.444 | INFO | PerfReport: 0.29%, AggregateStats -- 9.21e-02 / 1.04e-01, 1.02e-02 / 1.13e-02 | 20260506 17:46:59.444 | INFO | PerfReport: 11.61%, Baseline -- 3.64e+00 / 4.42e+00, 5.61e-01 / 3.43e-01 | 20260506 17:46:59.444 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.86e-01 / 1.44e-01 | 20260506 17:46:59.444 | INFO | PerfReport: 15.97%, DME -- 4.01e+01 / 4.47e+01, 4.14e+00 / 2.82e+00 | 20260506 17:46:59.444 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.46e+00 / 1.73e+00 | 20260506 17:46:59.444 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.26e+00 / 2.82e+00, 3.80e-01 / 2.38e-01 | 20260506 17:46:59.444 | INFO | PerfReport: 3.43%, Metrics -- 1.07e+00 / 1.42e+00, 2.81e-01 / 1.74e-01 | 20260506 17:46:59.444 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.74e-01 / 4.21e-01, 1.01e-01 / 2.40e-02 | 20260506 17:46:59.444 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:46:59.444 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:46:59.444 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:46:59.444 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 17:46:59.444 | INFO | PerfReport: 21.37%, Upload -- 2.08e+01 / 0.00e+00, 6.87e+00 / 0.00e+00 | 20260506 17:46:59.444 | INFO | PerfReport: 21.16%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 17:46:59.444 | INFO | PerfReport: 32.25%, Compute -- 3.14e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:46:59.444 | INFO | PerfReport: 1.30%, LockUpload -- 1.26e+00 / 0.00e+00, 5.44e+00 / 0.00e+00 | 20260506 17:46:59.444 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260506 17:46:59.444 | INFO | PerfReport: 22.68%, LockCompute -- 2.21e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 17:46:59.444 | INFO | PerfReport: 0.02%, Misc -- 1.52e-02 / 0.00e+00, 2.72e-02 / 0.00e+00 | 20260506 17:46:59.562 | INFO | Starting a round of Baz IO | 20260506 17:47:02.700 | INFO | Populating RTMetrics report | 20260506 17:47:02.701 | INFO | RTMetrics report populated | 20260506 17:47:02.701 | INFO | Opening tmp RTMetrics json file | 20260506 17:47:02.827 | INFO | Serializing RTMetrics to json | 20260506 17:47:02.837 | INFO | Serialization done, swapping tmp file now... | 20260506 17:47:02.839 | INFO | RTMetrics done | 20260506 17:47:02.874 | INFO | Finished a round of Baz IO: Wrote 2.056 GiB in 3.311 seconds | 20260506 17:47:02.874 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.44% / 0.01% / 64.55% | 20260506 17:47:02.874 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.28332% / 3.97364e-06% / 7.54992e-05% | 20260506 17:47:02.874 | INFO | Baz IO overall: | 20260506 17:47:02.874 | INFO | BytesWritten : 1720236052453 | 20260506 17:47:02.874 | INFO | NumEvents : 832124826716 | 20260506 17:47:04.585 | INFO | Analyzing chunk frames = [3260416,3260928) | 20260506 17:47:04.585 | INFO | MemInfo: { | 20260506 17:47:04.585 | INFO | "gpu" : " 17 GB", | 20260506 17:47:04.585 | INFO | "huge" : "", | 20260506 17:47:04.585 | INFO | "huge_shared" : " 27 GB", | 20260506 17:47:04.585 | INFO | "malloc" : " 14 GB", | 20260506 17:47:04.585 | INFO | "pinned" : " 29 GB" | 20260506 17:47:04.585 | INFO | } | 20260506 17:47:04.585 | INFO | Perf report for the last 5.14263 seconds: | 20260506 17:47:04.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:47:04.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:04.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:04.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:04.585 | INFO | ANALYSIS : 62.8%, 2.9, 95.1ms, 0 | 20260506 17:47:04.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:04.585 | INFO | RT_METRICS : 46.2%, 1.0, 24.3ms, 0 | 20260506 17:47:04.585 | INFO | PRE_HQ : 60.3%, 1.9, 58.8ms, 0 | 20260506 17:47:04.585 | INFO | BAZWRITER : 64.1%, 4.6, 150.7ms, 0 | 20260506 17:47:04.585 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 17:47:04.585 | INFO | Uploaded 36GiB to GPU in 2.05976s (17.4778GiB/s) | 20260506 17:47:04.585 | INFO | Downloaded 38GiB to GPU in 2.22454s (17.0822GiB/s) | 20260506 17:47:09.716 | INFO | Analyzing chunk frames = [3260928,3261440) | 20260506 17:47:09.716 | INFO | MemInfo: { | 20260506 17:47:09.716 | INFO | "gpu" : " 17 GB", | 20260506 17:47:09.716 | INFO | "huge" : "", | 20260506 17:47:09.716 | INFO | "huge_shared" : " 27 GB", | 20260506 17:47:09.717 | INFO | "malloc" : " 14 GB", | 20260506 17:47:09.717 | INFO | "pinned" : " 29 GB" | 20260506 17:47:09.717 | INFO | } | 20260506 17:47:09.717 | INFO | Perf report for the last 5.13189 seconds: | 20260506 17:47:09.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:47:09.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:09.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:09.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:09.717 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:47:09.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:09.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:09.717 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:47:09.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:09.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:09.717 | INFO | Uploaded 36GiB to GPU in 2.01274s (17.8861GiB/s) | 20260506 17:47:09.717 | INFO | Downloaded 34GiB to GPU in 1.92419s (17.6698GiB/s) | 20260506 17:47:14.853 | INFO | Analyzing chunk frames = [3261440,3261952) | 20260506 17:47:14.853 | INFO | MemInfo: { | 20260506 17:47:14.853 | INFO | "gpu" : " 17 GB", | 20260506 17:47:14.853 | INFO | "huge" : "", | 20260506 17:47:14.853 | INFO | "huge_shared" : " 27 GB", | 20260506 17:47:14.853 | INFO | "malloc" : " 14 GB", | 20260506 17:47:14.853 | INFO | "pinned" : " 29 GB" | 20260506 17:47:14.853 | INFO | } | 20260506 17:47:14.853 | INFO | Perf report for the last 5.13675 seconds: | 20260506 17:47:14.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:47:14.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:14.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:14.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:14.854 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:47:14.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:14.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:14.854 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:47:14.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:14.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:14.854 | INFO | Uploaded 36GiB to GPU in 1.97769s (18.2031GiB/s) | 20260506 17:47:14.854 | INFO | Downloaded 34GiB to GPU in 1.92677s (17.6461GiB/s) | 20260506 17:47:19.994 | INFO | Analyzing chunk frames = [3261952,3262464) | 20260506 17:47:19.994 | INFO | MemInfo: { | 20260506 17:47:19.994 | INFO | "gpu" : " 17 GB", | 20260506 17:47:19.994 | INFO | "huge" : "", | 20260506 17:47:19.994 | INFO | "huge_shared" : " 27 GB", | 20260506 17:47:19.994 | INFO | "malloc" : " 14 GB", | 20260506 17:47:19.994 | INFO | "pinned" : " 29 GB" | 20260506 17:47:19.994 | INFO | } | 20260506 17:47:19.994 | INFO | Perf report for the last 5.14084 seconds: | 20260506 17:47:19.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:47:19.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:19.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:19.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:19.994 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 17:47:19.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:19.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:19.994 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:47:19.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:19.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:19.995 | INFO | Uploaded 36GiB to GPU in 1.93636s (18.5916GiB/s) | 20260506 17:47:19.995 | INFO | Downloaded 34GiB to GPU in 1.86705s (18.2105GiB/s) | 20260506 17:47:25.135 | INFO | Analyzing chunk frames = [3262464,3262976) | 20260506 17:47:25.135 | INFO | MemInfo: { | 20260506 17:47:25.135 | INFO | "gpu" : " 17 GB", | 20260506 17:47:25.135 | INFO | "huge" : "", | 20260506 17:47:25.135 | INFO | "huge_shared" : " 27 GB", | 20260506 17:47:25.135 | INFO | "malloc" : " 14 GB", | 20260506 17:47:25.135 | INFO | "pinned" : " 29 GB" | 20260506 17:47:25.135 | INFO | } | 20260506 17:47:25.135 | INFO | Perf report for the last 5.14103 seconds: | 20260506 17:47:25.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:47:25.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:25.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:25.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:25.135 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 17:47:25.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:25.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:25.135 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:47:25.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:25.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:25.136 | INFO | Uploaded 36GiB to GPU in 2.02057s (17.8168GiB/s) | 20260506 17:47:25.136 | INFO | Downloaded 34GiB to GPU in 1.91347s (17.7688GiB/s) | 20260506 17:47:30.276 | INFO | Analyzing chunk frames = [3262976,3263488) | 20260506 17:47:30.276 | INFO | MemInfo: { | 20260506 17:47:30.276 | INFO | "gpu" : " 17 GB", | 20260506 17:47:30.276 | INFO | "huge" : "", | 20260506 17:47:30.276 | INFO | "huge_shared" : " 27 GB", | 20260506 17:47:30.277 | INFO | "malloc" : " 14 GB", | 20260506 17:47:30.277 | INFO | "pinned" : " 29 GB" | 20260506 17:47:30.277 | INFO | } | 20260506 17:47:30.277 | INFO | Perf report for the last 5.14124 seconds: | 20260506 17:47:30.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:47:30.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:30.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:30.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:30.277 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 17:47:30.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:30.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:30.277 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:47:30.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:30.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:30.277 | INFO | Uploaded 36GiB to GPU in 2.03564s (17.6849GiB/s) | 20260506 17:47:30.277 | INFO | Downloaded 34GiB to GPU in 1.94413s (17.4885GiB/s) | 20260506 17:47:35.415 | INFO | Analyzing chunk frames = [3263488,3264000) | 20260506 17:47:35.415 | INFO | MemInfo: { | 20260506 17:47:35.415 | INFO | "gpu" : " 17 GB", | 20260506 17:47:35.415 | INFO | "huge" : "", | 20260506 17:47:35.415 | INFO | "huge_shared" : " 27 GB", | 20260506 17:47:35.415 | INFO | "malloc" : " 14 GB", | 20260506 17:47:35.415 | INFO | "pinned" : " 29 GB" | 20260506 17:47:35.415 | INFO | } | 20260506 17:47:35.415 | INFO | Perf report for the last 5.13834 seconds: | 20260506 17:47:35.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:47:35.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:35.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:35.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:35.415 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:47:35.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:35.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:35.415 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:47:35.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:35.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:35.415 | INFO | Uploaded 36GiB to GPU in 2.00905s (17.9189GiB/s) | 20260506 17:47:35.415 | INFO | Downloaded 34GiB to GPU in 2.01709s (16.856GiB/s) | 20260506 17:47:40.554 | INFO | Analyzing chunk frames = [3264000,3264512) | 20260506 17:47:40.554 | INFO | MemInfo: { | 20260506 17:47:40.554 | INFO | "gpu" : " 17 GB", | 20260506 17:47:40.554 | INFO | "huge" : "", | 20260506 17:47:40.554 | INFO | "huge_shared" : " 27 GB", | 20260506 17:47:40.554 | INFO | "malloc" : " 14 GB", | 20260506 17:47:40.554 | INFO | "pinned" : " 29 GB" | 20260506 17:47:40.554 | INFO | } | 20260506 17:47:40.554 | INFO | Perf report for the last 5.13947 seconds: | 20260506 17:47:40.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:47:40.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:40.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:40.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:40.555 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:47:40.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:40.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:40.555 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:47:40.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:40.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:40.555 | INFO | Uploaded 36GiB to GPU in 2.01793s (17.8401GiB/s) | 20260506 17:47:40.555 | INFO | Downloaded 34GiB to GPU in 1.99638s (17.0308GiB/s) | 20260506 17:47:40.673 | INFO | Starting a round of Baz IO | 20260506 17:47:43.802 | INFO | Populating RTMetrics report | 20260506 17:47:43.802 | INFO | RTMetrics report populated | 20260506 17:47:43.802 | INFO | Opening tmp RTMetrics json file | 20260506 17:47:43.895 | INFO | Serializing RTMetrics to json | 20260506 17:47:44.028 | INFO | Serialization done, swapping tmp file now... | 20260506 17:47:44.032 | INFO | RTMetrics done | 20260506 17:47:44.238 | INFO | Finished a round of Baz IO: Wrote 2.057 GiB in 3.565 seconds | 20260506 17:47:44.238 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.43% / 0.01% / 64.56% | 20260506 17:47:44.238 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.28109% / 1.19209e-05% / 5.96046e-05% | 20260506 17:47:44.238 | INFO | Baz IO overall: | 20260506 17:47:44.238 | INFO | BytesWritten : 1722444799800 | 20260506 17:47:44.238 | INFO | NumEvents : 833190126998 | 20260506 17:47:45.694 | INFO | Analyzing chunk frames = [3264512,3265024) | 20260506 17:47:45.694 | INFO | MemInfo: { | 20260506 17:47:45.694 | INFO | "gpu" : " 17 GB", | 20260506 17:47:45.694 | INFO | "huge" : "", | 20260506 17:47:45.694 | INFO | "huge_shared" : " 27 GB", | 20260506 17:47:45.694 | INFO | "malloc" : " 14 GB", | 20260506 17:47:45.694 | INFO | "pinned" : " 29 GB" | 20260506 17:47:45.694 | INFO | } | 20260506 17:47:45.695 | INFO | Perf report for the last 5.14012 seconds: | 20260506 17:47:45.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:47:45.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:45.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:45.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:45.695 | INFO | ANALYSIS : 62.6%, 2.9, 95.4ms, 0 | 20260506 17:47:45.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:45.695 | INFO | RT_METRICS : 47.8%, 1.0, 25.1ms, 0 | 20260506 17:47:45.695 | INFO | PRE_HQ : 60.2%, 1.9, 57.9ms, 0 | 20260506 17:47:45.695 | INFO | BAZWRITER : 69.0%, 4.8, 171.4ms, 0 | 20260506 17:47:45.695 | INFO | BAZ_SAMPLING : 4.2%, 1.0, 2.2ms, 0 | 20260506 17:47:45.695 | INFO | Uploaded 36GiB to GPU in 2.04893s (17.5702GiB/s) | 20260506 17:47:45.695 | INFO | Downloaded 38GiB to GPU in 2.21814s (17.1315GiB/s) | 20260506 17:47:50.829 | INFO | Analyzing chunk frames = [3265024,3265536) | 20260506 17:47:50.829 | INFO | MemInfo: { | 20260506 17:47:50.829 | INFO | "gpu" : " 17 GB", | 20260506 17:47:50.829 | INFO | "huge" : "", | 20260506 17:47:50.829 | INFO | "huge_shared" : " 27 GB", | 20260506 17:47:50.829 | INFO | "malloc" : " 14 GB", | 20260506 17:47:50.829 | INFO | "pinned" : " 29 GB" | 20260506 17:47:50.830 | INFO | } | 20260506 17:47:50.830 | INFO | Perf report for the last 5.13497 seconds: | 20260506 17:47:50.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:47:50.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:50.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:50.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:50.830 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:47:50.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:50.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:50.830 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 17:47:50.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:50.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:50.830 | INFO | Uploaded 36GiB to GPU in 1.99897s (18.0093GiB/s) | 20260506 17:47:50.830 | INFO | Downloaded 34GiB to GPU in 1.94139s (17.5132GiB/s) | 20260506 17:47:55.970 | INFO | Analyzing chunk frames = [3265536,3266048) | 20260506 17:47:55.970 | INFO | MemInfo: { | 20260506 17:47:55.970 | INFO | "gpu" : " 17 GB", | 20260506 17:47:55.970 | INFO | "huge" : "", | 20260506 17:47:55.970 | INFO | "huge_shared" : " 27 GB", | 20260506 17:47:55.970 | INFO | "malloc" : " 14 GB", | 20260506 17:47:55.970 | INFO | "pinned" : " 29 GB" | 20260506 17:47:55.970 | INFO | } | 20260506 17:47:55.970 | INFO | Perf report for the last 5.14048 seconds: | 20260506 17:47:55.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:47:55.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:55.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:55.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:55.970 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:47:55.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:55.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:55.970 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 17:47:55.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:55.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:47:55.970 | INFO | Uploaded 36GiB to GPU in 1.97945s (18.1869GiB/s) | 20260506 17:47:55.970 | INFO | Downloaded 34GiB to GPU in 1.92788s (17.636GiB/s) | 20260506 17:48:01.106 | INFO | Analyzing chunk frames = [3266048,3266560) | 20260506 17:48:01.106 | INFO | MemInfo: { | 20260506 17:48:01.106 | INFO | "gpu" : " 17 GB", | 20260506 17:48:01.106 | INFO | "huge" : "", | 20260506 17:48:01.106 | INFO | "huge_shared" : " 27 GB", | 20260506 17:48:01.106 | INFO | "malloc" : " 14 GB", | 20260506 17:48:01.106 | INFO | "pinned" : " 29 GB" | 20260506 17:48:01.107 | INFO | } | 20260506 17:48:01.107 | INFO | Perf report for the last 5.1366 seconds: | 20260506 17:48:01.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:48:01.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:01.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:01.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:01.107 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 17:48:01.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:01.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:01.107 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:48:01.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:01.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:01.107 | INFO | Uploaded 36GiB to GPU in 1.92664s (18.6854GiB/s) | 20260506 17:48:01.107 | INFO | Downloaded 34GiB to GPU in 1.86834s (18.198GiB/s) | 20260506 17:48:06.249 | INFO | Analyzing chunk frames = [3266560,3267072) | 20260506 17:48:06.249 | INFO | MemInfo: { | 20260506 17:48:06.249 | INFO | "gpu" : " 17 GB", | 20260506 17:48:06.249 | INFO | "huge" : "", | 20260506 17:48:06.249 | INFO | "huge_shared" : " 27 GB", | 20260506 17:48:06.249 | INFO | "malloc" : " 14 GB", | 20260506 17:48:06.249 | INFO | "pinned" : " 29 GB" | 20260506 17:48:06.249 | INFO | } | 20260506 17:48:06.250 | INFO | Perf report for the last 5.14286 seconds: | 20260506 17:48:06.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:48:06.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:06.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:06.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:06.250 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 17:48:06.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:06.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:06.250 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:48:06.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:06.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:06.250 | INFO | Uploaded 36GiB to GPU in 1.96811s (18.2916GiB/s) | 20260506 17:48:06.250 | INFO | Downloaded 34GiB to GPU in 1.89865s (17.9074GiB/s) | 20260506 17:48:11.396 | INFO | Analyzing chunk frames = [3267072,3267584) | 20260506 17:48:11.397 | INFO | MemInfo: { | 20260506 17:48:11.397 | INFO | "gpu" : " 17 GB", | 20260506 17:48:11.397 | INFO | "huge" : "", | 20260506 17:48:11.397 | INFO | "huge_shared" : " 27 GB", | 20260506 17:48:11.397 | INFO | "malloc" : " 14 GB", | 20260506 17:48:11.397 | INFO | "pinned" : " 29 GB" | 20260506 17:48:11.397 | INFO | } | 20260506 17:48:11.397 | INFO | Perf report for the last 5.14728 seconds: | 20260506 17:48:11.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:48:11.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:11.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:11.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:11.397 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 17:48:11.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:11.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:11.397 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 17:48:11.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:11.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:11.397 | INFO | Uploaded 36GiB to GPU in 2.01049s (17.9061GiB/s) | 20260506 17:48:11.397 | INFO | Downloaded 34GiB to GPU in 1.94733s (17.4598GiB/s) | 20260506 17:48:16.531 | INFO | Analyzing chunk frames = [3267584,3268096) | 20260506 17:48:16.531 | INFO | MemInfo: { | 20260506 17:48:16.531 | INFO | "gpu" : " 17 GB", | 20260506 17:48:16.531 | INFO | "huge" : "", | 20260506 17:48:16.531 | INFO | "huge_shared" : " 27 GB", | 20260506 17:48:16.531 | INFO | "malloc" : " 14 GB", | 20260506 17:48:16.531 | INFO | "pinned" : " 29 GB" | 20260506 17:48:16.532 | INFO | } | 20260506 17:48:16.532 | INFO | Perf report for the last 5.13474 seconds: | 20260506 17:48:16.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:48:16.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:16.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:16.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:16.532 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 17:48:16.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:16.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:16.532 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:48:16.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:16.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:16.532 | INFO | Uploaded 36GiB to GPU in 1.9991s (18.0081GiB/s) | 20260506 17:48:16.532 | INFO | Downloaded 34GiB to GPU in 1.99352s (17.0552GiB/s) | 20260506 17:48:21.676 | INFO | Analyzing chunk frames = [3268096,3268608) | 20260506 17:48:21.676 | INFO | MemInfo: { | 20260506 17:48:21.676 | INFO | "gpu" : " 17 GB", | 20260506 17:48:21.676 | INFO | "huge" : "", | 20260506 17:48:21.676 | INFO | "huge_shared" : " 27 GB", | 20260506 17:48:21.676 | INFO | "malloc" : " 14 GB", | 20260506 17:48:21.676 | INFO | "pinned" : " 29 GB" | 20260506 17:48:21.676 | INFO | } | 20260506 17:48:21.677 | INFO | Perf report for the last 5.14493 seconds: | 20260506 17:48:21.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:48:21.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:21.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:21.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:21.677 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 17:48:21.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:21.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:21.677 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260506 17:48:21.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:21.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:21.677 | INFO | Uploaded 36GiB to GPU in 2.03708s (17.6724GiB/s) | 20260506 17:48:21.677 | INFO | Downloaded 34GiB to GPU in 1.98914s (17.0928GiB/s) | 20260506 17:48:21.677 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:48:21.677 | INFO | Total mem usage: 18 GiB | 20260506 17:48:21.677 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:48:21.677 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -18%) | 20260506 17:48:21.677 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:48:21.677 | INFO | High Water Breakdown: | 20260506 17:48:21.677 | INFO | 61% : 9.31 GiB ( 81.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:48:21.677 | INFO | 38.8% : 5.93 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:48:21.677 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:48:21.677 | INFO | ------------------------------------------ | 20260506 17:48:21.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:48:21.677 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:48:21.677 | INFO | Total mem usage: 26.5 GiB | 20260506 17:48:21.677 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:48:21.677 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:48:21.677 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:48:21.677 | INFO | High Water Breakdown: | 20260506 17:48:21.677 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:48:21.677 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:48:21.677 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:48:21.677 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:48:21.677 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:48:21.677 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:48:21.677 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:48:21.677 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:48:21.677 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:48:21.677 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:48:21.677 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:48:21.677 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:48:21.677 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:48:21.677 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:48:21.677 | INFO | ------------------------------------------ | 20260506 17:48:21.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:48:21.677 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:48:21.677 | INFO | Total mem usage: 25.6 GiB | 20260506 17:48:21.677 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:48:21.677 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:48:21.677 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:48:21.677 | INFO | High Water Breakdown: | 20260506 17:48:21.677 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:48:21.677 | INFO | ------------------------------------------ | 20260506 17:48:21.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:48:21.677 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:48:21.678 | INFO | Total mem usage: 15.7 GiB | 20260506 17:48:21.678 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:48:21.678 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:48:21.678 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 17:48:21.678 | INFO | High Water Breakdown: | 20260506 17:48:21.678 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:48:21.678 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:48:21.678 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:48:21.678 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:48:21.678 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:48:21.678 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:48:21.678 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:48:21.678 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:48:21.678 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:48:21.678 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:48:21.678 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:48:21.678 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:48:21.678 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:48:21.678 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:48:21.678 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:48:21.678 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:48:21.678 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:48:21.678 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:48:21.678 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:48:21.678 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:48:21.678 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:48:21.678 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:48:21.678 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:48:21.678 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:48:21.678 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:48:21.678 | INFO | ------------------------------------------ | 20260506 17:48:21.678 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:48:21.678 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:48:21.678 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:48:21.678 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:48:21.678 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.59e+01, 1.34e+01 / 2.51e+00 | 20260506 17:48:21.678 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 1.03e-02 / 5.56e-03 | 20260506 17:48:21.678 | INFO | PerfReport: 11.60%, Baseline -- 3.63e+00 / 4.42e+00, 5.58e-01 / 3.40e-01 | 20260506 17:48:21.678 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.83e-01 / 1.35e-01 | 20260506 17:48:21.678 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.46e+01, 4.16e+00 / 2.55e+00 | 20260506 17:48:21.678 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.43e+00 / 1.14e+00 | 20260506 17:48:21.678 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.27e+00 / 2.89e+00, 3.87e-01 / 1.59e-01 | 20260506 17:48:21.678 | INFO | PerfReport: 3.43%, Metrics -- 1.07e+00 / 1.43e+00, 2.80e-01 / 1.53e-01 | 20260506 17:48:21.678 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.72e-01 / 4.22e-01, 5.84e-02 / 1.73e-02 | 20260506 17:48:21.678 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:48:21.678 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:48:21.678 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:48:21.678 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 17:48:21.678 | INFO | PerfReport: 21.47%, Upload -- 2.09e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260506 17:48:21.678 | INFO | PerfReport: 21.18%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 17:48:21.678 | INFO | PerfReport: 32.23%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:48:21.678 | INFO | PerfReport: 1.34%, LockUpload -- 1.30e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260506 17:48:21.678 | INFO | PerfReport: 1.16%, LockDownload -- 1.13e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 17:48:21.678 | INFO | PerfReport: 22.60%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 17:48:21.678 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260506 17:48:21.796 | INFO | Starting a round of Baz IO | 20260506 17:48:24.949 | INFO | Populating RTMetrics report | 20260506 17:48:24.949 | INFO | RTMetrics report populated | 20260506 17:48:24.949 | INFO | Opening tmp RTMetrics json file | 20260506 17:48:24.961 | INFO | Serializing RTMetrics to json | 20260506 17:48:24.970 | INFO | Serialization done, swapping tmp file now... | 20260506 17:48:24.989 | INFO | RTMetrics done | 20260506 17:48:25.501 | INFO | Finished a round of Baz IO: Wrote 2.055 GiB in 3.704 seconds | 20260506 17:48:25.501 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.43% / 0.01% / 64.57% | 20260506 17:48:25.501 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.27488% / 3.97364e-06% / 6.35783e-05% | 20260506 17:48:25.501 | INFO | Baz IO overall: | 20260506 17:48:25.501 | INFO | BytesWritten : 1724651360593 | 20260506 17:48:25.501 | INFO | NumEvents : 834253693135 | 20260506 17:48:26.807 | INFO | Analyzing chunk frames = [3268608,3269120) | 20260506 17:48:26.807 | INFO | MemInfo: { | 20260506 17:48:26.807 | INFO | "gpu" : " 17 GB", | 20260506 17:48:26.807 | INFO | "huge" : "", | 20260506 17:48:26.807 | INFO | "huge_shared" : " 27 GB", | 20260506 17:48:26.807 | INFO | "malloc" : " 14 GB", | 20260506 17:48:26.807 | INFO | "pinned" : " 29 GB" | 20260506 17:48:26.807 | INFO | } | 20260506 17:48:26.808 | INFO | Perf report for the last 5.13099 seconds: | 20260506 17:48:26.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:48:26.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:26.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:26.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:26.808 | INFO | ANALYSIS : 63.2%, 2.8, 94.1ms, 0 | 20260506 17:48:26.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:26.808 | INFO | RT_METRICS : 47.6%, 1.0, 24.9ms, 0 | 20260506 17:48:26.808 | INFO | PRE_HQ : 61.1%, 2.0, 62.3ms, 0 | 20260506 17:48:26.808 | INFO | BAZWRITER : 71.8%, 4.8, 177.7ms, 0 | 20260506 17:48:26.808 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 17:48:26.808 | INFO | Uploaded 36GiB to GPU in 2.07089s (17.3838GiB/s) | 20260506 17:48:26.808 | INFO | Downloaded 38GiB to GPU in 2.22263s (17.0968GiB/s) | 20260506 17:48:31.946 | INFO | Analyzing chunk frames = [3269120,3269632) | 20260506 17:48:31.946 | INFO | MemInfo: { | 20260506 17:48:31.946 | INFO | "gpu" : " 17 GB", | 20260506 17:48:31.946 | INFO | "huge" : "", | 20260506 17:48:31.946 | INFO | "huge_shared" : " 27 GB", | 20260506 17:48:31.946 | INFO | "malloc" : " 14 GB", | 20260506 17:48:31.946 | INFO | "pinned" : " 29 GB" | 20260506 17:48:31.946 | INFO | } | 20260506 17:48:31.946 | INFO | Perf report for the last 5.1384 seconds: | 20260506 17:48:31.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:48:31.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:31.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:31.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:31.946 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:48:31.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:31.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:31.946 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:48:31.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:31.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:31.946 | INFO | Uploaded 36GiB to GPU in 1.99758s (18.0218GiB/s) | 20260506 17:48:31.946 | INFO | Downloaded 34GiB to GPU in 1.94367s (17.4926GiB/s) | 20260506 17:48:37.083 | INFO | Analyzing chunk frames = [3269632,3270144) | 20260506 17:48:37.083 | INFO | MemInfo: { | 20260506 17:48:37.083 | INFO | "gpu" : " 17 GB", | 20260506 17:48:37.083 | INFO | "huge" : "", | 20260506 17:48:37.083 | INFO | "huge_shared" : " 27 GB", | 20260506 17:48:37.083 | INFO | "malloc" : " 14 GB", | 20260506 17:48:37.083 | INFO | "pinned" : " 29 GB" | 20260506 17:48:37.083 | INFO | } | 20260506 17:48:37.083 | INFO | Perf report for the last 5.13709 seconds: | 20260506 17:48:37.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:48:37.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:37.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:37.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:37.083 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:48:37.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:37.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:37.083 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:48:37.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:37.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:37.083 | INFO | Uploaded 36GiB to GPU in 1.98035s (18.1786GiB/s) | 20260506 17:48:37.083 | INFO | Downloaded 34GiB to GPU in 1.92084s (17.7006GiB/s) | 20260506 17:48:42.225 | INFO | Analyzing chunk frames = [3270144,3270656) | 20260506 17:48:42.225 | INFO | MemInfo: { | 20260506 17:48:42.225 | INFO | "gpu" : " 17 GB", | 20260506 17:48:42.225 | INFO | "huge" : "", | 20260506 17:48:42.225 | INFO | "huge_shared" : " 27 GB", | 20260506 17:48:42.225 | INFO | "malloc" : " 14 GB", | 20260506 17:48:42.225 | INFO | "pinned" : " 29 GB" | 20260506 17:48:42.225 | INFO | } | 20260506 17:48:42.225 | INFO | Perf report for the last 5.1417 seconds: | 20260506 17:48:42.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:48:42.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:42.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:42.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:42.225 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 17:48:42.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:42.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:42.225 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:48:42.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:42.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:42.225 | INFO | Uploaded 36GiB to GPU in 1.92941s (18.6585GiB/s) | 20260506 17:48:42.225 | INFO | Downloaded 34GiB to GPU in 1.87312s (18.1516GiB/s) | 20260506 17:48:47.363 | INFO | Analyzing chunk frames = [3270656,3271168) | 20260506 17:48:47.363 | INFO | MemInfo: { | 20260506 17:48:47.363 | INFO | "gpu" : " 17 GB", | 20260506 17:48:47.363 | INFO | "huge" : "", | 20260506 17:48:47.363 | INFO | "huge_shared" : " 27 GB", | 20260506 17:48:47.363 | INFO | "malloc" : " 14 GB", | 20260506 17:48:47.363 | INFO | "pinned" : " 29 GB" | 20260506 17:48:47.363 | INFO | } | 20260506 17:48:47.363 | INFO | Perf report for the last 5.13839 seconds: | 20260506 17:48:47.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:48:47.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:47.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:47.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:47.363 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 17:48:47.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:47.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:47.363 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:48:47.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:47.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:47.363 | INFO | Uploaded 36GiB to GPU in 2.02034s (17.8188GiB/s) | 20260506 17:48:47.363 | INFO | Downloaded 34GiB to GPU in 1.91038s (17.7975GiB/s) | 20260506 17:48:52.502 | INFO | Analyzing chunk frames = [3271168,3271680) | 20260506 17:48:52.502 | INFO | MemInfo: { | 20260506 17:48:52.502 | INFO | "gpu" : " 17 GB", | 20260506 17:48:52.502 | INFO | "huge" : "", | 20260506 17:48:52.502 | INFO | "huge_shared" : " 27 GB", | 20260506 17:48:52.502 | INFO | "malloc" : " 14 GB", | 20260506 17:48:52.502 | INFO | "pinned" : " 29 GB" | 20260506 17:48:52.502 | INFO | } | 20260506 17:48:52.502 | INFO | Perf report for the last 5.13873 seconds: | 20260506 17:48:52.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:48:52.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:52.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:52.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:52.502 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 17:48:52.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:52.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:52.502 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:48:52.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:52.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:52.502 | INFO | Uploaded 36GiB to GPU in 2.0098s (17.9123GiB/s) | 20260506 17:48:52.502 | INFO | Downloaded 34GiB to GPU in 1.93801s (17.5438GiB/s) | 20260506 17:48:57.644 | INFO | Analyzing chunk frames = [3271680,3272192) | 20260506 17:48:57.644 | INFO | MemInfo: { | 20260506 17:48:57.644 | INFO | "gpu" : " 17 GB", | 20260506 17:48:57.644 | INFO | "huge" : "", | 20260506 17:48:57.644 | INFO | "huge_shared" : " 27 GB", | 20260506 17:48:57.644 | INFO | "malloc" : " 14 GB", | 20260506 17:48:57.644 | INFO | "pinned" : " 29 GB" | 20260506 17:48:57.644 | INFO | } | 20260506 17:48:57.644 | INFO | Perf report for the last 5.142 seconds: | 20260506 17:48:57.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:48:57.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:57.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:57.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:57.644 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 17:48:57.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:57.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:57.644 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:48:57.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:57.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:48:57.644 | INFO | Uploaded 36GiB to GPU in 1.98477s (18.1382GiB/s) | 20260506 17:48:57.644 | INFO | Downloaded 34GiB to GPU in 1.99708s (17.0248GiB/s) | 20260506 17:49:02.781 | INFO | Analyzing chunk frames = [3272192,3272704) | 20260506 17:49:02.781 | INFO | MemInfo: { | 20260506 17:49:02.781 | INFO | "gpu" : " 17 GB", | 20260506 17:49:02.781 | INFO | "huge" : "", | 20260506 17:49:02.782 | INFO | "huge_shared" : " 27 GB", | 20260506 17:49:02.782 | INFO | "malloc" : " 14 GB", | 20260506 17:49:02.782 | INFO | "pinned" : " 29 GB" | 20260506 17:49:02.782 | INFO | } | 20260506 17:49:02.782 | INFO | Perf report for the last 5.1378 seconds: | 20260506 17:49:02.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:49:02.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:02.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:02.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:02.782 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 17:49:02.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:02.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:02.782 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:49:02.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:02.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:02.782 | INFO | Uploaded 36GiB to GPU in 1.9842s (18.1433GiB/s) | 20260506 17:49:02.782 | INFO | Downloaded 34GiB to GPU in 1.98032s (17.1689GiB/s) | 20260506 17:49:02.895 | INFO | Starting a round of Baz IO | 20260506 17:49:06.042 | INFO | Populating RTMetrics report | 20260506 17:49:06.042 | INFO | RTMetrics report populated | 20260506 17:49:06.042 | INFO | Opening tmp RTMetrics json file | 20260506 17:49:06.088 | INFO | Serializing RTMetrics to json | 20260506 17:49:06.153 | INFO | Serialization done, swapping tmp file now... | 20260506 17:49:06.175 | INFO | RTMetrics done | 20260506 17:49:06.218 | INFO | Finished a round of Baz IO: Wrote 2.056 GiB in 3.322 seconds | 20260506 17:49:06.218 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.42% / 0.01% / 64.57% | 20260506 17:49:06.218 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.27482% / 1.19209e-05% / 8.34465e-05% | 20260506 17:49:06.218 | INFO | Baz IO overall: | 20260506 17:49:06.218 | INFO | BytesWritten : 1726859011082 | 20260506 17:49:06.218 | INFO | NumEvents : 835318163150 | 20260506 17:49:07.923 | INFO | Analyzing chunk frames = [3272704,3273216) | 20260506 17:49:07.923 | INFO | MemInfo: { | 20260506 17:49:07.923 | INFO | "gpu" : " 17 GB", | 20260506 17:49:07.923 | INFO | "huge" : "", | 20260506 17:49:07.923 | INFO | "huge_shared" : " 27 GB", | 20260506 17:49:07.923 | INFO | "malloc" : " 14 GB", | 20260506 17:49:07.923 | INFO | "pinned" : " 29 GB" | 20260506 17:49:07.923 | INFO | } | 20260506 17:49:07.923 | INFO | Perf report for the last 5.14104 seconds: | 20260506 17:49:07.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:49:07.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:07.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:07.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:07.923 | INFO | ANALYSIS : 62.8%, 2.9, 95.5ms, 0 | 20260506 17:49:07.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:07.923 | INFO | RT_METRICS : 46.0%, 1.0, 24.2ms, 0 | 20260506 17:49:07.923 | INFO | PRE_HQ : 60.3%, 1.9, 58.3ms, 0 | 20260506 17:49:07.923 | INFO | BAZWRITER : 64.3%, 4.8, 157.1ms, 0 | 20260506 17:49:07.923 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 17:49:07.923 | INFO | Uploaded 36GiB to GPU in 2.06991s (17.3921GiB/s) | 20260506 17:49:07.923 | INFO | Downloaded 38GiB to GPU in 2.2237s (17.0886GiB/s) | 20260506 17:49:13.060 | INFO | Analyzing chunk frames = [3273216,3273728) | 20260506 17:49:13.060 | INFO | MemInfo: { | 20260506 17:49:13.060 | INFO | "gpu" : " 17 GB", | 20260506 17:49:13.060 | INFO | "huge" : "", | 20260506 17:49:13.060 | INFO | "huge_shared" : " 27 GB", | 20260506 17:49:13.060 | INFO | "malloc" : " 14 GB", | 20260506 17:49:13.060 | INFO | "pinned" : " 29 GB" | 20260506 17:49:13.060 | INFO | } | 20260506 17:49:13.060 | INFO | Perf report for the last 5.13704 seconds: | 20260506 17:49:13.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:49:13.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:13.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:13.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:13.060 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:49:13.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:13.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:13.060 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:49:13.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:13.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:13.060 | INFO | Uploaded 36GiB to GPU in 2.00614s (17.945GiB/s) | 20260506 17:49:13.060 | INFO | Downloaded 34GiB to GPU in 1.9449s (17.4816GiB/s) | 20260506 17:49:18.198 | INFO | Analyzing chunk frames = [3273728,3274240) | 20260506 17:49:18.198 | INFO | MemInfo: { | 20260506 17:49:18.198 | INFO | "gpu" : " 17 GB", | 20260506 17:49:18.198 | INFO | "huge" : "", | 20260506 17:49:18.198 | INFO | "huge_shared" : " 27 GB", | 20260506 17:49:18.198 | INFO | "malloc" : " 14 GB", | 20260506 17:49:18.198 | INFO | "pinned" : " 29 GB" | 20260506 17:49:18.199 | INFO | } | 20260506 17:49:18.199 | INFO | Perf report for the last 5.13874 seconds: | 20260506 17:49:18.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:49:18.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:18.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:18.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:18.199 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:49:18.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:18.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:18.199 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 17:49:18.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:18.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:18.199 | INFO | Uploaded 36GiB to GPU in 1.97275s (18.2486GiB/s) | 20260506 17:49:18.199 | INFO | Downloaded 34GiB to GPU in 1.92818s (17.6332GiB/s) | 20260506 17:49:23.342 | INFO | Analyzing chunk frames = [3274240,3274752) | 20260506 17:49:23.342 | INFO | MemInfo: { | 20260506 17:49:23.342 | INFO | "gpu" : " 17 GB", | 20260506 17:49:23.342 | INFO | "huge" : "", | 20260506 17:49:23.342 | INFO | "huge_shared" : " 27 GB", | 20260506 17:49:23.342 | INFO | "malloc" : " 14 GB", | 20260506 17:49:23.342 | INFO | "pinned" : " 29 GB" | 20260506 17:49:23.342 | INFO | } | 20260506 17:49:23.343 | INFO | Perf report for the last 5.14396 seconds: | 20260506 17:49:23.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:49:23.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:23.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:23.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:23.343 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 17:49:23.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:23.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:23.343 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:49:23.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:23.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:23.343 | INFO | Uploaded 36GiB to GPU in 1.91775s (18.772GiB/s) | 20260506 17:49:23.343 | INFO | Downloaded 34GiB to GPU in 1.87293s (18.1534GiB/s) | 20260506 17:49:28.480 | INFO | Analyzing chunk frames = [3274752,3275264) | 20260506 17:49:28.480 | INFO | MemInfo: { | 20260506 17:49:28.480 | INFO | "gpu" : " 17 GB", | 20260506 17:49:28.480 | INFO | "huge" : "", | 20260506 17:49:28.480 | INFO | "huge_shared" : " 27 GB", | 20260506 17:49:28.480 | INFO | "malloc" : " 14 GB", | 20260506 17:49:28.480 | INFO | "pinned" : " 29 GB" | 20260506 17:49:28.480 | INFO | } | 20260506 17:49:28.480 | INFO | Perf report for the last 5.13761 seconds: | 20260506 17:49:28.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:49:28.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:28.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:28.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:28.480 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 17:49:28.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:28.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:28.480 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:49:28.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:28.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:28.480 | INFO | Uploaded 36GiB to GPU in 2.01616s (17.8557GiB/s) | 20260506 17:49:28.481 | INFO | Downloaded 34GiB to GPU in 1.91364s (17.7671GiB/s) | 20260506 17:49:33.621 | INFO | Analyzing chunk frames = [3275264,3275776) | 20260506 17:49:33.621 | INFO | MemInfo: { | 20260506 17:49:33.621 | INFO | "gpu" : " 17 GB", | 20260506 17:49:33.621 | INFO | "huge" : "", | 20260506 17:49:33.621 | INFO | "huge_shared" : " 27 GB", | 20260506 17:49:33.621 | INFO | "malloc" : " 14 GB", | 20260506 17:49:33.621 | INFO | "pinned" : " 29 GB" | 20260506 17:49:33.621 | INFO | } | 20260506 17:49:33.621 | INFO | Perf report for the last 5.14126 seconds: | 20260506 17:49:33.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:49:33.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:33.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:33.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:33.621 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:49:33.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:33.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:33.622 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 17:49:33.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:33.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:33.622 | INFO | Uploaded 36GiB to GPU in 2.057s (17.5012GiB/s) | 20260506 17:49:33.622 | INFO | Downloaded 34GiB to GPU in 1.97643s (17.2028GiB/s) | 20260506 17:49:38.763 | INFO | Analyzing chunk frames = [3275776,3276288) | 20260506 17:49:38.763 | INFO | MemInfo: { | 20260506 17:49:38.763 | INFO | "gpu" : " 17 GB", | 20260506 17:49:38.763 | INFO | "huge" : "", | 20260506 17:49:38.763 | INFO | "huge_shared" : " 27 GB", | 20260506 17:49:38.763 | INFO | "malloc" : " 14 GB", | 20260506 17:49:38.763 | INFO | "pinned" : " 29 GB" | 20260506 17:49:38.763 | INFO | } | 20260506 17:49:38.763 | INFO | Perf report for the last 5.14197 seconds: | 20260506 17:49:38.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:49:38.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:38.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:38.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:38.764 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:49:38.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:38.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:38.764 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 17:49:38.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:38.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:38.764 | INFO | Uploaded 36GiB to GPU in 1.99827s (18.0156GiB/s) | 20260506 17:49:38.764 | INFO | Downloaded 34GiB to GPU in 1.97882s (17.182GiB/s) | 20260506 17:49:43.897 | INFO | Analyzing chunk frames = [3276288,3276800) | 20260506 17:49:43.897 | INFO | MemInfo: { | 20260506 17:49:43.897 | INFO | "gpu" : " 17 GB", | 20260506 17:49:43.897 | INFO | "huge" : "", | 20260506 17:49:43.897 | INFO | "huge_shared" : " 27 GB", | 20260506 17:49:43.897 | INFO | "malloc" : " 14 GB", | 20260506 17:49:43.897 | INFO | "pinned" : " 29 GB" | 20260506 17:49:43.897 | INFO | } | 20260506 17:49:43.897 | INFO | Perf report for the last 5.13382 seconds: | 20260506 17:49:43.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:49:43.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:43.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:43.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:43.897 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 17:49:43.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:43.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:43.897 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:49:43.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:43.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:43.897 | INFO | Uploaded 36GiB to GPU in 1.9978s (18.0198GiB/s) | 20260506 17:49:43.898 | INFO | Downloaded 34GiB to GPU in 1.99387s (17.0522GiB/s) | 20260506 17:49:43.898 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:49:43.898 | INFO | Total mem usage: 18 GiB | 20260506 17:49:43.898 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:49:43.898 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -10.1%) | 20260506 17:49:43.898 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:49:43.898 | INFO | High Water Breakdown: | 20260506 17:49:43.898 | INFO | 62.1% : 10.2 GiB ( 89% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:49:43.898 | INFO | 37.8% : 6.18 GiB ( 92.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:49:43.898 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:49:43.898 | INFO | ------------------------------------------ | 20260506 17:49:43.898 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:49:43.898 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:49:43.898 | INFO | Total mem usage: 26.5 GiB | 20260506 17:49:43.898 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:49:43.898 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 17:49:43.898 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 17:49:43.898 | INFO | High Water Breakdown: | 20260506 17:49:43.898 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:49:43.898 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:49:43.898 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:49:43.898 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:49:43.898 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:49:43.898 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:49:43.898 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:49:43.898 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:49:43.898 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:49:43.898 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:49:43.898 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:49:43.898 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:49:43.898 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:49:43.898 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:49:43.899 | INFO | ------------------------------------------ | 20260506 17:49:43.899 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:49:43.899 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:49:43.899 | INFO | Total mem usage: 25.6 GiB | 20260506 17:49:43.899 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:49:43.899 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:49:43.899 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:49:43.899 | INFO | High Water Breakdown: | 20260506 17:49:43.899 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:49:43.899 | INFO | ------------------------------------------ | 20260506 17:49:43.899 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:49:43.899 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:49:43.899 | INFO | Total mem usage: 15.7 GiB | 20260506 17:49:43.899 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:49:43.899 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:49:43.899 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:49:43.899 | INFO | High Water Breakdown: | 20260506 17:49:43.899 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:49:43.899 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:49:43.899 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:49:43.899 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:49:43.899 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:49:43.899 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:49:43.899 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:49:43.899 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:49:43.899 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:49:43.899 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:49:43.899 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:49:43.899 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:49:43.899 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:49:43.899 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:49:43.899 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:49:43.899 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:49:43.899 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:49:43.899 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:49:43.899 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:49:43.899 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:49:43.899 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:49:43.899 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:49:43.899 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:49:43.899 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:49:43.899 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:49:43.899 | INFO | ------------------------------------------ | 20260506 17:49:43.899 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:49:43.899 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:49:43.899 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:49:43.899 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:49:43.899 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.55e+01, 1.34e+01 / 2.20e+00 | 20260506 17:49:43.899 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 9.99e-03 / 5.74e-03 | 20260506 17:49:43.899 | INFO | PerfReport: 11.62%, Baseline -- 3.64e+00 / 4.42e+00, 5.56e-01 / 3.36e-01 | 20260506 17:49:43.899 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.82e-01 / 1.32e-01 | 20260506 17:49:43.899 | INFO | PerfReport: 15.95%, DME -- 4.00e+01 / 4.43e+01, 3.88e+00 / 2.43e+00 | 20260506 17:49:43.899 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.41e+00 / 2.07e+00 | 20260506 17:49:43.899 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.27e+00 / 2.81e+00, 3.81e-01 / 2.65e-01 | 20260506 17:49:43.899 | INFO | PerfReport: 3.44%, Metrics -- 1.08e+00 / 1.42e+00, 2.81e-01 / 2.12e-01 | 20260506 17:49:43.899 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.72e-01 / 4.24e-01, 5.87e-02 / 3.03e-02 | 20260506 17:49:43.899 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:49:43.899 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:49:43.899 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:49:43.899 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 17:49:43.899 | INFO | PerfReport: 21.46%, Upload -- 2.08e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260506 17:49:43.899 | INFO | PerfReport: 21.20%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 17:49:43.899 | INFO | PerfReport: 32.26%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:49:43.899 | INFO | PerfReport: 1.40%, LockUpload -- 1.36e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260506 17:49:43.899 | INFO | PerfReport: 1.13%, LockDownload -- 1.09e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260506 17:49:43.899 | INFO | PerfReport: 22.54%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 17:49:43.899 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260506 17:49:44.015 | INFO | Starting a round of Baz IO | 20260506 17:49:47.154 | INFO | Populating RTMetrics report | 20260506 17:49:47.155 | INFO | RTMetrics report populated | 20260506 17:49:47.155 | INFO | Opening tmp RTMetrics json file | 20260506 17:49:47.221 | INFO | Serializing RTMetrics to json | 20260506 17:49:47.317 | INFO | Serialization done, swapping tmp file now... | 20260506 17:49:47.373 | INFO | RTMetrics done | 20260506 17:49:47.491 | INFO | Finished a round of Baz IO: Wrote 2.054 GiB in 3.475 seconds | 20260506 17:49:47.491 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.41% / 0.01% / 64.58% | 20260506 17:49:47.491 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.26863% / 1.58946e-05% / 7.15256e-05% | 20260506 17:49:47.491 | INFO | Baz IO overall: | 20260506 17:49:47.491 | INFO | BytesWritten : 1729064527022 | 20260506 17:49:47.491 | INFO | NumEvents : 836380888016 | 20260506 17:49:49.038 | INFO | Analyzing chunk frames = [3276800,3277312) | 20260506 17:49:49.038 | INFO | MemInfo: { | 20260506 17:49:49.038 | INFO | "gpu" : " 17 GB", | 20260506 17:49:49.038 | INFO | "huge" : "", | 20260506 17:49:49.038 | INFO | "huge_shared" : " 27 GB", | 20260506 17:49:49.038 | INFO | "malloc" : " 14 GB", | 20260506 17:49:49.038 | INFO | "pinned" : " 29 GB" | 20260506 17:49:49.038 | INFO | } | 20260506 17:49:49.038 | INFO | Perf report for the last 5.14086 seconds: | 20260506 17:49:49.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:49:49.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:49.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:49.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:49.038 | INFO | ANALYSIS : 62.8%, 2.9, 95.5ms, 0 | 20260506 17:49:49.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:49.038 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260506 17:49:49.038 | INFO | PRE_HQ : 60.2%, 1.9, 58.5ms, 0 | 20260506 17:49:49.038 | INFO | BAZWRITER : 67.2%, 4.8, 165.4ms, 0 | 20260506 17:49:49.038 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 17:49:49.038 | INFO | Uploaded 36GiB to GPU in 2.04485s (17.6052GiB/s) | 20260506 17:49:49.039 | INFO | Downloaded 38GiB to GPU in 2.21928s (17.1227GiB/s) | 20260506 17:49:54.175 | INFO | Analyzing chunk frames = [3277312,3277824) | 20260506 17:49:54.175 | INFO | MemInfo: { | 20260506 17:49:54.175 | INFO | "gpu" : " 17 GB", | 20260506 17:49:54.175 | INFO | "huge" : "", | 20260506 17:49:54.175 | INFO | "huge_shared" : " 27 GB", | 20260506 17:49:54.175 | INFO | "malloc" : " 14 GB", | 20260506 17:49:54.175 | INFO | "pinned" : " 29 GB" | 20260506 17:49:54.175 | INFO | } | 20260506 17:49:54.175 | INFO | Perf report for the last 5.13729 seconds: | 20260506 17:49:54.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:49:54.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:54.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:54.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:54.175 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:49:54.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:54.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:54.176 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:49:54.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:54.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:54.176 | INFO | Uploaded 36GiB to GPU in 1.99293s (18.0639GiB/s) | 20260506 17:49:54.176 | INFO | Downloaded 34GiB to GPU in 1.92162s (17.6934GiB/s) | 20260506 17:49:59.313 | INFO | Analyzing chunk frames = [3277824,3278336) | 20260506 17:49:59.313 | INFO | MemInfo: { | 20260506 17:49:59.313 | INFO | "gpu" : " 17 GB", | 20260506 17:49:59.313 | INFO | "huge" : "", | 20260506 17:49:59.313 | INFO | "huge_shared" : " 27 GB", | 20260506 17:49:59.313 | INFO | "malloc" : " 14 GB", | 20260506 17:49:59.313 | INFO | "pinned" : " 29 GB" | 20260506 17:49:59.313 | INFO | } | 20260506 17:49:59.313 | INFO | Perf report for the last 5.13751 seconds: | 20260506 17:49:59.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:49:59.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:59.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:59.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:59.313 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:49:59.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:59.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:59.313 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:49:59.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:59.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:49:59.313 | INFO | Uploaded 36GiB to GPU in 1.97829s (18.1975GiB/s) | 20260506 17:49:59.313 | INFO | Downloaded 34GiB to GPU in 1.92357s (17.6754GiB/s) | 20260506 17:50:04.454 | INFO | Analyzing chunk frames = [3278336,3278848) | 20260506 17:50:04.454 | INFO | MemInfo: { | 20260506 17:50:04.454 | INFO | "gpu" : " 17 GB", | 20260506 17:50:04.454 | INFO | "huge" : "", | 20260506 17:50:04.454 | INFO | "huge_shared" : " 27 GB", | 20260506 17:50:04.454 | INFO | "malloc" : " 14 GB", | 20260506 17:50:04.454 | INFO | "pinned" : " 29 GB" | 20260506 17:50:04.454 | INFO | } | 20260506 17:50:04.454 | INFO | Perf report for the last 5.14107 seconds: | 20260506 17:50:04.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:50:04.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:04.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:04.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:04.454 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 17:50:04.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:04.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:04.454 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:50:04.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:04.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:04.454 | INFO | Uploaded 36GiB to GPU in 1.93597s (18.5953GiB/s) | 20260506 17:50:04.454 | INFO | Downloaded 34GiB to GPU in 1.87415s (18.1416GiB/s) | 20260506 17:50:09.596 | INFO | Analyzing chunk frames = [3278848,3279360) | 20260506 17:50:09.596 | INFO | MemInfo: { | 20260506 17:50:09.596 | INFO | "gpu" : " 17 GB", | 20260506 17:50:09.596 | INFO | "huge" : "", | 20260506 17:50:09.596 | INFO | "huge_shared" : " 27 GB", | 20260506 17:50:09.596 | INFO | "malloc" : " 14 GB", | 20260506 17:50:09.596 | INFO | "pinned" : " 29 GB" | 20260506 17:50:09.596 | INFO | } | 20260506 17:50:09.596 | INFO | Perf report for the last 5.14201 seconds: | 20260506 17:50:09.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:50:09.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:09.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:09.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:09.596 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 17:50:09.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:09.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:09.596 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:50:09.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:09.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:09.596 | INFO | Uploaded 36GiB to GPU in 2.01933s (17.8277GiB/s) | 20260506 17:50:09.596 | INFO | Downloaded 34GiB to GPU in 1.91872s (17.7201GiB/s) | 20260506 17:50:14.737 | INFO | Analyzing chunk frames = [3279360,3279872) | 20260506 17:50:14.737 | INFO | MemInfo: { | 20260506 17:50:14.737 | INFO | "gpu" : " 17 GB", | 20260506 17:50:14.738 | INFO | "huge" : "", | 20260506 17:50:14.738 | INFO | "huge_shared" : " 27 GB", | 20260506 17:50:14.738 | INFO | "malloc" : " 14 GB", | 20260506 17:50:14.738 | INFO | "pinned" : " 29 GB" | 20260506 17:50:14.738 | INFO | } | 20260506 17:50:14.738 | INFO | Perf report for the last 5.14164 seconds: | 20260506 17:50:14.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:50:14.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:14.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:14.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:14.738 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 17:50:14.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:14.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:14.738 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 17:50:14.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:14.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:14.738 | INFO | Uploaded 36GiB to GPU in 2.02858s (17.7464GiB/s) | 20260506 17:50:14.738 | INFO | Downloaded 34GiB to GPU in 1.94388s (17.4908GiB/s) | 20260506 17:50:19.876 | INFO | Analyzing chunk frames = [3279872,3280384) | 20260506 17:50:19.876 | INFO | MemInfo: { | 20260506 17:50:19.876 | INFO | "gpu" : " 17 GB", | 20260506 17:50:19.876 | INFO | "huge" : "", | 20260506 17:50:19.876 | INFO | "huge_shared" : " 27 GB", | 20260506 17:50:19.876 | INFO | "malloc" : " 14 GB", | 20260506 17:50:19.876 | INFO | "pinned" : " 29 GB" | 20260506 17:50:19.876 | INFO | } | 20260506 17:50:19.876 | INFO | Perf report for the last 5.1381 seconds: | 20260506 17:50:19.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:50:19.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:19.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:19.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:19.876 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 17:50:19.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:19.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:19.876 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:50:19.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:19.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:19.876 | INFO | Uploaded 36GiB to GPU in 1.98827s (18.1062GiB/s) | 20260506 17:50:19.876 | INFO | Downloaded 34GiB to GPU in 1.96533s (17.2999GiB/s) | 20260506 17:50:25.014 | INFO | Analyzing chunk frames = [3280384,3280896) | 20260506 17:50:25.014 | INFO | MemInfo: { | 20260506 17:50:25.014 | INFO | "gpu" : " 17 GB", | 20260506 17:50:25.014 | INFO | "huge" : "", | 20260506 17:50:25.014 | INFO | "huge_shared" : " 27 GB", | 20260506 17:50:25.014 | INFO | "malloc" : " 14 GB", | 20260506 17:50:25.014 | INFO | "pinned" : " 29 GB" | 20260506 17:50:25.014 | INFO | } | 20260506 17:50:25.014 | INFO | Perf report for the last 5.13799 seconds: | 20260506 17:50:25.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:50:25.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:25.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:25.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:25.014 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:50:25.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:25.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:25.014 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:50:25.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:25.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:25.014 | INFO | Uploaded 36GiB to GPU in 2.0149s (17.8669GiB/s) | 20260506 17:50:25.014 | INFO | Downloaded 34GiB to GPU in 1.98986s (17.0866GiB/s) | 20260506 17:50:25.133 | INFO | Starting a round of Baz IO | 20260506 17:50:28.266 | INFO | Populating RTMetrics report | 20260506 17:50:28.267 | INFO | RTMetrics report populated | 20260506 17:50:28.267 | INFO | Opening tmp RTMetrics json file | 20260506 17:50:28.289 | INFO | Serializing RTMetrics to json | 20260506 17:50:28.294 | INFO | Serialization done, swapping tmp file now... | 20260506 17:50:28.296 | INFO | RTMetrics done | 20260506 17:50:28.941 | INFO | Finished a round of Baz IO: Wrote 2.057 GiB in 3.808 seconds | 20260506 17:50:28.941 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.41% / 0.01% / 64.59% | 20260506 17:50:28.941 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.26946% / 1.58946e-05% / 7.54992e-05% | 20260506 17:50:28.941 | INFO | Baz IO overall: | 20260506 17:50:28.941 | INFO | BytesWritten : 1731272847429 | 20260506 17:50:28.941 | INFO | NumEvents : 837445819979 | 20260506 17:50:30.150 | INFO | Analyzing chunk frames = [3280896,3281408) | 20260506 17:50:30.150 | INFO | MemInfo: { | 20260506 17:50:30.150 | INFO | "gpu" : " 17 GB", | 20260506 17:50:30.150 | INFO | "huge" : "", | 20260506 17:50:30.150 | INFO | "huge_shared" : " 27 GB", | 20260506 17:50:30.150 | INFO | "malloc" : " 14 GB", | 20260506 17:50:30.150 | INFO | "pinned" : " 29 GB" | 20260506 17:50:30.150 | INFO | } | 20260506 17:50:30.150 | INFO | Perf report for the last 5.13652 seconds: | 20260506 17:50:30.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:50:30.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:30.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:30.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:30.150 | INFO | ANALYSIS : 62.7%, 2.9, 95.4ms, 0 | 20260506 17:50:30.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:30.150 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 17:50:30.150 | INFO | PRE_HQ : 60.1%, 1.9, 57.8ms, 0 | 20260506 17:50:30.150 | INFO | BAZWRITER : 73.8%, 4.8, 182.6ms, 0 | 20260506 17:50:30.150 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 17:50:30.151 | INFO | Uploaded 36GiB to GPU in 2.0178s (17.8412GiB/s) | 20260506 17:50:30.151 | INFO | Downloaded 38GiB to GPU in 2.21528s (17.1536GiB/s) | 20260506 17:50:35.286 | INFO | Analyzing chunk frames = [3281408,3281920) | 20260506 17:50:35.286 | INFO | MemInfo: { | 20260506 17:50:35.286 | INFO | "gpu" : " 17 GB", | 20260506 17:50:35.286 | INFO | "huge" : "", | 20260506 17:50:35.286 | INFO | "huge_shared" : " 27 GB", | 20260506 17:50:35.286 | INFO | "malloc" : " 14 GB", | 20260506 17:50:35.286 | INFO | "pinned" : " 29 GB" | 20260506 17:50:35.286 | INFO | } | 20260506 17:50:35.286 | INFO | Perf report for the last 5.13581 seconds: | 20260506 17:50:35.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:50:35.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:35.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:35.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:35.286 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 17:50:35.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:35.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:35.286 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:50:35.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:35.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:35.286 | INFO | Uploaded 36GiB to GPU in 1.9826s (18.158GiB/s) | 20260506 17:50:35.286 | INFO | Downloaded 34GiB to GPU in 1.92939s (17.6221GiB/s) | 20260506 17:50:40.429 | INFO | Analyzing chunk frames = [3281920,3282432) | 20260506 17:50:40.429 | INFO | MemInfo: { | 20260506 17:50:40.429 | INFO | "gpu" : " 17 GB", | 20260506 17:50:40.429 | INFO | "huge" : "", | 20260506 17:50:40.429 | INFO | "huge_shared" : " 27 GB", | 20260506 17:50:40.429 | INFO | "malloc" : " 14 GB", | 20260506 17:50:40.429 | INFO | "pinned" : " 29 GB" | 20260506 17:50:40.429 | INFO | } | 20260506 17:50:40.430 | INFO | Perf report for the last 5.14333 seconds: | 20260506 17:50:40.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:50:40.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:40.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:40.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:40.430 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 17:50:40.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:40.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:40.430 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:50:40.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:40.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:40.430 | INFO | Uploaded 36GiB to GPU in 1.97278s (18.2484GiB/s) | 20260506 17:50:40.430 | INFO | Downloaded 34GiB to GPU in 1.92461s (17.6659GiB/s) | 20260506 17:50:45.569 | INFO | Analyzing chunk frames = [3282432,3282944) | 20260506 17:50:45.569 | INFO | MemInfo: { | 20260506 17:50:45.569 | INFO | "gpu" : " 17 GB", | 20260506 17:50:45.569 | INFO | "huge" : "", | 20260506 17:50:45.569 | INFO | "huge_shared" : " 27 GB", | 20260506 17:50:45.569 | INFO | "malloc" : " 14 GB", | 20260506 17:50:45.569 | INFO | "pinned" : " 29 GB" | 20260506 17:50:45.569 | INFO | } | 20260506 17:50:45.569 | INFO | Perf report for the last 5.13952 seconds: | 20260506 17:50:45.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:50:45.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:45.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:45.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:45.569 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 17:50:45.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:45.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:45.569 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:50:45.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:45.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:45.569 | INFO | Uploaded 36GiB to GPU in 1.9268s (18.6838GiB/s) | 20260506 17:50:45.569 | INFO | Downloaded 34GiB to GPU in 1.86983s (18.1834GiB/s) | 20260506 17:50:50.710 | INFO | Analyzing chunk frames = [3282944,3283456) | 20260506 17:50:50.710 | INFO | MemInfo: { | 20260506 17:50:50.710 | INFO | "gpu" : " 17 GB", | 20260506 17:50:50.710 | INFO | "huge" : "", | 20260506 17:50:50.710 | INFO | "huge_shared" : " 27 GB", | 20260506 17:50:50.710 | INFO | "malloc" : " 14 GB", | 20260506 17:50:50.710 | INFO | "pinned" : " 29 GB" | 20260506 17:50:50.710 | INFO | } | 20260506 17:50:50.710 | INFO | Perf report for the last 5.14134 seconds: | 20260506 17:50:50.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:50:50.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:50.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:50.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:50.710 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 17:50:50.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:50.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:50.710 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:50:50.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:50.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:50.711 | INFO | Uploaded 36GiB to GPU in 2.01655s (17.8522GiB/s) | 20260506 17:50:50.711 | INFO | Downloaded 34GiB to GPU in 1.91631s (17.7424GiB/s) | 20260506 17:50:55.853 | INFO | Analyzing chunk frames = [3283456,3283968) | 20260506 17:50:55.853 | INFO | MemInfo: { | 20260506 17:50:55.853 | INFO | "gpu" : " 17 GB", | 20260506 17:50:55.853 | INFO | "huge" : "", | 20260506 17:50:55.853 | INFO | "huge_shared" : " 27 GB", | 20260506 17:50:55.853 | INFO | "malloc" : " 14 GB", | 20260506 17:50:55.853 | INFO | "pinned" : " 29 GB" | 20260506 17:50:55.853 | INFO | } | 20260506 17:50:55.853 | INFO | Perf report for the last 5.14248 seconds: | 20260506 17:50:55.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:50:55.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:55.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:55.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:55.853 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 17:50:55.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:55.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:55.853 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:50:55.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:55.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:50:55.853 | INFO | Uploaded 36GiB to GPU in 1.99812s (18.0169GiB/s) | 20260506 17:50:55.853 | INFO | Downloaded 34GiB to GPU in 1.94478s (17.4827GiB/s) | 20260506 17:51:00.989 | INFO | Analyzing chunk frames = [3283968,3284480) | 20260506 17:51:00.989 | INFO | MemInfo: { | 20260506 17:51:00.989 | INFO | "gpu" : " 17 GB", | 20260506 17:51:00.989 | INFO | "huge" : "", | 20260506 17:51:00.989 | INFO | "huge_shared" : " 27 GB", | 20260506 17:51:00.989 | INFO | "malloc" : " 14 GB", | 20260506 17:51:00.989 | INFO | "pinned" : " 29 GB" | 20260506 17:51:00.989 | INFO | } | 20260506 17:51:00.989 | INFO | Perf report for the last 5.13614 seconds: | 20260506 17:51:00.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:51:00.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:00.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:00.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:00.989 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 17:51:00.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:00.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:00.989 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 17:51:00.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:00.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:00.989 | INFO | Uploaded 36GiB to GPU in 2.00474s (17.9575GiB/s) | 20260506 17:51:00.989 | INFO | Downloaded 34GiB to GPU in 2.01613s (16.864GiB/s) | 20260506 17:51:06.127 | INFO | Analyzing chunk frames = [3284480,3284992) | 20260506 17:51:06.127 | INFO | MemInfo: { | 20260506 17:51:06.127 | INFO | "gpu" : " 17 GB", | 20260506 17:51:06.127 | INFO | "huge" : "", | 20260506 17:51:06.127 | INFO | "huge_shared" : " 27 GB", | 20260506 17:51:06.127 | INFO | "malloc" : " 14 GB", | 20260506 17:51:06.127 | INFO | "pinned" : " 29 GB" | 20260506 17:51:06.127 | INFO | } | 20260506 17:51:06.127 | INFO | Perf report for the last 5.13786 seconds: | 20260506 17:51:06.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:51:06.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:06.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:06.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:06.127 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 17:51:06.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:06.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:06.127 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:51:06.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:06.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:06.127 | INFO | Uploaded 36GiB to GPU in 2.04759s (17.5816GiB/s) | 20260506 17:51:06.127 | INFO | Downloaded 34GiB to GPU in 1.98544s (17.1247GiB/s) | 20260506 17:51:06.127 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:51:06.127 | INFO | Total mem usage: 18 GiB | 20260506 17:51:06.127 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:51:06.127 | INFO | Recent High Water Marks Sum: 17 GiB (Savings: -5.75%) | 20260506 17:51:06.127 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:51:06.127 | INFO | High Water Breakdown: | 20260506 17:51:06.128 | INFO | 62.7% : 10.7 GiB ( 93.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:51:06.128 | INFO | 37.2% : 6.33 GiB ( 94.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:51:06.128 | INFO | 0.117% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:51:06.128 | INFO | ------------------------------------------ | 20260506 17:51:06.128 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:51:06.128 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:51:06.128 | INFO | Total mem usage: 26.5 GiB | 20260506 17:51:06.128 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:51:06.128 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:51:06.128 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:51:06.128 | INFO | High Water Breakdown: | 20260506 17:51:06.128 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:51:06.128 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:51:06.128 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:51:06.128 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:51:06.128 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:51:06.128 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:51:06.128 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:51:06.128 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:51:06.128 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:51:06.128 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:51:06.128 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:51:06.128 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:51:06.128 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:51:06.128 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:51:06.128 | INFO | ------------------------------------------ | 20260506 17:51:06.128 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:51:06.128 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:51:06.128 | INFO | Total mem usage: 25.6 GiB | 20260506 17:51:06.128 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:51:06.128 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:51:06.128 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:51:06.128 | INFO | High Water Breakdown: | 20260506 17:51:06.128 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:51:06.128 | INFO | ------------------------------------------ | 20260506 17:51:06.128 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:51:06.128 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:51:06.128 | INFO | Total mem usage: 15.7 GiB | 20260506 17:51:06.128 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:51:06.128 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:51:06.128 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 17:51:06.128 | INFO | High Water Breakdown: | 20260506 17:51:06.128 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:51:06.128 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:51:06.128 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:51:06.128 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:51:06.128 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:51:06.128 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:51:06.128 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:51:06.128 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:51:06.128 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:51:06.128 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:51:06.128 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:51:06.128 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:51:06.128 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:51:06.128 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:51:06.128 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:51:06.128 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:51:06.128 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:51:06.128 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:51:06.128 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:51:06.128 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:51:06.128 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:51:06.128 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:51:06.128 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:51:06.128 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:51:06.128 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:51:06.128 | INFO | ------------------------------------------ | 20260506 17:51:06.128 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:51:06.128 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:51:06.128 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:51:06.128 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:51:06.128 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.59e+01, 1.36e+01 / 2.69e+00 | 20260506 17:51:06.128 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.02e-01, 1.00e-02 / 5.49e-03 | 20260506 17:51:06.128 | INFO | PerfReport: 11.61%, Baseline -- 3.64e+00 / 4.41e+00, 5.64e-01 / 3.29e-01 | 20260506 17:51:06.128 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.47e+00, 1.87e-01 / 1.40e-01 | 20260506 17:51:06.128 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.47e+01, 4.14e+00 / 2.78e+00 | 20260506 17:51:06.128 | INFO | PerfReport: 57.02%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.46e+00 / 1.76e+00 | 20260506 17:51:06.128 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.27e+00 / 2.83e+00, 3.79e-01 / 2.46e-01 | 20260506 17:51:06.128 | INFO | PerfReport: 3.43%, Metrics -- 1.07e+00 / 1.41e+00, 2.82e-01 / 2.01e-01 | 20260506 17:51:06.128 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.72e-01 / 4.18e-01, 5.86e-02 / 2.79e-02 | 20260506 17:51:06.128 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:51:06.128 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:51:06.128 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:51:06.128 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 17:51:06.128 | INFO | PerfReport: 21.41%, Upload -- 2.08e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260506 17:51:06.129 | INFO | PerfReport: 21.14%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 17:51:06.129 | INFO | PerfReport: 32.26%, Compute -- 3.14e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 17:51:06.129 | INFO | PerfReport: 1.36%, LockUpload -- 1.32e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260506 17:51:06.129 | INFO | PerfReport: 1.14%, LockDownload -- 1.11e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 17:51:06.129 | INFO | PerfReport: 22.68%, LockCompute -- 2.20e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 17:51:06.129 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260506 17:51:06.246 | INFO | Starting a round of Baz IO | 20260506 17:51:09.378 | INFO | Populating RTMetrics report | 20260506 17:51:09.378 | INFO | RTMetrics report populated | 20260506 17:51:09.378 | INFO | Opening tmp RTMetrics json file | 20260506 17:51:09.629 | INFO | Serializing RTMetrics to json | 20260506 17:51:09.654 | INFO | Serialization done, swapping tmp file now... | 20260506 17:51:09.675 | INFO | RTMetrics done | 20260506 17:51:09.974 | INFO | Finished a round of Baz IO: Wrote 2.055 GiB in 3.727 seconds | 20260506 17:51:09.974 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.40% / 0.01% / 64.59% | 20260506 17:51:09.974 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.26608% / 1.58946e-05% / 5.96046e-05% | 20260506 17:51:09.974 | INFO | Baz IO overall: | 20260506 17:51:09.974 | INFO | BytesWritten : 1733479252596 | 20260506 17:51:09.974 | INFO | NumEvents : 838509753006 | 20260506 17:51:11.266 | INFO | Analyzing chunk frames = [3284992,3285504) | 20260506 17:51:11.266 | INFO | MemInfo: { | 20260506 17:51:11.266 | INFO | "gpu" : " 17 GB", | 20260506 17:51:11.266 | INFO | "huge" : "", | 20260506 17:51:11.266 | INFO | "huge_shared" : " 27 GB", | 20260506 17:51:11.266 | INFO | "malloc" : " 14 GB", | 20260506 17:51:11.266 | INFO | "pinned" : " 29 GB" | 20260506 17:51:11.266 | INFO | } | 20260506 17:51:11.266 | INFO | Perf report for the last 5.13892 seconds: | 20260506 17:51:11.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:51:11.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:11.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:11.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:11.266 | INFO | ANALYSIS : 62.7%, 2.9, 95.3ms, 0 | 20260506 17:51:11.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:11.266 | INFO | RT_METRICS : 49.2%, 1.0, 25.8ms, 0 | 20260506 17:51:11.266 | INFO | PRE_HQ : 60.1%, 1.9, 58.0ms, 0 | 20260506 17:51:11.266 | INFO | BAZWRITER : 72.1%, 4.8, 177.1ms, 0 | 20260506 17:51:11.266 | INFO | BAZ_SAMPLING : 4.9%, 1.0, 2.5ms, 0 | 20260506 17:51:11.266 | INFO | Uploaded 36GiB to GPU in 2.05406s (17.5263GiB/s) | 20260506 17:51:11.266 | INFO | Downloaded 38GiB to GPU in 2.22253s (17.0977GiB/s) | 20260506 17:51:16.404 | INFO | Analyzing chunk frames = [3285504,3286016) | 20260506 17:51:16.404 | INFO | MemInfo: { | 20260506 17:51:16.404 | INFO | "gpu" : " 17 GB", | 20260506 17:51:16.405 | INFO | "huge" : "", | 20260506 17:51:16.405 | INFO | "huge_shared" : " 27 GB", | 20260506 17:51:16.405 | INFO | "malloc" : " 14 GB", | 20260506 17:51:16.405 | INFO | "pinned" : " 29 GB" | 20260506 17:51:16.405 | INFO | } | 20260506 17:51:16.405 | INFO | Perf report for the last 5.13894 seconds: | 20260506 17:51:16.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:51:16.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:16.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:16.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:16.405 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:51:16.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:16.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:16.405 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 17:51:16.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:16.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:16.405 | INFO | Uploaded 36GiB to GPU in 2.00348s (17.9687GiB/s) | 20260506 17:51:16.405 | INFO | Downloaded 34GiB to GPU in 1.93621s (17.5601GiB/s) | 20260506 17:51:21.544 | INFO | Analyzing chunk frames = [3286016,3286528) | 20260506 17:51:21.544 | INFO | MemInfo: { | 20260506 17:51:21.544 | INFO | "gpu" : " 17 GB", | 20260506 17:51:21.544 | INFO | "huge" : "", | 20260506 17:51:21.544 | INFO | "huge_shared" : " 27 GB", | 20260506 17:51:21.544 | INFO | "malloc" : " 14 GB", | 20260506 17:51:21.544 | INFO | "pinned" : " 29 GB" | 20260506 17:51:21.544 | INFO | } | 20260506 17:51:21.544 | INFO | Perf report for the last 5.13915 seconds: | 20260506 17:51:21.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:51:21.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:21.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:21.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:21.544 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:51:21.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:21.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:21.544 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:51:21.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:21.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:21.544 | INFO | Uploaded 36GiB to GPU in 1.97689s (18.2104GiB/s) | 20260506 17:51:21.544 | INFO | Downloaded 34GiB to GPU in 1.92341s (17.677GiB/s) | 20260506 17:51:26.688 | INFO | Analyzing chunk frames = [3286528,3287040) | 20260506 17:51:26.688 | INFO | MemInfo: { | 20260506 17:51:26.688 | INFO | "gpu" : " 17 GB", | 20260506 17:51:26.688 | INFO | "huge" : "", | 20260506 17:51:26.688 | INFO | "huge_shared" : " 27 GB", | 20260506 17:51:26.688 | INFO | "malloc" : " 14 GB", | 20260506 17:51:26.688 | INFO | "pinned" : " 29 GB" | 20260506 17:51:26.688 | INFO | } | 20260506 17:51:26.688 | INFO | Perf report for the last 5.14418 seconds: | 20260506 17:51:26.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:51:26.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:26.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:26.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:26.688 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 17:51:26.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:26.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:26.688 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:51:26.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:26.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:26.688 | INFO | Uploaded 36GiB to GPU in 1.93846s (18.5714GiB/s) | 20260506 17:51:26.689 | INFO | Downloaded 34GiB to GPU in 1.86923s (18.1893GiB/s) | 20260506 17:51:31.827 | INFO | Analyzing chunk frames = [3287040,3287552) | 20260506 17:51:31.827 | INFO | MemInfo: { | 20260506 17:51:31.827 | INFO | "gpu" : " 17 GB", | 20260506 17:51:31.827 | INFO | "huge" : "", | 20260506 17:51:31.828 | INFO | "huge_shared" : " 27 GB", | 20260506 17:51:31.828 | INFO | "malloc" : " 14 GB", | 20260506 17:51:31.828 | INFO | "pinned" : " 29 GB" | 20260506 17:51:31.828 | INFO | } | 20260506 17:51:31.828 | INFO | Perf report for the last 5.13961 seconds: | 20260506 17:51:31.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:51:31.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:31.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:31.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:31.828 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 17:51:31.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:31.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:31.828 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:51:31.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:31.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:31.828 | INFO | Uploaded 36GiB to GPU in 2.02106s (17.8124GiB/s) | 20260506 17:51:31.828 | INFO | Downloaded 34GiB to GPU in 1.91187s (17.7836GiB/s) | 20260506 17:51:36.966 | INFO | Analyzing chunk frames = [3287552,3288064) | 20260506 17:51:36.966 | INFO | MemInfo: { | 20260506 17:51:36.966 | INFO | "gpu" : " 17 GB", | 20260506 17:51:36.966 | INFO | "huge" : "", | 20260506 17:51:36.966 | INFO | "huge_shared" : " 27 GB", | 20260506 17:51:36.966 | INFO | "malloc" : " 14 GB", | 20260506 17:51:36.966 | INFO | "pinned" : " 29 GB" | 20260506 17:51:36.966 | INFO | } | 20260506 17:51:36.967 | INFO | Perf report for the last 5.13873 seconds: | 20260506 17:51:36.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:51:36.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:36.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:36.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:36.967 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 17:51:36.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:36.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:36.967 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:51:36.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:36.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:36.967 | INFO | Uploaded 36GiB to GPU in 2.06844s (17.4044GiB/s) | 20260506 17:51:36.967 | INFO | Downloaded 34GiB to GPU in 1.96209s (17.3284GiB/s) | 20260506 17:51:42.106 | INFO | Analyzing chunk frames = [3288064,3288576) | 20260506 17:51:42.106 | INFO | MemInfo: { | 20260506 17:51:42.106 | INFO | "gpu" : " 17 GB", | 20260506 17:51:42.106 | INFO | "huge" : "", | 20260506 17:51:42.106 | INFO | "huge_shared" : " 27 GB", | 20260506 17:51:42.106 | INFO | "malloc" : " 14 GB", | 20260506 17:51:42.106 | INFO | "pinned" : " 29 GB" | 20260506 17:51:42.106 | INFO | } | 20260506 17:51:42.106 | INFO | Perf report for the last 5.13981 seconds: | 20260506 17:51:42.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:51:42.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:42.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:42.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:42.106 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 17:51:42.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:42.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:42.106 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 17:51:42.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:42.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:42.107 | INFO | Uploaded 36GiB to GPU in 1.9926s (18.0668GiB/s) | 20260506 17:51:42.107 | INFO | Downloaded 34GiB to GPU in 1.97312s (17.2316GiB/s) | 20260506 17:51:47.237 | INFO | Analyzing chunk frames = [3288576,3289088) | 20260506 17:51:47.237 | INFO | MemInfo: { | 20260506 17:51:47.237 | INFO | "gpu" : " 17 GB", | 20260506 17:51:47.237 | INFO | "huge" : "", | 20260506 17:51:47.237 | INFO | "huge_shared" : " 27 GB", | 20260506 17:51:47.237 | INFO | "malloc" : " 14 GB", | 20260506 17:51:47.237 | INFO | "pinned" : " 29 GB" | 20260506 17:51:47.237 | INFO | } | 20260506 17:51:47.237 | INFO | Perf report for the last 5.13103 seconds: | 20260506 17:51:47.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:51:47.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:47.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:47.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:47.237 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 17:51:47.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:47.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:47.237 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 17:51:47.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:47.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:47.238 | INFO | Uploaded 36GiB to GPU in 2.05489s (17.5192GiB/s) | 20260506 17:51:47.238 | INFO | Downloaded 34GiB to GPU in 1.98382s (17.1386GiB/s) | 20260506 17:51:47.357 | INFO | Starting a round of Baz IO | 20260506 17:51:50.503 | INFO | Populating RTMetrics report | 20260506 17:51:50.503 | INFO | RTMetrics report populated | 20260506 17:51:50.503 | INFO | Opening tmp RTMetrics json file | 20260506 17:51:50.505 | INFO | Serializing RTMetrics to json | 20260506 17:51:50.517 | INFO | Serialization done, swapping tmp file now... | 20260506 17:51:50.536 | INFO | RTMetrics done | 20260506 17:51:50.632 | INFO | Finished a round of Baz IO: Wrote 2.053 GiB in 3.275 seconds | 20260506 17:51:50.632 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.39% / 0.01% / 64.60% | 20260506 17:51:50.632 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.26292% / 0% / 5.96046e-05% | 20260506 17:51:50.632 | INFO | Baz IO overall: | 20260506 17:51:50.632 | INFO | BytesWritten : 1735684052958 | 20260506 17:51:50.632 | INFO | NumEvents : 839572944799 | 20260506 17:51:52.376 | INFO | Analyzing chunk frames = [3289088,3289600) | 20260506 17:51:52.376 | INFO | MemInfo: { | 20260506 17:51:52.376 | INFO | "gpu" : " 17 GB", | 20260506 17:51:52.376 | INFO | "huge" : "", | 20260506 17:51:52.377 | INFO | "huge_shared" : " 27 GB", | 20260506 17:51:52.377 | INFO | "malloc" : " 14 GB", | 20260506 17:51:52.377 | INFO | "pinned" : " 29 GB" | 20260506 17:51:52.377 | INFO | } | 20260506 17:51:52.377 | INFO | Perf report for the last 5.1393 seconds: | 20260506 17:51:52.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:51:52.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:52.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:52.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:52.377 | INFO | ANALYSIS : 63.0%, 2.9, 95.7ms, 0 | 20260506 17:51:52.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:52.377 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 17:51:52.377 | INFO | PRE_HQ : 60.5%, 1.9, 58.7ms, 0 | 20260506 17:51:52.377 | INFO | BAZWRITER : 63.2%, 4.8, 154.2ms, 0 | 20260506 17:51:52.377 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 17:51:52.377 | INFO | Uploaded 36GiB to GPU in 2.07469s (17.352GiB/s) | 20260506 17:51:52.377 | INFO | Downloaded 38GiB to GPU in 2.24042s (16.9611GiB/s) | 20260506 17:51:57.521 | INFO | Analyzing chunk frames = [3289600,3290112) | 20260506 17:51:57.521 | INFO | MemInfo: { | 20260506 17:51:57.521 | INFO | "gpu" : " 17 GB", | 20260506 17:51:57.521 | INFO | "huge" : "", | 20260506 17:51:57.522 | INFO | "huge_shared" : " 27 GB", | 20260506 17:51:57.522 | INFO | "malloc" : " 14 GB", | 20260506 17:51:57.522 | INFO | "pinned" : " 29 GB" | 20260506 17:51:57.522 | INFO | } | 20260506 17:51:57.522 | INFO | Perf report for the last 5.14507 seconds: | 20260506 17:51:57.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:51:57.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:57.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:57.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:57.522 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:51:57.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:57.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:57.522 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:51:57.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:57.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:51:57.522 | INFO | Uploaded 36GiB to GPU in 1.99865s (18.0122GiB/s) | 20260506 17:51:57.522 | INFO | Downloaded 34GiB to GPU in 1.93943s (17.5309GiB/s) | 20260506 17:52:02.660 | INFO | Analyzing chunk frames = [3290112,3290624) | 20260506 17:52:02.660 | INFO | MemInfo: { | 20260506 17:52:02.660 | INFO | "gpu" : " 17 GB", | 20260506 17:52:02.660 | INFO | "huge" : "", | 20260506 17:52:02.660 | INFO | "huge_shared" : " 27 GB", | 20260506 17:52:02.660 | INFO | "malloc" : " 14 GB", | 20260506 17:52:02.660 | INFO | "pinned" : " 29 GB" | 20260506 17:52:02.660 | INFO | } | 20260506 17:52:02.660 | INFO | Perf report for the last 5.13844 seconds: | 20260506 17:52:02.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:52:02.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:02.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:02.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:02.660 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:52:02.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:02.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:02.660 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:52:02.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:02.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:02.661 | INFO | Uploaded 36GiB to GPU in 1.98124s (18.1704GiB/s) | 20260506 17:52:02.661 | INFO | Downloaded 34GiB to GPU in 1.91959s (17.7121GiB/s) | 20260506 17:52:07.803 | INFO | Analyzing chunk frames = [3290624,3291136) | 20260506 17:52:07.803 | INFO | MemInfo: { | 20260506 17:52:07.803 | INFO | "gpu" : " 17 GB", | 20260506 17:52:07.803 | INFO | "huge" : "", | 20260506 17:52:07.803 | INFO | "huge_shared" : " 27 GB", | 20260506 17:52:07.803 | INFO | "malloc" : " 14 GB", | 20260506 17:52:07.803 | INFO | "pinned" : " 29 GB" | 20260506 17:52:07.803 | INFO | } | 20260506 17:52:07.803 | INFO | Perf report for the last 5.14271 seconds: | 20260506 17:52:07.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:52:07.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:07.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:07.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:07.803 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:52:07.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:07.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:07.803 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:52:07.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:07.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:07.803 | INFO | Uploaded 36GiB to GPU in 1.93876s (18.5686GiB/s) | 20260506 17:52:07.803 | INFO | Downloaded 34GiB to GPU in 1.87268s (18.1558GiB/s) | 20260506 17:52:12.943 | INFO | Analyzing chunk frames = [3291136,3291648) | 20260506 17:52:12.943 | INFO | MemInfo: { | 20260506 17:52:12.943 | INFO | "gpu" : " 17 GB", | 20260506 17:52:12.943 | INFO | "huge" : "", | 20260506 17:52:12.943 | INFO | "huge_shared" : " 27 GB", | 20260506 17:52:12.943 | INFO | "malloc" : " 14 GB", | 20260506 17:52:12.943 | INFO | "pinned" : " 29 GB" | 20260506 17:52:12.943 | INFO | } | 20260506 17:52:12.943 | INFO | Perf report for the last 5.14048 seconds: | 20260506 17:52:12.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:52:12.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:12.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:12.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:12.943 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 17:52:12.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:12.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:12.943 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:52:12.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:12.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:12.944 | INFO | Uploaded 36GiB to GPU in 2.0152s (17.8642GiB/s) | 20260506 17:52:12.944 | INFO | Downloaded 34GiB to GPU in 1.90578s (17.8404GiB/s) | 20260506 17:52:18.078 | INFO | Analyzing chunk frames = [3291648,3292160) | 20260506 17:52:18.078 | INFO | MemInfo: { | 20260506 17:52:18.078 | INFO | "gpu" : " 17 GB", | 20260506 17:52:18.078 | INFO | "huge" : "", | 20260506 17:52:18.078 | INFO | "huge_shared" : " 27 GB", | 20260506 17:52:18.078 | INFO | "malloc" : " 14 GB", | 20260506 17:52:18.078 | INFO | "pinned" : " 29 GB" | 20260506 17:52:18.078 | INFO | } | 20260506 17:52:18.078 | INFO | Perf report for the last 5.13471 seconds: | 20260506 17:52:18.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:52:18.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:18.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:18.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:18.078 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:52:18.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:18.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:18.078 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:52:18.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:18.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:18.078 | INFO | Uploaded 36GiB to GPU in 2.06797s (17.4083GiB/s) | 20260506 17:52:18.078 | INFO | Downloaded 34GiB to GPU in 1.95841s (17.361GiB/s) | 20260506 17:52:23.214 | INFO | Analyzing chunk frames = [3292160,3292672) | 20260506 17:52:23.214 | INFO | MemInfo: { | 20260506 17:52:23.214 | INFO | "gpu" : " 17 GB", | 20260506 17:52:23.214 | INFO | "huge" : "", | 20260506 17:52:23.214 | INFO | "huge_shared" : " 27 GB", | 20260506 17:52:23.214 | INFO | "malloc" : " 14 GB", | 20260506 17:52:23.214 | INFO | "pinned" : " 29 GB" | 20260506 17:52:23.214 | INFO | } | 20260506 17:52:23.214 | INFO | Perf report for the last 5.13602 seconds: | 20260506 17:52:23.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:52:23.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:23.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:23.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:23.214 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 17:52:23.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:23.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:23.214 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:52:23.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:23.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:23.215 | INFO | Uploaded 36GiB to GPU in 2.0325s (17.7122GiB/s) | 20260506 17:52:23.215 | INFO | Downloaded 34GiB to GPU in 1.99984s (17.0013GiB/s) | 20260506 17:52:28.353 | INFO | Analyzing chunk frames = [3292672,3293184) | 20260506 17:52:28.353 | INFO | MemInfo: { | 20260506 17:52:28.353 | INFO | "gpu" : " 17 GB", | 20260506 17:52:28.353 | INFO | "huge" : "", | 20260506 17:52:28.353 | INFO | "huge_shared" : " 27 GB", | 20260506 17:52:28.353 | INFO | "malloc" : " 14 GB", | 20260506 17:52:28.353 | INFO | "pinned" : " 29 GB" | 20260506 17:52:28.353 | INFO | } | 20260506 17:52:28.353 | INFO | Perf report for the last 5.13858 seconds: | 20260506 17:52:28.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:52:28.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:28.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:28.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:28.353 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:52:28.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:28.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:28.353 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 17:52:28.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:28.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:28.353 | INFO | Uploaded 36GiB to GPU in 2.0182s (17.8377GiB/s) | 20260506 17:52:28.353 | INFO | Downloaded 34GiB to GPU in 1.97516s (17.2138GiB/s) | 20260506 17:52:28.353 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:52:28.353 | INFO | Total mem usage: 18 GiB | 20260506 17:52:28.353 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:52:28.353 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -18.8%) | 20260506 17:52:28.353 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:52:28.353 | INFO | High Water Breakdown: | 20260506 17:52:28.353 | INFO | 60.9% : 9.24 GiB ( 80.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:52:28.353 | INFO | 38.9% : 5.9 GiB ( 88.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:52:28.353 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:52:28.353 | INFO | ------------------------------------------ | 20260506 17:52:28.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:52:28.353 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:52:28.353 | INFO | Total mem usage: 26.5 GiB | 20260506 17:52:28.353 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:52:28.353 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:52:28.353 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:52:28.353 | INFO | High Water Breakdown: | 20260506 17:52:28.353 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:52:28.353 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:52:28.353 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:52:28.353 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:52:28.353 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:52:28.353 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:52:28.353 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:52:28.353 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:52:28.353 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:52:28.353 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:52:28.353 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:52:28.353 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:52:28.353 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:52:28.353 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:52:28.353 | INFO | ------------------------------------------ | 20260506 17:52:28.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:52:28.353 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:52:28.353 | INFO | Total mem usage: 25.6 GiB | 20260506 17:52:28.353 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:52:28.353 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:52:28.354 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:52:28.354 | INFO | High Water Breakdown: | 20260506 17:52:28.354 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:52:28.354 | INFO | ------------------------------------------ | 20260506 17:52:28.354 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:52:28.354 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:52:28.354 | INFO | Total mem usage: 15.7 GiB | 20260506 17:52:28.354 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:52:28.354 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:52:28.354 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:52:28.354 | INFO | High Water Breakdown: | 20260506 17:52:28.354 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:52:28.354 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:52:28.354 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:52:28.354 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:52:28.354 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:52:28.354 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:52:28.354 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:52:28.354 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:52:28.354 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:52:28.354 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:52:28.354 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:52:28.354 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:52:28.354 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:52:28.354 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:52:28.354 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:52:28.354 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:52:28.354 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:52:28.354 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:52:28.354 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:52:28.354 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:52:28.354 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:52:28.354 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:52:28.354 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:52:28.354 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:52:28.354 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:52:28.354 | INFO | ------------------------------------------ | 20260506 17:52:28.354 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:52:28.354 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:52:28.354 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:52:28.354 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:52:28.354 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.55e+01, 1.31e+01 / 2.02e+00 | 20260506 17:52:28.354 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.02e-01, 1.03e-02 / 9.97e-03 | 20260506 17:52:28.354 | INFO | PerfReport: 11.61%, Baseline -- 3.64e+00 / 4.43e+00, 5.59e-01 / 3.31e-01 | 20260506 17:52:28.354 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.83e-01 / 1.41e-01 | 20260506 17:52:28.354 | INFO | PerfReport: 15.92%, DME -- 3.99e+01 / 4.44e+01, 3.87e+00 / 2.28e+00 | 20260506 17:52:28.354 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.47e+00 / 1.96e+00 | 20260506 17:52:28.354 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.27e+00 / 2.81e+00, 3.84e-01 / 2.32e-01 | 20260506 17:52:28.354 | INFO | PerfReport: 3.44%, Metrics -- 1.08e+00 / 1.42e+00, 2.84e-01 / 1.63e-01 | 20260506 17:52:28.354 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.72e-01 / 4.21e-01, 5.92e-02 / 1.87e-02 | 20260506 17:52:28.354 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:52:28.354 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:52:28.354 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:52:28.354 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 17:52:28.354 | INFO | PerfReport: 21.57%, Upload -- 2.10e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 17:52:28.354 | INFO | PerfReport: 21.14%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 17:52:28.354 | INFO | PerfReport: 32.21%, Compute -- 3.13e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 17:52:28.354 | INFO | PerfReport: 1.51%, LockUpload -- 1.47e+00 / 0.00e+00, 5.84e+00 / 0.00e+00 | 20260506 17:52:28.354 | INFO | PerfReport: 1.03%, LockDownload -- 1.00e+00 / 0.00e+00, 2.60e+00 / 0.00e+00 | 20260506 17:52:28.354 | INFO | PerfReport: 22.54%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 17:52:28.355 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260506 17:52:28.473 | INFO | Starting a round of Baz IO | 20260506 17:52:31.644 | INFO | Populating RTMetrics report | 20260506 17:52:31.644 | INFO | RTMetrics report populated | 20260506 17:52:31.644 | INFO | Opening tmp RTMetrics json file | 20260506 17:52:31.647 | INFO | Serializing RTMetrics to json | 20260506 17:52:31.652 | INFO | Serialization done, swapping tmp file now... | 20260506 17:52:31.656 | INFO | RTMetrics done | 20260506 17:52:32.539 | INFO | Finished a round of Baz IO: Wrote 2.054 GiB in 4.065 seconds | 20260506 17:52:32.539 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.39% / 0.01% / 64.61% | 20260506 17:52:32.539 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.26005% / 7.94729e-06% / 7.94729e-05% | 20260506 17:52:32.539 | INFO | Baz IO overall: | 20260506 17:52:32.539 | INFO | BytesWritten : 1737889706191 | 20260506 17:52:32.539 | INFO | NumEvents : 840636642143 | 20260506 17:52:33.494 | INFO | Analyzing chunk frames = [3293184,3293696) | 20260506 17:52:33.494 | INFO | MemInfo: { | 20260506 17:52:33.494 | INFO | "gpu" : " 17 GB", | 20260506 17:52:33.494 | INFO | "huge" : "", | 20260506 17:52:33.494 | INFO | "huge_shared" : " 27 GB", | 20260506 17:52:33.494 | INFO | "malloc" : " 14 GB", | 20260506 17:52:33.494 | INFO | "pinned" : " 29 GB" | 20260506 17:52:33.494 | INFO | } | 20260506 17:52:33.494 | INFO | Perf report for the last 5.1413 seconds: | 20260506 17:52:33.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:52:33.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:33.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:33.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:33.494 | INFO | ANALYSIS : 63.2%, 2.8, 94.3ms, 0 | 20260506 17:52:33.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:33.494 | INFO | RT_METRICS : 47.4%, 1.0, 24.9ms, 0 | 20260506 17:52:33.494 | INFO | PRE_HQ : 61.5%, 2.0, 63.0ms, 0 | 20260506 17:52:33.494 | INFO | BAZWRITER : 78.6%, 4.8, 194.9ms, 0 | 20260506 17:52:33.494 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 17:52:33.494 | INFO | Uploaded 36GiB to GPU in 2.07812s (17.3233GiB/s) | 20260506 17:52:33.494 | INFO | Downloaded 38GiB to GPU in 2.22679s (17.0649GiB/s) | 20260506 17:52:38.635 | INFO | Analyzing chunk frames = [3293696,3294208) | 20260506 17:52:38.635 | INFO | MemInfo: { | 20260506 17:52:38.635 | INFO | "gpu" : " 17 GB", | 20260506 17:52:38.635 | INFO | "huge" : "", | 20260506 17:52:38.635 | INFO | "huge_shared" : " 27 GB", | 20260506 17:52:38.635 | INFO | "malloc" : " 14 GB", | 20260506 17:52:38.635 | INFO | "pinned" : " 29 GB" | 20260506 17:52:38.635 | INFO | } | 20260506 17:52:38.635 | INFO | Perf report for the last 5.14075 seconds: | 20260506 17:52:38.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:52:38.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:38.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:38.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:38.635 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 17:52:38.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:38.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:38.635 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 17:52:38.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:38.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:38.635 | INFO | Uploaded 36GiB to GPU in 1.99948s (18.0046GiB/s) | 20260506 17:52:38.635 | INFO | Downloaded 34GiB to GPU in 1.94314s (17.4974GiB/s) | 20260506 17:52:43.779 | INFO | Analyzing chunk frames = [3294208,3294720) | 20260506 17:52:43.779 | INFO | MemInfo: { | 20260506 17:52:43.779 | INFO | "gpu" : " 17 GB", | 20260506 17:52:43.779 | INFO | "huge" : "", | 20260506 17:52:43.779 | INFO | "huge_shared" : " 27 GB", | 20260506 17:52:43.779 | INFO | "malloc" : " 14 GB", | 20260506 17:52:43.779 | INFO | "pinned" : " 29 GB" | 20260506 17:52:43.779 | INFO | } | 20260506 17:52:43.779 | INFO | Perf report for the last 5.14463 seconds: | 20260506 17:52:43.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:52:43.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:43.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:43.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:43.779 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 17:52:43.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:43.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:43.780 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:52:43.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:43.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:43.780 | INFO | Uploaded 36GiB to GPU in 1.98541s (18.1322GiB/s) | 20260506 17:52:43.780 | INFO | Downloaded 34GiB to GPU in 1.92322s (17.6787GiB/s) | 20260506 17:52:48.913 | INFO | Analyzing chunk frames = [3294720,3295232) | 20260506 17:52:48.913 | INFO | MemInfo: { | 20260506 17:52:48.913 | INFO | "gpu" : " 17 GB", | 20260506 17:52:48.913 | INFO | "huge" : "", | 20260506 17:52:48.913 | INFO | "huge_shared" : " 27 GB", | 20260506 17:52:48.914 | INFO | "malloc" : " 14 GB", | 20260506 17:52:48.914 | INFO | "pinned" : " 29 GB" | 20260506 17:52:48.914 | INFO | } | 20260506 17:52:48.914 | INFO | Perf report for the last 5.13416 seconds: | 20260506 17:52:48.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:52:48.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:48.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:48.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:48.914 | INFO | ANALYSIS : 60.1%, 3.0, 93.0ms, 0 | 20260506 17:52:48.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:48.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:48.914 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 17:52:48.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:48.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:48.914 | INFO | Uploaded 36GiB to GPU in 1.93528s (18.6019GiB/s) | 20260506 17:52:48.914 | INFO | Downloaded 34GiB to GPU in 1.86846s (18.1968GiB/s) | 20260506 17:52:54.054 | INFO | Analyzing chunk frames = [3295232,3295744) | 20260506 17:52:54.054 | INFO | MemInfo: { | 20260506 17:52:54.054 | INFO | "gpu" : " 17 GB", | 20260506 17:52:54.054 | INFO | "huge" : "", | 20260506 17:52:54.055 | INFO | "huge_shared" : " 27 GB", | 20260506 17:52:54.055 | INFO | "malloc" : " 14 GB", | 20260506 17:52:54.055 | INFO | "pinned" : " 29 GB" | 20260506 17:52:54.055 | INFO | } | 20260506 17:52:54.055 | INFO | Perf report for the last 5.14091 seconds: | 20260506 17:52:54.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:52:54.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:54.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:54.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:54.055 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 17:52:54.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:54.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:54.055 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:52:54.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:54.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:54.055 | INFO | Uploaded 36GiB to GPU in 2.01991s (17.8226GiB/s) | 20260506 17:52:54.055 | INFO | Downloaded 34GiB to GPU in 1.91246s (17.7782GiB/s) | 20260506 17:52:59.195 | INFO | Analyzing chunk frames = [3295744,3296256) | 20260506 17:52:59.195 | INFO | MemInfo: { | 20260506 17:52:59.195 | INFO | "gpu" : " 17 GB", | 20260506 17:52:59.195 | INFO | "huge" : "", | 20260506 17:52:59.195 | INFO | "huge_shared" : " 27 GB", | 20260506 17:52:59.195 | INFO | "malloc" : " 14 GB", | 20260506 17:52:59.195 | INFO | "pinned" : " 29 GB" | 20260506 17:52:59.195 | INFO | } | 20260506 17:52:59.195 | INFO | Perf report for the last 5.14037 seconds: | 20260506 17:52:59.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:52:59.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:59.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:59.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:59.195 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 17:52:59.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:59.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:59.195 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:52:59.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:59.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:52:59.195 | INFO | Uploaded 36GiB to GPU in 2.02782s (17.753GiB/s) | 20260506 17:52:59.195 | INFO | Downloaded 34GiB to GPU in 1.94984s (17.4373GiB/s) | 20260506 17:53:04.329 | INFO | Analyzing chunk frames = [3296256,3296768) | 20260506 17:53:04.329 | INFO | MemInfo: { | 20260506 17:53:04.329 | INFO | "gpu" : " 17 GB", | 20260506 17:53:04.329 | INFO | "huge" : "", | 20260506 17:53:04.329 | INFO | "huge_shared" : " 27 GB", | 20260506 17:53:04.329 | INFO | "malloc" : " 14 GB", | 20260506 17:53:04.329 | INFO | "pinned" : " 29 GB" | 20260506 17:53:04.329 | INFO | } | 20260506 17:53:04.329 | INFO | Perf report for the last 5.13402 seconds: | 20260506 17:53:04.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:53:04.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:04.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:04.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:04.329 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 17:53:04.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:04.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:04.329 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:53:04.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:04.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:04.329 | INFO | Uploaded 36GiB to GPU in 2.00095s (17.9914GiB/s) | 20260506 17:53:04.329 | INFO | Downloaded 34GiB to GPU in 1.99765s (17.02GiB/s) | 20260506 17:53:09.472 | INFO | Analyzing chunk frames = [3296768,3297280) | 20260506 17:53:09.472 | INFO | MemInfo: { | 20260506 17:53:09.472 | INFO | "gpu" : " 17 GB", | 20260506 17:53:09.472 | INFO | "huge" : "", | 20260506 17:53:09.472 | INFO | "huge_shared" : " 27 GB", | 20260506 17:53:09.472 | INFO | "malloc" : " 14 GB", | 20260506 17:53:09.472 | INFO | "pinned" : " 29 GB" | 20260506 17:53:09.472 | INFO | } | 20260506 17:53:09.472 | INFO | Perf report for the last 5.14342 seconds: | 20260506 17:53:09.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:53:09.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:09.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:09.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:09.473 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 17:53:09.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:09.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:09.473 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:53:09.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:09.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:09.473 | INFO | Uploaded 36GiB to GPU in 2.00388s (17.9652GiB/s) | 20260506 17:53:09.473 | INFO | Downloaded 34GiB to GPU in 1.98701s (17.1111GiB/s) | 20260506 17:53:09.589 | INFO | Starting a round of Baz IO | 20260506 17:53:12.724 | INFO | Populating RTMetrics report | 20260506 17:53:12.724 | INFO | RTMetrics report populated | 20260506 17:53:12.724 | INFO | Opening tmp RTMetrics json file | 20260506 17:53:12.795 | INFO | Serializing RTMetrics to json | 20260506 17:53:12.912 | INFO | Serialization done, swapping tmp file now... | 20260506 17:53:12.924 | INFO | RTMetrics done | 20260506 17:53:13.333 | INFO | Finished a round of Baz IO: Wrote 2.054 GiB in 3.743 seconds | 20260506 17:53:13.333 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.38% / 0.01% / 64.61% | 20260506 17:53:13.333 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.25898% / 3.97364e-06% / 5.5631e-05% | 20260506 17:53:13.333 | INFO | Baz IO overall: | 20260506 17:53:13.333 | INFO | BytesWritten : 1740095373936 | 20260506 17:53:13.333 | INFO | NumEvents : 841700709600 | 20260506 17:53:14.611 | INFO | Analyzing chunk frames = [3297280,3297792) | 20260506 17:53:14.611 | INFO | MemInfo: { | 20260506 17:53:14.611 | INFO | "gpu" : " 17 GB", | 20260506 17:53:14.611 | INFO | "huge" : "", | 20260506 17:53:14.611 | INFO | "huge_shared" : " 27 GB", | 20260506 17:53:14.611 | INFO | "malloc" : " 14 GB", | 20260506 17:53:14.611 | INFO | "pinned" : " 29 GB" | 20260506 17:53:14.611 | INFO | } | 20260506 17:53:14.611 | INFO | Perf report for the last 5.13851 seconds: | 20260506 17:53:14.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:53:14.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:14.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:14.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:14.611 | INFO | ANALYSIS : 62.7%, 2.9, 95.3ms, 0 | 20260506 17:53:14.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:14.611 | INFO | RT_METRICS : 47.2%, 1.0, 24.7ms, 0 | 20260506 17:53:14.611 | INFO | PRE_HQ : 60.1%, 1.9, 58.2ms, 0 | 20260506 17:53:14.611 | INFO | BAZWRITER : 72.5%, 4.8, 178.3ms, 0 | 20260506 17:53:14.611 | INFO | BAZ_SAMPLING : 3.6%, 1.0, 1.8ms, 0 | 20260506 17:53:14.611 | INFO | Uploaded 36GiB to GPU in 2.01807s (17.8388GiB/s) | 20260506 17:53:14.611 | INFO | Downloaded 38GiB to GPU in 2.20858s (17.2056GiB/s) | 20260506 17:53:19.746 | INFO | Analyzing chunk frames = [3297792,3298304) | 20260506 17:53:19.746 | INFO | MemInfo: { | 20260506 17:53:19.746 | INFO | "gpu" : " 17 GB", | 20260506 17:53:19.746 | INFO | "huge" : "", | 20260506 17:53:19.746 | INFO | "huge_shared" : " 27 GB", | 20260506 17:53:19.746 | INFO | "malloc" : " 14 GB", | 20260506 17:53:19.746 | INFO | "pinned" : " 29 GB" | 20260506 17:53:19.746 | INFO | } | 20260506 17:53:19.746 | INFO | Perf report for the last 5.13539 seconds: | 20260506 17:53:19.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:53:19.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:19.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:19.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:19.746 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:53:19.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:19.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:19.746 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:53:19.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:19.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:19.747 | INFO | Uploaded 36GiB to GPU in 1.99334s (18.0602GiB/s) | 20260506 17:53:19.747 | INFO | Downloaded 34GiB to GPU in 1.92937s (17.6223GiB/s) | 20260506 17:53:24.888 | INFO | Analyzing chunk frames = [3298304,3298816) | 20260506 17:53:24.888 | INFO | MemInfo: { | 20260506 17:53:24.888 | INFO | "gpu" : " 17 GB", | 20260506 17:53:24.888 | INFO | "huge" : "", | 20260506 17:53:24.888 | INFO | "huge_shared" : " 27 GB", | 20260506 17:53:24.888 | INFO | "malloc" : " 14 GB", | 20260506 17:53:24.888 | INFO | "pinned" : " 29 GB" | 20260506 17:53:24.888 | INFO | } | 20260506 17:53:24.888 | INFO | Perf report for the last 5.14205 seconds: | 20260506 17:53:24.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:53:24.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:24.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:24.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:24.888 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 17:53:24.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:24.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:24.889 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 17:53:24.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:24.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:24.889 | INFO | Uploaded 36GiB to GPU in 1.94053s (18.5516GiB/s) | 20260506 17:53:24.889 | INFO | Downloaded 34GiB to GPU in 1.88122s (18.0733GiB/s) | 20260506 17:53:30.029 | INFO | Analyzing chunk frames = [3298816,3299328) | 20260506 17:53:30.029 | INFO | MemInfo: { | 20260506 17:53:30.029 | INFO | "gpu" : " 17 GB", | 20260506 17:53:30.029 | INFO | "huge" : "", | 20260506 17:53:30.029 | INFO | "huge_shared" : " 27 GB", | 20260506 17:53:30.030 | INFO | "malloc" : " 14 GB", | 20260506 17:53:30.030 | INFO | "pinned" : " 29 GB" | 20260506 17:53:30.030 | INFO | } | 20260506 17:53:30.030 | INFO | Perf report for the last 5.1413 seconds: | 20260506 17:53:30.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:53:30.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:30.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:30.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:30.030 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 17:53:30.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:30.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:30.030 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:53:30.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:30.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:30.030 | INFO | Uploaded 36GiB to GPU in 1.92935s (18.6591GiB/s) | 20260506 17:53:30.030 | INFO | Downloaded 34GiB to GPU in 1.88142s (18.0714GiB/s) | 20260506 17:53:35.170 | INFO | Analyzing chunk frames = [3299328,3299840) | 20260506 17:53:35.170 | INFO | MemInfo: { | 20260506 17:53:35.170 | INFO | "gpu" : " 17 GB", | 20260506 17:53:35.170 | INFO | "huge" : "", | 20260506 17:53:35.170 | INFO | "huge_shared" : " 27 GB", | 20260506 17:53:35.171 | INFO | "malloc" : " 14 GB", | 20260506 17:53:35.171 | INFO | "pinned" : " 29 GB" | 20260506 17:53:35.171 | INFO | } | 20260506 17:53:35.171 | INFO | Perf report for the last 5.14088 seconds: | 20260506 17:53:35.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:53:35.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:35.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:35.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:35.171 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 17:53:35.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:35.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:35.171 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:53:35.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:35.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:35.171 | INFO | Uploaded 36GiB to GPU in 2.00484s (17.9565GiB/s) | 20260506 17:53:35.171 | INFO | Downloaded 34GiB to GPU in 1.91267s (17.7762GiB/s) | 20260506 17:53:40.308 | INFO | Analyzing chunk frames = [3299840,3300352) | 20260506 17:53:40.308 | INFO | MemInfo: { | 20260506 17:53:40.308 | INFO | "gpu" : " 17 GB", | 20260506 17:53:40.308 | INFO | "huge" : "", | 20260506 17:53:40.308 | INFO | "huge_shared" : " 27 GB", | 20260506 17:53:40.308 | INFO | "malloc" : " 14 GB", | 20260506 17:53:40.308 | INFO | "pinned" : " 29 GB" | 20260506 17:53:40.308 | INFO | } | 20260506 17:53:40.308 | INFO | Perf report for the last 5.1375 seconds: | 20260506 17:53:40.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:53:40.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:40.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:40.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:40.308 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 17:53:40.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:40.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:40.308 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:53:40.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:40.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:40.308 | INFO | Uploaded 36GiB to GPU in 2.02275s (17.7976GiB/s) | 20260506 17:53:40.308 | INFO | Downloaded 34GiB to GPU in 1.95216s (17.4166GiB/s) | 20260506 17:53:45.451 | INFO | Analyzing chunk frames = [3300352,3300864) | 20260506 17:53:45.451 | INFO | MemInfo: { | 20260506 17:53:45.451 | INFO | "gpu" : " 17 GB", | 20260506 17:53:45.451 | INFO | "huge" : "", | 20260506 17:53:45.451 | INFO | "huge_shared" : " 27 GB", | 20260506 17:53:45.451 | INFO | "malloc" : " 14 GB", | 20260506 17:53:45.451 | INFO | "pinned" : " 29 GB" | 20260506 17:53:45.451 | INFO | } | 20260506 17:53:45.451 | INFO | Perf report for the last 5.14271 seconds: | 20260506 17:53:45.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:53:45.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:45.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:45.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:45.451 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:53:45.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:45.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:45.451 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260506 17:53:45.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:45.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:45.451 | INFO | Uploaded 36GiB to GPU in 2.0014s (17.9874GiB/s) | 20260506 17:53:45.451 | INFO | Downloaded 34GiB to GPU in 1.98864s (17.0971GiB/s) | 20260506 17:53:50.589 | INFO | Analyzing chunk frames = [3300864,3301376) | 20260506 17:53:50.589 | INFO | MemInfo: { | 20260506 17:53:50.589 | INFO | "gpu" : " 17 GB", | 20260506 17:53:50.589 | INFO | "huge" : "", | 20260506 17:53:50.589 | INFO | "huge_shared" : " 27 GB", | 20260506 17:53:50.589 | INFO | "malloc" : " 14 GB", | 20260506 17:53:50.589 | INFO | "pinned" : " 29 GB" | 20260506 17:53:50.589 | INFO | } | 20260506 17:53:50.589 | INFO | Perf report for the last 5.13845 seconds: | 20260506 17:53:50.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:53:50.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:50.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:50.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:50.589 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 17:53:50.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:50.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:50.589 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:53:50.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:50.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:50.590 | INFO | Uploaded 36GiB to GPU in 2.03392s (17.6998GiB/s) | 20260506 17:53:50.590 | INFO | Downloaded 34GiB to GPU in 1.98372s (17.1395GiB/s) | 20260506 17:53:50.590 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:53:50.590 | INFO | Total mem usage: 18 GiB | 20260506 17:53:50.590 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:53:50.590 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.1%) | 20260506 17:53:50.590 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:53:50.590 | INFO | High Water Breakdown: | 20260506 17:53:50.590 | INFO | 61.7% : 9.82 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:53:50.590 | INFO | 38.2% : 6.08 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:53:50.590 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:53:50.590 | INFO | ------------------------------------------ | 20260506 17:53:50.590 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:53:50.590 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:53:50.590 | INFO | Total mem usage: 26.5 GiB | 20260506 17:53:50.590 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:53:50.590 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:53:50.590 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:53:50.590 | INFO | High Water Breakdown: | 20260506 17:53:50.590 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:53:50.590 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:53:50.590 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:53:50.590 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:53:50.590 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:53:50.590 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:53:50.590 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:53:50.590 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:53:50.590 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:53:50.590 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:53:50.590 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:53:50.590 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:53:50.590 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:53:50.590 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:53:50.590 | INFO | ------------------------------------------ | 20260506 17:53:50.590 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:53:50.590 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:53:50.590 | INFO | Total mem usage: 25.6 GiB | 20260506 17:53:50.590 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:53:50.590 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:53:50.590 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:53:50.590 | INFO | High Water Breakdown: | 20260506 17:53:50.590 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:53:50.590 | INFO | ------------------------------------------ | 20260506 17:53:50.590 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:53:50.590 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:53:50.590 | INFO | Total mem usage: 15.7 GiB | 20260506 17:53:50.590 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:53:50.590 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:53:50.590 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:53:50.590 | INFO | High Water Breakdown: | 20260506 17:53:50.590 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:53:50.590 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:53:50.590 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:53:50.590 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:53:50.590 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:53:50.590 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:53:50.590 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:53:50.590 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:53:50.590 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:53:50.590 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:53:50.590 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:53:50.590 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:53:50.590 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:53:50.590 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:53:50.591 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:53:50.591 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:53:50.591 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:53:50.591 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:53:50.591 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:53:50.591 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:53:50.591 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:53:50.591 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:53:50.591 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:53:50.591 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:53:50.591 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:53:50.591 | INFO | ------------------------------------------ | 20260506 17:53:50.591 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:53:50.591 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:53:50.591 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:53:50.591 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:53:50.591 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.57e+01, 1.35e+01 / 2.28e+00 | 20260506 17:53:50.591 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 1.06e-02 / 5.89e-03 | 20260506 17:53:50.591 | INFO | PerfReport: 11.62%, Baseline -- 3.64e+00 / 4.43e+00, 5.58e-01 / 3.12e-01 | 20260506 17:53:50.591 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.46e+00, 1.87e-01 / 1.33e-01 | 20260506 17:53:50.591 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.47e+01, 4.00e+00 / 2.49e+00 | 20260506 17:53:50.591 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.79e+01 / 2.03e+01, 1.45e+00 / 1.96e+00 | 20260506 17:53:50.591 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.27e+00 / 2.83e+00, 3.82e-01 / 2.24e-01 | 20260506 17:53:50.591 | INFO | PerfReport: 3.42%, Metrics -- 1.07e+00 / 1.41e+00, 2.84e-01 / 1.58e-01 | 20260506 17:53:50.591 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.71e-01 / 4.17e-01, 5.78e-02 / 2.50e-02 | 20260506 17:53:50.591 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:53:50.591 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:53:50.591 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:53:50.591 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 17:53:50.591 | INFO | PerfReport: 21.45%, Upload -- 2.08e+01 / 0.00e+00, 6.94e+00 / 0.00e+00 | 20260506 17:53:50.591 | INFO | PerfReport: 21.15%, Download -- 2.05e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 17:53:50.591 | INFO | PerfReport: 32.27%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 17:53:50.591 | INFO | PerfReport: 1.37%, LockUpload -- 1.33e+00 / 0.00e+00, 5.39e+00 / 0.00e+00 | 20260506 17:53:50.591 | INFO | PerfReport: 1.16%, LockDownload -- 1.13e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 17:53:50.591 | INFO | PerfReport: 22.57%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 17:53:50.591 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260506 17:53:50.710 | INFO | Starting a round of Baz IO | 20260506 17:53:53.844 | INFO | Populating RTMetrics report | 20260506 17:53:53.844 | INFO | RTMetrics report populated | 20260506 17:53:53.844 | INFO | Opening tmp RTMetrics json file | 20260506 17:53:53.954 | INFO | Serializing RTMetrics to json | 20260506 17:53:53.958 | INFO | Serialization done, swapping tmp file now... | 20260506 17:53:53.962 | INFO | RTMetrics done | 20260506 17:53:54.433 | INFO | Finished a round of Baz IO: Wrote 2.053 GiB in 3.722 seconds | 20260506 17:53:54.433 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.37% / 0.01% / 64.62% | 20260506 17:53:54.433 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.25481% / 7.94729e-06% / 4.76837e-05% | 20260506 17:53:54.433 | INFO | Baz IO overall: | 20260506 17:53:54.433 | INFO | BytesWritten : 1742300130301 | 20260506 17:53:54.433 | INFO | NumEvents : 842764065095 | 20260506 17:53:55.724 | INFO | Analyzing chunk frames = [3301376,3301888) | 20260506 17:53:55.724 | INFO | MemInfo: { | 20260506 17:53:55.724 | INFO | "gpu" : " 17 GB", | 20260506 17:53:55.724 | INFO | "huge" : "", | 20260506 17:53:55.724 | INFO | "huge_shared" : " 27 GB", | 20260506 17:53:55.724 | INFO | "malloc" : " 14 GB", | 20260506 17:53:55.724 | INFO | "pinned" : " 29 GB" | 20260506 17:53:55.724 | INFO | } | 20260506 17:53:55.724 | INFO | Perf report for the last 5.13434 seconds: | 20260506 17:53:55.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:53:55.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:55.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:55.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:55.724 | INFO | ANALYSIS : 62.8%, 2.9, 95.6ms, 0 | 20260506 17:53:55.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:53:55.724 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 17:53:55.724 | INFO | PRE_HQ : 60.3%, 1.9, 57.9ms, 0 | 20260506 17:53:55.724 | INFO | BAZWRITER : 72.2%, 4.8, 177.7ms, 0 | 20260506 17:53:55.724 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 17:53:55.724 | INFO | Uploaded 36GiB to GPU in 2.08241s (17.2877GiB/s) | 20260506 17:53:55.724 | INFO | Downloaded 38GiB to GPU in 2.22503s (17.0784GiB/s) | 20260506 17:54:00.862 | INFO | Analyzing chunk frames = [3301888,3302400) | 20260506 17:54:00.862 | INFO | MemInfo: { | 20260506 17:54:00.862 | INFO | "gpu" : " 17 GB", | 20260506 17:54:00.862 | INFO | "huge" : "", | 20260506 17:54:00.862 | INFO | "huge_shared" : " 27 GB", | 20260506 17:54:00.862 | INFO | "malloc" : " 14 GB", | 20260506 17:54:00.862 | INFO | "pinned" : " 29 GB" | 20260506 17:54:00.862 | INFO | } | 20260506 17:54:00.862 | INFO | Perf report for the last 5.13872 seconds: | 20260506 17:54:00.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:54:00.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:00.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:00.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:00.862 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:54:00.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:00.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:00.863 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 17:54:00.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:00.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:00.863 | INFO | Uploaded 36GiB to GPU in 2.01324s (17.8816GiB/s) | 20260506 17:54:00.863 | INFO | Downloaded 34GiB to GPU in 1.94825s (17.4516GiB/s) | 20260506 17:54:06.003 | INFO | Analyzing chunk frames = [3302400,3302912) | 20260506 17:54:06.003 | INFO | MemInfo: { | 20260506 17:54:06.003 | INFO | "gpu" : " 17 GB", | 20260506 17:54:06.003 | INFO | "huge" : "", | 20260506 17:54:06.003 | INFO | "huge_shared" : " 27 GB", | 20260506 17:54:06.003 | INFO | "malloc" : " 14 GB", | 20260506 17:54:06.003 | INFO | "pinned" : " 29 GB" | 20260506 17:54:06.003 | INFO | } | 20260506 17:54:06.003 | INFO | Perf report for the last 5.14067 seconds: | 20260506 17:54:06.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:54:06.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:06.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:06.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:06.003 | INFO | ANALYSIS : 61.1%, 3.0, 94.6ms, 0 | 20260506 17:54:06.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:06.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:06.003 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:54:06.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:06.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:06.003 | INFO | Uploaded 36GiB to GPU in 1.99613s (18.0349GiB/s) | 20260506 17:54:06.003 | INFO | Downloaded 34GiB to GPU in 1.92666s (17.6471GiB/s) | 20260506 17:54:11.146 | INFO | Analyzing chunk frames = [3302912,3303424) | 20260506 17:54:11.146 | INFO | MemInfo: { | 20260506 17:54:11.146 | INFO | "gpu" : " 17 GB", | 20260506 17:54:11.146 | INFO | "huge" : "", | 20260506 17:54:11.147 | INFO | "huge_shared" : " 27 GB", | 20260506 17:54:11.147 | INFO | "malloc" : " 14 GB", | 20260506 17:54:11.147 | INFO | "pinned" : " 29 GB" | 20260506 17:54:11.147 | INFO | } | 20260506 17:54:11.147 | INFO | Perf report for the last 5.14345 seconds: | 20260506 17:54:11.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:54:11.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:11.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:11.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:11.147 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260506 17:54:11.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:11.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:11.147 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:54:11.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:11.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:11.147 | INFO | Uploaded 36GiB to GPU in 1.92965s (18.6563GiB/s) | 20260506 17:54:11.147 | INFO | Downloaded 34GiB to GPU in 1.87056s (18.1764GiB/s) | 20260506 17:54:16.286 | INFO | Analyzing chunk frames = [3303424,3303936) | 20260506 17:54:16.286 | INFO | MemInfo: { | 20260506 17:54:16.286 | INFO | "gpu" : " 17 GB", | 20260506 17:54:16.286 | INFO | "huge" : "", | 20260506 17:54:16.286 | INFO | "huge_shared" : " 27 GB", | 20260506 17:54:16.286 | INFO | "malloc" : " 14 GB", | 20260506 17:54:16.286 | INFO | "pinned" : " 29 GB" | 20260506 17:54:16.286 | INFO | } | 20260506 17:54:16.286 | INFO | Perf report for the last 5.13975 seconds: | 20260506 17:54:16.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:54:16.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:16.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:16.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:16.286 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 17:54:16.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:16.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:16.286 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:54:16.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:16.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:16.287 | INFO | Uploaded 36GiB to GPU in 2.02211s (17.8032GiB/s) | 20260506 17:54:16.287 | INFO | Downloaded 34GiB to GPU in 1.91414s (17.7625GiB/s) | 20260506 17:54:21.420 | INFO | Analyzing chunk frames = [3303936,3304448) | 20260506 17:54:21.420 | INFO | MemInfo: { | 20260506 17:54:21.420 | INFO | "gpu" : " 17 GB", | 20260506 17:54:21.420 | INFO | "huge" : "", | 20260506 17:54:21.420 | INFO | "huge_shared" : " 27 GB", | 20260506 17:54:21.420 | INFO | "malloc" : " 14 GB", | 20260506 17:54:21.420 | INFO | "pinned" : " 29 GB" | 20260506 17:54:21.420 | INFO | } | 20260506 17:54:21.420 | INFO | Perf report for the last 5.13402 seconds: | 20260506 17:54:21.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:54:21.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:21.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:21.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:21.421 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:54:21.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:21.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:21.421 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260506 17:54:21.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:21.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:21.421 | INFO | Uploaded 36GiB to GPU in 2.07109s (17.3822GiB/s) | 20260506 17:54:21.421 | INFO | Downloaded 34GiB to GPU in 1.9579s (17.3655GiB/s) | 20260506 17:54:26.559 | INFO | Analyzing chunk frames = [3304448,3304960) | 20260506 17:54:26.559 | INFO | MemInfo: { | 20260506 17:54:26.559 | INFO | "gpu" : " 17 GB", | 20260506 17:54:26.559 | INFO | "huge" : "", | 20260506 17:54:26.559 | INFO | "huge_shared" : " 27 GB", | 20260506 17:54:26.559 | INFO | "malloc" : " 14 GB", | 20260506 17:54:26.559 | INFO | "pinned" : " 29 GB" | 20260506 17:54:26.559 | INFO | } | 20260506 17:54:26.559 | INFO | Perf report for the last 5.13849 seconds: | 20260506 17:54:26.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:54:26.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:26.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:26.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:26.559 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 17:54:26.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:26.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:26.559 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 17:54:26.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:26.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:26.559 | INFO | Uploaded 36GiB to GPU in 2.03703s (17.6728GiB/s) | 20260506 17:54:26.559 | INFO | Downloaded 34GiB to GPU in 1.99295s (17.0601GiB/s) | 20260506 17:54:31.696 | INFO | Analyzing chunk frames = [3304960,3305472) | 20260506 17:54:31.696 | INFO | MemInfo: { | 20260506 17:54:31.696 | INFO | "gpu" : " 17 GB", | 20260506 17:54:31.696 | INFO | "huge" : "", | 20260506 17:54:31.696 | INFO | "huge_shared" : " 27 GB", | 20260506 17:54:31.696 | INFO | "malloc" : " 14 GB", | 20260506 17:54:31.696 | INFO | "pinned" : " 29 GB" | 20260506 17:54:31.696 | INFO | } | 20260506 17:54:31.696 | INFO | Perf report for the last 5.13728 seconds: | 20260506 17:54:31.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:54:31.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:31.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:31.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:31.696 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:54:31.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:31.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:31.696 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 17:54:31.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:31.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:31.697 | INFO | Uploaded 36GiB to GPU in 1.99928s (18.0065GiB/s) | 20260506 17:54:31.697 | INFO | Downloaded 34GiB to GPU in 1.97598s (17.2066GiB/s) | 20260506 17:54:31.811 | INFO | Starting a round of Baz IO | 20260506 17:54:34.944 | INFO | Populating RTMetrics report | 20260506 17:54:34.945 | INFO | RTMetrics report populated | 20260506 17:54:34.945 | INFO | Opening tmp RTMetrics json file | 20260506 17:54:34.947 | INFO | Serializing RTMetrics to json | 20260506 17:54:34.953 | INFO | Serialization done, swapping tmp file now... | 20260506 17:54:34.958 | INFO | RTMetrics done | 20260506 17:54:35.487 | INFO | Finished a round of Baz IO: Wrote 2.053 GiB in 3.676 seconds | 20260506 17:54:35.487 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.37% / 0.01% / 64.63% | 20260506 17:54:35.487 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.25126% / 7.94729e-06% / 5.96046e-05% | 20260506 17:54:35.487 | INFO | Baz IO overall: | 20260506 17:54:35.488 | INFO | BytesWritten : 1744504295320 | 20260506 17:54:35.488 | INFO | NumEvents : 843826787053 | 20260506 17:54:36.835 | INFO | Analyzing chunk frames = [3305472,3305984) | 20260506 17:54:36.835 | INFO | MemInfo: { | 20260506 17:54:36.835 | INFO | "gpu" : " 17 GB", | 20260506 17:54:36.835 | INFO | "huge" : "", | 20260506 17:54:36.835 | INFO | "huge_shared" : " 27 GB", | 20260506 17:54:36.835 | INFO | "malloc" : " 14 GB", | 20260506 17:54:36.835 | INFO | "pinned" : " 29 GB" | 20260506 17:54:36.835 | INFO | } | 20260506 17:54:36.835 | INFO | Perf report for the last 5.13936 seconds: | 20260506 17:54:36.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:54:36.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:36.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:36.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:36.836 | INFO | ANALYSIS : 62.6%, 2.9, 95.2ms, 0 | 20260506 17:54:36.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:36.836 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 17:54:36.836 | INFO | PRE_HQ : 60.2%, 1.9, 58.2ms, 0 | 20260506 17:54:36.836 | INFO | BAZWRITER : 71.2%, 4.8, 175.7ms, 0 | 20260506 17:54:36.836 | INFO | BAZ_SAMPLING : 4.0%, 1.0, 2.1ms, 0 | 20260506 17:54:36.836 | INFO | Uploaded 36GiB to GPU in 2.00775s (17.9305GiB/s) | 20260506 17:54:36.836 | INFO | Downloaded 38GiB to GPU in 2.20897s (17.2026GiB/s) | 20260506 17:54:41.974 | INFO | Analyzing chunk frames = [3305984,3306496) | 20260506 17:54:41.974 | INFO | MemInfo: { | 20260506 17:54:41.974 | INFO | "gpu" : " 17 GB", | 20260506 17:54:41.974 | INFO | "huge" : "", | 20260506 17:54:41.974 | INFO | "huge_shared" : " 27 GB", | 20260506 17:54:41.974 | INFO | "malloc" : " 14 GB", | 20260506 17:54:41.974 | INFO | "pinned" : " 29 GB" | 20260506 17:54:41.974 | INFO | } | 20260506 17:54:41.974 | INFO | Perf report for the last 5.13853 seconds: | 20260506 17:54:41.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:54:41.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:41.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:41.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:41.974 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:54:41.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:41.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:41.974 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:54:41.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:41.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:41.974 | INFO | Uploaded 36GiB to GPU in 2.00081s (17.9927GiB/s) | 20260506 17:54:41.975 | INFO | Downloaded 34GiB to GPU in 1.92946s (17.6215GiB/s) | 20260506 17:54:47.120 | INFO | Analyzing chunk frames = [3306496,3307008) | 20260506 17:54:47.120 | INFO | MemInfo: { | 20260506 17:54:47.120 | INFO | "gpu" : " 17 GB", | 20260506 17:54:47.120 | INFO | "huge" : "", | 20260506 17:54:47.120 | INFO | "huge_shared" : " 27 GB", | 20260506 17:54:47.120 | INFO | "malloc" : " 14 GB", | 20260506 17:54:47.120 | INFO | "pinned" : " 29 GB" | 20260506 17:54:47.120 | INFO | } | 20260506 17:54:47.120 | INFO | Perf report for the last 5.14615 seconds: | 20260506 17:54:47.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:54:47.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:47.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:47.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:47.120 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 17:54:47.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:47.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:47.120 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:54:47.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:47.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:47.120 | INFO | Uploaded 36GiB to GPU in 1.96337s (18.3358GiB/s) | 20260506 17:54:47.121 | INFO | Downloaded 34GiB to GPU in 1.92262s (17.6842GiB/s) | 20260506 17:54:52.265 | INFO | Analyzing chunk frames = [3307008,3307520) | 20260506 17:54:52.265 | INFO | MemInfo: { | 20260506 17:54:52.265 | INFO | "gpu" : " 17 GB", | 20260506 17:54:52.265 | INFO | "huge" : "", | 20260506 17:54:52.265 | INFO | "huge_shared" : " 27 GB", | 20260506 17:54:52.266 | INFO | "malloc" : " 14 GB", | 20260506 17:54:52.266 | INFO | "pinned" : " 29 GB" | 20260506 17:54:52.266 | INFO | } | 20260506 17:54:52.266 | INFO | Perf report for the last 5.14532 seconds: | 20260506 17:54:52.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:54:52.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:52.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:52.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:52.266 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 17:54:52.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:52.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:52.266 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:54:52.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:52.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:52.266 | INFO | Uploaded 36GiB to GPU in 1.92528s (18.6985GiB/s) | 20260506 17:54:52.266 | INFO | Downloaded 34GiB to GPU in 1.87238s (18.1587GiB/s) | 20260506 17:54:57.404 | INFO | Analyzing chunk frames = [3307520,3308032) | 20260506 17:54:57.404 | INFO | MemInfo: { | 20260506 17:54:57.404 | INFO | "gpu" : " 17 GB", | 20260506 17:54:57.404 | INFO | "huge" : "", | 20260506 17:54:57.404 | INFO | "huge_shared" : " 27 GB", | 20260506 17:54:57.404 | INFO | "malloc" : " 14 GB", | 20260506 17:54:57.404 | INFO | "pinned" : " 29 GB" | 20260506 17:54:57.404 | INFO | } | 20260506 17:54:57.404 | INFO | Perf report for the last 5.13886 seconds: | 20260506 17:54:57.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:54:57.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:57.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:57.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:57.405 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 17:54:57.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:57.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:57.405 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:54:57.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:57.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:54:57.405 | INFO | Uploaded 36GiB to GPU in 1.9983s (18.0153GiB/s) | 20260506 17:54:57.405 | INFO | Downloaded 34GiB to GPU in 1.91696s (17.7364GiB/s) | 20260506 17:55:02.542 | INFO | Analyzing chunk frames = [3308032,3308544) | 20260506 17:55:02.542 | INFO | MemInfo: { | 20260506 17:55:02.542 | INFO | "gpu" : " 17 GB", | 20260506 17:55:02.542 | INFO | "huge" : "", | 20260506 17:55:02.542 | INFO | "huge_shared" : " 27 GB", | 20260506 17:55:02.542 | INFO | "malloc" : " 14 GB", | 20260506 17:55:02.542 | INFO | "pinned" : " 29 GB" | 20260506 17:55:02.542 | INFO | } | 20260506 17:55:02.542 | INFO | Perf report for the last 5.13794 seconds: | 20260506 17:55:02.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:55:02.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:02.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:02.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:02.543 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 17:55:02.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:02.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:02.543 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 17:55:02.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:02.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:02.543 | INFO | Uploaded 36GiB to GPU in 2.00953s (17.9147GiB/s) | 20260506 17:55:02.543 | INFO | Downloaded 34GiB to GPU in 1.95347s (17.4049GiB/s) | 20260506 17:55:07.676 | INFO | Analyzing chunk frames = [3308544,3309056) | 20260506 17:55:07.676 | INFO | MemInfo: { | 20260506 17:55:07.676 | INFO | "gpu" : " 17 GB", | 20260506 17:55:07.676 | INFO | "huge" : "", | 20260506 17:55:07.676 | INFO | "huge_shared" : " 27 GB", | 20260506 17:55:07.676 | INFO | "malloc" : " 14 GB", | 20260506 17:55:07.676 | INFO | "pinned" : " 29 GB" | 20260506 17:55:07.676 | INFO | } | 20260506 17:55:07.676 | INFO | Perf report for the last 5.13338 seconds: | 20260506 17:55:07.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:55:07.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:07.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:07.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:07.676 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:55:07.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:07.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:07.676 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 17:55:07.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:07.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:07.676 | INFO | Uploaded 36GiB to GPU in 2.00297s (17.9733GiB/s) | 20260506 17:55:07.676 | INFO | Downloaded 34GiB to GPU in 2.01404s (16.8815GiB/s) | 20260506 17:55:12.812 | INFO | Analyzing chunk frames = [3309056,3309568) | 20260506 17:55:12.812 | INFO | MemInfo: { | 20260506 17:55:12.812 | INFO | "gpu" : " 17 GB", | 20260506 17:55:12.812 | INFO | "huge" : "", | 20260506 17:55:12.812 | INFO | "huge_shared" : " 27 GB", | 20260506 17:55:12.812 | INFO | "malloc" : " 14 GB", | 20260506 17:55:12.812 | INFO | "pinned" : " 29 GB" | 20260506 17:55:12.812 | INFO | } | 20260506 17:55:12.812 | INFO | Perf report for the last 5.13642 seconds: | 20260506 17:55:12.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:55:12.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:12.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:12.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:12.812 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 17:55:12.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:12.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:12.812 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 17:55:12.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:12.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:12.812 | INFO | Uploaded 36GiB to GPU in 2.06867s (17.4025GiB/s) | 20260506 17:55:12.813 | INFO | Downloaded 34GiB to GPU in 1.97469s (17.2179GiB/s) | 20260506 17:55:12.813 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:55:12.813 | INFO | Total mem usage: 18 GiB | 20260506 17:55:12.813 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:55:12.813 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.3%) | 20260506 17:55:12.813 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:55:12.813 | INFO | High Water Breakdown: | 20260506 17:55:12.813 | INFO | 61.8% : 9.91 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:55:12.813 | INFO | 38.1% : 6.1 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:55:12.813 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:55:12.813 | INFO | ------------------------------------------ | 20260506 17:55:12.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:55:12.813 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:55:12.813 | INFO | Total mem usage: 26.5 GiB | 20260506 17:55:12.813 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:55:12.813 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:55:12.813 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:55:12.813 | INFO | High Water Breakdown: | 20260506 17:55:12.813 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:55:12.813 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:55:12.813 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:55:12.813 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:55:12.813 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:55:12.813 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:55:12.813 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:55:12.813 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:55:12.813 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:55:12.813 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:55:12.813 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:55:12.813 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:55:12.813 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:55:12.813 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:55:12.813 | INFO | ------------------------------------------ | 20260506 17:55:12.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:55:12.813 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:55:12.813 | INFO | Total mem usage: 25.6 GiB | 20260506 17:55:12.813 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:55:12.813 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:55:12.813 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:55:12.813 | INFO | High Water Breakdown: | 20260506 17:55:12.813 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:55:12.813 | INFO | ------------------------------------------ | 20260506 17:55:12.813 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:55:12.813 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:55:12.813 | INFO | Total mem usage: 15.7 GiB | 20260506 17:55:12.813 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:55:12.813 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:55:12.813 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 17:55:12.813 | INFO | High Water Breakdown: | 20260506 17:55:12.813 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:55:12.813 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:55:12.813 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:55:12.813 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:55:12.813 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:55:12.813 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:55:12.813 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:55:12.813 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:55:12.813 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:55:12.813 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:55:12.814 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:55:12.814 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:55:12.814 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:55:12.814 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:55:12.814 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:55:12.814 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:55:12.814 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:55:12.814 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:55:12.814 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:55:12.814 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:55:12.814 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:55:12.814 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:55:12.814 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:55:12.814 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:55:12.814 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:55:12.814 | INFO | ------------------------------------------ | 20260506 17:55:12.814 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:55:12.814 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:55:12.814 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:55:12.814 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:55:12.814 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.57e+01, 1.36e+01 / 2.52e+00 | 20260506 17:55:12.814 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.02e-01, 1.01e-02 / 7.16e-03 | 20260506 17:55:12.814 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.43e+00, 5.60e-01 / 3.31e-01 | 20260506 17:55:12.814 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.84e-01 / 1.32e-01 | 20260506 17:55:12.814 | INFO | PerfReport: 15.95%, DME -- 4.00e+01 / 4.45e+01, 4.13e+00 / 2.89e+00 | 20260506 17:55:12.814 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.44e+00 / 2.00e+00 | 20260506 17:55:12.814 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.26e+00 / 2.83e+00, 3.79e-01 / 2.15e-01 | 20260506 17:55:12.814 | INFO | PerfReport: 3.43%, Metrics -- 1.08e+00 / 1.40e+00, 2.85e-01 / 1.75e-01 | 20260506 17:55:12.814 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.71e-01 / 4.19e-01, 5.75e-02 / 2.93e-02 | 20260506 17:55:12.814 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:55:12.814 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:55:12.814 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:55:12.814 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 17:55:12.814 | INFO | PerfReport: 21.51%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 17:55:12.814 | INFO | PerfReport: 21.16%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 17:55:12.814 | INFO | PerfReport: 32.20%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 17:55:12.814 | INFO | PerfReport: 1.43%, LockUpload -- 1.39e+00 / 0.00e+00, 5.66e+00 / 0.00e+00 | 20260506 17:55:12.814 | INFO | PerfReport: 1.11%, LockDownload -- 1.08e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260506 17:55:12.814 | INFO | PerfReport: 22.57%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 17:55:12.814 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260506 17:55:12.923 | INFO | Starting a round of Baz IO | 20260506 17:55:16.103 | INFO | Populating RTMetrics report | 20260506 17:55:16.104 | INFO | RTMetrics report populated | 20260506 17:55:16.104 | INFO | Opening tmp RTMetrics json file | 20260506 17:55:16.132 | INFO | Serializing RTMetrics to json | 20260506 17:55:16.151 | INFO | Serialization done, swapping tmp file now... | 20260506 17:55:16.168 | INFO | RTMetrics done | 20260506 17:55:16.657 | INFO | Finished a round of Baz IO: Wrote 2.053 GiB in 3.733 seconds | 20260506 17:55:16.657 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.36% / 0.01% / 64.63% | 20260506 17:55:16.657 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.24927% / 1.19209e-05% / 5.16574e-05% | 20260506 17:55:16.657 | INFO | Baz IO overall: | 20260506 17:55:16.657 | INFO | BytesWritten : 1746708711667 | 20260506 17:55:16.657 | INFO | NumEvents : 844889885589 | 20260506 17:55:17.954 | INFO | Analyzing chunk frames = [3309568,3310080) | 20260506 17:55:17.954 | INFO | MemInfo: { | 20260506 17:55:17.954 | INFO | "gpu" : " 17 GB", | 20260506 17:55:17.954 | INFO | "huge" : "", | 20260506 17:55:17.954 | INFO | "huge_shared" : " 27 GB", | 20260506 17:55:17.954 | INFO | "malloc" : " 14 GB", | 20260506 17:55:17.954 | INFO | "pinned" : " 29 GB" | 20260506 17:55:17.954 | INFO | } | 20260506 17:55:17.954 | INFO | Perf report for the last 5.14223 seconds: | 20260506 17:55:17.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:55:17.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:17.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:17.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:17.955 | INFO | ANALYSIS : 63.1%, 2.8, 93.3ms, 0 | 20260506 17:55:17.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:17.955 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260506 17:55:17.955 | INFO | PRE_HQ : 60.9%, 2.0, 61.9ms, 0 | 20260506 17:55:17.955 | INFO | BAZWRITER : 72.2%, 4.8, 179.3ms, 0 | 20260506 17:55:17.955 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 17:55:17.955 | INFO | Uploaded 36GiB to GPU in 2.04953s (17.565GiB/s) | 20260506 17:55:17.955 | INFO | Downloaded 38GiB to GPU in 2.18026s (17.4291GiB/s) | 20260506 17:55:23.091 | INFO | Analyzing chunk frames = [3310080,3310592) | 20260506 17:55:23.091 | INFO | MemInfo: { | 20260506 17:55:23.091 | INFO | "gpu" : " 17 GB", | 20260506 17:55:23.091 | INFO | "huge" : "", | 20260506 17:55:23.091 | INFO | "huge_shared" : " 27 GB", | 20260506 17:55:23.091 | INFO | "malloc" : " 14 GB", | 20260506 17:55:23.091 | INFO | "pinned" : " 29 GB" | 20260506 17:55:23.091 | INFO | } | 20260506 17:55:23.092 | INFO | Perf report for the last 5.13716 seconds: | 20260506 17:55:23.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:55:23.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:23.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:23.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:23.092 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:55:23.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:23.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:23.092 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 17:55:23.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:23.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:23.092 | INFO | Uploaded 36GiB to GPU in 1.99259s (18.067GiB/s) | 20260506 17:55:23.092 | INFO | Downloaded 34GiB to GPU in 1.93168s (17.6013GiB/s) | 20260506 17:55:28.233 | INFO | Analyzing chunk frames = [3310592,3311104) | 20260506 17:55:28.233 | INFO | MemInfo: { | 20260506 17:55:28.233 | INFO | "gpu" : " 17 GB", | 20260506 17:55:28.233 | INFO | "huge" : "", | 20260506 17:55:28.233 | INFO | "huge_shared" : " 27 GB", | 20260506 17:55:28.233 | INFO | "malloc" : " 14 GB", | 20260506 17:55:28.233 | INFO | "pinned" : " 29 GB" | 20260506 17:55:28.233 | INFO | } | 20260506 17:55:28.233 | INFO | Perf report for the last 5.14158 seconds: | 20260506 17:55:28.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:55:28.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:28.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:28.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:28.233 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 17:55:28.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:28.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:28.233 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:55:28.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:28.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:28.233 | INFO | Uploaded 36GiB to GPU in 1.9693s (18.2806GiB/s) | 20260506 17:55:28.234 | INFO | Downloaded 34GiB to GPU in 1.92014s (17.7071GiB/s) | 20260506 17:55:33.373 | INFO | Analyzing chunk frames = [3311104,3311616) | 20260506 17:55:33.373 | INFO | MemInfo: { | 20260506 17:55:33.373 | INFO | "gpu" : " 17 GB", | 20260506 17:55:33.373 | INFO | "huge" : "", | 20260506 17:55:33.373 | INFO | "huge_shared" : " 27 GB", | 20260506 17:55:33.373 | INFO | "malloc" : " 14 GB", | 20260506 17:55:33.373 | INFO | "pinned" : " 29 GB" | 20260506 17:55:33.373 | INFO | } | 20260506 17:55:33.373 | INFO | Perf report for the last 5.13966 seconds: | 20260506 17:55:33.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:55:33.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:33.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:33.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:33.373 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:55:33.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:33.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:33.373 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:55:33.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:33.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:33.373 | INFO | Uploaded 36GiB to GPU in 1.93038s (18.6492GiB/s) | 20260506 17:55:33.373 | INFO | Downloaded 34GiB to GPU in 1.86797s (18.2016GiB/s) | 20260506 17:55:38.510 | INFO | Analyzing chunk frames = [3311616,3312128) | 20260506 17:55:38.510 | INFO | MemInfo: { | 20260506 17:55:38.510 | INFO | "gpu" : " 17 GB", | 20260506 17:55:38.510 | INFO | "huge" : "", | 20260506 17:55:38.510 | INFO | "huge_shared" : " 27 GB", | 20260506 17:55:38.510 | INFO | "malloc" : " 14 GB", | 20260506 17:55:38.510 | INFO | "pinned" : " 29 GB" | 20260506 17:55:38.510 | INFO | } | 20260506 17:55:38.510 | INFO | Perf report for the last 5.13756 seconds: | 20260506 17:55:38.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:55:38.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:38.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:38.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:38.511 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 17:55:38.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:38.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:38.511 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:55:38.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:38.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:38.511 | INFO | Uploaded 36GiB to GPU in 1.99275s (18.0655GiB/s) | 20260506 17:55:38.511 | INFO | Downloaded 34GiB to GPU in 1.92446s (17.6673GiB/s) | 20260506 17:55:43.651 | INFO | Analyzing chunk frames = [3312128,3312640) | 20260506 17:55:43.651 | INFO | MemInfo: { | 20260506 17:55:43.651 | INFO | "gpu" : " 17 GB", | 20260506 17:55:43.651 | INFO | "huge" : "", | 20260506 17:55:43.651 | INFO | "huge_shared" : " 27 GB", | 20260506 17:55:43.651 | INFO | "malloc" : " 14 GB", | 20260506 17:55:43.651 | INFO | "pinned" : " 29 GB" | 20260506 17:55:43.651 | INFO | } | 20260506 17:55:43.651 | INFO | Perf report for the last 5.14044 seconds: | 20260506 17:55:43.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:55:43.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:43.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:43.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:43.651 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 17:55:43.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:43.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:43.651 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:55:43.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:43.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:43.651 | INFO | Uploaded 36GiB to GPU in 2.02376s (17.7886GiB/s) | 20260506 17:55:43.651 | INFO | Downloaded 34GiB to GPU in 1.94374s (17.4921GiB/s) | 20260506 17:55:48.791 | INFO | Analyzing chunk frames = [3312640,3313152) | 20260506 17:55:48.791 | INFO | MemInfo: { | 20260506 17:55:48.791 | INFO | "gpu" : " 17 GB", | 20260506 17:55:48.791 | INFO | "huge" : "", | 20260506 17:55:48.791 | INFO | "huge_shared" : " 27 GB", | 20260506 17:55:48.791 | INFO | "malloc" : " 14 GB", | 20260506 17:55:48.791 | INFO | "pinned" : " 29 GB" | 20260506 17:55:48.791 | INFO | } | 20260506 17:55:48.791 | INFO | Perf report for the last 5.1403 seconds: | 20260506 17:55:48.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:55:48.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:48.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:48.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:48.791 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 17:55:48.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:48.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:48.791 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:55:48.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:48.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:48.792 | INFO | Uploaded 36GiB to GPU in 2.00042s (17.9962GiB/s) | 20260506 17:55:48.792 | INFO | Downloaded 34GiB to GPU in 2.01576s (16.8671GiB/s) | 20260506 17:55:53.930 | INFO | Analyzing chunk frames = [3313152,3313664) | 20260506 17:55:53.930 | INFO | MemInfo: { | 20260506 17:55:53.930 | INFO | "gpu" : " 17 GB", | 20260506 17:55:53.930 | INFO | "huge" : "", | 20260506 17:55:53.930 | INFO | "huge_shared" : " 27 GB", | 20260506 17:55:53.930 | INFO | "malloc" : " 14 GB", | 20260506 17:55:53.930 | INFO | "pinned" : " 29 GB" | 20260506 17:55:53.930 | INFO | } | 20260506 17:55:53.930 | INFO | Perf report for the last 5.13913 seconds: | 20260506 17:55:53.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:55:53.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:53.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:53.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:53.930 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 17:55:53.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:53.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:53.930 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:55:53.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:53.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:53.931 | INFO | Uploaded 36GiB to GPU in 2.04644s (17.5915GiB/s) | 20260506 17:55:53.931 | INFO | Downloaded 34GiB to GPU in 1.97598s (17.2067GiB/s) | 20260506 17:55:54.049 | INFO | Starting a round of Baz IO | 20260506 17:55:57.188 | INFO | Populating RTMetrics report | 20260506 17:55:57.188 | INFO | RTMetrics report populated | 20260506 17:55:57.188 | INFO | Opening tmp RTMetrics json file | 20260506 17:55:57.231 | INFO | Serializing RTMetrics to json | 20260506 17:55:57.344 | INFO | Serialization done, swapping tmp file now... | 20260506 17:55:57.349 | INFO | RTMetrics done | 20260506 17:55:57.663 | INFO | Finished a round of Baz IO: Wrote 2.053 GiB in 3.613 seconds | 20260506 17:55:57.663 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.35% / 0.01% / 64.64% | 20260506 17:55:57.663 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.24581% / 7.94729e-06% / 5.5631e-05% | 20260506 17:55:57.663 | INFO | Baz IO overall: | 20260506 17:55:57.663 | INFO | BytesWritten : 1748913343496 | 20260506 17:55:57.663 | INFO | NumEvents : 845953254484 | 20260506 17:55:59.067 | INFO | Analyzing chunk frames = [3313664,3314176) | 20260506 17:55:59.067 | INFO | MemInfo: { | 20260506 17:55:59.067 | INFO | "gpu" : " 17 GB", | 20260506 17:55:59.067 | INFO | "huge" : "", | 20260506 17:55:59.067 | INFO | "huge_shared" : " 27 GB", | 20260506 17:55:59.067 | INFO | "malloc" : " 14 GB", | 20260506 17:55:59.067 | INFO | "pinned" : " 29 GB" | 20260506 17:55:59.067 | INFO | } | 20260506 17:55:59.067 | INFO | Perf report for the last 5.13653 seconds: | 20260506 17:55:59.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:55:59.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:59.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:59.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:59.067 | INFO | ANALYSIS : 62.9%, 2.9, 95.6ms, 0 | 20260506 17:55:59.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:55:59.067 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260506 17:55:59.067 | INFO | PRE_HQ : 60.1%, 1.9, 58.0ms, 0 | 20260506 17:55:59.067 | INFO | BAZWRITER : 70.0%, 4.8, 171.6ms, 0 | 20260506 17:55:59.067 | INFO | BAZ_SAMPLING : 3.0%, 1.0, 1.6ms, 0 | 20260506 17:55:59.067 | INFO | Uploaded 36GiB to GPU in 2.07923s (17.3141GiB/s) | 20260506 17:55:59.067 | INFO | Downloaded 38GiB to GPU in 2.22656s (17.0667GiB/s) | 20260506 17:56:04.213 | INFO | Analyzing chunk frames = [3314176,3314688) | 20260506 17:56:04.213 | INFO | MemInfo: { | 20260506 17:56:04.213 | INFO | "gpu" : " 17 GB", | 20260506 17:56:04.213 | INFO | "huge" : "", | 20260506 17:56:04.213 | INFO | "huge_shared" : " 27 GB", | 20260506 17:56:04.213 | INFO | "malloc" : " 14 GB", | 20260506 17:56:04.213 | INFO | "pinned" : " 29 GB" | 20260506 17:56:04.213 | INFO | } | 20260506 17:56:04.214 | INFO | Perf report for the last 5.14665 seconds: | 20260506 17:56:04.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:56:04.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:04.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:04.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:04.214 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 17:56:04.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:04.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:04.214 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:56:04.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:04.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:04.214 | INFO | Uploaded 36GiB to GPU in 2.00878s (17.9213GiB/s) | 20260506 17:56:04.214 | INFO | Downloaded 34GiB to GPU in 1.94391s (17.4905GiB/s) | 20260506 17:56:09.350 | INFO | Analyzing chunk frames = [3314688,3315200) | 20260506 17:56:09.350 | INFO | MemInfo: { | 20260506 17:56:09.350 | INFO | "gpu" : " 17 GB", | 20260506 17:56:09.350 | INFO | "huge" : "", | 20260506 17:56:09.350 | INFO | "huge_shared" : " 27 GB", | 20260506 17:56:09.350 | INFO | "malloc" : " 14 GB", | 20260506 17:56:09.350 | INFO | "pinned" : " 29 GB" | 20260506 17:56:09.350 | INFO | } | 20260506 17:56:09.350 | INFO | Perf report for the last 5.13613 seconds: | 20260506 17:56:09.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:56:09.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:09.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:09.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:09.350 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 17:56:09.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:09.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:09.350 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:56:09.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:09.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:09.350 | INFO | Uploaded 36GiB to GPU in 1.98514s (18.1348GiB/s) | 20260506 17:56:09.350 | INFO | Downloaded 34GiB to GPU in 1.91807s (17.7261GiB/s) | 20260506 17:56:14.489 | INFO | Analyzing chunk frames = [3315200,3315712) | 20260506 17:56:14.489 | INFO | MemInfo: { | 20260506 17:56:14.489 | INFO | "gpu" : " 17 GB", | 20260506 17:56:14.489 | INFO | "huge" : "", | 20260506 17:56:14.489 | INFO | "huge_shared" : " 27 GB", | 20260506 17:56:14.489 | INFO | "malloc" : " 14 GB", | 20260506 17:56:14.489 | INFO | "pinned" : " 29 GB" | 20260506 17:56:14.489 | INFO | } | 20260506 17:56:14.489 | INFO | Perf report for the last 5.13917 seconds: | 20260506 17:56:14.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:56:14.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:14.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:14.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:14.489 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 17:56:14.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:14.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:14.489 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:56:14.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:14.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:14.489 | INFO | Uploaded 36GiB to GPU in 1.92598s (18.6918GiB/s) | 20260506 17:56:14.489 | INFO | Downloaded 34GiB to GPU in 1.87504s (18.1329GiB/s) | 20260506 17:56:19.626 | INFO | Analyzing chunk frames = [3315712,3316224) | 20260506 17:56:19.626 | INFO | MemInfo: { | 20260506 17:56:19.626 | INFO | "gpu" : " 17 GB", | 20260506 17:56:19.626 | INFO | "huge" : "", | 20260506 17:56:19.626 | INFO | "huge_shared" : " 27 GB", | 20260506 17:56:19.626 | INFO | "malloc" : " 14 GB", | 20260506 17:56:19.626 | INFO | "pinned" : " 29 GB" | 20260506 17:56:19.626 | INFO | } | 20260506 17:56:19.626 | INFO | Perf report for the last 5.13716 seconds: | 20260506 17:56:19.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:56:19.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:19.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:19.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:19.626 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 17:56:19.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:19.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:19.626 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:56:19.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:19.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:19.627 | INFO | Uploaded 36GiB to GPU in 2.01951s (17.8261GiB/s) | 20260506 17:56:19.627 | INFO | Downloaded 34GiB to GPU in 1.91052s (17.7962GiB/s) | 20260506 17:56:24.764 | INFO | Analyzing chunk frames = [3316224,3316736) | 20260506 17:56:24.764 | INFO | MemInfo: { | 20260506 17:56:24.764 | INFO | "gpu" : " 17 GB", | 20260506 17:56:24.764 | INFO | "huge" : "", | 20260506 17:56:24.764 | INFO | "huge_shared" : " 27 GB", | 20260506 17:56:24.764 | INFO | "malloc" : " 14 GB", | 20260506 17:56:24.764 | INFO | "pinned" : " 29 GB" | 20260506 17:56:24.764 | INFO | } | 20260506 17:56:24.764 | INFO | Perf report for the last 5.13816 seconds: | 20260506 17:56:24.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:56:24.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:24.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:24.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:24.764 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 17:56:24.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:24.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:24.764 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:56:24.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:24.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:24.765 | INFO | Uploaded 36GiB to GPU in 2.04235s (17.6268GiB/s) | 20260506 17:56:24.765 | INFO | Downloaded 34GiB to GPU in 1.95278s (17.4111GiB/s) | 20260506 17:56:29.904 | INFO | Analyzing chunk frames = [3316736,3317248) | 20260506 17:56:29.904 | INFO | MemInfo: { | 20260506 17:56:29.904 | INFO | "gpu" : " 17 GB", | 20260506 17:56:29.904 | INFO | "huge" : "", | 20260506 17:56:29.904 | INFO | "huge_shared" : " 27 GB", | 20260506 17:56:29.904 | INFO | "malloc" : " 14 GB", | 20260506 17:56:29.904 | INFO | "pinned" : " 29 GB" | 20260506 17:56:29.904 | INFO | } | 20260506 17:56:29.904 | INFO | Perf report for the last 5.14025 seconds: | 20260506 17:56:29.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:56:29.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:29.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:29.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:29.905 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:56:29.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:29.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:29.905 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:56:29.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:29.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:29.905 | INFO | Uploaded 36GiB to GPU in 2.0095s (17.9149GiB/s) | 20260506 17:56:29.905 | INFO | Downloaded 34GiB to GPU in 1.98968s (17.0882GiB/s) | 20260506 17:56:35.042 | INFO | Analyzing chunk frames = [3317248,3317760) | 20260506 17:56:35.042 | INFO | MemInfo: { | 20260506 17:56:35.042 | INFO | "gpu" : " 17 GB", | 20260506 17:56:35.042 | INFO | "huge" : "", | 20260506 17:56:35.042 | INFO | "huge_shared" : " 27 GB", | 20260506 17:56:35.042 | INFO | "malloc" : " 14 GB", | 20260506 17:56:35.042 | INFO | "pinned" : " 29 GB" | 20260506 17:56:35.042 | INFO | } | 20260506 17:56:35.042 | INFO | Perf report for the last 5.13796 seconds: | 20260506 17:56:35.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:56:35.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:35.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:35.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:35.043 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 17:56:35.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:35.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:35.043 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260506 17:56:35.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:35.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:35.043 | INFO | Uploaded 36GiB to GPU in 2.02501s (17.7777GiB/s) | 20260506 17:56:35.043 | INFO | Downloaded 34GiB to GPU in 1.99501s (17.0426GiB/s) | 20260506 17:56:35.043 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:56:35.043 | INFO | Total mem usage: 18 GiB | 20260506 17:56:35.043 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:56:35.043 | INFO | Recent High Water Marks Sum: 16.7 GiB (Savings: -7.84%) | 20260506 17:56:35.043 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:56:35.043 | INFO | High Water Breakdown: | 20260506 17:56:35.043 | INFO | 62.4% : 10.4 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:56:35.043 | INFO | 37.5% : 6.26 GiB ( 93.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:56:35.043 | INFO | 0.119% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:56:35.043 | INFO | ------------------------------------------ | 20260506 17:56:35.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:56:35.043 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:56:35.043 | INFO | Total mem usage: 26.5 GiB | 20260506 17:56:35.043 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:56:35.043 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:56:35.043 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:56:35.043 | INFO | High Water Breakdown: | 20260506 17:56:35.043 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:56:35.043 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:56:35.043 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:56:35.043 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:56:35.043 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:56:35.043 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:56:35.043 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:56:35.043 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:56:35.043 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:56:35.043 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:56:35.043 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:56:35.043 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:56:35.043 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:56:35.044 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:56:35.044 | INFO | ------------------------------------------ | 20260506 17:56:35.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:56:35.044 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:56:35.044 | INFO | Total mem usage: 25.6 GiB | 20260506 17:56:35.044 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:56:35.044 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:56:35.044 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:56:35.044 | INFO | High Water Breakdown: | 20260506 17:56:35.044 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:56:35.044 | INFO | ------------------------------------------ | 20260506 17:56:35.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:56:35.044 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:56:35.044 | INFO | Total mem usage: 15.7 GiB | 20260506 17:56:35.044 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:56:35.044 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:56:35.044 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:56:35.044 | INFO | High Water Breakdown: | 20260506 17:56:35.044 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:56:35.044 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:56:35.044 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:56:35.044 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:56:35.044 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:56:35.044 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:56:35.044 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:56:35.044 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:56:35.044 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:56:35.044 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:56:35.044 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:56:35.044 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:56:35.044 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:56:35.044 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:56:35.044 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:56:35.044 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:56:35.044 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:56:35.044 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:56:35.044 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:56:35.044 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:56:35.044 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:56:35.044 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:56:35.044 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:56:35.044 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:56:35.044 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:56:35.044 | INFO | ------------------------------------------ | 20260506 17:56:35.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:56:35.045 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:56:35.045 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:56:35.045 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:56:35.045 | INFO | PerfReport: Full Execution -- 3.14e+01 / 7.60e+01, 1.35e+01 / 2.61e+00 | 20260506 17:56:35.045 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 9.87e-02, 9.89e-03 / 5.92e-03 | 20260506 17:56:35.045 | INFO | PerfReport: 11.61%, Baseline -- 3.64e+00 / 4.40e+00, 5.59e-01 / 3.48e-01 | 20260506 17:56:35.045 | INFO | PerfReport: 3.27%, Histogram -- 1.03e+00 / 1.45e+00, 1.86e-01 / 1.55e-01 | 20260506 17:56:35.045 | INFO | PerfReport: 15.99%, DME -- 4.01e+01 / 4.49e+01, 4.20e+00 / 3.22e+00 | 20260506 17:56:35.045 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.46e+00 / 2.04e+00 | 20260506 17:56:35.045 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.26e+00 / 2.76e+00, 3.77e-01 / 2.59e-01 | 20260506 17:56:35.045 | INFO | PerfReport: 3.41%, Metrics -- 1.07e+00 / 1.40e+00, 2.78e-01 / 1.45e-01 | 20260506 17:56:35.045 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.71e-01 / 4.17e-01, 5.74e-02 / 2.55e-02 | 20260506 17:56:35.045 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:56:35.045 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:56:35.045 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:56:35.045 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 17:56:35.045 | INFO | PerfReport: 21.52%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 17:56:35.045 | INFO | PerfReport: 21.17%, Download -- 2.06e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 17:56:35.045 | INFO | PerfReport: 32.29%, Compute -- 3.14e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 17:56:35.045 | INFO | PerfReport: 1.47%, LockUpload -- 1.43e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260506 17:56:35.045 | INFO | PerfReport: 1.08%, LockDownload -- 1.05e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260506 17:56:35.045 | INFO | PerfReport: 22.45%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 17:56:35.045 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.51e-02 / 0.00e+00 | 20260506 17:56:35.161 | INFO | Starting a round of Baz IO | 20260506 17:56:38.310 | INFO | Populating RTMetrics report | 20260506 17:56:38.310 | INFO | RTMetrics report populated | 20260506 17:56:38.310 | INFO | Opening tmp RTMetrics json file | 20260506 17:56:38.324 | INFO | Serializing RTMetrics to json | 20260506 17:56:38.326 | INFO | Serialization done, swapping tmp file now... | 20260506 17:56:38.329 | INFO | RTMetrics done | 20260506 17:56:38.659 | INFO | Finished a round of Baz IO: Wrote 2.051 GiB in 3.498 seconds | 20260506 17:56:38.659 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.35% / 0.01% / 64.65% | 20260506 17:56:38.659 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.23992% / 7.94729e-06% / 7.15256e-05% | 20260506 17:56:38.660 | INFO | Baz IO overall: | 20260506 17:56:38.660 | INFO | BytesWritten : 1751115065038 | 20260506 17:56:38.660 | INFO | NumEvents : 847014420225 | 20260506 17:56:40.180 | INFO | Analyzing chunk frames = [3317760,3318272) | 20260506 17:56:40.180 | INFO | MemInfo: { | 20260506 17:56:40.180 | INFO | "gpu" : " 17 GB", | 20260506 17:56:40.180 | INFO | "huge" : "", | 20260506 17:56:40.180 | INFO | "huge_shared" : " 27 GB", | 20260506 17:56:40.180 | INFO | "malloc" : " 14 GB", | 20260506 17:56:40.180 | INFO | "pinned" : " 29 GB" | 20260506 17:56:40.180 | INFO | } | 20260506 17:56:40.180 | INFO | Perf report for the last 5.13784 seconds: | 20260506 17:56:40.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:56:40.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:40.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:40.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:40.181 | INFO | ANALYSIS : 62.8%, 2.9, 95.6ms, 0 | 20260506 17:56:40.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:40.181 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 17:56:40.181 | INFO | PRE_HQ : 60.5%, 1.9, 58.5ms, 0 | 20260506 17:56:40.181 | INFO | BAZWRITER : 67.7%, 4.8, 167.0ms, 0 | 20260506 17:56:40.181 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 17:56:40.181 | INFO | Uploaded 36GiB to GPU in 2.03196s (17.7169GiB/s) | 20260506 17:56:40.181 | INFO | Downloaded 38GiB to GPU in 2.23588s (16.9956GiB/s) | 20260506 17:56:45.321 | INFO | Analyzing chunk frames = [3318272,3318784) | 20260506 17:56:45.321 | INFO | MemInfo: { | 20260506 17:56:45.321 | INFO | "gpu" : " 17 GB", | 20260506 17:56:45.321 | INFO | "huge" : "", | 20260506 17:56:45.321 | INFO | "huge_shared" : " 27 GB", | 20260506 17:56:45.321 | INFO | "malloc" : " 14 GB", | 20260506 17:56:45.321 | INFO | "pinned" : " 29 GB" | 20260506 17:56:45.321 | INFO | } | 20260506 17:56:45.321 | INFO | Perf report for the last 5.14094 seconds: | 20260506 17:56:45.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:56:45.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:45.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:45.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:45.321 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 17:56:45.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:45.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:45.321 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:56:45.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:45.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:45.322 | INFO | Uploaded 36GiB to GPU in 2.0111s (17.9007GiB/s) | 20260506 17:56:45.322 | INFO | Downloaded 34GiB to GPU in 1.97818s (17.1875GiB/s) | 20260506 17:56:50.463 | INFO | Analyzing chunk frames = [3318784,3319296) | 20260506 17:56:50.463 | INFO | MemInfo: { | 20260506 17:56:50.463 | INFO | "gpu" : " 17 GB", | 20260506 17:56:50.463 | INFO | "huge" : "", | 20260506 17:56:50.463 | INFO | "huge_shared" : " 27 GB", | 20260506 17:56:50.463 | INFO | "malloc" : " 14 GB", | 20260506 17:56:50.463 | INFO | "pinned" : " 29 GB" | 20260506 17:56:50.463 | INFO | } | 20260506 17:56:50.463 | INFO | Perf report for the last 5.14177 seconds: | 20260506 17:56:50.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:56:50.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:50.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:50.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:50.463 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 17:56:50.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:50.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:50.463 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:56:50.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:50.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:50.463 | INFO | Uploaded 36GiB to GPU in 1.96702s (18.3018GiB/s) | 20260506 17:56:50.463 | INFO | Downloaded 34GiB to GPU in 1.92261s (17.6843GiB/s) | 20260506 17:56:55.606 | INFO | Analyzing chunk frames = [3319296,3319808) | 20260506 17:56:55.606 | INFO | MemInfo: { | 20260506 17:56:55.606 | INFO | "gpu" : " 17 GB", | 20260506 17:56:55.606 | INFO | "huge" : "", | 20260506 17:56:55.606 | INFO | "huge_shared" : " 27 GB", | 20260506 17:56:55.606 | INFO | "malloc" : " 14 GB", | 20260506 17:56:55.606 | INFO | "pinned" : " 29 GB" | 20260506 17:56:55.606 | INFO | } | 20260506 17:56:55.607 | INFO | Perf report for the last 5.14346 seconds: | 20260506 17:56:55.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:56:55.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:55.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:55.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:55.607 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 17:56:55.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:55.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:55.607 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:56:55.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:55.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:56:55.607 | INFO | Uploaded 36GiB to GPU in 1.92757s (18.6763GiB/s) | 20260506 17:56:55.607 | INFO | Downloaded 34GiB to GPU in 1.87387s (18.1443GiB/s) | 20260506 17:57:00.743 | INFO | Analyzing chunk frames = [3319808,3320320) | 20260506 17:57:00.743 | INFO | MemInfo: { | 20260506 17:57:00.743 | INFO | "gpu" : " 17 GB", | 20260506 17:57:00.743 | INFO | "huge" : "", | 20260506 17:57:00.743 | INFO | "huge_shared" : " 27 GB", | 20260506 17:57:00.743 | INFO | "malloc" : " 14 GB", | 20260506 17:57:00.743 | INFO | "pinned" : " 29 GB" | 20260506 17:57:00.743 | INFO | } | 20260506 17:57:00.743 | INFO | Perf report for the last 5.13642 seconds: | 20260506 17:57:00.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:57:00.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:00.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:00.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:00.743 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 17:57:00.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:00.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:00.743 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:57:00.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:00.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:00.743 | INFO | Uploaded 36GiB to GPU in 1.98682s (18.1194GiB/s) | 20260506 17:57:00.743 | INFO | Downloaded 34GiB to GPU in 1.9229s (17.6816GiB/s) | 20260506 17:57:05.881 | INFO | Analyzing chunk frames = [3320320,3320832) | 20260506 17:57:05.881 | INFO | MemInfo: { | 20260506 17:57:05.881 | INFO | "gpu" : " 17 GB", | 20260506 17:57:05.881 | INFO | "huge" : "", | 20260506 17:57:05.881 | INFO | "huge_shared" : " 27 GB", | 20260506 17:57:05.882 | INFO | "malloc" : " 14 GB", | 20260506 17:57:05.882 | INFO | "pinned" : " 29 GB" | 20260506 17:57:05.882 | INFO | } | 20260506 17:57:05.882 | INFO | Perf report for the last 5.13871 seconds: | 20260506 17:57:05.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:57:05.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:05.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:05.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:05.882 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 17:57:05.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:05.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:05.882 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 17:57:05.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:05.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:05.882 | INFO | Uploaded 36GiB to GPU in 2.0205s (17.8174GiB/s) | 20260506 17:57:05.882 | INFO | Downloaded 34GiB to GPU in 1.94631s (17.4689GiB/s) | 20260506 17:57:11.020 | INFO | Analyzing chunk frames = [3320832,3321344) | 20260506 17:57:11.020 | INFO | MemInfo: { | 20260506 17:57:11.020 | INFO | "gpu" : " 17 GB", | 20260506 17:57:11.020 | INFO | "huge" : "", | 20260506 17:57:11.020 | INFO | "huge_shared" : " 27 GB", | 20260506 17:57:11.020 | INFO | "malloc" : " 14 GB", | 20260506 17:57:11.020 | INFO | "pinned" : " 29 GB" | 20260506 17:57:11.020 | INFO | } | 20260506 17:57:11.020 | INFO | Perf report for the last 5.13827 seconds: | 20260506 17:57:11.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:57:11.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:11.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:11.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:11.020 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:57:11.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:11.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:11.020 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 17:57:11.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:11.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:11.020 | INFO | Uploaded 36GiB to GPU in 2.0097s (17.9131GiB/s) | 20260506 17:57:11.020 | INFO | Downloaded 34GiB to GPU in 1.99422s (17.0492GiB/s) | 20260506 17:57:16.155 | INFO | Analyzing chunk frames = [3321344,3321856) | 20260506 17:57:16.155 | INFO | MemInfo: { | 20260506 17:57:16.155 | INFO | "gpu" : " 17 GB", | 20260506 17:57:16.155 | INFO | "huge" : "", | 20260506 17:57:16.155 | INFO | "huge_shared" : " 27 GB", | 20260506 17:57:16.155 | INFO | "malloc" : " 14 GB", | 20260506 17:57:16.155 | INFO | "pinned" : " 29 GB" | 20260506 17:57:16.155 | INFO | } | 20260506 17:57:16.155 | INFO | Perf report for the last 5.13484 seconds: | 20260506 17:57:16.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:57:16.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:16.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:16.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:16.155 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:57:16.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:16.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:16.155 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 17:57:16.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:16.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:16.155 | INFO | Uploaded 36GiB to GPU in 1.98309s (18.1535GiB/s) | 20260506 17:57:16.155 | INFO | Downloaded 34GiB to GPU in 1.97878s (17.1823GiB/s) | 20260506 17:57:16.267 | INFO | Starting a round of Baz IO | 20260506 17:57:19.411 | INFO | Populating RTMetrics report | 20260506 17:57:19.411 | INFO | RTMetrics report populated | 20260506 17:57:19.411 | INFO | Opening tmp RTMetrics json file | 20260506 17:57:19.412 | INFO | Serializing RTMetrics to json | 20260506 17:57:19.413 | INFO | Serialization done, swapping tmp file now... | 20260506 17:57:19.414 | INFO | RTMetrics done | 20260506 17:57:19.883 | INFO | Finished a round of Baz IO: Wrote 2.052 GiB in 3.615 seconds | 20260506 17:57:19.883 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.34% / 0.01% / 64.65% | 20260506 17:57:19.883 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.24005% / 1.19209e-05% / 5.5631e-05% | 20260506 17:57:19.883 | INFO | Baz IO overall: | 20260506 17:57:19.883 | INFO | BytesWritten : 1753318721701 | 20260506 17:57:19.883 | INFO | NumEvents : 848077104961 | 20260506 17:57:21.296 | INFO | Analyzing chunk frames = [3321856,3322368) | 20260506 17:57:21.296 | INFO | MemInfo: { | 20260506 17:57:21.296 | INFO | "gpu" : " 17 GB", | 20260506 17:57:21.296 | INFO | "huge" : "", | 20260506 17:57:21.296 | INFO | "huge_shared" : " 27 GB", | 20260506 17:57:21.297 | INFO | "malloc" : " 14 GB", | 20260506 17:57:21.297 | INFO | "pinned" : " 29 GB" | 20260506 17:57:21.297 | INFO | } | 20260506 17:57:21.297 | INFO | Perf report for the last 5.1418 seconds: | 20260506 17:57:21.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:57:21.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:21.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:21.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:21.297 | INFO | ANALYSIS : 62.7%, 2.9, 95.6ms, 0 | 20260506 17:57:21.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:21.297 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260506 17:57:21.297 | INFO | PRE_HQ : 60.2%, 1.9, 57.9ms, 0 | 20260506 17:57:21.297 | INFO | BAZWRITER : 69.9%, 4.8, 172.7ms, 0 | 20260506 17:57:21.297 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 17:57:21.297 | INFO | Uploaded 36GiB to GPU in 2.05943s (17.4806GiB/s) | 20260506 17:57:21.297 | INFO | Downloaded 38GiB to GPU in 2.21219s (17.1775GiB/s) | 20260506 17:57:26.437 | INFO | Analyzing chunk frames = [3322368,3322880) | 20260506 17:57:26.437 | INFO | MemInfo: { | 20260506 17:57:26.437 | INFO | "gpu" : " 17 GB", | 20260506 17:57:26.437 | INFO | "huge" : "", | 20260506 17:57:26.437 | INFO | "huge_shared" : " 27 GB", | 20260506 17:57:26.437 | INFO | "malloc" : " 14 GB", | 20260506 17:57:26.437 | INFO | "pinned" : " 29 GB" | 20260506 17:57:26.437 | INFO | } | 20260506 17:57:26.437 | INFO | Perf report for the last 5.14012 seconds: | 20260506 17:57:26.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:57:26.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:26.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:26.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:26.437 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 17:57:26.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:26.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:26.437 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:57:26.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:26.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:26.437 | INFO | Uploaded 36GiB to GPU in 1.99518s (18.0435GiB/s) | 20260506 17:57:26.437 | INFO | Downloaded 34GiB to GPU in 1.91942s (17.7136GiB/s) | 20260506 17:57:31.579 | INFO | Analyzing chunk frames = [3322880,3323392) | 20260506 17:57:31.579 | INFO | MemInfo: { | 20260506 17:57:31.579 | INFO | "gpu" : " 17 GB", | 20260506 17:57:31.579 | INFO | "huge" : "", | 20260506 17:57:31.579 | INFO | "huge_shared" : " 27 GB", | 20260506 17:57:31.579 | INFO | "malloc" : " 14 GB", | 20260506 17:57:31.579 | INFO | "pinned" : " 29 GB" | 20260506 17:57:31.579 | INFO | } | 20260506 17:57:31.579 | INFO | Perf report for the last 5.14254 seconds: | 20260506 17:57:31.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:57:31.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:31.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:31.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:31.580 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 17:57:31.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:31.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:31.580 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:57:31.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:31.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:31.580 | INFO | Uploaded 36GiB to GPU in 1.9728s (18.2482GiB/s) | 20260506 17:57:31.580 | INFO | Downloaded 34GiB to GPU in 1.92374s (17.6739GiB/s) | 20260506 17:57:36.725 | INFO | Analyzing chunk frames = [3323392,3323904) | 20260506 17:57:36.725 | INFO | MemInfo: { | 20260506 17:57:36.725 | INFO | "gpu" : " 17 GB", | 20260506 17:57:36.725 | INFO | "huge" : "", | 20260506 17:57:36.725 | INFO | "huge_shared" : " 27 GB", | 20260506 17:57:36.725 | INFO | "malloc" : " 14 GB", | 20260506 17:57:36.725 | INFO | "pinned" : " 29 GB" | 20260506 17:57:36.725 | INFO | } | 20260506 17:57:36.725 | INFO | Perf report for the last 5.14587 seconds: | 20260506 17:57:36.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:57:36.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:36.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:36.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:36.725 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 17:57:36.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:36.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:36.725 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:57:36.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:36.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:36.726 | INFO | Uploaded 36GiB to GPU in 1.93506s (18.6041GiB/s) | 20260506 17:57:36.726 | INFO | Downloaded 34GiB to GPU in 1.86766s (18.2046GiB/s) | 20260506 17:57:41.859 | INFO | Analyzing chunk frames = [3323904,3324416) | 20260506 17:57:41.859 | INFO | MemInfo: { | 20260506 17:57:41.859 | INFO | "gpu" : " 17 GB", | 20260506 17:57:41.859 | INFO | "huge" : "", | 20260506 17:57:41.859 | INFO | "huge_shared" : " 27 GB", | 20260506 17:57:41.859 | INFO | "malloc" : " 14 GB", | 20260506 17:57:41.859 | INFO | "pinned" : " 29 GB" | 20260506 17:57:41.859 | INFO | } | 20260506 17:57:41.859 | INFO | Perf report for the last 5.13365 seconds: | 20260506 17:57:41.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:57:41.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:41.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:41.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:41.859 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 17:57:41.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:41.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:41.859 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:57:41.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:41.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:41.859 | INFO | Uploaded 36GiB to GPU in 2.01773s (17.8418GiB/s) | 20260506 17:57:41.859 | INFO | Downloaded 34GiB to GPU in 1.90775s (17.822GiB/s) | 20260506 17:57:46.994 | INFO | Analyzing chunk frames = [3324416,3324928) | 20260506 17:57:46.994 | INFO | MemInfo: { | 20260506 17:57:46.994 | INFO | "gpu" : " 17 GB", | 20260506 17:57:46.994 | INFO | "huge" : "", | 20260506 17:57:46.994 | INFO | "huge_shared" : " 27 GB", | 20260506 17:57:46.994 | INFO | "malloc" : " 14 GB", | 20260506 17:57:46.994 | INFO | "pinned" : " 29 GB" | 20260506 17:57:46.994 | INFO | } | 20260506 17:57:46.994 | INFO | Perf report for the last 5.13496 seconds: | 20260506 17:57:46.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:57:46.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:46.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:46.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:46.994 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:57:46.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:46.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:46.994 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:57:46.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:46.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:46.994 | INFO | Uploaded 36GiB to GPU in 2.07125s (17.3808GiB/s) | 20260506 17:57:46.994 | INFO | Downloaded 34GiB to GPU in 1.96251s (17.3247GiB/s) | 20260506 17:57:52.131 | INFO | Analyzing chunk frames = [3324928,3325440) | 20260506 17:57:52.131 | INFO | MemInfo: { | 20260506 17:57:52.131 | INFO | "gpu" : " 17 GB", | 20260506 17:57:52.131 | INFO | "huge" : "", | 20260506 17:57:52.131 | INFO | "huge_shared" : " 27 GB", | 20260506 17:57:52.131 | INFO | "malloc" : " 14 GB", | 20260506 17:57:52.131 | INFO | "pinned" : " 29 GB" | 20260506 17:57:52.131 | INFO | } | 20260506 17:57:52.131 | INFO | Perf report for the last 5.137 seconds: | 20260506 17:57:52.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:57:52.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:52.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:52.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:52.131 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 17:57:52.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:52.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:52.131 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 17:57:52.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:52.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:52.131 | INFO | Uploaded 36GiB to GPU in 2.02218s (17.8026GiB/s) | 20260506 17:57:52.131 | INFO | Downloaded 34GiB to GPU in 2.00844s (16.9285GiB/s) | 20260506 17:57:57.268 | INFO | Analyzing chunk frames = [3325440,3325952) | 20260506 17:57:57.268 | INFO | MemInfo: { | 20260506 17:57:57.268 | INFO | "gpu" : " 17 GB", | 20260506 17:57:57.268 | INFO | "huge" : "", | 20260506 17:57:57.268 | INFO | "huge_shared" : " 27 GB", | 20260506 17:57:57.268 | INFO | "malloc" : " 14 GB", | 20260506 17:57:57.268 | INFO | "pinned" : " 29 GB" | 20260506 17:57:57.268 | INFO | } | 20260506 17:57:57.268 | INFO | Perf report for the last 5.13755 seconds: | 20260506 17:57:57.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:57:57.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:57.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:57.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:57.268 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 17:57:57.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:57.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:57.268 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:57:57.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:57.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:57:57.269 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:57:57.269 | INFO | Total mem usage: 18 GiB | 20260506 17:57:57.269 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:57:57.269 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -13.8%) | 20260506 17:57:57.269 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:57:57.269 | INFO | High Water Breakdown: | 20260506 17:57:57.269 | INFO | 61.6% : 9.74 GiB ( 85.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:57:57.269 | INFO | 38.3% : 6.05 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:57:57.269 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:57:57.269 | INFO | ------------------------------------------ | 20260506 17:57:57.269 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:57:57.269 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:57:57.269 | INFO | Total mem usage: 26.5 GiB | 20260506 17:57:57.269 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:57:57.269 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:57:57.269 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:57:57.269 | INFO | High Water Breakdown: | 20260506 17:57:57.269 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:57:57.269 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:57:57.269 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:57:57.269 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:57:57.269 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:57:57.269 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:57:57.269 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:57:57.269 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:57:57.269 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:57:57.269 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:57:57.269 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:57:57.269 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:57:57.269 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:57:57.269 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:57:57.269 | INFO | ------------------------------------------ | 20260506 17:57:57.269 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:57:57.269 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:57:57.269 | INFO | Total mem usage: 25.6 GiB | 20260506 17:57:57.269 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:57:57.269 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:57:57.269 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:57:57.269 | INFO | High Water Breakdown: | 20260506 17:57:57.269 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:57:57.269 | INFO | ------------------------------------------ | 20260506 17:57:57.269 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:57:57.269 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:57:57.269 | INFO | Total mem usage: 15.7 GiB | 20260506 17:57:57.269 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:57:57.269 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:57:57.269 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:57:57.269 | INFO | High Water Breakdown: | 20260506 17:57:57.269 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:57:57.269 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:57:57.269 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:57:57.269 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:57:57.269 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:57:57.269 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:57:57.269 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:57:57.269 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:57:57.269 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:57:57.269 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:57:57.269 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:57:57.269 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:57:57.269 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:57:57.269 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:57:57.269 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:57:57.269 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:57:57.269 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:57:57.269 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:57:57.269 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:57:57.269 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:57:57.269 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:57:57.269 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:57:57.269 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:57:57.269 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:57:57.269 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:57:57.269 | INFO | ------------------------------------------ | 20260506 17:57:57.269 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:57:57.269 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:57:57.269 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:57:57.269 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:57:57.269 | INFO | Uploaded 36GiB to GPU in 1.99981s (18.0017GiB/s) | 20260506 17:57:57.269 | INFO | Downloaded 34GiB to GPU in 1.96931s (17.265GiB/s) | 20260506 17:57:57.269 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.59e+01, 1.35e+01 / 2.34e+00 | 20260506 17:57:57.269 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 9.84e-03 / 6.35e-03 | 20260506 17:57:57.269 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.47e+00, 5.54e-01 / 2.80e-01 | 20260506 17:57:57.270 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.81e-01 / 1.22e-01 | 20260506 17:57:57.270 | INFO | PerfReport: 15.95%, DME -- 3.99e+01 / 4.46e+01, 4.01e+00 / 2.48e+00 | 20260506 17:57:57.270 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.39e+00 / 2.12e+00 | 20260506 17:57:57.270 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.27e+00 / 2.78e+00, 3.88e-01 / 2.80e-01 | 20260506 17:57:57.270 | INFO | PerfReport: 3.43%, Metrics -- 1.07e+00 / 1.45e+00, 2.80e-01 / 1.99e-01 | 20260506 17:57:57.270 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.71e-01 / 4.20e-01, 5.79e-02 / 2.30e-02 | 20260506 17:57:57.270 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:57:57.270 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:57:57.270 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:57:57.270 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 17:57:57.270 | INFO | PerfReport: 21.44%, Upload -- 2.08e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260506 17:57:57.270 | INFO | PerfReport: 21.18%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 17:57:57.270 | INFO | PerfReport: 32.18%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 17:57:57.270 | INFO | PerfReport: 1.42%, LockUpload -- 1.38e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260506 17:57:57.270 | INFO | PerfReport: 1.17%, LockDownload -- 1.14e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 17:57:57.270 | INFO | PerfReport: 22.58%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 17:57:57.270 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.26e-02 / 0.00e+00 | 20260506 17:57:57.383 | INFO | Starting a round of Baz IO | 20260506 17:58:00.522 | INFO | Populating RTMetrics report | 20260506 17:58:00.522 | INFO | RTMetrics report populated | 20260506 17:58:00.522 | INFO | Opening tmp RTMetrics json file | 20260506 17:58:00.679 | INFO | Serializing RTMetrics to json | 20260506 17:58:00.721 | INFO | Serialization done, swapping tmp file now... | 20260506 17:58:00.725 | INFO | RTMetrics done | 20260506 17:58:00.750 | INFO | Finished a round of Baz IO: Wrote 2.055 GiB in 3.367 seconds | 20260506 17:58:00.750 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.33% / 0.01% / 64.66% | 20260506 17:58:00.750 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.2409% / 0% / 7.54992e-05% | 20260506 17:58:00.750 | INFO | Baz IO overall: | 20260506 17:58:00.750 | INFO | BytesWritten : 1755525118103 | 20260506 17:58:00.750 | INFO | NumEvents : 849141929246 | 20260506 17:58:02.412 | INFO | Analyzing chunk frames = [3325952,3326464) | 20260506 17:58:02.412 | INFO | MemInfo: { | 20260506 17:58:02.412 | INFO | "gpu" : " 17 GB", | 20260506 17:58:02.412 | INFO | "huge" : "", | 20260506 17:58:02.412 | INFO | "huge_shared" : " 27 GB", | 20260506 17:58:02.412 | INFO | "malloc" : " 14 GB", | 20260506 17:58:02.412 | INFO | "pinned" : " 29 GB" | 20260506 17:58:02.412 | INFO | } | 20260506 17:58:02.412 | INFO | Perf report for the last 5.14332 seconds: | 20260506 17:58:02.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:58:02.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:02.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:02.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:02.412 | INFO | ANALYSIS : 62.7%, 2.9, 95.4ms, 0 | 20260506 17:58:02.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:02.412 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260506 17:58:02.412 | INFO | PRE_HQ : 60.3%, 1.9, 58.1ms, 0 | 20260506 17:58:02.412 | INFO | BAZWRITER : 65.1%, 4.8, 160.3ms, 0 | 20260506 17:58:02.412 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.4ms, 0 | 20260506 17:58:02.412 | INFO | Uploaded 36GiB to GPU in 2.05947s (17.4802GiB/s) | 20260506 17:58:02.412 | INFO | Downloaded 38GiB to GPU in 2.21397s (17.1638GiB/s) | 20260506 17:58:07.554 | INFO | Analyzing chunk frames = [3326464,3326976) | 20260506 17:58:07.554 | INFO | MemInfo: { | 20260506 17:58:07.554 | INFO | "gpu" : " 17 GB", | 20260506 17:58:07.554 | INFO | "huge" : "", | 20260506 17:58:07.554 | INFO | "huge_shared" : " 27 GB", | 20260506 17:58:07.554 | INFO | "malloc" : " 14 GB", | 20260506 17:58:07.554 | INFO | "pinned" : " 29 GB" | 20260506 17:58:07.554 | INFO | } | 20260506 17:58:07.555 | INFO | Perf report for the last 5.14279 seconds: | 20260506 17:58:07.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:58:07.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:07.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:07.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:07.555 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 17:58:07.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:07.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:07.555 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 17:58:07.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:07.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:07.555 | INFO | Uploaded 36GiB to GPU in 1.99864s (18.0122GiB/s) | 20260506 17:58:07.555 | INFO | Downloaded 34GiB to GPU in 1.93471s (17.5737GiB/s) | 20260506 17:58:12.697 | INFO | Analyzing chunk frames = [3326976,3327488) | 20260506 17:58:12.697 | INFO | MemInfo: { | 20260506 17:58:12.697 | INFO | "gpu" : " 17 GB", | 20260506 17:58:12.697 | INFO | "huge" : "", | 20260506 17:58:12.697 | INFO | "huge_shared" : " 27 GB", | 20260506 17:58:12.697 | INFO | "malloc" : " 14 GB", | 20260506 17:58:12.697 | INFO | "pinned" : " 29 GB" | 20260506 17:58:12.697 | INFO | } | 20260506 17:58:12.697 | INFO | Perf report for the last 5.14261 seconds: | 20260506 17:58:12.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:58:12.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:12.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:12.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:12.697 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 17:58:12.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:12.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:12.697 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 17:58:12.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:12.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:12.697 | INFO | Uploaded 36GiB to GPU in 1.98235s (18.1603GiB/s) | 20260506 17:58:12.697 | INFO | Downloaded 34GiB to GPU in 1.91738s (17.7325GiB/s) | 20260506 17:58:17.835 | INFO | Analyzing chunk frames = [3327488,3328000) | 20260506 17:58:17.835 | INFO | MemInfo: { | 20260506 17:58:17.835 | INFO | "gpu" : " 17 GB", | 20260506 17:58:17.835 | INFO | "huge" : "", | 20260506 17:58:17.835 | INFO | "huge_shared" : " 27 GB", | 20260506 17:58:17.835 | INFO | "malloc" : " 14 GB", | 20260506 17:58:17.835 | INFO | "pinned" : " 29 GB" | 20260506 17:58:17.835 | INFO | } | 20260506 17:58:17.835 | INFO | Perf report for the last 5.13774 seconds: | 20260506 17:58:17.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:58:17.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:17.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:17.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:17.835 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 17:58:17.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:17.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:17.835 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 17:58:17.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:17.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:17.835 | INFO | Uploaded 36GiB to GPU in 1.94518s (18.5073GiB/s) | 20260506 17:58:17.835 | INFO | Downloaded 34GiB to GPU in 1.87053s (18.1767GiB/s) | 20260506 17:58:22.975 | INFO | Analyzing chunk frames = [3328000,3328512) | 20260506 17:58:22.975 | INFO | MemInfo: { | 20260506 17:58:22.975 | INFO | "gpu" : " 17 GB", | 20260506 17:58:22.975 | INFO | "huge" : "", | 20260506 17:58:22.975 | INFO | "huge_shared" : " 27 GB", | 20260506 17:58:22.975 | INFO | "malloc" : " 14 GB", | 20260506 17:58:22.975 | INFO | "pinned" : " 29 GB" | 20260506 17:58:22.975 | INFO | } | 20260506 17:58:22.975 | INFO | Perf report for the last 5.14002 seconds: | 20260506 17:58:22.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:58:22.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:22.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:22.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:22.975 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 17:58:22.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:22.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:22.975 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 17:58:22.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:22.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:22.975 | INFO | Uploaded 36GiB to GPU in 2.02068s (17.8158GiB/s) | 20260506 17:58:22.975 | INFO | Downloaded 34GiB to GPU in 1.90863s (17.8138GiB/s) | 20260506 17:58:28.107 | INFO | Analyzing chunk frames = [3328512,3329024) | 20260506 17:58:28.107 | INFO | MemInfo: { | 20260506 17:58:28.107 | INFO | "gpu" : " 17 GB", | 20260506 17:58:28.107 | INFO | "huge" : "", | 20260506 17:58:28.107 | INFO | "huge_shared" : " 27 GB", | 20260506 17:58:28.107 | INFO | "malloc" : " 14 GB", | 20260506 17:58:28.107 | INFO | "pinned" : " 29 GB" | 20260506 17:58:28.107 | INFO | } | 20260506 17:58:28.107 | INFO | Perf report for the last 5.13252 seconds: | 20260506 17:58:28.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:58:28.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:28.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:28.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:28.108 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 17:58:28.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:28.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:28.108 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 17:58:28.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:28.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:28.108 | INFO | Uploaded 36GiB to GPU in 2.07504s (17.349GiB/s) | 20260506 17:58:28.108 | INFO | Downloaded 34GiB to GPU in 1.96267s (17.3234GiB/s) | 20260506 17:58:33.245 | INFO | Analyzing chunk frames = [3329024,3329536) | 20260506 17:58:33.245 | INFO | MemInfo: { | 20260506 17:58:33.245 | INFO | "gpu" : " 17 GB", | 20260506 17:58:33.245 | INFO | "huge" : "", | 20260506 17:58:33.245 | INFO | "huge_shared" : " 27 GB", | 20260506 17:58:33.245 | INFO | "malloc" : " 14 GB", | 20260506 17:58:33.245 | INFO | "pinned" : " 29 GB" | 20260506 17:58:33.245 | INFO | } | 20260506 17:58:33.245 | INFO | Perf report for the last 5.1378 seconds: | 20260506 17:58:33.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:58:33.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:33.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:33.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:33.245 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 17:58:33.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:33.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:33.245 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260506 17:58:33.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:33.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:33.246 | INFO | Uploaded 36GiB to GPU in 2.00569s (17.9489GiB/s) | 20260506 17:58:33.246 | INFO | Downloaded 34GiB to GPU in 1.9789s (17.1813GiB/s) | 20260506 17:58:38.389 | INFO | Analyzing chunk frames = [3329536,3330048) | 20260506 17:58:38.389 | INFO | MemInfo: { | 20260506 17:58:38.389 | INFO | "gpu" : " 17 GB", | 20260506 17:58:38.389 | INFO | "huge" : "", | 20260506 17:58:38.389 | INFO | "huge_shared" : " 27 GB", | 20260506 17:58:38.390 | INFO | "malloc" : " 14 GB", | 20260506 17:58:38.390 | INFO | "pinned" : " 29 GB" | 20260506 17:58:38.390 | INFO | } | 20260506 17:58:38.390 | INFO | Perf report for the last 5.14433 seconds: | 20260506 17:58:38.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:58:38.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:38.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:38.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:38.390 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 17:58:38.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:38.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:38.390 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260506 17:58:38.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:38.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:38.390 | INFO | Uploaded 36GiB to GPU in 1.98971s (18.0931GiB/s) | 20260506 17:58:38.390 | INFO | Downloaded 34GiB to GPU in 1.98793s (17.1032GiB/s) | 20260506 17:58:38.505 | INFO | Starting a round of Baz IO | 20260506 17:58:41.639 | INFO | Populating RTMetrics report | 20260506 17:58:41.639 | INFO | RTMetrics report populated | 20260506 17:58:41.639 | INFO | Opening tmp RTMetrics json file | 20260506 17:58:41.659 | INFO | Serializing RTMetrics to json | 20260506 17:58:41.666 | INFO | Serialization done, swapping tmp file now... | 20260506 17:58:41.671 | INFO | RTMetrics done | 20260506 17:58:42.117 | INFO | Finished a round of Baz IO: Wrote 2.052 GiB in 3.611 seconds | 20260506 17:58:42.117 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.33% / 0.01% / 64.67% | 20260506 17:58:42.117 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.23715% / 0% / 5.5631e-05% | 20260506 17:58:42.117 | INFO | Baz IO overall: | 20260506 17:58:42.117 | INFO | BytesWritten : 1757728561597 | 20260506 17:58:42.117 | INFO | NumEvents : 850204519612 | 20260506 17:58:43.531 | INFO | Analyzing chunk frames = [3330048,3330560) | 20260506 17:58:43.531 | INFO | MemInfo: { | 20260506 17:58:43.531 | INFO | "gpu" : " 17 GB", | 20260506 17:58:43.531 | INFO | "huge" : "", | 20260506 17:58:43.531 | INFO | "huge_shared" : " 27 GB", | 20260506 17:58:43.531 | INFO | "malloc" : " 14 GB", | 20260506 17:58:43.531 | INFO | "pinned" : " 29 GB" | 20260506 17:58:43.531 | INFO | } | 20260506 17:58:43.531 | INFO | Perf report for the last 5.14141 seconds: | 20260506 17:58:43.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:58:43.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:43.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:43.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:43.531 | INFO | ANALYSIS : 62.6%, 2.9, 95.4ms, 0 | 20260506 17:58:43.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:43.531 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 17:58:43.531 | INFO | PRE_HQ : 60.0%, 1.9, 57.8ms, 0 | 20260506 17:58:43.531 | INFO | BAZWRITER : 70.0%, 4.8, 172.9ms, 0 | 20260506 17:58:43.531 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 17:58:43.531 | INFO | Uploaded 36GiB to GPU in 2.01107s (17.9009GiB/s) | 20260506 17:58:43.532 | INFO | Downloaded 38GiB to GPU in 2.21286s (17.1723GiB/s) | 20260506 17:58:48.670 | INFO | Analyzing chunk frames = [3330560,3331072) | 20260506 17:58:48.670 | INFO | MemInfo: { | 20260506 17:58:48.670 | INFO | "gpu" : " 17 GB", | 20260506 17:58:48.670 | INFO | "huge" : "", | 20260506 17:58:48.670 | INFO | "huge_shared" : " 27 GB", | 20260506 17:58:48.670 | INFO | "malloc" : " 14 GB", | 20260506 17:58:48.670 | INFO | "pinned" : " 29 GB" | 20260506 17:58:48.670 | INFO | } | 20260506 17:58:48.670 | INFO | Perf report for the last 5.13894 seconds: | 20260506 17:58:48.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:58:48.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:48.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:48.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:48.670 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 17:58:48.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:48.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:48.670 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:58:48.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:48.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:48.670 | INFO | Uploaded 36GiB to GPU in 1.99651s (18.0315GiB/s) | 20260506 17:58:48.670 | INFO | Downloaded 34GiB to GPU in 1.92371s (17.6741GiB/s) | 20260506 17:58:53.810 | INFO | Analyzing chunk frames = [3331072,3331584) | 20260506 17:58:53.810 | INFO | MemInfo: { | 20260506 17:58:53.810 | INFO | "gpu" : " 17 GB", | 20260506 17:58:53.810 | INFO | "huge" : "", | 20260506 17:58:53.810 | INFO | "huge_shared" : " 27 GB", | 20260506 17:58:53.810 | INFO | "malloc" : " 14 GB", | 20260506 17:58:53.810 | INFO | "pinned" : " 29 GB" | 20260506 17:58:53.810 | INFO | } | 20260506 17:58:53.810 | INFO | Perf report for the last 5.13975 seconds: | 20260506 17:58:53.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:58:53.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:53.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:53.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:53.810 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 17:58:53.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:53.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:53.810 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 17:58:53.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:53.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:53.810 | INFO | Uploaded 36GiB to GPU in 1.97437s (18.2337GiB/s) | 20260506 17:58:53.810 | INFO | Downloaded 34GiB to GPU in 1.92439s (17.6679GiB/s) | 20260506 17:58:58.944 | INFO | Analyzing chunk frames = [3331584,3332096) | 20260506 17:58:58.944 | INFO | MemInfo: { | 20260506 17:58:58.944 | INFO | "gpu" : " 17 GB", | 20260506 17:58:58.944 | INFO | "huge" : "", | 20260506 17:58:58.944 | INFO | "huge_shared" : " 27 GB", | 20260506 17:58:58.944 | INFO | "malloc" : " 14 GB", | 20260506 17:58:58.944 | INFO | "pinned" : " 29 GB" | 20260506 17:58:58.944 | INFO | } | 20260506 17:58:58.944 | INFO | Perf report for the last 5.13471 seconds: | 20260506 17:58:58.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:58:58.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:58.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:58.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:58.945 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 17:58:58.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:58.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:58.945 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:58:58.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:58.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:58:58.945 | INFO | Uploaded 36GiB to GPU in 1.92641s (18.6876GiB/s) | 20260506 17:58:58.945 | INFO | Downloaded 34GiB to GPU in 1.86767s (18.2045GiB/s) | 20260506 17:59:04.083 | INFO | Analyzing chunk frames = [3332096,3332608) | 20260506 17:59:04.083 | INFO | MemInfo: { | 20260506 17:59:04.083 | INFO | "gpu" : " 17 GB", | 20260506 17:59:04.083 | INFO | "huge" : "", | 20260506 17:59:04.084 | INFO | "huge_shared" : " 27 GB", | 20260506 17:59:04.084 | INFO | "malloc" : " 14 GB", | 20260506 17:59:04.084 | INFO | "pinned" : " 29 GB" | 20260506 17:59:04.084 | INFO | } | 20260506 17:59:04.084 | INFO | Perf report for the last 5.13912 seconds: | 20260506 17:59:04.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:59:04.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:04.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:04.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:04.084 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 17:59:04.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:04.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:04.084 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 17:59:04.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:04.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:04.084 | INFO | Uploaded 36GiB to GPU in 2.01553s (17.8613GiB/s) | 20260506 17:59:04.084 | INFO | Downloaded 34GiB to GPU in 1.91661s (17.7397GiB/s) | 20260506 17:59:09.223 | INFO | Analyzing chunk frames = [3332608,3333120) | 20260506 17:59:09.223 | INFO | MemInfo: { | 20260506 17:59:09.223 | INFO | "gpu" : " 17 GB", | 20260506 17:59:09.223 | INFO | "huge" : "", | 20260506 17:59:09.223 | INFO | "huge_shared" : " 27 GB", | 20260506 17:59:09.223 | INFO | "malloc" : " 14 GB", | 20260506 17:59:09.223 | INFO | "pinned" : " 29 GB" | 20260506 17:59:09.223 | INFO | } | 20260506 17:59:09.223 | INFO | Perf report for the last 5.13942 seconds: | 20260506 17:59:09.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:59:09.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:09.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:09.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:09.223 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 17:59:09.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:09.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:09.223 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:59:09.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:09.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:09.223 | INFO | Uploaded 36GiB to GPU in 2.02363s (17.7898GiB/s) | 20260506 17:59:09.223 | INFO | Downloaded 34GiB to GPU in 1.95114s (17.4257GiB/s) | 20260506 17:59:14.364 | INFO | Analyzing chunk frames = [3333120,3333632) | 20260506 17:59:14.364 | INFO | MemInfo: { | 20260506 17:59:14.364 | INFO | "gpu" : " 17 GB", | 20260506 17:59:14.364 | INFO | "huge" : "", | 20260506 17:59:14.364 | INFO | "huge_shared" : " 27 GB", | 20260506 17:59:14.364 | INFO | "malloc" : " 14 GB", | 20260506 17:59:14.364 | INFO | "pinned" : " 29 GB" | 20260506 17:59:14.364 | INFO | } | 20260506 17:59:14.364 | INFO | Perf report for the last 5.14068 seconds: | 20260506 17:59:14.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:59:14.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:14.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:14.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:14.364 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:59:14.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:14.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:14.364 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:59:14.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:14.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:14.364 | INFO | Uploaded 36GiB to GPU in 1.99455s (18.0492GiB/s) | 20260506 17:59:14.364 | INFO | Downloaded 34GiB to GPU in 2.00634s (16.9463GiB/s) | 20260506 17:59:19.504 | INFO | Analyzing chunk frames = [3333632,3334144) | 20260506 17:59:19.504 | INFO | MemInfo: { | 20260506 17:59:19.504 | INFO | "gpu" : " 17 GB", | 20260506 17:59:19.504 | INFO | "huge" : "", | 20260506 17:59:19.504 | INFO | "huge_shared" : " 27 GB", | 20260506 17:59:19.504 | INFO | "malloc" : " 14 GB", | 20260506 17:59:19.504 | INFO | "pinned" : " 29 GB" | 20260506 17:59:19.504 | INFO | } | 20260506 17:59:19.504 | INFO | Perf report for the last 5.14048 seconds: | 20260506 17:59:19.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:59:19.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:19.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:19.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:19.504 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 17:59:19.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:19.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:19.504 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 17:59:19.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:19.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:19.504 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 17:59:19.504 | INFO | Total mem usage: 18 GiB | 20260506 17:59:19.504 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 17:59:19.504 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.5%) | 20260506 17:59:19.504 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 17:59:19.504 | INFO | High Water Breakdown: | 20260506 17:59:19.504 | INFO | 61.4% : 9.56 GiB ( 83.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 17:59:19.504 | INFO | 38.5% : 6 GiB ( 89.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 17:59:19.504 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 17:59:19.504 | INFO | ------------------------------------------ | 20260506 17:59:19.504 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:59:19.504 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 17:59:19.504 | INFO | Total mem usage: 26.5 GiB | 20260506 17:59:19.504 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 17:59:19.505 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 17:59:19.505 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 17:59:19.505 | INFO | High Water Breakdown: | 20260506 17:59:19.505 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:59:19.505 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:59:19.505 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:59:19.505 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:59:19.505 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:59:19.505 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:59:19.505 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 17:59:19.505 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 17:59:19.505 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:59:19.505 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 17:59:19.505 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 17:59:19.505 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 17:59:19.505 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 17:59:19.505 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 17:59:19.505 | INFO | ------------------------------------------ | 20260506 17:59:19.505 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:59:19.505 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 17:59:19.505 | INFO | Total mem usage: 25.6 GiB | 20260506 17:59:19.505 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 17:59:19.505 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 17:59:19.505 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 17:59:19.505 | INFO | High Water Breakdown: | 20260506 17:59:19.505 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 17:59:19.505 | INFO | ------------------------------------------ | 20260506 17:59:19.505 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:59:19.505 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 17:59:19.505 | INFO | Total mem usage: 15.7 GiB | 20260506 17:59:19.505 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 17:59:19.505 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 17:59:19.505 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 17:59:19.505 | INFO | High Water Breakdown: | 20260506 17:59:19.505 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 17:59:19.505 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 17:59:19.505 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 17:59:19.505 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 17:59:19.505 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 17:59:19.505 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 17:59:19.505 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 17:59:19.505 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 17:59:19.505 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 17:59:19.505 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 17:59:19.505 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 17:59:19.505 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 17:59:19.505 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 17:59:19.505 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 17:59:19.505 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 17:59:19.505 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 17:59:19.505 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 17:59:19.505 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 17:59:19.505 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 17:59:19.505 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 17:59:19.505 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 17:59:19.505 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 17:59:19.505 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 17:59:19.505 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 17:59:19.505 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 17:59:19.505 | INFO | ------------------------------------------ | 20260506 17:59:19.505 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 17:59:19.505 | INFO | Uploaded 36GiB to GPU in 2.03785s (17.6657GiB/s) | 20260506 17:59:19.505 | INFO | Downloaded 34GiB to GPU in 1.99167s (17.0711GiB/s) | 20260506 17:59:19.505 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 17:59:19.505 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:59:19.505 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:59:19.505 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.61e+01, 1.34e+01 / 2.30e+00 | 20260506 17:59:19.505 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.02e-01, 1.04e-02 / 6.65e-03 | 20260506 17:59:19.505 | INFO | PerfReport: 11.61%, Baseline -- 3.64e+00 / 4.42e+00, 5.58e-01 / 3.25e-01 | 20260506 17:59:19.505 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.85e-01 / 1.27e-01 | 20260506 17:59:19.505 | INFO | PerfReport: 15.95%, DME -- 4.00e+01 / 4.47e+01, 3.96e+00 / 2.38e+00 | 20260506 17:59:19.505 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.46e+00 / 1.49e+00 | 20260506 17:59:19.505 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.26e+00 / 2.82e+00, 3.76e-01 / 1.67e-01 | 20260506 17:59:19.505 | INFO | PerfReport: 3.43%, Metrics -- 1.08e+00 / 1.41e+00, 2.85e-01 / 1.76e-01 | 20260506 17:59:19.505 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.71e-01 / 4.25e-01, 5.86e-02 / 2.45e-02 | 20260506 17:59:19.505 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 17:59:19.505 | INFO | PerfReport: Gathered 1536 samples. | 20260506 17:59:19.506 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 17:59:19.506 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 17:59:19.506 | INFO | PerfReport: 21.46%, Upload -- 2.09e+01 / 0.00e+00, 7.04e+00 / 0.00e+00 | 20260506 17:59:19.506 | INFO | PerfReport: 21.14%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 17:59:19.506 | INFO | PerfReport: 32.21%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 17:59:19.506 | INFO | PerfReport: 1.47%, LockUpload -- 1.43e+00 / 0.00e+00, 5.76e+00 / 0.00e+00 | 20260506 17:59:19.506 | INFO | PerfReport: 1.08%, LockDownload -- 1.05e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260506 17:59:19.506 | INFO | PerfReport: 22.63%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 17:59:19.506 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260506 17:59:19.621 | INFO | Starting a round of Baz IO | 20260506 17:59:22.752 | INFO | Populating RTMetrics report | 20260506 17:59:22.753 | INFO | RTMetrics report populated | 20260506 17:59:22.753 | INFO | Opening tmp RTMetrics json file | 20260506 17:59:22.757 | INFO | Serializing RTMetrics to json | 20260506 17:59:22.784 | INFO | Serialization done, swapping tmp file now... | 20260506 17:59:22.815 | INFO | RTMetrics done | 20260506 17:59:23.308 | INFO | Finished a round of Baz IO: Wrote 2.051 GiB in 3.686 seconds | 20260506 17:59:23.308 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.32% / 0.01% / 64.67% | 20260506 17:59:23.308 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.22947% / 3.97364e-06% / 7.15256e-05% | 20260506 17:59:23.308 | INFO | Baz IO overall: | 20260506 17:59:23.308 | INFO | BytesWritten : 1759930350360 | 20260506 17:59:23.308 | INFO | NumEvents : 851265734102 | 20260506 17:59:24.647 | INFO | Analyzing chunk frames = [3334144,3334656) | 20260506 17:59:24.647 | INFO | MemInfo: { | 20260506 17:59:24.647 | INFO | "gpu" : " 17 GB", | 20260506 17:59:24.647 | INFO | "huge" : "", | 20260506 17:59:24.647 | INFO | "huge_shared" : " 27 GB", | 20260506 17:59:24.647 | INFO | "malloc" : " 14 GB", | 20260506 17:59:24.647 | INFO | "pinned" : " 29 GB" | 20260506 17:59:24.647 | INFO | } | 20260506 17:59:24.647 | INFO | Perf report for the last 5.14256 seconds: | 20260506 17:59:24.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:59:24.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:24.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:24.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:24.647 | INFO | ANALYSIS : 62.6%, 2.9, 95.3ms, 0 | 20260506 17:59:24.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:24.647 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 17:59:24.647 | INFO | PRE_HQ : 60.0%, 1.9, 57.8ms, 0 | 20260506 17:59:24.647 | INFO | BAZWRITER : 71.3%, 4.8, 177.5ms, 0 | 20260506 17:59:24.647 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.2ms, 0 | 20260506 17:59:24.647 | INFO | Uploaded 36GiB to GPU in 2.06612s (17.424GiB/s) | 20260506 17:59:24.648 | INFO | Downloaded 38GiB to GPU in 2.2148s (17.1573GiB/s) | 20260506 17:59:29.786 | INFO | Analyzing chunk frames = [3334656,3335168) | 20260506 17:59:29.786 | INFO | MemInfo: { | 20260506 17:59:29.786 | INFO | "gpu" : " 17 GB", | 20260506 17:59:29.786 | INFO | "huge" : "", | 20260506 17:59:29.786 | INFO | "huge_shared" : " 27 GB", | 20260506 17:59:29.786 | INFO | "malloc" : " 14 GB", | 20260506 17:59:29.787 | INFO | "pinned" : " 29 GB" | 20260506 17:59:29.787 | INFO | } | 20260506 17:59:29.787 | INFO | Perf report for the last 5.1398 seconds: | 20260506 17:59:29.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:59:29.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:29.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:29.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:29.787 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 17:59:29.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:29.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:29.787 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 17:59:29.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:29.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:29.787 | INFO | Uploaded 36GiB to GPU in 2.00421s (17.9622GiB/s) | 20260506 17:59:29.787 | INFO | Downloaded 34GiB to GPU in 1.93734s (17.5498GiB/s) | 20260506 17:59:34.926 | INFO | Analyzing chunk frames = [3335168,3335680) | 20260506 17:59:34.926 | INFO | MemInfo: { | 20260506 17:59:34.926 | INFO | "gpu" : " 17 GB", | 20260506 17:59:34.926 | INFO | "huge" : "", | 20260506 17:59:34.926 | INFO | "huge_shared" : " 27 GB", | 20260506 17:59:34.926 | INFO | "malloc" : " 14 GB", | 20260506 17:59:34.926 | INFO | "pinned" : " 29 GB" | 20260506 17:59:34.926 | INFO | } | 20260506 17:59:34.926 | INFO | Perf report for the last 5.13929 seconds: | 20260506 17:59:34.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:59:34.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:34.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:34.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:34.926 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 17:59:34.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:34.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:34.926 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:59:34.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:34.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:34.926 | INFO | Uploaded 36GiB to GPU in 1.97806s (18.1996GiB/s) | 20260506 17:59:34.926 | INFO | Downloaded 34GiB to GPU in 1.92153s (17.6943GiB/s) | 20260506 17:59:40.062 | INFO | Analyzing chunk frames = [3335680,3336192) | 20260506 17:59:40.062 | INFO | MemInfo: { | 20260506 17:59:40.062 | INFO | "gpu" : " 17 GB", | 20260506 17:59:40.062 | INFO | "huge" : "", | 20260506 17:59:40.062 | INFO | "huge_shared" : " 27 GB", | 20260506 17:59:40.062 | INFO | "malloc" : " 14 GB", | 20260506 17:59:40.062 | INFO | "pinned" : " 29 GB" | 20260506 17:59:40.062 | INFO | } | 20260506 17:59:40.062 | INFO | Perf report for the last 5.13595 seconds: | 20260506 17:59:40.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:59:40.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:40.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:40.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:40.062 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 17:59:40.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:40.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:40.062 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 17:59:40.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:40.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:40.062 | INFO | Uploaded 36GiB to GPU in 1.94942s (18.467GiB/s) | 20260506 17:59:40.062 | INFO | Downloaded 34GiB to GPU in 1.86583s (18.2224GiB/s) | 20260506 17:59:45.198 | INFO | Analyzing chunk frames = [3336192,3336704) | 20260506 17:59:45.198 | INFO | MemInfo: { | 20260506 17:59:45.198 | INFO | "gpu" : " 17 GB", | 20260506 17:59:45.198 | INFO | "huge" : "", | 20260506 17:59:45.198 | INFO | "huge_shared" : " 27 GB", | 20260506 17:59:45.199 | INFO | "malloc" : " 14 GB", | 20260506 17:59:45.199 | INFO | "pinned" : " 29 GB" | 20260506 17:59:45.199 | INFO | } | 20260506 17:59:45.199 | INFO | Perf report for the last 5.13667 seconds: | 20260506 17:59:45.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:59:45.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:45.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:45.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:45.199 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 17:59:45.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:45.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:45.199 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 17:59:45.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:45.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:45.199 | INFO | Uploaded 36GiB to GPU in 2.00951s (17.9148GiB/s) | 20260506 17:59:45.199 | INFO | Downloaded 34GiB to GPU in 1.91375s (17.7662GiB/s) | 20260506 17:59:50.336 | INFO | Analyzing chunk frames = [3336704,3337216) | 20260506 17:59:50.336 | INFO | MemInfo: { | 20260506 17:59:50.336 | INFO | "gpu" : " 17 GB", | 20260506 17:59:50.336 | INFO | "huge" : "", | 20260506 17:59:50.336 | INFO | "huge_shared" : " 27 GB", | 20260506 17:59:50.336 | INFO | "malloc" : " 14 GB", | 20260506 17:59:50.336 | INFO | "pinned" : " 29 GB" | 20260506 17:59:50.336 | INFO | } | 20260506 17:59:50.336 | INFO | Perf report for the last 5.13726 seconds: | 20260506 17:59:50.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:59:50.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:50.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:50.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:50.336 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 17:59:50.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:50.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:50.336 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 17:59:50.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:50.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:50.336 | INFO | Uploaded 36GiB to GPU in 2.01875s (17.8329GiB/s) | 20260506 17:59:50.336 | INFO | Downloaded 34GiB to GPU in 1.93491s (17.5719GiB/s) | 20260506 17:59:55.481 | INFO | Analyzing chunk frames = [3337216,3337728) | 20260506 17:59:55.481 | INFO | MemInfo: { | 20260506 17:59:55.481 | INFO | "gpu" : " 17 GB", | 20260506 17:59:55.481 | INFO | "huge" : "", | 20260506 17:59:55.481 | INFO | "huge_shared" : " 27 GB", | 20260506 17:59:55.481 | INFO | "malloc" : " 14 GB", | 20260506 17:59:55.481 | INFO | "pinned" : " 29 GB" | 20260506 17:59:55.481 | INFO | } | 20260506 17:59:55.481 | INFO | Perf report for the last 5.14551 seconds: | 20260506 17:59:55.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 17:59:55.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:55.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:55.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:55.481 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 17:59:55.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:55.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:55.481 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 17:59:55.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:55.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 17:59:55.482 | INFO | Uploaded 36GiB to GPU in 2.00035s (17.9969GiB/s) | 20260506 17:59:55.482 | INFO | Downloaded 34GiB to GPU in 2.00933s (16.921GiB/s) | 20260506 18:00:00.619 | INFO | Analyzing chunk frames = [3337728,3338240) | 20260506 18:00:00.619 | INFO | MemInfo: { | 20260506 18:00:00.619 | INFO | "gpu" : " 17 GB", | 20260506 18:00:00.619 | INFO | "huge" : "", | 20260506 18:00:00.619 | INFO | "huge_shared" : " 27 GB", | 20260506 18:00:00.619 | INFO | "malloc" : " 14 GB", | 20260506 18:00:00.619 | INFO | "pinned" : " 29 GB" | 20260506 18:00:00.619 | INFO | } | 20260506 18:00:00.619 | INFO | Perf report for the last 5.13733 seconds: | 20260506 18:00:00.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:00:00.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:00.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:00.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:00.619 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:00:00.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:00.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:00.619 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:00:00.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:00.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:00.619 | INFO | Uploaded 36GiB to GPU in 1.98797s (18.1089GiB/s) | 20260506 18:00:00.619 | INFO | Downloaded 34GiB to GPU in 1.9775s (17.1935GiB/s) | 20260506 18:00:00.732 | INFO | Starting a round of Baz IO | 20260506 18:00:03.865 | INFO | Populating RTMetrics report | 20260506 18:00:03.865 | INFO | RTMetrics report populated | 20260506 18:00:03.865 | INFO | Opening tmp RTMetrics json file | 20260506 18:00:03.892 | INFO | Serializing RTMetrics to json | 20260506 18:00:03.952 | INFO | Serialization done, swapping tmp file now... | 20260506 18:00:03.983 | INFO | RTMetrics done | 20260506 18:00:04.472 | INFO | Finished a round of Baz IO: Wrote 2.051 GiB in 3.739 seconds | 20260506 18:00:04.472 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.31% / 0.01% / 64.68% | 20260506 18:00:04.472 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.22732% / 7.94729e-06% / 0.000127157% | 20260506 18:00:04.472 | INFO | Baz IO overall: | 20260506 18:00:04.472 | INFO | BytesWritten : 1762132793615 | 20260506 18:00:04.472 | INFO | NumEvents : 852327495490 | 20260506 18:00:05.757 | INFO | Analyzing chunk frames = [3338240,3338752) | 20260506 18:00:05.757 | INFO | MemInfo: { | 20260506 18:00:05.757 | INFO | "gpu" : " 17 GB", | 20260506 18:00:05.757 | INFO | "huge" : "", | 20260506 18:00:05.757 | INFO | "huge_shared" : " 27 GB", | 20260506 18:00:05.757 | INFO | "malloc" : " 14 GB", | 20260506 18:00:05.757 | INFO | "pinned" : " 29 GB" | 20260506 18:00:05.757 | INFO | } | 20260506 18:00:05.757 | INFO | Perf report for the last 5.13841 seconds: | 20260506 18:00:05.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:00:05.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:05.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:05.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:05.757 | INFO | ANALYSIS : 62.6%, 2.9, 95.2ms, 0 | 20260506 18:00:05.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:05.757 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 18:00:05.757 | INFO | PRE_HQ : 60.2%, 1.9, 57.9ms, 0 | 20260506 18:00:05.757 | INFO | BAZWRITER : 72.4%, 4.8, 178.5ms, 0 | 20260506 18:00:05.757 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 18:00:05.757 | INFO | Uploaded 36GiB to GPU in 2.05382s (17.5283GiB/s) | 20260506 18:00:05.758 | INFO | Downloaded 38GiB to GPU in 2.20512s (17.2326GiB/s) | 20260506 18:00:10.896 | INFO | Analyzing chunk frames = [3338752,3339264) | 20260506 18:00:10.896 | INFO | MemInfo: { | 20260506 18:00:10.896 | INFO | "gpu" : " 17 GB", | 20260506 18:00:10.896 | INFO | "huge" : "", | 20260506 18:00:10.896 | INFO | "huge_shared" : " 27 GB", | 20260506 18:00:10.896 | INFO | "malloc" : " 14 GB", | 20260506 18:00:10.896 | INFO | "pinned" : " 29 GB" | 20260506 18:00:10.896 | INFO | } | 20260506 18:00:10.896 | INFO | Perf report for the last 5.13923 seconds: | 20260506 18:00:10.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:00:10.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:10.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:10.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:10.896 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:00:10.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:10.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:10.896 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:00:10.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:10.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:10.897 | INFO | Uploaded 36GiB to GPU in 2.02602s (17.7689GiB/s) | 20260506 18:00:10.897 | INFO | Downloaded 34GiB to GPU in 1.97041s (17.2553GiB/s) | 20260506 18:00:16.038 | INFO | Analyzing chunk frames = [3339264,3339776) | 20260506 18:00:16.038 | INFO | MemInfo: { | 20260506 18:00:16.038 | INFO | "gpu" : " 17 GB", | 20260506 18:00:16.038 | INFO | "huge" : "", | 20260506 18:00:16.038 | INFO | "huge_shared" : " 27 GB", | 20260506 18:00:16.038 | INFO | "malloc" : " 14 GB", | 20260506 18:00:16.038 | INFO | "pinned" : " 29 GB" | 20260506 18:00:16.038 | INFO | } | 20260506 18:00:16.038 | INFO | Perf report for the last 5.14201 seconds: | 20260506 18:00:16.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:00:16.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:16.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:16.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:16.038 | INFO | ANALYSIS : 61.2%, 3.0, 94.7ms, 0 | 20260506 18:00:16.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:16.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:16.038 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:00:16.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:16.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:16.039 | INFO | Uploaded 36GiB to GPU in 2.00029s (17.9974GiB/s) | 20260506 18:00:16.039 | INFO | Downloaded 34GiB to GPU in 1.93117s (17.6059GiB/s) | 20260506 18:00:21.174 | INFO | Analyzing chunk frames = [3339776,3340288) | 20260506 18:00:21.174 | INFO | MemInfo: { | 20260506 18:00:21.174 | INFO | "gpu" : " 17 GB", | 20260506 18:00:21.174 | INFO | "huge" : "", | 20260506 18:00:21.174 | INFO | "huge_shared" : " 27 GB", | 20260506 18:00:21.174 | INFO | "malloc" : " 14 GB", | 20260506 18:00:21.175 | INFO | "pinned" : " 29 GB" | 20260506 18:00:21.175 | INFO | } | 20260506 18:00:21.175 | INFO | Perf report for the last 5.13617 seconds: | 20260506 18:00:21.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:00:21.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:21.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:21.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:21.175 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 18:00:21.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:21.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:21.175 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:00:21.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:21.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:21.175 | INFO | Uploaded 36GiB to GPU in 1.93916s (18.5647GiB/s) | 20260506 18:00:21.175 | INFO | Downloaded 34GiB to GPU in 1.87463s (18.1369GiB/s) | 20260506 18:00:26.317 | INFO | Analyzing chunk frames = [3340288,3340800) | 20260506 18:00:26.317 | INFO | MemInfo: { | 20260506 18:00:26.317 | INFO | "gpu" : " 17 GB", | 20260506 18:00:26.317 | INFO | "huge" : "", | 20260506 18:00:26.317 | INFO | "huge_shared" : " 27 GB", | 20260506 18:00:26.317 | INFO | "malloc" : " 14 GB", | 20260506 18:00:26.317 | INFO | "pinned" : " 29 GB" | 20260506 18:00:26.317 | INFO | } | 20260506 18:00:26.317 | INFO | Perf report for the last 5.14228 seconds: | 20260506 18:00:26.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:00:26.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:26.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:26.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:26.317 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 18:00:26.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:26.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:26.317 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:00:26.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:26.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:26.317 | INFO | Uploaded 36GiB to GPU in 2.00562s (17.9495GiB/s) | 20260506 18:00:26.317 | INFO | Downloaded 34GiB to GPU in 1.91272s (17.7758GiB/s) | 20260506 18:00:31.455 | INFO | Analyzing chunk frames = [3340800,3341312) | 20260506 18:00:31.455 | INFO | MemInfo: { | 20260506 18:00:31.455 | INFO | "gpu" : " 17 GB", | 20260506 18:00:31.455 | INFO | "huge" : "", | 20260506 18:00:31.455 | INFO | "huge_shared" : " 27 GB", | 20260506 18:00:31.455 | INFO | "malloc" : " 14 GB", | 20260506 18:00:31.455 | INFO | "pinned" : " 29 GB" | 20260506 18:00:31.455 | INFO | } | 20260506 18:00:31.455 | INFO | Perf report for the last 5.13779 seconds: | 20260506 18:00:31.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:00:31.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:31.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:31.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:31.455 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 18:00:31.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:31.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:31.455 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 18:00:31.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:31.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:31.455 | INFO | Uploaded 36GiB to GPU in 2.01352s (17.8791GiB/s) | 20260506 18:00:31.455 | INFO | Downloaded 34GiB to GPU in 1.95729s (17.3709GiB/s) | 20260506 18:00:36.599 | INFO | Analyzing chunk frames = [3341312,3341824) | 20260506 18:00:36.599 | INFO | MemInfo: { | 20260506 18:00:36.599 | INFO | "gpu" : " 17 GB", | 20260506 18:00:36.599 | INFO | "huge" : "", | 20260506 18:00:36.599 | INFO | "huge_shared" : " 27 GB", | 20260506 18:00:36.599 | INFO | "malloc" : " 14 GB", | 20260506 18:00:36.599 | INFO | "pinned" : " 29 GB" | 20260506 18:00:36.599 | INFO | } | 20260506 18:00:36.599 | INFO | Perf report for the last 5.14464 seconds: | 20260506 18:00:36.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:00:36.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:36.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:36.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:36.599 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:00:36.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:36.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:36.599 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 18:00:36.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:36.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:36.600 | INFO | Uploaded 36GiB to GPU in 1.99852s (18.0134GiB/s) | 20260506 18:00:36.600 | INFO | Downloaded 34GiB to GPU in 2.00692s (16.9414GiB/s) | 20260506 18:00:41.744 | INFO | Analyzing chunk frames = [3341824,3342336) | 20260506 18:00:41.744 | INFO | MemInfo: { | 20260506 18:00:41.744 | INFO | "gpu" : " 17 GB", | 20260506 18:00:41.744 | INFO | "huge" : "", | 20260506 18:00:41.744 | INFO | "huge_shared" : " 27 GB", | 20260506 18:00:41.744 | INFO | "malloc" : " 14 GB", | 20260506 18:00:41.744 | INFO | "pinned" : " 29 GB" | 20260506 18:00:41.744 | INFO | } | 20260506 18:00:41.744 | INFO | Perf report for the last 5.14476 seconds: | 20260506 18:00:41.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:00:41.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:41.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:41.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:41.744 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 18:00:41.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:41.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:41.744 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 18:00:41.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:41.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:41.744 | INFO | Uploaded 36GiB to GPU in 1.98877s (18.1016GiB/s) | 20260506 18:00:41.744 | INFO | Downloaded 34GiB to GPU in 1.99635s (17.0311GiB/s) | 20260506 18:00:41.744 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:00:41.744 | INFO | Total mem usage: 18 GiB | 20260506 18:00:41.744 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:00:41.744 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -10.1%) | 20260506 18:00:41.744 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:00:41.744 | INFO | High Water Breakdown: | 20260506 18:00:41.744 | INFO | 62.1% : 10.2 GiB ( 89% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:00:41.744 | INFO | 37.8% : 6.18 GiB ( 92.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:00:41.744 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:00:41.744 | INFO | ------------------------------------------ | 20260506 18:00:41.745 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:00:41.745 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:00:41.745 | INFO | Total mem usage: 26.5 GiB | 20260506 18:00:41.745 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:00:41.745 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:00:41.745 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:00:41.745 | INFO | High Water Breakdown: | 20260506 18:00:41.745 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:00:41.745 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:00:41.745 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:00:41.745 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:00:41.745 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:00:41.745 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:00:41.745 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:00:41.745 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:00:41.745 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:00:41.745 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:00:41.745 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:00:41.745 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:00:41.745 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:00:41.745 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:00:41.745 | INFO | ------------------------------------------ | 20260506 18:00:41.745 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:00:41.745 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:00:41.745 | INFO | Total mem usage: 25.6 GiB | 20260506 18:00:41.745 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:00:41.745 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:00:41.745 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:00:41.745 | INFO | High Water Breakdown: | 20260506 18:00:41.745 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:00:41.745 | INFO | ------------------------------------------ | 20260506 18:00:41.745 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:00:41.745 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:00:41.745 | INFO | Total mem usage: 15.7 GiB | 20260506 18:00:41.745 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:00:41.745 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:00:41.745 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 18:00:41.745 | INFO | High Water Breakdown: | 20260506 18:00:41.745 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:00:41.745 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:00:41.745 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:00:41.745 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:00:41.745 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:00:41.745 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:00:41.745 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:00:41.745 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:00:41.745 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:00:41.745 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:00:41.745 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:00:41.745 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:00:41.745 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:00:41.745 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:00:41.745 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:00:41.745 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:00:41.745 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:00:41.745 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:00:41.745 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:00:41.745 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:00:41.745 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:00:41.745 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:00:41.745 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:00:41.745 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:00:41.745 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:00:41.745 | INFO | ------------------------------------------ | 20260506 18:00:41.745 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:00:41.745 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:00:41.745 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:00:41.745 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:00:41.745 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.60e+01, 1.35e+01 / 2.80e+00 | 20260506 18:00:41.745 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.00e-01, 9.51e-03 / 5.24e-03 | 20260506 18:00:41.745 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.47e+00, 5.57e-01 / 2.79e-01 | 20260506 18:00:41.745 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.83e-01 / 1.28e-01 | 20260506 18:00:41.745 | INFO | PerfReport: 16.02%, DME -- 4.01e+01 / 4.49e+01, 4.30e+00 / 3.24e+00 | 20260506 18:00:41.745 | INFO | PerfReport: 56.92%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.32e+00 / 1.91e+00 | 20260506 18:00:41.745 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.27e+00 / 2.83e+00, 3.89e-01 / 2.20e-01 | 20260506 18:00:41.745 | INFO | PerfReport: 3.43%, Metrics -- 1.07e+00 / 1.42e+00, 2.80e-01 / 1.67e-01 | 20260506 18:00:41.745 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.70e-01 / 4.18e-01, 5.73e-02 / 1.95e-02 | 20260506 18:00:41.745 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:00:41.745 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:00:41.745 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:00:41.745 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 18:00:41.745 | INFO | PerfReport: 21.47%, Upload -- 2.09e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 18:00:41.745 | INFO | PerfReport: 21.19%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 18:00:41.745 | INFO | PerfReport: 32.18%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:00:41.745 | INFO | PerfReport: 1.41%, LockUpload -- 1.37e+00 / 0.00e+00, 5.37e+00 / 0.00e+00 | 20260506 18:00:41.745 | INFO | PerfReport: 1.13%, LockDownload -- 1.10e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260506 18:00:41.745 | INFO | PerfReport: 22.60%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:00:41.745 | INFO | PerfReport: 0.02%, Misc -- 1.53e-02 / 0.00e+00, 2.95e-02 / 0.00e+00 | 20260506 18:00:41.859 | INFO | Starting a round of Baz IO | 20260506 18:00:44.997 | INFO | Populating RTMetrics report | 20260506 18:00:44.998 | INFO | RTMetrics report populated | 20260506 18:00:44.998 | INFO | Opening tmp RTMetrics json file | 20260506 18:00:44.999 | INFO | Serializing RTMetrics to json | 20260506 18:00:45.008 | INFO | Serialization done, swapping tmp file now... | 20260506 18:00:45.034 | INFO | RTMetrics done | 20260506 18:00:45.372 | INFO | Finished a round of Baz IO: Wrote 2.051 GiB in 3.512 seconds | 20260506 18:00:45.372 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.31% / 0.01% / 64.69% | 20260506 18:00:45.372 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.22544% / 1.19209e-05% / 6.35783e-05% | 20260506 18:00:45.372 | INFO | Baz IO overall: | 20260506 18:00:45.372 | INFO | BytesWritten : 1764334666788 | 20260506 18:00:45.372 | INFO | NumEvents : 853388790013 | 20260506 18:00:46.873 | INFO | Analyzing chunk frames = [3342336,3342848) | 20260506 18:00:46.873 | INFO | MemInfo: { | 20260506 18:00:46.873 | INFO | "gpu" : " 17 GB", | 20260506 18:00:46.873 | INFO | "huge" : "", | 20260506 18:00:46.873 | INFO | "huge_shared" : " 27 GB", | 20260506 18:00:46.873 | INFO | "malloc" : " 14 GB", | 20260506 18:00:46.873 | INFO | "pinned" : " 29 GB" | 20260506 18:00:46.873 | INFO | } | 20260506 18:00:46.873 | INFO | Perf report for the last 5.12864 seconds: | 20260506 18:00:46.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:00:46.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:46.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:46.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:46.873 | INFO | ANALYSIS : 62.9%, 2.9, 95.4ms, 0 | 20260506 18:00:46.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:46.873 | INFO | RT_METRICS : 44.2%, 1.0, 23.1ms, 0 | 20260506 18:00:46.873 | INFO | PRE_HQ : 60.4%, 1.9, 58.3ms, 0 | 20260506 18:00:46.873 | INFO | BAZWRITER : 68.2%, 4.7, 166.0ms, 0 | 20260506 18:00:46.873 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 18:00:46.873 | INFO | Uploaded 36GiB to GPU in 2.06778s (17.41GiB/s) | 20260506 18:00:46.873 | INFO | Downloaded 38GiB to GPU in 2.21542s (17.1525GiB/s) | 20260506 18:00:52.010 | INFO | Analyzing chunk frames = [3342848,3343360) | 20260506 18:00:52.010 | INFO | MemInfo: { | 20260506 18:00:52.010 | INFO | "gpu" : " 17 GB", | 20260506 18:00:52.010 | INFO | "huge" : "", | 20260506 18:00:52.010 | INFO | "huge_shared" : " 27 GB", | 20260506 18:00:52.010 | INFO | "malloc" : " 14 GB", | 20260506 18:00:52.010 | INFO | "pinned" : " 29 GB" | 20260506 18:00:52.010 | INFO | } | 20260506 18:00:52.010 | INFO | Perf report for the last 5.13753 seconds: | 20260506 18:00:52.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:00:52.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:52.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:52.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:52.010 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:00:52.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:52.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:52.010 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:00:52.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:52.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:52.010 | INFO | Uploaded 36GiB to GPU in 2.00347s (17.9688GiB/s) | 20260506 18:00:52.011 | INFO | Downloaded 34GiB to GPU in 1.94486s (17.482GiB/s) | 20260506 18:00:57.150 | INFO | Analyzing chunk frames = [3343360,3343872) | 20260506 18:00:57.150 | INFO | MemInfo: { | 20260506 18:00:57.150 | INFO | "gpu" : " 17 GB", | 20260506 18:00:57.150 | INFO | "huge" : "", | 20260506 18:00:57.150 | INFO | "huge_shared" : " 27 GB", | 20260506 18:00:57.150 | INFO | "malloc" : " 14 GB", | 20260506 18:00:57.150 | INFO | "pinned" : " 29 GB" | 20260506 18:00:57.150 | INFO | } | 20260506 18:00:57.150 | INFO | Perf report for the last 5.14011 seconds: | 20260506 18:00:57.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:00:57.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:57.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:57.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:57.150 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:00:57.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:57.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:57.151 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:00:57.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:57.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:00:57.151 | INFO | Uploaded 36GiB to GPU in 1.96094s (18.3585GiB/s) | 20260506 18:00:57.151 | INFO | Downloaded 34GiB to GPU in 1.92418s (17.6698GiB/s) | 20260506 18:01:02.291 | INFO | Analyzing chunk frames = [3343872,3344384) | 20260506 18:01:02.291 | INFO | MemInfo: { | 20260506 18:01:02.291 | INFO | "gpu" : " 17 GB", | 20260506 18:01:02.291 | INFO | "huge" : "", | 20260506 18:01:02.291 | INFO | "huge_shared" : " 27 GB", | 20260506 18:01:02.291 | INFO | "malloc" : " 14 GB", | 20260506 18:01:02.291 | INFO | "pinned" : " 29 GB" | 20260506 18:01:02.291 | INFO | } | 20260506 18:01:02.291 | INFO | Perf report for the last 5.14066 seconds: | 20260506 18:01:02.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:01:02.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:02.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:02.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:02.291 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:01:02.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:02.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:02.291 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:01:02.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:02.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:02.291 | INFO | Uploaded 36GiB to GPU in 1.91666s (18.7827GiB/s) | 20260506 18:01:02.291 | INFO | Downloaded 34GiB to GPU in 1.86944s (18.1872GiB/s) | 20260506 18:01:07.430 | INFO | Analyzing chunk frames = [3344384,3344896) | 20260506 18:01:07.430 | INFO | MemInfo: { | 20260506 18:01:07.430 | INFO | "gpu" : " 17 GB", | 20260506 18:01:07.430 | INFO | "huge" : "", | 20260506 18:01:07.430 | INFO | "huge_shared" : " 27 GB", | 20260506 18:01:07.430 | INFO | "malloc" : " 14 GB", | 20260506 18:01:07.430 | INFO | "pinned" : " 29 GB" | 20260506 18:01:07.430 | INFO | } | 20260506 18:01:07.430 | INFO | Perf report for the last 5.13934 seconds: | 20260506 18:01:07.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:01:07.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:07.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:07.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:07.430 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260506 18:01:07.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:07.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:07.430 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:01:07.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:07.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:07.431 | INFO | Uploaded 36GiB to GPU in 1.95423s (18.4216GiB/s) | 20260506 18:01:07.431 | INFO | Downloaded 34GiB to GPU in 1.8866s (18.0218GiB/s) | 20260506 18:01:12.569 | INFO | Analyzing chunk frames = [3344896,3345408) | 20260506 18:01:12.569 | INFO | MemInfo: { | 20260506 18:01:12.569 | INFO | "gpu" : " 17 GB", | 20260506 18:01:12.569 | INFO | "huge" : "", | 20260506 18:01:12.569 | INFO | "huge_shared" : " 27 GB", | 20260506 18:01:12.569 | INFO | "malloc" : " 14 GB", | 20260506 18:01:12.569 | INFO | "pinned" : " 29 GB" | 20260506 18:01:12.569 | INFO | } | 20260506 18:01:12.569 | INFO | Perf report for the last 5.13885 seconds: | 20260506 18:01:12.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:01:12.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:12.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:12.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:12.569 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 18:01:12.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:12.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:12.569 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260506 18:01:12.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:12.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:12.569 | INFO | Uploaded 36GiB to GPU in 1.98523s (18.1339GiB/s) | 20260506 18:01:12.570 | INFO | Downloaded 34GiB to GPU in 1.95812s (17.3636GiB/s) | 20260506 18:01:17.705 | INFO | Analyzing chunk frames = [3345408,3345920) | 20260506 18:01:17.705 | INFO | MemInfo: { | 20260506 18:01:17.705 | INFO | "gpu" : " 17 GB", | 20260506 18:01:17.705 | INFO | "huge" : "", | 20260506 18:01:17.706 | INFO | "huge_shared" : " 27 GB", | 20260506 18:01:17.706 | INFO | "malloc" : " 14 GB", | 20260506 18:01:17.706 | INFO | "pinned" : " 29 GB" | 20260506 18:01:17.706 | INFO | } | 20260506 18:01:17.706 | INFO | Perf report for the last 5.13641 seconds: | 20260506 18:01:17.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:01:17.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:17.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:17.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:17.706 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 18:01:17.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:17.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:17.706 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 18:01:17.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:17.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:17.706 | INFO | Uploaded 36GiB to GPU in 1.99234s (18.0692GiB/s) | 20260506 18:01:17.706 | INFO | Downloaded 34GiB to GPU in 2.01144s (16.9033GiB/s) | 20260506 18:01:22.848 | INFO | Analyzing chunk frames = [3345920,3346432) | 20260506 18:01:22.848 | INFO | MemInfo: { | 20260506 18:01:22.848 | INFO | "gpu" : " 17 GB", | 20260506 18:01:22.848 | INFO | "huge" : "", | 20260506 18:01:22.848 | INFO | "huge_shared" : " 27 GB", | 20260506 18:01:22.848 | INFO | "malloc" : " 14 GB", | 20260506 18:01:22.848 | INFO | "pinned" : " 29 GB" | 20260506 18:01:22.848 | INFO | } | 20260506 18:01:22.848 | INFO | Perf report for the last 5.14225 seconds: | 20260506 18:01:22.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:01:22.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:22.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:22.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:22.848 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 18:01:22.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:22.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:22.848 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260506 18:01:22.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:22.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:22.848 | INFO | Uploaded 36GiB to GPU in 2.00318s (17.9714GiB/s) | 20260506 18:01:22.848 | INFO | Downloaded 34GiB to GPU in 1.97388s (17.225GiB/s) | 20260506 18:01:22.963 | INFO | Starting a round of Baz IO | 20260506 18:01:26.101 | INFO | Populating RTMetrics report | 20260506 18:01:26.102 | INFO | RTMetrics report populated | 20260506 18:01:26.102 | INFO | Opening tmp RTMetrics json file | 20260506 18:01:26.115 | INFO | Serializing RTMetrics to json | 20260506 18:01:26.331 | INFO | Serialization done, swapping tmp file now... | 20260506 18:01:26.332 | INFO | RTMetrics done | 20260506 18:01:26.846 | INFO | Finished a round of Baz IO: Wrote 2.052 GiB in 3.883 seconds | 20260506 18:01:26.846 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.30% / 0.01% / 64.69% | 20260506 18:01:26.846 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.22116% / 1.19209e-05% / 8.74201e-05% | 20260506 18:01:26.846 | INFO | Baz IO overall: | 20260506 18:01:26.846 | INFO | BytesWritten : 1766537479617 | 20260506 18:01:26.846 | INFO | NumEvents : 854451194129 | 20260506 18:01:27.991 | INFO | Analyzing chunk frames = [3346432,3346944) | 20260506 18:01:27.991 | INFO | MemInfo: { | 20260506 18:01:27.991 | INFO | "gpu" : " 17 GB", | 20260506 18:01:27.991 | INFO | "huge" : "", | 20260506 18:01:27.991 | INFO | "huge_shared" : " 27 GB", | 20260506 18:01:27.991 | INFO | "malloc" : " 14 GB", | 20260506 18:01:27.991 | INFO | "pinned" : " 29 GB" | 20260506 18:01:27.991 | INFO | } | 20260506 18:01:27.991 | INFO | Perf report for the last 5.14339 seconds: | 20260506 18:01:27.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:01:27.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:27.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:27.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:27.991 | INFO | ANALYSIS : 62.7%, 2.9, 95.2ms, 0 | 20260506 18:01:27.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:27.991 | INFO | RT_METRICS : 47.9%, 1.0, 25.1ms, 0 | 20260506 18:01:27.991 | INFO | PRE_HQ : 60.4%, 1.9, 58.3ms, 0 | 20260506 18:01:27.991 | INFO | BAZWRITER : 75.2%, 4.8, 186.3ms, 0 | 20260506 18:01:27.991 | INFO | BAZ_SAMPLING : 4.3%, 1.0, 2.2ms, 0 | 20260506 18:01:27.992 | INFO | Uploaded 36GiB to GPU in 2.0466s (17.5902GiB/s) | 20260506 18:01:27.992 | INFO | Downloaded 38GiB to GPU in 2.19704s (17.296GiB/s) | 20260506 18:01:33.128 | INFO | Analyzing chunk frames = [3346944,3347456) | 20260506 18:01:33.128 | INFO | MemInfo: { | 20260506 18:01:33.128 | INFO | "gpu" : " 17 GB", | 20260506 18:01:33.128 | INFO | "huge" : "", | 20260506 18:01:33.128 | INFO | "huge_shared" : " 27 GB", | 20260506 18:01:33.128 | INFO | "malloc" : " 14 GB", | 20260506 18:01:33.128 | INFO | "pinned" : " 29 GB" | 20260506 18:01:33.128 | INFO | } | 20260506 18:01:33.128 | INFO | Perf report for the last 5.13679 seconds: | 20260506 18:01:33.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:01:33.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:33.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:33.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:33.128 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:01:33.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:33.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:33.128 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:01:33.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:33.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:33.128 | INFO | Uploaded 36GiB to GPU in 2.01534s (17.863GiB/s) | 20260506 18:01:33.128 | INFO | Downloaded 34GiB to GPU in 1.93059s (17.6112GiB/s) | 20260506 18:01:38.265 | INFO | Analyzing chunk frames = [3347456,3347968) | 20260506 18:01:38.265 | INFO | MemInfo: { | 20260506 18:01:38.265 | INFO | "gpu" : " 17 GB", | 20260506 18:01:38.265 | INFO | "huge" : "", | 20260506 18:01:38.265 | INFO | "huge_shared" : " 27 GB", | 20260506 18:01:38.266 | INFO | "malloc" : " 14 GB", | 20260506 18:01:38.266 | INFO | "pinned" : " 29 GB" | 20260506 18:01:38.266 | INFO | } | 20260506 18:01:38.266 | INFO | Perf report for the last 5.13739 seconds: | 20260506 18:01:38.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:01:38.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:38.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:38.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:38.266 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 18:01:38.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:38.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:38.266 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:01:38.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:38.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:38.266 | INFO | Uploaded 36GiB to GPU in 1.94436s (18.5151GiB/s) | 20260506 18:01:38.266 | INFO | Downloaded 34GiB to GPU in 1.91838s (17.7233GiB/s) | 20260506 18:01:43.406 | INFO | Analyzing chunk frames = [3347968,3348480) | 20260506 18:01:43.406 | INFO | MemInfo: { | 20260506 18:01:43.406 | INFO | "gpu" : " 17 GB", | 20260506 18:01:43.406 | INFO | "huge" : "", | 20260506 18:01:43.406 | INFO | "huge_shared" : " 27 GB", | 20260506 18:01:43.406 | INFO | "malloc" : " 14 GB", | 20260506 18:01:43.406 | INFO | "pinned" : " 29 GB" | 20260506 18:01:43.406 | INFO | } | 20260506 18:01:43.406 | INFO | Perf report for the last 5.14023 seconds: | 20260506 18:01:43.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:01:43.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:43.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:43.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:43.406 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:01:43.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:43.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:43.406 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:01:43.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:43.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:43.407 | INFO | Uploaded 36GiB to GPU in 1.93509s (18.6038GiB/s) | 20260506 18:01:43.407 | INFO | Downloaded 34GiB to GPU in 1.86657s (18.2152GiB/s) | 20260506 18:01:48.547 | INFO | Analyzing chunk frames = [3348480,3348992) | 20260506 18:01:48.547 | INFO | MemInfo: { | 20260506 18:01:48.547 | INFO | "gpu" : " 17 GB", | 20260506 18:01:48.547 | INFO | "huge" : "", | 20260506 18:01:48.547 | INFO | "huge_shared" : " 27 GB", | 20260506 18:01:48.547 | INFO | "malloc" : " 14 GB", | 20260506 18:01:48.547 | INFO | "pinned" : " 29 GB" | 20260506 18:01:48.547 | INFO | } | 20260506 18:01:48.547 | INFO | Perf report for the last 5.14099 seconds: | 20260506 18:01:48.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:01:48.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:48.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:48.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:48.547 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260506 18:01:48.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:48.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:48.547 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:01:48.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:48.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:48.547 | INFO | Uploaded 36GiB to GPU in 1.95603s (18.4046GiB/s) | 20260506 18:01:48.547 | INFO | Downloaded 34GiB to GPU in 1.88075s (18.0779GiB/s) | 20260506 18:01:53.685 | INFO | Analyzing chunk frames = [3348992,3349504) | 20260506 18:01:53.685 | INFO | MemInfo: { | 20260506 18:01:53.685 | INFO | "gpu" : " 17 GB", | 20260506 18:01:53.686 | INFO | "huge" : "", | 20260506 18:01:53.686 | INFO | "huge_shared" : " 27 GB", | 20260506 18:01:53.686 | INFO | "malloc" : " 14 GB", | 20260506 18:01:53.686 | INFO | "pinned" : " 29 GB" | 20260506 18:01:53.686 | INFO | } | 20260506 18:01:53.686 | INFO | Perf report for the last 5.13877 seconds: | 20260506 18:01:53.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:01:53.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:53.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:53.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:53.686 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 18:01:53.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:53.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:53.686 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:01:53.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:53.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:53.686 | INFO | Uploaded 36GiB to GPU in 2.0152s (17.8642GiB/s) | 20260506 18:01:53.686 | INFO | Downloaded 34GiB to GPU in 1.95804s (17.3643GiB/s) | 20260506 18:01:58.827 | INFO | Analyzing chunk frames = [3349504,3350016) | 20260506 18:01:58.827 | INFO | MemInfo: { | 20260506 18:01:58.827 | INFO | "gpu" : " 17 GB", | 20260506 18:01:58.827 | INFO | "huge" : "", | 20260506 18:01:58.827 | INFO | "huge_shared" : " 27 GB", | 20260506 18:01:58.827 | INFO | "malloc" : " 14 GB", | 20260506 18:01:58.827 | INFO | "pinned" : " 29 GB" | 20260506 18:01:58.827 | INFO | } | 20260506 18:01:58.827 | INFO | Perf report for the last 5.14123 seconds: | 20260506 18:01:58.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:01:58.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:58.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:58.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:58.827 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:01:58.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:58.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:58.827 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:01:58.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:58.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:01:58.827 | INFO | Uploaded 36GiB to GPU in 2.01805s (17.839GiB/s) | 20260506 18:01:58.827 | INFO | Downloaded 34GiB to GPU in 2.00473s (16.9599GiB/s) | 20260506 18:02:03.961 | INFO | Analyzing chunk frames = [3350016,3350528) | 20260506 18:02:03.961 | INFO | MemInfo: { | 20260506 18:02:03.961 | INFO | "gpu" : " 17 GB", | 20260506 18:02:03.962 | INFO | "huge" : "", | 20260506 18:02:03.962 | INFO | "huge_shared" : " 27 GB", | 20260506 18:02:03.962 | INFO | "malloc" : " 14 GB", | 20260506 18:02:03.962 | INFO | "pinned" : " 29 GB" | 20260506 18:02:03.962 | INFO | } | 20260506 18:02:03.962 | INFO | Perf report for the last 5.13475 seconds: | 20260506 18:02:03.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:02:03.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:03.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:03.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:03.962 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 18:02:03.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:03.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:03.962 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:02:03.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:03.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:03.962 | INFO | Uploaded 36GiB to GPU in 2.0173s (17.8457GiB/s) | 20260506 18:02:03.962 | INFO | Downloaded 34GiB to GPU in 1.99624s (17.032GiB/s) | 20260506 18:02:03.962 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:02:03.962 | INFO | Total mem usage: 18 GiB | 20260506 18:02:03.962 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:02:03.962 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -13.9%) | 20260506 18:02:03.962 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:02:03.962 | INFO | High Water Breakdown: | 20260506 18:02:03.962 | INFO | 61.6% : 9.73 GiB ( 85.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:02:03.962 | INFO | 38.3% : 6.05 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:02:03.962 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:02:03.962 | INFO | ------------------------------------------ | 20260506 18:02:03.962 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:02:03.962 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:02:03.962 | INFO | Total mem usage: 26.5 GiB | 20260506 18:02:03.962 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:02:03.962 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:02:03.962 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:02:03.962 | INFO | High Water Breakdown: | 20260506 18:02:03.962 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:02:03.962 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:02:03.962 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:02:03.962 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:02:03.962 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:02:03.962 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:02:03.963 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:02:03.963 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:02:03.963 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:02:03.963 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:02:03.963 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:02:03.963 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:02:03.963 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:02:03.963 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:02:03.963 | INFO | ------------------------------------------ | 20260506 18:02:03.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:02:03.963 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:02:03.963 | INFO | Total mem usage: 25.6 GiB | 20260506 18:02:03.963 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:02:03.963 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:02:03.963 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:02:03.963 | INFO | High Water Breakdown: | 20260506 18:02:03.963 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:02:03.963 | INFO | ------------------------------------------ | 20260506 18:02:03.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:02:03.963 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:02:03.963 | INFO | Total mem usage: 15.7 GiB | 20260506 18:02:03.963 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:02:03.963 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:02:03.963 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:02:03.963 | INFO | High Water Breakdown: | 20260506 18:02:03.963 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:02:03.963 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:02:03.963 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:02:03.963 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:02:03.963 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:02:03.963 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:02:03.963 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:02:03.963 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:02:03.963 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:02:03.963 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:02:03.963 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:02:03.963 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:02:03.963 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:02:03.963 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:02:03.963 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:02:03.963 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:02:03.963 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:02:03.963 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:02:03.963 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:02:03.963 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:02:03.963 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:02:03.963 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:02:03.963 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:02:03.963 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:02:03.963 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:02:03.963 | INFO | ------------------------------------------ | 20260506 18:02:03.963 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:02:03.963 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:02:03.963 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:02:03.963 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:02:03.963 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.56e+01, 1.35e+01 / 2.87e+00 | 20260506 18:02:03.963 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.57e-03 / 6.07e-03 | 20260506 18:02:03.963 | INFO | PerfReport: 11.59%, Baseline -- 3.63e+00 / 4.38e+00, 5.50e-01 / 3.40e-01 | 20260506 18:02:03.963 | INFO | PerfReport: 3.24%, Histogram -- 1.01e+00 / 1.43e+00, 1.79e-01 / 1.19e-01 | 20260506 18:02:03.963 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.48e+01, 4.30e+00 / 3.09e+00 | 20260506 18:02:03.963 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.41e+00 / 1.79e+00 | 20260506 18:02:03.963 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.27e+00 / 2.87e+00, 3.91e-01 / 2.26e-01 | 20260506 18:02:03.963 | INFO | PerfReport: 3.42%, Metrics -- 1.07e+00 / 1.41e+00, 2.80e-01 / 1.45e-01 | 20260506 18:02:03.963 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.70e-01 / 4.20e-01, 5.86e-02 / 2.74e-02 | 20260506 18:02:03.963 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:02:03.963 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:02:03.963 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:02:03.963 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.31e+01 / 0.00e+00 | 20260506 18:02:03.963 | INFO | PerfReport: 21.37%, Upload -- 2.07e+01 / 0.00e+00, 6.80e+00 / 0.00e+00 | 20260506 18:02:03.963 | INFO | PerfReport: 21.17%, Download -- 2.05e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 18:02:03.963 | INFO | PerfReport: 32.26%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:02:03.963 | INFO | PerfReport: 1.28%, LockUpload -- 1.24e+00 / 0.00e+00, 5.29e+00 / 0.00e+00 | 20260506 18:02:03.963 | INFO | PerfReport: 1.21%, LockDownload -- 1.17e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 18:02:03.963 | INFO | PerfReport: 22.71%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:02:03.963 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260506 18:02:04.080 | INFO | Starting a round of Baz IO | 20260506 18:02:07.219 | INFO | Populating RTMetrics report | 20260506 18:02:07.219 | INFO | RTMetrics report populated | 20260506 18:02:07.219 | INFO | Opening tmp RTMetrics json file | 20260506 18:02:07.222 | INFO | Serializing RTMetrics to json | 20260506 18:02:07.227 | INFO | Serialization done, swapping tmp file now... | 20260506 18:02:07.236 | INFO | RTMetrics done | 20260506 18:02:07.635 | INFO | Finished a round of Baz IO: Wrote 2.05 GiB in 3.555 seconds | 20260506 18:02:07.635 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.30% / 0.01% / 64.70% | 20260506 18:02:07.635 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.21718% / 1.19209e-05% / 5.96046e-05% | 20260506 18:02:07.635 | INFO | Baz IO overall: | 20260506 18:02:07.635 | INFO | BytesWritten : 1768738355503 | 20260506 18:02:07.636 | INFO | NumEvents : 855512218800 | 20260506 18:02:09.103 | INFO | Analyzing chunk frames = [3350528,3351040) | 20260506 18:02:09.103 | INFO | MemInfo: { | 20260506 18:02:09.103 | INFO | "gpu" : " 17 GB", | 20260506 18:02:09.103 | INFO | "huge" : "", | 20260506 18:02:09.103 | INFO | "huge_shared" : " 27 GB", | 20260506 18:02:09.103 | INFO | "malloc" : " 14 GB", | 20260506 18:02:09.103 | INFO | "pinned" : " 29 GB" | 20260506 18:02:09.103 | INFO | } | 20260506 18:02:09.104 | INFO | Perf report for the last 5.14194 seconds: | 20260506 18:02:09.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:02:09.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:09.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:09.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:09.104 | INFO | ANALYSIS : 62.7%, 2.9, 95.6ms, 0 | 20260506 18:02:09.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:09.104 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 18:02:09.104 | INFO | PRE_HQ : 60.2%, 1.9, 58.0ms, 0 | 20260506 18:02:09.104 | INFO | BAZWRITER : 68.8%, 4.8, 169.8ms, 0 | 20260506 18:02:09.104 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 18:02:09.104 | INFO | Uploaded 36GiB to GPU in 2.03264s (17.7109GiB/s) | 20260506 18:02:09.104 | INFO | Downloaded 38GiB to GPU in 2.21797s (17.1328GiB/s) | 20260506 18:02:14.244 | INFO | Analyzing chunk frames = [3351040,3351552) | 20260506 18:02:14.244 | INFO | MemInfo: { | 20260506 18:02:14.244 | INFO | "gpu" : " 17 GB", | 20260506 18:02:14.244 | INFO | "huge" : "", | 20260506 18:02:14.244 | INFO | "huge_shared" : " 27 GB", | 20260506 18:02:14.244 | INFO | "malloc" : " 14 GB", | 20260506 18:02:14.245 | INFO | "pinned" : " 29 GB" | 20260506 18:02:14.245 | INFO | } | 20260506 18:02:14.245 | INFO | Perf report for the last 5.14102 seconds: | 20260506 18:02:14.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:02:14.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:14.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:14.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:14.245 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 18:02:14.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:14.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:14.245 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:02:14.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:14.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:14.245 | INFO | Uploaded 36GiB to GPU in 1.99992s (18.0007GiB/s) | 20260506 18:02:14.245 | INFO | Downloaded 34GiB to GPU in 1.92822s (17.6328GiB/s) | 20260506 18:02:19.379 | INFO | Analyzing chunk frames = [3351552,3352064) | 20260506 18:02:19.379 | INFO | MemInfo: { | 20260506 18:02:19.379 | INFO | "gpu" : " 17 GB", | 20260506 18:02:19.379 | INFO | "huge" : "", | 20260506 18:02:19.379 | INFO | "huge_shared" : " 27 GB", | 20260506 18:02:19.379 | INFO | "malloc" : " 14 GB", | 20260506 18:02:19.379 | INFO | "pinned" : " 29 GB" | 20260506 18:02:19.379 | INFO | } | 20260506 18:02:19.379 | INFO | Perf report for the last 5.13439 seconds: | 20260506 18:02:19.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:02:19.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:19.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:19.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:19.379 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 18:02:19.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:19.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:19.379 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:02:19.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:19.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:19.379 | INFO | Uploaded 36GiB to GPU in 1.9612s (18.3561GiB/s) | 20260506 18:02:19.379 | INFO | Downloaded 34GiB to GPU in 1.91795s (17.7273GiB/s) | 20260506 18:02:24.516 | INFO | Analyzing chunk frames = [3352064,3352576) | 20260506 18:02:24.516 | INFO | MemInfo: { | 20260506 18:02:24.516 | INFO | "gpu" : " 17 GB", | 20260506 18:02:24.516 | INFO | "huge" : "", | 20260506 18:02:24.516 | INFO | "huge_shared" : " 27 GB", | 20260506 18:02:24.517 | INFO | "malloc" : " 14 GB", | 20260506 18:02:24.517 | INFO | "pinned" : " 29 GB" | 20260506 18:02:24.517 | INFO | } | 20260506 18:02:24.517 | INFO | Perf report for the last 5.1376 seconds: | 20260506 18:02:24.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:02:24.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:24.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:24.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:24.517 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 18:02:24.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:24.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:24.517 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:02:24.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:24.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:24.517 | INFO | Uploaded 36GiB to GPU in 1.93217s (18.6319GiB/s) | 20260506 18:02:24.517 | INFO | Downloaded 34GiB to GPU in 1.87786s (18.1057GiB/s) | 20260506 18:02:29.657 | INFO | Analyzing chunk frames = [3352576,3353088) | 20260506 18:02:29.657 | INFO | MemInfo: { | 20260506 18:02:29.657 | INFO | "gpu" : " 17 GB", | 20260506 18:02:29.657 | INFO | "huge" : "", | 20260506 18:02:29.657 | INFO | "huge_shared" : " 27 GB", | 20260506 18:02:29.657 | INFO | "malloc" : " 14 GB", | 20260506 18:02:29.657 | INFO | "pinned" : " 29 GB" | 20260506 18:02:29.657 | INFO | } | 20260506 18:02:29.657 | INFO | Perf report for the last 5.14052 seconds: | 20260506 18:02:29.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:02:29.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:29.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:29.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:29.657 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 18:02:29.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:29.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:29.657 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:02:29.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:29.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:29.657 | INFO | Uploaded 36GiB to GPU in 1.94828s (18.4778GiB/s) | 20260506 18:02:29.657 | INFO | Downloaded 34GiB to GPU in 1.88523s (18.0349GiB/s) | 20260506 18:02:34.802 | INFO | Analyzing chunk frames = [3353088,3353600) | 20260506 18:02:34.802 | INFO | MemInfo: { | 20260506 18:02:34.802 | INFO | "gpu" : " 17 GB", | 20260506 18:02:34.802 | INFO | "huge" : "", | 20260506 18:02:34.802 | INFO | "huge_shared" : " 27 GB", | 20260506 18:02:34.802 | INFO | "malloc" : " 14 GB", | 20260506 18:02:34.802 | INFO | "pinned" : " 29 GB" | 20260506 18:02:34.802 | INFO | } | 20260506 18:02:34.802 | INFO | Perf report for the last 5.14505 seconds: | 20260506 18:02:34.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:02:34.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:34.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:34.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:34.802 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 18:02:34.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:34.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:34.802 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260506 18:02:34.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:34.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:34.803 | INFO | Uploaded 36GiB to GPU in 1.98584s (18.1284GiB/s) | 20260506 18:02:34.803 | INFO | Downloaded 34GiB to GPU in 1.95969s (17.3497GiB/s) | 20260506 18:02:39.940 | INFO | Analyzing chunk frames = [3353600,3354112) | 20260506 18:02:39.940 | INFO | MemInfo: { | 20260506 18:02:39.940 | INFO | "gpu" : " 17 GB", | 20260506 18:02:39.940 | INFO | "huge" : "", | 20260506 18:02:39.940 | INFO | "huge_shared" : " 27 GB", | 20260506 18:02:39.940 | INFO | "malloc" : " 14 GB", | 20260506 18:02:39.940 | INFO | "pinned" : " 29 GB" | 20260506 18:02:39.940 | INFO | } | 20260506 18:02:39.940 | INFO | Perf report for the last 5.13769 seconds: | 20260506 18:02:39.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:02:39.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:39.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:39.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:39.940 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 18:02:39.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:39.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:39.940 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:02:39.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:39.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:39.940 | INFO | Uploaded 36GiB to GPU in 1.98654s (18.1219GiB/s) | 20260506 18:02:39.940 | INFO | Downloaded 34GiB to GPU in 1.96637s (17.2908GiB/s) | 20260506 18:02:45.084 | INFO | Analyzing chunk frames = [3354112,3354624) | 20260506 18:02:45.084 | INFO | MemInfo: { | 20260506 18:02:45.084 | INFO | "gpu" : " 17 GB", | 20260506 18:02:45.084 | INFO | "huge" : "", | 20260506 18:02:45.084 | INFO | "huge_shared" : " 27 GB", | 20260506 18:02:45.084 | INFO | "malloc" : " 14 GB", | 20260506 18:02:45.084 | INFO | "pinned" : " 29 GB" | 20260506 18:02:45.084 | INFO | } | 20260506 18:02:45.084 | INFO | Perf report for the last 5.14389 seconds: | 20260506 18:02:45.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:02:45.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:45.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:45.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:45.084 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 18:02:45.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:45.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:45.084 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 18:02:45.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:45.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:45.084 | INFO | Uploaded 36GiB to GPU in 1.99176s (18.0745GiB/s) | 20260506 18:02:45.084 | INFO | Downloaded 34GiB to GPU in 1.99784s (17.0183GiB/s) | 20260506 18:02:45.199 | INFO | Starting a round of Baz IO | 20260506 18:02:48.336 | INFO | Populating RTMetrics report | 20260506 18:02:48.336 | INFO | RTMetrics report populated | 20260506 18:02:48.336 | INFO | Opening tmp RTMetrics json file | 20260506 18:02:48.352 | INFO | Serializing RTMetrics to json | 20260506 18:02:48.354 | INFO | Serialization done, swapping tmp file now... | 20260506 18:02:48.359 | INFO | RTMetrics done | 20260506 18:02:48.643 | INFO | Finished a round of Baz IO: Wrote 2.05 GiB in 3.444 seconds | 20260506 18:02:48.643 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.29% / 0.01% / 64.70% | 20260506 18:02:48.643 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.21601% / 1.98682e-05% / 7.54992e-05% | 20260506 18:02:48.643 | INFO | Baz IO overall: | 20260506 18:02:48.643 | INFO | BytesWritten : 1770939694247 | 20260506 18:02:48.643 | INFO | NumEvents : 856573694663 | 20260506 18:02:50.220 | INFO | Analyzing chunk frames = [3354624,3355136) | 20260506 18:02:50.220 | INFO | MemInfo: { | 20260506 18:02:50.220 | INFO | "gpu" : " 17 GB", | 20260506 18:02:50.220 | INFO | "huge" : "", | 20260506 18:02:50.220 | INFO | "huge_shared" : " 27 GB", | 20260506 18:02:50.220 | INFO | "malloc" : " 14 GB", | 20260506 18:02:50.220 | INFO | "pinned" : " 29 GB" | 20260506 18:02:50.220 | INFO | } | 20260506 18:02:50.220 | INFO | Perf report for the last 5.13619 seconds: | 20260506 18:02:50.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:02:50.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:50.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:50.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:50.220 | INFO | ANALYSIS : 62.7%, 2.9, 95.2ms, 0 | 20260506 18:02:50.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:50.220 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 18:02:50.220 | INFO | PRE_HQ : 60.1%, 1.9, 58.3ms, 0 | 20260506 18:02:50.220 | INFO | BAZWRITER : 66.7%, 4.8, 164.5ms, 0 | 20260506 18:02:50.220 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 18:02:50.220 | INFO | Uploaded 36GiB to GPU in 2.03479s (17.6922GiB/s) | 20260506 18:02:50.220 | INFO | Downloaded 38GiB to GPU in 2.20893s (17.2029GiB/s) | 20260506 18:02:55.357 | INFO | Analyzing chunk frames = [3355136,3355648) | 20260506 18:02:55.357 | INFO | MemInfo: { | 20260506 18:02:55.357 | INFO | "gpu" : " 17 GB", | 20260506 18:02:55.357 | INFO | "huge" : "", | 20260506 18:02:55.357 | INFO | "huge_shared" : " 27 GB", | 20260506 18:02:55.357 | INFO | "malloc" : " 14 GB", | 20260506 18:02:55.357 | INFO | "pinned" : " 29 GB" | 20260506 18:02:55.357 | INFO | } | 20260506 18:02:55.357 | INFO | Perf report for the last 5.13702 seconds: | 20260506 18:02:55.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:02:55.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:55.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:55.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:55.357 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 18:02:55.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:55.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:55.357 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:02:55.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:55.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:02:55.357 | INFO | Uploaded 36GiB to GPU in 2.00985s (17.9118GiB/s) | 20260506 18:02:55.357 | INFO | Downloaded 34GiB to GPU in 1.93591s (17.5628GiB/s) | 20260506 18:03:00.496 | INFO | Analyzing chunk frames = [3355648,3356160) | 20260506 18:03:00.496 | INFO | MemInfo: { | 20260506 18:03:00.496 | INFO | "gpu" : " 17 GB", | 20260506 18:03:00.496 | INFO | "huge" : "", | 20260506 18:03:00.496 | INFO | "huge_shared" : " 27 GB", | 20260506 18:03:00.496 | INFO | "malloc" : " 14 GB", | 20260506 18:03:00.496 | INFO | "pinned" : " 29 GB" | 20260506 18:03:00.496 | INFO | } | 20260506 18:03:00.496 | INFO | Perf report for the last 5.13918 seconds: | 20260506 18:03:00.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:03:00.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:00.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:00.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:00.496 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 18:03:00.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:00.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:00.496 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:03:00.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:00.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:00.496 | INFO | Uploaded 36GiB to GPU in 1.97651s (18.2139GiB/s) | 20260506 18:03:00.496 | INFO | Downloaded 34GiB to GPU in 1.92532s (17.6594GiB/s) | 20260506 18:03:05.633 | INFO | Analyzing chunk frames = [3356160,3356672) | 20260506 18:03:05.633 | INFO | MemInfo: { | 20260506 18:03:05.633 | INFO | "gpu" : " 17 GB", | 20260506 18:03:05.634 | INFO | "huge" : "", | 20260506 18:03:05.634 | INFO | "huge_shared" : " 27 GB", | 20260506 18:03:05.634 | INFO | "malloc" : " 14 GB", | 20260506 18:03:05.634 | INFO | "pinned" : " 29 GB" | 20260506 18:03:05.634 | INFO | } | 20260506 18:03:05.634 | INFO | Perf report for the last 5.1374 seconds: | 20260506 18:03:05.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:03:05.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:05.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:05.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:05.634 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:03:05.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:05.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:05.634 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:03:05.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:05.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:05.634 | INFO | Uploaded 36GiB to GPU in 1.91628s (18.7864GiB/s) | 20260506 18:03:05.634 | INFO | Downloaded 34GiB to GPU in 1.87103s (18.1718GiB/s) | 20260506 18:03:10.773 | INFO | Analyzing chunk frames = [3356672,3357184) | 20260506 18:03:10.773 | INFO | MemInfo: { | 20260506 18:03:10.773 | INFO | "gpu" : " 17 GB", | 20260506 18:03:10.773 | INFO | "huge" : "", | 20260506 18:03:10.773 | INFO | "huge_shared" : " 27 GB", | 20260506 18:03:10.773 | INFO | "malloc" : " 14 GB", | 20260506 18:03:10.773 | INFO | "pinned" : " 29 GB" | 20260506 18:03:10.773 | INFO | } | 20260506 18:03:10.773 | INFO | Perf report for the last 5.13956 seconds: | 20260506 18:03:10.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:03:10.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:10.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:10.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:10.773 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 18:03:10.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:10.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:10.773 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:03:10.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:10.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:10.773 | INFO | Uploaded 36GiB to GPU in 2.01202s (17.8925GiB/s) | 20260506 18:03:10.773 | INFO | Downloaded 34GiB to GPU in 1.91672s (17.7386GiB/s) | 20260506 18:03:15.917 | INFO | Analyzing chunk frames = [3357184,3357696) | 20260506 18:03:15.917 | INFO | MemInfo: { | 20260506 18:03:15.917 | INFO | "gpu" : " 17 GB", | 20260506 18:03:15.917 | INFO | "huge" : "", | 20260506 18:03:15.917 | INFO | "huge_shared" : " 27 GB", | 20260506 18:03:15.917 | INFO | "malloc" : " 14 GB", | 20260506 18:03:15.917 | INFO | "pinned" : " 29 GB" | 20260506 18:03:15.917 | INFO | } | 20260506 18:03:15.917 | INFO | Perf report for the last 5.14419 seconds: | 20260506 18:03:15.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:03:15.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:15.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:15.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:15.917 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 18:03:15.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:15.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:15.918 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:03:15.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:15.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:15.918 | INFO | Uploaded 36GiB to GPU in 2.01048s (17.9062GiB/s) | 20260506 18:03:15.918 | INFO | Downloaded 34GiB to GPU in 1.95616s (17.381GiB/s) | 20260506 18:03:21.056 | INFO | Analyzing chunk frames = [3357696,3358208) | 20260506 18:03:21.056 | INFO | MemInfo: { | 20260506 18:03:21.056 | INFO | "gpu" : " 17 GB", | 20260506 18:03:21.056 | INFO | "huge" : "", | 20260506 18:03:21.056 | INFO | "huge_shared" : " 27 GB", | 20260506 18:03:21.056 | INFO | "malloc" : " 14 GB", | 20260506 18:03:21.056 | INFO | "pinned" : " 29 GB" | 20260506 18:03:21.056 | INFO | } | 20260506 18:03:21.056 | INFO | Perf report for the last 5.13871 seconds: | 20260506 18:03:21.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:03:21.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:21.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:21.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:21.056 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:03:21.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:21.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:21.056 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:03:21.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:21.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:21.056 | INFO | Uploaded 36GiB to GPU in 1.99483s (18.0466GiB/s) | 20260506 18:03:21.056 | INFO | Downloaded 34GiB to GPU in 2.00708s (16.94GiB/s) | 20260506 18:03:26.194 | INFO | Analyzing chunk frames = [3358208,3358720) | 20260506 18:03:26.194 | INFO | MemInfo: { | 20260506 18:03:26.194 | INFO | "gpu" : " 17 GB", | 20260506 18:03:26.194 | INFO | "huge" : "", | 20260506 18:03:26.194 | INFO | "huge_shared" : " 27 GB", | 20260506 18:03:26.194 | INFO | "malloc" : " 14 GB", | 20260506 18:03:26.194 | INFO | "pinned" : " 29 GB" | 20260506 18:03:26.194 | INFO | } | 20260506 18:03:26.194 | INFO | Perf report for the last 5.1379 seconds: | 20260506 18:03:26.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:03:26.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:26.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:26.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:26.194 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 18:03:26.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:26.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:26.194 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:03:26.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:26.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:26.194 | INFO | Uploaded 36GiB to GPU in 2.06237s (17.4557GiB/s) | 20260506 18:03:26.194 | INFO | Downloaded 34GiB to GPU in 1.97921s (17.1785GiB/s) | 20260506 18:03:26.194 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:03:26.194 | INFO | Total mem usage: 18 GiB | 20260506 18:03:26.194 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:03:26.194 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -13.9%) | 20260506 18:03:26.194 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:03:26.194 | INFO | High Water Breakdown: | 20260506 18:03:26.194 | INFO | 61.6% : 9.74 GiB ( 85.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:03:26.194 | INFO | 38.3% : 6.05 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:03:26.194 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:03:26.194 | INFO | ------------------------------------------ | 20260506 18:03:26.194 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:03:26.194 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:03:26.194 | INFO | Total mem usage: 26.5 GiB | 20260506 18:03:26.194 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:03:26.194 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:03:26.194 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:03:26.194 | INFO | High Water Breakdown: | 20260506 18:03:26.194 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:03:26.194 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:03:26.195 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:03:26.195 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:03:26.195 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:03:26.195 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:03:26.195 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:03:26.195 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:03:26.195 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:03:26.195 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:03:26.195 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:03:26.195 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:03:26.195 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:03:26.195 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:03:26.195 | INFO | ------------------------------------------ | 20260506 18:03:26.195 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:03:26.195 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:03:26.195 | INFO | Total mem usage: 25.6 GiB | 20260506 18:03:26.195 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:03:26.195 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:03:26.195 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:03:26.195 | INFO | High Water Breakdown: | 20260506 18:03:26.195 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:03:26.195 | INFO | ------------------------------------------ | 20260506 18:03:26.195 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:03:26.195 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:03:26.195 | INFO | Total mem usage: 15.7 GiB | 20260506 18:03:26.195 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:03:26.195 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:03:26.195 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 18:03:26.195 | INFO | High Water Breakdown: | 20260506 18:03:26.195 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:03:26.195 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:03:26.195 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:03:26.195 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:03:26.195 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:03:26.195 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:03:26.195 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:03:26.195 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:03:26.195 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:03:26.195 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:03:26.195 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:03:26.195 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:03:26.195 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:03:26.195 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:03:26.195 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:03:26.195 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:03:26.195 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:03:26.195 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:03:26.195 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:03:26.195 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:03:26.195 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:03:26.195 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:03:26.195 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:03:26.195 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:03:26.195 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:03:26.195 | INFO | ------------------------------------------ | 20260506 18:03:26.195 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:03:26.195 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:03:26.195 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:03:26.195 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:03:26.195 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.58e+01, 1.33e+01 / 2.49e+00 | 20260506 18:03:26.195 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.00e-01, 1.07e-02 / 6.30e-03 | 20260506 18:03:26.195 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.41e+00, 5.59e-01 / 3.81e-01 | 20260506 18:03:26.195 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.48e+00, 1.85e-01 / 1.45e-01 | 20260506 18:03:26.195 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.47e+01, 4.12e+00 / 2.42e+00 | 20260506 18:03:26.195 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.44e+00 / 1.69e+00 | 20260506 18:03:26.195 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.26e+00 / 2.83e+00, 3.82e-01 / 2.65e-01 | 20260506 18:03:26.195 | INFO | PerfReport: 3.42%, Metrics -- 1.07e+00 / 1.43e+00, 2.80e-01 / 1.69e-01 | 20260506 18:03:26.195 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.70e-01 / 4.17e-01, 5.88e-02 / 2.10e-02 | 20260506 18:03:26.195 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:03:26.195 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:03:26.195 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:03:26.195 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 18:03:26.195 | INFO | PerfReport: 21.35%, Upload -- 2.07e+01 / 0.00e+00, 6.82e+00 / 0.00e+00 | 20260506 18:03:26.195 | INFO | PerfReport: 21.15%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 18:03:26.195 | INFO | PerfReport: 32.24%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 18:03:26.195 | INFO | PerfReport: 1.32%, LockUpload -- 1.28e+00 / 0.00e+00, 5.47e+00 / 0.00e+00 | 20260506 18:03:26.195 | INFO | PerfReport: 1.22%, LockDownload -- 1.19e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260506 18:03:26.195 | INFO | PerfReport: 22.70%, LockCompute -- 2.20e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 18:03:26.195 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260506 18:03:26.305 | INFO | Starting a round of Baz IO | 20260506 18:03:29.454 | INFO | Populating RTMetrics report | 20260506 18:03:29.454 | INFO | RTMetrics report populated | 20260506 18:03:29.454 | INFO | Opening tmp RTMetrics json file | 20260506 18:03:29.557 | INFO | Serializing RTMetrics to json | 20260506 18:03:29.595 | INFO | Serialization done, swapping tmp file now... | 20260506 18:03:29.596 | INFO | RTMetrics done | 20260506 18:03:30.057 | INFO | Finished a round of Baz IO: Wrote 2.051 GiB in 3.752 seconds | 20260506 18:03:30.057 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.28% / 0.01% / 64.71% | 20260506 18:03:30.057 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.21316% / 7.94729e-06% / 6.75519e-05% | 20260506 18:03:30.058 | INFO | Baz IO overall: | 20260506 18:03:30.058 | INFO | BytesWritten : 1773141527213 | 20260506 18:03:30.058 | INFO | NumEvents : 857635722723 | 20260506 18:03:31.336 | INFO | Analyzing chunk frames = [3358720,3359232) | 20260506 18:03:31.336 | INFO | MemInfo: { | 20260506 18:03:31.336 | INFO | "gpu" : " 17 GB", | 20260506 18:03:31.336 | INFO | "huge" : "", | 20260506 18:03:31.336 | INFO | "huge_shared" : " 27 GB", | 20260506 18:03:31.336 | INFO | "malloc" : " 14 GB", | 20260506 18:03:31.336 | INFO | "pinned" : " 29 GB" | 20260506 18:03:31.336 | INFO | } | 20260506 18:03:31.336 | INFO | Perf report for the last 5.14186 seconds: | 20260506 18:03:31.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:03:31.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:31.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:31.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:31.336 | INFO | ANALYSIS : 62.8%, 2.9, 95.5ms, 0 | 20260506 18:03:31.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:31.336 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260506 18:03:31.336 | INFO | PRE_HQ : 60.5%, 1.9, 58.8ms, 0 | 20260506 18:03:31.336 | INFO | BAZWRITER : 72.4%, 4.8, 178.8ms, 0 | 20260506 18:03:31.336 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 18:03:31.336 | INFO | Uploaded 36GiB to GPU in 2.05081s (17.554GiB/s) | 20260506 18:03:31.337 | INFO | Downloaded 38GiB to GPU in 2.22189s (17.1025GiB/s) | 20260506 18:03:36.467 | INFO | Analyzing chunk frames = [3359232,3359744) | 20260506 18:03:36.467 | INFO | MemInfo: { | 20260506 18:03:36.467 | INFO | "gpu" : " 17 GB", | 20260506 18:03:36.467 | INFO | "huge" : "", | 20260506 18:03:36.468 | INFO | "huge_shared" : " 27 GB", | 20260506 18:03:36.468 | INFO | "malloc" : " 14 GB", | 20260506 18:03:36.468 | INFO | "pinned" : " 29 GB" | 20260506 18:03:36.468 | INFO | } | 20260506 18:03:36.468 | INFO | Perf report for the last 5.13166 seconds: | 20260506 18:03:36.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:03:36.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:36.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:36.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:36.468 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 18:03:36.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:36.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:36.468 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 18:03:36.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:36.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:36.468 | INFO | Uploaded 36GiB to GPU in 2.0016s (17.9856GiB/s) | 20260506 18:03:36.468 | INFO | Downloaded 34GiB to GPU in 1.92464s (17.6656GiB/s) | 20260506 18:03:41.609 | INFO | Analyzing chunk frames = [3359744,3360256) | 20260506 18:03:41.609 | INFO | MemInfo: { | 20260506 18:03:41.609 | INFO | "gpu" : " 17 GB", | 20260506 18:03:41.609 | INFO | "huge" : "", | 20260506 18:03:41.609 | INFO | "huge_shared" : " 27 GB", | 20260506 18:03:41.609 | INFO | "malloc" : " 14 GB", | 20260506 18:03:41.609 | INFO | "pinned" : " 29 GB" | 20260506 18:03:41.609 | INFO | } | 20260506 18:03:41.609 | INFO | Perf report for the last 5.14162 seconds: | 20260506 18:03:41.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:03:41.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:41.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:41.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:41.609 | INFO | ANALYSIS : 61.0%, 3.0, 94.6ms, 0 | 20260506 18:03:41.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:41.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:41.609 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 18:03:41.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:41.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:41.609 | INFO | Uploaded 36GiB to GPU in 2.00038s (17.9966GiB/s) | 20260506 18:03:41.609 | INFO | Downloaded 34GiB to GPU in 1.93105s (17.607GiB/s) | 20260506 18:03:46.750 | INFO | Analyzing chunk frames = [3360256,3360768) | 20260506 18:03:46.750 | INFO | MemInfo: { | 20260506 18:03:46.750 | INFO | "gpu" : " 17 GB", | 20260506 18:03:46.750 | INFO | "huge" : "", | 20260506 18:03:46.750 | INFO | "huge_shared" : " 27 GB", | 20260506 18:03:46.750 | INFO | "malloc" : " 14 GB", | 20260506 18:03:46.750 | INFO | "pinned" : " 29 GB" | 20260506 18:03:46.750 | INFO | } | 20260506 18:03:46.750 | INFO | Perf report for the last 5.14094 seconds: | 20260506 18:03:46.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:03:46.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:46.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:46.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:46.750 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 18:03:46.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:46.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:46.750 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:03:46.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:46.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:46.750 | INFO | Uploaded 36GiB to GPU in 1.93511s (18.6036GiB/s) | 20260506 18:03:46.750 | INFO | Downloaded 34GiB to GPU in 1.8755s (18.1285GiB/s) | 20260506 18:03:51.892 | INFO | Analyzing chunk frames = [3360768,3361280) | 20260506 18:03:51.892 | INFO | MemInfo: { | 20260506 18:03:51.892 | INFO | "gpu" : " 17 GB", | 20260506 18:03:51.892 | INFO | "huge" : "", | 20260506 18:03:51.892 | INFO | "huge_shared" : " 27 GB", | 20260506 18:03:51.892 | INFO | "malloc" : " 14 GB", | 20260506 18:03:51.892 | INFO | "pinned" : " 29 GB" | 20260506 18:03:51.892 | INFO | } | 20260506 18:03:51.892 | INFO | Perf report for the last 5.1415 seconds: | 20260506 18:03:51.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:03:51.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:51.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:51.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:51.892 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 18:03:51.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:51.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:51.892 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:03:51.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:51.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:51.892 | INFO | Uploaded 36GiB to GPU in 2.02241s (17.8006GiB/s) | 20260506 18:03:51.892 | INFO | Downloaded 34GiB to GPU in 1.90916s (17.8089GiB/s) | 20260506 18:03:57.031 | INFO | Analyzing chunk frames = [3361280,3361792) | 20260506 18:03:57.031 | INFO | MemInfo: { | 20260506 18:03:57.031 | INFO | "gpu" : " 17 GB", | 20260506 18:03:57.031 | INFO | "huge" : "", | 20260506 18:03:57.031 | INFO | "huge_shared" : " 27 GB", | 20260506 18:03:57.031 | INFO | "malloc" : " 14 GB", | 20260506 18:03:57.031 | INFO | "pinned" : " 29 GB" | 20260506 18:03:57.031 | INFO | } | 20260506 18:03:57.031 | INFO | Perf report for the last 5.13896 seconds: | 20260506 18:03:57.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:03:57.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:57.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:57.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:57.031 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:03:57.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:57.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:57.031 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:03:57.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:57.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:03:57.031 | INFO | Uploaded 36GiB to GPU in 2.07079s (17.3846GiB/s) | 20260506 18:03:57.031 | INFO | Downloaded 34GiB to GPU in 1.95536s (17.3881GiB/s) | 20260506 18:04:02.174 | INFO | Analyzing chunk frames = [3361792,3362304) | 20260506 18:04:02.174 | INFO | MemInfo: { | 20260506 18:04:02.174 | INFO | "gpu" : " 17 GB", | 20260506 18:04:02.174 | INFO | "huge" : "", | 20260506 18:04:02.174 | INFO | "huge_shared" : " 27 GB", | 20260506 18:04:02.174 | INFO | "malloc" : " 14 GB", | 20260506 18:04:02.174 | INFO | "pinned" : " 29 GB" | 20260506 18:04:02.174 | INFO | } | 20260506 18:04:02.174 | INFO | Perf report for the last 5.14361 seconds: | 20260506 18:04:02.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:04:02.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:02.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:02.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:02.174 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 18:04:02.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:02.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:02.174 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260506 18:04:02.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:02.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:02.175 | INFO | Uploaded 36GiB to GPU in 1.99806s (18.0175GiB/s) | 20260506 18:04:02.175 | INFO | Downloaded 34GiB to GPU in 1.96069s (17.3408GiB/s) | 20260506 18:04:07.310 | INFO | Analyzing chunk frames = [3362304,3362816) | 20260506 18:04:07.310 | INFO | MemInfo: { | 20260506 18:04:07.310 | INFO | "gpu" : " 17 GB", | 20260506 18:04:07.310 | INFO | "huge" : "", | 20260506 18:04:07.310 | INFO | "huge_shared" : " 27 GB", | 20260506 18:04:07.310 | INFO | "malloc" : " 14 GB", | 20260506 18:04:07.310 | INFO | "pinned" : " 29 GB" | 20260506 18:04:07.310 | INFO | } | 20260506 18:04:07.310 | INFO | Perf report for the last 5.13564 seconds: | 20260506 18:04:07.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:04:07.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:07.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:07.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:07.310 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 18:04:07.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:07.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:07.310 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 18:04:07.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:07.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:07.310 | INFO | Uploaded 36GiB to GPU in 1.97339s (18.2427GiB/s) | 20260506 18:04:07.310 | INFO | Downloaded 34GiB to GPU in 1.99116s (17.0754GiB/s) | 20260506 18:04:07.425 | INFO | Starting a round of Baz IO | 20260506 18:04:10.563 | INFO | Populating RTMetrics report | 20260506 18:04:10.563 | INFO | RTMetrics report populated | 20260506 18:04:10.563 | INFO | Opening tmp RTMetrics json file | 20260506 18:04:10.595 | INFO | Serializing RTMetrics to json | 20260506 18:04:10.688 | INFO | Serialization done, swapping tmp file now... | 20260506 18:04:10.693 | INFO | RTMetrics done | 20260506 18:04:10.956 | INFO | Finished a round of Baz IO: Wrote 2.048 GiB in 3.531 seconds | 20260506 18:04:10.956 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.28% / 0.01% / 64.72% | 20260506 18:04:10.956 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.20991% / 3.97364e-06% / 5.96046e-05% | 20260506 18:04:10.956 | INFO | Baz IO overall: | 20260506 18:04:10.956 | INFO | BytesWritten : 1775340990622 | 20260506 18:04:10.956 | INFO | NumEvents : 858696026925 | 20260506 18:04:12.447 | INFO | Analyzing chunk frames = [3362816,3363328) | 20260506 18:04:12.447 | INFO | MemInfo: { | 20260506 18:04:12.447 | INFO | "gpu" : " 17 GB", | 20260506 18:04:12.447 | INFO | "huge" : "", | 20260506 18:04:12.447 | INFO | "huge_shared" : " 27 GB", | 20260506 18:04:12.447 | INFO | "malloc" : " 14 GB", | 20260506 18:04:12.447 | INFO | "pinned" : " 29 GB" | 20260506 18:04:12.447 | INFO | } | 20260506 18:04:12.447 | INFO | Perf report for the last 5.13746 seconds: | 20260506 18:04:12.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:04:12.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:12.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:12.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:12.448 | INFO | ANALYSIS : 62.7%, 2.9, 95.4ms, 0 | 20260506 18:04:12.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:12.448 | INFO | RT_METRICS : 45.9%, 1.0, 24.0ms, 0 | 20260506 18:04:12.448 | INFO | PRE_HQ : 60.2%, 1.9, 57.9ms, 0 | 20260506 18:04:12.448 | INFO | BAZWRITER : 68.4%, 4.8, 168.6ms, 0 | 20260506 18:04:12.448 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.3ms, 0 | 20260506 18:04:12.448 | INFO | Uploaded 36GiB to GPU in 2.02584s (17.7704GiB/s) | 20260506 18:04:12.448 | INFO | Downloaded 38GiB to GPU in 2.21432s (17.161GiB/s) | 20260506 18:04:17.585 | INFO | Analyzing chunk frames = [3363328,3363840) | 20260506 18:04:17.585 | INFO | MemInfo: { | 20260506 18:04:17.585 | INFO | "gpu" : " 17 GB", | 20260506 18:04:17.585 | INFO | "huge" : "", | 20260506 18:04:17.585 | INFO | "huge_shared" : " 27 GB", | 20260506 18:04:17.585 | INFO | "malloc" : " 14 GB", | 20260506 18:04:17.585 | INFO | "pinned" : " 29 GB" | 20260506 18:04:17.585 | INFO | } | 20260506 18:04:17.585 | INFO | Perf report for the last 5.13728 seconds: | 20260506 18:04:17.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:04:17.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:17.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:17.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:17.585 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:04:17.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:17.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:17.585 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:04:17.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:17.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:17.585 | INFO | Uploaded 36GiB to GPU in 1.97479s (18.2298GiB/s) | 20260506 18:04:17.585 | INFO | Downloaded 34GiB to GPU in 1.92533s (17.6593GiB/s) | 20260506 18:04:22.725 | INFO | Analyzing chunk frames = [3363840,3364352) | 20260506 18:04:22.725 | INFO | MemInfo: { | 20260506 18:04:22.725 | INFO | "gpu" : " 17 GB", | 20260506 18:04:22.725 | INFO | "huge" : "", | 20260506 18:04:22.725 | INFO | "huge_shared" : " 27 GB", | 20260506 18:04:22.726 | INFO | "malloc" : " 14 GB", | 20260506 18:04:22.726 | INFO | "pinned" : " 29 GB" | 20260506 18:04:22.726 | INFO | } | 20260506 18:04:22.726 | INFO | Perf report for the last 5.14089 seconds: | 20260506 18:04:22.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:04:22.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:22.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:22.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:22.726 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 18:04:22.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:22.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:22.726 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:04:22.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:22.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:22.726 | INFO | Uploaded 36GiB to GPU in 1.98618s (18.1253GiB/s) | 20260506 18:04:22.726 | INFO | Downloaded 34GiB to GPU in 1.92314s (17.6794GiB/s) | 20260506 18:04:27.872 | INFO | Analyzing chunk frames = [3364352,3364864) | 20260506 18:04:27.872 | INFO | MemInfo: { | 20260506 18:04:27.872 | INFO | "gpu" : " 17 GB", | 20260506 18:04:27.872 | INFO | "huge" : "", | 20260506 18:04:27.872 | INFO | "huge_shared" : " 27 GB", | 20260506 18:04:27.872 | INFO | "malloc" : " 14 GB", | 20260506 18:04:27.872 | INFO | "pinned" : " 29 GB" | 20260506 18:04:27.872 | INFO | } | 20260506 18:04:27.872 | INFO | Perf report for the last 5.14675 seconds: | 20260506 18:04:27.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:04:27.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:27.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:27.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:27.872 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 18:04:27.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:27.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:27.872 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:04:27.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:27.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:27.873 | INFO | Uploaded 36GiB to GPU in 1.93093s (18.6439GiB/s) | 20260506 18:04:27.873 | INFO | Downloaded 34GiB to GPU in 1.87148s (18.1674GiB/s) | 20260506 18:04:33.010 | INFO | Analyzing chunk frames = [3364864,3365376) | 20260506 18:04:33.010 | INFO | MemInfo: { | 20260506 18:04:33.010 | INFO | "gpu" : " 17 GB", | 20260506 18:04:33.010 | INFO | "huge" : "", | 20260506 18:04:33.010 | INFO | "huge_shared" : " 27 GB", | 20260506 18:04:33.010 | INFO | "malloc" : " 14 GB", | 20260506 18:04:33.010 | INFO | "pinned" : " 29 GB" | 20260506 18:04:33.010 | INFO | } | 20260506 18:04:33.010 | INFO | Perf report for the last 5.13802 seconds: | 20260506 18:04:33.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:04:33.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:33.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:33.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:33.010 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 18:04:33.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:33.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:33.011 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:04:33.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:33.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:33.011 | INFO | Uploaded 36GiB to GPU in 2.00737s (17.9339GiB/s) | 20260506 18:04:33.011 | INFO | Downloaded 34GiB to GPU in 1.91442s (17.7599GiB/s) | 20260506 18:04:38.145 | INFO | Analyzing chunk frames = [3365376,3365888) | 20260506 18:04:38.145 | INFO | MemInfo: { | 20260506 18:04:38.145 | INFO | "gpu" : " 17 GB", | 20260506 18:04:38.145 | INFO | "huge" : "", | 20260506 18:04:38.145 | INFO | "huge_shared" : " 27 GB", | 20260506 18:04:38.145 | INFO | "malloc" : " 14 GB", | 20260506 18:04:38.145 | INFO | "pinned" : " 29 GB" | 20260506 18:04:38.145 | INFO | } | 20260506 18:04:38.145 | INFO | Perf report for the last 5.1347 seconds: | 20260506 18:04:38.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:04:38.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:38.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:38.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:38.145 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 18:04:38.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:38.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:38.145 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:04:38.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:38.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:38.145 | INFO | Uploaded 36GiB to GPU in 2.00291s (17.9739GiB/s) | 20260506 18:04:38.145 | INFO | Downloaded 34GiB to GPU in 1.96099s (17.3382GiB/s) | 20260506 18:04:43.279 | INFO | Analyzing chunk frames = [3365888,3366400) | 20260506 18:04:43.279 | INFO | MemInfo: { | 20260506 18:04:43.279 | INFO | "gpu" : " 17 GB", | 20260506 18:04:43.279 | INFO | "huge" : "", | 20260506 18:04:43.279 | INFO | "huge_shared" : " 27 GB", | 20260506 18:04:43.279 | INFO | "malloc" : " 14 GB", | 20260506 18:04:43.279 | INFO | "pinned" : " 29 GB" | 20260506 18:04:43.279 | INFO | } | 20260506 18:04:43.279 | INFO | Perf report for the last 5.13397 seconds: | 20260506 18:04:43.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:04:43.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:43.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:43.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:43.279 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260506 18:04:43.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:43.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:43.279 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:04:43.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:43.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:43.279 | INFO | Uploaded 36GiB to GPU in 1.97917s (18.1894GiB/s) | 20260506 18:04:43.280 | INFO | Downloaded 34GiB to GPU in 2.00966s (16.9183GiB/s) | 20260506 18:04:48.418 | INFO | Analyzing chunk frames = [3366400,3366912) | 20260506 18:04:48.418 | INFO | MemInfo: { | 20260506 18:04:48.418 | INFO | "gpu" : " 17 GB", | 20260506 18:04:48.418 | INFO | "huge" : "", | 20260506 18:04:48.418 | INFO | "huge_shared" : " 27 GB", | 20260506 18:04:48.418 | INFO | "malloc" : " 14 GB", | 20260506 18:04:48.418 | INFO | "pinned" : " 29 GB" | 20260506 18:04:48.418 | INFO | } | 20260506 18:04:48.418 | INFO | Perf report for the last 5.13876 seconds: | 20260506 18:04:48.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:04:48.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:48.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:48.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:48.418 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 18:04:48.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:48.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:48.418 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:04:48.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:48.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:48.418 | INFO | Uploaded 36GiB to GPU in 2.02192s (17.8048GiB/s) | 20260506 18:04:48.418 | INFO | Downloaded 34GiB to GPU in 1.9865s (17.1155GiB/s) | 20260506 18:04:48.418 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:04:48.418 | INFO | Total mem usage: 18 GiB | 20260506 18:04:48.418 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:04:48.418 | INFO | Recent High Water Marks Sum: 16.7 GiB (Savings: -7.88%) | 20260506 18:04:48.418 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:04:48.418 | INFO | High Water Breakdown: | 20260506 18:04:48.418 | INFO | 62.4% : 10.4 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:04:48.418 | INFO | 37.5% : 6.25 GiB ( 93.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:04:48.418 | INFO | 0.119% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:04:48.418 | INFO | ------------------------------------------ | 20260506 18:04:48.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:04:48.418 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:04:48.418 | INFO | Total mem usage: 26.5 GiB | 20260506 18:04:48.418 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:04:48.418 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:04:48.418 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:04:48.418 | INFO | High Water Breakdown: | 20260506 18:04:48.418 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:04:48.418 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:04:48.418 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:04:48.418 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:04:48.418 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:04:48.418 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:04:48.418 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:04:48.418 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:04:48.418 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:04:48.418 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:04:48.418 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:04:48.418 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:04:48.419 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:04:48.419 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:04:48.419 | INFO | ------------------------------------------ | 20260506 18:04:48.419 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:04:48.419 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:04:48.419 | INFO | Total mem usage: 25.6 GiB | 20260506 18:04:48.419 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:04:48.419 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:04:48.419 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:04:48.419 | INFO | High Water Breakdown: | 20260506 18:04:48.419 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:04:48.419 | INFO | ------------------------------------------ | 20260506 18:04:48.419 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:04:48.419 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:04:48.419 | INFO | Total mem usage: 15.7 GiB | 20260506 18:04:48.419 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:04:48.419 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:04:48.419 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 18:04:48.419 | INFO | High Water Breakdown: | 20260506 18:04:48.419 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:04:48.419 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:04:48.419 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:04:48.419 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:04:48.419 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:04:48.419 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:04:48.419 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:04:48.419 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:04:48.419 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:04:48.419 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:04:48.419 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:04:48.419 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:04:48.419 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:04:48.419 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:04:48.419 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:04:48.419 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:04:48.419 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:04:48.419 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:04:48.419 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:04:48.419 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:04:48.419 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:04:48.419 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:04:48.419 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:04:48.419 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:04:48.419 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:04:48.419 | INFO | ------------------------------------------ | 20260506 18:04:48.419 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:04:48.419 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:04:48.419 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:04:48.419 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:04:48.419 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.58e+01, 1.34e+01 / 2.61e+00 | 20260506 18:04:48.419 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 9.94e-03 / 5.86e-03 | 20260506 18:04:48.419 | INFO | PerfReport: 11.63%, Baseline -- 3.64e+00 / 4.40e+00, 5.64e-01 / 3.45e-01 | 20260506 18:04:48.419 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.47e+00, 1.87e-01 / 1.35e-01 | 20260506 18:04:48.419 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.45e+01, 4.07e+00 / 2.46e+00 | 20260506 18:04:48.419 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.43e+00 / 2.11e+00 | 20260506 18:04:48.419 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.26e+00 / 2.81e+00, 3.77e-01 / 2.31e-01 | 20260506 18:04:48.419 | INFO | PerfReport: 3.42%, Metrics -- 1.07e+00 / 1.40e+00, 2.83e-01 / 1.99e-01 | 20260506 18:04:48.419 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.70e-01 / 4.13e-01, 5.77e-02 / 2.07e-02 | 20260506 18:04:48.419 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:04:48.419 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:04:48.419 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:04:48.419 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 18:04:48.419 | INFO | PerfReport: 21.43%, Upload -- 2.08e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 18:04:48.419 | INFO | PerfReport: 21.15%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 18:04:48.419 | INFO | PerfReport: 32.22%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 18:04:48.419 | INFO | PerfReport: 1.40%, LockUpload -- 1.36e+00 / 0.00e+00, 5.43e+00 / 0.00e+00 | 20260506 18:04:48.419 | INFO | PerfReport: 1.13%, LockDownload -- 1.10e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 18:04:48.419 | INFO | PerfReport: 22.66%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:04:48.419 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260506 18:04:48.538 | INFO | Starting a round of Baz IO | 20260506 18:04:51.674 | INFO | Populating RTMetrics report | 20260506 18:04:51.675 | INFO | RTMetrics report populated | 20260506 18:04:51.675 | INFO | Opening tmp RTMetrics json file | 20260506 18:04:51.700 | INFO | Serializing RTMetrics to json | 20260506 18:04:51.704 | INFO | Serialization done, swapping tmp file now... | 20260506 18:04:51.707 | INFO | RTMetrics done | 20260506 18:04:52.072 | INFO | Finished a round of Baz IO: Wrote 2.048 GiB in 3.534 seconds | 20260506 18:04:52.072 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.27% / 0.01% / 64.72% | 20260506 18:04:52.072 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.20632% / 1.19209e-05% / 4.76837e-05% | 20260506 18:04:52.072 | INFO | Baz IO overall: | 20260506 18:04:52.072 | INFO | BytesWritten : 1777539639408 | 20260506 18:04:52.072 | INFO | NumEvents : 859755712799 | 20260506 18:04:53.556 | INFO | Analyzing chunk frames = [3366912,3367424) | 20260506 18:04:53.556 | INFO | MemInfo: { | 20260506 18:04:53.556 | INFO | "gpu" : " 17 GB", | 20260506 18:04:53.556 | INFO | "huge" : "", | 20260506 18:04:53.556 | INFO | "huge_shared" : " 27 GB", | 20260506 18:04:53.556 | INFO | "malloc" : " 14 GB", | 20260506 18:04:53.556 | INFO | "pinned" : " 29 GB" | 20260506 18:04:53.556 | INFO | } | 20260506 18:04:53.556 | INFO | Perf report for the last 5.13837 seconds: | 20260506 18:04:53.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:04:53.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:53.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:53.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:53.556 | INFO | ANALYSIS : 62.8%, 2.9, 95.6ms, 0 | 20260506 18:04:53.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:53.556 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 18:04:53.556 | INFO | PRE_HQ : 60.3%, 1.9, 58.0ms, 0 | 20260506 18:04:53.556 | INFO | BAZWRITER : 68.4%, 4.8, 169.9ms, 0 | 20260506 18:04:53.556 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 18:04:53.557 | INFO | Uploaded 36GiB to GPU in 2.04891s (17.5703GiB/s) | 20260506 18:04:53.557 | INFO | Downloaded 38GiB to GPU in 2.21914s (17.1237GiB/s) | 20260506 18:04:58.697 | INFO | Analyzing chunk frames = [3367424,3367936) | 20260506 18:04:58.697 | INFO | MemInfo: { | 20260506 18:04:58.697 | INFO | "gpu" : " 17 GB", | 20260506 18:04:58.697 | INFO | "huge" : "", | 20260506 18:04:58.697 | INFO | "huge_shared" : " 27 GB", | 20260506 18:04:58.697 | INFO | "malloc" : " 14 GB", | 20260506 18:04:58.697 | INFO | "pinned" : " 29 GB" | 20260506 18:04:58.697 | INFO | } | 20260506 18:04:58.698 | INFO | Perf report for the last 5.14127 seconds: | 20260506 18:04:58.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:04:58.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:58.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:58.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:58.698 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:04:58.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:58.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:58.698 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:04:58.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:58.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:04:58.698 | INFO | Uploaded 36GiB to GPU in 2.0252s (17.776GiB/s) | 20260506 18:04:58.698 | INFO | Downloaded 34GiB to GPU in 1.97197s (17.2417GiB/s) | 20260506 18:05:03.844 | INFO | Analyzing chunk frames = [3367936,3368448) | 20260506 18:05:03.844 | INFO | MemInfo: { | 20260506 18:05:03.844 | INFO | "gpu" : " 17 GB", | 20260506 18:05:03.844 | INFO | "huge" : "", | 20260506 18:05:03.844 | INFO | "huge_shared" : " 27 GB", | 20260506 18:05:03.844 | INFO | "malloc" : " 14 GB", | 20260506 18:05:03.844 | INFO | "pinned" : " 29 GB" | 20260506 18:05:03.844 | INFO | } | 20260506 18:05:03.844 | INFO | Perf report for the last 5.14626 seconds: | 20260506 18:05:03.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:05:03.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:03.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:03.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:03.844 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 18:05:03.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:03.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:03.844 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:05:03.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:03.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:03.844 | INFO | Uploaded 36GiB to GPU in 1.97978s (18.1839GiB/s) | 20260506 18:05:03.844 | INFO | Downloaded 34GiB to GPU in 1.92781s (17.6366GiB/s) | 20260506 18:05:08.985 | INFO | Analyzing chunk frames = [3368448,3368960) | 20260506 18:05:08.985 | INFO | MemInfo: { | 20260506 18:05:08.985 | INFO | "gpu" : " 17 GB", | 20260506 18:05:08.985 | INFO | "huge" : "", | 20260506 18:05:08.985 | INFO | "huge_shared" : " 27 GB", | 20260506 18:05:08.985 | INFO | "malloc" : " 14 GB", | 20260506 18:05:08.985 | INFO | "pinned" : " 29 GB" | 20260506 18:05:08.985 | INFO | } | 20260506 18:05:08.985 | INFO | Perf report for the last 5.14146 seconds: | 20260506 18:05:08.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:05:08.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:08.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:08.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:08.985 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:05:08.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:08.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:08.985 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:05:08.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:08.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:08.985 | INFO | Uploaded 36GiB to GPU in 1.94832s (18.4774GiB/s) | 20260506 18:05:08.986 | INFO | Downloaded 34GiB to GPU in 1.86775s (18.2037GiB/s) | 20260506 18:05:14.124 | INFO | Analyzing chunk frames = [3368960,3369472) | 20260506 18:05:14.124 | INFO | MemInfo: { | 20260506 18:05:14.124 | INFO | "gpu" : " 17 GB", | 20260506 18:05:14.124 | INFO | "huge" : "", | 20260506 18:05:14.124 | INFO | "huge_shared" : " 27 GB", | 20260506 18:05:14.124 | INFO | "malloc" : " 14 GB", | 20260506 18:05:14.124 | INFO | "pinned" : " 29 GB" | 20260506 18:05:14.124 | INFO | } | 20260506 18:05:14.124 | INFO | Perf report for the last 5.13885 seconds: | 20260506 18:05:14.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:05:14.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:14.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:14.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:14.124 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 18:05:14.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:14.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:14.124 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 18:05:14.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:14.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:14.124 | INFO | Uploaded 36GiB to GPU in 2.01254s (17.8878GiB/s) | 20260506 18:05:14.124 | INFO | Downloaded 34GiB to GPU in 1.91609s (17.7445GiB/s) | 20260506 18:05:19.264 | INFO | Analyzing chunk frames = [3369472,3369984) | 20260506 18:05:19.264 | INFO | MemInfo: { | 20260506 18:05:19.264 | INFO | "gpu" : " 17 GB", | 20260506 18:05:19.264 | INFO | "huge" : "", | 20260506 18:05:19.264 | INFO | "huge_shared" : " 27 GB", | 20260506 18:05:19.264 | INFO | "malloc" : " 14 GB", | 20260506 18:05:19.264 | INFO | "pinned" : " 29 GB" | 20260506 18:05:19.264 | INFO | } | 20260506 18:05:19.264 | INFO | Perf report for the last 5.13987 seconds: | 20260506 18:05:19.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:05:19.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:19.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:19.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:19.264 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 18:05:19.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:19.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:19.264 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:05:19.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:19.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:19.264 | INFO | Uploaded 36GiB to GPU in 1.99686s (18.0283GiB/s) | 20260506 18:05:19.264 | INFO | Downloaded 34GiB to GPU in 1.93646s (17.5578GiB/s) | 20260506 18:05:24.395 | INFO | Analyzing chunk frames = [3369984,3370496) | 20260506 18:05:24.395 | INFO | MemInfo: { | 20260506 18:05:24.395 | INFO | "gpu" : " 17 GB", | 20260506 18:05:24.395 | INFO | "huge" : "", | 20260506 18:05:24.395 | INFO | "huge_shared" : " 27 GB", | 20260506 18:05:24.395 | INFO | "malloc" : " 14 GB", | 20260506 18:05:24.395 | INFO | "pinned" : " 29 GB" | 20260506 18:05:24.395 | INFO | } | 20260506 18:05:24.395 | INFO | Perf report for the last 5.13123 seconds: | 20260506 18:05:24.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:05:24.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:24.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:24.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:24.395 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260506 18:05:24.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:24.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:24.395 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:05:24.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:24.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:24.395 | INFO | Uploaded 36GiB to GPU in 1.99347s (18.059GiB/s) | 20260506 18:05:24.396 | INFO | Downloaded 34GiB to GPU in 1.99515s (17.0414GiB/s) | 20260506 18:05:29.533 | INFO | Analyzing chunk frames = [3370496,3371008) | 20260506 18:05:29.533 | INFO | MemInfo: { | 20260506 18:05:29.533 | INFO | "gpu" : " 17 GB", | 20260506 18:05:29.533 | INFO | "huge" : "", | 20260506 18:05:29.533 | INFO | "huge_shared" : " 27 GB", | 20260506 18:05:29.533 | INFO | "malloc" : " 14 GB", | 20260506 18:05:29.533 | INFO | "pinned" : " 29 GB" | 20260506 18:05:29.533 | INFO | } | 20260506 18:05:29.533 | INFO | Perf report for the last 5.13751 seconds: | 20260506 18:05:29.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:05:29.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:29.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:29.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:29.533 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:05:29.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:29.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:29.533 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 18:05:29.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:29.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:29.533 | INFO | Uploaded 36GiB to GPU in 1.99935s (18.0058GiB/s) | 20260506 18:05:29.533 | INFO | Downloaded 34GiB to GPU in 1.97446s (17.2199GiB/s) | 20260506 18:05:29.649 | INFO | Starting a round of Baz IO | 20260506 18:05:32.806 | INFO | Populating RTMetrics report | 20260506 18:05:32.807 | INFO | RTMetrics report populated | 20260506 18:05:32.807 | INFO | Opening tmp RTMetrics json file | 20260506 18:05:32.851 | INFO | Serializing RTMetrics to json | 20260506 18:05:32.885 | INFO | Serialization done, swapping tmp file now... | 20260506 18:05:32.891 | INFO | RTMetrics done | 20260506 18:05:33.016 | INFO | Finished a round of Baz IO: Wrote 2.047 GiB in 3.367 seconds | 20260506 18:05:33.016 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.26% / 0.01% / 64.73% | 20260506 18:05:33.017 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.20412% / 1.19209e-05% / 7.15256e-05% | 20260506 18:05:33.017 | INFO | Baz IO overall: | 20260506 18:05:33.017 | INFO | BytesWritten : 1779737938407 | 20260506 18:05:33.017 | INFO | NumEvents : 860815217134 | 20260506 18:05:34.678 | INFO | Analyzing chunk frames = [3371008,3371520) | 20260506 18:05:34.678 | INFO | MemInfo: { | 20260506 18:05:34.678 | INFO | "gpu" : " 17 GB", | 20260506 18:05:34.678 | INFO | "huge" : "", | 20260506 18:05:34.678 | INFO | "huge_shared" : " 27 GB", | 20260506 18:05:34.678 | INFO | "malloc" : " 14 GB", | 20260506 18:05:34.678 | INFO | "pinned" : " 29 GB" | 20260506 18:05:34.678 | INFO | } | 20260506 18:05:34.678 | INFO | Perf report for the last 5.14509 seconds: | 20260506 18:05:34.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:05:34.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:34.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:34.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:34.678 | INFO | ANALYSIS : 62.7%, 2.8, 93.7ms, 0 | 20260506 18:05:34.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:34.678 | INFO | RT_METRICS : 48.1%, 1.0, 25.2ms, 0 | 20260506 18:05:34.678 | INFO | PRE_HQ : 60.8%, 2.0, 62.1ms, 0 | 20260506 18:05:34.678 | INFO | BAZWRITER : 65.2%, 4.8, 160.5ms, 0 | 20260506 18:05:34.678 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 18:05:34.678 | INFO | Uploaded 36GiB to GPU in 2.01205s (17.8922GiB/s) | 20260506 18:05:34.678 | INFO | Downloaded 38GiB to GPU in 2.19241s (17.3325GiB/s) | 20260506 18:05:39.817 | INFO | Analyzing chunk frames = [3371520,3372032) | 20260506 18:05:39.817 | INFO | MemInfo: { | 20260506 18:05:39.817 | INFO | "gpu" : " 17 GB", | 20260506 18:05:39.817 | INFO | "huge" : "", | 20260506 18:05:39.817 | INFO | "huge_shared" : " 27 GB", | 20260506 18:05:39.817 | INFO | "malloc" : " 14 GB", | 20260506 18:05:39.818 | INFO | "pinned" : " 29 GB" | 20260506 18:05:39.818 | INFO | } | 20260506 18:05:39.818 | INFO | Perf report for the last 5.1397 seconds: | 20260506 18:05:39.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:05:39.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:39.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:39.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:39.818 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:05:39.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:39.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:39.818 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:05:39.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:39.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:39.818 | INFO | Uploaded 36GiB to GPU in 1.995s (18.0451GiB/s) | 20260506 18:05:39.818 | INFO | Downloaded 34GiB to GPU in 1.91883s (17.7191GiB/s) | 20260506 18:05:44.957 | INFO | Analyzing chunk frames = [3372032,3372544) | 20260506 18:05:44.957 | INFO | MemInfo: { | 20260506 18:05:44.957 | INFO | "gpu" : " 17 GB", | 20260506 18:05:44.957 | INFO | "huge" : "", | 20260506 18:05:44.957 | INFO | "huge_shared" : " 27 GB", | 20260506 18:05:44.957 | INFO | "malloc" : " 14 GB", | 20260506 18:05:44.957 | INFO | "pinned" : " 29 GB" | 20260506 18:05:44.957 | INFO | } | 20260506 18:05:44.957 | INFO | Perf report for the last 5.13923 seconds: | 20260506 18:05:44.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:05:44.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:44.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:44.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:44.957 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:05:44.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:44.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:44.957 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:05:44.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:44.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:44.957 | INFO | Uploaded 36GiB to GPU in 1.95909s (18.3758GiB/s) | 20260506 18:05:44.957 | INFO | Downloaded 34GiB to GPU in 1.92336s (17.6774GiB/s) | 20260506 18:05:50.097 | INFO | Analyzing chunk frames = [3372544,3373056) | 20260506 18:05:50.097 | INFO | MemInfo: { | 20260506 18:05:50.097 | INFO | "gpu" : " 17 GB", | 20260506 18:05:50.097 | INFO | "huge" : "", | 20260506 18:05:50.097 | INFO | "huge_shared" : " 27 GB", | 20260506 18:05:50.097 | INFO | "malloc" : " 14 GB", | 20260506 18:05:50.097 | INFO | "pinned" : " 29 GB" | 20260506 18:05:50.097 | INFO | } | 20260506 18:05:50.097 | INFO | Perf report for the last 5.13991 seconds: | 20260506 18:05:50.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:05:50.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:50.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:50.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:50.097 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:05:50.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:50.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:50.097 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:05:50.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:50.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:50.097 | INFO | Uploaded 36GiB to GPU in 1.93087s (18.6444GiB/s) | 20260506 18:05:50.097 | INFO | Downloaded 34GiB to GPU in 1.86998s (18.182GiB/s) | 20260506 18:05:55.234 | INFO | Analyzing chunk frames = [3373056,3373568) | 20260506 18:05:55.234 | INFO | MemInfo: { | 20260506 18:05:55.234 | INFO | "gpu" : " 17 GB", | 20260506 18:05:55.234 | INFO | "huge" : "", | 20260506 18:05:55.234 | INFO | "huge_shared" : " 27 GB", | 20260506 18:05:55.234 | INFO | "malloc" : " 14 GB", | 20260506 18:05:55.234 | INFO | "pinned" : " 29 GB" | 20260506 18:05:55.234 | INFO | } | 20260506 18:05:55.234 | INFO | Perf report for the last 5.13716 seconds: | 20260506 18:05:55.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:05:55.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:55.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:55.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:55.234 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 18:05:55.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:55.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:55.234 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:05:55.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:55.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:05:55.234 | INFO | Uploaded 36GiB to GPU in 1.95899s (18.3768GiB/s) | 20260506 18:05:55.234 | INFO | Downloaded 34GiB to GPU in 1.88143s (18.0713GiB/s) | 20260506 18:06:00.370 | INFO | Analyzing chunk frames = [3373568,3374080) | 20260506 18:06:00.370 | INFO | MemInfo: { | 20260506 18:06:00.370 | INFO | "gpu" : " 17 GB", | 20260506 18:06:00.370 | INFO | "huge" : "", | 20260506 18:06:00.370 | INFO | "huge_shared" : " 27 GB", | 20260506 18:06:00.370 | INFO | "malloc" : " 14 GB", | 20260506 18:06:00.370 | INFO | "pinned" : " 29 GB" | 20260506 18:06:00.370 | INFO | } | 20260506 18:06:00.370 | INFO | Perf report for the last 5.13658 seconds: | 20260506 18:06:00.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:06:00.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:00.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:00.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:00.371 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 18:06:00.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:00.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:00.371 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:06:00.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:00.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:00.371 | INFO | Uploaded 36GiB to GPU in 2.01775s (17.8416GiB/s) | 20260506 18:06:00.371 | INFO | Downloaded 34GiB to GPU in 1.93986s (17.5271GiB/s) | 20260506 18:06:05.512 | INFO | Analyzing chunk frames = [3374080,3374592) | 20260506 18:06:05.512 | INFO | MemInfo: { | 20260506 18:06:05.512 | INFO | "gpu" : " 17 GB", | 20260506 18:06:05.512 | INFO | "huge" : "", | 20260506 18:06:05.512 | INFO | "huge_shared" : " 27 GB", | 20260506 18:06:05.512 | INFO | "malloc" : " 14 GB", | 20260506 18:06:05.512 | INFO | "pinned" : " 29 GB" | 20260506 18:06:05.512 | INFO | } | 20260506 18:06:05.512 | INFO | Perf report for the last 5.14176 seconds: | 20260506 18:06:05.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:06:05.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:05.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:05.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:05.512 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:06:05.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:05.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:05.512 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 18:06:05.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:05.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:05.512 | INFO | Uploaded 36GiB to GPU in 2.00151s (17.9864GiB/s) | 20260506 18:06:05.513 | INFO | Downloaded 34GiB to GPU in 2.01019s (16.9139GiB/s) | 20260506 18:06:10.649 | INFO | Analyzing chunk frames = [3374592,3375104) | 20260506 18:06:10.649 | INFO | MemInfo: { | 20260506 18:06:10.649 | INFO | "gpu" : " 17 GB", | 20260506 18:06:10.649 | INFO | "huge" : "", | 20260506 18:06:10.649 | INFO | "huge_shared" : " 27 GB", | 20260506 18:06:10.649 | INFO | "malloc" : " 14 GB", | 20260506 18:06:10.649 | INFO | "pinned" : " 29 GB" | 20260506 18:06:10.649 | INFO | } | 20260506 18:06:10.649 | INFO | Perf report for the last 5.13648 seconds: | 20260506 18:06:10.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:06:10.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:10.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:10.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:10.649 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 18:06:10.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:10.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:10.649 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 18:06:10.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:10.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:10.649 | INFO | Uploaded 36GiB to GPU in 2.00968s (17.9133GiB/s) | 20260506 18:06:10.649 | INFO | Downloaded 34GiB to GPU in 1.99583s (17.0355GiB/s) | 20260506 18:06:10.649 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:06:10.649 | INFO | Total mem usage: 18 GiB | 20260506 18:06:10.649 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:06:10.649 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -13.9%) | 20260506 18:06:10.649 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:06:10.649 | INFO | High Water Breakdown: | 20260506 18:06:10.649 | INFO | 61.6% : 9.74 GiB ( 85.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:06:10.649 | INFO | 38.3% : 6.05 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:06:10.649 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:06:10.649 | INFO | ------------------------------------------ | 20260506 18:06:10.649 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:06:10.649 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:06:10.649 | INFO | Total mem usage: 26.5 GiB | 20260506 18:06:10.649 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:06:10.649 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:06:10.649 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:06:10.649 | INFO | High Water Breakdown: | 20260506 18:06:10.649 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:06:10.649 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:06:10.649 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:06:10.649 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:06:10.649 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:06:10.649 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:06:10.649 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:06:10.649 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:06:10.649 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:06:10.649 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:06:10.649 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:06:10.649 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:06:10.649 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:06:10.649 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:06:10.649 | INFO | ------------------------------------------ | 20260506 18:06:10.649 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:06:10.649 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:06:10.649 | INFO | Total mem usage: 25.6 GiB | 20260506 18:06:10.649 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:06:10.649 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:06:10.649 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:06:10.649 | INFO | High Water Breakdown: | 20260506 18:06:10.649 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:06:10.649 | INFO | ------------------------------------------ | 20260506 18:06:10.650 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:06:10.650 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:06:10.650 | INFO | Total mem usage: 15.7 GiB | 20260506 18:06:10.650 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:06:10.650 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:06:10.650 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:06:10.650 | INFO | High Water Breakdown: | 20260506 18:06:10.650 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:06:10.650 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:06:10.650 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:06:10.650 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:06:10.650 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:06:10.650 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:06:10.650 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:06:10.650 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:06:10.650 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:06:10.650 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:06:10.650 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:06:10.650 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:06:10.650 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:06:10.650 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:06:10.650 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:06:10.650 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:06:10.650 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:06:10.650 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:06:10.650 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:06:10.650 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:06:10.650 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:06:10.650 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:06:10.650 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:06:10.650 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:06:10.650 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:06:10.650 | INFO | ------------------------------------------ | 20260506 18:06:10.650 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:06:10.650 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:06:10.650 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:06:10.650 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:06:10.650 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.56e+01, 1.34e+01 / 2.70e+00 | 20260506 18:06:10.650 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 9.99e-02, 1.37e-02 / 6.34e-03 | 20260506 18:06:10.650 | INFO | PerfReport: 11.60%, Baseline -- 3.63e+00 / 4.42e+00, 5.52e-01 / 3.41e-01 | 20260506 18:06:10.650 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.82e-01 / 1.42e-01 | 20260506 18:06:10.650 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.49e+01, 4.29e+00 / 3.02e+00 | 20260506 18:06:10.650 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.78e+01 / 2.01e+01, 1.36e+00 / 2.03e+00 | 20260506 18:06:10.650 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.27e+00 / 2.83e+00, 3.93e-01 / 2.21e-01 | 20260506 18:06:10.650 | INFO | PerfReport: 3.42%, Metrics -- 1.07e+00 / 1.42e+00, 2.76e-01 / 1.82e-01 | 20260506 18:06:10.650 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.70e-01 / 4.22e-01, 5.78e-02 / 2.08e-02 | 20260506 18:06:10.650 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:06:10.650 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:06:10.650 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:06:10.650 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 18:06:10.650 | INFO | PerfReport: 21.41%, Upload -- 2.08e+01 / 0.00e+00, 6.90e+00 / 0.00e+00 | 20260506 18:06:10.650 | INFO | PerfReport: 21.17%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 18:06:10.650 | INFO | PerfReport: 32.24%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 18:06:10.650 | INFO | PerfReport: 1.42%, LockUpload -- 1.38e+00 / 0.00e+00, 5.55e+00 / 0.00e+00 | 20260506 18:06:10.650 | INFO | PerfReport: 1.15%, LockDownload -- 1.12e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 18:06:10.650 | INFO | PerfReport: 22.60%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 18:06:10.650 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260506 18:06:10.766 | INFO | Starting a round of Baz IO | 20260506 18:06:13.946 | INFO | Populating RTMetrics report | 20260506 18:06:13.947 | INFO | RTMetrics report populated | 20260506 18:06:13.947 | INFO | Opening tmp RTMetrics json file | 20260506 18:06:13.964 | INFO | Serializing RTMetrics to json | 20260506 18:06:13.968 | INFO | Serialization done, swapping tmp file now... | 20260506 18:06:13.971 | INFO | RTMetrics done | 20260506 18:06:14.663 | INFO | Finished a round of Baz IO: Wrote 2.048 GiB in 3.897 seconds | 20260506 18:06:14.663 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.26% / 0.01% / 64.74% | 20260506 18:06:14.663 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.20186% / 7.94729e-06% / 5.96046e-05% | 20260506 18:06:14.663 | INFO | Baz IO overall: | 20260506 18:06:14.663 | INFO | BytesWritten : 1781936873076 | 20260506 18:06:14.663 | INFO | NumEvents : 861874965554 | 20260506 18:06:15.787 | INFO | Analyzing chunk frames = [3375104,3375616) | 20260506 18:06:15.787 | INFO | MemInfo: { | 20260506 18:06:15.787 | INFO | "gpu" : " 17 GB", | 20260506 18:06:15.787 | INFO | "huge" : "", | 20260506 18:06:15.787 | INFO | "huge_shared" : " 27 GB", | 20260506 18:06:15.787 | INFO | "malloc" : " 14 GB", | 20260506 18:06:15.787 | INFO | "pinned" : " 29 GB" | 20260506 18:06:15.787 | INFO | } | 20260506 18:06:15.787 | INFO | Perf report for the last 5.13853 seconds: | 20260506 18:06:15.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:06:15.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:15.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:15.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:15.787 | INFO | ANALYSIS : 62.8%, 2.9, 95.7ms, 0 | 20260506 18:06:15.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:15.787 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260506 18:06:15.787 | INFO | PRE_HQ : 60.9%, 1.9, 60.3ms, 0 | 20260506 18:06:15.787 | INFO | BAZWRITER : 75.5%, 4.8, 186.7ms, 0 | 20260506 18:06:15.787 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 18:06:15.788 | INFO | Uploaded 36GiB to GPU in 2.03114s (17.7241GiB/s) | 20260506 18:06:15.788 | INFO | Downloaded 38GiB to GPU in 2.22584s (17.0722GiB/s) | 20260506 18:06:20.938 | INFO | Analyzing chunk frames = [3375616,3376128) | 20260506 18:06:20.938 | INFO | MemInfo: { | 20260506 18:06:20.938 | INFO | "gpu" : " 17 GB", | 20260506 18:06:20.938 | INFO | "huge" : "", | 20260506 18:06:20.938 | INFO | "huge_shared" : " 27 GB", | 20260506 18:06:20.938 | INFO | "malloc" : " 14 GB", | 20260506 18:06:20.938 | INFO | "pinned" : " 29 GB" | 20260506 18:06:20.938 | INFO | } | 20260506 18:06:20.938 | INFO | Perf report for the last 5.15091 seconds: | 20260506 18:06:20.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:06:20.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:20.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:20.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:20.938 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 18:06:20.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:20.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:20.938 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 18:06:20.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:20.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:20.938 | INFO | Uploaded 36GiB to GPU in 1.99088s (18.0824GiB/s) | 20260506 18:06:20.938 | INFO | Downloaded 34GiB to GPU in 1.93863s (17.5382GiB/s) | 20260506 18:06:26.073 | INFO | Analyzing chunk frames = [3376128,3376640) | 20260506 18:06:26.073 | INFO | MemInfo: { | 20260506 18:06:26.073 | INFO | "gpu" : " 17 GB", | 20260506 18:06:26.073 | INFO | "huge" : "", | 20260506 18:06:26.073 | INFO | "huge_shared" : " 27 GB", | 20260506 18:06:26.073 | INFO | "malloc" : " 14 GB", | 20260506 18:06:26.074 | INFO | "pinned" : " 29 GB" | 20260506 18:06:26.074 | INFO | } | 20260506 18:06:26.074 | INFO | Perf report for the last 5.13535 seconds: | 20260506 18:06:26.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:06:26.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:26.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:26.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:26.074 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 18:06:26.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:26.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:26.074 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:06:26.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:26.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:26.074 | INFO | Uploaded 36GiB to GPU in 1.96066s (18.3612GiB/s) | 20260506 18:06:26.074 | INFO | Downloaded 34GiB to GPU in 1.91857s (17.7215GiB/s) | 20260506 18:06:31.211 | INFO | Analyzing chunk frames = [3376640,3377152) | 20260506 18:06:31.211 | INFO | MemInfo: { | 20260506 18:06:31.211 | INFO | "gpu" : " 17 GB", | 20260506 18:06:31.211 | INFO | "huge" : "", | 20260506 18:06:31.211 | INFO | "huge_shared" : " 27 GB", | 20260506 18:06:31.211 | INFO | "malloc" : " 14 GB", | 20260506 18:06:31.211 | INFO | "pinned" : " 29 GB" | 20260506 18:06:31.211 | INFO | } | 20260506 18:06:31.211 | INFO | Perf report for the last 5.13736 seconds: | 20260506 18:06:31.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:06:31.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:31.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:31.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:31.211 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:06:31.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:31.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:31.211 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:06:31.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:31.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:31.212 | INFO | Uploaded 36GiB to GPU in 1.92505s (18.7008GiB/s) | 20260506 18:06:31.212 | INFO | Downloaded 34GiB to GPU in 1.86922s (18.1895GiB/s) | 20260506 18:06:36.347 | INFO | Analyzing chunk frames = [3377152,3377664) | 20260506 18:06:36.347 | INFO | MemInfo: { | 20260506 18:06:36.347 | INFO | "gpu" : " 17 GB", | 20260506 18:06:36.347 | INFO | "huge" : "", | 20260506 18:06:36.347 | INFO | "huge_shared" : " 27 GB", | 20260506 18:06:36.348 | INFO | "malloc" : " 14 GB", | 20260506 18:06:36.348 | INFO | "pinned" : " 29 GB" | 20260506 18:06:36.348 | INFO | } | 20260506 18:06:36.348 | INFO | Perf report for the last 5.13661 seconds: | 20260506 18:06:36.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:06:36.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:36.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:36.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:36.348 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 18:06:36.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:36.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:36.348 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:06:36.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:36.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:36.348 | INFO | Uploaded 36GiB to GPU in 2.01602s (17.857GiB/s) | 20260506 18:06:36.348 | INFO | Downloaded 34GiB to GPU in 1.91893s (17.7182GiB/s) | 20260506 18:06:41.484 | INFO | Analyzing chunk frames = [3377664,3378176) | 20260506 18:06:41.484 | INFO | MemInfo: { | 20260506 18:06:41.484 | INFO | "gpu" : " 17 GB", | 20260506 18:06:41.484 | INFO | "huge" : "", | 20260506 18:06:41.484 | INFO | "huge_shared" : " 27 GB", | 20260506 18:06:41.484 | INFO | "malloc" : " 14 GB", | 20260506 18:06:41.484 | INFO | "pinned" : " 29 GB" | 20260506 18:06:41.484 | INFO | } | 20260506 18:06:41.484 | INFO | Perf report for the last 5.13649 seconds: | 20260506 18:06:41.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:06:41.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:41.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:41.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:41.484 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 18:06:41.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:41.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:41.484 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 18:06:41.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:41.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:41.484 | INFO | Uploaded 36GiB to GPU in 2.00889s (17.9204GiB/s) | 20260506 18:06:41.484 | INFO | Downloaded 34GiB to GPU in 1.94719s (17.461GiB/s) | 20260506 18:06:46.626 | INFO | Analyzing chunk frames = [3378176,3378688) | 20260506 18:06:46.626 | INFO | MemInfo: { | 20260506 18:06:46.626 | INFO | "gpu" : " 17 GB", | 20260506 18:06:46.626 | INFO | "huge" : "", | 20260506 18:06:46.626 | INFO | "huge_shared" : " 27 GB", | 20260506 18:06:46.627 | INFO | "malloc" : " 14 GB", | 20260506 18:06:46.627 | INFO | "pinned" : " 29 GB" | 20260506 18:06:46.627 | INFO | } | 20260506 18:06:46.627 | INFO | Perf report for the last 5.14268 seconds: | 20260506 18:06:46.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:06:46.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:46.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:46.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:46.627 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:06:46.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:46.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:46.627 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:06:46.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:46.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:46.627 | INFO | Uploaded 36GiB to GPU in 1.96741s (18.2981GiB/s) | 20260506 18:06:46.627 | INFO | Downloaded 34GiB to GPU in 2.00318s (16.973GiB/s) | 20260506 18:06:51.773 | INFO | Analyzing chunk frames = [3378688,3379200) | 20260506 18:06:51.773 | INFO | MemInfo: { | 20260506 18:06:51.773 | INFO | "gpu" : " 17 GB", | 20260506 18:06:51.773 | INFO | "huge" : "", | 20260506 18:06:51.773 | INFO | "huge_shared" : " 27 GB", | 20260506 18:06:51.773 | INFO | "malloc" : " 14 GB", | 20260506 18:06:51.773 | INFO | "pinned" : " 29 GB" | 20260506 18:06:51.773 | INFO | } | 20260506 18:06:51.773 | INFO | Perf report for the last 5.14642 seconds: | 20260506 18:06:51.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:06:51.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:51.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:51.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:51.773 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 18:06:51.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:51.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:51.773 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:06:51.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:51.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:51.773 | INFO | Uploaded 36GiB to GPU in 2.03458s (17.6941GiB/s) | 20260506 18:06:51.773 | INFO | Downloaded 34GiB to GPU in 1.98691s (17.112GiB/s) | 20260506 18:06:51.887 | INFO | Starting a round of Baz IO | 20260506 18:06:55.025 | INFO | Populating RTMetrics report | 20260506 18:06:55.025 | INFO | RTMetrics report populated | 20260506 18:06:55.025 | INFO | Opening tmp RTMetrics json file | 20260506 18:06:55.054 | INFO | Serializing RTMetrics to json | 20260506 18:06:55.076 | INFO | Serialization done, swapping tmp file now... | 20260506 18:06:55.095 | INFO | RTMetrics done | 20260506 18:06:55.264 | INFO | Finished a round of Baz IO: Wrote 2.047 GiB in 3.376 seconds | 20260506 18:06:55.264 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.25% / 0.01% / 64.74% | 20260506 18:06:55.264 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.19661% / 1.98682e-05% / 6.75519e-05% | 20260506 18:06:55.264 | INFO | Baz IO overall: | 20260506 18:06:55.264 | INFO | BytesWritten : 1784134815478 | 20260506 18:06:55.264 | INFO | NumEvents : 862934309730 | 20260506 18:06:56.904 | INFO | Analyzing chunk frames = [3379200,3379712) | 20260506 18:06:56.904 | INFO | MemInfo: { | 20260506 18:06:56.904 | INFO | "gpu" : " 17 GB", | 20260506 18:06:56.904 | INFO | "huge" : "", | 20260506 18:06:56.904 | INFO | "huge_shared" : " 27 GB", | 20260506 18:06:56.904 | INFO | "malloc" : " 14 GB", | 20260506 18:06:56.904 | INFO | "pinned" : " 29 GB" | 20260506 18:06:56.904 | INFO | } | 20260506 18:06:56.904 | INFO | Perf report for the last 5.13113 seconds: | 20260506 18:06:56.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:06:56.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:56.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:56.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:56.904 | INFO | ANALYSIS : 62.7%, 2.9, 95.2ms, 0 | 20260506 18:06:56.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:06:56.904 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 18:06:56.904 | INFO | PRE_HQ : 60.3%, 1.9, 58.1ms, 0 | 20260506 18:06:56.904 | INFO | BAZWRITER : 65.5%, 4.8, 161.4ms, 0 | 20260506 18:06:56.904 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 18:06:56.905 | INFO | Uploaded 36GiB to GPU in 2.04151s (17.634GiB/s) | 20260506 18:06:56.905 | INFO | Downloaded 38GiB to GPU in 2.20058s (17.2681GiB/s) | 20260506 18:07:02.045 | INFO | Analyzing chunk frames = [3379712,3380224) | 20260506 18:07:02.045 | INFO | MemInfo: { | 20260506 18:07:02.045 | INFO | "gpu" : " 17 GB", | 20260506 18:07:02.045 | INFO | "huge" : "", | 20260506 18:07:02.045 | INFO | "huge_shared" : " 27 GB", | 20260506 18:07:02.045 | INFO | "malloc" : " 14 GB", | 20260506 18:07:02.045 | INFO | "pinned" : " 29 GB" | 20260506 18:07:02.045 | INFO | } | 20260506 18:07:02.045 | INFO | Perf report for the last 5.14102 seconds: | 20260506 18:07:02.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:07:02.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:02.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:02.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:02.045 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 18:07:02.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:02.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:02.045 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:07:02.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:02.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:02.046 | INFO | Uploaded 36GiB to GPU in 1.99531s (18.0423GiB/s) | 20260506 18:07:02.046 | INFO | Downloaded 34GiB to GPU in 1.91949s (17.713GiB/s) | 20260506 18:07:07.187 | INFO | Analyzing chunk frames = [3380224,3380736) | 20260506 18:07:07.187 | INFO | MemInfo: { | 20260506 18:07:07.187 | INFO | "gpu" : " 17 GB", | 20260506 18:07:07.187 | INFO | "huge" : "", | 20260506 18:07:07.187 | INFO | "huge_shared" : " 27 GB", | 20260506 18:07:07.187 | INFO | "malloc" : " 14 GB", | 20260506 18:07:07.187 | INFO | "pinned" : " 29 GB" | 20260506 18:07:07.187 | INFO | } | 20260506 18:07:07.187 | INFO | Perf report for the last 5.14135 seconds: | 20260506 18:07:07.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:07:07.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:07.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:07.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:07.187 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 18:07:07.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:07.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:07.187 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:07:07.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:07.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:07.187 | INFO | Uploaded 36GiB to GPU in 1.97241s (18.2518GiB/s) | 20260506 18:07:07.187 | INFO | Downloaded 34GiB to GPU in 1.91686s (17.7374GiB/s) | 20260506 18:07:12.328 | INFO | Analyzing chunk frames = [3380736,3381248) | 20260506 18:07:12.328 | INFO | MemInfo: { | 20260506 18:07:12.328 | INFO | "gpu" : " 17 GB", | 20260506 18:07:12.329 | INFO | "huge" : "", | 20260506 18:07:12.329 | INFO | "huge_shared" : " 27 GB", | 20260506 18:07:12.329 | INFO | "malloc" : " 14 GB", | 20260506 18:07:12.329 | INFO | "pinned" : " 29 GB" | 20260506 18:07:12.329 | INFO | } | 20260506 18:07:12.329 | INFO | Perf report for the last 5.14221 seconds: | 20260506 18:07:12.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:07:12.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:12.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:12.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:12.329 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:07:12.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:12.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:12.329 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:07:12.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:12.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:12.329 | INFO | Uploaded 36GiB to GPU in 1.92988s (18.654GiB/s) | 20260506 18:07:12.329 | INFO | Downloaded 34GiB to GPU in 1.87127s (18.1695GiB/s) | 20260506 18:07:17.462 | INFO | Analyzing chunk frames = [3381248,3381760) | 20260506 18:07:17.462 | INFO | MemInfo: { | 20260506 18:07:17.462 | INFO | "gpu" : " 17 GB", | 20260506 18:07:17.462 | INFO | "huge" : "", | 20260506 18:07:17.462 | INFO | "huge_shared" : " 27 GB", | 20260506 18:07:17.462 | INFO | "malloc" : " 14 GB", | 20260506 18:07:17.462 | INFO | "pinned" : " 29 GB" | 20260506 18:07:17.462 | INFO | } | 20260506 18:07:17.462 | INFO | Perf report for the last 5.13278 seconds: | 20260506 18:07:17.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:07:17.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:17.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:17.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:17.462 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 18:07:17.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:17.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:17.462 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:07:17.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:17.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:17.462 | INFO | Uploaded 36GiB to GPU in 2.01814s (17.8382GiB/s) | 20260506 18:07:17.462 | INFO | Downloaded 34GiB to GPU in 1.91633s (17.7423GiB/s) | 20260506 18:07:22.601 | INFO | Analyzing chunk frames = [3381760,3382272) | 20260506 18:07:22.601 | INFO | MemInfo: { | 20260506 18:07:22.601 | INFO | "gpu" : " 17 GB", | 20260506 18:07:22.601 | INFO | "huge" : "", | 20260506 18:07:22.601 | INFO | "huge_shared" : " 27 GB", | 20260506 18:07:22.601 | INFO | "malloc" : " 14 GB", | 20260506 18:07:22.601 | INFO | "pinned" : " 29 GB" | 20260506 18:07:22.601 | INFO | } | 20260506 18:07:22.601 | INFO | Perf report for the last 5.13946 seconds: | 20260506 18:07:22.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:07:22.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:22.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:22.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:22.601 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 18:07:22.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:22.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:22.601 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:07:22.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:22.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:22.601 | INFO | Uploaded 36GiB to GPU in 2.02695s (17.7606GiB/s) | 20260506 18:07:22.601 | INFO | Downloaded 34GiB to GPU in 1.94443s (17.4859GiB/s) | 20260506 18:07:27.738 | INFO | Analyzing chunk frames = [3382272,3382784) | 20260506 18:07:27.738 | INFO | MemInfo: { | 20260506 18:07:27.738 | INFO | "gpu" : " 17 GB", | 20260506 18:07:27.738 | INFO | "huge" : "", | 20260506 18:07:27.738 | INFO | "huge_shared" : " 27 GB", | 20260506 18:07:27.738 | INFO | "malloc" : " 14 GB", | 20260506 18:07:27.739 | INFO | "pinned" : " 29 GB" | 20260506 18:07:27.739 | INFO | } | 20260506 18:07:27.739 | INFO | Perf report for the last 5.1373 seconds: | 20260506 18:07:27.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:07:27.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:27.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:27.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:27.739 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 18:07:27.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:27.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:27.739 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 18:07:27.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:27.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:27.739 | INFO | Uploaded 36GiB to GPU in 2.00351s (17.9684GiB/s) | 20260506 18:07:27.739 | INFO | Downloaded 34GiB to GPU in 2.01067s (16.9098GiB/s) | 20260506 18:07:32.879 | INFO | Analyzing chunk frames = [3382784,3383296) | 20260506 18:07:32.879 | INFO | MemInfo: { | 20260506 18:07:32.879 | INFO | "gpu" : " 17 GB", | 20260506 18:07:32.879 | INFO | "huge" : "", | 20260506 18:07:32.879 | INFO | "huge_shared" : " 27 GB", | 20260506 18:07:32.879 | INFO | "malloc" : " 14 GB", | 20260506 18:07:32.879 | INFO | "pinned" : " 29 GB" | 20260506 18:07:32.879 | INFO | } | 20260506 18:07:32.879 | INFO | Perf report for the last 5.14019 seconds: | 20260506 18:07:32.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:07:32.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:32.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:32.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:32.879 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 18:07:32.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:32.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:32.879 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 18:07:32.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:32.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:32.879 | INFO | Uploaded 36GiB to GPU in 1.99929s (18.0064GiB/s) | 20260506 18:07:32.879 | INFO | Downloaded 34GiB to GPU in 1.9877s (17.1052GiB/s) | 20260506 18:07:32.879 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:07:32.879 | INFO | Total mem usage: 18 GiB | 20260506 18:07:32.879 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:07:32.879 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -10.9%) | 20260506 18:07:32.879 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:07:32.879 | INFO | High Water Breakdown: | 20260506 18:07:32.879 | INFO | 62% : 10.1 GiB ( 88.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:07:32.879 | INFO | 37.9% : 6.15 GiB ( 91.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:07:32.879 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:07:32.879 | INFO | ------------------------------------------ | 20260506 18:07:32.879 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:07:32.879 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:07:32.879 | INFO | Total mem usage: 26.5 GiB | 20260506 18:07:32.879 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:07:32.879 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:07:32.879 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:07:32.879 | INFO | High Water Breakdown: | 20260506 18:07:32.879 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:07:32.879 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:07:32.879 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:07:32.879 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:07:32.879 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:07:32.879 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:07:32.879 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:07:32.879 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:07:32.879 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:07:32.879 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:07:32.879 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:07:32.879 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:07:32.879 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:07:32.879 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:07:32.879 | INFO | ------------------------------------------ | 20260506 18:07:32.879 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:07:32.879 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:07:32.879 | INFO | Total mem usage: 25.6 GiB | 20260506 18:07:32.879 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:07:32.879 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:07:32.879 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:07:32.879 | INFO | High Water Breakdown: | 20260506 18:07:32.879 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:07:32.879 | INFO | ------------------------------------------ | 20260506 18:07:32.879 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:07:32.879 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:07:32.879 | INFO | Total mem usage: 15.7 GiB | 20260506 18:07:32.879 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:07:32.879 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:07:32.879 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 18:07:32.880 | INFO | High Water Breakdown: | 20260506 18:07:32.880 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:07:32.880 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:07:32.880 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:07:32.880 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:07:32.880 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:07:32.880 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:07:32.880 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:07:32.880 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:07:32.880 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:07:32.880 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:07:32.880 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:07:32.880 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:07:32.880 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:07:32.880 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:07:32.880 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:07:32.880 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:07:32.880 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:07:32.880 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:07:32.880 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:07:32.880 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:07:32.880 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:07:32.880 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:07:32.880 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:07:32.880 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:07:32.880 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:07:32.880 | INFO | ------------------------------------------ | 20260506 18:07:32.880 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:07:32.880 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:07:32.880 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:07:32.880 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:07:32.880 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.58e+01, 1.34e+01 / 2.30e+00 | 20260506 18:07:32.880 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.97e-03 / 6.41e-03 | 20260506 18:07:32.880 | INFO | PerfReport: 11.62%, Baseline -- 3.64e+00 / 4.40e+00, 5.63e-01 / 3.19e-01 | 20260506 18:07:32.880 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.46e+00, 1.87e-01 / 1.48e-01 | 20260506 18:07:32.880 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.47e+01, 4.07e+00 / 2.99e+00 | 20260506 18:07:32.880 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.43e+00 / 2.22e+00 | 20260506 18:07:32.880 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.26e+00 / 2.85e+00, 3.78e-01 / 2.31e-01 | 20260506 18:07:32.880 | INFO | PerfReport: 3.42%, Metrics -- 1.07e+00 / 1.42e+00, 2.82e-01 / 1.79e-01 | 20260506 18:07:32.880 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.69e-01 / 4.16e-01, 5.74e-02 / 2.84e-02 | 20260506 18:07:32.880 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:07:32.880 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:07:32.880 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:07:32.880 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 18:07:32.880 | INFO | PerfReport: 21.40%, Upload -- 2.08e+01 / 0.00e+00, 6.89e+00 / 0.00e+00 | 20260506 18:07:32.880 | INFO | PerfReport: 21.17%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 18:07:32.880 | INFO | PerfReport: 32.23%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 18:07:32.880 | INFO | PerfReport: 1.37%, LockUpload -- 1.33e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260506 18:07:32.880 | INFO | PerfReport: 1.16%, LockDownload -- 1.13e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260506 18:07:32.880 | INFO | PerfReport: 22.66%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 18:07:32.880 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.51e-02 / 0.00e+00 | 20260506 18:07:32.994 | INFO | Starting a round of Baz IO | 20260506 18:07:36.130 | INFO | Populating RTMetrics report | 20260506 18:07:36.131 | INFO | RTMetrics report populated | 20260506 18:07:36.131 | INFO | Opening tmp RTMetrics json file | 20260506 18:07:36.173 | INFO | Serializing RTMetrics to json | 20260506 18:07:36.225 | INFO | Serialization done, swapping tmp file now... | 20260506 18:07:36.228 | INFO | RTMetrics done | 20260506 18:07:36.491 | INFO | Finished a round of Baz IO: Wrote 2.045 GiB in 3.496 seconds | 20260506 18:07:36.491 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.25% / 0.01% / 64.75% | 20260506 18:07:36.491 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.19354% / 7.94729e-06% / 5.16574e-05% | 20260506 18:07:36.491 | INFO | Baz IO overall: | 20260506 18:07:36.491 | INFO | BytesWritten : 1786330373084 | 20260506 18:07:36.491 | INFO | NumEvents : 863991721435 | 20260506 18:07:38.027 | INFO | Analyzing chunk frames = [3383296,3383808) | 20260506 18:07:38.027 | INFO | MemInfo: { | 20260506 18:07:38.027 | INFO | "gpu" : " 17 GB", | 20260506 18:07:38.027 | INFO | "huge" : "", | 20260506 18:07:38.027 | INFO | "huge_shared" : " 27 GB", | 20260506 18:07:38.027 | INFO | "malloc" : " 14 GB", | 20260506 18:07:38.027 | INFO | "pinned" : " 29 GB" | 20260506 18:07:38.027 | INFO | } | 20260506 18:07:38.027 | INFO | Perf report for the last 5.14791 seconds: | 20260506 18:07:38.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:07:38.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:38.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:38.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:38.027 | INFO | ANALYSIS : 62.6%, 2.9, 95.6ms, 0 | 20260506 18:07:38.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:38.027 | INFO | RT_METRICS : 45.0%, 1.0, 23.7ms, 0 | 20260506 18:07:38.027 | INFO | PRE_HQ : 59.9%, 1.9, 57.7ms, 0 | 20260506 18:07:38.027 | INFO | BAZWRITER : 67.6%, 4.8, 167.2ms, 0 | 20260506 18:07:38.027 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 18:07:38.027 | INFO | Uploaded 36GiB to GPU in 2.01646s (17.8531GiB/s) | 20260506 18:07:38.027 | INFO | Downloaded 38GiB to GPU in 2.20701s (17.2179GiB/s) | 20260506 18:07:43.158 | INFO | Analyzing chunk frames = [3383808,3384320) | 20260506 18:07:43.158 | INFO | MemInfo: { | 20260506 18:07:43.158 | INFO | "gpu" : " 17 GB", | 20260506 18:07:43.158 | INFO | "huge" : "", | 20260506 18:07:43.158 | INFO | "huge_shared" : " 27 GB", | 20260506 18:07:43.158 | INFO | "malloc" : " 14 GB", | 20260506 18:07:43.158 | INFO | "pinned" : " 29 GB" | 20260506 18:07:43.158 | INFO | } | 20260506 18:07:43.158 | INFO | Perf report for the last 5.13105 seconds: | 20260506 18:07:43.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:07:43.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:43.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:43.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:43.158 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260506 18:07:43.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:43.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:43.158 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:07:43.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:43.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:43.158 | INFO | Uploaded 36GiB to GPU in 2.00965s (17.9135GiB/s) | 20260506 18:07:43.158 | INFO | Downloaded 34GiB to GPU in 1.92545s (17.6583GiB/s) | 20260506 18:07:48.299 | INFO | Analyzing chunk frames = [3384320,3384832) | 20260506 18:07:48.299 | INFO | MemInfo: { | 20260506 18:07:48.299 | INFO | "gpu" : " 17 GB", | 20260506 18:07:48.299 | INFO | "huge" : "", | 20260506 18:07:48.299 | INFO | "huge_shared" : " 27 GB", | 20260506 18:07:48.299 | INFO | "malloc" : " 14 GB", | 20260506 18:07:48.299 | INFO | "pinned" : " 29 GB" | 20260506 18:07:48.299 | INFO | } | 20260506 18:07:48.299 | INFO | Perf report for the last 5.14097 seconds: | 20260506 18:07:48.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:07:48.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:48.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:48.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:48.299 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 18:07:48.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:48.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:48.299 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:07:48.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:48.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:48.299 | INFO | Uploaded 36GiB to GPU in 1.96891s (18.2843GiB/s) | 20260506 18:07:48.299 | INFO | Downloaded 34GiB to GPU in 1.92768s (17.6378GiB/s) | 20260506 18:07:53.439 | INFO | Analyzing chunk frames = [3384832,3385344) | 20260506 18:07:53.439 | INFO | MemInfo: { | 20260506 18:07:53.439 | INFO | "gpu" : " 17 GB", | 20260506 18:07:53.439 | INFO | "huge" : "", | 20260506 18:07:53.439 | INFO | "huge_shared" : " 27 GB", | 20260506 18:07:53.439 | INFO | "malloc" : " 14 GB", | 20260506 18:07:53.439 | INFO | "pinned" : " 29 GB" | 20260506 18:07:53.439 | INFO | } | 20260506 18:07:53.439 | INFO | Perf report for the last 5.14054 seconds: | 20260506 18:07:53.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:07:53.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:53.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:53.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:53.439 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:07:53.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:53.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:53.439 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:07:53.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:53.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:53.440 | INFO | Uploaded 36GiB to GPU in 1.93028s (18.6501GiB/s) | 20260506 18:07:53.440 | INFO | Downloaded 34GiB to GPU in 1.873s (18.1527GiB/s) | 20260506 18:07:58.577 | INFO | Analyzing chunk frames = [3385344,3385856) | 20260506 18:07:58.577 | INFO | MemInfo: { | 20260506 18:07:58.577 | INFO | "gpu" : " 17 GB", | 20260506 18:07:58.577 | INFO | "huge" : "", | 20260506 18:07:58.577 | INFO | "huge_shared" : " 27 GB", | 20260506 18:07:58.578 | INFO | "malloc" : " 14 GB", | 20260506 18:07:58.578 | INFO | "pinned" : " 29 GB" | 20260506 18:07:58.578 | INFO | } | 20260506 18:07:58.578 | INFO | Perf report for the last 5.13831 seconds: | 20260506 18:07:58.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:07:58.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:58.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:58.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:58.578 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 18:07:58.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:58.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:58.578 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:07:58.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:58.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:07:58.578 | INFO | Uploaded 36GiB to GPU in 2.00701s (17.9371GiB/s) | 20260506 18:07:58.578 | INFO | Downloaded 34GiB to GPU in 1.90824s (17.8174GiB/s) | 20260506 18:08:03.716 | INFO | Analyzing chunk frames = [3385856,3386368) | 20260506 18:08:03.716 | INFO | MemInfo: { | 20260506 18:08:03.716 | INFO | "gpu" : " 17 GB", | 20260506 18:08:03.716 | INFO | "huge" : "", | 20260506 18:08:03.716 | INFO | "huge_shared" : " 27 GB", | 20260506 18:08:03.716 | INFO | "malloc" : " 14 GB", | 20260506 18:08:03.716 | INFO | "pinned" : " 29 GB" | 20260506 18:08:03.716 | INFO | } | 20260506 18:08:03.716 | INFO | Perf report for the last 5.13873 seconds: | 20260506 18:08:03.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:08:03.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:03.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:03.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:03.716 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 18:08:03.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:03.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:03.716 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:08:03.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:03.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:03.717 | INFO | Uploaded 36GiB to GPU in 2.01599s (17.8572GiB/s) | 20260506 18:08:03.717 | INFO | Downloaded 34GiB to GPU in 1.94621s (17.4699GiB/s) | 20260506 18:08:08.853 | INFO | Analyzing chunk frames = [3386368,3386880) | 20260506 18:08:08.853 | INFO | MemInfo: { | 20260506 18:08:08.853 | INFO | "gpu" : " 17 GB", | 20260506 18:08:08.853 | INFO | "huge" : "", | 20260506 18:08:08.853 | INFO | "huge_shared" : " 27 GB", | 20260506 18:08:08.853 | INFO | "malloc" : " 14 GB", | 20260506 18:08:08.853 | INFO | "pinned" : " 29 GB" | 20260506 18:08:08.853 | INFO | } | 20260506 18:08:08.853 | INFO | Perf report for the last 5.13698 seconds: | 20260506 18:08:08.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:08:08.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:08.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:08.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:08.853 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 18:08:08.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:08.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:08.853 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:08:08.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:08.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:08.854 | INFO | Uploaded 36GiB to GPU in 1.99745s (18.023GiB/s) | 20260506 18:08:08.854 | INFO | Downloaded 34GiB to GPU in 2.00832s (16.9296GiB/s) | 20260506 18:08:13.992 | INFO | Analyzing chunk frames = [3386880,3387392) | 20260506 18:08:13.992 | INFO | MemInfo: { | 20260506 18:08:13.992 | INFO | "gpu" : " 17 GB", | 20260506 18:08:13.992 | INFO | "huge" : "", | 20260506 18:08:13.992 | INFO | "huge_shared" : " 27 GB", | 20260506 18:08:13.992 | INFO | "malloc" : " 14 GB", | 20260506 18:08:13.992 | INFO | "pinned" : " 29 GB" | 20260506 18:08:13.992 | INFO | } | 20260506 18:08:13.992 | INFO | Perf report for the last 5.13883 seconds: | 20260506 18:08:13.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:08:13.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:13.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:13.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:13.992 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 18:08:13.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:13.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:13.992 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:08:13.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:13.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:13.992 | INFO | Uploaded 36GiB to GPU in 2.01191s (17.8935GiB/s) | 20260506 18:08:13.992 | INFO | Downloaded 34GiB to GPU in 1.97822s (17.1872GiB/s) | 20260506 18:08:14.107 | INFO | Starting a round of Baz IO | 20260506 18:08:17.234 | INFO | Populating RTMetrics report | 20260506 18:08:17.234 | INFO | RTMetrics report populated | 20260506 18:08:17.234 | INFO | Opening tmp RTMetrics json file | 20260506 18:08:17.243 | INFO | Serializing RTMetrics to json | 20260506 18:08:17.273 | INFO | Serialization done, swapping tmp file now... | 20260506 18:08:17.281 | INFO | RTMetrics done | 20260506 18:08:17.739 | INFO | Finished a round of Baz IO: Wrote 2.048 GiB in 3.631 seconds | 20260506 18:08:17.739 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.24% / 0.01% / 64.75% | 20260506 18:08:17.739 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.19294% / 7.94729e-06% / 5.5631e-05% | 20260506 18:08:17.739 | INFO | Baz IO overall: | 20260506 18:08:17.739 | INFO | BytesWritten : 1788528997437 | 20260506 18:08:17.739 | INFO | NumEvents : 865051338992 | 20260506 18:08:19.134 | INFO | Analyzing chunk frames = [3387392,3387904) | 20260506 18:08:19.134 | INFO | MemInfo: { | 20260506 18:08:19.134 | INFO | "gpu" : " 17 GB", | 20260506 18:08:19.134 | INFO | "huge" : "", | 20260506 18:08:19.134 | INFO | "huge_shared" : " 27 GB", | 20260506 18:08:19.134 | INFO | "malloc" : " 14 GB", | 20260506 18:08:19.134 | INFO | "pinned" : " 29 GB" | 20260506 18:08:19.134 | INFO | } | 20260506 18:08:19.134 | INFO | Perf report for the last 5.14214 seconds: | 20260506 18:08:19.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:08:19.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:19.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:19.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:19.134 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 18:08:19.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:19.134 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 18:08:19.134 | INFO | PRE_HQ : 59.9%, 1.8, 57.4ms, 0 | 20260506 18:08:19.134 | INFO | BAZWRITER : 70.3%, 4.8, 174.4ms, 0 | 20260506 18:08:19.134 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 18:08:19.135 | INFO | Uploaded 36GiB to GPU in 2.04536s (17.6008GiB/s) | 20260506 18:08:19.135 | INFO | Downloaded 38GiB to GPU in 2.19747s (17.2926GiB/s) | 20260506 18:08:24.278 | INFO | Analyzing chunk frames = [3387904,3388416) | 20260506 18:08:24.278 | INFO | MemInfo: { | 20260506 18:08:24.278 | INFO | "gpu" : " 17 GB", | 20260506 18:08:24.278 | INFO | "huge" : "", | 20260506 18:08:24.278 | INFO | "huge_shared" : " 27 GB", | 20260506 18:08:24.278 | INFO | "malloc" : " 14 GB", | 20260506 18:08:24.278 | INFO | "pinned" : " 29 GB" | 20260506 18:08:24.278 | INFO | } | 20260506 18:08:24.278 | INFO | Perf report for the last 5.14391 seconds: | 20260506 18:08:24.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:08:24.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:24.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:24.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:24.278 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:08:24.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:24.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:24.278 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:08:24.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:24.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:24.278 | INFO | Uploaded 36GiB to GPU in 2.01452s (17.8703GiB/s) | 20260506 18:08:24.279 | INFO | Downloaded 34GiB to GPU in 1.93749s (17.5485GiB/s) | 20260506 18:08:29.418 | INFO | Analyzing chunk frames = [3388416,3388928) | 20260506 18:08:29.418 | INFO | MemInfo: { | 20260506 18:08:29.418 | INFO | "gpu" : " 17 GB", | 20260506 18:08:29.418 | INFO | "huge" : "", | 20260506 18:08:29.418 | INFO | "huge_shared" : " 27 GB", | 20260506 18:08:29.418 | INFO | "malloc" : " 14 GB", | 20260506 18:08:29.418 | INFO | "pinned" : " 29 GB" | 20260506 18:08:29.418 | INFO | } | 20260506 18:08:29.418 | INFO | Perf report for the last 5.14015 seconds: | 20260506 18:08:29.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:08:29.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:29.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:29.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:29.418 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 18:08:29.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:29.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:29.418 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:08:29.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:29.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:29.419 | INFO | Uploaded 36GiB to GPU in 1.97494s (18.2284GiB/s) | 20260506 18:08:29.419 | INFO | Downloaded 34GiB to GPU in 1.91568s (17.7483GiB/s) | 20260506 18:08:34.553 | INFO | Analyzing chunk frames = [3388928,3389440) | 20260506 18:08:34.553 | INFO | MemInfo: { | 20260506 18:08:34.553 | INFO | "gpu" : " 17 GB", | 20260506 18:08:34.553 | INFO | "huge" : "", | 20260506 18:08:34.553 | INFO | "huge_shared" : " 27 GB", | 20260506 18:08:34.553 | INFO | "malloc" : " 14 GB", | 20260506 18:08:34.553 | INFO | "pinned" : " 29 GB" | 20260506 18:08:34.553 | INFO | } | 20260506 18:08:34.553 | INFO | Perf report for the last 5.13464 seconds: | 20260506 18:08:34.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:08:34.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:34.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:34.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:34.553 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 18:08:34.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:34.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:34.553 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:08:34.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:34.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:34.553 | INFO | Uploaded 36GiB to GPU in 1.96675s (18.3043GiB/s) | 20260506 18:08:34.553 | INFO | Downloaded 34GiB to GPU in 1.87078s (18.1743GiB/s) | 20260506 18:08:39.694 | INFO | Analyzing chunk frames = [3389440,3389952) | 20260506 18:08:39.694 | INFO | MemInfo: { | 20260506 18:08:39.694 | INFO | "gpu" : " 17 GB", | 20260506 18:08:39.694 | INFO | "huge" : "", | 20260506 18:08:39.694 | INFO | "huge_shared" : " 27 GB", | 20260506 18:08:39.694 | INFO | "malloc" : " 14 GB", | 20260506 18:08:39.694 | INFO | "pinned" : " 29 GB" | 20260506 18:08:39.694 | INFO | } | 20260506 18:08:39.694 | INFO | Perf report for the last 5.14066 seconds: | 20260506 18:08:39.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:08:39.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:39.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:39.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:39.694 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 18:08:39.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:39.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:39.694 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:08:39.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:39.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:39.694 | INFO | Uploaded 36GiB to GPU in 2.01942s (17.8269GiB/s) | 20260506 18:08:39.694 | INFO | Downloaded 34GiB to GPU in 1.91351s (17.7684GiB/s) | 20260506 18:08:44.828 | INFO | Analyzing chunk frames = [3389952,3390464) | 20260506 18:08:44.828 | INFO | MemInfo: { | 20260506 18:08:44.828 | INFO | "gpu" : " 17 GB", | 20260506 18:08:44.828 | INFO | "huge" : "", | 20260506 18:08:44.828 | INFO | "huge_shared" : " 27 GB", | 20260506 18:08:44.828 | INFO | "malloc" : " 14 GB", | 20260506 18:08:44.828 | INFO | "pinned" : " 29 GB" | 20260506 18:08:44.828 | INFO | } | 20260506 18:08:44.828 | INFO | Perf report for the last 5.13464 seconds: | 20260506 18:08:44.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:08:44.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:44.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:44.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:44.828 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 18:08:44.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:44.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:44.828 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:08:44.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:44.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:44.829 | INFO | Uploaded 36GiB to GPU in 2.02191s (17.805GiB/s) | 20260506 18:08:44.829 | INFO | Downloaded 34GiB to GPU in 1.94584s (17.4732GiB/s) | 20260506 18:08:49.967 | INFO | Analyzing chunk frames = [3390464,3390976) | 20260506 18:08:49.967 | INFO | MemInfo: { | 20260506 18:08:49.967 | INFO | "gpu" : " 17 GB", | 20260506 18:08:49.967 | INFO | "huge" : "", | 20260506 18:08:49.967 | INFO | "huge_shared" : " 27 GB", | 20260506 18:08:49.967 | INFO | "malloc" : " 14 GB", | 20260506 18:08:49.967 | INFO | "pinned" : " 29 GB" | 20260506 18:08:49.967 | INFO | } | 20260506 18:08:49.967 | INFO | Perf report for the last 5.13834 seconds: | 20260506 18:08:49.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:08:49.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:49.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:49.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:49.967 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 18:08:49.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:49.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:49.967 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 18:08:49.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:49.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:49.967 | INFO | Uploaded 36GiB to GPU in 1.99292s (18.0639GiB/s) | 20260506 18:08:49.967 | INFO | Downloaded 34GiB to GPU in 2.0186s (16.8434GiB/s) | 20260506 18:08:55.114 | INFO | Analyzing chunk frames = [3390976,3391488) | 20260506 18:08:55.114 | INFO | MemInfo: { | 20260506 18:08:55.114 | INFO | "gpu" : " 17 GB", | 20260506 18:08:55.114 | INFO | "huge" : "", | 20260506 18:08:55.114 | INFO | "huge_shared" : " 27 GB", | 20260506 18:08:55.114 | INFO | "malloc" : " 14 GB", | 20260506 18:08:55.114 | INFO | "pinned" : " 29 GB" | 20260506 18:08:55.114 | INFO | } | 20260506 18:08:55.114 | INFO | Perf report for the last 5.1472 seconds: | 20260506 18:08:55.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:08:55.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:55.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:55.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:55.114 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 18:08:55.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:55.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:55.114 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:08:55.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:55.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:08:55.114 | INFO | Uploaded 36GiB to GPU in 2.0112s (17.8997GiB/s) | 20260506 18:08:55.114 | INFO | Downloaded 34GiB to GPU in 1.97419s (17.2223GiB/s) | 20260506 18:08:55.114 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:08:55.114 | INFO | Total mem usage: 18 GiB | 20260506 18:08:55.114 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:08:55.114 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -10.1%) | 20260506 18:08:55.114 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:08:55.114 | INFO | High Water Breakdown: | 20260506 18:08:55.114 | INFO | 62.1% : 10.2 GiB ( 89% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:08:55.114 | INFO | 37.8% : 6.18 GiB ( 92.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:08:55.114 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:08:55.114 | INFO | ------------------------------------------ | 20260506 18:08:55.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:08:55.114 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:08:55.114 | INFO | Total mem usage: 26.5 GiB | 20260506 18:08:55.114 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:08:55.114 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:08:55.114 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:08:55.114 | INFO | High Water Breakdown: | 20260506 18:08:55.114 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:08:55.114 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:08:55.114 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:08:55.114 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:08:55.114 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:08:55.114 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:08:55.114 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:08:55.114 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:08:55.114 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:08:55.114 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:08:55.114 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:08:55.114 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:08:55.114 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:08:55.115 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:08:55.115 | INFO | ------------------------------------------ | 20260506 18:08:55.115 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:08:55.115 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:08:55.115 | INFO | Total mem usage: 25.6 GiB | 20260506 18:08:55.115 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:08:55.115 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:08:55.115 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:08:55.115 | INFO | High Water Breakdown: | 20260506 18:08:55.115 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:08:55.115 | INFO | ------------------------------------------ | 20260506 18:08:55.115 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:08:55.115 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:08:55.115 | INFO | Total mem usage: 15.7 GiB | 20260506 18:08:55.115 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:08:55.115 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:08:55.115 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 18:08:55.115 | INFO | High Water Breakdown: | 20260506 18:08:55.115 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:08:55.115 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:08:55.115 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:08:55.115 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:08:55.115 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:08:55.115 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:08:55.115 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:08:55.115 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:08:55.115 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:08:55.115 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:08:55.115 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:08:55.115 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:08:55.115 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:08:55.115 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:08:55.115 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:08:55.115 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:08:55.115 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:08:55.115 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:08:55.115 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:08:55.115 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:08:55.115 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:08:55.115 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:08:55.115 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:08:55.115 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:08:55.115 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:08:55.115 | INFO | ------------------------------------------ | 20260506 18:08:55.115 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:08:55.115 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:08:55.115 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:08:55.115 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:08:55.115 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.66e+01, 1.33e+01 / 2.72e+00 | 20260506 18:08:55.115 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.02e-01, 1.02e-02 / 5.73e-03 | 20260506 18:08:55.115 | INFO | PerfReport: 11.62%, Baseline -- 3.64e+00 / 4.44e+00, 5.60e-01 / 3.27e-01 | 20260506 18:08:55.115 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.48e+00, 1.86e-01 / 1.37e-01 | 20260506 18:08:55.115 | INFO | PerfReport: 16.01%, DME -- 4.01e+01 / 4.51e+01, 4.37e+00 / 2.73e+00 | 20260506 18:08:55.115 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.41e+00 / 1.81e+00 | 20260506 18:08:55.115 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.26e+00 / 2.84e+00, 3.81e-01 / 2.08e-01 | 20260506 18:08:55.115 | INFO | PerfReport: 3.42%, Metrics -- 1.07e+00 / 1.40e+00, 2.81e-01 / 1.50e-01 | 20260506 18:08:55.115 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.69e-01 / 4.20e-01, 5.79e-02 / 2.71e-02 | 20260506 18:08:55.115 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:08:55.115 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:08:55.115 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:08:55.115 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 18:08:55.115 | INFO | PerfReport: 21.45%, Upload -- 2.08e+01 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260506 18:08:55.115 | INFO | PerfReport: 21.14%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 18:08:55.115 | INFO | PerfReport: 32.22%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 18:08:55.115 | INFO | PerfReport: 1.38%, LockUpload -- 1.34e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260506 18:08:55.115 | INFO | PerfReport: 1.15%, LockDownload -- 1.12e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 18:08:55.115 | INFO | PerfReport: 22.66%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:08:55.115 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.15e-02 / 0.00e+00 | 20260506 18:08:55.229 | INFO | Starting a round of Baz IO | 20260506 18:08:58.375 | INFO | Populating RTMetrics report | 20260506 18:08:58.376 | INFO | RTMetrics report populated | 20260506 18:08:58.376 | INFO | Opening tmp RTMetrics json file | 20260506 18:08:58.380 | INFO | Serializing RTMetrics to json | 20260506 18:08:58.469 | INFO | Serialization done, swapping tmp file now... | 20260506 18:08:58.495 | INFO | RTMetrics done | 20260506 18:08:58.882 | INFO | Finished a round of Baz IO: Wrote 2.047 GiB in 3.653 seconds | 20260506 18:08:58.882 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.23% / 0.01% / 64.76% | 20260506 18:08:58.882 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.19043% / 1.98682e-05% / 5.5631e-05% | 20260506 18:08:58.882 | INFO | Baz IO overall: | 20260506 18:08:58.882 | INFO | BytesWritten : 1790727345868 | 20260506 18:08:58.882 | INFO | NumEvents : 866110669261 | 20260506 18:09:00.251 | INFO | Analyzing chunk frames = [3391488,3392000) | 20260506 18:09:00.251 | INFO | MemInfo: { | 20260506 18:09:00.251 | INFO | "gpu" : " 17 GB", | 20260506 18:09:00.251 | INFO | "huge" : "", | 20260506 18:09:00.251 | INFO | "huge_shared" : " 27 GB", | 20260506 18:09:00.251 | INFO | "malloc" : " 14 GB", | 20260506 18:09:00.251 | INFO | "pinned" : " 29 GB" | 20260506 18:09:00.251 | INFO | } | 20260506 18:09:00.251 | INFO | Perf report for the last 5.13729 seconds: | 20260506 18:09:00.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:09:00.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:00.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:00.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:00.251 | INFO | ANALYSIS : 62.6%, 2.9, 95.4ms, 0 | 20260506 18:09:00.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:00.251 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 18:09:00.251 | INFO | PRE_HQ : 60.0%, 1.9, 57.8ms, 0 | 20260506 18:09:00.251 | INFO | BAZWRITER : 70.8%, 4.8, 174.5ms, 0 | 20260506 18:09:00.251 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 18:09:00.251 | INFO | Uploaded 36GiB to GPU in 2.04481s (17.6055GiB/s) | 20260506 18:09:00.251 | INFO | Downloaded 38GiB to GPU in 2.20845s (17.2067GiB/s) | 20260506 18:09:05.396 | INFO | Analyzing chunk frames = [3392000,3392512) | 20260506 18:09:05.396 | INFO | MemInfo: { | 20260506 18:09:05.396 | INFO | "gpu" : " 17 GB", | 20260506 18:09:05.396 | INFO | "huge" : "", | 20260506 18:09:05.396 | INFO | "huge_shared" : " 27 GB", | 20260506 18:09:05.396 | INFO | "malloc" : " 14 GB", | 20260506 18:09:05.396 | INFO | "pinned" : " 29 GB" | 20260506 18:09:05.396 | INFO | } | 20260506 18:09:05.396 | INFO | Perf report for the last 5.14483 seconds: | 20260506 18:09:05.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:09:05.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:05.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:05.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:05.396 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 18:09:05.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:05.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:05.396 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 18:09:05.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:05.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:05.396 | INFO | Uploaded 36GiB to GPU in 2.01842s (17.8357GiB/s) | 20260506 18:09:05.396 | INFO | Downloaded 34GiB to GPU in 1.94516s (17.4793GiB/s) | 20260506 18:09:10.532 | INFO | Analyzing chunk frames = [3392512,3393024) | 20260506 18:09:10.532 | INFO | MemInfo: { | 20260506 18:09:10.532 | INFO | "gpu" : " 17 GB", | 20260506 18:09:10.532 | INFO | "huge" : "", | 20260506 18:09:10.532 | INFO | "huge_shared" : " 27 GB", | 20260506 18:09:10.532 | INFO | "malloc" : " 14 GB", | 20260506 18:09:10.532 | INFO | "pinned" : " 29 GB" | 20260506 18:09:10.532 | INFO | } | 20260506 18:09:10.532 | INFO | Perf report for the last 5.13635 seconds: | 20260506 18:09:10.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:09:10.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:10.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:10.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:10.533 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:09:10.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:10.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:10.533 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:09:10.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:10.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:10.533 | INFO | Uploaded 36GiB to GPU in 1.98042s (18.178GiB/s) | 20260506 18:09:10.533 | INFO | Downloaded 34GiB to GPU in 1.91621s (17.7434GiB/s) | 20260506 18:09:15.667 | INFO | Analyzing chunk frames = [3393024,3393536) | 20260506 18:09:15.667 | INFO | MemInfo: { | 20260506 18:09:15.667 | INFO | "gpu" : " 17 GB", | 20260506 18:09:15.667 | INFO | "huge" : "", | 20260506 18:09:15.667 | INFO | "huge_shared" : " 27 GB", | 20260506 18:09:15.667 | INFO | "malloc" : " 14 GB", | 20260506 18:09:15.667 | INFO | "pinned" : " 29 GB" | 20260506 18:09:15.667 | INFO | } | 20260506 18:09:15.667 | INFO | Perf report for the last 5.13465 seconds: | 20260506 18:09:15.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:09:15.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:15.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:15.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:15.667 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:09:15.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:15.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:15.667 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:09:15.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:15.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:15.667 | INFO | Uploaded 36GiB to GPU in 1.94258s (18.532GiB/s) | 20260506 18:09:15.667 | INFO | Downloaded 34GiB to GPU in 1.86827s (18.1987GiB/s) | 20260506 18:09:20.801 | INFO | Analyzing chunk frames = [3393536,3394048) | 20260506 18:09:20.801 | INFO | MemInfo: { | 20260506 18:09:20.801 | INFO | "gpu" : " 17 GB", | 20260506 18:09:20.801 | INFO | "huge" : "", | 20260506 18:09:20.801 | INFO | "huge_shared" : " 27 GB", | 20260506 18:09:20.802 | INFO | "malloc" : " 14 GB", | 20260506 18:09:20.802 | INFO | "pinned" : " 29 GB" | 20260506 18:09:20.802 | INFO | } | 20260506 18:09:20.802 | INFO | Perf report for the last 5.13497 seconds: | 20260506 18:09:20.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:09:20.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:20.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:20.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:20.802 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 18:09:20.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:20.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:20.802 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 18:09:20.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:20.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:20.802 | INFO | Uploaded 36GiB to GPU in 1.95941s (18.3729GiB/s) | 20260506 18:09:20.802 | INFO | Downloaded 34GiB to GPU in 1.88109s (18.0746GiB/s) | 20260506 18:09:25.941 | INFO | Analyzing chunk frames = [3394048,3394560) | 20260506 18:09:25.941 | INFO | MemInfo: { | 20260506 18:09:25.941 | INFO | "gpu" : " 17 GB", | 20260506 18:09:25.941 | INFO | "huge" : "", | 20260506 18:09:25.941 | INFO | "huge_shared" : " 27 GB", | 20260506 18:09:25.941 | INFO | "malloc" : " 14 GB", | 20260506 18:09:25.941 | INFO | "pinned" : " 29 GB" | 20260506 18:09:25.941 | INFO | } | 20260506 18:09:25.941 | INFO | Perf report for the last 5.13854 seconds: | 20260506 18:09:25.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:09:25.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:25.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:25.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:25.941 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 18:09:25.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:25.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:25.941 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:09:25.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:25.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:25.941 | INFO | Uploaded 36GiB to GPU in 2.01219s (17.891GiB/s) | 20260506 18:09:25.941 | INFO | Downloaded 34GiB to GPU in 1.95832s (17.3618GiB/s) | 20260506 18:09:31.084 | INFO | Analyzing chunk frames = [3394560,3395072) | 20260506 18:09:31.084 | INFO | MemInfo: { | 20260506 18:09:31.084 | INFO | "gpu" : " 17 GB", | 20260506 18:09:31.084 | INFO | "huge" : "", | 20260506 18:09:31.084 | INFO | "huge_shared" : " 27 GB", | 20260506 18:09:31.084 | INFO | "malloc" : " 14 GB", | 20260506 18:09:31.084 | INFO | "pinned" : " 29 GB" | 20260506 18:09:31.084 | INFO | } | 20260506 18:09:31.085 | INFO | Perf report for the last 5.14385 seconds: | 20260506 18:09:31.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:09:31.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:31.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:31.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:31.085 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 18:09:31.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:31.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:31.085 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:09:31.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:31.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:31.085 | INFO | Uploaded 36GiB to GPU in 1.98624s (18.1247GiB/s) | 20260506 18:09:31.085 | INFO | Downloaded 34GiB to GPU in 2.00704s (16.9404GiB/s) | 20260506 18:09:36.226 | INFO | Analyzing chunk frames = [3395072,3395584) | 20260506 18:09:36.226 | INFO | MemInfo: { | 20260506 18:09:36.226 | INFO | "gpu" : " 17 GB", | 20260506 18:09:36.226 | INFO | "huge" : "", | 20260506 18:09:36.226 | INFO | "huge_shared" : " 27 GB", | 20260506 18:09:36.226 | INFO | "malloc" : " 14 GB", | 20260506 18:09:36.226 | INFO | "pinned" : " 29 GB" | 20260506 18:09:36.226 | INFO | } | 20260506 18:09:36.226 | INFO | Perf report for the last 5.14167 seconds: | 20260506 18:09:36.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:09:36.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:36.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:36.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:36.226 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:09:36.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:36.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:36.226 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:09:36.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:36.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:36.226 | INFO | Uploaded 36GiB to GPU in 1.99777s (18.0201GiB/s) | 20260506 18:09:36.226 | INFO | Downloaded 34GiB to GPU in 2.00038s (16.9968GiB/s) | 20260506 18:09:36.340 | INFO | Starting a round of Baz IO | 20260506 18:09:39.481 | INFO | Populating RTMetrics report | 20260506 18:09:39.481 | INFO | RTMetrics report populated | 20260506 18:09:39.481 | INFO | Opening tmp RTMetrics json file | 20260506 18:09:39.496 | INFO | Serializing RTMetrics to json | 20260506 18:09:39.498 | INFO | Serialization done, swapping tmp file now... | 20260506 18:09:39.498 | INFO | RTMetrics done | 20260506 18:09:39.843 | INFO | Finished a round of Baz IO: Wrote 2.047 GiB in 3.502 seconds | 20260506 18:09:39.843 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.23% / 0.01% / 64.77% | 20260506 18:09:39.843 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.18632% / 3.97364e-06% / 8.74201e-05% | 20260506 18:09:39.843 | INFO | Baz IO overall: | 20260506 18:09:39.843 | INFO | BytesWritten : 1792925297850 | 20260506 18:09:39.843 | INFO | NumEvents : 867170052594 | 20260506 18:09:41.371 | INFO | Analyzing chunk frames = [3395584,3396096) | 20260506 18:09:41.371 | INFO | MemInfo: { | 20260506 18:09:41.371 | INFO | "gpu" : " 17 GB", | 20260506 18:09:41.371 | INFO | "huge" : "", | 20260506 18:09:41.371 | INFO | "huge_shared" : " 27 GB", | 20260506 18:09:41.371 | INFO | "malloc" : " 14 GB", | 20260506 18:09:41.371 | INFO | "pinned" : " 29 GB" | 20260506 18:09:41.371 | INFO | } | 20260506 18:09:41.371 | INFO | Perf report for the last 5.14519 seconds: | 20260506 18:09:41.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:09:41.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:41.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:41.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:41.371 | INFO | ANALYSIS : 62.7%, 2.9, 95.6ms, 0 | 20260506 18:09:41.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:41.371 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260506 18:09:41.371 | INFO | PRE_HQ : 60.0%, 1.9, 57.7ms, 0 | 20260506 18:09:41.371 | INFO | BAZWRITER : 67.8%, 4.8, 167.3ms, 0 | 20260506 18:09:41.371 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.3ms, 0 | 20260506 18:09:41.372 | INFO | Uploaded 36GiB to GPU in 2.06047s (17.4717GiB/s) | 20260506 18:09:41.372 | INFO | Downloaded 38GiB to GPU in 2.2239s (17.0871GiB/s) | 20260506 18:09:46.506 | INFO | Analyzing chunk frames = [3396096,3396608) | 20260506 18:09:46.506 | INFO | MemInfo: { | 20260506 18:09:46.506 | INFO | "gpu" : " 17 GB", | 20260506 18:09:46.506 | INFO | "huge" : "", | 20260506 18:09:46.506 | INFO | "huge_shared" : " 27 GB", | 20260506 18:09:46.506 | INFO | "malloc" : " 14 GB", | 20260506 18:09:46.506 | INFO | "pinned" : " 29 GB" | 20260506 18:09:46.506 | INFO | } | 20260506 18:09:46.506 | INFO | Perf report for the last 5.1348 seconds: | 20260506 18:09:46.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:09:46.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:46.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:46.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:46.506 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 18:09:46.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:46.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:46.506 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:09:46.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:46.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:46.507 | INFO | Uploaded 36GiB to GPU in 2.00154s (17.9862GiB/s) | 20260506 18:09:46.507 | INFO | Downloaded 34GiB to GPU in 1.94394s (17.4902GiB/s) | 20260506 18:09:51.645 | INFO | Analyzing chunk frames = [3396608,3397120) | 20260506 18:09:51.645 | INFO | MemInfo: { | 20260506 18:09:51.645 | INFO | "gpu" : " 17 GB", | 20260506 18:09:51.645 | INFO | "huge" : "", | 20260506 18:09:51.645 | INFO | "huge_shared" : " 27 GB", | 20260506 18:09:51.645 | INFO | "malloc" : " 14 GB", | 20260506 18:09:51.645 | INFO | "pinned" : " 29 GB" | 20260506 18:09:51.645 | INFO | } | 20260506 18:09:51.645 | INFO | Perf report for the last 5.13844 seconds: | 20260506 18:09:51.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:09:51.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:51.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:51.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:51.645 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:09:51.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:51.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:51.645 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:09:51.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:51.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:51.645 | INFO | Uploaded 36GiB to GPU in 1.95921s (18.3748GiB/s) | 20260506 18:09:51.645 | INFO | Downloaded 34GiB to GPU in 1.92181s (17.6917GiB/s) | 20260506 18:09:56.783 | INFO | Analyzing chunk frames = [3397120,3397632) | 20260506 18:09:56.783 | INFO | MemInfo: { | 20260506 18:09:56.783 | INFO | "gpu" : " 17 GB", | 20260506 18:09:56.783 | INFO | "huge" : "", | 20260506 18:09:56.783 | INFO | "huge_shared" : " 27 GB", | 20260506 18:09:56.783 | INFO | "malloc" : " 14 GB", | 20260506 18:09:56.783 | INFO | "pinned" : " 29 GB" | 20260506 18:09:56.783 | INFO | } | 20260506 18:09:56.783 | INFO | Perf report for the last 5.13848 seconds: | 20260506 18:09:56.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:09:56.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:56.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:56.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:56.783 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:09:56.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:56.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:56.783 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:09:56.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:56.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:09:56.783 | INFO | Uploaded 36GiB to GPU in 1.91594s (18.7898GiB/s) | 20260506 18:09:56.783 | INFO | Downloaded 34GiB to GPU in 1.87026s (18.1793GiB/s) | 20260506 18:10:01.917 | INFO | Analyzing chunk frames = [3397632,3398144) | 20260506 18:10:01.917 | INFO | MemInfo: { | 20260506 18:10:01.917 | INFO | "gpu" : " 17 GB", | 20260506 18:10:01.917 | INFO | "huge" : "", | 20260506 18:10:01.917 | INFO | "huge_shared" : " 27 GB", | 20260506 18:10:01.917 | INFO | "malloc" : " 14 GB", | 20260506 18:10:01.917 | INFO | "pinned" : " 29 GB" | 20260506 18:10:01.917 | INFO | } | 20260506 18:10:01.917 | INFO | Perf report for the last 5.13416 seconds: | 20260506 18:10:01.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:10:01.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:01.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:01.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:01.917 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 18:10:01.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:01.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:01.917 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 18:10:01.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:01.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:01.918 | INFO | Uploaded 36GiB to GPU in 1.95339s (18.4295GiB/s) | 20260506 18:10:01.918 | INFO | Downloaded 34GiB to GPU in 1.88563s (18.0311GiB/s) | 20260506 18:10:07.061 | INFO | Analyzing chunk frames = [3398144,3398656) | 20260506 18:10:07.061 | INFO | MemInfo: { | 20260506 18:10:07.061 | INFO | "gpu" : " 17 GB", | 20260506 18:10:07.061 | INFO | "huge" : "", | 20260506 18:10:07.061 | INFO | "huge_shared" : " 27 GB", | 20260506 18:10:07.061 | INFO | "malloc" : " 14 GB", | 20260506 18:10:07.061 | INFO | "pinned" : " 29 GB" | 20260506 18:10:07.061 | INFO | } | 20260506 18:10:07.061 | INFO | Perf report for the last 5.14358 seconds: | 20260506 18:10:07.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:10:07.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:07.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:07.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:07.061 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 18:10:07.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:07.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:07.061 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:10:07.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:07.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:07.061 | INFO | Uploaded 36GiB to GPU in 2.01603s (17.8568GiB/s) | 20260506 18:10:07.061 | INFO | Downloaded 34GiB to GPU in 1.95213s (17.4169GiB/s) | 20260506 18:10:12.205 | INFO | Analyzing chunk frames = [3398656,3399168) | 20260506 18:10:12.205 | INFO | MemInfo: { | 20260506 18:10:12.205 | INFO | "gpu" : " 17 GB", | 20260506 18:10:12.205 | INFO | "huge" : "", | 20260506 18:10:12.205 | INFO | "huge_shared" : " 27 GB", | 20260506 18:10:12.205 | INFO | "malloc" : " 14 GB", | 20260506 18:10:12.205 | INFO | "pinned" : " 29 GB" | 20260506 18:10:12.205 | INFO | } | 20260506 18:10:12.205 | INFO | Perf report for the last 5.14415 seconds: | 20260506 18:10:12.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:10:12.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:12.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:12.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:12.205 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:10:12.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:12.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:12.205 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:10:12.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:12.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:12.205 | INFO | Uploaded 36GiB to GPU in 1.99089s (18.0824GiB/s) | 20260506 18:10:12.205 | INFO | Downloaded 34GiB to GPU in 2.00978s (16.9173GiB/s) | 20260506 18:10:17.344 | INFO | Analyzing chunk frames = [3399168,3399680) | 20260506 18:10:17.344 | INFO | MemInfo: { | 20260506 18:10:17.344 | INFO | "gpu" : " 17 GB", | 20260506 18:10:17.344 | INFO | "huge" : "", | 20260506 18:10:17.344 | INFO | "huge_shared" : " 27 GB", | 20260506 18:10:17.344 | INFO | "malloc" : " 14 GB", | 20260506 18:10:17.344 | INFO | "pinned" : " 29 GB" | 20260506 18:10:17.344 | INFO | } | 20260506 18:10:17.344 | INFO | Perf report for the last 5.13886 seconds: | 20260506 18:10:17.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:10:17.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:17.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:17.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:17.344 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 18:10:17.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:17.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:17.344 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 18:10:17.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:17.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:17.344 | INFO | Uploaded 36GiB to GPU in 2.00078s (17.993GiB/s) | 20260506 18:10:17.344 | INFO | Downloaded 34GiB to GPU in 1.9945s (17.0468GiB/s) | 20260506 18:10:17.344 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:10:17.344 | INFO | Total mem usage: 18 GiB | 20260506 18:10:17.344 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:10:17.344 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.3%) | 20260506 18:10:17.345 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:10:17.345 | INFO | High Water Breakdown: | 20260506 18:10:17.345 | INFO | 61.3% : 9.48 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:10:17.345 | INFO | 38.6% : 5.97 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:10:17.345 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:10:17.345 | INFO | ------------------------------------------ | 20260506 18:10:17.345 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:10:17.345 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:10:17.345 | INFO | Total mem usage: 26.5 GiB | 20260506 18:10:17.345 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:10:17.345 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:10:17.345 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:10:17.345 | INFO | High Water Breakdown: | 20260506 18:10:17.345 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:10:17.345 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:10:17.345 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:10:17.345 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:10:17.345 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:10:17.345 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:10:17.345 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:10:17.345 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:10:17.345 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:10:17.345 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:10:17.345 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:10:17.345 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:10:17.345 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:10:17.345 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:10:17.345 | INFO | ------------------------------------------ | 20260506 18:10:17.345 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:10:17.345 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:10:17.345 | INFO | Total mem usage: 25.6 GiB | 20260506 18:10:17.345 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:10:17.345 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:10:17.345 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:10:17.345 | INFO | High Water Breakdown: | 20260506 18:10:17.345 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:10:17.345 | INFO | ------------------------------------------ | 20260506 18:10:17.345 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:10:17.345 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:10:17.345 | INFO | Total mem usage: 15.7 GiB | 20260506 18:10:17.345 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:10:17.345 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:10:17.345 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:10:17.345 | INFO | High Water Breakdown: | 20260506 18:10:17.345 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:10:17.345 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:10:17.345 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:10:17.345 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:10:17.345 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:10:17.345 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:10:17.345 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:10:17.345 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:10:17.345 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:10:17.345 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:10:17.345 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:10:17.345 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:10:17.345 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:10:17.345 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:10:17.345 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:10:17.345 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:10:17.345 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:10:17.345 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:10:17.345 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:10:17.345 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:10:17.345 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:10:17.345 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:10:17.345 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:10:17.345 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:10:17.345 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:10:17.345 | INFO | ------------------------------------------ | 20260506 18:10:17.345 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:10:17.346 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:10:17.346 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:10:17.346 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:10:17.346 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.58e+01, 1.35e+01 / 2.93e+00 | 20260506 18:10:17.346 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.00e-01, 9.95e-03 / 5.60e-03 | 20260506 18:10:17.346 | INFO | PerfReport: 11.59%, Baseline -- 3.62e+00 / 4.39e+00, 5.44e-01 / 3.35e-01 | 20260506 18:10:17.346 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.43e+00, 1.80e-01 / 1.40e-01 | 20260506 18:10:17.346 | INFO | PerfReport: 16.02%, DME -- 4.01e+01 / 4.50e+01, 4.37e+00 / 3.20e+00 | 20260506 18:10:17.346 | INFO | PerfReport: 56.99%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.37e+00 / 1.77e+00 | 20260506 18:10:17.346 | INFO | PerfReport: 7.27%, PulseAccumulating -- 2.27e+00 / 2.88e+00, 3.90e-01 / 1.99e-01 | 20260506 18:10:17.346 | INFO | PerfReport: 3.40%, Metrics -- 1.06e+00 / 1.40e+00, 2.74e-01 / 1.39e-01 | 20260506 18:10:17.346 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.69e-01 / 4.23e-01, 5.66e-02 / 2.34e-02 | 20260506 18:10:17.346 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:10:17.346 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:10:17.346 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:10:17.346 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 18:10:17.346 | INFO | PerfReport: 21.36%, Upload -- 2.07e+01 / 0.00e+00, 6.83e+00 / 0.00e+00 | 20260506 18:10:17.346 | INFO | PerfReport: 21.19%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 18:10:17.346 | INFO | PerfReport: 32.22%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:10:17.346 | INFO | PerfReport: 1.31%, LockUpload -- 1.27e+00 / 0.00e+00, 5.32e+00 / 0.00e+00 | 20260506 18:10:17.346 | INFO | PerfReport: 1.21%, LockDownload -- 1.18e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260506 18:10:17.346 | INFO | PerfReport: 22.70%, LockCompute -- 2.20e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 18:10:17.346 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260506 18:10:17.462 | INFO | Starting a round of Baz IO | 20260506 18:10:20.596 | INFO | Populating RTMetrics report | 20260506 18:10:20.597 | INFO | RTMetrics report populated | 20260506 18:10:20.597 | INFO | Opening tmp RTMetrics json file | 20260506 18:10:20.789 | INFO | Serializing RTMetrics to json | 20260506 18:10:20.826 | INFO | Serialization done, swapping tmp file now... | 20260506 18:10:20.847 | INFO | RTMetrics done | 20260506 18:10:21.032 | INFO | Finished a round of Baz IO: Wrote 2.046 GiB in 3.57 seconds | 20260506 18:10:21.032 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.22% / 0.01% / 64.77% | 20260506 18:10:21.032 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.18417% / 3.97364e-06% / 9.13938e-05% | 20260506 18:10:21.032 | INFO | Baz IO overall: | 20260506 18:10:21.032 | INFO | BytesWritten : 1795121951094 | 20260506 18:10:21.032 | INFO | NumEvents : 868228189346 | 20260506 18:10:22.481 | INFO | Analyzing chunk frames = [3399680,3400192) | 20260506 18:10:22.481 | INFO | MemInfo: { | 20260506 18:10:22.481 | INFO | "gpu" : " 17 GB", | 20260506 18:10:22.481 | INFO | "huge" : "", | 20260506 18:10:22.481 | INFO | "huge_shared" : " 27 GB", | 20260506 18:10:22.481 | INFO | "malloc" : " 14 GB", | 20260506 18:10:22.481 | INFO | "pinned" : " 29 GB" | 20260506 18:10:22.481 | INFO | } | 20260506 18:10:22.481 | INFO | Perf report for the last 5.13737 seconds: | 20260506 18:10:22.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:10:22.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:22.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:22.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:22.481 | INFO | ANALYSIS : 62.7%, 2.9, 95.3ms, 0 | 20260506 18:10:22.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:22.481 | INFO | RT_METRICS : 48.3%, 1.0, 25.3ms, 0 | 20260506 18:10:22.481 | INFO | PRE_HQ : 60.1%, 1.9, 58.0ms, 0 | 20260506 18:10:22.481 | INFO | BAZWRITER : 69.2%, 4.8, 169.4ms, 0 | 20260506 18:10:22.482 | INFO | BAZ_SAMPLING : 2.9%, 1.0, 1.5ms, 0 | 20260506 18:10:22.482 | INFO | Uploaded 36GiB to GPU in 2.07281s (17.3677GiB/s) | 20260506 18:10:22.482 | INFO | Downloaded 38GiB to GPU in 2.20811s (17.2093GiB/s) | 20260506 18:10:27.619 | INFO | Analyzing chunk frames = [3400192,3400704) | 20260506 18:10:27.619 | INFO | MemInfo: { | 20260506 18:10:27.619 | INFO | "gpu" : " 17 GB", | 20260506 18:10:27.619 | INFO | "huge" : "", | 20260506 18:10:27.619 | INFO | "huge_shared" : " 27 GB", | 20260506 18:10:27.619 | INFO | "malloc" : " 14 GB", | 20260506 18:10:27.619 | INFO | "pinned" : " 29 GB" | 20260506 18:10:27.619 | INFO | } | 20260506 18:10:27.619 | INFO | Perf report for the last 5.13802 seconds: | 20260506 18:10:27.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:10:27.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:27.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:27.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:27.620 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:10:27.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:27.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:27.620 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 18:10:27.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:27.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:27.620 | INFO | Uploaded 36GiB to GPU in 2.01284s (17.8852GiB/s) | 20260506 18:10:27.620 | INFO | Downloaded 34GiB to GPU in 1.9486s (17.4484GiB/s) | 20260506 18:10:32.756 | INFO | Analyzing chunk frames = [3400704,3401216) | 20260506 18:10:32.756 | INFO | MemInfo: { | 20260506 18:10:32.756 | INFO | "gpu" : " 17 GB", | 20260506 18:10:32.756 | INFO | "huge" : "", | 20260506 18:10:32.756 | INFO | "huge_shared" : " 27 GB", | 20260506 18:10:32.756 | INFO | "malloc" : " 14 GB", | 20260506 18:10:32.756 | INFO | "pinned" : " 29 GB" | 20260506 18:10:32.756 | INFO | } | 20260506 18:10:32.756 | INFO | Perf report for the last 5.13693 seconds: | 20260506 18:10:32.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:10:32.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:32.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:32.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:32.756 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 18:10:32.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:32.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:32.756 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:10:32.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:32.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:32.757 | INFO | Uploaded 36GiB to GPU in 1.98722s (18.1157GiB/s) | 20260506 18:10:32.757 | INFO | Downloaded 34GiB to GPU in 1.9248s (17.6642GiB/s) | 20260506 18:10:37.892 | INFO | Analyzing chunk frames = [3401216,3401728) | 20260506 18:10:37.892 | INFO | MemInfo: { | 20260506 18:10:37.892 | INFO | "gpu" : " 17 GB", | 20260506 18:10:37.892 | INFO | "huge" : "", | 20260506 18:10:37.892 | INFO | "huge_shared" : " 27 GB", | 20260506 18:10:37.892 | INFO | "malloc" : " 14 GB", | 20260506 18:10:37.892 | INFO | "pinned" : " 29 GB" | 20260506 18:10:37.892 | INFO | } | 20260506 18:10:37.892 | INFO | Perf report for the last 5.13571 seconds: | 20260506 18:10:37.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:10:37.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:37.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:37.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:37.892 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 18:10:37.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:37.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:37.892 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:10:37.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:37.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:37.892 | INFO | Uploaded 36GiB to GPU in 1.93255s (18.6283GiB/s) | 20260506 18:10:37.892 | INFO | Downloaded 34GiB to GPU in 1.87332s (18.1496GiB/s) | 20260506 18:10:43.037 | INFO | Analyzing chunk frames = [3401728,3402240) | 20260506 18:10:43.037 | INFO | MemInfo: { | 20260506 18:10:43.037 | INFO | "gpu" : " 17 GB", | 20260506 18:10:43.037 | INFO | "huge" : "", | 20260506 18:10:43.037 | INFO | "huge_shared" : " 27 GB", | 20260506 18:10:43.037 | INFO | "malloc" : " 14 GB", | 20260506 18:10:43.037 | INFO | "pinned" : " 29 GB" | 20260506 18:10:43.037 | INFO | } | 20260506 18:10:43.037 | INFO | Perf report for the last 5.14463 seconds: | 20260506 18:10:43.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:10:43.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:43.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:43.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:43.037 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 18:10:43.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:43.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:43.037 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:10:43.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:43.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:43.037 | INFO | Uploaded 36GiB to GPU in 1.9586s (18.3805GiB/s) | 20260506 18:10:43.037 | INFO | Downloaded 34GiB to GPU in 1.88094s (18.0761GiB/s) | 20260506 18:10:48.179 | INFO | Analyzing chunk frames = [3402240,3402752) | 20260506 18:10:48.179 | INFO | MemInfo: { | 20260506 18:10:48.179 | INFO | "gpu" : " 17 GB", | 20260506 18:10:48.179 | INFO | "huge" : "", | 20260506 18:10:48.179 | INFO | "huge_shared" : " 27 GB", | 20260506 18:10:48.179 | INFO | "malloc" : " 14 GB", | 20260506 18:10:48.179 | INFO | "pinned" : " 29 GB" | 20260506 18:10:48.179 | INFO | } | 20260506 18:10:48.179 | INFO | Perf report for the last 5.14277 seconds: | 20260506 18:10:48.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:10:48.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:48.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:48.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:48.180 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 18:10:48.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:48.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:48.180 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:10:48.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:48.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:48.180 | INFO | Uploaded 36GiB to GPU in 2.02234s (17.8012GiB/s) | 20260506 18:10:48.180 | INFO | Downloaded 34GiB to GPU in 1.95224s (17.4159GiB/s) | 20260506 18:10:53.321 | INFO | Analyzing chunk frames = [3402752,3403264) | 20260506 18:10:53.321 | INFO | MemInfo: { | 20260506 18:10:53.321 | INFO | "gpu" : " 17 GB", | 20260506 18:10:53.321 | INFO | "huge" : "", | 20260506 18:10:53.321 | INFO | "huge_shared" : " 27 GB", | 20260506 18:10:53.321 | INFO | "malloc" : " 14 GB", | 20260506 18:10:53.322 | INFO | "pinned" : " 29 GB" | 20260506 18:10:53.322 | INFO | } | 20260506 18:10:53.322 | INFO | Perf report for the last 5.14219 seconds: | 20260506 18:10:53.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:10:53.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:53.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:53.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:53.322 | INFO | ANALYSIS : 61.9%, 3.0, 95.4ms, 0 | 20260506 18:10:53.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:53.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:53.322 | INFO | PRE_HQ : 27.2%, 1.0, 14.0ms, 0 | 20260506 18:10:53.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:53.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:53.322 | INFO | Uploaded 36GiB to GPU in 2.00056s (17.995GiB/s) | 20260506 18:10:53.322 | INFO | Downloaded 34GiB to GPU in 1.98569s (17.1225GiB/s) | 20260506 18:10:58.465 | INFO | Analyzing chunk frames = [3403264,3403776) | 20260506 18:10:58.465 | INFO | MemInfo: { | 20260506 18:10:58.465 | INFO | "gpu" : " 17 GB", | 20260506 18:10:58.465 | INFO | "huge" : "", | 20260506 18:10:58.465 | INFO | "huge_shared" : " 27 GB", | 20260506 18:10:58.465 | INFO | "malloc" : " 14 GB", | 20260506 18:10:58.465 | INFO | "pinned" : " 29 GB" | 20260506 18:10:58.465 | INFO | } | 20260506 18:10:58.465 | INFO | Perf report for the last 5.14352 seconds: | 20260506 18:10:58.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:10:58.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:58.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:58.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:58.465 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 18:10:58.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:58.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:58.465 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 18:10:58.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:58.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:10:58.465 | INFO | Uploaded 36GiB to GPU in 1.98481s (18.1378GiB/s) | 20260506 18:10:58.465 | INFO | Downloaded 34GiB to GPU in 1.99654s (17.0294GiB/s) | 20260506 18:10:58.577 | INFO | Starting a round of Baz IO | 20260506 18:11:01.757 | INFO | Populating RTMetrics report | 20260506 18:11:01.758 | INFO | RTMetrics report populated | 20260506 18:11:01.758 | INFO | Opening tmp RTMetrics json file | 20260506 18:11:01.837 | INFO | Serializing RTMetrics to json | 20260506 18:11:01.914 | INFO | Serialization done, swapping tmp file now... | 20260506 18:11:01.962 | INFO | RTMetrics done | 20260506 18:11:02.023 | INFO | Finished a round of Baz IO: Wrote 2.046 GiB in 3.445 seconds | 20260506 18:11:02.023 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.22% / 0.01% / 64.78% | 20260506 18:11:02.023 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.18313% / 3.97364e-06% / 5.16574e-05% | 20260506 18:11:02.023 | INFO | Baz IO overall: | 20260506 18:11:02.023 | INFO | BytesWritten : 1797318779441 | 20260506 18:11:02.023 | INFO | NumEvents : 869286895528 | 20260506 18:11:03.597 | INFO | Analyzing chunk frames = [3403776,3404288) | 20260506 18:11:03.597 | INFO | MemInfo: { | 20260506 18:11:03.597 | INFO | "gpu" : " 17 GB", | 20260506 18:11:03.597 | INFO | "huge" : "", | 20260506 18:11:03.597 | INFO | "huge_shared" : " 27 GB", | 20260506 18:11:03.597 | INFO | "malloc" : " 14 GB", | 20260506 18:11:03.597 | INFO | "pinned" : " 29 GB" | 20260506 18:11:03.597 | INFO | } | 20260506 18:11:03.597 | INFO | Perf report for the last 5.13188 seconds: | 20260506 18:11:03.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:11:03.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:03.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:03.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:03.597 | INFO | ANALYSIS : 63.3%, 2.8, 93.8ms, 0 | 20260506 18:11:03.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:03.597 | INFO | RT_METRICS : 50.3%, 1.0, 26.4ms, 0 | 20260506 18:11:03.597 | INFO | PRE_HQ : 61.1%, 2.0, 62.5ms, 0 | 20260506 18:11:03.597 | INFO | BAZWRITER : 66.9%, 4.7, 162.6ms, 0 | 20260506 18:11:03.597 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 18:11:03.597 | INFO | Uploaded 36GiB to GPU in 2.04836s (17.575GiB/s) | 20260506 18:11:03.597 | INFO | Downloaded 38GiB to GPU in 2.18254s (17.4109GiB/s) | 20260506 18:11:08.731 | INFO | Analyzing chunk frames = [3404288,3404800) | 20260506 18:11:08.731 | INFO | MemInfo: { | 20260506 18:11:08.731 | INFO | "gpu" : " 17 GB", | 20260506 18:11:08.731 | INFO | "huge" : "", | 20260506 18:11:08.731 | INFO | "huge_shared" : " 27 GB", | 20260506 18:11:08.731 | INFO | "malloc" : " 14 GB", | 20260506 18:11:08.731 | INFO | "pinned" : " 29 GB" | 20260506 18:11:08.731 | INFO | } | 20260506 18:11:08.731 | INFO | Perf report for the last 5.13424 seconds: | 20260506 18:11:08.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:11:08.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:08.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:08.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:08.731 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260506 18:11:08.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:08.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:08.731 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 18:11:08.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:08.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:08.731 | INFO | Uploaded 36GiB to GPU in 1.99991s (18.0008GiB/s) | 20260506 18:11:08.732 | INFO | Downloaded 34GiB to GPU in 1.94545s (17.4767GiB/s) | 20260506 18:11:13.871 | INFO | Analyzing chunk frames = [3404800,3405312) | 20260506 18:11:13.871 | INFO | MemInfo: { | 20260506 18:11:13.871 | INFO | "gpu" : " 17 GB", | 20260506 18:11:13.871 | INFO | "huge" : "", | 20260506 18:11:13.871 | INFO | "huge_shared" : " 27 GB", | 20260506 18:11:13.871 | INFO | "malloc" : " 14 GB", | 20260506 18:11:13.871 | INFO | "pinned" : " 29 GB" | 20260506 18:11:13.871 | INFO | } | 20260506 18:11:13.871 | INFO | Perf report for the last 5.13934 seconds: | 20260506 18:11:13.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:11:13.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:13.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:13.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:13.871 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 18:11:13.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:13.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:13.871 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:11:13.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:13.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:13.871 | INFO | Uploaded 36GiB to GPU in 1.96804s (18.2923GiB/s) | 20260506 18:11:13.871 | INFO | Downloaded 34GiB to GPU in 1.92813s (17.6337GiB/s) | 20260506 18:11:19.008 | INFO | Analyzing chunk frames = [3405312,3405824) | 20260506 18:11:19.008 | INFO | MemInfo: { | 20260506 18:11:19.008 | INFO | "gpu" : " 17 GB", | 20260506 18:11:19.008 | INFO | "huge" : "", | 20260506 18:11:19.008 | INFO | "huge_shared" : " 27 GB", | 20260506 18:11:19.008 | INFO | "malloc" : " 14 GB", | 20260506 18:11:19.008 | INFO | "pinned" : " 29 GB" | 20260506 18:11:19.008 | INFO | } | 20260506 18:11:19.008 | INFO | Perf report for the last 5.13727 seconds: | 20260506 18:11:19.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:11:19.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:19.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:19.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:19.008 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:11:19.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:19.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:19.008 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:11:19.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:19.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:19.008 | INFO | Uploaded 36GiB to GPU in 1.91334s (18.8152GiB/s) | 20260506 18:11:19.008 | INFO | Downloaded 34GiB to GPU in 1.86762s (18.2049GiB/s) | 20260506 18:11:24.152 | INFO | Analyzing chunk frames = [3405824,3406336) | 20260506 18:11:24.152 | INFO | MemInfo: { | 20260506 18:11:24.152 | INFO | "gpu" : " 17 GB", | 20260506 18:11:24.152 | INFO | "huge" : "", | 20260506 18:11:24.152 | INFO | "huge_shared" : " 27 GB", | 20260506 18:11:24.152 | INFO | "malloc" : " 14 GB", | 20260506 18:11:24.152 | INFO | "pinned" : " 29 GB" | 20260506 18:11:24.152 | INFO | } | 20260506 18:11:24.152 | INFO | Perf report for the last 5.14434 seconds: | 20260506 18:11:24.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:11:24.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:24.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:24.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:24.152 | INFO | ANALYSIS : 59.9%, 3.0, 93.4ms, 0 | 20260506 18:11:24.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:24.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:24.152 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:11:24.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:24.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:24.153 | INFO | Uploaded 36GiB to GPU in 1.95073s (18.4546GiB/s) | 20260506 18:11:24.153 | INFO | Downloaded 34GiB to GPU in 1.88353s (18.0512GiB/s) | 20260506 18:11:29.295 | INFO | Analyzing chunk frames = [3406336,3406848) | 20260506 18:11:29.295 | INFO | MemInfo: { | 20260506 18:11:29.295 | INFO | "gpu" : " 17 GB", | 20260506 18:11:29.295 | INFO | "huge" : "", | 20260506 18:11:29.295 | INFO | "huge_shared" : " 27 GB", | 20260506 18:11:29.295 | INFO | "malloc" : " 14 GB", | 20260506 18:11:29.295 | INFO | "pinned" : " 29 GB" | 20260506 18:11:29.295 | INFO | } | 20260506 18:11:29.295 | INFO | Perf report for the last 5.14258 seconds: | 20260506 18:11:29.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:11:29.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:29.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:29.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:29.295 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 18:11:29.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:29.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:29.295 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:11:29.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:29.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:29.295 | INFO | Uploaded 36GiB to GPU in 1.97954s (18.1861GiB/s) | 20260506 18:11:29.295 | INFO | Downloaded 34GiB to GPU in 1.94506s (17.4802GiB/s) | 20260506 18:11:34.430 | INFO | Analyzing chunk frames = [3406848,3407360) | 20260506 18:11:34.430 | INFO | MemInfo: { | 20260506 18:11:34.430 | INFO | "gpu" : " 17 GB", | 20260506 18:11:34.430 | INFO | "huge" : "", | 20260506 18:11:34.430 | INFO | "huge_shared" : " 27 GB", | 20260506 18:11:34.430 | INFO | "malloc" : " 14 GB", | 20260506 18:11:34.430 | INFO | "pinned" : " 29 GB" | 20260506 18:11:34.430 | INFO | } | 20260506 18:11:34.430 | INFO | Perf report for the last 5.13494 seconds: | 20260506 18:11:34.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:11:34.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:34.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:34.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:34.430 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 18:11:34.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:34.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:34.430 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 18:11:34.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:34.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:34.430 | INFO | Uploaded 36GiB to GPU in 1.99716s (18.0256GiB/s) | 20260506 18:11:34.430 | INFO | Downloaded 34GiB to GPU in 2.01494s (16.874GiB/s) | 20260506 18:11:39.568 | INFO | Analyzing chunk frames = [3407360,3407872) | 20260506 18:11:39.568 | INFO | MemInfo: { | 20260506 18:11:39.568 | INFO | "gpu" : " 17 GB", | 20260506 18:11:39.568 | INFO | "huge" : "", | 20260506 18:11:39.568 | INFO | "huge_shared" : " 27 GB", | 20260506 18:11:39.568 | INFO | "malloc" : " 14 GB", | 20260506 18:11:39.568 | INFO | "pinned" : " 29 GB" | 20260506 18:11:39.568 | INFO | } | 20260506 18:11:39.568 | INFO | Perf report for the last 5.13787 seconds: | 20260506 18:11:39.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:11:39.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:39.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:39.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:39.568 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 18:11:39.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:39.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:39.568 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:11:39.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:39.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:39.568 | INFO | Uploaded 36GiB to GPU in 2.01742s (17.8446GiB/s) | 20260506 18:11:39.568 | INFO | Downloaded 34GiB to GPU in 1.97097s (17.2504GiB/s) | 20260506 18:11:39.568 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:11:39.568 | INFO | Total mem usage: 18 GiB | 20260506 18:11:39.568 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:11:39.568 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -17.2%) | 20260506 18:11:39.568 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:11:39.568 | INFO | High Water Breakdown: | 20260506 18:11:39.568 | INFO | 61.2% : 9.4 GiB ( 82.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:11:39.568 | INFO | 38.7% : 5.95 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:11:39.568 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:11:39.568 | INFO | ------------------------------------------ | 20260506 18:11:39.568 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:11:39.568 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:11:39.568 | INFO | Total mem usage: 26.5 GiB | 20260506 18:11:39.568 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:11:39.568 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 18:11:39.568 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 18:11:39.568 | INFO | High Water Breakdown: | 20260506 18:11:39.568 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:11:39.569 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:11:39.569 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:11:39.569 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:11:39.569 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:11:39.569 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:11:39.569 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:11:39.569 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:11:39.569 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:11:39.569 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:11:39.569 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:11:39.569 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:11:39.569 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:11:39.569 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:11:39.569 | INFO | ------------------------------------------ | 20260506 18:11:39.569 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:11:39.569 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:11:39.569 | INFO | Total mem usage: 25.6 GiB | 20260506 18:11:39.569 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:11:39.569 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:11:39.569 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:11:39.569 | INFO | High Water Breakdown: | 20260506 18:11:39.569 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:11:39.569 | INFO | ------------------------------------------ | 20260506 18:11:39.569 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:11:39.569 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:11:39.569 | INFO | Total mem usage: 15.7 GiB | 20260506 18:11:39.569 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:11:39.569 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:11:39.569 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 18:11:39.569 | INFO | High Water Breakdown: | 20260506 18:11:39.569 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:11:39.569 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:11:39.569 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:11:39.569 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:11:39.569 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:11:39.569 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:11:39.569 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:11:39.569 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:11:39.569 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:11:39.569 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:11:39.569 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:11:39.569 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:11:39.569 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:11:39.569 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:11:39.569 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:11:39.569 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:11:39.569 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:11:39.569 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:11:39.569 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:11:39.569 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:11:39.569 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:11:39.569 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:11:39.569 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:11:39.569 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:11:39.569 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:11:39.569 | INFO | ------------------------------------------ | 20260506 18:11:39.569 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:11:39.569 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:11:39.569 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:11:39.569 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:11:39.569 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.50e+01, 1.35e+01 / 2.29e+00 | 20260506 18:11:39.569 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 9.97e-02, 1.06e-02 / 5.86e-03 | 20260506 18:11:39.569 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.34e+00, 5.61e-01 / 4.33e-01 | 20260506 18:11:39.569 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.44e+00, 1.84e-01 / 1.55e-01 | 20260506 18:11:39.569 | INFO | PerfReport: 15.88%, DME -- 3.97e+01 / 4.37e+01, 3.78e+00 / 2.09e+00 | 20260506 18:11:39.569 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.46e+00 / 2.11e+00 | 20260506 18:11:39.569 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.26e+00 / 2.77e+00, 3.82e-01 / 2.81e-01 | 20260506 18:11:39.569 | INFO | PerfReport: 3.41%, Metrics -- 1.07e+00 / 1.40e+00, 2.75e-01 / 1.72e-01 | 20260506 18:11:39.569 | INFO | PerfReport: 1.19%, PulseSerializing -- 3.73e-01 / 5.67e-01, 1.92e-01 / 8.95e-01 | 20260506 18:11:39.570 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:11:39.570 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:11:39.570 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:11:39.570 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.30e+01 / 0.00e+00 | 20260506 18:11:39.570 | INFO | PerfReport: 21.41%, Upload -- 2.07e+01 / 0.00e+00, 6.87e+00 / 0.00e+00 | 20260506 18:11:39.570 | INFO | PerfReport: 21.19%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 18:11:39.570 | INFO | PerfReport: 32.31%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:11:39.570 | INFO | PerfReport: 1.26%, LockUpload -- 1.22e+00 / 0.00e+00, 5.18e+00 / 0.00e+00 | 20260506 18:11:39.570 | INFO | PerfReport: 1.15%, LockDownload -- 1.11e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 18:11:39.570 | INFO | PerfReport: 22.66%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 18:11:39.570 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260506 18:11:39.682 | INFO | Starting a round of Baz IO | 20260506 18:11:42.824 | INFO | Populating RTMetrics report | 20260506 18:11:42.824 | INFO | RTMetrics report populated | 20260506 18:11:42.824 | INFO | Opening tmp RTMetrics json file | 20260506 18:11:42.825 | INFO | Serializing RTMetrics to json | 20260506 18:11:42.848 | INFO | Serialization done, swapping tmp file now... | 20260506 18:11:42.850 | INFO | RTMetrics done | 20260506 18:11:43.698 | INFO | Finished a round of Baz IO: Wrote 2.047 GiB in 4.015 seconds | 20260506 18:11:43.698 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.21% / 0.01% / 64.78% | 20260506 18:11:43.698 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.18177% / 1.19209e-05% / 7.15256e-05% | 20260506 18:11:43.698 | INFO | Baz IO overall: | 20260506 18:11:43.698 | INFO | BytesWritten : 1799516388597 | 20260506 18:11:43.698 | INFO | NumEvents : 870346095978 | 20260506 18:11:44.706 | INFO | Analyzing chunk frames = [3407872,3408384) | 20260506 18:11:44.706 | INFO | MemInfo: { | 20260506 18:11:44.706 | INFO | "gpu" : " 17 GB", | 20260506 18:11:44.706 | INFO | "huge" : "", | 20260506 18:11:44.706 | INFO | "huge_shared" : " 27 GB", | 20260506 18:11:44.706 | INFO | "malloc" : " 14 GB", | 20260506 18:11:44.706 | INFO | "pinned" : " 29 GB" | 20260506 18:11:44.706 | INFO | } | 20260506 18:11:44.706 | INFO | Perf report for the last 5.13841 seconds: | 20260506 18:11:44.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:11:44.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:44.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:44.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:44.706 | INFO | ANALYSIS : 62.8%, 2.9, 95.3ms, 0 | 20260506 18:11:44.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:44.706 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 18:11:44.706 | INFO | PRE_HQ : 60.3%, 1.9, 57.9ms, 0 | 20260506 18:11:44.706 | INFO | BAZWRITER : 77.8%, 4.8, 193.5ms, 0 | 20260506 18:11:44.706 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 18:11:44.706 | INFO | Uploaded 36GiB to GPU in 2.04716s (17.5854GiB/s) | 20260506 18:11:44.706 | INFO | Downloaded 38GiB to GPU in 2.20693s (17.2185GiB/s) | 20260506 18:11:49.846 | INFO | Analyzing chunk frames = [3408384,3408896) | 20260506 18:11:49.846 | INFO | MemInfo: { | 20260506 18:11:49.846 | INFO | "gpu" : " 17 GB", | 20260506 18:11:49.846 | INFO | "huge" : "", | 20260506 18:11:49.846 | INFO | "huge_shared" : " 27 GB", | 20260506 18:11:49.846 | INFO | "malloc" : " 14 GB", | 20260506 18:11:49.846 | INFO | "pinned" : " 29 GB" | 20260506 18:11:49.846 | INFO | } | 20260506 18:11:49.846 | INFO | Perf report for the last 5.13973 seconds: | 20260506 18:11:49.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:11:49.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:49.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:49.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:49.846 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:11:49.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:49.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:49.846 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:11:49.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:49.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:49.846 | INFO | Uploaded 36GiB to GPU in 2.01131s (17.8987GiB/s) | 20260506 18:11:49.846 | INFO | Downloaded 34GiB to GPU in 1.93689s (17.5539GiB/s) | 20260506 18:11:54.986 | INFO | Analyzing chunk frames = [3408896,3409408) | 20260506 18:11:54.986 | INFO | MemInfo: { | 20260506 18:11:54.986 | INFO | "gpu" : " 17 GB", | 20260506 18:11:54.986 | INFO | "huge" : "", | 20260506 18:11:54.986 | INFO | "huge_shared" : " 27 GB", | 20260506 18:11:54.986 | INFO | "malloc" : " 14 GB", | 20260506 18:11:54.986 | INFO | "pinned" : " 29 GB" | 20260506 18:11:54.986 | INFO | } | 20260506 18:11:54.986 | INFO | Perf report for the last 5.13973 seconds: | 20260506 18:11:54.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:11:54.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:54.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:54.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:54.986 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 18:11:54.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:54.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:54.986 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:11:54.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:54.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:11:54.986 | INFO | Uploaded 36GiB to GPU in 1.97058s (18.2687GiB/s) | 20260506 18:11:54.986 | INFO | Downloaded 34GiB to GPU in 1.92166s (17.693GiB/s) | 20260506 18:12:00.130 | INFO | Analyzing chunk frames = [3409408,3409920) | 20260506 18:12:00.130 | INFO | MemInfo: { | 20260506 18:12:00.130 | INFO | "gpu" : " 17 GB", | 20260506 18:12:00.130 | INFO | "huge" : "", | 20260506 18:12:00.130 | INFO | "huge_shared" : " 27 GB", | 20260506 18:12:00.130 | INFO | "malloc" : " 14 GB", | 20260506 18:12:00.131 | INFO | "pinned" : " 29 GB" | 20260506 18:12:00.131 | INFO | } | 20260506 18:12:00.131 | INFO | Perf report for the last 5.14491 seconds: | 20260506 18:12:00.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:12:00.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:00.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:00.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:00.131 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:12:00.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:00.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:00.131 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:12:00.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:00.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:00.131 | INFO | Uploaded 36GiB to GPU in 1.94464s (18.5124GiB/s) | 20260506 18:12:00.131 | INFO | Downloaded 34GiB to GPU in 1.86738s (18.2073GiB/s) | 20260506 18:12:05.272 | INFO | Analyzing chunk frames = [3409920,3410432) | 20260506 18:12:05.272 | INFO | MemInfo: { | 20260506 18:12:05.272 | INFO | "gpu" : " 17 GB", | 20260506 18:12:05.272 | INFO | "huge" : "", | 20260506 18:12:05.272 | INFO | "huge_shared" : " 27 GB", | 20260506 18:12:05.272 | INFO | "malloc" : " 14 GB", | 20260506 18:12:05.272 | INFO | "pinned" : " 29 GB" | 20260506 18:12:05.272 | INFO | } | 20260506 18:12:05.272 | INFO | Perf report for the last 5.1416 seconds: | 20260506 18:12:05.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:12:05.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:05.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:05.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:05.272 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 18:12:05.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:05.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:05.272 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:12:05.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:05.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:05.272 | INFO | Uploaded 36GiB to GPU in 2.02096s (17.8133GiB/s) | 20260506 18:12:05.273 | INFO | Downloaded 34GiB to GPU in 1.9075s (17.8244GiB/s) | 20260506 18:12:10.411 | INFO | Analyzing chunk frames = [3410432,3410944) | 20260506 18:12:10.412 | INFO | MemInfo: { | 20260506 18:12:10.412 | INFO | "gpu" : " 17 GB", | 20260506 18:12:10.412 | INFO | "huge" : "", | 20260506 18:12:10.412 | INFO | "huge_shared" : " 27 GB", | 20260506 18:12:10.412 | INFO | "malloc" : " 14 GB", | 20260506 18:12:10.412 | INFO | "pinned" : " 29 GB" | 20260506 18:12:10.412 | INFO | } | 20260506 18:12:10.412 | INFO | Perf report for the last 5.13954 seconds: | 20260506 18:12:10.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:12:10.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:10.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:10.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:10.412 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:12:10.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:10.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:10.412 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:12:10.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:10.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:10.412 | INFO | Uploaded 36GiB to GPU in 2.07686s (17.3339GiB/s) | 20260506 18:12:10.412 | INFO | Downloaded 34GiB to GPU in 1.9668s (17.287GiB/s) | 20260506 18:12:15.552 | INFO | Analyzing chunk frames = [3410944,3411456) | 20260506 18:12:15.552 | INFO | MemInfo: { | 20260506 18:12:15.552 | INFO | "gpu" : " 17 GB", | 20260506 18:12:15.552 | INFO | "huge" : "", | 20260506 18:12:15.552 | INFO | "huge_shared" : " 27 GB", | 20260506 18:12:15.552 | INFO | "malloc" : " 14 GB", | 20260506 18:12:15.552 | INFO | "pinned" : " 29 GB" | 20260506 18:12:15.553 | INFO | } | 20260506 18:12:15.553 | INFO | Perf report for the last 5.1408 seconds: | 20260506 18:12:15.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:12:15.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:15.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:15.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:15.553 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:12:15.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:15.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:15.553 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 18:12:15.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:15.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:15.553 | INFO | Uploaded 36GiB to GPU in 2.01661s (17.8517GiB/s) | 20260506 18:12:15.553 | INFO | Downloaded 34GiB to GPU in 1.99551s (17.0382GiB/s) | 20260506 18:12:20.682 | INFO | Analyzing chunk frames = [3411456,3411968) | 20260506 18:12:20.682 | INFO | MemInfo: { | 20260506 18:12:20.682 | INFO | "gpu" : " 17 GB", | 20260506 18:12:20.682 | INFO | "huge" : "", | 20260506 18:12:20.682 | INFO | "huge_shared" : " 27 GB", | 20260506 18:12:20.682 | INFO | "malloc" : " 14 GB", | 20260506 18:12:20.682 | INFO | "pinned" : " 29 GB" | 20260506 18:12:20.682 | INFO | } | 20260506 18:12:20.682 | INFO | Perf report for the last 5.1296 seconds: | 20260506 18:12:20.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:12:20.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:20.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:20.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:20.682 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260506 18:12:20.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:20.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:20.682 | INFO | PRE_HQ : 31.0%, 1.0, 16.0ms, 0 | 20260506 18:12:20.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:20.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:20.683 | INFO | Uploaded 36GiB to GPU in 1.99347s (18.059GiB/s) | 20260506 18:12:20.683 | INFO | Downloaded 34GiB to GPU in 1.98226s (17.1521GiB/s) | 20260506 18:12:20.802 | INFO | Starting a round of Baz IO | 20260506 18:12:23.935 | INFO | Populating RTMetrics report | 20260506 18:12:23.936 | INFO | RTMetrics report populated | 20260506 18:12:23.936 | INFO | Opening tmp RTMetrics json file | 20260506 18:12:23.961 | INFO | Serializing RTMetrics to json | 20260506 18:12:23.991 | INFO | Serialization done, swapping tmp file now... | 20260506 18:12:23.995 | INFO | RTMetrics done | 20260506 18:12:24.080 | INFO | Finished a round of Baz IO: Wrote 2.045 GiB in 3.277 seconds | 20260506 18:12:24.080 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.20% / 0.01% / 64.79% | 20260506 18:12:24.080 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.17613% / 1.58946e-05% / 6.75519e-05% | 20260506 18:12:24.080 | INFO | Baz IO overall: | 20260506 18:12:24.080 | INFO | BytesWritten : 1801712578055 | 20260506 18:12:24.080 | INFO | NumEvents : 871403786159 | 20260506 18:12:25.821 | INFO | Analyzing chunk frames = [3411968,3412480) | 20260506 18:12:25.821 | INFO | MemInfo: { | 20260506 18:12:25.821 | INFO | "gpu" : " 17 GB", | 20260506 18:12:25.822 | INFO | "huge" : "", | 20260506 18:12:25.822 | INFO | "huge_shared" : " 27 GB", | 20260506 18:12:25.822 | INFO | "malloc" : " 14 GB", | 20260506 18:12:25.822 | INFO | "pinned" : " 29 GB" | 20260506 18:12:25.822 | INFO | } | 20260506 18:12:25.822 | INFO | Perf report for the last 5.1394 seconds: | 20260506 18:12:25.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:12:25.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:25.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:25.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:25.822 | INFO | ANALYSIS : 62.7%, 2.9, 95.3ms, 0 | 20260506 18:12:25.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:25.822 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 18:12:25.822 | INFO | PRE_HQ : 60.2%, 1.9, 58.2ms, 0 | 20260506 18:12:25.822 | INFO | BAZWRITER : 63.4%, 4.8, 156.5ms, 0 | 20260506 18:12:25.822 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 18:12:25.822 | INFO | Uploaded 36GiB to GPU in 2.0791s (17.3152GiB/s) | 20260506 18:12:25.822 | INFO | Downloaded 38GiB to GPU in 2.22172s (17.1039GiB/s) | 20260506 18:12:30.963 | INFO | Analyzing chunk frames = [3412480,3412992) | 20260506 18:12:30.963 | INFO | MemInfo: { | 20260506 18:12:30.963 | INFO | "gpu" : " 17 GB", | 20260506 18:12:30.963 | INFO | "huge" : "", | 20260506 18:12:30.963 | INFO | "huge_shared" : " 27 GB", | 20260506 18:12:30.964 | INFO | "malloc" : " 14 GB", | 20260506 18:12:30.964 | INFO | "pinned" : " 29 GB" | 20260506 18:12:30.964 | INFO | } | 20260506 18:12:30.964 | INFO | Perf report for the last 5.14194 seconds: | 20260506 18:12:30.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:12:30.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:30.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:30.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:30.964 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:12:30.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:30.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:30.964 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:12:30.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:30.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:30.964 | INFO | Uploaded 36GiB to GPU in 2.01263s (17.8871GiB/s) | 20260506 18:12:30.964 | INFO | Downloaded 34GiB to GPU in 1.94535s (17.4776GiB/s) | 20260506 18:12:36.099 | INFO | Analyzing chunk frames = [3412992,3413504) | 20260506 18:12:36.099 | INFO | MemInfo: { | 20260506 18:12:36.099 | INFO | "gpu" : " 17 GB", | 20260506 18:12:36.099 | INFO | "huge" : "", | 20260506 18:12:36.099 | INFO | "huge_shared" : " 27 GB", | 20260506 18:12:36.099 | INFO | "malloc" : " 14 GB", | 20260506 18:12:36.099 | INFO | "pinned" : " 29 GB" | 20260506 18:12:36.099 | INFO | } | 20260506 18:12:36.099 | INFO | Perf report for the last 5.13568 seconds: | 20260506 18:12:36.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:12:36.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:36.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:36.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:36.099 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 18:12:36.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:36.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:36.099 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:12:36.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:36.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:36.099 | INFO | Uploaded 36GiB to GPU in 1.98044s (18.1778GiB/s) | 20260506 18:12:36.099 | INFO | Downloaded 34GiB to GPU in 1.92353s (17.6758GiB/s) | 20260506 18:12:41.241 | INFO | Analyzing chunk frames = [3413504,3414016) | 20260506 18:12:41.241 | INFO | MemInfo: { | 20260506 18:12:41.241 | INFO | "gpu" : " 17 GB", | 20260506 18:12:41.241 | INFO | "huge" : "", | 20260506 18:12:41.241 | INFO | "huge_shared" : " 27 GB", | 20260506 18:12:41.241 | INFO | "malloc" : " 14 GB", | 20260506 18:12:41.241 | INFO | "pinned" : " 29 GB" | 20260506 18:12:41.241 | INFO | } | 20260506 18:12:41.241 | INFO | Perf report for the last 5.14194 seconds: | 20260506 18:12:41.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:12:41.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:41.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:41.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:41.241 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 18:12:41.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:41.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:41.241 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:12:41.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:41.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:41.241 | INFO | Uploaded 36GiB to GPU in 1.92777s (18.6744GiB/s) | 20260506 18:12:41.241 | INFO | Downloaded 34GiB to GPU in 1.87228s (18.1597GiB/s) | 20260506 18:12:46.381 | INFO | Analyzing chunk frames = [3414016,3414528) | 20260506 18:12:46.381 | INFO | MemInfo: { | 20260506 18:12:46.381 | INFO | "gpu" : " 17 GB", | 20260506 18:12:46.382 | INFO | "huge" : "", | 20260506 18:12:46.382 | INFO | "huge_shared" : " 27 GB", | 20260506 18:12:46.382 | INFO | "malloc" : " 14 GB", | 20260506 18:12:46.382 | INFO | "pinned" : " 29 GB" | 20260506 18:12:46.382 | INFO | } | 20260506 18:12:46.382 | INFO | Perf report for the last 5.14037 seconds: | 20260506 18:12:46.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:12:46.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:46.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:46.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:46.382 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 18:12:46.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:46.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:46.382 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 18:12:46.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:46.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:46.382 | INFO | Uploaded 36GiB to GPU in 1.96047s (18.363GiB/s) | 20260506 18:12:46.382 | INFO | Downloaded 34GiB to GPU in 1.88071s (18.0782GiB/s) | 20260506 18:12:51.522 | INFO | Analyzing chunk frames = [3414528,3415040) | 20260506 18:12:51.522 | INFO | MemInfo: { | 20260506 18:12:51.522 | INFO | "gpu" : " 17 GB", | 20260506 18:12:51.522 | INFO | "huge" : "", | 20260506 18:12:51.522 | INFO | "huge_shared" : " 27 GB", | 20260506 18:12:51.522 | INFO | "malloc" : " 14 GB", | 20260506 18:12:51.522 | INFO | "pinned" : " 29 GB" | 20260506 18:12:51.522 | INFO | } | 20260506 18:12:51.522 | INFO | Perf report for the last 5.14072 seconds: | 20260506 18:12:51.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:12:51.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:51.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:51.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:51.522 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 18:12:51.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:51.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:51.522 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:12:51.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:51.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:51.523 | INFO | Uploaded 36GiB to GPU in 2.01986s (17.823GiB/s) | 20260506 18:12:51.523 | INFO | Downloaded 34GiB to GPU in 1.95669s (17.3763GiB/s) | 20260506 18:12:56.657 | INFO | Analyzing chunk frames = [3415040,3415552) | 20260506 18:12:56.657 | INFO | MemInfo: { | 20260506 18:12:56.657 | INFO | "gpu" : " 17 GB", | 20260506 18:12:56.657 | INFO | "huge" : "", | 20260506 18:12:56.657 | INFO | "huge_shared" : " 27 GB", | 20260506 18:12:56.657 | INFO | "malloc" : " 14 GB", | 20260506 18:12:56.657 | INFO | "pinned" : " 29 GB" | 20260506 18:12:56.657 | INFO | } | 20260506 18:12:56.657 | INFO | Perf report for the last 5.13505 seconds: | 20260506 18:12:56.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:12:56.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:56.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:56.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:56.657 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 18:12:56.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:56.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:56.657 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:12:56.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:56.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:12:56.658 | INFO | Uploaded 36GiB to GPU in 2.01165s (17.8958GiB/s) | 20260506 18:12:56.658 | INFO | Downloaded 34GiB to GPU in 1.989s (17.094GiB/s) | 20260506 18:13:01.801 | INFO | Analyzing chunk frames = [3415552,3416064) | 20260506 18:13:01.801 | INFO | MemInfo: { | 20260506 18:13:01.801 | INFO | "gpu" : " 17 GB", | 20260506 18:13:01.801 | INFO | "huge" : "", | 20260506 18:13:01.801 | INFO | "huge_shared" : " 27 GB", | 20260506 18:13:01.801 | INFO | "malloc" : " 14 GB", | 20260506 18:13:01.801 | INFO | "pinned" : " 29 GB" | 20260506 18:13:01.801 | INFO | } | 20260506 18:13:01.801 | INFO | Perf report for the last 5.14326 seconds: | 20260506 18:13:01.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:13:01.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:01.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:01.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:01.801 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 18:13:01.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:01.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:01.801 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:13:01.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:01.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:01.801 | INFO | Uploaded 36GiB to GPU in 2.00419s (17.9624GiB/s) | 20260506 18:13:01.801 | INFO | Downloaded 34GiB to GPU in 1.9789s (17.1813GiB/s) | 20260506 18:13:01.801 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:13:01.801 | INFO | Total mem usage: 18 GiB | 20260506 18:13:01.801 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:13:01.801 | INFO | Recent High Water Marks Sum: 17.1 GiB (Savings: -5.19%) | 20260506 18:13:01.801 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:13:01.801 | INFO | High Water Breakdown: | 20260506 18:13:01.801 | INFO | 62.8% : 10.7 GiB ( 94.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:13:01.801 | INFO | 37.1% : 6.35 GiB ( 95% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:13:01.801 | INFO | 0.116% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:13:01.801 | INFO | ------------------------------------------ | 20260506 18:13:01.801 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:13:01.801 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:13:01.801 | INFO | Total mem usage: 26.5 GiB | 20260506 18:13:01.801 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:13:01.801 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:13:01.801 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:13:01.801 | INFO | High Water Breakdown: | 20260506 18:13:01.801 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:13:01.801 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:13:01.801 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:13:01.801 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:13:01.801 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:13:01.801 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:13:01.801 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:13:01.801 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:13:01.801 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:13:01.801 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:13:01.801 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:13:01.801 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:13:01.801 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:13:01.801 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:13:01.801 | INFO | ------------------------------------------ | 20260506 18:13:01.801 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:13:01.801 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:13:01.801 | INFO | Total mem usage: 25.6 GiB | 20260506 18:13:01.801 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:13:01.801 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:13:01.801 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:13:01.801 | INFO | High Water Breakdown: | 20260506 18:13:01.801 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:13:01.801 | INFO | ------------------------------------------ | 20260506 18:13:01.801 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:13:01.801 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:13:01.801 | INFO | Total mem usage: 15.7 GiB | 20260506 18:13:01.801 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:13:01.801 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:13:01.801 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:13:01.801 | INFO | High Water Breakdown: | 20260506 18:13:01.801 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:13:01.801 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:13:01.801 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:13:01.801 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:13:01.801 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:13:01.801 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:13:01.801 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:13:01.802 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:13:01.802 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:13:01.802 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:13:01.802 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:13:01.802 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:13:01.802 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:13:01.802 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:13:01.802 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:13:01.802 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:13:01.802 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:13:01.802 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:13:01.802 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:13:01.802 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:13:01.802 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:13:01.802 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:13:01.802 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:13:01.802 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:13:01.802 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:13:01.802 | INFO | ------------------------------------------ | 20260506 18:13:01.802 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:13:01.802 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:13:01.802 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:13:01.802 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:13:01.802 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.54e+01, 1.34e+01 / 2.23e+00 | 20260506 18:13:01.802 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 9.99e-03 / 8.01e-03 | 20260506 18:13:01.802 | INFO | PerfReport: 11.60%, Baseline -- 3.63e+00 / 4.43e+00, 5.50e-01 / 3.65e-01 | 20260506 18:13:01.802 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.82e-01 / 1.40e-01 | 20260506 18:13:01.802 | INFO | PerfReport: 15.94%, DME -- 3.99e+01 / 4.42e+01, 3.99e+00 / 2.45e+00 | 20260506 18:13:01.802 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.44e+00 / 1.86e+00 | 20260506 18:13:01.802 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.26e+00 / 2.84e+00, 3.86e-01 / 2.39e-01 | 20260506 18:13:01.802 | INFO | PerfReport: 3.41%, Metrics -- 1.07e+00 / 1.42e+00, 2.75e-01 / 1.79e-01 | 20260506 18:13:01.802 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.69e-01 / 4.13e-01, 5.75e-02 / 2.20e-02 | 20260506 18:13:01.802 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:13:01.802 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:13:01.802 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:13:01.802 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 18:13:01.802 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 18:13:01.802 | INFO | PerfReport: 21.15%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 18:13:01.802 | INFO | PerfReport: 32.20%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 18:13:01.802 | INFO | PerfReport: 1.47%, LockUpload -- 1.43e+00 / 0.00e+00, 5.56e+00 / 0.00e+00 | 20260506 18:13:01.802 | INFO | PerfReport: 1.08%, LockDownload -- 1.05e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260506 18:13:01.802 | INFO | PerfReport: 22.60%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 18:13:01.802 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260506 18:13:01.911 | INFO | Starting a round of Baz IO | 20260506 18:13:05.051 | INFO | Populating RTMetrics report | 20260506 18:13:05.051 | INFO | RTMetrics report populated | 20260506 18:13:05.051 | INFO | Opening tmp RTMetrics json file | 20260506 18:13:05.196 | INFO | Serializing RTMetrics to json | 20260506 18:13:05.239 | INFO | Serialization done, swapping tmp file now... | 20260506 18:13:05.240 | INFO | RTMetrics done | 20260506 18:13:05.272 | INFO | Finished a round of Baz IO: Wrote 2.044 GiB in 3.361 seconds | 20260506 18:13:05.272 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.20% / 0.01% / 64.80% | 20260506 18:13:05.272 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.17079% / 1.19209e-05% / 5.5631e-05% | 20260506 18:13:05.272 | INFO | Baz IO overall: | 20260506 18:13:05.272 | INFO | BytesWritten : 1803907348927 | 20260506 18:13:05.272 | INFO | NumEvents : 872460578122 | 20260506 18:13:06.937 | INFO | Analyzing chunk frames = [3416064,3416576) | 20260506 18:13:06.937 | INFO | MemInfo: { | 20260506 18:13:06.937 | INFO | "gpu" : " 17 GB", | 20260506 18:13:06.937 | INFO | "huge" : "", | 20260506 18:13:06.937 | INFO | "huge_shared" : " 27 GB", | 20260506 18:13:06.937 | INFO | "malloc" : " 14 GB", | 20260506 18:13:06.937 | INFO | "pinned" : " 29 GB" | 20260506 18:13:06.937 | INFO | } | 20260506 18:13:06.937 | INFO | Perf report for the last 5.13628 seconds: | 20260506 18:13:06.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:13:06.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:06.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:06.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:06.937 | INFO | ANALYSIS : 62.7%, 2.9, 95.3ms, 0 | 20260506 18:13:06.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:06.937 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260506 18:13:06.937 | INFO | PRE_HQ : 60.3%, 1.9, 58.3ms, 0 | 20260506 18:13:06.937 | INFO | BAZWRITER : 65.2%, 4.8, 160.5ms, 0 | 20260506 18:13:06.937 | INFO | BAZ_SAMPLING : 3.2%, 1.0, 1.7ms, 0 | 20260506 18:13:06.937 | INFO | Uploaded 36GiB to GPU in 2.07734s (17.3299GiB/s) | 20260506 18:13:06.937 | INFO | Downloaded 38GiB to GPU in 2.20362s (17.2444GiB/s) | 20260506 18:13:12.072 | INFO | Analyzing chunk frames = [3416576,3417088) | 20260506 18:13:12.072 | INFO | MemInfo: { | 20260506 18:13:12.072 | INFO | "gpu" : " 17 GB", | 20260506 18:13:12.072 | INFO | "huge" : "", | 20260506 18:13:12.072 | INFO | "huge_shared" : " 27 GB", | 20260506 18:13:12.072 | INFO | "malloc" : " 14 GB", | 20260506 18:13:12.072 | INFO | "pinned" : " 29 GB" | 20260506 18:13:12.072 | INFO | } | 20260506 18:13:12.072 | INFO | Perf report for the last 5.13538 seconds: | 20260506 18:13:12.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:13:12.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:12.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:12.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:12.072 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 18:13:12.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:12.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:12.072 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:13:12.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:12.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:12.073 | INFO | Uploaded 36GiB to GPU in 2.0128s (17.8856GiB/s) | 20260506 18:13:12.073 | INFO | Downloaded 34GiB to GPU in 1.9418s (17.5096GiB/s) | 20260506 18:13:17.228 | INFO | Analyzing chunk frames = [3417088,3417600) | 20260506 18:13:17.228 | INFO | MemInfo: { | 20260506 18:13:17.228 | INFO | "gpu" : " 17 GB", | 20260506 18:13:17.228 | INFO | "huge" : "", | 20260506 18:13:17.228 | INFO | "huge_shared" : " 27 GB", | 20260506 18:13:17.228 | INFO | "malloc" : " 14 GB", | 20260506 18:13:17.228 | INFO | "pinned" : " 29 GB" | 20260506 18:13:17.228 | INFO | } | 20260506 18:13:17.228 | INFO | Perf report for the last 5.15551 seconds: | 20260506 18:13:17.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:13:17.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:17.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:17.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:17.228 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260506 18:13:17.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:17.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:17.228 | INFO | PRE_HQ : 26.6%, 1.0, 13.9ms, 0 | 20260506 18:13:17.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:17.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:17.228 | INFO | Uploaded 36GiB to GPU in 1.97373s (18.2395GiB/s) | 20260506 18:13:17.228 | INFO | Downloaded 34GiB to GPU in 1.92476s (17.6645GiB/s) | 20260506 18:13:22.361 | INFO | Analyzing chunk frames = [3417600,3418112) | 20260506 18:13:22.361 | INFO | MemInfo: { | 20260506 18:13:22.361 | INFO | "gpu" : " 17 GB", | 20260506 18:13:22.361 | INFO | "huge" : "", | 20260506 18:13:22.361 | INFO | "huge_shared" : " 27 GB", | 20260506 18:13:22.361 | INFO | "malloc" : " 14 GB", | 20260506 18:13:22.361 | INFO | "pinned" : " 29 GB" | 20260506 18:13:22.361 | INFO | } | 20260506 18:13:22.361 | INFO | Perf report for the last 5.13358 seconds: | 20260506 18:13:22.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:13:22.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:22.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:22.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:22.362 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 18:13:22.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:22.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:22.362 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:13:22.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:22.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:22.362 | INFO | Uploaded 36GiB to GPU in 1.93622s (18.5929GiB/s) | 20260506 18:13:22.362 | INFO | Downloaded 34GiB to GPU in 1.86831s (18.1983GiB/s) | 20260506 18:13:27.497 | INFO | Analyzing chunk frames = [3418112,3418624) | 20260506 18:13:27.497 | INFO | MemInfo: { | 20260506 18:13:27.497 | INFO | "gpu" : " 17 GB", | 20260506 18:13:27.497 | INFO | "huge" : "", | 20260506 18:13:27.497 | INFO | "huge_shared" : " 27 GB", | 20260506 18:13:27.497 | INFO | "malloc" : " 14 GB", | 20260506 18:13:27.497 | INFO | "pinned" : " 29 GB" | 20260506 18:13:27.497 | INFO | } | 20260506 18:13:27.497 | INFO | Perf report for the last 5.13537 seconds: | 20260506 18:13:27.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:13:27.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:27.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:27.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:27.497 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 18:13:27.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:27.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:27.497 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:13:27.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:27.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:27.497 | INFO | Uploaded 36GiB to GPU in 2.00874s (17.9217GiB/s) | 20260506 18:13:27.497 | INFO | Downloaded 34GiB to GPU in 1.91612s (17.7442GiB/s) | 20260506 18:13:32.636 | INFO | Analyzing chunk frames = [3418624,3419136) | 20260506 18:13:32.636 | INFO | MemInfo: { | 20260506 18:13:32.636 | INFO | "gpu" : " 17 GB", | 20260506 18:13:32.636 | INFO | "huge" : "", | 20260506 18:13:32.636 | INFO | "huge_shared" : " 27 GB", | 20260506 18:13:32.636 | INFO | "malloc" : " 14 GB", | 20260506 18:13:32.636 | INFO | "pinned" : " 29 GB" | 20260506 18:13:32.636 | INFO | } | 20260506 18:13:32.636 | INFO | Perf report for the last 5.13961 seconds: | 20260506 18:13:32.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:13:32.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:32.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:32.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:32.637 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 18:13:32.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:32.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:32.637 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:13:32.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:32.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:32.637 | INFO | Uploaded 36GiB to GPU in 2.02696s (17.7606GiB/s) | 20260506 18:13:32.637 | INFO | Downloaded 34GiB to GPU in 1.94554s (17.4759GiB/s) | 20260506 18:13:37.772 | INFO | Analyzing chunk frames = [3419136,3419648) | 20260506 18:13:37.772 | INFO | MemInfo: { | 20260506 18:13:37.772 | INFO | "gpu" : " 17 GB", | 20260506 18:13:37.772 | INFO | "huge" : "", | 20260506 18:13:37.772 | INFO | "huge_shared" : " 27 GB", | 20260506 18:13:37.772 | INFO | "malloc" : " 14 GB", | 20260506 18:13:37.772 | INFO | "pinned" : " 29 GB" | 20260506 18:13:37.772 | INFO | } | 20260506 18:13:37.772 | INFO | Perf report for the last 5.13577 seconds: | 20260506 18:13:37.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:13:37.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:37.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:37.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:37.772 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 18:13:37.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:37.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:37.772 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 18:13:37.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:37.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:37.773 | INFO | Uploaded 36GiB to GPU in 2.00641s (17.9425GiB/s) | 20260506 18:13:37.773 | INFO | Downloaded 34GiB to GPU in 2.01403s (16.8816GiB/s) | 20260506 18:13:42.912 | INFO | Analyzing chunk frames = [3419648,3420160) | 20260506 18:13:42.912 | INFO | MemInfo: { | 20260506 18:13:42.912 | INFO | "gpu" : " 17 GB", | 20260506 18:13:42.912 | INFO | "huge" : "", | 20260506 18:13:42.912 | INFO | "huge_shared" : " 27 GB", | 20260506 18:13:42.912 | INFO | "malloc" : " 14 GB", | 20260506 18:13:42.912 | INFO | "pinned" : " 29 GB" | 20260506 18:13:42.912 | INFO | } | 20260506 18:13:42.912 | INFO | Perf report for the last 5.1402 seconds: | 20260506 18:13:42.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:13:42.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:42.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:42.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:42.912 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:13:42.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:42.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:42.913 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:13:42.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:42.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:42.913 | INFO | Uploaded 36GiB to GPU in 1.99791s (18.0188GiB/s) | 20260506 18:13:42.913 | INFO | Downloaded 34GiB to GPU in 1.98472s (17.1309GiB/s) | 20260506 18:13:43.026 | INFO | Starting a round of Baz IO | 20260506 18:13:46.164 | INFO | Populating RTMetrics report | 20260506 18:13:46.165 | INFO | RTMetrics report populated | 20260506 18:13:46.165 | INFO | Opening tmp RTMetrics json file | 20260506 18:13:46.198 | INFO | Serializing RTMetrics to json | 20260506 18:13:46.258 | INFO | Serialization done, swapping tmp file now... | 20260506 18:13:46.260 | INFO | RTMetrics done | 20260506 18:13:46.481 | INFO | Finished a round of Baz IO: Wrote 2.047 GiB in 3.455 seconds | 20260506 18:13:46.481 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.19% / 0.01% / 64.80% | 20260506 18:13:46.481 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.1745% / 1.19209e-05% / 7.94729e-05% | 20260506 18:13:46.481 | INFO | Baz IO overall: | 20260506 18:13:46.481 | INFO | BytesWritten : 1806104982292 | 20260506 18:13:46.481 | INFO | NumEvents : 873520232123 | 20260506 18:13:48.057 | INFO | Analyzing chunk frames = [3420160,3420672) | 20260506 18:13:48.057 | INFO | MemInfo: { | 20260506 18:13:48.057 | INFO | "gpu" : " 17 GB", | 20260506 18:13:48.057 | INFO | "huge" : "", | 20260506 18:13:48.057 | INFO | "huge_shared" : " 27 GB", | 20260506 18:13:48.057 | INFO | "malloc" : " 14 GB", | 20260506 18:13:48.057 | INFO | "pinned" : " 29 GB" | 20260506 18:13:48.057 | INFO | } | 20260506 18:13:48.057 | INFO | Perf report for the last 5.14442 seconds: | 20260506 18:13:48.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:13:48.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:48.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:48.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:48.057 | INFO | ANALYSIS : 62.6%, 2.9, 95.2ms, 0 | 20260506 18:13:48.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:48.057 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 18:13:48.057 | INFO | PRE_HQ : 60.1%, 1.9, 58.2ms, 0 | 20260506 18:13:48.057 | INFO | BAZWRITER : 66.9%, 4.8, 165.0ms, 0 | 20260506 18:13:48.057 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 18:13:48.057 | INFO | Uploaded 36GiB to GPU in 2.01475s (17.8682GiB/s) | 20260506 18:13:48.057 | INFO | Downloaded 38GiB to GPU in 2.21087s (17.1878GiB/s) | 20260506 18:13:53.190 | INFO | Analyzing chunk frames = [3420672,3421184) | 20260506 18:13:53.190 | INFO | MemInfo: { | 20260506 18:13:53.190 | INFO | "gpu" : " 17 GB", | 20260506 18:13:53.190 | INFO | "huge" : "", | 20260506 18:13:53.190 | INFO | "huge_shared" : " 27 GB", | 20260506 18:13:53.190 | INFO | "malloc" : " 14 GB", | 20260506 18:13:53.190 | INFO | "pinned" : " 29 GB" | 20260506 18:13:53.190 | INFO | } | 20260506 18:13:53.190 | INFO | Perf report for the last 5.13277 seconds: | 20260506 18:13:53.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:13:53.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:53.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:53.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:53.190 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 18:13:53.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:53.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:53.190 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:13:53.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:53.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:53.190 | INFO | Uploaded 36GiB to GPU in 1.98798s (18.1088GiB/s) | 20260506 18:13:53.190 | INFO | Downloaded 34GiB to GPU in 1.92164s (17.6932GiB/s) | 20260506 18:13:58.330 | INFO | Analyzing chunk frames = [3421184,3421696) | 20260506 18:13:58.330 | INFO | MemInfo: { | 20260506 18:13:58.330 | INFO | "gpu" : " 17 GB", | 20260506 18:13:58.330 | INFO | "huge" : "", | 20260506 18:13:58.330 | INFO | "huge_shared" : " 27 GB", | 20260506 18:13:58.330 | INFO | "malloc" : " 14 GB", | 20260506 18:13:58.330 | INFO | "pinned" : " 29 GB" | 20260506 18:13:58.330 | INFO | } | 20260506 18:13:58.330 | INFO | Perf report for the last 5.14055 seconds: | 20260506 18:13:58.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:13:58.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:58.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:58.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:58.330 | INFO | ANALYSIS : 60.2%, 3.0, 93.5ms, 0 | 20260506 18:13:58.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:58.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:58.330 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:13:58.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:58.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:13:58.330 | INFO | Uploaded 36GiB to GPU in 1.9416s (18.5414GiB/s) | 20260506 18:13:58.331 | INFO | Downloaded 34GiB to GPU in 1.88214s (18.0646GiB/s) | 20260506 18:14:03.475 | INFO | Analyzing chunk frames = [3421696,3422208) | 20260506 18:14:03.475 | INFO | MemInfo: { | 20260506 18:14:03.475 | INFO | "gpu" : " 17 GB", | 20260506 18:14:03.475 | INFO | "huge" : "", | 20260506 18:14:03.475 | INFO | "huge_shared" : " 27 GB", | 20260506 18:14:03.475 | INFO | "malloc" : " 14 GB", | 20260506 18:14:03.475 | INFO | "pinned" : " 29 GB" | 20260506 18:14:03.475 | INFO | } | 20260506 18:14:03.475 | INFO | Perf report for the last 5.14513 seconds: | 20260506 18:14:03.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:14:03.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:03.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:03.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:03.475 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:14:03.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:03.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:03.475 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:14:03.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:03.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:03.476 | INFO | Uploaded 36GiB to GPU in 1.91965s (18.7534GiB/s) | 20260506 18:14:03.476 | INFO | Downloaded 34GiB to GPU in 1.86543s (18.2264GiB/s) | 20260506 18:14:08.612 | INFO | Analyzing chunk frames = [3422208,3422720) | 20260506 18:14:08.612 | INFO | MemInfo: { | 20260506 18:14:08.612 | INFO | "gpu" : " 17 GB", | 20260506 18:14:08.612 | INFO | "huge" : "", | 20260506 18:14:08.612 | INFO | "huge_shared" : " 27 GB", | 20260506 18:14:08.612 | INFO | "malloc" : " 14 GB", | 20260506 18:14:08.612 | INFO | "pinned" : " 29 GB" | 20260506 18:14:08.612 | INFO | } | 20260506 18:14:08.612 | INFO | Perf report for the last 5.13642 seconds: | 20260506 18:14:08.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:14:08.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:08.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:08.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:08.612 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 18:14:08.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:08.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:08.612 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:14:08.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:08.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:08.612 | INFO | Uploaded 36GiB to GPU in 2.01389s (17.8758GiB/s) | 20260506 18:14:08.612 | INFO | Downloaded 34GiB to GPU in 1.91136s (17.7884GiB/s) | 20260506 18:14:13.749 | INFO | Analyzing chunk frames = [3422720,3423232) | 20260506 18:14:13.749 | INFO | MemInfo: { | 20260506 18:14:13.749 | INFO | "gpu" : " 17 GB", | 20260506 18:14:13.749 | INFO | "huge" : "", | 20260506 18:14:13.749 | INFO | "huge_shared" : " 27 GB", | 20260506 18:14:13.749 | INFO | "malloc" : " 14 GB", | 20260506 18:14:13.749 | INFO | "pinned" : " 29 GB" | 20260506 18:14:13.749 | INFO | } | 20260506 18:14:13.749 | INFO | Perf report for the last 5.137 seconds: | 20260506 18:14:13.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:14:13.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:13.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:13.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:13.749 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 18:14:13.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:13.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:13.749 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:14:13.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:13.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:13.749 | INFO | Uploaded 36GiB to GPU in 2.02911s (17.7418GiB/s) | 20260506 18:14:13.749 | INFO | Downloaded 34GiB to GPU in 1.94961s (17.4394GiB/s) | 20260506 18:14:18.891 | INFO | Analyzing chunk frames = [3423232,3423744) | 20260506 18:14:18.891 | INFO | MemInfo: { | 20260506 18:14:18.891 | INFO | "gpu" : " 17 GB", | 20260506 18:14:18.891 | INFO | "huge" : "", | 20260506 18:14:18.891 | INFO | "huge_shared" : " 27 GB", | 20260506 18:14:18.891 | INFO | "malloc" : " 14 GB", | 20260506 18:14:18.891 | INFO | "pinned" : " 29 GB" | 20260506 18:14:18.891 | INFO | } | 20260506 18:14:18.891 | INFO | Perf report for the last 5.14205 seconds: | 20260506 18:14:18.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:14:18.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:18.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:18.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:18.891 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 18:14:18.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:18.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:18.891 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 18:14:18.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:18.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:18.891 | INFO | Uploaded 36GiB to GPU in 1.9901s (18.0895GiB/s) | 20260506 18:14:18.891 | INFO | Downloaded 34GiB to GPU in 2.01731s (16.8541GiB/s) | 20260506 18:14:24.027 | INFO | Analyzing chunk frames = [3423744,3424256) | 20260506 18:14:24.027 | INFO | MemInfo: { | 20260506 18:14:24.027 | INFO | "gpu" : " 17 GB", | 20260506 18:14:24.027 | INFO | "huge" : "", | 20260506 18:14:24.027 | INFO | "huge_shared" : " 27 GB", | 20260506 18:14:24.027 | INFO | "malloc" : " 14 GB", | 20260506 18:14:24.027 | INFO | "pinned" : " 29 GB" | 20260506 18:14:24.027 | INFO | } | 20260506 18:14:24.027 | INFO | Perf report for the last 5.13608 seconds: | 20260506 18:14:24.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:14:24.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:24.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:24.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:24.027 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 18:14:24.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:24.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:24.027 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:14:24.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:24.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:24.027 | INFO | Uploaded 36GiB to GPU in 2.0074s (17.9337GiB/s) | 20260506 18:14:24.027 | INFO | Downloaded 34GiB to GPU in 1.98349s (17.1415GiB/s) | 20260506 18:14:24.027 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:14:24.027 | INFO | Total mem usage: 18 GiB | 20260506 18:14:24.027 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:14:24.027 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.3%) | 20260506 18:14:24.027 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:14:24.027 | INFO | High Water Breakdown: | 20260506 18:14:24.027 | INFO | 61.3% : 9.49 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:14:24.027 | INFO | 38.6% : 5.97 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:14:24.027 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:14:24.027 | INFO | ------------------------------------------ | 20260506 18:14:24.027 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:14:24.027 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:14:24.027 | INFO | Total mem usage: 26.5 GiB | 20260506 18:14:24.027 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:14:24.027 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:14:24.027 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:14:24.027 | INFO | High Water Breakdown: | 20260506 18:14:24.027 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:14:24.027 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:14:24.027 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:14:24.028 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:14:24.028 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:14:24.028 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:14:24.028 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:14:24.028 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:14:24.028 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:14:24.028 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:14:24.028 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:14:24.028 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:14:24.028 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:14:24.028 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:14:24.028 | INFO | ------------------------------------------ | 20260506 18:14:24.028 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:14:24.028 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:14:24.028 | INFO | Total mem usage: 25.6 GiB | 20260506 18:14:24.028 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:14:24.028 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:14:24.028 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:14:24.028 | INFO | High Water Breakdown: | 20260506 18:14:24.028 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:14:24.028 | INFO | ------------------------------------------ | 20260506 18:14:24.028 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:14:24.028 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:14:24.028 | INFO | Total mem usage: 15.7 GiB | 20260506 18:14:24.028 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:14:24.028 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:14:24.028 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 18:14:24.028 | INFO | High Water Breakdown: | 20260506 18:14:24.028 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:14:24.028 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:14:24.028 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:14:24.028 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:14:24.028 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:14:24.028 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:14:24.028 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:14:24.028 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:14:24.028 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:14:24.028 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:14:24.028 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:14:24.028 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:14:24.028 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:14:24.028 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:14:24.028 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:14:24.028 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:14:24.028 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:14:24.028 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:14:24.028 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:14:24.028 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:14:24.028 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:14:24.028 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:14:24.028 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:14:24.028 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:14:24.028 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:14:24.028 | INFO | ------------------------------------------ | 20260506 18:14:24.028 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:14:24.028 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:14:24.028 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:14:24.028 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:14:24.028 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.58e+01, 1.35e+01 / 2.98e+00 | 20260506 18:14:24.028 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.02e-01, 1.01e-02 / 6.16e-03 | 20260506 18:14:24.028 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.45e+00, 5.55e-01 / 2.92e-01 | 20260506 18:14:24.028 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.47e+00, 1.84e-01 / 1.11e-01 | 20260506 18:14:24.029 | INFO | PerfReport: 16.02%, DME -- 4.01e+01 / 4.51e+01, 4.23e+00 / 3.02e+00 | 20260506 18:14:24.029 | INFO | PerfReport: 56.94%, FrameLabeling -- 1.78e+01 / 2.00e+01, 1.34e+00 / 1.64e+00 | 20260506 18:14:24.029 | INFO | PerfReport: 7.26%, PulseAccumulating -- 2.27e+00 / 2.81e+00, 3.86e-01 / 2.09e-01 | 20260506 18:14:24.029 | INFO | PerfReport: 3.43%, Metrics -- 1.07e+00 / 1.40e+00, 2.82e-01 / 1.52e-01 | 20260506 18:14:24.029 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.69e-01 / 4.15e-01, 5.78e-02 / 1.91e-02 | 20260506 18:14:24.029 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:14:24.029 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:14:24.029 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:14:24.029 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 18:14:24.029 | INFO | PerfReport: 21.43%, Upload -- 2.08e+01 / 0.00e+00, 6.91e+00 / 0.00e+00 | 20260506 18:14:24.029 | INFO | PerfReport: 21.16%, Download -- 2.05e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260506 18:14:24.029 | INFO | PerfReport: 32.22%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:14:24.029 | INFO | PerfReport: 1.35%, LockUpload -- 1.31e+00 / 0.00e+00, 5.38e+00 / 0.00e+00 | 20260506 18:14:24.029 | INFO | PerfReport: 1.15%, LockDownload -- 1.11e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 18:14:24.029 | INFO | PerfReport: 22.68%, LockCompute -- 2.20e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 18:14:24.029 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260506 18:14:24.141 | INFO | Starting a round of Baz IO | 20260506 18:14:27.281 | INFO | Populating RTMetrics report | 20260506 18:14:27.282 | INFO | RTMetrics report populated | 20260506 18:14:27.282 | INFO | Opening tmp RTMetrics json file | 20260506 18:14:27.328 | INFO | Serializing RTMetrics to json | 20260506 18:14:27.334 | INFO | Serialization done, swapping tmp file now... | 20260506 18:14:27.339 | INFO | RTMetrics done | 20260506 18:14:27.741 | INFO | Finished a round of Baz IO: Wrote 2.046 GiB in 3.599 seconds | 20260506 18:14:27.741 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.19% / 0.01% / 64.81% | 20260506 18:14:27.741 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.17166% / 3.97364e-06% / 7.94729e-05% | 20260506 18:14:27.741 | INFO | Baz IO overall: | 20260506 18:14:27.741 | INFO | BytesWritten : 1808301700917 | 20260506 18:14:27.741 | INFO | NumEvents : 874579401600 | 20260506 18:14:29.165 | INFO | Analyzing chunk frames = [3424256,3424768) | 20260506 18:14:29.165 | INFO | MemInfo: { | 20260506 18:14:29.165 | INFO | "gpu" : " 17 GB", | 20260506 18:14:29.165 | INFO | "huge" : "", | 20260506 18:14:29.165 | INFO | "huge_shared" : " 27 GB", | 20260506 18:14:29.165 | INFO | "malloc" : " 14 GB", | 20260506 18:14:29.165 | INFO | "pinned" : " 29 GB" | 20260506 18:14:29.165 | INFO | } | 20260506 18:14:29.165 | INFO | Perf report for the last 5.13774 seconds: | 20260506 18:14:29.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:14:29.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:29.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:29.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:29.165 | INFO | ANALYSIS : 62.8%, 2.9, 95.5ms, 0 | 20260506 18:14:29.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:29.165 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 18:14:29.165 | INFO | PRE_HQ : 60.1%, 1.9, 57.7ms, 0 | 20260506 18:14:29.165 | INFO | BAZWRITER : 69.7%, 4.8, 171.9ms, 0 | 20260506 18:14:29.165 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 18:14:29.165 | INFO | Uploaded 36GiB to GPU in 2.06146s (17.4634GiB/s) | 20260506 18:14:29.165 | INFO | Downloaded 38GiB to GPU in 2.21785s (17.1337GiB/s) | 20260506 18:14:34.313 | INFO | Analyzing chunk frames = [3424768,3425280) | 20260506 18:14:34.313 | INFO | MemInfo: { | 20260506 18:14:34.313 | INFO | "gpu" : " 17 GB", | 20260506 18:14:34.313 | INFO | "huge" : "", | 20260506 18:14:34.313 | INFO | "huge_shared" : " 27 GB", | 20260506 18:14:34.313 | INFO | "malloc" : " 14 GB", | 20260506 18:14:34.313 | INFO | "pinned" : " 29 GB" | 20260506 18:14:34.313 | INFO | } | 20260506 18:14:34.313 | INFO | Perf report for the last 5.14834 seconds: | 20260506 18:14:34.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:14:34.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:34.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:34.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:34.313 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 18:14:34.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:34.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:34.313 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:14:34.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:34.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:34.313 | INFO | Uploaded 36GiB to GPU in 2.02321s (17.7935GiB/s) | 20260506 18:14:34.313 | INFO | Downloaded 34GiB to GPU in 1.97445s (17.22GiB/s) | 20260506 18:14:39.447 | INFO | Analyzing chunk frames = [3425280,3425792) | 20260506 18:14:39.447 | INFO | MemInfo: { | 20260506 18:14:39.447 | INFO | "gpu" : " 17 GB", | 20260506 18:14:39.447 | INFO | "huge" : "", | 20260506 18:14:39.447 | INFO | "huge_shared" : " 27 GB", | 20260506 18:14:39.447 | INFO | "malloc" : " 14 GB", | 20260506 18:14:39.447 | INFO | "pinned" : " 29 GB" | 20260506 18:14:39.447 | INFO | } | 20260506 18:14:39.447 | INFO | Perf report for the last 5.13392 seconds: | 20260506 18:14:39.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:14:39.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:39.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:39.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:39.447 | INFO | ANALYSIS : 61.1%, 3.0, 94.5ms, 0 | 20260506 18:14:39.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:39.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:39.447 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:14:39.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:39.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:39.447 | INFO | Uploaded 36GiB to GPU in 1.98642s (18.123GiB/s) | 20260506 18:14:39.447 | INFO | Downloaded 34GiB to GPU in 1.93125s (17.6052GiB/s) | 20260506 18:14:44.588 | INFO | Analyzing chunk frames = [3425792,3426304) | 20260506 18:14:44.588 | INFO | MemInfo: { | 20260506 18:14:44.588 | INFO | "gpu" : " 17 GB", | 20260506 18:14:44.588 | INFO | "huge" : "", | 20260506 18:14:44.588 | INFO | "huge_shared" : " 27 GB", | 20260506 18:14:44.588 | INFO | "malloc" : " 14 GB", | 20260506 18:14:44.588 | INFO | "pinned" : " 29 GB" | 20260506 18:14:44.588 | INFO | } | 20260506 18:14:44.589 | INFO | Perf report for the last 5.14149 seconds: | 20260506 18:14:44.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:14:44.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:44.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:44.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:44.589 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 18:14:44.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:44.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:44.589 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:14:44.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:44.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:44.589 | INFO | Uploaded 36GiB to GPU in 1.94628s (18.4969GiB/s) | 20260506 18:14:44.589 | INFO | Downloaded 34GiB to GPU in 1.87706s (18.1134GiB/s) | 20260506 18:14:49.729 | INFO | Analyzing chunk frames = [3426304,3426816) | 20260506 18:14:49.729 | INFO | MemInfo: { | 20260506 18:14:49.729 | INFO | "gpu" : " 17 GB", | 20260506 18:14:49.729 | INFO | "huge" : "", | 20260506 18:14:49.729 | INFO | "huge_shared" : " 27 GB", | 20260506 18:14:49.729 | INFO | "malloc" : " 14 GB", | 20260506 18:14:49.729 | INFO | "pinned" : " 29 GB" | 20260506 18:14:49.729 | INFO | } | 20260506 18:14:49.730 | INFO | Perf report for the last 5.14099 seconds: | 20260506 18:14:49.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:14:49.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:49.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:49.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:49.730 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 18:14:49.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:49.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:49.730 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:14:49.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:49.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:49.730 | INFO | Uploaded 36GiB to GPU in 2.01098s (17.9017GiB/s) | 20260506 18:14:49.730 | INFO | Downloaded 34GiB to GPU in 1.90873s (17.8129GiB/s) | 20260506 18:14:54.866 | INFO | Analyzing chunk frames = [3426816,3427328) | 20260506 18:14:54.866 | INFO | MemInfo: { | 20260506 18:14:54.866 | INFO | "gpu" : " 17 GB", | 20260506 18:14:54.866 | INFO | "huge" : "", | 20260506 18:14:54.866 | INFO | "huge_shared" : " 27 GB", | 20260506 18:14:54.866 | INFO | "malloc" : " 14 GB", | 20260506 18:14:54.866 | INFO | "pinned" : " 29 GB" | 20260506 18:14:54.866 | INFO | } | 20260506 18:14:54.866 | INFO | Perf report for the last 5.13658 seconds: | 20260506 18:14:54.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:14:54.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:54.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:54.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:54.866 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 18:14:54.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:54.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:54.866 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:14:54.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:54.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:14:54.866 | INFO | Uploaded 36GiB to GPU in 2.034s (17.6991GiB/s) | 20260506 18:14:54.866 | INFO | Downloaded 34GiB to GPU in 1.94484s (17.4821GiB/s) | 20260506 18:15:00.003 | INFO | Analyzing chunk frames = [3427328,3427840) | 20260506 18:15:00.003 | INFO | MemInfo: { | 20260506 18:15:00.003 | INFO | "gpu" : " 17 GB", | 20260506 18:15:00.003 | INFO | "huge" : "", | 20260506 18:15:00.003 | INFO | "huge_shared" : " 27 GB", | 20260506 18:15:00.003 | INFO | "malloc" : " 14 GB", | 20260506 18:15:00.003 | INFO | "pinned" : " 29 GB" | 20260506 18:15:00.003 | INFO | } | 20260506 18:15:00.003 | INFO | Perf report for the last 5.13682 seconds: | 20260506 18:15:00.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:15:00.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:00.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:00.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:00.003 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 18:15:00.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:00.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:00.003 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:15:00.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:00.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:00.003 | INFO | Uploaded 36GiB to GPU in 1.9983s (18.0153GiB/s) | 20260506 18:15:00.003 | INFO | Downloaded 34GiB to GPU in 2.01591s (16.8659GiB/s) | 20260506 18:15:05.138 | INFO | Analyzing chunk frames = [3427840,3428352) | 20260506 18:15:05.138 | INFO | MemInfo: { | 20260506 18:15:05.138 | INFO | "gpu" : " 17 GB", | 20260506 18:15:05.138 | INFO | "huge" : "", | 20260506 18:15:05.138 | INFO | "huge_shared" : " 27 GB", | 20260506 18:15:05.138 | INFO | "malloc" : " 14 GB", | 20260506 18:15:05.138 | INFO | "pinned" : " 29 GB" | 20260506 18:15:05.139 | INFO | } | 20260506 18:15:05.139 | INFO | Perf report for the last 5.13559 seconds: | 20260506 18:15:05.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:15:05.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:05.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:05.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:05.139 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 18:15:05.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:05.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:05.139 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:15:05.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:05.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:05.139 | INFO | Uploaded 36GiB to GPU in 2.01525s (17.8638GiB/s) | 20260506 18:15:05.139 | INFO | Downloaded 34GiB to GPU in 1.99235s (17.0653GiB/s) | 20260506 18:15:05.257 | INFO | Starting a round of Baz IO | 20260506 18:15:08.389 | INFO | Populating RTMetrics report | 20260506 18:15:08.389 | INFO | RTMetrics report populated | 20260506 18:15:08.389 | INFO | Opening tmp RTMetrics json file | 20260506 18:15:08.406 | INFO | Serializing RTMetrics to json | 20260506 18:15:08.436 | INFO | Serialization done, swapping tmp file now... | 20260506 18:15:08.460 | INFO | RTMetrics done | 20260506 18:15:08.782 | INFO | Finished a round of Baz IO: Wrote 2.043 GiB in 3.524 seconds | 20260506 18:15:08.782 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.18% / 0.01% / 64.81% | 20260506 18:15:08.782 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.16694% / 7.94729e-06% / 7.94729e-05% | 20260506 18:15:08.782 | INFO | Baz IO overall: | 20260506 18:15:08.782 | INFO | BytesWritten : 1810495383122 | 20260506 18:15:08.782 | INFO | NumEvents : 875636652724 | 20260506 18:15:10.280 | INFO | Analyzing chunk frames = [3428352,3428864) | 20260506 18:15:10.280 | INFO | MemInfo: { | 20260506 18:15:10.280 | INFO | "gpu" : " 17 GB", | 20260506 18:15:10.280 | INFO | "huge" : "", | 20260506 18:15:10.280 | INFO | "huge_shared" : " 27 GB", | 20260506 18:15:10.280 | INFO | "malloc" : " 14 GB", | 20260506 18:15:10.280 | INFO | "pinned" : " 29 GB" | 20260506 18:15:10.280 | INFO | } | 20260506 18:15:10.280 | INFO | Perf report for the last 5.14122 seconds: | 20260506 18:15:10.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:15:10.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:10.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:10.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:10.280 | INFO | ANALYSIS : 62.6%, 2.9, 95.6ms, 0 | 20260506 18:15:10.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:10.280 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260506 18:15:10.280 | INFO | PRE_HQ : 60.1%, 1.8, 57.7ms, 0 | 20260506 18:15:10.280 | INFO | BAZWRITER : 68.2%, 4.8, 168.3ms, 0 | 20260506 18:15:10.280 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 18:15:10.280 | INFO | Uploaded 36GiB to GPU in 2.01103s (17.9013GiB/s) | 20260506 18:15:10.280 | INFO | Downloaded 38GiB to GPU in 2.21276s (17.1731GiB/s) | 20260506 18:15:15.421 | INFO | Analyzing chunk frames = [3428864,3429376) | 20260506 18:15:15.421 | INFO | MemInfo: { | 20260506 18:15:15.421 | INFO | "gpu" : " 17 GB", | 20260506 18:15:15.421 | INFO | "huge" : "", | 20260506 18:15:15.421 | INFO | "huge_shared" : " 27 GB", | 20260506 18:15:15.421 | INFO | "malloc" : " 14 GB", | 20260506 18:15:15.421 | INFO | "pinned" : " 29 GB" | 20260506 18:15:15.421 | INFO | } | 20260506 18:15:15.421 | INFO | Perf report for the last 5.14146 seconds: | 20260506 18:15:15.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:15:15.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:15.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:15.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:15.421 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:15:15.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:15.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:15.421 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 18:15:15.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:15.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:15.421 | INFO | Uploaded 36GiB to GPU in 1.98402s (18.145GiB/s) | 20260506 18:15:15.421 | INFO | Downloaded 34GiB to GPU in 1.92492s (17.6631GiB/s) | 20260506 18:15:20.566 | INFO | Analyzing chunk frames = [3429376,3429888) | 20260506 18:15:20.566 | INFO | MemInfo: { | 20260506 18:15:20.566 | INFO | "gpu" : " 17 GB", | 20260506 18:15:20.566 | INFO | "huge" : "", | 20260506 18:15:20.566 | INFO | "huge_shared" : " 27 GB", | 20260506 18:15:20.566 | INFO | "malloc" : " 14 GB", | 20260506 18:15:20.566 | INFO | "pinned" : " 29 GB" | 20260506 18:15:20.566 | INFO | } | 20260506 18:15:20.566 | INFO | Perf report for the last 5.1448 seconds: | 20260506 18:15:20.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:15:20.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:20.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:20.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:20.566 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 18:15:20.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:20.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:20.566 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:15:20.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:20.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:20.566 | INFO | Uploaded 36GiB to GPU in 2.00029s (17.9974GiB/s) | 20260506 18:15:20.566 | INFO | Downloaded 34GiB to GPU in 1.92479s (17.6643GiB/s) | 20260506 18:15:25.702 | INFO | Analyzing chunk frames = [3429888,3430400) | 20260506 18:15:25.702 | INFO | MemInfo: { | 20260506 18:15:25.702 | INFO | "gpu" : " 17 GB", | 20260506 18:15:25.703 | INFO | "huge" : "", | 20260506 18:15:25.703 | INFO | "huge_shared" : " 27 GB", | 20260506 18:15:25.703 | INFO | "malloc" : " 14 GB", | 20260506 18:15:25.703 | INFO | "pinned" : " 29 GB" | 20260506 18:15:25.703 | INFO | } | 20260506 18:15:25.703 | INFO | Perf report for the last 5.13662 seconds: | 20260506 18:15:25.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:15:25.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:25.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:25.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:25.703 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:15:25.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:25.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:25.703 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:15:25.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:25.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:25.703 | INFO | Uploaded 36GiB to GPU in 1.93279s (18.626GiB/s) | 20260506 18:15:25.703 | INFO | Downloaded 34GiB to GPU in 1.87177s (18.1647GiB/s) | 20260506 18:15:30.840 | INFO | Analyzing chunk frames = [3430400,3430912) | 20260506 18:15:30.840 | INFO | MemInfo: { | 20260506 18:15:30.840 | INFO | "gpu" : " 17 GB", | 20260506 18:15:30.840 | INFO | "huge" : "", | 20260506 18:15:30.840 | INFO | "huge_shared" : " 27 GB", | 20260506 18:15:30.840 | INFO | "malloc" : " 14 GB", | 20260506 18:15:30.840 | INFO | "pinned" : " 29 GB" | 20260506 18:15:30.840 | INFO | } | 20260506 18:15:30.840 | INFO | Perf report for the last 5.13748 seconds: | 20260506 18:15:30.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:15:30.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:30.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:30.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:30.840 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 18:15:30.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:30.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:30.840 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:15:30.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:30.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:30.840 | INFO | Uploaded 36GiB to GPU in 2.00261s (17.9766GiB/s) | 20260506 18:15:30.841 | INFO | Downloaded 34GiB to GPU in 1.9107s (17.7945GiB/s) | 20260506 18:15:35.981 | INFO | Analyzing chunk frames = [3430912,3431424) | 20260506 18:15:35.981 | INFO | MemInfo: { | 20260506 18:15:35.981 | INFO | "gpu" : " 17 GB", | 20260506 18:15:35.981 | INFO | "huge" : "", | 20260506 18:15:35.981 | INFO | "huge_shared" : " 27 GB", | 20260506 18:15:35.981 | INFO | "malloc" : " 14 GB", | 20260506 18:15:35.981 | INFO | "pinned" : " 29 GB" | 20260506 18:15:35.981 | INFO | } | 20260506 18:15:35.981 | INFO | Perf report for the last 5.14072 seconds: | 20260506 18:15:35.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:15:35.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:35.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:35.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:35.981 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 18:15:35.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:35.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:35.981 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:15:35.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:35.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:35.981 | INFO | Uploaded 36GiB to GPU in 2.02767s (17.7543GiB/s) | 20260506 18:15:35.981 | INFO | Downloaded 34GiB to GPU in 1.94799s (17.4539GiB/s) | 20260506 18:15:41.117 | INFO | Analyzing chunk frames = [3431424,3431936) | 20260506 18:15:41.117 | INFO | MemInfo: { | 20260506 18:15:41.117 | INFO | "gpu" : " 17 GB", | 20260506 18:15:41.117 | INFO | "huge" : "", | 20260506 18:15:41.117 | INFO | "huge_shared" : " 27 GB", | 20260506 18:15:41.117 | INFO | "malloc" : " 14 GB", | 20260506 18:15:41.117 | INFO | "pinned" : " 29 GB" | 20260506 18:15:41.117 | INFO | } | 20260506 18:15:41.117 | INFO | Perf report for the last 5.13581 seconds: | 20260506 18:15:41.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:15:41.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:41.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:41.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:41.117 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 18:15:41.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:41.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:41.117 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:15:41.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:41.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:41.117 | INFO | Uploaded 36GiB to GPU in 1.99084s (18.0828GiB/s) | 20260506 18:15:41.117 | INFO | Downloaded 34GiB to GPU in 2.00855s (16.9276GiB/s) | 20260506 18:15:46.255 | INFO | Analyzing chunk frames = [3431936,3432448) | 20260506 18:15:46.255 | INFO | MemInfo: { | 20260506 18:15:46.255 | INFO | "gpu" : " 17 GB", | 20260506 18:15:46.255 | INFO | "huge" : "", | 20260506 18:15:46.255 | INFO | "huge_shared" : " 27 GB", | 20260506 18:15:46.255 | INFO | "malloc" : " 14 GB", | 20260506 18:15:46.255 | INFO | "pinned" : " 29 GB" | 20260506 18:15:46.255 | INFO | } | 20260506 18:15:46.255 | INFO | Perf report for the last 5.13828 seconds: | 20260506 18:15:46.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:15:46.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:46.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:46.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:46.255 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 18:15:46.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:46.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:46.255 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:15:46.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:46.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:46.255 | INFO | Uploaded 36GiB to GPU in 2.01002s (17.9103GiB/s) | 20260506 18:15:46.255 | INFO | Downloaded 34GiB to GPU in 1.98397s (17.1373GiB/s) | 20260506 18:15:46.255 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:15:46.255 | INFO | Total mem usage: 18 GiB | 20260506 18:15:46.255 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:15:46.255 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.3%) | 20260506 18:15:46.255 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:15:46.255 | INFO | High Water Breakdown: | 20260506 18:15:46.255 | INFO | 61.8% : 9.91 GiB ( 86.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:15:46.255 | INFO | 38.1% : 6.1 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:15:46.255 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:15:46.255 | INFO | ------------------------------------------ | 20260506 18:15:46.256 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:15:46.256 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:15:46.256 | INFO | Total mem usage: 26.5 GiB | 20260506 18:15:46.256 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:15:46.256 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:15:46.256 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:15:46.256 | INFO | High Water Breakdown: | 20260506 18:15:46.256 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:15:46.256 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:15:46.256 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:15:46.256 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:15:46.256 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:15:46.256 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:15:46.256 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:15:46.256 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:15:46.256 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:15:46.256 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:15:46.256 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:15:46.256 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:15:46.256 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:15:46.256 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:15:46.256 | INFO | ------------------------------------------ | 20260506 18:15:46.256 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:15:46.256 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:15:46.256 | INFO | Total mem usage: 25.6 GiB | 20260506 18:15:46.256 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:15:46.256 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:15:46.256 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:15:46.256 | INFO | High Water Breakdown: | 20260506 18:15:46.256 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:15:46.256 | INFO | ------------------------------------------ | 20260506 18:15:46.256 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:15:46.256 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:15:46.256 | INFO | Total mem usage: 15.7 GiB | 20260506 18:15:46.256 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:15:46.256 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:15:46.256 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:15:46.256 | INFO | High Water Breakdown: | 20260506 18:15:46.256 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:15:46.256 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:15:46.256 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:15:46.256 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:15:46.256 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:15:46.256 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:15:46.256 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:15:46.256 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:15:46.256 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:15:46.256 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:15:46.256 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:15:46.256 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:15:46.256 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:15:46.256 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:15:46.256 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:15:46.256 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:15:46.256 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:15:46.256 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:15:46.256 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:15:46.256 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:15:46.256 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:15:46.256 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:15:46.256 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:15:46.256 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:15:46.256 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:15:46.256 | INFO | ------------------------------------------ | 20260506 18:15:46.256 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:15:46.256 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:15:46.256 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:15:46.256 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:15:46.256 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.63e+01, 1.35e+01 / 2.36e+00 | 20260506 18:15:46.256 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.62e-03 / 6.00e-03 | 20260506 18:15:46.256 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.46e+00, 5.58e-01 / 2.82e-01 | 20260506 18:15:46.256 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.49e+00, 1.85e-01 / 1.14e-01 | 20260506 18:15:46.256 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.49e+01, 4.19e+00 / 2.54e+00 | 20260506 18:15:46.256 | INFO | PerfReport: 56.98%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.37e+00 / 1.85e+00 | 20260506 18:15:46.256 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.27e+00 / 2.84e+00, 3.88e-01 / 2.03e-01 | 20260506 18:15:46.256 | INFO | PerfReport: 3.42%, Metrics -- 1.07e+00 / 1.40e+00, 2.82e-01 / 1.91e-01 | 20260506 18:15:46.256 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.68e-01 / 4.19e-01, 5.77e-02 / 1.92e-02 | 20260506 18:15:46.256 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:15:46.256 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:15:46.256 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:15:46.256 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 18:15:46.256 | INFO | PerfReport: 21.46%, Upload -- 2.09e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260506 18:15:46.256 | INFO | PerfReport: 21.20%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 18:15:46.256 | INFO | PerfReport: 32.18%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:15:46.256 | INFO | PerfReport: 1.41%, LockUpload -- 1.37e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260506 18:15:46.256 | INFO | PerfReport: 1.16%, LockDownload -- 1.13e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260506 18:15:46.256 | INFO | PerfReport: 22.57%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 18:15:46.256 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260506 18:15:46.368 | INFO | Starting a round of Baz IO | 20260506 18:15:49.508 | INFO | Populating RTMetrics report | 20260506 18:15:49.509 | INFO | RTMetrics report populated | 20260506 18:15:49.509 | INFO | Opening tmp RTMetrics json file | 20260506 18:15:49.647 | INFO | Serializing RTMetrics to json | 20260506 18:15:49.656 | INFO | Serialization done, swapping tmp file now... | 20260506 18:15:49.661 | INFO | RTMetrics done | 20260506 18:15:50.135 | INFO | Finished a round of Baz IO: Wrote 2.044 GiB in 3.766 seconds | 20260506 18:15:50.135 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.17% / 0.01% / 64.82% | 20260506 18:15:50.135 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.1668% / 7.94729e-06% / 7.54992e-05% | 20260506 18:15:50.135 | INFO | Baz IO overall: | 20260506 18:15:50.135 | INFO | BytesWritten : 1812690202552 | 20260506 18:15:50.135 | INFO | NumEvents : 876694830113 | 20260506 18:15:51.399 | INFO | Analyzing chunk frames = [3432448,3432960) | 20260506 18:15:51.399 | INFO | MemInfo: { | 20260506 18:15:51.399 | INFO | "gpu" : " 17 GB", | 20260506 18:15:51.399 | INFO | "huge" : "", | 20260506 18:15:51.399 | INFO | "huge_shared" : " 27 GB", | 20260506 18:15:51.399 | INFO | "malloc" : " 14 GB", | 20260506 18:15:51.399 | INFO | "pinned" : " 29 GB" | 20260506 18:15:51.399 | INFO | } | 20260506 18:15:51.399 | INFO | Perf report for the last 5.14425 seconds: | 20260506 18:15:51.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:15:51.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:51.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:51.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:51.399 | INFO | ANALYSIS : 62.7%, 2.9, 95.6ms, 0 | 20260506 18:15:51.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:51.399 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260506 18:15:51.399 | INFO | PRE_HQ : 60.2%, 1.9, 57.9ms, 0 | 20260506 18:15:51.399 | INFO | BAZWRITER : 72.9%, 4.8, 180.7ms, 0 | 20260506 18:15:51.399 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 18:15:51.400 | INFO | Uploaded 36GiB to GPU in 2.0639s (17.4427GiB/s) | 20260506 18:15:51.400 | INFO | Downloaded 38GiB to GPU in 2.20541s (17.2304GiB/s) | 20260506 18:15:56.541 | INFO | Analyzing chunk frames = [3432960,3433472) | 20260506 18:15:56.541 | INFO | MemInfo: { | 20260506 18:15:56.541 | INFO | "gpu" : " 17 GB", | 20260506 18:15:56.541 | INFO | "huge" : "", | 20260506 18:15:56.541 | INFO | "huge_shared" : " 27 GB", | 20260506 18:15:56.541 | INFO | "malloc" : " 14 GB", | 20260506 18:15:56.541 | INFO | "pinned" : " 29 GB" | 20260506 18:15:56.541 | INFO | } | 20260506 18:15:56.541 | INFO | Perf report for the last 5.14217 seconds: | 20260506 18:15:56.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:15:56.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:56.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:56.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:56.542 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:15:56.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:56.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:56.542 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:15:56.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:56.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:15:56.542 | INFO | Uploaded 36GiB to GPU in 2.00283s (17.9746GiB/s) | 20260506 18:15:56.542 | INFO | Downloaded 34GiB to GPU in 1.93331s (17.5864GiB/s) | 20260506 18:16:01.680 | INFO | Analyzing chunk frames = [3433472,3433984) | 20260506 18:16:01.680 | INFO | MemInfo: { | 20260506 18:16:01.680 | INFO | "gpu" : " 17 GB", | 20260506 18:16:01.680 | INFO | "huge" : "", | 20260506 18:16:01.680 | INFO | "huge_shared" : " 27 GB", | 20260506 18:16:01.680 | INFO | "malloc" : " 14 GB", | 20260506 18:16:01.680 | INFO | "pinned" : " 29 GB" | 20260506 18:16:01.680 | INFO | } | 20260506 18:16:01.680 | INFO | Perf report for the last 5.13848 seconds: | 20260506 18:16:01.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:16:01.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:01.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:01.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:01.680 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:16:01.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:01.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:01.680 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:16:01.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:01.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:01.680 | INFO | Uploaded 36GiB to GPU in 1.98086s (18.1739GiB/s) | 20260506 18:16:01.680 | INFO | Downloaded 34GiB to GPU in 1.91454s (17.7588GiB/s) | 20260506 18:16:06.819 | INFO | Analyzing chunk frames = [3433984,3434496) | 20260506 18:16:06.819 | INFO | MemInfo: { | 20260506 18:16:06.819 | INFO | "gpu" : " 17 GB", | 20260506 18:16:06.819 | INFO | "huge" : "", | 20260506 18:16:06.819 | INFO | "huge_shared" : " 27 GB", | 20260506 18:16:06.819 | INFO | "malloc" : " 14 GB", | 20260506 18:16:06.819 | INFO | "pinned" : " 29 GB" | 20260506 18:16:06.819 | INFO | } | 20260506 18:16:06.819 | INFO | Perf report for the last 5.1388 seconds: | 20260506 18:16:06.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:16:06.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:06.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:06.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:06.819 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 18:16:06.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:06.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:06.819 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:16:06.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:06.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:06.819 | INFO | Uploaded 36GiB to GPU in 1.9698s (18.2759GiB/s) | 20260506 18:16:06.819 | INFO | Downloaded 34GiB to GPU in 1.8726s (18.1566GiB/s) | 20260506 18:16:11.955 | INFO | Analyzing chunk frames = [3434496,3435008) | 20260506 18:16:11.955 | INFO | MemInfo: { | 20260506 18:16:11.955 | INFO | "gpu" : " 17 GB", | 20260506 18:16:11.955 | INFO | "huge" : "", | 20260506 18:16:11.955 | INFO | "huge_shared" : " 27 GB", | 20260506 18:16:11.955 | INFO | "malloc" : " 14 GB", | 20260506 18:16:11.955 | INFO | "pinned" : " 29 GB" | 20260506 18:16:11.955 | INFO | } | 20260506 18:16:11.955 | INFO | Perf report for the last 5.13596 seconds: | 20260506 18:16:11.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:16:11.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:11.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:11.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:11.955 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260506 18:16:11.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:11.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:11.955 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:16:11.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:11.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:11.955 | INFO | Uploaded 36GiB to GPU in 2.01894s (17.8312GiB/s) | 20260506 18:16:11.955 | INFO | Downloaded 34GiB to GPU in 1.90853s (17.8148GiB/s) | 20260506 18:16:17.089 | INFO | Analyzing chunk frames = [3435008,3435520) | 20260506 18:16:17.089 | INFO | MemInfo: { | 20260506 18:16:17.089 | INFO | "gpu" : " 17 GB", | 20260506 18:16:17.089 | INFO | "huge" : "", | 20260506 18:16:17.089 | INFO | "huge_shared" : " 27 GB", | 20260506 18:16:17.089 | INFO | "malloc" : " 14 GB", | 20260506 18:16:17.089 | INFO | "pinned" : " 29 GB" | 20260506 18:16:17.089 | INFO | } | 20260506 18:16:17.089 | INFO | Perf report for the last 5.13434 seconds: | 20260506 18:16:17.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:16:17.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:17.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:17.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:17.089 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:16:17.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:17.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:17.089 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:16:17.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:17.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:17.089 | INFO | Uploaded 36GiB to GPU in 2.04158s (17.6334GiB/s) | 20260506 18:16:17.089 | INFO | Downloaded 34GiB to GPU in 1.94572s (17.4743GiB/s) | 20260506 18:16:22.230 | INFO | Analyzing chunk frames = [3435520,3436032) | 20260506 18:16:22.230 | INFO | MemInfo: { | 20260506 18:16:22.230 | INFO | "gpu" : " 17 GB", | 20260506 18:16:22.230 | INFO | "huge" : "", | 20260506 18:16:22.230 | INFO | "huge_shared" : " 27 GB", | 20260506 18:16:22.230 | INFO | "malloc" : " 14 GB", | 20260506 18:16:22.230 | INFO | "pinned" : " 29 GB" | 20260506 18:16:22.230 | INFO | } | 20260506 18:16:22.230 | INFO | Perf report for the last 5.14121 seconds: | 20260506 18:16:22.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:16:22.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:22.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:22.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:22.230 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 18:16:22.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:22.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:22.230 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:16:22.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:22.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:22.231 | INFO | Uploaded 36GiB to GPU in 2.02523s (17.7758GiB/s) | 20260506 18:16:22.231 | INFO | Downloaded 34GiB to GPU in 2.01959s (16.8351GiB/s) | 20260506 18:16:27.374 | INFO | Analyzing chunk frames = [3436032,3436544) | 20260506 18:16:27.374 | INFO | MemInfo: { | 20260506 18:16:27.374 | INFO | "gpu" : " 17 GB", | 20260506 18:16:27.374 | INFO | "huge" : "", | 20260506 18:16:27.374 | INFO | "huge_shared" : " 27 GB", | 20260506 18:16:27.374 | INFO | "malloc" : " 14 GB", | 20260506 18:16:27.374 | INFO | "pinned" : " 29 GB" | 20260506 18:16:27.374 | INFO | } | 20260506 18:16:27.374 | INFO | Perf report for the last 5.14407 seconds: | 20260506 18:16:27.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:16:27.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:27.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:27.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:27.374 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 18:16:27.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:27.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:27.374 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260506 18:16:27.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:27.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:27.375 | INFO | Uploaded 36GiB to GPU in 2.00741s (17.9336GiB/s) | 20260506 18:16:27.375 | INFO | Downloaded 34GiB to GPU in 1.97738s (17.1945GiB/s) | 20260506 18:16:27.490 | INFO | Starting a round of Baz IO | 20260506 18:16:30.626 | INFO | Populating RTMetrics report | 20260506 18:16:30.626 | INFO | RTMetrics report populated | 20260506 18:16:30.626 | INFO | Opening tmp RTMetrics json file | 20260506 18:16:30.629 | INFO | Serializing RTMetrics to json | 20260506 18:16:30.633 | INFO | Serialization done, swapping tmp file now... | 20260506 18:16:30.634 | INFO | RTMetrics done | 20260506 18:16:31.101 | INFO | Finished a round of Baz IO: Wrote 2.045 GiB in 3.611 seconds | 20260506 18:16:31.102 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.17% / 0.01% / 64.83% | 20260506 18:16:31.102 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.1651% / 7.94729e-06% / 6.75519e-05% | 20260506 18:16:31.102 | INFO | Baz IO overall: | 20260506 18:16:31.102 | INFO | BytesWritten : 1814886444128 | 20260506 18:16:31.102 | INFO | NumEvents : 877753855165 | 20260506 18:16:32.513 | INFO | Analyzing chunk frames = [3436544,3437056) | 20260506 18:16:32.513 | INFO | MemInfo: { | 20260506 18:16:32.513 | INFO | "gpu" : " 17 GB", | 20260506 18:16:32.513 | INFO | "huge" : "", | 20260506 18:16:32.513 | INFO | "huge_shared" : " 27 GB", | 20260506 18:16:32.513 | INFO | "malloc" : " 14 GB", | 20260506 18:16:32.513 | INFO | "pinned" : " 29 GB" | 20260506 18:16:32.513 | INFO | } | 20260506 18:16:32.513 | INFO | Perf report for the last 5.13887 seconds: | 20260506 18:16:32.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:16:32.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:32.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:32.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:32.513 | INFO | ANALYSIS : 62.7%, 2.9, 95.3ms, 0 | 20260506 18:16:32.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:32.513 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 18:16:32.513 | INFO | PRE_HQ : 60.1%, 1.9, 58.1ms, 0 | 20260506 18:16:32.513 | INFO | BAZWRITER : 69.9%, 4.8, 173.1ms, 0 | 20260506 18:16:32.513 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 18:16:32.514 | INFO | Uploaded 36GiB to GPU in 2.03346s (17.7038GiB/s) | 20260506 18:16:32.514 | INFO | Downloaded 38GiB to GPU in 2.21298s (17.1715GiB/s) | 20260506 18:16:37.658 | INFO | Analyzing chunk frames = [3437056,3437568) | 20260506 18:16:37.658 | INFO | MemInfo: { | 20260506 18:16:37.658 | INFO | "gpu" : " 17 GB", | 20260506 18:16:37.658 | INFO | "huge" : "", | 20260506 18:16:37.658 | INFO | "huge_shared" : " 27 GB", | 20260506 18:16:37.658 | INFO | "malloc" : " 14 GB", | 20260506 18:16:37.658 | INFO | "pinned" : " 29 GB" | 20260506 18:16:37.658 | INFO | } | 20260506 18:16:37.658 | INFO | Perf report for the last 5.14456 seconds: | 20260506 18:16:37.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:16:37.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:37.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:37.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:37.658 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 18:16:37.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:37.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:37.658 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:16:37.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:37.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:37.658 | INFO | Uploaded 36GiB to GPU in 1.9886s (18.1032GiB/s) | 20260506 18:16:37.658 | INFO | Downloaded 34GiB to GPU in 1.92248s (17.6855GiB/s) | 20260506 18:16:42.795 | INFO | Analyzing chunk frames = [3437568,3438080) | 20260506 18:16:42.795 | INFO | MemInfo: { | 20260506 18:16:42.795 | INFO | "gpu" : " 17 GB", | 20260506 18:16:42.795 | INFO | "huge" : "", | 20260506 18:16:42.795 | INFO | "huge_shared" : " 27 GB", | 20260506 18:16:42.795 | INFO | "malloc" : " 14 GB", | 20260506 18:16:42.795 | INFO | "pinned" : " 29 GB" | 20260506 18:16:42.795 | INFO | } | 20260506 18:16:42.795 | INFO | Perf report for the last 5.13725 seconds: | 20260506 18:16:42.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:16:42.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:42.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:42.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:42.795 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 18:16:42.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:42.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:42.795 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:16:42.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:42.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:42.795 | INFO | Uploaded 36GiB to GPU in 1.93532s (18.6016GiB/s) | 20260506 18:16:42.795 | INFO | Downloaded 34GiB to GPU in 1.91946s (17.7133GiB/s) | 20260506 18:16:47.936 | INFO | Analyzing chunk frames = [3438080,3438592) | 20260506 18:16:47.936 | INFO | MemInfo: { | 20260506 18:16:47.936 | INFO | "gpu" : " 17 GB", | 20260506 18:16:47.936 | INFO | "huge" : "", | 20260506 18:16:47.936 | INFO | "huge_shared" : " 27 GB", | 20260506 18:16:47.936 | INFO | "malloc" : " 14 GB", | 20260506 18:16:47.936 | INFO | "pinned" : " 29 GB" | 20260506 18:16:47.936 | INFO | } | 20260506 18:16:47.936 | INFO | Perf report for the last 5.14114 seconds: | 20260506 18:16:47.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:16:47.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:47.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:47.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:47.936 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 18:16:47.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:47.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:47.936 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:16:47.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:47.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:47.937 | INFO | Uploaded 36GiB to GPU in 1.92308s (18.7199GiB/s) | 20260506 18:16:47.937 | INFO | Downloaded 34GiB to GPU in 1.86732s (18.2079GiB/s) | 20260506 18:16:53.069 | INFO | Analyzing chunk frames = [3438592,3439104) | 20260506 18:16:53.069 | INFO | MemInfo: { | 20260506 18:16:53.069 | INFO | "gpu" : " 17 GB", | 20260506 18:16:53.069 | INFO | "huge" : "", | 20260506 18:16:53.069 | INFO | "huge_shared" : " 27 GB", | 20260506 18:16:53.069 | INFO | "malloc" : " 14 GB", | 20260506 18:16:53.069 | INFO | "pinned" : " 29 GB" | 20260506 18:16:53.069 | INFO | } | 20260506 18:16:53.069 | INFO | Perf report for the last 5.13246 seconds: | 20260506 18:16:53.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:16:53.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:53.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:53.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:53.069 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260506 18:16:53.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:53.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:53.069 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260506 18:16:53.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:53.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:53.069 | INFO | Uploaded 36GiB to GPU in 2.00979s (17.9123GiB/s) | 20260506 18:16:53.069 | INFO | Downloaded 34GiB to GPU in 1.91976s (17.7105GiB/s) | 20260506 18:16:58.204 | INFO | Analyzing chunk frames = [3439104,3439616) | 20260506 18:16:58.204 | INFO | MemInfo: { | 20260506 18:16:58.204 | INFO | "gpu" : " 17 GB", | 20260506 18:16:58.204 | INFO | "huge" : "", | 20260506 18:16:58.204 | INFO | "huge_shared" : " 27 GB", | 20260506 18:16:58.204 | INFO | "malloc" : " 14 GB", | 20260506 18:16:58.204 | INFO | "pinned" : " 29 GB" | 20260506 18:16:58.204 | INFO | } | 20260506 18:16:58.204 | INFO | Perf report for the last 5.13509 seconds: | 20260506 18:16:58.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:16:58.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:58.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:58.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:58.204 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 18:16:58.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:58.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:58.204 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:16:58.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:58.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:16:58.204 | INFO | Uploaded 36GiB to GPU in 2.01671s (17.8509GiB/s) | 20260506 18:16:58.204 | INFO | Downloaded 34GiB to GPU in 1.94242s (17.5039GiB/s) | 20260506 18:17:03.348 | INFO | Analyzing chunk frames = [3439616,3440128) | 20260506 18:17:03.348 | INFO | MemInfo: { | 20260506 18:17:03.348 | INFO | "gpu" : " 17 GB", | 20260506 18:17:03.348 | INFO | "huge" : "", | 20260506 18:17:03.348 | INFO | "huge_shared" : " 27 GB", | 20260506 18:17:03.348 | INFO | "malloc" : " 14 GB", | 20260506 18:17:03.348 | INFO | "pinned" : " 29 GB" | 20260506 18:17:03.348 | INFO | } | 20260506 18:17:03.348 | INFO | Perf report for the last 5.14459 seconds: | 20260506 18:17:03.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:17:03.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:03.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:03.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:03.349 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:17:03.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:03.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:03.349 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:17:03.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:03.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:03.349 | INFO | Uploaded 36GiB to GPU in 1.97402s (18.2369GiB/s) | 20260506 18:17:03.349 | INFO | Downloaded 34GiB to GPU in 2.00492s (16.9582GiB/s) | 20260506 18:17:08.494 | INFO | Analyzing chunk frames = [3440128,3440640) | 20260506 18:17:08.494 | INFO | MemInfo: { | 20260506 18:17:08.494 | INFO | "gpu" : " 17 GB", | 20260506 18:17:08.494 | INFO | "huge" : "", | 20260506 18:17:08.494 | INFO | "huge_shared" : " 27 GB", | 20260506 18:17:08.494 | INFO | "malloc" : " 14 GB", | 20260506 18:17:08.494 | INFO | "pinned" : " 29 GB" | 20260506 18:17:08.494 | INFO | } | 20260506 18:17:08.494 | INFO | Perf report for the last 5.14581 seconds: | 20260506 18:17:08.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:17:08.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:08.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:08.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:08.494 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 18:17:08.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:08.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:08.494 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:17:08.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:08.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:08.494 | INFO | Uploaded 36GiB to GPU in 1.99901s (18.0089GiB/s) | 20260506 18:17:08.495 | INFO | Downloaded 34GiB to GPU in 1.97747s (17.1937GiB/s) | 20260506 18:17:08.495 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:17:08.495 | INFO | Total mem usage: 18 GiB | 20260506 18:17:08.495 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:17:08.495 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -14%) | 20260506 18:17:08.495 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:17:08.495 | INFO | High Water Breakdown: | 20260506 18:17:08.495 | INFO | 61.6% : 9.72 GiB ( 85.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:17:08.495 | INFO | 38.3% : 6.05 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:17:08.495 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:17:08.495 | INFO | ------------------------------------------ | 20260506 18:17:08.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:17:08.495 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:17:08.495 | INFO | Total mem usage: 26.5 GiB | 20260506 18:17:08.495 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:17:08.495 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:17:08.495 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:17:08.495 | INFO | High Water Breakdown: | 20260506 18:17:08.495 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:17:08.495 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:17:08.495 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:17:08.495 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:17:08.495 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:17:08.495 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:17:08.495 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:17:08.495 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:17:08.495 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:17:08.495 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:17:08.495 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:17:08.495 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:17:08.495 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:17:08.495 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:17:08.495 | INFO | ------------------------------------------ | 20260506 18:17:08.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:17:08.495 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:17:08.495 | INFO | Total mem usage: 25.6 GiB | 20260506 18:17:08.495 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:17:08.495 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:17:08.495 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:17:08.495 | INFO | High Water Breakdown: | 20260506 18:17:08.495 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:17:08.495 | INFO | ------------------------------------------ | 20260506 18:17:08.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:17:08.495 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:17:08.495 | INFO | Total mem usage: 15.7 GiB | 20260506 18:17:08.495 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:17:08.495 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:17:08.495 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:17:08.495 | INFO | High Water Breakdown: | 20260506 18:17:08.495 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:17:08.495 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:17:08.495 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:17:08.495 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:17:08.495 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:17:08.495 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:17:08.495 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:17:08.495 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:17:08.495 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:17:08.495 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:17:08.495 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:17:08.495 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:17:08.495 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:17:08.495 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:17:08.495 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:17:08.495 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:17:08.495 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:17:08.495 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:17:08.495 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:17:08.495 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:17:08.495 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:17:08.495 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:17:08.495 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:17:08.495 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:17:08.495 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:17:08.495 | INFO | ------------------------------------------ | 20260506 18:17:08.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:17:08.495 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:17:08.495 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:17:08.495 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:17:08.495 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.60e+01, 1.35e+01 / 1.91e+00 | 20260506 18:17:08.495 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 1.01e-02 / 6.05e-03 | 20260506 18:17:08.495 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.43e+00, 5.64e-01 / 3.13e-01 | 20260506 18:17:08.496 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.84e-01 / 1.11e-01 | 20260506 18:17:08.496 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.47e+01, 4.14e+00 / 2.35e+00 | 20260506 18:17:08.496 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.40e+00 / 1.63e+00 | 20260506 18:17:08.496 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.26e+00 / 2.82e+00, 3.85e-01 / 1.92e-01 | 20260506 18:17:08.496 | INFO | PerfReport: 3.42%, Metrics -- 1.07e+00 / 1.43e+00, 2.81e-01 / 1.43e-01 | 20260506 18:17:08.496 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.69e-01 / 4.19e-01, 5.77e-02 / 1.87e-02 | 20260506 18:17:08.496 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:17:08.496 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:17:08.496 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:17:08.496 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 18:17:08.496 | INFO | PerfReport: 21.45%, Upload -- 2.08e+01 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260506 18:17:08.496 | INFO | PerfReport: 21.15%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 18:17:08.496 | INFO | PerfReport: 32.22%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:17:08.496 | INFO | PerfReport: 1.41%, LockUpload -- 1.37e+00 / 0.00e+00, 5.68e+00 / 0.00e+00 | 20260506 18:17:08.496 | INFO | PerfReport: 1.17%, LockDownload -- 1.14e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260506 18:17:08.496 | INFO | PerfReport: 22.59%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:17:08.496 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260506 18:17:08.604 | INFO | Starting a round of Baz IO | 20260506 18:17:11.739 | INFO | Populating RTMetrics report | 20260506 18:17:11.739 | INFO | RTMetrics report populated | 20260506 18:17:11.739 | INFO | Opening tmp RTMetrics json file | 20260506 18:17:11.792 | INFO | Serializing RTMetrics to json | 20260506 18:17:11.797 | INFO | Serialization done, swapping tmp file now... | 20260506 18:17:11.806 | INFO | RTMetrics done | 20260506 18:17:12.275 | INFO | Finished a round of Baz IO: Wrote 2.043 GiB in 3.671 seconds | 20260506 18:17:12.275 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.16% / 0.01% / 64.83% | 20260506 18:17:12.276 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.15944% / 1.58946e-05% / 6.75519e-05% | 20260506 18:17:12.276 | INFO | Baz IO overall: | 20260506 18:17:12.276 | INFO | BytesWritten : 1817079736220 | 20260506 18:17:12.276 | INFO | NumEvents : 878810825103 | 20260506 18:17:13.630 | INFO | Analyzing chunk frames = [3440640,3441152) | 20260506 18:17:13.630 | INFO | MemInfo: { | 20260506 18:17:13.630 | INFO | "gpu" : " 17 GB", | 20260506 18:17:13.630 | INFO | "huge" : "", | 20260506 18:17:13.630 | INFO | "huge_shared" : " 27 GB", | 20260506 18:17:13.630 | INFO | "malloc" : " 14 GB", | 20260506 18:17:13.630 | INFO | "pinned" : " 29 GB" | 20260506 18:17:13.630 | INFO | } | 20260506 18:17:13.630 | INFO | Perf report for the last 5.13585 seconds: | 20260506 18:17:13.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:17:13.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:13.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:13.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:13.630 | INFO | ANALYSIS : 62.6%, 2.9, 95.3ms, 0 | 20260506 18:17:13.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:13.630 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 18:17:13.630 | INFO | PRE_HQ : 60.2%, 1.9, 57.9ms, 0 | 20260506 18:17:13.630 | INFO | BAZWRITER : 71.2%, 4.8, 175.6ms, 0 | 20260506 18:17:13.630 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 18:17:13.630 | INFO | Uploaded 36GiB to GPU in 2.06596s (17.4254GiB/s) | 20260506 18:17:13.630 | INFO | Downloaded 38GiB to GPU in 2.19614s (17.3031GiB/s) | 20260506 18:17:18.768 | INFO | Analyzing chunk frames = [3441152,3441664) | 20260506 18:17:18.768 | INFO | MemInfo: { | 20260506 18:17:18.768 | INFO | "gpu" : " 17 GB", | 20260506 18:17:18.768 | INFO | "huge" : "", | 20260506 18:17:18.768 | INFO | "huge_shared" : " 27 GB", | 20260506 18:17:18.768 | INFO | "malloc" : " 14 GB", | 20260506 18:17:18.768 | INFO | "pinned" : " 29 GB" | 20260506 18:17:18.768 | INFO | } | 20260506 18:17:18.768 | INFO | Perf report for the last 5.13825 seconds: | 20260506 18:17:18.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:17:18.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:18.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:18.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:18.768 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 18:17:18.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:18.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:18.768 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:17:18.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:18.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:18.769 | INFO | Uploaded 36GiB to GPU in 2.01975s (17.824GiB/s) | 20260506 18:17:18.769 | INFO | Downloaded 34GiB to GPU in 1.9414s (17.5131GiB/s) | 20260506 18:17:23.905 | INFO | Analyzing chunk frames = [3441664,3442176) | 20260506 18:17:23.905 | INFO | MemInfo: { | 20260506 18:17:23.905 | INFO | "gpu" : " 17 GB", | 20260506 18:17:23.905 | INFO | "huge" : "", | 20260506 18:17:23.905 | INFO | "huge_shared" : " 27 GB", | 20260506 18:17:23.905 | INFO | "malloc" : " 14 GB", | 20260506 18:17:23.905 | INFO | "pinned" : " 29 GB" | 20260506 18:17:23.905 | INFO | } | 20260506 18:17:23.905 | INFO | Perf report for the last 5.13672 seconds: | 20260506 18:17:23.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:17:23.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:23.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:23.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:23.905 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 18:17:23.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:23.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:23.905 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:17:23.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:23.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:23.905 | INFO | Uploaded 36GiB to GPU in 1.97602s (18.2184GiB/s) | 20260506 18:17:23.905 | INFO | Downloaded 34GiB to GPU in 1.92035s (17.7051GiB/s) | 20260506 18:17:29.043 | INFO | Analyzing chunk frames = [3442176,3442688) | 20260506 18:17:29.043 | INFO | MemInfo: { | 20260506 18:17:29.043 | INFO | "gpu" : " 17 GB", | 20260506 18:17:29.043 | INFO | "huge" : "", | 20260506 18:17:29.043 | INFO | "huge_shared" : " 27 GB", | 20260506 18:17:29.043 | INFO | "malloc" : " 14 GB", | 20260506 18:17:29.043 | INFO | "pinned" : " 29 GB" | 20260506 18:17:29.043 | INFO | } | 20260506 18:17:29.043 | INFO | Perf report for the last 5.13799 seconds: | 20260506 18:17:29.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:17:29.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:29.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:29.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:29.043 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 18:17:29.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:29.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:29.043 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:17:29.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:29.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:29.043 | INFO | Uploaded 36GiB to GPU in 1.93136s (18.6398GiB/s) | 20260506 18:17:29.043 | INFO | Downloaded 34GiB to GPU in 1.86734s (18.2077GiB/s) | 20260506 18:17:34.180 | INFO | Analyzing chunk frames = [3442688,3443200) | 20260506 18:17:34.180 | INFO | MemInfo: { | 20260506 18:17:34.180 | INFO | "gpu" : " 17 GB", | 20260506 18:17:34.180 | INFO | "huge" : "", | 20260506 18:17:34.180 | INFO | "huge_shared" : " 27 GB", | 20260506 18:17:34.180 | INFO | "malloc" : " 14 GB", | 20260506 18:17:34.180 | INFO | "pinned" : " 29 GB" | 20260506 18:17:34.180 | INFO | } | 20260506 18:17:34.180 | INFO | Perf report for the last 5.13724 seconds: | 20260506 18:17:34.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:17:34.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:34.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:34.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:34.180 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 18:17:34.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:34.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:34.180 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:17:34.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:34.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:34.181 | INFO | Uploaded 36GiB to GPU in 2.0128s (17.8855GiB/s) | 20260506 18:17:34.181 | INFO | Downloaded 34GiB to GPU in 1.918s (17.7268GiB/s) | 20260506 18:17:39.322 | INFO | Analyzing chunk frames = [3443200,3443712) | 20260506 18:17:39.322 | INFO | MemInfo: { | 20260506 18:17:39.322 | INFO | "gpu" : " 17 GB", | 20260506 18:17:39.322 | INFO | "huge" : "", | 20260506 18:17:39.322 | INFO | "huge_shared" : " 27 GB", | 20260506 18:17:39.322 | INFO | "malloc" : " 14 GB", | 20260506 18:17:39.322 | INFO | "pinned" : " 29 GB" | 20260506 18:17:39.322 | INFO | } | 20260506 18:17:39.322 | INFO | Perf report for the last 5.14134 seconds: | 20260506 18:17:39.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:17:39.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:39.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:39.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:39.322 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 18:17:39.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:39.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:39.322 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:17:39.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:39.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:39.322 | INFO | Uploaded 36GiB to GPU in 2.02933s (17.7399GiB/s) | 20260506 18:17:39.322 | INFO | Downloaded 34GiB to GPU in 1.93742s (17.5491GiB/s) | 20260506 18:17:44.465 | INFO | Analyzing chunk frames = [3443712,3444224) | 20260506 18:17:44.465 | INFO | MemInfo: { | 20260506 18:17:44.465 | INFO | "gpu" : " 17 GB", | 20260506 18:17:44.465 | INFO | "huge" : "", | 20260506 18:17:44.465 | INFO | "huge_shared" : " 27 GB", | 20260506 18:17:44.465 | INFO | "malloc" : " 14 GB", | 20260506 18:17:44.466 | INFO | "pinned" : " 29 GB" | 20260506 18:17:44.466 | INFO | } | 20260506 18:17:44.466 | INFO | Perf report for the last 5.14389 seconds: | 20260506 18:17:44.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:17:44.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:44.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:44.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:44.466 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:17:44.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:44.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:44.466 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 18:17:44.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:44.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:44.466 | INFO | Uploaded 36GiB to GPU in 2.00968s (17.9133GiB/s) | 20260506 18:17:44.466 | INFO | Downloaded 34GiB to GPU in 2.01346s (16.8864GiB/s) | 20260506 18:17:49.609 | INFO | Analyzing chunk frames = [3444224,3444736) | 20260506 18:17:49.609 | INFO | MemInfo: { | 20260506 18:17:49.609 | INFO | "gpu" : " 17 GB", | 20260506 18:17:49.609 | INFO | "huge" : "", | 20260506 18:17:49.609 | INFO | "huge_shared" : " 27 GB", | 20260506 18:17:49.609 | INFO | "malloc" : " 14 GB", | 20260506 18:17:49.609 | INFO | "pinned" : " 29 GB" | 20260506 18:17:49.609 | INFO | } | 20260506 18:17:49.609 | INFO | Perf report for the last 5.14328 seconds: | 20260506 18:17:49.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:17:49.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:49.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:49.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:49.609 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:17:49.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:49.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:49.609 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:17:49.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:49.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:49.610 | INFO | Uploaded 36GiB to GPU in 1.99057s (18.0853GiB/s) | 20260506 18:17:49.610 | INFO | Downloaded 34GiB to GPU in 1.99104s (17.0765GiB/s) | 20260506 18:17:49.723 | INFO | Starting a round of Baz IO | 20260506 18:17:52.858 | INFO | Populating RTMetrics report | 20260506 18:17:52.858 | INFO | RTMetrics report populated | 20260506 18:17:52.858 | INFO | Opening tmp RTMetrics json file | 20260506 18:17:52.875 | INFO | Serializing RTMetrics to json | 20260506 18:17:52.901 | INFO | Serialization done, swapping tmp file now... | 20260506 18:17:52.904 | INFO | RTMetrics done | 20260506 18:17:53.199 | INFO | Finished a round of Baz IO: Wrote 2.042 GiB in 3.476 seconds | 20260506 18:17:53.199 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.15% / 0.01% / 64.84% | 20260506 18:17:53.200 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.15589% / 7.94729e-06% / 8.34465e-05% | 20260506 18:17:53.200 | INFO | Baz IO overall: | 20260506 18:17:53.200 | INFO | BytesWritten : 1819271924470 | 20260506 18:17:53.200 | INFO | NumEvents : 879867048842 | 20260506 18:17:54.752 | INFO | Analyzing chunk frames = [3444736,3445248) | 20260506 18:17:54.752 | INFO | MemInfo: { | 20260506 18:17:54.752 | INFO | "gpu" : " 17 GB", | 20260506 18:17:54.752 | INFO | "huge" : "", | 20260506 18:17:54.752 | INFO | "huge_shared" : " 27 GB", | 20260506 18:17:54.752 | INFO | "malloc" : " 14 GB", | 20260506 18:17:54.752 | INFO | "pinned" : " 29 GB" | 20260506 18:17:54.752 | INFO | } | 20260506 18:17:54.752 | INFO | Perf report for the last 5.14334 seconds: | 20260506 18:17:54.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:17:54.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:54.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:54.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:54.752 | INFO | ANALYSIS : 62.6%, 2.9, 95.3ms, 0 | 20260506 18:17:54.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:54.752 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 18:17:54.752 | INFO | PRE_HQ : 59.9%, 1.9, 57.9ms, 0 | 20260506 18:17:54.752 | INFO | BAZWRITER : 67.3%, 4.8, 166.3ms, 0 | 20260506 18:17:54.752 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 18:17:54.752 | INFO | Uploaded 36GiB to GPU in 2.03515s (17.6891GiB/s) | 20260506 18:17:54.753 | INFO | Downloaded 38GiB to GPU in 2.21513s (17.1547GiB/s) | 20260506 18:17:59.883 | INFO | Analyzing chunk frames = [3445248,3445760) | 20260506 18:17:59.883 | INFO | MemInfo: { | 20260506 18:17:59.883 | INFO | "gpu" : " 17 GB", | 20260506 18:17:59.883 | INFO | "huge" : "", | 20260506 18:17:59.883 | INFO | "huge_shared" : " 27 GB", | 20260506 18:17:59.883 | INFO | "malloc" : " 14 GB", | 20260506 18:17:59.883 | INFO | "pinned" : " 29 GB" | 20260506 18:17:59.883 | INFO | } | 20260506 18:17:59.883 | INFO | Perf report for the last 5.1306 seconds: | 20260506 18:17:59.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:17:59.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:59.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:59.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:59.883 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 18:17:59.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:59.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:59.883 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:17:59.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:59.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:17:59.883 | INFO | Uploaded 36GiB to GPU in 1.99019s (18.0887GiB/s) | 20260506 18:17:59.883 | INFO | Downloaded 34GiB to GPU in 1.92032s (17.7054GiB/s) | 20260506 18:18:05.019 | INFO | Analyzing chunk frames = [3445760,3446272) | 20260506 18:18:05.019 | INFO | MemInfo: { | 20260506 18:18:05.019 | INFO | "gpu" : " 17 GB", | 20260506 18:18:05.019 | INFO | "huge" : "", | 20260506 18:18:05.019 | INFO | "huge_shared" : " 27 GB", | 20260506 18:18:05.019 | INFO | "malloc" : " 14 GB", | 20260506 18:18:05.019 | INFO | "pinned" : " 29 GB" | 20260506 18:18:05.019 | INFO | } | 20260506 18:18:05.019 | INFO | Perf report for the last 5.13612 seconds: | 20260506 18:18:05.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:18:05.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:05.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:05.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:05.019 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 18:18:05.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:05.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:05.019 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:18:05.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:05.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:05.019 | INFO | Uploaded 36GiB to GPU in 1.97998s (18.182GiB/s) | 20260506 18:18:05.019 | INFO | Downloaded 34GiB to GPU in 1.92034s (17.7052GiB/s) | 20260506 18:18:10.157 | INFO | Analyzing chunk frames = [3446272,3446784) | 20260506 18:18:10.157 | INFO | MemInfo: { | 20260506 18:18:10.157 | INFO | "gpu" : " 17 GB", | 20260506 18:18:10.157 | INFO | "huge" : "", | 20260506 18:18:10.157 | INFO | "huge_shared" : " 27 GB", | 20260506 18:18:10.157 | INFO | "malloc" : " 14 GB", | 20260506 18:18:10.157 | INFO | "pinned" : " 29 GB" | 20260506 18:18:10.157 | INFO | } | 20260506 18:18:10.157 | INFO | Perf report for the last 5.13831 seconds: | 20260506 18:18:10.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:18:10.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:10.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:10.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:10.157 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:18:10.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:10.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:10.157 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:18:10.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:10.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:10.158 | INFO | Uploaded 36GiB to GPU in 1.91492s (18.7998GiB/s) | 20260506 18:18:10.158 | INFO | Downloaded 34GiB to GPU in 1.87436s (18.1396GiB/s) | 20260506 18:18:15.296 | INFO | Analyzing chunk frames = [3446784,3447296) | 20260506 18:18:15.296 | INFO | MemInfo: { | 20260506 18:18:15.296 | INFO | "gpu" : " 17 GB", | 20260506 18:18:15.296 | INFO | "huge" : "", | 20260506 18:18:15.296 | INFO | "huge_shared" : " 27 GB", | 20260506 18:18:15.296 | INFO | "malloc" : " 14 GB", | 20260506 18:18:15.296 | INFO | "pinned" : " 29 GB" | 20260506 18:18:15.296 | INFO | } | 20260506 18:18:15.296 | INFO | Perf report for the last 5.13886 seconds: | 20260506 18:18:15.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:18:15.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:15.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:15.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:15.296 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 18:18:15.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:15.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:15.296 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:18:15.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:15.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:15.296 | INFO | Uploaded 36GiB to GPU in 2.0169s (17.8492GiB/s) | 20260506 18:18:15.297 | INFO | Downloaded 34GiB to GPU in 1.91438s (17.7603GiB/s) | 20260506 18:18:20.443 | INFO | Analyzing chunk frames = [3447296,3447808) | 20260506 18:18:20.443 | INFO | MemInfo: { | 20260506 18:18:20.444 | INFO | "gpu" : " 17 GB", | 20260506 18:18:20.444 | INFO | "huge" : "", | 20260506 18:18:20.444 | INFO | "huge_shared" : " 27 GB", | 20260506 18:18:20.444 | INFO | "malloc" : " 14 GB", | 20260506 18:18:20.444 | INFO | "pinned" : " 29 GB" | 20260506 18:18:20.444 | INFO | } | 20260506 18:18:20.444 | INFO | Perf report for the last 5.14741 seconds: | 20260506 18:18:20.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:18:20.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:20.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:20.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:20.444 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 18:18:20.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:20.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:20.444 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:18:20.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:20.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:20.444 | INFO | Uploaded 36GiB to GPU in 2.01952s (17.826GiB/s) | 20260506 18:18:20.444 | INFO | Downloaded 34GiB to GPU in 1.96242s (17.3256GiB/s) | 20260506 18:18:25.579 | INFO | Analyzing chunk frames = [3447808,3448320) | 20260506 18:18:25.579 | INFO | MemInfo: { | 20260506 18:18:25.579 | INFO | "gpu" : " 17 GB", | 20260506 18:18:25.579 | INFO | "huge" : "", | 20260506 18:18:25.579 | INFO | "huge_shared" : " 27 GB", | 20260506 18:18:25.579 | INFO | "malloc" : " 14 GB", | 20260506 18:18:25.579 | INFO | "pinned" : " 29 GB" | 20260506 18:18:25.579 | INFO | } | 20260506 18:18:25.579 | INFO | Perf report for the last 5.13542 seconds: | 20260506 18:18:25.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:18:25.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:25.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:25.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:25.579 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 18:18:25.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:25.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:25.579 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:18:25.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:25.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:25.579 | INFO | Uploaded 36GiB to GPU in 2.00367s (17.967GiB/s) | 20260506 18:18:25.579 | INFO | Downloaded 34GiB to GPU in 1.98405s (17.1367GiB/s) | 20260506 18:18:30.719 | INFO | Analyzing chunk frames = [3448320,3448832) | 20260506 18:18:30.719 | INFO | MemInfo: { | 20260506 18:18:30.719 | INFO | "gpu" : " 17 GB", | 20260506 18:18:30.719 | INFO | "huge" : "", | 20260506 18:18:30.720 | INFO | "huge_shared" : " 27 GB", | 20260506 18:18:30.720 | INFO | "malloc" : " 14 GB", | 20260506 18:18:30.720 | INFO | "pinned" : " 29 GB" | 20260506 18:18:30.720 | INFO | } | 20260506 18:18:30.720 | INFO | Perf report for the last 5.14053 seconds: | 20260506 18:18:30.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:18:30.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:30.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:30.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:30.720 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:18:30.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:30.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:30.720 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:18:30.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:30.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:30.720 | INFO | Uploaded 36GiB to GPU in 1.99512s (18.044GiB/s) | 20260506 18:18:30.720 | INFO | Downloaded 34GiB to GPU in 1.99063s (17.08GiB/s) | 20260506 18:18:30.720 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:18:30.720 | INFO | Total mem usage: 18 GiB | 20260506 18:18:30.720 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:18:30.720 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.4%) | 20260506 18:18:30.720 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:18:30.720 | INFO | High Water Breakdown: | 20260506 18:18:30.720 | INFO | 61.8% : 9.9 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:18:30.720 | INFO | 38.1% : 6.1 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:18:30.720 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:18:30.720 | INFO | ------------------------------------------ | 20260506 18:18:30.720 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:18:30.720 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:18:30.720 | INFO | Total mem usage: 26.5 GiB | 20260506 18:18:30.720 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:18:30.720 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:18:30.720 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:18:30.720 | INFO | High Water Breakdown: | 20260506 18:18:30.720 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:18:30.720 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:18:30.720 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:18:30.720 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:18:30.720 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:18:30.720 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:18:30.720 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:18:30.720 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:18:30.720 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:18:30.720 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:18:30.720 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:18:30.720 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:18:30.720 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:18:30.720 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:18:30.720 | INFO | ------------------------------------------ | 20260506 18:18:30.720 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:18:30.720 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:18:30.720 | INFO | Total mem usage: 25.6 GiB | 20260506 18:18:30.720 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:18:30.720 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:18:30.720 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:18:30.720 | INFO | High Water Breakdown: | 20260506 18:18:30.720 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:18:30.720 | INFO | ------------------------------------------ | 20260506 18:18:30.720 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:18:30.720 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:18:30.720 | INFO | Total mem usage: 15.7 GiB | 20260506 18:18:30.720 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:18:30.720 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:18:30.720 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 18:18:30.720 | INFO | High Water Breakdown: | 20260506 18:18:30.720 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:18:30.720 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:18:30.721 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:18:30.721 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:18:30.721 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:18:30.721 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:18:30.721 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:18:30.721 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:18:30.721 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:18:30.721 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:18:30.721 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:18:30.721 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:18:30.721 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:18:30.721 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:18:30.721 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:18:30.721 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:18:30.721 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:18:30.721 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:18:30.721 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:18:30.721 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:18:30.721 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:18:30.721 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:18:30.721 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:18:30.721 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:18:30.721 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:18:30.721 | INFO | ------------------------------------------ | 20260506 18:18:30.721 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:18:30.721 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:18:30.721 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:18:30.721 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:18:30.721 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.60e+01, 1.35e+01 / 2.99e+00 | 20260506 18:18:30.721 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 1.07e-02 / 5.30e-03 | 20260506 18:18:30.721 | INFO | PerfReport: 11.60%, Baseline -- 3.63e+00 / 4.42e+00, 5.57e-01 / 3.46e-01 | 20260506 18:18:30.721 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.49e+00, 1.87e-01 / 1.19e-01 | 20260506 18:18:30.721 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.47e+01, 4.31e+00 / 3.24e+00 | 20260506 18:18:30.721 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.45e+00 / 1.82e+00 | 20260506 18:18:30.721 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.26e+00 / 2.80e+00, 3.81e-01 / 2.60e-01 | 20260506 18:18:30.721 | INFO | PerfReport: 3.40%, Metrics -- 1.06e+00 / 1.39e+00, 2.80e-01 / 1.93e-01 | 20260506 18:18:30.721 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.68e-01 / 4.18e-01, 5.87e-02 / 2.21e-02 | 20260506 18:18:30.721 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:18:30.721 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:18:30.721 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:18:30.721 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 18:18:30.721 | INFO | PerfReport: 21.44%, Upload -- 2.08e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260506 18:18:30.721 | INFO | PerfReport: 21.16%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 18:18:30.721 | INFO | PerfReport: 32.22%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:18:30.721 | INFO | PerfReport: 1.35%, LockUpload -- 1.31e+00 / 0.00e+00, 5.37e+00 / 0.00e+00 | 20260506 18:18:30.721 | INFO | PerfReport: 1.15%, LockDownload -- 1.12e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 18:18:30.721 | INFO | PerfReport: 22.65%, LockCompute -- 2.20e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 18:18:30.721 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.95e-02 / 0.00e+00 | 20260506 18:18:30.834 | INFO | Starting a round of Baz IO | 20260506 18:18:33.970 | INFO | Populating RTMetrics report | 20260506 18:18:33.970 | INFO | RTMetrics report populated | 20260506 18:18:33.970 | INFO | Opening tmp RTMetrics json file | 20260506 18:18:34.055 | INFO | Serializing RTMetrics to json | 20260506 18:18:34.084 | INFO | Serialization done, swapping tmp file now... | 20260506 18:18:34.087 | INFO | RTMetrics done | 20260506 18:18:34.507 | INFO | Finished a round of Baz IO: Wrote 2.044 GiB in 3.673 seconds | 20260506 18:18:34.507 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.15% / 0.01% / 64.85% | 20260506 18:18:34.507 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.155% / 3.97364e-06% / 9.13938e-05% | 20260506 18:18:34.507 | INFO | Baz IO overall: | 20260506 18:18:34.508 | INFO | BytesWritten : 1821466812308 | 20260506 18:18:34.508 | INFO | NumEvents : 880925384163 | 20260506 18:18:35.856 | INFO | Analyzing chunk frames = [3448832,3449344) | 20260506 18:18:35.856 | INFO | MemInfo: { | 20260506 18:18:35.856 | INFO | "gpu" : " 17 GB", | 20260506 18:18:35.856 | INFO | "huge" : "", | 20260506 18:18:35.856 | INFO | "huge_shared" : " 27 GB", | 20260506 18:18:35.856 | INFO | "malloc" : " 14 GB", | 20260506 18:18:35.856 | INFO | "pinned" : " 29 GB" | 20260506 18:18:35.856 | INFO | } | 20260506 18:18:35.856 | INFO | Perf report for the last 5.1368 seconds: | 20260506 18:18:35.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:18:35.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:35.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:35.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:35.856 | INFO | ANALYSIS : 62.7%, 2.9, 95.4ms, 0 | 20260506 18:18:35.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:35.856 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 18:18:35.856 | INFO | PRE_HQ : 60.1%, 1.9, 57.7ms, 0 | 20260506 18:18:35.857 | INFO | BAZWRITER : 71.2%, 4.8, 176.4ms, 0 | 20260506 18:18:35.857 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 18:18:35.857 | INFO | Uploaded 36GiB to GPU in 2.04369s (17.6152GiB/s) | 20260506 18:18:35.857 | INFO | Downloaded 38GiB to GPU in 2.21336s (17.1685GiB/s) | 20260506 18:18:40.995 | INFO | Analyzing chunk frames = [3449344,3449856) | 20260506 18:18:40.995 | INFO | MemInfo: { | 20260506 18:18:40.995 | INFO | "gpu" : " 17 GB", | 20260506 18:18:40.995 | INFO | "huge" : "", | 20260506 18:18:40.995 | INFO | "huge_shared" : " 27 GB", | 20260506 18:18:40.995 | INFO | "malloc" : " 14 GB", | 20260506 18:18:40.995 | INFO | "pinned" : " 29 GB" | 20260506 18:18:40.995 | INFO | } | 20260506 18:18:40.995 | INFO | Perf report for the last 5.13886 seconds: | 20260506 18:18:40.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:18:40.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:40.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:40.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:40.995 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:18:40.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:40.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:40.995 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:18:40.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:40.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:40.995 | INFO | Uploaded 36GiB to GPU in 1.9931s (18.0623GiB/s) | 20260506 18:18:40.996 | INFO | Downloaded 34GiB to GPU in 1.93835s (17.5407GiB/s) | 20260506 18:18:46.136 | INFO | Analyzing chunk frames = [3449856,3450368) | 20260506 18:18:46.136 | INFO | MemInfo: { | 20260506 18:18:46.136 | INFO | "gpu" : " 17 GB", | 20260506 18:18:46.136 | INFO | "huge" : "", | 20260506 18:18:46.136 | INFO | "huge_shared" : " 27 GB", | 20260506 18:18:46.136 | INFO | "malloc" : " 14 GB", | 20260506 18:18:46.136 | INFO | "pinned" : " 29 GB" | 20260506 18:18:46.136 | INFO | } | 20260506 18:18:46.136 | INFO | Perf report for the last 5.141 seconds: | 20260506 18:18:46.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:18:46.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:46.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:46.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:46.136 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:18:46.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:46.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:46.136 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:18:46.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:46.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:46.137 | INFO | Uploaded 36GiB to GPU in 1.95734s (18.3923GiB/s) | 20260506 18:18:46.137 | INFO | Downloaded 34GiB to GPU in 1.92102s (17.6989GiB/s) | 20260506 18:18:51.272 | INFO | Analyzing chunk frames = [3450368,3450880) | 20260506 18:18:51.272 | INFO | MemInfo: { | 20260506 18:18:51.272 | INFO | "gpu" : " 17 GB", | 20260506 18:18:51.272 | INFO | "huge" : "", | 20260506 18:18:51.272 | INFO | "huge_shared" : " 27 GB", | 20260506 18:18:51.272 | INFO | "malloc" : " 14 GB", | 20260506 18:18:51.272 | INFO | "pinned" : " 29 GB" | 20260506 18:18:51.272 | INFO | } | 20260506 18:18:51.272 | INFO | Perf report for the last 5.13613 seconds: | 20260506 18:18:51.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:18:51.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:51.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:51.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:51.273 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:18:51.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:51.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:51.273 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:18:51.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:51.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:51.273 | INFO | Uploaded 36GiB to GPU in 1.94464s (18.5124GiB/s) | 20260506 18:18:51.273 | INFO | Downloaded 34GiB to GPU in 1.8733s (18.1497GiB/s) | 20260506 18:18:56.416 | INFO | Analyzing chunk frames = [3450880,3451392) | 20260506 18:18:56.416 | INFO | MemInfo: { | 20260506 18:18:56.416 | INFO | "gpu" : " 17 GB", | 20260506 18:18:56.416 | INFO | "huge" : "", | 20260506 18:18:56.416 | INFO | "huge_shared" : " 27 GB", | 20260506 18:18:56.416 | INFO | "malloc" : " 14 GB", | 20260506 18:18:56.416 | INFO | "pinned" : " 29 GB" | 20260506 18:18:56.416 | INFO | } | 20260506 18:18:56.416 | INFO | Perf report for the last 5.14348 seconds: | 20260506 18:18:56.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:18:56.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:56.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:56.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:56.416 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 18:18:56.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:56.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:56.416 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:18:56.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:56.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:18:56.416 | INFO | Uploaded 36GiB to GPU in 2.01599s (17.8573GiB/s) | 20260506 18:18:56.416 | INFO | Downloaded 34GiB to GPU in 1.91717s (17.7345GiB/s) | 20260506 18:19:01.555 | INFO | Analyzing chunk frames = [3451392,3451904) | 20260506 18:19:01.555 | INFO | MemInfo: { | 20260506 18:19:01.555 | INFO | "gpu" : " 17 GB", | 20260506 18:19:01.556 | INFO | "huge" : "", | 20260506 18:19:01.556 | INFO | "huge_shared" : " 27 GB", | 20260506 18:19:01.556 | INFO | "malloc" : " 14 GB", | 20260506 18:19:01.556 | INFO | "pinned" : " 29 GB" | 20260506 18:19:01.556 | INFO | } | 20260506 18:19:01.556 | INFO | Perf report for the last 5.13974 seconds: | 20260506 18:19:01.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:19:01.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:01.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:01.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:01.556 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 18:19:01.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:01.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:01.556 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:19:01.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:01.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:01.556 | INFO | Uploaded 36GiB to GPU in 2.0384s (17.6609GiB/s) | 20260506 18:19:01.556 | INFO | Downloaded 34GiB to GPU in 1.95258s (17.4129GiB/s) | 20260506 18:19:06.697 | INFO | Analyzing chunk frames = [3451904,3452416) | 20260506 18:19:06.697 | INFO | MemInfo: { | 20260506 18:19:06.697 | INFO | "gpu" : " 17 GB", | 20260506 18:19:06.697 | INFO | "huge" : "", | 20260506 18:19:06.697 | INFO | "huge_shared" : " 27 GB", | 20260506 18:19:06.697 | INFO | "malloc" : " 14 GB", | 20260506 18:19:06.697 | INFO | "pinned" : " 29 GB" | 20260506 18:19:06.697 | INFO | } | 20260506 18:19:06.697 | INFO | Perf report for the last 5.14117 seconds: | 20260506 18:19:06.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:19:06.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:06.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:06.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:06.697 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 18:19:06.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:06.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:06.697 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 18:19:06.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:06.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:06.697 | INFO | Uploaded 36GiB to GPU in 1.99638s (18.0326GiB/s) | 20260506 18:19:06.697 | INFO | Downloaded 34GiB to GPU in 2.00665s (16.9437GiB/s) | 20260506 18:19:11.833 | INFO | Analyzing chunk frames = [3452416,3452928) | 20260506 18:19:11.833 | INFO | MemInfo: { | 20260506 18:19:11.833 | INFO | "gpu" : " 17 GB", | 20260506 18:19:11.833 | INFO | "huge" : "", | 20260506 18:19:11.833 | INFO | "huge_shared" : " 27 GB", | 20260506 18:19:11.833 | INFO | "malloc" : " 14 GB", | 20260506 18:19:11.833 | INFO | "pinned" : " 29 GB" | 20260506 18:19:11.833 | INFO | } | 20260506 18:19:11.833 | INFO | Perf report for the last 5.13612 seconds: | 20260506 18:19:11.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:19:11.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:11.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:11.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:11.833 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 18:19:11.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:11.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:11.833 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:19:11.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:11.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:11.833 | INFO | Uploaded 36GiB to GPU in 2.00445s (17.96GiB/s) | 20260506 18:19:11.833 | INFO | Downloaded 34GiB to GPU in 1.98506s (17.1279GiB/s) | 20260506 18:19:11.947 | INFO | Starting a round of Baz IO | 20260506 18:19:15.077 | INFO | Populating RTMetrics report | 20260506 18:19:15.077 | INFO | RTMetrics report populated | 20260506 18:19:15.077 | INFO | Opening tmp RTMetrics json file | 20260506 18:19:15.123 | INFO | Serializing RTMetrics to json | 20260506 18:19:15.160 | INFO | Serialization done, swapping tmp file now... | 20260506 18:19:15.168 | INFO | RTMetrics done | 20260506 18:19:15.530 | INFO | Finished a round of Baz IO: Wrote 2.042 GiB in 3.582 seconds | 20260506 18:19:15.530 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.14% / 0.01% / 64.85% | 20260506 18:19:15.530 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.15054% / 7.94729e-06% / 7.94729e-05% | 20260506 18:19:15.530 | INFO | Baz IO overall: | 20260506 18:19:15.530 | INFO | BytesWritten : 1823659791625 | 20260506 18:19:15.530 | INFO | NumEvents : 881982429656 | 20260506 18:19:16.971 | INFO | Analyzing chunk frames = [3452928,3453440) | 20260506 18:19:16.971 | INFO | MemInfo: { | 20260506 18:19:16.971 | INFO | "gpu" : " 17 GB", | 20260506 18:19:16.971 | INFO | "huge" : "", | 20260506 18:19:16.971 | INFO | "huge_shared" : " 27 GB", | 20260506 18:19:16.971 | INFO | "malloc" : " 14 GB", | 20260506 18:19:16.971 | INFO | "pinned" : " 29 GB" | 20260506 18:19:16.971 | INFO | } | 20260506 18:19:16.971 | INFO | Perf report for the last 5.13836 seconds: | 20260506 18:19:16.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:19:16.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:16.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:16.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:16.971 | INFO | ANALYSIS : 62.5%, 2.9, 95.0ms, 0 | 20260506 18:19:16.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:16.972 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260506 18:19:16.972 | INFO | PRE_HQ : 60.0%, 1.9, 58.0ms, 0 | 20260506 18:19:16.972 | INFO | BAZWRITER : 69.4%, 4.8, 171.7ms, 0 | 20260506 18:19:16.972 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 18:19:16.972 | INFO | Uploaded 36GiB to GPU in 2.06224s (17.4568GiB/s) | 20260506 18:19:16.972 | INFO | Downloaded 38GiB to GPU in 2.20838s (17.2072GiB/s) | 20260506 18:19:22.110 | INFO | Analyzing chunk frames = [3453440,3453952) | 20260506 18:19:22.110 | INFO | MemInfo: { | 20260506 18:19:22.110 | INFO | "gpu" : " 17 GB", | 20260506 18:19:22.110 | INFO | "huge" : "", | 20260506 18:19:22.110 | INFO | "huge_shared" : " 27 GB", | 20260506 18:19:22.110 | INFO | "malloc" : " 14 GB", | 20260506 18:19:22.110 | INFO | "pinned" : " 29 GB" | 20260506 18:19:22.110 | INFO | } | 20260506 18:19:22.110 | INFO | Perf report for the last 5.13866 seconds: | 20260506 18:19:22.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:19:22.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:22.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:22.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:22.110 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 18:19:22.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:22.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:22.110 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:19:22.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:22.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:22.110 | INFO | Uploaded 36GiB to GPU in 2.0235s (17.7909GiB/s) | 20260506 18:19:22.110 | INFO | Downloaded 34GiB to GPU in 1.93622s (17.56GiB/s) | 20260506 18:19:27.250 | INFO | Analyzing chunk frames = [3453952,3454464) | 20260506 18:19:27.250 | INFO | MemInfo: { | 20260506 18:19:27.250 | INFO | "gpu" : " 17 GB", | 20260506 18:19:27.250 | INFO | "huge" : "", | 20260506 18:19:27.250 | INFO | "huge_shared" : " 27 GB", | 20260506 18:19:27.250 | INFO | "malloc" : " 14 GB", | 20260506 18:19:27.251 | INFO | "pinned" : " 29 GB" | 20260506 18:19:27.251 | INFO | } | 20260506 18:19:27.251 | INFO | Perf report for the last 5.14053 seconds: | 20260506 18:19:27.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:19:27.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:27.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:27.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:27.251 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:19:27.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:27.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:27.251 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:19:27.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:27.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:27.251 | INFO | Uploaded 36GiB to GPU in 1.97753s (18.2045GiB/s) | 20260506 18:19:27.251 | INFO | Downloaded 34GiB to GPU in 1.91889s (17.7186GiB/s) | 20260506 18:19:32.388 | INFO | Analyzing chunk frames = [3454464,3454976) | 20260506 18:19:32.388 | INFO | MemInfo: { | 20260506 18:19:32.388 | INFO | "gpu" : " 17 GB", | 20260506 18:19:32.388 | INFO | "huge" : "", | 20260506 18:19:32.388 | INFO | "huge_shared" : " 27 GB", | 20260506 18:19:32.388 | INFO | "malloc" : " 14 GB", | 20260506 18:19:32.388 | INFO | "pinned" : " 29 GB" | 20260506 18:19:32.388 | INFO | } | 20260506 18:19:32.388 | INFO | Perf report for the last 5.1378 seconds: | 20260506 18:19:32.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:19:32.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:32.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:32.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:32.388 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:19:32.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:32.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:32.389 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:19:32.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:32.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:32.389 | INFO | Uploaded 36GiB to GPU in 1.94255s (18.5323GiB/s) | 20260506 18:19:32.389 | INFO | Downloaded 34GiB to GPU in 1.8691s (18.1906GiB/s) | 20260506 18:19:37.524 | INFO | Analyzing chunk frames = [3454976,3455488) | 20260506 18:19:37.524 | INFO | MemInfo: { | 20260506 18:19:37.524 | INFO | "gpu" : " 17 GB", | 20260506 18:19:37.525 | INFO | "huge" : "", | 20260506 18:19:37.525 | INFO | "huge_shared" : " 27 GB", | 20260506 18:19:37.525 | INFO | "malloc" : " 14 GB", | 20260506 18:19:37.525 | INFO | "pinned" : " 29 GB" | 20260506 18:19:37.525 | INFO | } | 20260506 18:19:37.525 | INFO | Perf report for the last 5.13626 seconds: | 20260506 18:19:37.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:19:37.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:37.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:37.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:37.525 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260506 18:19:37.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:37.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:37.525 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:19:37.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:37.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:37.525 | INFO | Uploaded 36GiB to GPU in 1.96291s (18.3401GiB/s) | 20260506 18:19:37.525 | INFO | Downloaded 34GiB to GPU in 1.87942s (18.0907GiB/s) | 20260506 18:19:42.668 | INFO | Analyzing chunk frames = [3455488,3456000) | 20260506 18:19:42.668 | INFO | MemInfo: { | 20260506 18:19:42.668 | INFO | "gpu" : " 17 GB", | 20260506 18:19:42.668 | INFO | "huge" : "", | 20260506 18:19:42.668 | INFO | "huge_shared" : " 27 GB", | 20260506 18:19:42.668 | INFO | "malloc" : " 14 GB", | 20260506 18:19:42.668 | INFO | "pinned" : " 29 GB" | 20260506 18:19:42.668 | INFO | } | 20260506 18:19:42.668 | INFO | Perf report for the last 5.1438 seconds: | 20260506 18:19:42.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:19:42.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:42.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:42.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:42.668 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 18:19:42.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:42.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:42.669 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:19:42.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:42.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:42.669 | INFO | Uploaded 36GiB to GPU in 2.02212s (17.8031GiB/s) | 20260506 18:19:42.669 | INFO | Downloaded 34GiB to GPU in 1.95093s (17.4276GiB/s) | 20260506 18:19:47.811 | INFO | Analyzing chunk frames = [3456000,3456512) | 20260506 18:19:47.811 | INFO | MemInfo: { | 20260506 18:19:47.811 | INFO | "gpu" : " 17 GB", | 20260506 18:19:47.811 | INFO | "huge" : "", | 20260506 18:19:47.811 | INFO | "huge_shared" : " 27 GB", | 20260506 18:19:47.811 | INFO | "malloc" : " 14 GB", | 20260506 18:19:47.811 | INFO | "pinned" : " 29 GB" | 20260506 18:19:47.811 | INFO | } | 20260506 18:19:47.811 | INFO | Perf report for the last 5.14262 seconds: | 20260506 18:19:47.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:19:47.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:47.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:47.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:47.811 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 18:19:47.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:47.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:47.811 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:19:47.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:47.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:47.811 | INFO | Uploaded 36GiB to GPU in 2.00903s (17.9191GiB/s) | 20260506 18:19:47.811 | INFO | Downloaded 34GiB to GPU in 2.01951s (16.8358GiB/s) | 20260506 18:19:52.947 | INFO | Analyzing chunk frames = [3456512,3457024) | 20260506 18:19:52.947 | INFO | MemInfo: { | 20260506 18:19:52.947 | INFO | "gpu" : " 17 GB", | 20260506 18:19:52.947 | INFO | "huge" : "", | 20260506 18:19:52.947 | INFO | "huge_shared" : " 27 GB", | 20260506 18:19:52.947 | INFO | "malloc" : " 14 GB", | 20260506 18:19:52.947 | INFO | "pinned" : " 29 GB" | 20260506 18:19:52.947 | INFO | } | 20260506 18:19:52.947 | INFO | Perf report for the last 5.13625 seconds: | 20260506 18:19:52.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:19:52.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:52.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:52.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:52.947 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 18:19:52.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:52.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:52.947 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 18:19:52.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:52.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:52.948 | INFO | Uploaded 36GiB to GPU in 2.01747s (17.8441GiB/s) | 20260506 18:19:52.948 | INFO | Downloaded 34GiB to GPU in 1.99071s (17.0793GiB/s) | 20260506 18:19:52.948 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:19:52.948 | INFO | Total mem usage: 18 GiB | 20260506 18:19:52.948 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:19:52.948 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: -9.34%) | 20260506 18:19:52.948 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:19:52.948 | INFO | High Water Breakdown: | 20260506 18:19:52.948 | INFO | 62.2% : 10.2 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:19:52.948 | INFO | 37.7% : 6.2 GiB ( 92.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:19:52.948 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:19:52.948 | INFO | ------------------------------------------ | 20260506 18:19:52.948 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:19:52.948 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:19:52.948 | INFO | Total mem usage: 26.5 GiB | 20260506 18:19:52.948 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:19:52.948 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:19:52.948 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:19:52.948 | INFO | High Water Breakdown: | 20260506 18:19:52.948 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:19:52.948 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:19:52.948 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:19:52.948 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:19:52.948 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:19:52.948 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:19:52.948 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:19:52.948 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:19:52.948 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:19:52.948 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:19:52.948 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:19:52.948 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:19:52.948 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:19:52.948 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:19:52.948 | INFO | ------------------------------------------ | 20260506 18:19:52.948 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:19:52.948 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:19:52.948 | INFO | Total mem usage: 25.6 GiB | 20260506 18:19:52.948 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:19:52.948 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:19:52.948 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:19:52.948 | INFO | High Water Breakdown: | 20260506 18:19:52.948 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:19:52.948 | INFO | ------------------------------------------ | 20260506 18:19:52.948 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:19:52.948 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:19:52.948 | INFO | Total mem usage: 15.7 GiB | 20260506 18:19:52.948 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:19:52.948 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:19:52.948 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:19:52.948 | INFO | High Water Breakdown: | 20260506 18:19:52.948 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:19:52.948 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:19:52.948 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:19:52.948 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:19:52.948 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:19:52.948 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:19:52.948 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:19:52.948 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:19:52.948 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:19:52.948 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:19:52.948 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:19:52.948 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:19:52.948 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:19:52.948 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:19:52.948 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:19:52.948 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:19:52.948 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:19:52.948 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:19:52.948 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:19:52.948 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:19:52.948 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:19:52.948 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:19:52.948 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:19:52.948 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:19:52.948 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:19:52.948 | INFO | ------------------------------------------ | 20260506 18:19:52.948 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:19:52.949 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:19:52.949 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:19:52.949 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:19:52.949 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.56e+01, 1.35e+01 / 2.65e+00 | 20260506 18:19:52.949 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 9.83e-02, 1.03e-02 / 6.86e-03 | 20260506 18:19:52.949 | INFO | PerfReport: 11.59%, Baseline -- 3.62e+00 / 4.37e+00, 5.52e-01 / 3.80e-01 | 20260506 18:19:52.949 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.43e+00, 1.82e-01 / 1.48e-01 | 20260506 18:19:52.949 | INFO | PerfReport: 15.97%, DME -- 3.99e+01 / 4.46e+01, 4.01e+00 / 2.39e+00 | 20260506 18:19:52.949 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.41e+00 / 1.64e+00 | 20260506 18:19:52.949 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.26e+00 / 2.82e+00, 3.81e-01 / 2.65e-01 | 20260506 18:19:52.949 | INFO | PerfReport: 3.41%, Metrics -- 1.07e+00 / 1.40e+00, 2.80e-01 / 1.89e-01 | 20260506 18:19:52.949 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.68e-01 / 4.16e-01, 5.76e-02 / 2.86e-02 | 20260506 18:19:52.949 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:19:52.949 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:19:52.949 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:19:52.949 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 18:19:52.949 | INFO | PerfReport: 21.48%, Upload -- 2.08e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260506 18:19:52.949 | INFO | PerfReport: 21.19%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 18:19:52.949 | INFO | PerfReport: 32.22%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:19:52.949 | INFO | PerfReport: 1.33%, LockUpload -- 1.29e+00 / 0.00e+00, 5.37e+00 / 0.00e+00 | 20260506 18:19:52.949 | INFO | PerfReport: 1.14%, LockDownload -- 1.11e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 18:19:52.949 | INFO | PerfReport: 22.63%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:19:52.949 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.24e-02 / 0.00e+00 | 20260506 18:19:53.064 | INFO | Starting a round of Baz IO | 20260506 18:19:56.200 | INFO | Populating RTMetrics report | 20260506 18:19:56.200 | INFO | RTMetrics report populated | 20260506 18:19:56.201 | INFO | Opening tmp RTMetrics json file | 20260506 18:19:56.212 | INFO | Serializing RTMetrics to json | 20260506 18:19:56.217 | INFO | Serialization done, swapping tmp file now... | 20260506 18:19:56.222 | INFO | RTMetrics done | 20260506 18:19:56.544 | INFO | Finished a round of Baz IO: Wrote 2.042 GiB in 3.48 seconds | 20260506 18:19:56.544 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.14% / 0.01% / 64.86% | 20260506 18:19:56.544 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.14901% / 3.97364e-06% / 4.37101e-05% | 20260506 18:19:56.544 | INFO | Baz IO overall: | 20260506 18:19:56.544 | INFO | BytesWritten : 1825852514797 | 20260506 18:19:56.544 | INFO | NumEvents : 883038999490 | 20260506 18:19:58.091 | INFO | Analyzing chunk frames = [3457024,3457536) | 20260506 18:19:58.091 | INFO | MemInfo: { | 20260506 18:19:58.091 | INFO | "gpu" : " 17 GB", | 20260506 18:19:58.091 | INFO | "huge" : "", | 20260506 18:19:58.091 | INFO | "huge_shared" : " 27 GB", | 20260506 18:19:58.091 | INFO | "malloc" : " 14 GB", | 20260506 18:19:58.091 | INFO | "pinned" : " 29 GB" | 20260506 18:19:58.091 | INFO | } | 20260506 18:19:58.091 | INFO | Perf report for the last 5.14345 seconds: | 20260506 18:19:58.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:19:58.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:58.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:58.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:58.091 | INFO | ANALYSIS : 62.6%, 2.9, 95.5ms, 0 | 20260506 18:19:58.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:19:58.091 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 18:19:58.091 | INFO | PRE_HQ : 60.0%, 1.9, 57.8ms, 0 | 20260506 18:19:58.091 | INFO | BAZWRITER : 67.3%, 4.8, 166.2ms, 0 | 20260506 18:19:58.091 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 18:19:58.091 | INFO | Uploaded 36GiB to GPU in 2.0385s (17.6601GiB/s) | 20260506 18:19:58.091 | INFO | Downloaded 38GiB to GPU in 2.21665s (17.143GiB/s) | 20260506 18:20:03.225 | INFO | Analyzing chunk frames = [3457536,3458048) | 20260506 18:20:03.225 | INFO | MemInfo: { | 20260506 18:20:03.225 | INFO | "gpu" : " 17 GB", | 20260506 18:20:03.225 | INFO | "huge" : "", | 20260506 18:20:03.225 | INFO | "huge_shared" : " 27 GB", | 20260506 18:20:03.225 | INFO | "malloc" : " 14 GB", | 20260506 18:20:03.225 | INFO | "pinned" : " 29 GB" | 20260506 18:20:03.225 | INFO | } | 20260506 18:20:03.226 | INFO | Perf report for the last 5.13468 seconds: | 20260506 18:20:03.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:20:03.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:03.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:03.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:03.226 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 18:20:03.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:03.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:03.226 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:20:03.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:03.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:03.226 | INFO | Uploaded 36GiB to GPU in 1.99678s (18.029GiB/s) | 20260506 18:20:03.226 | INFO | Downloaded 34GiB to GPU in 1.92251s (17.6852GiB/s) | 20260506 18:20:08.366 | INFO | Analyzing chunk frames = [3458048,3458560) | 20260506 18:20:08.366 | INFO | MemInfo: { | 20260506 18:20:08.366 | INFO | "gpu" : " 17 GB", | 20260506 18:20:08.366 | INFO | "huge" : "", | 20260506 18:20:08.366 | INFO | "huge_shared" : " 27 GB", | 20260506 18:20:08.366 | INFO | "malloc" : " 14 GB", | 20260506 18:20:08.366 | INFO | "pinned" : " 29 GB" | 20260506 18:20:08.366 | INFO | } | 20260506 18:20:08.366 | INFO | Perf report for the last 5.1404 seconds: | 20260506 18:20:08.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:20:08.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:08.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:08.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:08.366 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260506 18:20:08.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:08.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:08.366 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:20:08.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:08.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:08.366 | INFO | Uploaded 36GiB to GPU in 1.95298s (18.4334GiB/s) | 20260506 18:20:08.366 | INFO | Downloaded 34GiB to GPU in 1.90551s (17.843GiB/s) | 20260506 18:20:13.505 | INFO | Analyzing chunk frames = [3458560,3459072) | 20260506 18:20:13.505 | INFO | MemInfo: { | 20260506 18:20:13.505 | INFO | "gpu" : " 17 GB", | 20260506 18:20:13.505 | INFO | "huge" : "", | 20260506 18:20:13.505 | INFO | "huge_shared" : " 27 GB", | 20260506 18:20:13.505 | INFO | "malloc" : " 14 GB", | 20260506 18:20:13.505 | INFO | "pinned" : " 29 GB" | 20260506 18:20:13.505 | INFO | } | 20260506 18:20:13.505 | INFO | Perf report for the last 5.13952 seconds: | 20260506 18:20:13.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:20:13.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:13.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:13.506 | INFO | SAVE_TRACE : 0.1%, 1.0, 0.1ms, 0 | 20260506 18:20:13.506 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:20:13.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:13.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:13.506 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:20:13.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:13.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:13.506 | INFO | Uploaded 36GiB to GPU in 1.92853s (18.667GiB/s) | 20260506 18:20:13.506 | INFO | Downloaded 34GiB to GPU in 1.87954s (18.0896GiB/s) | 20260506 18:20:18.639 | INFO | Analyzing chunk frames = [3459072,3459584) | 20260506 18:20:18.640 | INFO | MemInfo: { | 20260506 18:20:18.640 | INFO | "gpu" : " 17 GB", | 20260506 18:20:18.640 | INFO | "huge" : "", | 20260506 18:20:18.640 | INFO | "huge_shared" : " 27 GB", | 20260506 18:20:18.640 | INFO | "malloc" : " 14 GB", | 20260506 18:20:18.640 | INFO | "pinned" : " 29 GB" | 20260506 18:20:18.640 | INFO | } | 20260506 18:20:18.640 | INFO | Perf report for the last 5.13417 seconds: | 20260506 18:20:18.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:20:18.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:18.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:18.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:18.640 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260506 18:20:18.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:18.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:18.640 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:20:18.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:18.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:18.640 | INFO | Uploaded 36GiB to GPU in 1.94938s (18.4674GiB/s) | 20260506 18:20:18.640 | INFO | Downloaded 34GiB to GPU in 1.88488s (18.0383GiB/s) | 20260506 18:20:23.785 | INFO | Analyzing chunk frames = [3459584,3460096) | 20260506 18:20:23.785 | INFO | MemInfo: { | 20260506 18:20:23.785 | INFO | "gpu" : " 17 GB", | 20260506 18:20:23.785 | INFO | "huge" : "", | 20260506 18:20:23.785 | INFO | "huge_shared" : " 27 GB", | 20260506 18:20:23.785 | INFO | "malloc" : " 14 GB", | 20260506 18:20:23.785 | INFO | "pinned" : " 29 GB" | 20260506 18:20:23.785 | INFO | } | 20260506 18:20:23.785 | INFO | Perf report for the last 5.14524 seconds: | 20260506 18:20:23.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:20:23.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:23.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:23.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:23.785 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 18:20:23.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:23.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:23.785 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:20:23.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:23.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:23.785 | INFO | Uploaded 36GiB to GPU in 2.00985s (17.9118GiB/s) | 20260506 18:20:23.785 | INFO | Downloaded 34GiB to GPU in 1.96152s (17.3335GiB/s) | 20260506 18:20:28.926 | INFO | Analyzing chunk frames = [3460096,3460608) | 20260506 18:20:28.926 | INFO | MemInfo: { | 20260506 18:20:28.926 | INFO | "gpu" : " 17 GB", | 20260506 18:20:28.926 | INFO | "huge" : "", | 20260506 18:20:28.926 | INFO | "huge_shared" : " 27 GB", | 20260506 18:20:28.926 | INFO | "malloc" : " 14 GB", | 20260506 18:20:28.926 | INFO | "pinned" : " 29 GB" | 20260506 18:20:28.926 | INFO | } | 20260506 18:20:28.926 | INFO | Perf report for the last 5.14111 seconds: | 20260506 18:20:28.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:20:28.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:28.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:28.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:28.926 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:20:28.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:28.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:28.926 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 18:20:28.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:28.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:28.926 | INFO | Uploaded 36GiB to GPU in 1.97314s (18.245GiB/s) | 20260506 18:20:28.926 | INFO | Downloaded 34GiB to GPU in 2.0088s (16.9255GiB/s) | 20260506 18:20:34.066 | INFO | Analyzing chunk frames = [3460608,3461120) | 20260506 18:20:34.066 | INFO | MemInfo: { | 20260506 18:20:34.066 | INFO | "gpu" : " 17 GB", | 20260506 18:20:34.066 | INFO | "huge" : "", | 20260506 18:20:34.066 | INFO | "huge_shared" : " 27 GB", | 20260506 18:20:34.066 | INFO | "malloc" : " 14 GB", | 20260506 18:20:34.066 | INFO | "pinned" : " 29 GB" | 20260506 18:20:34.066 | INFO | } | 20260506 18:20:34.066 | INFO | Perf report for the last 5.13977 seconds: | 20260506 18:20:34.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:20:34.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:34.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:34.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:34.066 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 18:20:34.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:34.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:34.066 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:20:34.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:34.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:34.066 | INFO | Uploaded 36GiB to GPU in 1.97405s (18.2367GiB/s) | 20260506 18:20:34.066 | INFO | Downloaded 34GiB to GPU in 1.98489s (17.1294GiB/s) | 20260506 18:20:34.180 | INFO | Starting a round of Baz IO | 20260506 18:20:37.322 | INFO | Populating RTMetrics report | 20260506 18:20:37.322 | INFO | RTMetrics report populated | 20260506 18:20:37.322 | INFO | Opening tmp RTMetrics json file | 20260506 18:20:37.371 | INFO | Serializing RTMetrics to json | 20260506 18:20:37.466 | INFO | Serialization done, swapping tmp file now... | 20260506 18:20:37.493 | INFO | RTMetrics done | 20260506 18:20:37.525 | INFO | Finished a round of Baz IO: Wrote 2.043 GiB in 3.345 seconds | 20260506 18:20:37.525 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.13% / 0.01% / 64.86% | 20260506 18:20:37.525 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.14713% / 1.19209e-05% / 6.75519e-05% | 20260506 18:20:37.525 | INFO | Baz IO overall: | 20260506 18:20:37.525 | INFO | BytesWritten : 1828046024724 | 20260506 18:20:37.525 | INFO | NumEvents : 884096187173 | 20260506 18:20:39.206 | INFO | Analyzing chunk frames = [3461120,3461632) | 20260506 18:20:39.206 | INFO | MemInfo: { | 20260506 18:20:39.206 | INFO | "gpu" : " 17 GB", | 20260506 18:20:39.206 | INFO | "huge" : "", | 20260506 18:20:39.206 | INFO | "huge_shared" : " 27 GB", | 20260506 18:20:39.206 | INFO | "malloc" : " 14 GB", | 20260506 18:20:39.206 | INFO | "pinned" : " 29 GB" | 20260506 18:20:39.206 | INFO | } | 20260506 18:20:39.206 | INFO | Perf report for the last 5.14022 seconds: | 20260506 18:20:39.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:20:39.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:39.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:39.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:39.206 | INFO | ANALYSIS : 62.8%, 2.9, 95.6ms, 0 | 20260506 18:20:39.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:39.206 | INFO | RT_METRICS : 46.6%, 1.0, 24.5ms, 0 | 20260506 18:20:39.206 | INFO | PRE_HQ : 60.0%, 1.9, 57.8ms, 0 | 20260506 18:20:39.206 | INFO | BAZWRITER : 64.8%, 4.8, 159.7ms, 0 | 20260506 18:20:39.206 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.4ms, 0 | 20260506 18:20:39.206 | INFO | Uploaded 36GiB to GPU in 2.07999s (17.3078GiB/s) | 20260506 18:20:39.206 | INFO | Downloaded 38GiB to GPU in 2.21771s (17.1348GiB/s) | 20260506 18:20:44.338 | INFO | Analyzing chunk frames = [3461632,3462144) | 20260506 18:20:44.338 | INFO | MemInfo: { | 20260506 18:20:44.338 | INFO | "gpu" : " 17 GB", | 20260506 18:20:44.338 | INFO | "huge" : "", | 20260506 18:20:44.338 | INFO | "huge_shared" : " 27 GB", | 20260506 18:20:44.338 | INFO | "malloc" : " 14 GB", | 20260506 18:20:44.338 | INFO | "pinned" : " 29 GB" | 20260506 18:20:44.338 | INFO | } | 20260506 18:20:44.338 | INFO | Perf report for the last 5.1324 seconds: | 20260506 18:20:44.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:20:44.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:44.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:44.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:44.339 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260506 18:20:44.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:44.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:44.339 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:20:44.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:44.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:44.339 | INFO | Uploaded 36GiB to GPU in 2.01527s (17.8636GiB/s) | 20260506 18:20:44.339 | INFO | Downloaded 34GiB to GPU in 1.93112s (17.6064GiB/s) | 20260506 18:20:49.475 | INFO | Analyzing chunk frames = [3462144,3462656) | 20260506 18:20:49.475 | INFO | MemInfo: { | 20260506 18:20:49.475 | INFO | "gpu" : " 17 GB", | 20260506 18:20:49.475 | INFO | "huge" : "", | 20260506 18:20:49.475 | INFO | "huge_shared" : " 27 GB", | 20260506 18:20:49.475 | INFO | "malloc" : " 14 GB", | 20260506 18:20:49.475 | INFO | "pinned" : " 29 GB" | 20260506 18:20:49.475 | INFO | } | 20260506 18:20:49.475 | INFO | Perf report for the last 5.13645 seconds: | 20260506 18:20:49.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:20:49.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:49.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:49.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:49.475 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 18:20:49.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:49.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:49.475 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:20:49.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:49.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:49.475 | INFO | Uploaded 36GiB to GPU in 1.97982s (18.1835GiB/s) | 20260506 18:20:49.475 | INFO | Downloaded 34GiB to GPU in 1.92209s (17.6891GiB/s) | 20260506 18:20:54.614 | INFO | Analyzing chunk frames = [3462656,3463168) | 20260506 18:20:54.614 | INFO | MemInfo: { | 20260506 18:20:54.614 | INFO | "gpu" : " 17 GB", | 20260506 18:20:54.614 | INFO | "huge" : "", | 20260506 18:20:54.614 | INFO | "huge_shared" : " 27 GB", | 20260506 18:20:54.614 | INFO | "malloc" : " 14 GB", | 20260506 18:20:54.614 | INFO | "pinned" : " 29 GB" | 20260506 18:20:54.614 | INFO | } | 20260506 18:20:54.614 | INFO | Perf report for the last 5.1394 seconds: | 20260506 18:20:54.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:20:54.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:54.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:54.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:54.614 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 18:20:54.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:54.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:54.614 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:20:54.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:54.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:54.615 | INFO | Uploaded 36GiB to GPU in 1.92943s (18.6584GiB/s) | 20260506 18:20:54.615 | INFO | Downloaded 34GiB to GPU in 1.878s (18.1044GiB/s) | 20260506 18:20:59.759 | INFO | Analyzing chunk frames = [3463168,3463680) | 20260506 18:20:59.759 | INFO | MemInfo: { | 20260506 18:20:59.759 | INFO | "gpu" : " 17 GB", | 20260506 18:20:59.759 | INFO | "huge" : "", | 20260506 18:20:59.759 | INFO | "huge_shared" : " 27 GB", | 20260506 18:20:59.759 | INFO | "malloc" : " 14 GB", | 20260506 18:20:59.759 | INFO | "pinned" : " 29 GB" | 20260506 18:20:59.759 | INFO | } | 20260506 18:20:59.759 | INFO | Perf report for the last 5.14464 seconds: | 20260506 18:20:59.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:20:59.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:59.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:59.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:59.759 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 18:20:59.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:59.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:59.759 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:20:59.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:59.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:20:59.759 | INFO | Uploaded 36GiB to GPU in 1.95883s (18.3783GiB/s) | 20260506 18:20:59.759 | INFO | Downloaded 34GiB to GPU in 1.88091s (18.0764GiB/s) | 20260506 18:21:04.900 | INFO | Analyzing chunk frames = [3463680,3464192) | 20260506 18:21:04.900 | INFO | MemInfo: { | 20260506 18:21:04.900 | INFO | "gpu" : " 17 GB", | 20260506 18:21:04.900 | INFO | "huge" : "", | 20260506 18:21:04.900 | INFO | "huge_shared" : " 27 GB", | 20260506 18:21:04.900 | INFO | "malloc" : " 14 GB", | 20260506 18:21:04.900 | INFO | "pinned" : " 29 GB" | 20260506 18:21:04.900 | INFO | } | 20260506 18:21:04.900 | INFO | Perf report for the last 5.14149 seconds: | 20260506 18:21:04.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:21:04.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:04.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:04.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:04.901 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 18:21:04.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:04.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:04.901 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:21:04.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:04.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:04.901 | INFO | Uploaded 36GiB to GPU in 1.99942s (18.0052GiB/s) | 20260506 18:21:04.901 | INFO | Downloaded 34GiB to GPU in 1.94288s (17.4998GiB/s) | 20260506 18:21:10.041 | INFO | Analyzing chunk frames = [3464192,3464704) | 20260506 18:21:10.041 | INFO | MemInfo: { | 20260506 18:21:10.041 | INFO | "gpu" : " 17 GB", | 20260506 18:21:10.041 | INFO | "huge" : "", | 20260506 18:21:10.041 | INFO | "huge_shared" : " 27 GB", | 20260506 18:21:10.041 | INFO | "malloc" : " 14 GB", | 20260506 18:21:10.041 | INFO | "pinned" : " 29 GB" | 20260506 18:21:10.041 | INFO | } | 20260506 18:21:10.041 | INFO | Perf report for the last 5.14052 seconds: | 20260506 18:21:10.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:21:10.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:10.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:10.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:10.041 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:21:10.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:10.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:10.041 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:21:10.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:10.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:10.041 | INFO | Uploaded 36GiB to GPU in 2.0166s (17.8518GiB/s) | 20260506 18:21:10.041 | INFO | Downloaded 34GiB to GPU in 1.99608s (17.0334GiB/s) | 20260506 18:21:15.181 | INFO | Analyzing chunk frames = [3464704,3465216) | 20260506 18:21:15.181 | INFO | MemInfo: { | 20260506 18:21:15.181 | INFO | "gpu" : " 17 GB", | 20260506 18:21:15.181 | INFO | "huge" : "", | 20260506 18:21:15.181 | INFO | "huge_shared" : " 27 GB", | 20260506 18:21:15.181 | INFO | "malloc" : " 14 GB", | 20260506 18:21:15.181 | INFO | "pinned" : " 29 GB" | 20260506 18:21:15.181 | INFO | } | 20260506 18:21:15.182 | INFO | Perf report for the last 5.14052 seconds: | 20260506 18:21:15.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:21:15.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:15.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:15.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:15.182 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 18:21:15.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:15.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:15.182 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 18:21:15.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:15.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:15.182 | INFO | Uploaded 36GiB to GPU in 2.00207s (17.9814GiB/s) | 20260506 18:21:15.182 | INFO | Downloaded 34GiB to GPU in 1.99076s (17.0789GiB/s) | 20260506 18:21:15.182 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:21:15.182 | INFO | Total mem usage: 18 GiB | 20260506 18:21:15.182 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:21:15.182 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -19.6%) | 20260506 18:21:15.182 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:21:15.182 | INFO | High Water Breakdown: | 20260506 18:21:15.182 | INFO | 60.9% : 9.16 GiB ( 80.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:21:15.182 | INFO | 39% : 5.87 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:21:15.182 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:21:15.182 | INFO | ------------------------------------------ | 20260506 18:21:15.182 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:21:15.182 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:21:15.182 | INFO | Total mem usage: 26.5 GiB | 20260506 18:21:15.182 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:21:15.182 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:21:15.182 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:21:15.182 | INFO | High Water Breakdown: | 20260506 18:21:15.182 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:21:15.182 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:21:15.182 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:21:15.182 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:21:15.182 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:21:15.182 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:21:15.182 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:21:15.182 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:21:15.182 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:21:15.182 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:21:15.182 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:21:15.182 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:21:15.182 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:21:15.182 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:21:15.182 | INFO | ------------------------------------------ | 20260506 18:21:15.182 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:21:15.182 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:21:15.182 | INFO | Total mem usage: 25.6 GiB | 20260506 18:21:15.182 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:21:15.182 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:21:15.182 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:21:15.182 | INFO | High Water Breakdown: | 20260506 18:21:15.182 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:21:15.182 | INFO | ------------------------------------------ | 20260506 18:21:15.182 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:21:15.182 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:21:15.182 | INFO | Total mem usage: 15.7 GiB | 20260506 18:21:15.182 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:21:15.182 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:21:15.182 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:21:15.182 | INFO | High Water Breakdown: | 20260506 18:21:15.182 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:21:15.183 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:21:15.183 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:21:15.183 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:21:15.183 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:21:15.183 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:21:15.183 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:21:15.183 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:21:15.183 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:21:15.183 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:21:15.183 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:21:15.183 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:21:15.183 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:21:15.183 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:21:15.183 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:21:15.183 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:21:15.183 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:21:15.183 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:21:15.183 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:21:15.183 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:21:15.183 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:21:15.183 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:21:15.183 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:21:15.183 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:21:15.183 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:21:15.183 | INFO | ------------------------------------------ | 20260506 18:21:15.183 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:21:15.183 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:21:15.183 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:21:15.183 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:21:15.183 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.54e+01, 1.34e+01 / 2.49e+00 | 20260506 18:21:15.183 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.01e-01, 9.82e-03 / 7.39e-03 | 20260506 18:21:15.183 | INFO | PerfReport: 11.58%, Baseline -- 3.62e+00 / 4.36e+00, 5.54e-01 / 4.07e-01 | 20260506 18:21:15.183 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.42e+00, 1.79e-01 / 1.30e-01 | 20260506 18:21:15.183 | INFO | PerfReport: 15.95%, DME -- 3.99e+01 / 4.44e+01, 4.09e+00 / 2.45e+00 | 20260506 18:21:15.183 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.79e+01 / 2.04e+01, 1.44e+00 / 1.97e+00 | 20260506 18:21:15.183 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.26e+00 / 2.83e+00, 3.89e-01 / 2.80e-01 | 20260506 18:21:15.183 | INFO | PerfReport: 3.41%, Metrics -- 1.06e+00 / 1.43e+00, 2.80e-01 / 1.85e-01 | 20260506 18:21:15.183 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.67e-01 / 4.15e-01, 5.84e-02 / 2.43e-02 | 20260506 18:21:15.183 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:21:15.183 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:21:15.183 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:21:15.183 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 18:21:15.183 | INFO | PerfReport: 21.36%, Upload -- 2.07e+01 / 0.00e+00, 6.84e+00 / 0.00e+00 | 20260506 18:21:15.183 | INFO | PerfReport: 21.17%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 18:21:15.183 | INFO | PerfReport: 32.25%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 18:21:15.183 | INFO | PerfReport: 1.27%, LockUpload -- 1.23e+00 / 0.00e+00, 5.09e+00 / 0.00e+00 | 20260506 18:21:15.183 | INFO | PerfReport: 1.18%, LockDownload -- 1.15e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 18:21:15.183 | INFO | PerfReport: 22.75%, LockCompute -- 2.21e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 18:21:15.183 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.20e-02 / 0.00e+00 | 20260506 18:21:15.296 | INFO | Starting a round of Baz IO | 20260506 18:21:18.440 | INFO | Populating RTMetrics report | 20260506 18:21:18.440 | INFO | RTMetrics report populated | 20260506 18:21:18.440 | INFO | Opening tmp RTMetrics json file | 20260506 18:21:18.550 | INFO | Serializing RTMetrics to json | 20260506 18:21:18.552 | INFO | Serialization done, swapping tmp file now... | 20260506 18:21:18.553 | INFO | RTMetrics done | 20260506 18:21:19.084 | INFO | Finished a round of Baz IO: Wrote 2.043 GiB in 3.787 seconds | 20260506 18:21:19.084 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.13% / 0.01% / 64.87% | 20260506 18:21:19.084 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.1438% / 7.94729e-06% / 6.75519e-05% | 20260506 18:21:19.084 | INFO | Baz IO overall: | 20260506 18:21:19.084 | INFO | BytesWritten : 1830239954123 | 20260506 18:21:19.084 | INFO | NumEvents : 885154091907 | 20260506 18:21:20.318 | INFO | Analyzing chunk frames = [3465216,3465728) | 20260506 18:21:20.318 | INFO | MemInfo: { | 20260506 18:21:20.318 | INFO | "gpu" : " 17 GB", | 20260506 18:21:20.318 | INFO | "huge" : "", | 20260506 18:21:20.318 | INFO | "huge_shared" : " 27 GB", | 20260506 18:21:20.318 | INFO | "malloc" : " 14 GB", | 20260506 18:21:20.318 | INFO | "pinned" : " 29 GB" | 20260506 18:21:20.318 | INFO | } | 20260506 18:21:20.318 | INFO | Perf report for the last 5.1369 seconds: | 20260506 18:21:20.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:21:20.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:20.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:20.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:20.319 | INFO | ANALYSIS : 62.8%, 2.9, 95.5ms, 0 | 20260506 18:21:20.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:20.319 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 18:21:20.319 | INFO | PRE_HQ : 60.2%, 1.8, 57.6ms, 0 | 20260506 18:21:20.319 | INFO | BAZWRITER : 73.4%, 4.8, 182.6ms, 0 | 20260506 18:21:20.319 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 18:21:20.319 | INFO | Uploaded 36GiB to GPU in 2.0537s (17.5293GiB/s) | 20260506 18:21:20.319 | INFO | Downloaded 38GiB to GPU in 2.22556s (17.0744GiB/s) | 20260506 18:21:25.454 | INFO | Analyzing chunk frames = [3465728,3466240) | 20260506 18:21:25.454 | INFO | MemInfo: { | 20260506 18:21:25.454 | INFO | "gpu" : " 17 GB", | 20260506 18:21:25.455 | INFO | "huge" : "", | 20260506 18:21:25.455 | INFO | "huge_shared" : " 27 GB", | 20260506 18:21:25.455 | INFO | "malloc" : " 14 GB", | 20260506 18:21:25.455 | INFO | "pinned" : " 29 GB" | 20260506 18:21:25.455 | INFO | } | 20260506 18:21:25.455 | INFO | Perf report for the last 5.13614 seconds: | 20260506 18:21:25.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:21:25.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:25.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:25.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:25.455 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 18:21:25.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:25.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:25.455 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:21:25.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:25.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:25.455 | INFO | Uploaded 36GiB to GPU in 2.00709s (17.9364GiB/s) | 20260506 18:21:25.455 | INFO | Downloaded 34GiB to GPU in 1.9355s (17.5665GiB/s) | 20260506 18:21:30.590 | INFO | Analyzing chunk frames = [3466240,3466752) | 20260506 18:21:30.590 | INFO | MemInfo: { | 20260506 18:21:30.590 | INFO | "gpu" : " 17 GB", | 20260506 18:21:30.590 | INFO | "huge" : "", | 20260506 18:21:30.590 | INFO | "huge_shared" : " 27 GB", | 20260506 18:21:30.590 | INFO | "malloc" : " 14 GB", | 20260506 18:21:30.590 | INFO | "pinned" : " 29 GB" | 20260506 18:21:30.590 | INFO | } | 20260506 18:21:30.590 | INFO | Perf report for the last 5.13585 seconds: | 20260506 18:21:30.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:21:30.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:30.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:30.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:30.591 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:21:30.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:30.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:30.591 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:21:30.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:30.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:30.591 | INFO | Uploaded 36GiB to GPU in 1.96638s (18.3077GiB/s) | 20260506 18:21:30.591 | INFO | Downloaded 34GiB to GPU in 1.92224s (17.6877GiB/s) | 20260506 18:21:35.735 | INFO | Analyzing chunk frames = [3466752,3467264) | 20260506 18:21:35.735 | INFO | MemInfo: { | 20260506 18:21:35.735 | INFO | "gpu" : " 17 GB", | 20260506 18:21:35.735 | INFO | "huge" : "", | 20260506 18:21:35.735 | INFO | "huge_shared" : " 27 GB", | 20260506 18:21:35.735 | INFO | "malloc" : " 14 GB", | 20260506 18:21:35.735 | INFO | "pinned" : " 29 GB" | 20260506 18:21:35.735 | INFO | } | 20260506 18:21:35.735 | INFO | Perf report for the last 5.14426 seconds: | 20260506 18:21:35.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:21:35.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:35.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:35.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:35.735 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260506 18:21:35.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:35.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:35.735 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:21:35.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:35.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:35.735 | INFO | Uploaded 36GiB to GPU in 1.92437s (18.7074GiB/s) | 20260506 18:21:35.735 | INFO | Downloaded 34GiB to GPU in 1.87692s (18.1148GiB/s) | 20260506 18:21:40.880 | INFO | Analyzing chunk frames = [3467264,3467776) | 20260506 18:21:40.880 | INFO | MemInfo: { | 20260506 18:21:40.880 | INFO | "gpu" : " 17 GB", | 20260506 18:21:40.880 | INFO | "huge" : "", | 20260506 18:21:40.880 | INFO | "huge_shared" : " 27 GB", | 20260506 18:21:40.880 | INFO | "malloc" : " 14 GB", | 20260506 18:21:40.880 | INFO | "pinned" : " 29 GB" | 20260506 18:21:40.880 | INFO | } | 20260506 18:21:40.880 | INFO | Perf report for the last 5.14528 seconds: | 20260506 18:21:40.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:21:40.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:40.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:40.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:40.880 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260506 18:21:40.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:40.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:40.880 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:21:40.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:40.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:40.880 | INFO | Uploaded 36GiB to GPU in 1.95449s (18.4191GiB/s) | 20260506 18:21:40.880 | INFO | Downloaded 34GiB to GPU in 1.92729s (17.6413GiB/s) | 20260506 18:21:46.017 | INFO | Analyzing chunk frames = [3467776,3468288) | 20260506 18:21:46.017 | INFO | MemInfo: { | 20260506 18:21:46.017 | INFO | "gpu" : " 17 GB", | 20260506 18:21:46.017 | INFO | "huge" : "", | 20260506 18:21:46.017 | INFO | "huge_shared" : " 27 GB", | 20260506 18:21:46.017 | INFO | "malloc" : " 14 GB", | 20260506 18:21:46.017 | INFO | "pinned" : " 29 GB" | 20260506 18:21:46.017 | INFO | } | 20260506 18:21:46.017 | INFO | Perf report for the last 5.13706 seconds: | 20260506 18:21:46.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:21:46.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:46.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:46.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:46.017 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 18:21:46.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:46.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:46.017 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:21:46.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:46.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:46.017 | INFO | Uploaded 36GiB to GPU in 2.00166s (17.9851GiB/s) | 20260506 18:21:46.017 | INFO | Downloaded 34GiB to GPU in 1.93847s (17.5396GiB/s) | 20260506 18:21:51.155 | INFO | Analyzing chunk frames = [3468288,3468800) | 20260506 18:21:51.155 | INFO | MemInfo: { | 20260506 18:21:51.155 | INFO | "gpu" : " 17 GB", | 20260506 18:21:51.155 | INFO | "huge" : "", | 20260506 18:21:51.155 | INFO | "huge_shared" : " 27 GB", | 20260506 18:21:51.155 | INFO | "malloc" : " 14 GB", | 20260506 18:21:51.155 | INFO | "pinned" : " 29 GB" | 20260506 18:21:51.155 | INFO | } | 20260506 18:21:51.155 | INFO | Perf report for the last 5.13774 seconds: | 20260506 18:21:51.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:21:51.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:51.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:51.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:51.155 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 18:21:51.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:51.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:51.155 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:21:51.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:51.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:51.155 | INFO | Uploaded 36GiB to GPU in 2.01312s (17.8827GiB/s) | 20260506 18:21:51.155 | INFO | Downloaded 34GiB to GPU in 1.98848s (17.0985GiB/s) | 20260506 18:21:56.290 | INFO | Analyzing chunk frames = [3468800,3469312) | 20260506 18:21:56.290 | INFO | MemInfo: { | 20260506 18:21:56.290 | INFO | "gpu" : " 17 GB", | 20260506 18:21:56.290 | INFO | "huge" : "", | 20260506 18:21:56.290 | INFO | "huge_shared" : " 27 GB", | 20260506 18:21:56.290 | INFO | "malloc" : " 14 GB", | 20260506 18:21:56.290 | INFO | "pinned" : " 29 GB" | 20260506 18:21:56.290 | INFO | } | 20260506 18:21:56.290 | INFO | Perf report for the last 5.13505 seconds: | 20260506 18:21:56.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:21:56.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:56.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:56.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:56.290 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 18:21:56.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:56.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:56.290 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 18:21:56.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:56.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:21:56.290 | INFO | Uploaded 36GiB to GPU in 2.03804s (17.664GiB/s) | 20260506 18:21:56.290 | INFO | Downloaded 34GiB to GPU in 1.98451s (17.1327GiB/s) | 20260506 18:21:56.410 | INFO | Starting a round of Baz IO | 20260506 18:21:59.541 | INFO | Populating RTMetrics report | 20260506 18:21:59.542 | INFO | RTMetrics report populated | 20260506 18:21:59.542 | INFO | Opening tmp RTMetrics json file | 20260506 18:21:59.545 | INFO | Serializing RTMetrics to json | 20260506 18:21:59.550 | INFO | Serialization done, swapping tmp file now... | 20260506 18:21:59.556 | INFO | RTMetrics done | 20260506 18:21:59.724 | INFO | Finished a round of Baz IO: Wrote 2.042 GiB in 3.313 seconds | 20260506 18:21:59.724 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.12% / 0.01% / 64.87% | 20260506 18:21:59.724 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.13972% / 7.94729e-06% / 6.35783e-05% | 20260506 18:21:59.724 | INFO | Baz IO overall: | 20260506 18:21:59.724 | INFO | BytesWritten : 1832432029282 | 20260506 18:21:59.724 | INFO | NumEvents : 886210288551 | 20260506 18:22:01.427 | INFO | Analyzing chunk frames = [3469312,3469824) | 20260506 18:22:01.427 | INFO | MemInfo: { | 20260506 18:22:01.427 | INFO | "gpu" : " 17 GB", | 20260506 18:22:01.427 | INFO | "huge" : "", | 20260506 18:22:01.427 | INFO | "huge_shared" : " 27 GB", | 20260506 18:22:01.427 | INFO | "malloc" : " 14 GB", | 20260506 18:22:01.427 | INFO | "pinned" : " 29 GB" | 20260506 18:22:01.427 | INFO | } | 20260506 18:22:01.427 | INFO | Perf report for the last 5.13746 seconds: | 20260506 18:22:01.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:22:01.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:01.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:01.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:01.427 | INFO | ANALYSIS : 62.7%, 2.9, 95.2ms, 0 | 20260506 18:22:01.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:01.427 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 18:22:01.427 | INFO | PRE_HQ : 60.0%, 1.9, 58.2ms, 0 | 20260506 18:22:01.427 | INFO | BAZWRITER : 64.1%, 4.8, 157.0ms, 0 | 20260506 18:22:01.427 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.3ms, 0 | 20260506 18:22:01.428 | INFO | Uploaded 36GiB to GPU in 2.06997s (17.3916GiB/s) | 20260506 18:22:01.428 | INFO | Downloaded 38GiB to GPU in 2.22298s (17.0942GiB/s) | 20260506 18:22:06.566 | INFO | Analyzing chunk frames = [3469824,3470336) | 20260506 18:22:06.566 | INFO | MemInfo: { | 20260506 18:22:06.566 | INFO | "gpu" : " 17 GB", | 20260506 18:22:06.566 | INFO | "huge" : "", | 20260506 18:22:06.566 | INFO | "huge_shared" : " 27 GB", | 20260506 18:22:06.566 | INFO | "malloc" : " 14 GB", | 20260506 18:22:06.566 | INFO | "pinned" : " 29 GB" | 20260506 18:22:06.566 | INFO | } | 20260506 18:22:06.566 | INFO | Perf report for the last 5.13833 seconds: | 20260506 18:22:06.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:22:06.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:06.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:06.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:06.566 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:22:06.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:06.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:06.566 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:22:06.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:06.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:06.566 | INFO | Uploaded 36GiB to GPU in 2.01367s (17.8778GiB/s) | 20260506 18:22:06.566 | INFO | Downloaded 34GiB to GPU in 1.94051s (17.5212GiB/s) | 20260506 18:22:11.704 | INFO | Analyzing chunk frames = [3470336,3470848) | 20260506 18:22:11.704 | INFO | MemInfo: { | 20260506 18:22:11.704 | INFO | "gpu" : " 17 GB", | 20260506 18:22:11.704 | INFO | "huge" : "", | 20260506 18:22:11.704 | INFO | "huge_shared" : " 27 GB", | 20260506 18:22:11.704 | INFO | "malloc" : " 14 GB", | 20260506 18:22:11.704 | INFO | "pinned" : " 29 GB" | 20260506 18:22:11.704 | INFO | } | 20260506 18:22:11.704 | INFO | Perf report for the last 5.13796 seconds: | 20260506 18:22:11.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:22:11.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:11.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:11.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:11.704 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260506 18:22:11.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:11.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:11.704 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:22:11.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:11.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:11.704 | INFO | Uploaded 36GiB to GPU in 1.94456s (18.5132GiB/s) | 20260506 18:22:11.704 | INFO | Downloaded 34GiB to GPU in 1.87992s (18.0859GiB/s) | 20260506 18:22:16.855 | INFO | Analyzing chunk frames = [3470848,3471360) | 20260506 18:22:16.855 | INFO | MemInfo: { | 20260506 18:22:16.855 | INFO | "gpu" : " 17 GB", | 20260506 18:22:16.855 | INFO | "huge" : "", | 20260506 18:22:16.855 | INFO | "huge_shared" : " 27 GB", | 20260506 18:22:16.855 | INFO | "malloc" : " 14 GB", | 20260506 18:22:16.855 | INFO | "pinned" : " 29 GB" | 20260506 18:22:16.855 | INFO | } | 20260506 18:22:16.855 | INFO | Perf report for the last 5.15126 seconds: | 20260506 18:22:16.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:22:16.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:16.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:16.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:16.855 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:22:16.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:16.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:16.855 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:22:16.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:16.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:16.855 | INFO | Uploaded 36GiB to GPU in 1.9261s (18.6907GiB/s) | 20260506 18:22:16.855 | INFO | Downloaded 34GiB to GPU in 1.86579s (18.2229GiB/s) | 20260506 18:22:21.996 | INFO | Analyzing chunk frames = [3471360,3471872) | 20260506 18:22:21.996 | INFO | MemInfo: { | 20260506 18:22:21.996 | INFO | "gpu" : " 17 GB", | 20260506 18:22:21.996 | INFO | "huge" : "", | 20260506 18:22:21.996 | INFO | "huge_shared" : " 27 GB", | 20260506 18:22:21.996 | INFO | "malloc" : " 14 GB", | 20260506 18:22:21.996 | INFO | "pinned" : " 29 GB" | 20260506 18:22:21.996 | INFO | } | 20260506 18:22:21.996 | INFO | Perf report for the last 5.1414 seconds: | 20260506 18:22:21.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:22:21.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:21.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:21.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:21.996 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 18:22:21.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:21.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:21.996 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:22:21.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:21.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:21.997 | INFO | Uploaded 36GiB to GPU in 2.01539s (17.8626GiB/s) | 20260506 18:22:21.997 | INFO | Downloaded 34GiB to GPU in 1.91025s (17.7987GiB/s) | 20260506 18:22:27.135 | INFO | Analyzing chunk frames = [3471872,3472384) | 20260506 18:22:27.135 | INFO | MemInfo: { | 20260506 18:22:27.135 | INFO | "gpu" : " 17 GB", | 20260506 18:22:27.136 | INFO | "huge" : "", | 20260506 18:22:27.136 | INFO | "huge_shared" : " 27 GB", | 20260506 18:22:27.136 | INFO | "malloc" : " 14 GB", | 20260506 18:22:27.136 | INFO | "pinned" : " 29 GB" | 20260506 18:22:27.136 | INFO | } | 20260506 18:22:27.136 | INFO | Perf report for the last 5.13918 seconds: | 20260506 18:22:27.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:22:27.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:27.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:27.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:27.136 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 18:22:27.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:27.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:27.136 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 18:22:27.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:27.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:27.136 | INFO | Uploaded 36GiB to GPU in 2.05467s (17.5211GiB/s) | 20260506 18:22:27.136 | INFO | Downloaded 34GiB to GPU in 1.95817s (17.3632GiB/s) | 20260506 18:22:32.268 | INFO | Analyzing chunk frames = [3472384,3472896) | 20260506 18:22:32.268 | INFO | MemInfo: { | 20260506 18:22:32.268 | INFO | "gpu" : " 17 GB", | 20260506 18:22:32.268 | INFO | "huge" : "", | 20260506 18:22:32.268 | INFO | "huge_shared" : " 27 GB", | 20260506 18:22:32.269 | INFO | "malloc" : " 14 GB", | 20260506 18:22:32.269 | INFO | "pinned" : " 29 GB" | 20260506 18:22:32.269 | INFO | } | 20260506 18:22:32.269 | INFO | Perf report for the last 5.13298 seconds: | 20260506 18:22:32.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:22:32.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:32.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:32.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:32.269 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 18:22:32.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:32.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:32.269 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 18:22:32.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:32.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:32.269 | INFO | Uploaded 36GiB to GPU in 2.01718s (17.8467GiB/s) | 20260506 18:22:32.269 | INFO | Downloaded 34GiB to GPU in 1.98954s (17.0894GiB/s) | 20260506 18:22:37.402 | INFO | Analyzing chunk frames = [3472896,3473408) | 20260506 18:22:37.402 | INFO | MemInfo: { | 20260506 18:22:37.402 | INFO | "gpu" : " 17 GB", | 20260506 18:22:37.403 | INFO | "huge" : "", | 20260506 18:22:37.403 | INFO | "huge_shared" : " 27 GB", | 20260506 18:22:37.403 | INFO | "malloc" : " 14 GB", | 20260506 18:22:37.403 | INFO | "pinned" : " 29 GB" | 20260506 18:22:37.403 | INFO | } | 20260506 18:22:37.403 | INFO | Perf report for the last 5.13398 seconds: | 20260506 18:22:37.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:22:37.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:37.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:37.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:37.403 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:22:37.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:37.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:37.403 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 18:22:37.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:37.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:37.403 | INFO | Uploaded 36GiB to GPU in 2.00271s (17.9757GiB/s) | 20260506 18:22:37.403 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:22:37.403 | INFO | Downloaded 34GiB to GPU in 1.98294s (17.1462GiB/s) | 20260506 18:22:37.403 | INFO | Total mem usage: 18 GiB | 20260506 18:22:37.403 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:22:37.403 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -13.9%) | 20260506 18:22:37.403 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:22:37.403 | INFO | High Water Breakdown: | 20260506 18:22:37.403 | INFO | 61.6% : 9.74 GiB ( 85.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:22:37.403 | INFO | 38.3% : 6.05 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:22:37.403 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:22:37.403 | INFO | ------------------------------------------ | 20260506 18:22:37.403 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:22:37.403 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:22:37.403 | INFO | Total mem usage: 26.5 GiB | 20260506 18:22:37.403 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:22:37.403 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:22:37.403 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:22:37.404 | INFO | High Water Breakdown: | 20260506 18:22:37.404 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:22:37.404 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:22:37.404 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:22:37.404 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:22:37.404 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:22:37.404 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:22:37.404 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:22:37.404 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:22:37.404 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:22:37.404 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:22:37.404 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:22:37.404 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:22:37.404 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:22:37.404 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:22:37.404 | INFO | ------------------------------------------ | 20260506 18:22:37.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:22:37.404 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:22:37.404 | INFO | Total mem usage: 25.6 GiB | 20260506 18:22:37.404 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:22:37.404 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:22:37.404 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:22:37.404 | INFO | High Water Breakdown: | 20260506 18:22:37.404 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:22:37.404 | INFO | ------------------------------------------ | 20260506 18:22:37.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:22:37.404 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:22:37.404 | INFO | Total mem usage: 15.7 GiB | 20260506 18:22:37.404 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:22:37.404 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:22:37.404 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:22:37.404 | INFO | High Water Breakdown: | 20260506 18:22:37.404 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:22:37.404 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:22:37.404 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:22:37.404 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:22:37.404 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:22:37.404 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:22:37.404 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:22:37.404 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:22:37.404 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:22:37.404 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:22:37.404 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:22:37.404 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:22:37.404 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:22:37.404 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:22:37.404 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:22:37.404 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:22:37.404 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:22:37.404 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:22:37.404 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:22:37.404 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:22:37.404 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:22:37.404 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:22:37.404 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:22:37.404 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:22:37.404 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:22:37.404 | INFO | ------------------------------------------ | 20260506 18:22:37.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:22:37.404 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:22:37.404 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:22:37.404 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:22:37.404 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.58e+01, 1.34e+01 / 2.58e+00 | 20260506 18:22:37.404 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 9.99e-02, 1.00e-02 / 4.80e-03 | 20260506 18:22:37.404 | INFO | PerfReport: 11.60%, Baseline -- 3.63e+00 / 4.46e+00, 5.53e-01 / 3.19e-01 | 20260506 18:22:37.404 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.46e+00, 1.80e-01 / 1.27e-01 | 20260506 18:22:37.404 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.50e+01, 4.06e+00 / 2.62e+00 | 20260506 18:22:37.404 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.78e+01 / 2.00e+01, 1.43e+00 / 1.68e+00 | 20260506 18:22:37.404 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.26e+00 / 2.86e+00, 3.85e-01 / 1.78e-01 | 20260506 18:22:37.404 | INFO | PerfReport: 3.40%, Metrics -- 1.06e+00 / 1.44e+00, 2.80e-01 / 1.25e-01 | 20260506 18:22:37.404 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.67e-01 / 4.26e-01, 5.87e-02 / 2.37e-02 | 20260506 18:22:37.404 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:22:37.404 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:22:37.404 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:22:37.404 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 18:22:37.404 | INFO | PerfReport: 21.47%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 18:22:37.404 | INFO | PerfReport: 21.17%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 18:22:37.404 | INFO | PerfReport: 32.23%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:22:37.404 | INFO | PerfReport: 1.35%, LockUpload -- 1.31e+00 / 0.00e+00, 5.32e+00 / 0.00e+00 | 20260506 18:22:37.404 | INFO | PerfReport: 1.17%, LockDownload -- 1.14e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260506 18:22:37.404 | INFO | PerfReport: 22.60%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 18:22:37.404 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.40e-02 / 0.00e+00 | 20260506 18:22:37.517 | INFO | Starting a round of Baz IO | 20260506 18:22:40.651 | INFO | Populating RTMetrics report | 20260506 18:22:40.652 | INFO | RTMetrics report populated | 20260506 18:22:40.652 | INFO | Opening tmp RTMetrics json file | 20260506 18:22:40.768 | INFO | Serializing RTMetrics to json | 20260506 18:22:40.772 | INFO | Serialization done, swapping tmp file now... | 20260506 18:22:40.775 | INFO | RTMetrics done | 20260506 18:22:41.220 | INFO | Finished a round of Baz IO: Wrote 2.043 GiB in 3.702 seconds | 20260506 18:22:41.220 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.11% / 0.01% / 64.88% | 20260506 18:22:41.220 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.1396% / 3.97364e-06% / 5.96046e-05% | 20260506 18:22:41.220 | INFO | Baz IO overall: | 20260506 18:22:41.220 | INFO | BytesWritten : 1834625346491 | 20260506 18:22:41.220 | INFO | NumEvents : 887267705307 | 20260506 18:22:42.542 | INFO | Analyzing chunk frames = [3473408,3473920) | 20260506 18:22:42.542 | INFO | MemInfo: { | 20260506 18:22:42.542 | INFO | "gpu" : " 17 GB", | 20260506 18:22:42.542 | INFO | "huge" : "", | 20260506 18:22:42.542 | INFO | "huge_shared" : " 27 GB", | 20260506 18:22:42.542 | INFO | "malloc" : " 14 GB", | 20260506 18:22:42.542 | INFO | "pinned" : " 29 GB" | 20260506 18:22:42.542 | INFO | } | 20260506 18:22:42.542 | INFO | Perf report for the last 5.1393 seconds: | 20260506 18:22:42.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:22:42.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:42.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:42.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:42.542 | INFO | ANALYSIS : 62.6%, 2.9, 95.6ms, 0 | 20260506 18:22:42.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:42.542 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260506 18:22:42.542 | INFO | PRE_HQ : 60.1%, 1.9, 57.8ms, 0 | 20260506 18:22:42.542 | INFO | BAZWRITER : 71.7%, 4.8, 176.5ms, 0 | 20260506 18:22:42.542 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 18:22:42.542 | INFO | Uploaded 36GiB to GPU in 2.01192s (17.8934GiB/s) | 20260506 18:22:42.542 | INFO | Downloaded 38GiB to GPU in 2.20858s (17.2056GiB/s) | 20260506 18:22:47.684 | INFO | Analyzing chunk frames = [3473920,3474432) | 20260506 18:22:47.684 | INFO | MemInfo: { | 20260506 18:22:47.684 | INFO | "gpu" : " 17 GB", | 20260506 18:22:47.684 | INFO | "huge" : "", | 20260506 18:22:47.684 | INFO | "huge_shared" : " 27 GB", | 20260506 18:22:47.684 | INFO | "malloc" : " 14 GB", | 20260506 18:22:47.684 | INFO | "pinned" : " 29 GB" | 20260506 18:22:47.684 | INFO | } | 20260506 18:22:47.685 | INFO | Perf report for the last 5.14261 seconds: | 20260506 18:22:47.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:22:47.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:47.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:47.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:47.685 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 18:22:47.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:47.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:47.685 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:22:47.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:47.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:47.685 | INFO | Uploaded 36GiB to GPU in 1.98866s (18.1027GiB/s) | 20260506 18:22:47.685 | INFO | Downloaded 34GiB to GPU in 1.92681s (17.6457GiB/s) | 20260506 18:22:52.825 | INFO | Analyzing chunk frames = [3474432,3474944) | 20260506 18:22:52.825 | INFO | MemInfo: { | 20260506 18:22:52.825 | INFO | "gpu" : " 17 GB", | 20260506 18:22:52.826 | INFO | "huge" : "", | 20260506 18:22:52.826 | INFO | "huge_shared" : " 27 GB", | 20260506 18:22:52.826 | INFO | "malloc" : " 14 GB", | 20260506 18:22:52.826 | INFO | "pinned" : " 29 GB" | 20260506 18:22:52.826 | INFO | } | 20260506 18:22:52.826 | INFO | Perf report for the last 5.14103 seconds: | 20260506 18:22:52.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:22:52.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:52.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:52.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:52.826 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:22:52.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:52.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:52.826 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:22:52.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:52.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:52.826 | INFO | Uploaded 36GiB to GPU in 1.96261s (18.3429GiB/s) | 20260506 18:22:52.826 | INFO | Downloaded 34GiB to GPU in 1.92229s (17.6872GiB/s) | 20260506 18:22:57.965 | INFO | Analyzing chunk frames = [3474944,3475456) | 20260506 18:22:57.965 | INFO | MemInfo: { | 20260506 18:22:57.965 | INFO | "gpu" : " 17 GB", | 20260506 18:22:57.965 | INFO | "huge" : "", | 20260506 18:22:57.965 | INFO | "huge_shared" : " 27 GB", | 20260506 18:22:57.965 | INFO | "malloc" : " 14 GB", | 20260506 18:22:57.965 | INFO | "pinned" : " 29 GB" | 20260506 18:22:57.965 | INFO | } | 20260506 18:22:57.965 | INFO | Perf report for the last 5.13988 seconds: | 20260506 18:22:57.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:22:57.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:57.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:57.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:57.966 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 18:22:57.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:57.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:57.966 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 18:22:57.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:57.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:22:57.966 | INFO | Uploaded 36GiB to GPU in 1.90928s (18.8552GiB/s) | 20260506 18:22:57.966 | INFO | Downloaded 34GiB to GPU in 1.8715s (18.1672GiB/s) | 20260506 18:23:03.103 | INFO | Analyzing chunk frames = [3475456,3475968) | 20260506 18:23:03.103 | INFO | MemInfo: { | 20260506 18:23:03.103 | INFO | "gpu" : " 17 GB", | 20260506 18:23:03.103 | INFO | "huge" : "", | 20260506 18:23:03.103 | INFO | "huge_shared" : " 27 GB", | 20260506 18:23:03.103 | INFO | "malloc" : " 14 GB", | 20260506 18:23:03.103 | INFO | "pinned" : " 29 GB" | 20260506 18:23:03.103 | INFO | } | 20260506 18:23:03.103 | INFO | Perf report for the last 5.13741 seconds: | 20260506 18:23:03.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:23:03.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:03.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:03.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:03.103 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 18:23:03.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:03.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:03.103 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:23:03.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:03.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:03.103 | INFO | Uploaded 36GiB to GPU in 2.01501s (17.8659GiB/s) | 20260506 18:23:03.103 | INFO | Downloaded 34GiB to GPU in 1.91665s (17.7393GiB/s) | 20260506 18:23:08.241 | INFO | Analyzing chunk frames = [3475968,3476480) | 20260506 18:23:08.241 | INFO | MemInfo: { | 20260506 18:23:08.241 | INFO | "gpu" : " 17 GB", | 20260506 18:23:08.241 | INFO | "huge" : "", | 20260506 18:23:08.241 | INFO | "huge_shared" : " 27 GB", | 20260506 18:23:08.241 | INFO | "malloc" : " 14 GB", | 20260506 18:23:08.241 | INFO | "pinned" : " 29 GB" | 20260506 18:23:08.241 | INFO | } | 20260506 18:23:08.241 | INFO | Perf report for the last 5.13817 seconds: | 20260506 18:23:08.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:23:08.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:08.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:08.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:08.241 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 18:23:08.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:08.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:08.241 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 18:23:08.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:08.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:08.241 | INFO | Uploaded 36GiB to GPU in 2.0318s (17.7183GiB/s) | 20260506 18:23:08.241 | INFO | Downloaded 34GiB to GPU in 1.96445s (17.3077GiB/s) | 20260506 18:23:13.380 | INFO | Analyzing chunk frames = [3476480,3476992) | 20260506 18:23:13.380 | INFO | MemInfo: { | 20260506 18:23:13.380 | INFO | "gpu" : " 17 GB", | 20260506 18:23:13.380 | INFO | "huge" : "", | 20260506 18:23:13.380 | INFO | "huge_shared" : " 27 GB", | 20260506 18:23:13.380 | INFO | "malloc" : " 14 GB", | 20260506 18:23:13.380 | INFO | "pinned" : " 29 GB" | 20260506 18:23:13.380 | INFO | } | 20260506 18:23:13.380 | INFO | Perf report for the last 5.13898 seconds: | 20260506 18:23:13.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:23:13.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:13.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:13.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:13.380 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 18:23:13.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:13.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:13.380 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:23:13.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:13.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:13.380 | INFO | Uploaded 36GiB to GPU in 2.02261s (17.7988GiB/s) | 20260506 18:23:13.380 | INFO | Downloaded 34GiB to GPU in 2.00062s (16.9948GiB/s) | 20260506 18:23:18.520 | INFO | Analyzing chunk frames = [3476992,3477504) | 20260506 18:23:18.520 | INFO | MemInfo: { | 20260506 18:23:18.520 | INFO | "gpu" : " 17 GB", | 20260506 18:23:18.520 | INFO | "huge" : "", | 20260506 18:23:18.520 | INFO | "huge_shared" : " 27 GB", | 20260506 18:23:18.521 | INFO | "malloc" : " 14 GB", | 20260506 18:23:18.521 | INFO | "pinned" : " 29 GB" | 20260506 18:23:18.521 | INFO | } | 20260506 18:23:18.521 | INFO | Perf report for the last 5.14058 seconds: | 20260506 18:23:18.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:23:18.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:18.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:18.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:18.521 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 18:23:18.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:18.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:18.521 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:23:18.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:18.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:18.521 | INFO | Uploaded 36GiB to GPU in 2.03816s (17.663GiB/s) | 20260506 18:23:18.521 | INFO | Downloaded 34GiB to GPU in 1.98877s (17.096GiB/s) | 20260506 18:23:18.636 | INFO | Starting a round of Baz IO | 20260506 18:23:21.767 | INFO | Populating RTMetrics report | 20260506 18:23:21.767 | INFO | RTMetrics report populated | 20260506 18:23:21.767 | INFO | Opening tmp RTMetrics json file | 20260506 18:23:21.791 | INFO | Serializing RTMetrics to json | 20260506 18:23:21.796 | INFO | Serialization done, swapping tmp file now... | 20260506 18:23:21.799 | INFO | RTMetrics done | 20260506 18:23:22.029 | INFO | Finished a round of Baz IO: Wrote 2.042 GiB in 3.392 seconds | 20260506 18:23:22.029 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.11% / 0.01% / 64.89% | 20260506 18:23:22.029 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.13777% / 0% / 5.16574e-05% | 20260506 18:23:22.029 | INFO | Baz IO overall: | 20260506 18:23:22.029 | INFO | BytesWritten : 1836818074548 | 20260506 18:23:22.029 | INFO | NumEvents : 888325046680 | 20260506 18:23:23.657 | INFO | Analyzing chunk frames = [3477504,3478016) | 20260506 18:23:23.657 | INFO | MemInfo: { | 20260506 18:23:23.657 | INFO | "gpu" : " 17 GB", | 20260506 18:23:23.657 | INFO | "huge" : "", | 20260506 18:23:23.657 | INFO | "huge_shared" : " 27 GB", | 20260506 18:23:23.657 | INFO | "malloc" : " 14 GB", | 20260506 18:23:23.657 | INFO | "pinned" : " 29 GB" | 20260506 18:23:23.657 | INFO | } | 20260506 18:23:23.657 | INFO | Perf report for the last 5.13657 seconds: | 20260506 18:23:23.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:23:23.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:23.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:23.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:23.657 | INFO | ANALYSIS : 62.6%, 2.9, 95.1ms, 0 | 20260506 18:23:23.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:23.657 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 18:23:23.657 | INFO | PRE_HQ : 60.2%, 1.9, 58.5ms, 0 | 20260506 18:23:23.657 | INFO | BAZWRITER : 65.6%, 4.8, 160.4ms, 0 | 20260506 18:23:23.657 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 18:23:23.657 | INFO | Uploaded 36GiB to GPU in 2.07544s (17.3457GiB/s) | 20260506 18:23:23.657 | INFO | Downloaded 38GiB to GPU in 2.21414s (17.1624GiB/s) | 20260506 18:23:28.797 | INFO | Analyzing chunk frames = [3478016,3478528) | 20260506 18:23:28.797 | INFO | MemInfo: { | 20260506 18:23:28.797 | INFO | "gpu" : " 17 GB", | 20260506 18:23:28.797 | INFO | "huge" : "", | 20260506 18:23:28.797 | INFO | "huge_shared" : " 27 GB", | 20260506 18:23:28.797 | INFO | "malloc" : " 14 GB", | 20260506 18:23:28.797 | INFO | "pinned" : " 29 GB" | 20260506 18:23:28.797 | INFO | } | 20260506 18:23:28.797 | INFO | Perf report for the last 5.13992 seconds: | 20260506 18:23:28.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:23:28.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:28.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:28.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:28.797 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:23:28.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:28.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:28.797 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:23:28.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:28.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:28.797 | INFO | Uploaded 36GiB to GPU in 2.01656s (17.8522GiB/s) | 20260506 18:23:28.797 | INFO | Downloaded 34GiB to GPU in 1.94027s (17.5233GiB/s) | 20260506 18:23:33.947 | INFO | Analyzing chunk frames = [3478528,3479040) | 20260506 18:23:33.947 | INFO | MemInfo: { | 20260506 18:23:33.947 | INFO | "gpu" : " 17 GB", | 20260506 18:23:33.947 | INFO | "huge" : "", | 20260506 18:23:33.947 | INFO | "huge_shared" : " 27 GB", | 20260506 18:23:33.947 | INFO | "malloc" : " 14 GB", | 20260506 18:23:33.947 | INFO | "pinned" : " 29 GB" | 20260506 18:23:33.947 | INFO | } | 20260506 18:23:33.947 | INFO | Perf report for the last 5.14999 seconds: | 20260506 18:23:33.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:23:33.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:33.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:33.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:33.947 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260506 18:23:33.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:33.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:33.947 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:23:33.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:33.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:33.947 | INFO | Uploaded 36GiB to GPU in 1.99937s (18.0056GiB/s) | 20260506 18:23:33.947 | INFO | Downloaded 34GiB to GPU in 1.92086s (17.7004GiB/s) | 20260506 18:23:39.083 | INFO | Analyzing chunk frames = [3479040,3479552) | 20260506 18:23:39.083 | INFO | MemInfo: { | 20260506 18:23:39.083 | INFO | "gpu" : " 17 GB", | 20260506 18:23:39.083 | INFO | "huge" : "", | 20260506 18:23:39.083 | INFO | "huge_shared" : " 27 GB", | 20260506 18:23:39.083 | INFO | "malloc" : " 14 GB", | 20260506 18:23:39.083 | INFO | "pinned" : " 29 GB" | 20260506 18:23:39.083 | INFO | } | 20260506 18:23:39.084 | INFO | Perf report for the last 5.13632 seconds: | 20260506 18:23:39.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:23:39.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:39.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:39.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:39.084 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:23:39.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:39.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:39.084 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:23:39.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:39.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:39.084 | INFO | Uploaded 36GiB to GPU in 1.95189s (18.4436GiB/s) | 20260506 18:23:39.084 | INFO | Downloaded 34GiB to GPU in 1.86908s (18.1908GiB/s) | 20260506 18:23:44.221 | INFO | Analyzing chunk frames = [3479552,3480064) | 20260506 18:23:44.221 | INFO | MemInfo: { | 20260506 18:23:44.221 | INFO | "gpu" : " 17 GB", | 20260506 18:23:44.221 | INFO | "huge" : "", | 20260506 18:23:44.221 | INFO | "huge_shared" : " 27 GB", | 20260506 18:23:44.221 | INFO | "malloc" : " 14 GB", | 20260506 18:23:44.221 | INFO | "pinned" : " 29 GB" | 20260506 18:23:44.221 | INFO | } | 20260506 18:23:44.221 | INFO | Perf report for the last 5.13781 seconds: | 20260506 18:23:44.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:23:44.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:44.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:44.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:44.221 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 18:23:44.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:44.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:44.221 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 18:23:44.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:44.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:44.222 | INFO | Uploaded 36GiB to GPU in 2.01218s (17.891GiB/s) | 20260506 18:23:44.222 | INFO | Downloaded 34GiB to GPU in 1.91487s (17.7558GiB/s) | 20260506 18:23:49.356 | INFO | Analyzing chunk frames = [3480064,3480576) | 20260506 18:23:49.356 | INFO | MemInfo: { | 20260506 18:23:49.356 | INFO | "gpu" : " 17 GB", | 20260506 18:23:49.356 | INFO | "huge" : "", | 20260506 18:23:49.356 | INFO | "huge_shared" : " 27 GB", | 20260506 18:23:49.356 | INFO | "malloc" : " 14 GB", | 20260506 18:23:49.356 | INFO | "pinned" : " 29 GB" | 20260506 18:23:49.356 | INFO | } | 20260506 18:23:49.356 | INFO | Perf report for the last 5.13491 seconds: | 20260506 18:23:49.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:23:49.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:49.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:49.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:49.356 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 18:23:49.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:49.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:49.356 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:23:49.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:49.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:49.356 | INFO | Uploaded 36GiB to GPU in 2.03878s (17.6576GiB/s) | 20260506 18:23:49.357 | INFO | Downloaded 34GiB to GPU in 1.94731s (17.46GiB/s) | 20260506 18:23:54.495 | INFO | Analyzing chunk frames = [3480576,3481088) | 20260506 18:23:54.495 | INFO | MemInfo: { | 20260506 18:23:54.495 | INFO | "gpu" : " 17 GB", | 20260506 18:23:54.495 | INFO | "huge" : "", | 20260506 18:23:54.495 | INFO | "huge_shared" : " 27 GB", | 20260506 18:23:54.495 | INFO | "malloc" : " 14 GB", | 20260506 18:23:54.495 | INFO | "pinned" : " 29 GB" | 20260506 18:23:54.495 | INFO | } | 20260506 18:23:54.495 | INFO | Perf report for the last 5.1385 seconds: | 20260506 18:23:54.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:23:54.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:54.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:54.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:54.495 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 18:23:54.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:54.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:54.495 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:23:54.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:54.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:54.495 | INFO | Uploaded 36GiB to GPU in 2.02925s (17.7405GiB/s) | 20260506 18:23:54.495 | INFO | Downloaded 34GiB to GPU in 2.00349s (16.9704GiB/s) | 20260506 18:23:59.637 | INFO | Analyzing chunk frames = [3481088,3481600) | 20260506 18:23:59.637 | INFO | MemInfo: { | 20260506 18:23:59.637 | INFO | "gpu" : " 17 GB", | 20260506 18:23:59.637 | INFO | "huge" : "", | 20260506 18:23:59.637 | INFO | "huge_shared" : " 27 GB", | 20260506 18:23:59.637 | INFO | "malloc" : " 14 GB", | 20260506 18:23:59.637 | INFO | "pinned" : " 29 GB" | 20260506 18:23:59.637 | INFO | } | 20260506 18:23:59.637 | INFO | Perf report for the last 5.14205 seconds: | 20260506 18:23:59.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:23:59.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:59.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:59.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:59.637 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:23:59.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:59.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:59.637 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 18:23:59.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:59.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:23:59.637 | INFO | Uploaded 36GiB to GPU in 1.99686s (18.0283GiB/s) | 20260506 18:23:59.637 | INFO | Downloaded 34GiB to GPU in 1.97162s (17.2447GiB/s) | 20260506 18:23:59.637 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:23:59.637 | INFO | Total mem usage: 18 GiB | 20260506 18:23:59.637 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:23:59.637 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -14.8%) | 20260506 18:23:59.637 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:23:59.637 | INFO | High Water Breakdown: | 20260506 18:23:59.637 | INFO | 61.4% : 9.63 GiB ( 84.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:23:59.637 | INFO | 38.4% : 6.03 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:23:59.637 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:23:59.637 | INFO | ------------------------------------------ | 20260506 18:23:59.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:23:59.637 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:23:59.637 | INFO | Total mem usage: 26.5 GiB | 20260506 18:23:59.638 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:23:59.638 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:23:59.638 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:23:59.638 | INFO | High Water Breakdown: | 20260506 18:23:59.638 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:23:59.638 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:23:59.638 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:23:59.638 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:23:59.638 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:23:59.638 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:23:59.638 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:23:59.638 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:23:59.638 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:23:59.638 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:23:59.638 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:23:59.638 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:23:59.638 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:23:59.638 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:23:59.638 | INFO | ------------------------------------------ | 20260506 18:23:59.638 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:23:59.638 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:23:59.638 | INFO | Total mem usage: 25.6 GiB | 20260506 18:23:59.638 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:23:59.638 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:23:59.638 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:23:59.638 | INFO | High Water Breakdown: | 20260506 18:23:59.638 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:23:59.638 | INFO | ------------------------------------------ | 20260506 18:23:59.638 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:23:59.638 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:23:59.638 | INFO | Total mem usage: 15.7 GiB | 20260506 18:23:59.638 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:23:59.638 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:23:59.638 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 18:23:59.638 | INFO | High Water Breakdown: | 20260506 18:23:59.638 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:23:59.638 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:23:59.638 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:23:59.638 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:23:59.638 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:23:59.638 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:23:59.638 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:23:59.638 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:23:59.638 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:23:59.638 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:23:59.638 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:23:59.638 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:23:59.638 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:23:59.638 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:23:59.638 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:23:59.638 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:23:59.638 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:23:59.638 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:23:59.638 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:23:59.638 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:23:59.638 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:23:59.638 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:23:59.638 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:23:59.638 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:23:59.638 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:23:59.638 | INFO | ------------------------------------------ | 20260506 18:23:59.638 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:23:59.638 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:23:59.638 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:23:59.638 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:23:59.638 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.57e+01, 1.35e+01 / 2.23e+00 | 20260506 18:23:59.638 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.01e-01, 9.89e-03 / 4.55e-03 | 20260506 18:23:59.638 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.43e+00, 5.58e-01 / 3.25e-01 | 20260506 18:23:59.638 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.84e-01 / 1.30e-01 | 20260506 18:23:59.638 | INFO | PerfReport: 15.96%, DME -- 3.99e+01 / 4.44e+01, 4.10e+00 / 2.52e+00 | 20260506 18:23:59.638 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.43e+00 / 1.98e+00 | 20260506 18:23:59.638 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.25e+00 / 2.79e+00, 3.75e-01 / 2.37e-01 | 20260506 18:23:59.638 | INFO | PerfReport: 3.41%, Metrics -- 1.07e+00 / 1.38e+00, 2.83e-01 / 1.47e-01 | 20260506 18:23:59.638 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.68e-01 / 4.16e-01, 5.75e-02 / 2.11e-02 | 20260506 18:23:59.638 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:23:59.638 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:23:59.638 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:23:59.638 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 18:23:59.638 | INFO | PerfReport: 21.51%, Upload -- 2.09e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260506 18:23:59.638 | INFO | PerfReport: 21.17%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 18:23:59.638 | INFO | PerfReport: 32.20%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:23:59.638 | INFO | PerfReport: 1.43%, LockUpload -- 1.39e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260506 18:23:59.638 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260506 18:23:59.638 | INFO | PerfReport: 22.58%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:23:59.638 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260506 18:23:59.746 | INFO | Starting a round of Baz IO | 20260506 18:24:02.884 | INFO | Populating RTMetrics report | 20260506 18:24:02.885 | INFO | RTMetrics report populated | 20260506 18:24:02.885 | INFO | Opening tmp RTMetrics json file | 20260506 18:24:02.998 | INFO | Serializing RTMetrics to json | 20260506 18:24:03.006 | INFO | Serialization done, swapping tmp file now... | 20260506 18:24:03.008 | INFO | RTMetrics done | 20260506 18:24:03.316 | INFO | Finished a round of Baz IO: Wrote 2.042 GiB in 3.57 seconds | 20260506 18:24:03.316 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.10% / 0.01% / 64.89% | 20260506 18:24:03.316 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.13343% / 2.38419e-05% / 4.76837e-05% | 20260506 18:24:03.316 | INFO | Baz IO overall: | 20260506 18:24:03.316 | INFO | BytesWritten : 1839010439571 | 20260506 18:24:03.316 | INFO | NumEvents : 889381917505 | 20260506 18:24:04.774 | INFO | Analyzing chunk frames = [3481600,3482112) | 20260506 18:24:04.774 | INFO | MemInfo: { | 20260506 18:24:04.774 | INFO | "gpu" : " 17 GB", | 20260506 18:24:04.774 | INFO | "huge" : "", | 20260506 18:24:04.774 | INFO | "huge_shared" : " 27 GB", | 20260506 18:24:04.774 | INFO | "malloc" : " 14 GB", | 20260506 18:24:04.774 | INFO | "pinned" : " 29 GB" | 20260506 18:24:04.774 | INFO | } | 20260506 18:24:04.774 | INFO | Perf report for the last 5.13675 seconds: | 20260506 18:24:04.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:24:04.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:04.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:04.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:04.774 | INFO | ANALYSIS : 62.7%, 2.9, 95.1ms, 0 | 20260506 18:24:04.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:04.774 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 18:24:04.774 | INFO | PRE_HQ : 60.4%, 1.9, 58.4ms, 0 | 20260506 18:24:04.774 | INFO | BAZWRITER : 69.2%, 4.8, 170.7ms, 0 | 20260506 18:24:04.774 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 18:24:04.774 | INFO | Uploaded 36GiB to GPU in 2.03704s (17.6727GiB/s) | 20260506 18:24:04.774 | INFO | Downloaded 38GiB to GPU in 2.20161s (17.2601GiB/s) | 20260506 18:24:09.916 | INFO | Analyzing chunk frames = [3482112,3482624) | 20260506 18:24:09.916 | INFO | MemInfo: { | 20260506 18:24:09.916 | INFO | "gpu" : " 17 GB", | 20260506 18:24:09.916 | INFO | "huge" : "", | 20260506 18:24:09.916 | INFO | "huge_shared" : " 27 GB", | 20260506 18:24:09.916 | INFO | "malloc" : " 14 GB", | 20260506 18:24:09.916 | INFO | "pinned" : " 29 GB" | 20260506 18:24:09.916 | INFO | } | 20260506 18:24:09.916 | INFO | Perf report for the last 5.14226 seconds: | 20260506 18:24:09.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:24:09.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:09.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:09.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:09.916 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:24:09.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:09.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:09.916 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:24:09.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:09.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:09.916 | INFO | Uploaded 36GiB to GPU in 1.99696s (18.0274GiB/s) | 20260506 18:24:09.917 | INFO | Downloaded 34GiB to GPU in 1.92857s (17.6297GiB/s) | 20260506 18:24:15.052 | INFO | Analyzing chunk frames = [3482624,3483136) | 20260506 18:24:15.052 | INFO | MemInfo: { | 20260506 18:24:15.052 | INFO | "gpu" : " 17 GB", | 20260506 18:24:15.052 | INFO | "huge" : "", | 20260506 18:24:15.052 | INFO | "huge_shared" : " 27 GB", | 20260506 18:24:15.053 | INFO | "malloc" : " 14 GB", | 20260506 18:24:15.053 | INFO | "pinned" : " 29 GB" | 20260506 18:24:15.053 | INFO | } | 20260506 18:24:15.053 | INFO | Perf report for the last 5.13665 seconds: | 20260506 18:24:15.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:24:15.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:15.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:15.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:15.053 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 18:24:15.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:15.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:15.053 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:24:15.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:15.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:15.053 | INFO | Uploaded 36GiB to GPU in 1.96852s (18.2878GiB/s) | 20260506 18:24:15.053 | INFO | Downloaded 34GiB to GPU in 1.91661s (17.7396GiB/s) | 20260506 18:24:20.195 | INFO | Analyzing chunk frames = [3483136,3483648) | 20260506 18:24:20.195 | INFO | MemInfo: { | 20260506 18:24:20.195 | INFO | "gpu" : " 17 GB", | 20260506 18:24:20.195 | INFO | "huge" : "", | 20260506 18:24:20.195 | INFO | "huge_shared" : " 27 GB", | 20260506 18:24:20.195 | INFO | "malloc" : " 14 GB", | 20260506 18:24:20.195 | INFO | "pinned" : " 29 GB" | 20260506 18:24:20.195 | INFO | } | 20260506 18:24:20.195 | INFO | Perf report for the last 5.14246 seconds: | 20260506 18:24:20.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:24:20.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:20.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:20.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:20.195 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 18:24:20.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:20.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:20.195 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:24:20.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:20.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:20.195 | INFO | Uploaded 36GiB to GPU in 1.934s (18.6143GiB/s) | 20260506 18:24:20.195 | INFO | Downloaded 34GiB to GPU in 1.86848s (18.1966GiB/s) | 20260506 18:24:25.332 | INFO | Analyzing chunk frames = [3483648,3484160) | 20260506 18:24:25.332 | INFO | MemInfo: { | 20260506 18:24:25.332 | INFO | "gpu" : " 17 GB", | 20260506 18:24:25.332 | INFO | "huge" : "", | 20260506 18:24:25.332 | INFO | "huge_shared" : " 27 GB", | 20260506 18:24:25.332 | INFO | "malloc" : " 14 GB", | 20260506 18:24:25.332 | INFO | "pinned" : " 29 GB" | 20260506 18:24:25.332 | INFO | } | 20260506 18:24:25.332 | INFO | Perf report for the last 5.13734 seconds: | 20260506 18:24:25.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:24:25.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:25.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:25.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:25.332 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 18:24:25.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:25.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:25.332 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 18:24:25.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:25.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:25.333 | INFO | Uploaded 36GiB to GPU in 2.01032s (17.9076GiB/s) | 20260506 18:24:25.333 | INFO | Downloaded 34GiB to GPU in 1.9156s (17.7491GiB/s) | 20260506 18:24:30.469 | INFO | Analyzing chunk frames = [3484160,3484672) | 20260506 18:24:30.469 | INFO | MemInfo: { | 20260506 18:24:30.469 | INFO | "gpu" : " 17 GB", | 20260506 18:24:30.469 | INFO | "huge" : "", | 20260506 18:24:30.469 | INFO | "huge_shared" : " 27 GB", | 20260506 18:24:30.469 | INFO | "malloc" : " 14 GB", | 20260506 18:24:30.469 | INFO | "pinned" : " 29 GB" | 20260506 18:24:30.469 | INFO | } | 20260506 18:24:30.469 | INFO | Perf report for the last 5.13693 seconds: | 20260506 18:24:30.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:24:30.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:30.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:30.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:30.469 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 18:24:30.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:30.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:30.469 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 18:24:30.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:30.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:30.470 | INFO | Uploaded 36GiB to GPU in 2.04312s (17.6201GiB/s) | 20260506 18:24:30.470 | INFO | Downloaded 34GiB to GPU in 1.94652s (17.467GiB/s) | 20260506 18:24:35.610 | INFO | Analyzing chunk frames = [3484672,3485184) | 20260506 18:24:35.610 | INFO | MemInfo: { | 20260506 18:24:35.610 | INFO | "gpu" : " 17 GB", | 20260506 18:24:35.610 | INFO | "huge" : "", | 20260506 18:24:35.610 | INFO | "huge_shared" : " 27 GB", | 20260506 18:24:35.610 | INFO | "malloc" : " 14 GB", | 20260506 18:24:35.610 | INFO | "pinned" : " 29 GB" | 20260506 18:24:35.610 | INFO | } | 20260506 18:24:35.610 | INFO | Perf report for the last 5.14065 seconds: | 20260506 18:24:35.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:24:35.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:35.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:35.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:35.610 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:24:35.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:35.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:35.610 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:24:35.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:35.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:35.610 | INFO | Uploaded 36GiB to GPU in 1.99801s (18.0179GiB/s) | 20260506 18:24:35.610 | INFO | Downloaded 34GiB to GPU in 1.953s (17.4091GiB/s) | 20260506 18:24:40.748 | INFO | Analyzing chunk frames = [3485184,3485696) | 20260506 18:24:40.748 | INFO | MemInfo: { | 20260506 18:24:40.748 | INFO | "gpu" : " 17 GB", | 20260506 18:24:40.748 | INFO | "huge" : "", | 20260506 18:24:40.748 | INFO | "huge_shared" : " 27 GB", | 20260506 18:24:40.748 | INFO | "malloc" : " 14 GB", | 20260506 18:24:40.748 | INFO | "pinned" : " 29 GB" | 20260506 18:24:40.748 | INFO | } | 20260506 18:24:40.748 | INFO | Perf report for the last 5.138 seconds: | 20260506 18:24:40.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:24:40.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:40.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:40.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:40.748 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 18:24:40.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:40.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:40.748 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 18:24:40.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:40.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:40.748 | INFO | Uploaded 36GiB to GPU in 1.98687s (18.119GiB/s) | 20260506 18:24:40.748 | INFO | Downloaded 34GiB to GPU in 1.97923s (17.1784GiB/s) | 20260506 18:24:40.863 | INFO | Starting a round of Baz IO | 20260506 18:24:43.998 | INFO | Populating RTMetrics report | 20260506 18:24:43.999 | INFO | RTMetrics report populated | 20260506 18:24:43.999 | INFO | Opening tmp RTMetrics json file | 20260506 18:24:44.069 | INFO | Serializing RTMetrics to json | 20260506 18:24:44.071 | INFO | Serialization done, swapping tmp file now... | 20260506 18:24:44.073 | INFO | RTMetrics done | 20260506 18:24:44.481 | INFO | Finished a round of Baz IO: Wrote 2.043 GiB in 3.618 seconds | 20260506 18:24:44.481 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.10% / 0.01% / 64.90% | 20260506 18:24:44.481 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.13241% / 3.97364e-06% / 6.35783e-05% | 20260506 18:24:44.481 | INFO | Baz IO overall: | 20260506 18:24:44.481 | INFO | BytesWritten : 1841203759484 | 20260506 18:24:44.481 | INFO | NumEvents : 890439655734 | 20260506 18:24:45.891 | INFO | Analyzing chunk frames = [3485696,3486208) | 20260506 18:24:45.891 | INFO | MemInfo: { | 20260506 18:24:45.891 | INFO | "gpu" : " 17 GB", | 20260506 18:24:45.891 | INFO | "huge" : "", | 20260506 18:24:45.891 | INFO | "huge_shared" : " 27 GB", | 20260506 18:24:45.891 | INFO | "malloc" : " 14 GB", | 20260506 18:24:45.891 | INFO | "pinned" : " 29 GB" | 20260506 18:24:45.891 | INFO | } | 20260506 18:24:45.891 | INFO | Perf report for the last 5.14316 seconds: | 20260506 18:24:45.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:24:45.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:45.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:45.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:45.891 | INFO | ANALYSIS : 62.6%, 2.9, 95.3ms, 0 | 20260506 18:24:45.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:45.891 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260506 18:24:45.891 | INFO | PRE_HQ : 60.2%, 1.9, 58.2ms, 0 | 20260506 18:24:45.891 | INFO | BAZWRITER : 70.0%, 4.7, 170.2ms, 0 | 20260506 18:24:45.891 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 18:24:45.891 | INFO | Uploaded 36GiB to GPU in 2.04625s (17.5931GiB/s) | 20260506 18:24:45.892 | INFO | Downloaded 38GiB to GPU in 2.20794s (17.2106GiB/s) | 20260506 18:24:51.029 | INFO | Analyzing chunk frames = [3486208,3486720) | 20260506 18:24:51.029 | INFO | MemInfo: { | 20260506 18:24:51.029 | INFO | "gpu" : " 17 GB", | 20260506 18:24:51.029 | INFO | "huge" : "", | 20260506 18:24:51.029 | INFO | "huge_shared" : " 27 GB", | 20260506 18:24:51.029 | INFO | "malloc" : " 14 GB", | 20260506 18:24:51.029 | INFO | "pinned" : " 29 GB" | 20260506 18:24:51.029 | INFO | } | 20260506 18:24:51.029 | INFO | Perf report for the last 5.13743 seconds: | 20260506 18:24:51.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:24:51.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:51.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:51.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:51.029 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 18:24:51.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:51.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:51.029 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 18:24:51.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:51.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:51.029 | INFO | Uploaded 36GiB to GPU in 2.01721s (17.8464GiB/s) | 20260506 18:24:51.029 | INFO | Downloaded 34GiB to GPU in 1.94096s (17.5171GiB/s) | 20260506 18:24:56.169 | INFO | Analyzing chunk frames = [3486720,3487232) | 20260506 18:24:56.169 | INFO | MemInfo: { | 20260506 18:24:56.169 | INFO | "gpu" : " 17 GB", | 20260506 18:24:56.169 | INFO | "huge" : "", | 20260506 18:24:56.169 | INFO | "huge_shared" : " 27 GB", | 20260506 18:24:56.169 | INFO | "malloc" : " 14 GB", | 20260506 18:24:56.169 | INFO | "pinned" : " 29 GB" | 20260506 18:24:56.169 | INFO | } | 20260506 18:24:56.169 | INFO | Perf report for the last 5.14054 seconds: | 20260506 18:24:56.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:24:56.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:56.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:56.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:56.169 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 18:24:56.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:56.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:56.169 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:24:56.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:56.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:24:56.169 | INFO | Uploaded 36GiB to GPU in 1.98148s (18.1682GiB/s) | 20260506 18:24:56.169 | INFO | Downloaded 34GiB to GPU in 1.92307s (17.6801GiB/s) | 20260506 18:25:01.312 | INFO | Analyzing chunk frames = [3487232,3487744) | 20260506 18:25:01.312 | INFO | MemInfo: { | 20260506 18:25:01.312 | INFO | "gpu" : " 17 GB", | 20260506 18:25:01.312 | INFO | "huge" : "", | 20260506 18:25:01.312 | INFO | "huge_shared" : " 27 GB", | 20260506 18:25:01.312 | INFO | "malloc" : " 14 GB", | 20260506 18:25:01.312 | INFO | "pinned" : " 29 GB" | 20260506 18:25:01.312 | INFO | } | 20260506 18:25:01.312 | INFO | Perf report for the last 5.14327 seconds: | 20260506 18:25:01.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:25:01.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:01.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:01.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:01.313 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:25:01.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:01.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:01.313 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:25:01.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:01.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:01.313 | INFO | Uploaded 36GiB to GPU in 1.94201s (18.5375GiB/s) | 20260506 18:25:01.313 | INFO | Downloaded 34GiB to GPU in 1.87336s (18.1492GiB/s) | 20260506 18:25:06.446 | INFO | Analyzing chunk frames = [3487744,3488256) | 20260506 18:25:06.446 | INFO | MemInfo: { | 20260506 18:25:06.446 | INFO | "gpu" : " 17 GB", | 20260506 18:25:06.446 | INFO | "huge" : "", | 20260506 18:25:06.446 | INFO | "huge_shared" : " 27 GB", | 20260506 18:25:06.446 | INFO | "malloc" : " 14 GB", | 20260506 18:25:06.446 | INFO | "pinned" : " 29 GB" | 20260506 18:25:06.446 | INFO | } | 20260506 18:25:06.446 | INFO | Perf report for the last 5.1339 seconds: | 20260506 18:25:06.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:25:06.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:06.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:06.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:06.446 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260506 18:25:06.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:06.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:06.446 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:25:06.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:06.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:06.447 | INFO | Uploaded 36GiB to GPU in 2.01627s (17.8548GiB/s) | 20260506 18:25:06.447 | INFO | Downloaded 34GiB to GPU in 1.90435s (17.8539GiB/s) | 20260506 18:25:11.587 | INFO | Analyzing chunk frames = [3488256,3488768) | 20260506 18:25:11.587 | INFO | MemInfo: { | 20260506 18:25:11.587 | INFO | "gpu" : " 17 GB", | 20260506 18:25:11.587 | INFO | "huge" : "", | 20260506 18:25:11.587 | INFO | "huge_shared" : " 27 GB", | 20260506 18:25:11.587 | INFO | "malloc" : " 14 GB", | 20260506 18:25:11.587 | INFO | "pinned" : " 29 GB" | 20260506 18:25:11.587 | INFO | } | 20260506 18:25:11.587 | INFO | Perf report for the last 5.14031 seconds: | 20260506 18:25:11.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:25:11.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:11.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:11.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:11.587 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 18:25:11.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:11.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:11.587 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:25:11.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:11.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:11.587 | INFO | Uploaded 36GiB to GPU in 2.02832s (17.7486GiB/s) | 20260506 18:25:11.587 | INFO | Downloaded 34GiB to GPU in 1.94705s (17.4623GiB/s) | 20260506 18:25:16.726 | INFO | Analyzing chunk frames = [3488768,3489280) | 20260506 18:25:16.726 | INFO | MemInfo: { | 20260506 18:25:16.726 | INFO | "gpu" : " 17 GB", | 20260506 18:25:16.726 | INFO | "huge" : "", | 20260506 18:25:16.726 | INFO | "huge_shared" : " 27 GB", | 20260506 18:25:16.726 | INFO | "malloc" : " 14 GB", | 20260506 18:25:16.726 | INFO | "pinned" : " 29 GB" | 20260506 18:25:16.726 | INFO | } | 20260506 18:25:16.726 | INFO | Perf report for the last 5.13943 seconds: | 20260506 18:25:16.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:25:16.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:16.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:16.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:16.726 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:25:16.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:16.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:16.726 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:25:16.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:16.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:16.726 | INFO | Uploaded 36GiB to GPU in 1.98661s (18.1213GiB/s) | 20260506 18:25:16.726 | INFO | Downloaded 34GiB to GPU in 2.00568s (16.9518GiB/s) | 20260506 18:25:21.861 | INFO | Analyzing chunk frames = [3489280,3489792) | 20260506 18:25:21.861 | INFO | MemInfo: { | 20260506 18:25:21.861 | INFO | "gpu" : " 17 GB", | 20260506 18:25:21.861 | INFO | "huge" : "", | 20260506 18:25:21.861 | INFO | "huge_shared" : " 27 GB", | 20260506 18:25:21.861 | INFO | "malloc" : " 14 GB", | 20260506 18:25:21.861 | INFO | "pinned" : " 29 GB" | 20260506 18:25:21.861 | INFO | } | 20260506 18:25:21.861 | INFO | Perf report for the last 5.13523 seconds: | 20260506 18:25:21.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:25:21.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:21.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:21.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:21.861 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260506 18:25:21.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:21.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:21.861 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 18:25:21.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:21.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:21.862 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:25:21.862 | INFO | Total mem usage: 18 GiB | 20260506 18:25:21.862 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:25:21.862 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -10.1%) | 20260506 18:25:21.862 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:25:21.862 | INFO | High Water Breakdown: | 20260506 18:25:21.862 | INFO | 62.1% : 10.2 GiB ( 89% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:25:21.862 | INFO | 37.8% : 6.18 GiB ( 92.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:25:21.862 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:25:21.862 | INFO | ------------------------------------------ | 20260506 18:25:21.862 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:25:21.862 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:25:21.862 | INFO | Total mem usage: 26.5 GiB | 20260506 18:25:21.862 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:25:21.862 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:25:21.862 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:25:21.862 | INFO | High Water Breakdown: | 20260506 18:25:21.862 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:25:21.862 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:25:21.862 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:25:21.862 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:25:21.862 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:25:21.862 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:25:21.862 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:25:21.862 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:25:21.862 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:25:21.862 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:25:21.862 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:25:21.862 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:25:21.862 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:25:21.862 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:25:21.862 | INFO | ------------------------------------------ | 20260506 18:25:21.862 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:25:21.862 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:25:21.862 | INFO | Total mem usage: 25.6 GiB | 20260506 18:25:21.862 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:25:21.862 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:25:21.862 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:25:21.862 | INFO | High Water Breakdown: | 20260506 18:25:21.862 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:25:21.862 | INFO | ------------------------------------------ | 20260506 18:25:21.862 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:25:21.862 | INFO | Uploaded 36GiB to GPU in 2.0053s (17.9524GiB/s) | 20260506 18:25:21.862 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:25:21.862 | INFO | Total mem usage: 15.7 GiB | 20260506 18:25:21.862 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:25:21.862 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:25:21.862 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:25:21.862 | INFO | High Water Breakdown: | 20260506 18:25:21.862 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:25:21.862 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:25:21.862 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:25:21.862 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:25:21.862 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:25:21.862 | INFO | Downloaded 34GiB to GPU in 1.99232s (17.0655GiB/s) | 20260506 18:25:21.862 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:25:21.862 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:25:21.862 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:25:21.862 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:25:21.862 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:25:21.862 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:25:21.862 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:25:21.862 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:25:21.862 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:25:21.862 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:25:21.862 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:25:21.862 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:25:21.862 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:25:21.862 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:25:21.862 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:25:21.862 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:25:21.862 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:25:21.862 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:25:21.862 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:25:21.862 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:25:21.862 | INFO | ------------------------------------------ | 20260506 18:25:21.862 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:25:21.862 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:25:21.862 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:25:21.862 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:25:21.862 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.53e+01, 1.33e+01 / 2.28e+00 | 20260506 18:25:21.862 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.02e-01, 1.03e-02 / 7.01e-03 | 20260506 18:25:21.863 | INFO | PerfReport: 11.62%, Baseline -- 3.64e+00 / 4.44e+00, 5.58e-01 / 3.24e-01 | 20260506 18:25:21.863 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.45e+00, 1.84e-01 / 1.36e-01 | 20260506 18:25:21.863 | INFO | PerfReport: 15.94%, DME -- 3.99e+01 / 4.41e+01, 4.02e+00 / 2.36e+00 | 20260506 18:25:21.863 | INFO | PerfReport: 57.05%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.42e+00 / 1.78e+00 | 20260506 18:25:21.863 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.26e+00 / 2.76e+00, 3.83e-01 / 2.29e-01 | 20260506 18:25:21.863 | INFO | PerfReport: 3.42%, Metrics -- 1.07e+00 / 1.38e+00, 2.81e-01 / 1.78e-01 | 20260506 18:25:21.863 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.68e-01 / 4.11e-01, 5.76e-02 / 2.26e-02 | 20260506 18:25:21.863 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:25:21.863 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:25:21.863 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:25:21.863 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 18:25:21.863 | INFO | PerfReport: 21.46%, Upload -- 2.08e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 18:25:21.863 | INFO | PerfReport: 21.13%, Download -- 2.05e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 18:25:21.863 | INFO | PerfReport: 32.22%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 18:25:21.863 | INFO | PerfReport: 1.43%, LockUpload -- 1.39e+00 / 0.00e+00, 5.51e+00 / 0.00e+00 | 20260506 18:25:21.863 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260506 18:25:21.863 | INFO | PerfReport: 22.66%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:25:21.863 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260506 18:25:21.974 | INFO | Starting a round of Baz IO | 20260506 18:25:25.112 | INFO | Populating RTMetrics report | 20260506 18:25:25.112 | INFO | RTMetrics report populated | 20260506 18:25:25.112 | INFO | Opening tmp RTMetrics json file | 20260506 18:25:25.161 | INFO | Serializing RTMetrics to json | 20260506 18:25:25.167 | INFO | Serialization done, swapping tmp file now... | 20260506 18:25:25.169 | INFO | RTMetrics done | 20260506 18:25:25.521 | INFO | Finished a round of Baz IO: Wrote 2.041 GiB in 3.547 seconds | 20260506 18:25:25.521 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.09% / 0.01% / 64.90% | 20260506 18:25:25.521 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.12833% / 7.94729e-06% / 7.15256e-05% | 20260506 18:25:25.522 | INFO | Baz IO overall: | 20260506 18:25:25.522 | INFO | BytesWritten : 1843394838451 | 20260506 18:25:25.522 | INFO | NumEvents : 891495781513 | 20260506 18:25:27.006 | INFO | Analyzing chunk frames = [3489792,3490304) | 20260506 18:25:27.006 | INFO | MemInfo: { | 20260506 18:25:27.006 | INFO | "gpu" : " 17 GB", | 20260506 18:25:27.006 | INFO | "huge" : "", | 20260506 18:25:27.006 | INFO | "huge_shared" : " 27 GB", | 20260506 18:25:27.006 | INFO | "malloc" : " 14 GB", | 20260506 18:25:27.006 | INFO | "pinned" : " 29 GB" | 20260506 18:25:27.006 | INFO | } | 20260506 18:25:27.006 | INFO | Perf report for the last 5.14508 seconds: | 20260506 18:25:27.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:25:27.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:27.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:27.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:27.007 | INFO | ANALYSIS : 62.6%, 2.9, 95.4ms, 0 | 20260506 18:25:27.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:27.007 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 18:25:27.007 | INFO | PRE_HQ : 60.3%, 1.8, 57.7ms, 0 | 20260506 18:25:27.007 | INFO | BAZWRITER : 68.6%, 4.8, 169.1ms, 0 | 20260506 18:25:27.007 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 18:25:27.007 | INFO | Uploaded 36GiB to GPU in 2.05523s (17.5163GiB/s) | 20260506 18:25:27.007 | INFO | Downloaded 38GiB to GPU in 2.21926s (17.1228GiB/s) | 20260506 18:25:32.147 | INFO | Analyzing chunk frames = [3490304,3490816) | 20260506 18:25:32.147 | INFO | MemInfo: { | 20260506 18:25:32.147 | INFO | "gpu" : " 17 GB", | 20260506 18:25:32.147 | INFO | "huge" : "", | 20260506 18:25:32.147 | INFO | "huge_shared" : " 27 GB", | 20260506 18:25:32.147 | INFO | "malloc" : " 14 GB", | 20260506 18:25:32.147 | INFO | "pinned" : " 29 GB" | 20260506 18:25:32.147 | INFO | } | 20260506 18:25:32.147 | INFO | Perf report for the last 5.14045 seconds: | 20260506 18:25:32.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:25:32.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:32.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:32.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:32.147 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 18:25:32.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:32.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:32.147 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:25:32.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:32.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:32.147 | INFO | Uploaded 36GiB to GPU in 2.00414s (17.9628GiB/s) | 20260506 18:25:32.147 | INFO | Downloaded 34GiB to GPU in 1.93198s (17.5986GiB/s) | 20260506 18:25:37.283 | INFO | Analyzing chunk frames = [3490816,3491328) | 20260506 18:25:37.283 | INFO | MemInfo: { | 20260506 18:25:37.283 | INFO | "gpu" : " 17 GB", | 20260506 18:25:37.283 | INFO | "huge" : "", | 20260506 18:25:37.283 | INFO | "huge_shared" : " 27 GB", | 20260506 18:25:37.283 | INFO | "malloc" : " 14 GB", | 20260506 18:25:37.283 | INFO | "pinned" : " 29 GB" | 20260506 18:25:37.283 | INFO | } | 20260506 18:25:37.283 | INFO | Perf report for the last 5.13646 seconds: | 20260506 18:25:37.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:25:37.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:37.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:37.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:37.283 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 18:25:37.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:37.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:37.284 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 18:25:37.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:37.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:37.284 | INFO | Uploaded 36GiB to GPU in 1.96586s (18.3126GiB/s) | 20260506 18:25:37.284 | INFO | Downloaded 34GiB to GPU in 1.92183s (17.6915GiB/s) | 20260506 18:25:42.423 | INFO | Analyzing chunk frames = [3491328,3491840) | 20260506 18:25:42.423 | INFO | MemInfo: { | 20260506 18:25:42.423 | INFO | "gpu" : " 17 GB", | 20260506 18:25:42.423 | INFO | "huge" : "", | 20260506 18:25:42.423 | INFO | "huge_shared" : " 27 GB", | 20260506 18:25:42.423 | INFO | "malloc" : " 14 GB", | 20260506 18:25:42.423 | INFO | "pinned" : " 29 GB" | 20260506 18:25:42.423 | INFO | } | 20260506 18:25:42.423 | INFO | Perf report for the last 5.13974 seconds: | 20260506 18:25:42.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:25:42.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:42.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:42.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:42.423 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:25:42.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:42.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:42.423 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:25:42.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:42.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:42.423 | INFO | Uploaded 36GiB to GPU in 1.93225s (18.6311GiB/s) | 20260506 18:25:42.423 | INFO | Downloaded 34GiB to GPU in 1.87374s (18.1455GiB/s) | 20260506 18:25:47.563 | INFO | Analyzing chunk frames = [3491840,3492352) | 20260506 18:25:47.563 | INFO | MemInfo: { | 20260506 18:25:47.563 | INFO | "gpu" : " 17 GB", | 20260506 18:25:47.563 | INFO | "huge" : "", | 20260506 18:25:47.563 | INFO | "huge_shared" : " 27 GB", | 20260506 18:25:47.563 | INFO | "malloc" : " 14 GB", | 20260506 18:25:47.563 | INFO | "pinned" : " 29 GB" | 20260506 18:25:47.563 | INFO | } | 20260506 18:25:47.563 | INFO | Perf report for the last 5.14003 seconds: | 20260506 18:25:47.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:25:47.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:47.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:47.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:47.563 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 18:25:47.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:47.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:47.564 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:25:47.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:47.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:47.564 | INFO | Uploaded 36GiB to GPU in 2.01578s (17.8591GiB/s) | 20260506 18:25:47.564 | INFO | Downloaded 34GiB to GPU in 1.91616s (17.7438GiB/s) | 20260506 18:25:52.702 | INFO | Analyzing chunk frames = [3492352,3492864) | 20260506 18:25:52.702 | INFO | MemInfo: { | 20260506 18:25:52.702 | INFO | "gpu" : " 17 GB", | 20260506 18:25:52.702 | INFO | "huge" : "", | 20260506 18:25:52.702 | INFO | "huge_shared" : " 27 GB", | 20260506 18:25:52.702 | INFO | "malloc" : " 14 GB", | 20260506 18:25:52.703 | INFO | "pinned" : " 29 GB" | 20260506 18:25:52.703 | INFO | } | 20260506 18:25:52.703 | INFO | Perf report for the last 5.13935 seconds: | 20260506 18:25:52.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:25:52.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:52.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:52.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:52.703 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 18:25:52.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:52.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:52.703 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:25:52.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:52.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:52.703 | INFO | Uploaded 36GiB to GPU in 2.02656s (17.7641GiB/s) | 20260506 18:25:52.703 | INFO | Downloaded 34GiB to GPU in 1.94962s (17.4393GiB/s) | 20260506 18:25:57.838 | INFO | Analyzing chunk frames = [3492864,3493376) | 20260506 18:25:57.838 | INFO | MemInfo: { | 20260506 18:25:57.838 | INFO | "gpu" : " 17 GB", | 20260506 18:25:57.838 | INFO | "huge" : "", | 20260506 18:25:57.838 | INFO | "huge_shared" : " 27 GB", | 20260506 18:25:57.838 | INFO | "malloc" : " 14 GB", | 20260506 18:25:57.838 | INFO | "pinned" : " 29 GB" | 20260506 18:25:57.838 | INFO | } | 20260506 18:25:57.838 | INFO | Perf report for the last 5.1356 seconds: | 20260506 18:25:57.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:25:57.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:57.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:57.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:57.838 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 18:25:57.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:57.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:57.838 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:25:57.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:57.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:25:57.839 | INFO | Uploaded 36GiB to GPU in 2.00351s (17.9684GiB/s) | 20260506 18:25:57.839 | INFO | Downloaded 34GiB to GPU in 2.01171s (16.901GiB/s) | 20260506 18:26:02.980 | INFO | Analyzing chunk frames = [3493376,3493888) | 20260506 18:26:02.980 | INFO | MemInfo: { | 20260506 18:26:02.980 | INFO | "gpu" : " 17 GB", | 20260506 18:26:02.980 | INFO | "huge" : "", | 20260506 18:26:02.980 | INFO | "huge_shared" : " 27 GB", | 20260506 18:26:02.980 | INFO | "malloc" : " 14 GB", | 20260506 18:26:02.980 | INFO | "pinned" : " 29 GB" | 20260506 18:26:02.980 | INFO | } | 20260506 18:26:02.980 | INFO | Perf report for the last 5.14186 seconds: | 20260506 18:26:02.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:26:02.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:02.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:02.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:02.980 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 18:26:02.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:02.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:02.980 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260506 18:26:02.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:02.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:02.981 | INFO | Uploaded 36GiB to GPU in 2.01266s (17.8868GiB/s) | 20260506 18:26:02.981 | INFO | Downloaded 34GiB to GPU in 1.99629s (17.0316GiB/s) | 20260506 18:26:03.094 | INFO | Starting a round of Baz IO | 20260506 18:26:06.226 | INFO | Populating RTMetrics report | 20260506 18:26:06.227 | INFO | RTMetrics report populated | 20260506 18:26:06.227 | INFO | Opening tmp RTMetrics json file | 20260506 18:26:06.255 | INFO | Serializing RTMetrics to json | 20260506 18:26:06.278 | INFO | Serialization done, swapping tmp file now... | 20260506 18:26:06.278 | INFO | RTMetrics done | 20260506 18:26:06.522 | INFO | Finished a round of Baz IO: Wrote 2.04 GiB in 3.428 seconds | 20260506 18:26:06.522 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.08% / 0.01% / 64.91% | 20260506 18:26:06.522 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.12535% / 7.94729e-06% / 7.94729e-05% | 20260506 18:26:06.522 | INFO | Baz IO overall: | 20260506 18:26:06.522 | INFO | BytesWritten : 1845584895987 | 20260506 18:26:06.522 | INFO | NumEvents : 892551246809 | 20260506 18:26:08.122 | INFO | Analyzing chunk frames = [3493888,3494400) | 20260506 18:26:08.122 | INFO | MemInfo: { | 20260506 18:26:08.122 | INFO | "gpu" : " 17 GB", | 20260506 18:26:08.122 | INFO | "huge" : "", | 20260506 18:26:08.122 | INFO | "huge_shared" : " 27 GB", | 20260506 18:26:08.122 | INFO | "malloc" : " 14 GB", | 20260506 18:26:08.122 | INFO | "pinned" : " 29 GB" | 20260506 18:26:08.122 | INFO | } | 20260506 18:26:08.122 | INFO | Perf report for the last 5.14189 seconds: | 20260506 18:26:08.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:26:08.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:08.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:08.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:08.122 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 18:26:08.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:08.122 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260506 18:26:08.122 | INFO | PRE_HQ : 60.2%, 1.8, 57.6ms, 0 | 20260506 18:26:08.122 | INFO | BAZWRITER : 66.3%, 4.8, 162.3ms, 0 | 20260506 18:26:08.122 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 18:26:08.122 | INFO | Uploaded 36GiB to GPU in 2.04869s (17.5722GiB/s) | 20260506 18:26:08.122 | INFO | Downloaded 38GiB to GPU in 2.21304s (17.171GiB/s) | 20260506 18:26:13.261 | INFO | Analyzing chunk frames = [3494400,3494912) | 20260506 18:26:13.261 | INFO | MemInfo: { | 20260506 18:26:13.261 | INFO | "gpu" : " 17 GB", | 20260506 18:26:13.261 | INFO | "huge" : "", | 20260506 18:26:13.261 | INFO | "huge_shared" : " 27 GB", | 20260506 18:26:13.261 | INFO | "malloc" : " 14 GB", | 20260506 18:26:13.261 | INFO | "pinned" : " 29 GB" | 20260506 18:26:13.261 | INFO | } | 20260506 18:26:13.261 | INFO | Perf report for the last 5.13935 seconds: | 20260506 18:26:13.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:26:13.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:13.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:13.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:13.262 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 18:26:13.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:13.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:13.262 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:26:13.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:13.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:13.262 | INFO | Uploaded 36GiB to GPU in 1.98791s (18.1095GiB/s) | 20260506 18:26:13.262 | INFO | Downloaded 34GiB to GPU in 1.93518s (17.5694GiB/s) | 20260506 18:26:18.402 | INFO | Analyzing chunk frames = [3494912,3495424) | 20260506 18:26:18.402 | INFO | MemInfo: { | 20260506 18:26:18.402 | INFO | "gpu" : " 17 GB", | 20260506 18:26:18.402 | INFO | "huge" : "", | 20260506 18:26:18.402 | INFO | "huge_shared" : " 27 GB", | 20260506 18:26:18.402 | INFO | "malloc" : " 14 GB", | 20260506 18:26:18.402 | INFO | "pinned" : " 29 GB" | 20260506 18:26:18.402 | INFO | } | 20260506 18:26:18.402 | INFO | Perf report for the last 5.1404 seconds: | 20260506 18:26:18.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:26:18.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:18.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:18.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:18.402 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:26:18.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:18.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:18.402 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:26:18.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:18.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:18.402 | INFO | Uploaded 36GiB to GPU in 1.98955s (18.0945GiB/s) | 20260506 18:26:18.402 | INFO | Downloaded 34GiB to GPU in 1.92376s (17.6737GiB/s) | 20260506 18:26:23.540 | INFO | Analyzing chunk frames = [3495424,3495936) | 20260506 18:26:23.540 | INFO | MemInfo: { | 20260506 18:26:23.540 | INFO | "gpu" : " 17 GB", | 20260506 18:26:23.540 | INFO | "huge" : "", | 20260506 18:26:23.540 | INFO | "huge_shared" : " 27 GB", | 20260506 18:26:23.540 | INFO | "malloc" : " 14 GB", | 20260506 18:26:23.540 | INFO | "pinned" : " 29 GB" | 20260506 18:26:23.540 | INFO | } | 20260506 18:26:23.540 | INFO | Perf report for the last 5.1386 seconds: | 20260506 18:26:23.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:26:23.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:23.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:23.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:23.540 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:26:23.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:23.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:23.540 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 18:26:23.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:23.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:23.541 | INFO | Uploaded 36GiB to GPU in 1.93598s (18.5953GiB/s) | 20260506 18:26:23.541 | INFO | Downloaded 34GiB to GPU in 1.87685s (18.1154GiB/s) | 20260506 18:26:28.674 | INFO | Analyzing chunk frames = [3495936,3496448) | 20260506 18:26:28.674 | INFO | MemInfo: { | 20260506 18:26:28.674 | INFO | "gpu" : " 17 GB", | 20260506 18:26:28.674 | INFO | "huge" : "", | 20260506 18:26:28.674 | INFO | "huge_shared" : " 27 GB", | 20260506 18:26:28.674 | INFO | "malloc" : " 14 GB", | 20260506 18:26:28.674 | INFO | "pinned" : " 29 GB" | 20260506 18:26:28.674 | INFO | } | 20260506 18:26:28.674 | INFO | Perf report for the last 5.13382 seconds: | 20260506 18:26:28.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:26:28.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:28.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:28.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:28.674 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 18:26:28.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:28.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:28.674 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:26:28.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:28.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:28.674 | INFO | Uploaded 36GiB to GPU in 2.01407s (17.8743GiB/s) | 20260506 18:26:28.674 | INFO | Downloaded 34GiB to GPU in 1.911s (17.7917GiB/s) | 20260506 18:26:33.810 | INFO | Analyzing chunk frames = [3496448,3496960) | 20260506 18:26:33.810 | INFO | MemInfo: { | 20260506 18:26:33.810 | INFO | "gpu" : " 17 GB", | 20260506 18:26:33.810 | INFO | "huge" : "", | 20260506 18:26:33.810 | INFO | "huge_shared" : " 27 GB", | 20260506 18:26:33.810 | INFO | "malloc" : " 14 GB", | 20260506 18:26:33.810 | INFO | "pinned" : " 29 GB" | 20260506 18:26:33.810 | INFO | } | 20260506 18:26:33.810 | INFO | Perf report for the last 5.13594 seconds: | 20260506 18:26:33.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:26:33.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:33.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:33.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:33.810 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 18:26:33.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:33.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:33.810 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 18:26:33.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:33.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:33.811 | INFO | Uploaded 36GiB to GPU in 2.03014s (17.7328GiB/s) | 20260506 18:26:33.811 | INFO | Downloaded 34GiB to GPU in 1.95646s (17.3783GiB/s) | 20260506 18:26:38.949 | INFO | Analyzing chunk frames = [3496960,3497472) | 20260506 18:26:38.949 | INFO | MemInfo: { | 20260506 18:26:38.949 | INFO | "gpu" : " 17 GB", | 20260506 18:26:38.949 | INFO | "huge" : "", | 20260506 18:26:38.949 | INFO | "huge_shared" : " 27 GB", | 20260506 18:26:38.949 | INFO | "malloc" : " 14 GB", | 20260506 18:26:38.949 | INFO | "pinned" : " 29 GB" | 20260506 18:26:38.950 | INFO | } | 20260506 18:26:38.950 | INFO | Perf report for the last 5.13958 seconds: | 20260506 18:26:38.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:26:38.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:38.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:38.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:38.950 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 18:26:38.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:38.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:38.950 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 18:26:38.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:38.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:38.950 | INFO | Uploaded 36GiB to GPU in 1.98351s (18.1497GiB/s) | 20260506 18:26:38.950 | INFO | Downloaded 34GiB to GPU in 2.00741s (16.9372GiB/s) | 20260506 18:26:44.093 | INFO | Analyzing chunk frames = [3497472,3497984) | 20260506 18:26:44.093 | INFO | MemInfo: { | 20260506 18:26:44.093 | INFO | "gpu" : " 17 GB", | 20260506 18:26:44.093 | INFO | "huge" : "", | 20260506 18:26:44.093 | INFO | "huge_shared" : " 27 GB", | 20260506 18:26:44.093 | INFO | "malloc" : " 14 GB", | 20260506 18:26:44.093 | INFO | "pinned" : " 29 GB" | 20260506 18:26:44.093 | INFO | } | 20260506 18:26:44.093 | INFO | Perf report for the last 5.14312 seconds: | 20260506 18:26:44.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:26:44.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:44.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:44.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:44.093 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 18:26:44.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:44.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:44.093 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 18:26:44.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:44.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:44.093 | INFO | Uploaded 36GiB to GPU in 2.00121s (17.9892GiB/s) | 20260506 18:26:44.093 | INFO | Downloaded 34GiB to GPU in 1.98298s (17.1459GiB/s) | 20260506 18:26:44.093 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:26:44.093 | INFO | Total mem usage: 18 GiB | 20260506 18:26:44.093 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:26:44.093 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -14.7%) | 20260506 18:26:44.093 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:26:44.093 | INFO | High Water Breakdown: | 20260506 18:26:44.093 | INFO | 61.5% : 9.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:26:44.093 | INFO | 38.4% : 6.02 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:26:44.093 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:26:44.093 | INFO | ------------------------------------------ | 20260506 18:26:44.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:26:44.094 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:26:44.094 | INFO | Total mem usage: 26.5 GiB | 20260506 18:26:44.094 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:26:44.094 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:26:44.094 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:26:44.094 | INFO | High Water Breakdown: | 20260506 18:26:44.094 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:26:44.094 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:26:44.094 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:26:44.094 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:26:44.094 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:26:44.094 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:26:44.094 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:26:44.094 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:26:44.094 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:26:44.094 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:26:44.094 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:26:44.094 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:26:44.094 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:26:44.094 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:26:44.094 | INFO | ------------------------------------------ | 20260506 18:26:44.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:26:44.094 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:26:44.094 | INFO | Total mem usage: 25.6 GiB | 20260506 18:26:44.094 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:26:44.094 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:26:44.094 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:26:44.094 | INFO | High Water Breakdown: | 20260506 18:26:44.094 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:26:44.094 | INFO | ------------------------------------------ | 20260506 18:26:44.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:26:44.094 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:26:44.094 | INFO | Total mem usage: 15.7 GiB | 20260506 18:26:44.094 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:26:44.094 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:26:44.094 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:26:44.094 | INFO | High Water Breakdown: | 20260506 18:26:44.094 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:26:44.094 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:26:44.094 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:26:44.094 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:26:44.094 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:26:44.094 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:26:44.094 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:26:44.094 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:26:44.094 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:26:44.094 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:26:44.094 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:26:44.094 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:26:44.094 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:26:44.094 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:26:44.094 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:26:44.094 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:26:44.094 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:26:44.094 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:26:44.094 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:26:44.094 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:26:44.094 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:26:44.094 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:26:44.094 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:26:44.094 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:26:44.094 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:26:44.094 | INFO | ------------------------------------------ | 20260506 18:26:44.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:26:44.094 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:26:44.094 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:26:44.094 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:26:44.094 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.62e+01, 1.36e+01 / 2.76e+00 | 20260506 18:26:44.094 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.00e-01, 9.60e-03 / 6.67e-03 | 20260506 18:26:44.094 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.42e+00, 5.58e-01 / 3.23e-01 | 20260506 18:26:44.094 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.83e-01 / 1.37e-01 | 20260506 18:26:44.094 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.50e+01, 4.25e+00 / 2.78e+00 | 20260506 18:26:44.094 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.42e+00 / 1.77e+00 | 20260506 18:26:44.094 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.26e+00 / 2.82e+00, 3.82e-01 / 1.92e-01 | 20260506 18:26:44.094 | INFO | PerfReport: 3.41%, Metrics -- 1.07e+00 / 1.41e+00, 2.78e-01 / 1.72e-01 | 20260506 18:26:44.094 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.68e-01 / 4.17e-01, 5.80e-02 / 1.73e-02 | 20260506 18:26:44.094 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:26:44.094 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:26:44.094 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:26:44.094 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 18:26:44.094 | INFO | PerfReport: 21.46%, Upload -- 2.08e+01 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260506 18:26:44.095 | INFO | PerfReport: 21.20%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 18:26:44.095 | INFO | PerfReport: 32.22%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:26:44.095 | INFO | PerfReport: 1.34%, LockUpload -- 1.30e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260506 18:26:44.095 | INFO | PerfReport: 1.18%, LockDownload -- 1.14e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260506 18:26:44.095 | INFO | PerfReport: 22.59%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:26:44.095 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260506 18:26:44.208 | INFO | Starting a round of Baz IO | 20260506 18:26:47.346 | INFO | Populating RTMetrics report | 20260506 18:26:47.347 | INFO | RTMetrics report populated | 20260506 18:26:47.347 | INFO | Opening tmp RTMetrics json file | 20260506 18:26:47.442 | INFO | Serializing RTMetrics to json | 20260506 18:26:47.447 | INFO | Serialization done, swapping tmp file now... | 20260506 18:26:47.451 | INFO | RTMetrics done | 20260506 18:26:47.826 | INFO | Finished a round of Baz IO: Wrote 2.04 GiB in 3.617 seconds | 20260506 18:26:47.826 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.08% / 0.01% / 64.92% | 20260506 18:26:47.826 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.12229% / 1.19209e-05% / 3.57628e-05% | 20260506 18:26:47.826 | INFO | Baz IO overall: | 20260506 18:26:47.826 | INFO | BytesWritten : 1847775100936 | 20260506 18:26:47.826 | INFO | NumEvents : 893606853380 | 20260506 18:26:49.237 | INFO | Analyzing chunk frames = [3497984,3498496) | 20260506 18:26:49.237 | INFO | MemInfo: { | 20260506 18:26:49.237 | INFO | "gpu" : " 17 GB", | 20260506 18:26:49.237 | INFO | "huge" : "", | 20260506 18:26:49.237 | INFO | "huge_shared" : " 27 GB", | 20260506 18:26:49.237 | INFO | "malloc" : " 14 GB", | 20260506 18:26:49.237 | INFO | "pinned" : " 29 GB" | 20260506 18:26:49.237 | INFO | } | 20260506 18:26:49.237 | INFO | Perf report for the last 5.14375 seconds: | 20260506 18:26:49.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:26:49.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:49.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:49.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:49.237 | INFO | ANALYSIS : 62.7%, 2.9, 95.4ms, 0 | 20260506 18:26:49.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:49.237 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 18:26:49.237 | INFO | PRE_HQ : 60.0%, 1.9, 57.8ms, 0 | 20260506 18:26:49.237 | INFO | BAZWRITER : 70.0%, 4.8, 173.4ms, 0 | 20260506 18:26:49.237 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 18:26:49.237 | INFO | Uploaded 36GiB to GPU in 2.05405s (17.5264GiB/s) | 20260506 18:26:49.237 | INFO | Downloaded 38GiB to GPU in 2.21618s (17.1466GiB/s) | 20260506 18:26:54.378 | INFO | Analyzing chunk frames = [3498496,3499008) | 20260506 18:26:54.378 | INFO | MemInfo: { | 20260506 18:26:54.378 | INFO | "gpu" : " 17 GB", | 20260506 18:26:54.378 | INFO | "huge" : "", | 20260506 18:26:54.378 | INFO | "huge_shared" : " 27 GB", | 20260506 18:26:54.378 | INFO | "malloc" : " 14 GB", | 20260506 18:26:54.378 | INFO | "pinned" : " 29 GB" | 20260506 18:26:54.378 | INFO | } | 20260506 18:26:54.378 | INFO | Perf report for the last 5.14124 seconds: | 20260506 18:26:54.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:26:54.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:54.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:54.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:54.378 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 18:26:54.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:54.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:54.378 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 18:26:54.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:54.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:54.378 | INFO | Uploaded 36GiB to GPU in 2.01066s (17.9046GiB/s) | 20260506 18:26:54.378 | INFO | Downloaded 34GiB to GPU in 1.92746s (17.6398GiB/s) | 20260506 18:26:59.516 | INFO | Analyzing chunk frames = [3499008,3499520) | 20260506 18:26:59.516 | INFO | MemInfo: { | 20260506 18:26:59.516 | INFO | "gpu" : " 17 GB", | 20260506 18:26:59.516 | INFO | "huge" : "", | 20260506 18:26:59.516 | INFO | "huge_shared" : " 27 GB", | 20260506 18:26:59.516 | INFO | "malloc" : " 14 GB", | 20260506 18:26:59.516 | INFO | "pinned" : " 29 GB" | 20260506 18:26:59.516 | INFO | } | 20260506 18:26:59.516 | INFO | Perf report for the last 5.13844 seconds: | 20260506 18:26:59.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:26:59.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:59.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:59.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:59.517 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:26:59.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:59.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:59.517 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:26:59.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:59.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:26:59.517 | INFO | Uploaded 36GiB to GPU in 1.99492s (18.0458GiB/s) | 20260506 18:26:59.517 | INFO | Downloaded 34GiB to GPU in 1.91878s (17.7196GiB/s) | 20260506 18:27:04.658 | INFO | Analyzing chunk frames = [3499520,3500032) | 20260506 18:27:04.658 | INFO | MemInfo: { | 20260506 18:27:04.658 | INFO | "gpu" : " 17 GB", | 20260506 18:27:04.658 | INFO | "huge" : "", | 20260506 18:27:04.658 | INFO | "huge_shared" : " 27 GB", | 20260506 18:27:04.658 | INFO | "malloc" : " 14 GB", | 20260506 18:27:04.658 | INFO | "pinned" : " 29 GB" | 20260506 18:27:04.658 | INFO | } | 20260506 18:27:04.658 | INFO | Perf report for the last 5.14176 seconds: | 20260506 18:27:04.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:27:04.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:04.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:04.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:04.658 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 18:27:04.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:04.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:04.658 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:27:04.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:04.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:04.658 | INFO | Uploaded 36GiB to GPU in 1.95616s (18.4034GiB/s) | 20260506 18:27:04.658 | INFO | Downloaded 34GiB to GPU in 1.86957s (18.186GiB/s) | 20260506 18:27:09.789 | INFO | Analyzing chunk frames = [3500032,3500544) | 20260506 18:27:09.789 | INFO | MemInfo: { | 20260506 18:27:09.789 | INFO | "gpu" : " 17 GB", | 20260506 18:27:09.789 | INFO | "huge" : "", | 20260506 18:27:09.789 | INFO | "huge_shared" : " 27 GB", | 20260506 18:27:09.790 | INFO | "malloc" : " 14 GB", | 20260506 18:27:09.790 | INFO | "pinned" : " 29 GB" | 20260506 18:27:09.790 | INFO | } | 20260506 18:27:09.790 | INFO | Perf report for the last 5.1316 seconds: | 20260506 18:27:09.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:27:09.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:09.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:09.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:09.790 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260506 18:27:09.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:09.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:09.790 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:27:09.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:09.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:09.790 | INFO | Uploaded 36GiB to GPU in 2.02087s (17.8141GiB/s) | 20260506 18:27:09.790 | INFO | Downloaded 34GiB to GPU in 1.91083s (17.7933GiB/s) | 20260506 18:27:14.925 | INFO | Analyzing chunk frames = [3500544,3501056) | 20260506 18:27:14.925 | INFO | MemInfo: { | 20260506 18:27:14.925 | INFO | "gpu" : " 17 GB", | 20260506 18:27:14.925 | INFO | "huge" : "", | 20260506 18:27:14.925 | INFO | "huge_shared" : " 27 GB", | 20260506 18:27:14.925 | INFO | "malloc" : " 14 GB", | 20260506 18:27:14.925 | INFO | "pinned" : " 29 GB" | 20260506 18:27:14.925 | INFO | } | 20260506 18:27:14.925 | INFO | Perf report for the last 5.13531 seconds: | 20260506 18:27:14.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:27:14.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:14.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:14.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:14.925 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:27:14.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:14.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:14.925 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:27:14.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:14.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:14.926 | INFO | Uploaded 36GiB to GPU in 2.07874s (17.3182GiB/s) | 20260506 18:27:14.926 | INFO | Downloaded 34GiB to GPU in 1.96181s (17.3309GiB/s) | 20260506 18:27:20.077 | INFO | Analyzing chunk frames = [3501056,3501568) | 20260506 18:27:20.077 | INFO | MemInfo: { | 20260506 18:27:20.077 | INFO | "gpu" : " 17 GB", | 20260506 18:27:20.077 | INFO | "huge" : "", | 20260506 18:27:20.077 | INFO | "huge_shared" : " 27 GB", | 20260506 18:27:20.077 | INFO | "malloc" : " 14 GB", | 20260506 18:27:20.077 | INFO | "pinned" : " 29 GB" | 20260506 18:27:20.077 | INFO | } | 20260506 18:27:20.077 | INFO | Perf report for the last 5.15212 seconds: | 20260506 18:27:20.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:27:20.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:20.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:20.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:20.077 | INFO | ANALYSIS : 62.1%, 3.0, 97.0ms, 0 | 20260506 18:27:20.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:20.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:20.077 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260506 18:27:20.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:20.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:20.078 | INFO | Uploaded 36GiB to GPU in 2.02471s (17.7803GiB/s) | 20260506 18:27:20.078 | INFO | Downloaded 34GiB to GPU in 1.96694s (17.2857GiB/s) | 20260506 18:27:25.212 | INFO | Analyzing chunk frames = [3501568,3502080) | 20260506 18:27:25.212 | INFO | MemInfo: { | 20260506 18:27:25.212 | INFO | "gpu" : " 17 GB", | 20260506 18:27:25.212 | INFO | "huge" : "", | 20260506 18:27:25.212 | INFO | "huge_shared" : " 27 GB", | 20260506 18:27:25.212 | INFO | "malloc" : " 14 GB", | 20260506 18:27:25.212 | INFO | "pinned" : " 29 GB" | 20260506 18:27:25.212 | INFO | } | 20260506 18:27:25.212 | INFO | Perf report for the last 5.13526 seconds: | 20260506 18:27:25.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:27:25.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:25.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:25.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:25.213 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:27:25.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:25.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:25.213 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 18:27:25.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:25.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:25.213 | INFO | Uploaded 36GiB to GPU in 1.99709s (18.0262GiB/s) | 20260506 18:27:25.213 | INFO | Downloaded 34GiB to GPU in 1.97273s (17.235GiB/s) | 20260506 18:27:25.330 | INFO | Starting a round of Baz IO | 20260506 18:27:28.494 | INFO | Populating RTMetrics report | 20260506 18:27:28.495 | INFO | RTMetrics report populated | 20260506 18:27:28.495 | INFO | Opening tmp RTMetrics json file | 20260506 18:27:28.552 | INFO | Serializing RTMetrics to json | 20260506 18:27:28.557 | INFO | Serialization done, swapping tmp file now... | 20260506 18:27:28.560 | INFO | RTMetrics done | 20260506 18:27:28.806 | INFO | Finished a round of Baz IO: Wrote 2.04 GiB in 3.476 seconds | 20260506 18:27:28.806 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.07% / 0.01% / 64.92% | 20260506 18:27:28.806 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.12208% / 0% / 6.75519e-05% | 20260506 18:27:28.806 | INFO | Baz IO overall: | 20260506 18:27:28.806 | INFO | BytesWritten : 1849965988420 | 20260506 18:27:28.806 | INFO | NumEvents : 894663120821 | 20260506 18:27:30.349 | INFO | Analyzing chunk frames = [3502080,3502592) | 20260506 18:27:30.349 | INFO | MemInfo: { | 20260506 18:27:30.349 | INFO | "gpu" : " 17 GB", | 20260506 18:27:30.349 | INFO | "huge" : "", | 20260506 18:27:30.349 | INFO | "huge_shared" : " 27 GB", | 20260506 18:27:30.349 | INFO | "malloc" : " 14 GB", | 20260506 18:27:30.349 | INFO | "pinned" : " 29 GB" | 20260506 18:27:30.349 | INFO | } | 20260506 18:27:30.349 | INFO | Perf report for the last 5.13701 seconds: | 20260506 18:27:30.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:27:30.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:30.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:30.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:30.350 | INFO | ANALYSIS : 62.8%, 2.9, 95.4ms, 0 | 20260506 18:27:30.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:30.350 | INFO | RT_METRICS : 47.3%, 1.0, 24.8ms, 0 | 20260506 18:27:30.350 | INFO | PRE_HQ : 60.9%, 1.9, 61.5ms, 0 | 20260506 18:27:30.350 | INFO | BAZWRITER : 67.3%, 4.8, 166.0ms, 0 | 20260506 18:27:30.350 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.8ms, 0 | 20260506 18:27:30.350 | INFO | Uploaded 36GiB to GPU in 2.07412s (17.3568GiB/s) | 20260506 18:27:30.350 | INFO | Downloaded 38GiB to GPU in 2.21791s (17.1333GiB/s) | 20260506 18:27:35.489 | INFO | Analyzing chunk frames = [3502592,3503104) | 20260506 18:27:35.489 | INFO | MemInfo: { | 20260506 18:27:35.489 | INFO | "gpu" : " 17 GB", | 20260506 18:27:35.489 | INFO | "huge" : "", | 20260506 18:27:35.489 | INFO | "huge_shared" : " 27 GB", | 20260506 18:27:35.489 | INFO | "malloc" : " 14 GB", | 20260506 18:27:35.489 | INFO | "pinned" : " 29 GB" | 20260506 18:27:35.489 | INFO | } | 20260506 18:27:35.489 | INFO | Perf report for the last 5.1396 seconds: | 20260506 18:27:35.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:27:35.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:35.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:35.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:35.489 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:27:35.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:35.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:35.489 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 18:27:35.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:35.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:35.489 | INFO | Uploaded 36GiB to GPU in 2.0156s (17.8607GiB/s) | 20260506 18:27:35.489 | INFO | Downloaded 34GiB to GPU in 1.93521s (17.5692GiB/s) | 20260506 18:27:40.627 | INFO | Analyzing chunk frames = [3503104,3503616) | 20260506 18:27:40.627 | INFO | MemInfo: { | 20260506 18:27:40.627 | INFO | "gpu" : " 17 GB", | 20260506 18:27:40.627 | INFO | "huge" : "", | 20260506 18:27:40.627 | INFO | "huge_shared" : " 27 GB", | 20260506 18:27:40.627 | INFO | "malloc" : " 14 GB", | 20260506 18:27:40.627 | INFO | "pinned" : " 29 GB" | 20260506 18:27:40.627 | INFO | } | 20260506 18:27:40.627 | INFO | Perf report for the last 5.1375 seconds: | 20260506 18:27:40.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:27:40.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:40.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:40.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:40.627 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 18:27:40.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:40.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:40.627 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 18:27:40.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:40.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:40.627 | INFO | Uploaded 36GiB to GPU in 1.99433s (18.0511GiB/s) | 20260506 18:27:40.627 | INFO | Downloaded 34GiB to GPU in 1.91578s (17.7473GiB/s) | 20260506 18:27:45.765 | INFO | Analyzing chunk frames = [3503616,3504128) | 20260506 18:27:45.765 | INFO | MemInfo: { | 20260506 18:27:45.765 | INFO | "gpu" : " 17 GB", | 20260506 18:27:45.765 | INFO | "huge" : "", | 20260506 18:27:45.765 | INFO | "huge_shared" : " 27 GB", | 20260506 18:27:45.765 | INFO | "malloc" : " 14 GB", | 20260506 18:27:45.765 | INFO | "pinned" : " 29 GB" | 20260506 18:27:45.765 | INFO | } | 20260506 18:27:45.765 | INFO | Perf report for the last 5.13807 seconds: | 20260506 18:27:45.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:27:45.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:45.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:45.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:45.765 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:27:45.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:45.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:45.765 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:27:45.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:45.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:45.765 | INFO | Uploaded 36GiB to GPU in 1.92663s (18.6855GiB/s) | 20260506 18:27:45.765 | INFO | Downloaded 34GiB to GPU in 1.8665s (18.2159GiB/s) | 20260506 18:27:50.905 | INFO | Analyzing chunk frames = [3504128,3504640) | 20260506 18:27:50.905 | INFO | MemInfo: { | 20260506 18:27:50.905 | INFO | "gpu" : " 17 GB", | 20260506 18:27:50.905 | INFO | "huge" : "", | 20260506 18:27:50.905 | INFO | "huge_shared" : " 27 GB", | 20260506 18:27:50.905 | INFO | "malloc" : " 14 GB", | 20260506 18:27:50.905 | INFO | "pinned" : " 29 GB" | 20260506 18:27:50.905 | INFO | } | 20260506 18:27:50.905 | INFO | Perf report for the last 5.14068 seconds: | 20260506 18:27:50.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:27:50.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:50.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:50.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:50.906 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 18:27:50.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:50.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:50.906 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:27:50.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:50.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:50.906 | INFO | Uploaded 36GiB to GPU in 2.01904s (17.8303GiB/s) | 20260506 18:27:50.906 | INFO | Downloaded 34GiB to GPU in 1.90878s (17.8124GiB/s) | 20260506 18:27:56.043 | INFO | Analyzing chunk frames = [3504640,3505152) | 20260506 18:27:56.043 | INFO | MemInfo: { | 20260506 18:27:56.043 | INFO | "gpu" : " 17 GB", | 20260506 18:27:56.043 | INFO | "huge" : "", | 20260506 18:27:56.043 | INFO | "huge_shared" : " 27 GB", | 20260506 18:27:56.043 | INFO | "malloc" : " 14 GB", | 20260506 18:27:56.043 | INFO | "pinned" : " 29 GB" | 20260506 18:27:56.043 | INFO | } | 20260506 18:27:56.043 | INFO | Perf report for the last 5.1377 seconds: | 20260506 18:27:56.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:27:56.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:56.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:56.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:56.043 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 18:27:56.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:56.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:56.043 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:27:56.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:56.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:27:56.044 | INFO | Uploaded 36GiB to GPU in 2.01866s (17.8336GiB/s) | 20260506 18:27:56.044 | INFO | Downloaded 34GiB to GPU in 1.94609s (17.471GiB/s) | 20260506 18:28:01.184 | INFO | Analyzing chunk frames = [3505152,3505664) | 20260506 18:28:01.184 | INFO | MemInfo: { | 20260506 18:28:01.184 | INFO | "gpu" : " 17 GB", | 20260506 18:28:01.184 | INFO | "huge" : "", | 20260506 18:28:01.184 | INFO | "huge_shared" : " 27 GB", | 20260506 18:28:01.184 | INFO | "malloc" : " 14 GB", | 20260506 18:28:01.184 | INFO | "pinned" : " 29 GB" | 20260506 18:28:01.184 | INFO | } | 20260506 18:28:01.184 | INFO | Perf report for the last 5.1407 seconds: | 20260506 18:28:01.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:28:01.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:01.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:01.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:01.185 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:28:01.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:01.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:01.185 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:28:01.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:01.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:01.185 | INFO | Uploaded 36GiB to GPU in 2.00134s (17.9879GiB/s) | 20260506 18:28:01.185 | INFO | Downloaded 34GiB to GPU in 2.01116s (16.9057GiB/s) | 20260506 18:28:06.333 | INFO | Analyzing chunk frames = [3505664,3506176) | 20260506 18:28:06.333 | INFO | MemInfo: { | 20260506 18:28:06.333 | INFO | "gpu" : " 17 GB", | 20260506 18:28:06.333 | INFO | "huge" : "", | 20260506 18:28:06.333 | INFO | "huge_shared" : " 27 GB", | 20260506 18:28:06.333 | INFO | "malloc" : " 14 GB", | 20260506 18:28:06.333 | INFO | "pinned" : " 29 GB" | 20260506 18:28:06.333 | INFO | } | 20260506 18:28:06.333 | INFO | Perf report for the last 5.14915 seconds: | 20260506 18:28:06.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:28:06.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:06.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:06.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:06.333 | INFO | ANALYSIS : 61.7%, 3.0, 96.3ms, 0 | 20260506 18:28:06.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:06.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:06.333 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 18:28:06.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:06.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:06.333 | INFO | Uploaded 36GiB to GPU in 2.01034s (17.9074GiB/s) | 20260506 18:28:06.333 | INFO | Downloaded 34GiB to GPU in 1.97604s (17.2061GiB/s) | 20260506 18:28:06.333 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:28:06.333 | INFO | Total mem usage: 18 GiB | 20260506 18:28:06.333 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:28:06.333 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -14.7%) | 20260506 18:28:06.333 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:28:06.333 | INFO | High Water Breakdown: | 20260506 18:28:06.333 | INFO | 61.5% : 9.65 GiB ( 84.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:28:06.333 | INFO | 38.4% : 6.02 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:28:06.333 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:28:06.333 | INFO | ------------------------------------------ | 20260506 18:28:06.334 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:28:06.334 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:28:06.334 | INFO | Total mem usage: 26.5 GiB | 20260506 18:28:06.334 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:28:06.334 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:28:06.334 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:28:06.334 | INFO | High Water Breakdown: | 20260506 18:28:06.334 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:28:06.334 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:28:06.334 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:28:06.334 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:28:06.334 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:28:06.334 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:28:06.334 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:28:06.334 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:28:06.334 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:28:06.334 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:28:06.334 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:28:06.334 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:28:06.334 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:28:06.334 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:28:06.334 | INFO | ------------------------------------------ | 20260506 18:28:06.334 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:28:06.334 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:28:06.334 | INFO | Total mem usage: 25.6 GiB | 20260506 18:28:06.334 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:28:06.334 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:28:06.334 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:28:06.334 | INFO | High Water Breakdown: | 20260506 18:28:06.334 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:28:06.334 | INFO | ------------------------------------------ | 20260506 18:28:06.334 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:28:06.334 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:28:06.334 | INFO | Total mem usage: 15.7 GiB | 20260506 18:28:06.334 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:28:06.334 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:28:06.334 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:28:06.334 | INFO | High Water Breakdown: | 20260506 18:28:06.334 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:28:06.334 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:28:06.334 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:28:06.334 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:28:06.334 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:28:06.334 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:28:06.334 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:28:06.334 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:28:06.334 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:28:06.334 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:28:06.334 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:28:06.334 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:28:06.334 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:28:06.334 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:28:06.334 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:28:06.334 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:28:06.334 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:28:06.334 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:28:06.334 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:28:06.334 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:28:06.334 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:28:06.334 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:28:06.334 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:28:06.334 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:28:06.334 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:28:06.334 | INFO | ------------------------------------------ | 20260506 18:28:06.334 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:28:06.334 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:28:06.334 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:28:06.334 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:28:06.334 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.57e+01, 1.34e+01 / 2.43e+00 | 20260506 18:28:06.334 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 1.02e-02 / 7.54e-03 | 20260506 18:28:06.334 | INFO | PerfReport: 11.61%, Baseline -- 3.64e+00 / 4.43e+00, 5.57e-01 / 3.23e-01 | 20260506 18:28:06.334 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.85e-01 / 1.28e-01 | 20260506 18:28:06.334 | INFO | PerfReport: 15.94%, DME -- 3.99e+01 / 4.44e+01, 4.08e+00 / 2.68e+00 | 20260506 18:28:06.334 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.50e+00 / 1.90e+00 | 20260506 18:28:06.334 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.25e+00 / 2.80e+00, 3.73e-01 / 2.42e-01 | 20260506 18:28:06.334 | INFO | PerfReport: 3.40%, Metrics -- 1.06e+00 / 1.41e+00, 2.79e-01 / 1.85e-01 | 20260506 18:28:06.334 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.68e-01 / 4.16e-01, 5.76e-02 / 2.36e-02 | 20260506 18:28:06.334 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:28:06.334 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:28:06.334 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:28:06.334 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 18:28:06.334 | INFO | PerfReport: 21.57%, Upload -- 2.10e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260506 18:28:06.334 | INFO | PerfReport: 21.12%, Download -- 2.05e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 18:28:06.334 | INFO | PerfReport: 32.22%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 18:28:06.335 | INFO | PerfReport: 1.48%, LockUpload -- 1.44e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260506 18:28:06.335 | INFO | PerfReport: 1.01%, LockDownload -- 9.80e-01 / 0.00e+00, 2.61e+00 / 0.00e+00 | 20260506 18:28:06.335 | INFO | PerfReport: 22.59%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:28:06.335 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260506 18:28:06.447 | INFO | Starting a round of Baz IO | 20260506 18:28:09.583 | INFO | Populating RTMetrics report | 20260506 18:28:09.583 | INFO | RTMetrics report populated | 20260506 18:28:09.583 | INFO | Opening tmp RTMetrics json file | 20260506 18:28:09.638 | INFO | Serializing RTMetrics to json | 20260506 18:28:09.643 | INFO | Serialization done, swapping tmp file now... | 20260506 18:28:09.643 | INFO | RTMetrics done | 20260506 18:28:10.153 | INFO | Finished a round of Baz IO: Wrote 2.04 GiB in 3.705 seconds | 20260506 18:28:10.153 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.07% / 0.01% / 64.93% | 20260506 18:28:10.153 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.11988% / 7.94729e-06% / 5.96046e-05% | 20260506 18:28:10.153 | INFO | Baz IO overall: | 20260506 18:28:10.153 | INFO | BytesWritten : 1852156653682 | 20260506 18:28:10.153 | INFO | NumEvents : 895719145306 | 20260506 18:28:11.466 | INFO | Analyzing chunk frames = [3506176,3506688) | 20260506 18:28:11.466 | INFO | MemInfo: { | 20260506 18:28:11.466 | INFO | "gpu" : " 17 GB", | 20260506 18:28:11.466 | INFO | "huge" : "", | 20260506 18:28:11.466 | INFO | "huge_shared" : " 27 GB", | 20260506 18:28:11.466 | INFO | "malloc" : " 14 GB", | 20260506 18:28:11.466 | INFO | "pinned" : " 29 GB" | 20260506 18:28:11.466 | INFO | } | 20260506 18:28:11.466 | INFO | Perf report for the last 5.13338 seconds: | 20260506 18:28:11.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:28:11.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:11.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:11.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:11.467 | INFO | ANALYSIS : 62.7%, 2.9, 95.6ms, 0 | 20260506 18:28:11.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:11.467 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 18:28:11.467 | INFO | PRE_HQ : 60.3%, 1.8, 57.7ms, 0 | 20260506 18:28:11.467 | INFO | BAZWRITER : 71.8%, 4.8, 176.7ms, 0 | 20260506 18:28:11.467 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 18:28:11.467 | INFO | Uploaded 36GiB to GPU in 2.05812s (17.4917GiB/s) | 20260506 18:28:11.467 | INFO | Downloaded 38GiB to GPU in 2.20393s (17.242GiB/s) | 20260506 18:28:16.602 | INFO | Analyzing chunk frames = [3506688,3507200) | 20260506 18:28:16.602 | INFO | MemInfo: { | 20260506 18:28:16.602 | INFO | "gpu" : " 17 GB", | 20260506 18:28:16.602 | INFO | "huge" : "", | 20260506 18:28:16.602 | INFO | "huge_shared" : " 27 GB", | 20260506 18:28:16.602 | INFO | "malloc" : " 14 GB", | 20260506 18:28:16.602 | INFO | "pinned" : " 29 GB" | 20260506 18:28:16.602 | INFO | } | 20260506 18:28:16.603 | INFO | Perf report for the last 5.1363 seconds: | 20260506 18:28:16.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:28:16.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:16.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:16.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:16.603 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 18:28:16.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:16.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:16.603 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:28:16.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:16.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:16.603 | INFO | Uploaded 36GiB to GPU in 2.02308s (17.7946GiB/s) | 20260506 18:28:16.603 | INFO | Downloaded 34GiB to GPU in 1.92713s (17.6428GiB/s) | 20260506 18:28:21.740 | INFO | Analyzing chunk frames = [3507200,3507712) | 20260506 18:28:21.740 | INFO | MemInfo: { | 20260506 18:28:21.740 | INFO | "gpu" : " 17 GB", | 20260506 18:28:21.740 | INFO | "huge" : "", | 20260506 18:28:21.740 | INFO | "huge_shared" : " 27 GB", | 20260506 18:28:21.740 | INFO | "malloc" : " 14 GB", | 20260506 18:28:21.740 | INFO | "pinned" : " 29 GB" | 20260506 18:28:21.740 | INFO | } | 20260506 18:28:21.740 | INFO | Perf report for the last 5.13775 seconds: | 20260506 18:28:21.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:28:21.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:21.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:21.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:21.740 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 18:28:21.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:21.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:21.740 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 18:28:21.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:21.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:21.741 | INFO | Uploaded 36GiB to GPU in 1.9781s (18.1993GiB/s) | 20260506 18:28:21.741 | INFO | Downloaded 34GiB to GPU in 1.92395s (17.672GiB/s) | 20260506 18:28:26.877 | INFO | Analyzing chunk frames = [3507712,3508224) | 20260506 18:28:26.877 | INFO | MemInfo: { | 20260506 18:28:26.877 | INFO | "gpu" : " 17 GB", | 20260506 18:28:26.877 | INFO | "huge" : "", | 20260506 18:28:26.877 | INFO | "huge_shared" : " 27 GB", | 20260506 18:28:26.877 | INFO | "malloc" : " 14 GB", | 20260506 18:28:26.877 | INFO | "pinned" : " 29 GB" | 20260506 18:28:26.877 | INFO | } | 20260506 18:28:26.877 | INFO | Perf report for the last 5.13693 seconds: | 20260506 18:28:26.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:28:26.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:26.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:26.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:26.878 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:28:26.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:26.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:26.878 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:28:26.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:26.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:26.878 | INFO | Uploaded 36GiB to GPU in 1.92683s (18.6836GiB/s) | 20260506 18:28:26.878 | INFO | Downloaded 34GiB to GPU in 1.86486s (18.232GiB/s) | 20260506 18:28:32.020 | INFO | Analyzing chunk frames = [3508224,3508736) | 20260506 18:28:32.020 | INFO | MemInfo: { | 20260506 18:28:32.020 | INFO | "gpu" : " 17 GB", | 20260506 18:28:32.020 | INFO | "huge" : "", | 20260506 18:28:32.020 | INFO | "huge_shared" : " 27 GB", | 20260506 18:28:32.020 | INFO | "malloc" : " 14 GB", | 20260506 18:28:32.020 | INFO | "pinned" : " 29 GB" | 20260506 18:28:32.020 | INFO | } | 20260506 18:28:32.020 | INFO | Perf report for the last 5.14287 seconds: | 20260506 18:28:32.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:28:32.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:32.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:32.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:32.020 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 18:28:32.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:32.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:32.020 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:28:32.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:32.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:32.020 | INFO | Uploaded 36GiB to GPU in 2.02299s (17.7954GiB/s) | 20260506 18:28:32.020 | INFO | Downloaded 34GiB to GPU in 1.91415s (17.7625GiB/s) | 20260506 18:28:37.166 | INFO | Analyzing chunk frames = [3508736,3509248) | 20260506 18:28:37.166 | INFO | MemInfo: { | 20260506 18:28:37.166 | INFO | "gpu" : " 17 GB", | 20260506 18:28:37.166 | INFO | "huge" : "", | 20260506 18:28:37.166 | INFO | "huge_shared" : " 27 GB", | 20260506 18:28:37.166 | INFO | "malloc" : " 14 GB", | 20260506 18:28:37.166 | INFO | "pinned" : " 29 GB" | 20260506 18:28:37.166 | INFO | } | 20260506 18:28:37.166 | INFO | Perf report for the last 5.14606 seconds: | 20260506 18:28:37.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:28:37.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:37.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:37.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:37.166 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 18:28:37.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:37.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:37.166 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:28:37.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:37.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:37.167 | INFO | Uploaded 36GiB to GPU in 2.03131s (17.7225GiB/s) | 20260506 18:28:37.167 | INFO | Downloaded 34GiB to GPU in 1.94534s (17.4777GiB/s) | 20260506 18:28:42.300 | INFO | Analyzing chunk frames = [3509248,3509760) | 20260506 18:28:42.300 | INFO | MemInfo: { | 20260506 18:28:42.300 | INFO | "gpu" : " 17 GB", | 20260506 18:28:42.300 | INFO | "huge" : "", | 20260506 18:28:42.300 | INFO | "huge_shared" : " 27 GB", | 20260506 18:28:42.300 | INFO | "malloc" : " 14 GB", | 20260506 18:28:42.300 | INFO | "pinned" : " 29 GB" | 20260506 18:28:42.300 | INFO | } | 20260506 18:28:42.300 | INFO | Perf report for the last 5.1335 seconds: | 20260506 18:28:42.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:28:42.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:42.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:42.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:42.300 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 18:28:42.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:42.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:42.300 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:28:42.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:42.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:42.300 | INFO | Uploaded 36GiB to GPU in 2.01238s (17.8893GiB/s) | 20260506 18:28:42.300 | INFO | Downloaded 34GiB to GPU in 1.98851s (17.0982GiB/s) | 20260506 18:28:47.441 | INFO | Analyzing chunk frames = [3509760,3510272) | 20260506 18:28:47.441 | INFO | MemInfo: { | 20260506 18:28:47.441 | INFO | "gpu" : " 17 GB", | 20260506 18:28:47.441 | INFO | "huge" : "", | 20260506 18:28:47.441 | INFO | "huge_shared" : " 27 GB", | 20260506 18:28:47.441 | INFO | "malloc" : " 14 GB", | 20260506 18:28:47.441 | INFO | "pinned" : " 29 GB" | 20260506 18:28:47.441 | INFO | } | 20260506 18:28:47.441 | INFO | Perf report for the last 5.1413 seconds: | 20260506 18:28:47.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:28:47.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:47.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:47.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:47.441 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 18:28:47.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:47.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:47.441 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 18:28:47.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:47.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:47.441 | INFO | Uploaded 36GiB to GPU in 2.01837s (17.8362GiB/s) | 20260506 18:28:47.442 | INFO | Downloaded 34GiB to GPU in 1.98078s (17.165GiB/s) | 20260506 18:28:47.555 | INFO | Starting a round of Baz IO | 20260506 18:28:50.691 | INFO | Populating RTMetrics report | 20260506 18:28:50.692 | INFO | RTMetrics report populated | 20260506 18:28:50.692 | INFO | Opening tmp RTMetrics json file | 20260506 18:28:50.716 | INFO | Serializing RTMetrics to json | 20260506 18:28:50.771 | INFO | Serialization done, swapping tmp file now... | 20260506 18:28:50.802 | INFO | RTMetrics done | 20260506 18:28:51.232 | INFO | Finished a round of Baz IO: Wrote 2.039 GiB in 3.676 seconds | 20260506 18:28:51.232 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.06% / 0.01% / 64.93% | 20260506 18:28:51.232 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.1164% / 7.94729e-06% / 8.74201e-05% | 20260506 18:28:51.232 | INFO | Baz IO overall: | 20260506 18:28:51.232 | INFO | BytesWritten : 1854345881474 | 20260506 18:28:51.232 | INFO | NumEvents : 896773985726 | 20260506 18:28:52.581 | INFO | Analyzing chunk frames = [3510272,3510784) | 20260506 18:28:52.581 | INFO | MemInfo: { | 20260506 18:28:52.581 | INFO | "gpu" : " 17 GB", | 20260506 18:28:52.581 | INFO | "huge" : "", | 20260506 18:28:52.581 | INFO | "huge_shared" : " 27 GB", | 20260506 18:28:52.581 | INFO | "malloc" : " 14 GB", | 20260506 18:28:52.581 | INFO | "pinned" : " 29 GB" | 20260506 18:28:52.581 | INFO | } | 20260506 18:28:52.581 | INFO | Perf report for the last 5.14032 seconds: | 20260506 18:28:52.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:28:52.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:52.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:52.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:52.581 | INFO | ANALYSIS : 62.6%, 2.9, 95.3ms, 0 | 20260506 18:28:52.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:52.582 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 18:28:52.582 | INFO | PRE_HQ : 60.0%, 1.9, 58.0ms, 0 | 20260506 18:28:52.582 | INFO | BAZWRITER : 70.9%, 4.8, 176.1ms, 0 | 20260506 18:28:52.582 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 18:28:52.582 | INFO | Uploaded 36GiB to GPU in 2.019s (17.8306GiB/s) | 20260506 18:28:52.582 | INFO | Downloaded 38GiB to GPU in 2.21257s (17.1746GiB/s) | 20260506 18:28:57.720 | INFO | Analyzing chunk frames = [3510784,3511296) | 20260506 18:28:57.720 | INFO | MemInfo: { | 20260506 18:28:57.720 | INFO | "gpu" : " 17 GB", | 20260506 18:28:57.720 | INFO | "huge" : "", | 20260506 18:28:57.720 | INFO | "huge_shared" : " 27 GB", | 20260506 18:28:57.721 | INFO | "malloc" : " 14 GB", | 20260506 18:28:57.721 | INFO | "pinned" : " 29 GB" | 20260506 18:28:57.721 | INFO | } | 20260506 18:28:57.721 | INFO | Perf report for the last 5.13912 seconds: | 20260506 18:28:57.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:28:57.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:57.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:57.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:57.721 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 18:28:57.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:57.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:57.721 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:28:57.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:57.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:28:57.721 | INFO | Uploaded 36GiB to GPU in 1.98554s (18.1311GiB/s) | 20260506 18:28:57.721 | INFO | Downloaded 34GiB to GPU in 1.91738s (17.7325GiB/s) | 20260506 18:29:02.855 | INFO | Analyzing chunk frames = [3511296,3511808) | 20260506 18:29:02.855 | INFO | MemInfo: { | 20260506 18:29:02.855 | INFO | "gpu" : " 17 GB", | 20260506 18:29:02.855 | INFO | "huge" : "", | 20260506 18:29:02.855 | INFO | "huge_shared" : " 27 GB", | 20260506 18:29:02.855 | INFO | "malloc" : " 14 GB", | 20260506 18:29:02.855 | INFO | "pinned" : " 29 GB" | 20260506 18:29:02.855 | INFO | } | 20260506 18:29:02.855 | INFO | Perf report for the last 5.13466 seconds: | 20260506 18:29:02.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:29:02.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:02.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:02.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:02.855 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 18:29:02.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:02.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:02.855 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:29:02.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:02.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:02.856 | INFO | Uploaded 36GiB to GPU in 1.97343s (18.2424GiB/s) | 20260506 18:29:02.856 | INFO | Downloaded 34GiB to GPU in 1.91926s (17.7152GiB/s) | 20260506 18:29:07.994 | INFO | Analyzing chunk frames = [3511808,3512320) | 20260506 18:29:07.994 | INFO | MemInfo: { | 20260506 18:29:07.994 | INFO | "gpu" : " 17 GB", | 20260506 18:29:07.994 | INFO | "huge" : "", | 20260506 18:29:07.994 | INFO | "huge_shared" : " 27 GB", | 20260506 18:29:07.994 | INFO | "malloc" : " 14 GB", | 20260506 18:29:07.994 | INFO | "pinned" : " 29 GB" | 20260506 18:29:07.994 | INFO | } | 20260506 18:29:07.994 | INFO | Perf report for the last 5.1391 seconds: | 20260506 18:29:07.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:29:07.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:07.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:07.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:07.994 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:29:07.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:07.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:07.995 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:29:07.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:07.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:07.995 | INFO | Uploaded 36GiB to GPU in 1.93538s (18.601GiB/s) | 20260506 18:29:07.995 | INFO | Downloaded 34GiB to GPU in 1.87414s (18.1416GiB/s) | 20260506 18:29:13.135 | INFO | Analyzing chunk frames = [3512320,3512832) | 20260506 18:29:13.135 | INFO | MemInfo: { | 20260506 18:29:13.135 | INFO | "gpu" : " 17 GB", | 20260506 18:29:13.135 | INFO | "huge" : "", | 20260506 18:29:13.135 | INFO | "huge_shared" : " 27 GB", | 20260506 18:29:13.135 | INFO | "malloc" : " 14 GB", | 20260506 18:29:13.135 | INFO | "pinned" : " 29 GB" | 20260506 18:29:13.135 | INFO | } | 20260506 18:29:13.135 | INFO | Perf report for the last 5.14057 seconds: | 20260506 18:29:13.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:29:13.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:13.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:13.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:13.135 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260506 18:29:13.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:13.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:13.135 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:29:13.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:13.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:13.135 | INFO | Uploaded 36GiB to GPU in 1.95311s (18.4321GiB/s) | 20260506 18:29:13.135 | INFO | Downloaded 34GiB to GPU in 1.88379s (18.0488GiB/s) | 20260506 18:29:18.274 | INFO | Analyzing chunk frames = [3512832,3513344) | 20260506 18:29:18.274 | INFO | MemInfo: { | 20260506 18:29:18.274 | INFO | "gpu" : " 17 GB", | 20260506 18:29:18.274 | INFO | "huge" : "", | 20260506 18:29:18.274 | INFO | "huge_shared" : " 27 GB", | 20260506 18:29:18.274 | INFO | "malloc" : " 14 GB", | 20260506 18:29:18.274 | INFO | "pinned" : " 29 GB" | 20260506 18:29:18.274 | INFO | } | 20260506 18:29:18.274 | INFO | Perf report for the last 5.13877 seconds: | 20260506 18:29:18.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:29:18.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:18.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:18.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:18.274 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 18:29:18.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:18.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:18.274 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:29:18.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:18.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:18.274 | INFO | Uploaded 36GiB to GPU in 1.98744s (18.1137GiB/s) | 20260506 18:29:18.274 | INFO | Downloaded 34GiB to GPU in 1.93732s (17.55GiB/s) | 20260506 18:29:23.417 | INFO | Analyzing chunk frames = [3513344,3513856) | 20260506 18:29:23.417 | INFO | MemInfo: { | 20260506 18:29:23.417 | INFO | "gpu" : " 17 GB", | 20260506 18:29:23.417 | INFO | "huge" : "", | 20260506 18:29:23.417 | INFO | "huge_shared" : " 27 GB", | 20260506 18:29:23.417 | INFO | "malloc" : " 14 GB", | 20260506 18:29:23.417 | INFO | "pinned" : " 29 GB" | 20260506 18:29:23.417 | INFO | } | 20260506 18:29:23.418 | INFO | Perf report for the last 5.14375 seconds: | 20260506 18:29:23.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:29:23.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:23.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:23.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:23.418 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 18:29:23.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:23.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:23.418 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 18:29:23.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:23.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:23.418 | INFO | Uploaded 36GiB to GPU in 1.98823s (18.1065GiB/s) | 20260506 18:29:23.418 | INFO | Downloaded 34GiB to GPU in 2.0081s (16.9314GiB/s) | 20260506 18:29:28.554 | INFO | Analyzing chunk frames = [3513856,3514368) | 20260506 18:29:28.554 | INFO | MemInfo: { | 20260506 18:29:28.554 | INFO | "gpu" : " 17 GB", | 20260506 18:29:28.554 | INFO | "huge" : "", | 20260506 18:29:28.554 | INFO | "huge_shared" : " 27 GB", | 20260506 18:29:28.554 | INFO | "malloc" : " 14 GB", | 20260506 18:29:28.554 | INFO | "pinned" : " 29 GB" | 20260506 18:29:28.554 | INFO | } | 20260506 18:29:28.554 | INFO | Perf report for the last 5.13626 seconds: | 20260506 18:29:28.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:29:28.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:28.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:28.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:28.554 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 18:29:28.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:28.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:28.554 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 18:29:28.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:28.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:28.554 | INFO | Uploaded 36GiB to GPU in 2.02174s (17.8064GiB/s) | 20260506 18:29:28.554 | INFO | Downloaded 34GiB to GPU in 1.99087s (17.0779GiB/s) | 20260506 18:29:28.554 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:29:28.554 | INFO | Total mem usage: 18 GiB | 20260506 18:29:28.554 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:29:28.554 | INFO | Recent High Water Marks Sum: 16.3 GiB (Savings: -10.8%) | 20260506 18:29:28.554 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:29:28.554 | INFO | High Water Breakdown: | 20260506 18:29:28.554 | INFO | 62% : 10.1 GiB ( 88.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:29:28.554 | INFO | 37.8% : 6.15 GiB ( 91.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:29:28.554 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:29:28.554 | INFO | ------------------------------------------ | 20260506 18:29:28.555 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:29:28.555 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:29:28.555 | INFO | Total mem usage: 26.5 GiB | 20260506 18:29:28.555 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:29:28.555 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:29:28.555 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:29:28.555 | INFO | High Water Breakdown: | 20260506 18:29:28.555 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:29:28.555 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:29:28.555 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:29:28.555 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:29:28.555 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:29:28.555 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:29:28.555 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:29:28.555 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:29:28.555 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:29:28.555 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:29:28.555 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:29:28.555 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:29:28.555 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:29:28.555 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:29:28.555 | INFO | ------------------------------------------ | 20260506 18:29:28.555 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:29:28.555 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:29:28.555 | INFO | Total mem usage: 25.6 GiB | 20260506 18:29:28.555 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:29:28.555 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:29:28.555 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:29:28.555 | INFO | High Water Breakdown: | 20260506 18:29:28.555 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:29:28.555 | INFO | ------------------------------------------ | 20260506 18:29:28.555 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:29:28.555 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:29:28.555 | INFO | Total mem usage: 15.7 GiB | 20260506 18:29:28.555 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:29:28.555 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:29:28.555 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 18:29:28.555 | INFO | High Water Breakdown: | 20260506 18:29:28.555 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:29:28.555 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:29:28.555 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:29:28.555 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:29:28.555 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:29:28.555 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:29:28.555 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:29:28.555 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:29:28.555 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:29:28.555 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:29:28.555 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:29:28.555 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:29:28.555 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:29:28.555 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:29:28.555 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:29:28.555 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:29:28.555 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:29:28.555 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:29:28.555 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:29:28.555 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:29:28.555 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:29:28.555 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:29:28.555 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:29:28.555 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:29:28.555 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:29:28.555 | INFO | ------------------------------------------ | 20260506 18:29:28.555 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:29:28.555 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:29:28.555 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:29:28.555 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:29:28.555 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.60e+01, 1.35e+01 / 2.77e+00 | 20260506 18:29:28.555 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.00e-01, 9.74e-03 / 5.95e-03 | 20260506 18:29:28.555 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.41e+00, 5.57e-01 / 3.45e-01 | 20260506 18:29:28.555 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.85e-01 / 1.24e-01 | 20260506 18:29:28.555 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.46e+01, 4.22e+00 / 2.87e+00 | 20260506 18:29:28.555 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.48e+00 / 1.89e+00 | 20260506 18:29:28.555 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.25e+00 / 2.86e+00, 3.78e-01 / 2.00e-01 | 20260506 18:29:28.555 | INFO | PerfReport: 3.40%, Metrics -- 1.06e+00 / 1.42e+00, 2.80e-01 / 1.67e-01 | 20260506 18:29:28.555 | INFO | PerfReport: 1.18%, PulseSerializing -- 3.68e-01 / 4.13e-01, 5.80e-02 / 2.08e-02 | 20260506 18:29:28.555 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:29:28.555 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:29:28.555 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:29:28.555 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 18:29:28.555 | INFO | PerfReport: 21.43%, Upload -- 2.08e+01 / 0.00e+00, 6.91e+00 / 0.00e+00 | 20260506 18:29:28.556 | INFO | PerfReport: 21.13%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 18:29:28.556 | INFO | PerfReport: 32.26%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:29:28.556 | INFO | PerfReport: 1.34%, LockUpload -- 1.30e+00 / 0.00e+00, 5.45e+00 / 0.00e+00 | 20260506 18:29:28.556 | INFO | PerfReport: 1.13%, LockDownload -- 1.10e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 18:29:28.556 | INFO | PerfReport: 22.70%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:29:28.556 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260506 18:29:28.668 | INFO | Starting a round of Baz IO | 20260506 18:29:31.821 | INFO | Populating RTMetrics report | 20260506 18:29:31.821 | INFO | RTMetrics report populated | 20260506 18:29:31.821 | INFO | Opening tmp RTMetrics json file | 20260506 18:29:31.826 | INFO | Serializing RTMetrics to json | 20260506 18:29:31.850 | INFO | Serialization done, swapping tmp file now... | 20260506 18:29:31.873 | INFO | RTMetrics done | 20260506 18:29:31.983 | INFO | Finished a round of Baz IO: Wrote 2.039 GiB in 3.315 seconds | 20260506 18:29:31.983 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.06% / 0.01% / 64.94% | 20260506 18:29:31.983 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.1116% / 1.19209e-05% / 9.53674e-05% | 20260506 18:29:31.983 | INFO | Baz IO overall: | 20260506 18:29:31.983 | INFO | BytesWritten : 1856535258764 | 20260506 18:29:31.983 | INFO | NumEvents : 897828928024 | 20260506 18:29:33.695 | INFO | Analyzing chunk frames = [3514368,3514880) | 20260506 18:29:33.695 | INFO | MemInfo: { | 20260506 18:29:33.695 | INFO | "gpu" : " 17 GB", | 20260506 18:29:33.695 | INFO | "huge" : "", | 20260506 18:29:33.695 | INFO | "huge_shared" : " 27 GB", | 20260506 18:29:33.695 | INFO | "malloc" : " 14 GB", | 20260506 18:29:33.695 | INFO | "pinned" : " 29 GB" | 20260506 18:29:33.695 | INFO | } | 20260506 18:29:33.695 | INFO | Perf report for the last 5.14104 seconds: | 20260506 18:29:33.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:29:33.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:33.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:33.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:33.695 | INFO | ANALYSIS : 62.8%, 2.9, 95.1ms, 0 | 20260506 18:29:33.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:33.695 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 18:29:33.695 | INFO | PRE_HQ : 60.4%, 1.9, 59.7ms, 0 | 20260506 18:29:33.695 | INFO | BAZWRITER : 64.1%, 4.7, 156.2ms, 0 | 20260506 18:29:33.695 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 18:29:33.695 | INFO | Uploaded 36GiB to GPU in 2.04426s (17.6103GiB/s) | 20260506 18:29:33.695 | INFO | Downloaded 38GiB to GPU in 2.21237s (17.1761GiB/s) | 20260506 18:29:38.836 | INFO | Analyzing chunk frames = [3514880,3515392) | 20260506 18:29:38.836 | INFO | MemInfo: { | 20260506 18:29:38.836 | INFO | "gpu" : " 17 GB", | 20260506 18:29:38.836 | INFO | "huge" : "", | 20260506 18:29:38.836 | INFO | "huge_shared" : " 27 GB", | 20260506 18:29:38.836 | INFO | "malloc" : " 14 GB", | 20260506 18:29:38.836 | INFO | "pinned" : " 29 GB" | 20260506 18:29:38.836 | INFO | } | 20260506 18:29:38.836 | INFO | Perf report for the last 5.14139 seconds: | 20260506 18:29:38.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:29:38.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:38.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:38.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:38.836 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:29:38.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:38.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:38.836 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:29:38.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:38.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:38.837 | INFO | Uploaded 36GiB to GPU in 2.01454s (17.8701GiB/s) | 20260506 18:29:38.837 | INFO | Downloaded 34GiB to GPU in 1.92825s (17.6326GiB/s) | 20260506 18:29:43.972 | INFO | Analyzing chunk frames = [3515392,3515904) | 20260506 18:29:43.972 | INFO | MemInfo: { | 20260506 18:29:43.972 | INFO | "gpu" : " 17 GB", | 20260506 18:29:43.972 | INFO | "huge" : "", | 20260506 18:29:43.972 | INFO | "huge_shared" : " 27 GB", | 20260506 18:29:43.972 | INFO | "malloc" : " 14 GB", | 20260506 18:29:43.972 | INFO | "pinned" : " 29 GB" | 20260506 18:29:43.972 | INFO | } | 20260506 18:29:43.972 | INFO | Perf report for the last 5.13568 seconds: | 20260506 18:29:43.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:29:43.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:43.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:43.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:43.972 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 18:29:43.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:43.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:43.972 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:29:43.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:43.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:43.972 | INFO | Uploaded 36GiB to GPU in 1.98728s (18.1152GiB/s) | 20260506 18:29:43.972 | INFO | Downloaded 34GiB to GPU in 1.91374s (17.7662GiB/s) | 20260506 18:29:49.109 | INFO | Analyzing chunk frames = [3515904,3516416) | 20260506 18:29:49.109 | INFO | MemInfo: { | 20260506 18:29:49.109 | INFO | "gpu" : " 17 GB", | 20260506 18:29:49.109 | INFO | "huge" : "", | 20260506 18:29:49.109 | INFO | "huge_shared" : " 27 GB", | 20260506 18:29:49.109 | INFO | "malloc" : " 14 GB", | 20260506 18:29:49.109 | INFO | "pinned" : " 29 GB" | 20260506 18:29:49.109 | INFO | } | 20260506 18:29:49.109 | INFO | Perf report for the last 5.13672 seconds: | 20260506 18:29:49.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:29:49.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:49.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:49.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:49.109 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 18:29:49.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:49.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:49.109 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:29:49.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:49.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:49.109 | INFO | Uploaded 36GiB to GPU in 1.94646s (18.4951GiB/s) | 20260506 18:29:49.109 | INFO | Downloaded 34GiB to GPU in 1.88046s (18.0807GiB/s) | 20260506 18:29:54.247 | INFO | Analyzing chunk frames = [3516416,3516928) | 20260506 18:29:54.247 | INFO | MemInfo: { | 20260506 18:29:54.247 | INFO | "gpu" : " 17 GB", | 20260506 18:29:54.247 | INFO | "huge" : "", | 20260506 18:29:54.247 | INFO | "huge_shared" : " 27 GB", | 20260506 18:29:54.247 | INFO | "malloc" : " 14 GB", | 20260506 18:29:54.247 | INFO | "pinned" : " 29 GB" | 20260506 18:29:54.247 | INFO | } | 20260506 18:29:54.247 | INFO | Perf report for the last 5.13846 seconds: | 20260506 18:29:54.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:29:54.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:54.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:54.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:54.247 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260506 18:29:54.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:54.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:54.247 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:29:54.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:54.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:54.247 | INFO | Uploaded 36GiB to GPU in 1.95719s (18.3937GiB/s) | 20260506 18:29:54.247 | INFO | Downloaded 34GiB to GPU in 1.88004s (18.0847GiB/s) | 20260506 18:29:59.387 | INFO | Analyzing chunk frames = [3516928,3517440) | 20260506 18:29:59.387 | INFO | MemInfo: { | 20260506 18:29:59.387 | INFO | "gpu" : " 17 GB", | 20260506 18:29:59.387 | INFO | "huge" : "", | 20260506 18:29:59.387 | INFO | "huge_shared" : " 27 GB", | 20260506 18:29:59.387 | INFO | "malloc" : " 14 GB", | 20260506 18:29:59.388 | INFO | "pinned" : " 29 GB" | 20260506 18:29:59.388 | INFO | } | 20260506 18:29:59.388 | INFO | Perf report for the last 5.14057 seconds: | 20260506 18:29:59.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:29:59.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:59.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:59.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:59.388 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 18:29:59.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:59.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:59.388 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:29:59.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:59.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:29:59.388 | INFO | Uploaded 36GiB to GPU in 2.02733s (17.7573GiB/s) | 20260506 18:29:59.388 | INFO | Downloaded 34GiB to GPU in 1.95049s (17.4315GiB/s) | 20260506 18:30:04.529 | INFO | Analyzing chunk frames = [3517440,3517952) | 20260506 18:30:04.529 | INFO | MemInfo: { | 20260506 18:30:04.529 | INFO | "gpu" : " 17 GB", | 20260506 18:30:04.529 | INFO | "huge" : "", | 20260506 18:30:04.529 | INFO | "huge_shared" : " 27 GB", | 20260506 18:30:04.529 | INFO | "malloc" : " 14 GB", | 20260506 18:30:04.529 | INFO | "pinned" : " 29 GB" | 20260506 18:30:04.529 | INFO | } | 20260506 18:30:04.529 | INFO | Perf report for the last 5.14126 seconds: | 20260506 18:30:04.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:30:04.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:04.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:04.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:04.529 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 18:30:04.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:04.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:04.529 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:30:04.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:04.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:04.529 | INFO | Uploaded 36GiB to GPU in 2.00931s (17.9166GiB/s) | 20260506 18:30:04.529 | INFO | Downloaded 34GiB to GPU in 2.0248s (16.7918GiB/s) | 20260506 18:30:09.672 | INFO | Analyzing chunk frames = [3517952,3518464) | 20260506 18:30:09.672 | INFO | MemInfo: { | 20260506 18:30:09.672 | INFO | "gpu" : " 17 GB", | 20260506 18:30:09.672 | INFO | "huge" : "", | 20260506 18:30:09.672 | INFO | "huge_shared" : " 27 GB", | 20260506 18:30:09.672 | INFO | "malloc" : " 14 GB", | 20260506 18:30:09.672 | INFO | "pinned" : " 29 GB" | 20260506 18:30:09.672 | INFO | } | 20260506 18:30:09.672 | INFO | Perf report for the last 5.14303 seconds: | 20260506 18:30:09.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:30:09.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:09.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:09.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:09.672 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 18:30:09.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:09.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:09.672 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 18:30:09.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:09.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:09.672 | INFO | Uploaded 36GiB to GPU in 2.0378s (17.6661GiB/s) | 20260506 18:30:09.673 | INFO | Downloaded 34GiB to GPU in 1.98376s (17.1392GiB/s) | 20260506 18:30:09.790 | INFO | Starting a round of Baz IO | 20260506 18:30:12.943 | INFO | Populating RTMetrics report | 20260506 18:30:12.943 | INFO | RTMetrics report populated | 20260506 18:30:12.943 | INFO | Opening tmp RTMetrics json file | 20260506 18:30:12.950 | INFO | Serializing RTMetrics to json | 20260506 18:30:12.974 | INFO | Serialization done, swapping tmp file now... | 20260506 18:30:12.975 | INFO | RTMetrics done | 20260506 18:30:13.399 | INFO | Finished a round of Baz IO: Wrote 2.039 GiB in 3.608 seconds | 20260506 18:30:13.399 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.05% / 0.01% / 64.94% | 20260506 18:30:13.399 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.11026% / 2.38419e-05% / 9.53674e-05% | 20260506 18:30:13.399 | INFO | Baz IO overall: | 20260506 18:30:13.399 | INFO | BytesWritten : 1858724757737 | 20260506 18:30:13.399 | INFO | NumEvents : 898884104577 | 20260506 18:30:14.812 | INFO | Analyzing chunk frames = [3518464,3518976) | 20260506 18:30:14.812 | INFO | MemInfo: { | 20260506 18:30:14.812 | INFO | "gpu" : " 17 GB", | 20260506 18:30:14.812 | INFO | "huge" : "", | 20260506 18:30:14.812 | INFO | "huge_shared" : " 27 GB", | 20260506 18:30:14.812 | INFO | "malloc" : " 14 GB", | 20260506 18:30:14.812 | INFO | "pinned" : " 29 GB" | 20260506 18:30:14.812 | INFO | } | 20260506 18:30:14.812 | INFO | Perf report for the last 5.14011 seconds: | 20260506 18:30:14.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:30:14.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:14.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:14.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:14.812 | INFO | ANALYSIS : 62.7%, 2.9, 95.0ms, 0 | 20260506 18:30:14.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:14.812 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 18:30:14.812 | INFO | PRE_HQ : 60.4%, 1.9, 59.0ms, 0 | 20260506 18:30:14.812 | INFO | BAZWRITER : 69.8%, 4.8, 172.3ms, 0 | 20260506 18:30:14.812 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 18:30:14.812 | INFO | Uploaded 36GiB to GPU in 2.03272s (17.7103GiB/s) | 20260506 18:30:14.812 | INFO | Downloaded 38GiB to GPU in 2.21704s (17.14GiB/s) | 20260506 18:30:19.949 | INFO | Analyzing chunk frames = [3518976,3519488) | 20260506 18:30:19.949 | INFO | MemInfo: { | 20260506 18:30:19.949 | INFO | "gpu" : " 17 GB", | 20260506 18:30:19.949 | INFO | "huge" : "", | 20260506 18:30:19.949 | INFO | "huge_shared" : " 27 GB", | 20260506 18:30:19.949 | INFO | "malloc" : " 14 GB", | 20260506 18:30:19.949 | INFO | "pinned" : " 29 GB" | 20260506 18:30:19.949 | INFO | } | 20260506 18:30:19.949 | INFO | Perf report for the last 5.13729 seconds: | 20260506 18:30:19.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:30:19.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:19.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:19.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:19.950 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 18:30:19.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:19.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:19.950 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:30:19.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:19.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:19.950 | INFO | Uploaded 36GiB to GPU in 2.00028s (17.9975GiB/s) | 20260506 18:30:19.950 | INFO | Downloaded 34GiB to GPU in 1.92911s (17.6247GiB/s) | 20260506 18:30:25.091 | INFO | Analyzing chunk frames = [3519488,3520000) | 20260506 18:30:25.091 | INFO | MemInfo: { | 20260506 18:30:25.091 | INFO | "gpu" : " 17 GB", | 20260506 18:30:25.091 | INFO | "huge" : "", | 20260506 18:30:25.091 | INFO | "huge_shared" : " 27 GB", | 20260506 18:30:25.091 | INFO | "malloc" : " 14 GB", | 20260506 18:30:25.091 | INFO | "pinned" : " 29 GB" | 20260506 18:30:25.091 | INFO | } | 20260506 18:30:25.091 | INFO | Perf report for the last 5.14139 seconds: | 20260506 18:30:25.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:30:25.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:25.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:25.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:25.091 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 18:30:25.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:25.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:25.091 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:30:25.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:25.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:25.091 | INFO | Uploaded 36GiB to GPU in 1.9386s (18.5701GiB/s) | 20260506 18:30:25.091 | INFO | Downloaded 34GiB to GPU in 1.91827s (17.7243GiB/s) | 20260506 18:30:30.223 | INFO | Analyzing chunk frames = [3520000,3520512) | 20260506 18:30:30.223 | INFO | MemInfo: { | 20260506 18:30:30.223 | INFO | "gpu" : " 17 GB", | 20260506 18:30:30.223 | INFO | "huge" : "", | 20260506 18:30:30.223 | INFO | "huge_shared" : " 27 GB", | 20260506 18:30:30.223 | INFO | "malloc" : " 14 GB", | 20260506 18:30:30.223 | INFO | "pinned" : " 29 GB" | 20260506 18:30:30.223 | INFO | } | 20260506 18:30:30.223 | INFO | Perf report for the last 5.13257 seconds: | 20260506 18:30:30.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:30:30.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:30.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:30.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:30.224 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:30:30.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:30.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:30.224 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:30:30.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:30.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:30.224 | INFO | Uploaded 36GiB to GPU in 1.91557s (18.7933GiB/s) | 20260506 18:30:30.224 | INFO | Downloaded 34GiB to GPU in 1.86603s (18.2205GiB/s) | 20260506 18:30:35.361 | INFO | Analyzing chunk frames = [3520512,3521024) | 20260506 18:30:35.361 | INFO | MemInfo: { | 20260506 18:30:35.361 | INFO | "gpu" : " 17 GB", | 20260506 18:30:35.361 | INFO | "huge" : "", | 20260506 18:30:35.361 | INFO | "huge_shared" : " 27 GB", | 20260506 18:30:35.361 | INFO | "malloc" : " 14 GB", | 20260506 18:30:35.361 | INFO | "pinned" : " 29 GB" | 20260506 18:30:35.361 | INFO | } | 20260506 18:30:35.361 | INFO | Perf report for the last 5.13756 seconds: | 20260506 18:30:35.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:30:35.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:35.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:35.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:35.361 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260506 18:30:35.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:35.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:35.361 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:30:35.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:35.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:35.361 | INFO | Uploaded 36GiB to GPU in 2.01795s (17.8399GiB/s) | 20260506 18:30:35.361 | INFO | Downloaded 34GiB to GPU in 1.91548s (17.7501GiB/s) | 20260506 18:30:40.507 | INFO | Analyzing chunk frames = [3521024,3521536) | 20260506 18:30:40.507 | INFO | MemInfo: { | 20260506 18:30:40.507 | INFO | "gpu" : " 17 GB", | 20260506 18:30:40.507 | INFO | "huge" : "", | 20260506 18:30:40.507 | INFO | "huge_shared" : " 27 GB", | 20260506 18:30:40.507 | INFO | "malloc" : " 14 GB", | 20260506 18:30:40.507 | INFO | "pinned" : " 29 GB" | 20260506 18:30:40.507 | INFO | } | 20260506 18:30:40.507 | INFO | Perf report for the last 5.14603 seconds: | 20260506 18:30:40.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:30:40.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:40.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:40.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:40.507 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 18:30:40.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:40.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:40.507 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:30:40.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:40.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:40.507 | INFO | Uploaded 36GiB to GPU in 2.00853s (17.9235GiB/s) | 20260506 18:30:40.507 | INFO | Downloaded 34GiB to GPU in 1.94625s (17.4695GiB/s) | 20260506 18:30:45.649 | INFO | Analyzing chunk frames = [3521536,3522048) | 20260506 18:30:45.649 | INFO | MemInfo: { | 20260506 18:30:45.649 | INFO | "gpu" : " 17 GB", | 20260506 18:30:45.649 | INFO | "huge" : "", | 20260506 18:30:45.649 | INFO | "huge_shared" : " 27 GB", | 20260506 18:30:45.649 | INFO | "malloc" : " 14 GB", | 20260506 18:30:45.649 | INFO | "pinned" : " 29 GB" | 20260506 18:30:45.649 | INFO | } | 20260506 18:30:45.649 | INFO | Perf report for the last 5.14185 seconds: | 20260506 18:30:45.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:30:45.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:45.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:45.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:45.649 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 18:30:45.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:45.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:45.649 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:30:45.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:45.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:45.649 | INFO | Uploaded 36GiB to GPU in 1.98566s (18.13GiB/s) | 20260506 18:30:45.649 | INFO | Downloaded 34GiB to GPU in 2.00848s (16.9282GiB/s) | 20260506 18:30:50.788 | INFO | Analyzing chunk frames = [3522048,3522560) | 20260506 18:30:50.788 | INFO | MemInfo: { | 20260506 18:30:50.788 | INFO | "gpu" : " 17 GB", | 20260506 18:30:50.788 | INFO | "huge" : "", | 20260506 18:30:50.788 | INFO | "huge_shared" : " 27 GB", | 20260506 18:30:50.788 | INFO | "malloc" : " 14 GB", | 20260506 18:30:50.788 | INFO | "pinned" : " 29 GB" | 20260506 18:30:50.788 | INFO | } | 20260506 18:30:50.788 | INFO | Perf report for the last 5.13868 seconds: | 20260506 18:30:50.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:30:50.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:50.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:50.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:50.788 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 18:30:50.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:50.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:50.788 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 18:30:50.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:50.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:50.788 | INFO | Uploaded 36GiB to GPU in 2.0271s (17.7594GiB/s) | 20260506 18:30:50.788 | INFO | Downloaded 34GiB to GPU in 1.97772s (17.1915GiB/s) | 20260506 18:30:50.788 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:30:50.788 | INFO | Total mem usage: 18 GiB | 20260506 18:30:50.788 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:30:50.788 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: -9.31%) | 20260506 18:30:50.788 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:30:50.788 | INFO | High Water Breakdown: | 20260506 18:30:50.788 | INFO | 62.2% : 10.3 GiB ( 89.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:30:50.788 | INFO | 37.6% : 6.2 GiB ( 92.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:30:50.788 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:30:50.788 | INFO | ------------------------------------------ | 20260506 18:30:50.788 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:30:50.788 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:30:50.788 | INFO | Total mem usage: 26.5 GiB | 20260506 18:30:50.788 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:30:50.788 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:30:50.788 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:30:50.788 | INFO | High Water Breakdown: | 20260506 18:30:50.788 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:30:50.788 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:30:50.788 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:30:50.788 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:30:50.788 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:30:50.788 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:30:50.789 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:30:50.789 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:30:50.789 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:30:50.789 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:30:50.789 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:30:50.789 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:30:50.789 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:30:50.789 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:30:50.789 | INFO | ------------------------------------------ | 20260506 18:30:50.789 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:30:50.789 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:30:50.789 | INFO | Total mem usage: 25.6 GiB | 20260506 18:30:50.789 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:30:50.789 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:30:50.789 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:30:50.789 | INFO | High Water Breakdown: | 20260506 18:30:50.789 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:30:50.789 | INFO | ------------------------------------------ | 20260506 18:30:50.789 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:30:50.789 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:30:50.789 | INFO | Total mem usage: 15.7 GiB | 20260506 18:30:50.789 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:30:50.789 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:30:50.789 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:30:50.789 | INFO | High Water Breakdown: | 20260506 18:30:50.789 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:30:50.789 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:30:50.789 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:30:50.789 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:30:50.789 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:30:50.789 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:30:50.789 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:30:50.789 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:30:50.789 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:30:50.789 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:30:50.789 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:30:50.789 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:30:50.789 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:30:50.789 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:30:50.789 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:30:50.789 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:30:50.789 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:30:50.789 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:30:50.789 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:30:50.789 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:30:50.789 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:30:50.789 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:30:50.789 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:30:50.789 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:30:50.789 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:30:50.789 | INFO | ------------------------------------------ | 20260506 18:30:50.789 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:30:50.789 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:30:50.789 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:30:50.789 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:30:50.789 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.58e+01, 1.35e+01 / 2.75e+00 | 20260506 18:30:50.789 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 9.89e-03 / 6.76e-03 | 20260506 18:30:50.789 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.34e+00, 5.62e-01 / 4.15e-01 | 20260506 18:30:50.789 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.45e+00, 1.87e-01 / 1.40e-01 | 20260506 18:30:50.789 | INFO | PerfReport: 15.96%, DME -- 4.00e+01 / 4.45e+01, 4.17e+00 / 2.85e+00 | 20260506 18:30:50.789 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.47e+00 / 1.77e+00 | 20260506 18:30:50.789 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.26e+00 / 2.84e+00, 3.82e-01 / 2.12e-01 | 20260506 18:30:50.789 | INFO | PerfReport: 3.39%, Metrics -- 1.06e+00 / 1.40e+00, 2.78e-01 / 1.56e-01 | 20260506 18:30:50.789 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.67e-01 / 4.10e-01, 5.77e-02 / 2.25e-02 | 20260506 18:30:50.789 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:30:50.789 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:30:50.790 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:30:50.790 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 18:30:50.790 | INFO | PerfReport: 21.45%, Upload -- 2.08e+01 / 0.00e+00, 6.88e+00 / 0.00e+00 | 20260506 18:30:50.790 | INFO | PerfReport: 21.18%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 18:30:50.790 | INFO | PerfReport: 32.28%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:30:50.790 | INFO | PerfReport: 1.36%, LockUpload -- 1.32e+00 / 0.00e+00, 5.50e+00 / 0.00e+00 | 20260506 18:30:50.790 | INFO | PerfReport: 1.15%, LockDownload -- 1.11e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260506 18:30:50.790 | INFO | PerfReport: 22.57%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 18:30:50.790 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260506 18:30:50.903 | INFO | Starting a round of Baz IO | 20260506 18:30:54.104 | INFO | Populating RTMetrics report | 20260506 18:30:54.104 | INFO | RTMetrics report populated | 20260506 18:30:54.104 | INFO | Opening tmp RTMetrics json file | 20260506 18:30:54.233 | INFO | Serializing RTMetrics to json | 20260506 18:30:54.261 | INFO | Serialization done, swapping tmp file now... | 20260506 18:30:54.264 | INFO | RTMetrics done | 20260506 18:30:54.666 | INFO | Finished a round of Baz IO: Wrote 2.037 GiB in 3.763 seconds | 20260506 18:30:54.666 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.04% / 0.01% / 64.95% | 20260506 18:30:54.666 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.10549% / 1.58946e-05% / 9.53674e-05% | 20260506 18:30:54.666 | INFO | Baz IO overall: | 20260506 18:30:54.666 | INFO | BytesWritten : 1860912333753 | 20260506 18:30:54.666 | INFO | NumEvents : 899937761920 | 20260506 18:30:55.927 | INFO | Analyzing chunk frames = [3522560,3523072) | 20260506 18:30:55.927 | INFO | MemInfo: { | 20260506 18:30:55.927 | INFO | "gpu" : " 17 GB", | 20260506 18:30:55.927 | INFO | "huge" : "", | 20260506 18:30:55.927 | INFO | "huge_shared" : " 27 GB", | 20260506 18:30:55.927 | INFO | "malloc" : " 14 GB", | 20260506 18:30:55.927 | INFO | "pinned" : " 29 GB" | 20260506 18:30:55.927 | INFO | } | 20260506 18:30:55.927 | INFO | Perf report for the last 5.13906 seconds: | 20260506 18:30:55.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:30:55.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:55.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:55.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:55.927 | INFO | ANALYSIS : 63.5%, 2.8, 94.1ms, 0 | 20260506 18:30:55.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:30:55.927 | INFO | RT_METRICS : 50.2%, 1.0, 26.3ms, 0 | 20260506 18:30:55.927 | INFO | PRE_HQ : 61.0%, 2.0, 63.9ms, 0 | 20260506 18:30:55.927 | INFO | BAZWRITER : 72.9%, 4.8, 180.4ms, 0 | 20260506 18:30:55.927 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 18:30:55.928 | INFO | Uploaded 36GiB to GPU in 2.09663s (17.1704GiB/s) | 20260506 18:30:55.928 | INFO | Downloaded 38GiB to GPU in 2.17021s (17.5098GiB/s) | 20260506 18:31:01.060 | INFO | Analyzing chunk frames = [3523072,3523584) | 20260506 18:31:01.060 | INFO | MemInfo: { | 20260506 18:31:01.060 | INFO | "gpu" : " 17 GB", | 20260506 18:31:01.060 | INFO | "huge" : "", | 20260506 18:31:01.060 | INFO | "huge_shared" : " 27 GB", | 20260506 18:31:01.060 | INFO | "malloc" : " 14 GB", | 20260506 18:31:01.060 | INFO | "pinned" : " 29 GB" | 20260506 18:31:01.060 | INFO | } | 20260506 18:31:01.060 | INFO | Perf report for the last 5.13299 seconds: | 20260506 18:31:01.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:31:01.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:01.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:01.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:01.060 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 18:31:01.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:01.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:01.060 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:31:01.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:01.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:01.060 | INFO | Uploaded 36GiB to GPU in 2.02095s (17.8134GiB/s) | 20260506 18:31:01.060 | INFO | Downloaded 34GiB to GPU in 1.941s (17.5167GiB/s) | 20260506 18:31:06.198 | INFO | Analyzing chunk frames = [3523584,3524096) | 20260506 18:31:06.198 | INFO | MemInfo: { | 20260506 18:31:06.198 | INFO | "gpu" : " 17 GB", | 20260506 18:31:06.198 | INFO | "huge" : "", | 20260506 18:31:06.198 | INFO | "huge_shared" : " 27 GB", | 20260506 18:31:06.198 | INFO | "malloc" : " 14 GB", | 20260506 18:31:06.198 | INFO | "pinned" : " 29 GB" | 20260506 18:31:06.198 | INFO | } | 20260506 18:31:06.198 | INFO | Perf report for the last 5.13858 seconds: | 20260506 18:31:06.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:31:06.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:06.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:06.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:06.198 | INFO | ANALYSIS : 61.0%, 3.0, 94.5ms, 0 | 20260506 18:31:06.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:06.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:06.199 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:31:06.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:06.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:06.199 | INFO | Uploaded 36GiB to GPU in 1.98456s (18.14GiB/s) | 20260506 18:31:06.199 | INFO | Downloaded 34GiB to GPU in 1.92259s (17.6845GiB/s) | 20260506 18:31:11.337 | INFO | Analyzing chunk frames = [3524096,3524608) | 20260506 18:31:11.337 | INFO | MemInfo: { | 20260506 18:31:11.337 | INFO | "gpu" : " 17 GB", | 20260506 18:31:11.337 | INFO | "huge" : "", | 20260506 18:31:11.337 | INFO | "huge_shared" : " 27 GB", | 20260506 18:31:11.337 | INFO | "malloc" : " 14 GB", | 20260506 18:31:11.337 | INFO | "pinned" : " 29 GB" | 20260506 18:31:11.337 | INFO | } | 20260506 18:31:11.337 | INFO | Perf report for the last 5.13844 seconds: | 20260506 18:31:11.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:31:11.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:11.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:11.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:11.337 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 18:31:11.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:11.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:11.337 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:31:11.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:11.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:11.337 | INFO | Uploaded 36GiB to GPU in 1.93083s (18.6449GiB/s) | 20260506 18:31:11.337 | INFO | Downloaded 34GiB to GPU in 1.87052s (18.1767GiB/s) | 20260506 18:31:16.478 | INFO | Analyzing chunk frames = [3524608,3525120) | 20260506 18:31:16.478 | INFO | MemInfo: { | 20260506 18:31:16.478 | INFO | "gpu" : " 17 GB", | 20260506 18:31:16.478 | INFO | "huge" : "", | 20260506 18:31:16.478 | INFO | "huge_shared" : " 27 GB", | 20260506 18:31:16.478 | INFO | "malloc" : " 14 GB", | 20260506 18:31:16.478 | INFO | "pinned" : " 29 GB" | 20260506 18:31:16.478 | INFO | } | 20260506 18:31:16.478 | INFO | Perf report for the last 5.14169 seconds: | 20260506 18:31:16.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:31:16.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:16.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:16.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:16.478 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 18:31:16.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:16.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:16.479 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:31:16.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:16.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:16.479 | INFO | Uploaded 36GiB to GPU in 1.9609s (18.3589GiB/s) | 20260506 18:31:16.479 | INFO | Downloaded 34GiB to GPU in 1.88219s (18.0641GiB/s) | 20260506 18:31:21.620 | INFO | Analyzing chunk frames = [3525120,3525632) | 20260506 18:31:21.620 | INFO | MemInfo: { | 20260506 18:31:21.620 | INFO | "gpu" : " 17 GB", | 20260506 18:31:21.620 | INFO | "huge" : "", | 20260506 18:31:21.620 | INFO | "huge_shared" : " 27 GB", | 20260506 18:31:21.620 | INFO | "malloc" : " 14 GB", | 20260506 18:31:21.620 | INFO | "pinned" : " 29 GB" | 20260506 18:31:21.620 | INFO | } | 20260506 18:31:21.620 | INFO | Perf report for the last 5.14165 seconds: | 20260506 18:31:21.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:31:21.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:21.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:21.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:21.620 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 18:31:21.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:21.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:21.620 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:31:21.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:21.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:21.620 | INFO | Uploaded 36GiB to GPU in 2.0191s (17.8297GiB/s) | 20260506 18:31:21.621 | INFO | Downloaded 34GiB to GPU in 1.94991s (17.4367GiB/s) | 20260506 18:31:26.767 | INFO | Analyzing chunk frames = [3525632,3526144) | 20260506 18:31:26.767 | INFO | MemInfo: { | 20260506 18:31:26.767 | INFO | "gpu" : " 17 GB", | 20260506 18:31:26.767 | INFO | "huge" : "", | 20260506 18:31:26.767 | INFO | "huge_shared" : " 27 GB", | 20260506 18:31:26.767 | INFO | "malloc" : " 14 GB", | 20260506 18:31:26.767 | INFO | "pinned" : " 29 GB" | 20260506 18:31:26.768 | INFO | } | 20260506 18:31:26.768 | INFO | Perf report for the last 5.14748 seconds: | 20260506 18:31:26.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:31:26.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:26.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:26.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:26.768 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 18:31:26.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:26.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:26.768 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:31:26.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:26.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:26.768 | INFO | Uploaded 36GiB to GPU in 2.01955s (17.8257GiB/s) | 20260506 18:31:26.768 | INFO | Downloaded 34GiB to GPU in 1.99009s (17.0847GiB/s) | 20260506 18:31:31.904 | INFO | Analyzing chunk frames = [3526144,3526656) | 20260506 18:31:31.904 | INFO | MemInfo: { | 20260506 18:31:31.904 | INFO | "gpu" : " 17 GB", | 20260506 18:31:31.904 | INFO | "huge" : "", | 20260506 18:31:31.904 | INFO | "huge_shared" : " 27 GB", | 20260506 18:31:31.904 | INFO | "malloc" : " 14 GB", | 20260506 18:31:31.904 | INFO | "pinned" : " 29 GB" | 20260506 18:31:31.904 | INFO | } | 20260506 18:31:31.904 | INFO | Perf report for the last 5.1362 seconds: | 20260506 18:31:31.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:31:31.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:31.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:31.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:31.904 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 18:31:31.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:31.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:31.904 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:31:31.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:31.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:31.904 | INFO | Uploaded 36GiB to GPU in 2.01212s (17.8915GiB/s) | 20260506 18:31:31.904 | INFO | Downloaded 34GiB to GPU in 1.9878s (17.1043GiB/s) | 20260506 18:31:32.019 | INFO | Starting a round of Baz IO | 20260506 18:31:35.159 | INFO | Populating RTMetrics report | 20260506 18:31:35.159 | INFO | RTMetrics report populated | 20260506 18:31:35.159 | INFO | Opening tmp RTMetrics json file | 20260506 18:31:35.160 | INFO | Serializing RTMetrics to json | 20260506 18:31:35.162 | INFO | Serialization done, swapping tmp file now... | 20260506 18:31:35.165 | INFO | RTMetrics done | 20260506 18:31:35.477 | INFO | Finished a round of Baz IO: Wrote 2.038 GiB in 3.458 seconds | 20260506 18:31:35.477 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.04% / 0.01% / 64.96% | 20260506 18:31:35.477 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.10523% / 1.19209e-05% / 8.74201e-05% | 20260506 18:31:35.477 | INFO | Baz IO overall: | 20260506 18:31:35.477 | INFO | BytesWritten : 1863100244748 | 20260506 18:31:35.477 | INFO | NumEvents : 900991623324 | 20260506 18:31:37.039 | INFO | Analyzing chunk frames = [3526656,3527168) | 20260506 18:31:37.039 | INFO | MemInfo: { | 20260506 18:31:37.039 | INFO | "gpu" : " 17 GB", | 20260506 18:31:37.039 | INFO | "huge" : "", | 20260506 18:31:37.039 | INFO | "huge_shared" : " 27 GB", | 20260506 18:31:37.039 | INFO | "malloc" : " 14 GB", | 20260506 18:31:37.039 | INFO | "pinned" : " 29 GB" | 20260506 18:31:37.039 | INFO | } | 20260506 18:31:37.039 | INFO | Perf report for the last 5.13568 seconds: | 20260506 18:31:37.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:31:37.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:37.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:37.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:37.040 | INFO | ANALYSIS : 62.8%, 2.9, 95.3ms, 0 | 20260506 18:31:37.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:37.040 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260506 18:31:37.040 | INFO | PRE_HQ : 60.2%, 1.9, 58.2ms, 0 | 20260506 18:31:37.040 | INFO | BAZWRITER : 67.0%, 4.8, 164.4ms, 0 | 20260506 18:31:37.040 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 18:31:37.040 | INFO | Uploaded 36GiB to GPU in 2.0749s (17.3502GiB/s) | 20260506 18:31:37.040 | INFO | Downloaded 38GiB to GPU in 2.21526s (17.1537GiB/s) | 20260506 18:31:42.179 | INFO | Analyzing chunk frames = [3527168,3527680) | 20260506 18:31:42.179 | INFO | MemInfo: { | 20260506 18:31:42.179 | INFO | "gpu" : " 17 GB", | 20260506 18:31:42.179 | INFO | "huge" : "", | 20260506 18:31:42.179 | INFO | "huge_shared" : " 27 GB", | 20260506 18:31:42.179 | INFO | "malloc" : " 14 GB", | 20260506 18:31:42.179 | INFO | "pinned" : " 29 GB" | 20260506 18:31:42.179 | INFO | } | 20260506 18:31:42.180 | INFO | Perf report for the last 5.14002 seconds: | 20260506 18:31:42.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:31:42.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:42.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:42.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:42.180 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:31:42.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:42.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:42.180 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:31:42.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:42.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:42.180 | INFO | Uploaded 36GiB to GPU in 2.01321s (17.8819GiB/s) | 20260506 18:31:42.180 | INFO | Downloaded 34GiB to GPU in 1.93318s (17.5876GiB/s) | 20260506 18:31:47.311 | INFO | Analyzing chunk frames = [3527680,3528192) | 20260506 18:31:47.311 | INFO | MemInfo: { | 20260506 18:31:47.311 | INFO | "gpu" : " 17 GB", | 20260506 18:31:47.311 | INFO | "huge" : "", | 20260506 18:31:47.311 | INFO | "huge_shared" : " 27 GB", | 20260506 18:31:47.311 | INFO | "malloc" : " 14 GB", | 20260506 18:31:47.311 | INFO | "pinned" : " 29 GB" | 20260506 18:31:47.311 | INFO | } | 20260506 18:31:47.312 | INFO | Perf report for the last 5.13202 seconds: | 20260506 18:31:47.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:31:47.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:47.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:47.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:47.312 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 18:31:47.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:47.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:47.312 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:31:47.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:47.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:47.312 | INFO | Uploaded 36GiB to GPU in 1.95585s (18.4064GiB/s) | 20260506 18:31:47.312 | INFO | Downloaded 34GiB to GPU in 1.91837s (17.7234GiB/s) | 20260506 18:31:52.454 | INFO | Analyzing chunk frames = [3528192,3528704) | 20260506 18:31:52.454 | INFO | MemInfo: { | 20260506 18:31:52.454 | INFO | "gpu" : " 17 GB", | 20260506 18:31:52.454 | INFO | "huge" : "", | 20260506 18:31:52.455 | INFO | "huge_shared" : " 27 GB", | 20260506 18:31:52.455 | INFO | "malloc" : " 14 GB", | 20260506 18:31:52.455 | INFO | "pinned" : " 29 GB" | 20260506 18:31:52.455 | INFO | } | 20260506 18:31:52.455 | INFO | Perf report for the last 5.14316 seconds: | 20260506 18:31:52.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:31:52.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:52.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:52.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:52.455 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:31:52.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:52.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:52.455 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:31:52.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:52.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:52.455 | INFO | Uploaded 36GiB to GPU in 1.93075s (18.6456GiB/s) | 20260506 18:31:52.455 | INFO | Downloaded 34GiB to GPU in 1.86663s (18.2147GiB/s) | 20260506 18:31:57.599 | INFO | Analyzing chunk frames = [3528704,3529216) | 20260506 18:31:57.600 | INFO | MemInfo: { | 20260506 18:31:57.600 | INFO | "gpu" : " 17 GB", | 20260506 18:31:57.600 | INFO | "huge" : "", | 20260506 18:31:57.600 | INFO | "huge_shared" : " 27 GB", | 20260506 18:31:57.600 | INFO | "malloc" : " 14 GB", | 20260506 18:31:57.600 | INFO | "pinned" : " 29 GB" | 20260506 18:31:57.600 | INFO | } | 20260506 18:31:57.600 | INFO | Perf report for the last 5.14512 seconds: | 20260506 18:31:57.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:31:57.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:57.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:57.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:57.600 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 18:31:57.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:57.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:57.600 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:31:57.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:57.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:31:57.600 | INFO | Uploaded 36GiB to GPU in 2.02395s (17.787GiB/s) | 20260506 18:31:57.600 | INFO | Downloaded 34GiB to GPU in 1.91398s (17.764GiB/s) | 20260506 18:32:02.733 | INFO | Analyzing chunk frames = [3529216,3529728) | 20260506 18:32:02.733 | INFO | MemInfo: { | 20260506 18:32:02.733 | INFO | "gpu" : " 17 GB", | 20260506 18:32:02.733 | INFO | "huge" : "", | 20260506 18:32:02.733 | INFO | "huge_shared" : " 27 GB", | 20260506 18:32:02.733 | INFO | "malloc" : " 14 GB", | 20260506 18:32:02.733 | INFO | "pinned" : " 29 GB" | 20260506 18:32:02.733 | INFO | } | 20260506 18:32:02.733 | INFO | Perf report for the last 5.13337 seconds: | 20260506 18:32:02.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:32:02.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:02.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:02.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:02.733 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 18:32:02.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:02.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:02.733 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 18:32:02.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:02.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:02.734 | INFO | Uploaded 36GiB to GPU in 2.07536s (17.3464GiB/s) | 20260506 18:32:02.734 | INFO | Downloaded 34GiB to GPU in 1.96666s (17.2882GiB/s) | 20260506 18:32:07.873 | INFO | Analyzing chunk frames = [3529728,3530240) | 20260506 18:32:07.873 | INFO | MemInfo: { | 20260506 18:32:07.873 | INFO | "gpu" : " 17 GB", | 20260506 18:32:07.873 | INFO | "huge" : "", | 20260506 18:32:07.873 | INFO | "huge_shared" : " 27 GB", | 20260506 18:32:07.873 | INFO | "malloc" : " 14 GB", | 20260506 18:32:07.873 | INFO | "pinned" : " 29 GB" | 20260506 18:32:07.873 | INFO | } | 20260506 18:32:07.873 | INFO | Perf report for the last 5.13941 seconds: | 20260506 18:32:07.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:32:07.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:07.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:07.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:07.873 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 18:32:07.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:07.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:07.873 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:32:07.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:07.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:07.873 | INFO | Uploaded 36GiB to GPU in 2.02242s (17.8005GiB/s) | 20260506 18:32:07.873 | INFO | Downloaded 34GiB to GPU in 1.97299s (17.2327GiB/s) | 20260506 18:32:13.013 | INFO | Analyzing chunk frames = [3530240,3530752) | 20260506 18:32:13.013 | INFO | MemInfo: { | 20260506 18:32:13.013 | INFO | "gpu" : " 17 GB", | 20260506 18:32:13.013 | INFO | "huge" : "", | 20260506 18:32:13.013 | INFO | "huge_shared" : " 27 GB", | 20260506 18:32:13.013 | INFO | "malloc" : " 14 GB", | 20260506 18:32:13.013 | INFO | "pinned" : " 29 GB" | 20260506 18:32:13.013 | INFO | } | 20260506 18:32:13.013 | INFO | Perf report for the last 5.14048 seconds: | 20260506 18:32:13.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:32:13.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:13.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:13.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:13.013 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 18:32:13.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:13.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:13.013 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 18:32:13.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:13.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:13.014 | INFO | Uploaded 36GiB to GPU in 2.03175s (17.7187GiB/s) | 20260506 18:32:13.014 | INFO | Downloaded 34GiB to GPU in 1.988s (17.1026GiB/s) | 20260506 18:32:13.014 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:32:13.014 | INFO | Total mem usage: 18 GiB | 20260506 18:32:13.014 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:32:13.014 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.1%) | 20260506 18:32:13.014 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:32:13.014 | INFO | High Water Breakdown: | 20260506 18:32:13.014 | INFO | 61.7% : 9.82 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:32:13.014 | INFO | 38.2% : 6.07 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:32:13.014 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:32:13.014 | INFO | ------------------------------------------ | 20260506 18:32:13.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:32:13.014 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:32:13.014 | INFO | Total mem usage: 26.5 GiB | 20260506 18:32:13.014 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:32:13.014 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.71%) | 20260506 18:32:13.014 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 18:32:13.014 | INFO | High Water Breakdown: | 20260506 18:32:13.014 | INFO | 56.1% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:32:13.014 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:32:13.014 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:32:13.014 | INFO | 1.6% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:32:13.014 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:32:13.014 | INFO | 0.837% : 219 MiB ( 100% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:32:13.014 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:32:13.014 | INFO | 9.36e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:32:13.014 | INFO | 8.74e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:32:13.014 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:32:13.014 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:32:13.014 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:32:13.014 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:32:13.014 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:32:13.014 | INFO | ------------------------------------------ | 20260506 18:32:13.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:32:13.014 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:32:13.014 | INFO | Total mem usage: 25.6 GiB | 20260506 18:32:13.014 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:32:13.014 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:32:13.014 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:32:13.014 | INFO | High Water Breakdown: | 20260506 18:32:13.014 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:32:13.014 | INFO | ------------------------------------------ | 20260506 18:32:13.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:32:13.014 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:32:13.014 | INFO | Total mem usage: 15.7 GiB | 20260506 18:32:13.014 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:32:13.014 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:32:13.014 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:32:13.014 | INFO | High Water Breakdown: | 20260506 18:32:13.014 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:32:13.014 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:32:13.014 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:32:13.014 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:32:13.014 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:32:13.014 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:32:13.015 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:32:13.015 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:32:13.015 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:32:13.015 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:32:13.015 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:32:13.015 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:32:13.015 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:32:13.015 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:32:13.015 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:32:13.015 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:32:13.015 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:32:13.015 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:32:13.015 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:32:13.015 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:32:13.015 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:32:13.015 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:32:13.015 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:32:13.015 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:32:13.015 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:32:13.015 | INFO | ------------------------------------------ | 20260506 18:32:13.015 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:32:13.015 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:32:13.015 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:32:13.015 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:32:13.015 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.53e+01, 1.34e+01 / 2.29e+00 | 20260506 18:32:13.015 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 9.98e-02, 1.05e-02 / 1.63e-02 | 20260506 18:32:13.015 | INFO | PerfReport: 11.59%, Baseline -- 3.62e+00 / 4.33e+00, 5.41e-01 / 3.91e-01 | 20260506 18:32:13.015 | INFO | PerfReport: 3.24%, Histogram -- 1.01e+00 / 1.42e+00, 1.79e-01 / 1.69e-01 | 20260506 18:32:13.015 | INFO | PerfReport: 15.93%, DME -- 3.99e+01 / 4.43e+01, 4.01e+00 / 2.40e+00 | 20260506 18:32:13.015 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.52e+00 / 1.82e+00 | 20260506 18:32:13.015 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.25e+00 / 2.83e+00, 3.77e-01 / 1.94e-01 | 20260506 18:32:13.015 | INFO | PerfReport: 3.39%, Metrics -- 1.06e+00 / 1.45e+00, 2.76e-01 / 2.12e-01 | 20260506 18:32:13.015 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.67e-01 / 4.17e-01, 5.67e-02 / 2.28e-02 | 20260506 18:32:13.015 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:32:13.015 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:32:13.015 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:32:13.015 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 18:32:13.015 | INFO | PerfReport: 21.58%, Upload -- 2.09e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260506 18:32:13.015 | INFO | PerfReport: 21.12%, Download -- 2.05e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 18:32:13.015 | INFO | PerfReport: 32.23%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 18:32:13.015 | INFO | PerfReport: 1.53%, LockUpload -- 1.48e+00 / 0.00e+00, 5.65e+00 / 0.00e+00 | 20260506 18:32:13.015 | INFO | PerfReport: 1.00%, LockDownload -- 9.66e-01 / 0.00e+00, 2.56e+00 / 0.00e+00 | 20260506 18:32:13.015 | INFO | PerfReport: 22.53%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:32:13.015 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.13e-02 / 0.00e+00 | 20260506 18:32:13.133 | INFO | Starting a round of Baz IO | 20260506 18:32:16.268 | INFO | Populating RTMetrics report | 20260506 18:32:16.268 | INFO | RTMetrics report populated | 20260506 18:32:16.268 | INFO | Opening tmp RTMetrics json file | 20260506 18:32:16.281 | INFO | Serializing RTMetrics to json | 20260506 18:32:16.294 | INFO | Serialization done, swapping tmp file now... | 20260506 18:32:16.308 | INFO | RTMetrics done | 20260506 18:32:16.590 | INFO | Finished a round of Baz IO: Wrote 2.038 GiB in 3.456 seconds | 20260506 18:32:16.590 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.03% / 0.01% / 64.96% | 20260506 18:32:16.590 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.1031% / 7.94729e-06% / 7.15256e-05% | 20260506 18:32:16.590 | INFO | Baz IO overall: | 20260506 18:32:16.590 | INFO | BytesWritten : 1865288188244 | 20260506 18:32:16.590 | INFO | NumEvents : 902045626485 | 20260506 18:32:18.151 | INFO | Analyzing chunk frames = [3530752,3531264) | 20260506 18:32:18.151 | INFO | MemInfo: { | 20260506 18:32:18.151 | INFO | "gpu" : " 17 GB", | 20260506 18:32:18.151 | INFO | "huge" : "", | 20260506 18:32:18.151 | INFO | "huge_shared" : " 27 GB", | 20260506 18:32:18.151 | INFO | "malloc" : " 14 GB", | 20260506 18:32:18.151 | INFO | "pinned" : " 29 GB" | 20260506 18:32:18.151 | INFO | } | 20260506 18:32:18.151 | INFO | Perf report for the last 5.13836 seconds: | 20260506 18:32:18.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:32:18.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:18.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:18.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:18.152 | INFO | ANALYSIS : 62.8%, 2.9, 95.5ms, 0 | 20260506 18:32:18.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:18.152 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 18:32:18.152 | INFO | PRE_HQ : 60.3%, 1.9, 57.9ms, 0 | 20260506 18:32:18.152 | INFO | BAZWRITER : 66.9%, 4.8, 165.3ms, 0 | 20260506 18:32:18.152 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 18:32:18.152 | INFO | Uploaded 36GiB to GPU in 2.06362s (17.4451GiB/s) | 20260506 18:32:18.152 | INFO | Downloaded 38GiB to GPU in 2.22279s (17.0956GiB/s) | 20260506 18:32:23.290 | INFO | Analyzing chunk frames = [3531264,3531776) | 20260506 18:32:23.290 | INFO | MemInfo: { | 20260506 18:32:23.290 | INFO | "gpu" : " 17 GB", | 20260506 18:32:23.290 | INFO | "huge" : "", | 20260506 18:32:23.290 | INFO | "huge_shared" : " 27 GB", | 20260506 18:32:23.290 | INFO | "malloc" : " 14 GB", | 20260506 18:32:23.290 | INFO | "pinned" : " 29 GB" | 20260506 18:32:23.290 | INFO | } | 20260506 18:32:23.290 | INFO | Perf report for the last 5.13847 seconds: | 20260506 18:32:23.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:32:23.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:23.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:23.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:23.290 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:32:23.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:23.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:23.290 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260506 18:32:23.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:23.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:23.290 | INFO | Uploaded 36GiB to GPU in 2.01366s (17.8779GiB/s) | 20260506 18:32:23.290 | INFO | Downloaded 34GiB to GPU in 1.93744s (17.549GiB/s) | 20260506 18:32:28.433 | INFO | Analyzing chunk frames = [3531776,3532288) | 20260506 18:32:28.433 | INFO | MemInfo: { | 20260506 18:32:28.433 | INFO | "gpu" : " 17 GB", | 20260506 18:32:28.433 | INFO | "huge" : "", | 20260506 18:32:28.433 | INFO | "huge_shared" : " 27 GB", | 20260506 18:32:28.433 | INFO | "malloc" : " 14 GB", | 20260506 18:32:28.433 | INFO | "pinned" : " 29 GB" | 20260506 18:32:28.433 | INFO | } | 20260506 18:32:28.433 | INFO | Perf report for the last 5.1428 seconds: | 20260506 18:32:28.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:32:28.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:28.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:28.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:28.433 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 18:32:28.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:28.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:28.433 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:32:28.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:28.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:28.433 | INFO | Uploaded 36GiB to GPU in 1.98697s (18.118GiB/s) | 20260506 18:32:28.433 | INFO | Downloaded 34GiB to GPU in 1.92205s (17.6895GiB/s) | 20260506 18:32:33.571 | INFO | Analyzing chunk frames = [3532288,3532800) | 20260506 18:32:33.571 | INFO | MemInfo: { | 20260506 18:32:33.571 | INFO | "gpu" : " 17 GB", | 20260506 18:32:33.571 | INFO | "huge" : "", | 20260506 18:32:33.571 | INFO | "huge_shared" : " 27 GB", | 20260506 18:32:33.571 | INFO | "malloc" : " 14 GB", | 20260506 18:32:33.571 | INFO | "pinned" : " 29 GB" | 20260506 18:32:33.571 | INFO | } | 20260506 18:32:33.571 | INFO | Perf report for the last 5.13815 seconds: | 20260506 18:32:33.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:32:33.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:33.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:33.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:33.571 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:32:33.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:33.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:33.571 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:32:33.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:33.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:33.571 | INFO | Uploaded 36GiB to GPU in 1.93755s (18.5802GiB/s) | 20260506 18:32:33.571 | INFO | Downloaded 34GiB to GPU in 1.86702s (18.2108GiB/s) | 20260506 18:32:38.713 | INFO | Analyzing chunk frames = [3532800,3533312) | 20260506 18:32:38.713 | INFO | MemInfo: { | 20260506 18:32:38.713 | INFO | "gpu" : " 17 GB", | 20260506 18:32:38.713 | INFO | "huge" : "", | 20260506 18:32:38.713 | INFO | "huge_shared" : " 27 GB", | 20260506 18:32:38.713 | INFO | "malloc" : " 14 GB", | 20260506 18:32:38.713 | INFO | "pinned" : " 29 GB" | 20260506 18:32:38.713 | INFO | } | 20260506 18:32:38.713 | INFO | Perf report for the last 5.14235 seconds: | 20260506 18:32:38.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:32:38.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:38.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:38.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:38.713 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 18:32:38.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:38.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:38.713 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:32:38.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:38.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:38.714 | INFO | Uploaded 36GiB to GPU in 2.01126s (17.8992GiB/s) | 20260506 18:32:38.714 | INFO | Downloaded 34GiB to GPU in 1.90438s (17.8535GiB/s) | 20260506 18:32:43.858 | INFO | Analyzing chunk frames = [3533312,3533824) | 20260506 18:32:43.858 | INFO | MemInfo: { | 20260506 18:32:43.858 | INFO | "gpu" : " 17 GB", | 20260506 18:32:43.858 | INFO | "huge" : "", | 20260506 18:32:43.858 | INFO | "huge_shared" : " 27 GB", | 20260506 18:32:43.858 | INFO | "malloc" : " 14 GB", | 20260506 18:32:43.858 | INFO | "pinned" : " 29 GB" | 20260506 18:32:43.858 | INFO | } | 20260506 18:32:43.858 | INFO | Perf report for the last 5.14434 seconds: | 20260506 18:32:43.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:32:43.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:43.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:43.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:43.858 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 18:32:43.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:43.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:43.858 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:32:43.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:43.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:43.858 | INFO | Uploaded 36GiB to GPU in 2.00169s (17.9848GiB/s) | 20260506 18:32:43.858 | INFO | Downloaded 34GiB to GPU in 1.94117s (17.5152GiB/s) | 20260506 18:32:48.990 | INFO | Analyzing chunk frames = [3533824,3534336) | 20260506 18:32:48.990 | INFO | MemInfo: { | 20260506 18:32:48.990 | INFO | "gpu" : " 17 GB", | 20260506 18:32:48.990 | INFO | "huge" : "", | 20260506 18:32:48.990 | INFO | "huge_shared" : " 27 GB", | 20260506 18:32:48.990 | INFO | "malloc" : " 14 GB", | 20260506 18:32:48.990 | INFO | "pinned" : " 29 GB" | 20260506 18:32:48.990 | INFO | } | 20260506 18:32:48.990 | INFO | Perf report for the last 5.1324 seconds: | 20260506 18:32:48.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:32:48.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:48.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:48.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:48.990 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 18:32:48.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:48.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:48.990 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:32:48.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:48.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:48.990 | INFO | Uploaded 36GiB to GPU in 1.97255s (18.2505GiB/s) | 20260506 18:32:48.990 | INFO | Downloaded 34GiB to GPU in 1.99876s (17.0105GiB/s) | 20260506 18:32:54.128 | INFO | Analyzing chunk frames = [3534336,3534848) | 20260506 18:32:54.128 | INFO | MemInfo: { | 20260506 18:32:54.128 | INFO | "gpu" : " 17 GB", | 20260506 18:32:54.128 | INFO | "huge" : "", | 20260506 18:32:54.128 | INFO | "huge_shared" : " 27 GB", | 20260506 18:32:54.128 | INFO | "malloc" : " 14 GB", | 20260506 18:32:54.128 | INFO | "pinned" : " 29 GB" | 20260506 18:32:54.128 | INFO | } | 20260506 18:32:54.128 | INFO | Perf report for the last 5.13834 seconds: | 20260506 18:32:54.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:32:54.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:54.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:54.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:54.129 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 18:32:54.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:54.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:54.129 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 18:32:54.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:54.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:54.129 | INFO | Uploaded 36GiB to GPU in 2.04433s (17.6097GiB/s) | 20260506 18:32:54.129 | INFO | Downloaded 34GiB to GPU in 1.98249s (17.1502GiB/s) | 20260506 18:32:54.247 | INFO | Starting a round of Baz IO | 20260506 18:32:57.382 | INFO | Populating RTMetrics report | 20260506 18:32:57.382 | INFO | RTMetrics report populated | 20260506 18:32:57.382 | INFO | Opening tmp RTMetrics json file | 20260506 18:32:57.397 | INFO | Serializing RTMetrics to json | 20260506 18:32:57.399 | INFO | Serialization done, swapping tmp file now... | 20260506 18:32:57.400 | INFO | RTMetrics done | 20260506 18:32:57.866 | INFO | Finished a round of Baz IO: Wrote 2.037 GiB in 3.618 seconds | 20260506 18:32:57.866 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.03% / 0.01% / 64.97% | 20260506 18:32:57.866 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.09878% / 1.98682e-05% / 4.37101e-05% | 20260506 18:32:57.866 | INFO | Baz IO overall: | 20260506 18:32:57.866 | INFO | BytesWritten : 1867475881213 | 20260506 18:32:57.866 | INFO | NumEvents : 903099477781 | 20260506 18:32:59.267 | INFO | Analyzing chunk frames = [3534848,3535360) | 20260506 18:32:59.267 | INFO | MemInfo: { | 20260506 18:32:59.267 | INFO | "gpu" : " 17 GB", | 20260506 18:32:59.267 | INFO | "huge" : "", | 20260506 18:32:59.267 | INFO | "huge_shared" : " 27 GB", | 20260506 18:32:59.267 | INFO | "malloc" : " 14 GB", | 20260506 18:32:59.267 | INFO | "pinned" : " 29 GB" | 20260506 18:32:59.267 | INFO | } | 20260506 18:32:59.267 | INFO | Perf report for the last 5.13864 seconds: | 20260506 18:32:59.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:32:59.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:59.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:59.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:59.267 | INFO | ANALYSIS : 62.7%, 2.9, 95.5ms, 0 | 20260506 18:32:59.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:32:59.267 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 18:32:59.267 | INFO | PRE_HQ : 59.9%, 1.9, 57.8ms, 0 | 20260506 18:32:59.267 | INFO | BAZWRITER : 69.9%, 4.8, 173.3ms, 0 | 20260506 18:32:59.267 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.3ms, 0 | 20260506 18:32:59.267 | INFO | Uploaded 36GiB to GPU in 2.04629s (17.5928GiB/s) | 20260506 18:32:59.267 | INFO | Downloaded 38GiB to GPU in 2.23153s (17.0287GiB/s) | 20260506 18:33:04.404 | INFO | Analyzing chunk frames = [3535360,3535872) | 20260506 18:33:04.404 | INFO | MemInfo: { | 20260506 18:33:04.404 | INFO | "gpu" : " 17 GB", | 20260506 18:33:04.404 | INFO | "huge" : "", | 20260506 18:33:04.404 | INFO | "huge_shared" : " 27 GB", | 20260506 18:33:04.404 | INFO | "malloc" : " 14 GB", | 20260506 18:33:04.404 | INFO | "pinned" : " 29 GB" | 20260506 18:33:04.404 | INFO | } | 20260506 18:33:04.404 | INFO | Perf report for the last 5.13694 seconds: | 20260506 18:33:04.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:33:04.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:04.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:04.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:04.404 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 18:33:04.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:04.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:04.404 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:33:04.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:04.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:04.404 | INFO | Uploaded 36GiB to GPU in 1.98717s (18.1162GiB/s) | 20260506 18:33:04.405 | INFO | Downloaded 34GiB to GPU in 1.92328s (17.6781GiB/s) | 20260506 18:33:09.544 | INFO | Analyzing chunk frames = [3535872,3536384) | 20260506 18:33:09.544 | INFO | MemInfo: { | 20260506 18:33:09.544 | INFO | "gpu" : " 17 GB", | 20260506 18:33:09.544 | INFO | "huge" : "", | 20260506 18:33:09.545 | INFO | "huge_shared" : " 27 GB", | 20260506 18:33:09.545 | INFO | "malloc" : " 14 GB", | 20260506 18:33:09.545 | INFO | "pinned" : " 29 GB" | 20260506 18:33:09.545 | INFO | } | 20260506 18:33:09.545 | INFO | Perf report for the last 5.14068 seconds: | 20260506 18:33:09.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:33:09.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:09.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:09.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:09.545 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:33:09.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:09.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:09.545 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:33:09.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:09.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:09.545 | INFO | Uploaded 36GiB to GPU in 1.97466s (18.231GiB/s) | 20260506 18:33:09.545 | INFO | Downloaded 34GiB to GPU in 1.91936s (17.7143GiB/s) | 20260506 18:33:14.682 | INFO | Analyzing chunk frames = [3536384,3536896) | 20260506 18:33:14.682 | INFO | MemInfo: { | 20260506 18:33:14.682 | INFO | "gpu" : " 17 GB", | 20260506 18:33:14.682 | INFO | "huge" : "", | 20260506 18:33:14.682 | INFO | "huge_shared" : " 27 GB", | 20260506 18:33:14.682 | INFO | "malloc" : " 14 GB", | 20260506 18:33:14.682 | INFO | "pinned" : " 29 GB" | 20260506 18:33:14.682 | INFO | } | 20260506 18:33:14.682 | INFO | Perf report for the last 5.13729 seconds: | 20260506 18:33:14.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:33:14.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:14.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:14.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:14.682 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 18:33:14.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:14.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:14.682 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:33:14.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:14.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:14.682 | INFO | Uploaded 36GiB to GPU in 1.92723s (18.6796GiB/s) | 20260506 18:33:14.682 | INFO | Downloaded 34GiB to GPU in 1.86915s (18.1901GiB/s) | 20260506 18:33:19.822 | INFO | Analyzing chunk frames = [3536896,3537408) | 20260506 18:33:19.822 | INFO | MemInfo: { | 20260506 18:33:19.822 | INFO | "gpu" : " 17 GB", | 20260506 18:33:19.822 | INFO | "huge" : "", | 20260506 18:33:19.822 | INFO | "huge_shared" : " 27 GB", | 20260506 18:33:19.823 | INFO | "malloc" : " 14 GB", | 20260506 18:33:19.823 | INFO | "pinned" : " 29 GB" | 20260506 18:33:19.823 | INFO | } | 20260506 18:33:19.823 | INFO | Perf report for the last 5.14054 seconds: | 20260506 18:33:19.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:33:19.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:19.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:19.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:19.823 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260506 18:33:19.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:19.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:19.823 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:33:19.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:19.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:19.823 | INFO | Uploaded 36GiB to GPU in 1.93851s (18.571GiB/s) | 20260506 18:33:19.823 | INFO | Downloaded 34GiB to GPU in 1.87309s (18.1518GiB/s) | 20260506 18:33:24.964 | INFO | Analyzing chunk frames = [3537408,3537920) | 20260506 18:33:24.964 | INFO | MemInfo: { | 20260506 18:33:24.964 | INFO | "gpu" : " 17 GB", | 20260506 18:33:24.964 | INFO | "huge" : "", | 20260506 18:33:24.964 | INFO | "huge_shared" : " 27 GB", | 20260506 18:33:24.964 | INFO | "malloc" : " 14 GB", | 20260506 18:33:24.964 | INFO | "pinned" : " 29 GB" | 20260506 18:33:24.964 | INFO | } | 20260506 18:33:24.964 | INFO | Perf report for the last 5.14135 seconds: | 20260506 18:33:24.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:33:24.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:24.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:24.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:24.964 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 18:33:24.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:24.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:24.964 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:33:24.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:24.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:24.964 | INFO | Uploaded 36GiB to GPU in 2.02493s (17.7784GiB/s) | 20260506 18:33:24.964 | INFO | Downloaded 34GiB to GPU in 1.96373s (17.314GiB/s) | 20260506 18:33:30.109 | INFO | Analyzing chunk frames = [3537920,3538432) | 20260506 18:33:30.109 | INFO | MemInfo: { | 20260506 18:33:30.109 | INFO | "gpu" : " 17 GB", | 20260506 18:33:30.109 | INFO | "huge" : "", | 20260506 18:33:30.109 | INFO | "huge_shared" : " 27 GB", | 20260506 18:33:30.109 | INFO | "malloc" : " 14 GB", | 20260506 18:33:30.109 | INFO | "pinned" : " 29 GB" | 20260506 18:33:30.109 | INFO | } | 20260506 18:33:30.109 | INFO | Perf report for the last 5.14525 seconds: | 20260506 18:33:30.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:33:30.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:30.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:30.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:30.109 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:33:30.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:30.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:30.109 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 18:33:30.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:30.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:30.110 | INFO | Uploaded 36GiB to GPU in 1.98177s (18.1656GiB/s) | 20260506 18:33:30.110 | INFO | Downloaded 34GiB to GPU in 2.00819s (16.9306GiB/s) | 20260506 18:33:35.245 | INFO | Analyzing chunk frames = [3538432,3538944) | 20260506 18:33:35.245 | INFO | MemInfo: { | 20260506 18:33:35.245 | INFO | "gpu" : " 17 GB", | 20260506 18:33:35.245 | INFO | "huge" : "", | 20260506 18:33:35.245 | INFO | "huge_shared" : " 27 GB", | 20260506 18:33:35.245 | INFO | "malloc" : " 14 GB", | 20260506 18:33:35.245 | INFO | "pinned" : " 29 GB" | 20260506 18:33:35.245 | INFO | } | 20260506 18:33:35.245 | INFO | Perf report for the last 5.13565 seconds: | 20260506 18:33:35.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:33:35.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:35.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:35.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:35.245 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 18:33:35.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:35.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:35.245 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:33:35.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:35.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:35.245 | INFO | Uploaded 36GiB to GPU in 2.04086s (17.6396GiB/s) | 20260506 18:33:35.245 | INFO | Downloaded 34GiB to GPU in 1.97867s (17.1832GiB/s) | 20260506 18:33:35.245 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:33:35.245 | INFO | Total mem usage: 18 GiB | 20260506 18:33:35.245 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:33:35.245 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -13.9%) | 20260506 18:33:35.245 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:33:35.245 | INFO | High Water Breakdown: | 20260506 18:33:35.245 | INFO | 61.6% : 9.74 GiB ( 85.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:33:35.245 | INFO | 38.3% : 6.05 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:33:35.245 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:33:35.245 | INFO | ------------------------------------------ | 20260506 18:33:35.246 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:33:35.246 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:33:35.246 | INFO | Total mem usage: 26.5 GiB | 20260506 18:33:35.246 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:33:35.246 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:33:35.246 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:33:35.246 | INFO | High Water Breakdown: | 20260506 18:33:35.246 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:33:35.246 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:33:35.246 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:33:35.246 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:33:35.246 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:33:35.246 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:33:35.246 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:33:35.246 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:33:35.246 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:33:35.246 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:33:35.246 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:33:35.246 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:33:35.246 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:33:35.246 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:33:35.246 | INFO | ------------------------------------------ | 20260506 18:33:35.246 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:33:35.246 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:33:35.246 | INFO | Total mem usage: 25.6 GiB | 20260506 18:33:35.246 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:33:35.246 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:33:35.246 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:33:35.246 | INFO | High Water Breakdown: | 20260506 18:33:35.246 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:33:35.246 | INFO | ------------------------------------------ | 20260506 18:33:35.246 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:33:35.246 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:33:35.246 | INFO | Total mem usage: 15.7 GiB | 20260506 18:33:35.246 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:33:35.246 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:33:35.246 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:33:35.246 | INFO | High Water Breakdown: | 20260506 18:33:35.246 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:33:35.246 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:33:35.246 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:33:35.246 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:33:35.246 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:33:35.246 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:33:35.246 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:33:35.246 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:33:35.246 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:33:35.246 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:33:35.246 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:33:35.246 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:33:35.246 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:33:35.246 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:33:35.246 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:33:35.246 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:33:35.246 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:33:35.246 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:33:35.246 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:33:35.246 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:33:35.246 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:33:35.246 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:33:35.246 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:33:35.246 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:33:35.246 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:33:35.246 | INFO | ------------------------------------------ | 20260506 18:33:35.246 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:33:35.246 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:33:35.246 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:33:35.246 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:33:35.246 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.57e+01, 1.36e+01 / 2.50e+00 | 20260506 18:33:35.246 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.02e-01, 1.00e-02 / 5.45e-03 | 20260506 18:33:35.246 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.43e+00, 5.62e-01 / 3.45e-01 | 20260506 18:33:35.246 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.83e-01 / 1.28e-01 | 20260506 18:33:35.246 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.49e+01, 4.36e+00 / 2.51e+00 | 20260506 18:33:35.246 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.79e+01 / 2.02e+01, 1.45e+00 / 1.73e+00 | 20260506 18:33:35.246 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.26e+00 / 2.79e+00, 3.82e-01 / 2.23e-01 | 20260506 18:33:35.246 | INFO | PerfReport: 3.39%, Metrics -- 1.06e+00 / 1.40e+00, 2.76e-01 / 1.61e-01 | 20260506 18:33:35.246 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.67e-01 / 4.12e-01, 5.80e-02 / 1.73e-02 | 20260506 18:33:35.247 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:33:35.247 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:33:35.247 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:33:35.247 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 18:33:35.247 | INFO | PerfReport: 21.44%, Upload -- 2.08e+01 / 0.00e+00, 6.91e+00 / 0.00e+00 | 20260506 18:33:35.247 | INFO | PerfReport: 21.17%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 18:33:35.247 | INFO | PerfReport: 32.26%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 18:33:35.247 | INFO | PerfReport: 1.33%, LockUpload -- 1.29e+00 / 0.00e+00, 5.33e+00 / 0.00e+00 | 20260506 18:33:35.247 | INFO | PerfReport: 1.18%, LockDownload -- 1.14e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 18:33:35.247 | INFO | PerfReport: 22.61%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:33:35.247 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260506 18:33:35.364 | INFO | Starting a round of Baz IO | 20260506 18:33:38.500 | INFO | Populating RTMetrics report | 20260506 18:33:38.501 | INFO | RTMetrics report populated | 20260506 18:33:38.501 | INFO | Opening tmp RTMetrics json file | 20260506 18:33:38.504 | INFO | Serializing RTMetrics to json | 20260506 18:33:38.505 | INFO | Serialization done, swapping tmp file now... | 20260506 18:33:38.506 | INFO | RTMetrics done | 20260506 18:33:39.013 | INFO | Finished a round of Baz IO: Wrote 2.037 GiB in 3.648 seconds | 20260506 18:33:39.013 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.02% / 0.01% / 64.97% | 20260506 18:33:39.013 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.09838% / 7.94729e-06% / 7.54992e-05% | 20260506 18:33:39.013 | INFO | Baz IO overall: | 20260506 18:33:39.013 | INFO | BytesWritten : 1869662666010 | 20260506 18:33:39.013 | INFO | NumEvents : 904152802035 | 20260506 18:33:40.382 | INFO | Analyzing chunk frames = [3538944,3539456) | 20260506 18:33:40.382 | INFO | MemInfo: { | 20260506 18:33:40.382 | INFO | "gpu" : " 17 GB", | 20260506 18:33:40.382 | INFO | "huge" : "", | 20260506 18:33:40.382 | INFO | "huge_shared" : " 27 GB", | 20260506 18:33:40.382 | INFO | "malloc" : " 14 GB", | 20260506 18:33:40.382 | INFO | "pinned" : " 29 GB" | 20260506 18:33:40.382 | INFO | } | 20260506 18:33:40.382 | INFO | Perf report for the last 5.13705 seconds: | 20260506 18:33:40.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:33:40.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:40.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:40.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:40.382 | INFO | ANALYSIS : 62.8%, 2.9, 95.5ms, 0 | 20260506 18:33:40.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:40.382 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260506 18:33:40.382 | INFO | PRE_HQ : 60.1%, 1.9, 57.8ms, 0 | 20260506 18:33:40.382 | INFO | BAZWRITER : 70.7%, 4.8, 173.7ms, 0 | 20260506 18:33:40.382 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260506 18:33:40.382 | INFO | Uploaded 36GiB to GPU in 2.08672s (17.252GiB/s) | 20260506 18:33:40.382 | INFO | Downloaded 38GiB to GPU in 2.22138s (17.1065GiB/s) | 20260506 18:33:45.522 | INFO | Analyzing chunk frames = [3539456,3539968) | 20260506 18:33:45.522 | INFO | MemInfo: { | 20260506 18:33:45.522 | INFO | "gpu" : " 17 GB", | 20260506 18:33:45.522 | INFO | "huge" : "", | 20260506 18:33:45.522 | INFO | "huge_shared" : " 27 GB", | 20260506 18:33:45.522 | INFO | "malloc" : " 14 GB", | 20260506 18:33:45.522 | INFO | "pinned" : " 29 GB" | 20260506 18:33:45.522 | INFO | } | 20260506 18:33:45.522 | INFO | Perf report for the last 5.14019 seconds: | 20260506 18:33:45.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:33:45.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:45.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:45.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:45.522 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:33:45.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:45.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:45.522 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:33:45.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:45.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:45.522 | INFO | Uploaded 36GiB to GPU in 2.01866s (17.8336GiB/s) | 20260506 18:33:45.523 | INFO | Downloaded 34GiB to GPU in 1.94049s (17.5214GiB/s) | 20260506 18:33:50.656 | INFO | Analyzing chunk frames = [3539968,3540480) | 20260506 18:33:50.656 | INFO | MemInfo: { | 20260506 18:33:50.656 | INFO | "gpu" : " 17 GB", | 20260506 18:33:50.656 | INFO | "huge" : "", | 20260506 18:33:50.656 | INFO | "huge_shared" : " 27 GB", | 20260506 18:33:50.656 | INFO | "malloc" : " 14 GB", | 20260506 18:33:50.656 | INFO | "pinned" : " 29 GB" | 20260506 18:33:50.656 | INFO | } | 20260506 18:33:50.656 | INFO | Perf report for the last 5.13419 seconds: | 20260506 18:33:50.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:33:50.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:50.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:50.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:50.656 | INFO | ANALYSIS : 61.1%, 3.0, 94.4ms, 0 | 20260506 18:33:50.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:50.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:50.656 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:33:50.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:50.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:50.657 | INFO | Uploaded 36GiB to GPU in 1.98351s (18.1497GiB/s) | 20260506 18:33:50.657 | INFO | Downloaded 34GiB to GPU in 1.9207s (17.7019GiB/s) | 20260506 18:33:55.797 | INFO | Analyzing chunk frames = [3540480,3540992) | 20260506 18:33:55.797 | INFO | MemInfo: { | 20260506 18:33:55.797 | INFO | "gpu" : " 17 GB", | 20260506 18:33:55.797 | INFO | "huge" : "", | 20260506 18:33:55.797 | INFO | "huge_shared" : " 27 GB", | 20260506 18:33:55.797 | INFO | "malloc" : " 14 GB", | 20260506 18:33:55.797 | INFO | "pinned" : " 29 GB" | 20260506 18:33:55.797 | INFO | } | 20260506 18:33:55.797 | INFO | Perf report for the last 5.14097 seconds: | 20260506 18:33:55.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:33:55.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:55.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:55.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:55.797 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 18:33:55.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:55.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:55.797 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:33:55.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:55.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:33:55.798 | INFO | Uploaded 36GiB to GPU in 1.92939s (18.6588GiB/s) | 20260506 18:33:55.798 | INFO | Downloaded 34GiB to GPU in 1.87278s (18.1548GiB/s) | 20260506 18:34:00.943 | INFO | Analyzing chunk frames = [3540992,3541504) | 20260506 18:34:00.943 | INFO | MemInfo: { | 20260506 18:34:00.943 | INFO | "gpu" : " 17 GB", | 20260506 18:34:00.943 | INFO | "huge" : "", | 20260506 18:34:00.943 | INFO | "huge_shared" : " 27 GB", | 20260506 18:34:00.943 | INFO | "malloc" : " 14 GB", | 20260506 18:34:00.943 | INFO | "pinned" : " 29 GB" | 20260506 18:34:00.943 | INFO | } | 20260506 18:34:00.943 | INFO | Perf report for the last 5.14591 seconds: | 20260506 18:34:00.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:34:00.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:00.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:00.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:00.943 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 18:34:00.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:00.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:00.943 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:34:00.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:00.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:00.944 | INFO | Uploaded 36GiB to GPU in 2.02371s (17.7892GiB/s) | 20260506 18:34:00.944 | INFO | Downloaded 34GiB to GPU in 1.91486s (17.7558GiB/s) | 20260506 18:34:06.078 | INFO | Analyzing chunk frames = [3541504,3542016) | 20260506 18:34:06.078 | INFO | MemInfo: { | 20260506 18:34:06.078 | INFO | "gpu" : " 17 GB", | 20260506 18:34:06.078 | INFO | "huge" : "", | 20260506 18:34:06.078 | INFO | "huge_shared" : " 27 GB", | 20260506 18:34:06.078 | INFO | "malloc" : " 14 GB", | 20260506 18:34:06.078 | INFO | "pinned" : " 29 GB" | 20260506 18:34:06.078 | INFO | } | 20260506 18:34:06.078 | INFO | Perf report for the last 5.13497 seconds: | 20260506 18:34:06.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:34:06.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:06.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:06.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:06.078 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 18:34:06.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:06.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:06.078 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:34:06.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:06.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:06.078 | INFO | Uploaded 36GiB to GPU in 2.0614s (17.4638GiB/s) | 20260506 18:34:06.078 | INFO | Downloaded 34GiB to GPU in 1.9597s (17.3496GiB/s) | 20260506 18:34:11.222 | INFO | Analyzing chunk frames = [3542016,3542528) | 20260506 18:34:11.222 | INFO | MemInfo: { | 20260506 18:34:11.222 | INFO | "gpu" : " 17 GB", | 20260506 18:34:11.222 | INFO | "huge" : "", | 20260506 18:34:11.222 | INFO | "huge_shared" : " 27 GB", | 20260506 18:34:11.222 | INFO | "malloc" : " 14 GB", | 20260506 18:34:11.222 | INFO | "pinned" : " 29 GB" | 20260506 18:34:11.222 | INFO | } | 20260506 18:34:11.222 | INFO | Perf report for the last 5.14387 seconds: | 20260506 18:34:11.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:34:11.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:11.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:11.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:11.222 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:34:11.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:11.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:11.222 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 18:34:11.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:11.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:11.223 | INFO | Uploaded 36GiB to GPU in 2.01939s (17.8272GiB/s) | 20260506 18:34:11.223 | INFO | Downloaded 34GiB to GPU in 1.98807s (17.102GiB/s) | 20260506 18:34:16.362 | INFO | Analyzing chunk frames = [3542528,3543040) | 20260506 18:34:16.362 | INFO | MemInfo: { | 20260506 18:34:16.362 | INFO | "gpu" : " 17 GB", | 20260506 18:34:16.362 | INFO | "huge" : "", | 20260506 18:34:16.362 | INFO | "huge_shared" : " 27 GB", | 20260506 18:34:16.362 | INFO | "malloc" : " 14 GB", | 20260506 18:34:16.362 | INFO | "pinned" : " 29 GB" | 20260506 18:34:16.362 | INFO | } | 20260506 18:34:16.362 | INFO | Perf report for the last 5.13955 seconds: | 20260506 18:34:16.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:34:16.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:16.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:16.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:16.362 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 18:34:16.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:16.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:16.362 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 18:34:16.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:16.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:16.362 | INFO | Uploaded 36GiB to GPU in 2.06403s (17.4416GiB/s) | 20260506 18:34:16.362 | INFO | Downloaded 34GiB to GPU in 1.99057s (17.0805GiB/s) | 20260506 18:34:16.480 | INFO | Starting a round of Baz IO | 20260506 18:34:19.618 | INFO | Populating RTMetrics report | 20260506 18:34:19.618 | INFO | RTMetrics report populated | 20260506 18:34:19.618 | INFO | Opening tmp RTMetrics json file | 20260506 18:34:19.639 | INFO | Serializing RTMetrics to json | 20260506 18:34:19.681 | INFO | Serialization done, swapping tmp file now... | 20260506 18:34:19.692 | INFO | RTMetrics done | 20260506 18:34:20.192 | INFO | Finished a round of Baz IO: Wrote 2.036 GiB in 3.712 seconds | 20260506 18:34:20.192 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.02% / 0.01% / 64.98% | 20260506 18:34:20.192 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.09535% / 3.97364e-06% / 5.96046e-05% | 20260506 18:34:20.192 | INFO | Baz IO overall: | 20260506 18:34:20.192 | INFO | BytesWritten : 1871848308811 | 20260506 18:34:20.192 | INFO | NumEvents : 905205302330 | 20260506 18:34:21.496 | INFO | Analyzing chunk frames = [3543040,3543552) | 20260506 18:34:21.496 | INFO | MemInfo: { | 20260506 18:34:21.496 | INFO | "gpu" : " 17 GB", | 20260506 18:34:21.496 | INFO | "huge" : "", | 20260506 18:34:21.496 | INFO | "huge_shared" : " 27 GB", | 20260506 18:34:21.496 | INFO | "malloc" : " 14 GB", | 20260506 18:34:21.496 | INFO | "pinned" : " 29 GB" | 20260506 18:34:21.496 | INFO | } | 20260506 18:34:21.496 | INFO | Perf report for the last 5.13422 seconds: | 20260506 18:34:21.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:34:21.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:21.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:21.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:21.496 | INFO | ANALYSIS : 62.8%, 2.9, 95.9ms, 0 | 20260506 18:34:21.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:21.496 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 18:34:21.496 | INFO | PRE_HQ : 60.1%, 1.8, 57.5ms, 0 | 20260506 18:34:21.496 | INFO | BAZWRITER : 72.0%, 4.8, 177.8ms, 0 | 20260506 18:34:21.496 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 18:34:21.496 | INFO | Uploaded 36GiB to GPU in 2.05536s (17.5152GiB/s) | 20260506 18:34:21.496 | INFO | Downloaded 38GiB to GPU in 2.21598s (17.1482GiB/s) | 20260506 18:34:26.632 | INFO | Analyzing chunk frames = [3543552,3544064) | 20260506 18:34:26.632 | INFO | MemInfo: { | 20260506 18:34:26.632 | INFO | "gpu" : " 17 GB", | 20260506 18:34:26.633 | INFO | "huge" : "", | 20260506 18:34:26.633 | INFO | "huge_shared" : " 27 GB", | 20260506 18:34:26.633 | INFO | "malloc" : " 14 GB", | 20260506 18:34:26.633 | INFO | "pinned" : " 29 GB" | 20260506 18:34:26.633 | INFO | } | 20260506 18:34:26.633 | INFO | Perf report for the last 5.13669 seconds: | 20260506 18:34:26.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:34:26.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:26.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:26.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:26.633 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 18:34:26.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:26.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:26.633 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:34:26.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:26.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:26.633 | INFO | Uploaded 36GiB to GPU in 2.01818s (17.8378GiB/s) | 20260506 18:34:26.633 | INFO | Downloaded 34GiB to GPU in 1.93071s (17.6101GiB/s) | 20260506 18:34:31.771 | INFO | Analyzing chunk frames = [3544064,3544576) | 20260506 18:34:31.771 | INFO | MemInfo: { | 20260506 18:34:31.771 | INFO | "gpu" : " 17 GB", | 20260506 18:34:31.771 | INFO | "huge" : "", | 20260506 18:34:31.771 | INFO | "huge_shared" : " 27 GB", | 20260506 18:34:31.771 | INFO | "malloc" : " 14 GB", | 20260506 18:34:31.771 | INFO | "pinned" : " 29 GB" | 20260506 18:34:31.771 | INFO | } | 20260506 18:34:31.771 | INFO | Perf report for the last 5.13874 seconds: | 20260506 18:34:31.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:34:31.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:31.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:31.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:31.771 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 18:34:31.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:31.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:31.772 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:34:31.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:31.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:31.772 | INFO | Uploaded 36GiB to GPU in 1.97846s (18.1959GiB/s) | 20260506 18:34:31.772 | INFO | Downloaded 34GiB to GPU in 1.9239s (17.6724GiB/s) | 20260506 18:34:36.913 | INFO | Analyzing chunk frames = [3544576,3545088) | 20260506 18:34:36.913 | INFO | MemInfo: { | 20260506 18:34:36.913 | INFO | "gpu" : " 17 GB", | 20260506 18:34:36.913 | INFO | "huge" : "", | 20260506 18:34:36.913 | INFO | "huge_shared" : " 27 GB", | 20260506 18:34:36.914 | INFO | "malloc" : " 14 GB", | 20260506 18:34:36.914 | INFO | "pinned" : " 29 GB" | 20260506 18:34:36.914 | INFO | } | 20260506 18:34:36.914 | INFO | Perf report for the last 5.14222 seconds: | 20260506 18:34:36.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:34:36.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:36.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:36.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:36.914 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 18:34:36.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:36.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:36.914 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:34:36.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:36.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:36.914 | INFO | Uploaded 36GiB to GPU in 1.92976s (18.6551GiB/s) | 20260506 18:34:36.914 | INFO | Downloaded 34GiB to GPU in 1.86795s (18.2017GiB/s) | 20260506 18:34:42.055 | INFO | Analyzing chunk frames = [3545088,3545600) | 20260506 18:34:42.056 | INFO | MemInfo: { | 20260506 18:34:42.056 | INFO | "gpu" : " 17 GB", | 20260506 18:34:42.056 | INFO | "huge" : "", | 20260506 18:34:42.056 | INFO | "huge_shared" : " 27 GB", | 20260506 18:34:42.056 | INFO | "malloc" : " 14 GB", | 20260506 18:34:42.056 | INFO | "pinned" : " 29 GB" | 20260506 18:34:42.056 | INFO | } | 20260506 18:34:42.056 | INFO | Perf report for the last 5.14218 seconds: | 20260506 18:34:42.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:34:42.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:42.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:42.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:42.056 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 18:34:42.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:42.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:42.056 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:34:42.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:42.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:42.056 | INFO | Uploaded 36GiB to GPU in 2.02s (17.8218GiB/s) | 20260506 18:34:42.056 | INFO | Downloaded 34GiB to GPU in 1.91076s (17.7939GiB/s) | 20260506 18:34:47.201 | INFO | Analyzing chunk frames = [3545600,3546112) | 20260506 18:34:47.201 | INFO | MemInfo: { | 20260506 18:34:47.201 | INFO | "gpu" : " 17 GB", | 20260506 18:34:47.201 | INFO | "huge" : "", | 20260506 18:34:47.201 | INFO | "huge_shared" : " 27 GB", | 20260506 18:34:47.201 | INFO | "malloc" : " 14 GB", | 20260506 18:34:47.201 | INFO | "pinned" : " 29 GB" | 20260506 18:34:47.201 | INFO | } | 20260506 18:34:47.201 | INFO | Perf report for the last 5.14564 seconds: | 20260506 18:34:47.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:34:47.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:47.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:47.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:47.201 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 18:34:47.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:47.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:47.201 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:34:47.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:47.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:47.202 | INFO | Uploaded 36GiB to GPU in 2.07022s (17.3894GiB/s) | 20260506 18:34:47.202 | INFO | Downloaded 34GiB to GPU in 1.95525s (17.3891GiB/s) | 20260506 18:34:52.338 | INFO | Analyzing chunk frames = [3546112,3546624) | 20260506 18:34:52.338 | INFO | MemInfo: { | 20260506 18:34:52.338 | INFO | "gpu" : " 17 GB", | 20260506 18:34:52.338 | INFO | "huge" : "", | 20260506 18:34:52.338 | INFO | "huge_shared" : " 27 GB", | 20260506 18:34:52.338 | INFO | "malloc" : " 14 GB", | 20260506 18:34:52.338 | INFO | "pinned" : " 29 GB" | 20260506 18:34:52.338 | INFO | } | 20260506 18:34:52.338 | INFO | Perf report for the last 5.13658 seconds: | 20260506 18:34:52.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:34:52.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:52.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:52.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:52.338 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:34:52.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:52.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:52.338 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 18:34:52.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:52.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:52.338 | INFO | Uploaded 36GiB to GPU in 1.97871s (18.1936GiB/s) | 20260506 18:34:52.338 | INFO | Downloaded 34GiB to GPU in 1.93992s (17.5265GiB/s) | 20260506 18:34:57.475 | INFO | Analyzing chunk frames = [3546624,3547136) | 20260506 18:34:57.475 | INFO | MemInfo: { | 20260506 18:34:57.475 | INFO | "gpu" : " 17 GB", | 20260506 18:34:57.475 | INFO | "huge" : "", | 20260506 18:34:57.475 | INFO | "huge_shared" : " 27 GB", | 20260506 18:34:57.475 | INFO | "malloc" : " 14 GB", | 20260506 18:34:57.475 | INFO | "pinned" : " 29 GB" | 20260506 18:34:57.475 | INFO | } | 20260506 18:34:57.475 | INFO | Perf report for the last 5.13711 seconds: | 20260506 18:34:57.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:34:57.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:57.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:57.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:57.475 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 18:34:57.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:57.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:57.475 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 18:34:57.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:57.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:34:57.475 | INFO | Uploaded 36GiB to GPU in 1.98406s (18.1447GiB/s) | 20260506 18:34:57.475 | INFO | Downloaded 34GiB to GPU in 1.98827s (17.1003GiB/s) | 20260506 18:34:57.476 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:34:57.476 | INFO | Total mem usage: 18 GiB | 20260506 18:34:57.476 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:34:57.476 | INFO | Recent High Water Marks Sum: 16.7 GiB (Savings: -7.87%) | 20260506 18:34:57.476 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:34:57.476 | INFO | High Water Breakdown: | 20260506 18:34:57.476 | INFO | 62.4% : 10.4 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:34:57.476 | INFO | 37.5% : 6.25 GiB ( 93.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:34:57.476 | INFO | 0.119% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:34:57.476 | INFO | ------------------------------------------ | 20260506 18:34:57.476 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:34:57.476 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:34:57.476 | INFO | Total mem usage: 26.5 GiB | 20260506 18:34:57.476 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:34:57.476 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:34:57.476 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:34:57.476 | INFO | High Water Breakdown: | 20260506 18:34:57.476 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:34:57.476 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:34:57.476 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:34:57.476 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:34:57.476 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:34:57.476 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:34:57.476 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:34:57.476 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:34:57.476 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:34:57.476 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:34:57.476 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:34:57.476 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:34:57.476 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:34:57.476 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:34:57.476 | INFO | ------------------------------------------ | 20260506 18:34:57.476 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:34:57.476 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:34:57.476 | INFO | Total mem usage: 25.6 GiB | 20260506 18:34:57.476 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:34:57.476 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:34:57.476 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:34:57.476 | INFO | High Water Breakdown: | 20260506 18:34:57.476 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:34:57.476 | INFO | ------------------------------------------ | 20260506 18:34:57.476 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:34:57.476 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:34:57.476 | INFO | Total mem usage: 15.7 GiB | 20260506 18:34:57.476 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:34:57.476 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:34:57.476 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:34:57.476 | INFO | High Water Breakdown: | 20260506 18:34:57.476 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:34:57.476 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:34:57.476 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:34:57.476 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:34:57.476 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:34:57.476 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:34:57.476 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:34:57.476 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:34:57.476 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:34:57.476 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:34:57.476 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:34:57.476 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:34:57.476 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:34:57.476 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:34:57.476 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:34:57.476 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:34:57.476 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:34:57.476 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:34:57.476 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:34:57.476 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:34:57.476 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:34:57.476 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:34:57.476 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:34:57.476 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:34:57.476 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:34:57.476 | INFO | ------------------------------------------ | 20260506 18:34:57.476 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:34:57.477 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:34:57.477 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:34:57.477 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:34:57.477 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.56e+01, 1.34e+01 / 2.20e+00 | 20260506 18:34:57.477 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.02e-01, 1.02e-02 / 7.84e-03 | 20260506 18:34:57.477 | INFO | PerfReport: 11.63%, Baseline -- 3.64e+00 / 4.45e+00, 5.61e-01 / 2.96e-01 | 20260506 18:34:57.477 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.82e-01 / 1.26e-01 | 20260506 18:34:57.477 | INFO | PerfReport: 15.94%, DME -- 3.99e+01 / 4.43e+01, 3.96e+00 / 2.05e+00 | 20260506 18:34:57.477 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.48e+00 / 1.72e+00 | 20260506 18:34:57.477 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.25e+00 / 2.82e+00, 3.75e-01 / 2.22e-01 | 20260506 18:34:57.477 | INFO | PerfReport: 3.39%, Metrics -- 1.06e+00 / 1.40e+00, 2.75e-01 / 1.53e-01 | 20260506 18:34:57.477 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.66e-01 / 4.06e-01, 5.72e-02 / 1.93e-02 | 20260506 18:34:57.477 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:34:57.477 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:34:57.477 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:34:57.477 | INFO | PerfReport: Full Execution -- 9.73e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 18:34:57.477 | INFO | PerfReport: 21.57%, Upload -- 2.10e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 18:34:57.477 | INFO | PerfReport: 21.12%, Download -- 2.05e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 18:34:57.477 | INFO | PerfReport: 32.18%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 18:34:57.477 | INFO | PerfReport: 1.53%, LockUpload -- 1.49e+00 / 0.00e+00, 5.69e+00 / 0.00e+00 | 20260506 18:34:57.477 | INFO | PerfReport: 1.01%, LockDownload -- 9.85e-01 / 0.00e+00, 2.55e+00 / 0.00e+00 | 20260506 18:34:57.477 | INFO | PerfReport: 22.57%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 18:34:57.477 | INFO | PerfReport: 0.02%, Misc -- 1.62e-02 / 0.00e+00, 3.46e-02 / 0.00e+00 | 20260506 18:34:57.589 | INFO | Starting a round of Baz IO | 20260506 18:35:00.730 | INFO | Populating RTMetrics report | 20260506 18:35:00.730 | INFO | RTMetrics report populated | 20260506 18:35:00.730 | INFO | Opening tmp RTMetrics json file | 20260506 18:35:01.123 | INFO | Serializing RTMetrics to json | 20260506 18:35:01.346 | INFO | Serialization done, swapping tmp file now... | 20260506 18:35:01.349 | INFO | RTMetrics done | 20260506 18:35:01.616 | INFO | Finished a round of Baz IO: Wrote 2.035 GiB in 4.026 seconds | 20260506 18:35:01.616 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.01% / 0.01% / 64.98% | 20260506 18:35:01.616 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.09261% / 3.97364e-06% / 7.15256e-05% | 20260506 18:35:01.616 | INFO | Baz IO overall: | 20260506 18:35:01.616 | INFO | BytesWritten : 1874033883964 | 20260506 18:35:01.616 | INFO | NumEvents : 906257847353 | 20260506 18:35:02.612 | INFO | Analyzing chunk frames = [3547136,3547648) | 20260506 18:35:02.612 | INFO | MemInfo: { | 20260506 18:35:02.612 | INFO | "gpu" : " 17 GB", | 20260506 18:35:02.612 | INFO | "huge" : "", | 20260506 18:35:02.612 | INFO | "huge_shared" : " 27 GB", | 20260506 18:35:02.612 | INFO | "malloc" : " 14 GB", | 20260506 18:35:02.612 | INFO | "pinned" : " 29 GB" | 20260506 18:35:02.612 | INFO | } | 20260506 18:35:02.612 | INFO | Perf report for the last 5.13683 seconds: | 20260506 18:35:02.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:35:02.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:02.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:02.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:02.612 | INFO | ANALYSIS : 62.8%, 2.9, 95.2ms, 0 | 20260506 18:35:02.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:02.612 | INFO | RT_METRICS : 57.8%, 1.0, 30.3ms, 0 | 20260506 18:35:02.612 | INFO | PRE_HQ : 61.3%, 1.9, 61.6ms, 0 | 20260506 18:35:02.612 | INFO | BAZWRITER : 78.0%, 4.8, 193.2ms, 0 | 20260506 18:35:02.612 | INFO | BAZ_SAMPLING : 7.5%, 1.0, 3.9ms, 0 | 20260506 18:35:02.612 | INFO | Uploaded 36GiB to GPU in 2.05246s (17.5399GiB/s) | 20260506 18:35:02.612 | INFO | Downloaded 38GiB to GPU in 2.20556s (17.2292GiB/s) | 20260506 18:35:07.747 | INFO | Analyzing chunk frames = [3547648,3548160) | 20260506 18:35:07.747 | INFO | MemInfo: { | 20260506 18:35:07.747 | INFO | "gpu" : " 17 GB", | 20260506 18:35:07.747 | INFO | "huge" : "", | 20260506 18:35:07.747 | INFO | "huge_shared" : " 27 GB", | 20260506 18:35:07.747 | INFO | "malloc" : " 14 GB", | 20260506 18:35:07.747 | INFO | "pinned" : " 29 GB" | 20260506 18:35:07.747 | INFO | } | 20260506 18:35:07.748 | INFO | Perf report for the last 5.13549 seconds: | 20260506 18:35:07.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:35:07.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:07.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:07.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:07.748 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:35:07.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:07.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:07.748 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:35:07.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:07.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:07.748 | INFO | Uploaded 36GiB to GPU in 2.00867s (17.9223GiB/s) | 20260506 18:35:07.748 | INFO | Downloaded 34GiB to GPU in 1.93151s (17.6028GiB/s) | 20260506 18:35:12.887 | INFO | Analyzing chunk frames = [3548160,3548672) | 20260506 18:35:12.887 | INFO | MemInfo: { | 20260506 18:35:12.887 | INFO | "gpu" : " 17 GB", | 20260506 18:35:12.887 | INFO | "huge" : "", | 20260506 18:35:12.888 | INFO | "huge_shared" : " 27 GB", | 20260506 18:35:12.888 | INFO | "malloc" : " 14 GB", | 20260506 18:35:12.888 | INFO | "pinned" : " 29 GB" | 20260506 18:35:12.888 | INFO | } | 20260506 18:35:12.888 | INFO | Perf report for the last 5.14004 seconds: | 20260506 18:35:12.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:35:12.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:12.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:12.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:12.888 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 18:35:12.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:12.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:12.888 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:35:12.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:12.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:12.888 | INFO | Uploaded 36GiB to GPU in 1.98858s (18.1034GiB/s) | 20260506 18:35:12.888 | INFO | Downloaded 34GiB to GPU in 1.92333s (17.6777GiB/s) | 20260506 18:35:18.035 | INFO | Analyzing chunk frames = [3548672,3549184) | 20260506 18:35:18.035 | INFO | MemInfo: { | 20260506 18:35:18.035 | INFO | "gpu" : " 17 GB", | 20260506 18:35:18.035 | INFO | "huge" : "", | 20260506 18:35:18.035 | INFO | "huge_shared" : " 27 GB", | 20260506 18:35:18.035 | INFO | "malloc" : " 14 GB", | 20260506 18:35:18.035 | INFO | "pinned" : " 29 GB" | 20260506 18:35:18.035 | INFO | } | 20260506 18:35:18.035 | INFO | Perf report for the last 5.14777 seconds: | 20260506 18:35:18.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:35:18.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:18.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:18.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:18.035 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260506 18:35:18.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:18.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:18.035 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:35:18.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:18.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:18.036 | INFO | Uploaded 36GiB to GPU in 1.93578s (18.5971GiB/s) | 20260506 18:35:18.036 | INFO | Downloaded 34GiB to GPU in 1.86623s (18.2185GiB/s) | 20260506 18:35:23.171 | INFO | Analyzing chunk frames = [3549184,3549696) | 20260506 18:35:23.171 | INFO | MemInfo: { | 20260506 18:35:23.171 | INFO | "gpu" : " 17 GB", | 20260506 18:35:23.171 | INFO | "huge" : "", | 20260506 18:35:23.171 | INFO | "huge_shared" : " 27 GB", | 20260506 18:35:23.171 | INFO | "malloc" : " 14 GB", | 20260506 18:35:23.171 | INFO | "pinned" : " 29 GB" | 20260506 18:35:23.171 | INFO | } | 20260506 18:35:23.171 | INFO | Perf report for the last 5.13608 seconds: | 20260506 18:35:23.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:35:23.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:23.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:23.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:23.171 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 18:35:23.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:23.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:23.172 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:35:23.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:23.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:23.172 | INFO | Uploaded 36GiB to GPU in 2.01561s (17.8606GiB/s) | 20260506 18:35:23.172 | INFO | Downloaded 34GiB to GPU in 1.90541s (17.8439GiB/s) | 20260506 18:35:28.311 | INFO | Analyzing chunk frames = [3549696,3550208) | 20260506 18:35:28.311 | INFO | MemInfo: { | 20260506 18:35:28.311 | INFO | "gpu" : " 17 GB", | 20260506 18:35:28.311 | INFO | "huge" : "", | 20260506 18:35:28.311 | INFO | "huge_shared" : " 27 GB", | 20260506 18:35:28.311 | INFO | "malloc" : " 14 GB", | 20260506 18:35:28.311 | INFO | "pinned" : " 29 GB" | 20260506 18:35:28.311 | INFO | } | 20260506 18:35:28.312 | INFO | Perf report for the last 5.1401 seconds: | 20260506 18:35:28.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:35:28.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:28.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:28.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:28.312 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 18:35:28.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:28.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:28.312 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:35:28.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:28.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:28.312 | INFO | Uploaded 36GiB to GPU in 2.03315s (17.7065GiB/s) | 20260506 18:35:28.312 | INFO | Downloaded 34GiB to GPU in 1.94755s (17.4579GiB/s) | 20260506 18:35:33.450 | INFO | Analyzing chunk frames = [3550208,3550720) | 20260506 18:35:33.450 | INFO | MemInfo: { | 20260506 18:35:33.450 | INFO | "gpu" : " 17 GB", | 20260506 18:35:33.450 | INFO | "huge" : "", | 20260506 18:35:33.450 | INFO | "huge_shared" : " 27 GB", | 20260506 18:35:33.450 | INFO | "malloc" : " 14 GB", | 20260506 18:35:33.450 | INFO | "pinned" : " 29 GB" | 20260506 18:35:33.450 | INFO | } | 20260506 18:35:33.450 | INFO | Perf report for the last 5.13833 seconds: | 20260506 18:35:33.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:35:33.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:33.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:33.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:33.450 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 18:35:33.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:33.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:33.450 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:35:33.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:33.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:33.450 | INFO | Uploaded 36GiB to GPU in 2.04747s (17.5827GiB/s) | 20260506 18:35:33.450 | INFO | Downloaded 34GiB to GPU in 1.99194s (17.0688GiB/s) | 20260506 18:35:38.585 | INFO | Analyzing chunk frames = [3550720,3551232) | 20260506 18:35:38.585 | INFO | MemInfo: { | 20260506 18:35:38.585 | INFO | "gpu" : " 17 GB", | 20260506 18:35:38.585 | INFO | "huge" : "", | 20260506 18:35:38.585 | INFO | "huge_shared" : " 27 GB", | 20260506 18:35:38.585 | INFO | "malloc" : " 14 GB", | 20260506 18:35:38.585 | INFO | "pinned" : " 29 GB" | 20260506 18:35:38.585 | INFO | } | 20260506 18:35:38.585 | INFO | Perf report for the last 5.13502 seconds: | 20260506 18:35:38.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:35:38.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:38.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:38.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:38.585 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 18:35:38.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:38.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:38.585 | INFO | PRE_HQ : 31.7%, 1.0, 16.5ms, 0 | 20260506 18:35:38.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:38.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:38.585 | INFO | Uploaded 36GiB to GPU in 2.00907s (17.9187GiB/s) | 20260506 18:35:38.585 | INFO | Downloaded 34GiB to GPU in 1.97779s (17.1909GiB/s) | 20260506 18:35:38.700 | INFO | Starting a round of Baz IO | 20260506 18:35:41.887 | INFO | Populating RTMetrics report | 20260506 18:35:41.887 | INFO | RTMetrics report populated | 20260506 18:35:41.887 | INFO | Opening tmp RTMetrics json file | 20260506 18:35:41.910 | INFO | Serializing RTMetrics to json | 20260506 18:35:41.989 | INFO | Serialization done, swapping tmp file now... | 20260506 18:35:42.028 | INFO | RTMetrics done | 20260506 18:35:42.116 | INFO | Finished a round of Baz IO: Wrote 2.035 GiB in 3.416 seconds | 20260506 18:35:42.116 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.00% / 0.01% / 64.99% | 20260506 18:35:42.116 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.08999% / 1.58946e-05% / 7.54992e-05% | 20260506 18:35:42.116 | INFO | Baz IO overall: | 20260506 18:35:42.116 | INFO | BytesWritten : 1876219293302 | 20260506 18:35:42.116 | INFO | NumEvents : 907310809469 | 20260506 18:35:43.723 | INFO | Analyzing chunk frames = [3551232,3551744) | 20260506 18:35:43.723 | INFO | MemInfo: { | 20260506 18:35:43.723 | INFO | "gpu" : " 17 GB", | 20260506 18:35:43.723 | INFO | "huge" : "", | 20260506 18:35:43.723 | INFO | "huge_shared" : " 27 GB", | 20260506 18:35:43.723 | INFO | "malloc" : " 14 GB", | 20260506 18:35:43.723 | INFO | "pinned" : " 29 GB" | 20260506 18:35:43.723 | INFO | } | 20260506 18:35:43.723 | INFO | Perf report for the last 5.13793 seconds: | 20260506 18:35:43.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:35:43.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:43.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:43.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:43.723 | INFO | ANALYSIS : 63.4%, 2.8, 94.4ms, 0 | 20260506 18:35:43.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:43.723 | INFO | RT_METRICS : 49.7%, 1.0, 26.1ms, 0 | 20260506 18:35:43.723 | INFO | PRE_HQ : 61.4%, 2.0, 63.5ms, 0 | 20260506 18:35:43.723 | INFO | BAZWRITER : 66.1%, 4.7, 160.9ms, 0 | 20260506 18:35:43.723 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.3ms, 0 | 20260506 18:35:43.723 | INFO | Uploaded 36GiB to GPU in 2.08824s (17.2394GiB/s) | 20260506 18:35:43.723 | INFO | Downloaded 38GiB to GPU in 2.20031s (17.2703GiB/s) | 20260506 18:35:48.861 | INFO | Analyzing chunk frames = [3551744,3552256) | 20260506 18:35:48.862 | INFO | MemInfo: { | 20260506 18:35:48.862 | INFO | "gpu" : " 17 GB", | 20260506 18:35:48.862 | INFO | "huge" : "", | 20260506 18:35:48.862 | INFO | "huge_shared" : " 27 GB", | 20260506 18:35:48.862 | INFO | "malloc" : " 14 GB", | 20260506 18:35:48.862 | INFO | "pinned" : " 29 GB" | 20260506 18:35:48.862 | INFO | } | 20260506 18:35:48.862 | INFO | Perf report for the last 5.13892 seconds: | 20260506 18:35:48.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:35:48.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:48.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:48.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:48.862 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:35:48.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:48.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:48.862 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:35:48.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:48.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:48.862 | INFO | Uploaded 36GiB to GPU in 1.99492s (18.0458GiB/s) | 20260506 18:35:48.862 | INFO | Downloaded 34GiB to GPU in 1.94652s (17.467GiB/s) | 20260506 18:35:54.006 | INFO | Analyzing chunk frames = [3552256,3552768) | 20260506 18:35:54.006 | INFO | MemInfo: { | 20260506 18:35:54.006 | INFO | "gpu" : " 17 GB", | 20260506 18:35:54.006 | INFO | "huge" : "", | 20260506 18:35:54.006 | INFO | "huge_shared" : " 27 GB", | 20260506 18:35:54.006 | INFO | "malloc" : " 14 GB", | 20260506 18:35:54.006 | INFO | "pinned" : " 29 GB" | 20260506 18:35:54.006 | INFO | } | 20260506 18:35:54.006 | INFO | Perf report for the last 5.14451 seconds: | 20260506 18:35:54.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:35:54.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:54.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:54.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:54.006 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 18:35:54.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:54.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:54.007 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 18:35:54.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:54.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:54.007 | INFO | Uploaded 36GiB to GPU in 1.98128s (18.17GiB/s) | 20260506 18:35:54.007 | INFO | Downloaded 34GiB to GPU in 1.92434s (17.6684GiB/s) | 20260506 18:35:59.146 | INFO | Analyzing chunk frames = [3552768,3553280) | 20260506 18:35:59.146 | INFO | MemInfo: { | 20260506 18:35:59.146 | INFO | "gpu" : " 17 GB", | 20260506 18:35:59.146 | INFO | "huge" : "", | 20260506 18:35:59.146 | INFO | "huge_shared" : " 27 GB", | 20260506 18:35:59.146 | INFO | "malloc" : " 14 GB", | 20260506 18:35:59.146 | INFO | "pinned" : " 29 GB" | 20260506 18:35:59.146 | INFO | } | 20260506 18:35:59.146 | INFO | Perf report for the last 5.14015 seconds: | 20260506 18:35:59.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:35:59.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:59.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:59.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:59.147 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 18:35:59.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:59.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:59.147 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:35:59.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:59.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:35:59.147 | INFO | Uploaded 36GiB to GPU in 1.94395s (18.519GiB/s) | 20260506 18:35:59.147 | INFO | Downloaded 34GiB to GPU in 1.86791s (18.2021GiB/s) | 20260506 18:36:04.290 | INFO | Analyzing chunk frames = [3553280,3553792) | 20260506 18:36:04.290 | INFO | MemInfo: { | 20260506 18:36:04.290 | INFO | "gpu" : " 17 GB", | 20260506 18:36:04.290 | INFO | "huge" : "", | 20260506 18:36:04.290 | INFO | "huge_shared" : " 27 GB", | 20260506 18:36:04.290 | INFO | "malloc" : " 14 GB", | 20260506 18:36:04.290 | INFO | "pinned" : " 29 GB" | 20260506 18:36:04.290 | INFO | } | 20260506 18:36:04.290 | INFO | Perf report for the last 5.14395 seconds: | 20260506 18:36:04.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:36:04.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:04.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:04.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:04.290 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 18:36:04.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:04.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:04.291 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:36:04.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:04.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:04.291 | INFO | Uploaded 36GiB to GPU in 2.0228s (17.7971GiB/s) | 20260506 18:36:04.291 | INFO | Downloaded 34GiB to GPU in 1.91098s (17.792GiB/s) | 20260506 18:36:09.424 | INFO | Analyzing chunk frames = [3553792,3554304) | 20260506 18:36:09.424 | INFO | MemInfo: { | 20260506 18:36:09.424 | INFO | "gpu" : " 17 GB", | 20260506 18:36:09.424 | INFO | "huge" : "", | 20260506 18:36:09.424 | INFO | "huge_shared" : " 27 GB", | 20260506 18:36:09.424 | INFO | "malloc" : " 14 GB", | 20260506 18:36:09.424 | INFO | "pinned" : " 29 GB" | 20260506 18:36:09.424 | INFO | } | 20260506 18:36:09.424 | INFO | Perf report for the last 5.13393 seconds: | 20260506 18:36:09.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:36:09.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:09.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:09.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:09.424 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:36:09.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:09.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:09.425 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 18:36:09.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:09.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:09.425 | INFO | Uploaded 36GiB to GPU in 2.07754s (17.3282GiB/s) | 20260506 18:36:09.425 | INFO | Downloaded 34GiB to GPU in 1.96603s (17.2937GiB/s) | 20260506 18:36:14.561 | INFO | Analyzing chunk frames = [3554304,3554816) | 20260506 18:36:14.561 | INFO | MemInfo: { | 20260506 18:36:14.561 | INFO | "gpu" : " 17 GB", | 20260506 18:36:14.561 | INFO | "huge" : "", | 20260506 18:36:14.561 | INFO | "huge_shared" : " 27 GB", | 20260506 18:36:14.561 | INFO | "malloc" : " 14 GB", | 20260506 18:36:14.561 | INFO | "pinned" : " 29 GB" | 20260506 18:36:14.561 | INFO | } | 20260506 18:36:14.561 | INFO | Perf report for the last 5.13646 seconds: | 20260506 18:36:14.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:36:14.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:14.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:14.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:14.561 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:36:14.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:14.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:14.561 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:36:14.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:14.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:14.561 | INFO | Uploaded 36GiB to GPU in 2.01618s (17.8555GiB/s) | 20260506 18:36:14.561 | INFO | Downloaded 34GiB to GPU in 2.01107s (16.9064GiB/s) | 20260506 18:36:19.703 | INFO | Analyzing chunk frames = [3554816,3555328) | 20260506 18:36:19.703 | INFO | MemInfo: { | 20260506 18:36:19.703 | INFO | "gpu" : " 17 GB", | 20260506 18:36:19.703 | INFO | "huge" : "", | 20260506 18:36:19.703 | INFO | "huge_shared" : " 27 GB", | 20260506 18:36:19.703 | INFO | "malloc" : " 14 GB", | 20260506 18:36:19.703 | INFO | "pinned" : " 29 GB" | 20260506 18:36:19.703 | INFO | } | 20260506 18:36:19.703 | INFO | Perf report for the last 5.14193 seconds: | 20260506 18:36:19.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:36:19.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:19.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:19.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:19.703 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 18:36:19.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:19.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:19.703 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 18:36:19.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:19.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:19.703 | INFO | Uploaded 36GiB to GPU in 2.0394s (17.6523GiB/s) | 20260506 18:36:19.703 | INFO | Downloaded 34GiB to GPU in 1.98762s (17.1059GiB/s) | 20260506 18:36:19.703 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:36:19.703 | INFO | Total mem usage: 18 GiB | 20260506 18:36:19.703 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:36:19.703 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.6%) | 20260506 18:36:19.703 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:36:19.703 | INFO | High Water Breakdown: | 20260506 18:36:19.703 | INFO | 61.3% : 9.56 GiB ( 83.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:36:19.703 | INFO | 38.5% : 6 GiB ( 89.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:36:19.703 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:36:19.703 | INFO | ------------------------------------------ | 20260506 18:36:19.703 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:36:19.703 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:36:19.703 | INFO | Total mem usage: 26.5 GiB | 20260506 18:36:19.703 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:36:19.703 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:36:19.703 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:36:19.703 | INFO | High Water Breakdown: | 20260506 18:36:19.703 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:36:19.703 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:36:19.704 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:36:19.704 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:36:19.704 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:36:19.704 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:36:19.704 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:36:19.704 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:36:19.704 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:36:19.704 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:36:19.704 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:36:19.704 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:36:19.704 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:36:19.704 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:36:19.704 | INFO | ------------------------------------------ | 20260506 18:36:19.704 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:36:19.704 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:36:19.704 | INFO | Total mem usage: 25.6 GiB | 20260506 18:36:19.704 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:36:19.704 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:36:19.704 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:36:19.704 | INFO | High Water Breakdown: | 20260506 18:36:19.704 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:36:19.704 | INFO | ------------------------------------------ | 20260506 18:36:19.704 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:36:19.704 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:36:19.704 | INFO | Total mem usage: 15.7 GiB | 20260506 18:36:19.704 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:36:19.704 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:36:19.704 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 18:36:19.704 | INFO | High Water Breakdown: | 20260506 18:36:19.704 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:36:19.704 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:36:19.704 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:36:19.704 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:36:19.704 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:36:19.704 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:36:19.704 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:36:19.704 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:36:19.704 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:36:19.704 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:36:19.704 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:36:19.704 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:36:19.704 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:36:19.704 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:36:19.704 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:36:19.704 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:36:19.704 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:36:19.704 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:36:19.704 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:36:19.704 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:36:19.704 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:36:19.704 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:36:19.704 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:36:19.704 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:36:19.704 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:36:19.704 | INFO | ------------------------------------------ | 20260506 18:36:19.704 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:36:19.704 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:36:19.704 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:36:19.704 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:36:19.704 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.52e+01, 1.33e+01 / 2.18e+00 | 20260506 18:36:19.704 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.01e-01, 1.09e-02 / 6.78e-03 | 20260506 18:36:19.704 | INFO | PerfReport: 11.60%, Baseline -- 3.63e+00 / 4.46e+00, 5.47e-01 / 3.52e-01 | 20260506 18:36:19.704 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.48e+00, 1.83e-01 / 1.40e-01 | 20260506 18:36:19.704 | INFO | PerfReport: 15.91%, DME -- 3.98e+01 / 4.38e+01, 3.82e+00 / 1.64e+00 | 20260506 18:36:19.704 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.52e+00 / 1.70e+00 | 20260506 18:36:19.704 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.26e+00 / 2.81e+00, 3.85e-01 / 1.94e-01 | 20260506 18:36:19.704 | INFO | PerfReport: 3.39%, Metrics -- 1.06e+00 / 1.37e+00, 2.76e-01 / 1.73e-01 | 20260506 18:36:19.704 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.67e-01 / 4.13e-01, 5.67e-02 / 2.35e-02 | 20260506 18:36:19.704 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:36:19.704 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:36:19.704 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:36:19.704 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 18:36:19.704 | INFO | PerfReport: 21.62%, Upload -- 2.10e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260506 18:36:19.704 | INFO | PerfReport: 21.16%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 18:36:19.704 | INFO | PerfReport: 32.23%, Compute -- 3.13e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 18:36:19.704 | INFO | PerfReport: 1.52%, LockUpload -- 1.48e+00 / 0.00e+00, 5.67e+00 / 0.00e+00 | 20260506 18:36:19.704 | INFO | PerfReport: 1.00%, LockDownload -- 9.75e-01 / 0.00e+00, 2.53e+00 / 0.00e+00 | 20260506 18:36:19.704 | INFO | PerfReport: 22.45%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 18:36:19.704 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260506 18:36:19.817 | INFO | Starting a round of Baz IO | 20260506 18:36:22.957 | INFO | Populating RTMetrics report | 20260506 18:36:22.957 | INFO | RTMetrics report populated | 20260506 18:36:22.957 | INFO | Opening tmp RTMetrics json file | 20260506 18:36:22.990 | INFO | Serializing RTMetrics to json | 20260506 18:36:22.996 | INFO | Serialization done, swapping tmp file now... | 20260506 18:36:22.998 | INFO | RTMetrics done | 20260506 18:36:23.439 | INFO | Finished a round of Baz IO: Wrote 2.035 GiB in 3.621 seconds | 20260506 18:36:23.439 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 35.00% / 0.01% / 65.00% | 20260506 18:36:23.439 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.0882% / 7.94729e-06% / 9.93411e-05% | 20260506 18:36:23.439 | INFO | Baz IO overall: | 20260506 18:36:23.439 | INFO | BytesWritten : 1878403992991 | 20260506 18:36:23.439 | INFO | NumEvents : 908363429908 | 20260506 18:36:24.839 | INFO | Analyzing chunk frames = [3555328,3555840) | 20260506 18:36:24.839 | INFO | MemInfo: { | 20260506 18:36:24.839 | INFO | "gpu" : " 17 GB", | 20260506 18:36:24.839 | INFO | "huge" : "", | 20260506 18:36:24.839 | INFO | "huge_shared" : " 27 GB", | 20260506 18:36:24.839 | INFO | "malloc" : " 14 GB", | 20260506 18:36:24.839 | INFO | "pinned" : " 29 GB" | 20260506 18:36:24.839 | INFO | } | 20260506 18:36:24.839 | INFO | Perf report for the last 5.13631 seconds: | 20260506 18:36:24.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:36:24.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:24.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:24.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:24.839 | INFO | ANALYSIS : 62.8%, 2.9, 95.5ms, 0 | 20260506 18:36:24.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:24.839 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 18:36:24.839 | INFO | PRE_HQ : 60.2%, 1.9, 58.1ms, 0 | 20260506 18:36:24.839 | INFO | BAZWRITER : 70.0%, 4.8, 173.0ms, 0 | 20260506 18:36:24.839 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 18:36:24.840 | INFO | Uploaded 36GiB to GPU in 2.03684s (17.6744GiB/s) | 20260506 18:36:24.840 | INFO | Downloaded 38GiB to GPU in 2.2171s (17.1395GiB/s) | 20260506 18:36:29.980 | INFO | Analyzing chunk frames = [3555840,3556352) | 20260506 18:36:29.980 | INFO | MemInfo: { | 20260506 18:36:29.980 | INFO | "gpu" : " 17 GB", | 20260506 18:36:29.980 | INFO | "huge" : "", | 20260506 18:36:29.980 | INFO | "huge_shared" : " 27 GB", | 20260506 18:36:29.981 | INFO | "malloc" : " 14 GB", | 20260506 18:36:29.981 | INFO | "pinned" : " 29 GB" | 20260506 18:36:29.981 | INFO | } | 20260506 18:36:29.981 | INFO | Perf report for the last 5.14155 seconds: | 20260506 18:36:29.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:36:29.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:29.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:29.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:29.981 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:36:29.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:29.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:29.981 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:36:29.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:29.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:29.981 | INFO | Uploaded 36GiB to GPU in 2.00342s (17.9692GiB/s) | 20260506 18:36:29.981 | INFO | Downloaded 34GiB to GPU in 1.92958s (17.6204GiB/s) | 20260506 18:36:35.119 | INFO | Analyzing chunk frames = [3556352,3556864) | 20260506 18:36:35.119 | INFO | MemInfo: { | 20260506 18:36:35.119 | INFO | "gpu" : " 17 GB", | 20260506 18:36:35.119 | INFO | "huge" : "", | 20260506 18:36:35.119 | INFO | "huge_shared" : " 27 GB", | 20260506 18:36:35.119 | INFO | "malloc" : " 14 GB", | 20260506 18:36:35.119 | INFO | "pinned" : " 29 GB" | 20260506 18:36:35.119 | INFO | } | 20260506 18:36:35.119 | INFO | Perf report for the last 5.13872 seconds: | 20260506 18:36:35.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:36:35.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:35.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:35.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:35.120 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:36:35.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:35.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:35.120 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:36:35.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:35.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:35.120 | INFO | Uploaded 36GiB to GPU in 1.95935s (18.3734GiB/s) | 20260506 18:36:35.120 | INFO | Downloaded 34GiB to GPU in 1.9206s (17.7028GiB/s) | 20260506 18:36:40.260 | INFO | Analyzing chunk frames = [3556864,3557376) | 20260506 18:36:40.260 | INFO | MemInfo: { | 20260506 18:36:40.260 | INFO | "gpu" : " 17 GB", | 20260506 18:36:40.260 | INFO | "huge" : "", | 20260506 18:36:40.260 | INFO | "huge_shared" : " 27 GB", | 20260506 18:36:40.260 | INFO | "malloc" : " 14 GB", | 20260506 18:36:40.260 | INFO | "pinned" : " 29 GB" | 20260506 18:36:40.260 | INFO | } | 20260506 18:36:40.260 | INFO | Perf report for the last 5.14038 seconds: | 20260506 18:36:40.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:36:40.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:40.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:40.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:40.260 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:36:40.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:40.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:40.260 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:36:40.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:40.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:40.260 | INFO | Uploaded 36GiB to GPU in 1.91504s (18.7986GiB/s) | 20260506 18:36:40.260 | INFO | Downloaded 34GiB to GPU in 1.8675s (18.2061GiB/s) | 20260506 18:36:45.399 | INFO | Analyzing chunk frames = [3557376,3557888) | 20260506 18:36:45.399 | INFO | MemInfo: { | 20260506 18:36:45.399 | INFO | "gpu" : " 17 GB", | 20260506 18:36:45.399 | INFO | "huge" : "", | 20260506 18:36:45.399 | INFO | "huge_shared" : " 27 GB", | 20260506 18:36:45.399 | INFO | "malloc" : " 14 GB", | 20260506 18:36:45.399 | INFO | "pinned" : " 29 GB" | 20260506 18:36:45.399 | INFO | } | 20260506 18:36:45.399 | INFO | Perf report for the last 5.13924 seconds: | 20260506 18:36:45.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:36:45.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:45.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:45.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:45.399 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260506 18:36:45.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:45.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:45.399 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:36:45.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:45.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:45.399 | INFO | Uploaded 36GiB to GPU in 1.94492s (18.5098GiB/s) | 20260506 18:36:45.399 | INFO | Downloaded 34GiB to GPU in 1.88087s (18.0767GiB/s) | 20260506 18:36:50.539 | INFO | Analyzing chunk frames = [3557888,3558400) | 20260506 18:36:50.539 | INFO | MemInfo: { | 20260506 18:36:50.539 | INFO | "gpu" : " 17 GB", | 20260506 18:36:50.539 | INFO | "huge" : "", | 20260506 18:36:50.539 | INFO | "huge_shared" : " 27 GB", | 20260506 18:36:50.539 | INFO | "malloc" : " 14 GB", | 20260506 18:36:50.539 | INFO | "pinned" : " 29 GB" | 20260506 18:36:50.539 | INFO | } | 20260506 18:36:50.539 | INFO | Perf report for the last 5.14024 seconds: | 20260506 18:36:50.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:36:50.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:50.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:50.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:50.539 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 18:36:50.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:50.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:50.539 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 18:36:50.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:50.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:50.540 | INFO | Uploaded 36GiB to GPU in 1.98942s (18.0957GiB/s) | 20260506 18:36:50.540 | INFO | Downloaded 34GiB to GPU in 1.95856s (17.3597GiB/s) | 20260506 18:36:55.676 | INFO | Analyzing chunk frames = [3558400,3558912) | 20260506 18:36:55.676 | INFO | MemInfo: { | 20260506 18:36:55.676 | INFO | "gpu" : " 17 GB", | 20260506 18:36:55.676 | INFO | "huge" : "", | 20260506 18:36:55.677 | INFO | "huge_shared" : " 27 GB", | 20260506 18:36:55.677 | INFO | "malloc" : " 14 GB", | 20260506 18:36:55.677 | INFO | "pinned" : " 29 GB" | 20260506 18:36:55.677 | INFO | } | 20260506 18:36:55.677 | INFO | Perf report for the last 5.13735 seconds: | 20260506 18:36:55.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:36:55.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:55.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:55.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:55.677 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:36:55.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:55.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:55.677 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 18:36:55.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:55.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:36:55.677 | INFO | Uploaded 36GiB to GPU in 2.01612s (17.8561GiB/s) | 20260506 18:36:55.677 | INFO | Downloaded 34GiB to GPU in 1.9905s (17.0812GiB/s) | 20260506 18:37:00.814 | INFO | Analyzing chunk frames = [3558912,3559424) | 20260506 18:37:00.814 | INFO | MemInfo: { | 20260506 18:37:00.814 | INFO | "gpu" : " 17 GB", | 20260506 18:37:00.815 | INFO | "huge" : "", | 20260506 18:37:00.815 | INFO | "huge_shared" : " 27 GB", | 20260506 18:37:00.815 | INFO | "malloc" : " 14 GB", | 20260506 18:37:00.815 | INFO | "pinned" : " 29 GB" | 20260506 18:37:00.815 | INFO | } | 20260506 18:37:00.815 | INFO | Perf report for the last 5.13807 seconds: | 20260506 18:37:00.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:37:00.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:00.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:00.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:00.815 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 18:37:00.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:00.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:00.815 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 18:37:00.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:00.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:00.815 | INFO | Uploaded 36GiB to GPU in 2.02714s (17.759GiB/s) | 20260506 18:37:00.815 | INFO | Downloaded 34GiB to GPU in 1.99382s (17.0527GiB/s) | 20260506 18:37:00.930 | INFO | Starting a round of Baz IO | 20260506 18:37:04.065 | INFO | Populating RTMetrics report | 20260506 18:37:04.065 | INFO | RTMetrics report populated | 20260506 18:37:04.065 | INFO | Opening tmp RTMetrics json file | 20260506 18:37:04.066 | INFO | Serializing RTMetrics to json | 20260506 18:37:04.081 | INFO | Serialization done, swapping tmp file now... | 20260506 18:37:04.122 | INFO | RTMetrics done | 20260506 18:37:04.224 | INFO | Finished a round of Baz IO: Wrote 2.033 GiB in 3.293 seconds | 20260506 18:37:04.224 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.99% / 0.01% / 65.00% | 20260506 18:37:04.224 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.0831% / 1.19209e-05% / 6.75519e-05% | 20260506 18:37:04.224 | INFO | Baz IO overall: | 20260506 18:37:04.224 | INFO | BytesWritten : 1880586933233 | 20260506 18:37:04.224 | INFO | NumEvents : 909414832538 | 20260506 18:37:05.954 | INFO | Analyzing chunk frames = [3559424,3559936) | 20260506 18:37:05.954 | INFO | MemInfo: { | 20260506 18:37:05.954 | INFO | "gpu" : " 17 GB", | 20260506 18:37:05.954 | INFO | "huge" : "", | 20260506 18:37:05.954 | INFO | "huge_shared" : " 27 GB", | 20260506 18:37:05.954 | INFO | "malloc" : " 14 GB", | 20260506 18:37:05.954 | INFO | "pinned" : " 29 GB" | 20260506 18:37:05.954 | INFO | } | 20260506 18:37:05.954 | INFO | Perf report for the last 5.13971 seconds: | 20260506 18:37:05.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:37:05.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:05.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:05.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:05.955 | INFO | ANALYSIS : 62.7%, 2.9, 95.5ms, 0 | 20260506 18:37:05.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:05.955 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 18:37:05.955 | INFO | PRE_HQ : 60.1%, 1.9, 57.9ms, 0 | 20260506 18:37:05.955 | INFO | BAZWRITER : 63.6%, 4.5, 145.4ms, 0 | 20260506 18:37:05.955 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 18:37:05.955 | INFO | Uploaded 36GiB to GPU in 2.07057s (17.3866GiB/s) | 20260506 18:37:05.955 | INFO | Downloaded 38GiB to GPU in 2.21194s (17.1795GiB/s) | 20260506 18:37:11.094 | INFO | Analyzing chunk frames = [3559936,3560448) | 20260506 18:37:11.094 | INFO | MemInfo: { | 20260506 18:37:11.094 | INFO | "gpu" : " 17 GB", | 20260506 18:37:11.094 | INFO | "huge" : "", | 20260506 18:37:11.094 | INFO | "huge_shared" : " 27 GB", | 20260506 18:37:11.094 | INFO | "malloc" : " 14 GB", | 20260506 18:37:11.094 | INFO | "pinned" : " 29 GB" | 20260506 18:37:11.094 | INFO | } | 20260506 18:37:11.094 | INFO | Perf report for the last 5.13986 seconds: | 20260506 18:37:11.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:37:11.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:11.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:11.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:11.094 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 18:37:11.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:11.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:11.094 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:37:11.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:11.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:11.095 | INFO | Uploaded 36GiB to GPU in 1.99746s (18.0229GiB/s) | 20260506 18:37:11.095 | INFO | Downloaded 34GiB to GPU in 1.93776s (17.546GiB/s) | 20260506 18:37:16.234 | INFO | Analyzing chunk frames = [3560448,3560960) | 20260506 18:37:16.234 | INFO | MemInfo: { | 20260506 18:37:16.234 | INFO | "gpu" : " 17 GB", | 20260506 18:37:16.234 | INFO | "huge" : "", | 20260506 18:37:16.234 | INFO | "huge_shared" : " 27 GB", | 20260506 18:37:16.234 | INFO | "malloc" : " 14 GB", | 20260506 18:37:16.234 | INFO | "pinned" : " 29 GB" | 20260506 18:37:16.235 | INFO | } | 20260506 18:37:16.235 | INFO | Perf report for the last 5.14019 seconds: | 20260506 18:37:16.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:37:16.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:16.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:16.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:16.235 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 18:37:16.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:16.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:16.235 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:37:16.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:16.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:16.235 | INFO | Uploaded 36GiB to GPU in 1.96245s (18.3445GiB/s) | 20260506 18:37:16.235 | INFO | Downloaded 34GiB to GPU in 1.92119s (17.6974GiB/s) | 20260506 18:37:21.380 | INFO | Analyzing chunk frames = [3560960,3561472) | 20260506 18:37:21.380 | INFO | MemInfo: { | 20260506 18:37:21.380 | INFO | "gpu" : " 17 GB", | 20260506 18:37:21.380 | INFO | "huge" : "", | 20260506 18:37:21.380 | INFO | "huge_shared" : " 27 GB", | 20260506 18:37:21.380 | INFO | "malloc" : " 14 GB", | 20260506 18:37:21.380 | INFO | "pinned" : " 29 GB" | 20260506 18:37:21.380 | INFO | } | 20260506 18:37:21.380 | INFO | Perf report for the last 5.14589 seconds: | 20260506 18:37:21.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:37:21.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:21.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:21.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:21.381 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260506 18:37:21.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:21.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:21.381 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:37:21.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:21.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:21.381 | INFO | Uploaded 36GiB to GPU in 1.93739s (18.5817GiB/s) | 20260506 18:37:21.381 | INFO | Downloaded 34GiB to GPU in 1.86945s (18.1872GiB/s) | 20260506 18:37:26.516 | INFO | Analyzing chunk frames = [3561472,3561984) | 20260506 18:37:26.516 | INFO | MemInfo: { | 20260506 18:37:26.516 | INFO | "gpu" : " 17 GB", | 20260506 18:37:26.516 | INFO | "huge" : "", | 20260506 18:37:26.516 | INFO | "huge_shared" : " 27 GB", | 20260506 18:37:26.516 | INFO | "malloc" : " 14 GB", | 20260506 18:37:26.516 | INFO | "pinned" : " 29 GB" | 20260506 18:37:26.516 | INFO | } | 20260506 18:37:26.516 | INFO | Perf report for the last 5.13536 seconds: | 20260506 18:37:26.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:37:26.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:26.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:26.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:26.516 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 18:37:26.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:26.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:26.516 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:37:26.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:26.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:26.516 | INFO | Uploaded 36GiB to GPU in 1.99941s (18.0053GiB/s) | 20260506 18:37:26.516 | INFO | Downloaded 34GiB to GPU in 1.92564s (17.6565GiB/s) | 20260506 18:37:31.650 | INFO | Analyzing chunk frames = [3561984,3562496) | 20260506 18:37:31.650 | INFO | MemInfo: { | 20260506 18:37:31.650 | INFO | "gpu" : " 17 GB", | 20260506 18:37:31.650 | INFO | "huge" : "", | 20260506 18:37:31.650 | INFO | "huge_shared" : " 27 GB", | 20260506 18:37:31.650 | INFO | "malloc" : " 14 GB", | 20260506 18:37:31.650 | INFO | "pinned" : " 29 GB" | 20260506 18:37:31.650 | INFO | } | 20260506 18:37:31.650 | INFO | Perf report for the last 5.1345 seconds: | 20260506 18:37:31.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:37:31.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:31.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:31.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:31.650 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 18:37:31.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:31.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:31.650 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:37:31.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:31.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:31.651 | INFO | Uploaded 36GiB to GPU in 2.01351s (17.8792GiB/s) | 20260506 18:37:31.651 | INFO | Downloaded 34GiB to GPU in 1.94045s (17.5217GiB/s) | 20260506 18:37:36.792 | INFO | Analyzing chunk frames = [3562496,3563008) | 20260506 18:37:36.792 | INFO | MemInfo: { | 20260506 18:37:36.792 | INFO | "gpu" : " 17 GB", | 20260506 18:37:36.792 | INFO | "huge" : "", | 20260506 18:37:36.792 | INFO | "huge_shared" : " 27 GB", | 20260506 18:37:36.792 | INFO | "malloc" : " 14 GB", | 20260506 18:37:36.792 | INFO | "pinned" : " 29 GB" | 20260506 18:37:36.792 | INFO | } | 20260506 18:37:36.792 | INFO | Perf report for the last 5.14146 seconds: | 20260506 18:37:36.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:37:36.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:36.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:36.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:36.792 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:37:36.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:36.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:36.792 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 18:37:36.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:36.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:36.792 | INFO | Uploaded 36GiB to GPU in 1.98733s (18.1148GiB/s) | 20260506 18:37:36.792 | INFO | Downloaded 34GiB to GPU in 2.00431s (16.9635GiB/s) | 20260506 18:37:41.928 | INFO | Analyzing chunk frames = [3563008,3563520) | 20260506 18:37:41.928 | INFO | MemInfo: { | 20260506 18:37:41.928 | INFO | "gpu" : " 17 GB", | 20260506 18:37:41.928 | INFO | "huge" : "", | 20260506 18:37:41.928 | INFO | "huge_shared" : " 27 GB", | 20260506 18:37:41.928 | INFO | "malloc" : " 14 GB", | 20260506 18:37:41.928 | INFO | "pinned" : " 29 GB" | 20260506 18:37:41.928 | INFO | } | 20260506 18:37:41.928 | INFO | Perf report for the last 5.13606 seconds: | 20260506 18:37:41.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:37:41.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:41.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:41.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:41.928 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 18:37:41.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:41.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:41.928 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:37:41.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:41.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:41.928 | INFO | Uploaded 36GiB to GPU in 2.03537s (17.6872GiB/s) | 20260506 18:37:41.928 | INFO | Downloaded 34GiB to GPU in 1.97639s (17.2031GiB/s) | 20260506 18:37:41.928 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:37:41.928 | INFO | Total mem usage: 18 GiB | 20260506 18:37:41.928 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:37:41.928 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -17.4%) | 20260506 18:37:41.928 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:37:41.928 | INFO | High Water Breakdown: | 20260506 18:37:41.928 | INFO | 61.1% : 9.37 GiB ( 82.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:37:41.928 | INFO | 38.8% : 5.95 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:37:41.928 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:37:41.928 | INFO | ------------------------------------------ | 20260506 18:37:41.928 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:37:41.928 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:37:41.928 | INFO | Total mem usage: 26.5 GiB | 20260506 18:37:41.928 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:37:41.928 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:37:41.928 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:37:41.928 | INFO | High Water Breakdown: | 20260506 18:37:41.929 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:37:41.929 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:37:41.929 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:37:41.929 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:37:41.929 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:37:41.929 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:37:41.929 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:37:41.929 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:37:41.929 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:37:41.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:37:41.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:37:41.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:37:41.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:37:41.929 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:37:41.929 | INFO | ------------------------------------------ | 20260506 18:37:41.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:37:41.929 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:37:41.929 | INFO | Total mem usage: 25.6 GiB | 20260506 18:37:41.929 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:37:41.929 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:37:41.929 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:37:41.929 | INFO | High Water Breakdown: | 20260506 18:37:41.929 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:37:41.929 | INFO | ------------------------------------------ | 20260506 18:37:41.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:37:41.929 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:37:41.929 | INFO | Total mem usage: 15.7 GiB | 20260506 18:37:41.929 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:37:41.929 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:37:41.929 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 18:37:41.929 | INFO | High Water Breakdown: | 20260506 18:37:41.929 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:37:41.929 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:37:41.929 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:37:41.929 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:37:41.929 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:37:41.929 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:37:41.929 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:37:41.929 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:37:41.929 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:37:41.929 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:37:41.929 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:37:41.929 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:37:41.929 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:37:41.929 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:37:41.929 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:37:41.929 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:37:41.929 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:37:41.929 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:37:41.929 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:37:41.929 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:37:41.929 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:37:41.929 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:37:41.929 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:37:41.929 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:37:41.929 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:37:41.929 | INFO | ------------------------------------------ | 20260506 18:37:41.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:37:41.929 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:37:41.929 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:37:41.929 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:37:41.929 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.52e+01, 1.34e+01 / 2.49e+00 | 20260506 18:37:41.929 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 9.96e-03 / 6.88e-03 | 20260506 18:37:41.929 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.37e+00, 5.55e-01 / 3.59e-01 | 20260506 18:37:41.929 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.46e+00, 1.84e-01 / 1.37e-01 | 20260506 18:37:41.929 | INFO | PerfReport: 15.95%, DME -- 3.99e+01 / 4.42e+01, 4.14e+00 / 2.75e+00 | 20260506 18:37:41.929 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.42e+00 / 2.06e+00 | 20260506 18:37:41.929 | INFO | PerfReport: 7.25%, PulseAccumulating -- 2.27e+00 / 2.86e+00, 3.91e-01 / 2.23e-01 | 20260506 18:37:41.929 | INFO | PerfReport: 3.39%, Metrics -- 1.06e+00 / 1.41e+00, 2.78e-01 / 1.53e-01 | 20260506 18:37:41.929 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.66e-01 / 4.11e-01, 5.66e-02 / 2.03e-02 | 20260506 18:37:41.929 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:37:41.929 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:37:41.929 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:37:41.929 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 18:37:41.929 | INFO | PerfReport: 21.41%, Upload -- 2.08e+01 / 0.00e+00, 6.87e+00 / 0.00e+00 | 20260506 18:37:41.929 | INFO | PerfReport: 21.18%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 18:37:41.929 | INFO | PerfReport: 32.24%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 18:37:41.929 | INFO | PerfReport: 1.30%, LockUpload -- 1.26e+00 / 0.00e+00, 5.32e+00 / 0.00e+00 | 20260506 18:37:41.929 | INFO | PerfReport: 1.20%, LockDownload -- 1.16e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260506 18:37:41.929 | INFO | PerfReport: 22.65%, LockCompute -- 2.20e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 18:37:41.929 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260506 18:37:42.059 | INFO | Starting a round of Baz IO | 20260506 18:37:45.182 | INFO | Populating RTMetrics report | 20260506 18:37:45.182 | INFO | RTMetrics report populated | 20260506 18:37:45.182 | INFO | Opening tmp RTMetrics json file | 20260506 18:37:45.224 | INFO | Serializing RTMetrics to json | 20260506 18:37:45.255 | INFO | Serialization done, swapping tmp file now... | 20260506 18:37:45.265 | INFO | RTMetrics done | 20260506 18:37:45.699 | INFO | Finished a round of Baz IO: Wrote 2.035 GiB in 3.64 seconds | 20260506 18:37:45.699 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.99% / 0.01% / 65.01% | 20260506 18:37:45.699 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.08462% / 0% / 7.94729e-05% | 20260506 18:37:45.699 | INFO | Baz IO overall: | 20260506 18:37:45.699 | INFO | BytesWritten : 1882771829400 | 20260506 18:37:45.699 | INFO | NumEvents : 910468056558 | 20260506 18:37:47.066 | INFO | Analyzing chunk frames = [3563520,3564032) | 20260506 18:37:47.066 | INFO | MemInfo: { | 20260506 18:37:47.066 | INFO | "gpu" : " 17 GB", | 20260506 18:37:47.066 | INFO | "huge" : "", | 20260506 18:37:47.066 | INFO | "huge_shared" : " 27 GB", | 20260506 18:37:47.066 | INFO | "malloc" : " 14 GB", | 20260506 18:37:47.066 | INFO | "pinned" : " 29 GB" | 20260506 18:37:47.066 | INFO | } | 20260506 18:37:47.066 | INFO | Perf report for the last 5.13828 seconds: | 20260506 18:37:47.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:37:47.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:47.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:47.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:47.066 | INFO | ANALYSIS : 62.7%, 2.9, 95.3ms, 0 | 20260506 18:37:47.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:47.066 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 18:37:47.066 | INFO | PRE_HQ : 60.0%, 1.9, 58.1ms, 0 | 20260506 18:37:47.066 | INFO | BAZWRITER : 70.6%, 4.8, 175.8ms, 0 | 20260506 18:37:47.066 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 18:37:47.066 | INFO | Uploaded 36GiB to GPU in 2.02384s (17.788GiB/s) | 20260506 18:37:47.067 | INFO | Downloaded 38GiB to GPU in 2.21695s (17.1407GiB/s) | 20260506 18:37:52.206 | INFO | Analyzing chunk frames = [3564032,3564544) | 20260506 18:37:52.206 | INFO | MemInfo: { | 20260506 18:37:52.206 | INFO | "gpu" : " 17 GB", | 20260506 18:37:52.206 | INFO | "huge" : "", | 20260506 18:37:52.206 | INFO | "huge_shared" : " 27 GB", | 20260506 18:37:52.206 | INFO | "malloc" : " 14 GB", | 20260506 18:37:52.206 | INFO | "pinned" : " 29 GB" | 20260506 18:37:52.206 | INFO | } | 20260506 18:37:52.206 | INFO | Perf report for the last 5.13958 seconds: | 20260506 18:37:52.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:37:52.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:52.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:52.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:52.206 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 18:37:52.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:52.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:52.206 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:37:52.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:52.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:52.206 | INFO | Uploaded 36GiB to GPU in 2.0206s (17.8165GiB/s) | 20260506 18:37:52.206 | INFO | Downloaded 34GiB to GPU in 1.96614s (17.2928GiB/s) | 20260506 18:37:57.351 | INFO | Analyzing chunk frames = [3564544,3565056) | 20260506 18:37:57.351 | INFO | MemInfo: { | 20260506 18:37:57.351 | INFO | "gpu" : " 17 GB", | 20260506 18:37:57.351 | INFO | "huge" : "", | 20260506 18:37:57.352 | INFO | "huge_shared" : " 27 GB", | 20260506 18:37:57.352 | INFO | "malloc" : " 14 GB", | 20260506 18:37:57.352 | INFO | "pinned" : " 29 GB" | 20260506 18:37:57.352 | INFO | } | 20260506 18:37:57.352 | INFO | Perf report for the last 5.14593 seconds: | 20260506 18:37:57.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:37:57.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:57.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:57.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:57.352 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260506 18:37:57.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:57.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:57.352 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:37:57.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:57.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:37:57.352 | INFO | Uploaded 36GiB to GPU in 1.97356s (18.2411GiB/s) | 20260506 18:37:57.352 | INFO | Downloaded 34GiB to GPU in 1.92631s (17.6503GiB/s) | 20260506 18:38:02.490 | INFO | Analyzing chunk frames = [3565056,3565568) | 20260506 18:38:02.490 | INFO | MemInfo: { | 20260506 18:38:02.490 | INFO | "gpu" : " 17 GB", | 20260506 18:38:02.490 | INFO | "huge" : "", | 20260506 18:38:02.490 | INFO | "huge_shared" : " 27 GB", | 20260506 18:38:02.490 | INFO | "malloc" : " 14 GB", | 20260506 18:38:02.490 | INFO | "pinned" : " 29 GB" | 20260506 18:38:02.490 | INFO | } | 20260506 18:38:02.490 | INFO | Perf report for the last 5.13858 seconds: | 20260506 18:38:02.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:38:02.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:02.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:02.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:02.490 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:38:02.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:02.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:02.490 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:38:02.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:02.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:02.491 | INFO | Uploaded 36GiB to GPU in 1.93636s (18.5916GiB/s) | 20260506 18:38:02.491 | INFO | Downloaded 34GiB to GPU in 1.87773s (18.107GiB/s) | 20260506 18:38:07.631 | INFO | Analyzing chunk frames = [3565568,3566080) | 20260506 18:38:07.631 | INFO | MemInfo: { | 20260506 18:38:07.631 | INFO | "gpu" : " 17 GB", | 20260506 18:38:07.631 | INFO | "huge" : "", | 20260506 18:38:07.631 | INFO | "huge_shared" : " 27 GB", | 20260506 18:38:07.631 | INFO | "malloc" : " 14 GB", | 20260506 18:38:07.631 | INFO | "pinned" : " 29 GB" | 20260506 18:38:07.631 | INFO | } | 20260506 18:38:07.631 | INFO | Perf report for the last 5.14065 seconds: | 20260506 18:38:07.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:38:07.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:07.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:07.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:07.631 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260506 18:38:07.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:07.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:07.631 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:38:07.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:07.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:07.631 | INFO | Uploaded 36GiB to GPU in 1.98766s (18.1117GiB/s) | 20260506 18:38:07.631 | INFO | Downloaded 34GiB to GPU in 1.91922s (17.7156GiB/s) | 20260506 18:38:12.767 | INFO | Analyzing chunk frames = [3566080,3566592) | 20260506 18:38:12.767 | INFO | MemInfo: { | 20260506 18:38:12.767 | INFO | "gpu" : " 17 GB", | 20260506 18:38:12.767 | INFO | "huge" : "", | 20260506 18:38:12.767 | INFO | "huge_shared" : " 27 GB", | 20260506 18:38:12.767 | INFO | "malloc" : " 14 GB", | 20260506 18:38:12.767 | INFO | "pinned" : " 29 GB" | 20260506 18:38:12.767 | INFO | } | 20260506 18:38:12.767 | INFO | Perf report for the last 5.13583 seconds: | 20260506 18:38:12.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:38:12.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:12.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:12.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:12.767 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 18:38:12.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:12.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:12.767 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 18:38:12.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:12.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:12.767 | INFO | Uploaded 36GiB to GPU in 1.98861s (18.1031GiB/s) | 20260506 18:38:12.767 | INFO | Downloaded 34GiB to GPU in 1.95221s (17.4161GiB/s) | 20260506 18:38:17.906 | INFO | Analyzing chunk frames = [3566592,3567104) | 20260506 18:38:17.906 | INFO | MemInfo: { | 20260506 18:38:17.906 | INFO | "gpu" : " 17 GB", | 20260506 18:38:17.906 | INFO | "huge" : "", | 20260506 18:38:17.907 | INFO | "huge_shared" : " 27 GB", | 20260506 18:38:17.907 | INFO | "malloc" : " 14 GB", | 20260506 18:38:17.907 | INFO | "pinned" : " 29 GB" | 20260506 18:38:17.907 | INFO | } | 20260506 18:38:17.907 | INFO | Perf report for the last 5.13986 seconds: | 20260506 18:38:17.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:38:17.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:17.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:17.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:17.907 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 18:38:17.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:17.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:17.907 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 18:38:17.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:17.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:17.907 | INFO | Uploaded 36GiB to GPU in 2.00828s (17.9258GiB/s) | 20260506 18:38:17.907 | INFO | Downloaded 34GiB to GPU in 1.97317s (17.2312GiB/s) | 20260506 18:38:23.046 | INFO | Analyzing chunk frames = [3567104,3567616) | 20260506 18:38:23.046 | INFO | MemInfo: { | 20260506 18:38:23.046 | INFO | "gpu" : " 17 GB", | 20260506 18:38:23.046 | INFO | "huge" : "", | 20260506 18:38:23.046 | INFO | "huge_shared" : " 27 GB", | 20260506 18:38:23.046 | INFO | "malloc" : " 14 GB", | 20260506 18:38:23.046 | INFO | "pinned" : " 29 GB" | 20260506 18:38:23.046 | INFO | } | 20260506 18:38:23.047 | INFO | Perf report for the last 5.13983 seconds: | 20260506 18:38:23.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:38:23.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:23.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:23.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:23.047 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 18:38:23.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:23.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:23.047 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 18:38:23.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:23.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:23.047 | INFO | Uploaded 36GiB to GPU in 2.04323s (17.6192GiB/s) | 20260506 18:38:23.047 | INFO | Downloaded 34GiB to GPU in 1.98348s (17.1416GiB/s) | 20260506 18:38:23.160 | INFO | Starting a round of Baz IO | 20260506 18:38:26.318 | INFO | Populating RTMetrics report | 20260506 18:38:26.318 | INFO | RTMetrics report populated | 20260506 18:38:26.318 | INFO | Opening tmp RTMetrics json file | 20260506 18:38:26.353 | INFO | Serializing RTMetrics to json | 20260506 18:38:26.383 | INFO | Serialization done, swapping tmp file now... | 20260506 18:38:26.384 | INFO | RTMetrics done | 20260506 18:38:26.820 | INFO | Finished a round of Baz IO: Wrote 2.035 GiB in 3.66 seconds | 20260506 18:38:26.820 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.98% / 0.01% / 65.01% | 20260506 18:38:26.820 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.08115% / 7.94729e-06% / 5.96046e-05% | 20260506 18:38:26.820 | INFO | Baz IO overall: | 20260506 18:38:26.820 | INFO | BytesWritten : 1884956363920 | 20260506 18:38:26.820 | INFO | NumEvents : 911521022723 | 20260506 18:38:28.181 | INFO | Analyzing chunk frames = [3567616,3568128) | 20260506 18:38:28.181 | INFO | MemInfo: { | 20260506 18:38:28.181 | INFO | "gpu" : " 17 GB", | 20260506 18:38:28.181 | INFO | "huge" : "", | 20260506 18:38:28.181 | INFO | "huge_shared" : " 27 GB", | 20260506 18:38:28.181 | INFO | "malloc" : " 14 GB", | 20260506 18:38:28.181 | INFO | "pinned" : " 29 GB" | 20260506 18:38:28.181 | INFO | } | 20260506 18:38:28.181 | INFO | Perf report for the last 5.13426 seconds: | 20260506 18:38:28.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:38:28.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:28.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:28.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:28.181 | INFO | ANALYSIS : 63.0%, 2.9, 94.5ms, 0 | 20260506 18:38:28.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:28.181 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260506 18:38:28.181 | INFO | PRE_HQ : 61.1%, 1.9, 60.7ms, 0 | 20260506 18:38:28.181 | INFO | BAZWRITER : 70.7%, 4.8, 174.6ms, 0 | 20260506 18:38:28.181 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 18:38:28.181 | INFO | Uploaded 36GiB to GPU in 2.03868s (17.6585GiB/s) | 20260506 18:38:28.181 | INFO | Downloaded 38GiB to GPU in 2.22377s (17.0881GiB/s) | 20260506 18:38:33.325 | INFO | Analyzing chunk frames = [3568128,3568640) | 20260506 18:38:33.325 | INFO | MemInfo: { | 20260506 18:38:33.325 | INFO | "gpu" : " 17 GB", | 20260506 18:38:33.325 | INFO | "huge" : "", | 20260506 18:38:33.325 | INFO | "huge_shared" : " 27 GB", | 20260506 18:38:33.325 | INFO | "malloc" : " 14 GB", | 20260506 18:38:33.326 | INFO | "pinned" : " 29 GB" | 20260506 18:38:33.326 | INFO | } | 20260506 18:38:33.326 | INFO | Perf report for the last 5.14473 seconds: | 20260506 18:38:33.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:38:33.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:33.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:33.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:33.326 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 18:38:33.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:33.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:33.326 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:38:33.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:33.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:33.326 | INFO | Uploaded 36GiB to GPU in 1.99387s (18.0553GiB/s) | 20260506 18:38:33.326 | INFO | Downloaded 34GiB to GPU in 1.93462s (17.5745GiB/s) | 20260506 18:38:38.471 | INFO | Analyzing chunk frames = [3568640,3569152) | 20260506 18:38:38.471 | INFO | MemInfo: { | 20260506 18:38:38.471 | INFO | "gpu" : " 17 GB", | 20260506 18:38:38.471 | INFO | "huge" : "", | 20260506 18:38:38.471 | INFO | "huge_shared" : " 27 GB", | 20260506 18:38:38.471 | INFO | "malloc" : " 14 GB", | 20260506 18:38:38.471 | INFO | "pinned" : " 29 GB" | 20260506 18:38:38.471 | INFO | } | 20260506 18:38:38.471 | INFO | Perf report for the last 5.1458 seconds: | 20260506 18:38:38.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:38:38.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:38.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:38.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:38.471 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:38:38.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:38.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:38.472 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:38:38.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:38.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:38.472 | INFO | Uploaded 36GiB to GPU in 1.98791s (18.1095GiB/s) | 20260506 18:38:38.472 | INFO | Downloaded 34GiB to GPU in 1.91618s (17.7437GiB/s) | 20260506 18:38:43.604 | INFO | Analyzing chunk frames = [3569152,3569664) | 20260506 18:38:43.605 | INFO | MemInfo: { | 20260506 18:38:43.605 | INFO | "gpu" : " 17 GB", | 20260506 18:38:43.605 | INFO | "huge" : "", | 20260506 18:38:43.605 | INFO | "huge_shared" : " 27 GB", | 20260506 18:38:43.605 | INFO | "malloc" : " 14 GB", | 20260506 18:38:43.605 | INFO | "pinned" : " 29 GB" | 20260506 18:38:43.605 | INFO | } | 20260506 18:38:43.605 | INFO | Perf report for the last 5.13337 seconds: | 20260506 18:38:43.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:38:43.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:43.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:43.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:43.605 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 18:38:43.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:43.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:43.605 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:38:43.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:43.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:43.605 | INFO | Uploaded 36GiB to GPU in 1.94172s (18.5402GiB/s) | 20260506 18:38:43.605 | INFO | Downloaded 34GiB to GPU in 1.86703s (18.2107GiB/s) | 20260506 18:38:48.743 | INFO | Analyzing chunk frames = [3569664,3570176) | 20260506 18:38:48.743 | INFO | MemInfo: { | 20260506 18:38:48.743 | INFO | "gpu" : " 17 GB", | 20260506 18:38:48.743 | INFO | "huge" : "", | 20260506 18:38:48.743 | INFO | "huge_shared" : " 27 GB", | 20260506 18:38:48.743 | INFO | "malloc" : " 14 GB", | 20260506 18:38:48.744 | INFO | "pinned" : " 29 GB" | 20260506 18:38:48.744 | INFO | } | 20260506 18:38:48.744 | INFO | Perf report for the last 5.13928 seconds: | 20260506 18:38:48.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:38:48.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:48.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:48.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:48.744 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 18:38:48.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:48.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:48.744 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:38:48.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:48.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:48.744 | INFO | Uploaded 36GiB to GPU in 2.00672s (17.9397GiB/s) | 20260506 18:38:48.744 | INFO | Downloaded 34GiB to GPU in 1.91636s (17.742GiB/s) | 20260506 18:38:53.881 | INFO | Analyzing chunk frames = [3570176,3570688) | 20260506 18:38:53.881 | INFO | MemInfo: { | 20260506 18:38:53.881 | INFO | "gpu" : " 17 GB", | 20260506 18:38:53.881 | INFO | "huge" : "", | 20260506 18:38:53.881 | INFO | "huge_shared" : " 27 GB", | 20260506 18:38:53.881 | INFO | "malloc" : " 14 GB", | 20260506 18:38:53.881 | INFO | "pinned" : " 29 GB" | 20260506 18:38:53.881 | INFO | } | 20260506 18:38:53.881 | INFO | Perf report for the last 5.13715 seconds: | 20260506 18:38:53.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:38:53.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:53.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:53.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:53.881 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 18:38:53.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:53.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:53.881 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 18:38:53.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:53.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:53.881 | INFO | Uploaded 36GiB to GPU in 2.00994s (17.9109GiB/s) | 20260506 18:38:53.881 | INFO | Downloaded 34GiB to GPU in 1.94421s (17.4878GiB/s) | 20260506 18:38:59.019 | INFO | Analyzing chunk frames = [3570688,3571200) | 20260506 18:38:59.019 | INFO | MemInfo: { | 20260506 18:38:59.019 | INFO | "gpu" : " 17 GB", | 20260506 18:38:59.019 | INFO | "huge" : "", | 20260506 18:38:59.019 | INFO | "huge_shared" : " 27 GB", | 20260506 18:38:59.019 | INFO | "malloc" : " 14 GB", | 20260506 18:38:59.019 | INFO | "pinned" : " 29 GB" | 20260506 18:38:59.019 | INFO | } | 20260506 18:38:59.020 | INFO | Perf report for the last 5.13824 seconds: | 20260506 18:38:59.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:38:59.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:59.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:59.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:59.020 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:38:59.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:59.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:59.020 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:38:59.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:59.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:38:59.020 | INFO | Uploaded 36GiB to GPU in 2.00903s (17.9191GiB/s) | 20260506 18:38:59.020 | INFO | Downloaded 34GiB to GPU in 2.00633s (16.9463GiB/s) | 20260506 18:39:04.157 | INFO | Analyzing chunk frames = [3571200,3571712) | 20260506 18:39:04.157 | INFO | MemInfo: { | 20260506 18:39:04.157 | INFO | "gpu" : " 17 GB", | 20260506 18:39:04.157 | INFO | "huge" : "", | 20260506 18:39:04.157 | INFO | "huge_shared" : " 27 GB", | 20260506 18:39:04.157 | INFO | "malloc" : " 14 GB", | 20260506 18:39:04.157 | INFO | "pinned" : " 29 GB" | 20260506 18:39:04.157 | INFO | } | 20260506 18:39:04.157 | INFO | Perf report for the last 5.13749 seconds: | 20260506 18:39:04.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:39:04.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:04.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:04.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:04.157 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:39:04.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:04.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:04.157 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:39:04.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:04.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:04.157 | INFO | Uploaded 36GiB to GPU in 2.01348s (17.8795GiB/s) | 20260506 18:39:04.157 | INFO | Downloaded 34GiB to GPU in 1.99296s (17.06GiB/s) | 20260506 18:39:04.157 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:39:04.157 | INFO | Total mem usage: 18 GiB | 20260506 18:39:04.157 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:39:04.157 | INFO | Recent High Water Marks Sum: 16.6 GiB (Savings: -8.73%) | 20260506 18:39:04.157 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:39:04.157 | INFO | High Water Breakdown: | 20260506 18:39:04.157 | INFO | 62.4% : 10.3 GiB ( 90.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:39:04.158 | INFO | 37.5% : 6.21 GiB ( 92.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:39:04.158 | INFO | 0.12% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:39:04.158 | INFO | ------------------------------------------ | 20260506 18:39:04.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:39:04.158 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:39:04.158 | INFO | Total mem usage: 26.5 GiB | 20260506 18:39:04.158 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:39:04.158 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:39:04.158 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:39:04.158 | INFO | High Water Breakdown: | 20260506 18:39:04.158 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:39:04.158 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:39:04.158 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:39:04.158 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:39:04.158 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:39:04.158 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:39:04.158 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:39:04.158 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:39:04.158 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:39:04.158 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:39:04.158 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:39:04.158 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:39:04.158 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:39:04.158 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:39:04.158 | INFO | ------------------------------------------ | 20260506 18:39:04.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:39:04.158 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:39:04.158 | INFO | Total mem usage: 25.6 GiB | 20260506 18:39:04.158 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:39:04.158 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:39:04.158 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:39:04.158 | INFO | High Water Breakdown: | 20260506 18:39:04.158 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:39:04.158 | INFO | ------------------------------------------ | 20260506 18:39:04.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:39:04.158 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:39:04.158 | INFO | Total mem usage: 15.7 GiB | 20260506 18:39:04.158 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:39:04.158 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:39:04.158 | INFO | Unused Cache: 862 MiB (Waste: 5.37%) | 20260506 18:39:04.158 | INFO | High Water Breakdown: | 20260506 18:39:04.158 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:39:04.158 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:39:04.158 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:39:04.158 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:39:04.158 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:39:04.158 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:39:04.158 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:39:04.158 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:39:04.158 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:39:04.158 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:39:04.159 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:39:04.159 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:39:04.159 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:39:04.159 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:39:04.159 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:39:04.159 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:39:04.159 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:39:04.159 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:39:04.159 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:39:04.159 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:39:04.159 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:39:04.159 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:39:04.159 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:39:04.159 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:39:04.159 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:39:04.159 | INFO | ------------------------------------------ | 20260506 18:39:04.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:39:04.159 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:39:04.159 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:39:04.159 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:39:04.159 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.58e+01, 1.35e+01 / 2.06e+00 | 20260506 18:39:04.159 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 1.10e-02 / 5.43e-03 | 20260506 18:39:04.159 | INFO | PerfReport: 11.63%, Baseline -- 3.64e+00 / 4.42e+00, 5.58e-01 / 2.95e-01 | 20260506 18:39:04.159 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.48e+00, 1.87e-01 / 1.33e-01 | 20260506 18:39:04.159 | INFO | PerfReport: 15.93%, DME -- 3.98e+01 / 4.43e+01, 4.06e+00 / 2.57e+00 | 20260506 18:39:04.159 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.42e+00 / 2.08e+00 | 20260506 18:39:04.159 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.26e+00 / 2.76e+00, 3.79e-01 / 2.65e-01 | 20260506 18:39:04.159 | INFO | PerfReport: 3.40%, Metrics -- 1.06e+00 / 1.40e+00, 2.80e-01 / 1.56e-01 | 20260506 18:39:04.159 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.67e-01 / 4.15e-01, 5.74e-02 / 2.50e-02 | 20260506 18:39:04.159 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:39:04.159 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:39:04.159 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:39:04.159 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 18:39:04.159 | INFO | PerfReport: 21.46%, Upload -- 2.08e+01 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260506 18:39:04.159 | INFO | PerfReport: 21.22%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 18:39:04.159 | INFO | PerfReport: 32.24%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:39:04.159 | INFO | PerfReport: 1.36%, LockUpload -- 1.32e+00 / 0.00e+00, 5.39e+00 / 0.00e+00 | 20260506 18:39:04.159 | INFO | PerfReport: 1.16%, LockDownload -- 1.12e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 18:39:04.159 | INFO | PerfReport: 22.55%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:39:04.159 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260506 18:39:04.271 | INFO | Starting a round of Baz IO | 20260506 18:39:07.407 | INFO | Populating RTMetrics report | 20260506 18:39:07.407 | INFO | RTMetrics report populated | 20260506 18:39:07.407 | INFO | Opening tmp RTMetrics json file | 20260506 18:39:07.470 | INFO | Serializing RTMetrics to json | 20260506 18:39:07.481 | INFO | Serialization done, swapping tmp file now... | 20260506 18:39:07.482 | INFO | RTMetrics done | 20260506 18:39:07.622 | INFO | Finished a round of Baz IO: Wrote 2.03 GiB in 3.351 seconds | 20260506 18:39:07.622 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.98% / 0.01% / 65.02% | 20260506 18:39:07.622 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.07421% / 3.97364e-06% / 8.34465e-05% | 20260506 18:39:07.622 | INFO | Baz IO overall: | 20260506 18:39:07.622 | INFO | BytesWritten : 1887136453392 | 20260506 18:39:07.622 | INFO | NumEvents : 912570468057 | 20260506 18:39:09.297 | INFO | Analyzing chunk frames = [3571712,3572224) | 20260506 18:39:09.297 | INFO | MemInfo: { | 20260506 18:39:09.297 | INFO | "gpu" : " 17 GB", | 20260506 18:39:09.297 | INFO | "huge" : "", | 20260506 18:39:09.297 | INFO | "huge_shared" : " 27 GB", | 20260506 18:39:09.297 | INFO | "malloc" : " 14 GB", | 20260506 18:39:09.297 | INFO | "pinned" : " 29 GB" | 20260506 18:39:09.297 | INFO | } | 20260506 18:39:09.297 | INFO | Perf report for the last 5.13993 seconds: | 20260506 18:39:09.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:39:09.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:09.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:09.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:09.297 | INFO | ANALYSIS : 62.6%, 2.9, 95.3ms, 0 | 20260506 18:39:09.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:09.297 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 18:39:09.297 | INFO | PRE_HQ : 60.2%, 1.8, 57.8ms, 0 | 20260506 18:39:09.297 | INFO | BAZWRITER : 64.8%, 4.8, 159.3ms, 0 | 20260506 18:39:09.297 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 18:39:09.297 | INFO | Uploaded 36GiB to GPU in 2.0179s (17.8403GiB/s) | 20260506 18:39:09.297 | INFO | Downloaded 38GiB to GPU in 2.21198s (17.1792GiB/s) | 20260506 18:39:14.447 | INFO | Analyzing chunk frames = [3572224,3572736) | 20260506 18:39:14.447 | INFO | MemInfo: { | 20260506 18:39:14.447 | INFO | "gpu" : " 17 GB", | 20260506 18:39:14.447 | INFO | "huge" : "", | 20260506 18:39:14.447 | INFO | "huge_shared" : " 27 GB", | 20260506 18:39:14.447 | INFO | "malloc" : " 14 GB", | 20260506 18:39:14.447 | INFO | "pinned" : " 29 GB" | 20260506 18:39:14.447 | INFO | } | 20260506 18:39:14.447 | INFO | Perf report for the last 5.15008 seconds: | 20260506 18:39:14.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:39:14.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:14.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:14.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:14.447 | INFO | ANALYSIS : 61.6%, 3.0, 96.2ms, 0 | 20260506 18:39:14.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:14.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:14.447 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 18:39:14.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:14.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:14.447 | INFO | Uploaded 36GiB to GPU in 2.02205s (17.8037GiB/s) | 20260506 18:39:14.447 | INFO | Downloaded 34GiB to GPU in 1.96008s (17.3462GiB/s) | 20260506 18:39:19.585 | INFO | Analyzing chunk frames = [3572736,3573248) | 20260506 18:39:19.585 | INFO | MemInfo: { | 20260506 18:39:19.585 | INFO | "gpu" : " 17 GB", | 20260506 18:39:19.585 | INFO | "huge" : "", | 20260506 18:39:19.585 | INFO | "huge_shared" : " 27 GB", | 20260506 18:39:19.585 | INFO | "malloc" : " 14 GB", | 20260506 18:39:19.586 | INFO | "pinned" : " 29 GB" | 20260506 18:39:19.586 | INFO | } | 20260506 18:39:19.586 | INFO | Perf report for the last 5.13849 seconds: | 20260506 18:39:19.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:39:19.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:19.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:19.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:19.586 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 18:39:19.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:19.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:19.586 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:39:19.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:19.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:19.586 | INFO | Uploaded 36GiB to GPU in 1.98815s (18.1073GiB/s) | 20260506 18:39:19.586 | INFO | Downloaded 34GiB to GPU in 1.95103s (17.4267GiB/s) | 20260506 18:39:24.724 | INFO | Analyzing chunk frames = [3573248,3573760) | 20260506 18:39:24.724 | INFO | MemInfo: { | 20260506 18:39:24.724 | INFO | "gpu" : " 17 GB", | 20260506 18:39:24.724 | INFO | "huge" : "", | 20260506 18:39:24.724 | INFO | "huge_shared" : " 27 GB", | 20260506 18:39:24.724 | INFO | "malloc" : " 14 GB", | 20260506 18:39:24.724 | INFO | "pinned" : " 29 GB" | 20260506 18:39:24.724 | INFO | } | 20260506 18:39:24.724 | INFO | Perf report for the last 5.13888 seconds: | 20260506 18:39:24.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:39:24.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:24.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:24.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:24.725 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 18:39:24.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:24.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:24.725 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:39:24.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:24.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:24.725 | INFO | Uploaded 36GiB to GPU in 1.9519s (18.4436GiB/s) | 20260506 18:39:24.725 | INFO | Downloaded 34GiB to GPU in 1.87524s (18.131GiB/s) | 20260506 18:39:29.854 | INFO | Analyzing chunk frames = [3573760,3574272) | 20260506 18:39:29.854 | INFO | MemInfo: { | 20260506 18:39:29.854 | INFO | "gpu" : " 17 GB", | 20260506 18:39:29.854 | INFO | "huge" : "", | 20260506 18:39:29.854 | INFO | "huge_shared" : " 27 GB", | 20260506 18:39:29.854 | INFO | "malloc" : " 14 GB", | 20260506 18:39:29.854 | INFO | "pinned" : " 29 GB" | 20260506 18:39:29.854 | INFO | } | 20260506 18:39:29.854 | INFO | Perf report for the last 5.12931 seconds: | 20260506 18:39:29.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:39:29.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:29.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:29.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:29.854 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260506 18:39:29.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:29.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:29.854 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:39:29.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:29.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:29.854 | INFO | Uploaded 36GiB to GPU in 2.01443s (17.8711GiB/s) | 20260506 18:39:29.854 | INFO | Downloaded 34GiB to GPU in 1.91541s (17.7508GiB/s) | 20260506 18:39:34.993 | INFO | Analyzing chunk frames = [3574272,3574784) | 20260506 18:39:34.993 | INFO | MemInfo: { | 20260506 18:39:34.993 | INFO | "gpu" : " 17 GB", | 20260506 18:39:34.993 | INFO | "huge" : "", | 20260506 18:39:34.993 | INFO | "huge_shared" : " 27 GB", | 20260506 18:39:34.994 | INFO | "malloc" : " 14 GB", | 20260506 18:39:34.994 | INFO | "pinned" : " 29 GB" | 20260506 18:39:34.994 | INFO | } | 20260506 18:39:34.994 | INFO | Perf report for the last 5.13986 seconds: | 20260506 18:39:34.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:39:34.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:34.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:34.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:34.994 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 18:39:34.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:34.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:34.994 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 18:39:34.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:34.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:34.994 | INFO | Uploaded 36GiB to GPU in 2.01148s (17.8972GiB/s) | 20260506 18:39:34.994 | INFO | Downloaded 34GiB to GPU in 1.94424s (17.4875GiB/s) | 20260506 18:39:40.132 | INFO | Analyzing chunk frames = [3574784,3575296) | 20260506 18:39:40.132 | INFO | MemInfo: { | 20260506 18:39:40.132 | INFO | "gpu" : " 17 GB", | 20260506 18:39:40.132 | INFO | "huge" : "", | 20260506 18:39:40.132 | INFO | "huge_shared" : " 27 GB", | 20260506 18:39:40.132 | INFO | "malloc" : " 14 GB", | 20260506 18:39:40.132 | INFO | "pinned" : " 29 GB" | 20260506 18:39:40.132 | INFO | } | 20260506 18:39:40.132 | INFO | Perf report for the last 5.1386 seconds: | 20260506 18:39:40.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:39:40.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:40.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:40.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:40.132 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 18:39:40.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:40.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:40.132 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 18:39:40.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:40.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:40.133 | INFO | Uploaded 36GiB to GPU in 1.98616s (18.1254GiB/s) | 20260506 18:39:40.133 | INFO | Downloaded 34GiB to GPU in 2.00773s (16.9346GiB/s) | 20260506 18:39:45.270 | INFO | Analyzing chunk frames = [3575296,3575808) | 20260506 18:39:45.270 | INFO | MemInfo: { | 20260506 18:39:45.270 | INFO | "gpu" : " 17 GB", | 20260506 18:39:45.270 | INFO | "huge" : "", | 20260506 18:39:45.270 | INFO | "huge_shared" : " 27 GB", | 20260506 18:39:45.270 | INFO | "malloc" : " 14 GB", | 20260506 18:39:45.270 | INFO | "pinned" : " 29 GB" | 20260506 18:39:45.270 | INFO | } | 20260506 18:39:45.270 | INFO | Perf report for the last 5.13816 seconds: | 20260506 18:39:45.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:39:45.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:45.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:45.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:45.270 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:39:45.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:45.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:45.271 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:39:45.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:45.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:45.271 | INFO | Uploaded 36GiB to GPU in 1.99788s (18.0191GiB/s) | 20260506 18:39:45.271 | INFO | Downloaded 34GiB to GPU in 1.9915s (17.0725GiB/s) | 20260506 18:39:45.385 | INFO | Starting a round of Baz IO | 20260506 18:39:48.516 | INFO | Populating RTMetrics report | 20260506 18:39:48.516 | INFO | RTMetrics report populated | 20260506 18:39:48.516 | INFO | Opening tmp RTMetrics json file | 20260506 18:39:48.518 | INFO | Serializing RTMetrics to json | 20260506 18:39:48.523 | INFO | Serialization done, swapping tmp file now... | 20260506 18:39:48.549 | INFO | RTMetrics done | 20260506 18:39:48.686 | INFO | Finished a round of Baz IO: Wrote 2.03 GiB in 3.301 seconds | 20260506 18:39:48.686 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.97% / 0.01% / 65.02% | 20260506 18:39:48.686 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.07037% / 7.94729e-06% / 7.15256e-05% | 20260506 18:39:48.686 | INFO | Baz IO overall: | 20260506 18:39:48.686 | INFO | BytesWritten : 1889316199003 | 20260506 18:39:48.686 | INFO | NumEvents : 913619470458 | 20260506 18:39:50.417 | INFO | Analyzing chunk frames = [3575808,3576320) | 20260506 18:39:50.417 | INFO | MemInfo: { | 20260506 18:39:50.417 | INFO | "gpu" : " 17 GB", | 20260506 18:39:50.417 | INFO | "huge" : "", | 20260506 18:39:50.417 | INFO | "huge_shared" : " 27 GB", | 20260506 18:39:50.417 | INFO | "malloc" : " 14 GB", | 20260506 18:39:50.417 | INFO | "pinned" : " 29 GB" | 20260506 18:39:50.417 | INFO | } | 20260506 18:39:50.417 | INFO | Perf report for the last 5.14689 seconds: | 20260506 18:39:50.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:39:50.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:50.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:50.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:50.417 | INFO | ANALYSIS : 62.5%, 2.9, 95.2ms, 0 | 20260506 18:39:50.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:50.417 | INFO | RT_METRICS : 43.7%, 1.0, 23.0ms, 0 | 20260506 18:39:50.417 | INFO | PRE_HQ : 60.1%, 1.8, 57.6ms, 0 | 20260506 18:39:50.417 | INFO | BAZWRITER : 63.8%, 4.6, 150.1ms, 0 | 20260506 18:39:50.417 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.4ms, 0 | 20260506 18:39:50.418 | INFO | Uploaded 36GiB to GPU in 2.01651s (17.8527GiB/s) | 20260506 18:39:50.418 | INFO | Downloaded 38GiB to GPU in 2.21104s (17.1865GiB/s) | 20260506 18:39:55.559 | INFO | Analyzing chunk frames = [3576320,3576832) | 20260506 18:39:55.559 | INFO | MemInfo: { | 20260506 18:39:55.559 | INFO | "gpu" : " 17 GB", | 20260506 18:39:55.559 | INFO | "huge" : "", | 20260506 18:39:55.559 | INFO | "huge_shared" : " 27 GB", | 20260506 18:39:55.559 | INFO | "malloc" : " 14 GB", | 20260506 18:39:55.559 | INFO | "pinned" : " 29 GB" | 20260506 18:39:55.559 | INFO | } | 20260506 18:39:55.559 | INFO | Perf report for the last 5.14168 seconds: | 20260506 18:39:55.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:39:55.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:55.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:55.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:55.559 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 18:39:55.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:55.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:55.559 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 18:39:55.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:55.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:39:55.559 | INFO | Uploaded 36GiB to GPU in 1.99876s (18.0112GiB/s) | 20260506 18:39:55.559 | INFO | Downloaded 34GiB to GPU in 1.92738s (17.6405GiB/s) | 20260506 18:40:00.696 | INFO | Analyzing chunk frames = [3576832,3577344) | 20260506 18:40:00.696 | INFO | MemInfo: { | 20260506 18:40:00.696 | INFO | "gpu" : " 17 GB", | 20260506 18:40:00.696 | INFO | "huge" : "", | 20260506 18:40:00.696 | INFO | "huge_shared" : " 27 GB", | 20260506 18:40:00.696 | INFO | "malloc" : " 14 GB", | 20260506 18:40:00.696 | INFO | "pinned" : " 29 GB" | 20260506 18:40:00.696 | INFO | } | 20260506 18:40:00.696 | INFO | Perf report for the last 5.13714 seconds: | 20260506 18:40:00.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:40:00.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:00.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:00.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:00.696 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 18:40:00.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:00.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:00.696 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:40:00.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:00.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:00.697 | INFO | Uploaded 36GiB to GPU in 1.97484s (18.2294GiB/s) | 20260506 18:40:00.697 | INFO | Downloaded 34GiB to GPU in 1.92007s (17.7077GiB/s) | 20260506 18:40:05.834 | INFO | Analyzing chunk frames = [3577344,3577856) | 20260506 18:40:05.834 | INFO | MemInfo: { | 20260506 18:40:05.834 | INFO | "gpu" : " 17 GB", | 20260506 18:40:05.834 | INFO | "huge" : "", | 20260506 18:40:05.834 | INFO | "huge_shared" : " 27 GB", | 20260506 18:40:05.834 | INFO | "malloc" : " 14 GB", | 20260506 18:40:05.834 | INFO | "pinned" : " 29 GB" | 20260506 18:40:05.834 | INFO | } | 20260506 18:40:05.834 | INFO | Perf report for the last 5.13776 seconds: | 20260506 18:40:05.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:40:05.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:05.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:05.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:05.834 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 18:40:05.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:05.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:05.834 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:40:05.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:05.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:05.834 | INFO | Uploaded 36GiB to GPU in 1.92771s (18.6751GiB/s) | 20260506 18:40:05.834 | INFO | Downloaded 34GiB to GPU in 1.87345s (18.1484GiB/s) | 20260506 18:40:10.971 | INFO | Analyzing chunk frames = [3577856,3578368) | 20260506 18:40:10.971 | INFO | MemInfo: { | 20260506 18:40:10.971 | INFO | "gpu" : " 17 GB", | 20260506 18:40:10.971 | INFO | "huge" : "", | 20260506 18:40:10.971 | INFO | "huge_shared" : " 27 GB", | 20260506 18:40:10.971 | INFO | "malloc" : " 14 GB", | 20260506 18:40:10.971 | INFO | "pinned" : " 29 GB" | 20260506 18:40:10.971 | INFO | } | 20260506 18:40:10.971 | INFO | Perf report for the last 5.13727 seconds: | 20260506 18:40:10.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:40:10.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:10.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:10.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:10.971 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260506 18:40:10.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:10.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:10.971 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:40:10.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:10.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:10.972 | INFO | Uploaded 36GiB to GPU in 1.95495s (18.4148GiB/s) | 20260506 18:40:10.972 | INFO | Downloaded 34GiB to GPU in 1.88896s (17.9994GiB/s) | 20260506 18:40:16.111 | INFO | Analyzing chunk frames = [3578368,3578880) | 20260506 18:40:16.111 | INFO | MemInfo: { | 20260506 18:40:16.111 | INFO | "gpu" : " 17 GB", | 20260506 18:40:16.111 | INFO | "huge" : "", | 20260506 18:40:16.112 | INFO | "huge_shared" : " 27 GB", | 20260506 18:40:16.112 | INFO | "malloc" : " 14 GB", | 20260506 18:40:16.112 | INFO | "pinned" : " 29 GB" | 20260506 18:40:16.112 | INFO | } | 20260506 18:40:16.112 | INFO | Perf report for the last 5.14049 seconds: | 20260506 18:40:16.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:40:16.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:16.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:16.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:16.112 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 18:40:16.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:16.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:16.112 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 18:40:16.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:16.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:16.112 | INFO | Uploaded 36GiB to GPU in 1.986s (18.1269GiB/s) | 20260506 18:40:16.112 | INFO | Downloaded 34GiB to GPU in 1.96163s (17.3325GiB/s) | 20260506 18:40:21.246 | INFO | Analyzing chunk frames = [3578880,3579392) | 20260506 18:40:21.246 | INFO | MemInfo: { | 20260506 18:40:21.246 | INFO | "gpu" : " 17 GB", | 20260506 18:40:21.246 | INFO | "huge" : "", | 20260506 18:40:21.247 | INFO | "huge_shared" : " 27 GB", | 20260506 18:40:21.247 | INFO | "malloc" : " 14 GB", | 20260506 18:40:21.247 | INFO | "pinned" : " 29 GB" | 20260506 18:40:21.247 | INFO | } | 20260506 18:40:21.247 | INFO | Perf report for the last 5.13506 seconds: | 20260506 18:40:21.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:40:21.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:21.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:21.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:21.247 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260506 18:40:21.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:21.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:21.247 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:40:21.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:21.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:21.247 | INFO | Uploaded 36GiB to GPU in 2.0138s (17.8766GiB/s) | 20260506 18:40:21.247 | INFO | Downloaded 34GiB to GPU in 1.99714s (17.0243GiB/s) | 20260506 18:40:26.392 | INFO | Analyzing chunk frames = [3579392,3579904) | 20260506 18:40:26.392 | INFO | MemInfo: { | 20260506 18:40:26.392 | INFO | "gpu" : " 17 GB", | 20260506 18:40:26.392 | INFO | "huge" : "", | 20260506 18:40:26.392 | INFO | "huge_shared" : " 27 GB", | 20260506 18:40:26.392 | INFO | "malloc" : " 14 GB", | 20260506 18:40:26.393 | INFO | "pinned" : " 29 GB" | 20260506 18:40:26.393 | INFO | } | 20260506 18:40:26.393 | INFO | Perf report for the last 5.14572 seconds: | 20260506 18:40:26.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:40:26.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:26.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:26.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:26.393 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 18:40:26.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:26.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:26.393 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 18:40:26.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:26.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:26.393 | INFO | Uploaded 36GiB to GPU in 2.07369s (17.3604GiB/s) | 20260506 18:40:26.393 | INFO | Downloaded 34GiB to GPU in 2.01s (16.9154GiB/s) | 20260506 18:40:26.393 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:40:26.393 | INFO | Total mem usage: 18 GiB | 20260506 18:40:26.393 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:40:26.393 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -19.7%) | 20260506 18:40:26.393 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:40:26.393 | INFO | High Water Breakdown: | 20260506 18:40:26.393 | INFO | 60.8% : 9.15 GiB ( 80.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:40:26.393 | INFO | 39% : 5.87 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:40:26.393 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:40:26.393 | INFO | ------------------------------------------ | 20260506 18:40:26.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:40:26.393 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:40:26.393 | INFO | Total mem usage: 26.5 GiB | 20260506 18:40:26.393 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:40:26.393 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:40:26.393 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:40:26.393 | INFO | High Water Breakdown: | 20260506 18:40:26.393 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:40:26.393 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:40:26.393 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:40:26.393 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:40:26.393 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:40:26.393 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:40:26.393 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:40:26.393 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:40:26.393 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:40:26.393 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:40:26.393 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:40:26.393 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:40:26.393 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:40:26.393 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:40:26.393 | INFO | ------------------------------------------ | 20260506 18:40:26.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:40:26.393 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:40:26.393 | INFO | Total mem usage: 25.6 GiB | 20260506 18:40:26.393 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:40:26.393 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:40:26.393 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:40:26.393 | INFO | High Water Breakdown: | 20260506 18:40:26.393 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:40:26.393 | INFO | ------------------------------------------ | 20260506 18:40:26.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:40:26.393 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:40:26.393 | INFO | Total mem usage: 15.7 GiB | 20260506 18:40:26.393 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:40:26.394 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:40:26.394 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 18:40:26.394 | INFO | High Water Breakdown: | 20260506 18:40:26.394 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:40:26.394 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:40:26.394 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:40:26.394 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:40:26.394 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:40:26.394 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:40:26.394 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:40:26.394 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:40:26.394 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:40:26.394 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:40:26.394 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:40:26.394 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:40:26.394 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:40:26.394 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:40:26.394 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:40:26.394 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:40:26.394 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:40:26.394 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:40:26.394 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:40:26.394 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:40:26.394 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:40:26.394 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:40:26.394 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:40:26.394 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:40:26.394 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:40:26.394 | INFO | ------------------------------------------ | 20260506 18:40:26.394 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:40:26.394 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:40:26.394 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:40:26.394 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:40:26.394 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.56e+01, 1.35e+01 / 2.29e+00 | 20260506 18:40:26.394 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.02e-01, 9.90e-03 / 7.52e-03 | 20260506 18:40:26.394 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.43e+00, 5.51e-01 / 3.16e-01 | 20260506 18:40:26.394 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.46e+00, 1.78e-01 / 1.27e-01 | 20260506 18:40:26.394 | INFO | PerfReport: 15.94%, DME -- 3.98e+01 / 4.44e+01, 3.87e+00 / 2.40e+00 | 20260506 18:40:26.394 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.38e+00 / 1.89e+00 | 20260506 18:40:26.394 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.26e+00 / 2.87e+00, 3.83e-01 / 1.87e-01 | 20260506 18:40:26.394 | INFO | PerfReport: 3.39%, Metrics -- 1.06e+00 / 1.40e+00, 2.75e-01 / 1.45e-01 | 20260506 18:40:26.394 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.66e-01 / 4.15e-01, 5.68e-02 / 2.46e-02 | 20260506 18:40:26.394 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:40:26.394 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:40:26.394 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:40:26.394 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 18:40:26.394 | INFO | PerfReport: 21.45%, Upload -- 2.08e+01 / 0.00e+00, 6.87e+00 / 0.00e+00 | 20260506 18:40:26.394 | INFO | PerfReport: 21.26%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 18:40:26.394 | INFO | PerfReport: 32.21%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:40:26.394 | INFO | PerfReport: 1.33%, LockUpload -- 1.29e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260506 18:40:26.394 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260506 18:40:26.394 | INFO | PerfReport: 22.51%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 18:40:26.394 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260506 18:40:26.512 | INFO | Starting a round of Baz IO | 20260506 18:40:29.678 | INFO | Populating RTMetrics report | 20260506 18:40:29.678 | INFO | RTMetrics report populated | 20260506 18:40:29.678 | INFO | Opening tmp RTMetrics json file | 20260506 18:40:29.732 | INFO | Serializing RTMetrics to json | 20260506 18:40:29.758 | INFO | Serialization done, swapping tmp file now... | 20260506 18:40:29.761 | INFO | RTMetrics done | 20260506 18:40:30.025 | INFO | Finished a round of Baz IO: Wrote 2.03 GiB in 3.513 seconds | 20260506 18:40:30.025 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.97% / 0.01% / 65.03% | 20260506 18:40:30.025 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.06885% / 1.19209e-05% / 5.96046e-05% | 20260506 18:40:30.025 | INFO | Baz IO overall: | 20260506 18:40:30.025 | INFO | BytesWritten : 1891496201814 | 20260506 18:40:30.025 | INFO | NumEvents : 914668803083 | 20260506 18:40:31.534 | INFO | Analyzing chunk frames = [3579904,3580416) | 20260506 18:40:31.534 | INFO | MemInfo: { | 20260506 18:40:31.534 | INFO | "gpu" : " 17 GB", | 20260506 18:40:31.534 | INFO | "huge" : "", | 20260506 18:40:31.534 | INFO | "huge_shared" : " 27 GB", | 20260506 18:40:31.534 | INFO | "malloc" : " 14 GB", | 20260506 18:40:31.534 | INFO | "pinned" : " 29 GB" | 20260506 18:40:31.534 | INFO | } | 20260506 18:40:31.534 | INFO | Perf report for the last 5.1418 seconds: | 20260506 18:40:31.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:40:31.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:31.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:31.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:31.534 | INFO | ANALYSIS : 63.0%, 2.9, 94.4ms, 0 | 20260506 18:40:31.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:31.534 | INFO | RT_METRICS : 48.2%, 1.0, 25.3ms, 0 | 20260506 18:40:31.534 | INFO | PRE_HQ : 61.2%, 1.9, 61.9ms, 0 | 20260506 18:40:31.534 | INFO | BAZWRITER : 67.8%, 4.8, 168.0ms, 0 | 20260506 18:40:31.534 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 18:40:31.535 | INFO | Uploaded 36GiB to GPU in 2.05762s (17.4959GiB/s) | 20260506 18:40:31.535 | INFO | Downloaded 38GiB to GPU in 2.21581s (17.1495GiB/s) | 20260506 18:40:36.672 | INFO | Analyzing chunk frames = [3580416,3580928) | 20260506 18:40:36.672 | INFO | MemInfo: { | 20260506 18:40:36.672 | INFO | "gpu" : " 17 GB", | 20260506 18:40:36.672 | INFO | "huge" : "", | 20260506 18:40:36.672 | INFO | "huge_shared" : " 27 GB", | 20260506 18:40:36.672 | INFO | "malloc" : " 14 GB", | 20260506 18:40:36.672 | INFO | "pinned" : " 29 GB" | 20260506 18:40:36.672 | INFO | } | 20260506 18:40:36.672 | INFO | Perf report for the last 5.13789 seconds: | 20260506 18:40:36.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:40:36.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:36.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:36.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:36.672 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:40:36.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:36.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:36.672 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 18:40:36.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:36.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:36.672 | INFO | Uploaded 36GiB to GPU in 2.01881s (17.8323GiB/s) | 20260506 18:40:36.672 | INFO | Downloaded 34GiB to GPU in 1.92488s (17.6635GiB/s) | 20260506 18:40:41.814 | INFO | Analyzing chunk frames = [3580928,3581440) | 20260506 18:40:41.814 | INFO | MemInfo: { | 20260506 18:40:41.814 | INFO | "gpu" : " 17 GB", | 20260506 18:40:41.814 | INFO | "huge" : "", | 20260506 18:40:41.814 | INFO | "huge_shared" : " 27 GB", | 20260506 18:40:41.814 | INFO | "malloc" : " 14 GB", | 20260506 18:40:41.814 | INFO | "pinned" : " 29 GB" | 20260506 18:40:41.814 | INFO | } | 20260506 18:40:41.814 | INFO | Perf report for the last 5.14218 seconds: | 20260506 18:40:41.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:40:41.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:41.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:41.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:41.814 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 18:40:41.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:41.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:41.815 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:40:41.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:41.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:41.815 | INFO | Uploaded 36GiB to GPU in 1.97845s (18.1961GiB/s) | 20260506 18:40:41.815 | INFO | Downloaded 34GiB to GPU in 1.91744s (17.7319GiB/s) | 20260506 18:40:46.948 | INFO | Analyzing chunk frames = [3581440,3581952) | 20260506 18:40:46.948 | INFO | MemInfo: { | 20260506 18:40:46.948 | INFO | "gpu" : " 17 GB", | 20260506 18:40:46.948 | INFO | "huge" : "", | 20260506 18:40:46.948 | INFO | "huge_shared" : " 27 GB", | 20260506 18:40:46.948 | INFO | "malloc" : " 14 GB", | 20260506 18:40:46.948 | INFO | "pinned" : " 29 GB" | 20260506 18:40:46.948 | INFO | } | 20260506 18:40:46.948 | INFO | Perf report for the last 5.13355 seconds: | 20260506 18:40:46.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:40:46.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:46.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:46.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:46.948 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:40:46.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:46.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:46.948 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:40:46.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:46.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:46.948 | INFO | Uploaded 36GiB to GPU in 1.96726s (18.2996GiB/s) | 20260506 18:40:46.948 | INFO | Downloaded 34GiB to GPU in 1.87551s (18.1284GiB/s) | 20260506 18:40:52.087 | INFO | Analyzing chunk frames = [3581952,3582464) | 20260506 18:40:52.087 | INFO | MemInfo: { | 20260506 18:40:52.087 | INFO | "gpu" : " 17 GB", | 20260506 18:40:52.087 | INFO | "huge" : "", | 20260506 18:40:52.087 | INFO | "huge_shared" : " 27 GB", | 20260506 18:40:52.087 | INFO | "malloc" : " 14 GB", | 20260506 18:40:52.087 | INFO | "pinned" : " 29 GB" | 20260506 18:40:52.087 | INFO | } | 20260506 18:40:52.087 | INFO | Perf report for the last 5.1388 seconds: | 20260506 18:40:52.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:40:52.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:52.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:52.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:52.087 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 18:40:52.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:52.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:52.087 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 18:40:52.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:52.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:52.087 | INFO | Uploaded 36GiB to GPU in 2.01668s (17.8511GiB/s) | 20260506 18:40:52.087 | INFO | Downloaded 34GiB to GPU in 1.90499s (17.8479GiB/s) | 20260506 18:40:57.227 | INFO | Analyzing chunk frames = [3582464,3582976) | 20260506 18:40:57.227 | INFO | MemInfo: { | 20260506 18:40:57.227 | INFO | "gpu" : " 17 GB", | 20260506 18:40:57.227 | INFO | "huge" : "", | 20260506 18:40:57.227 | INFO | "huge_shared" : " 27 GB", | 20260506 18:40:57.227 | INFO | "malloc" : " 14 GB", | 20260506 18:40:57.227 | INFO | "pinned" : " 29 GB" | 20260506 18:40:57.227 | INFO | } | 20260506 18:40:57.227 | INFO | Perf report for the last 5.14006 seconds: | 20260506 18:40:57.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:40:57.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:57.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:57.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:57.227 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 18:40:57.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:57.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:57.227 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:40:57.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:57.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:40:57.227 | INFO | Uploaded 36GiB to GPU in 2.02915s (17.7414GiB/s) | 20260506 18:40:57.227 | INFO | Downloaded 34GiB to GPU in 1.94328s (17.4962GiB/s) | 20260506 18:41:02.360 | INFO | Analyzing chunk frames = [3582976,3583488) | 20260506 18:41:02.360 | INFO | MemInfo: { | 20260506 18:41:02.360 | INFO | "gpu" : " 17 GB", | 20260506 18:41:02.360 | INFO | "huge" : "", | 20260506 18:41:02.360 | INFO | "huge_shared" : " 27 GB", | 20260506 18:41:02.361 | INFO | "malloc" : " 14 GB", | 20260506 18:41:02.361 | INFO | "pinned" : " 29 GB" | 20260506 18:41:02.361 | INFO | } | 20260506 18:41:02.361 | INFO | Perf report for the last 5.13379 seconds: | 20260506 18:41:02.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:41:02.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:02.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:02.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:02.361 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260506 18:41:02.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:02.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:02.361 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 18:41:02.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:02.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:02.361 | INFO | Uploaded 36GiB to GPU in 2.02531s (17.7751GiB/s) | 20260506 18:41:02.361 | INFO | Downloaded 34GiB to GPU in 1.98262s (17.149GiB/s) | 20260506 18:41:07.503 | INFO | Analyzing chunk frames = [3583488,3584000) | 20260506 18:41:07.503 | INFO | MemInfo: { | 20260506 18:41:07.503 | INFO | "gpu" : " 17 GB", | 20260506 18:41:07.503 | INFO | "huge" : "", | 20260506 18:41:07.503 | INFO | "huge_shared" : " 27 GB", | 20260506 18:41:07.503 | INFO | "malloc" : " 14 GB", | 20260506 18:41:07.503 | INFO | "pinned" : " 29 GB" | 20260506 18:41:07.503 | INFO | } | 20260506 18:41:07.503 | INFO | Perf report for the last 5.14207 seconds: | 20260506 18:41:07.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:41:07.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:07.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:07.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:07.503 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 18:41:07.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:07.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:07.503 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 18:41:07.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:07.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:07.503 | INFO | Uploaded 36GiB to GPU in 2.05241s (17.5404GiB/s) | 20260506 18:41:07.503 | INFO | Downloaded 34GiB to GPU in 1.98897s (17.0943GiB/s) | 20260506 18:41:07.619 | INFO | Starting a round of Baz IO | 20260506 18:41:10.760 | INFO | Populating RTMetrics report | 20260506 18:41:10.760 | INFO | RTMetrics report populated | 20260506 18:41:10.760 | INFO | Opening tmp RTMetrics json file | 20260506 18:41:10.778 | INFO | Serializing RTMetrics to json | 20260506 18:41:10.832 | INFO | Serialization done, swapping tmp file now... | 20260506 18:41:10.833 | INFO | RTMetrics done | 20260506 18:41:11.408 | INFO | Finished a round of Baz IO: Wrote 2.031 GiB in 3.789 seconds | 20260506 18:41:11.408 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.96% / 0.01% / 65.03% | 20260506 18:41:11.408 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.06744% / 1.19209e-05% / 6.75519e-05% | 20260506 18:41:11.408 | INFO | Baz IO overall: | 20260506 18:41:11.408 | INFO | BytesWritten : 1893676446028 | 20260506 18:41:11.408 | INFO | NumEvents : 915718492932 | 20260506 18:41:12.643 | INFO | Analyzing chunk frames = [3584000,3584512) | 20260506 18:41:12.643 | INFO | MemInfo: { | 20260506 18:41:12.643 | INFO | "gpu" : " 17 GB", | 20260506 18:41:12.643 | INFO | "huge" : "", | 20260506 18:41:12.643 | INFO | "huge_shared" : " 27 GB", | 20260506 18:41:12.643 | INFO | "malloc" : " 14 GB", | 20260506 18:41:12.643 | INFO | "pinned" : " 29 GB" | 20260506 18:41:12.643 | INFO | } | 20260506 18:41:12.643 | INFO | Perf report for the last 5.1403 seconds: | 20260506 18:41:12.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:41:12.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:12.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:12.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:12.643 | INFO | ANALYSIS : 62.6%, 2.9, 95.0ms, 0 | 20260506 18:41:12.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:12.643 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260506 18:41:12.643 | INFO | PRE_HQ : 60.3%, 1.9, 60.2ms, 0 | 20260506 18:41:12.643 | INFO | BAZWRITER : 73.3%, 4.8, 182.3ms, 0 | 20260506 18:41:12.643 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 18:41:12.643 | INFO | Uploaded 36GiB to GPU in 2.0423s (17.6272GiB/s) | 20260506 18:41:12.643 | INFO | Downloaded 38GiB to GPU in 2.20406s (17.2409GiB/s) | 20260506 18:41:17.784 | INFO | Analyzing chunk frames = [3584512,3585024) | 20260506 18:41:17.784 | INFO | MemInfo: { | 20260506 18:41:17.784 | INFO | "gpu" : " 17 GB", | 20260506 18:41:17.784 | INFO | "huge" : "", | 20260506 18:41:17.784 | INFO | "huge_shared" : " 27 GB", | 20260506 18:41:17.784 | INFO | "malloc" : " 14 GB", | 20260506 18:41:17.784 | INFO | "pinned" : " 29 GB" | 20260506 18:41:17.784 | INFO | } | 20260506 18:41:17.784 | INFO | Perf report for the last 5.14085 seconds: | 20260506 18:41:17.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:41:17.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:17.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:17.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:17.784 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 18:41:17.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:17.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:17.784 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:41:17.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:17.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:17.784 | INFO | Uploaded 36GiB to GPU in 1.9996s (18.0036GiB/s) | 20260506 18:41:17.784 | INFO | Downloaded 34GiB to GPU in 1.92994s (17.6171GiB/s) | 20260506 18:41:22.923 | INFO | Analyzing chunk frames = [3585024,3585536) | 20260506 18:41:22.923 | INFO | MemInfo: { | 20260506 18:41:22.923 | INFO | "gpu" : " 17 GB", | 20260506 18:41:22.923 | INFO | "huge" : "", | 20260506 18:41:22.923 | INFO | "huge_shared" : " 27 GB", | 20260506 18:41:22.923 | INFO | "malloc" : " 14 GB", | 20260506 18:41:22.923 | INFO | "pinned" : " 29 GB" | 20260506 18:41:22.923 | INFO | } | 20260506 18:41:22.923 | INFO | Perf report for the last 5.13872 seconds: | 20260506 18:41:22.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:41:22.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:22.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:22.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:22.923 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 18:41:22.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:22.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:22.923 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:41:22.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:22.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:22.923 | INFO | Uploaded 36GiB to GPU in 1.97928s (18.1884GiB/s) | 20260506 18:41:22.923 | INFO | Downloaded 34GiB to GPU in 1.91443s (17.7599GiB/s) | 20260506 18:41:28.062 | INFO | Analyzing chunk frames = [3585536,3586048) | 20260506 18:41:28.063 | INFO | MemInfo: { | 20260506 18:41:28.063 | INFO | "gpu" : " 17 GB", | 20260506 18:41:28.063 | INFO | "huge" : "", | 20260506 18:41:28.063 | INFO | "huge_shared" : " 27 GB", | 20260506 18:41:28.063 | INFO | "malloc" : " 14 GB", | 20260506 18:41:28.063 | INFO | "pinned" : " 29 GB" | 20260506 18:41:28.063 | INFO | } | 20260506 18:41:28.063 | INFO | Perf report for the last 5.14003 seconds: | 20260506 18:41:28.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:41:28.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:28.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:28.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:28.063 | INFO | ANALYSIS : 60.6%, 3.0, 94.0ms, 0 | 20260506 18:41:28.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:28.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:28.063 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:41:28.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:28.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:28.063 | INFO | Uploaded 36GiB to GPU in 1.99817s (18.0165GiB/s) | 20260506 18:41:28.063 | INFO | Downloaded 34GiB to GPU in 1.91304s (17.7728GiB/s) | 20260506 18:41:33.201 | INFO | Analyzing chunk frames = [3586048,3586560) | 20260506 18:41:33.201 | INFO | MemInfo: { | 20260506 18:41:33.201 | INFO | "gpu" : " 17 GB", | 20260506 18:41:33.201 | INFO | "huge" : "", | 20260506 18:41:33.201 | INFO | "huge_shared" : " 27 GB", | 20260506 18:41:33.201 | INFO | "malloc" : " 14 GB", | 20260506 18:41:33.201 | INFO | "pinned" : " 29 GB" | 20260506 18:41:33.201 | INFO | } | 20260506 18:41:33.201 | INFO | Perf report for the last 5.13853 seconds: | 20260506 18:41:33.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:41:33.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:33.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:33.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:33.201 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 18:41:33.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:33.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:33.201 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:41:33.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:33.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:33.202 | INFO | Uploaded 36GiB to GPU in 2.01379s (17.8767GiB/s) | 20260506 18:41:33.202 | INFO | Downloaded 34GiB to GPU in 1.91518s (17.7529GiB/s) | 20260506 18:41:38.341 | INFO | Analyzing chunk frames = [3586560,3587072) | 20260506 18:41:38.341 | INFO | MemInfo: { | 20260506 18:41:38.341 | INFO | "gpu" : " 17 GB", | 20260506 18:41:38.341 | INFO | "huge" : "", | 20260506 18:41:38.341 | INFO | "huge_shared" : " 27 GB", | 20260506 18:41:38.341 | INFO | "malloc" : " 14 GB", | 20260506 18:41:38.341 | INFO | "pinned" : " 29 GB" | 20260506 18:41:38.341 | INFO | } | 20260506 18:41:38.341 | INFO | Perf report for the last 5.13963 seconds: | 20260506 18:41:38.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:41:38.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:38.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:38.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:38.341 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 18:41:38.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:38.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:38.341 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:41:38.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:38.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:38.341 | INFO | Uploaded 36GiB to GPU in 2.0161s (17.8563GiB/s) | 20260506 18:41:38.341 | INFO | Downloaded 34GiB to GPU in 1.94039s (17.5222GiB/s) | 20260506 18:41:43.482 | INFO | Analyzing chunk frames = [3587072,3587584) | 20260506 18:41:43.482 | INFO | MemInfo: { | 20260506 18:41:43.482 | INFO | "gpu" : " 17 GB", | 20260506 18:41:43.482 | INFO | "huge" : "", | 20260506 18:41:43.482 | INFO | "huge_shared" : " 27 GB", | 20260506 18:41:43.482 | INFO | "malloc" : " 14 GB", | 20260506 18:41:43.482 | INFO | "pinned" : " 29 GB" | 20260506 18:41:43.482 | INFO | } | 20260506 18:41:43.482 | INFO | Perf report for the last 5.14131 seconds: | 20260506 18:41:43.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:41:43.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:43.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:43.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:43.482 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:41:43.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:43.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:43.482 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:41:43.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:43.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:43.483 | INFO | Uploaded 36GiB to GPU in 2.0135s (17.8793GiB/s) | 20260506 18:41:43.483 | INFO | Downloaded 34GiB to GPU in 2.00716s (16.9394GiB/s) | 20260506 18:41:48.619 | INFO | Analyzing chunk frames = [3587584,3588096) | 20260506 18:41:48.619 | INFO | MemInfo: { | 20260506 18:41:48.619 | INFO | "gpu" : " 17 GB", | 20260506 18:41:48.619 | INFO | "huge" : "", | 20260506 18:41:48.619 | INFO | "huge_shared" : " 27 GB", | 20260506 18:41:48.619 | INFO | "malloc" : " 14 GB", | 20260506 18:41:48.619 | INFO | "pinned" : " 29 GB" | 20260506 18:41:48.619 | INFO | } | 20260506 18:41:48.619 | INFO | Perf report for the last 5.13657 seconds: | 20260506 18:41:48.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:41:48.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:48.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:48.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:48.619 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 18:41:48.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:48.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:48.619 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260506 18:41:48.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:48.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:48.619 | INFO | Uploaded 36GiB to GPU in 1.99149s (18.0769GiB/s) | 20260506 18:41:48.619 | INFO | Downloaded 34GiB to GPU in 1.99105s (17.0764GiB/s) | 20260506 18:41:48.619 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:41:48.619 | INFO | Total mem usage: 18 GiB | 20260506 18:41:48.619 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:41:48.619 | INFO | Recent High Water Marks Sum: 16.7 GiB (Savings: -7.9%) | 20260506 18:41:48.619 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:41:48.619 | INFO | High Water Breakdown: | 20260506 18:41:48.619 | INFO | 62.4% : 10.4 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:41:48.620 | INFO | 37.5% : 6.25 GiB ( 93.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:41:48.620 | INFO | 0.119% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:41:48.620 | INFO | ------------------------------------------ | 20260506 18:41:48.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:41:48.620 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:41:48.620 | INFO | Total mem usage: 26.5 GiB | 20260506 18:41:48.620 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:41:48.620 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:41:48.620 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:41:48.620 | INFO | High Water Breakdown: | 20260506 18:41:48.620 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:41:48.620 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:41:48.620 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:41:48.620 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:41:48.620 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:41:48.620 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:41:48.620 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:41:48.620 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:41:48.620 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:41:48.620 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:41:48.620 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:41:48.620 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:41:48.620 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:41:48.620 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:41:48.620 | INFO | ------------------------------------------ | 20260506 18:41:48.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:41:48.620 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:41:48.620 | INFO | Total mem usage: 25.6 GiB | 20260506 18:41:48.620 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:41:48.620 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:41:48.620 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:41:48.620 | INFO | High Water Breakdown: | 20260506 18:41:48.620 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:41:48.620 | INFO | ------------------------------------------ | 20260506 18:41:48.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:41:48.620 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:41:48.620 | INFO | Total mem usage: 15.7 GiB | 20260506 18:41:48.620 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:41:48.620 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:41:48.620 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 18:41:48.620 | INFO | High Water Breakdown: | 20260506 18:41:48.620 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:41:48.620 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:41:48.620 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:41:48.620 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:41:48.620 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:41:48.620 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:41:48.620 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:41:48.620 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:41:48.620 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:41:48.620 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:41:48.620 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:41:48.620 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:41:48.620 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:41:48.620 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:41:48.620 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:41:48.620 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:41:48.620 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:41:48.620 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:41:48.620 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:41:48.620 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:41:48.620 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:41:48.620 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:41:48.620 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:41:48.620 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:41:48.620 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:41:48.620 | INFO | ------------------------------------------ | 20260506 18:41:48.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:41:48.620 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:41:48.620 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:41:48.620 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:41:48.620 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.62e+01, 1.36e+01 / 2.49e+00 | 20260506 18:41:48.620 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.02e-01, 1.09e-02 / 9.91e-03 | 20260506 18:41:48.620 | INFO | PerfReport: 11.62%, Baseline -- 3.64e+00 / 4.44e+00, 5.60e-01 / 2.96e-01 | 20260506 18:41:48.620 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.84e-01 / 1.29e-01 | 20260506 18:41:48.620 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.49e+01, 4.18e+00 / 2.52e+00 | 20260506 18:41:48.620 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.45e+00 / 1.63e+00 | 20260506 18:41:48.620 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.25e+00 / 2.77e+00, 3.82e-01 / 2.36e-01 | 20260506 18:41:48.620 | INFO | PerfReport: 3.39%, Metrics -- 1.06e+00 / 1.40e+00, 2.77e-01 / 1.32e-01 | 20260506 18:41:48.620 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.66e-01 / 4.17e-01, 5.70e-02 / 1.70e-02 | 20260506 18:41:48.620 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:41:48.620 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:41:48.620 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:41:48.620 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 18:41:48.621 | INFO | PerfReport: 21.59%, Upload -- 2.10e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260506 18:41:48.621 | INFO | PerfReport: 21.17%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 18:41:48.621 | INFO | PerfReport: 32.24%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 18:41:48.621 | INFO | PerfReport: 1.47%, LockUpload -- 1.42e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260506 18:41:48.621 | INFO | PerfReport: 1.12%, LockDownload -- 1.08e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 18:41:48.621 | INFO | PerfReport: 22.41%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 18:41:48.621 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260506 18:41:48.733 | INFO | Starting a round of Baz IO | 20260506 18:41:51.872 | INFO | Populating RTMetrics report | 20260506 18:41:51.872 | INFO | RTMetrics report populated | 20260506 18:41:51.872 | INFO | Opening tmp RTMetrics json file | 20260506 18:41:51.985 | INFO | Serializing RTMetrics to json | 20260506 18:41:51.992 | INFO | Serialization done, swapping tmp file now... | 20260506 18:41:52.001 | INFO | RTMetrics done | 20260506 18:41:52.201 | INFO | Finished a round of Baz IO: Wrote 2.028 GiB in 3.468 seconds | 20260506 18:41:52.201 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.96% / 0.01% / 65.04% | 20260506 18:41:52.201 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.06245% / 3.97364e-06% / 5.5631e-05% | 20260506 18:41:52.201 | INFO | Baz IO overall: | 20260506 18:41:52.201 | INFO | BytesWritten : 1895853525256 | 20260506 18:41:52.201 | INFO | NumEvents : 916765563823 | 20260506 18:41:53.759 | INFO | Analyzing chunk frames = [3588096,3588608) | 20260506 18:41:53.759 | INFO | MemInfo: { | 20260506 18:41:53.759 | INFO | "gpu" : " 17 GB", | 20260506 18:41:53.759 | INFO | "huge" : "", | 20260506 18:41:53.759 | INFO | "huge_shared" : " 27 GB", | 20260506 18:41:53.759 | INFO | "malloc" : " 14 GB", | 20260506 18:41:53.759 | INFO | "pinned" : " 29 GB" | 20260506 18:41:53.759 | INFO | } | 20260506 18:41:53.759 | INFO | Perf report for the last 5.14051 seconds: | 20260506 18:41:53.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:41:53.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:53.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:53.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:53.759 | INFO | ANALYSIS : 62.7%, 2.9, 95.6ms, 0 | 20260506 18:41:53.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:53.759 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 18:41:53.759 | INFO | PRE_HQ : 60.1%, 1.8, 57.7ms, 0 | 20260506 18:41:53.759 | INFO | BAZWRITER : 67.1%, 4.8, 165.1ms, 0 | 20260506 18:41:53.759 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.2ms, 0 | 20260506 18:41:53.760 | INFO | Uploaded 36GiB to GPU in 2.03309s (17.7071GiB/s) | 20260506 18:41:53.760 | INFO | Downloaded 38GiB to GPU in 2.21777s (17.1343GiB/s) | 20260506 18:41:58.903 | INFO | Analyzing chunk frames = [3588608,3589120) | 20260506 18:41:58.903 | INFO | MemInfo: { | 20260506 18:41:58.903 | INFO | "gpu" : " 17 GB", | 20260506 18:41:58.903 | INFO | "huge" : "", | 20260506 18:41:58.903 | INFO | "huge_shared" : " 27 GB", | 20260506 18:41:58.903 | INFO | "malloc" : " 14 GB", | 20260506 18:41:58.903 | INFO | "pinned" : " 29 GB" | 20260506 18:41:58.903 | INFO | } | 20260506 18:41:58.903 | INFO | Perf report for the last 5.14393 seconds: | 20260506 18:41:58.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:41:58.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:58.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:58.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:58.903 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 18:41:58.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:58.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:58.903 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 18:41:58.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:58.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:41:58.903 | INFO | Uploaded 36GiB to GPU in 1.99444s (18.0502GiB/s) | 20260506 18:41:58.904 | INFO | Downloaded 34GiB to GPU in 1.92963s (17.62GiB/s) | 20260506 18:42:04.039 | INFO | Analyzing chunk frames = [3589120,3589632) | 20260506 18:42:04.039 | INFO | MemInfo: { | 20260506 18:42:04.039 | INFO | "gpu" : " 17 GB", | 20260506 18:42:04.039 | INFO | "huge" : "", | 20260506 18:42:04.039 | INFO | "huge_shared" : " 27 GB", | 20260506 18:42:04.039 | INFO | "malloc" : " 14 GB", | 20260506 18:42:04.039 | INFO | "pinned" : " 29 GB" | 20260506 18:42:04.039 | INFO | } | 20260506 18:42:04.039 | INFO | Perf report for the last 5.1357 seconds: | 20260506 18:42:04.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:42:04.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:04.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:04.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:04.039 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 18:42:04.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:04.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:04.039 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:42:04.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:04.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:04.039 | INFO | Uploaded 36GiB to GPU in 1.96564s (18.3146GiB/s) | 20260506 18:42:04.039 | INFO | Downloaded 34GiB to GPU in 1.91913s (17.7164GiB/s) | 20260506 18:42:09.177 | INFO | Analyzing chunk frames = [3589632,3590144) | 20260506 18:42:09.177 | INFO | MemInfo: { | 20260506 18:42:09.177 | INFO | "gpu" : " 17 GB", | 20260506 18:42:09.177 | INFO | "huge" : "", | 20260506 18:42:09.177 | INFO | "huge_shared" : " 27 GB", | 20260506 18:42:09.177 | INFO | "malloc" : " 14 GB", | 20260506 18:42:09.177 | INFO | "pinned" : " 29 GB" | 20260506 18:42:09.177 | INFO | } | 20260506 18:42:09.177 | INFO | Perf report for the last 5.13811 seconds: | 20260506 18:42:09.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:42:09.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:09.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:09.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:09.177 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 18:42:09.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:09.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:09.177 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:42:09.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:09.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:09.177 | INFO | Uploaded 36GiB to GPU in 1.94034s (18.5535GiB/s) | 20260506 18:42:09.178 | INFO | Downloaded 34GiB to GPU in 1.87504s (18.1329GiB/s) | 20260506 18:42:14.317 | INFO | Analyzing chunk frames = [3590144,3590656) | 20260506 18:42:14.317 | INFO | MemInfo: { | 20260506 18:42:14.317 | INFO | "gpu" : " 17 GB", | 20260506 18:42:14.317 | INFO | "huge" : "", | 20260506 18:42:14.317 | INFO | "huge_shared" : " 27 GB", | 20260506 18:42:14.317 | INFO | "malloc" : " 14 GB", | 20260506 18:42:14.317 | INFO | "pinned" : " 29 GB" | 20260506 18:42:14.317 | INFO | } | 20260506 18:42:14.317 | INFO | Perf report for the last 5.14007 seconds: | 20260506 18:42:14.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:42:14.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:14.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:14.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:14.317 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 18:42:14.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:14.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:14.317 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:42:14.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:14.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:14.317 | INFO | Uploaded 36GiB to GPU in 2.01386s (17.8762GiB/s) | 20260506 18:42:14.318 | INFO | Downloaded 34GiB to GPU in 1.91161s (17.7861GiB/s) | 20260506 18:42:19.451 | INFO | Analyzing chunk frames = [3590656,3591168) | 20260506 18:42:19.451 | INFO | MemInfo: { | 20260506 18:42:19.451 | INFO | "gpu" : " 17 GB", | 20260506 18:42:19.451 | INFO | "huge" : "", | 20260506 18:42:19.451 | INFO | "huge_shared" : " 27 GB", | 20260506 18:42:19.451 | INFO | "malloc" : " 14 GB", | 20260506 18:42:19.451 | INFO | "pinned" : " 29 GB" | 20260506 18:42:19.451 | INFO | } | 20260506 18:42:19.451 | INFO | Perf report for the last 5.13353 seconds: | 20260506 18:42:19.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:42:19.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:19.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:19.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:19.451 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 18:42:19.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:19.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:19.451 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:42:19.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:19.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:19.451 | INFO | Uploaded 36GiB to GPU in 2.00231s (17.9792GiB/s) | 20260506 18:42:19.451 | INFO | Downloaded 34GiB to GPU in 1.94481s (17.4825GiB/s) | 20260506 18:42:24.594 | INFO | Analyzing chunk frames = [3591168,3591680) | 20260506 18:42:24.594 | INFO | MemInfo: { | 20260506 18:42:24.594 | INFO | "gpu" : " 17 GB", | 20260506 18:42:24.594 | INFO | "huge" : "", | 20260506 18:42:24.594 | INFO | "huge_shared" : " 27 GB", | 20260506 18:42:24.594 | INFO | "malloc" : " 14 GB", | 20260506 18:42:24.594 | INFO | "pinned" : " 29 GB" | 20260506 18:42:24.594 | INFO | } | 20260506 18:42:24.594 | INFO | Perf report for the last 5.14355 seconds: | 20260506 18:42:24.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:42:24.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:24.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:24.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:24.594 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:42:24.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:24.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:24.595 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:42:24.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:24.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:24.595 | INFO | Uploaded 36GiB to GPU in 1.99043s (18.0865GiB/s) | 20260506 18:42:24.595 | INFO | Downloaded 34GiB to GPU in 2.0084s (16.9289GiB/s) | 20260506 18:42:29.733 | INFO | Analyzing chunk frames = [3591680,3592192) | 20260506 18:42:29.733 | INFO | MemInfo: { | 20260506 18:42:29.733 | INFO | "gpu" : " 17 GB", | 20260506 18:42:29.733 | INFO | "huge" : "", | 20260506 18:42:29.733 | INFO | "huge_shared" : " 27 GB", | 20260506 18:42:29.733 | INFO | "malloc" : " 14 GB", | 20260506 18:42:29.733 | INFO | "pinned" : " 29 GB" | 20260506 18:42:29.733 | INFO | } | 20260506 18:42:29.733 | INFO | Perf report for the last 5.13916 seconds: | 20260506 18:42:29.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:42:29.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:29.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:29.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:29.733 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 18:42:29.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:29.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:29.734 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:42:29.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:29.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:29.734 | INFO | Uploaded 36GiB to GPU in 2.04887s (17.5707GiB/s) | 20260506 18:42:29.734 | INFO | Downloaded 34GiB to GPU in 1.97747s (17.1937GiB/s) | 20260506 18:42:29.853 | INFO | Starting a round of Baz IO | 20260506 18:42:33.024 | INFO | Populating RTMetrics report | 20260506 18:42:33.024 | INFO | RTMetrics report populated | 20260506 18:42:33.024 | INFO | Opening tmp RTMetrics json file | 20260506 18:42:33.026 | INFO | Serializing RTMetrics to json | 20260506 18:42:33.029 | INFO | Serialization done, swapping tmp file now... | 20260506 18:42:33.030 | INFO | RTMetrics done | 20260506 18:42:33.711 | INFO | Finished a round of Baz IO: Wrote 2.031 GiB in 3.857 seconds | 20260506 18:42:33.711 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.95% / 0.01% / 65.04% | 20260506 18:42:33.711 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.06272% / 1.58946e-05% / 5.96046e-05% | 20260506 18:42:33.711 | INFO | Baz IO overall: | 20260506 18:42:33.711 | INFO | BytesWritten : 1898034095314 | 20260506 18:42:33.711 | INFO | NumEvents : 917815388686 | 20260506 18:42:34.875 | INFO | Analyzing chunk frames = [3592192,3592704) | 20260506 18:42:34.875 | INFO | MemInfo: { | 20260506 18:42:34.875 | INFO | "gpu" : " 17 GB", | 20260506 18:42:34.875 | INFO | "huge" : "", | 20260506 18:42:34.875 | INFO | "huge_shared" : " 27 GB", | 20260506 18:42:34.875 | INFO | "malloc" : " 14 GB", | 20260506 18:42:34.875 | INFO | "pinned" : " 29 GB" | 20260506 18:42:34.875 | INFO | } | 20260506 18:42:34.876 | INFO | Perf report for the last 5.14203 seconds: | 20260506 18:42:34.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:42:34.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:34.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:34.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:34.876 | INFO | ANALYSIS : 63.1%, 2.8, 93.9ms, 0 | 20260506 18:42:34.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:34.876 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260506 18:42:34.876 | INFO | PRE_HQ : 60.9%, 1.9, 61.6ms, 0 | 20260506 18:42:34.876 | INFO | BAZWRITER : 74.5%, 4.8, 185.4ms, 0 | 20260506 18:42:34.876 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 18:42:34.876 | INFO | Uploaded 36GiB to GPU in 2.0512s (17.5507GiB/s) | 20260506 18:42:34.876 | INFO | Downloaded 38GiB to GPU in 2.18297s (17.4075GiB/s) | 20260506 18:42:40.018 | INFO | Analyzing chunk frames = [3592704,3593216) | 20260506 18:42:40.018 | INFO | MemInfo: { | 20260506 18:42:40.018 | INFO | "gpu" : " 17 GB", | 20260506 18:42:40.019 | INFO | "huge" : "", | 20260506 18:42:40.019 | INFO | "huge_shared" : " 27 GB", | 20260506 18:42:40.019 | INFO | "malloc" : " 14 GB", | 20260506 18:42:40.019 | INFO | "pinned" : " 29 GB" | 20260506 18:42:40.019 | INFO | } | 20260506 18:42:40.019 | INFO | Perf report for the last 5.1432 seconds: | 20260506 18:42:40.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:42:40.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:40.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:40.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:40.019 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 18:42:40.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:40.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:40.019 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:42:40.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:40.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:40.019 | INFO | Uploaded 36GiB to GPU in 1.99571s (18.0387GiB/s) | 20260506 18:42:40.019 | INFO | Downloaded 34GiB to GPU in 1.92942s (17.6218GiB/s) | 20260506 18:42:45.163 | INFO | Analyzing chunk frames = [3593216,3593728) | 20260506 18:42:45.163 | INFO | MemInfo: { | 20260506 18:42:45.163 | INFO | "gpu" : " 17 GB", | 20260506 18:42:45.163 | INFO | "huge" : "", | 20260506 18:42:45.163 | INFO | "huge_shared" : " 27 GB", | 20260506 18:42:45.163 | INFO | "malloc" : " 14 GB", | 20260506 18:42:45.163 | INFO | "pinned" : " 29 GB" | 20260506 18:42:45.163 | INFO | } | 20260506 18:42:45.163 | INFO | Perf report for the last 5.14467 seconds: | 20260506 18:42:45.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:42:45.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:45.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:45.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:45.164 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260506 18:42:45.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:45.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:45.164 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:42:45.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:45.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:45.164 | INFO | Uploaded 36GiB to GPU in 1.9781s (18.1993GiB/s) | 20260506 18:42:45.164 | INFO | Downloaded 34GiB to GPU in 1.9134s (17.7694GiB/s) | 20260506 18:42:50.294 | INFO | Analyzing chunk frames = [3593728,3594240) | 20260506 18:42:50.294 | INFO | MemInfo: { | 20260506 18:42:50.294 | INFO | "gpu" : " 17 GB", | 20260506 18:42:50.294 | INFO | "huge" : "", | 20260506 18:42:50.294 | INFO | "huge_shared" : " 27 GB", | 20260506 18:42:50.294 | INFO | "malloc" : " 14 GB", | 20260506 18:42:50.294 | INFO | "pinned" : " 29 GB" | 20260506 18:42:50.294 | INFO | } | 20260506 18:42:50.294 | INFO | Perf report for the last 5.13071 seconds: | 20260506 18:42:50.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:42:50.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:50.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:50.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:50.294 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 18:42:50.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:50.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:50.294 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:42:50.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:50.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:50.294 | INFO | Uploaded 36GiB to GPU in 1.97212s (18.2545GiB/s) | 20260506 18:42:50.294 | INFO | Downloaded 34GiB to GPU in 1.87081s (18.1739GiB/s) | 20260506 18:42:55.429 | INFO | Analyzing chunk frames = [3594240,3594752) | 20260506 18:42:55.429 | INFO | MemInfo: { | 20260506 18:42:55.429 | INFO | "gpu" : " 17 GB", | 20260506 18:42:55.429 | INFO | "huge" : "", | 20260506 18:42:55.429 | INFO | "huge_shared" : " 27 GB", | 20260506 18:42:55.429 | INFO | "malloc" : " 14 GB", | 20260506 18:42:55.429 | INFO | "pinned" : " 29 GB" | 20260506 18:42:55.429 | INFO | } | 20260506 18:42:55.429 | INFO | Perf report for the last 5.13456 seconds: | 20260506 18:42:55.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:42:55.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:55.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:55.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:55.429 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260506 18:42:55.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:55.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:55.429 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:42:55.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:55.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:42:55.429 | INFO | Uploaded 36GiB to GPU in 2.01655s (17.8523GiB/s) | 20260506 18:42:55.429 | INFO | Downloaded 34GiB to GPU in 1.90202s (17.8757GiB/s) | 20260506 18:43:00.567 | INFO | Analyzing chunk frames = [3594752,3595264) | 20260506 18:43:00.567 | INFO | MemInfo: { | 20260506 18:43:00.567 | INFO | "gpu" : " 17 GB", | 20260506 18:43:00.568 | INFO | "huge" : "", | 20260506 18:43:00.568 | INFO | "huge_shared" : " 27 GB", | 20260506 18:43:00.568 | INFO | "malloc" : " 14 GB", | 20260506 18:43:00.568 | INFO | "pinned" : " 29 GB" | 20260506 18:43:00.568 | INFO | } | 20260506 18:43:00.568 | INFO | Perf report for the last 5.13894 seconds: | 20260506 18:43:00.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:43:00.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:00.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:00.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:00.568 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 18:43:00.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:00.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:00.568 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:43:00.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:00.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:00.568 | INFO | Uploaded 36GiB to GPU in 2.03873s (17.6581GiB/s) | 20260506 18:43:00.568 | INFO | Downloaded 34GiB to GPU in 1.94631s (17.4689GiB/s) | 20260506 18:43:05.710 | INFO | Analyzing chunk frames = [3595264,3595776) | 20260506 18:43:05.710 | INFO | MemInfo: { | 20260506 18:43:05.710 | INFO | "gpu" : " 17 GB", | 20260506 18:43:05.710 | INFO | "huge" : "", | 20260506 18:43:05.710 | INFO | "huge_shared" : " 27 GB", | 20260506 18:43:05.710 | INFO | "malloc" : " 14 GB", | 20260506 18:43:05.710 | INFO | "pinned" : " 29 GB" | 20260506 18:43:05.710 | INFO | } | 20260506 18:43:05.710 | INFO | Perf report for the last 5.14212 seconds: | 20260506 18:43:05.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:43:05.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:05.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:05.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:05.710 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:43:05.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:05.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:05.710 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:43:05.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:05.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:05.710 | INFO | Uploaded 36GiB to GPU in 1.99751s (18.0225GiB/s) | 20260506 18:43:05.710 | INFO | Downloaded 34GiB to GPU in 2.01501s (16.8734GiB/s) | 20260506 18:43:10.853 | INFO | Analyzing chunk frames = [3595776,3596288) | 20260506 18:43:10.853 | INFO | MemInfo: { | 20260506 18:43:10.853 | INFO | "gpu" : " 17 GB", | 20260506 18:43:10.853 | INFO | "huge" : "", | 20260506 18:43:10.853 | INFO | "huge_shared" : " 27 GB", | 20260506 18:43:10.853 | INFO | "malloc" : " 14 GB", | 20260506 18:43:10.853 | INFO | "pinned" : " 29 GB" | 20260506 18:43:10.853 | INFO | } | 20260506 18:43:10.853 | INFO | Perf report for the last 5.14342 seconds: | 20260506 18:43:10.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:43:10.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:10.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:10.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:10.853 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 18:43:10.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:10.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:10.853 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:43:10.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:10.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:10.854 | INFO | Uploaded 36GiB to GPU in 1.98336s (18.151GiB/s) | 20260506 18:43:10.854 | INFO | Downloaded 34GiB to GPU in 1.99128s (17.0745GiB/s) | 20260506 18:43:10.854 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:43:10.854 | INFO | Total mem usage: 18 GiB | 20260506 18:43:10.854 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:43:10.854 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.4%) | 20260506 18:43:10.854 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:43:10.854 | INFO | High Water Breakdown: | 20260506 18:43:10.854 | INFO | 61.3% : 9.48 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:43:10.854 | INFO | 38.6% : 5.97 GiB ( 89.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:43:10.854 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:43:10.854 | INFO | ------------------------------------------ | 20260506 18:43:10.854 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:43:10.854 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:43:10.854 | INFO | Total mem usage: 26.5 GiB | 20260506 18:43:10.854 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:43:10.854 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 18:43:10.854 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260506 18:43:10.854 | INFO | High Water Breakdown: | 20260506 18:43:10.854 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:43:10.854 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:43:10.854 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:43:10.854 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:43:10.854 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:43:10.854 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:43:10.854 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:43:10.854 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:43:10.854 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:43:10.854 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:43:10.854 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:43:10.854 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:43:10.854 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:43:10.854 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:43:10.854 | INFO | ------------------------------------------ | 20260506 18:43:10.854 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:43:10.854 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:43:10.854 | INFO | Total mem usage: 25.6 GiB | 20260506 18:43:10.854 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:43:10.854 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:43:10.854 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:43:10.854 | INFO | High Water Breakdown: | 20260506 18:43:10.854 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:43:10.854 | INFO | ------------------------------------------ | 20260506 18:43:10.854 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:43:10.854 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:43:10.854 | INFO | Total mem usage: 15.7 GiB | 20260506 18:43:10.854 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:43:10.854 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:43:10.854 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:43:10.854 | INFO | High Water Breakdown: | 20260506 18:43:10.854 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:43:10.854 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:43:10.854 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:43:10.854 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:43:10.854 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:43:10.854 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:43:10.854 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:43:10.854 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:43:10.854 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:43:10.854 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:43:10.854 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:43:10.855 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:43:10.855 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:43:10.855 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:43:10.855 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:43:10.855 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:43:10.855 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:43:10.855 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:43:10.855 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:43:10.855 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:43:10.855 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:43:10.855 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:43:10.855 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:43:10.855 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:43:10.855 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:43:10.855 | INFO | ------------------------------------------ | 20260506 18:43:10.855 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:43:10.855 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:43:10.855 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:43:10.855 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:43:10.855 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.60e+01, 1.36e+01 / 2.60e+00 | 20260506 18:43:10.855 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.04e-01, 9.97e-03 / 1.23e-02 | 20260506 18:43:10.855 | INFO | PerfReport: 11.64%, Baseline -- 3.64e+00 / 4.42e+00, 5.62e-01 / 2.93e-01 | 20260506 18:43:10.855 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.84e-01 / 1.20e-01 | 20260506 18:43:10.855 | INFO | PerfReport: 16.03%, DME -- 4.01e+01 / 4.48e+01, 4.33e+00 / 2.97e+00 | 20260506 18:43:10.855 | INFO | PerfReport: 57.00%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.37e+00 / 1.96e+00 | 20260506 18:43:10.855 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.25e+00 / 2.80e+00, 3.82e-01 / 2.39e-01 | 20260506 18:43:10.855 | INFO | PerfReport: 3.40%, Metrics -- 1.06e+00 / 1.39e+00, 2.82e-01 / 1.50e-01 | 20260506 18:43:10.855 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.66e-01 / 4.13e-01, 5.75e-02 / 2.47e-02 | 20260506 18:43:10.855 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:43:10.855 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:43:10.855 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:43:10.855 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 18:43:10.855 | INFO | PerfReport: 21.49%, Upload -- 2.08e+01 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260506 18:43:10.855 | INFO | PerfReport: 21.17%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 18:43:10.855 | INFO | PerfReport: 32.24%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 18:43:10.855 | INFO | PerfReport: 1.43%, LockUpload -- 1.39e+00 / 0.00e+00, 5.52e+00 / 0.00e+00 | 20260506 18:43:10.855 | INFO | PerfReport: 1.18%, LockDownload -- 1.15e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 18:43:10.855 | INFO | PerfReport: 22.47%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:43:10.855 | INFO | PerfReport: 0.02%, Misc -- 1.59e-02 / 0.00e+00, 3.28e-02 / 0.00e+00 | 20260506 18:43:10.964 | INFO | Starting a round of Baz IO | 20260506 18:43:14.156 | INFO | Populating RTMetrics report | 20260506 18:43:14.156 | INFO | RTMetrics report populated | 20260506 18:43:14.156 | INFO | Opening tmp RTMetrics json file | 20260506 18:43:14.225 | INFO | Serializing RTMetrics to json | 20260506 18:43:14.251 | INFO | Serialization done, swapping tmp file now... | 20260506 18:43:14.256 | INFO | RTMetrics done | 20260506 18:43:14.546 | INFO | Finished a round of Baz IO: Wrote 2.031 GiB in 3.582 seconds | 20260506 18:43:14.546 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.94% / 0.01% / 65.05% | 20260506 18:43:14.546 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.06052% / 7.94729e-06% / 7.15256e-05% | 20260506 18:43:14.546 | INFO | Baz IO overall: | 20260506 18:43:14.546 | INFO | BytesWritten : 1900214502891 | 20260506 18:43:14.546 | INFO | NumEvents : 918865290479 | 20260506 18:43:15.991 | INFO | Analyzing chunk frames = [3596288,3596800) | 20260506 18:43:15.991 | INFO | MemInfo: { | 20260506 18:43:15.991 | INFO | "gpu" : " 17 GB", | 20260506 18:43:15.991 | INFO | "huge" : "", | 20260506 18:43:15.991 | INFO | "huge_shared" : " 27 GB", | 20260506 18:43:15.991 | INFO | "malloc" : " 14 GB", | 20260506 18:43:15.991 | INFO | "pinned" : " 29 GB" | 20260506 18:43:15.991 | INFO | } | 20260506 18:43:15.992 | INFO | Perf report for the last 5.1383 seconds: | 20260506 18:43:15.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:43:15.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:15.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:15.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:15.992 | INFO | ANALYSIS : 63.2%, 2.8, 94.2ms, 0 | 20260506 18:43:15.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:15.992 | INFO | RT_METRICS : 48.8%, 1.0, 25.6ms, 0 | 20260506 18:43:15.992 | INFO | PRE_HQ : 61.2%, 2.0, 62.9ms, 0 | 20260506 18:43:15.992 | INFO | BAZWRITER : 69.4%, 4.8, 170.3ms, 0 | 20260506 18:43:15.992 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 18:43:15.992 | INFO | Uploaded 36GiB to GPU in 2.07823s (17.3224GiB/s) | 20260506 18:43:15.992 | INFO | Downloaded 38GiB to GPU in 2.19695s (17.2967GiB/s) | 20260506 18:43:21.128 | INFO | Analyzing chunk frames = [3596800,3597312) | 20260506 18:43:21.128 | INFO | MemInfo: { | 20260506 18:43:21.128 | INFO | "gpu" : " 17 GB", | 20260506 18:43:21.129 | INFO | "huge" : "", | 20260506 18:43:21.129 | INFO | "huge_shared" : " 27 GB", | 20260506 18:43:21.129 | INFO | "malloc" : " 14 GB", | 20260506 18:43:21.129 | INFO | "pinned" : " 29 GB" | 20260506 18:43:21.129 | INFO | } | 20260506 18:43:21.129 | INFO | Perf report for the last 5.13712 seconds: | 20260506 18:43:21.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:43:21.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:21.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:21.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:21.129 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:43:21.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:21.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:21.129 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 18:43:21.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:21.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:21.129 | INFO | Uploaded 36GiB to GPU in 1.97825s (18.1979GiB/s) | 20260506 18:43:21.129 | INFO | Downloaded 34GiB to GPU in 1.95105s (17.4265GiB/s) | 20260506 18:43:26.266 | INFO | Analyzing chunk frames = [3597312,3597824) | 20260506 18:43:26.266 | INFO | MemInfo: { | 20260506 18:43:26.266 | INFO | "gpu" : " 17 GB", | 20260506 18:43:26.266 | INFO | "huge" : "", | 20260506 18:43:26.266 | INFO | "huge_shared" : " 27 GB", | 20260506 18:43:26.266 | INFO | "malloc" : " 14 GB", | 20260506 18:43:26.266 | INFO | "pinned" : " 29 GB" | 20260506 18:43:26.266 | INFO | } | 20260506 18:43:26.266 | INFO | Perf report for the last 5.13754 seconds: | 20260506 18:43:26.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:43:26.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:26.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:26.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:26.266 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 18:43:26.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:26.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:26.266 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:43:26.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:26.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:26.266 | INFO | Uploaded 36GiB to GPU in 1.95218s (18.4409GiB/s) | 20260506 18:43:26.267 | INFO | Downloaded 34GiB to GPU in 1.92039s (17.7048GiB/s) | 20260506 18:43:31.405 | INFO | Analyzing chunk frames = [3597824,3598336) | 20260506 18:43:31.405 | INFO | MemInfo: { | 20260506 18:43:31.405 | INFO | "gpu" : " 17 GB", | 20260506 18:43:31.405 | INFO | "huge" : "", | 20260506 18:43:31.405 | INFO | "huge_shared" : " 27 GB", | 20260506 18:43:31.405 | INFO | "malloc" : " 14 GB", | 20260506 18:43:31.405 | INFO | "pinned" : " 29 GB" | 20260506 18:43:31.405 | INFO | } | 20260506 18:43:31.405 | INFO | Perf report for the last 5.13876 seconds: | 20260506 18:43:31.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:43:31.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:31.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:31.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:31.405 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:43:31.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:31.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:31.405 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:43:31.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:31.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:31.405 | INFO | Uploaded 36GiB to GPU in 1.93153s (18.6381GiB/s) | 20260506 18:43:31.405 | INFO | Downloaded 34GiB to GPU in 1.87675s (18.1164GiB/s) | 20260506 18:43:36.544 | INFO | Analyzing chunk frames = [3598336,3598848) | 20260506 18:43:36.544 | INFO | MemInfo: { | 20260506 18:43:36.544 | INFO | "gpu" : " 17 GB", | 20260506 18:43:36.544 | INFO | "huge" : "", | 20260506 18:43:36.544 | INFO | "huge_shared" : " 27 GB", | 20260506 18:43:36.544 | INFO | "malloc" : " 14 GB", | 20260506 18:43:36.544 | INFO | "pinned" : " 29 GB" | 20260506 18:43:36.544 | INFO | } | 20260506 18:43:36.545 | INFO | Perf report for the last 5.13951 seconds: | 20260506 18:43:36.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:43:36.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:36.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:36.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:36.545 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 18:43:36.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:36.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:36.545 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:43:36.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:36.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:36.545 | INFO | Uploaded 36GiB to GPU in 2.01258s (17.8875GiB/s) | 20260506 18:43:36.545 | INFO | Downloaded 34GiB to GPU in 1.92358s (17.6753GiB/s) | 20260506 18:43:41.683 | INFO | Analyzing chunk frames = [3598848,3599360) | 20260506 18:43:41.683 | INFO | MemInfo: { | 20260506 18:43:41.683 | INFO | "gpu" : " 17 GB", | 20260506 18:43:41.683 | INFO | "huge" : "", | 20260506 18:43:41.683 | INFO | "huge_shared" : " 27 GB", | 20260506 18:43:41.683 | INFO | "malloc" : " 14 GB", | 20260506 18:43:41.683 | INFO | "pinned" : " 29 GB" | 20260506 18:43:41.683 | INFO | } | 20260506 18:43:41.683 | INFO | Perf report for the last 5.13852 seconds: | 20260506 18:43:41.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:43:41.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:41.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:41.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:41.683 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:43:41.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:41.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:41.683 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:43:41.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:41.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:41.683 | INFO | Uploaded 36GiB to GPU in 2.05969s (17.4783GiB/s) | 20260506 18:43:41.683 | INFO | Downloaded 34GiB to GPU in 1.96394s (17.3122GiB/s) | 20260506 18:43:46.827 | INFO | Analyzing chunk frames = [3599360,3599872) | 20260506 18:43:46.827 | INFO | MemInfo: { | 20260506 18:43:46.827 | INFO | "gpu" : " 17 GB", | 20260506 18:43:46.827 | INFO | "huge" : "", | 20260506 18:43:46.827 | INFO | "huge_shared" : " 27 GB", | 20260506 18:43:46.827 | INFO | "malloc" : " 14 GB", | 20260506 18:43:46.827 | INFO | "pinned" : " 29 GB" | 20260506 18:43:46.827 | INFO | } | 20260506 18:43:46.827 | INFO | Perf report for the last 5.14409 seconds: | 20260506 18:43:46.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:43:46.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:46.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:46.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:46.827 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 18:43:46.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:46.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:46.827 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:43:46.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:46.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:46.827 | INFO | Uploaded 36GiB to GPU in 2.01197s (17.8929GiB/s) | 20260506 18:43:46.827 | INFO | Downloaded 34GiB to GPU in 1.9969s (17.0264GiB/s) | 20260506 18:43:51.972 | INFO | Analyzing chunk frames = [3599872,3600384) | 20260506 18:43:51.972 | INFO | MemInfo: { | 20260506 18:43:51.972 | INFO | "gpu" : " 17 GB", | 20260506 18:43:51.972 | INFO | "huge" : "", | 20260506 18:43:51.972 | INFO | "huge_shared" : " 27 GB", | 20260506 18:43:51.973 | INFO | "malloc" : " 14 GB", | 20260506 18:43:51.973 | INFO | "pinned" : " 29 GB" | 20260506 18:43:51.973 | INFO | } | 20260506 18:43:51.973 | INFO | Perf report for the last 5.1455 seconds: | 20260506 18:43:51.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:43:51.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:51.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:51.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:51.973 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 18:43:51.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:51.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:51.973 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 18:43:51.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:51.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:51.973 | INFO | Uploaded 36GiB to GPU in 2.02019s (17.8201GiB/s) | 20260506 18:43:51.973 | INFO | Downloaded 34GiB to GPU in 1.98087s (17.1642GiB/s) | 20260506 18:43:52.087 | INFO | Starting a round of Baz IO | 20260506 18:43:55.227 | INFO | Populating RTMetrics report | 20260506 18:43:55.227 | INFO | RTMetrics report populated | 20260506 18:43:55.227 | INFO | Opening tmp RTMetrics json file | 20260506 18:43:55.274 | INFO | Serializing RTMetrics to json | 20260506 18:43:55.374 | INFO | Serialization done, swapping tmp file now... | 20260506 18:43:55.379 | INFO | RTMetrics done | 20260506 18:43:55.625 | INFO | Finished a round of Baz IO: Wrote 2.031 GiB in 3.538 seconds | 20260506 18:43:55.625 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.94% / 0.01% / 65.06% | 20260506 18:43:55.625 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.06099% / 1.58946e-05% / 5.5631e-05% | 20260506 18:43:55.625 | INFO | Baz IO overall: | 20260506 18:43:55.625 | INFO | BytesWritten : 1902395758236 | 20260506 18:43:55.625 | INFO | NumEvents : 919915699239 | 20260506 18:43:57.107 | INFO | Analyzing chunk frames = [3600384,3600896) | 20260506 18:43:57.107 | INFO | MemInfo: { | 20260506 18:43:57.107 | INFO | "gpu" : " 17 GB", | 20260506 18:43:57.107 | INFO | "huge" : "", | 20260506 18:43:57.107 | INFO | "huge_shared" : " 27 GB", | 20260506 18:43:57.107 | INFO | "malloc" : " 14 GB", | 20260506 18:43:57.107 | INFO | "pinned" : " 29 GB" | 20260506 18:43:57.107 | INFO | } | 20260506 18:43:57.107 | INFO | Perf report for the last 5.13476 seconds: | 20260506 18:43:57.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:43:57.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:57.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:57.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:57.107 | INFO | ANALYSIS : 62.8%, 2.9, 94.7ms, 0 | 20260506 18:43:57.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:43:57.107 | INFO | RT_METRICS : 48.2%, 1.0, 25.3ms, 0 | 20260506 18:43:57.107 | INFO | PRE_HQ : 60.9%, 1.9, 60.4ms, 0 | 20260506 18:43:57.108 | INFO | BAZWRITER : 68.6%, 4.8, 169.1ms, 0 | 20260506 18:43:57.108 | INFO | BAZ_SAMPLING : 2.9%, 1.0, 1.5ms, 0 | 20260506 18:43:57.108 | INFO | Uploaded 36GiB to GPU in 2.07263s (17.3693GiB/s) | 20260506 18:43:57.108 | INFO | Downloaded 38GiB to GPU in 2.21161s (17.1821GiB/s) | 20260506 18:44:02.250 | INFO | Analyzing chunk frames = [3600896,3601408) | 20260506 18:44:02.250 | INFO | MemInfo: { | 20260506 18:44:02.250 | INFO | "gpu" : " 17 GB", | 20260506 18:44:02.250 | INFO | "huge" : "", | 20260506 18:44:02.250 | INFO | "huge_shared" : " 27 GB", | 20260506 18:44:02.250 | INFO | "malloc" : " 14 GB", | 20260506 18:44:02.250 | INFO | "pinned" : " 29 GB" | 20260506 18:44:02.250 | INFO | } | 20260506 18:44:02.250 | INFO | Perf report for the last 5.1425 seconds: | 20260506 18:44:02.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:44:02.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:02.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:02.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:02.250 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:44:02.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:02.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:02.250 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 18:44:02.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:02.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:02.250 | INFO | Uploaded 36GiB to GPU in 2.02118s (17.8114GiB/s) | 20260506 18:44:02.250 | INFO | Downloaded 34GiB to GPU in 1.93688s (17.554GiB/s) | 20260506 18:44:07.380 | INFO | Analyzing chunk frames = [3601408,3601920) | 20260506 18:44:07.380 | INFO | MemInfo: { | 20260506 18:44:07.380 | INFO | "gpu" : " 17 GB", | 20260506 18:44:07.380 | INFO | "huge" : "", | 20260506 18:44:07.380 | INFO | "huge_shared" : " 27 GB", | 20260506 18:44:07.380 | INFO | "malloc" : " 14 GB", | 20260506 18:44:07.380 | INFO | "pinned" : " 29 GB" | 20260506 18:44:07.380 | INFO | } | 20260506 18:44:07.380 | INFO | Perf report for the last 5.13012 seconds: | 20260506 18:44:07.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:44:07.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:07.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:07.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:07.380 | INFO | ANALYSIS : 60.9%, 3.0, 94.1ms, 0 | 20260506 18:44:07.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:07.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:07.380 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:44:07.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:07.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:07.380 | INFO | Uploaded 36GiB to GPU in 1.97888s (18.1922GiB/s) | 20260506 18:44:07.380 | INFO | Downloaded 34GiB to GPU in 1.91548s (17.7501GiB/s) | 20260506 18:44:12.519 | INFO | Analyzing chunk frames = [3601920,3602432) | 20260506 18:44:12.519 | INFO | MemInfo: { | 20260506 18:44:12.519 | INFO | "gpu" : " 17 GB", | 20260506 18:44:12.519 | INFO | "huge" : "", | 20260506 18:44:12.519 | INFO | "huge_shared" : " 27 GB", | 20260506 18:44:12.519 | INFO | "malloc" : " 14 GB", | 20260506 18:44:12.519 | INFO | "pinned" : " 29 GB" | 20260506 18:44:12.519 | INFO | } | 20260506 18:44:12.519 | INFO | Perf report for the last 5.1389 seconds: | 20260506 18:44:12.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:44:12.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:12.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:12.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:12.520 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:44:12.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:12.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:12.520 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:44:12.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:12.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:12.520 | INFO | Uploaded 36GiB to GPU in 1.94763s (18.484GiB/s) | 20260506 18:44:12.520 | INFO | Downloaded 34GiB to GPU in 1.86901s (18.1915GiB/s) | 20260506 18:44:17.663 | INFO | Analyzing chunk frames = [3602432,3602944) | 20260506 18:44:17.663 | INFO | MemInfo: { | 20260506 18:44:17.663 | INFO | "gpu" : " 17 GB", | 20260506 18:44:17.663 | INFO | "huge" : "", | 20260506 18:44:17.663 | INFO | "huge_shared" : " 27 GB", | 20260506 18:44:17.663 | INFO | "malloc" : " 14 GB", | 20260506 18:44:17.663 | INFO | "pinned" : " 29 GB" | 20260506 18:44:17.663 | INFO | } | 20260506 18:44:17.663 | INFO | Perf report for the last 5.14387 seconds: | 20260506 18:44:17.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:44:17.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:17.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:17.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:17.663 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 18:44:17.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:17.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:17.663 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:44:17.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:17.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:17.663 | INFO | Uploaded 36GiB to GPU in 2.02185s (17.8054GiB/s) | 20260506 18:44:17.663 | INFO | Downloaded 34GiB to GPU in 1.91402s (17.7637GiB/s) | 20260506 18:44:22.799 | INFO | Analyzing chunk frames = [3602944,3603456) | 20260506 18:44:22.799 | INFO | MemInfo: { | 20260506 18:44:22.799 | INFO | "gpu" : " 17 GB", | 20260506 18:44:22.799 | INFO | "huge" : "", | 20260506 18:44:22.799 | INFO | "huge_shared" : " 27 GB", | 20260506 18:44:22.799 | INFO | "malloc" : " 14 GB", | 20260506 18:44:22.799 | INFO | "pinned" : " 29 GB" | 20260506 18:44:22.799 | INFO | } | 20260506 18:44:22.799 | INFO | Perf report for the last 5.13604 seconds: | 20260506 18:44:22.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:44:22.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:22.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:22.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:22.799 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 18:44:22.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:22.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:22.799 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:44:22.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:22.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:22.799 | INFO | Uploaded 36GiB to GPU in 2.03026s (17.7317GiB/s) | 20260506 18:44:22.799 | INFO | Downloaded 34GiB to GPU in 1.94339s (17.4952GiB/s) | 20260506 18:44:27.937 | INFO | Analyzing chunk frames = [3603456,3603968) | 20260506 18:44:27.937 | INFO | MemInfo: { | 20260506 18:44:27.937 | INFO | "gpu" : " 17 GB", | 20260506 18:44:27.937 | INFO | "huge" : "", | 20260506 18:44:27.937 | INFO | "huge_shared" : " 27 GB", | 20260506 18:44:27.937 | INFO | "malloc" : " 14 GB", | 20260506 18:44:27.937 | INFO | "pinned" : " 29 GB" | 20260506 18:44:27.937 | INFO | } | 20260506 18:44:27.937 | INFO | Perf report for the last 5.13797 seconds: | 20260506 18:44:27.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:44:27.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:27.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:27.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:27.937 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:44:27.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:27.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:27.937 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 18:44:27.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:27.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:27.937 | INFO | Uploaded 36GiB to GPU in 2.00814s (17.927GiB/s) | 20260506 18:44:27.937 | INFO | Downloaded 34GiB to GPU in 2.0083s (16.9298GiB/s) | 20260506 18:44:33.079 | INFO | Analyzing chunk frames = [3603968,3604480) | 20260506 18:44:33.079 | INFO | MemInfo: { | 20260506 18:44:33.079 | INFO | "gpu" : " 17 GB", | 20260506 18:44:33.079 | INFO | "huge" : "", | 20260506 18:44:33.079 | INFO | "huge_shared" : " 27 GB", | 20260506 18:44:33.079 | INFO | "malloc" : " 14 GB", | 20260506 18:44:33.079 | INFO | "pinned" : " 29 GB" | 20260506 18:44:33.079 | INFO | } | 20260506 18:44:33.079 | INFO | Perf report for the last 5.14231 seconds: | 20260506 18:44:33.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:44:33.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:33.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:33.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:33.079 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 18:44:33.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:33.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:33.079 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:44:33.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:33.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:33.080 | INFO | Uploaded 36GiB to GPU in 1.99648s (18.0317GiB/s) | 20260506 18:44:33.080 | INFO | Downloaded 34GiB to GPU in 1.99399s (17.0512GiB/s) | 20260506 18:44:33.080 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:44:33.080 | INFO | Total mem usage: 18 GiB | 20260506 18:44:33.080 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:44:33.080 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.2%) | 20260506 18:44:33.080 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:44:33.080 | INFO | High Water Breakdown: | 20260506 18:44:33.080 | INFO | 61.7% : 9.82 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:44:33.080 | INFO | 38.2% : 6.07 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:44:33.080 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:44:33.080 | INFO | ------------------------------------------ | 20260506 18:44:33.080 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:44:33.080 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:44:33.080 | INFO | Total mem usage: 26.5 GiB | 20260506 18:44:33.080 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:44:33.080 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:44:33.080 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:44:33.080 | INFO | High Water Breakdown: | 20260506 18:44:33.080 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:44:33.080 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:44:33.080 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:44:33.080 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:44:33.080 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:44:33.080 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:44:33.080 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:44:33.080 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:44:33.080 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:44:33.080 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:44:33.080 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:44:33.080 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:44:33.080 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:44:33.080 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:44:33.080 | INFO | ------------------------------------------ | 20260506 18:44:33.080 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:44:33.080 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:44:33.080 | INFO | Total mem usage: 25.6 GiB | 20260506 18:44:33.080 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:44:33.080 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:44:33.080 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:44:33.080 | INFO | High Water Breakdown: | 20260506 18:44:33.080 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:44:33.080 | INFO | ------------------------------------------ | 20260506 18:44:33.080 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:44:33.080 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:44:33.080 | INFO | Total mem usage: 15.7 GiB | 20260506 18:44:33.080 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:44:33.080 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:44:33.080 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 18:44:33.080 | INFO | High Water Breakdown: | 20260506 18:44:33.080 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:44:33.080 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:44:33.080 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:44:33.080 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:44:33.080 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:44:33.080 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:44:33.080 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:44:33.080 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:44:33.080 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:44:33.080 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:44:33.080 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:44:33.080 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:44:33.080 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:44:33.080 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:44:33.080 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:44:33.080 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:44:33.080 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:44:33.080 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:44:33.080 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:44:33.080 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:44:33.080 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:44:33.080 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:44:33.080 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:44:33.080 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:44:33.080 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:44:33.080 | INFO | ------------------------------------------ | 20260506 18:44:33.080 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:44:33.081 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:44:33.081 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:44:33.081 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:44:33.081 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.60e+01, 1.34e+01 / 2.38e+00 | 20260506 18:44:33.081 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.02e-01, 1.01e-02 / 7.61e-03 | 20260506 18:44:33.081 | INFO | PerfReport: 11.60%, Baseline -- 3.63e+00 / 4.44e+00, 5.48e-01 / 3.37e-01 | 20260506 18:44:33.081 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.49e+00, 1.85e-01 / 1.23e-01 | 20260506 18:44:33.081 | INFO | PerfReport: 16.01%, DME -- 4.00e+01 / 4.48e+01, 4.00e+00 / 2.25e+00 | 20260506 18:44:33.081 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.46e+00 / 2.03e+00 | 20260506 18:44:33.081 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.25e+00 / 2.82e+00, 3.78e-01 / 2.24e-01 | 20260506 18:44:33.081 | INFO | PerfReport: 3.38%, Metrics -- 1.06e+00 / 1.37e+00, 2.74e-01 / 1.93e-01 | 20260506 18:44:33.081 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.66e-01 / 4.16e-01, 5.68e-02 / 3.57e-02 | 20260506 18:44:33.081 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:44:33.081 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:44:33.081 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:44:33.081 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 18:44:33.081 | INFO | PerfReport: 21.55%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260506 18:44:33.081 | INFO | PerfReport: 21.21%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 18:44:33.081 | INFO | PerfReport: 32.23%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 18:44:33.081 | INFO | PerfReport: 1.43%, LockUpload -- 1.38e+00 / 0.00e+00, 5.53e+00 / 0.00e+00 | 20260506 18:44:33.081 | INFO | PerfReport: 1.12%, LockDownload -- 1.09e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 18:44:33.081 | INFO | PerfReport: 22.45%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:44:33.081 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260506 18:44:33.194 | INFO | Starting a round of Baz IO | 20260506 18:44:36.328 | INFO | Populating RTMetrics report | 20260506 18:44:36.329 | INFO | RTMetrics report populated | 20260506 18:44:36.329 | INFO | Opening tmp RTMetrics json file | 20260506 18:44:36.361 | INFO | Serializing RTMetrics to json | 20260506 18:44:36.364 | INFO | Serialization done, swapping tmp file now... | 20260506 18:44:36.364 | INFO | RTMetrics done | 20260506 18:44:36.945 | INFO | Finished a round of Baz IO: Wrote 2.029 GiB in 3.751 seconds | 20260506 18:44:36.945 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.93% / 0.01% / 65.06% | 20260506 18:44:36.945 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.05247% / 3.97364e-06% / 9.53674e-05% | 20260506 18:44:36.945 | INFO | Baz IO overall: | 20260506 18:44:36.945 | INFO | BytesWritten : 1904573952930 | 20260506 18:44:36.945 | INFO | NumEvents : 920963916717 | 20260506 18:44:38.220 | INFO | Analyzing chunk frames = [3604480,3604992) | 20260506 18:44:38.220 | INFO | MemInfo: { | 20260506 18:44:38.220 | INFO | "gpu" : " 17 GB", | 20260506 18:44:38.220 | INFO | "huge" : "", | 20260506 18:44:38.220 | INFO | "huge_shared" : " 27 GB", | 20260506 18:44:38.220 | INFO | "malloc" : " 14 GB", | 20260506 18:44:38.220 | INFO | "pinned" : " 29 GB" | 20260506 18:44:38.220 | INFO | } | 20260506 18:44:38.220 | INFO | Perf report for the last 5.14074 seconds: | 20260506 18:44:38.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:44:38.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:38.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:38.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:38.220 | INFO | ANALYSIS : 62.6%, 2.9, 95.7ms, 0 | 20260506 18:44:38.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:38.220 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 18:44:38.220 | INFO | PRE_HQ : 60.0%, 1.8, 57.6ms, 0 | 20260506 18:44:38.220 | INFO | BAZWRITER : 72.6%, 4.8, 180.6ms, 0 | 20260506 18:44:38.220 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 18:44:38.220 | INFO | Uploaded 36GiB to GPU in 2.02649s (17.7647GiB/s) | 20260506 18:44:38.220 | INFO | Downloaded 38GiB to GPU in 2.2034s (17.2461GiB/s) | 20260506 18:44:43.358 | INFO | Analyzing chunk frames = [3604992,3605504) | 20260506 18:44:43.358 | INFO | MemInfo: { | 20260506 18:44:43.358 | INFO | "gpu" : " 17 GB", | 20260506 18:44:43.358 | INFO | "huge" : "", | 20260506 18:44:43.358 | INFO | "huge_shared" : " 27 GB", | 20260506 18:44:43.358 | INFO | "malloc" : " 14 GB", | 20260506 18:44:43.358 | INFO | "pinned" : " 29 GB" | 20260506 18:44:43.358 | INFO | } | 20260506 18:44:43.358 | INFO | Perf report for the last 5.13827 seconds: | 20260506 18:44:43.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:44:43.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:43.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:43.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:43.358 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 18:44:43.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:43.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:43.358 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:44:43.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:43.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:43.359 | INFO | Uploaded 36GiB to GPU in 1.97585s (18.22GiB/s) | 20260506 18:44:43.359 | INFO | Downloaded 34GiB to GPU in 1.91616s (17.7438GiB/s) | 20260506 18:44:48.499 | INFO | Analyzing chunk frames = [3605504,3606016) | 20260506 18:44:48.499 | INFO | MemInfo: { | 20260506 18:44:48.499 | INFO | "gpu" : " 17 GB", | 20260506 18:44:48.499 | INFO | "huge" : "", | 20260506 18:44:48.500 | INFO | "huge_shared" : " 27 GB", | 20260506 18:44:48.500 | INFO | "malloc" : " 14 GB", | 20260506 18:44:48.500 | INFO | "pinned" : " 29 GB" | 20260506 18:44:48.500 | INFO | } | 20260506 18:44:48.500 | INFO | Perf report for the last 5.14142 seconds: | 20260506 18:44:48.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:44:48.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:48.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:48.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:48.500 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 18:44:48.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:48.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:48.500 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:44:48.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:48.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:48.500 | INFO | Uploaded 36GiB to GPU in 1.95784s (18.3876GiB/s) | 20260506 18:44:48.500 | INFO | Downloaded 34GiB to GPU in 1.92578s (17.6551GiB/s) | 20260506 18:44:53.634 | INFO | Analyzing chunk frames = [3606016,3606528) | 20260506 18:44:53.634 | INFO | MemInfo: { | 20260506 18:44:53.634 | INFO | "gpu" : " 17 GB", | 20260506 18:44:53.634 | INFO | "huge" : "", | 20260506 18:44:53.634 | INFO | "huge_shared" : " 27 GB", | 20260506 18:44:53.634 | INFO | "malloc" : " 14 GB", | 20260506 18:44:53.634 | INFO | "pinned" : " 29 GB" | 20260506 18:44:53.634 | INFO | } | 20260506 18:44:53.635 | INFO | Perf report for the last 5.13482 seconds: | 20260506 18:44:53.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:44:53.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:53.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:53.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:53.635 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 18:44:53.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:53.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:53.635 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 18:44:53.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:53.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:53.635 | INFO | Uploaded 36GiB to GPU in 1.93342s (18.6199GiB/s) | 20260506 18:44:53.635 | INFO | Downloaded 34GiB to GPU in 1.87089s (18.1731GiB/s) | 20260506 18:44:58.773 | INFO | Analyzing chunk frames = [3606528,3607040) | 20260506 18:44:58.773 | INFO | MemInfo: { | 20260506 18:44:58.773 | INFO | "gpu" : " 17 GB", | 20260506 18:44:58.773 | INFO | "huge" : "", | 20260506 18:44:58.773 | INFO | "huge_shared" : " 27 GB", | 20260506 18:44:58.773 | INFO | "malloc" : " 14 GB", | 20260506 18:44:58.773 | INFO | "pinned" : " 29 GB" | 20260506 18:44:58.773 | INFO | } | 20260506 18:44:58.773 | INFO | Perf report for the last 5.13869 seconds: | 20260506 18:44:58.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:44:58.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:58.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:58.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:58.773 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 18:44:58.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:58.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:58.773 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:44:58.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:58.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:44:58.774 | INFO | Uploaded 36GiB to GPU in 2.00882s (17.921GiB/s) | 20260506 18:44:58.774 | INFO | Downloaded 34GiB to GPU in 1.91252s (17.7776GiB/s) | 20260506 18:45:03.916 | INFO | Analyzing chunk frames = [3607040,3607552) | 20260506 18:45:03.916 | INFO | MemInfo: { | 20260506 18:45:03.916 | INFO | "gpu" : " 17 GB", | 20260506 18:45:03.917 | INFO | "huge" : "", | 20260506 18:45:03.917 | INFO | "huge_shared" : " 27 GB", | 20260506 18:45:03.917 | INFO | "malloc" : " 14 GB", | 20260506 18:45:03.917 | INFO | "pinned" : " 29 GB" | 20260506 18:45:03.917 | INFO | } | 20260506 18:45:03.917 | INFO | Perf report for the last 5.14349 seconds: | 20260506 18:45:03.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:45:03.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:03.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:03.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:03.917 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 18:45:03.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:03.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:03.917 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260506 18:45:03.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:03.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:03.917 | INFO | Uploaded 36GiB to GPU in 2.01523s (17.8639GiB/s) | 20260506 18:45:03.917 | INFO | Downloaded 34GiB to GPU in 1.94825s (17.4515GiB/s) | 20260506 18:45:09.054 | INFO | Analyzing chunk frames = [3607552,3608064) | 20260506 18:45:09.054 | INFO | MemInfo: { | 20260506 18:45:09.054 | INFO | "gpu" : " 17 GB", | 20260506 18:45:09.054 | INFO | "huge" : "", | 20260506 18:45:09.054 | INFO | "huge_shared" : " 27 GB", | 20260506 18:45:09.054 | INFO | "malloc" : " 14 GB", | 20260506 18:45:09.054 | INFO | "pinned" : " 29 GB" | 20260506 18:45:09.054 | INFO | } | 20260506 18:45:09.054 | INFO | Perf report for the last 5.13742 seconds: | 20260506 18:45:09.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:45:09.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:09.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:09.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:09.054 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:45:09.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:09.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:09.054 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:45:09.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:09.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:09.055 | INFO | Uploaded 36GiB to GPU in 1.97348s (18.2419GiB/s) | 20260506 18:45:09.055 | INFO | Downloaded 34GiB to GPU in 2.0073s (16.9381GiB/s) | 20260506 18:45:14.193 | INFO | Analyzing chunk frames = [3608064,3608576) | 20260506 18:45:14.193 | INFO | MemInfo: { | 20260506 18:45:14.193 | INFO | "gpu" : " 17 GB", | 20260506 18:45:14.193 | INFO | "huge" : "", | 20260506 18:45:14.193 | INFO | "huge_shared" : " 27 GB", | 20260506 18:45:14.193 | INFO | "malloc" : " 14 GB", | 20260506 18:45:14.193 | INFO | "pinned" : " 29 GB" | 20260506 18:45:14.193 | INFO | } | 20260506 18:45:14.193 | INFO | Perf report for the last 5.13897 seconds: | 20260506 18:45:14.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:45:14.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:14.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:14.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:14.193 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 18:45:14.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:14.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:14.193 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 18:45:14.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:14.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:14.194 | INFO | Uploaded 36GiB to GPU in 1.9854s (18.1324GiB/s) | 20260506 18:45:14.194 | INFO | Downloaded 34GiB to GPU in 1.98632s (17.1171GiB/s) | 20260506 18:45:14.307 | INFO | Starting a round of Baz IO | 20260506 18:45:17.442 | INFO | Populating RTMetrics report | 20260506 18:45:17.443 | INFO | RTMetrics report populated | 20260506 18:45:17.443 | INFO | Opening tmp RTMetrics json file | 20260506 18:45:17.444 | INFO | Serializing RTMetrics to json | 20260506 18:45:17.445 | INFO | Serialization done, swapping tmp file now... | 20260506 18:45:17.446 | INFO | RTMetrics done | 20260506 18:45:17.595 | INFO | Finished a round of Baz IO: Wrote 2.03 GiB in 3.288 seconds | 20260506 18:45:17.595 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.93% / 0.01% / 65.07% | 20260506 18:45:17.595 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.05453% / 3.97364e-06% / 7.94729e-05% | 20260506 18:45:17.595 | INFO | Baz IO overall: | 20260506 18:45:17.595 | INFO | BytesWritten : 1906753760454 | 20260506 18:45:17.595 | INFO | NumEvents : 922013505761 | 20260506 18:45:19.336 | INFO | Analyzing chunk frames = [3608576,3609088) | 20260506 18:45:19.336 | INFO | MemInfo: { | 20260506 18:45:19.336 | INFO | "gpu" : " 17 GB", | 20260506 18:45:19.336 | INFO | "huge" : "", | 20260506 18:45:19.337 | INFO | "huge_shared" : " 27 GB", | 20260506 18:45:19.337 | INFO | "malloc" : " 14 GB", | 20260506 18:45:19.337 | INFO | "pinned" : " 29 GB" | 20260506 18:45:19.337 | INFO | } | 20260506 18:45:19.337 | INFO | Perf report for the last 5.14349 seconds: | 20260506 18:45:19.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:45:19.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:19.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:19.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:19.337 | INFO | ANALYSIS : 62.6%, 2.9, 95.2ms, 0 | 20260506 18:45:19.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:19.337 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260506 18:45:19.337 | INFO | PRE_HQ : 60.2%, 1.9, 58.2ms, 0 | 20260506 18:45:19.337 | INFO | BAZWRITER : 63.7%, 4.6, 150.2ms, 0 | 20260506 18:45:19.337 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 18:45:19.337 | INFO | Uploaded 36GiB to GPU in 2.07416s (17.3564GiB/s) | 20260506 18:45:19.337 | INFO | Downloaded 38GiB to GPU in 2.20484s (17.2348GiB/s) | 20260506 18:45:24.471 | INFO | Analyzing chunk frames = [3609088,3609600) | 20260506 18:45:24.471 | INFO | MemInfo: { | 20260506 18:45:24.471 | INFO | "gpu" : " 17 GB", | 20260506 18:45:24.471 | INFO | "huge" : "", | 20260506 18:45:24.471 | INFO | "huge_shared" : " 27 GB", | 20260506 18:45:24.471 | INFO | "malloc" : " 14 GB", | 20260506 18:45:24.471 | INFO | "pinned" : " 29 GB" | 20260506 18:45:24.471 | INFO | } | 20260506 18:45:24.471 | INFO | Perf report for the last 5.13441 seconds: | 20260506 18:45:24.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:45:24.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:24.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:24.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:24.471 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 18:45:24.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:24.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:24.471 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 18:45:24.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:24.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:24.471 | INFO | Uploaded 36GiB to GPU in 2.02059s (17.8166GiB/s) | 20260506 18:45:24.471 | INFO | Downloaded 34GiB to GPU in 1.94013s (17.5246GiB/s) | 20260506 18:45:29.612 | INFO | Analyzing chunk frames = [3609600,3610112) | 20260506 18:45:29.612 | INFO | MemInfo: { | 20260506 18:45:29.612 | INFO | "gpu" : " 17 GB", | 20260506 18:45:29.612 | INFO | "huge" : "", | 20260506 18:45:29.612 | INFO | "huge_shared" : " 27 GB", | 20260506 18:45:29.612 | INFO | "malloc" : " 14 GB", | 20260506 18:45:29.612 | INFO | "pinned" : " 29 GB" | 20260506 18:45:29.613 | INFO | } | 20260506 18:45:29.613 | INFO | Perf report for the last 5.1415 seconds: | 20260506 18:45:29.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:45:29.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:29.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:29.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:29.613 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 18:45:29.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:29.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:29.613 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:45:29.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:29.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:29.613 | INFO | Uploaded 36GiB to GPU in 1.97673s (18.2119GiB/s) | 20260506 18:45:29.613 | INFO | Downloaded 34GiB to GPU in 1.9208s (17.701GiB/s) | 20260506 18:45:34.753 | INFO | Analyzing chunk frames = [3610112,3610624) | 20260506 18:45:34.753 | INFO | MemInfo: { | 20260506 18:45:34.753 | INFO | "gpu" : " 17 GB", | 20260506 18:45:34.753 | INFO | "huge" : "", | 20260506 18:45:34.753 | INFO | "huge_shared" : " 27 GB", | 20260506 18:45:34.753 | INFO | "malloc" : " 14 GB", | 20260506 18:45:34.753 | INFO | "pinned" : " 29 GB" | 20260506 18:45:34.753 | INFO | } | 20260506 18:45:34.753 | INFO | Perf report for the last 5.1408 seconds: | 20260506 18:45:34.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:45:34.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:34.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:34.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:34.753 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:45:34.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:34.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:34.754 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 18:45:34.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:34.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:34.754 | INFO | Uploaded 36GiB to GPU in 1.93103s (18.6429GiB/s) | 20260506 18:45:34.754 | INFO | Downloaded 34GiB to GPU in 1.86995s (18.1823GiB/s) | 20260506 18:45:39.888 | INFO | Analyzing chunk frames = [3610624,3611136) | 20260506 18:45:39.888 | INFO | MemInfo: { | 20260506 18:45:39.888 | INFO | "gpu" : " 17 GB", | 20260506 18:45:39.888 | INFO | "huge" : "", | 20260506 18:45:39.888 | INFO | "huge_shared" : " 27 GB", | 20260506 18:45:39.888 | INFO | "malloc" : " 14 GB", | 20260506 18:45:39.888 | INFO | "pinned" : " 29 GB" | 20260506 18:45:39.888 | INFO | } | 20260506 18:45:39.888 | INFO | Perf report for the last 5.13494 seconds: | 20260506 18:45:39.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:45:39.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:39.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:39.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:39.888 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260506 18:45:39.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:39.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:39.889 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:45:39.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:39.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:39.889 | INFO | Uploaded 36GiB to GPU in 2.0197s (17.8244GiB/s) | 20260506 18:45:39.889 | INFO | Downloaded 34GiB to GPU in 1.91206s (17.7818GiB/s) | 20260506 18:45:45.026 | INFO | Analyzing chunk frames = [3611136,3611648) | 20260506 18:45:45.026 | INFO | MemInfo: { | 20260506 18:45:45.026 | INFO | "gpu" : " 17 GB", | 20260506 18:45:45.027 | INFO | "huge" : "", | 20260506 18:45:45.027 | INFO | "huge_shared" : " 27 GB", | 20260506 18:45:45.027 | INFO | "malloc" : " 14 GB", | 20260506 18:45:45.027 | INFO | "pinned" : " 29 GB" | 20260506 18:45:45.027 | INFO | } | 20260506 18:45:45.027 | INFO | Perf report for the last 5.1383 seconds: | 20260506 18:45:45.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:45:45.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:45.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:45.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:45.027 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 18:45:45.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:45.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:45.027 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 18:45:45.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:45.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:45.027 | INFO | Uploaded 36GiB to GPU in 2.03533s (17.6875GiB/s) | 20260506 18:45:45.027 | INFO | Downloaded 34GiB to GPU in 1.94998s (17.4361GiB/s) | 20260506 18:45:50.172 | INFO | Analyzing chunk frames = [3611648,3612160) | 20260506 18:45:50.172 | INFO | MemInfo: { | 20260506 18:45:50.172 | INFO | "gpu" : " 17 GB", | 20260506 18:45:50.173 | INFO | "huge" : "", | 20260506 18:45:50.173 | INFO | "huge_shared" : " 27 GB", | 20260506 18:45:50.173 | INFO | "malloc" : " 14 GB", | 20260506 18:45:50.173 | INFO | "pinned" : " 29 GB" | 20260506 18:45:50.173 | INFO | } | 20260506 18:45:50.173 | INFO | Perf report for the last 5.146 seconds: | 20260506 18:45:50.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:45:50.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:50.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:50.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:50.173 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:45:50.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:50.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:50.173 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260506 18:45:50.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:50.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:50.173 | INFO | Uploaded 36GiB to GPU in 1.99836s (18.0147GiB/s) | 20260506 18:45:50.173 | INFO | Downloaded 34GiB to GPU in 2.01227s (16.8964GiB/s) | 20260506 18:45:55.310 | INFO | Analyzing chunk frames = [3612160,3612672) | 20260506 18:45:55.310 | INFO | MemInfo: { | 20260506 18:45:55.310 | INFO | "gpu" : " 17 GB", | 20260506 18:45:55.310 | INFO | "huge" : "", | 20260506 18:45:55.310 | INFO | "huge_shared" : " 27 GB", | 20260506 18:45:55.310 | INFO | "malloc" : " 14 GB", | 20260506 18:45:55.310 | INFO | "pinned" : " 29 GB" | 20260506 18:45:55.310 | INFO | } | 20260506 18:45:55.310 | INFO | Perf report for the last 5.13759 seconds: | 20260506 18:45:55.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:45:55.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:55.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:55.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:55.310 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 18:45:55.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:55.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:55.310 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:45:55.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:55.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:45:55.310 | INFO | Uploaded 36GiB to GPU in 1.97614s (18.2174GiB/s) | 20260506 18:45:55.311 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:45:55.311 | INFO | Total mem usage: 18 GiB | 20260506 18:45:55.311 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:45:55.311 | INFO | Recent High Water Marks Sum: 16.9 GiB (Savings: -6.59%) | 20260506 18:45:55.311 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:45:55.311 | INFO | High Water Breakdown: | 20260506 18:45:55.311 | INFO | 62.6% : 10.6 GiB ( 92.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:45:55.311 | INFO | 37.3% : 6.3 GiB ( 94.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:45:55.311 | INFO | Downloaded 34GiB to GPU in 1.98625s (17.1176GiB/s) | 20260506 18:45:55.311 | INFO | 0.118% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:45:55.311 | INFO | ------------------------------------------ | 20260506 18:45:55.311 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:45:55.311 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:45:55.311 | INFO | Total mem usage: 26.5 GiB | 20260506 18:45:55.311 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:45:55.311 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:45:55.311 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:45:55.311 | INFO | High Water Breakdown: | 20260506 18:45:55.311 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:45:55.311 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:45:55.311 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:45:55.311 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:45:55.311 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:45:55.311 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:45:55.311 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:45:55.311 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:45:55.311 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:45:55.311 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:45:55.311 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:45:55.311 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:45:55.311 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:45:55.311 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:45:55.311 | INFO | ------------------------------------------ | 20260506 18:45:55.311 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:45:55.311 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:45:55.311 | INFO | Total mem usage: 25.6 GiB | 20260506 18:45:55.311 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:45:55.311 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:45:55.311 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:45:55.311 | INFO | High Water Breakdown: | 20260506 18:45:55.311 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:45:55.311 | INFO | ------------------------------------------ | 20260506 18:45:55.311 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:45:55.311 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:45:55.311 | INFO | Total mem usage: 15.7 GiB | 20260506 18:45:55.311 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:45:55.311 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:45:55.311 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 18:45:55.311 | INFO | High Water Breakdown: | 20260506 18:45:55.311 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:45:55.311 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:45:55.311 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:45:55.311 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:45:55.311 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:45:55.311 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:45:55.311 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:45:55.311 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:45:55.311 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:45:55.311 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:45:55.311 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:45:55.311 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:45:55.311 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:45:55.311 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:45:55.311 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:45:55.311 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:45:55.311 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:45:55.311 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:45:55.311 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:45:55.311 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:45:55.311 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:45:55.311 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:45:55.311 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:45:55.311 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:45:55.311 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:45:55.311 | INFO | ------------------------------------------ | 20260506 18:45:55.311 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:45:55.311 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:45:55.311 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:45:55.311 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:45:55.311 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.62e+01, 1.35e+01 / 2.46e+00 | 20260506 18:45:55.311 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.00e-01, 9.90e-03 / 5.72e-03 | 20260506 18:45:55.311 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.42e+00, 5.56e-01 / 3.45e-01 | 20260506 18:45:55.311 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.48e+00, 1.86e-01 / 1.34e-01 | 20260506 18:45:55.312 | INFO | PerfReport: 16.03%, DME -- 4.01e+01 / 4.49e+01, 4.26e+00 / 2.95e+00 | 20260506 18:45:55.312 | INFO | PerfReport: 57.01%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.36e+00 / 1.67e+00 | 20260506 18:45:55.312 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.26e+00 / 2.85e+00, 3.84e-01 / 2.05e-01 | 20260506 18:45:55.312 | INFO | PerfReport: 3.39%, Metrics -- 1.06e+00 / 1.40e+00, 2.77e-01 / 1.19e-01 | 20260506 18:45:55.312 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.66e-01 / 4.15e-01, 5.62e-02 / 2.32e-02 | 20260506 18:45:55.312 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:45:55.312 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:45:55.312 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:45:55.312 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 18:45:55.312 | INFO | PerfReport: 21.41%, Upload -- 2.08e+01 / 0.00e+00, 6.88e+00 / 0.00e+00 | 20260506 18:45:55.312 | INFO | PerfReport: 21.19%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 18:45:55.312 | INFO | PerfReport: 32.21%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:45:55.312 | INFO | PerfReport: 1.34%, LockUpload -- 1.30e+00 / 0.00e+00, 5.30e+00 / 0.00e+00 | 20260506 18:45:55.312 | INFO | PerfReport: 1.19%, LockDownload -- 1.16e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 18:45:55.312 | INFO | PerfReport: 22.64%, LockCompute -- 2.20e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 18:45:55.312 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260506 18:45:55.425 | INFO | Starting a round of Baz IO | 20260506 18:45:58.561 | INFO | Populating RTMetrics report | 20260506 18:45:58.561 | INFO | RTMetrics report populated | 20260506 18:45:58.561 | INFO | Opening tmp RTMetrics json file | 20260506 18:45:58.579 | INFO | Serializing RTMetrics to json | 20260506 18:45:58.616 | INFO | Serialization done, swapping tmp file now... | 20260506 18:45:58.648 | INFO | RTMetrics done | 20260506 18:45:59.152 | INFO | Finished a round of Baz IO: Wrote 2.028 GiB in 3.727 seconds | 20260506 18:45:59.152 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.92% / 0.01% / 65.07% | 20260506 18:45:59.152 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.04957% / 0% / 5.5631e-05% | 20260506 18:45:59.152 | INFO | Baz IO overall: | 20260506 18:45:59.152 | INFO | BytesWritten : 1908931746052 | 20260506 18:45:59.152 | INFO | NumEvents : 923061866318 | 20260506 18:46:00.449 | INFO | Analyzing chunk frames = [3612672,3613184) | 20260506 18:46:00.449 | INFO | MemInfo: { | 20260506 18:46:00.449 | INFO | "gpu" : " 17 GB", | 20260506 18:46:00.449 | INFO | "huge" : "", | 20260506 18:46:00.449 | INFO | "huge_shared" : " 27 GB", | 20260506 18:46:00.449 | INFO | "malloc" : " 14 GB", | 20260506 18:46:00.449 | INFO | "pinned" : " 29 GB" | 20260506 18:46:00.449 | INFO | } | 20260506 18:46:00.449 | INFO | Perf report for the last 5.1387 seconds: | 20260506 18:46:00.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:46:00.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:00.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:00.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:00.449 | INFO | ANALYSIS : 62.7%, 2.9, 95.6ms, 0 | 20260506 18:46:00.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:00.449 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 18:46:00.449 | INFO | PRE_HQ : 60.1%, 1.8, 57.5ms, 0 | 20260506 18:46:00.449 | INFO | BAZWRITER : 72.2%, 4.8, 178.8ms, 0 | 20260506 18:46:00.449 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 18:46:00.449 | INFO | Uploaded 36GiB to GPU in 2.06373s (17.4441GiB/s) | 20260506 18:46:00.449 | INFO | Downloaded 38GiB to GPU in 2.21766s (17.1351GiB/s) | 20260506 18:46:05.587 | INFO | Analyzing chunk frames = [3613184,3613696) | 20260506 18:46:05.587 | INFO | MemInfo: { | 20260506 18:46:05.587 | INFO | "gpu" : " 17 GB", | 20260506 18:46:05.587 | INFO | "huge" : "", | 20260506 18:46:05.587 | INFO | "huge_shared" : " 27 GB", | 20260506 18:46:05.587 | INFO | "malloc" : " 14 GB", | 20260506 18:46:05.587 | INFO | "pinned" : " 29 GB" | 20260506 18:46:05.587 | INFO | } | 20260506 18:46:05.587 | INFO | Perf report for the last 5.13769 seconds: | 20260506 18:46:05.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:46:05.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:05.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:05.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:05.587 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 18:46:05.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:05.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:05.587 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:46:05.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:05.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:05.587 | INFO | Uploaded 36GiB to GPU in 1.97814s (18.1989GiB/s) | 20260506 18:46:05.587 | INFO | Downloaded 34GiB to GPU in 1.93767s (17.5468GiB/s) | 20260506 18:46:10.727 | INFO | Analyzing chunk frames = [3613696,3614208) | 20260506 18:46:10.727 | INFO | MemInfo: { | 20260506 18:46:10.727 | INFO | "gpu" : " 17 GB", | 20260506 18:46:10.727 | INFO | "huge" : "", | 20260506 18:46:10.727 | INFO | "huge_shared" : " 27 GB", | 20260506 18:46:10.727 | INFO | "malloc" : " 14 GB", | 20260506 18:46:10.727 | INFO | "pinned" : " 29 GB" | 20260506 18:46:10.727 | INFO | } | 20260506 18:46:10.727 | INFO | Perf report for the last 5.14069 seconds: | 20260506 18:46:10.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:46:10.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:10.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:10.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:10.728 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 18:46:10.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:10.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:10.728 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 18:46:10.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:10.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:10.728 | INFO | Uploaded 36GiB to GPU in 1.96901s (18.2833GiB/s) | 20260506 18:46:10.728 | INFO | Downloaded 34GiB to GPU in 1.92026s (17.7059GiB/s) | 20260506 18:46:15.863 | INFO | Analyzing chunk frames = [3614208,3614720) | 20260506 18:46:15.863 | INFO | MemInfo: { | 20260506 18:46:15.863 | INFO | "gpu" : " 17 GB", | 20260506 18:46:15.863 | INFO | "huge" : "", | 20260506 18:46:15.863 | INFO | "huge_shared" : " 27 GB", | 20260506 18:46:15.863 | INFO | "malloc" : " 14 GB", | 20260506 18:46:15.863 | INFO | "pinned" : " 29 GB" | 20260506 18:46:15.863 | INFO | } | 20260506 18:46:15.863 | INFO | Perf report for the last 5.13551 seconds: | 20260506 18:46:15.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:46:15.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:15.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:15.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:15.863 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 18:46:15.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:15.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:15.863 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 18:46:15.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:15.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:15.863 | INFO | Uploaded 36GiB to GPU in 1.91344s (18.8142GiB/s) | 20260506 18:46:15.863 | INFO | Downloaded 34GiB to GPU in 1.87133s (18.1689GiB/s) | 20260506 18:46:21.003 | INFO | Analyzing chunk frames = [3614720,3615232) | 20260506 18:46:21.003 | INFO | MemInfo: { | 20260506 18:46:21.003 | INFO | "gpu" : " 17 GB", | 20260506 18:46:21.003 | INFO | "huge" : "", | 20260506 18:46:21.003 | INFO | "huge_shared" : " 27 GB", | 20260506 18:46:21.003 | INFO | "malloc" : " 14 GB", | 20260506 18:46:21.003 | INFO | "pinned" : " 29 GB" | 20260506 18:46:21.003 | INFO | } | 20260506 18:46:21.003 | INFO | Perf report for the last 5.14037 seconds: | 20260506 18:46:21.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:46:21.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:21.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:21.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:21.003 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260506 18:46:21.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:21.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:21.003 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:46:21.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:21.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:21.004 | INFO | Uploaded 36GiB to GPU in 1.94743s (18.4859GiB/s) | 20260506 18:46:21.004 | INFO | Downloaded 34GiB to GPU in 1.88298s (18.0565GiB/s) | 20260506 18:46:26.144 | INFO | Analyzing chunk frames = [3615232,3615744) | 20260506 18:46:26.144 | INFO | MemInfo: { | 20260506 18:46:26.144 | INFO | "gpu" : " 17 GB", | 20260506 18:46:26.144 | INFO | "huge" : "", | 20260506 18:46:26.144 | INFO | "huge_shared" : " 27 GB", | 20260506 18:46:26.144 | INFO | "malloc" : " 14 GB", | 20260506 18:46:26.144 | INFO | "pinned" : " 29 GB" | 20260506 18:46:26.144 | INFO | } | 20260506 18:46:26.145 | INFO | Perf report for the last 5.14134 seconds: | 20260506 18:46:26.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:46:26.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:26.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:26.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:26.145 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 18:46:26.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:26.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:26.145 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 18:46:26.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:26.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:26.145 | INFO | Uploaded 36GiB to GPU in 1.98772s (18.1112GiB/s) | 20260506 18:46:26.145 | INFO | Downloaded 34GiB to GPU in 1.95752s (17.3689GiB/s) | 20260506 18:46:31.287 | INFO | Analyzing chunk frames = [3615744,3616256) | 20260506 18:46:31.287 | INFO | MemInfo: { | 20260506 18:46:31.287 | INFO | "gpu" : " 17 GB", | 20260506 18:46:31.287 | INFO | "huge" : "", | 20260506 18:46:31.287 | INFO | "huge_shared" : " 27 GB", | 20260506 18:46:31.287 | INFO | "malloc" : " 14 GB", | 20260506 18:46:31.287 | INFO | "pinned" : " 29 GB" | 20260506 18:46:31.287 | INFO | } | 20260506 18:46:31.287 | INFO | Perf report for the last 5.1423 seconds: | 20260506 18:46:31.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:46:31.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:31.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:31.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:31.287 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:46:31.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:31.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:31.287 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 18:46:31.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:31.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:31.287 | INFO | Uploaded 36GiB to GPU in 1.98387s (18.1464GiB/s) | 20260506 18:46:31.287 | INFO | Downloaded 34GiB to GPU in 2.01038s (16.9122GiB/s) | 20260506 18:46:36.430 | INFO | Analyzing chunk frames = [3616256,3616768) | 20260506 18:46:36.430 | INFO | MemInfo: { | 20260506 18:46:36.430 | INFO | "gpu" : " 17 GB", | 20260506 18:46:36.430 | INFO | "huge" : "", | 20260506 18:46:36.430 | INFO | "huge_shared" : " 27 GB", | 20260506 18:46:36.430 | INFO | "malloc" : " 14 GB", | 20260506 18:46:36.430 | INFO | "pinned" : " 29 GB" | 20260506 18:46:36.430 | INFO | } | 20260506 18:46:36.430 | INFO | Perf report for the last 5.14299 seconds: | 20260506 18:46:36.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:46:36.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:36.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:36.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:36.430 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 18:46:36.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:36.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:36.430 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260506 18:46:36.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:36.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:36.430 | INFO | Uploaded 36GiB to GPU in 1.99097s (18.0816GiB/s) | 20260506 18:46:36.430 | INFO | Downloaded 34GiB to GPU in 1.98544s (17.1246GiB/s) | 20260506 18:46:36.538 | INFO | Starting a round of Baz IO | 20260506 18:46:39.676 | INFO | Populating RTMetrics report | 20260506 18:46:39.677 | INFO | RTMetrics report populated | 20260506 18:46:39.677 | INFO | Opening tmp RTMetrics json file | 20260506 18:46:39.717 | INFO | Serializing RTMetrics to json | 20260506 18:46:39.748 | INFO | Serialization done, swapping tmp file now... | 20260506 18:46:39.749 | INFO | RTMetrics done | 20260506 18:46:40.224 | INFO | Finished a round of Baz IO: Wrote 2.028 GiB in 3.685 seconds | 20260506 18:46:40.224 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.92% / 0.01% / 65.08% | 20260506 18:46:40.224 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.04846% / 7.94729e-06% / 6.75519e-05% | 20260506 18:46:40.224 | INFO | Baz IO overall: | 20260506 18:46:40.224 | INFO | BytesWritten : 1911109243071 | 20260506 18:46:40.224 | INFO | NumEvents : 924109883732 | 20260506 18:46:41.565 | INFO | Analyzing chunk frames = [3616768,3617280) | 20260506 18:46:41.565 | INFO | MemInfo: { | 20260506 18:46:41.565 | INFO | "gpu" : " 17 GB", | 20260506 18:46:41.565 | INFO | "huge" : "", | 20260506 18:46:41.565 | INFO | "huge_shared" : " 27 GB", | 20260506 18:46:41.565 | INFO | "malloc" : " 14 GB", | 20260506 18:46:41.565 | INFO | "pinned" : " 29 GB" | 20260506 18:46:41.565 | INFO | } | 20260506 18:46:41.565 | INFO | Perf report for the last 5.13506 seconds: | 20260506 18:46:41.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:46:41.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:41.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:41.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:41.565 | INFO | ANALYSIS : 62.7%, 2.9, 95.5ms, 0 | 20260506 18:46:41.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:41.565 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 18:46:41.565 | INFO | PRE_HQ : 60.2%, 1.8, 57.6ms, 0 | 20260506 18:46:41.565 | INFO | BAZWRITER : 71.5%, 4.8, 175.2ms, 0 | 20260506 18:46:41.565 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 18:46:41.565 | INFO | Uploaded 36GiB to GPU in 2.06975s (17.3934GiB/s) | 20260506 18:46:41.565 | INFO | Downloaded 38GiB to GPU in 2.2065s (17.2219GiB/s) | 20260506 18:46:46.700 | INFO | Analyzing chunk frames = [3617280,3617792) | 20260506 18:46:46.700 | INFO | MemInfo: { | 20260506 18:46:46.700 | INFO | "gpu" : " 17 GB", | 20260506 18:46:46.700 | INFO | "huge" : "", | 20260506 18:46:46.700 | INFO | "huge_shared" : " 27 GB", | 20260506 18:46:46.701 | INFO | "malloc" : " 14 GB", | 20260506 18:46:46.701 | INFO | "pinned" : " 29 GB" | 20260506 18:46:46.701 | INFO | } | 20260506 18:46:46.701 | INFO | Perf report for the last 5.13563 seconds: | 20260506 18:46:46.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:46:46.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:46.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:46.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:46.701 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 18:46:46.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:46.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:46.701 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:46:46.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:46.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:46.701 | INFO | Uploaded 36GiB to GPU in 2.01562s (17.8605GiB/s) | 20260506 18:46:46.701 | INFO | Downloaded 34GiB to GPU in 1.93309s (17.5884GiB/s) | 20260506 18:46:51.836 | INFO | Analyzing chunk frames = [3617792,3618304) | 20260506 18:46:51.836 | INFO | MemInfo: { | 20260506 18:46:51.836 | INFO | "gpu" : " 17 GB", | 20260506 18:46:51.836 | INFO | "huge" : "", | 20260506 18:46:51.836 | INFO | "huge_shared" : " 27 GB", | 20260506 18:46:51.836 | INFO | "malloc" : " 14 GB", | 20260506 18:46:51.836 | INFO | "pinned" : " 29 GB" | 20260506 18:46:51.836 | INFO | } | 20260506 18:46:51.837 | INFO | Perf report for the last 5.13579 seconds: | 20260506 18:46:51.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:46:51.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:51.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:51.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:51.837 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 18:46:51.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:51.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:51.837 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:46:51.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:51.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:51.837 | INFO | Uploaded 36GiB to GPU in 1.99024s (18.0883GiB/s) | 20260506 18:46:51.837 | INFO | Downloaded 34GiB to GPU in 1.92176s (17.6922GiB/s) | 20260506 18:46:56.978 | INFO | Analyzing chunk frames = [3618304,3618816) | 20260506 18:46:56.978 | INFO | MemInfo: { | 20260506 18:46:56.978 | INFO | "gpu" : " 17 GB", | 20260506 18:46:56.978 | INFO | "huge" : "", | 20260506 18:46:56.978 | INFO | "huge_shared" : " 27 GB", | 20260506 18:46:56.978 | INFO | "malloc" : " 14 GB", | 20260506 18:46:56.978 | INFO | "pinned" : " 29 GB" | 20260506 18:46:56.978 | INFO | } | 20260506 18:46:56.978 | INFO | Perf report for the last 5.14156 seconds: | 20260506 18:46:56.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:46:56.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:56.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:56.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:56.978 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 18:46:56.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:56.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:56.978 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:46:56.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:56.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:46:56.978 | INFO | Uploaded 36GiB to GPU in 1.9295s (18.6577GiB/s) | 20260506 18:46:56.978 | INFO | Downloaded 34GiB to GPU in 1.86362s (18.2441GiB/s) | 20260506 18:47:02.118 | INFO | Analyzing chunk frames = [3618816,3619328) | 20260506 18:47:02.118 | INFO | MemInfo: { | 20260506 18:47:02.118 | INFO | "gpu" : " 17 GB", | 20260506 18:47:02.118 | INFO | "huge" : "", | 20260506 18:47:02.118 | INFO | "huge_shared" : " 27 GB", | 20260506 18:47:02.118 | INFO | "malloc" : " 14 GB", | 20260506 18:47:02.118 | INFO | "pinned" : " 29 GB" | 20260506 18:47:02.118 | INFO | } | 20260506 18:47:02.119 | INFO | Perf report for the last 5.14047 seconds: | 20260506 18:47:02.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:47:02.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:02.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:02.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:02.119 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260506 18:47:02.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:02.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:02.119 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:47:02.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:02.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:02.119 | INFO | Uploaded 36GiB to GPU in 1.96186s (18.3499GiB/s) | 20260506 18:47:02.119 | INFO | Downloaded 34GiB to GPU in 1.88103s (18.0752GiB/s) | 20260506 18:47:07.261 | INFO | Analyzing chunk frames = [3619328,3619840) | 20260506 18:47:07.261 | INFO | MemInfo: { | 20260506 18:47:07.261 | INFO | "gpu" : " 17 GB", | 20260506 18:47:07.261 | INFO | "huge" : "", | 20260506 18:47:07.261 | INFO | "huge_shared" : " 27 GB", | 20260506 18:47:07.261 | INFO | "malloc" : " 14 GB", | 20260506 18:47:07.261 | INFO | "pinned" : " 29 GB" | 20260506 18:47:07.261 | INFO | } | 20260506 18:47:07.261 | INFO | Perf report for the last 5.14235 seconds: | 20260506 18:47:07.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:47:07.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:07.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:07.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:07.261 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 18:47:07.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:07.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:07.261 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 18:47:07.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:07.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:07.261 | INFO | Uploaded 36GiB to GPU in 2.01818s (17.8379GiB/s) | 20260506 18:47:07.261 | INFO | Downloaded 34GiB to GPU in 1.94472s (17.4832GiB/s) | 20260506 18:47:12.404 | INFO | Analyzing chunk frames = [3619840,3620352) | 20260506 18:47:12.404 | INFO | MemInfo: { | 20260506 18:47:12.404 | INFO | "gpu" : " 17 GB", | 20260506 18:47:12.404 | INFO | "huge" : "", | 20260506 18:47:12.404 | INFO | "huge_shared" : " 27 GB", | 20260506 18:47:12.404 | INFO | "malloc" : " 14 GB", | 20260506 18:47:12.404 | INFO | "pinned" : " 29 GB" | 20260506 18:47:12.404 | INFO | } | 20260506 18:47:12.404 | INFO | Perf report for the last 5.14284 seconds: | 20260506 18:47:12.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:47:12.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:12.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:12.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:12.404 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 18:47:12.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:12.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:12.404 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:47:12.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:12.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:12.404 | INFO | Uploaded 36GiB to GPU in 1.99284s (18.0647GiB/s) | 20260506 18:47:12.404 | INFO | Downloaded 34GiB to GPU in 2.00247s (16.979GiB/s) | 20260506 18:47:17.543 | INFO | Analyzing chunk frames = [3620352,3620864) | 20260506 18:47:17.543 | INFO | MemInfo: { | 20260506 18:47:17.543 | INFO | "gpu" : " 17 GB", | 20260506 18:47:17.543 | INFO | "huge" : "", | 20260506 18:47:17.543 | INFO | "huge_shared" : " 27 GB", | 20260506 18:47:17.543 | INFO | "malloc" : " 14 GB", | 20260506 18:47:17.543 | INFO | "pinned" : " 29 GB" | 20260506 18:47:17.543 | INFO | } | 20260506 18:47:17.543 | INFO | Perf report for the last 5.13922 seconds: | 20260506 18:47:17.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:47:17.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:17.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:17.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:17.543 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:47:17.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:17.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:17.543 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:47:17.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:17.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:17.543 | INFO | Uploaded 36GiB to GPU in 2.00538s (17.9517GiB/s) | 20260506 18:47:17.543 | INFO | Downloaded 34GiB to GPU in 1.99876s (17.0105GiB/s) | 20260506 18:47:17.544 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:47:17.544 | INFO | Total mem usage: 18 GiB | 20260506 18:47:17.544 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:47:17.544 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -11.6%) | 20260506 18:47:17.544 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:47:17.544 | INFO | High Water Breakdown: | 20260506 18:47:17.544 | INFO | 61.9% : 9.99 GiB ( 87.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:47:17.544 | INFO | 38% : 6.12 GiB ( 91.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:47:17.544 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:47:17.544 | INFO | ------------------------------------------ | 20260506 18:47:17.544 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:47:17.544 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:47:17.544 | INFO | Total mem usage: 26.5 GiB | 20260506 18:47:17.544 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:47:17.544 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:47:17.544 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:47:17.544 | INFO | High Water Breakdown: | 20260506 18:47:17.544 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:47:17.544 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:47:17.544 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:47:17.544 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:47:17.544 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:47:17.544 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:47:17.544 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:47:17.544 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:47:17.544 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:47:17.544 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:47:17.544 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:47:17.544 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:47:17.544 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:47:17.544 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:47:17.544 | INFO | ------------------------------------------ | 20260506 18:47:17.544 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:47:17.544 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:47:17.544 | INFO | Total mem usage: 25.6 GiB | 20260506 18:47:17.544 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:47:17.544 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:47:17.544 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:47:17.544 | INFO | High Water Breakdown: | 20260506 18:47:17.544 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:47:17.544 | INFO | ------------------------------------------ | 20260506 18:47:17.544 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:47:17.544 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:47:17.544 | INFO | Total mem usage: 15.7 GiB | 20260506 18:47:17.544 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:47:17.544 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:47:17.544 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:47:17.544 | INFO | High Water Breakdown: | 20260506 18:47:17.544 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:47:17.544 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:47:17.544 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:47:17.544 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:47:17.544 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:47:17.544 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:47:17.544 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:47:17.544 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:47:17.544 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:47:17.544 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:47:17.544 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:47:17.544 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:47:17.545 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:47:17.545 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:47:17.545 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:47:17.545 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:47:17.545 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:47:17.545 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:47:17.545 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:47:17.545 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:47:17.545 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:47:17.545 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:47:17.545 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:47:17.545 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:47:17.545 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:47:17.545 | INFO | ------------------------------------------ | 20260506 18:47:17.545 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:47:17.545 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:47:17.545 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:47:17.545 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:47:17.545 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.55e+01, 1.35e+01 / 2.26e+00 | 20260506 18:47:17.545 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.00e-01, 1.00e-02 / 6.67e-03 | 20260506 18:47:17.545 | INFO | PerfReport: 11.60%, Baseline -- 3.62e+00 / 4.40e+00, 5.47e-01 / 3.52e-01 | 20260506 18:47:17.545 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.45e+00, 1.78e-01 / 1.25e-01 | 20260506 18:47:17.545 | INFO | PerfReport: 16.02%, DME -- 4.00e+01 / 4.47e+01, 4.17e+00 / 2.40e+00 | 20260506 18:47:17.545 | INFO | PerfReport: 57.04%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.39e+00 / 1.47e+00 | 20260506 18:47:17.545 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.26e+00 / 2.84e+00, 3.92e-01 / 1.98e-01 | 20260506 18:47:17.545 | INFO | PerfReport: 3.39%, Metrics -- 1.06e+00 / 1.39e+00, 2.77e-01 / 1.61e-01 | 20260506 18:47:17.545 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.65e-01 / 4.16e-01, 5.69e-02 / 2.43e-02 | 20260506 18:47:17.545 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:47:17.545 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:47:17.545 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:47:17.545 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 18:47:17.545 | INFO | PerfReport: 21.37%, Upload -- 2.07e+01 / 0.00e+00, 6.83e+00 / 0.00e+00 | 20260506 18:47:17.545 | INFO | PerfReport: 21.18%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 18:47:17.545 | INFO | PerfReport: 32.22%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:47:17.545 | INFO | PerfReport: 1.28%, LockUpload -- 1.24e+00 / 0.00e+00, 5.09e+00 / 0.00e+00 | 20260506 18:47:17.545 | INFO | PerfReport: 1.20%, LockDownload -- 1.16e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 18:47:17.545 | INFO | PerfReport: 22.73%, LockCompute -- 2.20e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 18:47:17.545 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260506 18:47:17.659 | INFO | Starting a round of Baz IO | 20260506 18:47:20.788 | INFO | Populating RTMetrics report | 20260506 18:47:20.789 | INFO | RTMetrics report populated | 20260506 18:47:20.789 | INFO | Opening tmp RTMetrics json file | 20260506 18:47:20.816 | INFO | Serializing RTMetrics to json | 20260506 18:47:20.822 | INFO | Serialization done, swapping tmp file now... | 20260506 18:47:20.828 | INFO | RTMetrics done | 20260506 18:47:21.040 | INFO | Finished a round of Baz IO: Wrote 2.027 GiB in 3.38 seconds | 20260506 18:47:21.040 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.91% / 0.01% / 65.08% | 20260506 18:47:21.040 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.04314% / 1.58946e-05% / 7.94729e-05% | 20260506 18:47:21.040 | INFO | Baz IO overall: | 20260506 18:47:21.040 | INFO | BytesWritten : 1913285696990 | 20260506 18:47:21.040 | INFO | NumEvents : 925157263712 | 20260506 18:47:22.683 | INFO | Analyzing chunk frames = [3620864,3621376) | 20260506 18:47:22.683 | INFO | MemInfo: { | 20260506 18:47:22.683 | INFO | "gpu" : " 17 GB", | 20260506 18:47:22.683 | INFO | "huge" : "", | 20260506 18:47:22.683 | INFO | "huge_shared" : " 27 GB", | 20260506 18:47:22.683 | INFO | "malloc" : " 14 GB", | 20260506 18:47:22.683 | INFO | "pinned" : " 29 GB" | 20260506 18:47:22.683 | INFO | } | 20260506 18:47:22.683 | INFO | Perf report for the last 5.14043 seconds: | 20260506 18:47:22.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:47:22.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:22.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:22.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:22.684 | INFO | ANALYSIS : 62.6%, 2.9, 95.0ms, 0 | 20260506 18:47:22.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:22.684 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 18:47:22.684 | INFO | PRE_HQ : 60.1%, 1.9, 58.0ms, 0 | 20260506 18:47:22.684 | INFO | BAZWRITER : 65.5%, 4.8, 161.1ms, 0 | 20260506 18:47:22.684 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 18:47:22.684 | INFO | Uploaded 36GiB to GPU in 2.06673s (17.4188GiB/s) | 20260506 18:47:22.684 | INFO | Downloaded 38GiB to GPU in 2.21108s (17.1862GiB/s) | 20260506 18:47:27.814 | INFO | Analyzing chunk frames = [3621376,3621888) | 20260506 18:47:27.814 | INFO | MemInfo: { | 20260506 18:47:27.814 | INFO | "gpu" : " 17 GB", | 20260506 18:47:27.814 | INFO | "huge" : "", | 20260506 18:47:27.814 | INFO | "huge_shared" : " 27 GB", | 20260506 18:47:27.814 | INFO | "malloc" : " 14 GB", | 20260506 18:47:27.814 | INFO | "pinned" : " 29 GB" | 20260506 18:47:27.814 | INFO | } | 20260506 18:47:27.814 | INFO | Perf report for the last 5.13096 seconds: | 20260506 18:47:27.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:47:27.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:27.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:27.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:27.814 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 18:47:27.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:27.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:27.815 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:47:27.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:27.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:27.815 | INFO | Uploaded 36GiB to GPU in 1.99392s (18.0549GiB/s) | 20260506 18:47:27.815 | INFO | Downloaded 34GiB to GPU in 1.93998s (17.5259GiB/s) | 20260506 18:47:32.952 | INFO | Analyzing chunk frames = [3621888,3622400) | 20260506 18:47:32.952 | INFO | MemInfo: { | 20260506 18:47:32.952 | INFO | "gpu" : " 17 GB", | 20260506 18:47:32.952 | INFO | "huge" : "", | 20260506 18:47:32.953 | INFO | "huge_shared" : " 27 GB", | 20260506 18:47:32.953 | INFO | "malloc" : " 14 GB", | 20260506 18:47:32.953 | INFO | "pinned" : " 29 GB" | 20260506 18:47:32.953 | INFO | } | 20260506 18:47:32.953 | INFO | Perf report for the last 5.13827 seconds: | 20260506 18:47:32.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:47:32.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:32.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:32.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:32.953 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:47:32.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:32.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:32.953 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:47:32.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:32.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:32.953 | INFO | Uploaded 36GiB to GPU in 1.97281s (18.248GiB/s) | 20260506 18:47:32.953 | INFO | Downloaded 34GiB to GPU in 1.92435s (17.6683GiB/s) | 20260506 18:47:38.097 | INFO | Analyzing chunk frames = [3622400,3622912) | 20260506 18:47:38.097 | INFO | MemInfo: { | 20260506 18:47:38.097 | INFO | "gpu" : " 17 GB", | 20260506 18:47:38.097 | INFO | "huge" : "", | 20260506 18:47:38.097 | INFO | "huge_shared" : " 27 GB", | 20260506 18:47:38.097 | INFO | "malloc" : " 14 GB", | 20260506 18:47:38.097 | INFO | "pinned" : " 29 GB" | 20260506 18:47:38.097 | INFO | } | 20260506 18:47:38.097 | INFO | Perf report for the last 5.14471 seconds: | 20260506 18:47:38.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:47:38.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:38.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:38.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:38.097 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 18:47:38.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:38.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:38.098 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:47:38.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:38.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:38.098 | INFO | Uploaded 36GiB to GPU in 1.9219s (18.7315GiB/s) | 20260506 18:47:38.098 | INFO | Downloaded 34GiB to GPU in 1.86833s (18.198GiB/s) | 20260506 18:47:43.237 | INFO | Analyzing chunk frames = [3622912,3623424) | 20260506 18:47:43.237 | INFO | MemInfo: { | 20260506 18:47:43.237 | INFO | "gpu" : " 17 GB", | 20260506 18:47:43.237 | INFO | "huge" : "", | 20260506 18:47:43.237 | INFO | "huge_shared" : " 27 GB", | 20260506 18:47:43.237 | INFO | "malloc" : " 14 GB", | 20260506 18:47:43.237 | INFO | "pinned" : " 29 GB" | 20260506 18:47:43.237 | INFO | } | 20260506 18:47:43.238 | INFO | Perf report for the last 5.14007 seconds: | 20260506 18:47:43.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:47:43.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:43.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:43.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:43.238 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 18:47:43.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:43.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:43.238 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:47:43.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:43.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:43.238 | INFO | Uploaded 36GiB to GPU in 2.00918s (17.9178GiB/s) | 20260506 18:47:43.238 | INFO | Downloaded 34GiB to GPU in 1.90755s (17.8239GiB/s) | 20260506 18:47:48.380 | INFO | Analyzing chunk frames = [3623424,3623936) | 20260506 18:47:48.380 | INFO | MemInfo: { | 20260506 18:47:48.380 | INFO | "gpu" : " 17 GB", | 20260506 18:47:48.380 | INFO | "huge" : "", | 20260506 18:47:48.380 | INFO | "huge_shared" : " 27 GB", | 20260506 18:47:48.380 | INFO | "malloc" : " 14 GB", | 20260506 18:47:48.380 | INFO | "pinned" : " 29 GB" | 20260506 18:47:48.380 | INFO | } | 20260506 18:47:48.380 | INFO | Perf report for the last 5.1423 seconds: | 20260506 18:47:48.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:47:48.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:48.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:48.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:48.380 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 18:47:48.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:48.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:48.380 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:47:48.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:48.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:48.380 | INFO | Uploaded 36GiB to GPU in 2.01936s (17.8275GiB/s) | 20260506 18:47:48.380 | INFO | Downloaded 34GiB to GPU in 1.95714s (17.3723GiB/s) | 20260506 18:47:53.514 | INFO | Analyzing chunk frames = [3623936,3624448) | 20260506 18:47:53.514 | INFO | MemInfo: { | 20260506 18:47:53.514 | INFO | "gpu" : " 17 GB", | 20260506 18:47:53.514 | INFO | "huge" : "", | 20260506 18:47:53.514 | INFO | "huge_shared" : " 27 GB", | 20260506 18:47:53.514 | INFO | "malloc" : " 14 GB", | 20260506 18:47:53.514 | INFO | "pinned" : " 29 GB" | 20260506 18:47:53.514 | INFO | } | 20260506 18:47:53.514 | INFO | Perf report for the last 5.13408 seconds: | 20260506 18:47:53.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:47:53.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:53.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:53.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:53.514 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 18:47:53.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:53.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:53.514 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:47:53.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:53.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:53.514 | INFO | Uploaded 36GiB to GPU in 1.9802s (18.1799GiB/s) | 20260506 18:47:53.514 | INFO | Downloaded 34GiB to GPU in 1.99862s (17.0117GiB/s) | 20260506 18:47:58.652 | INFO | Analyzing chunk frames = [3624448,3624960) | 20260506 18:47:58.652 | INFO | MemInfo: { | 20260506 18:47:58.652 | INFO | "gpu" : " 17 GB", | 20260506 18:47:58.652 | INFO | "huge" : "", | 20260506 18:47:58.652 | INFO | "huge_shared" : " 27 GB", | 20260506 18:47:58.652 | INFO | "malloc" : " 14 GB", | 20260506 18:47:58.652 | INFO | "pinned" : " 29 GB" | 20260506 18:47:58.652 | INFO | } | 20260506 18:47:58.652 | INFO | Perf report for the last 5.13807 seconds: | 20260506 18:47:58.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:47:58.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:58.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:58.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:58.652 | INFO | ANALYSIS : 62.5%, 3.0, 97.2ms, 0 | 20260506 18:47:58.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:58.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:58.652 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 18:47:58.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:58.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:47:58.652 | INFO | Uploaded 36GiB to GPU in 2.08662s (17.2528GiB/s) | 20260506 18:47:58.652 | INFO | Downloaded 34GiB to GPU in 2.0269s (16.7744GiB/s) | 20260506 18:47:58.766 | INFO | Starting a round of Baz IO | 20260506 18:48:01.961 | INFO | Populating RTMetrics report | 20260506 18:48:01.962 | INFO | RTMetrics report populated | 20260506 18:48:01.962 | INFO | Opening tmp RTMetrics json file | 20260506 18:48:01.977 | INFO | Serializing RTMetrics to json | 20260506 18:48:01.981 | INFO | Serialization done, swapping tmp file now... | 20260506 18:48:01.986 | INFO | RTMetrics done | 20260506 18:48:02.271 | INFO | Finished a round of Baz IO: Wrote 2.027 GiB in 3.504 seconds | 20260506 18:48:02.271 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.91% / 0.00% / 65.09% | 20260506 18:48:02.271 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.04172% / 7.94729e-06% / 5.5631e-05% | 20260506 18:48:02.271 | INFO | Baz IO overall: | 20260506 18:48:02.271 | INFO | BytesWritten : 1915461946760 | 20260506 18:48:02.271 | INFO | NumEvents : 926204729245 | 20260506 18:48:03.789 | INFO | Analyzing chunk frames = [3624960,3625472) | 20260506 18:48:03.789 | INFO | MemInfo: { | 20260506 18:48:03.789 | INFO | "gpu" : " 17 GB", | 20260506 18:48:03.789 | INFO | "huge" : "", | 20260506 18:48:03.789 | INFO | "huge_shared" : " 27 GB", | 20260506 18:48:03.789 | INFO | "malloc" : " 14 GB", | 20260506 18:48:03.789 | INFO | "pinned" : " 29 GB" | 20260506 18:48:03.789 | INFO | } | 20260506 18:48:03.790 | INFO | Perf report for the last 5.13753 seconds: | 20260506 18:48:03.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:48:03.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:03.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:03.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:03.790 | INFO | ANALYSIS : 63.2%, 2.8, 94.2ms, 0 | 20260506 18:48:03.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:03.790 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260506 18:48:03.790 | INFO | PRE_HQ : 61.1%, 2.0, 63.3ms, 0 | 20260506 18:48:03.790 | INFO | BAZWRITER : 67.6%, 4.8, 167.7ms, 0 | 20260506 18:48:03.790 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 18:48:03.790 | INFO | Uploaded 36GiB to GPU in 2.05921s (17.4824GiB/s) | 20260506 18:48:03.790 | INFO | Downloaded 38GiB to GPU in 2.19912s (17.2797GiB/s) | 20260506 18:48:08.929 | INFO | Analyzing chunk frames = [3625472,3625984) | 20260506 18:48:08.929 | INFO | MemInfo: { | 20260506 18:48:08.929 | INFO | "gpu" : " 17 GB", | 20260506 18:48:08.929 | INFO | "huge" : "", | 20260506 18:48:08.929 | INFO | "huge_shared" : " 27 GB", | 20260506 18:48:08.929 | INFO | "malloc" : " 14 GB", | 20260506 18:48:08.929 | INFO | "pinned" : " 29 GB" | 20260506 18:48:08.929 | INFO | } | 20260506 18:48:08.930 | INFO | Perf report for the last 5.13998 seconds: | 20260506 18:48:08.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:48:08.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:08.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:08.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:08.930 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 18:48:08.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:08.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:08.930 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 18:48:08.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:08.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:08.930 | INFO | Uploaded 36GiB to GPU in 1.99817s (18.0165GiB/s) | 20260506 18:48:08.930 | INFO | Downloaded 34GiB to GPU in 1.94327s (17.4963GiB/s) | 20260506 18:48:14.069 | INFO | Analyzing chunk frames = [3625984,3626496) | 20260506 18:48:14.069 | INFO | MemInfo: { | 20260506 18:48:14.069 | INFO | "gpu" : " 17 GB", | 20260506 18:48:14.069 | INFO | "huge" : "", | 20260506 18:48:14.069 | INFO | "huge_shared" : " 27 GB", | 20260506 18:48:14.069 | INFO | "malloc" : " 14 GB", | 20260506 18:48:14.069 | INFO | "pinned" : " 29 GB" | 20260506 18:48:14.069 | INFO | } | 20260506 18:48:14.069 | INFO | Perf report for the last 5.13963 seconds: | 20260506 18:48:14.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:48:14.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:14.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:14.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:14.069 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:48:14.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:14.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:14.069 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:48:14.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:14.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:14.069 | INFO | Uploaded 36GiB to GPU in 1.97533s (18.2248GiB/s) | 20260506 18:48:14.070 | INFO | Downloaded 34GiB to GPU in 1.91966s (17.7115GiB/s) | 20260506 18:48:19.209 | INFO | Analyzing chunk frames = [3626496,3627008) | 20260506 18:48:19.209 | INFO | MemInfo: { | 20260506 18:48:19.209 | INFO | "gpu" : " 17 GB", | 20260506 18:48:19.209 | INFO | "huge" : "", | 20260506 18:48:19.209 | INFO | "huge_shared" : " 27 GB", | 20260506 18:48:19.209 | INFO | "malloc" : " 14 GB", | 20260506 18:48:19.209 | INFO | "pinned" : " 29 GB" | 20260506 18:48:19.209 | INFO | } | 20260506 18:48:19.209 | INFO | Perf report for the last 5.13986 seconds: | 20260506 18:48:19.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:48:19.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:19.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:19.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:19.209 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 18:48:19.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:19.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:19.209 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:48:19.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:19.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:19.209 | INFO | Uploaded 36GiB to GPU in 1.93428s (18.6115GiB/s) | 20260506 18:48:19.209 | INFO | Downloaded 34GiB to GPU in 1.87393s (18.1437GiB/s) | 20260506 18:48:24.356 | INFO | Analyzing chunk frames = [3627008,3627520) | 20260506 18:48:24.356 | INFO | MemInfo: { | 20260506 18:48:24.356 | INFO | "gpu" : " 17 GB", | 20260506 18:48:24.356 | INFO | "huge" : "", | 20260506 18:48:24.356 | INFO | "huge_shared" : " 27 GB", | 20260506 18:48:24.356 | INFO | "malloc" : " 14 GB", | 20260506 18:48:24.356 | INFO | "pinned" : " 29 GB" | 20260506 18:48:24.356 | INFO | } | 20260506 18:48:24.356 | INFO | Perf report for the last 5.14736 seconds: | 20260506 18:48:24.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:48:24.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:24.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:24.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:24.357 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 18:48:24.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:24.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:24.357 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:48:24.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:24.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:24.357 | INFO | Uploaded 36GiB to GPU in 2.01767s (17.8423GiB/s) | 20260506 18:48:24.357 | INFO | Downloaded 34GiB to GPU in 1.91234s (17.7793GiB/s) | 20260506 18:48:29.491 | INFO | Analyzing chunk frames = [3627520,3628032) | 20260506 18:48:29.491 | INFO | MemInfo: { | 20260506 18:48:29.491 | INFO | "gpu" : " 17 GB", | 20260506 18:48:29.491 | INFO | "huge" : "", | 20260506 18:48:29.491 | INFO | "huge_shared" : " 27 GB", | 20260506 18:48:29.491 | INFO | "malloc" : " 14 GB", | 20260506 18:48:29.492 | INFO | "pinned" : " 29 GB" | 20260506 18:48:29.492 | INFO | } | 20260506 18:48:29.492 | INFO | Perf report for the last 5.13522 seconds: | 20260506 18:48:29.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:48:29.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:29.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:29.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:29.492 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 18:48:29.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:29.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:29.492 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:48:29.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:29.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:29.492 | INFO | Uploaded 36GiB to GPU in 1.9889s (18.1004GiB/s) | 20260506 18:48:29.492 | INFO | Downloaded 34GiB to GPU in 1.95717s (17.372GiB/s) | 20260506 18:48:34.632 | INFO | Analyzing chunk frames = [3628032,3628544) | 20260506 18:48:34.632 | INFO | MemInfo: { | 20260506 18:48:34.632 | INFO | "gpu" : " 17 GB", | 20260506 18:48:34.632 | INFO | "huge" : "", | 20260506 18:48:34.632 | INFO | "huge_shared" : " 27 GB", | 20260506 18:48:34.632 | INFO | "malloc" : " 14 GB", | 20260506 18:48:34.632 | INFO | "pinned" : " 29 GB" | 20260506 18:48:34.632 | INFO | } | 20260506 18:48:34.632 | INFO | Perf report for the last 5.14028 seconds: | 20260506 18:48:34.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:48:34.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:34.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:34.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:34.632 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 18:48:34.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:34.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:34.632 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:48:34.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:34.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:34.632 | INFO | Uploaded 36GiB to GPU in 1.9769s (18.2103GiB/s) | 20260506 18:48:34.632 | INFO | Downloaded 34GiB to GPU in 1.99865s (17.0115GiB/s) | 20260506 18:48:39.769 | INFO | Analyzing chunk frames = [3628544,3629056) | 20260506 18:48:39.769 | INFO | MemInfo: { | 20260506 18:48:39.769 | INFO | "gpu" : " 17 GB", | 20260506 18:48:39.769 | INFO | "huge" : "", | 20260506 18:48:39.769 | INFO | "huge_shared" : " 27 GB", | 20260506 18:48:39.769 | INFO | "malloc" : " 14 GB", | 20260506 18:48:39.769 | INFO | "pinned" : " 29 GB" | 20260506 18:48:39.769 | INFO | } | 20260506 18:48:39.769 | INFO | Perf report for the last 5.13745 seconds: | 20260506 18:48:39.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:48:39.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:39.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:39.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:39.769 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 18:48:39.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:39.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:39.770 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:48:39.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:39.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:39.770 | INFO | Uploaded 36GiB to GPU in 2.00755s (17.9323GiB/s) | 20260506 18:48:39.770 | INFO | Downloaded 34GiB to GPU in 1.98795s (17.103GiB/s) | 20260506 18:48:39.770 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:48:39.770 | INFO | Total mem usage: 18 GiB | 20260506 18:48:39.770 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:48:39.770 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.5%) | 20260506 18:48:39.770 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:48:39.770 | INFO | High Water Breakdown: | 20260506 18:48:39.770 | INFO | 61.4% : 9.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:48:39.770 | INFO | 38.5% : 6 GiB ( 89.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:48:39.770 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:48:39.770 | INFO | ------------------------------------------ | 20260506 18:48:39.770 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:48:39.770 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:48:39.770 | INFO | Total mem usage: 26.5 GiB | 20260506 18:48:39.770 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:48:39.770 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260506 18:48:39.770 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260506 18:48:39.770 | INFO | High Water Breakdown: | 20260506 18:48:39.770 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:48:39.770 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:48:39.770 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:48:39.770 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:48:39.770 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:48:39.770 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:48:39.770 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:48:39.770 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:48:39.770 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:48:39.770 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:48:39.770 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:48:39.770 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:48:39.770 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:48:39.770 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:48:39.770 | INFO | ------------------------------------------ | 20260506 18:48:39.770 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:48:39.770 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:48:39.770 | INFO | Total mem usage: 25.6 GiB | 20260506 18:48:39.770 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:48:39.770 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:48:39.770 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:48:39.770 | INFO | High Water Breakdown: | 20260506 18:48:39.770 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:48:39.770 | INFO | ------------------------------------------ | 20260506 18:48:39.770 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:48:39.770 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:48:39.770 | INFO | Total mem usage: 15.7 GiB | 20260506 18:48:39.770 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:48:39.770 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:48:39.770 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:48:39.770 | INFO | High Water Breakdown: | 20260506 18:48:39.770 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:48:39.770 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:48:39.770 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:48:39.770 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:48:39.770 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:48:39.770 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:48:39.770 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:48:39.770 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:48:39.770 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:48:39.770 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:48:39.770 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:48:39.770 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:48:39.770 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:48:39.770 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:48:39.770 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:48:39.770 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:48:39.770 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:48:39.770 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:48:39.770 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:48:39.770 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:48:39.770 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:48:39.770 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:48:39.770 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:48:39.770 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:48:39.770 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:48:39.771 | INFO | ------------------------------------------ | 20260506 18:48:39.771 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:48:39.771 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:48:39.771 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:48:39.771 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:48:39.771 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.56e+01, 1.35e+01 / 2.63e+00 | 20260506 18:48:39.771 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.01e-01, 1.09e-02 / 5.31e-03 | 20260506 18:48:39.771 | INFO | PerfReport: 11.60%, Baseline -- 3.63e+00 / 4.36e+00, 5.50e-01 / 3.50e-01 | 20260506 18:48:39.771 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.83e-01 / 1.24e-01 | 20260506 18:48:39.771 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.44e+01, 4.11e+00 / 3.00e+00 | 20260506 18:48:39.771 | INFO | PerfReport: 57.08%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.42e+00 / 2.14e+00 | 20260506 18:48:39.771 | INFO | PerfReport: 7.23%, PulseAccumulating -- 2.26e+00 / 2.77e+00, 3.86e-01 / 2.61e-01 | 20260506 18:48:39.771 | INFO | PerfReport: 3.38%, Metrics -- 1.06e+00 / 1.34e+00, 2.77e-01 / 1.63e-01 | 20260506 18:48:39.771 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.66e-01 / 4.11e-01, 5.80e-02 / 2.94e-02 | 20260506 18:48:39.771 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:48:39.771 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:48:39.771 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:48:39.771 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 18:48:39.771 | INFO | PerfReport: 21.50%, Upload -- 2.08e+01 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260506 18:48:39.771 | INFO | PerfReport: 21.24%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 18:48:39.771 | INFO | PerfReport: 32.25%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:48:39.771 | INFO | PerfReport: 1.32%, LockUpload -- 1.28e+00 / 0.00e+00, 5.34e+00 / 0.00e+00 | 20260506 18:48:39.771 | INFO | PerfReport: 1.21%, LockDownload -- 1.17e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 18:48:39.771 | INFO | PerfReport: 22.46%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 18:48:39.771 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260506 18:48:39.884 | INFO | Starting a round of Baz IO | 20260506 18:48:43.018 | INFO | Populating RTMetrics report | 20260506 18:48:43.019 | INFO | RTMetrics report populated | 20260506 18:48:43.019 | INFO | Opening tmp RTMetrics json file | 20260506 18:48:43.022 | INFO | Serializing RTMetrics to json | 20260506 18:48:43.025 | INFO | Serialization done, swapping tmp file now... | 20260506 18:48:43.026 | INFO | RTMetrics done | 20260506 18:48:43.278 | INFO | Finished a round of Baz IO: Wrote 2.026 GiB in 3.393 seconds | 20260506 18:48:43.278 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.90% / 0.01% / 65.09% | 20260506 18:48:43.278 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.03809% / 7.94729e-06% / 6.35783e-05% | 20260506 18:48:43.278 | INFO | Baz IO overall: | 20260506 18:48:43.278 | INFO | BytesWritten : 1917637677854 | 20260506 18:48:43.278 | INFO | NumEvents : 927251616128 | 20260506 18:48:44.903 | INFO | Analyzing chunk frames = [3629056,3629568) | 20260506 18:48:44.903 | INFO | MemInfo: { | 20260506 18:48:44.903 | INFO | "gpu" : " 17 GB", | 20260506 18:48:44.903 | INFO | "huge" : "", | 20260506 18:48:44.903 | INFO | "huge_shared" : " 27 GB", | 20260506 18:48:44.903 | INFO | "malloc" : " 14 GB", | 20260506 18:48:44.903 | INFO | "pinned" : " 29 GB" | 20260506 18:48:44.903 | INFO | } | 20260506 18:48:44.903 | INFO | Perf report for the last 5.13396 seconds: | 20260506 18:48:44.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:48:44.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:44.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:44.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:44.904 | INFO | ANALYSIS : 62.7%, 2.9, 95.2ms, 0 | 20260506 18:48:44.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:44.904 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260506 18:48:44.904 | INFO | PRE_HQ : 60.2%, 1.9, 58.2ms, 0 | 20260506 18:48:44.904 | INFO | BAZWRITER : 65.8%, 4.8, 161.6ms, 0 | 20260506 18:48:44.904 | INFO | BAZ_SAMPLING : 4.1%, 1.0, 2.1ms, 0 | 20260506 18:48:44.904 | INFO | Uploaded 36GiB to GPU in 2.03767s (17.6672GiB/s) | 20260506 18:48:44.904 | INFO | Downloaded 38GiB to GPU in 2.20329s (17.247GiB/s) | 20260506 18:48:50.043 | INFO | Analyzing chunk frames = [3629568,3630080) | 20260506 18:48:50.044 | INFO | MemInfo: { | 20260506 18:48:50.044 | INFO | "gpu" : " 17 GB", | 20260506 18:48:50.044 | INFO | "huge" : "", | 20260506 18:48:50.044 | INFO | "huge_shared" : " 27 GB", | 20260506 18:48:50.044 | INFO | "malloc" : " 14 GB", | 20260506 18:48:50.044 | INFO | "pinned" : " 29 GB" | 20260506 18:48:50.044 | INFO | } | 20260506 18:48:50.044 | INFO | Perf report for the last 5.14039 seconds: | 20260506 18:48:50.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:48:50.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:50.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:50.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:50.044 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 18:48:50.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:50.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:50.044 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:48:50.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:50.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:50.044 | INFO | Uploaded 36GiB to GPU in 1.99893s (18.0096GiB/s) | 20260506 18:48:50.044 | INFO | Downloaded 34GiB to GPU in 1.9279s (17.6358GiB/s) | 20260506 18:48:55.186 | INFO | Analyzing chunk frames = [3630080,3630592) | 20260506 18:48:55.186 | INFO | MemInfo: { | 20260506 18:48:55.186 | INFO | "gpu" : " 17 GB", | 20260506 18:48:55.186 | INFO | "huge" : "", | 20260506 18:48:55.186 | INFO | "huge_shared" : " 27 GB", | 20260506 18:48:55.186 | INFO | "malloc" : " 14 GB", | 20260506 18:48:55.186 | INFO | "pinned" : " 29 GB" | 20260506 18:48:55.186 | INFO | } | 20260506 18:48:55.186 | INFO | Perf report for the last 5.14204 seconds: | 20260506 18:48:55.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:48:55.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:55.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:55.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:55.186 | INFO | ANALYSIS : 60.9%, 3.0, 94.4ms, 0 | 20260506 18:48:55.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:55.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:55.186 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:48:55.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:55.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:48:55.186 | INFO | Uploaded 36GiB to GPU in 1.99005s (18.09GiB/s) | 20260506 18:48:55.186 | INFO | Downloaded 34GiB to GPU in 1.91963s (17.7118GiB/s) | 20260506 18:49:00.321 | INFO | Analyzing chunk frames = [3630592,3631104) | 20260506 18:49:00.321 | INFO | MemInfo: { | 20260506 18:49:00.321 | INFO | "gpu" : " 17 GB", | 20260506 18:49:00.321 | INFO | "huge" : "", | 20260506 18:49:00.321 | INFO | "huge_shared" : " 27 GB", | 20260506 18:49:00.321 | INFO | "malloc" : " 14 GB", | 20260506 18:49:00.321 | INFO | "pinned" : " 29 GB" | 20260506 18:49:00.321 | INFO | } | 20260506 18:49:00.321 | INFO | Perf report for the last 5.13559 seconds: | 20260506 18:49:00.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:49:00.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:00.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:00.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:00.322 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 18:49:00.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:00.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:00.322 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:49:00.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:00.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:00.322 | INFO | Uploaded 36GiB to GPU in 1.93523s (18.6024GiB/s) | 20260506 18:49:00.322 | INFO | Downloaded 34GiB to GPU in 1.87604s (18.1233GiB/s) | 20260506 18:49:05.465 | INFO | Analyzing chunk frames = [3631104,3631616) | 20260506 18:49:05.465 | INFO | MemInfo: { | 20260506 18:49:05.465 | INFO | "gpu" : " 17 GB", | 20260506 18:49:05.465 | INFO | "huge" : "", | 20260506 18:49:05.465 | INFO | "huge_shared" : " 27 GB", | 20260506 18:49:05.466 | INFO | "malloc" : " 14 GB", | 20260506 18:49:05.466 | INFO | "pinned" : " 29 GB" | 20260506 18:49:05.466 | INFO | } | 20260506 18:49:05.466 | INFO | Perf report for the last 5.14427 seconds: | 20260506 18:49:05.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:49:05.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:05.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:05.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:05.466 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260506 18:49:05.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:05.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:05.466 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:49:05.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:05.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:05.466 | INFO | Uploaded 36GiB to GPU in 1.93603s (18.5947GiB/s) | 20260506 18:49:05.466 | INFO | Downloaded 34GiB to GPU in 1.86271s (18.2529GiB/s) | 20260506 18:49:10.605 | INFO | Analyzing chunk frames = [3631616,3632128) | 20260506 18:49:10.605 | INFO | MemInfo: { | 20260506 18:49:10.605 | INFO | "gpu" : " 17 GB", | 20260506 18:49:10.605 | INFO | "huge" : "", | 20260506 18:49:10.605 | INFO | "huge_shared" : " 27 GB", | 20260506 18:49:10.605 | INFO | "malloc" : " 14 GB", | 20260506 18:49:10.605 | INFO | "pinned" : " 29 GB" | 20260506 18:49:10.605 | INFO | } | 20260506 18:49:10.605 | INFO | Perf report for the last 5.13931 seconds: | 20260506 18:49:10.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:49:10.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:10.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:10.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:10.605 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 18:49:10.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:10.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:10.605 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:49:10.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:10.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:10.605 | INFO | Uploaded 36GiB to GPU in 1.99313s (18.062GiB/s) | 20260506 18:49:10.605 | INFO | Downloaded 34GiB to GPU in 1.92791s (17.6357GiB/s) | 20260506 18:49:15.745 | INFO | Analyzing chunk frames = [3632128,3632640) | 20260506 18:49:15.745 | INFO | MemInfo: { | 20260506 18:49:15.745 | INFO | "gpu" : " 17 GB", | 20260506 18:49:15.745 | INFO | "huge" : "", | 20260506 18:49:15.745 | INFO | "huge_shared" : " 27 GB", | 20260506 18:49:15.745 | INFO | "malloc" : " 14 GB", | 20260506 18:49:15.745 | INFO | "pinned" : " 29 GB" | 20260506 18:49:15.745 | INFO | } | 20260506 18:49:15.745 | INFO | Perf report for the last 5.14005 seconds: | 20260506 18:49:15.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:49:15.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:15.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:15.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:15.745 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:49:15.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:15.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:15.745 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 18:49:15.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:15.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:15.745 | INFO | Uploaded 36GiB to GPU in 1.99403s (18.0539GiB/s) | 20260506 18:49:15.745 | INFO | Downloaded 34GiB to GPU in 2.01042s (16.9119GiB/s) | 20260506 18:49:20.880 | INFO | Analyzing chunk frames = [3632640,3633152) | 20260506 18:49:20.880 | INFO | MemInfo: { | 20260506 18:49:20.880 | INFO | "gpu" : " 17 GB", | 20260506 18:49:20.880 | INFO | "huge" : "", | 20260506 18:49:20.880 | INFO | "huge_shared" : " 27 GB", | 20260506 18:49:20.880 | INFO | "malloc" : " 14 GB", | 20260506 18:49:20.880 | INFO | "pinned" : " 29 GB" | 20260506 18:49:20.880 | INFO | } | 20260506 18:49:20.880 | INFO | Perf report for the last 5.13537 seconds: | 20260506 18:49:20.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:49:20.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:20.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:20.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:20.881 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 18:49:20.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:20.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:20.881 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 18:49:20.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:20.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:20.881 | INFO | Uploaded 36GiB to GPU in 2.04615s (17.594GiB/s) | 20260506 18:49:20.881 | INFO | Downloaded 34GiB to GPU in 2.00301s (16.9744GiB/s) | 20260506 18:49:20.999 | INFO | Starting a round of Baz IO | 20260506 18:49:24.161 | INFO | Populating RTMetrics report | 20260506 18:49:24.161 | INFO | RTMetrics report populated | 20260506 18:49:24.161 | INFO | Opening tmp RTMetrics json file | 20260506 18:49:24.176 | INFO | Serializing RTMetrics to json | 20260506 18:49:24.216 | INFO | Serialization done, swapping tmp file now... | 20260506 18:49:24.221 | INFO | RTMetrics done | 20260506 18:49:24.839 | INFO | Finished a round of Baz IO: Wrote 2.026 GiB in 3.839 seconds | 20260506 18:49:24.839 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.90% / 0.01% / 65.10% | 20260506 18:49:24.839 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.03552% / 1.19209e-05% / 8.74201e-05% | 20260506 18:49:24.839 | INFO | Baz IO overall: | 20260506 18:49:24.839 | INFO | BytesWritten : 1919812712532 | 20260506 18:49:24.839 | INFO | NumEvents : 928298269029 | 20260506 18:49:26.024 | INFO | Analyzing chunk frames = [3633152,3633664) | 20260506 18:49:26.024 | INFO | MemInfo: { | 20260506 18:49:26.024 | INFO | "gpu" : " 17 GB", | 20260506 18:49:26.024 | INFO | "huge" : "", | 20260506 18:49:26.024 | INFO | "huge_shared" : " 27 GB", | 20260506 18:49:26.024 | INFO | "malloc" : " 14 GB", | 20260506 18:49:26.024 | INFO | "pinned" : " 29 GB" | 20260506 18:49:26.024 | INFO | } | 20260506 18:49:26.024 | INFO | Perf report for the last 5.14371 seconds: | 20260506 18:49:26.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:49:26.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:26.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:26.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:26.024 | INFO | ANALYSIS : 62.7%, 2.9, 95.8ms, 0 | 20260506 18:49:26.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:26.024 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260506 18:49:26.024 | INFO | PRE_HQ : 60.4%, 1.9, 59.4ms, 0 | 20260506 18:49:26.024 | INFO | BAZWRITER : 74.3%, 4.8, 184.4ms, 0 | 20260506 18:49:26.024 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 18:49:26.024 | INFO | Uploaded 36GiB to GPU in 2.04326s (17.6189GiB/s) | 20260506 18:49:26.025 | INFO | Downloaded 38GiB to GPU in 2.23022s (17.0387GiB/s) | 20260506 18:49:31.160 | INFO | Analyzing chunk frames = [3633664,3634176) | 20260506 18:49:31.160 | INFO | MemInfo: { | 20260506 18:49:31.160 | INFO | "gpu" : " 17 GB", | 20260506 18:49:31.160 | INFO | "huge" : "", | 20260506 18:49:31.160 | INFO | "huge_shared" : " 27 GB", | 20260506 18:49:31.160 | INFO | "malloc" : " 14 GB", | 20260506 18:49:31.160 | INFO | "pinned" : " 29 GB" | 20260506 18:49:31.160 | INFO | } | 20260506 18:49:31.160 | INFO | Perf report for the last 5.13627 seconds: | 20260506 18:49:31.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:49:31.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:31.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:31.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:31.161 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:49:31.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:31.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:31.161 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 18:49:31.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:31.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:31.161 | INFO | Uploaded 36GiB to GPU in 2.00328s (17.9705GiB/s) | 20260506 18:49:31.161 | INFO | Downloaded 34GiB to GPU in 1.92494s (17.6629GiB/s) | 20260506 18:49:36.301 | INFO | Analyzing chunk frames = [3634176,3634688) | 20260506 18:49:36.301 | INFO | MemInfo: { | 20260506 18:49:36.301 | INFO | "gpu" : " 17 GB", | 20260506 18:49:36.301 | INFO | "huge" : "", | 20260506 18:49:36.301 | INFO | "huge_shared" : " 27 GB", | 20260506 18:49:36.301 | INFO | "malloc" : " 14 GB", | 20260506 18:49:36.301 | INFO | "pinned" : " 29 GB" | 20260506 18:49:36.301 | INFO | } | 20260506 18:49:36.301 | INFO | Perf report for the last 5.14077 seconds: | 20260506 18:49:36.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:49:36.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:36.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:36.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:36.301 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 18:49:36.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:36.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:36.301 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:49:36.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:36.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:36.302 | INFO | Uploaded 36GiB to GPU in 1.96139s (18.3544GiB/s) | 20260506 18:49:36.302 | INFO | Downloaded 34GiB to GPU in 1.92274s (17.6831GiB/s) | 20260506 18:49:41.447 | INFO | Analyzing chunk frames = [3634688,3635200) | 20260506 18:49:41.447 | INFO | MemInfo: { | 20260506 18:49:41.447 | INFO | "gpu" : " 17 GB", | 20260506 18:49:41.447 | INFO | "huge" : "", | 20260506 18:49:41.447 | INFO | "huge_shared" : " 27 GB", | 20260506 18:49:41.447 | INFO | "malloc" : " 14 GB", | 20260506 18:49:41.447 | INFO | "pinned" : " 29 GB" | 20260506 18:49:41.447 | INFO | } | 20260506 18:49:41.447 | INFO | Perf report for the last 5.14602 seconds: | 20260506 18:49:41.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:49:41.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:41.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:41.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:41.447 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 18:49:41.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:41.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:41.447 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:49:41.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:41.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:41.448 | INFO | Uploaded 36GiB to GPU in 1.91396s (18.8092GiB/s) | 20260506 18:49:41.448 | INFO | Downloaded 34GiB to GPU in 1.87184s (18.1639GiB/s) | 20260506 18:49:46.582 | INFO | Analyzing chunk frames = [3635200,3635712) | 20260506 18:49:46.582 | INFO | MemInfo: { | 20260506 18:49:46.582 | INFO | "gpu" : " 17 GB", | 20260506 18:49:46.582 | INFO | "huge" : "", | 20260506 18:49:46.582 | INFO | "huge_shared" : " 27 GB", | 20260506 18:49:46.582 | INFO | "malloc" : " 14 GB", | 20260506 18:49:46.582 | INFO | "pinned" : " 29 GB" | 20260506 18:49:46.582 | INFO | } | 20260506 18:49:46.582 | INFO | Perf report for the last 5.13454 seconds: | 20260506 18:49:46.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:49:46.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:46.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:46.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:46.582 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 18:49:46.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:46.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:46.582 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:49:46.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:46.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:46.582 | INFO | Uploaded 36GiB to GPU in 1.98402s (18.145GiB/s) | 20260506 18:49:46.582 | INFO | Downloaded 34GiB to GPU in 1.91802s (17.7266GiB/s) | 20260506 18:49:51.717 | INFO | Analyzing chunk frames = [3635712,3636224) | 20260506 18:49:51.717 | INFO | MemInfo: { | 20260506 18:49:51.717 | INFO | "gpu" : " 17 GB", | 20260506 18:49:51.717 | INFO | "huge" : "", | 20260506 18:49:51.717 | INFO | "huge_shared" : " 27 GB", | 20260506 18:49:51.717 | INFO | "malloc" : " 14 GB", | 20260506 18:49:51.717 | INFO | "pinned" : " 29 GB" | 20260506 18:49:51.717 | INFO | } | 20260506 18:49:51.717 | INFO | Perf report for the last 5.13535 seconds: | 20260506 18:49:51.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:49:51.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:51.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:51.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:51.717 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 18:49:51.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:51.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:51.717 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 18:49:51.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:51.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:51.718 | INFO | Uploaded 36GiB to GPU in 2.00678s (17.9392GiB/s) | 20260506 18:49:51.718 | INFO | Downloaded 34GiB to GPU in 1.96228s (17.3267GiB/s) | 20260506 18:49:56.857 | INFO | Analyzing chunk frames = [3636224,3636736) | 20260506 18:49:56.857 | INFO | MemInfo: { | 20260506 18:49:56.857 | INFO | "gpu" : " 17 GB", | 20260506 18:49:56.857 | INFO | "huge" : "", | 20260506 18:49:56.857 | INFO | "huge_shared" : " 27 GB", | 20260506 18:49:56.857 | INFO | "malloc" : " 14 GB", | 20260506 18:49:56.858 | INFO | "pinned" : " 29 GB" | 20260506 18:49:56.858 | INFO | } | 20260506 18:49:56.858 | INFO | Perf report for the last 5.14044 seconds: | 20260506 18:49:56.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:49:56.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:56.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:56.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:56.858 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 18:49:56.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:56.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:56.858 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:49:56.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:56.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:49:56.858 | INFO | Uploaded 36GiB to GPU in 1.99974s (18.0024GiB/s) | 20260506 18:49:56.858 | INFO | Downloaded 34GiB to GPU in 2.01125s (16.9049GiB/s) | 20260506 18:50:01.995 | INFO | Analyzing chunk frames = [3636736,3637248) | 20260506 18:50:01.995 | INFO | MemInfo: { | 20260506 18:50:01.995 | INFO | "gpu" : " 17 GB", | 20260506 18:50:01.995 | INFO | "huge" : "", | 20260506 18:50:01.995 | INFO | "huge_shared" : " 27 GB", | 20260506 18:50:01.995 | INFO | "malloc" : " 14 GB", | 20260506 18:50:01.995 | INFO | "pinned" : " 29 GB" | 20260506 18:50:01.995 | INFO | } | 20260506 18:50:01.995 | INFO | Perf report for the last 5.13759 seconds: | 20260506 18:50:01.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:50:01.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:01.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:01.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:01.995 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:50:01.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:01.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:01.995 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 18:50:01.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:01.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:01.996 | INFO | Uploaded 36GiB to GPU in 2.01131s (17.8988GiB/s) | 20260506 18:50:01.996 | INFO | Downloaded 34GiB to GPU in 1.99674s (17.0278GiB/s) | 20260506 18:50:01.996 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:50:01.996 | INFO | Total mem usage: 18 GiB | 20260506 18:50:01.996 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:50:01.996 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -10.9%) | 20260506 18:50:01.996 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:50:01.996 | INFO | High Water Breakdown: | 20260506 18:50:01.996 | INFO | 62% : 10.1 GiB ( 88.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:50:01.996 | INFO | 37.9% : 6.15 GiB ( 91.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:50:01.996 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:50:01.996 | INFO | ------------------------------------------ | 20260506 18:50:01.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:50:01.996 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:50:01.996 | INFO | Total mem usage: 26.5 GiB | 20260506 18:50:01.996 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:50:01.996 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:50:01.996 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:50:01.996 | INFO | High Water Breakdown: | 20260506 18:50:01.996 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:50:01.996 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:50:01.996 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:50:01.996 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:50:01.996 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:50:01.996 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:50:01.996 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:50:01.996 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:50:01.996 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:50:01.996 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:50:01.996 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:50:01.996 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:50:01.996 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:50:01.996 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:50:01.996 | INFO | ------------------------------------------ | 20260506 18:50:01.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:50:01.996 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:50:01.996 | INFO | Total mem usage: 25.6 GiB | 20260506 18:50:01.996 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:50:01.996 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:50:01.996 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:50:01.996 | INFO | High Water Breakdown: | 20260506 18:50:01.996 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:50:01.996 | INFO | ------------------------------------------ | 20260506 18:50:01.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:50:01.996 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:50:01.996 | INFO | Total mem usage: 15.7 GiB | 20260506 18:50:01.996 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:50:01.996 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:50:01.996 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 18:50:01.996 | INFO | High Water Breakdown: | 20260506 18:50:01.996 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:50:01.996 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:50:01.996 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:50:01.996 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:50:01.996 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:50:01.996 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:50:01.996 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:50:01.996 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:50:01.996 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:50:01.996 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:50:01.996 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:50:01.996 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:50:01.996 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:50:01.996 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:50:01.996 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:50:01.996 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:50:01.996 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:50:01.996 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:50:01.996 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:50:01.996 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:50:01.996 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:50:01.996 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:50:01.996 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:50:01.996 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:50:01.996 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:50:01.996 | INFO | ------------------------------------------ | 20260506 18:50:01.996 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:50:01.997 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:50:01.997 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:50:01.997 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:50:01.997 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.58e+01, 1.36e+01 / 2.97e+00 | 20260506 18:50:01.997 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.03e-01, 1.02e-02 / 5.23e-03 | 20260506 18:50:01.997 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.44e+00, 5.58e-01 / 3.44e-01 | 20260506 18:50:01.997 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.85e-01 / 1.43e-01 | 20260506 18:50:01.997 | INFO | PerfReport: 15.97%, DME -- 3.99e+01 / 4.45e+01, 4.20e+00 / 3.23e+00 | 20260506 18:50:01.997 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.44e+00 / 2.02e+00 | 20260506 18:50:01.997 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.25e+00 / 2.78e+00, 3.83e-01 / 2.58e-01 | 20260506 18:50:01.997 | INFO | PerfReport: 3.38%, Metrics -- 1.06e+00 / 1.38e+00, 2.80e-01 / 1.91e-01 | 20260506 18:50:01.997 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.65e-01 / 4.10e-01, 5.76e-02 / 2.72e-02 | 20260506 18:50:01.997 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:50:01.997 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:50:01.997 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:50:01.997 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 18:50:01.997 | INFO | PerfReport: 21.39%, Upload -- 2.07e+01 / 0.00e+00, 6.82e+00 / 0.00e+00 | 20260506 18:50:01.997 | INFO | PerfReport: 21.20%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 18:50:01.997 | INFO | PerfReport: 32.26%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 18:50:01.997 | INFO | PerfReport: 1.21%, LockUpload -- 1.17e+00 / 0.00e+00, 5.14e+00 / 0.00e+00 | 20260506 18:50:01.997 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260506 18:50:01.997 | INFO | PerfReport: 22.70%, LockCompute -- 2.20e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 18:50:01.997 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260506 18:50:02.110 | INFO | Starting a round of Baz IO | 20260506 18:50:05.269 | INFO | Populating RTMetrics report | 20260506 18:50:05.270 | INFO | RTMetrics report populated | 20260506 18:50:05.270 | INFO | Opening tmp RTMetrics json file | 20260506 18:50:05.305 | INFO | Serializing RTMetrics to json | 20260506 18:50:05.311 | INFO | Serialization done, swapping tmp file now... | 20260506 18:50:05.326 | INFO | RTMetrics done | 20260506 18:50:05.462 | INFO | Finished a round of Baz IO: Wrote 2.026 GiB in 3.352 seconds | 20260506 18:50:05.462 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.89% / 0.01% / 65.10% | 20260506 18:50:05.462 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.03214% / 1.19209e-05% / 8.74201e-05% | 20260506 18:50:05.462 | INFO | Baz IO overall: | 20260506 18:50:05.463 | INFO | BytesWritten : 1921987636946 | 20260506 18:50:05.463 | INFO | NumEvents : 929344840197 | 20260506 18:50:07.135 | INFO | Analyzing chunk frames = [3637248,3637760) | 20260506 18:50:07.135 | INFO | MemInfo: { | 20260506 18:50:07.135 | INFO | "gpu" : " 17 GB", | 20260506 18:50:07.135 | INFO | "huge" : "", | 20260506 18:50:07.135 | INFO | "huge_shared" : " 27 GB", | 20260506 18:50:07.135 | INFO | "malloc" : " 14 GB", | 20260506 18:50:07.135 | INFO | "pinned" : " 29 GB" | 20260506 18:50:07.135 | INFO | } | 20260506 18:50:07.135 | INFO | Perf report for the last 5.13978 seconds: | 20260506 18:50:07.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:50:07.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:07.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:07.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:07.135 | INFO | ANALYSIS : 63.1%, 2.8, 94.1ms, 0 | 20260506 18:50:07.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:07.135 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260506 18:50:07.135 | INFO | PRE_HQ : 61.2%, 1.9, 61.1ms, 0 | 20260506 18:50:07.135 | INFO | BAZWRITER : 64.9%, 4.8, 158.8ms, 0 | 20260506 18:50:07.135 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 18:50:07.135 | INFO | Uploaded 36GiB to GPU in 2.06989s (17.3922GiB/s) | 20260506 18:50:07.135 | INFO | Downloaded 38GiB to GPU in 2.19361s (17.323GiB/s) | 20260506 18:50:12.272 | INFO | Analyzing chunk frames = [3637760,3638272) | 20260506 18:50:12.272 | INFO | MemInfo: { | 20260506 18:50:12.272 | INFO | "gpu" : " 17 GB", | 20260506 18:50:12.272 | INFO | "huge" : "", | 20260506 18:50:12.272 | INFO | "huge_shared" : " 27 GB", | 20260506 18:50:12.272 | INFO | "malloc" : " 14 GB", | 20260506 18:50:12.272 | INFO | "pinned" : " 29 GB" | 20260506 18:50:12.272 | INFO | } | 20260506 18:50:12.273 | INFO | Perf report for the last 5.13746 seconds: | 20260506 18:50:12.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:50:12.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:12.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:12.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:12.273 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 18:50:12.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:12.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:12.273 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 18:50:12.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:12.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:12.273 | INFO | Uploaded 36GiB to GPU in 2.01271s (17.8863GiB/s) | 20260506 18:50:12.273 | INFO | Downloaded 34GiB to GPU in 1.94584s (17.4732GiB/s) | 20260506 18:50:17.410 | INFO | Analyzing chunk frames = [3638272,3638784) | 20260506 18:50:17.410 | INFO | MemInfo: { | 20260506 18:50:17.410 | INFO | "gpu" : " 17 GB", | 20260506 18:50:17.410 | INFO | "huge" : "", | 20260506 18:50:17.410 | INFO | "huge_shared" : " 27 GB", | 20260506 18:50:17.410 | INFO | "malloc" : " 14 GB", | 20260506 18:50:17.410 | INFO | "pinned" : " 29 GB" | 20260506 18:50:17.410 | INFO | } | 20260506 18:50:17.410 | INFO | Perf report for the last 5.13777 seconds: | 20260506 18:50:17.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:50:17.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:17.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:17.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:17.410 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:50:17.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:17.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:17.410 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:50:17.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:17.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:17.411 | INFO | Uploaded 36GiB to GPU in 1.96703s (18.3017GiB/s) | 20260506 18:50:17.411 | INFO | Downloaded 34GiB to GPU in 1.92371s (17.6742GiB/s) | 20260506 18:50:22.552 | INFO | Analyzing chunk frames = [3638784,3639296) | 20260506 18:50:22.552 | INFO | MemInfo: { | 20260506 18:50:22.552 | INFO | "gpu" : " 17 GB", | 20260506 18:50:22.552 | INFO | "huge" : "", | 20260506 18:50:22.553 | INFO | "huge_shared" : " 27 GB", | 20260506 18:50:22.553 | INFO | "malloc" : " 14 GB", | 20260506 18:50:22.553 | INFO | "pinned" : " 29 GB" | 20260506 18:50:22.553 | INFO | } | 20260506 18:50:22.553 | INFO | Perf report for the last 5.14242 seconds: | 20260506 18:50:22.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:50:22.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:22.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:22.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:22.553 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 18:50:22.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:22.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:22.553 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:50:22.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:22.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:22.553 | INFO | Uploaded 36GiB to GPU in 1.92105s (18.7397GiB/s) | 20260506 18:50:22.553 | INFO | Downloaded 34GiB to GPU in 1.87016s (18.1802GiB/s) | 20260506 18:50:27.699 | INFO | Analyzing chunk frames = [3639296,3639808) | 20260506 18:50:27.699 | INFO | MemInfo: { | 20260506 18:50:27.699 | INFO | "gpu" : " 17 GB", | 20260506 18:50:27.699 | INFO | "huge" : "", | 20260506 18:50:27.699 | INFO | "huge_shared" : " 27 GB", | 20260506 18:50:27.699 | INFO | "malloc" : " 14 GB", | 20260506 18:50:27.699 | INFO | "pinned" : " 29 GB" | 20260506 18:50:27.699 | INFO | } | 20260506 18:50:27.699 | INFO | Perf report for the last 5.14657 seconds: | 20260506 18:50:27.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:50:27.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:27.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:27.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:27.699 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 18:50:27.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:27.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:27.700 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:50:27.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:27.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:27.700 | INFO | Uploaded 36GiB to GPU in 2.01329s (17.8812GiB/s) | 20260506 18:50:27.700 | INFO | Downloaded 34GiB to GPU in 1.91817s (17.7252GiB/s) | 20260506 18:50:32.838 | INFO | Analyzing chunk frames = [3639808,3640320) | 20260506 18:50:32.838 | INFO | MemInfo: { | 20260506 18:50:32.838 | INFO | "gpu" : " 17 GB", | 20260506 18:50:32.838 | INFO | "huge" : "", | 20260506 18:50:32.838 | INFO | "huge_shared" : " 27 GB", | 20260506 18:50:32.838 | INFO | "malloc" : " 14 GB", | 20260506 18:50:32.838 | INFO | "pinned" : " 29 GB" | 20260506 18:50:32.838 | INFO | } | 20260506 18:50:32.838 | INFO | Perf report for the last 5.139 seconds: | 20260506 18:50:32.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:50:32.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:32.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:32.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:32.838 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 18:50:32.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:32.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:32.838 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:50:32.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:32.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:32.839 | INFO | Uploaded 36GiB to GPU in 2.02783s (17.7529GiB/s) | 20260506 18:50:32.839 | INFO | Downloaded 34GiB to GPU in 1.95864s (17.359GiB/s) | 20260506 18:50:37.972 | INFO | Analyzing chunk frames = [3640320,3640832) | 20260506 18:50:37.972 | INFO | MemInfo: { | 20260506 18:50:37.972 | INFO | "gpu" : " 17 GB", | 20260506 18:50:37.972 | INFO | "huge" : "", | 20260506 18:50:37.972 | INFO | "huge_shared" : " 27 GB", | 20260506 18:50:37.972 | INFO | "malloc" : " 14 GB", | 20260506 18:50:37.972 | INFO | "pinned" : " 29 GB" | 20260506 18:50:37.972 | INFO | } | 20260506 18:50:37.973 | INFO | Perf report for the last 5.13417 seconds: | 20260506 18:50:37.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:50:37.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:37.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:37.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:37.973 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 18:50:37.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:37.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:37.973 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:50:37.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:37.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:37.973 | INFO | Uploaded 36GiB to GPU in 1.995s (18.0451GiB/s) | 20260506 18:50:37.973 | INFO | Downloaded 34GiB to GPU in 1.97667s (17.2007GiB/s) | 20260506 18:50:43.115 | INFO | Analyzing chunk frames = [3640832,3641344) | 20260506 18:50:43.115 | INFO | MemInfo: { | 20260506 18:50:43.115 | INFO | "gpu" : " 17 GB", | 20260506 18:50:43.115 | INFO | "huge" : "", | 20260506 18:50:43.115 | INFO | "huge_shared" : " 27 GB", | 20260506 18:50:43.115 | INFO | "malloc" : " 14 GB", | 20260506 18:50:43.115 | INFO | "pinned" : " 29 GB" | 20260506 18:50:43.115 | INFO | } | 20260506 18:50:43.116 | INFO | Perf report for the last 5.14296 seconds: | 20260506 18:50:43.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:50:43.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:43.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:43.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:43.116 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 18:50:43.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:43.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:43.116 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:50:43.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:43.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:43.116 | INFO | Uploaded 36GiB to GPU in 2.0556s (17.5131GiB/s) | 20260506 18:50:43.116 | INFO | Downloaded 34GiB to GPU in 1.98651s (17.1155GiB/s) | 20260506 18:50:43.228 | INFO | Starting a round of Baz IO | 20260506 18:50:46.361 | INFO | Populating RTMetrics report | 20260506 18:50:46.362 | INFO | RTMetrics report populated | 20260506 18:50:46.362 | INFO | Opening tmp RTMetrics json file | 20260506 18:50:46.440 | INFO | Serializing RTMetrics to json | 20260506 18:50:46.516 | INFO | Serialization done, swapping tmp file now... | 20260506 18:50:46.537 | INFO | RTMetrics done | 20260506 18:50:47.050 | INFO | Finished a round of Baz IO: Wrote 2.025 GiB in 3.821 seconds | 20260506 18:50:47.050 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.89% / 0.01% / 65.11% | 20260506 18:50:47.050 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.0301% / 3.97364e-06% / 7.54992e-05% | 20260506 18:50:47.050 | INFO | Baz IO overall: | 20260506 18:50:47.050 | INFO | BytesWritten : 1924162470853 | 20260506 18:50:47.050 | INFO | NumEvents : 930390981141 | 20260506 18:50:48.248 | INFO | Analyzing chunk frames = [3641344,3641856) | 20260506 18:50:48.248 | INFO | MemInfo: { | 20260506 18:50:48.248 | INFO | "gpu" : " 17 GB", | 20260506 18:50:48.248 | INFO | "huge" : "", | 20260506 18:50:48.248 | INFO | "huge_shared" : " 27 GB", | 20260506 18:50:48.248 | INFO | "malloc" : " 14 GB", | 20260506 18:50:48.248 | INFO | "pinned" : " 29 GB" | 20260506 18:50:48.248 | INFO | } | 20260506 18:50:48.248 | INFO | Perf report for the last 5.13209 seconds: | 20260506 18:50:48.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:50:48.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:48.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:48.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:48.248 | INFO | ANALYSIS : 62.7%, 2.9, 95.1ms, 0 | 20260506 18:50:48.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:48.248 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260506 18:50:48.248 | INFO | PRE_HQ : 60.2%, 1.9, 57.9ms, 0 | 20260506 18:50:48.248 | INFO | BAZWRITER : 74.1%, 4.8, 183.5ms, 0 | 20260506 18:50:48.248 | INFO | BAZ_SAMPLING : 3.3%, 1.0, 1.7ms, 0 | 20260506 18:50:48.248 | INFO | Uploaded 36GiB to GPU in 2.05565s (17.5127GiB/s) | 20260506 18:50:48.248 | INFO | Downloaded 38GiB to GPU in 2.20774s (17.2122GiB/s) | 20260506 18:50:53.387 | INFO | Analyzing chunk frames = [3641856,3642368) | 20260506 18:50:53.387 | INFO | MemInfo: { | 20260506 18:50:53.387 | INFO | "gpu" : " 17 GB", | 20260506 18:50:53.387 | INFO | "huge" : "", | 20260506 18:50:53.387 | INFO | "huge_shared" : " 27 GB", | 20260506 18:50:53.387 | INFO | "malloc" : " 14 GB", | 20260506 18:50:53.387 | INFO | "pinned" : " 29 GB" | 20260506 18:50:53.388 | INFO | } | 20260506 18:50:53.388 | INFO | Perf report for the last 5.14001 seconds: | 20260506 18:50:53.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:50:53.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:53.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:53.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:53.388 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:50:53.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:53.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:53.388 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:50:53.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:53.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:53.388 | INFO | Uploaded 36GiB to GPU in 2.0046s (17.9587GiB/s) | 20260506 18:50:53.388 | INFO | Downloaded 34GiB to GPU in 1.94005s (17.5253GiB/s) | 20260506 18:50:58.534 | INFO | Analyzing chunk frames = [3642368,3642880) | 20260506 18:50:58.534 | INFO | MemInfo: { | 20260506 18:50:58.534 | INFO | "gpu" : " 17 GB", | 20260506 18:50:58.534 | INFO | "huge" : "", | 20260506 18:50:58.534 | INFO | "huge_shared" : " 27 GB", | 20260506 18:50:58.534 | INFO | "malloc" : " 14 GB", | 20260506 18:50:58.534 | INFO | "pinned" : " 29 GB" | 20260506 18:50:58.534 | INFO | } | 20260506 18:50:58.534 | INFO | Perf report for the last 5.14619 seconds: | 20260506 18:50:58.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:50:58.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:58.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:58.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:58.534 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 18:50:58.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:58.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:58.534 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:50:58.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:58.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:50:58.534 | INFO | Uploaded 36GiB to GPU in 1.97645s (18.2145GiB/s) | 20260506 18:50:58.534 | INFO | Downloaded 34GiB to GPU in 1.91489s (17.7556GiB/s) | 20260506 18:51:03.671 | INFO | Analyzing chunk frames = [3642880,3643392) | 20260506 18:51:03.671 | INFO | MemInfo: { | 20260506 18:51:03.671 | INFO | "gpu" : " 17 GB", | 20260506 18:51:03.671 | INFO | "huge" : "", | 20260506 18:51:03.672 | INFO | "huge_shared" : " 27 GB", | 20260506 18:51:03.672 | INFO | "malloc" : " 14 GB", | 20260506 18:51:03.672 | INFO | "pinned" : " 29 GB" | 20260506 18:51:03.672 | INFO | } | 20260506 18:51:03.672 | INFO | Perf report for the last 5.1379 seconds: | 20260506 18:51:03.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:51:03.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:03.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:03.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:03.672 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:51:03.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:03.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:03.672 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:51:03.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:03.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:03.672 | INFO | Uploaded 36GiB to GPU in 1.93692s (18.5862GiB/s) | 20260506 18:51:03.672 | INFO | Downloaded 34GiB to GPU in 1.8677s (18.2042GiB/s) | 20260506 18:51:08.814 | INFO | Analyzing chunk frames = [3643392,3643904) | 20260506 18:51:08.814 | INFO | MemInfo: { | 20260506 18:51:08.814 | INFO | "gpu" : " 17 GB", | 20260506 18:51:08.815 | INFO | "huge" : "", | 20260506 18:51:08.815 | INFO | "huge_shared" : " 27 GB", | 20260506 18:51:08.815 | INFO | "malloc" : " 14 GB", | 20260506 18:51:08.815 | INFO | "pinned" : " 29 GB" | 20260506 18:51:08.815 | INFO | } | 20260506 18:51:08.815 | INFO | Perf report for the last 5.1429 seconds: | 20260506 18:51:08.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:51:08.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:08.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:08.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:08.815 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 18:51:08.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:08.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:08.815 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:51:08.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:08.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:08.815 | INFO | Uploaded 36GiB to GPU in 2.00928s (17.9168GiB/s) | 20260506 18:51:08.815 | INFO | Downloaded 34GiB to GPU in 1.90796s (17.8201GiB/s) | 20260506 18:51:13.952 | INFO | Analyzing chunk frames = [3643904,3644416) | 20260506 18:51:13.952 | INFO | MemInfo: { | 20260506 18:51:13.952 | INFO | "gpu" : " 17 GB", | 20260506 18:51:13.952 | INFO | "huge" : "", | 20260506 18:51:13.953 | INFO | "huge_shared" : " 27 GB", | 20260506 18:51:13.953 | INFO | "malloc" : " 14 GB", | 20260506 18:51:13.953 | INFO | "pinned" : " 29 GB" | 20260506 18:51:13.953 | INFO | } | 20260506 18:51:13.953 | INFO | Perf report for the last 5.13795 seconds: | 20260506 18:51:13.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:51:13.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:13.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:13.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:13.953 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 18:51:13.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:13.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:13.953 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 18:51:13.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:13.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:13.953 | INFO | Uploaded 36GiB to GPU in 2.03405s (17.6987GiB/s) | 20260506 18:51:13.953 | INFO | Downloaded 34GiB to GPU in 1.95287s (17.4103GiB/s) | 20260506 18:51:19.087 | INFO | Analyzing chunk frames = [3644416,3644928) | 20260506 18:51:19.087 | INFO | MemInfo: { | 20260506 18:51:19.087 | INFO | "gpu" : " 17 GB", | 20260506 18:51:19.087 | INFO | "huge" : "", | 20260506 18:51:19.087 | INFO | "huge_shared" : " 27 GB", | 20260506 18:51:19.087 | INFO | "malloc" : " 14 GB", | 20260506 18:51:19.087 | INFO | "pinned" : " 29 GB" | 20260506 18:51:19.087 | INFO | } | 20260506 18:51:19.087 | INFO | Perf report for the last 5.13421 seconds: | 20260506 18:51:19.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:51:19.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:19.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:19.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:19.087 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 18:51:19.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:19.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:19.087 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 18:51:19.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:19.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:19.087 | INFO | Uploaded 36GiB to GPU in 2.02259s (17.7989GiB/s) | 20260506 18:51:19.087 | INFO | Downloaded 34GiB to GPU in 1.98708s (17.1106GiB/s) | 20260506 18:51:24.223 | INFO | Analyzing chunk frames = [3644928,3645440) | 20260506 18:51:24.223 | INFO | MemInfo: { | 20260506 18:51:24.223 | INFO | "gpu" : " 17 GB", | 20260506 18:51:24.223 | INFO | "huge" : "", | 20260506 18:51:24.224 | INFO | "huge_shared" : " 27 GB", | 20260506 18:51:24.224 | INFO | "malloc" : " 14 GB", | 20260506 18:51:24.224 | INFO | "pinned" : " 29 GB" | 20260506 18:51:24.224 | INFO | } | 20260506 18:51:24.224 | INFO | Perf report for the last 5.13681 seconds: | 20260506 18:51:24.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:51:24.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:24.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:24.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:24.224 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:51:24.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:24.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:24.224 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 18:51:24.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:24.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:24.224 | INFO | Uploaded 36GiB to GPU in 2.02081s (17.8146GiB/s) | 20260506 18:51:24.224 | INFO | Downloaded 34GiB to GPU in 1.99116s (17.0755GiB/s) | 20260506 18:51:24.224 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:51:24.224 | INFO | Total mem usage: 18 GiB | 20260506 18:51:24.224 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:51:24.224 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.2%) | 20260506 18:51:24.224 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:51:24.224 | INFO | High Water Breakdown: | 20260506 18:51:24.224 | INFO | 61.7% : 9.82 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:51:24.224 | INFO | 38.2% : 6.07 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:51:24.224 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:51:24.224 | INFO | ------------------------------------------ | 20260506 18:51:24.224 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:51:24.224 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:51:24.224 | INFO | Total mem usage: 26.5 GiB | 20260506 18:51:24.224 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:51:24.224 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:51:24.224 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:51:24.224 | INFO | High Water Breakdown: | 20260506 18:51:24.225 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:51:24.225 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:51:24.225 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:51:24.225 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:51:24.225 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:51:24.225 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:51:24.225 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:51:24.225 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:51:24.225 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:51:24.225 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:51:24.225 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:51:24.225 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:51:24.225 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:51:24.225 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:51:24.225 | INFO | ------------------------------------------ | 20260506 18:51:24.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:51:24.225 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:51:24.225 | INFO | Total mem usage: 25.6 GiB | 20260506 18:51:24.225 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:51:24.225 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:51:24.225 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:51:24.225 | INFO | High Water Breakdown: | 20260506 18:51:24.225 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:51:24.225 | INFO | ------------------------------------------ | 20260506 18:51:24.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:51:24.225 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:51:24.225 | INFO | Total mem usage: 15.7 GiB | 20260506 18:51:24.225 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:51:24.225 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:51:24.225 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:51:24.225 | INFO | High Water Breakdown: | 20260506 18:51:24.225 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:51:24.225 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:51:24.225 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:51:24.225 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:51:24.225 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:51:24.225 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:51:24.225 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:51:24.225 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:51:24.225 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:51:24.225 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:51:24.225 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:51:24.225 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:51:24.225 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:51:24.225 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:51:24.225 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:51:24.225 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:51:24.225 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:51:24.225 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:51:24.225 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:51:24.225 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:51:24.225 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:51:24.225 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:51:24.225 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:51:24.225 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:51:24.225 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:51:24.225 | INFO | ------------------------------------------ | 20260506 18:51:24.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:51:24.225 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:51:24.225 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:51:24.225 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:51:24.225 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.56e+01, 1.35e+01 / 2.12e+00 | 20260506 18:51:24.225 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.02e-01, 1.07e-02 / 8.47e-03 | 20260506 18:51:24.225 | INFO | PerfReport: 11.63%, Baseline -- 3.64e+00 / 4.45e+00, 5.57e-01 / 3.16e-01 | 20260506 18:51:24.225 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.82e-01 / 1.33e-01 | 20260506 18:51:24.225 | INFO | PerfReport: 15.95%, DME -- 3.99e+01 / 4.44e+01, 3.91e+00 / 2.09e+00 | 20260506 18:51:24.225 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.44e+00 / 2.10e+00 | 20260506 18:51:24.225 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.25e+00 / 2.83e+00, 3.83e-01 / 2.25e-01 | 20260506 18:51:24.225 | INFO | PerfReport: 3.39%, Metrics -- 1.06e+00 / 1.40e+00, 2.80e-01 / 1.57e-01 | 20260506 18:51:24.225 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.65e-01 / 4.13e-01, 5.69e-02 / 2.19e-02 | 20260506 18:51:24.225 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:51:24.225 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:51:24.225 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:51:24.225 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 18:51:24.225 | INFO | PerfReport: 21.56%, Upload -- 2.09e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260506 18:51:24.225 | INFO | PerfReport: 21.17%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 18:51:24.225 | INFO | PerfReport: 32.25%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:51:24.225 | INFO | PerfReport: 1.40%, LockUpload -- 1.36e+00 / 0.00e+00, 5.38e+00 / 0.00e+00 | 20260506 18:51:24.225 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260506 18:51:24.225 | INFO | PerfReport: 22.51%, LockCompute -- 2.18e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 18:51:24.225 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.55e-02 / 0.00e+00 | 20260506 18:51:24.360 | INFO | Starting a round of Baz IO | 20260506 18:51:27.477 | INFO | Populating RTMetrics report | 20260506 18:51:27.477 | INFO | RTMetrics report populated | 20260506 18:51:27.477 | INFO | Opening tmp RTMetrics json file | 20260506 18:51:27.611 | INFO | Serializing RTMetrics to json | 20260506 18:51:27.621 | INFO | Serialization done, swapping tmp file now... | 20260506 18:51:27.624 | INFO | RTMetrics done | 20260506 18:51:28.164 | INFO | Finished a round of Baz IO: Wrote 2.026 GiB in 3.804 seconds | 20260506 18:51:28.164 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.88% / 0.01% / 65.11% | 20260506 18:51:28.164 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.02999% / 7.94729e-06% / 9.53674e-05% | 20260506 18:51:28.164 | INFO | Baz IO overall: | 20260506 18:51:28.164 | INFO | BytesWritten : 1926338007624 | 20260506 18:51:28.164 | INFO | NumEvents : 931437807290 | 20260506 18:51:29.362 | INFO | Analyzing chunk frames = [3645440,3645952) | 20260506 18:51:29.362 | INFO | MemInfo: { | 20260506 18:51:29.362 | INFO | "gpu" : " 17 GB", | 20260506 18:51:29.362 | INFO | "huge" : "", | 20260506 18:51:29.362 | INFO | "huge_shared" : " 27 GB", | 20260506 18:51:29.362 | INFO | "malloc" : " 14 GB", | 20260506 18:51:29.362 | INFO | "pinned" : " 29 GB" | 20260506 18:51:29.362 | INFO | } | 20260506 18:51:29.362 | INFO | Perf report for the last 5.13843 seconds: | 20260506 18:51:29.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:51:29.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:29.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:29.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:29.362 | INFO | ANALYSIS : 62.7%, 2.9, 95.5ms, 0 | 20260506 18:51:29.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:29.362 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260506 18:51:29.362 | INFO | PRE_HQ : 60.0%, 1.8, 57.6ms, 0 | 20260506 18:51:29.362 | INFO | BAZWRITER : 74.0%, 4.8, 183.6ms, 0 | 20260506 18:51:29.362 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 18:51:29.362 | INFO | Uploaded 36GiB to GPU in 2.03806s (17.6639GiB/s) | 20260506 18:51:29.362 | INFO | Downloaded 38GiB to GPU in 2.21764s (17.1353GiB/s) | 20260506 18:51:34.503 | INFO | Analyzing chunk frames = [3645952,3646464) | 20260506 18:51:34.503 | INFO | MemInfo: { | 20260506 18:51:34.503 | INFO | "gpu" : " 17 GB", | 20260506 18:51:34.503 | INFO | "huge" : "", | 20260506 18:51:34.503 | INFO | "huge_shared" : " 27 GB", | 20260506 18:51:34.503 | INFO | "malloc" : " 14 GB", | 20260506 18:51:34.503 | INFO | "pinned" : " 29 GB" | 20260506 18:51:34.503 | INFO | } | 20260506 18:51:34.503 | INFO | Perf report for the last 5.14109 seconds: | 20260506 18:51:34.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:51:34.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:34.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:34.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:34.503 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 18:51:34.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:34.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:34.503 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:51:34.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:34.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:34.503 | INFO | Uploaded 36GiB to GPU in 1.99101s (18.0813GiB/s) | 20260506 18:51:34.504 | INFO | Downloaded 34GiB to GPU in 1.92002s (17.7082GiB/s) | 20260506 18:51:39.644 | INFO | Analyzing chunk frames = [3646464,3646976) | 20260506 18:51:39.644 | INFO | MemInfo: { | 20260506 18:51:39.644 | INFO | "gpu" : " 17 GB", | 20260506 18:51:39.644 | INFO | "huge" : "", | 20260506 18:51:39.644 | INFO | "huge_shared" : " 27 GB", | 20260506 18:51:39.644 | INFO | "malloc" : " 14 GB", | 20260506 18:51:39.644 | INFO | "pinned" : " 29 GB" | 20260506 18:51:39.644 | INFO | } | 20260506 18:51:39.644 | INFO | Perf report for the last 5.1412 seconds: | 20260506 18:51:39.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:51:39.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:39.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:39.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:39.645 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:51:39.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:39.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:39.645 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:51:39.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:39.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:39.645 | INFO | Uploaded 36GiB to GPU in 1.98066s (18.1757GiB/s) | 20260506 18:51:39.645 | INFO | Downloaded 34GiB to GPU in 1.9172s (17.7342GiB/s) | 20260506 18:51:44.791 | INFO | Analyzing chunk frames = [3646976,3647488) | 20260506 18:51:44.791 | INFO | MemInfo: { | 20260506 18:51:44.791 | INFO | "gpu" : " 17 GB", | 20260506 18:51:44.791 | INFO | "huge" : "", | 20260506 18:51:44.791 | INFO | "huge_shared" : " 27 GB", | 20260506 18:51:44.791 | INFO | "malloc" : " 14 GB", | 20260506 18:51:44.791 | INFO | "pinned" : " 29 GB" | 20260506 18:51:44.791 | INFO | } | 20260506 18:51:44.791 | INFO | Perf report for the last 5.14671 seconds: | 20260506 18:51:44.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:51:44.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:44.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:44.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:44.791 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 18:51:44.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:44.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:44.791 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:51:44.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:44.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:44.791 | INFO | Uploaded 36GiB to GPU in 1.92794s (18.6728GiB/s) | 20260506 18:51:44.792 | INFO | Downloaded 34GiB to GPU in 1.86663s (18.2147GiB/s) | 20260506 18:51:49.924 | INFO | Analyzing chunk frames = [3647488,3648000) | 20260506 18:51:49.924 | INFO | MemInfo: { | 20260506 18:51:49.924 | INFO | "gpu" : " 17 GB", | 20260506 18:51:49.924 | INFO | "huge" : "", | 20260506 18:51:49.924 | INFO | "huge_shared" : " 27 GB", | 20260506 18:51:49.924 | INFO | "malloc" : " 14 GB", | 20260506 18:51:49.924 | INFO | "pinned" : " 29 GB" | 20260506 18:51:49.924 | INFO | } | 20260506 18:51:49.924 | INFO | Perf report for the last 5.13324 seconds: | 20260506 18:51:49.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:51:49.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:49.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:49.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:49.925 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 18:51:49.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:49.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:49.925 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:51:49.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:49.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:49.925 | INFO | Uploaded 36GiB to GPU in 1.99753s (18.0223GiB/s) | 20260506 18:51:49.925 | INFO | Downloaded 34GiB to GPU in 1.91543s (17.7506GiB/s) | 20260506 18:51:55.064 | INFO | Analyzing chunk frames = [3648000,3648512) | 20260506 18:51:55.064 | INFO | MemInfo: { | 20260506 18:51:55.064 | INFO | "gpu" : " 17 GB", | 20260506 18:51:55.064 | INFO | "huge" : "", | 20260506 18:51:55.064 | INFO | "huge_shared" : " 27 GB", | 20260506 18:51:55.064 | INFO | "malloc" : " 14 GB", | 20260506 18:51:55.064 | INFO | "pinned" : " 29 GB" | 20260506 18:51:55.064 | INFO | } | 20260506 18:51:55.064 | INFO | Perf report for the last 5.13927 seconds: | 20260506 18:51:55.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:51:55.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:55.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:55.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:55.064 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 18:51:55.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:55.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:55.064 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 18:51:55.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:55.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:51:55.064 | INFO | Uploaded 36GiB to GPU in 2.01395s (17.8753GiB/s) | 20260506 18:51:55.064 | INFO | Downloaded 34GiB to GPU in 1.93984s (17.5272GiB/s) | 20260506 18:52:00.204 | INFO | Analyzing chunk frames = [3648512,3649024) | 20260506 18:52:00.204 | INFO | MemInfo: { | 20260506 18:52:00.204 | INFO | "gpu" : " 17 GB", | 20260506 18:52:00.204 | INFO | "huge" : "", | 20260506 18:52:00.204 | INFO | "huge_shared" : " 27 GB", | 20260506 18:52:00.204 | INFO | "malloc" : " 14 GB", | 20260506 18:52:00.204 | INFO | "pinned" : " 29 GB" | 20260506 18:52:00.204 | INFO | } | 20260506 18:52:00.204 | INFO | Perf report for the last 5.14019 seconds: | 20260506 18:52:00.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:52:00.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:00.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:00.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:00.204 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 18:52:00.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:00.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:00.204 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:52:00.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:00.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:00.204 | INFO | Uploaded 36GiB to GPU in 2.01731s (17.8455GiB/s) | 20260506 18:52:00.204 | INFO | Downloaded 34GiB to GPU in 1.98735s (17.1082GiB/s) | 20260506 18:52:05.339 | INFO | Analyzing chunk frames = [3649024,3649536) | 20260506 18:52:05.339 | INFO | MemInfo: { | 20260506 18:52:05.339 | INFO | "gpu" : " 17 GB", | 20260506 18:52:05.339 | INFO | "huge" : "", | 20260506 18:52:05.339 | INFO | "huge_shared" : " 27 GB", | 20260506 18:52:05.340 | INFO | "malloc" : " 14 GB", | 20260506 18:52:05.340 | INFO | "pinned" : " 29 GB" | 20260506 18:52:05.340 | INFO | } | 20260506 18:52:05.340 | INFO | Perf report for the last 5.13571 seconds: | 20260506 18:52:05.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:52:05.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:05.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:05.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:05.340 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 18:52:05.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:05.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:05.340 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:52:05.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:05.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:05.340 | INFO | Uploaded 36GiB to GPU in 2.0346s (17.6939GiB/s) | 20260506 18:52:05.340 | INFO | Downloaded 34GiB to GPU in 2.01213s (16.8976GiB/s) | 20260506 18:52:05.456 | INFO | Starting a round of Baz IO | 20260506 18:52:08.586 | INFO | Populating RTMetrics report | 20260506 18:52:08.587 | INFO | RTMetrics report populated | 20260506 18:52:08.587 | INFO | Opening tmp RTMetrics json file | 20260506 18:52:08.619 | INFO | Serializing RTMetrics to json | 20260506 18:52:08.680 | INFO | Serialization done, swapping tmp file now... | 20260506 18:52:08.681 | INFO | RTMetrics done | 20260506 18:52:09.059 | INFO | Finished a round of Baz IO: Wrote 2.027 GiB in 3.603 seconds | 20260506 18:52:09.059 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.87% / 0.01% / 65.12% | 20260506 18:52:09.059 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.02829% / 3.97364e-06% / 9.53674e-05% | 20260506 18:52:09.059 | INFO | Baz IO overall: | 20260506 18:52:09.059 | INFO | BytesWritten : 1928514709643 | 20260506 18:52:09.059 | INFO | NumEvents : 932485604386 | 20260506 18:52:10.478 | INFO | Analyzing chunk frames = [3649536,3650048) | 20260506 18:52:10.478 | INFO | MemInfo: { | 20260506 18:52:10.478 | INFO | "gpu" : " 17 GB", | 20260506 18:52:10.478 | INFO | "huge" : "", | 20260506 18:52:10.479 | INFO | "huge_shared" : " 27 GB", | 20260506 18:52:10.479 | INFO | "malloc" : " 14 GB", | 20260506 18:52:10.479 | INFO | "pinned" : " 29 GB" | 20260506 18:52:10.479 | INFO | } | 20260506 18:52:10.479 | INFO | Perf report for the last 5.13906 seconds: | 20260506 18:52:10.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:52:10.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:10.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:10.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:10.479 | INFO | ANALYSIS : 62.6%, 2.9, 95.4ms, 0 | 20260506 18:52:10.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:10.479 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260506 18:52:10.479 | INFO | PRE_HQ : 60.0%, 1.9, 57.8ms, 0 | 20260506 18:52:10.479 | INFO | BAZWRITER : 69.8%, 4.8, 172.6ms, 0 | 20260506 18:52:10.479 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 18:52:10.479 | INFO | Uploaded 36GiB to GPU in 2.03058s (17.729GiB/s) | 20260506 18:52:10.479 | INFO | Downloaded 38GiB to GPU in 2.22913s (17.047GiB/s) | 20260506 18:52:15.622 | INFO | Analyzing chunk frames = [3650048,3650560) | 20260506 18:52:15.622 | INFO | MemInfo: { | 20260506 18:52:15.622 | INFO | "gpu" : " 17 GB", | 20260506 18:52:15.622 | INFO | "huge" : "", | 20260506 18:52:15.622 | INFO | "huge_shared" : " 27 GB", | 20260506 18:52:15.622 | INFO | "malloc" : " 14 GB", | 20260506 18:52:15.622 | INFO | "pinned" : " 29 GB" | 20260506 18:52:15.622 | INFO | } | 20260506 18:52:15.622 | INFO | Perf report for the last 5.14364 seconds: | 20260506 18:52:15.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:52:15.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:15.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:15.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:15.622 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 18:52:15.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:15.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:15.622 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 18:52:15.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:15.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:15.623 | INFO | Uploaded 36GiB to GPU in 2.02077s (17.815GiB/s) | 20260506 18:52:15.623 | INFO | Downloaded 34GiB to GPU in 1.95891s (17.3566GiB/s) | 20260506 18:52:20.759 | INFO | Analyzing chunk frames = [3650560,3651072) | 20260506 18:52:20.759 | INFO | MemInfo: { | 20260506 18:52:20.759 | INFO | "gpu" : " 17 GB", | 20260506 18:52:20.759 | INFO | "huge" : "", | 20260506 18:52:20.760 | INFO | "huge_shared" : " 27 GB", | 20260506 18:52:20.760 | INFO | "malloc" : " 14 GB", | 20260506 18:52:20.760 | INFO | "pinned" : " 29 GB" | 20260506 18:52:20.760 | INFO | } | 20260506 18:52:20.760 | INFO | Perf report for the last 5.13727 seconds: | 20260506 18:52:20.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:52:20.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:20.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:20.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:20.760 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:52:20.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:20.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:20.760 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:52:20.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:20.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:20.760 | INFO | Uploaded 36GiB to GPU in 1.96979s (18.276GiB/s) | 20260506 18:52:20.760 | INFO | Downloaded 34GiB to GPU in 1.93298s (17.5895GiB/s) | 20260506 18:52:25.899 | INFO | Analyzing chunk frames = [3651072,3651584) | 20260506 18:52:25.899 | INFO | MemInfo: { | 20260506 18:52:25.899 | INFO | "gpu" : " 17 GB", | 20260506 18:52:25.899 | INFO | "huge" : "", | 20260506 18:52:25.899 | INFO | "huge_shared" : " 27 GB", | 20260506 18:52:25.899 | INFO | "malloc" : " 14 GB", | 20260506 18:52:25.899 | INFO | "pinned" : " 29 GB" | 20260506 18:52:25.899 | INFO | } | 20260506 18:52:25.899 | INFO | Perf report for the last 5.13948 seconds: | 20260506 18:52:25.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:52:25.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:25.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:25.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:25.899 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:52:25.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:25.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:25.899 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:52:25.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:25.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:25.899 | INFO | Uploaded 36GiB to GPU in 1.92897s (18.6628GiB/s) | 20260506 18:52:25.899 | INFO | Downloaded 34GiB to GPU in 1.87635s (18.1203GiB/s) | 20260506 18:52:31.039 | INFO | Analyzing chunk frames = [3651584,3652096) | 20260506 18:52:31.039 | INFO | MemInfo: { | 20260506 18:52:31.039 | INFO | "gpu" : " 17 GB", | 20260506 18:52:31.039 | INFO | "huge" : "", | 20260506 18:52:31.039 | INFO | "huge_shared" : " 27 GB", | 20260506 18:52:31.039 | INFO | "malloc" : " 14 GB", | 20260506 18:52:31.039 | INFO | "pinned" : " 29 GB" | 20260506 18:52:31.039 | INFO | } | 20260506 18:52:31.039 | INFO | Perf report for the last 5.13979 seconds: | 20260506 18:52:31.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:52:31.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:31.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:31.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:31.039 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 18:52:31.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:31.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:31.039 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:52:31.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:31.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:31.039 | INFO | Uploaded 36GiB to GPU in 2.01175s (17.8948GiB/s) | 20260506 18:52:31.039 | INFO | Downloaded 34GiB to GPU in 1.91473s (17.7571GiB/s) | 20260506 18:52:36.177 | INFO | Analyzing chunk frames = [3652096,3652608) | 20260506 18:52:36.177 | INFO | MemInfo: { | 20260506 18:52:36.177 | INFO | "gpu" : " 17 GB", | 20260506 18:52:36.177 | INFO | "huge" : "", | 20260506 18:52:36.177 | INFO | "huge_shared" : " 27 GB", | 20260506 18:52:36.178 | INFO | "malloc" : " 14 GB", | 20260506 18:52:36.178 | INFO | "pinned" : " 29 GB" | 20260506 18:52:36.178 | INFO | } | 20260506 18:52:36.178 | INFO | Perf report for the last 5.1388 seconds: | 20260506 18:52:36.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:52:36.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:36.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:36.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:36.178 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 18:52:36.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:36.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:36.178 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 18:52:36.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:36.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:36.178 | INFO | Uploaded 36GiB to GPU in 2.02033s (17.8189GiB/s) | 20260506 18:52:36.178 | INFO | Downloaded 34GiB to GPU in 1.9529s (17.41GiB/s) | 20260506 18:52:41.315 | INFO | Analyzing chunk frames = [3652608,3653120) | 20260506 18:52:41.315 | INFO | MemInfo: { | 20260506 18:52:41.315 | INFO | "gpu" : " 17 GB", | 20260506 18:52:41.315 | INFO | "huge" : "", | 20260506 18:52:41.315 | INFO | "huge_shared" : " 27 GB", | 20260506 18:52:41.315 | INFO | "malloc" : " 14 GB", | 20260506 18:52:41.315 | INFO | "pinned" : " 29 GB" | 20260506 18:52:41.315 | INFO | } | 20260506 18:52:41.315 | INFO | Perf report for the last 5.13763 seconds: | 20260506 18:52:41.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:52:41.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:41.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:41.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:41.315 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 18:52:41.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:41.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:41.316 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 18:52:41.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:41.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:41.316 | INFO | Uploaded 36GiB to GPU in 2.00926s (17.9171GiB/s) | 20260506 18:52:41.316 | INFO | Downloaded 34GiB to GPU in 1.98907s (17.0935GiB/s) | 20260506 18:52:46.454 | INFO | Analyzing chunk frames = [3653120,3653632) | 20260506 18:52:46.454 | INFO | MemInfo: { | 20260506 18:52:46.454 | INFO | "gpu" : " 17 GB", | 20260506 18:52:46.454 | INFO | "huge" : "", | 20260506 18:52:46.454 | INFO | "huge_shared" : " 27 GB", | 20260506 18:52:46.454 | INFO | "malloc" : " 14 GB", | 20260506 18:52:46.454 | INFO | "pinned" : " 29 GB" | 20260506 18:52:46.454 | INFO | } | 20260506 18:52:46.454 | INFO | Perf report for the last 5.13847 seconds: | 20260506 18:52:46.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:52:46.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:46.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:46.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:46.454 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 18:52:46.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:46.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:46.454 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 18:52:46.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:46.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:46.454 | INFO | Uploaded 36GiB to GPU in 2.05872s (17.4866GiB/s) | 20260506 18:52:46.454 | INFO | Downloaded 34GiB to GPU in 1.9944s (17.0478GiB/s) | 20260506 18:52:46.454 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:52:46.454 | INFO | Total mem usage: 18 GiB | 20260506 18:52:46.454 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:52:46.454 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -10.1%) | 20260506 18:52:46.454 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:52:46.454 | INFO | High Water Breakdown: | 20260506 18:52:46.454 | INFO | 62.1% : 10.2 GiB ( 89% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:52:46.455 | INFO | 37.8% : 6.17 GiB ( 92.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:52:46.455 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:52:46.455 | INFO | ------------------------------------------ | 20260506 18:52:46.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:52:46.455 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:52:46.455 | INFO | Total mem usage: 26.5 GiB | 20260506 18:52:46.455 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:52:46.455 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:52:46.455 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:52:46.455 | INFO | High Water Breakdown: | 20260506 18:52:46.455 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:52:46.455 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:52:46.455 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:52:46.455 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:52:46.455 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:52:46.455 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:52:46.455 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:52:46.455 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:52:46.455 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:52:46.455 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:52:46.455 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:52:46.455 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:52:46.455 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:52:46.455 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:52:46.455 | INFO | ------------------------------------------ | 20260506 18:52:46.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:52:46.455 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:52:46.455 | INFO | Total mem usage: 25.6 GiB | 20260506 18:52:46.455 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:52:46.455 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:52:46.455 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:52:46.455 | INFO | High Water Breakdown: | 20260506 18:52:46.455 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:52:46.455 | INFO | ------------------------------------------ | 20260506 18:52:46.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:52:46.455 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:52:46.455 | INFO | Total mem usage: 15.7 GiB | 20260506 18:52:46.455 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:52:46.455 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:52:46.455 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 18:52:46.455 | INFO | High Water Breakdown: | 20260506 18:52:46.455 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:52:46.455 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:52:46.455 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:52:46.455 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:52:46.455 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:52:46.455 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:52:46.455 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:52:46.455 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:52:46.455 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:52:46.455 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:52:46.455 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:52:46.455 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:52:46.455 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:52:46.455 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:52:46.455 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:52:46.455 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:52:46.455 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:52:46.455 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:52:46.455 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:52:46.455 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:52:46.455 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:52:46.455 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:52:46.455 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:52:46.455 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:52:46.455 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:52:46.455 | INFO | ------------------------------------------ | 20260506 18:52:46.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:52:46.455 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:52:46.455 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:52:46.455 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:52:46.455 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.58e+01, 1.36e+01 / 2.36e+00 | 20260506 18:52:46.455 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 1.00e-02 / 6.70e-03 | 20260506 18:52:46.455 | INFO | PerfReport: 11.59%, Baseline -- 3.63e+00 / 4.43e+00, 5.52e-01 / 3.42e-01 | 20260506 18:52:46.455 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.84e-01 / 1.40e-01 | 20260506 18:52:46.455 | INFO | PerfReport: 15.97%, DME -- 4.00e+01 / 4.45e+01, 4.04e+00 / 2.46e+00 | 20260506 18:52:46.455 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.48e+00 / 1.95e+00 | 20260506 18:52:46.455 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.25e+00 / 2.81e+00, 3.74e-01 / 2.18e-01 | 20260506 18:52:46.455 | INFO | PerfReport: 3.38%, Metrics -- 1.06e+00 / 1.39e+00, 2.79e-01 / 1.86e-01 | 20260506 18:52:46.455 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.65e-01 / 4.09e-01, 5.63e-02 / 2.62e-02 | 20260506 18:52:46.455 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:52:46.455 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:52:46.455 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:52:46.455 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 18:52:46.455 | INFO | PerfReport: 21.48%, Upload -- 2.09e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 18:52:46.455 | INFO | PerfReport: 21.20%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 18:52:46.455 | INFO | PerfReport: 32.21%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 18:52:46.455 | INFO | PerfReport: 1.44%, LockUpload -- 1.40e+00 / 0.00e+00, 5.60e+00 / 0.00e+00 | 20260506 18:52:46.455 | INFO | PerfReport: 1.15%, LockDownload -- 1.11e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 18:52:46.455 | INFO | PerfReport: 22.51%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:52:46.455 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260506 18:52:46.574 | INFO | Starting a round of Baz IO | 20260506 18:52:49.746 | INFO | Populating RTMetrics report | 20260506 18:52:49.747 | INFO | RTMetrics report populated | 20260506 18:52:49.747 | INFO | Opening tmp RTMetrics json file | 20260506 18:52:49.859 | INFO | Serializing RTMetrics to json | 20260506 18:52:49.865 | INFO | Serialization done, swapping tmp file now... | 20260506 18:52:49.871 | INFO | RTMetrics done | 20260506 18:52:50.035 | INFO | Finished a round of Baz IO: Wrote 2.024 GiB in 3.461 seconds | 20260506 18:52:50.035 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.87% / 0.01% / 65.13% | 20260506 18:52:50.035 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.02195% / 7.94729e-06% / 9.13938e-05% | 20260506 18:52:50.035 | INFO | Baz IO overall: | 20260506 18:52:50.035 | INFO | BytesWritten : 1930687712514 | 20260506 18:52:50.035 | INFO | NumEvents : 933530406589 | 20260506 18:52:51.594 | INFO | Analyzing chunk frames = [3653632,3654144) | 20260506 18:52:51.595 | INFO | MemInfo: { | 20260506 18:52:51.595 | INFO | "gpu" : " 17 GB", | 20260506 18:52:51.595 | INFO | "huge" : "", | 20260506 18:52:51.595 | INFO | "huge_shared" : " 27 GB", | 20260506 18:52:51.595 | INFO | "malloc" : " 14 GB", | 20260506 18:52:51.595 | INFO | "pinned" : " 29 GB" | 20260506 18:52:51.595 | INFO | } | 20260506 18:52:51.595 | INFO | Perf report for the last 5.1407 seconds: | 20260506 18:52:51.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:52:51.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:51.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:51.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:51.595 | INFO | ANALYSIS : 63.1%, 2.9, 94.5ms, 0 | 20260506 18:52:51.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:51.595 | INFO | RT_METRICS : 47.8%, 1.0, 25.1ms, 0 | 20260506 18:52:51.595 | INFO | PRE_HQ : 60.7%, 1.9, 61.3ms, 0 | 20260506 18:52:51.595 | INFO | BAZWRITER : 67.1%, 4.8, 165.6ms, 0 | 20260506 18:52:51.595 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.3ms, 0 | 20260506 18:52:51.595 | INFO | Uploaded 36GiB to GPU in 2.04698s (17.5868GiB/s) | 20260506 18:52:51.595 | INFO | Downloaded 38GiB to GPU in 2.18936s (17.3567GiB/s) | 20260506 18:52:56.733 | INFO | Analyzing chunk frames = [3654144,3654656) | 20260506 18:52:56.733 | INFO | MemInfo: { | 20260506 18:52:56.733 | INFO | "gpu" : " 17 GB", | 20260506 18:52:56.733 | INFO | "huge" : "", | 20260506 18:52:56.733 | INFO | "huge_shared" : " 27 GB", | 20260506 18:52:56.733 | INFO | "malloc" : " 14 GB", | 20260506 18:52:56.733 | INFO | "pinned" : " 29 GB" | 20260506 18:52:56.733 | INFO | } | 20260506 18:52:56.733 | INFO | Perf report for the last 5.13834 seconds: | 20260506 18:52:56.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:52:56.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:56.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:56.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:56.733 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:52:56.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:56.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:56.733 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:52:56.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:56.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:52:56.733 | INFO | Uploaded 36GiB to GPU in 2.00176s (17.9841GiB/s) | 20260506 18:52:56.733 | INFO | Downloaded 34GiB to GPU in 1.93232s (17.5954GiB/s) | 20260506 18:53:01.881 | INFO | Analyzing chunk frames = [3654656,3655168) | 20260506 18:53:01.881 | INFO | MemInfo: { | 20260506 18:53:01.881 | INFO | "gpu" : " 17 GB", | 20260506 18:53:01.881 | INFO | "huge" : "", | 20260506 18:53:01.881 | INFO | "huge_shared" : " 27 GB", | 20260506 18:53:01.881 | INFO | "malloc" : " 14 GB", | 20260506 18:53:01.881 | INFO | "pinned" : " 29 GB" | 20260506 18:53:01.881 | INFO | } | 20260506 18:53:01.882 | INFO | Perf report for the last 5.14854 seconds: | 20260506 18:53:01.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:53:01.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:01.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:01.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:01.882 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 18:53:01.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:01.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:01.882 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:53:01.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:01.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:01.882 | INFO | Uploaded 36GiB to GPU in 1.96774s (18.2951GiB/s) | 20260506 18:53:01.882 | INFO | Downloaded 34GiB to GPU in 1.92147s (17.6948GiB/s) | 20260506 18:53:07.016 | INFO | Analyzing chunk frames = [3655168,3655680) | 20260506 18:53:07.016 | INFO | MemInfo: { | 20260506 18:53:07.016 | INFO | "gpu" : " 17 GB", | 20260506 18:53:07.017 | INFO | "huge" : "", | 20260506 18:53:07.017 | INFO | "huge_shared" : " 27 GB", | 20260506 18:53:07.017 | INFO | "malloc" : " 14 GB", | 20260506 18:53:07.017 | INFO | "pinned" : " 29 GB" | 20260506 18:53:07.017 | INFO | } | 20260506 18:53:07.017 | INFO | Perf report for the last 5.13516 seconds: | 20260506 18:53:07.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:53:07.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:07.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:07.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:07.017 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:53:07.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:07.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:07.017 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:53:07.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:07.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:07.017 | INFO | Uploaded 36GiB to GPU in 1.96417s (18.3283GiB/s) | 20260506 18:53:07.017 | INFO | Downloaded 34GiB to GPU in 1.87413s (18.1418GiB/s) | 20260506 18:53:12.154 | INFO | Analyzing chunk frames = [3655680,3656192) | 20260506 18:53:12.154 | INFO | MemInfo: { | 20260506 18:53:12.154 | INFO | "gpu" : " 17 GB", | 20260506 18:53:12.154 | INFO | "huge" : "", | 20260506 18:53:12.154 | INFO | "huge_shared" : " 27 GB", | 20260506 18:53:12.154 | INFO | "malloc" : " 14 GB", | 20260506 18:53:12.154 | INFO | "pinned" : " 29 GB" | 20260506 18:53:12.154 | INFO | } | 20260506 18:53:12.154 | INFO | Perf report for the last 5.13768 seconds: | 20260506 18:53:12.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:53:12.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:12.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:12.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:12.154 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260506 18:53:12.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:12.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:12.154 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:53:12.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:12.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:12.155 | INFO | Uploaded 36GiB to GPU in 2.01451s (17.8704GiB/s) | 20260506 18:53:12.155 | INFO | Downloaded 34GiB to GPU in 1.91903s (17.7173GiB/s) | 20260506 18:53:17.294 | INFO | Analyzing chunk frames = [3656192,3656704) | 20260506 18:53:17.294 | INFO | MemInfo: { | 20260506 18:53:17.294 | INFO | "gpu" : " 17 GB", | 20260506 18:53:17.294 | INFO | "huge" : "", | 20260506 18:53:17.294 | INFO | "huge_shared" : " 27 GB", | 20260506 18:53:17.294 | INFO | "malloc" : " 14 GB", | 20260506 18:53:17.294 | INFO | "pinned" : " 29 GB" | 20260506 18:53:17.294 | INFO | } | 20260506 18:53:17.295 | INFO | Perf report for the last 5.14015 seconds: | 20260506 18:53:17.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:53:17.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:17.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:17.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:17.295 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 18:53:17.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:17.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:17.295 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:53:17.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:17.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:17.295 | INFO | Uploaded 36GiB to GPU in 2.01762s (17.8428GiB/s) | 20260506 18:53:17.295 | INFO | Downloaded 34GiB to GPU in 1.94088s (17.5178GiB/s) | 20260506 18:53:22.430 | INFO | Analyzing chunk frames = [3656704,3657216) | 20260506 18:53:22.430 | INFO | MemInfo: { | 20260506 18:53:22.430 | INFO | "gpu" : " 17 GB", | 20260506 18:53:22.430 | INFO | "huge" : "", | 20260506 18:53:22.430 | INFO | "huge_shared" : " 27 GB", | 20260506 18:53:22.430 | INFO | "malloc" : " 14 GB", | 20260506 18:53:22.430 | INFO | "pinned" : " 29 GB" | 20260506 18:53:22.430 | INFO | } | 20260506 18:53:22.430 | INFO | Perf report for the last 5.13556 seconds: | 20260506 18:53:22.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:53:22.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:22.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:22.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:22.430 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 18:53:22.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:22.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:22.430 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:53:22.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:22.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:22.430 | INFO | Uploaded 36GiB to GPU in 2.0082s (17.9265GiB/s) | 20260506 18:53:22.431 | INFO | Downloaded 34GiB to GPU in 1.98696s (17.1116GiB/s) | 20260506 18:53:27.567 | INFO | Analyzing chunk frames = [3657216,3657728) | 20260506 18:53:27.567 | INFO | MemInfo: { | 20260506 18:53:27.567 | INFO | "gpu" : " 17 GB", | 20260506 18:53:27.567 | INFO | "huge" : "", | 20260506 18:53:27.567 | INFO | "huge_shared" : " 27 GB", | 20260506 18:53:27.567 | INFO | "malloc" : " 14 GB", | 20260506 18:53:27.567 | INFO | "pinned" : " 29 GB" | 20260506 18:53:27.567 | INFO | } | 20260506 18:53:27.567 | INFO | Perf report for the last 5.13738 seconds: | 20260506 18:53:27.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:53:27.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:27.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:27.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:27.568 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 18:53:27.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:27.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:27.568 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260506 18:53:27.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:27.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:27.568 | INFO | Uploaded 36GiB to GPU in 2.04385s (17.6138GiB/s) | 20260506 18:53:27.568 | INFO | Downloaded 34GiB to GPU in 1.99157s (17.0719GiB/s) | 20260506 18:53:27.689 | INFO | Starting a round of Baz IO | 20260506 18:53:30.822 | INFO | Populating RTMetrics report | 20260506 18:53:30.822 | INFO | RTMetrics report populated | 20260506 18:53:30.822 | INFO | Opening tmp RTMetrics json file | 20260506 18:53:30.837 | INFO | Serializing RTMetrics to json | 20260506 18:53:30.893 | INFO | Serialization done, swapping tmp file now... | 20260506 18:53:30.928 | INFO | RTMetrics done | 20260506 18:53:31.345 | INFO | Finished a round of Baz IO: Wrote 2.025 GiB in 3.656 seconds | 20260506 18:53:31.345 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.86% / 0.00% / 65.13% | 20260506 18:53:31.345 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.02202% / 7.94729e-06% / 8.34465e-05% | 20260506 18:53:31.346 | INFO | Baz IO overall: | 20260506 18:53:31.346 | INFO | BytesWritten : 1932862393349 | 20260506 18:53:31.346 | INFO | NumEvents : 934576857872 | 20260506 18:53:32.707 | INFO | Analyzing chunk frames = [3657728,3658240) | 20260506 18:53:32.707 | INFO | MemInfo: { | 20260506 18:53:32.707 | INFO | "gpu" : " 17 GB", | 20260506 18:53:32.707 | INFO | "huge" : "", | 20260506 18:53:32.707 | INFO | "huge_shared" : " 27 GB", | 20260506 18:53:32.707 | INFO | "malloc" : " 14 GB", | 20260506 18:53:32.707 | INFO | "pinned" : " 29 GB" | 20260506 18:53:32.707 | INFO | } | 20260506 18:53:32.707 | INFO | Perf report for the last 5.13989 seconds: | 20260506 18:53:32.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:53:32.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:32.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:32.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:32.707 | INFO | ANALYSIS : 62.8%, 2.9, 94.9ms, 0 | 20260506 18:53:32.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:32.708 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260506 18:53:32.708 | INFO | PRE_HQ : 60.5%, 1.9, 59.5ms, 0 | 20260506 18:53:32.708 | INFO | BAZWRITER : 70.6%, 4.8, 174.9ms, 0 | 20260506 18:53:32.708 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 18:53:32.708 | INFO | Uploaded 36GiB to GPU in 2.06282s (17.4518GiB/s) | 20260506 18:53:32.708 | INFO | Downloaded 38GiB to GPU in 2.23075s (17.0347GiB/s) | 20260506 18:53:37.847 | INFO | Analyzing chunk frames = [3658240,3658752) | 20260506 18:53:37.847 | INFO | MemInfo: { | 20260506 18:53:37.847 | INFO | "gpu" : " 17 GB", | 20260506 18:53:37.847 | INFO | "huge" : "", | 20260506 18:53:37.847 | INFO | "huge_shared" : " 27 GB", | 20260506 18:53:37.847 | INFO | "malloc" : " 14 GB", | 20260506 18:53:37.847 | INFO | "pinned" : " 29 GB" | 20260506 18:53:37.847 | INFO | } | 20260506 18:53:37.847 | INFO | Perf report for the last 5.13937 seconds: | 20260506 18:53:37.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:53:37.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:37.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:37.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:37.847 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 18:53:37.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:37.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:37.847 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:53:37.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:37.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:37.847 | INFO | Uploaded 36GiB to GPU in 1.99742s (18.0232GiB/s) | 20260506 18:53:37.847 | INFO | Downloaded 34GiB to GPU in 1.93825s (17.5416GiB/s) | 20260506 18:53:42.988 | INFO | Analyzing chunk frames = [3658752,3659264) | 20260506 18:53:42.988 | INFO | MemInfo: { | 20260506 18:53:42.988 | INFO | "gpu" : " 17 GB", | 20260506 18:53:42.988 | INFO | "huge" : "", | 20260506 18:53:42.988 | INFO | "huge_shared" : " 27 GB", | 20260506 18:53:42.988 | INFO | "malloc" : " 14 GB", | 20260506 18:53:42.989 | INFO | "pinned" : " 29 GB" | 20260506 18:53:42.989 | INFO | } | 20260506 18:53:42.989 | INFO | Perf report for the last 5.14179 seconds: | 20260506 18:53:42.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:53:42.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:42.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:42.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:42.989 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260506 18:53:42.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:42.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:42.989 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:53:42.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:42.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:42.989 | INFO | Uploaded 36GiB to GPU in 1.9303s (18.6499GiB/s) | 20260506 18:53:42.989 | INFO | Downloaded 34GiB to GPU in 1.88049s (18.0804GiB/s) | 20260506 18:53:48.131 | INFO | Analyzing chunk frames = [3659264,3659776) | 20260506 18:53:48.131 | INFO | MemInfo: { | 20260506 18:53:48.131 | INFO | "gpu" : " 17 GB", | 20260506 18:53:48.131 | INFO | "huge" : "", | 20260506 18:53:48.131 | INFO | "huge_shared" : " 27 GB", | 20260506 18:53:48.131 | INFO | "malloc" : " 14 GB", | 20260506 18:53:48.131 | INFO | "pinned" : " 29 GB" | 20260506 18:53:48.131 | INFO | } | 20260506 18:53:48.131 | INFO | Perf report for the last 5.14271 seconds: | 20260506 18:53:48.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:53:48.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:48.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:48.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:48.131 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 18:53:48.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:48.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:48.131 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:53:48.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:48.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:48.132 | INFO | Uploaded 36GiB to GPU in 1.91329s (18.8157GiB/s) | 20260506 18:53:48.132 | INFO | Downloaded 34GiB to GPU in 1.87231s (18.1594GiB/s) | 20260506 18:53:53.270 | INFO | Analyzing chunk frames = [3659776,3660288) | 20260506 18:53:53.270 | INFO | MemInfo: { | 20260506 18:53:53.270 | INFO | "gpu" : " 17 GB", | 20260506 18:53:53.270 | INFO | "huge" : "", | 20260506 18:53:53.270 | INFO | "huge_shared" : " 27 GB", | 20260506 18:53:53.270 | INFO | "malloc" : " 14 GB", | 20260506 18:53:53.270 | INFO | "pinned" : " 29 GB" | 20260506 18:53:53.270 | INFO | } | 20260506 18:53:53.270 | INFO | Perf report for the last 5.13878 seconds: | 20260506 18:53:53.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:53:53.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:53.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:53.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:53.270 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 18:53:53.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:53.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:53.270 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:53:53.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:53.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:53.270 | INFO | Uploaded 36GiB to GPU in 2.01314s (17.8825GiB/s) | 20260506 18:53:53.270 | INFO | Downloaded 34GiB to GPU in 1.91584s (17.7468GiB/s) | 20260506 18:53:58.408 | INFO | Analyzing chunk frames = [3660288,3660800) | 20260506 18:53:58.408 | INFO | MemInfo: { | 20260506 18:53:58.408 | INFO | "gpu" : " 17 GB", | 20260506 18:53:58.408 | INFO | "huge" : "", | 20260506 18:53:58.408 | INFO | "huge_shared" : " 27 GB", | 20260506 18:53:58.409 | INFO | "malloc" : " 14 GB", | 20260506 18:53:58.409 | INFO | "pinned" : " 29 GB" | 20260506 18:53:58.409 | INFO | } | 20260506 18:53:58.409 | INFO | Perf report for the last 5.13853 seconds: | 20260506 18:53:58.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:53:58.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:58.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:58.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:58.409 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 18:53:58.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:58.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:58.409 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:53:58.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:58.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:53:58.409 | INFO | Uploaded 36GiB to GPU in 2.06749s (17.4125GiB/s) | 20260506 18:53:58.409 | INFO | Downloaded 34GiB to GPU in 1.96659s (17.2888GiB/s) | 20260506 18:54:03.550 | INFO | Analyzing chunk frames = [3660800,3661312) | 20260506 18:54:03.551 | INFO | MemInfo: { | 20260506 18:54:03.551 | INFO | "gpu" : " 17 GB", | 20260506 18:54:03.551 | INFO | "huge" : "", | 20260506 18:54:03.551 | INFO | "huge_shared" : " 27 GB", | 20260506 18:54:03.551 | INFO | "malloc" : " 14 GB", | 20260506 18:54:03.551 | INFO | "pinned" : " 29 GB" | 20260506 18:54:03.551 | INFO | } | 20260506 18:54:03.551 | INFO | Perf report for the last 5.14212 seconds: | 20260506 18:54:03.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:54:03.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:03.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:03.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:03.551 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 18:54:03.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:03.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:03.551 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:54:03.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:03.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:03.551 | INFO | Uploaded 36GiB to GPU in 1.98817s (18.1071GiB/s) | 20260506 18:54:03.551 | INFO | Downloaded 34GiB to GPU in 1.97572s (17.2089GiB/s) | 20260506 18:54:08.683 | INFO | Analyzing chunk frames = [3661312,3661824) | 20260506 18:54:08.683 | INFO | MemInfo: { | 20260506 18:54:08.683 | INFO | "gpu" : " 17 GB", | 20260506 18:54:08.683 | INFO | "huge" : "", | 20260506 18:54:08.683 | INFO | "huge_shared" : " 27 GB", | 20260506 18:54:08.683 | INFO | "malloc" : " 14 GB", | 20260506 18:54:08.684 | INFO | "pinned" : " 29 GB" | 20260506 18:54:08.684 | INFO | } | 20260506 18:54:08.684 | INFO | Perf report for the last 5.1328 seconds: | 20260506 18:54:08.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:54:08.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:08.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:08.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:08.684 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 18:54:08.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:08.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:08.684 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 18:54:08.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:08.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:08.684 | INFO | Uploaded 36GiB to GPU in 2.01746s (17.8442GiB/s) | 20260506 18:54:08.684 | INFO | Downloaded 34GiB to GPU in 1.98546s (17.1245GiB/s) | 20260506 18:54:08.684 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:54:08.684 | INFO | Total mem usage: 18 GiB | 20260506 18:54:08.684 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:54:08.684 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.2%) | 20260506 18:54:08.684 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:54:08.684 | INFO | High Water Breakdown: | 20260506 18:54:08.684 | INFO | 61.7% : 9.81 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:54:08.684 | INFO | 38.2% : 6.07 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:54:08.684 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:54:08.684 | INFO | ------------------------------------------ | 20260506 18:54:08.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:54:08.684 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:54:08.684 | INFO | Total mem usage: 26.5 GiB | 20260506 18:54:08.684 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:54:08.684 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260506 18:54:08.684 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260506 18:54:08.684 | INFO | High Water Breakdown: | 20260506 18:54:08.684 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:54:08.684 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:54:08.684 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:54:08.684 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:54:08.684 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:54:08.684 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:54:08.684 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:54:08.684 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:54:08.684 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:54:08.685 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:54:08.685 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:54:08.685 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:54:08.685 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:54:08.685 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:54:08.685 | INFO | ------------------------------------------ | 20260506 18:54:08.685 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:54:08.685 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:54:08.685 | INFO | Total mem usage: 25.6 GiB | 20260506 18:54:08.685 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:54:08.685 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:54:08.685 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:54:08.685 | INFO | High Water Breakdown: | 20260506 18:54:08.685 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:54:08.685 | INFO | ------------------------------------------ | 20260506 18:54:08.685 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:54:08.685 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:54:08.685 | INFO | Total mem usage: 15.7 GiB | 20260506 18:54:08.685 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:54:08.685 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:54:08.685 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 18:54:08.685 | INFO | High Water Breakdown: | 20260506 18:54:08.685 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:54:08.685 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:54:08.685 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:54:08.685 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:54:08.685 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:54:08.685 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:54:08.685 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:54:08.685 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:54:08.685 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:54:08.685 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:54:08.685 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:54:08.685 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:54:08.685 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:54:08.685 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:54:08.685 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:54:08.685 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:54:08.685 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:54:08.685 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:54:08.685 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:54:08.685 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:54:08.685 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:54:08.685 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:54:08.685 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:54:08.685 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:54:08.685 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:54:08.685 | INFO | ------------------------------------------ | 20260506 18:54:08.685 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:54:08.685 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:54:08.685 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:54:08.685 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:54:08.685 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.55e+01, 1.35e+01 / 1.93e+00 | 20260506 18:54:08.685 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 1.02e-02 / 4.75e-03 | 20260506 18:54:08.685 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.44e+00, 5.56e-01 / 3.38e-01 | 20260506 18:54:08.685 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.46e+00, 1.82e-01 / 1.35e-01 | 20260506 18:54:08.685 | INFO | PerfReport: 15.97%, DME -- 3.99e+01 / 4.46e+01, 4.09e+00 / 2.20e+00 | 20260506 18:54:08.685 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.79e+01 / 2.03e+01, 1.46e+00 / 1.96e+00 | 20260506 18:54:08.685 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.25e+00 / 2.79e+00, 3.85e-01 / 2.65e-01 | 20260506 18:54:08.685 | INFO | PerfReport: 3.37%, Metrics -- 1.06e+00 / 1.40e+00, 2.77e-01 / 2.00e-01 | 20260506 18:54:08.685 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.65e-01 / 4.12e-01, 5.64e-02 / 2.67e-02 | 20260506 18:54:08.685 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:54:08.685 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:54:08.685 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:54:08.685 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 18:54:08.685 | INFO | PerfReport: 21.52%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260506 18:54:08.685 | INFO | PerfReport: 21.16%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 18:54:08.685 | INFO | PerfReport: 32.24%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:54:08.685 | INFO | PerfReport: 1.46%, LockUpload -- 1.42e+00 / 0.00e+00, 5.62e+00 / 0.00e+00 | 20260506 18:54:08.685 | INFO | PerfReport: 1.12%, LockDownload -- 1.09e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260506 18:54:08.685 | INFO | PerfReport: 22.48%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:54:08.685 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260506 18:54:08.805 | INFO | Starting a round of Baz IO | 20260506 18:54:11.966 | INFO | Populating RTMetrics report | 20260506 18:54:11.967 | INFO | RTMetrics report populated | 20260506 18:54:11.967 | INFO | Opening tmp RTMetrics json file | 20260506 18:54:12.020 | INFO | Serializing RTMetrics to json | 20260506 18:54:12.024 | INFO | Serialization done, swapping tmp file now... | 20260506 18:54:12.029 | INFO | RTMetrics done | 20260506 18:54:12.318 | INFO | Finished a round of Baz IO: Wrote 2.023 GiB in 3.513 seconds | 20260506 18:54:12.318 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.86% / 0.01% / 65.14% | 20260506 18:54:12.318 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.01993% / 1.98682e-05% / 7.94729e-05% | 20260506 18:54:12.318 | INFO | Baz IO overall: | 20260506 18:54:12.318 | INFO | BytesWritten : 1935034880214 | 20260506 18:54:12.318 | INFO | NumEvents : 935621791994 | 20260506 18:54:13.823 | INFO | Analyzing chunk frames = [3661824,3662336) | 20260506 18:54:13.823 | INFO | MemInfo: { | 20260506 18:54:13.823 | INFO | "gpu" : " 17 GB", | 20260506 18:54:13.823 | INFO | "huge" : "", | 20260506 18:54:13.823 | INFO | "huge_shared" : " 27 GB", | 20260506 18:54:13.823 | INFO | "malloc" : " 14 GB", | 20260506 18:54:13.823 | INFO | "pinned" : " 29 GB" | 20260506 18:54:13.823 | INFO | } | 20260506 18:54:13.823 | INFO | Perf report for the last 5.13941 seconds: | 20260506 18:54:13.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:54:13.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:13.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:13.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:13.823 | INFO | ANALYSIS : 63.0%, 2.9, 94.4ms, 0 | 20260506 18:54:13.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:13.823 | INFO | RT_METRICS : 47.6%, 1.0, 25.0ms, 0 | 20260506 18:54:13.823 | INFO | PRE_HQ : 61.1%, 2.0, 62.0ms, 0 | 20260506 18:54:13.823 | INFO | BAZWRITER : 67.7%, 4.8, 166.9ms, 0 | 20260506 18:54:13.823 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 18:54:13.823 | INFO | Uploaded 36GiB to GPU in 2.05507s (17.5176GiB/s) | 20260506 18:54:13.823 | INFO | Downloaded 38GiB to GPU in 2.22018s (17.1157GiB/s) | 20260506 18:54:18.968 | INFO | Analyzing chunk frames = [3662336,3662848) | 20260506 18:54:18.968 | INFO | MemInfo: { | 20260506 18:54:18.968 | INFO | "gpu" : " 17 GB", | 20260506 18:54:18.968 | INFO | "huge" : "", | 20260506 18:54:18.968 | INFO | "huge_shared" : " 27 GB", | 20260506 18:54:18.968 | INFO | "malloc" : " 14 GB", | 20260506 18:54:18.968 | INFO | "pinned" : " 29 GB" | 20260506 18:54:18.968 | INFO | } | 20260506 18:54:18.968 | INFO | Perf report for the last 5.14522 seconds: | 20260506 18:54:18.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:54:18.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:18.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:18.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:18.968 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 18:54:18.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:18.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:18.968 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 18:54:18.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:18.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:18.969 | INFO | Uploaded 36GiB to GPU in 1.99143s (18.0775GiB/s) | 20260506 18:54:18.969 | INFO | Downloaded 34GiB to GPU in 1.93336s (17.586GiB/s) | 20260506 18:54:24.105 | INFO | Analyzing chunk frames = [3662848,3663360) | 20260506 18:54:24.105 | INFO | MemInfo: { | 20260506 18:54:24.105 | INFO | "gpu" : " 17 GB", | 20260506 18:54:24.105 | INFO | "huge" : "", | 20260506 18:54:24.105 | INFO | "huge_shared" : " 27 GB", | 20260506 18:54:24.105 | INFO | "malloc" : " 14 GB", | 20260506 18:54:24.105 | INFO | "pinned" : " 29 GB" | 20260506 18:54:24.105 | INFO | } | 20260506 18:54:24.106 | INFO | Perf report for the last 5.13728 seconds: | 20260506 18:54:24.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:54:24.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:24.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:24.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:24.106 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 18:54:24.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:24.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:24.106 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:54:24.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:24.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:24.106 | INFO | Uploaded 36GiB to GPU in 1.9546s (18.4181GiB/s) | 20260506 18:54:24.106 | INFO | Downloaded 34GiB to GPU in 1.92535s (17.6591GiB/s) | 20260506 18:54:29.243 | INFO | Analyzing chunk frames = [3663360,3663872) | 20260506 18:54:29.243 | INFO | MemInfo: { | 20260506 18:54:29.243 | INFO | "gpu" : " 17 GB", | 20260506 18:54:29.244 | INFO | "huge" : "", | 20260506 18:54:29.244 | INFO | "huge_shared" : " 27 GB", | 20260506 18:54:29.244 | INFO | "malloc" : " 14 GB", | 20260506 18:54:29.244 | INFO | "pinned" : " 29 GB" | 20260506 18:54:29.244 | INFO | } | 20260506 18:54:29.244 | INFO | Perf report for the last 5.13814 seconds: | 20260506 18:54:29.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:54:29.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:29.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:29.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:29.244 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 18:54:29.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:29.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:29.244 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:54:29.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:29.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:29.244 | INFO | Uploaded 36GiB to GPU in 1.93286s (18.6252GiB/s) | 20260506 18:54:29.244 | INFO | Downloaded 34GiB to GPU in 1.87761s (18.1082GiB/s) | 20260506 18:54:34.383 | INFO | Analyzing chunk frames = [3663872,3664384) | 20260506 18:54:34.383 | INFO | MemInfo: { | 20260506 18:54:34.383 | INFO | "gpu" : " 17 GB", | 20260506 18:54:34.383 | INFO | "huge" : "", | 20260506 18:54:34.383 | INFO | "huge_shared" : " 27 GB", | 20260506 18:54:34.383 | INFO | "malloc" : " 14 GB", | 20260506 18:54:34.383 | INFO | "pinned" : " 29 GB" | 20260506 18:54:34.383 | INFO | } | 20260506 18:54:34.384 | INFO | Perf report for the last 5.13981 seconds: | 20260506 18:54:34.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:54:34.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:34.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:34.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:34.384 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 18:54:34.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:34.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:34.384 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 18:54:34.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:34.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:34.384 | INFO | Uploaded 36GiB to GPU in 2.01069s (17.9043GiB/s) | 20260506 18:54:34.384 | INFO | Downloaded 34GiB to GPU in 1.91032s (17.798GiB/s) | 20260506 18:54:39.520 | INFO | Analyzing chunk frames = [3664384,3664896) | 20260506 18:54:39.520 | INFO | MemInfo: { | 20260506 18:54:39.520 | INFO | "gpu" : " 17 GB", | 20260506 18:54:39.520 | INFO | "huge" : "", | 20260506 18:54:39.521 | INFO | "huge_shared" : " 27 GB", | 20260506 18:54:39.521 | INFO | "malloc" : " 14 GB", | 20260506 18:54:39.521 | INFO | "pinned" : " 29 GB" | 20260506 18:54:39.521 | INFO | } | 20260506 18:54:39.521 | INFO | Perf report for the last 5.13721 seconds: | 20260506 18:54:39.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:54:39.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:39.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:39.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:39.521 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 18:54:39.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:39.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:39.521 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 18:54:39.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:39.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:39.521 | INFO | Uploaded 36GiB to GPU in 2.00554s (17.9503GiB/s) | 20260506 18:54:39.521 | INFO | Downloaded 34GiB to GPU in 1.9508s (17.4288GiB/s) | 20260506 18:54:44.659 | INFO | Analyzing chunk frames = [3664896,3665408) | 20260506 18:54:44.659 | INFO | MemInfo: { | 20260506 18:54:44.659 | INFO | "gpu" : " 17 GB", | 20260506 18:54:44.659 | INFO | "huge" : "", | 20260506 18:54:44.659 | INFO | "huge_shared" : " 27 GB", | 20260506 18:54:44.659 | INFO | "malloc" : " 14 GB", | 20260506 18:54:44.659 | INFO | "pinned" : " 29 GB" | 20260506 18:54:44.659 | INFO | } | 20260506 18:54:44.659 | INFO | Perf report for the last 5.13861 seconds: | 20260506 18:54:44.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:54:44.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:44.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:44.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:44.659 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:54:44.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:44.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:44.659 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:54:44.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:44.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:44.660 | INFO | Uploaded 36GiB to GPU in 1.97917s (18.1894GiB/s) | 20260506 18:54:44.660 | INFO | Downloaded 34GiB to GPU in 2.00783s (16.9337GiB/s) | 20260506 18:54:49.796 | INFO | Analyzing chunk frames = [3665408,3665920) | 20260506 18:54:49.796 | INFO | MemInfo: { | 20260506 18:54:49.796 | INFO | "gpu" : " 17 GB", | 20260506 18:54:49.796 | INFO | "huge" : "", | 20260506 18:54:49.796 | INFO | "huge_shared" : " 27 GB", | 20260506 18:54:49.796 | INFO | "malloc" : " 14 GB", | 20260506 18:54:49.796 | INFO | "pinned" : " 29 GB" | 20260506 18:54:49.796 | INFO | } | 20260506 18:54:49.796 | INFO | Perf report for the last 5.13684 seconds: | 20260506 18:54:49.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:54:49.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:49.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:49.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:49.796 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 18:54:49.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:49.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:49.796 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 18:54:49.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:49.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:49.796 | INFO | Uploaded 36GiB to GPU in 2.04449s (17.6083GiB/s) | 20260506 18:54:49.797 | INFO | Downloaded 34GiB to GPU in 1.98826s (17.1004GiB/s) | 20260506 18:54:49.915 | INFO | Starting a round of Baz IO | 20260506 18:54:53.049 | INFO | Populating RTMetrics report | 20260506 18:54:53.049 | INFO | RTMetrics report populated | 20260506 18:54:53.049 | INFO | Opening tmp RTMetrics json file | 20260506 18:54:53.056 | INFO | Serializing RTMetrics to json | 20260506 18:54:53.097 | INFO | Serialization done, swapping tmp file now... | 20260506 18:54:53.098 | INFO | RTMetrics done | 20260506 18:54:53.226 | INFO | Finished a round of Baz IO: Wrote 2.024 GiB in 3.311 seconds | 20260506 18:54:53.227 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.85% / 0.01% / 65.14% | 20260506 18:54:53.227 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.01815% / 1.19209e-05% / 7.54992e-05% | 20260506 18:54:53.227 | INFO | Baz IO overall: | 20260506 18:54:53.227 | INFO | BytesWritten : 1937208019486 | 20260506 18:54:53.227 | INFO | NumEvents : 936667096440 | 20260506 18:54:54.940 | INFO | Analyzing chunk frames = [3665920,3666432) | 20260506 18:54:54.940 | INFO | MemInfo: { | 20260506 18:54:54.940 | INFO | "gpu" : " 17 GB", | 20260506 18:54:54.940 | INFO | "huge" : "", | 20260506 18:54:54.940 | INFO | "huge_shared" : " 27 GB", | 20260506 18:54:54.940 | INFO | "malloc" : " 14 GB", | 20260506 18:54:54.940 | INFO | "pinned" : " 29 GB" | 20260506 18:54:54.940 | INFO | } | 20260506 18:54:54.940 | INFO | Perf report for the last 5.14405 seconds: | 20260506 18:54:54.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:54:54.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:54.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:54.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:54.940 | INFO | ANALYSIS : 62.7%, 2.9, 95.3ms, 0 | 20260506 18:54:54.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:54:54.940 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 18:54:54.940 | INFO | PRE_HQ : 60.2%, 1.9, 58.1ms, 0 | 20260506 18:54:54.940 | INFO | BAZWRITER : 63.8%, 4.8, 157.7ms, 0 | 20260506 18:54:54.940 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 18:54:54.941 | INFO | Uploaded 36GiB to GPU in 2.0633s (17.4478GiB/s) | 20260506 18:54:54.941 | INFO | Downloaded 38GiB to GPU in 2.22464s (17.0814GiB/s) | 20260506 18:55:00.082 | INFO | Analyzing chunk frames = [3666432,3666944) | 20260506 18:55:00.082 | INFO | MemInfo: { | 20260506 18:55:00.082 | INFO | "gpu" : " 17 GB", | 20260506 18:55:00.082 | INFO | "huge" : "", | 20260506 18:55:00.082 | INFO | "huge_shared" : " 27 GB", | 20260506 18:55:00.082 | INFO | "malloc" : " 14 GB", | 20260506 18:55:00.082 | INFO | "pinned" : " 29 GB" | 20260506 18:55:00.082 | INFO | } | 20260506 18:55:00.082 | INFO | Perf report for the last 5.14206 seconds: | 20260506 18:55:00.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:55:00.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:00.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:00.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:00.082 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 18:55:00.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:00.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:00.082 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:55:00.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:00.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:00.083 | INFO | Uploaded 36GiB to GPU in 2.01957s (17.8255GiB/s) | 20260506 18:55:00.083 | INFO | Downloaded 34GiB to GPU in 1.94056s (17.5208GiB/s) | 20260506 18:55:05.227 | INFO | Analyzing chunk frames = [3666944,3667456) | 20260506 18:55:05.227 | INFO | MemInfo: { | 20260506 18:55:05.227 | INFO | "gpu" : " 17 GB", | 20260506 18:55:05.227 | INFO | "huge" : "", | 20260506 18:55:05.227 | INFO | "huge_shared" : " 27 GB", | 20260506 18:55:05.227 | INFO | "malloc" : " 14 GB", | 20260506 18:55:05.227 | INFO | "pinned" : " 29 GB" | 20260506 18:55:05.227 | INFO | } | 20260506 18:55:05.227 | INFO | Perf report for the last 5.14503 seconds: | 20260506 18:55:05.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:55:05.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:05.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:05.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:05.227 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 18:55:05.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:05.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:05.228 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:55:05.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:05.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:05.228 | INFO | Uploaded 36GiB to GPU in 1.98548s (18.1316GiB/s) | 20260506 18:55:05.228 | INFO | Downloaded 34GiB to GPU in 1.91754s (17.731GiB/s) | 20260506 18:55:10.361 | INFO | Analyzing chunk frames = [3667456,3667968) | 20260506 18:55:10.361 | INFO | MemInfo: { | 20260506 18:55:10.361 | INFO | "gpu" : " 17 GB", | 20260506 18:55:10.361 | INFO | "huge" : "", | 20260506 18:55:10.361 | INFO | "huge_shared" : " 27 GB", | 20260506 18:55:10.361 | INFO | "malloc" : " 14 GB", | 20260506 18:55:10.361 | INFO | "pinned" : " 29 GB" | 20260506 18:55:10.361 | INFO | } | 20260506 18:55:10.361 | INFO | Perf report for the last 5.13375 seconds: | 20260506 18:55:10.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:55:10.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:10.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:10.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:10.361 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 18:55:10.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:10.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:10.361 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:55:10.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:10.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:10.361 | INFO | Uploaded 36GiB to GPU in 1.96334s (18.3361GiB/s) | 20260506 18:55:10.361 | INFO | Downloaded 34GiB to GPU in 1.87448s (18.1384GiB/s) | 20260506 18:55:15.496 | INFO | Analyzing chunk frames = [3667968,3668480) | 20260506 18:55:15.496 | INFO | MemInfo: { | 20260506 18:55:15.496 | INFO | "gpu" : " 17 GB", | 20260506 18:55:15.496 | INFO | "huge" : "", | 20260506 18:55:15.496 | INFO | "huge_shared" : " 27 GB", | 20260506 18:55:15.496 | INFO | "malloc" : " 14 GB", | 20260506 18:55:15.496 | INFO | "pinned" : " 29 GB" | 20260506 18:55:15.496 | INFO | } | 20260506 18:55:15.496 | INFO | Perf report for the last 5.13509 seconds: | 20260506 18:55:15.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:55:15.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:15.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:15.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:15.496 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 18:55:15.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:15.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:15.496 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 18:55:15.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:15.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:15.497 | INFO | Uploaded 36GiB to GPU in 2.02643s (17.7652GiB/s) | 20260506 18:55:15.497 | INFO | Downloaded 34GiB to GPU in 1.91462s (17.7581GiB/s) | 20260506 18:55:20.634 | INFO | Analyzing chunk frames = [3668480,3668992) | 20260506 18:55:20.634 | INFO | MemInfo: { | 20260506 18:55:20.634 | INFO | "gpu" : " 17 GB", | 20260506 18:55:20.634 | INFO | "huge" : "", | 20260506 18:55:20.634 | INFO | "huge_shared" : " 27 GB", | 20260506 18:55:20.634 | INFO | "malloc" : " 14 GB", | 20260506 18:55:20.634 | INFO | "pinned" : " 29 GB" | 20260506 18:55:20.634 | INFO | } | 20260506 18:55:20.634 | INFO | Perf report for the last 5.1375 seconds: | 20260506 18:55:20.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:55:20.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:20.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:20.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:20.634 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 18:55:20.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:20.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:20.634 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:55:20.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:20.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:20.634 | INFO | Uploaded 36GiB to GPU in 2.037s (17.673GiB/s) | 20260506 18:55:20.634 | INFO | Downloaded 34GiB to GPU in 1.94744s (17.4588GiB/s) | 20260506 18:55:25.769 | INFO | Analyzing chunk frames = [3668992,3669504) | 20260506 18:55:25.769 | INFO | MemInfo: { | 20260506 18:55:25.769 | INFO | "gpu" : " 17 GB", | 20260506 18:55:25.769 | INFO | "huge" : "", | 20260506 18:55:25.769 | INFO | "huge_shared" : " 27 GB", | 20260506 18:55:25.769 | INFO | "malloc" : " 14 GB", | 20260506 18:55:25.769 | INFO | "pinned" : " 29 GB" | 20260506 18:55:25.769 | INFO | } | 20260506 18:55:25.769 | INFO | Perf report for the last 5.13532 seconds: | 20260506 18:55:25.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:55:25.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:25.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:25.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:25.769 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 18:55:25.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:25.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:25.769 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 18:55:25.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:25.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:25.769 | INFO | Uploaded 36GiB to GPU in 1.98008s (18.1811GiB/s) | 20260506 18:55:25.769 | INFO | Downloaded 34GiB to GPU in 1.99582s (17.0356GiB/s) | 20260506 18:55:30.913 | INFO | Analyzing chunk frames = [3669504,3670016) | 20260506 18:55:30.913 | INFO | MemInfo: { | 20260506 18:55:30.913 | INFO | "gpu" : " 17 GB", | 20260506 18:55:30.913 | INFO | "huge" : "", | 20260506 18:55:30.913 | INFO | "huge_shared" : " 27 GB", | 20260506 18:55:30.913 | INFO | "malloc" : " 14 GB", | 20260506 18:55:30.913 | INFO | "pinned" : " 29 GB" | 20260506 18:55:30.913 | INFO | } | 20260506 18:55:30.913 | INFO | Perf report for the last 5.14429 seconds: | 20260506 18:55:30.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:55:30.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:30.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:30.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:30.913 | INFO | ANALYSIS : 62.4%, 3.0, 97.3ms, 0 | 20260506 18:55:30.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:30.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:30.914 | INFO | PRE_HQ : 32.1%, 1.0, 16.7ms, 0 | 20260506 18:55:30.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:30.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:30.914 | INFO | Uploaded 36GiB to GPU in 2.08276s (17.2848GiB/s) | 20260506 18:55:30.914 | INFO | Downloaded 34GiB to GPU in 2.02014s (16.8305GiB/s) | 20260506 18:55:30.914 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:55:30.914 | INFO | Total mem usage: 18 GiB | 20260506 18:55:30.914 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:55:30.914 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.4%) | 20260506 18:55:30.914 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:55:30.914 | INFO | High Water Breakdown: | 20260506 18:55:30.914 | INFO | 61.3% : 9.48 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:55:30.914 | INFO | 38.6% : 5.97 GiB ( 89.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:55:30.914 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:55:30.914 | INFO | ------------------------------------------ | 20260506 18:55:30.914 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:55:30.914 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:55:30.914 | INFO | Total mem usage: 26.5 GiB | 20260506 18:55:30.914 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:55:30.914 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:55:30.914 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:55:30.914 | INFO | High Water Breakdown: | 20260506 18:55:30.914 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:55:30.914 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:55:30.914 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:55:30.914 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:55:30.914 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:55:30.914 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:55:30.914 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:55:30.914 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:55:30.914 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:55:30.914 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:55:30.914 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:55:30.914 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:55:30.914 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:55:30.914 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:55:30.914 | INFO | ------------------------------------------ | 20260506 18:55:30.914 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:55:30.914 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:55:30.914 | INFO | Total mem usage: 25.6 GiB | 20260506 18:55:30.914 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:55:30.914 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:55:30.914 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:55:30.914 | INFO | High Water Breakdown: | 20260506 18:55:30.914 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:55:30.914 | INFO | ------------------------------------------ | 20260506 18:55:30.914 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:55:30.914 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:55:30.914 | INFO | Total mem usage: 15.7 GiB | 20260506 18:55:30.914 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:55:30.914 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:55:30.914 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 18:55:30.914 | INFO | High Water Breakdown: | 20260506 18:55:30.914 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:55:30.914 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:55:30.914 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:55:30.914 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:55:30.914 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:55:30.914 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:55:30.914 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:55:30.914 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:55:30.914 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:55:30.914 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:55:30.914 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:55:30.914 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:55:30.914 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:55:30.914 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:55:30.914 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:55:30.914 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:55:30.914 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:55:30.914 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:55:30.914 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:55:30.914 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:55:30.914 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:55:30.914 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:55:30.914 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:55:30.914 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:55:30.915 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:55:30.915 | INFO | ------------------------------------------ | 20260506 18:55:30.915 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:55:30.915 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:55:30.915 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:55:30.915 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:55:30.915 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.62e+01, 1.35e+01 / 2.48e+00 | 20260506 18:55:30.915 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.01e-01, 1.06e-02 / 6.46e-03 | 20260506 18:55:30.915 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.42e+00, 5.50e-01 / 3.10e-01 | 20260506 18:55:30.915 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.48e+00, 1.84e-01 / 1.33e-01 | 20260506 18:55:30.915 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.51e+01, 4.24e+00 / 2.44e+00 | 20260506 18:55:30.915 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.38e+00 / 1.07e+00 | 20260506 18:55:30.915 | INFO | PerfReport: 7.22%, PulseAccumulating -- 2.26e+00 / 2.85e+00, 3.85e-01 / 1.42e-01 | 20260506 18:55:30.915 | INFO | PerfReport: 3.38%, Metrics -- 1.06e+00 / 1.41e+00, 2.74e-01 / 1.46e-01 | 20260506 18:55:30.915 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.65e-01 / 4.13e-01, 5.63e-02 / 2.42e-02 | 20260506 18:55:30.915 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:55:30.915 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:55:30.915 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:55:30.915 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 18:55:30.915 | INFO | PerfReport: 21.56%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 18:55:30.915 | INFO | PerfReport: 21.24%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 18:55:30.915 | INFO | PerfReport: 32.21%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 18:55:30.915 | INFO | PerfReport: 1.44%, LockUpload -- 1.39e+00 / 0.00e+00, 5.58e+00 / 0.00e+00 | 20260506 18:55:30.915 | INFO | PerfReport: 1.18%, LockDownload -- 1.14e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 18:55:30.915 | INFO | PerfReport: 22.35%, LockCompute -- 2.17e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:55:30.915 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260506 18:55:31.030 | INFO | Starting a round of Baz IO | 20260506 18:55:34.173 | INFO | Populating RTMetrics report | 20260506 18:55:34.174 | INFO | RTMetrics report populated | 20260506 18:55:34.174 | INFO | Opening tmp RTMetrics json file | 20260506 18:55:34.201 | INFO | Serializing RTMetrics to json | 20260506 18:55:34.217 | INFO | Serialization done, swapping tmp file now... | 20260506 18:55:34.232 | INFO | RTMetrics done | 20260506 18:55:34.654 | INFO | Finished a round of Baz IO: Wrote 2.026 GiB in 3.623 seconds | 20260506 18:55:34.654 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.85% / 0.01% / 65.15% | 20260506 18:55:34.654 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.0184% / 1.58946e-05% / 7.54992e-05% | 20260506 18:55:34.654 | INFO | Baz IO overall: | 20260506 18:55:34.654 | INFO | BytesWritten : 1939383163720 | 20260506 18:55:34.654 | INFO | NumEvents : 937713998311 | 20260506 18:55:36.060 | INFO | Analyzing chunk frames = [3670016,3670528) | 20260506 18:55:36.060 | INFO | MemInfo: { | 20260506 18:55:36.060 | INFO | "gpu" : " 17 GB", | 20260506 18:55:36.060 | INFO | "huge" : "", | 20260506 18:55:36.060 | INFO | "huge_shared" : " 27 GB", | 20260506 18:55:36.060 | INFO | "malloc" : " 14 GB", | 20260506 18:55:36.060 | INFO | "pinned" : " 29 GB" | 20260506 18:55:36.060 | INFO | } | 20260506 18:55:36.060 | INFO | Perf report for the last 5.14675 seconds: | 20260506 18:55:36.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:55:36.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:36.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:36.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:36.060 | INFO | ANALYSIS : 62.8%, 2.9, 95.0ms, 0 | 20260506 18:55:36.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:36.060 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260506 18:55:36.060 | INFO | PRE_HQ : 60.6%, 1.9, 58.7ms, 0 | 20260506 18:55:36.060 | INFO | BAZWRITER : 69.9%, 4.8, 172.5ms, 0 | 20260506 18:55:36.060 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 18:55:36.061 | INFO | Uploaded 36GiB to GPU in 2.05392s (17.5275GiB/s) | 20260506 18:55:36.061 | INFO | Downloaded 38GiB to GPU in 2.22336s (17.0913GiB/s) | 20260506 18:55:41.196 | INFO | Analyzing chunk frames = [3670528,3671040) | 20260506 18:55:41.196 | INFO | MemInfo: { | 20260506 18:55:41.196 | INFO | "gpu" : " 17 GB", | 20260506 18:55:41.196 | INFO | "huge" : "", | 20260506 18:55:41.196 | INFO | "huge_shared" : " 27 GB", | 20260506 18:55:41.196 | INFO | "malloc" : " 14 GB", | 20260506 18:55:41.196 | INFO | "pinned" : " 29 GB" | 20260506 18:55:41.196 | INFO | } | 20260506 18:55:41.196 | INFO | Perf report for the last 5.1358 seconds: | 20260506 18:55:41.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:55:41.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:41.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:41.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:41.196 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 18:55:41.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:41.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:41.196 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:55:41.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:41.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:41.196 | INFO | Uploaded 36GiB to GPU in 2.01787s (17.8406GiB/s) | 20260506 18:55:41.196 | INFO | Downloaded 34GiB to GPU in 1.92372s (17.6741GiB/s) | 20260506 18:55:46.336 | INFO | Analyzing chunk frames = [3671040,3671552) | 20260506 18:55:46.336 | INFO | MemInfo: { | 20260506 18:55:46.336 | INFO | "gpu" : " 17 GB", | 20260506 18:55:46.336 | INFO | "huge" : "", | 20260506 18:55:46.336 | INFO | "huge_shared" : " 27 GB", | 20260506 18:55:46.336 | INFO | "malloc" : " 14 GB", | 20260506 18:55:46.336 | INFO | "pinned" : " 29 GB" | 20260506 18:55:46.336 | INFO | } | 20260506 18:55:46.336 | INFO | Perf report for the last 5.13989 seconds: | 20260506 18:55:46.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:55:46.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:46.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:46.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:46.336 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:55:46.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:46.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:46.336 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:55:46.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:46.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:46.336 | INFO | Uploaded 36GiB to GPU in 1.99181s (18.074GiB/s) | 20260506 18:55:46.336 | INFO | Downloaded 34GiB to GPU in 1.91532s (17.7516GiB/s) | 20260506 18:55:51.472 | INFO | Analyzing chunk frames = [3671552,3672064) | 20260506 18:55:51.472 | INFO | MemInfo: { | 20260506 18:55:51.472 | INFO | "gpu" : " 17 GB", | 20260506 18:55:51.472 | INFO | "huge" : "", | 20260506 18:55:51.473 | INFO | "huge_shared" : " 27 GB", | 20260506 18:55:51.473 | INFO | "malloc" : " 14 GB", | 20260506 18:55:51.473 | INFO | "pinned" : " 29 GB" | 20260506 18:55:51.473 | INFO | } | 20260506 18:55:51.473 | INFO | Perf report for the last 5.13678 seconds: | 20260506 18:55:51.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:55:51.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:51.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:51.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:51.473 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 18:55:51.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:51.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:51.473 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:55:51.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:51.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:51.473 | INFO | Uploaded 36GiB to GPU in 1.96695s (18.3024GiB/s) | 20260506 18:55:51.473 | INFO | Downloaded 34GiB to GPU in 1.88641s (18.0236GiB/s) | 20260506 18:55:56.610 | INFO | Analyzing chunk frames = [3672064,3672576) | 20260506 18:55:56.610 | INFO | MemInfo: { | 20260506 18:55:56.610 | INFO | "gpu" : " 17 GB", | 20260506 18:55:56.610 | INFO | "huge" : "", | 20260506 18:55:56.610 | INFO | "huge_shared" : " 27 GB", | 20260506 18:55:56.610 | INFO | "malloc" : " 14 GB", | 20260506 18:55:56.610 | INFO | "pinned" : " 29 GB" | 20260506 18:55:56.610 | INFO | } | 20260506 18:55:56.610 | INFO | Perf report for the last 5.1374 seconds: | 20260506 18:55:56.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:55:56.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:56.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:56.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:56.610 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260506 18:55:56.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:56.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:56.610 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:55:56.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:56.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:55:56.610 | INFO | Uploaded 36GiB to GPU in 2.00834s (17.9252GiB/s) | 20260506 18:55:56.610 | INFO | Downloaded 34GiB to GPU in 1.90454s (17.8521GiB/s) | 20260506 18:56:01.745 | INFO | Analyzing chunk frames = [3672576,3673088) | 20260506 18:56:01.745 | INFO | MemInfo: { | 20260506 18:56:01.745 | INFO | "gpu" : " 17 GB", | 20260506 18:56:01.745 | INFO | "huge" : "", | 20260506 18:56:01.745 | INFO | "huge_shared" : " 27 GB", | 20260506 18:56:01.745 | INFO | "malloc" : " 14 GB", | 20260506 18:56:01.745 | INFO | "pinned" : " 29 GB" | 20260506 18:56:01.745 | INFO | } | 20260506 18:56:01.745 | INFO | Perf report for the last 5.13473 seconds: | 20260506 18:56:01.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:56:01.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:01.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:01.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:01.745 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 18:56:01.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:01.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:01.745 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:56:01.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:01.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:01.745 | INFO | Uploaded 36GiB to GPU in 2.0043s (17.9613GiB/s) | 20260506 18:56:01.745 | INFO | Downloaded 34GiB to GPU in 1.94806s (17.4533GiB/s) | 20260506 18:56:06.890 | INFO | Analyzing chunk frames = [3673088,3673600) | 20260506 18:56:06.890 | INFO | MemInfo: { | 20260506 18:56:06.890 | INFO | "gpu" : " 17 GB", | 20260506 18:56:06.890 | INFO | "huge" : "", | 20260506 18:56:06.890 | INFO | "huge_shared" : " 27 GB", | 20260506 18:56:06.890 | INFO | "malloc" : " 14 GB", | 20260506 18:56:06.890 | INFO | "pinned" : " 29 GB" | 20260506 18:56:06.890 | INFO | } | 20260506 18:56:06.890 | INFO | Perf report for the last 5.14558 seconds: | 20260506 18:56:06.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:56:06.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:06.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:06.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:06.890 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 18:56:06.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:06.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:06.891 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 18:56:06.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:06.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:06.891 | INFO | Uploaded 36GiB to GPU in 1.99151s (18.0767GiB/s) | 20260506 18:56:06.891 | INFO | Downloaded 34GiB to GPU in 2.01075s (16.9091GiB/s) | 20260506 18:56:12.035 | INFO | Analyzing chunk frames = [3673600,3674112) | 20260506 18:56:12.035 | INFO | MemInfo: { | 20260506 18:56:12.035 | INFO | "gpu" : " 17 GB", | 20260506 18:56:12.035 | INFO | "huge" : "", | 20260506 18:56:12.035 | INFO | "huge_shared" : " 27 GB", | 20260506 18:56:12.035 | INFO | "malloc" : " 14 GB", | 20260506 18:56:12.035 | INFO | "pinned" : " 29 GB" | 20260506 18:56:12.035 | INFO | } | 20260506 18:56:12.035 | INFO | Perf report for the last 5.14476 seconds: | 20260506 18:56:12.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:56:12.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:12.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:12.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:12.035 | INFO | ANALYSIS : 61.9%, 3.0, 96.6ms, 0 | 20260506 18:56:12.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:12.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:12.035 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 18:56:12.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:12.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:12.035 | INFO | Uploaded 36GiB to GPU in 2.04169s (17.6325GiB/s) | 20260506 18:56:12.036 | INFO | Downloaded 34GiB to GPU in 1.98108s (17.1623GiB/s) | 20260506 18:56:12.149 | INFO | Starting a round of Baz IO | 20260506 18:56:15.284 | INFO | Populating RTMetrics report | 20260506 18:56:15.284 | INFO | RTMetrics report populated | 20260506 18:56:15.284 | INFO | Opening tmp RTMetrics json file | 20260506 18:56:15.285 | INFO | Serializing RTMetrics to json | 20260506 18:56:15.287 | INFO | Serialization done, swapping tmp file now... | 20260506 18:56:15.287 | INFO | RTMetrics done | 20260506 18:56:15.681 | INFO | Finished a round of Baz IO: Wrote 2.026 GiB in 3.532 seconds | 20260506 18:56:15.681 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.84% / 0.01% / 65.15% | 20260506 18:56:15.681 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.01768% / 1.58946e-05% / 8.74201e-05% | 20260506 18:56:15.681 | INFO | Baz IO overall: | 20260506 18:56:15.681 | INFO | BytesWritten : 1941558199067 | 20260506 18:56:15.681 | INFO | NumEvents : 938761054364 | 20260506 18:56:17.175 | INFO | Analyzing chunk frames = [3674112,3674624) | 20260506 18:56:17.175 | INFO | MemInfo: { | 20260506 18:56:17.175 | INFO | "gpu" : " 17 GB", | 20260506 18:56:17.175 | INFO | "huge" : "", | 20260506 18:56:17.175 | INFO | "huge_shared" : " 27 GB", | 20260506 18:56:17.175 | INFO | "malloc" : " 14 GB", | 20260506 18:56:17.175 | INFO | "pinned" : " 29 GB" | 20260506 18:56:17.175 | INFO | } | 20260506 18:56:17.175 | INFO | Perf report for the last 5.13984 seconds: | 20260506 18:56:17.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:56:17.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:17.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:17.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:17.175 | INFO | ANALYSIS : 62.6%, 2.9, 95.3ms, 0 | 20260506 18:56:17.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:17.175 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 18:56:17.175 | INFO | PRE_HQ : 60.3%, 1.9, 58.3ms, 0 | 20260506 18:56:17.175 | INFO | BAZWRITER : 68.4%, 4.8, 169.0ms, 0 | 20260506 18:56:17.175 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.3ms, 0 | 20260506 18:56:17.175 | INFO | Uploaded 36GiB to GPU in 2.03907s (17.6551GiB/s) | 20260506 18:56:17.175 | INFO | Downloaded 38GiB to GPU in 2.20952s (17.1983GiB/s) | 20260506 18:56:22.315 | INFO | Analyzing chunk frames = [3674624,3675136) | 20260506 18:56:22.315 | INFO | MemInfo: { | 20260506 18:56:22.315 | INFO | "gpu" : " 17 GB", | 20260506 18:56:22.315 | INFO | "huge" : "", | 20260506 18:56:22.315 | INFO | "huge_shared" : " 27 GB", | 20260506 18:56:22.315 | INFO | "malloc" : " 14 GB", | 20260506 18:56:22.315 | INFO | "pinned" : " 29 GB" | 20260506 18:56:22.315 | INFO | } | 20260506 18:56:22.315 | INFO | Perf report for the last 5.14036 seconds: | 20260506 18:56:22.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:56:22.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:22.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:22.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:22.315 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 18:56:22.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:22.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:22.316 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:56:22.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:22.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:22.316 | INFO | Uploaded 36GiB to GPU in 2.00209s (17.9812GiB/s) | 20260506 18:56:22.316 | INFO | Downloaded 34GiB to GPU in 1.92585s (17.6546GiB/s) | 20260506 18:56:27.448 | INFO | Analyzing chunk frames = [3675136,3675648) | 20260506 18:56:27.448 | INFO | MemInfo: { | 20260506 18:56:27.448 | INFO | "gpu" : " 17 GB", | 20260506 18:56:27.448 | INFO | "huge" : "", | 20260506 18:56:27.448 | INFO | "huge_shared" : " 27 GB", | 20260506 18:56:27.448 | INFO | "malloc" : " 14 GB", | 20260506 18:56:27.448 | INFO | "pinned" : " 29 GB" | 20260506 18:56:27.448 | INFO | } | 20260506 18:56:27.448 | INFO | Perf report for the last 5.13283 seconds: | 20260506 18:56:27.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:56:27.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:27.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:27.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:27.448 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 18:56:27.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:27.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:27.448 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 18:56:27.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:27.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:27.449 | INFO | Uploaded 36GiB to GPU in 1.95756s (18.3902GiB/s) | 20260506 18:56:27.449 | INFO | Downloaded 34GiB to GPU in 1.91731s (17.7331GiB/s) | 20260506 18:56:32.585 | INFO | Analyzing chunk frames = [3675648,3676160) | 20260506 18:56:32.585 | INFO | MemInfo: { | 20260506 18:56:32.585 | INFO | "gpu" : " 17 GB", | 20260506 18:56:32.585 | INFO | "huge" : "", | 20260506 18:56:32.585 | INFO | "huge_shared" : " 27 GB", | 20260506 18:56:32.585 | INFO | "malloc" : " 14 GB", | 20260506 18:56:32.585 | INFO | "pinned" : " 29 GB" | 20260506 18:56:32.585 | INFO | } | 20260506 18:56:32.585 | INFO | Perf report for the last 5.13675 seconds: | 20260506 18:56:32.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:56:32.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:32.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:32.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:32.585 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 18:56:32.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:32.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:32.585 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:56:32.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:32.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:32.585 | INFO | Uploaded 36GiB to GPU in 1.92848s (18.6676GiB/s) | 20260506 18:56:32.585 | INFO | Downloaded 34GiB to GPU in 1.87512s (18.1321GiB/s) | 20260506 18:56:37.724 | INFO | Analyzing chunk frames = [3676160,3676672) | 20260506 18:56:37.724 | INFO | MemInfo: { | 20260506 18:56:37.724 | INFO | "gpu" : " 17 GB", | 20260506 18:56:37.724 | INFO | "huge" : "", | 20260506 18:56:37.724 | INFO | "huge_shared" : " 27 GB", | 20260506 18:56:37.724 | INFO | "malloc" : " 14 GB", | 20260506 18:56:37.724 | INFO | "pinned" : " 29 GB" | 20260506 18:56:37.724 | INFO | } | 20260506 18:56:37.724 | INFO | Perf report for the last 5.13944 seconds: | 20260506 18:56:37.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:56:37.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:37.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:37.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:37.725 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 18:56:37.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:37.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:37.725 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 18:56:37.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:37.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:37.725 | INFO | Uploaded 36GiB to GPU in 2.01322s (17.8818GiB/s) | 20260506 18:56:37.725 | INFO | Downloaded 34GiB to GPU in 1.91796s (17.7272GiB/s) | 20260506 18:56:42.862 | INFO | Analyzing chunk frames = [3676672,3677184) | 20260506 18:56:42.862 | INFO | MemInfo: { | 20260506 18:56:42.862 | INFO | "gpu" : " 17 GB", | 20260506 18:56:42.862 | INFO | "huge" : "", | 20260506 18:56:42.862 | INFO | "huge_shared" : " 27 GB", | 20260506 18:56:42.862 | INFO | "malloc" : " 14 GB", | 20260506 18:56:42.862 | INFO | "pinned" : " 29 GB" | 20260506 18:56:42.862 | INFO | } | 20260506 18:56:42.862 | INFO | Perf report for the last 5.13755 seconds: | 20260506 18:56:42.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:56:42.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:42.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:42.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:42.862 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 18:56:42.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:42.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:42.862 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 18:56:42.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:42.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:42.862 | INFO | Uploaded 36GiB to GPU in 2.00389s (17.965GiB/s) | 20260506 18:56:42.862 | INFO | Downloaded 34GiB to GPU in 1.96051s (17.3424GiB/s) | 20260506 18:56:48.006 | INFO | Analyzing chunk frames = [3677184,3677696) | 20260506 18:56:48.006 | INFO | MemInfo: { | 20260506 18:56:48.006 | INFO | "gpu" : " 17 GB", | 20260506 18:56:48.006 | INFO | "huge" : "", | 20260506 18:56:48.006 | INFO | "huge_shared" : " 27 GB", | 20260506 18:56:48.006 | INFO | "malloc" : " 14 GB", | 20260506 18:56:48.006 | INFO | "pinned" : " 29 GB" | 20260506 18:56:48.006 | INFO | } | 20260506 18:56:48.006 | INFO | Perf report for the last 5.14407 seconds: | 20260506 18:56:48.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:56:48.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:48.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:48.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:48.006 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:56:48.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:48.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:48.006 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 18:56:48.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:48.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:48.006 | INFO | Uploaded 36GiB to GPU in 1.98875s (18.1018GiB/s) | 20260506 18:56:48.007 | INFO | Downloaded 34GiB to GPU in 2.01356s (16.8855GiB/s) | 20260506 18:56:53.152 | INFO | Analyzing chunk frames = [3677696,3678208) | 20260506 18:56:53.152 | INFO | MemInfo: { | 20260506 18:56:53.152 | INFO | "gpu" : " 17 GB", | 20260506 18:56:53.152 | INFO | "huge" : "", | 20260506 18:56:53.152 | INFO | "huge_shared" : " 27 GB", | 20260506 18:56:53.152 | INFO | "malloc" : " 14 GB", | 20260506 18:56:53.152 | INFO | "pinned" : " 29 GB" | 20260506 18:56:53.152 | INFO | } | 20260506 18:56:53.152 | INFO | Perf report for the last 5.14589 seconds: | 20260506 18:56:53.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:56:53.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:53.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:53.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:53.152 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 18:56:53.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:53.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:53.152 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:56:53.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:53.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:53.152 | INFO | Uploaded 36GiB to GPU in 1.9883s (18.1059GiB/s) | 20260506 18:56:53.152 | INFO | Downloaded 34GiB to GPU in 1.98269s (17.1484GiB/s) | 20260506 18:56:53.153 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:56:53.153 | INFO | Total mem usage: 18 GiB | 20260506 18:56:53.153 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:56:53.153 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -14%) | 20260506 18:56:53.153 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:56:53.153 | INFO | High Water Breakdown: | 20260506 18:56:53.153 | INFO | 61.6% : 9.73 GiB ( 85.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:56:53.153 | INFO | 38.3% : 6.04 GiB ( 90.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:56:53.153 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:56:53.153 | INFO | ------------------------------------------ | 20260506 18:56:53.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:56:53.153 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:56:53.153 | INFO | Total mem usage: 26.5 GiB | 20260506 18:56:53.153 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:56:53.153 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:56:53.153 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:56:53.153 | INFO | High Water Breakdown: | 20260506 18:56:53.153 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:56:53.153 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:56:53.153 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:56:53.153 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:56:53.153 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:56:53.153 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:56:53.153 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:56:53.153 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:56:53.153 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:56:53.153 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:56:53.153 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:56:53.153 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:56:53.153 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:56:53.153 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:56:53.153 | INFO | ------------------------------------------ | 20260506 18:56:53.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:56:53.153 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:56:53.153 | INFO | Total mem usage: 25.6 GiB | 20260506 18:56:53.153 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:56:53.153 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:56:53.153 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:56:53.153 | INFO | High Water Breakdown: | 20260506 18:56:53.153 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:56:53.153 | INFO | ------------------------------------------ | 20260506 18:56:53.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:56:53.153 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:56:53.153 | INFO | Total mem usage: 15.7 GiB | 20260506 18:56:53.153 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:56:53.153 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:56:53.153 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 18:56:53.153 | INFO | High Water Breakdown: | 20260506 18:56:53.153 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:56:53.153 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:56:53.153 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:56:53.153 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:56:53.153 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:56:53.153 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:56:53.153 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:56:53.153 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:56:53.153 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:56:53.153 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:56:53.153 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:56:53.153 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:56:53.153 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:56:53.153 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:56:53.153 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:56:53.153 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:56:53.153 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:56:53.153 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:56:53.153 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:56:53.153 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:56:53.153 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:56:53.153 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:56:53.153 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:56:53.153 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:56:53.153 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:56:53.153 | INFO | ------------------------------------------ | 20260506 18:56:53.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:56:53.153 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:56:53.153 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:56:53.153 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:56:53.153 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.64e+01, 1.36e+01 / 2.88e+00 | 20260506 18:56:53.153 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.02e-01, 1.04e-02 / 8.20e-03 | 20260506 18:56:53.153 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.41e+00, 5.52e-01 / 3.04e-01 | 20260506 18:56:53.153 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.48e+00, 1.86e-01 / 1.21e-01 | 20260506 18:56:53.153 | INFO | PerfReport: 16.04%, DME -- 4.01e+01 / 4.51e+01, 4.32e+00 / 3.00e+00 | 20260506 18:56:53.153 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.42e+00 / 1.56e+00 | 20260506 18:56:53.153 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.24e+00 / 2.82e+00, 3.74e-01 / 2.59e-01 | 20260506 18:56:53.153 | INFO | PerfReport: 3.37%, Metrics -- 1.05e+00 / 1.41e+00, 2.70e-01 / 1.60e-01 | 20260506 18:56:53.153 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.64e-01 / 4.11e-01, 5.76e-02 / 1.97e-02 | 20260506 18:56:53.153 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:56:53.153 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:56:53.153 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:56:53.153 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 18:56:53.153 | INFO | PerfReport: 21.47%, Upload -- 2.08e+01 / 0.00e+00, 6.88e+00 / 0.00e+00 | 20260506 18:56:53.153 | INFO | PerfReport: 21.20%, Download -- 2.06e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260506 18:56:53.153 | INFO | PerfReport: 32.22%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 18:56:53.153 | INFO | PerfReport: 1.35%, LockUpload -- 1.31e+00 / 0.00e+00, 5.36e+00 / 0.00e+00 | 20260506 18:56:53.153 | INFO | PerfReport: 1.21%, LockDownload -- 1.17e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 18:56:53.153 | INFO | PerfReport: 22.53%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 18:56:53.153 | INFO | PerfReport: 0.01%, Misc -- 1.27e-02 / 0.00e+00, 1.43e-02 / 0.00e+00 | 20260506 18:56:53.259 | INFO | Starting a round of Baz IO | 20260506 18:56:56.392 | INFO | Populating RTMetrics report | 20260506 18:56:56.393 | INFO | RTMetrics report populated | 20260506 18:56:56.393 | INFO | Opening tmp RTMetrics json file | 20260506 18:56:56.487 | INFO | Serializing RTMetrics to json | 20260506 18:56:56.533 | INFO | Serialization done, swapping tmp file now... | 20260506 18:56:56.537 | INFO | RTMetrics done | 20260506 18:56:56.557 | INFO | Finished a round of Baz IO: Wrote 2.022 GiB in 3.298 seconds | 20260506 18:56:56.557 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.84% / 0.01% / 65.16% | 20260506 18:56:56.557 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.01096% / 3.97364e-06% / 8.34465e-05% | 20260506 18:56:56.557 | INFO | Baz IO overall: | 20260506 18:56:56.558 | INFO | BytesWritten : 1943729122104 | 20260506 18:56:56.558 | INFO | NumEvents : 939805198992 | 20260506 18:56:58.286 | INFO | Analyzing chunk frames = [3678208,3678720) | 20260506 18:56:58.286 | INFO | MemInfo: { | 20260506 18:56:58.286 | INFO | "gpu" : " 17 GB", | 20260506 18:56:58.286 | INFO | "huge" : "", | 20260506 18:56:58.286 | INFO | "huge_shared" : " 27 GB", | 20260506 18:56:58.286 | INFO | "malloc" : " 14 GB", | 20260506 18:56:58.286 | INFO | "pinned" : " 29 GB" | 20260506 18:56:58.286 | INFO | } | 20260506 18:56:58.286 | INFO | Perf report for the last 5.13418 seconds: | 20260506 18:56:58.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:56:58.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:58.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:58.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:58.286 | INFO | ANALYSIS : 62.5%, 2.9, 95.1ms, 0 | 20260506 18:56:58.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:56:58.286 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260506 18:56:58.286 | INFO | PRE_HQ : 60.1%, 1.8, 57.6ms, 0 | 20260506 18:56:58.286 | INFO | BAZWRITER : 64.0%, 4.8, 156.5ms, 0 | 20260506 18:56:58.286 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 18:56:58.287 | INFO | Uploaded 36GiB to GPU in 2.06817s (17.4067GiB/s) | 20260506 18:56:58.287 | INFO | Downloaded 38GiB to GPU in 2.194s (17.32GiB/s) | 20260506 18:57:03.424 | INFO | Analyzing chunk frames = [3678720,3679232) | 20260506 18:57:03.424 | INFO | MemInfo: { | 20260506 18:57:03.424 | INFO | "gpu" : " 17 GB", | 20260506 18:57:03.424 | INFO | "huge" : "", | 20260506 18:57:03.424 | INFO | "huge_shared" : " 27 GB", | 20260506 18:57:03.424 | INFO | "malloc" : " 14 GB", | 20260506 18:57:03.424 | INFO | "pinned" : " 29 GB" | 20260506 18:57:03.424 | INFO | } | 20260506 18:57:03.424 | INFO | Perf report for the last 5.13798 seconds: | 20260506 18:57:03.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:57:03.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:03.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:03.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:03.424 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 18:57:03.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:03.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:03.424 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:57:03.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:03.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:03.424 | INFO | Uploaded 36GiB to GPU in 2.00919s (17.9176GiB/s) | 20260506 18:57:03.424 | INFO | Downloaded 34GiB to GPU in 1.93639s (17.5584GiB/s) | 20260506 18:57:08.562 | INFO | Analyzing chunk frames = [3679232,3679744) | 20260506 18:57:08.562 | INFO | MemInfo: { | 20260506 18:57:08.562 | INFO | "gpu" : " 17 GB", | 20260506 18:57:08.562 | INFO | "huge" : "", | 20260506 18:57:08.562 | INFO | "huge_shared" : " 27 GB", | 20260506 18:57:08.562 | INFO | "malloc" : " 14 GB", | 20260506 18:57:08.562 | INFO | "pinned" : " 29 GB" | 20260506 18:57:08.562 | INFO | } | 20260506 18:57:08.562 | INFO | Perf report for the last 5.13764 seconds: | 20260506 18:57:08.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:57:08.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:08.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:08.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:08.562 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 18:57:08.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:08.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:08.562 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 18:57:08.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:08.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:08.562 | INFO | Uploaded 36GiB to GPU in 1.98921s (18.0977GiB/s) | 20260506 18:57:08.562 | INFO | Downloaded 34GiB to GPU in 1.92036s (17.705GiB/s) | 20260506 18:57:13.700 | INFO | Analyzing chunk frames = [3679744,3680256) | 20260506 18:57:13.700 | INFO | MemInfo: { | 20260506 18:57:13.700 | INFO | "gpu" : " 17 GB", | 20260506 18:57:13.700 | INFO | "huge" : "", | 20260506 18:57:13.700 | INFO | "huge_shared" : " 27 GB", | 20260506 18:57:13.700 | INFO | "malloc" : " 14 GB", | 20260506 18:57:13.700 | INFO | "pinned" : " 29 GB" | 20260506 18:57:13.700 | INFO | } | 20260506 18:57:13.700 | INFO | Perf report for the last 5.13831 seconds: | 20260506 18:57:13.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:57:13.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:13.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:13.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:13.700 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 18:57:13.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:13.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:13.700 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:57:13.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:13.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:13.700 | INFO | Uploaded 36GiB to GPU in 1.95824s (18.3838GiB/s) | 20260506 18:57:13.701 | INFO | Downloaded 34GiB to GPU in 1.87371s (18.1458GiB/s) | 20260506 18:57:18.837 | INFO | Analyzing chunk frames = [3680256,3680768) | 20260506 18:57:18.837 | INFO | MemInfo: { | 20260506 18:57:18.837 | INFO | "gpu" : " 17 GB", | 20260506 18:57:18.837 | INFO | "huge" : "", | 20260506 18:57:18.837 | INFO | "huge_shared" : " 27 GB", | 20260506 18:57:18.837 | INFO | "malloc" : " 14 GB", | 20260506 18:57:18.837 | INFO | "pinned" : " 29 GB" | 20260506 18:57:18.837 | INFO | } | 20260506 18:57:18.837 | INFO | Perf report for the last 5.13711 seconds: | 20260506 18:57:18.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:57:18.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:18.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:18.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:18.837 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 18:57:18.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:18.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:18.837 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 18:57:18.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:18.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:18.838 | INFO | Uploaded 36GiB to GPU in 2.02288s (17.7964GiB/s) | 20260506 18:57:18.838 | INFO | Downloaded 34GiB to GPU in 1.9152s (17.7527GiB/s) | 20260506 18:57:23.984 | INFO | Analyzing chunk frames = [3680768,3681280) | 20260506 18:57:23.984 | INFO | MemInfo: { | 20260506 18:57:23.984 | INFO | "gpu" : " 17 GB", | 20260506 18:57:23.984 | INFO | "huge" : "", | 20260506 18:57:23.985 | INFO | "huge_shared" : " 27 GB", | 20260506 18:57:23.985 | INFO | "malloc" : " 14 GB", | 20260506 18:57:23.985 | INFO | "pinned" : " 29 GB" | 20260506 18:57:23.985 | INFO | } | 20260506 18:57:23.985 | INFO | Perf report for the last 5.14745 seconds: | 20260506 18:57:23.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:57:23.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:23.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:23.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:23.985 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 18:57:23.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:23.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:23.985 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:57:23.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:23.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:23.985 | INFO | Uploaded 36GiB to GPU in 2.02956s (17.7378GiB/s) | 20260506 18:57:23.985 | INFO | Downloaded 34GiB to GPU in 1.94407s (17.4891GiB/s) | 20260506 18:57:29.123 | INFO | Analyzing chunk frames = [3681280,3681792) | 20260506 18:57:29.123 | INFO | MemInfo: { | 20260506 18:57:29.123 | INFO | "gpu" : " 17 GB", | 20260506 18:57:29.123 | INFO | "huge" : "", | 20260506 18:57:29.123 | INFO | "huge_shared" : " 27 GB", | 20260506 18:57:29.123 | INFO | "malloc" : " 14 GB", | 20260506 18:57:29.123 | INFO | "pinned" : " 29 GB" | 20260506 18:57:29.123 | INFO | } | 20260506 18:57:29.123 | INFO | Perf report for the last 5.13862 seconds: | 20260506 18:57:29.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:57:29.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:29.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:29.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:29.124 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 18:57:29.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:29.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:29.124 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:57:29.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:29.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:29.124 | INFO | Uploaded 36GiB to GPU in 1.98978s (18.0925GiB/s) | 20260506 18:57:29.124 | INFO | Downloaded 34GiB to GPU in 1.99479s (17.0444GiB/s) | 20260506 18:57:34.264 | INFO | Analyzing chunk frames = [3681792,3682304) | 20260506 18:57:34.264 | INFO | MemInfo: { | 20260506 18:57:34.264 | INFO | "gpu" : " 17 GB", | 20260506 18:57:34.264 | INFO | "huge" : "", | 20260506 18:57:34.264 | INFO | "huge_shared" : " 27 GB", | 20260506 18:57:34.264 | INFO | "malloc" : " 14 GB", | 20260506 18:57:34.264 | INFO | "pinned" : " 29 GB" | 20260506 18:57:34.264 | INFO | } | 20260506 18:57:34.264 | INFO | Perf report for the last 5.14047 seconds: | 20260506 18:57:34.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:57:34.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:34.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:34.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:34.264 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 18:57:34.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:34.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:34.264 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:57:34.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:34.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:34.264 | INFO | Uploaded 36GiB to GPU in 2.03758s (17.668GiB/s) | 20260506 18:57:34.264 | INFO | Downloaded 34GiB to GPU in 1.99813s (17.0159GiB/s) | 20260506 18:57:34.381 | INFO | Starting a round of Baz IO | 20260506 18:57:37.516 | INFO | Populating RTMetrics report | 20260506 18:57:37.516 | INFO | RTMetrics report populated | 20260506 18:57:37.516 | INFO | Opening tmp RTMetrics json file | 20260506 18:57:37.541 | INFO | Serializing RTMetrics to json | 20260506 18:57:37.600 | INFO | Serialization done, swapping tmp file now... | 20260506 18:57:37.602 | INFO | RTMetrics done | 20260506 18:57:37.899 | INFO | Finished a round of Baz IO: Wrote 2.021 GiB in 3.518 seconds | 20260506 18:57:37.899 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.83% / 0.01% / 65.16% | 20260506 18:57:37.899 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.00845% / 3.97364e-06% / 0.000103315% | 20260506 18:57:37.899 | INFO | Baz IO overall: | 20260506 18:57:37.899 | INFO | BytesWritten : 1945899580066 | 20260506 18:57:37.899 | INFO | NumEvents : 940849188642 | 20260506 18:57:39.406 | INFO | Analyzing chunk frames = [3682304,3682816) | 20260506 18:57:39.406 | INFO | MemInfo: { | 20260506 18:57:39.406 | INFO | "gpu" : " 17 GB", | 20260506 18:57:39.406 | INFO | "huge" : "", | 20260506 18:57:39.406 | INFO | "huge_shared" : " 27 GB", | 20260506 18:57:39.406 | INFO | "malloc" : " 14 GB", | 20260506 18:57:39.406 | INFO | "pinned" : " 29 GB" | 20260506 18:57:39.406 | INFO | } | 20260506 18:57:39.406 | INFO | Perf report for the last 5.14207 seconds: | 20260506 18:57:39.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:57:39.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:39.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:39.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:39.406 | INFO | ANALYSIS : 62.6%, 2.9, 95.5ms, 0 | 20260506 18:57:39.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:39.406 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260506 18:57:39.406 | INFO | PRE_HQ : 59.8%, 1.9, 57.5ms, 0 | 20260506 18:57:39.406 | INFO | BAZWRITER : 68.1%, 4.8, 167.9ms, 0 | 20260506 18:57:39.406 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 18:57:39.406 | INFO | Uploaded 36GiB to GPU in 2.0403s (17.6445GiB/s) | 20260506 18:57:39.406 | INFO | Downloaded 38GiB to GPU in 2.22587s (17.072GiB/s) | 20260506 18:57:44.538 | INFO | Analyzing chunk frames = [3682816,3683328) | 20260506 18:57:44.538 | INFO | MemInfo: { | 20260506 18:57:44.538 | INFO | "gpu" : " 17 GB", | 20260506 18:57:44.538 | INFO | "huge" : "", | 20260506 18:57:44.538 | INFO | "huge_shared" : " 27 GB", | 20260506 18:57:44.538 | INFO | "malloc" : " 14 GB", | 20260506 18:57:44.538 | INFO | "pinned" : " 29 GB" | 20260506 18:57:44.538 | INFO | } | 20260506 18:57:44.538 | INFO | Perf report for the last 5.13195 seconds: | 20260506 18:57:44.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:57:44.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:44.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:44.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:44.538 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 18:57:44.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:44.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:44.538 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 18:57:44.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:44.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:44.538 | INFO | Uploaded 36GiB to GPU in 2.00375s (17.9663GiB/s) | 20260506 18:57:44.538 | INFO | Downloaded 34GiB to GPU in 1.92568s (17.6561GiB/s) | 20260506 18:57:49.679 | INFO | Analyzing chunk frames = [3683328,3683840) | 20260506 18:57:49.679 | INFO | MemInfo: { | 20260506 18:57:49.679 | INFO | "gpu" : " 17 GB", | 20260506 18:57:49.679 | INFO | "huge" : "", | 20260506 18:57:49.679 | INFO | "huge_shared" : " 27 GB", | 20260506 18:57:49.679 | INFO | "malloc" : " 14 GB", | 20260506 18:57:49.679 | INFO | "pinned" : " 29 GB" | 20260506 18:57:49.679 | INFO | } | 20260506 18:57:49.679 | INFO | Perf report for the last 5.14085 seconds: | 20260506 18:57:49.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:57:49.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:49.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:49.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:49.679 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 18:57:49.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:49.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:49.679 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 18:57:49.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:49.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:49.679 | INFO | Uploaded 36GiB to GPU in 1.95752s (18.3906GiB/s) | 20260506 18:57:49.679 | INFO | Downloaded 34GiB to GPU in 1.9233s (17.678GiB/s) | 20260506 18:57:54.817 | INFO | Analyzing chunk frames = [3683840,3684352) | 20260506 18:57:54.817 | INFO | MemInfo: { | 20260506 18:57:54.817 | INFO | "gpu" : " 17 GB", | 20260506 18:57:54.817 | INFO | "huge" : "", | 20260506 18:57:54.817 | INFO | "huge_shared" : " 27 GB", | 20260506 18:57:54.817 | INFO | "malloc" : " 14 GB", | 20260506 18:57:54.817 | INFO | "pinned" : " 29 GB" | 20260506 18:57:54.817 | INFO | } | 20260506 18:57:54.817 | INFO | Perf report for the last 5.1381 seconds: | 20260506 18:57:54.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:57:54.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:54.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:54.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:54.817 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 18:57:54.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:54.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:54.817 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:57:54.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:54.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:54.817 | INFO | Uploaded 36GiB to GPU in 1.93853s (18.5708GiB/s) | 20260506 18:57:54.817 | INFO | Downloaded 34GiB to GPU in 1.87231s (18.1594GiB/s) | 20260506 18:57:59.954 | INFO | Analyzing chunk frames = [3684352,3684864) | 20260506 18:57:59.954 | INFO | MemInfo: { | 20260506 18:57:59.954 | INFO | "gpu" : " 17 GB", | 20260506 18:57:59.954 | INFO | "huge" : "", | 20260506 18:57:59.954 | INFO | "huge_shared" : " 27 GB", | 20260506 18:57:59.954 | INFO | "malloc" : " 14 GB", | 20260506 18:57:59.954 | INFO | "pinned" : " 29 GB" | 20260506 18:57:59.954 | INFO | } | 20260506 18:57:59.954 | INFO | Perf report for the last 5.13724 seconds: | 20260506 18:57:59.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:57:59.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:59.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:59.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:59.954 | INFO | ANALYSIS : 59.9%, 3.0, 93.2ms, 0 | 20260506 18:57:59.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:59.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:59.954 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 18:57:59.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:59.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:57:59.955 | INFO | Uploaded 36GiB to GPU in 1.95124s (18.4498GiB/s) | 20260506 18:57:59.955 | INFO | Downloaded 34GiB to GPU in 1.88458s (18.0412GiB/s) | 20260506 18:58:05.094 | INFO | Analyzing chunk frames = [3684864,3685376) | 20260506 18:58:05.094 | INFO | MemInfo: { | 20260506 18:58:05.094 | INFO | "gpu" : " 17 GB", | 20260506 18:58:05.094 | INFO | "huge" : "", | 20260506 18:58:05.094 | INFO | "huge_shared" : " 27 GB", | 20260506 18:58:05.094 | INFO | "malloc" : " 14 GB", | 20260506 18:58:05.094 | INFO | "pinned" : " 29 GB" | 20260506 18:58:05.094 | INFO | } | 20260506 18:58:05.095 | INFO | Perf report for the last 5.14036 seconds: | 20260506 18:58:05.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:58:05.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:05.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:05.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:05.095 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 18:58:05.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:05.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:05.095 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 18:58:05.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:05.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:05.095 | INFO | Uploaded 36GiB to GPU in 2.0114s (17.8979GiB/s) | 20260506 18:58:05.095 | INFO | Downloaded 34GiB to GPU in 1.96256s (17.3243GiB/s) | 20260506 18:58:10.235 | INFO | Analyzing chunk frames = [3685376,3685888) | 20260506 18:58:10.235 | INFO | MemInfo: { | 20260506 18:58:10.235 | INFO | "gpu" : " 17 GB", | 20260506 18:58:10.235 | INFO | "huge" : "", | 20260506 18:58:10.235 | INFO | "huge_shared" : " 27 GB", | 20260506 18:58:10.235 | INFO | "malloc" : " 14 GB", | 20260506 18:58:10.236 | INFO | "pinned" : " 29 GB" | 20260506 18:58:10.236 | INFO | } | 20260506 18:58:10.236 | INFO | Perf report for the last 5.14121 seconds: | 20260506 18:58:10.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:58:10.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:10.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:10.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:10.236 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 18:58:10.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:10.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:10.236 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:58:10.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:10.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:10.236 | INFO | Uploaded 36GiB to GPU in 1.95395s (18.4242GiB/s) | 20260506 18:58:10.236 | INFO | Downloaded 34GiB to GPU in 1.9939s (17.052GiB/s) | 20260506 18:58:15.374 | INFO | Analyzing chunk frames = [3685888,3686400) | 20260506 18:58:15.374 | INFO | MemInfo: { | 20260506 18:58:15.374 | INFO | "gpu" : " 17 GB", | 20260506 18:58:15.374 | INFO | "huge" : "", | 20260506 18:58:15.374 | INFO | "huge_shared" : " 27 GB", | 20260506 18:58:15.374 | INFO | "malloc" : " 14 GB", | 20260506 18:58:15.374 | INFO | "pinned" : " 29 GB" | 20260506 18:58:15.374 | INFO | } | 20260506 18:58:15.374 | INFO | Perf report for the last 5.13819 seconds: | 20260506 18:58:15.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:58:15.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:15.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:15.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:15.374 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 18:58:15.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:15.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:15.374 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 18:58:15.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:15.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:15.374 | INFO | Uploaded 36GiB to GPU in 2.04043s (17.6434GiB/s) | 20260506 18:58:15.374 | INFO | Downloaded 34GiB to GPU in 1.97805s (17.1886GiB/s) | 20260506 18:58:15.374 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:58:15.374 | INFO | Total mem usage: 18 GiB | 20260506 18:58:15.374 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:58:15.374 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -11.2%) | 20260506 18:58:15.374 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:58:15.374 | INFO | High Water Breakdown: | 20260506 18:58:15.374 | INFO | 62% : 10 GiB ( 88% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:58:15.374 | INFO | 37.8% : 6.13 GiB ( 91.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:58:15.374 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:58:15.374 | INFO | ------------------------------------------ | 20260506 18:58:15.374 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:58:15.375 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:58:15.375 | INFO | Total mem usage: 26.5 GiB | 20260506 18:58:15.375 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:58:15.375 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 18:58:15.375 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 18:58:15.375 | INFO | High Water Breakdown: | 20260506 18:58:15.375 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:58:15.375 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:58:15.375 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:58:15.375 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:58:15.375 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:58:15.375 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:58:15.375 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:58:15.375 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:58:15.375 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:58:15.375 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:58:15.375 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:58:15.375 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:58:15.375 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:58:15.375 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:58:15.375 | INFO | ------------------------------------------ | 20260506 18:58:15.375 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:58:15.375 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:58:15.375 | INFO | Total mem usage: 25.6 GiB | 20260506 18:58:15.375 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:58:15.375 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:58:15.375 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:58:15.375 | INFO | High Water Breakdown: | 20260506 18:58:15.375 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:58:15.375 | INFO | ------------------------------------------ | 20260506 18:58:15.375 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:58:15.375 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:58:15.375 | INFO | Total mem usage: 15.7 GiB | 20260506 18:58:15.375 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:58:15.375 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:58:15.375 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 18:58:15.375 | INFO | High Water Breakdown: | 20260506 18:58:15.375 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:58:15.375 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:58:15.375 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:58:15.375 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:58:15.375 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:58:15.375 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:58:15.375 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:58:15.375 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:58:15.375 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:58:15.375 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:58:15.375 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:58:15.375 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:58:15.375 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:58:15.375 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:58:15.375 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:58:15.375 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:58:15.375 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:58:15.375 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:58:15.375 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:58:15.375 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:58:15.375 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:58:15.375 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:58:15.375 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:58:15.375 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:58:15.375 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:58:15.375 | INFO | ------------------------------------------ | 20260506 18:58:15.375 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:58:15.375 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:58:15.375 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:58:15.375 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:58:15.375 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.56e+01, 1.32e+01 / 2.69e+00 | 20260506 18:58:15.375 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 1.02e-02 / 6.73e-03 | 20260506 18:58:15.375 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.36e+00, 5.54e-01 / 3.82e-01 | 20260506 18:58:15.375 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.82e-01 / 1.36e-01 | 20260506 18:58:15.375 | INFO | PerfReport: 15.99%, DME -- 3.99e+01 / 4.46e+01, 4.23e+00 / 2.82e+00 | 20260506 18:58:15.375 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.38e+00 / 2.16e+00 | 20260506 18:58:15.375 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.26e+00 / 2.82e+00, 3.91e-01 / 2.27e-01 | 20260506 18:58:15.375 | INFO | PerfReport: 3.39%, Metrics -- 1.06e+00 / 1.42e+00, 2.81e-01 / 1.94e-01 | 20260506 18:58:15.376 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.64e-01 / 4.13e-01, 5.73e-02 / 2.69e-02 | 20260506 18:58:15.376 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:58:15.376 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:58:15.376 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:58:15.376 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 18:58:15.376 | INFO | PerfReport: 21.48%, Upload -- 2.08e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 18:58:15.376 | INFO | PerfReport: 21.17%, Download -- 2.05e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 18:58:15.376 | INFO | PerfReport: 32.19%, Compute -- 3.12e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 18:58:15.376 | INFO | PerfReport: 1.34%, LockUpload -- 1.30e+00 / 0.00e+00, 5.35e+00 / 0.00e+00 | 20260506 18:58:15.376 | INFO | PerfReport: 1.19%, LockDownload -- 1.16e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 18:58:15.376 | INFO | PerfReport: 22.61%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 18:58:15.376 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260506 18:58:15.509 | INFO | Starting a round of Baz IO | 20260506 18:58:18.633 | INFO | Populating RTMetrics report | 20260506 18:58:18.634 | INFO | RTMetrics report populated | 20260506 18:58:18.634 | INFO | Opening tmp RTMetrics json file | 20260506 18:58:18.635 | INFO | Serializing RTMetrics to json | 20260506 18:58:18.637 | INFO | Serialization done, swapping tmp file now... | 20260506 18:58:18.638 | INFO | RTMetrics done | 20260506 18:58:19.077 | INFO | Finished a round of Baz IO: Wrote 2.023 GiB in 3.567 seconds | 20260506 18:58:19.077 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.83% / 0.01% / 65.17% | 20260506 18:58:19.077 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.00791% / 7.94729e-06% / 9.13938e-05% | 20260506 18:58:19.077 | INFO | Baz IO overall: | 20260506 18:58:19.077 | INFO | BytesWritten : 1948071743748 | 20260506 18:58:19.077 | INFO | NumEvents : 941894771802 | 20260506 18:58:20.512 | INFO | Analyzing chunk frames = [3686400,3686912) | 20260506 18:58:20.512 | INFO | MemInfo: { | 20260506 18:58:20.512 | INFO | "gpu" : " 17 GB", | 20260506 18:58:20.512 | INFO | "huge" : "", | 20260506 18:58:20.512 | INFO | "huge_shared" : " 27 GB", | 20260506 18:58:20.512 | INFO | "malloc" : " 14 GB", | 20260506 18:58:20.512 | INFO | "pinned" : " 29 GB" | 20260506 18:58:20.512 | INFO | } | 20260506 18:58:20.512 | INFO | Perf report for the last 5.13811 seconds: | 20260506 18:58:20.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:58:20.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:20.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:20.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:20.512 | INFO | ANALYSIS : 62.8%, 2.9, 95.3ms, 0 | 20260506 18:58:20.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:20.512 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260506 18:58:20.512 | INFO | PRE_HQ : 60.3%, 1.9, 58.5ms, 0 | 20260506 18:58:20.512 | INFO | BAZWRITER : 69.4%, 4.7, 169.4ms, 0 | 20260506 18:58:20.512 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 18:58:20.512 | INFO | Uploaded 36GiB to GPU in 2.04578s (17.5972GiB/s) | 20260506 18:58:20.512 | INFO | Downloaded 38GiB to GPU in 2.20513s (17.2326GiB/s) | 20260506 18:58:25.653 | INFO | Analyzing chunk frames = [3686912,3687424) | 20260506 18:58:25.653 | INFO | MemInfo: { | 20260506 18:58:25.653 | INFO | "gpu" : " 17 GB", | 20260506 18:58:25.653 | INFO | "huge" : "", | 20260506 18:58:25.653 | INFO | "huge_shared" : " 27 GB", | 20260506 18:58:25.653 | INFO | "malloc" : " 14 GB", | 20260506 18:58:25.653 | INFO | "pinned" : " 29 GB" | 20260506 18:58:25.653 | INFO | } | 20260506 18:58:25.653 | INFO | Perf report for the last 5.14111 seconds: | 20260506 18:58:25.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:58:25.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:25.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:25.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:25.653 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 18:58:25.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:25.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:25.653 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:58:25.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:25.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:25.654 | INFO | Uploaded 36GiB to GPU in 2.00492s (17.9558GiB/s) | 20260506 18:58:25.654 | INFO | Downloaded 34GiB to GPU in 1.9374s (17.5493GiB/s) | 20260506 18:58:30.792 | INFO | Analyzing chunk frames = [3687424,3687936) | 20260506 18:58:30.792 | INFO | MemInfo: { | 20260506 18:58:30.792 | INFO | "gpu" : " 17 GB", | 20260506 18:58:30.792 | INFO | "huge" : "", | 20260506 18:58:30.792 | INFO | "huge_shared" : " 27 GB", | 20260506 18:58:30.792 | INFO | "malloc" : " 14 GB", | 20260506 18:58:30.792 | INFO | "pinned" : " 29 GB" | 20260506 18:58:30.792 | INFO | } | 20260506 18:58:30.792 | INFO | Perf report for the last 5.13895 seconds: | 20260506 18:58:30.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:58:30.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:30.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:30.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:30.792 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 18:58:30.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:30.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:30.792 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 18:58:30.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:30.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:30.792 | INFO | Uploaded 36GiB to GPU in 1.95779s (18.3881GiB/s) | 20260506 18:58:30.792 | INFO | Downloaded 34GiB to GPU in 1.92091s (17.7GiB/s) | 20260506 18:58:35.929 | INFO | Analyzing chunk frames = [3687936,3688448) | 20260506 18:58:35.929 | INFO | MemInfo: { | 20260506 18:58:35.929 | INFO | "gpu" : " 17 GB", | 20260506 18:58:35.929 | INFO | "huge" : "", | 20260506 18:58:35.929 | INFO | "huge_shared" : " 27 GB", | 20260506 18:58:35.929 | INFO | "malloc" : " 14 GB", | 20260506 18:58:35.929 | INFO | "pinned" : " 29 GB" | 20260506 18:58:35.929 | INFO | } | 20260506 18:58:35.929 | INFO | Perf report for the last 5.13692 seconds: | 20260506 18:58:35.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:58:35.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:35.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:35.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:35.929 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 18:58:35.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:35.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:35.929 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:58:35.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:35.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:35.930 | INFO | Uploaded 36GiB to GPU in 1.92655s (18.6863GiB/s) | 20260506 18:58:35.930 | INFO | Downloaded 34GiB to GPU in 1.87168s (18.1655GiB/s) | 20260506 18:58:41.075 | INFO | Analyzing chunk frames = [3688448,3688960) | 20260506 18:58:41.075 | INFO | MemInfo: { | 20260506 18:58:41.075 | INFO | "gpu" : " 17 GB", | 20260506 18:58:41.075 | INFO | "huge" : "", | 20260506 18:58:41.075 | INFO | "huge_shared" : " 27 GB", | 20260506 18:58:41.076 | INFO | "malloc" : " 14 GB", | 20260506 18:58:41.076 | INFO | "pinned" : " 29 GB" | 20260506 18:58:41.076 | INFO | } | 20260506 18:58:41.076 | INFO | Perf report for the last 5.1467 seconds: | 20260506 18:58:41.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:58:41.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:41.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:41.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:41.076 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260506 18:58:41.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:41.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:41.076 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:58:41.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:41.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:41.076 | INFO | Uploaded 36GiB to GPU in 1.9497s (18.4644GiB/s) | 20260506 18:58:41.076 | INFO | Downloaded 34GiB to GPU in 1.88453s (18.0417GiB/s) | 20260506 18:58:46.214 | INFO | Analyzing chunk frames = [3688960,3689472) | 20260506 18:58:46.214 | INFO | MemInfo: { | 20260506 18:58:46.214 | INFO | "gpu" : " 17 GB", | 20260506 18:58:46.214 | INFO | "huge" : "", | 20260506 18:58:46.214 | INFO | "huge_shared" : " 27 GB", | 20260506 18:58:46.214 | INFO | "malloc" : " 14 GB", | 20260506 18:58:46.214 | INFO | "pinned" : " 29 GB" | 20260506 18:58:46.214 | INFO | } | 20260506 18:58:46.214 | INFO | Perf report for the last 5.13858 seconds: | 20260506 18:58:46.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:58:46.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:46.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:46.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:46.214 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 18:58:46.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:46.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:46.214 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 18:58:46.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:46.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:46.215 | INFO | Uploaded 36GiB to GPU in 1.99119s (18.0796GiB/s) | 20260506 18:58:46.215 | INFO | Downloaded 34GiB to GPU in 1.96062s (17.3414GiB/s) | 20260506 18:58:51.353 | INFO | Analyzing chunk frames = [3689472,3689984) | 20260506 18:58:51.353 | INFO | MemInfo: { | 20260506 18:58:51.353 | INFO | "gpu" : " 17 GB", | 20260506 18:58:51.353 | INFO | "huge" : "", | 20260506 18:58:51.353 | INFO | "huge_shared" : " 27 GB", | 20260506 18:58:51.353 | INFO | "malloc" : " 14 GB", | 20260506 18:58:51.353 | INFO | "pinned" : " 29 GB" | 20260506 18:58:51.353 | INFO | } | 20260506 18:58:51.353 | INFO | Perf report for the last 5.13825 seconds: | 20260506 18:58:51.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:58:51.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:51.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:51.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:51.353 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 18:58:51.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:51.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:51.353 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 18:58:51.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:51.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:51.353 | INFO | Uploaded 36GiB to GPU in 2.00627s (17.9437GiB/s) | 20260506 18:58:51.353 | INFO | Downloaded 34GiB to GPU in 1.99579s (17.0359GiB/s) | 20260506 18:58:56.491 | INFO | Analyzing chunk frames = [3689984,3690496) | 20260506 18:58:56.491 | INFO | MemInfo: { | 20260506 18:58:56.491 | INFO | "gpu" : " 17 GB", | 20260506 18:58:56.491 | INFO | "huge" : "", | 20260506 18:58:56.491 | INFO | "huge_shared" : " 27 GB", | 20260506 18:58:56.491 | INFO | "malloc" : " 14 GB", | 20260506 18:58:56.491 | INFO | "pinned" : " 29 GB" | 20260506 18:58:56.491 | INFO | } | 20260506 18:58:56.491 | INFO | Perf report for the last 5.13859 seconds: | 20260506 18:58:56.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:58:56.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:56.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:56.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:56.491 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:58:56.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:56.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:56.492 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260506 18:58:56.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:56.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:58:56.492 | INFO | Uploaded 36GiB to GPU in 2.02413s (17.7854GiB/s) | 20260506 18:58:56.492 | INFO | Downloaded 34GiB to GPU in 1.98493s (17.129GiB/s) | 20260506 18:58:56.605 | INFO | Starting a round of Baz IO | 20260506 18:58:59.794 | INFO | Populating RTMetrics report | 20260506 18:58:59.795 | INFO | RTMetrics report populated | 20260506 18:58:59.795 | INFO | Opening tmp RTMetrics json file | 20260506 18:58:59.824 | INFO | Serializing RTMetrics to json | 20260506 18:58:59.851 | INFO | Serialization done, swapping tmp file now... | 20260506 18:58:59.853 | INFO | RTMetrics done | 20260506 18:59:00.467 | INFO | Finished a round of Baz IO: Wrote 2.023 GiB in 3.861 seconds | 20260506 18:59:00.467 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.82% / 0.01% / 65.17% | 20260506 18:59:00.467 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.00765% / 3.97364e-06% / 9.53674e-05% | 20260506 18:59:00.467 | INFO | Baz IO overall: | 20260506 18:59:00.467 | INFO | BytesWritten : 1950244150375 | 20260506 18:59:00.467 | INFO | NumEvents : 942940465346 | 20260506 18:59:01.626 | INFO | Analyzing chunk frames = [3690496,3691008) | 20260506 18:59:01.626 | INFO | MemInfo: { | 20260506 18:59:01.626 | INFO | "gpu" : " 17 GB", | 20260506 18:59:01.626 | INFO | "huge" : "", | 20260506 18:59:01.626 | INFO | "huge_shared" : " 27 GB", | 20260506 18:59:01.626 | INFO | "malloc" : " 14 GB", | 20260506 18:59:01.626 | INFO | "pinned" : " 29 GB" | 20260506 18:59:01.626 | INFO | } | 20260506 18:59:01.626 | INFO | Perf report for the last 5.13475 seconds: | 20260506 18:59:01.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:59:01.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:01.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:01.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:01.626 | INFO | ANALYSIS : 63.4%, 2.8, 93.5ms, 0 | 20260506 18:59:01.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:01.626 | INFO | RT_METRICS : 49.2%, 1.0, 25.8ms, 0 | 20260506 18:59:01.626 | INFO | PRE_HQ : 60.8%, 2.0, 64.6ms, 0 | 20260506 18:59:01.626 | INFO | BAZWRITER : 74.9%, 4.8, 185.6ms, 0 | 20260506 18:59:01.626 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 18:59:01.626 | INFO | Uploaded 36GiB to GPU in 2.0337s (17.7018GiB/s) | 20260506 18:59:01.626 | INFO | Downloaded 38GiB to GPU in 2.16983s (17.5129GiB/s) | 20260506 18:59:06.765 | INFO | Analyzing chunk frames = [3691008,3691520) | 20260506 18:59:06.765 | INFO | MemInfo: { | 20260506 18:59:06.765 | INFO | "gpu" : " 17 GB", | 20260506 18:59:06.765 | INFO | "huge" : "", | 20260506 18:59:06.765 | INFO | "huge_shared" : " 27 GB", | 20260506 18:59:06.765 | INFO | "malloc" : " 14 GB", | 20260506 18:59:06.765 | INFO | "pinned" : " 29 GB" | 20260506 18:59:06.765 | INFO | } | 20260506 18:59:06.765 | INFO | Perf report for the last 5.13934 seconds: | 20260506 18:59:06.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:59:06.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:06.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:06.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:06.766 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:59:06.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:06.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:06.766 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 18:59:06.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:06.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:06.766 | INFO | Uploaded 36GiB to GPU in 2.01796s (17.8398GiB/s) | 20260506 18:59:06.766 | INFO | Downloaded 34GiB to GPU in 1.92794s (17.6354GiB/s) | 20260506 18:59:11.911 | INFO | Analyzing chunk frames = [3691520,3692032) | 20260506 18:59:11.911 | INFO | MemInfo: { | 20260506 18:59:11.911 | INFO | "gpu" : " 17 GB", | 20260506 18:59:11.911 | INFO | "huge" : "", | 20260506 18:59:11.911 | INFO | "huge_shared" : " 27 GB", | 20260506 18:59:11.911 | INFO | "malloc" : " 14 GB", | 20260506 18:59:11.911 | INFO | "pinned" : " 29 GB" | 20260506 18:59:11.911 | INFO | } | 20260506 18:59:11.912 | INFO | Perf report for the last 5.14618 seconds: | 20260506 18:59:11.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:59:11.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:11.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:11.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:11.912 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 18:59:11.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:11.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:11.912 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 18:59:11.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:11.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:11.912 | INFO | Uploaded 36GiB to GPU in 1.98045s (18.1777GiB/s) | 20260506 18:59:11.912 | INFO | Downloaded 34GiB to GPU in 1.91609s (17.7445GiB/s) | 20260506 18:59:17.044 | INFO | Analyzing chunk frames = [3692032,3692544) | 20260506 18:59:17.044 | INFO | MemInfo: { | 20260506 18:59:17.044 | INFO | "gpu" : " 17 GB", | 20260506 18:59:17.044 | INFO | "huge" : "", | 20260506 18:59:17.044 | INFO | "huge_shared" : " 27 GB", | 20260506 18:59:17.044 | INFO | "malloc" : " 14 GB", | 20260506 18:59:17.044 | INFO | "pinned" : " 29 GB" | 20260506 18:59:17.044 | INFO | } | 20260506 18:59:17.044 | INFO | Perf report for the last 5.13258 seconds: | 20260506 18:59:17.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:59:17.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:17.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:17.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:17.044 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 18:59:17.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:17.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:17.044 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:59:17.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:17.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:17.044 | INFO | Uploaded 36GiB to GPU in 1.93316s (18.6224GiB/s) | 20260506 18:59:17.044 | INFO | Downloaded 34GiB to GPU in 1.86854s (18.1961GiB/s) | 20260506 18:59:22.182 | INFO | Analyzing chunk frames = [3692544,3693056) | 20260506 18:59:22.182 | INFO | MemInfo: { | 20260506 18:59:22.182 | INFO | "gpu" : " 17 GB", | 20260506 18:59:22.182 | INFO | "huge" : "", | 20260506 18:59:22.182 | INFO | "huge_shared" : " 27 GB", | 20260506 18:59:22.182 | INFO | "malloc" : " 14 GB", | 20260506 18:59:22.182 | INFO | "pinned" : " 29 GB" | 20260506 18:59:22.182 | INFO | } | 20260506 18:59:22.182 | INFO | Perf report for the last 5.13802 seconds: | 20260506 18:59:22.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:59:22.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:22.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:22.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:22.182 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 18:59:22.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:22.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:22.182 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 18:59:22.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:22.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:22.183 | INFO | Uploaded 36GiB to GPU in 2.02042s (17.8181GiB/s) | 20260506 18:59:22.183 | INFO | Downloaded 34GiB to GPU in 1.91263s (17.7766GiB/s) | 20260506 18:59:27.325 | INFO | Analyzing chunk frames = [3693056,3693568) | 20260506 18:59:27.325 | INFO | MemInfo: { | 20260506 18:59:27.325 | INFO | "gpu" : " 17 GB", | 20260506 18:59:27.325 | INFO | "huge" : "", | 20260506 18:59:27.325 | INFO | "huge_shared" : " 27 GB", | 20260506 18:59:27.325 | INFO | "malloc" : " 14 GB", | 20260506 18:59:27.325 | INFO | "pinned" : " 29 GB" | 20260506 18:59:27.325 | INFO | } | 20260506 18:59:27.325 | INFO | Perf report for the last 5.14325 seconds: | 20260506 18:59:27.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:59:27.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:27.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:27.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:27.326 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 18:59:27.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:27.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:27.326 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 18:59:27.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:27.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:27.326 | INFO | Uploaded 36GiB to GPU in 2.04162s (17.6331GiB/s) | 20260506 18:59:27.326 | INFO | Downloaded 34GiB to GPU in 1.94517s (17.4792GiB/s) | 20260506 18:59:32.468 | INFO | Analyzing chunk frames = [3693568,3694080) | 20260506 18:59:32.468 | INFO | MemInfo: { | 20260506 18:59:32.468 | INFO | "gpu" : " 17 GB", | 20260506 18:59:32.468 | INFO | "huge" : "", | 20260506 18:59:32.468 | INFO | "huge_shared" : " 27 GB", | 20260506 18:59:32.468 | INFO | "malloc" : " 14 GB", | 20260506 18:59:32.468 | INFO | "pinned" : " 29 GB" | 20260506 18:59:32.468 | INFO | } | 20260506 18:59:32.468 | INFO | Perf report for the last 5.14247 seconds: | 20260506 18:59:32.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:59:32.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:32.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:32.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:32.468 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 18:59:32.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:32.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:32.468 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 18:59:32.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:32.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:32.468 | INFO | Uploaded 36GiB to GPU in 1.99295s (18.0637GiB/s) | 20260506 18:59:32.468 | INFO | Downloaded 34GiB to GPU in 2.00927s (16.9216GiB/s) | 20260506 18:59:37.603 | INFO | Analyzing chunk frames = [3694080,3694592) | 20260506 18:59:37.603 | INFO | MemInfo: { | 20260506 18:59:37.603 | INFO | "gpu" : " 17 GB", | 20260506 18:59:37.603 | INFO | "huge" : "", | 20260506 18:59:37.603 | INFO | "huge_shared" : " 27 GB", | 20260506 18:59:37.603 | INFO | "malloc" : " 14 GB", | 20260506 18:59:37.603 | INFO | "pinned" : " 29 GB" | 20260506 18:59:37.603 | INFO | } | 20260506 18:59:37.603 | INFO | Perf report for the last 5.13551 seconds: | 20260506 18:59:37.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:59:37.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:37.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:37.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:37.603 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 18:59:37.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:37.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:37.604 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 18:59:37.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:37.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:37.604 | INFO | Uploaded 36GiB to GPU in 2.0179s (17.8403GiB/s) | 20260506 18:59:37.604 | INFO | Downloaded 34GiB to GPU in 1.98465s (17.1315GiB/s) | 20260506 18:59:37.604 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 18:59:37.604 | INFO | Total mem usage: 18 GiB | 20260506 18:59:37.604 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 18:59:37.604 | INFO | Recent High Water Marks Sum: 16.9 GiB (Savings: -6.48%) | 20260506 18:59:37.604 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 18:59:37.604 | INFO | High Water Breakdown: | 20260506 18:59:37.604 | INFO | 62.6% : 10.6 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 18:59:37.604 | INFO | 37.2% : 6.3 GiB ( 94.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 18:59:37.604 | INFO | 0.118% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 18:59:37.604 | INFO | ------------------------------------------ | 20260506 18:59:37.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:59:37.604 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 18:59:37.604 | INFO | Total mem usage: 26.5 GiB | 20260506 18:59:37.604 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 18:59:37.604 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260506 18:59:37.604 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260506 18:59:37.604 | INFO | High Water Breakdown: | 20260506 18:59:37.604 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:59:37.604 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:59:37.604 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:59:37.604 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:59:37.604 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:59:37.604 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:59:37.604 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 18:59:37.604 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 18:59:37.604 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:59:37.604 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 18:59:37.604 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 18:59:37.604 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 18:59:37.604 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 18:59:37.604 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 18:59:37.604 | INFO | ------------------------------------------ | 20260506 18:59:37.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:59:37.604 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 18:59:37.604 | INFO | Total mem usage: 25.6 GiB | 20260506 18:59:37.604 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 18:59:37.604 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 18:59:37.604 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 18:59:37.604 | INFO | High Water Breakdown: | 20260506 18:59:37.604 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 18:59:37.604 | INFO | ------------------------------------------ | 20260506 18:59:37.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:59:37.605 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 18:59:37.605 | INFO | Total mem usage: 15.7 GiB | 20260506 18:59:37.605 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 18:59:37.605 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 18:59:37.605 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 18:59:37.605 | INFO | High Water Breakdown: | 20260506 18:59:37.605 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 18:59:37.605 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 18:59:37.605 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 18:59:37.605 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 18:59:37.605 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 18:59:37.605 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 18:59:37.605 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 18:59:37.605 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 18:59:37.605 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 18:59:37.605 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 18:59:37.605 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 18:59:37.605 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 18:59:37.605 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 18:59:37.605 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 18:59:37.605 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 18:59:37.605 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 18:59:37.605 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 18:59:37.605 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 18:59:37.605 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 18:59:37.605 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 18:59:37.605 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 18:59:37.605 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 18:59:37.605 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 18:59:37.605 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 18:59:37.605 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 18:59:37.605 | INFO | ------------------------------------------ | 20260506 18:59:37.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 18:59:37.605 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 18:59:37.605 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:59:37.605 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:59:37.605 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.57e+01, 1.36e+01 / 2.53e+00 | 20260506 18:59:37.605 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.00e-01, 1.06e-02 / 6.82e-03 | 20260506 18:59:37.605 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.31e+00, 5.55e-01 / 3.92e-01 | 20260506 18:59:37.605 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.44e+00, 1.83e-01 / 1.43e-01 | 20260506 18:59:37.605 | INFO | PerfReport: 15.96%, DME -- 3.99e+01 / 4.46e+01, 4.06e+00 / 2.40e+00 | 20260506 18:59:37.605 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.47e+00 / 1.74e+00 | 20260506 18:59:37.605 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.24e+00 / 2.78e+00, 3.77e-01 / 2.31e-01 | 20260506 18:59:37.605 | INFO | PerfReport: 3.37%, Metrics -- 1.05e+00 / 1.42e+00, 2.77e-01 / 1.88e-01 | 20260506 18:59:37.605 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.64e-01 / 4.10e-01, 5.67e-02 / 2.16e-02 | 20260506 18:59:37.605 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 18:59:37.605 | INFO | PerfReport: Gathered 1536 samples. | 20260506 18:59:37.605 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 18:59:37.605 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 18:59:37.605 | INFO | PerfReport: 21.47%, Upload -- 2.08e+01 / 0.00e+00, 6.94e+00 / 0.00e+00 | 20260506 18:59:37.605 | INFO | PerfReport: 21.17%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 18:59:37.605 | INFO | PerfReport: 32.30%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 18:59:37.605 | INFO | PerfReport: 1.37%, LockUpload -- 1.33e+00 / 0.00e+00, 5.36e+00 / 0.00e+00 | 20260506 18:59:37.605 | INFO | PerfReport: 1.12%, LockDownload -- 1.08e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 18:59:37.605 | INFO | PerfReport: 22.54%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 18:59:37.605 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260506 18:59:37.718 | INFO | Starting a round of Baz IO | 20260506 18:59:40.851 | INFO | Populating RTMetrics report | 20260506 18:59:40.851 | INFO | RTMetrics report populated | 20260506 18:59:40.851 | INFO | Opening tmp RTMetrics json file | 20260506 18:59:40.993 | INFO | Serializing RTMetrics to json | 20260506 18:59:41.015 | INFO | Serialization done, swapping tmp file now... | 20260506 18:59:41.028 | INFO | RTMetrics done | 20260506 18:59:41.229 | INFO | Finished a round of Baz IO: Wrote 2.021 GiB in 3.511 seconds | 20260506 18:59:41.229 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.82% / 0.01% / 65.18% | 20260506 18:59:41.229 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.0023% / 3.97364e-06% / 9.13938e-05% | 20260506 18:59:41.229 | INFO | Baz IO overall: | 20260506 18:59:41.229 | INFO | BytesWritten : 1952414689569 | 20260506 18:59:41.229 | INFO | NumEvents : 943984946001 | 20260506 18:59:42.746 | INFO | Analyzing chunk frames = [3694592,3695104) | 20260506 18:59:42.746 | INFO | MemInfo: { | 20260506 18:59:42.746 | INFO | "gpu" : " 17 GB", | 20260506 18:59:42.746 | INFO | "huge" : "", | 20260506 18:59:42.746 | INFO | "huge_shared" : " 27 GB", | 20260506 18:59:42.746 | INFO | "malloc" : " 14 GB", | 20260506 18:59:42.746 | INFO | "pinned" : " 29 GB" | 20260506 18:59:42.746 | INFO | } | 20260506 18:59:42.746 | INFO | Perf report for the last 5.14262 seconds: | 20260506 18:59:42.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:59:42.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:42.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:42.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:42.746 | INFO | ANALYSIS : 62.6%, 2.9, 95.7ms, 0 | 20260506 18:59:42.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:42.746 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260506 18:59:42.746 | INFO | PRE_HQ : 60.1%, 1.8, 57.8ms, 0 | 20260506 18:59:42.746 | INFO | BAZWRITER : 67.9%, 4.8, 166.6ms, 0 | 20260506 18:59:42.746 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.4ms, 0 | 20260506 18:59:42.746 | INFO | Uploaded 36GiB to GPU in 2.07423s (17.3559GiB/s) | 20260506 18:59:42.746 | INFO | Downloaded 38GiB to GPU in 2.21352s (17.1673GiB/s) | 20260506 18:59:47.881 | INFO | Analyzing chunk frames = [3695104,3695616) | 20260506 18:59:47.881 | INFO | MemInfo: { | 20260506 18:59:47.881 | INFO | "gpu" : " 17 GB", | 20260506 18:59:47.881 | INFO | "huge" : "", | 20260506 18:59:47.881 | INFO | "huge_shared" : " 27 GB", | 20260506 18:59:47.881 | INFO | "malloc" : " 14 GB", | 20260506 18:59:47.881 | INFO | "pinned" : " 29 GB" | 20260506 18:59:47.881 | INFO | } | 20260506 18:59:47.881 | INFO | Perf report for the last 5.13447 seconds: | 20260506 18:59:47.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:59:47.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:47.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:47.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:47.881 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 18:59:47.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:47.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:47.881 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:59:47.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:47.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:47.881 | INFO | Uploaded 36GiB to GPU in 2.01982s (17.8234GiB/s) | 20260506 18:59:47.881 | INFO | Downloaded 34GiB to GPU in 1.94001s (17.5257GiB/s) | 20260506 18:59:53.020 | INFO | Analyzing chunk frames = [3695616,3696128) | 20260506 18:59:53.020 | INFO | MemInfo: { | 20260506 18:59:53.020 | INFO | "gpu" : " 17 GB", | 20260506 18:59:53.020 | INFO | "huge" : "", | 20260506 18:59:53.020 | INFO | "huge_shared" : " 27 GB", | 20260506 18:59:53.020 | INFO | "malloc" : " 14 GB", | 20260506 18:59:53.020 | INFO | "pinned" : " 29 GB" | 20260506 18:59:53.020 | INFO | } | 20260506 18:59:53.020 | INFO | Perf report for the last 5.13987 seconds: | 20260506 18:59:53.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:59:53.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:53.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:53.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:53.021 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 18:59:53.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:53.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:53.021 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 18:59:53.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:53.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:53.021 | INFO | Uploaded 36GiB to GPU in 1.98557s (18.1308GiB/s) | 20260506 18:59:53.021 | INFO | Downloaded 34GiB to GPU in 1.91918s (17.7159GiB/s) | 20260506 18:59:58.164 | INFO | Analyzing chunk frames = [3696128,3696640) | 20260506 18:59:58.164 | INFO | MemInfo: { | 20260506 18:59:58.164 | INFO | "gpu" : " 17 GB", | 20260506 18:59:58.164 | INFO | "huge" : "", | 20260506 18:59:58.164 | INFO | "huge_shared" : " 27 GB", | 20260506 18:59:58.164 | INFO | "malloc" : " 14 GB", | 20260506 18:59:58.164 | INFO | "pinned" : " 29 GB" | 20260506 18:59:58.164 | INFO | } | 20260506 18:59:58.164 | INFO | Perf report for the last 5.14378 seconds: | 20260506 18:59:58.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 18:59:58.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:58.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:58.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:58.165 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 18:59:58.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:58.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:58.165 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 18:59:58.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:58.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 18:59:58.165 | INFO | Uploaded 36GiB to GPU in 1.93412s (18.6131GiB/s) | 20260506 18:59:58.165 | INFO | Downloaded 34GiB to GPU in 1.86735s (18.2076GiB/s) | 20260506 19:00:03.300 | INFO | Analyzing chunk frames = [3696640,3697152) | 20260506 19:00:03.300 | INFO | MemInfo: { | 20260506 19:00:03.300 | INFO | "gpu" : " 17 GB", | 20260506 19:00:03.300 | INFO | "huge" : "", | 20260506 19:00:03.300 | INFO | "huge_shared" : " 27 GB", | 20260506 19:00:03.300 | INFO | "malloc" : " 14 GB", | 20260506 19:00:03.300 | INFO | "pinned" : " 29 GB" | 20260506 19:00:03.301 | INFO | } | 20260506 19:00:03.301 | INFO | Perf report for the last 5.13632 seconds: | 20260506 19:00:03.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:00:03.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:03.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:03.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:03.301 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 19:00:03.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:03.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:03.301 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:00:03.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:03.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:03.301 | INFO | Uploaded 36GiB to GPU in 2.02139s (17.8095GiB/s) | 20260506 19:00:03.301 | INFO | Downloaded 34GiB to GPU in 1.91126s (17.7893GiB/s) | 20260506 19:00:08.440 | INFO | Analyzing chunk frames = [3697152,3697664) | 20260506 19:00:08.440 | INFO | MemInfo: { | 20260506 19:00:08.440 | INFO | "gpu" : " 17 GB", | 20260506 19:00:08.440 | INFO | "huge" : "", | 20260506 19:00:08.440 | INFO | "huge_shared" : " 27 GB", | 20260506 19:00:08.440 | INFO | "malloc" : " 14 GB", | 20260506 19:00:08.441 | INFO | "pinned" : " 29 GB" | 20260506 19:00:08.441 | INFO | } | 20260506 19:00:08.441 | INFO | Perf report for the last 5.13998 seconds: | 20260506 19:00:08.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:00:08.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:08.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:08.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:08.441 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:00:08.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:08.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:08.441 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:00:08.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:08.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:08.441 | INFO | Uploaded 36GiB to GPU in 2.07728s (17.3303GiB/s) | 20260506 19:00:08.441 | INFO | Downloaded 34GiB to GPU in 1.96572s (17.2965GiB/s) | 20260506 19:00:13.581 | INFO | Analyzing chunk frames = [3697664,3698176) | 20260506 19:00:13.581 | INFO | MemInfo: { | 20260506 19:00:13.581 | INFO | "gpu" : " 17 GB", | 20260506 19:00:13.581 | INFO | "huge" : "", | 20260506 19:00:13.581 | INFO | "huge_shared" : " 27 GB", | 20260506 19:00:13.581 | INFO | "malloc" : " 14 GB", | 20260506 19:00:13.581 | INFO | "pinned" : " 29 GB" | 20260506 19:00:13.581 | INFO | } | 20260506 19:00:13.581 | INFO | Perf report for the last 5.1407 seconds: | 20260506 19:00:13.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:00:13.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:13.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:13.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:13.582 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 19:00:13.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:13.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:13.582 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:00:13.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:13.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:13.582 | INFO | Uploaded 36GiB to GPU in 2.00431s (17.9613GiB/s) | 20260506 19:00:13.582 | INFO | Downloaded 34GiB to GPU in 1.99224s (17.0662GiB/s) | 20260506 19:00:18.719 | INFO | Analyzing chunk frames = [3698176,3698688) | 20260506 19:00:18.719 | INFO | MemInfo: { | 20260506 19:00:18.719 | INFO | "gpu" : " 17 GB", | 20260506 19:00:18.719 | INFO | "huge" : "", | 20260506 19:00:18.719 | INFO | "huge_shared" : " 27 GB", | 20260506 19:00:18.719 | INFO | "malloc" : " 14 GB", | 20260506 19:00:18.719 | INFO | "pinned" : " 29 GB" | 20260506 19:00:18.719 | INFO | } | 20260506 19:00:18.719 | INFO | Perf report for the last 5.13753 seconds: | 20260506 19:00:18.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:00:18.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:18.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:18.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:18.719 | INFO | ANALYSIS : 62.1%, 3.0, 96.9ms, 0 | 20260506 19:00:18.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:18.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:18.719 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 19:00:18.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:18.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:18.719 | INFO | Uploaded 36GiB to GPU in 2.02623s (17.767GiB/s) | 20260506 19:00:18.719 | INFO | Downloaded 34GiB to GPU in 1.98554s (17.1238GiB/s) | 20260506 19:00:18.834 | INFO | Starting a round of Baz IO | 20260506 19:00:21.988 | INFO | Populating RTMetrics report | 20260506 19:00:21.989 | INFO | RTMetrics report populated | 20260506 19:00:21.989 | INFO | Opening tmp RTMetrics json file | 20260506 19:00:22.014 | INFO | Serializing RTMetrics to json | 20260506 19:00:22.081 | INFO | Serialization done, swapping tmp file now... | 20260506 19:00:22.113 | INFO | RTMetrics done | 20260506 19:00:22.147 | INFO | Finished a round of Baz IO: Wrote 2.024 GiB in 3.313 seconds | 20260506 19:00:22.147 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.81% / 0.00% / 65.18% | 20260506 19:00:22.147 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.0037% / 0% / 7.94729e-05% | 20260506 19:00:22.147 | INFO | Baz IO overall: | 20260506 19:00:22.147 | INFO | BytesWritten : 1954587561824 | 20260506 19:00:22.147 | INFO | NumEvents : 945031388474 | 20260506 19:00:23.858 | INFO | Analyzing chunk frames = [3698688,3699200) | 20260506 19:00:23.858 | INFO | MemInfo: { | 20260506 19:00:23.858 | INFO | "gpu" : " 17 GB", | 20260506 19:00:23.858 | INFO | "huge" : "", | 20260506 19:00:23.858 | INFO | "huge_shared" : " 27 GB", | 20260506 19:00:23.858 | INFO | "malloc" : " 14 GB", | 20260506 19:00:23.858 | INFO | "pinned" : " 29 GB" | 20260506 19:00:23.858 | INFO | } | 20260506 19:00:23.858 | INFO | Perf report for the last 5.13929 seconds: | 20260506 19:00:23.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:00:23.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:23.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:23.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:23.858 | INFO | ANALYSIS : 63.0%, 2.8, 93.9ms, 0 | 20260506 19:00:23.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:23.858 | INFO | RT_METRICS : 47.7%, 1.0, 25.0ms, 0 | 20260506 19:00:23.858 | INFO | PRE_HQ : 60.8%, 1.9, 60.5ms, 0 | 20260506 19:00:23.858 | INFO | BAZWRITER : 64.2%, 4.8, 157.9ms, 0 | 20260506 19:00:23.858 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 19:00:23.858 | INFO | Uploaded 36GiB to GPU in 2.06484s (17.4347GiB/s) | 20260506 19:00:23.858 | INFO | Downloaded 38GiB to GPU in 2.19625s (17.3022GiB/s) | 20260506 19:00:28.993 | INFO | Analyzing chunk frames = [3699200,3699712) | 20260506 19:00:28.993 | INFO | MemInfo: { | 20260506 19:00:28.993 | INFO | "gpu" : " 17 GB", | 20260506 19:00:28.993 | INFO | "huge" : "", | 20260506 19:00:28.993 | INFO | "huge_shared" : " 27 GB", | 20260506 19:00:28.993 | INFO | "malloc" : " 14 GB", | 20260506 19:00:28.993 | INFO | "pinned" : " 29 GB" | 20260506 19:00:28.993 | INFO | } | 20260506 19:00:28.993 | INFO | Perf report for the last 5.13514 seconds: | 20260506 19:00:28.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:00:28.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:28.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:28.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:28.993 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:00:28.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:28.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:28.993 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:00:28.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:28.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:28.995 | INFO | Uploaded 36GiB to GPU in 2.01007s (17.9098GiB/s) | 20260506 19:00:28.995 | INFO | Downloaded 34GiB to GPU in 1.94315s (17.4973GiB/s) | 20260506 19:00:34.132 | INFO | Analyzing chunk frames = [3699712,3700224) | 20260506 19:00:34.132 | INFO | MemInfo: { | 20260506 19:00:34.132 | INFO | "gpu" : " 17 GB", | 20260506 19:00:34.132 | INFO | "huge" : "", | 20260506 19:00:34.132 | INFO | "huge_shared" : " 27 GB", | 20260506 19:00:34.132 | INFO | "malloc" : " 14 GB", | 20260506 19:00:34.132 | INFO | "pinned" : " 29 GB" | 20260506 19:00:34.132 | INFO | } | 20260506 19:00:34.132 | INFO | Perf report for the last 5.1391 seconds: | 20260506 19:00:34.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:00:34.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:34.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:34.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:34.132 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 19:00:34.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:34.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:34.133 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:00:34.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:34.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:34.133 | INFO | Uploaded 36GiB to GPU in 1.96024s (18.3651GiB/s) | 20260506 19:00:34.133 | INFO | Downloaded 34GiB to GPU in 1.9198s (17.7102GiB/s) | 20260506 19:00:39.275 | INFO | Analyzing chunk frames = [3700224,3700736) | 20260506 19:00:39.275 | INFO | MemInfo: { | 20260506 19:00:39.275 | INFO | "gpu" : " 17 GB", | 20260506 19:00:39.275 | INFO | "huge" : "", | 20260506 19:00:39.275 | INFO | "huge_shared" : " 27 GB", | 20260506 19:00:39.275 | INFO | "malloc" : " 14 GB", | 20260506 19:00:39.275 | INFO | "pinned" : " 29 GB" | 20260506 19:00:39.275 | INFO | } | 20260506 19:00:39.276 | INFO | Perf report for the last 5.14315 seconds: | 20260506 19:00:39.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:00:39.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:39.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:39.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:39.276 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 19:00:39.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:39.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:39.276 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:00:39.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:39.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:39.276 | INFO | Uploaded 36GiB to GPU in 1.91472s (18.8017GiB/s) | 20260506 19:00:39.276 | INFO | Downloaded 34GiB to GPU in 1.86894s (18.1922GiB/s) | 20260506 19:00:44.421 | INFO | Analyzing chunk frames = [3700736,3701248) | 20260506 19:00:44.421 | INFO | MemInfo: { | 20260506 19:00:44.421 | INFO | "gpu" : " 17 GB", | 20260506 19:00:44.421 | INFO | "huge" : "", | 20260506 19:00:44.421 | INFO | "huge_shared" : " 27 GB", | 20260506 19:00:44.421 | INFO | "malloc" : " 14 GB", | 20260506 19:00:44.421 | INFO | "pinned" : " 29 GB" | 20260506 19:00:44.421 | INFO | } | 20260506 19:00:44.421 | INFO | Perf report for the last 5.14524 seconds: | 20260506 19:00:44.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:00:44.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:44.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:44.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:44.421 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 19:00:44.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:44.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:44.421 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 19:00:44.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:44.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:44.421 | INFO | Uploaded 36GiB to GPU in 2.01073s (17.9039GiB/s) | 20260506 19:00:44.421 | INFO | Downloaded 34GiB to GPU in 1.91604s (17.7449GiB/s) | 20260506 19:00:49.560 | INFO | Analyzing chunk frames = [3701248,3701760) | 20260506 19:00:49.560 | INFO | MemInfo: { | 20260506 19:00:49.560 | INFO | "gpu" : " 17 GB", | 20260506 19:00:49.560 | INFO | "huge" : "", | 20260506 19:00:49.560 | INFO | "huge_shared" : " 27 GB", | 20260506 19:00:49.560 | INFO | "malloc" : " 14 GB", | 20260506 19:00:49.560 | INFO | "pinned" : " 29 GB" | 20260506 19:00:49.560 | INFO | } | 20260506 19:00:49.560 | INFO | Perf report for the last 5.13909 seconds: | 20260506 19:00:49.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:00:49.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:49.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:49.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:49.560 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 19:00:49.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:49.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:49.560 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 19:00:49.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:49.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:49.560 | INFO | Uploaded 36GiB to GPU in 2.00649s (17.9417GiB/s) | 20260506 19:00:49.560 | INFO | Downloaded 34GiB to GPU in 1.95198s (17.4182GiB/s) | 20260506 19:00:54.694 | INFO | Analyzing chunk frames = [3701760,3702272) | 20260506 19:00:54.694 | INFO | MemInfo: { | 20260506 19:00:54.694 | INFO | "gpu" : " 17 GB", | 20260506 19:00:54.694 | INFO | "huge" : "", | 20260506 19:00:54.694 | INFO | "huge_shared" : " 27 GB", | 20260506 19:00:54.694 | INFO | "malloc" : " 14 GB", | 20260506 19:00:54.694 | INFO | "pinned" : " 29 GB" | 20260506 19:00:54.695 | INFO | } | 20260506 19:00:54.695 | INFO | Perf report for the last 5.13477 seconds: | 20260506 19:00:54.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:00:54.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:54.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:54.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:54.695 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 19:00:54.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:54.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:54.695 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:00:54.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:54.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:54.695 | INFO | Uploaded 36GiB to GPU in 1.98614s (18.1256GiB/s) | 20260506 19:00:54.695 | INFO | Downloaded 34GiB to GPU in 2.00856s (16.9276GiB/s) | 20260506 19:00:59.834 | INFO | Analyzing chunk frames = [3702272,3702784) | 20260506 19:00:59.834 | INFO | MemInfo: { | 20260506 19:00:59.834 | INFO | "gpu" : " 17 GB", | 20260506 19:00:59.834 | INFO | "huge" : "", | 20260506 19:00:59.834 | INFO | "huge_shared" : " 27 GB", | 20260506 19:00:59.834 | INFO | "malloc" : " 14 GB", | 20260506 19:00:59.834 | INFO | "pinned" : " 29 GB" | 20260506 19:00:59.834 | INFO | } | 20260506 19:00:59.834 | INFO | Perf report for the last 5.13964 seconds: | 20260506 19:00:59.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:00:59.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:59.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:59.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:59.834 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:00:59.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:59.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:59.835 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:00:59.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:59.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:00:59.835 | INFO | Uploaded 36GiB to GPU in 1.99431s (18.0513GiB/s) | 20260506 19:00:59.835 | INFO | Downloaded 34GiB to GPU in 1.98624s (17.1178GiB/s) | 20260506 19:00:59.835 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:00:59.835 | INFO | Total mem usage: 18 GiB | 20260506 19:00:59.835 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:00:59.835 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -17.2%) | 20260506 19:00:59.835 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:00:59.835 | INFO | High Water Breakdown: | 20260506 19:00:59.835 | INFO | 61.2% : 9.41 GiB ( 82.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:00:59.835 | INFO | 38.7% : 5.94 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:00:59.835 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:00:59.835 | INFO | ------------------------------------------ | 20260506 19:00:59.835 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:00:59.835 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:00:59.835 | INFO | Total mem usage: 26.5 GiB | 20260506 19:00:59.835 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:00:59.835 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:00:59.835 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:00:59.835 | INFO | High Water Breakdown: | 20260506 19:00:59.835 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:00:59.835 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:00:59.835 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:00:59.835 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:00:59.835 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:00:59.835 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:00:59.835 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:00:59.835 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:00:59.835 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:00:59.835 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:00:59.835 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:00:59.835 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:00:59.835 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:00:59.835 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:00:59.835 | INFO | ------------------------------------------ | 20260506 19:00:59.835 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:00:59.835 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:00:59.836 | INFO | Total mem usage: 25.6 GiB | 20260506 19:00:59.836 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:00:59.836 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:00:59.836 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:00:59.836 | INFO | High Water Breakdown: | 20260506 19:00:59.836 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:00:59.836 | INFO | ------------------------------------------ | 20260506 19:00:59.836 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:00:59.836 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:00:59.836 | INFO | Total mem usage: 15.7 GiB | 20260506 19:00:59.836 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:00:59.836 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:00:59.836 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:00:59.836 | INFO | High Water Breakdown: | 20260506 19:00:59.836 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:00:59.836 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:00:59.836 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:00:59.836 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:00:59.836 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:00:59.836 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:00:59.836 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:00:59.836 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:00:59.836 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:00:59.836 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:00:59.836 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:00:59.836 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:00:59.836 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:00:59.836 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:00:59.836 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:00:59.836 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:00:59.836 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:00:59.836 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:00:59.836 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:00:59.836 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:00:59.836 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:00:59.836 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:00:59.836 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:00:59.836 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:00:59.836 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:00:59.836 | INFO | ------------------------------------------ | 20260506 19:00:59.836 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:00:59.836 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:00:59.836 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:00:59.836 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:00:59.836 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.58e+01, 1.35e+01 / 2.56e+00 | 20260506 19:00:59.836 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.01e-01, 1.01e-02 / 5.13e-03 | 20260506 19:00:59.836 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.42e+00, 5.48e-01 / 3.63e-01 | 20260506 19:00:59.836 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.85e-01 / 1.44e-01 | 20260506 19:00:59.836 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.45e+01, 4.03e+00 / 2.61e+00 | 20260506 19:00:59.836 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.45e+00 / 1.73e+00 | 20260506 19:00:59.836 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.25e+00 / 2.79e+00, 3.77e-01 / 2.22e-01 | 20260506 19:00:59.836 | INFO | PerfReport: 3.38%, Metrics -- 1.06e+00 / 1.36e+00, 2.80e-01 / 1.25e-01 | 20260506 19:00:59.836 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.65e-01 / 4.09e-01, 5.72e-02 / 2.14e-02 | 20260506 19:00:59.836 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:00:59.836 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:00:59.836 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:00:59.836 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 19:00:59.836 | INFO | PerfReport: 21.53%, Upload -- 2.09e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 19:00:59.836 | INFO | PerfReport: 21.19%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 19:00:59.836 | INFO | PerfReport: 32.23%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 19:00:59.836 | INFO | PerfReport: 1.44%, LockUpload -- 1.39e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260506 19:00:59.836 | INFO | PerfReport: 1.07%, LockDownload -- 1.04e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260506 19:00:59.836 | INFO | PerfReport: 22.53%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 19:00:59.836 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260506 19:00:59.946 | INFO | Starting a round of Baz IO | 20260506 19:01:03.074 | INFO | Populating RTMetrics report | 20260506 19:01:03.074 | INFO | RTMetrics report populated | 20260506 19:01:03.074 | INFO | Opening tmp RTMetrics json file | 20260506 19:01:03.122 | INFO | Serializing RTMetrics to json | 20260506 19:01:03.144 | INFO | Serialization done, swapping tmp file now... | 20260506 19:01:03.144 | INFO | RTMetrics done | 20260506 19:01:03.460 | INFO | Finished a round of Baz IO: Wrote 2.023 GiB in 3.514 seconds | 20260506 19:01:03.460 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.81% / 0.01% / 65.19% | 20260506 19:01:03.460 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 1.00011% / 0% / 8.34465e-05% | 20260506 19:01:03.460 | INFO | Baz IO overall: | 20260506 19:01:03.460 | INFO | BytesWritten : 1956759309522 | 20260506 19:01:03.460 | INFO | NumEvents : 946076691366 | 20260506 19:01:04.968 | INFO | Analyzing chunk frames = [3702784,3703296) | 20260506 19:01:04.968 | INFO | MemInfo: { | 20260506 19:01:04.968 | INFO | "gpu" : " 17 GB", | 20260506 19:01:04.968 | INFO | "huge" : "", | 20260506 19:01:04.968 | INFO | "huge_shared" : " 27 GB", | 20260506 19:01:04.968 | INFO | "malloc" : " 14 GB", | 20260506 19:01:04.968 | INFO | "pinned" : " 29 GB" | 20260506 19:01:04.968 | INFO | } | 20260506 19:01:04.968 | INFO | Perf report for the last 5.13379 seconds: | 20260506 19:01:04.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:01:04.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:04.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:04.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:04.968 | INFO | ANALYSIS : 62.5%, 2.9, 95.2ms, 0 | 20260506 19:01:04.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:04.968 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 19:01:04.968 | INFO | PRE_HQ : 60.1%, 1.8, 57.6ms, 0 | 20260506 19:01:04.968 | INFO | BAZWRITER : 68.2%, 4.8, 168.4ms, 0 | 20260506 19:01:04.968 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 19:01:04.969 | INFO | Uploaded 36GiB to GPU in 2.04388s (17.6136GiB/s) | 20260506 19:01:04.969 | INFO | Downloaded 38GiB to GPU in 2.19839s (17.2854GiB/s) | 20260506 19:01:10.108 | INFO | Analyzing chunk frames = [3703296,3703808) | 20260506 19:01:10.108 | INFO | MemInfo: { | 20260506 19:01:10.108 | INFO | "gpu" : " 17 GB", | 20260506 19:01:10.108 | INFO | "huge" : "", | 20260506 19:01:10.108 | INFO | "huge_shared" : " 27 GB", | 20260506 19:01:10.108 | INFO | "malloc" : " 14 GB", | 20260506 19:01:10.109 | INFO | "pinned" : " 29 GB" | 20260506 19:01:10.109 | INFO | } | 20260506 19:01:10.109 | INFO | Perf report for the last 5.14051 seconds: | 20260506 19:01:10.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:01:10.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:10.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:10.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:10.109 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:01:10.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:10.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:10.109 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:01:10.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:10.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:10.109 | INFO | Uploaded 36GiB to GPU in 2.01267s (17.8867GiB/s) | 20260506 19:01:10.109 | INFO | Downloaded 34GiB to GPU in 1.94121s (17.5148GiB/s) | 20260506 19:01:15.253 | INFO | Analyzing chunk frames = [3703808,3704320) | 20260506 19:01:15.253 | INFO | MemInfo: { | 20260506 19:01:15.253 | INFO | "gpu" : " 17 GB", | 20260506 19:01:15.253 | INFO | "huge" : "", | 20260506 19:01:15.253 | INFO | "huge_shared" : " 27 GB", | 20260506 19:01:15.253 | INFO | "malloc" : " 14 GB", | 20260506 19:01:15.253 | INFO | "pinned" : " 29 GB" | 20260506 19:01:15.253 | INFO | } | 20260506 19:01:15.253 | INFO | Perf report for the last 5.14391 seconds: | 20260506 19:01:15.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:01:15.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:15.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:15.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:15.253 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 19:01:15.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:15.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:15.253 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:01:15.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:15.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:15.253 | INFO | Uploaded 36GiB to GPU in 1.97072s (18.2675GiB/s) | 20260506 19:01:15.253 | INFO | Downloaded 34GiB to GPU in 1.91736s (17.7327GiB/s) | 20260506 19:01:20.394 | INFO | Analyzing chunk frames = [3704320,3704832) | 20260506 19:01:20.394 | INFO | MemInfo: { | 20260506 19:01:20.394 | INFO | "gpu" : " 17 GB", | 20260506 19:01:20.394 | INFO | "huge" : "", | 20260506 19:01:20.394 | INFO | "huge_shared" : " 27 GB", | 20260506 19:01:20.394 | INFO | "malloc" : " 14 GB", | 20260506 19:01:20.394 | INFO | "pinned" : " 29 GB" | 20260506 19:01:20.394 | INFO | } | 20260506 19:01:20.394 | INFO | Perf report for the last 5.1413 seconds: | 20260506 19:01:20.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:01:20.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:20.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:20.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:20.394 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 19:01:20.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:20.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:20.394 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:01:20.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:20.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:20.394 | INFO | Uploaded 36GiB to GPU in 1.94599s (18.4996GiB/s) | 20260506 19:01:20.394 | INFO | Downloaded 34GiB to GPU in 1.87102s (18.1719GiB/s) | 20260506 19:01:25.532 | INFO | Analyzing chunk frames = [3704832,3705344) | 20260506 19:01:25.532 | INFO | MemInfo: { | 20260506 19:01:25.532 | INFO | "gpu" : " 17 GB", | 20260506 19:01:25.532 | INFO | "huge" : "", | 20260506 19:01:25.532 | INFO | "huge_shared" : " 27 GB", | 20260506 19:01:25.532 | INFO | "malloc" : " 14 GB", | 20260506 19:01:25.532 | INFO | "pinned" : " 29 GB" | 20260506 19:01:25.532 | INFO | } | 20260506 19:01:25.532 | INFO | Perf report for the last 5.13799 seconds: | 20260506 19:01:25.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:01:25.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:25.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:25.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:25.532 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 19:01:25.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:25.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:25.532 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:01:25.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:25.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:25.532 | INFO | Uploaded 36GiB to GPU in 2.01945s (17.8266GiB/s) | 20260506 19:01:25.532 | INFO | Downloaded 34GiB to GPU in 1.91058s (17.7957GiB/s) | 20260506 19:01:30.671 | INFO | Analyzing chunk frames = [3705344,3705856) | 20260506 19:01:30.671 | INFO | MemInfo: { | 20260506 19:01:30.671 | INFO | "gpu" : " 17 GB", | 20260506 19:01:30.671 | INFO | "huge" : "", | 20260506 19:01:30.671 | INFO | "huge_shared" : " 27 GB", | 20260506 19:01:30.671 | INFO | "malloc" : " 14 GB", | 20260506 19:01:30.671 | INFO | "pinned" : " 29 GB" | 20260506 19:01:30.671 | INFO | } | 20260506 19:01:30.671 | INFO | Perf report for the last 5.13931 seconds: | 20260506 19:01:30.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:01:30.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:30.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:30.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:30.671 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 19:01:30.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:30.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:30.671 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 19:01:30.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:30.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:30.672 | INFO | Uploaded 36GiB to GPU in 2.07921s (17.3143GiB/s) | 20260506 19:01:30.672 | INFO | Downloaded 34GiB to GPU in 1.97073s (17.2525GiB/s) | 20260506 19:01:35.809 | INFO | Analyzing chunk frames = [3705856,3706368) | 20260506 19:01:35.809 | INFO | MemInfo: { | 20260506 19:01:35.809 | INFO | "gpu" : " 17 GB", | 20260506 19:01:35.809 | INFO | "huge" : "", | 20260506 19:01:35.809 | INFO | "huge_shared" : " 27 GB", | 20260506 19:01:35.809 | INFO | "malloc" : " 14 GB", | 20260506 19:01:35.809 | INFO | "pinned" : " 29 GB" | 20260506 19:01:35.809 | INFO | } | 20260506 19:01:35.809 | INFO | Perf report for the last 5.13804 seconds: | 20260506 19:01:35.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:01:35.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:35.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:35.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:35.809 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 19:01:35.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:35.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:35.809 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 19:01:35.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:35.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:35.810 | INFO | Uploaded 36GiB to GPU in 2.01622s (17.8552GiB/s) | 20260506 19:01:35.810 | INFO | Downloaded 34GiB to GPU in 2.00023s (16.998GiB/s) | 20260506 19:01:40.942 | INFO | Analyzing chunk frames = [3706368,3706880) | 20260506 19:01:40.942 | INFO | MemInfo: { | 20260506 19:01:40.942 | INFO | "gpu" : " 17 GB", | 20260506 19:01:40.942 | INFO | "huge" : "", | 20260506 19:01:40.942 | INFO | "huge_shared" : " 27 GB", | 20260506 19:01:40.942 | INFO | "malloc" : " 14 GB", | 20260506 19:01:40.942 | INFO | "pinned" : " 29 GB" | 20260506 19:01:40.942 | INFO | } | 20260506 19:01:40.942 | INFO | Perf report for the last 5.13246 seconds: | 20260506 19:01:40.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:01:40.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:40.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:40.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:40.942 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 19:01:40.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:40.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:40.942 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 19:01:40.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:40.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:40.942 | INFO | Uploaded 36GiB to GPU in 1.98901s (18.0995GiB/s) | 20260506 19:01:40.942 | INFO | Downloaded 34GiB to GPU in 1.96983s (17.2604GiB/s) | 20260506 19:01:41.053 | INFO | Starting a round of Baz IO | 20260506 19:01:44.194 | INFO | Populating RTMetrics report | 20260506 19:01:44.194 | INFO | RTMetrics report populated | 20260506 19:01:44.194 | INFO | Opening tmp RTMetrics json file | 20260506 19:01:44.195 | INFO | Serializing RTMetrics to json | 20260506 19:01:44.197 | INFO | Serialization done, swapping tmp file now... | 20260506 19:01:44.197 | INFO | RTMetrics done | 20260506 19:01:44.326 | INFO | Finished a round of Baz IO: Wrote 2.024 GiB in 3.273 seconds | 20260506 19:01:44.327 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.80% / 0.00% / 65.19% | 20260506 19:01:44.327 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.999447% / 3.97364e-06% / 9.93411e-05% | 20260506 19:01:44.327 | INFO | Baz IO overall: | 20260506 19:01:44.327 | INFO | BytesWritten : 1958932877710 | 20260506 19:01:44.327 | INFO | NumEvents : 947123724546 | 20260506 19:01:46.083 | INFO | Analyzing chunk frames = [3706880,3707392) | 20260506 19:01:46.083 | INFO | MemInfo: { | 20260506 19:01:46.083 | INFO | "gpu" : " 17 GB", | 20260506 19:01:46.083 | INFO | "huge" : "", | 20260506 19:01:46.083 | INFO | "huge_shared" : " 27 GB", | 20260506 19:01:46.083 | INFO | "malloc" : " 14 GB", | 20260506 19:01:46.083 | INFO | "pinned" : " 29 GB" | 20260506 19:01:46.083 | INFO | } | 20260506 19:01:46.083 | INFO | Perf report for the last 5.14115 seconds: | 20260506 19:01:46.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:01:46.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:46.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:46.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:46.083 | INFO | ANALYSIS : 62.7%, 2.9, 95.7ms, 0 | 20260506 19:01:46.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:46.083 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 19:01:46.083 | INFO | PRE_HQ : 60.2%, 1.8, 57.7ms, 0 | 20260506 19:01:46.083 | INFO | BAZWRITER : 63.4%, 4.7, 152.9ms, 0 | 20260506 19:01:46.083 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 19:01:46.083 | INFO | Uploaded 36GiB to GPU in 2.04467s (17.6067GiB/s) | 20260506 19:01:46.083 | INFO | Downloaded 38GiB to GPU in 2.20331s (17.2468GiB/s) | 20260506 19:01:51.228 | INFO | Analyzing chunk frames = [3707392,3707904) | 20260506 19:01:51.228 | INFO | MemInfo: { | 20260506 19:01:51.228 | INFO | "gpu" : " 17 GB", | 20260506 19:01:51.228 | INFO | "huge" : "", | 20260506 19:01:51.228 | INFO | "huge_shared" : " 27 GB", | 20260506 19:01:51.228 | INFO | "malloc" : " 14 GB", | 20260506 19:01:51.228 | INFO | "pinned" : " 29 GB" | 20260506 19:01:51.228 | INFO | } | 20260506 19:01:51.228 | INFO | Perf report for the last 5.14496 seconds: | 20260506 19:01:51.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:01:51.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:51.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:51.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:51.228 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 19:01:51.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:51.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:51.228 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:01:51.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:51.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:51.228 | INFO | Uploaded 36GiB to GPU in 2.01052s (17.9058GiB/s) | 20260506 19:01:51.228 | INFO | Downloaded 34GiB to GPU in 1.93133s (17.6045GiB/s) | 20260506 19:01:56.366 | INFO | Analyzing chunk frames = [3707904,3708416) | 20260506 19:01:56.367 | INFO | MemInfo: { | 20260506 19:01:56.367 | INFO | "gpu" : " 17 GB", | 20260506 19:01:56.367 | INFO | "huge" : "", | 20260506 19:01:56.367 | INFO | "huge_shared" : " 27 GB", | 20260506 19:01:56.367 | INFO | "malloc" : " 14 GB", | 20260506 19:01:56.367 | INFO | "pinned" : " 29 GB" | 20260506 19:01:56.367 | INFO | } | 20260506 19:01:56.367 | INFO | Perf report for the last 5.13882 seconds: | 20260506 19:01:56.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:01:56.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:56.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:56.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:56.367 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 19:01:56.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:56.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:56.367 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:01:56.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:56.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:01:56.367 | INFO | Uploaded 36GiB to GPU in 1.98987s (18.0916GiB/s) | 20260506 19:01:56.367 | INFO | Downloaded 34GiB to GPU in 1.91369s (17.7667GiB/s) | 20260506 19:02:01.511 | INFO | Analyzing chunk frames = [3708416,3708928) | 20260506 19:02:01.511 | INFO | MemInfo: { | 20260506 19:02:01.511 | INFO | "gpu" : " 17 GB", | 20260506 19:02:01.511 | INFO | "huge" : "", | 20260506 19:02:01.511 | INFO | "huge_shared" : " 27 GB", | 20260506 19:02:01.511 | INFO | "malloc" : " 14 GB", | 20260506 19:02:01.511 | INFO | "pinned" : " 29 GB" | 20260506 19:02:01.511 | INFO | } | 20260506 19:02:01.511 | INFO | Perf report for the last 5.14446 seconds: | 20260506 19:02:01.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:02:01.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:01.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:01.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:01.511 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260506 19:02:01.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:01.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:01.511 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:02:01.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:01.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:01.512 | INFO | Uploaded 36GiB to GPU in 1.94048s (18.5521GiB/s) | 20260506 19:02:01.512 | INFO | Downloaded 34GiB to GPU in 1.8677s (18.2042GiB/s) | 20260506 19:02:06.649 | INFO | Analyzing chunk frames = [3708928,3709440) | 20260506 19:02:06.649 | INFO | MemInfo: { | 20260506 19:02:06.649 | INFO | "gpu" : " 17 GB", | 20260506 19:02:06.649 | INFO | "huge" : "", | 20260506 19:02:06.649 | INFO | "huge_shared" : " 27 GB", | 20260506 19:02:06.649 | INFO | "malloc" : " 14 GB", | 20260506 19:02:06.649 | INFO | "pinned" : " 29 GB" | 20260506 19:02:06.649 | INFO | } | 20260506 19:02:06.649 | INFO | Perf report for the last 5.13791 seconds: | 20260506 19:02:06.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:02:06.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:06.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:06.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:06.649 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 19:02:06.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:06.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:06.649 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:02:06.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:06.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:06.650 | INFO | Uploaded 36GiB to GPU in 2.00995s (17.9109GiB/s) | 20260506 19:02:06.650 | INFO | Downloaded 34GiB to GPU in 1.91455s (17.7588GiB/s) | 20260506 19:02:11.787 | INFO | Analyzing chunk frames = [3709440,3709952) | 20260506 19:02:11.787 | INFO | MemInfo: { | 20260506 19:02:11.787 | INFO | "gpu" : " 17 GB", | 20260506 19:02:11.787 | INFO | "huge" : "", | 20260506 19:02:11.787 | INFO | "huge_shared" : " 27 GB", | 20260506 19:02:11.787 | INFO | "malloc" : " 14 GB", | 20260506 19:02:11.787 | INFO | "pinned" : " 29 GB" | 20260506 19:02:11.787 | INFO | } | 20260506 19:02:11.787 | INFO | Perf report for the last 5.13768 seconds: | 20260506 19:02:11.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:02:11.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:11.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:11.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:11.787 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 19:02:11.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:11.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:11.787 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 19:02:11.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:11.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:11.787 | INFO | Uploaded 36GiB to GPU in 2.02899s (17.7429GiB/s) | 20260506 19:02:11.787 | INFO | Downloaded 34GiB to GPU in 1.94491s (17.4816GiB/s) | 20260506 19:02:16.921 | INFO | Analyzing chunk frames = [3709952,3710464) | 20260506 19:02:16.921 | INFO | MemInfo: { | 20260506 19:02:16.921 | INFO | "gpu" : " 17 GB", | 20260506 19:02:16.921 | INFO | "huge" : "", | 20260506 19:02:16.921 | INFO | "huge_shared" : " 27 GB", | 20260506 19:02:16.921 | INFO | "malloc" : " 14 GB", | 20260506 19:02:16.921 | INFO | "pinned" : " 29 GB" | 20260506 19:02:16.921 | INFO | } | 20260506 19:02:16.921 | INFO | Perf report for the last 5.13447 seconds: | 20260506 19:02:16.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:02:16.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:16.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:16.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:16.921 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 19:02:16.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:16.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:16.921 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:02:16.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:16.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:16.922 | INFO | Uploaded 36GiB to GPU in 2.00205s (17.9816GiB/s) | 20260506 19:02:16.922 | INFO | Downloaded 34GiB to GPU in 1.99784s (17.0184GiB/s) | 20260506 19:02:22.060 | INFO | Analyzing chunk frames = [3710464,3710976) | 20260506 19:02:22.060 | INFO | MemInfo: { | 20260506 19:02:22.060 | INFO | "gpu" : " 17 GB", | 20260506 19:02:22.060 | INFO | "huge" : "", | 20260506 19:02:22.060 | INFO | "huge_shared" : " 27 GB", | 20260506 19:02:22.060 | INFO | "malloc" : " 14 GB", | 20260506 19:02:22.060 | INFO | "pinned" : " 29 GB" | 20260506 19:02:22.060 | INFO | } | 20260506 19:02:22.060 | INFO | Perf report for the last 5.13853 seconds: | 20260506 19:02:22.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:02:22.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:22.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:22.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:22.060 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 19:02:22.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:22.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:22.060 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:02:22.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:22.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:22.060 | INFO | Uploaded 36GiB to GPU in 1.99449s (18.0497GiB/s) | 20260506 19:02:22.060 | INFO | Downloaded 34GiB to GPU in 1.98333s (17.1429GiB/s) | 20260506 19:02:22.060 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:02:22.060 | INFO | Total mem usage: 18 GiB | 20260506 19:02:22.060 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:02:22.060 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -10.9%) | 20260506 19:02:22.060 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:02:22.060 | INFO | High Water Breakdown: | 20260506 19:02:22.060 | INFO | 62% : 10.1 GiB ( 88.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:02:22.060 | INFO | 37.8% : 6.15 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:02:22.060 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:02:22.061 | INFO | ------------------------------------------ | 20260506 19:02:22.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:02:22.061 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:02:22.061 | INFO | Total mem usage: 26.5 GiB | 20260506 19:02:22.061 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:02:22.061 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:02:22.061 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:02:22.061 | INFO | High Water Breakdown: | 20260506 19:02:22.061 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:02:22.061 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:02:22.061 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:02:22.061 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:02:22.061 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:02:22.061 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:02:22.061 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:02:22.061 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:02:22.061 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:02:22.061 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:02:22.061 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:02:22.061 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:02:22.061 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:02:22.061 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:02:22.061 | INFO | ------------------------------------------ | 20260506 19:02:22.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:02:22.061 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:02:22.061 | INFO | Total mem usage: 25.6 GiB | 20260506 19:02:22.061 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:02:22.061 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:02:22.061 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:02:22.061 | INFO | High Water Breakdown: | 20260506 19:02:22.061 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:02:22.061 | INFO | ------------------------------------------ | 20260506 19:02:22.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:02:22.061 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:02:22.061 | INFO | Total mem usage: 15.7 GiB | 20260506 19:02:22.061 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:02:22.061 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:02:22.061 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:02:22.061 | INFO | High Water Breakdown: | 20260506 19:02:22.061 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:02:22.061 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:02:22.061 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:02:22.061 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:02:22.061 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:02:22.061 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:02:22.061 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:02:22.061 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:02:22.061 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:02:22.061 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:02:22.061 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:02:22.061 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:02:22.061 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:02:22.061 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:02:22.061 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:02:22.061 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:02:22.061 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:02:22.061 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:02:22.061 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:02:22.061 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:02:22.061 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:02:22.061 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:02:22.061 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:02:22.061 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:02:22.062 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:02:22.062 | INFO | ------------------------------------------ | 20260506 19:02:22.062 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:02:22.062 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:02:22.062 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:02:22.062 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:02:22.062 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.52e+01, 1.32e+01 / 2.04e+00 | 20260506 19:02:22.062 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 9.52e-03 / 4.84e-03 | 20260506 19:02:22.062 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.46e+00, 5.51e-01 / 2.97e-01 | 20260506 19:02:22.062 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.82e-01 / 1.28e-01 | 20260506 19:02:22.062 | INFO | PerfReport: 15.92%, DME -- 3.98e+01 / 4.39e+01, 3.94e+00 / 2.21e+00 | 20260506 19:02:22.062 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.48e+00 / 2.16e+00 | 20260506 19:02:22.062 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.24e+00 / 2.80e+00, 3.75e-01 / 2.45e-01 | 20260506 19:02:22.062 | INFO | PerfReport: 3.38%, Metrics -- 1.06e+00 / 1.39e+00, 2.80e-01 / 1.61e-01 | 20260506 19:02:22.062 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.64e-01 / 4.11e-01, 5.73e-02 / 3.02e-02 | 20260506 19:02:22.062 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:02:22.062 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:02:22.062 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:02:22.062 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 19:02:22.062 | INFO | PerfReport: 21.52%, Upload -- 2.09e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 19:02:22.062 | INFO | PerfReport: 21.15%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 19:02:22.062 | INFO | PerfReport: 32.18%, Compute -- 3.12e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 19:02:22.062 | INFO | PerfReport: 1.48%, LockUpload -- 1.43e+00 / 0.00e+00, 5.51e+00 / 0.00e+00 | 20260506 19:02:22.062 | INFO | PerfReport: 1.05%, LockDownload -- 1.02e+00 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260506 19:02:22.062 | INFO | PerfReport: 22.61%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 19:02:22.062 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260506 19:02:22.175 | INFO | Starting a round of Baz IO | 20260506 19:02:25.310 | INFO | Populating RTMetrics report | 20260506 19:02:25.310 | INFO | RTMetrics report populated | 20260506 19:02:25.310 | INFO | Opening tmp RTMetrics json file | 20260506 19:02:25.312 | INFO | Serializing RTMetrics to json | 20260506 19:02:25.328 | INFO | Serialization done, swapping tmp file now... | 20260506 19:02:25.350 | INFO | RTMetrics done | 20260506 19:02:25.479 | INFO | Finished a round of Baz IO: Wrote 2.022 GiB in 3.304 seconds | 20260506 19:02:25.479 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.80% / 0.01% / 65.20% | 20260506 19:02:25.479 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.994289% / 7.94729e-06% / 9.53674e-05% | 20260506 19:02:25.479 | INFO | Baz IO overall: | 20260506 19:02:25.479 | INFO | BytesWritten : 1961103577165 | 20260506 19:02:25.479 | INFO | NumEvents : 948168606539 | 20260506 19:02:27.199 | INFO | Analyzing chunk frames = [3710976,3711488) | 20260506 19:02:27.199 | INFO | MemInfo: { | 20260506 19:02:27.199 | INFO | "gpu" : " 17 GB", | 20260506 19:02:27.199 | INFO | "huge" : "", | 20260506 19:02:27.199 | INFO | "huge_shared" : " 27 GB", | 20260506 19:02:27.199 | INFO | "malloc" : " 14 GB", | 20260506 19:02:27.199 | INFO | "pinned" : " 29 GB" | 20260506 19:02:27.199 | INFO | } | 20260506 19:02:27.199 | INFO | Perf report for the last 5.13924 seconds: | 20260506 19:02:27.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:02:27.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:27.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:27.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:27.199 | INFO | ANALYSIS : 62.7%, 2.9, 95.4ms, 0 | 20260506 19:02:27.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:27.199 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 19:02:27.199 | INFO | PRE_HQ : 60.0%, 1.8, 57.6ms, 0 | 20260506 19:02:27.199 | INFO | BAZWRITER : 64.0%, 4.8, 156.5ms, 0 | 20260506 19:02:27.199 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 19:02:27.199 | INFO | Uploaded 36GiB to GPU in 2.051s (17.5525GiB/s) | 20260506 19:02:27.200 | INFO | Downloaded 38GiB to GPU in 2.21729s (17.1381GiB/s) | 20260506 19:02:32.344 | INFO | Analyzing chunk frames = [3711488,3712000) | 20260506 19:02:32.344 | INFO | MemInfo: { | 20260506 19:02:32.344 | INFO | "gpu" : " 17 GB", | 20260506 19:02:32.344 | INFO | "huge" : "", | 20260506 19:02:32.344 | INFO | "huge_shared" : " 27 GB", | 20260506 19:02:32.345 | INFO | "malloc" : " 14 GB", | 20260506 19:02:32.345 | INFO | "pinned" : " 29 GB" | 20260506 19:02:32.345 | INFO | } | 20260506 19:02:32.345 | INFO | Perf report for the last 5.14545 seconds: | 20260506 19:02:32.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:02:32.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:32.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:32.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:32.345 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 19:02:32.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:32.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:32.345 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:02:32.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:32.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:32.345 | INFO | Uploaded 36GiB to GPU in 2.01793s (17.8401GiB/s) | 20260506 19:02:32.345 | INFO | Downloaded 34GiB to GPU in 1.92859s (17.6295GiB/s) | 20260506 19:02:37.485 | INFO | Analyzing chunk frames = [3712000,3712512) | 20260506 19:02:37.485 | INFO | MemInfo: { | 20260506 19:02:37.485 | INFO | "gpu" : " 17 GB", | 20260506 19:02:37.485 | INFO | "huge" : "", | 20260506 19:02:37.485 | INFO | "huge_shared" : " 27 GB", | 20260506 19:02:37.485 | INFO | "malloc" : " 14 GB", | 20260506 19:02:37.485 | INFO | "pinned" : " 29 GB" | 20260506 19:02:37.485 | INFO | } | 20260506 19:02:37.485 | INFO | Perf report for the last 5.14043 seconds: | 20260506 19:02:37.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:02:37.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:37.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:37.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:37.485 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 19:02:37.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:37.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:37.485 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:02:37.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:37.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:37.485 | INFO | Uploaded 36GiB to GPU in 1.98568s (18.1298GiB/s) | 20260506 19:02:37.485 | INFO | Downloaded 34GiB to GPU in 1.91642s (17.7414GiB/s) | 20260506 19:02:42.622 | INFO | Analyzing chunk frames = [3712512,3713024) | 20260506 19:02:42.622 | INFO | MemInfo: { | 20260506 19:02:42.622 | INFO | "gpu" : " 17 GB", | 20260506 19:02:42.622 | INFO | "huge" : "", | 20260506 19:02:42.622 | INFO | "huge_shared" : " 27 GB", | 20260506 19:02:42.622 | INFO | "malloc" : " 14 GB", | 20260506 19:02:42.622 | INFO | "pinned" : " 29 GB" | 20260506 19:02:42.622 | INFO | } | 20260506 19:02:42.622 | INFO | Perf report for the last 5.13681 seconds: | 20260506 19:02:42.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:02:42.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:42.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:42.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:42.622 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 19:02:42.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:42.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:42.622 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:02:42.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:42.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:42.622 | INFO | Uploaded 36GiB to GPU in 1.93519s (18.6028GiB/s) | 20260506 19:02:42.622 | INFO | Downloaded 34GiB to GPU in 1.87187s (18.1637GiB/s) | 20260506 19:02:47.759 | INFO | Analyzing chunk frames = [3713024,3713536) | 20260506 19:02:47.759 | INFO | MemInfo: { | 20260506 19:02:47.759 | INFO | "gpu" : " 17 GB", | 20260506 19:02:47.759 | INFO | "huge" : "", | 20260506 19:02:47.759 | INFO | "huge_shared" : " 27 GB", | 20260506 19:02:47.759 | INFO | "malloc" : " 14 GB", | 20260506 19:02:47.759 | INFO | "pinned" : " 29 GB" | 20260506 19:02:47.759 | INFO | } | 20260506 19:02:47.760 | INFO | Perf report for the last 5.13766 seconds: | 20260506 19:02:47.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:02:47.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:47.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:47.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:47.760 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 19:02:47.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:47.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:47.760 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:02:47.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:47.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:47.760 | INFO | Uploaded 36GiB to GPU in 2.023s (17.7954GiB/s) | 20260506 19:02:47.760 | INFO | Downloaded 34GiB to GPU in 1.91103s (17.7915GiB/s) | 20260506 19:02:52.896 | INFO | Analyzing chunk frames = [3713536,3714048) | 20260506 19:02:52.896 | INFO | MemInfo: { | 20260506 19:02:52.896 | INFO | "gpu" : " 17 GB", | 20260506 19:02:52.896 | INFO | "huge" : "", | 20260506 19:02:52.896 | INFO | "huge_shared" : " 27 GB", | 20260506 19:02:52.896 | INFO | "malloc" : " 14 GB", | 20260506 19:02:52.896 | INFO | "pinned" : " 29 GB" | 20260506 19:02:52.896 | INFO | } | 20260506 19:02:52.896 | INFO | Perf report for the last 5.13678 seconds: | 20260506 19:02:52.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:02:52.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:52.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:52.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:52.896 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 19:02:52.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:52.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:52.896 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 19:02:52.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:52.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:52.897 | INFO | Uploaded 36GiB to GPU in 2.07497s (17.3497GiB/s) | 20260506 19:02:52.897 | INFO | Downloaded 34GiB to GPU in 1.96514s (17.3016GiB/s) | 20260506 19:02:58.036 | INFO | Analyzing chunk frames = [3714048,3714560) | 20260506 19:02:58.036 | INFO | MemInfo: { | 20260506 19:02:58.036 | INFO | "gpu" : " 17 GB", | 20260506 19:02:58.036 | INFO | "huge" : "", | 20260506 19:02:58.036 | INFO | "huge_shared" : " 27 GB", | 20260506 19:02:58.036 | INFO | "malloc" : " 14 GB", | 20260506 19:02:58.036 | INFO | "pinned" : " 29 GB" | 20260506 19:02:58.036 | INFO | } | 20260506 19:02:58.036 | INFO | Perf report for the last 5.13985 seconds: | 20260506 19:02:58.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:02:58.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:58.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:58.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:58.036 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 19:02:58.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:58.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:58.036 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:02:58.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:58.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:02:58.036 | INFO | Uploaded 36GiB to GPU in 2.01857s (17.8344GiB/s) | 20260506 19:02:58.036 | INFO | Downloaded 34GiB to GPU in 2.00648s (16.9451GiB/s) | 20260506 19:03:03.177 | INFO | Analyzing chunk frames = [3714560,3715072) | 20260506 19:03:03.177 | INFO | MemInfo: { | 20260506 19:03:03.177 | INFO | "gpu" : " 17 GB", | 20260506 19:03:03.177 | INFO | "huge" : "", | 20260506 19:03:03.177 | INFO | "huge_shared" : " 27 GB", | 20260506 19:03:03.177 | INFO | "malloc" : " 14 GB", | 20260506 19:03:03.177 | INFO | "pinned" : " 29 GB" | 20260506 19:03:03.177 | INFO | } | 20260506 19:03:03.177 | INFO | Perf report for the last 5.14118 seconds: | 20260506 19:03:03.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:03:03.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:03.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:03.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:03.177 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 19:03:03.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:03.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:03.177 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:03:03.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:03.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:03.178 | INFO | Uploaded 36GiB to GPU in 2.01049s (17.9061GiB/s) | 20260506 19:03:03.178 | INFO | Downloaded 34GiB to GPU in 1.99645s (17.0303GiB/s) | 20260506 19:03:03.293 | INFO | Starting a round of Baz IO | 20260506 19:03:06.423 | INFO | Populating RTMetrics report | 20260506 19:03:06.424 | INFO | RTMetrics report populated | 20260506 19:03:06.424 | INFO | Opening tmp RTMetrics json file | 20260506 19:03:06.449 | INFO | Serializing RTMetrics to json | 20260506 19:03:06.525 | INFO | Serialization done, swapping tmp file now... | 20260506 19:03:06.564 | INFO | RTMetrics done | 20260506 19:03:06.637 | INFO | Finished a round of Baz IO: Wrote 2.02 GiB in 3.343 seconds | 20260506 19:03:06.637 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.79% / 0.01% / 65.20% | 20260506 19:03:06.637 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.991873% / 1.19209e-05% / 7.94729e-05% | 20260506 19:03:06.637 | INFO | Baz IO overall: | 20260506 19:03:06.637 | INFO | BytesWritten : 1963272653041 | 20260506 19:03:06.637 | INFO | NumEvents : 949212198806 | 20260506 19:03:08.317 | INFO | Analyzing chunk frames = [3715072,3715584) | 20260506 19:03:08.317 | INFO | MemInfo: { | 20260506 19:03:08.317 | INFO | "gpu" : " 17 GB", | 20260506 19:03:08.317 | INFO | "huge" : "", | 20260506 19:03:08.317 | INFO | "huge_shared" : " 27 GB", | 20260506 19:03:08.317 | INFO | "malloc" : " 14 GB", | 20260506 19:03:08.317 | INFO | "pinned" : " 29 GB" | 20260506 19:03:08.317 | INFO | } | 20260506 19:03:08.317 | INFO | Perf report for the last 5.13931 seconds: | 20260506 19:03:08.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:03:08.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:08.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:08.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:08.317 | INFO | ANALYSIS : 62.6%, 2.9, 95.4ms, 0 | 20260506 19:03:08.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:08.317 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260506 19:03:08.317 | INFO | PRE_HQ : 60.0%, 1.9, 57.7ms, 0 | 20260506 19:03:08.317 | INFO | BAZWRITER : 64.7%, 4.8, 159.6ms, 0 | 20260506 19:03:08.317 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260506 19:03:08.317 | INFO | Uploaded 36GiB to GPU in 2.02754s (17.7555GiB/s) | 20260506 19:03:08.317 | INFO | Downloaded 38GiB to GPU in 2.2158s (17.1495GiB/s) | 20260506 19:03:13.460 | INFO | Analyzing chunk frames = [3715584,3716096) | 20260506 19:03:13.460 | INFO | MemInfo: { | 20260506 19:03:13.460 | INFO | "gpu" : " 17 GB", | 20260506 19:03:13.460 | INFO | "huge" : "", | 20260506 19:03:13.460 | INFO | "huge_shared" : " 27 GB", | 20260506 19:03:13.460 | INFO | "malloc" : " 14 GB", | 20260506 19:03:13.460 | INFO | "pinned" : " 29 GB" | 20260506 19:03:13.460 | INFO | } | 20260506 19:03:13.461 | INFO | Perf report for the last 5.14379 seconds: | 20260506 19:03:13.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:03:13.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:13.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:13.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:13.461 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 19:03:13.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:13.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:13.461 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 19:03:13.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:13.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:13.461 | INFO | Uploaded 36GiB to GPU in 1.993s (18.0632GiB/s) | 20260506 19:03:13.461 | INFO | Downloaded 34GiB to GPU in 1.92092s (17.6998GiB/s) | 20260506 19:03:18.602 | INFO | Analyzing chunk frames = [3716096,3716608) | 20260506 19:03:18.602 | INFO | MemInfo: { | 20260506 19:03:18.602 | INFO | "gpu" : " 17 GB", | 20260506 19:03:18.602 | INFO | "huge" : "", | 20260506 19:03:18.602 | INFO | "huge_shared" : " 27 GB", | 20260506 19:03:18.602 | INFO | "malloc" : " 14 GB", | 20260506 19:03:18.602 | INFO | "pinned" : " 29 GB" | 20260506 19:03:18.602 | INFO | } | 20260506 19:03:18.602 | INFO | Perf report for the last 5.14167 seconds: | 20260506 19:03:18.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:03:18.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:18.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:18.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:18.602 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 19:03:18.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:18.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:18.602 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:03:18.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:18.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:18.602 | INFO | Uploaded 36GiB to GPU in 1.95974s (18.3698GiB/s) | 20260506 19:03:18.602 | INFO | Downloaded 34GiB to GPU in 1.92257s (17.6846GiB/s) | 20260506 19:03:23.737 | INFO | Analyzing chunk frames = [3716608,3717120) | 20260506 19:03:23.737 | INFO | MemInfo: { | 20260506 19:03:23.737 | INFO | "gpu" : " 17 GB", | 20260506 19:03:23.737 | INFO | "huge" : "", | 20260506 19:03:23.738 | INFO | "huge_shared" : " 27 GB", | 20260506 19:03:23.738 | INFO | "malloc" : " 14 GB", | 20260506 19:03:23.738 | INFO | "pinned" : " 29 GB" | 20260506 19:03:23.738 | INFO | } | 20260506 19:03:23.738 | INFO | Perf report for the last 5.13554 seconds: | 20260506 19:03:23.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:03:23.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:23.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:23.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:23.738 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 19:03:23.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:23.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:23.738 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:03:23.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:23.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:23.738 | INFO | Uploaded 36GiB to GPU in 1.95493s (18.415GiB/s) | 20260506 19:03:23.738 | INFO | Downloaded 34GiB to GPU in 1.89404s (17.9511GiB/s) | 20260506 19:03:28.875 | INFO | Analyzing chunk frames = [3717120,3717632) | 20260506 19:03:28.875 | INFO | MemInfo: { | 20260506 19:03:28.875 | INFO | "gpu" : " 17 GB", | 20260506 19:03:28.875 | INFO | "huge" : "", | 20260506 19:03:28.875 | INFO | "huge_shared" : " 27 GB", | 20260506 19:03:28.875 | INFO | "malloc" : " 14 GB", | 20260506 19:03:28.875 | INFO | "pinned" : " 29 GB" | 20260506 19:03:28.875 | INFO | } | 20260506 19:03:28.875 | INFO | Perf report for the last 5.13734 seconds: | 20260506 19:03:28.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:03:28.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:28.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:28.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:28.875 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 19:03:28.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:28.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:28.875 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:03:28.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:28.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:28.875 | INFO | Uploaded 36GiB to GPU in 1.98495s (18.1364GiB/s) | 20260506 19:03:28.875 | INFO | Downloaded 34GiB to GPU in 1.91635s (17.7421GiB/s) | 20260506 19:03:34.009 | INFO | Analyzing chunk frames = [3717632,3718144) | 20260506 19:03:34.009 | INFO | MemInfo: { | 20260506 19:03:34.009 | INFO | "gpu" : " 17 GB", | 20260506 19:03:34.009 | INFO | "huge" : "", | 20260506 19:03:34.009 | INFO | "huge_shared" : " 27 GB", | 20260506 19:03:34.009 | INFO | "malloc" : " 14 GB", | 20260506 19:03:34.009 | INFO | "pinned" : " 29 GB" | 20260506 19:03:34.009 | INFO | } | 20260506 19:03:34.009 | INFO | Perf report for the last 5.1338 seconds: | 20260506 19:03:34.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:03:34.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:34.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:34.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:34.009 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 19:03:34.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:34.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:34.009 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 19:03:34.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:34.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:34.009 | INFO | Uploaded 36GiB to GPU in 2.0055s (17.9506GiB/s) | 20260506 19:03:34.009 | INFO | Downloaded 34GiB to GPU in 1.95546s (17.3872GiB/s) | 20260506 19:03:39.149 | INFO | Analyzing chunk frames = [3718144,3718656) | 20260506 19:03:39.149 | INFO | MemInfo: { | 20260506 19:03:39.149 | INFO | "gpu" : " 17 GB", | 20260506 19:03:39.149 | INFO | "huge" : "", | 20260506 19:03:39.149 | INFO | "huge_shared" : " 27 GB", | 20260506 19:03:39.149 | INFO | "malloc" : " 14 GB", | 20260506 19:03:39.149 | INFO | "pinned" : " 29 GB" | 20260506 19:03:39.149 | INFO | } | 20260506 19:03:39.149 | INFO | Perf report for the last 5.14046 seconds: | 20260506 19:03:39.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:03:39.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:39.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:39.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:39.149 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 19:03:39.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:39.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:39.149 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:03:39.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:39.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:39.150 | INFO | Uploaded 36GiB to GPU in 2.00223s (17.98GiB/s) | 20260506 19:03:39.150 | INFO | Downloaded 34GiB to GPU in 2.00975s (16.9175GiB/s) | 20260506 19:03:44.293 | INFO | Analyzing chunk frames = [3718656,3719168) | 20260506 19:03:44.293 | INFO | MemInfo: { | 20260506 19:03:44.293 | INFO | "gpu" : " 17 GB", | 20260506 19:03:44.293 | INFO | "huge" : "", | 20260506 19:03:44.293 | INFO | "huge_shared" : " 27 GB", | 20260506 19:03:44.293 | INFO | "malloc" : " 14 GB", | 20260506 19:03:44.294 | INFO | "pinned" : " 29 GB" | 20260506 19:03:44.294 | INFO | } | 20260506 19:03:44.294 | INFO | Perf report for the last 5.14428 seconds: | 20260506 19:03:44.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:03:44.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:44.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:44.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:44.294 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 19:03:44.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:44.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:44.294 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:03:44.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:44.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:44.294 | INFO | Uploaded 36GiB to GPU in 1.98316s (18.1529GiB/s) | 20260506 19:03:44.294 | INFO | Downloaded 34GiB to GPU in 1.99003s (17.0852GiB/s) | 20260506 19:03:44.294 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:03:44.294 | INFO | Total mem usage: 18 GiB | 20260506 19:03:44.294 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:03:44.294 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -18%) | 20260506 19:03:44.294 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:03:44.294 | INFO | High Water Breakdown: | 20260506 19:03:44.294 | INFO | 61.1% : 9.32 GiB ( 81.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:03:44.294 | INFO | 38.8% : 5.92 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:03:44.294 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:03:44.294 | INFO | ------------------------------------------ | 20260506 19:03:44.294 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:03:44.294 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:03:44.294 | INFO | Total mem usage: 26.5 GiB | 20260506 19:03:44.294 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:03:44.294 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:03:44.294 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:03:44.294 | INFO | High Water Breakdown: | 20260506 19:03:44.294 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:03:44.294 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:03:44.294 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:03:44.294 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:03:44.294 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:03:44.294 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:03:44.294 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:03:44.294 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:03:44.295 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:03:44.295 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:03:44.295 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:03:44.295 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:03:44.295 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:03:44.295 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:03:44.295 | INFO | ------------------------------------------ | 20260506 19:03:44.295 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:03:44.295 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:03:44.295 | INFO | Total mem usage: 25.6 GiB | 20260506 19:03:44.295 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:03:44.295 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:03:44.295 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:03:44.295 | INFO | High Water Breakdown: | 20260506 19:03:44.295 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:03:44.295 | INFO | ------------------------------------------ | 20260506 19:03:44.295 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:03:44.295 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:03:44.295 | INFO | Total mem usage: 15.7 GiB | 20260506 19:03:44.295 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:03:44.295 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:03:44.295 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:03:44.295 | INFO | High Water Breakdown: | 20260506 19:03:44.295 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:03:44.295 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:03:44.295 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:03:44.295 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:03:44.295 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:03:44.295 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:03:44.295 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:03:44.295 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:03:44.295 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:03:44.295 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:03:44.295 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:03:44.295 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:03:44.295 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:03:44.295 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:03:44.295 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:03:44.295 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:03:44.295 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:03:44.295 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:03:44.295 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:03:44.295 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:03:44.295 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:03:44.295 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:03:44.295 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:03:44.295 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:03:44.295 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:03:44.295 | INFO | ------------------------------------------ | 20260506 19:03:44.295 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:03:44.295 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:03:44.295 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:03:44.295 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:03:44.295 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.53e+01, 1.35e+01 / 2.37e+00 | 20260506 19:03:44.295 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.02e-01, 1.00e-02 / 6.25e-03 | 20260506 19:03:44.295 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.42e+00, 5.52e-01 / 3.56e-01 | 20260506 19:03:44.295 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.45e+00, 1.81e-01 / 1.37e-01 | 20260506 19:03:44.295 | INFO | PerfReport: 15.97%, DME -- 3.99e+01 / 4.42e+01, 4.16e+00 / 2.60e+00 | 20260506 19:03:44.295 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.42e+00 / 2.15e+00 | 20260506 19:03:44.295 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.25e+00 / 2.76e+00, 3.82e-01 / 2.83e-01 | 20260506 19:03:44.295 | INFO | PerfReport: 3.38%, Metrics -- 1.06e+00 / 1.36e+00, 2.78e-01 / 1.81e-01 | 20260506 19:03:44.295 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.64e-01 / 4.09e-01, 5.70e-02 / 2.80e-02 | 20260506 19:03:44.295 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:03:44.295 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:03:44.295 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:03:44.295 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 19:03:44.295 | INFO | PerfReport: 21.48%, Upload -- 2.09e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260506 19:03:44.295 | INFO | PerfReport: 21.22%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 19:03:44.295 | INFO | PerfReport: 32.18%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 19:03:44.295 | INFO | PerfReport: 1.38%, LockUpload -- 1.34e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260506 19:03:44.295 | INFO | PerfReport: 1.16%, LockDownload -- 1.12e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 19:03:44.295 | INFO | PerfReport: 22.57%, LockCompute -- 2.19e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 19:03:44.295 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.52e-02 / 0.00e+00 | 20260506 19:03:44.406 | INFO | Starting a round of Baz IO | 20260506 19:03:47.532 | INFO | Populating RTMetrics report | 20260506 19:03:47.532 | INFO | RTMetrics report populated | 20260506 19:03:47.532 | INFO | Opening tmp RTMetrics json file | 20260506 19:03:47.572 | INFO | Serializing RTMetrics to json | 20260506 19:03:47.576 | INFO | Serialization done, swapping tmp file now... | 20260506 19:03:47.580 | INFO | RTMetrics done | 20260506 19:03:48.141 | INFO | Finished a round of Baz IO: Wrote 2.02 GiB in 3.734 seconds | 20260506 19:03:48.141 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.79% / 0.01% / 65.21% | 20260506 19:03:48.141 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.989004% / 3.97364e-06% / 5.96046e-05% | 20260506 19:03:48.141 | INFO | Baz IO overall: | 20260506 19:03:48.141 | INFO | BytesWritten : 1965441178162 | 20260506 19:03:48.141 | INFO | NumEvents : 950255275699 | 20260506 19:03:49.430 | INFO | Analyzing chunk frames = [3719168,3719680) | 20260506 19:03:49.430 | INFO | MemInfo: { | 20260506 19:03:49.430 | INFO | "gpu" : " 17 GB", | 20260506 19:03:49.430 | INFO | "huge" : "", | 20260506 19:03:49.430 | INFO | "huge_shared" : " 27 GB", | 20260506 19:03:49.430 | INFO | "malloc" : " 14 GB", | 20260506 19:03:49.430 | INFO | "pinned" : " 29 GB" | 20260506 19:03:49.430 | INFO | } | 20260506 19:03:49.430 | INFO | Perf report for the last 5.13651 seconds: | 20260506 19:03:49.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:03:49.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:49.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:49.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:49.430 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 19:03:49.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:49.430 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 19:03:49.430 | INFO | PRE_HQ : 60.0%, 1.8, 57.4ms, 0 | 20260506 19:03:49.430 | INFO | BAZWRITER : 72.4%, 4.8, 177.9ms, 0 | 20260506 19:03:49.430 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 19:03:49.431 | INFO | Uploaded 36GiB to GPU in 2.0436s (17.616GiB/s) | 20260506 19:03:49.431 | INFO | Downloaded 38GiB to GPU in 2.20058s (17.2681GiB/s) | 20260506 19:03:54.570 | INFO | Analyzing chunk frames = [3719680,3720192) | 20260506 19:03:54.570 | INFO | MemInfo: { | 20260506 19:03:54.570 | INFO | "gpu" : " 17 GB", | 20260506 19:03:54.570 | INFO | "huge" : "", | 20260506 19:03:54.570 | INFO | "huge_shared" : " 27 GB", | 20260506 19:03:54.570 | INFO | "malloc" : " 14 GB", | 20260506 19:03:54.570 | INFO | "pinned" : " 29 GB" | 20260506 19:03:54.570 | INFO | } | 20260506 19:03:54.570 | INFO | Perf report for the last 5.14 seconds: | 20260506 19:03:54.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:03:54.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:54.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:54.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:54.570 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 19:03:54.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:54.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:54.570 | INFO | PRE_HQ : 26.4%, 1.0, 13.8ms, 0 | 20260506 19:03:54.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:54.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:54.570 | INFO | Uploaded 36GiB to GPU in 2.00819s (17.9266GiB/s) | 20260506 19:03:54.570 | INFO | Downloaded 34GiB to GPU in 1.93707s (17.5523GiB/s) | 20260506 19:03:59.708 | INFO | Analyzing chunk frames = [3720192,3720704) | 20260506 19:03:59.708 | INFO | MemInfo: { | 20260506 19:03:59.708 | INFO | "gpu" : " 17 GB", | 20260506 19:03:59.708 | INFO | "huge" : "", | 20260506 19:03:59.708 | INFO | "huge_shared" : " 27 GB", | 20260506 19:03:59.708 | INFO | "malloc" : " 14 GB", | 20260506 19:03:59.708 | INFO | "pinned" : " 29 GB" | 20260506 19:03:59.708 | INFO | } | 20260506 19:03:59.708 | INFO | Perf report for the last 5.13815 seconds: | 20260506 19:03:59.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:03:59.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:59.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:59.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:59.708 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:03:59.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:59.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:59.708 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:03:59.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:59.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:03:59.709 | INFO | Uploaded 36GiB to GPU in 1.96339s (18.3356GiB/s) | 20260506 19:03:59.709 | INFO | Downloaded 34GiB to GPU in 1.91885s (17.719GiB/s) | 20260506 19:04:04.854 | INFO | Analyzing chunk frames = [3720704,3721216) | 20260506 19:04:04.854 | INFO | MemInfo: { | 20260506 19:04:04.854 | INFO | "gpu" : " 17 GB", | 20260506 19:04:04.854 | INFO | "huge" : "", | 20260506 19:04:04.854 | INFO | "huge_shared" : " 27 GB", | 20260506 19:04:04.854 | INFO | "malloc" : " 14 GB", | 20260506 19:04:04.854 | INFO | "pinned" : " 29 GB" | 20260506 19:04:04.854 | INFO | } | 20260506 19:04:04.854 | INFO | Perf report for the last 5.14571 seconds: | 20260506 19:04:04.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:04:04.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:04.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:04.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:04.854 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 19:04:04.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:04.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:04.854 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:04:04.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:04.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:04.855 | INFO | Uploaded 36GiB to GPU in 1.93202s (18.6334GiB/s) | 20260506 19:04:04.855 | INFO | Downloaded 34GiB to GPU in 1.86903s (18.1913GiB/s) | 20260506 19:04:09.988 | INFO | Analyzing chunk frames = [3721216,3721728) | 20260506 19:04:09.988 | INFO | MemInfo: { | 20260506 19:04:09.988 | INFO | "gpu" : " 17 GB", | 20260506 19:04:09.988 | INFO | "huge" : "", | 20260506 19:04:09.988 | INFO | "huge_shared" : " 27 GB", | 20260506 19:04:09.988 | INFO | "malloc" : " 14 GB", | 20260506 19:04:09.988 | INFO | "pinned" : " 29 GB" | 20260506 19:04:09.988 | INFO | } | 20260506 19:04:09.988 | INFO | Perf report for the last 5.13404 seconds: | 20260506 19:04:09.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:04:09.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:09.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:09.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:09.988 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 19:04:09.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:09.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:09.988 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:04:09.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:09.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:09.989 | INFO | Uploaded 36GiB to GPU in 1.99562s (18.0395GiB/s) | 20260506 19:04:09.989 | INFO | Downloaded 34GiB to GPU in 1.90807s (17.8191GiB/s) | 20260506 19:04:15.128 | INFO | Analyzing chunk frames = [3721728,3722240) | 20260506 19:04:15.128 | INFO | MemInfo: { | 20260506 19:04:15.128 | INFO | "gpu" : " 17 GB", | 20260506 19:04:15.128 | INFO | "huge" : "", | 20260506 19:04:15.128 | INFO | "huge_shared" : " 27 GB", | 20260506 19:04:15.128 | INFO | "malloc" : " 14 GB", | 20260506 19:04:15.128 | INFO | "pinned" : " 29 GB" | 20260506 19:04:15.128 | INFO | } | 20260506 19:04:15.128 | INFO | Perf report for the last 5.14019 seconds: | 20260506 19:04:15.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:04:15.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:15.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:15.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:15.128 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 19:04:15.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:15.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:15.128 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 19:04:15.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:15.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:15.129 | INFO | Uploaded 36GiB to GPU in 2.0079s (17.9292GiB/s) | 20260506 19:04:15.129 | INFO | Downloaded 34GiB to GPU in 1.94294s (17.4993GiB/s) | 20260506 19:04:20.268 | INFO | Analyzing chunk frames = [3722240,3722752) | 20260506 19:04:20.268 | INFO | MemInfo: { | 20260506 19:04:20.268 | INFO | "gpu" : " 17 GB", | 20260506 19:04:20.268 | INFO | "huge" : "", | 20260506 19:04:20.268 | INFO | "huge_shared" : " 27 GB", | 20260506 19:04:20.268 | INFO | "malloc" : " 14 GB", | 20260506 19:04:20.268 | INFO | "pinned" : " 29 GB" | 20260506 19:04:20.268 | INFO | } | 20260506 19:04:20.268 | INFO | Perf report for the last 5.13978 seconds: | 20260506 19:04:20.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:04:20.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:20.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:20.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:20.268 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 19:04:20.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:20.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:20.268 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 19:04:20.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:20.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:20.268 | INFO | Uploaded 36GiB to GPU in 2.0068s (17.939GiB/s) | 20260506 19:04:20.268 | INFO | Downloaded 34GiB to GPU in 2.00785s (16.9335GiB/s) | 20260506 19:04:25.407 | INFO | Analyzing chunk frames = [3722752,3723264) | 20260506 19:04:25.407 | INFO | MemInfo: { | 20260506 19:04:25.407 | INFO | "gpu" : " 17 GB", | 20260506 19:04:25.407 | INFO | "huge" : "", | 20260506 19:04:25.407 | INFO | "huge_shared" : " 27 GB", | 20260506 19:04:25.407 | INFO | "malloc" : " 14 GB", | 20260506 19:04:25.407 | INFO | "pinned" : " 29 GB" | 20260506 19:04:25.407 | INFO | } | 20260506 19:04:25.407 | INFO | Perf report for the last 5.13923 seconds: | 20260506 19:04:25.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:04:25.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:25.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:25.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:25.407 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 19:04:25.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:25.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:25.407 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260506 19:04:25.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:25.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:25.408 | INFO | Uploaded 36GiB to GPU in 1.98072s (18.1753GiB/s) | 20260506 19:04:25.408 | INFO | Downloaded 34GiB to GPU in 1.98651s (17.1155GiB/s) | 20260506 19:04:25.522 | INFO | Starting a round of Baz IO | 20260506 19:04:28.707 | INFO | Populating RTMetrics report | 20260506 19:04:28.707 | INFO | RTMetrics report populated | 20260506 19:04:28.707 | INFO | Opening tmp RTMetrics json file | 20260506 19:04:28.710 | INFO | Serializing RTMetrics to json | 20260506 19:04:28.739 | INFO | Serialization done, swapping tmp file now... | 20260506 19:04:28.741 | INFO | RTMetrics done | 20260506 19:04:29.009 | INFO | Finished a round of Baz IO: Wrote 2.02 GiB in 3.487 seconds | 20260506 19:04:29.010 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.78% / 0.01% / 65.21% | 20260506 19:04:29.010 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.98664% / 0% / 3.97364e-05% | 20260506 19:04:29.010 | INFO | Baz IO overall: | 20260506 19:04:29.010 | INFO | BytesWritten : 1967609722669 | 20260506 19:04:29.010 | INFO | NumEvents : 951298389295 | 20260506 19:04:30.550 | INFO | Analyzing chunk frames = [3723264,3723776) | 20260506 19:04:30.550 | INFO | MemInfo: { | 20260506 19:04:30.550 | INFO | "gpu" : " 17 GB", | 20260506 19:04:30.550 | INFO | "huge" : "", | 20260506 19:04:30.550 | INFO | "huge_shared" : " 27 GB", | 20260506 19:04:30.550 | INFO | "malloc" : " 14 GB", | 20260506 19:04:30.550 | INFO | "pinned" : " 29 GB" | 20260506 19:04:30.550 | INFO | } | 20260506 19:04:30.550 | INFO | Perf report for the last 5.14292 seconds: | 20260506 19:04:30.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:04:30.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:30.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:30.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:30.550 | INFO | ANALYSIS : 63.0%, 2.8, 94.1ms, 0 | 20260506 19:04:30.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:30.550 | INFO | RT_METRICS : 47.8%, 1.0, 25.1ms, 0 | 20260506 19:04:30.550 | INFO | PRE_HQ : 61.1%, 2.0, 64.0ms, 0 | 20260506 19:04:30.550 | INFO | BAZWRITER : 67.5%, 4.8, 166.5ms, 0 | 20260506 19:04:30.550 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 19:04:30.551 | INFO | Uploaded 36GiB to GPU in 2.05679s (17.503GiB/s) | 20260506 19:04:30.551 | INFO | Downloaded 38GiB to GPU in 2.17526s (17.4692GiB/s) | 20260506 19:04:35.686 | INFO | Analyzing chunk frames = [3723776,3724288) | 20260506 19:04:35.686 | INFO | MemInfo: { | 20260506 19:04:35.686 | INFO | "gpu" : " 17 GB", | 20260506 19:04:35.686 | INFO | "huge" : "", | 20260506 19:04:35.686 | INFO | "huge_shared" : " 27 GB", | 20260506 19:04:35.686 | INFO | "malloc" : " 14 GB", | 20260506 19:04:35.686 | INFO | "pinned" : " 29 GB" | 20260506 19:04:35.686 | INFO | } | 20260506 19:04:35.686 | INFO | Perf report for the last 5.136 seconds: | 20260506 19:04:35.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:04:35.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:35.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:35.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:35.686 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:04:35.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:35.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:35.686 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 19:04:35.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:35.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:35.687 | INFO | Uploaded 36GiB to GPU in 2.01365s (17.878GiB/s) | 20260506 19:04:35.687 | INFO | Downloaded 34GiB to GPU in 1.93248s (17.594GiB/s) | 20260506 19:04:40.825 | INFO | Analyzing chunk frames = [3724288,3724800) | 20260506 19:04:40.825 | INFO | MemInfo: { | 20260506 19:04:40.825 | INFO | "gpu" : " 17 GB", | 20260506 19:04:40.825 | INFO | "huge" : "", | 20260506 19:04:40.825 | INFO | "huge_shared" : " 27 GB", | 20260506 19:04:40.825 | INFO | "malloc" : " 14 GB", | 20260506 19:04:40.825 | INFO | "pinned" : " 29 GB" | 20260506 19:04:40.825 | INFO | } | 20260506 19:04:40.825 | INFO | Perf report for the last 5.13869 seconds: | 20260506 19:04:40.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:04:40.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:40.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:40.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:40.825 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 19:04:40.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:40.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:40.825 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:04:40.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:40.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:40.825 | INFO | Uploaded 36GiB to GPU in 1.97643s (18.2147GiB/s) | 20260506 19:04:40.825 | INFO | Downloaded 34GiB to GPU in 1.91757s (17.7308GiB/s) | 20260506 19:04:45.964 | INFO | Analyzing chunk frames = [3724800,3725312) | 20260506 19:04:45.964 | INFO | MemInfo: { | 20260506 19:04:45.964 | INFO | "gpu" : " 17 GB", | 20260506 19:04:45.964 | INFO | "huge" : "", | 20260506 19:04:45.965 | INFO | "huge_shared" : " 27 GB", | 20260506 19:04:45.965 | INFO | "malloc" : " 14 GB", | 20260506 19:04:45.965 | INFO | "pinned" : " 29 GB" | 20260506 19:04:45.965 | INFO | } | 20260506 19:04:45.965 | INFO | Perf report for the last 5.13961 seconds: | 20260506 19:04:45.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:04:45.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:45.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:45.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:45.965 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 19:04:45.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:45.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:45.965 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:04:45.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:45.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:45.965 | INFO | Uploaded 36GiB to GPU in 1.95123s (18.4499GiB/s) | 20260506 19:04:45.965 | INFO | Downloaded 34GiB to GPU in 1.87205s (18.1619GiB/s) | 20260506 19:04:51.105 | INFO | Analyzing chunk frames = [3725312,3725824) | 20260506 19:04:51.105 | INFO | MemInfo: { | 20260506 19:04:51.105 | INFO | "gpu" : " 17 GB", | 20260506 19:04:51.105 | INFO | "huge" : "", | 20260506 19:04:51.105 | INFO | "huge_shared" : " 27 GB", | 20260506 19:04:51.105 | INFO | "malloc" : " 14 GB", | 20260506 19:04:51.105 | INFO | "pinned" : " 29 GB" | 20260506 19:04:51.105 | INFO | } | 20260506 19:04:51.105 | INFO | Perf report for the last 5.14014 seconds: | 20260506 19:04:51.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:04:51.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:51.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:51.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:51.105 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 19:04:51.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:51.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:51.105 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:04:51.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:51.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:51.105 | INFO | Uploaded 36GiB to GPU in 2.01939s (17.8272GiB/s) | 20260506 19:04:51.105 | INFO | Downloaded 34GiB to GPU in 1.91071s (17.7944GiB/s) | 20260506 19:04:56.242 | INFO | Analyzing chunk frames = [3725824,3726336) | 20260506 19:04:56.242 | INFO | MemInfo: { | 20260506 19:04:56.242 | INFO | "gpu" : " 17 GB", | 20260506 19:04:56.242 | INFO | "huge" : "", | 20260506 19:04:56.242 | INFO | "huge_shared" : " 27 GB", | 20260506 19:04:56.242 | INFO | "malloc" : " 14 GB", | 20260506 19:04:56.243 | INFO | "pinned" : " 29 GB" | 20260506 19:04:56.243 | INFO | } | 20260506 19:04:56.243 | INFO | Perf report for the last 5.13777 seconds: | 20260506 19:04:56.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:04:56.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:56.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:56.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:56.243 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:04:56.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:56.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:56.243 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:04:56.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:56.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:04:56.243 | INFO | Uploaded 36GiB to GPU in 2.06689s (17.4175GiB/s) | 20260506 19:04:56.243 | INFO | Downloaded 34GiB to GPU in 1.95443s (17.3964GiB/s) | 20260506 19:05:01.377 | INFO | Analyzing chunk frames = [3726336,3726848) | 20260506 19:05:01.377 | INFO | MemInfo: { | 20260506 19:05:01.377 | INFO | "gpu" : " 17 GB", | 20260506 19:05:01.377 | INFO | "huge" : "", | 20260506 19:05:01.377 | INFO | "huge_shared" : " 27 GB", | 20260506 19:05:01.377 | INFO | "malloc" : " 14 GB", | 20260506 19:05:01.377 | INFO | "pinned" : " 29 GB" | 20260506 19:05:01.377 | INFO | } | 20260506 19:05:01.377 | INFO | Perf report for the last 5.13434 seconds: | 20260506 19:05:01.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:05:01.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:01.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:01.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:01.377 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 19:05:01.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:01.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:01.377 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 19:05:01.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:01.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:01.377 | INFO | Uploaded 36GiB to GPU in 2.02447s (17.7824GiB/s) | 20260506 19:05:01.377 | INFO | Downloaded 34GiB to GPU in 2.00096s (16.9918GiB/s) | 20260506 19:05:06.518 | INFO | Analyzing chunk frames = [3726848,3727360) | 20260506 19:05:06.518 | INFO | MemInfo: { | 20260506 19:05:06.518 | INFO | "gpu" : " 17 GB", | 20260506 19:05:06.518 | INFO | "huge" : "", | 20260506 19:05:06.518 | INFO | "huge_shared" : " 27 GB", | 20260506 19:05:06.518 | INFO | "malloc" : " 14 GB", | 20260506 19:05:06.518 | INFO | "pinned" : " 29 GB" | 20260506 19:05:06.518 | INFO | } | 20260506 19:05:06.518 | INFO | Perf report for the last 5.14087 seconds: | 20260506 19:05:06.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:05:06.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:06.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:06.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:06.518 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 19:05:06.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:06.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:06.518 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260506 19:05:06.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:06.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:06.518 | INFO | Uploaded 36GiB to GPU in 2.00243s (17.9782GiB/s) | 20260506 19:05:06.518 | INFO | Downloaded 34GiB to GPU in 1.99572s (17.0364GiB/s) | 20260506 19:05:06.518 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:05:06.518 | INFO | Total mem usage: 18 GiB | 20260506 19:05:06.518 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:05:06.518 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -10.9%) | 20260506 19:05:06.518 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:05:06.518 | INFO | High Water Breakdown: | 20260506 19:05:06.518 | INFO | 62% : 10.1 GiB ( 88.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:05:06.518 | INFO | 37.9% : 6.15 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:05:06.518 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:05:06.518 | INFO | ------------------------------------------ | 20260506 19:05:06.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:05:06.518 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:05:06.518 | INFO | Total mem usage: 26.5 GiB | 20260506 19:05:06.518 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:05:06.518 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 19:05:06.518 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260506 19:05:06.518 | INFO | High Water Breakdown: | 20260506 19:05:06.518 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:05:06.518 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:05:06.518 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:05:06.518 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:05:06.519 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:05:06.519 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:05:06.519 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:05:06.519 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:05:06.519 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:05:06.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:05:06.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:05:06.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:05:06.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:05:06.519 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:05:06.519 | INFO | ------------------------------------------ | 20260506 19:05:06.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:05:06.519 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:05:06.519 | INFO | Total mem usage: 25.6 GiB | 20260506 19:05:06.519 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:05:06.519 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:05:06.519 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:05:06.519 | INFO | High Water Breakdown: | 20260506 19:05:06.519 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:05:06.519 | INFO | ------------------------------------------ | 20260506 19:05:06.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:05:06.519 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:05:06.519 | INFO | Total mem usage: 15.7 GiB | 20260506 19:05:06.519 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:05:06.519 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:05:06.519 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:05:06.519 | INFO | High Water Breakdown: | 20260506 19:05:06.519 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:05:06.519 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:05:06.519 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:05:06.519 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:05:06.519 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:05:06.519 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:05:06.519 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:05:06.519 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:05:06.519 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:05:06.519 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:05:06.519 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:05:06.519 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:05:06.519 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:05:06.519 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:05:06.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:05:06.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:05:06.519 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:05:06.519 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:05:06.519 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:05:06.519 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:05:06.519 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:05:06.519 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:05:06.519 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:05:06.519 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:05:06.519 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:05:06.519 | INFO | ------------------------------------------ | 20260506 19:05:06.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:05:06.519 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:05:06.519 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:05:06.519 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:05:06.519 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.57e+01, 1.35e+01 / 2.57e+00 | 20260506 19:05:06.519 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.01e-01, 1.00e-02 / 5.88e-03 | 20260506 19:05:06.519 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.42e+00, 5.55e-01 / 3.66e-01 | 20260506 19:05:06.519 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.47e+00, 1.85e-01 / 1.35e-01 | 20260506 19:05:06.519 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.45e+01, 4.21e+00 / 3.00e+00 | 20260506 19:05:06.519 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.44e+00 / 2.13e+00 | 20260506 19:05:06.519 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.24e+00 / 2.82e+00, 3.81e-01 / 2.44e-01 | 20260506 19:05:06.519 | INFO | PerfReport: 3.38%, Metrics -- 1.06e+00 / 1.41e+00, 2.80e-01 / 1.72e-01 | 20260506 19:05:06.519 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.64e-01 / 4.06e-01, 5.69e-02 / 1.82e-02 | 20260506 19:05:06.519 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:05:06.519 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:05:06.519 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:05:06.519 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 19:05:06.519 | INFO | PerfReport: 21.52%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260506 19:05:06.519 | INFO | PerfReport: 21.17%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 19:05:06.519 | INFO | PerfReport: 32.24%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 19:05:06.519 | INFO | PerfReport: 1.43%, LockUpload -- 1.38e+00 / 0.00e+00, 5.36e+00 / 0.00e+00 | 20260506 19:05:06.519 | INFO | PerfReport: 1.07%, LockDownload -- 1.04e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260506 19:05:06.519 | INFO | PerfReport: 22.57%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 19:05:06.519 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260506 19:05:06.632 | INFO | Starting a round of Baz IO | 20260506 19:05:09.769 | INFO | Populating RTMetrics report | 20260506 19:05:09.769 | INFO | RTMetrics report populated | 20260506 19:05:09.769 | INFO | Opening tmp RTMetrics json file | 20260506 19:05:09.898 | INFO | Serializing RTMetrics to json | 20260506 19:05:09.907 | INFO | Serialization done, swapping tmp file now... | 20260506 19:05:09.911 | INFO | RTMetrics done | 20260506 19:05:10.285 | INFO | Finished a round of Baz IO: Wrote 2.021 GiB in 3.653 seconds | 20260506 19:05:10.285 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.78% / 0.01% / 65.22% | 20260506 19:05:10.285 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.983604% / 3.97364e-06% / 6.75519e-05% | 20260506 19:05:10.285 | INFO | Baz IO overall: | 20260506 19:05:10.285 | INFO | BytesWritten : 1969779921760 | 20260506 19:05:10.285 | INFO | NumEvents : 952342835119 | 20260506 19:05:11.661 | INFO | Analyzing chunk frames = [3727360,3727872) | 20260506 19:05:11.661 | INFO | MemInfo: { | 20260506 19:05:11.661 | INFO | "gpu" : " 17 GB", | 20260506 19:05:11.661 | INFO | "huge" : "", | 20260506 19:05:11.661 | INFO | "huge_shared" : " 27 GB", | 20260506 19:05:11.661 | INFO | "malloc" : " 14 GB", | 20260506 19:05:11.661 | INFO | "pinned" : " 29 GB" | 20260506 19:05:11.661 | INFO | } | 20260506 19:05:11.662 | INFO | Perf report for the last 5.1437 seconds: | 20260506 19:05:11.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:05:11.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:11.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:11.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:11.662 | INFO | ANALYSIS : 62.6%, 2.9, 95.6ms, 0 | 20260506 19:05:11.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:11.662 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260506 19:05:11.662 | INFO | PRE_HQ : 60.3%, 1.9, 58.1ms, 0 | 20260506 19:05:11.662 | INFO | BAZWRITER : 70.7%, 4.8, 174.5ms, 0 | 20260506 19:05:11.662 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.2ms, 0 | 20260506 19:05:11.662 | INFO | Uploaded 36GiB to GPU in 2.00874s (17.9217GiB/s) | 20260506 19:05:11.662 | INFO | Downloaded 38GiB to GPU in 2.20754s (17.2137GiB/s) | 20260506 19:05:16.805 | INFO | Analyzing chunk frames = [3727872,3728384) | 20260506 19:05:16.805 | INFO | MemInfo: { | 20260506 19:05:16.805 | INFO | "gpu" : " 17 GB", | 20260506 19:05:16.805 | INFO | "huge" : "", | 20260506 19:05:16.805 | INFO | "huge_shared" : " 27 GB", | 20260506 19:05:16.805 | INFO | "malloc" : " 14 GB", | 20260506 19:05:16.806 | INFO | "pinned" : " 29 GB" | 20260506 19:05:16.806 | INFO | } | 20260506 19:05:16.806 | INFO | Perf report for the last 5.14411 seconds: | 20260506 19:05:16.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:05:16.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:16.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:16.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:16.806 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 19:05:16.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:16.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:16.806 | INFO | PRE_HQ : 26.9%, 1.0, 14.2ms, 0 | 20260506 19:05:16.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:16.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:16.806 | INFO | Uploaded 36GiB to GPU in 1.98889s (18.1005GiB/s) | 20260506 19:05:16.806 | INFO | Downloaded 34GiB to GPU in 1.92829s (17.6322GiB/s) | 20260506 19:05:21.944 | INFO | Analyzing chunk frames = [3728384,3728896) | 20260506 19:05:21.944 | INFO | MemInfo: { | 20260506 19:05:21.944 | INFO | "gpu" : " 17 GB", | 20260506 19:05:21.944 | INFO | "huge" : "", | 20260506 19:05:21.944 | INFO | "huge_shared" : " 27 GB", | 20260506 19:05:21.944 | INFO | "malloc" : " 14 GB", | 20260506 19:05:21.944 | INFO | "pinned" : " 29 GB" | 20260506 19:05:21.944 | INFO | } | 20260506 19:05:21.944 | INFO | Perf report for the last 5.13883 seconds: | 20260506 19:05:21.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:05:21.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:21.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:21.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:21.945 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 19:05:21.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:21.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:21.945 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:05:21.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:21.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:21.945 | INFO | Uploaded 36GiB to GPU in 1.96312s (18.3382GiB/s) | 20260506 19:05:21.945 | INFO | Downloaded 34GiB to GPU in 1.92674s (17.6464GiB/s) | 20260506 19:05:27.079 | INFO | Analyzing chunk frames = [3728896,3729408) | 20260506 19:05:27.079 | INFO | MemInfo: { | 20260506 19:05:27.079 | INFO | "gpu" : " 17 GB", | 20260506 19:05:27.079 | INFO | "huge" : "", | 20260506 19:05:27.079 | INFO | "huge_shared" : " 27 GB", | 20260506 19:05:27.079 | INFO | "malloc" : " 14 GB", | 20260506 19:05:27.079 | INFO | "pinned" : " 29 GB" | 20260506 19:05:27.079 | INFO | } | 20260506 19:05:27.079 | INFO | Perf report for the last 5.13486 seconds: | 20260506 19:05:27.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:05:27.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:27.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:27.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:27.079 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:05:27.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:27.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:27.079 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:05:27.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:27.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:27.080 | INFO | Uploaded 36GiB to GPU in 1.94322s (18.526GiB/s) | 20260506 19:05:27.080 | INFO | Downloaded 34GiB to GPU in 1.87526s (18.1308GiB/s) | 20260506 19:05:32.217 | INFO | Analyzing chunk frames = [3729408,3729920) | 20260506 19:05:32.217 | INFO | MemInfo: { | 20260506 19:05:32.217 | INFO | "gpu" : " 17 GB", | 20260506 19:05:32.217 | INFO | "huge" : "", | 20260506 19:05:32.217 | INFO | "huge_shared" : " 27 GB", | 20260506 19:05:32.217 | INFO | "malloc" : " 14 GB", | 20260506 19:05:32.217 | INFO | "pinned" : " 29 GB" | 20260506 19:05:32.217 | INFO | } | 20260506 19:05:32.217 | INFO | Perf report for the last 5.13755 seconds: | 20260506 19:05:32.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:05:32.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:32.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:32.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:32.217 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 19:05:32.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:32.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:32.217 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:05:32.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:32.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:32.217 | INFO | Uploaded 36GiB to GPU in 1.99918s (18.0074GiB/s) | 20260506 19:05:32.217 | INFO | Downloaded 34GiB to GPU in 1.91176s (17.7847GiB/s) | 20260506 19:05:37.354 | INFO | Analyzing chunk frames = [3729920,3730432) | 20260506 19:05:37.354 | INFO | MemInfo: { | 20260506 19:05:37.354 | INFO | "gpu" : " 17 GB", | 20260506 19:05:37.354 | INFO | "huge" : "", | 20260506 19:05:37.354 | INFO | "huge_shared" : " 27 GB", | 20260506 19:05:37.354 | INFO | "malloc" : " 14 GB", | 20260506 19:05:37.354 | INFO | "pinned" : " 29 GB" | 20260506 19:05:37.354 | INFO | } | 20260506 19:05:37.354 | INFO | Perf report for the last 5.13726 seconds: | 20260506 19:05:37.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:05:37.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:37.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:37.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:37.355 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 19:05:37.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:37.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:37.355 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 19:05:37.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:37.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:37.355 | INFO | Uploaded 36GiB to GPU in 1.99457s (18.049GiB/s) | 20260506 19:05:37.355 | INFO | Downloaded 34GiB to GPU in 1.96205s (17.3288GiB/s) | 20260506 19:05:42.495 | INFO | Analyzing chunk frames = [3730432,3730944) | 20260506 19:05:42.495 | INFO | MemInfo: { | 20260506 19:05:42.495 | INFO | "gpu" : " 17 GB", | 20260506 19:05:42.495 | INFO | "huge" : "", | 20260506 19:05:42.495 | INFO | "huge_shared" : " 27 GB", | 20260506 19:05:42.495 | INFO | "malloc" : " 14 GB", | 20260506 19:05:42.495 | INFO | "pinned" : " 29 GB" | 20260506 19:05:42.495 | INFO | } | 20260506 19:05:42.495 | INFO | Perf report for the last 5.14045 seconds: | 20260506 19:05:42.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:05:42.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:42.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:42.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:42.495 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 19:05:42.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:42.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:42.495 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 19:05:42.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:42.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:42.495 | INFO | Uploaded 36GiB to GPU in 1.99397s (18.0544GiB/s) | 20260506 19:05:42.495 | INFO | Downloaded 34GiB to GPU in 2.01442s (16.8783GiB/s) | 20260506 19:05:47.636 | INFO | Analyzing chunk frames = [3730944,3731456) | 20260506 19:05:47.636 | INFO | MemInfo: { | 20260506 19:05:47.636 | INFO | "gpu" : " 17 GB", | 20260506 19:05:47.636 | INFO | "huge" : "", | 20260506 19:05:47.636 | INFO | "huge_shared" : " 27 GB", | 20260506 19:05:47.636 | INFO | "malloc" : " 14 GB", | 20260506 19:05:47.636 | INFO | "pinned" : " 29 GB" | 20260506 19:05:47.636 | INFO | } | 20260506 19:05:47.636 | INFO | Perf report for the last 5.14143 seconds: | 20260506 19:05:47.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:05:47.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:47.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:47.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:47.636 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 19:05:47.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:47.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:47.636 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 19:05:47.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:47.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:47.637 | INFO | Uploaded 36GiB to GPU in 1.99426s (18.0518GiB/s) | 20260506 19:05:47.637 | INFO | Downloaded 34GiB to GPU in 1.98593s (17.1205GiB/s) | 20260506 19:05:47.760 | INFO | Starting a round of Baz IO | 20260506 19:05:50.889 | INFO | Populating RTMetrics report | 20260506 19:05:50.889 | INFO | RTMetrics report populated | 20260506 19:05:50.889 | INFO | Opening tmp RTMetrics json file | 20260506 19:05:50.908 | INFO | Serializing RTMetrics to json | 20260506 19:05:50.913 | INFO | Serialization done, swapping tmp file now... | 20260506 19:05:50.918 | INFO | RTMetrics done | 20260506 19:05:51.287 | INFO | Finished a round of Baz IO: Wrote 2.02 GiB in 3.527 seconds | 20260506 19:05:51.287 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.77% / 0.01% / 65.22% | 20260506 19:05:51.287 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.982312% / 7.94729e-06% / 5.16574e-05% | 20260506 19:05:51.287 | INFO | Baz IO overall: | 20260506 19:05:51.287 | INFO | BytesWritten : 1971949231487 | 20260506 19:05:51.287 | INFO | NumEvents : 953386949953 | 20260506 19:05:52.776 | INFO | Analyzing chunk frames = [3731456,3731968) | 20260506 19:05:52.776 | INFO | MemInfo: { | 20260506 19:05:52.776 | INFO | "gpu" : " 17 GB", | 20260506 19:05:52.776 | INFO | "huge" : "", | 20260506 19:05:52.776 | INFO | "huge_shared" : " 27 GB", | 20260506 19:05:52.776 | INFO | "malloc" : " 14 GB", | 20260506 19:05:52.776 | INFO | "pinned" : " 29 GB" | 20260506 19:05:52.776 | INFO | } | 20260506 19:05:52.776 | INFO | Perf report for the last 5.14039 seconds: | 20260506 19:05:52.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:05:52.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:52.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:52.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:52.777 | INFO | ANALYSIS : 62.7%, 2.9, 95.3ms, 0 | 20260506 19:05:52.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:52.777 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 19:05:52.777 | INFO | PRE_HQ : 60.0%, 1.9, 57.9ms, 0 | 20260506 19:05:52.777 | INFO | BAZWRITER : 68.6%, 4.8, 168.8ms, 0 | 20260506 19:05:52.777 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 19:05:52.777 | INFO | Uploaded 36GiB to GPU in 2.07029s (17.3889GiB/s) | 20260506 19:05:52.777 | INFO | Downloaded 38GiB to GPU in 2.20677s (17.2197GiB/s) | 20260506 19:05:57.918 | INFO | Analyzing chunk frames = [3731968,3732480) | 20260506 19:05:57.918 | INFO | MemInfo: { | 20260506 19:05:57.918 | INFO | "gpu" : " 17 GB", | 20260506 19:05:57.918 | INFO | "huge" : "", | 20260506 19:05:57.918 | INFO | "huge_shared" : " 27 GB", | 20260506 19:05:57.918 | INFO | "malloc" : " 14 GB", | 20260506 19:05:57.918 | INFO | "pinned" : " 29 GB" | 20260506 19:05:57.918 | INFO | } | 20260506 19:05:57.918 | INFO | Perf report for the last 5.14192 seconds: | 20260506 19:05:57.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:05:57.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:57.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:57.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:57.919 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 19:05:57.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:57.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:57.919 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:05:57.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:57.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:05:57.919 | INFO | Uploaded 36GiB to GPU in 2.01094s (17.902GiB/s) | 20260506 19:05:57.919 | INFO | Downloaded 34GiB to GPU in 1.93236s (17.5951GiB/s) | 20260506 19:06:03.058 | INFO | Analyzing chunk frames = [3732480,3732992) | 20260506 19:06:03.058 | INFO | MemInfo: { | 20260506 19:06:03.058 | INFO | "gpu" : " 17 GB", | 20260506 19:06:03.058 | INFO | "huge" : "", | 20260506 19:06:03.058 | INFO | "huge_shared" : " 27 GB", | 20260506 19:06:03.058 | INFO | "malloc" : " 14 GB", | 20260506 19:06:03.059 | INFO | "pinned" : " 29 GB" | 20260506 19:06:03.059 | INFO | } | 20260506 19:06:03.059 | INFO | Perf report for the last 5.14019 seconds: | 20260506 19:06:03.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:06:03.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:03.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:03.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:03.059 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 19:06:03.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:03.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:03.059 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:06:03.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:03.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:03.059 | INFO | Uploaded 36GiB to GPU in 1.992s (18.0723GiB/s) | 20260506 19:06:03.059 | INFO | Downloaded 34GiB to GPU in 1.92192s (17.6906GiB/s) | 20260506 19:06:08.194 | INFO | Analyzing chunk frames = [3732992,3733504) | 20260506 19:06:08.194 | INFO | MemInfo: { | 20260506 19:06:08.194 | INFO | "gpu" : " 17 GB", | 20260506 19:06:08.194 | INFO | "huge" : "", | 20260506 19:06:08.194 | INFO | "huge_shared" : " 27 GB", | 20260506 19:06:08.194 | INFO | "malloc" : " 14 GB", | 20260506 19:06:08.194 | INFO | "pinned" : " 29 GB" | 20260506 19:06:08.194 | INFO | } | 20260506 19:06:08.194 | INFO | Perf report for the last 5.13553 seconds: | 20260506 19:06:08.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:06:08.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:08.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:08.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:08.194 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 19:06:08.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:08.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:08.194 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:06:08.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:08.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:08.194 | INFO | Uploaded 36GiB to GPU in 1.93814s (18.5745GiB/s) | 20260506 19:06:08.194 | INFO | Downloaded 34GiB to GPU in 1.87158s (18.1665GiB/s) | 20260506 19:06:13.329 | INFO | Analyzing chunk frames = [3733504,3734016) | 20260506 19:06:13.329 | INFO | MemInfo: { | 20260506 19:06:13.329 | INFO | "gpu" : " 17 GB", | 20260506 19:06:13.329 | INFO | "huge" : "", | 20260506 19:06:13.329 | INFO | "huge_shared" : " 27 GB", | 20260506 19:06:13.329 | INFO | "malloc" : " 14 GB", | 20260506 19:06:13.329 | INFO | "pinned" : " 29 GB" | 20260506 19:06:13.329 | INFO | } | 20260506 19:06:13.329 | INFO | Perf report for the last 5.1351 seconds: | 20260506 19:06:13.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:06:13.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:13.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:13.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:13.329 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260506 19:06:13.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:13.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:13.330 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:06:13.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:13.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:13.330 | INFO | Uploaded 36GiB to GPU in 2.02062s (17.8163GiB/s) | 20260506 19:06:13.330 | INFO | Downloaded 34GiB to GPU in 1.91376s (17.7661GiB/s) | 20260506 19:06:18.471 | INFO | Analyzing chunk frames = [3734016,3734528) | 20260506 19:06:18.471 | INFO | MemInfo: { | 20260506 19:06:18.471 | INFO | "gpu" : " 17 GB", | 20260506 19:06:18.471 | INFO | "huge" : "", | 20260506 19:06:18.471 | INFO | "huge_shared" : " 27 GB", | 20260506 19:06:18.471 | INFO | "malloc" : " 14 GB", | 20260506 19:06:18.471 | INFO | "pinned" : " 29 GB" | 20260506 19:06:18.471 | INFO | } | 20260506 19:06:18.471 | INFO | Perf report for the last 5.14155 seconds: | 20260506 19:06:18.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:06:18.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:18.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:18.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:18.471 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:06:18.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:18.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:18.471 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 19:06:18.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:18.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:18.471 | INFO | Uploaded 36GiB to GPU in 2.04772s (17.5805GiB/s) | 20260506 19:06:18.471 | INFO | Downloaded 34GiB to GPU in 1.9514s (17.4233GiB/s) | 20260506 19:06:23.612 | INFO | Analyzing chunk frames = [3734528,3735040) | 20260506 19:06:23.612 | INFO | MemInfo: { | 20260506 19:06:23.612 | INFO | "gpu" : " 17 GB", | 20260506 19:06:23.612 | INFO | "huge" : "", | 20260506 19:06:23.612 | INFO | "huge_shared" : " 27 GB", | 20260506 19:06:23.612 | INFO | "malloc" : " 14 GB", | 20260506 19:06:23.612 | INFO | "pinned" : " 29 GB" | 20260506 19:06:23.612 | INFO | } | 20260506 19:06:23.612 | INFO | Perf report for the last 5.14122 seconds: | 20260506 19:06:23.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:06:23.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:23.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:23.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:23.612 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 19:06:23.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:23.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:23.612 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:06:23.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:23.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:23.613 | INFO | Uploaded 36GiB to GPU in 2.00799s (17.9284GiB/s) | 20260506 19:06:23.613 | INFO | Downloaded 34GiB to GPU in 2.01014s (16.9143GiB/s) | 20260506 19:06:28.752 | INFO | Analyzing chunk frames = [3735040,3735552) | 20260506 19:06:28.752 | INFO | MemInfo: { | 20260506 19:06:28.752 | INFO | "gpu" : " 17 GB", | 20260506 19:06:28.752 | INFO | "huge" : "", | 20260506 19:06:28.752 | INFO | "huge_shared" : " 27 GB", | 20260506 19:06:28.752 | INFO | "malloc" : " 14 GB", | 20260506 19:06:28.752 | INFO | "pinned" : " 29 GB" | 20260506 19:06:28.753 | INFO | } | 20260506 19:06:28.753 | INFO | Perf report for the last 5.14064 seconds: | 20260506 19:06:28.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:06:28.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:28.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:28.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:28.753 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 19:06:28.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:28.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:28.753 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 19:06:28.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:28.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:28.753 | INFO | Uploaded 36GiB to GPU in 1.9861s (18.1259GiB/s) | 20260506 19:06:28.753 | INFO | Downloaded 34GiB to GPU in 1.98909s (17.0933GiB/s) | 20260506 19:06:28.753 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:06:28.753 | INFO | Total mem usage: 18 GiB | 20260506 19:06:28.753 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:06:28.753 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -14.8%) | 20260506 19:06:28.753 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:06:28.753 | INFO | High Water Breakdown: | 20260506 19:06:28.753 | INFO | 61.5% : 9.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:06:28.753 | INFO | 38.4% : 6.02 GiB ( 89.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:06:28.753 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:06:28.753 | INFO | ------------------------------------------ | 20260506 19:06:28.753 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:06:28.753 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:06:28.753 | INFO | Total mem usage: 26.5 GiB | 20260506 19:06:28.753 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:06:28.753 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:06:28.753 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:06:28.753 | INFO | High Water Breakdown: | 20260506 19:06:28.753 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:06:28.753 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:06:28.753 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:06:28.753 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:06:28.753 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:06:28.753 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:06:28.753 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:06:28.753 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:06:28.753 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:06:28.753 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:06:28.753 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:06:28.754 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:06:28.754 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:06:28.754 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:06:28.754 | INFO | ------------------------------------------ | 20260506 19:06:28.754 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:06:28.754 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:06:28.754 | INFO | Total mem usage: 25.6 GiB | 20260506 19:06:28.754 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:06:28.754 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:06:28.754 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:06:28.754 | INFO | High Water Breakdown: | 20260506 19:06:28.754 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:06:28.754 | INFO | ------------------------------------------ | 20260506 19:06:28.754 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:06:28.754 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:06:28.754 | INFO | Total mem usage: 15.7 GiB | 20260506 19:06:28.754 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:06:28.754 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:06:28.754 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:06:28.754 | INFO | High Water Breakdown: | 20260506 19:06:28.754 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:06:28.754 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:06:28.754 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:06:28.754 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:06:28.754 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:06:28.754 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:06:28.754 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:06:28.754 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:06:28.754 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:06:28.754 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:06:28.754 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:06:28.754 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:06:28.754 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:06:28.754 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:06:28.754 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:06:28.754 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:06:28.754 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:06:28.754 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:06:28.754 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:06:28.754 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:06:28.754 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:06:28.754 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:06:28.754 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:06:28.754 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:06:28.754 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:06:28.754 | INFO | ------------------------------------------ | 20260506 19:06:28.754 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:06:28.755 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:06:28.755 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:06:28.755 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:06:28.755 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.61e+01, 1.35e+01 / 2.84e+00 | 20260506 19:06:28.755 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 1.00e-02 / 6.22e-03 | 20260506 19:06:28.755 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.40e+00, 5.59e-01 / 3.27e-01 | 20260506 19:06:28.755 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.44e+00, 1.83e-01 / 1.35e-01 | 20260506 19:06:28.755 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.47e+01, 4.03e+00 / 2.75e+00 | 20260506 19:06:28.755 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.41e+00 / 1.75e+00 | 20260506 19:06:28.755 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.25e+00 / 2.81e+00, 3.81e-01 / 2.14e-01 | 20260506 19:06:28.755 | INFO | PerfReport: 3.39%, Metrics -- 1.06e+00 / 1.41e+00, 3.07e-01 / 1.83e-01 | 20260506 19:06:28.755 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.63e-01 / 4.10e-01, 5.64e-02 / 1.93e-02 | 20260506 19:06:28.755 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:06:28.755 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:06:28.755 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:06:28.755 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 19:06:28.755 | INFO | PerfReport: 21.45%, Upload -- 2.08e+01 / 0.00e+00, 6.94e+00 / 0.00e+00 | 20260506 19:06:28.755 | INFO | PerfReport: 21.21%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 19:06:28.755 | INFO | PerfReport: 32.23%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 19:06:28.755 | INFO | PerfReport: 1.32%, LockUpload -- 1.28e+00 / 0.00e+00, 5.31e+00 / 0.00e+00 | 20260506 19:06:28.755 | INFO | PerfReport: 1.17%, LockDownload -- 1.14e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 19:06:28.755 | INFO | PerfReport: 22.61%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 19:06:28.755 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260506 19:06:28.873 | INFO | Starting a round of Baz IO | 20260506 19:06:32.001 | INFO | Populating RTMetrics report | 20260506 19:06:32.001 | INFO | RTMetrics report populated | 20260506 19:06:32.001 | INFO | Opening tmp RTMetrics json file | 20260506 19:06:32.013 | INFO | Serializing RTMetrics to json | 20260506 19:06:32.017 | INFO | Serialization done, swapping tmp file now... | 20260506 19:06:32.022 | INFO | RTMetrics done | 20260506 19:06:32.508 | INFO | Finished a round of Baz IO: Wrote 2.022 GiB in 3.635 seconds | 20260506 19:06:32.508 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.77% / 0.01% / 65.23% | 20260506 19:06:32.508 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.984395% / 7.94729e-06% / 8.74201e-05% | 20260506 19:06:32.508 | INFO | Baz IO overall: | 20260506 19:06:32.509 | INFO | BytesWritten : 1974120695228 | 20260506 19:06:32.509 | INFO | NumEvents : 954432667998 | 20260506 19:06:33.894 | INFO | Analyzing chunk frames = [3735552,3736064) | 20260506 19:06:33.894 | INFO | MemInfo: { | 20260506 19:06:33.894 | INFO | "gpu" : " 17 GB", | 20260506 19:06:33.894 | INFO | "huge" : "", | 20260506 19:06:33.894 | INFO | "huge_shared" : " 27 GB", | 20260506 19:06:33.894 | INFO | "malloc" : " 14 GB", | 20260506 19:06:33.894 | INFO | "pinned" : " 29 GB" | 20260506 19:06:33.894 | INFO | } | 20260506 19:06:33.894 | INFO | Perf report for the last 5.14171 seconds: | 20260506 19:06:33.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:06:33.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:33.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:33.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:33.895 | INFO | ANALYSIS : 62.6%, 2.9, 95.3ms, 0 | 20260506 19:06:33.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:33.895 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 19:06:33.895 | INFO | PRE_HQ : 60.1%, 1.9, 57.9ms, 0 | 20260506 19:06:33.895 | INFO | BAZWRITER : 70.4%, 4.8, 175.1ms, 0 | 20260506 19:06:33.895 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 19:06:33.895 | INFO | Uploaded 36GiB to GPU in 2.03978s (17.6489GiB/s) | 20260506 19:06:33.895 | INFO | Downloaded 38GiB to GPU in 2.21583s (17.1494GiB/s) | 20260506 19:06:39.033 | INFO | Analyzing chunk frames = [3736064,3736576) | 20260506 19:06:39.033 | INFO | MemInfo: { | 20260506 19:06:39.033 | INFO | "gpu" : " 17 GB", | 20260506 19:06:39.033 | INFO | "huge" : "", | 20260506 19:06:39.033 | INFO | "huge_shared" : " 27 GB", | 20260506 19:06:39.033 | INFO | "malloc" : " 14 GB", | 20260506 19:06:39.033 | INFO | "pinned" : " 29 GB" | 20260506 19:06:39.034 | INFO | } | 20260506 19:06:39.034 | INFO | Perf report for the last 5.1392 seconds: | 20260506 19:06:39.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:06:39.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:39.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:39.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:39.034 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:06:39.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:39.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:39.034 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:06:39.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:39.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:39.034 | INFO | Uploaded 36GiB to GPU in 2.01176s (17.8948GiB/s) | 20260506 19:06:39.034 | INFO | Downloaded 34GiB to GPU in 1.92892s (17.6264GiB/s) | 20260506 19:06:44.172 | INFO | Analyzing chunk frames = [3736576,3737088) | 20260506 19:06:44.172 | INFO | MemInfo: { | 20260506 19:06:44.172 | INFO | "gpu" : " 17 GB", | 20260506 19:06:44.172 | INFO | "huge" : "", | 20260506 19:06:44.172 | INFO | "huge_shared" : " 27 GB", | 20260506 19:06:44.172 | INFO | "malloc" : " 14 GB", | 20260506 19:06:44.172 | INFO | "pinned" : " 29 GB" | 20260506 19:06:44.172 | INFO | } | 20260506 19:06:44.172 | INFO | Perf report for the last 5.13827 seconds: | 20260506 19:06:44.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:06:44.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:44.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:44.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:44.172 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 19:06:44.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:44.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:44.172 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:06:44.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:44.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:44.172 | INFO | Uploaded 36GiB to GPU in 1.96573s (18.3138GiB/s) | 20260506 19:06:44.172 | INFO | Downloaded 34GiB to GPU in 1.9213s (17.6963GiB/s) | 20260506 19:06:49.310 | INFO | Analyzing chunk frames = [3737088,3737600) | 20260506 19:06:49.310 | INFO | MemInfo: { | 20260506 19:06:49.310 | INFO | "gpu" : " 17 GB", | 20260506 19:06:49.310 | INFO | "huge" : "", | 20260506 19:06:49.310 | INFO | "huge_shared" : " 27 GB", | 20260506 19:06:49.310 | INFO | "malloc" : " 14 GB", | 20260506 19:06:49.310 | INFO | "pinned" : " 29 GB" | 20260506 19:06:49.310 | INFO | } | 20260506 19:06:49.310 | INFO | Perf report for the last 5.13839 seconds: | 20260506 19:06:49.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:06:49.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:49.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:49.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:49.310 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:06:49.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:49.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:49.310 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:06:49.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:49.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:49.311 | INFO | Uploaded 36GiB to GPU in 1.91022s (18.846GiB/s) | 20260506 19:06:49.311 | INFO | Downloaded 34GiB to GPU in 1.87203s (18.1621GiB/s) | 20260506 19:06:54.443 | INFO | Analyzing chunk frames = [3737600,3738112) | 20260506 19:06:54.443 | INFO | MemInfo: { | 20260506 19:06:54.443 | INFO | "gpu" : " 17 GB", | 20260506 19:06:54.443 | INFO | "huge" : "", | 20260506 19:06:54.443 | INFO | "huge_shared" : " 27 GB", | 20260506 19:06:54.443 | INFO | "malloc" : " 14 GB", | 20260506 19:06:54.443 | INFO | "pinned" : " 29 GB" | 20260506 19:06:54.443 | INFO | } | 20260506 19:06:54.443 | INFO | Perf report for the last 5.13294 seconds: | 20260506 19:06:54.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:06:54.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:54.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:54.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:54.443 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 19:06:54.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:54.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:54.443 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:06:54.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:54.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:54.444 | INFO | Uploaded 36GiB to GPU in 2.0172s (17.8465GiB/s) | 20260506 19:06:54.444 | INFO | Downloaded 34GiB to GPU in 1.9153s (17.7517GiB/s) | 20260506 19:06:59.586 | INFO | Analyzing chunk frames = [3738112,3738624) | 20260506 19:06:59.586 | INFO | MemInfo: { | 20260506 19:06:59.586 | INFO | "gpu" : " 17 GB", | 20260506 19:06:59.586 | INFO | "huge" : "", | 20260506 19:06:59.586 | INFO | "huge_shared" : " 27 GB", | 20260506 19:06:59.586 | INFO | "malloc" : " 14 GB", | 20260506 19:06:59.586 | INFO | "pinned" : " 29 GB" | 20260506 19:06:59.586 | INFO | } | 20260506 19:06:59.586 | INFO | Perf report for the last 5.14268 seconds: | 20260506 19:06:59.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:06:59.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:59.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:59.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:59.586 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:06:59.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:59.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:59.586 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 19:06:59.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:59.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:06:59.586 | INFO | Uploaded 36GiB to GPU in 2.02842s (17.7478GiB/s) | 20260506 19:06:59.586 | INFO | Downloaded 34GiB to GPU in 1.96393s (17.3122GiB/s) | 20260506 19:07:04.733 | INFO | Analyzing chunk frames = [3738624,3739136) | 20260506 19:07:04.733 | INFO | MemInfo: { | 20260506 19:07:04.733 | INFO | "gpu" : " 17 GB", | 20260506 19:07:04.733 | INFO | "huge" : "", | 20260506 19:07:04.733 | INFO | "huge_shared" : " 27 GB", | 20260506 19:07:04.733 | INFO | "malloc" : " 14 GB", | 20260506 19:07:04.733 | INFO | "pinned" : " 29 GB" | 20260506 19:07:04.733 | INFO | } | 20260506 19:07:04.733 | INFO | Perf report for the last 5.14716 seconds: | 20260506 19:07:04.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:07:04.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:04.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:04.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:04.733 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 19:07:04.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:04.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:04.733 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 19:07:04.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:04.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:04.734 | INFO | Uploaded 36GiB to GPU in 1.98565s (18.1301GiB/s) | 20260506 19:07:04.734 | INFO | Downloaded 34GiB to GPU in 2.01453s (16.8774GiB/s) | 20260506 19:07:09.868 | INFO | Analyzing chunk frames = [3739136,3739648) | 20260506 19:07:09.868 | INFO | MemInfo: { | 20260506 19:07:09.868 | INFO | "gpu" : " 17 GB", | 20260506 19:07:09.868 | INFO | "huge" : "", | 20260506 19:07:09.868 | INFO | "huge_shared" : " 27 GB", | 20260506 19:07:09.868 | INFO | "malloc" : " 14 GB", | 20260506 19:07:09.868 | INFO | "pinned" : " 29 GB" | 20260506 19:07:09.869 | INFO | } | 20260506 19:07:09.869 | INFO | Perf report for the last 5.13538 seconds: | 20260506 19:07:09.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:07:09.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:09.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:09.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:09.869 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:07:09.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:09.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:09.869 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 19:07:09.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:09.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:09.869 | INFO | Uploaded 36GiB to GPU in 1.98925s (18.0973GiB/s) | 20260506 19:07:09.869 | INFO | Downloaded 34GiB to GPU in 1.98887s (17.0951GiB/s) | 20260506 19:07:09.983 | INFO | Starting a round of Baz IO | 20260506 19:07:13.120 | INFO | Populating RTMetrics report | 20260506 19:07:13.120 | INFO | RTMetrics report populated | 20260506 19:07:13.120 | INFO | Opening tmp RTMetrics json file | 20260506 19:07:13.318 | INFO | Serializing RTMetrics to json | 20260506 19:07:13.352 | INFO | Serialization done, swapping tmp file now... | 20260506 19:07:13.353 | INFO | RTMetrics done | 20260506 19:07:13.744 | INFO | Finished a round of Baz IO: Wrote 2.02 GiB in 3.761 seconds | 20260506 19:07:13.744 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.76% / 0.01% / 65.23% | 20260506 19:07:13.744 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.97934% / 7.94729e-06% / 9.53674e-05% | 20260506 19:07:13.744 | INFO | Baz IO overall: | 20260506 19:07:13.744 | INFO | BytesWritten : 1976289634343 | 20260506 19:07:13.744 | INFO | NumEvents : 955476525379 | 20260506 19:07:15.005 | INFO | Analyzing chunk frames = [3739648,3740160) | 20260506 19:07:15.005 | INFO | MemInfo: { | 20260506 19:07:15.005 | INFO | "gpu" : " 17 GB", | 20260506 19:07:15.005 | INFO | "huge" : "", | 20260506 19:07:15.005 | INFO | "huge_shared" : " 27 GB", | 20260506 19:07:15.005 | INFO | "malloc" : " 14 GB", | 20260506 19:07:15.005 | INFO | "pinned" : " 29 GB" | 20260506 19:07:15.005 | INFO | } | 20260506 19:07:15.005 | INFO | Perf report for the last 5.13674 seconds: | 20260506 19:07:15.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:07:15.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:15.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:15.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:15.006 | INFO | ANALYSIS : 62.7%, 2.9, 95.6ms, 0 | 20260506 19:07:15.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:15.006 | INFO | RT_METRICS : 47.6%, 1.0, 25.0ms, 0 | 20260506 19:07:15.006 | INFO | PRE_HQ : 60.1%, 1.8, 57.5ms, 0 | 20260506 19:07:15.006 | INFO | BAZWRITER : 72.9%, 4.8, 179.4ms, 0 | 20260506 19:07:15.006 | INFO | BAZ_SAMPLING : 4.3%, 1.0, 2.2ms, 0 | 20260506 19:07:15.006 | INFO | Uploaded 36GiB to GPU in 2.07549s (17.3453GiB/s) | 20260506 19:07:15.006 | INFO | Downloaded 38GiB to GPU in 2.2214s (17.1063GiB/s) | 20260506 19:07:20.143 | INFO | Analyzing chunk frames = [3740160,3740672) | 20260506 19:07:20.143 | INFO | MemInfo: { | 20260506 19:07:20.143 | INFO | "gpu" : " 17 GB", | 20260506 19:07:20.143 | INFO | "huge" : "", | 20260506 19:07:20.143 | INFO | "huge_shared" : " 27 GB", | 20260506 19:07:20.143 | INFO | "malloc" : " 14 GB", | 20260506 19:07:20.143 | INFO | "pinned" : " 29 GB" | 20260506 19:07:20.143 | INFO | } | 20260506 19:07:20.143 | INFO | Perf report for the last 5.13751 seconds: | 20260506 19:07:20.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:07:20.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:20.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:20.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:20.143 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:07:20.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:20.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:20.143 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:07:20.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:20.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:20.143 | INFO | Uploaded 36GiB to GPU in 2.00518s (17.9535GiB/s) | 20260506 19:07:20.143 | INFO | Downloaded 34GiB to GPU in 1.93052s (17.6119GiB/s) | 20260506 19:07:25.279 | INFO | Analyzing chunk frames = [3740672,3741184) | 20260506 19:07:25.279 | INFO | MemInfo: { | 20260506 19:07:25.279 | INFO | "gpu" : " 17 GB", | 20260506 19:07:25.279 | INFO | "huge" : "", | 20260506 19:07:25.279 | INFO | "huge_shared" : " 27 GB", | 20260506 19:07:25.279 | INFO | "malloc" : " 14 GB", | 20260506 19:07:25.279 | INFO | "pinned" : " 29 GB" | 20260506 19:07:25.279 | INFO | } | 20260506 19:07:25.279 | INFO | Perf report for the last 5.13621 seconds: | 20260506 19:07:25.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:07:25.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:25.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:25.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:25.279 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 19:07:25.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:25.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:25.279 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:07:25.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:25.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:25.279 | INFO | Uploaded 36GiB to GPU in 1.97783s (18.2017GiB/s) | 20260506 19:07:25.280 | INFO | Downloaded 34GiB to GPU in 1.91658s (17.7399GiB/s) | 20260506 19:07:30.421 | INFO | Analyzing chunk frames = [3741184,3741696) | 20260506 19:07:30.421 | INFO | MemInfo: { | 20260506 19:07:30.421 | INFO | "gpu" : " 17 GB", | 20260506 19:07:30.421 | INFO | "huge" : "", | 20260506 19:07:30.421 | INFO | "huge_shared" : " 27 GB", | 20260506 19:07:30.421 | INFO | "malloc" : " 14 GB", | 20260506 19:07:30.421 | INFO | "pinned" : " 29 GB" | 20260506 19:07:30.421 | INFO | } | 20260506 19:07:30.421 | INFO | Perf report for the last 5.14226 seconds: | 20260506 19:07:30.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:07:30.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:30.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:30.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:30.421 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 19:07:30.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:30.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:30.421 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:07:30.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:30.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:30.422 | INFO | Uploaded 36GiB to GPU in 1.93057s (18.6473GiB/s) | 20260506 19:07:30.422 | INFO | Downloaded 34GiB to GPU in 1.86973s (18.1844GiB/s) | 20260506 19:07:35.562 | INFO | Analyzing chunk frames = [3741696,3742208) | 20260506 19:07:35.562 | INFO | MemInfo: { | 20260506 19:07:35.562 | INFO | "gpu" : " 17 GB", | 20260506 19:07:35.562 | INFO | "huge" : "", | 20260506 19:07:35.562 | INFO | "huge_shared" : " 27 GB", | 20260506 19:07:35.562 | INFO | "malloc" : " 14 GB", | 20260506 19:07:35.562 | INFO | "pinned" : " 29 GB" | 20260506 19:07:35.562 | INFO | } | 20260506 19:07:35.562 | INFO | Perf report for the last 5.141 seconds: | 20260506 19:07:35.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:07:35.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:35.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:35.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:35.562 | INFO | ANALYSIS : 59.8%, 3.0, 93.2ms, 0 | 20260506 19:07:35.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:35.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:35.562 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:07:35.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:35.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:35.563 | INFO | Uploaded 36GiB to GPU in 1.94843s (18.4764GiB/s) | 20260506 19:07:35.563 | INFO | Downloaded 34GiB to GPU in 1.87891s (18.0956GiB/s) | 20260506 19:07:40.703 | INFO | Analyzing chunk frames = [3742208,3742720) | 20260506 19:07:40.703 | INFO | MemInfo: { | 20260506 19:07:40.703 | INFO | "gpu" : " 17 GB", | 20260506 19:07:40.703 | INFO | "huge" : "", | 20260506 19:07:40.703 | INFO | "huge_shared" : " 27 GB", | 20260506 19:07:40.703 | INFO | "malloc" : " 14 GB", | 20260506 19:07:40.703 | INFO | "pinned" : " 29 GB" | 20260506 19:07:40.703 | INFO | } | 20260506 19:07:40.703 | INFO | Perf report for the last 5.14057 seconds: | 20260506 19:07:40.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:07:40.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:40.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:40.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:40.703 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 19:07:40.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:40.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:40.703 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:07:40.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:40.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:40.703 | INFO | Uploaded 36GiB to GPU in 2.003s (17.973GiB/s) | 20260506 19:07:40.703 | INFO | Downloaded 34GiB to GPU in 1.95768s (17.3675GiB/s) | 20260506 19:07:45.846 | INFO | Analyzing chunk frames = [3742720,3743232) | 20260506 19:07:45.846 | INFO | MemInfo: { | 20260506 19:07:45.846 | INFO | "gpu" : " 17 GB", | 20260506 19:07:45.846 | INFO | "huge" : "", | 20260506 19:07:45.846 | INFO | "huge_shared" : " 27 GB", | 20260506 19:07:45.846 | INFO | "malloc" : " 14 GB", | 20260506 19:07:45.846 | INFO | "pinned" : " 29 GB" | 20260506 19:07:45.846 | INFO | } | 20260506 19:07:45.846 | INFO | Perf report for the last 5.14292 seconds: | 20260506 19:07:45.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:07:45.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:45.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:45.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:45.846 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 19:07:45.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:45.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:45.846 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:07:45.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:45.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:45.846 | INFO | Uploaded 36GiB to GPU in 1.97867s (18.194GiB/s) | 20260506 19:07:45.846 | INFO | Downloaded 34GiB to GPU in 2.00805s (16.9319GiB/s) | 20260506 19:07:50.991 | INFO | Analyzing chunk frames = [3743232,3743744) | 20260506 19:07:50.991 | INFO | MemInfo: { | 20260506 19:07:50.991 | INFO | "gpu" : " 17 GB", | 20260506 19:07:50.991 | INFO | "huge" : "", | 20260506 19:07:50.991 | INFO | "huge_shared" : " 27 GB", | 20260506 19:07:50.991 | INFO | "malloc" : " 14 GB", | 20260506 19:07:50.991 | INFO | "pinned" : " 29 GB" | 20260506 19:07:50.991 | INFO | } | 20260506 19:07:50.991 | INFO | Perf report for the last 5.14517 seconds: | 20260506 19:07:50.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:07:50.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:50.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:50.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:50.991 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 19:07:50.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:50.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:50.991 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 19:07:50.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:50.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:50.991 | INFO | Uploaded 36GiB to GPU in 1.96579s (18.3132GiB/s) | 20260506 19:07:50.991 | INFO | Downloaded 34GiB to GPU in 1.98445s (17.1332GiB/s) | 20260506 19:07:50.991 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:07:50.992 | INFO | Total mem usage: 18 GiB | 20260506 19:07:50.992 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:07:50.992 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -10.9%) | 20260506 19:07:50.992 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:07:50.992 | INFO | High Water Breakdown: | 20260506 19:07:50.992 | INFO | 62% : 10.1 GiB ( 88.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:07:50.992 | INFO | 37.8% : 6.15 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:07:50.992 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:07:50.992 | INFO | ------------------------------------------ | 20260506 19:07:50.992 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:07:50.992 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:07:50.992 | INFO | Total mem usage: 26.5 GiB | 20260506 19:07:50.992 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:07:50.992 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:07:50.992 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:07:50.992 | INFO | High Water Breakdown: | 20260506 19:07:50.992 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:07:50.992 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:07:50.992 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:07:50.992 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:07:50.992 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:07:50.992 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:07:50.992 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:07:50.992 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:07:50.992 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:07:50.992 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:07:50.992 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:07:50.992 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:07:50.992 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:07:50.992 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:07:50.992 | INFO | ------------------------------------------ | 20260506 19:07:50.992 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:07:50.992 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:07:50.992 | INFO | Total mem usage: 25.6 GiB | 20260506 19:07:50.992 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:07:50.992 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:07:50.992 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:07:50.992 | INFO | High Water Breakdown: | 20260506 19:07:50.992 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:07:50.992 | INFO | ------------------------------------------ | 20260506 19:07:50.992 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:07:50.992 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:07:50.993 | INFO | Total mem usage: 15.7 GiB | 20260506 19:07:50.993 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:07:50.993 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:07:50.993 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:07:50.993 | INFO | High Water Breakdown: | 20260506 19:07:50.993 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:07:50.993 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:07:50.993 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:07:50.993 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:07:50.993 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:07:50.993 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:07:50.993 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:07:50.993 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:07:50.993 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:07:50.993 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:07:50.993 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:07:50.993 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:07:50.993 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:07:50.993 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:07:50.993 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:07:50.993 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:07:50.993 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:07:50.993 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:07:50.993 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:07:50.993 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:07:50.993 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:07:50.993 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:07:50.993 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:07:50.993 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:07:50.993 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:07:50.993 | INFO | ------------------------------------------ | 20260506 19:07:50.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:07:50.993 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:07:50.993 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:07:50.993 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:07:50.993 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.61e+01, 1.35e+01 / 2.89e+00 | 20260506 19:07:50.993 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 9.66e-03 / 6.15e-03 | 20260506 19:07:50.993 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.40e+00, 5.54e-01 / 3.58e-01 | 20260506 19:07:50.993 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.44e+00, 1.81e-01 / 1.41e-01 | 20260506 19:07:50.993 | INFO | PerfReport: 16.02%, DME -- 4.00e+01 / 4.52e+01, 4.25e+00 / 2.98e+00 | 20260506 19:07:50.993 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.40e+00 / 1.88e+00 | 20260506 19:07:50.993 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.25e+00 / 2.83e+00, 3.82e-01 / 2.23e-01 | 20260506 19:07:50.993 | INFO | PerfReport: 3.37%, Metrics -- 1.05e+00 / 1.42e+00, 2.80e-01 / 1.72e-01 | 20260506 19:07:50.993 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.63e-01 / 4.11e-01, 5.61e-02 / 1.57e-02 | 20260506 19:07:50.993 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:07:50.993 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:07:50.993 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:07:50.993 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 19:07:50.993 | INFO | PerfReport: 21.39%, Upload -- 2.07e+01 / 0.00e+00, 6.85e+00 / 0.00e+00 | 20260506 19:07:50.993 | INFO | PerfReport: 21.22%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 19:07:50.993 | INFO | PerfReport: 32.22%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 19:07:50.993 | INFO | PerfReport: 1.27%, LockUpload -- 1.23e+00 / 0.00e+00, 5.08e+00 / 0.00e+00 | 20260506 19:07:50.993 | INFO | PerfReport: 1.23%, LockDownload -- 1.20e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260506 19:07:50.993 | INFO | PerfReport: 22.65%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 19:07:50.993 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260506 19:07:51.091 | INFO | Starting a round of Baz IO | 20260506 19:07:54.232 | INFO | Populating RTMetrics report | 20260506 19:07:54.233 | INFO | RTMetrics report populated | 20260506 19:07:54.233 | INFO | Opening tmp RTMetrics json file | 20260506 19:07:54.237 | INFO | Serializing RTMetrics to json | 20260506 19:07:54.264 | INFO | Serialization done, swapping tmp file now... | 20260506 19:07:54.281 | INFO | RTMetrics done | 20260506 19:07:54.384 | INFO | Finished a round of Baz IO: Wrote 2.018 GiB in 3.293 seconds | 20260506 19:07:54.385 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.76% / 0.01% / 65.24% | 20260506 19:07:54.385 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.977166% / 3.97364e-06% / 5.96046e-05% | 20260506 19:07:54.385 | INFO | Baz IO overall: | 20260506 19:07:54.385 | INFO | BytesWritten : 1978456979427 | 20260506 19:07:54.385 | INFO | NumEvents : 956519265967 | 20260506 19:07:56.121 | INFO | Analyzing chunk frames = [3743744,3744256) | 20260506 19:07:56.121 | INFO | MemInfo: { | 20260506 19:07:56.121 | INFO | "gpu" : " 17 GB", | 20260506 19:07:56.121 | INFO | "huge" : "", | 20260506 19:07:56.121 | INFO | "huge_shared" : " 27 GB", | 20260506 19:07:56.121 | INFO | "malloc" : " 14 GB", | 20260506 19:07:56.121 | INFO | "pinned" : " 29 GB" | 20260506 19:07:56.121 | INFO | } | 20260506 19:07:56.121 | INFO | Perf report for the last 5.13029 seconds: | 20260506 19:07:56.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:07:56.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:56.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:56.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:56.122 | INFO | ANALYSIS : 62.6%, 2.9, 95.1ms, 0 | 20260506 19:07:56.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:07:56.122 | INFO | RT_METRICS : 44.2%, 1.0, 23.1ms, 0 | 20260506 19:07:56.122 | INFO | PRE_HQ : 60.2%, 1.9, 58.0ms, 0 | 20260506 19:07:56.122 | INFO | BAZWRITER : 63.8%, 4.6, 149.2ms, 0 | 20260506 19:07:56.122 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 19:07:56.122 | INFO | Uploaded 36GiB to GPU in 2.03034s (17.731GiB/s) | 20260506 19:07:56.122 | INFO | Downloaded 38GiB to GPU in 2.19573s (17.3064GiB/s) | 20260506 19:08:01.257 | INFO | Analyzing chunk frames = [3744256,3744768) | 20260506 19:08:01.257 | INFO | MemInfo: { | 20260506 19:08:01.257 | INFO | "gpu" : " 17 GB", | 20260506 19:08:01.257 | INFO | "huge" : "", | 20260506 19:08:01.257 | INFO | "huge_shared" : " 27 GB", | 20260506 19:08:01.257 | INFO | "malloc" : " 14 GB", | 20260506 19:08:01.257 | INFO | "pinned" : " 29 GB" | 20260506 19:08:01.257 | INFO | } | 20260506 19:08:01.257 | INFO | Perf report for the last 5.13561 seconds: | 20260506 19:08:01.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:08:01.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:01.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:01.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:01.257 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:08:01.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:01.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:01.257 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:08:01.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:01.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:01.257 | INFO | Uploaded 36GiB to GPU in 1.99367s (18.0571GiB/s) | 20260506 19:08:01.257 | INFO | Downloaded 34GiB to GPU in 1.92158s (17.6937GiB/s) | 20260506 19:08:06.397 | INFO | Analyzing chunk frames = [3744768,3745280) | 20260506 19:08:06.397 | INFO | MemInfo: { | 20260506 19:08:06.397 | INFO | "gpu" : " 17 GB", | 20260506 19:08:06.397 | INFO | "huge" : "", | 20260506 19:08:06.397 | INFO | "huge_shared" : " 27 GB", | 20260506 19:08:06.397 | INFO | "malloc" : " 14 GB", | 20260506 19:08:06.397 | INFO | "pinned" : " 29 GB" | 20260506 19:08:06.397 | INFO | } | 20260506 19:08:06.397 | INFO | Perf report for the last 5.14023 seconds: | 20260506 19:08:06.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:08:06.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:06.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:06.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:06.397 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260506 19:08:06.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:06.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:06.397 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:08:06.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:06.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:06.398 | INFO | Uploaded 36GiB to GPU in 1.93952s (18.5613GiB/s) | 20260506 19:08:06.398 | INFO | Downloaded 34GiB to GPU in 1.87644s (18.1194GiB/s) | 20260506 19:08:11.537 | INFO | Analyzing chunk frames = [3745280,3745792) | 20260506 19:08:11.537 | INFO | MemInfo: { | 20260506 19:08:11.537 | INFO | "gpu" : " 17 GB", | 20260506 19:08:11.537 | INFO | "huge" : "", | 20260506 19:08:11.537 | INFO | "huge_shared" : " 27 GB", | 20260506 19:08:11.537 | INFO | "malloc" : " 14 GB", | 20260506 19:08:11.537 | INFO | "pinned" : " 29 GB" | 20260506 19:08:11.537 | INFO | } | 20260506 19:08:11.537 | INFO | Perf report for the last 5.13995 seconds: | 20260506 19:08:11.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:08:11.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:11.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:11.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:11.537 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:08:11.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:11.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:11.537 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:08:11.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:11.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:11.538 | INFO | Uploaded 36GiB to GPU in 1.92662s (18.6856GiB/s) | 20260506 19:08:11.538 | INFO | Downloaded 34GiB to GPU in 1.87131s (18.1691GiB/s) | 20260506 19:08:16.682 | INFO | Analyzing chunk frames = [3745792,3746304) | 20260506 19:08:16.682 | INFO | MemInfo: { | 20260506 19:08:16.682 | INFO | "gpu" : " 17 GB", | 20260506 19:08:16.682 | INFO | "huge" : "", | 20260506 19:08:16.682 | INFO | "huge_shared" : " 27 GB", | 20260506 19:08:16.682 | INFO | "malloc" : " 14 GB", | 20260506 19:08:16.682 | INFO | "pinned" : " 29 GB" | 20260506 19:08:16.682 | INFO | } | 20260506 19:08:16.682 | INFO | Perf report for the last 5.14495 seconds: | 20260506 19:08:16.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:08:16.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:16.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:16.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:16.682 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 19:08:16.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:16.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:16.682 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:08:16.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:16.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:16.682 | INFO | Uploaded 36GiB to GPU in 2.0149s (17.8669GiB/s) | 20260506 19:08:16.682 | INFO | Downloaded 34GiB to GPU in 1.9156s (17.749GiB/s) | 20260506 19:08:21.821 | INFO | Analyzing chunk frames = [3746304,3746816) | 20260506 19:08:21.821 | INFO | MemInfo: { | 20260506 19:08:21.821 | INFO | "gpu" : " 17 GB", | 20260506 19:08:21.821 | INFO | "huge" : "", | 20260506 19:08:21.821 | INFO | "huge_shared" : " 27 GB", | 20260506 19:08:21.821 | INFO | "malloc" : " 14 GB", | 20260506 19:08:21.821 | INFO | "pinned" : " 29 GB" | 20260506 19:08:21.821 | INFO | } | 20260506 19:08:21.821 | INFO | Perf report for the last 5.13864 seconds: | 20260506 19:08:21.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:08:21.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:21.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:21.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:21.821 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 19:08:21.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:21.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:21.821 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 19:08:21.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:21.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:21.821 | INFO | Uploaded 36GiB to GPU in 2.01773s (17.8419GiB/s) | 20260506 19:08:21.821 | INFO | Downloaded 34GiB to GPU in 1.96357s (17.3154GiB/s) | 20260506 19:08:26.958 | INFO | Analyzing chunk frames = [3746816,3747328) | 20260506 19:08:26.958 | INFO | MemInfo: { | 20260506 19:08:26.958 | INFO | "gpu" : " 17 GB", | 20260506 19:08:26.958 | INFO | "huge" : "", | 20260506 19:08:26.958 | INFO | "huge_shared" : " 27 GB", | 20260506 19:08:26.958 | INFO | "malloc" : " 14 GB", | 20260506 19:08:26.958 | INFO | "pinned" : " 29 GB" | 20260506 19:08:26.958 | INFO | } | 20260506 19:08:26.958 | INFO | Perf report for the last 5.13713 seconds: | 20260506 19:08:26.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:08:26.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:26.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:26.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:26.958 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:08:26.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:26.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:26.958 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:08:26.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:26.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:26.958 | INFO | Uploaded 36GiB to GPU in 1.98167s (18.1665GiB/s) | 20260506 19:08:26.958 | INFO | Downloaded 34GiB to GPU in 1.99812s (17.016GiB/s) | 20260506 19:08:32.095 | INFO | Analyzing chunk frames = [3747328,3747840) | 20260506 19:08:32.095 | INFO | MemInfo: { | 20260506 19:08:32.095 | INFO | "gpu" : " 17 GB", | 20260506 19:08:32.095 | INFO | "huge" : "", | 20260506 19:08:32.095 | INFO | "huge_shared" : " 27 GB", | 20260506 19:08:32.095 | INFO | "malloc" : " 14 GB", | 20260506 19:08:32.095 | INFO | "pinned" : " 29 GB" | 20260506 19:08:32.095 | INFO | } | 20260506 19:08:32.095 | INFO | Perf report for the last 5.13735 seconds: | 20260506 19:08:32.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:08:32.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:32.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:32.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:32.095 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 19:08:32.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:32.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:32.095 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260506 19:08:32.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:32.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:32.095 | INFO | Uploaded 36GiB to GPU in 2.02899s (17.7428GiB/s) | 20260506 19:08:32.096 | INFO | Downloaded 34GiB to GPU in 1.99309s (17.0589GiB/s) | 20260506 19:08:32.210 | INFO | Starting a round of Baz IO | 20260506 19:08:35.337 | INFO | Populating RTMetrics report | 20260506 19:08:35.337 | INFO | RTMetrics report populated | 20260506 19:08:35.337 | INFO | Opening tmp RTMetrics json file | 20260506 19:08:35.340 | INFO | Serializing RTMetrics to json | 20260506 19:08:35.342 | INFO | Serialization done, swapping tmp file now... | 20260506 19:08:35.346 | INFO | RTMetrics done | 20260506 19:08:35.595 | INFO | Finished a round of Baz IO: Wrote 2.018 GiB in 3.384 seconds | 20260506 19:08:35.595 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.75% / 0.01% / 65.24% | 20260506 19:08:35.595 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.974786% / 1.19209e-05% / 7.54992e-05% | 20260506 19:08:35.595 | INFO | Baz IO overall: | 20260506 19:08:35.595 | INFO | BytesWritten : 1980623956991 | 20260506 19:08:35.595 | INFO | NumEvents : 957562036793 | 20260506 19:08:37.234 | INFO | Analyzing chunk frames = [3747840,3748352) | 20260506 19:08:37.234 | INFO | MemInfo: { | 20260506 19:08:37.234 | INFO | "gpu" : " 17 GB", | 20260506 19:08:37.234 | INFO | "huge" : "", | 20260506 19:08:37.234 | INFO | "huge_shared" : " 27 GB", | 20260506 19:08:37.234 | INFO | "malloc" : " 14 GB", | 20260506 19:08:37.234 | INFO | "pinned" : " 29 GB" | 20260506 19:08:37.234 | INFO | } | 20260506 19:08:37.235 | INFO | Perf report for the last 5.13923 seconds: | 20260506 19:08:37.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:08:37.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:37.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:37.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:37.235 | INFO | ANALYSIS : 62.5%, 2.9, 94.9ms, 0 | 20260506 19:08:37.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:37.235 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260506 19:08:37.235 | INFO | PRE_HQ : 59.9%, 1.9, 58.0ms, 0 | 20260506 19:08:37.235 | INFO | BAZWRITER : 65.5%, 4.8, 161.4ms, 0 | 20260506 19:08:37.235 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 19:08:37.235 | INFO | Uploaded 36GiB to GPU in 2.04105s (17.638GiB/s) | 20260506 19:08:37.235 | INFO | Downloaded 38GiB to GPU in 2.20635s (17.223GiB/s) | 20260506 19:08:42.372 | INFO | Analyzing chunk frames = [3748352,3748864) | 20260506 19:08:42.372 | INFO | MemInfo: { | 20260506 19:08:42.372 | INFO | "gpu" : " 17 GB", | 20260506 19:08:42.372 | INFO | "huge" : "", | 20260506 19:08:42.372 | INFO | "huge_shared" : " 27 GB", | 20260506 19:08:42.372 | INFO | "malloc" : " 14 GB", | 20260506 19:08:42.372 | INFO | "pinned" : " 29 GB" | 20260506 19:08:42.372 | INFO | } | 20260506 19:08:42.372 | INFO | Perf report for the last 5.13785 seconds: | 20260506 19:08:42.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:08:42.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:42.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:42.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:42.372 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:08:42.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:42.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:42.373 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:08:42.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:42.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:42.373 | INFO | Uploaded 36GiB to GPU in 2.00448s (17.9597GiB/s) | 20260506 19:08:42.373 | INFO | Downloaded 34GiB to GPU in 1.93661s (17.5564GiB/s) | 20260506 19:08:47.508 | INFO | Analyzing chunk frames = [3748864,3749376) | 20260506 19:08:47.508 | INFO | MemInfo: { | 20260506 19:08:47.508 | INFO | "gpu" : " 17 GB", | 20260506 19:08:47.508 | INFO | "huge" : "", | 20260506 19:08:47.508 | INFO | "huge_shared" : " 27 GB", | 20260506 19:08:47.508 | INFO | "malloc" : " 14 GB", | 20260506 19:08:47.508 | INFO | "pinned" : " 29 GB" | 20260506 19:08:47.508 | INFO | } | 20260506 19:08:47.508 | INFO | Perf report for the last 5.13599 seconds: | 20260506 19:08:47.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:08:47.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:47.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:47.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:47.508 | INFO | ANALYSIS : 61.0%, 3.0, 94.3ms, 0 | 20260506 19:08:47.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:47.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:47.509 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:08:47.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:47.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:47.509 | INFO | Uploaded 36GiB to GPU in 1.9822s (18.1617GiB/s) | 20260506 19:08:47.509 | INFO | Downloaded 34GiB to GPU in 1.91987s (17.7096GiB/s) | 20260506 19:08:52.654 | INFO | Analyzing chunk frames = [3749376,3749888) | 20260506 19:08:52.654 | INFO | MemInfo: { | 20260506 19:08:52.654 | INFO | "gpu" : " 17 GB", | 20260506 19:08:52.654 | INFO | "huge" : "", | 20260506 19:08:52.654 | INFO | "huge_shared" : " 27 GB", | 20260506 19:08:52.654 | INFO | "malloc" : " 14 GB", | 20260506 19:08:52.654 | INFO | "pinned" : " 29 GB" | 20260506 19:08:52.654 | INFO | } | 20260506 19:08:52.654 | INFO | Perf report for the last 5.14569 seconds: | 20260506 19:08:52.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:08:52.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:52.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:52.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:52.654 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 19:08:52.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:52.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:52.654 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:08:52.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:52.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:52.654 | INFO | Uploaded 36GiB to GPU in 1.93221s (18.6315GiB/s) | 20260506 19:08:52.654 | INFO | Downloaded 34GiB to GPU in 1.87038s (18.1781GiB/s) | 20260506 19:08:57.800 | INFO | Analyzing chunk frames = [3749888,3750400) | 20260506 19:08:57.800 | INFO | MemInfo: { | 20260506 19:08:57.800 | INFO | "gpu" : " 17 GB", | 20260506 19:08:57.800 | INFO | "huge" : "", | 20260506 19:08:57.800 | INFO | "huge_shared" : " 27 GB", | 20260506 19:08:57.800 | INFO | "malloc" : " 14 GB", | 20260506 19:08:57.800 | INFO | "pinned" : " 29 GB" | 20260506 19:08:57.800 | INFO | } | 20260506 19:08:57.800 | INFO | Perf report for the last 5.14582 seconds: | 20260506 19:08:57.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:08:57.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:57.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:57.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:57.800 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 19:08:57.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:57.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:57.800 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:08:57.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:57.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:08:57.800 | INFO | Uploaded 36GiB to GPU in 2.01577s (17.8592GiB/s) | 20260506 19:08:57.800 | INFO | Downloaded 34GiB to GPU in 1.90474s (17.8502GiB/s) | 20260506 19:09:02.936 | INFO | Analyzing chunk frames = [3750400,3750912) | 20260506 19:09:02.936 | INFO | MemInfo: { | 20260506 19:09:02.936 | INFO | "gpu" : " 17 GB", | 20260506 19:09:02.936 | INFO | "huge" : "", | 20260506 19:09:02.937 | INFO | "huge_shared" : " 27 GB", | 20260506 19:09:02.937 | INFO | "malloc" : " 14 GB", | 20260506 19:09:02.937 | INFO | "pinned" : " 29 GB" | 20260506 19:09:02.937 | INFO | } | 20260506 19:09:02.937 | INFO | Perf report for the last 5.13673 seconds: | 20260506 19:09:02.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:09:02.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:02.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:02.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:02.937 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 19:09:02.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:02.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:02.937 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 19:09:02.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:02.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:02.937 | INFO | Uploaded 36GiB to GPU in 2.03368s (17.7019GiB/s) | 20260506 19:09:02.937 | INFO | Downloaded 34GiB to GPU in 1.94881s (17.4465GiB/s) | 20260506 19:09:08.079 | INFO | Analyzing chunk frames = [3750912,3751424) | 20260506 19:09:08.079 | INFO | MemInfo: { | 20260506 19:09:08.079 | INFO | "gpu" : " 17 GB", | 20260506 19:09:08.079 | INFO | "huge" : "", | 20260506 19:09:08.079 | INFO | "huge_shared" : " 27 GB", | 20260506 19:09:08.079 | INFO | "malloc" : " 14 GB", | 20260506 19:09:08.079 | INFO | "pinned" : " 29 GB" | 20260506 19:09:08.079 | INFO | } | 20260506 19:09:08.079 | INFO | Perf report for the last 5.14239 seconds: | 20260506 19:09:08.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:09:08.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:08.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:08.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:08.079 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 19:09:08.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:08.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:08.079 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:09:08.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:08.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:08.079 | INFO | Uploaded 36GiB to GPU in 2.01997s (17.8221GiB/s) | 20260506 19:09:08.079 | INFO | Downloaded 34GiB to GPU in 1.99119s (17.0752GiB/s) | 20260506 19:09:13.208 | INFO | Analyzing chunk frames = [3751424,3751936) | 20260506 19:09:13.208 | INFO | MemInfo: { | 20260506 19:09:13.208 | INFO | "gpu" : " 17 GB", | 20260506 19:09:13.208 | INFO | "huge" : "", | 20260506 19:09:13.208 | INFO | "huge_shared" : " 27 GB", | 20260506 19:09:13.208 | INFO | "malloc" : " 14 GB", | 20260506 19:09:13.208 | INFO | "pinned" : " 29 GB" | 20260506 19:09:13.208 | INFO | } | 20260506 19:09:13.208 | INFO | Perf report for the last 5.1293 seconds: | 20260506 19:09:13.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:09:13.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:13.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:13.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:13.208 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260506 19:09:13.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:13.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:13.209 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:09:13.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:13.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:13.209 | INFO | Uploaded 36GiB to GPU in 1.99003s (18.0902GiB/s) | 20260506 19:09:13.209 | INFO | Downloaded 34GiB to GPU in 1.98672s (17.1137GiB/s) | 20260506 19:09:13.209 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:09:13.209 | INFO | Total mem usage: 18 GiB | 20260506 19:09:13.209 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:09:13.209 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.4%) | 20260506 19:09:13.209 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:09:13.209 | INFO | High Water Breakdown: | 20260506 19:09:13.209 | INFO | 61.8% : 9.9 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:09:13.209 | INFO | 38.1% : 6.09 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:09:13.209 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:09:13.209 | INFO | ------------------------------------------ | 20260506 19:09:13.209 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:09:13.209 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:09:13.209 | INFO | Total mem usage: 26.5 GiB | 20260506 19:09:13.209 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:09:13.209 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:09:13.209 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:09:13.209 | INFO | High Water Breakdown: | 20260506 19:09:13.209 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:09:13.209 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:09:13.209 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:09:13.209 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:09:13.209 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:09:13.209 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:09:13.209 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:09:13.209 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:09:13.209 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:09:13.209 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:09:13.209 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:09:13.209 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:09:13.209 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:09:13.209 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:09:13.209 | INFO | ------------------------------------------ | 20260506 19:09:13.209 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:09:13.209 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:09:13.209 | INFO | Total mem usage: 25.6 GiB | 20260506 19:09:13.209 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:09:13.209 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:09:13.209 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:09:13.209 | INFO | High Water Breakdown: | 20260506 19:09:13.209 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:09:13.209 | INFO | ------------------------------------------ | 20260506 19:09:13.209 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:09:13.209 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:09:13.209 | INFO | Total mem usage: 15.7 GiB | 20260506 19:09:13.209 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:09:13.209 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:09:13.209 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 19:09:13.209 | INFO | High Water Breakdown: | 20260506 19:09:13.209 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:09:13.209 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:09:13.209 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:09:13.209 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:09:13.209 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:09:13.209 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:09:13.209 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:09:13.209 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:09:13.209 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:09:13.209 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:09:13.209 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:09:13.209 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:09:13.209 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:09:13.209 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:09:13.209 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:09:13.209 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:09:13.210 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:09:13.210 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:09:13.210 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:09:13.210 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:09:13.210 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:09:13.210 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:09:13.210 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:09:13.210 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:09:13.210 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:09:13.210 | INFO | ------------------------------------------ | 20260506 19:09:13.210 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:09:13.210 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:09:13.210 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:09:13.210 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:09:13.210 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.34e+01 / 2.24e+00 | 20260506 19:09:13.210 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.02e-01, 9.62e-03 / 6.03e-03 | 20260506 19:09:13.210 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.43e+00, 5.50e-01 / 2.94e-01 | 20260506 19:09:13.210 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.80e-01 / 1.31e-01 | 20260506 19:09:13.210 | INFO | PerfReport: 15.99%, DME -- 3.99e+01 / 4.50e+01, 4.07e+00 / 2.06e+00 | 20260506 19:09:13.210 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.39e+00 / 1.88e+00 | 20260506 19:09:13.210 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.25e+00 / 2.86e+00, 3.81e-01 / 2.01e-01 | 20260506 19:09:13.210 | INFO | PerfReport: 3.38%, Metrics -- 1.05e+00 / 1.39e+00, 2.79e-01 / 1.78e-01 | 20260506 19:09:13.210 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.63e-01 / 4.11e-01, 5.74e-02 / 2.23e-02 | 20260506 19:09:13.210 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:09:13.210 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:09:13.210 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:09:13.210 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 19:09:13.210 | INFO | PerfReport: 21.47%, Upload -- 2.08e+01 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260506 19:09:13.210 | INFO | PerfReport: 21.17%, Download -- 2.05e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 19:09:13.210 | INFO | PerfReport: 32.22%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 19:09:13.210 | INFO | PerfReport: 1.36%, LockUpload -- 1.32e+00 / 0.00e+00, 5.20e+00 / 0.00e+00 | 20260506 19:09:13.210 | INFO | PerfReport: 1.13%, LockDownload -- 1.09e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 19:09:13.210 | INFO | PerfReport: 22.63%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 19:09:13.210 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260506 19:09:13.322 | INFO | Starting a round of Baz IO | 20260506 19:09:16.463 | INFO | Populating RTMetrics report | 20260506 19:09:16.463 | INFO | RTMetrics report populated | 20260506 19:09:16.463 | INFO | Opening tmp RTMetrics json file | 20260506 19:09:16.534 | INFO | Serializing RTMetrics to json | 20260506 19:09:16.540 | INFO | Serialization done, swapping tmp file now... | 20260506 19:09:16.545 | INFO | RTMetrics done | 20260506 19:09:16.632 | INFO | Finished a round of Baz IO: Wrote 2.018 GiB in 3.309 seconds | 20260506 19:09:16.632 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.75% / 0.00% / 65.25% | 20260506 19:09:16.632 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.972982% / 7.94729e-06% / 5.5631e-05% | 20260506 19:09:16.632 | INFO | Baz IO overall: | 20260506 19:09:16.632 | INFO | BytesWritten : 1982790989049 | 20260506 19:09:16.632 | INFO | NumEvents : 958604781522 | 20260506 19:09:18.345 | INFO | Analyzing chunk frames = [3751936,3752448) | 20260506 19:09:18.345 | INFO | MemInfo: { | 20260506 19:09:18.345 | INFO | "gpu" : " 17 GB", | 20260506 19:09:18.345 | INFO | "huge" : "", | 20260506 19:09:18.345 | INFO | "huge_shared" : " 27 GB", | 20260506 19:09:18.345 | INFO | "malloc" : " 14 GB", | 20260506 19:09:18.345 | INFO | "pinned" : " 29 GB" | 20260506 19:09:18.345 | INFO | } | 20260506 19:09:18.345 | INFO | Perf report for the last 5.13686 seconds: | 20260506 19:09:18.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:09:18.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:18.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:18.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:18.345 | INFO | ANALYSIS : 62.8%, 2.9, 95.7ms, 0 | 20260506 19:09:18.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:18.345 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 19:09:18.345 | INFO | PRE_HQ : 60.2%, 1.9, 58.0ms, 0 | 20260506 19:09:18.345 | INFO | BAZWRITER : 64.1%, 4.8, 157.1ms, 0 | 20260506 19:09:18.345 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.4ms, 0 | 20260506 19:09:18.346 | INFO | Uploaded 36GiB to GPU in 2.04728s (17.5843GiB/s) | 20260506 19:09:18.346 | INFO | Downloaded 38GiB to GPU in 2.21425s (17.1616GiB/s) | 20260506 19:09:23.492 | INFO | Analyzing chunk frames = [3752448,3752960) | 20260506 19:09:23.492 | INFO | MemInfo: { | 20260506 19:09:23.492 | INFO | "gpu" : " 17 GB", | 20260506 19:09:23.492 | INFO | "huge" : "", | 20260506 19:09:23.492 | INFO | "huge_shared" : " 27 GB", | 20260506 19:09:23.492 | INFO | "malloc" : " 14 GB", | 20260506 19:09:23.492 | INFO | "pinned" : " 29 GB" | 20260506 19:09:23.492 | INFO | } | 20260506 19:09:23.493 | INFO | Perf report for the last 5.14721 seconds: | 20260506 19:09:23.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:09:23.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:23.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:23.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:23.493 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 19:09:23.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:23.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:23.493 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:09:23.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:23.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:23.493 | INFO | Uploaded 36GiB to GPU in 1.99753s (18.0222GiB/s) | 20260506 19:09:23.493 | INFO | Downloaded 34GiB to GPU in 1.92239s (17.6863GiB/s) | 20260506 19:09:28.630 | INFO | Analyzing chunk frames = [3752960,3753472) | 20260506 19:09:28.630 | INFO | MemInfo: { | 20260506 19:09:28.630 | INFO | "gpu" : " 17 GB", | 20260506 19:09:28.630 | INFO | "huge" : "", | 20260506 19:09:28.630 | INFO | "huge_shared" : " 27 GB", | 20260506 19:09:28.630 | INFO | "malloc" : " 14 GB", | 20260506 19:09:28.630 | INFO | "pinned" : " 29 GB" | 20260506 19:09:28.630 | INFO | } | 20260506 19:09:28.630 | INFO | Perf report for the last 5.13759 seconds: | 20260506 19:09:28.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:09:28.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:28.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:28.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:28.630 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 19:09:28.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:28.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:28.630 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:09:28.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:28.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:28.630 | INFO | Uploaded 36GiB to GPU in 1.98929s (18.0969GiB/s) | 20260506 19:09:28.630 | INFO | Downloaded 34GiB to GPU in 1.91653s (17.7404GiB/s) | 20260506 19:09:33.766 | INFO | Analyzing chunk frames = [3753472,3753984) | 20260506 19:09:33.766 | INFO | MemInfo: { | 20260506 19:09:33.766 | INFO | "gpu" : " 17 GB", | 20260506 19:09:33.766 | INFO | "huge" : "", | 20260506 19:09:33.766 | INFO | "huge_shared" : " 27 GB", | 20260506 19:09:33.766 | INFO | "malloc" : " 14 GB", | 20260506 19:09:33.766 | INFO | "pinned" : " 29 GB" | 20260506 19:09:33.766 | INFO | } | 20260506 19:09:33.766 | INFO | Perf report for the last 5.13598 seconds: | 20260506 19:09:33.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:09:33.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:33.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:33.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:33.766 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 19:09:33.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:33.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:33.766 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 19:09:33.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:33.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:33.766 | INFO | Uploaded 36GiB to GPU in 1.93164s (18.637GiB/s) | 20260506 19:09:33.766 | INFO | Downloaded 34GiB to GPU in 1.86861s (18.1954GiB/s) | 20260506 19:09:38.908 | INFO | Analyzing chunk frames = [3753984,3754496) | 20260506 19:09:38.908 | INFO | MemInfo: { | 20260506 19:09:38.908 | INFO | "gpu" : " 17 GB", | 20260506 19:09:38.908 | INFO | "huge" : "", | 20260506 19:09:38.908 | INFO | "huge_shared" : " 27 GB", | 20260506 19:09:38.908 | INFO | "malloc" : " 14 GB", | 20260506 19:09:38.908 | INFO | "pinned" : " 29 GB" | 20260506 19:09:38.908 | INFO | } | 20260506 19:09:38.909 | INFO | Perf report for the last 5.14242 seconds: | 20260506 19:09:38.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:09:38.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:38.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:38.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:38.909 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 19:09:38.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:38.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:38.909 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:09:38.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:38.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:38.909 | INFO | Uploaded 36GiB to GPU in 2.02372s (17.789GiB/s) | 20260506 19:09:38.909 | INFO | Downloaded 34GiB to GPU in 1.91446s (17.7596GiB/s) | 20260506 19:09:44.049 | INFO | Analyzing chunk frames = [3754496,3755008) | 20260506 19:09:44.049 | INFO | MemInfo: { | 20260506 19:09:44.049 | INFO | "gpu" : " 17 GB", | 20260506 19:09:44.049 | INFO | "huge" : "", | 20260506 19:09:44.049 | INFO | "huge_shared" : " 27 GB", | 20260506 19:09:44.049 | INFO | "malloc" : " 14 GB", | 20260506 19:09:44.049 | INFO | "pinned" : " 29 GB" | 20260506 19:09:44.049 | INFO | } | 20260506 19:09:44.049 | INFO | Perf report for the last 5.1408 seconds: | 20260506 19:09:44.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:09:44.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:44.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:44.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:44.049 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:09:44.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:44.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:44.049 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 19:09:44.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:44.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:44.050 | INFO | Uploaded 36GiB to GPU in 2.03939s (17.6523GiB/s) | 20260506 19:09:44.050 | INFO | Downloaded 34GiB to GPU in 1.94633s (17.4688GiB/s) | 20260506 19:09:49.185 | INFO | Analyzing chunk frames = [3755008,3755520) | 20260506 19:09:49.185 | INFO | MemInfo: { | 20260506 19:09:49.185 | INFO | "gpu" : " 17 GB", | 20260506 19:09:49.185 | INFO | "huge" : "", | 20260506 19:09:49.185 | INFO | "huge_shared" : " 27 GB", | 20260506 19:09:49.185 | INFO | "malloc" : " 14 GB", | 20260506 19:09:49.185 | INFO | "pinned" : " 29 GB" | 20260506 19:09:49.186 | INFO | } | 20260506 19:09:49.186 | INFO | Perf report for the last 5.13621 seconds: | 20260506 19:09:49.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:09:49.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:49.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:49.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:49.186 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 19:09:49.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:49.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:49.186 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 19:09:49.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:49.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:49.186 | INFO | Uploaded 36GiB to GPU in 1.99944s (18.0051GiB/s) | 20260506 19:09:49.186 | INFO | Downloaded 34GiB to GPU in 1.96224s (17.3272GiB/s) | 20260506 19:09:54.327 | INFO | Analyzing chunk frames = [3755520,3756032) | 20260506 19:09:54.327 | INFO | MemInfo: { | 20260506 19:09:54.327 | INFO | "gpu" : " 17 GB", | 20260506 19:09:54.327 | INFO | "huge" : "", | 20260506 19:09:54.327 | INFO | "huge_shared" : " 27 GB", | 20260506 19:09:54.327 | INFO | "malloc" : " 14 GB", | 20260506 19:09:54.327 | INFO | "pinned" : " 29 GB" | 20260506 19:09:54.327 | INFO | } | 20260506 19:09:54.327 | INFO | Perf report for the last 5.1414 seconds: | 20260506 19:09:54.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:09:54.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:54.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:54.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:54.327 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 19:09:54.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:54.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:54.327 | INFO | PRE_HQ : 31.5%, 1.0, 16.3ms, 0 | 20260506 19:09:54.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:54.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:54.327 | INFO | Uploaded 36GiB to GPU in 2.04342s (17.6176GiB/s) | 20260506 19:09:54.327 | INFO | Downloaded 34GiB to GPU in 1.98894s (17.0946GiB/s) | 20260506 19:09:54.456 | INFO | Starting a round of Baz IO | 20260506 19:09:57.606 | INFO | Populating RTMetrics report | 20260506 19:09:57.606 | INFO | RTMetrics report populated | 20260506 19:09:57.606 | INFO | Opening tmp RTMetrics json file | 20260506 19:09:57.637 | INFO | Serializing RTMetrics to json | 20260506 19:09:57.695 | INFO | Serialization done, swapping tmp file now... | 20260506 19:09:57.701 | INFO | RTMetrics done | 20260506 19:09:58.189 | INFO | Finished a round of Baz IO: Wrote 2.018 GiB in 3.733 seconds | 20260506 19:09:58.190 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.74% / 0.00% / 65.25% | 20260506 19:09:58.190 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.969338% / 2.38419e-05% / 5.5631e-05% | 20260506 19:09:58.190 | INFO | Baz IO overall: | 20260506 19:09:58.190 | INFO | BytesWritten : 1984957804425 | 20260506 19:09:58.190 | INFO | NumEvents : 959647524338 | 20260506 19:09:59.462 | INFO | Analyzing chunk frames = [3756032,3756544) | 20260506 19:09:59.462 | INFO | MemInfo: { | 20260506 19:09:59.462 | INFO | "gpu" : " 17 GB", | 20260506 19:09:59.462 | INFO | "huge" : "", | 20260506 19:09:59.462 | INFO | "huge_shared" : " 27 GB", | 20260506 19:09:59.462 | INFO | "malloc" : " 14 GB", | 20260506 19:09:59.462 | INFO | "pinned" : " 29 GB" | 20260506 19:09:59.462 | INFO | } | 20260506 19:09:59.462 | INFO | Perf report for the last 5.13507 seconds: | 20260506 19:09:59.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:09:59.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:59.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:59.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:59.462 | INFO | ANALYSIS : 63.1%, 2.8, 94.1ms, 0 | 20260506 19:09:59.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:09:59.462 | INFO | RT_METRICS : 48.3%, 1.0, 25.3ms, 0 | 20260506 19:09:59.462 | INFO | PRE_HQ : 61.2%, 2.0, 62.9ms, 0 | 20260506 19:09:59.462 | INFO | BAZWRITER : 72.7%, 4.8, 179.4ms, 0 | 20260506 19:09:59.462 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 19:09:59.463 | INFO | Uploaded 36GiB to GPU in 2.06933s (17.3969GiB/s) | 20260506 19:09:59.463 | INFO | Downloaded 38GiB to GPU in 2.21243s (17.1757GiB/s) | 20260506 19:10:04.601 | INFO | Analyzing chunk frames = [3756544,3757056) | 20260506 19:10:04.601 | INFO | MemInfo: { | 20260506 19:10:04.601 | INFO | "gpu" : " 17 GB", | 20260506 19:10:04.601 | INFO | "huge" : "", | 20260506 19:10:04.601 | INFO | "huge_shared" : " 27 GB", | 20260506 19:10:04.601 | INFO | "malloc" : " 14 GB", | 20260506 19:10:04.601 | INFO | "pinned" : " 29 GB" | 20260506 19:10:04.601 | INFO | } | 20260506 19:10:04.601 | INFO | Perf report for the last 5.13933 seconds: | 20260506 19:10:04.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:10:04.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:04.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:04.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:04.601 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:10:04.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:04.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:04.601 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 19:10:04.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:04.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:04.602 | INFO | Uploaded 36GiB to GPU in 2.0049s (17.956GiB/s) | 20260506 19:10:04.602 | INFO | Downloaded 34GiB to GPU in 1.94764s (17.457GiB/s) | 20260506 19:10:09.749 | INFO | Analyzing chunk frames = [3757056,3757568) | 20260506 19:10:09.749 | INFO | MemInfo: { | 20260506 19:10:09.749 | INFO | "gpu" : " 17 GB", | 20260506 19:10:09.749 | INFO | "huge" : "", | 20260506 19:10:09.749 | INFO | "huge_shared" : " 27 GB", | 20260506 19:10:09.749 | INFO | "malloc" : " 14 GB", | 20260506 19:10:09.749 | INFO | "pinned" : " 29 GB" | 20260506 19:10:09.749 | INFO | } | 20260506 19:10:09.749 | INFO | Perf report for the last 5.14781 seconds: | 20260506 19:10:09.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:10:09.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:09.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:09.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:09.749 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 19:10:09.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:09.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:09.749 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:10:09.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:09.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:09.749 | INFO | Uploaded 36GiB to GPU in 1.977s (18.2094GiB/s) | 20260506 19:10:09.750 | INFO | Downloaded 34GiB to GPU in 1.91931s (17.7147GiB/s) | 20260506 19:10:14.884 | INFO | Analyzing chunk frames = [3757568,3758080) | 20260506 19:10:14.884 | INFO | MemInfo: { | 20260506 19:10:14.884 | INFO | "gpu" : " 17 GB", | 20260506 19:10:14.884 | INFO | "huge" : "", | 20260506 19:10:14.884 | INFO | "huge_shared" : " 27 GB", | 20260506 19:10:14.884 | INFO | "malloc" : " 14 GB", | 20260506 19:10:14.884 | INFO | "pinned" : " 29 GB" | 20260506 19:10:14.884 | INFO | } | 20260506 19:10:14.885 | INFO | Perf report for the last 5.13548 seconds: | 20260506 19:10:14.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:10:14.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:14.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:14.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:14.885 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 19:10:14.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:14.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:14.885 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:10:14.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:14.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:14.885 | INFO | Uploaded 36GiB to GPU in 1.90793s (18.8686GiB/s) | 20260506 19:10:14.885 | INFO | Downloaded 34GiB to GPU in 1.87113s (18.1708GiB/s) | 20260506 19:10:20.022 | INFO | Analyzing chunk frames = [3758080,3758592) | 20260506 19:10:20.022 | INFO | MemInfo: { | 20260506 19:10:20.022 | INFO | "gpu" : " 17 GB", | 20260506 19:10:20.022 | INFO | "huge" : "", | 20260506 19:10:20.022 | INFO | "huge_shared" : " 27 GB", | 20260506 19:10:20.022 | INFO | "malloc" : " 14 GB", | 20260506 19:10:20.022 | INFO | "pinned" : " 29 GB" | 20260506 19:10:20.022 | INFO | } | 20260506 19:10:20.022 | INFO | Perf report for the last 5.1371 seconds: | 20260506 19:10:20.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:10:20.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:20.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:20.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:20.022 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 19:10:20.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:20.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:20.022 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 19:10:20.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:20.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:20.022 | INFO | Uploaded 36GiB to GPU in 2.01001s (17.9104GiB/s) | 20260506 19:10:20.022 | INFO | Downloaded 34GiB to GPU in 1.90763s (17.8231GiB/s) | 20260506 19:10:25.166 | INFO | Analyzing chunk frames = [3758592,3759104) | 20260506 19:10:25.166 | INFO | MemInfo: { | 20260506 19:10:25.166 | INFO | "gpu" : " 17 GB", | 20260506 19:10:25.166 | INFO | "huge" : "", | 20260506 19:10:25.166 | INFO | "huge_shared" : " 27 GB", | 20260506 19:10:25.166 | INFO | "malloc" : " 14 GB", | 20260506 19:10:25.166 | INFO | "pinned" : " 29 GB" | 20260506 19:10:25.166 | INFO | } | 20260506 19:10:25.166 | INFO | Perf report for the last 5.14399 seconds: | 20260506 19:10:25.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:10:25.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:25.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:25.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:25.166 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 19:10:25.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:25.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:25.166 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:10:25.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:25.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:25.166 | INFO | Uploaded 36GiB to GPU in 2.01749s (17.8439GiB/s) | 20260506 19:10:25.166 | INFO | Downloaded 34GiB to GPU in 1.95479s (17.3932GiB/s) | 20260506 19:10:30.301 | INFO | Analyzing chunk frames = [3759104,3759616) | 20260506 19:10:30.301 | INFO | MemInfo: { | 20260506 19:10:30.301 | INFO | "gpu" : " 17 GB", | 20260506 19:10:30.301 | INFO | "huge" : "", | 20260506 19:10:30.301 | INFO | "huge_shared" : " 27 GB", | 20260506 19:10:30.301 | INFO | "malloc" : " 14 GB", | 20260506 19:10:30.301 | INFO | "pinned" : " 29 GB" | 20260506 19:10:30.301 | INFO | } | 20260506 19:10:30.301 | INFO | Perf report for the last 5.13547 seconds: | 20260506 19:10:30.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:10:30.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:30.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:30.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:30.301 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260506 19:10:30.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:30.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:30.301 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:10:30.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:30.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:30.302 | INFO | Uploaded 36GiB to GPU in 1.96942s (18.2795GiB/s) | 20260506 19:10:30.302 | INFO | Downloaded 34GiB to GPU in 2.00626s (16.947GiB/s) | 20260506 19:10:35.440 | INFO | Analyzing chunk frames = [3759616,3760128) | 20260506 19:10:35.440 | INFO | MemInfo: { | 20260506 19:10:35.440 | INFO | "gpu" : " 17 GB", | 20260506 19:10:35.440 | INFO | "huge" : "", | 20260506 19:10:35.440 | INFO | "huge_shared" : " 27 GB", | 20260506 19:10:35.440 | INFO | "malloc" : " 14 GB", | 20260506 19:10:35.440 | INFO | "pinned" : " 29 GB" | 20260506 19:10:35.440 | INFO | } | 20260506 19:10:35.440 | INFO | Perf report for the last 5.13916 seconds: | 20260506 19:10:35.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:10:35.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:35.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:35.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:35.440 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 19:10:35.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:35.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:35.441 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 19:10:35.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:35.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:35.441 | INFO | Uploaded 36GiB to GPU in 2.01635s (17.8541GiB/s) | 20260506 19:10:35.441 | INFO | Downloaded 34GiB to GPU in 1.98619s (17.1182GiB/s) | 20260506 19:10:35.441 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:10:35.441 | INFO | Total mem usage: 18 GiB | 20260506 19:10:35.441 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:10:35.441 | INFO | Recent High Water Marks Sum: 16.6 GiB (Savings: -8.64%) | 20260506 19:10:35.441 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:10:35.441 | INFO | High Water Breakdown: | 20260506 19:10:35.441 | INFO | 62.3% : 10.3 GiB ( 90.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:10:35.441 | INFO | 37.5% : 6.22 GiB ( 92.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:10:35.441 | INFO | 0.12% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:10:35.441 | INFO | ------------------------------------------ | 20260506 19:10:35.441 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:10:35.441 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:10:35.441 | INFO | Total mem usage: 26.5 GiB | 20260506 19:10:35.441 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:10:35.441 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:10:35.441 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:10:35.441 | INFO | High Water Breakdown: | 20260506 19:10:35.441 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:10:35.441 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:10:35.441 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:10:35.441 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:10:35.441 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:10:35.441 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:10:35.441 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:10:35.441 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:10:35.441 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:10:35.441 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:10:35.441 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:10:35.441 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:10:35.441 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:10:35.441 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:10:35.441 | INFO | ------------------------------------------ | 20260506 19:10:35.441 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:10:35.441 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:10:35.441 | INFO | Total mem usage: 25.6 GiB | 20260506 19:10:35.441 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:10:35.441 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:10:35.441 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:10:35.441 | INFO | High Water Breakdown: | 20260506 19:10:35.441 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:10:35.441 | INFO | ------------------------------------------ | 20260506 19:10:35.441 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:10:35.441 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:10:35.441 | INFO | Total mem usage: 15.7 GiB | 20260506 19:10:35.441 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:10:35.441 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:10:35.441 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 19:10:35.441 | INFO | High Water Breakdown: | 20260506 19:10:35.441 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:10:35.441 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:10:35.441 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:10:35.442 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:10:35.442 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:10:35.442 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:10:35.442 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:10:35.442 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:10:35.442 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:10:35.442 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:10:35.442 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:10:35.442 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:10:35.442 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:10:35.442 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:10:35.442 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:10:35.442 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:10:35.442 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:10:35.442 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:10:35.442 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:10:35.442 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:10:35.442 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:10:35.442 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:10:35.442 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:10:35.442 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:10:35.442 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:10:35.442 | INFO | ------------------------------------------ | 20260506 19:10:35.442 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:10:35.442 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:10:35.442 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:10:35.442 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:10:35.442 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.59e+01, 1.35e+01 / 2.35e+00 | 20260506 19:10:35.442 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.01e-01, 9.79e-03 / 5.16e-03 | 20260506 19:10:35.442 | INFO | PerfReport: 11.64%, Baseline -- 3.64e+00 / 4.41e+00, 5.59e-01 / 3.33e-01 | 20260506 19:10:35.442 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.83e-01 / 1.28e-01 | 20260506 19:10:35.442 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.46e+01, 4.05e+00 / 2.40e+00 | 20260506 19:10:35.442 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.45e+00 / 1.80e+00 | 20260506 19:10:35.442 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.24e+00 / 2.78e+00, 3.76e-01 / 2.14e-01 | 20260506 19:10:35.442 | INFO | PerfReport: 3.38%, Metrics -- 1.06e+00 / 1.35e+00, 3.09e-01 / 1.41e-01 | 20260506 19:10:35.442 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.64e-01 / 4.09e-01, 5.69e-02 / 1.93e-02 | 20260506 19:10:35.442 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:10:35.442 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:10:35.442 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:10:35.442 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 19:10:35.442 | INFO | PerfReport: 21.51%, Upload -- 2.09e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 19:10:35.442 | INFO | PerfReport: 21.17%, Download -- 2.05e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 19:10:35.442 | INFO | PerfReport: 32.25%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 19:10:35.442 | INFO | PerfReport: 1.42%, LockUpload -- 1.38e+00 / 0.00e+00, 5.51e+00 / 0.00e+00 | 20260506 19:10:35.442 | INFO | PerfReport: 1.12%, LockDownload -- 1.09e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260506 19:10:35.442 | INFO | PerfReport: 22.52%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 19:10:35.442 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.37e-02 / 0.00e+00 | 20260506 19:10:35.557 | INFO | Starting a round of Baz IO | 20260506 19:10:38.684 | INFO | Populating RTMetrics report | 20260506 19:10:38.685 | INFO | RTMetrics report populated | 20260506 19:10:38.685 | INFO | Opening tmp RTMetrics json file | 20260506 19:10:38.730 | INFO | Serializing RTMetrics to json | 20260506 19:10:38.788 | INFO | Serialization done, swapping tmp file now... | 20260506 19:10:38.824 | INFO | RTMetrics done | 20260506 19:10:38.927 | INFO | Finished a round of Baz IO: Wrote 2.019 GiB in 3.37 seconds | 20260506 19:10:38.927 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.74% / 0.00% / 65.26% | 20260506 19:10:38.927 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.967626% / 3.97364e-06% / 6.75519e-05% | 20260506 19:10:38.927 | INFO | Baz IO overall: | 20260506 19:10:38.927 | INFO | BytesWritten : 1987125256179 | 20260506 19:10:38.927 | INFO | NumEvents : 960690600481 | 20260506 19:10:40.581 | INFO | Analyzing chunk frames = [3760128,3760640) | 20260506 19:10:40.581 | INFO | MemInfo: { | 20260506 19:10:40.581 | INFO | "gpu" : " 17 GB", | 20260506 19:10:40.581 | INFO | "huge" : "", | 20260506 19:10:40.581 | INFO | "huge_shared" : " 27 GB", | 20260506 19:10:40.581 | INFO | "malloc" : " 14 GB", | 20260506 19:10:40.581 | INFO | "pinned" : " 29 GB" | 20260506 19:10:40.581 | INFO | } | 20260506 19:10:40.581 | INFO | Perf report for the last 5.1406 seconds: | 20260506 19:10:40.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:10:40.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:40.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:40.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:40.581 | INFO | ANALYSIS : 62.5%, 2.9, 95.0ms, 0 | 20260506 19:10:40.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:40.581 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260506 19:10:40.581 | INFO | PRE_HQ : 60.1%, 1.9, 59.5ms, 0 | 20260506 19:10:40.581 | INFO | BAZWRITER : 65.2%, 4.8, 160.2ms, 0 | 20260506 19:10:40.581 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 19:10:40.582 | INFO | Uploaded 36GiB to GPU in 2.06473s (17.4357GiB/s) | 20260506 19:10:40.582 | INFO | Downloaded 38GiB to GPU in 2.21291s (17.172GiB/s) | 20260506 19:10:45.716 | INFO | Analyzing chunk frames = [3760640,3761152) | 20260506 19:10:45.716 | INFO | MemInfo: { | 20260506 19:10:45.716 | INFO | "gpu" : " 17 GB", | 20260506 19:10:45.716 | INFO | "huge" : "", | 20260506 19:10:45.716 | INFO | "huge_shared" : " 27 GB", | 20260506 19:10:45.716 | INFO | "malloc" : " 14 GB", | 20260506 19:10:45.716 | INFO | "pinned" : " 29 GB" | 20260506 19:10:45.716 | INFO | } | 20260506 19:10:45.716 | INFO | Perf report for the last 5.13515 seconds: | 20260506 19:10:45.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:10:45.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:45.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:45.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:45.716 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 19:10:45.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:45.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:45.716 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:10:45.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:45.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:45.716 | INFO | Uploaded 36GiB to GPU in 2.00137s (17.9877GiB/s) | 20260506 19:10:45.717 | INFO | Downloaded 34GiB to GPU in 1.94274s (17.501GiB/s) | 20260506 19:10:50.857 | INFO | Analyzing chunk frames = [3761152,3761664) | 20260506 19:10:50.857 | INFO | MemInfo: { | 20260506 19:10:50.857 | INFO | "gpu" : " 17 GB", | 20260506 19:10:50.857 | INFO | "huge" : "", | 20260506 19:10:50.857 | INFO | "huge_shared" : " 27 GB", | 20260506 19:10:50.858 | INFO | "malloc" : " 14 GB", | 20260506 19:10:50.858 | INFO | "pinned" : " 29 GB" | 20260506 19:10:50.858 | INFO | } | 20260506 19:10:50.858 | INFO | Perf report for the last 5.14137 seconds: | 20260506 19:10:50.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:10:50.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:50.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:50.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:50.858 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 19:10:50.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:50.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:50.858 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:10:50.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:50.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:50.858 | INFO | Uploaded 36GiB to GPU in 1.98249s (18.159GiB/s) | 20260506 19:10:50.858 | INFO | Downloaded 34GiB to GPU in 1.92095s (17.6996GiB/s) | 20260506 19:10:56.000 | INFO | Analyzing chunk frames = [3761664,3762176) | 20260506 19:10:56.000 | INFO | MemInfo: { | 20260506 19:10:56.000 | INFO | "gpu" : " 17 GB", | 20260506 19:10:56.000 | INFO | "huge" : "", | 20260506 19:10:56.000 | INFO | "huge_shared" : " 27 GB", | 20260506 19:10:56.000 | INFO | "malloc" : " 14 GB", | 20260506 19:10:56.000 | INFO | "pinned" : " 29 GB" | 20260506 19:10:56.000 | INFO | } | 20260506 19:10:56.000 | INFO | Perf report for the last 5.14212 seconds: | 20260506 19:10:56.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:10:56.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:56.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:56.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:56.000 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 19:10:56.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:56.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:56.000 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:10:56.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:56.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:10:56.000 | INFO | Uploaded 36GiB to GPU in 1.93079s (18.6452GiB/s) | 20260506 19:10:56.000 | INFO | Downloaded 34GiB to GPU in 1.87512s (18.1321GiB/s) | 20260506 19:11:01.138 | INFO | Analyzing chunk frames = [3762176,3762688) | 20260506 19:11:01.138 | INFO | MemInfo: { | 20260506 19:11:01.138 | INFO | "gpu" : " 17 GB", | 20260506 19:11:01.138 | INFO | "huge" : "", | 20260506 19:11:01.138 | INFO | "huge_shared" : " 27 GB", | 20260506 19:11:01.138 | INFO | "malloc" : " 14 GB", | 20260506 19:11:01.138 | INFO | "pinned" : " 29 GB" | 20260506 19:11:01.138 | INFO | } | 20260506 19:11:01.138 | INFO | Perf report for the last 5.13827 seconds: | 20260506 19:11:01.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:11:01.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:01.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:01.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:01.138 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 19:11:01.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:01.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:01.138 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:11:01.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:01.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:01.138 | INFO | Uploaded 36GiB to GPU in 2.01547s (17.8618GiB/s) | 20260506 19:11:01.138 | INFO | Downloaded 34GiB to GPU in 1.91562s (17.7488GiB/s) | 20260506 19:11:06.273 | INFO | Analyzing chunk frames = [3762688,3763200) | 20260506 19:11:06.273 | INFO | MemInfo: { | 20260506 19:11:06.273 | INFO | "gpu" : " 17 GB", | 20260506 19:11:06.273 | INFO | "huge" : "", | 20260506 19:11:06.273 | INFO | "huge_shared" : " 27 GB", | 20260506 19:11:06.273 | INFO | "malloc" : " 14 GB", | 20260506 19:11:06.273 | INFO | "pinned" : " 29 GB" | 20260506 19:11:06.273 | INFO | } | 20260506 19:11:06.273 | INFO | Perf report for the last 5.13513 seconds: | 20260506 19:11:06.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:11:06.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:06.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:06.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:06.273 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:11:06.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:06.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:06.273 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:11:06.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:06.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:06.273 | INFO | Uploaded 36GiB to GPU in 2.05339s (17.532GiB/s) | 20260506 19:11:06.274 | INFO | Downloaded 34GiB to GPU in 1.95978s (17.3489GiB/s) | 20260506 19:11:11.414 | INFO | Analyzing chunk frames = [3763200,3763712) | 20260506 19:11:11.414 | INFO | MemInfo: { | 20260506 19:11:11.414 | INFO | "gpu" : " 17 GB", | 20260506 19:11:11.414 | INFO | "huge" : "", | 20260506 19:11:11.414 | INFO | "huge_shared" : " 27 GB", | 20260506 19:11:11.414 | INFO | "malloc" : " 14 GB", | 20260506 19:11:11.414 | INFO | "pinned" : " 29 GB" | 20260506 19:11:11.414 | INFO | } | 20260506 19:11:11.414 | INFO | Perf report for the last 5.14084 seconds: | 20260506 19:11:11.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:11:11.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:11.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:11.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:11.414 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:11:11.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:11.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:11.414 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 19:11:11.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:11.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:11.415 | INFO | Uploaded 36GiB to GPU in 1.97338s (18.2428GiB/s) | 20260506 19:11:11.415 | INFO | Downloaded 34GiB to GPU in 2.00562s (16.9524GiB/s) | 20260506 19:11:16.554 | INFO | Analyzing chunk frames = [3763712,3764224) | 20260506 19:11:16.554 | INFO | MemInfo: { | 20260506 19:11:16.554 | INFO | "gpu" : " 17 GB", | 20260506 19:11:16.554 | INFO | "huge" : "", | 20260506 19:11:16.554 | INFO | "huge_shared" : " 27 GB", | 20260506 19:11:16.554 | INFO | "malloc" : " 14 GB", | 20260506 19:11:16.554 | INFO | "pinned" : " 29 GB" | 20260506 19:11:16.554 | INFO | } | 20260506 19:11:16.554 | INFO | Perf report for the last 5.14013 seconds: | 20260506 19:11:16.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:11:16.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:16.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:16.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:16.554 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 19:11:16.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:16.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:16.554 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 19:11:16.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:16.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:16.554 | INFO | Uploaded 36GiB to GPU in 2.02126s (17.8106GiB/s) | 20260506 19:11:16.555 | INFO | Downloaded 34GiB to GPU in 1.98197s (17.1546GiB/s) | 20260506 19:11:16.670 | INFO | Starting a round of Baz IO | 20260506 19:11:19.803 | INFO | Populating RTMetrics report | 20260506 19:11:19.803 | INFO | RTMetrics report populated | 20260506 19:11:19.803 | INFO | Opening tmp RTMetrics json file | 20260506 19:11:19.807 | INFO | Serializing RTMetrics to json | 20260506 19:11:19.809 | INFO | Serialization done, swapping tmp file now... | 20260506 19:11:19.809 | INFO | RTMetrics done | 20260506 19:11:20.215 | INFO | Finished a round of Baz IO: Wrote 2.016 GiB in 3.544 seconds | 20260506 19:11:20.215 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.73% / 0.00% / 65.26% | 20260506 19:11:20.215 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.963199% / 3.97364e-06% / 9.93411e-05% | 20260506 19:11:20.215 | INFO | Baz IO overall: | 20260506 19:11:20.215 | INFO | BytesWritten : 1989290125962 | 20260506 19:11:20.215 | INFO | NumEvents : 961731763983 | 20260506 19:11:21.692 | INFO | Analyzing chunk frames = [3764224,3764736) | 20260506 19:11:21.692 | INFO | MemInfo: { | 20260506 19:11:21.692 | INFO | "gpu" : " 17 GB", | 20260506 19:11:21.692 | INFO | "huge" : "", | 20260506 19:11:21.692 | INFO | "huge_shared" : " 27 GB", | 20260506 19:11:21.692 | INFO | "malloc" : " 14 GB", | 20260506 19:11:21.692 | INFO | "pinned" : " 29 GB" | 20260506 19:11:21.692 | INFO | } | 20260506 19:11:21.692 | INFO | Perf report for the last 5.13775 seconds: | 20260506 19:11:21.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:11:21.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:21.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:21.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:21.692 | INFO | ANALYSIS : 62.7%, 2.9, 95.2ms, 0 | 20260506 19:11:21.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:21.692 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 19:11:21.692 | INFO | PRE_HQ : 60.4%, 1.9, 58.0ms, 0 | 20260506 19:11:21.692 | INFO | BAZWRITER : 68.7%, 4.8, 169.6ms, 0 | 20260506 19:11:21.692 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.5ms, 0 | 20260506 19:11:21.692 | INFO | Uploaded 36GiB to GPU in 2.04606s (17.5948GiB/s) | 20260506 19:11:21.692 | INFO | Downloaded 38GiB to GPU in 2.21591s (17.1487GiB/s) | 20260506 19:11:26.833 | INFO | Analyzing chunk frames = [3764736,3765248) | 20260506 19:11:26.833 | INFO | MemInfo: { | 20260506 19:11:26.833 | INFO | "gpu" : " 17 GB", | 20260506 19:11:26.833 | INFO | "huge" : "", | 20260506 19:11:26.833 | INFO | "huge_shared" : " 27 GB", | 20260506 19:11:26.833 | INFO | "malloc" : " 14 GB", | 20260506 19:11:26.833 | INFO | "pinned" : " 29 GB" | 20260506 19:11:26.833 | INFO | } | 20260506 19:11:26.833 | INFO | Perf report for the last 5.14108 seconds: | 20260506 19:11:26.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:11:26.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:26.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:26.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:26.833 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 19:11:26.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:26.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:26.833 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:11:26.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:26.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:26.833 | INFO | Uploaded 36GiB to GPU in 2.00383s (17.9656GiB/s) | 20260506 19:11:26.833 | INFO | Downloaded 34GiB to GPU in 1.92853s (17.63GiB/s) | 20260506 19:11:31.973 | INFO | Analyzing chunk frames = [3765248,3765760) | 20260506 19:11:31.973 | INFO | MemInfo: { | 20260506 19:11:31.973 | INFO | "gpu" : " 17 GB", | 20260506 19:11:31.973 | INFO | "huge" : "", | 20260506 19:11:31.973 | INFO | "huge_shared" : " 27 GB", | 20260506 19:11:31.973 | INFO | "malloc" : " 14 GB", | 20260506 19:11:31.973 | INFO | "pinned" : " 29 GB" | 20260506 19:11:31.973 | INFO | } | 20260506 19:11:31.973 | INFO | Perf report for the last 5.13967 seconds: | 20260506 19:11:31.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:11:31.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:31.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:31.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:31.973 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 19:11:31.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:31.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:31.973 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:11:31.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:31.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:31.973 | INFO | Uploaded 36GiB to GPU in 1.97694s (18.21GiB/s) | 20260506 19:11:31.973 | INFO | Downloaded 34GiB to GPU in 1.91964s (17.7116GiB/s) | 20260506 19:11:37.116 | INFO | Analyzing chunk frames = [3765760,3766272) | 20260506 19:11:37.116 | INFO | MemInfo: { | 20260506 19:11:37.116 | INFO | "gpu" : " 17 GB", | 20260506 19:11:37.116 | INFO | "huge" : "", | 20260506 19:11:37.116 | INFO | "huge_shared" : " 27 GB", | 20260506 19:11:37.116 | INFO | "malloc" : " 14 GB", | 20260506 19:11:37.116 | INFO | "pinned" : " 29 GB" | 20260506 19:11:37.116 | INFO | } | 20260506 19:11:37.116 | INFO | Perf report for the last 5.14335 seconds: | 20260506 19:11:37.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:11:37.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:37.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:37.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:37.116 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260506 19:11:37.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:37.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:37.116 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:11:37.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:37.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:37.116 | INFO | Uploaded 36GiB to GPU in 1.94398s (18.5187GiB/s) | 20260506 19:11:37.117 | INFO | Downloaded 34GiB to GPU in 1.86973s (18.1844GiB/s) | 20260506 19:11:42.255 | INFO | Analyzing chunk frames = [3766272,3766784) | 20260506 19:11:42.255 | INFO | MemInfo: { | 20260506 19:11:42.255 | INFO | "gpu" : " 17 GB", | 20260506 19:11:42.255 | INFO | "huge" : "", | 20260506 19:11:42.255 | INFO | "huge_shared" : " 27 GB", | 20260506 19:11:42.255 | INFO | "malloc" : " 14 GB", | 20260506 19:11:42.255 | INFO | "pinned" : " 29 GB" | 20260506 19:11:42.255 | INFO | } | 20260506 19:11:42.255 | INFO | Perf report for the last 5.13924 seconds: | 20260506 19:11:42.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:11:42.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:42.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:42.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:42.255 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260506 19:11:42.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:42.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:42.255 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:11:42.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:42.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:42.256 | INFO | Uploaded 36GiB to GPU in 2.00981s (17.9121GiB/s) | 20260506 19:11:42.256 | INFO | Downloaded 34GiB to GPU in 1.90646s (17.8341GiB/s) | 20260506 19:11:47.391 | INFO | Analyzing chunk frames = [3766784,3767296) | 20260506 19:11:47.391 | INFO | MemInfo: { | 20260506 19:11:47.391 | INFO | "gpu" : " 17 GB", | 20260506 19:11:47.391 | INFO | "huge" : "", | 20260506 19:11:47.391 | INFO | "huge_shared" : " 27 GB", | 20260506 19:11:47.391 | INFO | "malloc" : " 14 GB", | 20260506 19:11:47.391 | INFO | "pinned" : " 29 GB" | 20260506 19:11:47.391 | INFO | } | 20260506 19:11:47.391 | INFO | Perf report for the last 5.13587 seconds: | 20260506 19:11:47.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:11:47.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:47.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:47.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:47.391 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 19:11:47.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:47.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:47.391 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:11:47.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:47.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:47.391 | INFO | Uploaded 36GiB to GPU in 2.02637s (17.7658GiB/s) | 20260506 19:11:47.391 | INFO | Downloaded 34GiB to GPU in 1.94472s (17.4833GiB/s) | 20260506 19:11:52.528 | INFO | Analyzing chunk frames = [3767296,3767808) | 20260506 19:11:52.528 | INFO | MemInfo: { | 20260506 19:11:52.528 | INFO | "gpu" : " 17 GB", | 20260506 19:11:52.528 | INFO | "huge" : "", | 20260506 19:11:52.528 | INFO | "huge_shared" : " 27 GB", | 20260506 19:11:52.528 | INFO | "malloc" : " 14 GB", | 20260506 19:11:52.528 | INFO | "pinned" : " 29 GB" | 20260506 19:11:52.528 | INFO | } | 20260506 19:11:52.528 | INFO | Perf report for the last 5.13711 seconds: | 20260506 19:11:52.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:11:52.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:52.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:52.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:52.529 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 19:11:52.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:52.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:52.529 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 19:11:52.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:52.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:52.529 | INFO | Uploaded 36GiB to GPU in 1.97936s (18.1877GiB/s) | 20260506 19:11:52.529 | INFO | Downloaded 34GiB to GPU in 2.00709s (16.9399GiB/s) | 20260506 19:11:57.663 | INFO | Analyzing chunk frames = [3767808,3768320) | 20260506 19:11:57.663 | INFO | MemInfo: { | 20260506 19:11:57.663 | INFO | "gpu" : " 17 GB", | 20260506 19:11:57.663 | INFO | "huge" : "", | 20260506 19:11:57.663 | INFO | "huge_shared" : " 27 GB", | 20260506 19:11:57.663 | INFO | "malloc" : " 14 GB", | 20260506 19:11:57.663 | INFO | "pinned" : " 29 GB" | 20260506 19:11:57.663 | INFO | } | 20260506 19:11:57.663 | INFO | Perf report for the last 5.1351 seconds: | 20260506 19:11:57.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:11:57.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:57.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:57.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:57.664 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 19:11:57.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:57.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:57.664 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:11:57.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:57.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:11:57.664 | INFO | Uploaded 36GiB to GPU in 1.99772s (18.0205GiB/s) | 20260506 19:11:57.664 | INFO | Downloaded 34GiB to GPU in 1.9832s (17.144GiB/s) | 20260506 19:11:57.664 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:11:57.664 | INFO | Total mem usage: 18 GiB | 20260506 19:11:57.664 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:11:57.664 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.7%) | 20260506 19:11:57.664 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:11:57.664 | INFO | High Water Breakdown: | 20260506 19:11:57.664 | INFO | 61.4% : 9.55 GiB ( 83.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:11:57.664 | INFO | 38.5% : 5.99 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:11:57.664 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:11:57.664 | INFO | ------------------------------------------ | 20260506 19:11:57.664 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:11:57.664 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:11:57.664 | INFO | Total mem usage: 26.5 GiB | 20260506 19:11:57.664 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:11:57.664 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 19:11:57.664 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260506 19:11:57.664 | INFO | High Water Breakdown: | 20260506 19:11:57.664 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:11:57.664 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:11:57.664 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:11:57.664 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:11:57.664 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:11:57.664 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:11:57.664 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:11:57.664 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:11:57.664 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:11:57.664 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:11:57.664 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:11:57.664 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:11:57.664 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:11:57.664 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:11:57.664 | INFO | ------------------------------------------ | 20260506 19:11:57.664 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:11:57.664 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:11:57.664 | INFO | Total mem usage: 25.6 GiB | 20260506 19:11:57.664 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:11:57.664 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:11:57.664 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:11:57.664 | INFO | High Water Breakdown: | 20260506 19:11:57.664 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:11:57.664 | INFO | ------------------------------------------ | 20260506 19:11:57.664 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:11:57.664 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:11:57.664 | INFO | Total mem usage: 15.7 GiB | 20260506 19:11:57.664 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:11:57.664 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:11:57.664 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:11:57.664 | INFO | High Water Breakdown: | 20260506 19:11:57.664 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:11:57.664 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:11:57.664 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:11:57.665 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:11:57.665 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:11:57.665 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:11:57.665 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:11:57.665 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:11:57.665 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:11:57.665 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:11:57.665 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:11:57.665 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:11:57.665 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:11:57.665 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:11:57.665 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:11:57.665 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:11:57.665 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:11:57.665 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:11:57.665 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:11:57.665 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:11:57.665 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:11:57.665 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:11:57.665 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:11:57.665 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:11:57.665 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:11:57.665 | INFO | ------------------------------------------ | 20260506 19:11:57.665 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:11:57.665 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:11:57.665 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:11:57.665 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:11:57.665 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.56e+01, 1.33e+01 / 2.34e+00 | 20260506 19:11:57.665 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 1.02e-02 / 5.79e-03 | 20260506 19:11:57.665 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.41e+00, 5.56e-01 / 3.24e-01 | 20260506 19:11:57.665 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.82e-01 / 1.17e-01 | 20260506 19:11:57.665 | INFO | PerfReport: 15.99%, DME -- 3.99e+01 / 4.44e+01, 4.20e+00 / 2.53e+00 | 20260506 19:11:57.665 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.40e+00 / 1.73e+00 | 20260506 19:11:57.665 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.25e+00 / 2.79e+00, 3.85e-01 / 2.20e-01 | 20260506 19:11:57.665 | INFO | PerfReport: 3.37%, Metrics -- 1.05e+00 / 1.36e+00, 2.75e-01 / 1.58e-01 | 20260506 19:11:57.665 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.62e-01 / 4.09e-01, 5.59e-02 / 2.24e-02 | 20260506 19:11:57.665 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:11:57.665 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:11:57.665 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:11:57.665 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 19:11:57.665 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 19:11:57.665 | INFO | PerfReport: 21.21%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 19:11:57.665 | INFO | PerfReport: 32.21%, Compute -- 3.12e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 19:11:57.665 | INFO | PerfReport: 1.37%, LockUpload -- 1.32e+00 / 0.00e+00, 5.41e+00 / 0.00e+00 | 20260506 19:11:57.665 | INFO | PerfReport: 1.14%, LockDownload -- 1.11e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 19:11:57.665 | INFO | PerfReport: 22.55%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 19:11:57.665 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260506 19:11:57.779 | INFO | Starting a round of Baz IO | 20260506 19:12:00.913 | INFO | Populating RTMetrics report | 20260506 19:12:00.913 | INFO | RTMetrics report populated | 20260506 19:12:00.913 | INFO | Opening tmp RTMetrics json file | 20260506 19:12:00.918 | INFO | Serializing RTMetrics to json | 20260506 19:12:00.922 | INFO | Serialization done, swapping tmp file now... | 20260506 19:12:00.923 | INFO | RTMetrics done | 20260506 19:12:01.541 | INFO | Finished a round of Baz IO: Wrote 2.016 GiB in 3.762 seconds | 20260506 19:12:01.541 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.73% / 0.00% / 65.27% | 20260506 19:12:01.541 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.962035% / 7.94729e-06% / 3.97364e-05% | 20260506 19:12:01.541 | INFO | Baz IO overall: | 20260506 19:12:01.541 | INFO | BytesWritten : 1991454898874 | 20260506 19:12:01.541 | INFO | NumEvents : 962772918499 | 20260506 19:12:02.805 | INFO | Analyzing chunk frames = [3768320,3768832) | 20260506 19:12:02.805 | INFO | MemInfo: { | 20260506 19:12:02.805 | INFO | "gpu" : " 17 GB", | 20260506 19:12:02.805 | INFO | "huge" : "", | 20260506 19:12:02.805 | INFO | "huge_shared" : " 27 GB", | 20260506 19:12:02.805 | INFO | "malloc" : " 14 GB", | 20260506 19:12:02.805 | INFO | "pinned" : " 29 GB" | 20260506 19:12:02.805 | INFO | } | 20260506 19:12:02.805 | INFO | Perf report for the last 5.14138 seconds: | 20260506 19:12:02.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:12:02.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:02.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:02.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:02.805 | INFO | ANALYSIS : 62.6%, 2.9, 95.4ms, 0 | 20260506 19:12:02.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:02.805 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 19:12:02.805 | INFO | PRE_HQ : 60.0%, 1.8, 57.5ms, 0 | 20260506 19:12:02.805 | INFO | BAZWRITER : 72.7%, 4.8, 180.1ms, 0 | 20260506 19:12:02.805 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 19:12:02.805 | INFO | Uploaded 36GiB to GPU in 2.01276s (17.8859GiB/s) | 20260506 19:12:02.805 | INFO | Downloaded 38GiB to GPU in 2.21196s (17.1793GiB/s) | 20260506 19:12:07.946 | INFO | Analyzing chunk frames = [3768832,3769344) | 20260506 19:12:07.946 | INFO | MemInfo: { | 20260506 19:12:07.946 | INFO | "gpu" : " 17 GB", | 20260506 19:12:07.946 | INFO | "huge" : "", | 20260506 19:12:07.946 | INFO | "huge_shared" : " 27 GB", | 20260506 19:12:07.946 | INFO | "malloc" : " 14 GB", | 20260506 19:12:07.946 | INFO | "pinned" : " 29 GB" | 20260506 19:12:07.946 | INFO | } | 20260506 19:12:07.946 | INFO | Perf report for the last 5.14115 seconds: | 20260506 19:12:07.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:12:07.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:07.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:07.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:07.946 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 19:12:07.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:07.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:07.946 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:12:07.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:07.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:07.946 | INFO | Uploaded 36GiB to GPU in 2.02s (17.8218GiB/s) | 20260506 19:12:07.946 | INFO | Downloaded 34GiB to GPU in 1.95891s (17.3566GiB/s) | 20260506 19:12:13.088 | INFO | Analyzing chunk frames = [3769344,3769856) | 20260506 19:12:13.088 | INFO | MemInfo: { | 20260506 19:12:13.088 | INFO | "gpu" : " 17 GB", | 20260506 19:12:13.088 | INFO | "huge" : "", | 20260506 19:12:13.088 | INFO | "huge_shared" : " 27 GB", | 20260506 19:12:13.088 | INFO | "malloc" : " 14 GB", | 20260506 19:12:13.088 | INFO | "pinned" : " 29 GB" | 20260506 19:12:13.088 | INFO | } | 20260506 19:12:13.088 | INFO | Perf report for the last 5.14182 seconds: | 20260506 19:12:13.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:12:13.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:13.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:13.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:13.088 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 19:12:13.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:13.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:13.088 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:12:13.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:13.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:13.088 | INFO | Uploaded 36GiB to GPU in 1.9658s (18.3131GiB/s) | 20260506 19:12:13.088 | INFO | Downloaded 34GiB to GPU in 1.94196s (17.5081GiB/s) | 20260506 19:12:18.232 | INFO | Analyzing chunk frames = [3769856,3770368) | 20260506 19:12:18.232 | INFO | MemInfo: { | 20260506 19:12:18.232 | INFO | "gpu" : " 17 GB", | 20260506 19:12:18.232 | INFO | "huge" : "", | 20260506 19:12:18.232 | INFO | "huge_shared" : " 27 GB", | 20260506 19:12:18.232 | INFO | "malloc" : " 14 GB", | 20260506 19:12:18.232 | INFO | "pinned" : " 29 GB" | 20260506 19:12:18.232 | INFO | } | 20260506 19:12:18.232 | INFO | Perf report for the last 5.14397 seconds: | 20260506 19:12:18.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:12:18.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:18.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:18.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:18.232 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:12:18.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:18.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:18.232 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:12:18.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:18.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:18.232 | INFO | Uploaded 36GiB to GPU in 1.92205s (18.73GiB/s) | 20260506 19:12:18.232 | INFO | Downloaded 34GiB to GPU in 1.8719s (18.1634GiB/s) | 20260506 19:12:23.371 | INFO | Analyzing chunk frames = [3770368,3770880) | 20260506 19:12:23.371 | INFO | MemInfo: { | 20260506 19:12:23.371 | INFO | "gpu" : " 17 GB", | 20260506 19:12:23.371 | INFO | "huge" : "", | 20260506 19:12:23.371 | INFO | "huge_shared" : " 27 GB", | 20260506 19:12:23.371 | INFO | "malloc" : " 14 GB", | 20260506 19:12:23.371 | INFO | "pinned" : " 29 GB" | 20260506 19:12:23.371 | INFO | } | 20260506 19:12:23.371 | INFO | Perf report for the last 5.13908 seconds: | 20260506 19:12:23.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:12:23.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:23.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:23.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:23.371 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 19:12:23.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:23.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:23.371 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:12:23.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:23.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:23.371 | INFO | Uploaded 36GiB to GPU in 1.99975s (18.0023GiB/s) | 20260506 19:12:23.371 | INFO | Downloaded 34GiB to GPU in 1.91802s (17.7267GiB/s) | 20260506 19:12:28.511 | INFO | Analyzing chunk frames = [3770880,3771392) | 20260506 19:12:28.511 | INFO | MemInfo: { | 20260506 19:12:28.511 | INFO | "gpu" : " 17 GB", | 20260506 19:12:28.511 | INFO | "huge" : "", | 20260506 19:12:28.511 | INFO | "huge_shared" : " 27 GB", | 20260506 19:12:28.512 | INFO | "malloc" : " 14 GB", | 20260506 19:12:28.512 | INFO | "pinned" : " 29 GB" | 20260506 19:12:28.512 | INFO | } | 20260506 19:12:28.512 | INFO | Perf report for the last 5.14076 seconds: | 20260506 19:12:28.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:12:28.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:28.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:28.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:28.512 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 19:12:28.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:28.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:28.512 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 19:12:28.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:28.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:28.512 | INFO | Uploaded 36GiB to GPU in 2.01309s (17.8829GiB/s) | 20260506 19:12:28.512 | INFO | Downloaded 34GiB to GPU in 1.95275s (17.4113GiB/s) | 20260506 19:12:33.641 | INFO | Analyzing chunk frames = [3771392,3771904) | 20260506 19:12:33.641 | INFO | MemInfo: { | 20260506 19:12:33.641 | INFO | "gpu" : " 17 GB", | 20260506 19:12:33.641 | INFO | "huge" : "", | 20260506 19:12:33.641 | INFO | "huge_shared" : " 27 GB", | 20260506 19:12:33.641 | INFO | "malloc" : " 14 GB", | 20260506 19:12:33.641 | INFO | "pinned" : " 29 GB" | 20260506 19:12:33.641 | INFO | } | 20260506 19:12:33.641 | INFO | Perf report for the last 5.12935 seconds: | 20260506 19:12:33.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:12:33.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:33.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:33.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:33.641 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 19:12:33.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:33.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:33.641 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 19:12:33.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:33.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:33.641 | INFO | Uploaded 36GiB to GPU in 1.97632s (18.2157GiB/s) | 20260506 19:12:33.641 | INFO | Downloaded 34GiB to GPU in 1.99878s (17.0104GiB/s) | 20260506 19:12:38.779 | INFO | Analyzing chunk frames = [3771904,3772416) | 20260506 19:12:38.779 | INFO | MemInfo: { | 20260506 19:12:38.779 | INFO | "gpu" : " 17 GB", | 20260506 19:12:38.779 | INFO | "huge" : "", | 20260506 19:12:38.779 | INFO | "huge_shared" : " 27 GB", | 20260506 19:12:38.779 | INFO | "malloc" : " 14 GB", | 20260506 19:12:38.779 | INFO | "pinned" : " 29 GB" | 20260506 19:12:38.779 | INFO | } | 20260506 19:12:38.779 | INFO | Perf report for the last 5.13815 seconds: | 20260506 19:12:38.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:12:38.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:38.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:38.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:38.779 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 19:12:38.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:38.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:38.780 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:12:38.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:38.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:38.780 | INFO | Uploaded 36GiB to GPU in 2.00829s (17.9257GiB/s) | 20260506 19:12:38.780 | INFO | Downloaded 34GiB to GPU in 1.97894s (17.1809GiB/s) | 20260506 19:12:38.892 | INFO | Starting a round of Baz IO | 20260506 19:12:42.025 | INFO | Populating RTMetrics report | 20260506 19:12:42.025 | INFO | RTMetrics report populated | 20260506 19:12:42.025 | INFO | Opening tmp RTMetrics json file | 20260506 19:12:42.030 | INFO | Serializing RTMetrics to json | 20260506 19:12:42.031 | INFO | Serialization done, swapping tmp file now... | 20260506 19:12:42.032 | INFO | RTMetrics done | 20260506 19:12:42.332 | INFO | Finished a round of Baz IO: Wrote 2.017 GiB in 3.44 seconds | 20260506 19:12:42.332 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.72% / 0.00% / 65.27% | 20260506 19:12:42.332 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.958753% / 0% / 7.94729e-05% | 20260506 19:12:42.332 | INFO | Baz IO overall: | 20260506 19:12:42.332 | INFO | BytesWritten : 1993620394035 | 20260506 19:12:42.332 | INFO | NumEvents : 963814422059 | 20260506 19:12:43.928 | INFO | Analyzing chunk frames = [3772416,3772928) | 20260506 19:12:43.928 | INFO | MemInfo: { | 20260506 19:12:43.928 | INFO | "gpu" : " 17 GB", | 20260506 19:12:43.928 | INFO | "huge" : "", | 20260506 19:12:43.928 | INFO | "huge_shared" : " 27 GB", | 20260506 19:12:43.928 | INFO | "malloc" : " 14 GB", | 20260506 19:12:43.928 | INFO | "pinned" : " 29 GB" | 20260506 19:12:43.928 | INFO | } | 20260506 19:12:43.929 | INFO | Perf report for the last 5.14929 seconds: | 20260506 19:12:43.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:12:43.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:43.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:43.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:43.929 | INFO | ANALYSIS : 62.5%, 2.9, 95.2ms, 0 | 20260506 19:12:43.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:43.929 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 19:12:43.929 | INFO | PRE_HQ : 60.0%, 1.8, 57.6ms, 0 | 20260506 19:12:43.929 | INFO | BAZWRITER : 66.5%, 4.8, 164.8ms, 0 | 20260506 19:12:43.929 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 19:12:43.929 | INFO | Uploaded 36GiB to GPU in 2.05106s (17.5519GiB/s) | 20260506 19:12:43.929 | INFO | Downloaded 38GiB to GPU in 2.20517s (17.2322GiB/s) | 20260506 19:12:49.069 | INFO | Analyzing chunk frames = [3772928,3773440) | 20260506 19:12:49.069 | INFO | MemInfo: { | 20260506 19:12:49.069 | INFO | "gpu" : " 17 GB", | 20260506 19:12:49.069 | INFO | "huge" : "", | 20260506 19:12:49.069 | INFO | "huge_shared" : " 27 GB", | 20260506 19:12:49.069 | INFO | "malloc" : " 14 GB", | 20260506 19:12:49.069 | INFO | "pinned" : " 29 GB" | 20260506 19:12:49.069 | INFO | } | 20260506 19:12:49.069 | INFO | Perf report for the last 5.14087 seconds: | 20260506 19:12:49.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:12:49.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:49.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:49.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:49.069 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:12:49.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:49.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:49.070 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:12:49.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:49.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:49.070 | INFO | Uploaded 36GiB to GPU in 2.01554s (17.8612GiB/s) | 20260506 19:12:49.070 | INFO | Downloaded 34GiB to GPU in 1.93406s (17.5796GiB/s) | 20260506 19:12:54.205 | INFO | Analyzing chunk frames = [3773440,3773952) | 20260506 19:12:54.205 | INFO | MemInfo: { | 20260506 19:12:54.205 | INFO | "gpu" : " 17 GB", | 20260506 19:12:54.205 | INFO | "huge" : "", | 20260506 19:12:54.205 | INFO | "huge_shared" : " 27 GB", | 20260506 19:12:54.205 | INFO | "malloc" : " 14 GB", | 20260506 19:12:54.205 | INFO | "pinned" : " 29 GB" | 20260506 19:12:54.205 | INFO | } | 20260506 19:12:54.205 | INFO | Perf report for the last 5.13561 seconds: | 20260506 19:12:54.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:12:54.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:54.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:54.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:54.205 | INFO | ANALYSIS : 60.9%, 3.0, 94.1ms, 0 | 20260506 19:12:54.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:54.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:54.205 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:12:54.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:54.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:54.205 | INFO | Uploaded 36GiB to GPU in 1.98385s (18.1465GiB/s) | 20260506 19:12:54.205 | INFO | Downloaded 34GiB to GPU in 1.9187s (17.7203GiB/s) | 20260506 19:12:59.348 | INFO | Analyzing chunk frames = [3773952,3774464) | 20260506 19:12:59.348 | INFO | MemInfo: { | 20260506 19:12:59.348 | INFO | "gpu" : " 17 GB", | 20260506 19:12:59.348 | INFO | "huge" : "", | 20260506 19:12:59.348 | INFO | "huge_shared" : " 27 GB", | 20260506 19:12:59.348 | INFO | "malloc" : " 14 GB", | 20260506 19:12:59.348 | INFO | "pinned" : " 29 GB" | 20260506 19:12:59.348 | INFO | } | 20260506 19:12:59.348 | INFO | Perf report for the last 5.14314 seconds: | 20260506 19:12:59.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:12:59.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:59.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:59.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:59.348 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 19:12:59.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:59.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:59.348 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:12:59.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:59.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:12:59.348 | INFO | Uploaded 36GiB to GPU in 1.96211s (18.3476GiB/s) | 20260506 19:12:59.348 | INFO | Downloaded 34GiB to GPU in 1.87106s (18.1715GiB/s) | 20260506 19:13:04.481 | INFO | Analyzing chunk frames = [3774464,3774976) | 20260506 19:13:04.481 | INFO | MemInfo: { | 20260506 19:13:04.481 | INFO | "gpu" : " 17 GB", | 20260506 19:13:04.481 | INFO | "huge" : "", | 20260506 19:13:04.481 | INFO | "huge_shared" : " 27 GB", | 20260506 19:13:04.481 | INFO | "malloc" : " 14 GB", | 20260506 19:13:04.481 | INFO | "pinned" : " 29 GB" | 20260506 19:13:04.481 | INFO | } | 20260506 19:13:04.481 | INFO | Perf report for the last 5.13307 seconds: | 20260506 19:13:04.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:13:04.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:04.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:04.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:04.481 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260506 19:13:04.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:04.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:04.481 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:13:04.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:04.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:04.482 | INFO | Uploaded 36GiB to GPU in 2.00672s (17.9397GiB/s) | 20260506 19:13:04.482 | INFO | Downloaded 34GiB to GPU in 1.91842s (17.7229GiB/s) | 20260506 19:13:09.617 | INFO | Analyzing chunk frames = [3774976,3775488) | 20260506 19:13:09.617 | INFO | MemInfo: { | 20260506 19:13:09.617 | INFO | "gpu" : " 17 GB", | 20260506 19:13:09.617 | INFO | "huge" : "", | 20260506 19:13:09.617 | INFO | "huge_shared" : " 27 GB", | 20260506 19:13:09.617 | INFO | "malloc" : " 14 GB", | 20260506 19:13:09.617 | INFO | "pinned" : " 29 GB" | 20260506 19:13:09.617 | INFO | } | 20260506 19:13:09.617 | INFO | Perf report for the last 5.13618 seconds: | 20260506 19:13:09.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:13:09.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:09.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:09.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:09.618 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 19:13:09.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:09.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:09.618 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:13:09.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:09.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:09.618 | INFO | Uploaded 36GiB to GPU in 2.0049s (17.956GiB/s) | 20260506 19:13:09.618 | INFO | Downloaded 34GiB to GPU in 1.93541s (17.5674GiB/s) | 20260506 19:13:14.757 | INFO | Analyzing chunk frames = [3775488,3776000) | 20260506 19:13:14.757 | INFO | MemInfo: { | 20260506 19:13:14.757 | INFO | "gpu" : " 17 GB", | 20260506 19:13:14.757 | INFO | "huge" : "", | 20260506 19:13:14.757 | INFO | "huge_shared" : " 27 GB", | 20260506 19:13:14.757 | INFO | "malloc" : " 14 GB", | 20260506 19:13:14.757 | INFO | "pinned" : " 29 GB" | 20260506 19:13:14.757 | INFO | } | 20260506 19:13:14.757 | INFO | Perf report for the last 5.13967 seconds: | 20260506 19:13:14.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:13:14.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:14.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:14.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:14.757 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 19:13:14.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:14.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:14.757 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 19:13:14.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:14.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:14.757 | INFO | Uploaded 36GiB to GPU in 2.01252s (17.888GiB/s) | 20260506 19:13:14.757 | INFO | Downloaded 34GiB to GPU in 2.01047s (16.9114GiB/s) | 20260506 19:13:19.897 | INFO | Analyzing chunk frames = [3776000,3776512) | 20260506 19:13:19.897 | INFO | MemInfo: { | 20260506 19:13:19.897 | INFO | "gpu" : " 17 GB", | 20260506 19:13:19.897 | INFO | "huge" : "", | 20260506 19:13:19.897 | INFO | "huge_shared" : " 27 GB", | 20260506 19:13:19.897 | INFO | "malloc" : " 14 GB", | 20260506 19:13:19.897 | INFO | "pinned" : " 29 GB" | 20260506 19:13:19.897 | INFO | } | 20260506 19:13:19.897 | INFO | Perf report for the last 5.13965 seconds: | 20260506 19:13:19.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:13:19.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:19.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:19.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:19.897 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:13:19.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:19.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:19.897 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:13:19.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:19.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:19.897 | INFO | Uploaded 36GiB to GPU in 1.98996s (18.0908GiB/s) | 20260506 19:13:19.897 | INFO | Downloaded 34GiB to GPU in 1.98405s (17.1367GiB/s) | 20260506 19:13:19.897 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:13:19.897 | INFO | Total mem usage: 18 GiB | 20260506 19:13:19.897 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:13:19.897 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -16.6%) | 20260506 19:13:19.897 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:13:19.897 | INFO | High Water Breakdown: | 20260506 19:13:19.897 | INFO | 61.2% : 9.45 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:13:19.897 | INFO | 38.6% : 5.97 GiB ( 89.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:13:19.897 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:13:19.897 | INFO | ------------------------------------------ | 20260506 19:13:19.897 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:13:19.897 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:13:19.897 | INFO | Total mem usage: 26.5 GiB | 20260506 19:13:19.897 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:13:19.897 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:13:19.897 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:13:19.897 | INFO | High Water Breakdown: | 20260506 19:13:19.898 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:13:19.898 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:13:19.898 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:13:19.898 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:13:19.898 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:13:19.898 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:13:19.898 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:13:19.898 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:13:19.898 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:13:19.898 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:13:19.898 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:13:19.898 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:13:19.898 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:13:19.898 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:13:19.898 | INFO | ------------------------------------------ | 20260506 19:13:19.898 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:13:19.898 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:13:19.898 | INFO | Total mem usage: 25.6 GiB | 20260506 19:13:19.898 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:13:19.898 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:13:19.898 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:13:19.898 | INFO | High Water Breakdown: | 20260506 19:13:19.898 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:13:19.898 | INFO | ------------------------------------------ | 20260506 19:13:19.898 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:13:19.898 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:13:19.898 | INFO | Total mem usage: 15.7 GiB | 20260506 19:13:19.898 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:13:19.898 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:13:19.898 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 19:13:19.898 | INFO | High Water Breakdown: | 20260506 19:13:19.898 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:13:19.898 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:13:19.898 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:13:19.898 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:13:19.898 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:13:19.898 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:13:19.898 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:13:19.898 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:13:19.898 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:13:19.898 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:13:19.898 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:13:19.898 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:13:19.898 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:13:19.898 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:13:19.898 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:13:19.898 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:13:19.898 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:13:19.898 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:13:19.898 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:13:19.898 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:13:19.898 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:13:19.898 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:13:19.898 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:13:19.898 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:13:19.898 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:13:19.898 | INFO | ------------------------------------------ | 20260506 19:13:19.898 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:13:19.898 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:13:19.898 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:13:19.898 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:13:19.898 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.61e+01, 1.36e+01 / 2.17e+00 | 20260506 19:13:19.898 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 9.86e-03 / 6.20e-03 | 20260506 19:13:19.898 | INFO | PerfReport: 11.61%, Baseline -- 3.62e+00 / 4.46e+00, 5.48e-01 / 3.05e-01 | 20260506 19:13:19.898 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.48e+00, 1.84e-01 / 1.32e-01 | 20260506 19:13:19.898 | INFO | PerfReport: 16.02%, DME -- 4.00e+01 / 4.49e+01, 4.06e+00 / 2.33e+00 | 20260506 19:13:19.898 | INFO | PerfReport: 57.03%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.32e+00 / 1.53e+00 | 20260506 19:13:19.898 | INFO | PerfReport: 7.24%, PulseAccumulating -- 2.26e+00 / 2.86e+00, 3.92e-01 / 2.06e-01 | 20260506 19:13:19.898 | INFO | PerfReport: 3.38%, Metrics -- 1.05e+00 / 1.41e+00, 2.75e-01 / 1.68e-01 | 20260506 19:13:19.898 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.62e-01 / 4.14e-01, 5.57e-02 / 2.78e-02 | 20260506 19:13:19.899 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:13:19.899 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:13:19.899 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:13:19.899 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 19:13:19.899 | INFO | PerfReport: 21.45%, Upload -- 2.08e+01 / 0.00e+00, 6.91e+00 / 0.00e+00 | 20260506 19:13:19.899 | INFO | PerfReport: 21.23%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 19:13:19.899 | INFO | PerfReport: 32.18%, Compute -- 3.12e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 19:13:19.899 | INFO | PerfReport: 1.35%, LockUpload -- 1.31e+00 / 0.00e+00, 5.37e+00 / 0.00e+00 | 20260506 19:13:19.899 | INFO | PerfReport: 1.20%, LockDownload -- 1.17e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260506 19:13:19.899 | INFO | PerfReport: 22.57%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 19:13:19.899 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260506 19:13:20.010 | INFO | Starting a round of Baz IO | 20260506 19:13:23.141 | INFO | Populating RTMetrics report | 20260506 19:13:23.142 | INFO | RTMetrics report populated | 20260506 19:13:23.142 | INFO | Opening tmp RTMetrics json file | 20260506 19:13:23.144 | INFO | Serializing RTMetrics to json | 20260506 19:13:23.149 | INFO | Serialization done, swapping tmp file now... | 20260506 19:13:23.153 | INFO | RTMetrics done | 20260506 19:13:23.403 | INFO | Finished a round of Baz IO: Wrote 2.018 GiB in 3.392 seconds | 20260506 19:13:23.403 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.72% / 0.01% / 65.28% | 20260506 19:13:23.403 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.957123% / 1.58946e-05% / 5.16574e-05% | 20260506 19:13:23.403 | INFO | Baz IO overall: | 20260506 19:13:23.403 | INFO | BytesWritten : 1995786878674 | 20260506 19:13:23.403 | INFO | NumEvents : 964856220658 | 20260506 19:13:25.039 | INFO | Analyzing chunk frames = [3776512,3777024) | 20260506 19:13:25.039 | INFO | MemInfo: { | 20260506 19:13:25.039 | INFO | "gpu" : " 17 GB", | 20260506 19:13:25.039 | INFO | "huge" : "", | 20260506 19:13:25.039 | INFO | "huge_shared" : " 27 GB", | 20260506 19:13:25.039 | INFO | "malloc" : " 14 GB", | 20260506 19:13:25.039 | INFO | "pinned" : " 29 GB" | 20260506 19:13:25.039 | INFO | } | 20260506 19:13:25.039 | INFO | Perf report for the last 5.14221 seconds: | 20260506 19:13:25.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:13:25.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:25.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:25.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:25.039 | INFO | ANALYSIS : 62.5%, 2.9, 95.5ms, 0 | 20260506 19:13:25.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:25.039 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 19:13:25.039 | INFO | PRE_HQ : 59.8%, 1.8, 57.4ms, 0 | 20260506 19:13:25.039 | INFO | BAZWRITER : 65.7%, 4.8, 161.2ms, 0 | 20260506 19:13:25.039 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 19:13:25.039 | INFO | Uploaded 36GiB to GPU in 2.02069s (17.8157GiB/s) | 20260506 19:13:25.039 | INFO | Downloaded 38GiB to GPU in 2.21049s (17.1908GiB/s) | 20260506 19:13:30.187 | INFO | Analyzing chunk frames = [3777024,3777536) | 20260506 19:13:30.187 | INFO | MemInfo: { | 20260506 19:13:30.187 | INFO | "gpu" : " 17 GB", | 20260506 19:13:30.187 | INFO | "huge" : "", | 20260506 19:13:30.187 | INFO | "huge_shared" : " 27 GB", | 20260506 19:13:30.187 | INFO | "malloc" : " 14 GB", | 20260506 19:13:30.187 | INFO | "pinned" : " 29 GB" | 20260506 19:13:30.187 | INFO | } | 20260506 19:13:30.187 | INFO | Perf report for the last 5.14781 seconds: | 20260506 19:13:30.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:13:30.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:30.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:30.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:30.187 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 19:13:30.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:30.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:30.187 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:13:30.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:30.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:30.187 | INFO | Uploaded 36GiB to GPU in 1.99282s (18.0649GiB/s) | 20260506 19:13:30.187 | INFO | Downloaded 34GiB to GPU in 1.92224s (17.6877GiB/s) | 20260506 19:13:35.321 | INFO | Analyzing chunk frames = [3777536,3778048) | 20260506 19:13:35.321 | INFO | MemInfo: { | 20260506 19:13:35.321 | INFO | "gpu" : " 17 GB", | 20260506 19:13:35.321 | INFO | "huge" : "", | 20260506 19:13:35.321 | INFO | "huge_shared" : " 27 GB", | 20260506 19:13:35.321 | INFO | "malloc" : " 14 GB", | 20260506 19:13:35.321 | INFO | "pinned" : " 29 GB" | 20260506 19:13:35.321 | INFO | } | 20260506 19:13:35.321 | INFO | Perf report for the last 5.13457 seconds: | 20260506 19:13:35.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:13:35.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:35.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:35.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:35.322 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:13:35.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:35.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:35.322 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:13:35.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:35.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:35.322 | INFO | Uploaded 36GiB to GPU in 1.97697s (18.2097GiB/s) | 20260506 19:13:35.322 | INFO | Downloaded 34GiB to GPU in 1.92789s (17.6359GiB/s) | 20260506 19:13:40.458 | INFO | Analyzing chunk frames = [3778048,3778560) | 20260506 19:13:40.459 | INFO | MemInfo: { | 20260506 19:13:40.459 | INFO | "gpu" : " 17 GB", | 20260506 19:13:40.459 | INFO | "huge" : "", | 20260506 19:13:40.459 | INFO | "huge_shared" : " 27 GB", | 20260506 19:13:40.459 | INFO | "malloc" : " 14 GB", | 20260506 19:13:40.459 | INFO | "pinned" : " 29 GB" | 20260506 19:13:40.459 | INFO | } | 20260506 19:13:40.459 | INFO | Perf report for the last 5.13736 seconds: | 20260506 19:13:40.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:13:40.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:40.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:40.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:40.459 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:13:40.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:40.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:40.459 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:13:40.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:40.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:40.459 | INFO | Uploaded 36GiB to GPU in 1.94518s (18.5072GiB/s) | 20260506 19:13:40.459 | INFO | Downloaded 34GiB to GPU in 1.87291s (18.1535GiB/s) | 20260506 19:13:45.596 | INFO | Analyzing chunk frames = [3778560,3779072) | 20260506 19:13:45.596 | INFO | MemInfo: { | 20260506 19:13:45.596 | INFO | "gpu" : " 17 GB", | 20260506 19:13:45.596 | INFO | "huge" : "", | 20260506 19:13:45.596 | INFO | "huge_shared" : " 27 GB", | 20260506 19:13:45.596 | INFO | "malloc" : " 14 GB", | 20260506 19:13:45.596 | INFO | "pinned" : " 29 GB" | 20260506 19:13:45.596 | INFO | } | 20260506 19:13:45.596 | INFO | Perf report for the last 5.13719 seconds: | 20260506 19:13:45.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:13:45.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:45.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:45.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:45.596 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 19:13:45.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:45.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:45.596 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:13:45.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:45.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:45.596 | INFO | Uploaded 36GiB to GPU in 2.01755s (17.8434GiB/s) | 20260506 19:13:45.597 | INFO | Downloaded 34GiB to GPU in 1.9153s (17.7518GiB/s) | 20260506 19:13:50.729 | INFO | Analyzing chunk frames = [3779072,3779584) | 20260506 19:13:50.729 | INFO | MemInfo: { | 20260506 19:13:50.729 | INFO | "gpu" : " 17 GB", | 20260506 19:13:50.729 | INFO | "huge" : "", | 20260506 19:13:50.729 | INFO | "huge_shared" : " 27 GB", | 20260506 19:13:50.729 | INFO | "malloc" : " 14 GB", | 20260506 19:13:50.729 | INFO | "pinned" : " 29 GB" | 20260506 19:13:50.729 | INFO | } | 20260506 19:13:50.730 | INFO | Perf report for the last 5.13351 seconds: | 20260506 19:13:50.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:13:50.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:50.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:50.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:50.730 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 19:13:50.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:50.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:50.730 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 19:13:50.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:50.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:50.730 | INFO | Uploaded 36GiB to GPU in 2.02024s (17.8197GiB/s) | 20260506 19:13:50.730 | INFO | Downloaded 34GiB to GPU in 1.94656s (17.4668GiB/s) | 20260506 19:13:55.873 | INFO | Analyzing chunk frames = [3779584,3780096) | 20260506 19:13:55.874 | INFO | MemInfo: { | 20260506 19:13:55.874 | INFO | "gpu" : " 17 GB", | 20260506 19:13:55.874 | INFO | "huge" : "", | 20260506 19:13:55.874 | INFO | "huge_shared" : " 27 GB", | 20260506 19:13:55.874 | INFO | "malloc" : " 14 GB", | 20260506 19:13:55.874 | INFO | "pinned" : " 29 GB" | 20260506 19:13:55.874 | INFO | } | 20260506 19:13:55.874 | INFO | Perf report for the last 5.14437 seconds: | 20260506 19:13:55.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:13:55.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:55.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:55.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:55.874 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 19:13:55.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:55.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:55.874 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 19:13:55.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:55.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:13:55.874 | INFO | Uploaded 36GiB to GPU in 1.9878s (18.1105GiB/s) | 20260506 19:13:55.874 | INFO | Downloaded 34GiB to GPU in 2.01528s (16.8711GiB/s) | 20260506 19:14:01.023 | INFO | Analyzing chunk frames = [3780096,3780608) | 20260506 19:14:01.023 | INFO | MemInfo: { | 20260506 19:14:01.023 | INFO | "gpu" : " 17 GB", | 20260506 19:14:01.023 | INFO | "huge" : "", | 20260506 19:14:01.023 | INFO | "huge_shared" : " 27 GB", | 20260506 19:14:01.023 | INFO | "malloc" : " 14 GB", | 20260506 19:14:01.023 | INFO | "pinned" : " 29 GB" | 20260506 19:14:01.023 | INFO | } | 20260506 19:14:01.023 | INFO | Perf report for the last 5.14957 seconds: | 20260506 19:14:01.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:14:01.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:01.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:01.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:01.024 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 19:14:01.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:01.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:01.024 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260506 19:14:01.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:01.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:01.024 | INFO | Uploaded 36GiB to GPU in 2.04676s (17.5888GiB/s) | 20260506 19:14:01.024 | INFO | Downloaded 34GiB to GPU in 1.9781s (17.1882GiB/s) | 20260506 19:14:01.140 | INFO | Starting a round of Baz IO | 20260506 19:14:04.273 | INFO | Populating RTMetrics report | 20260506 19:14:04.274 | INFO | RTMetrics report populated | 20260506 19:14:04.274 | INFO | Opening tmp RTMetrics json file | 20260506 19:14:04.275 | INFO | Serializing RTMetrics to json | 20260506 19:14:04.277 | INFO | Serialization done, swapping tmp file now... | 20260506 19:14:04.277 | INFO | RTMetrics done | 20260506 19:14:04.572 | INFO | Finished a round of Baz IO: Wrote 2.015 GiB in 3.431 seconds | 20260506 19:14:04.572 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.71% / 0.00% / 65.28% | 20260506 19:14:04.572 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.952776% / 1.19209e-05% / 5.5631e-05% | 20260506 19:14:04.572 | INFO | Baz IO overall: | 20260506 19:14:04.572 | INFO | BytesWritten : 1997950551508 | 20260506 19:14:04.572 | INFO | NumEvents : 965896130143 | 20260506 19:14:06.153 | INFO | Analyzing chunk frames = [3780608,3781120) | 20260506 19:14:06.153 | INFO | MemInfo: { | 20260506 19:14:06.153 | INFO | "gpu" : " 17 GB", | 20260506 19:14:06.153 | INFO | "huge" : "", | 20260506 19:14:06.153 | INFO | "huge_shared" : " 27 GB", | 20260506 19:14:06.153 | INFO | "malloc" : " 14 GB", | 20260506 19:14:06.153 | INFO | "pinned" : " 29 GB" | 20260506 19:14:06.153 | INFO | } | 20260506 19:14:06.153 | INFO | Perf report for the last 5.12996 seconds: | 20260506 19:14:06.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:14:06.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:06.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:06.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:06.154 | INFO | ANALYSIS : 62.8%, 2.9, 95.5ms, 0 | 20260506 19:14:06.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:06.154 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 19:14:06.154 | INFO | PRE_HQ : 60.5%, 1.8, 57.8ms, 0 | 20260506 19:14:06.154 | INFO | BAZWRITER : 66.5%, 4.8, 164.0ms, 0 | 20260506 19:14:06.154 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260506 19:14:06.154 | INFO | Uploaded 36GiB to GPU in 2.06245s (17.455GiB/s) | 20260506 19:14:06.154 | INFO | Downloaded 38GiB to GPU in 2.21024s (17.1927GiB/s) | 20260506 19:14:11.294 | INFO | Analyzing chunk frames = [3781120,3781632) | 20260506 19:14:11.294 | INFO | MemInfo: { | 20260506 19:14:11.294 | INFO | "gpu" : " 17 GB", | 20260506 19:14:11.294 | INFO | "huge" : "", | 20260506 19:14:11.294 | INFO | "huge_shared" : " 27 GB", | 20260506 19:14:11.294 | INFO | "malloc" : " 14 GB", | 20260506 19:14:11.294 | INFO | "pinned" : " 29 GB" | 20260506 19:14:11.294 | INFO | } | 20260506 19:14:11.294 | INFO | Perf report for the last 5.14097 seconds: | 20260506 19:14:11.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:14:11.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:11.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:11.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:11.295 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 19:14:11.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:11.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:11.295 | INFO | PRE_HQ : 26.5%, 1.0, 13.9ms, 0 | 20260506 19:14:11.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:11.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:11.295 | INFO | Uploaded 36GiB to GPU in 2.021s (17.813GiB/s) | 20260506 19:14:11.295 | INFO | Downloaded 34GiB to GPU in 1.93253s (17.5935GiB/s) | 20260506 19:14:16.436 | INFO | Analyzing chunk frames = [3781632,3782144) | 20260506 19:14:16.436 | INFO | MemInfo: { | 20260506 19:14:16.436 | INFO | "gpu" : " 17 GB", | 20260506 19:14:16.436 | INFO | "huge" : "", | 20260506 19:14:16.436 | INFO | "huge_shared" : " 27 GB", | 20260506 19:14:16.437 | INFO | "malloc" : " 14 GB", | 20260506 19:14:16.437 | INFO | "pinned" : " 29 GB" | 20260506 19:14:16.437 | INFO | } | 20260506 19:14:16.437 | INFO | Perf report for the last 5.14219 seconds: | 20260506 19:14:16.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:14:16.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:16.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:16.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:16.437 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 19:14:16.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:16.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:16.437 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 19:14:16.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:16.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:16.437 | INFO | Uploaded 36GiB to GPU in 1.9767s (18.2121GiB/s) | 20260506 19:14:16.437 | INFO | Downloaded 34GiB to GPU in 1.92052s (17.7036GiB/s) | 20260506 19:14:21.571 | INFO | Analyzing chunk frames = [3782144,3782656) | 20260506 19:14:21.571 | INFO | MemInfo: { | 20260506 19:14:21.571 | INFO | "gpu" : " 17 GB", | 20260506 19:14:21.571 | INFO | "huge" : "", | 20260506 19:14:21.572 | INFO | "huge_shared" : " 27 GB", | 20260506 19:14:21.572 | INFO | "malloc" : " 14 GB", | 20260506 19:14:21.572 | INFO | "pinned" : " 29 GB" | 20260506 19:14:21.572 | INFO | } | 20260506 19:14:21.572 | INFO | Perf report for the last 5.13505 seconds: | 20260506 19:14:21.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:14:21.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:21.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:21.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:21.572 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 19:14:21.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:21.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:21.572 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:14:21.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:21.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:21.572 | INFO | Uploaded 36GiB to GPU in 1.96265s (18.3426GiB/s) | 20260506 19:14:21.572 | INFO | Downloaded 34GiB to GPU in 1.87227s (18.1597GiB/s) | 20260506 19:14:26.708 | INFO | Analyzing chunk frames = [3782656,3783168) | 20260506 19:14:26.708 | INFO | MemInfo: { | 20260506 19:14:26.708 | INFO | "gpu" : " 17 GB", | 20260506 19:14:26.708 | INFO | "huge" : "", | 20260506 19:14:26.709 | INFO | "huge_shared" : " 27 GB", | 20260506 19:14:26.709 | INFO | "malloc" : " 14 GB", | 20260506 19:14:26.709 | INFO | "pinned" : " 29 GB" | 20260506 19:14:26.709 | INFO | } | 20260506 19:14:26.709 | INFO | Perf report for the last 5.13701 seconds: | 20260506 19:14:26.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:14:26.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:26.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:26.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:26.709 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 19:14:26.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:26.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:26.709 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:14:26.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:26.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:26.709 | INFO | Uploaded 36GiB to GPU in 2.01098s (17.9017GiB/s) | 20260506 19:14:26.709 | INFO | Downloaded 34GiB to GPU in 1.91101s (17.7916GiB/s) | 20260506 19:14:31.847 | INFO | Analyzing chunk frames = [3783168,3783680) | 20260506 19:14:31.847 | INFO | MemInfo: { | 20260506 19:14:31.847 | INFO | "gpu" : " 17 GB", | 20260506 19:14:31.847 | INFO | "huge" : "", | 20260506 19:14:31.847 | INFO | "huge_shared" : " 27 GB", | 20260506 19:14:31.847 | INFO | "malloc" : " 14 GB", | 20260506 19:14:31.847 | INFO | "pinned" : " 29 GB" | 20260506 19:14:31.847 | INFO | } | 20260506 19:14:31.848 | INFO | Perf report for the last 5.13878 seconds: | 20260506 19:14:31.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:14:31.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:31.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:31.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:31.848 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 19:14:31.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:31.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:31.848 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 19:14:31.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:31.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:31.848 | INFO | Uploaded 36GiB to GPU in 2.01969s (17.8245GiB/s) | 20260506 19:14:31.848 | INFO | Downloaded 34GiB to GPU in 1.94042s (17.522GiB/s) | 20260506 19:14:36.993 | INFO | Analyzing chunk frames = [3783680,3784192) | 20260506 19:14:36.993 | INFO | MemInfo: { | 20260506 19:14:36.993 | INFO | "gpu" : " 17 GB", | 20260506 19:14:36.993 | INFO | "huge" : "", | 20260506 19:14:36.993 | INFO | "huge_shared" : " 27 GB", | 20260506 19:14:36.993 | INFO | "malloc" : " 14 GB", | 20260506 19:14:36.993 | INFO | "pinned" : " 29 GB" | 20260506 19:14:36.993 | INFO | } | 20260506 19:14:36.993 | INFO | Perf report for the last 5.14567 seconds: | 20260506 19:14:36.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:14:36.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:36.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:36.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:36.993 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 19:14:36.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:36.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:36.993 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:14:36.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:36.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:36.994 | INFO | Uploaded 36GiB to GPU in 2.00018s (17.9984GiB/s) | 20260506 19:14:36.994 | INFO | Downloaded 34GiB to GPU in 2.00405s (16.9657GiB/s) | 20260506 19:14:42.132 | INFO | Analyzing chunk frames = [3784192,3784704) | 20260506 19:14:42.132 | INFO | MemInfo: { | 20260506 19:14:42.132 | INFO | "gpu" : " 17 GB", | 20260506 19:14:42.132 | INFO | "huge" : "", | 20260506 19:14:42.132 | INFO | "huge_shared" : " 27 GB", | 20260506 19:14:42.132 | INFO | "malloc" : " 14 GB", | 20260506 19:14:42.132 | INFO | "pinned" : " 29 GB" | 20260506 19:14:42.132 | INFO | } | 20260506 19:14:42.132 | INFO | Perf report for the last 5.13917 seconds: | 20260506 19:14:42.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:14:42.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:42.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:42.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:42.133 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 19:14:42.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:42.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:42.133 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:14:42.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:42.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:42.133 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:14:42.133 | INFO | Total mem usage: 18 GiB | 20260506 19:14:42.133 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:14:42.133 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.5%) | 20260506 19:14:42.133 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:14:42.133 | INFO | Uploaded 36GiB to GPU in 1.98021s (18.1799GiB/s) | 20260506 19:14:42.133 | INFO | High Water Breakdown: | 20260506 19:14:42.133 | INFO | 61.4% : 9.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:14:42.133 | INFO | 38.5% : 6 GiB ( 89.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:14:42.133 | INFO | Downloaded 34GiB to GPU in 1.99037s (17.0822GiB/s) | 20260506 19:14:42.133 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:14:42.133 | INFO | ------------------------------------------ | 20260506 19:14:42.133 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:14:42.133 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:14:42.133 | INFO | Total mem usage: 26.5 GiB | 20260506 19:14:42.133 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:14:42.133 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:14:42.133 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:14:42.133 | INFO | High Water Breakdown: | 20260506 19:14:42.133 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:14:42.133 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:14:42.133 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:14:42.133 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:14:42.133 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:14:42.133 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:14:42.133 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:14:42.133 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:14:42.133 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:14:42.133 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:14:42.133 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:14:42.133 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:14:42.133 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:14:42.133 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:14:42.133 | INFO | ------------------------------------------ | 20260506 19:14:42.133 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:14:42.133 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:14:42.133 | INFO | Total mem usage: 25.6 GiB | 20260506 19:14:42.133 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:14:42.133 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:14:42.133 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:14:42.133 | INFO | High Water Breakdown: | 20260506 19:14:42.133 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:14:42.133 | INFO | ------------------------------------------ | 20260506 19:14:42.133 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:14:42.133 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:14:42.133 | INFO | Total mem usage: 15.7 GiB | 20260506 19:14:42.133 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:14:42.134 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:14:42.134 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 19:14:42.134 | INFO | High Water Breakdown: | 20260506 19:14:42.134 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:14:42.134 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:14:42.134 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:14:42.134 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:14:42.134 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:14:42.134 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:14:42.134 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:14:42.134 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:14:42.134 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:14:42.134 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:14:42.134 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:14:42.134 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:14:42.134 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:14:42.134 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:14:42.134 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:14:42.134 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:14:42.134 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:14:42.134 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:14:42.134 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:14:42.134 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:14:42.134 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:14:42.134 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:14:42.134 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:14:42.134 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:14:42.134 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:14:42.134 | INFO | ------------------------------------------ | 20260506 19:14:42.134 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:14:42.134 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:14:42.134 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:14:42.134 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:14:42.134 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.62e+01, 1.36e+01 / 2.67e+00 | 20260506 19:14:42.134 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.02e-01, 9.92e-03 / 4.72e-03 | 20260506 19:14:42.134 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.46e+00, 5.59e-01 / 3.09e-01 | 20260506 19:14:42.134 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.48e+00, 1.86e-01 / 1.32e-01 | 20260506 19:14:42.134 | INFO | PerfReport: 16.04%, DME -- 4.01e+01 / 4.50e+01, 4.33e+00 / 2.61e+00 | 20260506 19:14:42.134 | INFO | PerfReport: 57.06%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.40e+00 / 1.75e+00 | 20260506 19:14:42.134 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.25e+00 / 2.80e+00, 3.82e-01 / 2.10e-01 | 20260506 19:14:42.134 | INFO | PerfReport: 3.37%, Metrics -- 1.05e+00 / 1.39e+00, 2.74e-01 / 1.73e-01 | 20260506 19:14:42.134 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.62e-01 / 4.11e-01, 5.66e-02 / 2.74e-02 | 20260506 19:14:42.134 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:14:42.134 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:14:42.134 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:14:42.134 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 19:14:42.134 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260506 19:14:42.134 | INFO | PerfReport: 21.18%, Download -- 2.06e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 19:14:42.134 | INFO | PerfReport: 32.20%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 19:14:42.134 | INFO | PerfReport: 1.38%, LockUpload -- 1.34e+00 / 0.00e+00, 5.36e+00 / 0.00e+00 | 20260506 19:14:42.134 | INFO | PerfReport: 1.18%, LockDownload -- 1.14e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 19:14:42.134 | INFO | PerfReport: 22.54%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 19:14:42.134 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260506 19:14:42.240 | INFO | Starting a round of Baz IO | 20260506 19:14:45.371 | INFO | Populating RTMetrics report | 20260506 19:14:45.372 | INFO | RTMetrics report populated | 20260506 19:14:45.372 | INFO | Opening tmp RTMetrics json file | 20260506 19:14:45.373 | INFO | Serializing RTMetrics to json | 20260506 19:14:45.376 | INFO | Serialization done, swapping tmp file now... | 20260506 19:14:45.379 | INFO | RTMetrics done | 20260506 19:14:45.510 | INFO | Finished a round of Baz IO: Wrote 2.017 GiB in 3.27 seconds | 20260506 19:14:45.511 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.71% / 0.00% / 65.29% | 20260506 19:14:45.511 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.952919% / 3.97364e-06% / 7.94729e-05% | 20260506 19:14:45.511 | INFO | Baz IO overall: | 20260506 19:14:45.511 | INFO | BytesWritten : 2000115965412 | 20260506 19:14:45.511 | INFO | NumEvents : 966937566711 | 20260506 19:14:47.277 | INFO | Analyzing chunk frames = [3784704,3785216) | 20260506 19:14:47.277 | INFO | MemInfo: { | 20260506 19:14:47.277 | INFO | "gpu" : " 17 GB", | 20260506 19:14:47.277 | INFO | "huge" : "", | 20260506 19:14:47.277 | INFO | "huge_shared" : " 27 GB", | 20260506 19:14:47.277 | INFO | "malloc" : " 14 GB", | 20260506 19:14:47.277 | INFO | "pinned" : " 29 GB" | 20260506 19:14:47.277 | INFO | } | 20260506 19:14:47.277 | INFO | Perf report for the last 5.14471 seconds: | 20260506 19:14:47.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:14:47.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:47.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:47.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:47.277 | INFO | ANALYSIS : 62.4%, 2.9, 94.7ms, 0 | 20260506 19:14:47.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:47.277 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260506 19:14:47.277 | INFO | PRE_HQ : 60.1%, 1.9, 58.2ms, 0 | 20260506 19:14:47.277 | INFO | BAZWRITER : 63.2%, 4.8, 155.4ms, 0 | 20260506 19:14:47.277 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 19:14:47.277 | INFO | Uploaded 36GiB to GPU in 2.04867s (17.5724GiB/s) | 20260506 19:14:47.277 | INFO | Downloaded 38GiB to GPU in 2.20479s (17.2352GiB/s) | 20260506 19:14:52.409 | INFO | Analyzing chunk frames = [3785216,3785728) | 20260506 19:14:52.409 | INFO | MemInfo: { | 20260506 19:14:52.409 | INFO | "gpu" : " 17 GB", | 20260506 19:14:52.409 | INFO | "huge" : "", | 20260506 19:14:52.409 | INFO | "huge_shared" : " 27 GB", | 20260506 19:14:52.409 | INFO | "malloc" : " 14 GB", | 20260506 19:14:52.409 | INFO | "pinned" : " 29 GB" | 20260506 19:14:52.409 | INFO | } | 20260506 19:14:52.409 | INFO | Perf report for the last 5.13165 seconds: | 20260506 19:14:52.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:14:52.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:52.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:52.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:52.409 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 19:14:52.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:52.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:52.409 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:14:52.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:52.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:52.409 | INFO | Uploaded 36GiB to GPU in 1.99024s (18.0883GiB/s) | 20260506 19:14:52.409 | INFO | Downloaded 34GiB to GPU in 1.9355s (17.5665GiB/s) | 20260506 19:14:57.545 | INFO | Analyzing chunk frames = [3785728,3786240) | 20260506 19:14:57.545 | INFO | MemInfo: { | 20260506 19:14:57.545 | INFO | "gpu" : " 17 GB", | 20260506 19:14:57.545 | INFO | "huge" : "", | 20260506 19:14:57.545 | INFO | "huge_shared" : " 27 GB", | 20260506 19:14:57.545 | INFO | "malloc" : " 14 GB", | 20260506 19:14:57.545 | INFO | "pinned" : " 29 GB" | 20260506 19:14:57.545 | INFO | } | 20260506 19:14:57.545 | INFO | Perf report for the last 5.1364 seconds: | 20260506 19:14:57.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:14:57.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:57.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:57.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:57.545 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 19:14:57.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:57.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:57.545 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:14:57.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:57.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:14:57.545 | INFO | Uploaded 36GiB to GPU in 1.9784s (18.1965GiB/s) | 20260506 19:14:57.545 | INFO | Downloaded 34GiB to GPU in 1.91948s (17.7131GiB/s) | 20260506 19:15:02.683 | INFO | Analyzing chunk frames = [3786240,3786752) | 20260506 19:15:02.683 | INFO | MemInfo: { | 20260506 19:15:02.683 | INFO | "gpu" : " 17 GB", | 20260506 19:15:02.683 | INFO | "huge" : "", | 20260506 19:15:02.683 | INFO | "huge_shared" : " 27 GB", | 20260506 19:15:02.683 | INFO | "malloc" : " 14 GB", | 20260506 19:15:02.683 | INFO | "pinned" : " 29 GB" | 20260506 19:15:02.683 | INFO | } | 20260506 19:15:02.683 | INFO | Perf report for the last 5.13781 seconds: | 20260506 19:15:02.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:15:02.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:02.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:02.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:02.683 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 19:15:02.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:02.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:02.683 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:15:02.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:02.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:02.683 | INFO | Uploaded 36GiB to GPU in 1.93451s (18.6093GiB/s) | 20260506 19:15:02.683 | INFO | Downloaded 34GiB to GPU in 1.87537s (18.1298GiB/s) | 20260506 19:15:07.823 | INFO | Analyzing chunk frames = [3786752,3787264) | 20260506 19:15:07.823 | INFO | MemInfo: { | 20260506 19:15:07.823 | INFO | "gpu" : " 17 GB", | 20260506 19:15:07.823 | INFO | "huge" : "", | 20260506 19:15:07.823 | INFO | "huge_shared" : " 27 GB", | 20260506 19:15:07.823 | INFO | "malloc" : " 14 GB", | 20260506 19:15:07.823 | INFO | "pinned" : " 29 GB" | 20260506 19:15:07.823 | INFO | } | 20260506 19:15:07.823 | INFO | Perf report for the last 5.14019 seconds: | 20260506 19:15:07.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:15:07.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:07.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:07.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:07.823 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 19:15:07.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:07.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:07.823 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:15:07.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:07.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:07.824 | INFO | Uploaded 36GiB to GPU in 1.98851s (18.104GiB/s) | 20260506 19:15:07.824 | INFO | Downloaded 34GiB to GPU in 1.90248s (17.8714GiB/s) | 20260506 19:15:12.965 | INFO | Analyzing chunk frames = [3787264,3787776) | 20260506 19:15:12.965 | INFO | MemInfo: { | 20260506 19:15:12.965 | INFO | "gpu" : " 17 GB", | 20260506 19:15:12.965 | INFO | "huge" : "", | 20260506 19:15:12.965 | INFO | "huge_shared" : " 27 GB", | 20260506 19:15:12.965 | INFO | "malloc" : " 14 GB", | 20260506 19:15:12.965 | INFO | "pinned" : " 29 GB" | 20260506 19:15:12.965 | INFO | } | 20260506 19:15:12.965 | INFO | Perf report for the last 5.14189 seconds: | 20260506 19:15:12.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:15:12.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:12.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:12.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:12.965 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 19:15:12.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:12.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:12.965 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 19:15:12.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:12.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:12.966 | INFO | Uploaded 36GiB to GPU in 2.02646s (17.765GiB/s) | 20260506 19:15:12.966 | INFO | Downloaded 34GiB to GPU in 1.94986s (17.4371GiB/s) | 20260506 19:15:18.104 | INFO | Analyzing chunk frames = [3787776,3788288) | 20260506 19:15:18.104 | INFO | MemInfo: { | 20260506 19:15:18.104 | INFO | "gpu" : " 17 GB", | 20260506 19:15:18.104 | INFO | "huge" : "", | 20260506 19:15:18.104 | INFO | "huge_shared" : " 27 GB", | 20260506 19:15:18.105 | INFO | "malloc" : " 14 GB", | 20260506 19:15:18.105 | INFO | "pinned" : " 29 GB" | 20260506 19:15:18.105 | INFO | } | 20260506 19:15:18.105 | INFO | Perf report for the last 5.13967 seconds: | 20260506 19:15:18.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:15:18.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:18.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:18.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:18.105 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:15:18.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:18.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:18.105 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 19:15:18.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:18.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:18.105 | INFO | Uploaded 36GiB to GPU in 1.97263s (18.2497GiB/s) | 20260506 19:15:18.105 | INFO | Downloaded 34GiB to GPU in 2.00024s (16.998GiB/s) | 20260506 19:15:23.246 | INFO | Analyzing chunk frames = [3788288,3788800) | 20260506 19:15:23.246 | INFO | MemInfo: { | 20260506 19:15:23.246 | INFO | "gpu" : " 17 GB", | 20260506 19:15:23.246 | INFO | "huge" : "", | 20260506 19:15:23.246 | INFO | "huge_shared" : " 27 GB", | 20260506 19:15:23.246 | INFO | "malloc" : " 14 GB", | 20260506 19:15:23.246 | INFO | "pinned" : " 29 GB" | 20260506 19:15:23.246 | INFO | } | 20260506 19:15:23.246 | INFO | Perf report for the last 5.14144 seconds: | 20260506 19:15:23.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:15:23.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:23.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:23.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:23.246 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 19:15:23.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:23.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:23.246 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 19:15:23.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:23.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:23.247 | INFO | Uploaded 36GiB to GPU in 2.01978s (17.8237GiB/s) | 20260506 19:15:23.247 | INFO | Downloaded 34GiB to GPU in 1.97752s (17.1932GiB/s) | 20260506 19:15:23.377 | INFO | Starting a round of Baz IO | 20260506 19:15:26.496 | INFO | Populating RTMetrics report | 20260506 19:15:26.496 | INFO | RTMetrics report populated | 20260506 19:15:26.496 | INFO | Opening tmp RTMetrics json file | 20260506 19:15:26.514 | INFO | Serializing RTMetrics to json | 20260506 19:15:26.520 | INFO | Serialization done, swapping tmp file now... | 20260506 19:15:26.524 | INFO | RTMetrics done | 20260506 19:15:26.780 | INFO | Finished a round of Baz IO: Wrote 2.016 GiB in 3.403 seconds | 20260506 19:15:26.780 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.70% / 0.00% / 65.29% | 20260506 19:15:26.781 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.951374% / 0% / 5.96046e-05% | 20260506 19:15:26.781 | INFO | Baz IO overall: | 20260506 19:15:26.781 | INFO | BytesWritten : 2002280271482 | 20260506 19:15:26.781 | INFO | NumEvents : 967978410465 | 20260506 19:15:28.382 | INFO | Analyzing chunk frames = [3788800,3789312) | 20260506 19:15:28.382 | INFO | MemInfo: { | 20260506 19:15:28.382 | INFO | "gpu" : " 17 GB", | 20260506 19:15:28.382 | INFO | "huge" : "", | 20260506 19:15:28.382 | INFO | "huge_shared" : " 27 GB", | 20260506 19:15:28.382 | INFO | "malloc" : " 14 GB", | 20260506 19:15:28.382 | INFO | "pinned" : " 29 GB" | 20260506 19:15:28.383 | INFO | } | 20260506 19:15:28.383 | INFO | Perf report for the last 5.13635 seconds: | 20260506 19:15:28.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:15:28.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:28.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:28.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:28.383 | INFO | ANALYSIS : 62.7%, 2.9, 95.6ms, 0 | 20260506 19:15:28.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:28.383 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 19:15:28.383 | INFO | PRE_HQ : 60.0%, 1.8, 57.6ms, 0 | 20260506 19:15:28.383 | INFO | BAZWRITER : 66.2%, 4.8, 162.9ms, 0 | 20260506 19:15:28.383 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 19:15:28.383 | INFO | Uploaded 36GiB to GPU in 2.06492s (17.4341GiB/s) | 20260506 19:15:28.383 | INFO | Downloaded 38GiB to GPU in 2.21611s (17.1472GiB/s) | 20260506 19:15:33.522 | INFO | Analyzing chunk frames = [3789312,3789824) | 20260506 19:15:33.522 | INFO | MemInfo: { | 20260506 19:15:33.522 | INFO | "gpu" : " 17 GB", | 20260506 19:15:33.522 | INFO | "huge" : "", | 20260506 19:15:33.522 | INFO | "huge_shared" : " 27 GB", | 20260506 19:15:33.522 | INFO | "malloc" : " 14 GB", | 20260506 19:15:33.522 | INFO | "pinned" : " 29 GB" | 20260506 19:15:33.522 | INFO | } | 20260506 19:15:33.522 | INFO | Perf report for the last 5.13953 seconds: | 20260506 19:15:33.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:15:33.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:33.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:33.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:33.522 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:15:33.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:33.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:33.522 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:15:33.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:33.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:33.523 | INFO | Uploaded 36GiB to GPU in 2.01671s (17.8508GiB/s) | 20260506 19:15:33.523 | INFO | Downloaded 34GiB to GPU in 1.93579s (17.5639GiB/s) | 20260506 19:15:38.660 | INFO | Analyzing chunk frames = [3789824,3790336) | 20260506 19:15:38.660 | INFO | MemInfo: { | 20260506 19:15:38.660 | INFO | "gpu" : " 17 GB", | 20260506 19:15:38.660 | INFO | "huge" : "", | 20260506 19:15:38.660 | INFO | "huge_shared" : " 27 GB", | 20260506 19:15:38.660 | INFO | "malloc" : " 14 GB", | 20260506 19:15:38.660 | INFO | "pinned" : " 29 GB" | 20260506 19:15:38.660 | INFO | } | 20260506 19:15:38.660 | INFO | Perf report for the last 5.13818 seconds: | 20260506 19:15:38.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:15:38.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:38.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:38.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:38.660 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 19:15:38.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:38.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:38.660 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:15:38.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:38.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:38.661 | INFO | Uploaded 36GiB to GPU in 1.9863s (18.1242GiB/s) | 20260506 19:15:38.661 | INFO | Downloaded 34GiB to GPU in 1.91722s (17.734GiB/s) | 20260506 19:15:43.797 | INFO | Analyzing chunk frames = [3790336,3790848) | 20260506 19:15:43.797 | INFO | MemInfo: { | 20260506 19:15:43.797 | INFO | "gpu" : " 17 GB", | 20260506 19:15:43.797 | INFO | "huge" : "", | 20260506 19:15:43.797 | INFO | "huge_shared" : " 27 GB", | 20260506 19:15:43.797 | INFO | "malloc" : " 14 GB", | 20260506 19:15:43.797 | INFO | "pinned" : " 29 GB" | 20260506 19:15:43.797 | INFO | } | 20260506 19:15:43.797 | INFO | Perf report for the last 5.13669 seconds: | 20260506 19:15:43.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:15:43.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:43.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:43.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:43.797 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:15:43.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:43.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:43.797 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:15:43.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:43.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:43.797 | INFO | Uploaded 36GiB to GPU in 1.95158s (18.4466GiB/s) | 20260506 19:15:43.797 | INFO | Downloaded 34GiB to GPU in 1.8724s (18.1585GiB/s) | 20260506 19:15:48.943 | INFO | Analyzing chunk frames = [3790848,3791360) | 20260506 19:15:48.943 | INFO | MemInfo: { | 20260506 19:15:48.943 | INFO | "gpu" : " 17 GB", | 20260506 19:15:48.943 | INFO | "huge" : "", | 20260506 19:15:48.943 | INFO | "huge_shared" : " 27 GB", | 20260506 19:15:48.943 | INFO | "malloc" : " 14 GB", | 20260506 19:15:48.943 | INFO | "pinned" : " 29 GB" | 20260506 19:15:48.943 | INFO | } | 20260506 19:15:48.943 | INFO | Perf report for the last 5.14599 seconds: | 20260506 19:15:48.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:15:48.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:48.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:48.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:48.943 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260506 19:15:48.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:48.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:48.943 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:15:48.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:48.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:48.943 | INFO | Uploaded 36GiB to GPU in 2.02307s (17.7947GiB/s) | 20260506 19:15:48.944 | INFO | Downloaded 34GiB to GPU in 1.91881s (17.7193GiB/s) | 20260506 19:15:54.080 | INFO | Analyzing chunk frames = [3791360,3791872) | 20260506 19:15:54.080 | INFO | MemInfo: { | 20260506 19:15:54.080 | INFO | "gpu" : " 17 GB", | 20260506 19:15:54.080 | INFO | "huge" : "", | 20260506 19:15:54.080 | INFO | "huge_shared" : " 27 GB", | 20260506 19:15:54.081 | INFO | "malloc" : " 14 GB", | 20260506 19:15:54.081 | INFO | "pinned" : " 29 GB" | 20260506 19:15:54.081 | INFO | } | 20260506 19:15:54.081 | INFO | Perf report for the last 5.13755 seconds: | 20260506 19:15:54.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:15:54.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:54.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:54.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:54.081 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 19:15:54.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:54.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:54.081 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 19:15:54.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:54.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:54.081 | INFO | Uploaded 36GiB to GPU in 2.01862s (17.834GiB/s) | 20260506 19:15:54.081 | INFO | Downloaded 34GiB to GPU in 1.94736s (17.4595GiB/s) | 20260506 19:15:59.222 | INFO | Analyzing chunk frames = [3791872,3792384) | 20260506 19:15:59.222 | INFO | MemInfo: { | 20260506 19:15:59.222 | INFO | "gpu" : " 17 GB", | 20260506 19:15:59.222 | INFO | "huge" : "", | 20260506 19:15:59.222 | INFO | "huge_shared" : " 27 GB", | 20260506 19:15:59.222 | INFO | "malloc" : " 14 GB", | 20260506 19:15:59.222 | INFO | "pinned" : " 29 GB" | 20260506 19:15:59.222 | INFO | } | 20260506 19:15:59.222 | INFO | Perf report for the last 5.14166 seconds: | 20260506 19:15:59.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:15:59.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:59.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:59.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:59.222 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 19:15:59.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:59.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:59.222 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 19:15:59.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:59.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:15:59.223 | INFO | Uploaded 36GiB to GPU in 2.01995s (17.8223GiB/s) | 20260506 19:15:59.223 | INFO | Downloaded 34GiB to GPU in 1.98759s (17.1062GiB/s) | 20260506 19:16:04.364 | INFO | Analyzing chunk frames = [3792384,3792896) | 20260506 19:16:04.364 | INFO | MemInfo: { | 20260506 19:16:04.364 | INFO | "gpu" : " 17 GB", | 20260506 19:16:04.364 | INFO | "huge" : "", | 20260506 19:16:04.364 | INFO | "huge_shared" : " 27 GB", | 20260506 19:16:04.364 | INFO | "malloc" : " 14 GB", | 20260506 19:16:04.364 | INFO | "pinned" : " 29 GB" | 20260506 19:16:04.364 | INFO | } | 20260506 19:16:04.364 | INFO | Perf report for the last 5.14171 seconds: | 20260506 19:16:04.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:16:04.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:04.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:04.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:04.364 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:16:04.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:04.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:04.364 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:16:04.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:04.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:04.364 | INFO | Uploaded 36GiB to GPU in 1.97455s (18.232GiB/s) | 20260506 19:16:04.364 | INFO | Downloaded 34GiB to GPU in 1.99843s (17.0134GiB/s) | 20260506 19:16:04.365 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:16:04.365 | INFO | Total mem usage: 18 GiB | 20260506 19:16:04.365 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:16:04.365 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.6%) | 20260506 19:16:04.365 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:16:04.365 | INFO | High Water Breakdown: | 20260506 19:16:04.365 | INFO | 61.4% : 9.56 GiB ( 83.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:16:04.365 | INFO | 38.5% : 5.99 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:16:04.365 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:16:04.365 | INFO | ------------------------------------------ | 20260506 19:16:04.365 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:16:04.365 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:16:04.365 | INFO | Total mem usage: 26.5 GiB | 20260506 19:16:04.365 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:16:04.365 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:16:04.365 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:16:04.365 | INFO | High Water Breakdown: | 20260506 19:16:04.365 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:16:04.365 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:16:04.365 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:16:04.365 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:16:04.365 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:16:04.365 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:16:04.365 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:16:04.365 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:16:04.365 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:16:04.365 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:16:04.365 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:16:04.365 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:16:04.365 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:16:04.365 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:16:04.365 | INFO | ------------------------------------------ | 20260506 19:16:04.365 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:16:04.365 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:16:04.365 | INFO | Total mem usage: 25.6 GiB | 20260506 19:16:04.365 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:16:04.365 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:16:04.365 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:16:04.365 | INFO | High Water Breakdown: | 20260506 19:16:04.365 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:16:04.365 | INFO | ------------------------------------------ | 20260506 19:16:04.365 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:16:04.365 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:16:04.365 | INFO | Total mem usage: 15.7 GiB | 20260506 19:16:04.365 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:16:04.365 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:16:04.365 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:16:04.365 | INFO | High Water Breakdown: | 20260506 19:16:04.365 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:16:04.365 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:16:04.365 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:16:04.365 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:16:04.365 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:16:04.365 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:16:04.365 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:16:04.365 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:16:04.365 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:16:04.365 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:16:04.365 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:16:04.365 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:16:04.365 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:16:04.365 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:16:04.365 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:16:04.365 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:16:04.365 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:16:04.365 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:16:04.365 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:16:04.365 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:16:04.365 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:16:04.365 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:16:04.365 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:16:04.365 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:16:04.365 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:16:04.366 | INFO | ------------------------------------------ | 20260506 19:16:04.366 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:16:04.366 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:16:04.366 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:16:04.366 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:16:04.366 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.56e+01, 1.34e+01 / 2.45e+00 | 20260506 19:16:04.366 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.03e-01, 1.05e-02 / 6.35e-03 | 20260506 19:16:04.366 | INFO | PerfReport: 11.60%, Baseline -- 3.62e+00 / 4.39e+00, 5.49e-01 / 3.80e-01 | 20260506 19:16:04.366 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.46e+00, 1.80e-01 / 1.40e-01 | 20260506 19:16:04.366 | INFO | PerfReport: 15.98%, DME -- 3.99e+01 / 4.44e+01, 4.08e+00 / 2.39e+00 | 20260506 19:16:04.366 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.42e+00 / 1.82e+00 | 20260506 19:16:04.366 | INFO | PerfReport: 7.21%, PulseAccumulating -- 2.25e+00 / 2.84e+00, 3.87e-01 / 2.20e-01 | 20260506 19:16:04.366 | INFO | PerfReport: 3.37%, Metrics -- 1.05e+00 / 1.39e+00, 2.77e-01 / 1.84e-01 | 20260506 19:16:04.366 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.62e-01 / 4.08e-01, 5.67e-02 / 2.43e-02 | 20260506 19:16:04.366 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:16:04.366 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:16:04.366 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:16:04.366 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 19:16:04.366 | INFO | PerfReport: 21.50%, Upload -- 2.08e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 19:16:04.366 | INFO | PerfReport: 21.20%, Download -- 2.05e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 19:16:04.366 | INFO | PerfReport: 32.21%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 19:16:04.366 | INFO | PerfReport: 1.37%, LockUpload -- 1.33e+00 / 0.00e+00, 5.31e+00 / 0.00e+00 | 20260506 19:16:04.366 | INFO | PerfReport: 1.14%, LockDownload -- 1.10e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260506 19:16:04.366 | INFO | PerfReport: 22.56%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 19:16:04.366 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260506 19:16:04.471 | INFO | Starting a round of Baz IO | 20260506 19:16:07.605 | INFO | Populating RTMetrics report | 20260506 19:16:07.605 | INFO | RTMetrics report populated | 20260506 19:16:07.605 | INFO | Opening tmp RTMetrics json file | 20260506 19:16:07.692 | INFO | Serializing RTMetrics to json | 20260506 19:16:07.743 | INFO | Serialization done, swapping tmp file now... | 20260506 19:16:07.748 | INFO | RTMetrics done | 20260506 19:16:07.842 | INFO | Finished a round of Baz IO: Wrote 2.014 GiB in 3.371 seconds | 20260506 19:16:07.843 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.70% / 0.00% / 65.30% | 20260506 19:16:07.843 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.947881% / 7.94729e-06% / 5.5631e-05% | 20260506 19:16:07.843 | INFO | Baz IO overall: | 20260506 19:16:07.843 | INFO | BytesWritten : 2004442966010 | 20260506 19:16:07.843 | INFO | NumEvents : 969018007288 | 20260506 19:16:09.498 | INFO | Analyzing chunk frames = [3792896,3793408) | 20260506 19:16:09.498 | INFO | MemInfo: { | 20260506 19:16:09.498 | INFO | "gpu" : " 17 GB", | 20260506 19:16:09.498 | INFO | "huge" : "", | 20260506 19:16:09.498 | INFO | "huge_shared" : " 27 GB", | 20260506 19:16:09.498 | INFO | "malloc" : " 14 GB", | 20260506 19:16:09.498 | INFO | "pinned" : " 29 GB" | 20260506 19:16:09.498 | INFO | } | 20260506 19:16:09.498 | INFO | Perf report for the last 5.13383 seconds: | 20260506 19:16:09.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:16:09.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:09.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:09.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:09.498 | INFO | ANALYSIS : 62.5%, 2.9, 94.9ms, 0 | 20260506 19:16:09.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:09.498 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260506 19:16:09.498 | INFO | PRE_HQ : 60.3%, 1.9, 57.9ms, 0 | 20260506 19:16:09.498 | INFO | BAZWRITER : 65.4%, 4.8, 160.4ms, 0 | 20260506 19:16:09.498 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 19:16:09.498 | INFO | Uploaded 36GiB to GPU in 2.01041s (17.9068GiB/s) | 20260506 19:16:09.498 | INFO | Downloaded 38GiB to GPU in 2.20302s (17.249GiB/s) | 20260506 19:16:14.638 | INFO | Analyzing chunk frames = [3793408,3793920) | 20260506 19:16:14.638 | INFO | MemInfo: { | 20260506 19:16:14.638 | INFO | "gpu" : " 17 GB", | 20260506 19:16:14.638 | INFO | "huge" : "", | 20260506 19:16:14.638 | INFO | "huge_shared" : " 27 GB", | 20260506 19:16:14.638 | INFO | "malloc" : " 14 GB", | 20260506 19:16:14.638 | INFO | "pinned" : " 29 GB" | 20260506 19:16:14.638 | INFO | } | 20260506 19:16:14.638 | INFO | Perf report for the last 5.14057 seconds: | 20260506 19:16:14.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:16:14.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:14.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:14.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:14.638 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 19:16:14.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:14.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:14.638 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:16:14.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:14.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:14.639 | INFO | Uploaded 36GiB to GPU in 2.00182s (17.9836GiB/s) | 20260506 19:16:14.639 | INFO | Downloaded 34GiB to GPU in 1.9262s (17.6513GiB/s) | 20260506 19:16:19.776 | INFO | Analyzing chunk frames = [3793920,3794432) | 20260506 19:16:19.776 | INFO | MemInfo: { | 20260506 19:16:19.776 | INFO | "gpu" : " 17 GB", | 20260506 19:16:19.776 | INFO | "huge" : "", | 20260506 19:16:19.776 | INFO | "huge_shared" : " 27 GB", | 20260506 19:16:19.776 | INFO | "malloc" : " 14 GB", | 20260506 19:16:19.776 | INFO | "pinned" : " 29 GB" | 20260506 19:16:19.776 | INFO | } | 20260506 19:16:19.776 | INFO | Perf report for the last 5.1377 seconds: | 20260506 19:16:19.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:16:19.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:19.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:19.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:19.776 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 19:16:19.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:19.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:19.776 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:16:19.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:19.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:19.776 | INFO | Uploaded 36GiB to GPU in 1.96931s (18.2805GiB/s) | 20260506 19:16:19.777 | INFO | Downloaded 34GiB to GPU in 1.92056s (17.7032GiB/s) | 20260506 19:16:24.911 | INFO | Analyzing chunk frames = [3794432,3794944) | 20260506 19:16:24.911 | INFO | MemInfo: { | 20260506 19:16:24.911 | INFO | "gpu" : " 17 GB", | 20260506 19:16:24.911 | INFO | "huge" : "", | 20260506 19:16:24.911 | INFO | "huge_shared" : " 27 GB", | 20260506 19:16:24.911 | INFO | "malloc" : " 14 GB", | 20260506 19:16:24.911 | INFO | "pinned" : " 29 GB" | 20260506 19:16:24.911 | INFO | } | 20260506 19:16:24.911 | INFO | Perf report for the last 5.13471 seconds: | 20260506 19:16:24.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:16:24.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:24.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:24.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:24.911 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 19:16:24.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:24.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:24.911 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 19:16:24.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:24.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:24.911 | INFO | Uploaded 36GiB to GPU in 1.93081s (18.645GiB/s) | 20260506 19:16:24.911 | INFO | Downloaded 34GiB to GPU in 1.87778s (18.1065GiB/s) | 20260506 19:16:30.052 | INFO | Analyzing chunk frames = [3794944,3795456) | 20260506 19:16:30.052 | INFO | MemInfo: { | 20260506 19:16:30.052 | INFO | "gpu" : " 17 GB", | 20260506 19:16:30.052 | INFO | "huge" : "", | 20260506 19:16:30.052 | INFO | "huge_shared" : " 27 GB", | 20260506 19:16:30.052 | INFO | "malloc" : " 14 GB", | 20260506 19:16:30.052 | INFO | "pinned" : " 29 GB" | 20260506 19:16:30.052 | INFO | } | 20260506 19:16:30.052 | INFO | Perf report for the last 5.14098 seconds: | 20260506 19:16:30.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:16:30.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:30.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:30.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:30.052 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 19:16:30.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:30.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:30.052 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:16:30.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:30.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:30.052 | INFO | Uploaded 36GiB to GPU in 1.97287s (18.2475GiB/s) | 20260506 19:16:30.052 | INFO | Downloaded 34GiB to GPU in 1.89879s (17.9062GiB/s) | 20260506 19:16:35.195 | INFO | Analyzing chunk frames = [3795456,3795968) | 20260506 19:16:35.195 | INFO | MemInfo: { | 20260506 19:16:35.195 | INFO | "gpu" : " 17 GB", | 20260506 19:16:35.195 | INFO | "huge" : "", | 20260506 19:16:35.195 | INFO | "huge_shared" : " 27 GB", | 20260506 19:16:35.195 | INFO | "malloc" : " 14 GB", | 20260506 19:16:35.195 | INFO | "pinned" : " 29 GB" | 20260506 19:16:35.195 | INFO | } | 20260506 19:16:35.195 | INFO | Perf report for the last 5.14347 seconds: | 20260506 19:16:35.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:16:35.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:35.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:35.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:35.195 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 19:16:35.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:35.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:35.195 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:16:35.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:35.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:35.196 | INFO | Uploaded 36GiB to GPU in 1.99928s (18.0065GiB/s) | 20260506 19:16:35.196 | INFO | Downloaded 34GiB to GPU in 1.95187s (17.4192GiB/s) | 20260506 19:16:40.334 | INFO | Analyzing chunk frames = [3795968,3796480) | 20260506 19:16:40.334 | INFO | MemInfo: { | 20260506 19:16:40.334 | INFO | "gpu" : " 17 GB", | 20260506 19:16:40.334 | INFO | "huge" : "", | 20260506 19:16:40.334 | INFO | "huge_shared" : " 27 GB", | 20260506 19:16:40.335 | INFO | "malloc" : " 14 GB", | 20260506 19:16:40.335 | INFO | "pinned" : " 29 GB" | 20260506 19:16:40.335 | INFO | } | 20260506 19:16:40.335 | INFO | Perf report for the last 5.13923 seconds: | 20260506 19:16:40.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:16:40.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:40.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:40.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:40.335 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:16:40.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:40.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:40.335 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 19:16:40.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:40.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:40.335 | INFO | Uploaded 36GiB to GPU in 1.97294s (18.2469GiB/s) | 20260506 19:16:40.335 | INFO | Downloaded 34GiB to GPU in 2.00389s (16.967GiB/s) | 20260506 19:16:45.474 | INFO | Analyzing chunk frames = [3796480,3796992) | 20260506 19:16:45.474 | INFO | MemInfo: { | 20260506 19:16:45.474 | INFO | "gpu" : " 17 GB", | 20260506 19:16:45.474 | INFO | "huge" : "", | 20260506 19:16:45.474 | INFO | "huge_shared" : " 27 GB", | 20260506 19:16:45.474 | INFO | "malloc" : " 14 GB", | 20260506 19:16:45.474 | INFO | "pinned" : " 29 GB" | 20260506 19:16:45.474 | INFO | } | 20260506 19:16:45.474 | INFO | Perf report for the last 5.13924 seconds: | 20260506 19:16:45.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:16:45.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:45.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:45.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:45.474 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 19:16:45.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:45.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:45.474 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 19:16:45.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:45.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:45.474 | INFO | Uploaded 36GiB to GPU in 2.02006s (17.8213GiB/s) | 20260506 19:16:45.474 | INFO | Downloaded 34GiB to GPU in 2.01196s (16.8989GiB/s) | 20260506 19:16:45.585 | INFO | Starting a round of Baz IO | 20260506 19:16:48.725 | INFO | Populating RTMetrics report | 20260506 19:16:48.725 | INFO | RTMetrics report populated | 20260506 19:16:48.725 | INFO | Opening tmp RTMetrics json file | 20260506 19:16:48.726 | INFO | Serializing RTMetrics to json | 20260506 19:16:48.727 | INFO | Serialization done, swapping tmp file now... | 20260506 19:16:48.728 | INFO | RTMetrics done | 20260506 19:16:48.924 | INFO | Finished a round of Baz IO: Wrote 2.014 GiB in 3.339 seconds | 20260506 19:16:48.924 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.69% / 0.00% / 65.30% | 20260506 19:16:48.924 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.945405% / 0% / 5.5631e-05% | 20260506 19:16:48.924 | INFO | Baz IO overall: | 20260506 19:16:48.924 | INFO | BytesWritten : 2006606006121 | 20260506 19:16:48.924 | INFO | NumEvents : 970058082842 | 20260506 19:16:50.613 | INFO | Analyzing chunk frames = [3796992,3797504) | 20260506 19:16:50.613 | INFO | MemInfo: { | 20260506 19:16:50.613 | INFO | "gpu" : " 17 GB", | 20260506 19:16:50.613 | INFO | "huge" : "", | 20260506 19:16:50.613 | INFO | "huge_shared" : " 27 GB", | 20260506 19:16:50.613 | INFO | "malloc" : " 14 GB", | 20260506 19:16:50.613 | INFO | "pinned" : " 29 GB" | 20260506 19:16:50.613 | INFO | } | 20260506 19:16:50.613 | INFO | Perf report for the last 5.13892 seconds: | 20260506 19:16:50.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:16:50.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:50.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:50.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:50.613 | INFO | ANALYSIS : 62.7%, 2.9, 95.4ms, 0 | 20260506 19:16:50.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:50.613 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260506 19:16:50.613 | INFO | PRE_HQ : 60.1%, 1.8, 57.7ms, 0 | 20260506 19:16:50.613 | INFO | BAZWRITER : 64.7%, 4.8, 159.3ms, 0 | 20260506 19:16:50.613 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 19:16:50.613 | INFO | Uploaded 36GiB to GPU in 2.06719s (17.415GiB/s) | 20260506 19:16:50.613 | INFO | Downloaded 38GiB to GPU in 2.21196s (17.1794GiB/s) | 20260506 19:16:55.751 | INFO | Analyzing chunk frames = [3797504,3798016) | 20260506 19:16:55.752 | INFO | MemInfo: { | 20260506 19:16:55.752 | INFO | "gpu" : " 17 GB", | 20260506 19:16:55.752 | INFO | "huge" : "", | 20260506 19:16:55.752 | INFO | "huge_shared" : " 27 GB", | 20260506 19:16:55.752 | INFO | "malloc" : " 14 GB", | 20260506 19:16:55.752 | INFO | "pinned" : " 29 GB" | 20260506 19:16:55.752 | INFO | } | 20260506 19:16:55.752 | INFO | Perf report for the last 5.13882 seconds: | 20260506 19:16:55.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:16:55.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:55.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:55.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:55.752 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 19:16:55.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:55.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:55.752 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:16:55.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:55.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:16:55.752 | INFO | Uploaded 36GiB to GPU in 1.99908s (18.0083GiB/s) | 20260506 19:16:55.752 | INFO | Downloaded 34GiB to GPU in 1.93266s (17.5923GiB/s) | 20260506 19:17:00.892 | INFO | Analyzing chunk frames = [3798016,3798528) | 20260506 19:17:00.892 | INFO | MemInfo: { | 20260506 19:17:00.892 | INFO | "gpu" : " 17 GB", | 20260506 19:17:00.892 | INFO | "huge" : "", | 20260506 19:17:00.892 | INFO | "huge_shared" : " 27 GB", | 20260506 19:17:00.892 | INFO | "malloc" : " 14 GB", | 20260506 19:17:00.892 | INFO | "pinned" : " 29 GB" | 20260506 19:17:00.892 | INFO | } | 20260506 19:17:00.892 | INFO | Perf report for the last 5.14068 seconds: | 20260506 19:17:00.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:17:00.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:00.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:00.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:00.892 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 19:17:00.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:00.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:00.892 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 19:17:00.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:00.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:00.893 | INFO | Uploaded 36GiB to GPU in 1.97006s (18.2736GiB/s) | 20260506 19:17:00.893 | INFO | Downloaded 34GiB to GPU in 1.92293s (17.6813GiB/s) | 20260506 19:17:06.033 | INFO | Analyzing chunk frames = [3798528,3799040) | 20260506 19:17:06.033 | INFO | MemInfo: { | 20260506 19:17:06.033 | INFO | "gpu" : " 17 GB", | 20260506 19:17:06.033 | INFO | "huge" : "", | 20260506 19:17:06.033 | INFO | "huge_shared" : " 27 GB", | 20260506 19:17:06.033 | INFO | "malloc" : " 14 GB", | 20260506 19:17:06.033 | INFO | "pinned" : " 29 GB" | 20260506 19:17:06.033 | INFO | } | 20260506 19:17:06.033 | INFO | Perf report for the last 5.14056 seconds: | 20260506 19:17:06.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:17:06.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:06.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:06.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:06.033 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 19:17:06.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:06.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:06.033 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:17:06.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:06.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:06.033 | INFO | Uploaded 36GiB to GPU in 1.91599s (18.7893GiB/s) | 20260506 19:17:06.033 | INFO | Downloaded 34GiB to GPU in 1.86644s (18.2165GiB/s) | 20260506 19:17:11.169 | INFO | Analyzing chunk frames = [3799040,3799552) | 20260506 19:17:11.169 | INFO | MemInfo: { | 20260506 19:17:11.169 | INFO | "gpu" : " 17 GB", | 20260506 19:17:11.169 | INFO | "huge" : "", | 20260506 19:17:11.169 | INFO | "huge_shared" : " 27 GB", | 20260506 19:17:11.169 | INFO | "malloc" : " 14 GB", | 20260506 19:17:11.169 | INFO | "pinned" : " 29 GB" | 20260506 19:17:11.170 | INFO | } | 20260506 19:17:11.170 | INFO | Perf report for the last 5.13669 seconds: | 20260506 19:17:11.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:17:11.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:11.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:11.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:11.170 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 19:17:11.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:11.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:11.170 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:17:11.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:11.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:11.170 | INFO | Uploaded 36GiB to GPU in 2.01062s (17.9049GiB/s) | 20260506 19:17:11.170 | INFO | Downloaded 34GiB to GPU in 1.91646s (17.7411GiB/s) | 20260506 19:17:16.312 | INFO | Analyzing chunk frames = [3799552,3800064) | 20260506 19:17:16.312 | INFO | MemInfo: { | 20260506 19:17:16.312 | INFO | "gpu" : " 17 GB", | 20260506 19:17:16.312 | INFO | "huge" : "", | 20260506 19:17:16.312 | INFO | "huge_shared" : " 27 GB", | 20260506 19:17:16.312 | INFO | "malloc" : " 14 GB", | 20260506 19:17:16.312 | INFO | "pinned" : " 29 GB" | 20260506 19:17:16.312 | INFO | } | 20260506 19:17:16.312 | INFO | Perf report for the last 5.14231 seconds: | 20260506 19:17:16.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:17:16.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:16.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:16.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:16.312 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 19:17:16.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:16.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:16.312 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 19:17:16.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:16.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:16.312 | INFO | Uploaded 36GiB to GPU in 2.02207s (17.8035GiB/s) | 20260506 19:17:16.312 | INFO | Downloaded 34GiB to GPU in 1.95699s (17.3737GiB/s) | 20260506 19:17:21.451 | INFO | Analyzing chunk frames = [3800064,3800576) | 20260506 19:17:21.451 | INFO | MemInfo: { | 20260506 19:17:21.451 | INFO | "gpu" : " 17 GB", | 20260506 19:17:21.451 | INFO | "huge" : "", | 20260506 19:17:21.451 | INFO | "huge_shared" : " 27 GB", | 20260506 19:17:21.451 | INFO | "malloc" : " 14 GB", | 20260506 19:17:21.451 | INFO | "pinned" : " 29 GB" | 20260506 19:17:21.452 | INFO | } | 20260506 19:17:21.452 | INFO | Perf report for the last 5.13966 seconds: | 20260506 19:17:21.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:17:21.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:21.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:21.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:21.452 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 19:17:21.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:21.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:21.452 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 19:17:21.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:21.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:21.452 | INFO | Uploaded 36GiB to GPU in 1.98099s (18.1727GiB/s) | 20260506 19:17:21.452 | INFO | Downloaded 34GiB to GPU in 1.96449s (17.3073GiB/s) | 20260506 19:17:26.589 | INFO | Analyzing chunk frames = [3800576,3801088) | 20260506 19:17:26.589 | INFO | MemInfo: { | 20260506 19:17:26.589 | INFO | "gpu" : " 17 GB", | 20260506 19:17:26.589 | INFO | "huge" : "", | 20260506 19:17:26.589 | INFO | "huge_shared" : " 27 GB", | 20260506 19:17:26.589 | INFO | "malloc" : " 14 GB", | 20260506 19:17:26.589 | INFO | "pinned" : " 29 GB" | 20260506 19:17:26.589 | INFO | } | 20260506 19:17:26.589 | INFO | Perf report for the last 5.13766 seconds: | 20260506 19:17:26.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:17:26.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:26.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:26.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:26.589 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 19:17:26.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:26.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:26.589 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 19:17:26.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:26.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:26.589 | INFO | Uploaded 36GiB to GPU in 2.05381s (17.5284GiB/s) | 20260506 19:17:26.589 | INFO | Downloaded 34GiB to GPU in 1.97606s (17.206GiB/s) | 20260506 19:17:26.590 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:17:26.590 | INFO | Total mem usage: 18 GiB | 20260506 19:17:26.590 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:17:26.590 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.4%) | 20260506 19:17:26.590 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:17:26.590 | INFO | High Water Breakdown: | 20260506 19:17:26.590 | INFO | 61.3% : 9.48 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:17:26.590 | INFO | 38.6% : 5.97 GiB ( 89.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:17:26.590 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:17:26.590 | INFO | ------------------------------------------ | 20260506 19:17:26.590 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:17:26.590 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:17:26.590 | INFO | Total mem usage: 26.5 GiB | 20260506 19:17:26.590 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:17:26.590 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:17:26.590 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:17:26.590 | INFO | High Water Breakdown: | 20260506 19:17:26.590 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:17:26.590 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:17:26.590 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:17:26.590 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:17:26.590 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:17:26.590 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:17:26.590 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:17:26.590 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:17:26.590 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:17:26.590 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:17:26.590 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:17:26.590 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:17:26.590 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:17:26.590 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:17:26.590 | INFO | ------------------------------------------ | 20260506 19:17:26.590 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:17:26.590 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:17:26.590 | INFO | Total mem usage: 25.6 GiB | 20260506 19:17:26.590 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:17:26.590 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:17:26.590 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:17:26.590 | INFO | High Water Breakdown: | 20260506 19:17:26.590 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:17:26.590 | INFO | ------------------------------------------ | 20260506 19:17:26.590 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:17:26.590 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:17:26.590 | INFO | Total mem usage: 15.7 GiB | 20260506 19:17:26.590 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:17:26.590 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:17:26.591 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:17:26.591 | INFO | High Water Breakdown: | 20260506 19:17:26.591 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:17:26.591 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:17:26.591 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:17:26.591 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:17:26.591 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:17:26.591 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:17:26.591 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:17:26.591 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:17:26.591 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:17:26.591 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:17:26.591 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:17:26.591 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:17:26.591 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:17:26.591 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:17:26.591 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:17:26.591 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:17:26.591 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:17:26.591 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:17:26.591 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:17:26.591 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:17:26.591 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:17:26.591 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:17:26.591 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:17:26.591 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:17:26.591 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:17:26.591 | INFO | ------------------------------------------ | 20260506 19:17:26.591 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:17:26.591 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:17:26.591 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:17:26.591 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:17:26.591 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.53e+01, 1.35e+01 / 2.32e+00 | 20260506 19:17:26.591 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 9.95e-03 / 6.48e-03 | 20260506 19:17:26.591 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.34e+00, 5.55e-01 / 4.19e-01 | 20260506 19:17:26.591 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.83e-01 / 1.51e-01 | 20260506 19:17:26.591 | INFO | PerfReport: 15.93%, DME -- 3.98e+01 / 4.40e+01, 3.86e+00 / 2.19e+00 | 20260506 19:17:26.591 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.79e+01 / 2.09e+01, 1.50e+00 / 1.84e+00 | 20260506 19:17:26.591 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.24e+00 / 2.78e+00, 3.81e-01 / 2.46e-01 | 20260506 19:17:26.591 | INFO | PerfReport: 3.36%, Metrics -- 1.05e+00 / 1.37e+00, 2.77e-01 / 1.81e-01 | 20260506 19:17:26.591 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.62e-01 / 4.06e-01, 5.65e-02 / 2.67e-02 | 20260506 19:17:26.591 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:17:26.591 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:17:26.591 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:17:26.591 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 19:17:26.591 | INFO | PerfReport: 21.44%, Upload -- 2.08e+01 / 0.00e+00, 6.89e+00 / 0.00e+00 | 20260506 19:17:26.591 | INFO | PerfReport: 21.20%, Download -- 2.05e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 19:17:26.591 | INFO | PerfReport: 32.26%, Compute -- 3.13e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 19:17:26.591 | INFO | PerfReport: 1.33%, LockUpload -- 1.29e+00 / 0.00e+00, 5.44e+00 / 0.00e+00 | 20260506 19:17:26.591 | INFO | PerfReport: 1.18%, LockDownload -- 1.14e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260506 19:17:26.591 | INFO | PerfReport: 22.58%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 19:17:26.591 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260506 19:17:26.710 | INFO | Starting a round of Baz IO | 20260506 19:17:29.846 | INFO | Populating RTMetrics report | 20260506 19:17:29.846 | INFO | RTMetrics report populated | 20260506 19:17:29.846 | INFO | Opening tmp RTMetrics json file | 20260506 19:17:29.910 | INFO | Serializing RTMetrics to json | 20260506 19:17:29.937 | INFO | Serialization done, swapping tmp file now... | 20260506 19:17:29.941 | INFO | RTMetrics done | 20260506 19:17:30.278 | INFO | Finished a round of Baz IO: Wrote 2.013 GiB in 3.567 seconds | 20260506 19:17:30.278 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.69% / 0.01% / 65.31% | 20260506 19:17:30.278 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.942504% / 7.94729e-06% / 8.34465e-05% | 20260506 19:17:30.278 | INFO | Baz IO overall: | 20260506 19:17:30.278 | INFO | BytesWritten : 2008767700456 | 20260506 19:17:30.278 | INFO | NumEvents : 971097089479 | 20260506 19:17:31.727 | INFO | Analyzing chunk frames = [3801088,3801600) | 20260506 19:17:31.727 | INFO | MemInfo: { | 20260506 19:17:31.727 | INFO | "gpu" : " 17 GB", | 20260506 19:17:31.727 | INFO | "huge" : "", | 20260506 19:17:31.727 | INFO | "huge_shared" : " 27 GB", | 20260506 19:17:31.727 | INFO | "malloc" : " 14 GB", | 20260506 19:17:31.727 | INFO | "pinned" : " 29 GB" | 20260506 19:17:31.727 | INFO | } | 20260506 19:17:31.727 | INFO | Perf report for the last 5.13756 seconds: | 20260506 19:17:31.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:17:31.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:31.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:31.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:31.727 | INFO | ANALYSIS : 62.8%, 2.9, 95.7ms, 0 | 20260506 19:17:31.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:31.727 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260506 19:17:31.727 | INFO | PRE_HQ : 60.2%, 1.9, 58.0ms, 0 | 20260506 19:17:31.727 | INFO | BAZWRITER : 69.1%, 4.8, 171.2ms, 0 | 20260506 19:17:31.727 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 19:17:31.727 | INFO | Uploaded 36GiB to GPU in 2.04825s (17.576GiB/s) | 20260506 19:17:31.727 | INFO | Downloaded 38GiB to GPU in 2.20744s (17.2145GiB/s) | 20260506 19:17:36.869 | INFO | Analyzing chunk frames = [3801600,3802112) | 20260506 19:17:36.869 | INFO | MemInfo: { | 20260506 19:17:36.869 | INFO | "gpu" : " 17 GB", | 20260506 19:17:36.869 | INFO | "huge" : "", | 20260506 19:17:36.869 | INFO | "huge_shared" : " 27 GB", | 20260506 19:17:36.869 | INFO | "malloc" : " 14 GB", | 20260506 19:17:36.869 | INFO | "pinned" : " 29 GB" | 20260506 19:17:36.869 | INFO | } | 20260506 19:17:36.869 | INFO | Perf report for the last 5.14251 seconds: | 20260506 19:17:36.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:17:36.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:36.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:36.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:36.870 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 19:17:36.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:36.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:36.870 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:17:36.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:36.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:36.870 | INFO | Uploaded 36GiB to GPU in 2.0222s (17.8024GiB/s) | 20260506 19:17:36.870 | INFO | Downloaded 34GiB to GPU in 1.95152s (17.4223GiB/s) | 20260506 19:17:42.001 | INFO | Analyzing chunk frames = [3802112,3802624) | 20260506 19:17:42.001 | INFO | MemInfo: { | 20260506 19:17:42.001 | INFO | "gpu" : " 17 GB", | 20260506 19:17:42.001 | INFO | "huge" : "", | 20260506 19:17:42.001 | INFO | "huge_shared" : " 27 GB", | 20260506 19:17:42.001 | INFO | "malloc" : " 14 GB", | 20260506 19:17:42.001 | INFO | "pinned" : " 29 GB" | 20260506 19:17:42.001 | INFO | } | 20260506 19:17:42.001 | INFO | Perf report for the last 5.13203 seconds: | 20260506 19:17:42.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:17:42.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:42.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:42.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:42.002 | INFO | ANALYSIS : 61.0%, 3.0, 94.4ms, 0 | 20260506 19:17:42.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:42.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:42.002 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:17:42.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:42.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:42.002 | INFO | Uploaded 36GiB to GPU in 1.99258s (18.067GiB/s) | 20260506 19:17:42.002 | INFO | Downloaded 34GiB to GPU in 1.93643s (17.5581GiB/s) | 20260506 19:17:47.140 | INFO | Analyzing chunk frames = [3802624,3803136) | 20260506 19:17:47.140 | INFO | MemInfo: { | 20260506 19:17:47.140 | INFO | "gpu" : " 17 GB", | 20260506 19:17:47.140 | INFO | "huge" : "", | 20260506 19:17:47.140 | INFO | "huge_shared" : " 27 GB", | 20260506 19:17:47.140 | INFO | "malloc" : " 14 GB", | 20260506 19:17:47.140 | INFO | "pinned" : " 29 GB" | 20260506 19:17:47.140 | INFO | } | 20260506 19:17:47.140 | INFO | Perf report for the last 5.13831 seconds: | 20260506 19:17:47.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:17:47.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:47.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:47.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:47.140 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 19:17:47.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:47.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:47.140 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:17:47.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:47.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:47.140 | INFO | Uploaded 36GiB to GPU in 1.93278s (18.626GiB/s) | 20260506 19:17:47.140 | INFO | Downloaded 34GiB to GPU in 1.86978s (18.1839GiB/s) | 20260506 19:17:52.285 | INFO | Analyzing chunk frames = [3803136,3803648) | 20260506 19:17:52.285 | INFO | MemInfo: { | 20260506 19:17:52.285 | INFO | "gpu" : " 17 GB", | 20260506 19:17:52.285 | INFO | "huge" : "", | 20260506 19:17:52.285 | INFO | "huge_shared" : " 27 GB", | 20260506 19:17:52.285 | INFO | "malloc" : " 14 GB", | 20260506 19:17:52.285 | INFO | "pinned" : " 29 GB" | 20260506 19:17:52.285 | INFO | } | 20260506 19:17:52.285 | INFO | Perf report for the last 5.14562 seconds: | 20260506 19:17:52.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:17:52.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:52.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:52.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:52.285 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260506 19:17:52.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:52.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:52.285 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:17:52.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:52.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:52.286 | INFO | Uploaded 36GiB to GPU in 1.98774s (18.111GiB/s) | 20260506 19:17:52.286 | INFO | Downloaded 34GiB to GPU in 1.89874s (17.9066GiB/s) | 20260506 19:17:57.427 | INFO | Analyzing chunk frames = [3803648,3804160) | 20260506 19:17:57.427 | INFO | MemInfo: { | 20260506 19:17:57.427 | INFO | "gpu" : " 17 GB", | 20260506 19:17:57.427 | INFO | "huge" : "", | 20260506 19:17:57.427 | INFO | "huge_shared" : " 27 GB", | 20260506 19:17:57.427 | INFO | "malloc" : " 14 GB", | 20260506 19:17:57.427 | INFO | "pinned" : " 29 GB" | 20260506 19:17:57.427 | INFO | } | 20260506 19:17:57.427 | INFO | Perf report for the last 5.14158 seconds: | 20260506 19:17:57.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:17:57.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:57.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:57.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:57.427 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 19:17:57.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:57.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:57.427 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 19:17:57.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:57.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:17:57.427 | INFO | Uploaded 36GiB to GPU in 2.01527s (17.8636GiB/s) | 20260506 19:17:57.427 | INFO | Downloaded 34GiB to GPU in 1.94864s (17.448GiB/s) | 20260506 19:18:02.564 | INFO | Analyzing chunk frames = [3804160,3804672) | 20260506 19:18:02.564 | INFO | MemInfo: { | 20260506 19:18:02.564 | INFO | "gpu" : " 17 GB", | 20260506 19:18:02.564 | INFO | "huge" : "", | 20260506 19:18:02.564 | INFO | "huge_shared" : " 27 GB", | 20260506 19:18:02.564 | INFO | "malloc" : " 14 GB", | 20260506 19:18:02.564 | INFO | "pinned" : " 29 GB" | 20260506 19:18:02.564 | INFO | } | 20260506 19:18:02.564 | INFO | Perf report for the last 5.1372 seconds: | 20260506 19:18:02.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:18:02.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:02.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:02.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:02.564 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:18:02.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:02.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:02.564 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 19:18:02.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:02.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:02.565 | INFO | Uploaded 36GiB to GPU in 1.98485s (18.1374GiB/s) | 20260506 19:18:02.565 | INFO | Downloaded 34GiB to GPU in 1.95611s (17.3815GiB/s) | 20260506 19:18:07.707 | INFO | Analyzing chunk frames = [3804672,3805184) | 20260506 19:18:07.707 | INFO | MemInfo: { | 20260506 19:18:07.707 | INFO | "gpu" : " 17 GB", | 20260506 19:18:07.707 | INFO | "huge" : "", | 20260506 19:18:07.707 | INFO | "huge_shared" : " 27 GB", | 20260506 19:18:07.707 | INFO | "malloc" : " 14 GB", | 20260506 19:18:07.707 | INFO | "pinned" : " 29 GB" | 20260506 19:18:07.707 | INFO | } | 20260506 19:18:07.707 | INFO | Perf report for the last 5.1431 seconds: | 20260506 19:18:07.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:18:07.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:07.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:07.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:07.707 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:18:07.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:07.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:07.708 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:18:07.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:07.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:07.708 | INFO | Uploaded 36GiB to GPU in 2.00056s (17.995GiB/s) | 20260506 19:18:07.708 | INFO | Downloaded 34GiB to GPU in 1.99631s (17.0314GiB/s) | 20260506 19:18:07.817 | INFO | Starting a round of Baz IO | 20260506 19:18:10.949 | INFO | Populating RTMetrics report | 20260506 19:18:10.949 | INFO | RTMetrics report populated | 20260506 19:18:10.949 | INFO | Opening tmp RTMetrics json file | 20260506 19:18:10.969 | INFO | Serializing RTMetrics to json | 20260506 19:18:10.973 | INFO | Serialization done, swapping tmp file now... | 20260506 19:18:10.977 | INFO | RTMetrics done | 20260506 19:18:11.167 | INFO | Finished a round of Baz IO: Wrote 2.016 GiB in 3.349 seconds | 20260506 19:18:11.167 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.68% / 0.01% / 65.31% | 20260506 19:18:11.167 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.944984% / 0% / 7.54992e-05% | 20260506 19:18:11.167 | INFO | Baz IO overall: | 20260506 19:18:11.167 | INFO | BytesWritten : 2010931999607 | 20260506 19:18:11.167 | INFO | NumEvents : 972138330551 | 20260506 19:18:12.843 | INFO | Analyzing chunk frames = [3805184,3805696) | 20260506 19:18:12.843 | INFO | MemInfo: { | 20260506 19:18:12.843 | INFO | "gpu" : " 17 GB", | 20260506 19:18:12.843 | INFO | "huge" : "", | 20260506 19:18:12.843 | INFO | "huge_shared" : " 27 GB", | 20260506 19:18:12.843 | INFO | "malloc" : " 14 GB", | 20260506 19:18:12.843 | INFO | "pinned" : " 29 GB" | 20260506 19:18:12.843 | INFO | } | 20260506 19:18:12.843 | INFO | Perf report for the last 5.13622 seconds: | 20260506 19:18:12.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:18:12.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:12.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:12.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:12.844 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 19:18:12.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:12.844 | INFO | RT_METRICS : 43.6%, 1.0, 22.8ms, 0 | 20260506 19:18:12.844 | INFO | PRE_HQ : 60.0%, 1.8, 57.6ms, 0 | 20260506 19:18:12.844 | INFO | BAZWRITER : 64.8%, 4.8, 158.6ms, 0 | 20260506 19:18:12.844 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260506 19:18:12.844 | INFO | Uploaded 36GiB to GPU in 2.07743s (17.3291GiB/s) | 20260506 19:18:12.844 | INFO | Downloaded 38GiB to GPU in 2.20536s (17.2308GiB/s) | 20260506 19:18:17.979 | INFO | Analyzing chunk frames = [3805696,3806208) | 20260506 19:18:17.979 | INFO | MemInfo: { | 20260506 19:18:17.979 | INFO | "gpu" : " 17 GB", | 20260506 19:18:17.979 | INFO | "huge" : "", | 20260506 19:18:17.979 | INFO | "huge_shared" : " 27 GB", | 20260506 19:18:17.979 | INFO | "malloc" : " 14 GB", | 20260506 19:18:17.979 | INFO | "pinned" : " 29 GB" | 20260506 19:18:17.979 | INFO | } | 20260506 19:18:17.979 | INFO | Perf report for the last 5.13585 seconds: | 20260506 19:18:17.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:18:17.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:17.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:17.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:17.979 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:18:17.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:17.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:17.979 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:18:17.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:17.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:17.980 | INFO | Uploaded 36GiB to GPU in 2.01537s (17.8627GiB/s) | 20260506 19:18:17.980 | INFO | Downloaded 34GiB to GPU in 1.94045s (17.5217GiB/s) | 20260506 19:18:23.119 | INFO | Analyzing chunk frames = [3806208,3806720) | 20260506 19:18:23.119 | INFO | MemInfo: { | 20260506 19:18:23.119 | INFO | "gpu" : " 17 GB", | 20260506 19:18:23.119 | INFO | "huge" : "", | 20260506 19:18:23.119 | INFO | "huge_shared" : " 27 GB", | 20260506 19:18:23.119 | INFO | "malloc" : " 14 GB", | 20260506 19:18:23.119 | INFO | "pinned" : " 29 GB" | 20260506 19:18:23.119 | INFO | } | 20260506 19:18:23.119 | INFO | Perf report for the last 5.1399 seconds: | 20260506 19:18:23.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:18:23.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:23.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:23.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:23.119 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 19:18:23.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:23.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:23.119 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:18:23.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:23.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:23.120 | INFO | Uploaded 36GiB to GPU in 1.978s (18.2002GiB/s) | 20260506 19:18:23.120 | INFO | Downloaded 34GiB to GPU in 1.9163s (17.7425GiB/s) | 20260506 19:18:28.260 | INFO | Analyzing chunk frames = [3806720,3807232) | 20260506 19:18:28.260 | INFO | MemInfo: { | 20260506 19:18:28.260 | INFO | "gpu" : " 17 GB", | 20260506 19:18:28.260 | INFO | "huge" : "", | 20260506 19:18:28.260 | INFO | "huge_shared" : " 27 GB", | 20260506 19:18:28.260 | INFO | "malloc" : " 14 GB", | 20260506 19:18:28.260 | INFO | "pinned" : " 29 GB" | 20260506 19:18:28.260 | INFO | } | 20260506 19:18:28.260 | INFO | Perf report for the last 5.14059 seconds: | 20260506 19:18:28.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:18:28.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:28.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:28.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:28.260 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 19:18:28.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:28.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:28.260 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:18:28.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:28.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:28.260 | INFO | Uploaded 36GiB to GPU in 1.92893s (18.6632GiB/s) | 20260506 19:18:28.260 | INFO | Downloaded 34GiB to GPU in 1.86941s (18.1875GiB/s) | 20260506 19:18:33.405 | INFO | Analyzing chunk frames = [3807232,3807744) | 20260506 19:18:33.405 | INFO | MemInfo: { | 20260506 19:18:33.405 | INFO | "gpu" : " 17 GB", | 20260506 19:18:33.405 | INFO | "huge" : "", | 20260506 19:18:33.405 | INFO | "huge_shared" : " 27 GB", | 20260506 19:18:33.405 | INFO | "malloc" : " 14 GB", | 20260506 19:18:33.406 | INFO | "pinned" : " 29 GB" | 20260506 19:18:33.406 | INFO | } | 20260506 19:18:33.406 | INFO | Perf report for the last 5.1457 seconds: | 20260506 19:18:33.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:18:33.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:33.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:33.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:33.406 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 19:18:33.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:33.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:33.406 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:18:33.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:33.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:33.406 | INFO | Uploaded 36GiB to GPU in 2.02218s (17.8025GiB/s) | 20260506 19:18:33.406 | INFO | Downloaded 34GiB to GPU in 1.91285s (17.7745GiB/s) | 20260506 19:18:38.541 | INFO | Analyzing chunk frames = [3807744,3808256) | 20260506 19:18:38.541 | INFO | MemInfo: { | 20260506 19:18:38.541 | INFO | "gpu" : " 17 GB", | 20260506 19:18:38.541 | INFO | "huge" : "", | 20260506 19:18:38.541 | INFO | "huge_shared" : " 27 GB", | 20260506 19:18:38.541 | INFO | "malloc" : " 14 GB", | 20260506 19:18:38.541 | INFO | "pinned" : " 29 GB" | 20260506 19:18:38.541 | INFO | } | 20260506 19:18:38.541 | INFO | Perf report for the last 5.13567 seconds: | 20260506 19:18:38.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:18:38.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:38.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:38.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:38.541 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260506 19:18:38.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:38.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:38.541 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 19:18:38.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:38.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:38.542 | INFO | Uploaded 36GiB to GPU in 2.0668s (17.4182GiB/s) | 20260506 19:18:38.542 | INFO | Downloaded 34GiB to GPU in 1.96792s (17.2771GiB/s) | 20260506 19:18:43.677 | INFO | Analyzing chunk frames = [3808256,3808768) | 20260506 19:18:43.677 | INFO | MemInfo: { | 20260506 19:18:43.677 | INFO | "gpu" : " 17 GB", | 20260506 19:18:43.677 | INFO | "huge" : "", | 20260506 19:18:43.677 | INFO | "huge_shared" : " 27 GB", | 20260506 19:18:43.677 | INFO | "malloc" : " 14 GB", | 20260506 19:18:43.677 | INFO | "pinned" : " 29 GB" | 20260506 19:18:43.677 | INFO | } | 20260506 19:18:43.677 | INFO | Perf report for the last 5.13595 seconds: | 20260506 19:18:43.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:18:43.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:43.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:43.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:43.677 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 19:18:43.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:43.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:43.677 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:18:43.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:43.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:43.678 | INFO | Uploaded 36GiB to GPU in 1.9953s (18.0424GiB/s) | 20260506 19:18:43.678 | INFO | Downloaded 34GiB to GPU in 1.97577s (17.2085GiB/s) | 20260506 19:18:48.814 | INFO | Analyzing chunk frames = [3808768,3809280) | 20260506 19:18:48.814 | INFO | MemInfo: { | 20260506 19:18:48.814 | INFO | "gpu" : " 17 GB", | 20260506 19:18:48.814 | INFO | "huge" : "", | 20260506 19:18:48.814 | INFO | "huge_shared" : " 27 GB", | 20260506 19:18:48.814 | INFO | "malloc" : " 14 GB", | 20260506 19:18:48.814 | INFO | "pinned" : " 29 GB" | 20260506 19:18:48.814 | INFO | } | 20260506 19:18:48.814 | INFO | Perf report for the last 5.13718 seconds: | 20260506 19:18:48.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:18:48.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:48.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:48.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:48.814 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 19:18:48.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:48.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:48.814 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 19:18:48.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:48.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:48.815 | INFO | Uploaded 36GiB to GPU in 1.98021s (18.1798GiB/s) | 20260506 19:18:48.815 | INFO | Downloaded 34GiB to GPU in 1.98142s (17.1594GiB/s) | 20260506 19:18:48.815 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:18:48.815 | INFO | Total mem usage: 18 GiB | 20260506 19:18:48.815 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:18:48.815 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.4%) | 20260506 19:18:48.815 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:18:48.815 | INFO | High Water Breakdown: | 20260506 19:18:48.815 | INFO | 61.8% : 9.9 GiB ( 86.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:18:48.815 | INFO | 38% : 6.09 GiB ( 91% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:18:48.815 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:18:48.815 | INFO | ------------------------------------------ | 20260506 19:18:48.815 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:18:48.815 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:18:48.815 | INFO | Total mem usage: 26.5 GiB | 20260506 19:18:48.815 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:18:48.815 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:18:48.815 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:18:48.815 | INFO | High Water Breakdown: | 20260506 19:18:48.815 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:18:48.815 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:18:48.815 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:18:48.815 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:18:48.815 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:18:48.815 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:18:48.815 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:18:48.815 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:18:48.815 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:18:48.815 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:18:48.815 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:18:48.815 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:18:48.815 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:18:48.815 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:18:48.815 | INFO | ------------------------------------------ | 20260506 19:18:48.815 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:18:48.815 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:18:48.815 | INFO | Total mem usage: 25.6 GiB | 20260506 19:18:48.815 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:18:48.815 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:18:48.815 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:18:48.815 | INFO | High Water Breakdown: | 20260506 19:18:48.816 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:18:48.816 | INFO | ------------------------------------------ | 20260506 19:18:48.816 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:18:48.816 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:18:48.816 | INFO | Total mem usage: 15.7 GiB | 20260506 19:18:48.816 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:18:48.816 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:18:48.816 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 19:18:48.816 | INFO | High Water Breakdown: | 20260506 19:18:48.816 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:18:48.816 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:18:48.816 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:18:48.816 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:18:48.816 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:18:48.816 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:18:48.816 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:18:48.816 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:18:48.816 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:18:48.816 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:18:48.816 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:18:48.816 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:18:48.816 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:18:48.816 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:18:48.816 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:18:48.816 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:18:48.816 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:18:48.816 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:18:48.816 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:18:48.816 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:18:48.816 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:18:48.816 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:18:48.816 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:18:48.816 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:18:48.816 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:18:48.816 | INFO | ------------------------------------------ | 20260506 19:18:48.816 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:18:48.816 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:18:48.816 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:18:48.816 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:18:48.816 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.54e+01, 1.34e+01 / 1.92e+00 | 20260506 19:18:48.816 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 9.98e-02, 9.83e-03 / 5.62e-03 | 20260506 19:18:48.816 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.45e+00, 5.52e-01 / 3.47e-01 | 20260506 19:18:48.816 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.83e-01 / 1.38e-01 | 20260506 19:18:48.816 | INFO | PerfReport: 15.97%, DME -- 3.99e+01 / 4.45e+01, 3.88e+00 / 1.86e+00 | 20260506 19:18:48.816 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.43e+00 / 1.83e+00 | 20260506 19:18:48.816 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.24e+00 / 2.85e+00, 3.78e-01 / 1.97e-01 | 20260506 19:18:48.816 | INFO | PerfReport: 3.37%, Metrics -- 1.05e+00 / 1.40e+00, 2.76e-01 / 1.53e-01 | 20260506 19:18:48.816 | INFO | PerfReport: 1.17%, PulseSerializing -- 3.66e-01 / 4.09e-01, 1.66e-01 / 2.32e-02 | 20260506 19:18:48.816 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:18:48.816 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:18:48.816 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:18:48.816 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 19:18:48.816 | INFO | PerfReport: 21.50%, Upload -- 2.09e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260506 19:18:48.816 | INFO | PerfReport: 21.16%, Download -- 2.05e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 19:18:48.816 | INFO | PerfReport: 32.19%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 19:18:48.816 | INFO | PerfReport: 1.48%, LockUpload -- 1.44e+00 / 0.00e+00, 5.36e+00 / 0.00e+00 | 20260506 19:18:48.816 | INFO | PerfReport: 1.02%, LockDownload -- 9.93e-01 / 0.00e+00, 2.62e+00 / 0.00e+00 | 20260506 19:18:48.816 | INFO | PerfReport: 22.63%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 19:18:48.816 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260506 19:18:48.924 | INFO | Starting a round of Baz IO | 20260506 19:18:52.055 | INFO | Populating RTMetrics report | 20260506 19:18:52.055 | INFO | RTMetrics report populated | 20260506 19:18:52.055 | INFO | Opening tmp RTMetrics json file | 20260506 19:18:52.083 | INFO | Serializing RTMetrics to json | 20260506 19:18:52.087 | INFO | Serialization done, swapping tmp file now... | 20260506 19:18:52.093 | INFO | RTMetrics done | 20260506 19:18:52.634 | INFO | Finished a round of Baz IO: Wrote 2.014 GiB in 3.709 seconds | 20260506 19:18:52.634 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.68% / 0.00% / 65.32% | 20260506 19:18:52.634 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.940808% / 1.19209e-05% / 7.54992e-05% | 20260506 19:18:52.634 | INFO | Baz IO overall: | 20260506 19:18:52.634 | INFO | BytesWritten : 2013094685572 | 20260506 19:18:52.634 | INFO | NumEvents : 973178422364 | 20260506 19:18:53.953 | INFO | Analyzing chunk frames = [3809280,3809792) | 20260506 19:18:53.953 | INFO | MemInfo: { | 20260506 19:18:53.953 | INFO | "gpu" : " 17 GB", | 20260506 19:18:53.953 | INFO | "huge" : "", | 20260506 19:18:53.953 | INFO | "huge_shared" : " 27 GB", | 20260506 19:18:53.953 | INFO | "malloc" : " 14 GB", | 20260506 19:18:53.953 | INFO | "pinned" : " 29 GB" | 20260506 19:18:53.953 | INFO | } | 20260506 19:18:53.953 | INFO | Perf report for the last 5.13898 seconds: | 20260506 19:18:53.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:18:53.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:53.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:53.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:53.954 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 19:18:53.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:53.954 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260506 19:18:53.954 | INFO | PRE_HQ : 60.2%, 1.8, 57.5ms, 0 | 20260506 19:18:53.954 | INFO | BAZWRITER : 71.9%, 4.8, 178.1ms, 0 | 20260506 19:18:53.954 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 19:18:53.954 | INFO | Uploaded 36GiB to GPU in 2.05093s (17.553GiB/s) | 20260506 19:18:53.954 | INFO | Downloaded 38GiB to GPU in 2.19625s (17.3022GiB/s) | 20260506 19:18:59.091 | INFO | Analyzing chunk frames = [3809792,3810304) | 20260506 19:18:59.091 | INFO | MemInfo: { | 20260506 19:18:59.091 | INFO | "gpu" : " 17 GB", | 20260506 19:18:59.091 | INFO | "huge" : "", | 20260506 19:18:59.091 | INFO | "huge_shared" : " 27 GB", | 20260506 19:18:59.091 | INFO | "malloc" : " 14 GB", | 20260506 19:18:59.091 | INFO | "pinned" : " 29 GB" | 20260506 19:18:59.091 | INFO | } | 20260506 19:18:59.091 | INFO | Perf report for the last 5.13773 seconds: | 20260506 19:18:59.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:18:59.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:59.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:59.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:59.091 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 19:18:59.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:59.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:59.091 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:18:59.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:59.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:18:59.092 | INFO | Uploaded 36GiB to GPU in 2.01262s (17.8871GiB/s) | 20260506 19:18:59.092 | INFO | Downloaded 34GiB to GPU in 1.94181s (17.5095GiB/s) | 20260506 19:19:04.233 | INFO | Analyzing chunk frames = [3810304,3810816) | 20260506 19:19:04.233 | INFO | MemInfo: { | 20260506 19:19:04.233 | INFO | "gpu" : " 17 GB", | 20260506 19:19:04.233 | INFO | "huge" : "", | 20260506 19:19:04.233 | INFO | "huge_shared" : " 27 GB", | 20260506 19:19:04.233 | INFO | "malloc" : " 14 GB", | 20260506 19:19:04.233 | INFO | "pinned" : " 29 GB" | 20260506 19:19:04.233 | INFO | } | 20260506 19:19:04.233 | INFO | Perf report for the last 5.14192 seconds: | 20260506 19:19:04.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:19:04.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:04.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:04.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:04.233 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 19:19:04.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:04.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:04.233 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:19:04.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:04.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:04.233 | INFO | Uploaded 36GiB to GPU in 1.99381s (18.0559GiB/s) | 20260506 19:19:04.234 | INFO | Downloaded 34GiB to GPU in 1.92103s (17.6988GiB/s) | 20260506 19:19:09.380 | INFO | Analyzing chunk frames = [3810816,3811328) | 20260506 19:19:09.380 | INFO | MemInfo: { | 20260506 19:19:09.380 | INFO | "gpu" : " 17 GB", | 20260506 19:19:09.380 | INFO | "huge" : "", | 20260506 19:19:09.380 | INFO | "huge_shared" : " 27 GB", | 20260506 19:19:09.380 | INFO | "malloc" : " 14 GB", | 20260506 19:19:09.381 | INFO | "pinned" : " 29 GB" | 20260506 19:19:09.381 | INFO | } | 20260506 19:19:09.381 | INFO | Perf report for the last 5.14753 seconds: | 20260506 19:19:09.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:19:09.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:09.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:09.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:09.381 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 19:19:09.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:09.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:09.381 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:19:09.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:09.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:09.381 | INFO | Uploaded 36GiB to GPU in 1.94486s (18.5103GiB/s) | 20260506 19:19:09.381 | INFO | Downloaded 34GiB to GPU in 1.87084s (18.1736GiB/s) | 20260506 19:19:14.518 | INFO | Analyzing chunk frames = [3811328,3811840) | 20260506 19:19:14.518 | INFO | MemInfo: { | 20260506 19:19:14.518 | INFO | "gpu" : " 17 GB", | 20260506 19:19:14.518 | INFO | "huge" : "", | 20260506 19:19:14.518 | INFO | "huge_shared" : " 27 GB", | 20260506 19:19:14.518 | INFO | "malloc" : " 14 GB", | 20260506 19:19:14.518 | INFO | "pinned" : " 29 GB" | 20260506 19:19:14.518 | INFO | } | 20260506 19:19:14.518 | INFO | Perf report for the last 5.13787 seconds: | 20260506 19:19:14.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:19:14.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:14.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:14.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:14.519 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 19:19:14.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:14.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:14.519 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 19:19:14.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:14.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:14.519 | INFO | Uploaded 36GiB to GPU in 2.00513s (17.954GiB/s) | 20260506 19:19:14.519 | INFO | Downloaded 34GiB to GPU in 1.91916s (17.7161GiB/s) | 20260506 19:19:19.659 | INFO | Analyzing chunk frames = [3811840,3812352) | 20260506 19:19:19.659 | INFO | MemInfo: { | 20260506 19:19:19.659 | INFO | "gpu" : " 17 GB", | 20260506 19:19:19.659 | INFO | "huge" : "", | 20260506 19:19:19.659 | INFO | "huge_shared" : " 27 GB", | 20260506 19:19:19.660 | INFO | "malloc" : " 14 GB", | 20260506 19:19:19.660 | INFO | "pinned" : " 29 GB" | 20260506 19:19:19.660 | INFO | } | 20260506 19:19:19.660 | INFO | Perf report for the last 5.14116 seconds: | 20260506 19:19:19.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:19:19.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:19.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:19.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:19.660 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:19:19.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:19.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:19.660 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 19:19:19.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:19.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:19.660 | INFO | Uploaded 36GiB to GPU in 2.06931s (17.3971GiB/s) | 20260506 19:19:19.660 | INFO | Downloaded 34GiB to GPU in 1.95866s (17.3588GiB/s) | 20260506 19:19:24.792 | INFO | Analyzing chunk frames = [3812352,3812864) | 20260506 19:19:24.792 | INFO | MemInfo: { | 20260506 19:19:24.792 | INFO | "gpu" : " 17 GB", | 20260506 19:19:24.792 | INFO | "huge" : "", | 20260506 19:19:24.792 | INFO | "huge_shared" : " 27 GB", | 20260506 19:19:24.792 | INFO | "malloc" : " 14 GB", | 20260506 19:19:24.793 | INFO | "pinned" : " 29 GB" | 20260506 19:19:24.793 | INFO | } | 20260506 19:19:24.793 | INFO | Perf report for the last 5.13291 seconds: | 20260506 19:19:24.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:19:24.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:24.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:24.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:24.793 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 19:19:24.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:24.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:24.793 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 19:19:24.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:24.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:24.793 | INFO | Uploaded 36GiB to GPU in 1.99655s (18.0311GiB/s) | 20260506 19:19:24.793 | INFO | Downloaded 34GiB to GPU in 1.97638s (17.2031GiB/s) | 20260506 19:19:29.929 | INFO | Analyzing chunk frames = [3812864,3813376) | 20260506 19:19:29.929 | INFO | MemInfo: { | 20260506 19:19:29.929 | INFO | "gpu" : " 17 GB", | 20260506 19:19:29.929 | INFO | "huge" : "", | 20260506 19:19:29.929 | INFO | "huge_shared" : " 27 GB", | 20260506 19:19:29.929 | INFO | "malloc" : " 14 GB", | 20260506 19:19:29.929 | INFO | "pinned" : " 29 GB" | 20260506 19:19:29.929 | INFO | } | 20260506 19:19:29.929 | INFO | Perf report for the last 5.13683 seconds: | 20260506 19:19:29.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:19:29.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:29.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:29.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:29.929 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 19:19:29.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:29.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:29.930 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 19:19:29.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:29.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:29.930 | INFO | Uploaded 36GiB to GPU in 1.98799s (18.1087GiB/s) | 20260506 19:19:29.930 | INFO | Downloaded 34GiB to GPU in 1.97513s (17.2141GiB/s) | 20260506 19:19:30.043 | INFO | Starting a round of Baz IO | 20260506 19:19:33.180 | INFO | Populating RTMetrics report | 20260506 19:19:33.180 | INFO | RTMetrics report populated | 20260506 19:19:33.180 | INFO | Opening tmp RTMetrics json file | 20260506 19:19:33.183 | INFO | Serializing RTMetrics to json | 20260506 19:19:33.208 | INFO | Serialization done, swapping tmp file now... | 20260506 19:19:33.213 | INFO | RTMetrics done | 20260506 19:19:33.802 | INFO | Finished a round of Baz IO: Wrote 2.013 GiB in 3.759 seconds | 20260506 19:19:33.802 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.67% / 0.01% / 65.32% | 20260506 19:19:33.802 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.937784% / 3.97364e-06% / 8.34465e-05% | 20260506 19:19:33.802 | INFO | Baz IO overall: | 20260506 19:19:33.802 | INFO | BytesWritten : 2015255750737 | 20260506 19:19:33.802 | INFO | NumEvents : 974217887332 | 20260506 19:19:35.068 | INFO | Analyzing chunk frames = [3813376,3813888) | 20260506 19:19:35.068 | INFO | MemInfo: { | 20260506 19:19:35.068 | INFO | "gpu" : " 17 GB", | 20260506 19:19:35.068 | INFO | "huge" : "", | 20260506 19:19:35.068 | INFO | "huge_shared" : " 27 GB", | 20260506 19:19:35.068 | INFO | "malloc" : " 14 GB", | 20260506 19:19:35.068 | INFO | "pinned" : " 29 GB" | 20260506 19:19:35.068 | INFO | } | 20260506 19:19:35.068 | INFO | Perf report for the last 5.1389 seconds: | 20260506 19:19:35.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:19:35.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:35.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:35.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:35.068 | INFO | ANALYSIS : 62.7%, 2.9, 95.3ms, 0 | 20260506 19:19:35.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:35.068 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 19:19:35.068 | INFO | PRE_HQ : 60.0%, 1.9, 58.0ms, 0 | 20260506 19:19:35.068 | INFO | BAZWRITER : 72.8%, 4.8, 180.6ms, 0 | 20260506 19:19:35.068 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260506 19:19:35.069 | INFO | Uploaded 36GiB to GPU in 2.05516s (17.5169GiB/s) | 20260506 19:19:35.069 | INFO | Downloaded 38GiB to GPU in 2.20749s (17.2141GiB/s) | 20260506 19:19:40.214 | INFO | Analyzing chunk frames = [3813888,3814400) | 20260506 19:19:40.214 | INFO | MemInfo: { | 20260506 19:19:40.214 | INFO | "gpu" : " 17 GB", | 20260506 19:19:40.214 | INFO | "huge" : "", | 20260506 19:19:40.214 | INFO | "huge_shared" : " 27 GB", | 20260506 19:19:40.214 | INFO | "malloc" : " 14 GB", | 20260506 19:19:40.214 | INFO | "pinned" : " 29 GB" | 20260506 19:19:40.214 | INFO | } | 20260506 19:19:40.214 | INFO | Perf report for the last 5.1459 seconds: | 20260506 19:19:40.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:19:40.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:40.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:40.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:40.214 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:19:40.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:40.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:40.214 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:19:40.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:40.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:40.214 | INFO | Uploaded 36GiB to GPU in 2.00742s (17.9335GiB/s) | 20260506 19:19:40.214 | INFO | Downloaded 34GiB to GPU in 1.93798s (17.544GiB/s) | 20260506 19:19:45.354 | INFO | Analyzing chunk frames = [3814400,3814912) | 20260506 19:19:45.354 | INFO | MemInfo: { | 20260506 19:19:45.354 | INFO | "gpu" : " 17 GB", | 20260506 19:19:45.354 | INFO | "huge" : "", | 20260506 19:19:45.354 | INFO | "huge_shared" : " 27 GB", | 20260506 19:19:45.354 | INFO | "malloc" : " 14 GB", | 20260506 19:19:45.354 | INFO | "pinned" : " 29 GB" | 20260506 19:19:45.354 | INFO | } | 20260506 19:19:45.354 | INFO | Perf report for the last 5.14 seconds: | 20260506 19:19:45.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:19:45.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:45.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:45.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:45.354 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 19:19:45.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:45.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:45.354 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:19:45.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:45.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:45.355 | INFO | Uploaded 36GiB to GPU in 1.97086s (18.2661GiB/s) | 20260506 19:19:45.355 | INFO | Downloaded 34GiB to GPU in 1.915s (17.7545GiB/s) | 20260506 19:19:50.494 | INFO | Analyzing chunk frames = [3814912,3815424) | 20260506 19:19:50.494 | INFO | MemInfo: { | 20260506 19:19:50.494 | INFO | "gpu" : " 17 GB", | 20260506 19:19:50.494 | INFO | "huge" : "", | 20260506 19:19:50.494 | INFO | "huge_shared" : " 27 GB", | 20260506 19:19:50.494 | INFO | "malloc" : " 14 GB", | 20260506 19:19:50.494 | INFO | "pinned" : " 29 GB" | 20260506 19:19:50.494 | INFO | } | 20260506 19:19:50.494 | INFO | Perf report for the last 5.13968 seconds: | 20260506 19:19:50.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:19:50.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:50.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:50.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:50.494 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 19:19:50.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:50.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:50.494 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 19:19:50.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:50.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:50.494 | INFO | Uploaded 36GiB to GPU in 1.94578s (18.5016GiB/s) | 20260506 19:19:50.494 | INFO | Downloaded 34GiB to GPU in 1.86931s (18.1885GiB/s) | 20260506 19:19:55.633 | INFO | Analyzing chunk frames = [3815424,3815936) | 20260506 19:19:55.633 | INFO | MemInfo: { | 20260506 19:19:55.633 | INFO | "gpu" : " 17 GB", | 20260506 19:19:55.633 | INFO | "huge" : "", | 20260506 19:19:55.633 | INFO | "huge_shared" : " 27 GB", | 20260506 19:19:55.633 | INFO | "malloc" : " 14 GB", | 20260506 19:19:55.633 | INFO | "pinned" : " 29 GB" | 20260506 19:19:55.633 | INFO | } | 20260506 19:19:55.633 | INFO | Perf report for the last 5.13947 seconds: | 20260506 19:19:55.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:19:55.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:55.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:55.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:55.634 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 19:19:55.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:55.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:55.634 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:19:55.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:55.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:19:55.634 | INFO | Uploaded 36GiB to GPU in 2.00771s (17.9309GiB/s) | 20260506 19:19:55.634 | INFO | Downloaded 34GiB to GPU in 1.91264s (17.7765GiB/s) | 20260506 19:20:00.771 | INFO | Analyzing chunk frames = [3815936,3816448) | 20260506 19:20:00.771 | INFO | MemInfo: { | 20260506 19:20:00.771 | INFO | "gpu" : " 17 GB", | 20260506 19:20:00.772 | INFO | "huge" : "", | 20260506 19:20:00.772 | INFO | "huge_shared" : " 27 GB", | 20260506 19:20:00.772 | INFO | "malloc" : " 14 GB", | 20260506 19:20:00.772 | INFO | "pinned" : " 29 GB" | 20260506 19:20:00.772 | INFO | } | 20260506 19:20:00.772 | INFO | Perf report for the last 5.13839 seconds: | 20260506 19:20:00.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:20:00.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:00.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:00.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:00.772 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 19:20:00.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:00.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:00.772 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 19:20:00.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:00.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:00.772 | INFO | Uploaded 36GiB to GPU in 2.01195s (17.8931GiB/s) | 20260506 19:20:00.772 | INFO | Downloaded 34GiB to GPU in 1.94112s (17.5157GiB/s) | 20260506 19:20:05.904 | INFO | Analyzing chunk frames = [3816448,3816960) | 20260506 19:20:05.904 | INFO | MemInfo: { | 20260506 19:20:05.904 | INFO | "gpu" : " 17 GB", | 20260506 19:20:05.904 | INFO | "huge" : "", | 20260506 19:20:05.904 | INFO | "huge_shared" : " 27 GB", | 20260506 19:20:05.904 | INFO | "malloc" : " 14 GB", | 20260506 19:20:05.904 | INFO | "pinned" : " 29 GB" | 20260506 19:20:05.904 | INFO | } | 20260506 19:20:05.904 | INFO | Perf report for the last 5.13247 seconds: | 20260506 19:20:05.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:20:05.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:05.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:05.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:05.904 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 19:20:05.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:05.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:05.904 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:20:05.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:05.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:05.905 | INFO | Uploaded 36GiB to GPU in 2.01405s (17.8744GiB/s) | 20260506 19:20:05.905 | INFO | Downloaded 34GiB to GPU in 1.9828s (17.1475GiB/s) | 20260506 19:20:11.042 | INFO | Analyzing chunk frames = [3816960,3817472) | 20260506 19:20:11.042 | INFO | MemInfo: { | 20260506 19:20:11.042 | INFO | "gpu" : " 17 GB", | 20260506 19:20:11.043 | INFO | "huge" : "", | 20260506 19:20:11.043 | INFO | "huge_shared" : " 27 GB", | 20260506 19:20:11.043 | INFO | "malloc" : " 14 GB", | 20260506 19:20:11.043 | INFO | "pinned" : " 29 GB" | 20260506 19:20:11.043 | INFO | } | 20260506 19:20:11.043 | INFO | Perf report for the last 5.13833 seconds: | 20260506 19:20:11.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:20:11.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:11.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:11.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:11.043 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 19:20:11.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:11.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:11.043 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 19:20:11.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:11.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:11.043 | INFO | Uploaded 36GiB to GPU in 2.02218s (17.8026GiB/s) | 20260506 19:20:11.043 | INFO | Downloaded 34GiB to GPU in 1.9868s (17.1129GiB/s) | 20260506 19:20:11.043 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:20:11.043 | INFO | Total mem usage: 18 GiB | 20260506 19:20:11.043 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:20:11.043 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: -9.41%) | 20260506 19:20:11.043 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:20:11.043 | INFO | High Water Breakdown: | 20260506 19:20:11.043 | INFO | 62.2% : 10.2 GiB ( 89.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:20:11.043 | INFO | 37.6% : 6.2 GiB ( 92.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:20:11.043 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:20:11.043 | INFO | ------------------------------------------ | 20260506 19:20:11.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:20:11.043 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:20:11.043 | INFO | Total mem usage: 26.5 GiB | 20260506 19:20:11.043 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:20:11.043 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:20:11.043 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:20:11.043 | INFO | High Water Breakdown: | 20260506 19:20:11.043 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:20:11.043 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:20:11.043 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:20:11.043 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:20:11.043 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:20:11.044 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:20:11.044 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:20:11.044 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:20:11.044 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:20:11.044 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:20:11.044 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:20:11.044 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:20:11.044 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:20:11.044 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:20:11.044 | INFO | ------------------------------------------ | 20260506 19:20:11.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:20:11.044 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:20:11.044 | INFO | Total mem usage: 25.6 GiB | 20260506 19:20:11.044 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:20:11.044 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:20:11.044 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:20:11.044 | INFO | High Water Breakdown: | 20260506 19:20:11.044 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:20:11.044 | INFO | ------------------------------------------ | 20260506 19:20:11.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:20:11.044 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:20:11.044 | INFO | Total mem usage: 15.7 GiB | 20260506 19:20:11.044 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:20:11.044 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:20:11.044 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:20:11.044 | INFO | High Water Breakdown: | 20260506 19:20:11.044 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:20:11.044 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:20:11.044 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:20:11.044 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:20:11.044 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:20:11.044 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:20:11.044 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:20:11.044 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:20:11.044 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:20:11.044 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:20:11.044 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:20:11.044 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:20:11.044 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:20:11.044 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:20:11.044 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:20:11.044 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:20:11.044 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:20:11.044 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:20:11.044 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:20:11.044 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:20:11.044 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:20:11.044 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:20:11.044 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:20:11.044 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:20:11.044 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:20:11.044 | INFO | ------------------------------------------ | 20260506 19:20:11.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:20:11.044 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:20:11.044 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:20:11.044 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:20:11.044 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.55e+01, 1.34e+01 / 2.12e+00 | 20260506 19:20:11.044 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.00e-01, 9.40e-03 / 5.59e-03 | 20260506 19:20:11.044 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.46e+00, 5.48e-01 / 2.95e-01 | 20260506 19:20:11.044 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.79e-01 / 1.20e-01 | 20260506 19:20:11.044 | INFO | PerfReport: 15.95%, DME -- 3.98e+01 / 4.42e+01, 4.02e+00 / 2.28e+00 | 20260506 19:20:11.044 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.44e+00 / 1.89e+00 | 20260506 19:20:11.044 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.24e+00 / 2.76e+00, 3.78e-01 / 2.11e-01 | 20260506 19:20:11.044 | INFO | PerfReport: 3.37%, Metrics -- 1.05e+00 / 1.38e+00, 2.75e-01 / 1.92e-01 | 20260506 19:20:11.044 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.62e-01 / 4.12e-01, 5.70e-02 / 2.46e-02 | 20260506 19:20:11.044 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:20:11.044 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:20:11.044 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:20:11.044 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 19:20:11.044 | INFO | PerfReport: 21.54%, Upload -- 2.09e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260506 19:20:11.044 | INFO | PerfReport: 21.15%, Download -- 2.05e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 19:20:11.044 | INFO | PerfReport: 32.19%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 19:20:11.044 | INFO | PerfReport: 1.47%, LockUpload -- 1.42e+00 / 0.00e+00, 5.28e+00 / 0.00e+00 | 20260506 19:20:11.044 | INFO | PerfReport: 1.05%, LockDownload -- 1.02e+00 / 0.00e+00, 2.66e+00 / 0.00e+00 | 20260506 19:20:11.044 | INFO | PerfReport: 22.57%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 19:20:11.044 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260506 19:20:11.157 | INFO | Starting a round of Baz IO | 20260506 19:20:14.287 | INFO | Populating RTMetrics report | 20260506 19:20:14.287 | INFO | RTMetrics report populated | 20260506 19:20:14.287 | INFO | Opening tmp RTMetrics json file | 20260506 19:20:14.308 | INFO | Serializing RTMetrics to json | 20260506 19:20:14.334 | INFO | Serialization done, swapping tmp file now... | 20260506 19:20:14.338 | INFO | RTMetrics done | 20260506 19:20:15.066 | INFO | Finished a round of Baz IO: Wrote 2.013 GiB in 3.909 seconds | 20260506 19:20:15.066 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.67% / 0.00% / 65.33% | 20260506 19:20:15.066 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.937831% / 1.19209e-05% / 5.96046e-05% | 20260506 19:20:15.066 | INFO | Baz IO overall: | 20260506 19:20:15.066 | INFO | BytesWritten : 2017417408536 | 20260506 19:20:15.066 | INFO | NumEvents : 975257868386 | 20260506 19:20:16.183 | INFO | Analyzing chunk frames = [3817472,3817984) | 20260506 19:20:16.183 | INFO | MemInfo: { | 20260506 19:20:16.183 | INFO | "gpu" : " 17 GB", | 20260506 19:20:16.183 | INFO | "huge" : "", | 20260506 19:20:16.183 | INFO | "huge_shared" : " 27 GB", | 20260506 19:20:16.183 | INFO | "malloc" : " 14 GB", | 20260506 19:20:16.183 | INFO | "pinned" : " 29 GB" | 20260506 19:20:16.183 | INFO | } | 20260506 19:20:16.183 | INFO | Perf report for the last 5.13999 seconds: | 20260506 19:20:16.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:20:16.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:16.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:16.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:16.183 | INFO | ANALYSIS : 62.6%, 2.9, 95.3ms, 0 | 20260506 19:20:16.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:16.183 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 19:20:16.183 | INFO | PRE_HQ : 60.1%, 1.9, 57.8ms, 0 | 20260506 19:20:16.183 | INFO | BAZWRITER : 75.7%, 4.8, 187.9ms, 0 | 20260506 19:20:16.183 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 19:20:16.183 | INFO | Uploaded 36GiB to GPU in 2.05536s (17.5152GiB/s) | 20260506 19:20:16.183 | INFO | Downloaded 38GiB to GPU in 2.21274s (17.1733GiB/s) | 20260506 19:20:21.327 | INFO | Analyzing chunk frames = [3817984,3818496) | 20260506 19:20:21.327 | INFO | MemInfo: { | 20260506 19:20:21.327 | INFO | "gpu" : " 17 GB", | 20260506 19:20:21.327 | INFO | "huge" : "", | 20260506 19:20:21.327 | INFO | "huge_shared" : " 27 GB", | 20260506 19:20:21.327 | INFO | "malloc" : " 14 GB", | 20260506 19:20:21.327 | INFO | "pinned" : " 29 GB" | 20260506 19:20:21.327 | INFO | } | 20260506 19:20:21.327 | INFO | Perf report for the last 5.14419 seconds: | 20260506 19:20:21.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:20:21.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:21.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:21.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:21.327 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 19:20:21.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:21.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:21.327 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:20:21.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:21.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:21.327 | INFO | Uploaded 36GiB to GPU in 1.99467s (18.0481GiB/s) | 20260506 19:20:21.327 | INFO | Downloaded 34GiB to GPU in 1.93329s (17.5866GiB/s) | 20260506 19:20:26.475 | INFO | Analyzing chunk frames = [3818496,3819008) | 20260506 19:20:26.475 | INFO | MemInfo: { | 20260506 19:20:26.475 | INFO | "gpu" : " 17 GB", | 20260506 19:20:26.475 | INFO | "huge" : "", | 20260506 19:20:26.475 | INFO | "huge_shared" : " 27 GB", | 20260506 19:20:26.475 | INFO | "malloc" : " 14 GB", | 20260506 19:20:26.475 | INFO | "pinned" : " 29 GB" | 20260506 19:20:26.475 | INFO | } | 20260506 19:20:26.475 | INFO | Perf report for the last 5.14829 seconds: | 20260506 19:20:26.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:20:26.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:26.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:26.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:26.475 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 19:20:26.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:26.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:26.475 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:20:26.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:26.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:26.475 | INFO | Uploaded 36GiB to GPU in 1.98223s (18.1613GiB/s) | 20260506 19:20:26.475 | INFO | Downloaded 34GiB to GPU in 1.91339s (17.7695GiB/s) | 20260506 19:20:31.609 | INFO | Analyzing chunk frames = [3819008,3819520) | 20260506 19:20:31.609 | INFO | MemInfo: { | 20260506 19:20:31.609 | INFO | "gpu" : " 17 GB", | 20260506 19:20:31.609 | INFO | "huge" : "", | 20260506 19:20:31.609 | INFO | "huge_shared" : " 27 GB", | 20260506 19:20:31.609 | INFO | "malloc" : " 14 GB", | 20260506 19:20:31.609 | INFO | "pinned" : " 29 GB" | 20260506 19:20:31.609 | INFO | } | 20260506 19:20:31.609 | INFO | Perf report for the last 5.13406 seconds: | 20260506 19:20:31.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:20:31.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:31.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:31.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:31.609 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 19:20:31.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:31.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:31.609 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 19:20:31.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:31.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:31.609 | INFO | Uploaded 36GiB to GPU in 2.02309s (17.7945GiB/s) | 20260506 19:20:31.609 | INFO | Downloaded 34GiB to GPU in 1.90579s (17.8404GiB/s) | 20260506 19:20:36.746 | INFO | Analyzing chunk frames = [3819520,3820032) | 20260506 19:20:36.746 | INFO | MemInfo: { | 20260506 19:20:36.746 | INFO | "gpu" : " 17 GB", | 20260506 19:20:36.746 | INFO | "huge" : "", | 20260506 19:20:36.746 | INFO | "huge_shared" : " 27 GB", | 20260506 19:20:36.746 | INFO | "malloc" : " 14 GB", | 20260506 19:20:36.746 | INFO | "pinned" : " 29 GB" | 20260506 19:20:36.746 | INFO | } | 20260506 19:20:36.747 | INFO | Perf report for the last 5.13735 seconds: | 20260506 19:20:36.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:20:36.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:36.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:36.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:36.747 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 19:20:36.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:36.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:36.747 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:20:36.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:36.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:36.747 | INFO | Uploaded 36GiB to GPU in 2.02154s (17.8082GiB/s) | 20260506 19:20:36.747 | INFO | Downloaded 34GiB to GPU in 1.91941s (17.7138GiB/s) | 20260506 19:20:41.882 | INFO | Analyzing chunk frames = [3820032,3820544) | 20260506 19:20:41.882 | INFO | MemInfo: { | 20260506 19:20:41.882 | INFO | "gpu" : " 17 GB", | 20260506 19:20:41.882 | INFO | "huge" : "", | 20260506 19:20:41.882 | INFO | "huge_shared" : " 27 GB", | 20260506 19:20:41.882 | INFO | "malloc" : " 14 GB", | 20260506 19:20:41.882 | INFO | "pinned" : " 29 GB" | 20260506 19:20:41.882 | INFO | } | 20260506 19:20:41.882 | INFO | Perf report for the last 5.13512 seconds: | 20260506 19:20:41.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:20:41.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:41.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:41.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:41.882 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 19:20:41.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:41.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:41.882 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:20:41.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:41.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:41.882 | INFO | Uploaded 36GiB to GPU in 2.01971s (17.8244GiB/s) | 20260506 19:20:41.882 | INFO | Downloaded 34GiB to GPU in 1.946s (17.4717GiB/s) | 20260506 19:20:47.018 | INFO | Analyzing chunk frames = [3820544,3821056) | 20260506 19:20:47.018 | INFO | MemInfo: { | 20260506 19:20:47.018 | INFO | "gpu" : " 17 GB", | 20260506 19:20:47.018 | INFO | "huge" : "", | 20260506 19:20:47.018 | INFO | "huge_shared" : " 27 GB", | 20260506 19:20:47.018 | INFO | "malloc" : " 14 GB", | 20260506 19:20:47.018 | INFO | "pinned" : " 29 GB" | 20260506 19:20:47.018 | INFO | } | 20260506 19:20:47.019 | INFO | Perf report for the last 5.13676 seconds: | 20260506 19:20:47.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:20:47.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:47.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:47.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:47.019 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 19:20:47.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:47.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:47.019 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 19:20:47.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:47.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:47.019 | INFO | Uploaded 36GiB to GPU in 2.02998s (17.7341GiB/s) | 20260506 19:20:47.019 | INFO | Downloaded 34GiB to GPU in 2.00253s (16.9785GiB/s) | 20260506 19:20:52.160 | INFO | Analyzing chunk frames = [3821056,3821568) | 20260506 19:20:52.160 | INFO | MemInfo: { | 20260506 19:20:52.160 | INFO | "gpu" : " 17 GB", | 20260506 19:20:52.161 | INFO | "huge" : "", | 20260506 19:20:52.161 | INFO | "huge_shared" : " 27 GB", | 20260506 19:20:52.161 | INFO | "malloc" : " 14 GB", | 20260506 19:20:52.161 | INFO | "pinned" : " 29 GB" | 20260506 19:20:52.161 | INFO | } | 20260506 19:20:52.161 | INFO | Perf report for the last 5.1422 seconds: | 20260506 19:20:52.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:20:52.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:52.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:52.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:52.161 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 19:20:52.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:52.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:52.161 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:20:52.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:52.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:52.161 | INFO | Uploaded 36GiB to GPU in 2.03768s (17.6672GiB/s) | 20260506 19:20:52.161 | INFO | Downloaded 34GiB to GPU in 1.98471s (17.131GiB/s) | 20260506 19:20:52.276 | INFO | Starting a round of Baz IO | 20260506 19:20:55.403 | INFO | Populating RTMetrics report | 20260506 19:20:55.403 | INFO | RTMetrics report populated | 20260506 19:20:55.403 | INFO | Opening tmp RTMetrics json file | 20260506 19:20:55.472 | INFO | Serializing RTMetrics to json | 20260506 19:20:55.535 | INFO | Serialization done, swapping tmp file now... | 20260506 19:20:55.538 | INFO | RTMetrics done | 20260506 19:20:55.769 | INFO | Finished a round of Baz IO: Wrote 2.01 GiB in 3.493 seconds | 20260506 19:20:55.769 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.66% / 0.00% / 65.33% | 20260506 19:20:55.769 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.93236% / 0% / 5.96046e-05% | 20260506 19:20:55.769 | INFO | Baz IO overall: | 20260506 19:20:55.770 | INFO | BytesWritten : 2019575774034 | 20260506 19:20:55.770 | INFO | NumEvents : 976295504773 | 20260506 19:20:57.306 | INFO | Analyzing chunk frames = [3821568,3822080) | 20260506 19:20:57.306 | INFO | MemInfo: { | 20260506 19:20:57.306 | INFO | "gpu" : " 17 GB", | 20260506 19:20:57.306 | INFO | "huge" : "", | 20260506 19:20:57.306 | INFO | "huge_shared" : " 27 GB", | 20260506 19:20:57.306 | INFO | "malloc" : " 14 GB", | 20260506 19:20:57.306 | INFO | "pinned" : " 29 GB" | 20260506 19:20:57.306 | INFO | } | 20260506 19:20:57.306 | INFO | Perf report for the last 5.14554 seconds: | 20260506 19:20:57.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:20:57.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:57.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:57.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:57.306 | INFO | ANALYSIS : 62.4%, 2.9, 95.3ms, 0 | 20260506 19:20:57.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:20:57.306 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260506 19:20:57.306 | INFO | PRE_HQ : 59.8%, 1.8, 57.2ms, 0 | 20260506 19:20:57.306 | INFO | BAZWRITER : 67.6%, 4.8, 166.9ms, 0 | 20260506 19:20:57.306 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 19:20:57.306 | INFO | Uploaded 36GiB to GPU in 2.06764s (17.4112GiB/s) | 20260506 19:20:57.307 | INFO | Downloaded 38GiB to GPU in 2.21236s (17.1763GiB/s) | 20260506 19:21:02.443 | INFO | Analyzing chunk frames = [3822080,3822592) | 20260506 19:21:02.443 | INFO | MemInfo: { | 20260506 19:21:02.443 | INFO | "gpu" : " 17 GB", | 20260506 19:21:02.443 | INFO | "huge" : "", | 20260506 19:21:02.443 | INFO | "huge_shared" : " 27 GB", | 20260506 19:21:02.444 | INFO | "malloc" : " 14 GB", | 20260506 19:21:02.444 | INFO | "pinned" : " 29 GB" | 20260506 19:21:02.444 | INFO | } | 20260506 19:21:02.444 | INFO | Perf report for the last 5.13741 seconds: | 20260506 19:21:02.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:21:02.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:02.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:02.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:02.444 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:21:02.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:02.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:02.444 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:21:02.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:02.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:02.444 | INFO | Uploaded 36GiB to GPU in 2.02044s (17.8179GiB/s) | 20260506 19:21:02.444 | INFO | Downloaded 34GiB to GPU in 1.93374s (17.5825GiB/s) | 20260506 19:21:07.583 | INFO | Analyzing chunk frames = [3822592,3823104) | 20260506 19:21:07.583 | INFO | MemInfo: { | 20260506 19:21:07.583 | INFO | "gpu" : " 17 GB", | 20260506 19:21:07.583 | INFO | "huge" : "", | 20260506 19:21:07.583 | INFO | "huge_shared" : " 27 GB", | 20260506 19:21:07.583 | INFO | "malloc" : " 14 GB", | 20260506 19:21:07.583 | INFO | "pinned" : " 29 GB" | 20260506 19:21:07.583 | INFO | } | 20260506 19:21:07.583 | INFO | Perf report for the last 5.13955 seconds: | 20260506 19:21:07.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:21:07.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:07.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:07.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:07.583 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 19:21:07.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:07.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:07.583 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:21:07.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:07.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:07.583 | INFO | Uploaded 36GiB to GPU in 1.98039s (18.1782GiB/s) | 20260506 19:21:07.583 | INFO | Downloaded 34GiB to GPU in 1.91575s (17.7476GiB/s) | 20260506 19:21:12.721 | INFO | Analyzing chunk frames = [3823104,3823616) | 20260506 19:21:12.721 | INFO | MemInfo: { | 20260506 19:21:12.721 | INFO | "gpu" : " 17 GB", | 20260506 19:21:12.721 | INFO | "huge" : "", | 20260506 19:21:12.721 | INFO | "huge_shared" : " 27 GB", | 20260506 19:21:12.721 | INFO | "malloc" : " 14 GB", | 20260506 19:21:12.721 | INFO | "pinned" : " 29 GB" | 20260506 19:21:12.721 | INFO | } | 20260506 19:21:12.721 | INFO | Perf report for the last 5.13792 seconds: | 20260506 19:21:12.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:21:12.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:12.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:12.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:12.721 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:21:12.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:12.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:12.721 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:21:12.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:12.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:12.721 | INFO | Uploaded 36GiB to GPU in 1.93762s (18.5795GiB/s) | 20260506 19:21:12.722 | INFO | Downloaded 34GiB to GPU in 1.86724s (18.2087GiB/s) | 20260506 19:21:17.858 | INFO | Analyzing chunk frames = [3823616,3824128) | 20260506 19:21:17.858 | INFO | MemInfo: { | 20260506 19:21:17.858 | INFO | "gpu" : " 17 GB", | 20260506 19:21:17.858 | INFO | "huge" : "", | 20260506 19:21:17.858 | INFO | "huge_shared" : " 27 GB", | 20260506 19:21:17.858 | INFO | "malloc" : " 14 GB", | 20260506 19:21:17.858 | INFO | "pinned" : " 29 GB" | 20260506 19:21:17.858 | INFO | } | 20260506 19:21:17.858 | INFO | Perf report for the last 5.13675 seconds: | 20260506 19:21:17.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:21:17.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:17.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:17.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:17.858 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 19:21:17.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:17.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:17.858 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:21:17.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:17.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:17.858 | INFO | Uploaded 36GiB to GPU in 2.02068s (17.8158GiB/s) | 20260506 19:21:17.858 | INFO | Downloaded 34GiB to GPU in 1.91319s (17.7714GiB/s) | 20260506 19:21:22.995 | INFO | Analyzing chunk frames = [3824128,3824640) | 20260506 19:21:22.995 | INFO | MemInfo: { | 20260506 19:21:22.995 | INFO | "gpu" : " 17 GB", | 20260506 19:21:22.995 | INFO | "huge" : "", | 20260506 19:21:22.995 | INFO | "huge_shared" : " 27 GB", | 20260506 19:21:22.995 | INFO | "malloc" : " 14 GB", | 20260506 19:21:22.995 | INFO | "pinned" : " 29 GB" | 20260506 19:21:22.995 | INFO | } | 20260506 19:21:22.995 | INFO | Perf report for the last 5.13725 seconds: | 20260506 19:21:22.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:21:22.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:22.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:22.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:22.995 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:21:22.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:22.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:22.995 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:21:22.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:22.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:22.996 | INFO | Uploaded 36GiB to GPU in 2.07524s (17.3474GiB/s) | 20260506 19:21:22.996 | INFO | Downloaded 34GiB to GPU in 1.96092s (17.3388GiB/s) | 20260506 19:21:28.135 | INFO | Analyzing chunk frames = [3824640,3825152) | 20260506 19:21:28.135 | INFO | MemInfo: { | 20260506 19:21:28.135 | INFO | "gpu" : " 17 GB", | 20260506 19:21:28.136 | INFO | "huge" : "", | 20260506 19:21:28.136 | INFO | "huge_shared" : " 27 GB", | 20260506 19:21:28.136 | INFO | "malloc" : " 14 GB", | 20260506 19:21:28.136 | INFO | "pinned" : " 29 GB" | 20260506 19:21:28.136 | INFO | } | 20260506 19:21:28.136 | INFO | Perf report for the last 5.14051 seconds: | 20260506 19:21:28.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:21:28.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:28.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:28.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:28.136 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 19:21:28.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:28.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:28.136 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 19:21:28.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:28.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:28.136 | INFO | Uploaded 36GiB to GPU in 2.01153s (17.8968GiB/s) | 20260506 19:21:28.136 | INFO | Downloaded 34GiB to GPU in 1.96297s (17.3207GiB/s) | 20260506 19:21:33.275 | INFO | Analyzing chunk frames = [3825152,3825664) | 20260506 19:21:33.275 | INFO | MemInfo: { | 20260506 19:21:33.275 | INFO | "gpu" : " 17 GB", | 20260506 19:21:33.275 | INFO | "huge" : "", | 20260506 19:21:33.275 | INFO | "huge_shared" : " 27 GB", | 20260506 19:21:33.275 | INFO | "malloc" : " 14 GB", | 20260506 19:21:33.275 | INFO | "pinned" : " 29 GB" | 20260506 19:21:33.275 | INFO | } | 20260506 19:21:33.275 | INFO | Perf report for the last 5.13918 seconds: | 20260506 19:21:33.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:21:33.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:33.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:33.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:33.275 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 19:21:33.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:33.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:33.275 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260506 19:21:33.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:33.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:33.275 | INFO | Uploaded 36GiB to GPU in 2.01831s (17.8367GiB/s) | 20260506 19:21:33.275 | INFO | Downloaded 34GiB to GPU in 1.98653s (17.1153GiB/s) | 20260506 19:21:33.275 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:21:33.275 | INFO | Total mem usage: 18 GiB | 20260506 19:21:33.275 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:21:33.275 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -11.8%) | 20260506 19:21:33.275 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:21:33.275 | INFO | High Water Breakdown: | 20260506 19:21:33.275 | INFO | 61.9% : 9.97 GiB ( 87.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:21:33.276 | INFO | 38% : 6.12 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:21:33.276 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:21:33.276 | INFO | ------------------------------------------ | 20260506 19:21:33.276 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:21:33.276 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:21:33.276 | INFO | Total mem usage: 26.5 GiB | 20260506 19:21:33.276 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:21:33.276 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:21:33.276 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:21:33.276 | INFO | High Water Breakdown: | 20260506 19:21:33.276 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:21:33.276 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:21:33.276 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:21:33.276 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:21:33.276 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:21:33.276 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:21:33.276 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:21:33.276 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:21:33.276 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:21:33.276 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:21:33.276 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:21:33.276 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:21:33.276 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:21:33.276 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:21:33.276 | INFO | ------------------------------------------ | 20260506 19:21:33.276 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:21:33.276 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:21:33.276 | INFO | Total mem usage: 25.6 GiB | 20260506 19:21:33.276 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:21:33.276 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:21:33.276 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:21:33.276 | INFO | High Water Breakdown: | 20260506 19:21:33.276 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:21:33.276 | INFO | ------------------------------------------ | 20260506 19:21:33.276 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:21:33.276 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:21:33.276 | INFO | Total mem usage: 15.7 GiB | 20260506 19:21:33.276 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:21:33.276 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:21:33.276 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 19:21:33.276 | INFO | High Water Breakdown: | 20260506 19:21:33.276 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:21:33.276 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:21:33.276 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:21:33.276 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:21:33.276 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:21:33.276 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:21:33.276 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:21:33.276 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:21:33.276 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:21:33.276 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:21:33.276 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:21:33.276 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:21:33.276 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:21:33.276 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:21:33.276 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:21:33.276 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:21:33.276 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:21:33.276 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:21:33.276 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:21:33.276 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:21:33.276 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:21:33.276 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:21:33.276 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:21:33.276 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:21:33.276 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:21:33.276 | INFO | ------------------------------------------ | 20260506 19:21:33.276 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:21:33.276 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:21:33.276 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:21:33.276 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:21:33.276 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.55e+01, 1.33e+01 / 2.13e+00 | 20260506 19:21:33.276 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.02e-01, 1.00e-02 / 8.50e-03 | 20260506 19:21:33.276 | INFO | PerfReport: 11.61%, Baseline -- 3.62e+00 / 4.37e+00, 5.47e-01 / 3.68e-01 | 20260506 19:21:33.276 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.82e-01 / 1.41e-01 | 20260506 19:21:33.276 | INFO | PerfReport: 15.99%, DME -- 3.99e+01 / 4.44e+01, 3.89e+00 / 2.24e+00 | 20260506 19:21:33.276 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.45e+00 / 2.04e+00 | 20260506 19:21:33.276 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.24e+00 / 2.80e+00, 3.94e-01 / 2.43e-01 | 20260506 19:21:33.276 | INFO | PerfReport: 3.36%, Metrics -- 1.05e+00 / 1.35e+00, 2.72e-01 / 1.68e-01 | 20260506 19:21:33.277 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.61e-01 / 4.05e-01, 5.66e-02 / 2.75e-02 | 20260506 19:21:33.277 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:21:33.277 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:21:33.277 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:21:33.277 | INFO | PerfReport: Full Execution -- 9.72e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 19:21:33.277 | INFO | PerfReport: 21.64%, Upload -- 2.10e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260506 19:21:33.277 | INFO | PerfReport: 21.15%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 19:21:33.277 | INFO | PerfReport: 32.14%, Compute -- 3.12e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 19:21:33.277 | INFO | PerfReport: 1.55%, LockUpload -- 1.51e+00 / 0.00e+00, 5.74e+00 / 0.00e+00 | 20260506 19:21:33.277 | INFO | PerfReport: 1.03%, LockDownload -- 1.01e+00 / 0.00e+00, 2.62e+00 / 0.00e+00 | 20260506 19:21:33.277 | INFO | PerfReport: 22.47%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 19:21:33.277 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.36e-02 / 0.00e+00 | 20260506 19:21:33.390 | INFO | Starting a round of Baz IO | 20260506 19:21:36.523 | INFO | Populating RTMetrics report | 20260506 19:21:36.523 | INFO | RTMetrics report populated | 20260506 19:21:36.523 | INFO | Opening tmp RTMetrics json file | 20260506 19:21:36.525 | INFO | Serializing RTMetrics to json | 20260506 19:21:36.531 | INFO | Serialization done, swapping tmp file now... | 20260506 19:21:36.553 | INFO | RTMetrics done | 20260506 19:21:36.676 | INFO | Finished a round of Baz IO: Wrote 2.01 GiB in 3.285 seconds | 20260506 19:21:36.676 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.66% / 0.00% / 65.34% | 20260506 19:21:36.676 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.929848% / 3.97364e-06% / 3.97364e-05% | 20260506 19:21:36.676 | INFO | Baz IO overall: | 20260506 19:21:36.676 | INFO | BytesWritten : 2021734515477 | 20260506 19:21:36.676 | INFO | NumEvents : 977333216758 | 20260506 19:21:38.419 | INFO | Analyzing chunk frames = [3825664,3826176) | 20260506 19:21:38.419 | INFO | MemInfo: { | 20260506 19:21:38.419 | INFO | "gpu" : " 17 GB", | 20260506 19:21:38.419 | INFO | "huge" : "", | 20260506 19:21:38.419 | INFO | "huge_shared" : " 27 GB", | 20260506 19:21:38.419 | INFO | "malloc" : " 14 GB", | 20260506 19:21:38.419 | INFO | "pinned" : " 29 GB" | 20260506 19:21:38.419 | INFO | } | 20260506 19:21:38.419 | INFO | Perf report for the last 5.14453 seconds: | 20260506 19:21:38.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:21:38.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:38.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:38.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:38.420 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 19:21:38.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:38.420 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 19:21:38.420 | INFO | PRE_HQ : 59.8%, 1.9, 57.9ms, 0 | 20260506 19:21:38.420 | INFO | BAZWRITER : 63.6%, 4.7, 152.4ms, 0 | 20260506 19:21:38.420 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260506 19:21:38.420 | INFO | Uploaded 36GiB to GPU in 2.02469s (17.7805GiB/s) | 20260506 19:21:38.420 | INFO | Downloaded 38GiB to GPU in 2.21364s (17.1663GiB/s) | 20260506 19:21:43.562 | INFO | Analyzing chunk frames = [3826176,3826688) | 20260506 19:21:43.562 | INFO | MemInfo: { | 20260506 19:21:43.562 | INFO | "gpu" : " 17 GB", | 20260506 19:21:43.562 | INFO | "huge" : "", | 20260506 19:21:43.562 | INFO | "huge_shared" : " 27 GB", | 20260506 19:21:43.563 | INFO | "malloc" : " 14 GB", | 20260506 19:21:43.563 | INFO | "pinned" : " 29 GB" | 20260506 19:21:43.563 | INFO | } | 20260506 19:21:43.563 | INFO | Perf report for the last 5.14322 seconds: | 20260506 19:21:43.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:21:43.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:43.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:43.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:43.563 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 19:21:43.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:43.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:43.563 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:21:43.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:43.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:43.563 | INFO | Uploaded 36GiB to GPU in 1.9991s (18.0081GiB/s) | 20260506 19:21:43.563 | INFO | Downloaded 34GiB to GPU in 1.92403s (17.6713GiB/s) | 20260506 19:21:48.695 | INFO | Analyzing chunk frames = [3826688,3827200) | 20260506 19:21:48.695 | INFO | MemInfo: { | 20260506 19:21:48.695 | INFO | "gpu" : " 17 GB", | 20260506 19:21:48.695 | INFO | "huge" : "", | 20260506 19:21:48.695 | INFO | "huge_shared" : " 27 GB", | 20260506 19:21:48.695 | INFO | "malloc" : " 14 GB", | 20260506 19:21:48.695 | INFO | "pinned" : " 29 GB" | 20260506 19:21:48.695 | INFO | } | 20260506 19:21:48.696 | INFO | Perf report for the last 5.13282 seconds: | 20260506 19:21:48.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:21:48.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:48.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:48.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:48.696 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:21:48.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:48.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:48.696 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:21:48.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:48.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:48.696 | INFO | Uploaded 36GiB to GPU in 1.96187s (18.3498GiB/s) | 20260506 19:21:48.696 | INFO | Downloaded 34GiB to GPU in 1.91783s (17.7284GiB/s) | 20260506 19:21:53.835 | INFO | Analyzing chunk frames = [3827200,3827712) | 20260506 19:21:53.835 | INFO | MemInfo: { | 20260506 19:21:53.835 | INFO | "gpu" : " 17 GB", | 20260506 19:21:53.835 | INFO | "huge" : "", | 20260506 19:21:53.835 | INFO | "huge_shared" : " 27 GB", | 20260506 19:21:53.835 | INFO | "malloc" : " 14 GB", | 20260506 19:21:53.835 | INFO | "pinned" : " 29 GB" | 20260506 19:21:53.835 | INFO | } | 20260506 19:21:53.835 | INFO | Perf report for the last 5.13909 seconds: | 20260506 19:21:53.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:21:53.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:53.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:53.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:53.835 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:21:53.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:53.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:53.835 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:21:53.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:53.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:53.835 | INFO | Uploaded 36GiB to GPU in 1.93432s (18.6112GiB/s) | 20260506 19:21:53.835 | INFO | Downloaded 34GiB to GPU in 1.87038s (18.1781GiB/s) | 20260506 19:21:58.972 | INFO | Analyzing chunk frames = [3827712,3828224) | 20260506 19:21:58.972 | INFO | MemInfo: { | 20260506 19:21:58.972 | INFO | "gpu" : " 17 GB", | 20260506 19:21:58.972 | INFO | "huge" : "", | 20260506 19:21:58.972 | INFO | "huge_shared" : " 27 GB", | 20260506 19:21:58.972 | INFO | "malloc" : " 14 GB", | 20260506 19:21:58.972 | INFO | "pinned" : " 29 GB" | 20260506 19:21:58.972 | INFO | } | 20260506 19:21:58.972 | INFO | Perf report for the last 5.13783 seconds: | 20260506 19:21:58.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:21:58.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:58.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:58.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:58.973 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 19:21:58.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:58.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:58.973 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:21:58.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:58.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:21:58.973 | INFO | Uploaded 36GiB to GPU in 2.00235s (17.9789GiB/s) | 20260506 19:21:58.973 | INFO | Downloaded 34GiB to GPU in 1.91757s (17.7308GiB/s) | 20260506 19:22:04.113 | INFO | Analyzing chunk frames = [3828224,3828736) | 20260506 19:22:04.113 | INFO | MemInfo: { | 20260506 19:22:04.113 | INFO | "gpu" : " 17 GB", | 20260506 19:22:04.113 | INFO | "huge" : "", | 20260506 19:22:04.113 | INFO | "huge_shared" : " 27 GB", | 20260506 19:22:04.113 | INFO | "malloc" : " 14 GB", | 20260506 19:22:04.113 | INFO | "pinned" : " 29 GB" | 20260506 19:22:04.113 | INFO | } | 20260506 19:22:04.113 | INFO | Perf report for the last 5.14038 seconds: | 20260506 19:22:04.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:22:04.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:04.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:04.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:04.113 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 19:22:04.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:04.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:04.113 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:22:04.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:04.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:04.113 | INFO | Uploaded 36GiB to GPU in 1.99887s (18.0102GiB/s) | 20260506 19:22:04.113 | INFO | Downloaded 34GiB to GPU in 1.94189s (17.5088GiB/s) | 20260506 19:22:09.251 | INFO | Analyzing chunk frames = [3828736,3829248) | 20260506 19:22:09.251 | INFO | MemInfo: { | 20260506 19:22:09.251 | INFO | "gpu" : " 17 GB", | 20260506 19:22:09.251 | INFO | "huge" : "", | 20260506 19:22:09.251 | INFO | "huge_shared" : " 27 GB", | 20260506 19:22:09.251 | INFO | "malloc" : " 14 GB", | 20260506 19:22:09.251 | INFO | "pinned" : " 29 GB" | 20260506 19:22:09.251 | INFO | } | 20260506 19:22:09.251 | INFO | Perf report for the last 5.13801 seconds: | 20260506 19:22:09.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:22:09.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:09.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:09.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:09.251 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 19:22:09.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:09.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:09.251 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:22:09.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:09.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:09.251 | INFO | Uploaded 36GiB to GPU in 1.94527s (18.5064GiB/s) | 20260506 19:22:09.251 | INFO | Downloaded 34GiB to GPU in 1.9944s (17.0477GiB/s) | 20260506 19:22:14.392 | INFO | Analyzing chunk frames = [3829248,3829760) | 20260506 19:22:14.392 | INFO | MemInfo: { | 20260506 19:22:14.392 | INFO | "gpu" : " 17 GB", | 20260506 19:22:14.392 | INFO | "huge" : "", | 20260506 19:22:14.392 | INFO | "huge_shared" : " 27 GB", | 20260506 19:22:14.392 | INFO | "malloc" : " 14 GB", | 20260506 19:22:14.392 | INFO | "pinned" : " 29 GB" | 20260506 19:22:14.392 | INFO | } | 20260506 19:22:14.392 | INFO | Perf report for the last 5.14075 seconds: | 20260506 19:22:14.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:22:14.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:14.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:14.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:14.392 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 19:22:14.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:14.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:14.392 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 19:22:14.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:14.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:14.392 | INFO | Uploaded 36GiB to GPU in 2.02466s (17.7807GiB/s) | 20260506 19:22:14.392 | INFO | Downloaded 34GiB to GPU in 2.00175s (16.9851GiB/s) | 20260506 19:22:14.506 | INFO | Starting a round of Baz IO | 20260506 19:22:17.638 | INFO | Populating RTMetrics report | 20260506 19:22:17.639 | INFO | RTMetrics report populated | 20260506 19:22:17.639 | INFO | Opening tmp RTMetrics json file | 20260506 19:22:17.651 | INFO | Serializing RTMetrics to json | 20260506 19:22:17.655 | INFO | Serialization done, swapping tmp file now... | 20260506 19:22:17.660 | INFO | RTMetrics done | 20260506 19:22:18.184 | INFO | Finished a round of Baz IO: Wrote 2.011 GiB in 3.677 seconds | 20260506 19:22:18.184 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.65% / 0.00% / 65.34% | 20260506 19:22:18.184 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.92953% / 3.97364e-06% / 6.35783e-05% | 20260506 19:22:18.185 | INFO | Baz IO overall: | 20260506 19:22:18.185 | INFO | BytesWritten : 2023894287247 | 20260506 19:22:18.185 | INFO | NumEvents : 978371883675 | 20260506 19:22:19.531 | INFO | Analyzing chunk frames = [3829760,3830272) | 20260506 19:22:19.531 | INFO | MemInfo: { | 20260506 19:22:19.531 | INFO | "gpu" : " 17 GB", | 20260506 19:22:19.531 | INFO | "huge" : "", | 20260506 19:22:19.531 | INFO | "huge_shared" : " 27 GB", | 20260506 19:22:19.531 | INFO | "malloc" : " 14 GB", | 20260506 19:22:19.532 | INFO | "pinned" : " 29 GB" | 20260506 19:22:19.532 | INFO | } | 20260506 19:22:19.532 | INFO | Perf report for the last 5.13997 seconds: | 20260506 19:22:19.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:22:19.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:19.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:19.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:19.532 | INFO | ANALYSIS : 62.6%, 2.9, 95.4ms, 0 | 20260506 19:22:19.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:19.532 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 19:22:19.532 | INFO | PRE_HQ : 59.9%, 1.9, 57.7ms, 0 | 20260506 19:22:19.532 | INFO | BAZWRITER : 71.2%, 4.8, 176.4ms, 0 | 20260506 19:22:19.532 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 19:22:19.532 | INFO | Uploaded 36GiB to GPU in 2.02207s (17.8036GiB/s) | 20260506 19:22:19.532 | INFO | Downloaded 38GiB to GPU in 2.20951s (17.1984GiB/s) | 20260506 19:22:24.669 | INFO | Analyzing chunk frames = [3830272,3830784) | 20260506 19:22:24.669 | INFO | MemInfo: { | 20260506 19:22:24.669 | INFO | "gpu" : " 17 GB", | 20260506 19:22:24.669 | INFO | "huge" : "", | 20260506 19:22:24.669 | INFO | "huge_shared" : " 27 GB", | 20260506 19:22:24.669 | INFO | "malloc" : " 14 GB", | 20260506 19:22:24.669 | INFO | "pinned" : " 29 GB" | 20260506 19:22:24.669 | INFO | } | 20260506 19:22:24.669 | INFO | Perf report for the last 5.13785 seconds: | 20260506 19:22:24.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:22:24.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:24.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:24.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:24.670 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:22:24.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:24.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:24.670 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:22:24.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:24.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:24.670 | INFO | Uploaded 36GiB to GPU in 2.00005s (17.9995GiB/s) | 20260506 19:22:24.670 | INFO | Downloaded 34GiB to GPU in 1.92722s (17.642GiB/s) | 20260506 19:22:29.811 | INFO | Analyzing chunk frames = [3830784,3831296) | 20260506 19:22:29.811 | INFO | MemInfo: { | 20260506 19:22:29.811 | INFO | "gpu" : " 17 GB", | 20260506 19:22:29.811 | INFO | "huge" : "", | 20260506 19:22:29.811 | INFO | "huge_shared" : " 27 GB", | 20260506 19:22:29.811 | INFO | "malloc" : " 14 GB", | 20260506 19:22:29.811 | INFO | "pinned" : " 29 GB" | 20260506 19:22:29.811 | INFO | } | 20260506 19:22:29.811 | INFO | Perf report for the last 5.14128 seconds: | 20260506 19:22:29.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:22:29.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:29.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:29.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:29.811 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:22:29.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:29.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:29.811 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:22:29.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:29.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:29.811 | INFO | Uploaded 36GiB to GPU in 1.95666s (18.3987GiB/s) | 20260506 19:22:29.811 | INFO | Downloaded 34GiB to GPU in 1.9216s (17.6936GiB/s) | 20260506 19:22:34.947 | INFO | Analyzing chunk frames = [3831296,3831808) | 20260506 19:22:34.947 | INFO | MemInfo: { | 20260506 19:22:34.947 | INFO | "gpu" : " 17 GB", | 20260506 19:22:34.947 | INFO | "huge" : "", | 20260506 19:22:34.947 | INFO | "huge_shared" : " 27 GB", | 20260506 19:22:34.947 | INFO | "malloc" : " 14 GB", | 20260506 19:22:34.947 | INFO | "pinned" : " 29 GB" | 20260506 19:22:34.947 | INFO | } | 20260506 19:22:34.947 | INFO | Perf report for the last 5.13648 seconds: | 20260506 19:22:34.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:22:34.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:34.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:34.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:34.948 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:22:34.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:34.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:34.948 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:22:34.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:34.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:34.948 | INFO | Uploaded 36GiB to GPU in 1.943s (18.5281GiB/s) | 20260506 19:22:34.948 | INFO | Downloaded 34GiB to GPU in 1.87397s (18.1433GiB/s) | 20260506 19:22:40.086 | INFO | Analyzing chunk frames = [3831808,3832320) | 20260506 19:22:40.086 | INFO | MemInfo: { | 20260506 19:22:40.086 | INFO | "gpu" : " 17 GB", | 20260506 19:22:40.086 | INFO | "huge" : "", | 20260506 19:22:40.086 | INFO | "huge_shared" : " 27 GB", | 20260506 19:22:40.086 | INFO | "malloc" : " 14 GB", | 20260506 19:22:40.086 | INFO | "pinned" : " 29 GB" | 20260506 19:22:40.086 | INFO | } | 20260506 19:22:40.086 | INFO | Perf report for the last 5.139 seconds: | 20260506 19:22:40.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:22:40.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:40.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:40.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:40.086 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 19:22:40.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:40.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:40.087 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 19:22:40.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:40.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:40.087 | INFO | Uploaded 36GiB to GPU in 2.00376s (17.9663GiB/s) | 20260506 19:22:40.087 | INFO | Downloaded 34GiB to GPU in 1.91207s (17.7817GiB/s) | 20260506 19:22:45.228 | INFO | Analyzing chunk frames = [3832320,3832832) | 20260506 19:22:45.228 | INFO | MemInfo: { | 20260506 19:22:45.228 | INFO | "gpu" : " 17 GB", | 20260506 19:22:45.228 | INFO | "huge" : "", | 20260506 19:22:45.228 | INFO | "huge_shared" : " 27 GB", | 20260506 19:22:45.228 | INFO | "malloc" : " 14 GB", | 20260506 19:22:45.228 | INFO | "pinned" : " 29 GB" | 20260506 19:22:45.228 | INFO | } | 20260506 19:22:45.228 | INFO | Perf report for the last 5.14183 seconds: | 20260506 19:22:45.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:22:45.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:45.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:45.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:45.228 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 19:22:45.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:45.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:45.228 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:22:45.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:45.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:45.229 | INFO | Uploaded 36GiB to GPU in 2.0241s (17.7857GiB/s) | 20260506 19:22:45.229 | INFO | Downloaded 34GiB to GPU in 1.96375s (17.3138GiB/s) | 20260506 19:22:50.366 | INFO | Analyzing chunk frames = [3832832,3833344) | 20260506 19:22:50.366 | INFO | MemInfo: { | 20260506 19:22:50.366 | INFO | "gpu" : " 17 GB", | 20260506 19:22:50.366 | INFO | "huge" : "", | 20260506 19:22:50.366 | INFO | "huge_shared" : " 27 GB", | 20260506 19:22:50.366 | INFO | "malloc" : " 14 GB", | 20260506 19:22:50.366 | INFO | "pinned" : " 29 GB" | 20260506 19:22:50.366 | INFO | } | 20260506 19:22:50.366 | INFO | Perf report for the last 5.13835 seconds: | 20260506 19:22:50.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:22:50.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:50.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:50.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:50.367 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:22:50.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:50.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:50.367 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 19:22:50.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:50.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:50.367 | INFO | Uploaded 36GiB to GPU in 1.96728s (18.2994GiB/s) | 20260506 19:22:50.367 | INFO | Downloaded 34GiB to GPU in 2.00588s (16.9501GiB/s) | 20260506 19:22:55.506 | INFO | Analyzing chunk frames = [3833344,3833856) | 20260506 19:22:55.506 | INFO | MemInfo: { | 20260506 19:22:55.506 | INFO | "gpu" : " 17 GB", | 20260506 19:22:55.506 | INFO | "huge" : "", | 20260506 19:22:55.506 | INFO | "huge_shared" : " 27 GB", | 20260506 19:22:55.506 | INFO | "malloc" : " 14 GB", | 20260506 19:22:55.506 | INFO | "pinned" : " 29 GB" | 20260506 19:22:55.506 | INFO | } | 20260506 19:22:55.506 | INFO | Perf report for the last 5.13947 seconds: | 20260506 19:22:55.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:22:55.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:55.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:55.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:55.506 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:22:55.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:55.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:55.506 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 19:22:55.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:55.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:22:55.506 | INFO | Uploaded 36GiB to GPU in 1.97997s (18.1821GiB/s) | 20260506 19:22:55.506 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:22:55.507 | INFO | Total mem usage: 18 GiB | 20260506 19:22:55.507 | INFO | Downloaded 34GiB to GPU in 1.97457s (17.2189GiB/s) | 20260506 19:22:55.507 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:22:55.507 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: -9.43%) | 20260506 19:22:55.507 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:22:55.507 | INFO | High Water Breakdown: | 20260506 19:22:55.507 | INFO | 62.2% : 10.2 GiB ( 89.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:22:55.507 | INFO | 37.7% : 6.19 GiB ( 92.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:22:55.507 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:22:55.507 | INFO | ------------------------------------------ | 20260506 19:22:55.507 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:22:55.507 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:22:55.507 | INFO | Total mem usage: 26.5 GiB | 20260506 19:22:55.507 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:22:55.507 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:22:55.507 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:22:55.507 | INFO | High Water Breakdown: | 20260506 19:22:55.507 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:22:55.507 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:22:55.507 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:22:55.507 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:22:55.507 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:22:55.507 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:22:55.507 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:22:55.507 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:22:55.507 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:22:55.507 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:22:55.507 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:22:55.507 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:22:55.507 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:22:55.507 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:22:55.507 | INFO | ------------------------------------------ | 20260506 19:22:55.507 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:22:55.507 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:22:55.507 | INFO | Total mem usage: 25.6 GiB | 20260506 19:22:55.507 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:22:55.507 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:22:55.507 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:22:55.507 | INFO | High Water Breakdown: | 20260506 19:22:55.507 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:22:55.507 | INFO | ------------------------------------------ | 20260506 19:22:55.507 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:22:55.507 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:22:55.507 | INFO | Total mem usage: 15.7 GiB | 20260506 19:22:55.507 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:22:55.507 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:22:55.507 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 19:22:55.507 | INFO | High Water Breakdown: | 20260506 19:22:55.507 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:22:55.507 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:22:55.507 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:22:55.507 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:22:55.507 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:22:55.507 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:22:55.507 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:22:55.507 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:22:55.507 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:22:55.507 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:22:55.507 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:22:55.507 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:22:55.507 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:22:55.507 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:22:55.507 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:22:55.507 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:22:55.507 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:22:55.508 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:22:55.508 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:22:55.508 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:22:55.508 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:22:55.508 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:22:55.508 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:22:55.508 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:22:55.508 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:22:55.508 | INFO | ------------------------------------------ | 20260506 19:22:55.508 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:22:55.508 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:22:55.508 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:22:55.508 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:22:55.508 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.60e+01, 1.34e+01 / 2.78e+00 | 20260506 19:22:55.508 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 1.03e-02 / 5.19e-03 | 20260506 19:22:55.508 | INFO | PerfReport: 11.64%, Baseline -- 3.64e+00 / 4.41e+00, 5.62e-01 / 3.18e-01 | 20260506 19:22:55.508 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.46e+00, 1.86e-01 / 1.41e-01 | 20260506 19:22:55.508 | INFO | PerfReport: 16.02%, DME -- 4.00e+01 / 4.49e+01, 4.31e+00 / 2.94e+00 | 20260506 19:22:55.508 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.40e+00 / 2.33e+00 | 20260506 19:22:55.508 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.24e+00 / 2.77e+00, 3.80e-01 / 2.75e-01 | 20260506 19:22:55.508 | INFO | PerfReport: 3.36%, Metrics -- 1.05e+00 / 1.38e+00, 2.78e-01 / 1.98e-01 | 20260506 19:22:55.508 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.62e-01 / 4.12e-01, 5.62e-02 / 2.62e-02 | 20260506 19:22:55.508 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:22:55.508 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:22:55.508 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:22:55.508 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 19:22:55.508 | INFO | PerfReport: 21.36%, Upload -- 2.07e+01 / 0.00e+00, 6.81e+00 / 0.00e+00 | 20260506 19:22:55.508 | INFO | PerfReport: 21.22%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 19:22:55.508 | INFO | PerfReport: 32.24%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 19:22:55.508 | INFO | PerfReport: 1.21%, LockUpload -- 1.17e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260506 19:22:55.508 | INFO | PerfReport: 1.29%, LockDownload -- 1.25e+00 / 0.00e+00, 3.16e+00 / 0.00e+00 | 20260506 19:22:55.508 | INFO | PerfReport: 22.66%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 19:22:55.508 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260506 19:22:55.611 | INFO | Starting a round of Baz IO | 20260506 19:22:58.747 | INFO | Populating RTMetrics report | 20260506 19:22:58.748 | INFO | RTMetrics report populated | 20260506 19:22:58.748 | INFO | Opening tmp RTMetrics json file | 20260506 19:22:58.796 | INFO | Serializing RTMetrics to json | 20260506 19:22:58.800 | INFO | Serialization done, swapping tmp file now... | 20260506 19:22:58.801 | INFO | RTMetrics done | 20260506 19:22:59.198 | INFO | Finished a round of Baz IO: Wrote 2.013 GiB in 3.587 seconds | 20260506 19:22:59.198 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.65% / 0.00% / 65.35% | 20260506 19:22:59.198 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.928402% / 7.94729e-06% / 6.75519e-05% | 20260506 19:22:59.198 | INFO | Baz IO overall: | 20260506 19:22:59.198 | INFO | BytesWritten : 2026055410729 | 20260506 19:22:59.198 | INFO | NumEvents : 979411600072 | 20260506 19:23:00.649 | INFO | Analyzing chunk frames = [3833856,3834368) | 20260506 19:23:00.649 | INFO | MemInfo: { | 20260506 19:23:00.649 | INFO | "gpu" : " 17 GB", | 20260506 19:23:00.649 | INFO | "huge" : "", | 20260506 19:23:00.649 | INFO | "huge_shared" : " 27 GB", | 20260506 19:23:00.649 | INFO | "malloc" : " 14 GB", | 20260506 19:23:00.649 | INFO | "pinned" : " 29 GB" | 20260506 19:23:00.649 | INFO | } | 20260506 19:23:00.649 | INFO | Perf report for the last 5.14261 seconds: | 20260506 19:23:00.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:23:00.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:00.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:00.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:00.649 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 19:23:00.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:00.649 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 19:23:00.649 | INFO | PRE_HQ : 60.2%, 1.9, 58.3ms, 0 | 20260506 19:23:00.649 | INFO | BAZWRITER : 69.4%, 4.7, 167.2ms, 0 | 20260506 19:23:00.649 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 19:23:00.649 | INFO | Uploaded 36GiB to GPU in 2.05216s (17.5425GiB/s) | 20260506 19:23:00.649 | INFO | Downloaded 38GiB to GPU in 2.19445s (17.3164GiB/s) | 20260506 19:23:05.788 | INFO | Analyzing chunk frames = [3834368,3834880) | 20260506 19:23:05.788 | INFO | MemInfo: { | 20260506 19:23:05.788 | INFO | "gpu" : " 17 GB", | 20260506 19:23:05.788 | INFO | "huge" : "", | 20260506 19:23:05.788 | INFO | "huge_shared" : " 27 GB", | 20260506 19:23:05.788 | INFO | "malloc" : " 14 GB", | 20260506 19:23:05.788 | INFO | "pinned" : " 29 GB" | 20260506 19:23:05.788 | INFO | } | 20260506 19:23:05.788 | INFO | Perf report for the last 5.13934 seconds: | 20260506 19:23:05.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:23:05.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:05.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:05.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:05.788 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 19:23:05.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:05.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:05.788 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 19:23:05.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:05.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:05.788 | INFO | Uploaded 36GiB to GPU in 2.01098s (17.9017GiB/s) | 20260506 19:23:05.788 | INFO | Downloaded 34GiB to GPU in 1.93472s (17.5736GiB/s) | 20260506 19:23:10.930 | INFO | Analyzing chunk frames = [3834880,3835392) | 20260506 19:23:10.930 | INFO | MemInfo: { | 20260506 19:23:10.930 | INFO | "gpu" : " 17 GB", | 20260506 19:23:10.930 | INFO | "huge" : "", | 20260506 19:23:10.930 | INFO | "huge_shared" : " 27 GB", | 20260506 19:23:10.930 | INFO | "malloc" : " 14 GB", | 20260506 19:23:10.930 | INFO | "pinned" : " 29 GB" | 20260506 19:23:10.930 | INFO | } | 20260506 19:23:10.930 | INFO | Perf report for the last 5.14163 seconds: | 20260506 19:23:10.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:23:10.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:10.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:10.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:10.930 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:23:10.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:10.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:10.930 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:23:10.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:10.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:10.930 | INFO | Uploaded 36GiB to GPU in 1.97678s (18.2114GiB/s) | 20260506 19:23:10.930 | INFO | Downloaded 34GiB to GPU in 1.92211s (17.6889GiB/s) | 20260506 19:23:16.064 | INFO | Analyzing chunk frames = [3835392,3835904) | 20260506 19:23:16.064 | INFO | MemInfo: { | 20260506 19:23:16.064 | INFO | "gpu" : " 17 GB", | 20260506 19:23:16.064 | INFO | "huge" : "", | 20260506 19:23:16.064 | INFO | "huge_shared" : " 27 GB", | 20260506 19:23:16.064 | INFO | "malloc" : " 14 GB", | 20260506 19:23:16.064 | INFO | "pinned" : " 29 GB" | 20260506 19:23:16.064 | INFO | } | 20260506 19:23:16.064 | INFO | Perf report for the last 5.13478 seconds: | 20260506 19:23:16.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:23:16.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:16.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:16.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:16.064 | INFO | ANALYSIS : 60.0%, 3.0, 92.8ms, 0 | 20260506 19:23:16.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:16.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:16.065 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:23:16.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:16.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:16.065 | INFO | Uploaded 36GiB to GPU in 1.9796s (18.1855GiB/s) | 20260506 19:23:16.065 | INFO | Downloaded 34GiB to GPU in 1.87602s (18.1235GiB/s) | 20260506 19:23:21.200 | INFO | Analyzing chunk frames = [3835904,3836416) | 20260506 19:23:21.200 | INFO | MemInfo: { | 20260506 19:23:21.200 | INFO | "gpu" : " 17 GB", | 20260506 19:23:21.200 | INFO | "huge" : "", | 20260506 19:23:21.200 | INFO | "huge_shared" : " 27 GB", | 20260506 19:23:21.200 | INFO | "malloc" : " 14 GB", | 20260506 19:23:21.200 | INFO | "pinned" : " 29 GB" | 20260506 19:23:21.200 | INFO | } | 20260506 19:23:21.200 | INFO | Perf report for the last 5.13608 seconds: | 20260506 19:23:21.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:23:21.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:21.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:21.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:21.201 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 19:23:21.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:21.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:21.201 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:23:21.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:21.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:21.201 | INFO | Uploaded 36GiB to GPU in 2.01303s (17.8835GiB/s) | 20260506 19:23:21.201 | INFO | Downloaded 34GiB to GPU in 1.91464s (17.7579GiB/s) | 20260506 19:23:26.335 | INFO | Analyzing chunk frames = [3836416,3836928) | 20260506 19:23:26.335 | INFO | MemInfo: { | 20260506 19:23:26.335 | INFO | "gpu" : " 17 GB", | 20260506 19:23:26.335 | INFO | "huge" : "", | 20260506 19:23:26.335 | INFO | "huge_shared" : " 27 GB", | 20260506 19:23:26.335 | INFO | "malloc" : " 14 GB", | 20260506 19:23:26.335 | INFO | "pinned" : " 29 GB" | 20260506 19:23:26.335 | INFO | } | 20260506 19:23:26.335 | INFO | Perf report for the last 5.1345 seconds: | 20260506 19:23:26.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:23:26.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:26.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:26.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:26.335 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 19:23:26.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:26.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:26.335 | INFO | PRE_HQ : 30.4%, 1.0, 15.7ms, 0 | 20260506 19:23:26.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:26.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:26.335 | INFO | Uploaded 36GiB to GPU in 2.02717s (17.7588GiB/s) | 20260506 19:23:26.335 | INFO | Downloaded 34GiB to GPU in 1.94767s (17.4568GiB/s) | 20260506 19:23:31.479 | INFO | Analyzing chunk frames = [3836928,3837440) | 20260506 19:23:31.479 | INFO | MemInfo: { | 20260506 19:23:31.479 | INFO | "gpu" : " 17 GB", | 20260506 19:23:31.479 | INFO | "huge" : "", | 20260506 19:23:31.479 | INFO | "huge_shared" : " 27 GB", | 20260506 19:23:31.479 | INFO | "malloc" : " 14 GB", | 20260506 19:23:31.479 | INFO | "pinned" : " 29 GB" | 20260506 19:23:31.479 | INFO | } | 20260506 19:23:31.479 | INFO | Perf report for the last 5.1443 seconds: | 20260506 19:23:31.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:23:31.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:31.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:31.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:31.479 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 19:23:31.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:31.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:31.480 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 19:23:31.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:31.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:31.480 | INFO | Uploaded 36GiB to GPU in 1.97907s (18.1904GiB/s) | 20260506 19:23:31.480 | INFO | Downloaded 34GiB to GPU in 2.00689s (16.9417GiB/s) | 20260506 19:23:36.622 | INFO | Analyzing chunk frames = [3837440,3837952) | 20260506 19:23:36.622 | INFO | MemInfo: { | 20260506 19:23:36.622 | INFO | "gpu" : " 17 GB", | 20260506 19:23:36.622 | INFO | "huge" : "", | 20260506 19:23:36.622 | INFO | "huge_shared" : " 27 GB", | 20260506 19:23:36.622 | INFO | "malloc" : " 14 GB", | 20260506 19:23:36.622 | INFO | "pinned" : " 29 GB" | 20260506 19:23:36.622 | INFO | } | 20260506 19:23:36.622 | INFO | Perf report for the last 5.14291 seconds: | 20260506 19:23:36.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:23:36.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:36.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:36.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:36.622 | INFO | ANALYSIS : 62.1%, 3.0, 96.8ms, 0 | 20260506 19:23:36.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:36.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:36.622 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 19:23:36.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:36.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:36.623 | INFO | Uploaded 36GiB to GPU in 2.04801s (17.578GiB/s) | 20260506 19:23:36.623 | INFO | Downloaded 34GiB to GPU in 1.97984s (17.1731GiB/s) | 20260506 19:23:36.740 | INFO | Starting a round of Baz IO | 20260506 19:23:39.870 | INFO | Populating RTMetrics report | 20260506 19:23:39.871 | INFO | RTMetrics report populated | 20260506 19:23:39.871 | INFO | Opening tmp RTMetrics json file | 20260506 19:23:39.895 | INFO | Serializing RTMetrics to json | 20260506 19:23:39.961 | INFO | Serialization done, swapping tmp file now... | 20260506 19:23:39.973 | INFO | RTMetrics done | 20260506 19:23:40.314 | INFO | Finished a round of Baz IO: Wrote 2.01 GiB in 3.573 seconds | 20260506 19:23:40.314 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.64% / 0.00% / 65.35% | 20260506 19:23:40.314 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.923852% / 7.94729e-06% / 6.35783e-05% | 20260506 19:23:40.314 | INFO | Baz IO overall: | 20260506 19:23:40.314 | INFO | BytesWritten : 2028213674725 | 20260506 19:23:40.314 | INFO | NumEvents : 980449192173 | 20260506 19:23:41.759 | INFO | Analyzing chunk frames = [3837952,3838464) | 20260506 19:23:41.759 | INFO | MemInfo: { | 20260506 19:23:41.759 | INFO | "gpu" : " 17 GB", | 20260506 19:23:41.759 | INFO | "huge" : "", | 20260506 19:23:41.759 | INFO | "huge_shared" : " 27 GB", | 20260506 19:23:41.759 | INFO | "malloc" : " 14 GB", | 20260506 19:23:41.759 | INFO | "pinned" : " 29 GB" | 20260506 19:23:41.759 | INFO | } | 20260506 19:23:41.759 | INFO | Perf report for the last 5.1367 seconds: | 20260506 19:23:41.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:23:41.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:41.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:41.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:41.759 | INFO | ANALYSIS : 62.7%, 2.9, 95.5ms, 0 | 20260506 19:23:41.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:41.759 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 19:23:41.759 | INFO | PRE_HQ : 60.0%, 1.8, 57.4ms, 0 | 20260506 19:23:41.759 | INFO | BAZWRITER : 69.2%, 4.8, 171.6ms, 0 | 20260506 19:23:41.759 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 19:23:41.759 | INFO | Uploaded 36GiB to GPU in 2.01513s (17.8649GiB/s) | 20260506 19:23:41.759 | INFO | Downloaded 38GiB to GPU in 2.21843s (17.1292GiB/s) | 20260506 19:23:46.906 | INFO | Analyzing chunk frames = [3838464,3838976) | 20260506 19:23:46.906 | INFO | MemInfo: { | 20260506 19:23:46.906 | INFO | "gpu" : " 17 GB", | 20260506 19:23:46.906 | INFO | "huge" : "", | 20260506 19:23:46.906 | INFO | "huge_shared" : " 27 GB", | 20260506 19:23:46.906 | INFO | "malloc" : " 14 GB", | 20260506 19:23:46.906 | INFO | "pinned" : " 29 GB" | 20260506 19:23:46.906 | INFO | } | 20260506 19:23:46.907 | INFO | Perf report for the last 5.14754 seconds: | 20260506 19:23:46.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:23:46.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:46.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:46.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:46.907 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 19:23:46.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:46.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:46.907 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:23:46.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:46.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:46.907 | INFO | Uploaded 36GiB to GPU in 1.98761s (18.1122GiB/s) | 20260506 19:23:46.907 | INFO | Downloaded 34GiB to GPU in 1.92101s (17.699GiB/s) | 20260506 19:23:52.042 | INFO | Analyzing chunk frames = [3838976,3839488) | 20260506 19:23:52.042 | INFO | MemInfo: { | 20260506 19:23:52.042 | INFO | "gpu" : " 17 GB", | 20260506 19:23:52.042 | INFO | "huge" : "", | 20260506 19:23:52.042 | INFO | "huge_shared" : " 27 GB", | 20260506 19:23:52.042 | INFO | "malloc" : " 14 GB", | 20260506 19:23:52.042 | INFO | "pinned" : " 29 GB" | 20260506 19:23:52.042 | INFO | } | 20260506 19:23:52.042 | INFO | Perf report for the last 5.1356 seconds: | 20260506 19:23:52.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:23:52.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:52.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:52.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:52.042 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:23:52.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:52.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:52.042 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:23:52.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:52.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:52.043 | INFO | Uploaded 36GiB to GPU in 1.96412s (18.3288GiB/s) | 20260506 19:23:52.043 | INFO | Downloaded 34GiB to GPU in 1.92396s (17.6719GiB/s) | 20260506 19:23:57.180 | INFO | Analyzing chunk frames = [3839488,3840000) | 20260506 19:23:57.180 | INFO | MemInfo: { | 20260506 19:23:57.180 | INFO | "gpu" : " 17 GB", | 20260506 19:23:57.180 | INFO | "huge" : "", | 20260506 19:23:57.180 | INFO | "huge_shared" : " 27 GB", | 20260506 19:23:57.180 | INFO | "malloc" : " 14 GB", | 20260506 19:23:57.180 | INFO | "pinned" : " 29 GB" | 20260506 19:23:57.180 | INFO | } | 20260506 19:23:57.180 | INFO | Perf report for the last 5.13832 seconds: | 20260506 19:23:57.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:23:57.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:57.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:57.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:57.181 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:23:57.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:57.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:57.181 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:23:57.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:57.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:23:57.181 | INFO | Uploaded 36GiB to GPU in 1.95613s (18.4037GiB/s) | 20260506 19:23:57.181 | INFO | Downloaded 34GiB to GPU in 1.87591s (18.1245GiB/s) | 20260506 19:24:02.316 | INFO | Analyzing chunk frames = [3840000,3840512) | 20260506 19:24:02.316 | INFO | MemInfo: { | 20260506 19:24:02.316 | INFO | "gpu" : " 17 GB", | 20260506 19:24:02.316 | INFO | "huge" : "", | 20260506 19:24:02.316 | INFO | "huge_shared" : " 27 GB", | 20260506 19:24:02.316 | INFO | "malloc" : " 14 GB", | 20260506 19:24:02.316 | INFO | "pinned" : " 29 GB" | 20260506 19:24:02.316 | INFO | } | 20260506 19:24:02.316 | INFO | Perf report for the last 5.13538 seconds: | 20260506 19:24:02.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:24:02.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:02.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:02.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:02.316 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 19:24:02.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:02.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:02.316 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:24:02.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:02.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:02.316 | INFO | Uploaded 36GiB to GPU in 1.98315s (18.1529GiB/s) | 20260506 19:24:02.316 | INFO | Downloaded 34GiB to GPU in 1.92095s (17.6996GiB/s) | 20260506 19:24:07.451 | INFO | Analyzing chunk frames = [3840512,3841024) | 20260506 19:24:07.451 | INFO | MemInfo: { | 20260506 19:24:07.451 | INFO | "gpu" : " 17 GB", | 20260506 19:24:07.451 | INFO | "huge" : "", | 20260506 19:24:07.451 | INFO | "huge_shared" : " 27 GB", | 20260506 19:24:07.451 | INFO | "malloc" : " 14 GB", | 20260506 19:24:07.451 | INFO | "pinned" : " 29 GB" | 20260506 19:24:07.451 | INFO | } | 20260506 19:24:07.451 | INFO | Perf report for the last 5.13525 seconds: | 20260506 19:24:07.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:24:07.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:07.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:07.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:07.451 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260506 19:24:07.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:07.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:07.451 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:24:07.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:07.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:07.451 | INFO | Uploaded 36GiB to GPU in 1.97923s (18.1889GiB/s) | 20260506 19:24:07.452 | INFO | Downloaded 34GiB to GPU in 1.93452s (17.5755GiB/s) | 20260506 19:24:12.596 | INFO | Analyzing chunk frames = [3841024,3841536) | 20260506 19:24:12.596 | INFO | MemInfo: { | 20260506 19:24:12.596 | INFO | "gpu" : " 17 GB", | 20260506 19:24:12.596 | INFO | "huge" : "", | 20260506 19:24:12.596 | INFO | "huge_shared" : " 27 GB", | 20260506 19:24:12.596 | INFO | "malloc" : " 14 GB", | 20260506 19:24:12.596 | INFO | "pinned" : " 29 GB" | 20260506 19:24:12.596 | INFO | } | 20260506 19:24:12.596 | INFO | Perf report for the last 5.14519 seconds: | 20260506 19:24:12.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:24:12.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:12.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:12.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:12.597 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 19:24:12.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:12.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:12.597 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 19:24:12.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:12.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:12.597 | INFO | Uploaded 36GiB to GPU in 2.00278s (17.975GiB/s) | 20260506 19:24:12.597 | INFO | Downloaded 34GiB to GPU in 2.00866s (16.9267GiB/s) | 20260506 19:24:17.741 | INFO | Analyzing chunk frames = [3841536,3842048) | 20260506 19:24:17.741 | INFO | MemInfo: { | 20260506 19:24:17.741 | INFO | "gpu" : " 17 GB", | 20260506 19:24:17.741 | INFO | "huge" : "", | 20260506 19:24:17.741 | INFO | "huge_shared" : " 27 GB", | 20260506 19:24:17.741 | INFO | "malloc" : " 14 GB", | 20260506 19:24:17.741 | INFO | "pinned" : " 29 GB" | 20260506 19:24:17.741 | INFO | } | 20260506 19:24:17.742 | INFO | Perf report for the last 5.14519 seconds: | 20260506 19:24:17.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:24:17.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:17.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:17.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:17.742 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 19:24:17.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:17.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:17.742 | INFO | PRE_HQ : 31.1%, 1.0, 16.2ms, 0 | 20260506 19:24:17.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:17.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:17.742 | INFO | Uploaded 36GiB to GPU in 2.05109s (17.5516GiB/s) | 20260506 19:24:17.742 | INFO | Downloaded 34GiB to GPU in 1.98405s (17.1366GiB/s) | 20260506 19:24:17.742 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:24:17.742 | INFO | Total mem usage: 18 GiB | 20260506 19:24:17.742 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:24:17.742 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -11.6%) | 20260506 19:24:17.742 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:24:17.742 | INFO | High Water Breakdown: | 20260506 19:24:17.742 | INFO | 61.9% : 9.99 GiB ( 87.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:24:17.742 | INFO | 37.9% : 6.12 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:24:17.742 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:24:17.742 | INFO | ------------------------------------------ | 20260506 19:24:17.742 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:24:17.742 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:24:17.742 | INFO | Total mem usage: 26.5 GiB | 20260506 19:24:17.742 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:24:17.742 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:24:17.742 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:24:17.742 | INFO | High Water Breakdown: | 20260506 19:24:17.742 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:24:17.742 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:24:17.742 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:24:17.742 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:24:17.742 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:24:17.742 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:24:17.742 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:24:17.742 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:24:17.742 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:24:17.742 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:24:17.742 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:24:17.742 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:24:17.742 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:24:17.742 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:24:17.742 | INFO | ------------------------------------------ | 20260506 19:24:17.742 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:24:17.742 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:24:17.742 | INFO | Total mem usage: 25.6 GiB | 20260506 19:24:17.742 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:24:17.742 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:24:17.743 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:24:17.743 | INFO | High Water Breakdown: | 20260506 19:24:17.743 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:24:17.743 | INFO | ------------------------------------------ | 20260506 19:24:17.743 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:24:17.743 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:24:17.743 | INFO | Total mem usage: 15.7 GiB | 20260506 19:24:17.743 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:24:17.743 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:24:17.743 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 19:24:17.743 | INFO | High Water Breakdown: | 20260506 19:24:17.743 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:24:17.743 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:24:17.743 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:24:17.743 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:24:17.743 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:24:17.743 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:24:17.743 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:24:17.743 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:24:17.743 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:24:17.743 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:24:17.743 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:24:17.743 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:24:17.743 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:24:17.743 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:24:17.743 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:24:17.743 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:24:17.743 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:24:17.743 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:24:17.743 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:24:17.743 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:24:17.743 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:24:17.743 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:24:17.743 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:24:17.743 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:24:17.743 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:24:17.743 | INFO | ------------------------------------------ | 20260506 19:24:17.743 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:24:17.743 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:24:17.743 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:24:17.743 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:24:17.743 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.36e+01 / 2.69e+00 | 20260506 19:24:17.743 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.02e-01, 1.05e-02 / 6.28e-03 | 20260506 19:24:17.743 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.41e+00, 5.54e-01 / 3.26e-01 | 20260506 19:24:17.743 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.49e+00, 1.87e-01 / 1.25e-01 | 20260506 19:24:17.743 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.45e+01, 3.96e+00 / 3.18e+00 | 20260506 19:24:17.743 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.43e+00 / 2.16e+00 | 20260506 19:24:17.743 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.25e+00 / 2.78e+00, 3.85e-01 / 2.84e-01 | 20260506 19:24:17.743 | INFO | PerfReport: 3.36%, Metrics -- 1.05e+00 / 1.37e+00, 2.76e-01 / 2.07e-01 | 20260506 19:24:17.743 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.61e-01 / 4.05e-01, 5.59e-02 / 2.39e-02 | 20260506 19:24:17.743 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:24:17.743 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:24:17.743 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:24:17.743 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 19:24:17.743 | INFO | PerfReport: 21.49%, Upload -- 2.09e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260506 19:24:17.743 | INFO | PerfReport: 21.18%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 19:24:17.743 | INFO | PerfReport: 32.21%, Compute -- 3.13e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 19:24:17.743 | INFO | PerfReport: 1.36%, LockUpload -- 1.32e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260506 19:24:17.743 | INFO | PerfReport: 1.19%, LockDownload -- 1.15e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260506 19:24:17.743 | INFO | PerfReport: 22.55%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 19:24:17.743 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.95e-02 / 0.00e+00 | 20260506 19:24:17.861 | INFO | Starting a round of Baz IO | 20260506 19:24:21.013 | INFO | Populating RTMetrics report | 20260506 19:24:21.013 | INFO | RTMetrics report populated | 20260506 19:24:21.013 | INFO | Opening tmp RTMetrics json file | 20260506 19:24:21.055 | INFO | Serializing RTMetrics to json | 20260506 19:24:21.059 | INFO | Serialization done, swapping tmp file now... | 20260506 19:24:21.061 | INFO | RTMetrics done | 20260506 19:24:21.451 | INFO | Finished a round of Baz IO: Wrote 2.009 GiB in 3.589 seconds | 20260506 19:24:21.451 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.64% / 0.00% / 65.36% | 20260506 19:24:21.451 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.921067% / 1.19209e-05% / 6.75519e-05% | 20260506 19:24:21.451 | INFO | Baz IO overall: | 20260506 19:24:21.451 | INFO | BytesWritten : 2030371333945 | 20260506 19:24:21.451 | INFO | NumEvents : 981486462388 | 20260506 19:24:22.878 | INFO | Analyzing chunk frames = [3842048,3842560) | 20260506 19:24:22.878 | INFO | MemInfo: { | 20260506 19:24:22.878 | INFO | "gpu" : " 17 GB", | 20260506 19:24:22.878 | INFO | "huge" : "", | 20260506 19:24:22.878 | INFO | "huge_shared" : " 27 GB", | 20260506 19:24:22.878 | INFO | "malloc" : " 14 GB", | 20260506 19:24:22.878 | INFO | "pinned" : " 29 GB" | 20260506 19:24:22.878 | INFO | } | 20260506 19:24:22.878 | INFO | Perf report for the last 5.13615 seconds: | 20260506 19:24:22.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:24:22.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:22.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:22.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:22.878 | INFO | ANALYSIS : 63.1%, 2.9, 94.4ms, 0 | 20260506 19:24:22.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:22.878 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260506 19:24:22.878 | INFO | PRE_HQ : 61.1%, 1.9, 61.5ms, 0 | 20260506 19:24:22.878 | INFO | BAZWRITER : 69.5%, 4.8, 172.4ms, 0 | 20260506 19:24:22.878 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 19:24:22.878 | INFO | Uploaded 36GiB to GPU in 2.07208s (17.3739GiB/s) | 20260506 19:24:22.878 | INFO | Downloaded 38GiB to GPU in 2.20418s (17.24GiB/s) | 20260506 19:24:28.015 | INFO | Analyzing chunk frames = [3842560,3843072) | 20260506 19:24:28.015 | INFO | MemInfo: { | 20260506 19:24:28.015 | INFO | "gpu" : " 17 GB", | 20260506 19:24:28.015 | INFO | "huge" : "", | 20260506 19:24:28.015 | INFO | "huge_shared" : " 27 GB", | 20260506 19:24:28.015 | INFO | "malloc" : " 14 GB", | 20260506 19:24:28.015 | INFO | "pinned" : " 29 GB" | 20260506 19:24:28.015 | INFO | } | 20260506 19:24:28.015 | INFO | Perf report for the last 5.13753 seconds: | 20260506 19:24:28.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:24:28.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:28.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:28.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:28.015 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 19:24:28.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:28.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:28.015 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:24:28.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:28.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:28.015 | INFO | Uploaded 36GiB to GPU in 1.98245s (18.1593GiB/s) | 20260506 19:24:28.016 | INFO | Downloaded 34GiB to GPU in 1.9361s (17.5611GiB/s) | 20260506 19:24:33.155 | INFO | Analyzing chunk frames = [3843072,3843584) | 20260506 19:24:33.155 | INFO | MemInfo: { | 20260506 19:24:33.155 | INFO | "gpu" : " 17 GB", | 20260506 19:24:33.155 | INFO | "huge" : "", | 20260506 19:24:33.155 | INFO | "huge_shared" : " 27 GB", | 20260506 19:24:33.155 | INFO | "malloc" : " 14 GB", | 20260506 19:24:33.156 | INFO | "pinned" : " 29 GB" | 20260506 19:24:33.156 | INFO | } | 20260506 19:24:33.156 | INFO | Perf report for the last 5.14051 seconds: | 20260506 19:24:33.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:24:33.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:33.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:33.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:33.156 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 19:24:33.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:33.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:33.156 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:24:33.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:33.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:33.156 | INFO | Uploaded 36GiB to GPU in 1.97745s (18.2053GiB/s) | 20260506 19:24:33.156 | INFO | Downloaded 34GiB to GPU in 1.92109s (17.6982GiB/s) | 20260506 19:24:38.291 | INFO | Analyzing chunk frames = [3843584,3844096) | 20260506 19:24:38.292 | INFO | MemInfo: { | 20260506 19:24:38.292 | INFO | "gpu" : " 17 GB", | 20260506 19:24:38.292 | INFO | "huge" : "", | 20260506 19:24:38.292 | INFO | "huge_shared" : " 27 GB", | 20260506 19:24:38.292 | INFO | "malloc" : " 14 GB", | 20260506 19:24:38.292 | INFO | "pinned" : " 29 GB" | 20260506 19:24:38.292 | INFO | } | 20260506 19:24:38.292 | INFO | Perf report for the last 5.13601 seconds: | 20260506 19:24:38.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:24:38.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:38.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:38.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:38.292 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:24:38.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:38.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:38.292 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:24:38.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:38.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:38.292 | INFO | Uploaded 36GiB to GPU in 1.93671s (18.5882GiB/s) | 20260506 19:24:38.292 | INFO | Downloaded 34GiB to GPU in 1.87389s (18.144GiB/s) | 20260506 19:24:43.429 | INFO | Analyzing chunk frames = [3844096,3844608) | 20260506 19:24:43.429 | INFO | MemInfo: { | 20260506 19:24:43.429 | INFO | "gpu" : " 17 GB", | 20260506 19:24:43.429 | INFO | "huge" : "", | 20260506 19:24:43.429 | INFO | "huge_shared" : " 27 GB", | 20260506 19:24:43.429 | INFO | "malloc" : " 14 GB", | 20260506 19:24:43.429 | INFO | "pinned" : " 29 GB" | 20260506 19:24:43.429 | INFO | } | 20260506 19:24:43.429 | INFO | Perf report for the last 5.13714 seconds: | 20260506 19:24:43.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:24:43.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:43.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:43.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:43.429 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 19:24:43.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:43.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:43.429 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:24:43.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:43.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:43.429 | INFO | Uploaded 36GiB to GPU in 1.99937s (18.0057GiB/s) | 20260506 19:24:43.429 | INFO | Downloaded 34GiB to GPU in 1.91318s (17.7715GiB/s) | 20260506 19:24:48.570 | INFO | Analyzing chunk frames = [3844608,3845120) | 20260506 19:24:48.571 | INFO | MemInfo: { | 20260506 19:24:48.571 | INFO | "gpu" : " 17 GB", | 20260506 19:24:48.571 | INFO | "huge" : "", | 20260506 19:24:48.571 | INFO | "huge_shared" : " 27 GB", | 20260506 19:24:48.571 | INFO | "malloc" : " 14 GB", | 20260506 19:24:48.571 | INFO | "pinned" : " 29 GB" | 20260506 19:24:48.571 | INFO | } | 20260506 19:24:48.571 | INFO | Perf report for the last 5.14189 seconds: | 20260506 19:24:48.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:24:48.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:48.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:48.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:48.571 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 19:24:48.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:48.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:48.571 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:24:48.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:48.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:48.571 | INFO | Uploaded 36GiB to GPU in 2.00567s (17.9491GiB/s) | 20260506 19:24:48.571 | INFO | Downloaded 34GiB to GPU in 1.96731s (17.2825GiB/s) | 20260506 19:24:53.710 | INFO | Analyzing chunk frames = [3845120,3845632) | 20260506 19:24:53.710 | INFO | MemInfo: { | 20260506 19:24:53.710 | INFO | "gpu" : " 17 GB", | 20260506 19:24:53.710 | INFO | "huge" : "", | 20260506 19:24:53.710 | INFO | "huge_shared" : " 27 GB", | 20260506 19:24:53.710 | INFO | "malloc" : " 14 GB", | 20260506 19:24:53.710 | INFO | "pinned" : " 29 GB" | 20260506 19:24:53.710 | INFO | } | 20260506 19:24:53.711 | INFO | Perf report for the last 5.1398 seconds: | 20260506 19:24:53.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:24:53.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:53.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:53.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:53.711 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:24:53.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:53.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:53.711 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:24:53.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:53.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:53.711 | INFO | Uploaded 36GiB to GPU in 1.97992s (18.1825GiB/s) | 20260506 19:24:53.711 | INFO | Downloaded 34GiB to GPU in 2.00259s (16.978GiB/s) | 20260506 19:24:58.856 | INFO | Analyzing chunk frames = [3845632,3846144) | 20260506 19:24:58.856 | INFO | MemInfo: { | 20260506 19:24:58.856 | INFO | "gpu" : " 17 GB", | 20260506 19:24:58.856 | INFO | "huge" : "", | 20260506 19:24:58.856 | INFO | "huge_shared" : " 27 GB", | 20260506 19:24:58.856 | INFO | "malloc" : " 14 GB", | 20260506 19:24:58.856 | INFO | "pinned" : " 29 GB" | 20260506 19:24:58.856 | INFO | } | 20260506 19:24:58.856 | INFO | Perf report for the last 5.14564 seconds: | 20260506 19:24:58.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:24:58.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:58.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:58.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:58.856 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 19:24:58.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:58.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:58.856 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 19:24:58.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:58.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:24:58.856 | INFO | Uploaded 36GiB to GPU in 1.98264s (18.1576GiB/s) | 20260506 19:24:58.857 | INFO | Downloaded 34GiB to GPU in 1.98686s (17.1124GiB/s) | 20260506 19:24:58.959 | INFO | Starting a round of Baz IO | 20260506 19:25:02.101 | INFO | Populating RTMetrics report | 20260506 19:25:02.101 | INFO | RTMetrics report populated | 20260506 19:25:02.101 | INFO | Opening tmp RTMetrics json file | 20260506 19:25:02.102 | INFO | Serializing RTMetrics to json | 20260506 19:25:02.104 | INFO | Serialization done, swapping tmp file now... | 20260506 19:25:02.105 | INFO | RTMetrics done | 20260506 19:25:02.284 | INFO | Finished a round of Baz IO: Wrote 2.011 GiB in 3.325 seconds | 20260506 19:25:02.284 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.63% / 0.00% / 65.36% | 20260506 19:25:02.284 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.922716% / 7.94729e-06% / 5.16574e-05% | 20260506 19:25:02.284 | INFO | Baz IO overall: | 20260506 19:25:02.284 | INFO | BytesWritten : 2032530653582 | 20260506 19:25:02.284 | INFO | NumEvents : 982525161536 | 20260506 19:25:03.999 | INFO | Analyzing chunk frames = [3846144,3846656) | 20260506 19:25:03.999 | INFO | MemInfo: { | 20260506 19:25:03.999 | INFO | "gpu" : " 17 GB", | 20260506 19:25:03.999 | INFO | "huge" : "", | 20260506 19:25:03.999 | INFO | "huge_shared" : " 27 GB", | 20260506 19:25:04.000 | INFO | "malloc" : " 14 GB", | 20260506 19:25:04.000 | INFO | "pinned" : " 29 GB" | 20260506 19:25:04.000 | INFO | } | 20260506 19:25:04.000 | INFO | Perf report for the last 5.14331 seconds: | 20260506 19:25:04.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:25:04.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:04.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:04.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:04.000 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 19:25:04.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:04.000 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 19:25:04.000 | INFO | PRE_HQ : 60.1%, 1.8, 57.6ms, 0 | 20260506 19:25:04.000 | INFO | BAZWRITER : 64.3%, 4.8, 158.3ms, 0 | 20260506 19:25:04.000 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 19:25:04.000 | INFO | Uploaded 36GiB to GPU in 2.06334s (17.4475GiB/s) | 20260506 19:25:04.000 | INFO | Downloaded 38GiB to GPU in 2.20562s (17.2287GiB/s) | 20260506 19:25:09.132 | INFO | Analyzing chunk frames = [3846656,3847168) | 20260506 19:25:09.132 | INFO | MemInfo: { | 20260506 19:25:09.132 | INFO | "gpu" : " 17 GB", | 20260506 19:25:09.132 | INFO | "huge" : "", | 20260506 19:25:09.132 | INFO | "huge_shared" : " 27 GB", | 20260506 19:25:09.132 | INFO | "malloc" : " 14 GB", | 20260506 19:25:09.132 | INFO | "pinned" : " 29 GB" | 20260506 19:25:09.132 | INFO | } | 20260506 19:25:09.133 | INFO | Perf report for the last 5.13306 seconds: | 20260506 19:25:09.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:25:09.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:09.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:09.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:09.133 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 19:25:09.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:09.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:09.133 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:25:09.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:09.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:09.133 | INFO | Uploaded 36GiB to GPU in 2.00747s (17.933GiB/s) | 20260506 19:25:09.133 | INFO | Downloaded 34GiB to GPU in 1.92571s (17.6558GiB/s) | 20260506 19:25:14.268 | INFO | Analyzing chunk frames = [3847168,3847680) | 20260506 19:25:14.268 | INFO | MemInfo: { | 20260506 19:25:14.268 | INFO | "gpu" : " 17 GB", | 20260506 19:25:14.268 | INFO | "huge" : "", | 20260506 19:25:14.268 | INFO | "huge_shared" : " 27 GB", | 20260506 19:25:14.268 | INFO | "malloc" : " 14 GB", | 20260506 19:25:14.268 | INFO | "pinned" : " 29 GB" | 20260506 19:25:14.268 | INFO | } | 20260506 19:25:14.268 | INFO | Perf report for the last 5.13579 seconds: | 20260506 19:25:14.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:25:14.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:14.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:14.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:14.268 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260506 19:25:14.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:14.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:14.268 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:25:14.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:14.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:14.269 | INFO | Uploaded 36GiB to GPU in 1.97158s (18.2595GiB/s) | 20260506 19:25:14.269 | INFO | Downloaded 34GiB to GPU in 1.91612s (17.7442GiB/s) | 20260506 19:25:19.403 | INFO | Analyzing chunk frames = [3847680,3848192) | 20260506 19:25:19.403 | INFO | MemInfo: { | 20260506 19:25:19.403 | INFO | "gpu" : " 17 GB", | 20260506 19:25:19.403 | INFO | "huge" : "", | 20260506 19:25:19.403 | INFO | "huge_shared" : " 27 GB", | 20260506 19:25:19.403 | INFO | "malloc" : " 14 GB", | 20260506 19:25:19.403 | INFO | "pinned" : " 29 GB" | 20260506 19:25:19.403 | INFO | } | 20260506 19:25:19.403 | INFO | Perf report for the last 5.13463 seconds: | 20260506 19:25:19.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:25:19.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:19.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:19.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:19.403 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 19:25:19.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:19.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:19.403 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:25:19.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:19.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:19.404 | INFO | Uploaded 36GiB to GPU in 1.94648s (18.4949GiB/s) | 20260506 19:25:19.404 | INFO | Downloaded 34GiB to GPU in 1.86958s (18.1859GiB/s) | 20260506 19:25:24.545 | INFO | Analyzing chunk frames = [3848192,3848704) | 20260506 19:25:24.545 | INFO | MemInfo: { | 20260506 19:25:24.545 | INFO | "gpu" : " 17 GB", | 20260506 19:25:24.545 | INFO | "huge" : "", | 20260506 19:25:24.545 | INFO | "huge_shared" : " 27 GB", | 20260506 19:25:24.545 | INFO | "malloc" : " 14 GB", | 20260506 19:25:24.545 | INFO | "pinned" : " 29 GB" | 20260506 19:25:24.545 | INFO | } | 20260506 19:25:24.545 | INFO | Perf report for the last 5.14215 seconds: | 20260506 19:25:24.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:25:24.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:24.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:24.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:24.545 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 19:25:24.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:24.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:24.545 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:25:24.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:24.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:24.546 | INFO | Uploaded 36GiB to GPU in 2.02472s (17.7803GiB/s) | 20260506 19:25:24.546 | INFO | Downloaded 34GiB to GPU in 1.91251s (17.7777GiB/s) | 20260506 19:25:29.685 | INFO | Analyzing chunk frames = [3848704,3849216) | 20260506 19:25:29.685 | INFO | MemInfo: { | 20260506 19:25:29.685 | INFO | "gpu" : " 17 GB", | 20260506 19:25:29.685 | INFO | "huge" : "", | 20260506 19:25:29.685 | INFO | "huge_shared" : " 27 GB", | 20260506 19:25:29.685 | INFO | "malloc" : " 14 GB", | 20260506 19:25:29.685 | INFO | "pinned" : " 29 GB" | 20260506 19:25:29.685 | INFO | } | 20260506 19:25:29.685 | INFO | Perf report for the last 5.13974 seconds: | 20260506 19:25:29.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:25:29.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:29.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:29.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:29.685 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 19:25:29.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:29.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:29.685 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 19:25:29.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:29.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:29.685 | INFO | Uploaded 36GiB to GPU in 2.07771s (17.3268GiB/s) | 20260506 19:25:29.685 | INFO | Downloaded 34GiB to GPU in 1.96407s (17.311GiB/s) | 20260506 19:25:34.821 | INFO | Analyzing chunk frames = [3849216,3849728) | 20260506 19:25:34.821 | INFO | MemInfo: { | 20260506 19:25:34.821 | INFO | "gpu" : " 17 GB", | 20260506 19:25:34.821 | INFO | "huge" : "", | 20260506 19:25:34.821 | INFO | "huge_shared" : " 27 GB", | 20260506 19:25:34.822 | INFO | "malloc" : " 14 GB", | 20260506 19:25:34.822 | INFO | "pinned" : " 29 GB" | 20260506 19:25:34.822 | INFO | } | 20260506 19:25:34.822 | INFO | Perf report for the last 5.1366 seconds: | 20260506 19:25:34.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:25:34.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:34.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:34.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:34.822 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 19:25:34.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:34.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:34.822 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 19:25:34.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:34.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:34.822 | INFO | Uploaded 36GiB to GPU in 2.01692s (17.849GiB/s) | 20260506 19:25:34.822 | INFO | Downloaded 34GiB to GPU in 1.97546s (17.2111GiB/s) | 20260506 19:25:39.963 | INFO | Analyzing chunk frames = [3849728,3850240) | 20260506 19:25:39.963 | INFO | MemInfo: { | 20260506 19:25:39.963 | INFO | "gpu" : " 17 GB", | 20260506 19:25:39.963 | INFO | "huge" : "", | 20260506 19:25:39.963 | INFO | "huge_shared" : " 27 GB", | 20260506 19:25:39.963 | INFO | "malloc" : " 14 GB", | 20260506 19:25:39.963 | INFO | "pinned" : " 29 GB" | 20260506 19:25:39.964 | INFO | } | 20260506 19:25:39.964 | INFO | Perf report for the last 5.14192 seconds: | 20260506 19:25:39.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:25:39.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:39.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:39.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:39.964 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:25:39.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:39.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:39.964 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 19:25:39.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:39.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:39.964 | INFO | Uploaded 36GiB to GPU in 2.0071s (17.9363GiB/s) | 20260506 19:25:39.964 | INFO | Downloaded 34GiB to GPU in 1.97435s (17.2209GiB/s) | 20260506 19:25:39.964 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:25:39.964 | INFO | Total mem usage: 18 GiB | 20260506 19:25:39.964 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:25:39.964 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.2%) | 20260506 19:25:39.964 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:25:39.964 | INFO | High Water Breakdown: | 20260506 19:25:39.964 | INFO | 61.7% : 9.82 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:25:39.964 | INFO | 38.1% : 6.07 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:25:39.964 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:25:39.964 | INFO | ------------------------------------------ | 20260506 19:25:39.964 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:25:39.964 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:25:39.964 | INFO | Total mem usage: 26.5 GiB | 20260506 19:25:39.964 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:25:39.964 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:25:39.964 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:25:39.964 | INFO | High Water Breakdown: | 20260506 19:25:39.964 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:25:39.964 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:25:39.964 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:25:39.964 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:25:39.964 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:25:39.964 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:25:39.964 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:25:39.964 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:25:39.964 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:25:39.964 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:25:39.964 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:25:39.964 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:25:39.964 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:25:39.964 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:25:39.964 | INFO | ------------------------------------------ | 20260506 19:25:39.964 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:25:39.964 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:25:39.964 | INFO | Total mem usage: 25.6 GiB | 20260506 19:25:39.964 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:25:39.964 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:25:39.964 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:25:39.964 | INFO | High Water Breakdown: | 20260506 19:25:39.964 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:25:39.964 | INFO | ------------------------------------------ | 20260506 19:25:39.964 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:25:39.965 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:25:39.965 | INFO | Total mem usage: 15.7 GiB | 20260506 19:25:39.965 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:25:39.965 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:25:39.965 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:25:39.965 | INFO | High Water Breakdown: | 20260506 19:25:39.965 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:25:39.965 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:25:39.965 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:25:39.965 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:25:39.965 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:25:39.965 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:25:39.965 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:25:39.965 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:25:39.965 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:25:39.965 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:25:39.965 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:25:39.965 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:25:39.965 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:25:39.965 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:25:39.965 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:25:39.965 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:25:39.965 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:25:39.965 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:25:39.965 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:25:39.965 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:25:39.965 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:25:39.965 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:25:39.965 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:25:39.965 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:25:39.965 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:25:39.965 | INFO | ------------------------------------------ | 20260506 19:25:39.965 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:25:39.965 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:25:39.965 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:25:39.965 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:25:39.965 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.61e+01, 1.34e+01 / 2.32e+00 | 20260506 19:25:39.965 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.02e-01, 1.05e-02 / 5.58e-03 | 20260506 19:25:39.965 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.43e+00, 5.51e-01 / 3.41e-01 | 20260506 19:25:39.965 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.49e+00, 1.83e-01 / 1.25e-01 | 20260506 19:25:39.965 | INFO | PerfReport: 15.98%, DME -- 4.00e+01 / 4.47e+01, 4.13e+00 / 1.94e+00 | 20260506 19:25:39.965 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.51e+00 / 1.64e+00 | 20260506 19:25:39.965 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.23e+00 / 2.79e+00, 3.75e-01 / 2.39e-01 | 20260506 19:25:39.965 | INFO | PerfReport: 3.34%, Metrics -- 1.05e+00 / 1.39e+00, 2.75e-01 / 1.63e-01 | 20260506 19:25:39.965 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.61e-01 / 4.15e-01, 5.75e-02 / 2.79e-02 | 20260506 19:25:39.965 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:25:39.965 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:25:39.965 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:25:39.965 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 19:25:39.965 | INFO | PerfReport: 21.52%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 19:25:39.965 | INFO | PerfReport: 21.19%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 19:25:39.965 | INFO | PerfReport: 32.24%, Compute -- 3.13e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 19:25:39.965 | INFO | PerfReport: 1.41%, LockUpload -- 1.37e+00 / 0.00e+00, 5.33e+00 / 0.00e+00 | 20260506 19:25:39.965 | INFO | PerfReport: 1.10%, LockDownload -- 1.06e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260506 19:25:39.965 | INFO | PerfReport: 22.52%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 19:25:39.965 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.30e-02 / 0.00e+00 | 20260506 19:25:40.071 | INFO | Starting a round of Baz IO | 20260506 19:25:43.214 | INFO | Populating RTMetrics report | 20260506 19:25:43.214 | INFO | RTMetrics report populated | 20260506 19:25:43.214 | INFO | Opening tmp RTMetrics json file | 20260506 19:25:43.220 | INFO | Serializing RTMetrics to json | 20260506 19:25:43.243 | INFO | Serialization done, swapping tmp file now... | 20260506 19:25:43.245 | INFO | RTMetrics done | 20260506 19:25:43.514 | INFO | Finished a round of Baz IO: Wrote 2.01 GiB in 3.443 seconds | 20260506 19:25:43.514 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.63% / 0.00% / 65.37% | 20260506 19:25:43.514 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.9196% / 7.94729e-06% / 5.96046e-05% | 20260506 19:25:43.514 | INFO | Baz IO overall: | 20260506 19:25:43.514 | INFO | BytesWritten : 2034689294586 | 20260506 19:25:43.514 | INFO | NumEvents : 983563500099 | 20260506 19:25:45.103 | INFO | Analyzing chunk frames = [3850240,3850752) | 20260506 19:25:45.103 | INFO | MemInfo: { | 20260506 19:25:45.103 | INFO | "gpu" : " 17 GB", | 20260506 19:25:45.103 | INFO | "huge" : "", | 20260506 19:25:45.103 | INFO | "huge_shared" : " 27 GB", | 20260506 19:25:45.103 | INFO | "malloc" : " 14 GB", | 20260506 19:25:45.103 | INFO | "pinned" : " 29 GB" | 20260506 19:25:45.103 | INFO | } | 20260506 19:25:45.103 | INFO | Perf report for the last 5.13962 seconds: | 20260506 19:25:45.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:25:45.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:45.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:45.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:45.103 | INFO | ANALYSIS : 62.7%, 2.9, 95.5ms, 0 | 20260506 19:25:45.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:45.103 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 19:25:45.103 | INFO | PRE_HQ : 60.2%, 1.8, 57.7ms, 0 | 20260506 19:25:45.103 | INFO | BAZWRITER : 66.7%, 4.8, 163.4ms, 0 | 20260506 19:25:45.103 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 19:25:45.104 | INFO | Uploaded 36GiB to GPU in 2.04228s (17.6273GiB/s) | 20260506 19:25:45.104 | INFO | Downloaded 38GiB to GPU in 2.19274s (17.3299GiB/s) | 20260506 19:25:50.246 | INFO | Analyzing chunk frames = [3850752,3851264) | 20260506 19:25:50.246 | INFO | MemInfo: { | 20260506 19:25:50.246 | INFO | "gpu" : " 17 GB", | 20260506 19:25:50.246 | INFO | "huge" : "", | 20260506 19:25:50.246 | INFO | "huge_shared" : " 27 GB", | 20260506 19:25:50.246 | INFO | "malloc" : " 14 GB", | 20260506 19:25:50.247 | INFO | "pinned" : " 29 GB" | 20260506 19:25:50.247 | INFO | } | 20260506 19:25:50.247 | INFO | Perf report for the last 5.14334 seconds: | 20260506 19:25:50.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:25:50.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:50.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:50.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:50.247 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 19:25:50.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:50.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:50.247 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:25:50.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:50.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:50.247 | INFO | Uploaded 36GiB to GPU in 2.01591s (17.858GiB/s) | 20260506 19:25:50.247 | INFO | Downloaded 34GiB to GPU in 1.92728s (17.6414GiB/s) | 20260506 19:25:55.384 | INFO | Analyzing chunk frames = [3851264,3851776) | 20260506 19:25:55.384 | INFO | MemInfo: { | 20260506 19:25:55.384 | INFO | "gpu" : " 17 GB", | 20260506 19:25:55.384 | INFO | "huge" : "", | 20260506 19:25:55.384 | INFO | "huge_shared" : " 27 GB", | 20260506 19:25:55.384 | INFO | "malloc" : " 14 GB", | 20260506 19:25:55.384 | INFO | "pinned" : " 29 GB" | 20260506 19:25:55.384 | INFO | } | 20260506 19:25:55.384 | INFO | Perf report for the last 5.1371 seconds: | 20260506 19:25:55.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:25:55.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:55.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:55.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:55.384 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 19:25:55.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:55.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:55.384 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:25:55.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:55.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:25:55.384 | INFO | Uploaded 36GiB to GPU in 1.98133s (18.1696GiB/s) | 20260506 19:25:55.384 | INFO | Downloaded 34GiB to GPU in 1.91603s (17.745GiB/s) | 20260506 19:26:00.521 | INFO | Analyzing chunk frames = [3851776,3852288) | 20260506 19:26:00.521 | INFO | MemInfo: { | 20260506 19:26:00.521 | INFO | "gpu" : " 17 GB", | 20260506 19:26:00.521 | INFO | "huge" : "", | 20260506 19:26:00.521 | INFO | "huge_shared" : " 27 GB", | 20260506 19:26:00.521 | INFO | "malloc" : " 14 GB", | 20260506 19:26:00.521 | INFO | "pinned" : " 29 GB" | 20260506 19:26:00.521 | INFO | } | 20260506 19:26:00.521 | INFO | Perf report for the last 5.13732 seconds: | 20260506 19:26:00.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:26:00.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:00.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:00.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:00.521 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 19:26:00.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:00.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:00.521 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:26:00.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:00.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:00.521 | INFO | Uploaded 36GiB to GPU in 1.94203s (18.5373GiB/s) | 20260506 19:26:00.521 | INFO | Downloaded 34GiB to GPU in 1.86768s (18.2044GiB/s) | 20260506 19:26:05.664 | INFO | Analyzing chunk frames = [3852288,3852800) | 20260506 19:26:05.664 | INFO | MemInfo: { | 20260506 19:26:05.664 | INFO | "gpu" : " 17 GB", | 20260506 19:26:05.664 | INFO | "huge" : "", | 20260506 19:26:05.664 | INFO | "huge_shared" : " 27 GB", | 20260506 19:26:05.664 | INFO | "malloc" : " 14 GB", | 20260506 19:26:05.664 | INFO | "pinned" : " 29 GB" | 20260506 19:26:05.664 | INFO | } | 20260506 19:26:05.664 | INFO | Perf report for the last 5.14349 seconds: | 20260506 19:26:05.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:26:05.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:05.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:05.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:05.665 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 19:26:05.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:05.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:05.665 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:26:05.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:05.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:05.665 | INFO | Uploaded 36GiB to GPU in 2.01927s (17.8283GiB/s) | 20260506 19:26:05.665 | INFO | Downloaded 34GiB to GPU in 1.90811s (17.8187GiB/s) | 20260506 19:26:10.799 | INFO | Analyzing chunk frames = [3852800,3853312) | 20260506 19:26:10.799 | INFO | MemInfo: { | 20260506 19:26:10.799 | INFO | "gpu" : " 17 GB", | 20260506 19:26:10.799 | INFO | "huge" : "", | 20260506 19:26:10.799 | INFO | "huge_shared" : " 27 GB", | 20260506 19:26:10.799 | INFO | "malloc" : " 14 GB", | 20260506 19:26:10.799 | INFO | "pinned" : " 29 GB" | 20260506 19:26:10.799 | INFO | } | 20260506 19:26:10.799 | INFO | Perf report for the last 5.13474 seconds: | 20260506 19:26:10.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:26:10.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:10.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:10.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:10.799 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 19:26:10.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:10.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:10.799 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 19:26:10.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:10.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:10.800 | INFO | Uploaded 36GiB to GPU in 2.04319s (17.6195GiB/s) | 20260506 19:26:10.800 | INFO | Downloaded 34GiB to GPU in 1.94302s (17.4985GiB/s) | 20260506 19:26:15.939 | INFO | Analyzing chunk frames = [3853312,3853824) | 20260506 19:26:15.939 | INFO | MemInfo: { | 20260506 19:26:15.939 | INFO | "gpu" : " 17 GB", | 20260506 19:26:15.939 | INFO | "huge" : "", | 20260506 19:26:15.939 | INFO | "huge_shared" : " 27 GB", | 20260506 19:26:15.939 | INFO | "malloc" : " 14 GB", | 20260506 19:26:15.939 | INFO | "pinned" : " 29 GB" | 20260506 19:26:15.939 | INFO | } | 20260506 19:26:15.939 | INFO | Perf report for the last 5.14008 seconds: | 20260506 19:26:15.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:26:15.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:15.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:15.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:15.939 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 19:26:15.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:15.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:15.939 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:26:15.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:15.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:15.940 | INFO | Uploaded 36GiB to GPU in 1.99486s (18.0464GiB/s) | 20260506 19:26:15.940 | INFO | Downloaded 34GiB to GPU in 2.00993s (16.916GiB/s) | 20260506 19:26:21.087 | INFO | Analyzing chunk frames = [3853824,3854336) | 20260506 19:26:21.087 | INFO | MemInfo: { | 20260506 19:26:21.087 | INFO | "gpu" : " 17 GB", | 20260506 19:26:21.087 | INFO | "huge" : "", | 20260506 19:26:21.087 | INFO | "huge_shared" : " 27 GB", | 20260506 19:26:21.087 | INFO | "malloc" : " 14 GB", | 20260506 19:26:21.087 | INFO | "pinned" : " 29 GB" | 20260506 19:26:21.087 | INFO | } | 20260506 19:26:21.087 | INFO | Perf report for the last 5.14771 seconds: | 20260506 19:26:21.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:26:21.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:21.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:21.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:21.087 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 19:26:21.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:21.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:21.087 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 19:26:21.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:21.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:21.087 | INFO | Uploaded 36GiB to GPU in 1.99808s (18.0173GiB/s) | 20260506 19:26:21.087 | INFO | Downloaded 34GiB to GPU in 1.98693s (17.1118GiB/s) | 20260506 19:26:21.198 | INFO | Starting a round of Baz IO | 20260506 19:26:24.334 | INFO | Populating RTMetrics report | 20260506 19:26:24.334 | INFO | RTMetrics report populated | 20260506 19:26:24.334 | INFO | Opening tmp RTMetrics json file | 20260506 19:26:24.335 | INFO | Serializing RTMetrics to json | 20260506 19:26:24.337 | INFO | Serialization done, swapping tmp file now... | 20260506 19:26:24.337 | INFO | RTMetrics done | 20260506 19:26:24.481 | INFO | Finished a round of Baz IO: Wrote 2.008 GiB in 3.282 seconds | 20260506 19:26:24.481 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.62% / 0.00% / 65.37% | 20260506 19:26:24.481 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.916306% / 1.98682e-05% / 3.97364e-05% | 20260506 19:26:24.481 | INFO | Baz IO overall: | 20260506 19:26:24.481 | INFO | BytesWritten : 2036845126818 | 20260506 19:26:24.481 | INFO | NumEvents : 984599403288 | 20260506 19:26:26.220 | INFO | Analyzing chunk frames = [3854336,3854848) | 20260506 19:26:26.220 | INFO | MemInfo: { | 20260506 19:26:26.220 | INFO | "gpu" : " 17 GB", | 20260506 19:26:26.220 | INFO | "huge" : "", | 20260506 19:26:26.220 | INFO | "huge_shared" : " 27 GB", | 20260506 19:26:26.220 | INFO | "malloc" : " 14 GB", | 20260506 19:26:26.220 | INFO | "pinned" : " 29 GB" | 20260506 19:26:26.220 | INFO | } | 20260506 19:26:26.220 | INFO | Perf report for the last 5.13278 seconds: | 20260506 19:26:26.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:26:26.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:26.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:26.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:26.220 | INFO | ANALYSIS : 62.7%, 2.9, 94.8ms, 0 | 20260506 19:26:26.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:26.220 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 19:26:26.220 | INFO | PRE_HQ : 60.7%, 1.9, 60.0ms, 0 | 20260506 19:26:26.220 | INFO | BAZWRITER : 63.7%, 4.7, 154.6ms, 0 | 20260506 19:26:26.220 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260506 19:26:26.220 | INFO | Uploaded 36GiB to GPU in 2.03824s (17.6623GiB/s) | 20260506 19:26:26.220 | INFO | Downloaded 38GiB to GPU in 2.21109s (17.1861GiB/s) | 20260506 19:26:31.359 | INFO | Analyzing chunk frames = [3854848,3855360) | 20260506 19:26:31.359 | INFO | MemInfo: { | 20260506 19:26:31.359 | INFO | "gpu" : " 17 GB", | 20260506 19:26:31.359 | INFO | "huge" : "", | 20260506 19:26:31.359 | INFO | "huge_shared" : " 27 GB", | 20260506 19:26:31.359 | INFO | "malloc" : " 14 GB", | 20260506 19:26:31.359 | INFO | "pinned" : " 29 GB" | 20260506 19:26:31.359 | INFO | } | 20260506 19:26:31.359 | INFO | Perf report for the last 5.13908 seconds: | 20260506 19:26:31.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:26:31.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:31.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:31.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:31.359 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:26:31.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:31.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:31.359 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 19:26:31.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:31.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:31.359 | INFO | Uploaded 36GiB to GPU in 1.99387s (18.0553GiB/s) | 20260506 19:26:31.359 | INFO | Downloaded 34GiB to GPU in 1.93696s (17.5533GiB/s) | 20260506 19:26:36.499 | INFO | Analyzing chunk frames = [3855360,3855872) | 20260506 19:26:36.499 | INFO | MemInfo: { | 20260506 19:26:36.499 | INFO | "gpu" : " 17 GB", | 20260506 19:26:36.499 | INFO | "huge" : "", | 20260506 19:26:36.499 | INFO | "huge_shared" : " 27 GB", | 20260506 19:26:36.499 | INFO | "malloc" : " 14 GB", | 20260506 19:26:36.500 | INFO | "pinned" : " 29 GB" | 20260506 19:26:36.500 | INFO | } | 20260506 19:26:36.500 | INFO | Perf report for the last 5.1406 seconds: | 20260506 19:26:36.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:26:36.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:36.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:36.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:36.500 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 19:26:36.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:36.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:36.500 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:26:36.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:36.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:36.500 | INFO | Uploaded 36GiB to GPU in 1.96637s (18.3079GiB/s) | 20260506 19:26:36.500 | INFO | Downloaded 34GiB to GPU in 1.92001s (17.7082GiB/s) | 20260506 19:26:41.637 | INFO | Analyzing chunk frames = [3855872,3856384) | 20260506 19:26:41.637 | INFO | MemInfo: { | 20260506 19:26:41.637 | INFO | "gpu" : " 17 GB", | 20260506 19:26:41.637 | INFO | "huge" : "", | 20260506 19:26:41.637 | INFO | "huge_shared" : " 27 GB", | 20260506 19:26:41.637 | INFO | "malloc" : " 14 GB", | 20260506 19:26:41.638 | INFO | "pinned" : " 29 GB" | 20260506 19:26:41.638 | INFO | } | 20260506 19:26:41.638 | INFO | Perf report for the last 5.13801 seconds: | 20260506 19:26:41.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:26:41.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:41.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:41.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:41.638 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:26:41.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:41.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:41.638 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:26:41.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:41.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:41.638 | INFO | Uploaded 36GiB to GPU in 1.93542s (18.6006GiB/s) | 20260506 19:26:41.638 | INFO | Downloaded 34GiB to GPU in 1.87382s (18.1448GiB/s) | 20260506 19:26:46.774 | INFO | Analyzing chunk frames = [3856384,3856896) | 20260506 19:26:46.774 | INFO | MemInfo: { | 20260506 19:26:46.774 | INFO | "gpu" : " 17 GB", | 20260506 19:26:46.774 | INFO | "huge" : "", | 20260506 19:26:46.774 | INFO | "huge_shared" : " 27 GB", | 20260506 19:26:46.774 | INFO | "malloc" : " 14 GB", | 20260506 19:26:46.774 | INFO | "pinned" : " 29 GB" | 20260506 19:26:46.774 | INFO | } | 20260506 19:26:46.774 | INFO | Perf report for the last 5.13661 seconds: | 20260506 19:26:46.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:26:46.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:46.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:46.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:46.774 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 19:26:46.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:46.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:46.774 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:26:46.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:46.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:46.774 | INFO | Uploaded 36GiB to GPU in 2.0107s (17.9042GiB/s) | 20260506 19:26:46.774 | INFO | Downloaded 34GiB to GPU in 1.91314s (17.7718GiB/s) | 20260506 19:26:51.912 | INFO | Analyzing chunk frames = [3856896,3857408) | 20260506 19:26:51.912 | INFO | MemInfo: { | 20260506 19:26:51.912 | INFO | "gpu" : " 17 GB", | 20260506 19:26:51.912 | INFO | "huge" : "", | 20260506 19:26:51.912 | INFO | "huge_shared" : " 27 GB", | 20260506 19:26:51.912 | INFO | "malloc" : " 14 GB", | 20260506 19:26:51.912 | INFO | "pinned" : " 29 GB" | 20260506 19:26:51.912 | INFO | } | 20260506 19:26:51.912 | INFO | Perf report for the last 5.13755 seconds: | 20260506 19:26:51.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:26:51.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:51.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:51.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:51.912 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 19:26:51.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:51.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:51.912 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:26:51.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:51.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:51.912 | INFO | Uploaded 36GiB to GPU in 1.97968s (18.1848GiB/s) | 20260506 19:26:51.912 | INFO | Downloaded 34GiB to GPU in 1.96229s (17.3267GiB/s) | 20260506 19:26:57.055 | INFO | Analyzing chunk frames = [3857408,3857920) | 20260506 19:26:57.055 | INFO | MemInfo: { | 20260506 19:26:57.055 | INFO | "gpu" : " 17 GB", | 20260506 19:26:57.055 | INFO | "huge" : "", | 20260506 19:26:57.055 | INFO | "huge_shared" : " 27 GB", | 20260506 19:26:57.055 | INFO | "malloc" : " 14 GB", | 20260506 19:26:57.055 | INFO | "pinned" : " 29 GB" | 20260506 19:26:57.055 | INFO | } | 20260506 19:26:57.055 | INFO | Perf report for the last 5.14352 seconds: | 20260506 19:26:57.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:26:57.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:57.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:57.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:57.055 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 19:26:57.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:57.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:57.056 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:26:57.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:57.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:26:57.056 | INFO | Uploaded 36GiB to GPU in 1.96548s (18.3161GiB/s) | 20260506 19:26:57.056 | INFO | Downloaded 34GiB to GPU in 1.99275s (17.0619GiB/s) | 20260506 19:27:02.197 | INFO | Analyzing chunk frames = [3857920,3858432) | 20260506 19:27:02.197 | INFO | MemInfo: { | 20260506 19:27:02.197 | INFO | "gpu" : " 17 GB", | 20260506 19:27:02.197 | INFO | "huge" : "", | 20260506 19:27:02.197 | INFO | "huge_shared" : " 27 GB", | 20260506 19:27:02.197 | INFO | "malloc" : " 14 GB", | 20260506 19:27:02.197 | INFO | "pinned" : " 29 GB" | 20260506 19:27:02.197 | INFO | } | 20260506 19:27:02.197 | INFO | Perf report for the last 5.14148 seconds: | 20260506 19:27:02.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:27:02.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:02.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:02.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:02.197 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 19:27:02.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:02.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:02.197 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 19:27:02.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:02.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:02.197 | INFO | Uploaded 36GiB to GPU in 2.03748s (17.6689GiB/s) | 20260506 19:27:02.197 | INFO | Downloaded 34GiB to GPU in 1.98875s (17.0962GiB/s) | 20260506 19:27:02.197 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:27:02.197 | INFO | Total mem usage: 18 GiB | 20260506 19:27:02.197 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:27:02.197 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -19.8%) | 20260506 19:27:02.197 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:27:02.197 | INFO | High Water Breakdown: | 20260506 19:27:02.197 | INFO | 60.8% : 9.15 GiB ( 80.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:27:02.197 | INFO | 39% : 5.86 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:27:02.197 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:27:02.197 | INFO | ------------------------------------------ | 20260506 19:27:02.197 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:27:02.197 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:27:02.197 | INFO | Total mem usage: 26.5 GiB | 20260506 19:27:02.197 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:27:02.197 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:27:02.197 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:27:02.197 | INFO | High Water Breakdown: | 20260506 19:27:02.197 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:27:02.197 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:27:02.197 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:27:02.197 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:27:02.197 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:27:02.197 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:27:02.197 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:27:02.197 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:27:02.197 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:27:02.197 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:27:02.197 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:27:02.197 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:27:02.197 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:27:02.197 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:27:02.197 | INFO | ------------------------------------------ | 20260506 19:27:02.197 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:27:02.197 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:27:02.197 | INFO | Total mem usage: 25.6 GiB | 20260506 19:27:02.197 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:27:02.197 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:27:02.197 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:27:02.197 | INFO | High Water Breakdown: | 20260506 19:27:02.197 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:27:02.197 | INFO | ------------------------------------------ | 20260506 19:27:02.197 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:27:02.197 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:27:02.198 | INFO | Total mem usage: 15.7 GiB | 20260506 19:27:02.198 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:27:02.198 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:27:02.198 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:27:02.198 | INFO | High Water Breakdown: | 20260506 19:27:02.198 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:27:02.198 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:27:02.198 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:27:02.198 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:27:02.198 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:27:02.198 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:27:02.198 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:27:02.198 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:27:02.198 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:27:02.198 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:27:02.198 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:27:02.198 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:27:02.198 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:27:02.198 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:27:02.198 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:27:02.198 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:27:02.198 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:27:02.198 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:27:02.198 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:27:02.198 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:27:02.198 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:27:02.198 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:27:02.198 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:27:02.198 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:27:02.198 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:27:02.198 | INFO | ------------------------------------------ | 20260506 19:27:02.198 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:27:02.198 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:27:02.198 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:27:02.198 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:27:02.198 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.62e+01, 1.31e+01 / 2.88e+00 | 20260506 19:27:02.198 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.01e-01, 1.04e-02 / 6.42e-03 | 20260506 19:27:02.198 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.40e+00, 5.57e-01 / 3.44e-01 | 20260506 19:27:02.198 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.46e+00, 1.85e-01 / 1.50e-01 | 20260506 19:27:02.198 | INFO | PerfReport: 16.02%, DME -- 4.01e+01 / 4.50e+01, 4.30e+00 / 3.20e+00 | 20260506 19:27:02.198 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.79e+01 / 2.06e+01, 1.45e+00 / 2.16e+00 | 20260506 19:27:02.198 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.24e+00 / 2.82e+00, 3.78e-01 / 2.27e-01 | 20260506 19:27:02.198 | INFO | PerfReport: 3.35%, Metrics -- 1.05e+00 / 1.42e+00, 2.75e-01 / 1.96e-01 | 20260506 19:27:02.198 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.62e-01 / 4.12e-01, 5.75e-02 / 2.97e-02 | 20260506 19:27:02.198 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:27:02.198 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:27:02.198 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:27:02.198 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 19:27:02.198 | INFO | PerfReport: 21.46%, Upload -- 2.08e+01 / 0.00e+00, 6.94e+00 / 0.00e+00 | 20260506 19:27:02.198 | INFO | PerfReport: 21.19%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 19:27:02.198 | INFO | PerfReport: 32.24%, Compute -- 3.13e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 19:27:02.198 | INFO | PerfReport: 1.32%, LockUpload -- 1.28e+00 / 0.00e+00, 5.37e+00 / 0.00e+00 | 20260506 19:27:02.198 | INFO | PerfReport: 1.18%, LockDownload -- 1.15e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260506 19:27:02.198 | INFO | PerfReport: 22.59%, LockCompute -- 2.19e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 19:27:02.198 | INFO | PerfReport: 0.02%, Misc -- 1.52e-02 / 0.00e+00, 2.44e-02 / 0.00e+00 | 20260506 19:27:02.312 | INFO | Starting a round of Baz IO | 20260506 19:27:05.504 | INFO | Populating RTMetrics report | 20260506 19:27:05.505 | INFO | RTMetrics report populated | 20260506 19:27:05.505 | INFO | Opening tmp RTMetrics json file | 20260506 19:27:05.507 | INFO | Serializing RTMetrics to json | 20260506 19:27:05.511 | INFO | Serialization done, swapping tmp file now... | 20260506 19:27:05.513 | INFO | RTMetrics done | 20260506 19:27:06.554 | INFO | Finished a round of Baz IO: Wrote 2.009 GiB in 4.241 seconds | 20260506 19:27:06.554 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.62% / 0.00% / 65.38% | 20260506 19:27:06.554 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.914784% / 1.19209e-05% / 4.37101e-05% | 20260506 19:27:06.554 | INFO | Baz IO overall: | 20260506 19:27:06.554 | INFO | BytesWritten : 2039002672813 | 20260506 19:27:06.554 | INFO | NumEvents : 985636581343 | 20260506 19:27:07.342 | INFO | Analyzing chunk frames = [3858432,3858944) | 20260506 19:27:07.342 | INFO | MemInfo: { | 20260506 19:27:07.342 | INFO | "gpu" : " 17 GB", | 20260506 19:27:07.342 | INFO | "huge" : "", | 20260506 19:27:07.342 | INFO | "huge_shared" : " 27 GB", | 20260506 19:27:07.342 | INFO | "malloc" : " 14 GB", | 20260506 19:27:07.342 | INFO | "pinned" : " 29 GB" | 20260506 19:27:07.342 | INFO | } | 20260506 19:27:07.342 | INFO | Perf report for the last 5.14495 seconds: | 20260506 19:27:07.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:27:07.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:07.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:07.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:07.342 | INFO | ANALYSIS : 62.8%, 2.8, 93.9ms, 0 | 20260506 19:27:07.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:07.342 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260506 19:27:07.342 | INFO | PRE_HQ : 61.2%, 2.0, 62.3ms, 0 | 20260506 19:27:07.342 | INFO | BAZWRITER : 82.1%, 4.8, 204.0ms, 0 | 20260506 19:27:07.342 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 19:27:07.342 | INFO | Uploaded 36GiB to GPU in 2.06352s (17.4459GiB/s) | 20260506 19:27:07.342 | INFO | Downloaded 38GiB to GPU in 2.20813s (17.2091GiB/s) | 20260506 19:27:12.473 | INFO | Analyzing chunk frames = [3858944,3859456) | 20260506 19:27:12.473 | INFO | MemInfo: { | 20260506 19:27:12.473 | INFO | "gpu" : " 17 GB", | 20260506 19:27:12.474 | INFO | "huge" : "", | 20260506 19:27:12.474 | INFO | "huge_shared" : " 27 GB", | 20260506 19:27:12.474 | INFO | "malloc" : " 14 GB", | 20260506 19:27:12.474 | INFO | "pinned" : " 29 GB" | 20260506 19:27:12.474 | INFO | } | 20260506 19:27:12.474 | INFO | Perf report for the last 5.13186 seconds: | 20260506 19:27:12.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:27:12.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:12.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:12.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:12.474 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 19:27:12.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:12.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:12.474 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 19:27:12.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:12.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:12.474 | INFO | Uploaded 36GiB to GPU in 2.0058s (17.9479GiB/s) | 20260506 19:27:12.474 | INFO | Downloaded 34GiB to GPU in 1.92636s (17.6498GiB/s) | 20260506 19:27:17.610 | INFO | Analyzing chunk frames = [3859456,3859968) | 20260506 19:27:17.610 | INFO | MemInfo: { | 20260506 19:27:17.610 | INFO | "gpu" : " 17 GB", | 20260506 19:27:17.610 | INFO | "huge" : "", | 20260506 19:27:17.610 | INFO | "huge_shared" : " 27 GB", | 20260506 19:27:17.610 | INFO | "malloc" : " 14 GB", | 20260506 19:27:17.610 | INFO | "pinned" : " 29 GB" | 20260506 19:27:17.610 | INFO | } | 20260506 19:27:17.611 | INFO | Perf report for the last 5.13687 seconds: | 20260506 19:27:17.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:27:17.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:17.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:17.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:17.611 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 19:27:17.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:17.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:17.611 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:27:17.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:17.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:17.611 | INFO | Uploaded 36GiB to GPU in 1.97876s (18.1932GiB/s) | 20260506 19:27:17.611 | INFO | Downloaded 34GiB to GPU in 1.914s (17.7639GiB/s) | 20260506 19:27:22.748 | INFO | Analyzing chunk frames = [3859968,3860480) | 20260506 19:27:22.748 | INFO | MemInfo: { | 20260506 19:27:22.748 | INFO | "gpu" : " 17 GB", | 20260506 19:27:22.748 | INFO | "huge" : "", | 20260506 19:27:22.748 | INFO | "huge_shared" : " 27 GB", | 20260506 19:27:22.748 | INFO | "malloc" : " 14 GB", | 20260506 19:27:22.748 | INFO | "pinned" : " 29 GB" | 20260506 19:27:22.748 | INFO | } | 20260506 19:27:22.749 | INFO | Perf report for the last 5.138 seconds: | 20260506 19:27:22.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:27:22.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:22.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:22.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:22.749 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 19:27:22.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:22.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:22.749 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:27:22.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:22.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:22.749 | INFO | Uploaded 36GiB to GPU in 1.93055s (18.6475GiB/s) | 20260506 19:27:22.749 | INFO | Downloaded 34GiB to GPU in 1.87165s (18.1657GiB/s) | 20260506 19:27:27.885 | INFO | Analyzing chunk frames = [3860480,3860992) | 20260506 19:27:27.885 | INFO | MemInfo: { | 20260506 19:27:27.885 | INFO | "gpu" : " 17 GB", | 20260506 19:27:27.885 | INFO | "huge" : "", | 20260506 19:27:27.885 | INFO | "huge_shared" : " 27 GB", | 20260506 19:27:27.885 | INFO | "malloc" : " 14 GB", | 20260506 19:27:27.885 | INFO | "pinned" : " 29 GB" | 20260506 19:27:27.885 | INFO | } | 20260506 19:27:27.885 | INFO | Perf report for the last 5.13639 seconds: | 20260506 19:27:27.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:27:27.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:27.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:27.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:27.885 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260506 19:27:27.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:27.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:27.885 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:27:27.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:27.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:27.885 | INFO | Uploaded 36GiB to GPU in 1.96332s (18.3363GiB/s) | 20260506 19:27:27.885 | INFO | Downloaded 34GiB to GPU in 1.8824s (18.062GiB/s) | 20260506 19:27:33.029 | INFO | Analyzing chunk frames = [3860992,3861504) | 20260506 19:27:33.029 | INFO | MemInfo: { | 20260506 19:27:33.029 | INFO | "gpu" : " 17 GB", | 20260506 19:27:33.029 | INFO | "huge" : "", | 20260506 19:27:33.029 | INFO | "huge_shared" : " 27 GB", | 20260506 19:27:33.029 | INFO | "malloc" : " 14 GB", | 20260506 19:27:33.029 | INFO | "pinned" : " 29 GB" | 20260506 19:27:33.029 | INFO | } | 20260506 19:27:33.029 | INFO | Perf report for the last 5.14429 seconds: | 20260506 19:27:33.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:27:33.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:33.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:33.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:33.029 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 19:27:33.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:33.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:33.029 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:27:33.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:33.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:33.030 | INFO | Uploaded 36GiB to GPU in 2.03414s (17.6979GiB/s) | 20260506 19:27:33.030 | INFO | Downloaded 34GiB to GPU in 1.956s (17.3824GiB/s) | 20260506 19:27:38.176 | INFO | Analyzing chunk frames = [3861504,3862016) | 20260506 19:27:38.176 | INFO | MemInfo: { | 20260506 19:27:38.176 | INFO | "gpu" : " 17 GB", | 20260506 19:27:38.176 | INFO | "huge" : "", | 20260506 19:27:38.176 | INFO | "huge_shared" : " 27 GB", | 20260506 19:27:38.176 | INFO | "malloc" : " 14 GB", | 20260506 19:27:38.176 | INFO | "pinned" : " 29 GB" | 20260506 19:27:38.176 | INFO | } | 20260506 19:27:38.176 | INFO | Perf report for the last 5.14702 seconds: | 20260506 19:27:38.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:27:38.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:38.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:38.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:38.176 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:27:38.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:38.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:38.176 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:27:38.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:38.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:38.176 | INFO | Uploaded 36GiB to GPU in 1.9895s (18.095GiB/s) | 20260506 19:27:38.177 | INFO | Downloaded 34GiB to GPU in 2.00158s (16.9866GiB/s) | 20260506 19:27:43.311 | INFO | Analyzing chunk frames = [3862016,3862528) | 20260506 19:27:43.311 | INFO | MemInfo: { | 20260506 19:27:43.311 | INFO | "gpu" : " 17 GB", | 20260506 19:27:43.311 | INFO | "huge" : "", | 20260506 19:27:43.311 | INFO | "huge_shared" : " 27 GB", | 20260506 19:27:43.311 | INFO | "malloc" : " 14 GB", | 20260506 19:27:43.311 | INFO | "pinned" : " 29 GB" | 20260506 19:27:43.311 | INFO | } | 20260506 19:27:43.311 | INFO | Perf report for the last 5.13497 seconds: | 20260506 19:27:43.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:27:43.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:43.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:43.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:43.311 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 19:27:43.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:43.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:43.311 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 19:27:43.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:43.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:43.312 | INFO | Uploaded 36GiB to GPU in 1.98483s (18.1376GiB/s) | 20260506 19:27:43.312 | INFO | Downloaded 34GiB to GPU in 1.99949s (17.0044GiB/s) | 20260506 19:27:43.427 | INFO | Starting a round of Baz IO | 20260506 19:27:46.560 | INFO | Populating RTMetrics report | 20260506 19:27:46.560 | INFO | RTMetrics report populated | 20260506 19:27:46.560 | INFO | Opening tmp RTMetrics json file | 20260506 19:27:46.561 | INFO | Serializing RTMetrics to json | 20260506 19:27:46.563 | INFO | Serialization done, swapping tmp file now... | 20260506 19:27:46.564 | INFO | RTMetrics done | 20260506 19:27:46.723 | INFO | Finished a round of Baz IO: Wrote 2.009 GiB in 3.295 seconds | 20260506 19:27:46.723 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.61% / 0.00% / 65.38% | 20260506 19:27:46.723 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.913692% / 1.19209e-05% / 9.53674e-05% | 20260506 19:27:46.723 | INFO | Baz IO overall: | 20260506 19:27:46.723 | INFO | BytesWritten : 2041160093671 | 20260506 19:27:46.723 | INFO | NumEvents : 986673796403 | 20260506 19:27:48.452 | INFO | Analyzing chunk frames = [3862528,3863040) | 20260506 19:27:48.452 | INFO | MemInfo: { | 20260506 19:27:48.452 | INFO | "gpu" : " 17 GB", | 20260506 19:27:48.452 | INFO | "huge" : "", | 20260506 19:27:48.452 | INFO | "huge_shared" : " 27 GB", | 20260506 19:27:48.452 | INFO | "malloc" : " 14 GB", | 20260506 19:27:48.452 | INFO | "pinned" : " 29 GB" | 20260506 19:27:48.452 | INFO | } | 20260506 19:27:48.452 | INFO | Perf report for the last 5.14069 seconds: | 20260506 19:27:48.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:27:48.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:48.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:48.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:48.452 | INFO | ANALYSIS : 62.6%, 2.9, 95.2ms, 0 | 20260506 19:27:48.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:48.452 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 19:27:48.452 | INFO | PRE_HQ : 60.0%, 1.9, 58.1ms, 0 | 20260506 19:27:48.452 | INFO | BAZWRITER : 63.9%, 4.7, 152.8ms, 0 | 20260506 19:27:48.452 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 19:27:48.452 | INFO | Uploaded 36GiB to GPU in 2.03491s (17.6912GiB/s) | 20260506 19:27:48.452 | INFO | Downloaded 38GiB to GPU in 2.20374s (17.2434GiB/s) | 20260506 19:27:53.588 | INFO | Analyzing chunk frames = [3863040,3863552) | 20260506 19:27:53.588 | INFO | MemInfo: { | 20260506 19:27:53.588 | INFO | "gpu" : " 17 GB", | 20260506 19:27:53.588 | INFO | "huge" : "", | 20260506 19:27:53.588 | INFO | "huge_shared" : " 27 GB", | 20260506 19:27:53.588 | INFO | "malloc" : " 14 GB", | 20260506 19:27:53.588 | INFO | "pinned" : " 29 GB" | 20260506 19:27:53.588 | INFO | } | 20260506 19:27:53.588 | INFO | Perf report for the last 5.13654 seconds: | 20260506 19:27:53.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:27:53.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:53.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:53.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:53.589 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:27:53.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:53.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:53.589 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:27:53.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:53.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:53.589 | INFO | Uploaded 36GiB to GPU in 2.00249s (17.9776GiB/s) | 20260506 19:27:53.589 | INFO | Downloaded 34GiB to GPU in 1.92796s (17.6352GiB/s) | 20260506 19:27:58.726 | INFO | Analyzing chunk frames = [3863552,3864064) | 20260506 19:27:58.726 | INFO | MemInfo: { | 20260506 19:27:58.726 | INFO | "gpu" : " 17 GB", | 20260506 19:27:58.726 | INFO | "huge" : "", | 20260506 19:27:58.726 | INFO | "huge_shared" : " 27 GB", | 20260506 19:27:58.726 | INFO | "malloc" : " 14 GB", | 20260506 19:27:58.726 | INFO | "pinned" : " 29 GB" | 20260506 19:27:58.726 | INFO | } | 20260506 19:27:58.726 | INFO | Perf report for the last 5.13748 seconds: | 20260506 19:27:58.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:27:58.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:58.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:58.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:58.726 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:27:58.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:58.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:58.726 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:27:58.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:58.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:27:58.726 | INFO | Uploaded 36GiB to GPU in 1.9693s (18.2806GiB/s) | 20260506 19:27:58.726 | INFO | Downloaded 34GiB to GPU in 1.92095s (17.6996GiB/s) | 20260506 19:28:03.862 | INFO | Analyzing chunk frames = [3864064,3864576) | 20260506 19:28:03.862 | INFO | MemInfo: { | 20260506 19:28:03.862 | INFO | "gpu" : " 17 GB", | 20260506 19:28:03.862 | INFO | "huge" : "", | 20260506 19:28:03.862 | INFO | "huge_shared" : " 27 GB", | 20260506 19:28:03.862 | INFO | "malloc" : " 14 GB", | 20260506 19:28:03.862 | INFO | "pinned" : " 29 GB" | 20260506 19:28:03.862 | INFO | } | 20260506 19:28:03.862 | INFO | Perf report for the last 5.13603 seconds: | 20260506 19:28:03.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:28:03.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:03.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:03.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:03.862 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:28:03.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:03.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:03.862 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:28:03.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:03.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:03.862 | INFO | Uploaded 36GiB to GPU in 1.9125s (18.8235GiB/s) | 20260506 19:28:03.862 | INFO | Downloaded 34GiB to GPU in 1.87221s (18.1604GiB/s) | 20260506 19:28:09.004 | INFO | Analyzing chunk frames = [3864576,3865088) | 20260506 19:28:09.004 | INFO | MemInfo: { | 20260506 19:28:09.004 | INFO | "gpu" : " 17 GB", | 20260506 19:28:09.004 | INFO | "huge" : "", | 20260506 19:28:09.004 | INFO | "huge_shared" : " 27 GB", | 20260506 19:28:09.004 | INFO | "malloc" : " 14 GB", | 20260506 19:28:09.004 | INFO | "pinned" : " 29 GB" | 20260506 19:28:09.004 | INFO | } | 20260506 19:28:09.004 | INFO | Perf report for the last 5.14218 seconds: | 20260506 19:28:09.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:28:09.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:09.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:09.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:09.005 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 19:28:09.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:09.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:09.005 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:28:09.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:09.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:09.005 | INFO | Uploaded 36GiB to GPU in 2.01658s (17.852GiB/s) | 20260506 19:28:09.005 | INFO | Downloaded 34GiB to GPU in 1.91738s (17.7325GiB/s) | 20260506 19:28:14.145 | INFO | Analyzing chunk frames = [3865088,3865600) | 20260506 19:28:14.146 | INFO | MemInfo: { | 20260506 19:28:14.146 | INFO | "gpu" : " 17 GB", | 20260506 19:28:14.146 | INFO | "huge" : "", | 20260506 19:28:14.146 | INFO | "huge_shared" : " 27 GB", | 20260506 19:28:14.146 | INFO | "malloc" : " 14 GB", | 20260506 19:28:14.146 | INFO | "pinned" : " 29 GB" | 20260506 19:28:14.146 | INFO | } | 20260506 19:28:14.146 | INFO | Perf report for the last 5.14142 seconds: | 20260506 19:28:14.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:28:14.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:14.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:14.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:14.146 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 19:28:14.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:14.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:14.146 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 19:28:14.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:14.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:14.146 | INFO | Uploaded 36GiB to GPU in 1.97866s (18.1941GiB/s) | 20260506 19:28:14.146 | INFO | Downloaded 34GiB to GPU in 1.9544s (17.3966GiB/s) | 20260506 19:28:19.289 | INFO | Analyzing chunk frames = [3865600,3866112) | 20260506 19:28:19.289 | INFO | MemInfo: { | 20260506 19:28:19.289 | INFO | "gpu" : " 17 GB", | 20260506 19:28:19.289 | INFO | "huge" : "", | 20260506 19:28:19.289 | INFO | "huge_shared" : " 27 GB", | 20260506 19:28:19.289 | INFO | "malloc" : " 14 GB", | 20260506 19:28:19.289 | INFO | "pinned" : " 29 GB" | 20260506 19:28:19.289 | INFO | } | 20260506 19:28:19.289 | INFO | Perf report for the last 5.14298 seconds: | 20260506 19:28:19.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:28:19.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:19.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:19.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:19.289 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 19:28:19.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:19.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:19.289 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 19:28:19.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:19.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:19.289 | INFO | Uploaded 36GiB to GPU in 1.95821s (18.3842GiB/s) | 20260506 19:28:19.289 | INFO | Downloaded 34GiB to GPU in 2.00332s (16.9718GiB/s) | 20260506 19:28:24.432 | INFO | Analyzing chunk frames = [3866112,3866624) | 20260506 19:28:24.432 | INFO | MemInfo: { | 20260506 19:28:24.432 | INFO | "gpu" : " 17 GB", | 20260506 19:28:24.432 | INFO | "huge" : "", | 20260506 19:28:24.432 | INFO | "huge_shared" : " 27 GB", | 20260506 19:28:24.432 | INFO | "malloc" : " 14 GB", | 20260506 19:28:24.432 | INFO | "pinned" : " 29 GB" | 20260506 19:28:24.432 | INFO | } | 20260506 19:28:24.432 | INFO | Perf report for the last 5.14309 seconds: | 20260506 19:28:24.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:28:24.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:24.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:24.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:24.432 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 19:28:24.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:24.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:24.432 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:28:24.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:24.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:24.432 | INFO | Uploaded 36GiB to GPU in 1.99034s (18.0874GiB/s) | 20260506 19:28:24.432 | INFO | Downloaded 34GiB to GPU in 1.96859s (17.2713GiB/s) | 20260506 19:28:24.432 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:28:24.432 | INFO | Total mem usage: 18 GiB | 20260506 19:28:24.432 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:28:24.432 | INFO | Recent High Water Marks Sum: 16.8 GiB (Savings: -7.37%) | 20260506 19:28:24.432 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:28:24.432 | INFO | High Water Breakdown: | 20260506 19:28:24.432 | INFO | 62.5% : 10.5 GiB ( 91.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:28:24.432 | INFO | 37.4% : 6.27 GiB ( 93.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:28:24.432 | INFO | 0.119% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:28:24.432 | INFO | ------------------------------------------ | 20260506 19:28:24.432 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:28:24.432 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:28:24.432 | INFO | Total mem usage: 26.5 GiB | 20260506 19:28:24.432 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:28:24.432 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:28:24.432 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:28:24.432 | INFO | High Water Breakdown: | 20260506 19:28:24.432 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:28:24.432 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:28:24.432 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:28:24.432 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:28:24.432 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:28:24.432 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:28:24.432 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:28:24.432 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:28:24.432 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:28:24.432 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:28:24.432 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:28:24.432 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:28:24.432 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:28:24.432 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:28:24.432 | INFO | ------------------------------------------ | 20260506 19:28:24.432 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:28:24.432 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:28:24.432 | INFO | Total mem usage: 25.6 GiB | 20260506 19:28:24.432 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:28:24.432 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:28:24.432 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:28:24.432 | INFO | High Water Breakdown: | 20260506 19:28:24.432 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:28:24.432 | INFO | ------------------------------------------ | 20260506 19:28:24.432 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:28:24.432 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:28:24.432 | INFO | Total mem usage: 15.7 GiB | 20260506 19:28:24.432 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:28:24.432 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:28:24.432 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:28:24.432 | INFO | High Water Breakdown: | 20260506 19:28:24.432 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:28:24.433 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:28:24.433 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:28:24.433 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:28:24.433 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:28:24.433 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:28:24.433 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:28:24.433 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:28:24.433 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:28:24.433 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:28:24.433 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:28:24.433 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:28:24.433 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:28:24.433 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:28:24.433 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:28:24.433 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:28:24.433 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:28:24.433 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:28:24.433 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:28:24.433 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:28:24.433 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:28:24.433 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:28:24.433 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:28:24.433 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:28:24.433 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:28:24.433 | INFO | ------------------------------------------ | 20260506 19:28:24.433 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:28:24.433 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:28:24.433 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:28:24.433 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:28:24.433 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.53e+01, 1.36e+01 / 2.36e+00 | 20260506 19:28:24.433 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 9.67e-03 / 5.35e-03 | 20260506 19:28:24.433 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.32e+00, 5.55e-01 / 4.02e-01 | 20260506 19:28:24.433 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.44e+00, 1.83e-01 / 1.43e-01 | 20260506 19:28:24.433 | INFO | PerfReport: 15.97%, DME -- 3.99e+01 / 4.41e+01, 4.03e+00 / 2.87e+00 | 20260506 19:28:24.433 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.45e+00 / 1.89e+00 | 20260506 19:28:24.433 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.24e+00 / 2.79e+00, 3.81e-01 / 2.18e-01 | 20260506 19:28:24.433 | INFO | PerfReport: 3.36%, Metrics -- 1.05e+00 / 1.39e+00, 2.76e-01 / 1.61e-01 | 20260506 19:28:24.433 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.60e-01 / 4.04e-01, 5.57e-02 / 1.83e-02 | 20260506 19:28:24.433 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:28:24.433 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:28:24.433 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:28:24.433 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.31e+01 / 0.00e+00 | 20260506 19:28:24.433 | INFO | PerfReport: 21.42%, Upload -- 2.07e+01 / 0.00e+00, 6.89e+00 / 0.00e+00 | 20260506 19:28:24.433 | INFO | PerfReport: 21.23%, Download -- 2.05e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 19:28:24.433 | INFO | PerfReport: 32.30%, Compute -- 3.12e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 19:28:24.433 | INFO | PerfReport: 1.26%, LockUpload -- 1.22e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260506 19:28:24.433 | INFO | PerfReport: 1.18%, LockDownload -- 1.14e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260506 19:28:24.433 | INFO | PerfReport: 22.59%, LockCompute -- 2.18e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 19:28:24.433 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.32e-02 / 0.00e+00 | 20260506 19:28:24.531 | INFO | Starting a round of Baz IO | 20260506 19:28:27.666 | INFO | Populating RTMetrics report | 20260506 19:28:27.666 | INFO | RTMetrics report populated | 20260506 19:28:27.666 | INFO | Opening tmp RTMetrics json file | 20260506 19:28:27.717 | INFO | Serializing RTMetrics to json | 20260506 19:28:27.719 | INFO | Serialization done, swapping tmp file now... | 20260506 19:28:27.719 | INFO | RTMetrics done | 20260506 19:28:27.923 | INFO | Finished a round of Baz IO: Wrote 2.01 GiB in 3.392 seconds | 20260506 19:28:27.923 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.61% / 0.00% / 65.39% | 20260506 19:28:27.923 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.910735% / 7.94729e-06% / 7.94729e-05% | 20260506 19:28:27.923 | INFO | Baz IO overall: | 20260506 19:28:27.923 | INFO | BytesWritten : 2043317903013 | 20260506 19:28:27.923 | INFO | NumEvents : 987711521971 | 20260506 19:28:29.563 | INFO | Analyzing chunk frames = [3866624,3867136) | 20260506 19:28:29.563 | INFO | MemInfo: { | 20260506 19:28:29.563 | INFO | "gpu" : " 17 GB", | 20260506 19:28:29.563 | INFO | "huge" : "", | 20260506 19:28:29.563 | INFO | "huge_shared" : " 27 GB", | 20260506 19:28:29.563 | INFO | "malloc" : " 14 GB", | 20260506 19:28:29.563 | INFO | "pinned" : " 29 GB" | 20260506 19:28:29.563 | INFO | } | 20260506 19:28:29.563 | INFO | Perf report for the last 5.13145 seconds: | 20260506 19:28:29.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:28:29.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:29.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:29.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:29.563 | INFO | ANALYSIS : 62.4%, 2.9, 95.1ms, 0 | 20260506 19:28:29.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:29.563 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 19:28:29.563 | INFO | PRE_HQ : 60.1%, 1.8, 57.4ms, 0 | 20260506 19:28:29.563 | INFO | BAZWRITER : 65.8%, 4.8, 161.6ms, 0 | 20260506 19:28:29.563 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 19:28:29.563 | INFO | Uploaded 36GiB to GPU in 2.05441s (17.5232GiB/s) | 20260506 19:28:29.563 | INFO | Downloaded 38GiB to GPU in 2.19336s (17.325GiB/s) | 20260506 19:28:34.699 | INFO | Analyzing chunk frames = [3867136,3867648) | 20260506 19:28:34.699 | INFO | MemInfo: { | 20260506 19:28:34.699 | INFO | "gpu" : " 17 GB", | 20260506 19:28:34.699 | INFO | "huge" : "", | 20260506 19:28:34.699 | INFO | "huge_shared" : " 27 GB", | 20260506 19:28:34.699 | INFO | "malloc" : " 14 GB", | 20260506 19:28:34.699 | INFO | "pinned" : " 29 GB" | 20260506 19:28:34.699 | INFO | } | 20260506 19:28:34.699 | INFO | Perf report for the last 5.13614 seconds: | 20260506 19:28:34.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:28:34.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:34.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:34.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:34.699 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:28:34.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:34.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:34.699 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:28:34.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:34.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:34.700 | INFO | Uploaded 36GiB to GPU in 2.01605s (17.8567GiB/s) | 20260506 19:28:34.700 | INFO | Downloaded 34GiB to GPU in 1.93808s (17.5432GiB/s) | 20260506 19:28:39.840 | INFO | Analyzing chunk frames = [3867648,3868160) | 20260506 19:28:39.840 | INFO | MemInfo: { | 20260506 19:28:39.840 | INFO | "gpu" : " 17 GB", | 20260506 19:28:39.840 | INFO | "huge" : "", | 20260506 19:28:39.840 | INFO | "huge_shared" : " 27 GB", | 20260506 19:28:39.840 | INFO | "malloc" : " 14 GB", | 20260506 19:28:39.840 | INFO | "pinned" : " 29 GB" | 20260506 19:28:39.840 | INFO | } | 20260506 19:28:39.840 | INFO | Perf report for the last 5.14103 seconds: | 20260506 19:28:39.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:28:39.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:39.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:39.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:39.840 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 19:28:39.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:39.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:39.841 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:28:39.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:39.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:39.841 | INFO | Uploaded 36GiB to GPU in 1.98202s (18.1633GiB/s) | 20260506 19:28:39.841 | INFO | Downloaded 34GiB to GPU in 1.91547s (17.7502GiB/s) | 20260506 19:28:44.978 | INFO | Analyzing chunk frames = [3868160,3868672) | 20260506 19:28:44.978 | INFO | MemInfo: { | 20260506 19:28:44.978 | INFO | "gpu" : " 17 GB", | 20260506 19:28:44.978 | INFO | "huge" : "", | 20260506 19:28:44.978 | INFO | "huge_shared" : " 27 GB", | 20260506 19:28:44.978 | INFO | "malloc" : " 14 GB", | 20260506 19:28:44.978 | INFO | "pinned" : " 29 GB" | 20260506 19:28:44.978 | INFO | } | 20260506 19:28:44.978 | INFO | Perf report for the last 5.13774 seconds: | 20260506 19:28:44.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:28:44.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:44.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:44.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:44.978 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 19:28:44.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:44.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:44.978 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 19:28:44.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:44.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:44.978 | INFO | Uploaded 36GiB to GPU in 1.93324s (18.6216GiB/s) | 20260506 19:28:44.978 | INFO | Downloaded 34GiB to GPU in 1.86994s (18.1824GiB/s) | 20260506 19:28:50.120 | INFO | Analyzing chunk frames = [3868672,3869184) | 20260506 19:28:50.120 | INFO | MemInfo: { | 20260506 19:28:50.120 | INFO | "gpu" : " 17 GB", | 20260506 19:28:50.120 | INFO | "huge" : "", | 20260506 19:28:50.120 | INFO | "huge_shared" : " 27 GB", | 20260506 19:28:50.120 | INFO | "malloc" : " 14 GB", | 20260506 19:28:50.120 | INFO | "pinned" : " 29 GB" | 20260506 19:28:50.120 | INFO | } | 20260506 19:28:50.120 | INFO | Perf report for the last 5.14225 seconds: | 20260506 19:28:50.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:28:50.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:50.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:50.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:50.120 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260506 19:28:50.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:50.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:50.120 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:28:50.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:50.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:50.121 | INFO | Uploaded 36GiB to GPU in 1.96411s (18.329GiB/s) | 20260506 19:28:50.121 | INFO | Downloaded 34GiB to GPU in 1.88185s (18.0673GiB/s) | 20260506 19:28:55.266 | INFO | Analyzing chunk frames = [3869184,3869696) | 20260506 19:28:55.266 | INFO | MemInfo: { | 20260506 19:28:55.266 | INFO | "gpu" : " 17 GB", | 20260506 19:28:55.266 | INFO | "huge" : "", | 20260506 19:28:55.266 | INFO | "huge_shared" : " 27 GB", | 20260506 19:28:55.266 | INFO | "malloc" : " 14 GB", | 20260506 19:28:55.266 | INFO | "pinned" : " 29 GB" | 20260506 19:28:55.266 | INFO | } | 20260506 19:28:55.267 | INFO | Perf report for the last 5.14608 seconds: | 20260506 19:28:55.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:28:55.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:55.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:55.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:55.267 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 19:28:55.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:55.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:55.267 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 19:28:55.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:55.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:28:55.267 | INFO | Uploaded 36GiB to GPU in 2.02575s (17.7712GiB/s) | 20260506 19:28:55.267 | INFO | Downloaded 34GiB to GPU in 1.9599s (17.3479GiB/s) | 20260506 19:29:00.402 | INFO | Analyzing chunk frames = [3869696,3870208) | 20260506 19:29:00.402 | INFO | MemInfo: { | 20260506 19:29:00.402 | INFO | "gpu" : " 17 GB", | 20260506 19:29:00.402 | INFO | "huge" : "", | 20260506 19:29:00.402 | INFO | "huge_shared" : " 27 GB", | 20260506 19:29:00.402 | INFO | "malloc" : " 14 GB", | 20260506 19:29:00.402 | INFO | "pinned" : " 29 GB" | 20260506 19:29:00.402 | INFO | } | 20260506 19:29:00.402 | INFO | Perf report for the last 5.13542 seconds: | 20260506 19:29:00.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:29:00.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:00.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:00.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:00.402 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 19:29:00.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:00.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:00.402 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 19:29:00.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:00.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:00.402 | INFO | Uploaded 36GiB to GPU in 2.00188s (17.9831GiB/s) | 20260506 19:29:00.402 | INFO | Downloaded 34GiB to GPU in 2.006s (16.9492GiB/s) | 20260506 19:29:05.539 | INFO | Analyzing chunk frames = [3870208,3870720) | 20260506 19:29:05.539 | INFO | MemInfo: { | 20260506 19:29:05.539 | INFO | "gpu" : " 17 GB", | 20260506 19:29:05.539 | INFO | "huge" : "", | 20260506 19:29:05.539 | INFO | "huge_shared" : " 27 GB", | 20260506 19:29:05.539 | INFO | "malloc" : " 14 GB", | 20260506 19:29:05.539 | INFO | "pinned" : " 29 GB" | 20260506 19:29:05.539 | INFO | } | 20260506 19:29:05.539 | INFO | Perf report for the last 5.13737 seconds: | 20260506 19:29:05.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:29:05.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:05.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:05.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:05.539 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 19:29:05.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:05.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:05.539 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:29:05.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:05.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:05.540 | INFO | Uploaded 36GiB to GPU in 1.97634s (18.2155GiB/s) | 20260506 19:29:05.540 | INFO | Downloaded 34GiB to GPU in 1.98206s (17.1539GiB/s) | 20260506 19:29:05.648 | INFO | Starting a round of Baz IO | 20260506 19:29:08.782 | INFO | Populating RTMetrics report | 20260506 19:29:08.782 | INFO | RTMetrics report populated | 20260506 19:29:08.782 | INFO | Opening tmp RTMetrics json file | 20260506 19:29:08.867 | INFO | Serializing RTMetrics to json | 20260506 19:29:08.987 | INFO | Serialization done, swapping tmp file now... | 20260506 19:29:09.042 | INFO | RTMetrics done | 20260506 19:29:09.227 | INFO | Finished a round of Baz IO: Wrote 2.007 GiB in 3.579 seconds | 20260506 19:29:09.227 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.60% / 0.00% / 65.39% | 20260506 19:29:09.227 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.906527% / 7.94729e-06% / 7.94729e-05% | 20260506 19:29:09.227 | INFO | Baz IO overall: | 20260506 19:29:09.227 | INFO | BytesWritten : 2045473366377 | 20260506 19:29:09.227 | INFO | NumEvents : 988747395430 | 20260506 19:29:10.678 | INFO | Analyzing chunk frames = [3870720,3871232) | 20260506 19:29:10.678 | INFO | MemInfo: { | 20260506 19:29:10.678 | INFO | "gpu" : " 17 GB", | 20260506 19:29:10.678 | INFO | "huge" : "", | 20260506 19:29:10.678 | INFO | "huge_shared" : " 27 GB", | 20260506 19:29:10.678 | INFO | "malloc" : " 14 GB", | 20260506 19:29:10.678 | INFO | "pinned" : " 29 GB" | 20260506 19:29:10.678 | INFO | } | 20260506 19:29:10.678 | INFO | Perf report for the last 5.13912 seconds: | 20260506 19:29:10.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:29:10.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:10.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:10.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:10.678 | INFO | ANALYSIS : 62.5%, 2.9, 95.2ms, 0 | 20260506 19:29:10.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:10.679 | INFO | RT_METRICS : 48.3%, 1.0, 25.3ms, 0 | 20260506 19:29:10.679 | INFO | PRE_HQ : 60.0%, 1.8, 57.5ms, 0 | 20260506 19:29:10.679 | INFO | BAZWRITER : 69.4%, 4.8, 169.8ms, 0 | 20260506 19:29:10.679 | INFO | BAZ_SAMPLING : 3.8%, 1.0, 2.0ms, 0 | 20260506 19:29:10.679 | INFO | Uploaded 36GiB to GPU in 2.00854s (17.9235GiB/s) | 20260506 19:29:10.679 | INFO | Downloaded 38GiB to GPU in 2.20461s (17.2366GiB/s) | 20260506 19:29:15.815 | INFO | Analyzing chunk frames = [3871232,3871744) | 20260506 19:29:15.815 | INFO | MemInfo: { | 20260506 19:29:15.815 | INFO | "gpu" : " 17 GB", | 20260506 19:29:15.815 | INFO | "huge" : "", | 20260506 19:29:15.815 | INFO | "huge_shared" : " 27 GB", | 20260506 19:29:15.815 | INFO | "malloc" : " 14 GB", | 20260506 19:29:15.815 | INFO | "pinned" : " 29 GB" | 20260506 19:29:15.815 | INFO | } | 20260506 19:29:15.816 | INFO | Perf report for the last 5.13708 seconds: | 20260506 19:29:15.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:29:15.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:15.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:15.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:15.816 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:29:15.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:15.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:15.816 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:29:15.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:15.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:15.816 | INFO | Uploaded 36GiB to GPU in 1.99268s (18.0662GiB/s) | 20260506 19:29:15.816 | INFO | Downloaded 34GiB to GPU in 1.92536s (17.6591GiB/s) | 20260506 19:29:20.953 | INFO | Analyzing chunk frames = [3871744,3872256) | 20260506 19:29:20.953 | INFO | MemInfo: { | 20260506 19:29:20.953 | INFO | "gpu" : " 17 GB", | 20260506 19:29:20.953 | INFO | "huge" : "", | 20260506 19:29:20.953 | INFO | "huge_shared" : " 27 GB", | 20260506 19:29:20.953 | INFO | "malloc" : " 14 GB", | 20260506 19:29:20.953 | INFO | "pinned" : " 29 GB" | 20260506 19:29:20.953 | INFO | } | 20260506 19:29:20.953 | INFO | Perf report for the last 5.13727 seconds: | 20260506 19:29:20.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:29:20.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:20.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:20.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:20.953 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 19:29:20.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:20.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:20.953 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:29:20.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:20.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:20.953 | INFO | Uploaded 36GiB to GPU in 1.95845s (18.3819GiB/s) | 20260506 19:29:20.953 | INFO | Downloaded 34GiB to GPU in 1.91856s (17.7216GiB/s) | 20260506 19:29:26.098 | INFO | Analyzing chunk frames = [3872256,3872768) | 20260506 19:29:26.098 | INFO | MemInfo: { | 20260506 19:29:26.098 | INFO | "gpu" : " 17 GB", | 20260506 19:29:26.098 | INFO | "huge" : "", | 20260506 19:29:26.098 | INFO | "huge_shared" : " 27 GB", | 20260506 19:29:26.098 | INFO | "malloc" : " 14 GB", | 20260506 19:29:26.098 | INFO | "pinned" : " 29 GB" | 20260506 19:29:26.098 | INFO | } | 20260506 19:29:26.098 | INFO | Perf report for the last 5.14544 seconds: | 20260506 19:29:26.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:29:26.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:26.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:26.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:26.098 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 19:29:26.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:26.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:26.098 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:29:26.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:26.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:26.098 | INFO | Uploaded 36GiB to GPU in 1.92193s (18.7312GiB/s) | 20260506 19:29:26.099 | INFO | Downloaded 34GiB to GPU in 1.86973s (18.1844GiB/s) | 20260506 19:29:31.238 | INFO | Analyzing chunk frames = [3872768,3873280) | 20260506 19:29:31.238 | INFO | MemInfo: { | 20260506 19:29:31.238 | INFO | "gpu" : " 17 GB", | 20260506 19:29:31.238 | INFO | "huge" : "", | 20260506 19:29:31.239 | INFO | "huge_shared" : " 27 GB", | 20260506 19:29:31.239 | INFO | "malloc" : " 14 GB", | 20260506 19:29:31.239 | INFO | "pinned" : " 29 GB" | 20260506 19:29:31.239 | INFO | } | 20260506 19:29:31.239 | INFO | Perf report for the last 5.14033 seconds: | 20260506 19:29:31.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:29:31.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:31.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:31.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:31.239 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 19:29:31.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:31.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:31.239 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:29:31.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:31.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:31.239 | INFO | Uploaded 36GiB to GPU in 2.01792s (17.8402GiB/s) | 20260506 19:29:31.239 | INFO | Downloaded 34GiB to GPU in 1.91575s (17.7476GiB/s) | 20260506 19:29:36.378 | INFO | Analyzing chunk frames = [3873280,3873792) | 20260506 19:29:36.378 | INFO | MemInfo: { | 20260506 19:29:36.378 | INFO | "gpu" : " 17 GB", | 20260506 19:29:36.378 | INFO | "huge" : "", | 20260506 19:29:36.378 | INFO | "huge_shared" : " 27 GB", | 20260506 19:29:36.378 | INFO | "malloc" : " 14 GB", | 20260506 19:29:36.378 | INFO | "pinned" : " 29 GB" | 20260506 19:29:36.378 | INFO | } | 20260506 19:29:36.378 | INFO | Perf report for the last 5.1395 seconds: | 20260506 19:29:36.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:29:36.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:36.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:36.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:36.378 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 19:29:36.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:36.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:36.378 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 19:29:36.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:36.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:36.378 | INFO | Uploaded 36GiB to GPU in 2.03327s (17.7055GiB/s) | 20260506 19:29:36.378 | INFO | Downloaded 34GiB to GPU in 1.95723s (17.3715GiB/s) | 20260506 19:29:41.517 | INFO | Analyzing chunk frames = [3873792,3874304) | 20260506 19:29:41.517 | INFO | MemInfo: { | 20260506 19:29:41.517 | INFO | "gpu" : " 17 GB", | 20260506 19:29:41.517 | INFO | "huge" : "", | 20260506 19:29:41.517 | INFO | "huge_shared" : " 27 GB", | 20260506 19:29:41.517 | INFO | "malloc" : " 14 GB", | 20260506 19:29:41.517 | INFO | "pinned" : " 29 GB" | 20260506 19:29:41.517 | INFO | } | 20260506 19:29:41.517 | INFO | Perf report for the last 5.13903 seconds: | 20260506 19:29:41.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:29:41.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:41.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:41.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:41.517 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 19:29:41.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:41.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:41.517 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:29:41.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:41.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:41.517 | INFO | Uploaded 36GiB to GPU in 2.00104s (17.9906GiB/s) | 20260506 19:29:41.517 | INFO | Downloaded 34GiB to GPU in 2.01194s (16.8992GiB/s) | 20260506 19:29:46.651 | INFO | Analyzing chunk frames = [3874304,3874816) | 20260506 19:29:46.651 | INFO | MemInfo: { | 20260506 19:29:46.651 | INFO | "gpu" : " 17 GB", | 20260506 19:29:46.651 | INFO | "huge" : "", | 20260506 19:29:46.651 | INFO | "huge_shared" : " 27 GB", | 20260506 19:29:46.651 | INFO | "malloc" : " 14 GB", | 20260506 19:29:46.651 | INFO | "pinned" : " 29 GB" | 20260506 19:29:46.651 | INFO | } | 20260506 19:29:46.651 | INFO | Perf report for the last 5.13395 seconds: | 20260506 19:29:46.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:29:46.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:46.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:46.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:46.651 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 19:29:46.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:46.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:46.651 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:29:46.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:46.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:46.651 | INFO | Uploaded 36GiB to GPU in 2.01442s (17.8711GiB/s) | 20260506 19:29:46.651 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:29:46.651 | INFO | Total mem usage: 18 GiB | 20260506 19:29:46.651 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:29:46.651 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.4%) | 20260506 19:29:46.651 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:29:46.651 | INFO | High Water Breakdown: | 20260506 19:29:46.651 | INFO | 61.3% : 9.48 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:29:46.651 | INFO | 38.6% : 5.97 GiB ( 89.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:29:46.651 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:29:46.651 | INFO | ------------------------------------------ | 20260506 19:29:46.651 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:29:46.651 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:29:46.651 | INFO | Total mem usage: 26.5 GiB | 20260506 19:29:46.651 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:29:46.651 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:29:46.651 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:29:46.651 | INFO | High Water Breakdown: | 20260506 19:29:46.651 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:29:46.651 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:29:46.651 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:29:46.651 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:29:46.651 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:29:46.651 | INFO | Downloaded 34GiB to GPU in 1.99414s (17.05GiB/s) | 20260506 19:29:46.651 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:29:46.651 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:29:46.652 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:29:46.652 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:29:46.652 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:29:46.652 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:29:46.652 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:29:46.652 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:29:46.652 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:29:46.652 | INFO | ------------------------------------------ | 20260506 19:29:46.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:29:46.652 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:29:46.652 | INFO | Total mem usage: 25.6 GiB | 20260506 19:29:46.652 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:29:46.652 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:29:46.652 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:29:46.652 | INFO | High Water Breakdown: | 20260506 19:29:46.652 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:29:46.652 | INFO | ------------------------------------------ | 20260506 19:29:46.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:29:46.652 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:29:46.652 | INFO | Total mem usage: 15.7 GiB | 20260506 19:29:46.652 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:29:46.652 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:29:46.652 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 19:29:46.652 | INFO | High Water Breakdown: | 20260506 19:29:46.652 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:29:46.652 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:29:46.652 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:29:46.652 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:29:46.652 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:29:46.652 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:29:46.652 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:29:46.652 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:29:46.652 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:29:46.652 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:29:46.652 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:29:46.652 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:29:46.652 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:29:46.652 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:29:46.652 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:29:46.652 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:29:46.652 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:29:46.652 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:29:46.652 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:29:46.652 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:29:46.652 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:29:46.652 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:29:46.652 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:29:46.652 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:29:46.652 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:29:46.652 | INFO | ------------------------------------------ | 20260506 19:29:46.652 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:29:46.652 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:29:46.652 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:29:46.652 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:29:46.652 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.61e+01, 1.34e+01 / 3.14e+00 | 20260506 19:29:46.652 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.51e-03 / 6.67e-03 | 20260506 19:29:46.652 | INFO | PerfReport: 11.60%, Baseline -- 3.62e+00 / 4.39e+00, 5.48e-01 / 3.58e-01 | 20260506 19:29:46.652 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.44e+00, 1.81e-01 / 1.31e-01 | 20260506 19:29:46.652 | INFO | PerfReport: 16.04%, DME -- 4.01e+01 / 4.50e+01, 4.32e+00 / 3.07e+00 | 20260506 19:29:46.652 | INFO | PerfReport: 57.09%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.39e+00 / 1.84e+00 | 20260506 19:29:46.652 | INFO | PerfReport: 7.20%, PulseAccumulating -- 2.25e+00 / 2.82e+00, 3.85e-01 / 1.77e-01 | 20260506 19:29:46.652 | INFO | PerfReport: 3.36%, Metrics -- 1.05e+00 / 1.38e+00, 2.78e-01 / 1.77e-01 | 20260506 19:29:46.652 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.61e-01 / 4.07e-01, 5.66e-02 / 2.10e-02 | 20260506 19:29:46.652 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:29:46.652 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:29:46.652 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:29:46.652 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 19:29:46.652 | INFO | PerfReport: 21.44%, Upload -- 2.08e+01 / 0.00e+00, 6.93e+00 / 0.00e+00 | 20260506 19:29:46.652 | INFO | PerfReport: 21.20%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 19:29:46.652 | INFO | PerfReport: 32.23%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 19:29:46.652 | INFO | PerfReport: 1.28%, LockUpload -- 1.24e+00 / 0.00e+00, 4.99e+00 / 0.00e+00 | 20260506 19:29:46.652 | INFO | PerfReport: 1.15%, LockDownload -- 1.12e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 19:29:46.652 | INFO | PerfReport: 22.68%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 19:29:46.652 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260506 19:29:46.765 | INFO | Starting a round of Baz IO | 20260506 19:29:49.906 | INFO | Populating RTMetrics report | 20260506 19:29:49.906 | INFO | RTMetrics report populated | 20260506 19:29:49.906 | INFO | Opening tmp RTMetrics json file | 20260506 19:29:49.967 | INFO | Serializing RTMetrics to json | 20260506 19:29:49.971 | INFO | Serialization done, swapping tmp file now... | 20260506 19:29:49.977 | INFO | RTMetrics done | 20260506 19:29:50.448 | INFO | Finished a round of Baz IO: Wrote 2.008 GiB in 3.683 seconds | 20260506 19:29:50.448 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.60% / 0.00% / 65.40% | 20260506 19:29:50.448 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.907207% / 1.58946e-05% / 6.35783e-05% | 20260506 19:29:50.448 | INFO | Baz IO overall: | 20260506 19:29:50.448 | INFO | BytesWritten : 2047629301727 | 20260506 19:29:50.448 | INFO | NumEvents : 989783881381 | 20260506 19:29:51.791 | INFO | Analyzing chunk frames = [3874816,3875328) | 20260506 19:29:51.791 | INFO | MemInfo: { | 20260506 19:29:51.791 | INFO | "gpu" : " 17 GB", | 20260506 19:29:51.791 | INFO | "huge" : "", | 20260506 19:29:51.791 | INFO | "huge_shared" : " 27 GB", | 20260506 19:29:51.791 | INFO | "malloc" : " 14 GB", | 20260506 19:29:51.791 | INFO | "pinned" : " 29 GB" | 20260506 19:29:51.791 | INFO | } | 20260506 19:29:51.791 | INFO | Perf report for the last 5.13972 seconds: | 20260506 19:29:51.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:29:51.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:51.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:51.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:51.791 | INFO | ANALYSIS : 62.7%, 2.9, 95.5ms, 0 | 20260506 19:29:51.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:51.791 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 19:29:51.791 | INFO | PRE_HQ : 60.1%, 1.9, 57.8ms, 0 | 20260506 19:29:51.791 | INFO | BAZWRITER : 71.3%, 4.8, 176.2ms, 0 | 20260506 19:29:51.791 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 19:29:51.791 | INFO | Uploaded 36GiB to GPU in 2.06673s (17.4188GiB/s) | 20260506 19:29:51.791 | INFO | Downloaded 38GiB to GPU in 2.21704s (17.1399GiB/s) | 20260506 19:29:56.932 | INFO | Analyzing chunk frames = [3875328,3875840) | 20260506 19:29:56.932 | INFO | MemInfo: { | 20260506 19:29:56.932 | INFO | "gpu" : " 17 GB", | 20260506 19:29:56.932 | INFO | "huge" : "", | 20260506 19:29:56.932 | INFO | "huge_shared" : " 27 GB", | 20260506 19:29:56.932 | INFO | "malloc" : " 14 GB", | 20260506 19:29:56.932 | INFO | "pinned" : " 29 GB" | 20260506 19:29:56.932 | INFO | } | 20260506 19:29:56.932 | INFO | Perf report for the last 5.14131 seconds: | 20260506 19:29:56.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:29:56.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:56.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:56.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:56.932 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 19:29:56.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:56.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:56.932 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:29:56.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:56.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:29:56.932 | INFO | Uploaded 36GiB to GPU in 2.00737s (17.9339GiB/s) | 20260506 19:29:56.932 | INFO | Downloaded 34GiB to GPU in 1.93027s (17.6141GiB/s) | 20260506 19:30:02.069 | INFO | Analyzing chunk frames = [3875840,3876352) | 20260506 19:30:02.069 | INFO | MemInfo: { | 20260506 19:30:02.069 | INFO | "gpu" : " 17 GB", | 20260506 19:30:02.069 | INFO | "huge" : "", | 20260506 19:30:02.069 | INFO | "huge_shared" : " 27 GB", | 20260506 19:30:02.069 | INFO | "malloc" : " 14 GB", | 20260506 19:30:02.069 | INFO | "pinned" : " 29 GB" | 20260506 19:30:02.069 | INFO | } | 20260506 19:30:02.069 | INFO | Perf report for the last 5.13719 seconds: | 20260506 19:30:02.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:30:02.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:02.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:02.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:02.069 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 19:30:02.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:02.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:02.069 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:30:02.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:02.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:02.070 | INFO | Uploaded 36GiB to GPU in 1.98183s (18.165GiB/s) | 20260506 19:30:02.070 | INFO | Downloaded 34GiB to GPU in 1.91694s (17.7366GiB/s) | 20260506 19:30:07.213 | INFO | Analyzing chunk frames = [3876352,3876864) | 20260506 19:30:07.213 | INFO | MemInfo: { | 20260506 19:30:07.213 | INFO | "gpu" : " 17 GB", | 20260506 19:30:07.213 | INFO | "huge" : "", | 20260506 19:30:07.213 | INFO | "huge_shared" : " 27 GB", | 20260506 19:30:07.213 | INFO | "malloc" : " 14 GB", | 20260506 19:30:07.213 | INFO | "pinned" : " 29 GB" | 20260506 19:30:07.213 | INFO | } | 20260506 19:30:07.213 | INFO | Perf report for the last 5.14392 seconds: | 20260506 19:30:07.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:30:07.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:07.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:07.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:07.213 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:30:07.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:07.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:07.213 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:30:07.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:07.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:07.214 | INFO | Uploaded 36GiB to GPU in 1.93584s (18.5965GiB/s) | 20260506 19:30:07.214 | INFO | Downloaded 34GiB to GPU in 1.86934s (18.1883GiB/s) | 20260506 19:30:12.354 | INFO | Analyzing chunk frames = [3876864,3877376) | 20260506 19:30:12.354 | INFO | MemInfo: { | 20260506 19:30:12.354 | INFO | "gpu" : " 17 GB", | 20260506 19:30:12.354 | INFO | "huge" : "", | 20260506 19:30:12.354 | INFO | "huge_shared" : " 27 GB", | 20260506 19:30:12.354 | INFO | "malloc" : " 14 GB", | 20260506 19:30:12.354 | INFO | "pinned" : " 29 GB" | 20260506 19:30:12.354 | INFO | } | 20260506 19:30:12.354 | INFO | Perf report for the last 5.14081 seconds: | 20260506 19:30:12.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:30:12.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:12.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:12.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:12.354 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 19:30:12.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:12.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:12.354 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:30:12.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:12.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:12.354 | INFO | Uploaded 36GiB to GPU in 2.02106s (17.8124GiB/s) | 20260506 19:30:12.355 | INFO | Downloaded 34GiB to GPU in 1.91007s (17.8004GiB/s) | 20260506 19:30:17.492 | INFO | Analyzing chunk frames = [3877376,3877888) | 20260506 19:30:17.492 | INFO | MemInfo: { | 20260506 19:30:17.492 | INFO | "gpu" : " 17 GB", | 20260506 19:30:17.492 | INFO | "huge" : "", | 20260506 19:30:17.492 | INFO | "huge_shared" : " 27 GB", | 20260506 19:30:17.492 | INFO | "malloc" : " 14 GB", | 20260506 19:30:17.492 | INFO | "pinned" : " 29 GB" | 20260506 19:30:17.492 | INFO | } | 20260506 19:30:17.492 | INFO | Perf report for the last 5.13789 seconds: | 20260506 19:30:17.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:30:17.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:17.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:17.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:17.492 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:30:17.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:17.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:17.492 | INFO | PRE_HQ : 31.3%, 1.0, 16.3ms, 0 | 20260506 19:30:17.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:17.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:17.492 | INFO | Uploaded 36GiB to GPU in 2.07278s (17.368GiB/s) | 20260506 19:30:17.492 | INFO | Downloaded 34GiB to GPU in 1.96808s (17.2757GiB/s) | 20260506 19:30:22.626 | INFO | Analyzing chunk frames = [3877888,3878400) | 20260506 19:30:22.626 | INFO | MemInfo: { | 20260506 19:30:22.626 | INFO | "gpu" : " 17 GB", | 20260506 19:30:22.626 | INFO | "huge" : "", | 20260506 19:30:22.626 | INFO | "huge_shared" : " 27 GB", | 20260506 19:30:22.626 | INFO | "malloc" : " 14 GB", | 20260506 19:30:22.626 | INFO | "pinned" : " 29 GB" | 20260506 19:30:22.626 | INFO | } | 20260506 19:30:22.626 | INFO | Perf report for the last 5.13372 seconds: | 20260506 19:30:22.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:30:22.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:22.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:22.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:22.626 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 19:30:22.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:22.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:22.626 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260506 19:30:22.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:22.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:22.626 | INFO | Uploaded 36GiB to GPU in 2.05482s (17.5198GiB/s) | 20260506 19:30:22.626 | INFO | Downloaded 34GiB to GPU in 2.00756s (16.936GiB/s) | 20260506 19:30:27.766 | INFO | Analyzing chunk frames = [3878400,3878912) | 20260506 19:30:27.766 | INFO | MemInfo: { | 20260506 19:30:27.766 | INFO | "gpu" : " 17 GB", | 20260506 19:30:27.766 | INFO | "huge" : "", | 20260506 19:30:27.766 | INFO | "huge_shared" : " 27 GB", | 20260506 19:30:27.766 | INFO | "malloc" : " 14 GB", | 20260506 19:30:27.766 | INFO | "pinned" : " 29 GB" | 20260506 19:30:27.766 | INFO | } | 20260506 19:30:27.767 | INFO | Perf report for the last 5.14073 seconds: | 20260506 19:30:27.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:30:27.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:27.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:27.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:27.767 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:30:27.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:27.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:27.767 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:30:27.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:27.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:27.767 | INFO | Uploaded 36GiB to GPU in 1.99752s (18.0224GiB/s) | 20260506 19:30:27.767 | INFO | Downloaded 34GiB to GPU in 1.97174s (17.2437GiB/s) | 20260506 19:30:27.870 | INFO | Starting a round of Baz IO | 20260506 19:30:31.014 | INFO | Populating RTMetrics report | 20260506 19:30:31.015 | INFO | RTMetrics report populated | 20260506 19:30:31.015 | INFO | Opening tmp RTMetrics json file | 20260506 19:30:31.071 | INFO | Serializing RTMetrics to json | 20260506 19:30:31.076 | INFO | Serialization done, swapping tmp file now... | 20260506 19:30:31.079 | INFO | RTMetrics done | 20260506 19:30:31.407 | INFO | Finished a round of Baz IO: Wrote 2.007 GiB in 3.537 seconds | 20260506 19:30:31.407 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.60% / 0.00% / 65.40% | 20260506 19:30:31.407 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.903479% / 1.19209e-05% / 7.54992e-05% | 20260506 19:30:31.407 | INFO | Baz IO overall: | 20260506 19:30:31.407 | INFO | BytesWritten : 2049783765872 | 20260506 19:30:31.407 | INFO | NumEvents : 990819347252 | 20260506 19:30:32.908 | INFO | Analyzing chunk frames = [3878912,3879424) | 20260506 19:30:32.908 | INFO | MemInfo: { | 20260506 19:30:32.908 | INFO | "gpu" : " 17 GB", | 20260506 19:30:32.908 | INFO | "huge" : "", | 20260506 19:30:32.908 | INFO | "huge_shared" : " 27 GB", | 20260506 19:30:32.908 | INFO | "malloc" : " 14 GB", | 20260506 19:30:32.908 | INFO | "pinned" : " 29 GB" | 20260506 19:30:32.908 | INFO | } | 20260506 19:30:32.908 | INFO | Perf report for the last 5.14164 seconds: | 20260506 19:30:32.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:30:32.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:32.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:32.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:32.908 | INFO | ANALYSIS : 62.6%, 2.9, 95.3ms, 0 | 20260506 19:30:32.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:32.908 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 19:30:32.908 | INFO | PRE_HQ : 60.4%, 1.8, 58.0ms, 0 | 20260506 19:30:32.908 | INFO | BAZWRITER : 68.4%, 4.8, 168.1ms, 0 | 20260506 19:30:32.908 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 19:30:32.908 | INFO | Uploaded 36GiB to GPU in 2.05723s (17.4992GiB/s) | 20260506 19:30:32.909 | INFO | Downloaded 38GiB to GPU in 2.19821s (17.2868GiB/s) | 20260506 19:30:38.045 | INFO | Analyzing chunk frames = [3879424,3879936) | 20260506 19:30:38.045 | INFO | MemInfo: { | 20260506 19:30:38.045 | INFO | "gpu" : " 17 GB", | 20260506 19:30:38.045 | INFO | "huge" : "", | 20260506 19:30:38.045 | INFO | "huge_shared" : " 27 GB", | 20260506 19:30:38.045 | INFO | "malloc" : " 14 GB", | 20260506 19:30:38.045 | INFO | "pinned" : " 29 GB" | 20260506 19:30:38.045 | INFO | } | 20260506 19:30:38.045 | INFO | Perf report for the last 5.13696 seconds: | 20260506 19:30:38.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:30:38.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:38.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:38.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:38.045 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:30:38.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:38.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:38.045 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:30:38.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:38.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:38.045 | INFO | Uploaded 36GiB to GPU in 2.00319s (17.9713GiB/s) | 20260506 19:30:38.045 | INFO | Downloaded 34GiB to GPU in 1.91826s (17.7244GiB/s) | 20260506 19:30:43.192 | INFO | Analyzing chunk frames = [3879936,3880448) | 20260506 19:30:43.192 | INFO | MemInfo: { | 20260506 19:30:43.192 | INFO | "gpu" : " 17 GB", | 20260506 19:30:43.192 | INFO | "huge" : "", | 20260506 19:30:43.192 | INFO | "huge_shared" : " 27 GB", | 20260506 19:30:43.192 | INFO | "malloc" : " 14 GB", | 20260506 19:30:43.192 | INFO | "pinned" : " 29 GB" | 20260506 19:30:43.192 | INFO | } | 20260506 19:30:43.192 | INFO | Perf report for the last 5.1467 seconds: | 20260506 19:30:43.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:30:43.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:43.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:43.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:43.192 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 19:30:43.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:43.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:43.192 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:30:43.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:43.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:43.192 | INFO | Uploaded 36GiB to GPU in 1.9786s (18.1947GiB/s) | 20260506 19:30:43.192 | INFO | Downloaded 34GiB to GPU in 1.91665s (17.7393GiB/s) | 20260506 19:30:48.328 | INFO | Analyzing chunk frames = [3880448,3880960) | 20260506 19:30:48.328 | INFO | MemInfo: { | 20260506 19:30:48.328 | INFO | "gpu" : " 17 GB", | 20260506 19:30:48.328 | INFO | "huge" : "", | 20260506 19:30:48.328 | INFO | "huge_shared" : " 27 GB", | 20260506 19:30:48.328 | INFO | "malloc" : " 14 GB", | 20260506 19:30:48.328 | INFO | "pinned" : " 29 GB" | 20260506 19:30:48.328 | INFO | } | 20260506 19:30:48.328 | INFO | Perf report for the last 5.13646 seconds: | 20260506 19:30:48.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:30:48.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:48.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:48.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:48.328 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 19:30:48.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:48.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:48.328 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 19:30:48.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:48.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:48.329 | INFO | Uploaded 36GiB to GPU in 1.93027s (18.6502GiB/s) | 20260506 19:30:48.329 | INFO | Downloaded 34GiB to GPU in 1.87078s (18.1742GiB/s) | 20260506 19:30:53.468 | INFO | Analyzing chunk frames = [3880960,3881472) | 20260506 19:30:53.468 | INFO | MemInfo: { | 20260506 19:30:53.468 | INFO | "gpu" : " 17 GB", | 20260506 19:30:53.468 | INFO | "huge" : "", | 20260506 19:30:53.468 | INFO | "huge_shared" : " 27 GB", | 20260506 19:30:53.468 | INFO | "malloc" : " 14 GB", | 20260506 19:30:53.468 | INFO | "pinned" : " 29 GB" | 20260506 19:30:53.468 | INFO | } | 20260506 19:30:53.468 | INFO | Perf report for the last 5.14001 seconds: | 20260506 19:30:53.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:30:53.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:53.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:53.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:53.468 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 19:30:53.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:53.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:53.468 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:30:53.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:53.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:53.469 | INFO | Uploaded 36GiB to GPU in 2.00118s (17.9894GiB/s) | 20260506 19:30:53.469 | INFO | Downloaded 34GiB to GPU in 1.90901s (17.8103GiB/s) | 20260506 19:30:58.608 | INFO | Analyzing chunk frames = [3881472,3881984) | 20260506 19:30:58.608 | INFO | MemInfo: { | 20260506 19:30:58.609 | INFO | "gpu" : " 17 GB", | 20260506 19:30:58.609 | INFO | "huge" : "", | 20260506 19:30:58.609 | INFO | "huge_shared" : " 27 GB", | 20260506 19:30:58.609 | INFO | "malloc" : " 14 GB", | 20260506 19:30:58.609 | INFO | "pinned" : " 29 GB" | 20260506 19:30:58.609 | INFO | } | 20260506 19:30:58.609 | INFO | Perf report for the last 5.14026 seconds: | 20260506 19:30:58.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:30:58.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:58.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:58.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:58.609 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 19:30:58.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:58.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:58.609 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 19:30:58.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:58.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:30:58.609 | INFO | Uploaded 36GiB to GPU in 2.02472s (17.7803GiB/s) | 20260506 19:30:58.609 | INFO | Downloaded 34GiB to GPU in 1.94753s (17.458GiB/s) | 20260506 19:31:03.740 | INFO | Analyzing chunk frames = [3881984,3882496) | 20260506 19:31:03.740 | INFO | MemInfo: { | 20260506 19:31:03.740 | INFO | "gpu" : " 17 GB", | 20260506 19:31:03.740 | INFO | "huge" : "", | 20260506 19:31:03.740 | INFO | "huge_shared" : " 27 GB", | 20260506 19:31:03.740 | INFO | "malloc" : " 14 GB", | 20260506 19:31:03.740 | INFO | "pinned" : " 29 GB" | 20260506 19:31:03.740 | INFO | } | 20260506 19:31:03.740 | INFO | Perf report for the last 5.13135 seconds: | 20260506 19:31:03.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:31:03.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:03.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:03.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:03.740 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260506 19:31:03.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:03.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:03.740 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 19:31:03.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:03.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:03.740 | INFO | Uploaded 36GiB to GPU in 2.01316s (17.8823GiB/s) | 20260506 19:31:03.740 | INFO | Downloaded 34GiB to GPU in 1.99996s (17.0003GiB/s) | 20260506 19:31:08.881 | INFO | Analyzing chunk frames = [3882496,3883008) | 20260506 19:31:08.881 | INFO | MemInfo: { | 20260506 19:31:08.881 | INFO | "gpu" : " 17 GB", | 20260506 19:31:08.881 | INFO | "huge" : "", | 20260506 19:31:08.881 | INFO | "huge_shared" : " 27 GB", | 20260506 19:31:08.881 | INFO | "malloc" : " 14 GB", | 20260506 19:31:08.881 | INFO | "pinned" : " 29 GB" | 20260506 19:31:08.881 | INFO | } | 20260506 19:31:08.881 | INFO | Perf report for the last 5.14145 seconds: | 20260506 19:31:08.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:31:08.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:08.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:08.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:08.881 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 19:31:08.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:08.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:08.882 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260506 19:31:08.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:08.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:08.882 | INFO | Uploaded 36GiB to GPU in 2.06186s (17.46GiB/s) | 20260506 19:31:08.882 | INFO | Downloaded 34GiB to GPU in 1.97034s (17.2559GiB/s) | 20260506 19:31:08.882 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:31:08.882 | INFO | Total mem usage: 18 GiB | 20260506 19:31:08.882 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:31:08.882 | INFO | Recent High Water Marks Sum: 16.7 GiB (Savings: -8%) | 20260506 19:31:08.882 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:31:08.882 | INFO | High Water Breakdown: | 20260506 19:31:08.882 | INFO | 62.4% : 10.4 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:31:08.882 | INFO | 37.5% : 6.24 GiB ( 93.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:31:08.882 | INFO | 0.119% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:31:08.882 | INFO | ------------------------------------------ | 20260506 19:31:08.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:31:08.882 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:31:08.882 | INFO | Total mem usage: 26.5 GiB | 20260506 19:31:08.882 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:31:08.882 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:31:08.882 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:31:08.882 | INFO | High Water Breakdown: | 20260506 19:31:08.882 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:31:08.882 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:31:08.882 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:31:08.882 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:31:08.882 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:31:08.882 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:31:08.882 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:31:08.882 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:31:08.882 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:31:08.882 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:31:08.882 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:31:08.882 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:31:08.882 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:31:08.882 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:31:08.882 | INFO | ------------------------------------------ | 20260506 19:31:08.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:31:08.882 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:31:08.882 | INFO | Total mem usage: 25.6 GiB | 20260506 19:31:08.882 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:31:08.882 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:31:08.882 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:31:08.882 | INFO | High Water Breakdown: | 20260506 19:31:08.882 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:31:08.882 | INFO | ------------------------------------------ | 20260506 19:31:08.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:31:08.882 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:31:08.882 | INFO | Total mem usage: 15.7 GiB | 20260506 19:31:08.882 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:31:08.882 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:31:08.882 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:31:08.882 | INFO | High Water Breakdown: | 20260506 19:31:08.882 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:31:08.882 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:31:08.882 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:31:08.882 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:31:08.882 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:31:08.882 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:31:08.882 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:31:08.882 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:31:08.882 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:31:08.882 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:31:08.882 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:31:08.882 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:31:08.882 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:31:08.882 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:31:08.882 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:31:08.882 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:31:08.882 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:31:08.882 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:31:08.882 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:31:08.882 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:31:08.882 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:31:08.882 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:31:08.882 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:31:08.882 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:31:08.882 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:31:08.882 | INFO | ------------------------------------------ | 20260506 19:31:08.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:31:08.883 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:31:08.883 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:31:08.883 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:31:08.883 | INFO | PerfReport: Full Execution -- 3.13e+01 / 7.57e+01, 1.31e+01 / 2.17e+00 | 20260506 19:31:08.883 | INFO | PerfReport: 0.29%, AggregateStats -- 9.19e-02 / 1.02e-01, 1.10e-02 / 5.69e-03 | 20260506 19:31:08.883 | INFO | PerfReport: 11.59%, Baseline -- 3.62e+00 / 4.45e+00, 5.46e-01 / 3.11e-01 | 20260506 19:31:08.883 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.82e-01 / 1.27e-01 | 20260506 19:31:08.883 | INFO | PerfReport: 15.96%, DME -- 3.99e+01 / 4.42e+01, 3.94e+00 / 2.05e+00 | 20260506 19:31:08.883 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.79e+01 / 2.09e+01, 1.51e+00 / 1.96e+00 | 20260506 19:31:08.883 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.24e+00 / 2.76e+00, 3.76e-01 / 2.29e-01 | 20260506 19:31:08.883 | INFO | PerfReport: 3.35%, Metrics -- 1.05e+00 / 1.40e+00, 2.76e-01 / 1.72e-01 | 20260506 19:31:08.883 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.61e-01 / 4.09e-01, 5.66e-02 / 2.07e-02 | 20260506 19:31:08.883 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:31:08.883 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:31:08.883 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:31:08.883 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260506 19:31:08.883 | INFO | PerfReport: 21.59%, Upload -- 2.10e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260506 19:31:08.883 | INFO | PerfReport: 21.14%, Download -- 2.05e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 19:31:08.883 | INFO | PerfReport: 32.19%, Compute -- 3.13e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 19:31:08.883 | INFO | PerfReport: 1.50%, LockUpload -- 1.46e+00 / 0.00e+00, 5.51e+00 / 0.00e+00 | 20260506 19:31:08.883 | INFO | PerfReport: 1.01%, LockDownload -- 9.77e-01 / 0.00e+00, 2.60e+00 / 0.00e+00 | 20260506 19:31:08.883 | INFO | PerfReport: 22.56%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 19:31:08.883 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260506 19:31:09.001 | INFO | Starting a round of Baz IO | 20260506 19:31:12.150 | INFO | Populating RTMetrics report | 20260506 19:31:12.150 | INFO | RTMetrics report populated | 20260506 19:31:12.150 | INFO | Opening tmp RTMetrics json file | 20260506 19:31:12.153 | INFO | Serializing RTMetrics to json | 20260506 19:31:12.158 | INFO | Serialization done, swapping tmp file now... | 20260506 19:31:12.162 | INFO | RTMetrics done | 20260506 19:31:12.720 | INFO | Finished a round of Baz IO: Wrote 2.006 GiB in 3.718 seconds | 20260506 19:31:12.720 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.59% / 0.00% / 65.40% | 20260506 19:31:12.720 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.902061% / 0% / 6.75519e-05% | 20260506 19:31:12.720 | INFO | Baz IO overall: | 20260506 19:31:12.720 | INFO | BytesWritten : 2051937768702 | 20260506 19:31:12.720 | INFO | NumEvents : 991854659077 | 20260506 19:31:14.024 | INFO | Analyzing chunk frames = [3883008,3883520) | 20260506 19:31:14.025 | INFO | MemInfo: { | 20260506 19:31:14.025 | INFO | "gpu" : " 17 GB", | 20260506 19:31:14.025 | INFO | "huge" : "", | 20260506 19:31:14.025 | INFO | "huge_shared" : " 27 GB", | 20260506 19:31:14.025 | INFO | "malloc" : " 14 GB", | 20260506 19:31:14.025 | INFO | "pinned" : " 29 GB" | 20260506 19:31:14.025 | INFO | } | 20260506 19:31:14.025 | INFO | Perf report for the last 5.14316 seconds: | 20260506 19:31:14.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:31:14.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:14.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:14.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:14.025 | INFO | ANALYSIS : 62.8%, 2.9, 95.1ms, 0 | 20260506 19:31:14.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:14.025 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 19:31:14.025 | INFO | PRE_HQ : 60.3%, 1.9, 58.9ms, 0 | 20260506 19:31:14.025 | INFO | BAZWRITER : 71.9%, 4.8, 178.6ms, 0 | 20260506 19:31:14.025 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 19:31:14.025 | INFO | Uploaded 36GiB to GPU in 2.06013s (17.4747GiB/s) | 20260506 19:31:14.025 | INFO | Downloaded 38GiB to GPU in 2.22129s (17.1072GiB/s) | 20260506 19:31:19.161 | INFO | Analyzing chunk frames = [3883520,3884032) | 20260506 19:31:19.161 | INFO | MemInfo: { | 20260506 19:31:19.161 | INFO | "gpu" : " 17 GB", | 20260506 19:31:19.161 | INFO | "huge" : "", | 20260506 19:31:19.161 | INFO | "huge_shared" : " 27 GB", | 20260506 19:31:19.161 | INFO | "malloc" : " 14 GB", | 20260506 19:31:19.161 | INFO | "pinned" : " 29 GB" | 20260506 19:31:19.161 | INFO | } | 20260506 19:31:19.161 | INFO | Perf report for the last 5.13684 seconds: | 20260506 19:31:19.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:31:19.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:19.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:19.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:19.162 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:31:19.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:19.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:19.162 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:31:19.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:19.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:19.162 | INFO | Uploaded 36GiB to GPU in 2.01602s (17.857GiB/s) | 20260506 19:31:19.162 | INFO | Downloaded 34GiB to GPU in 1.91988s (17.7095GiB/s) | 20260506 19:31:24.299 | INFO | Analyzing chunk frames = [3884032,3884544) | 20260506 19:31:24.299 | INFO | MemInfo: { | 20260506 19:31:24.299 | INFO | "gpu" : " 17 GB", | 20260506 19:31:24.299 | INFO | "huge" : "", | 20260506 19:31:24.299 | INFO | "huge_shared" : " 27 GB", | 20260506 19:31:24.299 | INFO | "malloc" : " 14 GB", | 20260506 19:31:24.299 | INFO | "pinned" : " 29 GB" | 20260506 19:31:24.299 | INFO | } | 20260506 19:31:24.299 | INFO | Perf report for the last 5.1375 seconds: | 20260506 19:31:24.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:31:24.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:24.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:24.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:24.299 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 19:31:24.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:24.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:24.299 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:31:24.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:24.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:24.299 | INFO | Uploaded 36GiB to GPU in 1.9811s (18.1717GiB/s) | 20260506 19:31:24.299 | INFO | Downloaded 34GiB to GPU in 1.91659s (17.7398GiB/s) | 20260506 19:31:29.442 | INFO | Analyzing chunk frames = [3884544,3885056) | 20260506 19:31:29.442 | INFO | MemInfo: { | 20260506 19:31:29.442 | INFO | "gpu" : " 17 GB", | 20260506 19:31:29.442 | INFO | "huge" : "", | 20260506 19:31:29.442 | INFO | "huge_shared" : " 27 GB", | 20260506 19:31:29.442 | INFO | "malloc" : " 14 GB", | 20260506 19:31:29.442 | INFO | "pinned" : " 29 GB" | 20260506 19:31:29.442 | INFO | } | 20260506 19:31:29.442 | INFO | Perf report for the last 5.14302 seconds: | 20260506 19:31:29.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:31:29.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:29.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:29.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:29.442 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 19:31:29.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:29.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:29.442 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:31:29.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:29.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:29.442 | INFO | Uploaded 36GiB to GPU in 1.92826s (18.6697GiB/s) | 20260506 19:31:29.442 | INFO | Downloaded 34GiB to GPU in 1.86568s (18.2239GiB/s) | 20260506 19:31:34.581 | INFO | Analyzing chunk frames = [3885056,3885568) | 20260506 19:31:34.581 | INFO | MemInfo: { | 20260506 19:31:34.581 | INFO | "gpu" : " 17 GB", | 20260506 19:31:34.581 | INFO | "huge" : "", | 20260506 19:31:34.581 | INFO | "huge_shared" : " 27 GB", | 20260506 19:31:34.582 | INFO | "malloc" : " 14 GB", | 20260506 19:31:34.582 | INFO | "pinned" : " 29 GB" | 20260506 19:31:34.582 | INFO | } | 20260506 19:31:34.582 | INFO | Perf report for the last 5.13952 seconds: | 20260506 19:31:34.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:31:34.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:34.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:34.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:34.582 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 19:31:34.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:34.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:34.582 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:31:34.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:34.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:34.582 | INFO | Uploaded 36GiB to GPU in 2.02254s (17.7994GiB/s) | 20260506 19:31:34.582 | INFO | Downloaded 34GiB to GPU in 1.91571s (17.748GiB/s) | 20260506 19:31:39.718 | INFO | Analyzing chunk frames = [3885568,3886080) | 20260506 19:31:39.718 | INFO | MemInfo: { | 20260506 19:31:39.718 | INFO | "gpu" : " 17 GB", | 20260506 19:31:39.718 | INFO | "huge" : "", | 20260506 19:31:39.718 | INFO | "huge_shared" : " 27 GB", | 20260506 19:31:39.718 | INFO | "malloc" : " 14 GB", | 20260506 19:31:39.718 | INFO | "pinned" : " 29 GB" | 20260506 19:31:39.718 | INFO | } | 20260506 19:31:39.718 | INFO | Perf report for the last 5.13685 seconds: | 20260506 19:31:39.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:31:39.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:39.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:39.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:39.718 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 19:31:39.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:39.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:39.718 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:31:39.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:39.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:39.719 | INFO | Uploaded 36GiB to GPU in 2.07244s (17.3708GiB/s) | 20260506 19:31:39.719 | INFO | Downloaded 34GiB to GPU in 1.9679s (17.2773GiB/s) | 20260506 19:31:44.859 | INFO | Analyzing chunk frames = [3886080,3886592) | 20260506 19:31:44.859 | INFO | MemInfo: { | 20260506 19:31:44.859 | INFO | "gpu" : " 17 GB", | 20260506 19:31:44.859 | INFO | "huge" : "", | 20260506 19:31:44.859 | INFO | "huge_shared" : " 27 GB", | 20260506 19:31:44.859 | INFO | "malloc" : " 14 GB", | 20260506 19:31:44.859 | INFO | "pinned" : " 29 GB" | 20260506 19:31:44.859 | INFO | } | 20260506 19:31:44.859 | INFO | Perf report for the last 5.14104 seconds: | 20260506 19:31:44.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:31:44.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:44.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:44.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:44.860 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 19:31:44.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:44.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:44.860 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:31:44.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:44.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:44.860 | INFO | Uploaded 36GiB to GPU in 2.00452s (17.9594GiB/s) | 20260506 19:31:44.860 | INFO | Downloaded 34GiB to GPU in 1.98735s (17.1082GiB/s) | 20260506 19:31:49.995 | INFO | Analyzing chunk frames = [3886592,3887104) | 20260506 19:31:49.995 | INFO | MemInfo: { | 20260506 19:31:49.995 | INFO | "gpu" : " 17 GB", | 20260506 19:31:49.995 | INFO | "huge" : "", | 20260506 19:31:49.995 | INFO | "huge_shared" : " 27 GB", | 20260506 19:31:49.995 | INFO | "malloc" : " 14 GB", | 20260506 19:31:49.995 | INFO | "pinned" : " 29 GB" | 20260506 19:31:49.995 | INFO | } | 20260506 19:31:49.995 | INFO | Perf report for the last 5.13574 seconds: | 20260506 19:31:49.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:31:49.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:49.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:49.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:49.995 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 19:31:49.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:49.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:49.995 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 19:31:49.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:49.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:49.996 | INFO | Uploaded 36GiB to GPU in 2.02802s (17.7513GiB/s) | 20260506 19:31:49.996 | INFO | Downloaded 34GiB to GPU in 1.98151s (17.1586GiB/s) | 20260506 19:31:50.110 | INFO | Starting a round of Baz IO | 20260506 19:31:53.241 | INFO | Populating RTMetrics report | 20260506 19:31:53.241 | INFO | RTMetrics report populated | 20260506 19:31:53.241 | INFO | Opening tmp RTMetrics json file | 20260506 19:31:53.263 | INFO | Serializing RTMetrics to json | 20260506 19:31:53.368 | INFO | Serialization done, swapping tmp file now... | 20260506 19:31:53.390 | INFO | RTMetrics done | 20260506 19:31:53.611 | INFO | Finished a round of Baz IO: Wrote 2.005 GiB in 3.5 seconds | 20260506 19:31:53.611 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.59% / 0.00% / 65.41% | 20260506 19:31:53.611 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.899895% / 1.19209e-05% / 4.37101e-05% | 20260506 19:31:53.611 | INFO | Baz IO overall: | 20260506 19:31:53.611 | INFO | BytesWritten : 2054091007944 | 20260506 19:31:53.611 | INFO | NumEvents : 992889450439 | 20260506 19:31:55.137 | INFO | Analyzing chunk frames = [3887104,3887616) | 20260506 19:31:55.137 | INFO | MemInfo: { | 20260506 19:31:55.137 | INFO | "gpu" : " 17 GB", | 20260506 19:31:55.137 | INFO | "huge" : "", | 20260506 19:31:55.138 | INFO | "huge_shared" : " 27 GB", | 20260506 19:31:55.138 | INFO | "malloc" : " 14 GB", | 20260506 19:31:55.138 | INFO | "pinned" : " 29 GB" | 20260506 19:31:55.138 | INFO | } | 20260506 19:31:55.138 | INFO | Perf report for the last 5.14239 seconds: | 20260506 19:31:55.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:31:55.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:55.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:55.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:55.138 | INFO | ANALYSIS : 62.6%, 2.9, 95.4ms, 0 | 20260506 19:31:55.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:31:55.138 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260506 19:31:55.138 | INFO | PRE_HQ : 59.9%, 1.8, 57.4ms, 0 | 20260506 19:31:55.138 | INFO | BAZWRITER : 67.8%, 4.8, 167.0ms, 0 | 20260506 19:31:55.138 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 19:31:55.138 | INFO | Uploaded 36GiB to GPU in 2.04541s (17.6003GiB/s) | 20260506 19:31:55.138 | INFO | Downloaded 38GiB to GPU in 2.21449s (17.1597GiB/s) | 20260506 19:32:00.286 | INFO | Analyzing chunk frames = [3887616,3888128) | 20260506 19:32:00.286 | INFO | MemInfo: { | 20260506 19:32:00.286 | INFO | "gpu" : " 17 GB", | 20260506 19:32:00.286 | INFO | "huge" : "", | 20260506 19:32:00.286 | INFO | "huge_shared" : " 27 GB", | 20260506 19:32:00.286 | INFO | "malloc" : " 14 GB", | 20260506 19:32:00.286 | INFO | "pinned" : " 29 GB" | 20260506 19:32:00.286 | INFO | } | 20260506 19:32:00.286 | INFO | Perf report for the last 5.14856 seconds: | 20260506 19:32:00.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:32:00.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:00.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:00.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:00.286 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260506 19:32:00.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:00.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:00.286 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:32:00.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:00.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:00.286 | INFO | Uploaded 36GiB to GPU in 2.02267s (17.7983GiB/s) | 20260506 19:32:00.286 | INFO | Downloaded 34GiB to GPU in 1.93297s (17.5895GiB/s) | 20260506 19:32:05.418 | INFO | Analyzing chunk frames = [3888128,3888640) | 20260506 19:32:05.418 | INFO | MemInfo: { | 20260506 19:32:05.418 | INFO | "gpu" : " 17 GB", | 20260506 19:32:05.418 | INFO | "huge" : "", | 20260506 19:32:05.418 | INFO | "huge_shared" : " 27 GB", | 20260506 19:32:05.418 | INFO | "malloc" : " 14 GB", | 20260506 19:32:05.419 | INFO | "pinned" : " 29 GB" | 20260506 19:32:05.419 | INFO | } | 20260506 19:32:05.419 | INFO | Perf report for the last 5.13245 seconds: | 20260506 19:32:05.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:32:05.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:05.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:05.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:05.419 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 19:32:05.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:05.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:05.419 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:32:05.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:05.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:05.419 | INFO | Uploaded 36GiB to GPU in 1.96316s (18.3378GiB/s) | 20260506 19:32:05.419 | INFO | Downloaded 34GiB to GPU in 1.9168s (17.7379GiB/s) | 20260506 19:32:10.556 | INFO | Analyzing chunk frames = [3888640,3889152) | 20260506 19:32:10.556 | INFO | MemInfo: { | 20260506 19:32:10.556 | INFO | "gpu" : " 17 GB", | 20260506 19:32:10.556 | INFO | "huge" : "", | 20260506 19:32:10.556 | INFO | "huge_shared" : " 27 GB", | 20260506 19:32:10.556 | INFO | "malloc" : " 14 GB", | 20260506 19:32:10.556 | INFO | "pinned" : " 29 GB" | 20260506 19:32:10.556 | INFO | } | 20260506 19:32:10.556 | INFO | Perf report for the last 5.1376 seconds: | 20260506 19:32:10.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:32:10.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:10.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:10.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:10.556 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 19:32:10.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:10.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:10.556 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:32:10.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:10.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:10.557 | INFO | Uploaded 36GiB to GPU in 1.97493s (18.2285GiB/s) | 20260506 19:32:10.557 | INFO | Downloaded 34GiB to GPU in 1.87452s (18.138GiB/s) | 20260506 19:32:15.693 | INFO | Analyzing chunk frames = [3889152,3889664) | 20260506 19:32:15.693 | INFO | MemInfo: { | 20260506 19:32:15.693 | INFO | "gpu" : " 17 GB", | 20260506 19:32:15.693 | INFO | "huge" : "", | 20260506 19:32:15.693 | INFO | "huge_shared" : " 27 GB", | 20260506 19:32:15.693 | INFO | "malloc" : " 14 GB", | 20260506 19:32:15.693 | INFO | "pinned" : " 29 GB" | 20260506 19:32:15.693 | INFO | } | 20260506 19:32:15.693 | INFO | Perf report for the last 5.13645 seconds: | 20260506 19:32:15.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:32:15.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:15.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:15.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:15.693 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 19:32:15.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:15.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:15.693 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:32:15.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:15.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:15.693 | INFO | Uploaded 36GiB to GPU in 2.00814s (17.9271GiB/s) | 20260506 19:32:15.693 | INFO | Downloaded 34GiB to GPU in 1.91014s (17.7998GiB/s) | 20260506 19:32:20.831 | INFO | Analyzing chunk frames = [3889664,3890176) | 20260506 19:32:20.831 | INFO | MemInfo: { | 20260506 19:32:20.831 | INFO | "gpu" : " 17 GB", | 20260506 19:32:20.831 | INFO | "huge" : "", | 20260506 19:32:20.832 | INFO | "huge_shared" : " 27 GB", | 20260506 19:32:20.832 | INFO | "malloc" : " 14 GB", | 20260506 19:32:20.832 | INFO | "pinned" : " 29 GB" | 20260506 19:32:20.832 | INFO | } | 20260506 19:32:20.832 | INFO | Perf report for the last 5.13899 seconds: | 20260506 19:32:20.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:32:20.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:20.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:20.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:20.832 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 19:32:20.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:20.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:20.832 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 19:32:20.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:20.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:20.832 | INFO | Uploaded 36GiB to GPU in 2.02987s (17.7351GiB/s) | 20260506 19:32:20.832 | INFO | Downloaded 34GiB to GPU in 1.94497s (17.481GiB/s) | 20260506 19:32:25.971 | INFO | Analyzing chunk frames = [3890176,3890688) | 20260506 19:32:25.971 | INFO | MemInfo: { | 20260506 19:32:25.971 | INFO | "gpu" : " 17 GB", | 20260506 19:32:25.972 | INFO | "huge" : "", | 20260506 19:32:25.972 | INFO | "huge_shared" : " 27 GB", | 20260506 19:32:25.972 | INFO | "malloc" : " 14 GB", | 20260506 19:32:25.972 | INFO | "pinned" : " 29 GB" | 20260506 19:32:25.972 | INFO | } | 20260506 19:32:25.972 | INFO | Perf report for the last 5.13981 seconds: | 20260506 19:32:25.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:32:25.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:25.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:25.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:25.972 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 19:32:25.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:25.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:25.972 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:32:25.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:25.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:25.972 | INFO | Uploaded 36GiB to GPU in 2.01385s (17.8762GiB/s) | 20260506 19:32:25.972 | INFO | Downloaded 34GiB to GPU in 1.99775s (17.0191GiB/s) | 20260506 19:32:31.109 | INFO | Analyzing chunk frames = [3890688,3891200) | 20260506 19:32:31.109 | INFO | MemInfo: { | 20260506 19:32:31.109 | INFO | "gpu" : " 17 GB", | 20260506 19:32:31.109 | INFO | "huge" : "", | 20260506 19:32:31.109 | INFO | "huge_shared" : " 27 GB", | 20260506 19:32:31.109 | INFO | "malloc" : " 14 GB", | 20260506 19:32:31.109 | INFO | "pinned" : " 29 GB" | 20260506 19:32:31.109 | INFO | } | 20260506 19:32:31.109 | INFO | Perf report for the last 5.13746 seconds: | 20260506 19:32:31.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:32:31.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:31.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:31.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:31.109 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 19:32:31.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:31.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:31.109 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 19:32:31.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:31.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:31.109 | INFO | Uploaded 36GiB to GPU in 2.0072s (17.9355GiB/s) | 20260506 19:32:31.109 | INFO | Downloaded 34GiB to GPU in 1.98105s (17.1626GiB/s) | 20260506 19:32:31.110 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:32:31.110 | INFO | Total mem usage: 18 GiB | 20260506 19:32:31.110 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:32:31.110 | INFO | Recent High Water Marks Sum: 16.9 GiB (Savings: -6.58%) | 20260506 19:32:31.110 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:32:31.110 | INFO | High Water Breakdown: | 20260506 19:32:31.110 | INFO | 62.6% : 10.6 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:32:31.110 | INFO | 37.3% : 6.29 GiB ( 94.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:32:31.110 | INFO | 0.118% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:32:31.110 | INFO | ------------------------------------------ | 20260506 19:32:31.110 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:32:31.110 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:32:31.110 | INFO | Total mem usage: 26.5 GiB | 20260506 19:32:31.110 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:32:31.110 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:32:31.110 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:32:31.110 | INFO | High Water Breakdown: | 20260506 19:32:31.110 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:32:31.110 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:32:31.110 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:32:31.110 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:32:31.110 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:32:31.110 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:32:31.110 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:32:31.110 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:32:31.110 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:32:31.110 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:32:31.110 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:32:31.110 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:32:31.110 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:32:31.110 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:32:31.110 | INFO | ------------------------------------------ | 20260506 19:32:31.110 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:32:31.110 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:32:31.110 | INFO | Total mem usage: 25.6 GiB | 20260506 19:32:31.110 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:32:31.110 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:32:31.110 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:32:31.110 | INFO | High Water Breakdown: | 20260506 19:32:31.110 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:32:31.110 | INFO | ------------------------------------------ | 20260506 19:32:31.110 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:32:31.110 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:32:31.110 | INFO | Total mem usage: 15.7 GiB | 20260506 19:32:31.110 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:32:31.110 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:32:31.110 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:32:31.110 | INFO | High Water Breakdown: | 20260506 19:32:31.110 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:32:31.110 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:32:31.110 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:32:31.110 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:32:31.110 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:32:31.110 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:32:31.110 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:32:31.110 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:32:31.110 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:32:31.110 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:32:31.111 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:32:31.111 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:32:31.111 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:32:31.111 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:32:31.111 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:32:31.111 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:32:31.111 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:32:31.111 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:32:31.111 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:32:31.111 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:32:31.111 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:32:31.111 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:32:31.111 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:32:31.111 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:32:31.111 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:32:31.111 | INFO | ------------------------------------------ | 20260506 19:32:31.111 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:32:31.111 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:32:31.111 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:32:31.111 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:32:31.111 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.55e+01, 1.33e+01 / 2.07e+00 | 20260506 19:32:31.111 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 9.71e-03 / 5.54e-03 | 20260506 19:32:31.111 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.43e+00, 5.49e-01 / 3.66e-01 | 20260506 19:32:31.111 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.83e-01 / 1.37e-01 | 20260506 19:32:31.111 | INFO | PerfReport: 15.96%, DME -- 3.99e+01 / 4.43e+01, 3.86e+00 / 2.13e+00 | 20260506 19:32:31.111 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.49e+00 / 2.14e+00 | 20260506 19:32:31.111 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.24e+00 / 2.76e+00, 3.77e-01 / 2.70e-01 | 20260506 19:32:31.111 | INFO | PerfReport: 3.36%, Metrics -- 1.05e+00 / 1.39e+00, 2.79e-01 / 1.84e-01 | 20260506 19:32:31.111 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.61e-01 / 4.12e-01, 5.72e-02 / 3.27e-02 | 20260506 19:32:31.111 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:32:31.111 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:32:31.111 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:32:31.111 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 19:32:31.111 | INFO | PerfReport: 21.59%, Upload -- 2.09e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260506 19:32:31.111 | INFO | PerfReport: 21.17%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 19:32:31.111 | INFO | PerfReport: 32.19%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 19:32:31.111 | INFO | PerfReport: 1.50%, LockUpload -- 1.46e+00 / 0.00e+00, 5.59e+00 / 0.00e+00 | 20260506 19:32:31.111 | INFO | PerfReport: 1.05%, LockDownload -- 1.02e+00 / 0.00e+00, 2.68e+00 / 0.00e+00 | 20260506 19:32:31.111 | INFO | PerfReport: 22.48%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 19:32:31.111 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260506 19:32:31.225 | INFO | Starting a round of Baz IO | 20260506 19:32:34.353 | INFO | Populating RTMetrics report | 20260506 19:32:34.353 | INFO | RTMetrics report populated | 20260506 19:32:34.353 | INFO | Opening tmp RTMetrics json file | 20260506 19:32:34.591 | INFO | Serializing RTMetrics to json | 20260506 19:32:34.638 | INFO | Serialization done, swapping tmp file now... | 20260506 19:32:34.667 | INFO | RTMetrics done | 20260506 19:32:34.702 | INFO | Finished a round of Baz IO: Wrote 2.006 GiB in 3.476 seconds | 20260506 19:32:34.702 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.58% / 0.00% / 65.41% | 20260506 19:32:34.702 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.898834% / 1.19209e-05% / 5.96046e-05% | 20260506 19:32:34.702 | INFO | Baz IO overall: | 20260506 19:32:34.702 | INFO | BytesWritten : 2056244982528 | 20260506 19:32:34.702 | INFO | NumEvents : 993924643835 | 20260506 19:32:36.249 | INFO | Analyzing chunk frames = [3891200,3891712) | 20260506 19:32:36.249 | INFO | MemInfo: { | 20260506 19:32:36.249 | INFO | "gpu" : " 17 GB", | 20260506 19:32:36.249 | INFO | "huge" : "", | 20260506 19:32:36.249 | INFO | "huge_shared" : " 27 GB", | 20260506 19:32:36.249 | INFO | "malloc" : " 14 GB", | 20260506 19:32:36.249 | INFO | "pinned" : " 29 GB" | 20260506 19:32:36.249 | INFO | } | 20260506 19:32:36.249 | INFO | Perf report for the last 5.14014 seconds: | 20260506 19:32:36.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:32:36.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:36.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:36.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:36.249 | INFO | ANALYSIS : 62.5%, 2.9, 95.2ms, 0 | 20260506 19:32:36.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:36.249 | INFO | RT_METRICS : 49.3%, 1.0, 25.9ms, 0 | 20260506 19:32:36.249 | INFO | PRE_HQ : 60.0%, 1.9, 57.8ms, 0 | 20260506 19:32:36.249 | INFO | BAZWRITER : 67.3%, 4.6, 157.5ms, 0 | 20260506 19:32:36.249 | INFO | BAZ_SAMPLING : 4.9%, 1.0, 2.5ms, 0 | 20260506 19:32:36.249 | INFO | Uploaded 36GiB to GPU in 1.99034s (18.0874GiB/s) | 20260506 19:32:36.250 | INFO | Downloaded 38GiB to GPU in 2.20547s (17.2299GiB/s) | 20260506 19:32:41.392 | INFO | Analyzing chunk frames = [3891712,3892224) | 20260506 19:32:41.392 | INFO | MemInfo: { | 20260506 19:32:41.392 | INFO | "gpu" : " 17 GB", | 20260506 19:32:41.392 | INFO | "huge" : "", | 20260506 19:32:41.392 | INFO | "huge_shared" : " 27 GB", | 20260506 19:32:41.392 | INFO | "malloc" : " 14 GB", | 20260506 19:32:41.392 | INFO | "pinned" : " 29 GB" | 20260506 19:32:41.392 | INFO | } | 20260506 19:32:41.392 | INFO | Perf report for the last 5.14297 seconds: | 20260506 19:32:41.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:32:41.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:41.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:41.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:41.392 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:32:41.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:41.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:41.392 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:32:41.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:41.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:41.392 | INFO | Uploaded 36GiB to GPU in 2.02129s (17.8104GiB/s) | 20260506 19:32:41.393 | INFO | Downloaded 34GiB to GPU in 1.963s (17.3204GiB/s) | 20260506 19:32:46.537 | INFO | Analyzing chunk frames = [3892224,3892736) | 20260506 19:32:46.537 | INFO | MemInfo: { | 20260506 19:32:46.537 | INFO | "gpu" : " 17 GB", | 20260506 19:32:46.537 | INFO | "huge" : "", | 20260506 19:32:46.537 | INFO | "huge_shared" : " 27 GB", | 20260506 19:32:46.537 | INFO | "malloc" : " 14 GB", | 20260506 19:32:46.537 | INFO | "pinned" : " 29 GB" | 20260506 19:32:46.537 | INFO | } | 20260506 19:32:46.537 | INFO | Perf report for the last 5.14455 seconds: | 20260506 19:32:46.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:32:46.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:46.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:46.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:46.537 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 19:32:46.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:46.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:46.537 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:32:46.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:46.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:46.537 | INFO | Uploaded 36GiB to GPU in 1.96557s (18.3153GiB/s) | 20260506 19:32:46.537 | INFO | Downloaded 34GiB to GPU in 1.9428s (17.5005GiB/s) | 20260506 19:32:51.673 | INFO | Analyzing chunk frames = [3892736,3893248) | 20260506 19:32:51.673 | INFO | MemInfo: { | 20260506 19:32:51.673 | INFO | "gpu" : " 17 GB", | 20260506 19:32:51.673 | INFO | "huge" : "", | 20260506 19:32:51.673 | INFO | "huge_shared" : " 27 GB", | 20260506 19:32:51.673 | INFO | "malloc" : " 14 GB", | 20260506 19:32:51.673 | INFO | "pinned" : " 29 GB" | 20260506 19:32:51.673 | INFO | } | 20260506 19:32:51.673 | INFO | Perf report for the last 5.13644 seconds: | 20260506 19:32:51.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:32:51.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:51.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:51.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:51.673 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 19:32:51.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:51.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:51.673 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:32:51.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:51.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:51.673 | INFO | Uploaded 36GiB to GPU in 1.97819s (18.1985GiB/s) | 20260506 19:32:51.674 | INFO | Downloaded 34GiB to GPU in 1.87829s (18.1016GiB/s) | 20260506 19:32:56.811 | INFO | Analyzing chunk frames = [3893248,3893760) | 20260506 19:32:56.811 | INFO | MemInfo: { | 20260506 19:32:56.811 | INFO | "gpu" : " 17 GB", | 20260506 19:32:56.811 | INFO | "huge" : "", | 20260506 19:32:56.811 | INFO | "huge_shared" : " 27 GB", | 20260506 19:32:56.811 | INFO | "malloc" : " 14 GB", | 20260506 19:32:56.811 | INFO | "pinned" : " 29 GB" | 20260506 19:32:56.811 | INFO | } | 20260506 19:32:56.811 | INFO | Perf report for the last 5.13828 seconds: | 20260506 19:32:56.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:32:56.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:56.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:56.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:56.812 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 19:32:56.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:56.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:56.812 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:32:56.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:56.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:32:56.812 | INFO | Uploaded 36GiB to GPU in 1.99091s (18.0822GiB/s) | 20260506 19:32:56.812 | INFO | Downloaded 34GiB to GPU in 1.92141s (17.6953GiB/s) | 20260506 19:33:01.952 | INFO | Analyzing chunk frames = [3893760,3894272) | 20260506 19:33:01.952 | INFO | MemInfo: { | 20260506 19:33:01.952 | INFO | "gpu" : " 17 GB", | 20260506 19:33:01.952 | INFO | "huge" : "", | 20260506 19:33:01.952 | INFO | "huge_shared" : " 27 GB", | 20260506 19:33:01.952 | INFO | "malloc" : " 14 GB", | 20260506 19:33:01.952 | INFO | "pinned" : " 29 GB" | 20260506 19:33:01.952 | INFO | } | 20260506 19:33:01.952 | INFO | Perf report for the last 5.14092 seconds: | 20260506 19:33:01.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:33:01.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:01.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:01.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:01.952 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 19:33:01.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:01.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:01.952 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 19:33:01.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:01.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:01.953 | INFO | Uploaded 36GiB to GPU in 2.002s (17.982GiB/s) | 20260506 19:33:01.953 | INFO | Downloaded 34GiB to GPU in 1.95244s (17.4141GiB/s) | 20260506 19:33:07.086 | INFO | Analyzing chunk frames = [3894272,3894784) | 20260506 19:33:07.086 | INFO | MemInfo: { | 20260506 19:33:07.086 | INFO | "gpu" : " 17 GB", | 20260506 19:33:07.086 | INFO | "huge" : "", | 20260506 19:33:07.086 | INFO | "huge_shared" : " 27 GB", | 20260506 19:33:07.086 | INFO | "malloc" : " 14 GB", | 20260506 19:33:07.086 | INFO | "pinned" : " 29 GB" | 20260506 19:33:07.086 | INFO | } | 20260506 19:33:07.086 | INFO | Perf report for the last 5.13368 seconds: | 20260506 19:33:07.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:33:07.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:07.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:07.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:07.086 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 19:33:07.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:07.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:07.086 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 19:33:07.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:07.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:07.087 | INFO | Uploaded 36GiB to GPU in 1.97597s (18.2189GiB/s) | 20260506 19:33:07.087 | INFO | Downloaded 34GiB to GPU in 2.00662s (16.9439GiB/s) | 20260506 19:33:12.224 | INFO | Analyzing chunk frames = [3894784,3895296) | 20260506 19:33:12.224 | INFO | MemInfo: { | 20260506 19:33:12.224 | INFO | "gpu" : " 17 GB", | 20260506 19:33:12.224 | INFO | "huge" : "", | 20260506 19:33:12.224 | INFO | "huge_shared" : " 27 GB", | 20260506 19:33:12.224 | INFO | "malloc" : " 14 GB", | 20260506 19:33:12.224 | INFO | "pinned" : " 29 GB" | 20260506 19:33:12.224 | INFO | } | 20260506 19:33:12.224 | INFO | Perf report for the last 5.13807 seconds: | 20260506 19:33:12.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:33:12.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:12.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:12.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:12.224 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 19:33:12.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:12.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:12.224 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:33:12.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:12.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:12.225 | INFO | Uploaded 36GiB to GPU in 1.98917s (18.098GiB/s) | 20260506 19:33:12.225 | INFO | Downloaded 34GiB to GPU in 1.97339s (17.2292GiB/s) | 20260506 19:33:12.332 | INFO | Starting a round of Baz IO | 20260506 19:33:15.462 | INFO | Populating RTMetrics report | 20260506 19:33:15.462 | INFO | RTMetrics report populated | 20260506 19:33:15.462 | INFO | Opening tmp RTMetrics json file | 20260506 19:33:15.464 | INFO | Serializing RTMetrics to json | 20260506 19:33:15.468 | INFO | Serialization done, swapping tmp file now... | 20260506 19:33:15.469 | INFO | RTMetrics done | 20260506 19:33:15.667 | INFO | Finished a round of Baz IO: Wrote 2.006 GiB in 3.335 seconds | 20260506 19:33:15.667 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.58% / 0.00% / 65.42% | 20260506 19:33:15.667 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.896927% / 1.19209e-05% / 6.35783e-05% | 20260506 19:33:15.667 | INFO | Baz IO overall: | 20260506 19:33:15.667 | INFO | BytesWritten : 2058399286136 | 20260506 19:33:15.667 | INFO | NumEvents : 994960586699 | 20260506 19:33:17.371 | INFO | Analyzing chunk frames = [3895296,3895808) | 20260506 19:33:17.371 | INFO | MemInfo: { | 20260506 19:33:17.371 | INFO | "gpu" : " 17 GB", | 20260506 19:33:17.371 | INFO | "huge" : "", | 20260506 19:33:17.371 | INFO | "huge_shared" : " 27 GB", | 20260506 19:33:17.371 | INFO | "malloc" : " 14 GB", | 20260506 19:33:17.372 | INFO | "pinned" : " 29 GB" | 20260506 19:33:17.372 | INFO | } | 20260506 19:33:17.372 | INFO | Perf report for the last 5.14742 seconds: | 20260506 19:33:17.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:33:17.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:17.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:17.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:17.372 | INFO | ANALYSIS : 62.3%, 2.9, 95.0ms, 0 | 20260506 19:33:17.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:17.372 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 19:33:17.372 | INFO | PRE_HQ : 60.0%, 1.8, 57.7ms, 0 | 20260506 19:33:17.372 | INFO | BAZWRITER : 64.4%, 4.8, 159.2ms, 0 | 20260506 19:33:17.372 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 19:33:17.372 | INFO | Uploaded 36GiB to GPU in 2.03804s (17.6641GiB/s) | 20260506 19:33:17.372 | INFO | Downloaded 38GiB to GPU in 2.19307s (17.3273GiB/s) | 20260506 19:33:22.506 | INFO | Analyzing chunk frames = [3895808,3896320) | 20260506 19:33:22.506 | INFO | MemInfo: { | 20260506 19:33:22.506 | INFO | "gpu" : " 17 GB", | 20260506 19:33:22.506 | INFO | "huge" : "", | 20260506 19:33:22.506 | INFO | "huge_shared" : " 27 GB", | 20260506 19:33:22.506 | INFO | "malloc" : " 14 GB", | 20260506 19:33:22.506 | INFO | "pinned" : " 29 GB" | 20260506 19:33:22.506 | INFO | } | 20260506 19:33:22.506 | INFO | Perf report for the last 5.13472 seconds: | 20260506 19:33:22.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:33:22.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:22.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:22.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:22.507 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 19:33:22.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:22.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:22.507 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 19:33:22.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:22.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:22.507 | INFO | Uploaded 36GiB to GPU in 2.01716s (17.8469GiB/s) | 20260506 19:33:22.507 | INFO | Downloaded 34GiB to GPU in 1.92775s (17.6372GiB/s) | 20260506 19:33:27.647 | INFO | Analyzing chunk frames = [3896320,3896832) | 20260506 19:33:27.647 | INFO | MemInfo: { | 20260506 19:33:27.647 | INFO | "gpu" : " 17 GB", | 20260506 19:33:27.647 | INFO | "huge" : "", | 20260506 19:33:27.647 | INFO | "huge_shared" : " 27 GB", | 20260506 19:33:27.647 | INFO | "malloc" : " 14 GB", | 20260506 19:33:27.647 | INFO | "pinned" : " 29 GB" | 20260506 19:33:27.647 | INFO | } | 20260506 19:33:27.647 | INFO | Perf report for the last 5.14061 seconds: | 20260506 19:33:27.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:33:27.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:27.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:27.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:27.647 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:33:27.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:27.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:27.647 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:33:27.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:27.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:27.647 | INFO | Uploaded 36GiB to GPU in 1.97235s (18.2523GiB/s) | 20260506 19:33:27.647 | INFO | Downloaded 34GiB to GPU in 1.91425s (17.7615GiB/s) | 20260506 19:33:32.789 | INFO | Analyzing chunk frames = [3896832,3897344) | 20260506 19:33:32.789 | INFO | MemInfo: { | 20260506 19:33:32.789 | INFO | "gpu" : " 17 GB", | 20260506 19:33:32.789 | INFO | "huge" : "", | 20260506 19:33:32.789 | INFO | "huge_shared" : " 27 GB", | 20260506 19:33:32.789 | INFO | "malloc" : " 14 GB", | 20260506 19:33:32.789 | INFO | "pinned" : " 29 GB" | 20260506 19:33:32.789 | INFO | } | 20260506 19:33:32.789 | INFO | Perf report for the last 5.14185 seconds: | 20260506 19:33:32.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:33:32.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:32.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:32.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:32.789 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 19:33:32.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:32.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:32.789 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:33:32.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:32.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:32.789 | INFO | Uploaded 36GiB to GPU in 1.94621s (18.4975GiB/s) | 20260506 19:33:32.789 | INFO | Downloaded 34GiB to GPU in 1.87258s (18.1568GiB/s) | 20260506 19:33:37.924 | INFO | Analyzing chunk frames = [3897344,3897856) | 20260506 19:33:37.924 | INFO | MemInfo: { | 20260506 19:33:37.924 | INFO | "gpu" : " 17 GB", | 20260506 19:33:37.924 | INFO | "huge" : "", | 20260506 19:33:37.924 | INFO | "huge_shared" : " 27 GB", | 20260506 19:33:37.924 | INFO | "malloc" : " 14 GB", | 20260506 19:33:37.924 | INFO | "pinned" : " 29 GB" | 20260506 19:33:37.924 | INFO | } | 20260506 19:33:37.924 | INFO | Perf report for the last 5.13499 seconds: | 20260506 19:33:37.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:33:37.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:37.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:37.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:37.924 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 19:33:37.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:37.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:37.924 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:33:37.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:37.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:37.924 | INFO | Uploaded 36GiB to GPU in 2.0034s (17.9694GiB/s) | 20260506 19:33:37.924 | INFO | Downloaded 34GiB to GPU in 1.91092s (17.7925GiB/s) | 20260506 19:33:43.061 | INFO | Analyzing chunk frames = [3897856,3898368) | 20260506 19:33:43.061 | INFO | MemInfo: { | 20260506 19:33:43.061 | INFO | "gpu" : " 17 GB", | 20260506 19:33:43.061 | INFO | "huge" : "", | 20260506 19:33:43.061 | INFO | "huge_shared" : " 27 GB", | 20260506 19:33:43.061 | INFO | "malloc" : " 14 GB", | 20260506 19:33:43.061 | INFO | "pinned" : " 29 GB" | 20260506 19:33:43.061 | INFO | } | 20260506 19:33:43.061 | INFO | Perf report for the last 5.13704 seconds: | 20260506 19:33:43.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:33:43.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:43.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:43.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:43.061 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 19:33:43.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:43.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:43.061 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 19:33:43.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:43.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:43.061 | INFO | Uploaded 36GiB to GPU in 1.99699s (18.0271GiB/s) | 20260506 19:33:43.061 | INFO | Downloaded 34GiB to GPU in 1.9441s (17.4888GiB/s) | 20260506 19:33:48.198 | INFO | Analyzing chunk frames = [3898368,3898880) | 20260506 19:33:48.198 | INFO | MemInfo: { | 20260506 19:33:48.198 | INFO | "gpu" : " 17 GB", | 20260506 19:33:48.198 | INFO | "huge" : "", | 20260506 19:33:48.198 | INFO | "huge_shared" : " 27 GB", | 20260506 19:33:48.198 | INFO | "malloc" : " 14 GB", | 20260506 19:33:48.198 | INFO | "pinned" : " 29 GB" | 20260506 19:33:48.198 | INFO | } | 20260506 19:33:48.198 | INFO | Perf report for the last 5.13685 seconds: | 20260506 19:33:48.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:33:48.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:48.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:48.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:48.198 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 19:33:48.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:48.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:48.198 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 19:33:48.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:48.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:48.198 | INFO | Uploaded 36GiB to GPU in 1.97103s (18.2646GiB/s) | 20260506 19:33:48.198 | INFO | Downloaded 34GiB to GPU in 2.00494s (16.9581GiB/s) | 20260506 19:33:53.343 | INFO | Analyzing chunk frames = [3898880,3899392) | 20260506 19:33:53.343 | INFO | MemInfo: { | 20260506 19:33:53.343 | INFO | "gpu" : " 17 GB", | 20260506 19:33:53.343 | INFO | "huge" : "", | 20260506 19:33:53.343 | INFO | "huge_shared" : " 27 GB", | 20260506 19:33:53.343 | INFO | "malloc" : " 14 GB", | 20260506 19:33:53.343 | INFO | "pinned" : " 29 GB" | 20260506 19:33:53.343 | INFO | } | 20260506 19:33:53.343 | INFO | Perf report for the last 5.14491 seconds: | 20260506 19:33:53.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:33:53.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:53.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:53.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:53.343 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 19:33:53.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:53.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:53.343 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 19:33:53.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:53.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:53.343 | INFO | Uploaded 36GiB to GPU in 2.05423s (17.5248GiB/s) | 20260506 19:33:53.343 | INFO | Downloaded 34GiB to GPU in 1.97909s (17.1796GiB/s) | 20260506 19:33:53.343 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:33:53.343 | INFO | Total mem usage: 18 GiB | 20260506 19:33:53.343 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:33:53.343 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.6%) | 20260506 19:33:53.343 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:33:53.343 | INFO | High Water Breakdown: | 20260506 19:33:53.343 | INFO | 61.4% : 9.56 GiB ( 83.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:33:53.343 | INFO | 38.5% : 5.99 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:33:53.343 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:33:53.343 | INFO | ------------------------------------------ | 20260506 19:33:53.343 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:33:53.343 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:33:53.343 | INFO | Total mem usage: 26.5 GiB | 20260506 19:33:53.343 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:33:53.343 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:33:53.343 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:33:53.343 | INFO | High Water Breakdown: | 20260506 19:33:53.343 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:33:53.344 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:33:53.344 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:33:53.344 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:33:53.344 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:33:53.344 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:33:53.344 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:33:53.344 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:33:53.344 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:33:53.344 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:33:53.344 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:33:53.344 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:33:53.344 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:33:53.344 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:33:53.344 | INFO | ------------------------------------------ | 20260506 19:33:53.344 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:33:53.344 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:33:53.344 | INFO | Total mem usage: 25.6 GiB | 20260506 19:33:53.344 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:33:53.344 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:33:53.344 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:33:53.344 | INFO | High Water Breakdown: | 20260506 19:33:53.344 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:33:53.344 | INFO | ------------------------------------------ | 20260506 19:33:53.344 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:33:53.344 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:33:53.344 | INFO | Total mem usage: 15.7 GiB | 20260506 19:33:53.344 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:33:53.344 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:33:53.344 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 19:33:53.344 | INFO | High Water Breakdown: | 20260506 19:33:53.344 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:33:53.344 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:33:53.344 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:33:53.344 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:33:53.344 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:33:53.344 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:33:53.344 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:33:53.344 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:33:53.344 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:33:53.344 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:33:53.344 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:33:53.344 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:33:53.344 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:33:53.344 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:33:53.344 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:33:53.344 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:33:53.344 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:33:53.344 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:33:53.344 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:33:53.344 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:33:53.344 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:33:53.344 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:33:53.344 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:33:53.344 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:33:53.344 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:33:53.344 | INFO | ------------------------------------------ | 20260506 19:33:53.344 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:33:53.344 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:33:53.344 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:33:53.344 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:33:53.344 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.63e+01, 1.34e+01 / 2.52e+00 | 20260506 19:33:53.344 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.02e-01, 9.67e-03 / 7.00e-03 | 20260506 19:33:53.344 | INFO | PerfReport: 11.61%, Baseline -- 3.62e+00 / 4.44e+00, 5.54e-01 / 3.14e-01 | 20260506 19:33:53.344 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.46e+00, 1.82e-01 / 1.33e-01 | 20260506 19:33:53.344 | INFO | PerfReport: 16.07%, DME -- 4.01e+01 / 4.53e+01, 4.49e+00 / 3.25e+00 | 20260506 19:33:53.344 | INFO | PerfReport: 57.07%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.37e+00 / 1.88e+00 | 20260506 19:33:53.344 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.24e+00 / 2.81e+00, 3.83e-01 / 2.23e-01 | 20260506 19:33:53.344 | INFO | PerfReport: 3.35%, Metrics -- 1.05e+00 / 1.37e+00, 2.74e-01 / 1.40e-01 | 20260506 19:33:53.344 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.60e-01 / 4.10e-01, 5.68e-02 / 2.77e-02 | 20260506 19:33:53.344 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:33:53.344 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:33:53.344 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:33:53.344 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 19:33:53.344 | INFO | PerfReport: 21.44%, Upload -- 2.08e+01 / 0.00e+00, 6.93e+00 / 0.00e+00 | 20260506 19:33:53.344 | INFO | PerfReport: 21.22%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 19:33:53.344 | INFO | PerfReport: 32.21%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 19:33:53.344 | INFO | PerfReport: 1.33%, LockUpload -- 1.29e+00 / 0.00e+00, 5.07e+00 / 0.00e+00 | 20260506 19:33:53.344 | INFO | PerfReport: 1.20%, LockDownload -- 1.16e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260506 19:33:53.344 | INFO | PerfReport: 22.58%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 19:33:53.345 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260506 19:33:53.458 | INFO | Starting a round of Baz IO | 20260506 19:33:56.590 | INFO | Populating RTMetrics report | 20260506 19:33:56.590 | INFO | RTMetrics report populated | 20260506 19:33:56.590 | INFO | Opening tmp RTMetrics json file | 20260506 19:33:56.591 | INFO | Serializing RTMetrics to json | 20260506 19:33:56.594 | INFO | Serialization done, swapping tmp file now... | 20260506 19:33:56.594 | INFO | RTMetrics done | 20260506 19:33:57.093 | INFO | Finished a round of Baz IO: Wrote 2.006 GiB in 3.635 seconds | 20260506 19:33:57.093 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.57% / 0.00% / 65.42% | 20260506 19:33:57.093 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.897038% / 3.97364e-06% / 8.74201e-05% | 20260506 19:33:57.093 | INFO | Baz IO overall: | 20260506 19:33:57.094 | INFO | BytesWritten : 2060553653240 | 20260506 19:33:57.094 | INFO | NumEvents : 995996400561 | 20260506 19:33:58.484 | INFO | Analyzing chunk frames = [3899392,3899904) | 20260506 19:33:58.484 | INFO | MemInfo: { | 20260506 19:33:58.484 | INFO | "gpu" : " 17 GB", | 20260506 19:33:58.484 | INFO | "huge" : "", | 20260506 19:33:58.484 | INFO | "huge_shared" : " 27 GB", | 20260506 19:33:58.484 | INFO | "malloc" : " 14 GB", | 20260506 19:33:58.484 | INFO | "pinned" : " 29 GB" | 20260506 19:33:58.484 | INFO | } | 20260506 19:33:58.484 | INFO | Perf report for the last 5.14181 seconds: | 20260506 19:33:58.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:33:58.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:58.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:58.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:58.485 | INFO | ANALYSIS : 62.6%, 2.9, 95.4ms, 0 | 20260506 19:33:58.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:33:58.485 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 19:33:58.485 | INFO | PRE_HQ : 60.1%, 1.8, 57.6ms, 0 | 20260506 19:33:58.485 | INFO | BAZWRITER : 70.3%, 4.8, 173.4ms, 0 | 20260506 19:33:58.485 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 19:33:58.485 | INFO | Uploaded 36GiB to GPU in 2.05744s (17.4975GiB/s) | 20260506 19:33:58.485 | INFO | Downloaded 38GiB to GPU in 2.20383s (17.2427GiB/s) | 20260506 19:34:03.631 | INFO | Analyzing chunk frames = [3899904,3900416) | 20260506 19:34:03.631 | INFO | MemInfo: { | 20260506 19:34:03.631 | INFO | "gpu" : " 17 GB", | 20260506 19:34:03.631 | INFO | "huge" : "", | 20260506 19:34:03.631 | INFO | "huge_shared" : " 27 GB", | 20260506 19:34:03.631 | INFO | "malloc" : " 14 GB", | 20260506 19:34:03.631 | INFO | "pinned" : " 29 GB" | 20260506 19:34:03.631 | INFO | } | 20260506 19:34:03.632 | INFO | Perf report for the last 5.14702 seconds: | 20260506 19:34:03.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:34:03.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:03.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:03.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:03.632 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 19:34:03.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:03.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:03.632 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:34:03.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:03.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:03.632 | INFO | Uploaded 36GiB to GPU in 2.00059s (17.9947GiB/s) | 20260506 19:34:03.632 | INFO | Downloaded 34GiB to GPU in 1.91863s (17.721GiB/s) | 20260506 19:34:08.762 | INFO | Analyzing chunk frames = [3900416,3900928) | 20260506 19:34:08.762 | INFO | MemInfo: { | 20260506 19:34:08.762 | INFO | "gpu" : " 17 GB", | 20260506 19:34:08.762 | INFO | "huge" : "", | 20260506 19:34:08.762 | INFO | "huge_shared" : " 27 GB", | 20260506 19:34:08.762 | INFO | "malloc" : " 14 GB", | 20260506 19:34:08.762 | INFO | "pinned" : " 29 GB" | 20260506 19:34:08.762 | INFO | } | 20260506 19:34:08.762 | INFO | Perf report for the last 5.13091 seconds: | 20260506 19:34:08.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:34:08.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:08.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:08.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:08.763 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260506 19:34:08.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:08.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:08.763 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:34:08.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:08.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:08.763 | INFO | Uploaded 36GiB to GPU in 1.99845s (18.0139GiB/s) | 20260506 19:34:08.763 | INFO | Downloaded 34GiB to GPU in 1.92917s (17.6241GiB/s) | 20260506 19:34:13.898 | INFO | Analyzing chunk frames = [3900928,3901440) | 20260506 19:34:13.898 | INFO | MemInfo: { | 20260506 19:34:13.898 | INFO | "gpu" : " 17 GB", | 20260506 19:34:13.898 | INFO | "huge" : "", | 20260506 19:34:13.898 | INFO | "huge_shared" : " 27 GB", | 20260506 19:34:13.898 | INFO | "malloc" : " 14 GB", | 20260506 19:34:13.898 | INFO | "pinned" : " 29 GB" | 20260506 19:34:13.898 | INFO | } | 20260506 19:34:13.898 | INFO | Perf report for the last 5.13598 seconds: | 20260506 19:34:13.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:34:13.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:13.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:13.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:13.898 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:34:13.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:13.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:13.898 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:34:13.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:13.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:13.899 | INFO | Uploaded 36GiB to GPU in 1.96415s (18.3286GiB/s) | 20260506 19:34:13.899 | INFO | Downloaded 34GiB to GPU in 1.87183s (18.1641GiB/s) | 20260506 19:34:19.037 | INFO | Analyzing chunk frames = [3901440,3901952) | 20260506 19:34:19.037 | INFO | MemInfo: { | 20260506 19:34:19.037 | INFO | "gpu" : " 17 GB", | 20260506 19:34:19.037 | INFO | "huge" : "", | 20260506 19:34:19.037 | INFO | "huge_shared" : " 27 GB", | 20260506 19:34:19.037 | INFO | "malloc" : " 14 GB", | 20260506 19:34:19.037 | INFO | "pinned" : " 29 GB" | 20260506 19:34:19.037 | INFO | } | 20260506 19:34:19.037 | INFO | Perf report for the last 5.13892 seconds: | 20260506 19:34:19.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:34:19.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:19.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:19.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:19.038 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 19:34:19.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:19.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:19.038 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:34:19.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:19.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:19.038 | INFO | Uploaded 36GiB to GPU in 2.02131s (17.8102GiB/s) | 20260506 19:34:19.038 | INFO | Downloaded 34GiB to GPU in 1.9117s (17.7852GiB/s) | 20260506 19:34:24.174 | INFO | Analyzing chunk frames = [3901952,3902464) | 20260506 19:34:24.174 | INFO | MemInfo: { | 20260506 19:34:24.174 | INFO | "gpu" : " 17 GB", | 20260506 19:34:24.174 | INFO | "huge" : "", | 20260506 19:34:24.174 | INFO | "huge_shared" : " 27 GB", | 20260506 19:34:24.174 | INFO | "malloc" : " 14 GB", | 20260506 19:34:24.174 | INFO | "pinned" : " 29 GB" | 20260506 19:34:24.174 | INFO | } | 20260506 19:34:24.174 | INFO | Perf report for the last 5.13638 seconds: | 20260506 19:34:24.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:34:24.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:24.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:24.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:24.174 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:34:24.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:24.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:24.174 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:34:24.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:24.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:24.174 | INFO | Uploaded 36GiB to GPU in 2.04984s (17.5623GiB/s) | 20260506 19:34:24.174 | INFO | Downloaded 34GiB to GPU in 1.94459s (17.4844GiB/s) | 20260506 19:34:29.316 | INFO | Analyzing chunk frames = [3902464,3902976) | 20260506 19:34:29.316 | INFO | MemInfo: { | 20260506 19:34:29.316 | INFO | "gpu" : " 17 GB", | 20260506 19:34:29.316 | INFO | "huge" : "", | 20260506 19:34:29.316 | INFO | "huge_shared" : " 27 GB", | 20260506 19:34:29.316 | INFO | "malloc" : " 14 GB", | 20260506 19:34:29.316 | INFO | "pinned" : " 29 GB" | 20260506 19:34:29.316 | INFO | } | 20260506 19:34:29.316 | INFO | Perf report for the last 5.14239 seconds: | 20260506 19:34:29.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:34:29.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:29.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:29.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:29.316 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 19:34:29.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:29.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:29.316 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 19:34:29.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:29.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:29.317 | INFO | Uploaded 36GiB to GPU in 1.99815s (18.0167GiB/s) | 20260506 19:34:29.317 | INFO | Downloaded 34GiB to GPU in 2.00897s (16.9241GiB/s) | 20260506 19:34:34.459 | INFO | Analyzing chunk frames = [3902976,3903488) | 20260506 19:34:34.459 | INFO | MemInfo: { | 20260506 19:34:34.459 | INFO | "gpu" : " 17 GB", | 20260506 19:34:34.459 | INFO | "huge" : "", | 20260506 19:34:34.459 | INFO | "huge_shared" : " 27 GB", | 20260506 19:34:34.459 | INFO | "malloc" : " 14 GB", | 20260506 19:34:34.459 | INFO | "pinned" : " 29 GB" | 20260506 19:34:34.459 | INFO | } | 20260506 19:34:34.459 | INFO | Perf report for the last 5.143 seconds: | 20260506 19:34:34.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:34:34.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:34.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:34.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:34.459 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 19:34:34.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:34.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:34.459 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 19:34:34.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:34.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:34.459 | INFO | Uploaded 36GiB to GPU in 1.97061s (18.2684GiB/s) | 20260506 19:34:34.460 | INFO | Downloaded 34GiB to GPU in 1.99144s (17.073GiB/s) | 20260506 19:34:34.564 | INFO | Starting a round of Baz IO | 20260506 19:34:37.706 | INFO | Populating RTMetrics report | 20260506 19:34:37.707 | INFO | RTMetrics report populated | 20260506 19:34:37.707 | INFO | Opening tmp RTMetrics json file | 20260506 19:34:37.748 | INFO | Serializing RTMetrics to json | 20260506 19:34:37.829 | INFO | Serialization done, swapping tmp file now... | 20260506 19:34:37.876 | INFO | RTMetrics done | 20260506 19:34:37.967 | INFO | Finished a round of Baz IO: Wrote 2.007 GiB in 3.402 seconds | 20260506 19:34:37.967 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.57% / 0.00% / 65.43% | 20260506 19:34:37.967 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.895325% / 3.97364e-06% / 7.94729e-05% | 20260506 19:34:37.967 | INFO | Baz IO overall: | 20260506 19:34:37.967 | INFO | BytesWritten : 2062708832990 | 20260506 19:34:37.967 | INFO | NumEvents : 997032827253 | 20260506 19:34:39.598 | INFO | Analyzing chunk frames = [3903488,3904000) | 20260506 19:34:39.598 | INFO | MemInfo: { | 20260506 19:34:39.598 | INFO | "gpu" : " 17 GB", | 20260506 19:34:39.598 | INFO | "huge" : "", | 20260506 19:34:39.598 | INFO | "huge_shared" : " 27 GB", | 20260506 19:34:39.598 | INFO | "malloc" : " 14 GB", | 20260506 19:34:39.598 | INFO | "pinned" : " 29 GB" | 20260506 19:34:39.598 | INFO | } | 20260506 19:34:39.598 | INFO | Perf report for the last 5.13864 seconds: | 20260506 19:34:39.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:34:39.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:39.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:39.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:39.598 | INFO | ANALYSIS : 62.6%, 2.9, 95.4ms, 0 | 20260506 19:34:39.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:39.598 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260506 19:34:39.598 | INFO | PRE_HQ : 60.1%, 1.9, 57.8ms, 0 | 20260506 19:34:39.598 | INFO | BAZWRITER : 65.9%, 4.8, 161.4ms, 0 | 20260506 19:34:39.598 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 19:34:39.598 | INFO | Uploaded 36GiB to GPU in 2.02707s (17.7596GiB/s) | 20260506 19:34:39.598 | INFO | Downloaded 38GiB to GPU in 2.20831s (17.2078GiB/s) | 20260506 19:34:44.738 | INFO | Analyzing chunk frames = [3904000,3904512) | 20260506 19:34:44.738 | INFO | MemInfo: { | 20260506 19:34:44.738 | INFO | "gpu" : " 17 GB", | 20260506 19:34:44.738 | INFO | "huge" : "", | 20260506 19:34:44.738 | INFO | "huge_shared" : " 27 GB", | 20260506 19:34:44.738 | INFO | "malloc" : " 14 GB", | 20260506 19:34:44.738 | INFO | "pinned" : " 29 GB" | 20260506 19:34:44.738 | INFO | } | 20260506 19:34:44.738 | INFO | Perf report for the last 5.14007 seconds: | 20260506 19:34:44.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:34:44.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:44.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:44.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:44.738 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 19:34:44.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:44.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:44.738 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:34:44.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:44.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:44.738 | INFO | Uploaded 36GiB to GPU in 1.99409s (18.0533GiB/s) | 20260506 19:34:44.738 | INFO | Downloaded 34GiB to GPU in 1.92312s (17.6796GiB/s) | 20260506 19:34:49.877 | INFO | Analyzing chunk frames = [3904512,3905024) | 20260506 19:34:49.877 | INFO | MemInfo: { | 20260506 19:34:49.877 | INFO | "gpu" : " 17 GB", | 20260506 19:34:49.877 | INFO | "huge" : "", | 20260506 19:34:49.877 | INFO | "huge_shared" : " 27 GB", | 20260506 19:34:49.877 | INFO | "malloc" : " 14 GB", | 20260506 19:34:49.877 | INFO | "pinned" : " 29 GB" | 20260506 19:34:49.877 | INFO | } | 20260506 19:34:49.877 | INFO | Perf report for the last 5.13949 seconds: | 20260506 19:34:49.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:34:49.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:49.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:49.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:49.877 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:34:49.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:49.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:49.877 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:34:49.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:49.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:49.878 | INFO | Uploaded 36GiB to GPU in 1.965s (18.3206GiB/s) | 20260506 19:34:49.878 | INFO | Downloaded 34GiB to GPU in 1.9201s (17.7074GiB/s) | 20260506 19:34:55.012 | INFO | Analyzing chunk frames = [3905024,3905536) | 20260506 19:34:55.012 | INFO | MemInfo: { | 20260506 19:34:55.012 | INFO | "gpu" : " 17 GB", | 20260506 19:34:55.012 | INFO | "huge" : "", | 20260506 19:34:55.012 | INFO | "huge_shared" : " 27 GB", | 20260506 19:34:55.012 | INFO | "malloc" : " 14 GB", | 20260506 19:34:55.012 | INFO | "pinned" : " 29 GB" | 20260506 19:34:55.012 | INFO | } | 20260506 19:34:55.012 | INFO | Perf report for the last 5.13484 seconds: | 20260506 19:34:55.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:34:55.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:55.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:55.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:55.012 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 19:34:55.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:55.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:55.012 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:34:55.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:55.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:34:55.013 | INFO | Uploaded 36GiB to GPU in 1.94467s (18.5121GiB/s) | 20260506 19:34:55.013 | INFO | Downloaded 34GiB to GPU in 1.8754s (18.1294GiB/s) | 20260506 19:35:00.147 | INFO | Analyzing chunk frames = [3905536,3906048) | 20260506 19:35:00.147 | INFO | MemInfo: { | 20260506 19:35:00.147 | INFO | "gpu" : " 17 GB", | 20260506 19:35:00.147 | INFO | "huge" : "", | 20260506 19:35:00.147 | INFO | "huge_shared" : " 27 GB", | 20260506 19:35:00.147 | INFO | "malloc" : " 14 GB", | 20260506 19:35:00.147 | INFO | "pinned" : " 29 GB" | 20260506 19:35:00.147 | INFO | } | 20260506 19:35:00.147 | INFO | Perf report for the last 5.13517 seconds: | 20260506 19:35:00.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:35:00.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:00.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:00.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:00.148 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 19:35:00.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:00.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:00.148 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:35:00.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:00.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:00.148 | INFO | Uploaded 36GiB to GPU in 2.01411s (17.8739GiB/s) | 20260506 19:35:00.148 | INFO | Downloaded 34GiB to GPU in 1.91812s (17.7257GiB/s) | 20260506 19:35:05.292 | INFO | Analyzing chunk frames = [3906048,3906560) | 20260506 19:35:05.292 | INFO | MemInfo: { | 20260506 19:35:05.292 | INFO | "gpu" : " 17 GB", | 20260506 19:35:05.292 | INFO | "huge" : "", | 20260506 19:35:05.292 | INFO | "huge_shared" : " 27 GB", | 20260506 19:35:05.292 | INFO | "malloc" : " 14 GB", | 20260506 19:35:05.292 | INFO | "pinned" : " 29 GB" | 20260506 19:35:05.292 | INFO | } | 20260506 19:35:05.292 | INFO | Perf report for the last 5.14429 seconds: | 20260506 19:35:05.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:35:05.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:05.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:05.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:05.292 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 19:35:05.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:05.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:05.292 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 19:35:05.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:05.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:05.292 | INFO | Uploaded 36GiB to GPU in 2.03728s (17.6706GiB/s) | 20260506 19:35:05.292 | INFO | Downloaded 34GiB to GPU in 1.95748s (17.3693GiB/s) | 20260506 19:35:10.435 | INFO | Analyzing chunk frames = [3906560,3907072) | 20260506 19:35:10.435 | INFO | MemInfo: { | 20260506 19:35:10.435 | INFO | "gpu" : " 17 GB", | 20260506 19:35:10.435 | INFO | "huge" : "", | 20260506 19:35:10.435 | INFO | "huge_shared" : " 27 GB", | 20260506 19:35:10.435 | INFO | "malloc" : " 14 GB", | 20260506 19:35:10.435 | INFO | "pinned" : " 29 GB" | 20260506 19:35:10.435 | INFO | } | 20260506 19:35:10.435 | INFO | Perf report for the last 5.14326 seconds: | 20260506 19:35:10.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:35:10.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:10.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:10.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:10.435 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 19:35:10.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:10.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:10.435 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:35:10.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:10.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:10.435 | INFO | Uploaded 36GiB to GPU in 1.97506s (18.2273GiB/s) | 20260506 19:35:10.435 | INFO | Downloaded 34GiB to GPU in 2.00202s (16.9828GiB/s) | 20260506 19:35:15.575 | INFO | Analyzing chunk frames = [3907072,3907584) | 20260506 19:35:15.575 | INFO | MemInfo: { | 20260506 19:35:15.575 | INFO | "gpu" : " 17 GB", | 20260506 19:35:15.575 | INFO | "huge" : "", | 20260506 19:35:15.575 | INFO | "huge_shared" : " 27 GB", | 20260506 19:35:15.575 | INFO | "malloc" : " 14 GB", | 20260506 19:35:15.575 | INFO | "pinned" : " 29 GB" | 20260506 19:35:15.575 | INFO | } | 20260506 19:35:15.575 | INFO | Perf report for the last 5.1403 seconds: | 20260506 19:35:15.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:35:15.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:15.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:15.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:15.575 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 19:35:15.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:15.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:15.575 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 19:35:15.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:15.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:15.576 | INFO | Uploaded 36GiB to GPU in 2.05702s (17.501GiB/s) | 20260506 19:35:15.576 | INFO | Downloaded 34GiB to GPU in 1.99481s (17.0443GiB/s) | 20260506 19:35:15.576 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:35:15.576 | INFO | Total mem usage: 18 GiB | 20260506 19:35:15.576 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:35:15.576 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.4%) | 20260506 19:35:15.576 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:35:15.576 | INFO | High Water Breakdown: | 20260506 19:35:15.576 | INFO | 61.3% : 9.48 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:35:15.576 | INFO | 38.6% : 5.96 GiB ( 89.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:35:15.576 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:35:15.576 | INFO | ------------------------------------------ | 20260506 19:35:15.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:35:15.576 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:35:15.576 | INFO | Total mem usage: 26.5 GiB | 20260506 19:35:15.576 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:35:15.576 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:35:15.576 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:35:15.576 | INFO | High Water Breakdown: | 20260506 19:35:15.576 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:35:15.576 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:35:15.576 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:35:15.576 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:35:15.576 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:35:15.576 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:35:15.576 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:35:15.576 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:35:15.576 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:35:15.576 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:35:15.576 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:35:15.576 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:35:15.576 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:35:15.576 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:35:15.576 | INFO | ------------------------------------------ | 20260506 19:35:15.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:35:15.576 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:35:15.576 | INFO | Total mem usage: 25.6 GiB | 20260506 19:35:15.577 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:35:15.577 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:35:15.577 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:35:15.577 | INFO | High Water Breakdown: | 20260506 19:35:15.577 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:35:15.577 | INFO | ------------------------------------------ | 20260506 19:35:15.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:35:15.577 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:35:15.577 | INFO | Total mem usage: 15.7 GiB | 20260506 19:35:15.577 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:35:15.577 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:35:15.577 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 19:35:15.577 | INFO | High Water Breakdown: | 20260506 19:35:15.577 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:35:15.577 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:35:15.577 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:35:15.577 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:35:15.577 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:35:15.577 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:35:15.577 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:35:15.577 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:35:15.577 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:35:15.577 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:35:15.577 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:35:15.577 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:35:15.577 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:35:15.577 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:35:15.577 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:35:15.577 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:35:15.577 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:35:15.577 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:35:15.577 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:35:15.577 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:35:15.577 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:35:15.577 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:35:15.577 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:35:15.577 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:35:15.577 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:35:15.577 | INFO | ------------------------------------------ | 20260506 19:35:15.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:35:15.577 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:35:15.577 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:35:15.577 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:35:15.577 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.62e+01, 1.32e+01 / 2.41e+00 | 20260506 19:35:15.577 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.03e-01, 1.01e-02 / 1.12e-02 | 20260506 19:35:15.577 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.40e+00, 5.52e-01 / 3.25e-01 | 20260506 19:35:15.577 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.49e+00, 1.83e-01 / 1.31e-01 | 20260506 19:35:15.577 | INFO | PerfReport: 16.02%, DME -- 4.00e+01 / 4.51e+01, 4.23e+00 / 2.51e+00 | 20260506 19:35:15.577 | INFO | PerfReport: 57.12%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.42e+00 / 2.04e+00 | 20260506 19:35:15.577 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.24e+00 / 2.89e+00, 3.81e-01 / 1.83e-01 | 20260506 19:35:15.577 | INFO | PerfReport: 3.36%, Metrics -- 1.05e+00 / 1.40e+00, 2.76e-01 / 1.84e-01 | 20260506 19:35:15.577 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.61e-01 / 4.08e-01, 5.74e-02 / 2.17e-02 | 20260506 19:35:15.577 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:35:15.577 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:35:15.577 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:35:15.577 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 19:35:15.577 | INFO | PerfReport: 21.52%, Upload -- 2.09e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260506 19:35:15.577 | INFO | PerfReport: 21.19%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 19:35:15.577 | INFO | PerfReport: 32.18%, Compute -- 3.12e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 19:35:15.577 | INFO | PerfReport: 1.42%, LockUpload -- 1.38e+00 / 0.00e+00, 5.54e+00 / 0.00e+00 | 20260506 19:35:15.577 | INFO | PerfReport: 1.15%, LockDownload -- 1.11e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 19:35:15.577 | INFO | PerfReport: 22.52%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 19:35:15.577 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.33e-02 / 0.00e+00 | 20260506 19:35:15.688 | INFO | Starting a round of Baz IO | 20260506 19:35:18.826 | INFO | Populating RTMetrics report | 20260506 19:35:18.827 | INFO | RTMetrics report populated | 20260506 19:35:18.827 | INFO | Opening tmp RTMetrics json file | 20260506 19:35:18.829 | INFO | Serializing RTMetrics to json | 20260506 19:35:18.835 | INFO | Serialization done, swapping tmp file now... | 20260506 19:35:18.836 | INFO | RTMetrics done | 20260506 19:35:19.430 | INFO | Finished a round of Baz IO: Wrote 2.007 GiB in 3.741 seconds | 20260506 19:35:19.430 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.56% / 0.00% / 65.43% | 20260506 19:35:19.430 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.892723% / 3.97364e-06% / 6.75519e-05% | 20260506 19:35:19.430 | INFO | Baz IO overall: | 20260506 19:35:19.430 | INFO | BytesWritten : 2064863547981 | 20260506 19:35:19.430 | INFO | NumEvents : 998068880508 | 20260506 19:35:20.718 | INFO | Analyzing chunk frames = [3907584,3908096) | 20260506 19:35:20.718 | INFO | MemInfo: { | 20260506 19:35:20.718 | INFO | "gpu" : " 17 GB", | 20260506 19:35:20.718 | INFO | "huge" : "", | 20260506 19:35:20.718 | INFO | "huge_shared" : " 27 GB", | 20260506 19:35:20.718 | INFO | "malloc" : " 14 GB", | 20260506 19:35:20.718 | INFO | "pinned" : " 29 GB" | 20260506 19:35:20.718 | INFO | } | 20260506 19:35:20.718 | INFO | Perf report for the last 5.14283 seconds: | 20260506 19:35:20.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:35:20.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:20.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:20.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:20.718 | INFO | ANALYSIS : 62.6%, 2.9, 95.5ms, 0 | 20260506 19:35:20.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:20.718 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 19:35:20.718 | INFO | PRE_HQ : 60.2%, 1.8, 57.3ms, 0 | 20260506 19:35:20.718 | INFO | BAZWRITER : 72.2%, 4.8, 178.7ms, 0 | 20260506 19:35:20.719 | INFO | BAZ_SAMPLING : 5.6%, 1.0, 2.9ms, 0 | 20260506 19:35:20.719 | INFO | Uploaded 36GiB to GPU in 2.05786s (17.4939GiB/s) | 20260506 19:35:20.719 | INFO | Downloaded 38GiB to GPU in 2.22513s (17.0776GiB/s) | 20260506 19:35:25.852 | INFO | Analyzing chunk frames = [3908096,3908608) | 20260506 19:35:25.852 | INFO | MemInfo: { | 20260506 19:35:25.852 | INFO | "gpu" : " 17 GB", | 20260506 19:35:25.852 | INFO | "huge" : "", | 20260506 19:35:25.852 | INFO | "huge_shared" : " 27 GB", | 20260506 19:35:25.852 | INFO | "malloc" : " 14 GB", | 20260506 19:35:25.852 | INFO | "pinned" : " 29 GB" | 20260506 19:35:25.852 | INFO | } | 20260506 19:35:25.852 | INFO | Perf report for the last 5.13361 seconds: | 20260506 19:35:25.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:35:25.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:25.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:25.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:25.852 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 19:35:25.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:25.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:25.852 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:35:25.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:25.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:25.852 | INFO | Uploaded 36GiB to GPU in 2.0149s (17.8669GiB/s) | 20260506 19:35:25.852 | INFO | Downloaded 34GiB to GPU in 1.92101s (17.699GiB/s) | 20260506 19:35:30.989 | INFO | Analyzing chunk frames = [3908608,3909120) | 20260506 19:35:30.989 | INFO | MemInfo: { | 20260506 19:35:30.989 | INFO | "gpu" : " 17 GB", | 20260506 19:35:30.989 | INFO | "huge" : "", | 20260506 19:35:30.989 | INFO | "huge_shared" : " 27 GB", | 20260506 19:35:30.989 | INFO | "malloc" : " 14 GB", | 20260506 19:35:30.989 | INFO | "pinned" : " 29 GB" | 20260506 19:35:30.989 | INFO | } | 20260506 19:35:30.989 | INFO | Perf report for the last 5.13736 seconds: | 20260506 19:35:30.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:35:30.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:30.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:30.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:30.989 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 19:35:30.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:30.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:30.989 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:35:30.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:30.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:30.990 | INFO | Uploaded 36GiB to GPU in 1.98301s (18.1542GiB/s) | 20260506 19:35:30.990 | INFO | Downloaded 34GiB to GPU in 1.91786s (17.7281GiB/s) | 20260506 19:35:36.128 | INFO | Analyzing chunk frames = [3909120,3909632) | 20260506 19:35:36.128 | INFO | MemInfo: { | 20260506 19:35:36.128 | INFO | "gpu" : " 17 GB", | 20260506 19:35:36.128 | INFO | "huge" : "", | 20260506 19:35:36.128 | INFO | "huge_shared" : " 27 GB", | 20260506 19:35:36.128 | INFO | "malloc" : " 14 GB", | 20260506 19:35:36.128 | INFO | "pinned" : " 29 GB" | 20260506 19:35:36.128 | INFO | } | 20260506 19:35:36.128 | INFO | Perf report for the last 5.1391 seconds: | 20260506 19:35:36.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:35:36.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:36.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:36.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:36.128 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:35:36.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:36.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:36.128 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:35:36.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:36.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:36.129 | INFO | Uploaded 36GiB to GPU in 1.94252s (18.5327GiB/s) | 20260506 19:35:36.129 | INFO | Downloaded 34GiB to GPU in 1.86821s (18.1992GiB/s) | 20260506 19:35:41.265 | INFO | Analyzing chunk frames = [3909632,3910144) | 20260506 19:35:41.265 | INFO | MemInfo: { | 20260506 19:35:41.265 | INFO | "gpu" : " 17 GB", | 20260506 19:35:41.265 | INFO | "huge" : "", | 20260506 19:35:41.265 | INFO | "huge_shared" : " 27 GB", | 20260506 19:35:41.265 | INFO | "malloc" : " 14 GB", | 20260506 19:35:41.265 | INFO | "pinned" : " 29 GB" | 20260506 19:35:41.265 | INFO | } | 20260506 19:35:41.265 | INFO | Perf report for the last 5.13642 seconds: | 20260506 19:35:41.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:35:41.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:41.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:41.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:41.265 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 19:35:41.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:41.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:41.265 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 19:35:41.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:41.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:41.265 | INFO | Uploaded 36GiB to GPU in 1.98159s (18.1672GiB/s) | 20260506 19:35:41.265 | INFO | Downloaded 34GiB to GPU in 1.92568s (17.6561GiB/s) | 20260506 19:35:46.405 | INFO | Analyzing chunk frames = [3910144,3910656) | 20260506 19:35:46.405 | INFO | MemInfo: { | 20260506 19:35:46.405 | INFO | "gpu" : " 17 GB", | 20260506 19:35:46.405 | INFO | "huge" : "", | 20260506 19:35:46.405 | INFO | "huge_shared" : " 27 GB", | 20260506 19:35:46.405 | INFO | "malloc" : " 14 GB", | 20260506 19:35:46.405 | INFO | "pinned" : " 29 GB" | 20260506 19:35:46.405 | INFO | } | 20260506 19:35:46.405 | INFO | Perf report for the last 5.1403 seconds: | 20260506 19:35:46.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:35:46.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:46.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:46.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:46.405 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 19:35:46.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:46.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:46.405 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:35:46.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:46.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:46.405 | INFO | Uploaded 36GiB to GPU in 2.0271s (17.7593GiB/s) | 20260506 19:35:46.405 | INFO | Downloaded 34GiB to GPU in 1.95504s (17.391GiB/s) | 20260506 19:35:51.551 | INFO | Analyzing chunk frames = [3910656,3911168) | 20260506 19:35:51.551 | INFO | MemInfo: { | 20260506 19:35:51.551 | INFO | "gpu" : " 17 GB", | 20260506 19:35:51.551 | INFO | "huge" : "", | 20260506 19:35:51.551 | INFO | "huge_shared" : " 27 GB", | 20260506 19:35:51.551 | INFO | "malloc" : " 14 GB", | 20260506 19:35:51.551 | INFO | "pinned" : " 29 GB" | 20260506 19:35:51.551 | INFO | } | 20260506 19:35:51.551 | INFO | Perf report for the last 5.14624 seconds: | 20260506 19:35:51.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:35:51.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:51.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:51.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:51.552 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 19:35:51.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:51.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:51.552 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260506 19:35:51.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:51.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:51.552 | INFO | Uploaded 36GiB to GPU in 1.99693s (18.0277GiB/s) | 20260506 19:35:51.552 | INFO | Downloaded 34GiB to GPU in 2.00022s (16.9981GiB/s) | 20260506 19:35:56.689 | INFO | Analyzing chunk frames = [3911168,3911680) | 20260506 19:35:56.689 | INFO | MemInfo: { | 20260506 19:35:56.689 | INFO | "gpu" : " 17 GB", | 20260506 19:35:56.689 | INFO | "huge" : "", | 20260506 19:35:56.689 | INFO | "huge_shared" : " 27 GB", | 20260506 19:35:56.690 | INFO | "malloc" : " 14 GB", | 20260506 19:35:56.690 | INFO | "pinned" : " 29 GB" | 20260506 19:35:56.690 | INFO | } | 20260506 19:35:56.690 | INFO | Perf report for the last 5.13835 seconds: | 20260506 19:35:56.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:35:56.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:56.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:56.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:56.690 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 19:35:56.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:56.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:56.690 | INFO | PRE_HQ : 31.9%, 1.0, 16.5ms, 0 | 20260506 19:35:56.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:56.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:35:56.690 | INFO | Uploaded 36GiB to GPU in 2.02845s (17.7476GiB/s) | 20260506 19:35:56.690 | INFO | Downloaded 34GiB to GPU in 1.98752s (17.1068GiB/s) | 20260506 19:35:56.805 | INFO | Starting a round of Baz IO | 20260506 19:35:59.962 | INFO | Populating RTMetrics report | 20260506 19:35:59.962 | INFO | RTMetrics report populated | 20260506 19:35:59.962 | INFO | Opening tmp RTMetrics json file | 20260506 19:35:59.999 | INFO | Serializing RTMetrics to json | 20260506 19:36:00.005 | INFO | Serialization done, swapping tmp file now... | 20260506 19:36:00.009 | INFO | RTMetrics done | 20260506 19:36:00.403 | INFO | Finished a round of Baz IO: Wrote 2.005 GiB in 3.598 seconds | 20260506 19:36:00.403 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.56% / 0.00% / 65.44% | 20260506 19:36:00.403 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.889039% / 7.94729e-06% / 7.15256e-05% | 20260506 19:36:00.403 | INFO | Baz IO overall: | 20260506 19:36:00.403 | INFO | BytesWritten : 2067016244144 | 20260506 19:36:00.403 | INFO | NumEvents : 999103403130 | 20260506 19:36:01.824 | INFO | Analyzing chunk frames = [3911680,3912192) | 20260506 19:36:01.824 | INFO | MemInfo: { | 20260506 19:36:01.824 | INFO | "gpu" : " 17 GB", | 20260506 19:36:01.824 | INFO | "huge" : "", | 20260506 19:36:01.824 | INFO | "huge_shared" : " 27 GB", | 20260506 19:36:01.824 | INFO | "malloc" : " 14 GB", | 20260506 19:36:01.824 | INFO | "pinned" : " 29 GB" | 20260506 19:36:01.824 | INFO | } | 20260506 19:36:01.824 | INFO | Perf report for the last 5.13443 seconds: | 20260506 19:36:01.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:36:01.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:01.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:01.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:01.824 | INFO | ANALYSIS : 63.1%, 2.9, 94.4ms, 0 | 20260506 19:36:01.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:01.824 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260506 19:36:01.824 | INFO | PRE_HQ : 61.4%, 2.0, 62.3ms, 0 | 20260506 19:36:01.824 | INFO | BAZWRITER : 69.8%, 4.8, 171.8ms, 0 | 20260506 19:36:01.825 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 19:36:01.825 | INFO | Uploaded 36GiB to GPU in 2.07434s (17.3549GiB/s) | 20260506 19:36:01.825 | INFO | Downloaded 38GiB to GPU in 2.2059s (17.2265GiB/s) | 20260506 19:36:06.964 | INFO | Analyzing chunk frames = [3912192,3912704) | 20260506 19:36:06.964 | INFO | MemInfo: { | 20260506 19:36:06.964 | INFO | "gpu" : " 17 GB", | 20260506 19:36:06.964 | INFO | "huge" : "", | 20260506 19:36:06.964 | INFO | "huge_shared" : " 27 GB", | 20260506 19:36:06.964 | INFO | "malloc" : " 14 GB", | 20260506 19:36:06.964 | INFO | "pinned" : " 29 GB" | 20260506 19:36:06.964 | INFO | } | 20260506 19:36:06.964 | INFO | Perf report for the last 5.13995 seconds: | 20260506 19:36:06.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:36:06.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:06.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:06.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:06.964 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:36:06.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:06.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:06.964 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:36:06.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:06.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:06.964 | INFO | Uploaded 36GiB to GPU in 1.99593s (18.0367GiB/s) | 20260506 19:36:06.964 | INFO | Downloaded 34GiB to GPU in 1.94608s (17.4711GiB/s) | 20260506 19:36:12.102 | INFO | Analyzing chunk frames = [3912704,3913216) | 20260506 19:36:12.102 | INFO | MemInfo: { | 20260506 19:36:12.102 | INFO | "gpu" : " 17 GB", | 20260506 19:36:12.102 | INFO | "huge" : "", | 20260506 19:36:12.102 | INFO | "huge_shared" : " 27 GB", | 20260506 19:36:12.102 | INFO | "malloc" : " 14 GB", | 20260506 19:36:12.102 | INFO | "pinned" : " 29 GB" | 20260506 19:36:12.102 | INFO | } | 20260506 19:36:12.102 | INFO | Perf report for the last 5.13818 seconds: | 20260506 19:36:12.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:36:12.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:12.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:12.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:12.102 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:36:12.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:12.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:12.102 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:36:12.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:12.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:12.103 | INFO | Uploaded 36GiB to GPU in 1.95481s (18.4161GiB/s) | 20260506 19:36:12.103 | INFO | Downloaded 34GiB to GPU in 1.92377s (17.6737GiB/s) | 20260506 19:36:17.238 | INFO | Analyzing chunk frames = [3913216,3913728) | 20260506 19:36:17.238 | INFO | MemInfo: { | 20260506 19:36:17.238 | INFO | "gpu" : " 17 GB", | 20260506 19:36:17.238 | INFO | "huge" : "", | 20260506 19:36:17.238 | INFO | "huge_shared" : " 27 GB", | 20260506 19:36:17.238 | INFO | "malloc" : " 14 GB", | 20260506 19:36:17.238 | INFO | "pinned" : " 29 GB" | 20260506 19:36:17.238 | INFO | } | 20260506 19:36:17.238 | INFO | Perf report for the last 5.13602 seconds: | 20260506 19:36:17.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:36:17.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:17.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:17.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:17.238 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:36:17.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:17.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:17.239 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:36:17.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:17.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:17.239 | INFO | Uploaded 36GiB to GPU in 1.93392s (18.615GiB/s) | 20260506 19:36:17.239 | INFO | Downloaded 34GiB to GPU in 1.87289s (18.1538GiB/s) | 20260506 19:36:22.385 | INFO | Analyzing chunk frames = [3913728,3914240) | 20260506 19:36:22.385 | INFO | MemInfo: { | 20260506 19:36:22.385 | INFO | "gpu" : " 17 GB", | 20260506 19:36:22.385 | INFO | "huge" : "", | 20260506 19:36:22.385 | INFO | "huge_shared" : " 27 GB", | 20260506 19:36:22.385 | INFO | "malloc" : " 14 GB", | 20260506 19:36:22.385 | INFO | "pinned" : " 29 GB" | 20260506 19:36:22.385 | INFO | } | 20260506 19:36:22.385 | INFO | Perf report for the last 5.14697 seconds: | 20260506 19:36:22.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:36:22.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:22.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:22.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:22.385 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 19:36:22.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:22.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:22.385 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:36:22.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:22.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:22.386 | INFO | Uploaded 36GiB to GPU in 2.01385s (17.8762GiB/s) | 20260506 19:36:22.386 | INFO | Downloaded 34GiB to GPU in 1.9147s (17.7573GiB/s) | 20260506 19:36:27.524 | INFO | Analyzing chunk frames = [3914240,3914752) | 20260506 19:36:27.524 | INFO | MemInfo: { | 20260506 19:36:27.524 | INFO | "gpu" : " 17 GB", | 20260506 19:36:27.524 | INFO | "huge" : "", | 20260506 19:36:27.524 | INFO | "huge_shared" : " 27 GB", | 20260506 19:36:27.525 | INFO | "malloc" : " 14 GB", | 20260506 19:36:27.525 | INFO | "pinned" : " 29 GB" | 20260506 19:36:27.525 | INFO | } | 20260506 19:36:27.525 | INFO | Perf report for the last 5.1393 seconds: | 20260506 19:36:27.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:36:27.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:27.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:27.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:27.525 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 19:36:27.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:27.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:27.525 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:36:27.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:27.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:27.525 | INFO | Uploaded 36GiB to GPU in 2.01615s (17.8558GiB/s) | 20260506 19:36:27.525 | INFO | Downloaded 34GiB to GPU in 1.93814s (17.5426GiB/s) | 20260506 19:36:32.665 | INFO | Analyzing chunk frames = [3914752,3915264) | 20260506 19:36:32.665 | INFO | MemInfo: { | 20260506 19:36:32.665 | INFO | "gpu" : " 17 GB", | 20260506 19:36:32.665 | INFO | "huge" : "", | 20260506 19:36:32.665 | INFO | "huge_shared" : " 27 GB", | 20260506 19:36:32.665 | INFO | "malloc" : " 14 GB", | 20260506 19:36:32.665 | INFO | "pinned" : " 29 GB" | 20260506 19:36:32.665 | INFO | } | 20260506 19:36:32.665 | INFO | Perf report for the last 5.14073 seconds: | 20260506 19:36:32.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:36:32.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:32.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:32.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:32.666 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 19:36:32.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:32.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:32.666 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:36:32.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:32.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:32.666 | INFO | Uploaded 36GiB to GPU in 1.97789s (18.2012GiB/s) | 20260506 19:36:32.666 | INFO | Downloaded 34GiB to GPU in 2.00791s (16.933GiB/s) | 20260506 19:36:37.806 | INFO | Analyzing chunk frames = [3915264,3915776) | 20260506 19:36:37.806 | INFO | MemInfo: { | 20260506 19:36:37.806 | INFO | "gpu" : " 17 GB", | 20260506 19:36:37.806 | INFO | "huge" : "", | 20260506 19:36:37.806 | INFO | "huge_shared" : " 27 GB", | 20260506 19:36:37.806 | INFO | "malloc" : " 14 GB", | 20260506 19:36:37.806 | INFO | "pinned" : " 29 GB" | 20260506 19:36:37.806 | INFO | } | 20260506 19:36:37.806 | INFO | Perf report for the last 5.14112 seconds: | 20260506 19:36:37.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:36:37.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:37.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:37.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:37.806 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 19:36:37.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:37.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:37.807 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:36:37.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:37.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:37.807 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:36:37.807 | INFO | Total mem usage: 18 GiB | 20260506 19:36:37.807 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:36:37.807 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -11.6%) | 20260506 19:36:37.807 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:36:37.807 | INFO | High Water Breakdown: | 20260506 19:36:37.807 | INFO | Uploaded 36GiB to GPU in 1.99441s (18.0505GiB/s) | 20260506 19:36:37.807 | INFO | 62% : 9.99 GiB ( 87.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:36:37.807 | INFO | 37.9% : 6.12 GiB ( 91.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:36:37.807 | INFO | Downloaded 34GiB to GPU in 1.99093s (17.0774GiB/s) | 20260506 19:36:37.807 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:36:37.807 | INFO | ------------------------------------------ | 20260506 19:36:37.807 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:36:37.807 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:36:37.807 | INFO | Total mem usage: 26.5 GiB | 20260506 19:36:37.807 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:36:37.807 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260506 19:36:37.807 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260506 19:36:37.807 | INFO | High Water Breakdown: | 20260506 19:36:37.807 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:36:37.807 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:36:37.807 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:36:37.807 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:36:37.807 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:36:37.807 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:36:37.807 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:36:37.807 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:36:37.807 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:36:37.807 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:36:37.807 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:36:37.807 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:36:37.807 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:36:37.807 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:36:37.807 | INFO | ------------------------------------------ | 20260506 19:36:37.807 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:36:37.807 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:36:37.807 | INFO | Total mem usage: 25.6 GiB | 20260506 19:36:37.807 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:36:37.807 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:36:37.807 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:36:37.807 | INFO | High Water Breakdown: | 20260506 19:36:37.807 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:36:37.807 | INFO | ------------------------------------------ | 20260506 19:36:37.807 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:36:37.807 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:36:37.807 | INFO | Total mem usage: 15.7 GiB | 20260506 19:36:37.807 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:36:37.807 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:36:37.807 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 19:36:37.808 | INFO | High Water Breakdown: | 20260506 19:36:37.808 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:36:37.808 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:36:37.808 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:36:37.808 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:36:37.808 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:36:37.808 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:36:37.808 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:36:37.808 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:36:37.808 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:36:37.808 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:36:37.808 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:36:37.808 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:36:37.808 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:36:37.808 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:36:37.808 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:36:37.808 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:36:37.808 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:36:37.808 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:36:37.808 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:36:37.808 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:36:37.808 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:36:37.808 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:36:37.808 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:36:37.808 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:36:37.808 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:36:37.808 | INFO | ------------------------------------------ | 20260506 19:36:37.808 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:36:37.808 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:36:37.808 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:36:37.808 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:36:37.808 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.57e+01, 1.35e+01 / 2.71e+00 | 20260506 19:36:37.808 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 9.80e-03 / 6.01e-03 | 20260506 19:36:37.808 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.42e+00, 5.57e-01 / 3.36e-01 | 20260506 19:36:37.808 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.85e-01 / 1.37e-01 | 20260506 19:36:37.808 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.44e+01, 4.25e+00 / 2.56e+00 | 20260506 19:36:37.808 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.42e+00 / 1.81e+00 | 20260506 19:36:37.808 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.24e+00 / 2.76e+00, 3.80e-01 / 2.34e-01 | 20260506 19:36:37.808 | INFO | PerfReport: 3.36%, Metrics -- 1.05e+00 / 1.38e+00, 2.78e-01 / 1.42e-01 | 20260506 19:36:37.808 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.61e-01 / 4.06e-01, 5.63e-02 / 2.33e-02 | 20260506 19:36:37.808 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:36:37.808 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:36:37.808 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:36:37.808 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 19:36:37.808 | INFO | PerfReport: 21.51%, Upload -- 2.08e+01 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260506 19:36:37.808 | INFO | PerfReport: 21.25%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 19:36:37.808 | INFO | PerfReport: 32.26%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 19:36:37.808 | INFO | PerfReport: 1.30%, LockUpload -- 1.26e+00 / 0.00e+00, 5.00e+00 / 0.00e+00 | 20260506 19:36:37.808 | INFO | PerfReport: 1.15%, LockDownload -- 1.12e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 19:36:37.808 | INFO | PerfReport: 22.51%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 19:36:37.808 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260506 19:36:37.912 | INFO | Starting a round of Baz IO | 20260506 19:36:41.051 | INFO | Populating RTMetrics report | 20260506 19:36:41.052 | INFO | RTMetrics report populated | 20260506 19:36:41.052 | INFO | Opening tmp RTMetrics json file | 20260506 19:36:41.053 | INFO | Serializing RTMetrics to json | 20260506 19:36:41.054 | INFO | Serialization done, swapping tmp file now... | 20260506 19:36:41.055 | INFO | RTMetrics done | 20260506 19:36:41.167 | INFO | Finished a round of Baz IO: Wrote 2.005 GiB in 3.254 seconds | 20260506 19:36:41.167 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.55% / 0.00% / 65.44% | 20260506 19:36:41.167 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.888602% / 7.94729e-06% / 6.75519e-05% | 20260506 19:36:41.167 | INFO | Baz IO overall: | 20260506 19:36:41.167 | INFO | BytesWritten : 2069169517180 | 20260506 19:36:41.167 | INFO | NumEvents : 1000138289838 | 20260506 19:36:42.941 | INFO | Analyzing chunk frames = [3915776,3916288) | 20260506 19:36:42.941 | INFO | MemInfo: { | 20260506 19:36:42.941 | INFO | "gpu" : " 17 GB", | 20260506 19:36:42.941 | INFO | "huge" : "", | 20260506 19:36:42.941 | INFO | "huge_shared" : " 27 GB", | 20260506 19:36:42.941 | INFO | "malloc" : " 14 GB", | 20260506 19:36:42.941 | INFO | "pinned" : " 29 GB" | 20260506 19:36:42.941 | INFO | } | 20260506 19:36:42.941 | INFO | Perf report for the last 5.13498 seconds: | 20260506 19:36:42.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:36:42.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:42.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:42.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:42.942 | INFO | ANALYSIS : 62.6%, 2.9, 95.6ms, 0 | 20260506 19:36:42.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:42.942 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 19:36:42.942 | INFO | PRE_HQ : 60.1%, 1.8, 57.2ms, 0 | 20260506 19:36:42.942 | INFO | BAZWRITER : 63.4%, 4.5, 147.6ms, 0 | 20260506 19:36:42.942 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 19:36:42.942 | INFO | Uploaded 36GiB to GPU in 2.04012s (17.646GiB/s) | 20260506 19:36:42.942 | INFO | Downloaded 38GiB to GPU in 2.19968s (17.2753GiB/s) | 20260506 19:36:48.080 | INFO | Analyzing chunk frames = [3916288,3916800) | 20260506 19:36:48.080 | INFO | MemInfo: { | 20260506 19:36:48.080 | INFO | "gpu" : " 17 GB", | 20260506 19:36:48.080 | INFO | "huge" : "", | 20260506 19:36:48.080 | INFO | "huge_shared" : " 27 GB", | 20260506 19:36:48.080 | INFO | "malloc" : " 14 GB", | 20260506 19:36:48.080 | INFO | "pinned" : " 29 GB" | 20260506 19:36:48.080 | INFO | } | 20260506 19:36:48.080 | INFO | Perf report for the last 5.13856 seconds: | 20260506 19:36:48.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:36:48.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:48.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:48.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:48.080 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:36:48.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:48.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:48.080 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:36:48.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:48.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:48.080 | INFO | Uploaded 36GiB to GPU in 2.01286s (17.885GiB/s) | 20260506 19:36:48.080 | INFO | Downloaded 34GiB to GPU in 1.92443s (17.6676GiB/s) | 20260506 19:36:53.221 | INFO | Analyzing chunk frames = [3916800,3917312) | 20260506 19:36:53.221 | INFO | MemInfo: { | 20260506 19:36:53.221 | INFO | "gpu" : " 17 GB", | 20260506 19:36:53.221 | INFO | "huge" : "", | 20260506 19:36:53.221 | INFO | "huge_shared" : " 27 GB", | 20260506 19:36:53.221 | INFO | "malloc" : " 14 GB", | 20260506 19:36:53.221 | INFO | "pinned" : " 29 GB" | 20260506 19:36:53.221 | INFO | } | 20260506 19:36:53.222 | INFO | Perf report for the last 5.14145 seconds: | 20260506 19:36:53.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:36:53.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:53.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:53.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:53.222 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 19:36:53.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:53.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:53.222 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:36:53.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:53.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:53.222 | INFO | Uploaded 36GiB to GPU in 1.98327s (18.1519GiB/s) | 20260506 19:36:53.222 | INFO | Downloaded 34GiB to GPU in 1.91714s (17.7347GiB/s) | 20260506 19:36:58.356 | INFO | Analyzing chunk frames = [3917312,3917824) | 20260506 19:36:58.356 | INFO | MemInfo: { | 20260506 19:36:58.356 | INFO | "gpu" : " 17 GB", | 20260506 19:36:58.356 | INFO | "huge" : "", | 20260506 19:36:58.356 | INFO | "huge_shared" : " 27 GB", | 20260506 19:36:58.356 | INFO | "malloc" : " 14 GB", | 20260506 19:36:58.356 | INFO | "pinned" : " 29 GB" | 20260506 19:36:58.356 | INFO | } | 20260506 19:36:58.356 | INFO | Perf report for the last 5.13474 seconds: | 20260506 19:36:58.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:36:58.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:58.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:58.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:58.356 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 19:36:58.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:58.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:58.356 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:36:58.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:58.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:36:58.357 | INFO | Uploaded 36GiB to GPU in 1.96123s (18.3558GiB/s) | 20260506 19:36:58.357 | INFO | Downloaded 34GiB to GPU in 1.87608s (18.1229GiB/s) | 20260506 19:37:03.494 | INFO | Analyzing chunk frames = [3917824,3918336) | 20260506 19:37:03.494 | INFO | MemInfo: { | 20260506 19:37:03.494 | INFO | "gpu" : " 17 GB", | 20260506 19:37:03.494 | INFO | "huge" : "", | 20260506 19:37:03.494 | INFO | "huge_shared" : " 27 GB", | 20260506 19:37:03.494 | INFO | "malloc" : " 14 GB", | 20260506 19:37:03.494 | INFO | "pinned" : " 29 GB" | 20260506 19:37:03.494 | INFO | } | 20260506 19:37:03.494 | INFO | Perf report for the last 5.1382 seconds: | 20260506 19:37:03.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:37:03.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:03.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:03.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:03.495 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 19:37:03.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:03.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:03.495 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:37:03.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:03.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:03.495 | INFO | Uploaded 36GiB to GPU in 2.02154s (17.8082GiB/s) | 20260506 19:37:03.495 | INFO | Downloaded 34GiB to GPU in 1.91604s (17.7449GiB/s) | 20260506 19:37:08.637 | INFO | Analyzing chunk frames = [3918336,3918848) | 20260506 19:37:08.637 | INFO | MemInfo: { | 20260506 19:37:08.637 | INFO | "gpu" : " 17 GB", | 20260506 19:37:08.637 | INFO | "huge" : "", | 20260506 19:37:08.637 | INFO | "huge_shared" : " 27 GB", | 20260506 19:37:08.637 | INFO | "malloc" : " 14 GB", | 20260506 19:37:08.637 | INFO | "pinned" : " 29 GB" | 20260506 19:37:08.638 | INFO | } | 20260506 19:37:08.638 | INFO | Perf report for the last 5.14318 seconds: | 20260506 19:37:08.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:37:08.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:08.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:08.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:08.638 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 19:37:08.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:08.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:08.638 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 19:37:08.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:08.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:08.638 | INFO | Uploaded 36GiB to GPU in 2.02838s (17.7482GiB/s) | 20260506 19:37:08.638 | INFO | Downloaded 34GiB to GPU in 1.94452s (17.485GiB/s) | 20260506 19:37:13.777 | INFO | Analyzing chunk frames = [3918848,3919360) | 20260506 19:37:13.777 | INFO | MemInfo: { | 20260506 19:37:13.777 | INFO | "gpu" : " 17 GB", | 20260506 19:37:13.777 | INFO | "huge" : "", | 20260506 19:37:13.777 | INFO | "huge_shared" : " 27 GB", | 20260506 19:37:13.777 | INFO | "malloc" : " 14 GB", | 20260506 19:37:13.777 | INFO | "pinned" : " 29 GB" | 20260506 19:37:13.777 | INFO | } | 20260506 19:37:13.777 | INFO | Perf report for the last 5.13927 seconds: | 20260506 19:37:13.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:37:13.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:13.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:13.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:13.777 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 19:37:13.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:13.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:13.777 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:37:13.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:13.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:13.777 | INFO | Uploaded 36GiB to GPU in 2.01006s (17.9099GiB/s) | 20260506 19:37:13.777 | INFO | Downloaded 34GiB to GPU in 2.01105s (16.9066GiB/s) | 20260506 19:37:18.916 | INFO | Analyzing chunk frames = [3919360,3919872) | 20260506 19:37:18.916 | INFO | MemInfo: { | 20260506 19:37:18.916 | INFO | "gpu" : " 17 GB", | 20260506 19:37:18.916 | INFO | "huge" : "", | 20260506 19:37:18.916 | INFO | "huge_shared" : " 27 GB", | 20260506 19:37:18.916 | INFO | "malloc" : " 14 GB", | 20260506 19:37:18.916 | INFO | "pinned" : " 29 GB" | 20260506 19:37:18.917 | INFO | } | 20260506 19:37:18.917 | INFO | Perf report for the last 5.13961 seconds: | 20260506 19:37:18.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:37:18.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:18.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:18.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:18.917 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 19:37:18.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:18.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:18.917 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 19:37:18.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:18.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:18.917 | INFO | Uploaded 36GiB to GPU in 2.02502s (17.7776GiB/s) | 20260506 19:37:18.917 | INFO | Downloaded 34GiB to GPU in 1.98299s (17.1458GiB/s) | 20260506 19:37:19.031 | INFO | Starting a round of Baz IO | 20260506 19:37:22.168 | INFO | Populating RTMetrics report | 20260506 19:37:22.169 | INFO | RTMetrics report populated | 20260506 19:37:22.169 | INFO | Opening tmp RTMetrics json file | 20260506 19:37:22.187 | INFO | Serializing RTMetrics to json | 20260506 19:37:22.191 | INFO | Serialization done, swapping tmp file now... | 20260506 19:37:22.196 | INFO | RTMetrics done | 20260506 19:37:22.520 | INFO | Finished a round of Baz IO: Wrote 2.007 GiB in 3.489 seconds | 20260506 19:37:22.520 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.55% / 0.00% / 65.45% | 20260506 19:37:22.520 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.88923% / 2.78155e-05% / 7.15256e-05% | 20260506 19:37:22.521 | INFO | Baz IO overall: | 20260506 19:37:22.521 | INFO | BytesWritten : 2071324891607 | 20260506 19:37:22.521 | INFO | NumEvents : 1001174935140 | 20260506 19:37:24.058 | INFO | Analyzing chunk frames = [3919872,3920384) | 20260506 19:37:24.058 | INFO | MemInfo: { | 20260506 19:37:24.058 | INFO | "gpu" : " 17 GB", | 20260506 19:37:24.058 | INFO | "huge" : "", | 20260506 19:37:24.058 | INFO | "huge_shared" : " 27 GB", | 20260506 19:37:24.058 | INFO | "malloc" : " 14 GB", | 20260506 19:37:24.058 | INFO | "pinned" : " 29 GB" | 20260506 19:37:24.058 | INFO | } | 20260506 19:37:24.058 | INFO | Perf report for the last 5.14137 seconds: | 20260506 19:37:24.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:37:24.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:24.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:24.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:24.058 | INFO | ANALYSIS : 62.7%, 2.9, 95.9ms, 0 | 20260506 19:37:24.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:24.058 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260506 19:37:24.058 | INFO | PRE_HQ : 59.7%, 1.8, 57.3ms, 0 | 20260506 19:37:24.058 | INFO | BAZWRITER : 67.6%, 4.8, 166.5ms, 0 | 20260506 19:37:24.058 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 19:37:24.058 | INFO | Uploaded 36GiB to GPU in 2.06793s (17.4087GiB/s) | 20260506 19:37:24.058 | INFO | Downloaded 38GiB to GPU in 2.21168s (17.1815GiB/s) | 20260506 19:37:29.196 | INFO | Analyzing chunk frames = [3920384,3920896) | 20260506 19:37:29.196 | INFO | MemInfo: { | 20260506 19:37:29.196 | INFO | "gpu" : " 17 GB", | 20260506 19:37:29.196 | INFO | "huge" : "", | 20260506 19:37:29.196 | INFO | "huge_shared" : " 27 GB", | 20260506 19:37:29.196 | INFO | "malloc" : " 14 GB", | 20260506 19:37:29.196 | INFO | "pinned" : " 29 GB" | 20260506 19:37:29.196 | INFO | } | 20260506 19:37:29.196 | INFO | Perf report for the last 5.13794 seconds: | 20260506 19:37:29.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:37:29.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:29.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:29.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:29.196 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:37:29.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:29.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:29.196 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:37:29.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:29.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:29.196 | INFO | Uploaded 36GiB to GPU in 2.01294s (17.8843GiB/s) | 20260506 19:37:29.196 | INFO | Downloaded 34GiB to GPU in 1.92622s (17.6512GiB/s) | 20260506 19:37:34.333 | INFO | Analyzing chunk frames = [3920896,3921408) | 20260506 19:37:34.333 | INFO | MemInfo: { | 20260506 19:37:34.333 | INFO | "gpu" : " 17 GB", | 20260506 19:37:34.333 | INFO | "huge" : "", | 20260506 19:37:34.333 | INFO | "huge_shared" : " 27 GB", | 20260506 19:37:34.333 | INFO | "malloc" : " 14 GB", | 20260506 19:37:34.333 | INFO | "pinned" : " 29 GB" | 20260506 19:37:34.333 | INFO | } | 20260506 19:37:34.333 | INFO | Perf report for the last 5.13752 seconds: | 20260506 19:37:34.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:37:34.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:34.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:34.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:34.333 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 19:37:34.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:34.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:34.334 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:37:34.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:34.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:34.334 | INFO | Uploaded 36GiB to GPU in 1.98607s (18.1262GiB/s) | 20260506 19:37:34.334 | INFO | Downloaded 34GiB to GPU in 1.91851s (17.7221GiB/s) | 20260506 19:37:39.474 | INFO | Analyzing chunk frames = [3921408,3921920) | 20260506 19:37:39.474 | INFO | MemInfo: { | 20260506 19:37:39.474 | INFO | "gpu" : " 17 GB", | 20260506 19:37:39.474 | INFO | "huge" : "", | 20260506 19:37:39.474 | INFO | "huge_shared" : " 27 GB", | 20260506 19:37:39.474 | INFO | "malloc" : " 14 GB", | 20260506 19:37:39.474 | INFO | "pinned" : " 29 GB" | 20260506 19:37:39.474 | INFO | } | 20260506 19:37:39.474 | INFO | Perf report for the last 5.1403 seconds: | 20260506 19:37:39.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:37:39.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:39.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:39.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:39.474 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 19:37:39.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:39.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:39.474 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:37:39.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:39.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:39.474 | INFO | Uploaded 36GiB to GPU in 1.93796s (18.5762GiB/s) | 20260506 19:37:39.474 | INFO | Downloaded 34GiB to GPU in 1.86728s (18.2083GiB/s) | 20260506 19:37:44.613 | INFO | Analyzing chunk frames = [3921920,3922432) | 20260506 19:37:44.613 | INFO | MemInfo: { | 20260506 19:37:44.613 | INFO | "gpu" : " 17 GB", | 20260506 19:37:44.613 | INFO | "huge" : "", | 20260506 19:37:44.613 | INFO | "huge_shared" : " 27 GB", | 20260506 19:37:44.613 | INFO | "malloc" : " 14 GB", | 20260506 19:37:44.613 | INFO | "pinned" : " 29 GB" | 20260506 19:37:44.613 | INFO | } | 20260506 19:37:44.613 | INFO | Perf report for the last 5.13967 seconds: | 20260506 19:37:44.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:37:44.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:44.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:44.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:44.614 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 19:37:44.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:44.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:44.614 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 19:37:44.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:44.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:44.614 | INFO | Uploaded 36GiB to GPU in 2.02004s (17.8214GiB/s) | 20260506 19:37:44.614 | INFO | Downloaded 34GiB to GPU in 1.90903s (17.8101GiB/s) | 20260506 19:37:49.755 | INFO | Analyzing chunk frames = [3922432,3922944) | 20260506 19:37:49.755 | INFO | MemInfo: { | 20260506 19:37:49.755 | INFO | "gpu" : " 17 GB", | 20260506 19:37:49.755 | INFO | "huge" : "", | 20260506 19:37:49.755 | INFO | "huge_shared" : " 27 GB", | 20260506 19:37:49.755 | INFO | "malloc" : " 14 GB", | 20260506 19:37:49.756 | INFO | "pinned" : " 29 GB" | 20260506 19:37:49.756 | INFO | } | 20260506 19:37:49.756 | INFO | Perf report for the last 5.14221 seconds: | 20260506 19:37:49.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:37:49.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:49.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:49.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:49.756 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 19:37:49.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:49.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:49.756 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 19:37:49.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:49.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:49.756 | INFO | Uploaded 36GiB to GPU in 2.03806s (17.6639GiB/s) | 20260506 19:37:49.756 | INFO | Downloaded 34GiB to GPU in 1.94596s (17.4721GiB/s) | 20260506 19:37:54.897 | INFO | Analyzing chunk frames = [3922944,3923456) | 20260506 19:37:54.897 | INFO | MemInfo: { | 20260506 19:37:54.897 | INFO | "gpu" : " 17 GB", | 20260506 19:37:54.897 | INFO | "huge" : "", | 20260506 19:37:54.897 | INFO | "huge_shared" : " 27 GB", | 20260506 19:37:54.897 | INFO | "malloc" : " 14 GB", | 20260506 19:37:54.897 | INFO | "pinned" : " 29 GB" | 20260506 19:37:54.897 | INFO | } | 20260506 19:37:54.897 | INFO | Perf report for the last 5.14093 seconds: | 20260506 19:37:54.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:37:54.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:54.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:54.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:54.897 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 19:37:54.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:54.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:54.897 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 19:37:54.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:54.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:37:54.897 | INFO | Uploaded 36GiB to GPU in 1.99605s (18.0356GiB/s) | 20260506 19:37:54.897 | INFO | Downloaded 34GiB to GPU in 1.96343s (17.3166GiB/s) | 20260506 19:38:00.031 | INFO | Analyzing chunk frames = [3923456,3923968) | 20260506 19:38:00.031 | INFO | MemInfo: { | 20260506 19:38:00.031 | INFO | "gpu" : " 17 GB", | 20260506 19:38:00.031 | INFO | "huge" : "", | 20260506 19:38:00.031 | INFO | "huge_shared" : " 27 GB", | 20260506 19:38:00.031 | INFO | "malloc" : " 14 GB", | 20260506 19:38:00.031 | INFO | "pinned" : " 29 GB" | 20260506 19:38:00.031 | INFO | } | 20260506 19:38:00.031 | INFO | Perf report for the last 5.13479 seconds: | 20260506 19:38:00.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:38:00.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:00.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:00.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:00.031 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 19:38:00.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:00.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:00.032 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:38:00.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:00.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:00.032 | INFO | Uploaded 36GiB to GPU in 1.97061s (18.2685GiB/s) | 20260506 19:38:00.032 | INFO | Downloaded 34GiB to GPU in 1.97171s (17.2439GiB/s) | 20260506 19:38:00.032 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:38:00.032 | INFO | Total mem usage: 18 GiB | 20260506 19:38:00.032 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:38:00.032 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -17.3%) | 20260506 19:38:00.032 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:38:00.032 | INFO | High Water Breakdown: | 20260506 19:38:00.032 | INFO | 61.2% : 9.39 GiB ( 82.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:38:00.032 | INFO | 38.7% : 5.94 GiB ( 88.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:38:00.032 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:38:00.032 | INFO | ------------------------------------------ | 20260506 19:38:00.032 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:38:00.032 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:38:00.032 | INFO | Total mem usage: 26.5 GiB | 20260506 19:38:00.032 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:38:00.032 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:38:00.032 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:38:00.032 | INFO | High Water Breakdown: | 20260506 19:38:00.032 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:38:00.032 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:38:00.032 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:38:00.032 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:38:00.032 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:38:00.032 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:38:00.032 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:38:00.032 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:38:00.032 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:38:00.032 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:38:00.032 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:38:00.032 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:38:00.032 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:38:00.032 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:38:00.032 | INFO | ------------------------------------------ | 20260506 19:38:00.032 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:38:00.032 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:38:00.032 | INFO | Total mem usage: 25.6 GiB | 20260506 19:38:00.032 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:38:00.032 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:38:00.032 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:38:00.032 | INFO | High Water Breakdown: | 20260506 19:38:00.032 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:38:00.032 | INFO | ------------------------------------------ | 20260506 19:38:00.032 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:38:00.032 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:38:00.032 | INFO | Total mem usage: 15.7 GiB | 20260506 19:38:00.032 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:38:00.032 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:38:00.032 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:38:00.032 | INFO | High Water Breakdown: | 20260506 19:38:00.032 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:38:00.032 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:38:00.032 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:38:00.032 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:38:00.032 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:38:00.032 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:38:00.033 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:38:00.033 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:38:00.033 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:38:00.033 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:38:00.033 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:38:00.033 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:38:00.033 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:38:00.033 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:38:00.033 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:38:00.033 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:38:00.033 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:38:00.033 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:38:00.033 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:38:00.033 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:38:00.033 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:38:00.033 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:38:00.033 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:38:00.033 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:38:00.033 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:38:00.033 | INFO | ------------------------------------------ | 20260506 19:38:00.033 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:38:00.033 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:38:00.033 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:38:00.033 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:38:00.033 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.57e+01, 1.33e+01 / 2.30e+00 | 20260506 19:38:00.033 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 9.96e-03 / 7.23e-03 | 20260506 19:38:00.033 | INFO | PerfReport: 11.63%, Baseline -- 3.63e+00 / 4.44e+00, 5.52e-01 / 3.17e-01 | 20260506 19:38:00.033 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.46e+00, 1.85e-01 / 1.38e-01 | 20260506 19:38:00.033 | INFO | PerfReport: 15.99%, DME -- 4.00e+01 / 4.47e+01, 4.09e+00 / 2.15e+00 | 20260506 19:38:00.033 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.44e+00 / 1.90e+00 | 20260506 19:38:00.033 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.24e+00 / 2.83e+00, 3.78e-01 / 1.94e-01 | 20260506 19:38:00.033 | INFO | PerfReport: 3.35%, Metrics -- 1.05e+00 / 1.40e+00, 2.75e-01 / 1.56e-01 | 20260506 19:38:00.033 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.60e-01 / 4.04e-01, 5.61e-02 / 1.81e-02 | 20260506 19:38:00.033 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:38:00.033 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:38:00.033 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:38:00.033 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 19:38:00.033 | INFO | PerfReport: 21.54%, Upload -- 2.09e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260506 19:38:00.033 | INFO | PerfReport: 21.13%, Download -- 2.05e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260506 19:38:00.033 | INFO | PerfReport: 32.18%, Compute -- 3.12e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 19:38:00.033 | INFO | PerfReport: 1.46%, LockUpload -- 1.41e+00 / 0.00e+00, 5.48e+00 / 0.00e+00 | 20260506 19:38:00.033 | INFO | PerfReport: 1.08%, LockDownload -- 1.04e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260506 19:38:00.033 | INFO | PerfReport: 22.60%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 19:38:00.033 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.52e-02 / 0.00e+00 | 20260506 19:38:00.133 | INFO | Starting a round of Baz IO | 20260506 19:38:03.272 | INFO | Populating RTMetrics report | 20260506 19:38:03.272 | INFO | RTMetrics report populated | 20260506 19:38:03.272 | INFO | Opening tmp RTMetrics json file | 20260506 19:38:03.284 | INFO | Serializing RTMetrics to json | 20260506 19:38:03.310 | INFO | Serialization done, swapping tmp file now... | 20260506 19:38:03.316 | INFO | RTMetrics done | 20260506 19:38:04.031 | INFO | Finished a round of Baz IO: Wrote 2.004 GiB in 3.897 seconds | 20260506 19:38:04.031 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.55% / 0.00% / 65.45% | 20260506 19:38:04.031 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.884128% / 1.19209e-05% / 9.13938e-05% | 20260506 19:38:04.031 | INFO | Baz IO overall: | 20260506 19:38:04.031 | INFO | BytesWritten : 2073477053014 | 20260506 19:38:04.031 | INFO | NumEvents : 1002208974323 | 20260506 19:38:05.169 | INFO | Analyzing chunk frames = [3923968,3924480) | 20260506 19:38:05.169 | INFO | MemInfo: { | 20260506 19:38:05.169 | INFO | "gpu" : " 17 GB", | 20260506 19:38:05.169 | INFO | "huge" : "", | 20260506 19:38:05.169 | INFO | "huge_shared" : " 27 GB", | 20260506 19:38:05.169 | INFO | "malloc" : " 14 GB", | 20260506 19:38:05.169 | INFO | "pinned" : " 29 GB" | 20260506 19:38:05.169 | INFO | } | 20260506 19:38:05.169 | INFO | Perf report for the last 5.13781 seconds: | 20260506 19:38:05.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:38:05.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:05.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:05.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:05.169 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 19:38:05.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:05.169 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 19:38:05.169 | INFO | PRE_HQ : 60.0%, 1.8, 57.2ms, 0 | 20260506 19:38:05.170 | INFO | BAZWRITER : 75.5%, 4.8, 186.3ms, 0 | 20260506 19:38:05.170 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260506 19:38:05.170 | INFO | Uploaded 36GiB to GPU in 2.05409s (17.526GiB/s) | 20260506 19:38:05.170 | INFO | Downloaded 38GiB to GPU in 2.19632s (17.3017GiB/s) | 20260506 19:38:10.311 | INFO | Analyzing chunk frames = [3924480,3924992) | 20260506 19:38:10.311 | INFO | MemInfo: { | 20260506 19:38:10.311 | INFO | "gpu" : " 17 GB", | 20260506 19:38:10.311 | INFO | "huge" : "", | 20260506 19:38:10.311 | INFO | "huge_shared" : " 27 GB", | 20260506 19:38:10.311 | INFO | "malloc" : " 14 GB", | 20260506 19:38:10.311 | INFO | "pinned" : " 29 GB" | 20260506 19:38:10.311 | INFO | } | 20260506 19:38:10.311 | INFO | Perf report for the last 5.1421 seconds: | 20260506 19:38:10.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:38:10.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:10.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:10.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:10.312 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:38:10.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:10.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:10.312 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:38:10.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:10.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:10.312 | INFO | Uploaded 36GiB to GPU in 2.0109s (17.9024GiB/s) | 20260506 19:38:10.312 | INFO | Downloaded 34GiB to GPU in 1.93804s (17.5435GiB/s) | 20260506 19:38:15.446 | INFO | Analyzing chunk frames = [3924992,3925504) | 20260506 19:38:15.446 | INFO | MemInfo: { | 20260506 19:38:15.446 | INFO | "gpu" : " 17 GB", | 20260506 19:38:15.446 | INFO | "huge" : "", | 20260506 19:38:15.446 | INFO | "huge_shared" : " 27 GB", | 20260506 19:38:15.446 | INFO | "malloc" : " 14 GB", | 20260506 19:38:15.447 | INFO | "pinned" : " 29 GB" | 20260506 19:38:15.447 | INFO | } | 20260506 19:38:15.447 | INFO | Perf report for the last 5.13525 seconds: | 20260506 19:38:15.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:38:15.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:15.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:15.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:15.447 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:38:15.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:15.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:15.447 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:38:15.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:15.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:15.447 | INFO | Uploaded 36GiB to GPU in 1.95328s (18.4305GiB/s) | 20260506 19:38:15.447 | INFO | Downloaded 34GiB to GPU in 1.91832s (17.7239GiB/s) | 20260506 19:38:20.582 | INFO | Analyzing chunk frames = [3925504,3926016) | 20260506 19:38:20.582 | INFO | MemInfo: { | 20260506 19:38:20.582 | INFO | "gpu" : " 17 GB", | 20260506 19:38:20.582 | INFO | "huge" : "", | 20260506 19:38:20.582 | INFO | "huge_shared" : " 27 GB", | 20260506 19:38:20.582 | INFO | "malloc" : " 14 GB", | 20260506 19:38:20.582 | INFO | "pinned" : " 29 GB" | 20260506 19:38:20.582 | INFO | } | 20260506 19:38:20.582 | INFO | Perf report for the last 5.13522 seconds: | 20260506 19:38:20.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:38:20.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:20.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:20.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:20.582 | INFO | ANALYSIS : 59.6%, 3.0, 92.3ms, 0 | 20260506 19:38:20.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:20.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:20.582 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:38:20.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:20.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:20.583 | INFO | Uploaded 36GiB to GPU in 1.9307s (18.6461GiB/s) | 20260506 19:38:20.583 | INFO | Downloaded 34GiB to GPU in 1.87395s (18.1435GiB/s) | 20260506 19:38:25.725 | INFO | Analyzing chunk frames = [3926016,3926528) | 20260506 19:38:25.725 | INFO | MemInfo: { | 20260506 19:38:25.725 | INFO | "gpu" : " 17 GB", | 20260506 19:38:25.725 | INFO | "huge" : "", | 20260506 19:38:25.725 | INFO | "huge_shared" : " 27 GB", | 20260506 19:38:25.726 | INFO | "malloc" : " 14 GB", | 20260506 19:38:25.726 | INFO | "pinned" : " 29 GB" | 20260506 19:38:25.726 | INFO | } | 20260506 19:38:25.726 | INFO | Perf report for the last 5.14381 seconds: | 20260506 19:38:25.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:38:25.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:25.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:25.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:25.726 | INFO | ANALYSIS : 59.8%, 3.0, 93.2ms, 0 | 20260506 19:38:25.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:25.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:25.726 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:38:25.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:25.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:25.726 | INFO | Uploaded 36GiB to GPU in 1.94696s (18.4903GiB/s) | 20260506 19:38:25.726 | INFO | Downloaded 34GiB to GPU in 1.88749s (18.0133GiB/s) | 20260506 19:38:30.870 | INFO | Analyzing chunk frames = [3926528,3927040) | 20260506 19:38:30.870 | INFO | MemInfo: { | 20260506 19:38:30.870 | INFO | "gpu" : " 17 GB", | 20260506 19:38:30.870 | INFO | "huge" : "", | 20260506 19:38:30.870 | INFO | "huge_shared" : " 27 GB", | 20260506 19:38:30.870 | INFO | "malloc" : " 14 GB", | 20260506 19:38:30.871 | INFO | "pinned" : " 29 GB" | 20260506 19:38:30.871 | INFO | } | 20260506 19:38:30.871 | INFO | Perf report for the last 5.14492 seconds: | 20260506 19:38:30.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:38:30.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:30.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:30.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:30.871 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 19:38:30.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:30.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:30.871 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 19:38:30.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:30.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:30.871 | INFO | Uploaded 36GiB to GPU in 1.98112s (18.1716GiB/s) | 20260506 19:38:30.871 | INFO | Downloaded 34GiB to GPU in 1.95313s (17.4079GiB/s) | 20260506 19:38:36.010 | INFO | Analyzing chunk frames = [3927040,3927552) | 20260506 19:38:36.010 | INFO | MemInfo: { | 20260506 19:38:36.010 | INFO | "gpu" : " 17 GB", | 20260506 19:38:36.010 | INFO | "huge" : "", | 20260506 19:38:36.010 | INFO | "huge_shared" : " 27 GB", | 20260506 19:38:36.010 | INFO | "malloc" : " 14 GB", | 20260506 19:38:36.010 | INFO | "pinned" : " 29 GB" | 20260506 19:38:36.010 | INFO | } | 20260506 19:38:36.010 | INFO | Perf report for the last 5.13936 seconds: | 20260506 19:38:36.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:38:36.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:36.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:36.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:36.010 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:38:36.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:36.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:36.010 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:38:36.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:36.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:36.010 | INFO | Uploaded 36GiB to GPU in 1.99516s (18.0437GiB/s) | 20260506 19:38:36.010 | INFO | Downloaded 34GiB to GPU in 2.01532s (16.8708GiB/s) | 20260506 19:38:41.152 | INFO | Analyzing chunk frames = [3927552,3928064) | 20260506 19:38:41.152 | INFO | MemInfo: { | 20260506 19:38:41.152 | INFO | "gpu" : " 17 GB", | 20260506 19:38:41.152 | INFO | "huge" : "", | 20260506 19:38:41.152 | INFO | "huge_shared" : " 27 GB", | 20260506 19:38:41.152 | INFO | "malloc" : " 14 GB", | 20260506 19:38:41.152 | INFO | "pinned" : " 29 GB" | 20260506 19:38:41.152 | INFO | } | 20260506 19:38:41.152 | INFO | Perf report for the last 5.14239 seconds: | 20260506 19:38:41.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:38:41.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:41.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:41.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:41.152 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 19:38:41.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:41.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:41.152 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 19:38:41.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:41.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:41.153 | INFO | Uploaded 36GiB to GPU in 2.0305s (17.7296GiB/s) | 20260506 19:38:41.153 | INFO | Downloaded 34GiB to GPU in 1.97965s (17.1748GiB/s) | 20260506 19:38:41.267 | INFO | Starting a round of Baz IO | 20260506 19:38:44.398 | INFO | Populating RTMetrics report | 20260506 19:38:44.398 | INFO | RTMetrics report populated | 20260506 19:38:44.398 | INFO | Opening tmp RTMetrics json file | 20260506 19:38:44.399 | INFO | Serializing RTMetrics to json | 20260506 19:38:44.400 | INFO | Serialization done, swapping tmp file now... | 20260506 19:38:44.401 | INFO | RTMetrics done | 20260506 19:38:44.564 | INFO | Finished a round of Baz IO: Wrote 2.004 GiB in 3.296 seconds | 20260506 19:38:44.564 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.54% / 0.00% / 65.45% | 20260506 19:38:44.564 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.88292% / 7.94729e-06% / 9.13938e-05% | 20260506 19:38:44.564 | INFO | Baz IO overall: | 20260506 19:38:44.564 | INFO | BytesWritten : 2075629286175 | 20260506 19:38:44.564 | INFO | NumEvents : 1003243259009 | 20260506 19:38:46.284 | INFO | Analyzing chunk frames = [3928064,3928576) | 20260506 19:38:46.284 | INFO | MemInfo: { | 20260506 19:38:46.284 | INFO | "gpu" : " 17 GB", | 20260506 19:38:46.284 | INFO | "huge" : "", | 20260506 19:38:46.284 | INFO | "huge_shared" : " 27 GB", | 20260506 19:38:46.284 | INFO | "malloc" : " 14 GB", | 20260506 19:38:46.284 | INFO | "pinned" : " 29 GB" | 20260506 19:38:46.284 | INFO | } | 20260506 19:38:46.284 | INFO | Perf report for the last 5.13171 seconds: | 20260506 19:38:46.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:38:46.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:46.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:46.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:46.284 | INFO | ANALYSIS : 62.7%, 2.9, 95.3ms, 0 | 20260506 19:38:46.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:46.284 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260506 19:38:46.284 | INFO | PRE_HQ : 60.3%, 1.9, 58.0ms, 0 | 20260506 19:38:46.284 | INFO | BAZWRITER : 63.9%, 4.6, 151.9ms, 0 | 20260506 19:38:46.284 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260506 19:38:46.284 | INFO | Uploaded 36GiB to GPU in 2.0352s (17.6887GiB/s) | 20260506 19:38:46.284 | INFO | Downloaded 38GiB to GPU in 2.2172s (17.1388GiB/s) | 20260506 19:38:51.420 | INFO | Analyzing chunk frames = [3928576,3929088) | 20260506 19:38:51.420 | INFO | MemInfo: { | 20260506 19:38:51.420 | INFO | "gpu" : " 17 GB", | 20260506 19:38:51.420 | INFO | "huge" : "", | 20260506 19:38:51.420 | INFO | "huge_shared" : " 27 GB", | 20260506 19:38:51.420 | INFO | "malloc" : " 14 GB", | 20260506 19:38:51.420 | INFO | "pinned" : " 29 GB" | 20260506 19:38:51.420 | INFO | } | 20260506 19:38:51.420 | INFO | Perf report for the last 5.13575 seconds: | 20260506 19:38:51.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:38:51.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:51.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:51.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:51.420 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:38:51.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:51.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:51.420 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:38:51.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:51.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:51.420 | INFO | Uploaded 36GiB to GPU in 1.99896s (18.0094GiB/s) | 20260506 19:38:51.420 | INFO | Downloaded 34GiB to GPU in 1.92655s (17.6481GiB/s) | 20260506 19:38:56.561 | INFO | Analyzing chunk frames = [3929088,3929600) | 20260506 19:38:56.561 | INFO | MemInfo: { | 20260506 19:38:56.561 | INFO | "gpu" : " 17 GB", | 20260506 19:38:56.561 | INFO | "huge" : "", | 20260506 19:38:56.561 | INFO | "huge_shared" : " 27 GB", | 20260506 19:38:56.562 | INFO | "malloc" : " 14 GB", | 20260506 19:38:56.562 | INFO | "pinned" : " 29 GB" | 20260506 19:38:56.562 | INFO | } | 20260506 19:38:56.562 | INFO | Perf report for the last 5.14184 seconds: | 20260506 19:38:56.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:38:56.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:56.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:56.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:56.562 | INFO | ANALYSIS : 60.0%, 3.0, 93.2ms, 0 | 20260506 19:38:56.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:56.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:56.562 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:38:56.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:56.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:38:56.562 | INFO | Uploaded 36GiB to GPU in 1.93095s (18.6437GiB/s) | 20260506 19:38:56.562 | INFO | Downloaded 34GiB to GPU in 1.87919s (18.0929GiB/s) | 20260506 19:39:01.700 | INFO | Analyzing chunk frames = [3929600,3930112) | 20260506 19:39:01.700 | INFO | MemInfo: { | 20260506 19:39:01.700 | INFO | "gpu" : " 17 GB", | 20260506 19:39:01.701 | INFO | "huge" : "", | 20260506 19:39:01.701 | INFO | "huge_shared" : " 27 GB", | 20260506 19:39:01.701 | INFO | "malloc" : " 14 GB", | 20260506 19:39:01.701 | INFO | "pinned" : " 29 GB" | 20260506 19:39:01.701 | INFO | } | 20260506 19:39:01.701 | INFO | Perf report for the last 5.13931 seconds: | 20260506 19:39:01.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:39:01.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:01.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:01.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:01.701 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:39:01.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:01.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:01.701 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 19:39:01.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:01.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:01.701 | INFO | Uploaded 36GiB to GPU in 1.90887s (18.8593GiB/s) | 20260506 19:39:01.701 | INFO | Downloaded 34GiB to GPU in 1.87308s (18.152GiB/s) | 20260506 19:39:06.844 | INFO | Analyzing chunk frames = [3930112,3930624) | 20260506 19:39:06.844 | INFO | MemInfo: { | 20260506 19:39:06.844 | INFO | "gpu" : " 17 GB", | 20260506 19:39:06.844 | INFO | "huge" : "", | 20260506 19:39:06.844 | INFO | "huge_shared" : " 27 GB", | 20260506 19:39:06.844 | INFO | "malloc" : " 14 GB", | 20260506 19:39:06.844 | INFO | "pinned" : " 29 GB" | 20260506 19:39:06.844 | INFO | } | 20260506 19:39:06.844 | INFO | Perf report for the last 5.1428 seconds: | 20260506 19:39:06.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:39:06.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:06.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:06.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:06.844 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 19:39:06.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:06.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:06.844 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:39:06.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:06.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:06.844 | INFO | Uploaded 36GiB to GPU in 2.01597s (17.8574GiB/s) | 20260506 19:39:06.844 | INFO | Downloaded 34GiB to GPU in 1.91623s (17.7432GiB/s) | 20260506 19:39:11.984 | INFO | Analyzing chunk frames = [3930624,3931136) | 20260506 19:39:11.984 | INFO | MemInfo: { | 20260506 19:39:11.984 | INFO | "gpu" : " 17 GB", | 20260506 19:39:11.984 | INFO | "huge" : "", | 20260506 19:39:11.984 | INFO | "huge_shared" : " 27 GB", | 20260506 19:39:11.984 | INFO | "malloc" : " 14 GB", | 20260506 19:39:11.984 | INFO | "pinned" : " 29 GB" | 20260506 19:39:11.984 | INFO | } | 20260506 19:39:11.984 | INFO | Perf report for the last 5.14045 seconds: | 20260506 19:39:11.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:39:11.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:11.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:11.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:11.984 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 19:39:11.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:11.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:11.984 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 19:39:11.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:11.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:11.985 | INFO | Uploaded 36GiB to GPU in 2.0074s (17.9337GiB/s) | 20260506 19:39:11.985 | INFO | Downloaded 34GiB to GPU in 1.95961s (17.3504GiB/s) | 20260506 19:39:17.126 | INFO | Analyzing chunk frames = [3931136,3931648) | 20260506 19:39:17.126 | INFO | MemInfo: { | 20260506 19:39:17.126 | INFO | "gpu" : " 17 GB", | 20260506 19:39:17.126 | INFO | "huge" : "", | 20260506 19:39:17.126 | INFO | "huge_shared" : " 27 GB", | 20260506 19:39:17.126 | INFO | "malloc" : " 14 GB", | 20260506 19:39:17.126 | INFO | "pinned" : " 29 GB" | 20260506 19:39:17.126 | INFO | } | 20260506 19:39:17.127 | INFO | Perf report for the last 5.14217 seconds: | 20260506 19:39:17.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:39:17.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:17.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:17.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:17.127 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 19:39:17.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:17.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:17.127 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 19:39:17.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:17.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:17.127 | INFO | Uploaded 36GiB to GPU in 1.97605s (18.2182GiB/s) | 20260506 19:39:17.127 | INFO | Downloaded 34GiB to GPU in 2.01207s (16.898GiB/s) | 20260506 19:39:22.260 | INFO | Analyzing chunk frames = [3931648,3932160) | 20260506 19:39:22.260 | INFO | MemInfo: { | 20260506 19:39:22.260 | INFO | "gpu" : " 17 GB", | 20260506 19:39:22.260 | INFO | "huge" : "", | 20260506 19:39:22.260 | INFO | "huge_shared" : " 27 GB", | 20260506 19:39:22.260 | INFO | "malloc" : " 14 GB", | 20260506 19:39:22.260 | INFO | "pinned" : " 29 GB" | 20260506 19:39:22.260 | INFO | } | 20260506 19:39:22.260 | INFO | Perf report for the last 5.13392 seconds: | 20260506 19:39:22.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:39:22.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:22.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:22.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:22.261 | INFO | ANALYSIS : 62.0%, 3.0, 96.7ms, 0 | 20260506 19:39:22.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:22.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:22.261 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 19:39:22.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:22.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:22.261 | INFO | Uploaded 36GiB to GPU in 2.04123s (17.6364GiB/s) | 20260506 19:39:22.261 | INFO | Downloaded 34GiB to GPU in 1.97727s (17.1954GiB/s) | 20260506 19:39:22.261 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:39:22.261 | INFO | Total mem usage: 18 GiB | 20260506 19:39:22.261 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:39:22.261 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -12%) | 20260506 19:39:22.261 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:39:22.261 | INFO | High Water Breakdown: | 20260506 19:39:22.261 | INFO | 61.9% : 9.96 GiB ( 87.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:39:22.261 | INFO | 37.9% : 6.1 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:39:22.261 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:39:22.261 | INFO | ------------------------------------------ | 20260506 19:39:22.261 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:39:22.261 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:39:22.261 | INFO | Total mem usage: 26.5 GiB | 20260506 19:39:22.261 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:39:22.261 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:39:22.261 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:39:22.261 | INFO | High Water Breakdown: | 20260506 19:39:22.261 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:39:22.261 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:39:22.261 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:39:22.261 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:39:22.261 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:39:22.261 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:39:22.261 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:39:22.261 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:39:22.261 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:39:22.261 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:39:22.261 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:39:22.261 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:39:22.261 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:39:22.261 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:39:22.261 | INFO | ------------------------------------------ | 20260506 19:39:22.261 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:39:22.261 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:39:22.261 | INFO | Total mem usage: 25.6 GiB | 20260506 19:39:22.261 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:39:22.261 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:39:22.261 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:39:22.261 | INFO | High Water Breakdown: | 20260506 19:39:22.261 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:39:22.261 | INFO | ------------------------------------------ | 20260506 19:39:22.261 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:39:22.261 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:39:22.261 | INFO | Total mem usage: 15.7 GiB | 20260506 19:39:22.261 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:39:22.261 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:39:22.261 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 19:39:22.261 | INFO | High Water Breakdown: | 20260506 19:39:22.261 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:39:22.261 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:39:22.261 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:39:22.262 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:39:22.262 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:39:22.262 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:39:22.262 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:39:22.262 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:39:22.262 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:39:22.262 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:39:22.262 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:39:22.262 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:39:22.262 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:39:22.262 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:39:22.262 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:39:22.262 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:39:22.262 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:39:22.262 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:39:22.262 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:39:22.262 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:39:22.262 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:39:22.262 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:39:22.262 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:39:22.262 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:39:22.262 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:39:22.262 | INFO | ------------------------------------------ | 20260506 19:39:22.262 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:39:22.262 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:39:22.262 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:39:22.262 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:39:22.262 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.57e+01, 1.34e+01 / 2.56e+00 | 20260506 19:39:22.262 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.98e-03 / 5.65e-03 | 20260506 19:39:22.262 | INFO | PerfReport: 11.61%, Baseline -- 3.62e+00 / 4.39e+00, 5.46e-01 / 3.46e-01 | 20260506 19:39:22.262 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.43e+00, 1.80e-01 / 1.27e-01 | 20260506 19:39:22.262 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.48e+01, 4.17e+00 / 2.66e+00 | 20260506 19:39:22.262 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.42e+00 / 1.96e+00 | 20260506 19:39:22.262 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.24e+00 / 2.82e+00, 3.78e-01 / 2.35e-01 | 20260506 19:39:22.262 | INFO | PerfReport: 3.35%, Metrics -- 1.05e+00 / 1.36e+00, 2.75e-01 / 1.58e-01 | 20260506 19:39:22.262 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.60e-01 / 4.04e-01, 5.59e-02 / 2.42e-02 | 20260506 19:39:22.262 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:39:22.262 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:39:22.262 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:39:22.262 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 19:39:22.262 | INFO | PerfReport: 21.40%, Upload -- 2.07e+01 / 0.00e+00, 6.79e+00 / 0.00e+00 | 20260506 19:39:22.262 | INFO | PerfReport: 21.21%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 19:39:22.262 | INFO | PerfReport: 32.26%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 19:39:22.262 | INFO | PerfReport: 1.25%, LockUpload -- 1.21e+00 / 0.00e+00, 5.19e+00 / 0.00e+00 | 20260506 19:39:22.262 | INFO | PerfReport: 1.21%, LockDownload -- 1.17e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260506 19:39:22.262 | INFO | PerfReport: 22.65%, LockCompute -- 2.19e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 19:39:22.262 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260506 19:39:22.375 | INFO | Starting a round of Baz IO | 20260506 19:39:25.498 | INFO | Populating RTMetrics report | 20260506 19:39:25.498 | INFO | RTMetrics report populated | 20260506 19:39:25.498 | INFO | Opening tmp RTMetrics json file | 20260506 19:39:25.521 | INFO | Serializing RTMetrics to json | 20260506 19:39:25.523 | INFO | Serialization done, swapping tmp file now... | 20260506 19:39:25.526 | INFO | RTMetrics done | 20260506 19:39:25.915 | INFO | Finished a round of Baz IO: Wrote 2.005 GiB in 3.54 seconds | 20260506 19:39:25.915 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.54% / 0.00% / 65.46% | 20260506 19:39:25.915 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.881199% / 0% / 7.15256e-05% | 20260506 19:39:25.915 | INFO | Baz IO overall: | 20260506 19:39:25.915 | INFO | BytesWritten : 2077781735088 | 20260506 19:39:25.915 | INFO | NumEvents : 1004277673587 | 20260506 19:39:27.397 | INFO | Analyzing chunk frames = [3932160,3932672) | 20260506 19:39:27.397 | INFO | MemInfo: { | 20260506 19:39:27.397 | INFO | "gpu" : " 17 GB", | 20260506 19:39:27.397 | INFO | "huge" : "", | 20260506 19:39:27.397 | INFO | "huge_shared" : " 27 GB", | 20260506 19:39:27.397 | INFO | "malloc" : " 14 GB", | 20260506 19:39:27.397 | INFO | "pinned" : " 29 GB" | 20260506 19:39:27.397 | INFO | } | 20260506 19:39:27.397 | INFO | Perf report for the last 5.13617 seconds: | 20260506 19:39:27.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:39:27.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:27.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:27.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:27.397 | INFO | ANALYSIS : 62.4%, 2.9, 95.1ms, 0 | 20260506 19:39:27.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:27.397 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 19:39:27.397 | INFO | PRE_HQ : 59.9%, 1.9, 57.5ms, 0 | 20260506 19:39:27.397 | INFO | BAZWRITER : 68.6%, 4.8, 169.3ms, 0 | 20260506 19:39:27.397 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 19:39:27.397 | INFO | Uploaded 36GiB to GPU in 2.03913s (17.6546GiB/s) | 20260506 19:39:27.397 | INFO | Downloaded 38GiB to GPU in 2.21425s (17.1616GiB/s) | 20260506 19:39:32.535 | INFO | Analyzing chunk frames = [3932672,3933184) | 20260506 19:39:32.535 | INFO | MemInfo: { | 20260506 19:39:32.535 | INFO | "gpu" : " 17 GB", | 20260506 19:39:32.535 | INFO | "huge" : "", | 20260506 19:39:32.535 | INFO | "huge_shared" : " 27 GB", | 20260506 19:39:32.535 | INFO | "malloc" : " 14 GB", | 20260506 19:39:32.536 | INFO | "pinned" : " 29 GB" | 20260506 19:39:32.536 | INFO | } | 20260506 19:39:32.536 | INFO | Perf report for the last 5.13902 seconds: | 20260506 19:39:32.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:39:32.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:32.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:32.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:32.536 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:39:32.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:32.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:32.536 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:39:32.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:32.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:32.536 | INFO | Uploaded 36GiB to GPU in 2.01812s (17.8384GiB/s) | 20260506 19:39:32.536 | INFO | Downloaded 34GiB to GPU in 1.93325s (17.587GiB/s) | 20260506 19:39:37.675 | INFO | Analyzing chunk frames = [3933184,3933696) | 20260506 19:39:37.675 | INFO | MemInfo: { | 20260506 19:39:37.675 | INFO | "gpu" : " 17 GB", | 20260506 19:39:37.675 | INFO | "huge" : "", | 20260506 19:39:37.675 | INFO | "huge_shared" : " 27 GB", | 20260506 19:39:37.675 | INFO | "malloc" : " 14 GB", | 20260506 19:39:37.675 | INFO | "pinned" : " 29 GB" | 20260506 19:39:37.675 | INFO | } | 20260506 19:39:37.675 | INFO | Perf report for the last 5.139 seconds: | 20260506 19:39:37.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:39:37.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:37.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:37.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:37.675 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 19:39:37.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:37.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:37.675 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:39:37.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:37.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:37.675 | INFO | Uploaded 36GiB to GPU in 1.98429s (18.1425GiB/s) | 20260506 19:39:37.675 | INFO | Downloaded 34GiB to GPU in 1.91533s (17.7515GiB/s) | 20260506 19:39:42.824 | INFO | Analyzing chunk frames = [3933696,3934208) | 20260506 19:39:42.824 | INFO | MemInfo: { | 20260506 19:39:42.824 | INFO | "gpu" : " 17 GB", | 20260506 19:39:42.824 | INFO | "huge" : "", | 20260506 19:39:42.824 | INFO | "huge_shared" : " 27 GB", | 20260506 19:39:42.824 | INFO | "malloc" : " 14 GB", | 20260506 19:39:42.824 | INFO | "pinned" : " 29 GB" | 20260506 19:39:42.824 | INFO | } | 20260506 19:39:42.824 | INFO | Perf report for the last 5.14913 seconds: | 20260506 19:39:42.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:39:42.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:42.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:42.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:42.824 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260506 19:39:42.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:42.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:42.824 | INFO | PRE_HQ : 26.6%, 1.0, 13.9ms, 0 | 20260506 19:39:42.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:42.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:42.824 | INFO | Uploaded 36GiB to GPU in 1.93218s (18.6318GiB/s) | 20260506 19:39:42.824 | INFO | Downloaded 34GiB to GPU in 1.8637s (18.2432GiB/s) | 20260506 19:39:47.960 | INFO | Analyzing chunk frames = [3934208,3934720) | 20260506 19:39:47.960 | INFO | MemInfo: { | 20260506 19:39:47.960 | INFO | "gpu" : " 17 GB", | 20260506 19:39:47.960 | INFO | "huge" : "", | 20260506 19:39:47.960 | INFO | "huge_shared" : " 27 GB", | 20260506 19:39:47.960 | INFO | "malloc" : " 14 GB", | 20260506 19:39:47.960 | INFO | "pinned" : " 29 GB" | 20260506 19:39:47.960 | INFO | } | 20260506 19:39:47.960 | INFO | Perf report for the last 5.13652 seconds: | 20260506 19:39:47.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:39:47.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:47.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:47.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:47.960 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 19:39:47.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:47.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:47.960 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:39:47.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:47.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:47.961 | INFO | Uploaded 36GiB to GPU in 2.01472s (17.8684GiB/s) | 20260506 19:39:47.961 | INFO | Downloaded 34GiB to GPU in 1.91893s (17.7182GiB/s) | 20260506 19:39:53.100 | INFO | Analyzing chunk frames = [3934720,3935232) | 20260506 19:39:53.100 | INFO | MemInfo: { | 20260506 19:39:53.100 | INFO | "gpu" : " 17 GB", | 20260506 19:39:53.100 | INFO | "huge" : "", | 20260506 19:39:53.100 | INFO | "huge_shared" : " 27 GB", | 20260506 19:39:53.100 | INFO | "malloc" : " 14 GB", | 20260506 19:39:53.100 | INFO | "pinned" : " 29 GB" | 20260506 19:39:53.100 | INFO | } | 20260506 19:39:53.100 | INFO | Perf report for the last 5.13942 seconds: | 20260506 19:39:53.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:39:53.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:53.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:53.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:53.100 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 19:39:53.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:53.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:53.100 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:39:53.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:53.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:53.100 | INFO | Uploaded 36GiB to GPU in 2.04095s (17.6388GiB/s) | 20260506 19:39:53.100 | INFO | Downloaded 34GiB to GPU in 1.94498s (17.4809GiB/s) | 20260506 19:39:58.238 | INFO | Analyzing chunk frames = [3935232,3935744) | 20260506 19:39:58.238 | INFO | MemInfo: { | 20260506 19:39:58.238 | INFO | "gpu" : " 17 GB", | 20260506 19:39:58.238 | INFO | "huge" : "", | 20260506 19:39:58.238 | INFO | "huge_shared" : " 27 GB", | 20260506 19:39:58.238 | INFO | "malloc" : " 14 GB", | 20260506 19:39:58.238 | INFO | "pinned" : " 29 GB" | 20260506 19:39:58.238 | INFO | } | 20260506 19:39:58.238 | INFO | Perf report for the last 5.13854 seconds: | 20260506 19:39:58.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:39:58.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:58.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:58.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:58.239 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 19:39:58.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:58.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:58.239 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 19:39:58.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:58.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:39:58.239 | INFO | Uploaded 36GiB to GPU in 2.02616s (17.7676GiB/s) | 20260506 19:39:58.239 | INFO | Downloaded 34GiB to GPU in 1.98902s (17.0939GiB/s) | 20260506 19:40:03.371 | INFO | Analyzing chunk frames = [3935744,3936256) | 20260506 19:40:03.371 | INFO | MemInfo: { | 20260506 19:40:03.371 | INFO | "gpu" : " 17 GB", | 20260506 19:40:03.371 | INFO | "huge" : "", | 20260506 19:40:03.371 | INFO | "huge_shared" : " 27 GB", | 20260506 19:40:03.371 | INFO | "malloc" : " 14 GB", | 20260506 19:40:03.371 | INFO | "pinned" : " 29 GB" | 20260506 19:40:03.371 | INFO | } | 20260506 19:40:03.371 | INFO | Perf report for the last 5.13258 seconds: | 20260506 19:40:03.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:40:03.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:03.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:03.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:03.371 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260506 19:40:03.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:03.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:03.371 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:40:03.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:03.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:03.371 | INFO | Uploaded 36GiB to GPU in 2.02962s (17.7373GiB/s) | 20260506 19:40:03.371 | INFO | Downloaded 34GiB to GPU in 1.99847s (17.013GiB/s) | 20260506 19:40:03.487 | INFO | Starting a round of Baz IO | 20260506 19:40:06.619 | INFO | Populating RTMetrics report | 20260506 19:40:06.620 | INFO | RTMetrics report populated | 20260506 19:40:06.620 | INFO | Opening tmp RTMetrics json file | 20260506 19:40:06.647 | INFO | Serializing RTMetrics to json | 20260506 19:40:06.705 | INFO | Serialization done, swapping tmp file now... | 20260506 19:40:06.722 | INFO | RTMetrics done | 20260506 19:40:06.812 | INFO | Finished a round of Baz IO: Wrote 2.004 GiB in 3.324 seconds | 20260506 19:40:06.812 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.53% / 0.00% / 65.46% | 20260506 19:40:06.812 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.879622% / 7.94729e-06% / 5.5631e-05% | 20260506 19:40:06.812 | INFO | Baz IO overall: | 20260506 19:40:06.812 | INFO | BytesWritten : 2079933762533 | 20260506 19:40:06.812 | INFO | NumEvents : 1005312021104 | 20260506 19:40:08.511 | INFO | Analyzing chunk frames = [3936256,3936768) | 20260506 19:40:08.511 | INFO | MemInfo: { | 20260506 19:40:08.511 | INFO | "gpu" : " 17 GB", | 20260506 19:40:08.511 | INFO | "huge" : "", | 20260506 19:40:08.511 | INFO | "huge_shared" : " 27 GB", | 20260506 19:40:08.511 | INFO | "malloc" : " 14 GB", | 20260506 19:40:08.511 | INFO | "pinned" : " 29 GB" | 20260506 19:40:08.511 | INFO | } | 20260506 19:40:08.512 | INFO | Perf report for the last 5.14062 seconds: | 20260506 19:40:08.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:40:08.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:08.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:08.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:08.512 | INFO | ANALYSIS : 62.6%, 2.9, 95.5ms, 0 | 20260506 19:40:08.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:08.512 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 19:40:08.512 | INFO | PRE_HQ : 59.8%, 1.8, 57.3ms, 0 | 20260506 19:40:08.512 | INFO | BAZWRITER : 64.3%, 4.8, 158.1ms, 0 | 20260506 19:40:08.512 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 19:40:08.512 | INFO | Uploaded 36GiB to GPU in 2.02719s (17.7586GiB/s) | 20260506 19:40:08.512 | INFO | Downloaded 38GiB to GPU in 2.2182s (17.131GiB/s) | 20260506 19:40:13.654 | INFO | Analyzing chunk frames = [3936768,3937280) | 20260506 19:40:13.654 | INFO | MemInfo: { | 20260506 19:40:13.654 | INFO | "gpu" : " 17 GB", | 20260506 19:40:13.654 | INFO | "huge" : "", | 20260506 19:40:13.654 | INFO | "huge_shared" : " 27 GB", | 20260506 19:40:13.654 | INFO | "malloc" : " 14 GB", | 20260506 19:40:13.654 | INFO | "pinned" : " 29 GB" | 20260506 19:40:13.654 | INFO | } | 20260506 19:40:13.654 | INFO | Perf report for the last 5.14223 seconds: | 20260506 19:40:13.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:40:13.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:13.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:13.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:13.654 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 19:40:13.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:13.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:13.654 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 19:40:13.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:13.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:13.654 | INFO | Uploaded 36GiB to GPU in 1.98231s (18.1607GiB/s) | 20260506 19:40:13.654 | INFO | Downloaded 34GiB to GPU in 1.92172s (17.6925GiB/s) | 20260506 19:40:18.792 | INFO | Analyzing chunk frames = [3937280,3937792) | 20260506 19:40:18.792 | INFO | MemInfo: { | 20260506 19:40:18.792 | INFO | "gpu" : " 17 GB", | 20260506 19:40:18.792 | INFO | "huge" : "", | 20260506 19:40:18.792 | INFO | "huge_shared" : " 27 GB", | 20260506 19:40:18.792 | INFO | "malloc" : " 14 GB", | 20260506 19:40:18.792 | INFO | "pinned" : " 29 GB" | 20260506 19:40:18.792 | INFO | } | 20260506 19:40:18.792 | INFO | Perf report for the last 5.13827 seconds: | 20260506 19:40:18.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:40:18.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:18.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:18.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:18.792 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:40:18.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:18.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:18.792 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:40:18.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:18.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:18.792 | INFO | Uploaded 36GiB to GPU in 1.9636s (18.3336GiB/s) | 20260506 19:40:18.792 | INFO | Downloaded 34GiB to GPU in 1.92475s (17.6646GiB/s) | 20260506 19:40:23.933 | INFO | Analyzing chunk frames = [3937792,3938304) | 20260506 19:40:23.933 | INFO | MemInfo: { | 20260506 19:40:23.933 | INFO | "gpu" : " 17 GB", | 20260506 19:40:23.933 | INFO | "huge" : "", | 20260506 19:40:23.933 | INFO | "huge_shared" : " 27 GB", | 20260506 19:40:23.933 | INFO | "malloc" : " 14 GB", | 20260506 19:40:23.933 | INFO | "pinned" : " 29 GB" | 20260506 19:40:23.933 | INFO | } | 20260506 19:40:23.933 | INFO | Perf report for the last 5.1412 seconds: | 20260506 19:40:23.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:40:23.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:23.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:23.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:23.933 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 19:40:23.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:23.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:23.933 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:40:23.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:23.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:23.934 | INFO | Uploaded 36GiB to GPU in 1.9255s (18.6964GiB/s) | 20260506 19:40:23.934 | INFO | Downloaded 34GiB to GPU in 1.87103s (18.1718GiB/s) | 20260506 19:40:29.077 | INFO | Analyzing chunk frames = [3938304,3938816) | 20260506 19:40:29.077 | INFO | MemInfo: { | 20260506 19:40:29.077 | INFO | "gpu" : " 17 GB", | 20260506 19:40:29.077 | INFO | "huge" : "", | 20260506 19:40:29.077 | INFO | "huge_shared" : " 27 GB", | 20260506 19:40:29.077 | INFO | "malloc" : " 14 GB", | 20260506 19:40:29.077 | INFO | "pinned" : " 29 GB" | 20260506 19:40:29.077 | INFO | } | 20260506 19:40:29.077 | INFO | Perf report for the last 5.14366 seconds: | 20260506 19:40:29.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:40:29.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:29.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:29.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:29.077 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 19:40:29.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:29.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:29.077 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:40:29.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:29.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:29.077 | INFO | Uploaded 36GiB to GPU in 1.95431s (18.4208GiB/s) | 20260506 19:40:29.077 | INFO | Downloaded 34GiB to GPU in 1.92209s (17.6891GiB/s) | 20260506 19:40:34.214 | INFO | Analyzing chunk frames = [3938816,3939328) | 20260506 19:40:34.214 | INFO | MemInfo: { | 20260506 19:40:34.214 | INFO | "gpu" : " 17 GB", | 20260506 19:40:34.214 | INFO | "huge" : "", | 20260506 19:40:34.214 | INFO | "huge_shared" : " 27 GB", | 20260506 19:40:34.214 | INFO | "malloc" : " 14 GB", | 20260506 19:40:34.214 | INFO | "pinned" : " 29 GB" | 20260506 19:40:34.214 | INFO | } | 20260506 19:40:34.215 | INFO | Perf report for the last 5.13759 seconds: | 20260506 19:40:34.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:40:34.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:34.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:34.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:34.215 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 19:40:34.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:34.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:34.215 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 19:40:34.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:34.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:34.215 | INFO | Uploaded 36GiB to GPU in 1.99024s (18.0882GiB/s) | 20260506 19:40:34.215 | INFO | Downloaded 34GiB to GPU in 1.94123s (17.5147GiB/s) | 20260506 19:40:39.347 | INFO | Analyzing chunk frames = [3939328,3939840) | 20260506 19:40:39.347 | INFO | MemInfo: { | 20260506 19:40:39.347 | INFO | "gpu" : " 17 GB", | 20260506 19:40:39.347 | INFO | "huge" : "", | 20260506 19:40:39.347 | INFO | "huge_shared" : " 27 GB", | 20260506 19:40:39.347 | INFO | "malloc" : " 14 GB", | 20260506 19:40:39.347 | INFO | "pinned" : " 29 GB" | 20260506 19:40:39.347 | INFO | } | 20260506 19:40:39.347 | INFO | Perf report for the last 5.1329 seconds: | 20260506 19:40:39.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:40:39.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:39.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:39.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:39.348 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260506 19:40:39.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:39.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:39.348 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 19:40:39.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:39.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:39.348 | INFO | Uploaded 36GiB to GPU in 1.97715s (18.208GiB/s) | 20260506 19:40:39.348 | INFO | Downloaded 34GiB to GPU in 2.00726s (16.9385GiB/s) | 20260506 19:40:44.486 | INFO | Analyzing chunk frames = [3939840,3940352) | 20260506 19:40:44.486 | INFO | MemInfo: { | 20260506 19:40:44.486 | INFO | "gpu" : " 17 GB", | 20260506 19:40:44.487 | INFO | "huge" : "", | 20260506 19:40:44.487 | INFO | "huge_shared" : " 27 GB", | 20260506 19:40:44.487 | INFO | "malloc" : " 14 GB", | 20260506 19:40:44.487 | INFO | "pinned" : " 29 GB" | 20260506 19:40:44.487 | INFO | } | 20260506 19:40:44.487 | INFO | Perf report for the last 5.13921 seconds: | 20260506 19:40:44.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:40:44.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:44.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:44.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:44.487 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 19:40:44.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:44.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:44.487 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 19:40:44.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:44.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:44.487 | INFO | Uploaded 36GiB to GPU in 2.0406s (17.6419GiB/s) | 20260506 19:40:44.487 | INFO | Downloaded 34GiB to GPU in 1.98438s (17.1338GiB/s) | 20260506 19:40:44.487 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:40:44.487 | INFO | Total mem usage: 18 GiB | 20260506 19:40:44.487 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:40:44.487 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -10.9%) | 20260506 19:40:44.487 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:40:44.487 | INFO | High Water Breakdown: | 20260506 19:40:44.487 | INFO | 62.1% : 10.1 GiB ( 88.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:40:44.487 | INFO | 37.8% : 6.14 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:40:44.487 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:40:44.487 | INFO | ------------------------------------------ | 20260506 19:40:44.487 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:40:44.487 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:40:44.487 | INFO | Total mem usage: 26.5 GiB | 20260506 19:40:44.487 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:40:44.487 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:40:44.487 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:40:44.487 | INFO | High Water Breakdown: | 20260506 19:40:44.487 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:40:44.487 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:40:44.487 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:40:44.487 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:40:44.487 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:40:44.487 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:40:44.487 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:40:44.487 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:40:44.487 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:40:44.487 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:40:44.487 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:40:44.487 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:40:44.487 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:40:44.487 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:40:44.488 | INFO | ------------------------------------------ | 20260506 19:40:44.488 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:40:44.488 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:40:44.488 | INFO | Total mem usage: 25.6 GiB | 20260506 19:40:44.488 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:40:44.488 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:40:44.488 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:40:44.488 | INFO | High Water Breakdown: | 20260506 19:40:44.488 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:40:44.488 | INFO | ------------------------------------------ | 20260506 19:40:44.488 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:40:44.488 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:40:44.488 | INFO | Total mem usage: 15.7 GiB | 20260506 19:40:44.488 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:40:44.488 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:40:44.488 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 19:40:44.488 | INFO | High Water Breakdown: | 20260506 19:40:44.488 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:40:44.488 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:40:44.488 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:40:44.488 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:40:44.488 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:40:44.488 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:40:44.488 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:40:44.488 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:40:44.488 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:40:44.488 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:40:44.488 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:40:44.488 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:40:44.488 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:40:44.488 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:40:44.488 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:40:44.488 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:40:44.488 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:40:44.488 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:40:44.488 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:40:44.488 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:40:44.488 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:40:44.488 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:40:44.488 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:40:44.488 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:40:44.488 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:40:44.488 | INFO | ------------------------------------------ | 20260506 19:40:44.488 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:40:44.488 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:40:44.488 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:40:44.488 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:40:44.488 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.56e+01, 1.33e+01 / 2.42e+00 | 20260506 19:40:44.488 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 1.07e-02 / 6.24e-03 | 20260506 19:40:44.488 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.41e+00, 5.55e-01 / 3.34e-01 | 20260506 19:40:44.488 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.82e-01 / 1.43e-01 | 20260506 19:40:44.488 | INFO | PerfReport: 16.02%, DME -- 4.00e+01 / 4.48e+01, 4.23e+00 / 2.59e+00 | 20260506 19:40:44.488 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.43e+00 / 1.66e+00 | 20260506 19:40:44.488 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.24e+00 / 2.76e+00, 3.84e-01 / 2.57e-01 | 20260506 19:40:44.488 | INFO | PerfReport: 3.35%, Metrics -- 1.05e+00 / 1.36e+00, 2.74e-01 / 1.92e-01 | 20260506 19:40:44.488 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.60e-01 / 4.00e-01, 5.61e-02 / 2.33e-02 | 20260506 19:40:44.488 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:40:44.488 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:40:44.488 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:40:44.488 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 19:40:44.488 | INFO | PerfReport: 21.45%, Upload -- 2.08e+01 / 0.00e+00, 6.89e+00 / 0.00e+00 | 20260506 19:40:44.488 | INFO | PerfReport: 21.20%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 19:40:44.488 | INFO | PerfReport: 32.22%, Compute -- 3.12e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 19:40:44.488 | INFO | PerfReport: 1.34%, LockUpload -- 1.30e+00 / 0.00e+00, 5.26e+00 / 0.00e+00 | 20260506 19:40:44.488 | INFO | PerfReport: 1.21%, LockDownload -- 1.17e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260506 19:40:44.488 | INFO | PerfReport: 22.57%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 19:40:44.488 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.63e-02 / 0.00e+00 | 20260506 19:40:44.600 | INFO | Starting a round of Baz IO | 20260506 19:40:47.739 | INFO | Populating RTMetrics report | 20260506 19:40:47.739 | INFO | RTMetrics report populated | 20260506 19:40:47.739 | INFO | Opening tmp RTMetrics json file | 20260506 19:40:47.968 | INFO | Serializing RTMetrics to json | 20260506 19:40:48.022 | INFO | Serialization done, swapping tmp file now... | 20260506 19:40:48.023 | INFO | RTMetrics done | 20260506 19:40:48.291 | INFO | Finished a round of Baz IO: Wrote 2.001 GiB in 3.69 seconds | 20260506 19:40:48.291 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.53% / 0.00% / 65.47% | 20260506 19:40:48.291 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.874317% / 3.97364e-06% / 5.16574e-05% | 20260506 19:40:48.291 | INFO | Baz IO overall: | 20260506 19:40:48.291 | INFO | BytesWritten : 2082082688910 | 20260506 19:40:48.291 | INFO | NumEvents : 1006344121151 | 20260506 19:40:49.627 | INFO | Analyzing chunk frames = [3940352,3940864) | 20260506 19:40:49.627 | INFO | MemInfo: { | 20260506 19:40:49.627 | INFO | "gpu" : " 17 GB", | 20260506 19:40:49.627 | INFO | "huge" : "", | 20260506 19:40:49.627 | INFO | "huge_shared" : " 27 GB", | 20260506 19:40:49.627 | INFO | "malloc" : " 14 GB", | 20260506 19:40:49.627 | INFO | "pinned" : " 29 GB" | 20260506 19:40:49.627 | INFO | } | 20260506 19:40:49.627 | INFO | Perf report for the last 5.14015 seconds: | 20260506 19:40:49.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:40:49.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:49.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:49.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:49.627 | INFO | ANALYSIS : 62.7%, 2.9, 95.9ms, 0 | 20260506 19:40:49.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:49.627 | INFO | RT_METRICS : 48.5%, 1.0, 25.4ms, 0 | 20260506 19:40:49.627 | INFO | PRE_HQ : 59.9%, 1.8, 57.4ms, 0 | 20260506 19:40:49.627 | INFO | BAZWRITER : 71.4%, 4.8, 175.6ms, 0 | 20260506 19:40:49.627 | INFO | BAZ_SAMPLING : 4.1%, 1.0, 2.1ms, 0 | 20260506 19:40:49.627 | INFO | Uploaded 36GiB to GPU in 2.03632s (17.6789GiB/s) | 20260506 19:40:49.627 | INFO | Downloaded 38GiB to GPU in 2.21002s (17.1944GiB/s) | 20260506 19:40:54.765 | INFO | Analyzing chunk frames = [3940864,3941376) | 20260506 19:40:54.765 | INFO | MemInfo: { | 20260506 19:40:54.765 | INFO | "gpu" : " 17 GB", | 20260506 19:40:54.765 | INFO | "huge" : "", | 20260506 19:40:54.765 | INFO | "huge_shared" : " 27 GB", | 20260506 19:40:54.766 | INFO | "malloc" : " 14 GB", | 20260506 19:40:54.766 | INFO | "pinned" : " 29 GB" | 20260506 19:40:54.766 | INFO | } | 20260506 19:40:54.766 | INFO | Perf report for the last 5.13878 seconds: | 20260506 19:40:54.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:40:54.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:54.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:54.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:54.766 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:40:54.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:54.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:54.766 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:40:54.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:54.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:54.766 | INFO | Uploaded 36GiB to GPU in 2.00139s (17.9875GiB/s) | 20260506 19:40:54.766 | INFO | Downloaded 34GiB to GPU in 1.92703s (17.6437GiB/s) | 20260506 19:40:59.913 | INFO | Analyzing chunk frames = [3941376,3941888) | 20260506 19:40:59.913 | INFO | MemInfo: { | 20260506 19:40:59.913 | INFO | "gpu" : " 17 GB", | 20260506 19:40:59.913 | INFO | "huge" : "", | 20260506 19:40:59.913 | INFO | "huge_shared" : " 27 GB", | 20260506 19:40:59.913 | INFO | "malloc" : " 14 GB", | 20260506 19:40:59.913 | INFO | "pinned" : " 29 GB" | 20260506 19:40:59.913 | INFO | } | 20260506 19:40:59.913 | INFO | Perf report for the last 5.14773 seconds: | 20260506 19:40:59.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:40:59.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:59.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:59.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:59.913 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 19:40:59.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:59.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:59.913 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:40:59.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:59.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:40:59.914 | INFO | Uploaded 36GiB to GPU in 1.97119s (18.2631GiB/s) | 20260506 19:40:59.914 | INFO | Downloaded 34GiB to GPU in 1.91896s (17.7179GiB/s) | 20260506 19:41:05.051 | INFO | Analyzing chunk frames = [3941888,3942400) | 20260506 19:41:05.051 | INFO | MemInfo: { | 20260506 19:41:05.051 | INFO | "gpu" : " 17 GB", | 20260506 19:41:05.051 | INFO | "huge" : "", | 20260506 19:41:05.051 | INFO | "huge_shared" : " 27 GB", | 20260506 19:41:05.051 | INFO | "malloc" : " 14 GB", | 20260506 19:41:05.052 | INFO | "pinned" : " 29 GB" | 20260506 19:41:05.052 | INFO | } | 20260506 19:41:05.052 | INFO | Perf report for the last 5.13822 seconds: | 20260506 19:41:05.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:41:05.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:05.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:05.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:05.052 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 19:41:05.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:05.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:05.052 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:41:05.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:05.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:05.052 | INFO | Uploaded 36GiB to GPU in 1.93361s (18.6181GiB/s) | 20260506 19:41:05.052 | INFO | Downloaded 34GiB to GPU in 1.87295s (18.1532GiB/s) | 20260506 19:41:10.189 | INFO | Analyzing chunk frames = [3942400,3942912) | 20260506 19:41:10.189 | INFO | MemInfo: { | 20260506 19:41:10.189 | INFO | "gpu" : " 17 GB", | 20260506 19:41:10.189 | INFO | "huge" : "", | 20260506 19:41:10.189 | INFO | "huge_shared" : " 27 GB", | 20260506 19:41:10.189 | INFO | "malloc" : " 14 GB", | 20260506 19:41:10.189 | INFO | "pinned" : " 29 GB" | 20260506 19:41:10.189 | INFO | } | 20260506 19:41:10.189 | INFO | Perf report for the last 5.13788 seconds: | 20260506 19:41:10.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:41:10.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:10.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:10.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:10.190 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 19:41:10.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:10.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:10.190 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:41:10.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:10.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:10.190 | INFO | Uploaded 36GiB to GPU in 2.01202s (17.8925GiB/s) | 20260506 19:41:10.190 | INFO | Downloaded 34GiB to GPU in 1.9154s (17.7509GiB/s) | 20260506 19:41:15.324 | INFO | Analyzing chunk frames = [3942912,3943424) | 20260506 19:41:15.324 | INFO | MemInfo: { | 20260506 19:41:15.324 | INFO | "gpu" : " 17 GB", | 20260506 19:41:15.324 | INFO | "huge" : "", | 20260506 19:41:15.324 | INFO | "huge_shared" : " 27 GB", | 20260506 19:41:15.324 | INFO | "malloc" : " 14 GB", | 20260506 19:41:15.324 | INFO | "pinned" : " 29 GB" | 20260506 19:41:15.324 | INFO | } | 20260506 19:41:15.324 | INFO | Perf report for the last 5.13487 seconds: | 20260506 19:41:15.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:41:15.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:15.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:15.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:15.324 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 19:41:15.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:15.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:15.324 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 19:41:15.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:15.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:15.325 | INFO | Uploaded 36GiB to GPU in 1.99657s (18.0309GiB/s) | 20260506 19:41:15.325 | INFO | Downloaded 34GiB to GPU in 1.96215s (17.328GiB/s) | 20260506 19:41:20.462 | INFO | Analyzing chunk frames = [3943424,3943936) | 20260506 19:41:20.462 | INFO | MemInfo: { | 20260506 19:41:20.462 | INFO | "gpu" : " 17 GB", | 20260506 19:41:20.462 | INFO | "huge" : "", | 20260506 19:41:20.462 | INFO | "huge_shared" : " 27 GB", | 20260506 19:41:20.462 | INFO | "malloc" : " 14 GB", | 20260506 19:41:20.462 | INFO | "pinned" : " 29 GB" | 20260506 19:41:20.462 | INFO | } | 20260506 19:41:20.462 | INFO | Perf report for the last 5.1377 seconds: | 20260506 19:41:20.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:41:20.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:20.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:20.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:20.462 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 19:41:20.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:20.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:20.462 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 19:41:20.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:20.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:20.462 | INFO | Uploaded 36GiB to GPU in 2.00989s (17.9114GiB/s) | 20260506 19:41:20.462 | INFO | Downloaded 34GiB to GPU in 1.98461s (17.1318GiB/s) | 20260506 19:41:25.603 | INFO | Analyzing chunk frames = [3943936,3944448) | 20260506 19:41:25.603 | INFO | MemInfo: { | 20260506 19:41:25.603 | INFO | "gpu" : " 17 GB", | 20260506 19:41:25.603 | INFO | "huge" : "", | 20260506 19:41:25.603 | INFO | "huge_shared" : " 27 GB", | 20260506 19:41:25.603 | INFO | "malloc" : " 14 GB", | 20260506 19:41:25.603 | INFO | "pinned" : " 29 GB" | 20260506 19:41:25.603 | INFO | } | 20260506 19:41:25.603 | INFO | Perf report for the last 5.14081 seconds: | 20260506 19:41:25.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:41:25.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:25.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:25.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:25.603 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 19:41:25.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:25.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:25.603 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:41:25.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:25.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:25.603 | INFO | Uploaded 36GiB to GPU in 2.03226s (17.7142GiB/s) | 20260506 19:41:25.603 | INFO | Downloaded 34GiB to GPU in 1.9772s (17.196GiB/s) | 20260506 19:41:25.718 | INFO | Starting a round of Baz IO | 20260506 19:41:28.860 | INFO | Populating RTMetrics report | 20260506 19:41:28.860 | INFO | RTMetrics report populated | 20260506 19:41:28.860 | INFO | Opening tmp RTMetrics json file | 20260506 19:41:28.873 | INFO | Serializing RTMetrics to json | 20260506 19:41:28.895 | INFO | Serialization done, swapping tmp file now... | 20260506 19:41:28.895 | INFO | RTMetrics done | 20260506 19:41:28.986 | INFO | Finished a round of Baz IO: Wrote 2.001 GiB in 3.268 seconds | 20260506 19:41:28.986 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.52% / 0.00% / 65.47% | 20260506 19:41:28.986 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.872175% / 1.19209e-05% / 6.75519e-05% | 20260506 19:41:28.986 | INFO | Baz IO overall: | 20260506 19:41:28.986 | INFO | BytesWritten : 2084231120522 | 20260506 19:41:28.986 | INFO | NumEvents : 1007376251525 | 20260506 19:41:30.746 | INFO | Analyzing chunk frames = [3944448,3944960) | 20260506 19:41:30.746 | INFO | MemInfo: { | 20260506 19:41:30.746 | INFO | "gpu" : " 17 GB", | 20260506 19:41:30.746 | INFO | "huge" : "", | 20260506 19:41:30.746 | INFO | "huge_shared" : " 27 GB", | 20260506 19:41:30.746 | INFO | "malloc" : " 14 GB", | 20260506 19:41:30.746 | INFO | "pinned" : " 29 GB" | 20260506 19:41:30.746 | INFO | } | 20260506 19:41:30.746 | INFO | Perf report for the last 5.14318 seconds: | 20260506 19:41:30.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:41:30.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:30.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:30.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:30.746 | INFO | ANALYSIS : 62.6%, 2.9, 95.6ms, 0 | 20260506 19:41:30.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:30.746 | INFO | RT_METRICS : 43.9%, 1.0, 23.1ms, 0 | 20260506 19:41:30.746 | INFO | PRE_HQ : 60.1%, 1.8, 57.6ms, 0 | 20260506 19:41:30.746 | INFO | BAZWRITER : 63.3%, 4.5, 147.3ms, 0 | 20260506 19:41:30.746 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 19:41:30.746 | INFO | Uploaded 36GiB to GPU in 2.05602s (17.5096GiB/s) | 20260506 19:41:30.746 | INFO | Downloaded 38GiB to GPU in 2.21668s (17.1428GiB/s) | 20260506 19:41:35.886 | INFO | Analyzing chunk frames = [3944960,3945472) | 20260506 19:41:35.886 | INFO | MemInfo: { | 20260506 19:41:35.886 | INFO | "gpu" : " 17 GB", | 20260506 19:41:35.886 | INFO | "huge" : "", | 20260506 19:41:35.886 | INFO | "huge_shared" : " 27 GB", | 20260506 19:41:35.887 | INFO | "malloc" : " 14 GB", | 20260506 19:41:35.887 | INFO | "pinned" : " 29 GB" | 20260506 19:41:35.887 | INFO | } | 20260506 19:41:35.887 | INFO | Perf report for the last 5.14052 seconds: | 20260506 19:41:35.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:41:35.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:35.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:35.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:35.887 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:41:35.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:35.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:35.887 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:41:35.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:35.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:35.887 | INFO | Uploaded 36GiB to GPU in 2.01819s (17.8377GiB/s) | 20260506 19:41:35.887 | INFO | Downloaded 34GiB to GPU in 1.92562s (17.6567GiB/s) | 20260506 19:41:41.021 | INFO | Analyzing chunk frames = [3945472,3945984) | 20260506 19:41:41.021 | INFO | MemInfo: { | 20260506 19:41:41.021 | INFO | "gpu" : " 17 GB", | 20260506 19:41:41.021 | INFO | "huge" : "", | 20260506 19:41:41.021 | INFO | "huge_shared" : " 27 GB", | 20260506 19:41:41.021 | INFO | "malloc" : " 14 GB", | 20260506 19:41:41.021 | INFO | "pinned" : " 29 GB" | 20260506 19:41:41.021 | INFO | } | 20260506 19:41:41.022 | INFO | Perf report for the last 5.1349 seconds: | 20260506 19:41:41.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:41:41.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:41.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:41.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:41.022 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 19:41:41.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:41.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:41.022 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:41:41.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:41.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:41.022 | INFO | Uploaded 36GiB to GPU in 1.98449s (18.1407GiB/s) | 20260506 19:41:41.022 | INFO | Downloaded 34GiB to GPU in 1.91633s (17.7423GiB/s) | 20260506 19:41:46.162 | INFO | Analyzing chunk frames = [3945984,3946496) | 20260506 19:41:46.162 | INFO | MemInfo: { | 20260506 19:41:46.162 | INFO | "gpu" : " 17 GB", | 20260506 19:41:46.162 | INFO | "huge" : "", | 20260506 19:41:46.162 | INFO | "huge_shared" : " 27 GB", | 20260506 19:41:46.162 | INFO | "malloc" : " 14 GB", | 20260506 19:41:46.162 | INFO | "pinned" : " 29 GB" | 20260506 19:41:46.162 | INFO | } | 20260506 19:41:46.162 | INFO | Perf report for the last 5.14039 seconds: | 20260506 19:41:46.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:41:46.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:46.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:46.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:46.162 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 19:41:46.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:46.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:46.162 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:41:46.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:46.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:46.162 | INFO | Uploaded 36GiB to GPU in 1.93601s (18.5949GiB/s) | 20260506 19:41:46.162 | INFO | Downloaded 34GiB to GPU in 1.87239s (18.1586GiB/s) | 20260506 19:41:51.302 | INFO | Analyzing chunk frames = [3946496,3947008) | 20260506 19:41:51.302 | INFO | MemInfo: { | 20260506 19:41:51.302 | INFO | "gpu" : " 17 GB", | 20260506 19:41:51.302 | INFO | "huge" : "", | 20260506 19:41:51.302 | INFO | "huge_shared" : " 27 GB", | 20260506 19:41:51.302 | INFO | "malloc" : " 14 GB", | 20260506 19:41:51.302 | INFO | "pinned" : " 29 GB" | 20260506 19:41:51.302 | INFO | } | 20260506 19:41:51.302 | INFO | Perf report for the last 5.14024 seconds: | 20260506 19:41:51.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:41:51.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:51.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:51.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:51.302 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 19:41:51.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:51.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:51.302 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:41:51.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:51.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:51.303 | INFO | Uploaded 36GiB to GPU in 2.02291s (17.7962GiB/s) | 20260506 19:41:51.303 | INFO | Downloaded 34GiB to GPU in 1.91253s (17.7775GiB/s) | 20260506 19:41:56.439 | INFO | Analyzing chunk frames = [3947008,3947520) | 20260506 19:41:56.439 | INFO | MemInfo: { | 20260506 19:41:56.439 | INFO | "gpu" : " 17 GB", | 20260506 19:41:56.439 | INFO | "huge" : "", | 20260506 19:41:56.439 | INFO | "huge_shared" : " 27 GB", | 20260506 19:41:56.439 | INFO | "malloc" : " 14 GB", | 20260506 19:41:56.439 | INFO | "pinned" : " 29 GB" | 20260506 19:41:56.439 | INFO | } | 20260506 19:41:56.439 | INFO | Perf report for the last 5.13672 seconds: | 20260506 19:41:56.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:41:56.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:56.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:56.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:56.439 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 19:41:56.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:56.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:56.439 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 19:41:56.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:56.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:41:56.439 | INFO | Uploaded 36GiB to GPU in 2.07396s (17.3581GiB/s) | 20260506 19:41:56.439 | INFO | Downloaded 34GiB to GPU in 1.96553s (17.2981GiB/s) | 20260506 19:42:01.582 | INFO | Analyzing chunk frames = [3947520,3948032) | 20260506 19:42:01.582 | INFO | MemInfo: { | 20260506 19:42:01.582 | INFO | "gpu" : " 17 GB", | 20260506 19:42:01.582 | INFO | "huge" : "", | 20260506 19:42:01.582 | INFO | "huge_shared" : " 27 GB", | 20260506 19:42:01.582 | INFO | "malloc" : " 14 GB", | 20260506 19:42:01.582 | INFO | "pinned" : " 29 GB" | 20260506 19:42:01.582 | INFO | } | 20260506 19:42:01.582 | INFO | Perf report for the last 5.14328 seconds: | 20260506 19:42:01.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:42:01.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:01.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:01.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:01.582 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 19:42:01.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:01.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:01.582 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 19:42:01.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:01.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:01.582 | INFO | Uploaded 36GiB to GPU in 2.02022s (17.8198GiB/s) | 20260506 19:42:01.582 | INFO | Downloaded 34GiB to GPU in 1.99758s (17.0206GiB/s) | 20260506 19:42:06.717 | INFO | Analyzing chunk frames = [3948032,3948544) | 20260506 19:42:06.717 | INFO | MemInfo: { | 20260506 19:42:06.717 | INFO | "gpu" : " 17 GB", | 20260506 19:42:06.717 | INFO | "huge" : "", | 20260506 19:42:06.717 | INFO | "huge_shared" : " 27 GB", | 20260506 19:42:06.717 | INFO | "malloc" : " 14 GB", | 20260506 19:42:06.717 | INFO | "pinned" : " 29 GB" | 20260506 19:42:06.717 | INFO | } | 20260506 19:42:06.717 | INFO | Perf report for the last 5.13506 seconds: | 20260506 19:42:06.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:42:06.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:06.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:06.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:06.717 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:42:06.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:06.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:06.717 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 19:42:06.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:06.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:06.718 | INFO | Uploaded 36GiB to GPU in 1.96283s (18.3409GiB/s) | 20260506 19:42:06.718 | INFO | Downloaded 34GiB to GPU in 1.98264s (17.1489GiB/s) | 20260506 19:42:06.718 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:42:06.718 | INFO | Total mem usage: 18 GiB | 20260506 19:42:06.718 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:42:06.718 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -10.9%) | 20260506 19:42:06.718 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:42:06.718 | INFO | High Water Breakdown: | 20260506 19:42:06.718 | INFO | 62% : 10.1 GiB ( 88.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:42:06.718 | INFO | 37.8% : 6.14 GiB ( 91.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:42:06.718 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:42:06.718 | INFO | ------------------------------------------ | 20260506 19:42:06.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:42:06.718 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:42:06.718 | INFO | Total mem usage: 26.5 GiB | 20260506 19:42:06.718 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:42:06.718 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:42:06.718 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:42:06.718 | INFO | High Water Breakdown: | 20260506 19:42:06.718 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:42:06.718 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:42:06.718 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:42:06.718 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:42:06.718 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:42:06.718 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:42:06.718 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:42:06.718 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:42:06.718 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:42:06.718 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:42:06.718 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:42:06.718 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:42:06.718 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:42:06.718 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:42:06.718 | INFO | ------------------------------------------ | 20260506 19:42:06.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:42:06.718 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:42:06.718 | INFO | Total mem usage: 25.6 GiB | 20260506 19:42:06.718 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:42:06.718 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:42:06.718 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:42:06.718 | INFO | High Water Breakdown: | 20260506 19:42:06.718 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:42:06.718 | INFO | ------------------------------------------ | 20260506 19:42:06.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:42:06.718 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:42:06.718 | INFO | Total mem usage: 15.7 GiB | 20260506 19:42:06.718 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:42:06.718 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:42:06.718 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 19:42:06.718 | INFO | High Water Breakdown: | 20260506 19:42:06.718 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:42:06.718 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:42:06.718 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:42:06.718 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:42:06.718 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:42:06.718 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:42:06.718 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:42:06.718 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:42:06.718 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:42:06.718 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:42:06.718 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:42:06.718 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:42:06.718 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:42:06.718 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:42:06.718 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:42:06.718 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:42:06.718 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:42:06.718 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:42:06.718 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:42:06.718 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:42:06.718 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:42:06.718 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:42:06.718 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:42:06.719 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:42:06.719 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:42:06.719 | INFO | ------------------------------------------ | 20260506 19:42:06.719 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:42:06.719 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:42:06.719 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:42:06.719 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:42:06.719 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.55e+01, 1.35e+01 / 2.45e+00 | 20260506 19:42:06.719 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 9.45e-03 / 5.02e-03 | 20260506 19:42:06.719 | INFO | PerfReport: 11.63%, Baseline -- 3.63e+00 / 4.42e+00, 5.52e-01 / 3.52e-01 | 20260506 19:42:06.719 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.49e+00, 1.85e-01 / 1.24e-01 | 20260506 19:42:06.719 | INFO | PerfReport: 15.98%, DME -- 3.99e+01 / 4.43e+01, 3.96e+00 / 2.46e+00 | 20260506 19:42:06.719 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.47e+00 / 2.38e+00 | 20260506 19:42:06.719 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.23e+00 / 2.80e+00, 3.73e-01 / 2.38e-01 | 20260506 19:42:06.719 | INFO | PerfReport: 3.34%, Metrics -- 1.04e+00 / 1.36e+00, 2.75e-01 / 1.76e-01 | 20260506 19:42:06.719 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.59e-01 / 4.00e-01, 5.58e-02 / 2.47e-02 | 20260506 19:42:06.719 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:42:06.719 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:42:06.719 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:42:06.719 | INFO | PerfReport: Full Execution -- 9.71e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 19:42:06.719 | INFO | PerfReport: 21.51%, Upload -- 2.09e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260506 19:42:06.719 | INFO | PerfReport: 21.17%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 19:42:06.719 | INFO | PerfReport: 32.18%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 19:42:06.719 | INFO | PerfReport: 1.43%, LockUpload -- 1.39e+00 / 0.00e+00, 5.46e+00 / 0.00e+00 | 20260506 19:42:06.719 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260506 19:42:06.719 | INFO | PerfReport: 22.59%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 19:42:06.719 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260506 19:42:06.825 | INFO | Starting a round of Baz IO | 20260506 19:42:09.963 | INFO | Populating RTMetrics report | 20260506 19:42:09.964 | INFO | RTMetrics report populated | 20260506 19:42:09.964 | INFO | Opening tmp RTMetrics json file | 20260506 19:42:09.968 | INFO | Serializing RTMetrics to json | 20260506 19:42:09.996 | INFO | Serialization done, swapping tmp file now... | 20260506 19:42:10.005 | INFO | RTMetrics done | 20260506 19:42:10.116 | INFO | Finished a round of Baz IO: Wrote 2.001 GiB in 3.291 seconds | 20260506 19:42:10.116 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.52% / 0.00% / 65.48% | 20260506 19:42:10.116 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.871952% / 1.19209e-05% / 7.54992e-05% | 20260506 19:42:10.116 | INFO | Baz IO overall: | 20260506 19:42:10.116 | INFO | BytesWritten : 2086380097940 | 20260506 19:42:10.116 | INFO | NumEvents : 1008408914646 | 20260506 19:42:11.860 | INFO | Analyzing chunk frames = [3948544,3949056) | 20260506 19:42:11.860 | INFO | MemInfo: { | 20260506 19:42:11.860 | INFO | "gpu" : " 17 GB", | 20260506 19:42:11.860 | INFO | "huge" : "", | 20260506 19:42:11.860 | INFO | "huge_shared" : " 27 GB", | 20260506 19:42:11.860 | INFO | "malloc" : " 14 GB", | 20260506 19:42:11.860 | INFO | "pinned" : " 29 GB" | 20260506 19:42:11.860 | INFO | } | 20260506 19:42:11.860 | INFO | Perf report for the last 5.1424 seconds: | 20260506 19:42:11.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:42:11.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:11.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:11.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:11.860 | INFO | ANALYSIS : 62.6%, 2.9, 95.1ms, 0 | 20260506 19:42:11.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:11.860 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 19:42:11.860 | INFO | PRE_HQ : 60.1%, 1.9, 57.9ms, 0 | 20260506 19:42:11.860 | INFO | BAZWRITER : 63.7%, 4.8, 156.5ms, 0 | 20260506 19:42:11.860 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 19:42:11.860 | INFO | Uploaded 36GiB to GPU in 2.04134s (17.6355GiB/s) | 20260506 19:42:11.860 | INFO | Downloaded 38GiB to GPU in 2.20395s (17.2418GiB/s) | 20260506 19:42:17.004 | INFO | Analyzing chunk frames = [3949056,3949568) | 20260506 19:42:17.004 | INFO | MemInfo: { | 20260506 19:42:17.004 | INFO | "gpu" : " 17 GB", | 20260506 19:42:17.004 | INFO | "huge" : "", | 20260506 19:42:17.004 | INFO | "huge_shared" : " 27 GB", | 20260506 19:42:17.004 | INFO | "malloc" : " 14 GB", | 20260506 19:42:17.004 | INFO | "pinned" : " 29 GB" | 20260506 19:42:17.004 | INFO | } | 20260506 19:42:17.004 | INFO | Perf report for the last 5.1442 seconds: | 20260506 19:42:17.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:42:17.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:17.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:17.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:17.004 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 19:42:17.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:17.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:17.004 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:42:17.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:17.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:17.004 | INFO | Uploaded 36GiB to GPU in 2.00406s (17.9635GiB/s) | 20260506 19:42:17.004 | INFO | Downloaded 34GiB to GPU in 1.93064s (17.6108GiB/s) | 20260506 19:42:22.139 | INFO | Analyzing chunk frames = [3949568,3950080) | 20260506 19:42:22.139 | INFO | MemInfo: { | 20260506 19:42:22.139 | INFO | "gpu" : " 17 GB", | 20260506 19:42:22.139 | INFO | "huge" : "", | 20260506 19:42:22.139 | INFO | "huge_shared" : " 27 GB", | 20260506 19:42:22.139 | INFO | "malloc" : " 14 GB", | 20260506 19:42:22.139 | INFO | "pinned" : " 29 GB" | 20260506 19:42:22.139 | INFO | } | 20260506 19:42:22.139 | INFO | Perf report for the last 5.13503 seconds: | 20260506 19:42:22.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:42:22.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:22.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:22.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:22.139 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260506 19:42:22.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:22.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:22.139 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:42:22.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:22.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:22.139 | INFO | Uploaded 36GiB to GPU in 1.97247s (18.2512GiB/s) | 20260506 19:42:22.139 | INFO | Downloaded 34GiB to GPU in 1.91674s (17.7385GiB/s) | 20260506 19:42:27.276 | INFO | Analyzing chunk frames = [3950080,3950592) | 20260506 19:42:27.276 | INFO | MemInfo: { | 20260506 19:42:27.276 | INFO | "gpu" : " 17 GB", | 20260506 19:42:27.276 | INFO | "huge" : "", | 20260506 19:42:27.276 | INFO | "huge_shared" : " 27 GB", | 20260506 19:42:27.276 | INFO | "malloc" : " 14 GB", | 20260506 19:42:27.276 | INFO | "pinned" : " 29 GB" | 20260506 19:42:27.276 | INFO | } | 20260506 19:42:27.277 | INFO | Perf report for the last 5.13757 seconds: | 20260506 19:42:27.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:42:27.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:27.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:27.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:27.277 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 19:42:27.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:27.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:27.277 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:42:27.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:27.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:27.277 | INFO | Uploaded 36GiB to GPU in 1.93145s (18.6389GiB/s) | 20260506 19:42:27.277 | INFO | Downloaded 34GiB to GPU in 1.87272s (18.1554GiB/s) | 20260506 19:42:32.413 | INFO | Analyzing chunk frames = [3950592,3951104) | 20260506 19:42:32.413 | INFO | MemInfo: { | 20260506 19:42:32.413 | INFO | "gpu" : " 17 GB", | 20260506 19:42:32.413 | INFO | "huge" : "", | 20260506 19:42:32.413 | INFO | "huge_shared" : " 27 GB", | 20260506 19:42:32.413 | INFO | "malloc" : " 14 GB", | 20260506 19:42:32.413 | INFO | "pinned" : " 29 GB" | 20260506 19:42:32.413 | INFO | } | 20260506 19:42:32.413 | INFO | Perf report for the last 5.13641 seconds: | 20260506 19:42:32.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:42:32.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:32.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:32.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:32.413 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 19:42:32.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:32.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:32.413 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:42:32.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:32.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:32.413 | INFO | Uploaded 36GiB to GPU in 2.01375s (17.8771GiB/s) | 20260506 19:42:32.413 | INFO | Downloaded 34GiB to GPU in 1.91316s (17.7717GiB/s) | 20260506 19:42:37.551 | INFO | Analyzing chunk frames = [3951104,3951616) | 20260506 19:42:37.551 | INFO | MemInfo: { | 20260506 19:42:37.551 | INFO | "gpu" : " 17 GB", | 20260506 19:42:37.551 | INFO | "huge" : "", | 20260506 19:42:37.551 | INFO | "huge_shared" : " 27 GB", | 20260506 19:42:37.551 | INFO | "malloc" : " 14 GB", | 20260506 19:42:37.551 | INFO | "pinned" : " 29 GB" | 20260506 19:42:37.551 | INFO | } | 20260506 19:42:37.552 | INFO | Perf report for the last 5.13857 seconds: | 20260506 19:42:37.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:42:37.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:37.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:37.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:37.552 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 19:42:37.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:37.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:37.552 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 19:42:37.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:37.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:37.552 | INFO | Uploaded 36GiB to GPU in 2.00581s (17.9479GiB/s) | 20260506 19:42:37.552 | INFO | Downloaded 34GiB to GPU in 1.95095s (17.4274GiB/s) | 20260506 19:42:42.699 | INFO | Analyzing chunk frames = [3951616,3952128) | 20260506 19:42:42.699 | INFO | MemInfo: { | 20260506 19:42:42.699 | INFO | "gpu" : " 17 GB", | 20260506 19:42:42.699 | INFO | "huge" : "", | 20260506 19:42:42.699 | INFO | "huge_shared" : " 27 GB", | 20260506 19:42:42.699 | INFO | "malloc" : " 14 GB", | 20260506 19:42:42.699 | INFO | "pinned" : " 29 GB" | 20260506 19:42:42.699 | INFO | } | 20260506 19:42:42.699 | INFO | Perf report for the last 5.14728 seconds: | 20260506 19:42:42.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:42:42.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:42.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:42.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:42.699 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 19:42:42.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:42.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:42.699 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 19:42:42.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:42.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:42.699 | INFO | Uploaded 36GiB to GPU in 1.95277s (18.4353GiB/s) | 20260506 19:42:42.699 | INFO | Downloaded 34GiB to GPU in 1.98812s (17.1016GiB/s) | 20260506 19:42:47.832 | INFO | Analyzing chunk frames = [3952128,3952640) | 20260506 19:42:47.832 | INFO | MemInfo: { | 20260506 19:42:47.832 | INFO | "gpu" : " 17 GB", | 20260506 19:42:47.832 | INFO | "huge" : "", | 20260506 19:42:47.832 | INFO | "huge_shared" : " 27 GB", | 20260506 19:42:47.832 | INFO | "malloc" : " 14 GB", | 20260506 19:42:47.832 | INFO | "pinned" : " 29 GB" | 20260506 19:42:47.832 | INFO | } | 20260506 19:42:47.832 | INFO | Perf report for the last 5.13303 seconds: | 20260506 19:42:47.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:42:47.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:47.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:47.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:47.832 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 19:42:47.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:47.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:47.832 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:42:47.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:47.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:47.832 | INFO | Uploaded 36GiB to GPU in 1.97347s (18.242GiB/s) | 20260506 19:42:47.832 | INFO | Downloaded 34GiB to GPU in 1.97391s (17.2247GiB/s) | 20260506 19:42:47.939 | INFO | Starting a round of Baz IO | 20260506 19:42:51.076 | INFO | Populating RTMetrics report | 20260506 19:42:51.077 | INFO | RTMetrics report populated | 20260506 19:42:51.077 | INFO | Opening tmp RTMetrics json file | 20260506 19:42:51.078 | INFO | Serializing RTMetrics to json | 20260506 19:42:51.079 | INFO | Serialization done, swapping tmp file now... | 20260506 19:42:51.083 | INFO | RTMetrics done | 20260506 19:42:51.213 | INFO | Finished a round of Baz IO: Wrote 1.999 GiB in 3.274 seconds | 20260506 19:42:51.213 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.51% / 0.00% / 65.48% | 20260506 19:42:51.213 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.868587% / 7.94729e-06% / 6.35783e-05% | 20260506 19:42:51.213 | INFO | Baz IO overall: | 20260506 19:42:51.213 | INFO | BytesWritten : 2088526513151 | 20260506 19:42:51.213 | INFO | NumEvents : 1009439708339 | 20260506 19:42:52.974 | INFO | Analyzing chunk frames = [3952640,3953152) | 20260506 19:42:52.974 | INFO | MemInfo: { | 20260506 19:42:52.974 | INFO | "gpu" : " 17 GB", | 20260506 19:42:52.974 | INFO | "huge" : "", | 20260506 19:42:52.974 | INFO | "huge_shared" : " 27 GB", | 20260506 19:42:52.974 | INFO | "malloc" : " 14 GB", | 20260506 19:42:52.974 | INFO | "pinned" : " 29 GB" | 20260506 19:42:52.974 | INFO | } | 20260506 19:42:52.974 | INFO | Perf report for the last 5.14224 seconds: | 20260506 19:42:52.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:42:52.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:52.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:52.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:52.974 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 19:42:52.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:52.974 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 19:42:52.974 | INFO | PRE_HQ : 60.2%, 1.8, 57.4ms, 0 | 20260506 19:42:52.974 | INFO | BAZWRITER : 63.4%, 4.6, 150.8ms, 0 | 20260506 19:42:52.974 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 19:42:52.974 | INFO | Uploaded 36GiB to GPU in 2.02613s (17.7678GiB/s) | 20260506 19:42:52.974 | INFO | Downloaded 38GiB to GPU in 2.1998s (17.2743GiB/s) | 20260506 19:42:58.112 | INFO | Analyzing chunk frames = [3953152,3953664) | 20260506 19:42:58.112 | INFO | MemInfo: { | 20260506 19:42:58.112 | INFO | "gpu" : " 17 GB", | 20260506 19:42:58.112 | INFO | "huge" : "", | 20260506 19:42:58.112 | INFO | "huge_shared" : " 27 GB", | 20260506 19:42:58.112 | INFO | "malloc" : " 14 GB", | 20260506 19:42:58.112 | INFO | "pinned" : " 29 GB" | 20260506 19:42:58.112 | INFO | } | 20260506 19:42:58.112 | INFO | Perf report for the last 5.13764 seconds: | 20260506 19:42:58.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:42:58.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:58.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:58.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:58.112 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:42:58.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:58.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:58.112 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:42:58.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:58.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:42:58.112 | INFO | Uploaded 36GiB to GPU in 1.98635s (18.1237GiB/s) | 20260506 19:42:58.112 | INFO | Downloaded 34GiB to GPU in 1.92343s (17.6767GiB/s) | 20260506 19:43:03.255 | INFO | Analyzing chunk frames = [3953664,3954176) | 20260506 19:43:03.255 | INFO | MemInfo: { | 20260506 19:43:03.255 | INFO | "gpu" : " 17 GB", | 20260506 19:43:03.255 | INFO | "huge" : "", | 20260506 19:43:03.255 | INFO | "huge_shared" : " 27 GB", | 20260506 19:43:03.255 | INFO | "malloc" : " 14 GB", | 20260506 19:43:03.255 | INFO | "pinned" : " 29 GB" | 20260506 19:43:03.255 | INFO | } | 20260506 19:43:03.255 | INFO | Perf report for the last 5.14333 seconds: | 20260506 19:43:03.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:43:03.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:03.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:03.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:03.255 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:43:03.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:03.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:03.255 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:43:03.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:03.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:03.255 | INFO | Uploaded 36GiB to GPU in 1.95569s (18.4078GiB/s) | 20260506 19:43:03.255 | INFO | Downloaded 34GiB to GPU in 1.92105s (17.6987GiB/s) | 20260506 19:43:08.394 | INFO | Analyzing chunk frames = [3954176,3954688) | 20260506 19:43:08.394 | INFO | MemInfo: { | 20260506 19:43:08.394 | INFO | "gpu" : " 17 GB", | 20260506 19:43:08.394 | INFO | "huge" : "", | 20260506 19:43:08.394 | INFO | "huge_shared" : " 27 GB", | 20260506 19:43:08.394 | INFO | "malloc" : " 14 GB", | 20260506 19:43:08.394 | INFO | "pinned" : " 29 GB" | 20260506 19:43:08.394 | INFO | } | 20260506 19:43:08.394 | INFO | Perf report for the last 5.13875 seconds: | 20260506 19:43:08.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:43:08.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:08.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:08.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:08.394 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:43:08.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:08.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:08.394 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:43:08.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:08.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:08.394 | INFO | Uploaded 36GiB to GPU in 1.95622s (18.4029GiB/s) | 20260506 19:43:08.394 | INFO | Downloaded 34GiB to GPU in 1.87466s (18.1366GiB/s) | 20260506 19:43:13.530 | INFO | Analyzing chunk frames = [3954688,3955200) | 20260506 19:43:13.530 | INFO | MemInfo: { | 20260506 19:43:13.530 | INFO | "gpu" : " 17 GB", | 20260506 19:43:13.530 | INFO | "huge" : "", | 20260506 19:43:13.530 | INFO | "huge_shared" : " 27 GB", | 20260506 19:43:13.530 | INFO | "malloc" : " 14 GB", | 20260506 19:43:13.530 | INFO | "pinned" : " 29 GB" | 20260506 19:43:13.530 | INFO | } | 20260506 19:43:13.531 | INFO | Perf report for the last 5.13666 seconds: | 20260506 19:43:13.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:43:13.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:13.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:13.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:13.531 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 19:43:13.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:13.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:13.531 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:43:13.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:13.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:13.531 | INFO | Uploaded 36GiB to GPU in 1.99523s (18.0431GiB/s) | 20260506 19:43:13.531 | INFO | Downloaded 34GiB to GPU in 1.91805s (17.7263GiB/s) | 20260506 19:43:18.670 | INFO | Analyzing chunk frames = [3955200,3955712) | 20260506 19:43:18.670 | INFO | MemInfo: { | 20260506 19:43:18.670 | INFO | "gpu" : " 17 GB", | 20260506 19:43:18.670 | INFO | "huge" : "", | 20260506 19:43:18.670 | INFO | "huge_shared" : " 27 GB", | 20260506 19:43:18.670 | INFO | "malloc" : " 14 GB", | 20260506 19:43:18.670 | INFO | "pinned" : " 29 GB" | 20260506 19:43:18.670 | INFO | } | 20260506 19:43:18.670 | INFO | Perf report for the last 5.1395 seconds: | 20260506 19:43:18.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:43:18.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:18.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:18.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:18.670 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 19:43:18.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:18.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:18.670 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 19:43:18.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:18.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:18.670 | INFO | Uploaded 36GiB to GPU in 2.00574s (17.9485GiB/s) | 20260506 19:43:18.670 | INFO | Downloaded 34GiB to GPU in 1.94722s (17.4608GiB/s) | 20260506 19:43:23.810 | INFO | Analyzing chunk frames = [3955712,3956224) | 20260506 19:43:23.810 | INFO | MemInfo: { | 20260506 19:43:23.810 | INFO | "gpu" : " 17 GB", | 20260506 19:43:23.810 | INFO | "huge" : "", | 20260506 19:43:23.810 | INFO | "huge_shared" : " 27 GB", | 20260506 19:43:23.810 | INFO | "malloc" : " 14 GB", | 20260506 19:43:23.810 | INFO | "pinned" : " 29 GB" | 20260506 19:43:23.810 | INFO | } | 20260506 19:43:23.810 | INFO | Perf report for the last 5.14039 seconds: | 20260506 19:43:23.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:43:23.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:23.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:23.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:23.810 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 19:43:23.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:23.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:23.811 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:43:23.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:23.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:23.811 | INFO | Uploaded 36GiB to GPU in 1.96045s (18.3632GiB/s) | 20260506 19:43:23.811 | INFO | Downloaded 34GiB to GPU in 1.99748s (17.0215GiB/s) | 20260506 19:43:28.952 | INFO | Analyzing chunk frames = [3956224,3956736) | 20260506 19:43:28.952 | INFO | MemInfo: { | 20260506 19:43:28.952 | INFO | "gpu" : " 17 GB", | 20260506 19:43:28.952 | INFO | "huge" : "", | 20260506 19:43:28.952 | INFO | "huge_shared" : " 27 GB", | 20260506 19:43:28.952 | INFO | "malloc" : " 14 GB", | 20260506 19:43:28.952 | INFO | "pinned" : " 29 GB" | 20260506 19:43:28.952 | INFO | } | 20260506 19:43:28.952 | INFO | Perf report for the last 5.14161 seconds: | 20260506 19:43:28.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:43:28.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:28.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:28.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:28.952 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:43:28.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:28.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:28.952 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:43:28.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:28.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:28.952 | INFO | Uploaded 36GiB to GPU in 1.98457s (18.14GiB/s) | 20260506 19:43:28.952 | INFO | Downloaded 34GiB to GPU in 1.97791s (17.1898GiB/s) | 20260506 19:43:28.952 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:43:28.952 | INFO | Total mem usage: 18 GiB | 20260506 19:43:28.952 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:43:28.952 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -18.1%) | 20260506 19:43:28.952 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:43:28.952 | INFO | High Water Breakdown: | 20260506 19:43:28.952 | INFO | 61.1% : 9.31 GiB ( 81.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:43:28.952 | INFO | 38.8% : 5.91 GiB ( 88.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:43:28.952 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:43:28.952 | INFO | ------------------------------------------ | 20260506 19:43:28.952 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:43:28.952 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:43:28.952 | INFO | Total mem usage: 26.5 GiB | 20260506 19:43:28.952 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:43:28.952 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:43:28.952 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:43:28.953 | INFO | High Water Breakdown: | 20260506 19:43:28.953 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:43:28.953 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:43:28.953 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:43:28.953 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:43:28.953 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:43:28.953 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:43:28.953 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:43:28.953 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:43:28.953 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:43:28.953 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:43:28.953 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:43:28.953 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:43:28.953 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:43:28.953 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:43:28.953 | INFO | ------------------------------------------ | 20260506 19:43:28.953 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:43:28.953 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:43:28.953 | INFO | Total mem usage: 25.6 GiB | 20260506 19:43:28.953 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:43:28.953 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:43:28.953 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:43:28.953 | INFO | High Water Breakdown: | 20260506 19:43:28.953 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:43:28.953 | INFO | ------------------------------------------ | 20260506 19:43:28.953 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:43:28.953 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:43:28.953 | INFO | Total mem usage: 15.7 GiB | 20260506 19:43:28.953 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:43:28.953 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:43:28.953 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:43:28.953 | INFO | High Water Breakdown: | 20260506 19:43:28.953 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:43:28.953 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:43:28.953 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:43:28.953 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:43:28.953 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:43:28.953 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:43:28.953 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:43:28.953 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:43:28.953 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:43:28.953 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:43:28.953 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:43:28.953 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:43:28.953 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:43:28.953 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:43:28.953 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:43:28.953 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:43:28.953 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:43:28.953 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:43:28.953 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:43:28.953 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:43:28.953 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:43:28.953 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:43:28.953 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:43:28.953 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:43:28.953 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:43:28.953 | INFO | ------------------------------------------ | 20260506 19:43:28.953 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:43:28.953 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:43:28.953 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:43:28.953 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:43:28.953 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.36e+01 / 2.55e+00 | 20260506 19:43:28.953 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 9.84e-03 / 5.62e-03 | 20260506 19:43:28.953 | INFO | PerfReport: 11.63%, Baseline -- 3.63e+00 / 4.40e+00, 5.60e-01 / 3.56e-01 | 20260506 19:43:28.953 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.46e+00, 1.85e-01 / 1.36e-01 | 20260506 19:43:28.953 | INFO | PerfReport: 15.99%, DME -- 3.99e+01 / 4.44e+01, 3.96e+00 / 2.85e+00 | 20260506 19:43:28.953 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.40e+00 / 1.16e+00 | 20260506 19:43:28.953 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.24e+00 / 2.78e+00, 3.80e-01 / 2.05e-01 | 20260506 19:43:28.953 | INFO | PerfReport: 3.35%, Metrics -- 1.05e+00 / 1.38e+00, 2.77e-01 / 1.61e-01 | 20260506 19:43:28.953 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.59e-01 / 4.10e-01, 5.58e-02 / 2.52e-02 | 20260506 19:43:28.953 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:43:28.953 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:43:28.953 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:43:28.953 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 19:43:28.953 | INFO | PerfReport: 21.36%, Upload -- 2.07e+01 / 0.00e+00, 6.78e+00 / 0.00e+00 | 20260506 19:43:28.953 | INFO | PerfReport: 21.19%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 19:43:28.953 | INFO | PerfReport: 32.25%, Compute -- 3.12e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 19:43:28.953 | INFO | PerfReport: 1.25%, LockUpload -- 1.21e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260506 19:43:28.953 | INFO | PerfReport: 1.24%, LockDownload -- 1.20e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260506 19:43:28.953 | INFO | PerfReport: 22.68%, LockCompute -- 2.20e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 19:43:28.953 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260506 19:43:29.054 | INFO | Starting a round of Baz IO | 20260506 19:43:32.196 | INFO | Populating RTMetrics report | 20260506 19:43:32.196 | INFO | RTMetrics report populated | 20260506 19:43:32.196 | INFO | Opening tmp RTMetrics json file | 20260506 19:43:32.367 | INFO | Serializing RTMetrics to json | 20260506 19:43:32.390 | INFO | Serialization done, swapping tmp file now... | 20260506 19:43:32.391 | INFO | RTMetrics done | 20260506 19:43:32.495 | INFO | Finished a round of Baz IO: Wrote 2.003 GiB in 3.441 seconds | 20260506 19:43:32.496 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.51% / 0.00% / 65.49% | 20260506 19:43:32.496 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.869699% / 1.58946e-05% / 6.35783e-05% | 20260506 19:43:32.496 | INFO | Baz IO overall: | 20260506 19:43:32.496 | INFO | BytesWritten : 2090677107668 | 20260506 19:43:32.496 | INFO | NumEvents : 1010473923407 | 20260506 19:43:34.090 | INFO | Analyzing chunk frames = [3956736,3957248) | 20260506 19:43:34.090 | INFO | MemInfo: { | 20260506 19:43:34.090 | INFO | "gpu" : " 17 GB", | 20260506 19:43:34.090 | INFO | "huge" : "", | 20260506 19:43:34.090 | INFO | "huge_shared" : " 27 GB", | 20260506 19:43:34.090 | INFO | "malloc" : " 14 GB", | 20260506 19:43:34.090 | INFO | "pinned" : " 29 GB" | 20260506 19:43:34.090 | INFO | } | 20260506 19:43:34.090 | INFO | Perf report for the last 5.138 seconds: | 20260506 19:43:34.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:43:34.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:34.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:34.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:34.090 | INFO | ANALYSIS : 62.6%, 2.9, 95.4ms, 0 | 20260506 19:43:34.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:34.090 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260506 19:43:34.090 | INFO | PRE_HQ : 60.0%, 1.8, 57.3ms, 0 | 20260506 19:43:34.090 | INFO | BAZWRITER : 66.6%, 4.8, 163.3ms, 0 | 20260506 19:43:34.090 | INFO | BAZ_SAMPLING : 3.2%, 1.0, 1.7ms, 0 | 20260506 19:43:34.090 | INFO | Uploaded 36GiB to GPU in 2.04918s (17.568GiB/s) | 20260506 19:43:34.090 | INFO | Downloaded 38GiB to GPU in 2.20104s (17.2646GiB/s) | 20260506 19:43:39.228 | INFO | Analyzing chunk frames = [3957248,3957760) | 20260506 19:43:39.228 | INFO | MemInfo: { | 20260506 19:43:39.228 | INFO | "gpu" : " 17 GB", | 20260506 19:43:39.228 | INFO | "huge" : "", | 20260506 19:43:39.228 | INFO | "huge_shared" : " 27 GB", | 20260506 19:43:39.228 | INFO | "malloc" : " 14 GB", | 20260506 19:43:39.228 | INFO | "pinned" : " 29 GB" | 20260506 19:43:39.228 | INFO | } | 20260506 19:43:39.228 | INFO | Perf report for the last 5.13761 seconds: | 20260506 19:43:39.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:43:39.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:39.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:39.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:39.228 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:43:39.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:39.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:39.228 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 19:43:39.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:39.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:39.228 | INFO | Uploaded 36GiB to GPU in 2.0199s (17.8227GiB/s) | 20260506 19:43:39.228 | INFO | Downloaded 34GiB to GPU in 1.92938s (17.6222GiB/s) | 20260506 19:43:44.367 | INFO | Analyzing chunk frames = [3957760,3958272) | 20260506 19:43:44.367 | INFO | MemInfo: { | 20260506 19:43:44.367 | INFO | "gpu" : " 17 GB", | 20260506 19:43:44.367 | INFO | "huge" : "", | 20260506 19:43:44.367 | INFO | "huge_shared" : " 27 GB", | 20260506 19:43:44.368 | INFO | "malloc" : " 14 GB", | 20260506 19:43:44.368 | INFO | "pinned" : " 29 GB" | 20260506 19:43:44.368 | INFO | } | 20260506 19:43:44.368 | INFO | Perf report for the last 5.13985 seconds: | 20260506 19:43:44.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:43:44.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:44.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:44.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:44.368 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:43:44.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:44.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:44.368 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:43:44.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:44.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:44.368 | INFO | Uploaded 36GiB to GPU in 1.98226s (18.1611GiB/s) | 20260506 19:43:44.368 | INFO | Downloaded 34GiB to GPU in 1.91544s (17.7505GiB/s) | 20260506 19:43:49.506 | INFO | Analyzing chunk frames = [3958272,3958784) | 20260506 19:43:49.506 | INFO | MemInfo: { | 20260506 19:43:49.506 | INFO | "gpu" : " 17 GB", | 20260506 19:43:49.506 | INFO | "huge" : "", | 20260506 19:43:49.506 | INFO | "huge_shared" : " 27 GB", | 20260506 19:43:49.506 | INFO | "malloc" : " 14 GB", | 20260506 19:43:49.507 | INFO | "pinned" : " 29 GB" | 20260506 19:43:49.507 | INFO | } | 20260506 19:43:49.507 | INFO | Perf report for the last 5.13896 seconds: | 20260506 19:43:49.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:43:49.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:49.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:49.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:49.507 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 19:43:49.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:49.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:49.507 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:43:49.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:49.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:49.507 | INFO | Uploaded 36GiB to GPU in 1.94239s (18.5338GiB/s) | 20260506 19:43:49.507 | INFO | Downloaded 34GiB to GPU in 1.86833s (18.1981GiB/s) | 20260506 19:43:54.644 | INFO | Analyzing chunk frames = [3958784,3959296) | 20260506 19:43:54.644 | INFO | MemInfo: { | 20260506 19:43:54.644 | INFO | "gpu" : " 17 GB", | 20260506 19:43:54.644 | INFO | "huge" : "", | 20260506 19:43:54.644 | INFO | "huge_shared" : " 27 GB", | 20260506 19:43:54.644 | INFO | "malloc" : " 14 GB", | 20260506 19:43:54.644 | INFO | "pinned" : " 29 GB" | 20260506 19:43:54.644 | INFO | } | 20260506 19:43:54.644 | INFO | Perf report for the last 5.13733 seconds: | 20260506 19:43:54.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:43:54.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:54.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:54.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:54.644 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 19:43:54.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:54.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:54.644 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:43:54.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:54.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:54.644 | INFO | Uploaded 36GiB to GPU in 2.01088s (17.9026GiB/s) | 20260506 19:43:54.644 | INFO | Downloaded 34GiB to GPU in 1.91307s (17.7725GiB/s) | 20260506 19:43:59.780 | INFO | Analyzing chunk frames = [3959296,3959808) | 20260506 19:43:59.780 | INFO | MemInfo: { | 20260506 19:43:59.780 | INFO | "gpu" : " 17 GB", | 20260506 19:43:59.780 | INFO | "huge" : "", | 20260506 19:43:59.780 | INFO | "huge_shared" : " 27 GB", | 20260506 19:43:59.780 | INFO | "malloc" : " 14 GB", | 20260506 19:43:59.780 | INFO | "pinned" : " 29 GB" | 20260506 19:43:59.780 | INFO | } | 20260506 19:43:59.780 | INFO | Perf report for the last 5.13586 seconds: | 20260506 19:43:59.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:43:59.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:59.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:59.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:59.780 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 19:43:59.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:59.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:59.780 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 19:43:59.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:59.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:43:59.780 | INFO | Uploaded 36GiB to GPU in 2.03329s (17.7053GiB/s) | 20260506 19:43:59.780 | INFO | Downloaded 34GiB to GPU in 1.93557s (17.5659GiB/s) | 20260506 19:44:04.921 | INFO | Analyzing chunk frames = [3959808,3960320) | 20260506 19:44:04.921 | INFO | MemInfo: { | 20260506 19:44:04.921 | INFO | "gpu" : " 17 GB", | 20260506 19:44:04.921 | INFO | "huge" : "", | 20260506 19:44:04.921 | INFO | "huge_shared" : " 27 GB", | 20260506 19:44:04.921 | INFO | "malloc" : " 14 GB", | 20260506 19:44:04.921 | INFO | "pinned" : " 29 GB" | 20260506 19:44:04.921 | INFO | } | 20260506 19:44:04.921 | INFO | Perf report for the last 5.14157 seconds: | 20260506 19:44:04.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:44:04.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:04.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:04.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:04.921 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 19:44:04.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:04.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:04.921 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:44:04.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:04.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:04.922 | INFO | Uploaded 36GiB to GPU in 1.99131s (18.0785GiB/s) | 20260506 19:44:04.922 | INFO | Downloaded 34GiB to GPU in 2.01089s (16.9079GiB/s) | 20260506 19:44:10.066 | INFO | Analyzing chunk frames = [3960320,3960832) | 20260506 19:44:10.066 | INFO | MemInfo: { | 20260506 19:44:10.066 | INFO | "gpu" : " 17 GB", | 20260506 19:44:10.066 | INFO | "huge" : "", | 20260506 19:44:10.066 | INFO | "huge_shared" : " 27 GB", | 20260506 19:44:10.066 | INFO | "malloc" : " 14 GB", | 20260506 19:44:10.066 | INFO | "pinned" : " 29 GB" | 20260506 19:44:10.066 | INFO | } | 20260506 19:44:10.066 | INFO | Perf report for the last 5.14486 seconds: | 20260506 19:44:10.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:44:10.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:10.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:10.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:10.066 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 19:44:10.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:10.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:10.066 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:44:10.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:10.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:10.067 | INFO | Uploaded 36GiB to GPU in 1.99256s (18.0672GiB/s) | 20260506 19:44:10.067 | INFO | Downloaded 34GiB to GPU in 1.97598s (17.2066GiB/s) | 20260506 19:44:10.167 | INFO | Starting a round of Baz IO | 20260506 19:44:13.305 | INFO | Populating RTMetrics report | 20260506 19:44:13.305 | INFO | RTMetrics report populated | 20260506 19:44:13.305 | INFO | Opening tmp RTMetrics json file | 20260506 19:44:13.353 | INFO | Serializing RTMetrics to json | 20260506 19:44:13.477 | INFO | Serialization done, swapping tmp file now... | 20260506 19:44:13.481 | INFO | RTMetrics done | 20260506 19:44:13.510 | INFO | Finished a round of Baz IO: Wrote 1.999 GiB in 3.342 seconds | 20260506 19:44:13.510 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.50% / 0.00% / 65.49% | 20260506 19:44:13.510 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.865229% / 3.97364e-06% / 3.57628e-05% | 20260506 19:44:13.510 | INFO | Baz IO overall: | 20260506 19:44:13.510 | INFO | BytesWritten : 2092823975087 | 20260506 19:44:13.510 | INFO | NumEvents : 1011505432484 | 20260506 19:44:15.206 | INFO | Analyzing chunk frames = [3960832,3961344) | 20260506 19:44:15.206 | INFO | MemInfo: { | 20260506 19:44:15.206 | INFO | "gpu" : " 17 GB", | 20260506 19:44:15.206 | INFO | "huge" : "", | 20260506 19:44:15.207 | INFO | "huge_shared" : " 27 GB", | 20260506 19:44:15.207 | INFO | "malloc" : " 14 GB", | 20260506 19:44:15.207 | INFO | "pinned" : " 29 GB" | 20260506 19:44:15.207 | INFO | } | 20260506 19:44:15.207 | INFO | Perf report for the last 5.14036 seconds: | 20260506 19:44:15.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:44:15.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:15.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:15.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:15.207 | INFO | ANALYSIS : 62.4%, 2.9, 95.5ms, 0 | 20260506 19:44:15.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:15.207 | INFO | RT_METRICS : 46.4%, 1.0, 24.3ms, 0 | 20260506 19:44:15.207 | INFO | PRE_HQ : 60.0%, 1.8, 57.5ms, 0 | 20260506 19:44:15.207 | INFO | BAZWRITER : 64.7%, 4.5, 150.0ms, 0 | 20260506 19:44:15.207 | INFO | BAZ_SAMPLING : 3.3%, 1.0, 1.7ms, 0 | 20260506 19:44:15.207 | INFO | Uploaded 36GiB to GPU in 2.05728s (17.4989GiB/s) | 20260506 19:44:15.207 | INFO | Downloaded 38GiB to GPU in 2.18609s (17.3827GiB/s) | 20260506 19:44:20.351 | INFO | Analyzing chunk frames = [3961344,3961856) | 20260506 19:44:20.351 | INFO | MemInfo: { | 20260506 19:44:20.351 | INFO | "gpu" : " 17 GB", | 20260506 19:44:20.351 | INFO | "huge" : "", | 20260506 19:44:20.351 | INFO | "huge_shared" : " 27 GB", | 20260506 19:44:20.351 | INFO | "malloc" : " 14 GB", | 20260506 19:44:20.352 | INFO | "pinned" : " 29 GB" | 20260506 19:44:20.352 | INFO | } | 20260506 19:44:20.352 | INFO | Perf report for the last 5.14495 seconds: | 20260506 19:44:20.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:44:20.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:20.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:20.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:20.352 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 19:44:20.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:20.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:20.352 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:44:20.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:20.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:20.352 | INFO | Uploaded 36GiB to GPU in 1.99853s (18.0132GiB/s) | 20260506 19:44:20.352 | INFO | Downloaded 34GiB to GPU in 1.92974s (17.6189GiB/s) | 20260506 19:44:25.485 | INFO | Analyzing chunk frames = [3961856,3962368) | 20260506 19:44:25.485 | INFO | MemInfo: { | 20260506 19:44:25.485 | INFO | "gpu" : " 17 GB", | 20260506 19:44:25.485 | INFO | "huge" : "", | 20260506 19:44:25.485 | INFO | "huge_shared" : " 27 GB", | 20260506 19:44:25.485 | INFO | "malloc" : " 14 GB", | 20260506 19:44:25.485 | INFO | "pinned" : " 29 GB" | 20260506 19:44:25.485 | INFO | } | 20260506 19:44:25.485 | INFO | Perf report for the last 5.13333 seconds: | 20260506 19:44:25.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:44:25.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:25.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:25.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:25.485 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260506 19:44:25.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:25.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:25.485 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:44:25.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:25.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:25.485 | INFO | Uploaded 36GiB to GPU in 1.99204s (18.072GiB/s) | 20260506 19:44:25.485 | INFO | Downloaded 34GiB to GPU in 1.92775s (17.6372GiB/s) | 20260506 19:44:30.619 | INFO | Analyzing chunk frames = [3962368,3962880) | 20260506 19:44:30.619 | INFO | MemInfo: { | 20260506 19:44:30.619 | INFO | "gpu" : " 17 GB", | 20260506 19:44:30.619 | INFO | "huge" : "", | 20260506 19:44:30.619 | INFO | "huge_shared" : " 27 GB", | 20260506 19:44:30.619 | INFO | "malloc" : " 14 GB", | 20260506 19:44:30.619 | INFO | "pinned" : " 29 GB" | 20260506 19:44:30.619 | INFO | } | 20260506 19:44:30.619 | INFO | Perf report for the last 5.13434 seconds: | 20260506 19:44:30.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:44:30.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:30.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:30.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:30.619 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 19:44:30.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:30.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:30.619 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:44:30.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:30.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:30.619 | INFO | Uploaded 36GiB to GPU in 1.9724s (18.2519GiB/s) | 20260506 19:44:30.619 | INFO | Downloaded 34GiB to GPU in 1.87526s (18.1309GiB/s) | 20260506 19:44:35.758 | INFO | Analyzing chunk frames = [3962880,3963392) | 20260506 19:44:35.758 | INFO | MemInfo: { | 20260506 19:44:35.758 | INFO | "gpu" : " 17 GB", | 20260506 19:44:35.758 | INFO | "huge" : "", | 20260506 19:44:35.758 | INFO | "huge_shared" : " 27 GB", | 20260506 19:44:35.758 | INFO | "malloc" : " 14 GB", | 20260506 19:44:35.758 | INFO | "pinned" : " 29 GB" | 20260506 19:44:35.758 | INFO | } | 20260506 19:44:35.758 | INFO | Perf report for the last 5.13871 seconds: | 20260506 19:44:35.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:44:35.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:35.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:35.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:35.758 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 19:44:35.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:35.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:35.758 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:44:35.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:35.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:35.758 | INFO | Uploaded 36GiB to GPU in 2.01964s (17.8249GiB/s) | 20260506 19:44:35.758 | INFO | Downloaded 34GiB to GPU in 1.91369s (17.7667GiB/s) | 20260506 19:44:40.894 | INFO | Analyzing chunk frames = [3963392,3963904) | 20260506 19:44:40.894 | INFO | MemInfo: { | 20260506 19:44:40.894 | INFO | "gpu" : " 17 GB", | 20260506 19:44:40.894 | INFO | "huge" : "", | 20260506 19:44:40.894 | INFO | "huge_shared" : " 27 GB", | 20260506 19:44:40.894 | INFO | "malloc" : " 14 GB", | 20260506 19:44:40.894 | INFO | "pinned" : " 29 GB" | 20260506 19:44:40.894 | INFO | } | 20260506 19:44:40.894 | INFO | Perf report for the last 5.13578 seconds: | 20260506 19:44:40.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:44:40.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:40.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:40.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:40.894 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 19:44:40.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:40.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:40.894 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 19:44:40.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:40.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:40.894 | INFO | Uploaded 36GiB to GPU in 2.06846s (17.4042GiB/s) | 20260506 19:44:40.894 | INFO | Downloaded 34GiB to GPU in 1.95681s (17.3752GiB/s) | 20260506 19:44:46.037 | INFO | Analyzing chunk frames = [3963904,3964416) | 20260506 19:44:46.037 | INFO | MemInfo: { | 20260506 19:44:46.037 | INFO | "gpu" : " 17 GB", | 20260506 19:44:46.037 | INFO | "huge" : "", | 20260506 19:44:46.037 | INFO | "huge_shared" : " 27 GB", | 20260506 19:44:46.037 | INFO | "malloc" : " 14 GB", | 20260506 19:44:46.037 | INFO | "pinned" : " 29 GB" | 20260506 19:44:46.037 | INFO | } | 20260506 19:44:46.037 | INFO | Perf report for the last 5.14348 seconds: | 20260506 19:44:46.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:44:46.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:46.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:46.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:46.037 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 19:44:46.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:46.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:46.037 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:44:46.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:46.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:46.038 | INFO | Uploaded 36GiB to GPU in 2.02841s (17.7479GiB/s) | 20260506 19:44:46.038 | INFO | Downloaded 34GiB to GPU in 1.97323s (17.2306GiB/s) | 20260506 19:44:51.180 | INFO | Analyzing chunk frames = [3964416,3964928) | 20260506 19:44:51.180 | INFO | MemInfo: { | 20260506 19:44:51.180 | INFO | "gpu" : " 17 GB", | 20260506 19:44:51.180 | INFO | "huge" : "", | 20260506 19:44:51.180 | INFO | "huge_shared" : " 27 GB", | 20260506 19:44:51.180 | INFO | "malloc" : " 14 GB", | 20260506 19:44:51.180 | INFO | "pinned" : " 29 GB" | 20260506 19:44:51.180 | INFO | } | 20260506 19:44:51.180 | INFO | Perf report for the last 5.14259 seconds: | 20260506 19:44:51.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:44:51.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:51.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:51.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:51.180 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 19:44:51.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:51.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:51.180 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260506 19:44:51.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:51.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:51.180 | INFO | Uploaded 36GiB to GPU in 1.98516s (18.1346GiB/s) | 20260506 19:44:51.180 | INFO | Downloaded 34GiB to GPU in 1.99449s (17.047GiB/s) | 20260506 19:44:51.180 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:44:51.180 | INFO | Total mem usage: 18 GiB | 20260506 19:44:51.180 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:44:51.180 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.6%) | 20260506 19:44:51.180 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:44:51.180 | INFO | High Water Breakdown: | 20260506 19:44:51.180 | INFO | 61.4% : 9.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:44:51.180 | INFO | 38.4% : 5.99 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:44:51.180 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:44:51.180 | INFO | ------------------------------------------ | 20260506 19:44:51.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:44:51.180 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:44:51.180 | INFO | Total mem usage: 26.5 GiB | 20260506 19:44:51.180 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:44:51.180 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:44:51.180 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:44:51.180 | INFO | High Water Breakdown: | 20260506 19:44:51.180 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:44:51.180 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:44:51.181 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:44:51.181 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:44:51.181 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:44:51.181 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:44:51.181 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:44:51.181 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:44:51.181 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:44:51.181 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:44:51.181 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:44:51.181 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:44:51.181 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:44:51.181 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:44:51.181 | INFO | ------------------------------------------ | 20260506 19:44:51.181 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:44:51.181 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:44:51.181 | INFO | Total mem usage: 25.6 GiB | 20260506 19:44:51.181 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:44:51.181 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:44:51.181 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:44:51.181 | INFO | High Water Breakdown: | 20260506 19:44:51.181 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:44:51.181 | INFO | ------------------------------------------ | 20260506 19:44:51.181 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:44:51.181 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:44:51.181 | INFO | Total mem usage: 15.7 GiB | 20260506 19:44:51.181 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:44:51.181 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:44:51.181 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:44:51.181 | INFO | High Water Breakdown: | 20260506 19:44:51.181 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:44:51.181 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:44:51.181 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:44:51.181 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:44:51.181 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:44:51.181 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:44:51.181 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:44:51.181 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:44:51.181 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:44:51.181 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:44:51.181 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:44:51.181 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:44:51.181 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:44:51.181 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:44:51.181 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:44:51.181 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:44:51.181 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:44:51.181 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:44:51.181 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:44:51.181 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:44:51.181 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:44:51.181 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:44:51.181 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:44:51.181 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:44:51.181 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:44:51.181 | INFO | ------------------------------------------ | 20260506 19:44:51.181 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:44:51.181 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:44:51.181 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:44:51.181 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:44:51.181 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.56e+01, 1.34e+01 / 2.40e+00 | 20260506 19:44:51.181 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 9.89e-03 / 5.74e-03 | 20260506 19:44:51.181 | INFO | PerfReport: 11.64%, Baseline -- 3.63e+00 / 4.43e+00, 5.55e-01 / 3.11e-01 | 20260506 19:44:51.181 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.81e-01 / 1.37e-01 | 20260506 19:44:51.181 | INFO | PerfReport: 16.02%, DME -- 4.00e+01 / 4.44e+01, 3.91e+00 / 2.60e+00 | 20260506 19:44:51.181 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.38e+00 / 1.82e+00 | 20260506 19:44:51.181 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.24e+00 / 2.82e+00, 3.84e-01 / 2.07e-01 | 20260506 19:44:51.181 | INFO | PerfReport: 3.35%, Metrics -- 1.04e+00 / 1.37e+00, 2.73e-01 / 1.63e-01 | 20260506 19:44:51.181 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.59e-01 / 4.08e-01, 5.60e-02 / 2.24e-02 | 20260506 19:44:51.181 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:44:51.181 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:44:51.181 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:44:51.181 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 19:44:51.181 | INFO | PerfReport: 21.57%, Upload -- 2.09e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 19:44:51.181 | INFO | PerfReport: 21.14%, Download -- 2.05e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 19:44:51.181 | INFO | PerfReport: 32.16%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 19:44:51.181 | INFO | PerfReport: 1.48%, LockUpload -- 1.43e+00 / 0.00e+00, 5.49e+00 / 0.00e+00 | 20260506 19:44:51.181 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 19:44:51.181 | INFO | PerfReport: 22.54%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 19:44:51.181 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260506 19:44:51.284 | INFO | Starting a round of Baz IO | 20260506 19:44:54.420 | INFO | Populating RTMetrics report | 20260506 19:44:54.420 | INFO | RTMetrics report populated | 20260506 19:44:54.420 | INFO | Opening tmp RTMetrics json file | 20260506 19:44:54.722 | INFO | Serializing RTMetrics to json | 20260506 19:44:54.755 | INFO | Serialization done, swapping tmp file now... | 20260506 19:44:54.766 | INFO | RTMetrics done | 20260506 19:44:54.878 | INFO | Finished a round of Baz IO: Wrote 2 GiB in 3.593 seconds | 20260506 19:44:54.878 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.50% / 0.00% / 65.50% | 20260506 19:44:54.878 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.8623% / 1.19209e-05% / 7.54992e-05% | 20260506 19:44:54.878 | INFO | Baz IO overall: | 20260506 19:44:54.878 | INFO | BytesWritten : 2094971242496 | 20260506 19:44:54.878 | INFO | NumEvents : 1012537334082 | 20260506 19:44:56.319 | INFO | Analyzing chunk frames = [3964928,3965440) | 20260506 19:44:56.319 | INFO | MemInfo: { | 20260506 19:44:56.319 | INFO | "gpu" : " 17 GB", | 20260506 19:44:56.319 | INFO | "huge" : "", | 20260506 19:44:56.319 | INFO | "huge_shared" : " 27 GB", | 20260506 19:44:56.319 | INFO | "malloc" : " 14 GB", | 20260506 19:44:56.319 | INFO | "pinned" : " 29 GB" | 20260506 19:44:56.319 | INFO | } | 20260506 19:44:56.320 | INFO | Perf report for the last 5.13966 seconds: | 20260506 19:44:56.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:44:56.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:56.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:56.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:56.320 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 19:44:56.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:44:56.320 | INFO | RT_METRICS : 49.8%, 1.0, 26.1ms, 0 | 20260506 19:44:56.320 | INFO | PRE_HQ : 59.9%, 1.8, 57.3ms, 0 | 20260506 19:44:56.320 | INFO | BAZWRITER : 69.6%, 4.8, 171.2ms, 0 | 20260506 19:44:56.320 | INFO | BAZ_SAMPLING : 5.6%, 1.0, 2.9ms, 0 | 20260506 19:44:56.320 | INFO | Uploaded 36GiB to GPU in 2.04601s (17.5952GiB/s) | 20260506 19:44:56.320 | INFO | Downloaded 38GiB to GPU in 2.19961s (17.2758GiB/s) | 20260506 19:45:01.460 | INFO | Analyzing chunk frames = [3965440,3965952) | 20260506 19:45:01.460 | INFO | MemInfo: { | 20260506 19:45:01.460 | INFO | "gpu" : " 17 GB", | 20260506 19:45:01.460 | INFO | "huge" : "", | 20260506 19:45:01.460 | INFO | "huge_shared" : " 27 GB", | 20260506 19:45:01.460 | INFO | "malloc" : " 14 GB", | 20260506 19:45:01.460 | INFO | "pinned" : " 29 GB" | 20260506 19:45:01.460 | INFO | } | 20260506 19:45:01.460 | INFO | Perf report for the last 5.14074 seconds: | 20260506 19:45:01.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:45:01.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:01.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:01.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:01.460 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:45:01.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:01.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:01.460 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 19:45:01.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:01.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:01.461 | INFO | Uploaded 36GiB to GPU in 2.0025s (17.9776GiB/s) | 20260506 19:45:01.461 | INFO | Downloaded 34GiB to GPU in 1.93861s (17.5383GiB/s) | 20260506 19:45:06.596 | INFO | Analyzing chunk frames = [3965952,3966464) | 20260506 19:45:06.596 | INFO | MemInfo: { | 20260506 19:45:06.596 | INFO | "gpu" : " 17 GB", | 20260506 19:45:06.596 | INFO | "huge" : "", | 20260506 19:45:06.596 | INFO | "huge_shared" : " 27 GB", | 20260506 19:45:06.596 | INFO | "malloc" : " 14 GB", | 20260506 19:45:06.596 | INFO | "pinned" : " 29 GB" | 20260506 19:45:06.596 | INFO | } | 20260506 19:45:06.596 | INFO | Perf report for the last 5.13599 seconds: | 20260506 19:45:06.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:45:06.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:06.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:06.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:06.596 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:45:06.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:06.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:06.596 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:45:06.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:06.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:06.596 | INFO | Uploaded 36GiB to GPU in 1.96567s (18.3144GiB/s) | 20260506 19:45:06.597 | INFO | Downloaded 34GiB to GPU in 1.92162s (17.6935GiB/s) | 20260506 19:45:11.732 | INFO | Analyzing chunk frames = [3966464,3966976) | 20260506 19:45:11.732 | INFO | MemInfo: { | 20260506 19:45:11.732 | INFO | "gpu" : " 17 GB", | 20260506 19:45:11.732 | INFO | "huge" : "", | 20260506 19:45:11.732 | INFO | "huge_shared" : " 27 GB", | 20260506 19:45:11.732 | INFO | "malloc" : " 14 GB", | 20260506 19:45:11.732 | INFO | "pinned" : " 29 GB" | 20260506 19:45:11.732 | INFO | } | 20260506 19:45:11.732 | INFO | Perf report for the last 5.1361 seconds: | 20260506 19:45:11.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:45:11.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:11.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:11.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:11.732 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 19:45:11.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:11.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:11.732 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:45:11.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:11.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:11.733 | INFO | Uploaded 36GiB to GPU in 1.91694s (18.78GiB/s) | 20260506 19:45:11.733 | INFO | Downloaded 34GiB to GPU in 1.8737s (18.146GiB/s) | 20260506 19:45:16.869 | INFO | Analyzing chunk frames = [3966976,3967488) | 20260506 19:45:16.869 | INFO | MemInfo: { | 20260506 19:45:16.869 | INFO | "gpu" : " 17 GB", | 20260506 19:45:16.869 | INFO | "huge" : "", | 20260506 19:45:16.869 | INFO | "huge_shared" : " 27 GB", | 20260506 19:45:16.869 | INFO | "malloc" : " 14 GB", | 20260506 19:45:16.869 | INFO | "pinned" : " 29 GB" | 20260506 19:45:16.869 | INFO | } | 20260506 19:45:16.869 | INFO | Perf report for the last 5.13662 seconds: | 20260506 19:45:16.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:45:16.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:16.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:16.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:16.869 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 19:45:16.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:16.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:16.869 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:45:16.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:16.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:16.869 | INFO | Uploaded 36GiB to GPU in 2.01488s (17.8671GiB/s) | 20260506 19:45:16.869 | INFO | Downloaded 34GiB to GPU in 1.91709s (17.7352GiB/s) | 20260506 19:45:22.015 | INFO | Analyzing chunk frames = [3967488,3968000) | 20260506 19:45:22.015 | INFO | MemInfo: { | 20260506 19:45:22.015 | INFO | "gpu" : " 17 GB", | 20260506 19:45:22.015 | INFO | "huge" : "", | 20260506 19:45:22.015 | INFO | "huge_shared" : " 27 GB", | 20260506 19:45:22.015 | INFO | "malloc" : " 14 GB", | 20260506 19:45:22.015 | INFO | "pinned" : " 29 GB" | 20260506 19:45:22.015 | INFO | } | 20260506 19:45:22.015 | INFO | Perf report for the last 5.14601 seconds: | 20260506 19:45:22.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:45:22.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:22.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:22.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:22.015 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 19:45:22.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:22.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:22.015 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 19:45:22.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:22.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:22.015 | INFO | Uploaded 36GiB to GPU in 2.01318s (17.8822GiB/s) | 20260506 19:45:22.015 | INFO | Downloaded 34GiB to GPU in 1.95583s (17.3839GiB/s) | 20260506 19:45:27.156 | INFO | Analyzing chunk frames = [3968000,3968512) | 20260506 19:45:27.156 | INFO | MemInfo: { | 20260506 19:45:27.156 | INFO | "gpu" : " 17 GB", | 20260506 19:45:27.156 | INFO | "huge" : "", | 20260506 19:45:27.156 | INFO | "huge_shared" : " 27 GB", | 20260506 19:45:27.156 | INFO | "malloc" : " 14 GB", | 20260506 19:45:27.156 | INFO | "pinned" : " 29 GB" | 20260506 19:45:27.156 | INFO | } | 20260506 19:45:27.156 | INFO | Perf report for the last 5.1413 seconds: | 20260506 19:45:27.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:45:27.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:27.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:27.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:27.156 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 19:45:27.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:27.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:27.156 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260506 19:45:27.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:27.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:27.157 | INFO | Uploaded 36GiB to GPU in 2.01115s (17.9002GiB/s) | 20260506 19:45:27.157 | INFO | Downloaded 34GiB to GPU in 1.98774s (17.1048GiB/s) | 20260506 19:45:32.299 | INFO | Analyzing chunk frames = [3968512,3969024) | 20260506 19:45:32.299 | INFO | MemInfo: { | 20260506 19:45:32.299 | INFO | "gpu" : " 17 GB", | 20260506 19:45:32.299 | INFO | "huge" : "", | 20260506 19:45:32.299 | INFO | "huge_shared" : " 27 GB", | 20260506 19:45:32.299 | INFO | "malloc" : " 14 GB", | 20260506 19:45:32.299 | INFO | "pinned" : " 29 GB" | 20260506 19:45:32.299 | INFO | } | 20260506 19:45:32.299 | INFO | Perf report for the last 5.14285 seconds: | 20260506 19:45:32.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:45:32.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:32.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:32.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:32.299 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 19:45:32.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:32.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:32.299 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:45:32.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:32.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:32.299 | INFO | Uploaded 36GiB to GPU in 2.01475s (17.8682GiB/s) | 20260506 19:45:32.299 | INFO | Downloaded 34GiB to GPU in 1.98786s (17.1038GiB/s) | 20260506 19:45:32.407 | INFO | Starting a round of Baz IO | 20260506 19:45:35.537 | INFO | Populating RTMetrics report | 20260506 19:45:35.537 | INFO | RTMetrics report populated | 20260506 19:45:35.537 | INFO | Opening tmp RTMetrics json file | 20260506 19:45:35.539 | INFO | Serializing RTMetrics to json | 20260506 19:45:35.554 | INFO | Serialization done, swapping tmp file now... | 20260506 19:45:35.576 | INFO | RTMetrics done | 20260506 19:45:35.687 | INFO | Finished a round of Baz IO: Wrote 1.998 GiB in 3.28 seconds | 20260506 19:45:35.687 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.50% / 0.00% / 65.50% | 20260506 19:45:35.687 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.860198% / 3.97364e-06% / 9.53674e-05% | 20260506 19:45:35.687 | INFO | Baz IO overall: | 20260506 19:45:35.687 | INFO | BytesWritten : 2097116467436 | 20260506 19:45:35.687 | INFO | NumEvents : 1013567541911 | 20260506 19:45:37.439 | INFO | Analyzing chunk frames = [3969024,3969536) | 20260506 19:45:37.439 | INFO | MemInfo: { | 20260506 19:45:37.439 | INFO | "gpu" : " 17 GB", | 20260506 19:45:37.439 | INFO | "huge" : "", | 20260506 19:45:37.439 | INFO | "huge_shared" : " 27 GB", | 20260506 19:45:37.439 | INFO | "malloc" : " 14 GB", | 20260506 19:45:37.439 | INFO | "pinned" : " 29 GB" | 20260506 19:45:37.439 | INFO | } | 20260506 19:45:37.439 | INFO | Perf report for the last 5.13955 seconds: | 20260506 19:45:37.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:45:37.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:37.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:37.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:37.439 | INFO | ANALYSIS : 62.4%, 2.9, 95.1ms, 0 | 20260506 19:45:37.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:37.439 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 19:45:37.439 | INFO | PRE_HQ : 60.0%, 1.8, 57.3ms, 0 | 20260506 19:45:37.439 | INFO | BAZWRITER : 63.6%, 4.5, 146.3ms, 0 | 20260506 19:45:37.439 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 19:45:37.439 | INFO | Uploaded 36GiB to GPU in 2.06827s (17.4059GiB/s) | 20260506 19:45:37.439 | INFO | Downloaded 38GiB to GPU in 2.19756s (17.2919GiB/s) | 20260506 19:45:42.574 | INFO | Analyzing chunk frames = [3969536,3970048) | 20260506 19:45:42.574 | INFO | MemInfo: { | 20260506 19:45:42.574 | INFO | "gpu" : " 17 GB", | 20260506 19:45:42.574 | INFO | "huge" : "", | 20260506 19:45:42.574 | INFO | "huge_shared" : " 27 GB", | 20260506 19:45:42.574 | INFO | "malloc" : " 14 GB", | 20260506 19:45:42.574 | INFO | "pinned" : " 29 GB" | 20260506 19:45:42.574 | INFO | } | 20260506 19:45:42.574 | INFO | Perf report for the last 5.13535 seconds: | 20260506 19:45:42.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:45:42.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:42.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:42.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:42.574 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:45:42.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:42.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:42.574 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 19:45:42.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:42.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:42.574 | INFO | Uploaded 36GiB to GPU in 2.02103s (17.8127GiB/s) | 20260506 19:45:42.574 | INFO | Downloaded 34GiB to GPU in 1.93071s (17.6101GiB/s) | 20260506 19:45:47.712 | INFO | Analyzing chunk frames = [3970048,3970560) | 20260506 19:45:47.712 | INFO | MemInfo: { | 20260506 19:45:47.712 | INFO | "gpu" : " 17 GB", | 20260506 19:45:47.712 | INFO | "huge" : "", | 20260506 19:45:47.712 | INFO | "huge_shared" : " 27 GB", | 20260506 19:45:47.712 | INFO | "malloc" : " 14 GB", | 20260506 19:45:47.712 | INFO | "pinned" : " 29 GB" | 20260506 19:45:47.712 | INFO | } | 20260506 19:45:47.712 | INFO | Perf report for the last 5.13782 seconds: | 20260506 19:45:47.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:45:47.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:47.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:47.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:47.712 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 19:45:47.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:47.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:47.712 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:45:47.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:47.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:47.712 | INFO | Uploaded 36GiB to GPU in 1.9957s (18.0388GiB/s) | 20260506 19:45:47.712 | INFO | Downloaded 34GiB to GPU in 1.92153s (17.6942GiB/s) | 20260506 19:45:52.846 | INFO | Analyzing chunk frames = [3970560,3971072) | 20260506 19:45:52.846 | INFO | MemInfo: { | 20260506 19:45:52.846 | INFO | "gpu" : " 17 GB", | 20260506 19:45:52.846 | INFO | "huge" : "", | 20260506 19:45:52.846 | INFO | "huge_shared" : " 27 GB", | 20260506 19:45:52.846 | INFO | "malloc" : " 14 GB", | 20260506 19:45:52.846 | INFO | "pinned" : " 29 GB" | 20260506 19:45:52.846 | INFO | } | 20260506 19:45:52.846 | INFO | Perf report for the last 5.13383 seconds: | 20260506 19:45:52.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:45:52.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:52.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:52.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:52.846 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 19:45:52.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:52.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:52.846 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:45:52.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:52.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:52.846 | INFO | Uploaded 36GiB to GPU in 1.94726s (18.4875GiB/s) | 20260506 19:45:52.846 | INFO | Downloaded 34GiB to GPU in 1.87232s (18.1593GiB/s) | 20260506 19:45:57.988 | INFO | Analyzing chunk frames = [3971072,3971584) | 20260506 19:45:57.988 | INFO | MemInfo: { | 20260506 19:45:57.988 | INFO | "gpu" : " 17 GB", | 20260506 19:45:57.989 | INFO | "huge" : "", | 20260506 19:45:57.989 | INFO | "huge_shared" : " 27 GB", | 20260506 19:45:57.989 | INFO | "malloc" : " 14 GB", | 20260506 19:45:57.989 | INFO | "pinned" : " 29 GB" | 20260506 19:45:57.989 | INFO | } | 20260506 19:45:57.989 | INFO | Perf report for the last 5.14278 seconds: | 20260506 19:45:57.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:45:57.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:57.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:57.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:57.989 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260506 19:45:57.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:57.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:57.989 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:45:57.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:57.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:45:57.989 | INFO | Uploaded 36GiB to GPU in 2.02337s (17.7921GiB/s) | 20260506 19:45:57.989 | INFO | Downloaded 34GiB to GPU in 1.91366s (17.767GiB/s) | 20260506 19:46:03.129 | INFO | Analyzing chunk frames = [3971584,3972096) | 20260506 19:46:03.129 | INFO | MemInfo: { | 20260506 19:46:03.129 | INFO | "gpu" : " 17 GB", | 20260506 19:46:03.129 | INFO | "huge" : "", | 20260506 19:46:03.129 | INFO | "huge_shared" : " 27 GB", | 20260506 19:46:03.129 | INFO | "malloc" : " 14 GB", | 20260506 19:46:03.129 | INFO | "pinned" : " 29 GB" | 20260506 19:46:03.129 | INFO | } | 20260506 19:46:03.129 | INFO | Perf report for the last 5.14065 seconds: | 20260506 19:46:03.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:46:03.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:03.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:03.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:03.129 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:46:03.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:03.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:03.129 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260506 19:46:03.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:03.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:03.129 | INFO | Uploaded 36GiB to GPU in 2.06515s (17.4321GiB/s) | 20260506 19:46:03.130 | INFO | Downloaded 34GiB to GPU in 1.94618s (17.4701GiB/s) | 20260506 19:46:08.272 | INFO | Analyzing chunk frames = [3972096,3972608) | 20260506 19:46:08.272 | INFO | MemInfo: { | 20260506 19:46:08.272 | INFO | "gpu" : " 17 GB", | 20260506 19:46:08.272 | INFO | "huge" : "", | 20260506 19:46:08.272 | INFO | "huge_shared" : " 27 GB", | 20260506 19:46:08.272 | INFO | "malloc" : " 14 GB", | 20260506 19:46:08.272 | INFO | "pinned" : " 29 GB" | 20260506 19:46:08.272 | INFO | } | 20260506 19:46:08.272 | INFO | Perf report for the last 5.14275 seconds: | 20260506 19:46:08.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:46:08.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:08.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:08.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:08.272 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 19:46:08.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:08.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:08.272 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:46:08.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:08.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:08.272 | INFO | Uploaded 36GiB to GPU in 2.02072s (17.8154GiB/s) | 20260506 19:46:08.272 | INFO | Downloaded 34GiB to GPU in 1.98503s (17.1282GiB/s) | 20260506 19:46:13.411 | INFO | Analyzing chunk frames = [3972608,3973120) | 20260506 19:46:13.411 | INFO | MemInfo: { | 20260506 19:46:13.411 | INFO | "gpu" : " 17 GB", | 20260506 19:46:13.412 | INFO | "huge" : "", | 20260506 19:46:13.412 | INFO | "huge_shared" : " 27 GB", | 20260506 19:46:13.412 | INFO | "malloc" : " 14 GB", | 20260506 19:46:13.412 | INFO | "pinned" : " 29 GB" | 20260506 19:46:13.412 | INFO | } | 20260506 19:46:13.412 | INFO | Perf report for the last 5.13955 seconds: | 20260506 19:46:13.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:46:13.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:13.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:13.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:13.412 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:46:13.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:13.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:13.412 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:46:13.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:13.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:13.412 | INFO | Uploaded 36GiB to GPU in 1.97192s (18.2563GiB/s) | 20260506 19:46:13.412 | INFO | Downloaded 34GiB to GPU in 1.97806s (17.1885GiB/s) | 20260506 19:46:13.412 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:46:13.412 | INFO | Total mem usage: 18 GiB | 20260506 19:46:13.412 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:46:13.412 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -18.1%) | 20260506 19:46:13.412 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:46:13.412 | INFO | High Water Breakdown: | 20260506 19:46:13.412 | INFO | 61.1% : 9.31 GiB ( 81.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:46:13.412 | INFO | 38.8% : 5.92 GiB ( 88.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:46:13.412 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:46:13.412 | INFO | ------------------------------------------ | 20260506 19:46:13.412 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:46:13.412 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:46:13.412 | INFO | Total mem usage: 26.5 GiB | 20260506 19:46:13.412 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:46:13.412 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:46:13.412 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:46:13.412 | INFO | High Water Breakdown: | 20260506 19:46:13.412 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:46:13.412 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:46:13.412 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:46:13.412 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:46:13.412 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:46:13.412 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:46:13.412 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:46:13.412 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:46:13.412 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:46:13.412 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:46:13.412 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:46:13.412 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:46:13.412 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:46:13.412 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:46:13.412 | INFO | ------------------------------------------ | 20260506 19:46:13.412 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:46:13.412 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:46:13.412 | INFO | Total mem usage: 25.6 GiB | 20260506 19:46:13.412 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:46:13.412 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:46:13.412 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:46:13.412 | INFO | High Water Breakdown: | 20260506 19:46:13.412 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:46:13.412 | INFO | ------------------------------------------ | 20260506 19:46:13.412 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:46:13.412 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:46:13.412 | INFO | Total mem usage: 15.7 GiB | 20260506 19:46:13.412 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:46:13.412 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:46:13.412 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 19:46:13.412 | INFO | High Water Breakdown: | 20260506 19:46:13.412 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:46:13.412 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:46:13.412 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:46:13.412 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:46:13.412 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:46:13.412 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:46:13.412 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:46:13.413 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:46:13.413 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:46:13.413 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:46:13.413 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:46:13.413 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:46:13.413 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:46:13.413 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:46:13.413 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:46:13.413 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:46:13.413 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:46:13.413 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:46:13.413 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:46:13.413 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:46:13.413 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:46:13.413 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:46:13.413 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:46:13.413 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:46:13.413 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:46:13.413 | INFO | ------------------------------------------ | 20260506 19:46:13.413 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:46:13.413 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:46:13.413 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:46:13.413 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:46:13.413 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.34e+01 / 2.51e+00 | 20260506 19:46:13.413 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.00e-01, 9.98e-03 / 4.41e-03 | 20260506 19:46:13.413 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.43e+00, 5.49e-01 / 3.15e-01 | 20260506 19:46:13.413 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.48e+00, 1.82e-01 / 1.19e-01 | 20260506 19:46:13.413 | INFO | PerfReport: 16.02%, DME -- 4.00e+01 / 4.46e+01, 4.03e+00 / 2.26e+00 | 20260506 19:46:13.413 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.79e+01 / 2.05e+01, 1.47e+00 / 2.14e+00 | 20260506 19:46:13.413 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.23e+00 / 2.87e+00, 3.79e-01 / 1.96e-01 | 20260506 19:46:13.413 | INFO | PerfReport: 3.34%, Metrics -- 1.04e+00 / 1.39e+00, 2.77e-01 / 1.22e-01 | 20260506 19:46:13.413 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.59e-01 / 4.06e-01, 5.76e-02 / 2.44e-02 | 20260506 19:46:13.413 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:46:13.413 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:46:13.413 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:46:13.413 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 19:46:13.413 | INFO | PerfReport: 21.54%, Upload -- 2.09e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260506 19:46:13.413 | INFO | PerfReport: 21.16%, Download -- 2.05e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260506 19:46:13.413 | INFO | PerfReport: 32.20%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 19:46:13.413 | INFO | PerfReport: 1.39%, LockUpload -- 1.35e+00 / 0.00e+00, 5.25e+00 / 0.00e+00 | 20260506 19:46:13.413 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260506 19:46:13.413 | INFO | PerfReport: 22.59%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 19:46:13.413 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260506 19:46:13.515 | INFO | Starting a round of Baz IO | 20260506 19:46:16.651 | INFO | Populating RTMetrics report | 20260506 19:46:16.652 | INFO | RTMetrics report populated | 20260506 19:46:16.652 | INFO | Opening tmp RTMetrics json file | 20260506 19:46:16.709 | INFO | Serializing RTMetrics to json | 20260506 19:46:16.714 | INFO | Serialization done, swapping tmp file now... | 20260506 19:46:16.719 | INFO | RTMetrics done | 20260506 19:46:17.360 | INFO | Finished a round of Baz IO: Wrote 2 GiB in 3.844 seconds | 20260506 19:46:17.360 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.49% / 0.00% / 65.50% | 20260506 19:46:17.360 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.860536% / 7.94729e-06% / 8.74201e-05% | 20260506 19:46:17.360 | INFO | Baz IO overall: | 20260506 19:46:17.360 | INFO | BytesWritten : 2099263759827 | 20260506 19:46:17.360 | INFO | NumEvents : 1014599608428 | 20260506 19:46:18.546 | INFO | Analyzing chunk frames = [3973120,3973632) | 20260506 19:46:18.546 | INFO | MemInfo: { | 20260506 19:46:18.546 | INFO | "gpu" : " 17 GB", | 20260506 19:46:18.546 | INFO | "huge" : "", | 20260506 19:46:18.546 | INFO | "huge_shared" : " 27 GB", | 20260506 19:46:18.546 | INFO | "malloc" : " 14 GB", | 20260506 19:46:18.547 | INFO | "pinned" : " 29 GB" | 20260506 19:46:18.547 | INFO | } | 20260506 19:46:18.547 | INFO | Perf report for the last 5.13495 seconds: | 20260506 19:46:18.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:46:18.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:18.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:18.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:18.547 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 19:46:18.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:18.547 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 19:46:18.547 | INFO | PRE_HQ : 59.9%, 1.8, 57.2ms, 0 | 20260506 19:46:18.547 | INFO | BAZWRITER : 74.5%, 4.8, 184.6ms, 0 | 20260506 19:46:18.547 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 19:46:18.547 | INFO | Uploaded 36GiB to GPU in 2.02024s (17.8196GiB/s) | 20260506 19:46:18.547 | INFO | Downloaded 38GiB to GPU in 2.19796s (17.2888GiB/s) | 20260506 19:46:23.685 | INFO | Analyzing chunk frames = [3973632,3974144) | 20260506 19:46:23.685 | INFO | MemInfo: { | 20260506 19:46:23.685 | INFO | "gpu" : " 17 GB", | 20260506 19:46:23.685 | INFO | "huge" : "", | 20260506 19:46:23.685 | INFO | "huge_shared" : " 27 GB", | 20260506 19:46:23.685 | INFO | "malloc" : " 14 GB", | 20260506 19:46:23.685 | INFO | "pinned" : " 29 GB" | 20260506 19:46:23.685 | INFO | } | 20260506 19:46:23.685 | INFO | Perf report for the last 5.13838 seconds: | 20260506 19:46:23.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:46:23.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:23.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:23.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:23.685 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:46:23.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:23.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:23.685 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:46:23.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:23.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:23.685 | INFO | Uploaded 36GiB to GPU in 1.99815s (18.0167GiB/s) | 20260506 19:46:23.685 | INFO | Downloaded 34GiB to GPU in 1.9243s (17.6687GiB/s) | 20260506 19:46:28.822 | INFO | Analyzing chunk frames = [3974144,3974656) | 20260506 19:46:28.822 | INFO | MemInfo: { | 20260506 19:46:28.822 | INFO | "gpu" : " 17 GB", | 20260506 19:46:28.822 | INFO | "huge" : "", | 20260506 19:46:28.822 | INFO | "huge_shared" : " 27 GB", | 20260506 19:46:28.822 | INFO | "malloc" : " 14 GB", | 20260506 19:46:28.822 | INFO | "pinned" : " 29 GB" | 20260506 19:46:28.822 | INFO | } | 20260506 19:46:28.822 | INFO | Perf report for the last 5.13694 seconds: | 20260506 19:46:28.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:46:28.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:28.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:28.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:28.822 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:46:28.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:28.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:28.822 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:46:28.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:28.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:28.822 | INFO | Uploaded 36GiB to GPU in 1.96782s (18.2944GiB/s) | 20260506 19:46:28.822 | INFO | Downloaded 34GiB to GPU in 1.92111s (17.6981GiB/s) | 20260506 19:46:33.960 | INFO | Analyzing chunk frames = [3974656,3975168) | 20260506 19:46:33.960 | INFO | MemInfo: { | 20260506 19:46:33.960 | INFO | "gpu" : " 17 GB", | 20260506 19:46:33.960 | INFO | "huge" : "", | 20260506 19:46:33.960 | INFO | "huge_shared" : " 27 GB", | 20260506 19:46:33.960 | INFO | "malloc" : " 14 GB", | 20260506 19:46:33.960 | INFO | "pinned" : " 29 GB" | 20260506 19:46:33.960 | INFO | } | 20260506 19:46:33.961 | INFO | Perf report for the last 5.13862 seconds: | 20260506 19:46:33.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:46:33.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:33.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:33.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:33.961 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 19:46:33.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:33.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:33.961 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:46:33.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:33.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:33.961 | INFO | Uploaded 36GiB to GPU in 1.92712s (18.6808GiB/s) | 20260506 19:46:33.961 | INFO | Downloaded 34GiB to GPU in 1.87315s (18.1512GiB/s) | 20260506 19:46:39.105 | INFO | Analyzing chunk frames = [3975168,3975680) | 20260506 19:46:39.105 | INFO | MemInfo: { | 20260506 19:46:39.105 | INFO | "gpu" : " 17 GB", | 20260506 19:46:39.105 | INFO | "huge" : "", | 20260506 19:46:39.105 | INFO | "huge_shared" : " 27 GB", | 20260506 19:46:39.105 | INFO | "malloc" : " 14 GB", | 20260506 19:46:39.105 | INFO | "pinned" : " 29 GB" | 20260506 19:46:39.105 | INFO | } | 20260506 19:46:39.105 | INFO | Perf report for the last 5.14477 seconds: | 20260506 19:46:39.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:46:39.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:39.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:39.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:39.105 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 19:46:39.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:39.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:39.105 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:46:39.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:39.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:39.106 | INFO | Uploaded 36GiB to GPU in 2.01758s (17.8431GiB/s) | 20260506 19:46:39.106 | INFO | Downloaded 34GiB to GPU in 1.91495s (17.7551GiB/s) | 20260506 19:46:44.249 | INFO | Analyzing chunk frames = [3975680,3976192) | 20260506 19:46:44.249 | INFO | MemInfo: { | 20260506 19:46:44.249 | INFO | "gpu" : " 17 GB", | 20260506 19:46:44.249 | INFO | "huge" : "", | 20260506 19:46:44.249 | INFO | "huge_shared" : " 27 GB", | 20260506 19:46:44.249 | INFO | "malloc" : " 14 GB", | 20260506 19:46:44.249 | INFO | "pinned" : " 29 GB" | 20260506 19:46:44.249 | INFO | } | 20260506 19:46:44.249 | INFO | Perf report for the last 5.14352 seconds: | 20260506 19:46:44.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:46:44.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:44.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:44.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:44.249 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 19:46:44.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:44.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:44.249 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 19:46:44.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:44.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:44.249 | INFO | Uploaded 36GiB to GPU in 2.04215s (17.6285GiB/s) | 20260506 19:46:44.249 | INFO | Downloaded 34GiB to GPU in 1.96122s (17.3361GiB/s) | 20260506 19:46:49.389 | INFO | Analyzing chunk frames = [3976192,3976704) | 20260506 19:46:49.389 | INFO | MemInfo: { | 20260506 19:46:49.389 | INFO | "gpu" : " 17 GB", | 20260506 19:46:49.389 | INFO | "huge" : "", | 20260506 19:46:49.389 | INFO | "huge_shared" : " 27 GB", | 20260506 19:46:49.389 | INFO | "malloc" : " 14 GB", | 20260506 19:46:49.389 | INFO | "pinned" : " 29 GB" | 20260506 19:46:49.389 | INFO | } | 20260506 19:46:49.389 | INFO | Perf report for the last 5.14038 seconds: | 20260506 19:46:49.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:46:49.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:49.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:49.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:49.389 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:46:49.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:49.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:49.389 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:46:49.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:49.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:49.390 | INFO | Uploaded 36GiB to GPU in 2.00526s (17.9528GiB/s) | 20260506 19:46:49.390 | INFO | Downloaded 34GiB to GPU in 1.98391s (17.1378GiB/s) | 20260506 19:46:54.528 | INFO | Analyzing chunk frames = [3976704,3977216) | 20260506 19:46:54.528 | INFO | MemInfo: { | 20260506 19:46:54.528 | INFO | "gpu" : " 17 GB", | 20260506 19:46:54.528 | INFO | "huge" : "", | 20260506 19:46:54.528 | INFO | "huge_shared" : " 27 GB", | 20260506 19:46:54.528 | INFO | "malloc" : " 14 GB", | 20260506 19:46:54.528 | INFO | "pinned" : " 29 GB" | 20260506 19:46:54.528 | INFO | } | 20260506 19:46:54.528 | INFO | Perf report for the last 5.13908 seconds: | 20260506 19:46:54.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:46:54.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:54.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:54.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:54.528 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:46:54.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:54.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:54.528 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:46:54.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:54.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:54.529 | INFO | Uploaded 36GiB to GPU in 1.96029s (18.3646GiB/s) | 20260506 19:46:54.529 | INFO | Downloaded 34GiB to GPU in 1.97787s (17.1902GiB/s) | 20260506 19:46:54.633 | INFO | Starting a round of Baz IO | 20260506 19:46:57.776 | INFO | Populating RTMetrics report | 20260506 19:46:57.776 | INFO | RTMetrics report populated | 20260506 19:46:57.776 | INFO | Opening tmp RTMetrics json file | 20260506 19:46:57.981 | INFO | Serializing RTMetrics to json | 20260506 19:46:58.046 | INFO | Serialization done, swapping tmp file now... | 20260506 19:46:58.046 | INFO | RTMetrics done | 20260506 19:46:58.360 | INFO | Finished a round of Baz IO: Wrote 1.998 GiB in 3.726 seconds | 20260506 19:46:58.360 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.49% / 0.00% / 65.51% | 20260506 19:46:58.360 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.857385% / 1.98682e-05% / 8.34465e-05% | 20260506 19:46:58.360 | INFO | Baz IO overall: | 20260506 19:46:58.360 | INFO | BytesWritten : 2101409421982 | 20260506 19:46:58.360 | INFO | NumEvents : 1015630470528 | 20260506 19:46:59.662 | INFO | Analyzing chunk frames = [3977216,3977728) | 20260506 19:46:59.662 | INFO | MemInfo: { | 20260506 19:46:59.662 | INFO | "gpu" : " 17 GB", | 20260506 19:46:59.662 | INFO | "huge" : "", | 20260506 19:46:59.662 | INFO | "huge_shared" : " 27 GB", | 20260506 19:46:59.662 | INFO | "malloc" : " 14 GB", | 20260506 19:46:59.662 | INFO | "pinned" : " 29 GB" | 20260506 19:46:59.662 | INFO | } | 20260506 19:46:59.662 | INFO | Perf report for the last 5.13362 seconds: | 20260506 19:46:59.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:46:59.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:59.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:59.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:59.662 | INFO | ANALYSIS : 62.7%, 2.9, 95.4ms, 0 | 20260506 19:46:59.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:46:59.662 | INFO | RT_METRICS : 48.2%, 1.0, 25.3ms, 0 | 20260506 19:46:59.662 | INFO | PRE_HQ : 60.1%, 1.8, 57.4ms, 0 | 20260506 19:46:59.662 | INFO | BAZWRITER : 72.3%, 4.8, 177.9ms, 0 | 20260506 19:46:59.662 | INFO | BAZ_SAMPLING : 5.1%, 1.0, 2.7ms, 0 | 20260506 19:46:59.662 | INFO | Uploaded 36GiB to GPU in 2.04724s (17.5846GiB/s) | 20260506 19:46:59.662 | INFO | Downloaded 38GiB to GPU in 2.19843s (17.2851GiB/s) | 20260506 19:47:04.802 | INFO | Analyzing chunk frames = [3977728,3978240) | 20260506 19:47:04.802 | INFO | MemInfo: { | 20260506 19:47:04.802 | INFO | "gpu" : " 17 GB", | 20260506 19:47:04.802 | INFO | "huge" : "", | 20260506 19:47:04.802 | INFO | "huge_shared" : " 27 GB", | 20260506 19:47:04.802 | INFO | "malloc" : " 14 GB", | 20260506 19:47:04.802 | INFO | "pinned" : " 29 GB" | 20260506 19:47:04.802 | INFO | } | 20260506 19:47:04.802 | INFO | Perf report for the last 5.14003 seconds: | 20260506 19:47:04.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:47:04.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:04.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:04.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:04.802 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:47:04.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:04.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:04.802 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:47:04.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:04.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:04.802 | INFO | Uploaded 36GiB to GPU in 2.01606s (17.8566GiB/s) | 20260506 19:47:04.802 | INFO | Downloaded 34GiB to GPU in 1.92622s (17.6512GiB/s) | 20260506 19:47:09.941 | INFO | Analyzing chunk frames = [3978240,3978752) | 20260506 19:47:09.941 | INFO | MemInfo: { | 20260506 19:47:09.941 | INFO | "gpu" : " 17 GB", | 20260506 19:47:09.941 | INFO | "huge" : "", | 20260506 19:47:09.941 | INFO | "huge_shared" : " 27 GB", | 20260506 19:47:09.941 | INFO | "malloc" : " 14 GB", | 20260506 19:47:09.941 | INFO | "pinned" : " 29 GB" | 20260506 19:47:09.941 | INFO | } | 20260506 19:47:09.941 | INFO | Perf report for the last 5.13867 seconds: | 20260506 19:47:09.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:47:09.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:09.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:09.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:09.941 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:47:09.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:09.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:09.941 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:47:09.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:09.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:09.941 | INFO | Uploaded 36GiB to GPU in 1.99106s (18.0809GiB/s) | 20260506 19:47:09.941 | INFO | Downloaded 34GiB to GPU in 1.9252s (17.6605GiB/s) | 20260506 19:47:15.078 | INFO | Analyzing chunk frames = [3978752,3979264) | 20260506 19:47:15.078 | INFO | MemInfo: { | 20260506 19:47:15.078 | INFO | "gpu" : " 17 GB", | 20260506 19:47:15.078 | INFO | "huge" : "", | 20260506 19:47:15.078 | INFO | "huge_shared" : " 27 GB", | 20260506 19:47:15.078 | INFO | "malloc" : " 14 GB", | 20260506 19:47:15.078 | INFO | "pinned" : " 29 GB" | 20260506 19:47:15.078 | INFO | } | 20260506 19:47:15.078 | INFO | Perf report for the last 5.13709 seconds: | 20260506 19:47:15.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:47:15.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:15.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:15.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:15.078 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 19:47:15.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:15.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:15.078 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:47:15.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:15.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:15.078 | INFO | Uploaded 36GiB to GPU in 1.93197s (18.6338GiB/s) | 20260506 19:47:15.078 | INFO | Downloaded 34GiB to GPU in 1.86794s (18.2019GiB/s) | 20260506 19:47:20.218 | INFO | Analyzing chunk frames = [3979264,3979776) | 20260506 19:47:20.218 | INFO | MemInfo: { | 20260506 19:47:20.218 | INFO | "gpu" : " 17 GB", | 20260506 19:47:20.218 | INFO | "huge" : "", | 20260506 19:47:20.218 | INFO | "huge_shared" : " 27 GB", | 20260506 19:47:20.218 | INFO | "malloc" : " 14 GB", | 20260506 19:47:20.218 | INFO | "pinned" : " 29 GB" | 20260506 19:47:20.218 | INFO | } | 20260506 19:47:20.218 | INFO | Perf report for the last 5.14013 seconds: | 20260506 19:47:20.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:47:20.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:20.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:20.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:20.218 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 19:47:20.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:20.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:20.218 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:47:20.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:20.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:20.218 | INFO | Uploaded 36GiB to GPU in 1.97565s (18.2218GiB/s) | 20260506 19:47:20.218 | INFO | Downloaded 34GiB to GPU in 1.92552s (17.6576GiB/s) | 20260506 19:47:25.359 | INFO | Analyzing chunk frames = [3979776,3980288) | 20260506 19:47:25.359 | INFO | MemInfo: { | 20260506 19:47:25.359 | INFO | "gpu" : " 17 GB", | 20260506 19:47:25.359 | INFO | "huge" : "", | 20260506 19:47:25.359 | INFO | "huge_shared" : " 27 GB", | 20260506 19:47:25.359 | INFO | "malloc" : " 14 GB", | 20260506 19:47:25.359 | INFO | "pinned" : " 29 GB" | 20260506 19:47:25.359 | INFO | } | 20260506 19:47:25.359 | INFO | Perf report for the last 5.14107 seconds: | 20260506 19:47:25.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:47:25.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:25.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:25.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:25.359 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 19:47:25.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:25.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:25.360 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 19:47:25.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:25.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:25.360 | INFO | Uploaded 36GiB to GPU in 2.0018s (17.9838GiB/s) | 20260506 19:47:25.360 | INFO | Downloaded 34GiB to GPU in 1.93968s (17.5287GiB/s) | 20260506 19:47:30.499 | INFO | Analyzing chunk frames = [3980288,3980800) | 20260506 19:47:30.499 | INFO | MemInfo: { | 20260506 19:47:30.499 | INFO | "gpu" : " 17 GB", | 20260506 19:47:30.499 | INFO | "huge" : "", | 20260506 19:47:30.500 | INFO | "huge_shared" : " 27 GB", | 20260506 19:47:30.500 | INFO | "malloc" : " 14 GB", | 20260506 19:47:30.500 | INFO | "pinned" : " 29 GB" | 20260506 19:47:30.500 | INFO | } | 20260506 19:47:30.500 | INFO | Perf report for the last 5.14061 seconds: | 20260506 19:47:30.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:47:30.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:30.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:30.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:30.500 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 19:47:30.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:30.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:30.500 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:47:30.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:30.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:30.500 | INFO | Uploaded 36GiB to GPU in 2.03452s (17.6946GiB/s) | 20260506 19:47:30.500 | INFO | Downloaded 34GiB to GPU in 1.9952s (17.0409GiB/s) | 20260506 19:47:35.636 | INFO | Analyzing chunk frames = [3980800,3981312) | 20260506 19:47:35.636 | INFO | MemInfo: { | 20260506 19:47:35.636 | INFO | "gpu" : " 17 GB", | 20260506 19:47:35.636 | INFO | "huge" : "", | 20260506 19:47:35.636 | INFO | "huge_shared" : " 27 GB", | 20260506 19:47:35.636 | INFO | "malloc" : " 14 GB", | 20260506 19:47:35.636 | INFO | "pinned" : " 29 GB" | 20260506 19:47:35.636 | INFO | } | 20260506 19:47:35.636 | INFO | Perf report for the last 5.1365 seconds: | 20260506 19:47:35.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:47:35.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:35.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:35.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:35.636 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 19:47:35.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:35.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:35.636 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:47:35.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:35.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:35.636 | INFO | Uploaded 36GiB to GPU in 2.04062s (17.6417GiB/s) | 20260506 19:47:35.636 | INFO | Downloaded 34GiB to GPU in 1.98747s (17.1072GiB/s) | 20260506 19:47:35.636 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:47:35.636 | INFO | Total mem usage: 18 GiB | 20260506 19:47:35.636 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:47:35.636 | INFO | Recent High Water Marks Sum: 17 GiB (Savings: -5.89%) | 20260506 19:47:35.636 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:47:35.636 | INFO | High Water Breakdown: | 20260506 19:47:35.636 | INFO | 62.7% : 10.7 GiB ( 93.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:47:35.637 | INFO | 37.2% : 6.32 GiB ( 94.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:47:35.637 | INFO | 0.117% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:47:35.637 | INFO | ------------------------------------------ | 20260506 19:47:35.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:47:35.637 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:47:35.637 | INFO | Total mem usage: 26.5 GiB | 20260506 19:47:35.637 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:47:35.637 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:47:35.637 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:47:35.637 | INFO | High Water Breakdown: | 20260506 19:47:35.637 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:47:35.637 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:47:35.637 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:47:35.637 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:47:35.637 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:47:35.637 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:47:35.637 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:47:35.637 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:47:35.637 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:47:35.637 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:47:35.637 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:47:35.637 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:47:35.637 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:47:35.637 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:47:35.637 | INFO | ------------------------------------------ | 20260506 19:47:35.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:47:35.637 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:47:35.637 | INFO | Total mem usage: 25.6 GiB | 20260506 19:47:35.637 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:47:35.637 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:47:35.637 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:47:35.637 | INFO | High Water Breakdown: | 20260506 19:47:35.637 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:47:35.637 | INFO | ------------------------------------------ | 20260506 19:47:35.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:47:35.637 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:47:35.637 | INFO | Total mem usage: 15.7 GiB | 20260506 19:47:35.637 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:47:35.637 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:47:35.637 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 19:47:35.637 | INFO | High Water Breakdown: | 20260506 19:47:35.637 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:47:35.637 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:47:35.637 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:47:35.637 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:47:35.637 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:47:35.637 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:47:35.637 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:47:35.637 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:47:35.637 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:47:35.637 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:47:35.637 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:47:35.637 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:47:35.637 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:47:35.637 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:47:35.637 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:47:35.637 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:47:35.637 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:47:35.637 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:47:35.637 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:47:35.637 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:47:35.637 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:47:35.637 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:47:35.637 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:47:35.637 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:47:35.637 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:47:35.637 | INFO | ------------------------------------------ | 20260506 19:47:35.637 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:47:35.637 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:47:35.637 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:47:35.637 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:47:35.637 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.33e+01 / 2.49e+00 | 20260506 19:47:35.637 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 9.65e-03 / 6.04e-03 | 20260506 19:47:35.637 | INFO | PerfReport: 11.61%, Baseline -- 3.63e+00 / 4.40e+00, 5.56e-01 / 3.83e-01 | 20260506 19:47:35.637 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.48e+00, 1.84e-01 / 1.36e-01 | 20260506 19:47:35.637 | INFO | PerfReport: 15.97%, DME -- 3.99e+01 / 4.44e+01, 4.04e+00 / 2.88e+00 | 20260506 19:47:35.637 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.47e+00 / 1.91e+00 | 20260506 19:47:35.637 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.23e+00 / 2.79e+00, 3.78e-01 / 2.45e-01 | 20260506 19:47:35.637 | INFO | PerfReport: 3.35%, Metrics -- 1.04e+00 / 1.41e+00, 2.77e-01 / 1.67e-01 | 20260506 19:47:35.637 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.59e-01 / 4.07e-01, 5.63e-02 / 2.88e-02 | 20260506 19:47:35.637 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:47:35.637 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:47:35.637 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:47:35.637 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 19:47:35.637 | INFO | PerfReport: 21.48%, Upload -- 2.08e+01 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260506 19:47:35.637 | INFO | PerfReport: 21.18%, Download -- 2.05e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 19:47:35.637 | INFO | PerfReport: 32.23%, Compute -- 3.12e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 19:47:35.637 | INFO | PerfReport: 1.33%, LockUpload -- 1.29e+00 / 0.00e+00, 5.24e+00 / 0.00e+00 | 20260506 19:47:35.637 | INFO | PerfReport: 1.13%, LockDownload -- 1.09e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 19:47:35.637 | INFO | PerfReport: 22.64%, LockCompute -- 2.19e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 19:47:35.637 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260506 19:47:35.753 | INFO | Starting a round of Baz IO | 20260506 19:47:38.872 | INFO | Populating RTMetrics report | 20260506 19:47:38.873 | INFO | RTMetrics report populated | 20260506 19:47:38.873 | INFO | Opening tmp RTMetrics json file | 20260506 19:47:38.946 | INFO | Serializing RTMetrics to json | 20260506 19:47:38.973 | INFO | Serialization done, swapping tmp file now... | 20260506 19:47:38.975 | INFO | RTMetrics done | 20260506 19:47:39.008 | INFO | Finished a round of Baz IO: Wrote 1.998 GiB in 3.255 seconds | 20260506 19:47:39.008 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.48% / 0.00% / 65.51% | 20260506 19:47:39.008 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.853058% / 1.19209e-05% / 9.53674e-05% | 20260506 19:47:39.008 | INFO | Baz IO overall: | 20260506 19:47:39.008 | INFO | BytesWritten : 2103554942378 | 20260506 19:47:39.008 | INFO | NumEvents : 1016661133194 | 20260506 19:47:40.778 | INFO | Analyzing chunk frames = [3981312,3981824) | 20260506 19:47:40.778 | INFO | MemInfo: { | 20260506 19:47:40.778 | INFO | "gpu" : " 17 GB", | 20260506 19:47:40.778 | INFO | "huge" : "", | 20260506 19:47:40.778 | INFO | "huge_shared" : " 27 GB", | 20260506 19:47:40.778 | INFO | "malloc" : " 14 GB", | 20260506 19:47:40.778 | INFO | "pinned" : " 29 GB" | 20260506 19:47:40.778 | INFO | } | 20260506 19:47:40.778 | INFO | Perf report for the last 5.14166 seconds: | 20260506 19:47:40.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:47:40.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:40.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:40.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:40.778 | INFO | ANALYSIS : 62.4%, 2.9, 95.1ms, 0 | 20260506 19:47:40.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:40.778 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260506 19:47:40.778 | INFO | PRE_HQ : 59.8%, 1.8, 57.4ms, 0 | 20260506 19:47:40.778 | INFO | BAZWRITER : 63.0%, 4.8, 155.0ms, 0 | 20260506 19:47:40.778 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 19:47:40.778 | INFO | Uploaded 36GiB to GPU in 2.05794s (17.4933GiB/s) | 20260506 19:47:40.778 | INFO | Downloaded 38GiB to GPU in 2.21347s (17.1676GiB/s) | 20260506 19:47:45.913 | INFO | Analyzing chunk frames = [3981824,3982336) | 20260506 19:47:45.913 | INFO | MemInfo: { | 20260506 19:47:45.913 | INFO | "gpu" : " 17 GB", | 20260506 19:47:45.913 | INFO | "huge" : "", | 20260506 19:47:45.913 | INFO | "huge_shared" : " 27 GB", | 20260506 19:47:45.913 | INFO | "malloc" : " 14 GB", | 20260506 19:47:45.913 | INFO | "pinned" : " 29 GB" | 20260506 19:47:45.913 | INFO | } | 20260506 19:47:45.913 | INFO | Perf report for the last 5.13489 seconds: | 20260506 19:47:45.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:47:45.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:45.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:45.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:45.913 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:47:45.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:45.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:45.913 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:47:45.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:45.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:45.913 | INFO | Uploaded 36GiB to GPU in 2.01601s (17.857GiB/s) | 20260506 19:47:45.913 | INFO | Downloaded 34GiB to GPU in 1.92903s (17.6255GiB/s) | 20260506 19:47:51.054 | INFO | Analyzing chunk frames = [3982336,3982848) | 20260506 19:47:51.054 | INFO | MemInfo: { | 20260506 19:47:51.054 | INFO | "gpu" : " 17 GB", | 20260506 19:47:51.054 | INFO | "huge" : "", | 20260506 19:47:51.054 | INFO | "huge_shared" : " 27 GB", | 20260506 19:47:51.054 | INFO | "malloc" : " 14 GB", | 20260506 19:47:51.054 | INFO | "pinned" : " 29 GB" | 20260506 19:47:51.054 | INFO | } | 20260506 19:47:51.054 | INFO | Perf report for the last 5.14112 seconds: | 20260506 19:47:51.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:47:51.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:51.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:51.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:51.054 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 19:47:51.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:51.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:51.054 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:47:51.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:51.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:51.054 | INFO | Uploaded 36GiB to GPU in 1.99004s (18.0901GiB/s) | 20260506 19:47:51.054 | INFO | Downloaded 34GiB to GPU in 1.91655s (17.7402GiB/s) | 20260506 19:47:56.195 | INFO | Analyzing chunk frames = [3982848,3983360) | 20260506 19:47:56.195 | INFO | MemInfo: { | 20260506 19:47:56.195 | INFO | "gpu" : " 17 GB", | 20260506 19:47:56.195 | INFO | "huge" : "", | 20260506 19:47:56.195 | INFO | "huge_shared" : " 27 GB", | 20260506 19:47:56.195 | INFO | "malloc" : " 14 GB", | 20260506 19:47:56.195 | INFO | "pinned" : " 29 GB" | 20260506 19:47:56.195 | INFO | } | 20260506 19:47:56.195 | INFO | Perf report for the last 5.14125 seconds: | 20260506 19:47:56.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:47:56.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:56.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:56.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:56.195 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 19:47:56.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:56.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:56.195 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:47:56.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:56.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:47:56.195 | INFO | Uploaded 36GiB to GPU in 1.94103s (18.5469GiB/s) | 20260506 19:47:56.195 | INFO | Downloaded 34GiB to GPU in 1.87133s (18.1689GiB/s) | 20260506 19:48:01.335 | INFO | Analyzing chunk frames = [3983360,3983872) | 20260506 19:48:01.335 | INFO | MemInfo: { | 20260506 19:48:01.335 | INFO | "gpu" : " 17 GB", | 20260506 19:48:01.335 | INFO | "huge" : "", | 20260506 19:48:01.335 | INFO | "huge_shared" : " 27 GB", | 20260506 19:48:01.335 | INFO | "malloc" : " 14 GB", | 20260506 19:48:01.335 | INFO | "pinned" : " 29 GB" | 20260506 19:48:01.335 | INFO | } | 20260506 19:48:01.335 | INFO | Perf report for the last 5.14003 seconds: | 20260506 19:48:01.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:48:01.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:01.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:01.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:01.335 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 19:48:01.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:01.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:01.335 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:48:01.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:01.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:01.335 | INFO | Uploaded 36GiB to GPU in 2.02058s (17.8167GiB/s) | 20260506 19:48:01.336 | INFO | Downloaded 34GiB to GPU in 1.91367s (17.7669GiB/s) | 20260506 19:48:06.477 | INFO | Analyzing chunk frames = [3983872,3984384) | 20260506 19:48:06.478 | INFO | MemInfo: { | 20260506 19:48:06.478 | INFO | "gpu" : " 17 GB", | 20260506 19:48:06.478 | INFO | "huge" : "", | 20260506 19:48:06.478 | INFO | "huge_shared" : " 27 GB", | 20260506 19:48:06.478 | INFO | "malloc" : " 14 GB", | 20260506 19:48:06.478 | INFO | "pinned" : " 29 GB" | 20260506 19:48:06.478 | INFO | } | 20260506 19:48:06.478 | INFO | Perf report for the last 5.14253 seconds: | 20260506 19:48:06.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:48:06.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:06.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:06.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:06.478 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:48:06.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:06.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:06.478 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260506 19:48:06.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:06.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:06.478 | INFO | Uploaded 36GiB to GPU in 2.06873s (17.402GiB/s) | 20260506 19:48:06.478 | INFO | Downloaded 34GiB to GPU in 1.96421s (17.3098GiB/s) | 20260506 19:48:11.615 | INFO | Analyzing chunk frames = [3984384,3984896) | 20260506 19:48:11.615 | INFO | MemInfo: { | 20260506 19:48:11.615 | INFO | "gpu" : " 17 GB", | 20260506 19:48:11.615 | INFO | "huge" : "", | 20260506 19:48:11.615 | INFO | "huge_shared" : " 27 GB", | 20260506 19:48:11.615 | INFO | "malloc" : " 14 GB", | 20260506 19:48:11.615 | INFO | "pinned" : " 29 GB" | 20260506 19:48:11.615 | INFO | } | 20260506 19:48:11.615 | INFO | Perf report for the last 5.13745 seconds: | 20260506 19:48:11.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:48:11.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:11.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:11.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:11.615 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 19:48:11.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:11.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:11.615 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:48:11.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:11.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:11.615 | INFO | Uploaded 36GiB to GPU in 1.96872s (18.286GiB/s) | 20260506 19:48:11.615 | INFO | Downloaded 34GiB to GPU in 1.98401s (17.137GiB/s) | 20260506 19:48:16.753 | INFO | Analyzing chunk frames = [3984896,3985408) | 20260506 19:48:16.753 | INFO | MemInfo: { | 20260506 19:48:16.753 | INFO | "gpu" : " 17 GB", | 20260506 19:48:16.753 | INFO | "huge" : "", | 20260506 19:48:16.753 | INFO | "huge_shared" : " 27 GB", | 20260506 19:48:16.753 | INFO | "malloc" : " 14 GB", | 20260506 19:48:16.753 | INFO | "pinned" : " 29 GB" | 20260506 19:48:16.753 | INFO | } | 20260506 19:48:16.753 | INFO | Perf report for the last 5.13812 seconds: | 20260506 19:48:16.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:48:16.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:16.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:16.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:16.753 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:48:16.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:16.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:16.753 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 19:48:16.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:16.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:16.754 | INFO | Uploaded 36GiB to GPU in 2.05025s (17.5589GiB/s) | 20260506 19:48:16.754 | INFO | Downloaded 34GiB to GPU in 1.99686s (17.0267GiB/s) | 20260506 19:48:16.873 | INFO | Starting a round of Baz IO | 20260506 19:48:20.002 | INFO | Populating RTMetrics report | 20260506 19:48:20.002 | INFO | RTMetrics report populated | 20260506 19:48:20.002 | INFO | Opening tmp RTMetrics json file | 20260506 19:48:20.038 | INFO | Serializing RTMetrics to json | 20260506 19:48:20.075 | INFO | Serialization done, swapping tmp file now... | 20260506 19:48:20.076 | INFO | RTMetrics done | 20260506 19:48:20.547 | INFO | Finished a round of Baz IO: Wrote 1.998 GiB in 3.673 seconds | 20260506 19:48:20.547 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.48% / 0.00% / 65.52% | 20260506 19:48:20.547 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.853117% / 1.98682e-05% / 7.54992e-05% | 20260506 19:48:20.547 | INFO | Baz IO overall: | 20260506 19:48:20.547 | INFO | BytesWritten : 2105700375516 | 20260506 19:48:20.547 | INFO | NumEvents : 1017691892477 | 20260506 19:48:21.890 | INFO | Analyzing chunk frames = [3985408,3985920) | 20260506 19:48:21.890 | INFO | MemInfo: { | 20260506 19:48:21.890 | INFO | "gpu" : " 17 GB", | 20260506 19:48:21.890 | INFO | "huge" : "", | 20260506 19:48:21.890 | INFO | "huge_shared" : " 27 GB", | 20260506 19:48:21.890 | INFO | "malloc" : " 14 GB", | 20260506 19:48:21.890 | INFO | "pinned" : " 29 GB" | 20260506 19:48:21.890 | INFO | } | 20260506 19:48:21.890 | INFO | Perf report for the last 5.13712 seconds: | 20260506 19:48:21.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:48:21.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:21.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:21.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:21.890 | INFO | ANALYSIS : 62.6%, 2.9, 95.3ms, 0 | 20260506 19:48:21.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:21.890 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 19:48:21.891 | INFO | PRE_HQ : 59.9%, 1.9, 57.7ms, 0 | 20260506 19:48:21.891 | INFO | BAZWRITER : 70.9%, 4.8, 174.6ms, 0 | 20260506 19:48:21.891 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 19:48:21.891 | INFO | Uploaded 36GiB to GPU in 2.06335s (17.4473GiB/s) | 20260506 19:48:21.891 | INFO | Downloaded 38GiB to GPU in 2.21854s (17.1284GiB/s) | 20260506 19:48:27.029 | INFO | Analyzing chunk frames = [3985920,3986432) | 20260506 19:48:27.029 | INFO | MemInfo: { | 20260506 19:48:27.029 | INFO | "gpu" : " 17 GB", | 20260506 19:48:27.029 | INFO | "huge" : "", | 20260506 19:48:27.029 | INFO | "huge_shared" : " 27 GB", | 20260506 19:48:27.029 | INFO | "malloc" : " 14 GB", | 20260506 19:48:27.029 | INFO | "pinned" : " 29 GB" | 20260506 19:48:27.029 | INFO | } | 20260506 19:48:27.029 | INFO | Perf report for the last 5.13893 seconds: | 20260506 19:48:27.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:48:27.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:27.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:27.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:27.029 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:48:27.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:27.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:27.029 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 19:48:27.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:27.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:27.030 | INFO | Uploaded 36GiB to GPU in 2.01629s (17.8545GiB/s) | 20260506 19:48:27.030 | INFO | Downloaded 34GiB to GPU in 1.92252s (17.6851GiB/s) | 20260506 19:48:32.165 | INFO | Analyzing chunk frames = [3986432,3986944) | 20260506 19:48:32.165 | INFO | MemInfo: { | 20260506 19:48:32.165 | INFO | "gpu" : " 17 GB", | 20260506 19:48:32.165 | INFO | "huge" : "", | 20260506 19:48:32.165 | INFO | "huge_shared" : " 27 GB", | 20260506 19:48:32.165 | INFO | "malloc" : " 14 GB", | 20260506 19:48:32.165 | INFO | "pinned" : " 29 GB" | 20260506 19:48:32.165 | INFO | } | 20260506 19:48:32.165 | INFO | Perf report for the last 5.13583 seconds: | 20260506 19:48:32.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:48:32.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:32.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:32.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:32.165 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 19:48:32.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:32.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:32.165 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:48:32.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:32.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:32.165 | INFO | Uploaded 36GiB to GPU in 1.98371s (18.1478GiB/s) | 20260506 19:48:32.166 | INFO | Downloaded 34GiB to GPU in 1.91668s (17.739GiB/s) | 20260506 19:48:37.305 | INFO | Analyzing chunk frames = [3986944,3987456) | 20260506 19:48:37.305 | INFO | MemInfo: { | 20260506 19:48:37.305 | INFO | "gpu" : " 17 GB", | 20260506 19:48:37.305 | INFO | "huge" : "", | 20260506 19:48:37.305 | INFO | "huge_shared" : " 27 GB", | 20260506 19:48:37.305 | INFO | "malloc" : " 14 GB", | 20260506 19:48:37.305 | INFO | "pinned" : " 29 GB" | 20260506 19:48:37.305 | INFO | } | 20260506 19:48:37.305 | INFO | Perf report for the last 5.13986 seconds: | 20260506 19:48:37.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:48:37.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:37.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:37.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:37.305 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:48:37.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:37.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:37.305 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 19:48:37.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:37.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:37.305 | INFO | Uploaded 36GiB to GPU in 1.93257s (18.6281GiB/s) | 20260506 19:48:37.305 | INFO | Downloaded 34GiB to GPU in 1.86443s (18.2361GiB/s) | 20260506 19:48:42.448 | INFO | Analyzing chunk frames = [3987456,3987968) | 20260506 19:48:42.448 | INFO | MemInfo: { | 20260506 19:48:42.448 | INFO | "gpu" : " 17 GB", | 20260506 19:48:42.448 | INFO | "huge" : "", | 20260506 19:48:42.448 | INFO | "huge_shared" : " 27 GB", | 20260506 19:48:42.448 | INFO | "malloc" : " 14 GB", | 20260506 19:48:42.448 | INFO | "pinned" : " 29 GB" | 20260506 19:48:42.448 | INFO | } | 20260506 19:48:42.449 | INFO | Perf report for the last 5.14341 seconds: | 20260506 19:48:42.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:48:42.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:42.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:42.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:42.449 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260506 19:48:42.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:42.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:42.449 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:48:42.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:42.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:42.449 | INFO | Uploaded 36GiB to GPU in 2.02311s (17.7944GiB/s) | 20260506 19:48:42.449 | INFO | Downloaded 34GiB to GPU in 1.91194s (17.783GiB/s) | 20260506 19:48:47.591 | INFO | Analyzing chunk frames = [3987968,3988480) | 20260506 19:48:47.591 | INFO | MemInfo: { | 20260506 19:48:47.591 | INFO | "gpu" : " 17 GB", | 20260506 19:48:47.591 | INFO | "huge" : "", | 20260506 19:48:47.591 | INFO | "huge_shared" : " 27 GB", | 20260506 19:48:47.591 | INFO | "malloc" : " 14 GB", | 20260506 19:48:47.591 | INFO | "pinned" : " 29 GB" | 20260506 19:48:47.591 | INFO | } | 20260506 19:48:47.591 | INFO | Perf report for the last 5.14282 seconds: | 20260506 19:48:47.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:48:47.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:47.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:47.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:47.591 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 19:48:47.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:47.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:47.591 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 19:48:47.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:47.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:47.592 | INFO | Uploaded 36GiB to GPU in 2.05547s (17.5143GiB/s) | 20260506 19:48:47.592 | INFO | Downloaded 34GiB to GPU in 1.9593s (17.3532GiB/s) | 20260506 19:48:52.731 | INFO | Analyzing chunk frames = [3988480,3988992) | 20260506 19:48:52.731 | INFO | MemInfo: { | 20260506 19:48:52.731 | INFO | "gpu" : " 17 GB", | 20260506 19:48:52.731 | INFO | "huge" : "", | 20260506 19:48:52.731 | INFO | "huge_shared" : " 27 GB", | 20260506 19:48:52.731 | INFO | "malloc" : " 14 GB", | 20260506 19:48:52.731 | INFO | "pinned" : " 29 GB" | 20260506 19:48:52.731 | INFO | } | 20260506 19:48:52.731 | INFO | Perf report for the last 5.14009 seconds: | 20260506 19:48:52.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:48:52.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:52.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:52.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:52.732 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:48:52.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:52.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:52.732 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:48:52.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:52.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:52.732 | INFO | Uploaded 36GiB to GPU in 2.00283s (17.9746GiB/s) | 20260506 19:48:52.732 | INFO | Downloaded 34GiB to GPU in 1.98319s (17.1441GiB/s) | 20260506 19:48:57.870 | INFO | Analyzing chunk frames = [3988992,3989504) | 20260506 19:48:57.870 | INFO | MemInfo: { | 20260506 19:48:57.870 | INFO | "gpu" : " 17 GB", | 20260506 19:48:57.870 | INFO | "huge" : "", | 20260506 19:48:57.870 | INFO | "huge_shared" : " 27 GB", | 20260506 19:48:57.870 | INFO | "malloc" : " 14 GB", | 20260506 19:48:57.870 | INFO | "pinned" : " 29 GB" | 20260506 19:48:57.870 | INFO | } | 20260506 19:48:57.870 | INFO | Perf report for the last 5.13903 seconds: | 20260506 19:48:57.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:48:57.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:57.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:57.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:57.871 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:48:57.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:57.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:57.871 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:48:57.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:57.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:48:57.871 | INFO | Uploaded 36GiB to GPU in 1.99985s (18.0014GiB/s) | 20260506 19:48:57.871 | INFO | Downloaded 34GiB to GPU in 1.98573s (17.1222GiB/s) | 20260506 19:48:57.871 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:48:57.871 | INFO | Total mem usage: 18 GiB | 20260506 19:48:57.871 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:48:57.871 | INFO | Recent High Water Marks Sum: 16.3 GiB (Savings: -10.2%) | 20260506 19:48:57.871 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:48:57.871 | INFO | High Water Breakdown: | 20260506 19:48:57.871 | INFO | 62.1% : 10.2 GiB ( 88.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:48:57.871 | INFO | 37.7% : 6.17 GiB ( 92.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:48:57.871 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:48:57.871 | INFO | ------------------------------------------ | 20260506 19:48:57.871 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:48:57.871 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:48:57.871 | INFO | Total mem usage: 26.5 GiB | 20260506 19:48:57.871 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:48:57.871 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:48:57.871 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:48:57.871 | INFO | High Water Breakdown: | 20260506 19:48:57.871 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:48:57.871 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:48:57.871 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:48:57.871 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:48:57.871 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:48:57.871 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:48:57.871 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:48:57.871 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:48:57.871 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:48:57.871 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:48:57.871 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:48:57.871 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:48:57.871 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:48:57.871 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:48:57.871 | INFO | ------------------------------------------ | 20260506 19:48:57.871 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:48:57.871 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:48:57.871 | INFO | Total mem usage: 25.6 GiB | 20260506 19:48:57.871 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:48:57.872 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:48:57.872 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:48:57.872 | INFO | High Water Breakdown: | 20260506 19:48:57.872 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:48:57.872 | INFO | ------------------------------------------ | 20260506 19:48:57.872 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:48:57.872 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:48:57.872 | INFO | Total mem usage: 15.7 GiB | 20260506 19:48:57.872 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:48:57.872 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:48:57.872 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:48:57.872 | INFO | High Water Breakdown: | 20260506 19:48:57.872 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:48:57.872 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:48:57.872 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:48:57.872 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:48:57.872 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:48:57.872 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:48:57.872 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:48:57.872 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:48:57.872 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:48:57.872 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:48:57.872 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:48:57.872 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:48:57.872 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:48:57.872 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:48:57.872 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:48:57.872 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:48:57.872 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:48:57.872 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:48:57.872 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:48:57.872 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:48:57.872 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:48:57.872 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:48:57.872 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:48:57.872 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:48:57.872 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:48:57.872 | INFO | ------------------------------------------ | 20260506 19:48:57.872 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:48:57.872 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:48:57.872 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:48:57.872 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:48:57.872 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.55e+01, 1.35e+01 / 2.35e+00 | 20260506 19:48:57.872 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 1.04e-02 / 8.41e-03 | 20260506 19:48:57.872 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.40e+00, 5.49e-01 / 3.66e-01 | 20260506 19:48:57.872 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.47e+00, 1.81e-01 / 1.37e-01 | 20260506 19:48:57.872 | INFO | PerfReport: 15.97%, DME -- 3.99e+01 / 4.41e+01, 3.92e+00 / 2.10e+00 | 20260506 19:48:57.872 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.79e+01 / 2.09e+01, 1.48e+00 / 2.01e+00 | 20260506 19:48:57.872 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.23e+00 / 2.83e+00, 3.73e-01 / 2.01e-01 | 20260506 19:48:57.872 | INFO | PerfReport: 3.33%, Metrics -- 1.04e+00 / 1.36e+00, 2.71e-01 / 1.78e-01 | 20260506 19:48:57.872 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.60e-01 / 4.03e-01, 5.70e-02 / 2.63e-02 | 20260506 19:48:57.872 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:48:57.872 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:48:57.872 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:48:57.872 | INFO | PerfReport: Full Execution -- 9.70e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 19:48:57.872 | INFO | PerfReport: 21.61%, Upload -- 2.10e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260506 19:48:57.872 | INFO | PerfReport: 21.18%, Download -- 2.05e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 19:48:57.872 | INFO | PerfReport: 32.18%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 19:48:57.872 | INFO | PerfReport: 1.50%, LockUpload -- 1.46e+00 / 0.00e+00, 5.37e+00 / 0.00e+00 | 20260506 19:48:57.872 | INFO | PerfReport: 1.03%, LockDownload -- 1.00e+00 / 0.00e+00, 2.66e+00 / 0.00e+00 | 20260506 19:48:57.872 | INFO | PerfReport: 22.48%, LockCompute -- 2.18e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260506 19:48:57.872 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260506 19:48:57.975 | INFO | Starting a round of Baz IO | 20260506 19:49:01.109 | INFO | Populating RTMetrics report | 20260506 19:49:01.109 | INFO | RTMetrics report populated | 20260506 19:49:01.109 | INFO | Opening tmp RTMetrics json file | 20260506 19:49:01.178 | INFO | Serializing RTMetrics to json | 20260506 19:49:01.181 | INFO | Serialization done, swapping tmp file now... | 20260506 19:49:01.185 | INFO | RTMetrics done | 20260506 19:49:01.420 | INFO | Finished a round of Baz IO: Wrote 1.997 GiB in 3.445 seconds | 20260506 19:49:01.420 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.47% / 0.00% / 65.52% | 20260506 19:49:01.420 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.848909% / 7.94729e-06% / 7.54992e-05% | 20260506 19:49:01.420 | INFO | Baz IO overall: | 20260506 19:49:01.420 | INFO | BytesWritten : 2107844442441 | 20260506 19:49:01.420 | INFO | NumEvents : 1018721565583 | 20260506 19:49:03.006 | INFO | Analyzing chunk frames = [3989504,3990016) | 20260506 19:49:03.006 | INFO | MemInfo: { | 20260506 19:49:03.006 | INFO | "gpu" : " 17 GB", | 20260506 19:49:03.006 | INFO | "huge" : "", | 20260506 19:49:03.006 | INFO | "huge_shared" : " 27 GB", | 20260506 19:49:03.006 | INFO | "malloc" : " 14 GB", | 20260506 19:49:03.006 | INFO | "pinned" : " 29 GB" | 20260506 19:49:03.007 | INFO | } | 20260506 19:49:03.007 | INFO | Perf report for the last 5.13605 seconds: | 20260506 19:49:03.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:49:03.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:03.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:03.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:03.007 | INFO | ANALYSIS : 62.5%, 2.9, 94.9ms, 0 | 20260506 19:49:03.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:03.007 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 19:49:03.007 | INFO | PRE_HQ : 60.1%, 1.9, 57.8ms, 0 | 20260506 19:49:03.007 | INFO | BAZWRITER : 67.1%, 4.8, 165.7ms, 0 | 20260506 19:49:03.007 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 19:49:03.007 | INFO | Uploaded 36GiB to GPU in 2.07005s (17.3909GiB/s) | 20260506 19:49:03.007 | INFO | Downloaded 38GiB to GPU in 2.18776s (17.3694GiB/s) | 20260506 19:49:08.143 | INFO | Analyzing chunk frames = [3990016,3990528) | 20260506 19:49:08.143 | INFO | MemInfo: { | 20260506 19:49:08.143 | INFO | "gpu" : " 17 GB", | 20260506 19:49:08.143 | INFO | "huge" : "", | 20260506 19:49:08.143 | INFO | "huge_shared" : " 27 GB", | 20260506 19:49:08.143 | INFO | "malloc" : " 14 GB", | 20260506 19:49:08.143 | INFO | "pinned" : " 29 GB" | 20260506 19:49:08.143 | INFO | } | 20260506 19:49:08.143 | INFO | Perf report for the last 5.13621 seconds: | 20260506 19:49:08.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:49:08.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:08.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:08.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:08.143 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:49:08.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:08.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:08.143 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 19:49:08.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:08.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:08.143 | INFO | Uploaded 36GiB to GPU in 2.01761s (17.8429GiB/s) | 20260506 19:49:08.143 | INFO | Downloaded 34GiB to GPU in 1.932s (17.5984GiB/s) | 20260506 19:49:13.282 | INFO | Analyzing chunk frames = [3990528,3991040) | 20260506 19:49:13.282 | INFO | MemInfo: { | 20260506 19:49:13.282 | INFO | "gpu" : " 17 GB", | 20260506 19:49:13.282 | INFO | "huge" : "", | 20260506 19:49:13.282 | INFO | "huge_shared" : " 27 GB", | 20260506 19:49:13.282 | INFO | "malloc" : " 14 GB", | 20260506 19:49:13.282 | INFO | "pinned" : " 29 GB" | 20260506 19:49:13.282 | INFO | } | 20260506 19:49:13.282 | INFO | Perf report for the last 5.13912 seconds: | 20260506 19:49:13.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:49:13.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:13.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:13.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:13.282 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 19:49:13.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:13.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:13.282 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:49:13.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:13.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:13.282 | INFO | Uploaded 36GiB to GPU in 1.97629s (18.216GiB/s) | 20260506 19:49:13.282 | INFO | Downloaded 34GiB to GPU in 1.92159s (17.6937GiB/s) | 20260506 19:49:18.426 | INFO | Analyzing chunk frames = [3991040,3991552) | 20260506 19:49:18.426 | INFO | MemInfo: { | 20260506 19:49:18.426 | INFO | "gpu" : " 17 GB", | 20260506 19:49:18.426 | INFO | "huge" : "", | 20260506 19:49:18.426 | INFO | "huge_shared" : " 27 GB", | 20260506 19:49:18.426 | INFO | "malloc" : " 14 GB", | 20260506 19:49:18.426 | INFO | "pinned" : " 29 GB" | 20260506 19:49:18.426 | INFO | } | 20260506 19:49:18.426 | INFO | Perf report for the last 5.14412 seconds: | 20260506 19:49:18.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:49:18.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:18.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:18.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:18.426 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260506 19:49:18.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:18.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:18.426 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:49:18.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:18.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:18.426 | INFO | Uploaded 36GiB to GPU in 1.93778s (18.578GiB/s) | 20260506 19:49:18.426 | INFO | Downloaded 34GiB to GPU in 1.8674s (18.2071GiB/s) | 20260506 19:49:23.568 | INFO | Analyzing chunk frames = [3991552,3992064) | 20260506 19:49:23.568 | INFO | MemInfo: { | 20260506 19:49:23.568 | INFO | "gpu" : " 17 GB", | 20260506 19:49:23.568 | INFO | "huge" : "", | 20260506 19:49:23.568 | INFO | "huge_shared" : " 27 GB", | 20260506 19:49:23.568 | INFO | "malloc" : " 14 GB", | 20260506 19:49:23.568 | INFO | "pinned" : " 29 GB" | 20260506 19:49:23.568 | INFO | } | 20260506 19:49:23.568 | INFO | Perf report for the last 5.14215 seconds: | 20260506 19:49:23.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:49:23.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:23.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:23.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:23.568 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 19:49:23.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:23.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:23.568 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:49:23.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:23.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:23.569 | INFO | Uploaded 36GiB to GPU in 2.01448s (17.8706GiB/s) | 20260506 19:49:23.569 | INFO | Downloaded 34GiB to GPU in 1.90995s (17.8016GiB/s) | 20260506 19:49:28.705 | INFO | Analyzing chunk frames = [3992064,3992576) | 20260506 19:49:28.705 | INFO | MemInfo: { | 20260506 19:49:28.705 | INFO | "gpu" : " 17 GB", | 20260506 19:49:28.705 | INFO | "huge" : "", | 20260506 19:49:28.705 | INFO | "huge_shared" : " 27 GB", | 20260506 19:49:28.705 | INFO | "malloc" : " 14 GB", | 20260506 19:49:28.705 | INFO | "pinned" : " 29 GB" | 20260506 19:49:28.705 | INFO | } | 20260506 19:49:28.705 | INFO | Perf report for the last 5.13709 seconds: | 20260506 19:49:28.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:49:28.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:28.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:28.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:28.705 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 19:49:28.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:28.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:28.705 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:49:28.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:28.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:28.706 | INFO | Uploaded 36GiB to GPU in 2.03155s (17.7205GiB/s) | 20260506 19:49:28.706 | INFO | Downloaded 34GiB to GPU in 1.94508s (17.48GiB/s) | 20260506 19:49:33.846 | INFO | Analyzing chunk frames = [3992576,3993088) | 20260506 19:49:33.846 | INFO | MemInfo: { | 20260506 19:49:33.846 | INFO | "gpu" : " 17 GB", | 20260506 19:49:33.846 | INFO | "huge" : "", | 20260506 19:49:33.846 | INFO | "huge_shared" : " 27 GB", | 20260506 19:49:33.846 | INFO | "malloc" : " 14 GB", | 20260506 19:49:33.846 | INFO | "pinned" : " 29 GB" | 20260506 19:49:33.846 | INFO | } | 20260506 19:49:33.846 | INFO | Perf report for the last 5.14096 seconds: | 20260506 19:49:33.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:49:33.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:33.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:33.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:33.846 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:49:33.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:33.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:33.846 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 19:49:33.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:33.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:33.847 | INFO | Uploaded 36GiB to GPU in 1.99881s (18.0108GiB/s) | 20260506 19:49:33.847 | INFO | Downloaded 34GiB to GPU in 2.01064s (16.91GiB/s) | 20260506 19:49:38.980 | INFO | Analyzing chunk frames = [3993088,3993600) | 20260506 19:49:38.980 | INFO | MemInfo: { | 20260506 19:49:38.980 | INFO | "gpu" : " 17 GB", | 20260506 19:49:38.980 | INFO | "huge" : "", | 20260506 19:49:38.980 | INFO | "huge_shared" : " 27 GB", | 20260506 19:49:38.980 | INFO | "malloc" : " 14 GB", | 20260506 19:49:38.980 | INFO | "pinned" : " 29 GB" | 20260506 19:49:38.980 | INFO | } | 20260506 19:49:38.980 | INFO | Perf report for the last 5.13409 seconds: | 20260506 19:49:38.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:49:38.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:38.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:38.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:38.980 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:49:38.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:38.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:38.980 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:49:38.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:38.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:38.981 | INFO | Uploaded 36GiB to GPU in 1.97977s (18.184GiB/s) | 20260506 19:49:38.981 | INFO | Downloaded 34GiB to GPU in 1.99022s (17.0836GiB/s) | 20260506 19:49:39.087 | INFO | Starting a round of Baz IO | 20260506 19:49:42.220 | INFO | Populating RTMetrics report | 20260506 19:49:42.220 | INFO | RTMetrics report populated | 20260506 19:49:42.220 | INFO | Opening tmp RTMetrics json file | 20260506 19:49:42.300 | INFO | Serializing RTMetrics to json | 20260506 19:49:42.307 | INFO | Serialization done, swapping tmp file now... | 20260506 19:49:42.313 | INFO | RTMetrics done | 20260506 19:49:42.687 | INFO | Finished a round of Baz IO: Wrote 1.999 GiB in 3.599 seconds | 20260506 19:49:42.687 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.47% / 0.00% / 65.53% | 20260506 19:49:42.687 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.84902% / 3.97364e-06% / 7.15256e-05% | 20260506 19:49:42.687 | INFO | Baz IO overall: | 20260506 19:49:42.687 | INFO | BytesWritten : 2109990358368 | 20260506 19:49:42.687 | INFO | NumEvents : 1019752635074 | 20260506 19:49:44.119 | INFO | Analyzing chunk frames = [3993600,3994112) | 20260506 19:49:44.119 | INFO | MemInfo: { | 20260506 19:49:44.119 | INFO | "gpu" : " 17 GB", | 20260506 19:49:44.119 | INFO | "huge" : "", | 20260506 19:49:44.120 | INFO | "huge_shared" : " 27 GB", | 20260506 19:49:44.120 | INFO | "malloc" : " 14 GB", | 20260506 19:49:44.120 | INFO | "pinned" : " 29 GB" | 20260506 19:49:44.120 | INFO | } | 20260506 19:49:44.120 | INFO | Perf report for the last 5.13921 seconds: | 20260506 19:49:44.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:49:44.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:44.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:44.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:44.120 | INFO | ANALYSIS : 62.5%, 2.9, 95.2ms, 0 | 20260506 19:49:44.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:44.120 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260506 19:49:44.120 | INFO | PRE_HQ : 59.9%, 1.8, 57.5ms, 0 | 20260506 19:49:44.120 | INFO | BAZWRITER : 69.8%, 4.8, 171.6ms, 0 | 20260506 19:49:44.120 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 19:49:44.120 | INFO | Uploaded 36GiB to GPU in 2.00628s (17.9437GiB/s) | 20260506 19:49:44.120 | INFO | Downloaded 38GiB to GPU in 2.19158s (17.3391GiB/s) | 20260506 19:49:49.257 | INFO | Analyzing chunk frames = [3994112,3994624) | 20260506 19:49:49.257 | INFO | MemInfo: { | 20260506 19:49:49.257 | INFO | "gpu" : " 17 GB", | 20260506 19:49:49.257 | INFO | "huge" : "", | 20260506 19:49:49.257 | INFO | "huge_shared" : " 27 GB", | 20260506 19:49:49.257 | INFO | "malloc" : " 14 GB", | 20260506 19:49:49.257 | INFO | "pinned" : " 29 GB" | 20260506 19:49:49.257 | INFO | } | 20260506 19:49:49.257 | INFO | Perf report for the last 5.13722 seconds: | 20260506 19:49:49.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:49:49.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:49.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:49.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:49.257 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:49:49.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:49.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:49.257 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:49:49.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:49.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:49.257 | INFO | Uploaded 36GiB to GPU in 1.99768s (18.0209GiB/s) | 20260506 19:49:49.257 | INFO | Downloaded 34GiB to GPU in 1.92975s (17.6189GiB/s) | 20260506 19:49:54.395 | INFO | Analyzing chunk frames = [3994624,3995136) | 20260506 19:49:54.395 | INFO | MemInfo: { | 20260506 19:49:54.395 | INFO | "gpu" : " 17 GB", | 20260506 19:49:54.395 | INFO | "huge" : "", | 20260506 19:49:54.396 | INFO | "huge_shared" : " 27 GB", | 20260506 19:49:54.396 | INFO | "malloc" : " 14 GB", | 20260506 19:49:54.396 | INFO | "pinned" : " 29 GB" | 20260506 19:49:54.396 | INFO | } | 20260506 19:49:54.396 | INFO | Perf report for the last 5.13875 seconds: | 20260506 19:49:54.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:49:54.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:54.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:54.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:54.396 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:49:54.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:54.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:54.396 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:49:54.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:54.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:54.396 | INFO | Uploaded 36GiB to GPU in 1.96388s (18.3311GiB/s) | 20260506 19:49:54.396 | INFO | Downloaded 34GiB to GPU in 1.92s (17.7083GiB/s) | 20260506 19:49:59.544 | INFO | Analyzing chunk frames = [3995136,3995648) | 20260506 19:49:59.544 | INFO | MemInfo: { | 20260506 19:49:59.544 | INFO | "gpu" : " 17 GB", | 20260506 19:49:59.544 | INFO | "huge" : "", | 20260506 19:49:59.544 | INFO | "huge_shared" : " 27 GB", | 20260506 19:49:59.544 | INFO | "malloc" : " 14 GB", | 20260506 19:49:59.544 | INFO | "pinned" : " 29 GB" | 20260506 19:49:59.544 | INFO | } | 20260506 19:49:59.544 | INFO | Perf report for the last 5.14821 seconds: | 20260506 19:49:59.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:49:59.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:59.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:59.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:59.544 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260506 19:49:59.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:59.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:59.544 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 19:49:59.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:59.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:49:59.544 | INFO | Uploaded 36GiB to GPU in 1.93081s (18.645GiB/s) | 20260506 19:49:59.544 | INFO | Downloaded 34GiB to GPU in 1.8696s (18.1857GiB/s) | 20260506 19:50:04.684 | INFO | Analyzing chunk frames = [3995648,3996160) | 20260506 19:50:04.684 | INFO | MemInfo: { | 20260506 19:50:04.684 | INFO | "gpu" : " 17 GB", | 20260506 19:50:04.684 | INFO | "huge" : "", | 20260506 19:50:04.684 | INFO | "huge_shared" : " 27 GB", | 20260506 19:50:04.684 | INFO | "malloc" : " 14 GB", | 20260506 19:50:04.684 | INFO | "pinned" : " 29 GB" | 20260506 19:50:04.684 | INFO | } | 20260506 19:50:04.685 | INFO | Perf report for the last 5.14066 seconds: | 20260506 19:50:04.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:50:04.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:04.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:04.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:04.685 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 19:50:04.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:04.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:04.685 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:50:04.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:04.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:04.685 | INFO | Uploaded 36GiB to GPU in 1.99529s (18.0425GiB/s) | 20260506 19:50:04.685 | INFO | Downloaded 34GiB to GPU in 1.91763s (17.7302GiB/s) | 20260506 19:50:09.823 | INFO | Analyzing chunk frames = [3996160,3996672) | 20260506 19:50:09.823 | INFO | MemInfo: { | 20260506 19:50:09.823 | INFO | "gpu" : " 17 GB", | 20260506 19:50:09.823 | INFO | "huge" : "", | 20260506 19:50:09.823 | INFO | "huge_shared" : " 27 GB", | 20260506 19:50:09.823 | INFO | "malloc" : " 14 GB", | 20260506 19:50:09.823 | INFO | "pinned" : " 29 GB" | 20260506 19:50:09.823 | INFO | } | 20260506 19:50:09.823 | INFO | Perf report for the last 5.13862 seconds: | 20260506 19:50:09.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:50:09.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:09.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:09.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:09.823 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 19:50:09.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:09.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:09.823 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:50:09.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:09.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:09.823 | INFO | Uploaded 36GiB to GPU in 2.01467s (17.8689GiB/s) | 20260506 19:50:09.823 | INFO | Downloaded 34GiB to GPU in 1.94928s (17.4423GiB/s) | 20260506 19:50:14.958 | INFO | Analyzing chunk frames = [3996672,3997184) | 20260506 19:50:14.958 | INFO | MemInfo: { | 20260506 19:50:14.958 | INFO | "gpu" : " 17 GB", | 20260506 19:50:14.958 | INFO | "huge" : "", | 20260506 19:50:14.958 | INFO | "huge_shared" : " 27 GB", | 20260506 19:50:14.958 | INFO | "malloc" : " 14 GB", | 20260506 19:50:14.958 | INFO | "pinned" : " 29 GB" | 20260506 19:50:14.958 | INFO | } | 20260506 19:50:14.958 | INFO | Perf report for the last 5.13505 seconds: | 20260506 19:50:14.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:50:14.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:14.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:14.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:14.958 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 19:50:14.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:14.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:14.958 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 19:50:14.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:14.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:14.958 | INFO | Uploaded 36GiB to GPU in 1.96859s (18.2872GiB/s) | 20260506 19:50:14.959 | INFO | Downloaded 34GiB to GPU in 2.00509s (16.9568GiB/s) | 20260506 19:50:20.093 | INFO | Analyzing chunk frames = [3997184,3997696) | 20260506 19:50:20.093 | INFO | MemInfo: { | 20260506 19:50:20.093 | INFO | "gpu" : " 17 GB", | 20260506 19:50:20.093 | INFO | "huge" : "", | 20260506 19:50:20.093 | INFO | "huge_shared" : " 27 GB", | 20260506 19:50:20.093 | INFO | "malloc" : " 14 GB", | 20260506 19:50:20.093 | INFO | "pinned" : " 29 GB" | 20260506 19:50:20.093 | INFO | } | 20260506 19:50:20.093 | INFO | Perf report for the last 5.13496 seconds: | 20260506 19:50:20.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:50:20.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:20.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:20.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:20.093 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 19:50:20.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:20.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:20.093 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 19:50:20.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:20.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:20.093 | INFO | Uploaded 36GiB to GPU in 1.98247s (18.1591GiB/s) | 20260506 19:50:20.094 | INFO | Downloaded 34GiB to GPU in 1.96878s (17.2695GiB/s) | 20260506 19:50:20.094 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:50:20.094 | INFO | Total mem usage: 18 GiB | 20260506 19:50:20.094 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:50:20.094 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -10.9%) | 20260506 19:50:20.094 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:50:20.094 | INFO | High Water Breakdown: | 20260506 19:50:20.094 | INFO | 62% : 10.1 GiB ( 88.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:50:20.094 | INFO | 37.8% : 6.14 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:50:20.094 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:50:20.094 | INFO | ------------------------------------------ | 20260506 19:50:20.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:50:20.094 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:50:20.094 | INFO | Total mem usage: 26.5 GiB | 20260506 19:50:20.094 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:50:20.094 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:50:20.094 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:50:20.094 | INFO | High Water Breakdown: | 20260506 19:50:20.094 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:50:20.094 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:50:20.094 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:50:20.094 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:50:20.094 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:50:20.094 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:50:20.094 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:50:20.094 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:50:20.094 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:50:20.094 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:50:20.094 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:50:20.094 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:50:20.094 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:50:20.094 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:50:20.094 | INFO | ------------------------------------------ | 20260506 19:50:20.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:50:20.094 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:50:20.094 | INFO | Total mem usage: 25.6 GiB | 20260506 19:50:20.094 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:50:20.094 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:50:20.094 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:50:20.094 | INFO | High Water Breakdown: | 20260506 19:50:20.094 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:50:20.094 | INFO | ------------------------------------------ | 20260506 19:50:20.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:50:20.094 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:50:20.094 | INFO | Total mem usage: 15.7 GiB | 20260506 19:50:20.094 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:50:20.094 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:50:20.094 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 19:50:20.094 | INFO | High Water Breakdown: | 20260506 19:50:20.094 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:50:20.094 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:50:20.094 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:50:20.094 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:50:20.094 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:50:20.094 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:50:20.094 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:50:20.094 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:50:20.094 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:50:20.094 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:50:20.094 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:50:20.094 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:50:20.094 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:50:20.094 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:50:20.094 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:50:20.094 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:50:20.094 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:50:20.094 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:50:20.094 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:50:20.094 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:50:20.094 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:50:20.094 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:50:20.094 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:50:20.094 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:50:20.094 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:50:20.094 | INFO | ------------------------------------------ | 20260506 19:50:20.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:50:20.094 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:50:20.094 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:50:20.094 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:50:20.094 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.61e+01, 1.34e+01 / 3.04e+00 | 20260506 19:50:20.094 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 1.00e-02 / 5.25e-03 | 20260506 19:50:20.094 | INFO | PerfReport: 11.63%, Baseline -- 3.63e+00 / 4.44e+00, 5.60e-01 / 3.25e-01 | 20260506 19:50:20.094 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.47e+00, 1.85e-01 / 1.25e-01 | 20260506 19:50:20.095 | INFO | PerfReport: 16.03%, DME -- 4.00e+01 / 4.49e+01, 4.38e+00 / 3.09e+00 | 20260506 19:50:20.095 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.40e+00 / 1.88e+00 | 20260506 19:50:20.095 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.24e+00 / 2.78e+00, 3.81e-01 / 2.43e-01 | 20260506 19:50:20.095 | INFO | PerfReport: 3.35%, Metrics -- 1.05e+00 / 1.38e+00, 2.78e-01 / 1.81e-01 | 20260506 19:50:20.095 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.59e-01 / 4.09e-01, 5.72e-02 / 2.74e-02 | 20260506 19:50:20.095 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:50:20.095 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:50:20.095 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:50:20.095 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 19:50:20.095 | INFO | PerfReport: 21.44%, Upload -- 2.08e+01 / 0.00e+00, 6.92e+00 / 0.00e+00 | 20260506 19:50:20.095 | INFO | PerfReport: 21.19%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 19:50:20.095 | INFO | PerfReport: 32.23%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 19:50:20.095 | INFO | PerfReport: 1.29%, LockUpload -- 1.25e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260506 19:50:20.095 | INFO | PerfReport: 1.16%, LockDownload -- 1.13e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 19:50:20.095 | INFO | PerfReport: 22.67%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 19:50:20.095 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260506 19:50:20.197 | INFO | Starting a round of Baz IO | 20260506 19:50:23.334 | INFO | Populating RTMetrics report | 20260506 19:50:23.334 | INFO | RTMetrics report populated | 20260506 19:50:23.334 | INFO | Opening tmp RTMetrics json file | 20260506 19:50:23.339 | INFO | Serializing RTMetrics to json | 20260506 19:50:23.374 | INFO | Serialization done, swapping tmp file now... | 20260506 19:50:23.400 | INFO | RTMetrics done | 20260506 19:50:23.492 | INFO | Finished a round of Baz IO: Wrote 1.997 GiB in 3.294 seconds | 20260506 19:50:23.492 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.47% / 0.00% / 65.53% | 20260506 19:50:23.492 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.846541% / 1.58946e-05% / 4.76837e-05% | 20260506 19:50:23.492 | INFO | Baz IO overall: | 20260506 19:50:23.492 | INFO | BytesWritten : 2112134884373 | 20260506 19:50:23.492 | INFO | NumEvents : 1020782604347 | 20260506 19:50:25.234 | INFO | Analyzing chunk frames = [3997696,3998208) | 20260506 19:50:25.234 | INFO | MemInfo: { | 20260506 19:50:25.234 | INFO | "gpu" : " 17 GB", | 20260506 19:50:25.234 | INFO | "huge" : "", | 20260506 19:50:25.234 | INFO | "huge_shared" : " 27 GB", | 20260506 19:50:25.234 | INFO | "malloc" : " 14 GB", | 20260506 19:50:25.234 | INFO | "pinned" : " 29 GB" | 20260506 19:50:25.234 | INFO | } | 20260506 19:50:25.234 | INFO | Perf report for the last 5.14107 seconds: | 20260506 19:50:25.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:50:25.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:25.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:25.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:25.234 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 19:50:25.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:25.234 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 19:50:25.234 | INFO | PRE_HQ : 60.1%, 1.8, 57.4ms, 0 | 20260506 19:50:25.234 | INFO | BAZWRITER : 63.8%, 4.6, 150.0ms, 0 | 20260506 19:50:25.234 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 19:50:25.235 | INFO | Uploaded 36GiB to GPU in 2.04543s (17.6002GiB/s) | 20260506 19:50:25.235 | INFO | Downloaded 38GiB to GPU in 2.19294s (17.3283GiB/s) | 20260506 19:50:30.375 | INFO | Analyzing chunk frames = [3998208,3998720) | 20260506 19:50:30.375 | INFO | MemInfo: { | 20260506 19:50:30.375 | INFO | "gpu" : " 17 GB", | 20260506 19:50:30.375 | INFO | "huge" : "", | 20260506 19:50:30.375 | INFO | "huge_shared" : " 27 GB", | 20260506 19:50:30.375 | INFO | "malloc" : " 14 GB", | 20260506 19:50:30.375 | INFO | "pinned" : " 29 GB" | 20260506 19:50:30.375 | INFO | } | 20260506 19:50:30.375 | INFO | Perf report for the last 5.14107 seconds: | 20260506 19:50:30.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:50:30.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:30.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:30.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:30.375 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:50:30.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:30.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:30.375 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:50:30.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:30.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:30.376 | INFO | Uploaded 36GiB to GPU in 2.01651s (17.8527GiB/s) | 20260506 19:50:30.376 | INFO | Downloaded 34GiB to GPU in 1.92636s (17.6498GiB/s) | 20260506 19:50:35.514 | INFO | Analyzing chunk frames = [3998720,3999232) | 20260506 19:50:35.514 | INFO | MemInfo: { | 20260506 19:50:35.514 | INFO | "gpu" : " 17 GB", | 20260506 19:50:35.514 | INFO | "huge" : "", | 20260506 19:50:35.514 | INFO | "huge_shared" : " 27 GB", | 20260506 19:50:35.514 | INFO | "malloc" : " 14 GB", | 20260506 19:50:35.514 | INFO | "pinned" : " 29 GB" | 20260506 19:50:35.514 | INFO | } | 20260506 19:50:35.515 | INFO | Perf report for the last 5.13918 seconds: | 20260506 19:50:35.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:50:35.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:35.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:35.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:35.515 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:50:35.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:35.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:35.515 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 19:50:35.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:35.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:35.515 | INFO | Uploaded 36GiB to GPU in 1.9665s (18.3066GiB/s) | 20260506 19:50:35.515 | INFO | Downloaded 34GiB to GPU in 1.91732s (17.7331GiB/s) | 20260506 19:50:40.659 | INFO | Analyzing chunk frames = [3999232,3999744) | 20260506 19:50:40.659 | INFO | MemInfo: { | 20260506 19:50:40.659 | INFO | "gpu" : " 17 GB", | 20260506 19:50:40.659 | INFO | "huge" : "", | 20260506 19:50:40.659 | INFO | "huge_shared" : " 27 GB", | 20260506 19:50:40.659 | INFO | "malloc" : " 14 GB", | 20260506 19:50:40.659 | INFO | "pinned" : " 29 GB" | 20260506 19:50:40.659 | INFO | } | 20260506 19:50:40.659 | INFO | Perf report for the last 5.14456 seconds: | 20260506 19:50:40.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:50:40.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:40.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:40.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:40.659 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 19:50:40.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:40.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:40.659 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:50:40.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:40.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:40.659 | INFO | Uploaded 36GiB to GPU in 1.92954s (18.6573GiB/s) | 20260506 19:50:40.659 | INFO | Downloaded 34GiB to GPU in 1.87504s (18.1329GiB/s) | 20260506 19:50:45.797 | INFO | Analyzing chunk frames = [3999744,4000256) | 20260506 19:50:45.797 | INFO | MemInfo: { | 20260506 19:50:45.797 | INFO | "gpu" : " 17 GB", | 20260506 19:50:45.798 | INFO | "huge" : "", | 20260506 19:50:45.798 | INFO | "huge_shared" : " 27 GB", | 20260506 19:50:45.798 | INFO | "malloc" : " 14 GB", | 20260506 19:50:45.798 | INFO | "pinned" : " 29 GB" | 20260506 19:50:45.798 | INFO | } | 20260506 19:50:45.798 | INFO | Perf report for the last 5.13849 seconds: | 20260506 19:50:45.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:50:45.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:45.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:45.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:45.798 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 19:50:45.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:45.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:45.798 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:50:45.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:45.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:45.798 | INFO | Uploaded 36GiB to GPU in 1.97232s (18.2526GiB/s) | 20260506 19:50:45.798 | INFO | Downloaded 34GiB to GPU in 1.93108s (17.6067GiB/s) | 20260506 19:50:50.933 | INFO | Analyzing chunk frames = [4000256,4000768) | 20260506 19:50:50.933 | INFO | MemInfo: { | 20260506 19:50:50.933 | INFO | "gpu" : " 17 GB", | 20260506 19:50:50.933 | INFO | "huge" : "", | 20260506 19:50:50.933 | INFO | "huge_shared" : " 27 GB", | 20260506 19:50:50.933 | INFO | "malloc" : " 14 GB", | 20260506 19:50:50.933 | INFO | "pinned" : " 29 GB" | 20260506 19:50:50.933 | INFO | } | 20260506 19:50:50.933 | INFO | Perf report for the last 5.13552 seconds: | 20260506 19:50:50.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:50:50.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:50.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:50.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:50.933 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 19:50:50.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:50.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:50.933 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:50:50.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:50.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:50.933 | INFO | Uploaded 36GiB to GPU in 1.98982s (18.0921GiB/s) | 20260506 19:50:50.933 | INFO | Downloaded 34GiB to GPU in 1.95415s (17.3989GiB/s) | 20260506 19:50:56.069 | INFO | Analyzing chunk frames = [4000768,4001280) | 20260506 19:50:56.069 | INFO | MemInfo: { | 20260506 19:50:56.069 | INFO | "gpu" : " 17 GB", | 20260506 19:50:56.069 | INFO | "huge" : "", | 20260506 19:50:56.069 | INFO | "huge_shared" : " 27 GB", | 20260506 19:50:56.069 | INFO | "malloc" : " 14 GB", | 20260506 19:50:56.069 | INFO | "pinned" : " 29 GB" | 20260506 19:50:56.069 | INFO | } | 20260506 19:50:56.070 | INFO | Perf report for the last 5.13638 seconds: | 20260506 19:50:56.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:50:56.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:56.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:56.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:56.070 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 19:50:56.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:56.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:56.070 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:50:56.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:56.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:50:56.070 | INFO | Uploaded 36GiB to GPU in 1.99906s (18.0085GiB/s) | 20260506 19:50:56.070 | INFO | Downloaded 34GiB to GPU in 1.98619s (17.1182GiB/s) | 20260506 19:51:01.208 | INFO | Analyzing chunk frames = [4001280,4001792) | 20260506 19:51:01.208 | INFO | MemInfo: { | 20260506 19:51:01.208 | INFO | "gpu" : " 17 GB", | 20260506 19:51:01.208 | INFO | "huge" : "", | 20260506 19:51:01.208 | INFO | "huge_shared" : " 27 GB", | 20260506 19:51:01.208 | INFO | "malloc" : " 14 GB", | 20260506 19:51:01.208 | INFO | "pinned" : " 29 GB" | 20260506 19:51:01.208 | INFO | } | 20260506 19:51:01.208 | INFO | Perf report for the last 5.1389 seconds: | 20260506 19:51:01.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:51:01.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:01.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:01.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:01.209 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 19:51:01.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:01.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:01.209 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 19:51:01.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:01.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:01.209 | INFO | Uploaded 36GiB to GPU in 2.05692s (17.5019GiB/s) | 20260506 19:51:01.209 | INFO | Downloaded 34GiB to GPU in 1.99445s (17.0473GiB/s) | 20260506 19:51:01.325 | INFO | Starting a round of Baz IO | 20260506 19:51:04.462 | INFO | Populating RTMetrics report | 20260506 19:51:04.462 | INFO | RTMetrics report populated | 20260506 19:51:04.462 | INFO | Opening tmp RTMetrics json file | 20260506 19:51:04.466 | INFO | Serializing RTMetrics to json | 20260506 19:51:04.470 | INFO | Serialization done, swapping tmp file now... | 20260506 19:51:04.474 | INFO | RTMetrics done | 20260506 19:51:04.842 | INFO | Finished a round of Baz IO: Wrote 1.998 GiB in 3.516 seconds | 20260506 19:51:04.842 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.46% / 0.00% / 65.53% | 20260506 19:51:04.842 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.846907% / 3.97364e-06% / 8.34465e-05% | 20260506 19:51:04.842 | INFO | Baz IO overall: | 20260506 19:51:04.842 | INFO | BytesWritten : 2114280667127 | 20260506 19:51:04.842 | INFO | NumEvents : 1021813597450 | 20260506 19:51:06.347 | INFO | Analyzing chunk frames = [4001792,4002304) | 20260506 19:51:06.347 | INFO | MemInfo: { | 20260506 19:51:06.347 | INFO | "gpu" : " 17 GB", | 20260506 19:51:06.347 | INFO | "huge" : "", | 20260506 19:51:06.347 | INFO | "huge_shared" : " 27 GB", | 20260506 19:51:06.347 | INFO | "malloc" : " 14 GB", | 20260506 19:51:06.347 | INFO | "pinned" : " 29 GB" | 20260506 19:51:06.347 | INFO | } | 20260506 19:51:06.347 | INFO | Perf report for the last 5.13851 seconds: | 20260506 19:51:06.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:51:06.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:06.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:06.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:06.347 | INFO | ANALYSIS : 62.7%, 2.9, 95.4ms, 0 | 20260506 19:51:06.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:06.347 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 19:51:06.347 | INFO | PRE_HQ : 60.5%, 1.9, 59.1ms, 0 | 20260506 19:51:06.347 | INFO | BAZWRITER : 68.1%, 4.8, 168.5ms, 0 | 20260506 19:51:06.347 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 19:51:06.347 | INFO | Uploaded 36GiB to GPU in 2.06089s (17.4682GiB/s) | 20260506 19:51:06.347 | INFO | Downloaded 38GiB to GPU in 2.21685s (17.1414GiB/s) | 20260506 19:51:11.489 | INFO | Analyzing chunk frames = [4002304,4002816) | 20260506 19:51:11.489 | INFO | MemInfo: { | 20260506 19:51:11.489 | INFO | "gpu" : " 17 GB", | 20260506 19:51:11.489 | INFO | "huge" : "", | 20260506 19:51:11.489 | INFO | "huge_shared" : " 27 GB", | 20260506 19:51:11.489 | INFO | "malloc" : " 14 GB", | 20260506 19:51:11.489 | INFO | "pinned" : " 29 GB" | 20260506 19:51:11.489 | INFO | } | 20260506 19:51:11.489 | INFO | Perf report for the last 5.14237 seconds: | 20260506 19:51:11.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:51:11.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:11.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:11.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:11.489 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:51:11.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:11.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:11.489 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:51:11.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:11.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:11.490 | INFO | Uploaded 36GiB to GPU in 2.0202s (17.82GiB/s) | 20260506 19:51:11.490 | INFO | Downloaded 34GiB to GPU in 1.92967s (17.6196GiB/s) | 20260506 19:51:16.635 | INFO | Analyzing chunk frames = [4002816,4003328) | 20260506 19:51:16.635 | INFO | MemInfo: { | 20260506 19:51:16.635 | INFO | "gpu" : " 17 GB", | 20260506 19:51:16.635 | INFO | "huge" : "", | 20260506 19:51:16.635 | INFO | "huge_shared" : " 27 GB", | 20260506 19:51:16.635 | INFO | "malloc" : " 14 GB", | 20260506 19:51:16.635 | INFO | "pinned" : " 29 GB" | 20260506 19:51:16.635 | INFO | } | 20260506 19:51:16.635 | INFO | Perf report for the last 5.14569 seconds: | 20260506 19:51:16.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:51:16.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:16.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:16.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:16.635 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 19:51:16.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:16.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:16.635 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:51:16.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:16.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:16.635 | INFO | Uploaded 36GiB to GPU in 1.99411s (18.0532GiB/s) | 20260506 19:51:16.635 | INFO | Downloaded 34GiB to GPU in 1.91907s (17.7169GiB/s) | 20260506 19:51:21.771 | INFO | Analyzing chunk frames = [4003328,4003840) | 20260506 19:51:21.771 | INFO | MemInfo: { | 20260506 19:51:21.771 | INFO | "gpu" : " 17 GB", | 20260506 19:51:21.771 | INFO | "huge" : "", | 20260506 19:51:21.771 | INFO | "huge_shared" : " 27 GB", | 20260506 19:51:21.771 | INFO | "malloc" : " 14 GB", | 20260506 19:51:21.771 | INFO | "pinned" : " 29 GB" | 20260506 19:51:21.771 | INFO | } | 20260506 19:51:21.771 | INFO | Perf report for the last 5.13564 seconds: | 20260506 19:51:21.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:51:21.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:21.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:21.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:21.771 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 19:51:21.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:21.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:21.771 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:51:21.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:21.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:21.771 | INFO | Uploaded 36GiB to GPU in 1.94888s (18.4721GiB/s) | 20260506 19:51:21.771 | INFO | Downloaded 34GiB to GPU in 1.87386s (18.1444GiB/s) | 20260506 19:51:26.913 | INFO | Analyzing chunk frames = [4003840,4004352) | 20260506 19:51:26.913 | INFO | MemInfo: { | 20260506 19:51:26.913 | INFO | "gpu" : " 17 GB", | 20260506 19:51:26.913 | INFO | "huge" : "", | 20260506 19:51:26.913 | INFO | "huge_shared" : " 27 GB", | 20260506 19:51:26.913 | INFO | "malloc" : " 14 GB", | 20260506 19:51:26.913 | INFO | "pinned" : " 29 GB" | 20260506 19:51:26.913 | INFO | } | 20260506 19:51:26.913 | INFO | Perf report for the last 5.14264 seconds: | 20260506 19:51:26.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:51:26.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:26.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:26.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:26.913 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 19:51:26.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:26.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:26.913 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:51:26.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:26.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:26.914 | INFO | Uploaded 36GiB to GPU in 2.0151s (17.8651GiB/s) | 20260506 19:51:26.914 | INFO | Downloaded 34GiB to GPU in 1.90684s (17.8305GiB/s) | 20260506 19:51:32.048 | INFO | Analyzing chunk frames = [4004352,4004864) | 20260506 19:51:32.048 | INFO | MemInfo: { | 20260506 19:51:32.048 | INFO | "gpu" : " 17 GB", | 20260506 19:51:32.048 | INFO | "huge" : "", | 20260506 19:51:32.048 | INFO | "huge_shared" : " 27 GB", | 20260506 19:51:32.048 | INFO | "malloc" : " 14 GB", | 20260506 19:51:32.048 | INFO | "pinned" : " 29 GB" | 20260506 19:51:32.048 | INFO | } | 20260506 19:51:32.048 | INFO | Perf report for the last 5.13437 seconds: | 20260506 19:51:32.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:51:32.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:32.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:32.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:32.048 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 19:51:32.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:32.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:32.048 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 19:51:32.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:32.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:32.048 | INFO | Uploaded 36GiB to GPU in 2.01701s (17.8482GiB/s) | 20260506 19:51:32.048 | INFO | Downloaded 34GiB to GPU in 1.93422s (17.5781GiB/s) | 20260506 19:51:37.185 | INFO | Analyzing chunk frames = [4004864,4005376) | 20260506 19:51:37.185 | INFO | MemInfo: { | 20260506 19:51:37.185 | INFO | "gpu" : " 17 GB", | 20260506 19:51:37.185 | INFO | "huge" : "", | 20260506 19:51:37.185 | INFO | "huge_shared" : " 27 GB", | 20260506 19:51:37.185 | INFO | "malloc" : " 14 GB", | 20260506 19:51:37.185 | INFO | "pinned" : " 29 GB" | 20260506 19:51:37.185 | INFO | } | 20260506 19:51:37.186 | INFO | Perf report for the last 5.13778 seconds: | 20260506 19:51:37.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:51:37.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:37.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:37.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:37.186 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 19:51:37.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:37.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:37.186 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 19:51:37.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:37.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:37.186 | INFO | Uploaded 36GiB to GPU in 2.0018s (17.9838GiB/s) | 20260506 19:51:37.186 | INFO | Downloaded 34GiB to GPU in 1.98744s (17.1074GiB/s) | 20260506 19:51:42.325 | INFO | Analyzing chunk frames = [4005376,4005888) | 20260506 19:51:42.325 | INFO | MemInfo: { | 20260506 19:51:42.325 | INFO | "gpu" : " 17 GB", | 20260506 19:51:42.325 | INFO | "huge" : "", | 20260506 19:51:42.325 | INFO | "huge_shared" : " 27 GB", | 20260506 19:51:42.325 | INFO | "malloc" : " 14 GB", | 20260506 19:51:42.325 | INFO | "pinned" : " 29 GB" | 20260506 19:51:42.325 | INFO | } | 20260506 19:51:42.325 | INFO | Perf report for the last 5.1397 seconds: | 20260506 19:51:42.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:51:42.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:42.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:42.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:42.325 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:51:42.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:42.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:42.325 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 19:51:42.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:42.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:42.325 | INFO | Uploaded 36GiB to GPU in 1.97908s (18.1903GiB/s) | 20260506 19:51:42.326 | INFO | Downloaded 34GiB to GPU in 1.97773s (17.1914GiB/s) | 20260506 19:51:42.326 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:51:42.326 | INFO | Total mem usage: 18 GiB | 20260506 19:51:42.326 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:51:42.326 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.2%) | 20260506 19:51:42.326 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:51:42.326 | INFO | High Water Breakdown: | 20260506 19:51:42.326 | INFO | 61.5% : 9.62 GiB ( 84.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:51:42.326 | INFO | 38.3% : 5.99 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:51:42.326 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:51:42.326 | INFO | ------------------------------------------ | 20260506 19:51:42.326 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:51:42.326 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:51:42.326 | INFO | Total mem usage: 26.5 GiB | 20260506 19:51:42.326 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:51:42.326 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:51:42.326 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:51:42.326 | INFO | High Water Breakdown: | 20260506 19:51:42.326 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:51:42.326 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:51:42.326 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:51:42.326 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:51:42.326 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:51:42.326 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:51:42.326 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:51:42.326 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:51:42.326 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:51:42.326 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:51:42.326 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:51:42.326 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:51:42.326 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:51:42.326 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:51:42.326 | INFO | ------------------------------------------ | 20260506 19:51:42.326 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:51:42.326 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:51:42.326 | INFO | Total mem usage: 25.6 GiB | 20260506 19:51:42.326 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:51:42.326 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:51:42.326 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:51:42.326 | INFO | High Water Breakdown: | 20260506 19:51:42.326 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:51:42.326 | INFO | ------------------------------------------ | 20260506 19:51:42.326 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:51:42.326 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:51:42.326 | INFO | Total mem usage: 15.7 GiB | 20260506 19:51:42.326 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:51:42.326 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:51:42.326 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:51:42.326 | INFO | High Water Breakdown: | 20260506 19:51:42.326 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:51:42.326 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:51:42.326 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:51:42.326 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:51:42.326 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:51:42.326 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:51:42.326 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:51:42.326 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:51:42.326 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:51:42.326 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:51:42.326 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:51:42.326 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:51:42.326 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:51:42.326 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:51:42.326 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:51:42.326 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:51:42.326 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:51:42.326 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:51:42.326 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:51:42.326 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:51:42.326 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:51:42.326 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:51:42.326 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:51:42.326 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:51:42.326 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:51:42.326 | INFO | ------------------------------------------ | 20260506 19:51:42.326 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:51:42.326 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:51:42.326 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:51:42.326 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:51:42.326 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.56e+01, 1.30e+01 / 2.24e+00 | 20260506 19:51:42.326 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.01e-01, 1.01e-02 / 7.13e-03 | 20260506 19:51:42.326 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.42e+00, 5.52e-01 / 3.46e-01 | 20260506 19:51:42.326 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.43e+00, 1.81e-01 / 1.46e-01 | 20260506 19:51:42.326 | INFO | PerfReport: 15.95%, DME -- 3.98e+01 / 4.43e+01, 3.93e+00 / 2.50e+00 | 20260506 19:51:42.326 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.47e+00 / 2.15e+00 | 20260506 19:51:42.326 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.23e+00 / 2.74e+00, 3.76e-01 / 2.49e-01 | 20260506 19:51:42.326 | INFO | PerfReport: 3.34%, Metrics -- 1.04e+00 / 1.40e+00, 2.79e-01 / 2.09e-01 | 20260506 19:51:42.326 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.59e-01 / 4.08e-01, 5.70e-02 / 2.18e-02 | 20260506 19:51:42.326 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:51:42.326 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:51:42.326 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:51:42.326 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 19:51:42.326 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260506 19:51:42.326 | INFO | PerfReport: 21.19%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 19:51:42.326 | INFO | PerfReport: 32.23%, Compute -- 3.12e+01 / 0.00e+00, 1.30e+01 / 0.00e+00 | 20260506 19:51:42.326 | INFO | PerfReport: 1.34%, LockUpload -- 1.29e+00 / 0.00e+00, 5.16e+00 / 0.00e+00 | 20260506 19:51:42.326 | INFO | PerfReport: 1.15%, LockDownload -- 1.11e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 19:51:42.327 | INFO | PerfReport: 22.55%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 19:51:42.327 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.34e-02 / 0.00e+00 | 20260506 19:51:42.425 | INFO | Starting a round of Baz IO | 20260506 19:51:45.570 | INFO | Populating RTMetrics report | 20260506 19:51:45.570 | INFO | RTMetrics report populated | 20260506 19:51:45.570 | INFO | Opening tmp RTMetrics json file | 20260506 19:51:45.608 | INFO | Serializing RTMetrics to json | 20260506 19:51:45.611 | INFO | Serialization done, swapping tmp file now... | 20260506 19:51:45.616 | INFO | RTMetrics done | 20260506 19:51:46.042 | INFO | Finished a round of Baz IO: Wrote 1.997 GiB in 3.616 seconds | 20260506 19:51:46.042 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.46% / 0.00% / 65.54% | 20260506 19:51:46.042 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.843783% / 7.94729e-06% / 7.54992e-05% | 20260506 19:51:46.042 | INFO | Baz IO overall: | 20260506 19:51:46.042 | INFO | BytesWritten : 2116424697173 | 20260506 19:51:46.042 | INFO | NumEvents : 1022843392763 | 20260506 19:51:47.464 | INFO | Analyzing chunk frames = [4005888,4006400) | 20260506 19:51:47.464 | INFO | MemInfo: { | 20260506 19:51:47.464 | INFO | "gpu" : " 17 GB", | 20260506 19:51:47.464 | INFO | "huge" : "", | 20260506 19:51:47.464 | INFO | "huge_shared" : " 27 GB", | 20260506 19:51:47.464 | INFO | "malloc" : " 14 GB", | 20260506 19:51:47.464 | INFO | "pinned" : " 29 GB" | 20260506 19:51:47.464 | INFO | } | 20260506 19:51:47.464 | INFO | Perf report for the last 5.13863 seconds: | 20260506 19:51:47.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:51:47.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:47.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:47.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:47.464 | INFO | ANALYSIS : 62.6%, 2.9, 95.5ms, 0 | 20260506 19:51:47.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:47.464 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 19:51:47.464 | INFO | PRE_HQ : 60.3%, 1.8, 57.5ms, 0 | 20260506 19:51:47.464 | INFO | BAZWRITER : 70.0%, 4.8, 172.7ms, 0 | 20260506 19:51:47.464 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 19:51:47.464 | INFO | Uploaded 36GiB to GPU in 2.04228s (17.6274GiB/s) | 20260506 19:51:47.464 | INFO | Downloaded 38GiB to GPU in 2.2051s (17.2328GiB/s) | 20260506 19:51:52.604 | INFO | Analyzing chunk frames = [4006400,4006912) | 20260506 19:51:52.604 | INFO | MemInfo: { | 20260506 19:51:52.604 | INFO | "gpu" : " 17 GB", | 20260506 19:51:52.604 | INFO | "huge" : "", | 20260506 19:51:52.604 | INFO | "huge_shared" : " 27 GB", | 20260506 19:51:52.604 | INFO | "malloc" : " 14 GB", | 20260506 19:51:52.604 | INFO | "pinned" : " 29 GB" | 20260506 19:51:52.604 | INFO | } | 20260506 19:51:52.605 | INFO | Perf report for the last 5.14065 seconds: | 20260506 19:51:52.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:51:52.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:52.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:52.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:52.605 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 19:51:52.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:52.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:52.605 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:51:52.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:52.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:52.605 | INFO | Uploaded 36GiB to GPU in 1.98457s (18.14GiB/s) | 20260506 19:51:52.605 | INFO | Downloaded 34GiB to GPU in 1.92071s (17.7018GiB/s) | 20260506 19:51:57.744 | INFO | Analyzing chunk frames = [4006912,4007424) | 20260506 19:51:57.744 | INFO | MemInfo: { | 20260506 19:51:57.744 | INFO | "gpu" : " 17 GB", | 20260506 19:51:57.744 | INFO | "huge" : "", | 20260506 19:51:57.744 | INFO | "huge_shared" : " 27 GB", | 20260506 19:51:57.744 | INFO | "malloc" : " 14 GB", | 20260506 19:51:57.744 | INFO | "pinned" : " 29 GB" | 20260506 19:51:57.744 | INFO | } | 20260506 19:51:57.744 | INFO | Perf report for the last 5.13929 seconds: | 20260506 19:51:57.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:51:57.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:57.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:57.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:57.744 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:51:57.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:57.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:57.744 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:51:57.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:57.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:51:57.744 | INFO | Uploaded 36GiB to GPU in 1.96204s (18.3482GiB/s) | 20260506 19:51:57.744 | INFO | Downloaded 34GiB to GPU in 1.91828s (17.7242GiB/s) | 20260506 19:52:02.885 | INFO | Analyzing chunk frames = [4007424,4007936) | 20260506 19:52:02.885 | INFO | MemInfo: { | 20260506 19:52:02.885 | INFO | "gpu" : " 17 GB", | 20260506 19:52:02.886 | INFO | "huge" : "", | 20260506 19:52:02.886 | INFO | "huge_shared" : " 27 GB", | 20260506 19:52:02.886 | INFO | "malloc" : " 14 GB", | 20260506 19:52:02.886 | INFO | "pinned" : " 29 GB" | 20260506 19:52:02.886 | INFO | } | 20260506 19:52:02.886 | INFO | Perf report for the last 5.14184 seconds: | 20260506 19:52:02.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:52:02.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:02.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:02.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:02.886 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:52:02.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:02.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:02.886 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:52:02.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:02.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:02.886 | INFO | Uploaded 36GiB to GPU in 1.92284s (18.7223GiB/s) | 20260506 19:52:02.886 | INFO | Downloaded 34GiB to GPU in 1.87268s (18.1558GiB/s) | 20260506 19:52:08.024 | INFO | Analyzing chunk frames = [4007936,4008448) | 20260506 19:52:08.024 | INFO | MemInfo: { | 20260506 19:52:08.024 | INFO | "gpu" : " 17 GB", | 20260506 19:52:08.024 | INFO | "huge" : "", | 20260506 19:52:08.024 | INFO | "huge_shared" : " 27 GB", | 20260506 19:52:08.024 | INFO | "malloc" : " 14 GB", | 20260506 19:52:08.024 | INFO | "pinned" : " 29 GB" | 20260506 19:52:08.024 | INFO | } | 20260506 19:52:08.025 | INFO | Perf report for the last 5.13884 seconds: | 20260506 19:52:08.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:52:08.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:08.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:08.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:08.025 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 19:52:08.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:08.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:08.025 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:52:08.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:08.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:08.025 | INFO | Uploaded 36GiB to GPU in 2.00618s (17.9445GiB/s) | 20260506 19:52:08.025 | INFO | Downloaded 34GiB to GPU in 1.90928s (17.8077GiB/s) | 20260506 19:52:13.160 | INFO | Analyzing chunk frames = [4008448,4008960) | 20260506 19:52:13.160 | INFO | MemInfo: { | 20260506 19:52:13.160 | INFO | "gpu" : " 17 GB", | 20260506 19:52:13.160 | INFO | "huge" : "", | 20260506 19:52:13.160 | INFO | "huge_shared" : " 27 GB", | 20260506 19:52:13.160 | INFO | "malloc" : " 14 GB", | 20260506 19:52:13.161 | INFO | "pinned" : " 29 GB" | 20260506 19:52:13.161 | INFO | } | 20260506 19:52:13.161 | INFO | Perf report for the last 5.13608 seconds: | 20260506 19:52:13.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:52:13.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:13.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:13.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:13.161 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 19:52:13.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:13.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:13.161 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 19:52:13.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:13.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:13.161 | INFO | Uploaded 36GiB to GPU in 2.00646s (17.9421GiB/s) | 20260506 19:52:13.161 | INFO | Downloaded 34GiB to GPU in 1.96481s (17.3045GiB/s) | 20260506 19:52:18.300 | INFO | Analyzing chunk frames = [4008960,4009472) | 20260506 19:52:18.300 | INFO | MemInfo: { | 20260506 19:52:18.301 | INFO | "gpu" : " 17 GB", | 20260506 19:52:18.301 | INFO | "huge" : "", | 20260506 19:52:18.301 | INFO | "huge_shared" : " 27 GB", | 20260506 19:52:18.301 | INFO | "malloc" : " 14 GB", | 20260506 19:52:18.301 | INFO | "pinned" : " 29 GB" | 20260506 19:52:18.301 | INFO | } | 20260506 19:52:18.301 | INFO | Perf report for the last 5.14009 seconds: | 20260506 19:52:18.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:52:18.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:18.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:18.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:18.301 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 19:52:18.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:18.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:18.301 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:52:18.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:18.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:18.301 | INFO | Uploaded 36GiB to GPU in 2.01778s (17.8414GiB/s) | 20260506 19:52:18.301 | INFO | Downloaded 34GiB to GPU in 1.99443s (17.0475GiB/s) | 20260506 19:52:23.439 | INFO | Analyzing chunk frames = [4009472,4009984) | 20260506 19:52:23.439 | INFO | MemInfo: { | 20260506 19:52:23.439 | INFO | "gpu" : " 17 GB", | 20260506 19:52:23.439 | INFO | "huge" : "", | 20260506 19:52:23.440 | INFO | "huge_shared" : " 27 GB", | 20260506 19:52:23.440 | INFO | "malloc" : " 14 GB", | 20260506 19:52:23.440 | INFO | "pinned" : " 29 GB" | 20260506 19:52:23.440 | INFO | } | 20260506 19:52:23.440 | INFO | Perf report for the last 5.13895 seconds: | 20260506 19:52:23.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:52:23.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:23.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:23.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:23.440 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 19:52:23.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:23.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:23.440 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:52:23.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:23.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:23.440 | INFO | Uploaded 36GiB to GPU in 1.98068s (18.1756GiB/s) | 20260506 19:52:23.440 | INFO | Downloaded 34GiB to GPU in 1.98828s (17.1002GiB/s) | 20260506 19:52:23.569 | INFO | Starting a round of Baz IO | 20260506 19:52:26.681 | INFO | Populating RTMetrics report | 20260506 19:52:26.681 | INFO | RTMetrics report populated | 20260506 19:52:26.681 | INFO | Opening tmp RTMetrics json file | 20260506 19:52:26.699 | INFO | Serializing RTMetrics to json | 20260506 19:52:26.706 | INFO | Serialization done, swapping tmp file now... | 20260506 19:52:26.711 | INFO | RTMetrics done | 20260506 19:52:26.944 | INFO | Finished a round of Baz IO: Wrote 1.996 GiB in 3.375 seconds | 20260506 19:52:26.944 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.45% / 0.00% / 65.54% | 20260506 19:52:26.944 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.841645% / 1.19209e-05% / 6.35783e-05% | 20260506 19:52:26.944 | INFO | Baz IO overall: | 20260506 19:52:26.944 | INFO | BytesWritten : 2118567662226 | 20260506 19:52:26.944 | INFO | NumEvents : 1023872642692 | 20260506 19:52:28.581 | INFO | Analyzing chunk frames = [4009984,4010496) | 20260506 19:52:28.581 | INFO | MemInfo: { | 20260506 19:52:28.581 | INFO | "gpu" : " 17 GB", | 20260506 19:52:28.581 | INFO | "huge" : "", | 20260506 19:52:28.581 | INFO | "huge_shared" : " 27 GB", | 20260506 19:52:28.581 | INFO | "malloc" : " 14 GB", | 20260506 19:52:28.581 | INFO | "pinned" : " 29 GB" | 20260506 19:52:28.581 | INFO | } | 20260506 19:52:28.581 | INFO | Perf report for the last 5.14137 seconds: | 20260506 19:52:28.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:52:28.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:28.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:28.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:28.581 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 19:52:28.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:28.581 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260506 19:52:28.581 | INFO | PRE_HQ : 59.8%, 1.8, 57.2ms, 0 | 20260506 19:52:28.581 | INFO | BAZWRITER : 65.7%, 4.8, 160.9ms, 0 | 20260506 19:52:28.581 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 19:52:28.581 | INFO | Uploaded 36GiB to GPU in 2.05828s (17.4903GiB/s) | 20260506 19:52:28.581 | INFO | Downloaded 38GiB to GPU in 2.20696s (17.2183GiB/s) | 20260506 19:52:33.721 | INFO | Analyzing chunk frames = [4010496,4011008) | 20260506 19:52:33.721 | INFO | MemInfo: { | 20260506 19:52:33.721 | INFO | "gpu" : " 17 GB", | 20260506 19:52:33.721 | INFO | "huge" : "", | 20260506 19:52:33.721 | INFO | "huge_shared" : " 27 GB", | 20260506 19:52:33.721 | INFO | "malloc" : " 14 GB", | 20260506 19:52:33.721 | INFO | "pinned" : " 29 GB" | 20260506 19:52:33.721 | INFO | } | 20260506 19:52:33.721 | INFO | Perf report for the last 5.14035 seconds: | 20260506 19:52:33.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:52:33.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:33.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:33.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:33.721 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 19:52:33.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:33.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:33.721 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:52:33.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:33.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:33.722 | INFO | Uploaded 36GiB to GPU in 2.01818s (17.8378GiB/s) | 20260506 19:52:33.722 | INFO | Downloaded 34GiB to GPU in 1.93031s (17.6137GiB/s) | 20260506 19:52:38.861 | INFO | Analyzing chunk frames = [4011008,4011520) | 20260506 19:52:38.861 | INFO | MemInfo: { | 20260506 19:52:38.861 | INFO | "gpu" : " 17 GB", | 20260506 19:52:38.861 | INFO | "huge" : "", | 20260506 19:52:38.861 | INFO | "huge_shared" : " 27 GB", | 20260506 19:52:38.861 | INFO | "malloc" : " 14 GB", | 20260506 19:52:38.861 | INFO | "pinned" : " 29 GB" | 20260506 19:52:38.861 | INFO | } | 20260506 19:52:38.861 | INFO | Perf report for the last 5.13993 seconds: | 20260506 19:52:38.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:52:38.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:38.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:38.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:38.861 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:52:38.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:38.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:38.861 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:52:38.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:38.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:38.862 | INFO | Uploaded 36GiB to GPU in 1.98431s (18.1423GiB/s) | 20260506 19:52:38.862 | INFO | Downloaded 34GiB to GPU in 1.91418s (17.7622GiB/s) | 20260506 19:52:44.002 | INFO | Analyzing chunk frames = [4011520,4012032) | 20260506 19:52:44.002 | INFO | MemInfo: { | 20260506 19:52:44.002 | INFO | "gpu" : " 17 GB", | 20260506 19:52:44.002 | INFO | "huge" : "", | 20260506 19:52:44.002 | INFO | "huge_shared" : " 27 GB", | 20260506 19:52:44.002 | INFO | "malloc" : " 14 GB", | 20260506 19:52:44.002 | INFO | "pinned" : " 29 GB" | 20260506 19:52:44.002 | INFO | } | 20260506 19:52:44.002 | INFO | Perf report for the last 5.14072 seconds: | 20260506 19:52:44.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:52:44.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:44.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:44.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:44.002 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 19:52:44.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:44.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:44.002 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:52:44.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:44.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:44.002 | INFO | Uploaded 36GiB to GPU in 1.96059s (18.3618GiB/s) | 20260506 19:52:44.002 | INFO | Downloaded 34GiB to GPU in 1.87579s (18.1257GiB/s) | 20260506 19:52:49.137 | INFO | Analyzing chunk frames = [4012032,4012544) | 20260506 19:52:49.137 | INFO | MemInfo: { | 20260506 19:52:49.137 | INFO | "gpu" : " 17 GB", | 20260506 19:52:49.137 | INFO | "huge" : "", | 20260506 19:52:49.137 | INFO | "huge_shared" : " 27 GB", | 20260506 19:52:49.137 | INFO | "malloc" : " 14 GB", | 20260506 19:52:49.137 | INFO | "pinned" : " 29 GB" | 20260506 19:52:49.137 | INFO | } | 20260506 19:52:49.137 | INFO | Perf report for the last 5.13509 seconds: | 20260506 19:52:49.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:52:49.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:49.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:49.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:49.137 | INFO | ANALYSIS : 60.5%, 3.0, 94.0ms, 0 | 20260506 19:52:49.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:49.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:49.137 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:52:49.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:49.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:49.138 | INFO | Uploaded 36GiB to GPU in 2.01479s (17.8679GiB/s) | 20260506 19:52:49.138 | INFO | Downloaded 34GiB to GPU in 1.91956s (17.7124GiB/s) | 20260506 19:52:54.275 | INFO | Analyzing chunk frames = [4012544,4013056) | 20260506 19:52:54.275 | INFO | MemInfo: { | 20260506 19:52:54.275 | INFO | "gpu" : " 17 GB", | 20260506 19:52:54.275 | INFO | "huge" : "", | 20260506 19:52:54.275 | INFO | "huge_shared" : " 27 GB", | 20260506 19:52:54.275 | INFO | "malloc" : " 14 GB", | 20260506 19:52:54.275 | INFO | "pinned" : " 29 GB" | 20260506 19:52:54.275 | INFO | } | 20260506 19:52:54.275 | INFO | Perf report for the last 5.13809 seconds: | 20260506 19:52:54.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:52:54.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:54.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:54.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:54.275 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 19:52:54.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:54.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:54.275 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 19:52:54.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:54.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:54.276 | INFO | Uploaded 36GiB to GPU in 2.02604s (17.7686GiB/s) | 20260506 19:52:54.276 | INFO | Downloaded 34GiB to GPU in 1.94623s (17.4697GiB/s) | 20260506 19:52:59.414 | INFO | Analyzing chunk frames = [4013056,4013568) | 20260506 19:52:59.414 | INFO | MemInfo: { | 20260506 19:52:59.414 | INFO | "gpu" : " 17 GB", | 20260506 19:52:59.414 | INFO | "huge" : "", | 20260506 19:52:59.414 | INFO | "huge_shared" : " 27 GB", | 20260506 19:52:59.414 | INFO | "malloc" : " 14 GB", | 20260506 19:52:59.414 | INFO | "pinned" : " 29 GB" | 20260506 19:52:59.414 | INFO | } | 20260506 19:52:59.414 | INFO | Perf report for the last 5.13915 seconds: | 20260506 19:52:59.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:52:59.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:59.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:59.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:59.415 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 19:52:59.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:59.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:59.415 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:52:59.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:59.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:52:59.415 | INFO | Uploaded 36GiB to GPU in 1.99266s (18.0663GiB/s) | 20260506 19:52:59.415 | INFO | Downloaded 34GiB to GPU in 2.0042s (16.9643GiB/s) | 20260506 19:53:04.553 | INFO | Analyzing chunk frames = [4013568,4014080) | 20260506 19:53:04.553 | INFO | MemInfo: { | 20260506 19:53:04.553 | INFO | "gpu" : " 17 GB", | 20260506 19:53:04.553 | INFO | "huge" : "", | 20260506 19:53:04.553 | INFO | "huge_shared" : " 27 GB", | 20260506 19:53:04.553 | INFO | "malloc" : " 14 GB", | 20260506 19:53:04.553 | INFO | "pinned" : " 29 GB" | 20260506 19:53:04.553 | INFO | } | 20260506 19:53:04.553 | INFO | Perf report for the last 5.13842 seconds: | 20260506 19:53:04.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:53:04.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:04.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:04.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:04.553 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 19:53:04.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:04.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:04.553 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:53:04.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:04.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:04.553 | INFO | Uploaded 36GiB to GPU in 1.97934s (18.1878GiB/s) | 20260506 19:53:04.553 | INFO | Downloaded 34GiB to GPU in 1.98858s (17.0976GiB/s) | 20260506 19:53:04.553 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:53:04.553 | INFO | Total mem usage: 18 GiB | 20260506 19:53:04.553 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:53:04.553 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.4%) | 20260506 19:53:04.553 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:53:04.553 | INFO | High Water Breakdown: | 20260506 19:53:04.553 | INFO | 61.3% : 9.48 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:53:04.553 | INFO | 38.5% : 5.96 GiB ( 89.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:53:04.553 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:53:04.553 | INFO | ------------------------------------------ | 20260506 19:53:04.553 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:53:04.553 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:53:04.553 | INFO | Total mem usage: 26.5 GiB | 20260506 19:53:04.553 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:53:04.553 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:53:04.554 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:53:04.554 | INFO | High Water Breakdown: | 20260506 19:53:04.554 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:53:04.554 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:53:04.554 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:53:04.554 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:53:04.554 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:53:04.554 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:53:04.554 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:53:04.554 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:53:04.554 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:53:04.554 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:53:04.554 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:53:04.554 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:53:04.554 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:53:04.554 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:53:04.554 | INFO | ------------------------------------------ | 20260506 19:53:04.554 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:53:04.554 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:53:04.554 | INFO | Total mem usage: 25.6 GiB | 20260506 19:53:04.554 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:53:04.554 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:53:04.554 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:53:04.554 | INFO | High Water Breakdown: | 20260506 19:53:04.554 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:53:04.554 | INFO | ------------------------------------------ | 20260506 19:53:04.554 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:53:04.554 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:53:04.554 | INFO | Total mem usage: 15.7 GiB | 20260506 19:53:04.554 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:53:04.554 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:53:04.554 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:53:04.554 | INFO | High Water Breakdown: | 20260506 19:53:04.554 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:53:04.554 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:53:04.554 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:53:04.554 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:53:04.554 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:53:04.554 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:53:04.554 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:53:04.554 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:53:04.554 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:53:04.554 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:53:04.554 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:53:04.554 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:53:04.554 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:53:04.554 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:53:04.554 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:53:04.554 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:53:04.554 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:53:04.554 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:53:04.554 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:53:04.554 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:53:04.554 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:53:04.554 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:53:04.554 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:53:04.554 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:53:04.554 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:53:04.554 | INFO | ------------------------------------------ | 20260506 19:53:04.554 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:53:04.554 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:53:04.554 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:53:04.554 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:53:04.554 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.34e+01 / 2.37e+00 | 20260506 19:53:04.554 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 9.48e-03 / 5.59e-03 | 20260506 19:53:04.554 | INFO | PerfReport: 11.63%, Baseline -- 3.63e+00 / 4.44e+00, 5.51e-01 / 3.28e-01 | 20260506 19:53:04.554 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.81e-01 / 1.27e-01 | 20260506 19:53:04.554 | INFO | PerfReport: 16.00%, DME -- 4.00e+01 / 4.47e+01, 4.10e+00 / 2.33e+00 | 20260506 19:53:04.554 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.44e+00 / 1.73e+00 | 20260506 19:53:04.554 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.23e+00 / 2.80e+00, 3.77e-01 / 2.23e-01 | 20260506 19:53:04.554 | INFO | PerfReport: 3.34%, Metrics -- 1.04e+00 / 1.36e+00, 2.76e-01 / 1.58e-01 | 20260506 19:53:04.554 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.58e-01 / 4.04e-01, 5.74e-02 / 2.34e-02 | 20260506 19:53:04.554 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:53:04.554 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:53:04.554 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:53:04.554 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 19:53:04.554 | INFO | PerfReport: 21.47%, Upload -- 2.08e+01 / 0.00e+00, 6.91e+00 / 0.00e+00 | 20260506 19:53:04.554 | INFO | PerfReport: 21.20%, Download -- 2.05e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 19:53:04.554 | INFO | PerfReport: 32.21%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 19:53:04.554 | INFO | PerfReport: 1.32%, LockUpload -- 1.28e+00 / 0.00e+00, 5.08e+00 / 0.00e+00 | 20260506 19:53:04.554 | INFO | PerfReport: 1.18%, LockDownload -- 1.14e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 19:53:04.554 | INFO | PerfReport: 22.61%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 19:53:04.554 | INFO | PerfReport: 0.02%, Misc -- 1.50e-02 / 0.00e+00, 2.94e-02 / 0.00e+00 | 20260506 19:53:04.661 | INFO | Starting a round of Baz IO | 20260506 19:53:07.793 | INFO | Populating RTMetrics report | 20260506 19:53:07.794 | INFO | RTMetrics report populated | 20260506 19:53:07.794 | INFO | Opening tmp RTMetrics json file | 20260506 19:53:07.843 | INFO | Serializing RTMetrics to json | 20260506 19:53:07.846 | INFO | Serialization done, swapping tmp file now... | 20260506 19:53:07.851 | INFO | RTMetrics done | 20260506 19:53:08.302 | INFO | Finished a round of Baz IO: Wrote 1.995 GiB in 3.641 seconds | 20260506 19:53:08.302 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.45% / 0.00% / 65.55% | 20260506 19:53:08.302 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.838502% / 7.94729e-06% / 5.16574e-05% | 20260506 19:53:08.302 | INFO | Baz IO overall: | 20260506 19:53:08.302 | INFO | BytesWritten : 2120710225927 | 20260506 19:53:08.302 | INFO | NumEvents : 1024901513229 | 20260506 19:53:09.695 | INFO | Analyzing chunk frames = [4014080,4014592) | 20260506 19:53:09.695 | INFO | MemInfo: { | 20260506 19:53:09.695 | INFO | "gpu" : " 17 GB", | 20260506 19:53:09.695 | INFO | "huge" : "", | 20260506 19:53:09.695 | INFO | "huge_shared" : " 27 GB", | 20260506 19:53:09.695 | INFO | "malloc" : " 14 GB", | 20260506 19:53:09.695 | INFO | "pinned" : " 29 GB" | 20260506 19:53:09.695 | INFO | } | 20260506 19:53:09.695 | INFO | Perf report for the last 5.14194 seconds: | 20260506 19:53:09.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:53:09.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:09.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:09.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:09.695 | INFO | ANALYSIS : 62.4%, 2.9, 95.2ms, 0 | 20260506 19:53:09.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:09.695 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 19:53:09.695 | INFO | PRE_HQ : 60.2%, 1.9, 58.0ms, 0 | 20260506 19:53:09.695 | INFO | BAZWRITER : 70.5%, 4.8, 172.8ms, 0 | 20260506 19:53:09.695 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 19:53:09.695 | INFO | Uploaded 36GiB to GPU in 2.00074s (17.9934GiB/s) | 20260506 19:53:09.695 | INFO | Downloaded 38GiB to GPU in 2.1918s (17.3373GiB/s) | 20260506 19:53:14.834 | INFO | Analyzing chunk frames = [4014592,4015104) | 20260506 19:53:14.834 | INFO | MemInfo: { | 20260506 19:53:14.834 | INFO | "gpu" : " 17 GB", | 20260506 19:53:14.834 | INFO | "huge" : "", | 20260506 19:53:14.834 | INFO | "huge_shared" : " 27 GB", | 20260506 19:53:14.834 | INFO | "malloc" : " 14 GB", | 20260506 19:53:14.834 | INFO | "pinned" : " 29 GB" | 20260506 19:53:14.834 | INFO | } | 20260506 19:53:14.834 | INFO | Perf report for the last 5.13932 seconds: | 20260506 19:53:14.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:53:14.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:14.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:14.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:14.834 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:53:14.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:14.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:14.834 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:53:14.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:14.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:14.835 | INFO | Uploaded 36GiB to GPU in 1.98708s (18.117GiB/s) | 20260506 19:53:14.835 | INFO | Downloaded 34GiB to GPU in 1.92726s (17.6416GiB/s) | 20260506 19:53:19.977 | INFO | Analyzing chunk frames = [4015104,4015616) | 20260506 19:53:19.977 | INFO | MemInfo: { | 20260506 19:53:19.977 | INFO | "gpu" : " 17 GB", | 20260506 19:53:19.977 | INFO | "huge" : "", | 20260506 19:53:19.977 | INFO | "huge_shared" : " 27 GB", | 20260506 19:53:19.977 | INFO | "malloc" : " 14 GB", | 20260506 19:53:19.977 | INFO | "pinned" : " 29 GB" | 20260506 19:53:19.977 | INFO | } | 20260506 19:53:19.977 | INFO | Perf report for the last 5.14313 seconds: | 20260506 19:53:19.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:53:19.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:19.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:19.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:19.977 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 19:53:19.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:19.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:19.977 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:53:19.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:19.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:19.978 | INFO | Uploaded 36GiB to GPU in 1.96847s (18.2883GiB/s) | 20260506 19:53:19.978 | INFO | Downloaded 34GiB to GPU in 1.92047s (17.704GiB/s) | 20260506 19:53:25.114 | INFO | Analyzing chunk frames = [4015616,4016128) | 20260506 19:53:25.114 | INFO | MemInfo: { | 20260506 19:53:25.114 | INFO | "gpu" : " 17 GB", | 20260506 19:53:25.114 | INFO | "huge" : "", | 20260506 19:53:25.114 | INFO | "huge_shared" : " 27 GB", | 20260506 19:53:25.114 | INFO | "malloc" : " 14 GB", | 20260506 19:53:25.114 | INFO | "pinned" : " 29 GB" | 20260506 19:53:25.114 | INFO | } | 20260506 19:53:25.114 | INFO | Perf report for the last 5.13646 seconds: | 20260506 19:53:25.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:53:25.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:25.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:25.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:25.114 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:53:25.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:25.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:25.114 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 19:53:25.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:25.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:25.114 | INFO | Uploaded 36GiB to GPU in 1.95695s (18.396GiB/s) | 20260506 19:53:25.114 | INFO | Downloaded 34GiB to GPU in 1.87675s (18.1164GiB/s) | 20260506 19:53:30.254 | INFO | Analyzing chunk frames = [4016128,4016640) | 20260506 19:53:30.254 | INFO | MemInfo: { | 20260506 19:53:30.254 | INFO | "gpu" : " 17 GB", | 20260506 19:53:30.254 | INFO | "huge" : "", | 20260506 19:53:30.254 | INFO | "huge_shared" : " 27 GB", | 20260506 19:53:30.254 | INFO | "malloc" : " 14 GB", | 20260506 19:53:30.254 | INFO | "pinned" : " 29 GB" | 20260506 19:53:30.254 | INFO | } | 20260506 19:53:30.254 | INFO | Perf report for the last 5.13996 seconds: | 20260506 19:53:30.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:53:30.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:30.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:30.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:30.254 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 19:53:30.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:30.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:30.254 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:53:30.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:30.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:30.254 | INFO | Uploaded 36GiB to GPU in 1.98482s (18.1377GiB/s) | 20260506 19:53:30.254 | INFO | Downloaded 34GiB to GPU in 1.92337s (17.6773GiB/s) | 20260506 19:53:35.392 | INFO | Analyzing chunk frames = [4016640,4017152) | 20260506 19:53:35.392 | INFO | MemInfo: { | 20260506 19:53:35.392 | INFO | "gpu" : " 17 GB", | 20260506 19:53:35.392 | INFO | "huge" : "", | 20260506 19:53:35.392 | INFO | "huge_shared" : " 27 GB", | 20260506 19:53:35.392 | INFO | "malloc" : " 14 GB", | 20260506 19:53:35.392 | INFO | "pinned" : " 29 GB" | 20260506 19:53:35.392 | INFO | } | 20260506 19:53:35.392 | INFO | Perf report for the last 5.13869 seconds: | 20260506 19:53:35.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:53:35.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:35.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:35.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:35.393 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 19:53:35.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:35.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:35.393 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:53:35.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:35.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:35.393 | INFO | Uploaded 36GiB to GPU in 2.00404s (17.9638GiB/s) | 20260506 19:53:35.393 | INFO | Downloaded 34GiB to GPU in 1.95229s (17.4155GiB/s) | 20260506 19:53:40.529 | INFO | Analyzing chunk frames = [4017152,4017664) | 20260506 19:53:40.529 | INFO | MemInfo: { | 20260506 19:53:40.529 | INFO | "gpu" : " 17 GB", | 20260506 19:53:40.529 | INFO | "huge" : "", | 20260506 19:53:40.529 | INFO | "huge_shared" : " 27 GB", | 20260506 19:53:40.529 | INFO | "malloc" : " 14 GB", | 20260506 19:53:40.529 | INFO | "pinned" : " 29 GB" | 20260506 19:53:40.529 | INFO | } | 20260506 19:53:40.529 | INFO | Perf report for the last 5.13671 seconds: | 20260506 19:53:40.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:53:40.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:40.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:40.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:40.529 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 19:53:40.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:40.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:40.529 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:53:40.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:40.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:40.529 | INFO | Uploaded 36GiB to GPU in 1.99016s (18.089GiB/s) | 20260506 19:53:40.530 | INFO | Downloaded 34GiB to GPU in 2.00532s (16.9549GiB/s) | 20260506 19:53:45.673 | INFO | Analyzing chunk frames = [4017664,4018176) | 20260506 19:53:45.673 | INFO | MemInfo: { | 20260506 19:53:45.673 | INFO | "gpu" : " 17 GB", | 20260506 19:53:45.673 | INFO | "huge" : "", | 20260506 19:53:45.673 | INFO | "huge_shared" : " 27 GB", | 20260506 19:53:45.673 | INFO | "malloc" : " 14 GB", | 20260506 19:53:45.673 | INFO | "pinned" : " 29 GB" | 20260506 19:53:45.673 | INFO | } | 20260506 19:53:45.673 | INFO | Perf report for the last 5.14406 seconds: | 20260506 19:53:45.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:53:45.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:45.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:45.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:45.673 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 19:53:45.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:45.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:45.673 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260506 19:53:45.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:45.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:45.674 | INFO | Uploaded 36GiB to GPU in 2.01025s (17.9082GiB/s) | 20260506 19:53:45.674 | INFO | Downloaded 34GiB to GPU in 1.98929s (17.0915GiB/s) | 20260506 19:53:45.788 | INFO | Starting a round of Baz IO | 20260506 19:53:48.956 | INFO | Populating RTMetrics report | 20260506 19:53:48.956 | INFO | RTMetrics report populated | 20260506 19:53:48.956 | INFO | Opening tmp RTMetrics json file | 20260506 19:53:48.979 | INFO | Serializing RTMetrics to json | 20260506 19:53:49.049 | INFO | Serialization done, swapping tmp file now... | 20260506 19:53:49.070 | INFO | RTMetrics done | 20260506 19:53:49.660 | INFO | Finished a round of Baz IO: Wrote 1.996 GiB in 3.871 seconds | 20260506 19:53:49.660 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.44% / 0.00% / 65.55% | 20260506 19:53:49.660 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.8382% / 3.97364e-06% / 5.5631e-05% | 20260506 19:53:49.660 | INFO | Baz IO overall: | 20260506 19:53:49.660 | INFO | BytesWritten : 2122853493295 | 20260506 19:53:49.660 | INFO | NumEvents : 1025931286148 | 20260506 19:53:50.809 | INFO | Analyzing chunk frames = [4018176,4018688) | 20260506 19:53:50.809 | INFO | MemInfo: { | 20260506 19:53:50.809 | INFO | "gpu" : " 17 GB", | 20260506 19:53:50.809 | INFO | "huge" : "", | 20260506 19:53:50.809 | INFO | "huge_shared" : " 27 GB", | 20260506 19:53:50.809 | INFO | "malloc" : " 14 GB", | 20260506 19:53:50.809 | INFO | "pinned" : " 29 GB" | 20260506 19:53:50.809 | INFO | } | 20260506 19:53:50.809 | INFO | Perf report for the last 5.13606 seconds: | 20260506 19:53:50.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:53:50.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:50.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:50.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:50.809 | INFO | ANALYSIS : 63.0%, 2.8, 93.8ms, 0 | 20260506 19:53:50.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:50.809 | INFO | RT_METRICS : 48.4%, 1.0, 25.4ms, 0 | 20260506 19:53:50.809 | INFO | PRE_HQ : 61.1%, 1.9, 61.4ms, 0 | 20260506 19:53:50.809 | INFO | BAZWRITER : 75.1%, 4.8, 186.5ms, 0 | 20260506 19:53:50.809 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 19:53:50.810 | INFO | Uploaded 36GiB to GPU in 2.05567s (17.5125GiB/s) | 20260506 19:53:50.810 | INFO | Downloaded 38GiB to GPU in 2.18908s (17.3589GiB/s) | 20260506 19:53:55.950 | INFO | Analyzing chunk frames = [4018688,4019200) | 20260506 19:53:55.950 | INFO | MemInfo: { | 20260506 19:53:55.950 | INFO | "gpu" : " 17 GB", | 20260506 19:53:55.950 | INFO | "huge" : "", | 20260506 19:53:55.950 | INFO | "huge_shared" : " 27 GB", | 20260506 19:53:55.950 | INFO | "malloc" : " 14 GB", | 20260506 19:53:55.950 | INFO | "pinned" : " 29 GB" | 20260506 19:53:55.950 | INFO | } | 20260506 19:53:55.951 | INFO | Perf report for the last 5.14117 seconds: | 20260506 19:53:55.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:53:55.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:55.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:55.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:55.951 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:53:55.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:55.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:55.951 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 19:53:55.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:55.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:53:55.951 | INFO | Uploaded 36GiB to GPU in 2.00183s (17.9835GiB/s) | 20260506 19:53:55.951 | INFO | Downloaded 34GiB to GPU in 1.93718s (17.5513GiB/s) | 20260506 19:54:01.091 | INFO | Analyzing chunk frames = [4019200,4019712) | 20260506 19:54:01.091 | INFO | MemInfo: { | 20260506 19:54:01.091 | INFO | "gpu" : " 17 GB", | 20260506 19:54:01.092 | INFO | "huge" : "", | 20260506 19:54:01.092 | INFO | "huge_shared" : " 27 GB", | 20260506 19:54:01.092 | INFO | "malloc" : " 14 GB", | 20260506 19:54:01.092 | INFO | "pinned" : " 29 GB" | 20260506 19:54:01.092 | INFO | } | 20260506 19:54:01.092 | INFO | Perf report for the last 5.14121 seconds: | 20260506 19:54:01.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:54:01.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:01.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:01.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:01.092 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 19:54:01.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:01.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:01.092 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 19:54:01.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:01.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:01.092 | INFO | Uploaded 36GiB to GPU in 1.9624s (18.3449GiB/s) | 20260506 19:54:01.092 | INFO | Downloaded 34GiB to GPU in 1.92084s (17.7006GiB/s) | 20260506 19:54:06.229 | INFO | Analyzing chunk frames = [4019712,4020224) | 20260506 19:54:06.229 | INFO | MemInfo: { | 20260506 19:54:06.229 | INFO | "gpu" : " 17 GB", | 20260506 19:54:06.229 | INFO | "huge" : "", | 20260506 19:54:06.229 | INFO | "huge_shared" : " 27 GB", | 20260506 19:54:06.230 | INFO | "malloc" : " 14 GB", | 20260506 19:54:06.230 | INFO | "pinned" : " 29 GB" | 20260506 19:54:06.230 | INFO | } | 20260506 19:54:06.230 | INFO | Perf report for the last 5.13787 seconds: | 20260506 19:54:06.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:54:06.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:06.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:06.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:06.230 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 19:54:06.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:06.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:06.230 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:54:06.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:06.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:06.230 | INFO | Uploaded 36GiB to GPU in 1.94303s (18.5278GiB/s) | 20260506 19:54:06.230 | INFO | Downloaded 34GiB to GPU in 1.87808s (18.1036GiB/s) | 20260506 19:54:11.366 | INFO | Analyzing chunk frames = [4020224,4020736) | 20260506 19:54:11.366 | INFO | MemInfo: { | 20260506 19:54:11.366 | INFO | "gpu" : " 17 GB", | 20260506 19:54:11.366 | INFO | "huge" : "", | 20260506 19:54:11.366 | INFO | "huge_shared" : " 27 GB", | 20260506 19:54:11.366 | INFO | "malloc" : " 14 GB", | 20260506 19:54:11.366 | INFO | "pinned" : " 29 GB" | 20260506 19:54:11.366 | INFO | } | 20260506 19:54:11.367 | INFO | Perf report for the last 5.13686 seconds: | 20260506 19:54:11.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:54:11.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:11.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:11.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:11.367 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 19:54:11.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:11.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:11.367 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:54:11.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:11.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:11.367 | INFO | Uploaded 36GiB to GPU in 1.98024s (18.1796GiB/s) | 20260506 19:54:11.367 | INFO | Downloaded 34GiB to GPU in 1.8974s (17.9192GiB/s) | 20260506 19:54:16.501 | INFO | Analyzing chunk frames = [4020736,4021248) | 20260506 19:54:16.501 | INFO | MemInfo: { | 20260506 19:54:16.501 | INFO | "gpu" : " 17 GB", | 20260506 19:54:16.501 | INFO | "huge" : "", | 20260506 19:54:16.501 | INFO | "huge_shared" : " 27 GB", | 20260506 19:54:16.501 | INFO | "malloc" : " 14 GB", | 20260506 19:54:16.501 | INFO | "pinned" : " 29 GB" | 20260506 19:54:16.501 | INFO | } | 20260506 19:54:16.501 | INFO | Perf report for the last 5.13429 seconds: | 20260506 19:54:16.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:54:16.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:16.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:16.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:16.501 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 19:54:16.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:16.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:16.501 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:54:16.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:16.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:16.501 | INFO | Uploaded 36GiB to GPU in 1.97978s (18.1838GiB/s) | 20260506 19:54:16.501 | INFO | Downloaded 34GiB to GPU in 1.92916s (17.6242GiB/s) | 20260506 19:54:21.642 | INFO | Analyzing chunk frames = [4021248,4021760) | 20260506 19:54:21.642 | INFO | MemInfo: { | 20260506 19:54:21.642 | INFO | "gpu" : " 17 GB", | 20260506 19:54:21.642 | INFO | "huge" : "", | 20260506 19:54:21.642 | INFO | "huge_shared" : " 27 GB", | 20260506 19:54:21.642 | INFO | "malloc" : " 14 GB", | 20260506 19:54:21.642 | INFO | "pinned" : " 29 GB" | 20260506 19:54:21.642 | INFO | } | 20260506 19:54:21.642 | INFO | Perf report for the last 5.14134 seconds: | 20260506 19:54:21.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:54:21.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:21.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:21.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:21.642 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 19:54:21.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:21.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:21.642 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:54:21.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:21.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:21.643 | INFO | Uploaded 36GiB to GPU in 1.996s (18.0361GiB/s) | 20260506 19:54:21.643 | INFO | Downloaded 34GiB to GPU in 2.0071s (16.9398GiB/s) | 20260506 19:54:26.786 | INFO | Analyzing chunk frames = [4021760,4022272) | 20260506 19:54:26.786 | INFO | MemInfo: { | 20260506 19:54:26.786 | INFO | "gpu" : " 17 GB", | 20260506 19:54:26.786 | INFO | "huge" : "", | 20260506 19:54:26.786 | INFO | "huge_shared" : " 27 GB", | 20260506 19:54:26.786 | INFO | "malloc" : " 14 GB", | 20260506 19:54:26.786 | INFO | "pinned" : " 29 GB" | 20260506 19:54:26.786 | INFO | } | 20260506 19:54:26.786 | INFO | Perf report for the last 5.14379 seconds: | 20260506 19:54:26.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:54:26.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:26.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:26.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:26.786 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 19:54:26.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:26.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:26.786 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:54:26.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:26.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:26.786 | INFO | Uploaded 36GiB to GPU in 1.98853s (18.1038GiB/s) | 20260506 19:54:26.786 | INFO | Downloaded 34GiB to GPU in 1.98572s (17.1223GiB/s) | 20260506 19:54:26.786 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:54:26.786 | INFO | Total mem usage: 18 GiB | 20260506 19:54:26.786 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:54:26.786 | INFO | Recent High Water Marks Sum: 17 GiB (Savings: -5.96%) | 20260506 19:54:26.786 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:54:26.786 | INFO | High Water Breakdown: | 20260506 19:54:26.786 | INFO | 62.7% : 10.7 GiB ( 93.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:54:26.786 | INFO | 37.2% : 6.32 GiB ( 94.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:54:26.786 | INFO | 0.117% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:54:26.786 | INFO | ------------------------------------------ | 20260506 19:54:26.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:54:26.786 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:54:26.787 | INFO | Total mem usage: 26.5 GiB | 20260506 19:54:26.787 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:54:26.787 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:54:26.787 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:54:26.787 | INFO | High Water Breakdown: | 20260506 19:54:26.787 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:54:26.787 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:54:26.787 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:54:26.787 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:54:26.787 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:54:26.787 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:54:26.787 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:54:26.787 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:54:26.787 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:54:26.787 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:54:26.787 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:54:26.787 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:54:26.787 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:54:26.787 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:54:26.787 | INFO | ------------------------------------------ | 20260506 19:54:26.787 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:54:26.787 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:54:26.787 | INFO | Total mem usage: 25.6 GiB | 20260506 19:54:26.787 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:54:26.787 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:54:26.787 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:54:26.787 | INFO | High Water Breakdown: | 20260506 19:54:26.787 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:54:26.787 | INFO | ------------------------------------------ | 20260506 19:54:26.787 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:54:26.787 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:54:26.787 | INFO | Total mem usage: 15.7 GiB | 20260506 19:54:26.787 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:54:26.787 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:54:26.787 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:54:26.787 | INFO | High Water Breakdown: | 20260506 19:54:26.787 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:54:26.787 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:54:26.787 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:54:26.787 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:54:26.787 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:54:26.787 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:54:26.787 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:54:26.787 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:54:26.787 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:54:26.787 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:54:26.787 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:54:26.787 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:54:26.787 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:54:26.787 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:54:26.787 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:54:26.787 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:54:26.787 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:54:26.787 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:54:26.787 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:54:26.787 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:54:26.787 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:54:26.787 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:54:26.787 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:54:26.787 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:54:26.787 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:54:26.787 | INFO | ------------------------------------------ | 20260506 19:54:26.787 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:54:26.787 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:54:26.787 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:54:26.787 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:54:26.787 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.36e+01 / 2.37e+00 | 20260506 19:54:26.787 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.03e-01, 9.45e-03 / 6.27e-03 | 20260506 19:54:26.787 | INFO | PerfReport: 11.62%, Baseline -- 3.62e+00 / 4.34e+00, 5.56e-01 / 3.88e-01 | 20260506 19:54:26.787 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.84e-01 / 1.35e-01 | 20260506 19:54:26.787 | INFO | PerfReport: 15.98%, DME -- 3.99e+01 / 4.41e+01, 4.13e+00 / 2.81e+00 | 20260506 19:54:26.787 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.78e+01 / 2.13e+01, 1.41e+00 / 1.85e+00 | 20260506 19:54:26.787 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.23e+00 / 2.73e+00, 3.79e-01 / 2.53e-01 | 20260506 19:54:26.787 | INFO | PerfReport: 3.34%, Metrics -- 1.04e+00 / 1.38e+00, 2.78e-01 / 1.24e-01 | 20260506 19:54:26.787 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.58e-01 / 4.04e-01, 5.75e-02 / 2.57e-02 | 20260506 19:54:26.787 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:54:26.787 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:54:26.787 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:54:26.787 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 19:54:26.787 | INFO | PerfReport: 21.42%, Upload -- 2.07e+01 / 0.00e+00, 6.81e+00 / 0.00e+00 | 20260506 19:54:26.787 | INFO | PerfReport: 21.23%, Download -- 2.05e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 19:54:26.787 | INFO | PerfReport: 32.27%, Compute -- 3.12e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 19:54:26.787 | INFO | PerfReport: 1.23%, LockUpload -- 1.19e+00 / 0.00e+00, 5.13e+00 / 0.00e+00 | 20260506 19:54:26.787 | INFO | PerfReport: 1.25%, LockDownload -- 1.21e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260506 19:54:26.787 | INFO | PerfReport: 22.58%, LockCompute -- 2.18e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 19:54:26.787 | INFO | PerfReport: 0.02%, Misc -- 1.45e-02 / 0.00e+00, 2.43e-02 / 0.00e+00 | 20260506 19:54:26.891 | INFO | Starting a round of Baz IO | 20260506 19:54:30.029 | INFO | Populating RTMetrics report | 20260506 19:54:30.029 | INFO | RTMetrics report populated | 20260506 19:54:30.030 | INFO | Opening tmp RTMetrics json file | 20260506 19:54:30.224 | INFO | Serializing RTMetrics to json | 20260506 19:54:30.226 | INFO | Serialization done, swapping tmp file now... | 20260506 19:54:30.227 | INFO | RTMetrics done | 20260506 19:54:30.256 | INFO | Finished a round of Baz IO: Wrote 1.996 GiB in 3.364 seconds | 20260506 19:54:30.256 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.44% / 0.00% / 65.56% | 20260506 19:54:30.256 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.837433% / 1.19209e-05% / 9.13938e-05% | 20260506 19:54:30.256 | INFO | Baz IO overall: | 20260506 19:54:30.256 | INFO | BytesWritten : 2124997015786 | 20260506 19:54:30.256 | INFO | NumEvents : 1026961115455 | 20260506 19:54:31.929 | INFO | Analyzing chunk frames = [4022272,4022784) | 20260506 19:54:31.929 | INFO | MemInfo: { | 20260506 19:54:31.929 | INFO | "gpu" : " 17 GB", | 20260506 19:54:31.929 | INFO | "huge" : "", | 20260506 19:54:31.929 | INFO | "huge_shared" : " 27 GB", | 20260506 19:54:31.929 | INFO | "malloc" : " 14 GB", | 20260506 19:54:31.929 | INFO | "pinned" : " 29 GB" | 20260506 19:54:31.929 | INFO | } | 20260506 19:54:31.929 | INFO | Perf report for the last 5.14317 seconds: | 20260506 19:54:31.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:54:31.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:31.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:31.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:31.929 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 19:54:31.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:31.929 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260506 19:54:31.929 | INFO | PRE_HQ : 60.1%, 1.8, 57.6ms, 0 | 20260506 19:54:31.929 | INFO | BAZWRITER : 65.1%, 4.7, 156.8ms, 0 | 20260506 19:54:31.929 | INFO | BAZ_SAMPLING : 3.6%, 1.0, 1.9ms, 0 | 20260506 19:54:31.930 | INFO | Uploaded 36GiB to GPU in 2.06285s (17.4516GiB/s) | 20260506 19:54:31.930 | INFO | Downloaded 38GiB to GPU in 2.20684s (17.2192GiB/s) | 20260506 19:54:37.069 | INFO | Analyzing chunk frames = [4022784,4023296) | 20260506 19:54:37.069 | INFO | MemInfo: { | 20260506 19:54:37.069 | INFO | "gpu" : " 17 GB", | 20260506 19:54:37.069 | INFO | "huge" : "", | 20260506 19:54:37.069 | INFO | "huge_shared" : " 27 GB", | 20260506 19:54:37.069 | INFO | "malloc" : " 14 GB", | 20260506 19:54:37.069 | INFO | "pinned" : " 29 GB" | 20260506 19:54:37.069 | INFO | } | 20260506 19:54:37.069 | INFO | Perf report for the last 5.13982 seconds: | 20260506 19:54:37.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:54:37.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:37.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:37.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:37.069 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:54:37.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:37.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:37.069 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:54:37.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:37.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:37.069 | INFO | Uploaded 36GiB to GPU in 2.01763s (17.8427GiB/s) | 20260506 19:54:37.069 | INFO | Downloaded 34GiB to GPU in 1.92391s (17.6724GiB/s) | 20260506 19:54:42.204 | INFO | Analyzing chunk frames = [4023296,4023808) | 20260506 19:54:42.204 | INFO | MemInfo: { | 20260506 19:54:42.204 | INFO | "gpu" : " 17 GB", | 20260506 19:54:42.204 | INFO | "huge" : "", | 20260506 19:54:42.204 | INFO | "huge_shared" : " 27 GB", | 20260506 19:54:42.204 | INFO | "malloc" : " 14 GB", | 20260506 19:54:42.204 | INFO | "pinned" : " 29 GB" | 20260506 19:54:42.204 | INFO | } | 20260506 19:54:42.204 | INFO | Perf report for the last 5.13506 seconds: | 20260506 19:54:42.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:54:42.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:42.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:42.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:42.204 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 19:54:42.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:42.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:42.204 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:54:42.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:42.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:42.204 | INFO | Uploaded 36GiB to GPU in 1.97568s (18.2216GiB/s) | 20260506 19:54:42.204 | INFO | Downloaded 34GiB to GPU in 1.92503s (17.662GiB/s) | 20260506 19:54:47.344 | INFO | Analyzing chunk frames = [4023808,4024320) | 20260506 19:54:47.344 | INFO | MemInfo: { | 20260506 19:54:47.344 | INFO | "gpu" : " 17 GB", | 20260506 19:54:47.344 | INFO | "huge" : "", | 20260506 19:54:47.344 | INFO | "huge_shared" : " 27 GB", | 20260506 19:54:47.344 | INFO | "malloc" : " 14 GB", | 20260506 19:54:47.344 | INFO | "pinned" : " 29 GB" | 20260506 19:54:47.344 | INFO | } | 20260506 19:54:47.344 | INFO | Perf report for the last 5.13982 seconds: | 20260506 19:54:47.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:54:47.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:47.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:47.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:47.344 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 19:54:47.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:47.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:47.344 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:54:47.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:47.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:47.344 | INFO | Uploaded 36GiB to GPU in 1.95568s (18.4079GiB/s) | 20260506 19:54:47.344 | INFO | Downloaded 34GiB to GPU in 1.87392s (18.1438GiB/s) | 20260506 19:54:52.478 | INFO | Analyzing chunk frames = [4024320,4024832) | 20260506 19:54:52.478 | INFO | MemInfo: { | 20260506 19:54:52.478 | INFO | "gpu" : " 17 GB", | 20260506 19:54:52.478 | INFO | "huge" : "", | 20260506 19:54:52.478 | INFO | "huge_shared" : " 27 GB", | 20260506 19:54:52.478 | INFO | "malloc" : " 14 GB", | 20260506 19:54:52.478 | INFO | "pinned" : " 29 GB" | 20260506 19:54:52.478 | INFO | } | 20260506 19:54:52.478 | INFO | Perf report for the last 5.13425 seconds: | 20260506 19:54:52.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:54:52.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:52.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:52.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:52.478 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260506 19:54:52.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:52.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:52.478 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:54:52.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:52.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:52.478 | INFO | Uploaded 36GiB to GPU in 2.01647s (17.8529GiB/s) | 20260506 19:54:52.479 | INFO | Downloaded 34GiB to GPU in 1.9102s (17.7992GiB/s) | 20260506 19:54:57.616 | INFO | Analyzing chunk frames = [4024832,4025344) | 20260506 19:54:57.616 | INFO | MemInfo: { | 20260506 19:54:57.616 | INFO | "gpu" : " 17 GB", | 20260506 19:54:57.616 | INFO | "huge" : "", | 20260506 19:54:57.617 | INFO | "huge_shared" : " 27 GB", | 20260506 19:54:57.617 | INFO | "malloc" : " 14 GB", | 20260506 19:54:57.617 | INFO | "pinned" : " 29 GB" | 20260506 19:54:57.617 | INFO | } | 20260506 19:54:57.617 | INFO | Perf report for the last 5.13846 seconds: | 20260506 19:54:57.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:54:57.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:57.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:57.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:57.617 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 19:54:57.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:57.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:57.617 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 19:54:57.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:57.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:54:57.617 | INFO | Uploaded 36GiB to GPU in 2.01574s (17.8594GiB/s) | 20260506 19:54:57.617 | INFO | Downloaded 34GiB to GPU in 1.95416s (17.3988GiB/s) | 20260506 19:55:02.764 | INFO | Analyzing chunk frames = [4025344,4025856) | 20260506 19:55:02.764 | INFO | MemInfo: { | 20260506 19:55:02.764 | INFO | "gpu" : " 17 GB", | 20260506 19:55:02.764 | INFO | "huge" : "", | 20260506 19:55:02.764 | INFO | "huge_shared" : " 27 GB", | 20260506 19:55:02.764 | INFO | "malloc" : " 14 GB", | 20260506 19:55:02.764 | INFO | "pinned" : " 29 GB" | 20260506 19:55:02.764 | INFO | } | 20260506 19:55:02.764 | INFO | Perf report for the last 5.14742 seconds: | 20260506 19:55:02.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:55:02.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:02.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:02.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:02.764 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 19:55:02.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:02.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:02.764 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260506 19:55:02.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:02.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:02.764 | INFO | Uploaded 36GiB to GPU in 2.00469s (17.9579GiB/s) | 20260506 19:55:02.764 | INFO | Downloaded 34GiB to GPU in 2.00773s (16.9345GiB/s) | 20260506 19:55:07.898 | INFO | Analyzing chunk frames = [4025856,4026368) | 20260506 19:55:07.898 | INFO | MemInfo: { | 20260506 19:55:07.898 | INFO | "gpu" : " 17 GB", | 20260506 19:55:07.898 | INFO | "huge" : "", | 20260506 19:55:07.898 | INFO | "huge_shared" : " 27 GB", | 20260506 19:55:07.898 | INFO | "malloc" : " 14 GB", | 20260506 19:55:07.898 | INFO | "pinned" : " 29 GB" | 20260506 19:55:07.898 | INFO | } | 20260506 19:55:07.898 | INFO | Perf report for the last 5.13431 seconds: | 20260506 19:55:07.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:55:07.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:07.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:07.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:07.898 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 19:55:07.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:07.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:07.898 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 19:55:07.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:07.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:07.899 | INFO | Uploaded 36GiB to GPU in 2.03181s (17.7182GiB/s) | 20260506 19:55:07.899 | INFO | Downloaded 34GiB to GPU in 1.98669s (17.1139GiB/s) | 20260506 19:55:08.017 | INFO | Starting a round of Baz IO | 20260506 19:55:11.175 | INFO | Populating RTMetrics report | 20260506 19:55:11.175 | INFO | RTMetrics report populated | 20260506 19:55:11.175 | INFO | Opening tmp RTMetrics json file | 20260506 19:55:11.199 | INFO | Serializing RTMetrics to json | 20260506 19:55:11.226 | INFO | Serialization done, swapping tmp file now... | 20260506 19:55:11.229 | INFO | RTMetrics done | 20260506 19:55:11.483 | INFO | Finished a round of Baz IO: Wrote 1.998 GiB in 3.466 seconds | 20260506 19:55:11.483 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.43% / 0.00% / 65.56% | 20260506 19:55:11.483 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.837064% / 3.97364e-06% / 5.96046e-05% | 20260506 19:55:11.483 | INFO | Baz IO overall: | 20260506 19:55:11.483 | INFO | BytesWritten : 2127142162971 | 20260506 19:55:11.483 | INFO | NumEvents : 1027992362911 | 20260506 19:55:13.042 | INFO | Analyzing chunk frames = [4026368,4026880) | 20260506 19:55:13.042 | INFO | MemInfo: { | 20260506 19:55:13.042 | INFO | "gpu" : " 17 GB", | 20260506 19:55:13.042 | INFO | "huge" : "", | 20260506 19:55:13.042 | INFO | "huge_shared" : " 27 GB", | 20260506 19:55:13.042 | INFO | "malloc" : " 14 GB", | 20260506 19:55:13.042 | INFO | "pinned" : " 29 GB" | 20260506 19:55:13.042 | INFO | } | 20260506 19:55:13.042 | INFO | Perf report for the last 5.14397 seconds: | 20260506 19:55:13.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:55:13.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:13.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:13.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:13.042 | INFO | ANALYSIS : 63.0%, 2.9, 94.4ms, 0 | 20260506 19:55:13.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:13.042 | INFO | RT_METRICS : 46.6%, 1.0, 24.5ms, 0 | 20260506 19:55:13.042 | INFO | PRE_HQ : 61.0%, 1.9, 61.4ms, 0 | 20260506 19:55:13.042 | INFO | BAZWRITER : 66.9%, 4.8, 165.4ms, 0 | 20260506 19:55:13.042 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 19:55:13.043 | INFO | Uploaded 36GiB to GPU in 2.07153s (17.3785GiB/s) | 20260506 19:55:13.043 | INFO | Downloaded 38GiB to GPU in 2.20041s (17.2695GiB/s) | 20260506 19:55:18.183 | INFO | Analyzing chunk frames = [4026880,4027392) | 20260506 19:55:18.183 | INFO | MemInfo: { | 20260506 19:55:18.183 | INFO | "gpu" : " 17 GB", | 20260506 19:55:18.183 | INFO | "huge" : "", | 20260506 19:55:18.183 | INFO | "huge_shared" : " 27 GB", | 20260506 19:55:18.183 | INFO | "malloc" : " 14 GB", | 20260506 19:55:18.183 | INFO | "pinned" : " 29 GB" | 20260506 19:55:18.183 | INFO | } | 20260506 19:55:18.183 | INFO | Perf report for the last 5.14089 seconds: | 20260506 19:55:18.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:55:18.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:18.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:18.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:18.183 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:55:18.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:18.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:18.183 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 19:55:18.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:18.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:18.184 | INFO | Uploaded 36GiB to GPU in 2.00379s (17.9659GiB/s) | 20260506 19:55:18.184 | INFO | Downloaded 34GiB to GPU in 1.93329s (17.5866GiB/s) | 20260506 19:55:23.316 | INFO | Analyzing chunk frames = [4027392,4027904) | 20260506 19:55:23.316 | INFO | MemInfo: { | 20260506 19:55:23.316 | INFO | "gpu" : " 17 GB", | 20260506 19:55:23.316 | INFO | "huge" : "", | 20260506 19:55:23.316 | INFO | "huge_shared" : " 27 GB", | 20260506 19:55:23.317 | INFO | "malloc" : " 14 GB", | 20260506 19:55:23.317 | INFO | "pinned" : " 29 GB" | 20260506 19:55:23.317 | INFO | } | 20260506 19:55:23.317 | INFO | Perf report for the last 5.13334 seconds: | 20260506 19:55:23.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:55:23.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:23.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:23.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:23.317 | INFO | ANALYSIS : 60.9%, 3.0, 94.1ms, 0 | 20260506 19:55:23.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:23.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:23.317 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:55:23.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:23.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:23.317 | INFO | Uploaded 36GiB to GPU in 1.98524s (18.1338GiB/s) | 20260506 19:55:23.317 | INFO | Downloaded 34GiB to GPU in 1.9205s (17.7038GiB/s) | 20260506 19:55:28.453 | INFO | Analyzing chunk frames = [4027904,4028416) | 20260506 19:55:28.453 | INFO | MemInfo: { | 20260506 19:55:28.453 | INFO | "gpu" : " 17 GB", | 20260506 19:55:28.453 | INFO | "huge" : "", | 20260506 19:55:28.453 | INFO | "huge_shared" : " 27 GB", | 20260506 19:55:28.453 | INFO | "malloc" : " 14 GB", | 20260506 19:55:28.453 | INFO | "pinned" : " 29 GB" | 20260506 19:55:28.453 | INFO | } | 20260506 19:55:28.453 | INFO | Perf report for the last 5.1362 seconds: | 20260506 19:55:28.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:55:28.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:28.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:28.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:28.453 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 19:55:28.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:28.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:28.453 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:55:28.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:28.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:28.453 | INFO | Uploaded 36GiB to GPU in 1.934s (18.6143GiB/s) | 20260506 19:55:28.453 | INFO | Downloaded 34GiB to GPU in 1.86761s (18.2051GiB/s) | 20260506 19:55:33.591 | INFO | Analyzing chunk frames = [4028416,4028928) | 20260506 19:55:33.591 | INFO | MemInfo: { | 20260506 19:55:33.591 | INFO | "gpu" : " 17 GB", | 20260506 19:55:33.591 | INFO | "huge" : "", | 20260506 19:55:33.591 | INFO | "huge_shared" : " 27 GB", | 20260506 19:55:33.591 | INFO | "malloc" : " 14 GB", | 20260506 19:55:33.591 | INFO | "pinned" : " 29 GB" | 20260506 19:55:33.591 | INFO | } | 20260506 19:55:33.591 | INFO | Perf report for the last 5.13814 seconds: | 20260506 19:55:33.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:55:33.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:33.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:33.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:33.591 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260506 19:55:33.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:33.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:33.591 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:55:33.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:33.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:33.591 | INFO | Uploaded 36GiB to GPU in 1.96214s (18.3473GiB/s) | 20260506 19:55:33.591 | INFO | Downloaded 34GiB to GPU in 1.8818s (18.0678GiB/s) | 20260506 19:55:38.735 | INFO | Analyzing chunk frames = [4028928,4029440) | 20260506 19:55:38.735 | INFO | MemInfo: { | 20260506 19:55:38.735 | INFO | "gpu" : " 17 GB", | 20260506 19:55:38.736 | INFO | "huge" : "", | 20260506 19:55:38.736 | INFO | "huge_shared" : " 27 GB", | 20260506 19:55:38.736 | INFO | "malloc" : " 14 GB", | 20260506 19:55:38.736 | INFO | "pinned" : " 29 GB" | 20260506 19:55:38.736 | INFO | } | 20260506 19:55:38.736 | INFO | Perf report for the last 5.14467 seconds: | 20260506 19:55:38.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:55:38.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:38.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:38.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:38.736 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 19:55:38.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:38.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:38.736 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 19:55:38.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:38.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:38.736 | INFO | Uploaded 36GiB to GPU in 2.00368s (17.9669GiB/s) | 20260506 19:55:38.736 | INFO | Downloaded 34GiB to GPU in 1.93243s (17.5944GiB/s) | 20260506 19:55:43.878 | INFO | Analyzing chunk frames = [4029440,4029952) | 20260506 19:55:43.878 | INFO | MemInfo: { | 20260506 19:55:43.878 | INFO | "gpu" : " 17 GB", | 20260506 19:55:43.878 | INFO | "huge" : "", | 20260506 19:55:43.878 | INFO | "huge_shared" : " 27 GB", | 20260506 19:55:43.878 | INFO | "malloc" : " 14 GB", | 20260506 19:55:43.878 | INFO | "pinned" : " 29 GB" | 20260506 19:55:43.878 | INFO | } | 20260506 19:55:43.878 | INFO | Perf report for the last 5.14216 seconds: | 20260506 19:55:43.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:55:43.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:43.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:43.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:43.878 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 19:55:43.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:43.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:43.878 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:55:43.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:43.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:43.878 | INFO | Uploaded 36GiB to GPU in 2.00608s (17.9454GiB/s) | 20260506 19:55:43.878 | INFO | Downloaded 34GiB to GPU in 1.99217s (17.0668GiB/s) | 20260506 19:55:49.023 | INFO | Analyzing chunk frames = [4029952,4030464) | 20260506 19:55:49.023 | INFO | MemInfo: { | 20260506 19:55:49.023 | INFO | "gpu" : " 17 GB", | 20260506 19:55:49.023 | INFO | "huge" : "", | 20260506 19:55:49.024 | INFO | "huge_shared" : " 27 GB", | 20260506 19:55:49.024 | INFO | "malloc" : " 14 GB", | 20260506 19:55:49.024 | INFO | "pinned" : " 29 GB" | 20260506 19:55:49.024 | INFO | } | 20260506 19:55:49.024 | INFO | Perf report for the last 5.14592 seconds: | 20260506 19:55:49.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:55:49.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:49.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:49.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:49.024 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 19:55:49.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:49.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:49.024 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:55:49.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:49.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:49.024 | INFO | Uploaded 36GiB to GPU in 2.00936s (17.9162GiB/s) | 20260506 19:55:49.024 | INFO | Downloaded 34GiB to GPU in 1.99238s (17.065GiB/s) | 20260506 19:55:49.024 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:55:49.024 | INFO | Total mem usage: 18 GiB | 20260506 19:55:49.024 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:55:49.024 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -12%) | 20260506 19:55:49.024 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:55:49.024 | INFO | High Water Breakdown: | 20260506 19:55:49.024 | INFO | 62% : 9.96 GiB ( 87.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:55:49.024 | INFO | 37.9% : 6.09 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:55:49.024 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:55:49.024 | INFO | ------------------------------------------ | 20260506 19:55:49.025 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:55:49.025 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:55:49.025 | INFO | Total mem usage: 26.5 GiB | 20260506 19:55:49.025 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:55:49.025 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:55:49.025 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:55:49.025 | INFO | High Water Breakdown: | 20260506 19:55:49.025 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:55:49.025 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:55:49.025 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:55:49.025 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:55:49.025 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:55:49.025 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:55:49.025 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:55:49.025 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:55:49.025 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:55:49.025 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:55:49.025 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:55:49.025 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:55:49.025 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:55:49.025 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:55:49.025 | INFO | ------------------------------------------ | 20260506 19:55:49.025 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:55:49.025 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:55:49.025 | INFO | Total mem usage: 25.6 GiB | 20260506 19:55:49.025 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:55:49.025 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:55:49.025 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:55:49.025 | INFO | High Water Breakdown: | 20260506 19:55:49.025 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:55:49.025 | INFO | ------------------------------------------ | 20260506 19:55:49.025 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:55:49.025 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:55:49.025 | INFO | Total mem usage: 15.7 GiB | 20260506 19:55:49.025 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:55:49.025 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:55:49.025 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 19:55:49.025 | INFO | High Water Breakdown: | 20260506 19:55:49.025 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:55:49.025 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:55:49.025 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:55:49.025 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:55:49.025 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:55:49.025 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:55:49.025 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:55:49.025 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:55:49.025 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:55:49.025 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:55:49.025 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:55:49.025 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:55:49.025 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:55:49.025 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:55:49.025 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:55:49.025 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:55:49.025 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:55:49.025 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:55:49.025 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:55:49.025 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:55:49.025 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:55:49.025 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:55:49.025 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:55:49.025 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:55:49.025 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:55:49.025 | INFO | ------------------------------------------ | 20260506 19:55:49.025 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:55:49.025 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:55:49.025 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:55:49.025 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:55:49.025 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.55e+01, 1.33e+01 / 2.96e+00 | 20260506 19:55:49.025 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.02e-01, 9.98e-03 / 5.97e-03 | 20260506 19:55:49.025 | INFO | PerfReport: 11.62%, Baseline -- 3.62e+00 / 4.42e+00, 5.49e-01 / 3.27e-01 | 20260506 19:55:49.025 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.82e-01 / 1.32e-01 | 20260506 19:55:49.025 | INFO | PerfReport: 15.99%, DME -- 3.99e+01 / 4.42e+01, 4.05e+00 / 2.99e+00 | 20260506 19:55:49.025 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.44e+00 / 1.81e+00 | 20260506 19:55:49.026 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.24e+00 / 2.81e+00, 3.80e-01 / 2.57e-01 | 20260506 19:55:49.026 | INFO | PerfReport: 3.33%, Metrics -- 1.04e+00 / 1.36e+00, 2.73e-01 / 1.70e-01 | 20260506 19:55:49.026 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.58e-01 / 4.06e-01, 5.65e-02 / 2.62e-02 | 20260506 19:55:49.026 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:55:49.026 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:55:49.026 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:55:49.026 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 19:55:49.026 | INFO | PerfReport: 21.56%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 19:55:49.026 | INFO | PerfReport: 21.19%, Download -- 2.05e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 19:55:49.026 | INFO | PerfReport: 32.23%, Compute -- 3.12e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 19:55:49.026 | INFO | PerfReport: 1.38%, LockUpload -- 1.33e+00 / 0.00e+00, 5.24e+00 / 0.00e+00 | 20260506 19:55:49.026 | INFO | PerfReport: 1.11%, LockDownload -- 1.07e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260506 19:55:49.026 | INFO | PerfReport: 22.51%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 19:55:49.026 | INFO | PerfReport: 0.02%, Misc -- 1.55e-02 / 0.00e+00, 2.60e-02 / 0.00e+00 | 20260506 19:55:49.130 | INFO | Starting a round of Baz IO | 20260506 19:55:52.268 | INFO | Populating RTMetrics report | 20260506 19:55:52.268 | INFO | RTMetrics report populated | 20260506 19:55:52.268 | INFO | Opening tmp RTMetrics json file | 20260506 19:55:52.288 | INFO | Serializing RTMetrics to json | 20260506 19:55:52.295 | INFO | Serialization done, swapping tmp file now... | 20260506 19:55:52.295 | INFO | RTMetrics done | 20260506 19:55:52.969 | INFO | Finished a round of Baz IO: Wrote 1.997 GiB in 3.838 seconds | 20260506 19:55:52.969 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.43% / 0.00% / 65.57% | 20260506 19:55:52.969 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.83522% / 0% / 3.57628e-05% | 20260506 19:55:52.969 | INFO | Baz IO overall: | 20260506 19:55:52.969 | INFO | BytesWritten : 2129286445765 | 20260506 19:55:52.969 | INFO | NumEvents : 1029022979046 | 20260506 19:55:54.159 | INFO | Analyzing chunk frames = [4030464,4030976) | 20260506 19:55:54.159 | INFO | MemInfo: { | 20260506 19:55:54.159 | INFO | "gpu" : " 17 GB", | 20260506 19:55:54.159 | INFO | "huge" : "", | 20260506 19:55:54.159 | INFO | "huge_shared" : " 27 GB", | 20260506 19:55:54.159 | INFO | "malloc" : " 14 GB", | 20260506 19:55:54.159 | INFO | "pinned" : " 29 GB" | 20260506 19:55:54.159 | INFO | } | 20260506 19:55:54.159 | INFO | Perf report for the last 5.13532 seconds: | 20260506 19:55:54.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:55:54.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:54.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:54.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:54.159 | INFO | ANALYSIS : 62.6%, 2.9, 95.4ms, 0 | 20260506 19:55:54.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:54.159 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260506 19:55:54.159 | INFO | PRE_HQ : 60.1%, 1.8, 57.1ms, 0 | 20260506 19:55:54.159 | INFO | BAZWRITER : 74.4%, 4.8, 183.8ms, 0 | 20260506 19:55:54.159 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 19:55:54.160 | INFO | Uploaded 36GiB to GPU in 2.05604s (17.5094GiB/s) | 20260506 19:55:54.160 | INFO | Downloaded 38GiB to GPU in 2.2044s (17.2382GiB/s) | 20260506 19:55:59.296 | INFO | Analyzing chunk frames = [4030976,4031488) | 20260506 19:55:59.296 | INFO | MemInfo: { | 20260506 19:55:59.296 | INFO | "gpu" : " 17 GB", | 20260506 19:55:59.296 | INFO | "huge" : "", | 20260506 19:55:59.296 | INFO | "huge_shared" : " 27 GB", | 20260506 19:55:59.296 | INFO | "malloc" : " 14 GB", | 20260506 19:55:59.296 | INFO | "pinned" : " 29 GB" | 20260506 19:55:59.296 | INFO | } | 20260506 19:55:59.296 | INFO | Perf report for the last 5.13699 seconds: | 20260506 19:55:59.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:55:59.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:59.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:59.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:59.296 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 19:55:59.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:59.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:59.296 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 19:55:59.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:59.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:55:59.296 | INFO | Uploaded 36GiB to GPU in 2.00313s (17.9719GiB/s) | 20260506 19:55:59.296 | INFO | Downloaded 34GiB to GPU in 1.92857s (17.6297GiB/s) | 20260506 19:56:04.433 | INFO | Analyzing chunk frames = [4031488,4032000) | 20260506 19:56:04.433 | INFO | MemInfo: { | 20260506 19:56:04.433 | INFO | "gpu" : " 17 GB", | 20260506 19:56:04.433 | INFO | "huge" : "", | 20260506 19:56:04.433 | INFO | "huge_shared" : " 27 GB", | 20260506 19:56:04.433 | INFO | "malloc" : " 14 GB", | 20260506 19:56:04.433 | INFO | "pinned" : " 29 GB" | 20260506 19:56:04.433 | INFO | } | 20260506 19:56:04.433 | INFO | Perf report for the last 5.13728 seconds: | 20260506 19:56:04.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:56:04.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:04.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:04.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:04.433 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:56:04.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:04.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:04.434 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:56:04.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:04.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:04.434 | INFO | Uploaded 36GiB to GPU in 1.98167s (18.1665GiB/s) | 20260506 19:56:04.434 | INFO | Downloaded 34GiB to GPU in 1.92122s (17.6971GiB/s) | 20260506 19:56:09.569 | INFO | Analyzing chunk frames = [4032000,4032512) | 20260506 19:56:09.569 | INFO | MemInfo: { | 20260506 19:56:09.569 | INFO | "gpu" : " 17 GB", | 20260506 19:56:09.569 | INFO | "huge" : "", | 20260506 19:56:09.569 | INFO | "huge_shared" : " 27 GB", | 20260506 19:56:09.569 | INFO | "malloc" : " 14 GB", | 20260506 19:56:09.569 | INFO | "pinned" : " 29 GB" | 20260506 19:56:09.569 | INFO | } | 20260506 19:56:09.569 | INFO | Perf report for the last 5.13558 seconds: | 20260506 19:56:09.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:56:09.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:09.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:09.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:09.569 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 19:56:09.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:09.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:09.569 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:56:09.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:09.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:09.569 | INFO | Uploaded 36GiB to GPU in 1.93775s (18.5783GiB/s) | 20260506 19:56:09.569 | INFO | Downloaded 34GiB to GPU in 1.87468s (18.1364GiB/s) | 20260506 19:56:14.709 | INFO | Analyzing chunk frames = [4032512,4033024) | 20260506 19:56:14.709 | INFO | MemInfo: { | 20260506 19:56:14.709 | INFO | "gpu" : " 17 GB", | 20260506 19:56:14.709 | INFO | "huge" : "", | 20260506 19:56:14.709 | INFO | "huge_shared" : " 27 GB", | 20260506 19:56:14.709 | INFO | "malloc" : " 14 GB", | 20260506 19:56:14.709 | INFO | "pinned" : " 29 GB" | 20260506 19:56:14.709 | INFO | } | 20260506 19:56:14.709 | INFO | Perf report for the last 5.14017 seconds: | 20260506 19:56:14.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:56:14.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:14.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:14.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:14.709 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 19:56:14.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:14.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:14.709 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:56:14.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:14.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:14.709 | INFO | Uploaded 36GiB to GPU in 2.00726s (17.9349GiB/s) | 20260506 19:56:14.709 | INFO | Downloaded 34GiB to GPU in 1.9116s (17.7862GiB/s) | 20260506 19:56:19.854 | INFO | Analyzing chunk frames = [4033024,4033536) | 20260506 19:56:19.854 | INFO | MemInfo: { | 20260506 19:56:19.854 | INFO | "gpu" : " 17 GB", | 20260506 19:56:19.854 | INFO | "huge" : "", | 20260506 19:56:19.854 | INFO | "huge_shared" : " 27 GB", | 20260506 19:56:19.854 | INFO | "malloc" : " 14 GB", | 20260506 19:56:19.854 | INFO | "pinned" : " 29 GB" | 20260506 19:56:19.854 | INFO | } | 20260506 19:56:19.854 | INFO | Perf report for the last 5.14489 seconds: | 20260506 19:56:19.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:56:19.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:19.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:19.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:19.854 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 19:56:19.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:19.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:19.854 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 19:56:19.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:19.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:19.854 | INFO | Uploaded 36GiB to GPU in 1.99535s (18.042GiB/s) | 20260506 19:56:19.855 | INFO | Downloaded 34GiB to GPU in 1.95785s (17.3659GiB/s) | 20260506 19:56:24.990 | INFO | Analyzing chunk frames = [4033536,4034048) | 20260506 19:56:24.990 | INFO | MemInfo: { | 20260506 19:56:24.990 | INFO | "gpu" : " 17 GB", | 20260506 19:56:24.990 | INFO | "huge" : "", | 20260506 19:56:24.990 | INFO | "huge_shared" : " 27 GB", | 20260506 19:56:24.990 | INFO | "malloc" : " 14 GB", | 20260506 19:56:24.990 | INFO | "pinned" : " 29 GB" | 20260506 19:56:24.990 | INFO | } | 20260506 19:56:24.990 | INFO | Perf report for the last 5.13605 seconds: | 20260506 19:56:24.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:56:24.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:24.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:24.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:24.990 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 19:56:24.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:24.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:24.990 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 19:56:24.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:24.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:24.990 | INFO | Uploaded 36GiB to GPU in 1.95849s (18.3815GiB/s) | 20260506 19:56:24.990 | INFO | Downloaded 34GiB to GPU in 1.99958s (17.0036GiB/s) | 20260506 19:56:30.135 | INFO | Analyzing chunk frames = [4034048,4034560) | 20260506 19:56:30.135 | INFO | MemInfo: { | 20260506 19:56:30.135 | INFO | "gpu" : " 17 GB", | 20260506 19:56:30.136 | INFO | "huge" : "", | 20260506 19:56:30.136 | INFO | "huge_shared" : " 27 GB", | 20260506 19:56:30.136 | INFO | "malloc" : " 14 GB", | 20260506 19:56:30.136 | INFO | "pinned" : " 29 GB" | 20260506 19:56:30.136 | INFO | } | 20260506 19:56:30.136 | INFO | Perf report for the last 5.1456 seconds: | 20260506 19:56:30.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:56:30.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:30.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:30.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:30.136 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 19:56:30.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:30.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:30.136 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 19:56:30.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:30.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:30.136 | INFO | Uploaded 36GiB to GPU in 2.04324s (17.6191GiB/s) | 20260506 19:56:30.136 | INFO | Downloaded 34GiB to GPU in 1.97111s (17.2491GiB/s) | 20260506 19:56:30.248 | INFO | Starting a round of Baz IO | 20260506 19:56:33.382 | INFO | Populating RTMetrics report | 20260506 19:56:33.382 | INFO | RTMetrics report populated | 20260506 19:56:33.382 | INFO | Opening tmp RTMetrics json file | 20260506 19:56:33.400 | INFO | Serializing RTMetrics to json | 20260506 19:56:33.433 | INFO | Serialization done, swapping tmp file now... | 20260506 19:56:33.438 | INFO | RTMetrics done | 20260506 19:56:33.718 | INFO | Finished a round of Baz IO: Wrote 1.995 GiB in 3.47 seconds | 20260506 19:56:33.718 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.43% / 0.00% / 65.57% | 20260506 19:56:33.718 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.832113% / 0% / 8.74201e-05% | 20260506 19:56:33.718 | INFO | Baz IO overall: | 20260506 19:56:33.718 | INFO | BytesWritten : 2131428532506 | 20260506 19:56:33.718 | INFO | NumEvents : 1030051955953 | 20260506 19:56:35.269 | INFO | Analyzing chunk frames = [4034560,4035072) | 20260506 19:56:35.269 | INFO | MemInfo: { | 20260506 19:56:35.269 | INFO | "gpu" : " 17 GB", | 20260506 19:56:35.269 | INFO | "huge" : "", | 20260506 19:56:35.270 | INFO | "huge_shared" : " 27 GB", | 20260506 19:56:35.270 | INFO | "malloc" : " 14 GB", | 20260506 19:56:35.270 | INFO | "pinned" : " 29 GB" | 20260506 19:56:35.270 | INFO | } | 20260506 19:56:35.270 | INFO | Perf report for the last 5.13389 seconds: | 20260506 19:56:35.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:56:35.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:35.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:35.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:35.270 | INFO | ANALYSIS : 62.6%, 2.9, 95.5ms, 0 | 20260506 19:56:35.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:35.270 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 19:56:35.270 | INFO | PRE_HQ : 60.1%, 1.8, 57.3ms, 0 | 20260506 19:56:35.270 | INFO | BAZWRITER : 67.3%, 4.8, 166.2ms, 0 | 20260506 19:56:35.270 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 19:56:35.270 | INFO | Uploaded 36GiB to GPU in 2.04884s (17.5709GiB/s) | 20260506 19:56:35.270 | INFO | Downloaded 38GiB to GPU in 2.20877s (17.2041GiB/s) | 20260506 19:56:40.405 | INFO | Analyzing chunk frames = [4035072,4035584) | 20260506 19:56:40.405 | INFO | MemInfo: { | 20260506 19:56:40.405 | INFO | "gpu" : " 17 GB", | 20260506 19:56:40.405 | INFO | "huge" : "", | 20260506 19:56:40.406 | INFO | "huge_shared" : " 27 GB", | 20260506 19:56:40.406 | INFO | "malloc" : " 14 GB", | 20260506 19:56:40.406 | INFO | "pinned" : " 29 GB" | 20260506 19:56:40.406 | INFO | } | 20260506 19:56:40.406 | INFO | Perf report for the last 5.1361 seconds: | 20260506 19:56:40.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:56:40.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:40.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:40.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:40.406 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:56:40.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:40.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:40.406 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 19:56:40.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:40.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:40.406 | INFO | Uploaded 36GiB to GPU in 2.01451s (17.8704GiB/s) | 20260506 19:56:40.406 | INFO | Downloaded 34GiB to GPU in 1.92178s (17.6919GiB/s) | 20260506 19:56:45.542 | INFO | Analyzing chunk frames = [4035584,4036096) | 20260506 19:56:45.542 | INFO | MemInfo: { | 20260506 19:56:45.542 | INFO | "gpu" : " 17 GB", | 20260506 19:56:45.542 | INFO | "huge" : "", | 20260506 19:56:45.542 | INFO | "huge_shared" : " 27 GB", | 20260506 19:56:45.542 | INFO | "malloc" : " 14 GB", | 20260506 19:56:45.542 | INFO | "pinned" : " 29 GB" | 20260506 19:56:45.542 | INFO | } | 20260506 19:56:45.542 | INFO | Perf report for the last 5.13655 seconds: | 20260506 19:56:45.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:56:45.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:45.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:45.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:45.542 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:56:45.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:45.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:45.542 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:56:45.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:45.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:45.543 | INFO | Uploaded 36GiB to GPU in 1.98588s (18.128GiB/s) | 20260506 19:56:45.543 | INFO | Downloaded 34GiB to GPU in 1.91604s (17.7449GiB/s) | 20260506 19:56:50.682 | INFO | Analyzing chunk frames = [4036096,4036608) | 20260506 19:56:50.682 | INFO | MemInfo: { | 20260506 19:56:50.682 | INFO | "gpu" : " 17 GB", | 20260506 19:56:50.682 | INFO | "huge" : "", | 20260506 19:56:50.682 | INFO | "huge_shared" : " 27 GB", | 20260506 19:56:50.682 | INFO | "malloc" : " 14 GB", | 20260506 19:56:50.682 | INFO | "pinned" : " 29 GB" | 20260506 19:56:50.683 | INFO | } | 20260506 19:56:50.683 | INFO | Perf report for the last 5.14037 seconds: | 20260506 19:56:50.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:56:50.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:50.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:50.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:50.683 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 19:56:50.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:50.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:50.683 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:56:50.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:50.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:50.683 | INFO | Uploaded 36GiB to GPU in 1.93454s (18.6091GiB/s) | 20260506 19:56:50.683 | INFO | Downloaded 34GiB to GPU in 1.86897s (18.1918GiB/s) | 20260506 19:56:55.826 | INFO | Analyzing chunk frames = [4036608,4037120) | 20260506 19:56:55.826 | INFO | MemInfo: { | 20260506 19:56:55.826 | INFO | "gpu" : " 17 GB", | 20260506 19:56:55.826 | INFO | "huge" : "", | 20260506 19:56:55.826 | INFO | "huge_shared" : " 27 GB", | 20260506 19:56:55.826 | INFO | "malloc" : " 14 GB", | 20260506 19:56:55.826 | INFO | "pinned" : " 29 GB" | 20260506 19:56:55.826 | INFO | } | 20260506 19:56:55.826 | INFO | Perf report for the last 5.1437 seconds: | 20260506 19:56:55.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:56:55.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:55.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:55.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:55.826 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 19:56:55.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:55.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:55.827 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:56:55.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:55.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:56:55.827 | INFO | Uploaded 36GiB to GPU in 2.02215s (17.8028GiB/s) | 20260506 19:56:55.827 | INFO | Downloaded 34GiB to GPU in 1.91875s (17.7199GiB/s) | 20260506 19:57:00.966 | INFO | Analyzing chunk frames = [4037120,4037632) | 20260506 19:57:00.966 | INFO | MemInfo: { | 20260506 19:57:00.966 | INFO | "gpu" : " 17 GB", | 20260506 19:57:00.966 | INFO | "huge" : "", | 20260506 19:57:00.966 | INFO | "huge_shared" : " 27 GB", | 20260506 19:57:00.966 | INFO | "malloc" : " 14 GB", | 20260506 19:57:00.966 | INFO | "pinned" : " 29 GB" | 20260506 19:57:00.966 | INFO | } | 20260506 19:57:00.966 | INFO | Perf report for the last 5.14006 seconds: | 20260506 19:57:00.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:57:00.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:00.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:00.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:00.967 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 19:57:00.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:00.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:00.967 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 19:57:00.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:00.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:00.967 | INFO | Uploaded 36GiB to GPU in 2.03473s (17.6928GiB/s) | 20260506 19:57:00.967 | INFO | Downloaded 34GiB to GPU in 1.95546s (17.3872GiB/s) | 20260506 19:57:06.116 | INFO | Analyzing chunk frames = [4037632,4038144) | 20260506 19:57:06.116 | INFO | MemInfo: { | 20260506 19:57:06.116 | INFO | "gpu" : " 17 GB", | 20260506 19:57:06.116 | INFO | "huge" : "", | 20260506 19:57:06.116 | INFO | "huge_shared" : " 27 GB", | 20260506 19:57:06.116 | INFO | "malloc" : " 14 GB", | 20260506 19:57:06.116 | INFO | "pinned" : " 29 GB" | 20260506 19:57:06.116 | INFO | } | 20260506 19:57:06.116 | INFO | Perf report for the last 5.14988 seconds: | 20260506 19:57:06.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:57:06.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:06.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:06.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:06.116 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 19:57:06.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:06.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:06.116 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:57:06.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:06.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:06.117 | INFO | Uploaded 36GiB to GPU in 1.97049s (18.2696GiB/s) | 20260506 19:57:06.117 | INFO | Downloaded 34GiB to GPU in 1.99283s (17.0612GiB/s) | 20260506 19:57:11.246 | INFO | Analyzing chunk frames = [4038144,4038656) | 20260506 19:57:11.246 | INFO | MemInfo: { | 20260506 19:57:11.246 | INFO | "gpu" : " 17 GB", | 20260506 19:57:11.246 | INFO | "huge" : "", | 20260506 19:57:11.246 | INFO | "huge_shared" : " 27 GB", | 20260506 19:57:11.246 | INFO | "malloc" : " 14 GB", | 20260506 19:57:11.246 | INFO | "pinned" : " 29 GB" | 20260506 19:57:11.246 | INFO | } | 20260506 19:57:11.246 | INFO | Perf report for the last 5.12955 seconds: | 20260506 19:57:11.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:57:11.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:11.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:11.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:11.246 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 19:57:11.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:11.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:11.246 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 19:57:11.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:11.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:11.246 | INFO | Uploaded 36GiB to GPU in 1.97226s (18.2532GiB/s) | 20260506 19:57:11.246 | INFO | Downloaded 34GiB to GPU in 1.98462s (17.1317GiB/s) | 20260506 19:57:11.246 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:57:11.246 | INFO | Total mem usage: 18 GiB | 20260506 19:57:11.246 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:57:11.246 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -11.1%) | 20260506 19:57:11.246 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:57:11.246 | INFO | High Water Breakdown: | 20260506 19:57:11.246 | INFO | 62% : 10.1 GiB ( 88.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:57:11.246 | INFO | 37.9% : 6.14 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:57:11.247 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:57:11.247 | INFO | ------------------------------------------ | 20260506 19:57:11.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:57:11.247 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:57:11.247 | INFO | Total mem usage: 26.5 GiB | 20260506 19:57:11.247 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:57:11.247 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:57:11.247 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:57:11.247 | INFO | High Water Breakdown: | 20260506 19:57:11.247 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:57:11.247 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:57:11.247 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:57:11.247 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:57:11.247 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:57:11.247 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:57:11.247 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:57:11.247 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:57:11.247 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:57:11.247 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:57:11.247 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:57:11.247 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:57:11.247 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:57:11.247 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:57:11.247 | INFO | ------------------------------------------ | 20260506 19:57:11.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:57:11.247 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:57:11.247 | INFO | Total mem usage: 25.6 GiB | 20260506 19:57:11.247 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:57:11.247 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:57:11.247 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:57:11.247 | INFO | High Water Breakdown: | 20260506 19:57:11.247 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:57:11.247 | INFO | ------------------------------------------ | 20260506 19:57:11.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:57:11.247 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:57:11.247 | INFO | Total mem usage: 15.7 GiB | 20260506 19:57:11.247 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:57:11.247 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:57:11.247 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 19:57:11.247 | INFO | High Water Breakdown: | 20260506 19:57:11.247 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:57:11.247 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:57:11.247 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:57:11.247 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:57:11.247 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:57:11.247 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:57:11.247 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:57:11.247 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:57:11.247 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:57:11.247 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:57:11.247 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:57:11.247 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:57:11.247 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:57:11.247 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:57:11.247 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:57:11.247 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:57:11.247 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:57:11.247 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:57:11.247 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:57:11.247 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:57:11.247 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:57:11.247 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:57:11.247 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:57:11.247 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:57:11.247 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:57:11.247 | INFO | ------------------------------------------ | 20260506 19:57:11.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:57:11.247 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:57:11.247 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:57:11.247 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:57:11.247 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.60e+01, 1.33e+01 / 2.62e+00 | 20260506 19:57:11.247 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.03e-01, 9.77e-03 / 5.35e-03 | 20260506 19:57:11.247 | INFO | PerfReport: 11.63%, Baseline -- 3.63e+00 / 4.41e+00, 5.48e-01 / 3.57e-01 | 20260506 19:57:11.247 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.48e+00, 1.80e-01 / 1.31e-01 | 20260506 19:57:11.247 | INFO | PerfReport: 16.01%, DME -- 4.00e+01 / 4.50e+01, 4.10e+00 / 2.68e+00 | 20260506 19:57:11.247 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.41e+00 / 2.20e+00 | 20260506 19:57:11.247 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.23e+00 / 2.76e+00, 3.73e-01 / 2.49e-01 | 20260506 19:57:11.248 | INFO | PerfReport: 3.34%, Metrics -- 1.04e+00 / 1.40e+00, 2.75e-01 / 2.04e-01 | 20260506 19:57:11.248 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.59e-01 / 4.05e-01, 5.80e-02 / 1.92e-02 | 20260506 19:57:11.248 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:57:11.248 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:57:11.248 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:57:11.248 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 19:57:11.248 | INFO | PerfReport: 21.49%, Upload -- 2.08e+01 / 0.00e+00, 6.86e+00 / 0.00e+00 | 20260506 19:57:11.248 | INFO | PerfReport: 21.20%, Download -- 2.05e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260506 19:57:11.248 | INFO | PerfReport: 32.21%, Compute -- 3.12e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 19:57:11.248 | INFO | PerfReport: 1.35%, LockUpload -- 1.31e+00 / 0.00e+00, 5.11e+00 / 0.00e+00 | 20260506 19:57:11.248 | INFO | PerfReport: 1.16%, LockDownload -- 1.13e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 19:57:11.248 | INFO | PerfReport: 22.58%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 19:57:11.248 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260506 19:57:11.353 | INFO | Starting a round of Baz IO | 20260506 19:57:14.488 | INFO | Populating RTMetrics report | 20260506 19:57:14.488 | INFO | RTMetrics report populated | 20260506 19:57:14.488 | INFO | Opening tmp RTMetrics json file | 20260506 19:57:14.506 | INFO | Serializing RTMetrics to json | 20260506 19:57:14.513 | INFO | Serialization done, swapping tmp file now... | 20260506 19:57:14.521 | INFO | RTMetrics done | 20260506 19:57:15.010 | INFO | Finished a round of Baz IO: Wrote 1.994 GiB in 3.657 seconds | 20260506 19:57:15.010 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.42% / 0.00% / 65.57% | 20260506 19:57:15.010 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.827559% / 3.97364e-06% / 9.13938e-05% | 20260506 19:57:15.010 | INFO | Baz IO overall: | 20260506 19:57:15.010 | INFO | BytesWritten : 2133569587360 | 20260506 19:57:15.010 | INFO | NumEvents : 1031080115275 | 20260506 19:57:16.385 | INFO | Analyzing chunk frames = [4038656,4039168) | 20260506 19:57:16.385 | INFO | MemInfo: { | 20260506 19:57:16.385 | INFO | "gpu" : " 17 GB", | 20260506 19:57:16.385 | INFO | "huge" : "", | 20260506 19:57:16.385 | INFO | "huge_shared" : " 27 GB", | 20260506 19:57:16.385 | INFO | "malloc" : " 14 GB", | 20260506 19:57:16.385 | INFO | "pinned" : " 29 GB" | 20260506 19:57:16.385 | INFO | } | 20260506 19:57:16.385 | INFO | Perf report for the last 5.13912 seconds: | 20260506 19:57:16.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:57:16.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:16.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:16.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:16.385 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 19:57:16.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:16.385 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 19:57:16.385 | INFO | PRE_HQ : 59.9%, 1.8, 57.4ms, 0 | 20260506 19:57:16.385 | INFO | BAZWRITER : 70.8%, 4.8, 174.9ms, 0 | 20260506 19:57:16.385 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 19:57:16.385 | INFO | Uploaded 36GiB to GPU in 2.03846s (17.6604GiB/s) | 20260506 19:57:16.386 | INFO | Downloaded 38GiB to GPU in 2.2084s (17.207GiB/s) | 20260506 19:57:21.521 | INFO | Analyzing chunk frames = [4039168,4039680) | 20260506 19:57:21.521 | INFO | MemInfo: { | 20260506 19:57:21.521 | INFO | "gpu" : " 17 GB", | 20260506 19:57:21.521 | INFO | "huge" : "", | 20260506 19:57:21.522 | INFO | "huge_shared" : " 27 GB", | 20260506 19:57:21.522 | INFO | "malloc" : " 14 GB", | 20260506 19:57:21.522 | INFO | "pinned" : " 29 GB" | 20260506 19:57:21.522 | INFO | } | 20260506 19:57:21.522 | INFO | Perf report for the last 5.13661 seconds: | 20260506 19:57:21.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:57:21.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:21.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:21.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:21.522 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 19:57:21.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:21.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:21.522 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 19:57:21.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:21.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:21.522 | INFO | Uploaded 36GiB to GPU in 1.99964s (18.0032GiB/s) | 20260506 19:57:21.522 | INFO | Downloaded 34GiB to GPU in 1.92952s (17.621GiB/s) | 20260506 19:57:26.660 | INFO | Analyzing chunk frames = [4039680,4040192) | 20260506 19:57:26.660 | INFO | MemInfo: { | 20260506 19:57:26.660 | INFO | "gpu" : " 17 GB", | 20260506 19:57:26.660 | INFO | "huge" : "", | 20260506 19:57:26.660 | INFO | "huge_shared" : " 27 GB", | 20260506 19:57:26.660 | INFO | "malloc" : " 14 GB", | 20260506 19:57:26.660 | INFO | "pinned" : " 29 GB" | 20260506 19:57:26.660 | INFO | } | 20260506 19:57:26.660 | INFO | Perf report for the last 5.13823 seconds: | 20260506 19:57:26.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:57:26.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:26.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:26.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:26.660 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:57:26.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:26.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:26.660 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 19:57:26.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:26.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:26.660 | INFO | Uploaded 36GiB to GPU in 1.97323s (18.2442GiB/s) | 20260506 19:57:26.660 | INFO | Downloaded 34GiB to GPU in 1.92081s (17.7009GiB/s) | 20260506 19:57:31.801 | INFO | Analyzing chunk frames = [4040192,4040704) | 20260506 19:57:31.801 | INFO | MemInfo: { | 20260506 19:57:31.801 | INFO | "gpu" : " 17 GB", | 20260506 19:57:31.801 | INFO | "huge" : "", | 20260506 19:57:31.801 | INFO | "huge_shared" : " 27 GB", | 20260506 19:57:31.801 | INFO | "malloc" : " 14 GB", | 20260506 19:57:31.801 | INFO | "pinned" : " 29 GB" | 20260506 19:57:31.801 | INFO | } | 20260506 19:57:31.801 | INFO | Perf report for the last 5.14093 seconds: | 20260506 19:57:31.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:57:31.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:31.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:31.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:31.801 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 19:57:31.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:31.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:31.801 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:57:31.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:31.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:31.801 | INFO | Uploaded 36GiB to GPU in 1.9408s (18.549GiB/s) | 20260506 19:57:31.801 | INFO | Downloaded 34GiB to GPU in 1.88812s (18.0073GiB/s) | 20260506 19:57:36.938 | INFO | Analyzing chunk frames = [4040704,4041216) | 20260506 19:57:36.938 | INFO | MemInfo: { | 20260506 19:57:36.938 | INFO | "gpu" : " 17 GB", | 20260506 19:57:36.938 | INFO | "huge" : "", | 20260506 19:57:36.938 | INFO | "huge_shared" : " 27 GB", | 20260506 19:57:36.938 | INFO | "malloc" : " 14 GB", | 20260506 19:57:36.938 | INFO | "pinned" : " 29 GB" | 20260506 19:57:36.938 | INFO | } | 20260506 19:57:36.938 | INFO | Perf report for the last 5.13744 seconds: | 20260506 19:57:36.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:57:36.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:36.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:36.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:36.939 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 19:57:36.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:36.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:36.939 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:57:36.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:36.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:36.939 | INFO | Uploaded 36GiB to GPU in 2.01413s (17.8738GiB/s) | 20260506 19:57:36.939 | INFO | Downloaded 34GiB to GPU in 1.91542s (17.7507GiB/s) | 20260506 19:57:42.079 | INFO | Analyzing chunk frames = [4041216,4041728) | 20260506 19:57:42.079 | INFO | MemInfo: { | 20260506 19:57:42.079 | INFO | "gpu" : " 17 GB", | 20260506 19:57:42.079 | INFO | "huge" : "", | 20260506 19:57:42.080 | INFO | "huge_shared" : " 27 GB", | 20260506 19:57:42.080 | INFO | "malloc" : " 14 GB", | 20260506 19:57:42.080 | INFO | "pinned" : " 29 GB" | 20260506 19:57:42.080 | INFO | } | 20260506 19:57:42.080 | INFO | Perf report for the last 5.14139 seconds: | 20260506 19:57:42.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:57:42.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:42.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:42.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:42.080 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 19:57:42.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:42.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:42.080 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 19:57:42.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:42.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:42.080 | INFO | Uploaded 36GiB to GPU in 2.00923s (17.9173GiB/s) | 20260506 19:57:42.080 | INFO | Downloaded 34GiB to GPU in 1.96302s (17.3202GiB/s) | 20260506 19:57:47.219 | INFO | Analyzing chunk frames = [4041728,4042240) | 20260506 19:57:47.219 | INFO | MemInfo: { | 20260506 19:57:47.219 | INFO | "gpu" : " 17 GB", | 20260506 19:57:47.219 | INFO | "huge" : "", | 20260506 19:57:47.219 | INFO | "huge_shared" : " 27 GB", | 20260506 19:57:47.219 | INFO | "malloc" : " 14 GB", | 20260506 19:57:47.219 | INFO | "pinned" : " 29 GB" | 20260506 19:57:47.219 | INFO | } | 20260506 19:57:47.219 | INFO | Perf report for the last 5.13923 seconds: | 20260506 19:57:47.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:57:47.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:47.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:47.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:47.219 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 19:57:47.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:47.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:47.219 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 19:57:47.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:47.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:47.219 | INFO | Uploaded 36GiB to GPU in 1.95964s (18.3707GiB/s) | 20260506 19:57:47.219 | INFO | Downloaded 34GiB to GPU in 1.99417s (17.0497GiB/s) | 20260506 19:57:52.359 | INFO | Analyzing chunk frames = [4042240,4042752) | 20260506 19:57:52.359 | INFO | MemInfo: { | 20260506 19:57:52.359 | INFO | "gpu" : " 17 GB", | 20260506 19:57:52.359 | INFO | "huge" : "", | 20260506 19:57:52.359 | INFO | "huge_shared" : " 27 GB", | 20260506 19:57:52.359 | INFO | "malloc" : " 14 GB", | 20260506 19:57:52.359 | INFO | "pinned" : " 29 GB" | 20260506 19:57:52.359 | INFO | } | 20260506 19:57:52.359 | INFO | Perf report for the last 5.1405 seconds: | 20260506 19:57:52.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:57:52.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:52.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:52.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:52.360 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 19:57:52.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:52.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:52.360 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:57:52.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:52.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:52.360 | INFO | Uploaded 36GiB to GPU in 1.97415s (18.2357GiB/s) | 20260506 19:57:52.360 | INFO | Downloaded 34GiB to GPU in 1.98233s (17.1515GiB/s) | 20260506 19:57:52.462 | INFO | Starting a round of Baz IO | 20260506 19:57:55.601 | INFO | Populating RTMetrics report | 20260506 19:57:55.601 | INFO | RTMetrics report populated | 20260506 19:57:55.601 | INFO | Opening tmp RTMetrics json file | 20260506 19:57:55.603 | INFO | Serializing RTMetrics to json | 20260506 19:57:55.607 | INFO | Serialization done, swapping tmp file now... | 20260506 19:57:55.630 | INFO | RTMetrics done | 20260506 19:57:55.768 | INFO | Finished a round of Baz IO: Wrote 1.994 GiB in 3.306 seconds | 20260506 19:57:55.768 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.42% / 0.00% / 65.58% | 20260506 19:57:55.768 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.826124% / 0% / 7.54992e-05% | 20260506 19:57:55.768 | INFO | Baz IO overall: | 20260506 19:57:55.769 | INFO | BytesWritten : 2135711021139 | 20260506 19:57:55.769 | INFO | NumEvents : 1032108734622 | 20260506 19:57:57.496 | INFO | Analyzing chunk frames = [4042752,4043264) | 20260506 19:57:57.496 | INFO | MemInfo: { | 20260506 19:57:57.496 | INFO | "gpu" : " 17 GB", | 20260506 19:57:57.496 | INFO | "huge" : "", | 20260506 19:57:57.496 | INFO | "huge_shared" : " 27 GB", | 20260506 19:57:57.496 | INFO | "malloc" : " 14 GB", | 20260506 19:57:57.496 | INFO | "pinned" : " 29 GB" | 20260506 19:57:57.496 | INFO | } | 20260506 19:57:57.496 | INFO | Perf report for the last 5.13673 seconds: | 20260506 19:57:57.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:57:57.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:57.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:57.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:57.496 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 19:57:57.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:57:57.496 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 19:57:57.496 | INFO | PRE_HQ : 60.1%, 1.8, 57.6ms, 0 | 20260506 19:57:57.496 | INFO | BAZWRITER : 64.0%, 4.4, 145.8ms, 0 | 20260506 19:57:57.496 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 19:57:57.496 | INFO | Uploaded 36GiB to GPU in 2.05964s (17.4788GiB/s) | 20260506 19:57:57.497 | INFO | Downloaded 38GiB to GPU in 2.19063s (17.3466GiB/s) | 20260506 19:58:02.636 | INFO | Analyzing chunk frames = [4043264,4043776) | 20260506 19:58:02.636 | INFO | MemInfo: { | 20260506 19:58:02.636 | INFO | "gpu" : " 17 GB", | 20260506 19:58:02.636 | INFO | "huge" : "", | 20260506 19:58:02.636 | INFO | "huge_shared" : " 27 GB", | 20260506 19:58:02.636 | INFO | "malloc" : " 14 GB", | 20260506 19:58:02.636 | INFO | "pinned" : " 29 GB" | 20260506 19:58:02.636 | INFO | } | 20260506 19:58:02.636 | INFO | Perf report for the last 5.13983 seconds: | 20260506 19:58:02.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:58:02.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:02.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:02.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:02.636 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 19:58:02.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:02.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:02.636 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:58:02.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:02.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:02.636 | INFO | Uploaded 36GiB to GPU in 2.01517s (17.8645GiB/s) | 20260506 19:58:02.636 | INFO | Downloaded 34GiB to GPU in 1.94099s (17.5169GiB/s) | 20260506 19:58:07.779 | INFO | Analyzing chunk frames = [4043776,4044288) | 20260506 19:58:07.779 | INFO | MemInfo: { | 20260506 19:58:07.779 | INFO | "gpu" : " 17 GB", | 20260506 19:58:07.779 | INFO | "huge" : "", | 20260506 19:58:07.779 | INFO | "huge_shared" : " 27 GB", | 20260506 19:58:07.779 | INFO | "malloc" : " 14 GB", | 20260506 19:58:07.779 | INFO | "pinned" : " 29 GB" | 20260506 19:58:07.779 | INFO | } | 20260506 19:58:07.779 | INFO | Perf report for the last 5.14281 seconds: | 20260506 19:58:07.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:58:07.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:07.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:07.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:07.779 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 19:58:07.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:07.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:07.779 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:58:07.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:07.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:07.779 | INFO | Uploaded 36GiB to GPU in 1.97966s (18.185GiB/s) | 20260506 19:58:07.779 | INFO | Downloaded 34GiB to GPU in 1.91677s (17.7381GiB/s) | 20260506 19:58:12.915 | INFO | Analyzing chunk frames = [4044288,4044800) | 20260506 19:58:12.915 | INFO | MemInfo: { | 20260506 19:58:12.915 | INFO | "gpu" : " 17 GB", | 20260506 19:58:12.915 | INFO | "huge" : "", | 20260506 19:58:12.915 | INFO | "huge_shared" : " 27 GB", | 20260506 19:58:12.915 | INFO | "malloc" : " 14 GB", | 20260506 19:58:12.915 | INFO | "pinned" : " 29 GB" | 20260506 19:58:12.915 | INFO | } | 20260506 19:58:12.915 | INFO | Perf report for the last 5.13627 seconds: | 20260506 19:58:12.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:58:12.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:12.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:12.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:12.915 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 19:58:12.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:12.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:12.915 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:58:12.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:12.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:12.915 | INFO | Uploaded 36GiB to GPU in 1.93606s (18.5945GiB/s) | 20260506 19:58:12.915 | INFO | Downloaded 34GiB to GPU in 1.86966s (18.1852GiB/s) | 20260506 19:58:18.055 | INFO | Analyzing chunk frames = [4044800,4045312) | 20260506 19:58:18.055 | INFO | MemInfo: { | 20260506 19:58:18.055 | INFO | "gpu" : " 17 GB", | 20260506 19:58:18.055 | INFO | "huge" : "", | 20260506 19:58:18.055 | INFO | "huge_shared" : " 27 GB", | 20260506 19:58:18.055 | INFO | "malloc" : " 14 GB", | 20260506 19:58:18.055 | INFO | "pinned" : " 29 GB" | 20260506 19:58:18.055 | INFO | } | 20260506 19:58:18.055 | INFO | Perf report for the last 5.1398 seconds: | 20260506 19:58:18.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:58:18.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:18.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:18.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:18.055 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 19:58:18.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:18.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:18.055 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:58:18.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:18.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:18.055 | INFO | Uploaded 36GiB to GPU in 2.02181s (17.8058GiB/s) | 20260506 19:58:18.055 | INFO | Downloaded 34GiB to GPU in 1.91129s (17.789GiB/s) | 20260506 19:58:23.203 | INFO | Analyzing chunk frames = [4045312,4045824) | 20260506 19:58:23.203 | INFO | MemInfo: { | 20260506 19:58:23.203 | INFO | "gpu" : " 17 GB", | 20260506 19:58:23.203 | INFO | "huge" : "", | 20260506 19:58:23.203 | INFO | "huge_shared" : " 27 GB", | 20260506 19:58:23.203 | INFO | "malloc" : " 14 GB", | 20260506 19:58:23.203 | INFO | "pinned" : " 29 GB" | 20260506 19:58:23.203 | INFO | } | 20260506 19:58:23.203 | INFO | Perf report for the last 5.14791 seconds: | 20260506 19:58:23.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:58:23.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:23.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:23.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:23.203 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 19:58:23.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:23.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:23.203 | INFO | PRE_HQ : 30.3%, 1.0, 15.8ms, 0 | 20260506 19:58:23.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:23.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:23.203 | INFO | Uploaded 36GiB to GPU in 2.04358s (17.6162GiB/s) | 20260506 19:58:23.203 | INFO | Downloaded 34GiB to GPU in 1.94697s (17.4631GiB/s) | 20260506 19:58:28.336 | INFO | Analyzing chunk frames = [4045824,4046336) | 20260506 19:58:28.336 | INFO | MemInfo: { | 20260506 19:58:28.336 | INFO | "gpu" : " 17 GB", | 20260506 19:58:28.336 | INFO | "huge" : "", | 20260506 19:58:28.336 | INFO | "huge_shared" : " 27 GB", | 20260506 19:58:28.336 | INFO | "malloc" : " 14 GB", | 20260506 19:58:28.336 | INFO | "pinned" : " 29 GB" | 20260506 19:58:28.336 | INFO | } | 20260506 19:58:28.336 | INFO | Perf report for the last 5.1332 seconds: | 20260506 19:58:28.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:58:28.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:28.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:28.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:28.336 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 19:58:28.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:28.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:28.336 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:58:28.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:28.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:28.336 | INFO | Uploaded 36GiB to GPU in 1.9717s (18.2584GiB/s) | 20260506 19:58:28.336 | INFO | Downloaded 34GiB to GPU in 1.95744s (17.3696GiB/s) | 20260506 19:58:33.473 | INFO | Analyzing chunk frames = [4046336,4046848) | 20260506 19:58:33.473 | INFO | MemInfo: { | 20260506 19:58:33.473 | INFO | "gpu" : " 17 GB", | 20260506 19:58:33.473 | INFO | "huge" : "", | 20260506 19:58:33.473 | INFO | "huge_shared" : " 27 GB", | 20260506 19:58:33.473 | INFO | "malloc" : " 14 GB", | 20260506 19:58:33.473 | INFO | "pinned" : " 29 GB" | 20260506 19:58:33.473 | INFO | } | 20260506 19:58:33.473 | INFO | Perf report for the last 5.13731 seconds: | 20260506 19:58:33.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:58:33.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:33.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:33.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:33.473 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:58:33.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:33.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:33.474 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 19:58:33.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:33.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:33.474 | INFO | Uploaded 36GiB to GPU in 2.00004s (17.9996GiB/s) | 20260506 19:58:33.474 | INFO | Downloaded 34GiB to GPU in 1.9767s (17.2004GiB/s) | 20260506 19:58:33.474 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:58:33.474 | INFO | Total mem usage: 18 GiB | 20260506 19:58:33.474 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:58:33.474 | INFO | Recent High Water Marks Sum: 16.7 GiB (Savings: -7.97%) | 20260506 19:58:33.474 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:58:33.474 | INFO | High Water Breakdown: | 20260506 19:58:33.474 | INFO | 62.5% : 10.4 GiB ( 91.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:58:33.474 | INFO | 37.4% : 6.24 GiB ( 93.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:58:33.474 | INFO | 0.119% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:58:33.474 | INFO | ------------------------------------------ | 20260506 19:58:33.474 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:58:33.474 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:58:33.474 | INFO | Total mem usage: 26.5 GiB | 20260506 19:58:33.474 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:58:33.474 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:58:33.474 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:58:33.474 | INFO | High Water Breakdown: | 20260506 19:58:33.474 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:58:33.474 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:58:33.474 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:58:33.474 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:58:33.474 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:58:33.474 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:58:33.474 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:58:33.474 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:58:33.474 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:58:33.474 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:58:33.474 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:58:33.474 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:58:33.474 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:58:33.474 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:58:33.474 | INFO | ------------------------------------------ | 20260506 19:58:33.474 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:58:33.474 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:58:33.474 | INFO | Total mem usage: 25.6 GiB | 20260506 19:58:33.474 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:58:33.474 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 19:58:33.474 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 19:58:33.474 | INFO | High Water Breakdown: | 20260506 19:58:33.474 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 19:58:33.474 | INFO | ------------------------------------------ | 20260506 19:58:33.474 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:58:33.474 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:58:33.474 | INFO | Total mem usage: 15.7 GiB | 20260506 19:58:33.475 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:58:33.475 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:58:33.475 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 19:58:33.475 | INFO | High Water Breakdown: | 20260506 19:58:33.475 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:58:33.475 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:58:33.475 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:58:33.475 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:58:33.475 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:58:33.475 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:58:33.475 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:58:33.475 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:58:33.475 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:58:33.475 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:58:33.475 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:58:33.475 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:58:33.475 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:58:33.475 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:58:33.475 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:58:33.475 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:58:33.475 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:58:33.475 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:58:33.475 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:58:33.475 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:58:33.475 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:58:33.475 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:58:33.475 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:58:33.475 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:58:33.475 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:58:33.475 | INFO | ------------------------------------------ | 20260506 19:58:33.475 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:58:33.475 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:58:33.475 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:58:33.475 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:58:33.475 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.35e+01 / 2.58e+00 | 20260506 19:58:33.475 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 9.94e-02, 1.04e-02 / 5.75e-03 | 20260506 19:58:33.475 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.35e+00, 5.53e-01 / 3.65e-01 | 20260506 19:58:33.475 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.83e-01 / 1.40e-01 | 20260506 19:58:33.475 | INFO | PerfReport: 15.99%, DME -- 3.99e+01 / 4.46e+01, 4.14e+00 / 2.78e+00 | 20260506 19:58:33.475 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.46e+00 / 2.04e+00 | 20260506 19:58:33.475 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.22e+00 / 2.80e+00, 3.73e-01 / 2.20e-01 | 20260506 19:58:33.475 | INFO | PerfReport: 3.33%, Metrics -- 1.04e+00 / 1.38e+00, 2.75e-01 / 1.44e-01 | 20260506 19:58:33.475 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.58e-01 / 4.03e-01, 5.73e-02 / 2.35e-02 | 20260506 19:58:33.475 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:58:33.475 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:58:33.475 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:58:33.475 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 19:58:33.475 | INFO | PerfReport: 21.47%, Upload -- 2.08e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260506 19:58:33.475 | INFO | PerfReport: 21.18%, Download -- 2.05e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 19:58:33.475 | INFO | PerfReport: 32.23%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 19:58:33.475 | INFO | PerfReport: 1.32%, LockUpload -- 1.28e+00 / 0.00e+00, 5.00e+00 / 0.00e+00 | 20260506 19:58:33.475 | INFO | PerfReport: 1.15%, LockDownload -- 1.12e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 19:58:33.475 | INFO | PerfReport: 22.64%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 19:58:33.475 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260506 19:58:33.575 | INFO | Starting a round of Baz IO | 20260506 19:58:36.719 | INFO | Populating RTMetrics report | 20260506 19:58:36.719 | INFO | RTMetrics report populated | 20260506 19:58:36.719 | INFO | Opening tmp RTMetrics json file | 20260506 19:58:36.724 | INFO | Serializing RTMetrics to json | 20260506 19:58:36.727 | INFO | Serialization done, swapping tmp file now... | 20260506 19:58:36.728 | INFO | RTMetrics done | 20260506 19:58:37.106 | INFO | Finished a round of Baz IO: Wrote 1.992 GiB in 3.531 seconds | 20260506 19:58:37.106 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.41% / 0.00% / 65.58% | 20260506 19:58:37.106 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.824412% / 1.58946e-05% / 6.35783e-05% | 20260506 19:58:37.106 | INFO | Baz IO overall: | 20260506 19:58:37.106 | INFO | BytesWritten : 2137849894426 | 20260506 19:58:37.106 | INFO | NumEvents : 1033135587640 | 20260506 19:58:38.613 | INFO | Analyzing chunk frames = [4046848,4047360) | 20260506 19:58:38.613 | INFO | MemInfo: { | 20260506 19:58:38.613 | INFO | "gpu" : " 17 GB", | 20260506 19:58:38.613 | INFO | "huge" : "", | 20260506 19:58:38.613 | INFO | "huge_shared" : " 27 GB", | 20260506 19:58:38.614 | INFO | "malloc" : " 14 GB", | 20260506 19:58:38.614 | INFO | "pinned" : " 29 GB" | 20260506 19:58:38.614 | INFO | } | 20260506 19:58:38.614 | INFO | Perf report for the last 5.14019 seconds: | 20260506 19:58:38.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:58:38.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:38.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:38.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:38.614 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 19:58:38.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:38.614 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 19:58:38.614 | INFO | PRE_HQ : 60.0%, 1.8, 57.2ms, 0 | 20260506 19:58:38.614 | INFO | BAZWRITER : 68.4%, 4.8, 168.2ms, 0 | 20260506 19:58:38.614 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.4ms, 0 | 20260506 19:58:38.614 | INFO | Uploaded 36GiB to GPU in 2.03181s (17.7182GiB/s) | 20260506 19:58:38.614 | INFO | Downloaded 38GiB to GPU in 2.19458s (17.3154GiB/s) | 20260506 19:58:43.749 | INFO | Analyzing chunk frames = [4047360,4047872) | 20260506 19:58:43.749 | INFO | MemInfo: { | 20260506 19:58:43.749 | INFO | "gpu" : " 17 GB", | 20260506 19:58:43.749 | INFO | "huge" : "", | 20260506 19:58:43.749 | INFO | "huge_shared" : " 27 GB", | 20260506 19:58:43.749 | INFO | "malloc" : " 14 GB", | 20260506 19:58:43.749 | INFO | "pinned" : " 29 GB" | 20260506 19:58:43.749 | INFO | } | 20260506 19:58:43.749 | INFO | Perf report for the last 5.13578 seconds: | 20260506 19:58:43.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:58:43.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:43.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:43.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:43.749 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 19:58:43.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:43.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:43.749 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 19:58:43.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:43.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:43.750 | INFO | Uploaded 36GiB to GPU in 2.01385s (17.8762GiB/s) | 20260506 19:58:43.750 | INFO | Downloaded 34GiB to GPU in 1.92222s (17.6879GiB/s) | 20260506 19:58:48.887 | INFO | Analyzing chunk frames = [4047872,4048384) | 20260506 19:58:48.887 | INFO | MemInfo: { | 20260506 19:58:48.887 | INFO | "gpu" : " 17 GB", | 20260506 19:58:48.887 | INFO | "huge" : "", | 20260506 19:58:48.887 | INFO | "huge_shared" : " 27 GB", | 20260506 19:58:48.887 | INFO | "malloc" : " 14 GB", | 20260506 19:58:48.887 | INFO | "pinned" : " 29 GB" | 20260506 19:58:48.887 | INFO | } | 20260506 19:58:48.887 | INFO | Perf report for the last 5.13776 seconds: | 20260506 19:58:48.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:58:48.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:48.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:48.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:48.887 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 19:58:48.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:48.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:48.887 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 19:58:48.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:48.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:48.887 | INFO | Uploaded 36GiB to GPU in 1.97878s (18.193GiB/s) | 20260506 19:58:48.888 | INFO | Downloaded 34GiB to GPU in 1.91893s (17.7182GiB/s) | 20260506 19:58:54.025 | INFO | Analyzing chunk frames = [4048384,4048896) | 20260506 19:58:54.025 | INFO | MemInfo: { | 20260506 19:58:54.025 | INFO | "gpu" : " 17 GB", | 20260506 19:58:54.025 | INFO | "huge" : "", | 20260506 19:58:54.025 | INFO | "huge_shared" : " 27 GB", | 20260506 19:58:54.025 | INFO | "malloc" : " 14 GB", | 20260506 19:58:54.025 | INFO | "pinned" : " 29 GB" | 20260506 19:58:54.025 | INFO | } | 20260506 19:58:54.025 | INFO | Perf report for the last 5.13796 seconds: | 20260506 19:58:54.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:58:54.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:54.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:54.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:54.025 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 19:58:54.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:54.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:54.025 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 19:58:54.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:54.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:54.025 | INFO | Uploaded 36GiB to GPU in 1.92664s (18.6854GiB/s) | 20260506 19:58:54.026 | INFO | Downloaded 34GiB to GPU in 1.86418s (18.2386GiB/s) | 20260506 19:58:59.171 | INFO | Analyzing chunk frames = [4048896,4049408) | 20260506 19:58:59.171 | INFO | MemInfo: { | 20260506 19:58:59.171 | INFO | "gpu" : " 17 GB", | 20260506 19:58:59.171 | INFO | "huge" : "", | 20260506 19:58:59.172 | INFO | "huge_shared" : " 27 GB", | 20260506 19:58:59.172 | INFO | "malloc" : " 14 GB", | 20260506 19:58:59.172 | INFO | "pinned" : " 29 GB" | 20260506 19:58:59.172 | INFO | } | 20260506 19:58:59.172 | INFO | Perf report for the last 5.14656 seconds: | 20260506 19:58:59.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:58:59.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:59.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:59.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:59.172 | INFO | ANALYSIS : 59.8%, 3.0, 93.2ms, 0 | 20260506 19:58:59.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:59.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:59.172 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:58:59.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:59.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:58:59.172 | INFO | Uploaded 36GiB to GPU in 1.95974s (18.3698GiB/s) | 20260506 19:58:59.172 | INFO | Downloaded 34GiB to GPU in 1.87829s (18.1016GiB/s) | 20260506 19:59:04.311 | INFO | Analyzing chunk frames = [4049408,4049920) | 20260506 19:59:04.311 | INFO | MemInfo: { | 20260506 19:59:04.311 | INFO | "gpu" : " 17 GB", | 20260506 19:59:04.311 | INFO | "huge" : "", | 20260506 19:59:04.311 | INFO | "huge_shared" : " 27 GB", | 20260506 19:59:04.311 | INFO | "malloc" : " 14 GB", | 20260506 19:59:04.311 | INFO | "pinned" : " 29 GB" | 20260506 19:59:04.311 | INFO | } | 20260506 19:59:04.311 | INFO | Perf report for the last 5.1395 seconds: | 20260506 19:59:04.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:59:04.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:04.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:04.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:04.311 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 19:59:04.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:04.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:04.311 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 19:59:04.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:04.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:04.312 | INFO | Uploaded 36GiB to GPU in 2.00002s (17.9998GiB/s) | 20260506 19:59:04.312 | INFO | Downloaded 34GiB to GPU in 1.95608s (17.3817GiB/s) | 20260506 19:59:09.453 | INFO | Analyzing chunk frames = [4049920,4050432) | 20260506 19:59:09.453 | INFO | MemInfo: { | 20260506 19:59:09.453 | INFO | "gpu" : " 17 GB", | 20260506 19:59:09.454 | INFO | "huge" : "", | 20260506 19:59:09.454 | INFO | "huge_shared" : " 27 GB", | 20260506 19:59:09.454 | INFO | "malloc" : " 14 GB", | 20260506 19:59:09.454 | INFO | "pinned" : " 29 GB" | 20260506 19:59:09.454 | INFO | } | 20260506 19:59:09.454 | INFO | Perf report for the last 5.14247 seconds: | 20260506 19:59:09.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:59:09.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:09.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:09.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:09.454 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 19:59:09.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:09.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:09.454 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 19:59:09.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:09.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:09.454 | INFO | Uploaded 36GiB to GPU in 1.98877s (18.1017GiB/s) | 20260506 19:59:09.454 | INFO | Downloaded 34GiB to GPU in 2.01021s (16.9136GiB/s) | 20260506 19:59:14.591 | INFO | Analyzing chunk frames = [4050432,4050944) | 20260506 19:59:14.592 | INFO | MemInfo: { | 20260506 19:59:14.592 | INFO | "gpu" : " 17 GB", | 20260506 19:59:14.592 | INFO | "huge" : "", | 20260506 19:59:14.592 | INFO | "huge_shared" : " 27 GB", | 20260506 19:59:14.592 | INFO | "malloc" : " 14 GB", | 20260506 19:59:14.592 | INFO | "pinned" : " 29 GB" | 20260506 19:59:14.592 | INFO | } | 20260506 19:59:14.592 | INFO | Perf report for the last 5.13804 seconds: | 20260506 19:59:14.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:59:14.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:14.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:14.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:14.592 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 19:59:14.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:14.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:14.592 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 19:59:14.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:14.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:14.592 | INFO | Uploaded 36GiB to GPU in 1.96675s (18.3043GiB/s) | 20260506 19:59:14.592 | INFO | Downloaded 34GiB to GPU in 1.98007s (17.1711GiB/s) | 20260506 19:59:14.701 | INFO | Starting a round of Baz IO | 20260506 19:59:17.829 | INFO | Populating RTMetrics report | 20260506 19:59:17.830 | INFO | RTMetrics report populated | 20260506 19:59:17.830 | INFO | Opening tmp RTMetrics json file | 20260506 19:59:17.834 | INFO | Serializing RTMetrics to json | 20260506 19:59:17.837 | INFO | Serialization done, swapping tmp file now... | 20260506 19:59:17.839 | INFO | RTMetrics done | 20260506 19:59:18.036 | INFO | Finished a round of Baz IO: Wrote 1.991 GiB in 3.335 seconds | 20260506 19:59:18.036 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.41% / 0.00% / 65.59% | 20260506 19:59:18.036 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.819981% / 7.94729e-06% / 5.5631e-05% | 20260506 19:59:18.036 | INFO | Baz IO overall: | 20260506 19:59:18.036 | INFO | BytesWritten : 2139988003579 | 20260506 19:59:18.036 | INFO | NumEvents : 1034161881633 | 20260506 19:59:19.728 | INFO | Analyzing chunk frames = [4050944,4051456) | 20260506 19:59:19.728 | INFO | MemInfo: { | 20260506 19:59:19.728 | INFO | "gpu" : " 17 GB", | 20260506 19:59:19.728 | INFO | "huge" : "", | 20260506 19:59:19.729 | INFO | "huge_shared" : " 27 GB", | 20260506 19:59:19.729 | INFO | "malloc" : " 14 GB", | 20260506 19:59:19.729 | INFO | "pinned" : " 29 GB" | 20260506 19:59:19.729 | INFO | } | 20260506 19:59:19.729 | INFO | Perf report for the last 5.13684 seconds: | 20260506 19:59:19.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:59:19.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:19.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:19.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:19.729 | INFO | ANALYSIS : 62.4%, 2.9, 95.7ms, 0 | 20260506 19:59:19.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:19.729 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 19:59:19.729 | INFO | PRE_HQ : 60.0%, 1.9, 57.6ms, 0 | 20260506 19:59:19.729 | INFO | BAZWRITER : 64.8%, 4.5, 150.2ms, 0 | 20260506 19:59:19.729 | INFO | BAZ_SAMPLING : 3.4%, 1.0, 1.8ms, 0 | 20260506 19:59:19.729 | INFO | Uploaded 36GiB to GPU in 2.02368s (17.7894GiB/s) | 20260506 19:59:19.729 | INFO | Downloaded 38GiB to GPU in 2.1853s (17.3889GiB/s) | 20260506 19:59:24.866 | INFO | Analyzing chunk frames = [4051456,4051968) | 20260506 19:59:24.866 | INFO | MemInfo: { | 20260506 19:59:24.866 | INFO | "gpu" : " 17 GB", | 20260506 19:59:24.866 | INFO | "huge" : "", | 20260506 19:59:24.866 | INFO | "huge_shared" : " 27 GB", | 20260506 19:59:24.866 | INFO | "malloc" : " 14 GB", | 20260506 19:59:24.866 | INFO | "pinned" : " 29 GB" | 20260506 19:59:24.866 | INFO | } | 20260506 19:59:24.867 | INFO | Perf report for the last 5.13787 seconds: | 20260506 19:59:24.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:59:24.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:24.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:24.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:24.867 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 19:59:24.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:24.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:24.867 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 19:59:24.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:24.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:24.867 | INFO | Uploaded 36GiB to GPU in 1.99514s (18.0439GiB/s) | 20260506 19:59:24.867 | INFO | Downloaded 34GiB to GPU in 1.92338s (17.6772GiB/s) | 20260506 19:59:30.000 | INFO | Analyzing chunk frames = [4051968,4052480) | 20260506 19:59:30.000 | INFO | MemInfo: { | 20260506 19:59:30.000 | INFO | "gpu" : " 17 GB", | 20260506 19:59:30.000 | INFO | "huge" : "", | 20260506 19:59:30.000 | INFO | "huge_shared" : " 27 GB", | 20260506 19:59:30.000 | INFO | "malloc" : " 14 GB", | 20260506 19:59:30.000 | INFO | "pinned" : " 29 GB" | 20260506 19:59:30.000 | INFO | } | 20260506 19:59:30.000 | INFO | Perf report for the last 5.13356 seconds: | 20260506 19:59:30.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:59:30.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:30.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:30.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:30.000 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 19:59:30.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:30.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:30.000 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 19:59:30.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:30.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:30.000 | INFO | Uploaded 36GiB to GPU in 1.96589s (18.3123GiB/s) | 20260506 19:59:30.000 | INFO | Downloaded 34GiB to GPU in 1.92329s (17.6781GiB/s) | 20260506 19:59:35.144 | INFO | Analyzing chunk frames = [4052480,4052992) | 20260506 19:59:35.144 | INFO | MemInfo: { | 20260506 19:59:35.144 | INFO | "gpu" : " 17 GB", | 20260506 19:59:35.144 | INFO | "huge" : "", | 20260506 19:59:35.144 | INFO | "huge_shared" : " 27 GB", | 20260506 19:59:35.144 | INFO | "malloc" : " 14 GB", | 20260506 19:59:35.144 | INFO | "pinned" : " 29 GB" | 20260506 19:59:35.144 | INFO | } | 20260506 19:59:35.144 | INFO | Perf report for the last 5.14424 seconds: | 20260506 19:59:35.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:59:35.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:35.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:35.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:35.144 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260506 19:59:35.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:35.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:35.144 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 19:59:35.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:35.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:35.145 | INFO | Uploaded 36GiB to GPU in 1.92851s (18.6673GiB/s) | 20260506 19:59:35.145 | INFO | Downloaded 34GiB to GPU in 1.87062s (18.1758GiB/s) | 20260506 19:59:40.293 | INFO | Analyzing chunk frames = [4052992,4053504) | 20260506 19:59:40.293 | INFO | MemInfo: { | 20260506 19:59:40.293 | INFO | "gpu" : " 17 GB", | 20260506 19:59:40.293 | INFO | "huge" : "", | 20260506 19:59:40.293 | INFO | "huge_shared" : " 27 GB", | 20260506 19:59:40.293 | INFO | "malloc" : " 14 GB", | 20260506 19:59:40.294 | INFO | "pinned" : " 29 GB" | 20260506 19:59:40.294 | INFO | } | 20260506 19:59:40.294 | INFO | Perf report for the last 5.14938 seconds: | 20260506 19:59:40.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:59:40.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:40.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:40.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:40.294 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260506 19:59:40.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:40.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:40.294 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260506 19:59:40.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:40.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:40.294 | INFO | Uploaded 36GiB to GPU in 2.01499s (17.8661GiB/s) | 20260506 19:59:40.294 | INFO | Downloaded 34GiB to GPU in 1.91268s (17.7761GiB/s) | 20260506 19:59:45.426 | INFO | Analyzing chunk frames = [4053504,4054016) | 20260506 19:59:45.426 | INFO | MemInfo: { | 20260506 19:59:45.426 | INFO | "gpu" : " 17 GB", | 20260506 19:59:45.426 | INFO | "huge" : "", | 20260506 19:59:45.426 | INFO | "huge_shared" : " 27 GB", | 20260506 19:59:45.426 | INFO | "malloc" : " 14 GB", | 20260506 19:59:45.426 | INFO | "pinned" : " 29 GB" | 20260506 19:59:45.426 | INFO | } | 20260506 19:59:45.426 | INFO | Perf report for the last 5.1324 seconds: | 20260506 19:59:45.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:59:45.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:45.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:45.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:45.426 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 19:59:45.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:45.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:45.426 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 19:59:45.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:45.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:45.426 | INFO | Uploaded 36GiB to GPU in 2.0134s (17.8802GiB/s) | 20260506 19:59:45.426 | INFO | Downloaded 34GiB to GPU in 1.95894s (17.3563GiB/s) | 20260506 19:59:50.564 | INFO | Analyzing chunk frames = [4054016,4054528) | 20260506 19:59:50.564 | INFO | MemInfo: { | 20260506 19:59:50.564 | INFO | "gpu" : " 17 GB", | 20260506 19:59:50.564 | INFO | "huge" : "", | 20260506 19:59:50.564 | INFO | "huge_shared" : " 27 GB", | 20260506 19:59:50.564 | INFO | "malloc" : " 14 GB", | 20260506 19:59:50.564 | INFO | "pinned" : " 29 GB" | 20260506 19:59:50.564 | INFO | } | 20260506 19:59:50.564 | INFO | Perf report for the last 5.13791 seconds: | 20260506 19:59:50.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:59:50.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:50.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:50.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:50.564 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 19:59:50.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:50.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:50.564 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 19:59:50.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:50.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:50.564 | INFO | Uploaded 36GiB to GPU in 1.96066s (18.3611GiB/s) | 20260506 19:59:50.564 | INFO | Downloaded 34GiB to GPU in 1.99705s (17.0251GiB/s) | 20260506 19:59:55.703 | INFO | Analyzing chunk frames = [4054528,4055040) | 20260506 19:59:55.703 | INFO | MemInfo: { | 20260506 19:59:55.703 | INFO | "gpu" : " 17 GB", | 20260506 19:59:55.703 | INFO | "huge" : "", | 20260506 19:59:55.703 | INFO | "huge_shared" : " 27 GB", | 20260506 19:59:55.703 | INFO | "malloc" : " 14 GB", | 20260506 19:59:55.703 | INFO | "pinned" : " 29 GB" | 20260506 19:59:55.703 | INFO | } | 20260506 19:59:55.703 | INFO | Perf report for the last 5.13942 seconds: | 20260506 19:59:55.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 19:59:55.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:55.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:55.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:55.704 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 19:59:55.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:55.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:55.704 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 19:59:55.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:55.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 19:59:55.704 | INFO | Uploaded 36GiB to GPU in 2.03251s (17.7121GiB/s) | 20260506 19:59:55.704 | INFO | Downloaded 34GiB to GPU in 1.98471s (17.1309GiB/s) | 20260506 19:59:55.704 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 19:59:55.704 | INFO | Total mem usage: 18 GiB | 20260506 19:59:55.704 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 19:59:55.704 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -9.49%) | 20260506 19:59:55.704 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 19:59:55.704 | INFO | High Water Breakdown: | 20260506 19:59:55.704 | INFO | 62.2% : 10.2 GiB ( 89.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 19:59:55.704 | INFO | 37.6% : 6.19 GiB ( 92.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 19:59:55.704 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 19:59:55.704 | INFO | ------------------------------------------ | 20260506 19:59:55.704 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:59:55.704 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 19:59:55.704 | INFO | Total mem usage: 26.5 GiB | 20260506 19:59:55.704 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 19:59:55.704 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 19:59:55.704 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 19:59:55.704 | INFO | High Water Breakdown: | 20260506 19:59:55.704 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:59:55.704 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:59:55.704 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:59:55.704 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:59:55.704 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:59:55.704 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:59:55.704 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 19:59:55.704 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 19:59:55.704 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:59:55.704 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 19:59:55.704 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 19:59:55.704 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 19:59:55.704 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 19:59:55.704 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 19:59:55.704 | INFO | ------------------------------------------ | 20260506 19:59:55.704 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:59:55.704 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 19:59:55.704 | INFO | Total mem usage: 25.6 GiB | 20260506 19:59:55.704 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 19:59:55.704 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 19:59:55.704 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 19:59:55.704 | INFO | High Water Breakdown: | 20260506 19:59:55.704 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 19:59:55.704 | INFO | ------------------------------------------ | 20260506 19:59:55.704 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:59:55.704 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 19:59:55.704 | INFO | Total mem usage: 15.7 GiB | 20260506 19:59:55.704 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 19:59:55.704 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 19:59:55.704 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 19:59:55.704 | INFO | High Water Breakdown: | 20260506 19:59:55.704 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 19:59:55.704 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 19:59:55.705 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 19:59:55.705 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 19:59:55.705 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 19:59:55.705 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 19:59:55.705 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 19:59:55.705 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 19:59:55.705 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 19:59:55.705 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 19:59:55.705 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 19:59:55.705 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 19:59:55.705 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 19:59:55.705 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 19:59:55.705 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 19:59:55.705 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 19:59:55.705 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 19:59:55.705 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 19:59:55.705 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 19:59:55.705 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 19:59:55.705 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 19:59:55.705 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 19:59:55.705 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 19:59:55.705 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 19:59:55.705 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 19:59:55.705 | INFO | ------------------------------------------ | 20260506 19:59:55.705 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 19:59:55.705 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 19:59:55.705 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:59:55.705 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:59:55.705 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.59e+01, 1.34e+01 / 2.88e+00 | 20260506 19:59:55.705 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.00e-01, 9.89e-03 / 5.05e-03 | 20260506 19:59:55.705 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.39e+00, 5.57e-01 / 3.50e-01 | 20260506 19:59:55.705 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.83e-01 / 1.30e-01 | 20260506 19:59:55.705 | INFO | PerfReport: 16.02%, DME -- 4.00e+01 / 4.50e+01, 4.27e+00 / 2.82e+00 | 20260506 19:59:55.705 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.42e+00 / 1.83e+00 | 20260506 19:59:55.705 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.24e+00 / 2.78e+00, 3.94e-01 / 1.89e-01 | 20260506 19:59:55.705 | INFO | PerfReport: 3.34%, Metrics -- 1.04e+00 / 1.38e+00, 2.79e-01 / 1.98e-01 | 20260506 19:59:55.705 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.58e-01 / 4.08e-01, 5.72e-02 / 2.32e-02 | 20260506 19:59:55.705 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 19:59:55.705 | INFO | PerfReport: Gathered 1536 samples. | 20260506 19:59:55.705 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 19:59:55.705 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 19:59:55.705 | INFO | PerfReport: 21.39%, Upload -- 2.07e+01 / 0.00e+00, 6.87e+00 / 0.00e+00 | 20260506 19:59:55.705 | INFO | PerfReport: 21.17%, Download -- 2.05e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 19:59:55.705 | INFO | PerfReport: 32.23%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 19:59:55.705 | INFO | PerfReport: 1.26%, LockUpload -- 1.22e+00 / 0.00e+00, 4.94e+00 / 0.00e+00 | 20260506 19:59:55.705 | INFO | PerfReport: 1.19%, LockDownload -- 1.15e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260506 19:59:55.705 | INFO | PerfReport: 22.74%, LockCompute -- 2.20e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 19:59:55.705 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260506 19:59:55.838 | INFO | Starting a round of Baz IO | 20260506 19:59:59.010 | INFO | Populating RTMetrics report | 20260506 19:59:59.010 | INFO | RTMetrics report populated | 20260506 19:59:59.010 | INFO | Opening tmp RTMetrics json file | 20260506 19:59:59.097 | INFO | Serializing RTMetrics to json | 20260506 19:59:59.124 | INFO | Serialization done, swapping tmp file now... | 20260506 19:59:59.128 | INFO | RTMetrics done | 20260506 19:59:59.616 | INFO | Finished a round of Baz IO: Wrote 1.993 GiB in 3.778 seconds | 20260506 19:59:59.617 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.40% / 0.00% / 65.59% | 20260506 19:59:59.617 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.819866% / 0% / 7.94729e-05% | 20260506 19:59:59.617 | INFO | Baz IO overall: | 20260506 19:59:59.617 | INFO | BytesWritten : 2142127878123 | 20260506 19:59:59.617 | INFO | NumEvents : 1035189469529 | 20260506 20:00:00.841 | INFO | Analyzing chunk frames = [4055040,4055552) | 20260506 20:00:00.841 | INFO | MemInfo: { | 20260506 20:00:00.841 | INFO | "gpu" : " 17 GB", | 20260506 20:00:00.841 | INFO | "huge" : "", | 20260506 20:00:00.841 | INFO | "huge_shared" : " 27 GB", | 20260506 20:00:00.841 | INFO | "malloc" : " 14 GB", | 20260506 20:00:00.841 | INFO | "pinned" : " 29 GB" | 20260506 20:00:00.841 | INFO | } | 20260506 20:00:00.841 | INFO | Perf report for the last 5.13781 seconds: | 20260506 20:00:00.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:00:00.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:00.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:00.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:00.841 | INFO | ANALYSIS : 63.3%, 2.8, 93.9ms, 0 | 20260506 20:00:00.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:00.841 | INFO | RT_METRICS : 49.8%, 1.0, 26.1ms, 0 | 20260506 20:00:00.841 | INFO | PRE_HQ : 61.0%, 2.0, 63.3ms, 0 | 20260506 20:00:00.841 | INFO | BAZWRITER : 73.6%, 4.8, 182.3ms, 0 | 20260506 20:00:00.841 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 20:00:00.842 | INFO | Uploaded 36GiB to GPU in 2.07809s (17.3236GiB/s) | 20260506 20:00:00.842 | INFO | Downloaded 38GiB to GPU in 2.17847s (17.4435GiB/s) | 20260506 20:00:05.979 | INFO | Analyzing chunk frames = [4055552,4056064) | 20260506 20:00:05.979 | INFO | MemInfo: { | 20260506 20:00:05.979 | INFO | "gpu" : " 17 GB", | 20260506 20:00:05.979 | INFO | "huge" : "", | 20260506 20:00:05.980 | INFO | "huge_shared" : " 27 GB", | 20260506 20:00:05.980 | INFO | "malloc" : " 14 GB", | 20260506 20:00:05.980 | INFO | "pinned" : " 29 GB" | 20260506 20:00:05.980 | INFO | } | 20260506 20:00:05.980 | INFO | Perf report for the last 5.1384 seconds: | 20260506 20:00:05.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:00:05.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:05.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:05.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:05.980 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:00:05.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:05.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:05.980 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:00:05.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:05.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:05.980 | INFO | Uploaded 36GiB to GPU in 1.9855s (18.1315GiB/s) | 20260506 20:00:05.980 | INFO | Downloaded 34GiB to GPU in 1.94514s (17.4794GiB/s) | 20260506 20:00:11.117 | INFO | Analyzing chunk frames = [4056064,4056576) | 20260506 20:00:11.117 | INFO | MemInfo: { | 20260506 20:00:11.117 | INFO | "gpu" : " 17 GB", | 20260506 20:00:11.117 | INFO | "huge" : "", | 20260506 20:00:11.117 | INFO | "huge_shared" : " 27 GB", | 20260506 20:00:11.117 | INFO | "malloc" : " 14 GB", | 20260506 20:00:11.117 | INFO | "pinned" : " 29 GB" | 20260506 20:00:11.117 | INFO | } | 20260506 20:00:11.118 | INFO | Perf report for the last 5.13776 seconds: | 20260506 20:00:11.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:00:11.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:11.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:11.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:11.118 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 20:00:11.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:11.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:11.118 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:00:11.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:11.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:11.118 | INFO | Uploaded 36GiB to GPU in 1.9629s (18.3402GiB/s) | 20260506 20:00:11.118 | INFO | Downloaded 34GiB to GPU in 1.91808s (17.7261GiB/s) | 20260506 20:00:16.263 | INFO | Analyzing chunk frames = [4056576,4057088) | 20260506 20:00:16.263 | INFO | MemInfo: { | 20260506 20:00:16.263 | INFO | "gpu" : " 17 GB", | 20260506 20:00:16.263 | INFO | "huge" : "", | 20260506 20:00:16.264 | INFO | "huge_shared" : " 27 GB", | 20260506 20:00:16.264 | INFO | "malloc" : " 14 GB", | 20260506 20:00:16.264 | INFO | "pinned" : " 29 GB" | 20260506 20:00:16.264 | INFO | } | 20260506 20:00:16.264 | INFO | Perf report for the last 5.14621 seconds: | 20260506 20:00:16.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:00:16.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:16.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:16.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:16.264 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:00:16.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:16.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:16.264 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:00:16.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:16.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:16.264 | INFO | Uploaded 36GiB to GPU in 1.91787s (18.7708GiB/s) | 20260506 20:00:16.264 | INFO | Downloaded 34GiB to GPU in 1.86908s (18.1908GiB/s) | 20260506 20:00:21.408 | INFO | Analyzing chunk frames = [4057088,4057600) | 20260506 20:00:21.408 | INFO | MemInfo: { | 20260506 20:00:21.408 | INFO | "gpu" : " 17 GB", | 20260506 20:00:21.408 | INFO | "huge" : "", | 20260506 20:00:21.408 | INFO | "huge_shared" : " 27 GB", | 20260506 20:00:21.408 | INFO | "malloc" : " 14 GB", | 20260506 20:00:21.408 | INFO | "pinned" : " 29 GB" | 20260506 20:00:21.408 | INFO | } | 20260506 20:00:21.408 | INFO | Perf report for the last 5.14475 seconds: | 20260506 20:00:21.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:00:21.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:21.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:21.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:21.409 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260506 20:00:21.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:21.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:21.409 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:00:21.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:21.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:21.409 | INFO | Uploaded 36GiB to GPU in 2.00597s (17.9464GiB/s) | 20260506 20:00:21.409 | INFO | Downloaded 34GiB to GPU in 1.92307s (17.68GiB/s) | 20260506 20:00:26.548 | INFO | Analyzing chunk frames = [4057600,4058112) | 20260506 20:00:26.548 | INFO | MemInfo: { | 20260506 20:00:26.548 | INFO | "gpu" : " 17 GB", | 20260506 20:00:26.548 | INFO | "huge" : "", | 20260506 20:00:26.548 | INFO | "huge_shared" : " 27 GB", | 20260506 20:00:26.548 | INFO | "malloc" : " 14 GB", | 20260506 20:00:26.548 | INFO | "pinned" : " 29 GB" | 20260506 20:00:26.548 | INFO | } | 20260506 20:00:26.548 | INFO | Perf report for the last 5.13962 seconds: | 20260506 20:00:26.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:00:26.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:26.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:26.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:26.548 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:00:26.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:26.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:26.548 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:00:26.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:26.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:26.548 | INFO | Uploaded 36GiB to GPU in 2.00998s (17.9106GiB/s) | 20260506 20:00:26.548 | INFO | Downloaded 34GiB to GPU in 1.93324s (17.587GiB/s) | 20260506 20:00:31.679 | INFO | Analyzing chunk frames = [4058112,4058624) | 20260506 20:00:31.679 | INFO | MemInfo: { | 20260506 20:00:31.679 | INFO | "gpu" : " 17 GB", | 20260506 20:00:31.679 | INFO | "huge" : "", | 20260506 20:00:31.679 | INFO | "huge_shared" : " 27 GB", | 20260506 20:00:31.679 | INFO | "malloc" : " 14 GB", | 20260506 20:00:31.679 | INFO | "pinned" : " 29 GB" | 20260506 20:00:31.679 | INFO | } | 20260506 20:00:31.679 | INFO | Perf report for the last 5.13084 seconds: | 20260506 20:00:31.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:00:31.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:31.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:31.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:31.679 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 20:00:31.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:31.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:31.679 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:00:31.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:31.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:31.679 | INFO | Uploaded 36GiB to GPU in 1.96833s (18.2897GiB/s) | 20260506 20:00:31.679 | INFO | Downloaded 34GiB to GPU in 2.00193s (16.9836GiB/s) | 20260506 20:00:36.814 | INFO | Analyzing chunk frames = [4058624,4059136) | 20260506 20:00:36.814 | INFO | MemInfo: { | 20260506 20:00:36.814 | INFO | "gpu" : " 17 GB", | 20260506 20:00:36.814 | INFO | "huge" : "", | 20260506 20:00:36.814 | INFO | "huge_shared" : " 27 GB", | 20260506 20:00:36.814 | INFO | "malloc" : " 14 GB", | 20260506 20:00:36.814 | INFO | "pinned" : " 29 GB" | 20260506 20:00:36.815 | INFO | } | 20260506 20:00:36.815 | INFO | Perf report for the last 5.13558 seconds: | 20260506 20:00:36.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:00:36.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:36.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:36.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:36.815 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 20:00:36.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:36.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:36.815 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:00:36.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:36.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:36.815 | INFO | Uploaded 36GiB to GPU in 1.99538s (18.0417GiB/s) | 20260506 20:00:36.815 | INFO | Downloaded 34GiB to GPU in 1.96755s (17.2804GiB/s) | 20260506 20:00:36.922 | INFO | Starting a round of Baz IO | 20260506 20:00:40.060 | INFO | Populating RTMetrics report | 20260506 20:00:40.060 | INFO | RTMetrics report populated | 20260506 20:00:40.060 | INFO | Opening tmp RTMetrics json file | 20260506 20:00:40.070 | INFO | Serializing RTMetrics to json | 20260506 20:00:40.075 | INFO | Serialization done, swapping tmp file now... | 20260506 20:00:40.081 | INFO | RTMetrics done | 20260506 20:00:40.235 | INFO | Finished a round of Baz IO: Wrote 1.991 GiB in 3.313 seconds | 20260506 20:00:40.235 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.40% / 0.00% / 65.60% | 20260506 20:00:40.235 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.817434% / 0% / 9.93411e-05% | 20260506 20:00:40.235 | INFO | Baz IO overall: | 20260506 20:00:40.235 | INFO | BytesWritten : 2144266227772 | 20260506 20:00:40.235 | INFO | NumEvents : 1036215949097 | 20260506 20:00:41.957 | INFO | Analyzing chunk frames = [4059136,4059648) | 20260506 20:00:41.957 | INFO | MemInfo: { | 20260506 20:00:41.957 | INFO | "gpu" : " 17 GB", | 20260506 20:00:41.957 | INFO | "huge" : "", | 20260506 20:00:41.957 | INFO | "huge_shared" : " 27 GB", | 20260506 20:00:41.957 | INFO | "malloc" : " 14 GB", | 20260506 20:00:41.957 | INFO | "pinned" : " 29 GB" | 20260506 20:00:41.957 | INFO | } | 20260506 20:00:41.957 | INFO | Perf report for the last 5.14228 seconds: | 20260506 20:00:41.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:00:41.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:41.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:41.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:41.957 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 20:00:41.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:41.957 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 20:00:41.957 | INFO | PRE_HQ : 60.0%, 1.8, 57.4ms, 0 | 20260506 20:00:41.957 | INFO | BAZWRITER : 64.1%, 4.8, 158.3ms, 0 | 20260506 20:00:41.957 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 20:00:41.957 | INFO | Uploaded 36GiB to GPU in 2.05089s (17.5533GiB/s) | 20260506 20:00:41.957 | INFO | Downloaded 38GiB to GPU in 2.20163s (17.26GiB/s) | 20260506 20:00:47.094 | INFO | Analyzing chunk frames = [4059648,4060160) | 20260506 20:00:47.094 | INFO | MemInfo: { | 20260506 20:00:47.094 | INFO | "gpu" : " 17 GB", | 20260506 20:00:47.094 | INFO | "huge" : "", | 20260506 20:00:47.094 | INFO | "huge_shared" : " 27 GB", | 20260506 20:00:47.094 | INFO | "malloc" : " 14 GB", | 20260506 20:00:47.094 | INFO | "pinned" : " 29 GB" | 20260506 20:00:47.094 | INFO | } | 20260506 20:00:47.094 | INFO | Perf report for the last 5.13753 seconds: | 20260506 20:00:47.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:00:47.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:47.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:47.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:47.094 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:00:47.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:47.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:47.094 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:00:47.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:47.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:47.095 | INFO | Uploaded 36GiB to GPU in 2.02138s (17.8096GiB/s) | 20260506 20:00:47.095 | INFO | Downloaded 34GiB to GPU in 1.92436s (17.6682GiB/s) | 20260506 20:00:52.236 | INFO | Analyzing chunk frames = [4060160,4060672) | 20260506 20:00:52.236 | INFO | MemInfo: { | 20260506 20:00:52.236 | INFO | "gpu" : " 17 GB", | 20260506 20:00:52.236 | INFO | "huge" : "", | 20260506 20:00:52.236 | INFO | "huge_shared" : " 27 GB", | 20260506 20:00:52.236 | INFO | "malloc" : " 14 GB", | 20260506 20:00:52.236 | INFO | "pinned" : " 29 GB" | 20260506 20:00:52.236 | INFO | } | 20260506 20:00:52.237 | INFO | Perf report for the last 5.14214 seconds: | 20260506 20:00:52.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:00:52.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:52.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:52.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:52.237 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 20:00:52.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:52.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:52.237 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:00:52.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:52.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:52.237 | INFO | Uploaded 36GiB to GPU in 1.98283s (18.1559GiB/s) | 20260506 20:00:52.237 | INFO | Downloaded 34GiB to GPU in 1.92108s (17.6984GiB/s) | 20260506 20:00:57.374 | INFO | Analyzing chunk frames = [4060672,4061184) | 20260506 20:00:57.374 | INFO | MemInfo: { | 20260506 20:00:57.374 | INFO | "gpu" : " 17 GB", | 20260506 20:00:57.374 | INFO | "huge" : "", | 20260506 20:00:57.375 | INFO | "huge_shared" : " 27 GB", | 20260506 20:00:57.375 | INFO | "malloc" : " 14 GB", | 20260506 20:00:57.375 | INFO | "pinned" : " 29 GB" | 20260506 20:00:57.375 | INFO | } | 20260506 20:00:57.375 | INFO | Perf report for the last 5.1381 seconds: | 20260506 20:00:57.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:00:57.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:57.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:57.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:57.375 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 20:00:57.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:57.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:57.375 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:00:57.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:57.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:00:57.375 | INFO | Uploaded 36GiB to GPU in 1.93326s (18.6214GiB/s) | 20260506 20:00:57.375 | INFO | Downloaded 34GiB to GPU in 1.86967s (18.1851GiB/s) | 20260506 20:01:02.513 | INFO | Analyzing chunk frames = [4061184,4061696) | 20260506 20:01:02.513 | INFO | MemInfo: { | 20260506 20:01:02.513 | INFO | "gpu" : " 17 GB", | 20260506 20:01:02.513 | INFO | "huge" : "", | 20260506 20:01:02.513 | INFO | "huge_shared" : " 27 GB", | 20260506 20:01:02.513 | INFO | "malloc" : " 14 GB", | 20260506 20:01:02.513 | INFO | "pinned" : " 29 GB" | 20260506 20:01:02.514 | INFO | } | 20260506 20:01:02.514 | INFO | Perf report for the last 5.13901 seconds: | 20260506 20:01:02.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:01:02.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:02.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:02.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:02.514 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 20:01:02.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:02.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:02.514 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:01:02.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:02.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:02.514 | INFO | Uploaded 36GiB to GPU in 2.01968s (17.8246GiB/s) | 20260506 20:01:02.514 | INFO | Downloaded 34GiB to GPU in 1.91345s (17.769GiB/s) | 20260506 20:01:07.655 | INFO | Analyzing chunk frames = [4061696,4062208) | 20260506 20:01:07.655 | INFO | MemInfo: { | 20260506 20:01:07.655 | INFO | "gpu" : " 17 GB", | 20260506 20:01:07.655 | INFO | "huge" : "", | 20260506 20:01:07.655 | INFO | "huge_shared" : " 27 GB", | 20260506 20:01:07.655 | INFO | "malloc" : " 14 GB", | 20260506 20:01:07.655 | INFO | "pinned" : " 29 GB" | 20260506 20:01:07.655 | INFO | } | 20260506 20:01:07.655 | INFO | Perf report for the last 5.1417 seconds: | 20260506 20:01:07.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:01:07.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:07.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:07.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:07.655 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 20:01:07.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:07.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:07.656 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 20:01:07.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:07.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:07.656 | INFO | Uploaded 36GiB to GPU in 2.02923s (17.7407GiB/s) | 20260506 20:01:07.656 | INFO | Downloaded 34GiB to GPU in 1.94976s (17.438GiB/s) | 20260506 20:01:12.793 | INFO | Analyzing chunk frames = [4062208,4062720) | 20260506 20:01:12.793 | INFO | MemInfo: { | 20260506 20:01:12.793 | INFO | "gpu" : " 17 GB", | 20260506 20:01:12.793 | INFO | "huge" : "", | 20260506 20:01:12.793 | INFO | "huge_shared" : " 27 GB", | 20260506 20:01:12.793 | INFO | "malloc" : " 14 GB", | 20260506 20:01:12.793 | INFO | "pinned" : " 29 GB" | 20260506 20:01:12.793 | INFO | } | 20260506 20:01:12.793 | INFO | Perf report for the last 5.13781 seconds: | 20260506 20:01:12.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:01:12.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:12.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:12.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:12.793 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 20:01:12.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:12.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:12.793 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 20:01:12.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:12.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:12.794 | INFO | Uploaded 36GiB to GPU in 2.00084s (17.9924GiB/s) | 20260506 20:01:12.794 | INFO | Downloaded 34GiB to GPU in 1.95409s (17.3994GiB/s) | 20260506 20:01:17.934 | INFO | Analyzing chunk frames = [4062720,4063232) | 20260506 20:01:17.934 | INFO | MemInfo: { | 20260506 20:01:17.934 | INFO | "gpu" : " 17 GB", | 20260506 20:01:17.934 | INFO | "huge" : "", | 20260506 20:01:17.934 | INFO | "huge_shared" : " 27 GB", | 20260506 20:01:17.934 | INFO | "malloc" : " 14 GB", | 20260506 20:01:17.934 | INFO | "pinned" : " 29 GB" | 20260506 20:01:17.934 | INFO | } | 20260506 20:01:17.934 | INFO | Perf report for the last 5.14111 seconds: | 20260506 20:01:17.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:01:17.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:17.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:17.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:17.934 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:01:17.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:17.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:17.934 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 20:01:17.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:17.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:17.935 | INFO | Uploaded 36GiB to GPU in 1.96557s (18.3153GiB/s) | 20260506 20:01:17.935 | INFO | Downloaded 34GiB to GPU in 1.97384s (17.2253GiB/s) | 20260506 20:01:17.935 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:01:17.935 | INFO | Total mem usage: 18 GiB | 20260506 20:01:17.935 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:01:17.935 | INFO | Recent High Water Marks Sum: 16.7 GiB (Savings: -8.03%) | 20260506 20:01:17.935 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:01:17.935 | INFO | High Water Breakdown: | 20260506 20:01:17.935 | INFO | 62.4% : 10.4 GiB ( 91.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:01:17.935 | INFO | 37.4% : 6.24 GiB ( 93.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:01:17.935 | INFO | 0.119% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:01:17.935 | INFO | ------------------------------------------ | 20260506 20:01:17.935 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:01:17.935 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:01:17.935 | INFO | Total mem usage: 26.5 GiB | 20260506 20:01:17.935 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:01:17.935 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 20:01:17.935 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 20:01:17.935 | INFO | High Water Breakdown: | 20260506 20:01:17.935 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:01:17.935 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:01:17.935 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:01:17.935 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:01:17.935 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:01:17.935 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:01:17.935 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:01:17.935 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:01:17.935 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:01:17.935 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:01:17.935 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:01:17.935 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:01:17.935 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:01:17.935 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:01:17.935 | INFO | ------------------------------------------ | 20260506 20:01:17.935 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:01:17.935 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:01:17.935 | INFO | Total mem usage: 25.6 GiB | 20260506 20:01:17.935 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:01:17.935 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:01:17.935 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:01:17.935 | INFO | High Water Breakdown: | 20260506 20:01:17.935 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:01:17.935 | INFO | ------------------------------------------ | 20260506 20:01:17.935 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:01:17.935 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:01:17.936 | INFO | Total mem usage: 15.7 GiB | 20260506 20:01:17.936 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:01:17.936 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:01:17.936 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:01:17.936 | INFO | High Water Breakdown: | 20260506 20:01:17.936 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:01:17.936 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:01:17.936 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:01:17.936 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:01:17.936 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:01:17.936 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:01:17.936 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:01:17.936 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:01:17.936 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:01:17.936 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:01:17.936 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:01:17.936 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:01:17.936 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:01:17.936 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:01:17.936 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:01:17.936 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:01:17.936 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:01:17.936 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:01:17.936 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:01:17.936 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:01:17.936 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:01:17.936 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:01:17.936 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:01:17.936 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:01:17.936 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:01:17.936 | INFO | ------------------------------------------ | 20260506 20:01:17.936 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:01:17.936 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:01:17.936 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:01:17.936 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:01:17.936 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.57e+01, 1.35e+01 / 2.37e+00 | 20260506 20:01:17.936 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.00e-01, 1.00e-02 / 6.15e-03 | 20260506 20:01:17.936 | INFO | PerfReport: 11.65%, Baseline -- 3.63e+00 / 4.41e+00, 5.57e-01 / 3.17e-01 | 20260506 20:01:17.936 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.44e+00, 1.84e-01 / 1.40e-01 | 20260506 20:01:17.936 | INFO | PerfReport: 16.01%, DME -- 4.00e+01 / 4.46e+01, 4.03e+00 / 2.44e+00 | 20260506 20:01:17.936 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.42e+00 / 1.71e+00 | 20260506 20:01:17.936 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.23e+00 / 2.80e+00, 3.77e-01 / 1.90e-01 | 20260506 20:01:17.936 | INFO | PerfReport: 3.34%, Metrics -- 1.04e+00 / 1.37e+00, 2.76e-01 / 1.66e-01 | 20260506 20:01:17.936 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.57e-01 / 4.04e-01, 5.73e-02 / 2.20e-02 | 20260506 20:01:17.936 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:01:17.936 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:01:17.936 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:01:17.936 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 20:01:17.936 | INFO | PerfReport: 21.48%, Upload -- 2.08e+01 / 0.00e+00, 6.93e+00 / 0.00e+00 | 20260506 20:01:17.936 | INFO | PerfReport: 21.16%, Download -- 2.05e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260506 20:01:17.936 | INFO | PerfReport: 32.26%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 20:01:17.936 | INFO | PerfReport: 1.38%, LockUpload -- 1.33e+00 / 0.00e+00, 5.10e+00 / 0.00e+00 | 20260506 20:01:17.936 | INFO | PerfReport: 1.15%, LockDownload -- 1.12e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 20:01:17.936 | INFO | PerfReport: 22.55%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 20:01:17.936 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.42e-02 / 0.00e+00 | 20260506 20:01:18.033 | INFO | Starting a round of Baz IO | 20260506 20:01:21.173 | INFO | Populating RTMetrics report | 20260506 20:01:21.173 | INFO | RTMetrics report populated | 20260506 20:01:21.173 | INFO | Opening tmp RTMetrics json file | 20260506 20:01:21.208 | INFO | Serializing RTMetrics to json | 20260506 20:01:21.233 | INFO | Serialization done, swapping tmp file now... | 20260506 20:01:21.239 | INFO | RTMetrics done | 20260506 20:01:21.428 | INFO | Finished a round of Baz IO: Wrote 1.992 GiB in 3.395 seconds | 20260506 20:01:21.428 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.40% / 0.00% / 65.60% | 20260506 20:01:21.428 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.816798% / 3.97364e-06% / 7.15256e-05% | 20260506 20:01:21.428 | INFO | Baz IO overall: | 20260506 20:01:21.429 | INFO | BytesWritten : 2146405207204 | 20260506 20:01:21.429 | INFO | NumEvents : 1037242927244 | 20260506 20:01:23.070 | INFO | Analyzing chunk frames = [4063232,4063744) | 20260506 20:01:23.070 | INFO | MemInfo: { | 20260506 20:01:23.070 | INFO | "gpu" : " 17 GB", | 20260506 20:01:23.070 | INFO | "huge" : "", | 20260506 20:01:23.070 | INFO | "huge_shared" : " 27 GB", | 20260506 20:01:23.070 | INFO | "malloc" : " 14 GB", | 20260506 20:01:23.070 | INFO | "pinned" : " 29 GB" | 20260506 20:01:23.070 | INFO | } | 20260506 20:01:23.070 | INFO | Perf report for the last 5.13592 seconds: | 20260506 20:01:23.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:01:23.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:23.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:23.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:23.070 | INFO | ANALYSIS : 62.5%, 2.9, 95.5ms, 0 | 20260506 20:01:23.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:23.070 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 20:01:23.070 | INFO | PRE_HQ : 59.9%, 1.8, 57.1ms, 0 | 20260506 20:01:23.070 | INFO | BAZWRITER : 65.7%, 4.8, 161.3ms, 0 | 20260506 20:01:23.070 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 20:01:23.071 | INFO | Uploaded 36GiB to GPU in 2.06729s (17.4141GiB/s) | 20260506 20:01:23.071 | INFO | Downloaded 38GiB to GPU in 2.20145s (17.2613GiB/s) | 20260506 20:01:28.209 | INFO | Analyzing chunk frames = [4063744,4064256) | 20260506 20:01:28.209 | INFO | MemInfo: { | 20260506 20:01:28.209 | INFO | "gpu" : " 17 GB", | 20260506 20:01:28.209 | INFO | "huge" : "", | 20260506 20:01:28.210 | INFO | "huge_shared" : " 27 GB", | 20260506 20:01:28.210 | INFO | "malloc" : " 14 GB", | 20260506 20:01:28.210 | INFO | "pinned" : " 29 GB" | 20260506 20:01:28.210 | INFO | } | 20260506 20:01:28.210 | INFO | Perf report for the last 5.13951 seconds: | 20260506 20:01:28.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:01:28.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:28.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:28.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:28.210 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 20:01:28.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:28.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:28.210 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:01:28.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:28.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:28.210 | INFO | Uploaded 36GiB to GPU in 1.99861s (18.0125GiB/s) | 20260506 20:01:28.210 | INFO | Downloaded 34GiB to GPU in 1.94184s (17.5092GiB/s) | 20260506 20:01:33.352 | INFO | Analyzing chunk frames = [4064256,4064768) | 20260506 20:01:33.352 | INFO | MemInfo: { | 20260506 20:01:33.352 | INFO | "gpu" : " 17 GB", | 20260506 20:01:33.352 | INFO | "huge" : "", | 20260506 20:01:33.352 | INFO | "huge_shared" : " 27 GB", | 20260506 20:01:33.352 | INFO | "malloc" : " 14 GB", | 20260506 20:01:33.352 | INFO | "pinned" : " 29 GB" | 20260506 20:01:33.352 | INFO | } | 20260506 20:01:33.352 | INFO | Perf report for the last 5.14207 seconds: | 20260506 20:01:33.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:01:33.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:33.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:33.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:33.352 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 20:01:33.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:33.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:33.352 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:01:33.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:33.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:33.352 | INFO | Uploaded 36GiB to GPU in 1.95789s (18.3872GiB/s) | 20260506 20:01:33.352 | INFO | Downloaded 34GiB to GPU in 1.91956s (17.7124GiB/s) | 20260506 20:01:38.490 | INFO | Analyzing chunk frames = [4064768,4065280) | 20260506 20:01:38.490 | INFO | MemInfo: { | 20260506 20:01:38.491 | INFO | "gpu" : " 17 GB", | 20260506 20:01:38.491 | INFO | "huge" : "", | 20260506 20:01:38.491 | INFO | "huge_shared" : " 27 GB", | 20260506 20:01:38.491 | INFO | "malloc" : " 14 GB", | 20260506 20:01:38.491 | INFO | "pinned" : " 29 GB" | 20260506 20:01:38.491 | INFO | } | 20260506 20:01:38.491 | INFO | Perf report for the last 5.1389 seconds: | 20260506 20:01:38.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:01:38.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:38.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:38.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:38.491 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 20:01:38.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:38.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:38.491 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:01:38.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:38.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:38.491 | INFO | Uploaded 36GiB to GPU in 1.91871s (18.7626GiB/s) | 20260506 20:01:38.491 | INFO | Downloaded 34GiB to GPU in 1.87299s (18.1528GiB/s) | 20260506 20:01:43.630 | INFO | Analyzing chunk frames = [4065280,4065792) | 20260506 20:01:43.630 | INFO | MemInfo: { | 20260506 20:01:43.630 | INFO | "gpu" : " 17 GB", | 20260506 20:01:43.630 | INFO | "huge" : "", | 20260506 20:01:43.631 | INFO | "huge_shared" : " 27 GB", | 20260506 20:01:43.631 | INFO | "malloc" : " 14 GB", | 20260506 20:01:43.631 | INFO | "pinned" : " 29 GB" | 20260506 20:01:43.631 | INFO | } | 20260506 20:01:43.631 | INFO | Perf report for the last 5.13984 seconds: | 20260506 20:01:43.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:01:43.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:43.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:43.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:43.631 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 20:01:43.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:43.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:43.631 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:01:43.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:43.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:43.631 | INFO | Uploaded 36GiB to GPU in 1.98972s (18.093GiB/s) | 20260506 20:01:43.631 | INFO | Downloaded 34GiB to GPU in 1.91041s (17.7973GiB/s) | 20260506 20:01:48.769 | INFO | Analyzing chunk frames = [4065792,4066304) | 20260506 20:01:48.769 | INFO | MemInfo: { | 20260506 20:01:48.769 | INFO | "gpu" : " 17 GB", | 20260506 20:01:48.769 | INFO | "huge" : "", | 20260506 20:01:48.769 | INFO | "huge_shared" : " 27 GB", | 20260506 20:01:48.769 | INFO | "malloc" : " 14 GB", | 20260506 20:01:48.769 | INFO | "pinned" : " 29 GB" | 20260506 20:01:48.769 | INFO | } | 20260506 20:01:48.769 | INFO | Perf report for the last 5.13857 seconds: | 20260506 20:01:48.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:01:48.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:48.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:48.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:48.769 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 20:01:48.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:48.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:48.769 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 20:01:48.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:48.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:48.769 | INFO | Uploaded 36GiB to GPU in 2.00936s (17.9161GiB/s) | 20260506 20:01:48.770 | INFO | Downloaded 34GiB to GPU in 1.941s (17.5167GiB/s) | 20260506 20:01:53.907 | INFO | Analyzing chunk frames = [4066304,4066816) | 20260506 20:01:53.907 | INFO | MemInfo: { | 20260506 20:01:53.907 | INFO | "gpu" : " 17 GB", | 20260506 20:01:53.907 | INFO | "huge" : "", | 20260506 20:01:53.907 | INFO | "huge_shared" : " 27 GB", | 20260506 20:01:53.907 | INFO | "malloc" : " 14 GB", | 20260506 20:01:53.907 | INFO | "pinned" : " 29 GB" | 20260506 20:01:53.907 | INFO | } | 20260506 20:01:53.907 | INFO | Perf report for the last 5.13792 seconds: | 20260506 20:01:53.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:01:53.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:53.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:53.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:53.907 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 20:01:53.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:53.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:53.907 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 20:01:53.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:53.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:53.907 | INFO | Uploaded 36GiB to GPU in 1.96878s (18.2854GiB/s) | 20260506 20:01:53.908 | INFO | Downloaded 34GiB to GPU in 1.99789s (17.018GiB/s) | 20260506 20:01:59.049 | INFO | Analyzing chunk frames = [4066816,4067328) | 20260506 20:01:59.049 | INFO | MemInfo: { | 20260506 20:01:59.049 | INFO | "gpu" : " 17 GB", | 20260506 20:01:59.049 | INFO | "huge" : "", | 20260506 20:01:59.049 | INFO | "huge_shared" : " 27 GB", | 20260506 20:01:59.049 | INFO | "malloc" : " 14 GB", | 20260506 20:01:59.049 | INFO | "pinned" : " 29 GB" | 20260506 20:01:59.049 | INFO | } | 20260506 20:01:59.049 | INFO | Perf report for the last 5.14207 seconds: | 20260506 20:01:59.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:01:59.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:59.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:59.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:59.049 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:01:59.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:59.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:59.049 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:01:59.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:59.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:01:59.050 | INFO | Uploaded 36GiB to GPU in 1.97499s (18.228GiB/s) | 20260506 20:01:59.050 | INFO | Downloaded 34GiB to GPU in 1.98332s (17.143GiB/s) | 20260506 20:01:59.151 | INFO | Starting a round of Baz IO | 20260506 20:02:02.287 | INFO | Populating RTMetrics report | 20260506 20:02:02.288 | INFO | RTMetrics report populated | 20260506 20:02:02.288 | INFO | Opening tmp RTMetrics json file | 20260506 20:02:02.320 | INFO | Serializing RTMetrics to json | 20260506 20:02:02.382 | INFO | Serialization done, swapping tmp file now... | 20260506 20:02:02.415 | INFO | RTMetrics done | 20260506 20:02:02.595 | INFO | Finished a round of Baz IO: Wrote 1.991 GiB in 3.443 seconds | 20260506 20:02:02.595 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.39% / 0.00% / 65.60% | 20260506 20:02:02.595 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.812991% / 1.19209e-05% / 6.35783e-05% | 20260506 20:02:02.595 | INFO | Baz IO overall: | 20260506 20:02:02.595 | INFO | BytesWritten : 2148542568941 | 20260506 20:02:02.595 | INFO | NumEvents : 1038268592735 | 20260506 20:02:04.186 | INFO | Analyzing chunk frames = [4067328,4067840) | 20260506 20:02:04.186 | INFO | MemInfo: { | 20260506 20:02:04.186 | INFO | "gpu" : " 17 GB", | 20260506 20:02:04.186 | INFO | "huge" : "", | 20260506 20:02:04.186 | INFO | "huge_shared" : " 27 GB", | 20260506 20:02:04.186 | INFO | "malloc" : " 14 GB", | 20260506 20:02:04.186 | INFO | "pinned" : " 29 GB" | 20260506 20:02:04.186 | INFO | } | 20260506 20:02:04.186 | INFO | Perf report for the last 5.13707 seconds: | 20260506 20:02:04.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:02:04.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:04.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:04.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:04.186 | INFO | ANALYSIS : 62.4%, 2.9, 95.2ms, 0 | 20260506 20:02:04.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:04.186 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 20:02:04.186 | INFO | PRE_HQ : 60.1%, 1.8, 57.3ms, 0 | 20260506 20:02:04.186 | INFO | BAZWRITER : 66.7%, 4.8, 163.5ms, 0 | 20260506 20:02:04.187 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 20:02:04.187 | INFO | Uploaded 36GiB to GPU in 2.06174s (17.4609GiB/s) | 20260506 20:02:04.187 | INFO | Downloaded 38GiB to GPU in 2.19088s (17.3446GiB/s) | 20260506 20:02:09.321 | INFO | Analyzing chunk frames = [4067840,4068352) | 20260506 20:02:09.321 | INFO | MemInfo: { | 20260506 20:02:09.321 | INFO | "gpu" : " 17 GB", | 20260506 20:02:09.321 | INFO | "huge" : "", | 20260506 20:02:09.321 | INFO | "huge_shared" : " 27 GB", | 20260506 20:02:09.321 | INFO | "malloc" : " 14 GB", | 20260506 20:02:09.321 | INFO | "pinned" : " 29 GB" | 20260506 20:02:09.321 | INFO | } | 20260506 20:02:09.322 | INFO | Perf report for the last 5.13517 seconds: | 20260506 20:02:09.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:02:09.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:09.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:09.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:09.322 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:02:09.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:09.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:09.322 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:02:09.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:09.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:09.322 | INFO | Uploaded 36GiB to GPU in 2.00961s (17.914GiB/s) | 20260506 20:02:09.322 | INFO | Downloaded 34GiB to GPU in 1.92843s (17.631GiB/s) | 20260506 20:02:14.460 | INFO | Analyzing chunk frames = [4068352,4068864) | 20260506 20:02:14.460 | INFO | MemInfo: { | 20260506 20:02:14.460 | INFO | "gpu" : " 17 GB", | 20260506 20:02:14.460 | INFO | "huge" : "", | 20260506 20:02:14.460 | INFO | "huge_shared" : " 27 GB", | 20260506 20:02:14.461 | INFO | "malloc" : " 14 GB", | 20260506 20:02:14.461 | INFO | "pinned" : " 29 GB" | 20260506 20:02:14.461 | INFO | } | 20260506 20:02:14.461 | INFO | Perf report for the last 5.13908 seconds: | 20260506 20:02:14.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:02:14.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:14.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:14.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:14.461 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 20:02:14.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:14.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:14.461 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:02:14.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:14.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:14.461 | INFO | Uploaded 36GiB to GPU in 1.97624s (18.2164GiB/s) | 20260506 20:02:14.461 | INFO | Downloaded 34GiB to GPU in 1.91339s (17.7695GiB/s) | 20260506 20:02:19.605 | INFO | Analyzing chunk frames = [4068864,4069376) | 20260506 20:02:19.605 | INFO | MemInfo: { | 20260506 20:02:19.605 | INFO | "gpu" : " 17 GB", | 20260506 20:02:19.605 | INFO | "huge" : "", | 20260506 20:02:19.606 | INFO | "huge_shared" : " 27 GB", | 20260506 20:02:19.606 | INFO | "malloc" : " 14 GB", | 20260506 20:02:19.606 | INFO | "pinned" : " 29 GB" | 20260506 20:02:19.606 | INFO | } | 20260506 20:02:19.606 | INFO | Perf report for the last 5.14512 seconds: | 20260506 20:02:19.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:02:19.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:19.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:19.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:19.606 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 20:02:19.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:19.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:19.606 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 20:02:19.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:19.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:19.606 | INFO | Uploaded 36GiB to GPU in 1.93583s (18.5967GiB/s) | 20260506 20:02:19.606 | INFO | Downloaded 34GiB to GPU in 1.86598s (18.221GiB/s) | 20260506 20:02:24.746 | INFO | Analyzing chunk frames = [4069376,4069888) | 20260506 20:02:24.746 | INFO | MemInfo: { | 20260506 20:02:24.746 | INFO | "gpu" : " 17 GB", | 20260506 20:02:24.746 | INFO | "huge" : "", | 20260506 20:02:24.746 | INFO | "huge_shared" : " 27 GB", | 20260506 20:02:24.746 | INFO | "malloc" : " 14 GB", | 20260506 20:02:24.746 | INFO | "pinned" : " 29 GB" | 20260506 20:02:24.746 | INFO | } | 20260506 20:02:24.746 | INFO | Perf report for the last 5.14029 seconds: | 20260506 20:02:24.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:02:24.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:24.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:24.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:24.746 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 20:02:24.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:24.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:24.746 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 20:02:24.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:24.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:24.746 | INFO | Uploaded 36GiB to GPU in 2.01488s (17.867GiB/s) | 20260506 20:02:24.746 | INFO | Downloaded 34GiB to GPU in 1.90852s (17.8148GiB/s) | 20260506 20:02:29.888 | INFO | Analyzing chunk frames = [4069888,4070400) | 20260506 20:02:29.888 | INFO | MemInfo: { | 20260506 20:02:29.888 | INFO | "gpu" : " 17 GB", | 20260506 20:02:29.888 | INFO | "huge" : "", | 20260506 20:02:29.888 | INFO | "huge_shared" : " 27 GB", | 20260506 20:02:29.888 | INFO | "malloc" : " 14 GB", | 20260506 20:02:29.888 | INFO | "pinned" : " 29 GB" | 20260506 20:02:29.888 | INFO | } | 20260506 20:02:29.888 | INFO | Perf report for the last 5.14181 seconds: | 20260506 20:02:29.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:02:29.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:29.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:29.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:29.888 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 20:02:29.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:29.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:29.888 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 20:02:29.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:29.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:29.888 | INFO | Uploaded 36GiB to GPU in 2.02447s (17.7824GiB/s) | 20260506 20:02:29.888 | INFO | Downloaded 34GiB to GPU in 1.94937s (17.4416GiB/s) | 20260506 20:02:35.025 | INFO | Analyzing chunk frames = [4070400,4070912) | 20260506 20:02:35.025 | INFO | MemInfo: { | 20260506 20:02:35.025 | INFO | "gpu" : " 17 GB", | 20260506 20:02:35.025 | INFO | "huge" : "", | 20260506 20:02:35.025 | INFO | "huge_shared" : " 27 GB", | 20260506 20:02:35.025 | INFO | "malloc" : " 14 GB", | 20260506 20:02:35.025 | INFO | "pinned" : " 29 GB" | 20260506 20:02:35.026 | INFO | } | 20260506 20:02:35.026 | INFO | Perf report for the last 5.1377 seconds: | 20260506 20:02:35.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:02:35.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:35.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:35.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:35.026 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 20:02:35.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:35.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:35.026 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:02:35.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:35.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:35.026 | INFO | Uploaded 36GiB to GPU in 2.0184s (17.8359GiB/s) | 20260506 20:02:35.026 | INFO | Downloaded 34GiB to GPU in 1.98825s (17.1005GiB/s) | 20260506 20:02:40.164 | INFO | Analyzing chunk frames = [4070912,4071424) | 20260506 20:02:40.164 | INFO | MemInfo: { | 20260506 20:02:40.164 | INFO | "gpu" : " 17 GB", | 20260506 20:02:40.164 | INFO | "huge" : "", | 20260506 20:02:40.164 | INFO | "huge_shared" : " 27 GB", | 20260506 20:02:40.164 | INFO | "malloc" : " 14 GB", | 20260506 20:02:40.164 | INFO | "pinned" : " 29 GB" | 20260506 20:02:40.164 | INFO | } | 20260506 20:02:40.164 | INFO | Perf report for the last 5.13824 seconds: | 20260506 20:02:40.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:02:40.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:40.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:40.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:40.164 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 20:02:40.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:40.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:40.164 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 20:02:40.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:40.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:40.164 | INFO | Uploaded 36GiB to GPU in 2.00342s (17.9693GiB/s) | 20260506 20:02:40.164 | INFO | Downloaded 34GiB to GPU in 1.98564s (17.1229GiB/s) | 20260506 20:02:40.164 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:02:40.164 | INFO | Total mem usage: 18 GiB | 20260506 20:02:40.164 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:02:40.164 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -11%) | 20260506 20:02:40.164 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:02:40.164 | INFO | High Water Breakdown: | 20260506 20:02:40.164 | INFO | 62% : 10.1 GiB ( 88.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:02:40.164 | INFO | 37.8% : 6.14 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:02:40.164 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:02:40.164 | INFO | ------------------------------------------ | 20260506 20:02:40.164 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:02:40.164 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:02:40.164 | INFO | Total mem usage: 26.5 GiB | 20260506 20:02:40.164 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:02:40.164 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:02:40.164 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:02:40.164 | INFO | High Water Breakdown: | 20260506 20:02:40.164 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:02:40.164 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:02:40.165 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:02:40.165 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:02:40.165 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:02:40.165 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:02:40.165 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:02:40.165 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:02:40.165 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:02:40.165 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:02:40.165 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:02:40.165 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:02:40.165 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:02:40.165 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:02:40.165 | INFO | ------------------------------------------ | 20260506 20:02:40.165 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:02:40.165 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:02:40.165 | INFO | Total mem usage: 25.6 GiB | 20260506 20:02:40.165 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:02:40.165 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:02:40.165 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:02:40.165 | INFO | High Water Breakdown: | 20260506 20:02:40.165 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:02:40.165 | INFO | ------------------------------------------ | 20260506 20:02:40.165 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:02:40.165 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:02:40.165 | INFO | Total mem usage: 15.7 GiB | 20260506 20:02:40.165 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:02:40.165 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:02:40.165 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 20:02:40.165 | INFO | High Water Breakdown: | 20260506 20:02:40.165 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:02:40.165 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:02:40.165 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:02:40.165 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:02:40.165 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:02:40.165 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:02:40.165 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:02:40.165 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:02:40.165 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:02:40.165 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:02:40.165 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:02:40.165 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:02:40.165 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:02:40.165 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:02:40.165 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:02:40.165 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:02:40.165 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:02:40.165 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:02:40.165 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:02:40.165 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:02:40.165 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:02:40.165 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:02:40.165 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:02:40.165 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:02:40.165 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:02:40.165 | INFO | ------------------------------------------ | 20260506 20:02:40.165 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:02:40.165 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:02:40.165 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:02:40.165 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:02:40.165 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.57e+01, 1.34e+01 / 2.31e+00 | 20260506 20:02:40.165 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 9.88e-03 / 9.10e-03 | 20260506 20:02:40.165 | INFO | PerfReport: 11.64%, Baseline -- 3.63e+00 / 4.43e+00, 5.55e-01 / 3.51e-01 | 20260506 20:02:40.165 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.82e-01 / 1.24e-01 | 20260506 20:02:40.165 | INFO | PerfReport: 16.00%, DME -- 3.99e+01 / 4.44e+01, 4.07e+00 / 2.42e+00 | 20260506 20:02:40.165 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.40e+00 / 2.03e+00 | 20260506 20:02:40.165 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.23e+00 / 2.74e+00, 3.78e-01 / 2.58e-01 | 20260506 20:02:40.165 | INFO | PerfReport: 3.33%, Metrics -- 1.04e+00 / 1.39e+00, 2.75e-01 / 1.84e-01 | 20260506 20:02:40.165 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.57e-01 / 4.06e-01, 5.65e-02 / 2.27e-02 | 20260506 20:02:40.165 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:02:40.165 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:02:40.165 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:02:40.165 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 20:02:40.165 | INFO | PerfReport: 21.47%, Upload -- 2.08e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260506 20:02:40.165 | INFO | PerfReport: 21.18%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 20:02:40.165 | INFO | PerfReport: 32.20%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 20:02:40.165 | INFO | PerfReport: 1.32%, LockUpload -- 1.27e+00 / 0.00e+00, 4.96e+00 / 0.00e+00 | 20260506 20:02:40.165 | INFO | PerfReport: 1.17%, LockDownload -- 1.13e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 20:02:40.165 | INFO | PerfReport: 22.65%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 20:02:40.165 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260506 20:02:40.274 | INFO | Starting a round of Baz IO | 20260506 20:02:43.409 | INFO | Populating RTMetrics report | 20260506 20:02:43.409 | INFO | RTMetrics report populated | 20260506 20:02:43.409 | INFO | Opening tmp RTMetrics json file | 20260506 20:02:43.412 | INFO | Serializing RTMetrics to json | 20260506 20:02:43.430 | INFO | Serialization done, swapping tmp file now... | 20260506 20:02:43.445 | INFO | RTMetrics done | 20260506 20:02:43.685 | INFO | Finished a round of Baz IO: Wrote 1.992 GiB in 3.411 seconds | 20260506 20:02:43.685 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.39% / 0.00% / 65.61% | 20260506 20:02:43.685 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.812777% / 1.19209e-05% / 6.75519e-05% | 20260506 20:02:43.685 | INFO | Baz IO overall: | 20260506 20:02:43.685 | INFO | BytesWritten : 2150681156128 | 20260506 20:02:43.685 | INFO | NumEvents : 1039295469445 | 20260506 20:02:45.303 | INFO | Analyzing chunk frames = [4071424,4071936) | 20260506 20:02:45.303 | INFO | MemInfo: { | 20260506 20:02:45.303 | INFO | "gpu" : " 17 GB", | 20260506 20:02:45.303 | INFO | "huge" : "", | 20260506 20:02:45.303 | INFO | "huge_shared" : " 27 GB", | 20260506 20:02:45.303 | INFO | "malloc" : " 14 GB", | 20260506 20:02:45.303 | INFO | "pinned" : " 29 GB" | 20260506 20:02:45.303 | INFO | } | 20260506 20:02:45.303 | INFO | Perf report for the last 5.13931 seconds: | 20260506 20:02:45.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:02:45.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:45.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:45.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:45.303 | INFO | ANALYSIS : 62.6%, 2.9, 95.4ms, 0 | 20260506 20:02:45.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:45.303 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 20:02:45.303 | INFO | PRE_HQ : 59.9%, 1.8, 57.1ms, 0 | 20260506 20:02:45.303 | INFO | BAZWRITER : 66.1%, 4.8, 163.3ms, 0 | 20260506 20:02:45.303 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 20:02:45.304 | INFO | Uploaded 36GiB to GPU in 2.06992s (17.392GiB/s) | 20260506 20:02:45.304 | INFO | Downloaded 38GiB to GPU in 2.2124s (17.1759GiB/s) | 20260506 20:02:50.435 | INFO | Analyzing chunk frames = [4071936,4072448) | 20260506 20:02:50.436 | INFO | MemInfo: { | 20260506 20:02:50.436 | INFO | "gpu" : " 17 GB", | 20260506 20:02:50.436 | INFO | "huge" : "", | 20260506 20:02:50.436 | INFO | "huge_shared" : " 27 GB", | 20260506 20:02:50.436 | INFO | "malloc" : " 14 GB", | 20260506 20:02:50.436 | INFO | "pinned" : " 29 GB" | 20260506 20:02:50.436 | INFO | } | 20260506 20:02:50.436 | INFO | Perf report for the last 5.1326 seconds: | 20260506 20:02:50.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:02:50.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:50.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:50.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:50.436 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:02:50.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:50.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:50.436 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:02:50.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:50.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:50.436 | INFO | Uploaded 36GiB to GPU in 2.00663s (17.9406GiB/s) | 20260506 20:02:50.436 | INFO | Downloaded 34GiB to GPU in 1.92073s (17.7016GiB/s) | 20260506 20:02:55.580 | INFO | Analyzing chunk frames = [4072448,4072960) | 20260506 20:02:55.580 | INFO | MemInfo: { | 20260506 20:02:55.580 | INFO | "gpu" : " 17 GB", | 20260506 20:02:55.580 | INFO | "huge" : "", | 20260506 20:02:55.580 | INFO | "huge_shared" : " 27 GB", | 20260506 20:02:55.580 | INFO | "malloc" : " 14 GB", | 20260506 20:02:55.580 | INFO | "pinned" : " 29 GB" | 20260506 20:02:55.580 | INFO | } | 20260506 20:02:55.580 | INFO | Perf report for the last 5.1443 seconds: | 20260506 20:02:55.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:02:55.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:55.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:55.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:55.580 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 20:02:55.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:55.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:55.580 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:02:55.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:55.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:02:55.580 | INFO | Uploaded 36GiB to GPU in 1.98606s (18.1263GiB/s) | 20260506 20:02:55.580 | INFO | Downloaded 34GiB to GPU in 1.9187s (17.7203GiB/s) | 20260506 20:03:00.727 | INFO | Analyzing chunk frames = [4072960,4073472) | 20260506 20:03:00.727 | INFO | MemInfo: { | 20260506 20:03:00.727 | INFO | "gpu" : " 17 GB", | 20260506 20:03:00.727 | INFO | "huge" : "", | 20260506 20:03:00.727 | INFO | "huge_shared" : " 27 GB", | 20260506 20:03:00.727 | INFO | "malloc" : " 14 GB", | 20260506 20:03:00.727 | INFO | "pinned" : " 29 GB" | 20260506 20:03:00.727 | INFO | } | 20260506 20:03:00.727 | INFO | Perf report for the last 5.14713 seconds: | 20260506 20:03:00.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:03:00.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:00.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:00.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:00.727 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 20:03:00.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:00.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:00.727 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:03:00.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:00.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:00.727 | INFO | Uploaded 36GiB to GPU in 1.95719s (18.3937GiB/s) | 20260506 20:03:00.728 | INFO | Downloaded 34GiB to GPU in 1.8743s (18.1401GiB/s) | 20260506 20:03:05.862 | INFO | Analyzing chunk frames = [4073472,4073984) | 20260506 20:03:05.862 | INFO | MemInfo: { | 20260506 20:03:05.862 | INFO | "gpu" : " 17 GB", | 20260506 20:03:05.862 | INFO | "huge" : "", | 20260506 20:03:05.862 | INFO | "huge_shared" : " 27 GB", | 20260506 20:03:05.862 | INFO | "malloc" : " 14 GB", | 20260506 20:03:05.862 | INFO | "pinned" : " 29 GB" | 20260506 20:03:05.862 | INFO | } | 20260506 20:03:05.862 | INFO | Perf report for the last 5.13493 seconds: | 20260506 20:03:05.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:03:05.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:05.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:05.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:05.862 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 20:03:05.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:05.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:05.862 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:03:05.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:05.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:05.863 | INFO | Uploaded 36GiB to GPU in 2.01447s (17.8707GiB/s) | 20260506 20:03:05.863 | INFO | Downloaded 34GiB to GPU in 1.92223s (17.6878GiB/s) | 20260506 20:03:11.001 | INFO | Analyzing chunk frames = [4073984,4074496) | 20260506 20:03:11.001 | INFO | MemInfo: { | 20260506 20:03:11.001 | INFO | "gpu" : " 17 GB", | 20260506 20:03:11.001 | INFO | "huge" : "", | 20260506 20:03:11.001 | INFO | "huge_shared" : " 27 GB", | 20260506 20:03:11.001 | INFO | "malloc" : " 14 GB", | 20260506 20:03:11.001 | INFO | "pinned" : " 29 GB" | 20260506 20:03:11.001 | INFO | } | 20260506 20:03:11.001 | INFO | Perf report for the last 5.13909 seconds: | 20260506 20:03:11.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:03:11.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:11.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:11.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:11.001 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:03:11.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:11.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:11.001 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 20:03:11.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:11.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:11.002 | INFO | Uploaded 36GiB to GPU in 2.01735s (17.8452GiB/s) | 20260506 20:03:11.002 | INFO | Downloaded 34GiB to GPU in 1.9455s (17.4763GiB/s) | 20260506 20:03:16.136 | INFO | Analyzing chunk frames = [4074496,4075008) | 20260506 20:03:16.136 | INFO | MemInfo: { | 20260506 20:03:16.136 | INFO | "gpu" : " 17 GB", | 20260506 20:03:16.136 | INFO | "huge" : "", | 20260506 20:03:16.136 | INFO | "huge_shared" : " 27 GB", | 20260506 20:03:16.136 | INFO | "malloc" : " 14 GB", | 20260506 20:03:16.136 | INFO | "pinned" : " 29 GB" | 20260506 20:03:16.136 | INFO | } | 20260506 20:03:16.136 | INFO | Perf report for the last 5.13458 seconds: | 20260506 20:03:16.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:03:16.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:16.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:16.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:16.136 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260506 20:03:16.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:16.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:16.136 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:03:16.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:16.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:16.136 | INFO | Uploaded 36GiB to GPU in 1.99974s (18.0023GiB/s) | 20260506 20:03:16.136 | INFO | Downloaded 34GiB to GPU in 2.00858s (16.9274GiB/s) | 20260506 20:03:21.270 | INFO | Analyzing chunk frames = [4075008,4075520) | 20260506 20:03:21.270 | INFO | MemInfo: { | 20260506 20:03:21.270 | INFO | "gpu" : " 17 GB", | 20260506 20:03:21.270 | INFO | "huge" : "", | 20260506 20:03:21.270 | INFO | "huge_shared" : " 27 GB", | 20260506 20:03:21.270 | INFO | "malloc" : " 14 GB", | 20260506 20:03:21.270 | INFO | "pinned" : " 29 GB" | 20260506 20:03:21.270 | INFO | } | 20260506 20:03:21.271 | INFO | Perf report for the last 5.13461 seconds: | 20260506 20:03:21.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:03:21.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:21.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:21.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:21.271 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 20:03:21.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:21.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:21.271 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 20:03:21.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:21.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:21.271 | INFO | Uploaded 36GiB to GPU in 1.98388s (18.1462GiB/s) | 20260506 20:03:21.271 | INFO | Downloaded 34GiB to GPU in 1.98126s (17.1608GiB/s) | 20260506 20:03:21.380 | INFO | Starting a round of Baz IO | 20260506 20:03:24.513 | INFO | Populating RTMetrics report | 20260506 20:03:24.514 | INFO | RTMetrics report populated | 20260506 20:03:24.514 | INFO | Opening tmp RTMetrics json file | 20260506 20:03:24.536 | INFO | Serializing RTMetrics to json | 20260506 20:03:24.540 | INFO | Serialization done, swapping tmp file now... | 20260506 20:03:24.540 | INFO | RTMetrics done | 20260506 20:03:24.779 | INFO | Finished a round of Baz IO: Wrote 1.991 GiB in 3.399 seconds | 20260506 20:03:24.779 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.38% / 0.00% / 65.61% | 20260506 20:03:24.779 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.812022% / 0% / 6.35783e-05% | 20260506 20:03:24.779 | INFO | Baz IO overall: | 20260506 20:03:24.779 | INFO | BytesWritten : 2152818451427 | 20260506 20:03:24.779 | INFO | NumEvents : 1040321736275 | 20260506 20:03:26.411 | INFO | Analyzing chunk frames = [4075520,4076032) | 20260506 20:03:26.411 | INFO | MemInfo: { | 20260506 20:03:26.411 | INFO | "gpu" : " 17 GB", | 20260506 20:03:26.411 | INFO | "huge" : "", | 20260506 20:03:26.411 | INFO | "huge_shared" : " 27 GB", | 20260506 20:03:26.411 | INFO | "malloc" : " 14 GB", | 20260506 20:03:26.411 | INFO | "pinned" : " 29 GB" | 20260506 20:03:26.411 | INFO | } | 20260506 20:03:26.411 | INFO | Perf report for the last 5.14024 seconds: | 20260506 20:03:26.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:03:26.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:26.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:26.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:26.411 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 20:03:26.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:26.411 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260506 20:03:26.411 | INFO | PRE_HQ : 60.0%, 1.8, 57.3ms, 0 | 20260506 20:03:26.411 | INFO | BAZWRITER : 65.9%, 4.8, 161.7ms, 0 | 20260506 20:03:26.411 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 20:03:26.411 | INFO | Uploaded 36GiB to GPU in 2.05054s (17.5564GiB/s) | 20260506 20:03:26.411 | INFO | Downloaded 38GiB to GPU in 2.19371s (17.3223GiB/s) | 20260506 20:03:31.551 | INFO | Analyzing chunk frames = [4076032,4076544) | 20260506 20:03:31.551 | INFO | MemInfo: { | 20260506 20:03:31.551 | INFO | "gpu" : " 17 GB", | 20260506 20:03:31.551 | INFO | "huge" : "", | 20260506 20:03:31.552 | INFO | "huge_shared" : " 27 GB", | 20260506 20:03:31.552 | INFO | "malloc" : " 14 GB", | 20260506 20:03:31.552 | INFO | "pinned" : " 29 GB" | 20260506 20:03:31.552 | INFO | } | 20260506 20:03:31.552 | INFO | Perf report for the last 5.14095 seconds: | 20260506 20:03:31.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:03:31.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:31.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:31.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:31.552 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 20:03:31.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:31.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:31.552 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:03:31.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:31.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:31.552 | INFO | Uploaded 36GiB to GPU in 2.01869s (17.8333GiB/s) | 20260506 20:03:31.552 | INFO | Downloaded 34GiB to GPU in 1.93714s (17.5517GiB/s) | 20260506 20:03:36.698 | INFO | Analyzing chunk frames = [4076544,4077056) | 20260506 20:03:36.698 | INFO | MemInfo: { | 20260506 20:03:36.698 | INFO | "gpu" : " 17 GB", | 20260506 20:03:36.698 | INFO | "huge" : "", | 20260506 20:03:36.698 | INFO | "huge_shared" : " 27 GB", | 20260506 20:03:36.698 | INFO | "malloc" : " 14 GB", | 20260506 20:03:36.698 | INFO | "pinned" : " 29 GB" | 20260506 20:03:36.698 | INFO | } | 20260506 20:03:36.698 | INFO | Perf report for the last 5.1463 seconds: | 20260506 20:03:36.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:03:36.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:36.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:36.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:36.698 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 20:03:36.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:36.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:36.698 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:03:36.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:36.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:36.698 | INFO | Uploaded 36GiB to GPU in 1.98026s (18.1795GiB/s) | 20260506 20:03:36.698 | INFO | Downloaded 34GiB to GPU in 1.91559s (17.7491GiB/s) | 20260506 20:03:41.839 | INFO | Analyzing chunk frames = [4077056,4077568) | 20260506 20:03:41.839 | INFO | MemInfo: { | 20260506 20:03:41.839 | INFO | "gpu" : " 17 GB", | 20260506 20:03:41.839 | INFO | "huge" : "", | 20260506 20:03:41.839 | INFO | "huge_shared" : " 27 GB", | 20260506 20:03:41.839 | INFO | "malloc" : " 14 GB", | 20260506 20:03:41.839 | INFO | "pinned" : " 29 GB" | 20260506 20:03:41.839 | INFO | } | 20260506 20:03:41.839 | INFO | Perf report for the last 5.14094 seconds: | 20260506 20:03:41.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:03:41.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:41.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:41.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:41.839 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 20:03:41.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:41.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:41.839 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 20:03:41.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:41.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:41.839 | INFO | Uploaded 36GiB to GPU in 1.96572s (18.3139GiB/s) | 20260506 20:03:41.839 | INFO | Downloaded 34GiB to GPU in 1.87263s (18.1563GiB/s) | 20260506 20:03:46.985 | INFO | Analyzing chunk frames = [4077568,4078080) | 20260506 20:03:46.985 | INFO | MemInfo: { | 20260506 20:03:46.985 | INFO | "gpu" : " 17 GB", | 20260506 20:03:46.985 | INFO | "huge" : "", | 20260506 20:03:46.985 | INFO | "huge_shared" : " 27 GB", | 20260506 20:03:46.985 | INFO | "malloc" : " 14 GB", | 20260506 20:03:46.985 | INFO | "pinned" : " 29 GB" | 20260506 20:03:46.985 | INFO | } | 20260506 20:03:46.985 | INFO | Perf report for the last 5.14583 seconds: | 20260506 20:03:46.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:03:46.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:46.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:46.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:46.985 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260506 20:03:46.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:46.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:46.985 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:03:46.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:46.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:46.985 | INFO | Uploaded 36GiB to GPU in 2.01603s (17.8569GiB/s) | 20260506 20:03:46.985 | INFO | Downloaded 34GiB to GPU in 1.91568s (17.7483GiB/s) | 20260506 20:03:52.116 | INFO | Analyzing chunk frames = [4078080,4078592) | 20260506 20:03:52.116 | INFO | MemInfo: { | 20260506 20:03:52.116 | INFO | "gpu" : " 17 GB", | 20260506 20:03:52.116 | INFO | "huge" : "", | 20260506 20:03:52.116 | INFO | "huge_shared" : " 27 GB", | 20260506 20:03:52.116 | INFO | "malloc" : " 14 GB", | 20260506 20:03:52.116 | INFO | "pinned" : " 29 GB" | 20260506 20:03:52.116 | INFO | } | 20260506 20:03:52.116 | INFO | Perf report for the last 5.13151 seconds: | 20260506 20:03:52.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:03:52.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:52.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:52.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:52.116 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 20:03:52.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:52.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:52.116 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 20:03:52.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:52.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:52.117 | INFO | Uploaded 36GiB to GPU in 2.05376s (17.5288GiB/s) | 20260506 20:03:52.117 | INFO | Downloaded 34GiB to GPU in 1.95128s (17.4245GiB/s) | 20260506 20:03:57.251 | INFO | Analyzing chunk frames = [4078592,4079104) | 20260506 20:03:57.251 | INFO | MemInfo: { | 20260506 20:03:57.251 | INFO | "gpu" : " 17 GB", | 20260506 20:03:57.251 | INFO | "huge" : "", | 20260506 20:03:57.251 | INFO | "huge_shared" : " 27 GB", | 20260506 20:03:57.251 | INFO | "malloc" : " 14 GB", | 20260506 20:03:57.251 | INFO | "pinned" : " 29 GB" | 20260506 20:03:57.252 | INFO | } | 20260506 20:03:57.252 | INFO | Perf report for the last 5.13521 seconds: | 20260506 20:03:57.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:03:57.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:57.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:57.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:57.252 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 20:03:57.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:57.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:57.252 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:03:57.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:57.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:03:57.252 | INFO | Uploaded 36GiB to GPU in 2.0152s (17.8643GiB/s) | 20260506 20:03:57.252 | INFO | Downloaded 34GiB to GPU in 1.99278s (17.0616GiB/s) | 20260506 20:04:02.389 | INFO | Analyzing chunk frames = [4079104,4079616) | 20260506 20:04:02.389 | INFO | MemInfo: { | 20260506 20:04:02.389 | INFO | "gpu" : " 17 GB", | 20260506 20:04:02.389 | INFO | "huge" : "", | 20260506 20:04:02.389 | INFO | "huge_shared" : " 27 GB", | 20260506 20:04:02.389 | INFO | "malloc" : " 14 GB", | 20260506 20:04:02.389 | INFO | "pinned" : " 29 GB" | 20260506 20:04:02.389 | INFO | } | 20260506 20:04:02.389 | INFO | Perf report for the last 5.13751 seconds: | 20260506 20:04:02.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:04:02.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:02.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:02.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:02.389 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 20:04:02.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:02.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:02.389 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 20:04:02.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:02.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:02.389 | INFO | Uploaded 36GiB to GPU in 2.02691s (17.761GiB/s) | 20260506 20:04:02.389 | INFO | Downloaded 34GiB to GPU in 1.99008s (17.0847GiB/s) | 20260506 20:04:02.389 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:04:02.389 | INFO | Total mem usage: 18 GiB | 20260506 20:04:02.389 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:04:02.389 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.3%) | 20260506 20:04:02.389 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:04:02.390 | INFO | High Water Breakdown: | 20260506 20:04:02.390 | INFO | 61.4% : 9.5 GiB ( 83.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:04:02.390 | INFO | 38.5% : 5.96 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:04:02.390 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:04:02.390 | INFO | ------------------------------------------ | 20260506 20:04:02.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:04:02.390 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:04:02.390 | INFO | Total mem usage: 26.5 GiB | 20260506 20:04:02.390 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:04:02.390 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:04:02.390 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:04:02.390 | INFO | High Water Breakdown: | 20260506 20:04:02.390 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:04:02.390 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:04:02.390 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:04:02.390 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:04:02.390 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:04:02.390 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:04:02.390 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:04:02.390 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:04:02.390 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:04:02.390 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:04:02.390 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:04:02.390 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:04:02.390 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:04:02.390 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:04:02.390 | INFO | ------------------------------------------ | 20260506 20:04:02.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:04:02.390 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:04:02.390 | INFO | Total mem usage: 25.6 GiB | 20260506 20:04:02.390 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:04:02.390 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:04:02.390 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:04:02.390 | INFO | High Water Breakdown: | 20260506 20:04:02.390 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:04:02.390 | INFO | ------------------------------------------ | 20260506 20:04:02.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:04:02.390 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:04:02.390 | INFO | Total mem usage: 15.7 GiB | 20260506 20:04:02.390 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:04:02.391 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:04:02.391 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:04:02.391 | INFO | High Water Breakdown: | 20260506 20:04:02.391 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:04:02.391 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:04:02.391 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:04:02.391 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:04:02.391 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:04:02.391 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:04:02.391 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:04:02.391 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:04:02.391 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:04:02.391 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:04:02.391 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:04:02.391 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:04:02.391 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:04:02.391 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:04:02.391 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:04:02.391 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:04:02.391 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:04:02.391 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:04:02.391 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:04:02.391 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:04:02.391 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:04:02.391 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:04:02.391 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:04:02.391 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:04:02.391 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:04:02.391 | INFO | ------------------------------------------ | 20260506 20:04:02.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:04:02.391 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:04:02.391 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:04:02.391 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:04:02.391 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.56e+01, 1.33e+01 / 2.49e+00 | 20260506 20:04:02.391 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 9.95e-03 / 5.41e-03 | 20260506 20:04:02.391 | INFO | PerfReport: 11.64%, Baseline -- 3.63e+00 / 4.43e+00, 5.57e-01 / 3.06e-01 | 20260506 20:04:02.391 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.81e-01 / 1.18e-01 | 20260506 20:04:02.391 | INFO | PerfReport: 16.01%, DME -- 3.99e+01 / 4.44e+01, 4.10e+00 / 2.91e+00 | 20260506 20:04:02.391 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.41e+00 / 2.07e+00 | 20260506 20:04:02.391 | INFO | PerfReport: 7.18%, PulseAccumulating -- 2.24e+00 / 2.74e+00, 3.82e-01 / 2.69e-01 | 20260506 20:04:02.391 | INFO | PerfReport: 3.33%, Metrics -- 1.04e+00 / 1.35e+00, 2.74e-01 / 2.01e-01 | 20260506 20:04:02.391 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.57e-01 / 4.03e-01, 5.66e-02 / 2.76e-02 | 20260506 20:04:02.391 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:04:02.391 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:04:02.391 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:04:02.391 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 20:04:02.391 | INFO | PerfReport: 21.60%, Upload -- 2.09e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 20:04:02.391 | INFO | PerfReport: 21.19%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 20:04:02.391 | INFO | PerfReport: 32.17%, Compute -- 3.12e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 20:04:02.391 | INFO | PerfReport: 1.41%, LockUpload -- 1.37e+00 / 0.00e+00, 5.17e+00 / 0.00e+00 | 20260506 20:04:02.391 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260506 20:04:02.391 | INFO | PerfReport: 22.52%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 20:04:02.391 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260506 20:04:02.523 | INFO | Starting a round of Baz IO | 20260506 20:04:05.631 | INFO | Populating RTMetrics report | 20260506 20:04:05.632 | INFO | RTMetrics report populated | 20260506 20:04:05.632 | INFO | Opening tmp RTMetrics json file | 20260506 20:04:05.720 | INFO | Serializing RTMetrics to json | 20260506 20:04:05.761 | INFO | Serialization done, swapping tmp file now... | 20260506 20:04:05.797 | INFO | RTMetrics done | 20260506 20:04:06.227 | INFO | Finished a round of Baz IO: Wrote 1.991 GiB in 3.703 seconds | 20260506 20:04:06.227 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.38% / 0.00% / 65.62% | 20260506 20:04:06.227 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.811835% / 7.94729e-06% / 6.75519e-05% | 20260506 20:04:06.227 | INFO | Baz IO overall: | 20260506 20:04:06.227 | INFO | BytesWritten : 2154956572942 | 20260506 20:04:06.227 | INFO | NumEvents : 1041348900039 | 20260506 20:04:07.531 | INFO | Analyzing chunk frames = [4079616,4080128) | 20260506 20:04:07.531 | INFO | MemInfo: { | 20260506 20:04:07.531 | INFO | "gpu" : " 17 GB", | 20260506 20:04:07.531 | INFO | "huge" : "", | 20260506 20:04:07.531 | INFO | "huge_shared" : " 27 GB", | 20260506 20:04:07.531 | INFO | "malloc" : " 14 GB", | 20260506 20:04:07.531 | INFO | "pinned" : " 29 GB" | 20260506 20:04:07.531 | INFO | } | 20260506 20:04:07.531 | INFO | Perf report for the last 5.14177 seconds: | 20260506 20:04:07.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:04:07.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:07.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:07.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:07.531 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 20:04:07.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:07.531 | INFO | RT_METRICS : 46.0%, 1.0, 24.2ms, 0 | 20260506 20:04:07.531 | INFO | PRE_HQ : 59.8%, 1.8, 57.3ms, 0 | 20260506 20:04:07.531 | INFO | BAZWRITER : 72.0%, 4.8, 178.0ms, 0 | 20260506 20:04:07.531 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 20:04:07.531 | INFO | Uploaded 36GiB to GPU in 2.03277s (17.7098GiB/s) | 20260506 20:04:07.531 | INFO | Downloaded 38GiB to GPU in 2.2059s (17.2265GiB/s) | 20260506 20:04:12.673 | INFO | Analyzing chunk frames = [4080128,4080640) | 20260506 20:04:12.673 | INFO | MemInfo: { | 20260506 20:04:12.673 | INFO | "gpu" : " 17 GB", | 20260506 20:04:12.673 | INFO | "huge" : "", | 20260506 20:04:12.673 | INFO | "huge_shared" : " 27 GB", | 20260506 20:04:12.673 | INFO | "malloc" : " 14 GB", | 20260506 20:04:12.673 | INFO | "pinned" : " 29 GB" | 20260506 20:04:12.673 | INFO | } | 20260506 20:04:12.673 | INFO | Perf report for the last 5.1423 seconds: | 20260506 20:04:12.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:04:12.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:12.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:12.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:12.673 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 20:04:12.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:12.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:12.673 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:04:12.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:12.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:12.673 | INFO | Uploaded 36GiB to GPU in 2.01911s (17.8296GiB/s) | 20260506 20:04:12.674 | INFO | Downloaded 34GiB to GPU in 1.92569s (17.656GiB/s) | 20260506 20:04:17.814 | INFO | Analyzing chunk frames = [4080640,4081152) | 20260506 20:04:17.814 | INFO | MemInfo: { | 20260506 20:04:17.814 | INFO | "gpu" : " 17 GB", | 20260506 20:04:17.814 | INFO | "huge" : "", | 20260506 20:04:17.814 | INFO | "huge_shared" : " 27 GB", | 20260506 20:04:17.814 | INFO | "malloc" : " 14 GB", | 20260506 20:04:17.814 | INFO | "pinned" : " 29 GB" | 20260506 20:04:17.815 | INFO | } | 20260506 20:04:17.815 | INFO | Perf report for the last 5.14137 seconds: | 20260506 20:04:17.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:04:17.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:17.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:17.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:17.815 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 20:04:17.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:17.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:17.815 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:04:17.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:17.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:17.815 | INFO | Uploaded 36GiB to GPU in 1.98006s (18.1813GiB/s) | 20260506 20:04:17.815 | INFO | Downloaded 34GiB to GPU in 1.91652s (17.7405GiB/s) | 20260506 20:04:22.948 | INFO | Analyzing chunk frames = [4081152,4081664) | 20260506 20:04:22.948 | INFO | MemInfo: { | 20260506 20:04:22.948 | INFO | "gpu" : " 17 GB", | 20260506 20:04:22.948 | INFO | "huge" : "", | 20260506 20:04:22.949 | INFO | "huge_shared" : " 27 GB", | 20260506 20:04:22.949 | INFO | "malloc" : " 14 GB", | 20260506 20:04:22.949 | INFO | "pinned" : " 29 GB" | 20260506 20:04:22.949 | INFO | } | 20260506 20:04:22.949 | INFO | Perf report for the last 5.13413 seconds: | 20260506 20:04:22.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:04:22.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:22.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:22.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:22.949 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 20:04:22.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:22.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:22.949 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:04:22.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:22.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:22.949 | INFO | Uploaded 36GiB to GPU in 1.95269s (18.4361GiB/s) | 20260506 20:04:22.949 | INFO | Downloaded 34GiB to GPU in 1.86867s (18.1947GiB/s) | 20260506 20:04:28.088 | INFO | Analyzing chunk frames = [4081664,4082176) | 20260506 20:04:28.088 | INFO | MemInfo: { | 20260506 20:04:28.088 | INFO | "gpu" : " 17 GB", | 20260506 20:04:28.088 | INFO | "huge" : "", | 20260506 20:04:28.088 | INFO | "huge_shared" : " 27 GB", | 20260506 20:04:28.088 | INFO | "malloc" : " 14 GB", | 20260506 20:04:28.088 | INFO | "pinned" : " 29 GB" | 20260506 20:04:28.088 | INFO | } | 20260506 20:04:28.088 | INFO | Perf report for the last 5.13945 seconds: | 20260506 20:04:28.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:04:28.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:28.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:28.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:28.088 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 20:04:28.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:28.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:28.088 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 20:04:28.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:28.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:28.088 | INFO | Uploaded 36GiB to GPU in 2.01886s (17.8318GiB/s) | 20260506 20:04:28.088 | INFO | Downloaded 34GiB to GPU in 1.91472s (17.7572GiB/s) | 20260506 20:04:33.226 | INFO | Analyzing chunk frames = [4082176,4082688) | 20260506 20:04:33.226 | INFO | MemInfo: { | 20260506 20:04:33.226 | INFO | "gpu" : " 17 GB", | 20260506 20:04:33.226 | INFO | "huge" : "", | 20260506 20:04:33.226 | INFO | "huge_shared" : " 27 GB", | 20260506 20:04:33.226 | INFO | "malloc" : " 14 GB", | 20260506 20:04:33.226 | INFO | "pinned" : " 29 GB" | 20260506 20:04:33.226 | INFO | } | 20260506 20:04:33.226 | INFO | Perf report for the last 5.13817 seconds: | 20260506 20:04:33.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:04:33.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:33.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:33.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:33.226 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 20:04:33.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:33.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:33.226 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 20:04:33.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:33.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:33.227 | INFO | Uploaded 36GiB to GPU in 2.04597s (17.5956GiB/s) | 20260506 20:04:33.227 | INFO | Downloaded 34GiB to GPU in 1.95555s (17.3864GiB/s) | 20260506 20:04:38.365 | INFO | Analyzing chunk frames = [4082688,4083200) | 20260506 20:04:38.365 | INFO | MemInfo: { | 20260506 20:04:38.365 | INFO | "gpu" : " 17 GB", | 20260506 20:04:38.365 | INFO | "huge" : "", | 20260506 20:04:38.366 | INFO | "huge_shared" : " 27 GB", | 20260506 20:04:38.366 | INFO | "malloc" : " 14 GB", | 20260506 20:04:38.366 | INFO | "pinned" : " 29 GB" | 20260506 20:04:38.366 | INFO | } | 20260506 20:04:38.366 | INFO | Perf report for the last 5.13928 seconds: | 20260506 20:04:38.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:04:38.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:38.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:38.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:38.366 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 20:04:38.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:38.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:38.366 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 20:04:38.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:38.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:38.366 | INFO | Uploaded 36GiB to GPU in 2.00434s (17.961GiB/s) | 20260506 20:04:38.366 | INFO | Downloaded 34GiB to GPU in 1.99581s (17.0357GiB/s) | 20260506 20:04:43.502 | INFO | Analyzing chunk frames = [4083200,4083712) | 20260506 20:04:43.502 | INFO | MemInfo: { | 20260506 20:04:43.502 | INFO | "gpu" : " 17 GB", | 20260506 20:04:43.503 | INFO | "huge" : "", | 20260506 20:04:43.503 | INFO | "huge_shared" : " 27 GB", | 20260506 20:04:43.503 | INFO | "malloc" : " 14 GB", | 20260506 20:04:43.503 | INFO | "pinned" : " 29 GB" | 20260506 20:04:43.503 | INFO | } | 20260506 20:04:43.503 | INFO | Perf report for the last 5.13704 seconds: | 20260506 20:04:43.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:04:43.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:43.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:43.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:43.503 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:04:43.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:43.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:43.503 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 20:04:43.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:43.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:43.503 | INFO | Uploaded 36GiB to GPU in 1.95628s (18.4023GiB/s) | 20260506 20:04:43.503 | INFO | Downloaded 34GiB to GPU in 1.98867s (17.0968GiB/s) | 20260506 20:04:43.606 | INFO | Starting a round of Baz IO | 20260506 20:04:46.740 | INFO | Populating RTMetrics report | 20260506 20:04:46.740 | INFO | RTMetrics report populated | 20260506 20:04:46.740 | INFO | Opening tmp RTMetrics json file | 20260506 20:04:46.741 | INFO | Serializing RTMetrics to json | 20260506 20:04:46.743 | INFO | Serialization done, swapping tmp file now... | 20260506 20:04:46.744 | INFO | RTMetrics done | 20260506 20:04:47.134 | INFO | Finished a round of Baz IO: Wrote 1.99 GiB in 3.528 seconds | 20260506 20:04:47.134 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.38% / 0.00% / 65.62% | 20260506 20:04:47.134 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.807226% / 0% / 4.76837e-05% | 20260506 20:04:47.134 | INFO | Baz IO overall: | 20260506 20:04:47.134 | INFO | BytesWritten : 2157093110570 | 20260506 20:04:47.134 | INFO | NumEvents : 1042374751849 | 20260506 20:04:48.647 | INFO | Analyzing chunk frames = [4083712,4084224) | 20260506 20:04:48.647 | INFO | MemInfo: { | 20260506 20:04:48.647 | INFO | "gpu" : " 17 GB", | 20260506 20:04:48.647 | INFO | "huge" : "", | 20260506 20:04:48.647 | INFO | "huge_shared" : " 27 GB", | 20260506 20:04:48.647 | INFO | "malloc" : " 14 GB", | 20260506 20:04:48.647 | INFO | "pinned" : " 29 GB" | 20260506 20:04:48.647 | INFO | } | 20260506 20:04:48.647 | INFO | Perf report for the last 5.14415 seconds: | 20260506 20:04:48.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:04:48.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:48.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:48.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:48.647 | INFO | ANALYSIS : 62.4%, 2.9, 95.1ms, 0 | 20260506 20:04:48.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:48.647 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 20:04:48.647 | INFO | PRE_HQ : 60.1%, 1.8, 57.6ms, 0 | 20260506 20:04:48.647 | INFO | BAZWRITER : 68.3%, 4.8, 168.3ms, 0 | 20260506 20:04:48.647 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 20:04:48.647 | INFO | Uploaded 36GiB to GPU in 2.03215s (17.7152GiB/s) | 20260506 20:04:48.647 | INFO | Downloaded 38GiB to GPU in 2.19534s (17.3094GiB/s) | 20260506 20:04:53.794 | INFO | Analyzing chunk frames = [4084224,4084736) | 20260506 20:04:53.794 | INFO | MemInfo: { | 20260506 20:04:53.794 | INFO | "gpu" : " 17 GB", | 20260506 20:04:53.794 | INFO | "huge" : "", | 20260506 20:04:53.794 | INFO | "huge_shared" : " 27 GB", | 20260506 20:04:53.794 | INFO | "malloc" : " 14 GB", | 20260506 20:04:53.794 | INFO | "pinned" : " 29 GB" | 20260506 20:04:53.794 | INFO | } | 20260506 20:04:53.794 | INFO | Perf report for the last 5.14699 seconds: | 20260506 20:04:53.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:04:53.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:53.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:53.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:53.794 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:04:53.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:53.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:53.794 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:04:53.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:53.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:53.794 | INFO | Uploaded 36GiB to GPU in 2.01141s (17.8979GiB/s) | 20260506 20:04:53.794 | INFO | Downloaded 34GiB to GPU in 1.93148s (17.6031GiB/s) | 20260506 20:04:58.930 | INFO | Analyzing chunk frames = [4084736,4085248) | 20260506 20:04:58.930 | INFO | MemInfo: { | 20260506 20:04:58.930 | INFO | "gpu" : " 17 GB", | 20260506 20:04:58.930 | INFO | "huge" : "", | 20260506 20:04:58.930 | INFO | "huge_shared" : " 27 GB", | 20260506 20:04:58.930 | INFO | "malloc" : " 14 GB", | 20260506 20:04:58.930 | INFO | "pinned" : " 29 GB" | 20260506 20:04:58.930 | INFO | } | 20260506 20:04:58.930 | INFO | Perf report for the last 5.13624 seconds: | 20260506 20:04:58.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:04:58.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:58.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:58.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:58.930 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 20:04:58.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:58.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:58.930 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:04:58.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:58.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:04:58.930 | INFO | Uploaded 36GiB to GPU in 1.97633s (18.2156GiB/s) | 20260506 20:04:58.930 | INFO | Downloaded 34GiB to GPU in 1.93003s (17.6163GiB/s) | 20260506 20:05:04.072 | INFO | Analyzing chunk frames = [4085248,4085760) | 20260506 20:05:04.072 | INFO | MemInfo: { | 20260506 20:05:04.072 | INFO | "gpu" : " 17 GB", | 20260506 20:05:04.072 | INFO | "huge" : "", | 20260506 20:05:04.073 | INFO | "huge_shared" : " 27 GB", | 20260506 20:05:04.073 | INFO | "malloc" : " 14 GB", | 20260506 20:05:04.073 | INFO | "pinned" : " 29 GB" | 20260506 20:05:04.073 | INFO | } | 20260506 20:05:04.073 | INFO | Perf report for the last 5.14254 seconds: | 20260506 20:05:04.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:05:04.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:04.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:04.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:04.073 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:05:04.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:04.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:04.073 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:05:04.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:04.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:04.073 | INFO | Uploaded 36GiB to GPU in 1.94292s (18.5288GiB/s) | 20260506 20:05:04.073 | INFO | Downloaded 34GiB to GPU in 1.8765s (18.1189GiB/s) | 20260506 20:05:09.202 | INFO | Analyzing chunk frames = [4085760,4086272) | 20260506 20:05:09.202 | INFO | MemInfo: { | 20260506 20:05:09.202 | INFO | "gpu" : " 17 GB", | 20260506 20:05:09.202 | INFO | "huge" : "", | 20260506 20:05:09.202 | INFO | "huge_shared" : " 27 GB", | 20260506 20:05:09.202 | INFO | "malloc" : " 14 GB", | 20260506 20:05:09.202 | INFO | "pinned" : " 29 GB" | 20260506 20:05:09.202 | INFO | } | 20260506 20:05:09.202 | INFO | Perf report for the last 5.12918 seconds: | 20260506 20:05:09.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:05:09.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:09.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:09.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:09.202 | INFO | ANALYSIS : 60.5%, 3.0, 94.0ms, 0 | 20260506 20:05:09.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:09.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:09.202 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:05:09.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:09.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:09.202 | INFO | Uploaded 36GiB to GPU in 2.01213s (17.8915GiB/s) | 20260506 20:05:09.202 | INFO | Downloaded 34GiB to GPU in 1.92209s (17.6891GiB/s) | 20260506 20:05:14.339 | INFO | Analyzing chunk frames = [4086272,4086784) | 20260506 20:05:14.339 | INFO | MemInfo: { | 20260506 20:05:14.339 | INFO | "gpu" : " 17 GB", | 20260506 20:05:14.339 | INFO | "huge" : "", | 20260506 20:05:14.340 | INFO | "huge_shared" : " 27 GB", | 20260506 20:05:14.340 | INFO | "malloc" : " 14 GB", | 20260506 20:05:14.340 | INFO | "pinned" : " 29 GB" | 20260506 20:05:14.340 | INFO | } | 20260506 20:05:14.340 | INFO | Perf report for the last 5.13781 seconds: | 20260506 20:05:14.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:05:14.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:14.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:14.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:14.340 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 20:05:14.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:14.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:14.340 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 20:05:14.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:14.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:14.340 | INFO | Uploaded 36GiB to GPU in 2.00153s (17.9863GiB/s) | 20260506 20:05:14.340 | INFO | Downloaded 34GiB to GPU in 1.94422s (17.4877GiB/s) | 20260506 20:05:19.486 | INFO | Analyzing chunk frames = [4086784,4087296) | 20260506 20:05:19.486 | INFO | MemInfo: { | 20260506 20:05:19.486 | INFO | "gpu" : " 17 GB", | 20260506 20:05:19.486 | INFO | "huge" : "", | 20260506 20:05:19.486 | INFO | "huge_shared" : " 27 GB", | 20260506 20:05:19.486 | INFO | "malloc" : " 14 GB", | 20260506 20:05:19.486 | INFO | "pinned" : " 29 GB" | 20260506 20:05:19.486 | INFO | } | 20260506 20:05:19.486 | INFO | Perf report for the last 5.14644 seconds: | 20260506 20:05:19.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:05:19.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:19.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:19.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:19.486 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 20:05:19.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:19.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:19.486 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:05:19.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:19.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:19.486 | INFO | Uploaded 36GiB to GPU in 1.9748s (18.2297GiB/s) | 20260506 20:05:19.487 | INFO | Downloaded 34GiB to GPU in 2.00802s (16.9321GiB/s) | 20260506 20:05:24.622 | INFO | Analyzing chunk frames = [4087296,4087808) | 20260506 20:05:24.622 | INFO | MemInfo: { | 20260506 20:05:24.622 | INFO | "gpu" : " 17 GB", | 20260506 20:05:24.622 | INFO | "huge" : "", | 20260506 20:05:24.622 | INFO | "huge_shared" : " 27 GB", | 20260506 20:05:24.622 | INFO | "malloc" : " 14 GB", | 20260506 20:05:24.622 | INFO | "pinned" : " 29 GB" | 20260506 20:05:24.622 | INFO | } | 20260506 20:05:24.622 | INFO | Perf report for the last 5.13615 seconds: | 20260506 20:05:24.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:05:24.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:24.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:24.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:24.622 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 20:05:24.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:24.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:24.622 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 20:05:24.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:24.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:24.623 | INFO | Uploaded 36GiB to GPU in 2.00405s (17.9636GiB/s) | 20260506 20:05:24.623 | INFO | Downloaded 34GiB to GPU in 1.99255s (17.0635GiB/s) | 20260506 20:05:24.623 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:05:24.623 | INFO | Total mem usage: 18 GiB | 20260506 20:05:24.623 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:05:24.623 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -10.9%) | 20260506 20:05:24.623 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:05:24.623 | INFO | High Water Breakdown: | 20260506 20:05:24.623 | INFO | 62.1% : 10.1 GiB ( 88.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:05:24.623 | INFO | 37.8% : 6.14 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:05:24.623 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:05:24.623 | INFO | ------------------------------------------ | 20260506 20:05:24.623 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:05:24.623 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:05:24.623 | INFO | Total mem usage: 26.5 GiB | 20260506 20:05:24.623 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:05:24.623 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:05:24.623 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:05:24.623 | INFO | High Water Breakdown: | 20260506 20:05:24.623 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:05:24.623 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:05:24.623 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:05:24.623 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:05:24.623 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:05:24.623 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:05:24.623 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:05:24.623 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:05:24.623 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:05:24.623 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:05:24.623 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:05:24.623 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:05:24.623 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:05:24.623 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:05:24.623 | INFO | ------------------------------------------ | 20260506 20:05:24.623 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:05:24.623 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:05:24.623 | INFO | Total mem usage: 25.6 GiB | 20260506 20:05:24.623 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:05:24.623 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:05:24.623 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:05:24.623 | INFO | High Water Breakdown: | 20260506 20:05:24.623 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:05:24.623 | INFO | ------------------------------------------ | 20260506 20:05:24.623 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:05:24.623 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:05:24.623 | INFO | Total mem usage: 15.7 GiB | 20260506 20:05:24.623 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:05:24.623 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:05:24.623 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:05:24.623 | INFO | High Water Breakdown: | 20260506 20:05:24.623 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:05:24.623 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:05:24.623 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:05:24.623 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:05:24.623 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:05:24.623 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:05:24.623 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:05:24.623 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:05:24.624 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:05:24.624 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:05:24.624 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:05:24.624 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:05:24.624 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:05:24.624 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:05:24.624 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:05:24.624 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:05:24.624 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:05:24.624 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:05:24.624 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:05:24.624 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:05:24.624 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:05:24.624 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:05:24.624 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:05:24.624 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:05:24.624 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:05:24.624 | INFO | ------------------------------------------ | 20260506 20:05:24.624 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:05:24.624 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:05:24.624 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:05:24.624 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:05:24.624 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.59e+01, 1.35e+01 / 2.10e+00 | 20260506 20:05:24.624 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.02e-01, 1.04e-02 / 6.62e-03 | 20260506 20:05:24.624 | INFO | PerfReport: 11.63%, Baseline -- 3.63e+00 / 4.41e+00, 5.54e-01 / 3.17e-01 | 20260506 20:05:24.624 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.82e-01 / 1.23e-01 | 20260506 20:05:24.624 | INFO | PerfReport: 16.01%, DME -- 3.99e+01 / 4.49e+01, 4.25e+00 / 2.55e+00 | 20260506 20:05:24.624 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.42e+00 / 1.89e+00 | 20260506 20:05:24.624 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.23e+00 / 2.80e+00, 3.79e-01 / 2.40e-01 | 20260506 20:05:24.624 | INFO | PerfReport: 3.33%, Metrics -- 1.04e+00 / 1.38e+00, 2.75e-01 / 9.96e-02 | 20260506 20:05:24.624 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.57e-01 / 4.07e-01, 5.73e-02 / 2.43e-02 | 20260506 20:05:24.624 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:05:24.624 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:05:24.624 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:05:24.624 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 20:05:24.624 | INFO | PerfReport: 21.47%, Upload -- 2.08e+01 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260506 20:05:24.624 | INFO | PerfReport: 21.21%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 20:05:24.624 | INFO | PerfReport: 32.19%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 20:05:24.624 | INFO | PerfReport: 1.35%, LockUpload -- 1.31e+00 / 0.00e+00, 5.28e+00 / 0.00e+00 | 20260506 20:05:24.624 | INFO | PerfReport: 1.21%, LockDownload -- 1.17e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260506 20:05:24.624 | INFO | PerfReport: 22.56%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 20:05:24.624 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260506 20:05:24.735 | INFO | Starting a round of Baz IO | 20260506 20:05:27.870 | INFO | Populating RTMetrics report | 20260506 20:05:27.870 | INFO | RTMetrics report populated | 20260506 20:05:27.870 | INFO | Opening tmp RTMetrics json file | 20260506 20:05:27.895 | INFO | Serializing RTMetrics to json | 20260506 20:05:27.900 | INFO | Serialization done, swapping tmp file now... | 20260506 20:05:27.905 | INFO | RTMetrics done | 20260506 20:05:28.335 | INFO | Finished a round of Baz IO: Wrote 1.988 GiB in 3.599 seconds | 20260506 20:05:28.335 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.37% / 0.00% / 65.62% | 20260506 20:05:28.335 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.806228% / 7.94729e-06% / 9.13938e-05% | 20260506 20:05:28.335 | INFO | Baz IO overall: | 20260506 20:05:28.335 | INFO | BytesWritten : 2159228077095 | 20260506 20:05:28.335 | INFO | NumEvents : 1043399696578 | 20260506 20:05:29.762 | INFO | Analyzing chunk frames = [4087808,4088320) | 20260506 20:05:29.762 | INFO | MemInfo: { | 20260506 20:05:29.762 | INFO | "gpu" : " 17 GB", | 20260506 20:05:29.762 | INFO | "huge" : "", | 20260506 20:05:29.762 | INFO | "huge_shared" : " 27 GB", | 20260506 20:05:29.762 | INFO | "malloc" : " 14 GB", | 20260506 20:05:29.762 | INFO | "pinned" : " 29 GB" | 20260506 20:05:29.762 | INFO | } | 20260506 20:05:29.762 | INFO | Perf report for the last 5.13991 seconds: | 20260506 20:05:29.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:05:29.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:29.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:29.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:29.762 | INFO | ANALYSIS : 62.6%, 2.9, 95.8ms, 0 | 20260506 20:05:29.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:29.762 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 20:05:29.762 | INFO | PRE_HQ : 59.8%, 1.8, 56.9ms, 0 | 20260506 20:05:29.762 | INFO | BAZWRITER : 69.8%, 4.8, 171.3ms, 0 | 20260506 20:05:29.762 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 20:05:29.763 | INFO | Uploaded 36GiB to GPU in 2.03645s (17.6778GiB/s) | 20260506 20:05:29.763 | INFO | Downloaded 38GiB to GPU in 2.21025s (17.1926GiB/s) | 20260506 20:05:34.902 | INFO | Analyzing chunk frames = [4088320,4088832) | 20260506 20:05:34.902 | INFO | MemInfo: { | 20260506 20:05:34.902 | INFO | "gpu" : " 17 GB", | 20260506 20:05:34.902 | INFO | "huge" : "", | 20260506 20:05:34.902 | INFO | "huge_shared" : " 27 GB", | 20260506 20:05:34.902 | INFO | "malloc" : " 14 GB", | 20260506 20:05:34.902 | INFO | "pinned" : " 29 GB" | 20260506 20:05:34.902 | INFO | } | 20260506 20:05:34.902 | INFO | Perf report for the last 5.1399 seconds: | 20260506 20:05:34.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:05:34.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:34.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:34.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:34.902 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:05:34.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:34.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:34.902 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:05:34.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:34.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:34.902 | INFO | Uploaded 36GiB to GPU in 2.00788s (17.9293GiB/s) | 20260506 20:05:34.902 | INFO | Downloaded 34GiB to GPU in 1.92739s (17.6405GiB/s) | 20260506 20:05:40.040 | INFO | Analyzing chunk frames = [4088832,4089344) | 20260506 20:05:40.040 | INFO | MemInfo: { | 20260506 20:05:40.040 | INFO | "gpu" : " 17 GB", | 20260506 20:05:40.040 | INFO | "huge" : "", | 20260506 20:05:40.040 | INFO | "huge_shared" : " 27 GB", | 20260506 20:05:40.040 | INFO | "malloc" : " 14 GB", | 20260506 20:05:40.040 | INFO | "pinned" : " 29 GB" | 20260506 20:05:40.040 | INFO | } | 20260506 20:05:40.040 | INFO | Perf report for the last 5.13761 seconds: | 20260506 20:05:40.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:05:40.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:40.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:40.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:40.040 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 20:05:40.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:40.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:40.040 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:05:40.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:40.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:40.040 | INFO | Uploaded 36GiB to GPU in 1.96276s (18.3415GiB/s) | 20260506 20:05:40.040 | INFO | Downloaded 34GiB to GPU in 1.91822s (17.7248GiB/s) | 20260506 20:05:45.179 | INFO | Analyzing chunk frames = [4089344,4089856) | 20260506 20:05:45.179 | INFO | MemInfo: { | 20260506 20:05:45.180 | INFO | "gpu" : " 17 GB", | 20260506 20:05:45.180 | INFO | "huge" : "", | 20260506 20:05:45.180 | INFO | "huge_shared" : " 27 GB", | 20260506 20:05:45.180 | INFO | "malloc" : " 14 GB", | 20260506 20:05:45.180 | INFO | "pinned" : " 29 GB" | 20260506 20:05:45.180 | INFO | } | 20260506 20:05:45.180 | INFO | Perf report for the last 5.13993 seconds: | 20260506 20:05:45.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:05:45.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:45.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:45.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:45.180 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 20:05:45.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:45.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:45.180 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:05:45.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:45.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:45.180 | INFO | Uploaded 36GiB to GPU in 1.91352s (18.8135GiB/s) | 20260506 20:05:45.180 | INFO | Downloaded 34GiB to GPU in 1.87275s (18.1552GiB/s) | 20260506 20:05:50.318 | INFO | Analyzing chunk frames = [4089856,4090368) | 20260506 20:05:50.318 | INFO | MemInfo: { | 20260506 20:05:50.318 | INFO | "gpu" : " 17 GB", | 20260506 20:05:50.318 | INFO | "huge" : "", | 20260506 20:05:50.318 | INFO | "huge_shared" : " 27 GB", | 20260506 20:05:50.318 | INFO | "malloc" : " 14 GB", | 20260506 20:05:50.318 | INFO | "pinned" : " 29 GB" | 20260506 20:05:50.319 | INFO | } | 20260506 20:05:50.319 | INFO | Perf report for the last 5.13885 seconds: | 20260506 20:05:50.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:05:50.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:50.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:50.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:50.319 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 20:05:50.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:50.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:50.319 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:05:50.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:50.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:50.319 | INFO | Uploaded 36GiB to GPU in 2.0002s (17.9982GiB/s) | 20260506 20:05:50.319 | INFO | Downloaded 34GiB to GPU in 1.91152s (17.7869GiB/s) | 20260506 20:05:55.457 | INFO | Analyzing chunk frames = [4090368,4090880) | 20260506 20:05:55.457 | INFO | MemInfo: { | 20260506 20:05:55.457 | INFO | "gpu" : " 17 GB", | 20260506 20:05:55.457 | INFO | "huge" : "", | 20260506 20:05:55.457 | INFO | "huge_shared" : " 27 GB", | 20260506 20:05:55.457 | INFO | "malloc" : " 14 GB", | 20260506 20:05:55.457 | INFO | "pinned" : " 29 GB" | 20260506 20:05:55.457 | INFO | } | 20260506 20:05:55.457 | INFO | Perf report for the last 5.1388 seconds: | 20260506 20:05:55.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:05:55.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:55.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:55.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:55.457 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:05:55.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:55.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:55.458 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 20:05:55.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:55.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:05:55.458 | INFO | Uploaded 36GiB to GPU in 1.99302s (18.0631GiB/s) | 20260506 20:05:55.458 | INFO | Downloaded 34GiB to GPU in 1.96236s (17.3261GiB/s) | 20260506 20:06:00.596 | INFO | Analyzing chunk frames = [4090880,4091392) | 20260506 20:06:00.596 | INFO | MemInfo: { | 20260506 20:06:00.596 | INFO | "gpu" : " 17 GB", | 20260506 20:06:00.596 | INFO | "huge" : "", | 20260506 20:06:00.596 | INFO | "huge_shared" : " 27 GB", | 20260506 20:06:00.596 | INFO | "malloc" : " 14 GB", | 20260506 20:06:00.596 | INFO | "pinned" : " 29 GB" | 20260506 20:06:00.596 | INFO | } | 20260506 20:06:00.596 | INFO | Perf report for the last 5.13852 seconds: | 20260506 20:06:00.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:06:00.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:00.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:00.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:00.596 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 20:06:00.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:00.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:00.596 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 20:06:00.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:00.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:00.596 | INFO | Uploaded 36GiB to GPU in 1.98088s (18.1738GiB/s) | 20260506 20:06:00.596 | INFO | Downloaded 34GiB to GPU in 2.00906s (16.9233GiB/s) | 20260506 20:06:05.745 | INFO | Analyzing chunk frames = [4091392,4091904) | 20260506 20:06:05.745 | INFO | MemInfo: { | 20260506 20:06:05.745 | INFO | "gpu" : " 17 GB", | 20260506 20:06:05.745 | INFO | "huge" : "", | 20260506 20:06:05.745 | INFO | "huge_shared" : " 27 GB", | 20260506 20:06:05.745 | INFO | "malloc" : " 14 GB", | 20260506 20:06:05.745 | INFO | "pinned" : " 29 GB" | 20260506 20:06:05.745 | INFO | } | 20260506 20:06:05.745 | INFO | Perf report for the last 5.14946 seconds: | 20260506 20:06:05.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:06:05.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:05.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:05.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:05.745 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 20:06:05.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:05.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:05.746 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260506 20:06:05.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:05.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:05.746 | INFO | Uploaded 36GiB to GPU in 2.00651s (17.9416GiB/s) | 20260506 20:06:05.746 | INFO | Downloaded 34GiB to GPU in 1.9662s (17.2923GiB/s) | 20260506 20:06:05.844 | INFO | Starting a round of Baz IO | 20260506 20:06:08.989 | INFO | Populating RTMetrics report | 20260506 20:06:08.989 | INFO | RTMetrics report populated | 20260506 20:06:08.989 | INFO | Opening tmp RTMetrics json file | 20260506 20:06:09.069 | INFO | Serializing RTMetrics to json | 20260506 20:06:09.109 | INFO | Serialization done, swapping tmp file now... | 20260506 20:06:09.111 | INFO | RTMetrics done | 20260506 20:06:09.646 | INFO | Finished a round of Baz IO: Wrote 1.991 GiB in 3.802 seconds | 20260506 20:06:09.646 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.37% / 0.00% / 65.63% | 20260506 20:06:09.646 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.80566% / 7.94729e-06% / 0.000103315% | 20260506 20:06:09.646 | INFO | Baz IO overall: | 20260506 20:06:09.646 | INFO | BytesWritten : 2161366207565 | 20260506 20:06:09.646 | INFO | NumEvents : 1044427147514 | 20260506 20:06:10.882 | INFO | Analyzing chunk frames = [4091904,4092416) | 20260506 20:06:10.882 | INFO | MemInfo: { | 20260506 20:06:10.882 | INFO | "gpu" : " 17 GB", | 20260506 20:06:10.882 | INFO | "huge" : "", | 20260506 20:06:10.882 | INFO | "huge_shared" : " 27 GB", | 20260506 20:06:10.882 | INFO | "malloc" : " 14 GB", | 20260506 20:06:10.882 | INFO | "pinned" : " 29 GB" | 20260506 20:06:10.882 | INFO | } | 20260506 20:06:10.882 | INFO | Perf report for the last 5.1369 seconds: | 20260506 20:06:10.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:06:10.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:10.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:10.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:10.882 | INFO | ANALYSIS : 62.6%, 2.9, 95.8ms, 0 | 20260506 20:06:10.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:10.882 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260506 20:06:10.882 | INFO | PRE_HQ : 60.1%, 1.8, 56.8ms, 0 | 20260506 20:06:10.882 | INFO | BAZWRITER : 73.7%, 4.8, 181.2ms, 0 | 20260506 20:06:10.882 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 20:06:10.883 | INFO | Uploaded 36GiB to GPU in 2.07235s (17.3716GiB/s) | 20260506 20:06:10.883 | INFO | Downloaded 38GiB to GPU in 2.1989s (17.2814GiB/s) | 20260506 20:06:16.016 | INFO | Analyzing chunk frames = [4092416,4092928) | 20260506 20:06:16.016 | INFO | MemInfo: { | 20260506 20:06:16.016 | INFO | "gpu" : " 17 GB", | 20260506 20:06:16.016 | INFO | "huge" : "", | 20260506 20:06:16.016 | INFO | "huge_shared" : " 27 GB", | 20260506 20:06:16.016 | INFO | "malloc" : " 14 GB", | 20260506 20:06:16.016 | INFO | "pinned" : " 29 GB" | 20260506 20:06:16.016 | INFO | } | 20260506 20:06:16.016 | INFO | Perf report for the last 5.13382 seconds: | 20260506 20:06:16.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:06:16.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:16.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:16.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:16.016 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:06:16.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:16.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:16.016 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:06:16.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:16.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:16.016 | INFO | Uploaded 36GiB to GPU in 1.99411s (18.0532GiB/s) | 20260506 20:06:16.016 | INFO | Downloaded 34GiB to GPU in 1.92469s (17.6652GiB/s) | 20260506 20:06:21.155 | INFO | Analyzing chunk frames = [4092928,4093440) | 20260506 20:06:21.155 | INFO | MemInfo: { | 20260506 20:06:21.155 | INFO | "gpu" : " 17 GB", | 20260506 20:06:21.155 | INFO | "huge" : "", | 20260506 20:06:21.155 | INFO | "huge_shared" : " 27 GB", | 20260506 20:06:21.155 | INFO | "malloc" : " 14 GB", | 20260506 20:06:21.155 | INFO | "pinned" : " 29 GB" | 20260506 20:06:21.155 | INFO | } | 20260506 20:06:21.155 | INFO | Perf report for the last 5.13871 seconds: | 20260506 20:06:21.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:06:21.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:21.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:21.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:21.155 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 20:06:21.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:21.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:21.155 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:06:21.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:21.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:21.155 | INFO | Uploaded 36GiB to GPU in 1.98422s (18.1432GiB/s) | 20260506 20:06:21.155 | INFO | Downloaded 34GiB to GPU in 1.91818s (17.7252GiB/s) | 20260506 20:06:26.292 | INFO | Analyzing chunk frames = [4093440,4093952) | 20260506 20:06:26.292 | INFO | MemInfo: { | 20260506 20:06:26.292 | INFO | "gpu" : " 17 GB", | 20260506 20:06:26.292 | INFO | "huge" : "", | 20260506 20:06:26.292 | INFO | "huge_shared" : " 27 GB", | 20260506 20:06:26.292 | INFO | "malloc" : " 14 GB", | 20260506 20:06:26.292 | INFO | "pinned" : " 29 GB" | 20260506 20:06:26.292 | INFO | } | 20260506 20:06:26.292 | INFO | Perf report for the last 5.13724 seconds: | 20260506 20:06:26.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:06:26.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:26.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:26.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:26.292 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 20:06:26.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:26.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:26.292 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:06:26.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:26.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:26.292 | INFO | Uploaded 36GiB to GPU in 1.94956s (18.4657GiB/s) | 20260506 20:06:26.292 | INFO | Downloaded 34GiB to GPU in 1.87585s (18.1251GiB/s) | 20260506 20:06:31.432 | INFO | Analyzing chunk frames = [4093952,4094464) | 20260506 20:06:31.432 | INFO | MemInfo: { | 20260506 20:06:31.432 | INFO | "gpu" : " 17 GB", | 20260506 20:06:31.432 | INFO | "huge" : "", | 20260506 20:06:31.432 | INFO | "huge_shared" : " 27 GB", | 20260506 20:06:31.432 | INFO | "malloc" : " 14 GB", | 20260506 20:06:31.432 | INFO | "pinned" : " 29 GB" | 20260506 20:06:31.432 | INFO | } | 20260506 20:06:31.432 | INFO | Perf report for the last 5.13959 seconds: | 20260506 20:06:31.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:06:31.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:31.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:31.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:31.432 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 20:06:31.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:31.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:31.432 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:06:31.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:31.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:31.432 | INFO | Uploaded 36GiB to GPU in 2.00613s (17.945GiB/s) | 20260506 20:06:31.432 | INFO | Downloaded 34GiB to GPU in 1.91155s (17.7866GiB/s) | 20260506 20:06:36.576 | INFO | Analyzing chunk frames = [4094464,4094976) | 20260506 20:06:36.576 | INFO | MemInfo: { | 20260506 20:06:36.576 | INFO | "gpu" : " 17 GB", | 20260506 20:06:36.576 | INFO | "huge" : "", | 20260506 20:06:36.576 | INFO | "huge_shared" : " 27 GB", | 20260506 20:06:36.576 | INFO | "malloc" : " 14 GB", | 20260506 20:06:36.576 | INFO | "pinned" : " 29 GB" | 20260506 20:06:36.576 | INFO | } | 20260506 20:06:36.576 | INFO | Perf report for the last 5.14431 seconds: | 20260506 20:06:36.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:06:36.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:36.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:36.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:36.576 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 20:06:36.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:36.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:36.576 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 20:06:36.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:36.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:36.576 | INFO | Uploaded 36GiB to GPU in 2.0047s (17.9578GiB/s) | 20260506 20:06:36.576 | INFO | Downloaded 34GiB to GPU in 1.9431s (17.4978GiB/s) | 20260506 20:06:41.709 | INFO | Analyzing chunk frames = [4094976,4095488) | 20260506 20:06:41.709 | INFO | MemInfo: { | 20260506 20:06:41.709 | INFO | "gpu" : " 17 GB", | 20260506 20:06:41.709 | INFO | "huge" : "", | 20260506 20:06:41.709 | INFO | "huge_shared" : " 27 GB", | 20260506 20:06:41.709 | INFO | "malloc" : " 14 GB", | 20260506 20:06:41.709 | INFO | "pinned" : " 29 GB" | 20260506 20:06:41.709 | INFO | } | 20260506 20:06:41.709 | INFO | Perf report for the last 5.13319 seconds: | 20260506 20:06:41.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:06:41.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:41.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:41.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:41.709 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 20:06:41.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:41.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:41.709 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:06:41.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:41.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:41.710 | INFO | Uploaded 36GiB to GPU in 1.95038s (18.458GiB/s) | 20260506 20:06:41.710 | INFO | Downloaded 34GiB to GPU in 2.00052s (16.9956GiB/s) | 20260506 20:06:46.850 | INFO | Analyzing chunk frames = [4095488,4096000) | 20260506 20:06:46.850 | INFO | MemInfo: { | 20260506 20:06:46.850 | INFO | "gpu" : " 17 GB", | 20260506 20:06:46.850 | INFO | "huge" : "", | 20260506 20:06:46.850 | INFO | "huge_shared" : " 27 GB", | 20260506 20:06:46.850 | INFO | "malloc" : " 14 GB", | 20260506 20:06:46.850 | INFO | "pinned" : " 29 GB" | 20260506 20:06:46.850 | INFO | } | 20260506 20:06:46.850 | INFO | Perf report for the last 5.14108 seconds: | 20260506 20:06:46.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:06:46.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:46.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:46.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:46.850 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 20:06:46.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:46.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:46.850 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 20:06:46.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:46.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:46.851 | INFO | Uploaded 36GiB to GPU in 1.99172s (18.0749GiB/s) | 20260506 20:06:46.851 | INFO | Downloaded 34GiB to GPU in 1.97998s (17.1719GiB/s) | 20260506 20:06:46.851 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:06:46.851 | INFO | Total mem usage: 18 GiB | 20260506 20:06:46.851 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:06:46.851 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -12.1%) | 20260506 20:06:46.851 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:06:46.851 | INFO | High Water Breakdown: | 20260506 20:06:46.851 | INFO | 61.9% : 9.95 GiB ( 87.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:06:46.851 | INFO | 37.9% : 6.09 GiB ( 91% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:06:46.851 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:06:46.851 | INFO | ------------------------------------------ | 20260506 20:06:46.851 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:06:46.851 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:06:46.851 | INFO | Total mem usage: 26.5 GiB | 20260506 20:06:46.851 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:06:46.851 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:06:46.851 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:06:46.851 | INFO | High Water Breakdown: | 20260506 20:06:46.851 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:06:46.851 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:06:46.851 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:06:46.851 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:06:46.851 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:06:46.851 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:06:46.851 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:06:46.851 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:06:46.851 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:06:46.851 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:06:46.851 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:06:46.851 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:06:46.851 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:06:46.851 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:06:46.851 | INFO | ------------------------------------------ | 20260506 20:06:46.851 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:06:46.851 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:06:46.851 | INFO | Total mem usage: 25.6 GiB | 20260506 20:06:46.851 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:06:46.851 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:06:46.851 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:06:46.851 | INFO | High Water Breakdown: | 20260506 20:06:46.851 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:06:46.851 | INFO | ------------------------------------------ | 20260506 20:06:46.851 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:06:46.851 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:06:46.851 | INFO | Total mem usage: 15.7 GiB | 20260506 20:06:46.851 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:06:46.851 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:06:46.851 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:06:46.851 | INFO | High Water Breakdown: | 20260506 20:06:46.851 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:06:46.851 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:06:46.851 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:06:46.851 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:06:46.851 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:06:46.851 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:06:46.851 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:06:46.851 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:06:46.851 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:06:46.851 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:06:46.851 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:06:46.851 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:06:46.851 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:06:46.851 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:06:46.851 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:06:46.851 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:06:46.851 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:06:46.851 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:06:46.851 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:06:46.851 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:06:46.852 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:06:46.852 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:06:46.852 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:06:46.852 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:06:46.852 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:06:46.852 | INFO | ------------------------------------------ | 20260506 20:06:46.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:06:46.852 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:06:46.852 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:06:46.852 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:06:46.852 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.64e+01, 1.36e+01 / 2.66e+00 | 20260506 20:06:46.852 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.02e-01, 9.99e-03 / 6.18e-03 | 20260506 20:06:46.852 | INFO | PerfReport: 11.64%, Baseline -- 3.63e+00 / 4.42e+00, 5.61e-01 / 3.32e-01 | 20260506 20:06:46.852 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.48e+00, 1.86e-01 / 1.24e-01 | 20260506 20:06:46.852 | INFO | PerfReport: 16.04%, DME -- 4.01e+01 / 4.51e+01, 4.23e+00 / 3.26e+00 | 20260506 20:06:46.852 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.43e+00 / 2.04e+00 | 20260506 20:06:46.852 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.23e+00 / 2.76e+00, 3.74e-01 / 2.36e-01 | 20260506 20:06:46.852 | INFO | PerfReport: 3.33%, Metrics -- 1.04e+00 / 1.40e+00, 2.78e-01 / 1.75e-01 | 20260506 20:06:46.852 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.57e-01 / 4.09e-01, 5.60e-02 / 2.41e-02 | 20260506 20:06:46.852 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:06:46.852 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:06:46.852 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:06:46.852 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 20:06:46.852 | INFO | PerfReport: 21.41%, Upload -- 2.07e+01 / 0.00e+00, 6.91e+00 / 0.00e+00 | 20260506 20:06:46.852 | INFO | PerfReport: 21.20%, Download -- 2.05e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 20:06:46.852 | INFO | PerfReport: 32.23%, Compute -- 3.12e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 20:06:46.852 | INFO | PerfReport: 1.26%, LockUpload -- 1.22e+00 / 0.00e+00, 4.87e+00 / 0.00e+00 | 20260506 20:06:46.852 | INFO | PerfReport: 1.21%, LockDownload -- 1.17e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260506 20:06:46.852 | INFO | PerfReport: 22.68%, LockCompute -- 2.20e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 20:06:46.852 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260506 20:06:46.954 | INFO | Starting a round of Baz IO | 20260506 20:06:50.088 | INFO | Populating RTMetrics report | 20260506 20:06:50.089 | INFO | RTMetrics report populated | 20260506 20:06:50.089 | INFO | Opening tmp RTMetrics json file | 20260506 20:06:50.235 | INFO | Serializing RTMetrics to json | 20260506 20:06:50.240 | INFO | Serialization done, swapping tmp file now... | 20260506 20:06:50.248 | INFO | RTMetrics done | 20260506 20:06:50.416 | INFO | Finished a round of Baz IO: Wrote 1.988 GiB in 3.461 seconds | 20260506 20:06:50.416 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.36% / 0.00% / 65.63% | 20260506 20:06:50.416 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.800455% / 0% / 5.5631e-05% | 20260506 20:06:50.416 | INFO | Baz IO overall: | 20260506 20:06:50.416 | INFO | BytesWritten : 2163500619152 | 20260506 20:06:50.416 | INFO | NumEvents : 1045451812731 | 20260506 20:06:51.993 | INFO | Analyzing chunk frames = [4096000,4096512) | 20260506 20:06:51.993 | INFO | MemInfo: { | 20260506 20:06:51.993 | INFO | "gpu" : " 17 GB", | 20260506 20:06:51.993 | INFO | "huge" : "", | 20260506 20:06:51.993 | INFO | "huge_shared" : " 27 GB", | 20260506 20:06:51.993 | INFO | "malloc" : " 14 GB", | 20260506 20:06:51.993 | INFO | "pinned" : " 29 GB" | 20260506 20:06:51.993 | INFO | } | 20260506 20:06:51.993 | INFO | Perf report for the last 5.14258 seconds: | 20260506 20:06:51.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:06:51.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:51.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:51.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:51.993 | INFO | ANALYSIS : 62.4%, 2.9, 95.3ms, 0 | 20260506 20:06:51.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:51.993 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 20:06:51.993 | INFO | PRE_HQ : 59.9%, 1.8, 57.0ms, 0 | 20260506 20:06:51.993 | INFO | BAZWRITER : 67.1%, 4.6, 160.2ms, 0 | 20260506 20:06:51.993 | INFO | BAZ_SAMPLING : 3.5%, 1.0, 1.8ms, 0 | 20260506 20:06:51.993 | INFO | Uploaded 36GiB to GPU in 2.00731s (17.9344GiB/s) | 20260506 20:06:51.993 | INFO | Downloaded 38GiB to GPU in 2.18605s (17.383GiB/s) | 20260506 20:06:57.132 | INFO | Analyzing chunk frames = [4096512,4097024) | 20260506 20:06:57.132 | INFO | MemInfo: { | 20260506 20:06:57.132 | INFO | "gpu" : " 17 GB", | 20260506 20:06:57.132 | INFO | "huge" : "", | 20260506 20:06:57.132 | INFO | "huge_shared" : " 27 GB", | 20260506 20:06:57.132 | INFO | "malloc" : " 14 GB", | 20260506 20:06:57.132 | INFO | "pinned" : " 29 GB" | 20260506 20:06:57.133 | INFO | } | 20260506 20:06:57.133 | INFO | Perf report for the last 5.13961 seconds: | 20260506 20:06:57.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:06:57.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:57.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:57.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:57.133 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:06:57.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:57.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:57.133 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:06:57.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:57.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:06:57.133 | INFO | Uploaded 36GiB to GPU in 1.99626s (18.0337GiB/s) | 20260506 20:06:57.133 | INFO | Downloaded 34GiB to GPU in 1.92251s (17.6852GiB/s) | 20260506 20:07:02.269 | INFO | Analyzing chunk frames = [4097024,4097536) | 20260506 20:07:02.269 | INFO | MemInfo: { | 20260506 20:07:02.269 | INFO | "gpu" : " 17 GB", | 20260506 20:07:02.269 | INFO | "huge" : "", | 20260506 20:07:02.269 | INFO | "huge_shared" : " 27 GB", | 20260506 20:07:02.269 | INFO | "malloc" : " 14 GB", | 20260506 20:07:02.269 | INFO | "pinned" : " 29 GB" | 20260506 20:07:02.269 | INFO | } | 20260506 20:07:02.269 | INFO | Perf report for the last 5.13631 seconds: | 20260506 20:07:02.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:07:02.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:02.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:02.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:02.269 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 20:07:02.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:02.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:02.269 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:07:02.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:02.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:02.269 | INFO | Uploaded 36GiB to GPU in 1.9609s (18.3589GiB/s) | 20260506 20:07:02.269 | INFO | Downloaded 34GiB to GPU in 1.92535s (17.6591GiB/s) | 20260506 20:07:07.407 | INFO | Analyzing chunk frames = [4097536,4098048) | 20260506 20:07:07.407 | INFO | MemInfo: { | 20260506 20:07:07.407 | INFO | "gpu" : " 17 GB", | 20260506 20:07:07.407 | INFO | "huge" : "", | 20260506 20:07:07.407 | INFO | "huge_shared" : " 27 GB", | 20260506 20:07:07.407 | INFO | "malloc" : " 14 GB", | 20260506 20:07:07.407 | INFO | "pinned" : " 29 GB" | 20260506 20:07:07.407 | INFO | } | 20260506 20:07:07.407 | INFO | Perf report for the last 5.13852 seconds: | 20260506 20:07:07.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:07:07.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:07.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:07.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:07.407 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 20:07:07.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:07.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:07.408 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:07:07.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:07.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:07.408 | INFO | Uploaded 36GiB to GPU in 1.96473s (18.3231GiB/s) | 20260506 20:07:07.408 | INFO | Downloaded 34GiB to GPU in 1.8746s (18.1372GiB/s) | 20260506 20:07:12.546 | INFO | Analyzing chunk frames = [4098048,4098560) | 20260506 20:07:12.546 | INFO | MemInfo: { | 20260506 20:07:12.546 | INFO | "gpu" : " 17 GB", | 20260506 20:07:12.546 | INFO | "huge" : "", | 20260506 20:07:12.546 | INFO | "huge_shared" : " 27 GB", | 20260506 20:07:12.546 | INFO | "malloc" : " 14 GB", | 20260506 20:07:12.546 | INFO | "pinned" : " 29 GB" | 20260506 20:07:12.546 | INFO | } | 20260506 20:07:12.546 | INFO | Perf report for the last 5.13862 seconds: | 20260506 20:07:12.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:07:12.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:12.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:12.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:12.546 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 20:07:12.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:12.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:12.546 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:07:12.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:12.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:12.546 | INFO | Uploaded 36GiB to GPU in 2.01077s (17.9036GiB/s) | 20260506 20:07:12.546 | INFO | Downloaded 34GiB to GPU in 1.92102s (17.699GiB/s) | 20260506 20:07:17.685 | INFO | Analyzing chunk frames = [4098560,4099072) | 20260506 20:07:17.685 | INFO | MemInfo: { | 20260506 20:07:17.685 | INFO | "gpu" : " 17 GB", | 20260506 20:07:17.685 | INFO | "huge" : "", | 20260506 20:07:17.685 | INFO | "huge_shared" : " 27 GB", | 20260506 20:07:17.685 | INFO | "malloc" : " 14 GB", | 20260506 20:07:17.685 | INFO | "pinned" : " 29 GB" | 20260506 20:07:17.685 | INFO | } | 20260506 20:07:17.685 | INFO | Perf report for the last 5.13936 seconds: | 20260506 20:07:17.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:07:17.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:17.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:17.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:17.685 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 20:07:17.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:17.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:17.686 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 20:07:17.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:17.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:17.686 | INFO | Uploaded 36GiB to GPU in 2.00766s (17.9313GiB/s) | 20260506 20:07:17.686 | INFO | Downloaded 34GiB to GPU in 1.97082s (17.2517GiB/s) | 20260506 20:07:22.829 | INFO | Analyzing chunk frames = [4099072,4099584) | 20260506 20:07:22.829 | INFO | MemInfo: { | 20260506 20:07:22.829 | INFO | "gpu" : " 17 GB", | 20260506 20:07:22.829 | INFO | "huge" : "", | 20260506 20:07:22.829 | INFO | "huge_shared" : " 27 GB", | 20260506 20:07:22.829 | INFO | "malloc" : " 14 GB", | 20260506 20:07:22.829 | INFO | "pinned" : " 29 GB" | 20260506 20:07:22.829 | INFO | } | 20260506 20:07:22.830 | INFO | Perf report for the last 5.14413 seconds: | 20260506 20:07:22.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:07:22.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:22.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:22.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:22.830 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 20:07:22.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:22.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:22.830 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260506 20:07:22.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:22.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:22.830 | INFO | Uploaded 36GiB to GPU in 1.95214s (18.4413GiB/s) | 20260506 20:07:22.830 | INFO | Downloaded 34GiB to GPU in 1.99575s (17.0362GiB/s) | 20260506 20:07:27.967 | INFO | Analyzing chunk frames = [4099584,4100096) | 20260506 20:07:27.967 | INFO | MemInfo: { | 20260506 20:07:27.967 | INFO | "gpu" : " 17 GB", | 20260506 20:07:27.967 | INFO | "huge" : "", | 20260506 20:07:27.967 | INFO | "huge_shared" : " 27 GB", | 20260506 20:07:27.967 | INFO | "malloc" : " 14 GB", | 20260506 20:07:27.967 | INFO | "pinned" : " 29 GB" | 20260506 20:07:27.967 | INFO | } | 20260506 20:07:27.967 | INFO | Perf report for the last 5.13729 seconds: | 20260506 20:07:27.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:07:27.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:27.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:27.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:27.967 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 20:07:27.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:27.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:27.967 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 20:07:27.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:27.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:27.967 | INFO | Uploaded 36GiB to GPU in 2.03527s (17.6881GiB/s) | 20260506 20:07:27.967 | INFO | Downloaded 34GiB to GPU in 1.99345s (17.0559GiB/s) | 20260506 20:07:28.081 | INFO | Starting a round of Baz IO | 20260506 20:07:31.213 | INFO | Populating RTMetrics report | 20260506 20:07:31.214 | INFO | RTMetrics report populated | 20260506 20:07:31.214 | INFO | Opening tmp RTMetrics json file | 20260506 20:07:31.224 | INFO | Serializing RTMetrics to json | 20260506 20:07:31.248 | INFO | Serialization done, swapping tmp file now... | 20260506 20:07:31.263 | INFO | RTMetrics done | 20260506 20:07:31.829 | INFO | Finished a round of Baz IO: Wrote 1.988 GiB in 3.748 seconds | 20260506 20:07:31.829 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.36% / 0.00% / 65.64% | 20260506 20:07:31.829 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.79941% / 1.19209e-05% / 6.75519e-05% | 20260506 20:07:31.829 | INFO | Baz IO overall: | 20260506 20:07:31.829 | INFO | BytesWritten : 2165635569298 | 20260506 20:07:31.829 | INFO | NumEvents : 1046476822160 | 20260506 20:07:33.107 | INFO | Analyzing chunk frames = [4100096,4100608) | 20260506 20:07:33.107 | INFO | MemInfo: { | 20260506 20:07:33.107 | INFO | "gpu" : " 17 GB", | 20260506 20:07:33.107 | INFO | "huge" : "", | 20260506 20:07:33.107 | INFO | "huge_shared" : " 27 GB", | 20260506 20:07:33.107 | INFO | "malloc" : " 14 GB", | 20260506 20:07:33.107 | INFO | "pinned" : " 29 GB" | 20260506 20:07:33.107 | INFO | } | 20260506 20:07:33.108 | INFO | Perf report for the last 5.14065 seconds: | 20260506 20:07:33.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:07:33.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:33.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:33.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:33.108 | INFO | ANALYSIS : 62.6%, 2.9, 95.7ms, 0 | 20260506 20:07:33.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:33.108 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 20:07:33.108 | INFO | PRE_HQ : 59.8%, 1.8, 57.3ms, 0 | 20260506 20:07:33.108 | INFO | BAZWRITER : 72.6%, 4.8, 179.6ms, 0 | 20260506 20:07:33.108 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 20:07:33.108 | INFO | Uploaded 36GiB to GPU in 2.05988s (17.4768GiB/s) | 20260506 20:07:33.108 | INFO | Downloaded 38GiB to GPU in 2.21618s (17.1466GiB/s) | 20260506 20:07:38.251 | INFO | Analyzing chunk frames = [4100608,4101120) | 20260506 20:07:38.251 | INFO | MemInfo: { | 20260506 20:07:38.251 | INFO | "gpu" : " 17 GB", | 20260506 20:07:38.251 | INFO | "huge" : "", | 20260506 20:07:38.251 | INFO | "huge_shared" : " 27 GB", | 20260506 20:07:38.251 | INFO | "malloc" : " 14 GB", | 20260506 20:07:38.251 | INFO | "pinned" : " 29 GB" | 20260506 20:07:38.251 | INFO | } | 20260506 20:07:38.251 | INFO | Perf report for the last 5.14327 seconds: | 20260506 20:07:38.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:07:38.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:38.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:38.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:38.251 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:07:38.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:38.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:38.251 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:07:38.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:38.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:38.251 | INFO | Uploaded 36GiB to GPU in 1.99781s (18.0198GiB/s) | 20260506 20:07:38.251 | INFO | Downloaded 34GiB to GPU in 1.92521s (17.6604GiB/s) | 20260506 20:07:43.384 | INFO | Analyzing chunk frames = [4101120,4101632) | 20260506 20:07:43.384 | INFO | MemInfo: { | 20260506 20:07:43.384 | INFO | "gpu" : " 17 GB", | 20260506 20:07:43.384 | INFO | "huge" : "", | 20260506 20:07:43.384 | INFO | "huge_shared" : " 27 GB", | 20260506 20:07:43.384 | INFO | "malloc" : " 14 GB", | 20260506 20:07:43.384 | INFO | "pinned" : " 29 GB" | 20260506 20:07:43.384 | INFO | } | 20260506 20:07:43.385 | INFO | Perf report for the last 5.13372 seconds: | 20260506 20:07:43.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:07:43.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:43.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:43.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:43.385 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 20:07:43.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:43.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:43.385 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:07:43.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:43.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:43.385 | INFO | Uploaded 36GiB to GPU in 1.97713s (18.2083GiB/s) | 20260506 20:07:43.385 | INFO | Downloaded 34GiB to GPU in 1.92987s (17.6178GiB/s) | 20260506 20:07:48.523 | INFO | Analyzing chunk frames = [4101632,4102144) | 20260506 20:07:48.523 | INFO | MemInfo: { | 20260506 20:07:48.523 | INFO | "gpu" : " 17 GB", | 20260506 20:07:48.523 | INFO | "huge" : "", | 20260506 20:07:48.523 | INFO | "huge_shared" : " 27 GB", | 20260506 20:07:48.523 | INFO | "malloc" : " 14 GB", | 20260506 20:07:48.523 | INFO | "pinned" : " 29 GB" | 20260506 20:07:48.523 | INFO | } | 20260506 20:07:48.523 | INFO | Perf report for the last 5.13864 seconds: | 20260506 20:07:48.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:07:48.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:48.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:48.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:48.523 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 20:07:48.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:48.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:48.523 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:07:48.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:48.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:48.523 | INFO | Uploaded 36GiB to GPU in 1.94168s (18.5407GiB/s) | 20260506 20:07:48.524 | INFO | Downloaded 34GiB to GPU in 1.87436s (18.1395GiB/s) | 20260506 20:07:53.658 | INFO | Analyzing chunk frames = [4102144,4102656) | 20260506 20:07:53.658 | INFO | MemInfo: { | 20260506 20:07:53.658 | INFO | "gpu" : " 17 GB", | 20260506 20:07:53.658 | INFO | "huge" : "", | 20260506 20:07:53.658 | INFO | "huge_shared" : " 27 GB", | 20260506 20:07:53.658 | INFO | "malloc" : " 14 GB", | 20260506 20:07:53.658 | INFO | "pinned" : " 29 GB" | 20260506 20:07:53.658 | INFO | } | 20260506 20:07:53.658 | INFO | Perf report for the last 5.13514 seconds: | 20260506 20:07:53.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:07:53.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:53.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:53.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:53.658 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 20:07:53.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:53.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:53.658 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:07:53.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:53.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:53.659 | INFO | Uploaded 36GiB to GPU in 2.00641s (17.9425GiB/s) | 20260506 20:07:53.659 | INFO | Downloaded 34GiB to GPU in 1.91448s (17.7594GiB/s) | 20260506 20:07:58.797 | INFO | Analyzing chunk frames = [4102656,4103168) | 20260506 20:07:58.797 | INFO | MemInfo: { | 20260506 20:07:58.797 | INFO | "gpu" : " 17 GB", | 20260506 20:07:58.797 | INFO | "huge" : "", | 20260506 20:07:58.798 | INFO | "huge_shared" : " 27 GB", | 20260506 20:07:58.798 | INFO | "malloc" : " 14 GB", | 20260506 20:07:58.798 | INFO | "pinned" : " 29 GB" | 20260506 20:07:58.798 | INFO | } | 20260506 20:07:58.798 | INFO | Perf report for the last 5.13929 seconds: | 20260506 20:07:58.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:07:58.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:58.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:58.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:58.798 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 20:07:58.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:58.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:58.798 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 20:07:58.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:58.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:07:58.798 | INFO | Uploaded 36GiB to GPU in 1.99029s (18.0878GiB/s) | 20260506 20:07:58.798 | INFO | Downloaded 34GiB to GPU in 1.94273s (17.5012GiB/s) | 20260506 20:08:03.942 | INFO | Analyzing chunk frames = [4103168,4103680) | 20260506 20:08:03.942 | INFO | MemInfo: { | 20260506 20:08:03.942 | INFO | "gpu" : " 17 GB", | 20260506 20:08:03.942 | INFO | "huge" : "", | 20260506 20:08:03.942 | INFO | "huge_shared" : " 27 GB", | 20260506 20:08:03.942 | INFO | "malloc" : " 14 GB", | 20260506 20:08:03.942 | INFO | "pinned" : " 29 GB" | 20260506 20:08:03.942 | INFO | } | 20260506 20:08:03.943 | INFO | Perf report for the last 5.14487 seconds: | 20260506 20:08:03.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:08:03.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:03.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:03.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:03.943 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:08:03.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:03.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:03.943 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 20:08:03.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:03.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:03.943 | INFO | Uploaded 36GiB to GPU in 1.98181s (18.1652GiB/s) | 20260506 20:08:03.943 | INFO | Downloaded 34GiB to GPU in 2.00644s (16.9455GiB/s) | 20260506 20:08:09.082 | INFO | Analyzing chunk frames = [4103680,4104192) | 20260506 20:08:09.082 | INFO | MemInfo: { | 20260506 20:08:09.082 | INFO | "gpu" : " 17 GB", | 20260506 20:08:09.082 | INFO | "huge" : "", | 20260506 20:08:09.082 | INFO | "huge_shared" : " 27 GB", | 20260506 20:08:09.082 | INFO | "malloc" : " 14 GB", | 20260506 20:08:09.082 | INFO | "pinned" : " 29 GB" | 20260506 20:08:09.082 | INFO | } | 20260506 20:08:09.082 | INFO | Perf report for the last 5.13965 seconds: | 20260506 20:08:09.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:08:09.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:09.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:09.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:09.082 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 20:08:09.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:09.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:09.082 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 20:08:09.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:09.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:09.083 | INFO | Uploaded 36GiB to GPU in 2.02791s (17.7522GiB/s) | 20260506 20:08:09.083 | INFO | Downloaded 34GiB to GPU in 1.98679s (17.113GiB/s) | 20260506 20:08:09.083 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:08:09.083 | INFO | Total mem usage: 18 GiB | 20260506 20:08:09.083 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:08:09.083 | INFO | Recent High Water Marks Sum: 16.6 GiB (Savings: -8.75%) | 20260506 20:08:09.083 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:08:09.083 | INFO | High Water Breakdown: | 20260506 20:08:09.083 | INFO | 62.4% : 10.3 GiB ( 90.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:08:09.083 | INFO | 37.5% : 6.21 GiB ( 92.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:08:09.083 | INFO | 0.12% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:08:09.083 | INFO | ------------------------------------------ | 20260506 20:08:09.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:08:09.083 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:08:09.083 | INFO | Total mem usage: 26.5 GiB | 20260506 20:08:09.083 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:08:09.083 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:08:09.083 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:08:09.083 | INFO | High Water Breakdown: | 20260506 20:08:09.083 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:08:09.083 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:08:09.083 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:08:09.083 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:08:09.083 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:08:09.083 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:08:09.083 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:08:09.083 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:08:09.083 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:08:09.083 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:08:09.083 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:08:09.083 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:08:09.083 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:08:09.083 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:08:09.083 | INFO | ------------------------------------------ | 20260506 20:08:09.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:08:09.083 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:08:09.083 | INFO | Total mem usage: 25.6 GiB | 20260506 20:08:09.083 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:08:09.083 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:08:09.083 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:08:09.083 | INFO | High Water Breakdown: | 20260506 20:08:09.083 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:08:09.083 | INFO | ------------------------------------------ | 20260506 20:08:09.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:08:09.083 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:08:09.083 | INFO | Total mem usage: 15.7 GiB | 20260506 20:08:09.083 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:08:09.083 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:08:09.083 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:08:09.083 | INFO | High Water Breakdown: | 20260506 20:08:09.083 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:08:09.083 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:08:09.083 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:08:09.083 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:08:09.083 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:08:09.083 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:08:09.083 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:08:09.083 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:08:09.083 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:08:09.083 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:08:09.083 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:08:09.083 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:08:09.083 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:08:09.083 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:08:09.083 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:08:09.083 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:08:09.083 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:08:09.083 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:08:09.083 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:08:09.083 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:08:09.083 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:08:09.083 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:08:09.083 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:08:09.083 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:08:09.083 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:08:09.083 | INFO | ------------------------------------------ | 20260506 20:08:09.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:08:09.084 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:08:09.084 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:08:09.084 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:08:09.084 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.56e+01, 1.35e+01 / 2.60e+00 | 20260506 20:08:09.084 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 9.46e-03 / 6.58e-03 | 20260506 20:08:09.084 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.34e+00, 5.42e-01 / 3.72e-01 | 20260506 20:08:09.084 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.43e+00, 1.79e-01 / 1.30e-01 | 20260506 20:08:09.084 | INFO | PerfReport: 15.98%, DME -- 3.99e+01 / 4.44e+01, 3.96e+00 / 2.84e+00 | 20260506 20:08:09.084 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.42e+00 / 1.83e+00 | 20260506 20:08:09.084 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.23e+00 / 2.75e+00, 3.69e-01 / 2.60e-01 | 20260506 20:08:09.084 | INFO | PerfReport: 3.33%, Metrics -- 1.04e+00 / 1.36e+00, 2.71e-01 / 1.75e-01 | 20260506 20:08:09.084 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.57e-01 / 3.98e-01, 5.58e-02 / 2.67e-02 | 20260506 20:08:09.084 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:08:09.084 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:08:09.084 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:08:09.084 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 20:08:09.084 | INFO | PerfReport: 21.46%, Upload -- 2.08e+01 / 0.00e+00, 6.74e+00 / 0.00e+00 | 20260506 20:08:09.084 | INFO | PerfReport: 21.24%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 20:08:09.084 | INFO | PerfReport: 32.23%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 20:08:09.084 | INFO | PerfReport: 1.23%, LockUpload -- 1.19e+00 / 0.00e+00, 5.18e+00 / 0.00e+00 | 20260506 20:08:09.084 | INFO | PerfReport: 1.24%, LockDownload -- 1.21e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260506 20:08:09.084 | INFO | PerfReport: 22.59%, LockCompute -- 2.19e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 20:08:09.084 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260506 20:08:09.198 | INFO | Starting a round of Baz IO | 20260506 20:08:12.329 | INFO | Populating RTMetrics report | 20260506 20:08:12.329 | INFO | RTMetrics report populated | 20260506 20:08:12.329 | INFO | Opening tmp RTMetrics json file | 20260506 20:08:12.332 | INFO | Serializing RTMetrics to json | 20260506 20:08:12.376 | INFO | Serialization done, swapping tmp file now... | 20260506 20:08:12.432 | INFO | RTMetrics done | 20260506 20:08:12.507 | INFO | Finished a round of Baz IO: Wrote 1.989 GiB in 3.309 seconds | 20260506 20:08:12.507 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.35% / 0.00% / 65.64% | 20260506 20:08:12.507 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.799898% / 7.94729e-06% / 5.5631e-05% | 20260506 20:08:12.507 | INFO | Baz IO overall: | 20260506 20:08:12.507 | INFO | BytesWritten : 2167770813425 | 20260506 20:08:12.507 | INFO | NumEvents : 1047501970175 | 20260506 20:08:14.226 | INFO | Analyzing chunk frames = [4104192,4104704) | 20260506 20:08:14.226 | INFO | MemInfo: { | 20260506 20:08:14.226 | INFO | "gpu" : " 17 GB", | 20260506 20:08:14.226 | INFO | "huge" : "", | 20260506 20:08:14.226 | INFO | "huge_shared" : " 27 GB", | 20260506 20:08:14.226 | INFO | "malloc" : " 14 GB", | 20260506 20:08:14.226 | INFO | "pinned" : " 29 GB" | 20260506 20:08:14.226 | INFO | } | 20260506 20:08:14.226 | INFO | Perf report for the last 5.14393 seconds: | 20260506 20:08:14.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:08:14.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:14.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:14.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:14.226 | INFO | ANALYSIS : 62.6%, 2.9, 95.6ms, 0 | 20260506 20:08:14.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:14.226 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260506 20:08:14.226 | INFO | PRE_HQ : 59.9%, 1.8, 57.4ms, 0 | 20260506 20:08:14.226 | INFO | BAZWRITER : 63.9%, 4.5, 147.2ms, 0 | 20260506 20:08:14.226 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.3ms, 0 | 20260506 20:08:14.226 | INFO | Uploaded 36GiB to GPU in 2.02267s (17.7983GiB/s) | 20260506 20:08:14.227 | INFO | Downloaded 38GiB to GPU in 2.20674s (17.22GiB/s) | 20260506 20:08:19.361 | INFO | Analyzing chunk frames = [4104704,4105216) | 20260506 20:08:19.361 | INFO | MemInfo: { | 20260506 20:08:19.361 | INFO | "gpu" : " 17 GB", | 20260506 20:08:19.362 | INFO | "huge" : "", | 20260506 20:08:19.362 | INFO | "huge_shared" : " 27 GB", | 20260506 20:08:19.362 | INFO | "malloc" : " 14 GB", | 20260506 20:08:19.362 | INFO | "pinned" : " 29 GB" | 20260506 20:08:19.362 | INFO | } | 20260506 20:08:19.362 | INFO | Perf report for the last 5.13555 seconds: | 20260506 20:08:19.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:08:19.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:19.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:19.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:19.362 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 20:08:19.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:19.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:19.362 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:08:19.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:19.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:19.362 | INFO | Uploaded 36GiB to GPU in 2.00567s (17.9492GiB/s) | 20260506 20:08:19.362 | INFO | Downloaded 34GiB to GPU in 1.9251s (17.6614GiB/s) | 20260506 20:08:24.502 | INFO | Analyzing chunk frames = [4105216,4105728) | 20260506 20:08:24.502 | INFO | MemInfo: { | 20260506 20:08:24.502 | INFO | "gpu" : " 17 GB", | 20260506 20:08:24.502 | INFO | "huge" : "", | 20260506 20:08:24.502 | INFO | "huge_shared" : " 27 GB", | 20260506 20:08:24.502 | INFO | "malloc" : " 14 GB", | 20260506 20:08:24.502 | INFO | "pinned" : " 29 GB" | 20260506 20:08:24.502 | INFO | } | 20260506 20:08:24.502 | INFO | Perf report for the last 5.14028 seconds: | 20260506 20:08:24.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:08:24.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:24.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:24.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:24.502 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 20:08:24.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:24.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:24.502 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:08:24.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:24.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:24.502 | INFO | Uploaded 36GiB to GPU in 1.95764s (18.3895GiB/s) | 20260506 20:08:24.502 | INFO | Downloaded 34GiB to GPU in 1.92227s (17.6875GiB/s) | 20260506 20:08:29.634 | INFO | Analyzing chunk frames = [4105728,4106240) | 20260506 20:08:29.634 | INFO | MemInfo: { | 20260506 20:08:29.634 | INFO | "gpu" : " 17 GB", | 20260506 20:08:29.634 | INFO | "huge" : "", | 20260506 20:08:29.634 | INFO | "huge_shared" : " 27 GB", | 20260506 20:08:29.634 | INFO | "malloc" : " 14 GB", | 20260506 20:08:29.634 | INFO | "pinned" : " 29 GB" | 20260506 20:08:29.634 | INFO | } | 20260506 20:08:29.634 | INFO | Perf report for the last 5.1318 seconds: | 20260506 20:08:29.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:08:29.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:29.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:29.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:29.634 | INFO | ANALYSIS : 59.8%, 3.0, 92.5ms, 0 | 20260506 20:08:29.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:29.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:29.634 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:08:29.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:29.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:29.634 | INFO | Uploaded 36GiB to GPU in 1.91283s (18.8203GiB/s) | 20260506 20:08:29.634 | INFO | Downloaded 34GiB to GPU in 1.87265s (18.1561GiB/s) | 20260506 20:08:34.771 | INFO | Analyzing chunk frames = [4106240,4106752) | 20260506 20:08:34.771 | INFO | MemInfo: { | 20260506 20:08:34.771 | INFO | "gpu" : " 17 GB", | 20260506 20:08:34.771 | INFO | "huge" : "", | 20260506 20:08:34.771 | INFO | "huge_shared" : " 27 GB", | 20260506 20:08:34.771 | INFO | "malloc" : " 14 GB", | 20260506 20:08:34.771 | INFO | "pinned" : " 29 GB" | 20260506 20:08:34.772 | INFO | } | 20260506 20:08:34.772 | INFO | Perf report for the last 5.13779 seconds: | 20260506 20:08:34.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:08:34.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:34.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:34.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:34.772 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 20:08:34.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:34.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:34.772 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:08:34.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:34.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:34.772 | INFO | Uploaded 36GiB to GPU in 1.96213s (18.3474GiB/s) | 20260506 20:08:34.772 | INFO | Downloaded 34GiB to GPU in 1.92475s (17.6646GiB/s) | 20260506 20:08:39.918 | INFO | Analyzing chunk frames = [4106752,4107264) | 20260506 20:08:39.918 | INFO | MemInfo: { | 20260506 20:08:39.918 | INFO | "gpu" : " 17 GB", | 20260506 20:08:39.918 | INFO | "huge" : "", | 20260506 20:08:39.918 | INFO | "huge_shared" : " 27 GB", | 20260506 20:08:39.918 | INFO | "malloc" : " 14 GB", | 20260506 20:08:39.918 | INFO | "pinned" : " 29 GB" | 20260506 20:08:39.918 | INFO | } | 20260506 20:08:39.919 | INFO | Perf report for the last 5.14692 seconds: | 20260506 20:08:39.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:08:39.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:39.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:39.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:39.919 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 20:08:39.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:39.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:39.919 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:08:39.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:39.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:39.919 | INFO | Uploaded 36GiB to GPU in 1.97753s (18.2045GiB/s) | 20260506 20:08:39.919 | INFO | Downloaded 34GiB to GPU in 1.92583s (17.6547GiB/s) | 20260506 20:08:45.062 | INFO | Analyzing chunk frames = [4107264,4107776) | 20260506 20:08:45.062 | INFO | MemInfo: { | 20260506 20:08:45.062 | INFO | "gpu" : " 17 GB", | 20260506 20:08:45.062 | INFO | "huge" : "", | 20260506 20:08:45.062 | INFO | "huge_shared" : " 27 GB", | 20260506 20:08:45.062 | INFO | "malloc" : " 14 GB", | 20260506 20:08:45.062 | INFO | "pinned" : " 29 GB" | 20260506 20:08:45.062 | INFO | } | 20260506 20:08:45.062 | INFO | Perf report for the last 5.14337 seconds: | 20260506 20:08:45.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:08:45.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:45.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:45.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:45.062 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 20:08:45.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:45.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:45.062 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:08:45.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:45.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:45.062 | INFO | Uploaded 36GiB to GPU in 1.97176s (18.2578GiB/s) | 20260506 20:08:45.062 | INFO | Downloaded 34GiB to GPU in 2.00877s (16.9258GiB/s) | 20260506 20:08:50.200 | INFO | Analyzing chunk frames = [4107776,4108288) | 20260506 20:08:50.200 | INFO | MemInfo: { | 20260506 20:08:50.200 | INFO | "gpu" : " 17 GB", | 20260506 20:08:50.201 | INFO | "huge" : "", | 20260506 20:08:50.201 | INFO | "huge_shared" : " 27 GB", | 20260506 20:08:50.201 | INFO | "malloc" : " 14 GB", | 20260506 20:08:50.201 | INFO | "pinned" : " 29 GB" | 20260506 20:08:50.201 | INFO | } | 20260506 20:08:50.201 | INFO | Perf report for the last 5.13873 seconds: | 20260506 20:08:50.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:08:50.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:50.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:50.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:50.201 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 20:08:50.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:50.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:50.201 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 20:08:50.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:50.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:50.201 | INFO | Uploaded 36GiB to GPU in 2.02231s (17.8014GiB/s) | 20260506 20:08:50.201 | INFO | Downloaded 34GiB to GPU in 1.97927s (17.178GiB/s) | 20260506 20:08:50.315 | INFO | Starting a round of Baz IO | 20260506 20:08:53.453 | INFO | Populating RTMetrics report | 20260506 20:08:53.453 | INFO | RTMetrics report populated | 20260506 20:08:53.453 | INFO | Opening tmp RTMetrics json file | 20260506 20:08:53.481 | INFO | Serializing RTMetrics to json | 20260506 20:08:53.508 | INFO | Serialization done, swapping tmp file now... | 20260506 20:08:53.509 | INFO | RTMetrics done | 20260506 20:08:53.768 | INFO | Finished a round of Baz IO: Wrote 1.989 GiB in 3.452 seconds | 20260506 20:08:53.768 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.35% / 0.00% / 65.65% | 20260506 20:08:53.768 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.798325% / 3.97364e-06% / 7.15256e-05% | 20260506 20:08:53.768 | INFO | Baz IO overall: | 20260506 20:08:53.768 | INFO | BytesWritten : 2169906101499 | 20260506 20:08:53.768 | INFO | NumEvents : 1048527286339 | 20260506 20:08:55.341 | INFO | Analyzing chunk frames = [4108288,4108800) | 20260506 20:08:55.341 | INFO | MemInfo: { | 20260506 20:08:55.341 | INFO | "gpu" : " 17 GB", | 20260506 20:08:55.341 | INFO | "huge" : "", | 20260506 20:08:55.341 | INFO | "huge_shared" : " 27 GB", | 20260506 20:08:55.341 | INFO | "malloc" : " 14 GB", | 20260506 20:08:55.341 | INFO | "pinned" : " 29 GB" | 20260506 20:08:55.341 | INFO | } | 20260506 20:08:55.341 | INFO | Perf report for the last 5.14016 seconds: | 20260506 20:08:55.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:08:55.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:55.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:55.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:55.341 | INFO | ANALYSIS : 62.7%, 2.9, 95.1ms, 0 | 20260506 20:08:55.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:08:55.341 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260506 20:08:55.341 | INFO | PRE_HQ : 60.6%, 1.9, 60.4ms, 0 | 20260506 20:08:55.341 | INFO | BAZWRITER : 66.8%, 4.8, 164.8ms, 0 | 20260506 20:08:55.341 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 20:08:55.341 | INFO | Uploaded 36GiB to GPU in 2.07069s (17.3855GiB/s) | 20260506 20:08:55.341 | INFO | Downloaded 38GiB to GPU in 2.20273s (17.2513GiB/s) | 20260506 20:09:00.473 | INFO | Analyzing chunk frames = [4108800,4109312) | 20260506 20:09:00.473 | INFO | MemInfo: { | 20260506 20:09:00.473 | INFO | "gpu" : " 17 GB", | 20260506 20:09:00.473 | INFO | "huge" : "", | 20260506 20:09:00.474 | INFO | "huge_shared" : " 27 GB", | 20260506 20:09:00.474 | INFO | "malloc" : " 14 GB", | 20260506 20:09:00.474 | INFO | "pinned" : " 29 GB" | 20260506 20:09:00.474 | INFO | } | 20260506 20:09:00.474 | INFO | Perf report for the last 5.13286 seconds: | 20260506 20:09:00.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:09:00.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:00.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:00.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:00.474 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 20:09:00.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:00.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:00.474 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 20:09:00.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:00.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:00.474 | INFO | Uploaded 36GiB to GPU in 1.9826s (18.1579GiB/s) | 20260506 20:09:00.474 | INFO | Downloaded 34GiB to GPU in 1.92627s (17.6507GiB/s) | 20260506 20:09:05.609 | INFO | Analyzing chunk frames = [4109312,4109824) | 20260506 20:09:05.609 | INFO | MemInfo: { | 20260506 20:09:05.609 | INFO | "gpu" : " 17 GB", | 20260506 20:09:05.609 | INFO | "huge" : "", | 20260506 20:09:05.609 | INFO | "huge_shared" : " 27 GB", | 20260506 20:09:05.610 | INFO | "malloc" : " 14 GB", | 20260506 20:09:05.610 | INFO | "pinned" : " 29 GB" | 20260506 20:09:05.610 | INFO | } | 20260506 20:09:05.610 | INFO | Perf report for the last 5.13584 seconds: | 20260506 20:09:05.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:09:05.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:05.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:05.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:05.610 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 20:09:05.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:05.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:05.610 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:09:05.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:05.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:05.610 | INFO | Uploaded 36GiB to GPU in 1.95944s (18.3726GiB/s) | 20260506 20:09:05.610 | INFO | Downloaded 34GiB to GPU in 1.92022s (17.7063GiB/s) | 20260506 20:09:10.747 | INFO | Analyzing chunk frames = [4109824,4110336) | 20260506 20:09:10.747 | INFO | MemInfo: { | 20260506 20:09:10.747 | INFO | "gpu" : " 17 GB", | 20260506 20:09:10.747 | INFO | "huge" : "", | 20260506 20:09:10.747 | INFO | "huge_shared" : " 27 GB", | 20260506 20:09:10.747 | INFO | "malloc" : " 14 GB", | 20260506 20:09:10.747 | INFO | "pinned" : " 29 GB" | 20260506 20:09:10.748 | INFO | } | 20260506 20:09:10.748 | INFO | Perf report for the last 5.13805 seconds: | 20260506 20:09:10.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:09:10.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:10.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:10.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:10.748 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 20:09:10.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:10.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:10.748 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:09:10.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:10.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:10.748 | INFO | Uploaded 36GiB to GPU in 1.92898s (18.6627GiB/s) | 20260506 20:09:10.748 | INFO | Downloaded 34GiB to GPU in 1.8739s (18.144GiB/s) | 20260506 20:09:15.889 | INFO | Analyzing chunk frames = [4110336,4110848) | 20260506 20:09:15.889 | INFO | MemInfo: { | 20260506 20:09:15.889 | INFO | "gpu" : " 17 GB", | 20260506 20:09:15.889 | INFO | "huge" : "", | 20260506 20:09:15.889 | INFO | "huge_shared" : " 27 GB", | 20260506 20:09:15.889 | INFO | "malloc" : " 14 GB", | 20260506 20:09:15.889 | INFO | "pinned" : " 29 GB" | 20260506 20:09:15.889 | INFO | } | 20260506 20:09:15.889 | INFO | Perf report for the last 5.14117 seconds: | 20260506 20:09:15.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:09:15.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:15.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:15.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:15.889 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 20:09:15.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:15.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:15.889 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 20:09:15.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:15.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:15.889 | INFO | Uploaded 36GiB to GPU in 2.01229s (17.8901GiB/s) | 20260506 20:09:15.889 | INFO | Downloaded 34GiB to GPU in 1.91524s (17.7524GiB/s) | 20260506 20:09:21.033 | INFO | Analyzing chunk frames = [4110848,4111360) | 20260506 20:09:21.033 | INFO | MemInfo: { | 20260506 20:09:21.033 | INFO | "gpu" : " 17 GB", | 20260506 20:09:21.033 | INFO | "huge" : "", | 20260506 20:09:21.033 | INFO | "huge_shared" : " 27 GB", | 20260506 20:09:21.033 | INFO | "malloc" : " 14 GB", | 20260506 20:09:21.033 | INFO | "pinned" : " 29 GB" | 20260506 20:09:21.033 | INFO | } | 20260506 20:09:21.033 | INFO | Perf report for the last 5.14417 seconds: | 20260506 20:09:21.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:09:21.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:21.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:21.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:21.033 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260506 20:09:21.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:21.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:21.033 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:09:21.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:21.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:21.033 | INFO | Uploaded 36GiB to GPU in 1.99005s (18.09GiB/s) | 20260506 20:09:21.033 | INFO | Downloaded 34GiB to GPU in 1.95795s (17.3651GiB/s) | 20260506 20:09:26.174 | INFO | Analyzing chunk frames = [4111360,4111872) | 20260506 20:09:26.174 | INFO | MemInfo: { | 20260506 20:09:26.174 | INFO | "gpu" : " 17 GB", | 20260506 20:09:26.175 | INFO | "huge" : "", | 20260506 20:09:26.175 | INFO | "huge_shared" : " 27 GB", | 20260506 20:09:26.175 | INFO | "malloc" : " 14 GB", | 20260506 20:09:26.175 | INFO | "pinned" : " 29 GB" | 20260506 20:09:26.175 | INFO | } | 20260506 20:09:26.175 | INFO | Perf report for the last 5.14184 seconds: | 20260506 20:09:26.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:09:26.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:26.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:26.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:26.175 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 20:09:26.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:26.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:26.175 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:09:26.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:26.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:26.175 | INFO | Uploaded 36GiB to GPU in 2.00714s (17.936GiB/s) | 20260506 20:09:26.175 | INFO | Downloaded 34GiB to GPU in 1.98865s (17.097GiB/s) | 20260506 20:09:31.316 | INFO | Analyzing chunk frames = [4111872,4112384) | 20260506 20:09:31.316 | INFO | MemInfo: { | 20260506 20:09:31.316 | INFO | "gpu" : " 17 GB", | 20260506 20:09:31.316 | INFO | "huge" : "", | 20260506 20:09:31.316 | INFO | "huge_shared" : " 27 GB", | 20260506 20:09:31.316 | INFO | "malloc" : " 14 GB", | 20260506 20:09:31.316 | INFO | "pinned" : " 29 GB" | 20260506 20:09:31.316 | INFO | } | 20260506 20:09:31.316 | INFO | Perf report for the last 5.14157 seconds: | 20260506 20:09:31.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:09:31.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:31.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:31.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:31.317 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:09:31.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:31.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:31.317 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:09:31.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:31.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:31.317 | INFO | Uploaded 36GiB to GPU in 1.97618s (18.217GiB/s) | 20260506 20:09:31.317 | INFO | Downloaded 34GiB to GPU in 1.98373s (17.1394GiB/s) | 20260506 20:09:31.317 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:09:31.317 | INFO | Total mem usage: 18 GiB | 20260506 20:09:31.317 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:09:31.317 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -21.8%) | 20260506 20:09:31.317 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:09:31.317 | INFO | High Water Breakdown: | 20260506 20:09:31.317 | INFO | 60.6% : 8.96 GiB ( 78.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:09:31.317 | INFO | 39.3% : 5.81 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:09:31.317 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:09:31.317 | INFO | ------------------------------------------ | 20260506 20:09:31.317 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:09:31.317 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:09:31.317 | INFO | Total mem usage: 26.5 GiB | 20260506 20:09:31.317 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:09:31.317 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:09:31.317 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:09:31.317 | INFO | High Water Breakdown: | 20260506 20:09:31.317 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:09:31.317 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:09:31.317 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:09:31.317 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:09:31.317 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:09:31.317 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:09:31.317 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:09:31.317 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:09:31.317 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:09:31.317 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:09:31.317 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:09:31.317 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:09:31.317 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:09:31.317 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:09:31.317 | INFO | ------------------------------------------ | 20260506 20:09:31.317 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:09:31.317 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:09:31.317 | INFO | Total mem usage: 25.6 GiB | 20260506 20:09:31.317 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:09:31.317 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:09:31.317 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:09:31.317 | INFO | High Water Breakdown: | 20260506 20:09:31.317 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:09:31.317 | INFO | ------------------------------------------ | 20260506 20:09:31.317 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:09:31.317 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:09:31.317 | INFO | Total mem usage: 15.7 GiB | 20260506 20:09:31.317 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:09:31.317 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:09:31.317 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 20:09:31.317 | INFO | High Water Breakdown: | 20260506 20:09:31.317 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:09:31.317 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:09:31.318 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:09:31.318 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:09:31.318 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:09:31.318 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:09:31.318 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:09:31.318 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:09:31.318 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:09:31.318 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:09:31.318 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:09:31.318 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:09:31.318 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:09:31.318 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:09:31.318 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:09:31.318 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:09:31.318 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:09:31.318 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:09:31.318 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:09:31.318 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:09:31.318 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:09:31.318 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:09:31.318 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:09:31.318 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:09:31.318 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:09:31.318 | INFO | ------------------------------------------ | 20260506 20:09:31.318 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:09:31.318 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:09:31.318 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:09:31.318 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:09:31.318 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.56e+01, 1.34e+01 / 2.30e+00 | 20260506 20:09:31.318 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.02e-01, 9.54e-03 / 6.20e-03 | 20260506 20:09:31.318 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.41e+00, 5.55e-01 / 3.26e-01 | 20260506 20:09:31.318 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.46e+00, 1.84e-01 / 1.26e-01 | 20260506 20:09:31.318 | INFO | PerfReport: 15.95%, DME -- 3.98e+01 / 4.42e+01, 4.04e+00 / 2.79e+00 | 20260506 20:09:31.318 | INFO | PerfReport: 57.26%, FrameLabeling -- 1.79e+01 / 2.10e+01, 1.49e+00 / 1.86e+00 | 20260506 20:09:31.318 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.23e+00 / 2.76e+00, 3.72e-01 / 2.84e-01 | 20260506 20:09:31.318 | INFO | PerfReport: 3.33%, Metrics -- 1.04e+00 / 1.38e+00, 2.77e-01 / 1.46e-01 | 20260506 20:09:31.318 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.56e-01 / 3.98e-01, 5.59e-02 / 2.33e-02 | 20260506 20:09:31.318 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:09:31.318 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:09:31.318 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:09:31.318 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.31e+01 / 0.00e+00 | 20260506 20:09:31.318 | INFO | PerfReport: 21.38%, Upload -- 2.07e+01 / 0.00e+00, 6.77e+00 / 0.00e+00 | 20260506 20:09:31.318 | INFO | PerfReport: 21.23%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 20:09:31.318 | INFO | PerfReport: 32.27%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 20:09:31.318 | INFO | PerfReport: 1.17%, LockUpload -- 1.13e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260506 20:09:31.318 | INFO | PerfReport: 1.27%, LockDownload -- 1.22e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260506 20:09:31.318 | INFO | PerfReport: 22.67%, LockCompute -- 2.19e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 20:09:31.318 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260506 20:09:31.418 | INFO | Starting a round of Baz IO | 20260506 20:09:34.559 | INFO | Populating RTMetrics report | 20260506 20:09:34.559 | INFO | RTMetrics report populated | 20260506 20:09:34.559 | INFO | Opening tmp RTMetrics json file | 20260506 20:09:34.562 | INFO | Serializing RTMetrics to json | 20260506 20:09:34.567 | INFO | Serialization done, swapping tmp file now... | 20260506 20:09:34.569 | INFO | RTMetrics done | 20260506 20:09:35.108 | INFO | Finished a round of Baz IO: Wrote 1.988 GiB in 3.69 seconds | 20260506 20:09:35.108 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.35% / 0.00% / 65.65% | 20260506 20:09:35.108 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.797053% / 0% / 5.16574e-05% | 20260506 20:09:35.108 | INFO | Baz IO overall: | 20260506 20:09:35.108 | INFO | BytesWritten : 2172041178329 | 20260506 20:09:35.108 | INFO | NumEvents : 1049552448408 | 20260506 20:09:36.453 | INFO | Analyzing chunk frames = [4112384,4112896) | 20260506 20:09:36.453 | INFO | MemInfo: { | 20260506 20:09:36.453 | INFO | "gpu" : " 17 GB", | 20260506 20:09:36.453 | INFO | "huge" : "", | 20260506 20:09:36.453 | INFO | "huge_shared" : " 27 GB", | 20260506 20:09:36.453 | INFO | "malloc" : " 14 GB", | 20260506 20:09:36.453 | INFO | "pinned" : " 29 GB" | 20260506 20:09:36.453 | INFO | } | 20260506 20:09:36.453 | INFO | Perf report for the last 5.13674 seconds: | 20260506 20:09:36.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:09:36.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:36.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:36.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:36.453 | INFO | ANALYSIS : 62.5%, 2.9, 95.6ms, 0 | 20260506 20:09:36.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:36.453 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 20:09:36.453 | INFO | PRE_HQ : 60.1%, 1.8, 57.0ms, 0 | 20260506 20:09:36.453 | INFO | BAZWRITER : 71.5%, 4.8, 174.7ms, 0 | 20260506 20:09:36.453 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 20:09:36.453 | INFO | Uploaded 36GiB to GPU in 2.03627s (17.6794GiB/s) | 20260506 20:09:36.453 | INFO | Downloaded 38GiB to GPU in 2.20223s (17.2552GiB/s) | 20260506 20:09:41.590 | INFO | Analyzing chunk frames = [4112896,4113408) | 20260506 20:09:41.590 | INFO | MemInfo: { | 20260506 20:09:41.590 | INFO | "gpu" : " 17 GB", | 20260506 20:09:41.590 | INFO | "huge" : "", | 20260506 20:09:41.590 | INFO | "huge_shared" : " 27 GB", | 20260506 20:09:41.590 | INFO | "malloc" : " 14 GB", | 20260506 20:09:41.590 | INFO | "pinned" : " 29 GB" | 20260506 20:09:41.590 | INFO | } | 20260506 20:09:41.590 | INFO | Perf report for the last 5.13686 seconds: | 20260506 20:09:41.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:09:41.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:41.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:41.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:41.590 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:09:41.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:41.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:41.590 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:09:41.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:41.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:41.590 | INFO | Uploaded 36GiB to GPU in 2.01231s (17.8899GiB/s) | 20260506 20:09:41.590 | INFO | Downloaded 34GiB to GPU in 1.92824s (17.6327GiB/s) | 20260506 20:09:46.724 | INFO | Analyzing chunk frames = [4113408,4113920) | 20260506 20:09:46.724 | INFO | MemInfo: { | 20260506 20:09:46.724 | INFO | "gpu" : " 17 GB", | 20260506 20:09:46.724 | INFO | "huge" : "", | 20260506 20:09:46.724 | INFO | "huge_shared" : " 27 GB", | 20260506 20:09:46.725 | INFO | "malloc" : " 14 GB", | 20260506 20:09:46.725 | INFO | "pinned" : " 29 GB" | 20260506 20:09:46.725 | INFO | } | 20260506 20:09:46.725 | INFO | Perf report for the last 5.13465 seconds: | 20260506 20:09:46.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:09:46.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:46.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:46.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:46.725 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 20:09:46.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:46.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:46.725 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:09:46.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:46.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:46.725 | INFO | Uploaded 36GiB to GPU in 1.97162s (18.2591GiB/s) | 20260506 20:09:46.725 | INFO | Downloaded 34GiB to GPU in 1.92579s (17.6551GiB/s) | 20260506 20:09:51.864 | INFO | Analyzing chunk frames = [4113920,4114432) | 20260506 20:09:51.864 | INFO | MemInfo: { | 20260506 20:09:51.864 | INFO | "gpu" : " 17 GB", | 20260506 20:09:51.864 | INFO | "huge" : "", | 20260506 20:09:51.864 | INFO | "huge_shared" : " 27 GB", | 20260506 20:09:51.864 | INFO | "malloc" : " 14 GB", | 20260506 20:09:51.864 | INFO | "pinned" : " 29 GB" | 20260506 20:09:51.864 | INFO | } | 20260506 20:09:51.864 | INFO | Perf report for the last 5.13956 seconds: | 20260506 20:09:51.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:09:51.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:51.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:51.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:51.864 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:09:51.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:51.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:51.864 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:09:51.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:51.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:51.865 | INFO | Uploaded 36GiB to GPU in 1.92944s (18.6583GiB/s) | 20260506 20:09:51.865 | INFO | Downloaded 34GiB to GPU in 1.87158s (18.1664GiB/s) | 20260506 20:09:57.014 | INFO | Analyzing chunk frames = [4114432,4114944) | 20260506 20:09:57.014 | INFO | MemInfo: { | 20260506 20:09:57.014 | INFO | "gpu" : " 17 GB", | 20260506 20:09:57.014 | INFO | "huge" : "", | 20260506 20:09:57.014 | INFO | "huge_shared" : " 27 GB", | 20260506 20:09:57.014 | INFO | "malloc" : " 14 GB", | 20260506 20:09:57.014 | INFO | "pinned" : " 29 GB" | 20260506 20:09:57.014 | INFO | } | 20260506 20:09:57.014 | INFO | Perf report for the last 5.14956 seconds: | 20260506 20:09:57.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:09:57.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:57.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:57.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:57.014 | INFO | ANALYSIS : 59.8%, 3.0, 93.2ms, 0 | 20260506 20:09:57.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:57.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:57.014 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:09:57.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:57.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:09:57.014 | INFO | Uploaded 36GiB to GPU in 1.95269s (18.4361GiB/s) | 20260506 20:09:57.015 | INFO | Downloaded 34GiB to GPU in 1.90133s (17.8823GiB/s) | 20260506 20:10:02.150 | INFO | Analyzing chunk frames = [4114944,4115456) | 20260506 20:10:02.150 | INFO | MemInfo: { | 20260506 20:10:02.150 | INFO | "gpu" : " 17 GB", | 20260506 20:10:02.150 | INFO | "huge" : "", | 20260506 20:10:02.150 | INFO | "huge_shared" : " 27 GB", | 20260506 20:10:02.150 | INFO | "malloc" : " 14 GB", | 20260506 20:10:02.150 | INFO | "pinned" : " 29 GB" | 20260506 20:10:02.150 | INFO | } | 20260506 20:10:02.150 | INFO | Perf report for the last 5.13636 seconds: | 20260506 20:10:02.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:10:02.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:02.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:02.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:02.150 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 20:10:02.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:02.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:02.150 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 20:10:02.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:02.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:02.150 | INFO | Uploaded 36GiB to GPU in 2.01305s (17.8833GiB/s) | 20260506 20:10:02.151 | INFO | Downloaded 34GiB to GPU in 1.93934s (17.5318GiB/s) | 20260506 20:10:07.286 | INFO | Analyzing chunk frames = [4115456,4115968) | 20260506 20:10:07.286 | INFO | MemInfo: { | 20260506 20:10:07.286 | INFO | "gpu" : " 17 GB", | 20260506 20:10:07.286 | INFO | "huge" : "", | 20260506 20:10:07.286 | INFO | "huge_shared" : " 27 GB", | 20260506 20:10:07.286 | INFO | "malloc" : " 14 GB", | 20260506 20:10:07.286 | INFO | "pinned" : " 29 GB" | 20260506 20:10:07.286 | INFO | } | 20260506 20:10:07.286 | INFO | Perf report for the last 5.13596 seconds: | 20260506 20:10:07.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:10:07.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:07.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:07.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:07.286 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260506 20:10:07.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:07.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:07.286 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 20:10:07.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:07.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:07.287 | INFO | Uploaded 36GiB to GPU in 2.01598s (17.8573GiB/s) | 20260506 20:10:07.287 | INFO | Downloaded 34GiB to GPU in 1.99006s (17.0849GiB/s) | 20260506 20:10:12.424 | INFO | Analyzing chunk frames = [4115968,4116480) | 20260506 20:10:12.424 | INFO | MemInfo: { | 20260506 20:10:12.424 | INFO | "gpu" : " 17 GB", | 20260506 20:10:12.424 | INFO | "huge" : "", | 20260506 20:10:12.424 | INFO | "huge_shared" : " 27 GB", | 20260506 20:10:12.424 | INFO | "malloc" : " 14 GB", | 20260506 20:10:12.424 | INFO | "pinned" : " 29 GB" | 20260506 20:10:12.424 | INFO | } | 20260506 20:10:12.424 | INFO | Perf report for the last 5.13803 seconds: | 20260506 20:10:12.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:10:12.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:12.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:12.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:12.424 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 20:10:12.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:12.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:12.424 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 20:10:12.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:12.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:12.425 | INFO | Uploaded 36GiB to GPU in 2.06843s (17.4045GiB/s) | 20260506 20:10:12.425 | INFO | Downloaded 34GiB to GPU in 1.98945s (17.0901GiB/s) | 20260506 20:10:12.535 | INFO | Starting a round of Baz IO | 20260506 20:10:15.677 | INFO | Populating RTMetrics report | 20260506 20:10:15.678 | INFO | RTMetrics report populated | 20260506 20:10:15.678 | INFO | Opening tmp RTMetrics json file | 20260506 20:10:15.717 | INFO | Serializing RTMetrics to json | 20260506 20:10:15.768 | INFO | Serialization done, swapping tmp file now... | 20260506 20:10:15.772 | INFO | RTMetrics done | 20260506 20:10:16.007 | INFO | Finished a round of Baz IO: Wrote 1.989 GiB in 3.472 seconds | 20260506 20:10:16.007 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.34% / 0.00% / 65.65% | 20260506 20:10:16.007 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.79548% / 3.97364e-06% / 6.35783e-05% | 20260506 20:10:16.007 | INFO | Baz IO overall: | 20260506 20:10:16.007 | INFO | BytesWritten : 2174176694675 | 20260506 20:10:16.007 | INFO | NumEvents : 1050578190187 | 20260506 20:10:17.565 | INFO | Analyzing chunk frames = [4116480,4116992) | 20260506 20:10:17.565 | INFO | MemInfo: { | 20260506 20:10:17.565 | INFO | "gpu" : " 17 GB", | 20260506 20:10:17.565 | INFO | "huge" : "", | 20260506 20:10:17.565 | INFO | "huge_shared" : " 27 GB", | 20260506 20:10:17.565 | INFO | "malloc" : " 14 GB", | 20260506 20:10:17.565 | INFO | "pinned" : " 29 GB" | 20260506 20:10:17.565 | INFO | } | 20260506 20:10:17.565 | INFO | Perf report for the last 5.14057 seconds: | 20260506 20:10:17.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:10:17.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:17.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:17.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:17.565 | INFO | ANALYSIS : 62.7%, 2.9, 95.2ms, 0 | 20260506 20:10:17.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:17.565 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260506 20:10:17.565 | INFO | PRE_HQ : 60.7%, 1.9, 59.3ms, 0 | 20260506 20:10:17.565 | INFO | BAZWRITER : 66.9%, 4.8, 165.7ms, 0 | 20260506 20:10:17.565 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.9ms, 0 | 20260506 20:10:17.565 | INFO | Uploaded 36GiB to GPU in 2.0697s (17.3938GiB/s) | 20260506 20:10:17.565 | INFO | Downloaded 38GiB to GPU in 2.22684s (17.0645GiB/s) | 20260506 20:10:22.700 | INFO | Analyzing chunk frames = [4116992,4117504) | 20260506 20:10:22.700 | INFO | MemInfo: { | 20260506 20:10:22.700 | INFO | "gpu" : " 17 GB", | 20260506 20:10:22.700 | INFO | "huge" : "", | 20260506 20:10:22.700 | INFO | "huge_shared" : " 27 GB", | 20260506 20:10:22.700 | INFO | "malloc" : " 14 GB", | 20260506 20:10:22.700 | INFO | "pinned" : " 29 GB" | 20260506 20:10:22.700 | INFO | } | 20260506 20:10:22.700 | INFO | Perf report for the last 5.13565 seconds: | 20260506 20:10:22.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:10:22.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:22.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:22.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:22.701 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:10:22.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:22.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:22.701 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:10:22.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:22.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:22.701 | INFO | Uploaded 36GiB to GPU in 2.00857s (17.9232GiB/s) | 20260506 20:10:22.701 | INFO | Downloaded 34GiB to GPU in 1.92387s (17.6727GiB/s) | 20260506 20:10:27.842 | INFO | Analyzing chunk frames = [4117504,4118016) | 20260506 20:10:27.842 | INFO | MemInfo: { | 20260506 20:10:27.842 | INFO | "gpu" : " 17 GB", | 20260506 20:10:27.842 | INFO | "huge" : "", | 20260506 20:10:27.842 | INFO | "huge_shared" : " 27 GB", | 20260506 20:10:27.842 | INFO | "malloc" : " 14 GB", | 20260506 20:10:27.842 | INFO | "pinned" : " 29 GB" | 20260506 20:10:27.842 | INFO | } | 20260506 20:10:27.842 | INFO | Perf report for the last 5.14148 seconds: | 20260506 20:10:27.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:10:27.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:27.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:27.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:27.842 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 20:10:27.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:27.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:27.842 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:10:27.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:27.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:27.842 | INFO | Uploaded 36GiB to GPU in 1.98278s (18.1563GiB/s) | 20260506 20:10:27.842 | INFO | Downloaded 34GiB to GPU in 1.91188s (17.7835GiB/s) | 20260506 20:10:32.982 | INFO | Analyzing chunk frames = [4118016,4118528) | 20260506 20:10:32.982 | INFO | MemInfo: { | 20260506 20:10:32.982 | INFO | "gpu" : " 17 GB", | 20260506 20:10:32.982 | INFO | "huge" : "", | 20260506 20:10:32.982 | INFO | "huge_shared" : " 27 GB", | 20260506 20:10:32.982 | INFO | "malloc" : " 14 GB", | 20260506 20:10:32.982 | INFO | "pinned" : " 29 GB" | 20260506 20:10:32.982 | INFO | } | 20260506 20:10:32.982 | INFO | Perf report for the last 5.13998 seconds: | 20260506 20:10:32.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:10:32.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:32.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:32.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:32.982 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 20:10:32.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:32.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:32.982 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 20:10:32.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:32.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:32.982 | INFO | Uploaded 36GiB to GPU in 1.94302s (18.5278GiB/s) | 20260506 20:10:32.982 | INFO | Downloaded 34GiB to GPU in 1.87133s (18.1689GiB/s) | 20260506 20:10:38.124 | INFO | Analyzing chunk frames = [4118528,4119040) | 20260506 20:10:38.124 | INFO | MemInfo: { | 20260506 20:10:38.124 | INFO | "gpu" : " 17 GB", | 20260506 20:10:38.124 | INFO | "huge" : "", | 20260506 20:10:38.124 | INFO | "huge_shared" : " 27 GB", | 20260506 20:10:38.124 | INFO | "malloc" : " 14 GB", | 20260506 20:10:38.125 | INFO | "pinned" : " 29 GB" | 20260506 20:10:38.125 | INFO | } | 20260506 20:10:38.125 | INFO | Perf report for the last 5.14319 seconds: | 20260506 20:10:38.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:10:38.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:38.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:38.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:38.125 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 20:10:38.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:38.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:38.125 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 20:10:38.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:38.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:38.125 | INFO | Uploaded 36GiB to GPU in 2.02078s (17.8149GiB/s) | 20260506 20:10:38.125 | INFO | Downloaded 34GiB to GPU in 1.91316s (17.7717GiB/s) | 20260506 20:10:43.269 | INFO | Analyzing chunk frames = [4119040,4119552) | 20260506 20:10:43.269 | INFO | MemInfo: { | 20260506 20:10:43.269 | INFO | "gpu" : " 17 GB", | 20260506 20:10:43.269 | INFO | "huge" : "", | 20260506 20:10:43.269 | INFO | "huge_shared" : " 27 GB", | 20260506 20:10:43.269 | INFO | "malloc" : " 14 GB", | 20260506 20:10:43.269 | INFO | "pinned" : " 29 GB" | 20260506 20:10:43.269 | INFO | } | 20260506 20:10:43.269 | INFO | Perf report for the last 5.14349 seconds: | 20260506 20:10:43.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:10:43.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:43.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:43.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:43.269 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 20:10:43.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:43.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:43.269 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:10:43.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:43.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:43.269 | INFO | Uploaded 36GiB to GPU in 2.03664s (17.6762GiB/s) | 20260506 20:10:43.269 | INFO | Downloaded 34GiB to GPU in 1.94315s (17.4974GiB/s) | 20260506 20:10:48.401 | INFO | Analyzing chunk frames = [4119552,4120064) | 20260506 20:10:48.401 | INFO | MemInfo: { | 20260506 20:10:48.401 | INFO | "gpu" : " 17 GB", | 20260506 20:10:48.401 | INFO | "huge" : "", | 20260506 20:10:48.401 | INFO | "huge_shared" : " 27 GB", | 20260506 20:10:48.401 | INFO | "malloc" : " 14 GB", | 20260506 20:10:48.401 | INFO | "pinned" : " 29 GB" | 20260506 20:10:48.401 | INFO | } | 20260506 20:10:48.401 | INFO | Perf report for the last 5.13276 seconds: | 20260506 20:10:48.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:10:48.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:48.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:48.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:48.402 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 20:10:48.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:48.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:48.402 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 20:10:48.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:48.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:48.402 | INFO | Uploaded 36GiB to GPU in 1.97898s (18.1912GiB/s) | 20260506 20:10:48.402 | INFO | Downloaded 34GiB to GPU in 1.99762s (17.0203GiB/s) | 20260506 20:10:53.539 | INFO | Analyzing chunk frames = [4120064,4120576) | 20260506 20:10:53.539 | INFO | MemInfo: { | 20260506 20:10:53.539 | INFO | "gpu" : " 17 GB", | 20260506 20:10:53.539 | INFO | "huge" : "", | 20260506 20:10:53.539 | INFO | "huge_shared" : " 27 GB", | 20260506 20:10:53.539 | INFO | "malloc" : " 14 GB", | 20260506 20:10:53.539 | INFO | "pinned" : " 29 GB" | 20260506 20:10:53.539 | INFO | } | 20260506 20:10:53.539 | INFO | Perf report for the last 5.1377 seconds: | 20260506 20:10:53.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:10:53.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:53.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:53.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:53.539 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 20:10:53.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:53.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:53.539 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 20:10:53.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:53.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:53.539 | INFO | Uploaded 36GiB to GPU in 1.99701s (18.0269GiB/s) | 20260506 20:10:53.540 | INFO | Downloaded 34GiB to GPU in 1.99379s (17.053GiB/s) | 20260506 20:10:53.540 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:10:53.540 | INFO | Total mem usage: 18 GiB | 20260506 20:10:53.540 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:10:53.540 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -11.8%) | 20260506 20:10:53.540 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:10:53.540 | INFO | High Water Breakdown: | 20260506 20:10:53.540 | INFO | 61.9% : 9.98 GiB ( 87.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:10:53.540 | INFO | 37.9% : 6.11 GiB ( 91.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:10:53.540 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:10:53.540 | INFO | ------------------------------------------ | 20260506 20:10:53.540 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:10:53.540 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:10:53.540 | INFO | Total mem usage: 26.5 GiB | 20260506 20:10:53.540 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:10:53.540 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 20:10:53.540 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260506 20:10:53.540 | INFO | High Water Breakdown: | 20260506 20:10:53.540 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:10:53.540 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:10:53.540 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:10:53.540 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:10:53.540 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:10:53.540 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:10:53.540 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:10:53.540 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:10:53.540 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:10:53.540 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:10:53.540 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:10:53.540 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:10:53.540 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:10:53.540 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:10:53.540 | INFO | ------------------------------------------ | 20260506 20:10:53.540 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:10:53.540 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:10:53.540 | INFO | Total mem usage: 25.6 GiB | 20260506 20:10:53.540 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:10:53.540 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:10:53.540 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:10:53.540 | INFO | High Water Breakdown: | 20260506 20:10:53.540 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:10:53.540 | INFO | ------------------------------------------ | 20260506 20:10:53.540 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:10:53.540 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:10:53.540 | INFO | Total mem usage: 15.7 GiB | 20260506 20:10:53.540 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:10:53.540 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:10:53.540 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 20:10:53.540 | INFO | High Water Breakdown: | 20260506 20:10:53.540 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:10:53.540 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:10:53.540 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:10:53.540 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:10:53.540 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:10:53.540 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:10:53.540 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:10:53.540 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:10:53.540 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:10:53.540 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:10:53.540 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:10:53.540 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:10:53.540 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:10:53.540 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:10:53.540 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:10:53.540 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:10:53.540 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:10:53.540 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:10:53.540 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:10:53.540 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:10:53.540 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:10:53.541 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:10:53.541 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:10:53.541 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:10:53.541 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:10:53.541 | INFO | ------------------------------------------ | 20260506 20:10:53.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:10:53.541 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:10:53.541 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:10:53.541 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:10:53.541 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.55e+01, 1.35e+01 / 2.67e+00 | 20260506 20:10:53.541 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.02e-01, 1.04e-02 / 7.11e-03 | 20260506 20:10:53.541 | INFO | PerfReport: 11.62%, Baseline -- 3.62e+00 / 4.38e+00, 5.48e-01 / 3.52e-01 | 20260506 20:10:53.541 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.49e+00, 1.81e-01 / 1.20e-01 | 20260506 20:10:53.541 | INFO | PerfReport: 16.02%, DME -- 4.00e+01 / 4.45e+01, 4.05e+00 / 2.57e+00 | 20260506 20:10:53.541 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.41e+00 / 1.89e+00 | 20260506 20:10:53.541 | INFO | PerfReport: 7.17%, PulseAccumulating -- 2.24e+00 / 2.84e+00, 3.86e-01 / 2.05e-01 | 20260506 20:10:53.541 | INFO | PerfReport: 3.32%, Metrics -- 1.04e+00 / 1.39e+00, 2.72e-01 / 1.92e-01 | 20260506 20:10:53.541 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.57e-01 / 4.08e-01, 5.70e-02 / 3.10e-02 | 20260506 20:10:53.541 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:10:53.541 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:10:53.541 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:10:53.541 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 20:10:53.541 | INFO | PerfReport: 21.53%, Upload -- 2.09e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260506 20:10:53.541 | INFO | PerfReport: 21.19%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 20:10:53.541 | INFO | PerfReport: 32.20%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 20:10:53.541 | INFO | PerfReport: 1.37%, LockUpload -- 1.33e+00 / 0.00e+00, 5.25e+00 / 0.00e+00 | 20260506 20:10:53.541 | INFO | PerfReport: 1.17%, LockDownload -- 1.13e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 20:10:53.541 | INFO | PerfReport: 22.52%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 20:10:53.541 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.93e-02 / 0.00e+00 | 20260506 20:10:53.648 | INFO | Starting a round of Baz IO | 20260506 20:10:56.779 | INFO | Populating RTMetrics report | 20260506 20:10:56.779 | INFO | RTMetrics report populated | 20260506 20:10:56.779 | INFO | Opening tmp RTMetrics json file | 20260506 20:10:56.781 | INFO | Serializing RTMetrics to json | 20260506 20:10:56.782 | INFO | Serialization done, swapping tmp file now... | 20260506 20:10:56.782 | INFO | RTMetrics done | 20260506 20:10:57.112 | INFO | Finished a round of Baz IO: Wrote 1.986 GiB in 3.464 seconds | 20260506 20:10:57.112 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.34% / 0.00% / 65.66% | 20260506 20:10:57.112 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.789404% / 0% / 4.76837e-05% | 20260506 20:10:57.112 | INFO | Baz IO overall: | 20260506 20:10:57.112 | INFO | BytesWritten : 2176308973114 | 20260506 20:10:57.112 | INFO | NumEvents : 1051601299542 | 20260506 20:10:58.678 | INFO | Analyzing chunk frames = [4120576,4121088) | 20260506 20:10:58.678 | INFO | MemInfo: { | 20260506 20:10:58.678 | INFO | "gpu" : " 17 GB", | 20260506 20:10:58.678 | INFO | "huge" : "", | 20260506 20:10:58.678 | INFO | "huge_shared" : " 27 GB", | 20260506 20:10:58.678 | INFO | "malloc" : " 14 GB", | 20260506 20:10:58.678 | INFO | "pinned" : " 29 GB" | 20260506 20:10:58.678 | INFO | } | 20260506 20:10:58.678 | INFO | Perf report for the last 5.13885 seconds: | 20260506 20:10:58.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:10:58.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:58.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:58.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:58.678 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 20:10:58.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:10:58.678 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 20:10:58.678 | INFO | PRE_HQ : 59.8%, 1.8, 57.2ms, 0 | 20260506 20:10:58.678 | INFO | BAZWRITER : 67.2%, 4.8, 166.1ms, 0 | 20260506 20:10:58.678 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 20:10:58.678 | INFO | Uploaded 36GiB to GPU in 2.0274s (17.7568GiB/s) | 20260506 20:10:58.678 | INFO | Downloaded 38GiB to GPU in 2.20283s (17.2505GiB/s) | 20260506 20:11:03.816 | INFO | Analyzing chunk frames = [4121088,4121600) | 20260506 20:11:03.816 | INFO | MemInfo: { | 20260506 20:11:03.816 | INFO | "gpu" : " 17 GB", | 20260506 20:11:03.816 | INFO | "huge" : "", | 20260506 20:11:03.816 | INFO | "huge_shared" : " 27 GB", | 20260506 20:11:03.816 | INFO | "malloc" : " 14 GB", | 20260506 20:11:03.816 | INFO | "pinned" : " 29 GB" | 20260506 20:11:03.816 | INFO | } | 20260506 20:11:03.816 | INFO | Perf report for the last 5.13813 seconds: | 20260506 20:11:03.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:11:03.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:03.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:03.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:03.816 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:11:03.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:03.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:03.816 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:11:03.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:03.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:03.816 | INFO | Uploaded 36GiB to GPU in 2.00441s (17.9604GiB/s) | 20260506 20:11:03.817 | INFO | Downloaded 34GiB to GPU in 1.92967s (17.6196GiB/s) | 20260506 20:11:08.957 | INFO | Analyzing chunk frames = [4121600,4122112) | 20260506 20:11:08.957 | INFO | MemInfo: { | 20260506 20:11:08.957 | INFO | "gpu" : " 17 GB", | 20260506 20:11:08.957 | INFO | "huge" : "", | 20260506 20:11:08.957 | INFO | "huge_shared" : " 27 GB", | 20260506 20:11:08.957 | INFO | "malloc" : " 14 GB", | 20260506 20:11:08.957 | INFO | "pinned" : " 29 GB" | 20260506 20:11:08.957 | INFO | } | 20260506 20:11:08.957 | INFO | Perf report for the last 5.14077 seconds: | 20260506 20:11:08.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:11:08.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:08.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:08.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:08.957 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 20:11:08.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:08.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:08.957 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:11:08.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:08.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:08.957 | INFO | Uploaded 36GiB to GPU in 1.97307s (18.2456GiB/s) | 20260506 20:11:08.958 | INFO | Downloaded 34GiB to GPU in 1.92113s (17.6979GiB/s) | 20260506 20:11:14.093 | INFO | Analyzing chunk frames = [4122112,4122624) | 20260506 20:11:14.093 | INFO | MemInfo: { | 20260506 20:11:14.093 | INFO | "gpu" : " 17 GB", | 20260506 20:11:14.093 | INFO | "huge" : "", | 20260506 20:11:14.093 | INFO | "huge_shared" : " 27 GB", | 20260506 20:11:14.093 | INFO | "malloc" : " 14 GB", | 20260506 20:11:14.093 | INFO | "pinned" : " 29 GB" | 20260506 20:11:14.093 | INFO | } | 20260506 20:11:14.093 | INFO | Perf report for the last 5.13626 seconds: | 20260506 20:11:14.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:11:14.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:14.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:14.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:14.093 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 20:11:14.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:14.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:14.093 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:11:14.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:14.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:14.094 | INFO | Uploaded 36GiB to GPU in 1.91499s (18.7991GiB/s) | 20260506 20:11:14.094 | INFO | Downloaded 34GiB to GPU in 1.87378s (18.1452GiB/s) | 20260506 20:11:19.234 | INFO | Analyzing chunk frames = [4122624,4123136) | 20260506 20:11:19.234 | INFO | MemInfo: { | 20260506 20:11:19.234 | INFO | "gpu" : " 17 GB", | 20260506 20:11:19.234 | INFO | "huge" : "", | 20260506 20:11:19.234 | INFO | "huge_shared" : " 27 GB", | 20260506 20:11:19.234 | INFO | "malloc" : " 14 GB", | 20260506 20:11:19.234 | INFO | "pinned" : " 29 GB" | 20260506 20:11:19.234 | INFO | } | 20260506 20:11:19.234 | INFO | Perf report for the last 5.14059 seconds: | 20260506 20:11:19.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:11:19.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:19.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:19.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:19.234 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260506 20:11:19.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:19.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:19.234 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:11:19.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:19.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:19.234 | INFO | Uploaded 36GiB to GPU in 1.96352s (18.3344GiB/s) | 20260506 20:11:19.234 | INFO | Downloaded 34GiB to GPU in 1.89774s (17.9161GiB/s) | 20260506 20:11:24.378 | INFO | Analyzing chunk frames = [4123136,4123648) | 20260506 20:11:24.378 | INFO | MemInfo: { | 20260506 20:11:24.378 | INFO | "gpu" : " 17 GB", | 20260506 20:11:24.378 | INFO | "huge" : "", | 20260506 20:11:24.378 | INFO | "huge_shared" : " 27 GB", | 20260506 20:11:24.378 | INFO | "malloc" : " 14 GB", | 20260506 20:11:24.378 | INFO | "pinned" : " 29 GB" | 20260506 20:11:24.378 | INFO | } | 20260506 20:11:24.378 | INFO | Perf report for the last 5.14448 seconds: | 20260506 20:11:24.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:11:24.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:24.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:24.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:24.379 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:11:24.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:24.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:24.379 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 20:11:24.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:24.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:24.379 | INFO | Uploaded 36GiB to GPU in 2.015s (17.866GiB/s) | 20260506 20:11:24.379 | INFO | Downloaded 34GiB to GPU in 1.96491s (17.3036GiB/s) | 20260506 20:11:29.519 | INFO | Analyzing chunk frames = [4123648,4124160) | 20260506 20:11:29.519 | INFO | MemInfo: { | 20260506 20:11:29.519 | INFO | "gpu" : " 17 GB", | 20260506 20:11:29.519 | INFO | "huge" : "", | 20260506 20:11:29.519 | INFO | "huge_shared" : " 27 GB", | 20260506 20:11:29.519 | INFO | "malloc" : " 14 GB", | 20260506 20:11:29.519 | INFO | "pinned" : " 29 GB" | 20260506 20:11:29.519 | INFO | } | 20260506 20:11:29.519 | INFO | Perf report for the last 5.14052 seconds: | 20260506 20:11:29.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:11:29.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:29.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:29.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:29.519 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 20:11:29.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:29.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:29.519 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:11:29.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:29.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:29.519 | INFO | Uploaded 36GiB to GPU in 1.95867s (18.3798GiB/s) | 20260506 20:11:29.519 | INFO | Downloaded 34GiB to GPU in 1.99412s (17.0501GiB/s) | 20260506 20:11:34.657 | INFO | Analyzing chunk frames = [4124160,4124672) | 20260506 20:11:34.657 | INFO | MemInfo: { | 20260506 20:11:34.657 | INFO | "gpu" : " 17 GB", | 20260506 20:11:34.657 | INFO | "huge" : "", | 20260506 20:11:34.657 | INFO | "huge_shared" : " 27 GB", | 20260506 20:11:34.657 | INFO | "malloc" : " 14 GB", | 20260506 20:11:34.657 | INFO | "pinned" : " 29 GB" | 20260506 20:11:34.657 | INFO | } | 20260506 20:11:34.657 | INFO | Perf report for the last 5.13839 seconds: | 20260506 20:11:34.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:11:34.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:34.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:34.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:34.657 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 20:11:34.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:34.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:34.657 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 20:11:34.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:34.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:34.657 | INFO | Uploaded 36GiB to GPU in 2.03911s (17.6548GiB/s) | 20260506 20:11:34.657 | INFO | Downloaded 34GiB to GPU in 2.02154s (16.8189GiB/s) | 20260506 20:11:34.768 | INFO | Starting a round of Baz IO | 20260506 20:11:37.899 | INFO | Populating RTMetrics report | 20260506 20:11:37.900 | INFO | RTMetrics report populated | 20260506 20:11:37.900 | INFO | Opening tmp RTMetrics json file | 20260506 20:11:37.930 | INFO | Serializing RTMetrics to json | 20260506 20:11:38.045 | INFO | Serialization done, swapping tmp file now... | 20260506 20:11:38.095 | INFO | RTMetrics done | 20260506 20:11:38.288 | INFO | Finished a round of Baz IO: Wrote 1.987 GiB in 3.519 seconds | 20260506 20:11:38.288 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.33% / 0.00% / 65.66% | 20260506 20:11:38.288 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.789948% / 3.97364e-06% / 0.000111262% | 20260506 20:11:38.288 | INFO | Baz IO overall: | 20260506 20:11:38.288 | INFO | BytesWritten : 2178442708955 | 20260506 20:11:38.288 | INFO | NumEvents : 1052625615024 | 20260506 20:11:39.792 | INFO | Analyzing chunk frames = [4124672,4125184) | 20260506 20:11:39.792 | INFO | MemInfo: { | 20260506 20:11:39.792 | INFO | "gpu" : " 17 GB", | 20260506 20:11:39.792 | INFO | "huge" : "", | 20260506 20:11:39.792 | INFO | "huge_shared" : " 27 GB", | 20260506 20:11:39.792 | INFO | "malloc" : " 14 GB", | 20260506 20:11:39.792 | INFO | "pinned" : " 29 GB" | 20260506 20:11:39.792 | INFO | } | 20260506 20:11:39.792 | INFO | Perf report for the last 5.13511 seconds: | 20260506 20:11:39.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:11:39.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:39.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:39.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:39.793 | INFO | ANALYSIS : 62.6%, 2.9, 95.6ms, 0 | 20260506 20:11:39.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:39.793 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260506 20:11:39.793 | INFO | PRE_HQ : 60.1%, 1.8, 57.2ms, 0 | 20260506 20:11:39.793 | INFO | BAZWRITER : 68.3%, 4.8, 167.1ms, 0 | 20260506 20:11:39.793 | INFO | BAZ_SAMPLING : 3.8%, 1.0, 1.9ms, 0 | 20260506 20:11:39.793 | INFO | Uploaded 36GiB to GPU in 2.04858s (17.5731GiB/s) | 20260506 20:11:39.793 | INFO | Downloaded 38GiB to GPU in 2.20657s (17.2213GiB/s) | 20260506 20:11:44.932 | INFO | Analyzing chunk frames = [4125184,4125696) | 20260506 20:11:44.932 | INFO | MemInfo: { | 20260506 20:11:44.932 | INFO | "gpu" : " 17 GB", | 20260506 20:11:44.932 | INFO | "huge" : "", | 20260506 20:11:44.932 | INFO | "huge_shared" : " 27 GB", | 20260506 20:11:44.932 | INFO | "malloc" : " 14 GB", | 20260506 20:11:44.932 | INFO | "pinned" : " 29 GB" | 20260506 20:11:44.932 | INFO | } | 20260506 20:11:44.932 | INFO | Perf report for the last 5.14006 seconds: | 20260506 20:11:44.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:11:44.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:44.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:44.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:44.933 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:11:44.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:44.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:44.933 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:11:44.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:44.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:44.933 | INFO | Uploaded 36GiB to GPU in 1.99811s (18.017GiB/s) | 20260506 20:11:44.933 | INFO | Downloaded 34GiB to GPU in 1.92484s (17.6638GiB/s) | 20260506 20:11:50.072 | INFO | Analyzing chunk frames = [4125696,4126208) | 20260506 20:11:50.072 | INFO | MemInfo: { | 20260506 20:11:50.072 | INFO | "gpu" : " 17 GB", | 20260506 20:11:50.072 | INFO | "huge" : "", | 20260506 20:11:50.072 | INFO | "huge_shared" : " 27 GB", | 20260506 20:11:50.072 | INFO | "malloc" : " 14 GB", | 20260506 20:11:50.072 | INFO | "pinned" : " 29 GB" | 20260506 20:11:50.072 | INFO | } | 20260506 20:11:50.073 | INFO | Perf report for the last 5.14008 seconds: | 20260506 20:11:50.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:11:50.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:50.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:50.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:50.073 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 20:11:50.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:50.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:50.073 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:11:50.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:50.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:50.073 | INFO | Uploaded 36GiB to GPU in 1.97387s (18.2383GiB/s) | 20260506 20:11:50.073 | INFO | Downloaded 34GiB to GPU in 1.91958s (17.7122GiB/s) | 20260506 20:11:55.211 | INFO | Analyzing chunk frames = [4126208,4126720) | 20260506 20:11:55.211 | INFO | MemInfo: { | 20260506 20:11:55.211 | INFO | "gpu" : " 17 GB", | 20260506 20:11:55.211 | INFO | "huge" : "", | 20260506 20:11:55.211 | INFO | "huge_shared" : " 27 GB", | 20260506 20:11:55.211 | INFO | "malloc" : " 14 GB", | 20260506 20:11:55.211 | INFO | "pinned" : " 29 GB" | 20260506 20:11:55.211 | INFO | } | 20260506 20:11:55.211 | INFO | Perf report for the last 5.1382 seconds: | 20260506 20:11:55.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:11:55.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:55.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:55.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:55.211 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 20:11:55.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:55.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:55.211 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 20:11:55.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:55.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:11:55.211 | INFO | Uploaded 36GiB to GPU in 1.92745s (18.6775GiB/s) | 20260506 20:11:55.211 | INFO | Downloaded 34GiB to GPU in 1.87111s (18.171GiB/s) | 20260506 20:12:00.356 | INFO | Analyzing chunk frames = [4126720,4127232) | 20260506 20:12:00.356 | INFO | MemInfo: { | 20260506 20:12:00.356 | INFO | "gpu" : " 17 GB", | 20260506 20:12:00.356 | INFO | "huge" : "", | 20260506 20:12:00.356 | INFO | "huge_shared" : " 27 GB", | 20260506 20:12:00.356 | INFO | "malloc" : " 14 GB", | 20260506 20:12:00.356 | INFO | "pinned" : " 29 GB" | 20260506 20:12:00.356 | INFO | } | 20260506 20:12:00.356 | INFO | Perf report for the last 5.14546 seconds: | 20260506 20:12:00.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:12:00.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:00.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:00.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:00.356 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260506 20:12:00.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:00.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:00.356 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 20:12:00.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:00.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:00.357 | INFO | Uploaded 36GiB to GPU in 2.00763s (17.9316GiB/s) | 20260506 20:12:00.357 | INFO | Downloaded 34GiB to GPU in 1.90763s (17.8232GiB/s) | 20260506 20:12:05.490 | INFO | Analyzing chunk frames = [4127232,4127744) | 20260506 20:12:05.490 | INFO | MemInfo: { | 20260506 20:12:05.490 | INFO | "gpu" : " 17 GB", | 20260506 20:12:05.490 | INFO | "huge" : "", | 20260506 20:12:05.490 | INFO | "huge_shared" : " 27 GB", | 20260506 20:12:05.490 | INFO | "malloc" : " 14 GB", | 20260506 20:12:05.490 | INFO | "pinned" : " 29 GB" | 20260506 20:12:05.490 | INFO | } | 20260506 20:12:05.490 | INFO | Perf report for the last 5.13371 seconds: | 20260506 20:12:05.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:12:05.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:05.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:05.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:05.490 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 20:12:05.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:05.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:05.490 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:12:05.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:05.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:05.490 | INFO | Uploaded 36GiB to GPU in 2.00375s (17.9663GiB/s) | 20260506 20:12:05.490 | INFO | Downloaded 34GiB to GPU in 1.9457s (17.4744GiB/s) | 20260506 20:12:10.630 | INFO | Analyzing chunk frames = [4127744,4128256) | 20260506 20:12:10.630 | INFO | MemInfo: { | 20260506 20:12:10.630 | INFO | "gpu" : " 17 GB", | 20260506 20:12:10.630 | INFO | "huge" : "", | 20260506 20:12:10.630 | INFO | "huge_shared" : " 27 GB", | 20260506 20:12:10.630 | INFO | "malloc" : " 14 GB", | 20260506 20:12:10.630 | INFO | "pinned" : " 29 GB" | 20260506 20:12:10.630 | INFO | } | 20260506 20:12:10.630 | INFO | Perf report for the last 5.14026 seconds: | 20260506 20:12:10.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:12:10.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:10.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:10.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:10.630 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 20:12:10.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:10.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:10.631 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:12:10.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:10.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:10.631 | INFO | Uploaded 36GiB to GPU in 1.96129s (18.3552GiB/s) | 20260506 20:12:10.631 | INFO | Downloaded 34GiB to GPU in 2.00652s (16.9447GiB/s) | 20260506 20:12:15.769 | INFO | Analyzing chunk frames = [4128256,4128768) | 20260506 20:12:15.769 | INFO | MemInfo: { | 20260506 20:12:15.769 | INFO | "gpu" : " 17 GB", | 20260506 20:12:15.769 | INFO | "huge" : "", | 20260506 20:12:15.769 | INFO | "huge_shared" : " 27 GB", | 20260506 20:12:15.769 | INFO | "malloc" : " 14 GB", | 20260506 20:12:15.769 | INFO | "pinned" : " 29 GB" | 20260506 20:12:15.769 | INFO | } | 20260506 20:12:15.769 | INFO | Perf report for the last 5.13923 seconds: | 20260506 20:12:15.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:12:15.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:15.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:15.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:15.770 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 20:12:15.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:15.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:15.770 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 20:12:15.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:15.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:15.770 | INFO | Uploaded 36GiB to GPU in 2.05315s (17.5341GiB/s) | 20260506 20:12:15.770 | INFO | Downloaded 34GiB to GPU in 1.9771s (17.1969GiB/s) | 20260506 20:12:15.770 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:12:15.770 | INFO | Total mem usage: 18 GiB | 20260506 20:12:15.770 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:12:15.770 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.9%) | 20260506 20:12:15.770 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:12:15.770 | INFO | High Water Breakdown: | 20260506 20:12:15.770 | INFO | 61.8% : 9.87 GiB ( 86.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:12:15.770 | INFO | 38% : 6.07 GiB ( 90.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:12:15.770 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:12:15.770 | INFO | ------------------------------------------ | 20260506 20:12:15.770 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:12:15.770 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:12:15.770 | INFO | Total mem usage: 26.5 GiB | 20260506 20:12:15.770 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:12:15.770 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:12:15.770 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:12:15.770 | INFO | High Water Breakdown: | 20260506 20:12:15.770 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:12:15.770 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:12:15.770 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:12:15.770 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:12:15.770 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:12:15.770 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:12:15.770 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:12:15.770 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:12:15.770 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:12:15.770 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:12:15.771 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:12:15.771 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:12:15.771 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:12:15.771 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:12:15.771 | INFO | ------------------------------------------ | 20260506 20:12:15.771 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:12:15.771 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:12:15.771 | INFO | Total mem usage: 25.6 GiB | 20260506 20:12:15.771 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:12:15.771 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:12:15.771 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:12:15.771 | INFO | High Water Breakdown: | 20260506 20:12:15.771 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:12:15.771 | INFO | ------------------------------------------ | 20260506 20:12:15.771 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:12:15.771 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:12:15.771 | INFO | Total mem usage: 15.7 GiB | 20260506 20:12:15.771 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:12:15.771 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:12:15.771 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 20:12:15.771 | INFO | High Water Breakdown: | 20260506 20:12:15.771 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:12:15.771 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:12:15.771 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:12:15.771 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:12:15.771 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:12:15.771 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:12:15.771 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:12:15.771 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:12:15.771 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:12:15.771 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:12:15.771 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:12:15.771 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:12:15.771 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:12:15.771 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:12:15.771 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:12:15.771 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:12:15.771 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:12:15.771 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:12:15.771 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:12:15.771 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:12:15.771 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:12:15.771 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:12:15.771 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:12:15.771 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:12:15.771 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:12:15.771 | INFO | ------------------------------------------ | 20260506 20:12:15.771 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:12:15.771 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:12:15.771 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:12:15.771 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:12:15.771 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.61e+01, 1.31e+01 / 2.52e+00 | 20260506 20:12:15.771 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 9.97e-02, 1.02e-02 / 5.68e-03 | 20260506 20:12:15.771 | INFO | PerfReport: 11.63%, Baseline -- 3.63e+00 / 4.34e+00, 5.57e-01 / 3.10e-01 | 20260506 20:12:15.771 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.82e-01 / 1.23e-01 | 20260506 20:12:15.771 | INFO | PerfReport: 16.04%, DME -- 4.00e+01 / 4.49e+01, 4.36e+00 / 2.93e+00 | 20260506 20:12:15.771 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.40e+00 / 1.80e+00 | 20260506 20:12:15.771 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.23e+00 / 2.80e+00, 3.83e-01 / 2.11e-01 | 20260506 20:12:15.771 | INFO | PerfReport: 3.33%, Metrics -- 1.04e+00 / 1.41e+00, 2.77e-01 / 1.66e-01 | 20260506 20:12:15.771 | INFO | PerfReport: 1.15%, PulseSerializing -- 3.60e-01 / 4.10e-01, 1.58e-01 / 2.56e-02 | 20260506 20:12:15.771 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:12:15.771 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:12:15.771 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:12:15.771 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 20:12:15.771 | INFO | PerfReport: 21.43%, Upload -- 2.07e+01 / 0.00e+00, 6.87e+00 / 0.00e+00 | 20260506 20:12:15.771 | INFO | PerfReport: 21.22%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 20:12:15.771 | INFO | PerfReport: 32.23%, Compute -- 3.12e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 20:12:15.771 | INFO | PerfReport: 1.25%, LockUpload -- 1.21e+00 / 0.00e+00, 5.07e+00 / 0.00e+00 | 20260506 20:12:15.771 | INFO | PerfReport: 1.23%, LockDownload -- 1.20e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260506 20:12:15.771 | INFO | PerfReport: 22.61%, LockCompute -- 2.19e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 20:12:15.771 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260506 20:12:15.884 | INFO | Starting a round of Baz IO | 20260506 20:12:19.023 | INFO | Populating RTMetrics report | 20260506 20:12:19.023 | INFO | RTMetrics report populated | 20260506 20:12:19.023 | INFO | Opening tmp RTMetrics json file | 20260506 20:12:19.025 | INFO | Serializing RTMetrics to json | 20260506 20:12:19.030 | INFO | Serialization done, swapping tmp file now... | 20260506 20:12:19.043 | INFO | RTMetrics done | 20260506 20:12:19.142 | INFO | Finished a round of Baz IO: Wrote 1.985 GiB in 3.258 seconds | 20260506 20:12:19.142 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.33% / 0.00% / 65.67% | 20260506 20:12:19.142 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.784981% / 1.98682e-05% / 8.74201e-05% | 20260506 20:12:19.142 | INFO | Baz IO overall: | 20260506 20:12:19.142 | INFO | BytesWritten : 2180574549956 | 20260506 20:12:19.142 | INFO | NumEvents : 1053648618596 | 20260506 20:12:20.906 | INFO | Analyzing chunk frames = [4128768,4129280) | 20260506 20:12:20.906 | INFO | MemInfo: { | 20260506 20:12:20.906 | INFO | "gpu" : " 17 GB", | 20260506 20:12:20.906 | INFO | "huge" : "", | 20260506 20:12:20.906 | INFO | "huge_shared" : " 27 GB", | 20260506 20:12:20.906 | INFO | "malloc" : " 14 GB", | 20260506 20:12:20.906 | INFO | "pinned" : " 29 GB" | 20260506 20:12:20.906 | INFO | } | 20260506 20:12:20.906 | INFO | Perf report for the last 5.13687 seconds: | 20260506 20:12:20.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:12:20.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:20.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:20.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:20.907 | INFO | ANALYSIS : 62.7%, 2.9, 95.8ms, 0 | 20260506 20:12:20.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:20.907 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 20:12:20.907 | INFO | PRE_HQ : 60.2%, 1.8, 57.6ms, 0 | 20260506 20:12:20.907 | INFO | BAZWRITER : 63.1%, 4.7, 152.7ms, 0 | 20260506 20:12:20.907 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 20:12:20.907 | INFO | Uploaded 36GiB to GPU in 2.07401s (17.3576GiB/s) | 20260506 20:12:20.907 | INFO | Downloaded 38GiB to GPU in 2.2179s (17.1333GiB/s) | 20260506 20:12:26.043 | INFO | Analyzing chunk frames = [4129280,4129792) | 20260506 20:12:26.043 | INFO | MemInfo: { | 20260506 20:12:26.043 | INFO | "gpu" : " 17 GB", | 20260506 20:12:26.043 | INFO | "huge" : "", | 20260506 20:12:26.043 | INFO | "huge_shared" : " 27 GB", | 20260506 20:12:26.043 | INFO | "malloc" : " 14 GB", | 20260506 20:12:26.043 | INFO | "pinned" : " 29 GB" | 20260506 20:12:26.043 | INFO | } | 20260506 20:12:26.043 | INFO | Perf report for the last 5.13701 seconds: | 20260506 20:12:26.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:12:26.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:26.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:26.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:26.043 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:12:26.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:26.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:26.044 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:12:26.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:26.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:26.044 | INFO | Uploaded 36GiB to GPU in 1.99774s (18.0204GiB/s) | 20260506 20:12:26.044 | INFO | Downloaded 34GiB to GPU in 1.91998s (17.7086GiB/s) | 20260506 20:12:31.181 | INFO | Analyzing chunk frames = [4129792,4130304) | 20260506 20:12:31.181 | INFO | MemInfo: { | 20260506 20:12:31.181 | INFO | "gpu" : " 17 GB", | 20260506 20:12:31.181 | INFO | "huge" : "", | 20260506 20:12:31.181 | INFO | "huge_shared" : " 27 GB", | 20260506 20:12:31.181 | INFO | "malloc" : " 14 GB", | 20260506 20:12:31.181 | INFO | "pinned" : " 29 GB" | 20260506 20:12:31.181 | INFO | } | 20260506 20:12:31.181 | INFO | Perf report for the last 5.13778 seconds: | 20260506 20:12:31.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:12:31.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:31.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:31.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:31.181 | INFO | ANALYSIS : 61.0%, 3.0, 94.3ms, 0 | 20260506 20:12:31.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:31.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:31.181 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:12:31.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:31.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:31.182 | INFO | Uploaded 36GiB to GPU in 1.98171s (18.1661GiB/s) | 20260506 20:12:31.182 | INFO | Downloaded 34GiB to GPU in 1.91605s (17.7448GiB/s) | 20260506 20:12:36.326 | INFO | Analyzing chunk frames = [4130304,4130816) | 20260506 20:12:36.326 | INFO | MemInfo: { | 20260506 20:12:36.326 | INFO | "gpu" : " 17 GB", | 20260506 20:12:36.326 | INFO | "huge" : "", | 20260506 20:12:36.326 | INFO | "huge_shared" : " 27 GB", | 20260506 20:12:36.326 | INFO | "malloc" : " 14 GB", | 20260506 20:12:36.326 | INFO | "pinned" : " 29 GB" | 20260506 20:12:36.326 | INFO | } | 20260506 20:12:36.327 | INFO | Perf report for the last 5.1455 seconds: | 20260506 20:12:36.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:12:36.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:36.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:36.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:36.327 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 20:12:36.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:36.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:36.327 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:12:36.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:36.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:36.327 | INFO | Uploaded 36GiB to GPU in 1.9355s (18.5998GiB/s) | 20260506 20:12:36.327 | INFO | Downloaded 34GiB to GPU in 1.86797s (18.2016GiB/s) | 20260506 20:12:41.467 | INFO | Analyzing chunk frames = [4130816,4131328) | 20260506 20:12:41.467 | INFO | MemInfo: { | 20260506 20:12:41.467 | INFO | "gpu" : " 17 GB", | 20260506 20:12:41.467 | INFO | "huge" : "", | 20260506 20:12:41.467 | INFO | "huge_shared" : " 27 GB", | 20260506 20:12:41.467 | INFO | "malloc" : " 14 GB", | 20260506 20:12:41.467 | INFO | "pinned" : " 29 GB" | 20260506 20:12:41.467 | INFO | } | 20260506 20:12:41.467 | INFO | Perf report for the last 5.14035 seconds: | 20260506 20:12:41.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:12:41.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:41.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:41.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:41.467 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 20:12:41.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:41.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:41.467 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:12:41.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:41.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:41.467 | INFO | Uploaded 36GiB to GPU in 2.00383s (17.9656GiB/s) | 20260506 20:12:41.467 | INFO | Downloaded 34GiB to GPU in 1.91475s (17.7569GiB/s) | 20260506 20:12:46.612 | INFO | Analyzing chunk frames = [4131328,4131840) | 20260506 20:12:46.612 | INFO | MemInfo: { | 20260506 20:12:46.612 | INFO | "gpu" : " 17 GB", | 20260506 20:12:46.612 | INFO | "huge" : "", | 20260506 20:12:46.612 | INFO | "huge_shared" : " 27 GB", | 20260506 20:12:46.612 | INFO | "malloc" : " 14 GB", | 20260506 20:12:46.612 | INFO | "pinned" : " 29 GB" | 20260506 20:12:46.612 | INFO | } | 20260506 20:12:46.612 | INFO | Perf report for the last 5.1453 seconds: | 20260506 20:12:46.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:12:46.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:46.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:46.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:46.612 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 20:12:46.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:46.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:46.612 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 20:12:46.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:46.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:46.613 | INFO | Uploaded 36GiB to GPU in 2.01648s (17.8529GiB/s) | 20260506 20:12:46.613 | INFO | Downloaded 34GiB to GPU in 1.95405s (17.3998GiB/s) | 20260506 20:12:51.747 | INFO | Analyzing chunk frames = [4131840,4132352) | 20260506 20:12:51.747 | INFO | MemInfo: { | 20260506 20:12:51.747 | INFO | "gpu" : " 17 GB", | 20260506 20:12:51.747 | INFO | "huge" : "", | 20260506 20:12:51.747 | INFO | "huge_shared" : " 27 GB", | 20260506 20:12:51.747 | INFO | "malloc" : " 14 GB", | 20260506 20:12:51.747 | INFO | "pinned" : " 29 GB" | 20260506 20:12:51.747 | INFO | } | 20260506 20:12:51.747 | INFO | Perf report for the last 5.13489 seconds: | 20260506 20:12:51.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:12:51.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:51.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:51.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:51.747 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 20:12:51.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:51.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:51.748 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 20:12:51.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:51.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:51.748 | INFO | Uploaded 36GiB to GPU in 2.01405s (17.8745GiB/s) | 20260506 20:12:51.748 | INFO | Downloaded 34GiB to GPU in 1.97858s (17.184GiB/s) | 20260506 20:12:56.884 | INFO | Analyzing chunk frames = [4132352,4132864) | 20260506 20:12:56.884 | INFO | MemInfo: { | 20260506 20:12:56.884 | INFO | "gpu" : " 17 GB", | 20260506 20:12:56.884 | INFO | "huge" : "", | 20260506 20:12:56.884 | INFO | "huge_shared" : " 27 GB", | 20260506 20:12:56.884 | INFO | "malloc" : " 14 GB", | 20260506 20:12:56.885 | INFO | "pinned" : " 29 GB" | 20260506 20:12:56.885 | INFO | } | 20260506 20:12:56.885 | INFO | Perf report for the last 5.13734 seconds: | 20260506 20:12:56.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:12:56.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:56.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:56.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:56.885 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 20:12:56.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:56.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:56.885 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 20:12:56.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:56.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:12:56.885 | INFO | Uploaded 36GiB to GPU in 2.01283s (17.8853GiB/s) | 20260506 20:12:56.885 | INFO | Downloaded 34GiB to GPU in 1.99374s (17.0534GiB/s) | 20260506 20:12:56.999 | INFO | Starting a round of Baz IO | 20260506 20:13:00.130 | INFO | Populating RTMetrics report | 20260506 20:13:00.130 | INFO | RTMetrics report populated | 20260506 20:13:00.130 | INFO | Opening tmp RTMetrics json file | 20260506 20:13:00.133 | INFO | Serializing RTMetrics to json | 20260506 20:13:00.180 | INFO | Serialization done, swapping tmp file now... | 20260506 20:13:00.220 | INFO | RTMetrics done | 20260506 20:13:00.280 | INFO | Finished a round of Baz IO: Wrote 1.99 GiB in 3.28 seconds | 20260506 20:13:00.280 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.33% / 0.00% / 65.67% | 20260506 20:13:00.280 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.783225% / 7.94729e-06% / 7.54992e-05% | 20260506 20:13:00.280 | INFO | Baz IO overall: | 20260506 20:13:00.280 | INFO | BytesWritten : 2182711018778 | 20260506 20:13:00.280 | INFO | NumEvents : 1054673297938 | 20260506 20:13:02.021 | INFO | Analyzing chunk frames = [4132864,4133376) | 20260506 20:13:02.021 | INFO | MemInfo: { | 20260506 20:13:02.021 | INFO | "gpu" : " 17 GB", | 20260506 20:13:02.021 | INFO | "huge" : "", | 20260506 20:13:02.021 | INFO | "huge_shared" : " 27 GB", | 20260506 20:13:02.021 | INFO | "malloc" : " 14 GB", | 20260506 20:13:02.021 | INFO | "pinned" : " 29 GB" | 20260506 20:13:02.021 | INFO | } | 20260506 20:13:02.021 | INFO | Perf report for the last 5.13681 seconds: | 20260506 20:13:02.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:13:02.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:02.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:02.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:02.022 | INFO | ANALYSIS : 62.6%, 2.9, 95.1ms, 0 | 20260506 20:13:02.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:02.022 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260506 20:13:02.022 | INFO | PRE_HQ : 60.3%, 1.9, 58.7ms, 0 | 20260506 20:13:02.022 | INFO | BAZWRITER : 63.5%, 4.6, 149.5ms, 0 | 20260506 20:13:02.022 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 20:13:02.022 | INFO | Uploaded 36GiB to GPU in 2.02141s (17.8094GiB/s) | 20260506 20:13:02.022 | INFO | Downloaded 38GiB to GPU in 2.20017s (17.2714GiB/s) | 20260506 20:13:07.157 | INFO | Analyzing chunk frames = [4133376,4133888) | 20260506 20:13:07.157 | INFO | MemInfo: { | 20260506 20:13:07.157 | INFO | "gpu" : " 17 GB", | 20260506 20:13:07.157 | INFO | "huge" : "", | 20260506 20:13:07.157 | INFO | "huge_shared" : " 27 GB", | 20260506 20:13:07.157 | INFO | "malloc" : " 14 GB", | 20260506 20:13:07.157 | INFO | "pinned" : " 29 GB" | 20260506 20:13:07.157 | INFO | } | 20260506 20:13:07.158 | INFO | Perf report for the last 5.13617 seconds: | 20260506 20:13:07.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:13:07.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:07.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:07.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:07.158 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 20:13:07.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:07.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:07.158 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:13:07.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:07.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:07.158 | INFO | Uploaded 36GiB to GPU in 2.01964s (17.8249GiB/s) | 20260506 20:13:07.158 | INFO | Downloaded 34GiB to GPU in 1.96185s (17.3306GiB/s) | 20260506 20:13:12.300 | INFO | Analyzing chunk frames = [4133888,4134400) | 20260506 20:13:12.300 | INFO | MemInfo: { | 20260506 20:13:12.300 | INFO | "gpu" : " 17 GB", | 20260506 20:13:12.301 | INFO | "huge" : "", | 20260506 20:13:12.301 | INFO | "huge_shared" : " 27 GB", | 20260506 20:13:12.301 | INFO | "malloc" : " 14 GB", | 20260506 20:13:12.301 | INFO | "pinned" : " 29 GB" | 20260506 20:13:12.301 | INFO | } | 20260506 20:13:12.301 | INFO | Perf report for the last 5.14305 seconds: | 20260506 20:13:12.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:13:12.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:12.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:12.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:12.301 | INFO | ANALYSIS : 60.8%, 3.0, 94.3ms, 0 | 20260506 20:13:12.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:12.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:12.301 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:13:12.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:12.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:12.301 | INFO | Uploaded 36GiB to GPU in 1.97136s (18.2615GiB/s) | 20260506 20:13:12.301 | INFO | Downloaded 34GiB to GPU in 1.93212s (17.5972GiB/s) | 20260506 20:13:17.447 | INFO | Analyzing chunk frames = [4134400,4134912) | 20260506 20:13:17.447 | INFO | MemInfo: { | 20260506 20:13:17.447 | INFO | "gpu" : " 17 GB", | 20260506 20:13:17.447 | INFO | "huge" : "", | 20260506 20:13:17.447 | INFO | "huge_shared" : " 27 GB", | 20260506 20:13:17.447 | INFO | "malloc" : " 14 GB", | 20260506 20:13:17.447 | INFO | "pinned" : " 29 GB" | 20260506 20:13:17.447 | INFO | } | 20260506 20:13:17.447 | INFO | Perf report for the last 5.1468 seconds: | 20260506 20:13:17.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:13:17.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:17.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:17.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:17.448 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260506 20:13:17.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:17.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:17.448 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:13:17.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:17.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:17.448 | INFO | Uploaded 36GiB to GPU in 1.9356s (18.5989GiB/s) | 20260506 20:13:17.448 | INFO | Downloaded 34GiB to GPU in 1.8745s (18.1381GiB/s) | 20260506 20:13:22.582 | INFO | Analyzing chunk frames = [4134912,4135424) | 20260506 20:13:22.582 | INFO | MemInfo: { | 20260506 20:13:22.582 | INFO | "gpu" : " 17 GB", | 20260506 20:13:22.582 | INFO | "huge" : "", | 20260506 20:13:22.582 | INFO | "huge_shared" : " 27 GB", | 20260506 20:13:22.582 | INFO | "malloc" : " 14 GB", | 20260506 20:13:22.582 | INFO | "pinned" : " 29 GB" | 20260506 20:13:22.582 | INFO | } | 20260506 20:13:22.582 | INFO | Perf report for the last 5.13448 seconds: | 20260506 20:13:22.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:13:22.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:22.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:22.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:22.582 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 20:13:22.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:22.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:22.582 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 20:13:22.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:22.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:22.582 | INFO | Uploaded 36GiB to GPU in 1.99323s (18.0612GiB/s) | 20260506 20:13:22.582 | INFO | Downloaded 34GiB to GPU in 1.92224s (17.6877GiB/s) | 20260506 20:13:27.725 | INFO | Analyzing chunk frames = [4135424,4135936) | 20260506 20:13:27.725 | INFO | MemInfo: { | 20260506 20:13:27.725 | INFO | "gpu" : " 17 GB", | 20260506 20:13:27.725 | INFO | "huge" : "", | 20260506 20:13:27.725 | INFO | "huge_shared" : " 27 GB", | 20260506 20:13:27.725 | INFO | "malloc" : " 14 GB", | 20260506 20:13:27.725 | INFO | "pinned" : " 29 GB" | 20260506 20:13:27.725 | INFO | } | 20260506 20:13:27.725 | INFO | Perf report for the last 5.14265 seconds: | 20260506 20:13:27.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:13:27.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:27.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:27.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:27.725 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:13:27.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:27.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:27.725 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:13:27.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:27.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:27.725 | INFO | Uploaded 36GiB to GPU in 2.01236s (17.8895GiB/s) | 20260506 20:13:27.725 | INFO | Downloaded 34GiB to GPU in 1.96103s (17.3378GiB/s) | 20260506 20:13:32.859 | INFO | Analyzing chunk frames = [4135936,4136448) | 20260506 20:13:32.859 | INFO | MemInfo: { | 20260506 20:13:32.859 | INFO | "gpu" : " 17 GB", | 20260506 20:13:32.859 | INFO | "huge" : "", | 20260506 20:13:32.859 | INFO | "huge_shared" : " 27 GB", | 20260506 20:13:32.859 | INFO | "malloc" : " 14 GB", | 20260506 20:13:32.859 | INFO | "pinned" : " 29 GB" | 20260506 20:13:32.859 | INFO | } | 20260506 20:13:32.859 | INFO | Perf report for the last 5.13447 seconds: | 20260506 20:13:32.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:13:32.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:32.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:32.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:32.859 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 20:13:32.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:32.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:32.859 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 20:13:32.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:32.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:32.859 | INFO | Uploaded 36GiB to GPU in 1.95561s (18.4086GiB/s) | 20260506 20:13:32.859 | INFO | Downloaded 34GiB to GPU in 1.99211s (17.0673GiB/s) | 20260506 20:13:37.996 | INFO | Analyzing chunk frames = [4136448,4136960) | 20260506 20:13:37.996 | INFO | MemInfo: { | 20260506 20:13:37.996 | INFO | "gpu" : " 17 GB", | 20260506 20:13:37.996 | INFO | "huge" : "", | 20260506 20:13:37.996 | INFO | "huge_shared" : " 27 GB", | 20260506 20:13:37.996 | INFO | "malloc" : " 14 GB", | 20260506 20:13:37.996 | INFO | "pinned" : " 29 GB" | 20260506 20:13:37.996 | INFO | } | 20260506 20:13:37.996 | INFO | Perf report for the last 5.13733 seconds: | 20260506 20:13:37.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:13:37.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:37.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:37.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:37.997 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 20:13:37.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:37.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:37.997 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 20:13:37.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:37.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:37.997 | INFO | Uploaded 36GiB to GPU in 2.05164s (17.5469GiB/s) | 20260506 20:13:37.997 | INFO | Downloaded 34GiB to GPU in 1.97644s (17.2027GiB/s) | 20260506 20:13:37.997 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:13:37.997 | INFO | Total mem usage: 18 GiB | 20260506 20:13:37.997 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:13:37.997 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -16.9%) | 20260506 20:13:37.997 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:13:37.997 | INFO | High Water Breakdown: | 20260506 20:13:37.997 | INFO | 61.3% : 9.45 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:13:37.997 | INFO | 38.6% : 5.94 GiB ( 88.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:13:37.997 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:13:37.997 | INFO | ------------------------------------------ | 20260506 20:13:37.997 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:13:37.997 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:13:37.997 | INFO | Total mem usage: 26.5 GiB | 20260506 20:13:37.997 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:13:37.997 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:13:37.997 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:13:37.997 | INFO | High Water Breakdown: | 20260506 20:13:37.997 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:13:37.997 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:13:37.997 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:13:37.997 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:13:37.997 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:13:37.997 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:13:37.997 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:13:37.997 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:13:37.997 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:13:37.997 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:13:37.997 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:13:37.997 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:13:37.997 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:13:37.997 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:13:37.997 | INFO | ------------------------------------------ | 20260506 20:13:37.998 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:13:37.998 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:13:37.998 | INFO | Total mem usage: 25.6 GiB | 20260506 20:13:37.998 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:13:37.998 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:13:37.998 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:13:37.998 | INFO | High Water Breakdown: | 20260506 20:13:37.998 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:13:37.998 | INFO | ------------------------------------------ | 20260506 20:13:37.998 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:13:37.998 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:13:37.998 | INFO | Total mem usage: 15.7 GiB | 20260506 20:13:37.998 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:13:37.998 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:13:37.998 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 20:13:37.998 | INFO | High Water Breakdown: | 20260506 20:13:37.998 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:13:37.998 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:13:37.998 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:13:37.998 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:13:37.998 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:13:37.998 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:13:37.998 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:13:37.998 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:13:37.998 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:13:37.998 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:13:37.998 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:13:37.998 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:13:37.998 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:13:37.998 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:13:37.998 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:13:37.998 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:13:37.998 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:13:37.998 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:13:37.998 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:13:37.998 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:13:37.998 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:13:37.998 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:13:37.998 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:13:37.998 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:13:37.998 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:13:37.998 | INFO | ------------------------------------------ | 20260506 20:13:37.998 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:13:37.998 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:13:37.998 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:13:37.998 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:13:37.998 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.59e+01, 1.34e+01 / 2.73e+00 | 20260506 20:13:37.998 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.03e-01, 9.98e-03 / 7.62e-03 | 20260506 20:13:37.998 | INFO | PerfReport: 11.63%, Baseline -- 3.63e+00 / 4.43e+00, 5.58e-01 / 3.14e-01 | 20260506 20:13:37.998 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.81e-01 / 1.39e-01 | 20260506 20:13:37.998 | INFO | PerfReport: 16.03%, DME -- 4.00e+01 / 4.47e+01, 4.19e+00 / 3.38e+00 | 20260506 20:13:37.998 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.43e+00 / 1.86e+00 | 20260506 20:13:37.998 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.23e+00 / 2.76e+00, 3.75e-01 / 2.37e-01 | 20260506 20:13:37.998 | INFO | PerfReport: 3.32%, Metrics -- 1.04e+00 / 1.36e+00, 2.76e-01 / 1.78e-01 | 20260506 20:13:37.998 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.57e-01 / 4.08e-01, 5.71e-02 / 2.88e-02 | 20260506 20:13:37.998 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:13:37.998 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:13:37.998 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:13:37.998 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 20:13:37.998 | INFO | PerfReport: 21.50%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 20:13:37.998 | INFO | PerfReport: 21.22%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 20:13:37.998 | INFO | PerfReport: 32.21%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 20:13:37.998 | INFO | PerfReport: 1.32%, LockUpload -- 1.28e+00 / 0.00e+00, 5.08e+00 / 0.00e+00 | 20260506 20:13:37.998 | INFO | PerfReport: 1.16%, LockDownload -- 1.13e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 20:13:37.998 | INFO | PerfReport: 22.58%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 20:13:37.998 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260506 20:13:38.115 | INFO | Starting a round of Baz IO | 20260506 20:13:41.245 | INFO | Populating RTMetrics report | 20260506 20:13:41.246 | INFO | RTMetrics report populated | 20260506 20:13:41.246 | INFO | Opening tmp RTMetrics json file | 20260506 20:13:41.260 | INFO | Serializing RTMetrics to json | 20260506 20:13:41.264 | INFO | Serialization done, swapping tmp file now... | 20260506 20:13:41.267 | INFO | RTMetrics done | 20260506 20:13:41.718 | INFO | Finished a round of Baz IO: Wrote 1.987 GiB in 3.603 seconds | 20260506 20:13:41.718 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.32% / 0.00% / 65.67% | 20260506 20:13:41.718 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.781441% / 3.57628e-05% / 7.94729e-05% | 20260506 20:13:41.718 | INFO | Baz IO overall: | 20260506 20:13:41.719 | INFO | BytesWritten : 2184844621642 | 20260506 20:13:41.719 | INFO | NumEvents : 1055696523873 | 20260506 20:13:43.134 | INFO | Analyzing chunk frames = [4136960,4137472) | 20260506 20:13:43.134 | INFO | MemInfo: { | 20260506 20:13:43.134 | INFO | "gpu" : " 17 GB", | 20260506 20:13:43.134 | INFO | "huge" : "", | 20260506 20:13:43.134 | INFO | "huge_shared" : " 27 GB", | 20260506 20:13:43.134 | INFO | "malloc" : " 14 GB", | 20260506 20:13:43.134 | INFO | "pinned" : " 29 GB" | 20260506 20:13:43.134 | INFO | } | 20260506 20:13:43.134 | INFO | Perf report for the last 5.13793 seconds: | 20260506 20:13:43.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:13:43.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:43.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:43.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:43.134 | INFO | ANALYSIS : 62.7%, 2.9, 94.8ms, 0 | 20260506 20:13:43.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:43.134 | INFO | RT_METRICS : 45.4%, 1.0, 23.8ms, 0 | 20260506 20:13:43.134 | INFO | PRE_HQ : 60.4%, 1.9, 60.4ms, 0 | 20260506 20:13:43.134 | INFO | BAZWRITER : 69.5%, 4.8, 171.8ms, 0 | 20260506 20:13:43.134 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 20:13:43.135 | INFO | Uploaded 36GiB to GPU in 2.06558s (17.4285GiB/s) | 20260506 20:13:43.135 | INFO | Downloaded 38GiB to GPU in 2.22057s (17.1127GiB/s) | 20260506 20:13:48.273 | INFO | Analyzing chunk frames = [4137472,4137984) | 20260506 20:13:48.273 | INFO | MemInfo: { | 20260506 20:13:48.273 | INFO | "gpu" : " 17 GB", | 20260506 20:13:48.273 | INFO | "huge" : "", | 20260506 20:13:48.273 | INFO | "huge_shared" : " 27 GB", | 20260506 20:13:48.273 | INFO | "malloc" : " 14 GB", | 20260506 20:13:48.273 | INFO | "pinned" : " 29 GB" | 20260506 20:13:48.273 | INFO | } | 20260506 20:13:48.273 | INFO | Perf report for the last 5.13903 seconds: | 20260506 20:13:48.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:13:48.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:48.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:48.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:48.273 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 20:13:48.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:48.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:48.273 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:13:48.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:48.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:48.274 | INFO | Uploaded 36GiB to GPU in 2.01786s (17.8407GiB/s) | 20260506 20:13:48.274 | INFO | Downloaded 34GiB to GPU in 1.9383s (17.5411GiB/s) | 20260506 20:13:53.420 | INFO | Analyzing chunk frames = [4137984,4138496) | 20260506 20:13:53.420 | INFO | MemInfo: { | 20260506 20:13:53.420 | INFO | "gpu" : " 17 GB", | 20260506 20:13:53.420 | INFO | "huge" : "", | 20260506 20:13:53.420 | INFO | "huge_shared" : " 27 GB", | 20260506 20:13:53.420 | INFO | "malloc" : " 14 GB", | 20260506 20:13:53.420 | INFO | "pinned" : " 29 GB" | 20260506 20:13:53.420 | INFO | } | 20260506 20:13:53.420 | INFO | Perf report for the last 5.14656 seconds: | 20260506 20:13:53.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:13:53.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:53.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:53.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:53.420 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 20:13:53.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:53.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:53.420 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:13:53.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:53.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:53.420 | INFO | Uploaded 36GiB to GPU in 1.97875s (18.1933GiB/s) | 20260506 20:13:53.420 | INFO | Downloaded 34GiB to GPU in 1.91721s (17.7341GiB/s) | 20260506 20:13:58.562 | INFO | Analyzing chunk frames = [4138496,4139008) | 20260506 20:13:58.562 | INFO | MemInfo: { | 20260506 20:13:58.562 | INFO | "gpu" : " 17 GB", | 20260506 20:13:58.562 | INFO | "huge" : "", | 20260506 20:13:58.562 | INFO | "huge_shared" : " 27 GB", | 20260506 20:13:58.562 | INFO | "malloc" : " 14 GB", | 20260506 20:13:58.562 | INFO | "pinned" : " 29 GB" | 20260506 20:13:58.562 | INFO | } | 20260506 20:13:58.562 | INFO | Perf report for the last 5.14199 seconds: | 20260506 20:13:58.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:13:58.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:58.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:58.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:58.562 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 20:13:58.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:58.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:58.562 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:13:58.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:58.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:13:58.562 | INFO | Uploaded 36GiB to GPU in 1.98391s (18.146GiB/s) | 20260506 20:13:58.562 | INFO | Downloaded 34GiB to GPU in 1.87759s (18.1083GiB/s) | 20260506 20:14:03.701 | INFO | Analyzing chunk frames = [4139008,4139520) | 20260506 20:14:03.701 | INFO | MemInfo: { | 20260506 20:14:03.701 | INFO | "gpu" : " 17 GB", | 20260506 20:14:03.701 | INFO | "huge" : "", | 20260506 20:14:03.701 | INFO | "huge_shared" : " 27 GB", | 20260506 20:14:03.701 | INFO | "malloc" : " 14 GB", | 20260506 20:14:03.701 | INFO | "pinned" : " 29 GB" | 20260506 20:14:03.701 | INFO | } | 20260506 20:14:03.701 | INFO | Perf report for the last 5.13885 seconds: | 20260506 20:14:03.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:14:03.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:03.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:03.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:03.701 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 20:14:03.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:03.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:03.701 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:14:03.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:03.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:03.701 | INFO | Uploaded 36GiB to GPU in 1.99712s (18.026GiB/s) | 20260506 20:14:03.701 | INFO | Downloaded 34GiB to GPU in 1.92557s (17.6571GiB/s) | 20260506 20:14:08.835 | INFO | Analyzing chunk frames = [4139520,4140032) | 20260506 20:14:08.835 | INFO | MemInfo: { | 20260506 20:14:08.835 | INFO | "gpu" : " 17 GB", | 20260506 20:14:08.835 | INFO | "huge" : "", | 20260506 20:14:08.835 | INFO | "huge_shared" : " 27 GB", | 20260506 20:14:08.835 | INFO | "malloc" : " 14 GB", | 20260506 20:14:08.835 | INFO | "pinned" : " 29 GB" | 20260506 20:14:08.835 | INFO | } | 20260506 20:14:08.835 | INFO | Perf report for the last 5.13438 seconds: | 20260506 20:14:08.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:14:08.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:08.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:08.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:08.835 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:14:08.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:08.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:08.835 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:14:08.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:08.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:08.835 | INFO | Uploaded 36GiB to GPU in 2.0069s (17.9381GiB/s) | 20260506 20:14:08.836 | INFO | Downloaded 34GiB to GPU in 1.94175s (17.51GiB/s) | 20260506 20:14:13.971 | INFO | Analyzing chunk frames = [4140032,4140544) | 20260506 20:14:13.971 | INFO | MemInfo: { | 20260506 20:14:13.971 | INFO | "gpu" : " 17 GB", | 20260506 20:14:13.971 | INFO | "huge" : "", | 20260506 20:14:13.971 | INFO | "huge_shared" : " 27 GB", | 20260506 20:14:13.971 | INFO | "malloc" : " 14 GB", | 20260506 20:14:13.971 | INFO | "pinned" : " 29 GB" | 20260506 20:14:13.971 | INFO | } | 20260506 20:14:13.971 | INFO | Perf report for the last 5.13551 seconds: | 20260506 20:14:13.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:14:13.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:13.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:13.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:13.971 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260506 20:14:13.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:13.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:13.971 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 20:14:13.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:13.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:13.971 | INFO | Uploaded 36GiB to GPU in 1.97709s (18.2085GiB/s) | 20260506 20:14:13.971 | INFO | Downloaded 34GiB to GPU in 2.00469s (16.9602GiB/s) | 20260506 20:14:19.111 | INFO | Analyzing chunk frames = [4140544,4141056) | 20260506 20:14:19.111 | INFO | MemInfo: { | 20260506 20:14:19.111 | INFO | "gpu" : " 17 GB", | 20260506 20:14:19.111 | INFO | "huge" : "", | 20260506 20:14:19.111 | INFO | "huge_shared" : " 27 GB", | 20260506 20:14:19.111 | INFO | "malloc" : " 14 GB", | 20260506 20:14:19.111 | INFO | "pinned" : " 29 GB" | 20260506 20:14:19.111 | INFO | } | 20260506 20:14:19.111 | INFO | Perf report for the last 5.14003 seconds: | 20260506 20:14:19.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:14:19.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:19.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:19.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:19.111 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 20:14:19.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:19.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:19.111 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 20:14:19.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:19.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:19.111 | INFO | Uploaded 36GiB to GPU in 2.0151s (17.8651GiB/s) | 20260506 20:14:19.111 | INFO | Downloaded 34GiB to GPU in 1.99222s (17.0664GiB/s) | 20260506 20:14:19.230 | INFO | Starting a round of Baz IO | 20260506 20:14:22.390 | INFO | Populating RTMetrics report | 20260506 20:14:22.390 | INFO | RTMetrics report populated | 20260506 20:14:22.390 | INFO | Opening tmp RTMetrics json file | 20260506 20:14:22.438 | INFO | Serializing RTMetrics to json | 20260506 20:14:22.539 | INFO | Serialization done, swapping tmp file now... | 20260506 20:14:22.601 | INFO | RTMetrics done | 20260506 20:14:22.635 | INFO | Finished a round of Baz IO: Wrote 1.986 GiB in 3.405 seconds | 20260506 20:14:22.635 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.32% / 0.00% / 65.68% | 20260506 20:14:22.635 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.779541% / 7.94729e-06% / 3.57628e-05% | 20260506 20:14:22.635 | INFO | Baz IO overall: | 20260506 20:14:22.635 | INFO | BytesWritten : 2186976794687 | 20260506 20:14:22.635 | INFO | NumEvents : 1056719642330 | 20260506 20:14:24.253 | INFO | Analyzing chunk frames = [4141056,4141568) | 20260506 20:14:24.253 | INFO | MemInfo: { | 20260506 20:14:24.253 | INFO | "gpu" : " 17 GB", | 20260506 20:14:24.253 | INFO | "huge" : "", | 20260506 20:14:24.253 | INFO | "huge_shared" : " 27 GB", | 20260506 20:14:24.253 | INFO | "malloc" : " 14 GB", | 20260506 20:14:24.253 | INFO | "pinned" : " 29 GB" | 20260506 20:14:24.253 | INFO | } | 20260506 20:14:24.253 | INFO | Perf report for the last 5.1424 seconds: | 20260506 20:14:24.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:14:24.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:24.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:24.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:24.253 | INFO | ANALYSIS : 62.9%, 2.8, 93.9ms, 0 | 20260506 20:14:24.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:24.253 | INFO | RT_METRICS : 50.5%, 1.0, 26.5ms, 0 | 20260506 20:14:24.253 | INFO | PRE_HQ : 60.9%, 2.0, 61.8ms, 0 | 20260506 20:14:24.253 | INFO | BAZWRITER : 65.7%, 4.8, 162.9ms, 0 | 20260506 20:14:24.253 | INFO | BAZ_SAMPLING : 3.1%, 1.0, 1.6ms, 0 | 20260506 20:14:24.253 | INFO | Uploaded 36GiB to GPU in 2.04938s (17.5663GiB/s) | 20260506 20:14:24.254 | INFO | Downloaded 38GiB to GPU in 2.18695s (17.3758GiB/s) | 20260506 20:14:29.392 | INFO | Analyzing chunk frames = [4141568,4142080) | 20260506 20:14:29.392 | INFO | MemInfo: { | 20260506 20:14:29.392 | INFO | "gpu" : " 17 GB", | 20260506 20:14:29.392 | INFO | "huge" : "", | 20260506 20:14:29.392 | INFO | "huge_shared" : " 27 GB", | 20260506 20:14:29.392 | INFO | "malloc" : " 14 GB", | 20260506 20:14:29.392 | INFO | "pinned" : " 29 GB" | 20260506 20:14:29.392 | INFO | } | 20260506 20:14:29.392 | INFO | Perf report for the last 5.13876 seconds: | 20260506 20:14:29.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:14:29.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:29.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:29.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:29.392 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:14:29.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:29.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:29.392 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 20:14:29.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:29.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:29.392 | INFO | Uploaded 36GiB to GPU in 1.9913s (18.0787GiB/s) | 20260506 20:14:29.392 | INFO | Downloaded 34GiB to GPU in 1.92591s (17.654GiB/s) | 20260506 20:14:34.532 | INFO | Analyzing chunk frames = [4142080,4142592) | 20260506 20:14:34.532 | INFO | MemInfo: { | 20260506 20:14:34.532 | INFO | "gpu" : " 17 GB", | 20260506 20:14:34.532 | INFO | "huge" : "", | 20260506 20:14:34.532 | INFO | "huge_shared" : " 27 GB", | 20260506 20:14:34.532 | INFO | "malloc" : " 14 GB", | 20260506 20:14:34.532 | INFO | "pinned" : " 29 GB" | 20260506 20:14:34.532 | INFO | } | 20260506 20:14:34.532 | INFO | Perf report for the last 5.14018 seconds: | 20260506 20:14:34.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:14:34.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:34.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:34.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:34.532 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 20:14:34.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:34.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:34.532 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:14:34.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:34.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:34.532 | INFO | Uploaded 36GiB to GPU in 1.96389s (18.3309GiB/s) | 20260506 20:14:34.532 | INFO | Downloaded 34GiB to GPU in 1.92266s (17.6838GiB/s) | 20260506 20:14:39.674 | INFO | Analyzing chunk frames = [4142592,4143104) | 20260506 20:14:39.674 | INFO | MemInfo: { | 20260506 20:14:39.674 | INFO | "gpu" : " 17 GB", | 20260506 20:14:39.674 | INFO | "huge" : "", | 20260506 20:14:39.674 | INFO | "huge_shared" : " 27 GB", | 20260506 20:14:39.674 | INFO | "malloc" : " 14 GB", | 20260506 20:14:39.674 | INFO | "pinned" : " 29 GB" | 20260506 20:14:39.674 | INFO | } | 20260506 20:14:39.674 | INFO | Perf report for the last 5.1418 seconds: | 20260506 20:14:39.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:14:39.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:39.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:39.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:39.674 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 20:14:39.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:39.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:39.674 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:14:39.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:39.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:39.674 | INFO | Uploaded 36GiB to GPU in 1.92769s (18.6752GiB/s) | 20260506 20:14:39.674 | INFO | Downloaded 34GiB to GPU in 1.87596s (18.124GiB/s) | 20260506 20:14:44.812 | INFO | Analyzing chunk frames = [4143104,4143616) | 20260506 20:14:44.812 | INFO | MemInfo: { | 20260506 20:14:44.812 | INFO | "gpu" : " 17 GB", | 20260506 20:14:44.812 | INFO | "huge" : "", | 20260506 20:14:44.812 | INFO | "huge_shared" : " 27 GB", | 20260506 20:14:44.812 | INFO | "malloc" : " 14 GB", | 20260506 20:14:44.812 | INFO | "pinned" : " 29 GB" | 20260506 20:14:44.812 | INFO | } | 20260506 20:14:44.812 | INFO | Perf report for the last 5.13803 seconds: | 20260506 20:14:44.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:14:44.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:44.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:44.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:44.812 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 20:14:44.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:44.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:44.812 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:14:44.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:44.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:44.812 | INFO | Uploaded 36GiB to GPU in 1.97481s (18.2296GiB/s) | 20260506 20:14:44.812 | INFO | Downloaded 34GiB to GPU in 1.9255s (17.6578GiB/s) | 20260506 20:14:49.948 | INFO | Analyzing chunk frames = [4143616,4144128) | 20260506 20:14:49.948 | INFO | MemInfo: { | 20260506 20:14:49.948 | INFO | "gpu" : " 17 GB", | 20260506 20:14:49.948 | INFO | "huge" : "", | 20260506 20:14:49.948 | INFO | "huge_shared" : " 27 GB", | 20260506 20:14:49.948 | INFO | "malloc" : " 14 GB", | 20260506 20:14:49.948 | INFO | "pinned" : " 29 GB" | 20260506 20:14:49.948 | INFO | } | 20260506 20:14:49.948 | INFO | Perf report for the last 5.13595 seconds: | 20260506 20:14:49.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:14:49.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:49.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:49.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:49.948 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 20:14:49.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:49.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:49.948 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:14:49.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:49.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:49.948 | INFO | Uploaded 36GiB to GPU in 2.00811s (17.9273GiB/s) | 20260506 20:14:49.948 | INFO | Downloaded 34GiB to GPU in 1.95214s (17.4167GiB/s) | 20260506 20:14:55.087 | INFO | Analyzing chunk frames = [4144128,4144640) | 20260506 20:14:55.087 | INFO | MemInfo: { | 20260506 20:14:55.087 | INFO | "gpu" : " 17 GB", | 20260506 20:14:55.087 | INFO | "huge" : "", | 20260506 20:14:55.087 | INFO | "huge_shared" : " 27 GB", | 20260506 20:14:55.087 | INFO | "malloc" : " 14 GB", | 20260506 20:14:55.087 | INFO | "pinned" : " 29 GB" | 20260506 20:14:55.087 | INFO | } | 20260506 20:14:55.087 | INFO | Perf report for the last 5.13914 seconds: | 20260506 20:14:55.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:14:55.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:55.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:55.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:55.087 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 20:14:55.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:55.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:55.087 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 20:14:55.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:55.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:14:55.087 | INFO | Uploaded 36GiB to GPU in 1.97034s (18.271GiB/s) | 20260506 20:14:55.087 | INFO | Downloaded 34GiB to GPU in 2.00781s (16.9338GiB/s) | 20260506 20:15:00.224 | INFO | Analyzing chunk frames = [4144640,4145152) | 20260506 20:15:00.224 | INFO | MemInfo: { | 20260506 20:15:00.224 | INFO | "gpu" : " 17 GB", | 20260506 20:15:00.224 | INFO | "huge" : "", | 20260506 20:15:00.224 | INFO | "huge_shared" : " 27 GB", | 20260506 20:15:00.224 | INFO | "malloc" : " 14 GB", | 20260506 20:15:00.224 | INFO | "pinned" : " 29 GB" | 20260506 20:15:00.224 | INFO | } | 20260506 20:15:00.224 | INFO | Perf report for the last 5.13728 seconds: | 20260506 20:15:00.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:15:00.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:00.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:00.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:00.225 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 20:15:00.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:00.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:00.225 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 20:15:00.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:00.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:00.225 | INFO | Uploaded 36GiB to GPU in 2.0259s (17.7699GiB/s) | 20260506 20:15:00.225 | INFO | Downloaded 34GiB to GPU in 2.00319s (16.9729GiB/s) | 20260506 20:15:00.225 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:15:00.225 | INFO | Total mem usage: 18 GiB | 20260506 20:15:00.225 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:15:00.225 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -11%) | 20260506 20:15:00.225 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:15:00.225 | INFO | High Water Breakdown: | 20260506 20:15:00.225 | INFO | 62% : 10.1 GiB ( 88.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:15:00.225 | INFO | 37.8% : 6.14 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:15:00.225 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:15:00.225 | INFO | ------------------------------------------ | 20260506 20:15:00.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:15:00.225 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:15:00.225 | INFO | Total mem usage: 26.5 GiB | 20260506 20:15:00.225 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:15:00.225 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:15:00.225 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:15:00.225 | INFO | High Water Breakdown: | 20260506 20:15:00.225 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:15:00.225 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:15:00.225 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:15:00.225 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:15:00.225 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:15:00.225 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:15:00.225 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:15:00.225 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:15:00.225 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:15:00.225 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:15:00.225 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:15:00.225 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:15:00.225 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:15:00.225 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:15:00.225 | INFO | ------------------------------------------ | 20260506 20:15:00.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:15:00.225 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:15:00.225 | INFO | Total mem usage: 25.6 GiB | 20260506 20:15:00.225 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:15:00.225 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:15:00.225 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:15:00.225 | INFO | High Water Breakdown: | 20260506 20:15:00.225 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:15:00.225 | INFO | ------------------------------------------ | 20260506 20:15:00.225 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:15:00.225 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:15:00.226 | INFO | Total mem usage: 15.7 GiB | 20260506 20:15:00.226 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:15:00.226 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:15:00.226 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:15:00.226 | INFO | High Water Breakdown: | 20260506 20:15:00.226 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:15:00.226 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:15:00.226 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:15:00.226 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:15:00.226 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:15:00.226 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:15:00.226 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:15:00.226 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:15:00.226 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:15:00.226 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:15:00.226 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:15:00.226 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:15:00.226 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:15:00.226 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:15:00.226 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:15:00.226 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:15:00.226 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:15:00.226 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:15:00.226 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:15:00.226 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:15:00.226 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:15:00.226 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:15:00.226 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:15:00.226 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:15:00.226 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:15:00.226 | INFO | ------------------------------------------ | 20260506 20:15:00.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:15:00.226 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:15:00.226 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:15:00.226 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:15:00.226 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.62e+01, 1.35e+01 / 3.04e+00 | 20260506 20:15:00.226 | INFO | PerfReport: 0.29%, AggregateStats -- 9.18e-02 / 1.03e-01, 1.01e-02 / 6.04e-03 | 20260506 20:15:00.226 | INFO | PerfReport: 11.63%, Baseline -- 3.63e+00 / 4.35e+00, 5.57e-01 / 3.57e-01 | 20260506 20:15:00.226 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.84e-01 / 1.22e-01 | 20260506 20:15:00.226 | INFO | PerfReport: 16.09%, DME -- 4.02e+01 / 4.52e+01, 4.60e+00 / 3.66e+00 | 20260506 20:15:00.226 | INFO | PerfReport: 57.10%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.41e+00 / 2.02e+00 | 20260506 20:15:00.226 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.24e+00 / 2.76e+00, 3.86e-01 / 2.44e-01 | 20260506 20:15:00.226 | INFO | PerfReport: 3.31%, Metrics -- 1.03e+00 / 1.37e+00, 2.73e-01 / 1.62e-01 | 20260506 20:15:00.226 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.56e-01 / 4.03e-01, 5.64e-02 / 2.40e-02 | 20260506 20:15:00.226 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:15:00.226 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:15:00.226 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:15:00.226 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 20:15:00.226 | INFO | PerfReport: 21.51%, Upload -- 2.08e+01 / 0.00e+00, 6.90e+00 / 0.00e+00 | 20260506 20:15:00.226 | INFO | PerfReport: 21.28%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 20:15:00.226 | INFO | PerfReport: 32.28%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 20:15:00.226 | INFO | PerfReport: 1.28%, LockUpload -- 1.24e+00 / 0.00e+00, 5.01e+00 / 0.00e+00 | 20260506 20:15:00.226 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260506 20:15:00.226 | INFO | PerfReport: 22.41%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 20:15:00.226 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260506 20:15:00.336 | INFO | Starting a round of Baz IO | 20260506 20:15:03.468 | INFO | Populating RTMetrics report | 20260506 20:15:03.468 | INFO | RTMetrics report populated | 20260506 20:15:03.468 | INFO | Opening tmp RTMetrics json file | 20260506 20:15:03.489 | INFO | Serializing RTMetrics to json | 20260506 20:15:03.496 | INFO | Serialization done, swapping tmp file now... | 20260506 20:15:03.499 | INFO | RTMetrics done | 20260506 20:15:03.840 | INFO | Finished a round of Baz IO: Wrote 1.986 GiB in 3.503 seconds | 20260506 20:15:03.840 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.31% / 0.00% / 65.68% | 20260506 20:15:03.840 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.78131% / 0% / 5.96046e-05% | 20260506 20:15:03.840 | INFO | Baz IO overall: | 20260506 20:15:03.840 | INFO | BytesWritten : 2189109534695 | 20260506 20:15:03.840 | INFO | NumEvents : 1057743516316 | 20260506 20:15:05.367 | INFO | Analyzing chunk frames = [4145152,4145664) | 20260506 20:15:05.367 | INFO | MemInfo: { | 20260506 20:15:05.367 | INFO | "gpu" : " 17 GB", | 20260506 20:15:05.367 | INFO | "huge" : "", | 20260506 20:15:05.367 | INFO | "huge_shared" : " 27 GB", | 20260506 20:15:05.367 | INFO | "malloc" : " 14 GB", | 20260506 20:15:05.367 | INFO | "pinned" : " 29 GB" | 20260506 20:15:05.367 | INFO | } | 20260506 20:15:05.367 | INFO | Perf report for the last 5.14289 seconds: | 20260506 20:15:05.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:15:05.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:05.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:05.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:05.367 | INFO | ANALYSIS : 62.5%, 2.9, 95.2ms, 0 | 20260506 20:15:05.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:05.368 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260506 20:15:05.368 | INFO | PRE_HQ : 59.6%, 1.8, 57.2ms, 0 | 20260506 20:15:05.368 | INFO | BAZWRITER : 67.8%, 4.8, 167.0ms, 0 | 20260506 20:15:05.368 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 20:15:05.368 | INFO | Uploaded 36GiB to GPU in 2.02257s (17.7991GiB/s) | 20260506 20:15:05.368 | INFO | Downloaded 38GiB to GPU in 2.19266s (17.3305GiB/s) | 20260506 20:15:10.511 | INFO | Analyzing chunk frames = [4145664,4146176) | 20260506 20:15:10.511 | INFO | MemInfo: { | 20260506 20:15:10.511 | INFO | "gpu" : " 17 GB", | 20260506 20:15:10.511 | INFO | "huge" : "", | 20260506 20:15:10.511 | INFO | "huge_shared" : " 27 GB", | 20260506 20:15:10.511 | INFO | "malloc" : " 14 GB", | 20260506 20:15:10.511 | INFO | "pinned" : " 29 GB" | 20260506 20:15:10.511 | INFO | } | 20260506 20:15:10.511 | INFO | Perf report for the last 5.14408 seconds: | 20260506 20:15:10.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:15:10.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:10.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:10.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:10.512 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 20:15:10.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:10.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:10.512 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:15:10.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:10.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:10.512 | INFO | Uploaded 36GiB to GPU in 1.99015s (18.0891GiB/s) | 20260506 20:15:10.512 | INFO | Downloaded 34GiB to GPU in 1.92596s (17.6536GiB/s) | 20260506 20:15:15.649 | INFO | Analyzing chunk frames = [4146176,4146688) | 20260506 20:15:15.649 | INFO | MemInfo: { | 20260506 20:15:15.649 | INFO | "gpu" : " 17 GB", | 20260506 20:15:15.649 | INFO | "huge" : "", | 20260506 20:15:15.649 | INFO | "huge_shared" : " 27 GB", | 20260506 20:15:15.649 | INFO | "malloc" : " 14 GB", | 20260506 20:15:15.649 | INFO | "pinned" : " 29 GB" | 20260506 20:15:15.649 | INFO | } | 20260506 20:15:15.649 | INFO | Perf report for the last 5.13795 seconds: | 20260506 20:15:15.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:15:15.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:15.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:15.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:15.649 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 20:15:15.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:15.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:15.649 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:15:15.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:15.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:15.650 | INFO | Uploaded 36GiB to GPU in 1.96388s (18.3311GiB/s) | 20260506 20:15:15.650 | INFO | Downloaded 34GiB to GPU in 1.92264s (17.684GiB/s) | 20260506 20:15:20.792 | INFO | Analyzing chunk frames = [4146688,4147200) | 20260506 20:15:20.792 | INFO | MemInfo: { | 20260506 20:15:20.792 | INFO | "gpu" : " 17 GB", | 20260506 20:15:20.792 | INFO | "huge" : "", | 20260506 20:15:20.792 | INFO | "huge_shared" : " 27 GB", | 20260506 20:15:20.792 | INFO | "malloc" : " 14 GB", | 20260506 20:15:20.792 | INFO | "pinned" : " 29 GB" | 20260506 20:15:20.792 | INFO | } | 20260506 20:15:20.792 | INFO | Perf report for the last 5.14274 seconds: | 20260506 20:15:20.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:15:20.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:20.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:20.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:20.792 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:15:20.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:20.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:20.792 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:15:20.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:20.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:20.793 | INFO | Uploaded 36GiB to GPU in 1.93477s (18.6069GiB/s) | 20260506 20:15:20.793 | INFO | Downloaded 34GiB to GPU in 1.87675s (18.1164GiB/s) | 20260506 20:15:25.924 | INFO | Analyzing chunk frames = [4147200,4147712) | 20260506 20:15:25.924 | INFO | MemInfo: { | 20260506 20:15:25.924 | INFO | "gpu" : " 17 GB", | 20260506 20:15:25.924 | INFO | "huge" : "", | 20260506 20:15:25.924 | INFO | "huge_shared" : " 27 GB", | 20260506 20:15:25.924 | INFO | "malloc" : " 14 GB", | 20260506 20:15:25.924 | INFO | "pinned" : " 29 GB" | 20260506 20:15:25.924 | INFO | } | 20260506 20:15:25.924 | INFO | Perf report for the last 5.13181 seconds: | 20260506 20:15:25.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:15:25.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:25.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:25.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:25.924 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 20:15:25.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:25.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:25.924 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:15:25.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:25.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:25.924 | INFO | Uploaded 36GiB to GPU in 2.01043s (17.9067GiB/s) | 20260506 20:15:25.924 | INFO | Downloaded 34GiB to GPU in 1.92216s (17.6885GiB/s) | 20260506 20:15:31.065 | INFO | Analyzing chunk frames = [4147712,4148224) | 20260506 20:15:31.065 | INFO | MemInfo: { | 20260506 20:15:31.065 | INFO | "gpu" : " 17 GB", | 20260506 20:15:31.065 | INFO | "huge" : "", | 20260506 20:15:31.065 | INFO | "huge_shared" : " 27 GB", | 20260506 20:15:31.065 | INFO | "malloc" : " 14 GB", | 20260506 20:15:31.065 | INFO | "pinned" : " 29 GB" | 20260506 20:15:31.065 | INFO | } | 20260506 20:15:31.065 | INFO | Perf report for the last 5.14138 seconds: | 20260506 20:15:31.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:15:31.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:31.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:31.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:31.065 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 20:15:31.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:31.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:31.065 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:15:31.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:31.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:31.066 | INFO | Uploaded 36GiB to GPU in 2.00367s (17.967GiB/s) | 20260506 20:15:31.066 | INFO | Downloaded 34GiB to GPU in 1.95929s (17.3532GiB/s) | 20260506 20:15:36.201 | INFO | Analyzing chunk frames = [4148224,4148736) | 20260506 20:15:36.201 | INFO | MemInfo: { | 20260506 20:15:36.201 | INFO | "gpu" : " 17 GB", | 20260506 20:15:36.201 | INFO | "huge" : "", | 20260506 20:15:36.201 | INFO | "huge_shared" : " 27 GB", | 20260506 20:15:36.201 | INFO | "malloc" : " 14 GB", | 20260506 20:15:36.201 | INFO | "pinned" : " 29 GB" | 20260506 20:15:36.201 | INFO | } | 20260506 20:15:36.201 | INFO | Perf report for the last 5.13554 seconds: | 20260506 20:15:36.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:15:36.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:36.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:36.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:36.201 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 20:15:36.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:36.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:36.201 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 20:15:36.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:36.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:36.201 | INFO | Uploaded 36GiB to GPU in 1.94103s (18.5468GiB/s) | 20260506 20:15:36.201 | INFO | Downloaded 34GiB to GPU in 1.98695s (17.1117GiB/s) | 20260506 20:15:41.341 | INFO | Analyzing chunk frames = [4148736,4149248) | 20260506 20:15:41.341 | INFO | MemInfo: { | 20260506 20:15:41.341 | INFO | "gpu" : " 17 GB", | 20260506 20:15:41.341 | INFO | "huge" : "", | 20260506 20:15:41.341 | INFO | "huge_shared" : " 27 GB", | 20260506 20:15:41.341 | INFO | "malloc" : " 14 GB", | 20260506 20:15:41.341 | INFO | "pinned" : " 29 GB" | 20260506 20:15:41.341 | INFO | } | 20260506 20:15:41.341 | INFO | Perf report for the last 5.14035 seconds: | 20260506 20:15:41.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:15:41.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:41.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:41.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:41.341 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:15:41.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:41.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:41.342 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:15:41.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:41.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:41.342 | INFO | Uploaded 36GiB to GPU in 2.00527s (17.9527GiB/s) | 20260506 20:15:41.342 | INFO | Downloaded 34GiB to GPU in 1.98093s (17.1637GiB/s) | 20260506 20:15:41.442 | INFO | Starting a round of Baz IO | 20260506 20:15:44.575 | INFO | Populating RTMetrics report | 20260506 20:15:44.575 | INFO | RTMetrics report populated | 20260506 20:15:44.575 | INFO | Opening tmp RTMetrics json file | 20260506 20:15:44.596 | INFO | Serializing RTMetrics to json | 20260506 20:15:44.625 | INFO | Serialization done, swapping tmp file now... | 20260506 20:15:44.627 | INFO | RTMetrics done | 20260506 20:15:45.316 | INFO | Finished a round of Baz IO: Wrote 1.984 GiB in 3.873 seconds | 20260506 20:15:45.316 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.31% / 0.00% / 65.69% | 20260506 20:15:45.316 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.777694% / 7.94729e-06% / 5.5631e-05% | 20260506 20:15:45.316 | INFO | Baz IO overall: | 20260506 20:15:45.316 | INFO | BytesWritten : 2191240263572 | 20260506 20:15:45.316 | INFO | NumEvents : 1058766153041 | 20260506 20:15:46.481 | INFO | Analyzing chunk frames = [4149248,4149760) | 20260506 20:15:46.481 | INFO | MemInfo: { | 20260506 20:15:46.481 | INFO | "gpu" : " 17 GB", | 20260506 20:15:46.481 | INFO | "huge" : "", | 20260506 20:15:46.481 | INFO | "huge_shared" : " 27 GB", | 20260506 20:15:46.481 | INFO | "malloc" : " 14 GB", | 20260506 20:15:46.481 | INFO | "pinned" : " 29 GB" | 20260506 20:15:46.481 | INFO | } | 20260506 20:15:46.481 | INFO | Perf report for the last 5.13978 seconds: | 20260506 20:15:46.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:15:46.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:46.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:46.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:46.481 | INFO | ANALYSIS : 62.3%, 2.9, 95.0ms, 0 | 20260506 20:15:46.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:46.481 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 20:15:46.481 | INFO | PRE_HQ : 60.1%, 1.8, 57.3ms, 0 | 20260506 20:15:46.481 | INFO | BAZWRITER : 75.0%, 4.7, 180.0ms, 0 | 20260506 20:15:46.481 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 20:15:46.481 | INFO | Uploaded 36GiB to GPU in 2.04407s (17.6119GiB/s) | 20260506 20:15:46.481 | INFO | Downloaded 38GiB to GPU in 2.18063s (17.4261GiB/s) | 20260506 20:15:51.618 | INFO | Analyzing chunk frames = [4149760,4150272) | 20260506 20:15:51.618 | INFO | MemInfo: { | 20260506 20:15:51.618 | INFO | "gpu" : " 17 GB", | 20260506 20:15:51.618 | INFO | "huge" : "", | 20260506 20:15:51.618 | INFO | "huge_shared" : " 27 GB", | 20260506 20:15:51.618 | INFO | "malloc" : " 14 GB", | 20260506 20:15:51.618 | INFO | "pinned" : " 29 GB" | 20260506 20:15:51.618 | INFO | } | 20260506 20:15:51.618 | INFO | Perf report for the last 5.13716 seconds: | 20260506 20:15:51.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:15:51.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:51.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:51.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:51.618 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:15:51.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:51.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:51.618 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 20:15:51.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:51.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:51.618 | INFO | Uploaded 36GiB to GPU in 1.99498s (18.0453GiB/s) | 20260506 20:15:51.619 | INFO | Downloaded 34GiB to GPU in 1.91903s (17.7173GiB/s) | 20260506 20:15:56.761 | INFO | Analyzing chunk frames = [4150272,4150784) | 20260506 20:15:56.761 | INFO | MemInfo: { | 20260506 20:15:56.761 | INFO | "gpu" : " 17 GB", | 20260506 20:15:56.761 | INFO | "huge" : "", | 20260506 20:15:56.762 | INFO | "huge_shared" : " 27 GB", | 20260506 20:15:56.762 | INFO | "malloc" : " 14 GB", | 20260506 20:15:56.762 | INFO | "pinned" : " 29 GB" | 20260506 20:15:56.762 | INFO | } | 20260506 20:15:56.762 | INFO | Perf report for the last 5.14331 seconds: | 20260506 20:15:56.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:15:56.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:56.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:56.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:56.762 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260506 20:15:56.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:56.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:56.762 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:15:56.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:56.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:15:56.762 | INFO | Uploaded 36GiB to GPU in 1.94064s (18.5506GiB/s) | 20260506 20:15:56.762 | INFO | Downloaded 34GiB to GPU in 1.87613s (18.1224GiB/s) | 20260506 20:16:01.900 | INFO | Analyzing chunk frames = [4150784,4151296) | 20260506 20:16:01.900 | INFO | MemInfo: { | 20260506 20:16:01.900 | INFO | "gpu" : " 17 GB", | 20260506 20:16:01.900 | INFO | "huge" : "", | 20260506 20:16:01.901 | INFO | "huge_shared" : " 27 GB", | 20260506 20:16:01.901 | INFO | "malloc" : " 14 GB", | 20260506 20:16:01.901 | INFO | "pinned" : " 29 GB" | 20260506 20:16:01.901 | INFO | } | 20260506 20:16:01.901 | INFO | Perf report for the last 5.13908 seconds: | 20260506 20:16:01.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:16:01.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:01.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:01.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:01.901 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 20:16:01.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:01.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:01.901 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:16:01.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:01.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:01.901 | INFO | Uploaded 36GiB to GPU in 1.9298s (18.6548GiB/s) | 20260506 20:16:01.901 | INFO | Downloaded 34GiB to GPU in 1.8695s (18.1866GiB/s) | 20260506 20:16:07.040 | INFO | Analyzing chunk frames = [4151296,4151808) | 20260506 20:16:07.040 | INFO | MemInfo: { | 20260506 20:16:07.040 | INFO | "gpu" : " 17 GB", | 20260506 20:16:07.040 | INFO | "huge" : "", | 20260506 20:16:07.040 | INFO | "huge_shared" : " 27 GB", | 20260506 20:16:07.040 | INFO | "malloc" : " 14 GB", | 20260506 20:16:07.040 | INFO | "pinned" : " 29 GB" | 20260506 20:16:07.040 | INFO | } | 20260506 20:16:07.040 | INFO | Perf report for the last 5.13929 seconds: | 20260506 20:16:07.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:16:07.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:07.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:07.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:07.040 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 20:16:07.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:07.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:07.040 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:16:07.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:07.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:07.040 | INFO | Uploaded 36GiB to GPU in 1.98951s (18.0949GiB/s) | 20260506 20:16:07.040 | INFO | Downloaded 34GiB to GPU in 1.92132s (17.6961GiB/s) | 20260506 20:16:12.179 | INFO | Analyzing chunk frames = [4151808,4152320) | 20260506 20:16:12.179 | INFO | MemInfo: { | 20260506 20:16:12.179 | INFO | "gpu" : " 17 GB", | 20260506 20:16:12.179 | INFO | "huge" : "", | 20260506 20:16:12.179 | INFO | "huge_shared" : " 27 GB", | 20260506 20:16:12.179 | INFO | "malloc" : " 14 GB", | 20260506 20:16:12.179 | INFO | "pinned" : " 29 GB" | 20260506 20:16:12.179 | INFO | } | 20260506 20:16:12.179 | INFO | Perf report for the last 5.13914 seconds: | 20260506 20:16:12.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:16:12.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:12.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:12.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:12.179 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 20:16:12.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:12.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:12.179 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 20:16:12.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:12.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:12.179 | INFO | Uploaded 36GiB to GPU in 2.0007s (17.9937GiB/s) | 20260506 20:16:12.179 | INFO | Downloaded 34GiB to GPU in 1.9455s (17.4762GiB/s) | 20260506 20:16:17.317 | INFO | Analyzing chunk frames = [4152320,4152832) | 20260506 20:16:17.317 | INFO | MemInfo: { | 20260506 20:16:17.317 | INFO | "gpu" : " 17 GB", | 20260506 20:16:17.317 | INFO | "huge" : "", | 20260506 20:16:17.317 | INFO | "huge_shared" : " 27 GB", | 20260506 20:16:17.317 | INFO | "malloc" : " 14 GB", | 20260506 20:16:17.317 | INFO | "pinned" : " 29 GB" | 20260506 20:16:17.317 | INFO | } | 20260506 20:16:17.317 | INFO | Perf report for the last 5.13754 seconds: | 20260506 20:16:17.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:16:17.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:17.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:17.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:17.317 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 20:16:17.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:17.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:17.317 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:16:17.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:17.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:17.317 | INFO | Uploaded 36GiB to GPU in 1.95922s (18.3747GiB/s) | 20260506 20:16:17.317 | INFO | Downloaded 34GiB to GPU in 1.99162s (17.0715GiB/s) | 20260506 20:16:22.457 | INFO | Analyzing chunk frames = [4152832,4153344) | 20260506 20:16:22.457 | INFO | MemInfo: { | 20260506 20:16:22.457 | INFO | "gpu" : " 17 GB", | 20260506 20:16:22.457 | INFO | "huge" : "", | 20260506 20:16:22.457 | INFO | "huge_shared" : " 27 GB", | 20260506 20:16:22.457 | INFO | "malloc" : " 14 GB", | 20260506 20:16:22.457 | INFO | "pinned" : " 29 GB" | 20260506 20:16:22.457 | INFO | } | 20260506 20:16:22.457 | INFO | Perf report for the last 5.14022 seconds: | 20260506 20:16:22.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:16:22.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:22.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:22.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:22.457 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:16:22.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:22.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:22.457 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:16:22.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:22.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:22.457 | INFO | Uploaded 36GiB to GPU in 1.98439s (18.1416GiB/s) | 20260506 20:16:22.457 | INFO | Downloaded 34GiB to GPU in 1.99412s (17.0501GiB/s) | 20260506 20:16:22.457 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:16:22.457 | INFO | Total mem usage: 18 GiB | 20260506 20:16:22.457 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:16:22.457 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -11.8%) | 20260506 20:16:22.457 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:16:22.457 | INFO | High Water Breakdown: | 20260506 20:16:22.457 | INFO | 61.9% : 9.97 GiB ( 87.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:16:22.457 | INFO | 37.9% : 6.11 GiB ( 91.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:16:22.457 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:16:22.457 | INFO | ------------------------------------------ | 20260506 20:16:22.457 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:16:22.457 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:16:22.457 | INFO | Total mem usage: 26.5 GiB | 20260506 20:16:22.457 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:16:22.457 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:16:22.457 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:16:22.457 | INFO | High Water Breakdown: | 20260506 20:16:22.457 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:16:22.457 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:16:22.458 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:16:22.458 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:16:22.458 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:16:22.458 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:16:22.458 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:16:22.458 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:16:22.458 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:16:22.458 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:16:22.458 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:16:22.458 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:16:22.458 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:16:22.458 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:16:22.458 | INFO | ------------------------------------------ | 20260506 20:16:22.458 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:16:22.458 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:16:22.458 | INFO | Total mem usage: 25.6 GiB | 20260506 20:16:22.458 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:16:22.458 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 20:16:22.458 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 20:16:22.458 | INFO | High Water Breakdown: | 20260506 20:16:22.458 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 20:16:22.458 | INFO | ------------------------------------------ | 20260506 20:16:22.458 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:16:22.458 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:16:22.458 | INFO | Total mem usage: 15.7 GiB | 20260506 20:16:22.458 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:16:22.458 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:16:22.458 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 20:16:22.458 | INFO | High Water Breakdown: | 20260506 20:16:22.458 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:16:22.458 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:16:22.458 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:16:22.458 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:16:22.458 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:16:22.458 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:16:22.458 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:16:22.458 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:16:22.458 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:16:22.458 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:16:22.458 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:16:22.458 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:16:22.458 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:16:22.458 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:16:22.458 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:16:22.458 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:16:22.458 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:16:22.458 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:16:22.458 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:16:22.458 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:16:22.458 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:16:22.458 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:16:22.458 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:16:22.458 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:16:22.458 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:16:22.458 | INFO | ------------------------------------------ | 20260506 20:16:22.458 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:16:22.458 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:16:22.458 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:16:22.458 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:16:22.458 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.34e+01 / 2.55e+00 | 20260506 20:16:22.458 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.73e-03 / 5.61e-03 | 20260506 20:16:22.458 | INFO | PerfReport: 11.64%, Baseline -- 3.63e+00 / 4.38e+00, 5.58e-01 / 3.08e-01 | 20260506 20:16:22.458 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.44e+00, 1.84e-01 / 1.33e-01 | 20260506 20:16:22.458 | INFO | PerfReport: 16.03%, DME -- 4.00e+01 / 4.52e+01, 4.30e+00 / 2.97e+00 | 20260506 20:16:22.458 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.78e+01 / 2.00e+01, 1.38e+00 / 1.74e+00 | 20260506 20:16:22.458 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.23e+00 / 2.85e+00, 3.80e-01 / 1.56e-01 | 20260506 20:16:22.458 | INFO | PerfReport: 3.32%, Metrics -- 1.03e+00 / 1.42e+00, 2.71e-01 / 1.35e-01 | 20260506 20:16:22.458 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.55e-01 / 4.01e-01, 5.62e-02 / 1.36e-02 | 20260506 20:16:22.458 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:16:22.458 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:16:22.458 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:16:22.458 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.31e+01 / 0.00e+00 | 20260506 20:16:22.458 | INFO | PerfReport: 21.38%, Upload -- 2.06e+01 / 0.00e+00, 6.77e+00 / 0.00e+00 | 20260506 20:16:22.458 | INFO | PerfReport: 21.21%, Download -- 2.05e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 20:16:22.458 | INFO | PerfReport: 32.27%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 20:16:22.458 | INFO | PerfReport: 1.20%, LockUpload -- 1.16e+00 / 0.00e+00, 4.73e+00 / 0.00e+00 | 20260506 20:16:22.458 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260506 20:16:22.458 | INFO | PerfReport: 22.71%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 20:16:22.458 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.98e-02 / 0.00e+00 | 20260506 20:16:22.556 | INFO | Starting a round of Baz IO | 20260506 20:16:25.698 | INFO | Populating RTMetrics report | 20260506 20:16:25.698 | INFO | RTMetrics report populated | 20260506 20:16:25.698 | INFO | Opening tmp RTMetrics json file | 20260506 20:16:25.772 | INFO | Serializing RTMetrics to json | 20260506 20:16:25.778 | INFO | Serialization done, swapping tmp file now... | 20260506 20:16:25.779 | INFO | RTMetrics done | 20260506 20:16:25.998 | INFO | Finished a round of Baz IO: Wrote 1.983 GiB in 3.441 seconds | 20260506 20:16:25.998 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.31% / 0.00% / 65.69% | 20260506 20:16:25.998 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.775135% / 3.97364e-06% / 6.75519e-05% | 20260506 20:16:25.998 | INFO | Baz IO overall: | 20260506 20:16:25.998 | INFO | BytesWritten : 2193369650263 | 20260506 20:16:25.998 | INFO | NumEvents : 1059788122016 | 20260506 20:16:27.595 | INFO | Analyzing chunk frames = [4153344,4153856) | 20260506 20:16:27.595 | INFO | MemInfo: { | 20260506 20:16:27.595 | INFO | "gpu" : " 17 GB", | 20260506 20:16:27.595 | INFO | "huge" : "", | 20260506 20:16:27.595 | INFO | "huge_shared" : " 27 GB", | 20260506 20:16:27.595 | INFO | "malloc" : " 14 GB", | 20260506 20:16:27.595 | INFO | "pinned" : " 29 GB" | 20260506 20:16:27.595 | INFO | } | 20260506 20:16:27.595 | INFO | Perf report for the last 5.1378 seconds: | 20260506 20:16:27.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:16:27.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:27.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:27.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:27.595 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 20:16:27.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:27.595 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 20:16:27.595 | INFO | PRE_HQ : 60.2%, 1.8, 57.3ms, 0 | 20260506 20:16:27.595 | INFO | BAZWRITER : 66.6%, 4.8, 163.8ms, 0 | 20260506 20:16:27.595 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 20:16:27.595 | INFO | Uploaded 36GiB to GPU in 2.02575s (17.7712GiB/s) | 20260506 20:16:27.595 | INFO | Downloaded 38GiB to GPU in 2.19969s (17.2751GiB/s) | 20260506 20:16:32.739 | INFO | Analyzing chunk frames = [4153856,4154368) | 20260506 20:16:32.739 | INFO | MemInfo: { | 20260506 20:16:32.739 | INFO | "gpu" : " 17 GB", | 20260506 20:16:32.739 | INFO | "huge" : "", | 20260506 20:16:32.739 | INFO | "huge_shared" : " 27 GB", | 20260506 20:16:32.739 | INFO | "malloc" : " 14 GB", | 20260506 20:16:32.739 | INFO | "pinned" : " 29 GB" | 20260506 20:16:32.739 | INFO | } | 20260506 20:16:32.739 | INFO | Perf report for the last 5.14416 seconds: | 20260506 20:16:32.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:16:32.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:32.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:32.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:32.739 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 20:16:32.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:32.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:32.739 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:16:32.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:32.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:32.739 | INFO | Uploaded 36GiB to GPU in 1.98566s (18.13GiB/s) | 20260506 20:16:32.739 | INFO | Downloaded 34GiB to GPU in 1.92764s (17.6382GiB/s) | 20260506 20:16:37.882 | INFO | Analyzing chunk frames = [4154368,4154880) | 20260506 20:16:37.882 | INFO | MemInfo: { | 20260506 20:16:37.882 | INFO | "gpu" : " 17 GB", | 20260506 20:16:37.882 | INFO | "huge" : "", | 20260506 20:16:37.882 | INFO | "huge_shared" : " 27 GB", | 20260506 20:16:37.882 | INFO | "malloc" : " 14 GB", | 20260506 20:16:37.882 | INFO | "pinned" : " 29 GB" | 20260506 20:16:37.882 | INFO | } | 20260506 20:16:37.882 | INFO | Perf report for the last 5.1435 seconds: | 20260506 20:16:37.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:16:37.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:37.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:37.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:37.883 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 20:16:37.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:37.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:37.883 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:16:37.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:37.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:37.883 | INFO | Uploaded 36GiB to GPU in 1.96297s (18.3396GiB/s) | 20260506 20:16:37.883 | INFO | Downloaded 34GiB to GPU in 1.92432s (17.6686GiB/s) | 20260506 20:16:43.017 | INFO | Analyzing chunk frames = [4154880,4155392) | 20260506 20:16:43.017 | INFO | MemInfo: { | 20260506 20:16:43.017 | INFO | "gpu" : " 17 GB", | 20260506 20:16:43.017 | INFO | "huge" : "", | 20260506 20:16:43.017 | INFO | "huge_shared" : " 27 GB", | 20260506 20:16:43.017 | INFO | "malloc" : " 14 GB", | 20260506 20:16:43.017 | INFO | "pinned" : " 29 GB" | 20260506 20:16:43.017 | INFO | } | 20260506 20:16:43.017 | INFO | Perf report for the last 5.1343 seconds: | 20260506 20:16:43.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:16:43.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:43.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:43.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:43.017 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 20:16:43.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:43.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:43.017 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:16:43.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:43.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:43.017 | INFO | Uploaded 36GiB to GPU in 2.01424s (17.8727GiB/s) | 20260506 20:16:43.017 | INFO | Downloaded 34GiB to GPU in 1.90815s (17.8183GiB/s) | 20260506 20:16:48.154 | INFO | Analyzing chunk frames = [4155392,4155904) | 20260506 20:16:48.154 | INFO | MemInfo: { | 20260506 20:16:48.154 | INFO | "gpu" : " 17 GB", | 20260506 20:16:48.154 | INFO | "huge" : "", | 20260506 20:16:48.154 | INFO | "huge_shared" : " 27 GB", | 20260506 20:16:48.154 | INFO | "malloc" : " 14 GB", | 20260506 20:16:48.154 | INFO | "pinned" : " 29 GB" | 20260506 20:16:48.154 | INFO | } | 20260506 20:16:48.154 | INFO | Perf report for the last 5.13721 seconds: | 20260506 20:16:48.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:16:48.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:48.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:48.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:48.154 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 20:16:48.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:48.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:48.154 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 20:16:48.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:48.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:48.154 | INFO | Uploaded 36GiB to GPU in 1.98806s (18.1081GiB/s) | 20260506 20:16:48.154 | INFO | Downloaded 34GiB to GPU in 1.91774s (17.7292GiB/s) | 20260506 20:16:53.290 | INFO | Analyzing chunk frames = [4155904,4156416) | 20260506 20:16:53.290 | INFO | MemInfo: { | 20260506 20:16:53.290 | INFO | "gpu" : " 17 GB", | 20260506 20:16:53.290 | INFO | "huge" : "", | 20260506 20:16:53.290 | INFO | "huge_shared" : " 27 GB", | 20260506 20:16:53.290 | INFO | "malloc" : " 14 GB", | 20260506 20:16:53.290 | INFO | "pinned" : " 29 GB" | 20260506 20:16:53.290 | INFO | } | 20260506 20:16:53.291 | INFO | Perf report for the last 5.13652 seconds: | 20260506 20:16:53.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:16:53.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:53.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:53.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:53.291 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:16:53.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:53.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:53.291 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 20:16:53.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:53.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:53.291 | INFO | Uploaded 36GiB to GPU in 1.9842s (18.1434GiB/s) | 20260506 20:16:53.291 | INFO | Downloaded 34GiB to GPU in 1.95937s (17.3525GiB/s) | 20260506 20:16:58.427 | INFO | Analyzing chunk frames = [4156416,4156928) | 20260506 20:16:58.427 | INFO | MemInfo: { | 20260506 20:16:58.427 | INFO | "gpu" : " 17 GB", | 20260506 20:16:58.427 | INFO | "huge" : "", | 20260506 20:16:58.427 | INFO | "huge_shared" : " 27 GB", | 20260506 20:16:58.427 | INFO | "malloc" : " 14 GB", | 20260506 20:16:58.427 | INFO | "pinned" : " 29 GB" | 20260506 20:16:58.427 | INFO | } | 20260506 20:16:58.427 | INFO | Perf report for the last 5.13625 seconds: | 20260506 20:16:58.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:16:58.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:58.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:58.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:58.427 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 20:16:58.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:58.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:58.427 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:16:58.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:58.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:16:58.427 | INFO | Uploaded 36GiB to GPU in 1.95613s (18.4037GiB/s) | 20260506 20:16:58.427 | INFO | Downloaded 34GiB to GPU in 1.98784s (17.104GiB/s) | 20260506 20:17:03.569 | INFO | Analyzing chunk frames = [4156928,4157440) | 20260506 20:17:03.569 | INFO | MemInfo: { | 20260506 20:17:03.569 | INFO | "gpu" : " 17 GB", | 20260506 20:17:03.569 | INFO | "huge" : "", | 20260506 20:17:03.569 | INFO | "huge_shared" : " 27 GB", | 20260506 20:17:03.569 | INFO | "malloc" : " 14 GB", | 20260506 20:17:03.569 | INFO | "pinned" : " 29 GB" | 20260506 20:17:03.569 | INFO | } | 20260506 20:17:03.570 | INFO | Perf report for the last 5.14278 seconds: | 20260506 20:17:03.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:17:03.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:03.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:03.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:03.570 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 20:17:03.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:03.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:03.570 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 20:17:03.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:03.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:03.570 | INFO | Uploaded 36GiB to GPU in 1.99156s (18.0763GiB/s) | 20260506 20:17:03.570 | INFO | Downloaded 34GiB to GPU in 1.97942s (17.1768GiB/s) | 20260506 20:17:03.674 | INFO | Starting a round of Baz IO | 20260506 20:17:06.801 | INFO | Populating RTMetrics report | 20260506 20:17:06.802 | INFO | RTMetrics report populated | 20260506 20:17:06.802 | INFO | Opening tmp RTMetrics json file | 20260506 20:17:06.829 | INFO | Serializing RTMetrics to json | 20260506 20:17:06.898 | INFO | Serialization done, swapping tmp file now... | 20260506 20:17:06.906 | INFO | RTMetrics done | 20260506 20:17:07.307 | INFO | Finished a round of Baz IO: Wrote 1.983 GiB in 3.632 seconds | 20260506 20:17:07.307 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.30% / 0.00% / 65.69% | 20260506 20:17:07.307 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.774308% / 7.94729e-06% / 5.5631e-05% | 20260506 20:17:07.307 | INFO | Baz IO overall: | 20260506 20:17:07.307 | INFO | BytesWritten : 2195498540308 | 20260506 20:17:07.307 | INFO | NumEvents : 1060809573316 | 20260506 20:17:08.710 | INFO | Analyzing chunk frames = [4157440,4157952) | 20260506 20:17:08.710 | INFO | MemInfo: { | 20260506 20:17:08.710 | INFO | "gpu" : " 17 GB", | 20260506 20:17:08.710 | INFO | "huge" : "", | 20260506 20:17:08.710 | INFO | "huge_shared" : " 27 GB", | 20260506 20:17:08.710 | INFO | "malloc" : " 14 GB", | 20260506 20:17:08.710 | INFO | "pinned" : " 29 GB" | 20260506 20:17:08.710 | INFO | } | 20260506 20:17:08.710 | INFO | Perf report for the last 5.1405 seconds: | 20260506 20:17:08.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:17:08.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:08.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:08.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:08.710 | INFO | ANALYSIS : 62.3%, 2.9, 95.0ms, 0 | 20260506 20:17:08.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:08.710 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260506 20:17:08.710 | INFO | PRE_HQ : 60.0%, 1.8, 57.4ms, 0 | 20260506 20:17:08.710 | INFO | BAZWRITER : 70.4%, 4.8, 173.9ms, 0 | 20260506 20:17:08.710 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 20:17:08.710 | INFO | Uploaded 36GiB to GPU in 2.03621s (17.6799GiB/s) | 20260506 20:17:08.710 | INFO | Downloaded 38GiB to GPU in 2.18789s (17.3683GiB/s) | 20260506 20:17:13.854 | INFO | Analyzing chunk frames = [4157952,4158464) | 20260506 20:17:13.854 | INFO | MemInfo: { | 20260506 20:17:13.854 | INFO | "gpu" : " 17 GB", | 20260506 20:17:13.854 | INFO | "huge" : "", | 20260506 20:17:13.854 | INFO | "huge_shared" : " 27 GB", | 20260506 20:17:13.854 | INFO | "malloc" : " 14 GB", | 20260506 20:17:13.854 | INFO | "pinned" : " 29 GB" | 20260506 20:17:13.854 | INFO | } | 20260506 20:17:13.854 | INFO | Perf report for the last 5.1443 seconds: | 20260506 20:17:13.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:17:13.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:13.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:13.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:13.854 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:17:13.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:13.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:13.854 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:17:13.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:13.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:13.855 | INFO | Uploaded 36GiB to GPU in 2.02337s (17.7921GiB/s) | 20260506 20:17:13.855 | INFO | Downloaded 34GiB to GPU in 1.92928s (17.6232GiB/s) | 20260506 20:17:18.994 | INFO | Analyzing chunk frames = [4158464,4158976) | 20260506 20:17:18.994 | INFO | MemInfo: { | 20260506 20:17:18.994 | INFO | "gpu" : " 17 GB", | 20260506 20:17:18.995 | INFO | "huge" : "", | 20260506 20:17:18.995 | INFO | "huge_shared" : " 27 GB", | 20260506 20:17:18.995 | INFO | "malloc" : " 14 GB", | 20260506 20:17:18.995 | INFO | "pinned" : " 29 GB" | 20260506 20:17:18.995 | INFO | } | 20260506 20:17:18.995 | INFO | Perf report for the last 5.14025 seconds: | 20260506 20:17:18.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:17:18.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:18.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:18.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:18.995 | INFO | ANALYSIS : 60.6%, 3.0, 94.0ms, 0 | 20260506 20:17:18.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:18.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:18.995 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:17:18.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:18.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:18.995 | INFO | Uploaded 36GiB to GPU in 1.98594s (18.1275GiB/s) | 20260506 20:17:18.995 | INFO | Downloaded 34GiB to GPU in 1.91598s (17.7455GiB/s) | 20260506 20:17:24.132 | INFO | Analyzing chunk frames = [4158976,4159488) | 20260506 20:17:24.132 | INFO | MemInfo: { | 20260506 20:17:24.132 | INFO | "gpu" : " 17 GB", | 20260506 20:17:24.132 | INFO | "huge" : "", | 20260506 20:17:24.132 | INFO | "huge_shared" : " 27 GB", | 20260506 20:17:24.132 | INFO | "malloc" : " 14 GB", | 20260506 20:17:24.132 | INFO | "pinned" : " 29 GB" | 20260506 20:17:24.132 | INFO | } | 20260506 20:17:24.132 | INFO | Perf report for the last 5.13758 seconds: | 20260506 20:17:24.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:17:24.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:24.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:24.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:24.132 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 20:17:24.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:24.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:24.132 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:17:24.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:24.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:24.133 | INFO | Uploaded 36GiB to GPU in 2.02018s (17.8202GiB/s) | 20260506 20:17:24.133 | INFO | Downloaded 34GiB to GPU in 1.9055s (17.8431GiB/s) | 20260506 20:17:29.272 | INFO | Analyzing chunk frames = [4159488,4160000) | 20260506 20:17:29.272 | INFO | MemInfo: { | 20260506 20:17:29.272 | INFO | "gpu" : " 17 GB", | 20260506 20:17:29.272 | INFO | "huge" : "", | 20260506 20:17:29.272 | INFO | "huge_shared" : " 27 GB", | 20260506 20:17:29.272 | INFO | "malloc" : " 14 GB", | 20260506 20:17:29.272 | INFO | "pinned" : " 29 GB" | 20260506 20:17:29.272 | INFO | } | 20260506 20:17:29.272 | INFO | Perf report for the last 5.13953 seconds: | 20260506 20:17:29.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:17:29.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:29.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:29.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:29.272 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 20:17:29.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:29.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:29.272 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 20:17:29.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:29.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:29.272 | INFO | Uploaded 36GiB to GPU in 2.01552s (17.8614GiB/s) | 20260506 20:17:29.272 | INFO | Downloaded 34GiB to GPU in 1.91824s (17.7246GiB/s) | 20260506 20:17:34.404 | INFO | Analyzing chunk frames = [4160000,4160512) | 20260506 20:17:34.404 | INFO | MemInfo: { | 20260506 20:17:34.404 | INFO | "gpu" : " 17 GB", | 20260506 20:17:34.404 | INFO | "huge" : "", | 20260506 20:17:34.405 | INFO | "huge_shared" : " 27 GB", | 20260506 20:17:34.405 | INFO | "malloc" : " 14 GB", | 20260506 20:17:34.405 | INFO | "pinned" : " 29 GB" | 20260506 20:17:34.405 | INFO | } | 20260506 20:17:34.405 | INFO | Perf report for the last 5.13281 seconds: | 20260506 20:17:34.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:17:34.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:34.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:34.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:34.405 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 20:17:34.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:34.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:34.405 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 20:17:34.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:34.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:34.405 | INFO | Uploaded 36GiB to GPU in 2.01617s (17.8557GiB/s) | 20260506 20:17:34.405 | INFO | Downloaded 34GiB to GPU in 1.95793s (17.3652GiB/s) | 20260506 20:17:39.546 | INFO | Analyzing chunk frames = [4160512,4161024) | 20260506 20:17:39.546 | INFO | MemInfo: { | 20260506 20:17:39.546 | INFO | "gpu" : " 17 GB", | 20260506 20:17:39.547 | INFO | "huge" : "", | 20260506 20:17:39.547 | INFO | "huge_shared" : " 27 GB", | 20260506 20:17:39.547 | INFO | "malloc" : " 14 GB", | 20260506 20:17:39.547 | INFO | "pinned" : " 29 GB" | 20260506 20:17:39.547 | INFO | } | 20260506 20:17:39.547 | INFO | Perf report for the last 5.14201 seconds: | 20260506 20:17:39.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:17:39.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:39.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:39.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:39.547 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 20:17:39.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:39.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:39.547 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:17:39.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:39.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:39.547 | INFO | Uploaded 36GiB to GPU in 1.97663s (18.2128GiB/s) | 20260506 20:17:39.547 | INFO | Downloaded 34GiB to GPU in 2.00269s (16.9772GiB/s) | 20260506 20:17:44.687 | INFO | Analyzing chunk frames = [4161024,4161536) | 20260506 20:17:44.687 | INFO | MemInfo: { | 20260506 20:17:44.687 | INFO | "gpu" : " 17 GB", | 20260506 20:17:44.687 | INFO | "huge" : "", | 20260506 20:17:44.687 | INFO | "huge_shared" : " 27 GB", | 20260506 20:17:44.687 | INFO | "malloc" : " 14 GB", | 20260506 20:17:44.687 | INFO | "pinned" : " 29 GB" | 20260506 20:17:44.687 | INFO | } | 20260506 20:17:44.687 | INFO | Perf report for the last 5.14072 seconds: | 20260506 20:17:44.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:17:44.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:44.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:44.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:44.687 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 20:17:44.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:44.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:44.687 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 20:17:44.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:44.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:44.688 | INFO | Uploaded 36GiB to GPU in 2.02976s (17.7361GiB/s) | 20260506 20:17:44.688 | INFO | Downloaded 34GiB to GPU in 1.99089s (17.0778GiB/s) | 20260506 20:17:44.688 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:17:44.688 | INFO | Total mem usage: 18 GiB | 20260506 20:17:44.688 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:17:44.688 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.5%) | 20260506 20:17:44.688 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:17:44.688 | INFO | High Water Breakdown: | 20260506 20:17:44.688 | INFO | 61.3% : 9.48 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:17:44.688 | INFO | 38.5% : 5.96 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:17:44.688 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:17:44.688 | INFO | ------------------------------------------ | 20260506 20:17:44.688 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:17:44.688 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:17:44.688 | INFO | Total mem usage: 26.5 GiB | 20260506 20:17:44.688 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:17:44.688 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:17:44.688 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:17:44.688 | INFO | High Water Breakdown: | 20260506 20:17:44.688 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:17:44.688 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:17:44.688 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:17:44.688 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:17:44.688 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:17:44.688 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:17:44.688 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:17:44.688 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:17:44.688 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:17:44.688 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:17:44.688 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:17:44.688 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:17:44.688 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:17:44.688 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:17:44.688 | INFO | ------------------------------------------ | 20260506 20:17:44.688 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:17:44.688 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:17:44.688 | INFO | Total mem usage: 25.6 GiB | 20260506 20:17:44.688 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:17:44.688 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:17:44.688 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:17:44.688 | INFO | High Water Breakdown: | 20260506 20:17:44.688 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:17:44.688 | INFO | ------------------------------------------ | 20260506 20:17:44.688 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:17:44.688 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:17:44.688 | INFO | Total mem usage: 15.7 GiB | 20260506 20:17:44.688 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:17:44.688 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:17:44.688 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 20:17:44.688 | INFO | High Water Breakdown: | 20260506 20:17:44.688 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:17:44.688 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:17:44.688 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:17:44.688 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:17:44.688 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:17:44.688 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:17:44.688 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:17:44.688 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:17:44.688 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:17:44.688 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:17:44.688 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:17:44.688 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:17:44.688 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:17:44.688 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:17:44.688 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:17:44.688 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:17:44.688 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:17:44.688 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:17:44.688 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:17:44.688 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:17:44.688 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:17:44.688 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:17:44.688 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:17:44.688 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:17:44.688 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:17:44.688 | INFO | ------------------------------------------ | 20260506 20:17:44.689 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:17:44.689 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:17:44.689 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:17:44.689 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:17:44.689 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.65e+01, 1.36e+01 / 2.79e+00 | 20260506 20:17:44.689 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.01e-01, 1.03e-02 / 7.12e-03 | 20260506 20:17:44.689 | INFO | PerfReport: 11.63%, Baseline -- 3.63e+00 / 4.35e+00, 5.57e-01 / 4.05e-01 | 20260506 20:17:44.689 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.44e+00, 1.85e-01 / 1.42e-01 | 20260506 20:17:44.689 | INFO | PerfReport: 16.02%, DME -- 4.00e+01 / 4.47e+01, 4.42e+00 / 3.34e+00 | 20260506 20:17:44.689 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.79e+01 / 2.13e+01, 1.44e+00 / 2.15e+00 | 20260506 20:17:44.689 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.22e+00 / 2.77e+00, 3.74e-01 / 2.25e-01 | 20260506 20:17:44.689 | INFO | PerfReport: 3.31%, Metrics -- 1.03e+00 / 1.37e+00, 2.74e-01 / 1.57e-01 | 20260506 20:17:44.689 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.55e-01 / 4.07e-01, 5.57e-02 / 3.03e-02 | 20260506 20:17:44.689 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:17:44.689 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:17:44.689 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:17:44.689 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 20:17:44.689 | INFO | PerfReport: 21.51%, Upload -- 2.08e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260506 20:17:44.689 | INFO | PerfReport: 21.24%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 20:17:44.689 | INFO | PerfReport: 32.20%, Compute -- 3.12e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 20:17:44.689 | INFO | PerfReport: 1.29%, LockUpload -- 1.25e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260506 20:17:44.689 | INFO | PerfReport: 1.20%, LockDownload -- 1.16e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260506 20:17:44.689 | INFO | PerfReport: 22.55%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 20:17:44.689 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260506 20:17:44.795 | INFO | Starting a round of Baz IO | 20260506 20:17:47.929 | INFO | Populating RTMetrics report | 20260506 20:17:47.929 | INFO | RTMetrics report populated | 20260506 20:17:47.929 | INFO | Opening tmp RTMetrics json file | 20260506 20:17:47.931 | INFO | Serializing RTMetrics to json | 20260506 20:17:47.953 | INFO | Serialization done, swapping tmp file now... | 20260506 20:17:47.963 | INFO | RTMetrics done | 20260506 20:17:48.061 | INFO | Finished a round of Baz IO: Wrote 1.983 GiB in 3.266 seconds | 20260506 20:17:48.061 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.30% / 0.00% / 65.70% | 20260506 20:17:48.061 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.772675% / 1.19209e-05% / 6.35783e-05% | 20260506 20:17:48.061 | INFO | Baz IO overall: | 20260506 20:17:48.062 | INFO | BytesWritten : 2197627869343 | 20260506 20:17:48.062 | INFO | NumEvents : 1061831414249 | 20260506 20:17:49.830 | INFO | Analyzing chunk frames = [4161536,4162048) | 20260506 20:17:49.830 | INFO | MemInfo: { | 20260506 20:17:49.830 | INFO | "gpu" : " 17 GB", | 20260506 20:17:49.830 | INFO | "huge" : "", | 20260506 20:17:49.830 | INFO | "huge_shared" : " 27 GB", | 20260506 20:17:49.830 | INFO | "malloc" : " 14 GB", | 20260506 20:17:49.830 | INFO | "pinned" : " 29 GB" | 20260506 20:17:49.830 | INFO | } | 20260506 20:17:49.831 | INFO | Perf report for the last 5.14313 seconds: | 20260506 20:17:49.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:17:49.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:49.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:49.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:49.831 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 20:17:49.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:49.831 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 20:17:49.831 | INFO | PRE_HQ : 59.9%, 1.8, 57.3ms, 0 | 20260506 20:17:49.831 | INFO | BAZWRITER : 63.2%, 4.7, 153.4ms, 0 | 20260506 20:17:49.831 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 20:17:49.831 | INFO | Uploaded 36GiB to GPU in 2.05741s (17.4977GiB/s) | 20260506 20:17:49.831 | INFO | Downloaded 38GiB to GPU in 2.20518s (17.2322GiB/s) | 20260506 20:17:54.973 | INFO | Analyzing chunk frames = [4162048,4162560) | 20260506 20:17:54.973 | INFO | MemInfo: { | 20260506 20:17:54.973 | INFO | "gpu" : " 17 GB", | 20260506 20:17:54.973 | INFO | "huge" : "", | 20260506 20:17:54.973 | INFO | "huge_shared" : " 27 GB", | 20260506 20:17:54.973 | INFO | "malloc" : " 14 GB", | 20260506 20:17:54.973 | INFO | "pinned" : " 29 GB" | 20260506 20:17:54.973 | INFO | } | 20260506 20:17:54.973 | INFO | Perf report for the last 5.14232 seconds: | 20260506 20:17:54.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:17:54.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:54.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:54.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:54.973 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 20:17:54.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:54.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:54.973 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:17:54.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:54.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:17:54.973 | INFO | Uploaded 36GiB to GPU in 2.00228s (17.9795GiB/s) | 20260506 20:17:54.973 | INFO | Downloaded 34GiB to GPU in 1.9202s (17.7065GiB/s) | 20260506 20:18:00.108 | INFO | Analyzing chunk frames = [4162560,4163072) | 20260506 20:18:00.108 | INFO | MemInfo: { | 20260506 20:18:00.108 | INFO | "gpu" : " 17 GB", | 20260506 20:18:00.108 | INFO | "huge" : "", | 20260506 20:18:00.108 | INFO | "huge_shared" : " 27 GB", | 20260506 20:18:00.108 | INFO | "malloc" : " 14 GB", | 20260506 20:18:00.108 | INFO | "pinned" : " 29 GB" | 20260506 20:18:00.108 | INFO | } | 20260506 20:18:00.108 | INFO | Perf report for the last 5.13482 seconds: | 20260506 20:18:00.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:18:00.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:00.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:00.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:00.108 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 20:18:00.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:00.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:00.108 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:18:00.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:00.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:00.108 | INFO | Uploaded 36GiB to GPU in 1.98961s (18.094GiB/s) | 20260506 20:18:00.108 | INFO | Downloaded 34GiB to GPU in 1.92578s (17.6552GiB/s) | 20260506 20:18:05.244 | INFO | Analyzing chunk frames = [4163072,4163584) | 20260506 20:18:05.244 | INFO | MemInfo: { | 20260506 20:18:05.244 | INFO | "gpu" : " 17 GB", | 20260506 20:18:05.244 | INFO | "huge" : "", | 20260506 20:18:05.244 | INFO | "huge_shared" : " 27 GB", | 20260506 20:18:05.244 | INFO | "malloc" : " 14 GB", | 20260506 20:18:05.244 | INFO | "pinned" : " 29 GB" | 20260506 20:18:05.244 | INFO | } | 20260506 20:18:05.244 | INFO | Perf report for the last 5.13636 seconds: | 20260506 20:18:05.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:18:05.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:05.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:05.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:05.244 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 20:18:05.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:05.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:05.244 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 20:18:05.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:05.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:05.244 | INFO | Uploaded 36GiB to GPU in 1.95093s (18.4527GiB/s) | 20260506 20:18:05.244 | INFO | Downloaded 34GiB to GPU in 1.8737s (18.1459GiB/s) | 20260506 20:18:10.381 | INFO | Analyzing chunk frames = [4163584,4164096) | 20260506 20:18:10.381 | INFO | MemInfo: { | 20260506 20:18:10.381 | INFO | "gpu" : " 17 GB", | 20260506 20:18:10.381 | INFO | "huge" : "", | 20260506 20:18:10.381 | INFO | "huge_shared" : " 27 GB", | 20260506 20:18:10.381 | INFO | "malloc" : " 14 GB", | 20260506 20:18:10.381 | INFO | "pinned" : " 29 GB" | 20260506 20:18:10.381 | INFO | } | 20260506 20:18:10.381 | INFO | Perf report for the last 5.13693 seconds: | 20260506 20:18:10.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:18:10.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:10.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:10.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:10.381 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 20:18:10.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:10.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:10.381 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:18:10.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:10.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:10.381 | INFO | Uploaded 36GiB to GPU in 1.968s (18.2927GiB/s) | 20260506 20:18:10.381 | INFO | Downloaded 34GiB to GPU in 1.9265s (17.6486GiB/s) | 20260506 20:18:15.520 | INFO | Analyzing chunk frames = [4164096,4164608) | 20260506 20:18:15.520 | INFO | MemInfo: { | 20260506 20:18:15.520 | INFO | "gpu" : " 17 GB", | 20260506 20:18:15.520 | INFO | "huge" : "", | 20260506 20:18:15.520 | INFO | "huge_shared" : " 27 GB", | 20260506 20:18:15.520 | INFO | "malloc" : " 14 GB", | 20260506 20:18:15.520 | INFO | "pinned" : " 29 GB" | 20260506 20:18:15.520 | INFO | } | 20260506 20:18:15.520 | INFO | Perf report for the last 5.13904 seconds: | 20260506 20:18:15.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:18:15.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:15.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:15.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:15.520 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 20:18:15.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:15.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:15.520 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 20:18:15.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:15.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:15.520 | INFO | Uploaded 36GiB to GPU in 2.00693s (17.9378GiB/s) | 20260506 20:18:15.520 | INFO | Downloaded 34GiB to GPU in 1.92192s (17.6906GiB/s) | 20260506 20:18:20.660 | INFO | Analyzing chunk frames = [4164608,4165120) | 20260506 20:18:20.660 | INFO | MemInfo: { | 20260506 20:18:20.660 | INFO | "gpu" : " 17 GB", | 20260506 20:18:20.660 | INFO | "huge" : "", | 20260506 20:18:20.660 | INFO | "huge_shared" : " 27 GB", | 20260506 20:18:20.660 | INFO | "malloc" : " 14 GB", | 20260506 20:18:20.660 | INFO | "pinned" : " 29 GB" | 20260506 20:18:20.660 | INFO | } | 20260506 20:18:20.660 | INFO | Perf report for the last 5.13969 seconds: | 20260506 20:18:20.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:18:20.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:20.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:20.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:20.660 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 20:18:20.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:20.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:20.660 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:18:20.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:20.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:20.660 | INFO | Uploaded 36GiB to GPU in 2.00455s (17.9592GiB/s) | 20260506 20:18:20.660 | INFO | Downloaded 34GiB to GPU in 2.00867s (16.9266GiB/s) | 20260506 20:18:25.805 | INFO | Analyzing chunk frames = [4165120,4165632) | 20260506 20:18:25.805 | INFO | MemInfo: { | 20260506 20:18:25.805 | INFO | "gpu" : " 17 GB", | 20260506 20:18:25.805 | INFO | "huge" : "", | 20260506 20:18:25.805 | INFO | "huge_shared" : " 27 GB", | 20260506 20:18:25.805 | INFO | "malloc" : " 14 GB", | 20260506 20:18:25.805 | INFO | "pinned" : " 29 GB" | 20260506 20:18:25.805 | INFO | } | 20260506 20:18:25.805 | INFO | Perf report for the last 5.14562 seconds: | 20260506 20:18:25.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:18:25.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:25.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:25.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:25.805 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:18:25.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:25.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:25.805 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:18:25.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:25.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:25.806 | INFO | Uploaded 36GiB to GPU in 1.96858s (18.2873GiB/s) | 20260506 20:18:25.806 | INFO | Downloaded 34GiB to GPU in 1.97318s (17.2311GiB/s) | 20260506 20:18:25.903 | INFO | Starting a round of Baz IO | 20260506 20:18:29.047 | INFO | Populating RTMetrics report | 20260506 20:18:29.047 | INFO | RTMetrics report populated | 20260506 20:18:29.047 | INFO | Opening tmp RTMetrics json file | 20260506 20:18:29.067 | INFO | Serializing RTMetrics to json | 20260506 20:18:29.094 | INFO | Serialization done, swapping tmp file now... | 20260506 20:18:29.111 | INFO | RTMetrics done | 20260506 20:18:29.756 | INFO | Finished a round of Baz IO: Wrote 1.982 GiB in 3.852 seconds | 20260506 20:18:29.756 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.29% / 0.00% / 65.70% | 20260506 20:18:29.756 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.769452% / 7.94729e-06% / 9.13938e-05% | 20260506 20:18:29.756 | INFO | Baz IO overall: | 20260506 20:18:29.756 | INFO | BytesWritten : 2199756192729 | 20260506 20:18:29.756 | INFO | NumEvents : 1062852433850 | 20260506 20:18:30.949 | INFO | Analyzing chunk frames = [4165632,4166144) | 20260506 20:18:30.949 | INFO | MemInfo: { | 20260506 20:18:30.949 | INFO | "gpu" : " 17 GB", | 20260506 20:18:30.949 | INFO | "huge" : "", | 20260506 20:18:30.949 | INFO | "huge_shared" : " 27 GB", | 20260506 20:18:30.949 | INFO | "malloc" : " 14 GB", | 20260506 20:18:30.949 | INFO | "pinned" : " 29 GB" | 20260506 20:18:30.949 | INFO | } | 20260506 20:18:30.949 | INFO | Perf report for the last 5.14331 seconds: | 20260506 20:18:30.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:18:30.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:30.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:30.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:30.949 | INFO | ANALYSIS : 62.5%, 2.9, 95.8ms, 0 | 20260506 20:18:30.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:30.949 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 20:18:30.949 | INFO | PRE_HQ : 60.0%, 1.8, 56.8ms, 0 | 20260506 20:18:30.949 | INFO | BAZWRITER : 74.5%, 4.8, 182.8ms, 0 | 20260506 20:18:30.949 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 20:18:30.949 | INFO | Uploaded 36GiB to GPU in 2.06175s (17.4609GiB/s) | 20260506 20:18:30.949 | INFO | Downloaded 38GiB to GPU in 2.19686s (17.2974GiB/s) | 20260506 20:18:36.086 | INFO | Analyzing chunk frames = [4166144,4166656) | 20260506 20:18:36.086 | INFO | MemInfo: { | 20260506 20:18:36.086 | INFO | "gpu" : " 17 GB", | 20260506 20:18:36.086 | INFO | "huge" : "", | 20260506 20:18:36.086 | INFO | "huge_shared" : " 27 GB", | 20260506 20:18:36.086 | INFO | "malloc" : " 14 GB", | 20260506 20:18:36.086 | INFO | "pinned" : " 29 GB" | 20260506 20:18:36.086 | INFO | } | 20260506 20:18:36.086 | INFO | Perf report for the last 5.13721 seconds: | 20260506 20:18:36.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:18:36.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:36.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:36.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:36.086 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 20:18:36.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:36.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:36.086 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:18:36.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:36.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:36.086 | INFO | Uploaded 36GiB to GPU in 2.00333s (17.9701GiB/s) | 20260506 20:18:36.086 | INFO | Downloaded 34GiB to GPU in 1.92365s (17.6747GiB/s) | 20260506 20:18:41.223 | INFO | Analyzing chunk frames = [4166656,4167168) | 20260506 20:18:41.223 | INFO | MemInfo: { | 20260506 20:18:41.223 | INFO | "gpu" : " 17 GB", | 20260506 20:18:41.223 | INFO | "huge" : "", | 20260506 20:18:41.223 | INFO | "huge_shared" : " 27 GB", | 20260506 20:18:41.223 | INFO | "malloc" : " 14 GB", | 20260506 20:18:41.223 | INFO | "pinned" : " 29 GB" | 20260506 20:18:41.223 | INFO | } | 20260506 20:18:41.223 | INFO | Perf report for the last 5.13721 seconds: | 20260506 20:18:41.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:18:41.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:41.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:41.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:41.223 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 20:18:41.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:41.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:41.223 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:18:41.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:41.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:41.223 | INFO | Uploaded 36GiB to GPU in 1.99044s (18.0864GiB/s) | 20260506 20:18:41.223 | INFO | Downloaded 34GiB to GPU in 1.91858s (17.7214GiB/s) | 20260506 20:18:46.355 | INFO | Analyzing chunk frames = [4167168,4167680) | 20260506 20:18:46.355 | INFO | MemInfo: { | 20260506 20:18:46.355 | INFO | "gpu" : " 17 GB", | 20260506 20:18:46.355 | INFO | "huge" : "", | 20260506 20:18:46.355 | INFO | "huge_shared" : " 27 GB", | 20260506 20:18:46.355 | INFO | "malloc" : " 14 GB", | 20260506 20:18:46.355 | INFO | "pinned" : " 29 GB" | 20260506 20:18:46.355 | INFO | } | 20260506 20:18:46.355 | INFO | Perf report for the last 5.13162 seconds: | 20260506 20:18:46.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:18:46.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:46.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:46.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:46.355 | INFO | ANALYSIS : 59.9%, 3.0, 92.6ms, 0 | 20260506 20:18:46.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:46.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:46.355 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:18:46.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:46.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:46.355 | INFO | Uploaded 36GiB to GPU in 1.94869s (18.474GiB/s) | 20260506 20:18:46.355 | INFO | Downloaded 34GiB to GPU in 1.87356s (18.1473GiB/s) | 20260506 20:18:51.494 | INFO | Analyzing chunk frames = [4167680,4168192) | 20260506 20:18:51.494 | INFO | MemInfo: { | 20260506 20:18:51.494 | INFO | "gpu" : " 17 GB", | 20260506 20:18:51.494 | INFO | "huge" : "", | 20260506 20:18:51.494 | INFO | "huge_shared" : " 27 GB", | 20260506 20:18:51.494 | INFO | "malloc" : " 14 GB", | 20260506 20:18:51.494 | INFO | "pinned" : " 29 GB" | 20260506 20:18:51.494 | INFO | } | 20260506 20:18:51.494 | INFO | Perf report for the last 5.1395 seconds: | 20260506 20:18:51.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:18:51.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:51.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:51.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:51.494 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 20:18:51.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:51.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:51.494 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:18:51.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:51.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:51.495 | INFO | Uploaded 36GiB to GPU in 2.02144s (17.8091GiB/s) | 20260506 20:18:51.495 | INFO | Downloaded 34GiB to GPU in 1.91378s (17.7659GiB/s) | 20260506 20:18:56.639 | INFO | Analyzing chunk frames = [4168192,4168704) | 20260506 20:18:56.639 | INFO | MemInfo: { | 20260506 20:18:56.639 | INFO | "gpu" : " 17 GB", | 20260506 20:18:56.639 | INFO | "huge" : "", | 20260506 20:18:56.639 | INFO | "huge_shared" : " 27 GB", | 20260506 20:18:56.639 | INFO | "malloc" : " 14 GB", | 20260506 20:18:56.639 | INFO | "pinned" : " 29 GB" | 20260506 20:18:56.639 | INFO | } | 20260506 20:18:56.639 | INFO | Perf report for the last 5.14474 seconds: | 20260506 20:18:56.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:18:56.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:56.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:56.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:56.639 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 20:18:56.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:56.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:56.639 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 20:18:56.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:56.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:18:56.639 | INFO | Uploaded 36GiB to GPU in 2.0305s (17.7296GiB/s) | 20260506 20:18:56.639 | INFO | Downloaded 34GiB to GPU in 1.94622s (17.4698GiB/s) | 20260506 20:19:01.778 | INFO | Analyzing chunk frames = [4168704,4169216) | 20260506 20:19:01.778 | INFO | MemInfo: { | 20260506 20:19:01.778 | INFO | "gpu" : " 17 GB", | 20260506 20:19:01.778 | INFO | "huge" : "", | 20260506 20:19:01.778 | INFO | "huge_shared" : " 27 GB", | 20260506 20:19:01.778 | INFO | "malloc" : " 14 GB", | 20260506 20:19:01.778 | INFO | "pinned" : " 29 GB" | 20260506 20:19:01.778 | INFO | } | 20260506 20:19:01.778 | INFO | Perf report for the last 5.13872 seconds: | 20260506 20:19:01.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:19:01.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:01.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:01.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:01.778 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 20:19:01.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:01.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:01.778 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:19:01.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:01.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:01.778 | INFO | Uploaded 36GiB to GPU in 1.98803s (18.1084GiB/s) | 20260506 20:19:01.778 | INFO | Downloaded 34GiB to GPU in 1.99852s (17.0126GiB/s) | 20260506 20:19:06.922 | INFO | Analyzing chunk frames = [4169216,4169728) | 20260506 20:19:06.922 | INFO | MemInfo: { | 20260506 20:19:06.922 | INFO | "gpu" : " 17 GB", | 20260506 20:19:06.922 | INFO | "huge" : "", | 20260506 20:19:06.922 | INFO | "huge_shared" : " 27 GB", | 20260506 20:19:06.922 | INFO | "malloc" : " 14 GB", | 20260506 20:19:06.922 | INFO | "pinned" : " 29 GB" | 20260506 20:19:06.922 | INFO | } | 20260506 20:19:06.922 | INFO | Perf report for the last 5.14443 seconds: | 20260506 20:19:06.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:19:06.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:06.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:06.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:06.922 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:19:06.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:06.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:06.922 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:19:06.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:06.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:06.923 | INFO | Uploaded 36GiB to GPU in 1.979s (18.191GiB/s) | 20260506 20:19:06.923 | INFO | Downloaded 34GiB to GPU in 1.97955s (17.1757GiB/s) | 20260506 20:19:06.923 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:19:06.923 | INFO | Total mem usage: 18 GiB | 20260506 20:19:06.923 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:19:06.923 | INFO | Recent High Water Marks Sum: 17 GiB (Savings: -5.93%) | 20260506 20:19:06.923 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:19:06.923 | INFO | High Water Breakdown: | 20260506 20:19:06.923 | INFO | 62.7% : 10.7 GiB ( 93.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:19:06.923 | INFO | 37.1% : 6.31 GiB ( 94.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:19:06.923 | INFO | 0.117% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:19:06.923 | INFO | ------------------------------------------ | 20260506 20:19:06.923 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:19:06.923 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:19:06.923 | INFO | Total mem usage: 26.5 GiB | 20260506 20:19:06.923 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:19:06.923 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:19:06.923 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:19:06.923 | INFO | High Water Breakdown: | 20260506 20:19:06.923 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:19:06.923 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:19:06.923 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:19:06.923 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:19:06.923 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:19:06.923 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:19:06.923 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:19:06.923 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:19:06.923 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:19:06.923 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:19:06.923 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:19:06.923 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:19:06.923 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:19:06.923 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:19:06.923 | INFO | ------------------------------------------ | 20260506 20:19:06.923 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:19:06.923 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:19:06.923 | INFO | Total mem usage: 25.6 GiB | 20260506 20:19:06.923 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:19:06.923 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:19:06.923 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:19:06.923 | INFO | High Water Breakdown: | 20260506 20:19:06.923 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:19:06.923 | INFO | ------------------------------------------ | 20260506 20:19:06.923 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:19:06.923 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:19:06.923 | INFO | Total mem usage: 15.7 GiB | 20260506 20:19:06.923 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:19:06.923 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:19:06.924 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:19:06.924 | INFO | High Water Breakdown: | 20260506 20:19:06.924 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:19:06.924 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:19:06.924 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:19:06.924 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:19:06.924 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:19:06.924 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:19:06.924 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:19:06.924 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:19:06.924 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:19:06.924 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:19:06.924 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:19:06.924 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:19:06.924 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:19:06.924 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:19:06.924 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:19:06.924 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:19:06.924 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:19:06.924 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:19:06.924 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:19:06.924 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:19:06.924 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:19:06.924 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:19:06.924 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:19:06.924 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:19:06.924 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:19:06.924 | INFO | ------------------------------------------ | 20260506 20:19:06.924 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:19:06.924 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:19:06.924 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:19:06.924 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:19:06.924 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.36e+01 / 2.52e+00 | 20260506 20:19:06.924 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 1.07e-02 / 6.53e-03 | 20260506 20:19:06.924 | INFO | PerfReport: 11.62%, Baseline -- 3.62e+00 / 4.40e+00, 5.54e-01 / 3.33e-01 | 20260506 20:19:06.924 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.44e+00, 1.83e-01 / 1.34e-01 | 20260506 20:19:06.924 | INFO | PerfReport: 16.03%, DME -- 4.00e+01 / 4.47e+01, 4.26e+00 / 2.83e+00 | 20260506 20:19:06.924 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.39e+00 / 1.97e+00 | 20260506 20:19:06.924 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.23e+00 / 2.79e+00, 3.78e-01 / 2.38e-01 | 20260506 20:19:06.924 | INFO | PerfReport: 3.32%, Metrics -- 1.04e+00 / 1.38e+00, 2.77e-01 / 2.08e-01 | 20260506 20:19:06.924 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.55e-01 / 3.97e-01, 5.56e-02 / 1.77e-02 | 20260506 20:19:06.924 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:19:06.924 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:19:06.924 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:19:06.924 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 20:19:06.924 | INFO | PerfReport: 21.51%, Upload -- 2.08e+01 / 0.00e+00, 6.94e+00 / 0.00e+00 | 20260506 20:19:06.924 | INFO | PerfReport: 21.19%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 20:19:06.924 | INFO | PerfReport: 32.21%, Compute -- 3.12e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 20:19:06.924 | INFO | PerfReport: 1.33%, LockUpload -- 1.28e+00 / 0.00e+00, 5.10e+00 / 0.00e+00 | 20260506 20:19:06.924 | INFO | PerfReport: 1.15%, LockDownload -- 1.11e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 20:19:06.924 | INFO | PerfReport: 22.60%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 20:19:06.924 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260506 20:19:07.020 | INFO | Starting a round of Baz IO | 20260506 20:19:10.175 | INFO | Populating RTMetrics report | 20260506 20:19:10.175 | INFO | RTMetrics report populated | 20260506 20:19:10.175 | INFO | Opening tmp RTMetrics json file | 20260506 20:19:10.223 | INFO | Serializing RTMetrics to json | 20260506 20:19:10.226 | INFO | Serialization done, swapping tmp file now... | 20260506 20:19:10.230 | INFO | RTMetrics done | 20260506 20:19:10.294 | INFO | Finished a round of Baz IO: Wrote 1.98 GiB in 3.273 seconds | 20260506 20:19:10.294 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.29% / 0.00% / 65.71% | 20260506 20:19:10.294 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.769107% / 1.19209e-05% / 5.5631e-05% | 20260506 20:19:10.294 | INFO | Baz IO overall: | 20260506 20:19:10.294 | INFO | BytesWritten : 2201882612401 | 20260506 20:19:10.294 | INFO | NumEvents : 1063872261502 | 20260506 20:19:12.061 | INFO | Analyzing chunk frames = [4169728,4170240) | 20260506 20:19:12.061 | INFO | MemInfo: { | 20260506 20:19:12.061 | INFO | "gpu" : " 17 GB", | 20260506 20:19:12.061 | INFO | "huge" : "", | 20260506 20:19:12.061 | INFO | "huge_shared" : " 27 GB", | 20260506 20:19:12.061 | INFO | "malloc" : " 14 GB", | 20260506 20:19:12.061 | INFO | "pinned" : " 29 GB" | 20260506 20:19:12.061 | INFO | } | 20260506 20:19:12.061 | INFO | Perf report for the last 5.13908 seconds: | 20260506 20:19:12.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:19:12.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:12.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:12.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:12.062 | INFO | ANALYSIS : 62.4%, 2.9, 95.2ms, 0 | 20260506 20:19:12.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:12.062 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 20:19:12.062 | INFO | PRE_HQ : 59.9%, 1.8, 57.2ms, 0 | 20260506 20:19:12.062 | INFO | BAZWRITER : 63.3%, 4.6, 149.4ms, 0 | 20260506 20:19:12.062 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 20:19:12.062 | INFO | Uploaded 36GiB to GPU in 2.02487s (17.7789GiB/s) | 20260506 20:19:12.062 | INFO | Downloaded 38GiB to GPU in 2.20057s (17.2683GiB/s) | 20260506 20:19:17.196 | INFO | Analyzing chunk frames = [4170240,4170752) | 20260506 20:19:17.196 | INFO | MemInfo: { | 20260506 20:19:17.196 | INFO | "gpu" : " 17 GB", | 20260506 20:19:17.196 | INFO | "huge" : "", | 20260506 20:19:17.196 | INFO | "huge_shared" : " 27 GB", | 20260506 20:19:17.196 | INFO | "malloc" : " 14 GB", | 20260506 20:19:17.196 | INFO | "pinned" : " 29 GB" | 20260506 20:19:17.196 | INFO | } | 20260506 20:19:17.196 | INFO | Perf report for the last 5.13503 seconds: | 20260506 20:19:17.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:19:17.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:17.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:17.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:17.197 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:19:17.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:17.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:17.197 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:19:17.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:17.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:17.197 | INFO | Uploaded 36GiB to GPU in 2.00272s (17.9756GiB/s) | 20260506 20:19:17.197 | INFO | Downloaded 34GiB to GPU in 1.93324s (17.5871GiB/s) | 20260506 20:19:22.334 | INFO | Analyzing chunk frames = [4170752,4171264) | 20260506 20:19:22.334 | INFO | MemInfo: { | 20260506 20:19:22.334 | INFO | "gpu" : " 17 GB", | 20260506 20:19:22.334 | INFO | "huge" : "", | 20260506 20:19:22.334 | INFO | "huge_shared" : " 27 GB", | 20260506 20:19:22.334 | INFO | "malloc" : " 14 GB", | 20260506 20:19:22.334 | INFO | "pinned" : " 29 GB" | 20260506 20:19:22.334 | INFO | } | 20260506 20:19:22.334 | INFO | Perf report for the last 5.13749 seconds: | 20260506 20:19:22.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:19:22.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:22.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:22.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:22.334 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 20:19:22.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:22.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:22.334 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:19:22.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:22.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:22.334 | INFO | Uploaded 36GiB to GPU in 1.96044s (18.3633GiB/s) | 20260506 20:19:22.334 | INFO | Downloaded 34GiB to GPU in 1.91877s (17.7196GiB/s) | 20260506 20:19:27.470 | INFO | Analyzing chunk frames = [4171264,4171776) | 20260506 20:19:27.470 | INFO | MemInfo: { | 20260506 20:19:27.470 | INFO | "gpu" : " 17 GB", | 20260506 20:19:27.470 | INFO | "huge" : "", | 20260506 20:19:27.470 | INFO | "huge_shared" : " 27 GB", | 20260506 20:19:27.470 | INFO | "malloc" : " 14 GB", | 20260506 20:19:27.470 | INFO | "pinned" : " 29 GB" | 20260506 20:19:27.470 | INFO | } | 20260506 20:19:27.470 | INFO | Perf report for the last 5.13645 seconds: | 20260506 20:19:27.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:19:27.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:27.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:27.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:27.470 | INFO | ANALYSIS : 59.8%, 3.0, 92.5ms, 0 | 20260506 20:19:27.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:27.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:27.470 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:19:27.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:27.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:27.471 | INFO | Uploaded 36GiB to GPU in 1.91459s (18.8029GiB/s) | 20260506 20:19:27.471 | INFO | Downloaded 34GiB to GPU in 1.87312s (18.1515GiB/s) | 20260506 20:19:32.612 | INFO | Analyzing chunk frames = [4171776,4172288) | 20260506 20:19:32.612 | INFO | MemInfo: { | 20260506 20:19:32.612 | INFO | "gpu" : " 17 GB", | 20260506 20:19:32.613 | INFO | "huge" : "", | 20260506 20:19:32.613 | INFO | "huge_shared" : " 27 GB", | 20260506 20:19:32.613 | INFO | "malloc" : " 14 GB", | 20260506 20:19:32.613 | INFO | "pinned" : " 29 GB" | 20260506 20:19:32.613 | INFO | } | 20260506 20:19:32.613 | INFO | Perf report for the last 5.14233 seconds: | 20260506 20:19:32.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:19:32.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:32.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:32.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:32.613 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 20:19:32.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:32.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:32.613 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:19:32.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:32.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:32.613 | INFO | Uploaded 36GiB to GPU in 2.01343s (17.88GiB/s) | 20260506 20:19:32.613 | INFO | Downloaded 34GiB to GPU in 1.91559s (17.7491GiB/s) | 20260506 20:19:37.754 | INFO | Analyzing chunk frames = [4172288,4172800) | 20260506 20:19:37.754 | INFO | MemInfo: { | 20260506 20:19:37.754 | INFO | "gpu" : " 17 GB", | 20260506 20:19:37.754 | INFO | "huge" : "", | 20260506 20:19:37.754 | INFO | "huge_shared" : " 27 GB", | 20260506 20:19:37.754 | INFO | "malloc" : " 14 GB", | 20260506 20:19:37.754 | INFO | "pinned" : " 29 GB" | 20260506 20:19:37.754 | INFO | } | 20260506 20:19:37.754 | INFO | Perf report for the last 5.14156 seconds: | 20260506 20:19:37.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:19:37.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:37.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:37.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:37.754 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:19:37.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:37.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:37.754 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 20:19:37.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:37.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:37.755 | INFO | Uploaded 36GiB to GPU in 1.99964s (18.0032GiB/s) | 20260506 20:19:37.755 | INFO | Downloaded 34GiB to GPU in 1.95816s (17.3633GiB/s) | 20260506 20:19:42.901 | INFO | Analyzing chunk frames = [4172800,4173312) | 20260506 20:19:42.901 | INFO | MemInfo: { | 20260506 20:19:42.901 | INFO | "gpu" : " 17 GB", | 20260506 20:19:42.901 | INFO | "huge" : "", | 20260506 20:19:42.901 | INFO | "huge_shared" : " 27 GB", | 20260506 20:19:42.901 | INFO | "malloc" : " 14 GB", | 20260506 20:19:42.901 | INFO | "pinned" : " 29 GB" | 20260506 20:19:42.901 | INFO | } | 20260506 20:19:42.901 | INFO | Perf report for the last 5.14708 seconds: | 20260506 20:19:42.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:19:42.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:42.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:42.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:42.901 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 20:19:42.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:42.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:42.901 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:19:42.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:42.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:42.902 | INFO | Uploaded 36GiB to GPU in 1.95639s (18.4012GiB/s) | 20260506 20:19:42.902 | INFO | Downloaded 34GiB to GPU in 1.99621s (17.0323GiB/s) | 20260506 20:19:48.035 | INFO | Analyzing chunk frames = [4173312,4173824) | 20260506 20:19:48.035 | INFO | MemInfo: { | 20260506 20:19:48.035 | INFO | "gpu" : " 17 GB", | 20260506 20:19:48.035 | INFO | "huge" : "", | 20260506 20:19:48.035 | INFO | "huge_shared" : " 27 GB", | 20260506 20:19:48.035 | INFO | "malloc" : " 14 GB", | 20260506 20:19:48.035 | INFO | "pinned" : " 29 GB" | 20260506 20:19:48.035 | INFO | } | 20260506 20:19:48.035 | INFO | Perf report for the last 5.13376 seconds: | 20260506 20:19:48.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:19:48.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:48.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:48.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:48.035 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 20:19:48.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:48.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:48.035 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:19:48.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:48.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:48.035 | INFO | Uploaded 36GiB to GPU in 1.98988s (18.0916GiB/s) | 20260506 20:19:48.036 | INFO | Downloaded 34GiB to GPU in 1.97391s (17.2247GiB/s) | 20260506 20:19:48.135 | INFO | Starting a round of Baz IO | 20260506 20:19:51.275 | INFO | Populating RTMetrics report | 20260506 20:19:51.276 | INFO | RTMetrics report populated | 20260506 20:19:51.276 | INFO | Opening tmp RTMetrics json file | 20260506 20:19:51.277 | INFO | Serializing RTMetrics to json | 20260506 20:19:51.279 | INFO | Serialization done, swapping tmp file now... | 20260506 20:19:51.280 | INFO | RTMetrics done | 20260506 20:19:51.805 | INFO | Finished a round of Baz IO: Wrote 1.98 GiB in 3.669 seconds | 20260506 20:19:51.805 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.29% / 0.00% / 65.71% | 20260506 20:19:51.805 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.766814% / 7.94729e-06% / 5.16574e-05% | 20260506 20:19:51.805 | INFO | Baz IO overall: | 20260506 20:19:51.805 | INFO | BytesWritten : 2204009064606 | 20260506 20:19:51.805 | INFO | NumEvents : 1064892244363 | 20260506 20:19:53.173 | INFO | Analyzing chunk frames = [4173824,4174336) | 20260506 20:19:53.173 | INFO | MemInfo: { | 20260506 20:19:53.173 | INFO | "gpu" : " 17 GB", | 20260506 20:19:53.173 | INFO | "huge" : "", | 20260506 20:19:53.173 | INFO | "huge_shared" : " 27 GB", | 20260506 20:19:53.173 | INFO | "malloc" : " 14 GB", | 20260506 20:19:53.173 | INFO | "pinned" : " 29 GB" | 20260506 20:19:53.173 | INFO | } | 20260506 20:19:53.173 | INFO | Perf report for the last 5.13831 seconds: | 20260506 20:19:53.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:19:53.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:53.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:53.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:53.173 | INFO | ANALYSIS : 62.5%, 2.9, 95.6ms, 0 | 20260506 20:19:53.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:53.174 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 20:19:53.174 | INFO | PRE_HQ : 60.0%, 1.8, 56.7ms, 0 | 20260506 20:19:53.174 | INFO | BAZWRITER : 71.0%, 4.8, 176.1ms, 0 | 20260506 20:19:53.174 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260506 20:19:53.174 | INFO | Uploaded 36GiB to GPU in 2.04638s (17.5921GiB/s) | 20260506 20:19:53.174 | INFO | Downloaded 38GiB to GPU in 2.19931s (17.2781GiB/s) | 20260506 20:19:58.310 | INFO | Analyzing chunk frames = [4174336,4174848) | 20260506 20:19:58.310 | INFO | MemInfo: { | 20260506 20:19:58.310 | INFO | "gpu" : " 17 GB", | 20260506 20:19:58.310 | INFO | "huge" : "", | 20260506 20:19:58.310 | INFO | "huge_shared" : " 27 GB", | 20260506 20:19:58.310 | INFO | "malloc" : " 14 GB", | 20260506 20:19:58.310 | INFO | "pinned" : " 29 GB" | 20260506 20:19:58.310 | INFO | } | 20260506 20:19:58.310 | INFO | Perf report for the last 5.13648 seconds: | 20260506 20:19:58.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:19:58.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:58.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:58.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:58.310 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:19:58.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:58.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:58.310 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:19:58.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:58.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:19:58.310 | INFO | Uploaded 36GiB to GPU in 2.01622s (17.8552GiB/s) | 20260506 20:19:58.310 | INFO | Downloaded 34GiB to GPU in 1.91761s (17.7304GiB/s) | 20260506 20:20:03.446 | INFO | Analyzing chunk frames = [4174848,4175360) | 20260506 20:20:03.446 | INFO | MemInfo: { | 20260506 20:20:03.446 | INFO | "gpu" : " 17 GB", | 20260506 20:20:03.446 | INFO | "huge" : "", | 20260506 20:20:03.446 | INFO | "huge_shared" : " 27 GB", | 20260506 20:20:03.446 | INFO | "malloc" : " 14 GB", | 20260506 20:20:03.446 | INFO | "pinned" : " 29 GB" | 20260506 20:20:03.446 | INFO | } | 20260506 20:20:03.446 | INFO | Perf report for the last 5.13573 seconds: | 20260506 20:20:03.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:20:03.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:03.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:03.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:03.446 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 20:20:03.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:03.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:03.446 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:20:03.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:03.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:03.446 | INFO | Uploaded 36GiB to GPU in 1.98126s (18.1703GiB/s) | 20260506 20:20:03.446 | INFO | Downloaded 34GiB to GPU in 1.91905s (17.7171GiB/s) | 20260506 20:20:08.586 | INFO | Analyzing chunk frames = [4175360,4175872) | 20260506 20:20:08.586 | INFO | MemInfo: { | 20260506 20:20:08.586 | INFO | "gpu" : " 17 GB", | 20260506 20:20:08.586 | INFO | "huge" : "", | 20260506 20:20:08.586 | INFO | "huge_shared" : " 27 GB", | 20260506 20:20:08.586 | INFO | "malloc" : " 14 GB", | 20260506 20:20:08.587 | INFO | "pinned" : " 29 GB" | 20260506 20:20:08.587 | INFO | } | 20260506 20:20:08.587 | INFO | Perf report for the last 5.14084 seconds: | 20260506 20:20:08.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:20:08.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:08.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:08.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:08.587 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 20:20:08.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:08.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:08.587 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:20:08.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:08.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:08.587 | INFO | Uploaded 36GiB to GPU in 1.92933s (18.6593GiB/s) | 20260506 20:20:08.587 | INFO | Downloaded 34GiB to GPU in 1.8686s (18.1955GiB/s) | 20260506 20:20:13.731 | INFO | Analyzing chunk frames = [4175872,4176384) | 20260506 20:20:13.731 | INFO | MemInfo: { | 20260506 20:20:13.731 | INFO | "gpu" : " 17 GB", | 20260506 20:20:13.731 | INFO | "huge" : "", | 20260506 20:20:13.731 | INFO | "huge_shared" : " 27 GB", | 20260506 20:20:13.731 | INFO | "malloc" : " 14 GB", | 20260506 20:20:13.731 | INFO | "pinned" : " 29 GB" | 20260506 20:20:13.731 | INFO | } | 20260506 20:20:13.731 | INFO | Perf report for the last 5.14449 seconds: | 20260506 20:20:13.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:20:13.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:13.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:13.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:13.731 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 20:20:13.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:13.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:13.731 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:20:13.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:13.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:13.731 | INFO | Uploaded 36GiB to GPU in 2.02088s (17.8141GiB/s) | 20260506 20:20:13.731 | INFO | Downloaded 34GiB to GPU in 1.91607s (17.7446GiB/s) | 20260506 20:20:18.870 | INFO | Analyzing chunk frames = [4176384,4176896) | 20260506 20:20:18.870 | INFO | MemInfo: { | 20260506 20:20:18.870 | INFO | "gpu" : " 17 GB", | 20260506 20:20:18.870 | INFO | "huge" : "", | 20260506 20:20:18.870 | INFO | "huge_shared" : " 27 GB", | 20260506 20:20:18.870 | INFO | "malloc" : " 14 GB", | 20260506 20:20:18.870 | INFO | "pinned" : " 29 GB" | 20260506 20:20:18.870 | INFO | } | 20260506 20:20:18.870 | INFO | Perf report for the last 5.13892 seconds: | 20260506 20:20:18.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:20:18.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:18.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:18.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:18.870 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 20:20:18.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:18.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:18.870 | INFO | PRE_HQ : 31.3%, 1.0, 16.2ms, 0 | 20260506 20:20:18.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:18.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:18.870 | INFO | Uploaded 36GiB to GPU in 2.03707s (17.6724GiB/s) | 20260506 20:20:18.871 | INFO | Downloaded 34GiB to GPU in 1.9569s (17.3744GiB/s) | 20260506 20:20:24.010 | INFO | Analyzing chunk frames = [4176896,4177408) | 20260506 20:20:24.010 | INFO | MemInfo: { | 20260506 20:20:24.010 | INFO | "gpu" : " 17 GB", | 20260506 20:20:24.010 | INFO | "huge" : "", | 20260506 20:20:24.010 | INFO | "huge_shared" : " 27 GB", | 20260506 20:20:24.010 | INFO | "malloc" : " 14 GB", | 20260506 20:20:24.010 | INFO | "pinned" : " 29 GB" | 20260506 20:20:24.010 | INFO | } | 20260506 20:20:24.010 | INFO | Perf report for the last 5.13993 seconds: | 20260506 20:20:24.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:20:24.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:24.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:24.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:24.010 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 20:20:24.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:24.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:24.010 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:20:24.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:24.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:24.010 | INFO | Uploaded 36GiB to GPU in 2.00575s (17.9484GiB/s) | 20260506 20:20:24.010 | INFO | Downloaded 34GiB to GPU in 1.98902s (17.0939GiB/s) | 20260506 20:20:29.148 | INFO | Analyzing chunk frames = [4177408,4177920) | 20260506 20:20:29.148 | INFO | MemInfo: { | 20260506 20:20:29.148 | INFO | "gpu" : " 17 GB", | 20260506 20:20:29.148 | INFO | "huge" : "", | 20260506 20:20:29.148 | INFO | "huge_shared" : " 27 GB", | 20260506 20:20:29.148 | INFO | "malloc" : " 14 GB", | 20260506 20:20:29.148 | INFO | "pinned" : " 29 GB" | 20260506 20:20:29.148 | INFO | } | 20260506 20:20:29.148 | INFO | Perf report for the last 5.13834 seconds: | 20260506 20:20:29.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:20:29.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:29.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:29.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:29.148 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 20:20:29.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:29.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:29.148 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 20:20:29.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:29.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:29.149 | INFO | Uploaded 36GiB to GPU in 1.99492s (18.0459GiB/s) | 20260506 20:20:29.149 | INFO | Downloaded 34GiB to GPU in 1.9919s (17.0692GiB/s) | 20260506 20:20:29.149 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:20:29.149 | INFO | Total mem usage: 18 GiB | 20260506 20:20:29.149 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:20:29.149 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -9.55%) | 20260506 20:20:29.149 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:20:29.149 | INFO | High Water Breakdown: | 20260506 20:20:29.149 | INFO | 62.2% : 10.2 GiB ( 89.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:20:29.149 | INFO | 37.6% : 6.19 GiB ( 92.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:20:29.149 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:20:29.149 | INFO | ------------------------------------------ | 20260506 20:20:29.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:20:29.149 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:20:29.149 | INFO | Total mem usage: 26.5 GiB | 20260506 20:20:29.149 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:20:29.149 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:20:29.149 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:20:29.149 | INFO | High Water Breakdown: | 20260506 20:20:29.149 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:20:29.149 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:20:29.149 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:20:29.149 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:20:29.149 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:20:29.149 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:20:29.149 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:20:29.149 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:20:29.149 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:20:29.149 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:20:29.149 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:20:29.149 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:20:29.149 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:20:29.149 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:20:29.149 | INFO | ------------------------------------------ | 20260506 20:20:29.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:20:29.149 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:20:29.149 | INFO | Total mem usage: 25.6 GiB | 20260506 20:20:29.149 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:20:29.149 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:20:29.149 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:20:29.149 | INFO | High Water Breakdown: | 20260506 20:20:29.149 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:20:29.149 | INFO | ------------------------------------------ | 20260506 20:20:29.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:20:29.149 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:20:29.149 | INFO | Total mem usage: 15.7 GiB | 20260506 20:20:29.149 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:20:29.149 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:20:29.149 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:20:29.149 | INFO | High Water Breakdown: | 20260506 20:20:29.149 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:20:29.149 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:20:29.149 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:20:29.149 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:20:29.149 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:20:29.149 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:20:29.149 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:20:29.149 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:20:29.149 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:20:29.149 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:20:29.149 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:20:29.149 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:20:29.149 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:20:29.149 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:20:29.149 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:20:29.149 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:20:29.150 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:20:29.150 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:20:29.150 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:20:29.150 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:20:29.150 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:20:29.150 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:20:29.150 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:20:29.150 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:20:29.150 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:20:29.150 | INFO | ------------------------------------------ | 20260506 20:20:29.150 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:20:29.150 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:20:29.150 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:20:29.150 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:20:29.150 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.59e+01, 1.35e+01 / 2.15e+00 | 20260506 20:20:29.150 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.02e-01, 9.86e-03 / 6.06e-03 | 20260506 20:20:29.150 | INFO | PerfReport: 11.62%, Baseline -- 3.63e+00 / 4.39e+00, 5.50e-01 / 3.31e-01 | 20260506 20:20:29.150 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.89e-01 / 1.29e-01 | 20260506 20:20:29.150 | INFO | PerfReport: 16.00%, DME -- 3.99e+01 / 4.46e+01, 4.14e+00 / 2.55e+00 | 20260506 20:20:29.150 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.49e+00 / 1.92e+00 | 20260506 20:20:29.150 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.22e+00 / 2.83e+00, 3.71e-01 / 2.07e-01 | 20260506 20:20:29.150 | INFO | PerfReport: 3.30%, Metrics -- 1.03e+00 / 1.39e+00, 2.71e-01 / 1.55e-01 | 20260506 20:20:29.150 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.55e-01 / 4.05e-01, 5.63e-02 / 2.76e-02 | 20260506 20:20:29.150 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:20:29.150 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:20:29.150 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:20:29.150 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 20:20:29.150 | INFO | PerfReport: 21.45%, Upload -- 2.08e+01 / 0.00e+00, 6.91e+00 / 0.00e+00 | 20260506 20:20:29.150 | INFO | PerfReport: 21.20%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 20:20:29.150 | INFO | PerfReport: 32.23%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 20:20:29.150 | INFO | PerfReport: 1.29%, LockUpload -- 1.25e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260506 20:20:29.150 | INFO | PerfReport: 1.17%, LockDownload -- 1.14e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260506 20:20:29.150 | INFO | PerfReport: 22.64%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 20:20:29.150 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.70e-02 / 0.00e+00 | 20260506 20:20:29.255 | INFO | Starting a round of Baz IO | 20260506 20:20:32.390 | INFO | Populating RTMetrics report | 20260506 20:20:32.391 | INFO | RTMetrics report populated | 20260506 20:20:32.391 | INFO | Opening tmp RTMetrics json file | 20260506 20:20:32.472 | INFO | Serializing RTMetrics to json | 20260506 20:20:32.474 | INFO | Serialization done, swapping tmp file now... | 20260506 20:20:32.477 | INFO | RTMetrics done | 20260506 20:20:32.582 | INFO | Finished a round of Baz IO: Wrote 1.981 GiB in 3.327 seconds | 20260506 20:20:32.582 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.28% / 0.00% / 65.71% | 20260506 20:20:32.582 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.76623% / 1.98682e-05% / 5.96046e-05% | 20260506 20:20:32.582 | INFO | Baz IO overall: | 20260506 20:20:32.582 | INFO | BytesWritten : 2206136619101 | 20260506 20:20:32.582 | INFO | NumEvents : 1065912807389 | 20260506 20:20:34.286 | INFO | Analyzing chunk frames = [4177920,4178432) | 20260506 20:20:34.286 | INFO | MemInfo: { | 20260506 20:20:34.286 | INFO | "gpu" : " 17 GB", | 20260506 20:20:34.286 | INFO | "huge" : "", | 20260506 20:20:34.286 | INFO | "huge_shared" : " 27 GB", | 20260506 20:20:34.286 | INFO | "malloc" : " 14 GB", | 20260506 20:20:34.286 | INFO | "pinned" : " 29 GB" | 20260506 20:20:34.286 | INFO | } | 20260506 20:20:34.286 | INFO | Perf report for the last 5.13748 seconds: | 20260506 20:20:34.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:20:34.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:34.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:34.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:34.286 | INFO | ANALYSIS : 62.5%, 2.9, 95.2ms, 0 | 20260506 20:20:34.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:34.286 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 20:20:34.286 | INFO | PRE_HQ : 59.9%, 1.8, 57.4ms, 0 | 20260506 20:20:34.286 | INFO | BAZWRITER : 64.4%, 4.8, 157.7ms, 0 | 20260506 20:20:34.286 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 20:20:34.286 | INFO | Uploaded 36GiB to GPU in 2.06924s (17.3977GiB/s) | 20260506 20:20:34.286 | INFO | Downloaded 38GiB to GPU in 2.20704s (17.2177GiB/s) | 20260506 20:20:39.421 | INFO | Analyzing chunk frames = [4178432,4178944) | 20260506 20:20:39.421 | INFO | MemInfo: { | 20260506 20:20:39.421 | INFO | "gpu" : " 17 GB", | 20260506 20:20:39.421 | INFO | "huge" : "", | 20260506 20:20:39.421 | INFO | "huge_shared" : " 27 GB", | 20260506 20:20:39.421 | INFO | "malloc" : " 14 GB", | 20260506 20:20:39.421 | INFO | "pinned" : " 29 GB" | 20260506 20:20:39.421 | INFO | } | 20260506 20:20:39.421 | INFO | Perf report for the last 5.13538 seconds: | 20260506 20:20:39.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:20:39.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:39.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:39.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:39.421 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:20:39.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:39.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:39.421 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:20:39.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:39.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:39.421 | INFO | Uploaded 36GiB to GPU in 2.02143s (17.8092GiB/s) | 20260506 20:20:39.421 | INFO | Downloaded 34GiB to GPU in 1.92121s (17.6972GiB/s) | 20260506 20:20:44.560 | INFO | Analyzing chunk frames = [4178944,4179456) | 20260506 20:20:44.560 | INFO | MemInfo: { | 20260506 20:20:44.560 | INFO | "gpu" : " 17 GB", | 20260506 20:20:44.560 | INFO | "huge" : "", | 20260506 20:20:44.560 | INFO | "huge_shared" : " 27 GB", | 20260506 20:20:44.561 | INFO | "malloc" : " 14 GB", | 20260506 20:20:44.561 | INFO | "pinned" : " 29 GB" | 20260506 20:20:44.561 | INFO | } | 20260506 20:20:44.561 | INFO | Perf report for the last 5.13944 seconds: | 20260506 20:20:44.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:20:44.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:44.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:44.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:44.561 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 20:20:44.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:44.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:44.561 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:20:44.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:44.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:44.561 | INFO | Uploaded 36GiB to GPU in 1.99045s (18.0863GiB/s) | 20260506 20:20:44.561 | INFO | Downloaded 34GiB to GPU in 1.91777s (17.7289GiB/s) | 20260506 20:20:49.705 | INFO | Analyzing chunk frames = [4179456,4179968) | 20260506 20:20:49.705 | INFO | MemInfo: { | 20260506 20:20:49.705 | INFO | "gpu" : " 17 GB", | 20260506 20:20:49.705 | INFO | "huge" : "", | 20260506 20:20:49.705 | INFO | "huge_shared" : " 27 GB", | 20260506 20:20:49.705 | INFO | "malloc" : " 14 GB", | 20260506 20:20:49.705 | INFO | "pinned" : " 29 GB" | 20260506 20:20:49.705 | INFO | } | 20260506 20:20:49.705 | INFO | Perf report for the last 5.14463 seconds: | 20260506 20:20:49.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:20:49.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:49.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:49.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:49.705 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260506 20:20:49.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:49.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:49.705 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:20:49.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:49.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:49.706 | INFO | Uploaded 36GiB to GPU in 1.92759s (18.6761GiB/s) | 20260506 20:20:49.706 | INFO | Downloaded 34GiB to GPU in 1.86318s (18.2484GiB/s) | 20260506 20:20:54.848 | INFO | Analyzing chunk frames = [4179968,4180480) | 20260506 20:20:54.848 | INFO | MemInfo: { | 20260506 20:20:54.848 | INFO | "gpu" : " 17 GB", | 20260506 20:20:54.848 | INFO | "huge" : "", | 20260506 20:20:54.848 | INFO | "huge_shared" : " 27 GB", | 20260506 20:20:54.848 | INFO | "malloc" : " 14 GB", | 20260506 20:20:54.848 | INFO | "pinned" : " 29 GB" | 20260506 20:20:54.848 | INFO | } | 20260506 20:20:54.848 | INFO | Perf report for the last 5.14259 seconds: | 20260506 20:20:54.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:20:54.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:54.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:54.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:54.848 | INFO | ANALYSIS : 60.3%, 3.0, 93.0ms, 0 | 20260506 20:20:54.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:54.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:54.848 | INFO | PRE_HQ : 26.6%, 1.0, 13.7ms, 0 | 20260506 20:20:54.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:54.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:54.848 | INFO | Uploaded 36GiB to GPU in 1.99674s (18.0294GiB/s) | 20260506 20:20:54.848 | INFO | Downloaded 34GiB to GPU in 1.93315s (17.5879GiB/s) | 20260506 20:20:59.990 | INFO | Analyzing chunk frames = [4180480,4180992) | 20260506 20:20:59.990 | INFO | MemInfo: { | 20260506 20:20:59.990 | INFO | "gpu" : " 17 GB", | 20260506 20:20:59.990 | INFO | "huge" : "", | 20260506 20:20:59.990 | INFO | "huge_shared" : " 27 GB", | 20260506 20:20:59.990 | INFO | "malloc" : " 14 GB", | 20260506 20:20:59.990 | INFO | "pinned" : " 29 GB" | 20260506 20:20:59.990 | INFO | } | 20260506 20:20:59.990 | INFO | Perf report for the last 5.1426 seconds: | 20260506 20:20:59.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:20:59.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:59.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:59.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:59.991 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 20:20:59.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:59.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:59.991 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 20:20:59.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:59.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:20:59.991 | INFO | Uploaded 36GiB to GPU in 2.02818s (17.7499GiB/s) | 20260506 20:20:59.991 | INFO | Downloaded 34GiB to GPU in 1.9335s (17.5847GiB/s) | 20260506 20:21:05.126 | INFO | Analyzing chunk frames = [4180992,4181504) | 20260506 20:21:05.126 | INFO | MemInfo: { | 20260506 20:21:05.126 | INFO | "gpu" : " 17 GB", | 20260506 20:21:05.126 | INFO | "huge" : "", | 20260506 20:21:05.126 | INFO | "huge_shared" : " 27 GB", | 20260506 20:21:05.126 | INFO | "malloc" : " 14 GB", | 20260506 20:21:05.126 | INFO | "pinned" : " 29 GB" | 20260506 20:21:05.126 | INFO | } | 20260506 20:21:05.126 | INFO | Perf report for the last 5.13528 seconds: | 20260506 20:21:05.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:21:05.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:05.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:05.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:05.126 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 20:21:05.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:05.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:05.126 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 20:21:05.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:05.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:05.126 | INFO | Uploaded 36GiB to GPU in 1.97584s (18.2201GiB/s) | 20260506 20:21:05.126 | INFO | Downloaded 34GiB to GPU in 2.00798s (16.9324GiB/s) | 20260506 20:21:10.261 | INFO | Analyzing chunk frames = [4181504,4182016) | 20260506 20:21:10.261 | INFO | MemInfo: { | 20260506 20:21:10.261 | INFO | "gpu" : " 17 GB", | 20260506 20:21:10.261 | INFO | "huge" : "", | 20260506 20:21:10.261 | INFO | "huge_shared" : " 27 GB", | 20260506 20:21:10.261 | INFO | "malloc" : " 14 GB", | 20260506 20:21:10.261 | INFO | "pinned" : " 29 GB" | 20260506 20:21:10.261 | INFO | } | 20260506 20:21:10.262 | INFO | Perf report for the last 5.13587 seconds: | 20260506 20:21:10.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:21:10.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:10.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:10.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:10.262 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 20:21:10.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:10.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:10.262 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 20:21:10.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:10.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:10.262 | INFO | Uploaded 36GiB to GPU in 2.05833s (17.4899GiB/s) | 20260506 20:21:10.262 | INFO | Downloaded 34GiB to GPU in 1.99572s (17.0365GiB/s) | 20260506 20:21:10.376 | INFO | Starting a round of Baz IO | 20260506 20:21:13.503 | INFO | Populating RTMetrics report | 20260506 20:21:13.503 | INFO | RTMetrics report populated | 20260506 20:21:13.503 | INFO | Opening tmp RTMetrics json file | 20260506 20:21:13.545 | INFO | Serializing RTMetrics to json | 20260506 20:21:13.557 | INFO | Serialization done, swapping tmp file now... | 20260506 20:21:13.561 | INFO | RTMetrics done | 20260506 20:21:14.357 | INFO | Finished a round of Baz IO: Wrote 1.981 GiB in 3.981 seconds | 20260506 20:21:14.357 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.28% / 0.00% / 65.72% | 20260506 20:21:14.357 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.764795% / 1.58946e-05% / 7.94729e-05% | 20260506 20:21:14.357 | INFO | Baz IO overall: | 20260506 20:21:14.357 | INFO | BytesWritten : 2208263657296 | 20260506 20:21:14.357 | INFO | NumEvents : 1066933050039 | 20260506 20:21:15.400 | INFO | Analyzing chunk frames = [4182016,4182528) | 20260506 20:21:15.400 | INFO | MemInfo: { | 20260506 20:21:15.400 | INFO | "gpu" : " 17 GB", | 20260506 20:21:15.400 | INFO | "huge" : "", | 20260506 20:21:15.400 | INFO | "huge_shared" : " 27 GB", | 20260506 20:21:15.400 | INFO | "malloc" : " 14 GB", | 20260506 20:21:15.400 | INFO | "pinned" : " 29 GB" | 20260506 20:21:15.400 | INFO | } | 20260506 20:21:15.400 | INFO | Perf report for the last 5.13864 seconds: | 20260506 20:21:15.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:21:15.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:15.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:15.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:15.400 | INFO | ANALYSIS : 62.5%, 2.9, 94.5ms, 0 | 20260506 20:21:15.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:15.400 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 20:21:15.400 | INFO | PRE_HQ : 60.1%, 1.9, 60.0ms, 0 | 20260506 20:21:15.400 | INFO | BAZWRITER : 77.2%, 4.8, 190.3ms, 0 | 20260506 20:21:15.400 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 20:21:15.401 | INFO | Uploaded 36GiB to GPU in 2.06266s (17.4532GiB/s) | 20260506 20:21:15.401 | INFO | Downloaded 38GiB to GPU in 2.20978s (17.1963GiB/s) | 20260506 20:21:20.539 | INFO | Analyzing chunk frames = [4182528,4183040) | 20260506 20:21:20.539 | INFO | MemInfo: { | 20260506 20:21:20.539 | INFO | "gpu" : " 17 GB", | 20260506 20:21:20.539 | INFO | "huge" : "", | 20260506 20:21:20.539 | INFO | "huge_shared" : " 27 GB", | 20260506 20:21:20.539 | INFO | "malloc" : " 14 GB", | 20260506 20:21:20.540 | INFO | "pinned" : " 29 GB" | 20260506 20:21:20.540 | INFO | } | 20260506 20:21:20.540 | INFO | Perf report for the last 5.13929 seconds: | 20260506 20:21:20.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:21:20.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:20.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:20.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:20.540 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:21:20.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:20.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:20.540 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:21:20.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:20.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:20.540 | INFO | Uploaded 36GiB to GPU in 2.01871s (17.8331GiB/s) | 20260506 20:21:20.540 | INFO | Downloaded 34GiB to GPU in 1.94105s (17.5163GiB/s) | 20260506 20:21:25.680 | INFO | Analyzing chunk frames = [4183040,4183552) | 20260506 20:21:25.680 | INFO | MemInfo: { | 20260506 20:21:25.680 | INFO | "gpu" : " 17 GB", | 20260506 20:21:25.680 | INFO | "huge" : "", | 20260506 20:21:25.680 | INFO | "huge_shared" : " 27 GB", | 20260506 20:21:25.680 | INFO | "malloc" : " 14 GB", | 20260506 20:21:25.680 | INFO | "pinned" : " 29 GB" | 20260506 20:21:25.680 | INFO | } | 20260506 20:21:25.680 | INFO | Perf report for the last 5.14046 seconds: | 20260506 20:21:25.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:21:25.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:25.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:25.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:25.680 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 20:21:25.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:25.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:25.680 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:21:25.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:25.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:25.680 | INFO | Uploaded 36GiB to GPU in 1.99021s (18.0885GiB/s) | 20260506 20:21:25.680 | INFO | Downloaded 34GiB to GPU in 1.91717s (17.7345GiB/s) | 20260506 20:21:30.819 | INFO | Analyzing chunk frames = [4183552,4184064) | 20260506 20:21:30.819 | INFO | MemInfo: { | 20260506 20:21:30.819 | INFO | "gpu" : " 17 GB", | 20260506 20:21:30.819 | INFO | "huge" : "", | 20260506 20:21:30.820 | INFO | "huge_shared" : " 27 GB", | 20260506 20:21:30.820 | INFO | "malloc" : " 14 GB", | 20260506 20:21:30.820 | INFO | "pinned" : " 29 GB" | 20260506 20:21:30.820 | INFO | } | 20260506 20:21:30.820 | INFO | Perf report for the last 5.13955 seconds: | 20260506 20:21:30.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:21:30.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:30.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:30.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:30.820 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 20:21:30.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:30.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:30.820 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:21:30.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:30.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:30.820 | INFO | Uploaded 36GiB to GPU in 1.945s (18.509GiB/s) | 20260506 20:21:30.820 | INFO | Downloaded 34GiB to GPU in 1.86958s (18.1859GiB/s) | 20260506 20:21:35.959 | INFO | Analyzing chunk frames = [4184064,4184576) | 20260506 20:21:35.959 | INFO | MemInfo: { | 20260506 20:21:35.959 | INFO | "gpu" : " 17 GB", | 20260506 20:21:35.959 | INFO | "huge" : "", | 20260506 20:21:35.959 | INFO | "huge_shared" : " 27 GB", | 20260506 20:21:35.959 | INFO | "malloc" : " 14 GB", | 20260506 20:21:35.959 | INFO | "pinned" : " 29 GB" | 20260506 20:21:35.960 | INFO | } | 20260506 20:21:35.960 | INFO | Perf report for the last 5.13993 seconds: | 20260506 20:21:35.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:21:35.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:35.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:35.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:35.960 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 20:21:35.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:35.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:35.960 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:21:35.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:35.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:35.960 | INFO | Uploaded 36GiB to GPU in 2.02224s (17.8021GiB/s) | 20260506 20:21:35.960 | INFO | Downloaded 34GiB to GPU in 1.9134s (17.7695GiB/s) | 20260506 20:21:41.099 | INFO | Analyzing chunk frames = [4184576,4185088) | 20260506 20:21:41.099 | INFO | MemInfo: { | 20260506 20:21:41.099 | INFO | "gpu" : " 17 GB", | 20260506 20:21:41.099 | INFO | "huge" : "", | 20260506 20:21:41.099 | INFO | "huge_shared" : " 27 GB", | 20260506 20:21:41.099 | INFO | "malloc" : " 14 GB", | 20260506 20:21:41.099 | INFO | "pinned" : " 29 GB" | 20260506 20:21:41.099 | INFO | } | 20260506 20:21:41.099 | INFO | Perf report for the last 5.13968 seconds: | 20260506 20:21:41.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:21:41.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:41.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:41.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:41.099 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 20:21:41.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:41.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:41.099 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:21:41.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:41.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:41.100 | INFO | Uploaded 36GiB to GPU in 2.03235s (17.7135GiB/s) | 20260506 20:21:41.100 | INFO | Downloaded 34GiB to GPU in 1.94663s (17.466GiB/s) | 20260506 20:21:46.235 | INFO | Analyzing chunk frames = [4185088,4185600) | 20260506 20:21:46.235 | INFO | MemInfo: { | 20260506 20:21:46.235 | INFO | "gpu" : " 17 GB", | 20260506 20:21:46.235 | INFO | "huge" : "", | 20260506 20:21:46.236 | INFO | "huge_shared" : " 27 GB", | 20260506 20:21:46.236 | INFO | "malloc" : " 14 GB", | 20260506 20:21:46.236 | INFO | "pinned" : " 29 GB" | 20260506 20:21:46.236 | INFO | } | 20260506 20:21:46.236 | INFO | Perf report for the last 5.13679 seconds: | 20260506 20:21:46.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:21:46.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:46.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:46.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:46.236 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 20:21:46.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:46.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:46.236 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:21:46.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:46.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:46.236 | INFO | Uploaded 36GiB to GPU in 1.99511s (18.0441GiB/s) | 20260506 20:21:46.236 | INFO | Downloaded 34GiB to GPU in 2.00733s (16.9379GiB/s) | 20260506 20:21:51.376 | INFO | Analyzing chunk frames = [4185600,4186112) | 20260506 20:21:51.376 | INFO | MemInfo: { | 20260506 20:21:51.376 | INFO | "gpu" : " 17 GB", | 20260506 20:21:51.376 | INFO | "huge" : "", | 20260506 20:21:51.376 | INFO | "huge_shared" : " 27 GB", | 20260506 20:21:51.376 | INFO | "malloc" : " 14 GB", | 20260506 20:21:51.376 | INFO | "pinned" : " 29 GB" | 20260506 20:21:51.376 | INFO | } | 20260506 20:21:51.376 | INFO | Perf report for the last 5.14001 seconds: | 20260506 20:21:51.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:21:51.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:51.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:51.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:51.376 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:21:51.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:51.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:51.376 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:21:51.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:51.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:51.376 | INFO | Uploaded 36GiB to GPU in 1.97274s (18.2488GiB/s) | 20260506 20:21:51.376 | INFO | Downloaded 34GiB to GPU in 1.98741s (17.1077GiB/s) | 20260506 20:21:51.377 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:21:51.377 | INFO | Total mem usage: 18 GiB | 20260506 20:21:51.377 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:21:51.377 | INFO | Recent High Water Marks Sum: 16.7 GiB (Savings: -8.12%) | 20260506 20:21:51.377 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:21:51.377 | INFO | High Water Breakdown: | 20260506 20:21:51.377 | INFO | 62.4% : 10.4 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:21:51.377 | INFO | 37.5% : 6.24 GiB ( 93.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:21:51.377 | INFO | 0.119% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:21:51.377 | INFO | ------------------------------------------ | 20260506 20:21:51.377 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:21:51.377 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:21:51.377 | INFO | Total mem usage: 26.5 GiB | 20260506 20:21:51.377 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:21:51.377 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:21:51.377 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:21:51.377 | INFO | High Water Breakdown: | 20260506 20:21:51.377 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:21:51.377 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:21:51.377 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:21:51.377 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:21:51.377 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:21:51.377 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:21:51.377 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:21:51.377 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:21:51.377 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:21:51.377 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:21:51.377 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:21:51.377 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:21:51.377 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:21:51.377 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:21:51.377 | INFO | ------------------------------------------ | 20260506 20:21:51.377 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:21:51.377 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:21:51.377 | INFO | Total mem usage: 25.6 GiB | 20260506 20:21:51.377 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:21:51.377 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:21:51.377 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:21:51.377 | INFO | High Water Breakdown: | 20260506 20:21:51.377 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:21:51.377 | INFO | ------------------------------------------ | 20260506 20:21:51.377 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:21:51.377 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:21:51.377 | INFO | Total mem usage: 15.7 GiB | 20260506 20:21:51.377 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:21:51.377 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:21:51.377 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:21:51.377 | INFO | High Water Breakdown: | 20260506 20:21:51.377 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:21:51.377 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:21:51.377 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:21:51.377 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:21:51.377 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:21:51.377 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:21:51.378 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:21:51.378 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:21:51.378 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:21:51.378 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:21:51.378 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:21:51.378 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:21:51.378 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:21:51.378 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:21:51.378 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:21:51.378 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:21:51.378 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:21:51.378 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:21:51.378 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:21:51.378 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:21:51.378 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:21:51.378 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:21:51.378 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:21:51.378 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:21:51.378 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:21:51.378 | INFO | ------------------------------------------ | 20260506 20:21:51.378 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:21:51.378 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:21:51.378 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:21:51.378 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:21:51.378 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.61e+01, 1.35e+01 / 2.48e+00 | 20260506 20:21:51.378 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 9.49e-03 / 4.85e-03 | 20260506 20:21:51.378 | INFO | PerfReport: 11.64%, Baseline -- 3.63e+00 / 4.43e+00, 5.53e-01 / 3.38e-01 | 20260506 20:21:51.378 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.81e-01 / 1.26e-01 | 20260506 20:21:51.378 | INFO | PerfReport: 16.08%, DME -- 4.01e+01 / 4.51e+01, 4.35e+00 / 3.09e+00 | 20260506 20:21:51.378 | INFO | PerfReport: 57.11%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.37e+00 / 2.07e+00 | 20260506 20:21:51.378 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.23e+00 / 2.81e+00, 3.81e-01 / 2.01e-01 | 20260506 20:21:51.378 | INFO | PerfReport: 3.31%, Metrics -- 1.03e+00 / 1.35e+00, 2.73e-01 / 1.24e-01 | 20260506 20:21:51.378 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.55e-01 / 3.98e-01, 5.59e-02 / 1.85e-02 | 20260506 20:21:51.378 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:21:51.378 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:21:51.378 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:21:51.378 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 20:21:51.378 | INFO | PerfReport: 21.59%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 20:21:51.378 | INFO | PerfReport: 21.23%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 20:21:51.378 | INFO | PerfReport: 32.19%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 20:21:51.378 | INFO | PerfReport: 1.39%, LockUpload -- 1.34e+00 / 0.00e+00, 5.15e+00 / 0.00e+00 | 20260506 20:21:51.378 | INFO | PerfReport: 1.12%, LockDownload -- 1.09e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260506 20:21:51.378 | INFO | PerfReport: 22.47%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 20:21:51.378 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260506 20:21:51.475 | INFO | Starting a round of Baz IO | 20260506 20:21:54.617 | INFO | Populating RTMetrics report | 20260506 20:21:54.617 | INFO | RTMetrics report populated | 20260506 20:21:54.617 | INFO | Opening tmp RTMetrics json file | 20260506 20:21:54.656 | INFO | Serializing RTMetrics to json | 20260506 20:21:54.659 | INFO | Serialization done, swapping tmp file now... | 20260506 20:21:54.665 | INFO | RTMetrics done | 20260506 20:21:55.103 | INFO | Finished a round of Baz IO: Wrote 1.981 GiB in 3.627 seconds | 20260506 20:21:55.103 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.27% / 0.00% / 65.72% | 20260506 20:21:55.103 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.76193% / 0% / 8.74201e-05% | 20260506 20:21:55.103 | INFO | Baz IO overall: | 20260506 20:21:55.103 | INFO | BytesWritten : 2210390925793 | 20260506 20:21:55.103 | INFO | NumEvents : 1067953376028 | 20260506 20:21:56.513 | INFO | Analyzing chunk frames = [4186112,4186624) | 20260506 20:21:56.513 | INFO | MemInfo: { | 20260506 20:21:56.513 | INFO | "gpu" : " 17 GB", | 20260506 20:21:56.513 | INFO | "huge" : "", | 20260506 20:21:56.513 | INFO | "huge_shared" : " 27 GB", | 20260506 20:21:56.513 | INFO | "malloc" : " 14 GB", | 20260506 20:21:56.513 | INFO | "pinned" : " 29 GB" | 20260506 20:21:56.513 | INFO | } | 20260506 20:21:56.513 | INFO | Perf report for the last 5.13727 seconds: | 20260506 20:21:56.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:21:56.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:56.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:56.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:56.514 | INFO | ANALYSIS : 62.5%, 2.9, 95.5ms, 0 | 20260506 20:21:56.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:21:56.514 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 20:21:56.514 | INFO | PRE_HQ : 60.0%, 1.8, 56.8ms, 0 | 20260506 20:21:56.514 | INFO | BAZWRITER : 70.2%, 4.8, 172.5ms, 0 | 20260506 20:21:56.514 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 20:21:56.514 | INFO | Uploaded 36GiB to GPU in 2.03131s (17.7226GiB/s) | 20260506 20:21:56.514 | INFO | Downloaded 38GiB to GPU in 2.19578s (17.3059GiB/s) | 20260506 20:22:01.656 | INFO | Analyzing chunk frames = [4186624,4187136) | 20260506 20:22:01.656 | INFO | MemInfo: { | 20260506 20:22:01.656 | INFO | "gpu" : " 17 GB", | 20260506 20:22:01.656 | INFO | "huge" : "", | 20260506 20:22:01.656 | INFO | "huge_shared" : " 27 GB", | 20260506 20:22:01.656 | INFO | "malloc" : " 14 GB", | 20260506 20:22:01.656 | INFO | "pinned" : " 29 GB" | 20260506 20:22:01.656 | INFO | } | 20260506 20:22:01.656 | INFO | Perf report for the last 5.14236 seconds: | 20260506 20:22:01.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:22:01.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:01.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:01.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:01.656 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:22:01.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:01.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:01.656 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:22:01.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:01.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:01.656 | INFO | Uploaded 36GiB to GPU in 2.0103s (17.9078GiB/s) | 20260506 20:22:01.656 | INFO | Downloaded 34GiB to GPU in 1.93051s (17.6119GiB/s) | 20260506 20:22:06.795 | INFO | Analyzing chunk frames = [4187136,4187648) | 20260506 20:22:06.795 | INFO | MemInfo: { | 20260506 20:22:06.795 | INFO | "gpu" : " 17 GB", | 20260506 20:22:06.795 | INFO | "huge" : "", | 20260506 20:22:06.795 | INFO | "huge_shared" : " 27 GB", | 20260506 20:22:06.795 | INFO | "malloc" : " 14 GB", | 20260506 20:22:06.795 | INFO | "pinned" : " 29 GB" | 20260506 20:22:06.795 | INFO | } | 20260506 20:22:06.795 | INFO | Perf report for the last 5.13941 seconds: | 20260506 20:22:06.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:22:06.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:06.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:06.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:06.795 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 20:22:06.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:06.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:06.795 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:22:06.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:06.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:06.795 | INFO | Uploaded 36GiB to GPU in 1.96288s (18.3404GiB/s) | 20260506 20:22:06.795 | INFO | Downloaded 34GiB to GPU in 1.92117s (17.6975GiB/s) | 20260506 20:22:11.936 | INFO | Analyzing chunk frames = [4187648,4188160) | 20260506 20:22:11.936 | INFO | MemInfo: { | 20260506 20:22:11.936 | INFO | "gpu" : " 17 GB", | 20260506 20:22:11.936 | INFO | "huge" : "", | 20260506 20:22:11.936 | INFO | "huge_shared" : " 27 GB", | 20260506 20:22:11.936 | INFO | "malloc" : " 14 GB", | 20260506 20:22:11.936 | INFO | "pinned" : " 29 GB" | 20260506 20:22:11.936 | INFO | } | 20260506 20:22:11.936 | INFO | Perf report for the last 5.14071 seconds: | 20260506 20:22:11.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:22:11.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:11.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:11.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:11.936 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:22:11.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:11.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:11.936 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:22:11.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:11.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:11.936 | INFO | Uploaded 36GiB to GPU in 1.92813s (18.6709GiB/s) | 20260506 20:22:11.936 | INFO | Downloaded 34GiB to GPU in 1.87253s (18.1573GiB/s) | 20260506 20:22:17.080 | INFO | Analyzing chunk frames = [4188160,4188672) | 20260506 20:22:17.080 | INFO | MemInfo: { | 20260506 20:22:17.080 | INFO | "gpu" : " 17 GB", | 20260506 20:22:17.080 | INFO | "huge" : "", | 20260506 20:22:17.080 | INFO | "huge_shared" : " 27 GB", | 20260506 20:22:17.080 | INFO | "malloc" : " 14 GB", | 20260506 20:22:17.080 | INFO | "pinned" : " 29 GB" | 20260506 20:22:17.080 | INFO | } | 20260506 20:22:17.080 | INFO | Perf report for the last 5.14425 seconds: | 20260506 20:22:17.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:22:17.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:17.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:17.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:17.080 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 20:22:17.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:17.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:17.080 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:22:17.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:17.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:17.081 | INFO | Uploaded 36GiB to GPU in 1.99518s (18.0435GiB/s) | 20260506 20:22:17.081 | INFO | Downloaded 34GiB to GPU in 1.92089s (17.7001GiB/s) | 20260506 20:22:22.212 | INFO | Analyzing chunk frames = [4188672,4189184) | 20260506 20:22:22.212 | INFO | MemInfo: { | 20260506 20:22:22.212 | INFO | "gpu" : " 17 GB", | 20260506 20:22:22.212 | INFO | "huge" : "", | 20260506 20:22:22.212 | INFO | "huge_shared" : " 27 GB", | 20260506 20:22:22.212 | INFO | "malloc" : " 14 GB", | 20260506 20:22:22.212 | INFO | "pinned" : " 29 GB" | 20260506 20:22:22.212 | INFO | } | 20260506 20:22:22.213 | INFO | Perf report for the last 5.13236 seconds: | 20260506 20:22:22.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:22:22.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:22.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:22.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:22.213 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 20:22:22.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:22.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:22.213 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 20:22:22.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:22.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:22.213 | INFO | Uploaded 36GiB to GPU in 2.02176s (17.8063GiB/s) | 20260506 20:22:22.213 | INFO | Downloaded 34GiB to GPU in 1.95534s (17.3882GiB/s) | 20260506 20:22:27.351 | INFO | Analyzing chunk frames = [4189184,4189696) | 20260506 20:22:27.351 | INFO | MemInfo: { | 20260506 20:22:27.351 | INFO | "gpu" : " 17 GB", | 20260506 20:22:27.351 | INFO | "huge" : "", | 20260506 20:22:27.351 | INFO | "huge_shared" : " 27 GB", | 20260506 20:22:27.351 | INFO | "malloc" : " 14 GB", | 20260506 20:22:27.351 | INFO | "pinned" : " 29 GB" | 20260506 20:22:27.351 | INFO | } | 20260506 20:22:27.351 | INFO | Perf report for the last 5.13822 seconds: | 20260506 20:22:27.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:22:27.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:27.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:27.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:27.351 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 20:22:27.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:27.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:27.351 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:22:27.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:27.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:27.351 | INFO | Uploaded 36GiB to GPU in 1.95848s (18.3816GiB/s) | 20260506 20:22:27.351 | INFO | Downloaded 34GiB to GPU in 1.99656s (17.0293GiB/s) | 20260506 20:22:32.490 | INFO | Analyzing chunk frames = [4189696,4190208) | 20260506 20:22:32.490 | INFO | MemInfo: { | 20260506 20:22:32.490 | INFO | "gpu" : " 17 GB", | 20260506 20:22:32.490 | INFO | "huge" : "", | 20260506 20:22:32.490 | INFO | "huge_shared" : " 27 GB", | 20260506 20:22:32.490 | INFO | "malloc" : " 14 GB", | 20260506 20:22:32.490 | INFO | "pinned" : " 29 GB" | 20260506 20:22:32.490 | INFO | } | 20260506 20:22:32.490 | INFO | Perf report for the last 5.13921 seconds: | 20260506 20:22:32.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:22:32.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:32.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:32.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:32.490 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 20:22:32.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:32.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:32.490 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:22:32.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:32.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:32.490 | INFO | Uploaded 36GiB to GPU in 1.99518s (18.0435GiB/s) | 20260506 20:22:32.490 | INFO | Downloaded 34GiB to GPU in 1.98334s (17.1428GiB/s) | 20260506 20:22:32.593 | INFO | Starting a round of Baz IO | 20260506 20:22:35.733 | INFO | Populating RTMetrics report | 20260506 20:22:35.734 | INFO | RTMetrics report populated | 20260506 20:22:35.734 | INFO | Opening tmp RTMetrics json file | 20260506 20:22:35.828 | INFO | Serializing RTMetrics to json | 20260506 20:22:35.885 | INFO | Serialization done, swapping tmp file now... | 20260506 20:22:35.888 | INFO | RTMetrics done | 20260506 20:22:36.128 | INFO | Finished a round of Baz IO: Wrote 1.982 GiB in 3.535 seconds | 20260506 20:22:36.128 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.27% / 0.00% / 65.73% | 20260506 20:22:36.128 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.762073% / 3.97364e-06% / 5.16574e-05% | 20260506 20:22:36.128 | INFO | Baz IO overall: | 20260506 20:22:36.128 | INFO | BytesWritten : 2212518660912 | 20260506 20:22:36.128 | INFO | NumEvents : 1068974455529 | 20260506 20:22:37.629 | INFO | Analyzing chunk frames = [4190208,4190720) | 20260506 20:22:37.629 | INFO | MemInfo: { | 20260506 20:22:37.629 | INFO | "gpu" : " 17 GB", | 20260506 20:22:37.629 | INFO | "huge" : "", | 20260506 20:22:37.629 | INFO | "huge_shared" : " 27 GB", | 20260506 20:22:37.629 | INFO | "malloc" : " 14 GB", | 20260506 20:22:37.629 | INFO | "pinned" : " 29 GB" | 20260506 20:22:37.629 | INFO | } | 20260506 20:22:37.629 | INFO | Perf report for the last 5.13905 seconds: | 20260506 20:22:37.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:22:37.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:37.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:37.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:37.629 | INFO | ANALYSIS : 62.5%, 2.9, 95.8ms, 0 | 20260506 20:22:37.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:37.629 | INFO | RT_METRICS : 45.9%, 1.0, 24.0ms, 0 | 20260506 20:22:37.629 | INFO | PRE_HQ : 60.0%, 1.8, 57.0ms, 0 | 20260506 20:22:37.629 | INFO | BAZWRITER : 68.4%, 4.7, 166.6ms, 0 | 20260506 20:22:37.629 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.5ms, 0 | 20260506 20:22:37.629 | INFO | Uploaded 36GiB to GPU in 2.03253s (17.7119GiB/s) | 20260506 20:22:37.629 | INFO | Downloaded 38GiB to GPU in 2.18692s (17.3761GiB/s) | 20260506 20:22:42.767 | INFO | Analyzing chunk frames = [4190720,4191232) | 20260506 20:22:42.767 | INFO | MemInfo: { | 20260506 20:22:42.767 | INFO | "gpu" : " 17 GB", | 20260506 20:22:42.767 | INFO | "huge" : "", | 20260506 20:22:42.767 | INFO | "huge_shared" : " 27 GB", | 20260506 20:22:42.767 | INFO | "malloc" : " 14 GB", | 20260506 20:22:42.767 | INFO | "pinned" : " 29 GB" | 20260506 20:22:42.767 | INFO | } | 20260506 20:22:42.767 | INFO | Perf report for the last 5.13818 seconds: | 20260506 20:22:42.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:22:42.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:42.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:42.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:42.767 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:22:42.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:42.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:42.767 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:22:42.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:42.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:42.768 | INFO | Uploaded 36GiB to GPU in 2.0055s (17.9506GiB/s) | 20260506 20:22:42.768 | INFO | Downloaded 34GiB to GPU in 1.92146s (17.6949GiB/s) | 20260506 20:22:47.905 | INFO | Analyzing chunk frames = [4191232,4191744) | 20260506 20:22:47.905 | INFO | MemInfo: { | 20260506 20:22:47.905 | INFO | "gpu" : " 17 GB", | 20260506 20:22:47.905 | INFO | "huge" : "", | 20260506 20:22:47.905 | INFO | "huge_shared" : " 27 GB", | 20260506 20:22:47.905 | INFO | "malloc" : " 14 GB", | 20260506 20:22:47.905 | INFO | "pinned" : " 29 GB" | 20260506 20:22:47.905 | INFO | } | 20260506 20:22:47.906 | INFO | Perf report for the last 5.13835 seconds: | 20260506 20:22:47.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:22:47.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:47.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:47.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:47.906 | INFO | ANALYSIS : 60.9%, 3.0, 94.2ms, 0 | 20260506 20:22:47.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:47.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:47.906 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:22:47.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:47.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:47.906 | INFO | Uploaded 36GiB to GPU in 1.98134s (18.1696GiB/s) | 20260506 20:22:47.906 | INFO | Downloaded 34GiB to GPU in 1.91735s (17.7328GiB/s) | 20260506 20:22:53.049 | INFO | Analyzing chunk frames = [4191744,4192256) | 20260506 20:22:53.049 | INFO | MemInfo: { | 20260506 20:22:53.049 | INFO | "gpu" : " 17 GB", | 20260506 20:22:53.049 | INFO | "huge" : "", | 20260506 20:22:53.049 | INFO | "huge_shared" : " 27 GB", | 20260506 20:22:53.049 | INFO | "malloc" : " 14 GB", | 20260506 20:22:53.049 | INFO | "pinned" : " 29 GB" | 20260506 20:22:53.049 | INFO | } | 20260506 20:22:53.049 | INFO | Perf report for the last 5.14352 seconds: | 20260506 20:22:53.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:22:53.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:53.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:53.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:53.049 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 20:22:53.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:53.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:53.049 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:22:53.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:53.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:53.049 | INFO | Uploaded 36GiB to GPU in 1.93156s (18.6377GiB/s) | 20260506 20:22:53.049 | INFO | Downloaded 34GiB to GPU in 1.86719s (18.2092GiB/s) | 20260506 20:22:58.190 | INFO | Analyzing chunk frames = [4192256,4192768) | 20260506 20:22:58.190 | INFO | MemInfo: { | 20260506 20:22:58.190 | INFO | "gpu" : " 17 GB", | 20260506 20:22:58.191 | INFO | "huge" : "", | 20260506 20:22:58.191 | INFO | "huge_shared" : " 27 GB", | 20260506 20:22:58.191 | INFO | "malloc" : " 14 GB", | 20260506 20:22:58.191 | INFO | "pinned" : " 29 GB" | 20260506 20:22:58.191 | INFO | } | 20260506 20:22:58.191 | INFO | Perf report for the last 5.14162 seconds: | 20260506 20:22:58.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:22:58.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:58.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:58.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:58.191 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 20:22:58.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:58.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:58.191 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:22:58.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:58.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:22:58.191 | INFO | Uploaded 36GiB to GPU in 2.01182s (17.8943GiB/s) | 20260506 20:22:58.191 | INFO | Downloaded 34GiB to GPU in 1.91596s (17.7456GiB/s) | 20260506 20:23:03.332 | INFO | Analyzing chunk frames = [4192768,4193280) | 20260506 20:23:03.332 | INFO | MemInfo: { | 20260506 20:23:03.332 | INFO | "gpu" : " 17 GB", | 20260506 20:23:03.332 | INFO | "huge" : "", | 20260506 20:23:03.332 | INFO | "huge_shared" : " 27 GB", | 20260506 20:23:03.332 | INFO | "malloc" : " 14 GB", | 20260506 20:23:03.332 | INFO | "pinned" : " 29 GB" | 20260506 20:23:03.332 | INFO | } | 20260506 20:23:03.332 | INFO | Perf report for the last 5.14098 seconds: | 20260506 20:23:03.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:23:03.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:03.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:03.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:03.332 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 20:23:03.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:03.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:03.332 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260506 20:23:03.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:03.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:03.332 | INFO | Uploaded 36GiB to GPU in 2.00909s (17.9186GiB/s) | 20260506 20:23:03.332 | INFO | Downloaded 34GiB to GPU in 1.93795s (17.5443GiB/s) | 20260506 20:23:08.466 | INFO | Analyzing chunk frames = [4193280,4193792) | 20260506 20:23:08.466 | INFO | MemInfo: { | 20260506 20:23:08.466 | INFO | "gpu" : " 17 GB", | 20260506 20:23:08.466 | INFO | "huge" : "", | 20260506 20:23:08.466 | INFO | "huge_shared" : " 27 GB", | 20260506 20:23:08.466 | INFO | "malloc" : " 14 GB", | 20260506 20:23:08.466 | INFO | "pinned" : " 29 GB" | 20260506 20:23:08.466 | INFO | } | 20260506 20:23:08.466 | INFO | Perf report for the last 5.13454 seconds: | 20260506 20:23:08.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:23:08.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:08.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:08.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:08.466 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 20:23:08.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:08.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:08.466 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 20:23:08.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:08.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:08.467 | INFO | Uploaded 36GiB to GPU in 1.96841s (18.2889GiB/s) | 20260506 20:23:08.467 | INFO | Downloaded 34GiB to GPU in 2.00725s (16.9386GiB/s) | 20260506 20:23:13.605 | INFO | Analyzing chunk frames = [4193792,4194304) | 20260506 20:23:13.605 | INFO | MemInfo: { | 20260506 20:23:13.605 | INFO | "gpu" : " 17 GB", | 20260506 20:23:13.605 | INFO | "huge" : "", | 20260506 20:23:13.605 | INFO | "huge_shared" : " 27 GB", | 20260506 20:23:13.605 | INFO | "malloc" : " 14 GB", | 20260506 20:23:13.605 | INFO | "pinned" : " 29 GB" | 20260506 20:23:13.605 | INFO | } | 20260506 20:23:13.606 | INFO | Perf report for the last 5.13919 seconds: | 20260506 20:23:13.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:23:13.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:13.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:13.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:13.606 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 20:23:13.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:13.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:13.606 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:23:13.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:13.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:13.606 | INFO | Uploaded 36GiB to GPU in 1.99286s (18.0645GiB/s) | 20260506 20:23:13.606 | INFO | Downloaded 34GiB to GPU in 1.97612s (17.2054GiB/s) | 20260506 20:23:13.606 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:23:13.606 | INFO | Total mem usage: 18 GiB | 20260506 20:23:13.606 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:23:13.606 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: -9.11%) | 20260506 20:23:13.606 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:23:13.606 | INFO | High Water Breakdown: | 20260506 20:23:13.606 | INFO | 62.4% : 10.3 GiB ( 90.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:23:13.606 | INFO | 37.5% : 6.19 GiB ( 92.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:23:13.606 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:23:13.606 | INFO | ------------------------------------------ | 20260506 20:23:13.606 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:23:13.606 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:23:13.606 | INFO | Total mem usage: 26.5 GiB | 20260506 20:23:13.606 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:23:13.606 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:23:13.606 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:23:13.606 | INFO | High Water Breakdown: | 20260506 20:23:13.606 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:23:13.606 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:23:13.606 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:23:13.606 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:23:13.606 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:23:13.606 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:23:13.606 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:23:13.606 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:23:13.606 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:23:13.606 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:23:13.606 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:23:13.606 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:23:13.607 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:23:13.607 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:23:13.607 | INFO | ------------------------------------------ | 20260506 20:23:13.607 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:23:13.607 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:23:13.607 | INFO | Total mem usage: 25.6 GiB | 20260506 20:23:13.607 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:23:13.607 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 20:23:13.607 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 20:23:13.607 | INFO | High Water Breakdown: | 20260506 20:23:13.607 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 20:23:13.607 | INFO | ------------------------------------------ | 20260506 20:23:13.607 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:23:13.607 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:23:13.607 | INFO | Total mem usage: 15.7 GiB | 20260506 20:23:13.607 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:23:13.607 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:23:13.607 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 20:23:13.607 | INFO | High Water Breakdown: | 20260506 20:23:13.607 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:23:13.607 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:23:13.607 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:23:13.607 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:23:13.607 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:23:13.607 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:23:13.607 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:23:13.607 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:23:13.607 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:23:13.607 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:23:13.607 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:23:13.607 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:23:13.607 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:23:13.607 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:23:13.607 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:23:13.607 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:23:13.607 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:23:13.607 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:23:13.607 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:23:13.607 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:23:13.607 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:23:13.607 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:23:13.607 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:23:13.607 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:23:13.607 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:23:13.607 | INFO | ------------------------------------------ | 20260506 20:23:13.607 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:23:13.607 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:23:13.607 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:23:13.607 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:23:13.607 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.36e+01 / 2.45e+00 | 20260506 20:23:13.607 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.00e-01, 9.42e-03 / 4.49e-03 | 20260506 20:23:13.607 | INFO | PerfReport: 11.64%, Baseline -- 3.63e+00 / 4.42e+00, 5.59e-01 / 3.54e-01 | 20260506 20:23:13.607 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.47e+00, 1.86e-01 / 1.26e-01 | 20260506 20:23:13.607 | INFO | PerfReport: 16.02%, DME -- 3.99e+01 / 4.45e+01, 4.18e+00 / 2.71e+00 | 20260506 20:23:13.607 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.39e+00 / 1.75e+00 | 20260506 20:23:13.607 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.23e+00 / 2.73e+00, 3.76e-01 / 2.30e-01 | 20260506 20:23:13.607 | INFO | PerfReport: 3.33%, Metrics -- 1.04e+00 / 1.39e+00, 2.78e-01 / 1.64e-01 | 20260506 20:23:13.607 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.55e-01 / 4.02e-01, 5.54e-02 / 1.95e-02 | 20260506 20:23:13.607 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:23:13.607 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:23:13.607 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:23:13.607 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 20:23:13.607 | INFO | PerfReport: 21.42%, Upload -- 2.07e+01 / 0.00e+00, 6.86e+00 / 0.00e+00 | 20260506 20:23:13.607 | INFO | PerfReport: 21.20%, Download -- 2.05e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 20:23:13.607 | INFO | PerfReport: 32.22%, Compute -- 3.12e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 20:23:13.607 | INFO | PerfReport: 1.28%, LockUpload -- 1.23e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260506 20:23:13.607 | INFO | PerfReport: 1.18%, LockDownload -- 1.14e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260506 20:23:13.607 | INFO | PerfReport: 22.68%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 20:23:13.607 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260506 20:23:13.704 | INFO | Starting a round of Baz IO | 20260506 20:23:16.839 | INFO | Populating RTMetrics report | 20260506 20:23:16.839 | INFO | RTMetrics report populated | 20260506 20:23:16.839 | INFO | Opening tmp RTMetrics json file | 20260506 20:23:16.843 | INFO | Serializing RTMetrics to json | 20260506 20:23:16.872 | INFO | Serialization done, swapping tmp file now... | 20260506 20:23:16.886 | INFO | RTMetrics done | 20260506 20:23:16.992 | INFO | Finished a round of Baz IO: Wrote 1.982 GiB in 3.287 seconds | 20260506 20:23:16.992 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.27% / 0.00% / 65.73% | 20260506 20:23:16.992 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.759792% / 3.97364e-06% / 3.97364e-05% | 20260506 20:23:16.992 | INFO | Baz IO overall: | 20260506 20:23:16.992 | INFO | BytesWritten : 2214646378708 | 20260506 20:23:16.992 | INFO | NumEvents : 1069995393847 | 20260506 20:23:18.746 | INFO | Analyzing chunk frames = [4194304,4194816) | 20260506 20:23:18.746 | INFO | MemInfo: { | 20260506 20:23:18.746 | INFO | "gpu" : " 17 GB", | 20260506 20:23:18.746 | INFO | "huge" : "", | 20260506 20:23:18.746 | INFO | "huge_shared" : " 27 GB", | 20260506 20:23:18.746 | INFO | "malloc" : " 14 GB", | 20260506 20:23:18.746 | INFO | "pinned" : " 29 GB" | 20260506 20:23:18.746 | INFO | } | 20260506 20:23:18.746 | INFO | Perf report for the last 5.14077 seconds: | 20260506 20:23:18.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:23:18.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:18.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:18.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:18.747 | INFO | ANALYSIS : 62.4%, 2.9, 95.1ms, 0 | 20260506 20:23:18.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:18.747 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 20:23:18.747 | INFO | PRE_HQ : 60.1%, 1.8, 57.3ms, 0 | 20260506 20:23:18.747 | INFO | BAZWRITER : 63.6%, 4.6, 151.5ms, 0 | 20260506 20:23:18.747 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260506 20:23:18.747 | INFO | Uploaded 36GiB to GPU in 2.04223s (17.6278GiB/s) | 20260506 20:23:18.747 | INFO | Downloaded 38GiB to GPU in 2.18879s (17.3612GiB/s) | 20260506 20:23:23.880 | INFO | Analyzing chunk frames = [4194816,4195328) | 20260506 20:23:23.880 | INFO | MemInfo: { | 20260506 20:23:23.880 | INFO | "gpu" : " 17 GB", | 20260506 20:23:23.880 | INFO | "huge" : "", | 20260506 20:23:23.880 | INFO | "huge_shared" : " 27 GB", | 20260506 20:23:23.881 | INFO | "malloc" : " 14 GB", | 20260506 20:23:23.881 | INFO | "pinned" : " 29 GB" | 20260506 20:23:23.881 | INFO | } | 20260506 20:23:23.881 | INFO | Perf report for the last 5.1343 seconds: | 20260506 20:23:23.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:23:23.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:23.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:23.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:23.881 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:23:23.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:23.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:23.881 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:23:23.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:23.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:23.881 | INFO | Uploaded 36GiB to GPU in 2.00544s (17.9511GiB/s) | 20260506 20:23:23.881 | INFO | Downloaded 34GiB to GPU in 1.9236s (17.6752GiB/s) | 20260506 20:23:29.025 | INFO | Analyzing chunk frames = [4195328,4195840) | 20260506 20:23:29.025 | INFO | MemInfo: { | 20260506 20:23:29.025 | INFO | "gpu" : " 17 GB", | 20260506 20:23:29.025 | INFO | "huge" : "", | 20260506 20:23:29.025 | INFO | "huge_shared" : " 27 GB", | 20260506 20:23:29.025 | INFO | "malloc" : " 14 GB", | 20260506 20:23:29.025 | INFO | "pinned" : " 29 GB" | 20260506 20:23:29.025 | INFO | } | 20260506 20:23:29.025 | INFO | Perf report for the last 5.14462 seconds: | 20260506 20:23:29.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:23:29.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:29.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:29.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:29.025 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 20:23:29.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:29.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:29.025 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:23:29.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:29.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:29.026 | INFO | Uploaded 36GiB to GPU in 1.98356s (18.1492GiB/s) | 20260506 20:23:29.026 | INFO | Downloaded 34GiB to GPU in 1.91621s (17.7434GiB/s) | 20260506 20:23:34.165 | INFO | Analyzing chunk frames = [4195840,4196352) | 20260506 20:23:34.165 | INFO | MemInfo: { | 20260506 20:23:34.165 | INFO | "gpu" : " 17 GB", | 20260506 20:23:34.165 | INFO | "huge" : "", | 20260506 20:23:34.165 | INFO | "huge_shared" : " 27 GB", | 20260506 20:23:34.165 | INFO | "malloc" : " 14 GB", | 20260506 20:23:34.165 | INFO | "pinned" : " 29 GB" | 20260506 20:23:34.165 | INFO | } | 20260506 20:23:34.166 | INFO | Perf report for the last 5.14028 seconds: | 20260506 20:23:34.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:23:34.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:34.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:34.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:34.166 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 20:23:34.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:34.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:34.166 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:23:34.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:34.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:34.166 | INFO | Uploaded 36GiB to GPU in 1.946s (18.4995GiB/s) | 20260506 20:23:34.166 | INFO | Downloaded 34GiB to GPU in 1.87071s (18.175GiB/s) | 20260506 20:23:39.301 | INFO | Analyzing chunk frames = [4196352,4196864) | 20260506 20:23:39.301 | INFO | MemInfo: { | 20260506 20:23:39.301 | INFO | "gpu" : " 17 GB", | 20260506 20:23:39.301 | INFO | "huge" : "", | 20260506 20:23:39.301 | INFO | "huge_shared" : " 27 GB", | 20260506 20:23:39.301 | INFO | "malloc" : " 14 GB", | 20260506 20:23:39.301 | INFO | "pinned" : " 29 GB" | 20260506 20:23:39.301 | INFO | } | 20260506 20:23:39.301 | INFO | Perf report for the last 5.13589 seconds: | 20260506 20:23:39.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:23:39.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:39.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:39.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:39.302 | INFO | ANALYSIS : 60.5%, 3.0, 94.0ms, 0 | 20260506 20:23:39.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:39.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:39.302 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:23:39.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:39.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:39.302 | INFO | Uploaded 36GiB to GPU in 2.01326s (17.8814GiB/s) | 20260506 20:23:39.302 | INFO | Downloaded 34GiB to GPU in 1.91029s (17.7984GiB/s) | 20260506 20:23:44.442 | INFO | Analyzing chunk frames = [4196864,4197376) | 20260506 20:23:44.442 | INFO | MemInfo: { | 20260506 20:23:44.442 | INFO | "gpu" : " 17 GB", | 20260506 20:23:44.442 | INFO | "huge" : "", | 20260506 20:23:44.442 | INFO | "huge_shared" : " 27 GB", | 20260506 20:23:44.442 | INFO | "malloc" : " 14 GB", | 20260506 20:23:44.442 | INFO | "pinned" : " 29 GB" | 20260506 20:23:44.442 | INFO | } | 20260506 20:23:44.442 | INFO | Perf report for the last 5.14066 seconds: | 20260506 20:23:44.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:23:44.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:44.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:44.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:44.442 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 20:23:44.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:44.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:44.442 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 20:23:44.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:44.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:44.442 | INFO | Uploaded 36GiB to GPU in 2.00754s (17.9324GiB/s) | 20260506 20:23:44.442 | INFO | Downloaded 34GiB to GPU in 1.93892s (17.5355GiB/s) | 20260506 20:23:49.578 | INFO | Analyzing chunk frames = [4197376,4197888) | 20260506 20:23:49.578 | INFO | MemInfo: { | 20260506 20:23:49.578 | INFO | "gpu" : " 17 GB", | 20260506 20:23:49.578 | INFO | "huge" : "", | 20260506 20:23:49.578 | INFO | "huge_shared" : " 27 GB", | 20260506 20:23:49.578 | INFO | "malloc" : " 14 GB", | 20260506 20:23:49.578 | INFO | "pinned" : " 29 GB" | 20260506 20:23:49.579 | INFO | } | 20260506 20:23:49.579 | INFO | Perf report for the last 5.13643 seconds: | 20260506 20:23:49.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:23:49.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:49.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:49.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:49.579 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260506 20:23:49.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:49.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:49.579 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 20:23:49.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:49.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:49.579 | INFO | Uploaded 36GiB to GPU in 1.9949s (18.046GiB/s) | 20260506 20:23:49.579 | INFO | Downloaded 34GiB to GPU in 2.00598s (16.9493GiB/s) | 20260506 20:23:54.715 | INFO | Analyzing chunk frames = [4197888,4198400) | 20260506 20:23:54.715 | INFO | MemInfo: { | 20260506 20:23:54.715 | INFO | "gpu" : " 17 GB", | 20260506 20:23:54.715 | INFO | "huge" : "", | 20260506 20:23:54.715 | INFO | "huge_shared" : " 27 GB", | 20260506 20:23:54.715 | INFO | "malloc" : " 14 GB", | 20260506 20:23:54.715 | INFO | "pinned" : " 29 GB" | 20260506 20:23:54.715 | INFO | } | 20260506 20:23:54.715 | INFO | Perf report for the last 5.13639 seconds: | 20260506 20:23:54.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:23:54.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:54.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:54.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:54.715 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 20:23:54.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:54.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:54.715 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 20:23:54.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:54.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:54.715 | INFO | Uploaded 36GiB to GPU in 2.03861s (17.6591GiB/s) | 20260506 20:23:54.715 | INFO | Downloaded 34GiB to GPU in 1.99452s (17.0467GiB/s) | 20260506 20:23:54.833 | INFO | Starting a round of Baz IO | 20260506 20:23:58.001 | INFO | Populating RTMetrics report | 20260506 20:23:58.002 | INFO | RTMetrics report populated | 20260506 20:23:58.002 | INFO | Opening tmp RTMetrics json file | 20260506 20:23:58.033 | INFO | Serializing RTMetrics to json | 20260506 20:23:58.097 | INFO | Serialization done, swapping tmp file now... | 20260506 20:23:58.121 | INFO | RTMetrics done | 20260506 20:23:58.376 | INFO | Finished a round of Baz IO: Wrote 1.982 GiB in 3.543 seconds | 20260506 20:23:58.376 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.26% / 0.00% / 65.73% | 20260506 20:23:58.376 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.758866% / 7.94729e-06% / 5.96046e-05% | 20260506 20:23:58.376 | INFO | Baz IO overall: | 20260506 20:23:58.377 | INFO | BytesWritten : 2216774344179 | 20260506 20:23:58.377 | INFO | NumEvents : 1071016654786 | 20260506 20:23:59.856 | INFO | Analyzing chunk frames = [4198400,4198912) | 20260506 20:23:59.856 | INFO | MemInfo: { | 20260506 20:23:59.856 | INFO | "gpu" : " 17 GB", | 20260506 20:23:59.856 | INFO | "huge" : "", | 20260506 20:23:59.856 | INFO | "huge_shared" : " 27 GB", | 20260506 20:23:59.856 | INFO | "malloc" : " 14 GB", | 20260506 20:23:59.856 | INFO | "pinned" : " 29 GB" | 20260506 20:23:59.856 | INFO | } | 20260506 20:23:59.856 | INFO | Perf report for the last 5.1408 seconds: | 20260506 20:23:59.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:23:59.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:59.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:59.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:59.856 | INFO | ANALYSIS : 63.0%, 2.8, 93.9ms, 0 | 20260506 20:23:59.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:23:59.856 | INFO | RT_METRICS : 48.1%, 1.0, 25.2ms, 0 | 20260506 20:23:59.856 | INFO | PRE_HQ : 60.5%, 1.9, 61.3ms, 0 | 20260506 20:23:59.856 | INFO | BAZWRITER : 68.6%, 4.8, 168.6ms, 0 | 20260506 20:23:59.856 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.1ms, 0 | 20260506 20:23:59.856 | INFO | Uploaded 36GiB to GPU in 2.04503s (17.6037GiB/s) | 20260506 20:23:59.856 | INFO | Downloaded 38GiB to GPU in 2.18416s (17.398GiB/s) | 20260506 20:24:04.999 | INFO | Analyzing chunk frames = [4198912,4199424) | 20260506 20:24:04.999 | INFO | MemInfo: { | 20260506 20:24:04.999 | INFO | "gpu" : " 17 GB", | 20260506 20:24:04.999 | INFO | "huge" : "", | 20260506 20:24:04.999 | INFO | "huge_shared" : " 27 GB", | 20260506 20:24:04.999 | INFO | "malloc" : " 14 GB", | 20260506 20:24:04.999 | INFO | "pinned" : " 29 GB" | 20260506 20:24:04.999 | INFO | } | 20260506 20:24:04.999 | INFO | Perf report for the last 5.14363 seconds: | 20260506 20:24:04.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:24:04.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:04.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:04.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:05.000 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 20:24:05.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:05.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:05.000 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:24:05.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:05.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:05.000 | INFO | Uploaded 36GiB to GPU in 1.99999s (18.0001GiB/s) | 20260506 20:24:05.000 | INFO | Downloaded 34GiB to GPU in 1.92444s (17.6675GiB/s) | 20260506 20:24:10.142 | INFO | Analyzing chunk frames = [4199424,4199936) | 20260506 20:24:10.142 | INFO | MemInfo: { | 20260506 20:24:10.142 | INFO | "gpu" : " 17 GB", | 20260506 20:24:10.142 | INFO | "huge" : "", | 20260506 20:24:10.142 | INFO | "huge_shared" : " 27 GB", | 20260506 20:24:10.142 | INFO | "malloc" : " 14 GB", | 20260506 20:24:10.142 | INFO | "pinned" : " 29 GB" | 20260506 20:24:10.142 | INFO | } | 20260506 20:24:10.142 | INFO | Perf report for the last 5.14266 seconds: | 20260506 20:24:10.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:24:10.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:10.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:10.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:10.142 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 20:24:10.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:10.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:10.142 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:24:10.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:10.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:10.142 | INFO | Uploaded 36GiB to GPU in 1.97586s (18.2199GiB/s) | 20260506 20:24:10.142 | INFO | Downloaded 34GiB to GPU in 1.92051s (17.7037GiB/s) | 20260506 20:24:15.284 | INFO | Analyzing chunk frames = [4199936,4200448) | 20260506 20:24:15.284 | INFO | MemInfo: { | 20260506 20:24:15.284 | INFO | "gpu" : " 17 GB", | 20260506 20:24:15.284 | INFO | "huge" : "", | 20260506 20:24:15.284 | INFO | "huge_shared" : " 27 GB", | 20260506 20:24:15.284 | INFO | "malloc" : " 14 GB", | 20260506 20:24:15.284 | INFO | "pinned" : " 29 GB" | 20260506 20:24:15.284 | INFO | } | 20260506 20:24:15.284 | INFO | Perf report for the last 5.14201 seconds: | 20260506 20:24:15.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:24:15.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:15.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:15.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:15.284 | INFO | ANALYSIS : 60.3%, 3.0, 93.6ms, 0 | 20260506 20:24:15.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:15.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:15.284 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:24:15.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:15.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:15.284 | INFO | Uploaded 36GiB to GPU in 2.0162s (17.8554GiB/s) | 20260506 20:24:15.284 | INFO | Downloaded 34GiB to GPU in 1.90781s (17.8215GiB/s) | 20260506 20:24:20.423 | INFO | Analyzing chunk frames = [4200448,4200960) | 20260506 20:24:20.423 | INFO | MemInfo: { | 20260506 20:24:20.423 | INFO | "gpu" : " 17 GB", | 20260506 20:24:20.424 | INFO | "huge" : "", | 20260506 20:24:20.424 | INFO | "huge_shared" : " 27 GB", | 20260506 20:24:20.424 | INFO | "malloc" : " 14 GB", | 20260506 20:24:20.424 | INFO | "pinned" : " 29 GB" | 20260506 20:24:20.424 | INFO | } | 20260506 20:24:20.424 | INFO | Perf report for the last 5.13954 seconds: | 20260506 20:24:20.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:24:20.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:20.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:20.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:20.424 | INFO | ANALYSIS : 60.2%, 3.0, 92.8ms, 0 | 20260506 20:24:20.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:20.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:20.424 | INFO | PRE_HQ : 26.3%, 1.0, 13.5ms, 0 | 20260506 20:24:20.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:20.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:20.424 | INFO | Uploaded 36GiB to GPU in 1.97718s (18.2078GiB/s) | 20260506 20:24:20.424 | INFO | Downloaded 34GiB to GPU in 1.92546s (17.6581GiB/s) | 20260506 20:24:25.556 | INFO | Analyzing chunk frames = [4200960,4201472) | 20260506 20:24:25.556 | INFO | MemInfo: { | 20260506 20:24:25.556 | INFO | "gpu" : " 17 GB", | 20260506 20:24:25.556 | INFO | "huge" : "", | 20260506 20:24:25.556 | INFO | "huge_shared" : " 27 GB", | 20260506 20:24:25.556 | INFO | "malloc" : " 14 GB", | 20260506 20:24:25.556 | INFO | "pinned" : " 29 GB" | 20260506 20:24:25.556 | INFO | } | 20260506 20:24:25.556 | INFO | Perf report for the last 5.13275 seconds: | 20260506 20:24:25.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:24:25.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:25.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:25.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:25.557 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 20:24:25.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:25.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:25.557 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:24:25.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:25.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:25.557 | INFO | Uploaded 36GiB to GPU in 1.99428s (18.0517GiB/s) | 20260506 20:24:25.557 | INFO | Downloaded 34GiB to GPU in 1.94636s (17.4685GiB/s) | 20260506 20:24:30.692 | INFO | Analyzing chunk frames = [4201472,4201984) | 20260506 20:24:30.692 | INFO | MemInfo: { | 20260506 20:24:30.692 | INFO | "gpu" : " 17 GB", | 20260506 20:24:30.692 | INFO | "huge" : "", | 20260506 20:24:30.692 | INFO | "huge_shared" : " 27 GB", | 20260506 20:24:30.692 | INFO | "malloc" : " 14 GB", | 20260506 20:24:30.692 | INFO | "pinned" : " 29 GB" | 20260506 20:24:30.692 | INFO | } | 20260506 20:24:30.692 | INFO | Perf report for the last 5.1357 seconds: | 20260506 20:24:30.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:24:30.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:30.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:30.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:30.692 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:24:30.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:30.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:30.692 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 20:24:30.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:30.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:30.692 | INFO | Uploaded 36GiB to GPU in 1.95496s (18.4147GiB/s) | 20260506 20:24:30.692 | INFO | Downloaded 34GiB to GPU in 1.99901s (17.0084GiB/s) | 20260506 20:24:35.832 | INFO | Analyzing chunk frames = [4201984,4202496) | 20260506 20:24:35.832 | INFO | MemInfo: { | 20260506 20:24:35.832 | INFO | "gpu" : " 17 GB", | 20260506 20:24:35.832 | INFO | "huge" : "", | 20260506 20:24:35.832 | INFO | "huge_shared" : " 27 GB", | 20260506 20:24:35.832 | INFO | "malloc" : " 14 GB", | 20260506 20:24:35.832 | INFO | "pinned" : " 29 GB" | 20260506 20:24:35.832 | INFO | } | 20260506 20:24:35.833 | INFO | Perf report for the last 5.14039 seconds: | 20260506 20:24:35.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:24:35.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:35.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:35.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:35.833 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 20:24:35.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:35.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:35.833 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 20:24:35.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:35.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:35.833 | INFO | Uploaded 36GiB to GPU in 1.99344s (18.0592GiB/s) | 20260506 20:24:35.833 | INFO | Downloaded 34GiB to GPU in 1.99297s (17.06GiB/s) | 20260506 20:24:35.833 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:24:35.833 | INFO | Total mem usage: 18 GiB | 20260506 20:24:35.833 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:24:35.833 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.7%) | 20260506 20:24:35.833 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:24:35.833 | INFO | High Water Breakdown: | 20260506 20:24:35.833 | INFO | 61.4% : 9.56 GiB ( 83.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:24:35.833 | INFO | 38.4% : 5.98 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:24:35.833 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:24:35.833 | INFO | ------------------------------------------ | 20260506 20:24:35.833 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:24:35.833 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:24:35.833 | INFO | Total mem usage: 26.5 GiB | 20260506 20:24:35.833 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:24:35.833 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 20:24:35.833 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260506 20:24:35.833 | INFO | High Water Breakdown: | 20260506 20:24:35.833 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:24:35.833 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:24:35.833 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:24:35.833 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:24:35.833 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:24:35.833 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:24:35.833 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:24:35.833 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:24:35.833 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:24:35.833 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:24:35.833 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:24:35.833 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:24:35.833 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:24:35.833 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:24:35.833 | INFO | ------------------------------------------ | 20260506 20:24:35.833 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:24:35.833 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:24:35.833 | INFO | Total mem usage: 25.6 GiB | 20260506 20:24:35.833 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:24:35.833 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:24:35.833 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:24:35.834 | INFO | High Water Breakdown: | 20260506 20:24:35.834 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:24:35.834 | INFO | ------------------------------------------ | 20260506 20:24:35.834 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:24:35.834 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:24:35.834 | INFO | Total mem usage: 15.7 GiB | 20260506 20:24:35.834 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:24:35.834 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:24:35.834 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:24:35.834 | INFO | High Water Breakdown: | 20260506 20:24:35.834 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:24:35.834 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:24:35.834 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:24:35.834 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:24:35.834 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:24:35.834 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:24:35.834 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:24:35.834 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:24:35.834 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:24:35.834 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:24:35.834 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:24:35.834 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:24:35.834 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:24:35.834 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:24:35.834 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:24:35.834 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:24:35.834 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:24:35.834 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:24:35.834 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:24:35.834 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:24:35.834 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:24:35.834 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:24:35.834 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:24:35.834 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:24:35.834 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:24:35.834 | INFO | ------------------------------------------ | 20260506 20:24:35.834 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:24:35.834 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:24:35.834 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:24:35.834 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:24:35.834 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.62e+01, 1.34e+01 / 2.41e+00 | 20260506 20:24:35.834 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.00e-01, 9.85e-03 / 6.39e-03 | 20260506 20:24:35.834 | INFO | PerfReport: 11.63%, Baseline -- 3.63e+00 / 4.34e+00, 5.58e-01 / 4.33e-01 | 20260506 20:24:35.834 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.43e+00, 1.83e-01 / 1.53e-01 | 20260506 20:24:35.834 | INFO | PerfReport: 16.06%, DME -- 4.01e+01 / 4.51e+01, 4.39e+00 / 2.61e+00 | 20260506 20:24:35.834 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.42e+00 / 1.96e+00 | 20260506 20:24:35.834 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.23e+00 / 2.75e+00, 3.78e-01 / 2.34e-01 | 20260506 20:24:35.834 | INFO | PerfReport: 3.32%, Metrics -- 1.04e+00 / 1.37e+00, 2.73e-01 / 1.47e-01 | 20260506 20:24:35.834 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.55e-01 / 3.99e-01, 5.58e-02 / 1.45e-02 | 20260506 20:24:35.834 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:24:35.834 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:24:35.834 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:24:35.834 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 20:24:35.834 | INFO | PerfReport: 21.51%, Upload -- 2.08e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260506 20:24:35.834 | INFO | PerfReport: 21.22%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 20:24:35.834 | INFO | PerfReport: 32.26%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 20:24:35.834 | INFO | PerfReport: 1.34%, LockUpload -- 1.30e+00 / 0.00e+00, 5.06e+00 / 0.00e+00 | 20260506 20:24:35.834 | INFO | PerfReport: 1.18%, LockDownload -- 1.15e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260506 20:24:35.834 | INFO | PerfReport: 22.47%, LockCompute -- 2.17e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 20:24:35.834 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.20e-02 / 0.00e+00 | 20260506 20:24:35.949 | INFO | Starting a round of Baz IO | 20260506 20:24:39.075 | INFO | Populating RTMetrics report | 20260506 20:24:39.075 | INFO | RTMetrics report populated | 20260506 20:24:39.075 | INFO | Opening tmp RTMetrics json file | 20260506 20:24:39.079 | INFO | Serializing RTMetrics to json | 20260506 20:24:39.107 | INFO | Serialization done, swapping tmp file now... | 20260506 20:24:39.110 | INFO | RTMetrics done | 20260506 20:24:39.209 | INFO | Finished a round of Baz IO: Wrote 1.98 GiB in 3.259 seconds | 20260506 20:24:39.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.26% / 0.00% / 65.74% | 20260506 20:24:39.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.75868% / 7.94729e-06% / 8.34465e-05% | 20260506 20:24:39.209 | INFO | Baz IO overall: | 20260506 20:24:39.209 | INFO | BytesWritten : 2218900742107 | 20260506 20:24:39.209 | INFO | NumEvents : 1072036936295 | 20260506 20:24:40.972 | INFO | Analyzing chunk frames = [4202496,4203008) | 20260506 20:24:40.972 | INFO | MemInfo: { | 20260506 20:24:40.972 | INFO | "gpu" : " 17 GB", | 20260506 20:24:40.972 | INFO | "huge" : "", | 20260506 20:24:40.972 | INFO | "huge_shared" : " 27 GB", | 20260506 20:24:40.972 | INFO | "malloc" : " 14 GB", | 20260506 20:24:40.972 | INFO | "pinned" : " 29 GB" | 20260506 20:24:40.972 | INFO | } | 20260506 20:24:40.972 | INFO | Perf report for the last 5.1393 seconds: | 20260506 20:24:40.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:24:40.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:40.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:40.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:40.972 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 20:24:40.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:40.972 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 20:24:40.972 | INFO | PRE_HQ : 59.8%, 1.9, 57.5ms, 0 | 20260506 20:24:40.972 | INFO | BAZWRITER : 63.1%, 4.4, 143.2ms, 0 | 20260506 20:24:40.972 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 20:24:40.972 | INFO | Uploaded 36GiB to GPU in 2.03513s (17.6893GiB/s) | 20260506 20:24:40.972 | INFO | Downloaded 38GiB to GPU in 2.22284s (17.0953GiB/s) | 20260506 20:24:46.117 | INFO | Analyzing chunk frames = [4203008,4203520) | 20260506 20:24:46.117 | INFO | MemInfo: { | 20260506 20:24:46.117 | INFO | "gpu" : " 17 GB", | 20260506 20:24:46.117 | INFO | "huge" : "", | 20260506 20:24:46.117 | INFO | "huge_shared" : " 27 GB", | 20260506 20:24:46.117 | INFO | "malloc" : " 14 GB", | 20260506 20:24:46.117 | INFO | "pinned" : " 29 GB" | 20260506 20:24:46.117 | INFO | } | 20260506 20:24:46.117 | INFO | Perf report for the last 5.14559 seconds: | 20260506 20:24:46.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:24:46.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:46.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:46.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:46.118 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 20:24:46.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:46.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:46.118 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:24:46.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:46.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:46.118 | INFO | Uploaded 36GiB to GPU in 1.99147s (18.0771GiB/s) | 20260506 20:24:46.118 | INFO | Downloaded 34GiB to GPU in 1.9265s (17.6486GiB/s) | 20260506 20:24:51.257 | INFO | Analyzing chunk frames = [4203520,4204032) | 20260506 20:24:51.257 | INFO | MemInfo: { | 20260506 20:24:51.257 | INFO | "gpu" : " 17 GB", | 20260506 20:24:51.257 | INFO | "huge" : "", | 20260506 20:24:51.257 | INFO | "huge_shared" : " 27 GB", | 20260506 20:24:51.257 | INFO | "malloc" : " 14 GB", | 20260506 20:24:51.257 | INFO | "pinned" : " 29 GB" | 20260506 20:24:51.257 | INFO | } | 20260506 20:24:51.257 | INFO | Perf report for the last 5.13982 seconds: | 20260506 20:24:51.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:24:51.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:51.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:51.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:51.257 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 20:24:51.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:51.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:51.257 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:24:51.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:51.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:51.258 | INFO | Uploaded 36GiB to GPU in 1.96728s (18.2994GiB/s) | 20260506 20:24:51.258 | INFO | Downloaded 34GiB to GPU in 1.92315s (17.6794GiB/s) | 20260506 20:24:56.393 | INFO | Analyzing chunk frames = [4204032,4204544) | 20260506 20:24:56.393 | INFO | MemInfo: { | 20260506 20:24:56.393 | INFO | "gpu" : " 17 GB", | 20260506 20:24:56.393 | INFO | "huge" : "", | 20260506 20:24:56.394 | INFO | "huge_shared" : " 27 GB", | 20260506 20:24:56.394 | INFO | "malloc" : " 14 GB", | 20260506 20:24:56.394 | INFO | "pinned" : " 29 GB" | 20260506 20:24:56.394 | INFO | } | 20260506 20:24:56.394 | INFO | Perf report for the last 5.13641 seconds: | 20260506 20:24:56.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:24:56.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:56.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:56.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:56.394 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 20:24:56.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:56.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:56.394 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:24:56.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:56.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:24:56.394 | INFO | Uploaded 36GiB to GPU in 1.96069s (18.3609GiB/s) | 20260506 20:24:56.394 | INFO | Downloaded 34GiB to GPU in 1.87716s (18.1125GiB/s) | 20260506 20:25:01.531 | INFO | Analyzing chunk frames = [4204544,4205056) | 20260506 20:25:01.531 | INFO | MemInfo: { | 20260506 20:25:01.531 | INFO | "gpu" : " 17 GB", | 20260506 20:25:01.531 | INFO | "huge" : "", | 20260506 20:25:01.531 | INFO | "huge_shared" : " 27 GB", | 20260506 20:25:01.531 | INFO | "malloc" : " 14 GB", | 20260506 20:25:01.531 | INFO | "pinned" : " 29 GB" | 20260506 20:25:01.531 | INFO | } | 20260506 20:25:01.531 | INFO | Perf report for the last 5.13741 seconds: | 20260506 20:25:01.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:25:01.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:01.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:01.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:01.531 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260506 20:25:01.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:01.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:01.531 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 20:25:01.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:01.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:01.531 | INFO | Uploaded 36GiB to GPU in 2.01341s (17.8801GiB/s) | 20260506 20:25:01.532 | INFO | Downloaded 34GiB to GPU in 1.92249s (17.6854GiB/s) | 20260506 20:25:06.668 | INFO | Analyzing chunk frames = [4205056,4205568) | 20260506 20:25:06.668 | INFO | MemInfo: { | 20260506 20:25:06.668 | INFO | "gpu" : " 17 GB", | 20260506 20:25:06.668 | INFO | "huge" : "", | 20260506 20:25:06.668 | INFO | "huge_shared" : " 27 GB", | 20260506 20:25:06.669 | INFO | "malloc" : " 14 GB", | 20260506 20:25:06.669 | INFO | "pinned" : " 29 GB" | 20260506 20:25:06.669 | INFO | } | 20260506 20:25:06.669 | INFO | Perf report for the last 5.13744 seconds: | 20260506 20:25:06.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:25:06.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:06.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:06.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:06.669 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 20:25:06.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:06.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:06.669 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:25:06.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:06.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:06.669 | INFO | Uploaded 36GiB to GPU in 2.00216s (17.9806GiB/s) | 20260506 20:25:06.669 | INFO | Downloaded 34GiB to GPU in 1.94147s (17.5125GiB/s) | 20260506 20:25:11.809 | INFO | Analyzing chunk frames = [4205568,4206080) | 20260506 20:25:11.809 | INFO | MemInfo: { | 20260506 20:25:11.809 | INFO | "gpu" : " 17 GB", | 20260506 20:25:11.809 | INFO | "huge" : "", | 20260506 20:25:11.809 | INFO | "huge_shared" : " 27 GB", | 20260506 20:25:11.809 | INFO | "malloc" : " 14 GB", | 20260506 20:25:11.809 | INFO | "pinned" : " 29 GB" | 20260506 20:25:11.809 | INFO | } | 20260506 20:25:11.809 | INFO | Perf report for the last 5.14048 seconds: | 20260506 20:25:11.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:25:11.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:11.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:11.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:11.809 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:25:11.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:11.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:11.809 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:25:11.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:11.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:11.809 | INFO | Uploaded 36GiB to GPU in 1.95244s (18.4385GiB/s) | 20260506 20:25:11.809 | INFO | Downloaded 34GiB to GPU in 1.99282s (17.0613GiB/s) | 20260506 20:25:16.945 | INFO | Analyzing chunk frames = [4206080,4206592) | 20260506 20:25:16.945 | INFO | MemInfo: { | 20260506 20:25:16.945 | INFO | "gpu" : " 17 GB", | 20260506 20:25:16.945 | INFO | "huge" : "", | 20260506 20:25:16.945 | INFO | "huge_shared" : " 27 GB", | 20260506 20:25:16.945 | INFO | "malloc" : " 14 GB", | 20260506 20:25:16.945 | INFO | "pinned" : " 29 GB" | 20260506 20:25:16.945 | INFO | } | 20260506 20:25:16.945 | INFO | Perf report for the last 5.13563 seconds: | 20260506 20:25:16.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:25:16.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:16.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:16.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:16.945 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 20:25:16.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:16.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:16.945 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:25:16.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:16.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:16.945 | INFO | Uploaded 36GiB to GPU in 2.00946s (17.9152GiB/s) | 20260506 20:25:16.945 | INFO | Downloaded 34GiB to GPU in 1.99851s (17.0127GiB/s) | 20260506 20:25:17.059 | INFO | Starting a round of Baz IO | 20260506 20:25:20.187 | INFO | Populating RTMetrics report | 20260506 20:25:20.187 | INFO | RTMetrics report populated | 20260506 20:25:20.187 | INFO | Opening tmp RTMetrics json file | 20260506 20:25:20.419 | INFO | Serializing RTMetrics to json | 20260506 20:25:20.505 | INFO | Serialization done, swapping tmp file now... | 20260506 20:25:20.506 | INFO | RTMetrics done | 20260506 20:25:20.803 | INFO | Finished a round of Baz IO: Wrote 1.98 GiB in 3.743 seconds | 20260506 20:25:20.803 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.25% / 0.00% / 65.74% | 20260506 20:25:20.803 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.7562% / 7.94729e-06% / 6.35783e-05% | 20260506 20:25:20.803 | INFO | Baz IO overall: | 20260506 20:25:20.803 | INFO | BytesWritten : 2221026592641 | 20260506 20:25:20.803 | INFO | NumEvents : 1073057351298 | 20260506 20:25:22.094 | INFO | Analyzing chunk frames = [4206592,4207104) | 20260506 20:25:22.094 | INFO | MemInfo: { | 20260506 20:25:22.094 | INFO | "gpu" : " 17 GB", | 20260506 20:25:22.094 | INFO | "huge" : "", | 20260506 20:25:22.094 | INFO | "huge_shared" : " 27 GB", | 20260506 20:25:22.094 | INFO | "malloc" : " 14 GB", | 20260506 20:25:22.094 | INFO | "pinned" : " 29 GB" | 20260506 20:25:22.094 | INFO | } | 20260506 20:25:22.094 | INFO | Perf report for the last 5.14927 seconds: | 20260506 20:25:22.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:25:22.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:22.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:22.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:22.094 | INFO | ANALYSIS : 62.4%, 2.9, 95.5ms, 0 | 20260506 20:25:22.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:22.094 | INFO | RT_METRICS : 49.1%, 1.0, 25.8ms, 0 | 20260506 20:25:22.094 | INFO | PRE_HQ : 59.6%, 1.8, 57.1ms, 0 | 20260506 20:25:22.094 | INFO | BAZWRITER : 72.4%, 4.8, 179.3ms, 0 | 20260506 20:25:22.094 | INFO | BAZ_SAMPLING : 6.0%, 1.0, 3.1ms, 0 | 20260506 20:25:22.094 | INFO | Uploaded 36GiB to GPU in 2.01462s (17.8693GiB/s) | 20260506 20:25:22.094 | INFO | Downloaded 38GiB to GPU in 2.21211s (17.1782GiB/s) | 20260506 20:25:27.239 | INFO | Analyzing chunk frames = [4207104,4207616) | 20260506 20:25:27.239 | INFO | MemInfo: { | 20260506 20:25:27.239 | INFO | "gpu" : " 17 GB", | 20260506 20:25:27.239 | INFO | "huge" : "", | 20260506 20:25:27.239 | INFO | "huge_shared" : " 27 GB", | 20260506 20:25:27.239 | INFO | "malloc" : " 14 GB", | 20260506 20:25:27.239 | INFO | "pinned" : " 29 GB" | 20260506 20:25:27.239 | INFO | } | 20260506 20:25:27.239 | INFO | Perf report for the last 5.14476 seconds: | 20260506 20:25:27.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:25:27.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:27.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:27.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:27.239 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 20:25:27.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:27.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:27.239 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:25:27.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:27.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:27.239 | INFO | Uploaded 36GiB to GPU in 2.00627s (17.9437GiB/s) | 20260506 20:25:27.239 | INFO | Downloaded 34GiB to GPU in 1.92776s (17.637GiB/s) | 20260506 20:25:32.372 | INFO | Analyzing chunk frames = [4207616,4208128) | 20260506 20:25:32.372 | INFO | MemInfo: { | 20260506 20:25:32.372 | INFO | "gpu" : " 17 GB", | 20260506 20:25:32.372 | INFO | "huge" : "", | 20260506 20:25:32.372 | INFO | "huge_shared" : " 27 GB", | 20260506 20:25:32.372 | INFO | "malloc" : " 14 GB", | 20260506 20:25:32.372 | INFO | "pinned" : " 29 GB" | 20260506 20:25:32.372 | INFO | } | 20260506 20:25:32.372 | INFO | Perf report for the last 5.13315 seconds: | 20260506 20:25:32.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:25:32.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:32.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:32.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:32.372 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 20:25:32.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:32.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:32.372 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:25:32.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:32.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:32.372 | INFO | Uploaded 36GiB to GPU in 1.97973s (18.1843GiB/s) | 20260506 20:25:32.372 | INFO | Downloaded 34GiB to GPU in 1.93843s (17.54GiB/s) | 20260506 20:25:37.513 | INFO | Analyzing chunk frames = [4208128,4208640) | 20260506 20:25:37.513 | INFO | MemInfo: { | 20260506 20:25:37.513 | INFO | "gpu" : " 17 GB", | 20260506 20:25:37.513 | INFO | "huge" : "", | 20260506 20:25:37.513 | INFO | "huge_shared" : " 27 GB", | 20260506 20:25:37.513 | INFO | "malloc" : " 14 GB", | 20260506 20:25:37.513 | INFO | "pinned" : " 29 GB" | 20260506 20:25:37.513 | INFO | } | 20260506 20:25:37.513 | INFO | Perf report for the last 5.14099 seconds: | 20260506 20:25:37.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:25:37.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:37.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:37.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:37.513 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 20:25:37.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:37.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:37.513 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:25:37.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:37.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:37.513 | INFO | Uploaded 36GiB to GPU in 1.93332s (18.6208GiB/s) | 20260506 20:25:37.513 | INFO | Downloaded 34GiB to GPU in 1.87259s (18.1567GiB/s) | 20260506 20:25:42.642 | INFO | Analyzing chunk frames = [4208640,4209152) | 20260506 20:25:42.642 | INFO | MemInfo: { | 20260506 20:25:42.642 | INFO | "gpu" : " 17 GB", | 20260506 20:25:42.643 | INFO | "huge" : "", | 20260506 20:25:42.643 | INFO | "huge_shared" : " 27 GB", | 20260506 20:25:42.643 | INFO | "malloc" : " 14 GB", | 20260506 20:25:42.643 | INFO | "pinned" : " 29 GB" | 20260506 20:25:42.643 | INFO | } | 20260506 20:25:42.643 | INFO | Perf report for the last 5.1297 seconds: | 20260506 20:25:42.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:25:42.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:42.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:42.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:42.643 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 20:25:42.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:42.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:42.643 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:25:42.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:42.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:42.643 | INFO | Uploaded 36GiB to GPU in 1.99664s (18.0303GiB/s) | 20260506 20:25:42.643 | INFO | Downloaded 34GiB to GPU in 1.92172s (17.6925GiB/s) | 20260506 20:25:47.782 | INFO | Analyzing chunk frames = [4209152,4209664) | 20260506 20:25:47.782 | INFO | MemInfo: { | 20260506 20:25:47.782 | INFO | "gpu" : " 17 GB", | 20260506 20:25:47.783 | INFO | "huge" : "", | 20260506 20:25:47.783 | INFO | "huge_shared" : " 27 GB", | 20260506 20:25:47.783 | INFO | "malloc" : " 14 GB", | 20260506 20:25:47.783 | INFO | "pinned" : " 29 GB" | 20260506 20:25:47.783 | INFO | } | 20260506 20:25:47.783 | INFO | Perf report for the last 5.13999 seconds: | 20260506 20:25:47.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:25:47.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:47.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:47.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:47.783 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:25:47.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:47.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:47.783 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 20:25:47.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:47.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:47.783 | INFO | Uploaded 36GiB to GPU in 2.00528s (17.9526GiB/s) | 20260506 20:25:47.783 | INFO | Downloaded 34GiB to GPU in 1.96758s (17.2801GiB/s) | 20260506 20:25:52.928 | INFO | Analyzing chunk frames = [4209664,4210176) | 20260506 20:25:52.928 | INFO | MemInfo: { | 20260506 20:25:52.928 | INFO | "gpu" : " 17 GB", | 20260506 20:25:52.928 | INFO | "huge" : "", | 20260506 20:25:52.928 | INFO | "huge_shared" : " 27 GB", | 20260506 20:25:52.928 | INFO | "malloc" : " 14 GB", | 20260506 20:25:52.928 | INFO | "pinned" : " 29 GB" | 20260506 20:25:52.928 | INFO | } | 20260506 20:25:52.928 | INFO | Perf report for the last 5.14557 seconds: | 20260506 20:25:52.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:25:52.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:52.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:52.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:52.928 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:25:52.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:52.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:52.928 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:25:52.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:52.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:52.929 | INFO | Uploaded 36GiB to GPU in 1.96219s (18.3468GiB/s) | 20260506 20:25:52.929 | INFO | Downloaded 34GiB to GPU in 1.99809s (17.0163GiB/s) | 20260506 20:25:58.064 | INFO | Analyzing chunk frames = [4210176,4210688) | 20260506 20:25:58.064 | INFO | MemInfo: { | 20260506 20:25:58.064 | INFO | "gpu" : " 17 GB", | 20260506 20:25:58.064 | INFO | "huge" : "", | 20260506 20:25:58.064 | INFO | "huge_shared" : " 27 GB", | 20260506 20:25:58.064 | INFO | "malloc" : " 14 GB", | 20260506 20:25:58.064 | INFO | "pinned" : " 29 GB" | 20260506 20:25:58.064 | INFO | } | 20260506 20:25:58.064 | INFO | Perf report for the last 5.1358 seconds: | 20260506 20:25:58.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:25:58.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:58.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:58.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:58.064 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 20:25:58.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:58.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:58.064 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:25:58.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:58.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:25:58.064 | INFO | Uploaded 36GiB to GPU in 2.04205s (17.6294GiB/s) | 20260506 20:25:58.064 | INFO | Downloaded 34GiB to GPU in 1.98157s (17.1581GiB/s) | 20260506 20:25:58.064 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:25:58.064 | INFO | Total mem usage: 18 GiB | 20260506 20:25:58.064 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:25:58.064 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.7%) | 20260506 20:25:58.065 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:25:58.065 | INFO | High Water Breakdown: | 20260506 20:25:58.065 | INFO | 61.4% : 9.56 GiB ( 83.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:25:58.065 | INFO | 38.4% : 5.98 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:25:58.065 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:25:58.065 | INFO | ------------------------------------------ | 20260506 20:25:58.065 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:25:58.065 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:25:58.065 | INFO | Total mem usage: 26.5 GiB | 20260506 20:25:58.065 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:25:58.065 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:25:58.065 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:25:58.065 | INFO | High Water Breakdown: | 20260506 20:25:58.065 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:25:58.065 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:25:58.065 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:25:58.065 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:25:58.065 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:25:58.065 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:25:58.065 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:25:58.065 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:25:58.065 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:25:58.065 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:25:58.065 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:25:58.065 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:25:58.065 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:25:58.065 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:25:58.065 | INFO | ------------------------------------------ | 20260506 20:25:58.065 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:25:58.065 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:25:58.065 | INFO | Total mem usage: 25.6 GiB | 20260506 20:25:58.065 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:25:58.065 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:25:58.065 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:25:58.065 | INFO | High Water Breakdown: | 20260506 20:25:58.065 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:25:58.065 | INFO | ------------------------------------------ | 20260506 20:25:58.065 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:25:58.065 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:25:58.065 | INFO | Total mem usage: 15.7 GiB | 20260506 20:25:58.065 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:25:58.065 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:25:58.065 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 20:25:58.065 | INFO | High Water Breakdown: | 20260506 20:25:58.065 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:25:58.065 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:25:58.065 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:25:58.065 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:25:58.065 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:25:58.065 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:25:58.065 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:25:58.065 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:25:58.065 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:25:58.065 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:25:58.065 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:25:58.065 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:25:58.065 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:25:58.065 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:25:58.065 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:25:58.065 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:25:58.065 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:25:58.065 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:25:58.065 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:25:58.065 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:25:58.065 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:25:58.065 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:25:58.065 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:25:58.065 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:25:58.065 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:25:58.065 | INFO | ------------------------------------------ | 20260506 20:25:58.065 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:25:58.065 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:25:58.065 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:25:58.065 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:25:58.066 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.36e+01 / 2.52e+00 | 20260506 20:25:58.066 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.03e-01, 9.96e-03 / 9.83e-03 | 20260506 20:25:58.066 | INFO | PerfReport: 11.64%, Baseline -- 3.63e+00 / 4.41e+00, 5.61e-01 / 3.18e-01 | 20260506 20:25:58.066 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.84e-01 / 1.30e-01 | 20260506 20:25:58.066 | INFO | PerfReport: 16.05%, DME -- 4.00e+01 / 4.48e+01, 4.30e+00 / 2.58e+00 | 20260506 20:25:58.066 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.42e+00 / 1.60e+00 | 20260506 20:25:58.066 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.22e+00 / 2.81e+00, 3.79e-01 / 2.24e-01 | 20260506 20:25:58.066 | INFO | PerfReport: 3.31%, Metrics -- 1.03e+00 / 1.37e+00, 2.76e-01 / 2.06e-01 | 20260506 20:25:58.066 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.54e-01 / 3.97e-01, 5.54e-02 / 2.63e-02 | 20260506 20:25:58.066 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:25:58.066 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:25:58.066 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:25:58.066 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 20:25:58.066 | INFO | PerfReport: 21.44%, Upload -- 2.08e+01 / 0.00e+00, 6.86e+00 / 0.00e+00 | 20260506 20:25:58.066 | INFO | PerfReport: 21.28%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 20:25:58.066 | INFO | PerfReport: 32.22%, Compute -- 3.12e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 20:25:58.066 | INFO | PerfReport: 1.20%, LockUpload -- 1.16e+00 / 0.00e+00, 4.75e+00 / 0.00e+00 | 20260506 20:25:58.066 | INFO | PerfReport: 1.29%, LockDownload -- 1.25e+00 / 0.00e+00, 3.18e+00 / 0.00e+00 | 20260506 20:25:58.066 | INFO | PerfReport: 22.55%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 20:25:58.066 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260506 20:25:58.178 | INFO | Starting a round of Baz IO | 20260506 20:26:01.306 | INFO | Populating RTMetrics report | 20260506 20:26:01.306 | INFO | RTMetrics report populated | 20260506 20:26:01.306 | INFO | Opening tmp RTMetrics json file | 20260506 20:26:01.307 | INFO | Serializing RTMetrics to json | 20260506 20:26:01.309 | INFO | Serialization done, swapping tmp file now... | 20260506 20:26:01.309 | INFO | RTMetrics done | 20260506 20:26:01.434 | INFO | Finished a round of Baz IO: Wrote 1.978 GiB in 3.255 seconds | 20260506 20:26:01.434 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.25% / 0.00% / 65.75% | 20260506 20:26:01.434 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.752223% / 0% / 7.94729e-05% | 20260506 20:26:01.434 | INFO | Baz IO overall: | 20260506 20:26:01.434 | INFO | BytesWritten : 2223150182695 | 20260506 20:26:01.434 | INFO | NumEvents : 1074075945152 | 20260506 20:26:03.206 | INFO | Analyzing chunk frames = [4210688,4211200) | 20260506 20:26:03.207 | INFO | MemInfo: { | 20260506 20:26:03.207 | INFO | "gpu" : " 17 GB", | 20260506 20:26:03.207 | INFO | "huge" : "", | 20260506 20:26:03.207 | INFO | "huge_shared" : " 27 GB", | 20260506 20:26:03.207 | INFO | "malloc" : " 14 GB", | 20260506 20:26:03.207 | INFO | "pinned" : " 29 GB" | 20260506 20:26:03.207 | INFO | } | 20260506 20:26:03.207 | INFO | Perf report for the last 5.1425 seconds: | 20260506 20:26:03.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:26:03.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:03.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:03.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:03.207 | INFO | ANALYSIS : 62.5%, 2.9, 95.2ms, 0 | 20260506 20:26:03.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:03.207 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 20:26:03.207 | INFO | PRE_HQ : 60.0%, 1.8, 57.5ms, 0 | 20260506 20:26:03.207 | INFO | BAZWRITER : 63.0%, 4.7, 150.9ms, 0 | 20260506 20:26:03.207 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 20:26:03.207 | INFO | Uploaded 36GiB to GPU in 2.08302s (17.2826GiB/s) | 20260506 20:26:03.207 | INFO | Downloaded 38GiB to GPU in 2.21895s (17.1252GiB/s) | 20260506 20:26:08.346 | INFO | Analyzing chunk frames = [4211200,4211712) | 20260506 20:26:08.346 | INFO | MemInfo: { | 20260506 20:26:08.346 | INFO | "gpu" : " 17 GB", | 20260506 20:26:08.346 | INFO | "huge" : "", | 20260506 20:26:08.346 | INFO | "huge_shared" : " 27 GB", | 20260506 20:26:08.346 | INFO | "malloc" : " 14 GB", | 20260506 20:26:08.346 | INFO | "pinned" : " 29 GB" | 20260506 20:26:08.346 | INFO | } | 20260506 20:26:08.346 | INFO | Perf report for the last 5.13974 seconds: | 20260506 20:26:08.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:26:08.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:08.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:08.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:08.346 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:26:08.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:08.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:08.346 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:26:08.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:08.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:08.347 | INFO | Uploaded 36GiB to GPU in 2.01772s (17.8419GiB/s) | 20260506 20:26:08.347 | INFO | Downloaded 34GiB to GPU in 1.929s (17.6257GiB/s) | 20260506 20:26:13.483 | INFO | Analyzing chunk frames = [4211712,4212224) | 20260506 20:26:13.483 | INFO | MemInfo: { | 20260506 20:26:13.483 | INFO | "gpu" : " 17 GB", | 20260506 20:26:13.483 | INFO | "huge" : "", | 20260506 20:26:13.483 | INFO | "huge_shared" : " 27 GB", | 20260506 20:26:13.483 | INFO | "malloc" : " 14 GB", | 20260506 20:26:13.483 | INFO | "pinned" : " 29 GB" | 20260506 20:26:13.483 | INFO | } | 20260506 20:26:13.483 | INFO | Perf report for the last 5.13654 seconds: | 20260506 20:26:13.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:26:13.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:13.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:13.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:13.483 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 20:26:13.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:13.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:13.483 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:26:13.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:13.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:13.483 | INFO | Uploaded 36GiB to GPU in 1.98451s (18.1405GiB/s) | 20260506 20:26:13.483 | INFO | Downloaded 34GiB to GPU in 1.92035s (17.7051GiB/s) | 20260506 20:26:18.620 | INFO | Analyzing chunk frames = [4212224,4212736) | 20260506 20:26:18.620 | INFO | MemInfo: { | 20260506 20:26:18.620 | INFO | "gpu" : " 17 GB", | 20260506 20:26:18.620 | INFO | "huge" : "", | 20260506 20:26:18.621 | INFO | "huge_shared" : " 27 GB", | 20260506 20:26:18.621 | INFO | "malloc" : " 14 GB", | 20260506 20:26:18.621 | INFO | "pinned" : " 29 GB" | 20260506 20:26:18.621 | INFO | } | 20260506 20:26:18.621 | INFO | Perf report for the last 5.13785 seconds: | 20260506 20:26:18.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:26:18.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:18.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:18.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:18.621 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 20:26:18.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:18.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:18.621 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:26:18.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:18.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:18.621 | INFO | Uploaded 36GiB to GPU in 1.92524s (18.699GiB/s) | 20260506 20:26:18.621 | INFO | Downloaded 34GiB to GPU in 1.86703s (18.2107GiB/s) | 20260506 20:26:23.759 | INFO | Analyzing chunk frames = [4212736,4213248) | 20260506 20:26:23.759 | INFO | MemInfo: { | 20260506 20:26:23.759 | INFO | "gpu" : " 17 GB", | 20260506 20:26:23.759 | INFO | "huge" : "", | 20260506 20:26:23.759 | INFO | "huge_shared" : " 27 GB", | 20260506 20:26:23.759 | INFO | "malloc" : " 14 GB", | 20260506 20:26:23.759 | INFO | "pinned" : " 29 GB" | 20260506 20:26:23.759 | INFO | } | 20260506 20:26:23.759 | INFO | Perf report for the last 5.1382 seconds: | 20260506 20:26:23.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:26:23.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:23.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:23.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:23.759 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 20:26:23.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:23.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:23.759 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:26:23.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:23.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:23.759 | INFO | Uploaded 36GiB to GPU in 1.98493s (18.1366GiB/s) | 20260506 20:26:23.759 | INFO | Downloaded 34GiB to GPU in 1.92342s (17.6768GiB/s) | 20260506 20:26:28.897 | INFO | Analyzing chunk frames = [4213248,4213760) | 20260506 20:26:28.897 | INFO | MemInfo: { | 20260506 20:26:28.897 | INFO | "gpu" : " 17 GB", | 20260506 20:26:28.897 | INFO | "huge" : "", | 20260506 20:26:28.897 | INFO | "huge_shared" : " 27 GB", | 20260506 20:26:28.897 | INFO | "malloc" : " 14 GB", | 20260506 20:26:28.897 | INFO | "pinned" : " 29 GB" | 20260506 20:26:28.897 | INFO | } | 20260506 20:26:28.897 | INFO | Perf report for the last 5.13781 seconds: | 20260506 20:26:28.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:26:28.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:28.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:28.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:28.897 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:26:28.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:28.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:28.897 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 20:26:28.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:28.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:28.897 | INFO | Uploaded 36GiB to GPU in 2.01231s (17.8899GiB/s) | 20260506 20:26:28.897 | INFO | Downloaded 34GiB to GPU in 1.95448s (17.3959GiB/s) | 20260506 20:26:34.040 | INFO | Analyzing chunk frames = [4213760,4214272) | 20260506 20:26:34.040 | INFO | MemInfo: { | 20260506 20:26:34.040 | INFO | "gpu" : " 17 GB", | 20260506 20:26:34.040 | INFO | "huge" : "", | 20260506 20:26:34.040 | INFO | "huge_shared" : " 27 GB", | 20260506 20:26:34.040 | INFO | "malloc" : " 14 GB", | 20260506 20:26:34.040 | INFO | "pinned" : " 29 GB" | 20260506 20:26:34.040 | INFO | } | 20260506 20:26:34.040 | INFO | Perf report for the last 5.14307 seconds: | 20260506 20:26:34.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:26:34.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:34.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:34.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:34.040 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 20:26:34.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:34.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:34.040 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:26:34.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:34.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:34.040 | INFO | Uploaded 36GiB to GPU in 1.95692s (18.3962GiB/s) | 20260506 20:26:34.040 | INFO | Downloaded 34GiB to GPU in 1.9974s (17.0221GiB/s) | 20260506 20:26:39.187 | INFO | Analyzing chunk frames = [4214272,4214784) | 20260506 20:26:39.187 | INFO | MemInfo: { | 20260506 20:26:39.187 | INFO | "gpu" : " 17 GB", | 20260506 20:26:39.187 | INFO | "huge" : "", | 20260506 20:26:39.187 | INFO | "huge_shared" : " 27 GB", | 20260506 20:26:39.187 | INFO | "malloc" : " 14 GB", | 20260506 20:26:39.187 | INFO | "pinned" : " 29 GB" | 20260506 20:26:39.187 | INFO | } | 20260506 20:26:39.187 | INFO | Perf report for the last 5.14743 seconds: | 20260506 20:26:39.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:26:39.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:39.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:39.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:39.187 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 20:26:39.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:39.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:39.187 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 20:26:39.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:39.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:39.188 | INFO | Uploaded 36GiB to GPU in 2.01752s (17.8437GiB/s) | 20260506 20:26:39.188 | INFO | Downloaded 34GiB to GPU in 1.99027s (17.0831GiB/s) | 20260506 20:26:39.293 | INFO | Starting a round of Baz IO | 20260506 20:26:42.460 | INFO | Populating RTMetrics report | 20260506 20:26:42.460 | INFO | RTMetrics report populated | 20260506 20:26:42.460 | INFO | Opening tmp RTMetrics json file | 20260506 20:26:42.505 | INFO | Serializing RTMetrics to json | 20260506 20:26:42.576 | INFO | Serialization done, swapping tmp file now... | 20260506 20:26:42.578 | INFO | RTMetrics done | 20260506 20:26:43.065 | INFO | Finished a round of Baz IO: Wrote 1.977 GiB in 3.772 seconds | 20260506 20:26:43.065 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.25% / 0.00% / 65.75% | 20260506 20:26:43.065 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.751054% / 7.94729e-06% / 5.96046e-05% | 20260506 20:26:43.065 | INFO | Baz IO overall: | 20260506 20:26:43.065 | INFO | BytesWritten : 2225272593351 | 20260506 20:26:43.065 | INFO | NumEvents : 1075093794535 | 20260506 20:26:44.323 | INFO | Analyzing chunk frames = [4214784,4215296) | 20260506 20:26:44.323 | INFO | MemInfo: { | 20260506 20:26:44.323 | INFO | "gpu" : " 17 GB", | 20260506 20:26:44.323 | INFO | "huge" : "", | 20260506 20:26:44.323 | INFO | "huge_shared" : " 27 GB", | 20260506 20:26:44.323 | INFO | "malloc" : " 14 GB", | 20260506 20:26:44.323 | INFO | "pinned" : " 29 GB" | 20260506 20:26:44.323 | INFO | } | 20260506 20:26:44.323 | INFO | Perf report for the last 5.13562 seconds: | 20260506 20:26:44.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:26:44.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:44.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:44.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:44.323 | INFO | ANALYSIS : 62.8%, 2.9, 94.4ms, 0 | 20260506 20:26:44.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:44.323 | INFO | RT_METRICS : 48.8%, 1.0, 25.6ms, 0 | 20260506 20:26:44.323 | INFO | PRE_HQ : 60.8%, 2.0, 62.3ms, 0 | 20260506 20:26:44.323 | INFO | BAZWRITER : 73.1%, 4.8, 179.5ms, 0 | 20260506 20:26:44.323 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 20:26:44.323 | INFO | Uploaded 36GiB to GPU in 2.04826s (17.5759GiB/s) | 20260506 20:26:44.323 | INFO | Downloaded 38GiB to GPU in 2.17838s (17.4442GiB/s) | 20260506 20:26:49.460 | INFO | Analyzing chunk frames = [4215296,4215808) | 20260506 20:26:49.460 | INFO | MemInfo: { | 20260506 20:26:49.460 | INFO | "gpu" : " 17 GB", | 20260506 20:26:49.460 | INFO | "huge" : "", | 20260506 20:26:49.460 | INFO | "huge_shared" : " 27 GB", | 20260506 20:26:49.460 | INFO | "malloc" : " 14 GB", | 20260506 20:26:49.460 | INFO | "pinned" : " 29 GB" | 20260506 20:26:49.460 | INFO | } | 20260506 20:26:49.461 | INFO | Perf report for the last 5.13757 seconds: | 20260506 20:26:49.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:26:49.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:49.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:49.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:49.461 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:26:49.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:49.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:49.461 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 20:26:49.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:49.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:49.461 | INFO | Uploaded 36GiB to GPU in 2.00577s (17.9482GiB/s) | 20260506 20:26:49.461 | INFO | Downloaded 34GiB to GPU in 1.931s (17.6075GiB/s) | 20260506 20:26:54.600 | INFO | Analyzing chunk frames = [4215808,4216320) | 20260506 20:26:54.600 | INFO | MemInfo: { | 20260506 20:26:54.600 | INFO | "gpu" : " 17 GB", | 20260506 20:26:54.600 | INFO | "huge" : "", | 20260506 20:26:54.600 | INFO | "huge_shared" : " 27 GB", | 20260506 20:26:54.600 | INFO | "malloc" : " 14 GB", | 20260506 20:26:54.600 | INFO | "pinned" : " 29 GB" | 20260506 20:26:54.600 | INFO | } | 20260506 20:26:54.600 | INFO | Perf report for the last 5.1392 seconds: | 20260506 20:26:54.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:26:54.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:54.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:54.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:54.600 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 20:26:54.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:54.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:54.600 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:26:54.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:54.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:54.600 | INFO | Uploaded 36GiB to GPU in 1.97496s (18.2282GiB/s) | 20260506 20:26:54.600 | INFO | Downloaded 34GiB to GPU in 1.92138s (17.6956GiB/s) | 20260506 20:26:59.735 | INFO | Analyzing chunk frames = [4216320,4216832) | 20260506 20:26:59.735 | INFO | MemInfo: { | 20260506 20:26:59.735 | INFO | "gpu" : " 17 GB", | 20260506 20:26:59.735 | INFO | "huge" : "", | 20260506 20:26:59.735 | INFO | "huge_shared" : " 27 GB", | 20260506 20:26:59.735 | INFO | "malloc" : " 14 GB", | 20260506 20:26:59.735 | INFO | "pinned" : " 29 GB" | 20260506 20:26:59.735 | INFO | } | 20260506 20:26:59.736 | INFO | Perf report for the last 5.13581 seconds: | 20260506 20:26:59.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:26:59.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:59.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:59.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:59.736 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 20:26:59.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:59.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:59.736 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:26:59.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:59.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:26:59.736 | INFO | Uploaded 36GiB to GPU in 1.9318s (18.6354GiB/s) | 20260506 20:26:59.736 | INFO | Downloaded 34GiB to GPU in 1.87367s (18.1462GiB/s) | 20260506 20:27:04.874 | INFO | Analyzing chunk frames = [4216832,4217344) | 20260506 20:27:04.874 | INFO | MemInfo: { | 20260506 20:27:04.874 | INFO | "gpu" : " 17 GB", | 20260506 20:27:04.874 | INFO | "huge" : "", | 20260506 20:27:04.874 | INFO | "huge_shared" : " 27 GB", | 20260506 20:27:04.874 | INFO | "malloc" : " 14 GB", | 20260506 20:27:04.874 | INFO | "pinned" : " 29 GB" | 20260506 20:27:04.874 | INFO | } | 20260506 20:27:04.874 | INFO | Perf report for the last 5.13875 seconds: | 20260506 20:27:04.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:27:04.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:04.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:04.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:04.874 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 20:27:04.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:04.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:04.874 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:27:04.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:04.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:04.875 | INFO | Uploaded 36GiB to GPU in 2.00404s (17.9637GiB/s) | 20260506 20:27:04.875 | INFO | Downloaded 34GiB to GPU in 1.90932s (17.8074GiB/s) | 20260506 20:27:10.021 | INFO | Analyzing chunk frames = [4217344,4217856) | 20260506 20:27:10.021 | INFO | MemInfo: { | 20260506 20:27:10.021 | INFO | "gpu" : " 17 GB", | 20260506 20:27:10.021 | INFO | "huge" : "", | 20260506 20:27:10.021 | INFO | "huge_shared" : " 27 GB", | 20260506 20:27:10.021 | INFO | "malloc" : " 14 GB", | 20260506 20:27:10.021 | INFO | "pinned" : " 29 GB" | 20260506 20:27:10.021 | INFO | } | 20260506 20:27:10.021 | INFO | Perf report for the last 5.14708 seconds: | 20260506 20:27:10.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:27:10.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:10.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:10.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:10.021 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260506 20:27:10.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:10.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:10.022 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:27:10.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:10.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:10.022 | INFO | Uploaded 36GiB to GPU in 1.9909s (18.0823GiB/s) | 20260506 20:27:10.022 | INFO | Downloaded 34GiB to GPU in 1.95263s (17.4124GiB/s) | 20260506 20:27:15.156 | INFO | Analyzing chunk frames = [4217856,4218368) | 20260506 20:27:15.156 | INFO | MemInfo: { | 20260506 20:27:15.156 | INFO | "gpu" : " 17 GB", | 20260506 20:27:15.156 | INFO | "huge" : "", | 20260506 20:27:15.157 | INFO | "huge_shared" : " 27 GB", | 20260506 20:27:15.157 | INFO | "malloc" : " 14 GB", | 20260506 20:27:15.157 | INFO | "pinned" : " 29 GB" | 20260506 20:27:15.157 | INFO | } | 20260506 20:27:15.157 | INFO | Perf report for the last 5.13521 seconds: | 20260506 20:27:15.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:27:15.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:15.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:15.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:15.157 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 20:27:15.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:15.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:15.157 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 20:27:15.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:15.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:15.157 | INFO | Uploaded 36GiB to GPU in 1.96602s (18.3111GiB/s) | 20260506 20:27:15.157 | INFO | Downloaded 34GiB to GPU in 1.99844s (17.0133GiB/s) | 20260506 20:27:20.296 | INFO | Analyzing chunk frames = [4218368,4218880) | 20260506 20:27:20.296 | INFO | MemInfo: { | 20260506 20:27:20.296 | INFO | "gpu" : " 17 GB", | 20260506 20:27:20.296 | INFO | "huge" : "", | 20260506 20:27:20.296 | INFO | "huge_shared" : " 27 GB", | 20260506 20:27:20.297 | INFO | "malloc" : " 14 GB", | 20260506 20:27:20.297 | INFO | "pinned" : " 29 GB" | 20260506 20:27:20.297 | INFO | } | 20260506 20:27:20.297 | INFO | Perf report for the last 5.13994 seconds: | 20260506 20:27:20.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:27:20.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:20.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:20.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:20.297 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260506 20:27:20.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:20.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:20.297 | INFO | PRE_HQ : 32.0%, 1.0, 16.6ms, 0 | 20260506 20:27:20.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:20.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:20.297 | INFO | Uploaded 36GiB to GPU in 2.05753s (17.4967GiB/s) | 20260506 20:27:20.297 | INFO | Downloaded 34GiB to GPU in 1.99787s (17.0181GiB/s) | 20260506 20:27:20.297 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:27:20.297 | INFO | Total mem usage: 18 GiB | 20260506 20:27:20.297 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:27:20.297 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.4%) | 20260506 20:27:20.297 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:27:20.297 | INFO | High Water Breakdown: | 20260506 20:27:20.297 | INFO | 61.7% : 9.8 GiB ( 85.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:27:20.297 | INFO | 38.2% : 6.06 GiB ( 90.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:27:20.297 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:27:20.297 | INFO | ------------------------------------------ | 20260506 20:27:20.297 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:27:20.297 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:27:20.297 | INFO | Total mem usage: 26.5 GiB | 20260506 20:27:20.297 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:27:20.297 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:27:20.297 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:27:20.297 | INFO | High Water Breakdown: | 20260506 20:27:20.297 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:27:20.297 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:27:20.297 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:27:20.297 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:27:20.297 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:27:20.297 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:27:20.297 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:27:20.297 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:27:20.297 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:27:20.297 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:27:20.297 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:27:20.297 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:27:20.297 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:27:20.297 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:27:20.298 | INFO | ------------------------------------------ | 20260506 20:27:20.298 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:27:20.298 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:27:20.298 | INFO | Total mem usage: 25.6 GiB | 20260506 20:27:20.298 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:27:20.298 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:27:20.298 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:27:20.298 | INFO | High Water Breakdown: | 20260506 20:27:20.298 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:27:20.298 | INFO | ------------------------------------------ | 20260506 20:27:20.298 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:27:20.298 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:27:20.298 | INFO | Total mem usage: 15.7 GiB | 20260506 20:27:20.298 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:27:20.298 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:27:20.298 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:27:20.298 | INFO | High Water Breakdown: | 20260506 20:27:20.298 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:27:20.298 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:27:20.298 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:27:20.298 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:27:20.298 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:27:20.298 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:27:20.298 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:27:20.298 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:27:20.298 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:27:20.298 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:27:20.298 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:27:20.298 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:27:20.298 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:27:20.298 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:27:20.298 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:27:20.298 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:27:20.298 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:27:20.298 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:27:20.298 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:27:20.298 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:27:20.298 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:27:20.298 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:27:20.298 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:27:20.298 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:27:20.298 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:27:20.298 | INFO | ------------------------------------------ | 20260506 20:27:20.298 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:27:20.298 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:27:20.298 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:27:20.298 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:27:20.298 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.54e+01, 1.35e+01 / 2.49e+00 | 20260506 20:27:20.298 | INFO | PerfReport: 0.29%, AggregateStats -- 9.20e-02 / 1.01e-01, 1.11e-02 / 5.87e-03 | 20260506 20:27:20.298 | INFO | PerfReport: 11.64%, Baseline -- 3.63e+00 / 4.32e+00, 5.54e-01 / 4.05e-01 | 20260506 20:27:20.298 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.43e+00, 1.83e-01 / 1.43e-01 | 20260506 20:27:20.298 | INFO | PerfReport: 15.98%, DME -- 3.99e+01 / 4.44e+01, 4.09e+00 / 2.79e+00 | 20260506 20:27:20.298 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.42e+00 / 1.84e+00 | 20260506 20:27:20.298 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.23e+00 / 2.75e+00, 3.80e-01 / 2.51e-01 | 20260506 20:27:20.298 | INFO | PerfReport: 3.31%, Metrics -- 1.03e+00 / 1.34e+00, 2.66e-01 / 1.09e-01 | 20260506 20:27:20.298 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.55e-01 / 4.02e-01, 5.52e-02 / 2.73e-02 | 20260506 20:27:20.298 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:27:20.298 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:27:20.298 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:27:20.298 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 20:27:20.298 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 6.90e+00 / 0.00e+00 | 20260506 20:27:20.298 | INFO | PerfReport: 21.26%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 20:27:20.298 | INFO | PerfReport: 32.25%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 20:27:20.298 | INFO | PerfReport: 1.28%, LockUpload -- 1.24e+00 / 0.00e+00, 5.18e+00 / 0.00e+00 | 20260506 20:27:20.298 | INFO | PerfReport: 1.20%, LockDownload -- 1.16e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 20:27:20.298 | INFO | PerfReport: 22.47%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 20:27:20.298 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260506 20:27:20.419 | INFO | Starting a round of Baz IO | 20260506 20:27:23.538 | INFO | Populating RTMetrics report | 20260506 20:27:23.538 | INFO | RTMetrics report populated | 20260506 20:27:23.538 | INFO | Opening tmp RTMetrics json file | 20260506 20:27:23.540 | INFO | Serializing RTMetrics to json | 20260506 20:27:23.568 | INFO | Serialization done, swapping tmp file now... | 20260506 20:27:23.597 | INFO | RTMetrics done | 20260506 20:27:23.733 | INFO | Finished a round of Baz IO: Wrote 1.978 GiB in 3.314 seconds | 20260506 20:27:23.734 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.24% / 0.00% / 65.75% | 20260506 20:27:23.734 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.749199% / 1.19209e-05% / 7.15256e-05% | 20260506 20:27:23.734 | INFO | Baz IO overall: | 20260506 20:27:23.734 | INFO | BytesWritten : 2227396680253 | 20260506 20:27:23.734 | INFO | NumEvents : 1076113318791 | 20260506 20:27:25.436 | INFO | Analyzing chunk frames = [4218880,4219392) | 20260506 20:27:25.436 | INFO | MemInfo: { | 20260506 20:27:25.436 | INFO | "gpu" : " 17 GB", | 20260506 20:27:25.436 | INFO | "huge" : "", | 20260506 20:27:25.436 | INFO | "huge_shared" : " 27 GB", | 20260506 20:27:25.436 | INFO | "malloc" : " 14 GB", | 20260506 20:27:25.436 | INFO | "pinned" : " 29 GB" | 20260506 20:27:25.436 | INFO | } | 20260506 20:27:25.436 | INFO | Perf report for the last 5.13908 seconds: | 20260506 20:27:25.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:27:25.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:25.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:25.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:25.436 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 20:27:25.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:25.436 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260506 20:27:25.436 | INFO | PRE_HQ : 59.8%, 1.9, 57.5ms, 0 | 20260506 20:27:25.436 | INFO | BAZWRITER : 64.3%, 4.7, 156.3ms, 0 | 20260506 20:27:25.436 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.5ms, 0 | 20260506 20:27:25.436 | INFO | Uploaded 36GiB to GPU in 2.07322s (17.3643GiB/s) | 20260506 20:27:25.436 | INFO | Downloaded 38GiB to GPU in 2.21488s (17.1567GiB/s) | 20260506 20:27:30.571 | INFO | Analyzing chunk frames = [4219392,4219904) | 20260506 20:27:30.571 | INFO | MemInfo: { | 20260506 20:27:30.571 | INFO | "gpu" : " 17 GB", | 20260506 20:27:30.571 | INFO | "huge" : "", | 20260506 20:27:30.571 | INFO | "huge_shared" : " 27 GB", | 20260506 20:27:30.572 | INFO | "malloc" : " 14 GB", | 20260506 20:27:30.572 | INFO | "pinned" : " 29 GB" | 20260506 20:27:30.572 | INFO | } | 20260506 20:27:30.572 | INFO | Perf report for the last 5.136 seconds: | 20260506 20:27:30.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:27:30.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:30.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:30.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:30.572 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 20:27:30.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:30.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:30.572 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:27:30.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:30.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:30.572 | INFO | Uploaded 36GiB to GPU in 2.01367s (17.8778GiB/s) | 20260506 20:27:30.572 | INFO | Downloaded 34GiB to GPU in 1.93691s (17.5537GiB/s) | 20260506 20:27:35.709 | INFO | Analyzing chunk frames = [4219904,4220416) | 20260506 20:27:35.709 | INFO | MemInfo: { | 20260506 20:27:35.709 | INFO | "gpu" : " 17 GB", | 20260506 20:27:35.709 | INFO | "huge" : "", | 20260506 20:27:35.709 | INFO | "huge_shared" : " 27 GB", | 20260506 20:27:35.709 | INFO | "malloc" : " 14 GB", | 20260506 20:27:35.709 | INFO | "pinned" : " 29 GB" | 20260506 20:27:35.709 | INFO | } | 20260506 20:27:35.709 | INFO | Perf report for the last 5.13746 seconds: | 20260506 20:27:35.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:27:35.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:35.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:35.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:35.709 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 20:27:35.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:35.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:35.709 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:27:35.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:35.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:35.709 | INFO | Uploaded 36GiB to GPU in 1.98015s (18.1804GiB/s) | 20260506 20:27:35.710 | INFO | Downloaded 34GiB to GPU in 1.91841s (17.723GiB/s) | 20260506 20:27:40.850 | INFO | Analyzing chunk frames = [4220416,4220928) | 20260506 20:27:40.850 | INFO | MemInfo: { | 20260506 20:27:40.850 | INFO | "gpu" : " 17 GB", | 20260506 20:27:40.850 | INFO | "huge" : "", | 20260506 20:27:40.850 | INFO | "huge_shared" : " 27 GB", | 20260506 20:27:40.850 | INFO | "malloc" : " 14 GB", | 20260506 20:27:40.850 | INFO | "pinned" : " 29 GB" | 20260506 20:27:40.850 | INFO | } | 20260506 20:27:40.850 | INFO | Perf report for the last 5.14082 seconds: | 20260506 20:27:40.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:27:40.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:40.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:40.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:40.850 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 20:27:40.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:40.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:40.850 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:27:40.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:40.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:40.850 | INFO | Uploaded 36GiB to GPU in 1.96096s (18.3584GiB/s) | 20260506 20:27:40.850 | INFO | Downloaded 34GiB to GPU in 1.87362s (18.1466GiB/s) | 20260506 20:27:45.991 | INFO | Analyzing chunk frames = [4220928,4221440) | 20260506 20:27:45.991 | INFO | MemInfo: { | 20260506 20:27:45.991 | INFO | "gpu" : " 17 GB", | 20260506 20:27:45.991 | INFO | "huge" : "", | 20260506 20:27:45.991 | INFO | "huge_shared" : " 27 GB", | 20260506 20:27:45.991 | INFO | "malloc" : " 14 GB", | 20260506 20:27:45.991 | INFO | "pinned" : " 29 GB" | 20260506 20:27:45.991 | INFO | } | 20260506 20:27:45.991 | INFO | Perf report for the last 5.14094 seconds: | 20260506 20:27:45.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:27:45.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:45.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:45.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:45.991 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 20:27:45.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:45.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:45.991 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:27:45.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:45.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:45.991 | INFO | Uploaded 36GiB to GPU in 2.02058s (17.8167GiB/s) | 20260506 20:27:45.991 | INFO | Downloaded 34GiB to GPU in 1.9111s (17.7908GiB/s) | 20260506 20:27:51.130 | INFO | Analyzing chunk frames = [4221440,4221952) | 20260506 20:27:51.130 | INFO | MemInfo: { | 20260506 20:27:51.130 | INFO | "gpu" : " 17 GB", | 20260506 20:27:51.130 | INFO | "huge" : "", | 20260506 20:27:51.130 | INFO | "huge_shared" : " 27 GB", | 20260506 20:27:51.130 | INFO | "malloc" : " 14 GB", | 20260506 20:27:51.130 | INFO | "pinned" : " 29 GB" | 20260506 20:27:51.130 | INFO | } | 20260506 20:27:51.130 | INFO | Perf report for the last 5.13901 seconds: | 20260506 20:27:51.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:27:51.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:51.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:51.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:51.130 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 20:27:51.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:51.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:51.130 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:27:51.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:51.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:51.130 | INFO | Uploaded 36GiB to GPU in 2.02506s (17.7773GiB/s) | 20260506 20:27:51.130 | INFO | Downloaded 34GiB to GPU in 1.94136s (17.5135GiB/s) | 20260506 20:27:56.276 | INFO | Analyzing chunk frames = [4221952,4222464) | 20260506 20:27:56.276 | INFO | MemInfo: { | 20260506 20:27:56.276 | INFO | "gpu" : " 17 GB", | 20260506 20:27:56.276 | INFO | "huge" : "", | 20260506 20:27:56.277 | INFO | "huge_shared" : " 27 GB", | 20260506 20:27:56.277 | INFO | "malloc" : " 14 GB", | 20260506 20:27:56.277 | INFO | "pinned" : " 29 GB" | 20260506 20:27:56.277 | INFO | } | 20260506 20:27:56.277 | INFO | Perf report for the last 5.14667 seconds: | 20260506 20:27:56.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:27:56.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:56.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:56.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:56.277 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 20:27:56.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:56.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:56.277 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 20:27:56.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:56.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:27:56.277 | INFO | Uploaded 36GiB to GPU in 1.97656s (18.2134GiB/s) | 20260506 20:27:56.277 | INFO | Downloaded 34GiB to GPU in 2.00014s (16.9988GiB/s) | 20260506 20:28:01.409 | INFO | Analyzing chunk frames = [4222464,4222976) | 20260506 20:28:01.409 | INFO | MemInfo: { | 20260506 20:28:01.409 | INFO | "gpu" : " 17 GB", | 20260506 20:28:01.409 | INFO | "huge" : "", | 20260506 20:28:01.409 | INFO | "huge_shared" : " 27 GB", | 20260506 20:28:01.409 | INFO | "malloc" : " 14 GB", | 20260506 20:28:01.409 | INFO | "pinned" : " 29 GB" | 20260506 20:28:01.409 | INFO | } | 20260506 20:28:01.409 | INFO | Perf report for the last 5.13256 seconds: | 20260506 20:28:01.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:28:01.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:01.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:01.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:01.409 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:28:01.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:01.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:01.409 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 20:28:01.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:01.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:01.409 | INFO | Uploaded 36GiB to GPU in 1.98985s (18.0918GiB/s) | 20260506 20:28:01.409 | INFO | Downloaded 34GiB to GPU in 1.98764s (17.1057GiB/s) | 20260506 20:28:01.509 | INFO | Starting a round of Baz IO | 20260506 20:28:04.651 | INFO | Populating RTMetrics report | 20260506 20:28:04.652 | INFO | RTMetrics report populated | 20260506 20:28:04.652 | INFO | Opening tmp RTMetrics json file | 20260506 20:28:04.653 | INFO | Serializing RTMetrics to json | 20260506 20:28:04.657 | INFO | Serialization done, swapping tmp file now... | 20260506 20:28:04.658 | INFO | RTMetrics done | 20260506 20:28:05.057 | INFO | Finished a round of Baz IO: Wrote 1.978 GiB in 3.547 seconds | 20260506 20:28:05.057 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.24% / 0.00% / 65.76% | 20260506 20:28:05.057 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.749203% / 7.94729e-06% / 0.000107288% | 20260506 20:28:05.057 | INFO | Baz IO overall: | 20260506 20:28:05.057 | INFO | BytesWritten : 2229520874871 | 20260506 20:28:05.057 | INFO | NumEvents : 1077132603435 | 20260506 20:28:06.549 | INFO | Analyzing chunk frames = [4222976,4223488) | 20260506 20:28:06.549 | INFO | MemInfo: { | 20260506 20:28:06.549 | INFO | "gpu" : " 17 GB", | 20260506 20:28:06.549 | INFO | "huge" : "", | 20260506 20:28:06.549 | INFO | "huge_shared" : " 27 GB", | 20260506 20:28:06.549 | INFO | "malloc" : " 14 GB", | 20260506 20:28:06.549 | INFO | "pinned" : " 29 GB" | 20260506 20:28:06.549 | INFO | } | 20260506 20:28:06.549 | INFO | Perf report for the last 5.13972 seconds: | 20260506 20:28:06.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:28:06.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:06.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:06.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:06.549 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 20:28:06.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:06.549 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 20:28:06.549 | INFO | PRE_HQ : 60.0%, 1.8, 57.2ms, 0 | 20260506 20:28:06.549 | INFO | BAZWRITER : 68.6%, 4.8, 169.2ms, 0 | 20260506 20:28:06.549 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 20:28:06.549 | INFO | Uploaded 36GiB to GPU in 2.05888s (17.4852GiB/s) | 20260506 20:28:06.549 | INFO | Downloaded 38GiB to GPU in 2.19188s (17.3367GiB/s) | 20260506 20:28:11.690 | INFO | Analyzing chunk frames = [4223488,4224000) | 20260506 20:28:11.690 | INFO | MemInfo: { | 20260506 20:28:11.690 | INFO | "gpu" : " 17 GB", | 20260506 20:28:11.690 | INFO | "huge" : "", | 20260506 20:28:11.690 | INFO | "huge_shared" : " 27 GB", | 20260506 20:28:11.690 | INFO | "malloc" : " 14 GB", | 20260506 20:28:11.690 | INFO | "pinned" : " 29 GB" | 20260506 20:28:11.690 | INFO | } | 20260506 20:28:11.690 | INFO | Perf report for the last 5.14153 seconds: | 20260506 20:28:11.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:28:11.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:11.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:11.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:11.691 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:28:11.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:11.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:11.691 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:28:11.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:11.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:11.691 | INFO | Uploaded 36GiB to GPU in 1.99911s (18.008GiB/s) | 20260506 20:28:11.691 | INFO | Downloaded 34GiB to GPU in 1.91758s (17.7307GiB/s) | 20260506 20:28:16.825 | INFO | Analyzing chunk frames = [4224000,4224512) | 20260506 20:28:16.825 | INFO | MemInfo: { | 20260506 20:28:16.825 | INFO | "gpu" : " 17 GB", | 20260506 20:28:16.825 | INFO | "huge" : "", | 20260506 20:28:16.825 | INFO | "huge_shared" : " 27 GB", | 20260506 20:28:16.825 | INFO | "malloc" : " 14 GB", | 20260506 20:28:16.825 | INFO | "pinned" : " 29 GB" | 20260506 20:28:16.825 | INFO | } | 20260506 20:28:16.825 | INFO | Perf report for the last 5.13456 seconds: | 20260506 20:28:16.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:28:16.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:16.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:16.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:16.825 | INFO | ANALYSIS : 60.9%, 3.0, 94.1ms, 0 | 20260506 20:28:16.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:16.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:16.825 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:28:16.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:16.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:16.825 | INFO | Uploaded 36GiB to GPU in 1.9925s (18.0677GiB/s) | 20260506 20:28:16.825 | INFO | Downloaded 34GiB to GPU in 1.92184s (17.6914GiB/s) | 20260506 20:28:21.965 | INFO | Analyzing chunk frames = [4224512,4225024) | 20260506 20:28:21.965 | INFO | MemInfo: { | 20260506 20:28:21.965 | INFO | "gpu" : " 17 GB", | 20260506 20:28:21.965 | INFO | "huge" : "", | 20260506 20:28:21.965 | INFO | "huge_shared" : " 27 GB", | 20260506 20:28:21.965 | INFO | "malloc" : " 14 GB", | 20260506 20:28:21.965 | INFO | "pinned" : " 29 GB" | 20260506 20:28:21.965 | INFO | } | 20260506 20:28:21.965 | INFO | Perf report for the last 5.14017 seconds: | 20260506 20:28:21.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:28:21.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:21.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:21.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:21.965 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 20:28:21.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:21.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:21.965 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:28:21.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:21.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:21.965 | INFO | Uploaded 36GiB to GPU in 1.93771s (18.5786GiB/s) | 20260506 20:28:21.966 | INFO | Downloaded 34GiB to GPU in 1.87065s (18.1755GiB/s) | 20260506 20:28:27.101 | INFO | Analyzing chunk frames = [4225024,4225536) | 20260506 20:28:27.101 | INFO | MemInfo: { | 20260506 20:28:27.101 | INFO | "gpu" : " 17 GB", | 20260506 20:28:27.101 | INFO | "huge" : "", | 20260506 20:28:27.101 | INFO | "huge_shared" : " 27 GB", | 20260506 20:28:27.101 | INFO | "malloc" : " 14 GB", | 20260506 20:28:27.101 | INFO | "pinned" : " 29 GB" | 20260506 20:28:27.101 | INFO | } | 20260506 20:28:27.101 | INFO | Perf report for the last 5.13599 seconds: | 20260506 20:28:27.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:28:27.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:27.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:27.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:27.101 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 20:28:27.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:27.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:27.101 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:28:27.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:27.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:27.101 | INFO | Uploaded 36GiB to GPU in 1.99579s (18.0379GiB/s) | 20260506 20:28:27.102 | INFO | Downloaded 34GiB to GPU in 1.90558s (17.8424GiB/s) | 20260506 20:28:32.244 | INFO | Analyzing chunk frames = [4225536,4226048) | 20260506 20:28:32.244 | INFO | MemInfo: { | 20260506 20:28:32.244 | INFO | "gpu" : " 17 GB", | 20260506 20:28:32.244 | INFO | "huge" : "", | 20260506 20:28:32.244 | INFO | "huge_shared" : " 27 GB", | 20260506 20:28:32.244 | INFO | "malloc" : " 14 GB", | 20260506 20:28:32.244 | INFO | "pinned" : " 29 GB" | 20260506 20:28:32.244 | INFO | } | 20260506 20:28:32.244 | INFO | Perf report for the last 5.14242 seconds: | 20260506 20:28:32.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:28:32.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:32.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:32.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:32.244 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 20:28:32.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:32.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:32.244 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 20:28:32.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:32.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:32.244 | INFO | Uploaded 36GiB to GPU in 2.01191s (17.8935GiB/s) | 20260506 20:28:32.244 | INFO | Downloaded 34GiB to GPU in 1.94148s (17.5124GiB/s) | 20260506 20:28:37.386 | INFO | Analyzing chunk frames = [4226048,4226560) | 20260506 20:28:37.386 | INFO | MemInfo: { | 20260506 20:28:37.386 | INFO | "gpu" : " 17 GB", | 20260506 20:28:37.386 | INFO | "huge" : "", | 20260506 20:28:37.386 | INFO | "huge_shared" : " 27 GB", | 20260506 20:28:37.386 | INFO | "malloc" : " 14 GB", | 20260506 20:28:37.386 | INFO | "pinned" : " 29 GB" | 20260506 20:28:37.386 | INFO | } | 20260506 20:28:37.386 | INFO | Perf report for the last 5.14215 seconds: | 20260506 20:28:37.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:28:37.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:37.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:37.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:37.386 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 20:28:37.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:37.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:37.386 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:28:37.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:37.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:37.386 | INFO | Uploaded 36GiB to GPU in 2.00264s (17.9763GiB/s) | 20260506 20:28:37.386 | INFO | Downloaded 34GiB to GPU in 2.007s (16.9407GiB/s) | 20260506 20:28:42.524 | INFO | Analyzing chunk frames = [4226560,4227072) | 20260506 20:28:42.524 | INFO | MemInfo: { | 20260506 20:28:42.524 | INFO | "gpu" : " 17 GB", | 20260506 20:28:42.524 | INFO | "huge" : "", | 20260506 20:28:42.524 | INFO | "huge_shared" : " 27 GB", | 20260506 20:28:42.524 | INFO | "malloc" : " 14 GB", | 20260506 20:28:42.524 | INFO | "pinned" : " 29 GB" | 20260506 20:28:42.524 | INFO | } | 20260506 20:28:42.524 | INFO | Perf report for the last 5.1379 seconds: | 20260506 20:28:42.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:28:42.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:42.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:42.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:42.524 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 20:28:42.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:42.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:42.524 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 20:28:42.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:42.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:42.524 | INFO | Uploaded 36GiB to GPU in 2.0692s (17.398GiB/s) | 20260506 20:28:42.524 | INFO | Downloaded 34GiB to GPU in 1.99666s (17.0284GiB/s) | 20260506 20:28:42.524 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:28:42.524 | INFO | Total mem usage: 18 GiB | 20260506 20:28:42.524 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:28:42.524 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.5%) | 20260506 20:28:42.524 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:28:42.524 | INFO | High Water Breakdown: | 20260506 20:28:42.524 | INFO | 61.8% : 9.9 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:28:42.524 | INFO | 38% : 6.08 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:28:42.524 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:28:42.524 | INFO | ------------------------------------------ | 20260506 20:28:42.524 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:28:42.524 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:28:42.524 | INFO | Total mem usage: 26.5 GiB | 20260506 20:28:42.524 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:28:42.524 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:28:42.524 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:28:42.524 | INFO | High Water Breakdown: | 20260506 20:28:42.524 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:28:42.524 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:28:42.524 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:28:42.524 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:28:42.524 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:28:42.524 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:28:42.524 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:28:42.524 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:28:42.524 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:28:42.524 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:28:42.524 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:28:42.524 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:28:42.524 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:28:42.524 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:28:42.524 | INFO | ------------------------------------------ | 20260506 20:28:42.525 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:28:42.525 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:28:42.525 | INFO | Total mem usage: 25.6 GiB | 20260506 20:28:42.525 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:28:42.525 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:28:42.525 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:28:42.525 | INFO | High Water Breakdown: | 20260506 20:28:42.525 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:28:42.525 | INFO | ------------------------------------------ | 20260506 20:28:42.525 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:28:42.525 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:28:42.525 | INFO | Total mem usage: 15.7 GiB | 20260506 20:28:42.525 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:28:42.525 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:28:42.525 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 20:28:42.525 | INFO | High Water Breakdown: | 20260506 20:28:42.525 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:28:42.525 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:28:42.525 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:28:42.525 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:28:42.525 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:28:42.525 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:28:42.525 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:28:42.525 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:28:42.525 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:28:42.525 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:28:42.525 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:28:42.525 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:28:42.525 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:28:42.525 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:28:42.525 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:28:42.525 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:28:42.525 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:28:42.525 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:28:42.525 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:28:42.525 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:28:42.525 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:28:42.525 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:28:42.525 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:28:42.525 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:28:42.525 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:28:42.525 | INFO | ------------------------------------------ | 20260506 20:28:42.525 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:28:42.525 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:28:42.525 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:28:42.525 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:28:42.525 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.61e+01, 1.36e+01 / 2.63e+00 | 20260506 20:28:42.525 | INFO | PerfReport: 0.31%, AggregateStats -- 9.68e-02 / 2.71e-01, 2.09e-01 / 9.24e-01 | 20260506 20:28:42.526 | INFO | PerfReport: 11.61%, Baseline -- 3.62e+00 / 4.39e+00, 5.49e-01 / 3.86e-01 | 20260506 20:28:42.526 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.44e+00, 1.83e-01 / 1.43e-01 | 20260506 20:28:42.526 | INFO | PerfReport: 16.01%, DME -- 3.99e+01 / 4.47e+01, 4.15e+00 / 2.52e+00 | 20260506 20:28:42.526 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.43e+00 / 1.99e+00 | 20260506 20:28:42.526 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.22e+00 / 2.75e+00, 3.77e-01 / 2.73e-01 | 20260506 20:28:42.526 | INFO | PerfReport: 3.31%, Metrics -- 1.03e+00 / 1.35e+00, 2.73e-01 / 1.89e-01 | 20260506 20:28:42.526 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.55e-01 / 4.01e-01, 5.68e-02 / 3.09e-02 | 20260506 20:28:42.526 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:28:42.526 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:28:42.526 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:28:42.526 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 20:28:42.526 | INFO | PerfReport: 21.57%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 20:28:42.526 | INFO | PerfReport: 21.19%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 20:28:42.526 | INFO | PerfReport: 32.19%, Compute -- 3.12e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 20:28:42.526 | INFO | PerfReport: 1.39%, LockUpload -- 1.34e+00 / 0.00e+00, 5.21e+00 / 0.00e+00 | 20260506 20:28:42.526 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260506 20:28:42.526 | INFO | PerfReport: 22.54%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 20:28:42.526 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260506 20:28:42.656 | INFO | Starting a round of Baz IO | 20260506 20:28:45.776 | INFO | Populating RTMetrics report | 20260506 20:28:45.776 | INFO | RTMetrics report populated | 20260506 20:28:45.776 | INFO | Opening tmp RTMetrics json file | 20260506 20:28:46.081 | INFO | Serializing RTMetrics to json | 20260506 20:28:46.083 | INFO | Serialization done, swapping tmp file now... | 20260506 20:28:46.084 | INFO | RTMetrics done | 20260506 20:28:46.109 | INFO | Finished a round of Baz IO: Wrote 1.975 GiB in 3.452 seconds | 20260506 20:28:46.109 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.24% / 0.00% / 65.76% | 20260506 20:28:46.109 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.744808% / 1.19209e-05% / 5.96046e-05% | 20260506 20:28:46.109 | INFO | Baz IO overall: | 20260506 20:28:46.109 | INFO | BytesWritten : 2231641790122 | 20260506 20:28:46.109 | INFO | NumEvents : 1078149414575 | 20260506 20:28:47.663 | INFO | Analyzing chunk frames = [4227072,4227584) | 20260506 20:28:47.663 | INFO | MemInfo: { | 20260506 20:28:47.663 | INFO | "gpu" : " 17 GB", | 20260506 20:28:47.663 | INFO | "huge" : "", | 20260506 20:28:47.663 | INFO | "huge_shared" : " 27 GB", | 20260506 20:28:47.663 | INFO | "malloc" : " 14 GB", | 20260506 20:28:47.663 | INFO | "pinned" : " 29 GB" | 20260506 20:28:47.663 | INFO | } | 20260506 20:28:47.663 | INFO | Perf report for the last 5.13912 seconds: | 20260506 20:28:47.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:28:47.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:47.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:47.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:47.663 | INFO | ANALYSIS : 62.7%, 2.9, 95.3ms, 0 | 20260506 20:28:47.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:47.663 | INFO | RT_METRICS : 49.5%, 1.0, 26.0ms, 0 | 20260506 20:28:47.663 | INFO | PRE_HQ : 60.5%, 1.9, 58.3ms, 0 | 20260506 20:28:47.663 | INFO | BAZWRITER : 66.8%, 4.8, 165.7ms, 0 | 20260506 20:28:47.663 | INFO | BAZ_SAMPLING : 5.7%, 1.0, 3.0ms, 0 | 20260506 20:28:47.663 | INFO | Uploaded 36GiB to GPU in 2.08927s (17.2309GiB/s) | 20260506 20:28:47.663 | INFO | Downloaded 38GiB to GPU in 2.23618s (16.9933GiB/s) | 20260506 20:28:52.800 | INFO | Analyzing chunk frames = [4227584,4228096) | 20260506 20:28:52.800 | INFO | MemInfo: { | 20260506 20:28:52.800 | INFO | "gpu" : " 17 GB", | 20260506 20:28:52.800 | INFO | "huge" : "", | 20260506 20:28:52.800 | INFO | "huge_shared" : " 27 GB", | 20260506 20:28:52.800 | INFO | "malloc" : " 14 GB", | 20260506 20:28:52.800 | INFO | "pinned" : " 29 GB" | 20260506 20:28:52.800 | INFO | } | 20260506 20:28:52.800 | INFO | Perf report for the last 5.13737 seconds: | 20260506 20:28:52.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:28:52.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:52.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:52.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:52.800 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:28:52.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:52.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:52.800 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:28:52.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:52.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:52.801 | INFO | Uploaded 36GiB to GPU in 2.00546s (17.951GiB/s) | 20260506 20:28:52.801 | INFO | Downloaded 34GiB to GPU in 1.92461s (17.6659GiB/s) | 20260506 20:28:57.942 | INFO | Analyzing chunk frames = [4228096,4228608) | 20260506 20:28:57.942 | INFO | MemInfo: { | 20260506 20:28:57.942 | INFO | "gpu" : " 17 GB", | 20260506 20:28:57.942 | INFO | "huge" : "", | 20260506 20:28:57.942 | INFO | "huge_shared" : " 27 GB", | 20260506 20:28:57.942 | INFO | "malloc" : " 14 GB", | 20260506 20:28:57.942 | INFO | "pinned" : " 29 GB" | 20260506 20:28:57.942 | INFO | } | 20260506 20:28:57.942 | INFO | Perf report for the last 5.14168 seconds: | 20260506 20:28:57.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:28:57.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:57.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:57.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:57.942 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 20:28:57.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:57.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:57.942 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:28:57.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:57.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:28:57.942 | INFO | Uploaded 36GiB to GPU in 1.95877s (18.3789GiB/s) | 20260506 20:28:57.942 | INFO | Downloaded 34GiB to GPU in 1.91456s (17.7587GiB/s) | 20260506 20:29:03.081 | INFO | Analyzing chunk frames = [4228608,4229120) | 20260506 20:29:03.081 | INFO | MemInfo: { | 20260506 20:29:03.081 | INFO | "gpu" : " 17 GB", | 20260506 20:29:03.081 | INFO | "huge" : "", | 20260506 20:29:03.081 | INFO | "huge_shared" : " 27 GB", | 20260506 20:29:03.081 | INFO | "malloc" : " 14 GB", | 20260506 20:29:03.081 | INFO | "pinned" : " 29 GB" | 20260506 20:29:03.081 | INFO | } | 20260506 20:29:03.081 | INFO | Perf report for the last 5.13899 seconds: | 20260506 20:29:03.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:29:03.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:03.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:03.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:03.081 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:29:03.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:03.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:03.081 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:29:03.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:03.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:03.081 | INFO | Uploaded 36GiB to GPU in 1.93369s (18.6173GiB/s) | 20260506 20:29:03.081 | INFO | Downloaded 34GiB to GPU in 1.87138s (18.1684GiB/s) | 20260506 20:29:08.216 | INFO | Analyzing chunk frames = [4229120,4229632) | 20260506 20:29:08.216 | INFO | MemInfo: { | 20260506 20:29:08.216 | INFO | "gpu" : " 17 GB", | 20260506 20:29:08.216 | INFO | "huge" : "", | 20260506 20:29:08.216 | INFO | "huge_shared" : " 27 GB", | 20260506 20:29:08.217 | INFO | "malloc" : " 14 GB", | 20260506 20:29:08.217 | INFO | "pinned" : " 29 GB" | 20260506 20:29:08.217 | INFO | } | 20260506 20:29:08.217 | INFO | Perf report for the last 5.13555 seconds: | 20260506 20:29:08.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:29:08.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:08.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:08.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:08.217 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 20:29:08.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:08.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:08.217 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:29:08.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:08.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:08.217 | INFO | Uploaded 36GiB to GPU in 1.97339s (18.2427GiB/s) | 20260506 20:29:08.217 | INFO | Downloaded 34GiB to GPU in 1.92503s (17.6621GiB/s) | 20260506 20:29:13.367 | INFO | Analyzing chunk frames = [4229632,4230144) | 20260506 20:29:13.367 | INFO | MemInfo: { | 20260506 20:29:13.367 | INFO | "gpu" : " 17 GB", | 20260506 20:29:13.368 | INFO | "huge" : "", | 20260506 20:29:13.368 | INFO | "huge_shared" : " 27 GB", | 20260506 20:29:13.368 | INFO | "malloc" : " 14 GB", | 20260506 20:29:13.368 | INFO | "pinned" : " 29 GB" | 20260506 20:29:13.368 | INFO | } | 20260506 20:29:13.368 | INFO | Perf report for the last 5.15106 seconds: | 20260506 20:29:13.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:29:13.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:13.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:13.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:13.368 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260506 20:29:13.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:13.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:13.368 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:29:13.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:13.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:13.368 | INFO | Uploaded 36GiB to GPU in 2.00061s (17.9945GiB/s) | 20260506 20:29:13.368 | INFO | Downloaded 34GiB to GPU in 1.92931s (17.6229GiB/s) | 20260506 20:29:18.502 | INFO | Analyzing chunk frames = [4230144,4230656) | 20260506 20:29:18.502 | INFO | MemInfo: { | 20260506 20:29:18.502 | INFO | "gpu" : " 17 GB", | 20260506 20:29:18.502 | INFO | "huge" : "", | 20260506 20:29:18.502 | INFO | "huge_shared" : " 27 GB", | 20260506 20:29:18.502 | INFO | "malloc" : " 14 GB", | 20260506 20:29:18.502 | INFO | "pinned" : " 29 GB" | 20260506 20:29:18.502 | INFO | } | 20260506 20:29:18.502 | INFO | Perf report for the last 5.1348 seconds: | 20260506 20:29:18.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:29:18.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:18.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:18.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:18.503 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 20:29:18.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:18.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:18.503 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:29:18.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:18.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:18.503 | INFO | Uploaded 36GiB to GPU in 2.00968s (17.9133GiB/s) | 20260506 20:29:18.503 | INFO | Downloaded 34GiB to GPU in 1.99081s (17.0785GiB/s) | 20260506 20:29:23.641 | INFO | Analyzing chunk frames = [4230656,4231168) | 20260506 20:29:23.641 | INFO | MemInfo: { | 20260506 20:29:23.641 | INFO | "gpu" : " 17 GB", | 20260506 20:29:23.641 | INFO | "huge" : "", | 20260506 20:29:23.641 | INFO | "huge_shared" : " 27 GB", | 20260506 20:29:23.641 | INFO | "malloc" : " 14 GB", | 20260506 20:29:23.641 | INFO | "pinned" : " 29 GB" | 20260506 20:29:23.641 | INFO | } | 20260506 20:29:23.641 | INFO | Perf report for the last 5.13834 seconds: | 20260506 20:29:23.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:29:23.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:23.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:23.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:23.641 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:29:23.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:23.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:23.641 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 20:29:23.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:23.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:23.641 | INFO | Uploaded 36GiB to GPU in 1.98574s (18.1293GiB/s) | 20260506 20:29:23.641 | INFO | Downloaded 34GiB to GPU in 1.97281s (17.2343GiB/s) | 20260506 20:29:23.739 | INFO | Starting a round of Baz IO | 20260506 20:29:26.878 | INFO | Populating RTMetrics report | 20260506 20:29:26.879 | INFO | RTMetrics report populated | 20260506 20:29:26.879 | INFO | Opening tmp RTMetrics json file | 20260506 20:29:26.910 | INFO | Serializing RTMetrics to json | 20260506 20:29:26.914 | INFO | Serialization done, swapping tmp file now... | 20260506 20:29:26.918 | INFO | RTMetrics done | 20260506 20:29:27.121 | INFO | Finished a round of Baz IO: Wrote 1.976 GiB in 3.382 seconds | 20260506 20:29:27.121 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.23% / 0.00% / 65.76% | 20260506 20:29:27.121 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.743099% / 7.94729e-06% / 5.5631e-05% | 20260506 20:29:27.121 | INFO | Baz IO overall: | 20260506 20:29:27.121 | INFO | BytesWritten : 2233763658312 | 20260506 20:29:27.121 | INFO | NumEvents : 1079167000021 | 20260506 20:29:28.781 | INFO | Analyzing chunk frames = [4231168,4231680) | 20260506 20:29:28.781 | INFO | MemInfo: { | 20260506 20:29:28.781 | INFO | "gpu" : " 17 GB", | 20260506 20:29:28.781 | INFO | "huge" : "", | 20260506 20:29:28.781 | INFO | "huge_shared" : " 27 GB", | 20260506 20:29:28.781 | INFO | "malloc" : " 14 GB", | 20260506 20:29:28.781 | INFO | "pinned" : " 29 GB" | 20260506 20:29:28.781 | INFO | } | 20260506 20:29:28.781 | INFO | Perf report for the last 5.14003 seconds: | 20260506 20:29:28.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:29:28.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:28.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:28.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:28.781 | INFO | ANALYSIS : 62.4%, 2.9, 95.1ms, 0 | 20260506 20:29:28.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:28.781 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 20:29:28.781 | INFO | PRE_HQ : 59.9%, 1.8, 57.0ms, 0 | 20260506 20:29:28.781 | INFO | BAZWRITER : 65.8%, 4.8, 162.8ms, 0 | 20260506 20:29:28.781 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 20:29:28.781 | INFO | Uploaded 36GiB to GPU in 2.04133s (17.6356GiB/s) | 20260506 20:29:28.781 | INFO | Downloaded 38GiB to GPU in 2.20474s (17.2356GiB/s) | 20260506 20:29:33.915 | INFO | Analyzing chunk frames = [4231680,4232192) | 20260506 20:29:33.915 | INFO | MemInfo: { | 20260506 20:29:33.915 | INFO | "gpu" : " 17 GB", | 20260506 20:29:33.915 | INFO | "huge" : "", | 20260506 20:29:33.915 | INFO | "huge_shared" : " 27 GB", | 20260506 20:29:33.915 | INFO | "malloc" : " 14 GB", | 20260506 20:29:33.915 | INFO | "pinned" : " 29 GB" | 20260506 20:29:33.915 | INFO | } | 20260506 20:29:33.915 | INFO | Perf report for the last 5.13425 seconds: | 20260506 20:29:33.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:29:33.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:33.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:33.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:33.915 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:29:33.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:33.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:33.915 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:29:33.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:33.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:33.915 | INFO | Uploaded 36GiB to GPU in 2.01339s (17.8803GiB/s) | 20260506 20:29:33.915 | INFO | Downloaded 34GiB to GPU in 1.92133s (17.6961GiB/s) | 20260506 20:29:39.052 | INFO | Analyzing chunk frames = [4232192,4232704) | 20260506 20:29:39.052 | INFO | MemInfo: { | 20260506 20:29:39.052 | INFO | "gpu" : " 17 GB", | 20260506 20:29:39.052 | INFO | "huge" : "", | 20260506 20:29:39.052 | INFO | "huge_shared" : " 27 GB", | 20260506 20:29:39.052 | INFO | "malloc" : " 14 GB", | 20260506 20:29:39.052 | INFO | "pinned" : " 29 GB" | 20260506 20:29:39.052 | INFO | } | 20260506 20:29:39.052 | INFO | Perf report for the last 5.13692 seconds: | 20260506 20:29:39.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:29:39.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:39.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:39.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:39.052 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 20:29:39.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:39.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:39.052 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:29:39.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:39.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:39.052 | INFO | Uploaded 36GiB to GPU in 1.97186s (18.2569GiB/s) | 20260506 20:29:39.052 | INFO | Downloaded 34GiB to GPU in 1.91665s (17.7393GiB/s) | 20260506 20:29:44.191 | INFO | Analyzing chunk frames = [4232704,4233216) | 20260506 20:29:44.191 | INFO | MemInfo: { | 20260506 20:29:44.191 | INFO | "gpu" : " 17 GB", | 20260506 20:29:44.191 | INFO | "huge" : "", | 20260506 20:29:44.191 | INFO | "huge_shared" : " 27 GB", | 20260506 20:29:44.191 | INFO | "malloc" : " 14 GB", | 20260506 20:29:44.191 | INFO | "pinned" : " 29 GB" | 20260506 20:29:44.191 | INFO | } | 20260506 20:29:44.191 | INFO | Perf report for the last 5.13943 seconds: | 20260506 20:29:44.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:29:44.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:44.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:44.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:44.192 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 20:29:44.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:44.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:44.192 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:29:44.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:44.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:44.192 | INFO | Uploaded 36GiB to GPU in 1.93263s (18.6275GiB/s) | 20260506 20:29:44.192 | INFO | Downloaded 34GiB to GPU in 1.8696s (18.1857GiB/s) | 20260506 20:29:49.336 | INFO | Analyzing chunk frames = [4233216,4233728) | 20260506 20:29:49.336 | INFO | MemInfo: { | 20260506 20:29:49.336 | INFO | "gpu" : " 17 GB", | 20260506 20:29:49.337 | INFO | "huge" : "", | 20260506 20:29:49.337 | INFO | "huge_shared" : " 27 GB", | 20260506 20:29:49.337 | INFO | "malloc" : " 14 GB", | 20260506 20:29:49.337 | INFO | "pinned" : " 29 GB" | 20260506 20:29:49.337 | INFO | } | 20260506 20:29:49.337 | INFO | Perf report for the last 5.14513 seconds: | 20260506 20:29:49.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:29:49.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:49.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:49.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:49.337 | INFO | ANALYSIS : 59.8%, 3.0, 93.2ms, 0 | 20260506 20:29:49.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:49.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:49.337 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 20:29:49.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:49.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:49.337 | INFO | Uploaded 36GiB to GPU in 1.95525s (18.412GiB/s) | 20260506 20:29:49.337 | INFO | Downloaded 34GiB to GPU in 1.88305s (18.0558GiB/s) | 20260506 20:29:54.476 | INFO | Analyzing chunk frames = [4233728,4234240) | 20260506 20:29:54.476 | INFO | MemInfo: { | 20260506 20:29:54.476 | INFO | "gpu" : " 17 GB", | 20260506 20:29:54.476 | INFO | "huge" : "", | 20260506 20:29:54.476 | INFO | "huge_shared" : " 27 GB", | 20260506 20:29:54.476 | INFO | "malloc" : " 14 GB", | 20260506 20:29:54.476 | INFO | "pinned" : " 29 GB" | 20260506 20:29:54.476 | INFO | } | 20260506 20:29:54.476 | INFO | Perf report for the last 5.13977 seconds: | 20260506 20:29:54.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:29:54.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:54.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:54.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:54.476 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:29:54.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:54.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:54.477 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 20:29:54.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:54.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:54.477 | INFO | Uploaded 36GiB to GPU in 2.00903s (17.9191GiB/s) | 20260506 20:29:54.477 | INFO | Downloaded 34GiB to GPU in 1.95049s (17.4315GiB/s) | 20260506 20:29:59.619 | INFO | Analyzing chunk frames = [4234240,4234752) | 20260506 20:29:59.619 | INFO | MemInfo: { | 20260506 20:29:59.619 | INFO | "gpu" : " 17 GB", | 20260506 20:29:59.619 | INFO | "huge" : "", | 20260506 20:29:59.619 | INFO | "huge_shared" : " 27 GB", | 20260506 20:29:59.619 | INFO | "malloc" : " 14 GB", | 20260506 20:29:59.619 | INFO | "pinned" : " 29 GB" | 20260506 20:29:59.619 | INFO | } | 20260506 20:29:59.619 | INFO | Perf report for the last 5.14277 seconds: | 20260506 20:29:59.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:29:59.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:59.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:59.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:59.619 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:29:59.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:59.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:59.619 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:29:59.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:59.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:29:59.619 | INFO | Uploaded 36GiB to GPU in 1.9876s (18.1123GiB/s) | 20260506 20:29:59.619 | INFO | Downloaded 34GiB to GPU in 2.00146s (16.9876GiB/s) | 20260506 20:30:04.757 | INFO | Analyzing chunk frames = [4234752,4235264) | 20260506 20:30:04.757 | INFO | MemInfo: { | 20260506 20:30:04.757 | INFO | "gpu" : " 17 GB", | 20260506 20:30:04.757 | INFO | "huge" : "", | 20260506 20:30:04.757 | INFO | "huge_shared" : " 27 GB", | 20260506 20:30:04.757 | INFO | "malloc" : " 14 GB", | 20260506 20:30:04.757 | INFO | "pinned" : " 29 GB" | 20260506 20:30:04.757 | INFO | } | 20260506 20:30:04.757 | INFO | Perf report for the last 5.13802 seconds: | 20260506 20:30:04.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:30:04.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:04.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:04.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:04.757 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:30:04.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:04.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:04.757 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 20:30:04.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:04.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:04.757 | INFO | Uploaded 36GiB to GPU in 1.98372s (18.1478GiB/s) | 20260506 20:30:04.757 | INFO | Downloaded 34GiB to GPU in 1.981s (17.163GiB/s) | 20260506 20:30:04.758 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:30:04.758 | INFO | Total mem usage: 18 GiB | 20260506 20:30:04.758 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:30:04.758 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.3%) | 20260506 20:30:04.758 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:30:04.758 | INFO | High Water Breakdown: | 20260506 20:30:04.758 | INFO | 61.7% : 9.81 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:30:04.758 | INFO | 38.1% : 6.06 GiB ( 90.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:30:04.758 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:30:04.758 | INFO | ------------------------------------------ | 20260506 20:30:04.758 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:30:04.758 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:30:04.758 | INFO | Total mem usage: 26.5 GiB | 20260506 20:30:04.758 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:30:04.758 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:30:04.758 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:30:04.758 | INFO | High Water Breakdown: | 20260506 20:30:04.758 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:30:04.758 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:30:04.758 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:30:04.758 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:30:04.758 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:30:04.758 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:30:04.758 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:30:04.758 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:30:04.758 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:30:04.758 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:30:04.758 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:30:04.758 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:30:04.758 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:30:04.758 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:30:04.758 | INFO | ------------------------------------------ | 20260506 20:30:04.758 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:30:04.758 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:30:04.758 | INFO | Total mem usage: 25.6 GiB | 20260506 20:30:04.758 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:30:04.758 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 20:30:04.758 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 20:30:04.758 | INFO | High Water Breakdown: | 20260506 20:30:04.758 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 20:30:04.758 | INFO | ------------------------------------------ | 20260506 20:30:04.758 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:30:04.758 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:30:04.758 | INFO | Total mem usage: 15.7 GiB | 20260506 20:30:04.758 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:30:04.758 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:30:04.758 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 20:30:04.758 | INFO | High Water Breakdown: | 20260506 20:30:04.758 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:30:04.758 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:30:04.758 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:30:04.758 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:30:04.758 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:30:04.758 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:30:04.758 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:30:04.758 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:30:04.758 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:30:04.758 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:30:04.758 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:30:04.758 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:30:04.758 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:30:04.758 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:30:04.758 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:30:04.758 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:30:04.758 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:30:04.758 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:30:04.758 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:30:04.758 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:30:04.758 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:30:04.758 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:30:04.758 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:30:04.758 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:30:04.758 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:30:04.758 | INFO | ------------------------------------------ | 20260506 20:30:04.758 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:30:04.758 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:30:04.758 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:30:04.758 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:30:04.758 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.57e+01, 1.33e+01 / 2.56e+00 | 20260506 20:30:04.758 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.00e-01, 1.03e-02 / 7.14e-03 | 20260506 20:30:04.758 | INFO | PerfReport: 11.60%, Baseline -- 3.62e+00 / 4.32e+00, 5.47e-01 / 3.84e-01 | 20260506 20:30:04.758 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.44e+00, 1.82e-01 / 1.54e-01 | 20260506 20:30:04.758 | INFO | PerfReport: 16.01%, DME -- 3.99e+01 / 4.46e+01, 4.14e+00 / 2.93e+00 | 20260506 20:30:04.758 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.79e+01 / 2.07e+01, 1.47e+00 / 1.92e+00 | 20260506 20:30:04.758 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.22e+00 / 2.80e+00, 3.79e-01 / 2.24e-01 | 20260506 20:30:04.758 | INFO | PerfReport: 3.30%, Metrics -- 1.03e+00 / 1.38e+00, 2.73e-01 / 1.66e-01 | 20260506 20:30:04.758 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.54e-01 / 4.03e-01, 5.56e-02 / 2.02e-02 | 20260506 20:30:04.758 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:30:04.758 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:30:04.758 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:30:04.759 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 20:30:04.759 | INFO | PerfReport: 21.47%, Upload -- 2.07e+01 / 0.00e+00, 6.89e+00 / 0.00e+00 | 20260506 20:30:04.759 | INFO | PerfReport: 21.22%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 20:30:04.759 | INFO | PerfReport: 32.28%, Compute -- 3.12e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 20:30:04.759 | INFO | PerfReport: 1.24%, LockUpload -- 1.19e+00 / 0.00e+00, 4.62e+00 / 0.00e+00 | 20260506 20:30:04.759 | INFO | PerfReport: 1.16%, LockDownload -- 1.12e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 20:30:04.759 | INFO | PerfReport: 22.62%, LockCompute -- 2.19e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 20:30:04.759 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260506 20:30:04.856 | INFO | Starting a round of Baz IO | 20260506 20:30:07.990 | INFO | Populating RTMetrics report | 20260506 20:30:07.991 | INFO | RTMetrics report populated | 20260506 20:30:07.991 | INFO | Opening tmp RTMetrics json file | 20260506 20:30:08.032 | INFO | Serializing RTMetrics to json | 20260506 20:30:08.036 | INFO | Serialization done, swapping tmp file now... | 20260506 20:30:08.039 | INFO | RTMetrics done | 20260506 20:30:08.313 | INFO | Finished a round of Baz IO: Wrote 1.975 GiB in 3.456 seconds | 20260506 20:30:08.313 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.23% / 0.00% / 65.77% | 20260506 20:30:08.313 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.741259% / 3.97364e-06% / 6.35783e-05% | 20260506 20:30:08.313 | INFO | Baz IO overall: | 20260506 20:30:08.313 | INFO | BytesWritten : 2235884316713 | 20260506 20:30:08.313 | INFO | NumEvents : 1080183890179 | 20260506 20:30:09.895 | INFO | Analyzing chunk frames = [4235264,4235776) | 20260506 20:30:09.895 | INFO | MemInfo: { | 20260506 20:30:09.895 | INFO | "gpu" : " 17 GB", | 20260506 20:30:09.895 | INFO | "huge" : "", | 20260506 20:30:09.895 | INFO | "huge_shared" : " 27 GB", | 20260506 20:30:09.895 | INFO | "malloc" : " 14 GB", | 20260506 20:30:09.895 | INFO | "pinned" : " 29 GB" | 20260506 20:30:09.895 | INFO | } | 20260506 20:30:09.895 | INFO | Perf report for the last 5.1377 seconds: | 20260506 20:30:09.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:30:09.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:09.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:09.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:09.895 | INFO | ANALYSIS : 62.4%, 2.9, 95.2ms, 0 | 20260506 20:30:09.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:09.895 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260506 20:30:09.895 | INFO | PRE_HQ : 60.0%, 1.8, 57.1ms, 0 | 20260506 20:30:09.895 | INFO | BAZWRITER : 66.9%, 4.8, 164.3ms, 0 | 20260506 20:30:09.895 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 20:30:09.895 | INFO | Uploaded 36GiB to GPU in 2.07681s (17.3342GiB/s) | 20260506 20:30:09.895 | INFO | Downloaded 38GiB to GPU in 2.19738s (17.2933GiB/s) | 20260506 20:30:15.031 | INFO | Analyzing chunk frames = [4235776,4236288) | 20260506 20:30:15.031 | INFO | MemInfo: { | 20260506 20:30:15.031 | INFO | "gpu" : " 17 GB", | 20260506 20:30:15.031 | INFO | "huge" : "", | 20260506 20:30:15.031 | INFO | "huge_shared" : " 27 GB", | 20260506 20:30:15.031 | INFO | "malloc" : " 14 GB", | 20260506 20:30:15.031 | INFO | "pinned" : " 29 GB" | 20260506 20:30:15.031 | INFO | } | 20260506 20:30:15.031 | INFO | Perf report for the last 5.13609 seconds: | 20260506 20:30:15.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:30:15.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:15.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:15.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:15.031 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:30:15.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:15.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:15.031 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:30:15.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:15.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:15.031 | INFO | Uploaded 36GiB to GPU in 2.01573s (17.8595GiB/s) | 20260506 20:30:15.031 | INFO | Downloaded 34GiB to GPU in 1.92617s (17.6516GiB/s) | 20260506 20:30:20.164 | INFO | Analyzing chunk frames = [4236288,4236800) | 20260506 20:30:20.164 | INFO | MemInfo: { | 20260506 20:30:20.164 | INFO | "gpu" : " 17 GB", | 20260506 20:30:20.164 | INFO | "huge" : "", | 20260506 20:30:20.164 | INFO | "huge_shared" : " 27 GB", | 20260506 20:30:20.164 | INFO | "malloc" : " 14 GB", | 20260506 20:30:20.164 | INFO | "pinned" : " 29 GB" | 20260506 20:30:20.164 | INFO | } | 20260506 20:30:20.164 | INFO | Perf report for the last 5.13293 seconds: | 20260506 20:30:20.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:30:20.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:20.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:20.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:20.164 | INFO | ANALYSIS : 60.9%, 3.0, 94.1ms, 0 | 20260506 20:30:20.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:20.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:20.164 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 20:30:20.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:20.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:20.164 | INFO | Uploaded 36GiB to GPU in 1.97774s (18.2026GiB/s) | 20260506 20:30:20.164 | INFO | Downloaded 34GiB to GPU in 1.9168s (17.7379GiB/s) | 20260506 20:30:25.311 | INFO | Analyzing chunk frames = [4236800,4237312) | 20260506 20:30:25.311 | INFO | MemInfo: { | 20260506 20:30:25.311 | INFO | "gpu" : " 17 GB", | 20260506 20:30:25.311 | INFO | "huge" : "", | 20260506 20:30:25.311 | INFO | "huge_shared" : " 27 GB", | 20260506 20:30:25.311 | INFO | "malloc" : " 14 GB", | 20260506 20:30:25.311 | INFO | "pinned" : " 29 GB" | 20260506 20:30:25.311 | INFO | } | 20260506 20:30:25.311 | INFO | Perf report for the last 5.1471 seconds: | 20260506 20:30:25.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:30:25.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:25.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:25.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:25.311 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:30:25.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:25.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:25.311 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:30:25.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:25.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:25.311 | INFO | Uploaded 36GiB to GPU in 1.93316s (18.6224GiB/s) | 20260506 20:30:25.311 | INFO | Downloaded 34GiB to GPU in 1.87032s (18.1787GiB/s) | 20260506 20:30:30.459 | INFO | Analyzing chunk frames = [4237312,4237824) | 20260506 20:30:30.459 | INFO | MemInfo: { | 20260506 20:30:30.459 | INFO | "gpu" : " 17 GB", | 20260506 20:30:30.459 | INFO | "huge" : "", | 20260506 20:30:30.459 | INFO | "huge_shared" : " 27 GB", | 20260506 20:30:30.459 | INFO | "malloc" : " 14 GB", | 20260506 20:30:30.459 | INFO | "pinned" : " 29 GB" | 20260506 20:30:30.459 | INFO | } | 20260506 20:30:30.459 | INFO | Perf report for the last 5.14807 seconds: | 20260506 20:30:30.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:30:30.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:30.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:30.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:30.459 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260506 20:30:30.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:30.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:30.459 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:30:30.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:30.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:30.459 | INFO | Uploaded 36GiB to GPU in 2.00145s (17.987GiB/s) | 20260506 20:30:30.460 | INFO | Downloaded 34GiB to GPU in 1.90638s (17.8348GiB/s) | 20260506 20:30:35.592 | INFO | Analyzing chunk frames = [4237824,4238336) | 20260506 20:30:35.592 | INFO | MemInfo: { | 20260506 20:30:35.592 | INFO | "gpu" : " 17 GB", | 20260506 20:30:35.592 | INFO | "huge" : "", | 20260506 20:30:35.592 | INFO | "huge_shared" : " 27 GB", | 20260506 20:30:35.592 | INFO | "malloc" : " 14 GB", | 20260506 20:30:35.592 | INFO | "pinned" : " 29 GB" | 20260506 20:30:35.592 | INFO | } | 20260506 20:30:35.592 | INFO | Perf report for the last 5.13279 seconds: | 20260506 20:30:35.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:30:35.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:35.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:35.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:35.592 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 20:30:35.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:35.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:35.592 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 20:30:35.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:35.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:35.592 | INFO | Uploaded 36GiB to GPU in 2.03267s (17.7107GiB/s) | 20260506 20:30:35.592 | INFO | Downloaded 34GiB to GPU in 1.93658s (17.5568GiB/s) | 20260506 20:30:40.729 | INFO | Analyzing chunk frames = [4238336,4238848) | 20260506 20:30:40.729 | INFO | MemInfo: { | 20260506 20:30:40.729 | INFO | "gpu" : " 17 GB", | 20260506 20:30:40.729 | INFO | "huge" : "", | 20260506 20:30:40.729 | INFO | "huge_shared" : " 27 GB", | 20260506 20:30:40.729 | INFO | "malloc" : " 14 GB", | 20260506 20:30:40.729 | INFO | "pinned" : " 29 GB" | 20260506 20:30:40.729 | INFO | } | 20260506 20:30:40.729 | INFO | Perf report for the last 5.13738 seconds: | 20260506 20:30:40.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:30:40.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:40.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:40.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:40.729 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 20:30:40.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:40.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:40.729 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 20:30:40.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:40.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:40.730 | INFO | Uploaded 36GiB to GPU in 2.00843s (17.9245GiB/s) | 20260506 20:30:40.730 | INFO | Downloaded 34GiB to GPU in 1.98809s (17.1019GiB/s) | 20260506 20:30:45.867 | INFO | Analyzing chunk frames = [4238848,4239360) | 20260506 20:30:45.867 | INFO | MemInfo: { | 20260506 20:30:45.867 | INFO | "gpu" : " 17 GB", | 20260506 20:30:45.867 | INFO | "huge" : "", | 20260506 20:30:45.867 | INFO | "huge_shared" : " 27 GB", | 20260506 20:30:45.867 | INFO | "malloc" : " 14 GB", | 20260506 20:30:45.867 | INFO | "pinned" : " 29 GB" | 20260506 20:30:45.867 | INFO | } | 20260506 20:30:45.867 | INFO | Perf report for the last 5.13808 seconds: | 20260506 20:30:45.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:30:45.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:45.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:45.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:45.867 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 20:30:45.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:45.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:45.867 | INFO | PRE_HQ : 31.4%, 1.0, 16.3ms, 0 | 20260506 20:30:45.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:45.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:45.868 | INFO | Uploaded 36GiB to GPU in 2.04476s (17.606GiB/s) | 20260506 20:30:45.868 | INFO | Downloaded 34GiB to GPU in 1.97555s (17.2104GiB/s) | 20260506 20:30:45.983 | INFO | Starting a round of Baz IO | 20260506 20:30:49.104 | INFO | Populating RTMetrics report | 20260506 20:30:49.105 | INFO | RTMetrics report populated | 20260506 20:30:49.105 | INFO | Opening tmp RTMetrics json file | 20260506 20:30:49.105 | INFO | Serializing RTMetrics to json | 20260506 20:30:49.107 | INFO | Serialization done, swapping tmp file now... | 20260506 20:30:49.108 | INFO | RTMetrics done | 20260506 20:30:49.276 | INFO | Finished a round of Baz IO: Wrote 1.975 GiB in 3.293 seconds | 20260506 20:30:49.276 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.22% / 0.00% / 65.77% | 20260506 20:30:49.276 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.740115% / 1.19209e-05% / 6.75519e-05% | 20260506 20:30:49.276 | INFO | Baz IO overall: | 20260506 20:30:49.276 | INFO | BytesWritten : 2238004975609 | 20260506 20:30:49.276 | INFO | NumEvents : 1081200707395 | 20260506 20:30:51.006 | INFO | Analyzing chunk frames = [4239360,4239872) | 20260506 20:30:51.006 | INFO | MemInfo: { | 20260506 20:30:51.006 | INFO | "gpu" : " 17 GB", | 20260506 20:30:51.006 | INFO | "huge" : "", | 20260506 20:30:51.006 | INFO | "huge_shared" : " 27 GB", | 20260506 20:30:51.006 | INFO | "malloc" : " 14 GB", | 20260506 20:30:51.006 | INFO | "pinned" : " 29 GB" | 20260506 20:30:51.006 | INFO | } | 20260506 20:30:51.006 | INFO | Perf report for the last 5.1383 seconds: | 20260506 20:30:51.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:30:51.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:51.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:51.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:51.006 | INFO | ANALYSIS : 62.4%, 2.9, 95.8ms, 0 | 20260506 20:30:51.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:51.006 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 20:30:51.006 | INFO | PRE_HQ : 59.6%, 1.9, 57.4ms, 0 | 20260506 20:30:51.006 | INFO | BAZWRITER : 63.7%, 4.3, 139.3ms, 0 | 20260506 20:30:51.006 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 20:30:51.006 | INFO | Uploaded 36GiB to GPU in 2.0468s (17.5885GiB/s) | 20260506 20:30:51.006 | INFO | Downloaded 38GiB to GPU in 2.19799s (17.2885GiB/s) | 20260506 20:30:56.143 | INFO | Analyzing chunk frames = [4239872,4240384) | 20260506 20:30:56.143 | INFO | MemInfo: { | 20260506 20:30:56.143 | INFO | "gpu" : " 17 GB", | 20260506 20:30:56.143 | INFO | "huge" : "", | 20260506 20:30:56.144 | INFO | "huge_shared" : " 27 GB", | 20260506 20:30:56.144 | INFO | "malloc" : " 14 GB", | 20260506 20:30:56.144 | INFO | "pinned" : " 29 GB" | 20260506 20:30:56.144 | INFO | } | 20260506 20:30:56.144 | INFO | Perf report for the last 5.13784 seconds: | 20260506 20:30:56.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:30:56.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:56.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:56.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:56.144 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:30:56.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:56.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:56.144 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:30:56.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:56.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:30:56.144 | INFO | Uploaded 36GiB to GPU in 2.0187s (17.8333GiB/s) | 20260506 20:30:56.144 | INFO | Downloaded 34GiB to GPU in 1.92975s (17.6189GiB/s) | 20260506 20:31:01.282 | INFO | Analyzing chunk frames = [4240384,4240896) | 20260506 20:31:01.282 | INFO | MemInfo: { | 20260506 20:31:01.282 | INFO | "gpu" : " 17 GB", | 20260506 20:31:01.282 | INFO | "huge" : "", | 20260506 20:31:01.282 | INFO | "huge_shared" : " 27 GB", | 20260506 20:31:01.282 | INFO | "malloc" : " 14 GB", | 20260506 20:31:01.282 | INFO | "pinned" : " 29 GB" | 20260506 20:31:01.282 | INFO | } | 20260506 20:31:01.282 | INFO | Perf report for the last 5.13843 seconds: | 20260506 20:31:01.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:31:01.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:01.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:01.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:01.282 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 20:31:01.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:01.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:01.282 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:31:01.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:01.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:01.283 | INFO | Uploaded 36GiB to GPU in 1.98268s (18.1572GiB/s) | 20260506 20:31:01.283 | INFO | Downloaded 34GiB to GPU in 1.91933s (17.7145GiB/s) | 20260506 20:31:06.430 | INFO | Analyzing chunk frames = [4240896,4241408) | 20260506 20:31:06.430 | INFO | MemInfo: { | 20260506 20:31:06.430 | INFO | "gpu" : " 17 GB", | 20260506 20:31:06.430 | INFO | "huge" : "", | 20260506 20:31:06.430 | INFO | "huge_shared" : " 27 GB", | 20260506 20:31:06.430 | INFO | "malloc" : " 14 GB", | 20260506 20:31:06.430 | INFO | "pinned" : " 29 GB" | 20260506 20:31:06.430 | INFO | } | 20260506 20:31:06.430 | INFO | Perf report for the last 5.14789 seconds: | 20260506 20:31:06.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:31:06.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:06.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:06.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:06.431 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:31:06.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:06.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:06.431 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:31:06.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:06.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:06.431 | INFO | Uploaded 36GiB to GPU in 1.93158s (18.6375GiB/s) | 20260506 20:31:06.431 | INFO | Downloaded 34GiB to GPU in 1.86794s (18.2019GiB/s) | 20260506 20:31:11.573 | INFO | Analyzing chunk frames = [4241408,4241920) | 20260506 20:31:11.573 | INFO | MemInfo: { | 20260506 20:31:11.573 | INFO | "gpu" : " 17 GB", | 20260506 20:31:11.573 | INFO | "huge" : "", | 20260506 20:31:11.573 | INFO | "huge_shared" : " 27 GB", | 20260506 20:31:11.573 | INFO | "malloc" : " 14 GB", | 20260506 20:31:11.573 | INFO | "pinned" : " 29 GB" | 20260506 20:31:11.573 | INFO | } | 20260506 20:31:11.573 | INFO | Perf report for the last 5.14329 seconds: | 20260506 20:31:11.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:31:11.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:11.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:11.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:11.573 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 20:31:11.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:11.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:11.573 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:31:11.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:11.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:11.574 | INFO | Uploaded 36GiB to GPU in 1.9902s (18.0887GiB/s) | 20260506 20:31:11.574 | INFO | Downloaded 34GiB to GPU in 1.92054s (17.7033GiB/s) | 20260506 20:31:16.712 | INFO | Analyzing chunk frames = [4241920,4242432) | 20260506 20:31:16.712 | INFO | MemInfo: { | 20260506 20:31:16.712 | INFO | "gpu" : " 17 GB", | 20260506 20:31:16.712 | INFO | "huge" : "", | 20260506 20:31:16.712 | INFO | "huge_shared" : " 27 GB", | 20260506 20:31:16.712 | INFO | "malloc" : " 14 GB", | 20260506 20:31:16.712 | INFO | "pinned" : " 29 GB" | 20260506 20:31:16.712 | INFO | } | 20260506 20:31:16.712 | INFO | Perf report for the last 5.1389 seconds: | 20260506 20:31:16.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:31:16.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:16.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:16.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:16.712 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 20:31:16.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:16.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:16.712 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:31:16.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:16.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:16.712 | INFO | Uploaded 36GiB to GPU in 2.05089s (17.5533GiB/s) | 20260506 20:31:16.712 | INFO | Downloaded 34GiB to GPU in 1.94595s (17.4722GiB/s) | 20260506 20:31:21.843 | INFO | Analyzing chunk frames = [4242432,4242944) | 20260506 20:31:21.843 | INFO | MemInfo: { | 20260506 20:31:21.843 | INFO | "gpu" : " 17 GB", | 20260506 20:31:21.843 | INFO | "huge" : "", | 20260506 20:31:21.843 | INFO | "huge_shared" : " 27 GB", | 20260506 20:31:21.843 | INFO | "malloc" : " 14 GB", | 20260506 20:31:21.843 | INFO | "pinned" : " 29 GB" | 20260506 20:31:21.843 | INFO | } | 20260506 20:31:21.843 | INFO | Perf report for the last 5.13133 seconds: | 20260506 20:31:21.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:31:21.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:21.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:21.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:21.844 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260506 20:31:21.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:21.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:21.844 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 20:31:21.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:21.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:21.844 | INFO | Uploaded 36GiB to GPU in 1.99432s (18.0513GiB/s) | 20260506 20:31:21.844 | INFO | Downloaded 34GiB to GPU in 2.00137s (16.9884GiB/s) | 20260506 20:31:26.979 | INFO | Analyzing chunk frames = [4242944,4243456) | 20260506 20:31:26.979 | INFO | MemInfo: { | 20260506 20:31:26.979 | INFO | "gpu" : " 17 GB", | 20260506 20:31:26.979 | INFO | "huge" : "", | 20260506 20:31:26.979 | INFO | "huge_shared" : " 27 GB", | 20260506 20:31:26.979 | INFO | "malloc" : " 14 GB", | 20260506 20:31:26.979 | INFO | "pinned" : " 29 GB" | 20260506 20:31:26.979 | INFO | } | 20260506 20:31:26.979 | INFO | Perf report for the last 5.13538 seconds: | 20260506 20:31:26.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:31:26.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:26.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:26.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:26.979 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 20:31:26.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:26.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:26.979 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 20:31:26.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:26.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:26.979 | INFO | Uploaded 36GiB to GPU in 1.98485s (18.1374GiB/s) | 20260506 20:31:26.979 | INFO | Downloaded 34GiB to GPU in 1.97455s (17.2191GiB/s) | 20260506 20:31:26.979 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:31:26.979 | INFO | Total mem usage: 18 GiB | 20260506 20:31:26.979 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:31:26.979 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -14.9%) | 20260506 20:31:26.979 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:31:26.979 | INFO | High Water Breakdown: | 20260506 20:31:26.979 | INFO | 61.5% : 9.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:31:26.979 | INFO | 38.3% : 6.01 GiB ( 89.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:31:26.979 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:31:26.979 | INFO | ------------------------------------------ | 20260506 20:31:26.979 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:31:26.979 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:31:26.980 | INFO | Total mem usage: 26.5 GiB | 20260506 20:31:26.980 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:31:26.980 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:31:26.980 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:31:26.980 | INFO | High Water Breakdown: | 20260506 20:31:26.980 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:31:26.980 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:31:26.980 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:31:26.980 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:31:26.980 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:31:26.980 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:31:26.980 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:31:26.980 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:31:26.980 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:31:26.980 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:31:26.980 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:31:26.980 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:31:26.980 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:31:26.980 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:31:26.980 | INFO | ------------------------------------------ | 20260506 20:31:26.980 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:31:26.980 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:31:26.980 | INFO | Total mem usage: 25.6 GiB | 20260506 20:31:26.980 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:31:26.980 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:31:26.980 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:31:26.980 | INFO | High Water Breakdown: | 20260506 20:31:26.980 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:31:26.980 | INFO | ------------------------------------------ | 20260506 20:31:26.980 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:31:26.980 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:31:26.980 | INFO | Total mem usage: 15.7 GiB | 20260506 20:31:26.980 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:31:26.980 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:31:26.980 | INFO | Unused Cache: 866 MiB (Waste: 5.4%) | 20260506 20:31:26.980 | INFO | High Water Breakdown: | 20260506 20:31:26.980 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:31:26.980 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:31:26.980 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:31:26.980 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:31:26.980 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:31:26.980 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:31:26.980 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:31:26.980 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:31:26.980 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:31:26.980 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:31:26.980 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:31:26.980 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:31:26.980 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:31:26.980 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:31:26.980 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:31:26.980 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:31:26.980 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:31:26.980 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:31:26.980 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:31:26.980 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:31:26.980 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:31:26.980 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:31:26.980 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:31:26.980 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:31:26.980 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:31:26.980 | INFO | ------------------------------------------ | 20260506 20:31:26.980 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:31:26.981 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:31:26.981 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:31:26.981 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:31:26.981 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.33e+01 / 2.22e+00 | 20260506 20:31:26.981 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.00e-01, 1.00e-02 / 5.30e-03 | 20260506 20:31:26.981 | INFO | PerfReport: 11.65%, Baseline -- 3.63e+00 / 4.45e+00, 5.59e-01 / 3.56e-01 | 20260506 20:31:26.981 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.81e-01 / 1.42e-01 | 20260506 20:31:26.981 | INFO | PerfReport: 16.03%, DME -- 4.00e+01 / 4.49e+01, 4.11e+00 / 2.09e+00 | 20260506 20:31:26.981 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.38e+00 / 1.52e+00 | 20260506 20:31:26.981 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.23e+00 / 2.75e+00, 3.83e-01 / 2.23e-01 | 20260506 20:31:26.981 | INFO | PerfReport: 3.31%, Metrics -- 1.03e+00 / 1.36e+00, 2.73e-01 / 1.62e-01 | 20260506 20:31:26.981 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.54e-01 / 4.00e-01, 5.52e-02 / 1.69e-02 | 20260506 20:31:26.981 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:31:26.981 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:31:26.981 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:31:26.981 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 20:31:26.981 | INFO | PerfReport: 21.57%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260506 20:31:26.981 | INFO | PerfReport: 21.15%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 20:31:26.981 | INFO | PerfReport: 32.17%, Compute -- 3.12e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 20:31:26.981 | INFO | PerfReport: 1.41%, LockUpload -- 1.36e+00 / 0.00e+00, 5.21e+00 / 0.00e+00 | 20260506 20:31:26.981 | INFO | PerfReport: 1.11%, LockDownload -- 1.08e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260506 20:31:26.981 | INFO | PerfReport: 22.58%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 20:31:26.981 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260506 20:31:27.082 | INFO | Starting a round of Baz IO | 20260506 20:31:30.219 | INFO | Populating RTMetrics report | 20260506 20:31:30.220 | INFO | RTMetrics report populated | 20260506 20:31:30.220 | INFO | Opening tmp RTMetrics json file | 20260506 20:31:30.311 | INFO | Serializing RTMetrics to json | 20260506 20:31:30.316 | INFO | Serialization done, swapping tmp file now... | 20260506 20:31:30.317 | INFO | RTMetrics done | 20260506 20:31:30.524 | INFO | Finished a round of Baz IO: Wrote 1.975 GiB in 3.441 seconds | 20260506 20:31:30.524 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.22% / 0.00% / 65.78% | 20260506 20:31:30.524 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.738509% / 0% / 5.96046e-05% | 20260506 20:31:30.524 | INFO | Baz IO overall: | 20260506 20:31:30.524 | INFO | BytesWritten : 2240125670948 | 20260506 20:31:30.524 | INFO | NumEvents : 1082217576944 | 20260506 20:31:32.119 | INFO | Analyzing chunk frames = [4243456,4243968) | 20260506 20:31:32.119 | INFO | MemInfo: { | 20260506 20:31:32.119 | INFO | "gpu" : " 17 GB", | 20260506 20:31:32.119 | INFO | "huge" : "", | 20260506 20:31:32.120 | INFO | "huge_shared" : " 27 GB", | 20260506 20:31:32.120 | INFO | "malloc" : " 14 GB", | 20260506 20:31:32.120 | INFO | "pinned" : " 29 GB" | 20260506 20:31:32.120 | INFO | } | 20260506 20:31:32.120 | INFO | Perf report for the last 5.14064 seconds: | 20260506 20:31:32.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:31:32.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:32.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:32.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:32.120 | INFO | ANALYSIS : 62.4%, 2.9, 95.1ms, 0 | 20260506 20:31:32.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:32.120 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 20:31:32.120 | INFO | PRE_HQ : 60.0%, 1.8, 57.6ms, 0 | 20260506 20:31:32.120 | INFO | BAZWRITER : 66.6%, 4.8, 164.6ms, 0 | 20260506 20:31:32.120 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 20:31:32.120 | INFO | Uploaded 36GiB to GPU in 2.05293s (17.5359GiB/s) | 20260506 20:31:32.120 | INFO | Downloaded 38GiB to GPU in 2.19543s (17.3087GiB/s) | 20260506 20:31:37.259 | INFO | Analyzing chunk frames = [4243968,4244480) | 20260506 20:31:37.259 | INFO | MemInfo: { | 20260506 20:31:37.259 | INFO | "gpu" : " 17 GB", | 20260506 20:31:37.259 | INFO | "huge" : "", | 20260506 20:31:37.259 | INFO | "huge_shared" : " 27 GB", | 20260506 20:31:37.259 | INFO | "malloc" : " 14 GB", | 20260506 20:31:37.259 | INFO | "pinned" : " 29 GB" | 20260506 20:31:37.259 | INFO | } | 20260506 20:31:37.259 | INFO | Perf report for the last 5.13926 seconds: | 20260506 20:31:37.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:31:37.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:37.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:37.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:37.259 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:31:37.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:37.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:37.259 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:31:37.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:37.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:37.259 | INFO | Uploaded 36GiB to GPU in 2.01464s (17.8692GiB/s) | 20260506 20:31:37.259 | INFO | Downloaded 34GiB to GPU in 1.93062s (17.6109GiB/s) | 20260506 20:31:42.402 | INFO | Analyzing chunk frames = [4244480,4244992) | 20260506 20:31:42.402 | INFO | MemInfo: { | 20260506 20:31:42.402 | INFO | "gpu" : " 17 GB", | 20260506 20:31:42.402 | INFO | "huge" : "", | 20260506 20:31:42.402 | INFO | "huge_shared" : " 27 GB", | 20260506 20:31:42.402 | INFO | "malloc" : " 14 GB", | 20260506 20:31:42.402 | INFO | "pinned" : " 29 GB" | 20260506 20:31:42.403 | INFO | } | 20260506 20:31:42.403 | INFO | Perf report for the last 5.14386 seconds: | 20260506 20:31:42.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:31:42.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:42.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:42.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:42.403 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 20:31:42.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:42.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:42.403 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:31:42.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:42.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:42.403 | INFO | Uploaded 36GiB to GPU in 1.9808s (18.1745GiB/s) | 20260506 20:31:42.403 | INFO | Downloaded 34GiB to GPU in 1.91845s (17.7227GiB/s) | 20260506 20:31:47.543 | INFO | Analyzing chunk frames = [4244992,4245504) | 20260506 20:31:47.543 | INFO | MemInfo: { | 20260506 20:31:47.543 | INFO | "gpu" : " 17 GB", | 20260506 20:31:47.543 | INFO | "huge" : "", | 20260506 20:31:47.543 | INFO | "huge_shared" : " 27 GB", | 20260506 20:31:47.543 | INFO | "malloc" : " 14 GB", | 20260506 20:31:47.543 | INFO | "pinned" : " 29 GB" | 20260506 20:31:47.543 | INFO | } | 20260506 20:31:47.543 | INFO | Perf report for the last 5.1405 seconds: | 20260506 20:31:47.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:31:47.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:47.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:47.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:47.543 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:31:47.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:47.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:47.543 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 20:31:47.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:47.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:47.544 | INFO | Uploaded 36GiB to GPU in 1.94012s (18.5555GiB/s) | 20260506 20:31:47.544 | INFO | Downloaded 34GiB to GPU in 1.87064s (18.1756GiB/s) | 20260506 20:31:52.680 | INFO | Analyzing chunk frames = [4245504,4246016) | 20260506 20:31:52.680 | INFO | MemInfo: { | 20260506 20:31:52.680 | INFO | "gpu" : " 17 GB", | 20260506 20:31:52.680 | INFO | "huge" : "", | 20260506 20:31:52.680 | INFO | "huge_shared" : " 27 GB", | 20260506 20:31:52.680 | INFO | "malloc" : " 14 GB", | 20260506 20:31:52.680 | INFO | "pinned" : " 29 GB" | 20260506 20:31:52.680 | INFO | } | 20260506 20:31:52.680 | INFO | Perf report for the last 5.13664 seconds: | 20260506 20:31:52.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:31:52.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:52.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:52.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:52.680 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 20:31:52.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:52.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:52.680 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:31:52.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:52.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:52.680 | INFO | Uploaded 36GiB to GPU in 2.00783s (17.9298GiB/s) | 20260506 20:31:52.680 | INFO | Downloaded 34GiB to GPU in 1.91543s (17.7505GiB/s) | 20260506 20:31:57.820 | INFO | Analyzing chunk frames = [4246016,4246528) | 20260506 20:31:57.820 | INFO | MemInfo: { | 20260506 20:31:57.820 | INFO | "gpu" : " 17 GB", | 20260506 20:31:57.820 | INFO | "huge" : "", | 20260506 20:31:57.820 | INFO | "huge_shared" : " 27 GB", | 20260506 20:31:57.820 | INFO | "malloc" : " 14 GB", | 20260506 20:31:57.820 | INFO | "pinned" : " 29 GB" | 20260506 20:31:57.820 | INFO | } | 20260506 20:31:57.820 | INFO | Perf report for the last 5.13987 seconds: | 20260506 20:31:57.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:31:57.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:57.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:57.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:57.820 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:31:57.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:57.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:57.820 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 20:31:57.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:57.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:31:57.820 | INFO | Uploaded 36GiB to GPU in 2.02363s (17.7898GiB/s) | 20260506 20:31:57.820 | INFO | Downloaded 34GiB to GPU in 1.94303s (17.4984GiB/s) | 20260506 20:32:02.956 | INFO | Analyzing chunk frames = [4246528,4247040) | 20260506 20:32:02.956 | INFO | MemInfo: { | 20260506 20:32:02.956 | INFO | "gpu" : " 17 GB", | 20260506 20:32:02.956 | INFO | "huge" : "", | 20260506 20:32:02.956 | INFO | "huge_shared" : " 27 GB", | 20260506 20:32:02.956 | INFO | "malloc" : " 14 GB", | 20260506 20:32:02.956 | INFO | "pinned" : " 29 GB" | 20260506 20:32:02.956 | INFO | } | 20260506 20:32:02.956 | INFO | Perf report for the last 5.13612 seconds: | 20260506 20:32:02.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:32:02.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:02.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:02.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:02.956 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 20:32:02.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:02.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:02.956 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:32:02.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:02.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:02.956 | INFO | Uploaded 36GiB to GPU in 1.98627s (18.1245GiB/s) | 20260506 20:32:02.956 | INFO | Downloaded 34GiB to GPU in 2.00406s (16.9656GiB/s) | 20260506 20:32:08.098 | INFO | Analyzing chunk frames = [4247040,4247552) | 20260506 20:32:08.098 | INFO | MemInfo: { | 20260506 20:32:08.098 | INFO | "gpu" : " 17 GB", | 20260506 20:32:08.098 | INFO | "huge" : "", | 20260506 20:32:08.098 | INFO | "huge_shared" : " 27 GB", | 20260506 20:32:08.098 | INFO | "malloc" : " 14 GB", | 20260506 20:32:08.098 | INFO | "pinned" : " 29 GB" | 20260506 20:32:08.098 | INFO | } | 20260506 20:32:08.098 | INFO | Perf report for the last 5.14223 seconds: | 20260506 20:32:08.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:32:08.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:08.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:08.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:08.098 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:32:08.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:08.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:08.098 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:32:08.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:08.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:08.099 | INFO | Uploaded 36GiB to GPU in 2.00406s (17.9635GiB/s) | 20260506 20:32:08.099 | INFO | Downloaded 34GiB to GPU in 1.96994s (17.2594GiB/s) | 20260506 20:32:08.196 | INFO | Starting a round of Baz IO | 20260506 20:32:11.327 | INFO | Populating RTMetrics report | 20260506 20:32:11.327 | INFO | RTMetrics report populated | 20260506 20:32:11.327 | INFO | Opening tmp RTMetrics json file | 20260506 20:32:11.334 | INFO | Serializing RTMetrics to json | 20260506 20:32:11.364 | INFO | Serialization done, swapping tmp file now... | 20260506 20:32:11.369 | INFO | RTMetrics done | 20260506 20:32:11.471 | INFO | Finished a round of Baz IO: Wrote 1.975 GiB in 3.274 seconds | 20260506 20:32:11.471 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.22% / 0.00% / 65.78% | 20260506 20:32:11.471 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.736833% / 3.97364e-06% / 5.16574e-05% | 20260506 20:32:11.471 | INFO | Baz IO overall: | 20260506 20:32:11.471 | INFO | BytesWritten : 2242246326069 | 20260506 20:32:11.471 | INFO | NumEvents : 1083234604119 | 20260506 20:32:13.233 | INFO | Analyzing chunk frames = [4247552,4248064) | 20260506 20:32:13.233 | INFO | MemInfo: { | 20260506 20:32:13.233 | INFO | "gpu" : " 17 GB", | 20260506 20:32:13.233 | INFO | "huge" : "", | 20260506 20:32:13.233 | INFO | "huge_shared" : " 27 GB", | 20260506 20:32:13.233 | INFO | "malloc" : " 14 GB", | 20260506 20:32:13.233 | INFO | "pinned" : " 29 GB" | 20260506 20:32:13.233 | INFO | } | 20260506 20:32:13.233 | INFO | Perf report for the last 5.13526 seconds: | 20260506 20:32:13.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:32:13.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:13.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:13.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:13.233 | INFO | ANALYSIS : 62.3%, 2.9, 95.2ms, 0 | 20260506 20:32:13.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:13.233 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 20:32:13.233 | INFO | PRE_HQ : 59.8%, 1.8, 57.0ms, 0 | 20260506 20:32:13.233 | INFO | BAZWRITER : 63.4%, 4.7, 152.0ms, 0 | 20260506 20:32:13.234 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 20:32:13.234 | INFO | Uploaded 36GiB to GPU in 2.05771s (17.4952GiB/s) | 20260506 20:32:13.234 | INFO | Downloaded 38GiB to GPU in 2.19476s (17.314GiB/s) | 20260506 20:32:18.375 | INFO | Analyzing chunk frames = [4248064,4248576) | 20260506 20:32:18.375 | INFO | MemInfo: { | 20260506 20:32:18.375 | INFO | "gpu" : " 17 GB", | 20260506 20:32:18.375 | INFO | "huge" : "", | 20260506 20:32:18.375 | INFO | "huge_shared" : " 27 GB", | 20260506 20:32:18.375 | INFO | "malloc" : " 14 GB", | 20260506 20:32:18.375 | INFO | "pinned" : " 29 GB" | 20260506 20:32:18.375 | INFO | } | 20260506 20:32:18.375 | INFO | Perf report for the last 5.14126 seconds: | 20260506 20:32:18.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:32:18.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:18.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:18.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:18.375 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:32:18.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:18.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:18.375 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:32:18.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:18.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:18.375 | INFO | Uploaded 36GiB to GPU in 2.01983s (17.8233GiB/s) | 20260506 20:32:18.375 | INFO | Downloaded 34GiB to GPU in 1.92914s (17.6244GiB/s) | 20260506 20:32:23.521 | INFO | Analyzing chunk frames = [4248576,4249088) | 20260506 20:32:23.521 | INFO | MemInfo: { | 20260506 20:32:23.521 | INFO | "gpu" : " 17 GB", | 20260506 20:32:23.521 | INFO | "huge" : "", | 20260506 20:32:23.521 | INFO | "huge_shared" : " 27 GB", | 20260506 20:32:23.521 | INFO | "malloc" : " 14 GB", | 20260506 20:32:23.521 | INFO | "pinned" : " 29 GB" | 20260506 20:32:23.521 | INFO | } | 20260506 20:32:23.521 | INFO | Perf report for the last 5.14665 seconds: | 20260506 20:32:23.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:32:23.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:23.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:23.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:23.521 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 20:32:23.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:23.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:23.521 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:32:23.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:23.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:23.522 | INFO | Uploaded 36GiB to GPU in 1.98764s (18.112GiB/s) | 20260506 20:32:23.522 | INFO | Downloaded 34GiB to GPU in 1.91926s (17.7151GiB/s) | 20260506 20:32:28.658 | INFO | Analyzing chunk frames = [4249088,4249600) | 20260506 20:32:28.658 | INFO | MemInfo: { | 20260506 20:32:28.658 | INFO | "gpu" : " 17 GB", | 20260506 20:32:28.658 | INFO | "huge" : "", | 20260506 20:32:28.658 | INFO | "huge_shared" : " 27 GB", | 20260506 20:32:28.658 | INFO | "malloc" : " 14 GB", | 20260506 20:32:28.658 | INFO | "pinned" : " 29 GB" | 20260506 20:32:28.658 | INFO | } | 20260506 20:32:28.658 | INFO | Perf report for the last 5.13707 seconds: | 20260506 20:32:28.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:32:28.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:28.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:28.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:28.658 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 20:32:28.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:28.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:28.658 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:32:28.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:28.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:28.659 | INFO | Uploaded 36GiB to GPU in 1.94433s (18.5154GiB/s) | 20260506 20:32:28.659 | INFO | Downloaded 34GiB to GPU in 1.86962s (18.1855GiB/s) | 20260506 20:32:33.796 | INFO | Analyzing chunk frames = [4249600,4250112) | 20260506 20:32:33.796 | INFO | MemInfo: { | 20260506 20:32:33.796 | INFO | "gpu" : " 17 GB", | 20260506 20:32:33.796 | INFO | "huge" : "", | 20260506 20:32:33.796 | INFO | "huge_shared" : " 27 GB", | 20260506 20:32:33.796 | INFO | "malloc" : " 14 GB", | 20260506 20:32:33.796 | INFO | "pinned" : " 29 GB" | 20260506 20:32:33.796 | INFO | } | 20260506 20:32:33.796 | INFO | Perf report for the last 5.13741 seconds: | 20260506 20:32:33.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:32:33.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:33.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:33.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:33.796 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 20:32:33.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:33.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:33.796 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:32:33.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:33.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:33.796 | INFO | Uploaded 36GiB to GPU in 2.01133s (17.8986GiB/s) | 20260506 20:32:33.796 | INFO | Downloaded 34GiB to GPU in 1.91526s (17.7521GiB/s) | 20260506 20:32:38.932 | INFO | Analyzing chunk frames = [4250112,4250624) | 20260506 20:32:38.932 | INFO | MemInfo: { | 20260506 20:32:38.932 | INFO | "gpu" : " 17 GB", | 20260506 20:32:38.932 | INFO | "huge" : "", | 20260506 20:32:38.932 | INFO | "huge_shared" : " 27 GB", | 20260506 20:32:38.932 | INFO | "malloc" : " 14 GB", | 20260506 20:32:38.932 | INFO | "pinned" : " 29 GB" | 20260506 20:32:38.932 | INFO | } | 20260506 20:32:38.932 | INFO | Perf report for the last 5.13653 seconds: | 20260506 20:32:38.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:32:38.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:38.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:38.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:38.933 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:32:38.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:38.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:38.933 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 20:32:38.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:38.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:38.933 | INFO | Uploaded 36GiB to GPU in 2.01115s (17.9002GiB/s) | 20260506 20:32:38.933 | INFO | Downloaded 34GiB to GPU in 1.94463s (17.4841GiB/s) | 20260506 20:32:44.070 | INFO | Analyzing chunk frames = [4250624,4251136) | 20260506 20:32:44.070 | INFO | MemInfo: { | 20260506 20:32:44.070 | INFO | "gpu" : " 17 GB", | 20260506 20:32:44.070 | INFO | "huge" : "", | 20260506 20:32:44.070 | INFO | "huge_shared" : " 27 GB", | 20260506 20:32:44.070 | INFO | "malloc" : " 14 GB", | 20260506 20:32:44.070 | INFO | "pinned" : " 29 GB" | 20260506 20:32:44.070 | INFO | } | 20260506 20:32:44.070 | INFO | Perf report for the last 5.13813 seconds: | 20260506 20:32:44.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:32:44.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:44.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:44.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:44.071 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 20:32:44.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:44.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:44.071 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:32:44.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:44.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:44.071 | INFO | Uploaded 36GiB to GPU in 1.98607s (18.1263GiB/s) | 20260506 20:32:44.071 | INFO | Downloaded 34GiB to GPU in 2.00511s (16.9566GiB/s) | 20260506 20:32:49.212 | INFO | Analyzing chunk frames = [4251136,4251648) | 20260506 20:32:49.212 | INFO | MemInfo: { | 20260506 20:32:49.212 | INFO | "gpu" : " 17 GB", | 20260506 20:32:49.212 | INFO | "huge" : "", | 20260506 20:32:49.212 | INFO | "huge_shared" : " 27 GB", | 20260506 20:32:49.212 | INFO | "malloc" : " 14 GB", | 20260506 20:32:49.212 | INFO | "pinned" : " 29 GB" | 20260506 20:32:49.212 | INFO | } | 20260506 20:32:49.212 | INFO | Perf report for the last 5.14194 seconds: | 20260506 20:32:49.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:32:49.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:49.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:49.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:49.212 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 20:32:49.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:49.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:49.213 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 20:32:49.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:49.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:49.213 | INFO | Uploaded 36GiB to GPU in 1.99449s (18.0497GiB/s) | 20260506 20:32:49.213 | INFO | Downloaded 34GiB to GPU in 1.98216s (17.153GiB/s) | 20260506 20:32:49.213 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:32:49.213 | INFO | Total mem usage: 18 GiB | 20260506 20:32:49.213 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:32:49.213 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -11%) | 20260506 20:32:49.213 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:32:49.213 | INFO | High Water Breakdown: | 20260506 20:32:49.213 | INFO | 62% : 10.1 GiB ( 88.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:32:49.213 | INFO | 37.8% : 6.14 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:32:49.213 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:32:49.213 | INFO | ------------------------------------------ | 20260506 20:32:49.213 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:32:49.213 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:32:49.213 | INFO | Total mem usage: 26.5 GiB | 20260506 20:32:49.213 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:32:49.213 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:32:49.213 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:32:49.213 | INFO | High Water Breakdown: | 20260506 20:32:49.213 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:32:49.213 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:32:49.213 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:32:49.213 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:32:49.213 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:32:49.213 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:32:49.213 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:32:49.213 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:32:49.213 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:32:49.213 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:32:49.213 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:32:49.213 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:32:49.213 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:32:49.213 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:32:49.213 | INFO | ------------------------------------------ | 20260506 20:32:49.213 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:32:49.213 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:32:49.213 | INFO | Total mem usage: 25.6 GiB | 20260506 20:32:49.214 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:32:49.214 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 20:32:49.214 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 20:32:49.214 | INFO | High Water Breakdown: | 20260506 20:32:49.214 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 20:32:49.214 | INFO | ------------------------------------------ | 20260506 20:32:49.214 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:32:49.214 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:32:49.214 | INFO | Total mem usage: 15.7 GiB | 20260506 20:32:49.214 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:32:49.214 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:32:49.214 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 20:32:49.214 | INFO | High Water Breakdown: | 20260506 20:32:49.214 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:32:49.214 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:32:49.214 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:32:49.214 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:32:49.214 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:32:49.214 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:32:49.214 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:32:49.214 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:32:49.214 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:32:49.214 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:32:49.214 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:32:49.214 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:32:49.214 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:32:49.214 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:32:49.214 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:32:49.214 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:32:49.214 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:32:49.214 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:32:49.214 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:32:49.214 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:32:49.214 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:32:49.214 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:32:49.214 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:32:49.214 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:32:49.214 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:32:49.214 | INFO | ------------------------------------------ | 20260506 20:32:49.214 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:32:49.214 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:32:49.214 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:32:49.214 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:32:49.214 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.33e+01 / 2.65e+00 | 20260506 20:32:49.214 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 1.03e-02 / 5.70e-03 | 20260506 20:32:49.214 | INFO | PerfReport: 11.63%, Baseline -- 3.62e+00 / 4.45e+00, 5.51e-01 / 3.28e-01 | 20260506 20:32:49.214 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.81e-01 / 1.48e-01 | 20260506 20:32:49.214 | INFO | PerfReport: 16.05%, DME -- 4.00e+01 / 4.48e+01, 4.26e+00 / 2.69e+00 | 20260506 20:32:49.214 | INFO | PerfReport: 57.14%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.35e+00 / 1.69e+00 | 20260506 20:32:49.214 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.23e+00 / 2.78e+00, 3.91e-01 / 2.29e-01 | 20260506 20:32:49.214 | INFO | PerfReport: 3.32%, Metrics -- 1.03e+00 / 1.36e+00, 2.73e-01 / 7.98e-02 | 20260506 20:32:49.214 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.54e-01 / 4.03e-01, 5.57e-02 / 1.93e-02 | 20260506 20:32:49.214 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:32:49.214 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:32:49.214 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:32:49.214 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 20:32:49.214 | INFO | PerfReport: 21.54%, Upload -- 2.08e+01 / 0.00e+00, 6.94e+00 / 0.00e+00 | 20260506 20:32:49.214 | INFO | PerfReport: 21.19%, Download -- 2.05e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260506 20:32:49.214 | INFO | PerfReport: 32.19%, Compute -- 3.12e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 20:32:49.214 | INFO | PerfReport: 1.36%, LockUpload -- 1.32e+00 / 0.00e+00, 5.00e+00 / 0.00e+00 | 20260506 20:32:49.214 | INFO | PerfReport: 1.14%, LockDownload -- 1.10e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 20:32:49.214 | INFO | PerfReport: 22.57%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 20:32:49.214 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260506 20:32:49.313 | INFO | Starting a round of Baz IO | 20260506 20:32:52.454 | INFO | Populating RTMetrics report | 20260506 20:32:52.454 | INFO | RTMetrics report populated | 20260506 20:32:52.454 | INFO | Opening tmp RTMetrics json file | 20260506 20:32:52.611 | INFO | Serializing RTMetrics to json | 20260506 20:32:52.617 | INFO | Serialization done, swapping tmp file now... | 20260506 20:32:52.623 | INFO | RTMetrics done | 20260506 20:32:52.756 | INFO | Finished a round of Baz IO: Wrote 1.976 GiB in 3.442 seconds | 20260506 20:32:52.756 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.21% / 0.00% / 65.78% | 20260506 20:32:52.756 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.736447% / 1.19209e-05% / 6.75519e-05% | 20260506 20:32:52.756 | INFO | Baz IO overall: | 20260506 20:32:52.756 | INFO | BytesWritten : 2244367841535 | 20260506 20:32:52.756 | INFO | NumEvents : 1084252323500 | 20260506 20:32:54.349 | INFO | Analyzing chunk frames = [4251648,4252160) | 20260506 20:32:54.349 | INFO | MemInfo: { | 20260506 20:32:54.349 | INFO | "gpu" : " 17 GB", | 20260506 20:32:54.349 | INFO | "huge" : "", | 20260506 20:32:54.349 | INFO | "huge_shared" : " 27 GB", | 20260506 20:32:54.349 | INFO | "malloc" : " 14 GB", | 20260506 20:32:54.349 | INFO | "pinned" : " 29 GB" | 20260506 20:32:54.349 | INFO | } | 20260506 20:32:54.349 | INFO | Perf report for the last 5.13682 seconds: | 20260506 20:32:54.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:32:54.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:54.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:54.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:54.350 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 20:32:54.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:54.350 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260506 20:32:54.350 | INFO | PRE_HQ : 60.4%, 1.8, 57.5ms, 0 | 20260506 20:32:54.350 | INFO | BAZWRITER : 66.6%, 4.7, 162.5ms, 0 | 20260506 20:32:54.350 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 20:32:54.350 | INFO | Uploaded 36GiB to GPU in 2.04998s (17.5611GiB/s) | 20260506 20:32:54.350 | INFO | Downloaded 38GiB to GPU in 2.19234s (17.3331GiB/s) | 20260506 20:32:59.488 | INFO | Analyzing chunk frames = [4252160,4252672) | 20260506 20:32:59.488 | INFO | MemInfo: { | 20260506 20:32:59.488 | INFO | "gpu" : " 17 GB", | 20260506 20:32:59.488 | INFO | "huge" : "", | 20260506 20:32:59.488 | INFO | "huge_shared" : " 27 GB", | 20260506 20:32:59.488 | INFO | "malloc" : " 14 GB", | 20260506 20:32:59.488 | INFO | "pinned" : " 29 GB" | 20260506 20:32:59.488 | INFO | } | 20260506 20:32:59.488 | INFO | Perf report for the last 5.13876 seconds: | 20260506 20:32:59.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:32:59.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:59.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:59.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:59.488 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:32:59.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:59.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:59.488 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:32:59.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:59.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:32:59.488 | INFO | Uploaded 36GiB to GPU in 2.01058s (17.9053GiB/s) | 20260506 20:32:59.488 | INFO | Downloaded 34GiB to GPU in 1.92488s (17.6635GiB/s) | 20260506 20:33:04.627 | INFO | Analyzing chunk frames = [4252672,4253184) | 20260506 20:33:04.627 | INFO | MemInfo: { | 20260506 20:33:04.627 | INFO | "gpu" : " 17 GB", | 20260506 20:33:04.628 | INFO | "huge" : "", | 20260506 20:33:04.628 | INFO | "huge_shared" : " 27 GB", | 20260506 20:33:04.628 | INFO | "malloc" : " 14 GB", | 20260506 20:33:04.628 | INFO | "pinned" : " 29 GB" | 20260506 20:33:04.628 | INFO | } | 20260506 20:33:04.628 | INFO | Perf report for the last 5.13967 seconds: | 20260506 20:33:04.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:33:04.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:04.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:04.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:04.628 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 20:33:04.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:04.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:04.628 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:33:04.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:04.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:04.628 | INFO | Uploaded 36GiB to GPU in 1.99955s (18.004GiB/s) | 20260506 20:33:04.628 | INFO | Downloaded 34GiB to GPU in 1.91854s (17.7218GiB/s) | 20260506 20:33:09.770 | INFO | Analyzing chunk frames = [4253184,4253696) | 20260506 20:33:09.770 | INFO | MemInfo: { | 20260506 20:33:09.770 | INFO | "gpu" : " 17 GB", | 20260506 20:33:09.770 | INFO | "huge" : "", | 20260506 20:33:09.770 | INFO | "huge_shared" : " 27 GB", | 20260506 20:33:09.770 | INFO | "malloc" : " 14 GB", | 20260506 20:33:09.770 | INFO | "pinned" : " 29 GB" | 20260506 20:33:09.770 | INFO | } | 20260506 20:33:09.770 | INFO | Perf report for the last 5.14273 seconds: | 20260506 20:33:09.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:33:09.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:09.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:09.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:09.771 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 20:33:09.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:09.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:09.771 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 20:33:09.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:09.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:09.771 | INFO | Uploaded 36GiB to GPU in 1.94268s (18.5311GiB/s) | 20260506 20:33:09.771 | INFO | Downloaded 34GiB to GPU in 1.87095s (18.1726GiB/s) | 20260506 20:33:14.910 | INFO | Analyzing chunk frames = [4253696,4254208) | 20260506 20:33:14.910 | INFO | MemInfo: { | 20260506 20:33:14.910 | INFO | "gpu" : " 17 GB", | 20260506 20:33:14.910 | INFO | "huge" : "", | 20260506 20:33:14.910 | INFO | "huge_shared" : " 27 GB", | 20260506 20:33:14.910 | INFO | "malloc" : " 14 GB", | 20260506 20:33:14.910 | INFO | "pinned" : " 29 GB" | 20260506 20:33:14.910 | INFO | } | 20260506 20:33:14.910 | INFO | Perf report for the last 5.13934 seconds: | 20260506 20:33:14.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:33:14.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:14.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:14.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:14.910 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 20:33:14.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:14.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:14.910 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:33:14.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:14.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:14.910 | INFO | Uploaded 36GiB to GPU in 1.9983s (18.0153GiB/s) | 20260506 20:33:14.910 | INFO | Downloaded 34GiB to GPU in 1.9143s (17.7611GiB/s) | 20260506 20:33:20.050 | INFO | Analyzing chunk frames = [4254208,4254720) | 20260506 20:33:20.050 | INFO | MemInfo: { | 20260506 20:33:20.050 | INFO | "gpu" : " 17 GB", | 20260506 20:33:20.050 | INFO | "huge" : "", | 20260506 20:33:20.050 | INFO | "huge_shared" : " 27 GB", | 20260506 20:33:20.050 | INFO | "malloc" : " 14 GB", | 20260506 20:33:20.050 | INFO | "pinned" : " 29 GB" | 20260506 20:33:20.050 | INFO | } | 20260506 20:33:20.050 | INFO | Perf report for the last 5.14011 seconds: | 20260506 20:33:20.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:33:20.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:20.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:20.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:20.050 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 20:33:20.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:20.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:20.050 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260506 20:33:20.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:20.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:20.050 | INFO | Uploaded 36GiB to GPU in 2.00165s (17.9852GiB/s) | 20260506 20:33:20.050 | INFO | Downloaded 34GiB to GPU in 1.93784s (17.5453GiB/s) | 20260506 20:33:25.188 | INFO | Analyzing chunk frames = [4254720,4255232) | 20260506 20:33:25.188 | INFO | MemInfo: { | 20260506 20:33:25.188 | INFO | "gpu" : " 17 GB", | 20260506 20:33:25.188 | INFO | "huge" : "", | 20260506 20:33:25.188 | INFO | "huge_shared" : " 27 GB", | 20260506 20:33:25.188 | INFO | "malloc" : " 14 GB", | 20260506 20:33:25.188 | INFO | "pinned" : " 29 GB" | 20260506 20:33:25.188 | INFO | } | 20260506 20:33:25.188 | INFO | Perf report for the last 5.1383 seconds: | 20260506 20:33:25.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:33:25.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:25.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:25.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:25.189 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:33:25.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:25.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:25.189 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:33:25.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:25.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:25.189 | INFO | Uploaded 36GiB to GPU in 1.96429s (18.3273GiB/s) | 20260506 20:33:25.190 | INFO | Downloaded 34GiB to GPU in 1.99242s (17.0647GiB/s) | 20260506 20:33:30.327 | INFO | Analyzing chunk frames = [4255232,4255744) | 20260506 20:33:30.327 | INFO | MemInfo: { | 20260506 20:33:30.327 | INFO | "gpu" : " 17 GB", | 20260506 20:33:30.327 | INFO | "huge" : "", | 20260506 20:33:30.327 | INFO | "huge_shared" : " 27 GB", | 20260506 20:33:30.327 | INFO | "malloc" : " 14 GB", | 20260506 20:33:30.327 | INFO | "pinned" : " 29 GB" | 20260506 20:33:30.327 | INFO | } | 20260506 20:33:30.327 | INFO | Perf report for the last 5.13892 seconds: | 20260506 20:33:30.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:33:30.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:30.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:30.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:30.327 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 20:33:30.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:30.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:30.327 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 20:33:30.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:30.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:30.327 | INFO | Uploaded 36GiB to GPU in 2.07225s (17.3724GiB/s) | 20260506 20:33:30.328 | INFO | Downloaded 34GiB to GPU in 2.00915s (16.9225GiB/s) | 20260506 20:33:30.447 | INFO | Starting a round of Baz IO | 20260506 20:33:33.574 | INFO | Populating RTMetrics report | 20260506 20:33:33.574 | INFO | RTMetrics report populated | 20260506 20:33:33.574 | INFO | Opening tmp RTMetrics json file | 20260506 20:33:33.669 | INFO | Serializing RTMetrics to json | 20260506 20:33:33.705 | INFO | Serialization done, swapping tmp file now... | 20260506 20:33:33.708 | INFO | RTMetrics done | 20260506 20:33:34.012 | INFO | Finished a round of Baz IO: Wrote 1.974 GiB in 3.564 seconds | 20260506 20:33:34.012 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.21% / 0.00% / 65.79% | 20260506 20:33:34.012 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.732974% / 0% / 6.35783e-05% | 20260506 20:33:34.012 | INFO | Baz IO overall: | 20260506 20:33:34.012 | INFO | BytesWritten : 2246486967020 | 20260506 20:33:34.012 | INFO | NumEvents : 1085268461863 | 20260506 20:33:35.468 | INFO | Analyzing chunk frames = [4255744,4256256) | 20260506 20:33:35.468 | INFO | MemInfo: { | 20260506 20:33:35.468 | INFO | "gpu" : " 17 GB", | 20260506 20:33:35.468 | INFO | "huge" : "", | 20260506 20:33:35.468 | INFO | "huge_shared" : " 27 GB", | 20260506 20:33:35.468 | INFO | "malloc" : " 14 GB", | 20260506 20:33:35.468 | INFO | "pinned" : " 29 GB" | 20260506 20:33:35.468 | INFO | } | 20260506 20:33:35.468 | INFO | Perf report for the last 5.14091 seconds: | 20260506 20:33:35.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:33:35.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:35.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:35.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:35.468 | INFO | ANALYSIS : 62.6%, 2.9, 95.9ms, 0 | 20260506 20:33:35.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:35.468 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260506 20:33:35.468 | INFO | PRE_HQ : 59.8%, 1.8, 57.1ms, 0 | 20260506 20:33:35.468 | INFO | BAZWRITER : 68.9%, 4.8, 168.5ms, 0 | 20260506 20:33:35.468 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 20:33:35.469 | INFO | Uploaded 36GiB to GPU in 2.05533s (17.5154GiB/s) | 20260506 20:33:35.469 | INFO | Downloaded 38GiB to GPU in 2.21681s (17.1418GiB/s) | 20260506 20:33:40.602 | INFO | Analyzing chunk frames = [4256256,4256768) | 20260506 20:33:40.602 | INFO | MemInfo: { | 20260506 20:33:40.602 | INFO | "gpu" : " 17 GB", | 20260506 20:33:40.602 | INFO | "huge" : "", | 20260506 20:33:40.602 | INFO | "huge_shared" : " 27 GB", | 20260506 20:33:40.602 | INFO | "malloc" : " 14 GB", | 20260506 20:33:40.602 | INFO | "pinned" : " 29 GB" | 20260506 20:33:40.602 | INFO | } | 20260506 20:33:40.602 | INFO | Perf report for the last 5.13398 seconds: | 20260506 20:33:40.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:33:40.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:40.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:40.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:40.602 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:33:40.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:40.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:40.602 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:33:40.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:40.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:40.603 | INFO | Uploaded 36GiB to GPU in 2.01466s (17.869GiB/s) | 20260506 20:33:40.603 | INFO | Downloaded 34GiB to GPU in 1.92774s (17.6373GiB/s) | 20260506 20:33:45.746 | INFO | Analyzing chunk frames = [4256768,4257280) | 20260506 20:33:45.746 | INFO | MemInfo: { | 20260506 20:33:45.746 | INFO | "gpu" : " 17 GB", | 20260506 20:33:45.746 | INFO | "huge" : "", | 20260506 20:33:45.746 | INFO | "huge_shared" : " 27 GB", | 20260506 20:33:45.746 | INFO | "malloc" : " 14 GB", | 20260506 20:33:45.746 | INFO | "pinned" : " 29 GB" | 20260506 20:33:45.746 | INFO | } | 20260506 20:33:45.746 | INFO | Perf report for the last 5.14401 seconds: | 20260506 20:33:45.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:33:45.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:45.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:45.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:45.746 | INFO | ANALYSIS : 60.8%, 3.0, 94.2ms, 0 | 20260506 20:33:45.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:45.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:45.746 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:33:45.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:45.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:45.746 | INFO | Uploaded 36GiB to GPU in 1.98401s (18.145GiB/s) | 20260506 20:33:45.746 | INFO | Downloaded 34GiB to GPU in 1.91947s (17.7132GiB/s) | 20260506 20:33:50.890 | INFO | Analyzing chunk frames = [4257280,4257792) | 20260506 20:33:50.890 | INFO | MemInfo: { | 20260506 20:33:50.890 | INFO | "gpu" : " 17 GB", | 20260506 20:33:50.890 | INFO | "huge" : "", | 20260506 20:33:50.890 | INFO | "huge_shared" : " 27 GB", | 20260506 20:33:50.890 | INFO | "malloc" : " 14 GB", | 20260506 20:33:50.890 | INFO | "pinned" : " 29 GB" | 20260506 20:33:50.890 | INFO | } | 20260506 20:33:50.890 | INFO | Perf report for the last 5.14412 seconds: | 20260506 20:33:50.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:33:50.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:50.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:50.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:50.890 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:33:50.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:50.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:50.891 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:33:50.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:50.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:50.891 | INFO | Uploaded 36GiB to GPU in 1.94868s (18.474GiB/s) | 20260506 20:33:50.891 | INFO | Downloaded 34GiB to GPU in 1.87225s (18.1599GiB/s) | 20260506 20:33:56.025 | INFO | Analyzing chunk frames = [4257792,4258304) | 20260506 20:33:56.025 | INFO | MemInfo: { | 20260506 20:33:56.025 | INFO | "gpu" : " 17 GB", | 20260506 20:33:56.025 | INFO | "huge" : "", | 20260506 20:33:56.025 | INFO | "huge_shared" : " 27 GB", | 20260506 20:33:56.025 | INFO | "malloc" : " 14 GB", | 20260506 20:33:56.025 | INFO | "pinned" : " 29 GB" | 20260506 20:33:56.025 | INFO | } | 20260506 20:33:56.025 | INFO | Perf report for the last 5.13501 seconds: | 20260506 20:33:56.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:33:56.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:56.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:56.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:56.025 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 20:33:56.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:56.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:56.026 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:33:56.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:56.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:33:56.026 | INFO | Uploaded 36GiB to GPU in 1.99493s (18.0457GiB/s) | 20260506 20:33:56.026 | INFO | Downloaded 34GiB to GPU in 1.92608s (17.6524GiB/s) | 20260506 20:34:01.164 | INFO | Analyzing chunk frames = [4258304,4258816) | 20260506 20:34:01.164 | INFO | MemInfo: { | 20260506 20:34:01.164 | INFO | "gpu" : " 17 GB", | 20260506 20:34:01.164 | INFO | "huge" : "", | 20260506 20:34:01.164 | INFO | "huge_shared" : " 27 GB", | 20260506 20:34:01.164 | INFO | "malloc" : " 14 GB", | 20260506 20:34:01.164 | INFO | "pinned" : " 29 GB" | 20260506 20:34:01.164 | INFO | } | 20260506 20:34:01.164 | INFO | Perf report for the last 5.13913 seconds: | 20260506 20:34:01.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:34:01.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:01.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:01.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:01.165 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 20:34:01.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:01.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:01.165 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:34:01.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:01.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:01.165 | INFO | Uploaded 36GiB to GPU in 2.02542s (17.7741GiB/s) | 20260506 20:34:01.165 | INFO | Downloaded 34GiB to GPU in 1.93847s (17.5396GiB/s) | 20260506 20:34:06.300 | INFO | Analyzing chunk frames = [4258816,4259328) | 20260506 20:34:06.300 | INFO | MemInfo: { | 20260506 20:34:06.300 | INFO | "gpu" : " 17 GB", | 20260506 20:34:06.300 | INFO | "huge" : "", | 20260506 20:34:06.300 | INFO | "huge_shared" : " 27 GB", | 20260506 20:34:06.300 | INFO | "malloc" : " 14 GB", | 20260506 20:34:06.300 | INFO | "pinned" : " 29 GB" | 20260506 20:34:06.300 | INFO | } | 20260506 20:34:06.300 | INFO | Perf report for the last 5.13582 seconds: | 20260506 20:34:06.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:34:06.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:06.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:06.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:06.300 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 20:34:06.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:06.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:06.300 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 20:34:06.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:06.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:06.301 | INFO | Uploaded 36GiB to GPU in 1.98394s (18.1457GiB/s) | 20260506 20:34:06.301 | INFO | Downloaded 34GiB to GPU in 2.0058s (16.9508GiB/s) | 20260506 20:34:11.435 | INFO | Analyzing chunk frames = [4259328,4259840) | 20260506 20:34:11.435 | INFO | MemInfo: { | 20260506 20:34:11.435 | INFO | "gpu" : " 17 GB", | 20260506 20:34:11.435 | INFO | "huge" : "", | 20260506 20:34:11.435 | INFO | "huge_shared" : " 27 GB", | 20260506 20:34:11.435 | INFO | "malloc" : " 14 GB", | 20260506 20:34:11.435 | INFO | "pinned" : " 29 GB" | 20260506 20:34:11.435 | INFO | } | 20260506 20:34:11.435 | INFO | Perf report for the last 5.13486 seconds: | 20260506 20:34:11.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:34:11.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:11.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:11.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:11.435 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 20:34:11.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:11.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:11.435 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 20:34:11.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:11.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:11.436 | INFO | Uploaded 36GiB to GPU in 1.98509s (18.1352GiB/s) | 20260506 20:34:11.436 | INFO | Downloaded 34GiB to GPU in 1.98985s (17.0867GiB/s) | 20260506 20:34:11.436 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:34:11.436 | INFO | Total mem usage: 18 GiB | 20260506 20:34:11.436 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:34:11.436 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.2%) | 20260506 20:34:11.436 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:34:11.436 | INFO | High Water Breakdown: | 20260506 20:34:11.436 | INFO | 61.8% : 9.83 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:34:11.436 | INFO | 38.1% : 6.06 GiB ( 90.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:34:11.436 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:34:11.436 | INFO | ------------------------------------------ | 20260506 20:34:11.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:34:11.436 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:34:11.436 | INFO | Total mem usage: 26.5 GiB | 20260506 20:34:11.436 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:34:11.436 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:34:11.436 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:34:11.436 | INFO | High Water Breakdown: | 20260506 20:34:11.436 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:34:11.436 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:34:11.436 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:34:11.436 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:34:11.436 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:34:11.436 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:34:11.436 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:34:11.436 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:34:11.436 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:34:11.436 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:34:11.436 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:34:11.436 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:34:11.436 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:34:11.436 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:34:11.436 | INFO | ------------------------------------------ | 20260506 20:34:11.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:34:11.436 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:34:11.436 | INFO | Total mem usage: 25.6 GiB | 20260506 20:34:11.436 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:34:11.436 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:34:11.436 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:34:11.436 | INFO | High Water Breakdown: | 20260506 20:34:11.436 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:34:11.436 | INFO | ------------------------------------------ | 20260506 20:34:11.436 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:34:11.436 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:34:11.436 | INFO | Total mem usage: 15.7 GiB | 20260506 20:34:11.436 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:34:11.436 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:34:11.436 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 20:34:11.436 | INFO | High Water Breakdown: | 20260506 20:34:11.436 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:34:11.436 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:34:11.436 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:34:11.436 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:34:11.436 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:34:11.436 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:34:11.436 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:34:11.436 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:34:11.436 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:34:11.436 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:34:11.436 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:34:11.437 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:34:11.437 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:34:11.437 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:34:11.437 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:34:11.437 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:34:11.437 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:34:11.437 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:34:11.437 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:34:11.437 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:34:11.437 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:34:11.437 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:34:11.437 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:34:11.437 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:34:11.437 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:34:11.437 | INFO | ------------------------------------------ | 20260506 20:34:11.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:34:11.437 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:34:11.437 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:34:11.437 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:34:11.437 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.34e+01 / 2.31e+00 | 20260506 20:34:11.437 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 9.95e-03 / 5.15e-03 | 20260506 20:34:11.437 | INFO | PerfReport: 11.63%, Baseline -- 3.63e+00 / 4.43e+00, 5.51e-01 / 3.10e-01 | 20260506 20:34:11.437 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.48e+00, 1.83e-01 / 1.34e-01 | 20260506 20:34:11.437 | INFO | PerfReport: 16.03%, DME -- 4.00e+01 / 4.48e+01, 4.24e+00 / 2.28e+00 | 20260506 20:34:11.437 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.40e+00 / 1.96e+00 | 20260506 20:34:11.437 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.23e+00 / 2.78e+00, 3.80e-01 / 2.07e-01 | 20260506 20:34:11.437 | INFO | PerfReport: 3.31%, Metrics -- 1.03e+00 / 1.37e+00, 2.74e-01 / 1.72e-01 | 20260506 20:34:11.437 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.54e-01 / 4.05e-01, 5.56e-02 / 2.39e-02 | 20260506 20:34:11.437 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:34:11.437 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:34:11.437 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:34:11.437 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 20:34:11.437 | INFO | PerfReport: 21.53%, Upload -- 2.09e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260506 20:34:11.437 | INFO | PerfReport: 21.21%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 20:34:11.437 | INFO | PerfReport: 32.19%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 20:34:11.437 | INFO | PerfReport: 1.40%, LockUpload -- 1.36e+00 / 0.00e+00, 5.14e+00 / 0.00e+00 | 20260506 20:34:11.437 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 20:34:11.437 | INFO | PerfReport: 22.56%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 20:34:11.437 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260506 20:34:11.546 | INFO | Starting a round of Baz IO | 20260506 20:34:14.670 | INFO | Populating RTMetrics report | 20260506 20:34:14.670 | INFO | RTMetrics report populated | 20260506 20:34:14.670 | INFO | Opening tmp RTMetrics json file | 20260506 20:34:14.672 | INFO | Serializing RTMetrics to json | 20260506 20:34:14.706 | INFO | Serialization done, swapping tmp file now... | 20260506 20:34:14.709 | INFO | RTMetrics done | 20260506 20:34:14.845 | INFO | Finished a round of Baz IO: Wrote 1.974 GiB in 3.299 seconds | 20260506 20:34:14.845 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.20% / 0.00% / 65.79% | 20260506 20:34:14.845 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.732021% / 0% / 7.94729e-05% | 20260506 20:34:14.845 | INFO | Baz IO overall: | 20260506 20:34:14.845 | INFO | BytesWritten : 2248606371280 | 20260506 20:34:14.845 | INFO | NumEvents : 1086284477252 | 20260506 20:34:16.577 | INFO | Analyzing chunk frames = [4259840,4260352) | 20260506 20:34:16.577 | INFO | MemInfo: { | 20260506 20:34:16.577 | INFO | "gpu" : " 17 GB", | 20260506 20:34:16.577 | INFO | "huge" : "", | 20260506 20:34:16.577 | INFO | "huge_shared" : " 27 GB", | 20260506 20:34:16.577 | INFO | "malloc" : " 14 GB", | 20260506 20:34:16.577 | INFO | "pinned" : " 29 GB" | 20260506 20:34:16.577 | INFO | } | 20260506 20:34:16.577 | INFO | Perf report for the last 5.14181 seconds: | 20260506 20:34:16.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:34:16.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:16.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:16.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:16.577 | INFO | ANALYSIS : 62.3%, 2.9, 94.9ms, 0 | 20260506 20:34:16.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:16.577 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 20:34:16.577 | INFO | PRE_HQ : 60.1%, 1.9, 57.7ms, 0 | 20260506 20:34:16.577 | INFO | BAZWRITER : 63.9%, 4.7, 155.0ms, 0 | 20260506 20:34:16.577 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 20:34:16.577 | INFO | Uploaded 36GiB to GPU in 2.04243s (17.6261GiB/s) | 20260506 20:34:16.577 | INFO | Downloaded 38GiB to GPU in 2.18969s (17.354GiB/s) | 20260506 20:34:21.719 | INFO | Analyzing chunk frames = [4260352,4260864) | 20260506 20:34:21.719 | INFO | MemInfo: { | 20260506 20:34:21.719 | INFO | "gpu" : " 17 GB", | 20260506 20:34:21.719 | INFO | "huge" : "", | 20260506 20:34:21.719 | INFO | "huge_shared" : " 27 GB", | 20260506 20:34:21.719 | INFO | "malloc" : " 14 GB", | 20260506 20:34:21.719 | INFO | "pinned" : " 29 GB" | 20260506 20:34:21.719 | INFO | } | 20260506 20:34:21.719 | INFO | Perf report for the last 5.14188 seconds: | 20260506 20:34:21.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:34:21.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:21.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:21.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:21.719 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:34:21.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:21.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:21.719 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:34:21.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:21.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:21.719 | INFO | Uploaded 36GiB to GPU in 2.01082s (17.9032GiB/s) | 20260506 20:34:21.719 | INFO | Downloaded 34GiB to GPU in 1.92402s (17.6713GiB/s) | 20260506 20:34:26.863 | INFO | Analyzing chunk frames = [4260864,4261376) | 20260506 20:34:26.863 | INFO | MemInfo: { | 20260506 20:34:26.863 | INFO | "gpu" : " 17 GB", | 20260506 20:34:26.863 | INFO | "huge" : "", | 20260506 20:34:26.863 | INFO | "huge_shared" : " 27 GB", | 20260506 20:34:26.863 | INFO | "malloc" : " 14 GB", | 20260506 20:34:26.863 | INFO | "pinned" : " 29 GB" | 20260506 20:34:26.863 | INFO | } | 20260506 20:34:26.863 | INFO | Perf report for the last 5.14392 seconds: | 20260506 20:34:26.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:34:26.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:26.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:26.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:26.863 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 20:34:26.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:26.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:26.863 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:34:26.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:26.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:26.863 | INFO | Uploaded 36GiB to GPU in 1.99319s (18.0615GiB/s) | 20260506 20:34:26.863 | INFO | Downloaded 34GiB to GPU in 1.91931s (17.7147GiB/s) | 20260506 20:34:32.003 | INFO | Analyzing chunk frames = [4261376,4261888) | 20260506 20:34:32.003 | INFO | MemInfo: { | 20260506 20:34:32.003 | INFO | "gpu" : " 17 GB", | 20260506 20:34:32.003 | INFO | "huge" : "", | 20260506 20:34:32.003 | INFO | "huge_shared" : " 27 GB", | 20260506 20:34:32.003 | INFO | "malloc" : " 14 GB", | 20260506 20:34:32.003 | INFO | "pinned" : " 29 GB" | 20260506 20:34:32.003 | INFO | } | 20260506 20:34:32.003 | INFO | Perf report for the last 5.14022 seconds: | 20260506 20:34:32.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:34:32.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:32.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:32.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:32.003 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 20:34:32.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:32.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:32.003 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:34:32.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:32.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:32.003 | INFO | Uploaded 36GiB to GPU in 2.02117s (17.8115GiB/s) | 20260506 20:34:32.003 | INFO | Downloaded 34GiB to GPU in 1.90645s (17.8342GiB/s) | 20260506 20:34:37.144 | INFO | Analyzing chunk frames = [4261888,4262400) | 20260506 20:34:37.144 | INFO | MemInfo: { | 20260506 20:34:37.144 | INFO | "gpu" : " 17 GB", | 20260506 20:34:37.144 | INFO | "huge" : "", | 20260506 20:34:37.144 | INFO | "huge_shared" : " 27 GB", | 20260506 20:34:37.144 | INFO | "malloc" : " 14 GB", | 20260506 20:34:37.144 | INFO | "pinned" : " 29 GB" | 20260506 20:34:37.144 | INFO | } | 20260506 20:34:37.144 | INFO | Perf report for the last 5.14123 seconds: | 20260506 20:34:37.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:34:37.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:37.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:37.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:37.145 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 20:34:37.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:37.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:37.145 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:34:37.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:37.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:37.145 | INFO | Uploaded 36GiB to GPU in 1.99851s (18.0134GiB/s) | 20260506 20:34:37.145 | INFO | Downloaded 34GiB to GPU in 1.9248s (17.6642GiB/s) | 20260506 20:34:42.280 | INFO | Analyzing chunk frames = [4262400,4262912) | 20260506 20:34:42.280 | INFO | MemInfo: { | 20260506 20:34:42.280 | INFO | "gpu" : " 17 GB", | 20260506 20:34:42.280 | INFO | "huge" : "", | 20260506 20:34:42.280 | INFO | "huge_shared" : " 27 GB", | 20260506 20:34:42.280 | INFO | "malloc" : " 14 GB", | 20260506 20:34:42.280 | INFO | "pinned" : " 29 GB" | 20260506 20:34:42.280 | INFO | } | 20260506 20:34:42.280 | INFO | Perf report for the last 5.13572 seconds: | 20260506 20:34:42.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:34:42.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:42.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:42.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:42.280 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 20:34:42.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:42.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:42.280 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 20:34:42.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:42.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:42.280 | INFO | Uploaded 36GiB to GPU in 2.0141s (17.874GiB/s) | 20260506 20:34:42.280 | INFO | Downloaded 34GiB to GPU in 1.94136s (17.5135GiB/s) | 20260506 20:34:47.415 | INFO | Analyzing chunk frames = [4262912,4263424) | 20260506 20:34:47.415 | INFO | MemInfo: { | 20260506 20:34:47.415 | INFO | "gpu" : " 17 GB", | 20260506 20:34:47.415 | INFO | "huge" : "", | 20260506 20:34:47.415 | INFO | "huge_shared" : " 27 GB", | 20260506 20:34:47.415 | INFO | "malloc" : " 14 GB", | 20260506 20:34:47.415 | INFO | "pinned" : " 29 GB" | 20260506 20:34:47.415 | INFO | } | 20260506 20:34:47.415 | INFO | Perf report for the last 5.13542 seconds: | 20260506 20:34:47.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:34:47.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:47.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:47.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:47.416 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260506 20:34:47.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:47.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:47.416 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:34:47.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:47.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:47.416 | INFO | Uploaded 36GiB to GPU in 1.99167s (18.0753GiB/s) | 20260506 20:34:47.416 | INFO | Downloaded 34GiB to GPU in 2.00484s (16.959GiB/s) | 20260506 20:34:52.554 | INFO | Analyzing chunk frames = [4263424,4263936) | 20260506 20:34:52.554 | INFO | MemInfo: { | 20260506 20:34:52.554 | INFO | "gpu" : " 17 GB", | 20260506 20:34:52.554 | INFO | "huge" : "", | 20260506 20:34:52.554 | INFO | "huge_shared" : " 27 GB", | 20260506 20:34:52.554 | INFO | "malloc" : " 14 GB", | 20260506 20:34:52.554 | INFO | "pinned" : " 29 GB" | 20260506 20:34:52.554 | INFO | } | 20260506 20:34:52.555 | INFO | Perf report for the last 5.13923 seconds: | 20260506 20:34:52.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:34:52.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:52.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:52.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:52.555 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 20:34:52.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:52.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:52.555 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:34:52.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:52.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:52.555 | INFO | Uploaded 36GiB to GPU in 1.99676s (18.0292GiB/s) | 20260506 20:34:52.555 | INFO | Downloaded 34GiB to GPU in 1.97205s (17.2409GiB/s) | 20260506 20:34:52.654 | INFO | Starting a round of Baz IO | 20260506 20:34:55.795 | INFO | Populating RTMetrics report | 20260506 20:34:55.795 | INFO | RTMetrics report populated | 20260506 20:34:55.795 | INFO | Opening tmp RTMetrics json file | 20260506 20:34:55.800 | INFO | Serializing RTMetrics to json | 20260506 20:34:55.806 | INFO | Serialization done, swapping tmp file now... | 20260506 20:34:55.810 | INFO | RTMetrics done | 20260506 20:34:56.306 | INFO | Finished a round of Baz IO: Wrote 1.976 GiB in 3.651 seconds | 20260506 20:34:56.306 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.20% / 0.00% / 65.80% | 20260506 20:34:56.306 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.73328% / 3.97364e-06% / 8.74201e-05% | 20260506 20:34:56.306 | INFO | Baz IO overall: | 20260506 20:34:56.306 | INFO | BytesWritten : 2250728346094 | 20260506 20:34:56.306 | INFO | NumEvents : 1087302745739 | 20260506 20:34:57.695 | INFO | Analyzing chunk frames = [4263936,4264448) | 20260506 20:34:57.695 | INFO | MemInfo: { | 20260506 20:34:57.695 | INFO | "gpu" : " 17 GB", | 20260506 20:34:57.695 | INFO | "huge" : "", | 20260506 20:34:57.695 | INFO | "huge_shared" : " 27 GB", | 20260506 20:34:57.695 | INFO | "malloc" : " 14 GB", | 20260506 20:34:57.695 | INFO | "pinned" : " 29 GB" | 20260506 20:34:57.695 | INFO | } | 20260506 20:34:57.695 | INFO | Perf report for the last 5.14074 seconds: | 20260506 20:34:57.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:34:57.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:57.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:57.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:57.696 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 20:34:57.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:34:57.696 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 20:34:57.696 | INFO | PRE_HQ : 60.0%, 1.8, 57.0ms, 0 | 20260506 20:34:57.696 | INFO | BAZWRITER : 70.7%, 4.8, 173.8ms, 0 | 20260506 20:34:57.696 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 20:34:57.696 | INFO | Uploaded 36GiB to GPU in 2.05108s (17.5517GiB/s) | 20260506 20:34:57.696 | INFO | Downloaded 38GiB to GPU in 2.19889s (17.2814GiB/s) | 20260506 20:35:02.840 | INFO | Analyzing chunk frames = [4264448,4264960) | 20260506 20:35:02.840 | INFO | MemInfo: { | 20260506 20:35:02.840 | INFO | "gpu" : " 17 GB", | 20260506 20:35:02.840 | INFO | "huge" : "", | 20260506 20:35:02.840 | INFO | "huge_shared" : " 27 GB", | 20260506 20:35:02.840 | INFO | "malloc" : " 14 GB", | 20260506 20:35:02.840 | INFO | "pinned" : " 29 GB" | 20260506 20:35:02.840 | INFO | } | 20260506 20:35:02.840 | INFO | Perf report for the last 5.14451 seconds: | 20260506 20:35:02.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:35:02.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:02.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:02.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:02.840 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:35:02.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:02.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:02.840 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:35:02.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:02.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:02.840 | INFO | Uploaded 36GiB to GPU in 2.01379s (17.8767GiB/s) | 20260506 20:35:02.840 | INFO | Downloaded 34GiB to GPU in 1.92374s (17.6739GiB/s) | 20260506 20:35:07.981 | INFO | Analyzing chunk frames = [4264960,4265472) | 20260506 20:35:07.981 | INFO | MemInfo: { | 20260506 20:35:07.981 | INFO | "gpu" : " 17 GB", | 20260506 20:35:07.981 | INFO | "huge" : "", | 20260506 20:35:07.981 | INFO | "huge_shared" : " 27 GB", | 20260506 20:35:07.981 | INFO | "malloc" : " 14 GB", | 20260506 20:35:07.981 | INFO | "pinned" : " 29 GB" | 20260506 20:35:07.981 | INFO | } | 20260506 20:35:07.981 | INFO | Perf report for the last 5.14078 seconds: | 20260506 20:35:07.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:35:07.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:07.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:07.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:07.981 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 20:35:07.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:07.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:07.981 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:35:07.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:07.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:07.981 | INFO | Uploaded 36GiB to GPU in 1.9918s (18.0741GiB/s) | 20260506 20:35:07.981 | INFO | Downloaded 34GiB to GPU in 1.92279s (17.6827GiB/s) | 20260506 20:35:13.115 | INFO | Analyzing chunk frames = [4265472,4265984) | 20260506 20:35:13.115 | INFO | MemInfo: { | 20260506 20:35:13.115 | INFO | "gpu" : " 17 GB", | 20260506 20:35:13.115 | INFO | "huge" : "", | 20260506 20:35:13.115 | INFO | "huge_shared" : " 27 GB", | 20260506 20:35:13.115 | INFO | "malloc" : " 14 GB", | 20260506 20:35:13.115 | INFO | "pinned" : " 29 GB" | 20260506 20:35:13.115 | INFO | } | 20260506 20:35:13.115 | INFO | Perf report for the last 5.13452 seconds: | 20260506 20:35:13.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:35:13.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:13.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:13.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:13.115 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 20:35:13.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:13.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:13.115 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:35:13.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:13.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:13.116 | INFO | Uploaded 36GiB to GPU in 1.97338s (18.2428GiB/s) | 20260506 20:35:13.116 | INFO | Downloaded 34GiB to GPU in 1.89059s (17.9838GiB/s) | 20260506 20:35:18.252 | INFO | Analyzing chunk frames = [4265984,4266496) | 20260506 20:35:18.252 | INFO | MemInfo: { | 20260506 20:35:18.252 | INFO | "gpu" : " 17 GB", | 20260506 20:35:18.252 | INFO | "huge" : "", | 20260506 20:35:18.252 | INFO | "huge_shared" : " 27 GB", | 20260506 20:35:18.252 | INFO | "malloc" : " 14 GB", | 20260506 20:35:18.252 | INFO | "pinned" : " 29 GB" | 20260506 20:35:18.252 | INFO | } | 20260506 20:35:18.252 | INFO | Perf report for the last 5.13699 seconds: | 20260506 20:35:18.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:35:18.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:18.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:18.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:18.253 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 20:35:18.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:18.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:18.253 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:35:18.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:18.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:18.254 | INFO | Uploaded 36GiB to GPU in 2.0183s (17.8368GiB/s) | 20260506 20:35:18.254 | INFO | Downloaded 34GiB to GPU in 1.90771s (17.8224GiB/s) | 20260506 20:35:23.390 | INFO | Analyzing chunk frames = [4266496,4267008) | 20260506 20:35:23.390 | INFO | MemInfo: { | 20260506 20:35:23.390 | INFO | "gpu" : " 17 GB", | 20260506 20:35:23.390 | INFO | "huge" : "", | 20260506 20:35:23.390 | INFO | "huge_shared" : " 27 GB", | 20260506 20:35:23.390 | INFO | "malloc" : " 14 GB", | 20260506 20:35:23.390 | INFO | "pinned" : " 29 GB" | 20260506 20:35:23.390 | INFO | } | 20260506 20:35:23.390 | INFO | Perf report for the last 5.1373 seconds: | 20260506 20:35:23.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:35:23.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:23.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:23.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:23.390 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:35:23.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:23.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:23.390 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 20:35:23.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:23.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:23.390 | INFO | Uploaded 36GiB to GPU in 2.02638s (17.7657GiB/s) | 20260506 20:35:23.390 | INFO | Downloaded 34GiB to GPU in 1.93676s (17.5551GiB/s) | 20260506 20:35:28.529 | INFO | Analyzing chunk frames = [4267008,4267520) | 20260506 20:35:28.529 | INFO | MemInfo: { | 20260506 20:35:28.529 | INFO | "gpu" : " 17 GB", | 20260506 20:35:28.529 | INFO | "huge" : "", | 20260506 20:35:28.529 | INFO | "huge_shared" : " 27 GB", | 20260506 20:35:28.529 | INFO | "malloc" : " 14 GB", | 20260506 20:35:28.529 | INFO | "pinned" : " 29 GB" | 20260506 20:35:28.529 | INFO | } | 20260506 20:35:28.529 | INFO | Perf report for the last 5.1393 seconds: | 20260506 20:35:28.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:35:28.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:28.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:28.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:28.529 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 20:35:28.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:28.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:28.529 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 20:35:28.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:28.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:28.530 | INFO | Uploaded 36GiB to GPU in 1.9665s (18.3067GiB/s) | 20260506 20:35:28.530 | INFO | Downloaded 34GiB to GPU in 1.99196s (17.0686GiB/s) | 20260506 20:35:33.668 | INFO | Analyzing chunk frames = [4267520,4268032) | 20260506 20:35:33.668 | INFO | MemInfo: { | 20260506 20:35:33.668 | INFO | "gpu" : " 17 GB", | 20260506 20:35:33.668 | INFO | "huge" : "", | 20260506 20:35:33.668 | INFO | "huge_shared" : " 27 GB", | 20260506 20:35:33.668 | INFO | "malloc" : " 14 GB", | 20260506 20:35:33.668 | INFO | "pinned" : " 29 GB" | 20260506 20:35:33.668 | INFO | } | 20260506 20:35:33.668 | INFO | Perf report for the last 5.13953 seconds: | 20260506 20:35:33.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:35:33.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:33.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:33.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:33.669 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:35:33.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:33.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:33.669 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 20:35:33.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:33.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:33.669 | INFO | Uploaded 36GiB to GPU in 1.98839s (18.1051GiB/s) | 20260506 20:35:33.669 | INFO | Downloaded 34GiB to GPU in 1.97789s (17.1901GiB/s) | 20260506 20:35:33.669 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:35:33.669 | INFO | Total mem usage: 18 GiB | 20260506 20:35:33.669 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:35:33.669 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -15%) | 20260506 20:35:33.669 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:35:33.669 | INFO | High Water Breakdown: | 20260506 20:35:33.669 | INFO | 61.5% : 9.63 GiB ( 84.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:35:33.669 | INFO | 38.4% : 6.01 GiB ( 89.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:35:33.669 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:35:33.669 | INFO | ------------------------------------------ | 20260506 20:35:33.669 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:35:33.669 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:35:33.669 | INFO | Total mem usage: 26.5 GiB | 20260506 20:35:33.669 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:35:33.669 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:35:33.669 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:35:33.669 | INFO | High Water Breakdown: | 20260506 20:35:33.669 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:35:33.669 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:35:33.669 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:35:33.669 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:35:33.669 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:35:33.669 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:35:33.669 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:35:33.669 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:35:33.669 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:35:33.669 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:35:33.669 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:35:33.669 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:35:33.669 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:35:33.669 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:35:33.669 | INFO | ------------------------------------------ | 20260506 20:35:33.669 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:35:33.669 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:35:33.669 | INFO | Total mem usage: 25.6 GiB | 20260506 20:35:33.669 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:35:33.669 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 20:35:33.669 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 20:35:33.669 | INFO | High Water Breakdown: | 20260506 20:35:33.669 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 20:35:33.669 | INFO | ------------------------------------------ | 20260506 20:35:33.669 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:35:33.670 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:35:33.670 | INFO | Total mem usage: 15.7 GiB | 20260506 20:35:33.670 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:35:33.670 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:35:33.670 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:35:33.670 | INFO | High Water Breakdown: | 20260506 20:35:33.670 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:35:33.670 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:35:33.670 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:35:33.670 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:35:33.670 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:35:33.670 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:35:33.670 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:35:33.670 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:35:33.670 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:35:33.670 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:35:33.670 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:35:33.670 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:35:33.670 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:35:33.670 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:35:33.670 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:35:33.670 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:35:33.670 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:35:33.670 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:35:33.670 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:35:33.670 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:35:33.670 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:35:33.670 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:35:33.670 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:35:33.670 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:35:33.670 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:35:33.670 | INFO | ------------------------------------------ | 20260506 20:35:33.670 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:35:33.670 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:35:33.670 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:35:33.670 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:35:33.670 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.61e+01, 1.35e+01 / 2.62e+00 | 20260506 20:35:33.670 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 1.03e-02 / 8.14e-03 | 20260506 20:35:33.670 | INFO | PerfReport: 11.65%, Baseline -- 3.63e+00 / 4.38e+00, 5.53e-01 / 3.81e-01 | 20260506 20:35:33.670 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.43e+00, 1.81e-01 / 1.52e-01 | 20260506 20:35:33.670 | INFO | PerfReport: 16.02%, DME -- 3.99e+01 / 4.49e+01, 4.16e+00 / 3.05e+00 | 20260506 20:35:33.670 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.41e+00 / 1.90e+00 | 20260506 20:35:33.670 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.22e+00 / 2.76e+00, 3.74e-01 / 2.37e-01 | 20260506 20:35:33.670 | INFO | PerfReport: 3.31%, Metrics -- 1.03e+00 / 1.35e+00, 2.69e-01 / 1.15e-01 | 20260506 20:35:33.670 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.55e-01 / 4.06e-01, 5.69e-02 / 2.29e-02 | 20260506 20:35:33.670 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:35:33.670 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:35:33.670 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:35:33.670 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 20:35:33.670 | INFO | PerfReport: 21.58%, Upload -- 2.09e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260506 20:35:33.670 | INFO | PerfReport: 21.20%, Download -- 2.05e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 20:35:33.670 | INFO | PerfReport: 32.19%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 20:35:33.670 | INFO | PerfReport: 1.40%, LockUpload -- 1.36e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260506 20:35:33.670 | INFO | PerfReport: 1.11%, LockDownload -- 1.07e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260506 20:35:33.670 | INFO | PerfReport: 22.49%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 20:35:33.670 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.55e-02 / 0.00e+00 | 20260506 20:35:33.784 | INFO | Starting a round of Baz IO | 20260506 20:35:36.909 | INFO | Populating RTMetrics report | 20260506 20:35:36.910 | INFO | RTMetrics report populated | 20260506 20:35:36.910 | INFO | Opening tmp RTMetrics json file | 20260506 20:35:36.955 | INFO | Serializing RTMetrics to json | 20260506 20:35:36.997 | INFO | Serialization done, swapping tmp file now... | 20260506 20:35:37.037 | INFO | RTMetrics done | 20260506 20:35:37.360 | INFO | Finished a round of Baz IO: Wrote 1.974 GiB in 3.575 seconds | 20260506 20:35:37.360 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.20% / 0.00% / 65.80% | 20260506 20:35:37.360 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.730828% / 0% / 4.76837e-05% | 20260506 20:35:37.360 | INFO | Baz IO overall: | 20260506 20:35:37.360 | INFO | BytesWritten : 2252848429570 | 20260506 20:35:37.360 | INFO | NumEvents : 1088319849439 | 20260506 20:35:38.815 | INFO | Analyzing chunk frames = [4268032,4268544) | 20260506 20:35:38.815 | INFO | MemInfo: { | 20260506 20:35:38.816 | INFO | "gpu" : " 17 GB", | 20260506 20:35:38.816 | INFO | "huge" : "", | 20260506 20:35:38.816 | INFO | "huge_shared" : " 27 GB", | 20260506 20:35:38.816 | INFO | "malloc" : " 14 GB", | 20260506 20:35:38.816 | INFO | "pinned" : " 29 GB" | 20260506 20:35:38.816 | INFO | } | 20260506 20:35:38.816 | INFO | Perf report for the last 5.14707 seconds: | 20260506 20:35:38.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:35:38.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:38.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:38.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:38.816 | INFO | ANALYSIS : 62.4%, 2.9, 95.3ms, 0 | 20260506 20:35:38.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:38.816 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260506 20:35:38.816 | INFO | PRE_HQ : 60.2%, 1.9, 58.0ms, 0 | 20260506 20:35:38.816 | INFO | BAZWRITER : 69.1%, 4.8, 171.1ms, 0 | 20260506 20:35:38.816 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 20:35:38.816 | INFO | Uploaded 36GiB to GPU in 2.05678s (17.5031GiB/s) | 20260506 20:35:38.816 | INFO | Downloaded 38GiB to GPU in 2.20208s (17.2564GiB/s) | 20260506 20:35:43.957 | INFO | Analyzing chunk frames = [4268544,4269056) | 20260506 20:35:43.957 | INFO | MemInfo: { | 20260506 20:35:43.957 | INFO | "gpu" : " 17 GB", | 20260506 20:35:43.957 | INFO | "huge" : "", | 20260506 20:35:43.957 | INFO | "huge_shared" : " 27 GB", | 20260506 20:35:43.957 | INFO | "malloc" : " 14 GB", | 20260506 20:35:43.957 | INFO | "pinned" : " 29 GB" | 20260506 20:35:43.957 | INFO | } | 20260506 20:35:43.957 | INFO | Perf report for the last 5.14146 seconds: | 20260506 20:35:43.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:35:43.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:43.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:43.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:43.957 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 20:35:43.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:43.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:43.957 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:35:43.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:43.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:43.957 | INFO | Uploaded 36GiB to GPU in 2.00122s (17.989GiB/s) | 20260506 20:35:43.957 | INFO | Downloaded 34GiB to GPU in 1.92119s (17.6974GiB/s) | 20260506 20:35:49.096 | INFO | Analyzing chunk frames = [4269056,4269568) | 20260506 20:35:49.096 | INFO | MemInfo: { | 20260506 20:35:49.096 | INFO | "gpu" : " 17 GB", | 20260506 20:35:49.096 | INFO | "huge" : "", | 20260506 20:35:49.096 | INFO | "huge_shared" : " 27 GB", | 20260506 20:35:49.096 | INFO | "malloc" : " 14 GB", | 20260506 20:35:49.096 | INFO | "pinned" : " 29 GB" | 20260506 20:35:49.096 | INFO | } | 20260506 20:35:49.096 | INFO | Perf report for the last 5.13877 seconds: | 20260506 20:35:49.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:35:49.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:49.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:49.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:49.096 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260506 20:35:49.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:49.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:49.096 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:35:49.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:49.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:49.096 | INFO | Uploaded 36GiB to GPU in 2.00582s (17.9478GiB/s) | 20260506 20:35:49.096 | INFO | Downloaded 34GiB to GPU in 1.93003s (17.6163GiB/s) | 20260506 20:35:54.235 | INFO | Analyzing chunk frames = [4269568,4270080) | 20260506 20:35:54.235 | INFO | MemInfo: { | 20260506 20:35:54.235 | INFO | "gpu" : " 17 GB", | 20260506 20:35:54.235 | INFO | "huge" : "", | 20260506 20:35:54.235 | INFO | "huge_shared" : " 27 GB", | 20260506 20:35:54.235 | INFO | "malloc" : " 14 GB", | 20260506 20:35:54.235 | INFO | "pinned" : " 29 GB" | 20260506 20:35:54.235 | INFO | } | 20260506 20:35:54.235 | INFO | Perf report for the last 5.1389 seconds: | 20260506 20:35:54.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:35:54.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:54.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:54.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:54.235 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 20:35:54.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:54.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:54.235 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:35:54.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:54.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:54.235 | INFO | Uploaded 36GiB to GPU in 1.95865s (18.38GiB/s) | 20260506 20:35:54.235 | INFO | Downloaded 34GiB to GPU in 1.87472s (18.1361GiB/s) | 20260506 20:35:59.364 | INFO | Analyzing chunk frames = [4270080,4270592) | 20260506 20:35:59.364 | INFO | MemInfo: { | 20260506 20:35:59.364 | INFO | "gpu" : " 17 GB", | 20260506 20:35:59.364 | INFO | "huge" : "", | 20260506 20:35:59.364 | INFO | "huge_shared" : " 27 GB", | 20260506 20:35:59.364 | INFO | "malloc" : " 14 GB", | 20260506 20:35:59.364 | INFO | "pinned" : " 29 GB" | 20260506 20:35:59.364 | INFO | } | 20260506 20:35:59.364 | INFO | Perf report for the last 5.12916 seconds: | 20260506 20:35:59.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:35:59.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:59.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:59.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:59.364 | INFO | ANALYSIS : 60.5%, 3.0, 94.0ms, 0 | 20260506 20:35:59.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:59.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:59.364 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 20:35:59.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:59.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:35:59.364 | INFO | Uploaded 36GiB to GPU in 2.00777s (17.9304GiB/s) | 20260506 20:35:59.364 | INFO | Downloaded 34GiB to GPU in 1.90925s (17.8081GiB/s) | 20260506 20:36:04.503 | INFO | Analyzing chunk frames = [4270592,4271104) | 20260506 20:36:04.503 | INFO | MemInfo: { | 20260506 20:36:04.503 | INFO | "gpu" : " 17 GB", | 20260506 20:36:04.503 | INFO | "huge" : "", | 20260506 20:36:04.503 | INFO | "huge_shared" : " 27 GB", | 20260506 20:36:04.503 | INFO | "malloc" : " 14 GB", | 20260506 20:36:04.503 | INFO | "pinned" : " 29 GB" | 20260506 20:36:04.504 | INFO | } | 20260506 20:36:04.504 | INFO | Perf report for the last 5.13972 seconds: | 20260506 20:36:04.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:36:04.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:04.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:04.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:04.504 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:36:04.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:04.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:04.504 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:36:04.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:04.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:04.504 | INFO | Uploaded 36GiB to GPU in 2.01253s (17.8879GiB/s) | 20260506 20:36:04.504 | INFO | Downloaded 34GiB to GPU in 1.95204s (17.4177GiB/s) | 20260506 20:36:09.649 | INFO | Analyzing chunk frames = [4271104,4271616) | 20260506 20:36:09.649 | INFO | MemInfo: { | 20260506 20:36:09.649 | INFO | "gpu" : " 17 GB", | 20260506 20:36:09.649 | INFO | "huge" : "", | 20260506 20:36:09.649 | INFO | "huge_shared" : " 27 GB", | 20260506 20:36:09.649 | INFO | "malloc" : " 14 GB", | 20260506 20:36:09.649 | INFO | "pinned" : " 29 GB" | 20260506 20:36:09.649 | INFO | } | 20260506 20:36:09.649 | INFO | Perf report for the last 5.14552 seconds: | 20260506 20:36:09.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:36:09.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:09.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:09.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:09.649 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 20:36:09.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:09.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:09.649 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 20:36:09.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:09.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:09.650 | INFO | Uploaded 36GiB to GPU in 1.95539s (18.4107GiB/s) | 20260506 20:36:09.650 | INFO | Downloaded 34GiB to GPU in 1.99171s (17.0708GiB/s) | 20260506 20:36:14.787 | INFO | Analyzing chunk frames = [4271616,4272128) | 20260506 20:36:14.787 | INFO | MemInfo: { | 20260506 20:36:14.787 | INFO | "gpu" : " 17 GB", | 20260506 20:36:14.787 | INFO | "huge" : "", | 20260506 20:36:14.787 | INFO | "huge_shared" : " 27 GB", | 20260506 20:36:14.787 | INFO | "malloc" : " 14 GB", | 20260506 20:36:14.787 | INFO | "pinned" : " 29 GB" | 20260506 20:36:14.787 | INFO | } | 20260506 20:36:14.787 | INFO | Perf report for the last 5.13779 seconds: | 20260506 20:36:14.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:36:14.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:14.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:14.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:14.787 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 20:36:14.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:14.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:14.787 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 20:36:14.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:14.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:14.787 | INFO | Uploaded 36GiB to GPU in 2.00986s (17.9117GiB/s) | 20260506 20:36:14.787 | INFO | Downloaded 34GiB to GPU in 1.99242s (17.0646GiB/s) | 20260506 20:36:14.900 | INFO | Starting a round of Baz IO | 20260506 20:36:18.053 | INFO | Populating RTMetrics report | 20260506 20:36:18.054 | INFO | RTMetrics report populated | 20260506 20:36:18.054 | INFO | Opening tmp RTMetrics json file | 20260506 20:36:18.057 | INFO | Serializing RTMetrics to json | 20260506 20:36:18.084 | INFO | Serialization done, swapping tmp file now... | 20260506 20:36:18.099 | INFO | RTMetrics done | 20260506 20:36:18.387 | INFO | Finished a round of Baz IO: Wrote 1.975 GiB in 3.486 seconds | 20260506 20:36:18.387 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.19% / 0.00% / 65.80% | 20260506 20:36:18.387 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.728504% / 3.97364e-06% / 7.54992e-05% | 20260506 20:36:18.387 | INFO | Baz IO overall: | 20260506 20:36:18.387 | INFO | BytesWritten : 2254968679667 | 20260506 20:36:18.387 | INFO | NumEvents : 1089337262763 | 20260506 20:36:19.926 | INFO | Analyzing chunk frames = [4272128,4272640) | 20260506 20:36:19.926 | INFO | MemInfo: { | 20260506 20:36:19.926 | INFO | "gpu" : " 17 GB", | 20260506 20:36:19.926 | INFO | "huge" : "", | 20260506 20:36:19.926 | INFO | "huge_shared" : " 27 GB", | 20260506 20:36:19.926 | INFO | "malloc" : " 14 GB", | 20260506 20:36:19.926 | INFO | "pinned" : " 29 GB" | 20260506 20:36:19.926 | INFO | } | 20260506 20:36:19.926 | INFO | Perf report for the last 5.13952 seconds: | 20260506 20:36:19.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:36:19.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:19.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:19.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:19.927 | INFO | ANALYSIS : 63.0%, 2.9, 94.1ms, 0 | 20260506 20:36:19.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:19.927 | INFO | RT_METRICS : 47.4%, 1.0, 24.8ms, 0 | 20260506 20:36:19.927 | INFO | PRE_HQ : 61.3%, 1.9, 62.0ms, 0 | 20260506 20:36:19.927 | INFO | BAZWRITER : 67.6%, 4.8, 165.9ms, 0 | 20260506 20:36:19.927 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 20:36:19.927 | INFO | Uploaded 36GiB to GPU in 2.08551s (17.2619GiB/s) | 20260506 20:36:19.927 | INFO | Downloaded 38GiB to GPU in 2.19302s (17.3277GiB/s) | 20260506 20:36:25.066 | INFO | Analyzing chunk frames = [4272640,4273152) | 20260506 20:36:25.066 | INFO | MemInfo: { | 20260506 20:36:25.066 | INFO | "gpu" : " 17 GB", | 20260506 20:36:25.066 | INFO | "huge" : "", | 20260506 20:36:25.066 | INFO | "huge_shared" : " 27 GB", | 20260506 20:36:25.066 | INFO | "malloc" : " 14 GB", | 20260506 20:36:25.066 | INFO | "pinned" : " 29 GB" | 20260506 20:36:25.066 | INFO | } | 20260506 20:36:25.066 | INFO | Perf report for the last 5.13979 seconds: | 20260506 20:36:25.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:36:25.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:25.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:25.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:25.066 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:36:25.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:25.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:25.066 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:36:25.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:25.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:25.067 | INFO | Uploaded 36GiB to GPU in 2.01315s (17.8824GiB/s) | 20260506 20:36:25.067 | INFO | Downloaded 34GiB to GPU in 1.93208s (17.5976GiB/s) | 20260506 20:36:30.202 | INFO | Analyzing chunk frames = [4273152,4273664) | 20260506 20:36:30.202 | INFO | MemInfo: { | 20260506 20:36:30.202 | INFO | "gpu" : " 17 GB", | 20260506 20:36:30.202 | INFO | "huge" : "", | 20260506 20:36:30.202 | INFO | "huge_shared" : " 27 GB", | 20260506 20:36:30.202 | INFO | "malloc" : " 14 GB", | 20260506 20:36:30.202 | INFO | "pinned" : " 29 GB" | 20260506 20:36:30.202 | INFO | } | 20260506 20:36:30.202 | INFO | Perf report for the last 5.13569 seconds: | 20260506 20:36:30.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:36:30.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:30.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:30.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:30.202 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 20:36:30.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:30.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:30.202 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:36:30.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:30.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:30.202 | INFO | Uploaded 36GiB to GPU in 1.97409s (18.2363GiB/s) | 20260506 20:36:30.203 | INFO | Downloaded 34GiB to GPU in 1.91962s (17.7118GiB/s) | 20260506 20:36:35.340 | INFO | Analyzing chunk frames = [4273664,4274176) | 20260506 20:36:35.340 | INFO | MemInfo: { | 20260506 20:36:35.340 | INFO | "gpu" : " 17 GB", | 20260506 20:36:35.340 | INFO | "huge" : "", | 20260506 20:36:35.340 | INFO | "huge_shared" : " 27 GB", | 20260506 20:36:35.340 | INFO | "malloc" : " 14 GB", | 20260506 20:36:35.341 | INFO | "pinned" : " 29 GB" | 20260506 20:36:35.341 | INFO | } | 20260506 20:36:35.341 | INFO | Perf report for the last 5.13853 seconds: | 20260506 20:36:35.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:36:35.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:35.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:35.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:35.341 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:36:35.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:35.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:35.341 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:36:35.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:35.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:35.341 | INFO | Uploaded 36GiB to GPU in 1.95102s (18.4519GiB/s) | 20260506 20:36:35.341 | INFO | Downloaded 34GiB to GPU in 1.87309s (18.1518GiB/s) | 20260506 20:36:40.482 | INFO | Analyzing chunk frames = [4274176,4274688) | 20260506 20:36:40.482 | INFO | MemInfo: { | 20260506 20:36:40.482 | INFO | "gpu" : " 17 GB", | 20260506 20:36:40.482 | INFO | "huge" : "", | 20260506 20:36:40.482 | INFO | "huge_shared" : " 27 GB", | 20260506 20:36:40.482 | INFO | "malloc" : " 14 GB", | 20260506 20:36:40.482 | INFO | "pinned" : " 29 GB" | 20260506 20:36:40.482 | INFO | } | 20260506 20:36:40.482 | INFO | Perf report for the last 5.14187 seconds: | 20260506 20:36:40.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:36:40.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:40.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:40.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:40.482 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 20:36:40.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:40.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:40.482 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:36:40.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:40.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:40.483 | INFO | Uploaded 36GiB to GPU in 1.995s (18.0451GiB/s) | 20260506 20:36:40.483 | INFO | Downloaded 34GiB to GPU in 1.91874s (17.7199GiB/s) | 20260506 20:36:45.621 | INFO | Analyzing chunk frames = [4274688,4275200) | 20260506 20:36:45.621 | INFO | MemInfo: { | 20260506 20:36:45.621 | INFO | "gpu" : " 17 GB", | 20260506 20:36:45.621 | INFO | "huge" : "", | 20260506 20:36:45.621 | INFO | "huge_shared" : " 27 GB", | 20260506 20:36:45.621 | INFO | "malloc" : " 14 GB", | 20260506 20:36:45.621 | INFO | "pinned" : " 29 GB" | 20260506 20:36:45.621 | INFO | } | 20260506 20:36:45.621 | INFO | Perf report for the last 5.13889 seconds: | 20260506 20:36:45.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:36:45.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:45.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:45.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:45.621 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:36:45.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:45.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:45.621 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:36:45.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:45.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:45.622 | INFO | Uploaded 36GiB to GPU in 2.02609s (17.7682GiB/s) | 20260506 20:36:45.622 | INFO | Downloaded 34GiB to GPU in 1.94693s (17.4634GiB/s) | 20260506 20:36:50.761 | INFO | Analyzing chunk frames = [4275200,4275712) | 20260506 20:36:50.761 | INFO | MemInfo: { | 20260506 20:36:50.761 | INFO | "gpu" : " 17 GB", | 20260506 20:36:50.761 | INFO | "huge" : "", | 20260506 20:36:50.761 | INFO | "huge_shared" : " 27 GB", | 20260506 20:36:50.761 | INFO | "malloc" : " 14 GB", | 20260506 20:36:50.761 | INFO | "pinned" : " 29 GB" | 20260506 20:36:50.761 | INFO | } | 20260506 20:36:50.761 | INFO | Perf report for the last 5.13978 seconds: | 20260506 20:36:50.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:36:50.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:50.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:50.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:50.761 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:36:50.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:50.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:50.761 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 20:36:50.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:50.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:50.762 | INFO | Uploaded 36GiB to GPU in 1.98005s (18.1813GiB/s) | 20260506 20:36:50.762 | INFO | Downloaded 34GiB to GPU in 2.00269s (16.9771GiB/s) | 20260506 20:36:55.911 | INFO | Analyzing chunk frames = [4275712,4276224) | 20260506 20:36:55.911 | INFO | MemInfo: { | 20260506 20:36:55.911 | INFO | "gpu" : " 17 GB", | 20260506 20:36:55.911 | INFO | "huge" : "", | 20260506 20:36:55.911 | INFO | "huge_shared" : " 27 GB", | 20260506 20:36:55.911 | INFO | "malloc" : " 14 GB", | 20260506 20:36:55.911 | INFO | "pinned" : " 29 GB" | 20260506 20:36:55.911 | INFO | } | 20260506 20:36:55.911 | INFO | Perf report for the last 5.14955 seconds: | 20260506 20:36:55.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:36:55.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:55.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:55.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:55.911 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 20:36:55.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:55.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:55.911 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 20:36:55.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:55.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:36:55.911 | INFO | Uploaded 36GiB to GPU in 2.04977s (17.5629GiB/s) | 20260506 20:36:55.911 | INFO | Downloaded 34GiB to GPU in 1.99909s (17.0077GiB/s) | 20260506 20:36:55.911 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:36:55.911 | INFO | Total mem usage: 18 GiB | 20260506 20:36:55.911 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:36:55.911 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -11%) | 20260506 20:36:55.911 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:36:55.911 | INFO | High Water Breakdown: | 20260506 20:36:55.911 | INFO | 62.1% : 10.1 GiB ( 88.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:36:55.911 | INFO | 37.8% : 6.13 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:36:55.911 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:36:55.911 | INFO | ------------------------------------------ | 20260506 20:36:55.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:36:55.911 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:36:55.912 | INFO | Total mem usage: 26.5 GiB | 20260506 20:36:55.912 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:36:55.912 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:36:55.912 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:36:55.912 | INFO | High Water Breakdown: | 20260506 20:36:55.912 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:36:55.912 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:36:55.912 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:36:55.912 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:36:55.912 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:36:55.912 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:36:55.912 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:36:55.912 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:36:55.912 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:36:55.912 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:36:55.912 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:36:55.912 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:36:55.912 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:36:55.912 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:36:55.912 | INFO | ------------------------------------------ | 20260506 20:36:55.912 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:36:55.912 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:36:55.912 | INFO | Total mem usage: 25.6 GiB | 20260506 20:36:55.912 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:36:55.912 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:36:55.912 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:36:55.912 | INFO | High Water Breakdown: | 20260506 20:36:55.912 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:36:55.912 | INFO | ------------------------------------------ | 20260506 20:36:55.912 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:36:55.912 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:36:55.912 | INFO | Total mem usage: 15.7 GiB | 20260506 20:36:55.912 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:36:55.912 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:36:55.912 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:36:55.912 | INFO | High Water Breakdown: | 20260506 20:36:55.912 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:36:55.912 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:36:55.912 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:36:55.912 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:36:55.912 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:36:55.912 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:36:55.912 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:36:55.912 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:36:55.912 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:36:55.912 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:36:55.912 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:36:55.912 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:36:55.912 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:36:55.912 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:36:55.912 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:36:55.912 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:36:55.912 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:36:55.912 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:36:55.912 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:36:55.912 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:36:55.912 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:36:55.912 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:36:55.912 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:36:55.912 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:36:55.912 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:36:55.912 | INFO | ------------------------------------------ | 20260506 20:36:55.912 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:36:55.912 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:36:55.912 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:36:55.912 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:36:55.912 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.58e+01, 1.36e+01 / 2.66e+00 | 20260506 20:36:55.912 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 1.09e-02 / 5.78e-03 | 20260506 20:36:55.912 | INFO | PerfReport: 11.63%, Baseline -- 3.62e+00 / 4.40e+00, 5.46e-01 / 2.71e-01 | 20260506 20:36:55.912 | INFO | PerfReport: 3.24%, Histogram -- 1.01e+00 / 1.44e+00, 1.76e-01 / 1.35e-01 | 20260506 20:36:55.912 | INFO | PerfReport: 16.06%, DME -- 4.00e+01 / 4.50e+01, 4.31e+00 / 2.80e+00 | 20260506 20:36:55.912 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.38e+00 / 1.91e+00 | 20260506 20:36:55.912 | INFO | PerfReport: 7.16%, PulseAccumulating -- 2.23e+00 / 2.83e+00, 3.88e-01 / 2.31e-01 | 20260506 20:36:55.912 | INFO | PerfReport: 3.31%, Metrics -- 1.03e+00 / 1.36e+00, 2.69e-01 / 1.31e-01 | 20260506 20:36:55.912 | INFO | PerfReport: 1.14%, PulseSerializing -- 3.53e-01 / 4.03e-01, 5.54e-02 / 1.99e-02 | 20260506 20:36:55.912 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:36:55.912 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:36:55.912 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:36:55.913 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 20:36:55.913 | INFO | PerfReport: 21.61%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260506 20:36:55.913 | INFO | PerfReport: 21.26%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 20:36:55.913 | INFO | PerfReport: 32.22%, Compute -- 3.11e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 20:36:55.913 | INFO | PerfReport: 1.31%, LockUpload -- 1.27e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260506 20:36:55.913 | INFO | PerfReport: 1.15%, LockDownload -- 1.11e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260506 20:36:55.913 | INFO | PerfReport: 22.44%, LockCompute -- 2.17e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 20:36:55.913 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260506 20:36:56.026 | INFO | Starting a round of Baz IO | 20260506 20:36:59.154 | INFO | Populating RTMetrics report | 20260506 20:36:59.154 | INFO | RTMetrics report populated | 20260506 20:36:59.154 | INFO | Opening tmp RTMetrics json file | 20260506 20:36:59.205 | INFO | Serializing RTMetrics to json | 20260506 20:36:59.256 | INFO | Serialization done, swapping tmp file now... | 20260506 20:36:59.286 | INFO | RTMetrics done | 20260506 20:36:59.424 | INFO | Finished a round of Baz IO: Wrote 1.973 GiB in 3.398 seconds | 20260506 20:36:59.424 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.19% / 0.00% / 65.81% | 20260506 20:36:59.424 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.725607% / 7.94729e-06% / 8.34465e-05% | 20260506 20:36:59.424 | INFO | Baz IO overall: | 20260506 20:36:59.424 | INFO | BytesWritten : 2257087333887 | 20260506 20:36:59.424 | INFO | NumEvents : 1090353375507 | 20260506 20:37:01.045 | INFO | Analyzing chunk frames = [4276224,4276736) | 20260506 20:37:01.045 | INFO | MemInfo: { | 20260506 20:37:01.045 | INFO | "gpu" : " 17 GB", | 20260506 20:37:01.045 | INFO | "huge" : "", | 20260506 20:37:01.045 | INFO | "huge_shared" : " 27 GB", | 20260506 20:37:01.045 | INFO | "malloc" : " 14 GB", | 20260506 20:37:01.045 | INFO | "pinned" : " 29 GB" | 20260506 20:37:01.045 | INFO | } | 20260506 20:37:01.045 | INFO | Perf report for the last 5.13419 seconds: | 20260506 20:37:01.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:37:01.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:01.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:01.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:01.045 | INFO | ANALYSIS : 62.6%, 2.9, 95.2ms, 0 | 20260506 20:37:01.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:01.045 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 20:37:01.045 | INFO | PRE_HQ : 60.0%, 1.8, 57.4ms, 0 | 20260506 20:37:01.045 | INFO | BAZWRITER : 65.9%, 4.8, 162.4ms, 0 | 20260506 20:37:01.045 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260506 20:37:01.046 | INFO | Uploaded 36GiB to GPU in 2.07757s (17.328GiB/s) | 20260506 20:37:01.046 | INFO | Downloaded 38GiB to GPU in 2.2162s (17.1465GiB/s) | 20260506 20:37:06.180 | INFO | Analyzing chunk frames = [4276736,4277248) | 20260506 20:37:06.180 | INFO | MemInfo: { | 20260506 20:37:06.180 | INFO | "gpu" : " 17 GB", | 20260506 20:37:06.181 | INFO | "huge" : "", | 20260506 20:37:06.181 | INFO | "huge_shared" : " 27 GB", | 20260506 20:37:06.181 | INFO | "malloc" : " 14 GB", | 20260506 20:37:06.181 | INFO | "pinned" : " 29 GB" | 20260506 20:37:06.181 | INFO | } | 20260506 20:37:06.181 | INFO | Perf report for the last 5.13578 seconds: | 20260506 20:37:06.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:37:06.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:06.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:06.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:06.181 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:37:06.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:06.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:06.181 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:37:06.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:06.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:06.181 | INFO | Uploaded 36GiB to GPU in 2.01596s (17.8575GiB/s) | 20260506 20:37:06.181 | INFO | Downloaded 34GiB to GPU in 1.92706s (17.6435GiB/s) | 20260506 20:37:11.321 | INFO | Analyzing chunk frames = [4277248,4277760) | 20260506 20:37:11.321 | INFO | MemInfo: { | 20260506 20:37:11.321 | INFO | "gpu" : " 17 GB", | 20260506 20:37:11.321 | INFO | "huge" : "", | 20260506 20:37:11.321 | INFO | "huge_shared" : " 27 GB", | 20260506 20:37:11.321 | INFO | "malloc" : " 14 GB", | 20260506 20:37:11.321 | INFO | "pinned" : " 29 GB" | 20260506 20:37:11.321 | INFO | } | 20260506 20:37:11.321 | INFO | Perf report for the last 5.14076 seconds: | 20260506 20:37:11.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:37:11.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:11.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:11.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:11.321 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 20:37:11.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:11.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:11.321 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:37:11.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:11.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:11.322 | INFO | Uploaded 36GiB to GPU in 1.97829s (18.1976GiB/s) | 20260506 20:37:11.322 | INFO | Downloaded 34GiB to GPU in 1.9148s (17.7564GiB/s) | 20260506 20:37:16.455 | INFO | Analyzing chunk frames = [4277760,4278272) | 20260506 20:37:16.455 | INFO | MemInfo: { | 20260506 20:37:16.455 | INFO | "gpu" : " 17 GB", | 20260506 20:37:16.455 | INFO | "huge" : "", | 20260506 20:37:16.455 | INFO | "huge_shared" : " 27 GB", | 20260506 20:37:16.455 | INFO | "malloc" : " 14 GB", | 20260506 20:37:16.455 | INFO | "pinned" : " 29 GB" | 20260506 20:37:16.455 | INFO | } | 20260506 20:37:16.455 | INFO | Perf report for the last 5.13347 seconds: | 20260506 20:37:16.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:37:16.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:16.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:16.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:16.455 | INFO | ANALYSIS : 59.7%, 3.0, 92.4ms, 0 | 20260506 20:37:16.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:16.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:16.455 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:37:16.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:16.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:16.455 | INFO | Uploaded 36GiB to GPU in 1.96011s (18.3663GiB/s) | 20260506 20:37:16.455 | INFO | Downloaded 34GiB to GPU in 1.87663s (18.1176GiB/s) | 20260506 20:37:21.596 | INFO | Analyzing chunk frames = [4278272,4278784) | 20260506 20:37:21.596 | INFO | MemInfo: { | 20260506 20:37:21.596 | INFO | "gpu" : " 17 GB", | 20260506 20:37:21.596 | INFO | "huge" : "", | 20260506 20:37:21.596 | INFO | "huge_shared" : " 27 GB", | 20260506 20:37:21.596 | INFO | "malloc" : " 14 GB", | 20260506 20:37:21.597 | INFO | "pinned" : " 29 GB" | 20260506 20:37:21.597 | INFO | } | 20260506 20:37:21.597 | INFO | Perf report for the last 5.1418 seconds: | 20260506 20:37:21.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:37:21.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:21.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:21.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:21.597 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260506 20:37:21.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:21.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:21.597 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:37:21.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:21.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:21.597 | INFO | Uploaded 36GiB to GPU in 2.01162s (17.896GiB/s) | 20260506 20:37:21.597 | INFO | Downloaded 34GiB to GPU in 1.90572s (17.841GiB/s) | 20260506 20:37:26.741 | INFO | Analyzing chunk frames = [4278784,4279296) | 20260506 20:37:26.741 | INFO | MemInfo: { | 20260506 20:37:26.741 | INFO | "gpu" : " 17 GB", | 20260506 20:37:26.741 | INFO | "huge" : "", | 20260506 20:37:26.741 | INFO | "huge_shared" : " 27 GB", | 20260506 20:37:26.741 | INFO | "malloc" : " 14 GB", | 20260506 20:37:26.741 | INFO | "pinned" : " 29 GB" | 20260506 20:37:26.741 | INFO | } | 20260506 20:37:26.741 | INFO | Perf report for the last 5.14406 seconds: | 20260506 20:37:26.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:37:26.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:26.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:26.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:26.741 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260506 20:37:26.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:26.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:26.741 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:37:26.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:26.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:26.741 | INFO | Uploaded 36GiB to GPU in 2.01493s (17.8666GiB/s) | 20260506 20:37:26.741 | INFO | Downloaded 34GiB to GPU in 1.94844s (17.4498GiB/s) | 20260506 20:37:31.875 | INFO | Analyzing chunk frames = [4279296,4279808) | 20260506 20:37:31.875 | INFO | MemInfo: { | 20260506 20:37:31.875 | INFO | "gpu" : " 17 GB", | 20260506 20:37:31.875 | INFO | "huge" : "", | 20260506 20:37:31.875 | INFO | "huge_shared" : " 27 GB", | 20260506 20:37:31.875 | INFO | "malloc" : " 14 GB", | 20260506 20:37:31.875 | INFO | "pinned" : " 29 GB" | 20260506 20:37:31.875 | INFO | } | 20260506 20:37:31.875 | INFO | Perf report for the last 5.13443 seconds: | 20260506 20:37:31.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:37:31.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:31.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:31.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:31.875 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 20:37:31.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:31.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:31.875 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 20:37:31.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:31.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:31.876 | INFO | Uploaded 36GiB to GPU in 2.02189s (17.8051GiB/s) | 20260506 20:37:31.876 | INFO | Downloaded 34GiB to GPU in 1.98677s (17.1132GiB/s) | 20260506 20:37:37.019 | INFO | Analyzing chunk frames = [4279808,4280320) | 20260506 20:37:37.019 | INFO | MemInfo: { | 20260506 20:37:37.019 | INFO | "gpu" : " 17 GB", | 20260506 20:37:37.019 | INFO | "huge" : "", | 20260506 20:37:37.019 | INFO | "huge_shared" : " 27 GB", | 20260506 20:37:37.019 | INFO | "malloc" : " 14 GB", | 20260506 20:37:37.019 | INFO | "pinned" : " 29 GB" | 20260506 20:37:37.019 | INFO | } | 20260506 20:37:37.020 | INFO | Perf report for the last 5.14428 seconds: | 20260506 20:37:37.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:37:37.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:37.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:37.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:37.020 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 20:37:37.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:37.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:37.020 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 20:37:37.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:37.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:37.020 | INFO | Uploaded 36GiB to GPU in 2.0596s (17.4791GiB/s) | 20260506 20:37:37.020 | INFO | Downloaded 34GiB to GPU in 1.99873s (17.0108GiB/s) | 20260506 20:37:37.148 | INFO | Starting a round of Baz IO | 20260506 20:37:40.257 | INFO | Populating RTMetrics report | 20260506 20:37:40.257 | INFO | RTMetrics report populated | 20260506 20:37:40.257 | INFO | Opening tmp RTMetrics json file | 20260506 20:37:40.275 | INFO | Serializing RTMetrics to json | 20260506 20:37:40.279 | INFO | Serialization done, swapping tmp file now... | 20260506 20:37:40.286 | INFO | RTMetrics done | 20260506 20:37:40.595 | INFO | Finished a round of Baz IO: Wrote 1.972 GiB in 3.447 seconds | 20260506 20:37:40.595 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.19% / 0.00% / 65.81% | 20260506 20:37:40.595 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.724522% / 3.97364e-06% / 5.96046e-05% | 20260506 20:37:40.596 | INFO | Baz IO overall: | 20260506 20:37:40.596 | INFO | BytesWritten : 2259204386217 | 20260506 20:37:40.596 | INFO | NumEvents : 1091368626329 | 20260506 20:37:42.158 | INFO | Analyzing chunk frames = [4280320,4280832) | 20260506 20:37:42.158 | INFO | MemInfo: { | 20260506 20:37:42.158 | INFO | "gpu" : " 17 GB", | 20260506 20:37:42.158 | INFO | "huge" : "", | 20260506 20:37:42.158 | INFO | "huge_shared" : " 27 GB", | 20260506 20:37:42.158 | INFO | "malloc" : " 14 GB", | 20260506 20:37:42.158 | INFO | "pinned" : " 29 GB" | 20260506 20:37:42.158 | INFO | } | 20260506 20:37:42.158 | INFO | Perf report for the last 5.13875 seconds: | 20260506 20:37:42.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:37:42.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:42.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:42.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:42.158 | INFO | ANALYSIS : 62.4%, 2.9, 95.5ms, 0 | 20260506 20:37:42.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:42.158 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 20:37:42.158 | INFO | PRE_HQ : 60.0%, 1.8, 57.6ms, 0 | 20260506 20:37:42.158 | INFO | BAZWRITER : 66.7%, 4.8, 163.8ms, 0 | 20260506 20:37:42.158 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260506 20:37:42.159 | INFO | Uploaded 36GiB to GPU in 2.06458s (17.437GiB/s) | 20260506 20:37:42.159 | INFO | Downloaded 38GiB to GPU in 2.20684s (17.2192GiB/s) | 20260506 20:37:47.296 | INFO | Analyzing chunk frames = [4280832,4281344) | 20260506 20:37:47.296 | INFO | MemInfo: { | 20260506 20:37:47.296 | INFO | "gpu" : " 17 GB", | 20260506 20:37:47.296 | INFO | "huge" : "", | 20260506 20:37:47.296 | INFO | "huge_shared" : " 27 GB", | 20260506 20:37:47.296 | INFO | "malloc" : " 14 GB", | 20260506 20:37:47.296 | INFO | "pinned" : " 29 GB" | 20260506 20:37:47.296 | INFO | } | 20260506 20:37:47.296 | INFO | Perf report for the last 5.13754 seconds: | 20260506 20:37:47.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:37:47.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:47.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:47.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:47.296 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:37:47.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:47.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:47.296 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:37:47.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:47.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:47.296 | INFO | Uploaded 36GiB to GPU in 1.99839s (18.0145GiB/s) | 20260506 20:37:47.296 | INFO | Downloaded 34GiB to GPU in 1.92651s (17.6485GiB/s) | 20260506 20:37:52.432 | INFO | Analyzing chunk frames = [4281344,4281856) | 20260506 20:37:52.432 | INFO | MemInfo: { | 20260506 20:37:52.432 | INFO | "gpu" : " 17 GB", | 20260506 20:37:52.432 | INFO | "huge" : "", | 20260506 20:37:52.432 | INFO | "huge_shared" : " 27 GB", | 20260506 20:37:52.432 | INFO | "malloc" : " 14 GB", | 20260506 20:37:52.432 | INFO | "pinned" : " 29 GB" | 20260506 20:37:52.432 | INFO | } | 20260506 20:37:52.432 | INFO | Perf report for the last 5.13598 seconds: | 20260506 20:37:52.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:37:52.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:52.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:52.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:52.432 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 20:37:52.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:52.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:52.432 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:37:52.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:52.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:52.432 | INFO | Uploaded 36GiB to GPU in 1.97834s (18.197GiB/s) | 20260506 20:37:52.432 | INFO | Downloaded 34GiB to GPU in 1.92094s (17.6997GiB/s) | 20260506 20:37:57.570 | INFO | Analyzing chunk frames = [4281856,4282368) | 20260506 20:37:57.570 | INFO | MemInfo: { | 20260506 20:37:57.570 | INFO | "gpu" : " 17 GB", | 20260506 20:37:57.570 | INFO | "huge" : "", | 20260506 20:37:57.570 | INFO | "huge_shared" : " 27 GB", | 20260506 20:37:57.570 | INFO | "malloc" : " 14 GB", | 20260506 20:37:57.570 | INFO | "pinned" : " 29 GB" | 20260506 20:37:57.570 | INFO | } | 20260506 20:37:57.570 | INFO | Perf report for the last 5.13857 seconds: | 20260506 20:37:57.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:37:57.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:57.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:57.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:57.571 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 20:37:57.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:57.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:57.571 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:37:57.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:57.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:37:57.571 | INFO | Uploaded 36GiB to GPU in 1.96238s (18.345GiB/s) | 20260506 20:37:57.571 | INFO | Downloaded 34GiB to GPU in 1.87638s (18.12GiB/s) | 20260506 20:38:02.713 | INFO | Analyzing chunk frames = [4282368,4282880) | 20260506 20:38:02.713 | INFO | MemInfo: { | 20260506 20:38:02.713 | INFO | "gpu" : " 17 GB", | 20260506 20:38:02.713 | INFO | "huge" : "", | 20260506 20:38:02.713 | INFO | "huge_shared" : " 27 GB", | 20260506 20:38:02.713 | INFO | "malloc" : " 14 GB", | 20260506 20:38:02.713 | INFO | "pinned" : " 29 GB" | 20260506 20:38:02.713 | INFO | } | 20260506 20:38:02.713 | INFO | Perf report for the last 5.14239 seconds: | 20260506 20:38:02.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:38:02.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:02.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:02.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:02.713 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260506 20:38:02.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:02.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:02.713 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:38:02.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:02.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:02.714 | INFO | Uploaded 36GiB to GPU in 2.01661s (17.8518GiB/s) | 20260506 20:38:02.714 | INFO | Downloaded 34GiB to GPU in 1.90742s (17.8251GiB/s) | 20260506 20:38:07.851 | INFO | Analyzing chunk frames = [4282880,4283392) | 20260506 20:38:07.851 | INFO | MemInfo: { | 20260506 20:38:07.851 | INFO | "gpu" : " 17 GB", | 20260506 20:38:07.851 | INFO | "huge" : "", | 20260506 20:38:07.851 | INFO | "huge_shared" : " 27 GB", | 20260506 20:38:07.851 | INFO | "malloc" : " 14 GB", | 20260506 20:38:07.851 | INFO | "pinned" : " 29 GB" | 20260506 20:38:07.851 | INFO | } | 20260506 20:38:07.851 | INFO | Perf report for the last 5.13853 seconds: | 20260506 20:38:07.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:38:07.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:07.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:07.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:07.851 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 20:38:07.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:07.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:07.851 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:38:07.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:07.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:07.852 | INFO | Uploaded 36GiB to GPU in 2.01014s (17.9092GiB/s) | 20260506 20:38:07.852 | INFO | Downloaded 34GiB to GPU in 1.94772s (17.4563GiB/s) | 20260506 20:38:12.999 | INFO | Analyzing chunk frames = [4283392,4283904) | 20260506 20:38:12.999 | INFO | MemInfo: { | 20260506 20:38:12.999 | INFO | "gpu" : " 17 GB", | 20260506 20:38:12.999 | INFO | "huge" : "", | 20260506 20:38:13.000 | INFO | "huge_shared" : " 27 GB", | 20260506 20:38:13.000 | INFO | "malloc" : " 14 GB", | 20260506 20:38:13.000 | INFO | "pinned" : " 29 GB" | 20260506 20:38:13.000 | INFO | } | 20260506 20:38:13.000 | INFO | Perf report for the last 5.1483 seconds: | 20260506 20:38:13.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:38:13.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:13.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:13.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:13.000 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 20:38:13.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:13.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:13.000 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 20:38:13.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:13.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:13.000 | INFO | Uploaded 36GiB to GPU in 1.96664s (18.3054GiB/s) | 20260506 20:38:13.000 | INFO | Downloaded 34GiB to GPU in 1.99443s (17.0475GiB/s) | 20260506 20:38:18.132 | INFO | Analyzing chunk frames = [4283904,4284416) | 20260506 20:38:18.132 | INFO | MemInfo: { | 20260506 20:38:18.132 | INFO | "gpu" : " 17 GB", | 20260506 20:38:18.132 | INFO | "huge" : "", | 20260506 20:38:18.132 | INFO | "huge_shared" : " 27 GB", | 20260506 20:38:18.132 | INFO | "malloc" : " 14 GB", | 20260506 20:38:18.133 | INFO | "pinned" : " 29 GB" | 20260506 20:38:18.133 | INFO | } | 20260506 20:38:18.133 | INFO | Perf report for the last 5.1329 seconds: | 20260506 20:38:18.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:38:18.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:18.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:18.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:18.133 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 20:38:18.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:18.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:18.133 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 20:38:18.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:18.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:18.133 | INFO | Uploaded 36GiB to GPU in 2.02085s (17.8143GiB/s) | 20260506 20:38:18.133 | INFO | Downloaded 34GiB to GPU in 1.99118s (17.0753GiB/s) | 20260506 20:38:18.133 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:38:18.133 | INFO | Total mem usage: 18 GiB | 20260506 20:38:18.133 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:38:18.133 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.8%) | 20260506 20:38:18.133 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:38:18.133 | INFO | High Water Breakdown: | 20260506 20:38:18.133 | INFO | 61.4% : 9.55 GiB ( 83.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:38:18.133 | INFO | 38.5% : 5.98 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:38:18.133 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:38:18.133 | INFO | ------------------------------------------ | 20260506 20:38:18.133 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:38:18.133 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:38:18.133 | INFO | Total mem usage: 26.5 GiB | 20260506 20:38:18.133 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:38:18.133 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:38:18.133 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:38:18.133 | INFO | High Water Breakdown: | 20260506 20:38:18.133 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:38:18.133 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:38:18.133 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:38:18.133 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:38:18.133 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:38:18.133 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:38:18.133 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:38:18.133 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:38:18.133 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:38:18.133 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:38:18.133 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:38:18.133 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:38:18.133 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:38:18.133 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:38:18.133 | INFO | ------------------------------------------ | 20260506 20:38:18.134 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:38:18.134 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:38:18.134 | INFO | Total mem usage: 25.6 GiB | 20260506 20:38:18.134 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:38:18.134 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:38:18.134 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:38:18.134 | INFO | High Water Breakdown: | 20260506 20:38:18.134 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:38:18.134 | INFO | ------------------------------------------ | 20260506 20:38:18.134 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:38:18.134 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:38:18.134 | INFO | Total mem usage: 15.7 GiB | 20260506 20:38:18.134 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:38:18.134 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:38:18.134 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 20:38:18.134 | INFO | High Water Breakdown: | 20260506 20:38:18.134 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:38:18.134 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:38:18.134 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:38:18.134 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:38:18.134 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:38:18.134 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:38:18.134 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:38:18.134 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:38:18.134 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:38:18.134 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:38:18.134 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:38:18.134 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:38:18.134 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:38:18.134 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:38:18.134 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:38:18.134 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:38:18.134 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:38:18.134 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:38:18.134 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:38:18.134 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:38:18.134 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:38:18.134 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:38:18.134 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:38:18.134 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:38:18.134 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:38:18.134 | INFO | ------------------------------------------ | 20260506 20:38:18.134 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:38:18.134 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:38:18.134 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:38:18.134 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:38:18.134 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.56e+01, 1.34e+01 / 2.40e+00 | 20260506 20:38:18.134 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 1.06e-02 / 6.75e-03 | 20260506 20:38:18.134 | INFO | PerfReport: 11.65%, Baseline -- 3.63e+00 / 4.43e+00, 5.53e-01 / 3.46e-01 | 20260506 20:38:18.134 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.83e-01 / 1.37e-01 | 20260506 20:38:18.134 | INFO | PerfReport: 15.98%, DME -- 3.98e+01 / 4.42e+01, 4.02e+00 / 2.58e+00 | 20260506 20:38:18.134 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.41e+00 / 2.12e+00 | 20260506 20:38:18.134 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.23e+00 / 2.79e+00, 3.80e-01 / 2.37e-01 | 20260506 20:38:18.134 | INFO | PerfReport: 3.32%, Metrics -- 1.03e+00 / 1.37e+00, 2.74e-01 / 1.60e-01 | 20260506 20:38:18.134 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.53e-01 / 3.96e-01, 5.46e-02 / 1.71e-02 | 20260506 20:38:18.134 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:38:18.134 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:38:18.134 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:38:18.134 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 20:38:18.134 | INFO | PerfReport: 21.64%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 20:38:18.134 | INFO | PerfReport: 21.23%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 20:38:18.134 | INFO | PerfReport: 32.19%, Compute -- 3.11e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 20:38:18.134 | INFO | PerfReport: 1.37%, LockUpload -- 1.33e+00 / 0.00e+00, 5.15e+00 / 0.00e+00 | 20260506 20:38:18.134 | INFO | PerfReport: 1.13%, LockDownload -- 1.09e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260506 20:38:18.134 | INFO | PerfReport: 22.43%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 20:38:18.134 | INFO | PerfReport: 0.02%, Misc -- 1.45e-02 / 0.00e+00, 2.55e-02 / 0.00e+00 | 20260506 20:38:18.242 | INFO | Starting a round of Baz IO | 20260506 20:38:21.371 | INFO | Populating RTMetrics report | 20260506 20:38:21.371 | INFO | RTMetrics report populated | 20260506 20:38:21.371 | INFO | Opening tmp RTMetrics json file | 20260506 20:38:21.444 | INFO | Serializing RTMetrics to json | 20260506 20:38:21.448 | INFO | Serialization done, swapping tmp file now... | 20260506 20:38:21.453 | INFO | RTMetrics done | 20260506 20:38:21.661 | INFO | Finished a round of Baz IO: Wrote 1.974 GiB in 3.418 seconds | 20260506 20:38:21.661 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.18% / 0.00% / 65.81% | 20260506 20:38:21.661 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.724371% / 1.58946e-05% / 8.34465e-05% | 20260506 20:38:21.661 | INFO | Baz IO overall: | 20260506 20:38:21.661 | INFO | BytesWritten : 2261324325875 | 20260506 20:38:21.662 | INFO | NumEvents : 1092386124703 | 20260506 20:38:23.268 | INFO | Analyzing chunk frames = [4284416,4284928) | 20260506 20:38:23.269 | INFO | MemInfo: { | 20260506 20:38:23.269 | INFO | "gpu" : " 17 GB", | 20260506 20:38:23.269 | INFO | "huge" : "", | 20260506 20:38:23.269 | INFO | "huge_shared" : " 27 GB", | 20260506 20:38:23.269 | INFO | "malloc" : " 14 GB", | 20260506 20:38:23.269 | INFO | "pinned" : " 29 GB" | 20260506 20:38:23.269 | INFO | } | 20260506 20:38:23.269 | INFO | Perf report for the last 5.13596 seconds: | 20260506 20:38:23.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:38:23.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:23.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:23.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:23.269 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 20:38:23.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:23.269 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 20:38:23.269 | INFO | PRE_HQ : 59.9%, 1.8, 57.2ms, 0 | 20260506 20:38:23.269 | INFO | BAZWRITER : 66.4%, 4.8, 163.2ms, 0 | 20260506 20:38:23.269 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 20:38:23.269 | INFO | Uploaded 36GiB to GPU in 2.05347s (17.5313GiB/s) | 20260506 20:38:23.269 | INFO | Downloaded 38GiB to GPU in 2.20661s (17.221GiB/s) | 20260506 20:38:28.411 | INFO | Analyzing chunk frames = [4284928,4285440) | 20260506 20:38:28.411 | INFO | MemInfo: { | 20260506 20:38:28.411 | INFO | "gpu" : " 17 GB", | 20260506 20:38:28.411 | INFO | "huge" : "", | 20260506 20:38:28.411 | INFO | "huge_shared" : " 27 GB", | 20260506 20:38:28.411 | INFO | "malloc" : " 14 GB", | 20260506 20:38:28.411 | INFO | "pinned" : " 29 GB" | 20260506 20:38:28.411 | INFO | } | 20260506 20:38:28.411 | INFO | Perf report for the last 5.14265 seconds: | 20260506 20:38:28.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:38:28.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:28.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:28.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:28.411 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:38:28.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:28.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:28.411 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 20:38:28.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:28.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:28.412 | INFO | Uploaded 36GiB to GPU in 2.01917s (17.8291GiB/s) | 20260506 20:38:28.412 | INFO | Downloaded 34GiB to GPU in 1.92736s (17.6408GiB/s) | 20260506 20:38:33.547 | INFO | Analyzing chunk frames = [4285440,4285952) | 20260506 20:38:33.547 | INFO | MemInfo: { | 20260506 20:38:33.547 | INFO | "gpu" : " 17 GB", | 20260506 20:38:33.547 | INFO | "huge" : "", | 20260506 20:38:33.547 | INFO | "huge_shared" : " 27 GB", | 20260506 20:38:33.547 | INFO | "malloc" : " 14 GB", | 20260506 20:38:33.547 | INFO | "pinned" : " 29 GB" | 20260506 20:38:33.547 | INFO | } | 20260506 20:38:33.547 | INFO | Perf report for the last 5.13595 seconds: | 20260506 20:38:33.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:38:33.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:33.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:33.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:33.547 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 20:38:33.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:33.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:33.547 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:38:33.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:33.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:33.548 | INFO | Uploaded 36GiB to GPU in 1.98216s (18.162GiB/s) | 20260506 20:38:33.548 | INFO | Downloaded 34GiB to GPU in 1.91702s (17.7359GiB/s) | 20260506 20:38:38.686 | INFO | Analyzing chunk frames = [4285952,4286464) | 20260506 20:38:38.686 | INFO | MemInfo: { | 20260506 20:38:38.686 | INFO | "gpu" : " 17 GB", | 20260506 20:38:38.686 | INFO | "huge" : "", | 20260506 20:38:38.686 | INFO | "huge_shared" : " 27 GB", | 20260506 20:38:38.686 | INFO | "malloc" : " 14 GB", | 20260506 20:38:38.686 | INFO | "pinned" : " 29 GB" | 20260506 20:38:38.686 | INFO | } | 20260506 20:38:38.686 | INFO | Perf report for the last 5.13918 seconds: | 20260506 20:38:38.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:38:38.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:38.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:38.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:38.686 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:38:38.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:38.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:38.687 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:38:38.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:38.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:38.687 | INFO | Uploaded 36GiB to GPU in 1.94126s (18.5447GiB/s) | 20260506 20:38:38.687 | INFO | Downloaded 34GiB to GPU in 1.868s (18.2013GiB/s) | 20260506 20:38:43.821 | INFO | Analyzing chunk frames = [4286464,4286976) | 20260506 20:38:43.821 | INFO | MemInfo: { | 20260506 20:38:43.821 | INFO | "gpu" : " 17 GB", | 20260506 20:38:43.821 | INFO | "huge" : "", | 20260506 20:38:43.821 | INFO | "huge_shared" : " 27 GB", | 20260506 20:38:43.821 | INFO | "malloc" : " 14 GB", | 20260506 20:38:43.821 | INFO | "pinned" : " 29 GB" | 20260506 20:38:43.822 | INFO | } | 20260506 20:38:43.822 | INFO | Perf report for the last 5.13517 seconds: | 20260506 20:38:43.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:38:43.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:43.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:43.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:43.822 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 20:38:43.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:43.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:43.822 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:38:43.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:43.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:43.822 | INFO | Uploaded 36GiB to GPU in 2.02063s (17.8162GiB/s) | 20260506 20:38:43.822 | INFO | Downloaded 34GiB to GPU in 1.91144s (17.7877GiB/s) | 20260506 20:38:48.963 | INFO | Analyzing chunk frames = [4286976,4287488) | 20260506 20:38:48.963 | INFO | MemInfo: { | 20260506 20:38:48.963 | INFO | "gpu" : " 17 GB", | 20260506 20:38:48.963 | INFO | "huge" : "", | 20260506 20:38:48.963 | INFO | "huge_shared" : " 27 GB", | 20260506 20:38:48.963 | INFO | "malloc" : " 14 GB", | 20260506 20:38:48.963 | INFO | "pinned" : " 29 GB" | 20260506 20:38:48.963 | INFO | } | 20260506 20:38:48.963 | INFO | Perf report for the last 5.1417 seconds: | 20260506 20:38:48.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:38:48.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:48.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:48.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:48.963 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:38:48.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:48.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:48.963 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 20:38:48.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:48.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:48.964 | INFO | Uploaded 36GiB to GPU in 2.03359s (17.7027GiB/s) | 20260506 20:38:48.964 | INFO | Downloaded 34GiB to GPU in 1.9491s (17.4439GiB/s) | 20260506 20:38:54.110 | INFO | Analyzing chunk frames = [4287488,4288000) | 20260506 20:38:54.110 | INFO | MemInfo: { | 20260506 20:38:54.110 | INFO | "gpu" : " 17 GB", | 20260506 20:38:54.110 | INFO | "huge" : "", | 20260506 20:38:54.110 | INFO | "huge_shared" : " 27 GB", | 20260506 20:38:54.110 | INFO | "malloc" : " 14 GB", | 20260506 20:38:54.110 | INFO | "pinned" : " 29 GB" | 20260506 20:38:54.110 | INFO | } | 20260506 20:38:54.110 | INFO | Perf report for the last 5.14715 seconds: | 20260506 20:38:54.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:38:54.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:54.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:54.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:54.111 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260506 20:38:54.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:54.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:54.111 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:38:54.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:54.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:54.111 | INFO | Uploaded 36GiB to GPU in 1.99945s (18.0049GiB/s) | 20260506 20:38:54.111 | INFO | Downloaded 34GiB to GPU in 2.00634s (16.9463GiB/s) | 20260506 20:38:59.250 | INFO | Analyzing chunk frames = [4288000,4288512) | 20260506 20:38:59.250 | INFO | MemInfo: { | 20260506 20:38:59.250 | INFO | "gpu" : " 17 GB", | 20260506 20:38:59.250 | INFO | "huge" : "", | 20260506 20:38:59.250 | INFO | "huge_shared" : " 27 GB", | 20260506 20:38:59.250 | INFO | "malloc" : " 14 GB", | 20260506 20:38:59.250 | INFO | "pinned" : " 29 GB" | 20260506 20:38:59.250 | INFO | } | 20260506 20:38:59.250 | INFO | Perf report for the last 5.13965 seconds: | 20260506 20:38:59.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:38:59.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:59.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:59.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:59.250 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:38:59.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:59.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:59.250 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:38:59.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:59.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:38:59.250 | INFO | Uploaded 36GiB to GPU in 1.99436s (18.0509GiB/s) | 20260506 20:38:59.250 | INFO | Downloaded 34GiB to GPU in 1.98091s (17.1638GiB/s) | 20260506 20:38:59.349 | INFO | Starting a round of Baz IO | 20260506 20:39:02.493 | INFO | Populating RTMetrics report | 20260506 20:39:02.493 | INFO | RTMetrics report populated | 20260506 20:39:02.493 | INFO | Opening tmp RTMetrics json file | 20260506 20:39:02.577 | INFO | Serializing RTMetrics to json | 20260506 20:39:02.707 | INFO | Serialization done, swapping tmp file now... | 20260506 20:39:02.734 | INFO | RTMetrics done | 20260506 20:39:02.906 | INFO | Finished a round of Baz IO: Wrote 1.973 GiB in 3.556 seconds | 20260506 20:39:02.906 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.18% / 0.00% / 65.82% | 20260506 20:39:02.906 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.720747% / 3.97364e-06% / 8.74201e-05% | 20260506 20:39:02.906 | INFO | Baz IO overall: | 20260506 20:39:02.906 | INFO | BytesWritten : 2263442519760 | 20260506 20:39:02.906 | INFO | NumEvents : 1093402240085 | 20260506 20:39:04.388 | INFO | Analyzing chunk frames = [4288512,4289024) | 20260506 20:39:04.388 | INFO | MemInfo: { | 20260506 20:39:04.388 | INFO | "gpu" : " 17 GB", | 20260506 20:39:04.388 | INFO | "huge" : "", | 20260506 20:39:04.388 | INFO | "huge_shared" : " 27 GB", | 20260506 20:39:04.388 | INFO | "malloc" : " 14 GB", | 20260506 20:39:04.388 | INFO | "pinned" : " 29 GB" | 20260506 20:39:04.388 | INFO | } | 20260506 20:39:04.389 | INFO | Perf report for the last 5.13839 seconds: | 20260506 20:39:04.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:39:04.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:04.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:04.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:04.389 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 20:39:04.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:04.389 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260506 20:39:04.389 | INFO | PRE_HQ : 60.1%, 1.8, 57.3ms, 0 | 20260506 20:39:04.389 | INFO | BAZWRITER : 68.9%, 4.7, 168.0ms, 0 | 20260506 20:39:04.389 | INFO | BAZ_SAMPLING : 4.0%, 1.0, 2.1ms, 0 | 20260506 20:39:04.389 | INFO | Uploaded 36GiB to GPU in 2.07241s (17.3711GiB/s) | 20260506 20:39:04.389 | INFO | Downloaded 38GiB to GPU in 2.20318s (17.2478GiB/s) | 20260506 20:39:09.524 | INFO | Analyzing chunk frames = [4289024,4289536) | 20260506 20:39:09.524 | INFO | MemInfo: { | 20260506 20:39:09.524 | INFO | "gpu" : " 17 GB", | 20260506 20:39:09.524 | INFO | "huge" : "", | 20260506 20:39:09.524 | INFO | "huge_shared" : " 27 GB", | 20260506 20:39:09.524 | INFO | "malloc" : " 14 GB", | 20260506 20:39:09.524 | INFO | "pinned" : " 29 GB" | 20260506 20:39:09.524 | INFO | } | 20260506 20:39:09.524 | INFO | Perf report for the last 5.13588 seconds: | 20260506 20:39:09.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:39:09.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:09.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:09.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:09.524 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:39:09.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:09.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:09.525 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 20:39:09.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:09.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:09.525 | INFO | Uploaded 36GiB to GPU in 2.00481s (17.9568GiB/s) | 20260506 20:39:09.525 | INFO | Downloaded 34GiB to GPU in 1.92686s (17.6453GiB/s) | 20260506 20:39:14.667 | INFO | Analyzing chunk frames = [4289536,4290048) | 20260506 20:39:14.667 | INFO | MemInfo: { | 20260506 20:39:14.667 | INFO | "gpu" : " 17 GB", | 20260506 20:39:14.667 | INFO | "huge" : "", | 20260506 20:39:14.667 | INFO | "huge_shared" : " 27 GB", | 20260506 20:39:14.667 | INFO | "malloc" : " 14 GB", | 20260506 20:39:14.667 | INFO | "pinned" : " 29 GB" | 20260506 20:39:14.667 | INFO | } | 20260506 20:39:14.667 | INFO | Perf report for the last 5.14245 seconds: | 20260506 20:39:14.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:39:14.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:14.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:14.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:14.667 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 20:39:14.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:14.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:14.667 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:39:14.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:14.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:14.668 | INFO | Uploaded 36GiB to GPU in 1.98069s (18.1754GiB/s) | 20260506 20:39:14.668 | INFO | Downloaded 34GiB to GPU in 1.92064s (17.7024GiB/s) | 20260506 20:39:19.800 | INFO | Analyzing chunk frames = [4290048,4290560) | 20260506 20:39:19.800 | INFO | MemInfo: { | 20260506 20:39:19.800 | INFO | "gpu" : " 17 GB", | 20260506 20:39:19.800 | INFO | "huge" : "", | 20260506 20:39:19.800 | INFO | "huge_shared" : " 27 GB", | 20260506 20:39:19.800 | INFO | "malloc" : " 14 GB", | 20260506 20:39:19.800 | INFO | "pinned" : " 29 GB" | 20260506 20:39:19.800 | INFO | } | 20260506 20:39:19.800 | INFO | Perf report for the last 5.13341 seconds: | 20260506 20:39:19.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:39:19.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:19.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:19.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:19.800 | INFO | ANALYSIS : 59.7%, 3.0, 92.4ms, 0 | 20260506 20:39:19.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:19.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:19.800 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 20:39:19.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:19.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:19.801 | INFO | Uploaded 36GiB to GPU in 1.94415s (18.5171GiB/s) | 20260506 20:39:19.801 | INFO | Downloaded 34GiB to GPU in 1.87538s (18.1297GiB/s) | 20260506 20:39:24.936 | INFO | Analyzing chunk frames = [4290560,4291072) | 20260506 20:39:24.936 | INFO | MemInfo: { | 20260506 20:39:24.936 | INFO | "gpu" : " 17 GB", | 20260506 20:39:24.936 | INFO | "huge" : "", | 20260506 20:39:24.936 | INFO | "huge_shared" : " 27 GB", | 20260506 20:39:24.936 | INFO | "malloc" : " 14 GB", | 20260506 20:39:24.936 | INFO | "pinned" : " 29 GB" | 20260506 20:39:24.936 | INFO | } | 20260506 20:39:24.937 | INFO | Perf report for the last 5.13619 seconds: | 20260506 20:39:24.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:39:24.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:24.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:24.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:24.937 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 20:39:24.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:24.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:24.937 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 20:39:24.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:24.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:24.937 | INFO | Uploaded 36GiB to GPU in 2.01824s (17.8373GiB/s) | 20260506 20:39:24.937 | INFO | Downloaded 34GiB to GPU in 1.91364s (17.7671GiB/s) | 20260506 20:39:30.083 | INFO | Analyzing chunk frames = [4291072,4291584) | 20260506 20:39:30.083 | INFO | MemInfo: { | 20260506 20:39:30.083 | INFO | "gpu" : " 17 GB", | 20260506 20:39:30.083 | INFO | "huge" : "", | 20260506 20:39:30.083 | INFO | "huge_shared" : " 27 GB", | 20260506 20:39:30.083 | INFO | "malloc" : " 14 GB", | 20260506 20:39:30.083 | INFO | "pinned" : " 29 GB" | 20260506 20:39:30.083 | INFO | } | 20260506 20:39:30.083 | INFO | Perf report for the last 5.1464 seconds: | 20260506 20:39:30.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:39:30.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:30.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:30.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:30.083 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 20:39:30.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:30.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:30.083 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 20:39:30.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:30.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:30.083 | INFO | Uploaded 36GiB to GPU in 2.03465s (17.6934GiB/s) | 20260506 20:39:30.083 | INFO | Downloaded 34GiB to GPU in 1.93574s (17.5643GiB/s) | 20260506 20:39:35.225 | INFO | Analyzing chunk frames = [4291584,4292096) | 20260506 20:39:35.225 | INFO | MemInfo: { | 20260506 20:39:35.225 | INFO | "gpu" : " 17 GB", | 20260506 20:39:35.225 | INFO | "huge" : "", | 20260506 20:39:35.225 | INFO | "huge_shared" : " 27 GB", | 20260506 20:39:35.225 | INFO | "malloc" : " 14 GB", | 20260506 20:39:35.225 | INFO | "pinned" : " 29 GB" | 20260506 20:39:35.225 | INFO | } | 20260506 20:39:35.225 | INFO | Perf report for the last 5.14188 seconds: | 20260506 20:39:35.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:39:35.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:35.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:35.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:35.225 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 20:39:35.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:35.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:35.225 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 20:39:35.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:35.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:35.225 | INFO | Uploaded 36GiB to GPU in 1.98011s (18.1808GiB/s) | 20260506 20:39:35.225 | INFO | Downloaded 34GiB to GPU in 1.99362s (17.0544GiB/s) | 20260506 20:39:40.364 | INFO | Analyzing chunk frames = [4292096,4292608) | 20260506 20:39:40.364 | INFO | MemInfo: { | 20260506 20:39:40.364 | INFO | "gpu" : " 17 GB", | 20260506 20:39:40.364 | INFO | "huge" : "", | 20260506 20:39:40.364 | INFO | "huge_shared" : " 27 GB", | 20260506 20:39:40.364 | INFO | "malloc" : " 14 GB", | 20260506 20:39:40.364 | INFO | "pinned" : " 29 GB" | 20260506 20:39:40.364 | INFO | } | 20260506 20:39:40.365 | INFO | Perf report for the last 5.13972 seconds: | 20260506 20:39:40.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:39:40.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:40.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:40.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:40.365 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:39:40.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:40.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:40.365 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 20:39:40.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:40.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:40.365 | INFO | Uploaded 36GiB to GPU in 1.99172s (18.0748GiB/s) | 20260506 20:39:40.365 | INFO | Downloaded 34GiB to GPU in 1.98087s (17.1642GiB/s) | 20260506 20:39:40.365 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:39:40.365 | INFO | Total mem usage: 18 GiB | 20260506 20:39:40.365 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:39:40.365 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -14.1%) | 20260506 20:39:40.365 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:39:40.365 | INFO | High Water Breakdown: | 20260506 20:39:40.365 | INFO | 61.6% : 9.73 GiB ( 85.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:39:40.365 | INFO | 38.2% : 6.03 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:39:40.365 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:39:40.365 | INFO | ------------------------------------------ | 20260506 20:39:40.365 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:39:40.365 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:39:40.365 | INFO | Total mem usage: 26.5 GiB | 20260506 20:39:40.365 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:39:40.365 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:39:40.365 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:39:40.365 | INFO | High Water Breakdown: | 20260506 20:39:40.365 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:39:40.365 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:39:40.365 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:39:40.365 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:39:40.365 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:39:40.365 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:39:40.365 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:39:40.365 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:39:40.365 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:39:40.365 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:39:40.365 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:39:40.365 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:39:40.365 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:39:40.365 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:39:40.365 | INFO | ------------------------------------------ | 20260506 20:39:40.365 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:39:40.365 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:39:40.365 | INFO | Total mem usage: 25.6 GiB | 20260506 20:39:40.365 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:39:40.365 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:39:40.365 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:39:40.365 | INFO | High Water Breakdown: | 20260506 20:39:40.365 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:39:40.365 | INFO | ------------------------------------------ | 20260506 20:39:40.365 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:39:40.365 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:39:40.365 | INFO | Total mem usage: 15.7 GiB | 20260506 20:39:40.365 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:39:40.365 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:39:40.365 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:39:40.365 | INFO | High Water Breakdown: | 20260506 20:39:40.365 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:39:40.365 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:39:40.365 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:39:40.365 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:39:40.365 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:39:40.365 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:39:40.365 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:39:40.365 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:39:40.366 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:39:40.366 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:39:40.366 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:39:40.366 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:39:40.366 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:39:40.366 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:39:40.366 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:39:40.366 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:39:40.366 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:39:40.366 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:39:40.366 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:39:40.366 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:39:40.366 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:39:40.366 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:39:40.366 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:39:40.366 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:39:40.366 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:39:40.366 | INFO | ------------------------------------------ | 20260506 20:39:40.366 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:39:40.366 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:39:40.366 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:39:40.366 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:39:40.366 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.60e+01, 1.35e+01 / 2.72e+00 | 20260506 20:39:40.366 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 9.99e-03 / 7.07e-03 | 20260506 20:39:40.366 | INFO | PerfReport: 11.62%, Baseline -- 3.62e+00 / 4.41e+00, 5.49e-01 / 3.45e-01 | 20260506 20:39:40.366 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.48e+00, 1.82e-01 / 1.35e-01 | 20260506 20:39:40.366 | INFO | PerfReport: 16.03%, DME -- 4.00e+01 / 4.48e+01, 4.22e+00 / 2.49e+00 | 20260506 20:39:40.366 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.43e+00 / 1.84e+00 | 20260506 20:39:40.366 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.22e+00 / 2.82e+00, 3.81e-01 / 1.88e-01 | 20260506 20:39:40.366 | INFO | PerfReport: 3.31%, Metrics -- 1.03e+00 / 1.37e+00, 2.72e-01 / 1.65e-01 | 20260506 20:39:40.366 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.53e-01 / 3.98e-01, 5.55e-02 / 2.14e-02 | 20260506 20:39:40.366 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:39:40.366 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:39:40.366 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:39:40.366 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 20:39:40.366 | INFO | PerfReport: 21.56%, Upload -- 2.09e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260506 20:39:40.366 | INFO | PerfReport: 21.19%, Download -- 2.05e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 20:39:40.366 | INFO | PerfReport: 32.19%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 20:39:40.366 | INFO | PerfReport: 1.41%, LockUpload -- 1.37e+00 / 0.00e+00, 5.18e+00 / 0.00e+00 | 20260506 20:39:40.366 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260506 20:39:40.366 | INFO | PerfReport: 22.53%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 20:39:40.366 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260506 20:39:40.464 | INFO | Starting a round of Baz IO | 20260506 20:39:43.606 | INFO | Populating RTMetrics report | 20260506 20:39:43.606 | INFO | RTMetrics report populated | 20260506 20:39:43.606 | INFO | Opening tmp RTMetrics json file | 20260506 20:39:43.614 | INFO | Serializing RTMetrics to json | 20260506 20:39:43.616 | INFO | Serialization done, swapping tmp file now... | 20260506 20:39:43.617 | INFO | RTMetrics done | 20260506 20:39:44.053 | INFO | Finished a round of Baz IO: Wrote 1.971 GiB in 3.589 seconds | 20260506 20:39:44.053 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.18% / 0.00% / 65.82% | 20260506 20:39:44.053 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.718538% / 0% / 5.16574e-05% | 20260506 20:39:44.053 | INFO | Baz IO overall: | 20260506 20:39:44.053 | INFO | BytesWritten : 2265558397417 | 20260506 20:39:44.053 | INFO | NumEvents : 1094416715302 | 20260506 20:39:45.503 | INFO | Analyzing chunk frames = [4292608,4293120) | 20260506 20:39:45.503 | INFO | MemInfo: { | 20260506 20:39:45.503 | INFO | "gpu" : " 17 GB", | 20260506 20:39:45.503 | INFO | "huge" : "", | 20260506 20:39:45.503 | INFO | "huge_shared" : " 27 GB", | 20260506 20:39:45.503 | INFO | "malloc" : " 14 GB", | 20260506 20:39:45.503 | INFO | "pinned" : " 29 GB" | 20260506 20:39:45.503 | INFO | } | 20260506 20:39:45.503 | INFO | Perf report for the last 5.13872 seconds: | 20260506 20:39:45.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:39:45.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:45.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:45.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:45.503 | INFO | ANALYSIS : 62.5%, 2.9, 95.5ms, 0 | 20260506 20:39:45.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:45.503 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 20:39:45.503 | INFO | PRE_HQ : 60.1%, 1.8, 57.2ms, 0 | 20260506 20:39:45.503 | INFO | BAZWRITER : 69.5%, 4.8, 171.7ms, 0 | 20260506 20:39:45.503 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 20:39:45.504 | INFO | Uploaded 36GiB to GPU in 2.05615s (17.5084GiB/s) | 20260506 20:39:45.504 | INFO | Downloaded 38GiB to GPU in 2.19705s (17.2959GiB/s) | 20260506 20:39:50.638 | INFO | Analyzing chunk frames = [4293120,4293632) | 20260506 20:39:50.638 | INFO | MemInfo: { | 20260506 20:39:50.638 | INFO | "gpu" : " 17 GB", | 20260506 20:39:50.639 | INFO | "huge" : "", | 20260506 20:39:50.639 | INFO | "huge_shared" : " 27 GB", | 20260506 20:39:50.639 | INFO | "malloc" : " 14 GB", | 20260506 20:39:50.639 | INFO | "pinned" : " 29 GB" | 20260506 20:39:50.639 | INFO | } | 20260506 20:39:50.639 | INFO | Perf report for the last 5.13544 seconds: | 20260506 20:39:50.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:39:50.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:50.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:50.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:50.639 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 20:39:50.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:50.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:50.639 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:39:50.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:50.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:50.639 | INFO | Uploaded 36GiB to GPU in 2.01685s (17.8496GiB/s) | 20260506 20:39:50.639 | INFO | Downloaded 34GiB to GPU in 1.9239s (17.6724GiB/s) | 20260506 20:39:55.773 | INFO | Analyzing chunk frames = [4293632,4294144) | 20260506 20:39:55.773 | INFO | MemInfo: { | 20260506 20:39:55.773 | INFO | "gpu" : " 17 GB", | 20260506 20:39:55.774 | INFO | "huge" : "", | 20260506 20:39:55.774 | INFO | "huge_shared" : " 27 GB", | 20260506 20:39:55.774 | INFO | "malloc" : " 14 GB", | 20260506 20:39:55.774 | INFO | "pinned" : " 29 GB" | 20260506 20:39:55.774 | INFO | } | 20260506 20:39:55.774 | INFO | Perf report for the last 5.13494 seconds: | 20260506 20:39:55.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:39:55.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:55.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:55.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:55.774 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 20:39:55.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:55.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:55.774 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:39:55.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:55.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:39:55.774 | INFO | Uploaded 36GiB to GPU in 1.96996s (18.2745GiB/s) | 20260506 20:39:55.774 | INFO | Downloaded 34GiB to GPU in 1.91771s (17.7295GiB/s) | 20260506 20:40:00.915 | INFO | Analyzing chunk frames = [4294144,4294656) | 20260506 20:40:00.915 | INFO | MemInfo: { | 20260506 20:40:00.915 | INFO | "gpu" : " 17 GB", | 20260506 20:40:00.915 | INFO | "huge" : "", | 20260506 20:40:00.915 | INFO | "huge_shared" : " 27 GB", | 20260506 20:40:00.915 | INFO | "malloc" : " 14 GB", | 20260506 20:40:00.915 | INFO | "pinned" : " 29 GB" | 20260506 20:40:00.915 | INFO | } | 20260506 20:40:00.915 | INFO | Perf report for the last 5.14134 seconds: | 20260506 20:40:00.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:40:00.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:00.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:00.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:00.915 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 20:40:00.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:00.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:00.915 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 20:40:00.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:00.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:00.915 | INFO | Uploaded 36GiB to GPU in 1.93187s (18.6348GiB/s) | 20260506 20:40:00.915 | INFO | Downloaded 34GiB to GPU in 1.8701s (18.1809GiB/s) | 20260506 20:40:06.056 | INFO | Analyzing chunk frames = [4294656,4295168) | 20260506 20:40:06.056 | INFO | MemInfo: { | 20260506 20:40:06.056 | INFO | "gpu" : " 17 GB", | 20260506 20:40:06.056 | INFO | "huge" : "", | 20260506 20:40:06.056 | INFO | "huge_shared" : " 27 GB", | 20260506 20:40:06.056 | INFO | "malloc" : " 14 GB", | 20260506 20:40:06.056 | INFO | "pinned" : " 29 GB" | 20260506 20:40:06.056 | INFO | } | 20260506 20:40:06.056 | INFO | Perf report for the last 5.14145 seconds: | 20260506 20:40:06.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:40:06.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:06.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:06.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:06.057 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 20:40:06.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:06.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:06.057 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 20:40:06.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:06.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:06.057 | INFO | Uploaded 36GiB to GPU in 2.01071s (17.9041GiB/s) | 20260506 20:40:06.057 | INFO | Downloaded 34GiB to GPU in 1.91096s (17.7921GiB/s) | 20260506 20:40:11.197 | INFO | Analyzing chunk frames = [4295168,4295680) | 20260506 20:40:11.197 | INFO | MemInfo: { | 20260506 20:40:11.197 | INFO | "gpu" : " 17 GB", | 20260506 20:40:11.197 | INFO | "huge" : "", | 20260506 20:40:11.197 | INFO | "huge_shared" : " 27 GB", | 20260506 20:40:11.197 | INFO | "malloc" : " 14 GB", | 20260506 20:40:11.197 | INFO | "pinned" : " 29 GB" | 20260506 20:40:11.197 | INFO | } | 20260506 20:40:11.197 | INFO | Perf report for the last 5.14069 seconds: | 20260506 20:40:11.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:40:11.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:11.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:11.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:11.197 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 20:40:11.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:11.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:11.197 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:40:11.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:11.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:11.197 | INFO | Uploaded 36GiB to GPU in 2.00253s (17.9773GiB/s) | 20260506 20:40:11.198 | INFO | Downloaded 34GiB to GPU in 1.95585s (17.3837GiB/s) | 20260506 20:40:16.338 | INFO | Analyzing chunk frames = [4295680,4296192) | 20260506 20:40:16.338 | INFO | MemInfo: { | 20260506 20:40:16.338 | INFO | "gpu" : " 17 GB", | 20260506 20:40:16.338 | INFO | "huge" : "", | 20260506 20:40:16.338 | INFO | "huge_shared" : " 27 GB", | 20260506 20:40:16.338 | INFO | "malloc" : " 14 GB", | 20260506 20:40:16.338 | INFO | "pinned" : " 29 GB" | 20260506 20:40:16.338 | INFO | } | 20260506 20:40:16.338 | INFO | Perf report for the last 5.14119 seconds: | 20260506 20:40:16.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:40:16.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:16.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:16.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:16.338 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:40:16.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:16.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:16.339 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:40:16.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:16.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:16.339 | INFO | Uploaded 36GiB to GPU in 1.96236s (18.3453GiB/s) | 20260506 20:40:16.339 | INFO | Downloaded 34GiB to GPU in 1.99211s (17.0674GiB/s) | 20260506 20:40:21.480 | INFO | Analyzing chunk frames = [4296192,4296704) | 20260506 20:40:21.480 | INFO | MemInfo: { | 20260506 20:40:21.480 | INFO | "gpu" : " 17 GB", | 20260506 20:40:21.480 | INFO | "huge" : "", | 20260506 20:40:21.480 | INFO | "huge_shared" : " 27 GB", | 20260506 20:40:21.480 | INFO | "malloc" : " 14 GB", | 20260506 20:40:21.480 | INFO | "pinned" : " 29 GB" | 20260506 20:40:21.480 | INFO | } | 20260506 20:40:21.480 | INFO | Perf report for the last 5.14175 seconds: | 20260506 20:40:21.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:40:21.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:21.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:21.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:21.480 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 20:40:21.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:21.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:21.480 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 20:40:21.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:21.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:21.480 | INFO | Uploaded 36GiB to GPU in 1.97976s (18.184GiB/s) | 20260506 20:40:21.481 | INFO | Downloaded 34GiB to GPU in 1.97303s (17.2324GiB/s) | 20260506 20:40:21.579 | INFO | Starting a round of Baz IO | 20260506 20:40:24.717 | INFO | Populating RTMetrics report | 20260506 20:40:24.717 | INFO | RTMetrics report populated | 20260506 20:40:24.717 | INFO | Opening tmp RTMetrics json file | 20260506 20:40:24.765 | INFO | Serializing RTMetrics to json | 20260506 20:40:24.810 | INFO | Serialization done, swapping tmp file now... | 20260506 20:40:24.811 | INFO | RTMetrics done | 20260506 20:40:24.918 | INFO | Finished a round of Baz IO: Wrote 1.972 GiB in 3.339 seconds | 20260506 20:40:24.918 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.17% / 0.00% / 65.82% | 20260506 20:40:24.918 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.718486% / 0% / 5.96046e-05% | 20260506 20:40:24.919 | INFO | Baz IO overall: | 20260506 20:40:24.919 | INFO | BytesWritten : 2267675954068 | 20260506 20:40:24.919 | INFO | NumEvents : 1095432354723 | 20260506 20:40:26.615 | INFO | Analyzing chunk frames = [4296704,4297216) | 20260506 20:40:26.615 | INFO | MemInfo: { | 20260506 20:40:26.615 | INFO | "gpu" : " 17 GB", | 20260506 20:40:26.615 | INFO | "huge" : "", | 20260506 20:40:26.615 | INFO | "huge_shared" : " 27 GB", | 20260506 20:40:26.615 | INFO | "malloc" : " 14 GB", | 20260506 20:40:26.615 | INFO | "pinned" : " 29 GB" | 20260506 20:40:26.615 | INFO | } | 20260506 20:40:26.615 | INFO | Perf report for the last 5.13458 seconds: | 20260506 20:40:26.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:40:26.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:26.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:26.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:26.615 | INFO | ANALYSIS : 62.5%, 2.9, 95.2ms, 0 | 20260506 20:40:26.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:26.615 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 20:40:26.615 | INFO | PRE_HQ : 59.9%, 1.8, 57.1ms, 0 | 20260506 20:40:26.615 | INFO | BAZWRITER : 64.6%, 4.7, 156.8ms, 0 | 20260506 20:40:26.615 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 20:40:26.615 | INFO | Uploaded 36GiB to GPU in 2.04112s (17.6374GiB/s) | 20260506 20:40:26.615 | INFO | Downloaded 38GiB to GPU in 2.20075s (17.2668GiB/s) | 20260506 20:40:31.754 | INFO | Analyzing chunk frames = [4297216,4297728) | 20260506 20:40:31.754 | INFO | MemInfo: { | 20260506 20:40:31.754 | INFO | "gpu" : " 17 GB", | 20260506 20:40:31.754 | INFO | "huge" : "", | 20260506 20:40:31.755 | INFO | "huge_shared" : " 27 GB", | 20260506 20:40:31.755 | INFO | "malloc" : " 14 GB", | 20260506 20:40:31.755 | INFO | "pinned" : " 29 GB" | 20260506 20:40:31.755 | INFO | } | 20260506 20:40:31.755 | INFO | Perf report for the last 5.13988 seconds: | 20260506 20:40:31.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:40:31.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:31.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:31.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:31.755 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:40:31.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:31.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:31.755 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:40:31.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:31.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:31.755 | INFO | Uploaded 36GiB to GPU in 2.01673s (17.8507GiB/s) | 20260506 20:40:31.755 | INFO | Downloaded 34GiB to GPU in 1.92471s (17.665GiB/s) | 20260506 20:40:36.891 | INFO | Analyzing chunk frames = [4297728,4298240) | 20260506 20:40:36.891 | INFO | MemInfo: { | 20260506 20:40:36.891 | INFO | "gpu" : " 17 GB", | 20260506 20:40:36.892 | INFO | "huge" : "", | 20260506 20:40:36.892 | INFO | "huge_shared" : " 27 GB", | 20260506 20:40:36.892 | INFO | "malloc" : " 14 GB", | 20260506 20:40:36.892 | INFO | "pinned" : " 29 GB" | 20260506 20:40:36.892 | INFO | } | 20260506 20:40:36.892 | INFO | Perf report for the last 5.13702 seconds: | 20260506 20:40:36.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:40:36.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:36.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:36.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:36.892 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260506 20:40:36.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:36.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:36.892 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:40:36.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:36.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:36.892 | INFO | Uploaded 36GiB to GPU in 1.98288s (18.1554GiB/s) | 20260506 20:40:36.892 | INFO | Downloaded 34GiB to GPU in 1.9198s (17.7102GiB/s) | 20260506 20:40:42.030 | INFO | Analyzing chunk frames = [4298240,4298752) | 20260506 20:40:42.030 | INFO | MemInfo: { | 20260506 20:40:42.030 | INFO | "gpu" : " 17 GB", | 20260506 20:40:42.030 | INFO | "huge" : "", | 20260506 20:40:42.030 | INFO | "huge_shared" : " 27 GB", | 20260506 20:40:42.030 | INFO | "malloc" : " 14 GB", | 20260506 20:40:42.030 | INFO | "pinned" : " 29 GB" | 20260506 20:40:42.030 | INFO | } | 20260506 20:40:42.030 | INFO | Perf report for the last 5.13805 seconds: | 20260506 20:40:42.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:40:42.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:42.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:42.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:42.030 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 20:40:42.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:42.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:42.030 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:40:42.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:42.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:42.030 | INFO | Uploaded 36GiB to GPU in 1.95134s (18.4489GiB/s) | 20260506 20:40:42.030 | INFO | Downloaded 34GiB to GPU in 1.8709s (18.1731GiB/s) | 20260506 20:40:47.173 | INFO | Analyzing chunk frames = [4298752,4299264) | 20260506 20:40:47.173 | INFO | MemInfo: { | 20260506 20:40:47.173 | INFO | "gpu" : " 17 GB", | 20260506 20:40:47.173 | INFO | "huge" : "", | 20260506 20:40:47.173 | INFO | "huge_shared" : " 27 GB", | 20260506 20:40:47.173 | INFO | "malloc" : " 14 GB", | 20260506 20:40:47.173 | INFO | "pinned" : " 29 GB" | 20260506 20:40:47.173 | INFO | } | 20260506 20:40:47.173 | INFO | Perf report for the last 5.14367 seconds: | 20260506 20:40:47.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:40:47.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:47.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:47.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:47.173 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 20:40:47.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:47.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:47.174 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:40:47.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:47.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:47.174 | INFO | Uploaded 36GiB to GPU in 2.02059s (17.8166GiB/s) | 20260506 20:40:47.174 | INFO | Downloaded 34GiB to GPU in 1.91243s (17.7784GiB/s) | 20260506 20:40:52.313 | INFO | Analyzing chunk frames = [4299264,4299776) | 20260506 20:40:52.313 | INFO | MemInfo: { | 20260506 20:40:52.313 | INFO | "gpu" : " 17 GB", | 20260506 20:40:52.313 | INFO | "huge" : "", | 20260506 20:40:52.313 | INFO | "huge_shared" : " 27 GB", | 20260506 20:40:52.313 | INFO | "malloc" : " 14 GB", | 20260506 20:40:52.313 | INFO | "pinned" : " 29 GB" | 20260506 20:40:52.313 | INFO | } | 20260506 20:40:52.313 | INFO | Perf report for the last 5.14002 seconds: | 20260506 20:40:52.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:40:52.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:52.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:52.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:52.314 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 20:40:52.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:52.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:52.314 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 20:40:52.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:52.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:52.314 | INFO | Uploaded 36GiB to GPU in 2.014s (17.8749GiB/s) | 20260506 20:40:52.314 | INFO | Downloaded 34GiB to GPU in 1.94194s (17.5083GiB/s) | 20260506 20:40:57.452 | INFO | Analyzing chunk frames = [4299776,4300288) | 20260506 20:40:57.452 | INFO | MemInfo: { | 20260506 20:40:57.452 | INFO | "gpu" : " 17 GB", | 20260506 20:40:57.452 | INFO | "huge" : "", | 20260506 20:40:57.452 | INFO | "huge_shared" : " 27 GB", | 20260506 20:40:57.452 | INFO | "malloc" : " 14 GB", | 20260506 20:40:57.452 | INFO | "pinned" : " 29 GB" | 20260506 20:40:57.452 | INFO | } | 20260506 20:40:57.452 | INFO | Perf report for the last 5.13882 seconds: | 20260506 20:40:57.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:40:57.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:57.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:57.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:57.452 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 20:40:57.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:57.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:57.452 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:40:57.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:57.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:40:57.453 | INFO | Uploaded 36GiB to GPU in 1.96798s (18.2928GiB/s) | 20260506 20:40:57.453 | INFO | Downloaded 34GiB to GPU in 1.99318s (17.0582GiB/s) | 20260506 20:41:02.588 | INFO | Analyzing chunk frames = [4300288,4300800) | 20260506 20:41:02.588 | INFO | MemInfo: { | 20260506 20:41:02.588 | INFO | "gpu" : " 17 GB", | 20260506 20:41:02.588 | INFO | "huge" : "", | 20260506 20:41:02.588 | INFO | "huge_shared" : " 27 GB", | 20260506 20:41:02.588 | INFO | "malloc" : " 14 GB", | 20260506 20:41:02.588 | INFO | "pinned" : " 29 GB" | 20260506 20:41:02.588 | INFO | } | 20260506 20:41:02.588 | INFO | Perf report for the last 5.13576 seconds: | 20260506 20:41:02.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:41:02.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:02.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:02.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:02.588 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 20:41:02.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:02.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:02.588 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 20:41:02.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:02.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:02.588 | INFO | Uploaded 36GiB to GPU in 1.99363s (18.0575GiB/s) | 20260506 20:41:02.588 | INFO | Downloaded 34GiB to GPU in 1.9927s (17.0623GiB/s) | 20260506 20:41:02.589 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:41:02.589 | INFO | Total mem usage: 18 GiB | 20260506 20:41:02.589 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:41:02.589 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -11.7%) | 20260506 20:41:02.589 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:41:02.589 | INFO | High Water Breakdown: | 20260506 20:41:02.589 | INFO | 62% : 9.98 GiB ( 87.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:41:02.589 | INFO | 37.9% : 6.11 GiB ( 91.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:41:02.589 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:41:02.589 | INFO | ------------------------------------------ | 20260506 20:41:02.589 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:41:02.589 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:41:02.589 | INFO | Total mem usage: 26.5 GiB | 20260506 20:41:02.589 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:41:02.589 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:41:02.589 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:41:02.589 | INFO | High Water Breakdown: | 20260506 20:41:02.589 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:41:02.589 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:41:02.589 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:41:02.589 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:41:02.589 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:41:02.589 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:41:02.589 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:41:02.589 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:41:02.589 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:41:02.589 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:41:02.589 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:41:02.589 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:41:02.589 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:41:02.589 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:41:02.589 | INFO | ------------------------------------------ | 20260506 20:41:02.589 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:41:02.589 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:41:02.589 | INFO | Total mem usage: 25.6 GiB | 20260506 20:41:02.589 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:41:02.589 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 20:41:02.589 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 20:41:02.589 | INFO | High Water Breakdown: | 20260506 20:41:02.589 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 20:41:02.589 | INFO | ------------------------------------------ | 20260506 20:41:02.589 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:41:02.589 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:41:02.589 | INFO | Total mem usage: 15.7 GiB | 20260506 20:41:02.589 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:41:02.589 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:41:02.589 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 20:41:02.589 | INFO | High Water Breakdown: | 20260506 20:41:02.589 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:41:02.589 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:41:02.589 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:41:02.589 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:41:02.589 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:41:02.589 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:41:02.589 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:41:02.589 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:41:02.589 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:41:02.589 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:41:02.589 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:41:02.589 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:41:02.589 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:41:02.589 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:41:02.589 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:41:02.589 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:41:02.589 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:41:02.589 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:41:02.589 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:41:02.589 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:41:02.589 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:41:02.589 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:41:02.589 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:41:02.589 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:41:02.589 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:41:02.589 | INFO | ------------------------------------------ | 20260506 20:41:02.589 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:41:02.589 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:41:02.589 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:41:02.589 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:41:02.589 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.34e+01 / 2.71e+00 | 20260506 20:41:02.589 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.03e-01, 9.65e-03 / 6.13e-03 | 20260506 20:41:02.589 | INFO | PerfReport: 11.65%, Baseline -- 3.63e+00 / 4.40e+00, 5.61e-01 / 3.81e-01 | 20260506 20:41:02.589 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.83e-01 / 1.30e-01 | 20260506 20:41:02.590 | INFO | PerfReport: 16.02%, DME -- 3.99e+01 / 4.46e+01, 4.18e+00 / 2.97e+00 | 20260506 20:41:02.590 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.44e+00 / 2.34e+00 | 20260506 20:41:02.590 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.22e+00 / 2.76e+00, 3.75e-01 / 2.38e-01 | 20260506 20:41:02.590 | INFO | PerfReport: 3.31%, Metrics -- 1.03e+00 / 1.36e+00, 2.76e-01 / 1.25e-01 | 20260506 20:41:02.590 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.53e-01 / 3.99e-01, 5.60e-02 / 1.75e-02 | 20260506 20:41:02.590 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:41:02.590 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:41:02.590 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:41:02.590 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 20:41:02.590 | INFO | PerfReport: 21.49%, Upload -- 2.08e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260506 20:41:02.590 | INFO | PerfReport: 21.20%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 20:41:02.590 | INFO | PerfReport: 32.24%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 20:41:02.590 | INFO | PerfReport: 1.30%, LockUpload -- 1.26e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260506 20:41:02.590 | INFO | PerfReport: 1.12%, LockDownload -- 1.08e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 20:41:02.590 | INFO | PerfReport: 22.64%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 20:41:02.590 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260506 20:41:02.695 | INFO | Starting a round of Baz IO | 20260506 20:41:05.833 | INFO | Populating RTMetrics report | 20260506 20:41:05.834 | INFO | RTMetrics report populated | 20260506 20:41:05.834 | INFO | Opening tmp RTMetrics json file | 20260506 20:41:05.844 | INFO | Serializing RTMetrics to json | 20260506 20:41:05.845 | INFO | Serialization done, swapping tmp file now... | 20260506 20:41:05.846 | INFO | RTMetrics done | 20260506 20:41:06.266 | INFO | Finished a round of Baz IO: Wrote 1.97 GiB in 3.571 seconds | 20260506 20:41:06.266 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.17% / 0.00% / 65.83% | 20260506 20:41:06.266 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.715756% / 1.19209e-05% / 8.34465e-05% | 20260506 20:41:06.266 | INFO | Baz IO overall: | 20260506 20:41:06.267 | INFO | BytesWritten : 2269791383001 | 20260506 20:41:06.267 | INFO | NumEvents : 1096446499893 | 20260506 20:41:07.727 | INFO | Analyzing chunk frames = [4300800,4301312) | 20260506 20:41:07.727 | INFO | MemInfo: { | 20260506 20:41:07.727 | INFO | "gpu" : " 17 GB", | 20260506 20:41:07.727 | INFO | "huge" : "", | 20260506 20:41:07.727 | INFO | "huge_shared" : " 27 GB", | 20260506 20:41:07.727 | INFO | "malloc" : " 14 GB", | 20260506 20:41:07.727 | INFO | "pinned" : " 29 GB" | 20260506 20:41:07.727 | INFO | } | 20260506 20:41:07.727 | INFO | Perf report for the last 5.13926 seconds: | 20260506 20:41:07.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:41:07.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:07.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:07.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:07.727 | INFO | ANALYSIS : 62.5%, 2.9, 95.5ms, 0 | 20260506 20:41:07.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:07.727 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 20:41:07.727 | INFO | PRE_HQ : 60.2%, 1.8, 57.1ms, 0 | 20260506 20:41:07.727 | INFO | BAZWRITER : 69.2%, 4.8, 170.0ms, 0 | 20260506 20:41:07.727 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 20:41:07.728 | INFO | Uploaded 36GiB to GPU in 2.04967s (17.5638GiB/s) | 20260506 20:41:07.728 | INFO | Downloaded 38GiB to GPU in 2.20702s (17.2178GiB/s) | 20260506 20:41:12.866 | INFO | Analyzing chunk frames = [4301312,4301824) | 20260506 20:41:12.866 | INFO | MemInfo: { | 20260506 20:41:12.866 | INFO | "gpu" : " 17 GB", | 20260506 20:41:12.866 | INFO | "huge" : "", | 20260506 20:41:12.866 | INFO | "huge_shared" : " 27 GB", | 20260506 20:41:12.866 | INFO | "malloc" : " 14 GB", | 20260506 20:41:12.866 | INFO | "pinned" : " 29 GB" | 20260506 20:41:12.866 | INFO | } | 20260506 20:41:12.866 | INFO | Perf report for the last 5.1391 seconds: | 20260506 20:41:12.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:41:12.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:12.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:12.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:12.867 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:41:12.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:12.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:12.867 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:41:12.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:12.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:12.867 | INFO | Uploaded 36GiB to GPU in 2.01618s (17.8555GiB/s) | 20260506 20:41:12.867 | INFO | Downloaded 34GiB to GPU in 1.92823s (17.6328GiB/s) | 20260506 20:41:18.006 | INFO | Analyzing chunk frames = [4301824,4302336) | 20260506 20:41:18.006 | INFO | MemInfo: { | 20260506 20:41:18.006 | INFO | "gpu" : " 17 GB", | 20260506 20:41:18.006 | INFO | "huge" : "", | 20260506 20:41:18.006 | INFO | "huge_shared" : " 27 GB", | 20260506 20:41:18.006 | INFO | "malloc" : " 14 GB", | 20260506 20:41:18.006 | INFO | "pinned" : " 29 GB" | 20260506 20:41:18.006 | INFO | } | 20260506 20:41:18.006 | INFO | Perf report for the last 5.13963 seconds: | 20260506 20:41:18.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:41:18.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:18.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:18.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:18.006 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260506 20:41:18.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:18.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:18.006 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:41:18.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:18.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:18.006 | INFO | Uploaded 36GiB to GPU in 1.98905s (18.0991GiB/s) | 20260506 20:41:18.006 | INFO | Downloaded 34GiB to GPU in 1.91743s (17.7321GiB/s) | 20260506 20:41:23.148 | INFO | Analyzing chunk frames = [4302336,4302848) | 20260506 20:41:23.148 | INFO | MemInfo: { | 20260506 20:41:23.148 | INFO | "gpu" : " 17 GB", | 20260506 20:41:23.148 | INFO | "huge" : "", | 20260506 20:41:23.148 | INFO | "huge_shared" : " 27 GB", | 20260506 20:41:23.148 | INFO | "malloc" : " 14 GB", | 20260506 20:41:23.148 | INFO | "pinned" : " 29 GB" | 20260506 20:41:23.148 | INFO | } | 20260506 20:41:23.148 | INFO | Perf report for the last 5.14233 seconds: | 20260506 20:41:23.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:41:23.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:23.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:23.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:23.148 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:41:23.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:23.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:23.148 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 20:41:23.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:23.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:23.149 | INFO | Uploaded 36GiB to GPU in 1.95082s (18.4538GiB/s) | 20260506 20:41:23.149 | INFO | Downloaded 34GiB to GPU in 1.87325s (18.1503GiB/s) | 20260506 20:41:28.289 | INFO | Analyzing chunk frames = [4302848,4303360) | 20260506 20:41:28.289 | INFO | MemInfo: { | 20260506 20:41:28.289 | INFO | "gpu" : " 17 GB", | 20260506 20:41:28.289 | INFO | "huge" : "", | 20260506 20:41:28.289 | INFO | "huge_shared" : " 27 GB", | 20260506 20:41:28.289 | INFO | "malloc" : " 14 GB", | 20260506 20:41:28.289 | INFO | "pinned" : " 29 GB" | 20260506 20:41:28.289 | INFO | } | 20260506 20:41:28.289 | INFO | Perf report for the last 5.14041 seconds: | 20260506 20:41:28.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:41:28.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:28.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:28.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:28.289 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 20:41:28.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:28.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:28.289 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:41:28.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:28.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:28.289 | INFO | Uploaded 36GiB to GPU in 2.01064s (17.9047GiB/s) | 20260506 20:41:28.289 | INFO | Downloaded 34GiB to GPU in 1.91444s (17.7598GiB/s) | 20260506 20:41:33.429 | INFO | Analyzing chunk frames = [4303360,4303872) | 20260506 20:41:33.429 | INFO | MemInfo: { | 20260506 20:41:33.429 | INFO | "gpu" : " 17 GB", | 20260506 20:41:33.429 | INFO | "huge" : "", | 20260506 20:41:33.429 | INFO | "huge_shared" : " 27 GB", | 20260506 20:41:33.429 | INFO | "malloc" : " 14 GB", | 20260506 20:41:33.429 | INFO | "pinned" : " 29 GB" | 20260506 20:41:33.429 | INFO | } | 20260506 20:41:33.430 | INFO | Perf report for the last 5.14067 seconds: | 20260506 20:41:33.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:41:33.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:33.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:33.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:33.430 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:41:33.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:33.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:33.430 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 20:41:33.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:33.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:33.430 | INFO | Uploaded 36GiB to GPU in 2.02688s (17.7613GiB/s) | 20260506 20:41:33.430 | INFO | Downloaded 34GiB to GPU in 1.94686s (17.464GiB/s) | 20260506 20:41:38.569 | INFO | Analyzing chunk frames = [4303872,4304384) | 20260506 20:41:38.569 | INFO | MemInfo: { | 20260506 20:41:38.569 | INFO | "gpu" : " 17 GB", | 20260506 20:41:38.569 | INFO | "huge" : "", | 20260506 20:41:38.569 | INFO | "huge_shared" : " 27 GB", | 20260506 20:41:38.569 | INFO | "malloc" : " 14 GB", | 20260506 20:41:38.569 | INFO | "pinned" : " 29 GB" | 20260506 20:41:38.569 | INFO | } | 20260506 20:41:38.569 | INFO | Perf report for the last 5.13993 seconds: | 20260506 20:41:38.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:41:38.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:38.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:38.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:38.570 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 20:41:38.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:38.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:38.570 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:41:38.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:38.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:38.570 | INFO | Uploaded 36GiB to GPU in 2.00328s (17.9705GiB/s) | 20260506 20:41:38.570 | INFO | Downloaded 34GiB to GPU in 2.00661s (16.944GiB/s) | 20260506 20:41:43.708 | INFO | Analyzing chunk frames = [4304384,4304896) | 20260506 20:41:43.708 | INFO | MemInfo: { | 20260506 20:41:43.708 | INFO | "gpu" : " 17 GB", | 20260506 20:41:43.708 | INFO | "huge" : "", | 20260506 20:41:43.708 | INFO | "huge_shared" : " 27 GB", | 20260506 20:41:43.708 | INFO | "malloc" : " 14 GB", | 20260506 20:41:43.708 | INFO | "pinned" : " 29 GB" | 20260506 20:41:43.708 | INFO | } | 20260506 20:41:43.708 | INFO | Perf report for the last 5.13885 seconds: | 20260506 20:41:43.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:41:43.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:43.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:43.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:43.708 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 20:41:43.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:43.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:43.709 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:41:43.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:43.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:43.709 | INFO | Uploaded 36GiB to GPU in 2.00982s (17.9121GiB/s) | 20260506 20:41:43.709 | INFO | Downloaded 34GiB to GPU in 2.00083s (16.9929GiB/s) | 20260506 20:41:43.811 | INFO | Starting a round of Baz IO | 20260506 20:41:46.946 | INFO | Populating RTMetrics report | 20260506 20:41:46.947 | INFO | RTMetrics report populated | 20260506 20:41:46.947 | INFO | Opening tmp RTMetrics json file | 20260506 20:41:46.947 | INFO | Serializing RTMetrics to json | 20260506 20:41:46.965 | INFO | Serialization done, swapping tmp file now... | 20260506 20:41:46.995 | INFO | RTMetrics done | 20260506 20:41:47.105 | INFO | Finished a round of Baz IO: Wrote 1.97 GiB in 3.294 seconds | 20260506 20:41:47.105 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.16% / 0.00% / 65.83% | 20260506 20:41:47.105 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.714302% / 0% / 7.94729e-05% | 20260506 20:41:47.105 | INFO | Baz IO overall: | 20260506 20:41:47.105 | INFO | BytesWritten : 2271906517367 | 20260506 20:41:47.105 | INFO | NumEvents : 1097460264404 | 20260506 20:41:48.844 | INFO | Analyzing chunk frames = [4304896,4305408) | 20260506 20:41:48.844 | INFO | MemInfo: { | 20260506 20:41:48.844 | INFO | "gpu" : " 17 GB", | 20260506 20:41:48.844 | INFO | "huge" : "", | 20260506 20:41:48.845 | INFO | "huge_shared" : " 27 GB", | 20260506 20:41:48.845 | INFO | "malloc" : " 14 GB", | 20260506 20:41:48.845 | INFO | "pinned" : " 29 GB" | 20260506 20:41:48.845 | INFO | } | 20260506 20:41:48.845 | INFO | Perf report for the last 5.13629 seconds: | 20260506 20:41:48.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:41:48.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:48.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:48.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:48.845 | INFO | ANALYSIS : 62.5%, 2.9, 95.7ms, 0 | 20260506 20:41:48.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:48.845 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 20:41:48.845 | INFO | PRE_HQ : 60.1%, 1.8, 57.2ms, 0 | 20260506 20:41:48.845 | INFO | BAZWRITER : 63.7%, 4.3, 141.3ms, 0 | 20260506 20:41:48.845 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.3ms, 0 | 20260506 20:41:48.845 | INFO | Uploaded 36GiB to GPU in 2.04865s (17.5726GiB/s) | 20260506 20:41:48.845 | INFO | Downloaded 38GiB to GPU in 2.19826s (17.2864GiB/s) | 20260506 20:41:53.983 | INFO | Analyzing chunk frames = [4305408,4305920) | 20260506 20:41:53.983 | INFO | MemInfo: { | 20260506 20:41:53.983 | INFO | "gpu" : " 17 GB", | 20260506 20:41:53.983 | INFO | "huge" : "", | 20260506 20:41:53.983 | INFO | "huge_shared" : " 27 GB", | 20260506 20:41:53.983 | INFO | "malloc" : " 14 GB", | 20260506 20:41:53.983 | INFO | "pinned" : " 29 GB" | 20260506 20:41:53.983 | INFO | } | 20260506 20:41:53.983 | INFO | Perf report for the last 5.13874 seconds: | 20260506 20:41:53.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:41:53.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:53.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:53.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:53.983 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:41:53.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:53.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:53.984 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:41:53.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:53.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:53.984 | INFO | Uploaded 36GiB to GPU in 1.99475s (18.0473GiB/s) | 20260506 20:41:53.984 | INFO | Downloaded 34GiB to GPU in 1.92201s (17.6898GiB/s) | 20260506 20:41:59.118 | INFO | Analyzing chunk frames = [4305920,4306432) | 20260506 20:41:59.118 | INFO | MemInfo: { | 20260506 20:41:59.118 | INFO | "gpu" : " 17 GB", | 20260506 20:41:59.118 | INFO | "huge" : "", | 20260506 20:41:59.118 | INFO | "huge_shared" : " 27 GB", | 20260506 20:41:59.118 | INFO | "malloc" : " 14 GB", | 20260506 20:41:59.118 | INFO | "pinned" : " 29 GB" | 20260506 20:41:59.118 | INFO | } | 20260506 20:41:59.118 | INFO | Perf report for the last 5.13503 seconds: | 20260506 20:41:59.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:41:59.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:59.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:59.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:59.118 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260506 20:41:59.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:59.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:59.119 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:41:59.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:59.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:41:59.119 | INFO | Uploaded 36GiB to GPU in 1.98304s (18.1539GiB/s) | 20260506 20:41:59.119 | INFO | Downloaded 34GiB to GPU in 1.91311s (17.7721GiB/s) | 20260506 20:42:04.257 | INFO | Analyzing chunk frames = [4306432,4306944) | 20260506 20:42:04.257 | INFO | MemInfo: { | 20260506 20:42:04.257 | INFO | "gpu" : " 17 GB", | 20260506 20:42:04.257 | INFO | "huge" : "", | 20260506 20:42:04.257 | INFO | "huge_shared" : " 27 GB", | 20260506 20:42:04.257 | INFO | "malloc" : " 14 GB", | 20260506 20:42:04.257 | INFO | "pinned" : " 29 GB" | 20260506 20:42:04.257 | INFO | } | 20260506 20:42:04.257 | INFO | Perf report for the last 5.13846 seconds: | 20260506 20:42:04.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:42:04.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:04.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:04.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:04.257 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 20:42:04.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:04.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:04.257 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:42:04.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:04.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:04.257 | INFO | Uploaded 36GiB to GPU in 1.93183s (18.6351GiB/s) | 20260506 20:42:04.257 | INFO | Downloaded 34GiB to GPU in 1.87135s (18.1687GiB/s) | 20260506 20:42:09.401 | INFO | Analyzing chunk frames = [4306944,4307456) | 20260506 20:42:09.401 | INFO | MemInfo: { | 20260506 20:42:09.401 | INFO | "gpu" : " 17 GB", | 20260506 20:42:09.401 | INFO | "huge" : "", | 20260506 20:42:09.401 | INFO | "huge_shared" : " 27 GB", | 20260506 20:42:09.401 | INFO | "malloc" : " 14 GB", | 20260506 20:42:09.401 | INFO | "pinned" : " 29 GB" | 20260506 20:42:09.401 | INFO | } | 20260506 20:42:09.401 | INFO | Perf report for the last 5.14447 seconds: | 20260506 20:42:09.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:42:09.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:09.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:09.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:09.401 | INFO | ANALYSIS : 59.8%, 3.0, 93.2ms, 0 | 20260506 20:42:09.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:09.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:09.402 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:42:09.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:09.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:09.402 | INFO | Uploaded 36GiB to GPU in 1.95971s (18.3701GiB/s) | 20260506 20:42:09.402 | INFO | Downloaded 34GiB to GPU in 1.88097s (18.0758GiB/s) | 20260506 20:42:14.541 | INFO | Analyzing chunk frames = [4307456,4307968) | 20260506 20:42:14.541 | INFO | MemInfo: { | 20260506 20:42:14.541 | INFO | "gpu" : " 17 GB", | 20260506 20:42:14.541 | INFO | "huge" : "", | 20260506 20:42:14.541 | INFO | "huge_shared" : " 27 GB", | 20260506 20:42:14.541 | INFO | "malloc" : " 14 GB", | 20260506 20:42:14.541 | INFO | "pinned" : " 29 GB" | 20260506 20:42:14.541 | INFO | } | 20260506 20:42:14.541 | INFO | Perf report for the last 5.13985 seconds: | 20260506 20:42:14.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:42:14.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:14.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:14.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:14.541 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 20:42:14.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:14.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:14.541 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 20:42:14.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:14.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:14.542 | INFO | Uploaded 36GiB to GPU in 2.0343s (17.6965GiB/s) | 20260506 20:42:14.542 | INFO | Downloaded 34GiB to GPU in 1.95534s (17.3883GiB/s) | 20260506 20:42:19.678 | INFO | Analyzing chunk frames = [4307968,4308480) | 20260506 20:42:19.678 | INFO | MemInfo: { | 20260506 20:42:19.678 | INFO | "gpu" : " 17 GB", | 20260506 20:42:19.678 | INFO | "huge" : "", | 20260506 20:42:19.678 | INFO | "huge_shared" : " 27 GB", | 20260506 20:42:19.678 | INFO | "malloc" : " 14 GB", | 20260506 20:42:19.678 | INFO | "pinned" : " 29 GB" | 20260506 20:42:19.678 | INFO | } | 20260506 20:42:19.678 | INFO | Perf report for the last 5.13705 seconds: | 20260506 20:42:19.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:42:19.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:19.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:19.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:19.678 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 20:42:19.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:19.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:19.678 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:42:19.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:19.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:19.679 | INFO | Uploaded 36GiB to GPU in 1.96758s (18.2966GiB/s) | 20260506 20:42:19.679 | INFO | Downloaded 34GiB to GPU in 2.00331s (16.9719GiB/s) | 20260506 20:42:24.825 | INFO | Analyzing chunk frames = [4308480,4308992) | 20260506 20:42:24.825 | INFO | MemInfo: { | 20260506 20:42:24.825 | INFO | "gpu" : " 17 GB", | 20260506 20:42:24.825 | INFO | "huge" : "", | 20260506 20:42:24.825 | INFO | "huge_shared" : " 27 GB", | 20260506 20:42:24.825 | INFO | "malloc" : " 14 GB", | 20260506 20:42:24.825 | INFO | "pinned" : " 29 GB" | 20260506 20:42:24.826 | INFO | } | 20260506 20:42:24.826 | INFO | Perf report for the last 5.14722 seconds: | 20260506 20:42:24.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:42:24.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:24.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:24.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:24.826 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260506 20:42:24.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:24.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:24.826 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:42:24.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:24.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:24.826 | INFO | Uploaded 36GiB to GPU in 2.00193s (17.9826GiB/s) | 20260506 20:42:24.826 | INFO | Downloaded 34GiB to GPU in 1.99659s (17.029GiB/s) | 20260506 20:42:24.826 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:42:24.826 | INFO | Total mem usage: 18 GiB | 20260506 20:42:24.826 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:42:24.826 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.5%) | 20260506 20:42:24.826 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:42:24.826 | INFO | High Water Breakdown: | 20260506 20:42:24.826 | INFO | 61.9% : 9.9 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:42:24.826 | INFO | 38% : 6.08 GiB ( 90.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:42:24.826 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:42:24.826 | INFO | ------------------------------------------ | 20260506 20:42:24.826 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:42:24.826 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:42:24.826 | INFO | Total mem usage: 26.5 GiB | 20260506 20:42:24.826 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:42:24.826 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:42:24.826 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:42:24.826 | INFO | High Water Breakdown: | 20260506 20:42:24.826 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:42:24.826 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:42:24.826 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:42:24.826 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:42:24.826 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:42:24.826 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:42:24.826 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:42:24.826 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:42:24.826 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:42:24.826 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:42:24.826 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:42:24.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:42:24.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:42:24.827 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:42:24.827 | INFO | ------------------------------------------ | 20260506 20:42:24.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:42:24.827 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:42:24.827 | INFO | Total mem usage: 25.6 GiB | 20260506 20:42:24.827 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:42:24.827 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:42:24.827 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:42:24.827 | INFO | High Water Breakdown: | 20260506 20:42:24.827 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:42:24.827 | INFO | ------------------------------------------ | 20260506 20:42:24.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:42:24.827 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:42:24.827 | INFO | Total mem usage: 15.7 GiB | 20260506 20:42:24.827 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:42:24.827 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:42:24.827 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 20:42:24.827 | INFO | High Water Breakdown: | 20260506 20:42:24.827 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:42:24.827 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:42:24.827 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:42:24.827 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:42:24.827 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:42:24.827 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:42:24.827 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:42:24.827 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:42:24.827 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:42:24.827 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:42:24.827 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:42:24.827 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:42:24.827 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:42:24.827 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:42:24.827 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:42:24.827 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:42:24.827 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:42:24.827 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:42:24.827 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:42:24.827 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:42:24.827 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:42:24.827 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:42:24.827 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:42:24.827 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:42:24.827 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:42:24.827 | INFO | ------------------------------------------ | 20260506 20:42:24.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:42:24.827 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:42:24.827 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:42:24.827 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:42:24.827 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.59e+01, 1.34e+01 / 2.32e+00 | 20260506 20:42:24.827 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.02e-01, 9.78e-03 / 6.61e-03 | 20260506 20:42:24.827 | INFO | PerfReport: 11.63%, Baseline -- 3.62e+00 / 4.40e+00, 5.48e-01 / 3.25e-01 | 20260506 20:42:24.827 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.46e+00, 1.81e-01 / 1.43e-01 | 20260506 20:42:24.827 | INFO | PerfReport: 16.06%, DME -- 4.00e+01 / 4.51e+01, 4.19e+00 / 2.44e+00 | 20260506 20:42:24.827 | INFO | PerfReport: 57.17%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.37e+00 / 1.68e+00 | 20260506 20:42:24.827 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.23e+00 / 2.81e+00, 3.82e-01 / 1.90e-01 | 20260506 20:42:24.827 | INFO | PerfReport: 3.30%, Metrics -- 1.03e+00 / 1.38e+00, 2.71e-01 / 1.34e-01 | 20260506 20:42:24.827 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.53e-01 / 4.06e-01, 5.48e-02 / 2.70e-02 | 20260506 20:42:24.827 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:42:24.827 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:42:24.827 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:42:24.827 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 20:42:24.827 | INFO | PerfReport: 21.51%, Upload -- 2.08e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 20:42:24.827 | INFO | PerfReport: 21.21%, Download -- 2.05e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 20:42:24.827 | INFO | PerfReport: 32.18%, Compute -- 3.11e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 20:42:24.827 | INFO | PerfReport: 1.33%, LockUpload -- 1.29e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260506 20:42:24.827 | INFO | PerfReport: 1.14%, LockDownload -- 1.11e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260506 20:42:24.827 | INFO | PerfReport: 22.62%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 20:42:24.827 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260506 20:42:24.926 | INFO | Starting a round of Baz IO | 20260506 20:42:28.066 | INFO | Populating RTMetrics report | 20260506 20:42:28.066 | INFO | RTMetrics report populated | 20260506 20:42:28.066 | INFO | Opening tmp RTMetrics json file | 20260506 20:42:28.219 | INFO | Serializing RTMetrics to json | 20260506 20:42:28.319 | INFO | Serialization done, swapping tmp file now... | 20260506 20:42:28.372 | INFO | RTMetrics done | 20260506 20:42:28.539 | INFO | Finished a round of Baz IO: Wrote 1.973 GiB in 3.612 seconds | 20260506 20:42:28.539 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.16% / 0.00% / 65.84% | 20260506 20:42:28.539 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.715295% / 0% / 8.74201e-05% | 20260506 20:42:28.539 | INFO | Baz IO overall: | 20260506 20:42:28.539 | INFO | BytesWritten : 2274024501191 | 20260506 20:42:28.539 | INFO | NumEvents : 1098476518801 | 20260506 20:42:29.961 | INFO | Analyzing chunk frames = [4308992,4309504) | 20260506 20:42:29.961 | INFO | MemInfo: { | 20260506 20:42:29.961 | INFO | "gpu" : " 17 GB", | 20260506 20:42:29.961 | INFO | "huge" : "", | 20260506 20:42:29.961 | INFO | "huge_shared" : " 27 GB", | 20260506 20:42:29.961 | INFO | "malloc" : " 14 GB", | 20260506 20:42:29.961 | INFO | "pinned" : " 29 GB" | 20260506 20:42:29.961 | INFO | } | 20260506 20:42:29.961 | INFO | Perf report for the last 5.13573 seconds: | 20260506 20:42:29.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:42:29.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:29.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:29.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:29.961 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 20:42:29.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:29.961 | INFO | RT_METRICS : 48.7%, 1.0, 25.5ms, 0 | 20260506 20:42:29.961 | INFO | PRE_HQ : 60.1%, 1.8, 57.1ms, 0 | 20260506 20:42:29.961 | INFO | BAZWRITER : 70.0%, 4.8, 172.5ms, 0 | 20260506 20:42:29.961 | INFO | BAZ_SAMPLING : 3.4%, 1.0, 1.8ms, 0 | 20260506 20:42:29.962 | INFO | Uploaded 36GiB to GPU in 2.07786s (17.3255GiB/s) | 20260506 20:42:29.962 | INFO | Downloaded 38GiB to GPU in 2.18989s (17.3525GiB/s) | 20260506 20:42:35.101 | INFO | Analyzing chunk frames = [4309504,4310016) | 20260506 20:42:35.101 | INFO | MemInfo: { | 20260506 20:42:35.101 | INFO | "gpu" : " 17 GB", | 20260506 20:42:35.101 | INFO | "huge" : "", | 20260506 20:42:35.101 | INFO | "huge_shared" : " 27 GB", | 20260506 20:42:35.101 | INFO | "malloc" : " 14 GB", | 20260506 20:42:35.101 | INFO | "pinned" : " 29 GB" | 20260506 20:42:35.101 | INFO | } | 20260506 20:42:35.101 | INFO | Perf report for the last 5.13971 seconds: | 20260506 20:42:35.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:42:35.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:35.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:35.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:35.101 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 20:42:35.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:35.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:35.101 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 20:42:35.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:35.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:35.101 | INFO | Uploaded 36GiB to GPU in 2.00442s (17.9603GiB/s) | 20260506 20:42:35.101 | INFO | Downloaded 34GiB to GPU in 1.92362s (17.6751GiB/s) | 20260506 20:42:40.236 | INFO | Analyzing chunk frames = [4310016,4310528) | 20260506 20:42:40.236 | INFO | MemInfo: { | 20260506 20:42:40.236 | INFO | "gpu" : " 17 GB", | 20260506 20:42:40.236 | INFO | "huge" : "", | 20260506 20:42:40.236 | INFO | "huge_shared" : " 27 GB", | 20260506 20:42:40.236 | INFO | "malloc" : " 14 GB", | 20260506 20:42:40.236 | INFO | "pinned" : " 29 GB" | 20260506 20:42:40.236 | INFO | } | 20260506 20:42:40.237 | INFO | Perf report for the last 5.13549 seconds: | 20260506 20:42:40.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:42:40.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:40.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:40.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:40.237 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 20:42:40.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:40.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:40.237 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:42:40.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:40.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:40.237 | INFO | Uploaded 36GiB to GPU in 1.9815s (18.168GiB/s) | 20260506 20:42:40.237 | INFO | Downloaded 34GiB to GPU in 1.91788s (17.7279GiB/s) | 20260506 20:42:45.374 | INFO | Analyzing chunk frames = [4310528,4311040) | 20260506 20:42:45.374 | INFO | MemInfo: { | 20260506 20:42:45.374 | INFO | "gpu" : " 17 GB", | 20260506 20:42:45.374 | INFO | "huge" : "", | 20260506 20:42:45.374 | INFO | "huge_shared" : " 27 GB", | 20260506 20:42:45.374 | INFO | "malloc" : " 14 GB", | 20260506 20:42:45.374 | INFO | "pinned" : " 29 GB" | 20260506 20:42:45.374 | INFO | } | 20260506 20:42:45.374 | INFO | Perf report for the last 5.13778 seconds: | 20260506 20:42:45.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:42:45.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:45.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:45.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:45.374 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 20:42:45.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:45.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:45.374 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:42:45.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:45.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:45.375 | INFO | Uploaded 36GiB to GPU in 1.93432s (18.6112GiB/s) | 20260506 20:42:45.375 | INFO | Downloaded 34GiB to GPU in 1.87084s (18.1737GiB/s) | 20260506 20:42:50.519 | INFO | Analyzing chunk frames = [4311040,4311552) | 20260506 20:42:50.519 | INFO | MemInfo: { | 20260506 20:42:50.519 | INFO | "gpu" : " 17 GB", | 20260506 20:42:50.519 | INFO | "huge" : "", | 20260506 20:42:50.519 | INFO | "huge_shared" : " 27 GB", | 20260506 20:42:50.519 | INFO | "malloc" : " 14 GB", | 20260506 20:42:50.519 | INFO | "pinned" : " 29 GB" | 20260506 20:42:50.519 | INFO | } | 20260506 20:42:50.519 | INFO | Perf report for the last 5.14454 seconds: | 20260506 20:42:50.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:42:50.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:50.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:50.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:50.519 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260506 20:42:50.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:50.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:50.519 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:42:50.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:50.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:50.519 | INFO | Uploaded 36GiB to GPU in 2.01244s (17.8887GiB/s) | 20260506 20:42:50.519 | INFO | Downloaded 34GiB to GPU in 1.9036s (17.8609GiB/s) | 20260506 20:42:55.661 | INFO | Analyzing chunk frames = [4311552,4312064) | 20260506 20:42:55.661 | INFO | MemInfo: { | 20260506 20:42:55.661 | INFO | "gpu" : " 17 GB", | 20260506 20:42:55.661 | INFO | "huge" : "", | 20260506 20:42:55.661 | INFO | "huge_shared" : " 27 GB", | 20260506 20:42:55.661 | INFO | "malloc" : " 14 GB", | 20260506 20:42:55.661 | INFO | "pinned" : " 29 GB" | 20260506 20:42:55.661 | INFO | } | 20260506 20:42:55.661 | INFO | Perf report for the last 5.14236 seconds: | 20260506 20:42:55.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:42:55.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:55.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:55.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:55.661 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:42:55.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:55.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:55.661 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 20:42:55.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:55.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:42:55.662 | INFO | Uploaded 36GiB to GPU in 2.02824s (17.7494GiB/s) | 20260506 20:42:55.662 | INFO | Downloaded 34GiB to GPU in 1.94665s (17.4659GiB/s) | 20260506 20:43:00.799 | INFO | Analyzing chunk frames = [4312064,4312576) | 20260506 20:43:00.799 | INFO | MemInfo: { | 20260506 20:43:00.799 | INFO | "gpu" : " 17 GB", | 20260506 20:43:00.799 | INFO | "huge" : "", | 20260506 20:43:00.799 | INFO | "huge_shared" : " 27 GB", | 20260506 20:43:00.799 | INFO | "malloc" : " 14 GB", | 20260506 20:43:00.799 | INFO | "pinned" : " 29 GB" | 20260506 20:43:00.799 | INFO | } | 20260506 20:43:00.799 | INFO | Perf report for the last 5.13765 seconds: | 20260506 20:43:00.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:43:00.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:00.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:00.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:00.799 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:43:00.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:00.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:00.799 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:43:00.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:00.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:00.799 | INFO | Uploaded 36GiB to GPU in 1.9722s (18.2538GiB/s) | 20260506 20:43:00.799 | INFO | Downloaded 34GiB to GPU in 2.0014s (16.9881GiB/s) | 20260506 20:43:05.936 | INFO | Analyzing chunk frames = [4312576,4313088) | 20260506 20:43:05.936 | INFO | MemInfo: { | 20260506 20:43:05.936 | INFO | "gpu" : " 17 GB", | 20260506 20:43:05.936 | INFO | "huge" : "", | 20260506 20:43:05.937 | INFO | "huge_shared" : " 27 GB", | 20260506 20:43:05.937 | INFO | "malloc" : " 14 GB", | 20260506 20:43:05.937 | INFO | "pinned" : " 29 GB" | 20260506 20:43:05.937 | INFO | } | 20260506 20:43:05.937 | INFO | Perf report for the last 5.1378 seconds: | 20260506 20:43:05.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:43:05.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:05.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:05.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:05.937 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 20:43:05.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:05.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:05.937 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:43:05.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:05.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:05.937 | INFO | Uploaded 36GiB to GPU in 1.98964s (18.0937GiB/s) | 20260506 20:43:05.937 | INFO | Downloaded 34GiB to GPU in 1.97264s (17.2358GiB/s) | 20260506 20:43:06.034 | INFO | Starting a round of Baz IO | 20260506 20:43:09.166 | INFO | Populating RTMetrics report | 20260506 20:43:09.166 | INFO | RTMetrics report populated | 20260506 20:43:09.166 | INFO | Opening tmp RTMetrics json file | 20260506 20:43:09.187 | INFO | Serializing RTMetrics to json | 20260506 20:43:09.196 | INFO | Serialization done, swapping tmp file now... | 20260506 20:43:09.220 | INFO | RTMetrics done | 20260506 20:43:09.646 | INFO | Finished a round of Baz IO: Wrote 1.971 GiB in 3.611 seconds | 20260506 20:43:09.646 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.16% / 0.00% / 65.84% | 20260506 20:43:09.646 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.71371% / 0% / 6.75519e-05% | 20260506 20:43:09.646 | INFO | Baz IO overall: | 20260506 20:43:09.646 | INFO | BytesWritten : 2276140869686 | 20260506 20:43:09.646 | INFO | NumEvents : 1099491439785 | 20260506 20:43:11.076 | INFO | Analyzing chunk frames = [4313088,4313600) | 20260506 20:43:11.076 | INFO | MemInfo: { | 20260506 20:43:11.076 | INFO | "gpu" : " 17 GB", | 20260506 20:43:11.076 | INFO | "huge" : "", | 20260506 20:43:11.076 | INFO | "huge_shared" : " 27 GB", | 20260506 20:43:11.076 | INFO | "malloc" : " 14 GB", | 20260506 20:43:11.076 | INFO | "pinned" : " 29 GB" | 20260506 20:43:11.076 | INFO | } | 20260506 20:43:11.076 | INFO | Perf report for the last 5.13961 seconds: | 20260506 20:43:11.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:43:11.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:11.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:11.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:11.076 | INFO | ANALYSIS : 62.3%, 2.9, 95.2ms, 0 | 20260506 20:43:11.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:11.076 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 20:43:11.076 | INFO | PRE_HQ : 59.9%, 1.8, 56.9ms, 0 | 20260506 20:43:11.076 | INFO | BAZWRITER : 69.9%, 4.8, 173.2ms, 0 | 20260506 20:43:11.077 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 20:43:11.077 | INFO | Uploaded 36GiB to GPU in 2.05115s (17.5512GiB/s) | 20260506 20:43:11.077 | INFO | Downloaded 38GiB to GPU in 2.18797s (17.3677GiB/s) | 20260506 20:43:16.209 | INFO | Analyzing chunk frames = [4313600,4314112) | 20260506 20:43:16.209 | INFO | MemInfo: { | 20260506 20:43:16.209 | INFO | "gpu" : " 17 GB", | 20260506 20:43:16.209 | INFO | "huge" : "", | 20260506 20:43:16.209 | INFO | "huge_shared" : " 27 GB", | 20260506 20:43:16.209 | INFO | "malloc" : " 14 GB", | 20260506 20:43:16.209 | INFO | "pinned" : " 29 GB" | 20260506 20:43:16.209 | INFO | } | 20260506 20:43:16.209 | INFO | Perf report for the last 5.13303 seconds: | 20260506 20:43:16.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:43:16.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:16.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:16.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:16.209 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:43:16.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:16.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:16.210 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:43:16.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:16.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:16.210 | INFO | Uploaded 36GiB to GPU in 2.0182s (17.8377GiB/s) | 20260506 20:43:16.210 | INFO | Downloaded 34GiB to GPU in 1.92588s (17.6543GiB/s) | 20260506 20:43:21.345 | INFO | Analyzing chunk frames = [4314112,4314624) | 20260506 20:43:21.345 | INFO | MemInfo: { | 20260506 20:43:21.345 | INFO | "gpu" : " 17 GB", | 20260506 20:43:21.345 | INFO | "huge" : "", | 20260506 20:43:21.345 | INFO | "huge_shared" : " 27 GB", | 20260506 20:43:21.345 | INFO | "malloc" : " 14 GB", | 20260506 20:43:21.345 | INFO | "pinned" : " 29 GB" | 20260506 20:43:21.345 | INFO | } | 20260506 20:43:21.345 | INFO | Perf report for the last 5.13563 seconds: | 20260506 20:43:21.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:43:21.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:21.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:21.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:21.345 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 20:43:21.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:21.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:21.345 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:43:21.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:21.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:21.345 | INFO | Uploaded 36GiB to GPU in 1.98847s (18.1044GiB/s) | 20260506 20:43:21.345 | INFO | Downloaded 34GiB to GPU in 1.91577s (17.7474GiB/s) | 20260506 20:43:26.489 | INFO | Analyzing chunk frames = [4314624,4315136) | 20260506 20:43:26.489 | INFO | MemInfo: { | 20260506 20:43:26.489 | INFO | "gpu" : " 17 GB", | 20260506 20:43:26.489 | INFO | "huge" : "", | 20260506 20:43:26.489 | INFO | "huge_shared" : " 27 GB", | 20260506 20:43:26.489 | INFO | "malloc" : " 14 GB", | 20260506 20:43:26.489 | INFO | "pinned" : " 29 GB" | 20260506 20:43:26.489 | INFO | } | 20260506 20:43:26.489 | INFO | Perf report for the last 5.14444 seconds: | 20260506 20:43:26.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:43:26.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:26.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:26.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:26.490 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:43:26.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:26.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:26.490 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:43:26.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:26.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:26.490 | INFO | Uploaded 36GiB to GPU in 1.93846s (18.5714GiB/s) | 20260506 20:43:26.491 | INFO | Downloaded 34GiB to GPU in 1.8776s (18.1082GiB/s) | 20260506 20:43:31.634 | INFO | Analyzing chunk frames = [4315136,4315648) | 20260506 20:43:31.634 | INFO | MemInfo: { | 20260506 20:43:31.634 | INFO | "gpu" : " 17 GB", | 20260506 20:43:31.634 | INFO | "huge" : "", | 20260506 20:43:31.634 | INFO | "huge_shared" : " 27 GB", | 20260506 20:43:31.634 | INFO | "malloc" : " 14 GB", | 20260506 20:43:31.634 | INFO | "pinned" : " 29 GB" | 20260506 20:43:31.634 | INFO | } | 20260506 20:43:31.634 | INFO | Perf report for the last 5.14485 seconds: | 20260506 20:43:31.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:43:31.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:31.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:31.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:31.634 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260506 20:43:31.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:31.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:31.634 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:43:31.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:31.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:31.635 | INFO | Uploaded 36GiB to GPU in 2.01445s (17.8709GiB/s) | 20260506 20:43:31.635 | INFO | Downloaded 34GiB to GPU in 1.9151s (17.7537GiB/s) | 20260506 20:43:36.774 | INFO | Analyzing chunk frames = [4315648,4316160) | 20260506 20:43:36.774 | INFO | MemInfo: { | 20260506 20:43:36.774 | INFO | "gpu" : " 17 GB", | 20260506 20:43:36.774 | INFO | "huge" : "", | 20260506 20:43:36.774 | INFO | "huge_shared" : " 27 GB", | 20260506 20:43:36.774 | INFO | "malloc" : " 14 GB", | 20260506 20:43:36.774 | INFO | "pinned" : " 29 GB" | 20260506 20:43:36.774 | INFO | } | 20260506 20:43:36.774 | INFO | Perf report for the last 5.13987 seconds: | 20260506 20:43:36.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:43:36.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:36.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:36.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:36.774 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:43:36.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:36.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:36.774 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 20:43:36.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:36.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:36.775 | INFO | Uploaded 36GiB to GPU in 2.01725s (17.8461GiB/s) | 20260506 20:43:36.775 | INFO | Downloaded 34GiB to GPU in 1.9428s (17.5005GiB/s) | 20260506 20:43:41.917 | INFO | Analyzing chunk frames = [4316160,4316672) | 20260506 20:43:41.917 | INFO | MemInfo: { | 20260506 20:43:41.917 | INFO | "gpu" : " 17 GB", | 20260506 20:43:41.917 | INFO | "huge" : "", | 20260506 20:43:41.917 | INFO | "huge_shared" : " 27 GB", | 20260506 20:43:41.917 | INFO | "malloc" : " 14 GB", | 20260506 20:43:41.917 | INFO | "pinned" : " 29 GB" | 20260506 20:43:41.917 | INFO | } | 20260506 20:43:41.917 | INFO | Perf report for the last 5.14325 seconds: | 20260506 20:43:41.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:43:41.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:41.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:41.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:41.918 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:43:41.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:41.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:41.918 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:43:41.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:41.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:41.918 | INFO | Uploaded 36GiB to GPU in 1.99262s (18.0667GiB/s) | 20260506 20:43:41.918 | INFO | Downloaded 34GiB to GPU in 2.00506s (16.9571GiB/s) | 20260506 20:43:47.051 | INFO | Analyzing chunk frames = [4316672,4317184) | 20260506 20:43:47.051 | INFO | MemInfo: { | 20260506 20:43:47.051 | INFO | "gpu" : " 17 GB", | 20260506 20:43:47.051 | INFO | "huge" : "", | 20260506 20:43:47.052 | INFO | "huge_shared" : " 27 GB", | 20260506 20:43:47.052 | INFO | "malloc" : " 14 GB", | 20260506 20:43:47.052 | INFO | "pinned" : " 29 GB" | 20260506 20:43:47.052 | INFO | } | 20260506 20:43:47.052 | INFO | Perf report for the last 5.13416 seconds: | 20260506 20:43:47.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:43:47.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:47.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:47.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:47.052 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 20:43:47.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:47.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:47.052 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 20:43:47.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:47.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:47.052 | INFO | Uploaded 36GiB to GPU in 2.04712s (17.5857GiB/s) | 20260506 20:43:47.052 | INFO | Downloaded 34GiB to GPU in 1.99361s (17.0545GiB/s) | 20260506 20:43:47.052 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:43:47.052 | INFO | Total mem usage: 18 GiB | 20260506 20:43:47.052 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:43:47.052 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -11%) | 20260506 20:43:47.052 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:43:47.052 | INFO | High Water Breakdown: | 20260506 20:43:47.052 | INFO | 62.1% : 10.1 GiB ( 88.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:43:47.052 | INFO | 37.8% : 6.13 GiB ( 91.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:43:47.052 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:43:47.052 | INFO | ------------------------------------------ | 20260506 20:43:47.052 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:43:47.052 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:43:47.052 | INFO | Total mem usage: 26.5 GiB | 20260506 20:43:47.052 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:43:47.052 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:43:47.052 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:43:47.052 | INFO | High Water Breakdown: | 20260506 20:43:47.052 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:43:47.052 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:43:47.052 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:43:47.052 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:43:47.052 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:43:47.052 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:43:47.052 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:43:47.052 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:43:47.052 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:43:47.052 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:43:47.052 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:43:47.052 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:43:47.052 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:43:47.052 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:43:47.052 | INFO | ------------------------------------------ | 20260506 20:43:47.052 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:43:47.052 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:43:47.052 | INFO | Total mem usage: 25.6 GiB | 20260506 20:43:47.052 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:43:47.052 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 20:43:47.052 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 20:43:47.052 | INFO | High Water Breakdown: | 20260506 20:43:47.053 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 20:43:47.053 | INFO | ------------------------------------------ | 20260506 20:43:47.053 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:43:47.053 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:43:47.053 | INFO | Total mem usage: 15.7 GiB | 20260506 20:43:47.053 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:43:47.053 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:43:47.053 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 20:43:47.053 | INFO | High Water Breakdown: | 20260506 20:43:47.053 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:43:47.053 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:43:47.053 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:43:47.053 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:43:47.053 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:43:47.053 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:43:47.053 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:43:47.053 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:43:47.053 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:43:47.053 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:43:47.053 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:43:47.053 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:43:47.053 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:43:47.053 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:43:47.053 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:43:47.053 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:43:47.053 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:43:47.053 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:43:47.053 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:43:47.053 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:43:47.053 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:43:47.053 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:43:47.053 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:43:47.053 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:43:47.053 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:43:47.053 | INFO | ------------------------------------------ | 20260506 20:43:47.053 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:43:47.053 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:43:47.053 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:43:47.053 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:43:47.053 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.59e+01, 1.31e+01 / 2.79e+00 | 20260506 20:43:47.053 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.02e-01, 9.86e-03 / 9.13e-03 | 20260506 20:43:47.053 | INFO | PerfReport: 11.64%, Baseline -- 3.62e+00 / 4.45e+00, 5.48e-01 / 3.44e-01 | 20260506 20:43:47.053 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.48e+00, 1.80e-01 / 1.30e-01 | 20260506 20:43:47.053 | INFO | PerfReport: 16.06%, DME -- 4.00e+01 / 4.49e+01, 4.07e+00 / 2.98e+00 | 20260506 20:43:47.053 | INFO | PerfReport: 57.15%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.34e+00 / 1.82e+00 | 20260506 20:43:47.053 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.23e+00 / 2.82e+00, 3.80e-01 / 1.85e-01 | 20260506 20:43:47.053 | INFO | PerfReport: 3.31%, Metrics -- 1.03e+00 / 1.39e+00, 2.73e-01 / 1.88e-01 | 20260506 20:43:47.053 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.53e-01 / 4.07e-01, 5.53e-02 / 3.02e-02 | 20260506 20:43:47.053 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:43:47.053 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:43:47.053 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:43:47.053 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 20:43:47.053 | INFO | PerfReport: 21.59%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 20:43:47.053 | INFO | PerfReport: 21.20%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 20:43:47.053 | INFO | PerfReport: 32.18%, Compute -- 3.11e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 20:43:47.053 | INFO | PerfReport: 1.36%, LockUpload -- 1.32e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260506 20:43:47.053 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 20:43:47.053 | INFO | PerfReport: 22.55%, LockCompute -- 2.18e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 20:43:47.053 | INFO | PerfReport: 0.02%, Misc -- 1.60e-02 / 0.00e+00, 4.88e-02 / 0.00e+00 | 20260506 20:43:47.169 | INFO | Starting a round of Baz IO | 20260506 20:43:50.297 | INFO | Populating RTMetrics report | 20260506 20:43:50.297 | INFO | RTMetrics report populated | 20260506 20:43:50.297 | INFO | Opening tmp RTMetrics json file | 20260506 20:43:50.430 | INFO | Serializing RTMetrics to json | 20260506 20:43:50.437 | INFO | Serialization done, swapping tmp file now... | 20260506 20:43:50.439 | INFO | RTMetrics done | 20260506 20:43:50.456 | INFO | Finished a round of Baz IO: Wrote 1.969 GiB in 3.287 seconds | 20260506 20:43:50.457 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.15% / 0.00% / 65.84% | 20260506 20:43:50.457 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.711242% / 7.94729e-06% / 5.5631e-05% | 20260506 20:43:50.457 | INFO | Baz IO overall: | 20260506 20:43:50.457 | INFO | BytesWritten : 2278255498330 | 20260506 20:43:50.457 | INFO | NumEvents : 1100505241999 | 20260506 20:43:52.185 | INFO | Analyzing chunk frames = [4317184,4317696) | 20260506 20:43:52.185 | INFO | MemInfo: { | 20260506 20:43:52.185 | INFO | "gpu" : " 17 GB", | 20260506 20:43:52.185 | INFO | "huge" : "", | 20260506 20:43:52.185 | INFO | "huge_shared" : " 27 GB", | 20260506 20:43:52.185 | INFO | "malloc" : " 14 GB", | 20260506 20:43:52.185 | INFO | "pinned" : " 29 GB" | 20260506 20:43:52.185 | INFO | } | 20260506 20:43:52.185 | INFO | Perf report for the last 5.13361 seconds: | 20260506 20:43:52.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:43:52.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:52.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:52.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:52.185 | INFO | ANALYSIS : 62.6%, 2.9, 95.7ms, 0 | 20260506 20:43:52.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:52.185 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 20:43:52.185 | INFO | PRE_HQ : 59.9%, 1.8, 57.1ms, 0 | 20260506 20:43:52.185 | INFO | BAZWRITER : 63.7%, 4.6, 149.6ms, 0 | 20260506 20:43:52.185 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.3ms, 0 | 20260506 20:43:52.186 | INFO | Uploaded 36GiB to GPU in 2.05082s (17.5539GiB/s) | 20260506 20:43:52.186 | INFO | Downloaded 38GiB to GPU in 2.21976s (17.119GiB/s) | 20260506 20:43:57.323 | INFO | Analyzing chunk frames = [4317696,4318208) | 20260506 20:43:57.323 | INFO | MemInfo: { | 20260506 20:43:57.323 | INFO | "gpu" : " 17 GB", | 20260506 20:43:57.323 | INFO | "huge" : "", | 20260506 20:43:57.323 | INFO | "huge_shared" : " 27 GB", | 20260506 20:43:57.323 | INFO | "malloc" : " 14 GB", | 20260506 20:43:57.323 | INFO | "pinned" : " 29 GB" | 20260506 20:43:57.323 | INFO | } | 20260506 20:43:57.323 | INFO | Perf report for the last 5.1374 seconds: | 20260506 20:43:57.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:43:57.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:57.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:57.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:57.323 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:43:57.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:57.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:57.323 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 20:43:57.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:57.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:43:57.323 | INFO | Uploaded 36GiB to GPU in 2.0158s (17.8589GiB/s) | 20260506 20:43:57.323 | INFO | Downloaded 34GiB to GPU in 1.92508s (17.6616GiB/s) | 20260506 20:44:02.465 | INFO | Analyzing chunk frames = [4318208,4318720) | 20260506 20:44:02.465 | INFO | MemInfo: { | 20260506 20:44:02.465 | INFO | "gpu" : " 17 GB", | 20260506 20:44:02.465 | INFO | "huge" : "", | 20260506 20:44:02.465 | INFO | "huge_shared" : " 27 GB", | 20260506 20:44:02.465 | INFO | "malloc" : " 14 GB", | 20260506 20:44:02.465 | INFO | "pinned" : " 29 GB" | 20260506 20:44:02.465 | INFO | } | 20260506 20:44:02.465 | INFO | Perf report for the last 5.14239 seconds: | 20260506 20:44:02.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:44:02.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:02.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:02.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:02.465 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 20:44:02.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:02.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:02.465 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:44:02.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:02.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:02.466 | INFO | Uploaded 36GiB to GPU in 1.98089s (18.1736GiB/s) | 20260506 20:44:02.466 | INFO | Downloaded 34GiB to GPU in 1.92098s (17.6993GiB/s) | 20260506 20:44:07.612 | INFO | Analyzing chunk frames = [4318720,4319232) | 20260506 20:44:07.612 | INFO | MemInfo: { | 20260506 20:44:07.612 | INFO | "gpu" : " 17 GB", | 20260506 20:44:07.612 | INFO | "huge" : "", | 20260506 20:44:07.612 | INFO | "huge_shared" : " 27 GB", | 20260506 20:44:07.612 | INFO | "malloc" : " 14 GB", | 20260506 20:44:07.612 | INFO | "pinned" : " 29 GB" | 20260506 20:44:07.612 | INFO | } | 20260506 20:44:07.612 | INFO | Perf report for the last 5.14667 seconds: | 20260506 20:44:07.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:44:07.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:07.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:07.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:07.612 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:44:07.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:07.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:07.612 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:44:07.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:07.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:07.612 | INFO | Uploaded 36GiB to GPU in 1.95493s (18.415GiB/s) | 20260506 20:44:07.612 | INFO | Downloaded 34GiB to GPU in 1.87376s (18.1454GiB/s) | 20260506 20:44:12.748 | INFO | Analyzing chunk frames = [4319232,4319744) | 20260506 20:44:12.748 | INFO | MemInfo: { | 20260506 20:44:12.748 | INFO | "gpu" : " 17 GB", | 20260506 20:44:12.748 | INFO | "huge" : "", | 20260506 20:44:12.748 | INFO | "huge_shared" : " 27 GB", | 20260506 20:44:12.748 | INFO | "malloc" : " 14 GB", | 20260506 20:44:12.748 | INFO | "pinned" : " 29 GB" | 20260506 20:44:12.748 | INFO | } | 20260506 20:44:12.748 | INFO | Perf report for the last 5.13664 seconds: | 20260506 20:44:12.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:44:12.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:12.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:12.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:12.748 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 20:44:12.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:12.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:12.749 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:44:12.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:12.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:12.749 | INFO | Uploaded 36GiB to GPU in 1.96181s (18.3504GiB/s) | 20260506 20:44:12.749 | INFO | Downloaded 34GiB to GPU in 1.92164s (17.6932GiB/s) | 20260506 20:44:17.887 | INFO | Analyzing chunk frames = [4319744,4320256) | 20260506 20:44:17.887 | INFO | MemInfo: { | 20260506 20:44:17.887 | INFO | "gpu" : " 17 GB", | 20260506 20:44:17.887 | INFO | "huge" : "", | 20260506 20:44:17.887 | INFO | "huge_shared" : " 27 GB", | 20260506 20:44:17.887 | INFO | "malloc" : " 14 GB", | 20260506 20:44:17.887 | INFO | "pinned" : " 29 GB" | 20260506 20:44:17.887 | INFO | } | 20260506 20:44:17.887 | INFO | Perf report for the last 5.13893 seconds: | 20260506 20:44:17.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:44:17.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:17.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:17.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:17.887 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:44:17.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:17.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:17.887 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 20:44:17.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:17.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:17.888 | INFO | Uploaded 36GiB to GPU in 2.01573s (17.8595GiB/s) | 20260506 20:44:17.888 | INFO | Downloaded 34GiB to GPU in 1.94687s (17.4639GiB/s) | 20260506 20:44:23.022 | INFO | Analyzing chunk frames = [4320256,4320768) | 20260506 20:44:23.022 | INFO | MemInfo: { | 20260506 20:44:23.022 | INFO | "gpu" : " 17 GB", | 20260506 20:44:23.022 | INFO | "huge" : "", | 20260506 20:44:23.022 | INFO | "huge_shared" : " 27 GB", | 20260506 20:44:23.022 | INFO | "malloc" : " 14 GB", | 20260506 20:44:23.022 | INFO | "pinned" : " 29 GB" | 20260506 20:44:23.022 | INFO | } | 20260506 20:44:23.022 | INFO | Perf report for the last 5.13468 seconds: | 20260506 20:44:23.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:44:23.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:23.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:23.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:23.022 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:44:23.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:23.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:23.022 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 20:44:23.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:23.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:23.022 | INFO | Uploaded 36GiB to GPU in 1.94084s (18.5487GiB/s) | 20260506 20:44:23.022 | INFO | Downloaded 34GiB to GPU in 1.98946s (17.0901GiB/s) | 20260506 20:44:28.162 | INFO | Analyzing chunk frames = [4320768,4321280) | 20260506 20:44:28.162 | INFO | MemInfo: { | 20260506 20:44:28.162 | INFO | "gpu" : " 17 GB", | 20260506 20:44:28.162 | INFO | "huge" : "", | 20260506 20:44:28.162 | INFO | "huge_shared" : " 27 GB", | 20260506 20:44:28.162 | INFO | "malloc" : " 14 GB", | 20260506 20:44:28.162 | INFO | "pinned" : " 29 GB" | 20260506 20:44:28.162 | INFO | } | 20260506 20:44:28.162 | INFO | Perf report for the last 5.14011 seconds: | 20260506 20:44:28.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:44:28.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:28.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:28.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:28.162 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 20:44:28.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:28.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:28.163 | INFO | PRE_HQ : 30.9%, 1.0, 16.1ms, 0 | 20260506 20:44:28.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:28.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:28.163 | INFO | Uploaded 36GiB to GPU in 2.03432s (17.6963GiB/s) | 20260506 20:44:28.163 | INFO | Downloaded 34GiB to GPU in 1.99054s (17.0808GiB/s) | 20260506 20:44:28.277 | INFO | Starting a round of Baz IO | 20260506 20:44:31.430 | INFO | Populating RTMetrics report | 20260506 20:44:31.431 | INFO | RTMetrics report populated | 20260506 20:44:31.431 | INFO | Opening tmp RTMetrics json file | 20260506 20:44:31.465 | INFO | Serializing RTMetrics to json | 20260506 20:44:31.492 | INFO | Serialization done, swapping tmp file now... | 20260506 20:44:31.500 | INFO | RTMetrics done | 20260506 20:44:31.774 | INFO | Finished a round of Baz IO: Wrote 1.968 GiB in 3.496 seconds | 20260506 20:44:31.774 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.15% / 0.00% / 65.85% | 20260506 20:44:31.774 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.706379% / 7.94729e-06% / 9.93411e-05% | 20260506 20:44:31.774 | INFO | Baz IO overall: | 20260506 20:44:31.774 | INFO | BytesWritten : 2280368785205 | 20260506 20:44:31.774 | INFO | NumEvents : 1101517864801 | 20260506 20:44:33.300 | INFO | Analyzing chunk frames = [4321280,4321792) | 20260506 20:44:33.300 | INFO | MemInfo: { | 20260506 20:44:33.300 | INFO | "gpu" : " 17 GB", | 20260506 20:44:33.301 | INFO | "huge" : "", | 20260506 20:44:33.301 | INFO | "huge_shared" : " 27 GB", | 20260506 20:44:33.301 | INFO | "malloc" : " 14 GB", | 20260506 20:44:33.301 | INFO | "pinned" : " 29 GB" | 20260506 20:44:33.301 | INFO | } | 20260506 20:44:33.301 | INFO | Perf report for the last 5.1384 seconds: | 20260506 20:44:33.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:44:33.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:33.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:33.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:33.301 | INFO | ANALYSIS : 62.7%, 2.8, 93.7ms, 0 | 20260506 20:44:33.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:33.301 | INFO | RT_METRICS : 47.8%, 1.0, 25.1ms, 0 | 20260506 20:44:33.301 | INFO | PRE_HQ : 60.9%, 2.0, 62.4ms, 0 | 20260506 20:44:33.301 | INFO | BAZWRITER : 67.8%, 4.8, 166.1ms, 0 | 20260506 20:44:33.301 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 20:44:33.301 | INFO | Uploaded 36GiB to GPU in 2.04552s (17.5994GiB/s) | 20260506 20:44:33.301 | INFO | Downloaded 38GiB to GPU in 2.1964s (17.301GiB/s) | 20260506 20:44:38.438 | INFO | Analyzing chunk frames = [4321792,4322304) | 20260506 20:44:38.438 | INFO | MemInfo: { | 20260506 20:44:38.438 | INFO | "gpu" : " 17 GB", | 20260506 20:44:38.438 | INFO | "huge" : "", | 20260506 20:44:38.438 | INFO | "huge_shared" : " 27 GB", | 20260506 20:44:38.438 | INFO | "malloc" : " 14 GB", | 20260506 20:44:38.438 | INFO | "pinned" : " 29 GB" | 20260506 20:44:38.438 | INFO | } | 20260506 20:44:38.438 | INFO | Perf report for the last 5.13735 seconds: | 20260506 20:44:38.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:44:38.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:38.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:38.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:38.438 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:44:38.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:38.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:38.438 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 20:44:38.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:38.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:38.438 | INFO | Uploaded 36GiB to GPU in 2.01803s (17.8392GiB/s) | 20260506 20:44:38.439 | INFO | Downloaded 34GiB to GPU in 1.92374s (17.6739GiB/s) | 20260506 20:44:43.582 | INFO | Analyzing chunk frames = [4322304,4322816) | 20260506 20:44:43.582 | INFO | MemInfo: { | 20260506 20:44:43.582 | INFO | "gpu" : " 17 GB", | 20260506 20:44:43.582 | INFO | "huge" : "", | 20260506 20:44:43.582 | INFO | "huge_shared" : " 27 GB", | 20260506 20:44:43.583 | INFO | "malloc" : " 14 GB", | 20260506 20:44:43.583 | INFO | "pinned" : " 29 GB" | 20260506 20:44:43.583 | INFO | } | 20260506 20:44:43.583 | INFO | Perf report for the last 5.14457 seconds: | 20260506 20:44:43.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:44:43.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:43.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:43.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:43.583 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 20:44:43.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:43.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:43.583 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:44:43.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:43.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:43.583 | INFO | Uploaded 36GiB to GPU in 1.98194s (18.1641GiB/s) | 20260506 20:44:43.583 | INFO | Downloaded 34GiB to GPU in 1.91967s (17.7114GiB/s) | 20260506 20:44:48.723 | INFO | Analyzing chunk frames = [4322816,4323328) | 20260506 20:44:48.723 | INFO | MemInfo: { | 20260506 20:44:48.723 | INFO | "gpu" : " 17 GB", | 20260506 20:44:48.723 | INFO | "huge" : "", | 20260506 20:44:48.723 | INFO | "huge_shared" : " 27 GB", | 20260506 20:44:48.723 | INFO | "malloc" : " 14 GB", | 20260506 20:44:48.723 | INFO | "pinned" : " 29 GB" | 20260506 20:44:48.723 | INFO | } | 20260506 20:44:48.724 | INFO | Perf report for the last 5.14095 seconds: | 20260506 20:44:48.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:44:48.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:48.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:48.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:48.724 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:44:48.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:48.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:48.724 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:44:48.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:48.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:48.724 | INFO | Uploaded 36GiB to GPU in 1.96097s (18.3583GiB/s) | 20260506 20:44:48.724 | INFO | Downloaded 34GiB to GPU in 1.87203s (18.1621GiB/s) | 20260506 20:44:53.865 | INFO | Analyzing chunk frames = [4323328,4323840) | 20260506 20:44:53.865 | INFO | MemInfo: { | 20260506 20:44:53.865 | INFO | "gpu" : " 17 GB", | 20260506 20:44:53.865 | INFO | "huge" : "", | 20260506 20:44:53.865 | INFO | "huge_shared" : " 27 GB", | 20260506 20:44:53.865 | INFO | "malloc" : " 14 GB", | 20260506 20:44:53.865 | INFO | "pinned" : " 29 GB" | 20260506 20:44:53.865 | INFO | } | 20260506 20:44:53.865 | INFO | Perf report for the last 5.14121 seconds: | 20260506 20:44:53.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:44:53.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:53.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:53.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:53.865 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260506 20:44:53.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:53.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:53.865 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:44:53.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:53.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:53.865 | INFO | Uploaded 36GiB to GPU in 2.00608s (17.9454GiB/s) | 20260506 20:44:53.865 | INFO | Downloaded 34GiB to GPU in 1.91416s (17.7624GiB/s) | 20260506 20:44:59.004 | INFO | Analyzing chunk frames = [4323840,4324352) | 20260506 20:44:59.004 | INFO | MemInfo: { | 20260506 20:44:59.004 | INFO | "gpu" : " 17 GB", | 20260506 20:44:59.004 | INFO | "huge" : "", | 20260506 20:44:59.004 | INFO | "huge_shared" : " 27 GB", | 20260506 20:44:59.004 | INFO | "malloc" : " 14 GB", | 20260506 20:44:59.004 | INFO | "pinned" : " 29 GB" | 20260506 20:44:59.004 | INFO | } | 20260506 20:44:59.004 | INFO | Perf report for the last 5.13927 seconds: | 20260506 20:44:59.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:44:59.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:59.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:59.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:59.004 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:44:59.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:59.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:59.004 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 20:44:59.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:59.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:44:59.004 | INFO | Uploaded 36GiB to GPU in 2.01413s (17.8737GiB/s) | 20260506 20:44:59.004 | INFO | Downloaded 34GiB to GPU in 1.9447s (17.4834GiB/s) | 20260506 20:45:04.137 | INFO | Analyzing chunk frames = [4324352,4324864) | 20260506 20:45:04.137 | INFO | MemInfo: { | 20260506 20:45:04.137 | INFO | "gpu" : " 17 GB", | 20260506 20:45:04.137 | INFO | "huge" : "", | 20260506 20:45:04.137 | INFO | "huge_shared" : " 27 GB", | 20260506 20:45:04.137 | INFO | "malloc" : " 14 GB", | 20260506 20:45:04.137 | INFO | "pinned" : " 29 GB" | 20260506 20:45:04.137 | INFO | } | 20260506 20:45:04.137 | INFO | Perf report for the last 5.13298 seconds: | 20260506 20:45:04.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:45:04.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:04.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:04.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:04.137 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 20:45:04.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:04.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:04.137 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:45:04.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:04.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:04.137 | INFO | Uploaded 36GiB to GPU in 1.96047s (18.3629GiB/s) | 20260506 20:45:04.137 | INFO | Downloaded 34GiB to GPU in 1.9935s (17.0554GiB/s) | 20260506 20:45:09.276 | INFO | Analyzing chunk frames = [4324864,4325376) | 20260506 20:45:09.276 | INFO | MemInfo: { | 20260506 20:45:09.276 | INFO | "gpu" : " 17 GB", | 20260506 20:45:09.276 | INFO | "huge" : "", | 20260506 20:45:09.276 | INFO | "huge_shared" : " 27 GB", | 20260506 20:45:09.276 | INFO | "malloc" : " 14 GB", | 20260506 20:45:09.276 | INFO | "pinned" : " 29 GB" | 20260506 20:45:09.276 | INFO | } | 20260506 20:45:09.276 | INFO | Perf report for the last 5.13925 seconds: | 20260506 20:45:09.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:45:09.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:09.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:09.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:09.276 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 20:45:09.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:09.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:09.276 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 20:45:09.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:09.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:09.276 | INFO | Uploaded 36GiB to GPU in 2.00545s (17.9511GiB/s) | 20260506 20:45:09.277 | INFO | Downloaded 34GiB to GPU in 1.98629s (17.1173GiB/s) | 20260506 20:45:09.277 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:45:09.277 | INFO | Total mem usage: 18 GiB | 20260506 20:45:09.277 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:45:09.277 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -14.1%) | 20260506 20:45:09.277 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:45:09.277 | INFO | High Water Breakdown: | 20260506 20:45:09.277 | INFO | 61.6% : 9.73 GiB ( 85.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:45:09.277 | INFO | 38.2% : 6.03 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:45:09.277 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:45:09.277 | INFO | ------------------------------------------ | 20260506 20:45:09.277 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:45:09.277 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:45:09.277 | INFO | Total mem usage: 26.5 GiB | 20260506 20:45:09.277 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:45:09.277 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 20:45:09.277 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 20:45:09.277 | INFO | High Water Breakdown: | 20260506 20:45:09.277 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:45:09.277 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:45:09.277 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:45:09.277 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:45:09.277 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:45:09.277 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:45:09.277 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:45:09.277 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:45:09.277 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:45:09.277 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:45:09.277 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:45:09.277 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:45:09.277 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:45:09.277 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:45:09.277 | INFO | ------------------------------------------ | 20260506 20:45:09.277 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:45:09.277 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:45:09.277 | INFO | Total mem usage: 25.6 GiB | 20260506 20:45:09.277 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:45:09.277 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:45:09.277 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:45:09.277 | INFO | High Water Breakdown: | 20260506 20:45:09.277 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:45:09.277 | INFO | ------------------------------------------ | 20260506 20:45:09.277 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:45:09.277 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:45:09.277 | INFO | Total mem usage: 15.7 GiB | 20260506 20:45:09.277 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:45:09.277 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:45:09.277 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:45:09.277 | INFO | High Water Breakdown: | 20260506 20:45:09.277 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:45:09.277 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:45:09.277 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:45:09.277 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:45:09.277 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:45:09.277 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:45:09.277 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:45:09.277 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:45:09.277 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:45:09.277 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:45:09.277 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:45:09.277 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:45:09.277 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:45:09.277 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:45:09.277 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:45:09.277 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:45:09.277 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:45:09.277 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:45:09.277 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:45:09.277 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:45:09.277 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:45:09.277 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:45:09.277 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:45:09.277 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:45:09.277 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:45:09.277 | INFO | ------------------------------------------ | 20260506 20:45:09.277 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:45:09.277 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:45:09.277 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:45:09.277 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:45:09.278 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.56e+01, 1.33e+01 / 2.70e+00 | 20260506 20:45:09.278 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.00e-01, 1.05e-02 / 7.38e-03 | 20260506 20:45:09.278 | INFO | PerfReport: 11.64%, Baseline -- 3.63e+00 / 4.34e+00, 5.50e-01 / 3.89e-01 | 20260506 20:45:09.278 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.82e-01 / 1.53e-01 | 20260506 20:45:09.278 | INFO | PerfReport: 16.03%, DME -- 4.00e+01 / 4.44e+01, 4.20e+00 / 3.14e+00 | 20260506 20:45:09.278 | INFO | PerfReport: 57.18%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.39e+00 / 1.82e+00 | 20260506 20:45:09.278 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.23e+00 / 2.71e+00, 3.85e-01 / 2.80e-01 | 20260506 20:45:09.278 | INFO | PerfReport: 3.31%, Metrics -- 1.03e+00 / 1.36e+00, 2.74e-01 / 1.16e-01 | 20260506 20:45:09.278 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.53e-01 / 3.96e-01, 5.44e-02 / 1.76e-02 | 20260506 20:45:09.278 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:45:09.278 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:45:09.278 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:45:09.278 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.31e+01 / 0.00e+00 | 20260506 20:45:09.278 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260506 20:45:09.278 | INFO | PerfReport: 21.25%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 20:45:09.278 | INFO | PerfReport: 32.24%, Compute -- 3.12e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 20:45:09.278 | INFO | PerfReport: 1.31%, LockUpload -- 1.27e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260506 20:45:09.278 | INFO | PerfReport: 1.18%, LockDownload -- 1.14e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 20:45:09.278 | INFO | PerfReport: 22.49%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 20:45:09.278 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.43e-02 / 0.00e+00 | 20260506 20:45:09.377 | INFO | Starting a round of Baz IO | 20260506 20:45:12.558 | INFO | Populating RTMetrics report | 20260506 20:45:12.559 | INFO | RTMetrics report populated | 20260506 20:45:12.559 | INFO | Opening tmp RTMetrics json file | 20260506 20:45:12.562 | INFO | Serializing RTMetrics to json | 20260506 20:45:12.566 | INFO | Serialization done, swapping tmp file now... | 20260506 20:45:12.571 | INFO | RTMetrics done | 20260506 20:45:12.909 | INFO | Finished a round of Baz IO: Wrote 1.969 GiB in 3.532 seconds | 20260506 20:45:12.909 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.15% / 0.00% / 65.85% | 20260506 20:45:12.909 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.707392% / 7.94729e-06% / 5.16574e-05% | 20260506 20:45:12.909 | INFO | Baz IO overall: | 20260506 20:45:12.909 | INFO | BytesWritten : 2282483077618 | 20260506 20:45:12.910 | INFO | NumEvents : 1102531449705 | 20260506 20:45:14.420 | INFO | Analyzing chunk frames = [4325376,4325888) | 20260506 20:45:14.420 | INFO | MemInfo: { | 20260506 20:45:14.420 | INFO | "gpu" : " 17 GB", | 20260506 20:45:14.420 | INFO | "huge" : "", | 20260506 20:45:14.420 | INFO | "huge_shared" : " 27 GB", | 20260506 20:45:14.420 | INFO | "malloc" : " 14 GB", | 20260506 20:45:14.420 | INFO | "pinned" : " 29 GB" | 20260506 20:45:14.420 | INFO | } | 20260506 20:45:14.420 | INFO | Perf report for the last 5.14396 seconds: | 20260506 20:45:14.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:45:14.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:14.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:14.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:14.420 | INFO | ANALYSIS : 62.8%, 2.8, 93.1ms, 0 | 20260506 20:45:14.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:14.420 | INFO | RT_METRICS : 46.3%, 1.0, 24.3ms, 0 | 20260506 20:45:14.420 | INFO | PRE_HQ : 60.6%, 2.0, 61.7ms, 0 | 20260506 20:45:14.420 | INFO | BAZWRITER : 68.3%, 4.8, 169.2ms, 0 | 20260506 20:45:14.420 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 20:45:14.421 | INFO | Uploaded 36GiB to GPU in 2.06294s (17.4508GiB/s) | 20260506 20:45:14.421 | INFO | Downloaded 38GiB to GPU in 2.1622s (17.5747GiB/s) | 20260506 20:45:19.557 | INFO | Analyzing chunk frames = [4325888,4326400) | 20260506 20:45:19.557 | INFO | MemInfo: { | 20260506 20:45:19.557 | INFO | "gpu" : " 17 GB", | 20260506 20:45:19.557 | INFO | "huge" : "", | 20260506 20:45:19.557 | INFO | "huge_shared" : " 27 GB", | 20260506 20:45:19.557 | INFO | "malloc" : " 14 GB", | 20260506 20:45:19.557 | INFO | "pinned" : " 29 GB" | 20260506 20:45:19.557 | INFO | } | 20260506 20:45:19.557 | INFO | Perf report for the last 5.1369 seconds: | 20260506 20:45:19.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:45:19.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:19.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:19.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:19.557 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:45:19.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:19.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:19.557 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:45:19.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:19.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:19.557 | INFO | Uploaded 36GiB to GPU in 2.0164s (17.8536GiB/s) | 20260506 20:45:19.557 | INFO | Downloaded 34GiB to GPU in 1.9238s (17.6734GiB/s) | 20260506 20:45:24.698 | INFO | Analyzing chunk frames = [4326400,4326912) | 20260506 20:45:24.698 | INFO | MemInfo: { | 20260506 20:45:24.698 | INFO | "gpu" : " 17 GB", | 20260506 20:45:24.698 | INFO | "huge" : "", | 20260506 20:45:24.698 | INFO | "huge_shared" : " 27 GB", | 20260506 20:45:24.698 | INFO | "malloc" : " 14 GB", | 20260506 20:45:24.698 | INFO | "pinned" : " 29 GB" | 20260506 20:45:24.698 | INFO | } | 20260506 20:45:24.698 | INFO | Perf report for the last 5.141 seconds: | 20260506 20:45:24.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:45:24.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:24.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:24.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:24.698 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 20:45:24.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:24.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:24.698 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:45:24.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:24.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:24.699 | INFO | Uploaded 36GiB to GPU in 1.98037s (18.1784GiB/s) | 20260506 20:45:24.699 | INFO | Downloaded 34GiB to GPU in 1.9162s (17.7434GiB/s) | 20260506 20:45:29.837 | INFO | Analyzing chunk frames = [4326912,4327424) | 20260506 20:45:29.837 | INFO | MemInfo: { | 20260506 20:45:29.837 | INFO | "gpu" : " 17 GB", | 20260506 20:45:29.837 | INFO | "huge" : "", | 20260506 20:45:29.837 | INFO | "huge_shared" : " 27 GB", | 20260506 20:45:29.837 | INFO | "malloc" : " 14 GB", | 20260506 20:45:29.837 | INFO | "pinned" : " 29 GB" | 20260506 20:45:29.837 | INFO | } | 20260506 20:45:29.837 | INFO | Perf report for the last 5.13898 seconds: | 20260506 20:45:29.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:45:29.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:29.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:29.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:29.837 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:45:29.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:29.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:29.837 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 20:45:29.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:29.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:29.837 | INFO | Uploaded 36GiB to GPU in 1.94532s (18.5059GiB/s) | 20260506 20:45:29.837 | INFO | Downloaded 34GiB to GPU in 1.87165s (18.1658GiB/s) | 20260506 20:45:34.979 | INFO | Analyzing chunk frames = [4327424,4327936) | 20260506 20:45:34.979 | INFO | MemInfo: { | 20260506 20:45:34.979 | INFO | "gpu" : " 17 GB", | 20260506 20:45:34.979 | INFO | "huge" : "", | 20260506 20:45:34.979 | INFO | "huge_shared" : " 27 GB", | 20260506 20:45:34.979 | INFO | "malloc" : " 14 GB", | 20260506 20:45:34.979 | INFO | "pinned" : " 29 GB" | 20260506 20:45:34.979 | INFO | } | 20260506 20:45:34.979 | INFO | Perf report for the last 5.14226 seconds: | 20260506 20:45:34.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:45:34.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:34.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:34.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:34.979 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 20:45:34.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:34.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:34.980 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:45:34.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:34.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:34.980 | INFO | Uploaded 36GiB to GPU in 2.01814s (17.8382GiB/s) | 20260506 20:45:34.980 | INFO | Downloaded 34GiB to GPU in 1.90529s (17.8451GiB/s) | 20260506 20:45:40.113 | INFO | Analyzing chunk frames = [4327936,4328448) | 20260506 20:45:40.113 | INFO | MemInfo: { | 20260506 20:45:40.114 | INFO | "gpu" : " 17 GB", | 20260506 20:45:40.114 | INFO | "huge" : "", | 20260506 20:45:40.114 | INFO | "huge_shared" : " 27 GB", | 20260506 20:45:40.114 | INFO | "malloc" : " 14 GB", | 20260506 20:45:40.114 | INFO | "pinned" : " 29 GB" | 20260506 20:45:40.114 | INFO | } | 20260506 20:45:40.114 | INFO | Perf report for the last 5.13418 seconds: | 20260506 20:45:40.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:45:40.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:40.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:40.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:40.114 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 20:45:40.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:40.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:40.114 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:45:40.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:40.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:40.114 | INFO | Uploaded 36GiB to GPU in 2.05932s (17.4815GiB/s) | 20260506 20:45:40.114 | INFO | Downloaded 34GiB to GPU in 1.96084s (17.3395GiB/s) | 20260506 20:45:45.252 | INFO | Analyzing chunk frames = [4328448,4328960) | 20260506 20:45:45.252 | INFO | MemInfo: { | 20260506 20:45:45.252 | INFO | "gpu" : " 17 GB", | 20260506 20:45:45.253 | INFO | "huge" : "", | 20260506 20:45:45.253 | INFO | "huge_shared" : " 27 GB", | 20260506 20:45:45.253 | INFO | "malloc" : " 14 GB", | 20260506 20:45:45.253 | INFO | "pinned" : " 29 GB" | 20260506 20:45:45.253 | INFO | } | 20260506 20:45:45.253 | INFO | Perf report for the last 5.13901 seconds: | 20260506 20:45:45.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:45:45.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:45.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:45.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:45.253 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 20:45:45.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:45.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:45.253 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 20:45:45.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:45.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:45.253 | INFO | Uploaded 36GiB to GPU in 1.99273s (18.0656GiB/s) | 20260506 20:45:45.253 | INFO | Downloaded 34GiB to GPU in 1.95559s (17.386GiB/s) | 20260506 20:45:50.388 | INFO | Analyzing chunk frames = [4328960,4329472) | 20260506 20:45:50.388 | INFO | MemInfo: { | 20260506 20:45:50.388 | INFO | "gpu" : " 17 GB", | 20260506 20:45:50.388 | INFO | "huge" : "", | 20260506 20:45:50.388 | INFO | "huge_shared" : " 27 GB", | 20260506 20:45:50.388 | INFO | "malloc" : " 14 GB", | 20260506 20:45:50.388 | INFO | "pinned" : " 29 GB" | 20260506 20:45:50.388 | INFO | } | 20260506 20:45:50.388 | INFO | Perf report for the last 5.13576 seconds: | 20260506 20:45:50.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:45:50.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:50.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:50.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:50.388 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 20:45:50.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:50.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:50.388 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 20:45:50.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:50.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:50.389 | INFO | Uploaded 36GiB to GPU in 2.0483s (17.5756GiB/s) | 20260506 20:45:50.389 | INFO | Downloaded 34GiB to GPU in 1.9928s (17.0614GiB/s) | 20260506 20:45:50.506 | INFO | Starting a round of Baz IO | 20260506 20:45:53.633 | INFO | Populating RTMetrics report | 20260506 20:45:53.634 | INFO | RTMetrics report populated | 20260506 20:45:53.634 | INFO | Opening tmp RTMetrics json file | 20260506 20:45:53.635 | INFO | Serializing RTMetrics to json | 20260506 20:45:53.637 | INFO | Serialization done, swapping tmp file now... | 20260506 20:45:53.637 | INFO | RTMetrics done | 20260506 20:45:54.224 | INFO | Finished a round of Baz IO: Wrote 1.969 GiB in 3.718 seconds | 20260506 20:45:54.224 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.14% / 0.00% / 65.85% | 20260506 20:45:54.224 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.705918% / 0% / 7.94729e-05% | 20260506 20:45:54.224 | INFO | Baz IO overall: | 20260506 20:45:54.224 | INFO | BytesWritten : 2284597762331 | 20260506 20:45:54.224 | INFO | NumEvents : 1103545354355 | 20260506 20:45:55.528 | INFO | Analyzing chunk frames = [4329472,4329984) | 20260506 20:45:55.528 | INFO | MemInfo: { | 20260506 20:45:55.528 | INFO | "gpu" : " 17 GB", | 20260506 20:45:55.528 | INFO | "huge" : "", | 20260506 20:45:55.528 | INFO | "huge_shared" : " 27 GB", | 20260506 20:45:55.528 | INFO | "malloc" : " 14 GB", | 20260506 20:45:55.528 | INFO | "pinned" : " 29 GB" | 20260506 20:45:55.528 | INFO | } | 20260506 20:45:55.528 | INFO | Perf report for the last 5.13995 seconds: | 20260506 20:45:55.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:45:55.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:55.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:55.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:55.528 | INFO | ANALYSIS : 62.6%, 2.9, 95.7ms, 0 | 20260506 20:45:55.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:45:55.528 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 20:45:55.528 | INFO | PRE_HQ : 59.7%, 1.8, 56.9ms, 0 | 20260506 20:45:55.528 | INFO | BAZWRITER : 72.0%, 4.8, 175.9ms, 0 | 20260506 20:45:55.528 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 20:45:55.529 | INFO | Uploaded 36GiB to GPU in 2.06969s (17.3939GiB/s) | 20260506 20:45:55.529 | INFO | Downloaded 38GiB to GPU in 2.22059s (17.1126GiB/s) | 20260506 20:46:00.681 | INFO | Analyzing chunk frames = [4329984,4330496) | 20260506 20:46:00.681 | INFO | MemInfo: { | 20260506 20:46:00.681 | INFO | "gpu" : " 17 GB", | 20260506 20:46:00.681 | INFO | "huge" : "", | 20260506 20:46:00.681 | INFO | "huge_shared" : " 27 GB", | 20260506 20:46:00.681 | INFO | "malloc" : " 14 GB", | 20260506 20:46:00.681 | INFO | "pinned" : " 29 GB" | 20260506 20:46:00.681 | INFO | } | 20260506 20:46:00.681 | INFO | Perf report for the last 5.15275 seconds: | 20260506 20:46:00.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:46:00.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:00.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:00.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:00.681 | INFO | ANALYSIS : 61.3%, 3.0, 95.8ms, 0 | 20260506 20:46:00.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:00.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:00.681 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 20:46:00.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:00.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:00.681 | INFO | Uploaded 36GiB to GPU in 2.01957s (17.8256GiB/s) | 20260506 20:46:00.681 | INFO | Downloaded 34GiB to GPU in 1.92606s (17.6526GiB/s) | 20260506 20:46:05.814 | INFO | Analyzing chunk frames = [4330496,4331008) | 20260506 20:46:05.814 | INFO | MemInfo: { | 20260506 20:46:05.814 | INFO | "gpu" : " 17 GB", | 20260506 20:46:05.814 | INFO | "huge" : "", | 20260506 20:46:05.814 | INFO | "huge_shared" : " 27 GB", | 20260506 20:46:05.814 | INFO | "malloc" : " 14 GB", | 20260506 20:46:05.814 | INFO | "pinned" : " 29 GB" | 20260506 20:46:05.814 | INFO | } | 20260506 20:46:05.814 | INFO | Perf report for the last 5.13273 seconds: | 20260506 20:46:05.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:46:05.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:05.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:05.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:05.814 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 20:46:05.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:05.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:05.814 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:46:05.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:05.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:05.814 | INFO | Uploaded 36GiB to GPU in 1.98973s (18.0929GiB/s) | 20260506 20:46:05.814 | INFO | Downloaded 34GiB to GPU in 1.92634s (17.65GiB/s) | 20260506 20:46:10.952 | INFO | Analyzing chunk frames = [4331008,4331520) | 20260506 20:46:10.952 | INFO | MemInfo: { | 20260506 20:46:10.952 | INFO | "gpu" : " 17 GB", | 20260506 20:46:10.952 | INFO | "huge" : "", | 20260506 20:46:10.952 | INFO | "huge_shared" : " 27 GB", | 20260506 20:46:10.952 | INFO | "malloc" : " 14 GB", | 20260506 20:46:10.952 | INFO | "pinned" : " 29 GB" | 20260506 20:46:10.952 | INFO | } | 20260506 20:46:10.952 | INFO | Perf report for the last 5.13848 seconds: | 20260506 20:46:10.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:46:10.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:10.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:10.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:10.952 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 20:46:10.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:10.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:10.952 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:46:10.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:10.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:10.953 | INFO | Uploaded 36GiB to GPU in 1.97848s (18.1958GiB/s) | 20260506 20:46:10.953 | INFO | Downloaded 34GiB to GPU in 1.87167s (18.1656GiB/s) | 20260506 20:46:16.090 | INFO | Analyzing chunk frames = [4331520,4332032) | 20260506 20:46:16.090 | INFO | MemInfo: { | 20260506 20:46:16.090 | INFO | "gpu" : " 17 GB", | 20260506 20:46:16.090 | INFO | "huge" : "", | 20260506 20:46:16.090 | INFO | "huge_shared" : " 27 GB", | 20260506 20:46:16.090 | INFO | "malloc" : " 14 GB", | 20260506 20:46:16.090 | INFO | "pinned" : " 29 GB" | 20260506 20:46:16.090 | INFO | } | 20260506 20:46:16.090 | INFO | Perf report for the last 5.13756 seconds: | 20260506 20:46:16.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:46:16.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:16.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:16.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:16.090 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 20:46:16.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:16.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:16.090 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:46:16.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:16.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:16.090 | INFO | Uploaded 36GiB to GPU in 2.01272s (17.8862GiB/s) | 20260506 20:46:16.090 | INFO | Downloaded 34GiB to GPU in 1.91582s (17.747GiB/s) | 20260506 20:46:21.228 | INFO | Analyzing chunk frames = [4332032,4332544) | 20260506 20:46:21.228 | INFO | MemInfo: { | 20260506 20:46:21.228 | INFO | "gpu" : " 17 GB", | 20260506 20:46:21.228 | INFO | "huge" : "", | 20260506 20:46:21.228 | INFO | "huge_shared" : " 27 GB", | 20260506 20:46:21.228 | INFO | "malloc" : " 14 GB", | 20260506 20:46:21.228 | INFO | "pinned" : " 29 GB" | 20260506 20:46:21.228 | INFO | } | 20260506 20:46:21.228 | INFO | Perf report for the last 5.13808 seconds: | 20260506 20:46:21.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:46:21.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:21.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:21.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:21.228 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:46:21.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:21.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:21.228 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 20:46:21.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:21.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:21.228 | INFO | Uploaded 36GiB to GPU in 2.00532s (17.9523GiB/s) | 20260506 20:46:21.228 | INFO | Downloaded 34GiB to GPU in 1.94854s (17.4489GiB/s) | 20260506 20:46:26.367 | INFO | Analyzing chunk frames = [4332544,4333056) | 20260506 20:46:26.367 | INFO | MemInfo: { | 20260506 20:46:26.367 | INFO | "gpu" : " 17 GB", | 20260506 20:46:26.367 | INFO | "huge" : "", | 20260506 20:46:26.367 | INFO | "huge_shared" : " 27 GB", | 20260506 20:46:26.367 | INFO | "malloc" : " 14 GB", | 20260506 20:46:26.367 | INFO | "pinned" : " 29 GB" | 20260506 20:46:26.367 | INFO | } | 20260506 20:46:26.367 | INFO | Perf report for the last 5.13944 seconds: | 20260506 20:46:26.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:46:26.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:26.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:26.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:26.368 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 20:46:26.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:26.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:26.368 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 20:46:26.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:26.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:26.368 | INFO | Uploaded 36GiB to GPU in 1.98234s (18.1604GiB/s) | 20260506 20:46:26.368 | INFO | Downloaded 34GiB to GPU in 1.99854s (17.0124GiB/s) | 20260506 20:46:31.511 | INFO | Analyzing chunk frames = [4333056,4333568) | 20260506 20:46:31.511 | INFO | MemInfo: { | 20260506 20:46:31.511 | INFO | "gpu" : " 17 GB", | 20260506 20:46:31.511 | INFO | "huge" : "", | 20260506 20:46:31.511 | INFO | "huge_shared" : " 27 GB", | 20260506 20:46:31.511 | INFO | "malloc" : " 14 GB", | 20260506 20:46:31.511 | INFO | "pinned" : " 29 GB" | 20260506 20:46:31.511 | INFO | } | 20260506 20:46:31.511 | INFO | Perf report for the last 5.14355 seconds: | 20260506 20:46:31.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:46:31.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:31.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:31.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:31.511 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:46:31.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:31.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:31.511 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 20:46:31.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:31.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:31.511 | INFO | Uploaded 36GiB to GPU in 1.98741s (18.114GiB/s) | 20260506 20:46:31.511 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:46:31.511 | INFO | Downloaded 34GiB to GPU in 1.98865s (17.097GiB/s) | 20260506 20:46:31.511 | INFO | Total mem usage: 18 GiB | 20260506 20:46:31.511 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:46:31.511 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -13.6%) | 20260506 20:46:31.511 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:46:31.511 | INFO | High Water Breakdown: | 20260506 20:46:31.511 | INFO | 61.8% : 9.78 GiB ( 85.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:46:31.511 | INFO | 38.1% : 6.04 GiB ( 90.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:46:31.511 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:46:31.511 | INFO | ------------------------------------------ | 20260506 20:46:31.511 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:46:31.512 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:46:31.512 | INFO | Total mem usage: 26.5 GiB | 20260506 20:46:31.512 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:46:31.512 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260506 20:46:31.512 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260506 20:46:31.512 | INFO | High Water Breakdown: | 20260506 20:46:31.512 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:46:31.512 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:46:31.512 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:46:31.512 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:46:31.512 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:46:31.512 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:46:31.512 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:46:31.512 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:46:31.512 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:46:31.512 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:46:31.512 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:46:31.512 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:46:31.512 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:46:31.512 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:46:31.512 | INFO | ------------------------------------------ | 20260506 20:46:31.512 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:46:31.512 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:46:31.512 | INFO | Total mem usage: 25.6 GiB | 20260506 20:46:31.512 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:46:31.512 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:46:31.512 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:46:31.512 | INFO | High Water Breakdown: | 20260506 20:46:31.512 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:46:31.512 | INFO | ------------------------------------------ | 20260506 20:46:31.512 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:46:31.512 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:46:31.512 | INFO | Total mem usage: 15.7 GiB | 20260506 20:46:31.512 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:46:31.512 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:46:31.512 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:46:31.512 | INFO | High Water Breakdown: | 20260506 20:46:31.512 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:46:31.512 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:46:31.512 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:46:31.512 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:46:31.512 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:46:31.512 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:46:31.512 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:46:31.512 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:46:31.512 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:46:31.512 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:46:31.512 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:46:31.512 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:46:31.512 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:46:31.512 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:46:31.512 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:46:31.512 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:46:31.512 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:46:31.512 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:46:31.512 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:46:31.512 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:46:31.512 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:46:31.512 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:46:31.512 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:46:31.512 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:46:31.512 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:46:31.512 | INFO | ------------------------------------------ | 20260506 20:46:31.512 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:46:31.512 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:46:31.512 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:46:31.512 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:46:31.512 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.53e+01, 1.35e+01 / 2.40e+00 | 20260506 20:46:31.512 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 1.02e-02 / 4.91e-03 | 20260506 20:46:31.512 | INFO | PerfReport: 11.64%, Baseline -- 3.62e+00 / 4.33e+00, 5.48e-01 / 3.48e-01 | 20260506 20:46:31.512 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.82e-01 / 1.20e-01 | 20260506 20:46:31.512 | INFO | PerfReport: 15.99%, DME -- 3.98e+01 / 4.41e+01, 3.83e+00 / 2.38e+00 | 20260506 20:46:31.512 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.42e+00 / 1.96e+00 | 20260506 20:46:31.512 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.22e+00 / 2.76e+00, 3.80e-01 / 2.33e-01 | 20260506 20:46:31.512 | INFO | PerfReport: 3.30%, Metrics -- 1.03e+00 / 1.37e+00, 2.69e-01 / 1.51e-01 | 20260506 20:46:31.512 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.52e-01 / 3.97e-01, 5.43e-02 / 2.12e-02 | 20260506 20:46:31.512 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:46:31.512 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:46:31.512 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:46:31.512 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 20:46:31.512 | INFO | PerfReport: 21.67%, Upload -- 2.09e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260506 20:46:31.513 | INFO | PerfReport: 21.21%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 20:46:31.513 | INFO | PerfReport: 32.19%, Compute -- 3.11e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 20:46:31.513 | INFO | PerfReport: 1.45%, LockUpload -- 1.40e+00 / 0.00e+00, 5.07e+00 / 0.00e+00 | 20260506 20:46:31.513 | INFO | PerfReport: 1.11%, LockDownload -- 1.07e+00 / 0.00e+00, 2.70e+00 / 0.00e+00 | 20260506 20:46:31.513 | INFO | PerfReport: 22.36%, LockCompute -- 2.16e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 20:46:31.513 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260506 20:46:31.610 | INFO | Starting a round of Baz IO | 20260506 20:46:34.747 | INFO | Populating RTMetrics report | 20260506 20:46:34.747 | INFO | RTMetrics report populated | 20260506 20:46:34.747 | INFO | Opening tmp RTMetrics json file | 20260506 20:46:34.754 | INFO | Serializing RTMetrics to json | 20260506 20:46:34.758 | INFO | Serialization done, swapping tmp file now... | 20260506 20:46:34.758 | INFO | RTMetrics done | 20260506 20:46:35.313 | INFO | Finished a round of Baz IO: Wrote 1.969 GiB in 3.703 seconds | 20260506 20:46:35.313 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.14% / 0.00% / 65.86% | 20260506 20:46:35.313 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.704137% / 1.19209e-05% / 4.76837e-05% | 20260506 20:46:35.313 | INFO | Baz IO overall: | 20260506 20:46:35.313 | INFO | BytesWritten : 2286711524275 | 20260506 20:46:35.313 | INFO | NumEvents : 1104558810913 | 20260506 20:46:36.646 | INFO | Analyzing chunk frames = [4333568,4334080) | 20260506 20:46:36.646 | INFO | MemInfo: { | 20260506 20:46:36.646 | INFO | "gpu" : " 17 GB", | 20260506 20:46:36.646 | INFO | "huge" : "", | 20260506 20:46:36.646 | INFO | "huge_shared" : " 27 GB", | 20260506 20:46:36.646 | INFO | "malloc" : " 14 GB", | 20260506 20:46:36.646 | INFO | "pinned" : " 29 GB" | 20260506 20:46:36.646 | INFO | } | 20260506 20:46:36.646 | INFO | Perf report for the last 5.13463 seconds: | 20260506 20:46:36.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:46:36.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:36.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:36.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:36.646 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 20:46:36.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:36.646 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 20:46:36.646 | INFO | PRE_HQ : 60.0%, 1.8, 57.0ms, 0 | 20260506 20:46:36.646 | INFO | BAZWRITER : 71.7%, 4.8, 176.8ms, 0 | 20260506 20:46:36.646 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 20:46:36.646 | INFO | Uploaded 36GiB to GPU in 2.06898s (17.3999GiB/s) | 20260506 20:46:36.646 | INFO | Downloaded 38GiB to GPU in 2.19567s (17.3068GiB/s) | 20260506 20:46:41.785 | INFO | Analyzing chunk frames = [4334080,4334592) | 20260506 20:46:41.785 | INFO | MemInfo: { | 20260506 20:46:41.785 | INFO | "gpu" : " 17 GB", | 20260506 20:46:41.785 | INFO | "huge" : "", | 20260506 20:46:41.785 | INFO | "huge_shared" : " 27 GB", | 20260506 20:46:41.785 | INFO | "malloc" : " 14 GB", | 20260506 20:46:41.785 | INFO | "pinned" : " 29 GB" | 20260506 20:46:41.785 | INFO | } | 20260506 20:46:41.785 | INFO | Perf report for the last 5.13942 seconds: | 20260506 20:46:41.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:46:41.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:41.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:41.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:41.785 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:46:41.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:41.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:41.785 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 20:46:41.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:41.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:41.785 | INFO | Uploaded 36GiB to GPU in 2.01779s (17.8413GiB/s) | 20260506 20:46:41.785 | INFO | Downloaded 34GiB to GPU in 1.9343s (17.5775GiB/s) | 20260506 20:46:46.930 | INFO | Analyzing chunk frames = [4334592,4335104) | 20260506 20:46:46.930 | INFO | MemInfo: { | 20260506 20:46:46.930 | INFO | "gpu" : " 17 GB", | 20260506 20:46:46.930 | INFO | "huge" : "", | 20260506 20:46:46.930 | INFO | "huge_shared" : " 27 GB", | 20260506 20:46:46.930 | INFO | "malloc" : " 14 GB", | 20260506 20:46:46.930 | INFO | "pinned" : " 29 GB" | 20260506 20:46:46.930 | INFO | } | 20260506 20:46:46.931 | INFO | Perf report for the last 5.14537 seconds: | 20260506 20:46:46.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:46:46.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:46.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:46.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:46.931 | INFO | ANALYSIS : 60.7%, 3.0, 94.1ms, 0 | 20260506 20:46:46.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:46.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:46.931 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:46:46.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:46.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:46.931 | INFO | Uploaded 36GiB to GPU in 1.98003s (18.1816GiB/s) | 20260506 20:46:46.931 | INFO | Downloaded 34GiB to GPU in 1.91944s (17.7135GiB/s) | 20260506 20:46:52.068 | INFO | Analyzing chunk frames = [4335104,4335616) | 20260506 20:46:52.068 | INFO | MemInfo: { | 20260506 20:46:52.068 | INFO | "gpu" : " 17 GB", | 20260506 20:46:52.068 | INFO | "huge" : "", | 20260506 20:46:52.068 | INFO | "huge_shared" : " 27 GB", | 20260506 20:46:52.068 | INFO | "malloc" : " 14 GB", | 20260506 20:46:52.068 | INFO | "pinned" : " 29 GB" | 20260506 20:46:52.068 | INFO | } | 20260506 20:46:52.068 | INFO | Perf report for the last 5.13738 seconds: | 20260506 20:46:52.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:46:52.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:52.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:52.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:52.068 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 20:46:52.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:52.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:52.068 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 20:46:52.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:52.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:52.068 | INFO | Uploaded 36GiB to GPU in 1.9456s (18.5033GiB/s) | 20260506 20:46:52.068 | INFO | Downloaded 34GiB to GPU in 1.86972s (18.1845GiB/s) | 20260506 20:46:57.203 | INFO | Analyzing chunk frames = [4335616,4336128) | 20260506 20:46:57.203 | INFO | MemInfo: { | 20260506 20:46:57.203 | INFO | "gpu" : " 17 GB", | 20260506 20:46:57.203 | INFO | "huge" : "", | 20260506 20:46:57.203 | INFO | "huge_shared" : " 27 GB", | 20260506 20:46:57.203 | INFO | "malloc" : " 14 GB", | 20260506 20:46:57.203 | INFO | "pinned" : " 29 GB" | 20260506 20:46:57.203 | INFO | } | 20260506 20:46:57.203 | INFO | Perf report for the last 5.13546 seconds: | 20260506 20:46:57.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:46:57.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:57.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:57.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:57.203 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 20:46:57.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:57.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:57.204 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:46:57.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:57.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:46:57.204 | INFO | Uploaded 36GiB to GPU in 2.00172s (17.9845GiB/s) | 20260506 20:46:57.204 | INFO | Downloaded 34GiB to GPU in 1.93084s (17.6089GiB/s) | 20260506 20:47:02.343 | INFO | Analyzing chunk frames = [4336128,4336640) | 20260506 20:47:02.343 | INFO | MemInfo: { | 20260506 20:47:02.343 | INFO | "gpu" : " 17 GB", | 20260506 20:47:02.343 | INFO | "huge" : "", | 20260506 20:47:02.343 | INFO | "huge_shared" : " 27 GB", | 20260506 20:47:02.343 | INFO | "malloc" : " 14 GB", | 20260506 20:47:02.343 | INFO | "pinned" : " 29 GB" | 20260506 20:47:02.343 | INFO | } | 20260506 20:47:02.343 | INFO | Perf report for the last 5.13937 seconds: | 20260506 20:47:02.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:47:02.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:02.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:02.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:02.343 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 20:47:02.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:02.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:02.343 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 20:47:02.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:02.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:02.343 | INFO | Uploaded 36GiB to GPU in 2.01392s (17.8756GiB/s) | 20260506 20:47:02.343 | INFO | Downloaded 34GiB to GPU in 1.94923s (17.4427GiB/s) | 20260506 20:47:07.480 | INFO | Analyzing chunk frames = [4336640,4337152) | 20260506 20:47:07.480 | INFO | MemInfo: { | 20260506 20:47:07.480 | INFO | "gpu" : " 17 GB", | 20260506 20:47:07.480 | INFO | "huge" : "", | 20260506 20:47:07.480 | INFO | "huge_shared" : " 27 GB", | 20260506 20:47:07.480 | INFO | "malloc" : " 14 GB", | 20260506 20:47:07.480 | INFO | "pinned" : " 29 GB" | 20260506 20:47:07.480 | INFO | } | 20260506 20:47:07.480 | INFO | Perf report for the last 5.13692 seconds: | 20260506 20:47:07.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:47:07.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:07.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:07.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:07.480 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 20:47:07.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:07.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:07.480 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:47:07.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:07.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:07.480 | INFO | Uploaded 36GiB to GPU in 1.9811s (18.1717GiB/s) | 20260506 20:47:07.480 | INFO | Downloaded 34GiB to GPU in 2.00594s (16.9497GiB/s) | 20260506 20:47:12.622 | INFO | Analyzing chunk frames = [4337152,4337664) | 20260506 20:47:12.622 | INFO | MemInfo: { | 20260506 20:47:12.622 | INFO | "gpu" : " 17 GB", | 20260506 20:47:12.622 | INFO | "huge" : "", | 20260506 20:47:12.622 | INFO | "huge_shared" : " 27 GB", | 20260506 20:47:12.622 | INFO | "malloc" : " 14 GB", | 20260506 20:47:12.622 | INFO | "pinned" : " 29 GB" | 20260506 20:47:12.622 | INFO | } | 20260506 20:47:12.622 | INFO | Perf report for the last 5.14227 seconds: | 20260506 20:47:12.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:47:12.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:12.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:12.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:12.622 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 20:47:12.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:12.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:12.622 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 20:47:12.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:12.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:12.622 | INFO | Uploaded 36GiB to GPU in 2.02207s (17.8035GiB/s) | 20260506 20:47:12.623 | INFO | Downloaded 34GiB to GPU in 1.99187s (17.0694GiB/s) | 20260506 20:47:12.731 | INFO | Starting a round of Baz IO | 20260506 20:47:15.860 | INFO | Populating RTMetrics report | 20260506 20:47:15.860 | INFO | RTMetrics report populated | 20260506 20:47:15.860 | INFO | Opening tmp RTMetrics json file | 20260506 20:47:15.867 | INFO | Serializing RTMetrics to json | 20260506 20:47:15.894 | INFO | Serialization done, swapping tmp file now... | 20260506 20:47:15.896 | INFO | RTMetrics done | 20260506 20:47:16.011 | INFO | Finished a round of Baz IO: Wrote 1.967 GiB in 3.279 seconds | 20260506 20:47:16.011 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.13% / 0.00% / 65.86% | 20260506 20:47:16.011 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.702071% / 3.97364e-06% / 6.75519e-05% | 20260506 20:47:16.011 | INFO | Baz IO overall: | 20260506 20:47:16.011 | INFO | BytesWritten : 2288824064586 | 20260506 20:47:16.011 | INFO | NumEvents : 1105571830571 | 20260506 20:47:17.768 | INFO | Analyzing chunk frames = [4337664,4338176) | 20260506 20:47:17.768 | INFO | MemInfo: { | 20260506 20:47:17.768 | INFO | "gpu" : " 17 GB", | 20260506 20:47:17.768 | INFO | "huge" : "", | 20260506 20:47:17.768 | INFO | "huge_shared" : " 27 GB", | 20260506 20:47:17.768 | INFO | "malloc" : " 14 GB", | 20260506 20:47:17.768 | INFO | "pinned" : " 29 GB" | 20260506 20:47:17.768 | INFO | } | 20260506 20:47:17.768 | INFO | Perf report for the last 5.14621 seconds: | 20260506 20:47:17.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:47:17.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:17.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:17.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:17.768 | INFO | ANALYSIS : 62.4%, 2.9, 95.0ms, 0 | 20260506 20:47:17.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:17.768 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 20:47:17.768 | INFO | PRE_HQ : 60.0%, 1.9, 57.7ms, 0 | 20260506 20:47:17.768 | INFO | BAZWRITER : 63.3%, 4.6, 149.7ms, 0 | 20260506 20:47:17.768 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 20:47:17.768 | INFO | Uploaded 36GiB to GPU in 2.05975s (17.4778GiB/s) | 20260506 20:47:17.769 | INFO | Downloaded 38GiB to GPU in 2.20405s (17.241GiB/s) | 20260506 20:47:22.901 | INFO | Analyzing chunk frames = [4338176,4338688) | 20260506 20:47:22.901 | INFO | MemInfo: { | 20260506 20:47:22.901 | INFO | "gpu" : " 17 GB", | 20260506 20:47:22.901 | INFO | "huge" : "", | 20260506 20:47:22.901 | INFO | "huge_shared" : " 27 GB", | 20260506 20:47:22.901 | INFO | "malloc" : " 14 GB", | 20260506 20:47:22.901 | INFO | "pinned" : " 29 GB" | 20260506 20:47:22.901 | INFO | } | 20260506 20:47:22.901 | INFO | Perf report for the last 5.13263 seconds: | 20260506 20:47:22.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:47:22.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:22.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:22.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:22.901 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 20:47:22.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:22.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:22.901 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:47:22.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:22.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:22.901 | INFO | Uploaded 36GiB to GPU in 2.01141s (17.8979GiB/s) | 20260506 20:47:22.901 | INFO | Downloaded 34GiB to GPU in 1.92453s (17.6667GiB/s) | 20260506 20:47:28.042 | INFO | Analyzing chunk frames = [4338688,4339200) | 20260506 20:47:28.042 | INFO | MemInfo: { | 20260506 20:47:28.042 | INFO | "gpu" : " 17 GB", | 20260506 20:47:28.042 | INFO | "huge" : "", | 20260506 20:47:28.042 | INFO | "huge_shared" : " 27 GB", | 20260506 20:47:28.042 | INFO | "malloc" : " 14 GB", | 20260506 20:47:28.042 | INFO | "pinned" : " 29 GB" | 20260506 20:47:28.042 | INFO | } | 20260506 20:47:28.042 | INFO | Perf report for the last 5.14101 seconds: | 20260506 20:47:28.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:47:28.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:28.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:28.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:28.042 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 20:47:28.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:28.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:28.042 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 20:47:28.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:28.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:28.042 | INFO | Uploaded 36GiB to GPU in 1.98126s (18.1703GiB/s) | 20260506 20:47:28.042 | INFO | Downloaded 34GiB to GPU in 1.91854s (17.7218GiB/s) | 20260506 20:47:33.180 | INFO | Analyzing chunk frames = [4339200,4339712) | 20260506 20:47:33.180 | INFO | MemInfo: { | 20260506 20:47:33.180 | INFO | "gpu" : " 17 GB", | 20260506 20:47:33.180 | INFO | "huge" : "", | 20260506 20:47:33.180 | INFO | "huge_shared" : " 27 GB", | 20260506 20:47:33.180 | INFO | "malloc" : " 14 GB", | 20260506 20:47:33.180 | INFO | "pinned" : " 29 GB" | 20260506 20:47:33.180 | INFO | } | 20260506 20:47:33.180 | INFO | Perf report for the last 5.13849 seconds: | 20260506 20:47:33.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:47:33.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:33.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:33.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:33.180 | INFO | ANALYSIS : 59.7%, 3.0, 91.7ms, 0 | 20260506 20:47:33.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:33.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:33.180 | INFO | PRE_HQ : 26.4%, 1.0, 13.6ms, 0 | 20260506 20:47:33.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:33.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:33.181 | INFO | Uploaded 36GiB to GPU in 1.97045s (18.27GiB/s) | 20260506 20:47:33.181 | INFO | Downloaded 34GiB to GPU in 1.87613s (18.1224GiB/s) | 20260506 20:47:38.321 | INFO | Analyzing chunk frames = [4339712,4340224) | 20260506 20:47:38.321 | INFO | MemInfo: { | 20260506 20:47:38.321 | INFO | "gpu" : " 17 GB", | 20260506 20:47:38.321 | INFO | "huge" : "", | 20260506 20:47:38.321 | INFO | "huge_shared" : " 27 GB", | 20260506 20:47:38.321 | INFO | "malloc" : " 14 GB", | 20260506 20:47:38.321 | INFO | "pinned" : " 29 GB" | 20260506 20:47:38.321 | INFO | } | 20260506 20:47:38.321 | INFO | Perf report for the last 5.14045 seconds: | 20260506 20:47:38.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:47:38.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:38.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:38.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:38.321 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260506 20:47:38.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:38.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:38.321 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:47:38.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:38.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:38.321 | INFO | Uploaded 36GiB to GPU in 2.01424s (17.8727GiB/s) | 20260506 20:47:38.321 | INFO | Downloaded 34GiB to GPU in 1.91743s (17.732GiB/s) | 20260506 20:47:43.457 | INFO | Analyzing chunk frames = [4340224,4340736) | 20260506 20:47:43.457 | INFO | MemInfo: { | 20260506 20:47:43.457 | INFO | "gpu" : " 17 GB", | 20260506 20:47:43.457 | INFO | "huge" : "", | 20260506 20:47:43.457 | INFO | "huge_shared" : " 27 GB", | 20260506 20:47:43.457 | INFO | "malloc" : " 14 GB", | 20260506 20:47:43.457 | INFO | "pinned" : " 29 GB" | 20260506 20:47:43.457 | INFO | } | 20260506 20:47:43.457 | INFO | Perf report for the last 5.13618 seconds: | 20260506 20:47:43.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:47:43.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:43.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:43.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:43.457 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:47:43.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:43.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:43.457 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:47:43.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:43.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:43.457 | INFO | Uploaded 36GiB to GPU in 2.01908s (17.8299GiB/s) | 20260506 20:47:43.457 | INFO | Downloaded 34GiB to GPU in 1.9531s (17.4083GiB/s) | 20260506 20:47:48.593 | INFO | Analyzing chunk frames = [4340736,4341248) | 20260506 20:47:48.593 | INFO | MemInfo: { | 20260506 20:47:48.593 | INFO | "gpu" : " 17 GB", | 20260506 20:47:48.594 | INFO | "huge" : "", | 20260506 20:47:48.594 | INFO | "huge_shared" : " 27 GB", | 20260506 20:47:48.594 | INFO | "malloc" : " 14 GB", | 20260506 20:47:48.594 | INFO | "pinned" : " 29 GB" | 20260506 20:47:48.594 | INFO | } | 20260506 20:47:48.594 | INFO | Perf report for the last 5.13667 seconds: | 20260506 20:47:48.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:47:48.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:48.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:48.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:48.594 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 20:47:48.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:48.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:48.594 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 20:47:48.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:48.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:48.594 | INFO | Uploaded 36GiB to GPU in 1.96392s (18.3307GiB/s) | 20260506 20:47:48.594 | INFO | Downloaded 34GiB to GPU in 1.99554s (17.038GiB/s) | 20260506 20:47:53.734 | INFO | Analyzing chunk frames = [4341248,4341760) | 20260506 20:47:53.734 | INFO | MemInfo: { | 20260506 20:47:53.734 | INFO | "gpu" : " 17 GB", | 20260506 20:47:53.734 | INFO | "huge" : "", | 20260506 20:47:53.734 | INFO | "huge_shared" : " 27 GB", | 20260506 20:47:53.734 | INFO | "malloc" : " 14 GB", | 20260506 20:47:53.734 | INFO | "pinned" : " 29 GB" | 20260506 20:47:53.734 | INFO | } | 20260506 20:47:53.734 | INFO | Perf report for the last 5.14042 seconds: | 20260506 20:47:53.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:47:53.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:53.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:53.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:53.734 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 20:47:53.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:53.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:53.734 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 20:47:53.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:53.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:53.734 | INFO | Uploaded 36GiB to GPU in 2.00116s (17.9896GiB/s) | 20260506 20:47:53.734 | INFO | Downloaded 34GiB to GPU in 1.99222s (17.0664GiB/s) | 20260506 20:47:53.735 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:47:53.735 | INFO | Total mem usage: 18 GiB | 20260506 20:47:53.735 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:47:53.735 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -12%) | 20260506 20:47:53.735 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:47:53.735 | INFO | High Water Breakdown: | 20260506 20:47:53.735 | INFO | 62% : 9.97 GiB ( 87.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:47:53.735 | INFO | 37.9% : 6.09 GiB ( 91% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:47:53.735 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:47:53.735 | INFO | ------------------------------------------ | 20260506 20:47:53.735 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:47:53.735 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:47:53.735 | INFO | Total mem usage: 26.5 GiB | 20260506 20:47:53.735 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:47:53.735 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:47:53.735 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:47:53.735 | INFO | High Water Breakdown: | 20260506 20:47:53.735 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:47:53.735 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:47:53.735 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:47:53.735 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:47:53.735 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:47:53.735 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:47:53.735 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:47:53.735 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:47:53.735 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:47:53.735 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:47:53.735 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:47:53.735 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:47:53.735 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:47:53.735 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:47:53.735 | INFO | ------------------------------------------ | 20260506 20:47:53.735 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:47:53.735 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:47:53.735 | INFO | Total mem usage: 25.6 GiB | 20260506 20:47:53.735 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:47:53.735 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:47:53.735 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:47:53.735 | INFO | High Water Breakdown: | 20260506 20:47:53.735 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:47:53.735 | INFO | ------------------------------------------ | 20260506 20:47:53.735 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:47:53.735 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:47:53.735 | INFO | Total mem usage: 15.7 GiB | 20260506 20:47:53.735 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:47:53.735 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:47:53.735 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 20:47:53.735 | INFO | High Water Breakdown: | 20260506 20:47:53.735 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:47:53.735 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:47:53.735 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:47:53.735 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:47:53.735 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:47:53.735 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:47:53.735 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:47:53.735 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:47:53.735 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:47:53.735 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:47:53.735 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:47:53.735 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:47:53.735 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:47:53.735 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:47:53.735 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:47:53.735 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:47:53.735 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:47:53.735 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:47:53.735 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:47:53.735 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:47:53.735 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:47:53.735 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:47:53.735 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:47:53.735 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:47:53.735 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:47:53.735 | INFO | ------------------------------------------ | 20260506 20:47:53.735 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:47:53.735 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:47:53.735 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:47:53.735 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:47:53.735 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.63e+01, 1.35e+01 / 2.76e+00 | 20260506 20:47:53.735 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.00e-01, 9.88e-03 / 5.96e-03 | 20260506 20:47:53.735 | INFO | PerfReport: 11.64%, Baseline -- 3.63e+00 / 4.38e+00, 5.52e-01 / 3.28e-01 | 20260506 20:47:53.735 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.43e+00, 1.79e-01 / 1.31e-01 | 20260506 20:47:53.735 | INFO | PerfReport: 16.09%, DME -- 4.01e+01 / 4.55e+01, 4.46e+00 / 3.07e+00 | 20260506 20:47:53.735 | INFO | PerfReport: 57.13%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.34e+00 / 2.03e+00 | 20260506 20:47:53.736 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.23e+00 / 2.78e+00, 3.84e-01 / 2.54e-01 | 20260506 20:47:53.736 | INFO | PerfReport: 3.30%, Metrics -- 1.03e+00 / 1.36e+00, 2.70e-01 / 1.46e-01 | 20260506 20:47:53.736 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.53e-01 / 3.97e-01, 5.46e-02 / 1.64e-02 | 20260506 20:47:53.736 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:47:53.736 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:47:53.736 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:47:53.736 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 20:47:53.736 | INFO | PerfReport: 21.56%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 20:47:53.736 | INFO | PerfReport: 21.24%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 20:47:53.736 | INFO | PerfReport: 32.18%, Compute -- 3.11e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 20:47:53.736 | INFO | PerfReport: 1.33%, LockUpload -- 1.28e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260506 20:47:53.736 | INFO | PerfReport: 1.17%, LockDownload -- 1.13e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 20:47:53.736 | INFO | PerfReport: 22.51%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 20:47:53.736 | INFO | PerfReport: 0.02%, Misc -- 1.50e-02 / 0.00e+00, 2.67e-02 / 0.00e+00 | 20260506 20:47:53.843 | INFO | Starting a round of Baz IO | 20260506 20:47:56.981 | INFO | Populating RTMetrics report | 20260506 20:47:56.982 | INFO | RTMetrics report populated | 20260506 20:47:56.982 | INFO | Opening tmp RTMetrics json file | 20260506 20:47:56.986 | INFO | Serializing RTMetrics to json | 20260506 20:47:57.021 | INFO | Serialization done, swapping tmp file now... | 20260506 20:47:57.049 | INFO | RTMetrics done | 20260506 20:47:57.159 | INFO | Finished a round of Baz IO: Wrote 1.967 GiB in 3.315 seconds | 20260506 20:47:57.159 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.13% / 0.00% / 65.87% | 20260506 20:47:57.159 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.698451% / 7.94729e-06% / 3.57628e-05% | 20260506 20:47:57.159 | INFO | Baz IO overall: | 20260506 20:47:57.159 | INFO | BytesWritten : 2290935779707 | 20260506 20:47:57.159 | INFO | NumEvents : 1106584228702 | 20260506 20:47:58.876 | INFO | Analyzing chunk frames = [4341760,4342272) | 20260506 20:47:58.876 | INFO | MemInfo: { | 20260506 20:47:58.876 | INFO | "gpu" : " 17 GB", | 20260506 20:47:58.876 | INFO | "huge" : "", | 20260506 20:47:58.876 | INFO | "huge_shared" : " 27 GB", | 20260506 20:47:58.876 | INFO | "malloc" : " 14 GB", | 20260506 20:47:58.876 | INFO | "pinned" : " 29 GB" | 20260506 20:47:58.876 | INFO | } | 20260506 20:47:58.876 | INFO | Perf report for the last 5.14159 seconds: | 20260506 20:47:58.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:47:58.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:58.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:58.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:58.876 | INFO | ANALYSIS : 62.6%, 2.9, 95.2ms, 0 | 20260506 20:47:58.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:47:58.876 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 20:47:58.876 | INFO | PRE_HQ : 60.1%, 1.8, 57.6ms, 0 | 20260506 20:47:58.876 | INFO | BAZWRITER : 64.1%, 4.5, 148.9ms, 0 | 20260506 20:47:58.876 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 20:47:58.876 | INFO | Uploaded 36GiB to GPU in 2.05908s (17.4835GiB/s) | 20260506 20:47:58.876 | INFO | Downloaded 38GiB to GPU in 2.19939s (17.2775GiB/s) | 20260506 20:48:04.018 | INFO | Analyzing chunk frames = [4342272,4342784) | 20260506 20:48:04.018 | INFO | MemInfo: { | 20260506 20:48:04.018 | INFO | "gpu" : " 17 GB", | 20260506 20:48:04.018 | INFO | "huge" : "", | 20260506 20:48:04.018 | INFO | "huge_shared" : " 27 GB", | 20260506 20:48:04.018 | INFO | "malloc" : " 14 GB", | 20260506 20:48:04.018 | INFO | "pinned" : " 29 GB" | 20260506 20:48:04.019 | INFO | } | 20260506 20:48:04.019 | INFO | Perf report for the last 5.14292 seconds: | 20260506 20:48:04.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:48:04.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:04.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:04.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:04.019 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:48:04.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:04.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:04.019 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:48:04.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:04.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:04.019 | INFO | Uploaded 36GiB to GPU in 2.02105s (17.8126GiB/s) | 20260506 20:48:04.019 | INFO | Downloaded 34GiB to GPU in 1.92946s (17.6215GiB/s) | 20260506 20:48:09.157 | INFO | Analyzing chunk frames = [4342784,4343296) | 20260506 20:48:09.157 | INFO | MemInfo: { | 20260506 20:48:09.157 | INFO | "gpu" : " 17 GB", | 20260506 20:48:09.157 | INFO | "huge" : "", | 20260506 20:48:09.157 | INFO | "huge_shared" : " 27 GB", | 20260506 20:48:09.157 | INFO | "malloc" : " 14 GB", | 20260506 20:48:09.157 | INFO | "pinned" : " 29 GB" | 20260506 20:48:09.157 | INFO | } | 20260506 20:48:09.157 | INFO | Perf report for the last 5.13824 seconds: | 20260506 20:48:09.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:48:09.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:09.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:09.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:09.157 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 20:48:09.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:09.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:09.157 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:48:09.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:09.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:09.157 | INFO | Uploaded 36GiB to GPU in 1.99076s (18.0835GiB/s) | 20260506 20:48:09.157 | INFO | Downloaded 34GiB to GPU in 1.92136s (17.6958GiB/s) | 20260506 20:48:14.294 | INFO | Analyzing chunk frames = [4343296,4343808) | 20260506 20:48:14.294 | INFO | MemInfo: { | 20260506 20:48:14.294 | INFO | "gpu" : " 17 GB", | 20260506 20:48:14.294 | INFO | "huge" : "", | 20260506 20:48:14.294 | INFO | "huge_shared" : " 27 GB", | 20260506 20:48:14.294 | INFO | "malloc" : " 14 GB", | 20260506 20:48:14.294 | INFO | "pinned" : " 29 GB" | 20260506 20:48:14.294 | INFO | } | 20260506 20:48:14.295 | INFO | Perf report for the last 5.13758 seconds: | 20260506 20:48:14.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:48:14.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:14.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:14.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:14.295 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 20:48:14.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:14.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:14.295 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:48:14.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:14.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:14.295 | INFO | Uploaded 36GiB to GPU in 2.02184s (17.8056GiB/s) | 20260506 20:48:14.295 | INFO | Downloaded 34GiB to GPU in 1.90683s (17.8306GiB/s) | 20260506 20:48:19.436 | INFO | Analyzing chunk frames = [4343808,4344320) | 20260506 20:48:19.437 | INFO | MemInfo: { | 20260506 20:48:19.437 | INFO | "gpu" : " 17 GB", | 20260506 20:48:19.437 | INFO | "huge" : "", | 20260506 20:48:19.437 | INFO | "huge_shared" : " 27 GB", | 20260506 20:48:19.437 | INFO | "malloc" : " 14 GB", | 20260506 20:48:19.437 | INFO | "pinned" : " 29 GB" | 20260506 20:48:19.437 | INFO | } | 20260506 20:48:19.437 | INFO | Perf report for the last 5.14202 seconds: | 20260506 20:48:19.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:48:19.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:19.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:19.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:19.437 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 20:48:19.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:19.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:19.437 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:48:19.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:19.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:19.437 | INFO | Uploaded 36GiB to GPU in 2.01604s (17.8568GiB/s) | 20260506 20:48:19.437 | INFO | Downloaded 34GiB to GPU in 1.91688s (17.7372GiB/s) | 20260506 20:48:24.574 | INFO | Analyzing chunk frames = [4344320,4344832) | 20260506 20:48:24.574 | INFO | MemInfo: { | 20260506 20:48:24.574 | INFO | "gpu" : " 17 GB", | 20260506 20:48:24.574 | INFO | "huge" : "", | 20260506 20:48:24.574 | INFO | "huge_shared" : " 27 GB", | 20260506 20:48:24.574 | INFO | "malloc" : " 14 GB", | 20260506 20:48:24.574 | INFO | "pinned" : " 29 GB" | 20260506 20:48:24.574 | INFO | } | 20260506 20:48:24.574 | INFO | Perf report for the last 5.1376 seconds: | 20260506 20:48:24.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:48:24.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:24.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:24.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:24.574 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:48:24.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:24.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:24.574 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:48:24.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:24.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:24.574 | INFO | Uploaded 36GiB to GPU in 2.02754s (17.7555GiB/s) | 20260506 20:48:24.574 | INFO | Downloaded 34GiB to GPU in 1.96171s (17.3319GiB/s) | 20260506 20:48:29.711 | INFO | Analyzing chunk frames = [4344832,4345344) | 20260506 20:48:29.711 | INFO | MemInfo: { | 20260506 20:48:29.711 | INFO | "gpu" : " 17 GB", | 20260506 20:48:29.711 | INFO | "huge" : "", | 20260506 20:48:29.711 | INFO | "huge_shared" : " 27 GB", | 20260506 20:48:29.711 | INFO | "malloc" : " 14 GB", | 20260506 20:48:29.711 | INFO | "pinned" : " 29 GB" | 20260506 20:48:29.711 | INFO | } | 20260506 20:48:29.711 | INFO | Perf report for the last 5.137 seconds: | 20260506 20:48:29.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:48:29.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:29.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:29.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:29.711 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 20:48:29.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:29.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:29.711 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 20:48:29.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:29.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:29.711 | INFO | Uploaded 36GiB to GPU in 1.94446s (18.5142GiB/s) | 20260506 20:48:29.712 | INFO | Downloaded 34GiB to GPU in 1.98859s (17.0976GiB/s) | 20260506 20:48:34.855 | INFO | Analyzing chunk frames = [4345344,4345856) | 20260506 20:48:34.855 | INFO | MemInfo: { | 20260506 20:48:34.855 | INFO | "gpu" : " 17 GB", | 20260506 20:48:34.855 | INFO | "huge" : "", | 20260506 20:48:34.855 | INFO | "huge_shared" : " 27 GB", | 20260506 20:48:34.855 | INFO | "malloc" : " 14 GB", | 20260506 20:48:34.855 | INFO | "pinned" : " 29 GB" | 20260506 20:48:34.855 | INFO | } | 20260506 20:48:34.855 | INFO | Perf report for the last 5.14364 seconds: | 20260506 20:48:34.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:48:34.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:34.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:34.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:34.855 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260506 20:48:34.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:34.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:34.855 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 20:48:34.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:34.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:34.855 | INFO | Uploaded 36GiB to GPU in 2.06676s (17.4185GiB/s) | 20260506 20:48:34.855 | INFO | Downloaded 34GiB to GPU in 1.99778s (17.0189GiB/s) | 20260506 20:48:34.972 | INFO | Starting a round of Baz IO | 20260506 20:48:38.103 | INFO | Populating RTMetrics report | 20260506 20:48:38.103 | INFO | RTMetrics report populated | 20260506 20:48:38.103 | INFO | Opening tmp RTMetrics json file | 20260506 20:48:38.104 | INFO | Serializing RTMetrics to json | 20260506 20:48:38.117 | INFO | Serialization done, swapping tmp file now... | 20260506 20:48:38.132 | INFO | RTMetrics done | 20260506 20:48:38.229 | INFO | Finished a round of Baz IO: Wrote 1.967 GiB in 3.256 seconds | 20260506 20:48:38.229 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.13% / 0.00% / 65.87% | 20260506 20:48:38.229 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.698467% / 7.94729e-06% / 4.37101e-05% | 20260506 20:48:38.229 | INFO | Baz IO overall: | 20260506 20:48:38.229 | INFO | BytesWritten : 2293047359136 | 20260506 20:48:38.229 | INFO | NumEvents : 1107596701931 | 20260506 20:48:39.996 | INFO | Analyzing chunk frames = [4345856,4346368) | 20260506 20:48:39.996 | INFO | MemInfo: { | 20260506 20:48:39.996 | INFO | "gpu" : " 17 GB", | 20260506 20:48:39.996 | INFO | "huge" : "", | 20260506 20:48:39.996 | INFO | "huge_shared" : " 27 GB", | 20260506 20:48:39.996 | INFO | "malloc" : " 14 GB", | 20260506 20:48:39.996 | INFO | "pinned" : " 29 GB" | 20260506 20:48:39.996 | INFO | } | 20260506 20:48:39.996 | INFO | Perf report for the last 5.14129 seconds: | 20260506 20:48:39.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:48:39.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:39.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:39.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:39.996 | INFO | ANALYSIS : 62.6%, 2.9, 95.3ms, 0 | 20260506 20:48:39.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:39.996 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 20:48:39.996 | INFO | PRE_HQ : 60.0%, 1.8, 57.2ms, 0 | 20260506 20:48:39.996 | INFO | BAZWRITER : 63.0%, 4.7, 153.7ms, 0 | 20260506 20:48:39.996 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 20:48:39.996 | INFO | Uploaded 36GiB to GPU in 2.06597s (17.4252GiB/s) | 20260506 20:48:39.996 | INFO | Downloaded 38GiB to GPU in 2.21255s (17.1748GiB/s) | 20260506 20:48:45.138 | INFO | Analyzing chunk frames = [4346368,4346880) | 20260506 20:48:45.138 | INFO | MemInfo: { | 20260506 20:48:45.138 | INFO | "gpu" : " 17 GB", | 20260506 20:48:45.138 | INFO | "huge" : "", | 20260506 20:48:45.138 | INFO | "huge_shared" : " 27 GB", | 20260506 20:48:45.138 | INFO | "malloc" : " 14 GB", | 20260506 20:48:45.138 | INFO | "pinned" : " 29 GB" | 20260506 20:48:45.138 | INFO | } | 20260506 20:48:45.138 | INFO | Perf report for the last 5.14226 seconds: | 20260506 20:48:45.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:48:45.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:45.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:45.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:45.138 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 20:48:45.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:45.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:45.139 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:48:45.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:45.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:45.139 | INFO | Uploaded 36GiB to GPU in 2.00428s (17.9615GiB/s) | 20260506 20:48:45.139 | INFO | Downloaded 34GiB to GPU in 1.92391s (17.6723GiB/s) | 20260506 20:48:50.272 | INFO | Analyzing chunk frames = [4346880,4347392) | 20260506 20:48:50.272 | INFO | MemInfo: { | 20260506 20:48:50.272 | INFO | "gpu" : " 17 GB", | 20260506 20:48:50.272 | INFO | "huge" : "", | 20260506 20:48:50.273 | INFO | "huge_shared" : " 27 GB", | 20260506 20:48:50.273 | INFO | "malloc" : " 14 GB", | 20260506 20:48:50.273 | INFO | "pinned" : " 29 GB" | 20260506 20:48:50.273 | INFO | } | 20260506 20:48:50.273 | INFO | Perf report for the last 5.13429 seconds: | 20260506 20:48:50.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:48:50.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:50.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:50.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:50.273 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 20:48:50.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:50.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:50.273 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:48:50.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:50.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:50.273 | INFO | Uploaded 36GiB to GPU in 2.00984s (17.9119GiB/s) | 20260506 20:48:50.273 | INFO | Downloaded 34GiB to GPU in 1.93042s (17.6127GiB/s) | 20260506 20:48:55.407 | INFO | Analyzing chunk frames = [4347392,4347904) | 20260506 20:48:55.407 | INFO | MemInfo: { | 20260506 20:48:55.407 | INFO | "gpu" : " 17 GB", | 20260506 20:48:55.407 | INFO | "huge" : "", | 20260506 20:48:55.407 | INFO | "huge_shared" : " 27 GB", | 20260506 20:48:55.407 | INFO | "malloc" : " 14 GB", | 20260506 20:48:55.407 | INFO | "pinned" : " 29 GB" | 20260506 20:48:55.407 | INFO | } | 20260506 20:48:55.407 | INFO | Perf report for the last 5.13422 seconds: | 20260506 20:48:55.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:48:55.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:55.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:55.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:55.407 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260506 20:48:55.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:55.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:55.407 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 20:48:55.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:55.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:48:55.407 | INFO | Uploaded 36GiB to GPU in 1.9584s (18.3824GiB/s) | 20260506 20:48:55.407 | INFO | Downloaded 34GiB to GPU in 1.87444s (18.1388GiB/s) | 20260506 20:49:00.541 | INFO | Analyzing chunk frames = [4347904,4348416) | 20260506 20:49:00.541 | INFO | MemInfo: { | 20260506 20:49:00.541 | INFO | "gpu" : " 17 GB", | 20260506 20:49:00.541 | INFO | "huge" : "", | 20260506 20:49:00.541 | INFO | "huge_shared" : " 27 GB", | 20260506 20:49:00.541 | INFO | "malloc" : " 14 GB", | 20260506 20:49:00.541 | INFO | "pinned" : " 29 GB" | 20260506 20:49:00.541 | INFO | } | 20260506 20:49:00.541 | INFO | Perf report for the last 5.13445 seconds: | 20260506 20:49:00.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:49:00.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:00.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:00.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:00.542 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260506 20:49:00.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:00.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:00.542 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:49:00.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:00.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:00.542 | INFO | Uploaded 36GiB to GPU in 2.02109s (17.8121GiB/s) | 20260506 20:49:00.542 | INFO | Downloaded 34GiB to GPU in 1.9177s (17.7296GiB/s) | 20260506 20:49:05.685 | INFO | Analyzing chunk frames = [4348416,4348928) | 20260506 20:49:05.685 | INFO | MemInfo: { | 20260506 20:49:05.685 | INFO | "gpu" : " 17 GB", | 20260506 20:49:05.685 | INFO | "huge" : "", | 20260506 20:49:05.685 | INFO | "huge_shared" : " 27 GB", | 20260506 20:49:05.685 | INFO | "malloc" : " 14 GB", | 20260506 20:49:05.685 | INFO | "pinned" : " 29 GB" | 20260506 20:49:05.685 | INFO | } | 20260506 20:49:05.685 | INFO | Perf report for the last 5.14354 seconds: | 20260506 20:49:05.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:49:05.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:05.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:05.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:05.685 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 20:49:05.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:05.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:05.685 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:49:05.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:05.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:05.685 | INFO | Uploaded 36GiB to GPU in 2.03352s (17.7033GiB/s) | 20260506 20:49:05.685 | INFO | Downloaded 34GiB to GPU in 1.9579s (17.3655GiB/s) | 20260506 20:49:10.828 | INFO | Analyzing chunk frames = [4348928,4349440) | 20260506 20:49:10.828 | INFO | MemInfo: { | 20260506 20:49:10.828 | INFO | "gpu" : " 17 GB", | 20260506 20:49:10.828 | INFO | "huge" : "", | 20260506 20:49:10.828 | INFO | "huge_shared" : " 27 GB", | 20260506 20:49:10.828 | INFO | "malloc" : " 14 GB", | 20260506 20:49:10.828 | INFO | "pinned" : " 29 GB" | 20260506 20:49:10.828 | INFO | } | 20260506 20:49:10.828 | INFO | Perf report for the last 5.14315 seconds: | 20260506 20:49:10.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:49:10.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:10.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:10.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:10.828 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 20:49:10.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:10.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:10.828 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 20:49:10.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:10.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:10.828 | INFO | Uploaded 36GiB to GPU in 1.95161s (18.4463GiB/s) | 20260506 20:49:10.828 | INFO | Downloaded 34GiB to GPU in 1.99069s (17.0795GiB/s) | 20260506 20:49:15.968 | INFO | Analyzing chunk frames = [4349440,4349952) | 20260506 20:49:15.968 | INFO | MemInfo: { | 20260506 20:49:15.968 | INFO | "gpu" : " 17 GB", | 20260506 20:49:15.968 | INFO | "huge" : "", | 20260506 20:49:15.968 | INFO | "huge_shared" : " 27 GB", | 20260506 20:49:15.968 | INFO | "malloc" : " 14 GB", | 20260506 20:49:15.968 | INFO | "pinned" : " 29 GB" | 20260506 20:49:15.968 | INFO | } | 20260506 20:49:15.968 | INFO | Perf report for the last 5.14005 seconds: | 20260506 20:49:15.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:49:15.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:15.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:15.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:15.968 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 20:49:15.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:15.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:15.968 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260506 20:49:15.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:15.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:15.968 | INFO | Uploaded 36GiB to GPU in 2.06744s (17.4128GiB/s) | 20260506 20:49:15.968 | INFO | Downloaded 34GiB to GPU in 2.01053s (16.911GiB/s) | 20260506 20:49:15.969 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:49:15.969 | INFO | Total mem usage: 18 GiB | 20260506 20:49:15.969 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:49:15.969 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -18.2%) | 20260506 20:49:15.969 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:49:15.969 | INFO | High Water Breakdown: | 20260506 20:49:15.969 | INFO | 61.1% : 9.31 GiB ( 81.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:49:15.969 | INFO | 38.8% : 5.9 GiB ( 88.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:49:15.969 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:49:15.969 | INFO | ------------------------------------------ | 20260506 20:49:15.969 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:49:15.969 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:49:15.969 | INFO | Total mem usage: 26.5 GiB | 20260506 20:49:15.969 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:49:15.969 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:49:15.969 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:49:15.969 | INFO | High Water Breakdown: | 20260506 20:49:15.969 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:49:15.969 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:49:15.969 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:49:15.969 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:49:15.969 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:49:15.969 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:49:15.969 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:49:15.969 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:49:15.969 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:49:15.969 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:49:15.969 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:49:15.969 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:49:15.969 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:49:15.969 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:49:15.969 | INFO | ------------------------------------------ | 20260506 20:49:15.969 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:49:15.969 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:49:15.969 | INFO | Total mem usage: 25.6 GiB | 20260506 20:49:15.969 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:49:15.969 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:49:15.969 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:49:15.969 | INFO | High Water Breakdown: | 20260506 20:49:15.969 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:49:15.969 | INFO | ------------------------------------------ | 20260506 20:49:15.969 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:49:15.969 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:49:15.969 | INFO | Total mem usage: 15.7 GiB | 20260506 20:49:15.969 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:49:15.969 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:49:15.969 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:49:15.969 | INFO | High Water Breakdown: | 20260506 20:49:15.969 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:49:15.969 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:49:15.969 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:49:15.969 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:49:15.969 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:49:15.969 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:49:15.969 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:49:15.969 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:49:15.969 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:49:15.969 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:49:15.969 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:49:15.969 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:49:15.969 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:49:15.969 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:49:15.969 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:49:15.969 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:49:15.969 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:49:15.969 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:49:15.969 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:49:15.969 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:49:15.969 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:49:15.969 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:49:15.969 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:49:15.969 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:49:15.969 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:49:15.969 | INFO | ------------------------------------------ | 20260506 20:49:15.969 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:49:15.969 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:49:15.969 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:49:15.969 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:49:15.970 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.63e+01, 1.36e+01 / 2.61e+00 | 20260506 20:49:15.970 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 9.92e-02, 1.02e-02 / 6.28e-03 | 20260506 20:49:15.970 | INFO | PerfReport: 11.62%, Baseline -- 3.62e+00 / 4.37e+00, 5.47e-01 / 3.77e-01 | 20260506 20:49:15.970 | INFO | PerfReport: 3.25%, Histogram -- 1.02e+00 / 1.43e+00, 1.79e-01 / 1.54e-01 | 20260506 20:49:15.970 | INFO | PerfReport: 16.05%, DME -- 4.00e+01 / 4.51e+01, 4.31e+00 / 2.98e+00 | 20260506 20:49:15.970 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.79e+01 / 2.08e+01, 1.45e+00 / 1.81e+00 | 20260506 20:49:15.970 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.21e+00 / 2.78e+00, 3.71e-01 / 2.23e-01 | 20260506 20:49:15.970 | INFO | PerfReport: 3.28%, Metrics -- 1.02e+00 / 1.36e+00, 2.69e-01 / 1.68e-01 | 20260506 20:49:15.970 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.52e-01 / 3.99e-01, 5.49e-02 / 3.24e-02 | 20260506 20:49:15.970 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:49:15.970 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:49:15.970 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:49:15.970 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 20:49:15.970 | INFO | PerfReport: 21.67%, Upload -- 2.10e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260506 20:49:15.970 | INFO | PerfReport: 21.25%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 20:49:15.970 | INFO | PerfReport: 32.18%, Compute -- 3.12e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 20:49:15.970 | INFO | PerfReport: 1.42%, LockUpload -- 1.38e+00 / 0.00e+00, 5.34e+00 / 0.00e+00 | 20260506 20:49:15.970 | INFO | PerfReport: 1.09%, LockDownload -- 1.06e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 20:49:15.970 | INFO | PerfReport: 22.38%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 20:49:15.970 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260506 20:49:16.087 | INFO | Starting a round of Baz IO | 20260506 20:49:19.226 | INFO | Populating RTMetrics report | 20260506 20:49:19.226 | INFO | RTMetrics report populated | 20260506 20:49:19.226 | INFO | Opening tmp RTMetrics json file | 20260506 20:49:19.259 | INFO | Serializing RTMetrics to json | 20260506 20:49:19.290 | INFO | Serialization done, swapping tmp file now... | 20260506 20:49:19.317 | INFO | RTMetrics done | 20260506 20:49:19.439 | INFO | Finished a round of Baz IO: Wrote 1.964 GiB in 3.351 seconds | 20260506 20:49:19.439 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.12% / 0.00% / 65.87% | 20260506 20:49:19.439 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.694124% / 3.97364e-06% / 7.15256e-05% | 20260506 20:49:19.439 | INFO | Baz IO overall: | 20260506 20:49:19.439 | INFO | BytesWritten : 2295156029837 | 20260506 20:49:19.439 | INFO | NumEvents : 1108607054999 | 20260506 20:49:21.114 | INFO | Analyzing chunk frames = [4349952,4350464) | 20260506 20:49:21.114 | INFO | MemInfo: { | 20260506 20:49:21.114 | INFO | "gpu" : " 17 GB", | 20260506 20:49:21.114 | INFO | "huge" : "", | 20260506 20:49:21.114 | INFO | "huge_shared" : " 27 GB", | 20260506 20:49:21.114 | INFO | "malloc" : " 14 GB", | 20260506 20:49:21.114 | INFO | "pinned" : " 29 GB" | 20260506 20:49:21.114 | INFO | } | 20260506 20:49:21.115 | INFO | Perf report for the last 5.14631 seconds: | 20260506 20:49:21.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:49:21.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:21.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:21.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:21.115 | INFO | ANALYSIS : 62.6%, 2.9, 94.3ms, 0 | 20260506 20:49:21.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:21.115 | INFO | RT_METRICS : 46.4%, 1.0, 24.4ms, 0 | 20260506 20:49:21.115 | INFO | PRE_HQ : 60.2%, 1.9, 60.9ms, 0 | 20260506 20:49:21.115 | INFO | BAZWRITER : 64.8%, 4.8, 159.2ms, 0 | 20260506 20:49:21.115 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 20:49:21.115 | INFO | Uploaded 36GiB to GPU in 2.05679s (17.503GiB/s) | 20260506 20:49:21.115 | INFO | Downloaded 38GiB to GPU in 2.22081s (17.1109GiB/s) | 20260506 20:49:26.249 | INFO | Analyzing chunk frames = [4350464,4350976) | 20260506 20:49:26.249 | INFO | MemInfo: { | 20260506 20:49:26.249 | INFO | "gpu" : " 17 GB", | 20260506 20:49:26.249 | INFO | "huge" : "", | 20260506 20:49:26.249 | INFO | "huge_shared" : " 27 GB", | 20260506 20:49:26.249 | INFO | "malloc" : " 14 GB", | 20260506 20:49:26.249 | INFO | "pinned" : " 29 GB" | 20260506 20:49:26.249 | INFO | } | 20260506 20:49:26.249 | INFO | Perf report for the last 5.13489 seconds: | 20260506 20:49:26.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:49:26.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:26.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:26.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:26.250 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 20:49:26.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:26.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:26.250 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:49:26.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:26.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:26.250 | INFO | Uploaded 36GiB to GPU in 1.99s (18.0904GiB/s) | 20260506 20:49:26.250 | INFO | Downloaded 34GiB to GPU in 1.92786s (17.6361GiB/s) | 20260506 20:49:31.388 | INFO | Analyzing chunk frames = [4350976,4351488) | 20260506 20:49:31.388 | INFO | MemInfo: { | 20260506 20:49:31.388 | INFO | "gpu" : " 17 GB", | 20260506 20:49:31.388 | INFO | "huge" : "", | 20260506 20:49:31.388 | INFO | "huge_shared" : " 27 GB", | 20260506 20:49:31.388 | INFO | "malloc" : " 14 GB", | 20260506 20:49:31.388 | INFO | "pinned" : " 29 GB" | 20260506 20:49:31.388 | INFO | } | 20260506 20:49:31.388 | INFO | Perf report for the last 5.13844 seconds: | 20260506 20:49:31.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:49:31.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:31.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:31.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:31.388 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 20:49:31.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:31.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:31.388 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:49:31.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:31.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:31.389 | INFO | Uploaded 36GiB to GPU in 1.98204s (18.1631GiB/s) | 20260506 20:49:31.389 | INFO | Downloaded 34GiB to GPU in 1.93145s (17.6034GiB/s) | 20260506 20:49:36.524 | INFO | Analyzing chunk frames = [4351488,4352000) | 20260506 20:49:36.524 | INFO | MemInfo: { | 20260506 20:49:36.524 | INFO | "gpu" : " 17 GB", | 20260506 20:49:36.524 | INFO | "huge" : "", | 20260506 20:49:36.524 | INFO | "huge_shared" : " 27 GB", | 20260506 20:49:36.524 | INFO | "malloc" : " 14 GB", | 20260506 20:49:36.524 | INFO | "pinned" : " 29 GB" | 20260506 20:49:36.524 | INFO | } | 20260506 20:49:36.524 | INFO | Perf report for the last 5.13604 seconds: | 20260506 20:49:36.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:49:36.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:36.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:36.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:36.524 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 20:49:36.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:36.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:36.524 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:49:36.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:36.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:36.524 | INFO | Uploaded 36GiB to GPU in 1.95791s (18.3869GiB/s) | 20260506 20:49:36.524 | INFO | Downloaded 34GiB to GPU in 1.87639s (18.1199GiB/s) | 20260506 20:49:41.658 | INFO | Analyzing chunk frames = [4352000,4352512) | 20260506 20:49:41.658 | INFO | MemInfo: { | 20260506 20:49:41.658 | INFO | "gpu" : " 17 GB", | 20260506 20:49:41.658 | INFO | "huge" : "", | 20260506 20:49:41.658 | INFO | "huge_shared" : " 27 GB", | 20260506 20:49:41.658 | INFO | "malloc" : " 14 GB", | 20260506 20:49:41.658 | INFO | "pinned" : " 29 GB" | 20260506 20:49:41.658 | INFO | } | 20260506 20:49:41.658 | INFO | Perf report for the last 5.13437 seconds: | 20260506 20:49:41.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:49:41.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:41.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:41.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:41.658 | INFO | ANALYSIS : 60.5%, 3.0, 94.0ms, 0 | 20260506 20:49:41.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:41.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:41.658 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:49:41.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:41.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:41.659 | INFO | Uploaded 36GiB to GPU in 2.00201s (17.9819GiB/s) | 20260506 20:49:41.659 | INFO | Downloaded 34GiB to GPU in 1.90033s (17.8916GiB/s) | 20260506 20:49:46.804 | INFO | Analyzing chunk frames = [4352512,4353024) | 20260506 20:49:46.804 | INFO | MemInfo: { | 20260506 20:49:46.804 | INFO | "gpu" : " 17 GB", | 20260506 20:49:46.804 | INFO | "huge" : "", | 20260506 20:49:46.804 | INFO | "huge_shared" : " 27 GB", | 20260506 20:49:46.804 | INFO | "malloc" : " 14 GB", | 20260506 20:49:46.804 | INFO | "pinned" : " 29 GB" | 20260506 20:49:46.804 | INFO | } | 20260506 20:49:46.804 | INFO | Perf report for the last 5.14548 seconds: | 20260506 20:49:46.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:49:46.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:46.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:46.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:46.804 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260506 20:49:46.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:46.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:46.804 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 20:49:46.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:46.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:46.804 | INFO | Uploaded 36GiB to GPU in 1.9808s (18.1745GiB/s) | 20260506 20:49:46.804 | INFO | Downloaded 34GiB to GPU in 1.95114s (17.4257GiB/s) | 20260506 20:49:51.945 | INFO | Analyzing chunk frames = [4353024,4353536) | 20260506 20:49:51.945 | INFO | MemInfo: { | 20260506 20:49:51.945 | INFO | "gpu" : " 17 GB", | 20260506 20:49:51.945 | INFO | "huge" : "", | 20260506 20:49:51.945 | INFO | "huge_shared" : " 27 GB", | 20260506 20:49:51.945 | INFO | "malloc" : " 14 GB", | 20260506 20:49:51.945 | INFO | "pinned" : " 29 GB" | 20260506 20:49:51.945 | INFO | } | 20260506 20:49:51.946 | INFO | Perf report for the last 5.14168 seconds: | 20260506 20:49:51.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:49:51.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:51.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:51.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:51.946 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 20:49:51.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:51.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:51.946 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:49:51.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:51.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:51.946 | INFO | Uploaded 36GiB to GPU in 1.96189s (18.3497GiB/s) | 20260506 20:49:51.946 | INFO | Downloaded 34GiB to GPU in 1.99996s (17.0004GiB/s) | 20260506 20:49:57.084 | INFO | Analyzing chunk frames = [4353536,4354048) | 20260506 20:49:57.084 | INFO | MemInfo: { | 20260506 20:49:57.084 | INFO | "gpu" : " 17 GB", | 20260506 20:49:57.084 | INFO | "huge" : "", | 20260506 20:49:57.084 | INFO | "huge_shared" : " 27 GB", | 20260506 20:49:57.084 | INFO | "malloc" : " 14 GB", | 20260506 20:49:57.084 | INFO | "pinned" : " 29 GB" | 20260506 20:49:57.084 | INFO | } | 20260506 20:49:57.084 | INFO | Perf report for the last 5.13889 seconds: | 20260506 20:49:57.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:49:57.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:57.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:57.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:57.085 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 20:49:57.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:57.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:57.085 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 20:49:57.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:57.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:49:57.085 | INFO | Uploaded 36GiB to GPU in 2.04234s (17.6268GiB/s) | 20260506 20:49:57.085 | INFO | Downloaded 34GiB to GPU in 1.97664s (17.2009GiB/s) | 20260506 20:49:57.197 | INFO | Starting a round of Baz IO | 20260506 20:50:00.323 | INFO | Populating RTMetrics report | 20260506 20:50:00.323 | INFO | RTMetrics report populated | 20260506 20:50:00.323 | INFO | Opening tmp RTMetrics json file | 20260506 20:50:00.328 | INFO | Serializing RTMetrics to json | 20260506 20:50:00.355 | INFO | Serialization done, swapping tmp file now... | 20260506 20:50:00.358 | INFO | RTMetrics done | 20260506 20:50:00.463 | INFO | Finished a round of Baz IO: Wrote 1.966 GiB in 3.266 seconds | 20260506 20:50:00.463 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.12% / 0.00% / 65.88% | 20260506 20:50:00.463 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.695761% / 1.19209e-05% / 8.74201e-05% | 20260506 20:50:00.463 | INFO | Baz IO overall: | 20260506 20:50:00.463 | INFO | BytesWritten : 2297267049391 | 20260506 20:50:00.463 | INFO | NumEvents : 1109619235866 | 20260506 20:50:02.222 | INFO | Analyzing chunk frames = [4354048,4354560) | 20260506 20:50:02.222 | INFO | MemInfo: { | 20260506 20:50:02.222 | INFO | "gpu" : " 17 GB", | 20260506 20:50:02.222 | INFO | "huge" : "", | 20260506 20:50:02.222 | INFO | "huge_shared" : " 27 GB", | 20260506 20:50:02.222 | INFO | "malloc" : " 14 GB", | 20260506 20:50:02.222 | INFO | "pinned" : " 29 GB" | 20260506 20:50:02.222 | INFO | } | 20260506 20:50:02.222 | INFO | Perf report for the last 5.13795 seconds: | 20260506 20:50:02.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:50:02.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:02.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:02.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:02.222 | INFO | ANALYSIS : 62.5%, 2.9, 95.0ms, 0 | 20260506 20:50:02.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:02.223 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 20:50:02.223 | INFO | PRE_HQ : 59.7%, 1.9, 57.4ms, 0 | 20260506 20:50:02.223 | INFO | BAZWRITER : 63.3%, 4.6, 150.7ms, 0 | 20260506 20:50:02.223 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 20:50:02.223 | INFO | Uploaded 36GiB to GPU in 2.08107s (17.2988GiB/s) | 20260506 20:50:02.223 | INFO | Downloaded 38GiB to GPU in 2.2013s (17.2625GiB/s) | 20260506 20:50:07.359 | INFO | Analyzing chunk frames = [4354560,4355072) | 20260506 20:50:07.359 | INFO | MemInfo: { | 20260506 20:50:07.359 | INFO | "gpu" : " 17 GB", | 20260506 20:50:07.359 | INFO | "huge" : "", | 20260506 20:50:07.359 | INFO | "huge_shared" : " 27 GB", | 20260506 20:50:07.359 | INFO | "malloc" : " 14 GB", | 20260506 20:50:07.359 | INFO | "pinned" : " 29 GB" | 20260506 20:50:07.359 | INFO | } | 20260506 20:50:07.359 | INFO | Perf report for the last 5.13668 seconds: | 20260506 20:50:07.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:50:07.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:07.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:07.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:07.359 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:50:07.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:07.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:07.359 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 20:50:07.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:07.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:07.359 | INFO | Uploaded 36GiB to GPU in 2.0127s (17.8864GiB/s) | 20260506 20:50:07.359 | INFO | Downloaded 34GiB to GPU in 1.93898s (17.535GiB/s) | 20260506 20:50:12.495 | INFO | Analyzing chunk frames = [4355072,4355584) | 20260506 20:50:12.495 | INFO | MemInfo: { | 20260506 20:50:12.495 | INFO | "gpu" : " 17 GB", | 20260506 20:50:12.495 | INFO | "huge" : "", | 20260506 20:50:12.495 | INFO | "huge_shared" : " 27 GB", | 20260506 20:50:12.495 | INFO | "malloc" : " 14 GB", | 20260506 20:50:12.495 | INFO | "pinned" : " 29 GB" | 20260506 20:50:12.495 | INFO | } | 20260506 20:50:12.495 | INFO | Perf report for the last 5.13624 seconds: | 20260506 20:50:12.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:50:12.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:12.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:12.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:12.495 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 20:50:12.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:12.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:12.495 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:50:12.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:12.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:12.496 | INFO | Uploaded 36GiB to GPU in 1.97315s (18.2449GiB/s) | 20260506 20:50:12.496 | INFO | Downloaded 34GiB to GPU in 1.91712s (17.7349GiB/s) | 20260506 20:50:17.634 | INFO | Analyzing chunk frames = [4355584,4356096) | 20260506 20:50:17.634 | INFO | MemInfo: { | 20260506 20:50:17.634 | INFO | "gpu" : " 17 GB", | 20260506 20:50:17.634 | INFO | "huge" : "", | 20260506 20:50:17.634 | INFO | "huge_shared" : " 27 GB", | 20260506 20:50:17.634 | INFO | "malloc" : " 14 GB", | 20260506 20:50:17.634 | INFO | "pinned" : " 29 GB" | 20260506 20:50:17.634 | INFO | } | 20260506 20:50:17.634 | INFO | Perf report for the last 5.13884 seconds: | 20260506 20:50:17.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:50:17.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:17.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:17.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:17.634 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 20:50:17.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:17.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:17.634 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 20:50:17.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:17.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:17.634 | INFO | Uploaded 36GiB to GPU in 1.9479s (18.4815GiB/s) | 20260506 20:50:17.634 | INFO | Downloaded 34GiB to GPU in 1.87871s (18.0975GiB/s) | 20260506 20:50:22.777 | INFO | Analyzing chunk frames = [4356096,4356608) | 20260506 20:50:22.777 | INFO | MemInfo: { | 20260506 20:50:22.777 | INFO | "gpu" : " 17 GB", | 20260506 20:50:22.777 | INFO | "huge" : "", | 20260506 20:50:22.777 | INFO | "huge_shared" : " 27 GB", | 20260506 20:50:22.777 | INFO | "malloc" : " 14 GB", | 20260506 20:50:22.777 | INFO | "pinned" : " 29 GB" | 20260506 20:50:22.777 | INFO | } | 20260506 20:50:22.777 | INFO | Perf report for the last 5.14285 seconds: | 20260506 20:50:22.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:50:22.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:22.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:22.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:22.777 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 20:50:22.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:22.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:22.777 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:50:22.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:22.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:22.777 | INFO | Uploaded 36GiB to GPU in 2.01726s (17.846GiB/s) | 20260506 20:50:22.777 | INFO | Downloaded 34GiB to GPU in 1.90797s (17.8199GiB/s) | 20260506 20:50:27.920 | INFO | Analyzing chunk frames = [4356608,4357120) | 20260506 20:50:27.920 | INFO | MemInfo: { | 20260506 20:50:27.920 | INFO | "gpu" : " 17 GB", | 20260506 20:50:27.920 | INFO | "huge" : "", | 20260506 20:50:27.920 | INFO | "huge_shared" : " 27 GB", | 20260506 20:50:27.920 | INFO | "malloc" : " 14 GB", | 20260506 20:50:27.920 | INFO | "pinned" : " 29 GB" | 20260506 20:50:27.920 | INFO | } | 20260506 20:50:27.920 | INFO | Perf report for the last 5.14282 seconds: | 20260506 20:50:27.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:50:27.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:27.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:27.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:27.920 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260506 20:50:27.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:27.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:27.920 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 20:50:27.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:27.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:27.920 | INFO | Uploaded 36GiB to GPU in 2.01591s (17.858GiB/s) | 20260506 20:50:27.920 | INFO | Downloaded 34GiB to GPU in 1.95139s (17.4235GiB/s) | 20260506 20:50:33.064 | INFO | Analyzing chunk frames = [4357120,4357632) | 20260506 20:50:33.064 | INFO | MemInfo: { | 20260506 20:50:33.064 | INFO | "gpu" : " 17 GB", | 20260506 20:50:33.064 | INFO | "huge" : "", | 20260506 20:50:33.064 | INFO | "huge_shared" : " 27 GB", | 20260506 20:50:33.064 | INFO | "malloc" : " 14 GB", | 20260506 20:50:33.064 | INFO | "pinned" : " 29 GB" | 20260506 20:50:33.064 | INFO | } | 20260506 20:50:33.064 | INFO | Perf report for the last 5.14411 seconds: | 20260506 20:50:33.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:50:33.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:33.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:33.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:33.064 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 20:50:33.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:33.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:33.064 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 20:50:33.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:33.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:33.064 | INFO | Uploaded 36GiB to GPU in 2.00712s (17.9361GiB/s) | 20260506 20:50:33.065 | INFO | Downloaded 34GiB to GPU in 1.99114s (17.0756GiB/s) | 20260506 20:50:38.205 | INFO | Analyzing chunk frames = [4357632,4358144) | 20260506 20:50:38.205 | INFO | MemInfo: { | 20260506 20:50:38.205 | INFO | "gpu" : " 17 GB", | 20260506 20:50:38.205 | INFO | "huge" : "", | 20260506 20:50:38.205 | INFO | "huge_shared" : " 27 GB", | 20260506 20:50:38.205 | INFO | "malloc" : " 14 GB", | 20260506 20:50:38.205 | INFO | "pinned" : " 29 GB" | 20260506 20:50:38.205 | INFO | } | 20260506 20:50:38.205 | INFO | Perf report for the last 5.14091 seconds: | 20260506 20:50:38.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:50:38.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:38.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:38.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:38.205 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 20:50:38.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:38.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:38.205 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 20:50:38.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:38.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:38.205 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:50:38.205 | INFO | Total mem usage: 18 GiB | 20260506 20:50:38.205 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:50:38.205 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -19%) | 20260506 20:50:38.205 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:50:38.205 | INFO | High Water Breakdown: | 20260506 20:50:38.205 | INFO | 61% : 9.23 GiB ( 80.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:50:38.205 | INFO | 38.9% : 5.88 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:50:38.205 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:50:38.206 | INFO | ------------------------------------------ | 20260506 20:50:38.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:50:38.206 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:50:38.206 | INFO | Total mem usage: 26.5 GiB | 20260506 20:50:38.206 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:50:38.205 | INFO | Uploaded 36GiB to GPU in 2.02064s (17.8161GiB/s) | 20260506 20:50:38.206 | INFO | Downloaded 34GiB to GPU in 1.99259s (17.0632GiB/s) | 20260506 20:50:38.206 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 20:50:38.206 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260506 20:50:38.206 | INFO | High Water Breakdown: | 20260506 20:50:38.206 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:50:38.206 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:50:38.206 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:50:38.206 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:50:38.206 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:50:38.206 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:50:38.206 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:50:38.206 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:50:38.206 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:50:38.206 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:50:38.206 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:50:38.206 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:50:38.206 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:50:38.206 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:50:38.206 | INFO | ------------------------------------------ | 20260506 20:50:38.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:50:38.206 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:50:38.206 | INFO | Total mem usage: 25.6 GiB | 20260506 20:50:38.206 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:50:38.206 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:50:38.206 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:50:38.206 | INFO | High Water Breakdown: | 20260506 20:50:38.206 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:50:38.206 | INFO | ------------------------------------------ | 20260506 20:50:38.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:50:38.206 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:50:38.206 | INFO | Total mem usage: 15.7 GiB | 20260506 20:50:38.206 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:50:38.206 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:50:38.206 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:50:38.206 | INFO | High Water Breakdown: | 20260506 20:50:38.206 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:50:38.206 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:50:38.206 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:50:38.206 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:50:38.206 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:50:38.206 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:50:38.206 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:50:38.206 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:50:38.206 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:50:38.206 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:50:38.206 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:50:38.206 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:50:38.206 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:50:38.206 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:50:38.206 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:50:38.206 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:50:38.206 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:50:38.206 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:50:38.206 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:50:38.206 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:50:38.206 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:50:38.206 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:50:38.206 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:50:38.206 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:50:38.206 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:50:38.206 | INFO | ------------------------------------------ | 20260506 20:50:38.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:50:38.206 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:50:38.206 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:50:38.206 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:50:38.206 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.52e+01, 1.35e+01 / 2.19e+00 | 20260506 20:50:38.206 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.02e-01, 1.01e-02 / 6.27e-03 | 20260506 20:50:38.206 | INFO | PerfReport: 11.62%, Baseline -- 3.62e+00 / 4.39e+00, 5.50e-01 / 3.68e-01 | 20260506 20:50:38.206 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.81e-01 / 1.20e-01 | 20260506 20:50:38.207 | INFO | PerfReport: 15.95%, DME -- 3.97e+01 / 4.38e+01, 3.72e+00 / 1.95e+00 | 20260506 20:50:38.207 | INFO | PerfReport: 57.31%, FrameLabeling -- 1.79e+01 / 2.10e+01, 1.44e+00 / 1.51e+00 | 20260506 20:50:38.207 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.22e+00 / 2.74e+00, 3.76e-01 / 2.16e-01 | 20260506 20:50:38.207 | INFO | PerfReport: 3.30%, Metrics -- 1.03e+00 / 1.34e+00, 2.71e-01 / 1.70e-01 | 20260506 20:50:38.207 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.52e-01 / 4.04e-01, 5.52e-02 / 2.88e-02 | 20260506 20:50:38.207 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:50:38.207 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:50:38.207 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:50:38.207 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 20:50:38.207 | INFO | PerfReport: 21.60%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 20:50:38.207 | INFO | PerfReport: 21.27%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 20:50:38.207 | INFO | PerfReport: 32.24%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 20:50:38.207 | INFO | PerfReport: 1.29%, LockUpload -- 1.25e+00 / 0.00e+00, 5.10e+00 / 0.00e+00 | 20260506 20:50:38.207 | INFO | PerfReport: 1.18%, LockDownload -- 1.14e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260506 20:50:38.207 | INFO | PerfReport: 22.40%, LockCompute -- 2.16e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 20:50:38.207 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260506 20:50:38.315 | INFO | Starting a round of Baz IO | 20260506 20:50:41.447 | INFO | Populating RTMetrics report | 20260506 20:50:41.447 | INFO | RTMetrics report populated | 20260506 20:50:41.447 | INFO | Opening tmp RTMetrics json file | 20260506 20:50:41.513 | INFO | Serializing RTMetrics to json | 20260506 20:50:41.517 | INFO | Serialization done, swapping tmp file now... | 20260506 20:50:41.521 | INFO | RTMetrics done | 20260506 20:50:41.852 | INFO | Finished a round of Baz IO: Wrote 1.963 GiB in 3.537 seconds | 20260506 20:50:41.852 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.12% / 0.00% / 65.88% | 20260506 20:50:41.853 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.692801% / 3.97364e-06% / 7.54992e-05% | 20260506 20:50:41.853 | INFO | Baz IO overall: | 20260506 20:50:41.853 | INFO | BytesWritten : 2299375255512 | 20260506 20:50:41.853 | INFO | NumEvents : 1110629202128 | 20260506 20:50:43.339 | INFO | Analyzing chunk frames = [4358144,4358656) | 20260506 20:50:43.339 | INFO | MemInfo: { | 20260506 20:50:43.339 | INFO | "gpu" : " 17 GB", | 20260506 20:50:43.339 | INFO | "huge" : "", | 20260506 20:50:43.339 | INFO | "huge_shared" : " 27 GB", | 20260506 20:50:43.339 | INFO | "malloc" : " 14 GB", | 20260506 20:50:43.339 | INFO | "pinned" : " 29 GB" | 20260506 20:50:43.339 | INFO | } | 20260506 20:50:43.339 | INFO | Perf report for the last 5.13436 seconds: | 20260506 20:50:43.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:50:43.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:43.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:43.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:43.340 | INFO | ANALYSIS : 62.6%, 2.9, 95.2ms, 0 | 20260506 20:50:43.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:43.340 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 20:50:43.340 | INFO | PRE_HQ : 60.0%, 1.8, 57.5ms, 0 | 20260506 20:50:43.340 | INFO | BAZWRITER : 68.6%, 4.8, 169.0ms, 0 | 20260506 20:50:43.340 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 20:50:43.340 | INFO | Uploaded 36GiB to GPU in 2.06069s (17.4699GiB/s) | 20260506 20:50:43.340 | INFO | Downloaded 38GiB to GPU in 2.20889s (17.2032GiB/s) | 20260506 20:50:48.473 | INFO | Analyzing chunk frames = [4358656,4359168) | 20260506 20:50:48.473 | INFO | MemInfo: { | 20260506 20:50:48.473 | INFO | "gpu" : " 17 GB", | 20260506 20:50:48.473 | INFO | "huge" : "", | 20260506 20:50:48.473 | INFO | "huge_shared" : " 27 GB", | 20260506 20:50:48.473 | INFO | "malloc" : " 14 GB", | 20260506 20:50:48.473 | INFO | "pinned" : " 29 GB" | 20260506 20:50:48.474 | INFO | } | 20260506 20:50:48.474 | INFO | Perf report for the last 5.13421 seconds: | 20260506 20:50:48.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:50:48.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:48.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:48.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:48.474 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:50:48.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:48.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:48.474 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:50:48.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:48.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:48.475 | INFO | Uploaded 36GiB to GPU in 2.01514s (17.8648GiB/s) | 20260506 20:50:48.475 | INFO | Downloaded 34GiB to GPU in 1.93368s (17.5831GiB/s) | 20260506 20:50:53.610 | INFO | Analyzing chunk frames = [4359168,4359680) | 20260506 20:50:53.610 | INFO | MemInfo: { | 20260506 20:50:53.610 | INFO | "gpu" : " 17 GB", | 20260506 20:50:53.610 | INFO | "huge" : "", | 20260506 20:50:53.610 | INFO | "huge_shared" : " 27 GB", | 20260506 20:50:53.610 | INFO | "malloc" : " 14 GB", | 20260506 20:50:53.610 | INFO | "pinned" : " 29 GB" | 20260506 20:50:53.610 | INFO | } | 20260506 20:50:53.610 | INFO | Perf report for the last 5.13637 seconds: | 20260506 20:50:53.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:50:53.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:53.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:53.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:53.610 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 20:50:53.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:53.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:53.610 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:50:53.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:53.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:53.610 | INFO | Uploaded 36GiB to GPU in 1.9702s (18.2723GiB/s) | 20260506 20:50:53.610 | INFO | Downloaded 34GiB to GPU in 1.91854s (17.7218GiB/s) | 20260506 20:50:58.751 | INFO | Analyzing chunk frames = [4359680,4360192) | 20260506 20:50:58.751 | INFO | MemInfo: { | 20260506 20:50:58.751 | INFO | "gpu" : " 17 GB", | 20260506 20:50:58.751 | INFO | "huge" : "", | 20260506 20:50:58.751 | INFO | "huge_shared" : " 27 GB", | 20260506 20:50:58.751 | INFO | "malloc" : " 14 GB", | 20260506 20:50:58.751 | INFO | "pinned" : " 29 GB" | 20260506 20:50:58.751 | INFO | } | 20260506 20:50:58.751 | INFO | Perf report for the last 5.14087 seconds: | 20260506 20:50:58.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:50:58.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:58.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:58.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:58.751 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 20:50:58.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:58.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:58.751 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:50:58.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:58.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:50:58.751 | INFO | Uploaded 36GiB to GPU in 1.92288s (18.7219GiB/s) | 20260506 20:50:58.751 | INFO | Downloaded 34GiB to GPU in 1.87575s (18.1261GiB/s) | 20260506 20:51:03.899 | INFO | Analyzing chunk frames = [4360192,4360704) | 20260506 20:51:03.899 | INFO | MemInfo: { | 20260506 20:51:03.899 | INFO | "gpu" : " 17 GB", | 20260506 20:51:03.899 | INFO | "huge" : "", | 20260506 20:51:03.899 | INFO | "huge_shared" : " 27 GB", | 20260506 20:51:03.899 | INFO | "malloc" : " 14 GB", | 20260506 20:51:03.899 | INFO | "pinned" : " 29 GB" | 20260506 20:51:03.899 | INFO | } | 20260506 20:51:03.899 | INFO | Perf report for the last 5.14845 seconds: | 20260506 20:51:03.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:51:03.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:03.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:03.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:03.900 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260506 20:51:03.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:03.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:03.900 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:51:03.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:03.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:03.900 | INFO | Uploaded 36GiB to GPU in 2.00495s (17.9555GiB/s) | 20260506 20:51:03.900 | INFO | Downloaded 34GiB to GPU in 1.91578s (17.7473GiB/s) | 20260506 20:51:09.036 | INFO | Analyzing chunk frames = [4360704,4361216) | 20260506 20:51:09.036 | INFO | MemInfo: { | 20260506 20:51:09.036 | INFO | "gpu" : " 17 GB", | 20260506 20:51:09.036 | INFO | "huge" : "", | 20260506 20:51:09.036 | INFO | "huge_shared" : " 27 GB", | 20260506 20:51:09.036 | INFO | "malloc" : " 14 GB", | 20260506 20:51:09.036 | INFO | "pinned" : " 29 GB" | 20260506 20:51:09.036 | INFO | } | 20260506 20:51:09.036 | INFO | Perf report for the last 5.1368 seconds: | 20260506 20:51:09.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:51:09.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:09.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:09.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:09.036 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 20:51:09.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:09.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:09.036 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:51:09.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:09.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:09.037 | INFO | Uploaded 36GiB to GPU in 2.01238s (17.8893GiB/s) | 20260506 20:51:09.037 | INFO | Downloaded 34GiB to GPU in 1.93926s (17.5324GiB/s) | 20260506 20:51:14.176 | INFO | Analyzing chunk frames = [4361216,4361728) | 20260506 20:51:14.176 | INFO | MemInfo: { | 20260506 20:51:14.176 | INFO | "gpu" : " 17 GB", | 20260506 20:51:14.176 | INFO | "huge" : "", | 20260506 20:51:14.176 | INFO | "huge_shared" : " 27 GB", | 20260506 20:51:14.176 | INFO | "malloc" : " 14 GB", | 20260506 20:51:14.176 | INFO | "pinned" : " 29 GB" | 20260506 20:51:14.176 | INFO | } | 20260506 20:51:14.176 | INFO | Perf report for the last 5.14012 seconds: | 20260506 20:51:14.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:51:14.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:14.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:14.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:14.176 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:51:14.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:14.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:14.177 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 20:51:14.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:14.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:14.177 | INFO | Uploaded 36GiB to GPU in 1.95391s (18.4246GiB/s) | 20260506 20:51:14.177 | INFO | Downloaded 34GiB to GPU in 1.98991s (17.0862GiB/s) | 20260506 20:51:19.310 | INFO | Analyzing chunk frames = [4361728,4362240) | 20260506 20:51:19.310 | INFO | MemInfo: { | 20260506 20:51:19.310 | INFO | "gpu" : " 17 GB", | 20260506 20:51:19.310 | INFO | "huge" : "", | 20260506 20:51:19.310 | INFO | "huge_shared" : " 27 GB", | 20260506 20:51:19.310 | INFO | "malloc" : " 14 GB", | 20260506 20:51:19.310 | INFO | "pinned" : " 29 GB" | 20260506 20:51:19.310 | INFO | } | 20260506 20:51:19.310 | INFO | Perf report for the last 5.13415 seconds: | 20260506 20:51:19.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:51:19.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:19.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:19.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:19.310 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260506 20:51:19.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:19.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:19.310 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 20:51:19.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:19.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:19.311 | INFO | Uploaded 36GiB to GPU in 2.00026s (17.9977GiB/s) | 20260506 20:51:19.311 | INFO | Downloaded 34GiB to GPU in 1.99165s (17.0712GiB/s) | 20260506 20:51:19.418 | INFO | Starting a round of Baz IO | 20260506 20:51:22.547 | INFO | Populating RTMetrics report | 20260506 20:51:22.547 | INFO | RTMetrics report populated | 20260506 20:51:22.547 | INFO | Opening tmp RTMetrics json file | 20260506 20:51:22.583 | INFO | Serializing RTMetrics to json | 20260506 20:51:22.645 | INFO | Serialization done, swapping tmp file now... | 20260506 20:51:22.649 | INFO | RTMetrics done | 20260506 20:51:23.242 | INFO | Finished a round of Baz IO: Wrote 1.965 GiB in 3.824 seconds | 20260506 20:51:23.242 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.11% / 0.00% / 65.88% | 20260506 20:51:23.242 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.693023% / 1.19209e-05% / 4.37101e-05% | 20260506 20:51:23.242 | INFO | Baz IO overall: | 20260506 20:51:23.242 | INFO | BytesWritten : 2301484798924 | 20260506 20:51:23.242 | INFO | NumEvents : 1111640124079 | 20260506 20:51:24.449 | INFO | Analyzing chunk frames = [4362240,4362752) | 20260506 20:51:24.449 | INFO | MemInfo: { | 20260506 20:51:24.449 | INFO | "gpu" : " 17 GB", | 20260506 20:51:24.450 | INFO | "huge" : "", | 20260506 20:51:24.450 | INFO | "huge_shared" : " 27 GB", | 20260506 20:51:24.450 | INFO | "malloc" : " 14 GB", | 20260506 20:51:24.450 | INFO | "pinned" : " 29 GB" | 20260506 20:51:24.450 | INFO | } | 20260506 20:51:24.450 | INFO | Perf report for the last 5.13927 seconds: | 20260506 20:51:24.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:51:24.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:24.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:24.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:24.450 | INFO | ANALYSIS : 62.4%, 2.9, 95.2ms, 0 | 20260506 20:51:24.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:24.450 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 20:51:24.450 | INFO | PRE_HQ : 59.9%, 1.8, 57.2ms, 0 | 20260506 20:51:24.450 | INFO | BAZWRITER : 73.9%, 4.8, 183.3ms, 0 | 20260506 20:51:24.450 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 20:51:24.450 | INFO | Uploaded 36GiB to GPU in 2.06211s (17.4579GiB/s) | 20260506 20:51:24.450 | INFO | Downloaded 38GiB to GPU in 2.19939s (17.2775GiB/s) | 20260506 20:51:29.587 | INFO | Analyzing chunk frames = [4362752,4363264) | 20260506 20:51:29.587 | INFO | MemInfo: { | 20260506 20:51:29.587 | INFO | "gpu" : " 17 GB", | 20260506 20:51:29.587 | INFO | "huge" : "", | 20260506 20:51:29.587 | INFO | "huge_shared" : " 27 GB", | 20260506 20:51:29.587 | INFO | "malloc" : " 14 GB", | 20260506 20:51:29.587 | INFO | "pinned" : " 29 GB" | 20260506 20:51:29.587 | INFO | } | 20260506 20:51:29.587 | INFO | Perf report for the last 5.13749 seconds: | 20260506 20:51:29.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:51:29.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:29.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:29.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:29.587 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:51:29.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:29.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:29.587 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 20:51:29.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:29.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:29.588 | INFO | Uploaded 36GiB to GPU in 2.02233s (17.8012GiB/s) | 20260506 20:51:29.588 | INFO | Downloaded 34GiB to GPU in 1.92755s (17.639GiB/s) | 20260506 20:51:34.724 | INFO | Analyzing chunk frames = [4363264,4363776) | 20260506 20:51:34.724 | INFO | MemInfo: { | 20260506 20:51:34.724 | INFO | "gpu" : " 17 GB", | 20260506 20:51:34.724 | INFO | "huge" : "", | 20260506 20:51:34.724 | INFO | "huge_shared" : " 27 GB", | 20260506 20:51:34.724 | INFO | "malloc" : " 14 GB", | 20260506 20:51:34.724 | INFO | "pinned" : " 29 GB" | 20260506 20:51:34.724 | INFO | } | 20260506 20:51:34.724 | INFO | Perf report for the last 5.1369 seconds: | 20260506 20:51:34.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:51:34.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:34.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:34.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:34.724 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 20:51:34.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:34.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:34.724 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:51:34.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:34.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:34.725 | INFO | Uploaded 36GiB to GPU in 1.9876s (18.1123GiB/s) | 20260506 20:51:34.725 | INFO | Downloaded 34GiB to GPU in 1.91971s (17.711GiB/s) | 20260506 20:51:39.875 | INFO | Analyzing chunk frames = [4363776,4364288) | 20260506 20:51:39.875 | INFO | MemInfo: { | 20260506 20:51:39.875 | INFO | "gpu" : " 17 GB", | 20260506 20:51:39.875 | INFO | "huge" : "", | 20260506 20:51:39.875 | INFO | "huge_shared" : " 27 GB", | 20260506 20:51:39.875 | INFO | "malloc" : " 14 GB", | 20260506 20:51:39.875 | INFO | "pinned" : " 29 GB" | 20260506 20:51:39.875 | INFO | } | 20260506 20:51:39.875 | INFO | Perf report for the last 5.15112 seconds: | 20260506 20:51:39.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:51:39.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:39.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:39.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:39.875 | INFO | ANALYSIS : 59.5%, 3.0, 92.5ms, 0 | 20260506 20:51:39.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:39.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:39.876 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:51:39.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:39.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:39.876 | INFO | Uploaded 36GiB to GPU in 1.94126s (18.5447GiB/s) | 20260506 20:51:39.876 | INFO | Downloaded 34GiB to GPU in 1.8732s (18.1507GiB/s) | 20260506 20:51:45.015 | INFO | Analyzing chunk frames = [4364288,4364800) | 20260506 20:51:45.015 | INFO | MemInfo: { | 20260506 20:51:45.015 | INFO | "gpu" : " 17 GB", | 20260506 20:51:45.015 | INFO | "huge" : "", | 20260506 20:51:45.015 | INFO | "huge_shared" : " 27 GB", | 20260506 20:51:45.015 | INFO | "malloc" : " 14 GB", | 20260506 20:51:45.015 | INFO | "pinned" : " 29 GB" | 20260506 20:51:45.015 | INFO | } | 20260506 20:51:45.015 | INFO | Perf report for the last 5.13952 seconds: | 20260506 20:51:45.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:51:45.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:45.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:45.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:45.015 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 20:51:45.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:45.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:45.015 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:51:45.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:45.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:45.015 | INFO | Uploaded 36GiB to GPU in 1.98945s (18.0954GiB/s) | 20260506 20:51:45.015 | INFO | Downloaded 34GiB to GPU in 1.92215s (17.6885GiB/s) | 20260506 20:51:50.155 | INFO | Analyzing chunk frames = [4364800,4365312) | 20260506 20:51:50.155 | INFO | MemInfo: { | 20260506 20:51:50.155 | INFO | "gpu" : " 17 GB", | 20260506 20:51:50.155 | INFO | "huge" : "", | 20260506 20:51:50.155 | INFO | "huge_shared" : " 27 GB", | 20260506 20:51:50.155 | INFO | "malloc" : " 14 GB", | 20260506 20:51:50.155 | INFO | "pinned" : " 29 GB" | 20260506 20:51:50.155 | INFO | } | 20260506 20:51:50.156 | INFO | Perf report for the last 5.14073 seconds: | 20260506 20:51:50.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:51:50.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:50.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:50.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:50.156 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:51:50.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:50.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:50.156 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 20:51:50.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:50.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:50.156 | INFO | Uploaded 36GiB to GPU in 2.05149s (17.5482GiB/s) | 20260506 20:51:50.156 | INFO | Downloaded 34GiB to GPU in 1.94783s (17.4554GiB/s) | 20260506 20:51:55.288 | INFO | Analyzing chunk frames = [4365312,4365824) | 20260506 20:51:55.288 | INFO | MemInfo: { | 20260506 20:51:55.288 | INFO | "gpu" : " 17 GB", | 20260506 20:51:55.288 | INFO | "huge" : "", | 20260506 20:51:55.288 | INFO | "huge_shared" : " 27 GB", | 20260506 20:51:55.288 | INFO | "malloc" : " 14 GB", | 20260506 20:51:55.288 | INFO | "pinned" : " 29 GB" | 20260506 20:51:55.288 | INFO | } | 20260506 20:51:55.288 | INFO | Perf report for the last 5.13286 seconds: | 20260506 20:51:55.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:51:55.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:55.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:55.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:55.289 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 20:51:55.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:55.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:55.289 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 20:51:55.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:55.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:51:55.289 | INFO | Uploaded 36GiB to GPU in 2.0097s (17.9131GiB/s) | 20260506 20:51:55.289 | INFO | Downloaded 34GiB to GPU in 1.99521s (17.0408GiB/s) | 20260506 20:52:00.423 | INFO | Analyzing chunk frames = [4365824,4366336) | 20260506 20:52:00.423 | INFO | MemInfo: { | 20260506 20:52:00.423 | INFO | "gpu" : " 17 GB", | 20260506 20:52:00.423 | INFO | "huge" : "", | 20260506 20:52:00.423 | INFO | "huge_shared" : " 27 GB", | 20260506 20:52:00.423 | INFO | "malloc" : " 14 GB", | 20260506 20:52:00.423 | INFO | "pinned" : " 29 GB" | 20260506 20:52:00.423 | INFO | } | 20260506 20:52:00.423 | INFO | Perf report for the last 5.13445 seconds: | 20260506 20:52:00.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:52:00.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:00.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:00.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:00.423 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 20:52:00.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:00.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:00.423 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:52:00.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:00.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:00.423 | INFO | Uploaded 36GiB to GPU in 1.96262s (18.3428GiB/s) | 20260506 20:52:00.423 | INFO | Downloaded 34GiB to GPU in 1.9852s (17.1267GiB/s) | 20260506 20:52:00.423 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:52:00.423 | INFO | Total mem usage: 18 GiB | 20260506 20:52:00.423 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:52:00.423 | INFO | Recent High Water Marks Sum: 16.6 GiB (Savings: -8.77%) | 20260506 20:52:00.423 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:52:00.423 | INFO | High Water Breakdown: | 20260506 20:52:00.423 | INFO | 62.4% : 10.3 GiB ( 90.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:52:00.424 | INFO | 37.5% : 6.21 GiB ( 92.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:52:00.424 | INFO | 0.12% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:52:00.424 | INFO | ------------------------------------------ | 20260506 20:52:00.424 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:52:00.424 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:52:00.424 | INFO | Total mem usage: 26.5 GiB | 20260506 20:52:00.424 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:52:00.424 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:52:00.424 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:52:00.424 | INFO | High Water Breakdown: | 20260506 20:52:00.424 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:52:00.424 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:52:00.424 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:52:00.424 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:52:00.424 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:52:00.424 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:52:00.424 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:52:00.424 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:52:00.424 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:52:00.424 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:52:00.424 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:52:00.424 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:52:00.424 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:52:00.424 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:52:00.424 | INFO | ------------------------------------------ | 20260506 20:52:00.424 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:52:00.424 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:52:00.424 | INFO | Total mem usage: 25.6 GiB | 20260506 20:52:00.424 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:52:00.424 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:52:00.424 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:52:00.424 | INFO | High Water Breakdown: | 20260506 20:52:00.424 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:52:00.424 | INFO | ------------------------------------------ | 20260506 20:52:00.424 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:52:00.424 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:52:00.424 | INFO | Total mem usage: 15.7 GiB | 20260506 20:52:00.424 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:52:00.424 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:52:00.424 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 20:52:00.424 | INFO | High Water Breakdown: | 20260506 20:52:00.424 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:52:00.424 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:52:00.424 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:52:00.424 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:52:00.424 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:52:00.424 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:52:00.424 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:52:00.424 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:52:00.424 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:52:00.424 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:52:00.424 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:52:00.424 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:52:00.424 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:52:00.424 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:52:00.424 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:52:00.424 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:52:00.424 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:52:00.424 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:52:00.424 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:52:00.424 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:52:00.424 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:52:00.424 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:52:00.424 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:52:00.424 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:52:00.424 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:52:00.424 | INFO | ------------------------------------------ | 20260506 20:52:00.424 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:52:00.424 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:52:00.424 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:52:00.424 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:52:00.424 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.56e+01, 1.33e+01 / 2.30e+00 | 20260506 20:52:00.424 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.02e-01, 9.70e-03 / 5.54e-03 | 20260506 20:52:00.424 | INFO | PerfReport: 11.65%, Baseline -- 3.63e+00 / 4.44e+00, 5.57e-01 / 3.63e-01 | 20260506 20:52:00.424 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.82e-01 / 1.36e-01 | 20260506 20:52:00.424 | INFO | PerfReport: 16.01%, DME -- 3.99e+01 / 4.45e+01, 3.98e+00 / 2.62e+00 | 20260506 20:52:00.424 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.38e+00 / 2.01e+00 | 20260506 20:52:00.424 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.23e+00 / 2.80e+00, 3.83e-01 / 2.32e-01 | 20260506 20:52:00.424 | INFO | PerfReport: 3.30%, Metrics -- 1.03e+00 / 1.36e+00, 2.71e-01 / 1.90e-01 | 20260506 20:52:00.424 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.52e-01 / 4.01e-01, 5.45e-02 / 2.32e-02 | 20260506 20:52:00.425 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:52:00.425 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:52:00.425 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:52:00.425 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.30e+01 / 0.00e+00 | 20260506 20:52:00.425 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260506 20:52:00.425 | INFO | PerfReport: 21.24%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 20:52:00.425 | INFO | PerfReport: 32.20%, Compute -- 3.11e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 20:52:00.425 | INFO | PerfReport: 1.32%, LockUpload -- 1.28e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260506 20:52:00.425 | INFO | PerfReport: 1.17%, LockDownload -- 1.13e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 20:52:00.425 | INFO | PerfReport: 22.53%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 20:52:00.425 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.38e-02 / 0.00e+00 | 20260506 20:52:00.523 | INFO | Starting a round of Baz IO | 20260506 20:52:03.664 | INFO | Populating RTMetrics report | 20260506 20:52:03.664 | INFO | RTMetrics report populated | 20260506 20:52:03.664 | INFO | Opening tmp RTMetrics json file | 20260506 20:52:03.734 | INFO | Serializing RTMetrics to json | 20260506 20:52:03.778 | INFO | Serialization done, swapping tmp file now... | 20260506 20:52:03.823 | INFO | RTMetrics done | 20260506 20:52:03.845 | INFO | Finished a round of Baz IO: Wrote 1.964 GiB in 3.321 seconds | 20260506 20:52:03.845 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.11% / 0.00% / 65.89% | 20260506 20:52:03.845 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.689479% / 3.97364e-06% / 8.34465e-05% | 20260506 20:52:03.845 | INFO | Baz IO overall: | 20260506 20:52:03.845 | INFO | BytesWritten : 2303593485748 | 20260506 20:52:03.845 | INFO | NumEvents : 1112650397005 | 20260506 20:52:05.561 | INFO | Analyzing chunk frames = [4366336,4366848) | 20260506 20:52:05.561 | INFO | MemInfo: { | 20260506 20:52:05.561 | INFO | "gpu" : " 17 GB", | 20260506 20:52:05.561 | INFO | "huge" : "", | 20260506 20:52:05.561 | INFO | "huge_shared" : " 27 GB", | 20260506 20:52:05.561 | INFO | "malloc" : " 14 GB", | 20260506 20:52:05.561 | INFO | "pinned" : " 29 GB" | 20260506 20:52:05.561 | INFO | } | 20260506 20:52:05.562 | INFO | Perf report for the last 5.1387 seconds: | 20260506 20:52:05.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:52:05.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:05.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:05.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:05.562 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 20:52:05.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:05.562 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260506 20:52:05.562 | INFO | PRE_HQ : 60.0%, 1.8, 57.1ms, 0 | 20260506 20:52:05.562 | INFO | BAZWRITER : 64.2%, 4.8, 157.7ms, 0 | 20260506 20:52:05.562 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 20:52:05.562 | INFO | Uploaded 36GiB to GPU in 2.01883s (17.8321GiB/s) | 20260506 20:52:05.562 | INFO | Downloaded 38GiB to GPU in 2.20041s (17.2695GiB/s) | 20260506 20:52:10.709 | INFO | Analyzing chunk frames = [4366848,4367360) | 20260506 20:52:10.709 | INFO | MemInfo: { | 20260506 20:52:10.709 | INFO | "gpu" : " 17 GB", | 20260506 20:52:10.709 | INFO | "huge" : "", | 20260506 20:52:10.709 | INFO | "huge_shared" : " 27 GB", | 20260506 20:52:10.709 | INFO | "malloc" : " 14 GB", | 20260506 20:52:10.709 | INFO | "pinned" : " 29 GB" | 20260506 20:52:10.709 | INFO | } | 20260506 20:52:10.710 | INFO | Perf report for the last 5.14792 seconds: | 20260506 20:52:10.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:52:10.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:10.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:10.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:10.710 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 20:52:10.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:10.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:10.710 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:52:10.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:10.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:10.710 | INFO | Uploaded 36GiB to GPU in 2.00384s (17.9655GiB/s) | 20260506 20:52:10.710 | INFO | Downloaded 34GiB to GPU in 1.92858s (17.6295GiB/s) | 20260506 20:52:15.846 | INFO | Analyzing chunk frames = [4367360,4367872) | 20260506 20:52:15.846 | INFO | MemInfo: { | 20260506 20:52:15.846 | INFO | "gpu" : " 17 GB", | 20260506 20:52:15.846 | INFO | "huge" : "", | 20260506 20:52:15.846 | INFO | "huge_shared" : " 27 GB", | 20260506 20:52:15.846 | INFO | "malloc" : " 14 GB", | 20260506 20:52:15.846 | INFO | "pinned" : " 29 GB" | 20260506 20:52:15.846 | INFO | } | 20260506 20:52:15.846 | INFO | Perf report for the last 5.13677 seconds: | 20260506 20:52:15.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:52:15.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:15.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:15.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:15.847 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 20:52:15.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:15.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:15.847 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:52:15.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:15.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:15.847 | INFO | Uploaded 36GiB to GPU in 1.96976s (18.2763GiB/s) | 20260506 20:52:15.847 | INFO | Downloaded 34GiB to GPU in 1.92212s (17.6888GiB/s) | 20260506 20:52:20.987 | INFO | Analyzing chunk frames = [4367872,4368384) | 20260506 20:52:20.987 | INFO | MemInfo: { | 20260506 20:52:20.987 | INFO | "gpu" : " 17 GB", | 20260506 20:52:20.987 | INFO | "huge" : "", | 20260506 20:52:20.987 | INFO | "huge_shared" : " 27 GB", | 20260506 20:52:20.987 | INFO | "malloc" : " 14 GB", | 20260506 20:52:20.987 | INFO | "pinned" : " 29 GB" | 20260506 20:52:20.987 | INFO | } | 20260506 20:52:20.987 | INFO | Perf report for the last 5.14093 seconds: | 20260506 20:52:20.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:52:20.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:20.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:20.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:20.987 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 20:52:20.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:20.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:20.987 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:52:20.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:20.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:20.988 | INFO | Uploaded 36GiB to GPU in 1.94228s (18.5349GiB/s) | 20260506 20:52:20.988 | INFO | Downloaded 34GiB to GPU in 1.8716s (18.1662GiB/s) | 20260506 20:52:26.129 | INFO | Analyzing chunk frames = [4368384,4368896) | 20260506 20:52:26.129 | INFO | MemInfo: { | 20260506 20:52:26.129 | INFO | "gpu" : " 17 GB", | 20260506 20:52:26.129 | INFO | "huge" : "", | 20260506 20:52:26.129 | INFO | "huge_shared" : " 27 GB", | 20260506 20:52:26.130 | INFO | "malloc" : " 14 GB", | 20260506 20:52:26.130 | INFO | "pinned" : " 29 GB" | 20260506 20:52:26.130 | INFO | } | 20260506 20:52:26.130 | INFO | Perf report for the last 5.14245 seconds: | 20260506 20:52:26.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:52:26.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:26.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:26.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:26.130 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 20:52:26.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:26.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:26.130 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:52:26.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:26.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:26.130 | INFO | Uploaded 36GiB to GPU in 1.99479s (18.047GiB/s) | 20260506 20:52:26.130 | INFO | Downloaded 34GiB to GPU in 1.92299s (17.6808GiB/s) | 20260506 20:52:31.265 | INFO | Analyzing chunk frames = [4368896,4369408) | 20260506 20:52:31.265 | INFO | MemInfo: { | 20260506 20:52:31.265 | INFO | "gpu" : " 17 GB", | 20260506 20:52:31.265 | INFO | "huge" : "", | 20260506 20:52:31.265 | INFO | "huge_shared" : " 27 GB", | 20260506 20:52:31.265 | INFO | "malloc" : " 14 GB", | 20260506 20:52:31.265 | INFO | "pinned" : " 29 GB" | 20260506 20:52:31.265 | INFO | } | 20260506 20:52:31.265 | INFO | Perf report for the last 5.13538 seconds: | 20260506 20:52:31.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:52:31.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:31.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:31.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:31.265 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:52:31.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:31.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:31.265 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 20:52:31.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:31.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:31.265 | INFO | Uploaded 36GiB to GPU in 2.01022s (17.9085GiB/s) | 20260506 20:52:31.265 | INFO | Downloaded 34GiB to GPU in 1.9429s (17.4996GiB/s) | 20260506 20:52:36.399 | INFO | Analyzing chunk frames = [4369408,4369920) | 20260506 20:52:36.399 | INFO | MemInfo: { | 20260506 20:52:36.399 | INFO | "gpu" : " 17 GB", | 20260506 20:52:36.399 | INFO | "huge" : "", | 20260506 20:52:36.399 | INFO | "huge_shared" : " 27 GB", | 20260506 20:52:36.399 | INFO | "malloc" : " 14 GB", | 20260506 20:52:36.399 | INFO | "pinned" : " 29 GB" | 20260506 20:52:36.399 | INFO | } | 20260506 20:52:36.399 | INFO | Perf report for the last 5.13427 seconds: | 20260506 20:52:36.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:52:36.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:36.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:36.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:36.399 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:52:36.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:36.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:36.399 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:52:36.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:36.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:36.400 | INFO | Uploaded 36GiB to GPU in 1.9615s (18.3533GiB/s) | 20260506 20:52:36.400 | INFO | Downloaded 34GiB to GPU in 1.99681s (17.0271GiB/s) | 20260506 20:52:41.539 | INFO | Analyzing chunk frames = [4369920,4370432) | 20260506 20:52:41.539 | INFO | MemInfo: { | 20260506 20:52:41.539 | INFO | "gpu" : " 17 GB", | 20260506 20:52:41.540 | INFO | "huge" : "", | 20260506 20:52:41.540 | INFO | "huge_shared" : " 27 GB", | 20260506 20:52:41.540 | INFO | "malloc" : " 14 GB", | 20260506 20:52:41.540 | INFO | "pinned" : " 29 GB" | 20260506 20:52:41.540 | INFO | } | 20260506 20:52:41.540 | INFO | Perf report for the last 5.14033 seconds: | 20260506 20:52:41.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:52:41.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:41.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:41.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:41.540 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 20:52:41.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:41.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:41.540 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:52:41.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:41.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:41.540 | INFO | Uploaded 36GiB to GPU in 1.99057s (18.0852GiB/s) | 20260506 20:52:41.540 | INFO | Downloaded 34GiB to GPU in 1.98994s (17.0859GiB/s) | 20260506 20:52:41.643 | INFO | Starting a round of Baz IO | 20260506 20:52:44.775 | INFO | Populating RTMetrics report | 20260506 20:52:44.776 | INFO | RTMetrics report populated | 20260506 20:52:44.776 | INFO | Opening tmp RTMetrics json file | 20260506 20:52:44.793 | INFO | Serializing RTMetrics to json | 20260506 20:52:44.811 | INFO | Serialization done, swapping tmp file now... | 20260506 20:52:44.813 | INFO | RTMetrics done | 20260506 20:52:44.938 | INFO | Finished a round of Baz IO: Wrote 1.963 GiB in 3.294 seconds | 20260506 20:52:44.938 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.11% / 0.00% / 65.89% | 20260506 20:52:44.938 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.689578% / 7.94729e-06% / 6.35783e-05% | 20260506 20:52:44.938 | INFO | Baz IO overall: | 20260506 20:52:44.938 | INFO | BytesWritten : 2305701455257 | 20260506 20:52:44.938 | INFO | NumEvents : 1113660266370 | 20260506 20:52:46.676 | INFO | Analyzing chunk frames = [4370432,4370944) | 20260506 20:52:46.676 | INFO | MemInfo: { | 20260506 20:52:46.676 | INFO | "gpu" : " 17 GB", | 20260506 20:52:46.676 | INFO | "huge" : "", | 20260506 20:52:46.676 | INFO | "huge_shared" : " 27 GB", | 20260506 20:52:46.676 | INFO | "malloc" : " 14 GB", | 20260506 20:52:46.676 | INFO | "pinned" : " 29 GB" | 20260506 20:52:46.676 | INFO | } | 20260506 20:52:46.676 | INFO | Perf report for the last 5.13629 seconds: | 20260506 20:52:46.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:52:46.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:46.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:46.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:46.676 | INFO | ANALYSIS : 62.4%, 2.9, 94.8ms, 0 | 20260506 20:52:46.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:46.676 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 20:52:46.676 | INFO | PRE_HQ : 59.8%, 1.9, 57.5ms, 0 | 20260506 20:52:46.676 | INFO | BAZWRITER : 63.8%, 4.8, 157.0ms, 0 | 20260506 20:52:46.676 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260506 20:52:46.676 | INFO | Uploaded 36GiB to GPU in 2.06825s (17.406GiB/s) | 20260506 20:52:46.676 | INFO | Downloaded 38GiB to GPU in 2.19559s (17.3074GiB/s) | 20260506 20:52:51.818 | INFO | Analyzing chunk frames = [4370944,4371456) | 20260506 20:52:51.818 | INFO | MemInfo: { | 20260506 20:52:51.818 | INFO | "gpu" : " 17 GB", | 20260506 20:52:51.818 | INFO | "huge" : "", | 20260506 20:52:51.818 | INFO | "huge_shared" : " 27 GB", | 20260506 20:52:51.818 | INFO | "malloc" : " 14 GB", | 20260506 20:52:51.818 | INFO | "pinned" : " 29 GB" | 20260506 20:52:51.818 | INFO | } | 20260506 20:52:51.818 | INFO | Perf report for the last 5.14153 seconds: | 20260506 20:52:51.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:52:51.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:51.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:51.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:51.818 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:52:51.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:51.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:51.818 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 20:52:51.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:51.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:51.818 | INFO | Uploaded 36GiB to GPU in 2.01717s (17.8468GiB/s) | 20260506 20:52:51.818 | INFO | Downloaded 34GiB to GPU in 1.93087s (17.6086GiB/s) | 20260506 20:52:56.969 | INFO | Analyzing chunk frames = [4371456,4371968) | 20260506 20:52:56.969 | INFO | MemInfo: { | 20260506 20:52:56.969 | INFO | "gpu" : " 17 GB", | 20260506 20:52:56.969 | INFO | "huge" : "", | 20260506 20:52:56.969 | INFO | "huge_shared" : " 27 GB", | 20260506 20:52:56.969 | INFO | "malloc" : " 14 GB", | 20260506 20:52:56.969 | INFO | "pinned" : " 29 GB" | 20260506 20:52:56.969 | INFO | } | 20260506 20:52:56.969 | INFO | Perf report for the last 5.15107 seconds: | 20260506 20:52:56.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:52:56.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:56.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:56.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:56.969 | INFO | ANALYSIS : 60.6%, 3.0, 94.0ms, 0 | 20260506 20:52:56.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:56.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:56.969 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:52:56.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:56.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:52:56.969 | INFO | Uploaded 36GiB to GPU in 1.98746s (18.1136GiB/s) | 20260506 20:52:56.969 | INFO | Downloaded 34GiB to GPU in 1.9176s (17.7305GiB/s) | 20260506 20:53:02.101 | INFO | Analyzing chunk frames = [4371968,4372480) | 20260506 20:53:02.101 | INFO | MemInfo: { | 20260506 20:53:02.101 | INFO | "gpu" : " 17 GB", | 20260506 20:53:02.101 | INFO | "huge" : "", | 20260506 20:53:02.101 | INFO | "huge_shared" : " 27 GB", | 20260506 20:53:02.101 | INFO | "malloc" : " 14 GB", | 20260506 20:53:02.101 | INFO | "pinned" : " 29 GB" | 20260506 20:53:02.101 | INFO | } | 20260506 20:53:02.101 | INFO | Perf report for the last 5.13257 seconds: | 20260506 20:53:02.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:53:02.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:02.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:02.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:02.101 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 20:53:02.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:02.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:02.101 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:53:02.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:02.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:02.102 | INFO | Uploaded 36GiB to GPU in 2.01753s (17.8436GiB/s) | 20260506 20:53:02.102 | INFO | Downloaded 34GiB to GPU in 1.90408s (17.8564GiB/s) | 20260506 20:53:07.240 | INFO | Analyzing chunk frames = [4372480,4372992) | 20260506 20:53:07.240 | INFO | MemInfo: { | 20260506 20:53:07.240 | INFO | "gpu" : " 17 GB", | 20260506 20:53:07.240 | INFO | "huge" : "", | 20260506 20:53:07.240 | INFO | "huge_shared" : " 27 GB", | 20260506 20:53:07.240 | INFO | "malloc" : " 14 GB", | 20260506 20:53:07.240 | INFO | "pinned" : " 29 GB" | 20260506 20:53:07.240 | INFO | } | 20260506 20:53:07.240 | INFO | Perf report for the last 5.139 seconds: | 20260506 20:53:07.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:53:07.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:07.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:07.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:07.240 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 20:53:07.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:07.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:07.241 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:53:07.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:07.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:07.241 | INFO | Uploaded 36GiB to GPU in 2.01534s (17.863GiB/s) | 20260506 20:53:07.241 | INFO | Downloaded 34GiB to GPU in 1.9076s (17.8235GiB/s) | 20260506 20:53:12.377 | INFO | Analyzing chunk frames = [4372992,4373504) | 20260506 20:53:12.377 | INFO | MemInfo: { | 20260506 20:53:12.377 | INFO | "gpu" : " 17 GB", | 20260506 20:53:12.378 | INFO | "huge" : "", | 20260506 20:53:12.378 | INFO | "huge_shared" : " 27 GB", | 20260506 20:53:12.378 | INFO | "malloc" : " 14 GB", | 20260506 20:53:12.378 | INFO | "pinned" : " 29 GB" | 20260506 20:53:12.378 | INFO | } | 20260506 20:53:12.378 | INFO | Perf report for the last 5.13729 seconds: | 20260506 20:53:12.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:53:12.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:12.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:12.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:12.378 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:53:12.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:12.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:12.378 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 20:53:12.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:12.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:12.378 | INFO | Uploaded 36GiB to GPU in 2.00532s (17.9523GiB/s) | 20260506 20:53:12.378 | INFO | Downloaded 34GiB to GPU in 1.94049s (17.5213GiB/s) | 20260506 20:53:17.514 | INFO | Analyzing chunk frames = [4373504,4374016) | 20260506 20:53:17.514 | INFO | MemInfo: { | 20260506 20:53:17.514 | INFO | "gpu" : " 17 GB", | 20260506 20:53:17.514 | INFO | "huge" : "", | 20260506 20:53:17.514 | INFO | "huge_shared" : " 27 GB", | 20260506 20:53:17.514 | INFO | "malloc" : " 14 GB", | 20260506 20:53:17.514 | INFO | "pinned" : " 29 GB" | 20260506 20:53:17.514 | INFO | } | 20260506 20:53:17.514 | INFO | Perf report for the last 5.13681 seconds: | 20260506 20:53:17.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:53:17.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:17.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:17.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:17.515 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:53:17.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:17.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:17.515 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 20:53:17.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:17.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:17.515 | INFO | Uploaded 36GiB to GPU in 1.95885s (18.3781GiB/s) | 20260506 20:53:17.515 | INFO | Downloaded 34GiB to GPU in 1.99631s (17.0314GiB/s) | 20260506 20:53:22.653 | INFO | Analyzing chunk frames = [4374016,4374528) | 20260506 20:53:22.653 | INFO | MemInfo: { | 20260506 20:53:22.653 | INFO | "gpu" : " 17 GB", | 20260506 20:53:22.653 | INFO | "huge" : "", | 20260506 20:53:22.653 | INFO | "huge_shared" : " 27 GB", | 20260506 20:53:22.653 | INFO | "malloc" : " 14 GB", | 20260506 20:53:22.653 | INFO | "pinned" : " 29 GB" | 20260506 20:53:22.653 | INFO | } | 20260506 20:53:22.653 | INFO | Perf report for the last 5.13852 seconds: | 20260506 20:53:22.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:53:22.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:22.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:22.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:22.653 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 20:53:22.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:22.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:22.653 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:53:22.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:22.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:22.653 | INFO | Uploaded 36GiB to GPU in 2.01027s (17.908GiB/s) | 20260506 20:53:22.653 | INFO | Downloaded 34GiB to GPU in 1.99677s (17.0275GiB/s) | 20260506 20:53:22.653 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:53:22.654 | INFO | Total mem usage: 18 GiB | 20260506 20:53:22.654 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:53:22.654 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.7%) | 20260506 20:53:22.654 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:53:22.654 | INFO | High Water Breakdown: | 20260506 20:53:22.654 | INFO | 61.4% : 9.56 GiB ( 83.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:53:22.654 | INFO | 38.4% : 5.98 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:53:22.654 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:53:22.654 | INFO | ------------------------------------------ | 20260506 20:53:22.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:53:22.654 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:53:22.654 | INFO | Total mem usage: 26.5 GiB | 20260506 20:53:22.654 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:53:22.654 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:53:22.654 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:53:22.654 | INFO | High Water Breakdown: | 20260506 20:53:22.654 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:53:22.654 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:53:22.654 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:53:22.654 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:53:22.654 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:53:22.654 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:53:22.654 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:53:22.654 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:53:22.654 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:53:22.654 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:53:22.654 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:53:22.654 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:53:22.654 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:53:22.654 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:53:22.654 | INFO | ------------------------------------------ | 20260506 20:53:22.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:53:22.654 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:53:22.654 | INFO | Total mem usage: 25.6 GiB | 20260506 20:53:22.654 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:53:22.654 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:53:22.654 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:53:22.654 | INFO | High Water Breakdown: | 20260506 20:53:22.654 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:53:22.654 | INFO | ------------------------------------------ | 20260506 20:53:22.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:53:22.654 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:53:22.654 | INFO | Total mem usage: 15.7 GiB | 20260506 20:53:22.654 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:53:22.654 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:53:22.654 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 20:53:22.654 | INFO | High Water Breakdown: | 20260506 20:53:22.654 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:53:22.654 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:53:22.654 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:53:22.654 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:53:22.654 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:53:22.654 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:53:22.654 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:53:22.654 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:53:22.654 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:53:22.654 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:53:22.654 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:53:22.654 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:53:22.654 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:53:22.654 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:53:22.654 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:53:22.654 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:53:22.654 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:53:22.654 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:53:22.654 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:53:22.654 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:53:22.654 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:53:22.654 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:53:22.654 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:53:22.654 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:53:22.654 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:53:22.654 | INFO | ------------------------------------------ | 20260506 20:53:22.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:53:22.654 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:53:22.654 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:53:22.654 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:53:22.654 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.59e+01, 1.35e+01 / 2.13e+00 | 20260506 20:53:22.654 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 1.01e-02 / 6.97e-03 | 20260506 20:53:22.654 | INFO | PerfReport: 11.64%, Baseline -- 3.62e+00 / 4.38e+00, 5.51e-01 / 3.78e-01 | 20260506 20:53:22.654 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.83e-01 / 1.41e-01 | 20260506 20:53:22.654 | INFO | PerfReport: 16.02%, DME -- 3.99e+01 / 4.45e+01, 4.17e+00 / 2.61e+00 | 20260506 20:53:22.654 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.37e+00 / 1.95e+00 | 20260506 20:53:22.655 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.22e+00 / 2.76e+00, 3.80e-01 / 2.33e-01 | 20260506 20:53:22.655 | INFO | PerfReport: 3.30%, Metrics -- 1.03e+00 / 1.36e+00, 2.72e-01 / 1.62e-01 | 20260506 20:53:22.655 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.52e-01 / 4.02e-01, 5.55e-02 / 2.09e-02 | 20260506 20:53:22.655 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:53:22.655 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:53:22.655 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:53:22.655 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 20:53:22.655 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 20:53:22.655 | INFO | PerfReport: 21.25%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 20:53:22.655 | INFO | PerfReport: 32.17%, Compute -- 3.11e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 20:53:22.655 | INFO | PerfReport: 1.32%, LockUpload -- 1.28e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260506 20:53:22.655 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260506 20:53:22.655 | INFO | PerfReport: 22.51%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 20:53:22.655 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260506 20:53:22.758 | INFO | Starting a round of Baz IO | 20260506 20:53:25.889 | INFO | Populating RTMetrics report | 20260506 20:53:25.889 | INFO | RTMetrics report populated | 20260506 20:53:25.889 | INFO | Opening tmp RTMetrics json file | 20260506 20:53:25.894 | INFO | Serializing RTMetrics to json | 20260506 20:53:25.897 | INFO | Serialization done, swapping tmp file now... | 20260506 20:53:25.908 | INFO | RTMetrics done | 20260506 20:53:26.154 | INFO | Finished a round of Baz IO: Wrote 1.965 GiB in 3.396 seconds | 20260506 20:53:26.154 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.10% / 0.00% / 65.89% | 20260506 20:53:26.154 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.690178% / 3.97364e-06% / 7.94729e-05% | 20260506 20:53:26.154 | INFO | Baz IO overall: | 20260506 20:53:26.154 | INFO | BytesWritten : 2307811767532 | 20260506 20:53:26.154 | INFO | NumEvents : 1114671847494 | 20260506 20:53:27.798 | INFO | Analyzing chunk frames = [4374528,4375040) | 20260506 20:53:27.798 | INFO | MemInfo: { | 20260506 20:53:27.798 | INFO | "gpu" : " 17 GB", | 20260506 20:53:27.798 | INFO | "huge" : "", | 20260506 20:53:27.798 | INFO | "huge_shared" : " 27 GB", | 20260506 20:53:27.798 | INFO | "malloc" : " 14 GB", | 20260506 20:53:27.798 | INFO | "pinned" : " 29 GB" | 20260506 20:53:27.798 | INFO | } | 20260506 20:53:27.798 | INFO | Perf report for the last 5.14497 seconds: | 20260506 20:53:27.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:53:27.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:27.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:27.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:27.798 | INFO | ANALYSIS : 62.3%, 2.9, 95.6ms, 0 | 20260506 20:53:27.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:27.798 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 20:53:27.798 | INFO | PRE_HQ : 59.8%, 1.8, 57.0ms, 0 | 20260506 20:53:27.798 | INFO | BAZWRITER : 65.6%, 4.8, 161.0ms, 0 | 20260506 20:53:27.798 | INFO | BAZ_SAMPLING : 3.0%, 1.0, 1.6ms, 0 | 20260506 20:53:27.798 | INFO | Uploaded 36GiB to GPU in 2.0502s (17.5592GiB/s) | 20260506 20:53:27.799 | INFO | Downloaded 38GiB to GPU in 2.19262s (17.3309GiB/s) | 20260506 20:53:32.934 | INFO | Analyzing chunk frames = [4375040,4375552) | 20260506 20:53:32.934 | INFO | MemInfo: { | 20260506 20:53:32.934 | INFO | "gpu" : " 17 GB", | 20260506 20:53:32.934 | INFO | "huge" : "", | 20260506 20:53:32.934 | INFO | "huge_shared" : " 27 GB", | 20260506 20:53:32.934 | INFO | "malloc" : " 14 GB", | 20260506 20:53:32.934 | INFO | "pinned" : " 29 GB" | 20260506 20:53:32.934 | INFO | } | 20260506 20:53:32.934 | INFO | Perf report for the last 5.13612 seconds: | 20260506 20:53:32.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:53:32.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:32.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:32.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:32.934 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 20:53:32.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:32.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:32.934 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:53:32.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:32.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:32.934 | INFO | Uploaded 36GiB to GPU in 2.0169s (17.8491GiB/s) | 20260506 20:53:32.934 | INFO | Downloaded 34GiB to GPU in 1.92717s (17.6425GiB/s) | 20260506 20:53:38.071 | INFO | Analyzing chunk frames = [4375552,4376064) | 20260506 20:53:38.071 | INFO | MemInfo: { | 20260506 20:53:38.071 | INFO | "gpu" : " 17 GB", | 20260506 20:53:38.071 | INFO | "huge" : "", | 20260506 20:53:38.071 | INFO | "huge_shared" : " 27 GB", | 20260506 20:53:38.071 | INFO | "malloc" : " 14 GB", | 20260506 20:53:38.071 | INFO | "pinned" : " 29 GB" | 20260506 20:53:38.071 | INFO | } | 20260506 20:53:38.071 | INFO | Perf report for the last 5.13733 seconds: | 20260506 20:53:38.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:53:38.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:38.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:38.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:38.071 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260506 20:53:38.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:38.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:38.072 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 20:53:38.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:38.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:38.072 | INFO | Uploaded 36GiB to GPU in 1.97934s (18.1879GiB/s) | 20260506 20:53:38.072 | INFO | Downloaded 34GiB to GPU in 1.91553s (17.7496GiB/s) | 20260506 20:53:43.217 | INFO | Analyzing chunk frames = [4376064,4376576) | 20260506 20:53:43.217 | INFO | MemInfo: { | 20260506 20:53:43.217 | INFO | "gpu" : " 17 GB", | 20260506 20:53:43.217 | INFO | "huge" : "", | 20260506 20:53:43.217 | INFO | "huge_shared" : " 27 GB", | 20260506 20:53:43.217 | INFO | "malloc" : " 14 GB", | 20260506 20:53:43.217 | INFO | "pinned" : " 29 GB" | 20260506 20:53:43.217 | INFO | } | 20260506 20:53:43.217 | INFO | Perf report for the last 5.14582 seconds: | 20260506 20:53:43.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:53:43.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:43.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:43.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:43.217 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:53:43.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:43.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:43.217 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:53:43.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:43.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:43.218 | INFO | Uploaded 36GiB to GPU in 1.93037s (18.6493GiB/s) | 20260506 20:53:43.218 | INFO | Downloaded 34GiB to GPU in 1.86998s (18.182GiB/s) | 20260506 20:53:48.357 | INFO | Analyzing chunk frames = [4376576,4377088) | 20260506 20:53:48.357 | INFO | MemInfo: { | 20260506 20:53:48.357 | INFO | "gpu" : " 17 GB", | 20260506 20:53:48.357 | INFO | "huge" : "", | 20260506 20:53:48.357 | INFO | "huge_shared" : " 27 GB", | 20260506 20:53:48.357 | INFO | "malloc" : " 14 GB", | 20260506 20:53:48.357 | INFO | "pinned" : " 29 GB" | 20260506 20:53:48.357 | INFO | } | 20260506 20:53:48.357 | INFO | Perf report for the last 5.13989 seconds: | 20260506 20:53:48.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:53:48.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:48.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:48.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:48.357 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 20:53:48.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:48.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:48.357 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:53:48.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:48.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:48.357 | INFO | Uploaded 36GiB to GPU in 2.01421s (17.873GiB/s) | 20260506 20:53:48.358 | INFO | Downloaded 34GiB to GPU in 1.91811s (17.7258GiB/s) | 20260506 20:53:53.491 | INFO | Analyzing chunk frames = [4377088,4377600) | 20260506 20:53:53.491 | INFO | MemInfo: { | 20260506 20:53:53.491 | INFO | "gpu" : " 17 GB", | 20260506 20:53:53.491 | INFO | "huge" : "", | 20260506 20:53:53.491 | INFO | "huge_shared" : " 27 GB", | 20260506 20:53:53.491 | INFO | "malloc" : " 14 GB", | 20260506 20:53:53.491 | INFO | "pinned" : " 29 GB" | 20260506 20:53:53.491 | INFO | } | 20260506 20:53:53.491 | INFO | Perf report for the last 5.13377 seconds: | 20260506 20:53:53.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:53:53.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:53.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:53.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:53.491 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 20:53:53.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:53.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:53.491 | INFO | PRE_HQ : 31.1%, 1.0, 16.1ms, 0 | 20260506 20:53:53.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:53.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:53.491 | INFO | Uploaded 36GiB to GPU in 2.02062s (17.8163GiB/s) | 20260506 20:53:53.491 | INFO | Downloaded 34GiB to GPU in 1.94891s (17.4456GiB/s) | 20260506 20:53:58.631 | INFO | Analyzing chunk frames = [4377600,4378112) | 20260506 20:53:58.631 | INFO | MemInfo: { | 20260506 20:53:58.631 | INFO | "gpu" : " 17 GB", | 20260506 20:53:58.631 | INFO | "huge" : "", | 20260506 20:53:58.631 | INFO | "huge_shared" : " 27 GB", | 20260506 20:53:58.631 | INFO | "malloc" : " 14 GB", | 20260506 20:53:58.631 | INFO | "pinned" : " 29 GB" | 20260506 20:53:58.631 | INFO | } | 20260506 20:53:58.631 | INFO | Perf report for the last 5.13977 seconds: | 20260506 20:53:58.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:53:58.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:58.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:58.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:58.631 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:53:58.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:58.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:58.631 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 20:53:58.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:58.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:53:58.631 | INFO | Uploaded 36GiB to GPU in 1.98375s (18.1474GiB/s) | 20260506 20:53:58.631 | INFO | Downloaded 34GiB to GPU in 2.00569s (16.9518GiB/s) | 20260506 20:54:03.766 | INFO | Analyzing chunk frames = [4378112,4378624) | 20260506 20:54:03.766 | INFO | MemInfo: { | 20260506 20:54:03.766 | INFO | "gpu" : " 17 GB", | 20260506 20:54:03.766 | INFO | "huge" : "", | 20260506 20:54:03.766 | INFO | "huge_shared" : " 27 GB", | 20260506 20:54:03.766 | INFO | "malloc" : " 14 GB", | 20260506 20:54:03.766 | INFO | "pinned" : " 29 GB" | 20260506 20:54:03.766 | INFO | } | 20260506 20:54:03.766 | INFO | Perf report for the last 5.13557 seconds: | 20260506 20:54:03.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:54:03.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:03.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:03.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:03.766 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:54:03.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:03.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:03.766 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 20:54:03.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:03.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:03.767 | INFO | Uploaded 36GiB to GPU in 1.99332s (18.0604GiB/s) | 20260506 20:54:03.767 | INFO | Downloaded 34GiB to GPU in 1.98114s (17.1618GiB/s) | 20260506 20:54:03.866 | INFO | Starting a round of Baz IO | 20260506 20:54:07.000 | INFO | Populating RTMetrics report | 20260506 20:54:07.000 | INFO | RTMetrics report populated | 20260506 20:54:07.000 | INFO | Opening tmp RTMetrics json file | 20260506 20:54:07.028 | INFO | Serializing RTMetrics to json | 20260506 20:54:07.060 | INFO | Serialization done, swapping tmp file now... | 20260506 20:54:07.063 | INFO | RTMetrics done | 20260506 20:54:07.660 | INFO | Finished a round of Baz IO: Wrote 1.964 GiB in 3.794 seconds | 20260506 20:54:07.660 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.10% / 0.00% / 65.90% | 20260506 20:54:07.660 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.686173% / 0% / 7.54992e-05% | 20260506 20:54:07.660 | INFO | Baz IO overall: | 20260506 20:54:07.660 | INFO | BytesWritten : 2309920099865 | 20260506 20:54:07.660 | INFO | NumEvents : 1115682233770 | 20260506 20:54:08.907 | INFO | Analyzing chunk frames = [4378624,4379136) | 20260506 20:54:08.907 | INFO | MemInfo: { | 20260506 20:54:08.907 | INFO | "gpu" : " 17 GB", | 20260506 20:54:08.907 | INFO | "huge" : "", | 20260506 20:54:08.907 | INFO | "huge_shared" : " 27 GB", | 20260506 20:54:08.907 | INFO | "malloc" : " 14 GB", | 20260506 20:54:08.907 | INFO | "pinned" : " 29 GB" | 20260506 20:54:08.907 | INFO | } | 20260506 20:54:08.907 | INFO | Perf report for the last 5.1404 seconds: | 20260506 20:54:08.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:54:08.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:08.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:08.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:08.907 | INFO | ANALYSIS : 62.4%, 2.9, 95.4ms, 0 | 20260506 20:54:08.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:08.907 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 20:54:08.907 | INFO | PRE_HQ : 59.9%, 1.8, 56.9ms, 0 | 20260506 20:54:08.907 | INFO | BAZWRITER : 73.5%, 4.8, 181.3ms, 0 | 20260506 20:54:08.907 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 20:54:08.907 | INFO | Uploaded 36GiB to GPU in 2.04374s (17.6148GiB/s) | 20260506 20:54:08.907 | INFO | Downloaded 38GiB to GPU in 2.18563s (17.3863GiB/s) | 20260506 20:54:14.054 | INFO | Analyzing chunk frames = [4379136,4379648) | 20260506 20:54:14.054 | INFO | MemInfo: { | 20260506 20:54:14.054 | INFO | "gpu" : " 17 GB", | 20260506 20:54:14.054 | INFO | "huge" : "", | 20260506 20:54:14.054 | INFO | "huge_shared" : " 27 GB", | 20260506 20:54:14.054 | INFO | "malloc" : " 14 GB", | 20260506 20:54:14.055 | INFO | "pinned" : " 29 GB" | 20260506 20:54:14.055 | INFO | } | 20260506 20:54:14.055 | INFO | Perf report for the last 5.14797 seconds: | 20260506 20:54:14.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:54:14.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:14.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:14.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:14.055 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:54:14.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:14.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:14.055 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:54:14.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:14.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:14.055 | INFO | Uploaded 36GiB to GPU in 1.99984s (18.0015GiB/s) | 20260506 20:54:14.055 | INFO | Downloaded 34GiB to GPU in 1.92396s (17.6718GiB/s) | 20260506 20:54:19.189 | INFO | Analyzing chunk frames = [4379648,4380160) | 20260506 20:54:19.189 | INFO | MemInfo: { | 20260506 20:54:19.189 | INFO | "gpu" : " 17 GB", | 20260506 20:54:19.189 | INFO | "huge" : "", | 20260506 20:54:19.189 | INFO | "huge_shared" : " 27 GB", | 20260506 20:54:19.189 | INFO | "malloc" : " 14 GB", | 20260506 20:54:19.189 | INFO | "pinned" : " 29 GB" | 20260506 20:54:19.189 | INFO | } | 20260506 20:54:19.189 | INFO | Perf report for the last 5.13406 seconds: | 20260506 20:54:19.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:54:19.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:19.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:19.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:19.189 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 20:54:19.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:19.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:19.189 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:54:19.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:19.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:19.189 | INFO | Uploaded 36GiB to GPU in 1.98769s (18.1114GiB/s) | 20260506 20:54:19.189 | INFO | Downloaded 34GiB to GPU in 1.92642s (17.6493GiB/s) | 20260506 20:54:24.329 | INFO | Analyzing chunk frames = [4380160,4380672) | 20260506 20:54:24.329 | INFO | MemInfo: { | 20260506 20:54:24.329 | INFO | "gpu" : " 17 GB", | 20260506 20:54:24.329 | INFO | "huge" : "", | 20260506 20:54:24.329 | INFO | "huge_shared" : " 27 GB", | 20260506 20:54:24.329 | INFO | "malloc" : " 14 GB", | 20260506 20:54:24.329 | INFO | "pinned" : " 29 GB" | 20260506 20:54:24.329 | INFO | } | 20260506 20:54:24.329 | INFO | Perf report for the last 5.14068 seconds: | 20260506 20:54:24.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:54:24.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:24.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:24.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:24.330 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 20:54:24.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:24.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:24.330 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:54:24.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:24.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:24.330 | INFO | Uploaded 36GiB to GPU in 1.96966s (18.2772GiB/s) | 20260506 20:54:24.330 | INFO | Downloaded 34GiB to GPU in 1.8754s (18.1294GiB/s) | 20260506 20:54:29.470 | INFO | Analyzing chunk frames = [4380672,4381184) | 20260506 20:54:29.470 | INFO | MemInfo: { | 20260506 20:54:29.470 | INFO | "gpu" : " 17 GB", | 20260506 20:54:29.470 | INFO | "huge" : "", | 20260506 20:54:29.470 | INFO | "huge_shared" : " 27 GB", | 20260506 20:54:29.470 | INFO | "malloc" : " 14 GB", | 20260506 20:54:29.470 | INFO | "pinned" : " 29 GB" | 20260506 20:54:29.470 | INFO | } | 20260506 20:54:29.470 | INFO | Perf report for the last 5.14064 seconds: | 20260506 20:54:29.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:54:29.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:29.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:29.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:29.470 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 20:54:29.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:29.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:29.470 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:54:29.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:29.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:29.471 | INFO | Uploaded 36GiB to GPU in 2.01264s (17.887GiB/s) | 20260506 20:54:29.471 | INFO | Downloaded 34GiB to GPU in 1.91646s (17.7411GiB/s) | 20260506 20:54:34.607 | INFO | Analyzing chunk frames = [4381184,4381696) | 20260506 20:54:34.607 | INFO | MemInfo: { | 20260506 20:54:34.607 | INFO | "gpu" : " 17 GB", | 20260506 20:54:34.607 | INFO | "huge" : "", | 20260506 20:54:34.607 | INFO | "huge_shared" : " 27 GB", | 20260506 20:54:34.607 | INFO | "malloc" : " 14 GB", | 20260506 20:54:34.608 | INFO | "pinned" : " 29 GB" | 20260506 20:54:34.608 | INFO | } | 20260506 20:54:34.608 | INFO | Perf report for the last 5.13768 seconds: | 20260506 20:54:34.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:54:34.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:34.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:34.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:34.608 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 20:54:34.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:34.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:34.608 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 20:54:34.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:34.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:34.608 | INFO | Uploaded 36GiB to GPU in 2.02703s (17.76GiB/s) | 20260506 20:54:34.608 | INFO | Downloaded 34GiB to GPU in 1.93104s (17.6071GiB/s) | 20260506 20:54:39.744 | INFO | Analyzing chunk frames = [4381696,4382208) | 20260506 20:54:39.744 | INFO | MemInfo: { | 20260506 20:54:39.744 | INFO | "gpu" : " 17 GB", | 20260506 20:54:39.744 | INFO | "huge" : "", | 20260506 20:54:39.744 | INFO | "huge_shared" : " 27 GB", | 20260506 20:54:39.744 | INFO | "malloc" : " 14 GB", | 20260506 20:54:39.744 | INFO | "pinned" : " 29 GB" | 20260506 20:54:39.745 | INFO | } | 20260506 20:54:39.745 | INFO | Perf report for the last 5.13683 seconds: | 20260506 20:54:39.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:54:39.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:39.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:39.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:39.745 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 20:54:39.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:39.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:39.745 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 20:54:39.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:39.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:39.745 | INFO | Uploaded 36GiB to GPU in 1.98435s (18.142GiB/s) | 20260506 20:54:39.745 | INFO | Downloaded 34GiB to GPU in 2.00495s (16.958GiB/s) | 20260506 20:54:44.880 | INFO | Analyzing chunk frames = [4382208,4382720) | 20260506 20:54:44.880 | INFO | MemInfo: { | 20260506 20:54:44.880 | INFO | "gpu" : " 17 GB", | 20260506 20:54:44.880 | INFO | "huge" : "", | 20260506 20:54:44.880 | INFO | "huge_shared" : " 27 GB", | 20260506 20:54:44.880 | INFO | "malloc" : " 14 GB", | 20260506 20:54:44.880 | INFO | "pinned" : " 29 GB" | 20260506 20:54:44.880 | INFO | } | 20260506 20:54:44.880 | INFO | Perf report for the last 5.13586 seconds: | 20260506 20:54:44.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:54:44.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:44.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:44.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:44.881 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:54:44.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:44.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:44.881 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 20:54:44.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:44.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:44.881 | INFO | Uploaded 36GiB to GPU in 1.98684s (18.1192GiB/s) | 20260506 20:54:44.881 | INFO | Downloaded 34GiB to GPU in 1.97486s (17.2164GiB/s) | 20260506 20:54:44.881 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:54:44.881 | INFO | Total mem usage: 18 GiB | 20260506 20:54:44.881 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:54:44.881 | INFO | Recent High Water Marks Sum: 16.8 GiB (Savings: -7.38%) | 20260506 20:54:44.881 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:54:44.881 | INFO | High Water Breakdown: | 20260506 20:54:44.881 | INFO | 62.5% : 10.5 GiB ( 91.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:54:44.881 | INFO | 37.3% : 6.26 GiB ( 93.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:54:44.881 | INFO | 0.119% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:54:44.881 | INFO | ------------------------------------------ | 20260506 20:54:44.881 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:54:44.881 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:54:44.881 | INFO | Total mem usage: 26.5 GiB | 20260506 20:54:44.881 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:54:44.881 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:54:44.881 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:54:44.881 | INFO | High Water Breakdown: | 20260506 20:54:44.881 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:54:44.881 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:54:44.881 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:54:44.881 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:54:44.881 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:54:44.881 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:54:44.881 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:54:44.881 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:54:44.881 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:54:44.881 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:54:44.881 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:54:44.881 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:54:44.881 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:54:44.881 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:54:44.881 | INFO | ------------------------------------------ | 20260506 20:54:44.881 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:54:44.881 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:54:44.881 | INFO | Total mem usage: 25.6 GiB | 20260506 20:54:44.881 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:54:44.881 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 20:54:44.881 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 20:54:44.882 | INFO | High Water Breakdown: | 20260506 20:54:44.882 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 20:54:44.882 | INFO | ------------------------------------------ | 20260506 20:54:44.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:54:44.882 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:54:44.882 | INFO | Total mem usage: 15.7 GiB | 20260506 20:54:44.882 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:54:44.882 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:54:44.882 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 20:54:44.882 | INFO | High Water Breakdown: | 20260506 20:54:44.882 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:54:44.882 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:54:44.882 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:54:44.882 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:54:44.882 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:54:44.882 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:54:44.882 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:54:44.882 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:54:44.882 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:54:44.882 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:54:44.882 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:54:44.882 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:54:44.882 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:54:44.882 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:54:44.882 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:54:44.882 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:54:44.882 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:54:44.882 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:54:44.882 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:54:44.882 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:54:44.882 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:54:44.882 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:54:44.882 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:54:44.882 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:54:44.882 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:54:44.882 | INFO | ------------------------------------------ | 20260506 20:54:44.882 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:54:44.882 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:54:44.882 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:54:44.882 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:54:44.882 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.60e+01, 1.36e+01 / 2.61e+00 | 20260506 20:54:44.882 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 9.78e-03 / 6.22e-03 | 20260506 20:54:44.882 | INFO | PerfReport: 11.64%, Baseline -- 3.62e+00 / 4.41e+00, 5.51e-01 / 3.23e-01 | 20260506 20:54:44.882 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.47e+00, 1.83e-01 / 1.34e-01 | 20260506 20:54:44.882 | INFO | PerfReport: 16.06%, DME -- 4.00e+01 / 4.50e+01, 4.34e+00 / 3.40e+00 | 20260506 20:54:44.882 | INFO | PerfReport: 57.16%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.34e+00 / 2.07e+00 | 20260506 20:54:44.882 | INFO | PerfReport: 7.15%, PulseAccumulating -- 2.22e+00 / 2.78e+00, 3.83e-01 / 2.39e-01 | 20260506 20:54:44.882 | INFO | PerfReport: 3.30%, Metrics -- 1.03e+00 / 1.35e+00, 2.72e-01 / 1.16e-01 | 20260506 20:54:44.882 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.52e-01 / 3.99e-01, 5.49e-02 / 1.95e-02 | 20260506 20:54:44.882 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:54:44.882 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:54:44.882 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:54:44.882 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 20:54:44.882 | INFO | PerfReport: 21.53%, Upload -- 2.08e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260506 20:54:44.882 | INFO | PerfReport: 21.20%, Download -- 2.05e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260506 20:54:44.882 | INFO | PerfReport: 32.18%, Compute -- 3.11e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 20:54:44.882 | INFO | PerfReport: 1.33%, LockUpload -- 1.29e+00 / 0.00e+00, 4.86e+00 / 0.00e+00 | 20260506 20:54:44.882 | INFO | PerfReport: 1.16%, LockDownload -- 1.12e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 20:54:44.882 | INFO | PerfReport: 22.59%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 20:54:44.882 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260506 20:54:44.981 | INFO | Starting a round of Baz IO | 20260506 20:54:48.185 | INFO | Populating RTMetrics report | 20260506 20:54:48.185 | INFO | RTMetrics report populated | 20260506 20:54:48.185 | INFO | Opening tmp RTMetrics json file | 20260506 20:54:48.187 | INFO | Serializing RTMetrics to json | 20260506 20:54:48.195 | INFO | Serialization done, swapping tmp file now... | 20260506 20:54:48.199 | INFO | RTMetrics done | 20260506 20:54:48.458 | INFO | Finished a round of Baz IO: Wrote 1.962 GiB in 3.476 seconds | 20260506 20:54:48.458 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.09% / 0.00% / 65.90% | 20260506 20:54:48.458 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.684444% / 0% / 7.94729e-05% | 20260506 20:54:48.458 | INFO | Baz IO overall: | 20260506 20:54:48.458 | INFO | BytesWritten : 2312026779201 | 20260506 20:54:48.458 | INFO | NumEvents : 1116691337262 | 20260506 20:54:50.022 | INFO | Analyzing chunk frames = [4382720,4383232) | 20260506 20:54:50.022 | INFO | MemInfo: { | 20260506 20:54:50.022 | INFO | "gpu" : " 17 GB", | 20260506 20:54:50.022 | INFO | "huge" : "", | 20260506 20:54:50.022 | INFO | "huge_shared" : " 27 GB", | 20260506 20:54:50.022 | INFO | "malloc" : " 14 GB", | 20260506 20:54:50.022 | INFO | "pinned" : " 29 GB" | 20260506 20:54:50.022 | INFO | } | 20260506 20:54:50.022 | INFO | Perf report for the last 5.14172 seconds: | 20260506 20:54:50.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:54:50.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:50.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:50.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:50.022 | INFO | ANALYSIS : 62.9%, 2.8, 94.0ms, 0 | 20260506 20:54:50.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:50.022 | INFO | RT_METRICS : 46.7%, 1.0, 24.5ms, 0 | 20260506 20:54:50.022 | INFO | PRE_HQ : 61.6%, 1.9, 62.0ms, 0 | 20260506 20:54:50.022 | INFO | BAZWRITER : 67.2%, 4.8, 165.3ms, 0 | 20260506 20:54:50.022 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 20:54:50.023 | INFO | Uploaded 36GiB to GPU in 2.07538s (17.3462GiB/s) | 20260506 20:54:50.023 | INFO | Downloaded 38GiB to GPU in 2.17149s (17.4995GiB/s) | 20260506 20:54:55.165 | INFO | Analyzing chunk frames = [4383232,4383744) | 20260506 20:54:55.165 | INFO | MemInfo: { | 20260506 20:54:55.165 | INFO | "gpu" : " 17 GB", | 20260506 20:54:55.165 | INFO | "huge" : "", | 20260506 20:54:55.165 | INFO | "huge_shared" : " 27 GB", | 20260506 20:54:55.165 | INFO | "malloc" : " 14 GB", | 20260506 20:54:55.165 | INFO | "pinned" : " 29 GB" | 20260506 20:54:55.165 | INFO | } | 20260506 20:54:55.165 | INFO | Perf report for the last 5.14312 seconds: | 20260506 20:54:55.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:54:55.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:55.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:55.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:55.165 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:54:55.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:55.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:55.166 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:54:55.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:55.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:54:55.166 | INFO | Uploaded 36GiB to GPU in 2.01319s (17.882GiB/s) | 20260506 20:54:55.166 | INFO | Downloaded 34GiB to GPU in 1.93067s (17.6104GiB/s) | 20260506 20:55:00.308 | INFO | Analyzing chunk frames = [4383744,4384256) | 20260506 20:55:00.308 | INFO | MemInfo: { | 20260506 20:55:00.308 | INFO | "gpu" : " 17 GB", | 20260506 20:55:00.308 | INFO | "huge" : "", | 20260506 20:55:00.308 | INFO | "huge_shared" : " 27 GB", | 20260506 20:55:00.308 | INFO | "malloc" : " 14 GB", | 20260506 20:55:00.308 | INFO | "pinned" : " 29 GB" | 20260506 20:55:00.308 | INFO | } | 20260506 20:55:00.308 | INFO | Perf report for the last 5.14294 seconds: | 20260506 20:55:00.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:55:00.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:00.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:00.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:00.308 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 20:55:00.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:00.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:00.308 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:55:00.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:00.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:00.309 | INFO | Uploaded 36GiB to GPU in 1.98955s (18.0946GiB/s) | 20260506 20:55:00.309 | INFO | Downloaded 34GiB to GPU in 1.91911s (17.7165GiB/s) | 20260506 20:55:05.447 | INFO | Analyzing chunk frames = [4384256,4384768) | 20260506 20:55:05.447 | INFO | MemInfo: { | 20260506 20:55:05.447 | INFO | "gpu" : " 17 GB", | 20260506 20:55:05.447 | INFO | "huge" : "", | 20260506 20:55:05.447 | INFO | "huge_shared" : " 27 GB", | 20260506 20:55:05.447 | INFO | "malloc" : " 14 GB", | 20260506 20:55:05.447 | INFO | "pinned" : " 29 GB" | 20260506 20:55:05.447 | INFO | } | 20260506 20:55:05.447 | INFO | Perf report for the last 5.13871 seconds: | 20260506 20:55:05.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:55:05.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:05.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:05.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:05.447 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 20:55:05.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:05.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:05.447 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:55:05.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:05.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:05.447 | INFO | Uploaded 36GiB to GPU in 1.97013s (18.2729GiB/s) | 20260506 20:55:05.447 | INFO | Downloaded 34GiB to GPU in 1.8739s (18.144GiB/s) | 20260506 20:55:10.583 | INFO | Analyzing chunk frames = [4384768,4385280) | 20260506 20:55:10.583 | INFO | MemInfo: { | 20260506 20:55:10.583 | INFO | "gpu" : " 17 GB", | 20260506 20:55:10.583 | INFO | "huge" : "", | 20260506 20:55:10.583 | INFO | "huge_shared" : " 27 GB", | 20260506 20:55:10.584 | INFO | "malloc" : " 14 GB", | 20260506 20:55:10.584 | INFO | "pinned" : " 29 GB" | 20260506 20:55:10.584 | INFO | } | 20260506 20:55:10.584 | INFO | Perf report for the last 5.13658 seconds: | 20260506 20:55:10.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:55:10.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:10.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:10.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:10.584 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 20:55:10.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:10.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:10.584 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:55:10.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:10.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:10.584 | INFO | Uploaded 36GiB to GPU in 1.99676s (18.0292GiB/s) | 20260506 20:55:10.584 | INFO | Downloaded 34GiB to GPU in 1.92415s (17.6701GiB/s) | 20260506 20:55:15.725 | INFO | Analyzing chunk frames = [4385280,4385792) | 20260506 20:55:15.725 | INFO | MemInfo: { | 20260506 20:55:15.725 | INFO | "gpu" : " 17 GB", | 20260506 20:55:15.725 | INFO | "huge" : "", | 20260506 20:55:15.725 | INFO | "huge_shared" : " 27 GB", | 20260506 20:55:15.725 | INFO | "malloc" : " 14 GB", | 20260506 20:55:15.725 | INFO | "pinned" : " 29 GB" | 20260506 20:55:15.725 | INFO | } | 20260506 20:55:15.725 | INFO | Perf report for the last 5.14106 seconds: | 20260506 20:55:15.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:55:15.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:15.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:15.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:15.725 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 20:55:15.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:15.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:15.725 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:55:15.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:15.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:15.725 | INFO | Uploaded 36GiB to GPU in 2.01221s (17.8908GiB/s) | 20260506 20:55:15.725 | INFO | Downloaded 34GiB to GPU in 1.94108s (17.516GiB/s) | 20260506 20:55:20.858 | INFO | Analyzing chunk frames = [4385792,4386304) | 20260506 20:55:20.858 | INFO | MemInfo: { | 20260506 20:55:20.858 | INFO | "gpu" : " 17 GB", | 20260506 20:55:20.858 | INFO | "huge" : "", | 20260506 20:55:20.858 | INFO | "huge_shared" : " 27 GB", | 20260506 20:55:20.858 | INFO | "malloc" : " 14 GB", | 20260506 20:55:20.858 | INFO | "pinned" : " 29 GB" | 20260506 20:55:20.858 | INFO | } | 20260506 20:55:20.858 | INFO | Perf report for the last 5.13312 seconds: | 20260506 20:55:20.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:55:20.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:20.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:20.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:20.858 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 20:55:20.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:20.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:20.858 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 20:55:20.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:20.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:20.858 | INFO | Uploaded 36GiB to GPU in 1.95161s (18.4463GiB/s) | 20260506 20:55:20.858 | INFO | Downloaded 34GiB to GPU in 1.98891s (17.0948GiB/s) | 20260506 20:55:25.995 | INFO | Analyzing chunk frames = [4386304,4386816) | 20260506 20:55:25.995 | INFO | MemInfo: { | 20260506 20:55:25.995 | INFO | "gpu" : " 17 GB", | 20260506 20:55:25.995 | INFO | "huge" : "", | 20260506 20:55:25.995 | INFO | "huge_shared" : " 27 GB", | 20260506 20:55:25.995 | INFO | "malloc" : " 14 GB", | 20260506 20:55:25.995 | INFO | "pinned" : " 29 GB" | 20260506 20:55:25.995 | INFO | } | 20260506 20:55:25.996 | INFO | Perf report for the last 5.13769 seconds: | 20260506 20:55:25.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:55:25.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:25.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:25.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:25.996 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 20:55:25.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:25.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:25.996 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 20:55:25.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:25.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:25.996 | INFO | Uploaded 36GiB to GPU in 1.99354s (18.0583GiB/s) | 20260506 20:55:25.996 | INFO | Downloaded 34GiB to GPU in 1.98256s (17.1495GiB/s) | 20260506 20:55:26.096 | INFO | Starting a round of Baz IO | 20260506 20:55:29.239 | INFO | Populating RTMetrics report | 20260506 20:55:29.240 | INFO | RTMetrics report populated | 20260506 20:55:29.240 | INFO | Opening tmp RTMetrics json file | 20260506 20:55:29.287 | INFO | Serializing RTMetrics to json | 20260506 20:55:29.435 | INFO | Serialization done, swapping tmp file now... | 20260506 20:55:29.436 | INFO | RTMetrics done | 20260506 20:55:29.459 | INFO | Finished a round of Baz IO: Wrote 1.965 GiB in 3.363 seconds | 20260506 20:55:29.459 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.09% / 0.00% / 65.91% | 20260506 20:55:29.459 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.684579% / 0% / 5.5631e-05% | 20260506 20:55:29.459 | INFO | Baz IO overall: | 20260506 20:55:29.460 | INFO | BytesWritten : 2314136468661 | 20260506 20:55:29.460 | INFO | NumEvents : 1117702793310 | 20260506 20:55:31.140 | INFO | Analyzing chunk frames = [4386816,4387328) | 20260506 20:55:31.140 | INFO | MemInfo: { | 20260506 20:55:31.140 | INFO | "gpu" : " 17 GB", | 20260506 20:55:31.140 | INFO | "huge" : "", | 20260506 20:55:31.140 | INFO | "huge_shared" : " 27 GB", | 20260506 20:55:31.140 | INFO | "malloc" : " 14 GB", | 20260506 20:55:31.140 | INFO | "pinned" : " 29 GB" | 20260506 20:55:31.140 | INFO | } | 20260506 20:55:31.140 | INFO | Perf report for the last 5.1444 seconds: | 20260506 20:55:31.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:55:31.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:31.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:31.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:31.140 | INFO | ANALYSIS : 62.5%, 2.9, 95.7ms, 0 | 20260506 20:55:31.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:31.140 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260506 20:55:31.140 | INFO | PRE_HQ : 60.1%, 1.8, 57.4ms, 0 | 20260506 20:55:31.140 | INFO | BAZWRITER : 65.0%, 4.6, 154.8ms, 0 | 20260506 20:55:31.140 | INFO | BAZ_SAMPLING : 3.6%, 1.0, 1.9ms, 0 | 20260506 20:55:31.140 | INFO | Uploaded 36GiB to GPU in 2.01454s (17.8701GiB/s) | 20260506 20:55:31.140 | INFO | Downloaded 38GiB to GPU in 2.1915s (17.3397GiB/s) | 20260506 20:55:36.283 | INFO | Analyzing chunk frames = [4387328,4387840) | 20260506 20:55:36.283 | INFO | MemInfo: { | 20260506 20:55:36.283 | INFO | "gpu" : " 17 GB", | 20260506 20:55:36.283 | INFO | "huge" : "", | 20260506 20:55:36.283 | INFO | "huge_shared" : " 27 GB", | 20260506 20:55:36.283 | INFO | "malloc" : " 14 GB", | 20260506 20:55:36.283 | INFO | "pinned" : " 29 GB" | 20260506 20:55:36.283 | INFO | } | 20260506 20:55:36.283 | INFO | Perf report for the last 5.14288 seconds: | 20260506 20:55:36.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:55:36.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:36.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:36.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:36.283 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 20:55:36.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:36.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:36.283 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 20:55:36.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:36.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:36.283 | INFO | Uploaded 36GiB to GPU in 2.01025s (17.9082GiB/s) | 20260506 20:55:36.283 | INFO | Downloaded 34GiB to GPU in 1.92738s (17.6405GiB/s) | 20260506 20:55:41.422 | INFO | Analyzing chunk frames = [4387840,4388352) | 20260506 20:55:41.422 | INFO | MemInfo: { | 20260506 20:55:41.422 | INFO | "gpu" : " 17 GB", | 20260506 20:55:41.422 | INFO | "huge" : "", | 20260506 20:55:41.422 | INFO | "huge_shared" : " 27 GB", | 20260506 20:55:41.422 | INFO | "malloc" : " 14 GB", | 20260506 20:55:41.422 | INFO | "pinned" : " 29 GB" | 20260506 20:55:41.422 | INFO | } | 20260506 20:55:41.422 | INFO | Perf report for the last 5.13949 seconds: | 20260506 20:55:41.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:55:41.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:41.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:41.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:41.423 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 20:55:41.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:41.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:41.423 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:55:41.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:41.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:41.423 | INFO | Uploaded 36GiB to GPU in 1.96611s (18.3103GiB/s) | 20260506 20:55:41.423 | INFO | Downloaded 34GiB to GPU in 1.92922s (17.6237GiB/s) | 20260506 20:55:46.562 | INFO | Analyzing chunk frames = [4388352,4388864) | 20260506 20:55:46.562 | INFO | MemInfo: { | 20260506 20:55:46.562 | INFO | "gpu" : " 17 GB", | 20260506 20:55:46.562 | INFO | "huge" : "", | 20260506 20:55:46.562 | INFO | "huge_shared" : " 27 GB", | 20260506 20:55:46.562 | INFO | "malloc" : " 14 GB", | 20260506 20:55:46.562 | INFO | "pinned" : " 29 GB" | 20260506 20:55:46.562 | INFO | } | 20260506 20:55:46.562 | INFO | Perf report for the last 5.14007 seconds: | 20260506 20:55:46.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:55:46.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:46.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:46.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:46.562 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 20:55:46.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:46.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:46.563 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:55:46.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:46.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:46.563 | INFO | Uploaded 36GiB to GPU in 2.01211s (17.8917GiB/s) | 20260506 20:55:46.563 | INFO | Downloaded 34GiB to GPU in 1.91011s (17.8GiB/s) | 20260506 20:55:51.697 | INFO | Analyzing chunk frames = [4388864,4389376) | 20260506 20:55:51.697 | INFO | MemInfo: { | 20260506 20:55:51.697 | INFO | "gpu" : " 17 GB", | 20260506 20:55:51.697 | INFO | "huge" : "", | 20260506 20:55:51.697 | INFO | "huge_shared" : " 27 GB", | 20260506 20:55:51.697 | INFO | "malloc" : " 14 GB", | 20260506 20:55:51.697 | INFO | "pinned" : " 29 GB" | 20260506 20:55:51.697 | INFO | } | 20260506 20:55:51.697 | INFO | Perf report for the last 5.13462 seconds: | 20260506 20:55:51.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:55:51.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:51.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:51.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:51.697 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 20:55:51.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:51.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:51.697 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:55:51.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:51.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:51.698 | INFO | Uploaded 36GiB to GPU in 2.00855s (17.9234GiB/s) | 20260506 20:55:51.698 | INFO | Downloaded 34GiB to GPU in 1.9225s (17.6853GiB/s) | 20260506 20:55:56.830 | INFO | Analyzing chunk frames = [4389376,4389888) | 20260506 20:55:56.830 | INFO | MemInfo: { | 20260506 20:55:56.830 | INFO | "gpu" : " 17 GB", | 20260506 20:55:56.830 | INFO | "huge" : "", | 20260506 20:55:56.830 | INFO | "huge_shared" : " 27 GB", | 20260506 20:55:56.830 | INFO | "malloc" : " 14 GB", | 20260506 20:55:56.830 | INFO | "pinned" : " 29 GB" | 20260506 20:55:56.830 | INFO | } | 20260506 20:55:56.830 | INFO | Perf report for the last 5.13265 seconds: | 20260506 20:55:56.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:55:56.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:56.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:56.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:56.830 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 20:55:56.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:56.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:56.830 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 20:55:56.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:56.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:55:56.830 | INFO | Uploaded 36GiB to GPU in 1.99412s (18.0531GiB/s) | 20260506 20:55:56.830 | INFO | Downloaded 34GiB to GPU in 1.94564s (17.475GiB/s) | 20260506 20:56:01.972 | INFO | Analyzing chunk frames = [4389888,4390400) | 20260506 20:56:01.972 | INFO | MemInfo: { | 20260506 20:56:01.972 | INFO | "gpu" : " 17 GB", | 20260506 20:56:01.972 | INFO | "huge" : "", | 20260506 20:56:01.972 | INFO | "huge_shared" : " 27 GB", | 20260506 20:56:01.972 | INFO | "malloc" : " 14 GB", | 20260506 20:56:01.972 | INFO | "pinned" : " 29 GB" | 20260506 20:56:01.972 | INFO | } | 20260506 20:56:01.972 | INFO | Perf report for the last 5.14234 seconds: | 20260506 20:56:01.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:56:01.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:01.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:01.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:01.972 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:56:01.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:01.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:01.972 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:56:01.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:01.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:01.972 | INFO | Uploaded 36GiB to GPU in 1.95217s (18.4411GiB/s) | 20260506 20:56:01.973 | INFO | Downloaded 34GiB to GPU in 1.99631s (17.0314GiB/s) | 20260506 20:56:07.113 | INFO | Analyzing chunk frames = [4390400,4390912) | 20260506 20:56:07.113 | INFO | MemInfo: { | 20260506 20:56:07.113 | INFO | "gpu" : " 17 GB", | 20260506 20:56:07.113 | INFO | "huge" : "", | 20260506 20:56:07.113 | INFO | "huge_shared" : " 27 GB", | 20260506 20:56:07.113 | INFO | "malloc" : " 14 GB", | 20260506 20:56:07.114 | INFO | "pinned" : " 29 GB" | 20260506 20:56:07.114 | INFO | } | 20260506 20:56:07.114 | INFO | Perf report for the last 5.14152 seconds: | 20260506 20:56:07.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:56:07.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:07.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:07.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:07.114 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:56:07.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:07.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:07.114 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:56:07.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:07.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:07.114 | INFO | Uploaded 36GiB to GPU in 1.99693s (18.0277GiB/s) | 20260506 20:56:07.114 | INFO | Downloaded 34GiB to GPU in 1.97146s (17.2461GiB/s) | 20260506 20:56:07.114 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:56:07.114 | INFO | Total mem usage: 18 GiB | 20260506 20:56:07.114 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:56:07.114 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.5%) | 20260506 20:56:07.114 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:56:07.114 | INFO | High Water Breakdown: | 20260506 20:56:07.114 | INFO | 61.5% : 9.58 GiB ( 83.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:56:07.114 | INFO | 38.4% : 5.99 GiB ( 89.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:56:07.114 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:56:07.114 | INFO | ------------------------------------------ | 20260506 20:56:07.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:56:07.114 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:56:07.114 | INFO | Total mem usage: 26.5 GiB | 20260506 20:56:07.114 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:56:07.114 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260506 20:56:07.114 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260506 20:56:07.114 | INFO | High Water Breakdown: | 20260506 20:56:07.114 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:56:07.114 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:56:07.114 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:56:07.114 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:56:07.114 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:56:07.114 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:56:07.114 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:56:07.114 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:56:07.115 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:56:07.115 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:56:07.115 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:56:07.115 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:56:07.115 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:56:07.115 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:56:07.115 | INFO | ------------------------------------------ | 20260506 20:56:07.115 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:56:07.115 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:56:07.115 | INFO | Total mem usage: 25.6 GiB | 20260506 20:56:07.115 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:56:07.115 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 20:56:07.115 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 20:56:07.115 | INFO | High Water Breakdown: | 20260506 20:56:07.115 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 20:56:07.115 | INFO | ------------------------------------------ | 20260506 20:56:07.115 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:56:07.115 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:56:07.115 | INFO | Total mem usage: 15.7 GiB | 20260506 20:56:07.115 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:56:07.115 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:56:07.115 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:56:07.115 | INFO | High Water Breakdown: | 20260506 20:56:07.115 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:56:07.115 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:56:07.115 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:56:07.115 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:56:07.115 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:56:07.115 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:56:07.115 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:56:07.115 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:56:07.115 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:56:07.115 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:56:07.115 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:56:07.115 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:56:07.115 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:56:07.115 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:56:07.115 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:56:07.115 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:56:07.115 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:56:07.115 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:56:07.115 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:56:07.115 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:56:07.115 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:56:07.115 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:56:07.115 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:56:07.115 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:56:07.115 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:56:07.115 | INFO | ------------------------------------------ | 20260506 20:56:07.115 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:56:07.115 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:56:07.115 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:56:07.115 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:56:07.115 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.61e+01, 1.35e+01 / 2.71e+00 | 20260506 20:56:07.115 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.02e-01, 1.00e-02 / 5.72e-03 | 20260506 20:56:07.115 | INFO | PerfReport: 11.66%, Baseline -- 3.63e+00 / 4.37e+00, 5.53e-01 / 3.45e-01 | 20260506 20:56:07.115 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.45e+00, 1.83e-01 / 1.18e-01 | 20260506 20:56:07.115 | INFO | PerfReport: 16.01%, DME -- 3.99e+01 / 4.46e+01, 4.17e+00 / 3.13e+00 | 20260506 20:56:07.115 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.39e+00 / 1.76e+00 | 20260506 20:56:07.115 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.22e+00 / 2.79e+00, 3.77e-01 / 2.19e-01 | 20260506 20:56:07.115 | INFO | PerfReport: 3.29%, Metrics -- 1.03e+00 / 1.37e+00, 2.70e-01 / 1.61e-01 | 20260506 20:56:07.115 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.51e-01 / 3.96e-01, 5.43e-02 / 1.85e-02 | 20260506 20:56:07.115 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:56:07.115 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:56:07.115 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:56:07.115 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 20:56:07.115 | INFO | PerfReport: 21.53%, Upload -- 2.08e+01 / 0.00e+00, 6.93e+00 / 0.00e+00 | 20260506 20:56:07.115 | INFO | PerfReport: 21.24%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 20:56:07.115 | INFO | PerfReport: 32.24%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 20:56:07.116 | INFO | PerfReport: 1.25%, LockUpload -- 1.21e+00 / 0.00e+00, 4.69e+00 / 0.00e+00 | 20260506 20:56:07.116 | INFO | PerfReport: 1.21%, LockDownload -- 1.17e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260506 20:56:07.116 | INFO | PerfReport: 22.51%, LockCompute -- 2.18e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 20:56:07.116 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.05e-02 / 0.00e+00 | 20260506 20:56:07.212 | INFO | Starting a round of Baz IO | 20260506 20:56:10.342 | INFO | Populating RTMetrics report | 20260506 20:56:10.343 | INFO | RTMetrics report populated | 20260506 20:56:10.343 | INFO | Opening tmp RTMetrics json file | 20260506 20:56:10.344 | INFO | Serializing RTMetrics to json | 20260506 20:56:10.362 | INFO | Serialization done, swapping tmp file now... | 20260506 20:56:10.372 | INFO | RTMetrics done | 20260506 20:56:11.200 | INFO | Finished a round of Baz IO: Wrote 1.963 GiB in 3.987 seconds | 20260506 20:56:11.200 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.09% / 0.00% / 65.91% | 20260506 20:56:11.200 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.682886% / 3.97364e-06% / 5.5631e-05% | 20260506 20:56:11.200 | INFO | Baz IO overall: | 20260506 20:56:11.200 | INFO | BytesWritten : 2316244734197 | 20260506 20:56:11.200 | INFO | NumEvents : 1118712993522 | 20260506 20:56:12.253 | INFO | Analyzing chunk frames = [4390912,4391424) | 20260506 20:56:12.253 | INFO | MemInfo: { | 20260506 20:56:12.253 | INFO | "gpu" : " 17 GB", | 20260506 20:56:12.253 | INFO | "huge" : "", | 20260506 20:56:12.253 | INFO | "huge_shared" : " 27 GB", | 20260506 20:56:12.253 | INFO | "malloc" : " 14 GB", | 20260506 20:56:12.253 | INFO | "pinned" : " 29 GB" | 20260506 20:56:12.253 | INFO | } | 20260506 20:56:12.253 | INFO | Perf report for the last 5.13972 seconds: | 20260506 20:56:12.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:56:12.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:12.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:12.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:12.253 | INFO | ANALYSIS : 62.2%, 2.9, 95.3ms, 0 | 20260506 20:56:12.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:12.253 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 20:56:12.254 | INFO | PRE_HQ : 59.9%, 1.8, 56.7ms, 0 | 20260506 20:56:12.254 | INFO | BAZWRITER : 77.2%, 4.8, 190.2ms, 0 | 20260506 20:56:12.254 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 20:56:12.254 | INFO | Uploaded 36GiB to GPU in 2.04611s (17.5944GiB/s) | 20260506 20:56:12.254 | INFO | Downloaded 38GiB to GPU in 2.1924s (17.3326GiB/s) | 20260506 20:56:17.403 | INFO | Analyzing chunk frames = [4391424,4391936) | 20260506 20:56:17.403 | INFO | MemInfo: { | 20260506 20:56:17.403 | INFO | "gpu" : " 17 GB", | 20260506 20:56:17.403 | INFO | "huge" : "", | 20260506 20:56:17.403 | INFO | "huge_shared" : " 27 GB", | 20260506 20:56:17.403 | INFO | "malloc" : " 14 GB", | 20260506 20:56:17.403 | INFO | "pinned" : " 29 GB" | 20260506 20:56:17.403 | INFO | } | 20260506 20:56:17.403 | INFO | Perf report for the last 5.14969 seconds: | 20260506 20:56:17.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:56:17.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:17.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:17.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:17.403 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 20:56:17.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:17.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:17.403 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:56:17.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:17.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:17.403 | INFO | Uploaded 36GiB to GPU in 2.01707s (17.8476GiB/s) | 20260506 20:56:17.403 | INFO | Downloaded 34GiB to GPU in 1.9265s (17.6485GiB/s) | 20260506 20:56:22.539 | INFO | Analyzing chunk frames = [4391936,4392448) | 20260506 20:56:22.539 | INFO | MemInfo: { | 20260506 20:56:22.539 | INFO | "gpu" : " 17 GB", | 20260506 20:56:22.539 | INFO | "huge" : "", | 20260506 20:56:22.539 | INFO | "huge_shared" : " 27 GB", | 20260506 20:56:22.539 | INFO | "malloc" : " 14 GB", | 20260506 20:56:22.539 | INFO | "pinned" : " 29 GB" | 20260506 20:56:22.539 | INFO | } | 20260506 20:56:22.539 | INFO | Perf report for the last 5.13577 seconds: | 20260506 20:56:22.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:56:22.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:22.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:22.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:22.539 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 20:56:22.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:22.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:22.539 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:56:22.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:22.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:22.539 | INFO | Uploaded 36GiB to GPU in 1.97744s (18.2054GiB/s) | 20260506 20:56:22.539 | INFO | Downloaded 34GiB to GPU in 1.91771s (17.7295GiB/s) | 20260506 20:56:27.676 | INFO | Analyzing chunk frames = [4392448,4392960) | 20260506 20:56:27.676 | INFO | MemInfo: { | 20260506 20:56:27.676 | INFO | "gpu" : " 17 GB", | 20260506 20:56:27.676 | INFO | "huge" : "", | 20260506 20:56:27.676 | INFO | "huge_shared" : " 27 GB", | 20260506 20:56:27.676 | INFO | "malloc" : " 14 GB", | 20260506 20:56:27.676 | INFO | "pinned" : " 29 GB" | 20260506 20:56:27.676 | INFO | } | 20260506 20:56:27.676 | INFO | Perf report for the last 5.13746 seconds: | 20260506 20:56:27.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:56:27.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:27.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:27.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:27.676 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260506 20:56:27.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:27.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:27.676 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 20:56:27.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:27.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:27.677 | INFO | Uploaded 36GiB to GPU in 2.01952s (17.826GiB/s) | 20260506 20:56:27.677 | INFO | Downloaded 34GiB to GPU in 1.90571s (17.8412GiB/s) | 20260506 20:56:32.814 | INFO | Analyzing chunk frames = [4392960,4393472) | 20260506 20:56:32.814 | INFO | MemInfo: { | 20260506 20:56:32.814 | INFO | "gpu" : " 17 GB", | 20260506 20:56:32.814 | INFO | "huge" : "", | 20260506 20:56:32.814 | INFO | "huge_shared" : " 27 GB", | 20260506 20:56:32.814 | INFO | "malloc" : " 14 GB", | 20260506 20:56:32.814 | INFO | "pinned" : " 29 GB" | 20260506 20:56:32.814 | INFO | } | 20260506 20:56:32.814 | INFO | Perf report for the last 5.13801 seconds: | 20260506 20:56:32.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:56:32.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:32.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:32.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:32.814 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 20:56:32.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:32.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:32.814 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:56:32.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:32.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:32.815 | INFO | Uploaded 36GiB to GPU in 2.0113s (17.8989GiB/s) | 20260506 20:56:32.815 | INFO | Downloaded 34GiB to GPU in 1.91938s (17.7141GiB/s) | 20260506 20:56:37.946 | INFO | Analyzing chunk frames = [4393472,4393984) | 20260506 20:56:37.946 | INFO | MemInfo: { | 20260506 20:56:37.946 | INFO | "gpu" : " 17 GB", | 20260506 20:56:37.946 | INFO | "huge" : "", | 20260506 20:56:37.946 | INFO | "huge_shared" : " 27 GB", | 20260506 20:56:37.946 | INFO | "malloc" : " 14 GB", | 20260506 20:56:37.946 | INFO | "pinned" : " 29 GB" | 20260506 20:56:37.946 | INFO | } | 20260506 20:56:37.946 | INFO | Perf report for the last 5.132 seconds: | 20260506 20:56:37.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:56:37.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:37.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:37.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:37.946 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 20:56:37.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:37.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:37.946 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 20:56:37.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:37.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:37.947 | INFO | Uploaded 36GiB to GPU in 2.0047s (17.9578GiB/s) | 20260506 20:56:37.947 | INFO | Downloaded 34GiB to GPU in 1.94489s (17.4817GiB/s) | 20260506 20:56:43.089 | INFO | Analyzing chunk frames = [4393984,4394496) | 20260506 20:56:43.089 | INFO | MemInfo: { | 20260506 20:56:43.089 | INFO | "gpu" : " 17 GB", | 20260506 20:56:43.089 | INFO | "huge" : "", | 20260506 20:56:43.089 | INFO | "huge_shared" : " 27 GB", | 20260506 20:56:43.089 | INFO | "malloc" : " 14 GB", | 20260506 20:56:43.089 | INFO | "pinned" : " 29 GB" | 20260506 20:56:43.089 | INFO | } | 20260506 20:56:43.089 | INFO | Perf report for the last 5.1425 seconds: | 20260506 20:56:43.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:56:43.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:43.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:43.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:43.089 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 20:56:43.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:43.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:43.089 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:56:43.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:43.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:43.089 | INFO | Uploaded 36GiB to GPU in 1.96529s (18.3179GiB/s) | 20260506 20:56:43.089 | INFO | Downloaded 34GiB to GPU in 1.99791s (17.0178GiB/s) | 20260506 20:56:48.230 | INFO | Analyzing chunk frames = [4394496,4395008) | 20260506 20:56:48.230 | INFO | MemInfo: { | 20260506 20:56:48.230 | INFO | "gpu" : " 17 GB", | 20260506 20:56:48.230 | INFO | "huge" : "", | 20260506 20:56:48.230 | INFO | "huge_shared" : " 27 GB", | 20260506 20:56:48.231 | INFO | "malloc" : " 14 GB", | 20260506 20:56:48.231 | INFO | "pinned" : " 29 GB" | 20260506 20:56:48.231 | INFO | } | 20260506 20:56:48.231 | INFO | Perf report for the last 5.1418 seconds: | 20260506 20:56:48.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:56:48.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:48.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:48.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:48.231 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:56:48.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:48.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:48.231 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 20:56:48.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:48.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:48.231 | INFO | Uploaded 36GiB to GPU in 1.98476s (18.1383GiB/s) | 20260506 20:56:48.231 | INFO | Downloaded 34GiB to GPU in 1.98086s (17.1643GiB/s) | 20260506 20:56:48.329 | INFO | Starting a round of Baz IO | 20260506 20:56:51.470 | INFO | Populating RTMetrics report | 20260506 20:56:51.470 | INFO | RTMetrics report populated | 20260506 20:56:51.470 | INFO | Opening tmp RTMetrics json file | 20260506 20:56:51.537 | INFO | Serializing RTMetrics to json | 20260506 20:56:51.603 | INFO | Serialization done, swapping tmp file now... | 20260506 20:56:51.615 | INFO | RTMetrics done | 20260506 20:56:51.691 | INFO | Finished a round of Baz IO: Wrote 1.963 GiB in 3.362 seconds | 20260506 20:56:51.691 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.08% / 0.00% / 65.91% | 20260506 20:56:51.691 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.680419% / 3.97364e-06% / 5.5631e-05% | 20260506 20:56:51.691 | INFO | Baz IO overall: | 20260506 20:56:51.691 | INFO | BytesWritten : 2318352247179 | 20260506 20:56:51.691 | INFO | NumEvents : 1119722956055 | 20260506 20:56:53.370 | INFO | Analyzing chunk frames = [4395008,4395520) | 20260506 20:56:53.370 | INFO | MemInfo: { | 20260506 20:56:53.370 | INFO | "gpu" : " 17 GB", | 20260506 20:56:53.370 | INFO | "huge" : "", | 20260506 20:56:53.370 | INFO | "huge_shared" : " 27 GB", | 20260506 20:56:53.370 | INFO | "malloc" : " 14 GB", | 20260506 20:56:53.370 | INFO | "pinned" : " 29 GB" | 20260506 20:56:53.370 | INFO | } | 20260506 20:56:53.370 | INFO | Perf report for the last 5.13917 seconds: | 20260506 20:56:53.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:56:53.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:53.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:53.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:53.370 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 20:56:53.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:53.370 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 20:56:53.370 | INFO | PRE_HQ : 60.2%, 1.8, 57.2ms, 0 | 20260506 20:56:53.370 | INFO | BAZWRITER : 65.1%, 4.7, 158.1ms, 0 | 20260506 20:56:53.370 | INFO | BAZ_SAMPLING : 2.4%, 1.0, 1.2ms, 0 | 20260506 20:56:53.370 | INFO | Uploaded 36GiB to GPU in 2.04691s (17.5875GiB/s) | 20260506 20:56:53.370 | INFO | Downloaded 38GiB to GPU in 2.20193s (17.2575GiB/s) | 20260506 20:56:58.506 | INFO | Analyzing chunk frames = [4395520,4396032) | 20260506 20:56:58.506 | INFO | MemInfo: { | 20260506 20:56:58.506 | INFO | "gpu" : " 17 GB", | 20260506 20:56:58.506 | INFO | "huge" : "", | 20260506 20:56:58.506 | INFO | "huge_shared" : " 27 GB", | 20260506 20:56:58.506 | INFO | "malloc" : " 14 GB", | 20260506 20:56:58.506 | INFO | "pinned" : " 29 GB" | 20260506 20:56:58.506 | INFO | } | 20260506 20:56:58.507 | INFO | Perf report for the last 5.13668 seconds: | 20260506 20:56:58.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:56:58.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:58.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:58.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:58.507 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 20:56:58.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:58.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:58.507 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 20:56:58.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:58.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:56:58.507 | INFO | Uploaded 36GiB to GPU in 2.01574s (17.8595GiB/s) | 20260506 20:56:58.507 | INFO | Downloaded 34GiB to GPU in 1.92164s (17.6932GiB/s) | 20260506 20:57:03.648 | INFO | Analyzing chunk frames = [4396032,4396544) | 20260506 20:57:03.648 | INFO | MemInfo: { | 20260506 20:57:03.648 | INFO | "gpu" : " 17 GB", | 20260506 20:57:03.648 | INFO | "huge" : "", | 20260506 20:57:03.648 | INFO | "huge_shared" : " 27 GB", | 20260506 20:57:03.649 | INFO | "malloc" : " 14 GB", | 20260506 20:57:03.649 | INFO | "pinned" : " 29 GB" | 20260506 20:57:03.649 | INFO | } | 20260506 20:57:03.649 | INFO | Perf report for the last 5.14203 seconds: | 20260506 20:57:03.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:57:03.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:03.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:03.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:03.649 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 20:57:03.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:03.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:03.649 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:57:03.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:03.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:03.649 | INFO | Uploaded 36GiB to GPU in 1.98196s (18.1639GiB/s) | 20260506 20:57:03.649 | INFO | Downloaded 34GiB to GPU in 1.91639s (17.7417GiB/s) | 20260506 20:57:08.787 | INFO | Analyzing chunk frames = [4396544,4397056) | 20260506 20:57:08.787 | INFO | MemInfo: { | 20260506 20:57:08.787 | INFO | "gpu" : " 17 GB", | 20260506 20:57:08.787 | INFO | "huge" : "", | 20260506 20:57:08.787 | INFO | "huge_shared" : " 27 GB", | 20260506 20:57:08.787 | INFO | "malloc" : " 14 GB", | 20260506 20:57:08.787 | INFO | "pinned" : " 29 GB" | 20260506 20:57:08.787 | INFO | } | 20260506 20:57:08.787 | INFO | Perf report for the last 5.1388 seconds: | 20260506 20:57:08.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:57:08.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:08.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:08.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:08.787 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 20:57:08.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:08.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:08.787 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 20:57:08.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:08.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:08.788 | INFO | Uploaded 36GiB to GPU in 1.96212s (18.3475GiB/s) | 20260506 20:57:08.788 | INFO | Downloaded 34GiB to GPU in 1.87356s (18.1472GiB/s) | 20260506 20:57:13.924 | INFO | Analyzing chunk frames = [4397056,4397568) | 20260506 20:57:13.924 | INFO | MemInfo: { | 20260506 20:57:13.924 | INFO | "gpu" : " 17 GB", | 20260506 20:57:13.924 | INFO | "huge" : "", | 20260506 20:57:13.924 | INFO | "huge_shared" : " 27 GB", | 20260506 20:57:13.924 | INFO | "malloc" : " 14 GB", | 20260506 20:57:13.924 | INFO | "pinned" : " 29 GB" | 20260506 20:57:13.924 | INFO | } | 20260506 20:57:13.924 | INFO | Perf report for the last 5.137 seconds: | 20260506 20:57:13.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:57:13.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:13.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:13.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:13.924 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 20:57:13.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:13.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:13.924 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:57:13.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:13.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:13.925 | INFO | Uploaded 36GiB to GPU in 1.99383s (18.0557GiB/s) | 20260506 20:57:13.925 | INFO | Downloaded 34GiB to GPU in 1.92108s (17.6984GiB/s) | 20260506 20:57:19.066 | INFO | Analyzing chunk frames = [4397568,4398080) | 20260506 20:57:19.066 | INFO | MemInfo: { | 20260506 20:57:19.066 | INFO | "gpu" : " 17 GB", | 20260506 20:57:19.066 | INFO | "huge" : "", | 20260506 20:57:19.066 | INFO | "huge_shared" : " 27 GB", | 20260506 20:57:19.066 | INFO | "malloc" : " 14 GB", | 20260506 20:57:19.066 | INFO | "pinned" : " 29 GB" | 20260506 20:57:19.066 | INFO | } | 20260506 20:57:19.067 | INFO | Perf report for the last 5.14212 seconds: | 20260506 20:57:19.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:57:19.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:19.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:19.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:19.067 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:57:19.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:19.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:19.067 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:57:19.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:19.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:19.067 | INFO | Uploaded 36GiB to GPU in 2.0245s (17.7822GiB/s) | 20260506 20:57:19.067 | INFO | Downloaded 34GiB to GPU in 1.95558s (17.3862GiB/s) | 20260506 20:57:24.203 | INFO | Analyzing chunk frames = [4398080,4398592) | 20260506 20:57:24.203 | INFO | MemInfo: { | 20260506 20:57:24.203 | INFO | "gpu" : " 17 GB", | 20260506 20:57:24.203 | INFO | "huge" : "", | 20260506 20:57:24.203 | INFO | "huge_shared" : " 27 GB", | 20260506 20:57:24.203 | INFO | "malloc" : " 14 GB", | 20260506 20:57:24.203 | INFO | "pinned" : " 29 GB" | 20260506 20:57:24.203 | INFO | } | 20260506 20:57:24.203 | INFO | Perf report for the last 5.13633 seconds: | 20260506 20:57:24.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:57:24.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:24.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:24.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:24.203 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 20:57:24.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:24.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:24.203 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:57:24.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:24.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:24.203 | INFO | Uploaded 36GiB to GPU in 1.98109s (18.1718GiB/s) | 20260506 20:57:24.203 | INFO | Downloaded 34GiB to GPU in 2.00763s (16.9354GiB/s) | 20260506 20:57:29.345 | INFO | Analyzing chunk frames = [4398592,4399104) | 20260506 20:57:29.345 | INFO | MemInfo: { | 20260506 20:57:29.345 | INFO | "gpu" : " 17 GB", | 20260506 20:57:29.345 | INFO | "huge" : "", | 20260506 20:57:29.345 | INFO | "huge_shared" : " 27 GB", | 20260506 20:57:29.345 | INFO | "malloc" : " 14 GB", | 20260506 20:57:29.345 | INFO | "pinned" : " 29 GB" | 20260506 20:57:29.345 | INFO | } | 20260506 20:57:29.346 | INFO | Perf report for the last 5.14265 seconds: | 20260506 20:57:29.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:57:29.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:29.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:29.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:29.346 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 20:57:29.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:29.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:29.346 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 20:57:29.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:29.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:29.346 | INFO | Uploaded 36GiB to GPU in 2.00697s (17.9374GiB/s) | 20260506 20:57:29.346 | INFO | Downloaded 34GiB to GPU in 1.99329s (17.0572GiB/s) | 20260506 20:57:29.346 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:57:29.346 | INFO | Total mem usage: 18 GiB | 20260506 20:57:29.346 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:57:29.346 | INFO | Recent High Water Marks Sum: 17.3 GiB (Savings: -3.92%) | 20260506 20:57:29.346 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:57:29.346 | INFO | High Water Breakdown: | 20260506 20:57:29.346 | INFO | 63% : 10.9 GiB ( 95.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:57:29.346 | INFO | 36.9% : 6.39 GiB ( 95.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:57:29.346 | INFO | 0.115% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:57:29.346 | INFO | ------------------------------------------ | 20260506 20:57:29.346 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:57:29.346 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:57:29.346 | INFO | Total mem usage: 26.5 GiB | 20260506 20:57:29.346 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:57:29.346 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:57:29.346 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:57:29.346 | INFO | High Water Breakdown: | 20260506 20:57:29.346 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:57:29.346 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:57:29.346 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:57:29.346 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:57:29.346 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:57:29.346 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:57:29.346 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:57:29.346 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:57:29.346 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:57:29.346 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:57:29.346 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:57:29.347 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:57:29.347 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:57:29.347 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:57:29.347 | INFO | ------------------------------------------ | 20260506 20:57:29.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:57:29.347 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:57:29.347 | INFO | Total mem usage: 25.6 GiB | 20260506 20:57:29.347 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:57:29.347 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:57:29.347 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:57:29.347 | INFO | High Water Breakdown: | 20260506 20:57:29.347 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:57:29.347 | INFO | ------------------------------------------ | 20260506 20:57:29.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:57:29.347 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:57:29.347 | INFO | Total mem usage: 15.7 GiB | 20260506 20:57:29.347 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:57:29.347 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:57:29.347 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 20:57:29.347 | INFO | High Water Breakdown: | 20260506 20:57:29.347 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:57:29.347 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:57:29.347 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:57:29.347 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:57:29.347 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:57:29.347 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:57:29.347 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:57:29.347 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:57:29.347 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:57:29.347 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:57:29.347 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:57:29.347 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:57:29.347 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:57:29.347 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:57:29.347 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:57:29.347 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:57:29.347 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:57:29.347 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:57:29.347 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:57:29.347 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:57:29.347 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:57:29.347 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:57:29.347 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:57:29.347 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:57:29.347 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:57:29.347 | INFO | ------------------------------------------ | 20260506 20:57:29.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:57:29.347 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:57:29.347 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:57:29.347 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:57:29.347 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.60e+01, 1.34e+01 / 2.78e+00 | 20260506 20:57:29.347 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 1.06e-02 / 5.42e-03 | 20260506 20:57:29.347 | INFO | PerfReport: 11.63%, Baseline -- 3.62e+00 / 4.38e+00, 5.53e-01 / 3.53e-01 | 20260506 20:57:29.349 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.43e+00, 1.82e-01 / 1.40e-01 | 20260506 20:57:29.349 | INFO | PerfReport: 16.04%, DME -- 4.00e+01 / 4.46e+01, 4.31e+00 / 3.13e+00 | 20260506 20:57:29.349 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.44e+00 / 1.72e+00 | 20260506 20:57:29.349 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.21e+00 / 2.78e+00, 3.76e-01 / 2.15e-01 | 20260506 20:57:29.349 | INFO | PerfReport: 3.29%, Metrics -- 1.02e+00 / 1.34e+00, 2.72e-01 / 1.38e-01 | 20260506 20:57:29.349 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.51e-01 / 3.96e-01, 5.48e-02 / 2.25e-02 | 20260506 20:57:29.349 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:57:29.349 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:57:29.349 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:57:29.349 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 20:57:29.349 | INFO | PerfReport: 21.55%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260506 20:57:29.349 | INFO | PerfReport: 21.24%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 20:57:29.349 | INFO | PerfReport: 32.19%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 20:57:29.349 | INFO | PerfReport: 1.35%, LockUpload -- 1.31e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260506 20:57:29.349 | INFO | PerfReport: 1.12%, LockDownload -- 1.08e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 20:57:29.349 | INFO | PerfReport: 22.54%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 20:57:29.349 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260506 20:57:29.453 | INFO | Starting a round of Baz IO | 20260506 20:57:32.583 | INFO | Populating RTMetrics report | 20260506 20:57:32.583 | INFO | RTMetrics report populated | 20260506 20:57:32.583 | INFO | Opening tmp RTMetrics json file | 20260506 20:57:32.590 | INFO | Serializing RTMetrics to json | 20260506 20:57:32.615 | INFO | Serialization done, swapping tmp file now... | 20260506 20:57:32.618 | INFO | RTMetrics done | 20260506 20:57:32.721 | INFO | Finished a round of Baz IO: Wrote 1.963 GiB in 3.268 seconds | 20260506 20:57:32.721 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.08% / 0.00% / 65.92% | 20260506 20:57:32.721 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.679723% / 1.98682e-05% / 4.37101e-05% | 20260506 20:57:32.721 | INFO | Baz IO overall: | 20260506 20:57:32.721 | INFO | BytesWritten : 2320459941334 | 20260506 20:57:32.721 | INFO | NumEvents : 1120733236925 | 20260506 20:57:34.492 | INFO | Analyzing chunk frames = [4399104,4399616) | 20260506 20:57:34.492 | INFO | MemInfo: { | 20260506 20:57:34.492 | INFO | "gpu" : " 17 GB", | 20260506 20:57:34.492 | INFO | "huge" : "", | 20260506 20:57:34.492 | INFO | "huge_shared" : " 27 GB", | 20260506 20:57:34.492 | INFO | "malloc" : " 14 GB", | 20260506 20:57:34.492 | INFO | "pinned" : " 29 GB" | 20260506 20:57:34.492 | INFO | } | 20260506 20:57:34.492 | INFO | Perf report for the last 5.14662 seconds: | 20260506 20:57:34.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:57:34.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:34.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:34.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:34.492 | INFO | ANALYSIS : 62.3%, 2.9, 95.3ms, 0 | 20260506 20:57:34.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:34.492 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 20:57:34.492 | INFO | PRE_HQ : 59.9%, 1.8, 57.3ms, 0 | 20260506 20:57:34.492 | INFO | BAZWRITER : 63.2%, 4.7, 154.3ms, 0 | 20260506 20:57:34.492 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260506 20:57:34.492 | INFO | Uploaded 36GiB to GPU in 2.07166s (17.3774GiB/s) | 20260506 20:57:34.492 | INFO | Downloaded 38GiB to GPU in 2.19899s (17.2807GiB/s) | 20260506 20:57:39.626 | INFO | Analyzing chunk frames = [4399616,4400128) | 20260506 20:57:39.626 | INFO | MemInfo: { | 20260506 20:57:39.626 | INFO | "gpu" : " 17 GB", | 20260506 20:57:39.626 | INFO | "huge" : "", | 20260506 20:57:39.626 | INFO | "huge_shared" : " 27 GB", | 20260506 20:57:39.626 | INFO | "malloc" : " 14 GB", | 20260506 20:57:39.626 | INFO | "pinned" : " 29 GB" | 20260506 20:57:39.626 | INFO | } | 20260506 20:57:39.626 | INFO | Perf report for the last 5.13389 seconds: | 20260506 20:57:39.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:57:39.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:39.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:39.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:39.626 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 20:57:39.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:39.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:39.626 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 20:57:39.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:39.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:39.626 | INFO | Uploaded 36GiB to GPU in 2.00151s (17.9864GiB/s) | 20260506 20:57:39.626 | INFO | Downloaded 34GiB to GPU in 1.91977s (17.7104GiB/s) | 20260506 20:57:44.764 | INFO | Analyzing chunk frames = [4400128,4400640) | 20260506 20:57:44.764 | INFO | MemInfo: { | 20260506 20:57:44.764 | INFO | "gpu" : " 17 GB", | 20260506 20:57:44.764 | INFO | "huge" : "", | 20260506 20:57:44.764 | INFO | "huge_shared" : " 27 GB", | 20260506 20:57:44.764 | INFO | "malloc" : " 14 GB", | 20260506 20:57:44.764 | INFO | "pinned" : " 29 GB" | 20260506 20:57:44.765 | INFO | } | 20260506 20:57:44.765 | INFO | Perf report for the last 5.13854 seconds: | 20260506 20:57:44.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:57:44.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:44.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:44.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:44.765 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 20:57:44.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:44.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:44.765 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:57:44.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:44.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:44.765 | INFO | Uploaded 36GiB to GPU in 1.98398s (18.1453GiB/s) | 20260506 20:57:44.765 | INFO | Downloaded 34GiB to GPU in 1.92933s (17.6227GiB/s) | 20260506 20:57:49.902 | INFO | Analyzing chunk frames = [4400640,4401152) | 20260506 20:57:49.902 | INFO | MemInfo: { | 20260506 20:57:49.902 | INFO | "gpu" : " 17 GB", | 20260506 20:57:49.902 | INFO | "huge" : "", | 20260506 20:57:49.902 | INFO | "huge_shared" : " 27 GB", | 20260506 20:57:49.902 | INFO | "malloc" : " 14 GB", | 20260506 20:57:49.902 | INFO | "pinned" : " 29 GB" | 20260506 20:57:49.902 | INFO | } | 20260506 20:57:49.902 | INFO | Perf report for the last 5.13735 seconds: | 20260506 20:57:49.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:57:49.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:49.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:49.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:49.902 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 20:57:49.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:49.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:49.902 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 20:57:49.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:49.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:49.902 | INFO | Uploaded 36GiB to GPU in 2.02054s (17.817GiB/s) | 20260506 20:57:49.902 | INFO | Downloaded 34GiB to GPU in 1.90758s (17.8237GiB/s) | 20260506 20:57:55.038 | INFO | Analyzing chunk frames = [4401152,4401664) | 20260506 20:57:55.038 | INFO | MemInfo: { | 20260506 20:57:55.038 | INFO | "gpu" : " 17 GB", | 20260506 20:57:55.038 | INFO | "huge" : "", | 20260506 20:57:55.038 | INFO | "huge_shared" : " 27 GB", | 20260506 20:57:55.038 | INFO | "malloc" : " 14 GB", | 20260506 20:57:55.038 | INFO | "pinned" : " 29 GB" | 20260506 20:57:55.038 | INFO | } | 20260506 20:57:55.038 | INFO | Perf report for the last 5.1357 seconds: | 20260506 20:57:55.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:57:55.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:55.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:55.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:55.038 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 20:57:55.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:55.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:55.038 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:57:55.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:55.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:57:55.038 | INFO | Uploaded 36GiB to GPU in 1.99196s (18.0727GiB/s) | 20260506 20:57:55.038 | INFO | Downloaded 34GiB to GPU in 1.91885s (17.7189GiB/s) | 20260506 20:58:00.177 | INFO | Analyzing chunk frames = [4401664,4402176) | 20260506 20:58:00.177 | INFO | MemInfo: { | 20260506 20:58:00.177 | INFO | "gpu" : " 17 GB", | 20260506 20:58:00.177 | INFO | "huge" : "", | 20260506 20:58:00.177 | INFO | "huge_shared" : " 27 GB", | 20260506 20:58:00.177 | INFO | "malloc" : " 14 GB", | 20260506 20:58:00.177 | INFO | "pinned" : " 29 GB" | 20260506 20:58:00.177 | INFO | } | 20260506 20:58:00.177 | INFO | Perf report for the last 5.13938 seconds: | 20260506 20:58:00.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:58:00.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:00.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:00.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:00.177 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 20:58:00.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:00.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:00.177 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:58:00.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:00.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:00.178 | INFO | Uploaded 36GiB to GPU in 2.00003s (17.9997GiB/s) | 20260506 20:58:00.178 | INFO | Downloaded 34GiB to GPU in 1.95417s (17.3987GiB/s) | 20260506 20:58:05.315 | INFO | Analyzing chunk frames = [4402176,4402688) | 20260506 20:58:05.315 | INFO | MemInfo: { | 20260506 20:58:05.315 | INFO | "gpu" : " 17 GB", | 20260506 20:58:05.315 | INFO | "huge" : "", | 20260506 20:58:05.315 | INFO | "huge_shared" : " 27 GB", | 20260506 20:58:05.315 | INFO | "malloc" : " 14 GB", | 20260506 20:58:05.315 | INFO | "pinned" : " 29 GB" | 20260506 20:58:05.315 | INFO | } | 20260506 20:58:05.315 | INFO | Perf report for the last 5.13784 seconds: | 20260506 20:58:05.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:58:05.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:05.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:05.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:05.315 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 20:58:05.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:05.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:05.315 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 20:58:05.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:05.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:05.315 | INFO | Uploaded 36GiB to GPU in 1.96883s (18.285GiB/s) | 20260506 20:58:05.315 | INFO | Downloaded 34GiB to GPU in 1.99836s (17.0139GiB/s) | 20260506 20:58:10.457 | INFO | Analyzing chunk frames = [4402688,4403200) | 20260506 20:58:10.457 | INFO | MemInfo: { | 20260506 20:58:10.457 | INFO | "gpu" : " 17 GB", | 20260506 20:58:10.457 | INFO | "huge" : "", | 20260506 20:58:10.457 | INFO | "huge_shared" : " 27 GB", | 20260506 20:58:10.457 | INFO | "malloc" : " 14 GB", | 20260506 20:58:10.457 | INFO | "pinned" : " 29 GB" | 20260506 20:58:10.457 | INFO | } | 20260506 20:58:10.457 | INFO | Perf report for the last 5.14224 seconds: | 20260506 20:58:10.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:58:10.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:10.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:10.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:10.457 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:58:10.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:10.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:10.457 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 20:58:10.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:10.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:10.457 | INFO | Uploaded 36GiB to GPU in 1.97474s (18.2303GiB/s) | 20260506 20:58:10.457 | INFO | Downloaded 34GiB to GPU in 1.98325s (17.1435GiB/s) | 20260506 20:58:10.556 | INFO | Starting a round of Baz IO | 20260506 20:58:13.694 | INFO | Populating RTMetrics report | 20260506 20:58:13.695 | INFO | RTMetrics report populated | 20260506 20:58:13.695 | INFO | Opening tmp RTMetrics json file | 20260506 20:58:13.700 | INFO | Serializing RTMetrics to json | 20260506 20:58:13.706 | INFO | Serialization done, swapping tmp file now... | 20260506 20:58:13.709 | INFO | RTMetrics done | 20260506 20:58:13.994 | INFO | Finished a round of Baz IO: Wrote 1.964 GiB in 3.437 seconds | 20260506 20:58:13.994 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.08% / 0.00% / 65.92% | 20260506 20:58:13.994 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.679394% / 3.97364e-06% / 5.96046e-05% | 20260506 20:58:13.994 | INFO | Baz IO overall: | 20260506 20:58:13.994 | INFO | BytesWritten : 2322568371577 | 20260506 20:58:13.994 | INFO | NumEvents : 1121744305321 | 20260506 20:58:15.599 | INFO | Analyzing chunk frames = [4403200,4403712) | 20260506 20:58:15.599 | INFO | MemInfo: { | 20260506 20:58:15.599 | INFO | "gpu" : " 17 GB", | 20260506 20:58:15.599 | INFO | "huge" : "", | 20260506 20:58:15.599 | INFO | "huge_shared" : " 27 GB", | 20260506 20:58:15.599 | INFO | "malloc" : " 14 GB", | 20260506 20:58:15.599 | INFO | "pinned" : " 29 GB" | 20260506 20:58:15.599 | INFO | } | 20260506 20:58:15.599 | INFO | Perf report for the last 5.14205 seconds: | 20260506 20:58:15.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:58:15.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:15.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:15.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:15.599 | INFO | ANALYSIS : 62.4%, 2.9, 95.3ms, 0 | 20260506 20:58:15.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:15.599 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 20:58:15.599 | INFO | PRE_HQ : 60.0%, 1.8, 57.4ms, 0 | 20260506 20:58:15.599 | INFO | BAZWRITER : 66.5%, 4.7, 161.1ms, 0 | 20260506 20:58:15.599 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260506 20:58:15.600 | INFO | Uploaded 36GiB to GPU in 2s (18GiB/s) | 20260506 20:58:15.600 | INFO | Downloaded 38GiB to GPU in 2.19692s (17.297GiB/s) | 20260506 20:58:20.742 | INFO | Analyzing chunk frames = [4403712,4404224) | 20260506 20:58:20.742 | INFO | MemInfo: { | 20260506 20:58:20.742 | INFO | "gpu" : " 17 GB", | 20260506 20:58:20.742 | INFO | "huge" : "", | 20260506 20:58:20.742 | INFO | "huge_shared" : " 27 GB", | 20260506 20:58:20.742 | INFO | "malloc" : " 14 GB", | 20260506 20:58:20.742 | INFO | "pinned" : " 29 GB" | 20260506 20:58:20.742 | INFO | } | 20260506 20:58:20.742 | INFO | Perf report for the last 5.14303 seconds: | 20260506 20:58:20.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:58:20.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:20.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:20.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:20.742 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 20:58:20.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:20.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:20.742 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 20:58:20.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:20.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:20.742 | INFO | Uploaded 36GiB to GPU in 2.00072s (17.9935GiB/s) | 20260506 20:58:20.743 | INFO | Downloaded 34GiB to GPU in 1.93014s (17.6153GiB/s) | 20260506 20:58:25.880 | INFO | Analyzing chunk frames = [4404224,4404736) | 20260506 20:58:25.880 | INFO | MemInfo: { | 20260506 20:58:25.880 | INFO | "gpu" : " 17 GB", | 20260506 20:58:25.880 | INFO | "huge" : "", | 20260506 20:58:25.880 | INFO | "huge_shared" : " 27 GB", | 20260506 20:58:25.880 | INFO | "malloc" : " 14 GB", | 20260506 20:58:25.880 | INFO | "pinned" : " 29 GB" | 20260506 20:58:25.880 | INFO | } | 20260506 20:58:25.880 | INFO | Perf report for the last 5.13811 seconds: | 20260506 20:58:25.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:58:25.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:25.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:25.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:25.880 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 20:58:25.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:25.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:25.880 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:58:25.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:25.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:25.881 | INFO | Uploaded 36GiB to GPU in 1.95042s (18.4576GiB/s) | 20260506 20:58:25.881 | INFO | Downloaded 34GiB to GPU in 1.92823s (17.6328GiB/s) | 20260506 20:58:31.017 | INFO | Analyzing chunk frames = [4404736,4405248) | 20260506 20:58:31.017 | INFO | MemInfo: { | 20260506 20:58:31.017 | INFO | "gpu" : " 17 GB", | 20260506 20:58:31.017 | INFO | "huge" : "", | 20260506 20:58:31.017 | INFO | "huge_shared" : " 27 GB", | 20260506 20:58:31.017 | INFO | "malloc" : " 14 GB", | 20260506 20:58:31.017 | INFO | "pinned" : " 29 GB" | 20260506 20:58:31.017 | INFO | } | 20260506 20:58:31.017 | INFO | Perf report for the last 5.13688 seconds: | 20260506 20:58:31.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:58:31.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:31.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:31.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:31.017 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260506 20:58:31.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:31.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:31.017 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 20:58:31.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:31.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:31.018 | INFO | Uploaded 36GiB to GPU in 2.00063s (17.9943GiB/s) | 20260506 20:58:31.018 | INFO | Downloaded 34GiB to GPU in 1.91146s (17.7875GiB/s) | 20260506 20:58:36.161 | INFO | Analyzing chunk frames = [4405248,4405760) | 20260506 20:58:36.161 | INFO | MemInfo: { | 20260506 20:58:36.161 | INFO | "gpu" : " 17 GB", | 20260506 20:58:36.161 | INFO | "huge" : "", | 20260506 20:58:36.161 | INFO | "huge_shared" : " 27 GB", | 20260506 20:58:36.161 | INFO | "malloc" : " 14 GB", | 20260506 20:58:36.161 | INFO | "pinned" : " 29 GB" | 20260506 20:58:36.161 | INFO | } | 20260506 20:58:36.161 | INFO | Perf report for the last 5.14385 seconds: | 20260506 20:58:36.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:58:36.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:36.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:36.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:36.161 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 20:58:36.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:36.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:36.161 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 20:58:36.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:36.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:36.162 | INFO | Uploaded 36GiB to GPU in 1.99845s (18.0139GiB/s) | 20260506 20:58:36.162 | INFO | Downloaded 34GiB to GPU in 1.91822s (17.7248GiB/s) | 20260506 20:58:41.293 | INFO | Analyzing chunk frames = [4405760,4406272) | 20260506 20:58:41.293 | INFO | MemInfo: { | 20260506 20:58:41.293 | INFO | "gpu" : " 17 GB", | 20260506 20:58:41.293 | INFO | "huge" : "", | 20260506 20:58:41.293 | INFO | "huge_shared" : " 27 GB", | 20260506 20:58:41.293 | INFO | "malloc" : " 14 GB", | 20260506 20:58:41.293 | INFO | "pinned" : " 29 GB" | 20260506 20:58:41.293 | INFO | } | 20260506 20:58:41.293 | INFO | Perf report for the last 5.13206 seconds: | 20260506 20:58:41.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:58:41.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:41.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:41.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:41.293 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 20:58:41.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:41.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:41.293 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 20:58:41.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:41.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:41.293 | INFO | Uploaded 36GiB to GPU in 1.99296s (18.0636GiB/s) | 20260506 20:58:41.294 | INFO | Downloaded 34GiB to GPU in 1.95401s (17.4002GiB/s) | 20260506 20:58:46.431 | INFO | Analyzing chunk frames = [4406272,4406784) | 20260506 20:58:46.431 | INFO | MemInfo: { | 20260506 20:58:46.431 | INFO | "gpu" : " 17 GB", | 20260506 20:58:46.431 | INFO | "huge" : "", | 20260506 20:58:46.431 | INFO | "huge_shared" : " 27 GB", | 20260506 20:58:46.431 | INFO | "malloc" : " 14 GB", | 20260506 20:58:46.431 | INFO | "pinned" : " 29 GB" | 20260506 20:58:46.431 | INFO | } | 20260506 20:58:46.431 | INFO | Perf report for the last 5.13801 seconds: | 20260506 20:58:46.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:58:46.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:46.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:46.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:46.431 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 20:58:46.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:46.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:46.431 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 20:58:46.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:46.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:46.432 | INFO | Uploaded 36GiB to GPU in 1.95168s (18.4456GiB/s) | 20260506 20:58:46.432 | INFO | Downloaded 34GiB to GPU in 1.99857s (17.0121GiB/s) | 20260506 20:58:51.579 | INFO | Analyzing chunk frames = [4406784,4407296) | 20260506 20:58:51.579 | INFO | MemInfo: { | 20260506 20:58:51.579 | INFO | "gpu" : " 17 GB", | 20260506 20:58:51.579 | INFO | "huge" : "", | 20260506 20:58:51.579 | INFO | "huge_shared" : " 27 GB", | 20260506 20:58:51.579 | INFO | "malloc" : " 14 GB", | 20260506 20:58:51.580 | INFO | "pinned" : " 29 GB" | 20260506 20:58:51.580 | INFO | } | 20260506 20:58:51.580 | INFO | Perf report for the last 5.14835 seconds: | 20260506 20:58:51.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:58:51.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:51.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:51.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:51.580 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260506 20:58:51.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:51.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:51.580 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 20:58:51.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:51.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:51.580 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 20:58:51.580 | INFO | Total mem usage: 18 GiB | 20260506 20:58:51.580 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 20:58:51.580 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -19.1%) | 20260506 20:58:51.580 | INFO | Uploaded 36GiB to GPU in 1.99316s (18.0618GiB/s) | 20260506 20:58:51.580 | INFO | Downloaded 34GiB to GPU in 1.97198s (17.2415GiB/s) | 20260506 20:58:51.580 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 20:58:51.580 | INFO | High Water Breakdown: | 20260506 20:58:51.580 | INFO | 61% : 9.22 GiB ( 80.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 20:58:51.580 | INFO | 38.9% : 5.88 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 20:58:51.580 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 20:58:51.580 | INFO | ------------------------------------------ | 20260506 20:58:51.580 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:58:51.580 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 20:58:51.580 | INFO | Total mem usage: 26.5 GiB | 20260506 20:58:51.580 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 20:58:51.580 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 20:58:51.580 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 20:58:51.580 | INFO | High Water Breakdown: | 20260506 20:58:51.580 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:58:51.580 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:58:51.580 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:58:51.580 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:58:51.580 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:58:51.580 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:58:51.580 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 20:58:51.580 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 20:58:51.580 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:58:51.580 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 20:58:51.580 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 20:58:51.580 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 20:58:51.580 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 20:58:51.580 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 20:58:51.581 | INFO | ------------------------------------------ | 20260506 20:58:51.581 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:58:51.581 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 20:58:51.581 | INFO | Total mem usage: 25.6 GiB | 20260506 20:58:51.581 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 20:58:51.581 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 20:58:51.581 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 20:58:51.581 | INFO | High Water Breakdown: | 20260506 20:58:51.581 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 20:58:51.581 | INFO | ------------------------------------------ | 20260506 20:58:51.581 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:58:51.581 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 20:58:51.581 | INFO | Total mem usage: 15.7 GiB | 20260506 20:58:51.581 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 20:58:51.581 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 20:58:51.581 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 20:58:51.581 | INFO | High Water Breakdown: | 20260506 20:58:51.581 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 20:58:51.581 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 20:58:51.581 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 20:58:51.581 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 20:58:51.581 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 20:58:51.581 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 20:58:51.581 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 20:58:51.581 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 20:58:51.581 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 20:58:51.581 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 20:58:51.581 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 20:58:51.581 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 20:58:51.581 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 20:58:51.581 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 20:58:51.581 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 20:58:51.581 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 20:58:51.581 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 20:58:51.581 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 20:58:51.581 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 20:58:51.581 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 20:58:51.581 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 20:58:51.581 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 20:58:51.581 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 20:58:51.581 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 20:58:51.581 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 20:58:51.581 | INFO | ------------------------------------------ | 20260506 20:58:51.581 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 20:58:51.581 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 20:58:51.581 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:58:51.581 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:58:51.581 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.63e+01, 1.36e+01 / 2.27e+00 | 20260506 20:58:51.581 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 9.75e-03 / 5.58e-03 | 20260506 20:58:51.581 | INFO | PerfReport: 11.65%, Baseline -- 3.63e+00 / 4.32e+00, 5.54e-01 / 3.94e-01 | 20260506 20:58:51.581 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.43e+00, 1.84e-01 / 1.37e-01 | 20260506 20:58:51.581 | INFO | PerfReport: 16.04%, DME -- 4.00e+01 / 4.48e+01, 4.48e+00 / 2.35e+00 | 20260506 20:58:51.581 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.78e+01 / 2.11e+01, 1.40e+00 / 1.50e+00 | 20260506 20:58:51.581 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.22e+00 / 2.79e+00, 3.77e-01 / 1.78e-01 | 20260506 20:58:51.581 | INFO | PerfReport: 3.29%, Metrics -- 1.02e+00 / 1.35e+00, 2.70e-01 / 1.50e-01 | 20260506 20:58:51.581 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.51e-01 / 4.03e-01, 5.57e-02 / 2.60e-02 | 20260506 20:58:51.581 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 20:58:51.581 | INFO | PerfReport: Gathered 1536 samples. | 20260506 20:58:51.581 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 20:58:51.581 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 20:58:51.581 | INFO | PerfReport: 21.46%, Upload -- 2.08e+01 / 0.00e+00, 6.90e+00 / 0.00e+00 | 20260506 20:58:51.581 | INFO | PerfReport: 21.27%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 20:58:51.581 | INFO | PerfReport: 32.21%, Compute -- 3.12e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 20:58:51.581 | INFO | PerfReport: 1.24%, LockUpload -- 1.20e+00 / 0.00e+00, 4.75e+00 / 0.00e+00 | 20260506 20:58:51.581 | INFO | PerfReport: 1.27%, LockDownload -- 1.23e+00 / 0.00e+00, 3.10e+00 / 0.00e+00 | 20260506 20:58:51.581 | INFO | PerfReport: 22.54%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 20:58:51.581 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.20e-02 / 0.00e+00 | 20260506 20:58:51.680 | INFO | Starting a round of Baz IO | 20260506 20:58:54.820 | INFO | Populating RTMetrics report | 20260506 20:58:54.820 | INFO | RTMetrics report populated | 20260506 20:58:54.820 | INFO | Opening tmp RTMetrics json file | 20260506 20:58:54.835 | INFO | Serializing RTMetrics to json | 20260506 20:58:54.838 | INFO | Serialization done, swapping tmp file now... | 20260506 20:58:54.843 | INFO | RTMetrics done | 20260506 20:58:55.122 | INFO | Finished a round of Baz IO: Wrote 1.962 GiB in 3.441 seconds | 20260506 20:58:55.122 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.07% / 0.00% / 65.92% | 20260506 20:58:55.122 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.677129% / 1.19209e-05% / 8.34465e-05% | 20260506 20:58:55.122 | INFO | Baz IO overall: | 20260506 20:58:55.122 | INFO | BytesWritten : 2324675326129 | 20260506 20:58:55.122 | INFO | NumEvents : 1122754456089 | 20260506 20:58:56.713 | INFO | Analyzing chunk frames = [4407296,4407808) | 20260506 20:58:56.713 | INFO | MemInfo: { | 20260506 20:58:56.713 | INFO | "gpu" : " 17 GB", | 20260506 20:58:56.713 | INFO | "huge" : "", | 20260506 20:58:56.713 | INFO | "huge_shared" : " 27 GB", | 20260506 20:58:56.713 | INFO | "malloc" : " 14 GB", | 20260506 20:58:56.713 | INFO | "pinned" : " 29 GB" | 20260506 20:58:56.713 | INFO | } | 20260506 20:58:56.714 | INFO | Perf report for the last 5.1339 seconds: | 20260506 20:58:56.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:58:56.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:56.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:56.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:56.714 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 20:58:56.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:58:56.714 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 20:58:56.714 | INFO | PRE_HQ : 60.1%, 1.8, 57.0ms, 0 | 20260506 20:58:56.714 | INFO | BAZWRITER : 66.7%, 4.8, 164.6ms, 0 | 20260506 20:58:56.714 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 20:58:56.714 | INFO | Uploaded 36GiB to GPU in 2.06045s (17.4719GiB/s) | 20260506 20:58:56.714 | INFO | Downloaded 38GiB to GPU in 2.20273s (17.2513GiB/s) | 20260506 20:59:01.855 | INFO | Analyzing chunk frames = [4407808,4408320) | 20260506 20:59:01.855 | INFO | MemInfo: { | 20260506 20:59:01.855 | INFO | "gpu" : " 17 GB", | 20260506 20:59:01.855 | INFO | "huge" : "", | 20260506 20:59:01.855 | INFO | "huge_shared" : " 27 GB", | 20260506 20:59:01.855 | INFO | "malloc" : " 14 GB", | 20260506 20:59:01.855 | INFO | "pinned" : " 29 GB" | 20260506 20:59:01.855 | INFO | } | 20260506 20:59:01.855 | INFO | Perf report for the last 5.1416 seconds: | 20260506 20:59:01.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:59:01.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:01.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:01.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:01.855 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 20:59:01.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:01.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:01.855 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 20:59:01.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:01.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:01.855 | INFO | Uploaded 36GiB to GPU in 2.0222s (17.8024GiB/s) | 20260506 20:59:01.855 | INFO | Downloaded 34GiB to GPU in 1.92898s (17.6259GiB/s) | 20260506 20:59:06.994 | INFO | Analyzing chunk frames = [4408320,4408832) | 20260506 20:59:06.994 | INFO | MemInfo: { | 20260506 20:59:06.994 | INFO | "gpu" : " 17 GB", | 20260506 20:59:06.994 | INFO | "huge" : "", | 20260506 20:59:06.994 | INFO | "huge_shared" : " 27 GB", | 20260506 20:59:06.994 | INFO | "malloc" : " 14 GB", | 20260506 20:59:06.994 | INFO | "pinned" : " 29 GB" | 20260506 20:59:06.994 | INFO | } | 20260506 20:59:06.994 | INFO | Perf report for the last 5.13919 seconds: | 20260506 20:59:06.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:59:06.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:06.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:06.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:06.994 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 20:59:06.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:06.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:06.994 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:59:06.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:06.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:06.995 | INFO | Uploaded 36GiB to GPU in 1.98364s (18.1485GiB/s) | 20260506 20:59:06.995 | INFO | Downloaded 34GiB to GPU in 1.92704s (17.6437GiB/s) | 20260506 20:59:12.131 | INFO | Analyzing chunk frames = [4408832,4409344) | 20260506 20:59:12.131 | INFO | MemInfo: { | 20260506 20:59:12.131 | INFO | "gpu" : " 17 GB", | 20260506 20:59:12.131 | INFO | "huge" : "", | 20260506 20:59:12.132 | INFO | "huge_shared" : " 27 GB", | 20260506 20:59:12.132 | INFO | "malloc" : " 14 GB", | 20260506 20:59:12.132 | INFO | "pinned" : " 29 GB" | 20260506 20:59:12.132 | INFO | } | 20260506 20:59:12.132 | INFO | Perf report for the last 5.13723 seconds: | 20260506 20:59:12.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:59:12.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:12.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:12.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:12.132 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 20:59:12.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:12.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:12.132 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:59:12.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:12.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:12.132 | INFO | Uploaded 36GiB to GPU in 1.97155s (18.2597GiB/s) | 20260506 20:59:12.132 | INFO | Downloaded 34GiB to GPU in 1.8736s (18.1468GiB/s) | 20260506 20:59:17.269 | INFO | Analyzing chunk frames = [4409344,4409856) | 20260506 20:59:17.269 | INFO | MemInfo: { | 20260506 20:59:17.269 | INFO | "gpu" : " 17 GB", | 20260506 20:59:17.269 | INFO | "huge" : "", | 20260506 20:59:17.269 | INFO | "huge_shared" : " 27 GB", | 20260506 20:59:17.269 | INFO | "malloc" : " 14 GB", | 20260506 20:59:17.269 | INFO | "pinned" : " 29 GB" | 20260506 20:59:17.269 | INFO | } | 20260506 20:59:17.269 | INFO | Perf report for the last 5.13734 seconds: | 20260506 20:59:17.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:59:17.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:17.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:17.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:17.269 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 20:59:17.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:17.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:17.269 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 20:59:17.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:17.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:17.270 | INFO | Uploaded 36GiB to GPU in 2.02093s (17.8136GiB/s) | 20260506 20:59:17.270 | INFO | Downloaded 34GiB to GPU in 1.91383s (17.7654GiB/s) | 20260506 20:59:22.406 | INFO | Analyzing chunk frames = [4409856,4410368) | 20260506 20:59:22.406 | INFO | MemInfo: { | 20260506 20:59:22.406 | INFO | "gpu" : " 17 GB", | 20260506 20:59:22.406 | INFO | "huge" : "", | 20260506 20:59:22.406 | INFO | "huge_shared" : " 27 GB", | 20260506 20:59:22.406 | INFO | "malloc" : " 14 GB", | 20260506 20:59:22.406 | INFO | "pinned" : " 29 GB" | 20260506 20:59:22.406 | INFO | } | 20260506 20:59:22.406 | INFO | Perf report for the last 5.13751 seconds: | 20260506 20:59:22.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:59:22.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:22.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:22.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:22.407 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 20:59:22.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:22.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:22.407 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 20:59:22.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:22.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:22.407 | INFO | Uploaded 36GiB to GPU in 2.01657s (17.8521GiB/s) | 20260506 20:59:22.407 | INFO | Downloaded 34GiB to GPU in 1.93554s (17.5661GiB/s) | 20260506 20:59:27.547 | INFO | Analyzing chunk frames = [4410368,4410880) | 20260506 20:59:27.547 | INFO | MemInfo: { | 20260506 20:59:27.547 | INFO | "gpu" : " 17 GB", | 20260506 20:59:27.547 | INFO | "huge" : "", | 20260506 20:59:27.548 | INFO | "huge_shared" : " 27 GB", | 20260506 20:59:27.548 | INFO | "malloc" : " 14 GB", | 20260506 20:59:27.548 | INFO | "pinned" : " 29 GB" | 20260506 20:59:27.548 | INFO | } | 20260506 20:59:27.548 | INFO | Perf report for the last 5.14122 seconds: | 20260506 20:59:27.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:59:27.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:27.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:27.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:27.548 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 20:59:27.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:27.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:27.548 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 20:59:27.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:27.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:27.548 | INFO | Uploaded 36GiB to GPU in 1.96058s (18.3619GiB/s) | 20260506 20:59:27.548 | INFO | Downloaded 34GiB to GPU in 1.99765s (17.02GiB/s) | 20260506 20:59:32.692 | INFO | Analyzing chunk frames = [4410880,4411392) | 20260506 20:59:32.692 | INFO | MemInfo: { | 20260506 20:59:32.692 | INFO | "gpu" : " 17 GB", | 20260506 20:59:32.692 | INFO | "huge" : "", | 20260506 20:59:32.692 | INFO | "huge_shared" : " 27 GB", | 20260506 20:59:32.692 | INFO | "malloc" : " 14 GB", | 20260506 20:59:32.692 | INFO | "pinned" : " 29 GB" | 20260506 20:59:32.692 | INFO | } | 20260506 20:59:32.692 | INFO | Perf report for the last 5.14475 seconds: | 20260506 20:59:32.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:59:32.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:32.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:32.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:32.693 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 20:59:32.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:32.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:32.693 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 20:59:32.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:32.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:32.693 | INFO | Uploaded 36GiB to GPU in 1.98831s (18.1059GiB/s) | 20260506 20:59:32.693 | INFO | Downloaded 34GiB to GPU in 1.99208s (17.0676GiB/s) | 20260506 20:59:32.791 | INFO | Starting a round of Baz IO | 20260506 20:59:35.929 | INFO | Populating RTMetrics report | 20260506 20:59:35.929 | INFO | RTMetrics report populated | 20260506 20:59:35.929 | INFO | Opening tmp RTMetrics json file | 20260506 20:59:36.136 | INFO | Serializing RTMetrics to json | 20260506 20:59:36.260 | INFO | Serialization done, swapping tmp file now... | 20260506 20:59:36.266 | INFO | RTMetrics done | 20260506 20:59:36.315 | INFO | Finished a round of Baz IO: Wrote 1.96 GiB in 3.523 seconds | 20260506 20:59:36.315 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.07% / 0.00% / 65.93% | 20260506 20:59:36.315 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.674768% / 0% / 5.96046e-05% | 20260506 20:59:36.315 | INFO | Baz IO overall: | 20260506 20:59:36.315 | INFO | BytesWritten : 2326780044718 | 20260506 20:59:36.315 | INFO | NumEvents : 1123763046975 | 20260506 20:59:37.840 | INFO | Analyzing chunk frames = [4411392,4411904) | 20260506 20:59:37.840 | INFO | MemInfo: { | 20260506 20:59:37.840 | INFO | "gpu" : " 17 GB", | 20260506 20:59:37.840 | INFO | "huge" : "", | 20260506 20:59:37.840 | INFO | "huge_shared" : " 27 GB", | 20260506 20:59:37.840 | INFO | "malloc" : " 14 GB", | 20260506 20:59:37.840 | INFO | "pinned" : " 29 GB" | 20260506 20:59:37.840 | INFO | } | 20260506 20:59:37.840 | INFO | Perf report for the last 5.14771 seconds: | 20260506 20:59:37.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:59:37.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:37.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:37.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:37.840 | INFO | ANALYSIS : 62.3%, 2.9, 95.5ms, 0 | 20260506 20:59:37.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:37.840 | INFO | RT_METRICS : 49.2%, 1.0, 25.9ms, 0 | 20260506 20:59:37.840 | INFO | PRE_HQ : 59.8%, 1.8, 57.0ms, 0 | 20260506 20:59:37.840 | INFO | BAZWRITER : 68.1%, 4.7, 165.6ms, 0 | 20260506 20:59:37.840 | INFO | BAZ_SAMPLING : 6.3%, 1.0, 3.3ms, 0 | 20260506 20:59:37.840 | INFO | Uploaded 36GiB to GPU in 2.03558s (17.6854GiB/s) | 20260506 20:59:37.841 | INFO | Downloaded 38GiB to GPU in 2.19997s (17.273GiB/s) | 20260506 20:59:42.972 | INFO | Analyzing chunk frames = [4411904,4412416) | 20260506 20:59:42.972 | INFO | MemInfo: { | 20260506 20:59:42.972 | INFO | "gpu" : " 17 GB", | 20260506 20:59:42.972 | INFO | "huge" : "", | 20260506 20:59:42.972 | INFO | "huge_shared" : " 27 GB", | 20260506 20:59:42.972 | INFO | "malloc" : " 14 GB", | 20260506 20:59:42.972 | INFO | "pinned" : " 29 GB" | 20260506 20:59:42.972 | INFO | } | 20260506 20:59:42.972 | INFO | Perf report for the last 5.13151 seconds: | 20260506 20:59:42.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:59:42.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:42.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:42.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:42.972 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 20:59:42.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:42.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:42.972 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 20:59:42.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:42.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:42.972 | INFO | Uploaded 36GiB to GPU in 2.00237s (17.9787GiB/s) | 20260506 20:59:42.972 | INFO | Downloaded 34GiB to GPU in 1.92631s (17.6504GiB/s) | 20260506 20:59:48.106 | INFO | Analyzing chunk frames = [4412416,4412928) | 20260506 20:59:48.106 | INFO | MemInfo: { | 20260506 20:59:48.106 | INFO | "gpu" : " 17 GB", | 20260506 20:59:48.106 | INFO | "huge" : "", | 20260506 20:59:48.106 | INFO | "huge_shared" : " 27 GB", | 20260506 20:59:48.106 | INFO | "malloc" : " 14 GB", | 20260506 20:59:48.106 | INFO | "pinned" : " 29 GB" | 20260506 20:59:48.106 | INFO | } | 20260506 20:59:48.107 | INFO | Perf report for the last 5.13491 seconds: | 20260506 20:59:48.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:59:48.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:48.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:48.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:48.107 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 20:59:48.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:48.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:48.107 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:59:48.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:48.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:48.107 | INFO | Uploaded 36GiB to GPU in 1.96803s (18.2924GiB/s) | 20260506 20:59:48.107 | INFO | Downloaded 34GiB to GPU in 1.93159s (17.6021GiB/s) | 20260506 20:59:53.245 | INFO | Analyzing chunk frames = [4412928,4413440) | 20260506 20:59:53.245 | INFO | MemInfo: { | 20260506 20:59:53.245 | INFO | "gpu" : " 17 GB", | 20260506 20:59:53.245 | INFO | "huge" : "", | 20260506 20:59:53.245 | INFO | "huge_shared" : " 27 GB", | 20260506 20:59:53.245 | INFO | "malloc" : " 14 GB", | 20260506 20:59:53.245 | INFO | "pinned" : " 29 GB" | 20260506 20:59:53.245 | INFO | } | 20260506 20:59:53.245 | INFO | Perf report for the last 5.13833 seconds: | 20260506 20:59:53.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:59:53.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:53.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:53.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:53.245 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 20:59:53.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:53.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:53.245 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 20:59:53.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:53.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:53.245 | INFO | Uploaded 36GiB to GPU in 1.93121s (18.6412GiB/s) | 20260506 20:59:53.245 | INFO | Downloaded 34GiB to GPU in 1.87332s (18.1496GiB/s) | 20260506 20:59:58.381 | INFO | Analyzing chunk frames = [4413440,4413952) | 20260506 20:59:58.381 | INFO | MemInfo: { | 20260506 20:59:58.381 | INFO | "gpu" : " 17 GB", | 20260506 20:59:58.381 | INFO | "huge" : "", | 20260506 20:59:58.381 | INFO | "huge_shared" : " 27 GB", | 20260506 20:59:58.381 | INFO | "malloc" : " 14 GB", | 20260506 20:59:58.381 | INFO | "pinned" : " 29 GB" | 20260506 20:59:58.381 | INFO | } | 20260506 20:59:58.382 | INFO | Perf report for the last 5.13654 seconds: | 20260506 20:59:58.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 20:59:58.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:58.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:58.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:58.382 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 20:59:58.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:58.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:58.382 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 20:59:58.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:58.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 20:59:58.382 | INFO | Uploaded 36GiB to GPU in 2.01403s (17.8746GiB/s) | 20260506 20:59:58.382 | INFO | Downloaded 34GiB to GPU in 1.91432s (17.7608GiB/s) | 20260506 21:00:03.524 | INFO | Analyzing chunk frames = [4413952,4414464) | 20260506 21:00:03.524 | INFO | MemInfo: { | 20260506 21:00:03.524 | INFO | "gpu" : " 17 GB", | 20260506 21:00:03.524 | INFO | "huge" : "", | 20260506 21:00:03.524 | INFO | "huge_shared" : " 27 GB", | 20260506 21:00:03.524 | INFO | "malloc" : " 14 GB", | 20260506 21:00:03.524 | INFO | "pinned" : " 29 GB" | 20260506 21:00:03.524 | INFO | } | 20260506 21:00:03.524 | INFO | Perf report for the last 5.14275 seconds: | 20260506 21:00:03.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:00:03.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:03.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:03.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:03.524 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 21:00:03.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:03.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:03.524 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 21:00:03.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:03.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:03.525 | INFO | Uploaded 36GiB to GPU in 2.01118s (17.8999GiB/s) | 20260506 21:00:03.525 | INFO | Downloaded 34GiB to GPU in 1.95494s (17.3918GiB/s) | 20260506 21:00:08.669 | INFO | Analyzing chunk frames = [4414464,4414976) | 20260506 21:00:08.669 | INFO | MemInfo: { | 20260506 21:00:08.669 | INFO | "gpu" : " 17 GB", | 20260506 21:00:08.669 | INFO | "huge" : "", | 20260506 21:00:08.669 | INFO | "huge_shared" : " 27 GB", | 20260506 21:00:08.669 | INFO | "malloc" : " 14 GB", | 20260506 21:00:08.669 | INFO | "pinned" : " 29 GB" | 20260506 21:00:08.669 | INFO | } | 20260506 21:00:08.669 | INFO | Perf report for the last 5.14509 seconds: | 20260506 21:00:08.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:00:08.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:08.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:08.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:08.670 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 21:00:08.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:08.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:08.670 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:00:08.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:08.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:08.670 | INFO | Uploaded 36GiB to GPU in 1.95354s (18.428GiB/s) | 20260506 21:00:08.670 | INFO | Downloaded 34GiB to GPU in 1.98996s (17.0858GiB/s) | 20260506 21:00:13.806 | INFO | Analyzing chunk frames = [4414976,4415488) | 20260506 21:00:13.806 | INFO | MemInfo: { | 20260506 21:00:13.806 | INFO | "gpu" : " 17 GB", | 20260506 21:00:13.806 | INFO | "huge" : "", | 20260506 21:00:13.806 | INFO | "huge_shared" : " 27 GB", | 20260506 21:00:13.806 | INFO | "malloc" : " 14 GB", | 20260506 21:00:13.806 | INFO | "pinned" : " 29 GB" | 20260506 21:00:13.806 | INFO | } | 20260506 21:00:13.806 | INFO | Perf report for the last 5.13677 seconds: | 20260506 21:00:13.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:00:13.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:13.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:13.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:13.806 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 21:00:13.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:13.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:13.806 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 21:00:13.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:13.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:13.806 | INFO | Uploaded 36GiB to GPU in 2.04434s (17.6096GiB/s) | 20260506 21:00:13.806 | INFO | Downloaded 34GiB to GPU in 2.00163s (16.9862GiB/s) | 20260506 21:00:13.807 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:00:13.807 | INFO | Total mem usage: 18 GiB | 20260506 21:00:13.807 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:00:13.807 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -14.1%) | 20260506 21:00:13.807 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:00:13.807 | INFO | High Water Breakdown: | 20260506 21:00:13.807 | INFO | 61.7% : 9.73 GiB ( 85.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:00:13.807 | INFO | 38.2% : 6.03 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:00:13.807 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:00:13.807 | INFO | ------------------------------------------ | 20260506 21:00:13.807 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:00:13.807 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:00:13.807 | INFO | Total mem usage: 26.5 GiB | 20260506 21:00:13.807 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:00:13.807 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:00:13.807 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:00:13.807 | INFO | High Water Breakdown: | 20260506 21:00:13.807 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:00:13.807 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:00:13.807 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:00:13.807 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:00:13.807 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:00:13.807 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:00:13.807 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:00:13.807 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:00:13.807 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:00:13.807 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:00:13.807 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:00:13.807 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:00:13.807 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:00:13.807 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:00:13.807 | INFO | ------------------------------------------ | 20260506 21:00:13.807 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:00:13.807 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:00:13.807 | INFO | Total mem usage: 25.6 GiB | 20260506 21:00:13.807 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:00:13.808 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 21:00:13.808 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 21:00:13.808 | INFO | High Water Breakdown: | 20260506 21:00:13.808 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 21:00:13.808 | INFO | ------------------------------------------ | 20260506 21:00:13.808 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:00:13.808 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:00:13.808 | INFO | Total mem usage: 15.7 GiB | 20260506 21:00:13.808 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:00:13.808 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:00:13.808 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:00:13.808 | INFO | High Water Breakdown: | 20260506 21:00:13.808 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:00:13.808 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:00:13.808 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:00:13.808 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:00:13.808 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:00:13.808 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:00:13.808 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:00:13.808 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:00:13.808 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:00:13.808 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:00:13.808 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:00:13.808 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:00:13.808 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:00:13.808 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:00:13.808 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:00:13.808 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:00:13.808 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:00:13.808 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:00:13.808 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:00:13.808 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:00:13.808 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:00:13.808 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:00:13.808 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:00:13.808 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:00:13.808 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:00:13.808 | INFO | ------------------------------------------ | 20260506 21:00:13.808 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:00:13.808 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:00:13.808 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:00:13.808 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:00:13.808 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.59e+01, 1.34e+01 / 2.50e+00 | 20260506 21:00:13.808 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.02e-01, 1.09e-02 / 7.69e-03 | 20260506 21:00:13.808 | INFO | PerfReport: 11.63%, Baseline -- 3.62e+00 / 4.43e+00, 5.46e-01 / 3.16e-01 | 20260506 21:00:13.808 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.81e-01 / 1.26e-01 | 20260506 21:00:13.808 | INFO | PerfReport: 16.03%, DME -- 3.99e+01 / 4.48e+01, 4.11e+00 / 2.57e+00 | 20260506 21:00:13.808 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.39e+00 / 2.02e+00 | 20260506 21:00:13.808 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.22e+00 / 2.84e+00, 3.78e-01 / 1.58e-01 | 20260506 21:00:13.808 | INFO | PerfReport: 3.29%, Metrics -- 1.02e+00 / 1.35e+00, 2.68e-01 / 1.63e-01 | 20260506 21:00:13.808 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.52e-01 / 3.99e-01, 5.50e-02 / 2.19e-02 | 20260506 21:00:13.808 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:00:13.808 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:00:13.808 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:00:13.808 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 21:00:13.808 | INFO | PerfReport: 21.53%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 21:00:13.808 | INFO | PerfReport: 21.25%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 21:00:13.808 | INFO | PerfReport: 32.17%, Compute -- 3.11e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 21:00:13.808 | INFO | PerfReport: 1.32%, LockUpload -- 1.28e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260506 21:00:13.808 | INFO | PerfReport: 1.19%, LockDownload -- 1.16e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260506 21:00:13.808 | INFO | PerfReport: 22.52%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 21:00:13.808 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260506 21:00:13.923 | INFO | Starting a round of Baz IO | 20260506 21:00:17.051 | INFO | Populating RTMetrics report | 20260506 21:00:17.051 | INFO | RTMetrics report populated | 20260506 21:00:17.052 | INFO | Opening tmp RTMetrics json file | 20260506 21:00:17.066 | INFO | Serializing RTMetrics to json | 20260506 21:00:17.068 | INFO | Serialization done, swapping tmp file now... | 20260506 21:00:17.070 | INFO | RTMetrics done | 20260506 21:00:17.383 | INFO | Finished a round of Baz IO: Wrote 1.959 GiB in 3.46 seconds | 20260506 21:00:17.383 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.07% / 0.00% / 65.93% | 20260506 21:00:17.383 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.670421% / 3.97364e-06% / 7.54992e-05% | 20260506 21:00:17.383 | INFO | Baz IO overall: | 20260506 21:00:17.383 | INFO | BytesWritten : 2328884021808 | 20260506 21:00:17.383 | INFO | NumEvents : 1124770994636 | 20260506 21:00:18.947 | INFO | Analyzing chunk frames = [4415488,4416000) | 20260506 21:00:18.947 | INFO | MemInfo: { | 20260506 21:00:18.947 | INFO | "gpu" : " 17 GB", | 20260506 21:00:18.947 | INFO | "huge" : "", | 20260506 21:00:18.947 | INFO | "huge_shared" : " 27 GB", | 20260506 21:00:18.947 | INFO | "malloc" : " 14 GB", | 20260506 21:00:18.947 | INFO | "pinned" : " 29 GB" | 20260506 21:00:18.947 | INFO | } | 20260506 21:00:18.947 | INFO | Perf report for the last 5.14057 seconds: | 20260506 21:00:18.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:00:18.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:18.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:18.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:18.947 | INFO | ANALYSIS : 62.5%, 2.9, 95.7ms, 0 | 20260506 21:00:18.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:18.947 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 21:00:18.947 | INFO | PRE_HQ : 59.8%, 1.8, 56.8ms, 0 | 20260506 21:00:18.947 | INFO | BAZWRITER : 67.0%, 4.8, 165.7ms, 0 | 20260506 21:00:18.947 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 21:00:18.947 | INFO | Uploaded 36GiB to GPU in 2.06999s (17.3914GiB/s) | 20260506 21:00:18.947 | INFO | Downloaded 38GiB to GPU in 2.21152s (17.1828GiB/s) | 20260506 21:00:24.087 | INFO | Analyzing chunk frames = [4416000,4416512) | 20260506 21:00:24.087 | INFO | MemInfo: { | 20260506 21:00:24.087 | INFO | "gpu" : " 17 GB", | 20260506 21:00:24.087 | INFO | "huge" : "", | 20260506 21:00:24.087 | INFO | "huge_shared" : " 27 GB", | 20260506 21:00:24.087 | INFO | "malloc" : " 14 GB", | 20260506 21:00:24.087 | INFO | "pinned" : " 29 GB" | 20260506 21:00:24.087 | INFO | } | 20260506 21:00:24.088 | INFO | Perf report for the last 5.14074 seconds: | 20260506 21:00:24.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:00:24.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:24.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:24.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:24.088 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 21:00:24.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:24.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:24.088 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:00:24.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:24.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:24.088 | INFO | Uploaded 36GiB to GPU in 2.01256s (17.8876GiB/s) | 20260506 21:00:24.088 | INFO | Downloaded 34GiB to GPU in 1.92589s (17.6542GiB/s) | 20260506 21:00:29.220 | INFO | Analyzing chunk frames = [4416512,4417024) | 20260506 21:00:29.220 | INFO | MemInfo: { | 20260506 21:00:29.220 | INFO | "gpu" : " 17 GB", | 20260506 21:00:29.220 | INFO | "huge" : "", | 20260506 21:00:29.220 | INFO | "huge_shared" : " 27 GB", | 20260506 21:00:29.220 | INFO | "malloc" : " 14 GB", | 20260506 21:00:29.220 | INFO | "pinned" : " 29 GB" | 20260506 21:00:29.220 | INFO | } | 20260506 21:00:29.220 | INFO | Perf report for the last 5.13233 seconds: | 20260506 21:00:29.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:00:29.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:29.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:29.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:29.220 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260506 21:00:29.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:29.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:29.220 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:00:29.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:29.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:29.220 | INFO | Uploaded 36GiB to GPU in 1.99756s (18.0219GiB/s) | 20260506 21:00:29.220 | INFO | Downloaded 34GiB to GPU in 1.92651s (17.6485GiB/s) | 20260506 21:00:34.355 | INFO | Analyzing chunk frames = [4417024,4417536) | 20260506 21:00:34.355 | INFO | MemInfo: { | 20260506 21:00:34.355 | INFO | "gpu" : " 17 GB", | 20260506 21:00:34.355 | INFO | "huge" : "", | 20260506 21:00:34.355 | INFO | "huge_shared" : " 27 GB", | 20260506 21:00:34.355 | INFO | "malloc" : " 14 GB", | 20260506 21:00:34.355 | INFO | "pinned" : " 29 GB" | 20260506 21:00:34.355 | INFO | } | 20260506 21:00:34.355 | INFO | Perf report for the last 5.13517 seconds: | 20260506 21:00:34.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:00:34.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:34.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:34.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:34.355 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 21:00:34.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:34.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:34.355 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 21:00:34.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:34.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:34.356 | INFO | Uploaded 36GiB to GPU in 1.95601s (18.4048GiB/s) | 20260506 21:00:34.356 | INFO | Downloaded 34GiB to GPU in 1.8744s (18.1391GiB/s) | 20260506 21:00:39.497 | INFO | Analyzing chunk frames = [4417536,4418048) | 20260506 21:00:39.497 | INFO | MemInfo: { | 20260506 21:00:39.497 | INFO | "gpu" : " 17 GB", | 20260506 21:00:39.497 | INFO | "huge" : "", | 20260506 21:00:39.497 | INFO | "huge_shared" : " 27 GB", | 20260506 21:00:39.497 | INFO | "malloc" : " 14 GB", | 20260506 21:00:39.497 | INFO | "pinned" : " 29 GB" | 20260506 21:00:39.497 | INFO | } | 20260506 21:00:39.498 | INFO | Perf report for the last 5.14248 seconds: | 20260506 21:00:39.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:00:39.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:39.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:39.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:39.498 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 21:00:39.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:39.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:39.498 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:00:39.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:39.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:39.498 | INFO | Uploaded 36GiB to GPU in 2.02063s (17.8162GiB/s) | 20260506 21:00:39.498 | INFO | Downloaded 34GiB to GPU in 1.91032s (17.7981GiB/s) | 20260506 21:00:44.641 | INFO | Analyzing chunk frames = [4418048,4418560) | 20260506 21:00:44.641 | INFO | MemInfo: { | 20260506 21:00:44.641 | INFO | "gpu" : " 17 GB", | 20260506 21:00:44.641 | INFO | "huge" : "", | 20260506 21:00:44.641 | INFO | "huge_shared" : " 27 GB", | 20260506 21:00:44.641 | INFO | "malloc" : " 14 GB", | 20260506 21:00:44.641 | INFO | "pinned" : " 29 GB" | 20260506 21:00:44.641 | INFO | } | 20260506 21:00:44.642 | INFO | Perf report for the last 5.14394 seconds: | 20260506 21:00:44.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:00:44.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:44.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:44.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:44.642 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 21:00:44.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:44.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:44.642 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 21:00:44.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:44.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:44.642 | INFO | Uploaded 36GiB to GPU in 2.01352s (17.8791GiB/s) | 20260506 21:00:44.642 | INFO | Downloaded 34GiB to GPU in 1.95448s (17.396GiB/s) | 20260506 21:00:49.782 | INFO | Analyzing chunk frames = [4418560,4419072) | 20260506 21:00:49.782 | INFO | MemInfo: { | 20260506 21:00:49.782 | INFO | "gpu" : " 17 GB", | 20260506 21:00:49.782 | INFO | "huge" : "", | 20260506 21:00:49.782 | INFO | "huge_shared" : " 27 GB", | 20260506 21:00:49.782 | INFO | "malloc" : " 14 GB", | 20260506 21:00:49.782 | INFO | "pinned" : " 29 GB" | 20260506 21:00:49.782 | INFO | } | 20260506 21:00:49.782 | INFO | Perf report for the last 5.14052 seconds: | 20260506 21:00:49.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:00:49.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:49.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:49.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:49.782 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:00:49.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:49.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:49.782 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 21:00:49.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:49.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:49.782 | INFO | Uploaded 36GiB to GPU in 1.95424s (18.4215GiB/s) | 20260506 21:00:49.783 | INFO | Downloaded 34GiB to GPU in 2.00467s (16.9604GiB/s) | 20260506 21:00:54.924 | INFO | Analyzing chunk frames = [4419072,4419584) | 20260506 21:00:54.924 | INFO | MemInfo: { | 20260506 21:00:54.924 | INFO | "gpu" : " 17 GB", | 20260506 21:00:54.924 | INFO | "huge" : "", | 20260506 21:00:54.924 | INFO | "huge_shared" : " 27 GB", | 20260506 21:00:54.924 | INFO | "malloc" : " 14 GB", | 20260506 21:00:54.924 | INFO | "pinned" : " 29 GB" | 20260506 21:00:54.924 | INFO | } | 20260506 21:00:54.924 | INFO | Perf report for the last 5.14232 seconds: | 20260506 21:00:54.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:00:54.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:54.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:54.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:54.925 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:00:54.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:54.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:54.925 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 21:00:54.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:54.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:00:54.925 | INFO | Uploaded 36GiB to GPU in 2.03674s (17.6753GiB/s) | 20260506 21:00:54.925 | INFO | Downloaded 34GiB to GPU in 1.99825s (17.0149GiB/s) | 20260506 21:00:55.038 | INFO | Starting a round of Baz IO | 20260506 21:00:58.166 | INFO | Populating RTMetrics report | 20260506 21:00:58.166 | INFO | RTMetrics report populated | 20260506 21:00:58.166 | INFO | Opening tmp RTMetrics json file | 20260506 21:00:58.322 | INFO | Serializing RTMetrics to json | 20260506 21:00:58.497 | INFO | Serialization done, swapping tmp file now... | 20260506 21:00:58.530 | INFO | RTMetrics done | 20260506 21:00:58.572 | INFO | Finished a round of Baz IO: Wrote 1.96 GiB in 3.534 seconds | 20260506 21:00:58.572 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.06% / 0.00% / 65.93% | 20260506 21:00:58.572 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.670838% / 0% / 7.54992e-05% | 20260506 21:00:58.572 | INFO | Baz IO overall: | 20260506 21:00:58.572 | INFO | BytesWritten : 2330988546859 | 20260506 21:00:58.572 | INFO | NumEvents : 1125779425519 | 20260506 21:01:00.059 | INFO | Analyzing chunk frames = [4419584,4420096) | 20260506 21:01:00.059 | INFO | MemInfo: { | 20260506 21:01:00.059 | INFO | "gpu" : " 17 GB", | 20260506 21:01:00.059 | INFO | "huge" : "", | 20260506 21:01:00.059 | INFO | "huge_shared" : " 27 GB", | 20260506 21:01:00.059 | INFO | "malloc" : " 14 GB", | 20260506 21:01:00.060 | INFO | "pinned" : " 29 GB" | 20260506 21:01:00.060 | INFO | } | 20260506 21:01:00.060 | INFO | Perf report for the last 5.1353 seconds: | 20260506 21:01:00.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:01:00.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:00.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:00.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:00.060 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 21:01:00.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:00.060 | INFO | RT_METRICS : 49.8%, 1.0, 26.1ms, 0 | 20260506 21:01:00.060 | INFO | PRE_HQ : 59.7%, 1.8, 57.3ms, 0 | 20260506 21:01:00.060 | INFO | BAZWRITER : 68.4%, 4.8, 167.5ms, 0 | 20260506 21:01:00.060 | INFO | BAZ_SAMPLING : 5.2%, 1.0, 2.7ms, 0 | 20260506 21:01:00.060 | INFO | Uploaded 36GiB to GPU in 2.04231s (17.6271GiB/s) | 20260506 21:01:00.060 | INFO | Downloaded 38GiB to GPU in 2.21326s (17.1692GiB/s) | 20260506 21:01:05.196 | INFO | Analyzing chunk frames = [4420096,4420608) | 20260506 21:01:05.196 | INFO | MemInfo: { | 20260506 21:01:05.196 | INFO | "gpu" : " 17 GB", | 20260506 21:01:05.196 | INFO | "huge" : "", | 20260506 21:01:05.196 | INFO | "huge_shared" : " 27 GB", | 20260506 21:01:05.196 | INFO | "malloc" : " 14 GB", | 20260506 21:01:05.196 | INFO | "pinned" : " 29 GB" | 20260506 21:01:05.196 | INFO | } | 20260506 21:01:05.196 | INFO | Perf report for the last 5.13662 seconds: | 20260506 21:01:05.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:01:05.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:05.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:05.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:05.196 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:01:05.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:05.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:05.197 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:01:05.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:05.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:05.197 | INFO | Uploaded 36GiB to GPU in 2.00239s (17.9785GiB/s) | 20260506 21:01:05.197 | INFO | Downloaded 34GiB to GPU in 1.92947s (17.6214GiB/s) | 20260506 21:01:10.336 | INFO | Analyzing chunk frames = [4420608,4421120) | 20260506 21:01:10.336 | INFO | MemInfo: { | 20260506 21:01:10.336 | INFO | "gpu" : " 17 GB", | 20260506 21:01:10.336 | INFO | "huge" : "", | 20260506 21:01:10.336 | INFO | "huge_shared" : " 27 GB", | 20260506 21:01:10.336 | INFO | "malloc" : " 14 GB", | 20260506 21:01:10.336 | INFO | "pinned" : " 29 GB" | 20260506 21:01:10.336 | INFO | } | 20260506 21:01:10.336 | INFO | Perf report for the last 5.13992 seconds: | 20260506 21:01:10.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:01:10.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:10.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:10.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:10.336 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 21:01:10.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:10.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:10.336 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:01:10.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:10.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:10.337 | INFO | Uploaded 36GiB to GPU in 1.97578s (18.2207GiB/s) | 20260506 21:01:10.337 | INFO | Downloaded 34GiB to GPU in 1.92285s (17.682GiB/s) | 20260506 21:01:15.473 | INFO | Analyzing chunk frames = [4421120,4421632) | 20260506 21:01:15.473 | INFO | MemInfo: { | 20260506 21:01:15.473 | INFO | "gpu" : " 17 GB", | 20260506 21:01:15.473 | INFO | "huge" : "", | 20260506 21:01:15.473 | INFO | "huge_shared" : " 27 GB", | 20260506 21:01:15.473 | INFO | "malloc" : " 14 GB", | 20260506 21:01:15.473 | INFO | "pinned" : " 29 GB" | 20260506 21:01:15.473 | INFO | } | 20260506 21:01:15.473 | INFO | Perf report for the last 5.13721 seconds: | 20260506 21:01:15.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:01:15.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:15.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:15.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:15.474 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 21:01:15.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:15.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:15.474 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:01:15.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:15.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:15.474 | INFO | Uploaded 36GiB to GPU in 1.93876s (18.5686GiB/s) | 20260506 21:01:15.474 | INFO | Downloaded 34GiB to GPU in 1.8731s (18.1517GiB/s) | 20260506 21:01:20.612 | INFO | Analyzing chunk frames = [4421632,4422144) | 20260506 21:01:20.612 | INFO | MemInfo: { | 20260506 21:01:20.612 | INFO | "gpu" : " 17 GB", | 20260506 21:01:20.612 | INFO | "huge" : "", | 20260506 21:01:20.612 | INFO | "huge_shared" : " 27 GB", | 20260506 21:01:20.612 | INFO | "malloc" : " 14 GB", | 20260506 21:01:20.612 | INFO | "pinned" : " 29 GB" | 20260506 21:01:20.612 | INFO | } | 20260506 21:01:20.612 | INFO | Perf report for the last 5.13882 seconds: | 20260506 21:01:20.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:01:20.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:20.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:20.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:20.612 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 21:01:20.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:20.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:20.613 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 21:01:20.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:20.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:20.613 | INFO | Uploaded 36GiB to GPU in 1.99455s (18.0491GiB/s) | 20260506 21:01:20.613 | INFO | Downloaded 34GiB to GPU in 1.91626s (17.7429GiB/s) | 20260506 21:01:25.757 | INFO | Analyzing chunk frames = [4422144,4422656) | 20260506 21:01:25.757 | INFO | MemInfo: { | 20260506 21:01:25.757 | INFO | "gpu" : " 17 GB", | 20260506 21:01:25.757 | INFO | "huge" : "", | 20260506 21:01:25.757 | INFO | "huge_shared" : " 27 GB", | 20260506 21:01:25.757 | INFO | "malloc" : " 14 GB", | 20260506 21:01:25.757 | INFO | "pinned" : " 29 GB" | 20260506 21:01:25.757 | INFO | } | 20260506 21:01:25.757 | INFO | Perf report for the last 5.14459 seconds: | 20260506 21:01:25.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:01:25.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:25.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:25.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:25.757 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 21:01:25.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:25.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:25.757 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260506 21:01:25.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:25.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:25.757 | INFO | Uploaded 36GiB to GPU in 1.99241s (18.0686GiB/s) | 20260506 21:01:25.757 | INFO | Downloaded 34GiB to GPU in 1.94954s (17.4401GiB/s) | 20260506 21:01:30.897 | INFO | Analyzing chunk frames = [4422656,4423168) | 20260506 21:01:30.897 | INFO | MemInfo: { | 20260506 21:01:30.897 | INFO | "gpu" : " 17 GB", | 20260506 21:01:30.897 | INFO | "huge" : "", | 20260506 21:01:30.897 | INFO | "huge_shared" : " 27 GB", | 20260506 21:01:30.897 | INFO | "malloc" : " 14 GB", | 20260506 21:01:30.897 | INFO | "pinned" : " 29 GB" | 20260506 21:01:30.897 | INFO | } | 20260506 21:01:30.898 | INFO | Perf report for the last 5.14064 seconds: | 20260506 21:01:30.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:01:30.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:30.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:30.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:30.898 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:01:30.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:30.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:30.898 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 21:01:30.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:30.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:30.898 | INFO | Uploaded 36GiB to GPU in 1.97157s (18.2596GiB/s) | 20260506 21:01:30.898 | INFO | Downloaded 34GiB to GPU in 1.99883s (17.0099GiB/s) | 20260506 21:01:36.033 | INFO | Analyzing chunk frames = [4423168,4423680) | 20260506 21:01:36.033 | INFO | MemInfo: { | 20260506 21:01:36.033 | INFO | "gpu" : " 17 GB", | 20260506 21:01:36.033 | INFO | "huge" : "", | 20260506 21:01:36.033 | INFO | "huge_shared" : " 27 GB", | 20260506 21:01:36.033 | INFO | "malloc" : " 14 GB", | 20260506 21:01:36.033 | INFO | "pinned" : " 29 GB" | 20260506 21:01:36.033 | INFO | } | 20260506 21:01:36.033 | INFO | Perf report for the last 5.13594 seconds: | 20260506 21:01:36.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:01:36.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:36.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:36.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:36.034 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 21:01:36.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:36.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:36.034 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 21:01:36.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:36.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:36.034 | INFO | Uploaded 36GiB to GPU in 2.03312s (17.7068GiB/s) | 20260506 21:01:36.034 | INFO | Downloaded 34GiB to GPU in 2.00761s (16.9355GiB/s) | 20260506 21:01:36.034 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:01:36.034 | INFO | Total mem usage: 18 GiB | 20260506 21:01:36.034 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:01:36.034 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -9.51%) | 20260506 21:01:36.034 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:01:36.034 | INFO | High Water Breakdown: | 20260506 21:01:36.034 | INFO | 62.3% : 10.2 GiB ( 89.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:01:36.034 | INFO | 37.6% : 6.18 GiB ( 92.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:01:36.034 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:01:36.034 | INFO | ------------------------------------------ | 20260506 21:01:36.034 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:01:36.034 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:01:36.034 | INFO | Total mem usage: 26.5 GiB | 20260506 21:01:36.034 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:01:36.034 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:01:36.034 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:01:36.034 | INFO | High Water Breakdown: | 20260506 21:01:36.034 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:01:36.034 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:01:36.034 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:01:36.034 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:01:36.034 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:01:36.034 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:01:36.034 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:01:36.034 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:01:36.034 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:01:36.034 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:01:36.034 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:01:36.034 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:01:36.034 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:01:36.034 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:01:36.034 | INFO | ------------------------------------------ | 20260506 21:01:36.034 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:01:36.035 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:01:36.035 | INFO | Total mem usage: 25.6 GiB | 20260506 21:01:36.035 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:01:36.035 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:01:36.035 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:01:36.035 | INFO | High Water Breakdown: | 20260506 21:01:36.035 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:01:36.035 | INFO | ------------------------------------------ | 20260506 21:01:36.035 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:01:36.035 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:01:36.035 | INFO | Total mem usage: 15.7 GiB | 20260506 21:01:36.035 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:01:36.035 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:01:36.035 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:01:36.035 | INFO | High Water Breakdown: | 20260506 21:01:36.035 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:01:36.035 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:01:36.035 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:01:36.035 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:01:36.035 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:01:36.035 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:01:36.035 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:01:36.035 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:01:36.035 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:01:36.035 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:01:36.035 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:01:36.035 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:01:36.035 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:01:36.035 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:01:36.035 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:01:36.035 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:01:36.035 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:01:36.035 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:01:36.035 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:01:36.035 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:01:36.035 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:01:36.035 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:01:36.035 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:01:36.035 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:01:36.035 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:01:36.035 | INFO | ------------------------------------------ | 20260506 21:01:36.035 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:01:36.035 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:01:36.035 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:01:36.035 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:01:36.035 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.59e+01, 1.35e+01 / 2.65e+00 | 20260506 21:01:36.035 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 9.48e-03 / 6.29e-03 | 20260506 21:01:36.035 | INFO | PerfReport: 11.63%, Baseline -- 3.62e+00 / 4.41e+00, 5.53e-01 / 3.47e-01 | 20260506 21:01:36.035 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.47e+00, 1.84e-01 / 1.27e-01 | 20260506 21:01:36.035 | INFO | PerfReport: 16.01%, DME -- 3.99e+01 / 4.43e+01, 4.05e+00 / 2.51e+00 | 20260506 21:01:36.035 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.78e+01 / 2.11e+01, 1.44e+00 / 1.33e+00 | 20260506 21:01:36.035 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.21e+00 / 2.76e+00, 3.70e-01 / 1.99e-01 | 20260506 21:01:36.035 | INFO | PerfReport: 3.28%, Metrics -- 1.02e+00 / 1.37e+00, 2.72e-01 / 1.81e-01 | 20260506 21:01:36.035 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.51e-01 / 4.00e-01, 5.46e-02 / 2.54e-02 | 20260506 21:01:36.035 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:01:36.035 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:01:36.035 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:01:36.035 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 21:01:36.035 | INFO | PerfReport: 21.55%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 21:01:36.036 | INFO | PerfReport: 21.28%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 21:01:36.036 | INFO | PerfReport: 32.21%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 21:01:36.036 | INFO | PerfReport: 1.25%, LockUpload -- 1.20e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260506 21:01:36.036 | INFO | PerfReport: 1.20%, LockDownload -- 1.16e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260506 21:01:36.036 | INFO | PerfReport: 22.51%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 21:01:36.036 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260506 21:01:36.148 | INFO | Starting a round of Baz IO | 20260506 21:01:39.271 | INFO | Populating RTMetrics report | 20260506 21:01:39.271 | INFO | RTMetrics report populated | 20260506 21:01:39.271 | INFO | Opening tmp RTMetrics json file | 20260506 21:01:39.273 | INFO | Serializing RTMetrics to json | 20260506 21:01:39.274 | INFO | Serialization done, swapping tmp file now... | 20260506 21:01:39.275 | INFO | RTMetrics done | 20260506 21:01:39.889 | INFO | Finished a round of Baz IO: Wrote 1.96 GiB in 3.741 seconds | 20260506 21:01:39.889 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.06% / 0.00% / 65.94% | 20260506 21:01:39.889 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.668617% / 7.94729e-06% / 5.5631e-05% | 20260506 21:01:39.889 | INFO | Baz IO overall: | 20260506 21:01:39.889 | INFO | BytesWritten : 2333092549468 | 20260506 21:01:39.889 | INFO | NumEvents : 1126787459985 | 20260506 21:01:41.173 | INFO | Analyzing chunk frames = [4423680,4424192) | 20260506 21:01:41.173 | INFO | MemInfo: { | 20260506 21:01:41.173 | INFO | "gpu" : " 17 GB", | 20260506 21:01:41.173 | INFO | "huge" : "", | 20260506 21:01:41.173 | INFO | "huge_shared" : " 27 GB", | 20260506 21:01:41.173 | INFO | "malloc" : " 14 GB", | 20260506 21:01:41.173 | INFO | "pinned" : " 29 GB" | 20260506 21:01:41.173 | INFO | } | 20260506 21:01:41.173 | INFO | Perf report for the last 5.13912 seconds: | 20260506 21:01:41.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:01:41.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:41.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:41.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:41.173 | INFO | ANALYSIS : 62.4%, 2.9, 95.5ms, 0 | 20260506 21:01:41.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:41.173 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 21:01:41.173 | INFO | PRE_HQ : 59.5%, 1.8, 56.7ms, 0 | 20260506 21:01:41.173 | INFO | BAZWRITER : 72.5%, 4.8, 179.2ms, 0 | 20260506 21:01:41.173 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 21:01:41.173 | INFO | Uploaded 36GiB to GPU in 2.01724s (17.8461GiB/s) | 20260506 21:01:41.173 | INFO | Downloaded 38GiB to GPU in 2.21245s (17.1755GiB/s) | 20260506 21:01:46.311 | INFO | Analyzing chunk frames = [4424192,4424704) | 20260506 21:01:46.311 | INFO | MemInfo: { | 20260506 21:01:46.311 | INFO | "gpu" : " 17 GB", | 20260506 21:01:46.311 | INFO | "huge" : "", | 20260506 21:01:46.311 | INFO | "huge_shared" : " 27 GB", | 20260506 21:01:46.311 | INFO | "malloc" : " 14 GB", | 20260506 21:01:46.311 | INFO | "pinned" : " 29 GB" | 20260506 21:01:46.311 | INFO | } | 20260506 21:01:46.311 | INFO | Perf report for the last 5.13862 seconds: | 20260506 21:01:46.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:01:46.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:46.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:46.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:46.311 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:01:46.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:46.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:46.311 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:01:46.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:46.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:46.312 | INFO | Uploaded 36GiB to GPU in 2.01019s (17.9087GiB/s) | 20260506 21:01:46.312 | INFO | Downloaded 34GiB to GPU in 1.93161s (17.6019GiB/s) | 20260506 21:01:51.446 | INFO | Analyzing chunk frames = [4424704,4425216) | 20260506 21:01:51.446 | INFO | MemInfo: { | 20260506 21:01:51.446 | INFO | "gpu" : " 17 GB", | 20260506 21:01:51.446 | INFO | "huge" : "", | 20260506 21:01:51.446 | INFO | "huge_shared" : " 27 GB", | 20260506 21:01:51.446 | INFO | "malloc" : " 14 GB", | 20260506 21:01:51.446 | INFO | "pinned" : " 29 GB" | 20260506 21:01:51.446 | INFO | } | 20260506 21:01:51.447 | INFO | Perf report for the last 5.13525 seconds: | 20260506 21:01:51.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:01:51.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:51.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:51.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:51.447 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 21:01:51.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:51.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:51.447 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:01:51.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:51.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:51.447 | INFO | Uploaded 36GiB to GPU in 1.95688s (18.3966GiB/s) | 20260506 21:01:51.447 | INFO | Downloaded 34GiB to GPU in 1.92644s (17.6492GiB/s) | 20260506 21:01:56.593 | INFO | Analyzing chunk frames = [4425216,4425728) | 20260506 21:01:56.593 | INFO | MemInfo: { | 20260506 21:01:56.593 | INFO | "gpu" : " 17 GB", | 20260506 21:01:56.593 | INFO | "huge" : "", | 20260506 21:01:56.593 | INFO | "huge_shared" : " 27 GB", | 20260506 21:01:56.593 | INFO | "malloc" : " 14 GB", | 20260506 21:01:56.593 | INFO | "pinned" : " 29 GB" | 20260506 21:01:56.593 | INFO | } | 20260506 21:01:56.593 | INFO | Perf report for the last 5.1463 seconds: | 20260506 21:01:56.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:01:56.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:56.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:56.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:56.593 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 21:01:56.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:56.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:56.593 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 21:01:56.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:56.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:01:56.593 | INFO | Uploaded 36GiB to GPU in 1.93701s (18.5853GiB/s) | 20260506 21:01:56.593 | INFO | Downloaded 34GiB to GPU in 1.87353s (18.1476GiB/s) | 20260506 21:02:01.727 | INFO | Analyzing chunk frames = [4425728,4426240) | 20260506 21:02:01.727 | INFO | MemInfo: { | 20260506 21:02:01.727 | INFO | "gpu" : " 17 GB", | 20260506 21:02:01.727 | INFO | "huge" : "", | 20260506 21:02:01.727 | INFO | "huge_shared" : " 27 GB", | 20260506 21:02:01.727 | INFO | "malloc" : " 14 GB", | 20260506 21:02:01.727 | INFO | "pinned" : " 29 GB" | 20260506 21:02:01.727 | INFO | } | 20260506 21:02:01.727 | INFO | Perf report for the last 5.13418 seconds: | 20260506 21:02:01.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:02:01.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:01.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:01.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:01.728 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 21:02:01.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:01.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:01.728 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:02:01.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:01.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:01.728 | INFO | Uploaded 36GiB to GPU in 2.00138s (17.9876GiB/s) | 20260506 21:02:01.728 | INFO | Downloaded 34GiB to GPU in 1.91465s (17.7579GiB/s) | 20260506 21:02:06.869 | INFO | Analyzing chunk frames = [4426240,4426752) | 20260506 21:02:06.869 | INFO | MemInfo: { | 20260506 21:02:06.869 | INFO | "gpu" : " 17 GB", | 20260506 21:02:06.869 | INFO | "huge" : "", | 20260506 21:02:06.869 | INFO | "huge_shared" : " 27 GB", | 20260506 21:02:06.869 | INFO | "malloc" : " 14 GB", | 20260506 21:02:06.869 | INFO | "pinned" : " 29 GB" | 20260506 21:02:06.869 | INFO | } | 20260506 21:02:06.869 | INFO | Perf report for the last 5.14224 seconds: | 20260506 21:02:06.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:02:06.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:06.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:06.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:06.869 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260506 21:02:06.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:06.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:06.869 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 21:02:06.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:06.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:06.870 | INFO | Uploaded 36GiB to GPU in 2.00318s (17.9714GiB/s) | 20260506 21:02:06.870 | INFO | Downloaded 34GiB to GPU in 1.95269s (17.4119GiB/s) | 20260506 21:02:12.016 | INFO | Analyzing chunk frames = [4426752,4427264) | 20260506 21:02:12.016 | INFO | MemInfo: { | 20260506 21:02:12.016 | INFO | "gpu" : " 17 GB", | 20260506 21:02:12.016 | INFO | "huge" : "", | 20260506 21:02:12.016 | INFO | "huge_shared" : " 27 GB", | 20260506 21:02:12.016 | INFO | "malloc" : " 14 GB", | 20260506 21:02:12.016 | INFO | "pinned" : " 29 GB" | 20260506 21:02:12.016 | INFO | } | 20260506 21:02:12.017 | INFO | Perf report for the last 5.14723 seconds: | 20260506 21:02:12.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:02:12.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:12.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:12.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:12.017 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 21:02:12.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:12.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:12.017 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 21:02:12.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:12.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:12.017 | INFO | Uploaded 36GiB to GPU in 2.00292s (17.9738GiB/s) | 20260506 21:02:12.017 | INFO | Downloaded 34GiB to GPU in 1.98537s (17.1253GiB/s) | 20260506 21:02:17.152 | INFO | Analyzing chunk frames = [4427264,4427776) | 20260506 21:02:17.152 | INFO | MemInfo: { | 20260506 21:02:17.152 | INFO | "gpu" : " 17 GB", | 20260506 21:02:17.152 | INFO | "huge" : "", | 20260506 21:02:17.152 | INFO | "huge_shared" : " 27 GB", | 20260506 21:02:17.152 | INFO | "malloc" : " 14 GB", | 20260506 21:02:17.152 | INFO | "pinned" : " 29 GB" | 20260506 21:02:17.152 | INFO | } | 20260506 21:02:17.152 | INFO | Perf report for the last 5.1354 seconds: | 20260506 21:02:17.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:02:17.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:17.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:17.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:17.152 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:02:17.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:17.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:17.152 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 21:02:17.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:17.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:17.152 | INFO | Uploaded 36GiB to GPU in 1.9627s (18.3421GiB/s) | 20260506 21:02:17.152 | INFO | Downloaded 34GiB to GPU in 1.98458s (17.1321GiB/s) | 20260506 21:02:17.251 | INFO | Starting a round of Baz IO | 20260506 21:02:20.383 | INFO | Populating RTMetrics report | 20260506 21:02:20.383 | INFO | RTMetrics report populated | 20260506 21:02:20.383 | INFO | Opening tmp RTMetrics json file | 20260506 21:02:20.444 | INFO | Serializing RTMetrics to json | 20260506 21:02:20.504 | INFO | Serialization done, swapping tmp file now... | 20260506 21:02:20.513 | INFO | RTMetrics done | 20260506 21:02:20.749 | INFO | Finished a round of Baz IO: Wrote 1.958 GiB in 3.498 seconds | 20260506 21:02:20.749 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.06% / 0.00% / 65.94% | 20260506 21:02:20.749 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.667302% / 7.94729e-06% / 6.75519e-05% | 20260506 21:02:20.749 | INFO | Baz IO overall: | 20260506 21:02:20.749 | INFO | BytesWritten : 2335194547946 | 20260506 21:02:20.749 | INFO | NumEvents : 1127793859860 | 20260506 21:02:22.289 | INFO | Analyzing chunk frames = [4427776,4428288) | 20260506 21:02:22.289 | INFO | MemInfo: { | 20260506 21:02:22.289 | INFO | "gpu" : " 17 GB", | 20260506 21:02:22.289 | INFO | "huge" : "", | 20260506 21:02:22.289 | INFO | "huge_shared" : " 27 GB", | 20260506 21:02:22.289 | INFO | "malloc" : " 14 GB", | 20260506 21:02:22.289 | INFO | "pinned" : " 29 GB" | 20260506 21:02:22.289 | INFO | } | 20260506 21:02:22.289 | INFO | Perf report for the last 5.13696 seconds: | 20260506 21:02:22.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:02:22.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:22.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:22.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:22.289 | INFO | ANALYSIS : 62.3%, 2.9, 95.0ms, 0 | 20260506 21:02:22.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:22.289 | INFO | RT_METRICS : 45.3%, 1.0, 23.7ms, 0 | 20260506 21:02:22.289 | INFO | PRE_HQ : 60.0%, 1.8, 57.1ms, 0 | 20260506 21:02:22.289 | INFO | BAZWRITER : 67.7%, 4.8, 165.3ms, 0 | 20260506 21:02:22.289 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260506 21:02:22.289 | INFO | Uploaded 36GiB to GPU in 2.02809s (17.7507GiB/s) | 20260506 21:02:22.289 | INFO | Downloaded 38GiB to GPU in 2.19652s (17.3001GiB/s) | 20260506 21:02:27.430 | INFO | Analyzing chunk frames = [4428288,4428800) | 20260506 21:02:27.430 | INFO | MemInfo: { | 20260506 21:02:27.430 | INFO | "gpu" : " 17 GB", | 20260506 21:02:27.430 | INFO | "huge" : "", | 20260506 21:02:27.430 | INFO | "huge_shared" : " 27 GB", | 20260506 21:02:27.430 | INFO | "malloc" : " 14 GB", | 20260506 21:02:27.430 | INFO | "pinned" : " 29 GB" | 20260506 21:02:27.430 | INFO | } | 20260506 21:02:27.430 | INFO | Perf report for the last 5.14097 seconds: | 20260506 21:02:27.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:02:27.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:27.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:27.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:27.430 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:02:27.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:27.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:27.430 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:02:27.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:27.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:27.430 | INFO | Uploaded 36GiB to GPU in 1.99088s (18.0824GiB/s) | 20260506 21:02:27.430 | INFO | Downloaded 34GiB to GPU in 1.93008s (17.6159GiB/s) | 20260506 21:02:32.564 | INFO | Analyzing chunk frames = [4428800,4429312) | 20260506 21:02:32.564 | INFO | MemInfo: { | 20260506 21:02:32.564 | INFO | "gpu" : " 17 GB", | 20260506 21:02:32.564 | INFO | "huge" : "", | 20260506 21:02:32.564 | INFO | "huge_shared" : " 27 GB", | 20260506 21:02:32.564 | INFO | "malloc" : " 14 GB", | 20260506 21:02:32.564 | INFO | "pinned" : " 29 GB" | 20260506 21:02:32.564 | INFO | } | 20260506 21:02:32.564 | INFO | Perf report for the last 5.13397 seconds: | 20260506 21:02:32.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:02:32.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:32.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:32.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:32.564 | INFO | ANALYSIS : 60.7%, 3.0, 93.8ms, 0 | 20260506 21:02:32.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:32.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:32.564 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:02:32.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:32.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:32.564 | INFO | Uploaded 36GiB to GPU in 1.96284s (18.3408GiB/s) | 20260506 21:02:32.564 | INFO | Downloaded 34GiB to GPU in 1.9232s (17.6789GiB/s) | 20260506 21:02:37.703 | INFO | Analyzing chunk frames = [4429312,4429824) | 20260506 21:02:37.703 | INFO | MemInfo: { | 20260506 21:02:37.703 | INFO | "gpu" : " 17 GB", | 20260506 21:02:37.703 | INFO | "huge" : "", | 20260506 21:02:37.703 | INFO | "huge_shared" : " 27 GB", | 20260506 21:02:37.703 | INFO | "malloc" : " 14 GB", | 20260506 21:02:37.703 | INFO | "pinned" : " 29 GB" | 20260506 21:02:37.703 | INFO | } | 20260506 21:02:37.703 | INFO | Perf report for the last 5.13946 seconds: | 20260506 21:02:37.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:02:37.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:37.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:37.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:37.703 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:02:37.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:37.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:37.703 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:02:37.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:37.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:37.704 | INFO | Uploaded 36GiB to GPU in 1.9339s (18.6152GiB/s) | 20260506 21:02:37.704 | INFO | Downloaded 34GiB to GPU in 1.87586s (18.125GiB/s) | 20260506 21:02:42.841 | INFO | Analyzing chunk frames = [4429824,4430336) | 20260506 21:02:42.841 | INFO | MemInfo: { | 20260506 21:02:42.841 | INFO | "gpu" : " 17 GB", | 20260506 21:02:42.841 | INFO | "huge" : "", | 20260506 21:02:42.841 | INFO | "huge_shared" : " 27 GB", | 20260506 21:02:42.841 | INFO | "malloc" : " 14 GB", | 20260506 21:02:42.841 | INFO | "pinned" : " 29 GB" | 20260506 21:02:42.842 | INFO | } | 20260506 21:02:42.842 | INFO | Perf report for the last 5.13819 seconds: | 20260506 21:02:42.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:02:42.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:42.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:42.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:42.842 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 21:02:42.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:42.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:42.842 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:02:42.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:42.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:42.842 | INFO | Uploaded 36GiB to GPU in 2.01415s (17.8735GiB/s) | 20260506 21:02:42.842 | INFO | Downloaded 34GiB to GPU in 1.92031s (17.7055GiB/s) | 20260506 21:02:47.982 | INFO | Analyzing chunk frames = [4430336,4430848) | 20260506 21:02:47.982 | INFO | MemInfo: { | 20260506 21:02:47.982 | INFO | "gpu" : " 17 GB", | 20260506 21:02:47.982 | INFO | "huge" : "", | 20260506 21:02:47.983 | INFO | "huge_shared" : " 27 GB", | 20260506 21:02:47.983 | INFO | "malloc" : " 14 GB", | 20260506 21:02:47.983 | INFO | "pinned" : " 29 GB" | 20260506 21:02:47.983 | INFO | } | 20260506 21:02:47.983 | INFO | Perf report for the last 5.14113 seconds: | 20260506 21:02:47.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:02:47.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:47.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:47.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:47.983 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 21:02:47.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:47.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:47.983 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 21:02:47.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:47.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:47.983 | INFO | Uploaded 36GiB to GPU in 2.01463s (17.8693GiB/s) | 20260506 21:02:47.983 | INFO | Downloaded 34GiB to GPU in 1.96725s (17.283GiB/s) | 20260506 21:02:53.124 | INFO | Analyzing chunk frames = [4430848,4431360) | 20260506 21:02:53.124 | INFO | MemInfo: { | 20260506 21:02:53.124 | INFO | "gpu" : " 17 GB", | 20260506 21:02:53.124 | INFO | "huge" : "", | 20260506 21:02:53.124 | INFO | "huge_shared" : " 27 GB", | 20260506 21:02:53.124 | INFO | "malloc" : " 14 GB", | 20260506 21:02:53.124 | INFO | "pinned" : " 29 GB" | 20260506 21:02:53.124 | INFO | } | 20260506 21:02:53.124 | INFO | Perf report for the last 5.14174 seconds: | 20260506 21:02:53.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:02:53.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:53.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:53.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:53.125 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 21:02:53.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:53.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:53.125 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:02:53.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:53.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:53.125 | INFO | Uploaded 36GiB to GPU in 1.95113s (18.4508GiB/s) | 20260506 21:02:53.125 | INFO | Downloaded 34GiB to GPU in 1.99357s (17.0548GiB/s) | 20260506 21:02:58.268 | INFO | Analyzing chunk frames = [4431360,4431872) | 20260506 21:02:58.268 | INFO | MemInfo: { | 20260506 21:02:58.268 | INFO | "gpu" : " 17 GB", | 20260506 21:02:58.268 | INFO | "huge" : "", | 20260506 21:02:58.269 | INFO | "huge_shared" : " 27 GB", | 20260506 21:02:58.269 | INFO | "malloc" : " 14 GB", | 20260506 21:02:58.269 | INFO | "pinned" : " 29 GB" | 20260506 21:02:58.269 | INFO | } | 20260506 21:02:58.269 | INFO | Perf report for the last 5.14427 seconds: | 20260506 21:02:58.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:02:58.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:58.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:58.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:58.269 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 21:02:58.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:58.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:58.269 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 21:02:58.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:58.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:02:58.269 | INFO | Uploaded 36GiB to GPU in 2.01545s (17.862GiB/s) | 20260506 21:02:58.269 | INFO | Downloaded 34GiB to GPU in 1.98781s (17.1043GiB/s) | 20260506 21:02:58.269 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:02:58.269 | INFO | Total mem usage: 18 GiB | 20260506 21:02:58.269 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:02:58.269 | INFO | Recent High Water Marks Sum: 17.1 GiB (Savings: -5.34%) | 20260506 21:02:58.269 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:02:58.269 | INFO | High Water Breakdown: | 20260506 21:02:58.269 | INFO | 62.8% : 10.7 GiB ( 94.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:02:58.269 | INFO | 37.1% : 6.34 GiB ( 94.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:02:58.269 | INFO | 0.116% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:02:58.269 | INFO | ------------------------------------------ | 20260506 21:02:58.269 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:02:58.269 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:02:58.269 | INFO | Total mem usage: 26.5 GiB | 20260506 21:02:58.269 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:02:58.269 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:02:58.269 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:02:58.269 | INFO | High Water Breakdown: | 20260506 21:02:58.269 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:02:58.269 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:02:58.269 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:02:58.269 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:02:58.269 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:02:58.269 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:02:58.270 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:02:58.270 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:02:58.270 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:02:58.270 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:02:58.270 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:02:58.270 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:02:58.270 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:02:58.270 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:02:58.270 | INFO | ------------------------------------------ | 20260506 21:02:58.270 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:02:58.270 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:02:58.270 | INFO | Total mem usage: 25.6 GiB | 20260506 21:02:58.270 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:02:58.270 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:02:58.270 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:02:58.270 | INFO | High Water Breakdown: | 20260506 21:02:58.270 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:02:58.270 | INFO | ------------------------------------------ | 20260506 21:02:58.270 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:02:58.270 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:02:58.270 | INFO | Total mem usage: 15.7 GiB | 20260506 21:02:58.270 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:02:58.270 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:02:58.270 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:02:58.270 | INFO | High Water Breakdown: | 20260506 21:02:58.270 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:02:58.270 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:02:58.270 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:02:58.270 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:02:58.270 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:02:58.270 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:02:58.270 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:02:58.270 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:02:58.270 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:02:58.270 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:02:58.270 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:02:58.270 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:02:58.270 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:02:58.270 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:02:58.270 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:02:58.270 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:02:58.270 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:02:58.270 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:02:58.270 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:02:58.270 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:02:58.270 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:02:58.270 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:02:58.270 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:02:58.270 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:02:58.270 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:02:58.270 | INFO | ------------------------------------------ | 20260506 21:02:58.270 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:02:58.270 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:02:58.270 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:02:58.270 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:02:58.270 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.34e+01 / 2.32e+00 | 20260506 21:02:58.270 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.03e-01, 1.04e-02 / 6.97e-03 | 20260506 21:02:58.270 | INFO | PerfReport: 11.64%, Baseline -- 3.63e+00 / 4.40e+00, 5.55e-01 / 3.65e-01 | 20260506 21:02:58.270 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.46e+00, 1.84e-01 / 1.32e-01 | 20260506 21:02:58.270 | INFO | PerfReport: 15.97%, DME -- 3.98e+01 / 4.43e+01, 3.99e+00 / 2.60e+00 | 20260506 21:02:58.270 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.79e+01 / 2.10e+01, 1.47e+00 / 1.87e+00 | 20260506 21:02:58.270 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.22e+00 / 2.78e+00, 3.78e-01 / 2.07e-01 | 20260506 21:02:58.270 | INFO | PerfReport: 3.29%, Metrics -- 1.02e+00 / 1.38e+00, 2.75e-01 / 1.68e-01 | 20260506 21:02:58.270 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.51e-01 / 4.02e-01, 5.51e-02 / 2.17e-02 | 20260506 21:02:58.270 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:02:58.270 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:02:58.270 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:02:58.270 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.31e+01 / 0.00e+00 | 20260506 21:02:58.270 | INFO | PerfReport: 21.42%, Upload -- 2.07e+01 / 0.00e+00, 6.81e+00 / 0.00e+00 | 20260506 21:02:58.270 | INFO | PerfReport: 21.27%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 21:02:58.270 | INFO | PerfReport: 32.25%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 21:02:58.270 | INFO | PerfReport: 1.17%, LockUpload -- 1.13e+00 / 0.00e+00, 4.70e+00 / 0.00e+00 | 20260506 21:02:58.270 | INFO | PerfReport: 1.26%, LockDownload -- 1.21e+00 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260506 21:02:58.270 | INFO | PerfReport: 22.61%, LockCompute -- 2.18e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 21:02:58.270 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.13e-02 / 0.00e+00 | 20260506 21:02:58.369 | INFO | Starting a round of Baz IO | 20260506 21:03:01.501 | INFO | Populating RTMetrics report | 20260506 21:03:01.502 | INFO | RTMetrics report populated | 20260506 21:03:01.502 | INFO | Opening tmp RTMetrics json file | 20260506 21:03:01.549 | INFO | Serializing RTMetrics to json | 20260506 21:03:01.596 | INFO | Serialization done, swapping tmp file now... | 20260506 21:03:01.607 | INFO | RTMetrics done | 20260506 21:03:02.101 | INFO | Finished a round of Baz IO: Wrote 1.959 GiB in 3.731 seconds | 20260506 21:03:02.101 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.05% / 0.00% / 65.94% | 20260506 21:03:02.101 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.667222% / 7.94729e-06% / 8.34465e-05% | 20260506 21:03:02.101 | INFO | Baz IO overall: | 20260506 21:03:02.101 | INFO | BytesWritten : 2337297850823 | 20260506 21:03:02.101 | INFO | NumEvents : 1128801482418 | 20260506 21:03:03.404 | INFO | Analyzing chunk frames = [4431872,4432384) | 20260506 21:03:03.404 | INFO | MemInfo: { | 20260506 21:03:03.404 | INFO | "gpu" : " 17 GB", | 20260506 21:03:03.404 | INFO | "huge" : "", | 20260506 21:03:03.404 | INFO | "huge_shared" : " 27 GB", | 20260506 21:03:03.404 | INFO | "malloc" : " 14 GB", | 20260506 21:03:03.404 | INFO | "pinned" : " 29 GB" | 20260506 21:03:03.404 | INFO | } | 20260506 21:03:03.404 | INFO | Perf report for the last 5.13569 seconds: | 20260506 21:03:03.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:03:03.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:03.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:03.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:03.405 | INFO | ANALYSIS : 62.4%, 2.9, 94.9ms, 0 | 20260506 21:03:03.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:03.405 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260506 21:03:03.405 | INFO | PRE_HQ : 60.1%, 1.8, 57.3ms, 0 | 20260506 21:03:03.405 | INFO | BAZWRITER : 72.3%, 4.8, 177.2ms, 0 | 20260506 21:03:03.405 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 1.0ms, 0 | 20260506 21:03:03.405 | INFO | Uploaded 36GiB to GPU in 2.04573s (17.5976GiB/s) | 20260506 21:03:03.405 | INFO | Downloaded 38GiB to GPU in 2.19512s (17.3111GiB/s) | 20260506 21:03:08.546 | INFO | Analyzing chunk frames = [4432384,4432896) | 20260506 21:03:08.546 | INFO | MemInfo: { | 20260506 21:03:08.546 | INFO | "gpu" : " 17 GB", | 20260506 21:03:08.546 | INFO | "huge" : "", | 20260506 21:03:08.546 | INFO | "huge_shared" : " 27 GB", | 20260506 21:03:08.546 | INFO | "malloc" : " 14 GB", | 20260506 21:03:08.546 | INFO | "pinned" : " 29 GB" | 20260506 21:03:08.546 | INFO | } | 20260506 21:03:08.546 | INFO | Perf report for the last 5.14154 seconds: | 20260506 21:03:08.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:03:08.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:08.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:08.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:08.546 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:03:08.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:08.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:08.546 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:03:08.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:08.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:08.546 | INFO | Uploaded 36GiB to GPU in 2.01697s (17.8485GiB/s) | 20260506 21:03:08.546 | INFO | Downloaded 34GiB to GPU in 1.92362s (17.675GiB/s) | 20260506 21:03:13.679 | INFO | Analyzing chunk frames = [4432896,4433408) | 20260506 21:03:13.679 | INFO | MemInfo: { | 20260506 21:03:13.679 | INFO | "gpu" : " 17 GB", | 20260506 21:03:13.679 | INFO | "huge" : "", | 20260506 21:03:13.679 | INFO | "huge_shared" : " 27 GB", | 20260506 21:03:13.679 | INFO | "malloc" : " 14 GB", | 20260506 21:03:13.679 | INFO | "pinned" : " 29 GB" | 20260506 21:03:13.679 | INFO | } | 20260506 21:03:13.679 | INFO | Perf report for the last 5.13309 seconds: | 20260506 21:03:13.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:03:13.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:13.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:13.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:13.679 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 21:03:13.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:13.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:13.679 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 21:03:13.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:13.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:13.679 | INFO | Uploaded 36GiB to GPU in 1.99688s (18.0281GiB/s) | 20260506 21:03:13.679 | INFO | Downloaded 34GiB to GPU in 1.91842s (17.7229GiB/s) | 20260506 21:03:18.813 | INFO | Analyzing chunk frames = [4433408,4433920) | 20260506 21:03:18.813 | INFO | MemInfo: { | 20260506 21:03:18.813 | INFO | "gpu" : " 17 GB", | 20260506 21:03:18.813 | INFO | "huge" : "", | 20260506 21:03:18.813 | INFO | "huge_shared" : " 27 GB", | 20260506 21:03:18.813 | INFO | "malloc" : " 14 GB", | 20260506 21:03:18.813 | INFO | "pinned" : " 29 GB" | 20260506 21:03:18.813 | INFO | } | 20260506 21:03:18.813 | INFO | Perf report for the last 5.13421 seconds: | 20260506 21:03:18.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:03:18.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:18.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:18.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:18.813 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:03:18.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:18.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:18.813 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:03:18.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:18.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:18.814 | INFO | Uploaded 36GiB to GPU in 1.94673s (18.4925GiB/s) | 20260506 21:03:18.814 | INFO | Downloaded 34GiB to GPU in 1.86948s (18.1868GiB/s) | 20260506 21:03:23.961 | INFO | Analyzing chunk frames = [4433920,4434432) | 20260506 21:03:23.961 | INFO | MemInfo: { | 20260506 21:03:23.961 | INFO | "gpu" : " 17 GB", | 20260506 21:03:23.961 | INFO | "huge" : "", | 20260506 21:03:23.961 | INFO | "huge_shared" : " 27 GB", | 20260506 21:03:23.961 | INFO | "malloc" : " 14 GB", | 20260506 21:03:23.961 | INFO | "pinned" : " 29 GB" | 20260506 21:03:23.961 | INFO | } | 20260506 21:03:23.961 | INFO | Perf report for the last 5.14756 seconds: | 20260506 21:03:23.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:03:23.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:23.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:23.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:23.961 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 21:03:23.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:23.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:23.961 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:03:23.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:23.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:23.961 | INFO | Uploaded 36GiB to GPU in 2.02527s (17.7754GiB/s) | 20260506 21:03:23.961 | INFO | Downloaded 34GiB to GPU in 1.91477s (17.7567GiB/s) | 20260506 21:03:29.106 | INFO | Analyzing chunk frames = [4434432,4434944) | 20260506 21:03:29.106 | INFO | MemInfo: { | 20260506 21:03:29.106 | INFO | "gpu" : " 17 GB", | 20260506 21:03:29.106 | INFO | "huge" : "", | 20260506 21:03:29.106 | INFO | "huge_shared" : " 27 GB", | 20260506 21:03:29.106 | INFO | "malloc" : " 14 GB", | 20260506 21:03:29.106 | INFO | "pinned" : " 29 GB" | 20260506 21:03:29.106 | INFO | } | 20260506 21:03:29.106 | INFO | Perf report for the last 5.14519 seconds: | 20260506 21:03:29.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:03:29.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:29.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:29.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:29.106 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:03:29.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:29.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:29.106 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:03:29.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:29.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:29.106 | INFO | Uploaded 36GiB to GPU in 1.99667s (18.03GiB/s) | 20260506 21:03:29.106 | INFO | Downloaded 34GiB to GPU in 1.94515s (17.4794GiB/s) | 20260506 21:03:34.239 | INFO | Analyzing chunk frames = [4434944,4435456) | 20260506 21:03:34.239 | INFO | MemInfo: { | 20260506 21:03:34.239 | INFO | "gpu" : " 17 GB", | 20260506 21:03:34.239 | INFO | "huge" : "", | 20260506 21:03:34.239 | INFO | "huge_shared" : " 27 GB", | 20260506 21:03:34.239 | INFO | "malloc" : " 14 GB", | 20260506 21:03:34.239 | INFO | "pinned" : " 29 GB" | 20260506 21:03:34.239 | INFO | } | 20260506 21:03:34.240 | INFO | Perf report for the last 5.13343 seconds: | 20260506 21:03:34.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:03:34.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:34.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:34.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:34.240 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 21:03:34.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:34.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:34.240 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 21:03:34.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:34.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:34.240 | INFO | Uploaded 36GiB to GPU in 1.96473s (18.3232GiB/s) | 20260506 21:03:34.240 | INFO | Downloaded 34GiB to GPU in 1.99716s (17.0242GiB/s) | 20260506 21:03:39.378 | INFO | Analyzing chunk frames = [4435456,4435968) | 20260506 21:03:39.378 | INFO | MemInfo: { | 20260506 21:03:39.378 | INFO | "gpu" : " 17 GB", | 20260506 21:03:39.378 | INFO | "huge" : "", | 20260506 21:03:39.379 | INFO | "huge_shared" : " 27 GB", | 20260506 21:03:39.379 | INFO | "malloc" : " 14 GB", | 20260506 21:03:39.379 | INFO | "pinned" : " 29 GB" | 20260506 21:03:39.379 | INFO | } | 20260506 21:03:39.379 | INFO | Perf report for the last 5.13915 seconds: | 20260506 21:03:39.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:03:39.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:39.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:39.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:39.379 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:03:39.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:39.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:39.379 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:03:39.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:39.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:39.379 | INFO | Uploaded 36GiB to GPU in 1.98425s (18.1429GiB/s) | 20260506 21:03:39.379 | INFO | Downloaded 34GiB to GPU in 1.98299s (17.1458GiB/s) | 20260506 21:03:39.477 | INFO | Starting a round of Baz IO | 20260506 21:03:42.622 | INFO | Populating RTMetrics report | 20260506 21:03:42.622 | INFO | RTMetrics report populated | 20260506 21:03:42.622 | INFO | Opening tmp RTMetrics json file | 20260506 21:03:42.626 | INFO | Serializing RTMetrics to json | 20260506 21:03:42.647 | INFO | Serialization done, swapping tmp file now... | 20260506 21:03:42.647 | INFO | RTMetrics done | 20260506 21:03:42.754 | INFO | Finished a round of Baz IO: Wrote 1.958 GiB in 3.277 seconds | 20260506 21:03:42.755 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.05% / 0.00% / 65.95% | 20260506 21:03:42.755 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.66483% / 3.97364e-06% / 4.37101e-05% | 20260506 21:03:42.755 | INFO | Baz IO overall: | 20260506 21:03:42.755 | INFO | BytesWritten : 2339399736883 | 20260506 21:03:42.755 | INFO | NumEvents : 1129808137763 | 20260506 21:03:44.518 | INFO | Analyzing chunk frames = [4435968,4436480) | 20260506 21:03:44.518 | INFO | MemInfo: { | 20260506 21:03:44.518 | INFO | "gpu" : " 17 GB", | 20260506 21:03:44.518 | INFO | "huge" : "", | 20260506 21:03:44.518 | INFO | "huge_shared" : " 27 GB", | 20260506 21:03:44.518 | INFO | "malloc" : " 14 GB", | 20260506 21:03:44.518 | INFO | "pinned" : " 29 GB" | 20260506 21:03:44.518 | INFO | } | 20260506 21:03:44.518 | INFO | Perf report for the last 5.13965 seconds: | 20260506 21:03:44.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:03:44.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:44.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:44.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:44.518 | INFO | ANALYSIS : 62.5%, 2.9, 95.6ms, 0 | 20260506 21:03:44.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:44.518 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 21:03:44.518 | INFO | PRE_HQ : 59.9%, 1.8, 56.9ms, 0 | 20260506 21:03:44.518 | INFO | BAZWRITER : 63.5%, 4.6, 149.6ms, 0 | 20260506 21:03:44.519 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 21:03:44.519 | INFO | Uploaded 36GiB to GPU in 2.06978s (17.3932GiB/s) | 20260506 21:03:44.519 | INFO | Downloaded 38GiB to GPU in 2.20712s (17.217GiB/s) | 20260506 21:03:49.658 | INFO | Analyzing chunk frames = [4436480,4436992) | 20260506 21:03:49.658 | INFO | MemInfo: { | 20260506 21:03:49.658 | INFO | "gpu" : " 17 GB", | 20260506 21:03:49.658 | INFO | "huge" : "", | 20260506 21:03:49.658 | INFO | "huge_shared" : " 27 GB", | 20260506 21:03:49.658 | INFO | "malloc" : " 14 GB", | 20260506 21:03:49.658 | INFO | "pinned" : " 29 GB" | 20260506 21:03:49.658 | INFO | } | 20260506 21:03:49.658 | INFO | Perf report for the last 5.13977 seconds: | 20260506 21:03:49.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:03:49.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:49.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:49.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:49.658 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:03:49.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:49.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:49.658 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 21:03:49.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:49.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:49.658 | INFO | Uploaded 36GiB to GPU in 2.01974s (17.8241GiB/s) | 20260506 21:03:49.659 | INFO | Downloaded 34GiB to GPU in 1.9269s (17.6449GiB/s) | 20260506 21:03:54.793 | INFO | Analyzing chunk frames = [4436992,4437504) | 20260506 21:03:54.793 | INFO | MemInfo: { | 20260506 21:03:54.793 | INFO | "gpu" : " 17 GB", | 20260506 21:03:54.793 | INFO | "huge" : "", | 20260506 21:03:54.793 | INFO | "huge_shared" : " 27 GB", | 20260506 21:03:54.793 | INFO | "malloc" : " 14 GB", | 20260506 21:03:54.793 | INFO | "pinned" : " 29 GB" | 20260506 21:03:54.793 | INFO | } | 20260506 21:03:54.793 | INFO | Perf report for the last 5.13467 seconds: | 20260506 21:03:54.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:03:54.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:54.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:54.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:54.793 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260506 21:03:54.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:54.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:54.793 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:03:54.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:54.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:54.793 | INFO | Uploaded 36GiB to GPU in 1.99202s (18.0721GiB/s) | 20260506 21:03:54.793 | INFO | Downloaded 34GiB to GPU in 1.92201s (17.6898GiB/s) | 20260506 21:03:59.932 | INFO | Analyzing chunk frames = [4437504,4438016) | 20260506 21:03:59.932 | INFO | MemInfo: { | 20260506 21:03:59.932 | INFO | "gpu" : " 17 GB", | 20260506 21:03:59.932 | INFO | "huge" : "", | 20260506 21:03:59.932 | INFO | "huge_shared" : " 27 GB", | 20260506 21:03:59.932 | INFO | "malloc" : " 14 GB", | 20260506 21:03:59.932 | INFO | "pinned" : " 29 GB" | 20260506 21:03:59.932 | INFO | } | 20260506 21:03:59.932 | INFO | Perf report for the last 5.13931 seconds: | 20260506 21:03:59.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:03:59.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:59.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:59.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:59.932 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 21:03:59.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:59.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:59.932 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 21:03:59.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:59.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:03:59.932 | INFO | Uploaded 36GiB to GPU in 1.96585s (18.3127GiB/s) | 20260506 21:03:59.932 | INFO | Downloaded 34GiB to GPU in 1.87556s (18.1279GiB/s) | 20260506 21:04:05.078 | INFO | Analyzing chunk frames = [4438016,4438528) | 20260506 21:04:05.078 | INFO | MemInfo: { | 20260506 21:04:05.078 | INFO | "gpu" : " 17 GB", | 20260506 21:04:05.078 | INFO | "huge" : "", | 20260506 21:04:05.079 | INFO | "huge_shared" : " 27 GB", | 20260506 21:04:05.079 | INFO | "malloc" : " 14 GB", | 20260506 21:04:05.079 | INFO | "pinned" : " 29 GB" | 20260506 21:04:05.079 | INFO | } | 20260506 21:04:05.079 | INFO | Perf report for the last 5.14649 seconds: | 20260506 21:04:05.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:04:05.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:05.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:05.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:05.079 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260506 21:04:05.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:05.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:05.079 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:04:05.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:05.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:05.079 | INFO | Uploaded 36GiB to GPU in 2.00514s (17.9539GiB/s) | 20260506 21:04:05.079 | INFO | Downloaded 34GiB to GPU in 1.90768s (17.8227GiB/s) | 20260506 21:04:10.219 | INFO | Analyzing chunk frames = [4438528,4439040) | 20260506 21:04:10.219 | INFO | MemInfo: { | 20260506 21:04:10.219 | INFO | "gpu" : " 17 GB", | 20260506 21:04:10.219 | INFO | "huge" : "", | 20260506 21:04:10.219 | INFO | "huge_shared" : " 27 GB", | 20260506 21:04:10.219 | INFO | "malloc" : " 14 GB", | 20260506 21:04:10.219 | INFO | "pinned" : " 29 GB" | 20260506 21:04:10.219 | INFO | } | 20260506 21:04:10.219 | INFO | Perf report for the last 5.14033 seconds: | 20260506 21:04:10.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:04:10.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:10.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:10.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:10.219 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 21:04:10.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:10.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:10.219 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260506 21:04:10.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:10.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:10.219 | INFO | Uploaded 36GiB to GPU in 2.02524s (17.7757GiB/s) | 20260506 21:04:10.219 | INFO | Downloaded 34GiB to GPU in 1.94217s (17.5062GiB/s) | 20260506 21:04:15.363 | INFO | Analyzing chunk frames = [4439040,4439552) | 20260506 21:04:15.363 | INFO | MemInfo: { | 20260506 21:04:15.363 | INFO | "gpu" : " 17 GB", | 20260506 21:04:15.363 | INFO | "huge" : "", | 20260506 21:04:15.363 | INFO | "huge_shared" : " 27 GB", | 20260506 21:04:15.363 | INFO | "malloc" : " 14 GB", | 20260506 21:04:15.363 | INFO | "pinned" : " 29 GB" | 20260506 21:04:15.363 | INFO | } | 20260506 21:04:15.363 | INFO | Perf report for the last 5.14419 seconds: | 20260506 21:04:15.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:04:15.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:15.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:15.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:15.363 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 21:04:15.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:15.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:15.363 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 21:04:15.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:15.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:15.364 | INFO | Uploaded 36GiB to GPU in 2.00169s (17.9848GiB/s) | 20260506 21:04:15.364 | INFO | Downloaded 34GiB to GPU in 1.99752s (17.0211GiB/s) | 20260506 21:04:20.492 | INFO | Analyzing chunk frames = [4439552,4440064) | 20260506 21:04:20.492 | INFO | MemInfo: { | 20260506 21:04:20.492 | INFO | "gpu" : " 17 GB", | 20260506 21:04:20.492 | INFO | "huge" : "", | 20260506 21:04:20.493 | INFO | "huge_shared" : " 27 GB", | 20260506 21:04:20.493 | INFO | "malloc" : " 14 GB", | 20260506 21:04:20.493 | INFO | "pinned" : " 29 GB" | 20260506 21:04:20.493 | INFO | } | 20260506 21:04:20.493 | INFO | Perf report for the last 5.12944 seconds: | 20260506 21:04:20.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:04:20.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:20.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:20.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:20.493 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260506 21:04:20.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:20.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:20.493 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 21:04:20.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:20.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:20.493 | INFO | Uploaded 36GiB to GPU in 2.02553s (17.7731GiB/s) | 20260506 21:04:20.493 | INFO | Downloaded 34GiB to GPU in 1.9883s (17.1GiB/s) | 20260506 21:04:20.493 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:04:20.493 | INFO | Total mem usage: 18 GiB | 20260506 21:04:20.493 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:04:20.493 | INFO | Recent High Water Marks Sum: 16.6 GiB (Savings: -8.36%) | 20260506 21:04:20.493 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:04:20.493 | INFO | High Water Breakdown: | 20260506 21:04:20.493 | INFO | 62.5% : 10.4 GiB ( 90.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:04:20.493 | INFO | 37.4% : 6.21 GiB ( 92.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:04:20.493 | INFO | 0.12% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:04:20.493 | INFO | ------------------------------------------ | 20260506 21:04:20.493 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:04:20.493 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:04:20.493 | INFO | Total mem usage: 26.5 GiB | 20260506 21:04:20.493 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:04:20.493 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:04:20.493 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:04:20.493 | INFO | High Water Breakdown: | 20260506 21:04:20.493 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:04:20.493 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:04:20.493 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:04:20.493 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:04:20.493 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:04:20.493 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:04:20.493 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:04:20.493 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:04:20.493 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:04:20.493 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:04:20.493 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:04:20.493 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:04:20.493 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:04:20.493 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:04:20.493 | INFO | ------------------------------------------ | 20260506 21:04:20.493 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:04:20.493 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:04:20.493 | INFO | Total mem usage: 25.6 GiB | 20260506 21:04:20.493 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:04:20.493 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 21:04:20.494 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 21:04:20.494 | INFO | High Water Breakdown: | 20260506 21:04:20.494 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 21:04:20.494 | INFO | ------------------------------------------ | 20260506 21:04:20.494 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:04:20.494 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:04:20.494 | INFO | Total mem usage: 15.7 GiB | 20260506 21:04:20.494 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:04:20.494 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:04:20.494 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:04:20.494 | INFO | High Water Breakdown: | 20260506 21:04:20.494 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:04:20.494 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:04:20.494 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:04:20.494 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:04:20.494 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:04:20.494 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:04:20.494 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:04:20.494 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:04:20.494 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:04:20.494 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:04:20.494 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:04:20.494 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:04:20.494 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:04:20.494 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:04:20.494 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:04:20.494 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:04:20.494 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:04:20.494 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:04:20.494 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:04:20.494 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:04:20.494 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:04:20.494 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:04:20.494 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:04:20.494 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:04:20.494 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:04:20.494 | INFO | ------------------------------------------ | 20260506 21:04:20.494 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:04:20.494 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:04:20.494 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:04:20.494 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:04:20.494 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.61e+01, 1.34e+01 / 2.48e+00 | 20260506 21:04:20.494 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 9.89e-03 / 4.64e-03 | 20260506 21:04:20.494 | INFO | PerfReport: 11.63%, Baseline -- 3.62e+00 / 4.43e+00, 5.53e-01 / 3.47e-01 | 20260506 21:04:20.494 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.47e+00, 1.82e-01 / 1.28e-01 | 20260506 21:04:20.494 | INFO | PerfReport: 16.06%, DME -- 4.00e+01 / 4.50e+01, 4.31e+00 / 2.53e+00 | 20260506 21:04:20.494 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.42e+00 / 1.38e+00 | 20260506 21:04:20.494 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.21e+00 / 2.79e+00, 3.74e-01 / 2.31e-01 | 20260506 21:04:20.494 | INFO | PerfReport: 3.29%, Metrics -- 1.02e+00 / 1.36e+00, 2.70e-01 / 1.72e-01 | 20260506 21:04:20.494 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.51e-01 / 3.99e-01, 5.53e-02 / 2.88e-02 | 20260506 21:04:20.494 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:04:20.494 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:04:20.494 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:04:20.494 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 21:04:20.494 | INFO | PerfReport: 21.59%, Upload -- 2.09e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260506 21:04:20.494 | INFO | PerfReport: 21.21%, Download -- 2.05e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 21:04:20.494 | INFO | PerfReport: 32.20%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 21:04:20.494 | INFO | PerfReport: 1.38%, LockUpload -- 1.34e+00 / 0.00e+00, 5.04e+00 / 0.00e+00 | 20260506 21:04:20.494 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260506 21:04:20.494 | INFO | PerfReport: 22.51%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:04:20.494 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260506 21:04:20.608 | INFO | Starting a round of Baz IO | 20260506 21:04:23.740 | INFO | Populating RTMetrics report | 20260506 21:04:23.740 | INFO | RTMetrics report populated | 20260506 21:04:23.740 | INFO | Opening tmp RTMetrics json file | 20260506 21:04:23.853 | INFO | Serializing RTMetrics to json | 20260506 21:04:23.891 | INFO | Serialization done, swapping tmp file now... | 20260506 21:04:23.891 | INFO | RTMetrics done | 20260506 21:04:23.924 | INFO | Finished a round of Baz IO: Wrote 1.958 GiB in 3.316 seconds | 20260506 21:04:23.924 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.05% / 0.00% / 65.95% | 20260506 21:04:23.924 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.662895% / 1.19209e-05% / 5.16574e-05% | 20260506 21:04:23.924 | INFO | Baz IO overall: | 20260506 21:04:23.924 | INFO | BytesWritten : 2341502563658 | 20260506 21:04:23.924 | INFO | NumEvents : 1130815459583 | 20260506 21:04:25.629 | INFO | Analyzing chunk frames = [4440064,4440576) | 20260506 21:04:25.629 | INFO | MemInfo: { | 20260506 21:04:25.629 | INFO | "gpu" : " 17 GB", | 20260506 21:04:25.629 | INFO | "huge" : "", | 20260506 21:04:25.629 | INFO | "huge_shared" : " 27 GB", | 20260506 21:04:25.629 | INFO | "malloc" : " 14 GB", | 20260506 21:04:25.629 | INFO | "pinned" : " 29 GB" | 20260506 21:04:25.629 | INFO | } | 20260506 21:04:25.629 | INFO | Perf report for the last 5.13646 seconds: | 20260506 21:04:25.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:04:25.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:25.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:25.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:25.629 | INFO | ANALYSIS : 62.7%, 2.9, 95.6ms, 0 | 20260506 21:04:25.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:25.629 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260506 21:04:25.629 | INFO | PRE_HQ : 59.9%, 1.8, 57.2ms, 0 | 20260506 21:04:25.629 | INFO | BAZWRITER : 64.2%, 4.8, 158.0ms, 0 | 20260506 21:04:25.629 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 21:04:25.630 | INFO | Uploaded 36GiB to GPU in 2.06973s (17.3936GiB/s) | 20260506 21:04:25.630 | INFO | Downloaded 38GiB to GPU in 2.22427s (17.0843GiB/s) | 20260506 21:04:30.769 | INFO | Analyzing chunk frames = [4440576,4441088) | 20260506 21:04:30.769 | INFO | MemInfo: { | 20260506 21:04:30.769 | INFO | "gpu" : " 17 GB", | 20260506 21:04:30.769 | INFO | "huge" : "", | 20260506 21:04:30.769 | INFO | "huge_shared" : " 27 GB", | 20260506 21:04:30.769 | INFO | "malloc" : " 14 GB", | 20260506 21:04:30.769 | INFO | "pinned" : " 29 GB" | 20260506 21:04:30.769 | INFO | } | 20260506 21:04:30.769 | INFO | Perf report for the last 5.13977 seconds: | 20260506 21:04:30.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:04:30.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:30.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:30.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:30.769 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 21:04:30.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:30.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:30.769 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:04:30.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:30.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:30.769 | INFO | Uploaded 36GiB to GPU in 1.99918s (18.0074GiB/s) | 20260506 21:04:30.769 | INFO | Downloaded 34GiB to GPU in 1.93471s (17.5737GiB/s) | 20260506 21:04:35.907 | INFO | Analyzing chunk frames = [4441088,4441600) | 20260506 21:04:35.907 | INFO | MemInfo: { | 20260506 21:04:35.907 | INFO | "gpu" : " 17 GB", | 20260506 21:04:35.907 | INFO | "huge" : "", | 20260506 21:04:35.907 | INFO | "huge_shared" : " 27 GB", | 20260506 21:04:35.907 | INFO | "malloc" : " 14 GB", | 20260506 21:04:35.907 | INFO | "pinned" : " 29 GB" | 20260506 21:04:35.907 | INFO | } | 20260506 21:04:35.907 | INFO | Perf report for the last 5.13781 seconds: | 20260506 21:04:35.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:04:35.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:35.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:35.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:35.907 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 21:04:35.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:35.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:35.907 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:04:35.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:35.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:35.907 | INFO | Uploaded 36GiB to GPU in 1.94868s (18.4741GiB/s) | 20260506 21:04:35.907 | INFO | Downloaded 34GiB to GPU in 1.9293s (17.623GiB/s) | 20260506 21:04:41.050 | INFO | Analyzing chunk frames = [4441600,4442112) | 20260506 21:04:41.050 | INFO | MemInfo: { | 20260506 21:04:41.050 | INFO | "gpu" : " 17 GB", | 20260506 21:04:41.050 | INFO | "huge" : "", | 20260506 21:04:41.050 | INFO | "huge_shared" : " 27 GB", | 20260506 21:04:41.050 | INFO | "malloc" : " 14 GB", | 20260506 21:04:41.050 | INFO | "pinned" : " 29 GB" | 20260506 21:04:41.050 | INFO | } | 20260506 21:04:41.050 | INFO | Perf report for the last 5.14314 seconds: | 20260506 21:04:41.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:04:41.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:41.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:41.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:41.050 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 21:04:41.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:41.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:41.050 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:04:41.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:41.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:41.050 | INFO | Uploaded 36GiB to GPU in 1.90998s (18.8484GiB/s) | 20260506 21:04:41.050 | INFO | Downloaded 34GiB to GPU in 1.87159s (18.1663GiB/s) | 20260506 21:04:46.193 | INFO | Analyzing chunk frames = [4442112,4442624) | 20260506 21:04:46.193 | INFO | MemInfo: { | 20260506 21:04:46.193 | INFO | "gpu" : " 17 GB", | 20260506 21:04:46.193 | INFO | "huge" : "", | 20260506 21:04:46.193 | INFO | "huge_shared" : " 27 GB", | 20260506 21:04:46.193 | INFO | "malloc" : " 14 GB", | 20260506 21:04:46.193 | INFO | "pinned" : " 29 GB" | 20260506 21:04:46.193 | INFO | } | 20260506 21:04:46.193 | INFO | Perf report for the last 5.14335 seconds: | 20260506 21:04:46.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:04:46.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:46.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:46.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:46.193 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 21:04:46.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:46.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:46.193 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:04:46.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:46.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:46.194 | INFO | Uploaded 36GiB to GPU in 1.99451s (18.0496GiB/s) | 20260506 21:04:46.194 | INFO | Downloaded 34GiB to GPU in 1.91878s (17.7196GiB/s) | 20260506 21:04:51.332 | INFO | Analyzing chunk frames = [4442624,4443136) | 20260506 21:04:51.332 | INFO | MemInfo: { | 20260506 21:04:51.332 | INFO | "gpu" : " 17 GB", | 20260506 21:04:51.332 | INFO | "huge" : "", | 20260506 21:04:51.333 | INFO | "huge_shared" : " 27 GB", | 20260506 21:04:51.333 | INFO | "malloc" : " 14 GB", | 20260506 21:04:51.333 | INFO | "pinned" : " 29 GB" | 20260506 21:04:51.333 | INFO | } | 20260506 21:04:51.333 | INFO | Perf report for the last 5.13942 seconds: | 20260506 21:04:51.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:04:51.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:51.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:51.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:51.333 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 21:04:51.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:51.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:51.333 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 21:04:51.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:51.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:51.333 | INFO | Uploaded 36GiB to GPU in 2.00436s (17.9609GiB/s) | 20260506 21:04:51.333 | INFO | Downloaded 34GiB to GPU in 1.95248s (17.4137GiB/s) | 20260506 21:04:56.467 | INFO | Analyzing chunk frames = [4443136,4443648) | 20260506 21:04:56.467 | INFO | MemInfo: { | 20260506 21:04:56.467 | INFO | "gpu" : " 17 GB", | 20260506 21:04:56.467 | INFO | "huge" : "", | 20260506 21:04:56.467 | INFO | "huge_shared" : " 27 GB", | 20260506 21:04:56.467 | INFO | "malloc" : " 14 GB", | 20260506 21:04:56.467 | INFO | "pinned" : " 29 GB" | 20260506 21:04:56.467 | INFO | } | 20260506 21:04:56.467 | INFO | Perf report for the last 5.13442 seconds: | 20260506 21:04:56.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:04:56.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:56.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:56.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:56.467 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:04:56.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:56.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:56.467 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 21:04:56.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:56.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:04:56.468 | INFO | Uploaded 36GiB to GPU in 1.97708s (18.2087GiB/s) | 20260506 21:04:56.468 | INFO | Downloaded 34GiB to GPU in 2.00893s (16.9245GiB/s) | 20260506 21:05:01.605 | INFO | Analyzing chunk frames = [4443648,4444160) | 20260506 21:05:01.605 | INFO | MemInfo: { | 20260506 21:05:01.605 | INFO | "gpu" : " 17 GB", | 20260506 21:05:01.605 | INFO | "huge" : "", | 20260506 21:05:01.605 | INFO | "huge_shared" : " 27 GB", | 20260506 21:05:01.605 | INFO | "malloc" : " 14 GB", | 20260506 21:05:01.605 | INFO | "pinned" : " 29 GB" | 20260506 21:05:01.605 | INFO | } | 20260506 21:05:01.606 | INFO | Perf report for the last 5.13849 seconds: | 20260506 21:05:01.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:05:01.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:01.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:01.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:01.606 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 21:05:01.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:01.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:01.606 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 21:05:01.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:01.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:01.606 | INFO | Uploaded 36GiB to GPU in 2.03046s (17.7299GiB/s) | 20260506 21:05:01.606 | INFO | Downloaded 34GiB to GPU in 1.98939s (17.0907GiB/s) | 20260506 21:05:01.718 | INFO | Starting a round of Baz IO | 20260506 21:05:04.850 | INFO | Populating RTMetrics report | 20260506 21:05:04.850 | INFO | RTMetrics report populated | 20260506 21:05:04.850 | INFO | Opening tmp RTMetrics json file | 20260506 21:05:04.860 | INFO | Serializing RTMetrics to json | 20260506 21:05:04.873 | INFO | Serialization done, swapping tmp file now... | 20260506 21:05:04.887 | INFO | RTMetrics done | 20260506 21:05:05.224 | INFO | Finished a round of Baz IO: Wrote 1.956 GiB in 3.506 seconds | 20260506 21:05:05.225 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.04% / 0.00% / 65.95% | 20260506 21:05:05.225 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.659895% / 3.97364e-06% / 5.5631e-05% | 20260506 21:05:05.225 | INFO | Baz IO overall: | 20260506 21:05:05.225 | INFO | BytesWritten : 2343603036883 | 20260506 21:05:05.225 | INFO | NumEvents : 1131820865064 | 20260506 21:05:06.745 | INFO | Analyzing chunk frames = [4444160,4444672) | 20260506 21:05:06.745 | INFO | MemInfo: { | 20260506 21:05:06.745 | INFO | "gpu" : " 17 GB", | 20260506 21:05:06.745 | INFO | "huge" : "", | 20260506 21:05:06.745 | INFO | "huge_shared" : " 27 GB", | 20260506 21:05:06.745 | INFO | "malloc" : " 14 GB", | 20260506 21:05:06.745 | INFO | "pinned" : " 29 GB" | 20260506 21:05:06.745 | INFO | } | 20260506 21:05:06.745 | INFO | Perf report for the last 5.13912 seconds: | 20260506 21:05:06.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:05:06.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:06.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:06.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:06.745 | INFO | ANALYSIS : 62.6%, 2.9, 95.4ms, 0 | 20260506 21:05:06.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:06.745 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 21:05:06.745 | INFO | PRE_HQ : 60.1%, 1.9, 58.4ms, 0 | 20260506 21:05:06.745 | INFO | BAZWRITER : 67.8%, 4.7, 165.5ms, 0 | 20260506 21:05:06.745 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260506 21:05:06.745 | INFO | Uploaded 36GiB to GPU in 2.06831s (17.4055GiB/s) | 20260506 21:05:06.745 | INFO | Downloaded 38GiB to GPU in 2.21015s (17.1934GiB/s) | 20260506 21:05:11.881 | INFO | Analyzing chunk frames = [4444672,4445184) | 20260506 21:05:11.881 | INFO | MemInfo: { | 20260506 21:05:11.881 | INFO | "gpu" : " 17 GB", | 20260506 21:05:11.881 | INFO | "huge" : "", | 20260506 21:05:11.881 | INFO | "huge_shared" : " 27 GB", | 20260506 21:05:11.881 | INFO | "malloc" : " 14 GB", | 20260506 21:05:11.881 | INFO | "pinned" : " 29 GB" | 20260506 21:05:11.881 | INFO | } | 20260506 21:05:11.881 | INFO | Perf report for the last 5.13657 seconds: | 20260506 21:05:11.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:05:11.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:11.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:11.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:11.881 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:05:11.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:11.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:11.881 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:05:11.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:11.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:11.882 | INFO | Uploaded 36GiB to GPU in 2.02148s (17.8087GiB/s) | 20260506 21:05:11.882 | INFO | Downloaded 34GiB to GPU in 1.92466s (17.6654GiB/s) | 20260506 21:05:17.031 | INFO | Analyzing chunk frames = [4445184,4445696) | 20260506 21:05:17.031 | INFO | MemInfo: { | 20260506 21:05:17.031 | INFO | "gpu" : " 17 GB", | 20260506 21:05:17.031 | INFO | "huge" : "", | 20260506 21:05:17.031 | INFO | "huge_shared" : " 27 GB", | 20260506 21:05:17.031 | INFO | "malloc" : " 14 GB", | 20260506 21:05:17.031 | INFO | "pinned" : " 29 GB" | 20260506 21:05:17.031 | INFO | } | 20260506 21:05:17.031 | INFO | Perf report for the last 5.14948 seconds: | 20260506 21:05:17.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:05:17.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:17.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:17.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:17.031 | INFO | ANALYSIS : 60.6%, 3.0, 94.0ms, 0 | 20260506 21:05:17.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:17.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:17.031 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:05:17.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:17.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:17.031 | INFO | Uploaded 36GiB to GPU in 1.97106s (18.2643GiB/s) | 20260506 21:05:17.031 | INFO | Downloaded 34GiB to GPU in 1.918s (17.7268GiB/s) | 20260506 21:05:22.170 | INFO | Analyzing chunk frames = [4445696,4446208) | 20260506 21:05:22.170 | INFO | MemInfo: { | 20260506 21:05:22.170 | INFO | "gpu" : " 17 GB", | 20260506 21:05:22.171 | INFO | "huge" : "", | 20260506 21:05:22.171 | INFO | "huge_shared" : " 27 GB", | 20260506 21:05:22.171 | INFO | "malloc" : " 14 GB", | 20260506 21:05:22.171 | INFO | "pinned" : " 29 GB" | 20260506 21:05:22.171 | INFO | } | 20260506 21:05:22.171 | INFO | Perf report for the last 5.1402 seconds: | 20260506 21:05:22.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:05:22.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:22.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:22.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:22.171 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 21:05:22.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:22.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:22.171 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:05:22.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:22.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:22.171 | INFO | Uploaded 36GiB to GPU in 1.97349s (18.2418GiB/s) | 20260506 21:05:22.171 | INFO | Downloaded 34GiB to GPU in 1.87454s (18.1378GiB/s) | 20260506 21:05:27.307 | INFO | Analyzing chunk frames = [4446208,4446720) | 20260506 21:05:27.307 | INFO | MemInfo: { | 20260506 21:05:27.307 | INFO | "gpu" : " 17 GB", | 20260506 21:05:27.307 | INFO | "huge" : "", | 20260506 21:05:27.307 | INFO | "huge_shared" : " 27 GB", | 20260506 21:05:27.307 | INFO | "malloc" : " 14 GB", | 20260506 21:05:27.307 | INFO | "pinned" : " 29 GB" | 20260506 21:05:27.307 | INFO | } | 20260506 21:05:27.307 | INFO | Perf report for the last 5.13592 seconds: | 20260506 21:05:27.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:05:27.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:27.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:27.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:27.307 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 21:05:27.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:27.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:27.307 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:05:27.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:27.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:27.307 | INFO | Uploaded 36GiB to GPU in 2.00999s (17.9105GiB/s) | 20260506 21:05:27.307 | INFO | Downloaded 34GiB to GPU in 1.92031s (17.7055GiB/s) | 20260506 21:05:32.448 | INFO | Analyzing chunk frames = [4446720,4447232) | 20260506 21:05:32.448 | INFO | MemInfo: { | 20260506 21:05:32.448 | INFO | "gpu" : " 17 GB", | 20260506 21:05:32.448 | INFO | "huge" : "", | 20260506 21:05:32.448 | INFO | "huge_shared" : " 27 GB", | 20260506 21:05:32.448 | INFO | "malloc" : " 14 GB", | 20260506 21:05:32.448 | INFO | "pinned" : " 29 GB" | 20260506 21:05:32.448 | INFO | } | 20260506 21:05:32.448 | INFO | Perf report for the last 5.14144 seconds: | 20260506 21:05:32.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:05:32.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:32.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:32.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:32.449 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:05:32.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:32.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:32.449 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:05:32.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:32.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:32.449 | INFO | Uploaded 36GiB to GPU in 2.0107s (17.9043GiB/s) | 20260506 21:05:32.449 | INFO | Downloaded 34GiB to GPU in 1.94421s (17.4878GiB/s) | 20260506 21:05:37.583 | INFO | Analyzing chunk frames = [4447232,4447744) | 20260506 21:05:37.583 | INFO | MemInfo: { | 20260506 21:05:37.583 | INFO | "gpu" : " 17 GB", | 20260506 21:05:37.583 | INFO | "huge" : "", | 20260506 21:05:37.583 | INFO | "huge_shared" : " 27 GB", | 20260506 21:05:37.583 | INFO | "malloc" : " 14 GB", | 20260506 21:05:37.583 | INFO | "pinned" : " 29 GB" | 20260506 21:05:37.583 | INFO | } | 20260506 21:05:37.583 | INFO | Perf report for the last 5.13474 seconds: | 20260506 21:05:37.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:05:37.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:37.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:37.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:37.583 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 21:05:37.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:37.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:37.583 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:05:37.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:37.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:37.584 | INFO | Uploaded 36GiB to GPU in 1.97315s (18.245GiB/s) | 20260506 21:05:37.584 | INFO | Downloaded 34GiB to GPU in 1.99799s (17.0171GiB/s) | 20260506 21:05:42.722 | INFO | Analyzing chunk frames = [4447744,4448256) | 20260506 21:05:42.722 | INFO | MemInfo: { | 20260506 21:05:42.722 | INFO | "gpu" : " 17 GB", | 20260506 21:05:42.722 | INFO | "huge" : "", | 20260506 21:05:42.722 | INFO | "huge_shared" : " 27 GB", | 20260506 21:05:42.722 | INFO | "malloc" : " 14 GB", | 20260506 21:05:42.722 | INFO | "pinned" : " 29 GB" | 20260506 21:05:42.722 | INFO | } | 20260506 21:05:42.722 | INFO | Perf report for the last 5.13883 seconds: | 20260506 21:05:42.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:05:42.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:42.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:42.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:42.722 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 21:05:42.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:42.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:42.722 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 21:05:42.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:42.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:42.722 | INFO | Uploaded 36GiB to GPU in 2.0173s (17.8457GiB/s) | 20260506 21:05:42.722 | INFO | Downloaded 34GiB to GPU in 1.98314s (17.1445GiB/s) | 20260506 21:05:42.722 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:05:42.722 | INFO | Total mem usage: 18 GiB | 20260506 21:05:42.722 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:05:42.722 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.5%) | 20260506 21:05:42.722 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:05:42.722 | INFO | High Water Breakdown: | 20260506 21:05:42.722 | INFO | 61.3% : 9.48 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:05:42.723 | INFO | 38.5% : 5.95 GiB ( 89% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:05:42.723 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:05:42.723 | INFO | ------------------------------------------ | 20260506 21:05:42.723 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:05:42.723 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:05:42.723 | INFO | Total mem usage: 26.5 GiB | 20260506 21:05:42.723 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:05:42.723 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:05:42.723 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:05:42.723 | INFO | High Water Breakdown: | 20260506 21:05:42.723 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:05:42.723 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:05:42.723 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:05:42.723 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:05:42.723 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:05:42.723 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:05:42.723 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:05:42.723 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:05:42.723 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:05:42.723 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:05:42.723 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:05:42.723 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:05:42.723 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:05:42.723 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:05:42.723 | INFO | ------------------------------------------ | 20260506 21:05:42.723 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:05:42.723 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:05:42.723 | INFO | Total mem usage: 25.6 GiB | 20260506 21:05:42.723 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:05:42.723 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:05:42.723 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:05:42.723 | INFO | High Water Breakdown: | 20260506 21:05:42.723 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:05:42.723 | INFO | ------------------------------------------ | 20260506 21:05:42.723 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:05:42.723 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:05:42.723 | INFO | Total mem usage: 15.7 GiB | 20260506 21:05:42.723 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:05:42.723 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:05:42.723 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:05:42.723 | INFO | High Water Breakdown: | 20260506 21:05:42.723 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:05:42.723 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:05:42.723 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:05:42.723 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:05:42.723 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:05:42.723 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:05:42.723 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:05:42.723 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:05:42.723 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:05:42.723 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:05:42.723 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:05:42.723 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:05:42.723 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:05:42.723 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:05:42.723 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:05:42.723 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:05:42.723 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:05:42.723 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:05:42.723 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:05:42.723 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:05:42.723 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:05:42.723 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:05:42.723 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:05:42.723 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:05:42.723 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:05:42.723 | INFO | ------------------------------------------ | 20260506 21:05:42.723 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:05:42.724 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:05:42.724 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:05:42.724 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:05:42.724 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.59e+01, 1.35e+01 / 2.62e+00 | 20260506 21:05:42.724 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.01e-01, 1.03e-02 / 5.39e-03 | 20260506 21:05:42.724 | INFO | PerfReport: 11.64%, Baseline -- 3.63e+00 / 4.40e+00, 5.51e-01 / 3.25e-01 | 20260506 21:05:42.724 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.45e+00, 1.82e-01 / 1.36e-01 | 20260506 21:05:42.724 | INFO | PerfReport: 16.06%, DME -- 4.00e+01 / 4.48e+01, 4.29e+00 / 3.31e+00 | 20260506 21:05:42.724 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.41e+00 / 1.83e+00 | 20260506 21:05:42.724 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.22e+00 / 2.76e+00, 3.80e-01 / 2.25e-01 | 20260506 21:05:42.724 | INFO | PerfReport: 3.28%, Metrics -- 1.02e+00 / 1.36e+00, 2.68e-01 / 1.35e-01 | 20260506 21:05:42.724 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.51e-01 / 3.92e-01, 5.53e-02 / 2.02e-02 | 20260506 21:05:42.724 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:05:42.724 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:05:42.724 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:05:42.724 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 21:05:42.724 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 6.92e+00 / 0.00e+00 | 20260506 21:05:42.724 | INFO | PerfReport: 21.26%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 21:05:42.724 | INFO | PerfReport: 32.20%, Compute -- 3.12e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 21:05:42.724 | INFO | PerfReport: 1.30%, LockUpload -- 1.26e+00 / 0.00e+00, 5.09e+00 / 0.00e+00 | 20260506 21:05:42.724 | INFO | PerfReport: 1.21%, LockDownload -- 1.17e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260506 21:05:42.724 | INFO | PerfReport: 22.50%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:05:42.724 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260506 21:05:42.825 | INFO | Starting a round of Baz IO | 20260506 21:05:46.009 | INFO | Populating RTMetrics report | 20260506 21:05:46.010 | INFO | RTMetrics report populated | 20260506 21:05:46.010 | INFO | Opening tmp RTMetrics json file | 20260506 21:05:46.029 | INFO | Serializing RTMetrics to json | 20260506 21:05:46.030 | INFO | Serialization done, swapping tmp file now... | 20260506 21:05:46.031 | INFO | RTMetrics done | 20260506 21:05:46.175 | INFO | Finished a round of Baz IO: Wrote 1.956 GiB in 3.35 seconds | 20260506 21:05:46.175 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.04% / 0.00% / 65.96% | 20260506 21:05:46.175 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.658361% / 0% / 8.34465e-05% | 20260506 21:05:46.175 | INFO | Baz IO overall: | 20260506 21:05:46.175 | INFO | BytesWritten : 2345703549075 | 20260506 21:05:46.175 | INFO | NumEvents : 1132826446120 | 20260506 21:05:47.863 | INFO | Analyzing chunk frames = [4448256,4448768) | 20260506 21:05:47.863 | INFO | MemInfo: { | 20260506 21:05:47.863 | INFO | "gpu" : " 17 GB", | 20260506 21:05:47.863 | INFO | "huge" : "", | 20260506 21:05:47.863 | INFO | "huge_shared" : " 27 GB", | 20260506 21:05:47.863 | INFO | "malloc" : " 14 GB", | 20260506 21:05:47.863 | INFO | "pinned" : " 29 GB" | 20260506 21:05:47.863 | INFO | } | 20260506 21:05:47.863 | INFO | Perf report for the last 5.1414 seconds: | 20260506 21:05:47.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:05:47.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:47.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:47.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:47.864 | INFO | ANALYSIS : 63.0%, 2.8, 93.5ms, 0 | 20260506 21:05:47.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:47.864 | INFO | RT_METRICS : 47.8%, 1.0, 25.1ms, 0 | 20260506 21:05:47.864 | INFO | PRE_HQ : 61.3%, 2.0, 63.5ms, 0 | 20260506 21:05:47.864 | INFO | BAZWRITER : 64.8%, 4.8, 159.6ms, 0 | 20260506 21:05:47.864 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 21:05:47.864 | INFO | Uploaded 36GiB to GPU in 2.07025s (17.3892GiB/s) | 20260506 21:05:47.864 | INFO | Downloaded 38GiB to GPU in 2.17122s (17.5017GiB/s) | 20260506 21:05:52.996 | INFO | Analyzing chunk frames = [4448768,4449280) | 20260506 21:05:52.996 | INFO | MemInfo: { | 20260506 21:05:52.996 | INFO | "gpu" : " 17 GB", | 20260506 21:05:52.996 | INFO | "huge" : "", | 20260506 21:05:52.996 | INFO | "huge_shared" : " 27 GB", | 20260506 21:05:52.996 | INFO | "malloc" : " 14 GB", | 20260506 21:05:52.996 | INFO | "pinned" : " 29 GB" | 20260506 21:05:52.996 | INFO | } | 20260506 21:05:52.996 | INFO | Perf report for the last 5.13225 seconds: | 20260506 21:05:52.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:05:52.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:52.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:52.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:52.996 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:05:52.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:52.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:52.996 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 21:05:52.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:52.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:52.996 | INFO | Uploaded 36GiB to GPU in 1.99585s (18.0375GiB/s) | 20260506 21:05:52.996 | INFO | Downloaded 34GiB to GPU in 1.92933s (17.6227GiB/s) | 20260506 21:05:58.137 | INFO | Analyzing chunk frames = [4449280,4449792) | 20260506 21:05:58.137 | INFO | MemInfo: { | 20260506 21:05:58.137 | INFO | "gpu" : " 17 GB", | 20260506 21:05:58.137 | INFO | "huge" : "", | 20260506 21:05:58.137 | INFO | "huge_shared" : " 27 GB", | 20260506 21:05:58.137 | INFO | "malloc" : " 14 GB", | 20260506 21:05:58.137 | INFO | "pinned" : " 29 GB" | 20260506 21:05:58.137 | INFO | } | 20260506 21:05:58.137 | INFO | Perf report for the last 5.14138 seconds: | 20260506 21:05:58.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:05:58.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:58.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:58.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:58.137 | INFO | ANALYSIS : 60.6%, 3.0, 94.0ms, 0 | 20260506 21:05:58.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:58.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:58.137 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:05:58.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:58.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:05:58.137 | INFO | Uploaded 36GiB to GPU in 1.97832s (18.1973GiB/s) | 20260506 21:05:58.137 | INFO | Downloaded 34GiB to GPU in 1.92233s (17.6869GiB/s) | 20260506 21:06:03.282 | INFO | Analyzing chunk frames = [4449792,4450304) | 20260506 21:06:03.282 | INFO | MemInfo: { | 20260506 21:06:03.282 | INFO | "gpu" : " 17 GB", | 20260506 21:06:03.282 | INFO | "huge" : "", | 20260506 21:06:03.282 | INFO | "huge_shared" : " 27 GB", | 20260506 21:06:03.282 | INFO | "malloc" : " 14 GB", | 20260506 21:06:03.282 | INFO | "pinned" : " 29 GB" | 20260506 21:06:03.282 | INFO | } | 20260506 21:06:03.282 | INFO | Perf report for the last 5.14512 seconds: | 20260506 21:06:03.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:06:03.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:03.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:03.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:03.282 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260506 21:06:03.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:03.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:03.282 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:06:03.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:03.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:03.283 | INFO | Uploaded 36GiB to GPU in 1.91322s (18.8165GiB/s) | 20260506 21:06:03.283 | INFO | Downloaded 34GiB to GPU in 1.87383s (18.1447GiB/s) | 20260506 21:06:08.422 | INFO | Analyzing chunk frames = [4450304,4450816) | 20260506 21:06:08.422 | INFO | MemInfo: { | 20260506 21:06:08.422 | INFO | "gpu" : " 17 GB", | 20260506 21:06:08.422 | INFO | "huge" : "", | 20260506 21:06:08.422 | INFO | "huge_shared" : " 27 GB", | 20260506 21:06:08.422 | INFO | "malloc" : " 14 GB", | 20260506 21:06:08.422 | INFO | "pinned" : " 29 GB" | 20260506 21:06:08.422 | INFO | } | 20260506 21:06:08.422 | INFO | Perf report for the last 5.13979 seconds: | 20260506 21:06:08.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:06:08.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:08.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:08.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:08.422 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 21:06:08.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:08.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:08.422 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:06:08.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:08.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:08.422 | INFO | Uploaded 36GiB to GPU in 2.00729s (17.9346GiB/s) | 20260506 21:06:08.422 | INFO | Downloaded 34GiB to GPU in 1.91184s (17.784GiB/s) | 20260506 21:06:13.558 | INFO | Analyzing chunk frames = [4450816,4451328) | 20260506 21:06:13.558 | INFO | MemInfo: { | 20260506 21:06:13.558 | INFO | "gpu" : " 17 GB", | 20260506 21:06:13.558 | INFO | "huge" : "", | 20260506 21:06:13.558 | INFO | "huge_shared" : " 27 GB", | 20260506 21:06:13.558 | INFO | "malloc" : " 14 GB", | 20260506 21:06:13.558 | INFO | "pinned" : " 29 GB" | 20260506 21:06:13.558 | INFO | } | 20260506 21:06:13.558 | INFO | Perf report for the last 5.13619 seconds: | 20260506 21:06:13.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:06:13.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:13.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:13.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:13.558 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 21:06:13.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:13.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:13.558 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:06:13.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:13.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:13.558 | INFO | Uploaded 36GiB to GPU in 2.01479s (17.8679GiB/s) | 20260506 21:06:13.559 | INFO | Downloaded 34GiB to GPU in 1.9371s (17.552GiB/s) | 20260506 21:06:18.699 | INFO | Analyzing chunk frames = [4451328,4451840) | 20260506 21:06:18.699 | INFO | MemInfo: { | 20260506 21:06:18.699 | INFO | "gpu" : " 17 GB", | 20260506 21:06:18.699 | INFO | "huge" : "", | 20260506 21:06:18.699 | INFO | "huge_shared" : " 27 GB", | 20260506 21:06:18.699 | INFO | "malloc" : " 14 GB", | 20260506 21:06:18.699 | INFO | "pinned" : " 29 GB" | 20260506 21:06:18.699 | INFO | } | 20260506 21:06:18.699 | INFO | Perf report for the last 5.14088 seconds: | 20260506 21:06:18.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:06:18.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:18.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:18.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:18.699 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:06:18.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:18.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:18.699 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:06:18.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:18.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:18.700 | INFO | Uploaded 36GiB to GPU in 1.96584s (18.3127GiB/s) | 20260506 21:06:18.700 | INFO | Downloaded 34GiB to GPU in 2.00349s (16.9704GiB/s) | 20260506 21:06:23.835 | INFO | Analyzing chunk frames = [4451840,4452352) | 20260506 21:06:23.835 | INFO | MemInfo: { | 20260506 21:06:23.835 | INFO | "gpu" : " 17 GB", | 20260506 21:06:23.835 | INFO | "huge" : "", | 20260506 21:06:23.835 | INFO | "huge_shared" : " 27 GB", | 20260506 21:06:23.835 | INFO | "malloc" : " 14 GB", | 20260506 21:06:23.835 | INFO | "pinned" : " 29 GB" | 20260506 21:06:23.835 | INFO | } | 20260506 21:06:23.835 | INFO | Perf report for the last 5.13618 seconds: | 20260506 21:06:23.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:06:23.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:23.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:23.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:23.835 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 21:06:23.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:23.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:23.835 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:06:23.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:23.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:23.836 | INFO | Uploaded 36GiB to GPU in 1.97793s (18.2009GiB/s) | 20260506 21:06:23.836 | INFO | Downloaded 34GiB to GPU in 1.97465s (17.2182GiB/s) | 20260506 21:06:23.933 | INFO | Starting a round of Baz IO | 20260506 21:06:27.069 | INFO | Populating RTMetrics report | 20260506 21:06:27.070 | INFO | RTMetrics report populated | 20260506 21:06:27.070 | INFO | Opening tmp RTMetrics json file | 20260506 21:06:27.070 | INFO | Serializing RTMetrics to json | 20260506 21:06:27.072 | INFO | Serialization done, swapping tmp file now... | 20260506 21:06:27.073 | INFO | RTMetrics done | 20260506 21:06:27.295 | INFO | Finished a round of Baz IO: Wrote 1.957 GiB in 3.362 seconds | 20260506 21:06:27.295 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.04% / 0.00% / 65.96% | 20260506 21:06:27.295 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.659736% / 3.97364e-06% / 7.54992e-05% | 20260506 21:06:27.295 | INFO | Baz IO overall: | 20260506 21:06:27.295 | INFO | BytesWritten : 2347805243301 | 20260506 21:06:27.295 | INFO | NumEvents : 1133833199108 | 20260506 21:06:28.976 | INFO | Analyzing chunk frames = [4452352,4452864) | 20260506 21:06:28.976 | INFO | MemInfo: { | 20260506 21:06:28.976 | INFO | "gpu" : " 17 GB", | 20260506 21:06:28.976 | INFO | "huge" : "", | 20260506 21:06:28.976 | INFO | "huge_shared" : " 27 GB", | 20260506 21:06:28.976 | INFO | "malloc" : " 14 GB", | 20260506 21:06:28.976 | INFO | "pinned" : " 29 GB" | 20260506 21:06:28.976 | INFO | } | 20260506 21:06:28.976 | INFO | Perf report for the last 5.14041 seconds: | 20260506 21:06:28.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:06:28.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:28.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:28.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:28.976 | INFO | ANALYSIS : 62.3%, 2.9, 95.4ms, 0 | 20260506 21:06:28.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:28.976 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 21:06:28.976 | INFO | PRE_HQ : 59.6%, 1.8, 56.7ms, 0 | 20260506 21:06:28.976 | INFO | BAZWRITER : 65.0%, 4.8, 159.5ms, 0 | 20260506 21:06:28.976 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260506 21:06:28.976 | INFO | Uploaded 36GiB to GPU in 2.01073s (17.904GiB/s) | 20260506 21:06:28.976 | INFO | Downloaded 38GiB to GPU in 2.20079s (17.2665GiB/s) | 20260506 21:06:34.119 | INFO | Analyzing chunk frames = [4452864,4453376) | 20260506 21:06:34.119 | INFO | MemInfo: { | 20260506 21:06:34.119 | INFO | "gpu" : " 17 GB", | 20260506 21:06:34.119 | INFO | "huge" : "", | 20260506 21:06:34.119 | INFO | "huge_shared" : " 27 GB", | 20260506 21:06:34.119 | INFO | "malloc" : " 14 GB", | 20260506 21:06:34.119 | INFO | "pinned" : " 29 GB" | 20260506 21:06:34.119 | INFO | } | 20260506 21:06:34.120 | INFO | Perf report for the last 5.14377 seconds: | 20260506 21:06:34.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:06:34.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:34.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:34.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:34.120 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 21:06:34.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:34.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:34.120 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:06:34.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:34.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:34.120 | INFO | Uploaded 36GiB to GPU in 2.00284s (17.9745GiB/s) | 20260506 21:06:34.120 | INFO | Downloaded 34GiB to GPU in 1.92687s (17.6452GiB/s) | 20260506 21:06:39.255 | INFO | Analyzing chunk frames = [4453376,4453888) | 20260506 21:06:39.255 | INFO | MemInfo: { | 20260506 21:06:39.255 | INFO | "gpu" : " 17 GB", | 20260506 21:06:39.255 | INFO | "huge" : "", | 20260506 21:06:39.255 | INFO | "huge_shared" : " 27 GB", | 20260506 21:06:39.255 | INFO | "malloc" : " 14 GB", | 20260506 21:06:39.255 | INFO | "pinned" : " 29 GB" | 20260506 21:06:39.255 | INFO | } | 20260506 21:06:39.255 | INFO | Perf report for the last 5.1356 seconds: | 20260506 21:06:39.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:06:39.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:39.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:39.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:39.255 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 21:06:39.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:39.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:39.255 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:06:39.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:39.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:39.255 | INFO | Uploaded 36GiB to GPU in 1.96429s (18.3272GiB/s) | 20260506 21:06:39.255 | INFO | Downloaded 34GiB to GPU in 1.92003s (17.7081GiB/s) | 20260506 21:06:44.399 | INFO | Analyzing chunk frames = [4453888,4454400) | 20260506 21:06:44.399 | INFO | MemInfo: { | 20260506 21:06:44.399 | INFO | "gpu" : " 17 GB", | 20260506 21:06:44.399 | INFO | "huge" : "", | 20260506 21:06:44.399 | INFO | "huge_shared" : " 27 GB", | 20260506 21:06:44.399 | INFO | "malloc" : " 14 GB", | 20260506 21:06:44.399 | INFO | "pinned" : " 29 GB" | 20260506 21:06:44.399 | INFO | } | 20260506 21:06:44.399 | INFO | Perf report for the last 5.14389 seconds: | 20260506 21:06:44.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:06:44.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:44.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:44.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:44.399 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 21:06:44.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:44.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:44.399 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:06:44.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:44.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:44.399 | INFO | Uploaded 36GiB to GPU in 1.93671s (18.5882GiB/s) | 20260506 21:06:44.399 | INFO | Downloaded 34GiB to GPU in 1.87214s (18.1611GiB/s) | 20260506 21:06:49.538 | INFO | Analyzing chunk frames = [4454400,4454912) | 20260506 21:06:49.538 | INFO | MemInfo: { | 20260506 21:06:49.538 | INFO | "gpu" : " 17 GB", | 20260506 21:06:49.538 | INFO | "huge" : "", | 20260506 21:06:49.538 | INFO | "huge_shared" : " 27 GB", | 20260506 21:06:49.538 | INFO | "malloc" : " 14 GB", | 20260506 21:06:49.538 | INFO | "pinned" : " 29 GB" | 20260506 21:06:49.538 | INFO | } | 20260506 21:06:49.538 | INFO | Perf report for the last 5.13907 seconds: | 20260506 21:06:49.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:06:49.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:49.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:49.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:49.538 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 21:06:49.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:49.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:49.538 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:06:49.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:49.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:49.538 | INFO | Uploaded 36GiB to GPU in 1.9831s (18.1534GiB/s) | 20260506 21:06:49.538 | INFO | Downloaded 34GiB to GPU in 1.92996s (17.6169GiB/s) | 20260506 21:06:54.673 | INFO | Analyzing chunk frames = [4454912,4455424) | 20260506 21:06:54.673 | INFO | MemInfo: { | 20260506 21:06:54.673 | INFO | "gpu" : " 17 GB", | 20260506 21:06:54.673 | INFO | "huge" : "", | 20260506 21:06:54.673 | INFO | "huge_shared" : " 27 GB", | 20260506 21:06:54.673 | INFO | "malloc" : " 14 GB", | 20260506 21:06:54.673 | INFO | "pinned" : " 29 GB" | 20260506 21:06:54.674 | INFO | } | 20260506 21:06:54.674 | INFO | Perf report for the last 5.13556 seconds: | 20260506 21:06:54.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:06:54.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:54.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:54.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:54.674 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 21:06:54.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:54.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:54.674 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 21:06:54.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:54.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:54.674 | INFO | Uploaded 36GiB to GPU in 2.01892s (17.8313GiB/s) | 20260506 21:06:54.674 | INFO | Downloaded 34GiB to GPU in 1.94717s (17.4612GiB/s) | 20260506 21:06:59.815 | INFO | Analyzing chunk frames = [4455424,4455936) | 20260506 21:06:59.815 | INFO | MemInfo: { | 20260506 21:06:59.815 | INFO | "gpu" : " 17 GB", | 20260506 21:06:59.815 | INFO | "huge" : "", | 20260506 21:06:59.815 | INFO | "huge_shared" : " 27 GB", | 20260506 21:06:59.815 | INFO | "malloc" : " 14 GB", | 20260506 21:06:59.815 | INFO | "pinned" : " 29 GB" | 20260506 21:06:59.815 | INFO | } | 20260506 21:06:59.815 | INFO | Perf report for the last 5.14092 seconds: | 20260506 21:06:59.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:06:59.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:59.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:59.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:59.815 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 21:06:59.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:59.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:59.815 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:06:59.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:59.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:06:59.815 | INFO | Uploaded 36GiB to GPU in 1.98345s (18.1502GiB/s) | 20260506 21:06:59.815 | INFO | Downloaded 34GiB to GPU in 2.01118s (16.9055GiB/s) | 20260506 21:07:04.952 | INFO | Analyzing chunk frames = [4455936,4456448) | 20260506 21:07:04.952 | INFO | MemInfo: { | 20260506 21:07:04.952 | INFO | "gpu" : " 17 GB", | 20260506 21:07:04.952 | INFO | "huge" : "", | 20260506 21:07:04.952 | INFO | "huge_shared" : " 27 GB", | 20260506 21:07:04.952 | INFO | "malloc" : " 14 GB", | 20260506 21:07:04.952 | INFO | "pinned" : " 29 GB" | 20260506 21:07:04.952 | INFO | } | 20260506 21:07:04.952 | INFO | Perf report for the last 5.13763 seconds: | 20260506 21:07:04.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:07:04.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:04.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:04.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:04.952 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:07:04.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:04.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:04.952 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:07:04.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:04.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:04.953 | INFO | Uploaded 36GiB to GPU in 1.98974s (18.0929GiB/s) | 20260506 21:07:04.953 | INFO | Downloaded 34GiB to GPU in 1.97942s (17.1768GiB/s) | 20260506 21:07:04.953 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:07:04.953 | INFO | Total mem usage: 18 GiB | 20260506 21:07:04.953 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:07:04.953 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -14.1%) | 20260506 21:07:04.953 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:07:04.953 | INFO | High Water Breakdown: | 20260506 21:07:04.953 | INFO | 61.7% : 9.73 GiB ( 85.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:07:04.953 | INFO | 38.2% : 6.03 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:07:04.953 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:07:04.953 | INFO | ------------------------------------------ | 20260506 21:07:04.953 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:07:04.953 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:07:04.953 | INFO | Total mem usage: 26.5 GiB | 20260506 21:07:04.953 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:07:04.953 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:07:04.953 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:07:04.953 | INFO | High Water Breakdown: | 20260506 21:07:04.953 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:07:04.953 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:07:04.953 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:07:04.953 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:07:04.953 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:07:04.953 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:07:04.953 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:07:04.953 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:07:04.953 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:07:04.953 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:07:04.953 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:07:04.953 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:07:04.953 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:07:04.953 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:07:04.953 | INFO | ------------------------------------------ | 20260506 21:07:04.953 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:07:04.953 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:07:04.953 | INFO | Total mem usage: 25.6 GiB | 20260506 21:07:04.953 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:07:04.953 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 21:07:04.953 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 21:07:04.953 | INFO | High Water Breakdown: | 20260506 21:07:04.953 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 21:07:04.953 | INFO | ------------------------------------------ | 20260506 21:07:04.953 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:07:04.953 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:07:04.953 | INFO | Total mem usage: 15.7 GiB | 20260506 21:07:04.953 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:07:04.953 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:07:04.953 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:07:04.953 | INFO | High Water Breakdown: | 20260506 21:07:04.953 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:07:04.953 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:07:04.953 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:07:04.953 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:07:04.953 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:07:04.953 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:07:04.953 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:07:04.953 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:07:04.953 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:07:04.953 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:07:04.953 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:07:04.953 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:07:04.954 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:07:04.954 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:07:04.954 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:07:04.954 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:07:04.954 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:07:04.954 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:07:04.954 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:07:04.954 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:07:04.954 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:07:04.954 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:07:04.954 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:07:04.954 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:07:04.954 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:07:04.954 | INFO | ------------------------------------------ | 20260506 21:07:04.954 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:07:04.954 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:07:04.954 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:07:04.954 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:07:04.954 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.58e+01, 1.36e+01 / 2.46e+00 | 20260506 21:07:04.954 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 9.85e-03 / 7.10e-03 | 20260506 21:07:04.954 | INFO | PerfReport: 11.65%, Baseline -- 3.63e+00 / 4.38e+00, 5.57e-01 / 3.30e-01 | 20260506 21:07:04.954 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.82e-01 / 1.40e-01 | 20260506 21:07:04.954 | INFO | PerfReport: 16.06%, DME -- 4.00e+01 / 4.50e+01, 4.34e+00 / 2.96e+00 | 20260506 21:07:04.954 | INFO | PerfReport: 57.20%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.40e+00 / 1.85e+00 | 20260506 21:07:04.954 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.22e+00 / 2.74e+00, 3.76e-01 / 2.31e-01 | 20260506 21:07:04.954 | INFO | PerfReport: 3.28%, Metrics -- 1.02e+00 / 1.31e+00, 2.70e-01 / 1.72e-01 | 20260506 21:07:04.954 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.51e-01 / 4.02e-01, 5.52e-02 / 2.71e-02 | 20260506 21:07:04.954 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:07:04.954 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:07:04.954 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:07:04.954 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 21:07:04.954 | INFO | PerfReport: 21.46%, Upload -- 2.07e+01 / 0.00e+00, 6.80e+00 / 0.00e+00 | 20260506 21:07:04.954 | INFO | PerfReport: 21.26%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 21:07:04.954 | INFO | PerfReport: 32.29%, Compute -- 3.12e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 21:07:04.954 | INFO | PerfReport: 1.23%, LockUpload -- 1.19e+00 / 0.00e+00, 4.63e+00 / 0.00e+00 | 20260506 21:07:04.954 | INFO | PerfReport: 1.25%, LockDownload -- 1.20e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260506 21:07:04.954 | INFO | PerfReport: 22.51%, LockCompute -- 2.17e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 21:07:04.954 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.22e-02 / 0.00e+00 | 20260506 21:07:05.050 | INFO | Starting a round of Baz IO | 20260506 21:07:08.189 | INFO | Populating RTMetrics report | 20260506 21:07:08.190 | INFO | RTMetrics report populated | 20260506 21:07:08.190 | INFO | Opening tmp RTMetrics json file | 20260506 21:07:08.195 | INFO | Serializing RTMetrics to json | 20260506 21:07:08.198 | INFO | Serialization done, swapping tmp file now... | 20260506 21:07:08.198 | INFO | RTMetrics done | 20260506 21:07:08.379 | INFO | Finished a round of Baz IO: Wrote 1.956 GiB in 3.328 seconds | 20260506 21:07:08.379 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.03% / 0.00% / 65.96% | 20260506 21:07:08.379 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.657248% / 7.94729e-06% / 6.75519e-05% | 20260506 21:07:08.379 | INFO | Baz IO overall: | 20260506 21:07:08.379 | INFO | BytesWritten : 2349905771075 | 20260506 21:07:08.379 | INFO | NumEvents : 1134838769396 | 20260506 21:07:10.088 | INFO | Analyzing chunk frames = [4456448,4456960) | 20260506 21:07:10.088 | INFO | MemInfo: { | 20260506 21:07:10.088 | INFO | "gpu" : " 17 GB", | 20260506 21:07:10.088 | INFO | "huge" : "", | 20260506 21:07:10.088 | INFO | "huge_shared" : " 27 GB", | 20260506 21:07:10.088 | INFO | "malloc" : " 14 GB", | 20260506 21:07:10.088 | INFO | "pinned" : " 29 GB" | 20260506 21:07:10.088 | INFO | } | 20260506 21:07:10.088 | INFO | Perf report for the last 5.13603 seconds: | 20260506 21:07:10.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:07:10.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:10.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:10.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:10.088 | INFO | ANALYSIS : 62.5%, 2.9, 95.5ms, 0 | 20260506 21:07:10.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:10.089 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 21:07:10.089 | INFO | PRE_HQ : 59.8%, 1.8, 56.6ms, 0 | 20260506 21:07:10.089 | INFO | BAZWRITER : 64.5%, 4.8, 157.5ms, 0 | 20260506 21:07:10.089 | INFO | BAZ_SAMPLING : 2.8%, 1.0, 1.4ms, 0 | 20260506 21:07:10.089 | INFO | Uploaded 36GiB to GPU in 2.05198s (17.5441GiB/s) | 20260506 21:07:10.089 | INFO | Downloaded 38GiB to GPU in 2.20138s (17.2619GiB/s) | 20260506 21:07:15.226 | INFO | Analyzing chunk frames = [4456960,4457472) | 20260506 21:07:15.226 | INFO | MemInfo: { | 20260506 21:07:15.226 | INFO | "gpu" : " 17 GB", | 20260506 21:07:15.226 | INFO | "huge" : "", | 20260506 21:07:15.226 | INFO | "huge_shared" : " 27 GB", | 20260506 21:07:15.226 | INFO | "malloc" : " 14 GB", | 20260506 21:07:15.226 | INFO | "pinned" : " 29 GB" | 20260506 21:07:15.226 | INFO | } | 20260506 21:07:15.226 | INFO | Perf report for the last 5.13795 seconds: | 20260506 21:07:15.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:07:15.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:15.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:15.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:15.226 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 21:07:15.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:15.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:15.226 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:07:15.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:15.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:15.227 | INFO | Uploaded 36GiB to GPU in 2.0055s (17.9507GiB/s) | 20260506 21:07:15.227 | INFO | Downloaded 34GiB to GPU in 1.92188s (17.691GiB/s) | 20260506 21:07:20.371 | INFO | Analyzing chunk frames = [4457472,4457984) | 20260506 21:07:20.371 | INFO | MemInfo: { | 20260506 21:07:20.371 | INFO | "gpu" : " 17 GB", | 20260506 21:07:20.371 | INFO | "huge" : "", | 20260506 21:07:20.371 | INFO | "huge_shared" : " 27 GB", | 20260506 21:07:20.371 | INFO | "malloc" : " 14 GB", | 20260506 21:07:20.371 | INFO | "pinned" : " 29 GB" | 20260506 21:07:20.371 | INFO | } | 20260506 21:07:20.371 | INFO | Perf report for the last 5.14505 seconds: | 20260506 21:07:20.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:07:20.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:20.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:20.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:20.371 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260506 21:07:20.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:20.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:20.371 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:07:20.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:20.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:20.372 | INFO | Uploaded 36GiB to GPU in 1.97991s (18.1827GiB/s) | 20260506 21:07:20.372 | INFO | Downloaded 34GiB to GPU in 1.91586s (17.7466GiB/s) | 20260506 21:07:25.514 | INFO | Analyzing chunk frames = [4457984,4458496) | 20260506 21:07:25.514 | INFO | MemInfo: { | 20260506 21:07:25.514 | INFO | "gpu" : " 17 GB", | 20260506 21:07:25.514 | INFO | "huge" : "", | 20260506 21:07:25.514 | INFO | "huge_shared" : " 27 GB", | 20260506 21:07:25.514 | INFO | "malloc" : " 14 GB", | 20260506 21:07:25.514 | INFO | "pinned" : " 29 GB" | 20260506 21:07:25.514 | INFO | } | 20260506 21:07:25.514 | INFO | Perf report for the last 5.14271 seconds: | 20260506 21:07:25.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:07:25.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:25.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:25.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:25.514 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:07:25.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:25.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:25.514 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:07:25.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:25.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:25.514 | INFO | Uploaded 36GiB to GPU in 1.93396s (18.6146GiB/s) | 20260506 21:07:25.514 | INFO | Downloaded 34GiB to GPU in 1.87056s (18.1764GiB/s) | 20260506 21:07:30.653 | INFO | Analyzing chunk frames = [4458496,4459008) | 20260506 21:07:30.653 | INFO | MemInfo: { | 20260506 21:07:30.653 | INFO | "gpu" : " 17 GB", | 20260506 21:07:30.653 | INFO | "huge" : "", | 20260506 21:07:30.653 | INFO | "huge_shared" : " 27 GB", | 20260506 21:07:30.653 | INFO | "malloc" : " 14 GB", | 20260506 21:07:30.653 | INFO | "pinned" : " 29 GB" | 20260506 21:07:30.653 | INFO | } | 20260506 21:07:30.653 | INFO | Perf report for the last 5.13883 seconds: | 20260506 21:07:30.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:07:30.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:30.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:30.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:30.653 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 21:07:30.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:30.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:30.653 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:07:30.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:30.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:30.653 | INFO | Uploaded 36GiB to GPU in 2.00137s (17.9877GiB/s) | 20260506 21:07:30.653 | INFO | Downloaded 34GiB to GPU in 1.93336s (17.586GiB/s) | 20260506 21:07:35.794 | INFO | Analyzing chunk frames = [4459008,4459520) | 20260506 21:07:35.794 | INFO | MemInfo: { | 20260506 21:07:35.794 | INFO | "gpu" : " 17 GB", | 20260506 21:07:35.794 | INFO | "huge" : "", | 20260506 21:07:35.794 | INFO | "huge_shared" : " 27 GB", | 20260506 21:07:35.794 | INFO | "malloc" : " 14 GB", | 20260506 21:07:35.794 | INFO | "pinned" : " 29 GB" | 20260506 21:07:35.794 | INFO | } | 20260506 21:07:35.794 | INFO | Perf report for the last 5.14105 seconds: | 20260506 21:07:35.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:07:35.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:35.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:35.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:35.794 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 21:07:35.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:35.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:35.794 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 21:07:35.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:35.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:35.794 | INFO | Uploaded 36GiB to GPU in 2.05507s (17.5176GiB/s) | 20260506 21:07:35.794 | INFO | Downloaded 34GiB to GPU in 1.9526s (17.4127GiB/s) | 20260506 21:07:40.930 | INFO | Analyzing chunk frames = [4459520,4460032) | 20260506 21:07:40.930 | INFO | MemInfo: { | 20260506 21:07:40.930 | INFO | "gpu" : " 17 GB", | 20260506 21:07:40.930 | INFO | "huge" : "", | 20260506 21:07:40.930 | INFO | "huge_shared" : " 27 GB", | 20260506 21:07:40.930 | INFO | "malloc" : " 14 GB", | 20260506 21:07:40.930 | INFO | "pinned" : " 29 GB" | 20260506 21:07:40.930 | INFO | } | 20260506 21:07:40.930 | INFO | Perf report for the last 5.13568 seconds: | 20260506 21:07:40.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:07:40.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:40.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:40.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:40.930 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 21:07:40.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:40.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:40.930 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:07:40.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:40.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:40.930 | INFO | Uploaded 36GiB to GPU in 1.97141s (18.2611GiB/s) | 20260506 21:07:40.930 | INFO | Downloaded 34GiB to GPU in 2.00832s (16.9296GiB/s) | 20260506 21:07:46.065 | INFO | Analyzing chunk frames = [4460032,4460544) | 20260506 21:07:46.065 | INFO | MemInfo: { | 20260506 21:07:46.065 | INFO | "gpu" : " 17 GB", | 20260506 21:07:46.066 | INFO | "huge" : "", | 20260506 21:07:46.066 | INFO | "huge_shared" : " 27 GB", | 20260506 21:07:46.066 | INFO | "malloc" : " 14 GB", | 20260506 21:07:46.066 | INFO | "pinned" : " 29 GB" | 20260506 21:07:46.066 | INFO | } | 20260506 21:07:46.066 | INFO | Perf report for the last 5.1361 seconds: | 20260506 21:07:46.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:07:46.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:46.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:46.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:46.066 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:07:46.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:46.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:46.066 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 21:07:46.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:46.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:46.066 | INFO | Uploaded 36GiB to GPU in 1.97235s (18.2523GiB/s) | 20260506 21:07:46.066 | INFO | Downloaded 34GiB to GPU in 1.97921s (17.1786GiB/s) | 20260506 21:07:46.164 | INFO | Starting a round of Baz IO | 20260506 21:07:49.304 | INFO | Populating RTMetrics report | 20260506 21:07:49.304 | INFO | RTMetrics report populated | 20260506 21:07:49.304 | INFO | Opening tmp RTMetrics json file | 20260506 21:07:49.321 | INFO | Serializing RTMetrics to json | 20260506 21:07:49.324 | INFO | Serialization done, swapping tmp file now... | 20260506 21:07:49.333 | INFO | RTMetrics done | 20260506 21:07:49.639 | INFO | Finished a round of Baz IO: Wrote 1.957 GiB in 3.475 seconds | 20260506 21:07:49.639 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.03% / 0.00% / 65.97% | 20260506 21:07:49.639 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.657276% / 1.19209e-05% / 5.16574e-05% | 20260506 21:07:49.639 | INFO | Baz IO overall: | 20260506 21:07:49.639 | INFO | BytesWritten : 2352007619832 | 20260506 21:07:49.639 | INFO | NumEvents : 1135845520636 | 20260506 21:07:51.204 | INFO | Analyzing chunk frames = [4460544,4461056) | 20260506 21:07:51.204 | INFO | MemInfo: { | 20260506 21:07:51.204 | INFO | "gpu" : " 17 GB", | 20260506 21:07:51.204 | INFO | "huge" : "", | 20260506 21:07:51.204 | INFO | "huge_shared" : " 27 GB", | 20260506 21:07:51.204 | INFO | "malloc" : " 14 GB", | 20260506 21:07:51.204 | INFO | "pinned" : " 29 GB" | 20260506 21:07:51.204 | INFO | } | 20260506 21:07:51.204 | INFO | Perf report for the last 5.13854 seconds: | 20260506 21:07:51.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:07:51.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:51.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:51.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:51.204 | INFO | ANALYSIS : 62.4%, 2.9, 95.5ms, 0 | 20260506 21:07:51.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:51.204 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 21:07:51.204 | INFO | PRE_HQ : 60.0%, 1.8, 56.8ms, 0 | 20260506 21:07:51.204 | INFO | BAZWRITER : 67.2%, 4.8, 165.6ms, 0 | 20260506 21:07:51.204 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 21:07:51.205 | INFO | Uploaded 36GiB to GPU in 2.00721s (17.9354GiB/s) | 20260506 21:07:51.205 | INFO | Downloaded 38GiB to GPU in 2.2008s (17.2665GiB/s) | 20260506 21:07:56.345 | INFO | Analyzing chunk frames = [4461056,4461568) | 20260506 21:07:56.345 | INFO | MemInfo: { | 20260506 21:07:56.345 | INFO | "gpu" : " 17 GB", | 20260506 21:07:56.345 | INFO | "huge" : "", | 20260506 21:07:56.345 | INFO | "huge_shared" : " 27 GB", | 20260506 21:07:56.345 | INFO | "malloc" : " 14 GB", | 20260506 21:07:56.346 | INFO | "pinned" : " 29 GB" | 20260506 21:07:56.346 | INFO | } | 20260506 21:07:56.346 | INFO | Perf report for the last 5.14124 seconds: | 20260506 21:07:56.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:07:56.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:56.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:56.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:56.346 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:07:56.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:56.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:56.346 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:07:56.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:56.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:07:56.346 | INFO | Uploaded 36GiB to GPU in 1.99386s (18.0554GiB/s) | 20260506 21:07:56.346 | INFO | Downloaded 34GiB to GPU in 1.92735s (17.6408GiB/s) | 20260506 21:08:01.490 | INFO | Analyzing chunk frames = [4461568,4462080) | 20260506 21:08:01.490 | INFO | MemInfo: { | 20260506 21:08:01.490 | INFO | "gpu" : " 17 GB", | 20260506 21:08:01.490 | INFO | "huge" : "", | 20260506 21:08:01.490 | INFO | "huge_shared" : " 27 GB", | 20260506 21:08:01.490 | INFO | "malloc" : " 14 GB", | 20260506 21:08:01.490 | INFO | "pinned" : " 29 GB" | 20260506 21:08:01.490 | INFO | } | 20260506 21:08:01.490 | INFO | Perf report for the last 5.14464 seconds: | 20260506 21:08:01.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:08:01.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:01.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:01.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:01.490 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 21:08:01.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:01.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:01.490 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 21:08:01.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:01.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:01.490 | INFO | Uploaded 36GiB to GPU in 1.97805s (18.1997GiB/s) | 20260506 21:08:01.491 | INFO | Downloaded 34GiB to GPU in 1.92749s (17.6396GiB/s) | 20260506 21:08:06.626 | INFO | Analyzing chunk frames = [4462080,4462592) | 20260506 21:08:06.626 | INFO | MemInfo: { | 20260506 21:08:06.626 | INFO | "gpu" : " 17 GB", | 20260506 21:08:06.626 | INFO | "huge" : "", | 20260506 21:08:06.626 | INFO | "huge_shared" : " 27 GB", | 20260506 21:08:06.626 | INFO | "malloc" : " 14 GB", | 20260506 21:08:06.626 | INFO | "pinned" : " 29 GB" | 20260506 21:08:06.626 | INFO | } | 20260506 21:08:06.626 | INFO | Perf report for the last 5.13625 seconds: | 20260506 21:08:06.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:08:06.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:06.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:06.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:06.626 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 21:08:06.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:06.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:06.627 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 21:08:06.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:06.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:06.627 | INFO | Uploaded 36GiB to GPU in 1.97348s (18.2419GiB/s) | 20260506 21:08:06.627 | INFO | Downloaded 34GiB to GPU in 1.9071s (17.8281GiB/s) | 20260506 21:08:11.765 | INFO | Analyzing chunk frames = [4462592,4463104) | 20260506 21:08:11.765 | INFO | MemInfo: { | 20260506 21:08:11.765 | INFO | "gpu" : " 17 GB", | 20260506 21:08:11.765 | INFO | "huge" : "", | 20260506 21:08:11.765 | INFO | "huge_shared" : " 27 GB", | 20260506 21:08:11.765 | INFO | "malloc" : " 14 GB", | 20260506 21:08:11.765 | INFO | "pinned" : " 29 GB" | 20260506 21:08:11.765 | INFO | } | 20260506 21:08:11.765 | INFO | Perf report for the last 5.13879 seconds: | 20260506 21:08:11.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:08:11.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:11.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:11.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:11.765 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 21:08:11.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:11.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:11.766 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:08:11.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:11.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:11.766 | INFO | Uploaded 36GiB to GPU in 1.99857s (18.0129GiB/s) | 20260506 21:08:11.766 | INFO | Downloaded 34GiB to GPU in 1.92469s (17.6652GiB/s) | 20260506 21:08:16.902 | INFO | Analyzing chunk frames = [4463104,4463616) | 20260506 21:08:16.902 | INFO | MemInfo: { | 20260506 21:08:16.902 | INFO | "gpu" : " 17 GB", | 20260506 21:08:16.902 | INFO | "huge" : "", | 20260506 21:08:16.902 | INFO | "huge_shared" : " 27 GB", | 20260506 21:08:16.902 | INFO | "malloc" : " 14 GB", | 20260506 21:08:16.902 | INFO | "pinned" : " 29 GB" | 20260506 21:08:16.902 | INFO | } | 20260506 21:08:16.902 | INFO | Perf report for the last 5.13658 seconds: | 20260506 21:08:16.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:08:16.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:16.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:16.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:16.902 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:08:16.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:16.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:16.902 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 21:08:16.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:16.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:16.902 | INFO | Uploaded 36GiB to GPU in 1.98384s (18.1466GiB/s) | 20260506 21:08:16.902 | INFO | Downloaded 34GiB to GPU in 1.93908s (17.5341GiB/s) | 20260506 21:08:22.042 | INFO | Analyzing chunk frames = [4463616,4464128) | 20260506 21:08:22.042 | INFO | MemInfo: { | 20260506 21:08:22.042 | INFO | "gpu" : " 17 GB", | 20260506 21:08:22.042 | INFO | "huge" : "", | 20260506 21:08:22.042 | INFO | "huge_shared" : " 27 GB", | 20260506 21:08:22.042 | INFO | "malloc" : " 14 GB", | 20260506 21:08:22.042 | INFO | "pinned" : " 29 GB" | 20260506 21:08:22.042 | INFO | } | 20260506 21:08:22.042 | INFO | Perf report for the last 5.14014 seconds: | 20260506 21:08:22.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:08:22.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:22.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:22.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:22.042 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 21:08:22.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:22.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:22.042 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 21:08:22.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:22.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:22.042 | INFO | Uploaded 36GiB to GPU in 1.93713s (18.5842GiB/s) | 20260506 21:08:22.042 | INFO | Downloaded 34GiB to GPU in 1.98661s (17.1146GiB/s) | 20260506 21:08:27.176 | INFO | Analyzing chunk frames = [4464128,4464640) | 20260506 21:08:27.176 | INFO | MemInfo: { | 20260506 21:08:27.176 | INFO | "gpu" : " 17 GB", | 20260506 21:08:27.177 | INFO | "huge" : "", | 20260506 21:08:27.177 | INFO | "huge_shared" : " 27 GB", | 20260506 21:08:27.177 | INFO | "malloc" : " 14 GB", | 20260506 21:08:27.177 | INFO | "pinned" : " 29 GB" | 20260506 21:08:27.177 | INFO | } | 20260506 21:08:27.177 | INFO | Perf report for the last 5.1346 seconds: | 20260506 21:08:27.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:08:27.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:27.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:27.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:27.177 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:08:27.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:27.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:27.177 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 21:08:27.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:27.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:27.177 | INFO | Uploaded 36GiB to GPU in 2.0491s (17.5687GiB/s) | 20260506 21:08:27.177 | INFO | Downloaded 34GiB to GPU in 1.99323s (17.0577GiB/s) | 20260506 21:08:27.177 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:08:27.177 | INFO | Total mem usage: 18 GiB | 20260506 21:08:27.177 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:08:27.177 | INFO | Recent High Water Marks Sum: 16.3 GiB (Savings: -10.2%) | 20260506 21:08:27.177 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:08:27.177 | INFO | High Water Breakdown: | 20260506 21:08:27.177 | INFO | 62.2% : 10.2 GiB ( 89% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:08:27.177 | INFO | 37.7% : 6.16 GiB ( 92% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:08:27.177 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:08:27.177 | INFO | ------------------------------------------ | 20260506 21:08:27.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:08:27.177 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:08:27.177 | INFO | Total mem usage: 26.5 GiB | 20260506 21:08:27.177 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:08:27.177 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:08:27.177 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:08:27.177 | INFO | High Water Breakdown: | 20260506 21:08:27.177 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:08:27.177 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:08:27.177 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:08:27.177 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:08:27.177 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:08:27.177 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:08:27.177 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:08:27.177 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:08:27.177 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:08:27.177 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:08:27.177 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:08:27.177 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:08:27.177 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:08:27.177 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:08:27.177 | INFO | ------------------------------------------ | 20260506 21:08:27.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:08:27.177 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:08:27.177 | INFO | Total mem usage: 25.6 GiB | 20260506 21:08:27.177 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:08:27.177 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 21:08:27.177 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 21:08:27.177 | INFO | High Water Breakdown: | 20260506 21:08:27.177 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 21:08:27.177 | INFO | ------------------------------------------ | 20260506 21:08:27.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:08:27.177 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:08:27.177 | INFO | Total mem usage: 15.7 GiB | 20260506 21:08:27.177 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:08:27.177 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:08:27.177 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:08:27.177 | INFO | High Water Breakdown: | 20260506 21:08:27.177 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:08:27.177 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:08:27.178 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:08:27.178 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:08:27.178 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:08:27.178 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:08:27.178 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:08:27.178 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:08:27.178 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:08:27.178 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:08:27.178 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:08:27.178 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:08:27.178 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:08:27.178 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:08:27.178 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:08:27.178 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:08:27.178 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:08:27.178 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:08:27.178 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:08:27.178 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:08:27.178 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:08:27.178 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:08:27.178 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:08:27.178 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:08:27.178 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:08:27.178 | INFO | ------------------------------------------ | 20260506 21:08:27.178 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:08:27.178 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:08:27.178 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:08:27.178 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:08:27.178 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.55e+01, 1.35e+01 / 2.30e+00 | 20260506 21:08:27.178 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.02e-01, 9.98e-03 / 9.56e-03 | 20260506 21:08:27.178 | INFO | PerfReport: 11.64%, Baseline -- 3.62e+00 / 4.35e+00, 5.48e-01 / 3.86e-01 | 20260506 21:08:27.178 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.42e+00, 1.79e-01 / 1.55e-01 | 20260506 21:08:27.178 | INFO | PerfReport: 16.04%, DME -- 3.99e+01 / 4.44e+01, 4.25e+00 / 2.98e+00 | 20260506 21:08:27.178 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.36e+00 / 1.97e+00 | 20260506 21:08:27.178 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.22e+00 / 2.74e+00, 3.80e-01 / 2.07e-01 | 20260506 21:08:27.178 | INFO | PerfReport: 3.29%, Metrics -- 1.02e+00 / 1.34e+00, 2.72e-01 / 1.59e-01 | 20260506 21:08:27.178 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.51e-01 / 3.95e-01, 5.47e-02 / 2.03e-02 | 20260506 21:08:27.178 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:08:27.178 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:08:27.178 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:08:27.178 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 21:08:27.178 | INFO | PerfReport: 21.47%, Upload -- 2.08e+01 / 0.00e+00, 6.89e+00 / 0.00e+00 | 20260506 21:08:27.178 | INFO | PerfReport: 21.27%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 21:08:27.178 | INFO | PerfReport: 32.17%, Compute -- 3.11e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 21:08:27.178 | INFO | PerfReport: 1.30%, LockUpload -- 1.26e+00 / 0.00e+00, 4.79e+00 / 0.00e+00 | 20260506 21:08:27.178 | INFO | PerfReport: 1.23%, LockDownload -- 1.19e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260506 21:08:27.178 | INFO | PerfReport: 22.54%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 21:08:27.178 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260506 21:08:27.291 | INFO | Starting a round of Baz IO | 20260506 21:08:30.425 | INFO | Populating RTMetrics report | 20260506 21:08:30.426 | INFO | RTMetrics report populated | 20260506 21:08:30.426 | INFO | Opening tmp RTMetrics json file | 20260506 21:08:30.484 | INFO | Serializing RTMetrics to json | 20260506 21:08:30.511 | INFO | Serialization done, swapping tmp file now... | 20260506 21:08:30.512 | INFO | RTMetrics done | 20260506 21:08:30.688 | INFO | Finished a round of Baz IO: Wrote 1.958 GiB in 3.397 seconds | 20260506 21:08:30.688 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.03% / 0.00% / 65.97% | 20260506 21:08:30.688 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.657419% / 7.94729e-06% / 6.75519e-05% | 20260506 21:08:30.688 | INFO | Baz IO overall: | 20260506 21:08:30.688 | INFO | BytesWritten : 2354110398097 | 20260506 21:08:30.688 | INFO | NumEvents : 1136853416511 | 20260506 21:08:32.319 | INFO | Analyzing chunk frames = [4464640,4465152) | 20260506 21:08:32.319 | INFO | MemInfo: { | 20260506 21:08:32.319 | INFO | "gpu" : " 17 GB", | 20260506 21:08:32.319 | INFO | "huge" : "", | 20260506 21:08:32.319 | INFO | "huge_shared" : " 27 GB", | 20260506 21:08:32.319 | INFO | "malloc" : " 14 GB", | 20260506 21:08:32.319 | INFO | "pinned" : " 29 GB" | 20260506 21:08:32.319 | INFO | } | 20260506 21:08:32.319 | INFO | Perf report for the last 5.14202 seconds: | 20260506 21:08:32.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:08:32.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:32.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:32.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:32.319 | INFO | ANALYSIS : 62.6%, 2.9, 95.6ms, 0 | 20260506 21:08:32.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:32.319 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260506 21:08:32.319 | INFO | PRE_HQ : 59.9%, 1.9, 57.7ms, 0 | 20260506 21:08:32.319 | INFO | BAZWRITER : 65.7%, 4.7, 159.7ms, 0 | 20260506 21:08:32.319 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 21:08:32.319 | INFO | Uploaded 36GiB to GPU in 2.05639s (17.5064GiB/s) | 20260506 21:08:32.319 | INFO | Downloaded 38GiB to GPU in 2.2112s (17.1852GiB/s) | 20260506 21:08:37.469 | INFO | Analyzing chunk frames = [4465152,4465664) | 20260506 21:08:37.469 | INFO | MemInfo: { | 20260506 21:08:37.469 | INFO | "gpu" : " 17 GB", | 20260506 21:08:37.469 | INFO | "huge" : "", | 20260506 21:08:37.469 | INFO | "huge_shared" : " 27 GB", | 20260506 21:08:37.469 | INFO | "malloc" : " 14 GB", | 20260506 21:08:37.469 | INFO | "pinned" : " 29 GB" | 20260506 21:08:37.469 | INFO | } | 20260506 21:08:37.469 | INFO | Perf report for the last 5.15044 seconds: | 20260506 21:08:37.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:08:37.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:37.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:37.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:37.469 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 21:08:37.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:37.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:37.469 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:08:37.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:37.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:37.469 | INFO | Uploaded 36GiB to GPU in 2.00295s (17.9735GiB/s) | 20260506 21:08:37.470 | INFO | Downloaded 34GiB to GPU in 1.91891s (17.7184GiB/s) | 20260506 21:08:42.606 | INFO | Analyzing chunk frames = [4465664,4466176) | 20260506 21:08:42.606 | INFO | MemInfo: { | 20260506 21:08:42.606 | INFO | "gpu" : " 17 GB", | 20260506 21:08:42.606 | INFO | "huge" : "", | 20260506 21:08:42.606 | INFO | "huge_shared" : " 27 GB", | 20260506 21:08:42.606 | INFO | "malloc" : " 14 GB", | 20260506 21:08:42.606 | INFO | "pinned" : " 29 GB" | 20260506 21:08:42.606 | INFO | } | 20260506 21:08:42.606 | INFO | Perf report for the last 5.13662 seconds: | 20260506 21:08:42.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:08:42.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:42.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:42.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:42.606 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 21:08:42.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:42.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:42.606 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:08:42.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:42.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:42.606 | INFO | Uploaded 36GiB to GPU in 1.98978s (18.0924GiB/s) | 20260506 21:08:42.606 | INFO | Downloaded 34GiB to GPU in 1.92535s (17.6592GiB/s) | 20260506 21:08:47.747 | INFO | Analyzing chunk frames = [4466176,4466688) | 20260506 21:08:47.747 | INFO | MemInfo: { | 20260506 21:08:47.747 | INFO | "gpu" : " 17 GB", | 20260506 21:08:47.747 | INFO | "huge" : "", | 20260506 21:08:47.747 | INFO | "huge_shared" : " 27 GB", | 20260506 21:08:47.747 | INFO | "malloc" : " 14 GB", | 20260506 21:08:47.747 | INFO | "pinned" : " 29 GB" | 20260506 21:08:47.747 | INFO | } | 20260506 21:08:47.747 | INFO | Perf report for the last 5.14118 seconds: | 20260506 21:08:47.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:08:47.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:47.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:47.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:47.747 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260506 21:08:47.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:47.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:47.747 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:08:47.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:47.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:47.747 | INFO | Uploaded 36GiB to GPU in 2.00806s (17.9277GiB/s) | 20260506 21:08:47.747 | INFO | Downloaded 34GiB to GPU in 1.90642s (17.8345GiB/s) | 20260506 21:08:52.882 | INFO | Analyzing chunk frames = [4466688,4467200) | 20260506 21:08:52.882 | INFO | MemInfo: { | 20260506 21:08:52.882 | INFO | "gpu" : " 17 GB", | 20260506 21:08:52.882 | INFO | "huge" : "", | 20260506 21:08:52.882 | INFO | "huge_shared" : " 27 GB", | 20260506 21:08:52.882 | INFO | "malloc" : " 14 GB", | 20260506 21:08:52.882 | INFO | "pinned" : " 29 GB" | 20260506 21:08:52.882 | INFO | } | 20260506 21:08:52.882 | INFO | Perf report for the last 5.13538 seconds: | 20260506 21:08:52.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:08:52.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:52.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:52.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:52.882 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 21:08:52.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:52.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:52.883 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:08:52.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:52.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:52.883 | INFO | Uploaded 36GiB to GPU in 1.99778s (18.02GiB/s) | 20260506 21:08:52.883 | INFO | Downloaded 34GiB to GPU in 1.92316s (17.6793GiB/s) | 20260506 21:08:58.017 | INFO | Analyzing chunk frames = [4467200,4467712) | 20260506 21:08:58.017 | INFO | MemInfo: { | 20260506 21:08:58.017 | INFO | "gpu" : " 17 GB", | 20260506 21:08:58.017 | INFO | "huge" : "", | 20260506 21:08:58.017 | INFO | "huge_shared" : " 27 GB", | 20260506 21:08:58.017 | INFO | "malloc" : " 14 GB", | 20260506 21:08:58.017 | INFO | "pinned" : " 29 GB" | 20260506 21:08:58.017 | INFO | } | 20260506 21:08:58.018 | INFO | Perf report for the last 5.13517 seconds: | 20260506 21:08:58.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:08:58.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:58.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:58.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:58.018 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 21:08:58.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:58.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:58.018 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 21:08:58.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:58.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:08:58.018 | INFO | Uploaded 36GiB to GPU in 2.01291s (17.8846GiB/s) | 20260506 21:08:58.018 | INFO | Downloaded 34GiB to GPU in 1.94429s (17.4871GiB/s) | 20260506 21:09:03.153 | INFO | Analyzing chunk frames = [4467712,4468224) | 20260506 21:09:03.153 | INFO | MemInfo: { | 20260506 21:09:03.153 | INFO | "gpu" : " 17 GB", | 20260506 21:09:03.153 | INFO | "huge" : "", | 20260506 21:09:03.153 | INFO | "huge_shared" : " 27 GB", | 20260506 21:09:03.153 | INFO | "malloc" : " 14 GB", | 20260506 21:09:03.153 | INFO | "pinned" : " 29 GB" | 20260506 21:09:03.153 | INFO | } | 20260506 21:09:03.154 | INFO | Perf report for the last 5.13595 seconds: | 20260506 21:09:03.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:09:03.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:03.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:03.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:03.154 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 21:09:03.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:03.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:03.154 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 21:09:03.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:03.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:03.154 | INFO | Uploaded 36GiB to GPU in 1.97459s (18.2316GiB/s) | 20260506 21:09:03.154 | INFO | Downloaded 34GiB to GPU in 2.00249s (16.9789GiB/s) | 20260506 21:09:08.298 | INFO | Analyzing chunk frames = [4468224,4468736) | 20260506 21:09:08.298 | INFO | MemInfo: { | 20260506 21:09:08.298 | INFO | "gpu" : " 17 GB", | 20260506 21:09:08.298 | INFO | "huge" : "", | 20260506 21:09:08.298 | INFO | "huge_shared" : " 27 GB", | 20260506 21:09:08.298 | INFO | "malloc" : " 14 GB", | 20260506 21:09:08.298 | INFO | "pinned" : " 29 GB" | 20260506 21:09:08.298 | INFO | } | 20260506 21:09:08.298 | INFO | Perf report for the last 5.14429 seconds: | 20260506 21:09:08.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:09:08.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:08.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:08.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:08.298 | INFO | ANALYSIS : 61.4%, 3.0, 94.7ms, 0 | 20260506 21:09:08.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:08.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:08.298 | INFO | PRE_HQ : 27.0%, 1.0, 13.9ms, 0 | 20260506 21:09:08.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:08.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:08.298 | INFO | Uploaded 36GiB to GPU in 1.99036s (18.0872GiB/s) | 20260506 21:09:08.298 | INFO | Downloaded 34GiB to GPU in 1.97742s (17.1941GiB/s) | 20260506 21:09:08.396 | INFO | Starting a round of Baz IO | 20260506 21:09:11.533 | INFO | Populating RTMetrics report | 20260506 21:09:11.534 | INFO | RTMetrics report populated | 20260506 21:09:11.534 | INFO | Opening tmp RTMetrics json file | 20260506 21:09:11.560 | INFO | Serializing RTMetrics to json | 20260506 21:09:11.595 | INFO | Serialization done, swapping tmp file now... | 20260506 21:09:11.600 | INFO | RTMetrics done | 20260506 21:09:11.884 | INFO | Finished a round of Baz IO: Wrote 1.957 GiB in 3.487 seconds | 20260506 21:09:11.884 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.02% / 0.00% / 65.97% | 20260506 21:09:11.884 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.655615% / 7.94729e-06% / 7.94729e-05% | 20260506 21:09:11.884 | INFO | Baz IO overall: | 20260506 21:09:11.884 | INFO | BytesWritten : 2356211481344 | 20260506 21:09:11.884 | INFO | NumEvents : 1137860185725 | 20260506 21:09:13.435 | INFO | Analyzing chunk frames = [4468736,4469248) | 20260506 21:09:13.435 | INFO | MemInfo: { | 20260506 21:09:13.435 | INFO | "gpu" : " 17 GB", | 20260506 21:09:13.436 | INFO | "huge" : "", | 20260506 21:09:13.436 | INFO | "huge_shared" : " 27 GB", | 20260506 21:09:13.436 | INFO | "malloc" : " 14 GB", | 20260506 21:09:13.436 | INFO | "pinned" : " 29 GB" | 20260506 21:09:13.436 | INFO | } | 20260506 21:09:13.436 | INFO | Perf report for the last 5.13782 seconds: | 20260506 21:09:13.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:09:13.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:13.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:13.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:13.436 | INFO | ANALYSIS : 62.4%, 2.9, 95.1ms, 0 | 20260506 21:09:13.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:13.436 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 21:09:13.436 | INFO | PRE_HQ : 60.0%, 1.8, 57.2ms, 0 | 20260506 21:09:13.436 | INFO | BAZWRITER : 67.5%, 4.8, 165.7ms, 0 | 20260506 21:09:13.436 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 21:09:13.436 | INFO | Uploaded 36GiB to GPU in 2.04685s (17.588GiB/s) | 20260506 21:09:13.436 | INFO | Downloaded 38GiB to GPU in 2.19928s (17.2783GiB/s) | 20260506 21:09:18.579 | INFO | Analyzing chunk frames = [4469248,4469760) | 20260506 21:09:18.579 | INFO | MemInfo: { | 20260506 21:09:18.579 | INFO | "gpu" : " 17 GB", | 20260506 21:09:18.579 | INFO | "huge" : "", | 20260506 21:09:18.579 | INFO | "huge_shared" : " 27 GB", | 20260506 21:09:18.579 | INFO | "malloc" : " 14 GB", | 20260506 21:09:18.579 | INFO | "pinned" : " 29 GB" | 20260506 21:09:18.579 | INFO | } | 20260506 21:09:18.579 | INFO | Perf report for the last 5.14304 seconds: | 20260506 21:09:18.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:09:18.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:18.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:18.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:18.579 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 21:09:18.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:18.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:18.579 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 21:09:18.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:18.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:18.579 | INFO | Uploaded 36GiB to GPU in 2.01334s (17.8807GiB/s) | 20260506 21:09:18.579 | INFO | Downloaded 34GiB to GPU in 1.92511s (17.6613GiB/s) | 20260506 21:09:23.717 | INFO | Analyzing chunk frames = [4469760,4470272) | 20260506 21:09:23.717 | INFO | MemInfo: { | 20260506 21:09:23.717 | INFO | "gpu" : " 17 GB", | 20260506 21:09:23.717 | INFO | "huge" : "", | 20260506 21:09:23.717 | INFO | "huge_shared" : " 27 GB", | 20260506 21:09:23.717 | INFO | "malloc" : " 14 GB", | 20260506 21:09:23.717 | INFO | "pinned" : " 29 GB" | 20260506 21:09:23.717 | INFO | } | 20260506 21:09:23.717 | INFO | Perf report for the last 5.1387 seconds: | 20260506 21:09:23.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:09:23.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:23.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:23.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:23.717 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260506 21:09:23.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:23.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:23.717 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 21:09:23.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:23.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:23.718 | INFO | Uploaded 36GiB to GPU in 1.98889s (18.1006GiB/s) | 20260506 21:09:23.718 | INFO | Downloaded 34GiB to GPU in 1.91747s (17.7317GiB/s) | 20260506 21:09:28.856 | INFO | Analyzing chunk frames = [4470272,4470784) | 20260506 21:09:28.856 | INFO | MemInfo: { | 20260506 21:09:28.856 | INFO | "gpu" : " 17 GB", | 20260506 21:09:28.856 | INFO | "huge" : "", | 20260506 21:09:28.856 | INFO | "huge_shared" : " 27 GB", | 20260506 21:09:28.856 | INFO | "malloc" : " 14 GB", | 20260506 21:09:28.856 | INFO | "pinned" : " 29 GB" | 20260506 21:09:28.856 | INFO | } | 20260506 21:09:28.856 | INFO | Perf report for the last 5.1389 seconds: | 20260506 21:09:28.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:09:28.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:28.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:28.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:28.856 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 21:09:28.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:28.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:28.856 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 21:09:28.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:28.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:28.856 | INFO | Uploaded 36GiB to GPU in 1.96749s (18.2974GiB/s) | 20260506 21:09:28.856 | INFO | Downloaded 34GiB to GPU in 1.87651s (18.1187GiB/s) | 20260506 21:09:33.991 | INFO | Analyzing chunk frames = [4470784,4471296) | 20260506 21:09:33.991 | INFO | MemInfo: { | 20260506 21:09:33.991 | INFO | "gpu" : " 17 GB", | 20260506 21:09:33.994 | INFO | "huge" : "", | 20260506 21:09:33.994 | INFO | "huge_shared" : " 27 GB", | 20260506 21:09:33.994 | INFO | "malloc" : " 14 GB", | 20260506 21:09:33.994 | INFO | "pinned" : " 29 GB" | 20260506 21:09:33.994 | INFO | } | 20260506 21:09:33.994 | INFO | Perf report for the last 5.13764 seconds: | 20260506 21:09:33.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:09:33.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:33.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:33.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:33.994 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 21:09:33.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:33.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:33.994 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 21:09:33.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:33.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:33.994 | INFO | Uploaded 36GiB to GPU in 2.01215s (17.8913GiB/s) | 20260506 21:09:33.994 | INFO | Downloaded 34GiB to GPU in 1.92291s (17.6816GiB/s) | 20260506 21:09:39.131 | INFO | Analyzing chunk frames = [4471296,4471808) | 20260506 21:09:39.131 | INFO | MemInfo: { | 20260506 21:09:39.131 | INFO | "gpu" : " 17 GB", | 20260506 21:09:39.131 | INFO | "huge" : "", | 20260506 21:09:39.131 | INFO | "huge_shared" : " 27 GB", | 20260506 21:09:39.131 | INFO | "malloc" : " 14 GB", | 20260506 21:09:39.131 | INFO | "pinned" : " 29 GB" | 20260506 21:09:39.131 | INFO | } | 20260506 21:09:39.131 | INFO | Perf report for the last 5.13739 seconds: | 20260506 21:09:39.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:09:39.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:39.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:39.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:39.131 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 21:09:39.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:39.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:39.131 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 21:09:39.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:39.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:39.132 | INFO | Uploaded 36GiB to GPU in 2.03763s (17.6676GiB/s) | 20260506 21:09:39.132 | INFO | Downloaded 34GiB to GPU in 1.94491s (17.4816GiB/s) | 20260506 21:09:44.270 | INFO | Analyzing chunk frames = [4471808,4472320) | 20260506 21:09:44.270 | INFO | MemInfo: { | 20260506 21:09:44.270 | INFO | "gpu" : " 17 GB", | 20260506 21:09:44.270 | INFO | "huge" : "", | 20260506 21:09:44.270 | INFO | "huge_shared" : " 27 GB", | 20260506 21:09:44.270 | INFO | "malloc" : " 14 GB", | 20260506 21:09:44.270 | INFO | "pinned" : " 29 GB" | 20260506 21:09:44.270 | INFO | } | 20260506 21:09:44.270 | INFO | Perf report for the last 5.13838 seconds: | 20260506 21:09:44.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:09:44.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:44.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:44.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:44.270 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:09:44.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:44.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:44.270 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 21:09:44.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:44.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:44.270 | INFO | Uploaded 36GiB to GPU in 1.96929s (18.2807GiB/s) | 20260506 21:09:44.270 | INFO | Downloaded 34GiB to GPU in 2.00247s (16.979GiB/s) | 20260506 21:09:49.408 | INFO | Analyzing chunk frames = [4472320,4472832) | 20260506 21:09:49.408 | INFO | MemInfo: { | 20260506 21:09:49.408 | INFO | "gpu" : " 17 GB", | 20260506 21:09:49.408 | INFO | "huge" : "", | 20260506 21:09:49.408 | INFO | "huge_shared" : " 27 GB", | 20260506 21:09:49.408 | INFO | "malloc" : " 14 GB", | 20260506 21:09:49.408 | INFO | "pinned" : " 29 GB" | 20260506 21:09:49.408 | INFO | } | 20260506 21:09:49.408 | INFO | Perf report for the last 5.13843 seconds: | 20260506 21:09:49.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:09:49.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:49.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:49.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:49.408 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 21:09:49.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:49.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:49.408 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 21:09:49.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:49.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:49.408 | INFO | Uploaded 36GiB to GPU in 2.02702s (17.7601GiB/s) | 20260506 21:09:49.408 | INFO | Downloaded 34GiB to GPU in 2.0012s (16.9898GiB/s) | 20260506 21:09:49.409 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:09:49.409 | INFO | Total mem usage: 18 GiB | 20260506 21:09:49.409 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:09:49.409 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -14.1%) | 20260506 21:09:49.409 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:09:49.409 | INFO | High Water Breakdown: | 20260506 21:09:49.409 | INFO | 61.7% : 9.73 GiB ( 85.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:09:49.409 | INFO | 38.2% : 6.03 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:09:49.409 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:09:49.409 | INFO | ------------------------------------------ | 20260506 21:09:49.409 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:09:49.409 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:09:49.409 | INFO | Total mem usage: 26.5 GiB | 20260506 21:09:49.409 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:09:49.409 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:09:49.409 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:09:49.409 | INFO | High Water Breakdown: | 20260506 21:09:49.409 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:09:49.409 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:09:49.409 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:09:49.409 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:09:49.409 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:09:49.409 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:09:49.409 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:09:49.409 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:09:49.409 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:09:49.409 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:09:49.409 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:09:49.409 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:09:49.409 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:09:49.409 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:09:49.409 | INFO | ------------------------------------------ | 20260506 21:09:49.409 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:09:49.409 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:09:49.409 | INFO | Total mem usage: 25.6 GiB | 20260506 21:09:49.409 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:09:49.409 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:09:49.409 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:09:49.409 | INFO | High Water Breakdown: | 20260506 21:09:49.409 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:09:49.409 | INFO | ------------------------------------------ | 20260506 21:09:49.409 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:09:49.409 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:09:49.409 | INFO | Total mem usage: 15.7 GiB | 20260506 21:09:49.409 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:09:49.409 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:09:49.409 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:09:49.409 | INFO | High Water Breakdown: | 20260506 21:09:49.409 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:09:49.409 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:09:49.409 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:09:49.409 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:09:49.409 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:09:49.409 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:09:49.409 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:09:49.409 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:09:49.409 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:09:49.409 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:09:49.409 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:09:49.409 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:09:49.409 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:09:49.409 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:09:49.409 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:09:49.409 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:09:49.409 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:09:49.409 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:09:49.409 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:09:49.409 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:09:49.409 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:09:49.409 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:09:49.409 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:09:49.409 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:09:49.409 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:09:49.409 | INFO | ------------------------------------------ | 20260506 21:09:49.409 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:09:49.410 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:09:49.410 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:09:49.410 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:09:49.410 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.63e+01, 1.34e+01 / 2.39e+00 | 20260506 21:09:49.410 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 1.00e-02 / 6.23e-03 | 20260506 21:09:49.410 | INFO | PerfReport: 11.65%, Baseline -- 3.63e+00 / 4.37e+00, 5.56e-01 / 3.80e-01 | 20260506 21:09:49.410 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.82e-01 / 1.44e-01 | 20260506 21:09:49.410 | INFO | PerfReport: 16.04%, DME -- 4.00e+01 / 4.50e+01, 4.26e+00 / 2.80e+00 | 20260506 21:09:49.410 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.43e+00 / 1.84e+00 | 20260506 21:09:49.410 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.22e+00 / 2.75e+00, 3.77e-01 / 2.26e-01 | 20260506 21:09:49.410 | INFO | PerfReport: 3.29%, Metrics -- 1.03e+00 / 1.37e+00, 2.72e-01 / 1.34e-01 | 20260506 21:09:49.410 | INFO | PerfReport: 1.13%, PulseSerializing -- 3.51e-01 / 4.00e-01, 5.44e-02 / 1.98e-02 | 20260506 21:09:49.410 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:09:49.410 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:09:49.410 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:09:49.410 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 21:09:49.410 | INFO | PerfReport: 21.59%, Upload -- 2.09e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260506 21:09:49.410 | INFO | PerfReport: 21.25%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 21:09:49.410 | INFO | PerfReport: 32.19%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 21:09:49.410 | INFO | PerfReport: 1.35%, LockUpload -- 1.30e+00 / 0.00e+00, 4.94e+00 / 0.00e+00 | 20260506 21:09:49.410 | INFO | PerfReport: 1.18%, LockDownload -- 1.14e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260506 21:09:49.410 | INFO | PerfReport: 22.43%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:09:49.410 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260506 21:09:49.530 | INFO | Starting a round of Baz IO | 20260506 21:09:52.652 | INFO | Populating RTMetrics report | 20260506 21:09:52.652 | INFO | RTMetrics report populated | 20260506 21:09:52.652 | INFO | Opening tmp RTMetrics json file | 20260506 21:09:52.759 | INFO | Serializing RTMetrics to json | 20260506 21:09:52.802 | INFO | Serialization done, swapping tmp file now... | 20260506 21:09:52.852 | INFO | RTMetrics done | 20260506 21:09:53.245 | INFO | Finished a round of Baz IO: Wrote 1.956 GiB in 3.715 seconds | 20260506 21:09:53.246 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.02% / 0.00% / 65.98% | 20260506 21:09:53.246 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.653207% / 1.19209e-05% / 5.5631e-05% | 20260506 21:09:53.246 | INFO | Baz IO overall: | 20260506 21:09:53.246 | INFO | BytesWritten : 2358311876461 | 20260506 21:09:53.246 | INFO | NumEvents : 1138866615704 | 20260506 21:09:54.563 | INFO | Analyzing chunk frames = [4472832,4473344) | 20260506 21:09:54.563 | INFO | MemInfo: { | 20260506 21:09:54.563 | INFO | "gpu" : " 17 GB", | 20260506 21:09:54.563 | INFO | "huge" : "", | 20260506 21:09:54.563 | INFO | "huge_shared" : " 27 GB", | 20260506 21:09:54.563 | INFO | "malloc" : " 14 GB", | 20260506 21:09:54.563 | INFO | "pinned" : " 29 GB" | 20260506 21:09:54.563 | INFO | } | 20260506 21:09:54.563 | INFO | Perf report for the last 5.15513 seconds: | 20260506 21:09:54.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:09:54.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:54.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:54.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:54.563 | INFO | ANALYSIS : 62.4%, 2.9, 95.6ms, 0 | 20260506 21:09:54.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:54.563 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260506 21:09:54.563 | INFO | PRE_HQ : 59.6%, 1.8, 57.2ms, 0 | 20260506 21:09:54.563 | INFO | BAZWRITER : 72.1%, 4.8, 176.9ms, 0 | 20260506 21:09:54.563 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 21:09:54.564 | INFO | Uploaded 36GiB to GPU in 2.03007s (17.7334GiB/s) | 20260506 21:09:54.564 | INFO | Downloaded 38GiB to GPU in 2.20828s (17.2079GiB/s) | 20260506 21:09:59.694 | INFO | Analyzing chunk frames = [4473344,4473856) | 20260506 21:09:59.694 | INFO | MemInfo: { | 20260506 21:09:59.694 | INFO | "gpu" : " 17 GB", | 20260506 21:09:59.694 | INFO | "huge" : "", | 20260506 21:09:59.694 | INFO | "huge_shared" : " 27 GB", | 20260506 21:09:59.694 | INFO | "malloc" : " 14 GB", | 20260506 21:09:59.694 | INFO | "pinned" : " 29 GB" | 20260506 21:09:59.694 | INFO | } | 20260506 21:09:59.694 | INFO | Perf report for the last 5.1309 seconds: | 20260506 21:09:59.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:09:59.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:59.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:59.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:59.694 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 21:09:59.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:59.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:59.694 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:09:59.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:59.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:09:59.694 | INFO | Uploaded 36GiB to GPU in 1.98147s (18.1684GiB/s) | 20260506 21:09:59.694 | INFO | Downloaded 34GiB to GPU in 1.92146s (17.6949GiB/s) | 20260506 21:10:04.832 | INFO | Analyzing chunk frames = [4473856,4474368) | 20260506 21:10:04.832 | INFO | MemInfo: { | 20260506 21:10:04.832 | INFO | "gpu" : " 17 GB", | 20260506 21:10:04.832 | INFO | "huge" : "", | 20260506 21:10:04.832 | INFO | "huge_shared" : " 27 GB", | 20260506 21:10:04.832 | INFO | "malloc" : " 14 GB", | 20260506 21:10:04.832 | INFO | "pinned" : " 29 GB" | 20260506 21:10:04.832 | INFO | } | 20260506 21:10:04.832 | INFO | Perf report for the last 5.13802 seconds: | 20260506 21:10:04.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:10:04.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:04.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:04.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:04.832 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 21:10:04.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:04.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:04.832 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:10:04.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:04.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:04.832 | INFO | Uploaded 36GiB to GPU in 1.97431s (18.2342GiB/s) | 20260506 21:10:04.833 | INFO | Downloaded 34GiB to GPU in 1.92025s (17.706GiB/s) | 20260506 21:10:09.972 | INFO | Analyzing chunk frames = [4474368,4474880) | 20260506 21:10:09.972 | INFO | MemInfo: { | 20260506 21:10:09.972 | INFO | "gpu" : " 17 GB", | 20260506 21:10:09.972 | INFO | "huge" : "", | 20260506 21:10:09.972 | INFO | "huge_shared" : " 27 GB", | 20260506 21:10:09.972 | INFO | "malloc" : " 14 GB", | 20260506 21:10:09.972 | INFO | "pinned" : " 29 GB" | 20260506 21:10:09.972 | INFO | } | 20260506 21:10:09.972 | INFO | Perf report for the last 5.14015 seconds: | 20260506 21:10:09.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:10:09.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:09.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:09.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:09.973 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 21:10:09.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:09.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:09.973 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 21:10:09.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:09.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:09.973 | INFO | Uploaded 36GiB to GPU in 1.96766s (18.2958GiB/s) | 20260506 21:10:09.973 | INFO | Downloaded 34GiB to GPU in 1.87973s (18.0877GiB/s) | 20260506 21:10:15.107 | INFO | Analyzing chunk frames = [4474880,4475392) | 20260506 21:10:15.107 | INFO | MemInfo: { | 20260506 21:10:15.107 | INFO | "gpu" : " 17 GB", | 20260506 21:10:15.107 | INFO | "huge" : "", | 20260506 21:10:15.109 | INFO | "huge_shared" : " 27 GB", | 20260506 21:10:15.109 | INFO | "malloc" : " 14 GB", | 20260506 21:10:15.109 | INFO | "pinned" : " 29 GB" | 20260506 21:10:15.109 | INFO | } | 20260506 21:10:15.109 | INFO | Perf report for the last 5.13637 seconds: | 20260506 21:10:15.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:10:15.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:15.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:15.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:15.109 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 21:10:15.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:15.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:15.109 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 21:10:15.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:15.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:15.109 | INFO | Uploaded 36GiB to GPU in 2.00433s (17.9611GiB/s) | 20260506 21:10:15.109 | INFO | Downloaded 34GiB to GPU in 1.92255s (17.6848GiB/s) | 20260506 21:10:20.246 | INFO | Analyzing chunk frames = [4475392,4475904) | 20260506 21:10:20.246 | INFO | MemInfo: { | 20260506 21:10:20.246 | INFO | "gpu" : " 17 GB", | 20260506 21:10:20.246 | INFO | "huge" : "", | 20260506 21:10:20.246 | INFO | "huge_shared" : " 27 GB", | 20260506 21:10:20.246 | INFO | "malloc" : " 14 GB", | 20260506 21:10:20.246 | INFO | "pinned" : " 29 GB" | 20260506 21:10:20.246 | INFO | } | 20260506 21:10:20.246 | INFO | Perf report for the last 5.13695 seconds: | 20260506 21:10:20.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:10:20.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:20.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:20.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:20.246 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:10:20.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:20.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:20.246 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 21:10:20.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:20.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:20.246 | INFO | Uploaded 36GiB to GPU in 1.99445s (18.0501GiB/s) | 20260506 21:10:20.246 | INFO | Downloaded 34GiB to GPU in 1.95489s (17.3923GiB/s) | 20260506 21:10:25.390 | INFO | Analyzing chunk frames = [4475904,4476416) | 20260506 21:10:25.390 | INFO | MemInfo: { | 20260506 21:10:25.390 | INFO | "gpu" : " 17 GB", | 20260506 21:10:25.390 | INFO | "huge" : "", | 20260506 21:10:25.391 | INFO | "huge_shared" : " 27 GB", | 20260506 21:10:25.391 | INFO | "malloc" : " 14 GB", | 20260506 21:10:25.391 | INFO | "pinned" : " 29 GB" | 20260506 21:10:25.391 | INFO | } | 20260506 21:10:25.391 | INFO | Perf report for the last 5.14478 seconds: | 20260506 21:10:25.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:10:25.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:25.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:25.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:25.391 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:10:25.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:25.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:25.391 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 21:10:25.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:25.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:25.391 | INFO | Uploaded 36GiB to GPU in 1.96939s (18.2798GiB/s) | 20260506 21:10:25.391 | INFO | Downloaded 34GiB to GPU in 1.99812s (17.016GiB/s) | 20260506 21:10:30.525 | INFO | Analyzing chunk frames = [4476416,4476928) | 20260506 21:10:30.525 | INFO | MemInfo: { | 20260506 21:10:30.525 | INFO | "gpu" : " 17 GB", | 20260506 21:10:30.525 | INFO | "huge" : "", | 20260506 21:10:30.525 | INFO | "huge_shared" : " 27 GB", | 20260506 21:10:30.525 | INFO | "malloc" : " 14 GB", | 20260506 21:10:30.525 | INFO | "pinned" : " 29 GB" | 20260506 21:10:30.525 | INFO | } | 20260506 21:10:30.525 | INFO | Perf report for the last 5.13429 seconds: | 20260506 21:10:30.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:10:30.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:30.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:30.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:30.525 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 21:10:30.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:30.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:30.525 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:10:30.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:30.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:30.525 | INFO | Uploaded 36GiB to GPU in 2.03483s (17.6919GiB/s) | 20260506 21:10:30.525 | INFO | Downloaded 34GiB to GPU in 1.99149s (17.0726GiB/s) | 20260506 21:10:30.639 | INFO | Starting a round of Baz IO | 20260506 21:10:33.769 | INFO | Populating RTMetrics report | 20260506 21:10:33.769 | INFO | RTMetrics report populated | 20260506 21:10:33.769 | INFO | Opening tmp RTMetrics json file | 20260506 21:10:33.839 | INFO | Serializing RTMetrics to json | 20260506 21:10:33.844 | INFO | Serialization done, swapping tmp file now... | 20260506 21:10:33.845 | INFO | RTMetrics done | 20260506 21:10:34.002 | INFO | Finished a round of Baz IO: Wrote 1.955 GiB in 3.363 seconds | 20260506 21:10:34.002 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.02% / 0.00% / 65.98% | 20260506 21:10:34.002 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.651165% / 1.19209e-05% / 7.15256e-05% | 20260506 21:10:34.002 | INFO | Baz IO overall: | 20260506 21:10:34.003 | INFO | BytesWritten : 2360411502455 | 20260506 21:10:34.003 | INFO | NumEvents : 1139872653984 | 20260506 21:10:35.663 | INFO | Analyzing chunk frames = [4476928,4477440) | 20260506 21:10:35.663 | INFO | MemInfo: { | 20260506 21:10:35.663 | INFO | "gpu" : " 17 GB", | 20260506 21:10:35.663 | INFO | "huge" : "", | 20260506 21:10:35.663 | INFO | "huge_shared" : " 27 GB", | 20260506 21:10:35.663 | INFO | "malloc" : " 14 GB", | 20260506 21:10:35.663 | INFO | "pinned" : " 29 GB" | 20260506 21:10:35.663 | INFO | } | 20260506 21:10:35.663 | INFO | Perf report for the last 5.13823 seconds: | 20260506 21:10:35.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:10:35.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:35.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:35.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:35.663 | INFO | ANALYSIS : 62.6%, 2.9, 95.7ms, 0 | 20260506 21:10:35.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:35.663 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 21:10:35.663 | INFO | PRE_HQ : 59.6%, 1.8, 56.8ms, 0 | 20260506 21:10:35.663 | INFO | BAZWRITER : 65.1%, 4.8, 159.4ms, 0 | 20260506 21:10:35.663 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 21:10:35.663 | INFO | Uploaded 36GiB to GPU in 2.04438s (17.6092GiB/s) | 20260506 21:10:35.663 | INFO | Downloaded 38GiB to GPU in 2.20645s (17.2223GiB/s) | 20260506 21:10:40.809 | INFO | Analyzing chunk frames = [4477440,4477952) | 20260506 21:10:40.809 | INFO | MemInfo: { | 20260506 21:10:40.809 | INFO | "gpu" : " 17 GB", | 20260506 21:10:40.809 | INFO | "huge" : "", | 20260506 21:10:40.809 | INFO | "huge_shared" : " 27 GB", | 20260506 21:10:40.809 | INFO | "malloc" : " 14 GB", | 20260506 21:10:40.809 | INFO | "pinned" : " 29 GB" | 20260506 21:10:40.809 | INFO | } | 20260506 21:10:40.809 | INFO | Perf report for the last 5.14631 seconds: | 20260506 21:10:40.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:10:40.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:40.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:40.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:40.810 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 21:10:40.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:40.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:40.810 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 21:10:40.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:40.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:40.810 | INFO | Uploaded 36GiB to GPU in 2.006s (17.9462GiB/s) | 20260506 21:10:40.810 | INFO | Downloaded 34GiB to GPU in 1.92218s (17.6882GiB/s) | 20260506 21:10:45.946 | INFO | Analyzing chunk frames = [4477952,4478464) | 20260506 21:10:45.946 | INFO | MemInfo: { | 20260506 21:10:45.946 | INFO | "gpu" : " 17 GB", | 20260506 21:10:45.946 | INFO | "huge" : "", | 20260506 21:10:45.947 | INFO | "huge_shared" : " 27 GB", | 20260506 21:10:45.947 | INFO | "malloc" : " 14 GB", | 20260506 21:10:45.947 | INFO | "pinned" : " 29 GB" | 20260506 21:10:45.947 | INFO | } | 20260506 21:10:45.947 | INFO | Perf report for the last 5.13719 seconds: | 20260506 21:10:45.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:10:45.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:45.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:45.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:45.947 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 21:10:45.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:45.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:45.947 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:10:45.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:45.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:45.947 | INFO | Uploaded 36GiB to GPU in 1.99452s (18.0494GiB/s) | 20260506 21:10:45.947 | INFO | Downloaded 34GiB to GPU in 1.92281s (17.6825GiB/s) | 20260506 21:10:51.082 | INFO | Analyzing chunk frames = [4478464,4478976) | 20260506 21:10:51.082 | INFO | MemInfo: { | 20260506 21:10:51.082 | INFO | "gpu" : " 17 GB", | 20260506 21:10:51.082 | INFO | "huge" : "", | 20260506 21:10:51.082 | INFO | "huge_shared" : " 27 GB", | 20260506 21:10:51.082 | INFO | "malloc" : " 14 GB", | 20260506 21:10:51.082 | INFO | "pinned" : " 29 GB" | 20260506 21:10:51.082 | INFO | } | 20260506 21:10:51.082 | INFO | Perf report for the last 5.13557 seconds: | 20260506 21:10:51.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:10:51.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:51.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:51.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:51.082 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:10:51.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:51.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:51.082 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:10:51.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:51.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:51.082 | INFO | Uploaded 36GiB to GPU in 1.97437s (18.2337GiB/s) | 20260506 21:10:51.082 | INFO | Downloaded 34GiB to GPU in 1.87524s (18.131GiB/s) | 20260506 21:10:56.223 | INFO | Analyzing chunk frames = [4478976,4479488) | 20260506 21:10:56.223 | INFO | MemInfo: { | 20260506 21:10:56.223 | INFO | "gpu" : " 17 GB", | 20260506 21:10:56.223 | INFO | "huge" : "", | 20260506 21:10:56.224 | INFO | "huge_shared" : " 27 GB", | 20260506 21:10:56.224 | INFO | "malloc" : " 14 GB", | 20260506 21:10:56.224 | INFO | "pinned" : " 29 GB" | 20260506 21:10:56.224 | INFO | } | 20260506 21:10:56.224 | INFO | Perf report for the last 5.14136 seconds: | 20260506 21:10:56.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:10:56.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:56.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:56.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:56.224 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 21:10:56.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:56.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:56.224 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:10:56.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:56.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:10:56.224 | INFO | Uploaded 36GiB to GPU in 2.00947s (17.9152GiB/s) | 20260506 21:10:56.224 | INFO | Downloaded 34GiB to GPU in 1.91222s (17.7804GiB/s) | 20260506 21:11:01.359 | INFO | Analyzing chunk frames = [4479488,4480000) | 20260506 21:11:01.359 | INFO | MemInfo: { | 20260506 21:11:01.359 | INFO | "gpu" : " 17 GB", | 20260506 21:11:01.359 | INFO | "huge" : "", | 20260506 21:11:01.359 | INFO | "huge_shared" : " 27 GB", | 20260506 21:11:01.359 | INFO | "malloc" : " 14 GB", | 20260506 21:11:01.359 | INFO | "pinned" : " 29 GB" | 20260506 21:11:01.359 | INFO | } | 20260506 21:11:01.359 | INFO | Perf report for the last 5.13517 seconds: | 20260506 21:11:01.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:11:01.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:01.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:01.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:01.359 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 21:11:01.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:01.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:01.359 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 21:11:01.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:01.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:01.359 | INFO | Uploaded 36GiB to GPU in 2.01418s (17.8733GiB/s) | 20260506 21:11:01.359 | INFO | Downloaded 34GiB to GPU in 1.95098s (17.4271GiB/s) | 20260506 21:11:06.502 | INFO | Analyzing chunk frames = [4480000,4480512) | 20260506 21:11:06.502 | INFO | MemInfo: { | 20260506 21:11:06.502 | INFO | "gpu" : " 17 GB", | 20260506 21:11:06.502 | INFO | "huge" : "", | 20260506 21:11:06.502 | INFO | "huge_shared" : " 27 GB", | 20260506 21:11:06.502 | INFO | "malloc" : " 14 GB", | 20260506 21:11:06.502 | INFO | "pinned" : " 29 GB" | 20260506 21:11:06.502 | INFO | } | 20260506 21:11:06.502 | INFO | Perf report for the last 5.14301 seconds: | 20260506 21:11:06.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:11:06.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:06.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:06.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:06.502 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 21:11:06.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:06.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:06.502 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:11:06.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:06.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:06.502 | INFO | Uploaded 36GiB to GPU in 1.96909s (18.2825GiB/s) | 20260506 21:11:06.502 | INFO | Downloaded 34GiB to GPU in 2.00507s (16.957GiB/s) | 20260506 21:11:11.646 | INFO | Analyzing chunk frames = [4480512,4481024) | 20260506 21:11:11.646 | INFO | MemInfo: { | 20260506 21:11:11.646 | INFO | "gpu" : " 17 GB", | 20260506 21:11:11.646 | INFO | "huge" : "", | 20260506 21:11:11.646 | INFO | "huge_shared" : " 27 GB", | 20260506 21:11:11.646 | INFO | "malloc" : " 14 GB", | 20260506 21:11:11.646 | INFO | "pinned" : " 29 GB" | 20260506 21:11:11.646 | INFO | } | 20260506 21:11:11.646 | INFO | Perf report for the last 5.14451 seconds: | 20260506 21:11:11.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:11:11.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:11.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:11.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:11.646 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:11:11.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:11.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:11.646 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 21:11:11.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:11.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:11.647 | INFO | Uploaded 36GiB to GPU in 1.9997s (18.0027GiB/s) | 20260506 21:11:11.647 | INFO | Downloaded 34GiB to GPU in 1.9757s (17.2091GiB/s) | 20260506 21:11:11.647 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:11:11.647 | INFO | Total mem usage: 18 GiB | 20260506 21:11:11.647 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:11:11.647 | INFO | Recent High Water Marks Sum: 16.3 GiB (Savings: -10.3%) | 20260506 21:11:11.647 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:11:11.647 | INFO | High Water Breakdown: | 20260506 21:11:11.647 | INFO | 62.2% : 10.2 GiB ( 88.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:11:11.647 | INFO | 37.7% : 6.16 GiB ( 92% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:11:11.647 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:11:11.647 | INFO | ------------------------------------------ | 20260506 21:11:11.647 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:11:11.647 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:11:11.647 | INFO | Total mem usage: 26.5 GiB | 20260506 21:11:11.647 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:11:11.647 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:11:11.647 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:11:11.647 | INFO | High Water Breakdown: | 20260506 21:11:11.647 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:11:11.647 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:11:11.647 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:11:11.647 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:11:11.647 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:11:11.647 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:11:11.647 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:11:11.647 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:11:11.647 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:11:11.647 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:11:11.647 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:11:11.647 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:11:11.647 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:11:11.647 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:11:11.647 | INFO | ------------------------------------------ | 20260506 21:11:11.647 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:11:11.647 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:11:11.647 | INFO | Total mem usage: 25.6 GiB | 20260506 21:11:11.647 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:11:11.647 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:11:11.647 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:11:11.647 | INFO | High Water Breakdown: | 20260506 21:11:11.647 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:11:11.647 | INFO | ------------------------------------------ | 20260506 21:11:11.647 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:11:11.647 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:11:11.647 | INFO | Total mem usage: 15.7 GiB | 20260506 21:11:11.647 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:11:11.647 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:11:11.647 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:11:11.647 | INFO | High Water Breakdown: | 20260506 21:11:11.647 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:11:11.647 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:11:11.647 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:11:11.647 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:11:11.647 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:11:11.647 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:11:11.647 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:11:11.647 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:11:11.647 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:11:11.647 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:11:11.647 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:11:11.647 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:11:11.647 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:11:11.647 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:11:11.647 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:11:11.647 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:11:11.647 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:11:11.647 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:11:11.647 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:11:11.647 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:11:11.647 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:11:11.647 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:11:11.647 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:11:11.647 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:11:11.647 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:11:11.647 | INFO | ------------------------------------------ | 20260506 21:11:11.647 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:11:11.647 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:11:11.647 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:11:11.647 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:11:11.647 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.62e+01, 1.36e+01 / 2.66e+00 | 20260506 21:11:11.647 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 9.83e-03 / 5.31e-03 | 20260506 21:11:11.647 | INFO | PerfReport: 11.63%, Baseline -- 3.62e+00 / 4.38e+00, 5.52e-01 / 3.28e-01 | 20260506 21:11:11.647 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.45e+00, 1.83e-01 / 1.16e-01 | 20260506 21:11:11.647 | INFO | PerfReport: 16.09%, DME -- 4.01e+01 / 4.53e+01, 4.43e+00 / 2.40e+00 | 20260506 21:11:11.647 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.40e+00 / 1.77e+00 | 20260506 21:11:11.648 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.21e+00 / 2.74e+00, 3.78e-01 / 2.00e-01 | 20260506 21:11:11.648 | INFO | PerfReport: 3.28%, Metrics -- 1.02e+00 / 1.36e+00, 2.71e-01 / 1.69e-01 | 20260506 21:11:11.648 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.50e-01 / 3.98e-01, 5.47e-02 / 2.20e-02 | 20260506 21:11:11.648 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:11:11.648 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:11:11.648 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:11:11.648 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 21:11:11.648 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260506 21:11:11.648 | INFO | PerfReport: 21.25%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 21:11:11.648 | INFO | PerfReport: 32.21%, Compute -- 3.12e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 21:11:11.648 | INFO | PerfReport: 1.25%, LockUpload -- 1.21e+00 / 0.00e+00, 4.73e+00 / 0.00e+00 | 20260506 21:11:11.648 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260506 21:11:11.648 | INFO | PerfReport: 22.54%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:11:11.648 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260506 21:11:11.752 | INFO | Starting a round of Baz IO | 20260506 21:11:14.895 | INFO | Populating RTMetrics report | 20260506 21:11:14.895 | INFO | RTMetrics report populated | 20260506 21:11:14.895 | INFO | Opening tmp RTMetrics json file | 20260506 21:11:14.973 | INFO | Serializing RTMetrics to json | 20260506 21:11:14.997 | INFO | Serialization done, swapping tmp file now... | 20260506 21:11:15.005 | INFO | RTMetrics done | 20260506 21:11:15.027 | INFO | Finished a round of Baz IO: Wrote 1.956 GiB in 3.274 seconds | 20260506 21:11:15.027 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.01% / 0.00% / 65.98% | 20260506 21:11:15.027 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.650398% / 0% / 6.75519e-05% | 20260506 21:11:15.027 | INFO | Baz IO overall: | 20260506 21:11:15.027 | INFO | BytesWritten : 2362511309436 | 20260506 21:11:15.027 | INFO | NumEvents : 1140878830577 | 20260506 21:11:16.781 | INFO | Analyzing chunk frames = [4481024,4481536) | 20260506 21:11:16.782 | INFO | MemInfo: { | 20260506 21:11:16.782 | INFO | "gpu" : " 17 GB", | 20260506 21:11:16.782 | INFO | "huge" : "", | 20260506 21:11:16.782 | INFO | "huge_shared" : " 27 GB", | 20260506 21:11:16.782 | INFO | "malloc" : " 14 GB", | 20260506 21:11:16.782 | INFO | "pinned" : " 29 GB" | 20260506 21:11:16.782 | INFO | } | 20260506 21:11:16.782 | INFO | Perf report for the last 5.1353 seconds: | 20260506 21:11:16.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:11:16.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:16.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:16.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:16.782 | INFO | ANALYSIS : 62.6%, 2.9, 95.2ms, 0 | 20260506 21:11:16.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:16.782 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 21:11:16.782 | INFO | PRE_HQ : 59.9%, 1.9, 57.6ms, 0 | 20260506 21:11:16.782 | INFO | BAZWRITER : 63.5%, 4.8, 156.2ms, 0 | 20260506 21:11:16.782 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 21:11:16.782 | INFO | Uploaded 36GiB to GPU in 2.06773s (17.4104GiB/s) | 20260506 21:11:16.782 | INFO | Downloaded 38GiB to GPU in 2.21485s (17.1569GiB/s) | 20260506 21:11:21.921 | INFO | Analyzing chunk frames = [4481536,4482048) | 20260506 21:11:21.921 | INFO | MemInfo: { | 20260506 21:11:21.921 | INFO | "gpu" : " 17 GB", | 20260506 21:11:21.921 | INFO | "huge" : "", | 20260506 21:11:21.921 | INFO | "huge_shared" : " 27 GB", | 20260506 21:11:21.921 | INFO | "malloc" : " 14 GB", | 20260506 21:11:21.921 | INFO | "pinned" : " 29 GB" | 20260506 21:11:21.921 | INFO | } | 20260506 21:11:21.921 | INFO | Perf report for the last 5.13916 seconds: | 20260506 21:11:21.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:11:21.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:21.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:21.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:21.921 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:11:21.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:21.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:21.921 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:11:21.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:21.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:21.921 | INFO | Uploaded 36GiB to GPU in 1.99678s (18.029GiB/s) | 20260506 21:11:21.921 | INFO | Downloaded 34GiB to GPU in 1.93567s (17.565GiB/s) | 20260506 21:11:27.063 | INFO | Analyzing chunk frames = [4482048,4482560) | 20260506 21:11:27.063 | INFO | MemInfo: { | 20260506 21:11:27.063 | INFO | "gpu" : " 17 GB", | 20260506 21:11:27.063 | INFO | "huge" : "", | 20260506 21:11:27.063 | INFO | "huge_shared" : " 27 GB", | 20260506 21:11:27.063 | INFO | "malloc" : " 14 GB", | 20260506 21:11:27.063 | INFO | "pinned" : " 29 GB" | 20260506 21:11:27.063 | INFO | } | 20260506 21:11:27.063 | INFO | Perf report for the last 5.1424 seconds: | 20260506 21:11:27.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:11:27.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:27.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:27.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:27.063 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 21:11:27.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:27.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:27.063 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:11:27.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:27.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:27.063 | INFO | Uploaded 36GiB to GPU in 1.96252s (18.3438GiB/s) | 20260506 21:11:27.063 | INFO | Downloaded 34GiB to GPU in 1.92719s (17.6423GiB/s) | 20260506 21:11:32.200 | INFO | Analyzing chunk frames = [4482560,4483072) | 20260506 21:11:32.200 | INFO | MemInfo: { | 20260506 21:11:32.200 | INFO | "gpu" : " 17 GB", | 20260506 21:11:32.200 | INFO | "huge" : "", | 20260506 21:11:32.200 | INFO | "huge_shared" : " 27 GB", | 20260506 21:11:32.200 | INFO | "malloc" : " 14 GB", | 20260506 21:11:32.200 | INFO | "pinned" : " 29 GB" | 20260506 21:11:32.200 | INFO | } | 20260506 21:11:32.200 | INFO | Perf report for the last 5.13682 seconds: | 20260506 21:11:32.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:11:32.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:32.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:32.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:32.200 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:11:32.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:32.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:32.200 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:11:32.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:32.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:32.200 | INFO | Uploaded 36GiB to GPU in 1.96359s (18.3337GiB/s) | 20260506 21:11:32.200 | INFO | Downloaded 34GiB to GPU in 1.87807s (18.1037GiB/s) | 20260506 21:11:37.337 | INFO | Analyzing chunk frames = [4483072,4483584) | 20260506 21:11:37.337 | INFO | MemInfo: { | 20260506 21:11:37.337 | INFO | "gpu" : " 17 GB", | 20260506 21:11:37.337 | INFO | "huge" : "", | 20260506 21:11:37.337 | INFO | "huge_shared" : " 27 GB", | 20260506 21:11:37.337 | INFO | "malloc" : " 14 GB", | 20260506 21:11:37.337 | INFO | "pinned" : " 29 GB" | 20260506 21:11:37.337 | INFO | } | 20260506 21:11:37.338 | INFO | Perf report for the last 5.13749 seconds: | 20260506 21:11:37.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:11:37.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:37.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:37.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:37.338 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 21:11:37.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:37.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:37.338 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:11:37.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:37.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:37.338 | INFO | Uploaded 36GiB to GPU in 1.98094s (18.1732GiB/s) | 20260506 21:11:37.338 | INFO | Downloaded 34GiB to GPU in 1.92022s (17.7063GiB/s) | 20260506 21:11:42.474 | INFO | Analyzing chunk frames = [4483584,4484096) | 20260506 21:11:42.474 | INFO | MemInfo: { | 20260506 21:11:42.474 | INFO | "gpu" : " 17 GB", | 20260506 21:11:42.474 | INFO | "huge" : "", | 20260506 21:11:42.474 | INFO | "huge_shared" : " 27 GB", | 20260506 21:11:42.475 | INFO | "malloc" : " 14 GB", | 20260506 21:11:42.475 | INFO | "pinned" : " 29 GB" | 20260506 21:11:42.475 | INFO | } | 20260506 21:11:42.475 | INFO | Perf report for the last 5.13701 seconds: | 20260506 21:11:42.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:11:42.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:42.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:42.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:42.475 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 21:11:42.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:42.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:42.475 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 21:11:42.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:42.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:42.475 | INFO | Uploaded 36GiB to GPU in 2.01027s (17.9081GiB/s) | 20260506 21:11:42.475 | INFO | Downloaded 34GiB to GPU in 1.94943s (17.441GiB/s) | 20260506 21:11:47.614 | INFO | Analyzing chunk frames = [4484096,4484608) | 20260506 21:11:47.614 | INFO | MemInfo: { | 20260506 21:11:47.614 | INFO | "gpu" : " 17 GB", | 20260506 21:11:47.614 | INFO | "huge" : "", | 20260506 21:11:47.614 | INFO | "huge_shared" : " 27 GB", | 20260506 21:11:47.614 | INFO | "malloc" : " 14 GB", | 20260506 21:11:47.614 | INFO | "pinned" : " 29 GB" | 20260506 21:11:47.614 | INFO | } | 20260506 21:11:47.614 | INFO | Perf report for the last 5.13977 seconds: | 20260506 21:11:47.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:11:47.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:47.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:47.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:47.614 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 21:11:47.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:47.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:47.614 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 21:11:47.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:47.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:47.615 | INFO | Uploaded 36GiB to GPU in 1.97755s (18.2043GiB/s) | 20260506 21:11:47.615 | INFO | Downloaded 34GiB to GPU in 2.00739s (16.9375GiB/s) | 20260506 21:11:52.756 | INFO | Analyzing chunk frames = [4484608,4485120) | 20260506 21:11:52.756 | INFO | MemInfo: { | 20260506 21:11:52.756 | INFO | "gpu" : " 17 GB", | 20260506 21:11:52.756 | INFO | "huge" : "", | 20260506 21:11:52.756 | INFO | "huge_shared" : " 27 GB", | 20260506 21:11:52.756 | INFO | "malloc" : " 14 GB", | 20260506 21:11:52.756 | INFO | "pinned" : " 29 GB" | 20260506 21:11:52.756 | INFO | } | 20260506 21:11:52.756 | INFO | Perf report for the last 5.14172 seconds: | 20260506 21:11:52.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:11:52.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:52.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:52.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:52.756 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:11:52.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:52.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:52.756 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:11:52.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:52.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:52.756 | INFO | Uploaded 36GiB to GPU in 1.99791s (18.0189GiB/s) | 20260506 21:11:52.756 | INFO | Downloaded 34GiB to GPU in 1.98738s (17.108GiB/s) | 20260506 21:11:52.854 | INFO | Starting a round of Baz IO | 20260506 21:11:55.995 | INFO | Populating RTMetrics report | 20260506 21:11:55.995 | INFO | RTMetrics report populated | 20260506 21:11:55.995 | INFO | Opening tmp RTMetrics json file | 20260506 21:11:56.030 | INFO | Serializing RTMetrics to json | 20260506 21:11:56.060 | INFO | Serialization done, swapping tmp file now... | 20260506 21:11:56.063 | INFO | RTMetrics done | 20260506 21:11:56.196 | INFO | Finished a round of Baz IO: Wrote 1.956 GiB in 3.341 seconds | 20260506 21:11:56.196 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.01% / 0.00% / 65.99% | 20260506 21:11:56.196 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.650732% / 7.94729e-06% / 9.53674e-05% | 20260506 21:11:56.196 | INFO | Baz IO overall: | 20260506 21:11:56.196 | INFO | BytesWritten : 2364612029187 | 20260506 21:11:56.196 | INFO | NumEvents : 1141885819518 | 20260506 21:11:57.900 | INFO | Analyzing chunk frames = [4485120,4485632) | 20260506 21:11:57.900 | INFO | MemInfo: { | 20260506 21:11:57.900 | INFO | "gpu" : " 17 GB", | 20260506 21:11:57.900 | INFO | "huge" : "", | 20260506 21:11:57.900 | INFO | "huge_shared" : " 27 GB", | 20260506 21:11:57.900 | INFO | "malloc" : " 14 GB", | 20260506 21:11:57.900 | INFO | "pinned" : " 29 GB" | 20260506 21:11:57.900 | INFO | } | 20260506 21:11:57.900 | INFO | Perf report for the last 5.14404 seconds: | 20260506 21:11:57.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:11:57.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:57.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:57.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:57.900 | INFO | ANALYSIS : 62.4%, 2.9, 95.3ms, 0 | 20260506 21:11:57.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:11:57.900 | INFO | RT_METRICS : 43.9%, 1.0, 23.1ms, 0 | 20260506 21:11:57.900 | INFO | PRE_HQ : 59.7%, 1.8, 56.9ms, 0 | 20260506 21:11:57.900 | INFO | BAZWRITER : 64.6%, 4.8, 158.1ms, 0 | 20260506 21:11:57.900 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 21:11:57.900 | INFO | Uploaded 36GiB to GPU in 2.02979s (17.7358GiB/s) | 20260506 21:11:57.900 | INFO | Downloaded 38GiB to GPU in 2.19632s (17.3017GiB/s) | 20260506 21:12:03.037 | INFO | Analyzing chunk frames = [4485632,4486144) | 20260506 21:12:03.037 | INFO | MemInfo: { | 20260506 21:12:03.037 | INFO | "gpu" : " 17 GB", | 20260506 21:12:03.037 | INFO | "huge" : "", | 20260506 21:12:03.037 | INFO | "huge_shared" : " 27 GB", | 20260506 21:12:03.037 | INFO | "malloc" : " 14 GB", | 20260506 21:12:03.037 | INFO | "pinned" : " 29 GB" | 20260506 21:12:03.037 | INFO | } | 20260506 21:12:03.037 | INFO | Perf report for the last 5.13699 seconds: | 20260506 21:12:03.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:12:03.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:03.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:03.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:03.037 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 21:12:03.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:03.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:03.037 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:12:03.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:03.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:03.037 | INFO | Uploaded 36GiB to GPU in 1.98571s (18.1296GiB/s) | 20260506 21:12:03.037 | INFO | Downloaded 34GiB to GPU in 1.93s (17.6166GiB/s) | 20260506 21:12:08.175 | INFO | Analyzing chunk frames = [4486144,4486656) | 20260506 21:12:08.175 | INFO | MemInfo: { | 20260506 21:12:08.175 | INFO | "gpu" : " 17 GB", | 20260506 21:12:08.175 | INFO | "huge" : "", | 20260506 21:12:08.175 | INFO | "huge_shared" : " 27 GB", | 20260506 21:12:08.175 | INFO | "malloc" : " 14 GB", | 20260506 21:12:08.175 | INFO | "pinned" : " 29 GB" | 20260506 21:12:08.175 | INFO | } | 20260506 21:12:08.175 | INFO | Perf report for the last 5.13774 seconds: | 20260506 21:12:08.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:12:08.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:08.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:08.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:08.175 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 21:12:08.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:08.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:08.175 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:12:08.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:08.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:08.175 | INFO | Uploaded 36GiB to GPU in 1.98207s (18.1629GiB/s) | 20260506 21:12:08.175 | INFO | Downloaded 34GiB to GPU in 1.93174s (17.6007GiB/s) | 20260506 21:12:13.316 | INFO | Analyzing chunk frames = [4486656,4487168) | 20260506 21:12:13.316 | INFO | MemInfo: { | 20260506 21:12:13.316 | INFO | "gpu" : " 17 GB", | 20260506 21:12:13.316 | INFO | "huge" : "", | 20260506 21:12:13.316 | INFO | "huge_shared" : " 27 GB", | 20260506 21:12:13.316 | INFO | "malloc" : " 14 GB", | 20260506 21:12:13.316 | INFO | "pinned" : " 29 GB" | 20260506 21:12:13.316 | INFO | } | 20260506 21:12:13.316 | INFO | Perf report for the last 5.1412 seconds: | 20260506 21:12:13.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:12:13.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:13.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:13.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:13.316 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 21:12:13.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:13.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:13.316 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:12:13.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:13.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:13.316 | INFO | Uploaded 36GiB to GPU in 1.94894s (18.4716GiB/s) | 20260506 21:12:13.316 | INFO | Downloaded 34GiB to GPU in 1.87478s (18.1355GiB/s) | 20260506 21:12:18.449 | INFO | Analyzing chunk frames = [4487168,4487680) | 20260506 21:12:18.449 | INFO | MemInfo: { | 20260506 21:12:18.449 | INFO | "gpu" : " 17 GB", | 20260506 21:12:18.449 | INFO | "huge" : "", | 20260506 21:12:18.449 | INFO | "huge_shared" : " 27 GB", | 20260506 21:12:18.449 | INFO | "malloc" : " 14 GB", | 20260506 21:12:18.449 | INFO | "pinned" : " 29 GB" | 20260506 21:12:18.449 | INFO | } | 20260506 21:12:18.449 | INFO | Perf report for the last 5.13327 seconds: | 20260506 21:12:18.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:12:18.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:18.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:18.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:18.449 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 21:12:18.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:18.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:18.449 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:12:18.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:18.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:18.450 | INFO | Uploaded 36GiB to GPU in 2.00031s (17.9972GiB/s) | 20260506 21:12:18.450 | INFO | Downloaded 34GiB to GPU in 1.92363s (17.6749GiB/s) | 20260506 21:12:23.587 | INFO | Analyzing chunk frames = [4487680,4488192) | 20260506 21:12:23.587 | INFO | MemInfo: { | 20260506 21:12:23.587 | INFO | "gpu" : " 17 GB", | 20260506 21:12:23.587 | INFO | "huge" : "", | 20260506 21:12:23.587 | INFO | "huge_shared" : " 27 GB", | 20260506 21:12:23.587 | INFO | "malloc" : " 14 GB", | 20260506 21:12:23.587 | INFO | "pinned" : " 29 GB" | 20260506 21:12:23.587 | INFO | } | 20260506 21:12:23.587 | INFO | Perf report for the last 5.13736 seconds: | 20260506 21:12:23.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:12:23.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:23.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:23.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:23.587 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 21:12:23.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:23.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:23.587 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 21:12:23.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:23.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:23.587 | INFO | Uploaded 36GiB to GPU in 2.01328s (17.8813GiB/s) | 20260506 21:12:23.587 | INFO | Downloaded 34GiB to GPU in 1.95455s (17.3953GiB/s) | 20260506 21:12:28.732 | INFO | Analyzing chunk frames = [4488192,4488704) | 20260506 21:12:28.732 | INFO | MemInfo: { | 20260506 21:12:28.732 | INFO | "gpu" : " 17 GB", | 20260506 21:12:28.732 | INFO | "huge" : "", | 20260506 21:12:28.732 | INFO | "huge_shared" : " 27 GB", | 20260506 21:12:28.732 | INFO | "malloc" : " 14 GB", | 20260506 21:12:28.732 | INFO | "pinned" : " 29 GB" | 20260506 21:12:28.732 | INFO | } | 20260506 21:12:28.732 | INFO | Perf report for the last 5.14544 seconds: | 20260506 21:12:28.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:12:28.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:28.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:28.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:28.732 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 21:12:28.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:28.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:28.732 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 21:12:28.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:28.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:28.732 | INFO | Uploaded 36GiB to GPU in 1.98272s (18.1569GiB/s) | 20260506 21:12:28.732 | INFO | Downloaded 34GiB to GPU in 2.00819s (16.9307GiB/s) | 20260506 21:12:33.873 | INFO | Analyzing chunk frames = [4488704,4489216) | 20260506 21:12:33.873 | INFO | MemInfo: { | 20260506 21:12:33.873 | INFO | "gpu" : " 17 GB", | 20260506 21:12:33.873 | INFO | "huge" : "", | 20260506 21:12:33.873 | INFO | "huge_shared" : " 27 GB", | 20260506 21:12:33.873 | INFO | "malloc" : " 14 GB", | 20260506 21:12:33.873 | INFO | "pinned" : " 29 GB" | 20260506 21:12:33.873 | INFO | } | 20260506 21:12:33.873 | INFO | Perf report for the last 5.14111 seconds: | 20260506 21:12:33.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:12:33.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:33.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:33.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:33.873 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 21:12:33.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:33.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:33.873 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 21:12:33.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:33.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:33.874 | INFO | Uploaded 36GiB to GPU in 1.99006s (18.0899GiB/s) | 20260506 21:12:33.874 | INFO | Downloaded 34GiB to GPU in 1.98916s (17.0926GiB/s) | 20260506 21:12:33.874 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:12:33.874 | INFO | Total mem usage: 18 GiB | 20260506 21:12:33.874 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:12:33.874 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -14.8%) | 20260506 21:12:33.874 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:12:33.874 | INFO | High Water Breakdown: | 20260506 21:12:33.874 | INFO | 61.6% : 9.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:12:33.874 | INFO | 38.3% : 6.01 GiB ( 89.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:12:33.874 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:12:33.874 | INFO | ------------------------------------------ | 20260506 21:12:33.874 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:12:33.874 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:12:33.874 | INFO | Total mem usage: 26.5 GiB | 20260506 21:12:33.874 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:12:33.874 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:12:33.874 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:12:33.874 | INFO | High Water Breakdown: | 20260506 21:12:33.874 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:12:33.874 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:12:33.874 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:12:33.874 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:12:33.874 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:12:33.874 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:12:33.874 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:12:33.874 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:12:33.874 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:12:33.874 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:12:33.874 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:12:33.874 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:12:33.874 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:12:33.874 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:12:33.874 | INFO | ------------------------------------------ | 20260506 21:12:33.874 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:12:33.874 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:12:33.874 | INFO | Total mem usage: 25.6 GiB | 20260506 21:12:33.874 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:12:33.874 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:12:33.874 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:12:33.874 | INFO | High Water Breakdown: | 20260506 21:12:33.874 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:12:33.874 | INFO | ------------------------------------------ | 20260506 21:12:33.874 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:12:33.874 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:12:33.874 | INFO | Total mem usage: 15.7 GiB | 20260506 21:12:33.874 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:12:33.874 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:12:33.874 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:12:33.874 | INFO | High Water Breakdown: | 20260506 21:12:33.874 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:12:33.874 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:12:33.874 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:12:33.874 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:12:33.874 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:12:33.874 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:12:33.874 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:12:33.874 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:12:33.874 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:12:33.874 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:12:33.874 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:12:33.874 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:12:33.874 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:12:33.874 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:12:33.874 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:12:33.874 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:12:33.874 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:12:33.874 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:12:33.874 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:12:33.874 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:12:33.874 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:12:33.874 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:12:33.874 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:12:33.874 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:12:33.874 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:12:33.874 | INFO | ------------------------------------------ | 20260506 21:12:33.874 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:12:33.874 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:12:33.874 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:12:33.874 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:12:33.874 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.59e+01, 1.31e+01 / 2.66e+00 | 20260506 21:12:33.874 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.03e-01, 1.08e-02 / 8.05e-03 | 20260506 21:12:33.875 | INFO | PerfReport: 11.63%, Baseline -- 3.62e+00 / 4.36e+00, 5.52e-01 / 3.83e-01 | 20260506 21:12:33.875 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.47e+00, 1.84e-01 / 1.45e-01 | 20260506 21:12:33.875 | INFO | PerfReport: 15.97%, DME -- 3.98e+01 / 4.41e+01, 4.02e+00 / 2.98e+00 | 20260506 21:12:33.875 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.78e+01 / 2.13e+01, 1.41e+00 / 1.98e+00 | 20260506 21:12:33.875 | INFO | PerfReport: 7.14%, PulseAccumulating -- 2.22e+00 / 2.75e+00, 3.82e-01 / 2.39e-01 | 20260506 21:12:33.875 | INFO | PerfReport: 3.28%, Metrics -- 1.02e+00 / 1.36e+00, 2.70e-01 / 1.50e-01 | 20260506 21:12:33.875 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.50e-01 / 3.98e-01, 5.40e-02 / 2.18e-02 | 20260506 21:12:33.875 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:12:33.875 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:12:33.875 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:12:33.875 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.31e+01 / 0.00e+00 | 20260506 21:12:33.875 | INFO | PerfReport: 21.49%, Upload -- 2.08e+01 / 0.00e+00, 6.82e+00 / 0.00e+00 | 20260506 21:12:33.875 | INFO | PerfReport: 21.32%, Download -- 2.06e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 21:12:33.875 | INFO | PerfReport: 32.21%, Compute -- 3.11e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 21:12:33.875 | INFO | PerfReport: 1.17%, LockUpload -- 1.13e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260506 21:12:33.875 | INFO | PerfReport: 1.30%, LockDownload -- 1.26e+00 / 0.00e+00, 3.15e+00 / 0.00e+00 | 20260506 21:12:33.875 | INFO | PerfReport: 22.49%, LockCompute -- 2.17e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 21:12:33.875 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260506 21:12:33.971 | INFO | Starting a round of Baz IO | 20260506 21:12:37.105 | INFO | Populating RTMetrics report | 20260506 21:12:37.106 | INFO | RTMetrics report populated | 20260506 21:12:37.106 | INFO | Opening tmp RTMetrics json file | 20260506 21:12:37.108 | INFO | Serializing RTMetrics to json | 20260506 21:12:37.118 | INFO | Serialization done, swapping tmp file now... | 20260506 21:12:37.145 | INFO | RTMetrics done | 20260506 21:12:37.233 | INFO | Finished a round of Baz IO: Wrote 1.953 GiB in 3.261 seconds | 20260506 21:12:37.233 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.01% / 0.00% / 65.99% | 20260506 21:12:37.233 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.64439% / 1.19209e-05% / 7.15256e-05% | 20260506 21:12:37.233 | INFO | Baz IO overall: | 20260506 21:12:37.233 | INFO | BytesWritten : 2366709322885 | 20260506 21:12:37.233 | INFO | NumEvents : 1142890084850 | 20260506 21:12:39.014 | INFO | Analyzing chunk frames = [4489216,4489728) | 20260506 21:12:39.014 | INFO | MemInfo: { | 20260506 21:12:39.014 | INFO | "gpu" : " 17 GB", | 20260506 21:12:39.014 | INFO | "huge" : "", | 20260506 21:12:39.014 | INFO | "huge_shared" : " 27 GB", | 20260506 21:12:39.014 | INFO | "malloc" : " 14 GB", | 20260506 21:12:39.014 | INFO | "pinned" : " 29 GB" | 20260506 21:12:39.014 | INFO | } | 20260506 21:12:39.014 | INFO | Perf report for the last 5.14067 seconds: | 20260506 21:12:39.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:12:39.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:39.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:39.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:39.014 | INFO | ANALYSIS : 62.3%, 2.9, 95.2ms, 0 | 20260506 21:12:39.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:39.014 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 21:12:39.014 | INFO | PRE_HQ : 59.7%, 1.8, 56.9ms, 0 | 20260506 21:12:39.014 | INFO | BAZWRITER : 63.1%, 4.7, 151.6ms, 0 | 20260506 21:12:39.014 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 21:12:39.014 | INFO | Uploaded 36GiB to GPU in 2.02955s (17.7379GiB/s) | 20260506 21:12:39.014 | INFO | Downloaded 38GiB to GPU in 2.19922s (17.2789GiB/s) | 20260506 21:12:44.154 | INFO | Analyzing chunk frames = [4489728,4490240) | 20260506 21:12:44.154 | INFO | MemInfo: { | 20260506 21:12:44.154 | INFO | "gpu" : " 17 GB", | 20260506 21:12:44.154 | INFO | "huge" : "", | 20260506 21:12:44.154 | INFO | "huge_shared" : " 27 GB", | 20260506 21:12:44.154 | INFO | "malloc" : " 14 GB", | 20260506 21:12:44.154 | INFO | "pinned" : " 29 GB" | 20260506 21:12:44.154 | INFO | } | 20260506 21:12:44.154 | INFO | Perf report for the last 5.13993 seconds: | 20260506 21:12:44.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:12:44.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:44.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:44.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:44.154 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 21:12:44.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:44.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:44.154 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:12:44.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:44.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:44.154 | INFO | Uploaded 36GiB to GPU in 1.991s (18.0814GiB/s) | 20260506 21:12:44.154 | INFO | Downloaded 34GiB to GPU in 1.92693s (17.6446GiB/s) | 20260506 21:12:49.289 | INFO | Analyzing chunk frames = [4490240,4490752) | 20260506 21:12:49.289 | INFO | MemInfo: { | 20260506 21:12:49.289 | INFO | "gpu" : " 17 GB", | 20260506 21:12:49.289 | INFO | "huge" : "", | 20260506 21:12:49.289 | INFO | "huge_shared" : " 27 GB", | 20260506 21:12:49.289 | INFO | "malloc" : " 14 GB", | 20260506 21:12:49.289 | INFO | "pinned" : " 29 GB" | 20260506 21:12:49.289 | INFO | } | 20260506 21:12:49.289 | INFO | Perf report for the last 5.13469 seconds: | 20260506 21:12:49.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:12:49.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:49.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:49.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:49.289 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 21:12:49.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:49.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:49.289 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:12:49.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:49.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:49.289 | INFO | Uploaded 36GiB to GPU in 1.9752s (18.226GiB/s) | 20260506 21:12:49.289 | INFO | Downloaded 34GiB to GPU in 1.92742s (17.6402GiB/s) | 20260506 21:12:54.423 | INFO | Analyzing chunk frames = [4490752,4491264) | 20260506 21:12:54.423 | INFO | MemInfo: { | 20260506 21:12:54.423 | INFO | "gpu" : " 17 GB", | 20260506 21:12:54.423 | INFO | "huge" : "", | 20260506 21:12:54.423 | INFO | "huge_shared" : " 27 GB", | 20260506 21:12:54.423 | INFO | "malloc" : " 14 GB", | 20260506 21:12:54.423 | INFO | "pinned" : " 29 GB" | 20260506 21:12:54.423 | INFO | } | 20260506 21:12:54.423 | INFO | Perf report for the last 5.1343 seconds: | 20260506 21:12:54.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:12:54.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:54.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:54.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:54.423 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 21:12:54.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:54.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:54.423 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 21:12:54.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:54.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:54.423 | INFO | Uploaded 36GiB to GPU in 1.96201s (18.3485GiB/s) | 20260506 21:12:54.423 | INFO | Downloaded 34GiB to GPU in 1.87838s (18.1007GiB/s) | 20260506 21:12:59.562 | INFO | Analyzing chunk frames = [4491264,4491776) | 20260506 21:12:59.562 | INFO | MemInfo: { | 20260506 21:12:59.562 | INFO | "gpu" : " 17 GB", | 20260506 21:12:59.562 | INFO | "huge" : "", | 20260506 21:12:59.562 | INFO | "huge_shared" : " 27 GB", | 20260506 21:12:59.562 | INFO | "malloc" : " 14 GB", | 20260506 21:12:59.562 | INFO | "pinned" : " 29 GB" | 20260506 21:12:59.562 | INFO | } | 20260506 21:12:59.562 | INFO | Perf report for the last 5.13883 seconds: | 20260506 21:12:59.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:12:59.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:59.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:59.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:59.562 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 21:12:59.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:59.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:59.562 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 21:12:59.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:59.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:12:59.562 | INFO | Uploaded 36GiB to GPU in 2.01258s (17.8875GiB/s) | 20260506 21:12:59.562 | INFO | Downloaded 34GiB to GPU in 1.91898s (17.7178GiB/s) | 20260506 21:13:04.707 | INFO | Analyzing chunk frames = [4491776,4492288) | 20260506 21:13:04.707 | INFO | MemInfo: { | 20260506 21:13:04.707 | INFO | "gpu" : " 17 GB", | 20260506 21:13:04.707 | INFO | "huge" : "", | 20260506 21:13:04.707 | INFO | "huge_shared" : " 27 GB", | 20260506 21:13:04.707 | INFO | "malloc" : " 14 GB", | 20260506 21:13:04.707 | INFO | "pinned" : " 29 GB" | 20260506 21:13:04.707 | INFO | } | 20260506 21:13:04.707 | INFO | Perf report for the last 5.14482 seconds: | 20260506 21:13:04.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:13:04.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:04.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:04.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:04.707 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 21:13:04.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:04.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:04.707 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:13:04.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:04.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:04.707 | INFO | Uploaded 36GiB to GPU in 1.99133s (18.0784GiB/s) | 20260506 21:13:04.707 | INFO | Downloaded 34GiB to GPU in 1.95912s (17.3547GiB/s) | 20260506 21:13:09.845 | INFO | Analyzing chunk frames = [4492288,4492800) | 20260506 21:13:09.845 | INFO | MemInfo: { | 20260506 21:13:09.845 | INFO | "gpu" : " 17 GB", | 20260506 21:13:09.845 | INFO | "huge" : "", | 20260506 21:13:09.845 | INFO | "huge_shared" : " 27 GB", | 20260506 21:13:09.846 | INFO | "malloc" : " 14 GB", | 20260506 21:13:09.846 | INFO | "pinned" : " 29 GB" | 20260506 21:13:09.846 | INFO | } | 20260506 21:13:09.846 | INFO | Perf report for the last 5.1389 seconds: | 20260506 21:13:09.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:13:09.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:09.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:09.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:09.846 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:13:09.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:09.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:09.846 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 21:13:09.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:09.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:09.846 | INFO | Uploaded 36GiB to GPU in 1.97537s (18.2244GiB/s) | 20260506 21:13:09.846 | INFO | Downloaded 34GiB to GPU in 2.01051s (16.9111GiB/s) | 20260506 21:13:14.990 | INFO | Analyzing chunk frames = [4492800,4493312) | 20260506 21:13:14.990 | INFO | MemInfo: { | 20260506 21:13:14.990 | INFO | "gpu" : " 17 GB", | 20260506 21:13:14.990 | INFO | "huge" : "", | 20260506 21:13:14.990 | INFO | "huge_shared" : " 27 GB", | 20260506 21:13:14.990 | INFO | "malloc" : " 14 GB", | 20260506 21:13:14.990 | INFO | "pinned" : " 29 GB" | 20260506 21:13:14.990 | INFO | } | 20260506 21:13:14.990 | INFO | Perf report for the last 5.14446 seconds: | 20260506 21:13:14.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:13:14.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:14.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:14.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:14.990 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 21:13:14.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:14.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:14.990 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:13:14.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:14.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:14.990 | INFO | Uploaded 36GiB to GPU in 1.98885s (18.1009GiB/s) | 20260506 21:13:14.990 | INFO | Downloaded 34GiB to GPU in 1.97682s (17.1993GiB/s) | 20260506 21:13:15.088 | INFO | Starting a round of Baz IO | 20260506 21:13:18.229 | INFO | Populating RTMetrics report | 20260506 21:13:18.229 | INFO | RTMetrics report populated | 20260506 21:13:18.229 | INFO | Opening tmp RTMetrics json file | 20260506 21:13:18.249 | INFO | Serializing RTMetrics to json | 20260506 21:13:18.250 | INFO | Serialization done, swapping tmp file now... | 20260506 21:13:18.255 | INFO | RTMetrics done | 20260506 21:13:18.405 | INFO | Finished a round of Baz IO: Wrote 1.952 GiB in 3.316 seconds | 20260506 21:13:18.405 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.00% / 0.00% / 65.99% | 20260506 21:13:18.405 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.638803% / 1.19209e-05% / 5.5631e-05% | 20260506 21:13:18.405 | INFO | Baz IO overall: | 20260506 21:13:18.405 | INFO | BytesWritten : 2368805055558 | 20260506 21:13:18.405 | INFO | NumEvents : 1143892926617 | 20260506 21:13:20.130 | INFO | Analyzing chunk frames = [4493312,4493824) | 20260506 21:13:20.130 | INFO | MemInfo: { | 20260506 21:13:20.130 | INFO | "gpu" : " 17 GB", | 20260506 21:13:20.130 | INFO | "huge" : "", | 20260506 21:13:20.130 | INFO | "huge_shared" : " 27 GB", | 20260506 21:13:20.130 | INFO | "malloc" : " 14 GB", | 20260506 21:13:20.130 | INFO | "pinned" : " 29 GB" | 20260506 21:13:20.130 | INFO | } | 20260506 21:13:20.130 | INFO | Perf report for the last 5.14011 seconds: | 20260506 21:13:20.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:13:20.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:20.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:20.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:20.130 | INFO | ANALYSIS : 62.5%, 2.9, 95.2ms, 0 | 20260506 21:13:20.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:20.130 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 21:13:20.130 | INFO | PRE_HQ : 59.9%, 1.8, 57.1ms, 0 | 20260506 21:13:20.130 | INFO | BAZWRITER : 64.3%, 4.8, 157.3ms, 0 | 20260506 21:13:20.130 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 21:13:20.130 | INFO | Uploaded 36GiB to GPU in 2.06526s (17.4312GiB/s) | 20260506 21:13:20.130 | INFO | Downloaded 38GiB to GPU in 2.20408s (17.2408GiB/s) | 20260506 21:13:25.267 | INFO | Analyzing chunk frames = [4493824,4494336) | 20260506 21:13:25.267 | INFO | MemInfo: { | 20260506 21:13:25.267 | INFO | "gpu" : " 17 GB", | 20260506 21:13:25.267 | INFO | "huge" : "", | 20260506 21:13:25.267 | INFO | "huge_shared" : " 27 GB", | 20260506 21:13:25.267 | INFO | "malloc" : " 14 GB", | 20260506 21:13:25.267 | INFO | "pinned" : " 29 GB" | 20260506 21:13:25.267 | INFO | } | 20260506 21:13:25.267 | INFO | Perf report for the last 5.13665 seconds: | 20260506 21:13:25.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:13:25.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:25.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:25.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:25.267 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 21:13:25.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:25.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:25.267 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:13:25.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:25.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:25.267 | INFO | Uploaded 36GiB to GPU in 1.99822s (18.0161GiB/s) | 20260506 21:13:25.267 | INFO | Downloaded 34GiB to GPU in 1.92392s (17.6722GiB/s) | 20260506 21:13:30.403 | INFO | Analyzing chunk frames = [4494336,4494848) | 20260506 21:13:30.403 | INFO | MemInfo: { | 20260506 21:13:30.403 | INFO | "gpu" : " 17 GB", | 20260506 21:13:30.403 | INFO | "huge" : "", | 20260506 21:13:30.403 | INFO | "huge_shared" : " 27 GB", | 20260506 21:13:30.403 | INFO | "malloc" : " 14 GB", | 20260506 21:13:30.403 | INFO | "pinned" : " 29 GB" | 20260506 21:13:30.403 | INFO | } | 20260506 21:13:30.403 | INFO | Perf report for the last 5.13633 seconds: | 20260506 21:13:30.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:13:30.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:30.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:30.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:30.403 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260506 21:13:30.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:30.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:30.403 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:13:30.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:30.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:30.403 | INFO | Uploaded 36GiB to GPU in 1.98862s (18.103GiB/s) | 20260506 21:13:30.403 | INFO | Downloaded 34GiB to GPU in 1.91754s (17.7311GiB/s) | 20260506 21:13:35.538 | INFO | Analyzing chunk frames = [4494848,4495360) | 20260506 21:13:35.538 | INFO | MemInfo: { | 20260506 21:13:35.538 | INFO | "gpu" : " 17 GB", | 20260506 21:13:35.538 | INFO | "huge" : "", | 20260506 21:13:35.538 | INFO | "huge_shared" : " 27 GB", | 20260506 21:13:35.538 | INFO | "malloc" : " 14 GB", | 20260506 21:13:35.538 | INFO | "pinned" : " 29 GB" | 20260506 21:13:35.538 | INFO | } | 20260506 21:13:35.538 | INFO | Perf report for the last 5.13468 seconds: | 20260506 21:13:35.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:13:35.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:35.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:35.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:35.538 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 21:13:35.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:35.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:35.538 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:13:35.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:35.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:35.538 | INFO | Uploaded 36GiB to GPU in 1.94108s (18.5464GiB/s) | 20260506 21:13:35.538 | INFO | Downloaded 34GiB to GPU in 1.87115s (18.1707GiB/s) | 20260506 21:13:40.680 | INFO | Analyzing chunk frames = [4495360,4495872) | 20260506 21:13:40.680 | INFO | MemInfo: { | 20260506 21:13:40.680 | INFO | "gpu" : " 17 GB", | 20260506 21:13:40.680 | INFO | "huge" : "", | 20260506 21:13:40.680 | INFO | "huge_shared" : " 27 GB", | 20260506 21:13:40.680 | INFO | "malloc" : " 14 GB", | 20260506 21:13:40.680 | INFO | "pinned" : " 29 GB" | 20260506 21:13:40.680 | INFO | } | 20260506 21:13:40.680 | INFO | Perf report for the last 5.14244 seconds: | 20260506 21:13:40.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:13:40.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:40.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:40.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:40.680 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260506 21:13:40.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:40.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:40.680 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 21:13:40.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:40.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:40.681 | INFO | Uploaded 36GiB to GPU in 2.01356s (17.8788GiB/s) | 20260506 21:13:40.681 | INFO | Downloaded 34GiB to GPU in 1.91296s (17.7735GiB/s) | 20260506 21:13:45.826 | INFO | Analyzing chunk frames = [4495872,4496384) | 20260506 21:13:45.826 | INFO | MemInfo: { | 20260506 21:13:45.826 | INFO | "gpu" : " 17 GB", | 20260506 21:13:45.826 | INFO | "huge" : "", | 20260506 21:13:45.826 | INFO | "huge_shared" : " 27 GB", | 20260506 21:13:45.826 | INFO | "malloc" : " 14 GB", | 20260506 21:13:45.826 | INFO | "pinned" : " 29 GB" | 20260506 21:13:45.826 | INFO | } | 20260506 21:13:45.826 | INFO | Perf report for the last 5.1454 seconds: | 20260506 21:13:45.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:13:45.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:45.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:45.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:45.826 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260506 21:13:45.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:45.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:45.826 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 21:13:45.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:45.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:45.826 | INFO | Uploaded 36GiB to GPU in 2.00206s (17.9815GiB/s) | 20260506 21:13:45.826 | INFO | Downloaded 34GiB to GPU in 1.94928s (17.4423GiB/s) | 20260506 21:13:50.962 | INFO | Analyzing chunk frames = [4496384,4496896) | 20260506 21:13:50.962 | INFO | MemInfo: { | 20260506 21:13:50.962 | INFO | "gpu" : " 17 GB", | 20260506 21:13:50.962 | INFO | "huge" : "", | 20260506 21:13:50.962 | INFO | "huge_shared" : " 27 GB", | 20260506 21:13:50.962 | INFO | "malloc" : " 14 GB", | 20260506 21:13:50.962 | INFO | "pinned" : " 29 GB" | 20260506 21:13:50.962 | INFO | } | 20260506 21:13:50.962 | INFO | Perf report for the last 5.13661 seconds: | 20260506 21:13:50.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:13:50.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:50.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:50.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:50.962 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 21:13:50.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:50.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:50.962 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 21:13:50.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:50.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:50.963 | INFO | Uploaded 36GiB to GPU in 1.99777s (18.0201GiB/s) | 20260506 21:13:50.963 | INFO | Downloaded 34GiB to GPU in 1.99333s (17.0569GiB/s) | 20260506 21:13:56.101 | INFO | Analyzing chunk frames = [4496896,4497408) | 20260506 21:13:56.101 | INFO | MemInfo: { | 20260506 21:13:56.101 | INFO | "gpu" : " 17 GB", | 20260506 21:13:56.101 | INFO | "huge" : "", | 20260506 21:13:56.101 | INFO | "huge_shared" : " 27 GB", | 20260506 21:13:56.101 | INFO | "malloc" : " 14 GB", | 20260506 21:13:56.101 | INFO | "pinned" : " 29 GB" | 20260506 21:13:56.101 | INFO | } | 20260506 21:13:56.101 | INFO | Perf report for the last 5.1389 seconds: | 20260506 21:13:56.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:13:56.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:56.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:56.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:56.101 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 21:13:56.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:56.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:56.101 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 21:13:56.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:56.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:13:56.101 | INFO | Uploaded 36GiB to GPU in 1.999s (18.009GiB/s) | 20260506 21:13:56.101 | INFO | Downloaded 34GiB to GPU in 1.99078s (17.0787GiB/s) | 20260506 21:13:56.102 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:13:56.102 | INFO | Total mem usage: 18 GiB | 20260506 21:13:56.102 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:13:56.102 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -17.3%) | 20260506 21:13:56.102 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:13:56.102 | INFO | High Water Breakdown: | 20260506 21:13:56.102 | INFO | 61.2% : 9.4 GiB ( 82.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:13:56.102 | INFO | 38.6% : 5.93 GiB ( 88.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:13:56.102 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:13:56.102 | INFO | ------------------------------------------ | 20260506 21:13:56.102 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:13:56.102 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:13:56.102 | INFO | Total mem usage: 26.5 GiB | 20260506 21:13:56.102 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:13:56.102 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:13:56.102 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:13:56.102 | INFO | High Water Breakdown: | 20260506 21:13:56.102 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:13:56.102 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:13:56.102 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:13:56.102 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:13:56.102 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:13:56.102 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:13:56.102 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:13:56.102 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:13:56.102 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:13:56.102 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:13:56.102 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:13:56.102 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:13:56.102 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:13:56.102 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:13:56.102 | INFO | ------------------------------------------ | 20260506 21:13:56.102 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:13:56.102 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:13:56.102 | INFO | Total mem usage: 25.6 GiB | 20260506 21:13:56.102 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:13:56.102 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:13:56.102 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:13:56.102 | INFO | High Water Breakdown: | 20260506 21:13:56.102 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:13:56.102 | INFO | ------------------------------------------ | 20260506 21:13:56.102 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:13:56.102 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:13:56.102 | INFO | Total mem usage: 15.7 GiB | 20260506 21:13:56.102 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:13:56.102 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:13:56.102 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:13:56.102 | INFO | High Water Breakdown: | 20260506 21:13:56.102 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:13:56.102 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:13:56.102 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:13:56.102 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:13:56.102 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:13:56.102 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:13:56.102 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:13:56.102 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:13:56.102 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:13:56.102 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:13:56.102 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:13:56.102 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:13:56.102 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:13:56.102 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:13:56.102 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:13:56.102 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:13:56.102 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:13:56.102 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:13:56.102 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:13:56.102 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:13:56.102 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:13:56.102 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:13:56.102 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:13:56.102 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:13:56.102 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:13:56.102 | INFO | ------------------------------------------ | 20260506 21:13:56.102 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:13:56.102 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:13:56.102 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:13:56.102 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:13:56.102 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.57e+01, 1.34e+01 / 2.32e+00 | 20260506 21:13:56.102 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.03e-01, 9.51e-03 / 5.34e-03 | 20260506 21:13:56.102 | INFO | PerfReport: 11.64%, Baseline -- 3.62e+00 / 4.42e+00, 5.52e-01 / 3.60e-01 | 20260506 21:13:56.102 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.43e+00, 1.80e-01 / 1.38e-01 | 20260506 21:13:56.102 | INFO | PerfReport: 15.98%, DME -- 3.98e+01 / 4.40e+01, 3.94e+00 / 2.80e+00 | 20260506 21:13:56.102 | INFO | PerfReport: 57.35%, FrameLabeling -- 1.79e+01 / 2.13e+01, 1.47e+00 / 1.88e+00 | 20260506 21:13:56.102 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.21e+00 / 2.77e+00, 3.69e-01 / 2.22e-01 | 20260506 21:13:56.102 | INFO | PerfReport: 3.27%, Metrics -- 1.02e+00 / 1.33e+00, 2.71e-01 / 1.90e-01 | 20260506 21:13:56.102 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.49e-01 / 3.99e-01, 5.48e-02 / 3.10e-02 | 20260506 21:13:56.102 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:13:56.102 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:13:56.102 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:13:56.102 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 21:13:56.102 | INFO | PerfReport: 21.51%, Upload -- 2.08e+01 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260506 21:13:56.102 | INFO | PerfReport: 21.26%, Download -- 2.05e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 21:13:56.102 | INFO | PerfReport: 32.23%, Compute -- 3.11e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 21:13:56.102 | INFO | PerfReport: 1.21%, LockUpload -- 1.17e+00 / 0.00e+00, 4.65e+00 / 0.00e+00 | 20260506 21:13:56.103 | INFO | PerfReport: 1.19%, LockDownload -- 1.15e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260506 21:13:56.103 | INFO | PerfReport: 22.57%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 21:13:56.103 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260506 21:13:56.202 | INFO | Starting a round of Baz IO | 20260506 21:13:59.386 | INFO | Populating RTMetrics report | 20260506 21:13:59.387 | INFO | RTMetrics report populated | 20260506 21:13:59.387 | INFO | Opening tmp RTMetrics json file | 20260506 21:13:59.432 | INFO | Serializing RTMetrics to json | 20260506 21:13:59.482 | INFO | Serialization done, swapping tmp file now... | 20260506 21:13:59.488 | INFO | RTMetrics done | 20260506 21:13:59.673 | INFO | Finished a round of Baz IO: Wrote 1.953 GiB in 3.47 seconds | 20260506 21:13:59.673 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 34.00% / 0.00% / 66.00% | 20260506 21:13:59.673 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.639196% / 7.94729e-06% / 7.54992e-05% | 20260506 21:13:59.673 | INFO | Baz IO overall: | 20260506 21:13:59.673 | INFO | BytesWritten : 2370902584510 | 20260506 21:13:59.673 | INFO | NumEvents : 1144897749953 | 20260506 21:14:01.242 | INFO | Analyzing chunk frames = [4497408,4497920) | 20260506 21:14:01.242 | INFO | MemInfo: { | 20260506 21:14:01.242 | INFO | "gpu" : " 17 GB", | 20260506 21:14:01.242 | INFO | "huge" : "", | 20260506 21:14:01.242 | INFO | "huge_shared" : " 27 GB", | 20260506 21:14:01.242 | INFO | "malloc" : " 14 GB", | 20260506 21:14:01.242 | INFO | "pinned" : " 29 GB" | 20260506 21:14:01.242 | INFO | } | 20260506 21:14:01.242 | INFO | Perf report for the last 5.14082 seconds: | 20260506 21:14:01.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:14:01.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:01.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:01.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:01.242 | INFO | ANALYSIS : 62.7%, 2.9, 93.8ms, 0 | 20260506 21:14:01.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:01.242 | INFO | RT_METRICS : 48.7%, 1.0, 25.5ms, 0 | 20260506 21:14:01.242 | INFO | PRE_HQ : 61.4%, 1.9, 62.0ms, 0 | 20260506 21:14:01.242 | INFO | BAZWRITER : 67.1%, 4.7, 163.4ms, 0 | 20260506 21:14:01.242 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 21:14:01.242 | INFO | Uploaded 36GiB to GPU in 2.05307s (17.5347GiB/s) | 20260506 21:14:01.243 | INFO | Downloaded 38GiB to GPU in 2.18663s (17.3784GiB/s) | 20260506 21:14:06.379 | INFO | Analyzing chunk frames = [4497920,4498432) | 20260506 21:14:06.379 | INFO | MemInfo: { | 20260506 21:14:06.379 | INFO | "gpu" : " 17 GB", | 20260506 21:14:06.379 | INFO | "huge" : "", | 20260506 21:14:06.379 | INFO | "huge_shared" : " 27 GB", | 20260506 21:14:06.379 | INFO | "malloc" : " 14 GB", | 20260506 21:14:06.379 | INFO | "pinned" : " 29 GB" | 20260506 21:14:06.379 | INFO | } | 20260506 21:14:06.379 | INFO | Perf report for the last 5.1371 seconds: | 20260506 21:14:06.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:14:06.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:06.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:06.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:06.379 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:14:06.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:06.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:06.379 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:14:06.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:06.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:06.379 | INFO | Uploaded 36GiB to GPU in 2.01284s (17.8852GiB/s) | 20260506 21:14:06.379 | INFO | Downloaded 34GiB to GPU in 1.92565s (17.6564GiB/s) | 20260506 21:14:11.517 | INFO | Analyzing chunk frames = [4498432,4498944) | 20260506 21:14:11.517 | INFO | MemInfo: { | 20260506 21:14:11.517 | INFO | "gpu" : " 17 GB", | 20260506 21:14:11.517 | INFO | "huge" : "", | 20260506 21:14:11.517 | INFO | "huge_shared" : " 27 GB", | 20260506 21:14:11.517 | INFO | "malloc" : " 14 GB", | 20260506 21:14:11.517 | INFO | "pinned" : " 29 GB" | 20260506 21:14:11.517 | INFO | } | 20260506 21:14:11.517 | INFO | Perf report for the last 5.13787 seconds: | 20260506 21:14:11.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:14:11.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:11.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:11.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:11.517 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 21:14:11.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:11.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:11.517 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:14:11.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:11.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:11.517 | INFO | Uploaded 36GiB to GPU in 1.97663s (18.2128GiB/s) | 20260506 21:14:11.518 | INFO | Downloaded 34GiB to GPU in 1.92303s (17.6805GiB/s) | 20260506 21:14:16.657 | INFO | Analyzing chunk frames = [4498944,4499456) | 20260506 21:14:16.657 | INFO | MemInfo: { | 20260506 21:14:16.657 | INFO | "gpu" : " 17 GB", | 20260506 21:14:16.657 | INFO | "huge" : "", | 20260506 21:14:16.657 | INFO | "huge_shared" : " 27 GB", | 20260506 21:14:16.658 | INFO | "malloc" : " 14 GB", | 20260506 21:14:16.658 | INFO | "pinned" : " 29 GB" | 20260506 21:14:16.658 | INFO | } | 20260506 21:14:16.658 | INFO | Perf report for the last 5.14056 seconds: | 20260506 21:14:16.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:14:16.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:16.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:16.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:16.658 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:14:16.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:16.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:16.658 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:14:16.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:16.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:16.658 | INFO | Uploaded 36GiB to GPU in 1.9663s (18.3085GiB/s) | 20260506 21:14:16.658 | INFO | Downloaded 34GiB to GPU in 1.87527s (18.1307GiB/s) | 20260506 21:14:21.797 | INFO | Analyzing chunk frames = [4499456,4499968) | 20260506 21:14:21.797 | INFO | MemInfo: { | 20260506 21:14:21.797 | INFO | "gpu" : " 17 GB", | 20260506 21:14:21.797 | INFO | "huge" : "", | 20260506 21:14:21.797 | INFO | "huge_shared" : " 27 GB", | 20260506 21:14:21.797 | INFO | "malloc" : " 14 GB", | 20260506 21:14:21.797 | INFO | "pinned" : " 29 GB" | 20260506 21:14:21.797 | INFO | } | 20260506 21:14:21.797 | INFO | Perf report for the last 5.13972 seconds: | 20260506 21:14:21.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:14:21.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:21.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:21.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:21.797 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 21:14:21.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:21.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:21.797 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:14:21.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:21.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:21.798 | INFO | Uploaded 36GiB to GPU in 2.00953s (17.9146GiB/s) | 20260506 21:14:21.798 | INFO | Downloaded 34GiB to GPU in 1.91632s (17.7423GiB/s) | 20260506 21:14:26.939 | INFO | Analyzing chunk frames = [4499968,4500480) | 20260506 21:14:26.939 | INFO | MemInfo: { | 20260506 21:14:26.939 | INFO | "gpu" : " 17 GB", | 20260506 21:14:26.939 | INFO | "huge" : "", | 20260506 21:14:26.939 | INFO | "huge_shared" : " 27 GB", | 20260506 21:14:26.939 | INFO | "malloc" : " 14 GB", | 20260506 21:14:26.939 | INFO | "pinned" : " 29 GB" | 20260506 21:14:26.939 | INFO | } | 20260506 21:14:26.939 | INFO | Perf report for the last 5.14189 seconds: | 20260506 21:14:26.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:14:26.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:26.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:26.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:26.939 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 21:14:26.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:26.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:26.939 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:14:26.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:26.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:26.939 | INFO | Uploaded 36GiB to GPU in 2.02435s (17.7835GiB/s) | 20260506 21:14:26.939 | INFO | Downloaded 34GiB to GPU in 1.95847s (17.3605GiB/s) | 20260506 21:14:32.087 | INFO | Analyzing chunk frames = [4500480,4500992) | 20260506 21:14:32.087 | INFO | MemInfo: { | 20260506 21:14:32.087 | INFO | "gpu" : " 17 GB", | 20260506 21:14:32.087 | INFO | "huge" : "", | 20260506 21:14:32.087 | INFO | "huge_shared" : " 27 GB", | 20260506 21:14:32.087 | INFO | "malloc" : " 14 GB", | 20260506 21:14:32.087 | INFO | "pinned" : " 29 GB" | 20260506 21:14:32.087 | INFO | } | 20260506 21:14:32.087 | INFO | Perf report for the last 5.1478 seconds: | 20260506 21:14:32.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:14:32.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:32.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:32.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:32.087 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 21:14:32.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:32.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:32.087 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 21:14:32.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:32.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:32.087 | INFO | Uploaded 36GiB to GPU in 1.98134s (18.1695GiB/s) | 20260506 21:14:32.087 | INFO | Downloaded 34GiB to GPU in 1.99579s (17.0359GiB/s) | 20260506 21:14:37.218 | INFO | Analyzing chunk frames = [4500992,4501504) | 20260506 21:14:37.218 | INFO | MemInfo: { | 20260506 21:14:37.218 | INFO | "gpu" : " 17 GB", | 20260506 21:14:37.218 | INFO | "huge" : "", | 20260506 21:14:37.218 | INFO | "huge_shared" : " 27 GB", | 20260506 21:14:37.218 | INFO | "malloc" : " 14 GB", | 20260506 21:14:37.218 | INFO | "pinned" : " 29 GB" | 20260506 21:14:37.218 | INFO | } | 20260506 21:14:37.219 | INFO | Perf report for the last 5.13137 seconds: | 20260506 21:14:37.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:14:37.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:37.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:37.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:37.219 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 21:14:37.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:37.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:37.219 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:14:37.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:37.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:37.219 | INFO | Uploaded 36GiB to GPU in 1.97309s (18.2455GiB/s) | 20260506 21:14:37.219 | INFO | Downloaded 34GiB to GPU in 1.9955s (17.0383GiB/s) | 20260506 21:14:37.317 | INFO | Starting a round of Baz IO | 20260506 21:14:40.453 | INFO | Populating RTMetrics report | 20260506 21:14:40.454 | INFO | RTMetrics report populated | 20260506 21:14:40.454 | INFO | Opening tmp RTMetrics json file | 20260506 21:14:40.509 | INFO | Serializing RTMetrics to json | 20260506 21:14:40.516 | INFO | Serialization done, swapping tmp file now... | 20260506 21:14:40.521 | INFO | RTMetrics done | 20260506 21:14:40.750 | INFO | Finished a round of Baz IO: Wrote 1.953 GiB in 3.433 seconds | 20260506 21:14:40.750 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.99% / 0.00% / 66.00% | 20260506 21:14:40.750 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.638636% / 2.38419e-05% / 6.35783e-05% | 20260506 21:14:40.750 | INFO | Baz IO overall: | 20260506 21:14:40.750 | INFO | BytesWritten : 2372999090868 | 20260506 21:14:40.750 | INFO | NumEvents : 1145901963791 | 20260506 21:14:42.354 | INFO | Analyzing chunk frames = [4501504,4502016) | 20260506 21:14:42.354 | INFO | MemInfo: { | 20260506 21:14:42.354 | INFO | "gpu" : " 17 GB", | 20260506 21:14:42.354 | INFO | "huge" : "", | 20260506 21:14:42.354 | INFO | "huge_shared" : " 27 GB", | 20260506 21:14:42.354 | INFO | "malloc" : " 14 GB", | 20260506 21:14:42.354 | INFO | "pinned" : " 29 GB" | 20260506 21:14:42.354 | INFO | } | 20260506 21:14:42.354 | INFO | Perf report for the last 5.13573 seconds: | 20260506 21:14:42.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:14:42.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:42.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:42.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:42.354 | INFO | ANALYSIS : 62.4%, 2.9, 95.3ms, 0 | 20260506 21:14:42.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:42.354 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 21:14:42.354 | INFO | PRE_HQ : 59.9%, 1.8, 56.9ms, 0 | 20260506 21:14:42.354 | INFO | BAZWRITER : 66.5%, 4.8, 163.4ms, 0 | 20260506 21:14:42.354 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 21:14:42.354 | INFO | Uploaded 36GiB to GPU in 2.01887s (17.8318GiB/s) | 20260506 21:14:42.354 | INFO | Downloaded 38GiB to GPU in 2.18844s (17.364GiB/s) | 20260506 21:14:47.492 | INFO | Analyzing chunk frames = [4502016,4502528) | 20260506 21:14:47.492 | INFO | MemInfo: { | 20260506 21:14:47.492 | INFO | "gpu" : " 17 GB", | 20260506 21:14:47.492 | INFO | "huge" : "", | 20260506 21:14:47.492 | INFO | "huge_shared" : " 27 GB", | 20260506 21:14:47.492 | INFO | "malloc" : " 14 GB", | 20260506 21:14:47.492 | INFO | "pinned" : " 29 GB" | 20260506 21:14:47.492 | INFO | } | 20260506 21:14:47.492 | INFO | Perf report for the last 5.13793 seconds: | 20260506 21:14:47.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:14:47.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:47.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:47.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:47.492 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:14:47.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:47.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:47.492 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:14:47.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:47.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:47.492 | INFO | Uploaded 36GiB to GPU in 2.00795s (17.9287GiB/s) | 20260506 21:14:47.493 | INFO | Downloaded 34GiB to GPU in 1.93055s (17.6116GiB/s) | 20260506 21:14:52.628 | INFO | Analyzing chunk frames = [4502528,4503040) | 20260506 21:14:52.629 | INFO | MemInfo: { | 20260506 21:14:52.629 | INFO | "gpu" : " 17 GB", | 20260506 21:14:52.629 | INFO | "huge" : "", | 20260506 21:14:52.629 | INFO | "huge_shared" : " 27 GB", | 20260506 21:14:52.629 | INFO | "malloc" : " 14 GB", | 20260506 21:14:52.629 | INFO | "pinned" : " 29 GB" | 20260506 21:14:52.629 | INFO | } | 20260506 21:14:52.629 | INFO | Perf report for the last 5.13643 seconds: | 20260506 21:14:52.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:14:52.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:52.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:52.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:52.629 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 21:14:52.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:52.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:52.629 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:14:52.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:52.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:52.629 | INFO | Uploaded 36GiB to GPU in 1.97171s (18.2583GiB/s) | 20260506 21:14:52.629 | INFO | Downloaded 34GiB to GPU in 1.91925s (17.7153GiB/s) | 20260506 21:14:57.773 | INFO | Analyzing chunk frames = [4503040,4503552) | 20260506 21:14:57.773 | INFO | MemInfo: { | 20260506 21:14:57.773 | INFO | "gpu" : " 17 GB", | 20260506 21:14:57.773 | INFO | "huge" : "", | 20260506 21:14:57.773 | INFO | "huge_shared" : " 27 GB", | 20260506 21:14:57.773 | INFO | "malloc" : " 14 GB", | 20260506 21:14:57.773 | INFO | "pinned" : " 29 GB" | 20260506 21:14:57.773 | INFO | } | 20260506 21:14:57.773 | INFO | Perf report for the last 5.14405 seconds: | 20260506 21:14:57.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:14:57.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:57.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:57.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:57.773 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 21:14:57.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:57.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:57.773 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:14:57.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:57.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:14:57.773 | INFO | Uploaded 36GiB to GPU in 1.94737s (18.4864GiB/s) | 20260506 21:14:57.773 | INFO | Downloaded 34GiB to GPU in 1.87464s (18.1368GiB/s) | 20260506 21:15:02.911 | INFO | Analyzing chunk frames = [4503552,4504064) | 20260506 21:15:02.911 | INFO | MemInfo: { | 20260506 21:15:02.911 | INFO | "gpu" : " 17 GB", | 20260506 21:15:02.911 | INFO | "huge" : "", | 20260506 21:15:02.911 | INFO | "huge_shared" : " 27 GB", | 20260506 21:15:02.911 | INFO | "malloc" : " 14 GB", | 20260506 21:15:02.912 | INFO | "pinned" : " 29 GB" | 20260506 21:15:02.912 | INFO | } | 20260506 21:15:02.912 | INFO | Perf report for the last 5.13884 seconds: | 20260506 21:15:02.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:15:02.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:02.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:02.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:02.912 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 21:15:02.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:02.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:02.912 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 21:15:02.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:02.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:02.912 | INFO | Uploaded 36GiB to GPU in 1.971s (18.2648GiB/s) | 20260506 21:15:02.912 | INFO | Downloaded 34GiB to GPU in 1.9325s (17.5938GiB/s) | 20260506 21:15:08.052 | INFO | Analyzing chunk frames = [4504064,4504576) | 20260506 21:15:08.052 | INFO | MemInfo: { | 20260506 21:15:08.052 | INFO | "gpu" : " 17 GB", | 20260506 21:15:08.052 | INFO | "huge" : "", | 20260506 21:15:08.052 | INFO | "huge_shared" : " 27 GB", | 20260506 21:15:08.052 | INFO | "malloc" : " 14 GB", | 20260506 21:15:08.052 | INFO | "pinned" : " 29 GB" | 20260506 21:15:08.052 | INFO | } | 20260506 21:15:08.052 | INFO | Perf report for the last 5.14021 seconds: | 20260506 21:15:08.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:15:08.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:08.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:08.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:08.052 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 21:15:08.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:08.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:08.052 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 21:15:08.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:08.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:08.052 | INFO | Uploaded 36GiB to GPU in 1.98898s (18.0997GiB/s) | 20260506 21:15:08.052 | INFO | Downloaded 34GiB to GPU in 1.9455s (17.4762GiB/s) | 20260506 21:15:13.190 | INFO | Analyzing chunk frames = [4504576,4505088) | 20260506 21:15:13.190 | INFO | MemInfo: { | 20260506 21:15:13.190 | INFO | "gpu" : " 17 GB", | 20260506 21:15:13.190 | INFO | "huge" : "", | 20260506 21:15:13.190 | INFO | "huge_shared" : " 27 GB", | 20260506 21:15:13.190 | INFO | "malloc" : " 14 GB", | 20260506 21:15:13.190 | INFO | "pinned" : " 29 GB" | 20260506 21:15:13.190 | INFO | } | 20260506 21:15:13.190 | INFO | Perf report for the last 5.13821 seconds: | 20260506 21:15:13.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:15:13.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:13.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:13.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:13.190 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 21:15:13.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:13.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:13.190 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 21:15:13.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:13.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:13.190 | INFO | Uploaded 36GiB to GPU in 1.96955s (18.2782GiB/s) | 20260506 21:15:13.190 | INFO | Downloaded 34GiB to GPU in 2.00699s (16.9408GiB/s) | 20260506 21:15:18.333 | INFO | Analyzing chunk frames = [4505088,4505600) | 20260506 21:15:18.333 | INFO | MemInfo: { | 20260506 21:15:18.333 | INFO | "gpu" : " 17 GB", | 20260506 21:15:18.333 | INFO | "huge" : "", | 20260506 21:15:18.333 | INFO | "huge_shared" : " 27 GB", | 20260506 21:15:18.333 | INFO | "malloc" : " 14 GB", | 20260506 21:15:18.333 | INFO | "pinned" : " 29 GB" | 20260506 21:15:18.333 | INFO | } | 20260506 21:15:18.333 | INFO | Perf report for the last 5.14305 seconds: | 20260506 21:15:18.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:15:18.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:18.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:18.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:18.333 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 21:15:18.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:18.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:18.333 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:15:18.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:18.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:18.333 | INFO | Uploaded 36GiB to GPU in 2.01597s (17.8574GiB/s) | 20260506 21:15:18.333 | INFO | Downloaded 34GiB to GPU in 1.99838s (17.0138GiB/s) | 20260506 21:15:18.333 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:15:18.333 | INFO | Total mem usage: 18 GiB | 20260506 21:15:18.333 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:15:18.334 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -14%) | 20260506 21:15:18.334 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:15:18.334 | INFO | High Water Breakdown: | 20260506 21:15:18.334 | INFO | 61.7% : 9.74 GiB ( 85.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:15:18.334 | INFO | 38.2% : 6.03 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:15:18.334 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:15:18.334 | INFO | ------------------------------------------ | 20260506 21:15:18.334 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:15:18.334 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:15:18.334 | INFO | Total mem usage: 26.5 GiB | 20260506 21:15:18.334 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:15:18.334 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:15:18.334 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:15:18.334 | INFO | High Water Breakdown: | 20260506 21:15:18.334 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:15:18.334 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:15:18.334 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:15:18.334 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:15:18.334 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:15:18.334 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:15:18.334 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:15:18.334 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:15:18.334 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:15:18.334 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:15:18.334 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:15:18.334 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:15:18.334 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:15:18.334 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:15:18.334 | INFO | ------------------------------------------ | 20260506 21:15:18.334 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:15:18.334 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:15:18.334 | INFO | Total mem usage: 25.6 GiB | 20260506 21:15:18.334 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:15:18.334 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:15:18.334 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:15:18.334 | INFO | High Water Breakdown: | 20260506 21:15:18.334 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:15:18.334 | INFO | ------------------------------------------ | 20260506 21:15:18.334 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:15:18.334 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:15:18.334 | INFO | Total mem usage: 15.7 GiB | 20260506 21:15:18.334 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:15:18.334 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:15:18.334 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:15:18.334 | INFO | High Water Breakdown: | 20260506 21:15:18.334 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:15:18.334 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:15:18.334 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:15:18.334 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:15:18.334 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:15:18.334 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:15:18.334 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:15:18.334 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:15:18.334 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:15:18.334 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:15:18.334 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:15:18.334 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:15:18.334 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:15:18.334 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:15:18.334 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:15:18.334 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:15:18.334 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:15:18.334 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:15:18.334 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:15:18.334 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:15:18.334 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:15:18.334 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:15:18.334 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:15:18.334 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:15:18.334 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:15:18.334 | INFO | ------------------------------------------ | 20260506 21:15:18.334 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:15:18.334 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:15:18.334 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:15:18.334 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:15:18.334 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.60e+01, 1.32e+01 / 2.55e+00 | 20260506 21:15:18.334 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 1.02e-02 / 4.74e-03 | 20260506 21:15:18.334 | INFO | PerfReport: 11.64%, Baseline -- 3.62e+00 / 4.38e+00, 5.50e-01 / 3.59e-01 | 20260506 21:15:18.334 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.47e+00, 2.17e-01 / 1.27e-01 | 20260506 21:15:18.334 | INFO | PerfReport: 16.04%, DME -- 3.99e+01 / 4.49e+01, 4.15e+00 / 2.81e+00 | 20260506 21:15:18.334 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.38e+00 / 1.59e+00 | 20260506 21:15:18.334 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.22e+00 / 2.78e+00, 3.80e-01 / 2.21e-01 | 20260506 21:15:18.334 | INFO | PerfReport: 3.28%, Metrics -- 1.02e+00 / 1.38e+00, 2.70e-01 / 1.37e-01 | 20260506 21:15:18.334 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.49e-01 / 4.04e-01, 5.42e-02 / 1.83e-02 | 20260506 21:15:18.334 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:15:18.334 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:15:18.334 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:15:18.334 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 21:15:18.334 | INFO | PerfReport: 21.51%, Upload -- 2.08e+01 / 0.00e+00, 6.87e+00 / 0.00e+00 | 20260506 21:15:18.334 | INFO | PerfReport: 21.29%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 21:15:18.334 | INFO | PerfReport: 32.22%, Compute -- 3.11e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 21:15:18.334 | INFO | PerfReport: 1.23%, LockUpload -- 1.19e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260506 21:15:18.334 | INFO | PerfReport: 1.26%, LockDownload -- 1.21e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260506 21:15:18.334 | INFO | PerfReport: 22.47%, LockCompute -- 2.17e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 21:15:18.334 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260506 21:15:18.437 | INFO | Starting a round of Baz IO | 20260506 21:15:21.567 | INFO | Populating RTMetrics report | 20260506 21:15:21.567 | INFO | RTMetrics report populated | 20260506 21:15:21.567 | INFO | Opening tmp RTMetrics json file | 20260506 21:15:21.670 | INFO | Serializing RTMetrics to json | 20260506 21:15:21.675 | INFO | Serialization done, swapping tmp file now... | 20260506 21:15:21.680 | INFO | RTMetrics done | 20260506 21:15:21.900 | INFO | Finished a round of Baz IO: Wrote 1.954 GiB in 3.463 seconds | 20260506 21:15:21.900 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.99% / 0.00% / 66.01% | 20260506 21:15:21.900 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.638123% / 1.19209e-05% / 5.96046e-05% | 20260506 21:15:21.900 | INFO | Baz IO overall: | 20260506 21:15:21.900 | INFO | BytesWritten : 2375096752178 | 20260506 21:15:21.900 | INFO | NumEvents : 1146906948519 | 20260506 21:15:23.470 | INFO | Analyzing chunk frames = [4505600,4506112) | 20260506 21:15:23.470 | INFO | MemInfo: { | 20260506 21:15:23.470 | INFO | "gpu" : " 17 GB", | 20260506 21:15:23.470 | INFO | "huge" : "", | 20260506 21:15:23.470 | INFO | "huge_shared" : " 27 GB", | 20260506 21:15:23.470 | INFO | "malloc" : " 14 GB", | 20260506 21:15:23.470 | INFO | "pinned" : " 29 GB" | 20260506 21:15:23.470 | INFO | } | 20260506 21:15:23.470 | INFO | Perf report for the last 5.1369 seconds: | 20260506 21:15:23.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:15:23.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:23.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:23.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:23.470 | INFO | ANALYSIS : 62.4%, 2.9, 95.1ms, 0 | 20260506 21:15:23.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:23.470 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 21:15:23.470 | INFO | PRE_HQ : 60.0%, 1.8, 57.2ms, 0 | 20260506 21:15:23.470 | INFO | BAZWRITER : 67.2%, 4.8, 165.0ms, 0 | 20260506 21:15:23.470 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.1ms, 0 | 20260506 21:15:23.470 | INFO | Uploaded 36GiB to GPU in 2.02029s (17.8192GiB/s) | 20260506 21:15:23.470 | INFO | Downloaded 38GiB to GPU in 2.20274s (17.2512GiB/s) | 20260506 21:15:28.609 | INFO | Analyzing chunk frames = [4506112,4506624) | 20260506 21:15:28.609 | INFO | MemInfo: { | 20260506 21:15:28.609 | INFO | "gpu" : " 17 GB", | 20260506 21:15:28.609 | INFO | "huge" : "", | 20260506 21:15:28.609 | INFO | "huge_shared" : " 27 GB", | 20260506 21:15:28.609 | INFO | "malloc" : " 14 GB", | 20260506 21:15:28.609 | INFO | "pinned" : " 29 GB" | 20260506 21:15:28.609 | INFO | } | 20260506 21:15:28.609 | INFO | Perf report for the last 5.13877 seconds: | 20260506 21:15:28.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:15:28.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:28.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:28.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:28.609 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 21:15:28.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:28.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:28.609 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 21:15:28.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:28.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:28.609 | INFO | Uploaded 36GiB to GPU in 1.99018s (18.0888GiB/s) | 20260506 21:15:28.609 | INFO | Downloaded 34GiB to GPU in 1.92913s (17.6245GiB/s) | 20260506 21:15:33.749 | INFO | Analyzing chunk frames = [4506624,4507136) | 20260506 21:15:33.749 | INFO | MemInfo: { | 20260506 21:15:33.749 | INFO | "gpu" : " 17 GB", | 20260506 21:15:33.749 | INFO | "huge" : "", | 20260506 21:15:33.749 | INFO | "huge_shared" : " 27 GB", | 20260506 21:15:33.749 | INFO | "malloc" : " 14 GB", | 20260506 21:15:33.749 | INFO | "pinned" : " 29 GB" | 20260506 21:15:33.749 | INFO | } | 20260506 21:15:33.749 | INFO | Perf report for the last 5.14069 seconds: | 20260506 21:15:33.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:15:33.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:33.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:33.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:33.750 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 21:15:33.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:33.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:33.750 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:15:33.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:33.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:33.750 | INFO | Uploaded 36GiB to GPU in 1.96558s (18.3152GiB/s) | 20260506 21:15:33.750 | INFO | Downloaded 34GiB to GPU in 1.93005s (17.6161GiB/s) | 20260506 21:15:38.887 | INFO | Analyzing chunk frames = [4507136,4507648) | 20260506 21:15:38.887 | INFO | MemInfo: { | 20260506 21:15:38.887 | INFO | "gpu" : " 17 GB", | 20260506 21:15:38.887 | INFO | "huge" : "", | 20260506 21:15:38.887 | INFO | "huge_shared" : " 27 GB", | 20260506 21:15:38.887 | INFO | "malloc" : " 14 GB", | 20260506 21:15:38.887 | INFO | "pinned" : " 29 GB" | 20260506 21:15:38.887 | INFO | } | 20260506 21:15:38.887 | INFO | Perf report for the last 5.13776 seconds: | 20260506 21:15:38.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:15:38.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:38.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:38.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:38.887 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:15:38.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:38.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:38.887 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:15:38.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:38.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:38.887 | INFO | Uploaded 36GiB to GPU in 1.93893s (18.5669GiB/s) | 20260506 21:15:38.887 | INFO | Downloaded 34GiB to GPU in 1.87637s (18.1201GiB/s) | 20260506 21:15:44.030 | INFO | Analyzing chunk frames = [4507648,4508160) | 20260506 21:15:44.030 | INFO | MemInfo: { | 20260506 21:15:44.030 | INFO | "gpu" : " 17 GB", | 20260506 21:15:44.030 | INFO | "huge" : "", | 20260506 21:15:44.030 | INFO | "huge_shared" : " 27 GB", | 20260506 21:15:44.030 | INFO | "malloc" : " 14 GB", | 20260506 21:15:44.030 | INFO | "pinned" : " 29 GB" | 20260506 21:15:44.030 | INFO | } | 20260506 21:15:44.030 | INFO | Perf report for the last 5.1428 seconds: | 20260506 21:15:44.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:15:44.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:44.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:44.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:44.030 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 21:15:44.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:44.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:44.030 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:15:44.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:44.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:44.030 | INFO | Uploaded 36GiB to GPU in 2.00046s (17.9959GiB/s) | 20260506 21:15:44.030 | INFO | Downloaded 34GiB to GPU in 1.91252s (17.7775GiB/s) | 20260506 21:15:49.171 | INFO | Analyzing chunk frames = [4508160,4508672) | 20260506 21:15:49.171 | INFO | MemInfo: { | 20260506 21:15:49.171 | INFO | "gpu" : " 17 GB", | 20260506 21:15:49.171 | INFO | "huge" : "", | 20260506 21:15:49.171 | INFO | "huge_shared" : " 27 GB", | 20260506 21:15:49.171 | INFO | "malloc" : " 14 GB", | 20260506 21:15:49.171 | INFO | "pinned" : " 29 GB" | 20260506 21:15:49.171 | INFO | } | 20260506 21:15:49.171 | INFO | Perf report for the last 5.14094 seconds: | 20260506 21:15:49.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:15:49.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:49.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:49.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:49.171 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 21:15:49.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:49.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:49.171 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 21:15:49.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:49.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:49.171 | INFO | Uploaded 36GiB to GPU in 2.0031s (17.9721GiB/s) | 20260506 21:15:49.171 | INFO | Downloaded 34GiB to GPU in 1.9512s (17.4252GiB/s) | 20260506 21:15:54.303 | INFO | Analyzing chunk frames = [4508672,4509184) | 20260506 21:15:54.303 | INFO | MemInfo: { | 20260506 21:15:54.304 | INFO | "gpu" : " 17 GB", | 20260506 21:15:54.304 | INFO | "huge" : "", | 20260506 21:15:54.304 | INFO | "huge_shared" : " 27 GB", | 20260506 21:15:54.304 | INFO | "malloc" : " 14 GB", | 20260506 21:15:54.304 | INFO | "pinned" : " 29 GB" | 20260506 21:15:54.304 | INFO | } | 20260506 21:15:54.304 | INFO | Perf report for the last 5.13262 seconds: | 20260506 21:15:54.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:15:54.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:54.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:54.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:54.304 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 21:15:54.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:54.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:54.304 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 21:15:54.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:54.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:54.304 | INFO | Uploaded 36GiB to GPU in 1.94125s (18.5448GiB/s) | 20260506 21:15:54.304 | INFO | Downloaded 34GiB to GPU in 1.99153s (17.0723GiB/s) | 20260506 21:15:59.443 | INFO | Analyzing chunk frames = [4509184,4509696) | 20260506 21:15:59.443 | INFO | MemInfo: { | 20260506 21:15:59.443 | INFO | "gpu" : " 17 GB", | 20260506 21:15:59.443 | INFO | "huge" : "", | 20260506 21:15:59.443 | INFO | "huge_shared" : " 27 GB", | 20260506 21:15:59.443 | INFO | "malloc" : " 14 GB", | 20260506 21:15:59.443 | INFO | "pinned" : " 29 GB" | 20260506 21:15:59.443 | INFO | } | 20260506 21:15:59.443 | INFO | Perf report for the last 5.13907 seconds: | 20260506 21:15:59.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:15:59.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:59.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:59.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:59.443 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 21:15:59.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:59.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:59.443 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:15:59.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:59.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:15:59.443 | INFO | Uploaded 36GiB to GPU in 1.9987s (18.0118GiB/s) | 20260506 21:15:59.443 | INFO | Downloaded 34GiB to GPU in 1.98474s (17.1307GiB/s) | 20260506 21:15:59.544 | INFO | Starting a round of Baz IO | 20260506 21:16:02.690 | INFO | Populating RTMetrics report | 20260506 21:16:02.691 | INFO | RTMetrics report populated | 20260506 21:16:02.691 | INFO | Opening tmp RTMetrics json file | 20260506 21:16:02.692 | INFO | Serializing RTMetrics to json | 20260506 21:16:02.694 | INFO | Serialization done, swapping tmp file now... | 20260506 21:16:02.694 | INFO | RTMetrics done | 20260506 21:16:03.257 | INFO | Finished a round of Baz IO: Wrote 1.951 GiB in 3.713 seconds | 20260506 21:16:03.257 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.99% / 0.00% / 66.01% | 20260506 21:16:03.258 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.635596% / 3.97364e-06% / 7.54992e-05% | 20260506 21:16:03.258 | INFO | Baz IO overall: | 20260506 21:16:03.258 | INFO | BytesWritten : 2377191533569 | 20260506 21:16:03.258 | INFO | NumEvents : 1147909860790 | 20260506 21:16:04.585 | INFO | Analyzing chunk frames = [4509696,4510208) | 20260506 21:16:04.585 | INFO | MemInfo: { | 20260506 21:16:04.585 | INFO | "gpu" : " 17 GB", | 20260506 21:16:04.585 | INFO | "huge" : "", | 20260506 21:16:04.585 | INFO | "huge_shared" : " 27 GB", | 20260506 21:16:04.585 | INFO | "malloc" : " 14 GB", | 20260506 21:16:04.585 | INFO | "pinned" : " 29 GB" | 20260506 21:16:04.585 | INFO | } | 20260506 21:16:04.585 | INFO | Perf report for the last 5.14257 seconds: | 20260506 21:16:04.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:16:04.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:04.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:04.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:04.585 | INFO | ANALYSIS : 62.3%, 2.9, 95.2ms, 0 | 20260506 21:16:04.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:04.585 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 21:16:04.585 | INFO | PRE_HQ : 60.0%, 1.8, 56.9ms, 0 | 20260506 21:16:04.585 | INFO | BAZWRITER : 71.9%, 4.8, 176.5ms, 0 | 20260506 21:16:04.585 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 21:16:04.586 | INFO | Uploaded 36GiB to GPU in 2.06996s (17.3916GiB/s) | 20260506 21:16:04.586 | INFO | Downloaded 38GiB to GPU in 2.19904s (17.2803GiB/s) | 20260506 21:16:09.721 | INFO | Analyzing chunk frames = [4510208,4510720) | 20260506 21:16:09.721 | INFO | MemInfo: { | 20260506 21:16:09.721 | INFO | "gpu" : " 17 GB", | 20260506 21:16:09.721 | INFO | "huge" : "", | 20260506 21:16:09.721 | INFO | "huge_shared" : " 27 GB", | 20260506 21:16:09.721 | INFO | "malloc" : " 14 GB", | 20260506 21:16:09.721 | INFO | "pinned" : " 29 GB" | 20260506 21:16:09.721 | INFO | } | 20260506 21:16:09.721 | INFO | Perf report for the last 5.13603 seconds: | 20260506 21:16:09.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:16:09.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:09.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:09.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:09.721 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:16:09.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:09.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:09.721 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:16:09.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:09.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:09.722 | INFO | Uploaded 36GiB to GPU in 2.01418s (17.8733GiB/s) | 20260506 21:16:09.722 | INFO | Downloaded 34GiB to GPU in 1.92341s (17.677GiB/s) | 20260506 21:16:14.860 | INFO | Analyzing chunk frames = [4510720,4511232) | 20260506 21:16:14.860 | INFO | MemInfo: { | 20260506 21:16:14.860 | INFO | "gpu" : " 17 GB", | 20260506 21:16:14.860 | INFO | "huge" : "", | 20260506 21:16:14.860 | INFO | "huge_shared" : " 27 GB", | 20260506 21:16:14.860 | INFO | "malloc" : " 14 GB", | 20260506 21:16:14.860 | INFO | "pinned" : " 29 GB" | 20260506 21:16:14.860 | INFO | } | 20260506 21:16:14.860 | INFO | Perf report for the last 5.13843 seconds: | 20260506 21:16:14.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:16:14.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:14.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:14.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:14.860 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 21:16:14.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:14.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:14.860 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:16:14.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:14.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:14.860 | INFO | Uploaded 36GiB to GPU in 1.98403s (18.1449GiB/s) | 20260506 21:16:14.860 | INFO | Downloaded 34GiB to GPU in 1.91796s (17.7272GiB/s) | 20260506 21:16:20.001 | INFO | Analyzing chunk frames = [4511232,4511744) | 20260506 21:16:20.001 | INFO | MemInfo: { | 20260506 21:16:20.001 | INFO | "gpu" : " 17 GB", | 20260506 21:16:20.001 | INFO | "huge" : "", | 20260506 21:16:20.001 | INFO | "huge_shared" : " 27 GB", | 20260506 21:16:20.001 | INFO | "malloc" : " 14 GB", | 20260506 21:16:20.001 | INFO | "pinned" : " 29 GB" | 20260506 21:16:20.001 | INFO | } | 20260506 21:16:20.001 | INFO | Perf report for the last 5.14144 seconds: | 20260506 21:16:20.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:16:20.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:20.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:20.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:20.001 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 21:16:20.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:20.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:20.001 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:16:20.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:20.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:20.001 | INFO | Uploaded 36GiB to GPU in 1.92504s (18.7009GiB/s) | 20260506 21:16:20.001 | INFO | Downloaded 34GiB to GPU in 1.86893s (18.1922GiB/s) | 20260506 21:16:25.143 | INFO | Analyzing chunk frames = [4511744,4512256) | 20260506 21:16:25.143 | INFO | MemInfo: { | 20260506 21:16:25.143 | INFO | "gpu" : " 17 GB", | 20260506 21:16:25.143 | INFO | "huge" : "", | 20260506 21:16:25.143 | INFO | "huge_shared" : " 27 GB", | 20260506 21:16:25.143 | INFO | "malloc" : " 14 GB", | 20260506 21:16:25.143 | INFO | "pinned" : " 29 GB" | 20260506 21:16:25.143 | INFO | } | 20260506 21:16:25.143 | INFO | Perf report for the last 5.14187 seconds: | 20260506 21:16:25.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:16:25.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:25.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:25.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:25.143 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 21:16:25.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:25.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:25.143 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:16:25.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:25.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:25.143 | INFO | Uploaded 36GiB to GPU in 2.01937s (17.8273GiB/s) | 20260506 21:16:25.143 | INFO | Downloaded 34GiB to GPU in 1.91415s (17.7625GiB/s) | 20260506 21:16:30.282 | INFO | Analyzing chunk frames = [4512256,4512768) | 20260506 21:16:30.282 | INFO | MemInfo: { | 20260506 21:16:30.282 | INFO | "gpu" : " 17 GB", | 20260506 21:16:30.282 | INFO | "huge" : "", | 20260506 21:16:30.282 | INFO | "huge_shared" : " 27 GB", | 20260506 21:16:30.282 | INFO | "malloc" : " 14 GB", | 20260506 21:16:30.282 | INFO | "pinned" : " 29 GB" | 20260506 21:16:30.282 | INFO | } | 20260506 21:16:30.282 | INFO | Perf report for the last 5.13888 seconds: | 20260506 21:16:30.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:16:30.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:30.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:30.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:30.282 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 21:16:30.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:30.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:30.282 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 21:16:30.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:30.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:30.282 | INFO | Uploaded 36GiB to GPU in 2.03387s (17.7002GiB/s) | 20260506 21:16:30.282 | INFO | Downloaded 34GiB to GPU in 1.94717s (17.4613GiB/s) | 20260506 21:16:35.424 | INFO | Analyzing chunk frames = [4512768,4513280) | 20260506 21:16:35.424 | INFO | MemInfo: { | 20260506 21:16:35.424 | INFO | "gpu" : " 17 GB", | 20260506 21:16:35.424 | INFO | "huge" : "", | 20260506 21:16:35.424 | INFO | "huge_shared" : " 27 GB", | 20260506 21:16:35.424 | INFO | "malloc" : " 14 GB", | 20260506 21:16:35.424 | INFO | "pinned" : " 29 GB" | 20260506 21:16:35.424 | INFO | } | 20260506 21:16:35.424 | INFO | Perf report for the last 5.14196 seconds: | 20260506 21:16:35.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:16:35.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:35.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:35.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:35.424 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:16:35.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:35.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:35.424 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:16:35.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:35.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:35.424 | INFO | Uploaded 36GiB to GPU in 1.97332s (18.2434GiB/s) | 20260506 21:16:35.424 | INFO | Downloaded 34GiB to GPU in 2.0112s (16.9053GiB/s) | 20260506 21:16:40.560 | INFO | Analyzing chunk frames = [4513280,4513792) | 20260506 21:16:40.560 | INFO | MemInfo: { | 20260506 21:16:40.560 | INFO | "gpu" : " 17 GB", | 20260506 21:16:40.560 | INFO | "huge" : "", | 20260506 21:16:40.560 | INFO | "huge_shared" : " 27 GB", | 20260506 21:16:40.560 | INFO | "malloc" : " 14 GB", | 20260506 21:16:40.560 | INFO | "pinned" : " 29 GB" | 20260506 21:16:40.560 | INFO | } | 20260506 21:16:40.560 | INFO | Perf report for the last 5.13603 seconds: | 20260506 21:16:40.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:16:40.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:40.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:40.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:40.560 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 21:16:40.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:40.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:40.560 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 21:16:40.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:40.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:40.560 | INFO | Uploaded 36GiB to GPU in 2.03711s (17.6721GiB/s) | 20260506 21:16:40.560 | INFO | Downloaded 34GiB to GPU in 2.00263s (16.9777GiB/s) | 20260506 21:16:40.560 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:16:40.560 | INFO | Total mem usage: 18 GiB | 20260506 21:16:40.560 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:16:40.560 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.1%) | 20260506 21:16:40.560 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:16:40.560 | INFO | High Water Breakdown: | 20260506 21:16:40.560 | INFO | 61.8% : 9.84 GiB ( 86.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:16:40.560 | INFO | 38.1% : 6.06 GiB ( 90.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:16:40.560 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:16:40.560 | INFO | ------------------------------------------ | 20260506 21:16:40.560 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:16:40.560 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:16:40.561 | INFO | Total mem usage: 26.5 GiB | 20260506 21:16:40.561 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:16:40.561 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:16:40.561 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:16:40.561 | INFO | High Water Breakdown: | 20260506 21:16:40.561 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:16:40.561 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:16:40.561 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:16:40.561 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:16:40.561 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:16:40.561 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:16:40.561 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:16:40.561 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:16:40.561 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:16:40.561 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:16:40.561 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:16:40.561 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:16:40.561 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:16:40.561 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:16:40.561 | INFO | ------------------------------------------ | 20260506 21:16:40.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:16:40.561 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:16:40.561 | INFO | Total mem usage: 25.6 GiB | 20260506 21:16:40.561 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:16:40.561 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:16:40.561 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:16:40.561 | INFO | High Water Breakdown: | 20260506 21:16:40.561 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:16:40.561 | INFO | ------------------------------------------ | 20260506 21:16:40.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:16:40.561 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:16:40.561 | INFO | Total mem usage: 15.7 GiB | 20260506 21:16:40.561 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:16:40.561 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:16:40.561 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:16:40.561 | INFO | High Water Breakdown: | 20260506 21:16:40.561 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:16:40.561 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:16:40.561 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:16:40.561 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:16:40.561 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:16:40.561 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:16:40.561 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:16:40.561 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:16:40.561 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:16:40.561 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:16:40.561 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:16:40.561 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:16:40.561 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:16:40.561 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:16:40.561 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:16:40.561 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:16:40.561 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:16:40.561 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:16:40.561 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:16:40.561 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:16:40.561 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:16:40.561 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:16:40.561 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:16:40.561 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:16:40.561 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:16:40.561 | INFO | ------------------------------------------ | 20260506 21:16:40.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:16:40.561 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:16:40.561 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:16:40.561 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:16:40.561 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.61e+01, 1.32e+01 / 2.63e+00 | 20260506 21:16:40.561 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 9.98e-03 / 5.05e-03 | 20260506 21:16:40.561 | INFO | PerfReport: 11.66%, Baseline -- 3.63e+00 / 4.44e+00, 5.56e-01 / 3.20e-01 | 20260506 21:16:40.561 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.81e-01 / 1.30e-01 | 20260506 21:16:40.561 | INFO | PerfReport: 16.06%, DME -- 4.00e+01 / 4.49e+01, 4.19e+00 / 3.20e+00 | 20260506 21:16:40.561 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.37e+00 / 2.11e+00 | 20260506 21:16:40.561 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.22e+00 / 2.78e+00, 3.82e-01 / 2.16e-01 | 20260506 21:16:40.561 | INFO | PerfReport: 3.28%, Metrics -- 1.02e+00 / 1.36e+00, 2.70e-01 / 1.41e-01 | 20260506 21:16:40.561 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.49e-01 / 4.01e-01, 5.46e-02 / 2.52e-02 | 20260506 21:16:40.561 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:16:40.561 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:16:40.561 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:16:40.561 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 21:16:40.561 | INFO | PerfReport: 21.49%, Upload -- 2.08e+01 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260506 21:16:40.561 | INFO | PerfReport: 21.26%, Download -- 2.05e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 21:16:40.561 | INFO | PerfReport: 32.19%, Compute -- 3.11e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 21:16:40.561 | INFO | PerfReport: 1.28%, LockUpload -- 1.24e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260506 21:16:40.561 | INFO | PerfReport: 1.23%, LockDownload -- 1.19e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260506 21:16:40.561 | INFO | PerfReport: 22.54%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 21:16:40.561 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260506 21:16:40.674 | INFO | Starting a round of Baz IO | 20260506 21:16:43.797 | INFO | Populating RTMetrics report | 20260506 21:16:43.797 | INFO | RTMetrics report populated | 20260506 21:16:43.797 | INFO | Opening tmp RTMetrics json file | 20260506 21:16:43.851 | INFO | Serializing RTMetrics to json | 20260506 21:16:43.887 | INFO | Serialization done, swapping tmp file now... | 20260506 21:16:43.893 | INFO | RTMetrics done | 20260506 21:16:44.586 | INFO | Finished a round of Baz IO: Wrote 1.952 GiB in 3.912 seconds | 20260506 21:16:44.586 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.98% / 0.00% / 66.01% | 20260506 21:16:44.586 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.634229% / 3.97364e-06% / 5.5631e-05% | 20260506 21:16:44.586 | INFO | Baz IO overall: | 20260506 21:16:44.586 | INFO | BytesWritten : 2379286976745 | 20260506 21:16:44.586 | INFO | NumEvents : 1148913310215 | 20260506 21:16:45.696 | INFO | Analyzing chunk frames = [4513792,4514304) | 20260506 21:16:45.696 | INFO | MemInfo: { | 20260506 21:16:45.696 | INFO | "gpu" : " 17 GB", | 20260506 21:16:45.696 | INFO | "huge" : "", | 20260506 21:16:45.696 | INFO | "huge_shared" : " 27 GB", | 20260506 21:16:45.696 | INFO | "malloc" : " 14 GB", | 20260506 21:16:45.696 | INFO | "pinned" : " 29 GB" | 20260506 21:16:45.696 | INFO | } | 20260506 21:16:45.697 | INFO | Perf report for the last 5.13651 seconds: | 20260506 21:16:45.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:16:45.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:45.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:45.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:45.697 | INFO | ANALYSIS : 62.4%, 2.9, 95.2ms, 0 | 20260506 21:16:45.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:45.697 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 21:16:45.697 | INFO | PRE_HQ : 59.9%, 1.8, 57.2ms, 0 | 20260506 21:16:45.697 | INFO | BAZWRITER : 75.7%, 4.8, 186.2ms, 0 | 20260506 21:16:45.697 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 21:16:45.697 | INFO | Uploaded 36GiB to GPU in 2.0559s (17.5106GiB/s) | 20260506 21:16:45.697 | INFO | Downloaded 38GiB to GPU in 2.20773s (17.2122GiB/s) | 20260506 21:16:50.840 | INFO | Analyzing chunk frames = [4514304,4514816) | 20260506 21:16:50.840 | INFO | MemInfo: { | 20260506 21:16:50.840 | INFO | "gpu" : " 17 GB", | 20260506 21:16:50.840 | INFO | "huge" : "", | 20260506 21:16:50.840 | INFO | "huge_shared" : " 27 GB", | 20260506 21:16:50.840 | INFO | "malloc" : " 14 GB", | 20260506 21:16:50.840 | INFO | "pinned" : " 29 GB" | 20260506 21:16:50.840 | INFO | } | 20260506 21:16:50.840 | INFO | Perf report for the last 5.1436 seconds: | 20260506 21:16:50.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:16:50.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:50.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:50.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:50.840 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 21:16:50.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:50.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:50.840 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:16:50.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:50.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:50.841 | INFO | Uploaded 36GiB to GPU in 2.00035s (17.9969GiB/s) | 20260506 21:16:50.841 | INFO | Downloaded 34GiB to GPU in 1.9296s (17.6202GiB/s) | 20260506 21:16:55.975 | INFO | Analyzing chunk frames = [4514816,4515328) | 20260506 21:16:55.975 | INFO | MemInfo: { | 20260506 21:16:55.975 | INFO | "gpu" : " 17 GB", | 20260506 21:16:55.975 | INFO | "huge" : "", | 20260506 21:16:55.975 | INFO | "huge_shared" : " 27 GB", | 20260506 21:16:55.975 | INFO | "malloc" : " 14 GB", | 20260506 21:16:55.975 | INFO | "pinned" : " 29 GB" | 20260506 21:16:55.975 | INFO | } | 20260506 21:16:55.975 | INFO | Perf report for the last 5.13482 seconds: | 20260506 21:16:55.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:16:55.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:55.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:55.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:55.975 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 21:16:55.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:55.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:55.975 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:16:55.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:55.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:16:55.975 | INFO | Uploaded 36GiB to GPU in 1.95094s (18.4526GiB/s) | 20260506 21:16:55.975 | INFO | Downloaded 34GiB to GPU in 1.9195s (17.713GiB/s) | 20260506 21:17:01.117 | INFO | Analyzing chunk frames = [4515328,4515840) | 20260506 21:17:01.117 | INFO | MemInfo: { | 20260506 21:17:01.117 | INFO | "gpu" : " 17 GB", | 20260506 21:17:01.117 | INFO | "huge" : "", | 20260506 21:17:01.117 | INFO | "huge_shared" : " 27 GB", | 20260506 21:17:01.117 | INFO | "malloc" : " 14 GB", | 20260506 21:17:01.117 | INFO | "pinned" : " 29 GB" | 20260506 21:17:01.117 | INFO | } | 20260506 21:17:01.117 | INFO | Perf report for the last 5.14215 seconds: | 20260506 21:17:01.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:17:01.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:01.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:01.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:01.117 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 21:17:01.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:01.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:01.117 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:17:01.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:01.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:01.117 | INFO | Uploaded 36GiB to GPU in 1.93139s (18.6394GiB/s) | 20260506 21:17:01.117 | INFO | Downloaded 34GiB to GPU in 1.87245s (18.1581GiB/s) | 20260506 21:17:06.258 | INFO | Analyzing chunk frames = [4515840,4516352) | 20260506 21:17:06.258 | INFO | MemInfo: { | 20260506 21:17:06.258 | INFO | "gpu" : " 17 GB", | 20260506 21:17:06.258 | INFO | "huge" : "", | 20260506 21:17:06.258 | INFO | "huge_shared" : " 27 GB", | 20260506 21:17:06.258 | INFO | "malloc" : " 14 GB", | 20260506 21:17:06.258 | INFO | "pinned" : " 29 GB" | 20260506 21:17:06.258 | INFO | } | 20260506 21:17:06.258 | INFO | Perf report for the last 5.14097 seconds: | 20260506 21:17:06.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:17:06.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:06.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:06.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:06.258 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 21:17:06.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:06.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:06.258 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:17:06.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:06.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:06.258 | INFO | Uploaded 36GiB to GPU in 2.00689s (17.9382GiB/s) | 20260506 21:17:06.258 | INFO | Downloaded 34GiB to GPU in 1.92339s (17.6771GiB/s) | 20260506 21:17:11.397 | INFO | Analyzing chunk frames = [4516352,4516864) | 20260506 21:17:11.397 | INFO | MemInfo: { | 20260506 21:17:11.397 | INFO | "gpu" : " 17 GB", | 20260506 21:17:11.397 | INFO | "huge" : "", | 20260506 21:17:11.397 | INFO | "huge_shared" : " 27 GB", | 20260506 21:17:11.397 | INFO | "malloc" : " 14 GB", | 20260506 21:17:11.397 | INFO | "pinned" : " 29 GB" | 20260506 21:17:11.397 | INFO | } | 20260506 21:17:11.397 | INFO | Perf report for the last 5.13891 seconds: | 20260506 21:17:11.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:17:11.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:11.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:11.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:11.397 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 21:17:11.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:11.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:11.397 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:17:11.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:11.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:11.397 | INFO | Uploaded 36GiB to GPU in 1.9966s (18.0307GiB/s) | 20260506 21:17:11.397 | INFO | Downloaded 34GiB to GPU in 1.95097s (17.4273GiB/s) | 20260506 21:17:16.537 | INFO | Analyzing chunk frames = [4516864,4517376) | 20260506 21:17:16.537 | INFO | MemInfo: { | 20260506 21:17:16.537 | INFO | "gpu" : " 17 GB", | 20260506 21:17:16.537 | INFO | "huge" : "", | 20260506 21:17:16.537 | INFO | "huge_shared" : " 27 GB", | 20260506 21:17:16.537 | INFO | "malloc" : " 14 GB", | 20260506 21:17:16.537 | INFO | "pinned" : " 29 GB" | 20260506 21:17:16.537 | INFO | } | 20260506 21:17:16.537 | INFO | Perf report for the last 5.13985 seconds: | 20260506 21:17:16.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:17:16.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:16.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:16.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:16.537 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 21:17:16.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:16.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:16.537 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 21:17:16.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:16.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:16.537 | INFO | Uploaded 36GiB to GPU in 1.94938s (18.4674GiB/s) | 20260506 21:17:16.537 | INFO | Downloaded 34GiB to GPU in 1.98571s (17.1223GiB/s) | 20260506 21:17:21.676 | INFO | Analyzing chunk frames = [4517376,4517888) | 20260506 21:17:21.676 | INFO | MemInfo: { | 20260506 21:17:21.676 | INFO | "gpu" : " 17 GB", | 20260506 21:17:21.676 | INFO | "huge" : "", | 20260506 21:17:21.676 | INFO | "huge_shared" : " 27 GB", | 20260506 21:17:21.676 | INFO | "malloc" : " 14 GB", | 20260506 21:17:21.676 | INFO | "pinned" : " 29 GB" | 20260506 21:17:21.676 | INFO | } | 20260506 21:17:21.676 | INFO | Perf report for the last 5.13878 seconds: | 20260506 21:17:21.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:17:21.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:21.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:21.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:21.676 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 21:17:21.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:21.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:21.676 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 21:17:21.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:21.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:21.676 | INFO | Uploaded 36GiB to GPU in 2.00187s (17.9832GiB/s) | 20260506 21:17:21.676 | INFO | Downloaded 34GiB to GPU in 1.98562s (17.1231GiB/s) | 20260506 21:17:21.775 | INFO | Starting a round of Baz IO | 20260506 21:17:24.914 | INFO | Populating RTMetrics report | 20260506 21:17:24.914 | INFO | RTMetrics report populated | 20260506 21:17:24.914 | INFO | Opening tmp RTMetrics json file | 20260506 21:17:24.955 | INFO | Serializing RTMetrics to json | 20260506 21:17:24.998 | INFO | Serialization done, swapping tmp file now... | 20260506 21:17:25.006 | INFO | RTMetrics done | 20260506 21:17:25.273 | INFO | Finished a round of Baz IO: Wrote 1.95 GiB in 3.497 seconds | 20260506 21:17:25.273 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.98% / 0.00% / 66.02% | 20260506 21:17:25.273 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.63262% / 3.97364e-06% / 6.75519e-05% | 20260506 21:17:25.273 | INFO | Baz IO overall: | 20260506 21:17:25.273 | INFO | BytesWritten : 2381380236799 | 20260506 21:17:25.273 | INFO | NumEvents : 1149915242479 | 20260506 21:17:26.811 | INFO | Analyzing chunk frames = [4517888,4518400) | 20260506 21:17:26.811 | INFO | MemInfo: { | 20260506 21:17:26.811 | INFO | "gpu" : " 17 GB", | 20260506 21:17:26.811 | INFO | "huge" : "", | 20260506 21:17:26.811 | INFO | "huge_shared" : " 27 GB", | 20260506 21:17:26.811 | INFO | "malloc" : " 14 GB", | 20260506 21:17:26.811 | INFO | "pinned" : " 29 GB" | 20260506 21:17:26.811 | INFO | } | 20260506 21:17:26.811 | INFO | Perf report for the last 5.13568 seconds: | 20260506 21:17:26.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:17:26.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:26.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:26.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:26.812 | INFO | ANALYSIS : 62.5%, 2.9, 95.7ms, 0 | 20260506 21:17:26.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:26.812 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 21:17:26.812 | INFO | PRE_HQ : 60.0%, 1.8, 57.0ms, 0 | 20260506 21:17:26.812 | INFO | BAZWRITER : 67.7%, 4.8, 165.3ms, 0 | 20260506 21:17:26.812 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 21:17:26.812 | INFO | Uploaded 36GiB to GPU in 2.05542s (17.5146GiB/s) | 20260506 21:17:26.812 | INFO | Downloaded 38GiB to GPU in 2.19331s (17.3254GiB/s) | 20260506 21:17:31.945 | INFO | Analyzing chunk frames = [4518400,4518912) | 20260506 21:17:31.945 | INFO | MemInfo: { | 20260506 21:17:31.945 | INFO | "gpu" : " 17 GB", | 20260506 21:17:31.945 | INFO | "huge" : "", | 20260506 21:17:31.945 | INFO | "huge_shared" : " 27 GB", | 20260506 21:17:31.945 | INFO | "malloc" : " 14 GB", | 20260506 21:17:31.945 | INFO | "pinned" : " 29 GB" | 20260506 21:17:31.945 | INFO | } | 20260506 21:17:31.945 | INFO | Perf report for the last 5.1338 seconds: | 20260506 21:17:31.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:17:31.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:31.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:31.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:31.945 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 21:17:31.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:31.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:31.945 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:17:31.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:31.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:31.945 | INFO | Uploaded 36GiB to GPU in 2.01685s (17.8496GiB/s) | 20260506 21:17:31.945 | INFO | Downloaded 34GiB to GPU in 1.92695s (17.6444GiB/s) | 20260506 21:17:37.092 | INFO | Analyzing chunk frames = [4518912,4519424) | 20260506 21:17:37.092 | INFO | MemInfo: { | 20260506 21:17:37.092 | INFO | "gpu" : " 17 GB", | 20260506 21:17:37.092 | INFO | "huge" : "", | 20260506 21:17:37.092 | INFO | "huge_shared" : " 27 GB", | 20260506 21:17:37.092 | INFO | "malloc" : " 14 GB", | 20260506 21:17:37.092 | INFO | "pinned" : " 29 GB" | 20260506 21:17:37.092 | INFO | } | 20260506 21:17:37.092 | INFO | Perf report for the last 5.14642 seconds: | 20260506 21:17:37.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:17:37.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:37.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:37.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:37.092 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260506 21:17:37.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:37.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:37.092 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:17:37.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:37.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:37.092 | INFO | Uploaded 36GiB to GPU in 1.99277s (18.0653GiB/s) | 20260506 21:17:37.092 | INFO | Downloaded 34GiB to GPU in 1.9193s (17.7148GiB/s) | 20260506 21:17:42.238 | INFO | Analyzing chunk frames = [4519424,4519936) | 20260506 21:17:42.238 | INFO | MemInfo: { | 20260506 21:17:42.238 | INFO | "gpu" : " 17 GB", | 20260506 21:17:42.238 | INFO | "huge" : "", | 20260506 21:17:42.238 | INFO | "huge_shared" : " 27 GB", | 20260506 21:17:42.238 | INFO | "malloc" : " 14 GB", | 20260506 21:17:42.238 | INFO | "pinned" : " 29 GB" | 20260506 21:17:42.238 | INFO | } | 20260506 21:17:42.238 | INFO | Perf report for the last 5.14609 seconds: | 20260506 21:17:42.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:17:42.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:42.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:42.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:42.238 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 21:17:42.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:42.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:42.238 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:17:42.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:42.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:42.238 | INFO | Uploaded 36GiB to GPU in 1.9558s (18.4068GiB/s) | 20260506 21:17:42.238 | INFO | Downloaded 34GiB to GPU in 1.87685s (18.1154GiB/s) | 20260506 21:17:47.375 | INFO | Analyzing chunk frames = [4519936,4520448) | 20260506 21:17:47.375 | INFO | MemInfo: { | 20260506 21:17:47.375 | INFO | "gpu" : " 17 GB", | 20260506 21:17:47.375 | INFO | "huge" : "", | 20260506 21:17:47.375 | INFO | "huge_shared" : " 27 GB", | 20260506 21:17:47.375 | INFO | "malloc" : " 14 GB", | 20260506 21:17:47.375 | INFO | "pinned" : " 29 GB" | 20260506 21:17:47.375 | INFO | } | 20260506 21:17:47.375 | INFO | Perf report for the last 5.13737 seconds: | 20260506 21:17:47.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:17:47.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:47.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:47.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:47.375 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 21:17:47.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:47.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:47.375 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 21:17:47.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:47.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:47.375 | INFO | Uploaded 36GiB to GPU in 2.00483s (17.9567GiB/s) | 20260506 21:17:47.375 | INFO | Downloaded 34GiB to GPU in 1.93346s (17.5851GiB/s) | 20260506 21:17:52.517 | INFO | Analyzing chunk frames = [4520448,4520960) | 20260506 21:17:52.517 | INFO | MemInfo: { | 20260506 21:17:52.517 | INFO | "gpu" : " 17 GB", | 20260506 21:17:52.517 | INFO | "huge" : "", | 20260506 21:17:52.517 | INFO | "huge_shared" : " 27 GB", | 20260506 21:17:52.517 | INFO | "malloc" : " 14 GB", | 20260506 21:17:52.517 | INFO | "pinned" : " 29 GB" | 20260506 21:17:52.517 | INFO | } | 20260506 21:17:52.517 | INFO | Perf report for the last 5.14185 seconds: | 20260506 21:17:52.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:17:52.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:52.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:52.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:52.517 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:17:52.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:52.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:52.517 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 21:17:52.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:52.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:52.517 | INFO | Uploaded 36GiB to GPU in 2.02033s (17.8189GiB/s) | 20260506 21:17:52.517 | INFO | Downloaded 34GiB to GPU in 1.94187s (17.5089GiB/s) | 20260506 21:17:57.650 | INFO | Analyzing chunk frames = [4520960,4521472) | 20260506 21:17:57.650 | INFO | MemInfo: { | 20260506 21:17:57.650 | INFO | "gpu" : " 17 GB", | 20260506 21:17:57.650 | INFO | "huge" : "", | 20260506 21:17:57.650 | INFO | "huge_shared" : " 27 GB", | 20260506 21:17:57.650 | INFO | "malloc" : " 14 GB", | 20260506 21:17:57.650 | INFO | "pinned" : " 29 GB" | 20260506 21:17:57.650 | INFO | } | 20260506 21:17:57.650 | INFO | Perf report for the last 5.13334 seconds: | 20260506 21:17:57.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:17:57.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:57.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:57.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:57.650 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 21:17:57.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:57.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:57.650 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 21:17:57.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:57.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:17:57.651 | INFO | Uploaded 36GiB to GPU in 1.94247s (18.5331GiB/s) | 20260506 21:17:57.651 | INFO | Downloaded 34GiB to GPU in 1.98865s (17.097GiB/s) | 20260506 21:18:02.787 | INFO | Analyzing chunk frames = [4521472,4521984) | 20260506 21:18:02.787 | INFO | MemInfo: { | 20260506 21:18:02.787 | INFO | "gpu" : " 17 GB", | 20260506 21:18:02.787 | INFO | "huge" : "", | 20260506 21:18:02.787 | INFO | "huge_shared" : " 27 GB", | 20260506 21:18:02.787 | INFO | "malloc" : " 14 GB", | 20260506 21:18:02.787 | INFO | "pinned" : " 29 GB" | 20260506 21:18:02.787 | INFO | } | 20260506 21:18:02.787 | INFO | Perf report for the last 5.13643 seconds: | 20260506 21:18:02.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:18:02.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:02.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:02.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:02.787 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 21:18:02.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:02.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:02.787 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 21:18:02.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:02.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:02.787 | INFO | Uploaded 36GiB to GPU in 2.01554s (17.8612GiB/s) | 20260506 21:18:02.787 | INFO | Downloaded 34GiB to GPU in 1.99525s (17.0405GiB/s) | 20260506 21:18:02.787 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:18:02.787 | INFO | Total mem usage: 18 GiB | 20260506 21:18:02.787 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:18:02.787 | INFO | Recent High Water Marks Sum: 16.9 GiB (Savings: -6.61%) | 20260506 21:18:02.787 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:18:02.787 | INFO | High Water Breakdown: | 20260506 21:18:02.787 | INFO | 62.7% : 10.6 GiB ( 92.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:18:02.787 | INFO | 37.2% : 6.28 GiB ( 93.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:18:02.787 | INFO | 0.118% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:18:02.787 | INFO | ------------------------------------------ | 20260506 21:18:02.787 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:18:02.787 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:18:02.787 | INFO | Total mem usage: 26.5 GiB | 20260506 21:18:02.787 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:18:02.787 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:18:02.787 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:18:02.787 | INFO | High Water Breakdown: | 20260506 21:18:02.787 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:18:02.787 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:18:02.787 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:18:02.787 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:18:02.787 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:18:02.787 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:18:02.787 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:18:02.787 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:18:02.787 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:18:02.787 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:18:02.787 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:18:02.787 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:18:02.787 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:18:02.787 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:18:02.787 | INFO | ------------------------------------------ | 20260506 21:18:02.787 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:18:02.787 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:18:02.787 | INFO | Total mem usage: 25.6 GiB | 20260506 21:18:02.787 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:18:02.787 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:18:02.787 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:18:02.787 | INFO | High Water Breakdown: | 20260506 21:18:02.787 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:18:02.788 | INFO | ------------------------------------------ | 20260506 21:18:02.788 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:18:02.788 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:18:02.788 | INFO | Total mem usage: 15.7 GiB | 20260506 21:18:02.788 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:18:02.788 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:18:02.788 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:18:02.788 | INFO | High Water Breakdown: | 20260506 21:18:02.788 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:18:02.788 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:18:02.788 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:18:02.788 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:18:02.788 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:18:02.788 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:18:02.788 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:18:02.788 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:18:02.788 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:18:02.788 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:18:02.788 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:18:02.788 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:18:02.788 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:18:02.788 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:18:02.788 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:18:02.788 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:18:02.788 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:18:02.788 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:18:02.788 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:18:02.788 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:18:02.788 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:18:02.788 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:18:02.788 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:18:02.788 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:18:02.788 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:18:02.788 | INFO | ------------------------------------------ | 20260506 21:18:02.788 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:18:02.788 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:18:02.788 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:18:02.788 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:18:02.788 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.53e+01, 1.34e+01 / 2.45e+00 | 20260506 21:18:02.788 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.02e-01, 1.01e-02 / 7.36e-03 | 20260506 21:18:02.788 | INFO | PerfReport: 11.66%, Baseline -- 3.63e+00 / 4.38e+00, 5.50e-01 / 3.80e-01 | 20260506 21:18:02.788 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.45e+00, 1.83e-01 / 1.55e-01 | 20260506 21:18:02.788 | INFO | PerfReport: 16.04%, DME -- 3.99e+01 / 4.44e+01, 3.97e+00 / 2.80e+00 | 20260506 21:18:02.788 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.38e+00 / 1.85e+00 | 20260506 21:18:02.788 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.21e+00 / 2.75e+00, 3.74e-01 / 2.34e-01 | 20260506 21:18:02.788 | INFO | PerfReport: 3.27%, Metrics -- 1.02e+00 / 1.34e+00, 2.63e-01 / 1.55e-01 | 20260506 21:18:02.788 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.49e-01 / 3.96e-01, 5.49e-02 / 2.35e-02 | 20260506 21:18:02.788 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:18:02.788 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:18:02.788 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:18:02.788 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.31e+01 / 0.00e+00 | 20260506 21:18:02.788 | INFO | PerfReport: 21.48%, Upload -- 2.08e+01 / 0.00e+00, 6.87e+00 / 0.00e+00 | 20260506 21:18:02.788 | INFO | PerfReport: 21.25%, Download -- 2.05e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260506 21:18:02.788 | INFO | PerfReport: 32.18%, Compute -- 3.11e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 21:18:02.788 | INFO | PerfReport: 1.27%, LockUpload -- 1.23e+00 / 0.00e+00, 4.87e+00 / 0.00e+00 | 20260506 21:18:02.788 | INFO | PerfReport: 1.24%, LockDownload -- 1.19e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260506 21:18:02.788 | INFO | PerfReport: 22.57%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 21:18:02.788 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260506 21:18:02.913 | INFO | Starting a round of Baz IO | 20260506 21:18:06.029 | INFO | Populating RTMetrics report | 20260506 21:18:06.029 | INFO | RTMetrics report populated | 20260506 21:18:06.030 | INFO | Opening tmp RTMetrics json file | 20260506 21:18:06.031 | INFO | Serializing RTMetrics to json | 20260506 21:18:06.032 | INFO | Serialization done, swapping tmp file now... | 20260506 21:18:06.033 | INFO | RTMetrics done | 20260506 21:18:06.162 | INFO | Finished a round of Baz IO: Wrote 1.952 GiB in 3.249 seconds | 20260506 21:18:06.162 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.98% / 0.00% / 66.02% | 20260506 21:18:06.162 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.632215% / 3.97364e-06% / 9.13938e-05% | 20260506 21:18:06.162 | INFO | Baz IO overall: | 20260506 21:18:06.162 | INFO | BytesWritten : 2383475726437 | 20260506 21:18:06.162 | INFO | NumEvents : 1150918843391 | 20260506 21:18:07.928 | INFO | Analyzing chunk frames = [4521984,4522496) | 20260506 21:18:07.928 | INFO | MemInfo: { | 20260506 21:18:07.928 | INFO | "gpu" : " 17 GB", | 20260506 21:18:07.928 | INFO | "huge" : "", | 20260506 21:18:07.928 | INFO | "huge_shared" : " 27 GB", | 20260506 21:18:07.928 | INFO | "malloc" : " 14 GB", | 20260506 21:18:07.928 | INFO | "pinned" : " 29 GB" | 20260506 21:18:07.928 | INFO | } | 20260506 21:18:07.928 | INFO | Perf report for the last 5.14125 seconds: | 20260506 21:18:07.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:18:07.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:07.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:07.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:07.928 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 21:18:07.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:07.928 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260506 21:18:07.928 | INFO | PRE_HQ : 60.0%, 1.9, 57.7ms, 0 | 20260506 21:18:07.928 | INFO | BAZWRITER : 62.9%, 4.6, 149.5ms, 0 | 20260506 21:18:07.928 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 21:18:07.928 | INFO | Uploaded 36GiB to GPU in 2.02592s (17.7697GiB/s) | 20260506 21:18:07.928 | INFO | Downloaded 38GiB to GPU in 2.22709s (17.0627GiB/s) | 20260506 21:18:13.065 | INFO | Analyzing chunk frames = [4522496,4523008) | 20260506 21:18:13.065 | INFO | MemInfo: { | 20260506 21:18:13.065 | INFO | "gpu" : " 17 GB", | 20260506 21:18:13.065 | INFO | "huge" : "", | 20260506 21:18:13.066 | INFO | "huge_shared" : " 27 GB", | 20260506 21:18:13.066 | INFO | "malloc" : " 14 GB", | 20260506 21:18:13.066 | INFO | "pinned" : " 29 GB" | 20260506 21:18:13.066 | INFO | } | 20260506 21:18:13.066 | INFO | Perf report for the last 5.13751 seconds: | 20260506 21:18:13.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:18:13.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:13.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:13.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:13.066 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:18:13.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:13.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:13.066 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 21:18:13.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:13.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:13.066 | INFO | Uploaded 36GiB to GPU in 1.99126s (18.079GiB/s) | 20260506 21:18:13.066 | INFO | Downloaded 34GiB to GPU in 1.92719s (17.6422GiB/s) | 20260506 21:18:18.209 | INFO | Analyzing chunk frames = [4523008,4523520) | 20260506 21:18:18.209 | INFO | MemInfo: { | 20260506 21:18:18.209 | INFO | "gpu" : " 17 GB", | 20260506 21:18:18.209 | INFO | "huge" : "", | 20260506 21:18:18.209 | INFO | "huge_shared" : " 27 GB", | 20260506 21:18:18.209 | INFO | "malloc" : " 14 GB", | 20260506 21:18:18.209 | INFO | "pinned" : " 29 GB" | 20260506 21:18:18.209 | INFO | } | 20260506 21:18:18.209 | INFO | Perf report for the last 5.14387 seconds: | 20260506 21:18:18.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:18:18.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:18.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:18.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:18.210 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 21:18:18.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:18.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:18.210 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:18:18.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:18.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:18.210 | INFO | Uploaded 36GiB to GPU in 1.96949s (18.2788GiB/s) | 20260506 21:18:18.210 | INFO | Downloaded 34GiB to GPU in 1.92646s (17.649GiB/s) | 20260506 21:18:23.353 | INFO | Analyzing chunk frames = [4523520,4524032) | 20260506 21:18:23.353 | INFO | MemInfo: { | 20260506 21:18:23.353 | INFO | "gpu" : " 17 GB", | 20260506 21:18:23.353 | INFO | "huge" : "", | 20260506 21:18:23.353 | INFO | "huge_shared" : " 27 GB", | 20260506 21:18:23.353 | INFO | "malloc" : " 14 GB", | 20260506 21:18:23.353 | INFO | "pinned" : " 29 GB" | 20260506 21:18:23.353 | INFO | } | 20260506 21:18:23.353 | INFO | Perf report for the last 5.14371 seconds: | 20260506 21:18:23.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:18:23.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:23.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:23.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:23.353 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 21:18:23.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:23.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:23.353 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:18:23.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:23.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:23.353 | INFO | Uploaded 36GiB to GPU in 1.94989s (18.4626GiB/s) | 20260506 21:18:23.354 | INFO | Downloaded 34GiB to GPU in 1.882s (18.0659GiB/s) | 20260506 21:18:28.491 | INFO | Analyzing chunk frames = [4524032,4524544) | 20260506 21:18:28.491 | INFO | MemInfo: { | 20260506 21:18:28.491 | INFO | "gpu" : " 17 GB", | 20260506 21:18:28.491 | INFO | "huge" : "", | 20260506 21:18:28.491 | INFO | "huge_shared" : " 27 GB", | 20260506 21:18:28.491 | INFO | "malloc" : " 14 GB", | 20260506 21:18:28.491 | INFO | "pinned" : " 29 GB" | 20260506 21:18:28.491 | INFO | } | 20260506 21:18:28.491 | INFO | Perf report for the last 5.13755 seconds: | 20260506 21:18:28.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:18:28.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:28.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:28.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:28.491 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 21:18:28.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:28.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:28.491 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:18:28.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:28.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:28.491 | INFO | Uploaded 36GiB to GPU in 1.97574s (18.2211GiB/s) | 20260506 21:18:28.491 | INFO | Downloaded 34GiB to GPU in 1.92618s (17.6516GiB/s) | 20260506 21:18:33.624 | INFO | Analyzing chunk frames = [4524544,4525056) | 20260506 21:18:33.624 | INFO | MemInfo: { | 20260506 21:18:33.624 | INFO | "gpu" : " 17 GB", | 20260506 21:18:33.624 | INFO | "huge" : "", | 20260506 21:18:33.624 | INFO | "huge_shared" : " 27 GB", | 20260506 21:18:33.624 | INFO | "malloc" : " 14 GB", | 20260506 21:18:33.624 | INFO | "pinned" : " 29 GB" | 20260506 21:18:33.624 | INFO | } | 20260506 21:18:33.624 | INFO | Perf report for the last 5.13287 seconds: | 20260506 21:18:33.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:18:33.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:33.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:33.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:33.624 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 21:18:33.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:33.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:33.624 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:18:33.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:33.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:33.624 | INFO | Uploaded 36GiB to GPU in 1.98921s (18.0977GiB/s) | 20260506 21:18:33.624 | INFO | Downloaded 34GiB to GPU in 1.95379s (17.402GiB/s) | 20260506 21:18:38.762 | INFO | Analyzing chunk frames = [4525056,4525568) | 20260506 21:18:38.762 | INFO | MemInfo: { | 20260506 21:18:38.762 | INFO | "gpu" : " 17 GB", | 20260506 21:18:38.762 | INFO | "huge" : "", | 20260506 21:18:38.762 | INFO | "huge_shared" : " 27 GB", | 20260506 21:18:38.762 | INFO | "malloc" : " 14 GB", | 20260506 21:18:38.762 | INFO | "pinned" : " 29 GB" | 20260506 21:18:38.762 | INFO | } | 20260506 21:18:38.762 | INFO | Perf report for the last 5.13881 seconds: | 20260506 21:18:38.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:18:38.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:38.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:38.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:38.762 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:18:38.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:38.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:38.763 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 21:18:38.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:38.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:38.763 | INFO | Uploaded 36GiB to GPU in 1.95862s (18.3803GiB/s) | 20260506 21:18:38.763 | INFO | Downloaded 34GiB to GPU in 2.00199s (16.9831GiB/s) | 20260506 21:18:43.899 | INFO | Analyzing chunk frames = [4525568,4526080) | 20260506 21:18:43.899 | INFO | MemInfo: { | 20260506 21:18:43.899 | INFO | "gpu" : " 17 GB", | 20260506 21:18:43.900 | INFO | "huge" : "", | 20260506 21:18:43.900 | INFO | "huge_shared" : " 27 GB", | 20260506 21:18:43.900 | INFO | "malloc" : " 14 GB", | 20260506 21:18:43.900 | INFO | "pinned" : " 29 GB" | 20260506 21:18:43.900 | INFO | } | 20260506 21:18:43.900 | INFO | Perf report for the last 5.13717 seconds: | 20260506 21:18:43.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:18:43.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:43.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:43.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:43.900 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 21:18:43.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:43.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:43.900 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 21:18:43.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:43.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:43.900 | INFO | Uploaded 36GiB to GPU in 1.99141s (18.0776GiB/s) | 20260506 21:18:43.900 | INFO | Downloaded 34GiB to GPU in 1.99716s (17.0242GiB/s) | 20260506 21:18:44.020 | INFO | Starting a round of Baz IO | 20260506 21:18:47.154 | INFO | Populating RTMetrics report | 20260506 21:18:47.155 | INFO | RTMetrics report populated | 20260506 21:18:47.155 | INFO | Opening tmp RTMetrics json file | 20260506 21:18:47.172 | INFO | Serializing RTMetrics to json | 20260506 21:18:47.203 | INFO | Serialization done, swapping tmp file now... | 20260506 21:18:47.207 | INFO | RTMetrics done | 20260506 21:18:47.505 | INFO | Finished a round of Baz IO: Wrote 1.95 GiB in 3.485 seconds | 20260506 21:18:47.505 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.97% / 0.00% / 66.02% | 20260506 21:18:47.505 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.630673% / 0% / 4.37101e-05% | 20260506 21:18:47.505 | INFO | Baz IO overall: | 20260506 21:18:47.506 | INFO | BytesWritten : 2385569816521 | 20260506 21:18:47.506 | INFO | NumEvents : 1151921190652 | 20260506 21:18:49.038 | INFO | Analyzing chunk frames = [4526080,4526592) | 20260506 21:18:49.038 | INFO | MemInfo: { | 20260506 21:18:49.038 | INFO | "gpu" : " 17 GB", | 20260506 21:18:49.038 | INFO | "huge" : "", | 20260506 21:18:49.038 | INFO | "huge_shared" : " 27 GB", | 20260506 21:18:49.038 | INFO | "malloc" : " 14 GB", | 20260506 21:18:49.038 | INFO | "pinned" : " 29 GB" | 20260506 21:18:49.038 | INFO | } | 20260506 21:18:49.038 | INFO | Perf report for the last 5.13857 seconds: | 20260506 21:18:49.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:18:49.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:49.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:49.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:49.038 | INFO | ANALYSIS : 62.6%, 2.9, 94.4ms, 0 | 20260506 21:18:49.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:49.038 | INFO | RT_METRICS : 46.2%, 1.0, 24.2ms, 0 | 20260506 21:18:49.038 | INFO | PRE_HQ : 60.8%, 1.9, 60.4ms, 0 | 20260506 21:18:49.038 | INFO | BAZWRITER : 67.3%, 4.8, 167.7ms, 0 | 20260506 21:18:49.038 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 21:18:49.039 | INFO | Uploaded 36GiB to GPU in 2.03049s (17.7297GiB/s) | 20260506 21:18:49.039 | INFO | Downloaded 38GiB to GPU in 2.1981s (17.2877GiB/s) | 20260506 21:18:54.189 | INFO | Analyzing chunk frames = [4526592,4527104) | 20260506 21:18:54.189 | INFO | MemInfo: { | 20260506 21:18:54.189 | INFO | "gpu" : " 17 GB", | 20260506 21:18:54.189 | INFO | "huge" : "", | 20260506 21:18:54.189 | INFO | "huge_shared" : " 27 GB", | 20260506 21:18:54.189 | INFO | "malloc" : " 14 GB", | 20260506 21:18:54.189 | INFO | "pinned" : " 29 GB" | 20260506 21:18:54.189 | INFO | } | 20260506 21:18:54.189 | INFO | Perf report for the last 5.15088 seconds: | 20260506 21:18:54.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:18:54.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:54.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:54.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:54.189 | INFO | ANALYSIS : 61.2%, 3.0, 95.6ms, 0 | 20260506 21:18:54.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:54.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:54.189 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:18:54.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:54.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:54.189 | INFO | Uploaded 36GiB to GPU in 2.01039s (17.9069GiB/s) | 20260506 21:18:54.189 | INFO | Downloaded 34GiB to GPU in 1.92329s (17.6781GiB/s) | 20260506 21:18:59.325 | INFO | Analyzing chunk frames = [4527104,4527616) | 20260506 21:18:59.325 | INFO | MemInfo: { | 20260506 21:18:59.325 | INFO | "gpu" : " 17 GB", | 20260506 21:18:59.325 | INFO | "huge" : "", | 20260506 21:18:59.325 | INFO | "huge_shared" : " 27 GB", | 20260506 21:18:59.325 | INFO | "malloc" : " 14 GB", | 20260506 21:18:59.325 | INFO | "pinned" : " 29 GB" | 20260506 21:18:59.325 | INFO | } | 20260506 21:18:59.326 | INFO | Perf report for the last 5.13641 seconds: | 20260506 21:18:59.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:18:59.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:59.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:59.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:59.326 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 21:18:59.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:59.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:59.326 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 21:18:59.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:59.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:18:59.326 | INFO | Uploaded 36GiB to GPU in 1.99702s (18.0269GiB/s) | 20260506 21:18:59.326 | INFO | Downloaded 34GiB to GPU in 1.92334s (17.6776GiB/s) | 20260506 21:19:04.465 | INFO | Analyzing chunk frames = [4527616,4528128) | 20260506 21:19:04.465 | INFO | MemInfo: { | 20260506 21:19:04.465 | INFO | "gpu" : " 17 GB", | 20260506 21:19:04.465 | INFO | "huge" : "", | 20260506 21:19:04.465 | INFO | "huge_shared" : " 27 GB", | 20260506 21:19:04.465 | INFO | "malloc" : " 14 GB", | 20260506 21:19:04.465 | INFO | "pinned" : " 29 GB" | 20260506 21:19:04.465 | INFO | } | 20260506 21:19:04.465 | INFO | Perf report for the last 5.13954 seconds: | 20260506 21:19:04.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:19:04.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:04.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:04.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:04.465 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 21:19:04.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:04.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:04.465 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:19:04.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:04.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:04.465 | INFO | Uploaded 36GiB to GPU in 1.9839s (18.1461GiB/s) | 20260506 21:19:04.465 | INFO | Downloaded 34GiB to GPU in 1.90452s (17.8523GiB/s) | 20260506 21:19:09.606 | INFO | Analyzing chunk frames = [4528128,4528640) | 20260506 21:19:09.606 | INFO | MemInfo: { | 20260506 21:19:09.606 | INFO | "gpu" : " 17 GB", | 20260506 21:19:09.606 | INFO | "huge" : "", | 20260506 21:19:09.606 | INFO | "huge_shared" : " 27 GB", | 20260506 21:19:09.606 | INFO | "malloc" : " 14 GB", | 20260506 21:19:09.606 | INFO | "pinned" : " 29 GB" | 20260506 21:19:09.606 | INFO | } | 20260506 21:19:09.606 | INFO | Perf report for the last 5.14058 seconds: | 20260506 21:19:09.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:19:09.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:09.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:09.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:09.606 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 21:19:09.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:09.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:09.606 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:19:09.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:09.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:09.606 | INFO | Uploaded 36GiB to GPU in 2.01069s (17.9043GiB/s) | 20260506 21:19:09.606 | INFO | Downloaded 34GiB to GPU in 1.91658s (17.7399GiB/s) | 20260506 21:19:14.740 | INFO | Analyzing chunk frames = [4528640,4529152) | 20260506 21:19:14.740 | INFO | MemInfo: { | 20260506 21:19:14.740 | INFO | "gpu" : " 17 GB", | 20260506 21:19:14.740 | INFO | "huge" : "", | 20260506 21:19:14.740 | INFO | "huge_shared" : " 27 GB", | 20260506 21:19:14.740 | INFO | "malloc" : " 14 GB", | 20260506 21:19:14.740 | INFO | "pinned" : " 29 GB" | 20260506 21:19:14.740 | INFO | } | 20260506 21:19:14.740 | INFO | Perf report for the last 5.13437 seconds: | 20260506 21:19:14.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:19:14.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:14.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:14.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:14.740 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 21:19:14.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:14.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:14.740 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:19:14.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:14.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:14.740 | INFO | Uploaded 36GiB to GPU in 2.01652s (17.8525GiB/s) | 20260506 21:19:14.740 | INFO | Downloaded 34GiB to GPU in 1.95472s (17.3938GiB/s) | 20260506 21:19:19.877 | INFO | Analyzing chunk frames = [4529152,4529664) | 20260506 21:19:19.877 | INFO | MemInfo: { | 20260506 21:19:19.877 | INFO | "gpu" : " 17 GB", | 20260506 21:19:19.877 | INFO | "huge" : "", | 20260506 21:19:19.877 | INFO | "huge_shared" : " 27 GB", | 20260506 21:19:19.877 | INFO | "malloc" : " 14 GB", | 20260506 21:19:19.877 | INFO | "pinned" : " 29 GB" | 20260506 21:19:19.877 | INFO | } | 20260506 21:19:19.877 | INFO | Perf report for the last 5.13723 seconds: | 20260506 21:19:19.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:19:19.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:19.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:19.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:19.877 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 21:19:19.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:19.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:19.877 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 21:19:19.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:19.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:19.878 | INFO | Uploaded 36GiB to GPU in 1.95698s (18.3957GiB/s) | 20260506 21:19:19.878 | INFO | Downloaded 34GiB to GPU in 1.99762s (17.0203GiB/s) | 20260506 21:19:25.017 | INFO | Analyzing chunk frames = [4529664,4530176) | 20260506 21:19:25.017 | INFO | MemInfo: { | 20260506 21:19:25.017 | INFO | "gpu" : " 17 GB", | 20260506 21:19:25.017 | INFO | "huge" : "", | 20260506 21:19:25.017 | INFO | "huge_shared" : " 27 GB", | 20260506 21:19:25.017 | INFO | "malloc" : " 14 GB", | 20260506 21:19:25.017 | INFO | "pinned" : " 29 GB" | 20260506 21:19:25.017 | INFO | } | 20260506 21:19:25.017 | INFO | Perf report for the last 5.13996 seconds: | 20260506 21:19:25.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:19:25.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:25.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:25.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:25.017 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 21:19:25.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:25.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:25.017 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 21:19:25.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:25.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:25.017 | INFO | Uploaded 36GiB to GPU in 1.99728s (18.0245GiB/s) | 20260506 21:19:25.018 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:19:25.018 | INFO | Total mem usage: 18 GiB | 20260506 21:19:25.018 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:19:25.018 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -11.7%) | 20260506 21:19:25.018 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:19:25.018 | INFO | High Water Breakdown: | 20260506 21:19:25.018 | INFO | 62% : 9.99 GiB ( 87.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:19:25.018 | INFO | 37.9% : 6.11 GiB ( 91.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:19:25.018 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:19:25.018 | INFO | ------------------------------------------ | 20260506 21:19:25.018 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:19:25.018 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:19:25.018 | INFO | Total mem usage: 26.5 GiB | 20260506 21:19:25.018 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:19:25.018 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 21:19:25.018 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260506 21:19:25.018 | INFO | High Water Breakdown: | 20260506 21:19:25.018 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:19:25.018 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:19:25.018 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:19:25.018 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:19:25.018 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:19:25.018 | INFO | Downloaded 34GiB to GPU in 1.99026s (17.0832GiB/s) | 20260506 21:19:25.018 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:19:25.018 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:19:25.018 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:19:25.018 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:19:25.018 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:19:25.018 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:19:25.018 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:19:25.018 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:19:25.018 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:19:25.018 | INFO | ------------------------------------------ | 20260506 21:19:25.018 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:19:25.018 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:19:25.018 | INFO | Total mem usage: 25.6 GiB | 20260506 21:19:25.018 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:19:25.018 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:19:25.018 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:19:25.018 | INFO | High Water Breakdown: | 20260506 21:19:25.018 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:19:25.018 | INFO | ------------------------------------------ | 20260506 21:19:25.018 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:19:25.018 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:19:25.018 | INFO | Total mem usage: 15.7 GiB | 20260506 21:19:25.018 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:19:25.018 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:19:25.018 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:19:25.018 | INFO | High Water Breakdown: | 20260506 21:19:25.018 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:19:25.018 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:19:25.018 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:19:25.018 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:19:25.018 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:19:25.018 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:19:25.018 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:19:25.018 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:19:25.018 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:19:25.018 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:19:25.018 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:19:25.018 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:19:25.018 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:19:25.018 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:19:25.018 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:19:25.019 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:19:25.019 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:19:25.019 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:19:25.019 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:19:25.019 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:19:25.019 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:19:25.019 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:19:25.019 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:19:25.019 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:19:25.019 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:19:25.019 | INFO | ------------------------------------------ | 20260506 21:19:25.019 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:19:25.019 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:19:25.019 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:19:25.019 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:19:25.019 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.61e+01, 1.36e+01 / 2.53e+00 | 20260506 21:19:25.019 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 1.05e-02 / 5.86e-03 | 20260506 21:19:25.019 | INFO | PerfReport: 11.66%, Baseline -- 3.63e+00 / 4.29e+00, 5.59e-01 / 4.02e-01 | 20260506 21:19:25.019 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.43e+00, 1.83e-01 / 1.48e-01 | 20260506 21:19:25.019 | INFO | PerfReport: 16.05%, DME -- 4.00e+01 / 4.50e+01, 4.40e+00 / 2.59e+00 | 20260506 21:19:25.019 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.41e+00 / 1.96e+00 | 20260506 21:19:25.019 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.21e+00 / 2.73e+00, 3.74e-01 / 2.36e-01 | 20260506 21:19:25.019 | INFO | PerfReport: 3.27%, Metrics -- 1.02e+00 / 1.37e+00, 2.72e-01 / 9.52e-02 | 20260506 21:19:25.019 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.48e-01 / 3.96e-01, 5.50e-02 / 1.91e-02 | 20260506 21:19:25.019 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:19:25.019 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:19:25.019 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:19:25.019 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 21:19:25.019 | INFO | PerfReport: 21.46%, Upload -- 2.07e+01 / 0.00e+00, 6.82e+00 / 0.00e+00 | 20260506 21:19:25.019 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 21:19:25.019 | INFO | PerfReport: 32.24%, Compute -- 3.12e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 21:19:25.019 | INFO | PerfReport: 1.23%, LockUpload -- 1.19e+00 / 0.00e+00, 4.75e+00 / 0.00e+00 | 20260506 21:19:25.019 | INFO | PerfReport: 1.27%, LockDownload -- 1.23e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260506 21:19:25.019 | INFO | PerfReport: 22.45%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:19:25.019 | INFO | PerfReport: 0.02%, Misc -- 1.45e-02 / 0.00e+00, 2.15e-02 / 0.00e+00 | 20260506 21:19:25.117 | INFO | Starting a round of Baz IO | 20260506 21:19:28.249 | INFO | Populating RTMetrics report | 20260506 21:19:28.249 | INFO | RTMetrics report populated | 20260506 21:19:28.249 | INFO | Opening tmp RTMetrics json file | 20260506 21:19:28.424 | INFO | Serializing RTMetrics to json | 20260506 21:19:28.431 | INFO | Serialization done, swapping tmp file now... | 20260506 21:19:28.434 | INFO | RTMetrics done | 20260506 21:19:28.611 | INFO | Finished a round of Baz IO: Wrote 1.949 GiB in 3.494 seconds | 20260506 21:19:28.611 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.97% / 0.00% / 66.03% | 20260506 21:19:28.611 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.62751% / 3.97364e-06% / 8.34465e-05% | 20260506 21:19:28.612 | INFO | Baz IO overall: | 20260506 21:19:28.612 | INFO | BytesWritten : 2387662648790 | 20260506 21:19:28.612 | INFO | NumEvents : 1152922734706 | 20260506 21:19:30.161 | INFO | Analyzing chunk frames = [4530176,4530688) | 20260506 21:19:30.162 | INFO | MemInfo: { | 20260506 21:19:30.162 | INFO | "gpu" : " 17 GB", | 20260506 21:19:30.162 | INFO | "huge" : "", | 20260506 21:19:30.162 | INFO | "huge_shared" : " 27 GB", | 20260506 21:19:30.162 | INFO | "malloc" : " 14 GB", | 20260506 21:19:30.162 | INFO | "pinned" : " 29 GB" | 20260506 21:19:30.162 | INFO | } | 20260506 21:19:30.162 | INFO | Perf report for the last 5.14436 seconds: | 20260506 21:19:30.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:19:30.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:30.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:30.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:30.162 | INFO | ANALYSIS : 62.3%, 2.9, 95.5ms, 0 | 20260506 21:19:30.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:30.162 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260506 21:19:30.162 | INFO | PRE_HQ : 59.6%, 1.8, 56.6ms, 0 | 20260506 21:19:30.162 | INFO | BAZWRITER : 67.4%, 4.7, 164.6ms, 0 | 20260506 21:19:30.162 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.4ms, 0 | 20260506 21:19:30.162 | INFO | Uploaded 36GiB to GPU in 1.99407s (18.0536GiB/s) | 20260506 21:19:30.162 | INFO | Downloaded 38GiB to GPU in 2.19254s (17.3315GiB/s) | 20260506 21:19:35.302 | INFO | Analyzing chunk frames = [4530688,4531200) | 20260506 21:19:35.302 | INFO | MemInfo: { | 20260506 21:19:35.302 | INFO | "gpu" : " 17 GB", | 20260506 21:19:35.302 | INFO | "huge" : "", | 20260506 21:19:35.302 | INFO | "huge_shared" : " 27 GB", | 20260506 21:19:35.302 | INFO | "malloc" : " 14 GB", | 20260506 21:19:35.302 | INFO | "pinned" : " 29 GB" | 20260506 21:19:35.302 | INFO | } | 20260506 21:19:35.303 | INFO | Perf report for the last 5.14083 seconds: | 20260506 21:19:35.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:19:35.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:35.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:35.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:35.303 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 21:19:35.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:35.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:35.303 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 21:19:35.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:35.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:35.303 | INFO | Uploaded 36GiB to GPU in 1.98154s (18.1676GiB/s) | 20260506 21:19:35.303 | INFO | Downloaded 34GiB to GPU in 1.92s (17.7083GiB/s) | 20260506 21:19:40.442 | INFO | Analyzing chunk frames = [4531200,4531712) | 20260506 21:19:40.442 | INFO | MemInfo: { | 20260506 21:19:40.442 | INFO | "gpu" : " 17 GB", | 20260506 21:19:40.442 | INFO | "huge" : "", | 20260506 21:19:40.442 | INFO | "huge_shared" : " 27 GB", | 20260506 21:19:40.442 | INFO | "malloc" : " 14 GB", | 20260506 21:19:40.442 | INFO | "pinned" : " 29 GB" | 20260506 21:19:40.442 | INFO | } | 20260506 21:19:40.442 | INFO | Perf report for the last 5.13979 seconds: | 20260506 21:19:40.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:19:40.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:40.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:40.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:40.442 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 21:19:40.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:40.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:40.442 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:19:40.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:40.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:40.442 | INFO | Uploaded 36GiB to GPU in 1.96004s (18.367GiB/s) | 20260506 21:19:40.443 | INFO | Downloaded 34GiB to GPU in 1.92718s (17.6424GiB/s) | 20260506 21:19:45.578 | INFO | Analyzing chunk frames = [4531712,4532224) | 20260506 21:19:45.578 | INFO | MemInfo: { | 20260506 21:19:45.578 | INFO | "gpu" : " 17 GB", | 20260506 21:19:45.578 | INFO | "huge" : "", | 20260506 21:19:45.578 | INFO | "huge_shared" : " 27 GB", | 20260506 21:19:45.578 | INFO | "malloc" : " 14 GB", | 20260506 21:19:45.578 | INFO | "pinned" : " 29 GB" | 20260506 21:19:45.578 | INFO | } | 20260506 21:19:45.578 | INFO | Perf report for the last 5.13552 seconds: | 20260506 21:19:45.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:19:45.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:45.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:45.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:45.578 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260506 21:19:45.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:45.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:45.578 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:19:45.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:45.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:45.578 | INFO | Uploaded 36GiB to GPU in 1.96115s (18.3566GiB/s) | 20260506 21:19:45.578 | INFO | Downloaded 34GiB to GPU in 1.89288s (17.9621GiB/s) | 20260506 21:19:50.711 | INFO | Analyzing chunk frames = [4532224,4532736) | 20260506 21:19:50.712 | INFO | MemInfo: { | 20260506 21:19:50.712 | INFO | "gpu" : " 17 GB", | 20260506 21:19:50.712 | INFO | "huge" : "", | 20260506 21:19:50.712 | INFO | "huge_shared" : " 27 GB", | 20260506 21:19:50.712 | INFO | "malloc" : " 14 GB", | 20260506 21:19:50.712 | INFO | "pinned" : " 29 GB" | 20260506 21:19:50.712 | INFO | } | 20260506 21:19:50.712 | INFO | Perf report for the last 5.13382 seconds: | 20260506 21:19:50.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:19:50.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:50.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:50.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:50.712 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 21:19:50.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:50.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:50.712 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:19:50.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:50.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:50.712 | INFO | Uploaded 36GiB to GPU in 2.00485s (17.9564GiB/s) | 20260506 21:19:50.712 | INFO | Downloaded 34GiB to GPU in 1.91165s (17.7857GiB/s) | 20260506 21:19:55.851 | INFO | Analyzing chunk frames = [4532736,4533248) | 20260506 21:19:55.851 | INFO | MemInfo: { | 20260506 21:19:55.851 | INFO | "gpu" : " 17 GB", | 20260506 21:19:55.851 | INFO | "huge" : "", | 20260506 21:19:55.851 | INFO | "huge_shared" : " 27 GB", | 20260506 21:19:55.851 | INFO | "malloc" : " 14 GB", | 20260506 21:19:55.851 | INFO | "pinned" : " 29 GB" | 20260506 21:19:55.851 | INFO | } | 20260506 21:19:55.851 | INFO | Perf report for the last 5.13948 seconds: | 20260506 21:19:55.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:19:55.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:55.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:55.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:55.851 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:19:55.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:55.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:55.851 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:19:55.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:55.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:19:55.852 | INFO | Uploaded 36GiB to GPU in 1.99664s (18.0303GiB/s) | 20260506 21:19:55.852 | INFO | Downloaded 34GiB to GPU in 1.96011s (17.3459GiB/s) | 20260506 21:20:00.992 | INFO | Analyzing chunk frames = [4533248,4533760) | 20260506 21:20:00.992 | INFO | MemInfo: { | 20260506 21:20:00.992 | INFO | "gpu" : " 17 GB", | 20260506 21:20:00.992 | INFO | "huge" : "", | 20260506 21:20:00.992 | INFO | "huge_shared" : " 27 GB", | 20260506 21:20:00.992 | INFO | "malloc" : " 14 GB", | 20260506 21:20:00.992 | INFO | "pinned" : " 29 GB" | 20260506 21:20:00.992 | INFO | } | 20260506 21:20:00.992 | INFO | Perf report for the last 5.14068 seconds: | 20260506 21:20:00.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:20:00.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:00.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:00.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:00.992 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:20:00.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:00.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:00.992 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 21:20:00.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:00.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:00.992 | INFO | Uploaded 36GiB to GPU in 1.97559s (18.2224GiB/s) | 20260506 21:20:00.992 | INFO | Downloaded 34GiB to GPU in 2.00889s (16.9248GiB/s) | 20260506 21:20:06.131 | INFO | Analyzing chunk frames = [4533760,4534272) | 20260506 21:20:06.131 | INFO | MemInfo: { | 20260506 21:20:06.131 | INFO | "gpu" : " 17 GB", | 20260506 21:20:06.131 | INFO | "huge" : "", | 20260506 21:20:06.131 | INFO | "huge_shared" : " 27 GB", | 20260506 21:20:06.131 | INFO | "malloc" : " 14 GB", | 20260506 21:20:06.131 | INFO | "pinned" : " 29 GB" | 20260506 21:20:06.131 | INFO | } | 20260506 21:20:06.131 | INFO | Perf report for the last 5.13879 seconds: | 20260506 21:20:06.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:20:06.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:06.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:06.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:06.131 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:20:06.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:06.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:06.131 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:20:06.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:06.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:06.131 | INFO | Uploaded 36GiB to GPU in 1.99095s (18.0818GiB/s) | 20260506 21:20:06.131 | INFO | Downloaded 34GiB to GPU in 1.97406s (17.2234GiB/s) | 20260506 21:20:06.229 | INFO | Starting a round of Baz IO | 20260506 21:20:09.360 | INFO | Populating RTMetrics report | 20260506 21:20:09.360 | INFO | RTMetrics report populated | 20260506 21:20:09.360 | INFO | Opening tmp RTMetrics json file | 20260506 21:20:09.418 | INFO | Serializing RTMetrics to json | 20260506 21:20:09.471 | INFO | Serialization done, swapping tmp file now... | 20260506 21:20:09.472 | INFO | RTMetrics done | 20260506 21:20:10.009 | INFO | Finished a round of Baz IO: Wrote 1.949 GiB in 3.78 seconds | 20260506 21:20:10.009 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.97% / 0.00% / 66.03% | 20260506 21:20:10.009 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.626965% / 7.94729e-06% / 5.96046e-05% | 20260506 21:20:10.010 | INFO | Baz IO overall: | 20260506 21:20:10.010 | INFO | BytesWritten : 2389755340318 | 20260506 21:20:10.010 | INFO | NumEvents : 1153923905108 | 20260506 21:20:11.281 | INFO | Analyzing chunk frames = [4534272,4534784) | 20260506 21:20:11.281 | INFO | MemInfo: { | 20260506 21:20:11.281 | INFO | "gpu" : " 17 GB", | 20260506 21:20:11.281 | INFO | "huge" : "", | 20260506 21:20:11.282 | INFO | "huge_shared" : " 27 GB", | 20260506 21:20:11.282 | INFO | "malloc" : " 14 GB", | 20260506 21:20:11.282 | INFO | "pinned" : " 29 GB" | 20260506 21:20:11.282 | INFO | } | 20260506 21:20:11.282 | INFO | Perf report for the last 5.1509 seconds: | 20260506 21:20:11.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:20:11.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:11.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:11.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:11.282 | INFO | ANALYSIS : 62.1%, 2.9, 95.1ms, 0 | 20260506 21:20:11.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:11.282 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 21:20:11.282 | INFO | PRE_HQ : 59.6%, 1.8, 56.5ms, 0 | 20260506 21:20:11.282 | INFO | BAZWRITER : 73.0%, 4.8, 180.1ms, 0 | 20260506 21:20:11.282 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 21:20:11.282 | INFO | Uploaded 36GiB to GPU in 2.04756s (17.5819GiB/s) | 20260506 21:20:11.282 | INFO | Downloaded 38GiB to GPU in 2.19598s (17.3043GiB/s) | 20260506 21:20:16.418 | INFO | Analyzing chunk frames = [4534784,4535296) | 20260506 21:20:16.418 | INFO | MemInfo: { | 20260506 21:20:16.418 | INFO | "gpu" : " 17 GB", | 20260506 21:20:16.418 | INFO | "huge" : "", | 20260506 21:20:16.418 | INFO | "huge_shared" : " 27 GB", | 20260506 21:20:16.418 | INFO | "malloc" : " 14 GB", | 20260506 21:20:16.418 | INFO | "pinned" : " 29 GB" | 20260506 21:20:16.418 | INFO | } | 20260506 21:20:16.418 | INFO | Perf report for the last 5.13613 seconds: | 20260506 21:20:16.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:20:16.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:16.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:16.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:16.418 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 21:20:16.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:16.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:16.418 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:20:16.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:16.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:16.418 | INFO | Uploaded 36GiB to GPU in 1.99938s (18.0056GiB/s) | 20260506 21:20:16.418 | INFO | Downloaded 34GiB to GPU in 1.92495s (17.6628GiB/s) | 20260506 21:20:21.556 | INFO | Analyzing chunk frames = [4535296,4535808) | 20260506 21:20:21.556 | INFO | MemInfo: { | 20260506 21:20:21.556 | INFO | "gpu" : " 17 GB", | 20260506 21:20:21.556 | INFO | "huge" : "", | 20260506 21:20:21.556 | INFO | "huge_shared" : " 27 GB", | 20260506 21:20:21.556 | INFO | "malloc" : " 14 GB", | 20260506 21:20:21.556 | INFO | "pinned" : " 29 GB" | 20260506 21:20:21.556 | INFO | } | 20260506 21:20:21.556 | INFO | Perf report for the last 5.13792 seconds: | 20260506 21:20:21.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:20:21.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:21.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:21.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:21.556 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 21:20:21.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:21.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:21.556 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:20:21.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:21.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:21.556 | INFO | Uploaded 36GiB to GPU in 1.98792s (18.1094GiB/s) | 20260506 21:20:21.556 | INFO | Downloaded 34GiB to GPU in 1.9268s (17.6458GiB/s) | 20260506 21:20:26.691 | INFO | Analyzing chunk frames = [4535808,4536320) | 20260506 21:20:26.691 | INFO | MemInfo: { | 20260506 21:20:26.691 | INFO | "gpu" : " 17 GB", | 20260506 21:20:26.691 | INFO | "huge" : "", | 20260506 21:20:26.691 | INFO | "huge_shared" : " 27 GB", | 20260506 21:20:26.691 | INFO | "malloc" : " 14 GB", | 20260506 21:20:26.691 | INFO | "pinned" : " 29 GB" | 20260506 21:20:26.691 | INFO | } | 20260506 21:20:26.691 | INFO | Perf report for the last 5.13532 seconds: | 20260506 21:20:26.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:20:26.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:26.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:26.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:26.691 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 21:20:26.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:26.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:26.691 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:20:26.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:26.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:26.691 | INFO | Uploaded 36GiB to GPU in 1.97419s (18.2353GiB/s) | 20260506 21:20:26.691 | INFO | Downloaded 34GiB to GPU in 1.8737s (18.1459GiB/s) | 20260506 21:20:31.829 | INFO | Analyzing chunk frames = [4536320,4536832) | 20260506 21:20:31.829 | INFO | MemInfo: { | 20260506 21:20:31.829 | INFO | "gpu" : " 17 GB", | 20260506 21:20:31.829 | INFO | "huge" : "", | 20260506 21:20:31.829 | INFO | "huge_shared" : " 27 GB", | 20260506 21:20:31.829 | INFO | "malloc" : " 14 GB", | 20260506 21:20:31.829 | INFO | "pinned" : " 29 GB" | 20260506 21:20:31.829 | INFO | } | 20260506 21:20:31.829 | INFO | Perf report for the last 5.13773 seconds: | 20260506 21:20:31.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:20:31.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:31.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:31.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:31.829 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 21:20:31.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:31.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:31.829 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:20:31.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:31.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:31.829 | INFO | Uploaded 36GiB to GPU in 2.02247s (17.8GiB/s) | 20260506 21:20:31.829 | INFO | Downloaded 34GiB to GPU in 1.91282s (17.7748GiB/s) | 20260506 21:20:36.966 | INFO | Analyzing chunk frames = [4536832,4537344) | 20260506 21:20:36.966 | INFO | MemInfo: { | 20260506 21:20:36.966 | INFO | "gpu" : " 17 GB", | 20260506 21:20:36.966 | INFO | "huge" : "", | 20260506 21:20:36.966 | INFO | "huge_shared" : " 27 GB", | 20260506 21:20:36.966 | INFO | "malloc" : " 14 GB", | 20260506 21:20:36.966 | INFO | "pinned" : " 29 GB" | 20260506 21:20:36.966 | INFO | } | 20260506 21:20:36.966 | INFO | Perf report for the last 5.13728 seconds: | 20260506 21:20:36.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:20:36.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:36.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:36.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:36.966 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 21:20:36.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:36.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:36.966 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 21:20:36.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:36.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:36.966 | INFO | Uploaded 36GiB to GPU in 2.02744s (17.7564GiB/s) | 20260506 21:20:36.966 | INFO | Downloaded 34GiB to GPU in 1.95505s (17.3909GiB/s) | 20260506 21:20:42.111 | INFO | Analyzing chunk frames = [4537344,4537856) | 20260506 21:20:42.111 | INFO | MemInfo: { | 20260506 21:20:42.111 | INFO | "gpu" : " 17 GB", | 20260506 21:20:42.111 | INFO | "huge" : "", | 20260506 21:20:42.111 | INFO | "huge_shared" : " 27 GB", | 20260506 21:20:42.111 | INFO | "malloc" : " 14 GB", | 20260506 21:20:42.111 | INFO | "pinned" : " 29 GB" | 20260506 21:20:42.111 | INFO | } | 20260506 21:20:42.112 | INFO | Perf report for the last 5.14551 seconds: | 20260506 21:20:42.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:20:42.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:42.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:42.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:42.112 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 21:20:42.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:42.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:42.112 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 21:20:42.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:42.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:42.112 | INFO | Uploaded 36GiB to GPU in 1.97429s (18.2344GiB/s) | 20260506 21:20:42.112 | INFO | Downloaded 34GiB to GPU in 2.00241s (16.9795GiB/s) | 20260506 21:20:47.249 | INFO | Analyzing chunk frames = [4537856,4538368) | 20260506 21:20:47.249 | INFO | MemInfo: { | 20260506 21:20:47.249 | INFO | "gpu" : " 17 GB", | 20260506 21:20:47.249 | INFO | "huge" : "", | 20260506 21:20:47.249 | INFO | "huge_shared" : " 27 GB", | 20260506 21:20:47.249 | INFO | "malloc" : " 14 GB", | 20260506 21:20:47.249 | INFO | "pinned" : " 29 GB" | 20260506 21:20:47.249 | INFO | } | 20260506 21:20:47.249 | INFO | Perf report for the last 5.1377 seconds: | 20260506 21:20:47.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:20:47.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:47.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:47.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:47.249 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:20:47.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:47.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:47.249 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:20:47.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:47.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:47.249 | INFO | Uploaded 36GiB to GPU in 2.03391s (17.6999GiB/s) | 20260506 21:20:47.250 | INFO | Downloaded 34GiB to GPU in 1.99528s (17.0402GiB/s) | 20260506 21:20:47.250 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:20:47.250 | INFO | Total mem usage: 18 GiB | 20260506 21:20:47.250 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:20:47.250 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.4%) | 20260506 21:20:47.250 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:20:47.250 | INFO | High Water Breakdown: | 20260506 21:20:47.250 | INFO | 61.7% : 9.8 GiB ( 85.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:20:47.250 | INFO | 38.1% : 6.06 GiB ( 90.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:20:47.250 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:20:47.250 | INFO | ------------------------------------------ | 20260506 21:20:47.250 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:20:47.250 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:20:47.250 | INFO | Total mem usage: 26.5 GiB | 20260506 21:20:47.250 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:20:47.250 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:20:47.250 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:20:47.250 | INFO | High Water Breakdown: | 20260506 21:20:47.250 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:20:47.250 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:20:47.250 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:20:47.250 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:20:47.250 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:20:47.250 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:20:47.250 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:20:47.250 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:20:47.250 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:20:47.250 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:20:47.250 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:20:47.250 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:20:47.250 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:20:47.250 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:20:47.250 | INFO | ------------------------------------------ | 20260506 21:20:47.250 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:20:47.250 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:20:47.250 | INFO | Total mem usage: 25.6 GiB | 20260506 21:20:47.250 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:20:47.250 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 21:20:47.250 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 21:20:47.250 | INFO | High Water Breakdown: | 20260506 21:20:47.250 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 21:20:47.250 | INFO | ------------------------------------------ | 20260506 21:20:47.250 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:20:47.250 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:20:47.250 | INFO | Total mem usage: 15.7 GiB | 20260506 21:20:47.250 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:20:47.250 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:20:47.250 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:20:47.250 | INFO | High Water Breakdown: | 20260506 21:20:47.250 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:20:47.250 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:20:47.250 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:20:47.250 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:20:47.250 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:20:47.250 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:20:47.250 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:20:47.250 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:20:47.250 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:20:47.250 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:20:47.250 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:20:47.250 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:20:47.250 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:20:47.250 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:20:47.250 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:20:47.250 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:20:47.250 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:20:47.250 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:20:47.250 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:20:47.250 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:20:47.250 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:20:47.250 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:20:47.250 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:20:47.250 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:20:47.250 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:20:47.250 | INFO | ------------------------------------------ | 20260506 21:20:47.250 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:20:47.250 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:20:47.250 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:20:47.250 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:20:47.250 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.63e+01, 1.35e+01 / 3.02e+00 | 20260506 21:20:47.250 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.03e-01, 9.80e-03 / 9.68e-03 | 20260506 21:20:47.250 | INFO | PerfReport: 11.66%, Baseline -- 3.63e+00 / 4.41e+00, 5.54e-01 / 3.61e-01 | 20260506 21:20:47.250 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.80e-01 / 1.41e-01 | 20260506 21:20:47.250 | INFO | PerfReport: 16.08%, DME -- 4.00e+01 / 4.52e+01, 4.22e+00 / 3.01e+00 | 20260506 21:20:47.250 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.38e+00 / 1.77e+00 | 20260506 21:20:47.250 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.21e+00 / 2.84e+00, 3.76e-01 / 1.83e-01 | 20260506 21:20:47.250 | INFO | PerfReport: 3.26%, Metrics -- 1.01e+00 / 1.36e+00, 2.67e-01 / 1.61e-01 | 20260506 21:20:47.250 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.48e-01 / 3.95e-01, 5.45e-02 / 2.75e-02 | 20260506 21:20:47.250 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:20:47.250 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:20:47.250 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:20:47.250 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 21:20:47.250 | INFO | PerfReport: 21.50%, Upload -- 2.08e+01 / 0.00e+00, 6.90e+00 / 0.00e+00 | 20260506 21:20:47.250 | INFO | PerfReport: 21.26%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 21:20:47.250 | INFO | PerfReport: 32.18%, Compute -- 3.11e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 21:20:47.250 | INFO | PerfReport: 1.24%, LockUpload -- 1.20e+00 / 0.00e+00, 4.67e+00 / 0.00e+00 | 20260506 21:20:47.250 | INFO | PerfReport: 1.26%, LockDownload -- 1.22e+00 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260506 21:20:47.251 | INFO | PerfReport: 22.53%, LockCompute -- 2.18e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 21:20:47.251 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.39e-02 / 0.00e+00 | 20260506 21:20:47.364 | INFO | Starting a round of Baz IO | 20260506 21:20:50.494 | INFO | Populating RTMetrics report | 20260506 21:20:50.494 | INFO | RTMetrics report populated | 20260506 21:20:50.494 | INFO | Opening tmp RTMetrics json file | 20260506 21:20:50.534 | INFO | Serializing RTMetrics to json | 20260506 21:20:50.570 | INFO | Serialization done, swapping tmp file now... | 20260506 21:20:50.582 | INFO | RTMetrics done | 20260506 21:20:50.888 | INFO | Finished a round of Baz IO: Wrote 1.949 GiB in 3.524 seconds | 20260506 21:20:50.888 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.97% / 0.00% / 66.03% | 20260506 21:20:50.888 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.62503% / 0% / 5.16574e-05% | 20260506 21:20:50.889 | INFO | Baz IO overall: | 20260506 21:20:50.889 | INFO | BytesWritten : 2391847676086 | 20260506 21:20:50.889 | INFO | NumEvents : 1154925267072 | 20260506 21:20:52.388 | INFO | Analyzing chunk frames = [4538368,4538880) | 20260506 21:20:52.388 | INFO | MemInfo: { | 20260506 21:20:52.388 | INFO | "gpu" : " 17 GB", | 20260506 21:20:52.388 | INFO | "huge" : "", | 20260506 21:20:52.388 | INFO | "huge_shared" : " 27 GB", | 20260506 21:20:52.388 | INFO | "malloc" : " 14 GB", | 20260506 21:20:52.388 | INFO | "pinned" : " 29 GB" | 20260506 21:20:52.388 | INFO | } | 20260506 21:20:52.388 | INFO | Perf report for the last 5.13859 seconds: | 20260506 21:20:52.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:20:52.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:52.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:52.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:52.388 | INFO | ANALYSIS : 62.6%, 2.9, 95.6ms, 0 | 20260506 21:20:52.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:52.388 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260506 21:20:52.388 | INFO | PRE_HQ : 59.7%, 1.8, 56.7ms, 0 | 20260506 21:20:52.388 | INFO | BAZWRITER : 68.3%, 4.8, 167.5ms, 0 | 20260506 21:20:52.388 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 21:20:52.388 | INFO | Uploaded 36GiB to GPU in 2.04503s (17.6037GiB/s) | 20260506 21:20:52.388 | INFO | Downloaded 38GiB to GPU in 2.22108s (17.1088GiB/s) | 20260506 21:20:57.532 | INFO | Analyzing chunk frames = [4538880,4539392) | 20260506 21:20:57.532 | INFO | MemInfo: { | 20260506 21:20:57.532 | INFO | "gpu" : " 17 GB", | 20260506 21:20:57.532 | INFO | "huge" : "", | 20260506 21:20:57.532 | INFO | "huge_shared" : " 27 GB", | 20260506 21:20:57.532 | INFO | "malloc" : " 14 GB", | 20260506 21:20:57.532 | INFO | "pinned" : " 29 GB" | 20260506 21:20:57.533 | INFO | } | 20260506 21:20:57.533 | INFO | Perf report for the last 5.14468 seconds: | 20260506 21:20:57.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:20:57.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:57.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:57.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:57.533 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 21:20:57.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:57.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:57.533 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 21:20:57.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:57.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:20:57.533 | INFO | Uploaded 36GiB to GPU in 2.00794s (17.9289GiB/s) | 20260506 21:20:57.533 | INFO | Downloaded 34GiB to GPU in 1.92977s (17.6187GiB/s) | 20260506 21:21:02.667 | INFO | Analyzing chunk frames = [4539392,4539904) | 20260506 21:21:02.667 | INFO | MemInfo: { | 20260506 21:21:02.667 | INFO | "gpu" : " 17 GB", | 20260506 21:21:02.667 | INFO | "huge" : "", | 20260506 21:21:02.667 | INFO | "huge_shared" : " 27 GB", | 20260506 21:21:02.667 | INFO | "malloc" : " 14 GB", | 20260506 21:21:02.667 | INFO | "pinned" : " 29 GB" | 20260506 21:21:02.667 | INFO | } | 20260506 21:21:02.667 | INFO | Perf report for the last 5.13438 seconds: | 20260506 21:21:02.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:21:02.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:02.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:02.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:02.667 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 21:21:02.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:02.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:02.667 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:21:02.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:02.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:02.667 | INFO | Uploaded 36GiB to GPU in 1.96516s (18.3191GiB/s) | 20260506 21:21:02.667 | INFO | Downloaded 34GiB to GPU in 1.93203s (17.5981GiB/s) | 20260506 21:21:07.807 | INFO | Analyzing chunk frames = [4539904,4540416) | 20260506 21:21:07.807 | INFO | MemInfo: { | 20260506 21:21:07.807 | INFO | "gpu" : " 17 GB", | 20260506 21:21:07.807 | INFO | "huge" : "", | 20260506 21:21:07.807 | INFO | "huge_shared" : " 27 GB", | 20260506 21:21:07.807 | INFO | "malloc" : " 14 GB", | 20260506 21:21:07.807 | INFO | "pinned" : " 29 GB" | 20260506 21:21:07.807 | INFO | } | 20260506 21:21:07.807 | INFO | Perf report for the last 5.14026 seconds: | 20260506 21:21:07.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:21:07.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:07.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:07.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:07.807 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 21:21:07.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:07.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:07.807 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:21:07.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:07.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:07.808 | INFO | Uploaded 36GiB to GPU in 1.95527s (18.4118GiB/s) | 20260506 21:21:07.808 | INFO | Downloaded 34GiB to GPU in 1.8787s (18.0976GiB/s) | 20260506 21:21:12.946 | INFO | Analyzing chunk frames = [4540416,4540928) | 20260506 21:21:12.946 | INFO | MemInfo: { | 20260506 21:21:12.946 | INFO | "gpu" : " 17 GB", | 20260506 21:21:12.946 | INFO | "huge" : "", | 20260506 21:21:12.946 | INFO | "huge_shared" : " 27 GB", | 20260506 21:21:12.946 | INFO | "malloc" : " 14 GB", | 20260506 21:21:12.946 | INFO | "pinned" : " 29 GB" | 20260506 21:21:12.946 | INFO | } | 20260506 21:21:12.946 | INFO | Perf report for the last 5.1387 seconds: | 20260506 21:21:12.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:21:12.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:12.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:12.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:12.946 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 21:21:12.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:12.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:12.946 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:21:12.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:12.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:12.946 | INFO | Uploaded 36GiB to GPU in 1.98038s (18.1784GiB/s) | 20260506 21:21:12.946 | INFO | Downloaded 34GiB to GPU in 1.93003s (17.6163GiB/s) | 20260506 21:21:18.081 | INFO | Analyzing chunk frames = [4540928,4541440) | 20260506 21:21:18.081 | INFO | MemInfo: { | 20260506 21:21:18.081 | INFO | "gpu" : " 17 GB", | 20260506 21:21:18.081 | INFO | "huge" : "", | 20260506 21:21:18.081 | INFO | "huge_shared" : " 27 GB", | 20260506 21:21:18.081 | INFO | "malloc" : " 14 GB", | 20260506 21:21:18.081 | INFO | "pinned" : " 29 GB" | 20260506 21:21:18.081 | INFO | } | 20260506 21:21:18.081 | INFO | Perf report for the last 5.13527 seconds: | 20260506 21:21:18.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:21:18.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:18.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:18.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:18.081 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 21:21:18.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:18.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:18.081 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 21:21:18.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:18.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:18.081 | INFO | Uploaded 36GiB to GPU in 1.99721s (18.0252GiB/s) | 20260506 21:21:18.082 | INFO | Downloaded 34GiB to GPU in 1.95393s (17.4009GiB/s) | 20260506 21:21:23.224 | INFO | Analyzing chunk frames = [4541440,4541952) | 20260506 21:21:23.224 | INFO | MemInfo: { | 20260506 21:21:23.224 | INFO | "gpu" : " 17 GB", | 20260506 21:21:23.224 | INFO | "huge" : "", | 20260506 21:21:23.224 | INFO | "huge_shared" : " 27 GB", | 20260506 21:21:23.224 | INFO | "malloc" : " 14 GB", | 20260506 21:21:23.224 | INFO | "pinned" : " 29 GB" | 20260506 21:21:23.224 | INFO | } | 20260506 21:21:23.224 | INFO | Perf report for the last 5.14254 seconds: | 20260506 21:21:23.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:21:23.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:23.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:23.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:23.224 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:21:23.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:23.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:23.224 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:21:23.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:23.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:23.224 | INFO | Uploaded 36GiB to GPU in 1.95585s (18.4063GiB/s) | 20260506 21:21:23.224 | INFO | Downloaded 34GiB to GPU in 2.00096s (16.9918GiB/s) | 20260506 21:21:28.373 | INFO | Analyzing chunk frames = [4541952,4542464) | 20260506 21:21:28.373 | INFO | MemInfo: { | 20260506 21:21:28.373 | INFO | "gpu" : " 17 GB", | 20260506 21:21:28.373 | INFO | "huge" : "", | 20260506 21:21:28.373 | INFO | "huge_shared" : " 27 GB", | 20260506 21:21:28.373 | INFO | "malloc" : " 14 GB", | 20260506 21:21:28.373 | INFO | "pinned" : " 29 GB" | 20260506 21:21:28.373 | INFO | } | 20260506 21:21:28.373 | INFO | Perf report for the last 5.14888 seconds: | 20260506 21:21:28.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:21:28.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:28.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:28.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:28.373 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 21:21:28.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:28.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:28.373 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 21:21:28.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:28.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:28.373 | INFO | Uploaded 36GiB to GPU in 2.05077s (17.5544GiB/s) | 20260506 21:21:28.373 | INFO | Downloaded 34GiB to GPU in 1.99652s (17.0296GiB/s) | 20260506 21:21:28.482 | INFO | Starting a round of Baz IO | 20260506 21:21:31.613 | INFO | Populating RTMetrics report | 20260506 21:21:31.613 | INFO | RTMetrics report populated | 20260506 21:21:31.613 | INFO | Opening tmp RTMetrics json file | 20260506 21:21:31.620 | INFO | Serializing RTMetrics to json | 20260506 21:21:31.653 | INFO | Serialization done, swapping tmp file now... | 20260506 21:21:31.670 | INFO | RTMetrics done | 20260506 21:21:31.781 | INFO | Finished a round of Baz IO: Wrote 1.948 GiB in 3.299 seconds | 20260506 21:21:31.781 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.96% / 0.00% / 66.03% | 20260506 21:21:31.781 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.623949% / 0% / 2.78155e-05% | 20260506 21:21:31.781 | INFO | Baz IO overall: | 20260506 21:21:31.781 | INFO | BytesWritten : 2393939766370 | 20260506 21:21:31.781 | INFO | NumEvents : 1155927063414 | 20260506 21:21:33.505 | INFO | Analyzing chunk frames = [4542464,4542976) | 20260506 21:21:33.505 | INFO | MemInfo: { | 20260506 21:21:33.505 | INFO | "gpu" : " 17 GB", | 20260506 21:21:33.505 | INFO | "huge" : "", | 20260506 21:21:33.505 | INFO | "huge_shared" : " 27 GB", | 20260506 21:21:33.505 | INFO | "malloc" : " 14 GB", | 20260506 21:21:33.505 | INFO | "pinned" : " 29 GB" | 20260506 21:21:33.505 | INFO | } | 20260506 21:21:33.505 | INFO | Perf report for the last 5.13243 seconds: | 20260506 21:21:33.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:21:33.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:33.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:33.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:33.505 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 21:21:33.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:33.505 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260506 21:21:33.505 | INFO | PRE_HQ : 59.9%, 1.8, 57.0ms, 0 | 20260506 21:21:33.505 | INFO | BAZWRITER : 64.0%, 4.5, 146.8ms, 0 | 20260506 21:21:33.505 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 21:21:33.506 | INFO | Uploaded 36GiB to GPU in 2.0633s (17.4478GiB/s) | 20260506 21:21:33.506 | INFO | Downloaded 38GiB to GPU in 2.20195s (17.2574GiB/s) | 20260506 21:21:38.643 | INFO | Analyzing chunk frames = [4542976,4543488) | 20260506 21:21:38.643 | INFO | MemInfo: { | 20260506 21:21:38.643 | INFO | "gpu" : " 17 GB", | 20260506 21:21:38.643 | INFO | "huge" : "", | 20260506 21:21:38.643 | INFO | "huge_shared" : " 27 GB", | 20260506 21:21:38.643 | INFO | "malloc" : " 14 GB", | 20260506 21:21:38.643 | INFO | "pinned" : " 29 GB" | 20260506 21:21:38.643 | INFO | } | 20260506 21:21:38.643 | INFO | Perf report for the last 5.13766 seconds: | 20260506 21:21:38.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:21:38.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:38.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:38.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:38.643 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:21:38.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:38.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:38.643 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:21:38.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:38.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:38.643 | INFO | Uploaded 36GiB to GPU in 2.02164s (17.8073GiB/s) | 20260506 21:21:38.643 | INFO | Downloaded 34GiB to GPU in 1.9275s (17.6395GiB/s) | 20260506 21:21:43.780 | INFO | Analyzing chunk frames = [4543488,4544000) | 20260506 21:21:43.780 | INFO | MemInfo: { | 20260506 21:21:43.780 | INFO | "gpu" : " 17 GB", | 20260506 21:21:43.780 | INFO | "huge" : "", | 20260506 21:21:43.780 | INFO | "huge_shared" : " 27 GB", | 20260506 21:21:43.780 | INFO | "malloc" : " 14 GB", | 20260506 21:21:43.780 | INFO | "pinned" : " 29 GB" | 20260506 21:21:43.780 | INFO | } | 20260506 21:21:43.780 | INFO | Perf report for the last 5.13703 seconds: | 20260506 21:21:43.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:21:43.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:43.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:43.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:43.780 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 21:21:43.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:43.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:43.780 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:21:43.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:43.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:43.780 | INFO | Uploaded 36GiB to GPU in 1.99248s (18.0679GiB/s) | 20260506 21:21:43.780 | INFO | Downloaded 34GiB to GPU in 1.92097s (17.6994GiB/s) | 20260506 21:21:48.919 | INFO | Analyzing chunk frames = [4544000,4544512) | 20260506 21:21:48.919 | INFO | MemInfo: { | 20260506 21:21:48.919 | INFO | "gpu" : " 17 GB", | 20260506 21:21:48.919 | INFO | "huge" : "", | 20260506 21:21:48.919 | INFO | "huge_shared" : " 27 GB", | 20260506 21:21:48.919 | INFO | "malloc" : " 14 GB", | 20260506 21:21:48.919 | INFO | "pinned" : " 29 GB" | 20260506 21:21:48.919 | INFO | } | 20260506 21:21:48.920 | INFO | Perf report for the last 5.13968 seconds: | 20260506 21:21:48.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:21:48.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:48.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:48.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:48.920 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 21:21:48.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:48.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:48.920 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:21:48.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:48.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:48.920 | INFO | Uploaded 36GiB to GPU in 1.94261s (18.5318GiB/s) | 20260506 21:21:48.920 | INFO | Downloaded 34GiB to GPU in 1.86936s (18.188GiB/s) | 20260506 21:21:54.059 | INFO | Analyzing chunk frames = [4544512,4545024) | 20260506 21:21:54.059 | INFO | MemInfo: { | 20260506 21:21:54.059 | INFO | "gpu" : " 17 GB", | 20260506 21:21:54.059 | INFO | "huge" : "", | 20260506 21:21:54.059 | INFO | "huge_shared" : " 27 GB", | 20260506 21:21:54.059 | INFO | "malloc" : " 14 GB", | 20260506 21:21:54.059 | INFO | "pinned" : " 29 GB" | 20260506 21:21:54.059 | INFO | } | 20260506 21:21:54.059 | INFO | Perf report for the last 5.13929 seconds: | 20260506 21:21:54.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:21:54.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:54.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:54.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:54.059 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 21:21:54.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:54.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:54.059 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:21:54.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:54.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:54.059 | INFO | Uploaded 36GiB to GPU in 1.9995s (18.0045GiB/s) | 20260506 21:21:54.059 | INFO | Downloaded 34GiB to GPU in 1.92002s (17.7082GiB/s) | 20260506 21:21:59.197 | INFO | Analyzing chunk frames = [4545024,4545536) | 20260506 21:21:59.197 | INFO | MemInfo: { | 20260506 21:21:59.197 | INFO | "gpu" : " 17 GB", | 20260506 21:21:59.197 | INFO | "huge" : "", | 20260506 21:21:59.197 | INFO | "huge_shared" : " 27 GB", | 20260506 21:21:59.197 | INFO | "malloc" : " 14 GB", | 20260506 21:21:59.197 | INFO | "pinned" : " 29 GB" | 20260506 21:21:59.197 | INFO | } | 20260506 21:21:59.197 | INFO | Perf report for the last 5.13806 seconds: | 20260506 21:21:59.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:21:59.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:59.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:59.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:59.197 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260506 21:21:59.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:59.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:59.197 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 21:21:59.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:59.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:21:59.197 | INFO | Uploaded 36GiB to GPU in 1.99831s (18.0152GiB/s) | 20260506 21:21:59.197 | INFO | Downloaded 34GiB to GPU in 1.93244s (17.5944GiB/s) | 20260506 21:22:04.339 | INFO | Analyzing chunk frames = [4545536,4546048) | 20260506 21:22:04.339 | INFO | MemInfo: { | 20260506 21:22:04.339 | INFO | "gpu" : " 17 GB", | 20260506 21:22:04.339 | INFO | "huge" : "", | 20260506 21:22:04.339 | INFO | "huge_shared" : " 27 GB", | 20260506 21:22:04.339 | INFO | "malloc" : " 14 GB", | 20260506 21:22:04.339 | INFO | "pinned" : " 29 GB" | 20260506 21:22:04.339 | INFO | } | 20260506 21:22:04.339 | INFO | Perf report for the last 5.14219 seconds: | 20260506 21:22:04.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:22:04.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:04.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:04.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:04.339 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 21:22:04.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:04.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:04.339 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 21:22:04.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:04.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:04.339 | INFO | Uploaded 36GiB to GPU in 1.97316s (18.2449GiB/s) | 20260506 21:22:04.339 | INFO | Downloaded 34GiB to GPU in 2.00344s (16.9708GiB/s) | 20260506 21:22:09.478 | INFO | Analyzing chunk frames = [4546048,4546560) | 20260506 21:22:09.478 | INFO | MemInfo: { | 20260506 21:22:09.478 | INFO | "gpu" : " 17 GB", | 20260506 21:22:09.478 | INFO | "huge" : "", | 20260506 21:22:09.478 | INFO | "huge_shared" : " 27 GB", | 20260506 21:22:09.478 | INFO | "malloc" : " 14 GB", | 20260506 21:22:09.478 | INFO | "pinned" : " 29 GB" | 20260506 21:22:09.478 | INFO | } | 20260506 21:22:09.478 | INFO | Perf report for the last 5.13905 seconds: | 20260506 21:22:09.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:22:09.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:09.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:09.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:09.478 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260506 21:22:09.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:09.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:09.478 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 21:22:09.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:09.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:09.478 | INFO | Uploaded 36GiB to GPU in 2.03367s (17.702GiB/s) | 20260506 21:22:09.478 | INFO | Downloaded 34GiB to GPU in 1.98758s (17.1062GiB/s) | 20260506 21:22:09.478 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:22:09.478 | INFO | Total mem usage: 18 GiB | 20260506 21:22:09.478 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:22:09.478 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.6%) | 20260506 21:22:09.478 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:22:09.478 | INFO | High Water Breakdown: | 20260506 21:22:09.479 | INFO | 61.5% : 9.57 GiB ( 83.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:22:09.479 | INFO | 38.4% : 5.98 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:22:09.479 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:22:09.479 | INFO | ------------------------------------------ | 20260506 21:22:09.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:22:09.479 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:22:09.479 | INFO | Total mem usage: 26.5 GiB | 20260506 21:22:09.479 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:22:09.479 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:22:09.479 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:22:09.479 | INFO | High Water Breakdown: | 20260506 21:22:09.479 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:22:09.479 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:22:09.479 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:22:09.479 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:22:09.479 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:22:09.479 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:22:09.479 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:22:09.479 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:22:09.479 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:22:09.479 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:22:09.479 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:22:09.479 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:22:09.479 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:22:09.479 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:22:09.479 | INFO | ------------------------------------------ | 20260506 21:22:09.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:22:09.479 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:22:09.479 | INFO | Total mem usage: 25.6 GiB | 20260506 21:22:09.479 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:22:09.479 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:22:09.479 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:22:09.479 | INFO | High Water Breakdown: | 20260506 21:22:09.479 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:22:09.479 | INFO | ------------------------------------------ | 20260506 21:22:09.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:22:09.479 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:22:09.479 | INFO | Total mem usage: 15.7 GiB | 20260506 21:22:09.479 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:22:09.479 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:22:09.479 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:22:09.479 | INFO | High Water Breakdown: | 20260506 21:22:09.479 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:22:09.479 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:22:09.479 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:22:09.479 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:22:09.479 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:22:09.479 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:22:09.479 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:22:09.479 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:22:09.479 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:22:09.479 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:22:09.479 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:22:09.479 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:22:09.479 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:22:09.479 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:22:09.479 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:22:09.479 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:22:09.479 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:22:09.479 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:22:09.479 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:22:09.479 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:22:09.479 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:22:09.479 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:22:09.479 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:22:09.479 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:22:09.479 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:22:09.479 | INFO | ------------------------------------------ | 20260506 21:22:09.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:22:09.479 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:22:09.479 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:22:09.479 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:22:09.479 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.57e+01, 1.34e+01 / 2.91e+00 | 20260506 21:22:09.479 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 1.01e-02 / 5.79e-03 | 20260506 21:22:09.480 | INFO | PerfReport: 11.63%, Baseline -- 3.62e+00 / 4.36e+00, 5.53e-01 / 3.66e-01 | 20260506 21:22:09.480 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.44e+00, 1.83e-01 / 1.43e-01 | 20260506 21:22:09.480 | INFO | PerfReport: 16.03%, DME -- 3.99e+01 / 4.47e+01, 4.16e+00 / 2.96e+00 | 20260506 21:22:09.480 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.44e+00 / 1.83e+00 | 20260506 21:22:09.480 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.21e+00 / 2.72e+00, 3.76e-01 / 2.40e-01 | 20260506 21:22:09.480 | INFO | PerfReport: 3.27%, Metrics -- 1.02e+00 / 1.34e+00, 2.69e-01 / 1.74e-01 | 20260506 21:22:09.480 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.48e-01 / 3.95e-01, 5.37e-02 / 2.63e-02 | 20260506 21:22:09.480 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:22:09.480 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:22:09.480 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:22:09.480 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.31e+01 / 0.00e+00 | 20260506 21:22:09.480 | INFO | PerfReport: 21.54%, Upload -- 2.08e+01 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260506 21:22:09.480 | INFO | PerfReport: 21.29%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 21:22:09.480 | INFO | PerfReport: 32.20%, Compute -- 3.11e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 21:22:09.480 | INFO | PerfReport: 1.25%, LockUpload -- 1.21e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260506 21:22:09.480 | INFO | PerfReport: 1.23%, LockDownload -- 1.19e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260506 21:22:09.480 | INFO | PerfReport: 22.49%, LockCompute -- 2.17e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 21:22:09.480 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.31e-02 / 0.00e+00 | 20260506 21:22:09.592 | INFO | Starting a round of Baz IO | 20260506 21:22:12.723 | INFO | Populating RTMetrics report | 20260506 21:22:12.723 | INFO | RTMetrics report populated | 20260506 21:22:12.723 | INFO | Opening tmp RTMetrics json file | 20260506 21:22:12.726 | INFO | Serializing RTMetrics to json | 20260506 21:22:12.758 | INFO | Serialization done, swapping tmp file now... | 20260506 21:22:12.761 | INFO | RTMetrics done | 20260506 21:22:12.867 | INFO | Finished a round of Baz IO: Wrote 1.948 GiB in 3.274 seconds | 20260506 21:22:12.867 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.96% / 0.00% / 66.04% | 20260506 21:22:12.867 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.623246% / 7.94729e-06% / 5.96046e-05% | 20260506 21:22:12.867 | INFO | Baz IO overall: | 20260506 21:22:12.867 | INFO | BytesWritten : 2396031017418 | 20260506 21:22:12.867 | INFO | NumEvents : 1156928142937 | 20260506 21:22:14.623 | INFO | Analyzing chunk frames = [4546560,4547072) | 20260506 21:22:14.623 | INFO | MemInfo: { | 20260506 21:22:14.623 | INFO | "gpu" : " 17 GB", | 20260506 21:22:14.623 | INFO | "huge" : "", | 20260506 21:22:14.623 | INFO | "huge_shared" : " 27 GB", | 20260506 21:22:14.623 | INFO | "malloc" : " 14 GB", | 20260506 21:22:14.623 | INFO | "pinned" : " 29 GB" | 20260506 21:22:14.623 | INFO | } | 20260506 21:22:14.623 | INFO | Perf report for the last 5.14529 seconds: | 20260506 21:22:14.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:22:14.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:14.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:14.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:14.623 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 21:22:14.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:14.623 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260506 21:22:14.624 | INFO | PRE_HQ : 60.1%, 1.8, 57.3ms, 0 | 20260506 21:22:14.624 | INFO | BAZWRITER : 63.4%, 4.4, 144.3ms, 0 | 20260506 21:22:14.624 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 21:22:14.624 | INFO | Uploaded 36GiB to GPU in 2.08098s (17.2995GiB/s) | 20260506 21:22:14.624 | INFO | Downloaded 38GiB to GPU in 2.21303s (17.1711GiB/s) | 20260506 21:22:19.761 | INFO | Analyzing chunk frames = [4547072,4547584) | 20260506 21:22:19.761 | INFO | MemInfo: { | 20260506 21:22:19.761 | INFO | "gpu" : " 17 GB", | 20260506 21:22:19.761 | INFO | "huge" : "", | 20260506 21:22:19.761 | INFO | "huge_shared" : " 27 GB", | 20260506 21:22:19.761 | INFO | "malloc" : " 14 GB", | 20260506 21:22:19.761 | INFO | "pinned" : " 29 GB" | 20260506 21:22:19.761 | INFO | } | 20260506 21:22:19.761 | INFO | Perf report for the last 5.13735 seconds: | 20260506 21:22:19.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:22:19.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:19.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:19.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:19.761 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 21:22:19.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:19.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:19.761 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:22:19.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:19.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:19.761 | INFO | Uploaded 36GiB to GPU in 2.01199s (17.8927GiB/s) | 20260506 21:22:19.761 | INFO | Downloaded 34GiB to GPU in 1.92653s (17.6483GiB/s) | 20260506 21:22:24.898 | INFO | Analyzing chunk frames = [4547584,4548096) | 20260506 21:22:24.898 | INFO | MemInfo: { | 20260506 21:22:24.898 | INFO | "gpu" : " 17 GB", | 20260506 21:22:24.898 | INFO | "huge" : "", | 20260506 21:22:24.898 | INFO | "huge_shared" : " 27 GB", | 20260506 21:22:24.898 | INFO | "malloc" : " 14 GB", | 20260506 21:22:24.898 | INFO | "pinned" : " 29 GB" | 20260506 21:22:24.898 | INFO | } | 20260506 21:22:24.898 | INFO | Perf report for the last 5.13726 seconds: | 20260506 21:22:24.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:22:24.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:24.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:24.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:24.898 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 21:22:24.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:24.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:24.898 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:22:24.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:24.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:24.899 | INFO | Uploaded 36GiB to GPU in 1.99781s (18.0197GiB/s) | 20260506 21:22:24.899 | INFO | Downloaded 34GiB to GPU in 1.92838s (17.6314GiB/s) | 20260506 21:22:30.040 | INFO | Analyzing chunk frames = [4548096,4548608) | 20260506 21:22:30.040 | INFO | MemInfo: { | 20260506 21:22:30.040 | INFO | "gpu" : " 17 GB", | 20260506 21:22:30.040 | INFO | "huge" : "", | 20260506 21:22:30.040 | INFO | "huge_shared" : " 27 GB", | 20260506 21:22:30.040 | INFO | "malloc" : " 14 GB", | 20260506 21:22:30.040 | INFO | "pinned" : " 29 GB" | 20260506 21:22:30.040 | INFO | } | 20260506 21:22:30.040 | INFO | Perf report for the last 5.14185 seconds: | 20260506 21:22:30.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:22:30.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:30.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:30.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:30.040 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 21:22:30.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:30.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:30.040 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:22:30.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:30.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:30.040 | INFO | Uploaded 36GiB to GPU in 1.96642s (18.3074GiB/s) | 20260506 21:22:30.040 | INFO | Downloaded 34GiB to GPU in 1.87359s (18.1469GiB/s) | 20260506 21:22:35.172 | INFO | Analyzing chunk frames = [4548608,4549120) | 20260506 21:22:35.172 | INFO | MemInfo: { | 20260506 21:22:35.172 | INFO | "gpu" : " 17 GB", | 20260506 21:22:35.172 | INFO | "huge" : "", | 20260506 21:22:35.173 | INFO | "huge_shared" : " 27 GB", | 20260506 21:22:35.173 | INFO | "malloc" : " 14 GB", | 20260506 21:22:35.173 | INFO | "pinned" : " 29 GB" | 20260506 21:22:35.173 | INFO | } | 20260506 21:22:35.173 | INFO | Perf report for the last 5.1327 seconds: | 20260506 21:22:35.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:22:35.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:35.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:35.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:35.173 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 21:22:35.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:35.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:35.173 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:22:35.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:35.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:35.173 | INFO | Uploaded 36GiB to GPU in 1.99997s (18.0003GiB/s) | 20260506 21:22:35.173 | INFO | Downloaded 34GiB to GPU in 1.91974s (17.7107GiB/s) | 20260506 21:22:40.310 | INFO | Analyzing chunk frames = [4549120,4549632) | 20260506 21:22:40.310 | INFO | MemInfo: { | 20260506 21:22:40.310 | INFO | "gpu" : " 17 GB", | 20260506 21:22:40.310 | INFO | "huge" : "", | 20260506 21:22:40.310 | INFO | "huge_shared" : " 27 GB", | 20260506 21:22:40.310 | INFO | "malloc" : " 14 GB", | 20260506 21:22:40.310 | INFO | "pinned" : " 29 GB" | 20260506 21:22:40.310 | INFO | } | 20260506 21:22:40.310 | INFO | Perf report for the last 5.1377 seconds: | 20260506 21:22:40.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:22:40.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:40.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:40.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:40.310 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 21:22:40.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:40.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:40.310 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:22:40.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:40.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:40.311 | INFO | Uploaded 36GiB to GPU in 1.99302s (18.063GiB/s) | 20260506 21:22:40.311 | INFO | Downloaded 34GiB to GPU in 1.93301s (17.5892GiB/s) | 20260506 21:22:45.457 | INFO | Analyzing chunk frames = [4549632,4550144) | 20260506 21:22:45.457 | INFO | MemInfo: { | 20260506 21:22:45.457 | INFO | "gpu" : " 17 GB", | 20260506 21:22:45.457 | INFO | "huge" : "", | 20260506 21:22:45.457 | INFO | "huge_shared" : " 27 GB", | 20260506 21:22:45.457 | INFO | "malloc" : " 14 GB", | 20260506 21:22:45.457 | INFO | "pinned" : " 29 GB" | 20260506 21:22:45.457 | INFO | } | 20260506 21:22:45.457 | INFO | Perf report for the last 5.14687 seconds: | 20260506 21:22:45.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:22:45.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:45.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:45.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:45.457 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260506 21:22:45.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:45.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:45.457 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:22:45.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:45.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:45.458 | INFO | Uploaded 36GiB to GPU in 1.97692s (18.2102GiB/s) | 20260506 21:22:45.458 | INFO | Downloaded 34GiB to GPU in 2.00058s (16.9951GiB/s) | 20260506 21:22:50.593 | INFO | Analyzing chunk frames = [4550144,4550656) | 20260506 21:22:50.593 | INFO | MemInfo: { | 20260506 21:22:50.593 | INFO | "gpu" : " 17 GB", | 20260506 21:22:50.593 | INFO | "huge" : "", | 20260506 21:22:50.593 | INFO | "huge_shared" : " 27 GB", | 20260506 21:22:50.593 | INFO | "malloc" : " 14 GB", | 20260506 21:22:50.593 | INFO | "pinned" : " 29 GB" | 20260506 21:22:50.593 | INFO | } | 20260506 21:22:50.593 | INFO | Perf report for the last 5.13548 seconds: | 20260506 21:22:50.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:22:50.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:50.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:50.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:50.593 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 21:22:50.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:50.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:50.593 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 21:22:50.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:50.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:50.594 | INFO | Uploaded 36GiB to GPU in 2.03757s (17.6681GiB/s) | 20260506 21:22:50.594 | INFO | Downloaded 34GiB to GPU in 2.00114s (16.9903GiB/s) | 20260506 21:22:50.702 | INFO | Starting a round of Baz IO | 20260506 21:22:53.832 | INFO | Populating RTMetrics report | 20260506 21:22:53.832 | INFO | RTMetrics report populated | 20260506 21:22:53.832 | INFO | Opening tmp RTMetrics json file | 20260506 21:22:53.834 | INFO | Serializing RTMetrics to json | 20260506 21:22:53.854 | INFO | Serialization done, swapping tmp file now... | 20260506 21:22:53.859 | INFO | RTMetrics done | 20260506 21:22:54.341 | INFO | Finished a round of Baz IO: Wrote 1.947 GiB in 3.639 seconds | 20260506 21:22:54.342 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.96% / 0.00% / 66.04% | 20260506 21:22:54.342 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.621327% / 1.19209e-05% / 7.15256e-05% | 20260506 21:22:54.342 | INFO | Baz IO overall: | 20260506 21:22:54.342 | INFO | BytesWritten : 2398121513655 | 20260506 21:22:54.342 | INFO | NumEvents : 1157928622520 | 20260506 21:22:55.734 | INFO | Analyzing chunk frames = [4550656,4551168) | 20260506 21:22:55.734 | INFO | MemInfo: { | 20260506 21:22:55.734 | INFO | "gpu" : " 17 GB", | 20260506 21:22:55.734 | INFO | "huge" : "", | 20260506 21:22:55.734 | INFO | "huge_shared" : " 27 GB", | 20260506 21:22:55.734 | INFO | "malloc" : " 14 GB", | 20260506 21:22:55.734 | INFO | "pinned" : " 29 GB" | 20260506 21:22:55.734 | INFO | } | 20260506 21:22:55.734 | INFO | Perf report for the last 5.14126 seconds: | 20260506 21:22:55.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:22:55.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:55.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:55.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:55.734 | INFO | ANALYSIS : 62.5%, 2.9, 95.7ms, 0 | 20260506 21:22:55.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:22:55.734 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 21:22:55.734 | INFO | PRE_HQ : 59.5%, 1.8, 56.5ms, 0 | 20260506 21:22:55.734 | INFO | BAZWRITER : 70.5%, 4.8, 173.7ms, 0 | 20260506 21:22:55.734 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 21:22:55.734 | INFO | Uploaded 36GiB to GPU in 2.07417s (17.3563GiB/s) | 20260506 21:22:55.734 | INFO | Downloaded 38GiB to GPU in 2.20353s (17.245GiB/s) | 20260506 21:23:00.876 | INFO | Analyzing chunk frames = [4551168,4551680) | 20260506 21:23:00.876 | INFO | MemInfo: { | 20260506 21:23:00.876 | INFO | "gpu" : " 17 GB", | 20260506 21:23:00.876 | INFO | "huge" : "", | 20260506 21:23:00.876 | INFO | "huge_shared" : " 27 GB", | 20260506 21:23:00.876 | INFO | "malloc" : " 14 GB", | 20260506 21:23:00.876 | INFO | "pinned" : " 29 GB" | 20260506 21:23:00.876 | INFO | } | 20260506 21:23:00.876 | INFO | Perf report for the last 5.1417 seconds: | 20260506 21:23:00.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:23:00.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:00.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:00.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:00.876 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 21:23:00.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:00.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:00.876 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:23:00.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:00.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:00.876 | INFO | Uploaded 36GiB to GPU in 2.01583s (17.8586GiB/s) | 20260506 21:23:00.876 | INFO | Downloaded 34GiB to GPU in 1.9268s (17.6458GiB/s) | 20260506 21:23:06.013 | INFO | Analyzing chunk frames = [4551680,4552192) | 20260506 21:23:06.013 | INFO | MemInfo: { | 20260506 21:23:06.013 | INFO | "gpu" : " 17 GB", | 20260506 21:23:06.013 | INFO | "huge" : "", | 20260506 21:23:06.013 | INFO | "huge_shared" : " 27 GB", | 20260506 21:23:06.013 | INFO | "malloc" : " 14 GB", | 20260506 21:23:06.013 | INFO | "pinned" : " 29 GB" | 20260506 21:23:06.013 | INFO | } | 20260506 21:23:06.013 | INFO | Perf report for the last 5.13729 seconds: | 20260506 21:23:06.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:23:06.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:06.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:06.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:06.013 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260506 21:23:06.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:06.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:06.013 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:23:06.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:06.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:06.013 | INFO | Uploaded 36GiB to GPU in 1.99611s (18.0351GiB/s) | 20260506 21:23:06.013 | INFO | Downloaded 34GiB to GPU in 1.92717s (17.6424GiB/s) | 20260506 21:23:11.148 | INFO | Analyzing chunk frames = [4552192,4552704) | 20260506 21:23:11.148 | INFO | MemInfo: { | 20260506 21:23:11.148 | INFO | "gpu" : " 17 GB", | 20260506 21:23:11.148 | INFO | "huge" : "", | 20260506 21:23:11.148 | INFO | "huge_shared" : " 27 GB", | 20260506 21:23:11.148 | INFO | "malloc" : " 14 GB", | 20260506 21:23:11.148 | INFO | "pinned" : " 29 GB" | 20260506 21:23:11.148 | INFO | } | 20260506 21:23:11.148 | INFO | Perf report for the last 5.13483 seconds: | 20260506 21:23:11.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:23:11.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:11.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:11.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:11.148 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 21:23:11.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:11.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:11.148 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 21:23:11.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:11.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:11.148 | INFO | Uploaded 36GiB to GPU in 1.96538s (18.3171GiB/s) | 20260506 21:23:11.148 | INFO | Downloaded 34GiB to GPU in 1.87525s (18.131GiB/s) | 20260506 21:23:16.285 | INFO | Analyzing chunk frames = [4552704,4553216) | 20260506 21:23:16.285 | INFO | MemInfo: { | 20260506 21:23:16.285 | INFO | "gpu" : " 17 GB", | 20260506 21:23:16.285 | INFO | "huge" : "", | 20260506 21:23:16.285 | INFO | "huge_shared" : " 27 GB", | 20260506 21:23:16.285 | INFO | "malloc" : " 14 GB", | 20260506 21:23:16.285 | INFO | "pinned" : " 29 GB" | 20260506 21:23:16.285 | INFO | } | 20260506 21:23:16.285 | INFO | Perf report for the last 5.13698 seconds: | 20260506 21:23:16.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:23:16.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:16.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:16.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:16.285 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260506 21:23:16.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:16.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:16.285 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:23:16.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:16.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:16.285 | INFO | Uploaded 36GiB to GPU in 2.02151s (17.8085GiB/s) | 20260506 21:23:16.285 | INFO | Downloaded 34GiB to GPU in 1.91323s (17.771GiB/s) | 20260506 21:23:21.432 | INFO | Analyzing chunk frames = [4553216,4553728) | 20260506 21:23:21.432 | INFO | MemInfo: { | 20260506 21:23:21.432 | INFO | "gpu" : " 17 GB", | 20260506 21:23:21.432 | INFO | "huge" : "", | 20260506 21:23:21.432 | INFO | "huge_shared" : " 27 GB", | 20260506 21:23:21.432 | INFO | "malloc" : " 14 GB", | 20260506 21:23:21.432 | INFO | "pinned" : " 29 GB" | 20260506 21:23:21.432 | INFO | } | 20260506 21:23:21.432 | INFO | Perf report for the last 5.1469 seconds: | 20260506 21:23:21.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:23:21.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:21.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:21.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:21.432 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 21:23:21.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:21.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:21.432 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 21:23:21.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:21.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:21.432 | INFO | Uploaded 36GiB to GPU in 2.01291s (17.8845GiB/s) | 20260506 21:23:21.432 | INFO | Downloaded 34GiB to GPU in 1.94597s (17.472GiB/s) | 20260506 21:23:26.571 | INFO | Analyzing chunk frames = [4553728,4554240) | 20260506 21:23:26.571 | INFO | MemInfo: { | 20260506 21:23:26.571 | INFO | "gpu" : " 17 GB", | 20260506 21:23:26.571 | INFO | "huge" : "", | 20260506 21:23:26.571 | INFO | "huge_shared" : " 27 GB", | 20260506 21:23:26.571 | INFO | "malloc" : " 14 GB", | 20260506 21:23:26.571 | INFO | "pinned" : " 29 GB" | 20260506 21:23:26.571 | INFO | } | 20260506 21:23:26.571 | INFO | Perf report for the last 5.13895 seconds: | 20260506 21:23:26.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:23:26.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:26.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:26.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:26.571 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:23:26.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:26.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:26.571 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 21:23:26.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:26.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:26.571 | INFO | Uploaded 36GiB to GPU in 1.98123s (18.1705GiB/s) | 20260506 21:23:26.571 | INFO | Downloaded 34GiB to GPU in 2.01032s (16.9128GiB/s) | 20260506 21:23:31.717 | INFO | Analyzing chunk frames = [4554240,4554752) | 20260506 21:23:31.717 | INFO | MemInfo: { | 20260506 21:23:31.717 | INFO | "gpu" : " 17 GB", | 20260506 21:23:31.717 | INFO | "huge" : "", | 20260506 21:23:31.717 | INFO | "huge_shared" : " 27 GB", | 20260506 21:23:31.717 | INFO | "malloc" : " 14 GB", | 20260506 21:23:31.717 | INFO | "pinned" : " 29 GB" | 20260506 21:23:31.717 | INFO | } | 20260506 21:23:31.717 | INFO | Perf report for the last 5.14675 seconds: | 20260506 21:23:31.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:23:31.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:31.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:31.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:31.718 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 21:23:31.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:31.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:31.718 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:23:31.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:31.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:31.718 | INFO | Uploaded 36GiB to GPU in 1.97678s (18.2114GiB/s) | 20260506 21:23:31.718 | INFO | Downloaded 34GiB to GPU in 1.98449s (17.1328GiB/s) | 20260506 21:23:31.718 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:23:31.718 | INFO | Total mem usage: 18 GiB | 20260506 21:23:31.718 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:23:31.718 | INFO | Recent High Water Marks Sum: 16.6 GiB (Savings: -8.76%) | 20260506 21:23:31.718 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:23:31.718 | INFO | High Water Breakdown: | 20260506 21:23:31.718 | INFO | 62.4% : 10.3 GiB ( 90.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:23:31.718 | INFO | 37.5% : 6.21 GiB ( 92.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:23:31.718 | INFO | 0.12% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:23:31.718 | INFO | ------------------------------------------ | 20260506 21:23:31.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:23:31.718 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:23:31.718 | INFO | Total mem usage: 26.5 GiB | 20260506 21:23:31.718 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:23:31.718 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:23:31.718 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:23:31.718 | INFO | High Water Breakdown: | 20260506 21:23:31.718 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:23:31.718 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:23:31.718 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:23:31.718 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:23:31.718 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:23:31.718 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:23:31.718 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:23:31.718 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:23:31.718 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:23:31.718 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:23:31.718 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:23:31.718 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:23:31.718 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:23:31.718 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:23:31.718 | INFO | ------------------------------------------ | 20260506 21:23:31.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:23:31.718 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:23:31.718 | INFO | Total mem usage: 25.6 GiB | 20260506 21:23:31.718 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:23:31.718 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:23:31.718 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:23:31.718 | INFO | High Water Breakdown: | 20260506 21:23:31.718 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:23:31.718 | INFO | ------------------------------------------ | 20260506 21:23:31.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:23:31.718 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:23:31.718 | INFO | Total mem usage: 15.7 GiB | 20260506 21:23:31.718 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:23:31.718 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:23:31.718 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:23:31.718 | INFO | High Water Breakdown: | 20260506 21:23:31.718 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:23:31.718 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:23:31.718 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:23:31.718 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:23:31.718 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:23:31.718 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:23:31.718 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:23:31.718 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:23:31.718 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:23:31.718 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:23:31.718 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:23:31.718 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:23:31.718 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:23:31.718 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:23:31.718 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:23:31.718 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:23:31.718 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:23:31.718 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:23:31.718 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:23:31.718 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:23:31.718 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:23:31.718 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:23:31.718 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:23:31.719 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:23:31.719 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:23:31.719 | INFO | ------------------------------------------ | 20260506 21:23:31.719 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:23:31.719 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:23:31.719 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:23:31.719 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:23:31.719 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.60e+01, 1.35e+01 / 2.28e+00 | 20260506 21:23:31.719 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.74e-03 / 5.25e-03 | 20260506 21:23:31.719 | INFO | PerfReport: 11.63%, Baseline -- 3.62e+00 / 4.41e+00, 5.47e-01 / 3.23e-01 | 20260506 21:23:31.719 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.46e+00, 1.78e-01 / 1.22e-01 | 20260506 21:23:31.719 | INFO | PerfReport: 16.05%, DME -- 3.99e+01 / 4.47e+01, 4.14e+00 / 2.68e+00 | 20260506 21:23:31.719 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.41e+00 / 1.76e+00 | 20260506 21:23:31.719 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.21e+00 / 2.77e+00, 3.74e-01 / 1.89e-01 | 20260506 21:23:31.719 | INFO | PerfReport: 3.27%, Metrics -- 1.02e+00 / 1.36e+00, 2.65e-01 / 2.06e-01 | 20260506 21:23:31.719 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.48e-01 / 3.96e-01, 5.45e-02 / 2.34e-02 | 20260506 21:23:31.719 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:23:31.719 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:23:31.719 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:23:31.719 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 21:23:31.719 | INFO | PerfReport: 21.62%, Upload -- 2.09e+01 / 0.00e+00, 7.04e+00 / 0.00e+00 | 20260506 21:23:31.719 | INFO | PerfReport: 21.26%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 21:23:31.719 | INFO | PerfReport: 32.17%, Compute -- 3.11e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 21:23:31.719 | INFO | PerfReport: 1.34%, LockUpload -- 1.30e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260506 21:23:31.719 | INFO | PerfReport: 1.16%, LockDownload -- 1.12e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260506 21:23:31.719 | INFO | PerfReport: 22.43%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:23:31.719 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260506 21:23:31.816 | INFO | Starting a round of Baz IO | 20260506 21:23:34.951 | INFO | Populating RTMetrics report | 20260506 21:23:34.951 | INFO | RTMetrics report populated | 20260506 21:23:34.951 | INFO | Opening tmp RTMetrics json file | 20260506 21:23:34.954 | INFO | Serializing RTMetrics to json | 20260506 21:23:34.983 | INFO | Serialization done, swapping tmp file now... | 20260506 21:23:34.985 | INFO | RTMetrics done | 20260506 21:23:35.087 | INFO | Finished a round of Baz IO: Wrote 1.95 GiB in 3.27 seconds | 20260506 21:23:35.087 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.95% / 0.00% / 66.04% | 20260506 21:23:35.087 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.622038% / 1.58946e-05% / 7.54992e-05% | 20260506 21:23:35.087 | INFO | Baz IO overall: | 20260506 21:23:35.087 | INFO | BytesWritten : 2400215732104 | 20260506 21:23:35.087 | INFO | NumEvents : 1158931846901 | 20260506 21:23:36.850 | INFO | Analyzing chunk frames = [4554752,4555264) | 20260506 21:23:36.850 | INFO | MemInfo: { | 20260506 21:23:36.850 | INFO | "gpu" : " 17 GB", | 20260506 21:23:36.850 | INFO | "huge" : "", | 20260506 21:23:36.850 | INFO | "huge_shared" : " 27 GB", | 20260506 21:23:36.850 | INFO | "malloc" : " 14 GB", | 20260506 21:23:36.850 | INFO | "pinned" : " 29 GB" | 20260506 21:23:36.850 | INFO | } | 20260506 21:23:36.850 | INFO | Perf report for the last 5.13279 seconds: | 20260506 21:23:36.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:23:36.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:36.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:36.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:36.850 | INFO | ANALYSIS : 62.4%, 2.9, 95.4ms, 0 | 20260506 21:23:36.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:36.850 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 21:23:36.850 | INFO | PRE_HQ : 59.9%, 1.8, 56.6ms, 0 | 20260506 21:23:36.850 | INFO | BAZWRITER : 63.3%, 4.4, 144.6ms, 0 | 20260506 21:23:36.850 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 21:23:36.851 | INFO | Uploaded 36GiB to GPU in 2.0291s (17.7419GiB/s) | 20260506 21:23:36.851 | INFO | Downloaded 38GiB to GPU in 2.19414s (17.3189GiB/s) | 20260506 21:23:41.987 | INFO | Analyzing chunk frames = [4555264,4555776) | 20260506 21:23:41.987 | INFO | MemInfo: { | 20260506 21:23:41.987 | INFO | "gpu" : " 17 GB", | 20260506 21:23:41.987 | INFO | "huge" : "", | 20260506 21:23:41.987 | INFO | "huge_shared" : " 27 GB", | 20260506 21:23:41.987 | INFO | "malloc" : " 14 GB", | 20260506 21:23:41.987 | INFO | "pinned" : " 29 GB" | 20260506 21:23:41.987 | INFO | } | 20260506 21:23:41.987 | INFO | Perf report for the last 5.13649 seconds: | 20260506 21:23:41.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:23:41.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:41.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:41.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:41.987 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 21:23:41.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:41.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:41.987 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 21:23:41.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:41.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:41.987 | INFO | Uploaded 36GiB to GPU in 2.00069s (17.9938GiB/s) | 20260506 21:23:41.987 | INFO | Downloaded 34GiB to GPU in 1.92557s (17.6571GiB/s) | 20260506 21:23:47.126 | INFO | Analyzing chunk frames = [4555776,4556288) | 20260506 21:23:47.126 | INFO | MemInfo: { | 20260506 21:23:47.126 | INFO | "gpu" : " 17 GB", | 20260506 21:23:47.126 | INFO | "huge" : "", | 20260506 21:23:47.126 | INFO | "huge_shared" : " 27 GB", | 20260506 21:23:47.126 | INFO | "malloc" : " 14 GB", | 20260506 21:23:47.126 | INFO | "pinned" : " 29 GB" | 20260506 21:23:47.126 | INFO | } | 20260506 21:23:47.126 | INFO | Perf report for the last 5.13945 seconds: | 20260506 21:23:47.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:23:47.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:47.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:47.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:47.126 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 21:23:47.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:47.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:47.126 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 21:23:47.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:47.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:47.126 | INFO | Uploaded 36GiB to GPU in 1.96988s (18.2752GiB/s) | 20260506 21:23:47.127 | INFO | Downloaded 34GiB to GPU in 1.92096s (17.6995GiB/s) | 20260506 21:23:52.261 | INFO | Analyzing chunk frames = [4556288,4556800) | 20260506 21:23:52.261 | INFO | MemInfo: { | 20260506 21:23:52.261 | INFO | "gpu" : " 17 GB", | 20260506 21:23:52.261 | INFO | "huge" : "", | 20260506 21:23:52.261 | INFO | "huge_shared" : " 27 GB", | 20260506 21:23:52.261 | INFO | "malloc" : " 14 GB", | 20260506 21:23:52.261 | INFO | "pinned" : " 29 GB" | 20260506 21:23:52.261 | INFO | } | 20260506 21:23:52.261 | INFO | Perf report for the last 5.13457 seconds: | 20260506 21:23:52.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:23:52.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:52.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:52.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:52.261 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:23:52.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:52.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:52.261 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 21:23:52.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:52.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:52.261 | INFO | Uploaded 36GiB to GPU in 1.92751s (18.6769GiB/s) | 20260506 21:23:52.261 | INFO | Downloaded 34GiB to GPU in 1.87358s (18.1471GiB/s) | 20260506 21:23:57.404 | INFO | Analyzing chunk frames = [4556800,4557312) | 20260506 21:23:57.404 | INFO | MemInfo: { | 20260506 21:23:57.404 | INFO | "gpu" : " 17 GB", | 20260506 21:23:57.404 | INFO | "huge" : "", | 20260506 21:23:57.404 | INFO | "huge_shared" : " 27 GB", | 20260506 21:23:57.404 | INFO | "malloc" : " 14 GB", | 20260506 21:23:57.404 | INFO | "pinned" : " 29 GB" | 20260506 21:23:57.404 | INFO | } | 20260506 21:23:57.404 | INFO | Perf report for the last 5.14306 seconds: | 20260506 21:23:57.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:23:57.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:57.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:57.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:57.404 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260506 21:23:57.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:57.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:57.404 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:23:57.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:57.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:23:57.404 | INFO | Uploaded 36GiB to GPU in 2.00716s (17.9358GiB/s) | 20260506 21:23:57.404 | INFO | Downloaded 34GiB to GPU in 1.91091s (17.7926GiB/s) | 20260506 21:24:02.547 | INFO | Analyzing chunk frames = [4557312,4557824) | 20260506 21:24:02.547 | INFO | MemInfo: { | 20260506 21:24:02.547 | INFO | "gpu" : " 17 GB", | 20260506 21:24:02.547 | INFO | "huge" : "", | 20260506 21:24:02.547 | INFO | "huge_shared" : " 27 GB", | 20260506 21:24:02.547 | INFO | "malloc" : " 14 GB", | 20260506 21:24:02.547 | INFO | "pinned" : " 29 GB" | 20260506 21:24:02.547 | INFO | } | 20260506 21:24:02.547 | INFO | Perf report for the last 5.14343 seconds: | 20260506 21:24:02.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:24:02.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:02.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:02.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:02.547 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:24:02.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:02.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:02.547 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 21:24:02.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:02.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:02.548 | INFO | Uploaded 36GiB to GPU in 1.99237s (18.0689GiB/s) | 20260506 21:24:02.548 | INFO | Downloaded 34GiB to GPU in 1.95671s (17.3761GiB/s) | 20260506 21:24:07.684 | INFO | Analyzing chunk frames = [4557824,4558336) | 20260506 21:24:07.684 | INFO | MemInfo: { | 20260506 21:24:07.684 | INFO | "gpu" : " 17 GB", | 20260506 21:24:07.684 | INFO | "huge" : "", | 20260506 21:24:07.684 | INFO | "huge_shared" : " 27 GB", | 20260506 21:24:07.684 | INFO | "malloc" : " 14 GB", | 20260506 21:24:07.684 | INFO | "pinned" : " 29 GB" | 20260506 21:24:07.684 | INFO | } | 20260506 21:24:07.684 | INFO | Perf report for the last 5.13638 seconds: | 20260506 21:24:07.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:24:07.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:07.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:07.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:07.684 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 21:24:07.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:07.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:07.684 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:24:07.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:07.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:07.684 | INFO | Uploaded 36GiB to GPU in 1.95577s (18.4071GiB/s) | 20260506 21:24:07.684 | INFO | Downloaded 34GiB to GPU in 1.985s (17.1284GiB/s) | 20260506 21:24:12.825 | INFO | Analyzing chunk frames = [4558336,4558848) | 20260506 21:24:12.825 | INFO | MemInfo: { | 20260506 21:24:12.825 | INFO | "gpu" : " 17 GB", | 20260506 21:24:12.825 | INFO | "huge" : "", | 20260506 21:24:12.825 | INFO | "huge_shared" : " 27 GB", | 20260506 21:24:12.825 | INFO | "malloc" : " 14 GB", | 20260506 21:24:12.825 | INFO | "pinned" : " 29 GB" | 20260506 21:24:12.825 | INFO | } | 20260506 21:24:12.825 | INFO | Perf report for the last 5.1415 seconds: | 20260506 21:24:12.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:24:12.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:12.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:12.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:12.825 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:24:12.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:12.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:12.825 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:24:12.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:12.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:12.825 | INFO | Uploaded 36GiB to GPU in 1.996s (18.0361GiB/s) | 20260506 21:24:12.826 | INFO | Downloaded 34GiB to GPU in 1.99368s (17.0539GiB/s) | 20260506 21:24:12.923 | INFO | Starting a round of Baz IO | 20260506 21:24:16.063 | INFO | Populating RTMetrics report | 20260506 21:24:16.063 | INFO | RTMetrics report populated | 20260506 21:24:16.063 | INFO | Opening tmp RTMetrics json file | 20260506 21:24:16.084 | INFO | Serializing RTMetrics to json | 20260506 21:24:16.088 | INFO | Serialization done, swapping tmp file now... | 20260506 21:24:16.091 | INFO | RTMetrics done | 20260506 21:24:16.390 | INFO | Finished a round of Baz IO: Wrote 1.948 GiB in 3.466 seconds | 20260506 21:24:16.390 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.95% / 0.00% / 66.05% | 20260506 21:24:16.390 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.619845% / 0% / 6.35783e-05% | 20260506 21:24:16.390 | INFO | Baz IO overall: | 20260506 21:24:16.390 | INFO | BytesWritten : 2402307716969 | 20260506 21:24:16.390 | INFO | NumEvents : 1159933562149 | 20260506 21:24:17.966 | INFO | Analyzing chunk frames = [4558848,4559360) | 20260506 21:24:17.966 | INFO | MemInfo: { | 20260506 21:24:17.966 | INFO | "gpu" : " 17 GB", | 20260506 21:24:17.966 | INFO | "huge" : "", | 20260506 21:24:17.966 | INFO | "huge_shared" : " 27 GB", | 20260506 21:24:17.966 | INFO | "malloc" : " 14 GB", | 20260506 21:24:17.967 | INFO | "pinned" : " 29 GB" | 20260506 21:24:17.967 | INFO | } | 20260506 21:24:17.967 | INFO | Perf report for the last 5.14129 seconds: | 20260506 21:24:17.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:24:17.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:17.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:17.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:17.967 | INFO | ANALYSIS : 62.4%, 2.9, 95.3ms, 0 | 20260506 21:24:17.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:17.967 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260506 21:24:17.967 | INFO | PRE_HQ : 59.8%, 1.8, 57.0ms, 0 | 20260506 21:24:17.967 | INFO | BAZWRITER : 67.1%, 4.8, 164.3ms, 0 | 20260506 21:24:17.967 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 21:24:17.967 | INFO | Uploaded 36GiB to GPU in 2.04752s (17.5822GiB/s) | 20260506 21:24:17.967 | INFO | Downloaded 38GiB to GPU in 2.20095s (17.2653GiB/s) | 20260506 21:24:23.101 | INFO | Analyzing chunk frames = [4559360,4559872) | 20260506 21:24:23.101 | INFO | MemInfo: { | 20260506 21:24:23.101 | INFO | "gpu" : " 17 GB", | 20260506 21:24:23.101 | INFO | "huge" : "", | 20260506 21:24:23.101 | INFO | "huge_shared" : " 27 GB", | 20260506 21:24:23.101 | INFO | "malloc" : " 14 GB", | 20260506 21:24:23.101 | INFO | "pinned" : " 29 GB" | 20260506 21:24:23.101 | INFO | } | 20260506 21:24:23.101 | INFO | Perf report for the last 5.1343 seconds: | 20260506 21:24:23.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:24:23.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:23.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:23.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:23.101 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:24:23.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:23.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:23.101 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:24:23.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:23.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:23.101 | INFO | Uploaded 36GiB to GPU in 2.00787s (17.9295GiB/s) | 20260506 21:24:23.101 | INFO | Downloaded 34GiB to GPU in 1.92611s (17.6522GiB/s) | 20260506 21:24:28.237 | INFO | Analyzing chunk frames = [4559872,4560384) | 20260506 21:24:28.237 | INFO | MemInfo: { | 20260506 21:24:28.237 | INFO | "gpu" : " 17 GB", | 20260506 21:24:28.237 | INFO | "huge" : "", | 20260506 21:24:28.237 | INFO | "huge_shared" : " 27 GB", | 20260506 21:24:28.237 | INFO | "malloc" : " 14 GB", | 20260506 21:24:28.237 | INFO | "pinned" : " 29 GB" | 20260506 21:24:28.237 | INFO | } | 20260506 21:24:28.237 | INFO | Perf report for the last 5.13581 seconds: | 20260506 21:24:28.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:24:28.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:28.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:28.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:28.237 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 21:24:28.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:28.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:28.237 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:24:28.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:28.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:28.237 | INFO | Uploaded 36GiB to GPU in 1.99206s (18.0718GiB/s) | 20260506 21:24:28.237 | INFO | Downloaded 34GiB to GPU in 1.92312s (17.6796GiB/s) | 20260506 21:24:33.377 | INFO | Analyzing chunk frames = [4560384,4560896) | 20260506 21:24:33.377 | INFO | MemInfo: { | 20260506 21:24:33.377 | INFO | "gpu" : " 17 GB", | 20260506 21:24:33.377 | INFO | "huge" : "", | 20260506 21:24:33.377 | INFO | "huge_shared" : " 27 GB", | 20260506 21:24:33.377 | INFO | "malloc" : " 14 GB", | 20260506 21:24:33.377 | INFO | "pinned" : " 29 GB" | 20260506 21:24:33.377 | INFO | } | 20260506 21:24:33.377 | INFO | Perf report for the last 5.14018 seconds: | 20260506 21:24:33.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:24:33.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:33.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:33.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:33.377 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 21:24:33.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:33.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:33.377 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:24:33.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:33.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:33.377 | INFO | Uploaded 36GiB to GPU in 1.9544s (18.42GiB/s) | 20260506 21:24:33.377 | INFO | Downloaded 34GiB to GPU in 1.87564s (18.1271GiB/s) | 20260506 21:24:38.521 | INFO | Analyzing chunk frames = [4560896,4561408) | 20260506 21:24:38.521 | INFO | MemInfo: { | 20260506 21:24:38.521 | INFO | "gpu" : " 17 GB", | 20260506 21:24:38.521 | INFO | "huge" : "", | 20260506 21:24:38.521 | INFO | "huge_shared" : " 27 GB", | 20260506 21:24:38.521 | INFO | "malloc" : " 14 GB", | 20260506 21:24:38.521 | INFO | "pinned" : " 29 GB" | 20260506 21:24:38.521 | INFO | } | 20260506 21:24:38.521 | INFO | Perf report for the last 5.1444 seconds: | 20260506 21:24:38.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:24:38.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:38.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:38.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:38.521 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 21:24:38.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:38.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:38.521 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 21:24:38.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:38.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:38.522 | INFO | Uploaded 36GiB to GPU in 2.02087s (17.8142GiB/s) | 20260506 21:24:38.522 | INFO | Downloaded 34GiB to GPU in 1.91725s (17.7337GiB/s) | 20260506 21:24:43.661 | INFO | Analyzing chunk frames = [4561408,4561920) | 20260506 21:24:43.661 | INFO | MemInfo: { | 20260506 21:24:43.661 | INFO | "gpu" : " 17 GB", | 20260506 21:24:43.661 | INFO | "huge" : "", | 20260506 21:24:43.661 | INFO | "huge_shared" : " 27 GB", | 20260506 21:24:43.661 | INFO | "malloc" : " 14 GB", | 20260506 21:24:43.661 | INFO | "pinned" : " 29 GB" | 20260506 21:24:43.661 | INFO | } | 20260506 21:24:43.661 | INFO | Perf report for the last 5.13937 seconds: | 20260506 21:24:43.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:24:43.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:43.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:43.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:43.661 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 21:24:43.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:43.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:43.661 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 21:24:43.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:43.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:43.661 | INFO | Uploaded 36GiB to GPU in 2.01128s (17.8991GiB/s) | 20260506 21:24:43.661 | INFO | Downloaded 34GiB to GPU in 1.95021s (17.434GiB/s) | 20260506 21:24:48.805 | INFO | Analyzing chunk frames = [4561920,4562432) | 20260506 21:24:48.805 | INFO | MemInfo: { | 20260506 21:24:48.805 | INFO | "gpu" : " 17 GB", | 20260506 21:24:48.805 | INFO | "huge" : "", | 20260506 21:24:48.805 | INFO | "huge_shared" : " 27 GB", | 20260506 21:24:48.805 | INFO | "malloc" : " 14 GB", | 20260506 21:24:48.805 | INFO | "pinned" : " 29 GB" | 20260506 21:24:48.805 | INFO | } | 20260506 21:24:48.805 | INFO | Perf report for the last 5.14429 seconds: | 20260506 21:24:48.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:24:48.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:48.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:48.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:48.805 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 21:24:48.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:48.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:48.805 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:24:48.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:48.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:48.805 | INFO | Uploaded 36GiB to GPU in 1.95685s (18.3969GiB/s) | 20260506 21:24:48.805 | INFO | Downloaded 34GiB to GPU in 2.00253s (16.9785GiB/s) | 20260506 21:24:53.940 | INFO | Analyzing chunk frames = [4562432,4562944) | 20260506 21:24:53.940 | INFO | MemInfo: { | 20260506 21:24:53.940 | INFO | "gpu" : " 17 GB", | 20260506 21:24:53.940 | INFO | "huge" : "", | 20260506 21:24:53.940 | INFO | "huge_shared" : " 27 GB", | 20260506 21:24:53.940 | INFO | "malloc" : " 14 GB", | 20260506 21:24:53.940 | INFO | "pinned" : " 29 GB" | 20260506 21:24:53.940 | INFO | } | 20260506 21:24:53.940 | INFO | Perf report for the last 5.13501 seconds: | 20260506 21:24:53.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:24:53.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:53.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:53.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:53.940 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:24:53.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:53.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:53.940 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:24:53.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:53.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:53.940 | INFO | Uploaded 36GiB to GPU in 2.0318s (17.7183GiB/s) | 20260506 21:24:53.940 | INFO | Downloaded 34GiB to GPU in 2.00196s (16.9834GiB/s) | 20260506 21:24:53.940 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:24:53.940 | INFO | Total mem usage: 18 GiB | 20260506 21:24:53.940 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:24:53.940 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.3%) | 20260506 21:24:53.940 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:24:53.940 | INFO | High Water Breakdown: | 20260506 21:24:53.940 | INFO | 61.8% : 9.82 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:24:53.940 | INFO | 38.1% : 6.06 GiB ( 90.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:24:53.940 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:24:53.940 | INFO | ------------------------------------------ | 20260506 21:24:53.940 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:24:53.941 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:24:53.941 | INFO | Total mem usage: 26.5 GiB | 20260506 21:24:53.941 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:24:53.941 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:24:53.941 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:24:53.941 | INFO | High Water Breakdown: | 20260506 21:24:53.941 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:24:53.941 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:24:53.941 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:24:53.941 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:24:53.941 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:24:53.941 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:24:53.941 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:24:53.941 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:24:53.941 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:24:53.941 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:24:53.941 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:24:53.941 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:24:53.941 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:24:53.941 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:24:53.941 | INFO | ------------------------------------------ | 20260506 21:24:53.941 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:24:53.941 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:24:53.941 | INFO | Total mem usage: 25.6 GiB | 20260506 21:24:53.941 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:24:53.941 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 21:24:53.941 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 21:24:53.941 | INFO | High Water Breakdown: | 20260506 21:24:53.941 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 21:24:53.941 | INFO | ------------------------------------------ | 20260506 21:24:53.941 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:24:53.941 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:24:53.941 | INFO | Total mem usage: 15.7 GiB | 20260506 21:24:53.941 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:24:53.941 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:24:53.941 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:24:53.941 | INFO | High Water Breakdown: | 20260506 21:24:53.941 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:24:53.941 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:24:53.941 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:24:53.941 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:24:53.941 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:24:53.941 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:24:53.941 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:24:53.941 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:24:53.941 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:24:53.941 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:24:53.941 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:24:53.941 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:24:53.941 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:24:53.941 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:24:53.941 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:24:53.941 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:24:53.941 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:24:53.941 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:24:53.941 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:24:53.941 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:24:53.941 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:24:53.941 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:24:53.941 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:24:53.941 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:24:53.941 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:24:53.941 | INFO | ------------------------------------------ | 20260506 21:24:53.941 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:24:53.941 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:24:53.941 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:24:53.941 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:24:53.941 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.64e+01, 1.37e+01 / 2.59e+00 | 20260506 21:24:53.941 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.00e-01, 1.04e-02 / 5.47e-03 | 20260506 21:24:53.941 | INFO | PerfReport: 11.63%, Baseline -- 3.62e+00 / 4.38e+00, 5.48e-01 / 3.49e-01 | 20260506 21:24:53.941 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.44e+00, 1.82e-01 / 1.37e-01 | 20260506 21:24:53.941 | INFO | PerfReport: 16.05%, DME -- 4.00e+01 / 4.52e+01, 4.27e+00 / 2.24e+00 | 20260506 21:24:53.941 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.41e+00 / 1.64e+00 | 20260506 21:24:53.941 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.21e+00 / 2.77e+00, 3.72e-01 / 1.98e-01 | 20260506 21:24:53.941 | INFO | PerfReport: 3.27%, Metrics -- 1.02e+00 / 1.38e+00, 2.68e-01 / 1.54e-01 | 20260506 21:24:53.941 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.48e-01 / 3.97e-01, 5.39e-02 / 1.73e-02 | 20260506 21:24:53.941 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:24:53.941 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:24:53.941 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:24:53.941 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 21:24:53.941 | INFO | PerfReport: 21.49%, Upload -- 2.08e+01 / 0.00e+00, 6.93e+00 / 0.00e+00 | 20260506 21:24:53.941 | INFO | PerfReport: 21.26%, Download -- 2.05e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 21:24:53.941 | INFO | PerfReport: 32.21%, Compute -- 3.11e+01 / 0.00e+00, 1.37e+01 / 0.00e+00 | 20260506 21:24:53.941 | INFO | PerfReport: 1.22%, LockUpload -- 1.18e+00 / 0.00e+00, 4.58e+00 / 0.00e+00 | 20260506 21:24:53.941 | INFO | PerfReport: 1.24%, LockDownload -- 1.20e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260506 21:24:53.941 | INFO | PerfReport: 22.56%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:24:53.941 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260506 21:24:54.054 | INFO | Starting a round of Baz IO | 20260506 21:24:57.176 | INFO | Populating RTMetrics report | 20260506 21:24:57.177 | INFO | RTMetrics report populated | 20260506 21:24:57.177 | INFO | Opening tmp RTMetrics json file | 20260506 21:24:57.290 | INFO | Serializing RTMetrics to json | 20260506 21:24:57.301 | INFO | Serialization done, swapping tmp file now... | 20260506 21:24:57.302 | INFO | RTMetrics done | 20260506 21:24:57.560 | INFO | Finished a round of Baz IO: Wrote 1.947 GiB in 3.505 seconds | 20260506 21:24:57.560 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.95% / 0.00% / 66.05% | 20260506 21:24:57.560 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.618136% / 7.94729e-06% / 7.94729e-05% | 20260506 21:24:57.560 | INFO | Baz IO overall: | 20260506 21:24:57.560 | INFO | BytesWritten : 2404397818248 | 20260506 21:24:57.560 | INFO | NumEvents : 1160933808921 | 20260506 21:24:59.077 | INFO | Analyzing chunk frames = [4562944,4563456) | 20260506 21:24:59.077 | INFO | MemInfo: { | 20260506 21:24:59.077 | INFO | "gpu" : " 17 GB", | 20260506 21:24:59.077 | INFO | "huge" : "", | 20260506 21:24:59.077 | INFO | "huge_shared" : " 27 GB", | 20260506 21:24:59.077 | INFO | "malloc" : " 14 GB", | 20260506 21:24:59.077 | INFO | "pinned" : " 29 GB" | 20260506 21:24:59.077 | INFO | } | 20260506 21:24:59.077 | INFO | Perf report for the last 5.13744 seconds: | 20260506 21:24:59.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:24:59.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:59.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:59.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:59.078 | INFO | ANALYSIS : 62.4%, 2.9, 95.4ms, 0 | 20260506 21:24:59.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:24:59.078 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 21:24:59.078 | INFO | PRE_HQ : 59.6%, 1.8, 56.7ms, 0 | 20260506 21:24:59.078 | INFO | BAZWRITER : 67.9%, 4.8, 167.4ms, 0 | 20260506 21:24:59.078 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 21:24:59.078 | INFO | Uploaded 36GiB to GPU in 2.03421s (17.6973GiB/s) | 20260506 21:24:59.078 | INFO | Downloaded 38GiB to GPU in 2.20936s (17.1995GiB/s) | 20260506 21:25:04.217 | INFO | Analyzing chunk frames = [4563456,4563968) | 20260506 21:25:04.217 | INFO | MemInfo: { | 20260506 21:25:04.217 | INFO | "gpu" : " 17 GB", | 20260506 21:25:04.217 | INFO | "huge" : "", | 20260506 21:25:04.217 | INFO | "huge_shared" : " 27 GB", | 20260506 21:25:04.217 | INFO | "malloc" : " 14 GB", | 20260506 21:25:04.217 | INFO | "pinned" : " 29 GB" | 20260506 21:25:04.217 | INFO | } | 20260506 21:25:04.217 | INFO | Perf report for the last 5.13985 seconds: | 20260506 21:25:04.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:25:04.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:04.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:04.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:04.217 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 21:25:04.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:04.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:04.217 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 21:25:04.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:04.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:04.217 | INFO | Uploaded 36GiB to GPU in 1.99761s (18.0216GiB/s) | 20260506 21:25:04.218 | INFO | Downloaded 34GiB to GPU in 1.92685s (17.6454GiB/s) | 20260506 21:25:09.352 | INFO | Analyzing chunk frames = [4563968,4564480) | 20260506 21:25:09.352 | INFO | MemInfo: { | 20260506 21:25:09.352 | INFO | "gpu" : " 17 GB", | 20260506 21:25:09.352 | INFO | "huge" : "", | 20260506 21:25:09.352 | INFO | "huge_shared" : " 27 GB", | 20260506 21:25:09.352 | INFO | "malloc" : " 14 GB", | 20260506 21:25:09.352 | INFO | "pinned" : " 29 GB" | 20260506 21:25:09.352 | INFO | } | 20260506 21:25:09.352 | INFO | Perf report for the last 5.13505 seconds: | 20260506 21:25:09.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:25:09.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:09.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:09.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:09.352 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 21:25:09.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:09.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:09.352 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:25:09.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:09.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:09.353 | INFO | Uploaded 36GiB to GPU in 1.94302s (18.5279GiB/s) | 20260506 21:25:09.353 | INFO | Downloaded 34GiB to GPU in 1.92598s (17.6534GiB/s) | 20260506 21:25:14.493 | INFO | Analyzing chunk frames = [4564480,4564992) | 20260506 21:25:14.493 | INFO | MemInfo: { | 20260506 21:25:14.493 | INFO | "gpu" : " 17 GB", | 20260506 21:25:14.493 | INFO | "huge" : "", | 20260506 21:25:14.493 | INFO | "huge_shared" : " 27 GB", | 20260506 21:25:14.493 | INFO | "malloc" : " 14 GB", | 20260506 21:25:14.493 | INFO | "pinned" : " 29 GB" | 20260506 21:25:14.493 | INFO | } | 20260506 21:25:14.493 | INFO | Perf report for the last 5.14075 seconds: | 20260506 21:25:14.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:25:14.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:14.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:14.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:14.493 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:25:14.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:14.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:14.493 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:25:14.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:14.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:14.493 | INFO | Uploaded 36GiB to GPU in 1.92985s (18.6543GiB/s) | 20260506 21:25:14.493 | INFO | Downloaded 34GiB to GPU in 1.87425s (18.1406GiB/s) | 20260506 21:25:19.631 | INFO | Analyzing chunk frames = [4564992,4565504) | 20260506 21:25:19.631 | INFO | MemInfo: { | 20260506 21:25:19.631 | INFO | "gpu" : " 17 GB", | 20260506 21:25:19.631 | INFO | "huge" : "", | 20260506 21:25:19.631 | INFO | "huge_shared" : " 27 GB", | 20260506 21:25:19.631 | INFO | "malloc" : " 14 GB", | 20260506 21:25:19.631 | INFO | "pinned" : " 29 GB" | 20260506 21:25:19.631 | INFO | } | 20260506 21:25:19.631 | INFO | Perf report for the last 5.13765 seconds: | 20260506 21:25:19.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:25:19.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:19.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:19.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:19.631 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 21:25:19.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:19.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:19.631 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:25:19.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:19.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:19.631 | INFO | Uploaded 36GiB to GPU in 1.99828s (18.0155GiB/s) | 20260506 21:25:19.631 | INFO | Downloaded 34GiB to GPU in 1.91047s (17.7967GiB/s) | 20260506 21:25:24.773 | INFO | Analyzing chunk frames = [4565504,4566016) | 20260506 21:25:24.773 | INFO | MemInfo: { | 20260506 21:25:24.773 | INFO | "gpu" : " 17 GB", | 20260506 21:25:24.773 | INFO | "huge" : "", | 20260506 21:25:24.773 | INFO | "huge_shared" : " 27 GB", | 20260506 21:25:24.773 | INFO | "malloc" : " 14 GB", | 20260506 21:25:24.773 | INFO | "pinned" : " 29 GB" | 20260506 21:25:24.773 | INFO | } | 20260506 21:25:24.773 | INFO | Perf report for the last 5.14265 seconds: | 20260506 21:25:24.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:25:24.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:24.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:24.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:24.774 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 21:25:24.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:24.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:24.774 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 21:25:24.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:24.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:24.774 | INFO | Uploaded 36GiB to GPU in 2.01088s (17.9026GiB/s) | 20260506 21:25:24.774 | INFO | Downloaded 34GiB to GPU in 1.95277s (17.4111GiB/s) | 20260506 21:25:29.912 | INFO | Analyzing chunk frames = [4566016,4566528) | 20260506 21:25:29.912 | INFO | MemInfo: { | 20260506 21:25:29.912 | INFO | "gpu" : " 17 GB", | 20260506 21:25:29.912 | INFO | "huge" : "", | 20260506 21:25:29.912 | INFO | "huge_shared" : " 27 GB", | 20260506 21:25:29.912 | INFO | "malloc" : " 14 GB", | 20260506 21:25:29.912 | INFO | "pinned" : " 29 GB" | 20260506 21:25:29.912 | INFO | } | 20260506 21:25:29.912 | INFO | Perf report for the last 5.13835 seconds: | 20260506 21:25:29.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:25:29.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:29.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:29.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:29.912 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 21:25:29.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:29.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:29.912 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 21:25:29.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:29.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:29.912 | INFO | Uploaded 36GiB to GPU in 1.9804s (18.1782GiB/s) | 20260506 21:25:29.912 | INFO | Downloaded 34GiB to GPU in 2.00742s (16.9372GiB/s) | 20260506 21:25:35.055 | INFO | Analyzing chunk frames = [4566528,4567040) | 20260506 21:25:35.055 | INFO | MemInfo: { | 20260506 21:25:35.055 | INFO | "gpu" : " 17 GB", | 20260506 21:25:35.056 | INFO | "huge" : "", | 20260506 21:25:35.056 | INFO | "huge_shared" : " 27 GB", | 20260506 21:25:35.056 | INFO | "malloc" : " 14 GB", | 20260506 21:25:35.056 | INFO | "pinned" : " 29 GB" | 20260506 21:25:35.056 | INFO | } | 20260506 21:25:35.056 | INFO | Perf report for the last 5.1439 seconds: | 20260506 21:25:35.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:25:35.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:35.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:35.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:35.056 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 21:25:35.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:35.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:35.056 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 21:25:35.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:35.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:35.056 | INFO | Uploaded 36GiB to GPU in 2.03022s (17.7321GiB/s) | 20260506 21:25:35.056 | INFO | Downloaded 34GiB to GPU in 1.99852s (17.0126GiB/s) | 20260506 21:25:35.162 | INFO | Starting a round of Baz IO | 20260506 21:25:38.290 | INFO | Populating RTMetrics report | 20260506 21:25:38.291 | INFO | RTMetrics report populated | 20260506 21:25:38.291 | INFO | Opening tmp RTMetrics json file | 20260506 21:25:38.292 | INFO | Serializing RTMetrics to json | 20260506 21:25:38.296 | INFO | Serialization done, swapping tmp file now... | 20260506 21:25:38.303 | INFO | RTMetrics done | 20260506 21:25:38.572 | INFO | Finished a round of Baz IO: Wrote 1.947 GiB in 3.409 seconds | 20260506 21:25:38.572 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.94% / 0.00% / 66.05% | 20260506 21:25:38.572 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.617766% / 3.97364e-06% / 5.96046e-05% | 20260506 21:25:38.572 | INFO | Baz IO overall: | 20260506 21:25:38.572 | INFO | BytesWritten : 2406488762336 | 20260506 21:25:38.572 | INFO | NumEvents : 1161934602465 | 20260506 21:25:40.199 | INFO | Analyzing chunk frames = [4567040,4567552) | 20260506 21:25:40.199 | INFO | MemInfo: { | 20260506 21:25:40.199 | INFO | "gpu" : " 17 GB", | 20260506 21:25:40.199 | INFO | "huge" : "", | 20260506 21:25:40.199 | INFO | "huge_shared" : " 27 GB", | 20260506 21:25:40.199 | INFO | "malloc" : " 14 GB", | 20260506 21:25:40.199 | INFO | "pinned" : " 29 GB" | 20260506 21:25:40.199 | INFO | } | 20260506 21:25:40.199 | INFO | Perf report for the last 5.14364 seconds: | 20260506 21:25:40.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:25:40.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:40.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:40.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:40.199 | INFO | ANALYSIS : 62.3%, 2.9, 95.5ms, 0 | 20260506 21:25:40.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:40.199 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 21:25:40.199 | INFO | PRE_HQ : 59.7%, 1.8, 56.9ms, 0 | 20260506 21:25:40.199 | INFO | BAZWRITER : 65.9%, 4.4, 149.0ms, 0 | 20260506 21:25:40.199 | INFO | BAZ_SAMPLING : 4.5%, 1.0, 2.3ms, 0 | 20260506 21:25:40.200 | INFO | Uploaded 36GiB to GPU in 2.02896s (17.743GiB/s) | 20260506 21:25:40.200 | INFO | Downloaded 38GiB to GPU in 2.20869s (17.2048GiB/s) | 20260506 21:25:45.335 | INFO | Analyzing chunk frames = [4567552,4568064) | 20260506 21:25:45.335 | INFO | MemInfo: { | 20260506 21:25:45.335 | INFO | "gpu" : " 17 GB", | 20260506 21:25:45.335 | INFO | "huge" : "", | 20260506 21:25:45.335 | INFO | "huge_shared" : " 27 GB", | 20260506 21:25:45.335 | INFO | "malloc" : " 14 GB", | 20260506 21:25:45.335 | INFO | "pinned" : " 29 GB" | 20260506 21:25:45.335 | INFO | } | 20260506 21:25:45.335 | INFO | Perf report for the last 5.13533 seconds: | 20260506 21:25:45.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:25:45.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:45.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:45.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:45.335 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 21:25:45.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:45.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:45.335 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:25:45.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:45.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:45.335 | INFO | Uploaded 36GiB to GPU in 2.0025s (17.9775GiB/s) | 20260506 21:25:45.335 | INFO | Downloaded 34GiB to GPU in 1.93027s (17.6142GiB/s) | 20260506 21:25:50.476 | INFO | Analyzing chunk frames = [4568064,4568576) | 20260506 21:25:50.476 | INFO | MemInfo: { | 20260506 21:25:50.476 | INFO | "gpu" : " 17 GB", | 20260506 21:25:50.476 | INFO | "huge" : "", | 20260506 21:25:50.476 | INFO | "huge_shared" : " 27 GB", | 20260506 21:25:50.476 | INFO | "malloc" : " 14 GB", | 20260506 21:25:50.476 | INFO | "pinned" : " 29 GB" | 20260506 21:25:50.476 | INFO | } | 20260506 21:25:50.476 | INFO | Perf report for the last 5.14167 seconds: | 20260506 21:25:50.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:25:50.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:50.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:50.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:50.476 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 21:25:50.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:50.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:50.477 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:25:50.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:50.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:50.477 | INFO | Uploaded 36GiB to GPU in 1.98028s (18.1792GiB/s) | 20260506 21:25:50.477 | INFO | Downloaded 34GiB to GPU in 1.93279s (17.5911GiB/s) | 20260506 21:25:55.608 | INFO | Analyzing chunk frames = [4568576,4569088) | 20260506 21:25:55.608 | INFO | MemInfo: { | 20260506 21:25:55.608 | INFO | "gpu" : " 17 GB", | 20260506 21:25:55.608 | INFO | "huge" : "", | 20260506 21:25:55.608 | INFO | "huge_shared" : " 27 GB", | 20260506 21:25:55.608 | INFO | "malloc" : " 14 GB", | 20260506 21:25:55.608 | INFO | "pinned" : " 29 GB" | 20260506 21:25:55.608 | INFO | } | 20260506 21:25:55.608 | INFO | Perf report for the last 5.13183 seconds: | 20260506 21:25:55.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:25:55.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:55.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:55.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:55.608 | INFO | ANALYSIS : 59.7%, 3.0, 92.4ms, 0 | 20260506 21:25:55.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:55.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:55.608 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:25:55.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:55.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:25:55.609 | INFO | Uploaded 36GiB to GPU in 1.9555s (18.4096GiB/s) | 20260506 21:25:55.609 | INFO | Downloaded 34GiB to GPU in 1.87751s (18.1091GiB/s) | 20260506 21:26:00.744 | INFO | Analyzing chunk frames = [4569088,4569600) | 20260506 21:26:00.744 | INFO | MemInfo: { | 20260506 21:26:00.744 | INFO | "gpu" : " 17 GB", | 20260506 21:26:00.744 | INFO | "huge" : "", | 20260506 21:26:00.744 | INFO | "huge_shared" : " 27 GB", | 20260506 21:26:00.744 | INFO | "malloc" : " 14 GB", | 20260506 21:26:00.744 | INFO | "pinned" : " 29 GB" | 20260506 21:26:00.744 | INFO | } | 20260506 21:26:00.744 | INFO | Perf report for the last 5.13585 seconds: | 20260506 21:26:00.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:26:00.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:00.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:00.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:00.744 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 21:26:00.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:00.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:00.744 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:26:00.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:00.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:00.744 | INFO | Uploaded 36GiB to GPU in 1.99749s (18.0226GiB/s) | 20260506 21:26:00.744 | INFO | Downloaded 34GiB to GPU in 1.9172s (17.7342GiB/s) | 20260506 21:26:05.891 | INFO | Analyzing chunk frames = [4569600,4570112) | 20260506 21:26:05.891 | INFO | MemInfo: { | 20260506 21:26:05.891 | INFO | "gpu" : " 17 GB", | 20260506 21:26:05.891 | INFO | "huge" : "", | 20260506 21:26:05.891 | INFO | "huge_shared" : " 27 GB", | 20260506 21:26:05.891 | INFO | "malloc" : " 14 GB", | 20260506 21:26:05.891 | INFO | "pinned" : " 29 GB" | 20260506 21:26:05.891 | INFO | } | 20260506 21:26:05.891 | INFO | Perf report for the last 5.14708 seconds: | 20260506 21:26:05.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:26:05.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:05.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:05.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:05.891 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260506 21:26:05.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:05.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:05.891 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 21:26:05.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:05.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:05.891 | INFO | Uploaded 36GiB to GPU in 2.00329s (17.9704GiB/s) | 20260506 21:26:05.891 | INFO | Downloaded 34GiB to GPU in 1.95187s (17.4192GiB/s) | 20260506 21:26:11.030 | INFO | Analyzing chunk frames = [4570112,4570624) | 20260506 21:26:11.030 | INFO | MemInfo: { | 20260506 21:26:11.030 | INFO | "gpu" : " 17 GB", | 20260506 21:26:11.030 | INFO | "huge" : "", | 20260506 21:26:11.030 | INFO | "huge_shared" : " 27 GB", | 20260506 21:26:11.030 | INFO | "malloc" : " 14 GB", | 20260506 21:26:11.030 | INFO | "pinned" : " 29 GB" | 20260506 21:26:11.030 | INFO | } | 20260506 21:26:11.030 | INFO | Perf report for the last 5.1391 seconds: | 20260506 21:26:11.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:26:11.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:11.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:11.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:11.030 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 21:26:11.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:11.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:11.030 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 21:26:11.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:11.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:11.031 | INFO | Uploaded 36GiB to GPU in 2.00562s (17.9496GiB/s) | 20260506 21:26:11.031 | INFO | Downloaded 34GiB to GPU in 1.99351s (17.0554GiB/s) | 20260506 21:26:16.173 | INFO | Analyzing chunk frames = [4570624,4571136) | 20260506 21:26:16.173 | INFO | MemInfo: { | 20260506 21:26:16.173 | INFO | "gpu" : " 17 GB", | 20260506 21:26:16.173 | INFO | "huge" : "", | 20260506 21:26:16.173 | INFO | "huge_shared" : " 27 GB", | 20260506 21:26:16.173 | INFO | "malloc" : " 14 GB", | 20260506 21:26:16.173 | INFO | "pinned" : " 29 GB" | 20260506 21:26:16.173 | INFO | } | 20260506 21:26:16.173 | INFO | Perf report for the last 5.14274 seconds: | 20260506 21:26:16.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:26:16.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:16.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:16.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:16.173 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 21:26:16.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:16.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:16.173 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 21:26:16.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:16.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:16.173 | INFO | Uploaded 36GiB to GPU in 2.00776s (17.9304GiB/s) | 20260506 21:26:16.173 | INFO | Downloaded 34GiB to GPU in 1.98676s (17.1133GiB/s) | 20260506 21:26:16.173 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:26:16.173 | INFO | Total mem usage: 18 GiB | 20260506 21:26:16.173 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:26:16.173 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -20.7%) | 20260506 21:26:16.173 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:26:16.173 | INFO | High Water Breakdown: | 20260506 21:26:16.173 | INFO | 60.8% : 9.07 GiB ( 79.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:26:16.173 | INFO | 39.1% : 5.83 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:26:16.174 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:26:16.174 | INFO | ------------------------------------------ | 20260506 21:26:16.174 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:26:16.174 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:26:16.174 | INFO | Total mem usage: 26.5 GiB | 20260506 21:26:16.174 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:26:16.174 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:26:16.174 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:26:16.174 | INFO | High Water Breakdown: | 20260506 21:26:16.174 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:26:16.174 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:26:16.174 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:26:16.174 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:26:16.174 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:26:16.174 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:26:16.174 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:26:16.174 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:26:16.174 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:26:16.174 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:26:16.174 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:26:16.174 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:26:16.174 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:26:16.174 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:26:16.174 | INFO | ------------------------------------------ | 20260506 21:26:16.174 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:26:16.174 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:26:16.174 | INFO | Total mem usage: 25.6 GiB | 20260506 21:26:16.174 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:26:16.174 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:26:16.174 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:26:16.174 | INFO | High Water Breakdown: | 20260506 21:26:16.174 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:26:16.174 | INFO | ------------------------------------------ | 20260506 21:26:16.174 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:26:16.174 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:26:16.174 | INFO | Total mem usage: 15.7 GiB | 20260506 21:26:16.174 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:26:16.174 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:26:16.174 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:26:16.174 | INFO | High Water Breakdown: | 20260506 21:26:16.174 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:26:16.174 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:26:16.174 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:26:16.174 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:26:16.174 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:26:16.174 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:26:16.174 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:26:16.174 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:26:16.175 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:26:16.175 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:26:16.175 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:26:16.175 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:26:16.175 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:26:16.175 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:26:16.175 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:26:16.175 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:26:16.175 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:26:16.175 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:26:16.175 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:26:16.175 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:26:16.175 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:26:16.175 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:26:16.175 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:26:16.175 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:26:16.175 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:26:16.175 | INFO | ------------------------------------------ | 20260506 21:26:16.175 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:26:16.175 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:26:16.175 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:26:16.175 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:26:16.175 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.57e+01, 1.33e+01 / 2.53e+00 | 20260506 21:26:16.175 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.04e-01, 9.98e-03 / 8.85e-03 | 20260506 21:26:16.175 | INFO | PerfReport: 11.65%, Baseline -- 3.63e+00 / 4.40e+00, 5.52e-01 / 3.38e-01 | 20260506 21:26:16.175 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.49e+00, 1.85e-01 / 1.34e-01 | 20260506 21:26:16.175 | INFO | PerfReport: 15.95%, DME -- 3.97e+01 / 4.41e+01, 3.82e+00 / 2.16e+00 | 20260506 21:26:16.175 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.78e+01 / 2.12e+01, 1.46e+00 / 1.47e+00 | 20260506 21:26:16.175 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.21e+00 / 2.70e+00, 3.77e-01 / 2.51e-01 | 20260506 21:26:16.175 | INFO | PerfReport: 3.27%, Metrics -- 1.02e+00 / 1.36e+00, 2.65e-01 / 1.81e-01 | 20260506 21:26:16.175 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.48e-01 / 3.96e-01, 5.34e-02 / 2.33e-02 | 20260506 21:26:16.175 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:26:16.175 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:26:16.175 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:26:16.175 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.30e+01 / 0.00e+00 | 20260506 21:26:16.175 | INFO | PerfReport: 21.49%, Upload -- 2.08e+01 / 0.00e+00, 6.83e+00 / 0.00e+00 | 20260506 21:26:16.175 | INFO | PerfReport: 21.29%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 21:26:16.175 | INFO | PerfReport: 32.21%, Compute -- 3.11e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 21:26:16.175 | INFO | PerfReport: 1.19%, LockUpload -- 1.15e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260506 21:26:16.175 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.10e+00 / 0.00e+00 | 20260506 21:26:16.175 | INFO | PerfReport: 22.52%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:26:16.175 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260506 21:26:16.271 | INFO | Starting a round of Baz IO | 20260506 21:26:19.409 | INFO | Populating RTMetrics report | 20260506 21:26:19.409 | INFO | RTMetrics report populated | 20260506 21:26:19.409 | INFO | Opening tmp RTMetrics json file | 20260506 21:26:19.520 | INFO | Serializing RTMetrics to json | 20260506 21:26:19.522 | INFO | Serialization done, swapping tmp file now... | 20260506 21:26:19.523 | INFO | RTMetrics done | 20260506 21:26:19.951 | INFO | Finished a round of Baz IO: Wrote 1.945 GiB in 3.679 seconds | 20260506 21:26:19.951 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.94% / 0.00% / 66.06% | 20260506 21:26:19.951 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.615907% / 3.97364e-06% / 6.35783e-05% | 20260506 21:26:19.951 | INFO | Baz IO overall: | 20260506 21:26:19.951 | INFO | BytesWritten : 2408577627833 | 20260506 21:26:19.951 | INFO | NumEvents : 1162933873285 | 20260506 21:26:21.312 | INFO | Analyzing chunk frames = [4571136,4571648) | 20260506 21:26:21.312 | INFO | MemInfo: { | 20260506 21:26:21.312 | INFO | "gpu" : " 17 GB", | 20260506 21:26:21.312 | INFO | "huge" : "", | 20260506 21:26:21.312 | INFO | "huge_shared" : " 27 GB", | 20260506 21:26:21.312 | INFO | "malloc" : " 14 GB", | 20260506 21:26:21.312 | INFO | "pinned" : " 29 GB" | 20260506 21:26:21.312 | INFO | } | 20260506 21:26:21.312 | INFO | Perf report for the last 5.13902 seconds: | 20260506 21:26:21.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:26:21.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:21.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:21.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:21.312 | INFO | ANALYSIS : 62.4%, 2.9, 95.4ms, 0 | 20260506 21:26:21.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:21.312 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 21:26:21.312 | INFO | PRE_HQ : 59.9%, 1.8, 56.6ms, 0 | 20260506 21:26:21.312 | INFO | BAZWRITER : 71.2%, 4.8, 176.1ms, 0 | 20260506 21:26:21.312 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 21:26:21.312 | INFO | Uploaded 36GiB to GPU in 2.04611s (17.5944GiB/s) | 20260506 21:26:21.312 | INFO | Downloaded 38GiB to GPU in 2.20662s (17.2209GiB/s) | 20260506 21:26:26.444 | INFO | Analyzing chunk frames = [4571648,4572160) | 20260506 21:26:26.444 | INFO | MemInfo: { | 20260506 21:26:26.444 | INFO | "gpu" : " 17 GB", | 20260506 21:26:26.444 | INFO | "huge" : "", | 20260506 21:26:26.444 | INFO | "huge_shared" : " 27 GB", | 20260506 21:26:26.444 | INFO | "malloc" : " 14 GB", | 20260506 21:26:26.444 | INFO | "pinned" : " 29 GB" | 20260506 21:26:26.444 | INFO | } | 20260506 21:26:26.444 | INFO | Perf report for the last 5.13177 seconds: | 20260506 21:26:26.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:26:26.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:26.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:26.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:26.444 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 21:26:26.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:26.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:26.444 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 21:26:26.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:26.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:26.444 | INFO | Uploaded 36GiB to GPU in 2.02036s (17.8186GiB/s) | 20260506 21:26:26.444 | INFO | Downloaded 34GiB to GPU in 1.927s (17.644GiB/s) | 20260506 21:26:31.582 | INFO | Analyzing chunk frames = [4572160,4572672) | 20260506 21:26:31.582 | INFO | MemInfo: { | 20260506 21:26:31.582 | INFO | "gpu" : " 17 GB", | 20260506 21:26:31.582 | INFO | "huge" : "", | 20260506 21:26:31.582 | INFO | "huge_shared" : " 27 GB", | 20260506 21:26:31.582 | INFO | "malloc" : " 14 GB", | 20260506 21:26:31.582 | INFO | "pinned" : " 29 GB" | 20260506 21:26:31.582 | INFO | } | 20260506 21:26:31.582 | INFO | Perf report for the last 5.13791 seconds: | 20260506 21:26:31.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:26:31.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:31.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:31.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:31.582 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 21:26:31.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:31.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:31.582 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 21:26:31.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:31.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:31.582 | INFO | Uploaded 36GiB to GPU in 1.98099s (18.1728GiB/s) | 20260506 21:26:31.582 | INFO | Downloaded 34GiB to GPU in 1.91442s (17.7599GiB/s) | 20260506 21:26:36.720 | INFO | Analyzing chunk frames = [4572672,4573184) | 20260506 21:26:36.720 | INFO | MemInfo: { | 20260506 21:26:36.720 | INFO | "gpu" : " 17 GB", | 20260506 21:26:36.720 | INFO | "huge" : "", | 20260506 21:26:36.720 | INFO | "huge_shared" : " 27 GB", | 20260506 21:26:36.720 | INFO | "malloc" : " 14 GB", | 20260506 21:26:36.720 | INFO | "pinned" : " 29 GB" | 20260506 21:26:36.720 | INFO | } | 20260506 21:26:36.720 | INFO | Perf report for the last 5.13822 seconds: | 20260506 21:26:36.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:26:36.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:36.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:36.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:36.720 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 21:26:36.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:36.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:36.720 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:26:36.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:36.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:36.720 | INFO | Uploaded 36GiB to GPU in 1.95623s (18.4028GiB/s) | 20260506 21:26:36.720 | INFO | Downloaded 34GiB to GPU in 1.87384s (18.1446GiB/s) | 20260506 21:26:41.861 | INFO | Analyzing chunk frames = [4573184,4573696) | 20260506 21:26:41.861 | INFO | MemInfo: { | 20260506 21:26:41.861 | INFO | "gpu" : " 17 GB", | 20260506 21:26:41.861 | INFO | "huge" : "", | 20260506 21:26:41.861 | INFO | "huge_shared" : " 27 GB", | 20260506 21:26:41.861 | INFO | "malloc" : " 14 GB", | 20260506 21:26:41.861 | INFO | "pinned" : " 29 GB" | 20260506 21:26:41.861 | INFO | } | 20260506 21:26:41.861 | INFO | Perf report for the last 5.14084 seconds: | 20260506 21:26:41.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:26:41.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:41.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:41.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:41.861 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 21:26:41.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:41.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:41.861 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:26:41.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:41.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:41.861 | INFO | Uploaded 36GiB to GPU in 2.01925s (17.8284GiB/s) | 20260506 21:26:41.861 | INFO | Downloaded 34GiB to GPU in 1.91371s (17.7666GiB/s) | 20260506 21:26:47.004 | INFO | Analyzing chunk frames = [4573696,4574208) | 20260506 21:26:47.004 | INFO | MemInfo: { | 20260506 21:26:47.004 | INFO | "gpu" : " 17 GB", | 20260506 21:26:47.004 | INFO | "huge" : "", | 20260506 21:26:47.004 | INFO | "huge_shared" : " 27 GB", | 20260506 21:26:47.004 | INFO | "malloc" : " 14 GB", | 20260506 21:26:47.004 | INFO | "pinned" : " 29 GB" | 20260506 21:26:47.004 | INFO | } | 20260506 21:26:47.004 | INFO | Perf report for the last 5.14292 seconds: | 20260506 21:26:47.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:26:47.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:47.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:47.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:47.004 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 21:26:47.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:47.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:47.004 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 21:26:47.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:47.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:47.004 | INFO | Uploaded 36GiB to GPU in 2.03283s (17.7093GiB/s) | 20260506 21:26:47.004 | INFO | Downloaded 34GiB to GPU in 1.94829s (17.4512GiB/s) | 20260506 21:26:52.157 | INFO | Analyzing chunk frames = [4574208,4574720) | 20260506 21:26:52.157 | INFO | MemInfo: { | 20260506 21:26:52.157 | INFO | "gpu" : " 17 GB", | 20260506 21:26:52.157 | INFO | "huge" : "", | 20260506 21:26:52.157 | INFO | "huge_shared" : " 27 GB", | 20260506 21:26:52.157 | INFO | "malloc" : " 14 GB", | 20260506 21:26:52.157 | INFO | "pinned" : " 29 GB" | 20260506 21:26:52.157 | INFO | } | 20260506 21:26:52.157 | INFO | Perf report for the last 5.15307 seconds: | 20260506 21:26:52.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:26:52.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:52.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:52.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:52.157 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 21:26:52.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:52.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:52.157 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 21:26:52.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:52.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:52.157 | INFO | Uploaded 36GiB to GPU in 1.96568s (18.3142GiB/s) | 20260506 21:26:52.157 | INFO | Downloaded 34GiB to GPU in 1.98634s (17.1169GiB/s) | 20260506 21:26:57.288 | INFO | Analyzing chunk frames = [4574720,4575232) | 20260506 21:26:57.288 | INFO | MemInfo: { | 20260506 21:26:57.288 | INFO | "gpu" : " 17 GB", | 20260506 21:26:57.288 | INFO | "huge" : "", | 20260506 21:26:57.288 | INFO | "huge_shared" : " 27 GB", | 20260506 21:26:57.288 | INFO | "malloc" : " 14 GB", | 20260506 21:26:57.288 | INFO | "pinned" : " 29 GB" | 20260506 21:26:57.288 | INFO | } | 20260506 21:26:57.288 | INFO | Perf report for the last 5.13139 seconds: | 20260506 21:26:57.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:26:57.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:57.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:57.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:57.288 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 21:26:57.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:57.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:57.289 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 21:26:57.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:57.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:26:57.289 | INFO | Uploaded 36GiB to GPU in 1.96562s (18.3148GiB/s) | 20260506 21:26:57.289 | INFO | Downloaded 34GiB to GPU in 1.9969s (17.0264GiB/s) | 20260506 21:26:57.386 | INFO | Starting a round of Baz IO | 20260506 21:27:00.518 | INFO | Populating RTMetrics report | 20260506 21:27:00.518 | INFO | RTMetrics report populated | 20260506 21:27:00.518 | INFO | Opening tmp RTMetrics json file | 20260506 21:27:00.525 | INFO | Serializing RTMetrics to json | 20260506 21:27:00.548 | INFO | Serialization done, swapping tmp file now... | 20260506 21:27:00.550 | INFO | RTMetrics done | 20260506 21:27:00.781 | INFO | Finished a round of Baz IO: Wrote 1.946 GiB in 3.394 seconds | 20260506 21:27:00.781 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.94% / 0.00% / 66.06% | 20260506 21:27:00.781 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.614385% / 7.94729e-06% / 5.96046e-05% | 20260506 21:27:00.781 | INFO | Baz IO overall: | 20260506 21:27:00.781 | INFO | BytesWritten : 2410667415354 | 20260506 21:27:00.781 | INFO | NumEvents : 1163934016106 | 20260506 21:27:02.425 | INFO | Analyzing chunk frames = [4575232,4575744) | 20260506 21:27:02.425 | INFO | MemInfo: { | 20260506 21:27:02.425 | INFO | "gpu" : " 17 GB", | 20260506 21:27:02.425 | INFO | "huge" : "", | 20260506 21:27:02.425 | INFO | "huge_shared" : " 27 GB", | 20260506 21:27:02.425 | INFO | "malloc" : " 14 GB", | 20260506 21:27:02.425 | INFO | "pinned" : " 29 GB" | 20260506 21:27:02.425 | INFO | } | 20260506 21:27:02.425 | INFO | Perf report for the last 5.13703 seconds: | 20260506 21:27:02.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:27:02.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:02.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:02.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:02.425 | INFO | ANALYSIS : 62.3%, 2.9, 95.1ms, 0 | 20260506 21:27:02.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:02.425 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260506 21:27:02.425 | INFO | PRE_HQ : 60.0%, 1.8, 57.0ms, 0 | 20260506 21:27:02.425 | INFO | BAZWRITER : 65.7%, 4.8, 161.8ms, 0 | 20260506 21:27:02.425 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 21:27:02.426 | INFO | Uploaded 36GiB to GPU in 2.06092s (17.468GiB/s) | 20260506 21:27:02.426 | INFO | Downloaded 38GiB to GPU in 2.19967s (17.2753GiB/s) | 20260506 21:27:07.563 | INFO | Analyzing chunk frames = [4575744,4576256) | 20260506 21:27:07.563 | INFO | MemInfo: { | 20260506 21:27:07.563 | INFO | "gpu" : " 17 GB", | 20260506 21:27:07.563 | INFO | "huge" : "", | 20260506 21:27:07.563 | INFO | "huge_shared" : " 27 GB", | 20260506 21:27:07.563 | INFO | "malloc" : " 14 GB", | 20260506 21:27:07.563 | INFO | "pinned" : " 29 GB" | 20260506 21:27:07.563 | INFO | } | 20260506 21:27:07.563 | INFO | Perf report for the last 5.13753 seconds: | 20260506 21:27:07.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:27:07.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:07.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:07.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:07.563 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:27:07.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:07.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:07.563 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:27:07.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:07.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:07.563 | INFO | Uploaded 36GiB to GPU in 2.00383s (17.9656GiB/s) | 20260506 21:27:07.563 | INFO | Downloaded 34GiB to GPU in 1.93221s (17.5964GiB/s) | 20260506 21:27:12.697 | INFO | Analyzing chunk frames = [4576256,4576768) | 20260506 21:27:12.697 | INFO | MemInfo: { | 20260506 21:27:12.697 | INFO | "gpu" : " 17 GB", | 20260506 21:27:12.698 | INFO | "huge" : "", | 20260506 21:27:12.698 | INFO | "huge_shared" : " 27 GB", | 20260506 21:27:12.698 | INFO | "malloc" : " 14 GB", | 20260506 21:27:12.698 | INFO | "pinned" : " 29 GB" | 20260506 21:27:12.698 | INFO | } | 20260506 21:27:12.698 | INFO | Perf report for the last 5.13467 seconds: | 20260506 21:27:12.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:27:12.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:12.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:12.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:12.698 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 21:27:12.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:12.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:12.698 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:27:12.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:12.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:12.698 | INFO | Uploaded 36GiB to GPU in 1.97489s (18.2289GiB/s) | 20260506 21:27:12.698 | INFO | Downloaded 34GiB to GPU in 1.92555s (17.6573GiB/s) | 20260506 21:27:17.836 | INFO | Analyzing chunk frames = [4576768,4577280) | 20260506 21:27:17.836 | INFO | MemInfo: { | 20260506 21:27:17.836 | INFO | "gpu" : " 17 GB", | 20260506 21:27:17.837 | INFO | "huge" : "", | 20260506 21:27:17.837 | INFO | "huge_shared" : " 27 GB", | 20260506 21:27:17.837 | INFO | "malloc" : " 14 GB", | 20260506 21:27:17.837 | INFO | "pinned" : " 29 GB" | 20260506 21:27:17.837 | INFO | } | 20260506 21:27:17.837 | INFO | Perf report for the last 5.13902 seconds: | 20260506 21:27:17.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:27:17.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:17.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:17.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:17.837 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 21:27:17.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:17.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:17.837 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:27:17.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:17.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:17.837 | INFO | Uploaded 36GiB to GPU in 1.96607s (18.3106GiB/s) | 20260506 21:27:17.837 | INFO | Downloaded 34GiB to GPU in 1.87913s (18.0935GiB/s) | 20260506 21:27:22.980 | INFO | Analyzing chunk frames = [4577280,4577792) | 20260506 21:27:22.980 | INFO | MemInfo: { | 20260506 21:27:22.980 | INFO | "gpu" : " 17 GB", | 20260506 21:27:22.980 | INFO | "huge" : "", | 20260506 21:27:22.980 | INFO | "huge_shared" : " 27 GB", | 20260506 21:27:22.980 | INFO | "malloc" : " 14 GB", | 20260506 21:27:22.980 | INFO | "pinned" : " 29 GB" | 20260506 21:27:22.980 | INFO | } | 20260506 21:27:22.980 | INFO | Perf report for the last 5.14354 seconds: | 20260506 21:27:22.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:27:22.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:22.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:22.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:22.980 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 21:27:22.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:22.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:22.980 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:27:22.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:22.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:22.981 | INFO | Uploaded 36GiB to GPU in 1.95193s (18.4433GiB/s) | 20260506 21:27:22.981 | INFO | Downloaded 34GiB to GPU in 1.92829s (17.6322GiB/s) | 20260506 21:27:28.115 | INFO | Analyzing chunk frames = [4577792,4578304) | 20260506 21:27:28.115 | INFO | MemInfo: { | 20260506 21:27:28.115 | INFO | "gpu" : " 17 GB", | 20260506 21:27:28.115 | INFO | "huge" : "", | 20260506 21:27:28.115 | INFO | "huge_shared" : " 27 GB", | 20260506 21:27:28.115 | INFO | "malloc" : " 14 GB", | 20260506 21:27:28.116 | INFO | "pinned" : " 29 GB" | 20260506 21:27:28.116 | INFO | } | 20260506 21:27:28.116 | INFO | Perf report for the last 5.13543 seconds: | 20260506 21:27:28.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:27:28.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:28.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:28.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:28.116 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 21:27:28.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:28.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:28.116 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 21:27:28.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:28.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:28.116 | INFO | Uploaded 36GiB to GPU in 1.98975s (18.0927GiB/s) | 20260506 21:27:28.116 | INFO | Downloaded 34GiB to GPU in 1.94606s (17.4712GiB/s) | 20260506 21:27:33.257 | INFO | Analyzing chunk frames = [4578304,4578816) | 20260506 21:27:33.258 | INFO | MemInfo: { | 20260506 21:27:33.258 | INFO | "gpu" : " 17 GB", | 20260506 21:27:33.258 | INFO | "huge" : "", | 20260506 21:27:33.258 | INFO | "huge_shared" : " 27 GB", | 20260506 21:27:33.258 | INFO | "malloc" : " 14 GB", | 20260506 21:27:33.258 | INFO | "pinned" : " 29 GB" | 20260506 21:27:33.258 | INFO | } | 20260506 21:27:33.258 | INFO | Perf report for the last 5.14213 seconds: | 20260506 21:27:33.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:27:33.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:33.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:33.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:33.258 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 21:27:33.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:33.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:33.258 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 21:27:33.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:33.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:33.258 | INFO | Uploaded 36GiB to GPU in 1.99511s (18.0442GiB/s) | 20260506 21:27:33.258 | INFO | Downloaded 34GiB to GPU in 2.01041s (16.9119GiB/s) | 20260506 21:27:38.398 | INFO | Analyzing chunk frames = [4578816,4579328) | 20260506 21:27:38.398 | INFO | MemInfo: { | 20260506 21:27:38.398 | INFO | "gpu" : " 17 GB", | 20260506 21:27:38.398 | INFO | "huge" : "", | 20260506 21:27:38.398 | INFO | "huge_shared" : " 27 GB", | 20260506 21:27:38.398 | INFO | "malloc" : " 14 GB", | 20260506 21:27:38.399 | INFO | "pinned" : " 29 GB" | 20260506 21:27:38.399 | INFO | } | 20260506 21:27:38.399 | INFO | Perf report for the last 5.14081 seconds: | 20260506 21:27:38.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:27:38.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:38.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:38.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:38.399 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 21:27:38.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:38.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:38.399 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:27:38.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:38.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:38.399 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:27:38.399 | INFO | Total mem usage: 18 GiB | 20260506 21:27:38.399 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:27:38.399 | INFO | Uploaded 36GiB to GPU in 1.99864s (18.0122GiB/s) | 20260506 21:27:38.399 | INFO | Recent High Water Marks Sum: 16.4 GiB (Savings: -9.54%) | 20260506 21:27:38.399 | INFO | Downloaded 34GiB to GPU in 1.97339s (17.2293GiB/s) | 20260506 21:27:38.399 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:27:38.399 | INFO | High Water Breakdown: | 20260506 21:27:38.399 | INFO | 62.3% : 10.2 GiB ( 89.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:27:38.399 | INFO | 37.6% : 6.18 GiB ( 92.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:27:38.399 | INFO | 0.121% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:27:38.399 | INFO | ------------------------------------------ | 20260506 21:27:38.399 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:27:38.399 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:27:38.399 | INFO | Total mem usage: 26.5 GiB | 20260506 21:27:38.399 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:27:38.399 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:27:38.399 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:27:38.399 | INFO | High Water Breakdown: | 20260506 21:27:38.399 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:27:38.399 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:27:38.399 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:27:38.399 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:27:38.399 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:27:38.399 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:27:38.399 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:27:38.399 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:27:38.399 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:27:38.399 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:27:38.399 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:27:38.399 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:27:38.399 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:27:38.399 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:27:38.399 | INFO | ------------------------------------------ | 20260506 21:27:38.399 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:27:38.399 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:27:38.399 | INFO | Total mem usage: 25.6 GiB | 20260506 21:27:38.399 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:27:38.399 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:27:38.399 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:27:38.399 | INFO | High Water Breakdown: | 20260506 21:27:38.399 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:27:38.399 | INFO | ------------------------------------------ | 20260506 21:27:38.399 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:27:38.399 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:27:38.399 | INFO | Total mem usage: 15.7 GiB | 20260506 21:27:38.399 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:27:38.399 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:27:38.399 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:27:38.399 | INFO | High Water Breakdown: | 20260506 21:27:38.399 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:27:38.399 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:27:38.399 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:27:38.399 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:27:38.399 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:27:38.399 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:27:38.399 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:27:38.400 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:27:38.400 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:27:38.400 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:27:38.400 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:27:38.400 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:27:38.400 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:27:38.400 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:27:38.400 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:27:38.400 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:27:38.400 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:27:38.400 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:27:38.400 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:27:38.400 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:27:38.400 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:27:38.400 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:27:38.400 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:27:38.400 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:27:38.400 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:27:38.400 | INFO | ------------------------------------------ | 20260506 21:27:38.400 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:27:38.400 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:27:38.400 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:27:38.400 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:27:38.400 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.58e+01, 1.32e+01 / 2.88e+00 | 20260506 21:27:38.400 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.02e-01, 1.00e-02 / 7.18e-03 | 20260506 21:27:38.400 | INFO | PerfReport: 11.63%, Baseline -- 3.62e+00 / 4.35e+00, 5.50e-01 / 4.01e-01 | 20260506 21:27:38.400 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.45e+00, 1.80e-01 / 1.37e-01 | 20260506 21:27:38.400 | INFO | PerfReport: 16.02%, DME -- 3.99e+01 / 4.45e+01, 4.12e+00 / 2.94e+00 | 20260506 21:27:38.400 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.40e+00 / 1.90e+00 | 20260506 21:27:38.400 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.21e+00 / 2.79e+00, 3.78e-01 / 2.16e-01 | 20260506 21:27:38.400 | INFO | PerfReport: 3.27%, Metrics -- 1.02e+00 / 1.35e+00, 2.68e-01 / 1.66e-01 | 20260506 21:27:38.400 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.49e-01 / 4.01e-01, 5.46e-02 / 3.39e-02 | 20260506 21:27:38.400 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:27:38.400 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:27:38.400 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:27:38.400 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 21:27:38.400 | INFO | PerfReport: 21.51%, Upload -- 2.08e+01 / 0.00e+00, 6.92e+00 / 0.00e+00 | 20260506 21:27:38.400 | INFO | PerfReport: 21.27%, Download -- 2.05e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 21:27:38.400 | INFO | PerfReport: 32.20%, Compute -- 3.11e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 21:27:38.400 | INFO | PerfReport: 1.24%, LockUpload -- 1.20e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260506 21:27:38.400 | INFO | PerfReport: 1.22%, LockDownload -- 1.17e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260506 21:27:38.400 | INFO | PerfReport: 22.55%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 21:27:38.400 | INFO | PerfReport: 0.02%, Misc -- 1.51e-02 / 0.00e+00, 2.37e-02 / 0.00e+00 | 20260506 21:27:38.496 | INFO | Starting a round of Baz IO | 20260506 21:27:41.634 | INFO | Populating RTMetrics report | 20260506 21:27:41.634 | INFO | RTMetrics report populated | 20260506 21:27:41.634 | INFO | Opening tmp RTMetrics json file | 20260506 21:27:41.635 | INFO | Serializing RTMetrics to json | 20260506 21:27:41.637 | INFO | Serialization done, swapping tmp file now... | 20260506 21:27:41.637 | INFO | RTMetrics done | 20260506 21:27:41.879 | INFO | Finished a round of Baz IO: Wrote 1.947 GiB in 3.382 seconds | 20260506 21:27:41.879 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.93% / 0.00% / 66.06% | 20260506 21:27:41.879 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.614365% / 0% / 4.76837e-05% | 20260506 21:27:41.879 | INFO | Baz IO overall: | 20260506 21:27:41.879 | INFO | BytesWritten : 2412757833077 | 20260506 21:27:41.879 | INFO | NumEvents : 1164934765558 | 20260506 21:27:43.537 | INFO | Analyzing chunk frames = [4579328,4579840) | 20260506 21:27:43.537 | INFO | MemInfo: { | 20260506 21:27:43.537 | INFO | "gpu" : " 17 GB", | 20260506 21:27:43.537 | INFO | "huge" : "", | 20260506 21:27:43.537 | INFO | "huge_shared" : " 27 GB", | 20260506 21:27:43.538 | INFO | "malloc" : " 14 GB", | 20260506 21:27:43.538 | INFO | "pinned" : " 29 GB" | 20260506 21:27:43.538 | INFO | } | 20260506 21:27:43.538 | INFO | Perf report for the last 5.13923 seconds: | 20260506 21:27:43.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:27:43.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:43.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:43.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:43.538 | INFO | ANALYSIS : 62.4%, 2.9, 95.3ms, 0 | 20260506 21:27:43.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:43.538 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 21:27:43.538 | INFO | PRE_HQ : 59.9%, 1.8, 56.8ms, 0 | 20260506 21:27:43.538 | INFO | BAZWRITER : 65.5%, 4.8, 161.2ms, 0 | 20260506 21:27:43.538 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 21:27:43.538 | INFO | Uploaded 36GiB to GPU in 2.05184s (17.5453GiB/s) | 20260506 21:27:43.538 | INFO | Downloaded 38GiB to GPU in 2.19679s (17.298GiB/s) | 20260506 21:27:48.675 | INFO | Analyzing chunk frames = [4579840,4580352) | 20260506 21:27:48.675 | INFO | MemInfo: { | 20260506 21:27:48.675 | INFO | "gpu" : " 17 GB", | 20260506 21:27:48.675 | INFO | "huge" : "", | 20260506 21:27:48.675 | INFO | "huge_shared" : " 27 GB", | 20260506 21:27:48.675 | INFO | "malloc" : " 14 GB", | 20260506 21:27:48.675 | INFO | "pinned" : " 29 GB" | 20260506 21:27:48.675 | INFO | } | 20260506 21:27:48.675 | INFO | Perf report for the last 5.13698 seconds: | 20260506 21:27:48.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:27:48.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:48.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:48.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:48.675 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 21:27:48.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:48.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:48.675 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:27:48.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:48.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:48.675 | INFO | Uploaded 36GiB to GPU in 1.98873s (18.102GiB/s) | 20260506 21:27:48.675 | INFO | Downloaded 34GiB to GPU in 1.91739s (17.7324GiB/s) | 20260506 21:27:53.815 | INFO | Analyzing chunk frames = [4580352,4580864) | 20260506 21:27:53.815 | INFO | MemInfo: { | 20260506 21:27:53.815 | INFO | "gpu" : " 17 GB", | 20260506 21:27:53.815 | INFO | "huge" : "", | 20260506 21:27:53.815 | INFO | "huge_shared" : " 27 GB", | 20260506 21:27:53.815 | INFO | "malloc" : " 14 GB", | 20260506 21:27:53.815 | INFO | "pinned" : " 29 GB" | 20260506 21:27:53.815 | INFO | } | 20260506 21:27:53.815 | INFO | Perf report for the last 5.14056 seconds: | 20260506 21:27:53.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:27:53.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:53.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:53.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:53.815 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 21:27:53.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:53.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:53.815 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:27:53.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:53.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:53.816 | INFO | Uploaded 36GiB to GPU in 1.975s (18.2279GiB/s) | 20260506 21:27:53.816 | INFO | Downloaded 34GiB to GPU in 1.92264s (17.684GiB/s) | 20260506 21:27:58.952 | INFO | Analyzing chunk frames = [4580864,4581376) | 20260506 21:27:58.952 | INFO | MemInfo: { | 20260506 21:27:58.952 | INFO | "gpu" : " 17 GB", | 20260506 21:27:58.952 | INFO | "huge" : "", | 20260506 21:27:58.952 | INFO | "huge_shared" : " 27 GB", | 20260506 21:27:58.952 | INFO | "malloc" : " 14 GB", | 20260506 21:27:58.952 | INFO | "pinned" : " 29 GB" | 20260506 21:27:58.952 | INFO | } | 20260506 21:27:58.952 | INFO | Perf report for the last 5.13657 seconds: | 20260506 21:27:58.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:27:58.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:58.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:58.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:58.952 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:27:58.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:58.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:58.952 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:27:58.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:58.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:27:58.952 | INFO | Uploaded 36GiB to GPU in 1.95691s (18.3964GiB/s) | 20260506 21:27:58.952 | INFO | Downloaded 34GiB to GPU in 1.87339s (18.1489GiB/s) | 20260506 21:28:04.095 | INFO | Analyzing chunk frames = [4581376,4581888) | 20260506 21:28:04.095 | INFO | MemInfo: { | 20260506 21:28:04.095 | INFO | "gpu" : " 17 GB", | 20260506 21:28:04.095 | INFO | "huge" : "", | 20260506 21:28:04.095 | INFO | "huge_shared" : " 27 GB", | 20260506 21:28:04.095 | INFO | "malloc" : " 14 GB", | 20260506 21:28:04.095 | INFO | "pinned" : " 29 GB" | 20260506 21:28:04.095 | INFO | } | 20260506 21:28:04.095 | INFO | Perf report for the last 5.14321 seconds: | 20260506 21:28:04.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:28:04.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:04.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:04.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:04.095 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 21:28:04.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:04.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:04.095 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:28:04.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:04.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:04.095 | INFO | Uploaded 36GiB to GPU in 2.003s (17.973GiB/s) | 20260506 21:28:04.095 | INFO | Downloaded 34GiB to GPU in 1.9177s (17.7295GiB/s) | 20260506 21:28:09.245 | INFO | Analyzing chunk frames = [4581888,4582400) | 20260506 21:28:09.245 | INFO | MemInfo: { | 20260506 21:28:09.245 | INFO | "gpu" : " 17 GB", | 20260506 21:28:09.245 | INFO | "huge" : "", | 20260506 21:28:09.245 | INFO | "huge_shared" : " 27 GB", | 20260506 21:28:09.245 | INFO | "malloc" : " 14 GB", | 20260506 21:28:09.245 | INFO | "pinned" : " 29 GB" | 20260506 21:28:09.245 | INFO | } | 20260506 21:28:09.245 | INFO | Perf report for the last 5.15002 seconds: | 20260506 21:28:09.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:28:09.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:09.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:09.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:09.245 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 21:28:09.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:09.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:09.245 | INFO | PRE_HQ : 30.1%, 1.0, 15.7ms, 0 | 20260506 21:28:09.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:09.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:09.245 | INFO | Uploaded 36GiB to GPU in 2.00731s (17.9345GiB/s) | 20260506 21:28:09.245 | INFO | Downloaded 34GiB to GPU in 1.95372s (17.4027GiB/s) | 20260506 21:28:14.374 | INFO | Analyzing chunk frames = [4582400,4582912) | 20260506 21:28:14.374 | INFO | MemInfo: { | 20260506 21:28:14.374 | INFO | "gpu" : " 17 GB", | 20260506 21:28:14.374 | INFO | "huge" : "", | 20260506 21:28:14.374 | INFO | "huge_shared" : " 27 GB", | 20260506 21:28:14.374 | INFO | "malloc" : " 14 GB", | 20260506 21:28:14.374 | INFO | "pinned" : " 29 GB" | 20260506 21:28:14.374 | INFO | } | 20260506 21:28:14.375 | INFO | Perf report for the last 5.1293 seconds: | 20260506 21:28:14.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:28:14.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:14.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:14.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:14.375 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 21:28:14.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:14.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:14.375 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:28:14.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:14.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:14.375 | INFO | Uploaded 36GiB to GPU in 1.98298s (18.1545GiB/s) | 20260506 21:28:14.375 | INFO | Downloaded 34GiB to GPU in 1.99495s (17.043GiB/s) | 20260506 21:28:19.510 | INFO | Analyzing chunk frames = [4582912,4583424) | 20260506 21:28:19.510 | INFO | MemInfo: { | 20260506 21:28:19.510 | INFO | "gpu" : " 17 GB", | 20260506 21:28:19.510 | INFO | "huge" : "", | 20260506 21:28:19.510 | INFO | "huge_shared" : " 27 GB", | 20260506 21:28:19.510 | INFO | "malloc" : " 14 GB", | 20260506 21:28:19.510 | INFO | "pinned" : " 29 GB" | 20260506 21:28:19.510 | INFO | } | 20260506 21:28:19.510 | INFO | Perf report for the last 5.13581 seconds: | 20260506 21:28:19.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:28:19.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:19.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:19.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:19.510 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 21:28:19.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:19.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:19.510 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260506 21:28:19.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:19.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:19.511 | INFO | Uploaded 36GiB to GPU in 2.04367s (17.6154GiB/s) | 20260506 21:28:19.511 | INFO | Downloaded 34GiB to GPU in 1.99959s (17.0035GiB/s) | 20260506 21:28:19.625 | INFO | Starting a round of Baz IO | 20260506 21:28:22.789 | INFO | Populating RTMetrics report | 20260506 21:28:22.789 | INFO | RTMetrics report populated | 20260506 21:28:22.789 | INFO | Opening tmp RTMetrics json file | 20260506 21:28:22.832 | INFO | Serializing RTMetrics to json | 20260506 21:28:22.892 | INFO | Serialization done, swapping tmp file now... | 20260506 21:28:22.896 | INFO | RTMetrics done | 20260506 21:28:23.112 | INFO | Finished a round of Baz IO: Wrote 1.943 GiB in 3.487 seconds | 20260506 21:28:23.112 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.93% / 0.00% / 66.07% | 20260506 21:28:23.112 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.611266% / 0% / 5.96046e-05% | 20260506 21:28:23.112 | INFO | Baz IO overall: | 20260506 21:28:23.112 | INFO | BytesWritten : 2414844222680 | 20260506 21:28:23.112 | INFO | NumEvents : 1165932499746 | 20260506 21:28:24.650 | INFO | Analyzing chunk frames = [4583424,4583936) | 20260506 21:28:24.650 | INFO | MemInfo: { | 20260506 21:28:24.650 | INFO | "gpu" : " 17 GB", | 20260506 21:28:24.650 | INFO | "huge" : "", | 20260506 21:28:24.650 | INFO | "huge_shared" : " 27 GB", | 20260506 21:28:24.650 | INFO | "malloc" : " 14 GB", | 20260506 21:28:24.650 | INFO | "pinned" : " 29 GB" | 20260506 21:28:24.650 | INFO | } | 20260506 21:28:24.650 | INFO | Perf report for the last 5.13997 seconds: | 20260506 21:28:24.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:28:24.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:24.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:24.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:24.650 | INFO | ANALYSIS : 62.9%, 2.8, 93.4ms, 0 | 20260506 21:28:24.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:24.650 | INFO | RT_METRICS : 48.5%, 1.0, 25.4ms, 0 | 20260506 21:28:24.650 | INFO | PRE_HQ : 60.3%, 2.0, 61.7ms, 0 | 20260506 21:28:24.650 | INFO | BAZWRITER : 67.5%, 4.8, 166.8ms, 0 | 20260506 21:28:24.650 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 21:28:24.651 | INFO | Uploaded 36GiB to GPU in 2.07704s (17.3324GiB/s) | 20260506 21:28:24.651 | INFO | Downloaded 38GiB to GPU in 2.18182s (17.4166GiB/s) | 20260506 21:28:29.788 | INFO | Analyzing chunk frames = [4583936,4584448) | 20260506 21:28:29.788 | INFO | MemInfo: { | 20260506 21:28:29.788 | INFO | "gpu" : " 17 GB", | 20260506 21:28:29.788 | INFO | "huge" : "", | 20260506 21:28:29.788 | INFO | "huge_shared" : " 27 GB", | 20260506 21:28:29.788 | INFO | "malloc" : " 14 GB", | 20260506 21:28:29.788 | INFO | "pinned" : " 29 GB" | 20260506 21:28:29.788 | INFO | } | 20260506 21:28:29.789 | INFO | Perf report for the last 5.13818 seconds: | 20260506 21:28:29.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:28:29.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:29.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:29.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:29.789 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:28:29.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:29.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:29.789 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 21:28:29.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:29.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:29.789 | INFO | Uploaded 36GiB to GPU in 2.01721s (17.8464GiB/s) | 20260506 21:28:29.789 | INFO | Downloaded 34GiB to GPU in 1.92278s (17.6828GiB/s) | 20260506 21:28:34.926 | INFO | Analyzing chunk frames = [4584448,4584960) | 20260506 21:28:34.926 | INFO | MemInfo: { | 20260506 21:28:34.926 | INFO | "gpu" : " 17 GB", | 20260506 21:28:34.926 | INFO | "huge" : "", | 20260506 21:28:34.926 | INFO | "huge_shared" : " 27 GB", | 20260506 21:28:34.926 | INFO | "malloc" : " 14 GB", | 20260506 21:28:34.926 | INFO | "pinned" : " 29 GB" | 20260506 21:28:34.926 | INFO | } | 20260506 21:28:34.926 | INFO | Perf report for the last 5.13771 seconds: | 20260506 21:28:34.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:28:34.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:34.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:34.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:34.926 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 21:28:34.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:34.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:34.926 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:28:34.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:34.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:34.927 | INFO | Uploaded 36GiB to GPU in 1.98954s (18.0946GiB/s) | 20260506 21:28:34.927 | INFO | Downloaded 34GiB to GPU in 1.92029s (17.7057GiB/s) | 20260506 21:28:40.066 | INFO | Analyzing chunk frames = [4584960,4585472) | 20260506 21:28:40.066 | INFO | MemInfo: { | 20260506 21:28:40.066 | INFO | "gpu" : " 17 GB", | 20260506 21:28:40.066 | INFO | "huge" : "", | 20260506 21:28:40.066 | INFO | "huge_shared" : " 27 GB", | 20260506 21:28:40.066 | INFO | "malloc" : " 14 GB", | 20260506 21:28:40.066 | INFO | "pinned" : " 29 GB" | 20260506 21:28:40.066 | INFO | } | 20260506 21:28:40.066 | INFO | Perf report for the last 5.14018 seconds: | 20260506 21:28:40.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:28:40.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:40.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:40.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:40.066 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:28:40.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:40.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:40.067 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:28:40.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:40.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:40.067 | INFO | Uploaded 36GiB to GPU in 1.94277s (18.5303GiB/s) | 20260506 21:28:40.067 | INFO | Downloaded 34GiB to GPU in 1.86765s (18.2047GiB/s) | 20260506 21:28:45.208 | INFO | Analyzing chunk frames = [4585472,4585984) | 20260506 21:28:45.208 | INFO | MemInfo: { | 20260506 21:28:45.208 | INFO | "gpu" : " 17 GB", | 20260506 21:28:45.208 | INFO | "huge" : "", | 20260506 21:28:45.208 | INFO | "huge_shared" : " 27 GB", | 20260506 21:28:45.208 | INFO | "malloc" : " 14 GB", | 20260506 21:28:45.208 | INFO | "pinned" : " 29 GB" | 20260506 21:28:45.208 | INFO | } | 20260506 21:28:45.208 | INFO | Perf report for the last 5.14177 seconds: | 20260506 21:28:45.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:28:45.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:45.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:45.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:45.208 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 21:28:45.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:45.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:45.208 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:28:45.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:45.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:45.208 | INFO | Uploaded 36GiB to GPU in 2.02064s (17.8161GiB/s) | 20260506 21:28:45.208 | INFO | Downloaded 34GiB to GPU in 1.91395s (17.7643GiB/s) | 20260506 21:28:50.346 | INFO | Analyzing chunk frames = [4585984,4586496) | 20260506 21:28:50.346 | INFO | MemInfo: { | 20260506 21:28:50.346 | INFO | "gpu" : " 17 GB", | 20260506 21:28:50.347 | INFO | "huge" : "", | 20260506 21:28:50.347 | INFO | "huge_shared" : " 27 GB", | 20260506 21:28:50.347 | INFO | "malloc" : " 14 GB", | 20260506 21:28:50.347 | INFO | "pinned" : " 29 GB" | 20260506 21:28:50.347 | INFO | } | 20260506 21:28:50.347 | INFO | Perf report for the last 5.13839 seconds: | 20260506 21:28:50.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:28:50.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:50.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:50.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:50.347 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 21:28:50.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:50.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:50.347 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:28:50.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:50.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:50.347 | INFO | Uploaded 36GiB to GPU in 2.01455s (17.87GiB/s) | 20260506 21:28:50.347 | INFO | Downloaded 34GiB to GPU in 1.96068s (17.3409GiB/s) | 20260506 21:28:55.490 | INFO | Analyzing chunk frames = [4586496,4587008) | 20260506 21:28:55.490 | INFO | MemInfo: { | 20260506 21:28:55.490 | INFO | "gpu" : " 17 GB", | 20260506 21:28:55.490 | INFO | "huge" : "", | 20260506 21:28:55.490 | INFO | "huge_shared" : " 27 GB", | 20260506 21:28:55.490 | INFO | "malloc" : " 14 GB", | 20260506 21:28:55.490 | INFO | "pinned" : " 29 GB" | 20260506 21:28:55.490 | INFO | } | 20260506 21:28:55.490 | INFO | Perf report for the last 5.14344 seconds: | 20260506 21:28:55.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:28:55.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:55.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:55.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:55.490 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 21:28:55.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:55.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:55.490 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:28:55.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:55.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:28:55.490 | INFO | Uploaded 36GiB to GPU in 1.9693s (18.2807GiB/s) | 20260506 21:28:55.490 | INFO | Downloaded 34GiB to GPU in 2.00144s (16.9878GiB/s) | 20260506 21:29:00.627 | INFO | Analyzing chunk frames = [4587008,4587520) | 20260506 21:29:00.627 | INFO | MemInfo: { | 20260506 21:29:00.627 | INFO | "gpu" : " 17 GB", | 20260506 21:29:00.627 | INFO | "huge" : "", | 20260506 21:29:00.627 | INFO | "huge_shared" : " 27 GB", | 20260506 21:29:00.627 | INFO | "malloc" : " 14 GB", | 20260506 21:29:00.627 | INFO | "pinned" : " 29 GB" | 20260506 21:29:00.627 | INFO | } | 20260506 21:29:00.627 | INFO | Perf report for the last 5.13674 seconds: | 20260506 21:29:00.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:29:00.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:00.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:00.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:00.627 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:29:00.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:00.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:00.627 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 21:29:00.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:00.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:00.627 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:29:00.627 | INFO | Total mem usage: 18 GiB | 20260506 21:29:00.627 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:29:00.627 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.2%) | 20260506 21:29:00.627 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:29:00.627 | INFO | High Water Breakdown: | 20260506 21:29:00.627 | INFO | 61.8% : 9.83 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:29:00.627 | INFO | 38.1% : 6.06 GiB ( 90.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:29:00.627 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:29:00.627 | INFO | ------------------------------------------ | 20260506 21:29:00.627 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:29:00.627 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:29:00.627 | INFO | Total mem usage: 26.5 GiB | 20260506 21:29:00.627 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:29:00.627 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260506 21:29:00.627 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260506 21:29:00.627 | INFO | High Water Breakdown: | 20260506 21:29:00.627 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:29:00.627 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:29:00.627 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:29:00.627 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:29:00.627 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:29:00.627 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:29:00.627 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:29:00.627 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:29:00.627 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:29:00.627 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:29:00.627 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:29:00.627 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:29:00.627 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:29:00.627 | INFO | Uploaded 36GiB to GPU in 2.04539s (17.6006GiB/s) | 20260506 21:29:00.628 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:29:00.628 | INFO | ------------------------------------------ | 20260506 21:29:00.628 | INFO | Downloaded 34GiB to GPU in 2.00545s (16.9538GiB/s) | 20260506 21:29:00.628 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:29:00.628 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:29:00.628 | INFO | Total mem usage: 25.6 GiB | 20260506 21:29:00.628 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:29:00.628 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:29:00.628 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:29:00.628 | INFO | High Water Breakdown: | 20260506 21:29:00.628 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:29:00.628 | INFO | ------------------------------------------ | 20260506 21:29:00.628 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:29:00.628 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:29:00.628 | INFO | Total mem usage: 15.7 GiB | 20260506 21:29:00.628 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:29:00.628 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:29:00.628 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:29:00.628 | INFO | High Water Breakdown: | 20260506 21:29:00.628 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:29:00.628 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:29:00.628 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:29:00.628 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:29:00.628 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:29:00.628 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:29:00.628 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:29:00.628 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:29:00.628 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:29:00.628 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:29:00.628 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:29:00.628 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:29:00.628 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:29:00.628 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:29:00.628 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:29:00.628 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:29:00.628 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:29:00.628 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:29:00.628 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:29:00.628 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:29:00.628 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:29:00.628 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:29:00.628 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:29:00.628 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:29:00.628 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:29:00.628 | INFO | ------------------------------------------ | 20260506 21:29:00.628 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:29:00.628 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:29:00.628 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:29:00.628 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:29:00.628 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.61e+01, 1.35e+01 / 3.05e+00 | 20260506 21:29:00.628 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.00e-01, 9.52e-03 / 6.63e-03 | 20260506 21:29:00.628 | INFO | PerfReport: 11.64%, Baseline -- 3.62e+00 / 4.35e+00, 5.48e-01 / 3.46e-01 | 20260506 21:29:00.628 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.43e+00, 1.80e-01 / 1.27e-01 | 20260506 21:29:00.628 | INFO | PerfReport: 16.08%, DME -- 4.00e+01 / 4.52e+01, 4.36e+00 / 3.07e+00 | 20260506 21:29:00.628 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.42e+00 / 1.11e+00 | 20260506 21:29:00.628 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.20e+00 / 2.77e+00, 3.71e-01 / 1.84e-01 | 20260506 21:29:00.628 | INFO | PerfReport: 3.25%, Metrics -- 1.01e+00 / 1.37e+00, 2.66e-01 / 1.46e-01 | 20260506 21:29:00.628 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.48e-01 / 3.96e-01, 5.34e-02 / 2.72e-02 | 20260506 21:29:00.628 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:29:00.628 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:29:00.628 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:29:00.628 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 21:29:00.628 | INFO | PerfReport: 21.63%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 21:29:00.628 | INFO | PerfReport: 21.27%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 21:29:00.628 | INFO | PerfReport: 32.22%, Compute -- 3.11e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 21:29:00.628 | INFO | PerfReport: 1.30%, LockUpload -- 1.25e+00 / 0.00e+00, 4.78e+00 / 0.00e+00 | 20260506 21:29:00.628 | INFO | PerfReport: 1.19%, LockDownload -- 1.15e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 21:29:00.628 | INFO | PerfReport: 22.37%, LockCompute -- 2.16e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 21:29:00.628 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260506 21:29:00.743 | INFO | Starting a round of Baz IO | 20260506 21:29:03.896 | INFO | Populating RTMetrics report | 20260506 21:29:03.896 | INFO | RTMetrics report populated | 20260506 21:29:03.896 | INFO | Opening tmp RTMetrics json file | 20260506 21:29:03.932 | INFO | Serializing RTMetrics to json | 20260506 21:29:03.935 | INFO | Serialization done, swapping tmp file now... | 20260506 21:29:03.935 | INFO | RTMetrics done | 20260506 21:29:04.392 | INFO | Finished a round of Baz IO: Wrote 1.944 GiB in 3.649 seconds | 20260506 21:29:04.392 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.93% / 0.00% / 66.07% | 20260506 21:29:04.393 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.609056% / 3.97364e-06% / 8.34465e-05% | 20260506 21:29:04.393 | INFO | Baz IO overall: | 20260506 21:29:04.393 | INFO | BytesWritten : 2416931615130 | 20260506 21:29:04.393 | INFO | NumEvents : 1166930773407 | 20260506 21:29:05.763 | INFO | Analyzing chunk frames = [4587520,4588032) | 20260506 21:29:05.763 | INFO | MemInfo: { | 20260506 21:29:05.763 | INFO | "gpu" : " 17 GB", | 20260506 21:29:05.763 | INFO | "huge" : "", | 20260506 21:29:05.763 | INFO | "huge_shared" : " 27 GB", | 20260506 21:29:05.763 | INFO | "malloc" : " 14 GB", | 20260506 21:29:05.763 | INFO | "pinned" : " 29 GB" | 20260506 21:29:05.763 | INFO | } | 20260506 21:29:05.763 | INFO | Perf report for the last 5.1361 seconds: | 20260506 21:29:05.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:29:05.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:05.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:05.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:05.763 | INFO | ANALYSIS : 62.8%, 2.9, 93.9ms, 0 | 20260506 21:29:05.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:05.763 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 21:29:05.763 | INFO | PRE_HQ : 60.8%, 1.9, 60.3ms, 0 | 20260506 21:29:05.763 | INFO | BAZWRITER : 70.7%, 4.8, 175.7ms, 0 | 20260506 21:29:05.763 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 21:29:05.763 | INFO | Uploaded 36GiB to GPU in 2.02368s (17.7894GiB/s) | 20260506 21:29:05.763 | INFO | Downloaded 38GiB to GPU in 2.20848s (17.2064GiB/s) | 20260506 21:29:10.905 | INFO | Analyzing chunk frames = [4588032,4588544) | 20260506 21:29:10.905 | INFO | MemInfo: { | 20260506 21:29:10.905 | INFO | "gpu" : " 17 GB", | 20260506 21:29:10.905 | INFO | "huge" : "", | 20260506 21:29:10.905 | INFO | "huge_shared" : " 27 GB", | 20260506 21:29:10.905 | INFO | "malloc" : " 14 GB", | 20260506 21:29:10.905 | INFO | "pinned" : " 29 GB" | 20260506 21:29:10.905 | INFO | } | 20260506 21:29:10.905 | INFO | Perf report for the last 5.14255 seconds: | 20260506 21:29:10.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:29:10.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:10.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:10.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:10.906 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 21:29:10.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:10.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:10.906 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 21:29:10.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:10.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:10.906 | INFO | Uploaded 36GiB to GPU in 1.99698s (18.0272GiB/s) | 20260506 21:29:10.906 | INFO | Downloaded 34GiB to GPU in 1.93001s (17.6165GiB/s) | 20260506 21:29:16.041 | INFO | Analyzing chunk frames = [4588544,4589056) | 20260506 21:29:16.041 | INFO | MemInfo: { | 20260506 21:29:16.041 | INFO | "gpu" : " 17 GB", | 20260506 21:29:16.041 | INFO | "huge" : "", | 20260506 21:29:16.041 | INFO | "huge_shared" : " 27 GB", | 20260506 21:29:16.041 | INFO | "malloc" : " 14 GB", | 20260506 21:29:16.041 | INFO | "pinned" : " 29 GB" | 20260506 21:29:16.041 | INFO | } | 20260506 21:29:16.041 | INFO | Perf report for the last 5.13599 seconds: | 20260506 21:29:16.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:29:16.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:16.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:16.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:16.042 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 21:29:16.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:16.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:16.042 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:29:16.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:16.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:16.042 | INFO | Uploaded 36GiB to GPU in 1.95948s (18.3722GiB/s) | 20260506 21:29:16.042 | INFO | Downloaded 34GiB to GPU in 1.92109s (17.6983GiB/s) | 20260506 21:29:21.183 | INFO | Analyzing chunk frames = [4589056,4589568) | 20260506 21:29:21.183 | INFO | MemInfo: { | 20260506 21:29:21.183 | INFO | "gpu" : " 17 GB", | 20260506 21:29:21.183 | INFO | "huge" : "", | 20260506 21:29:21.183 | INFO | "huge_shared" : " 27 GB", | 20260506 21:29:21.183 | INFO | "malloc" : " 14 GB", | 20260506 21:29:21.183 | INFO | "pinned" : " 29 GB" | 20260506 21:29:21.183 | INFO | } | 20260506 21:29:21.183 | INFO | Perf report for the last 5.14145 seconds: | 20260506 21:29:21.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:29:21.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:21.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:21.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:21.183 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:29:21.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:21.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:21.183 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:29:21.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:21.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:21.183 | INFO | Uploaded 36GiB to GPU in 1.92877s (18.6647GiB/s) | 20260506 21:29:21.183 | INFO | Downloaded 34GiB to GPU in 1.87221s (18.1604GiB/s) | 20260506 21:29:26.328 | INFO | Analyzing chunk frames = [4589568,4590080) | 20260506 21:29:26.328 | INFO | MemInfo: { | 20260506 21:29:26.328 | INFO | "gpu" : " 17 GB", | 20260506 21:29:26.328 | INFO | "huge" : "", | 20260506 21:29:26.328 | INFO | "huge_shared" : " 27 GB", | 20260506 21:29:26.328 | INFO | "malloc" : " 14 GB", | 20260506 21:29:26.328 | INFO | "pinned" : " 29 GB" | 20260506 21:29:26.328 | INFO | } | 20260506 21:29:26.328 | INFO | Perf report for the last 5.1455 seconds: | 20260506 21:29:26.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:29:26.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:26.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:26.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:26.329 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260506 21:29:26.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:26.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:26.329 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:29:26.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:26.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:26.329 | INFO | Uploaded 36GiB to GPU in 1.98823s (18.1066GiB/s) | 20260506 21:29:26.329 | INFO | Downloaded 34GiB to GPU in 1.92024s (17.7061GiB/s) | 20260506 21:29:31.465 | INFO | Analyzing chunk frames = [4590080,4590592) | 20260506 21:29:31.465 | INFO | MemInfo: { | 20260506 21:29:31.465 | INFO | "gpu" : " 17 GB", | 20260506 21:29:31.465 | INFO | "huge" : "", | 20260506 21:29:31.465 | INFO | "huge_shared" : " 27 GB", | 20260506 21:29:31.465 | INFO | "malloc" : " 14 GB", | 20260506 21:29:31.465 | INFO | "pinned" : " 29 GB" | 20260506 21:29:31.465 | INFO | } | 20260506 21:29:31.465 | INFO | Perf report for the last 5.13622 seconds: | 20260506 21:29:31.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:29:31.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:31.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:31.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:31.465 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 21:29:31.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:31.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:31.465 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:29:31.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:31.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:31.465 | INFO | Uploaded 36GiB to GPU in 1.99195s (18.0727GiB/s) | 20260506 21:29:31.465 | INFO | Downloaded 34GiB to GPU in 1.95276s (17.4112GiB/s) | 20260506 21:29:36.606 | INFO | Analyzing chunk frames = [4590592,4591104) | 20260506 21:29:36.606 | INFO | MemInfo: { | 20260506 21:29:36.606 | INFO | "gpu" : " 17 GB", | 20260506 21:29:36.606 | INFO | "huge" : "", | 20260506 21:29:36.606 | INFO | "huge_shared" : " 27 GB", | 20260506 21:29:36.606 | INFO | "malloc" : " 14 GB", | 20260506 21:29:36.606 | INFO | "pinned" : " 29 GB" | 20260506 21:29:36.606 | INFO | } | 20260506 21:29:36.606 | INFO | Perf report for the last 5.14135 seconds: | 20260506 21:29:36.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:29:36.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:36.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:36.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:36.606 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 21:29:36.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:36.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:36.606 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:29:36.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:36.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:36.606 | INFO | Uploaded 36GiB to GPU in 1.97703s (18.2091GiB/s) | 20260506 21:29:36.606 | INFO | Downloaded 34GiB to GPU in 2.00639s (16.9458GiB/s) | 20260506 21:29:41.746 | INFO | Analyzing chunk frames = [4591104,4591616) | 20260506 21:29:41.746 | INFO | MemInfo: { | 20260506 21:29:41.746 | INFO | "gpu" : " 17 GB", | 20260506 21:29:41.746 | INFO | "huge" : "", | 20260506 21:29:41.746 | INFO | "huge_shared" : " 27 GB", | 20260506 21:29:41.746 | INFO | "malloc" : " 14 GB", | 20260506 21:29:41.746 | INFO | "pinned" : " 29 GB" | 20260506 21:29:41.746 | INFO | } | 20260506 21:29:41.746 | INFO | Perf report for the last 5.13964 seconds: | 20260506 21:29:41.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:29:41.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:41.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:41.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:41.746 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 21:29:41.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:41.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:41.746 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 21:29:41.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:41.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:41.746 | INFO | Uploaded 36GiB to GPU in 1.9772s (18.2075GiB/s) | 20260506 21:29:41.746 | INFO | Downloaded 34GiB to GPU in 1.9745s (17.2195GiB/s) | 20260506 21:29:41.844 | INFO | Starting a round of Baz IO | 20260506 21:29:44.985 | INFO | Populating RTMetrics report | 20260506 21:29:44.986 | INFO | RTMetrics report populated | 20260506 21:29:44.986 | INFO | Opening tmp RTMetrics json file | 20260506 21:29:45.168 | INFO | Serializing RTMetrics to json | 20260506 21:29:45.282 | INFO | Serialization done, swapping tmp file now... | 20260506 21:29:45.288 | INFO | RTMetrics done | 20260506 21:29:45.306 | INFO | Finished a round of Baz IO: Wrote 1.945 GiB in 3.461 seconds | 20260506 21:29:45.306 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.92% / 0.00% / 66.07% | 20260506 21:29:45.306 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.608853% / 3.97364e-06% / 3.97364e-05% | 20260506 21:29:45.306 | INFO | Baz IO overall: | 20260506 21:29:45.306 | INFO | BytesWritten : 2419020149167 | 20260506 21:29:45.306 | INFO | NumEvents : 1167930164444 | 20260506 21:29:46.880 | INFO | Analyzing chunk frames = [4591616,4592128) | 20260506 21:29:46.880 | INFO | MemInfo: { | 20260506 21:29:46.880 | INFO | "gpu" : " 17 GB", | 20260506 21:29:46.880 | INFO | "huge" : "", | 20260506 21:29:46.880 | INFO | "huge_shared" : " 27 GB", | 20260506 21:29:46.880 | INFO | "malloc" : " 14 GB", | 20260506 21:29:46.880 | INFO | "pinned" : " 29 GB" | 20260506 21:29:46.880 | INFO | } | 20260506 21:29:46.880 | INFO | Perf report for the last 5.13466 seconds: | 20260506 21:29:46.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:29:46.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:46.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:46.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:46.880 | INFO | ANALYSIS : 62.5%, 2.9, 95.2ms, 0 | 20260506 21:29:46.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:46.880 | INFO | RT_METRICS : 48.8%, 1.0, 25.6ms, 0 | 20260506 21:29:46.880 | INFO | PRE_HQ : 60.0%, 1.8, 57.1ms, 0 | 20260506 21:29:46.880 | INFO | BAZWRITER : 67.0%, 4.7, 160.2ms, 0 | 20260506 21:29:46.880 | INFO | BAZ_SAMPLING : 5.7%, 1.0, 2.9ms, 0 | 20260506 21:29:46.881 | INFO | Uploaded 36GiB to GPU in 2.06307s (17.4497GiB/s) | 20260506 21:29:46.881 | INFO | Downloaded 38GiB to GPU in 2.20254s (17.2528GiB/s) | 20260506 21:29:52.016 | INFO | Analyzing chunk frames = [4592128,4592640) | 20260506 21:29:52.016 | INFO | MemInfo: { | 20260506 21:29:52.016 | INFO | "gpu" : " 17 GB", | 20260506 21:29:52.016 | INFO | "huge" : "", | 20260506 21:29:52.016 | INFO | "huge_shared" : " 27 GB", | 20260506 21:29:52.016 | INFO | "malloc" : " 14 GB", | 20260506 21:29:52.016 | INFO | "pinned" : " 29 GB" | 20260506 21:29:52.016 | INFO | } | 20260506 21:29:52.016 | INFO | Perf report for the last 5.13544 seconds: | 20260506 21:29:52.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:29:52.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:52.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:52.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:52.016 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:29:52.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:52.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:52.016 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:29:52.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:52.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:52.016 | INFO | Uploaded 36GiB to GPU in 2.0149s (17.8669GiB/s) | 20260506 21:29:52.016 | INFO | Downloaded 34GiB to GPU in 1.92276s (17.6829GiB/s) | 20260506 21:29:57.154 | INFO | Analyzing chunk frames = [4592640,4593152) | 20260506 21:29:57.154 | INFO | MemInfo: { | 20260506 21:29:57.154 | INFO | "gpu" : " 17 GB", | 20260506 21:29:57.155 | INFO | "huge" : "", | 20260506 21:29:57.155 | INFO | "huge_shared" : " 27 GB", | 20260506 21:29:57.155 | INFO | "malloc" : " 14 GB", | 20260506 21:29:57.155 | INFO | "pinned" : " 29 GB" | 20260506 21:29:57.155 | INFO | } | 20260506 21:29:57.155 | INFO | Perf report for the last 5.13873 seconds: | 20260506 21:29:57.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:29:57.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:57.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:57.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:57.155 | INFO | ANALYSIS : 60.9%, 3.0, 94.3ms, 0 | 20260506 21:29:57.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:57.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:57.155 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:29:57.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:57.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:29:57.155 | INFO | Uploaded 36GiB to GPU in 2.00046s (17.9958GiB/s) | 20260506 21:29:57.155 | INFO | Downloaded 34GiB to GPU in 1.92008s (17.7076GiB/s) | 20260506 21:30:02.299 | INFO | Analyzing chunk frames = [4593152,4593664) | 20260506 21:30:02.299 | INFO | MemInfo: { | 20260506 21:30:02.299 | INFO | "gpu" : " 17 GB", | 20260506 21:30:02.299 | INFO | "huge" : "", | 20260506 21:30:02.299 | INFO | "huge_shared" : " 27 GB", | 20260506 21:30:02.299 | INFO | "malloc" : " 14 GB", | 20260506 21:30:02.299 | INFO | "pinned" : " 29 GB" | 20260506 21:30:02.299 | INFO | } | 20260506 21:30:02.300 | INFO | Perf report for the last 5.14492 seconds: | 20260506 21:30:02.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:30:02.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:02.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:02.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:02.300 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260506 21:30:02.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:02.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:02.300 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:30:02.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:02.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:02.300 | INFO | Uploaded 36GiB to GPU in 1.93253s (18.6284GiB/s) | 20260506 21:30:02.300 | INFO | Downloaded 34GiB to GPU in 1.86526s (18.228GiB/s) | 20260506 21:30:07.438 | INFO | Analyzing chunk frames = [4593664,4594176) | 20260506 21:30:07.438 | INFO | MemInfo: { | 20260506 21:30:07.438 | INFO | "gpu" : " 17 GB", | 20260506 21:30:07.438 | INFO | "huge" : "", | 20260506 21:30:07.438 | INFO | "huge_shared" : " 27 GB", | 20260506 21:30:07.438 | INFO | "malloc" : " 14 GB", | 20260506 21:30:07.438 | INFO | "pinned" : " 29 GB" | 20260506 21:30:07.438 | INFO | } | 20260506 21:30:07.438 | INFO | Perf report for the last 5.13855 seconds: | 20260506 21:30:07.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:30:07.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:07.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:07.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:07.438 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 21:30:07.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:07.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:07.438 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:30:07.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:07.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:07.438 | INFO | Uploaded 36GiB to GPU in 2.00799s (17.9284GiB/s) | 20260506 21:30:07.438 | INFO | Downloaded 34GiB to GPU in 1.91634s (17.7421GiB/s) | 20260506 21:30:12.582 | INFO | Analyzing chunk frames = [4594176,4594688) | 20260506 21:30:12.582 | INFO | MemInfo: { | 20260506 21:30:12.582 | INFO | "gpu" : " 17 GB", | 20260506 21:30:12.582 | INFO | "huge" : "", | 20260506 21:30:12.582 | INFO | "huge_shared" : " 27 GB", | 20260506 21:30:12.582 | INFO | "malloc" : " 14 GB", | 20260506 21:30:12.582 | INFO | "pinned" : " 29 GB" | 20260506 21:30:12.582 | INFO | } | 20260506 21:30:12.582 | INFO | Perf report for the last 5.14435 seconds: | 20260506 21:30:12.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:30:12.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:12.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:12.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:12.583 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 21:30:12.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:12.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:12.583 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:30:12.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:12.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:12.583 | INFO | Uploaded 36GiB to GPU in 2.01499s (17.8661GiB/s) | 20260506 21:30:12.583 | INFO | Downloaded 34GiB to GPU in 1.92779s (17.6368GiB/s) | 20260506 21:30:17.720 | INFO | Analyzing chunk frames = [4594688,4595200) | 20260506 21:30:17.720 | INFO | MemInfo: { | 20260506 21:30:17.720 | INFO | "gpu" : " 17 GB", | 20260506 21:30:17.720 | INFO | "huge" : "", | 20260506 21:30:17.721 | INFO | "huge_shared" : " 27 GB", | 20260506 21:30:17.721 | INFO | "malloc" : " 14 GB", | 20260506 21:30:17.721 | INFO | "pinned" : " 29 GB" | 20260506 21:30:17.721 | INFO | } | 20260506 21:30:17.721 | INFO | Perf report for the last 5.13825 seconds: | 20260506 21:30:17.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:30:17.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:17.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:17.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:17.721 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:30:17.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:17.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:17.721 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 21:30:17.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:17.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:17.721 | INFO | Uploaded 36GiB to GPU in 1.96867s (18.2864GiB/s) | 20260506 21:30:17.721 | INFO | Downloaded 34GiB to GPU in 1.99781s (17.0187GiB/s) | 20260506 21:30:22.857 | INFO | Analyzing chunk frames = [4595200,4595712) | 20260506 21:30:22.857 | INFO | MemInfo: { | 20260506 21:30:22.857 | INFO | "gpu" : " 17 GB", | 20260506 21:30:22.857 | INFO | "huge" : "", | 20260506 21:30:22.857 | INFO | "huge_shared" : " 27 GB", | 20260506 21:30:22.857 | INFO | "malloc" : " 14 GB", | 20260506 21:30:22.857 | INFO | "pinned" : " 29 GB" | 20260506 21:30:22.857 | INFO | } | 20260506 21:30:22.857 | INFO | Perf report for the last 5.13615 seconds: | 20260506 21:30:22.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:30:22.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:22.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:22.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:22.857 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 21:30:22.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:22.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:22.857 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:30:22.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:22.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:22.857 | INFO | Uploaded 36GiB to GPU in 1.98928s (18.097GiB/s) | 20260506 21:30:22.857 | INFO | Downloaded 34GiB to GPU in 1.98201s (17.1543GiB/s) | 20260506 21:30:22.857 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:30:22.857 | INFO | Total mem usage: 18 GiB | 20260506 21:30:22.857 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:30:22.857 | INFO | Recent High Water Marks Sum: 16.6 GiB (Savings: -8.73%) | 20260506 21:30:22.857 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:30:22.857 | INFO | High Water Breakdown: | 20260506 21:30:22.857 | INFO | 62.4% : 10.3 GiB ( 90.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:30:22.857 | INFO | 37.5% : 6.21 GiB ( 92.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:30:22.857 | INFO | 0.12% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:30:22.857 | INFO | ------------------------------------------ | 20260506 21:30:22.857 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:30:22.857 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:30:22.857 | INFO | Total mem usage: 26.5 GiB | 20260506 21:30:22.857 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:30:22.857 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:30:22.857 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:30:22.857 | INFO | High Water Breakdown: | 20260506 21:30:22.857 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:30:22.857 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:30:22.857 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:30:22.857 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:30:22.857 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:30:22.857 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:30:22.857 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:30:22.857 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:30:22.857 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:30:22.857 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:30:22.857 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:30:22.857 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:30:22.857 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:30:22.857 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:30:22.857 | INFO | ------------------------------------------ | 20260506 21:30:22.857 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:30:22.857 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:30:22.857 | INFO | Total mem usage: 25.6 GiB | 20260506 21:30:22.857 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:30:22.857 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 21:30:22.857 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 21:30:22.857 | INFO | High Water Breakdown: | 20260506 21:30:22.858 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 21:30:22.858 | INFO | ------------------------------------------ | 20260506 21:30:22.858 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:30:22.858 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:30:22.858 | INFO | Total mem usage: 15.7 GiB | 20260506 21:30:22.858 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:30:22.858 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:30:22.858 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:30:22.858 | INFO | High Water Breakdown: | 20260506 21:30:22.858 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:30:22.858 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:30:22.858 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:30:22.858 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:30:22.858 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:30:22.858 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:30:22.858 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:30:22.858 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:30:22.858 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:30:22.858 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:30:22.858 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:30:22.858 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:30:22.858 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:30:22.858 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:30:22.858 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:30:22.858 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:30:22.858 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:30:22.858 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:30:22.858 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:30:22.858 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:30:22.858 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:30:22.858 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:30:22.858 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:30:22.858 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:30:22.858 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:30:22.858 | INFO | ------------------------------------------ | 20260506 21:30:22.858 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:30:22.858 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:30:22.858 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:30:22.858 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:30:22.858 | INFO | PerfReport: Full Execution -- 3.12e+01 / 7.62e+01, 1.34e+01 / 2.92e+00 | 20260506 21:30:22.858 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 9.98e-02, 9.69e-03 / 5.38e-03 | 20260506 21:30:22.858 | INFO | PerfReport: 11.65%, Baseline -- 3.63e+00 / 4.38e+00, 5.58e-01 / 3.45e-01 | 20260506 21:30:22.858 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.45e+00, 1.84e-01 / 1.29e-01 | 20260506 21:30:22.858 | INFO | PerfReport: 16.07%, DME -- 4.01e+01 / 4.51e+01, 4.31e+00 / 3.06e+00 | 20260506 21:30:22.858 | INFO | PerfReport: 57.26%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.43e+00 / 1.98e+00 | 20260506 21:30:22.858 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.21e+00 / 2.70e+00, 3.72e-01 / 2.69e-01 | 20260506 21:30:22.858 | INFO | PerfReport: 3.26%, Metrics -- 1.02e+00 / 1.33e+00, 2.72e-01 / 1.73e-01 | 20260506 21:30:22.858 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.48e-01 / 4.00e-01, 5.40e-02 / 2.44e-02 | 20260506 21:30:22.858 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:30:22.858 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:30:22.858 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:30:22.858 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 21:30:22.858 | INFO | PerfReport: 21.48%, Upload -- 2.07e+01 / 0.00e+00, 6.88e+00 / 0.00e+00 | 20260506 21:30:22.858 | INFO | PerfReport: 21.26%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 21:30:22.858 | INFO | PerfReport: 32.29%, Compute -- 3.12e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 21:30:22.858 | INFO | PerfReport: 1.22%, LockUpload -- 1.18e+00 / 0.00e+00, 4.53e+00 / 0.00e+00 | 20260506 21:30:22.858 | INFO | PerfReport: 1.21%, LockDownload -- 1.16e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260506 21:30:22.859 | INFO | PerfReport: 22.53%, LockCompute -- 2.17e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 21:30:22.859 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260506 21:30:22.954 | INFO | Starting a round of Baz IO | 20260506 21:30:26.088 | INFO | Populating RTMetrics report | 20260506 21:30:26.088 | INFO | RTMetrics report populated | 20260506 21:30:26.088 | INFO | Opening tmp RTMetrics json file | 20260506 21:30:26.188 | INFO | Serializing RTMetrics to json | 20260506 21:30:26.193 | INFO | Serialization done, swapping tmp file now... | 20260506 21:30:26.193 | INFO | RTMetrics done | 20260506 21:30:26.316 | INFO | Finished a round of Baz IO: Wrote 1.943 GiB in 3.361 seconds | 20260506 21:30:26.316 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.92% / 0.00% / 66.08% | 20260506 21:30:26.316 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.605818% / 7.94729e-06% / 7.54992e-05% | 20260506 21:30:26.316 | INFO | Baz IO overall: | 20260506 21:30:26.316 | INFO | BytesWritten : 2421105989940 | 20260506 21:30:26.316 | INFO | NumEvents : 1168927633933 | 20260506 21:30:27.995 | INFO | Analyzing chunk frames = [4595712,4596224) | 20260506 21:30:27.995 | INFO | MemInfo: { | 20260506 21:30:27.995 | INFO | "gpu" : " 17 GB", | 20260506 21:30:27.995 | INFO | "huge" : "", | 20260506 21:30:27.995 | INFO | "huge_shared" : " 27 GB", | 20260506 21:30:27.995 | INFO | "malloc" : " 14 GB", | 20260506 21:30:27.995 | INFO | "pinned" : " 29 GB" | 20260506 21:30:27.995 | INFO | } | 20260506 21:30:27.995 | INFO | Perf report for the last 5.13816 seconds: | 20260506 21:30:27.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:30:27.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:27.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:27.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:27.995 | INFO | ANALYSIS : 62.3%, 2.9, 95.3ms, 0 | 20260506 21:30:27.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:27.995 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 21:30:27.995 | INFO | PRE_HQ : 59.7%, 1.8, 56.7ms, 0 | 20260506 21:30:27.995 | INFO | BAZWRITER : 65.1%, 4.8, 159.8ms, 0 | 20260506 21:30:27.995 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 21:30:27.995 | INFO | Uploaded 36GiB to GPU in 1.99605s (18.0356GiB/s) | 20260506 21:30:27.995 | INFO | Downloaded 38GiB to GPU in 2.1931s (17.3271GiB/s) | 20260506 21:30:33.130 | INFO | Analyzing chunk frames = [4596224,4596736) | 20260506 21:30:33.130 | INFO | MemInfo: { | 20260506 21:30:33.130 | INFO | "gpu" : " 17 GB", | 20260506 21:30:33.130 | INFO | "huge" : "", | 20260506 21:30:33.130 | INFO | "huge_shared" : " 27 GB", | 20260506 21:30:33.130 | INFO | "malloc" : " 14 GB", | 20260506 21:30:33.130 | INFO | "pinned" : " 29 GB" | 20260506 21:30:33.130 | INFO | } | 20260506 21:30:33.130 | INFO | Perf report for the last 5.13525 seconds: | 20260506 21:30:33.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:30:33.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:33.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:33.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:33.130 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 21:30:33.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:33.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:33.130 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:30:33.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:33.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:33.131 | INFO | Uploaded 36GiB to GPU in 1.99658s (18.0308GiB/s) | 20260506 21:30:33.131 | INFO | Downloaded 34GiB to GPU in 1.9287s (17.6285GiB/s) | 20260506 21:30:38.275 | INFO | Analyzing chunk frames = [4596736,4597248) | 20260506 21:30:38.275 | INFO | MemInfo: { | 20260506 21:30:38.275 | INFO | "gpu" : " 17 GB", | 20260506 21:30:38.275 | INFO | "huge" : "", | 20260506 21:30:38.275 | INFO | "huge_shared" : " 27 GB", | 20260506 21:30:38.275 | INFO | "malloc" : " 14 GB", | 20260506 21:30:38.275 | INFO | "pinned" : " 29 GB" | 20260506 21:30:38.275 | INFO | } | 20260506 21:30:38.275 | INFO | Perf report for the last 5.14492 seconds: | 20260506 21:30:38.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:30:38.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:38.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:38.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:38.275 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 21:30:38.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:38.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:38.275 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 21:30:38.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:38.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:38.275 | INFO | Uploaded 36GiB to GPU in 1.96423s (18.3278GiB/s) | 20260506 21:30:38.275 | INFO | Downloaded 34GiB to GPU in 1.92154s (17.6942GiB/s) | 20260506 21:30:43.419 | INFO | Analyzing chunk frames = [4597248,4597760) | 20260506 21:30:43.419 | INFO | MemInfo: { | 20260506 21:30:43.419 | INFO | "gpu" : " 17 GB", | 20260506 21:30:43.419 | INFO | "huge" : "", | 20260506 21:30:43.419 | INFO | "huge_shared" : " 27 GB", | 20260506 21:30:43.419 | INFO | "malloc" : " 14 GB", | 20260506 21:30:43.419 | INFO | "pinned" : " 29 GB" | 20260506 21:30:43.419 | INFO | } | 20260506 21:30:43.419 | INFO | Perf report for the last 5.14357 seconds: | 20260506 21:30:43.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:30:43.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:43.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:43.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:43.419 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 21:30:43.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:43.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:43.419 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:30:43.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:43.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:43.419 | INFO | Uploaded 36GiB to GPU in 1.94328s (18.5254GiB/s) | 20260506 21:30:43.419 | INFO | Downloaded 34GiB to GPU in 1.8763s (18.1207GiB/s) | 20260506 21:30:48.557 | INFO | Analyzing chunk frames = [4597760,4598272) | 20260506 21:30:48.557 | INFO | MemInfo: { | 20260506 21:30:48.557 | INFO | "gpu" : " 17 GB", | 20260506 21:30:48.557 | INFO | "huge" : "", | 20260506 21:30:48.557 | INFO | "huge_shared" : " 27 GB", | 20260506 21:30:48.557 | INFO | "malloc" : " 14 GB", | 20260506 21:30:48.557 | INFO | "pinned" : " 29 GB" | 20260506 21:30:48.557 | INFO | } | 20260506 21:30:48.557 | INFO | Perf report for the last 5.13866 seconds: | 20260506 21:30:48.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:30:48.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:48.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:48.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:48.558 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 21:30:48.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:48.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:48.558 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:30:48.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:48.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:48.558 | INFO | Uploaded 36GiB to GPU in 1.99601s (18.036GiB/s) | 20260506 21:30:48.558 | INFO | Downloaded 34GiB to GPU in 1.92682s (17.6457GiB/s) | 20260506 21:30:53.694 | INFO | Analyzing chunk frames = [4598272,4598784) | 20260506 21:30:53.694 | INFO | MemInfo: { | 20260506 21:30:53.694 | INFO | "gpu" : " 17 GB", | 20260506 21:30:53.694 | INFO | "huge" : "", | 20260506 21:30:53.694 | INFO | "huge_shared" : " 27 GB", | 20260506 21:30:53.694 | INFO | "malloc" : " 14 GB", | 20260506 21:30:53.694 | INFO | "pinned" : " 29 GB" | 20260506 21:30:53.694 | INFO | } | 20260506 21:30:53.694 | INFO | Perf report for the last 5.13661 seconds: | 20260506 21:30:53.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:30:53.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:53.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:53.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:53.694 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 21:30:53.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:53.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:53.694 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 21:30:53.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:53.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:53.694 | INFO | Uploaded 36GiB to GPU in 2.04128s (17.636GiB/s) | 20260506 21:30:53.694 | INFO | Downloaded 34GiB to GPU in 1.95781s (17.3663GiB/s) | 20260506 21:30:58.833 | INFO | Analyzing chunk frames = [4598784,4599296) | 20260506 21:30:58.833 | INFO | MemInfo: { | 20260506 21:30:58.833 | INFO | "gpu" : " 17 GB", | 20260506 21:30:58.833 | INFO | "huge" : "", | 20260506 21:30:58.833 | INFO | "huge_shared" : " 27 GB", | 20260506 21:30:58.833 | INFO | "malloc" : " 14 GB", | 20260506 21:30:58.833 | INFO | "pinned" : " 29 GB" | 20260506 21:30:58.833 | INFO | } | 20260506 21:30:58.833 | INFO | Perf report for the last 5.13905 seconds: | 20260506 21:30:58.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:30:58.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:58.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:58.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:58.833 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 21:30:58.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:58.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:58.833 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 21:30:58.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:58.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:30:58.833 | INFO | Uploaded 36GiB to GPU in 1.98437s (18.1417GiB/s) | 20260506 21:30:58.833 | INFO | Downloaded 34GiB to GPU in 2.01424s (16.8798GiB/s) | 20260506 21:31:03.969 | INFO | Analyzing chunk frames = [4599296,4599808) | 20260506 21:31:03.969 | INFO | MemInfo: { | 20260506 21:31:03.969 | INFO | "gpu" : " 17 GB", | 20260506 21:31:03.969 | INFO | "huge" : "", | 20260506 21:31:03.969 | INFO | "huge_shared" : " 27 GB", | 20260506 21:31:03.969 | INFO | "malloc" : " 14 GB", | 20260506 21:31:03.969 | INFO | "pinned" : " 29 GB" | 20260506 21:31:03.969 | INFO | } | 20260506 21:31:03.969 | INFO | Perf report for the last 5.13603 seconds: | 20260506 21:31:03.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:31:03.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:03.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:03.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:03.969 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 21:31:03.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:03.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:03.969 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:31:03.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:03.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:03.969 | INFO | Uploaded 36GiB to GPU in 2.00669s (17.94GiB/s) | 20260506 21:31:03.969 | INFO | Downloaded 34GiB to GPU in 1.99462s (17.0458GiB/s) | 20260506 21:31:04.073 | INFO | Starting a round of Baz IO | 20260506 21:31:07.200 | INFO | Populating RTMetrics report | 20260506 21:31:07.200 | INFO | RTMetrics report populated | 20260506 21:31:07.200 | INFO | Opening tmp RTMetrics json file | 20260506 21:31:07.214 | INFO | Serializing RTMetrics to json | 20260506 21:31:07.267 | INFO | Serialization done, swapping tmp file now... | 20260506 21:31:07.295 | INFO | RTMetrics done | 20260506 21:31:07.566 | INFO | Finished a round of Baz IO: Wrote 1.943 GiB in 3.492 seconds | 20260506 21:31:07.566 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.92% / 0.00% / 66.08% | 20260506 21:31:07.566 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.60341% / 7.94729e-06% / 2.78155e-05% | 20260506 21:31:07.566 | INFO | Baz IO overall: | 20260506 21:31:07.566 | INFO | BytesWritten : 2423191749469 | 20260506 21:31:07.566 | INFO | NumEvents : 1169925389762 | 20260506 21:31:09.104 | INFO | Analyzing chunk frames = [4599808,4600320) | 20260506 21:31:09.104 | INFO | MemInfo: { | 20260506 21:31:09.104 | INFO | "gpu" : " 17 GB", | 20260506 21:31:09.104 | INFO | "huge" : "", | 20260506 21:31:09.104 | INFO | "huge_shared" : " 27 GB", | 20260506 21:31:09.104 | INFO | "malloc" : " 14 GB", | 20260506 21:31:09.104 | INFO | "pinned" : " 29 GB" | 20260506 21:31:09.104 | INFO | } | 20260506 21:31:09.104 | INFO | Perf report for the last 5.13458 seconds: | 20260506 21:31:09.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:31:09.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:09.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:09.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:09.104 | INFO | ANALYSIS : 62.4%, 2.9, 95.2ms, 0 | 20260506 21:31:09.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:09.104 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 21:31:09.104 | INFO | PRE_HQ : 59.7%, 1.8, 56.7ms, 0 | 20260506 21:31:09.104 | INFO | BAZWRITER : 67.6%, 4.8, 167.8ms, 0 | 20260506 21:31:09.104 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 21:31:09.104 | INFO | Uploaded 36GiB to GPU in 2.03328s (17.7053GiB/s) | 20260506 21:31:09.104 | INFO | Downloaded 38GiB to GPU in 2.19682s (17.2978GiB/s) | 20260506 21:31:14.249 | INFO | Analyzing chunk frames = [4600320,4600832) | 20260506 21:31:14.249 | INFO | MemInfo: { | 20260506 21:31:14.249 | INFO | "gpu" : " 17 GB", | 20260506 21:31:14.249 | INFO | "huge" : "", | 20260506 21:31:14.249 | INFO | "huge_shared" : " 27 GB", | 20260506 21:31:14.249 | INFO | "malloc" : " 14 GB", | 20260506 21:31:14.249 | INFO | "pinned" : " 29 GB" | 20260506 21:31:14.249 | INFO | } | 20260506 21:31:14.249 | INFO | Perf report for the last 5.14516 seconds: | 20260506 21:31:14.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:31:14.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:14.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:14.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:14.249 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 21:31:14.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:14.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:14.249 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:31:14.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:14.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:14.249 | INFO | Uploaded 36GiB to GPU in 2.00983s (17.912GiB/s) | 20260506 21:31:14.249 | INFO | Downloaded 34GiB to GPU in 1.93378s (17.5821GiB/s) | 20260506 21:31:19.391 | INFO | Analyzing chunk frames = [4600832,4601344) | 20260506 21:31:19.391 | INFO | MemInfo: { | 20260506 21:31:19.391 | INFO | "gpu" : " 17 GB", | 20260506 21:31:19.391 | INFO | "huge" : "", | 20260506 21:31:19.391 | INFO | "huge_shared" : " 27 GB", | 20260506 21:31:19.391 | INFO | "malloc" : " 14 GB", | 20260506 21:31:19.391 | INFO | "pinned" : " 29 GB" | 20260506 21:31:19.391 | INFO | } | 20260506 21:31:19.391 | INFO | Perf report for the last 5.1422 seconds: | 20260506 21:31:19.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:31:19.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:19.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:19.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:19.391 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 21:31:19.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:19.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:19.391 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:31:19.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:19.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:19.391 | INFO | Uploaded 36GiB to GPU in 1.97584s (18.2201GiB/s) | 20260506 21:31:19.392 | INFO | Downloaded 34GiB to GPU in 1.92124s (17.6969GiB/s) | 20260506 21:31:24.533 | INFO | Analyzing chunk frames = [4601344,4601856) | 20260506 21:31:24.533 | INFO | MemInfo: { | 20260506 21:31:24.533 | INFO | "gpu" : " 17 GB", | 20260506 21:31:24.533 | INFO | "huge" : "", | 20260506 21:31:24.533 | INFO | "huge_shared" : " 27 GB", | 20260506 21:31:24.533 | INFO | "malloc" : " 14 GB", | 20260506 21:31:24.533 | INFO | "pinned" : " 29 GB" | 20260506 21:31:24.533 | INFO | } | 20260506 21:31:24.533 | INFO | Perf report for the last 5.14162 seconds: | 20260506 21:31:24.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:31:24.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:24.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:24.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:24.533 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:31:24.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:24.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:24.533 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:31:24.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:24.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:24.533 | INFO | Uploaded 36GiB to GPU in 1.94814s (18.4792GiB/s) | 20260506 21:31:24.533 | INFO | Downloaded 34GiB to GPU in 1.87702s (18.1138GiB/s) | 20260506 21:31:29.675 | INFO | Analyzing chunk frames = [4601856,4602368) | 20260506 21:31:29.675 | INFO | MemInfo: { | 20260506 21:31:29.675 | INFO | "gpu" : " 17 GB", | 20260506 21:31:29.675 | INFO | "huge" : "", | 20260506 21:31:29.675 | INFO | "huge_shared" : " 27 GB", | 20260506 21:31:29.675 | INFO | "malloc" : " 14 GB", | 20260506 21:31:29.675 | INFO | "pinned" : " 29 GB" | 20260506 21:31:29.675 | INFO | } | 20260506 21:31:29.675 | INFO | Perf report for the last 5.14201 seconds: | 20260506 21:31:29.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:31:29.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:29.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:29.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:29.675 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 21:31:29.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:29.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:29.675 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:31:29.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:29.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:29.675 | INFO | Uploaded 36GiB to GPU in 1.97686s (18.2107GiB/s) | 20260506 21:31:29.675 | INFO | Downloaded 34GiB to GPU in 1.93167s (17.6014GiB/s) | 20260506 21:31:34.810 | INFO | Analyzing chunk frames = [4602368,4602880) | 20260506 21:31:34.810 | INFO | MemInfo: { | 20260506 21:31:34.810 | INFO | "gpu" : " 17 GB", | 20260506 21:31:34.810 | INFO | "huge" : "", | 20260506 21:31:34.810 | INFO | "huge_shared" : " 27 GB", | 20260506 21:31:34.810 | INFO | "malloc" : " 14 GB", | 20260506 21:31:34.810 | INFO | "pinned" : " 29 GB" | 20260506 21:31:34.810 | INFO | } | 20260506 21:31:34.810 | INFO | Perf report for the last 5.13501 seconds: | 20260506 21:31:34.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:31:34.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:34.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:34.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:34.810 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 21:31:34.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:34.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:34.810 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 21:31:34.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:34.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:34.810 | INFO | Uploaded 36GiB to GPU in 2.00086s (17.9923GiB/s) | 20260506 21:31:34.810 | INFO | Downloaded 34GiB to GPU in 1.94219s (17.506GiB/s) | 20260506 21:31:39.945 | INFO | Analyzing chunk frames = [4602880,4603392) | 20260506 21:31:39.945 | INFO | MemInfo: { | 20260506 21:31:39.945 | INFO | "gpu" : " 17 GB", | 20260506 21:31:39.945 | INFO | "huge" : "", | 20260506 21:31:39.945 | INFO | "huge_shared" : " 27 GB", | 20260506 21:31:39.945 | INFO | "malloc" : " 14 GB", | 20260506 21:31:39.945 | INFO | "pinned" : " 29 GB" | 20260506 21:31:39.945 | INFO | } | 20260506 21:31:39.946 | INFO | Perf report for the last 5.13563 seconds: | 20260506 21:31:39.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:31:39.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:39.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:39.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:39.946 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 21:31:39.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:39.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:39.946 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 21:31:39.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:39.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:39.946 | INFO | Uploaded 36GiB to GPU in 1.95308s (18.4325GiB/s) | 20260506 21:31:39.946 | INFO | Downloaded 34GiB to GPU in 1.9935s (17.0554GiB/s) | 20260506 21:31:45.082 | INFO | Analyzing chunk frames = [4603392,4603904) | 20260506 21:31:45.082 | INFO | MemInfo: { | 20260506 21:31:45.082 | INFO | "gpu" : " 17 GB", | 20260506 21:31:45.082 | INFO | "huge" : "", | 20260506 21:31:45.082 | INFO | "huge_shared" : " 27 GB", | 20260506 21:31:45.082 | INFO | "malloc" : " 14 GB", | 20260506 21:31:45.082 | INFO | "pinned" : " 29 GB" | 20260506 21:31:45.082 | INFO | } | 20260506 21:31:45.082 | INFO | Perf report for the last 5.1366 seconds: | 20260506 21:31:45.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:31:45.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:45.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:45.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:45.082 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 21:31:45.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:45.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:45.082 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:31:45.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:45.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:45.082 | INFO | Uploaded 36GiB to GPU in 2.00404s (17.9637GiB/s) | 20260506 21:31:45.082 | INFO | Downloaded 34GiB to GPU in 1.98141s (17.1595GiB/s) | 20260506 21:31:45.083 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:31:45.083 | INFO | Total mem usage: 18 GiB | 20260506 21:31:45.083 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:31:45.083 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -11.4%) | 20260506 21:31:45.083 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:31:45.083 | INFO | High Water Breakdown: | 20260506 21:31:45.083 | INFO | 62.1% : 10 GiB ( 87.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:31:45.083 | INFO | 37.8% : 6.11 GiB ( 91.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:31:45.083 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:31:45.083 | INFO | ------------------------------------------ | 20260506 21:31:45.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:31:45.083 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:31:45.083 | INFO | Total mem usage: 26.5 GiB | 20260506 21:31:45.083 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:31:45.083 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:31:45.083 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:31:45.083 | INFO | High Water Breakdown: | 20260506 21:31:45.083 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:31:45.083 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:31:45.083 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:31:45.083 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:31:45.083 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:31:45.083 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:31:45.083 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:31:45.083 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:31:45.083 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:31:45.083 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:31:45.083 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:31:45.083 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:31:45.083 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:31:45.083 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:31:45.083 | INFO | ------------------------------------------ | 20260506 21:31:45.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:31:45.083 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:31:45.083 | INFO | Total mem usage: 25.6 GiB | 20260506 21:31:45.083 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:31:45.083 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:31:45.083 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:31:45.083 | INFO | High Water Breakdown: | 20260506 21:31:45.083 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:31:45.083 | INFO | ------------------------------------------ | 20260506 21:31:45.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:31:45.083 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:31:45.083 | INFO | Total mem usage: 15.7 GiB | 20260506 21:31:45.083 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:31:45.083 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:31:45.083 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:31:45.083 | INFO | High Water Breakdown: | 20260506 21:31:45.083 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:31:45.083 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:31:45.083 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:31:45.083 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:31:45.083 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:31:45.083 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:31:45.083 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:31:45.083 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:31:45.083 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:31:45.083 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:31:45.083 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:31:45.083 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:31:45.083 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:31:45.083 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:31:45.083 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:31:45.083 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:31:45.083 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:31:45.083 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:31:45.083 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:31:45.083 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:31:45.083 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:31:45.083 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:31:45.083 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:31:45.083 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:31:45.083 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:31:45.083 | INFO | ------------------------------------------ | 20260506 21:31:45.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:31:45.083 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:31:45.083 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:31:45.083 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:31:45.083 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.58e+01, 1.36e+01 / 2.48e+00 | 20260506 21:31:45.083 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 1.06e-02 / 6.29e-03 | 20260506 21:31:45.083 | INFO | PerfReport: 11.66%, Baseline -- 3.63e+00 / 4.39e+00, 5.57e-01 / 3.62e-01 | 20260506 21:31:45.083 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.45e+00, 1.81e-01 / 1.37e-01 | 20260506 21:31:45.083 | INFO | PerfReport: 16.04%, DME -- 3.99e+01 / 4.47e+01, 4.18e+00 / 2.95e+00 | 20260506 21:31:45.083 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.36e+00 / 2.12e+00 | 20260506 21:31:45.083 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.22e+00 / 2.76e+00, 3.82e-01 / 2.51e-01 | 20260506 21:31:45.083 | INFO | PerfReport: 3.27%, Metrics -- 1.02e+00 / 1.35e+00, 2.67e-01 / 1.84e-01 | 20260506 21:31:45.083 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.48e-01 / 3.98e-01, 5.37e-02 / 2.52e-02 | 20260506 21:31:45.083 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:31:45.083 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:31:45.083 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:31:45.083 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 21:31:45.083 | INFO | PerfReport: 21.44%, Upload -- 2.07e+01 / 0.00e+00, 6.84e+00 / 0.00e+00 | 20260506 21:31:45.083 | INFO | PerfReport: 21.28%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 21:31:45.084 | INFO | PerfReport: 32.19%, Compute -- 3.11e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 21:31:45.084 | INFO | PerfReport: 1.26%, LockUpload -- 1.21e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260506 21:31:45.084 | INFO | PerfReport: 1.26%, LockDownload -- 1.22e+00 / 0.00e+00, 3.10e+00 / 0.00e+00 | 20260506 21:31:45.084 | INFO | PerfReport: 22.57%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:31:45.084 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260506 21:31:45.181 | INFO | Starting a round of Baz IO | 20260506 21:31:48.315 | INFO | Populating RTMetrics report | 20260506 21:31:48.315 | INFO | RTMetrics report populated | 20260506 21:31:48.315 | INFO | Opening tmp RTMetrics json file | 20260506 21:31:48.356 | INFO | Serializing RTMetrics to json | 20260506 21:31:48.387 | INFO | Serialization done, swapping tmp file now... | 20260506 21:31:48.398 | INFO | RTMetrics done | 20260506 21:31:48.778 | INFO | Finished a round of Baz IO: Wrote 1.942 GiB in 3.596 seconds | 20260506 21:31:48.778 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.91% / 0.00% / 66.08% | 20260506 21:31:48.778 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.603052% / 1.19209e-05% / 5.96046e-05% | 20260506 21:31:48.778 | INFO | Baz IO overall: | 20260506 21:31:48.778 | INFO | BytesWritten : 2425277374080 | 20260506 21:31:48.778 | INFO | NumEvents : 1170923350325 | 20260506 21:31:50.224 | INFO | Analyzing chunk frames = [4603904,4604416) | 20260506 21:31:50.224 | INFO | MemInfo: { | 20260506 21:31:50.224 | INFO | "gpu" : " 17 GB", | 20260506 21:31:50.224 | INFO | "huge" : "", | 20260506 21:31:50.224 | INFO | "huge_shared" : " 27 GB", | 20260506 21:31:50.224 | INFO | "malloc" : " 14 GB", | 20260506 21:31:50.224 | INFO | "pinned" : " 29 GB" | 20260506 21:31:50.224 | INFO | } | 20260506 21:31:50.224 | INFO | Perf report for the last 5.14151 seconds: | 20260506 21:31:50.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:31:50.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:50.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:50.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:50.224 | INFO | ANALYSIS : 62.3%, 2.9, 95.2ms, 0 | 20260506 21:31:50.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:50.224 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 21:31:50.224 | INFO | PRE_HQ : 59.9%, 1.8, 56.6ms, 0 | 20260506 21:31:50.224 | INFO | BAZWRITER : 69.6%, 4.8, 171.2ms, 0 | 20260506 21:31:50.224 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.8ms, 0 | 20260506 21:31:50.224 | INFO | Uploaded 36GiB to GPU in 2.04708s (17.586GiB/s) | 20260506 21:31:50.224 | INFO | Downloaded 38GiB to GPU in 2.19046s (17.348GiB/s) | 20260506 21:31:55.371 | INFO | Analyzing chunk frames = [4604416,4604928) | 20260506 21:31:55.371 | INFO | MemInfo: { | 20260506 21:31:55.371 | INFO | "gpu" : " 17 GB", | 20260506 21:31:55.371 | INFO | "huge" : "", | 20260506 21:31:55.371 | INFO | "huge_shared" : " 27 GB", | 20260506 21:31:55.371 | INFO | "malloc" : " 14 GB", | 20260506 21:31:55.371 | INFO | "pinned" : " 29 GB" | 20260506 21:31:55.371 | INFO | } | 20260506 21:31:55.371 | INFO | Perf report for the last 5.14735 seconds: | 20260506 21:31:55.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:31:55.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:55.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:55.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:55.371 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 21:31:55.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:55.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:55.371 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 21:31:55.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:55.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:31:55.371 | INFO | Uploaded 36GiB to GPU in 2.02959s (17.7376GiB/s) | 20260506 21:31:55.371 | INFO | Downloaded 34GiB to GPU in 1.92653s (17.6483GiB/s) | 20260506 21:32:00.508 | INFO | Analyzing chunk frames = [4604928,4605440) | 20260506 21:32:00.508 | INFO | MemInfo: { | 20260506 21:32:00.508 | INFO | "gpu" : " 17 GB", | 20260506 21:32:00.508 | INFO | "huge" : "", | 20260506 21:32:00.508 | INFO | "huge_shared" : " 27 GB", | 20260506 21:32:00.508 | INFO | "malloc" : " 14 GB", | 20260506 21:32:00.508 | INFO | "pinned" : " 29 GB" | 20260506 21:32:00.508 | INFO | } | 20260506 21:32:00.508 | INFO | Perf report for the last 5.13699 seconds: | 20260506 21:32:00.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:32:00.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:00.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:00.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:00.508 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 21:32:00.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:00.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:00.508 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:32:00.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:00.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:00.508 | INFO | Uploaded 36GiB to GPU in 1.98598s (18.1271GiB/s) | 20260506 21:32:00.508 | INFO | Downloaded 34GiB to GPU in 1.92525s (17.6601GiB/s) | 20260506 21:32:05.647 | INFO | Analyzing chunk frames = [4605440,4605952) | 20260506 21:32:05.647 | INFO | MemInfo: { | 20260506 21:32:05.647 | INFO | "gpu" : " 17 GB", | 20260506 21:32:05.647 | INFO | "huge" : "", | 20260506 21:32:05.647 | INFO | "huge_shared" : " 27 GB", | 20260506 21:32:05.647 | INFO | "malloc" : " 14 GB", | 20260506 21:32:05.647 | INFO | "pinned" : " 29 GB" | 20260506 21:32:05.647 | INFO | } | 20260506 21:32:05.647 | INFO | Perf report for the last 5.13881 seconds: | 20260506 21:32:05.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:32:05.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:05.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:05.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:05.647 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 21:32:05.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:05.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:05.647 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:32:05.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:05.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:05.647 | INFO | Uploaded 36GiB to GPU in 2.02132s (17.8102GiB/s) | 20260506 21:32:05.647 | INFO | Downloaded 34GiB to GPU in 1.90586s (17.8397GiB/s) | 20260506 21:32:10.779 | INFO | Analyzing chunk frames = [4605952,4606464) | 20260506 21:32:10.779 | INFO | MemInfo: { | 20260506 21:32:10.779 | INFO | "gpu" : " 17 GB", | 20260506 21:32:10.779 | INFO | "huge" : "", | 20260506 21:32:10.779 | INFO | "huge_shared" : " 27 GB", | 20260506 21:32:10.779 | INFO | "malloc" : " 14 GB", | 20260506 21:32:10.779 | INFO | "pinned" : " 29 GB" | 20260506 21:32:10.779 | INFO | } | 20260506 21:32:10.779 | INFO | Perf report for the last 5.13226 seconds: | 20260506 21:32:10.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:32:10.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:10.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:10.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:10.779 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 21:32:10.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:10.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:10.779 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:32:10.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:10.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:10.779 | INFO | Uploaded 36GiB to GPU in 1.99968s (18.0028GiB/s) | 20260506 21:32:10.779 | INFO | Downloaded 34GiB to GPU in 1.93224s (17.5962GiB/s) | 20260506 21:32:15.917 | INFO | Analyzing chunk frames = [4606464,4606976) | 20260506 21:32:15.917 | INFO | MemInfo: { | 20260506 21:32:15.917 | INFO | "gpu" : " 17 GB", | 20260506 21:32:15.917 | INFO | "huge" : "", | 20260506 21:32:15.917 | INFO | "huge_shared" : " 27 GB", | 20260506 21:32:15.917 | INFO | "malloc" : " 14 GB", | 20260506 21:32:15.917 | INFO | "pinned" : " 29 GB" | 20260506 21:32:15.917 | INFO | } | 20260506 21:32:15.917 | INFO | Perf report for the last 5.13776 seconds: | 20260506 21:32:15.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:32:15.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:15.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:15.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:15.917 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 21:32:15.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:15.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:15.917 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 21:32:15.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:15.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:15.917 | INFO | Uploaded 36GiB to GPU in 2.00145s (17.9869GiB/s) | 20260506 21:32:15.917 | INFO | Downloaded 34GiB to GPU in 1.94033s (17.5228GiB/s) | 20260506 21:32:21.057 | INFO | Analyzing chunk frames = [4606976,4607488) | 20260506 21:32:21.057 | INFO | MemInfo: { | 20260506 21:32:21.057 | INFO | "gpu" : " 17 GB", | 20260506 21:32:21.057 | INFO | "huge" : "", | 20260506 21:32:21.057 | INFO | "huge_shared" : " 27 GB", | 20260506 21:32:21.057 | INFO | "malloc" : " 14 GB", | 20260506 21:32:21.057 | INFO | "pinned" : " 29 GB" | 20260506 21:32:21.057 | INFO | } | 20260506 21:32:21.057 | INFO | Perf report for the last 5.1403 seconds: | 20260506 21:32:21.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:32:21.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:21.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:21.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:21.057 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 21:32:21.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:21.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:21.057 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 21:32:21.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:21.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:21.057 | INFO | Uploaded 36GiB to GPU in 1.99362s (18.0576GiB/s) | 20260506 21:32:21.058 | INFO | Downloaded 34GiB to GPU in 2.01059s (16.9105GiB/s) | 20260506 21:32:26.197 | INFO | Analyzing chunk frames = [4607488,4608000) | 20260506 21:32:26.197 | INFO | MemInfo: { | 20260506 21:32:26.197 | INFO | "gpu" : " 17 GB", | 20260506 21:32:26.197 | INFO | "huge" : "", | 20260506 21:32:26.197 | INFO | "huge_shared" : " 27 GB", | 20260506 21:32:26.197 | INFO | "malloc" : " 14 GB", | 20260506 21:32:26.197 | INFO | "pinned" : " 29 GB" | 20260506 21:32:26.197 | INFO | } | 20260506 21:32:26.197 | INFO | Perf report for the last 5.13966 seconds: | 20260506 21:32:26.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:32:26.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:26.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:26.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:26.197 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 21:32:26.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:26.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:26.197 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 21:32:26.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:26.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:26.197 | INFO | Uploaded 36GiB to GPU in 2.01998s (17.8219GiB/s) | 20260506 21:32:26.197 | INFO | Downloaded 34GiB to GPU in 1.99499s (17.0427GiB/s) | 20260506 21:32:26.306 | INFO | Starting a round of Baz IO | 20260506 21:32:29.437 | INFO | Populating RTMetrics report | 20260506 21:32:29.437 | INFO | RTMetrics report populated | 20260506 21:32:29.437 | INFO | Opening tmp RTMetrics json file | 20260506 21:32:29.442 | INFO | Serializing RTMetrics to json | 20260506 21:32:29.466 | INFO | Serialization done, swapping tmp file now... | 20260506 21:32:29.468 | INFO | RTMetrics done | 20260506 21:32:29.594 | INFO | Finished a round of Baz IO: Wrote 1.944 GiB in 3.288 seconds | 20260506 21:32:29.594 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.91% / 0.00% / 66.09% | 20260506 21:32:29.594 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.60308% / 0% / 4.37101e-05% | 20260506 21:32:29.594 | INFO | Baz IO overall: | 20260506 21:32:29.594 | INFO | BytesWritten : 2427365070719 | 20260506 21:32:29.594 | INFO | NumEvents : 1171923157222 | 20260506 21:32:31.340 | INFO | Analyzing chunk frames = [4608000,4608512) | 20260506 21:32:31.340 | INFO | MemInfo: { | 20260506 21:32:31.340 | INFO | "gpu" : " 17 GB", | 20260506 21:32:31.340 | INFO | "huge" : "", | 20260506 21:32:31.340 | INFO | "huge_shared" : " 27 GB", | 20260506 21:32:31.340 | INFO | "malloc" : " 14 GB", | 20260506 21:32:31.340 | INFO | "pinned" : " 29 GB" | 20260506 21:32:31.340 | INFO | } | 20260506 21:32:31.340 | INFO | Perf report for the last 5.14344 seconds: | 20260506 21:32:31.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:32:31.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:31.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:31.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:31.340 | INFO | ANALYSIS : 62.4%, 2.9, 95.4ms, 0 | 20260506 21:32:31.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:31.340 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260506 21:32:31.340 | INFO | PRE_HQ : 59.6%, 1.9, 57.4ms, 0 | 20260506 21:32:31.340 | INFO | BAZWRITER : 63.6%, 4.4, 144.6ms, 0 | 20260506 21:32:31.340 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 21:32:31.341 | INFO | Uploaded 36GiB to GPU in 2.03008s (17.7333GiB/s) | 20260506 21:32:31.341 | INFO | Downloaded 38GiB to GPU in 2.20824s (17.2083GiB/s) | 20260506 21:32:36.485 | INFO | Analyzing chunk frames = [4608512,4609024) | 20260506 21:32:36.485 | INFO | MemInfo: { | 20260506 21:32:36.485 | INFO | "gpu" : " 17 GB", | 20260506 21:32:36.485 | INFO | "huge" : "", | 20260506 21:32:36.485 | INFO | "huge_shared" : " 27 GB", | 20260506 21:32:36.485 | INFO | "malloc" : " 14 GB", | 20260506 21:32:36.485 | INFO | "pinned" : " 29 GB" | 20260506 21:32:36.485 | INFO | } | 20260506 21:32:36.486 | INFO | Perf report for the last 5.14515 seconds: | 20260506 21:32:36.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:32:36.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:36.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:36.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:36.486 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 21:32:36.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:36.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:36.486 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 21:32:36.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:36.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:36.486 | INFO | Uploaded 36GiB to GPU in 2.00422s (17.9621GiB/s) | 20260506 21:32:36.486 | INFO | Downloaded 34GiB to GPU in 1.9312s (17.6057GiB/s) | 20260506 21:32:41.625 | INFO | Analyzing chunk frames = [4609024,4609536) | 20260506 21:32:41.625 | INFO | MemInfo: { | 20260506 21:32:41.625 | INFO | "gpu" : " 17 GB", | 20260506 21:32:41.625 | INFO | "huge" : "", | 20260506 21:32:41.625 | INFO | "huge_shared" : " 27 GB", | 20260506 21:32:41.625 | INFO | "malloc" : " 14 GB", | 20260506 21:32:41.625 | INFO | "pinned" : " 29 GB" | 20260506 21:32:41.625 | INFO | } | 20260506 21:32:41.626 | INFO | Perf report for the last 5.14008 seconds: | 20260506 21:32:41.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:32:41.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:41.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:41.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:41.626 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 21:32:41.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:41.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:41.626 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:32:41.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:41.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:41.626 | INFO | Uploaded 36GiB to GPU in 1.9621s (18.3477GiB/s) | 20260506 21:32:41.626 | INFO | Downloaded 34GiB to GPU in 1.93306s (17.5887GiB/s) | 20260506 21:32:46.761 | INFO | Analyzing chunk frames = [4609536,4610048) | 20260506 21:32:46.761 | INFO | MemInfo: { | 20260506 21:32:46.761 | INFO | "gpu" : " 17 GB", | 20260506 21:32:46.761 | INFO | "huge" : "", | 20260506 21:32:46.761 | INFO | "huge_shared" : " 27 GB", | 20260506 21:32:46.761 | INFO | "malloc" : " 14 GB", | 20260506 21:32:46.761 | INFO | "pinned" : " 29 GB" | 20260506 21:32:46.761 | INFO | } | 20260506 21:32:46.762 | INFO | Perf report for the last 5.13596 seconds: | 20260506 21:32:46.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:32:46.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:46.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:46.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:46.762 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260506 21:32:46.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:46.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:46.762 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:32:46.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:46.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:46.762 | INFO | Uploaded 36GiB to GPU in 1.99119s (18.0796GiB/s) | 20260506 21:32:46.762 | INFO | Downloaded 34GiB to GPU in 1.90917s (17.8088GiB/s) | 20260506 21:32:51.893 | INFO | Analyzing chunk frames = [4610048,4610560) | 20260506 21:32:51.893 | INFO | MemInfo: { | 20260506 21:32:51.893 | INFO | "gpu" : " 17 GB", | 20260506 21:32:51.893 | INFO | "huge" : "", | 20260506 21:32:51.893 | INFO | "huge_shared" : " 27 GB", | 20260506 21:32:51.893 | INFO | "malloc" : " 14 GB", | 20260506 21:32:51.893 | INFO | "pinned" : " 29 GB" | 20260506 21:32:51.893 | INFO | } | 20260506 21:32:51.894 | INFO | Perf report for the last 5.13194 seconds: | 20260506 21:32:51.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:32:51.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:51.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:51.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:51.894 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 21:32:51.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:51.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:51.894 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 21:32:51.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:51.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:51.894 | INFO | Uploaded 36GiB to GPU in 1.99948s (18.0047GiB/s) | 20260506 21:32:51.894 | INFO | Downloaded 34GiB to GPU in 1.91475s (17.7569GiB/s) | 20260506 21:32:57.031 | INFO | Analyzing chunk frames = [4610560,4611072) | 20260506 21:32:57.031 | INFO | MemInfo: { | 20260506 21:32:57.031 | INFO | "gpu" : " 17 GB", | 20260506 21:32:57.031 | INFO | "huge" : "", | 20260506 21:32:57.031 | INFO | "huge_shared" : " 27 GB", | 20260506 21:32:57.031 | INFO | "malloc" : " 14 GB", | 20260506 21:32:57.031 | INFO | "pinned" : " 29 GB" | 20260506 21:32:57.031 | INFO | } | 20260506 21:32:57.032 | INFO | Perf report for the last 5.13799 seconds: | 20260506 21:32:57.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:32:57.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:57.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:57.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:57.032 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 21:32:57.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:57.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:57.032 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:32:57.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:57.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:32:57.032 | INFO | Uploaded 36GiB to GPU in 1.9828s (18.1561GiB/s) | 20260506 21:32:57.032 | INFO | Downloaded 34GiB to GPU in 1.94604s (17.4714GiB/s) | 20260506 21:33:02.182 | INFO | Analyzing chunk frames = [4611072,4611584) | 20260506 21:33:02.182 | INFO | MemInfo: { | 20260506 21:33:02.182 | INFO | "gpu" : " 17 GB", | 20260506 21:33:02.182 | INFO | "huge" : "", | 20260506 21:33:02.182 | INFO | "huge_shared" : " 27 GB", | 20260506 21:33:02.182 | INFO | "malloc" : " 14 GB", | 20260506 21:33:02.182 | INFO | "pinned" : " 29 GB" | 20260506 21:33:02.182 | INFO | } | 20260506 21:33:02.182 | INFO | Perf report for the last 5.15051 seconds: | 20260506 21:33:02.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:33:02.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:02.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:02.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:02.182 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 21:33:02.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:02.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:02.182 | INFO | PRE_HQ : 26.6%, 1.0, 13.9ms, 0 | 20260506 21:33:02.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:02.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:02.182 | INFO | Uploaded 36GiB to GPU in 1.95222s (18.4405GiB/s) | 20260506 21:33:02.182 | INFO | Downloaded 34GiB to GPU in 1.99628s (17.0317GiB/s) | 20260506 21:33:07.320 | INFO | Analyzing chunk frames = [4611584,4612096) | 20260506 21:33:07.320 | INFO | MemInfo: { | 20260506 21:33:07.320 | INFO | "gpu" : " 17 GB", | 20260506 21:33:07.320 | INFO | "huge" : "", | 20260506 21:33:07.320 | INFO | "huge_shared" : " 27 GB", | 20260506 21:33:07.320 | INFO | "malloc" : " 14 GB", | 20260506 21:33:07.320 | INFO | "pinned" : " 29 GB" | 20260506 21:33:07.320 | INFO | } | 20260506 21:33:07.320 | INFO | Perf report for the last 5.13805 seconds: | 20260506 21:33:07.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:33:07.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:07.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:07.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:07.320 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 21:33:07.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:07.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:07.320 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 21:33:07.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:07.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:07.320 | INFO | Uploaded 36GiB to GPU in 2.03413s (17.6979GiB/s) | 20260506 21:33:07.320 | INFO | Downloaded 34GiB to GPU in 1.98671s (17.1137GiB/s) | 20260506 21:33:07.321 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:33:07.321 | INFO | Total mem usage: 18 GiB | 20260506 21:33:07.321 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:33:07.321 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -14.1%) | 20260506 21:33:07.321 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:33:07.321 | INFO | High Water Breakdown: | 20260506 21:33:07.321 | INFO | 61.7% : 9.73 GiB ( 85.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:33:07.321 | INFO | 38.2% : 6.03 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:33:07.321 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:33:07.321 | INFO | ------------------------------------------ | 20260506 21:33:07.321 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:33:07.321 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:33:07.321 | INFO | Total mem usage: 26.5 GiB | 20260506 21:33:07.321 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:33:07.321 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:33:07.321 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:33:07.321 | INFO | High Water Breakdown: | 20260506 21:33:07.321 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:33:07.321 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:33:07.321 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:33:07.321 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:33:07.321 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:33:07.321 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:33:07.321 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:33:07.321 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:33:07.321 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:33:07.321 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:33:07.321 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:33:07.321 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:33:07.321 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:33:07.321 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:33:07.321 | INFO | ------------------------------------------ | 20260506 21:33:07.321 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:33:07.321 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:33:07.321 | INFO | Total mem usage: 25.6 GiB | 20260506 21:33:07.321 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:33:07.321 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:33:07.321 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:33:07.321 | INFO | High Water Breakdown: | 20260506 21:33:07.321 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:33:07.321 | INFO | ------------------------------------------ | 20260506 21:33:07.321 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:33:07.321 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:33:07.321 | INFO | Total mem usage: 15.7 GiB | 20260506 21:33:07.321 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:33:07.321 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:33:07.321 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:33:07.321 | INFO | High Water Breakdown: | 20260506 21:33:07.321 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:33:07.321 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:33:07.321 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:33:07.321 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:33:07.321 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:33:07.321 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:33:07.321 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:33:07.321 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:33:07.321 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:33:07.321 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:33:07.321 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:33:07.321 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:33:07.321 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:33:07.321 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:33:07.321 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:33:07.321 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:33:07.321 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:33:07.321 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:33:07.321 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:33:07.321 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:33:07.321 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:33:07.321 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:33:07.321 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:33:07.322 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:33:07.322 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:33:07.322 | INFO | ------------------------------------------ | 20260506 21:33:07.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:33:07.322 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:33:07.322 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:33:07.322 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:33:07.322 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.59e+01, 1.32e+01 / 2.41e+00 | 20260506 21:33:07.322 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.02e-01, 9.75e-03 / 8.68e-03 | 20260506 21:33:07.322 | INFO | PerfReport: 11.64%, Baseline -- 3.62e+00 / 4.35e+00, 5.50e-01 / 4.22e-01 | 20260506 21:33:07.322 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.44e+00, 1.80e-01 / 1.50e-01 | 20260506 21:33:07.322 | INFO | PerfReport: 15.96%, DME -- 3.97e+01 / 4.39e+01, 3.96e+00 / 2.76e+00 | 20260506 21:33:07.322 | INFO | PerfReport: 57.35%, FrameLabeling -- 1.78e+01 / 2.16e+01, 1.41e+00 / 2.18e+00 | 20260506 21:33:07.322 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.21e+00 / 2.71e+00, 3.72e-01 / 2.56e-01 | 20260506 21:33:07.322 | INFO | PerfReport: 3.28%, Metrics -- 1.02e+00 / 1.35e+00, 2.72e-01 / 2.20e-01 | 20260506 21:33:07.322 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.48e-01 / 3.92e-01, 5.38e-02 / 3.00e-02 | 20260506 21:33:07.322 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:33:07.322 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:33:07.322 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:33:07.322 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 21:33:07.322 | INFO | PerfReport: 21.57%, Upload -- 2.09e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260506 21:33:07.322 | INFO | PerfReport: 21.30%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 21:33:07.322 | INFO | PerfReport: 32.15%, Compute -- 3.11e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 21:33:07.322 | INFO | PerfReport: 1.32%, LockUpload -- 1.28e+00 / 0.00e+00, 5.16e+00 / 0.00e+00 | 20260506 21:33:07.322 | INFO | PerfReport: 1.24%, LockDownload -- 1.19e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260506 21:33:07.322 | INFO | PerfReport: 22.41%, LockCompute -- 2.17e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 21:33:07.322 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260506 21:33:07.430 | INFO | Starting a round of Baz IO | 20260506 21:33:10.559 | INFO | Populating RTMetrics report | 20260506 21:33:10.559 | INFO | RTMetrics report populated | 20260506 21:33:10.559 | INFO | Opening tmp RTMetrics json file | 20260506 21:33:10.603 | INFO | Serializing RTMetrics to json | 20260506 21:33:10.608 | INFO | Serialization done, swapping tmp file now... | 20260506 21:33:10.613 | INFO | RTMetrics done | 20260506 21:33:10.903 | INFO | Finished a round of Baz IO: Wrote 1.944 GiB in 3.472 seconds | 20260506 21:33:10.903 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.91% / 0.00% / 66.09% | 20260506 21:33:10.903 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.603608% / 3.97364e-06% / 7.54992e-05% | 20260506 21:33:10.903 | INFO | Baz IO overall: | 20260506 21:33:10.903 | INFO | BytesWritten : 2429452889213 | 20260506 21:33:10.903 | INFO | NumEvents : 1172923218562 | 20260506 21:33:12.459 | INFO | Analyzing chunk frames = [4612096,4612608) | 20260506 21:33:12.459 | INFO | MemInfo: { | 20260506 21:33:12.459 | INFO | "gpu" : " 17 GB", | 20260506 21:33:12.459 | INFO | "huge" : "", | 20260506 21:33:12.459 | INFO | "huge_shared" : " 27 GB", | 20260506 21:33:12.459 | INFO | "malloc" : " 14 GB", | 20260506 21:33:12.459 | INFO | "pinned" : " 29 GB" | 20260506 21:33:12.459 | INFO | } | 20260506 21:33:12.459 | INFO | Perf report for the last 5.13844 seconds: | 20260506 21:33:12.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:33:12.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:12.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:12.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:12.459 | INFO | ANALYSIS : 62.5%, 2.9, 95.5ms, 0 | 20260506 21:33:12.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:12.459 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 21:33:12.459 | INFO | PRE_HQ : 60.0%, 1.8, 57.0ms, 0 | 20260506 21:33:12.459 | INFO | BAZWRITER : 67.3%, 4.8, 165.7ms, 0 | 20260506 21:33:12.459 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 21:33:12.459 | INFO | Uploaded 36GiB to GPU in 2.06809s (17.4074GiB/s) | 20260506 21:33:12.459 | INFO | Downloaded 38GiB to GPU in 2.20158s (17.2603GiB/s) | 20260506 21:33:17.600 | INFO | Analyzing chunk frames = [4612608,4613120) | 20260506 21:33:17.600 | INFO | MemInfo: { | 20260506 21:33:17.600 | INFO | "gpu" : " 17 GB", | 20260506 21:33:17.600 | INFO | "huge" : "", | 20260506 21:33:17.600 | INFO | "huge_shared" : " 27 GB", | 20260506 21:33:17.600 | INFO | "malloc" : " 14 GB", | 20260506 21:33:17.600 | INFO | "pinned" : " 29 GB" | 20260506 21:33:17.600 | INFO | } | 20260506 21:33:17.600 | INFO | Perf report for the last 5.14182 seconds: | 20260506 21:33:17.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:33:17.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:17.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:17.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:17.600 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:33:17.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:17.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:17.601 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 21:33:17.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:17.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:17.601 | INFO | Uploaded 36GiB to GPU in 2.00022s (17.998GiB/s) | 20260506 21:33:17.601 | INFO | Downloaded 34GiB to GPU in 1.92494s (17.6629GiB/s) | 20260506 21:33:22.736 | INFO | Analyzing chunk frames = [4613120,4613632) | 20260506 21:33:22.736 | INFO | MemInfo: { | 20260506 21:33:22.736 | INFO | "gpu" : " 17 GB", | 20260506 21:33:22.736 | INFO | "huge" : "", | 20260506 21:33:22.736 | INFO | "huge_shared" : " 27 GB", | 20260506 21:33:22.736 | INFO | "malloc" : " 14 GB", | 20260506 21:33:22.736 | INFO | "pinned" : " 29 GB" | 20260506 21:33:22.736 | INFO | } | 20260506 21:33:22.736 | INFO | Perf report for the last 5.13589 seconds: | 20260506 21:33:22.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:33:22.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:22.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:22.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:22.736 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 21:33:22.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:22.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:22.736 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:33:22.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:22.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:22.737 | INFO | Uploaded 36GiB to GPU in 1.98892s (18.1003GiB/s) | 20260506 21:33:22.737 | INFO | Downloaded 34GiB to GPU in 1.92656s (17.648GiB/s) | 20260506 21:33:27.870 | INFO | Analyzing chunk frames = [4613632,4614144) | 20260506 21:33:27.870 | INFO | MemInfo: { | 20260506 21:33:27.870 | INFO | "gpu" : " 17 GB", | 20260506 21:33:27.870 | INFO | "huge" : "", | 20260506 21:33:27.870 | INFO | "huge_shared" : " 27 GB", | 20260506 21:33:27.870 | INFO | "malloc" : " 14 GB", | 20260506 21:33:27.870 | INFO | "pinned" : " 29 GB" | 20260506 21:33:27.870 | INFO | } | 20260506 21:33:27.870 | INFO | Perf report for the last 5.13378 seconds: | 20260506 21:33:27.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:33:27.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:27.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:27.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:27.870 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 21:33:27.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:27.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:27.870 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:33:27.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:27.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:27.871 | INFO | Uploaded 36GiB to GPU in 2.01525s (17.8637GiB/s) | 20260506 21:33:27.871 | INFO | Downloaded 34GiB to GPU in 1.90532s (17.8447GiB/s) | 20260506 21:33:33.010 | INFO | Analyzing chunk frames = [4614144,4614656) | 20260506 21:33:33.010 | INFO | MemInfo: { | 20260506 21:33:33.010 | INFO | "gpu" : " 17 GB", | 20260506 21:33:33.010 | INFO | "huge" : "", | 20260506 21:33:33.010 | INFO | "huge_shared" : " 27 GB", | 20260506 21:33:33.010 | INFO | "malloc" : " 14 GB", | 20260506 21:33:33.010 | INFO | "pinned" : " 29 GB" | 20260506 21:33:33.010 | INFO | } | 20260506 21:33:33.010 | INFO | Perf report for the last 5.14024 seconds: | 20260506 21:33:33.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:33:33.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:33.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:33.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:33.010 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 21:33:33.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:33.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:33.010 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 21:33:33.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:33.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:33.011 | INFO | Uploaded 36GiB to GPU in 1.9892s (18.0977GiB/s) | 20260506 21:33:33.011 | INFO | Downloaded 34GiB to GPU in 1.92258s (17.6845GiB/s) | 20260506 21:33:38.150 | INFO | Analyzing chunk frames = [4614656,4615168) | 20260506 21:33:38.150 | INFO | MemInfo: { | 20260506 21:33:38.150 | INFO | "gpu" : " 17 GB", | 20260506 21:33:38.150 | INFO | "huge" : "", | 20260506 21:33:38.150 | INFO | "huge_shared" : " 27 GB", | 20260506 21:33:38.150 | INFO | "malloc" : " 14 GB", | 20260506 21:33:38.150 | INFO | "pinned" : " 29 GB" | 20260506 21:33:38.150 | INFO | } | 20260506 21:33:38.150 | INFO | Perf report for the last 5.13935 seconds: | 20260506 21:33:38.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:33:38.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:38.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:38.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:38.150 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 21:33:38.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:38.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:38.150 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 21:33:38.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:38.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:38.150 | INFO | Uploaded 36GiB to GPU in 2.0235s (17.7909GiB/s) | 20260506 21:33:38.150 | INFO | Downloaded 34GiB to GPU in 1.94627s (17.4693GiB/s) | 20260506 21:33:43.292 | INFO | Analyzing chunk frames = [4615168,4615680) | 20260506 21:33:43.292 | INFO | MemInfo: { | 20260506 21:33:43.292 | INFO | "gpu" : " 17 GB", | 20260506 21:33:43.292 | INFO | "huge" : "", | 20260506 21:33:43.292 | INFO | "huge_shared" : " 27 GB", | 20260506 21:33:43.292 | INFO | "malloc" : " 14 GB", | 20260506 21:33:43.292 | INFO | "pinned" : " 29 GB" | 20260506 21:33:43.292 | INFO | } | 20260506 21:33:43.292 | INFO | Perf report for the last 5.1424 seconds: | 20260506 21:33:43.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:33:43.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:43.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:43.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:43.292 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260506 21:33:43.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:43.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:43.292 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260506 21:33:43.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:43.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:43.293 | INFO | Uploaded 36GiB to GPU in 2.04898s (17.5697GiB/s) | 20260506 21:33:43.293 | INFO | Downloaded 34GiB to GPU in 1.96938s (17.2643GiB/s) | 20260506 21:33:48.441 | INFO | Analyzing chunk frames = [4615680,4616192) | 20260506 21:33:48.441 | INFO | MemInfo: { | 20260506 21:33:48.441 | INFO | "gpu" : " 17 GB", | 20260506 21:33:48.441 | INFO | "huge" : "", | 20260506 21:33:48.441 | INFO | "huge_shared" : " 27 GB", | 20260506 21:33:48.441 | INFO | "malloc" : " 14 GB", | 20260506 21:33:48.441 | INFO | "pinned" : " 29 GB" | 20260506 21:33:48.441 | INFO | } | 20260506 21:33:48.441 | INFO | Perf report for the last 5.14926 seconds: | 20260506 21:33:48.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:33:48.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:48.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:48.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:48.441 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 21:33:48.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:48.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:48.442 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 21:33:48.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:48.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:48.442 | INFO | Uploaded 36GiB to GPU in 1.9889s (18.1004GiB/s) | 20260506 21:33:48.442 | INFO | Downloaded 34GiB to GPU in 1.98854s (17.0979GiB/s) | 20260506 21:33:48.540 | INFO | Starting a round of Baz IO | 20260506 21:33:51.674 | INFO | Populating RTMetrics report | 20260506 21:33:51.675 | INFO | RTMetrics report populated | 20260506 21:33:51.675 | INFO | Opening tmp RTMetrics json file | 20260506 21:33:51.705 | INFO | Serializing RTMetrics to json | 20260506 21:33:51.731 | INFO | Serialization done, swapping tmp file now... | 20260506 21:33:51.733 | INFO | RTMetrics done | 20260506 21:33:51.835 | INFO | Finished a round of Baz IO: Wrote 1.941 GiB in 3.294 seconds | 20260506 21:33:51.835 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.90% / 0.00% / 66.09% | 20260506 21:33:51.835 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.599694% / 7.94729e-06% / 9.93411e-05% | 20260506 21:33:51.835 | INFO | Baz IO overall: | 20260506 21:33:51.835 | INFO | BytesWritten : 2431537522953 | 20260506 21:33:51.835 | INFO | NumEvents : 1173920788191 | 20260506 21:33:53.575 | INFO | Analyzing chunk frames = [4616192,4616704) | 20260506 21:33:53.575 | INFO | MemInfo: { | 20260506 21:33:53.575 | INFO | "gpu" : " 17 GB", | 20260506 21:33:53.575 | INFO | "huge" : "", | 20260506 21:33:53.575 | INFO | "huge_shared" : " 27 GB", | 20260506 21:33:53.575 | INFO | "malloc" : " 14 GB", | 20260506 21:33:53.575 | INFO | "pinned" : " 29 GB" | 20260506 21:33:53.575 | INFO | } | 20260506 21:33:53.575 | INFO | Perf report for the last 5.13362 seconds: | 20260506 21:33:53.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:33:53.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:53.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:53.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:53.575 | INFO | ANALYSIS : 62.4%, 2.9, 95.4ms, 0 | 20260506 21:33:53.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:53.575 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 21:33:53.575 | INFO | PRE_HQ : 60.1%, 1.8, 56.8ms, 0 | 20260506 21:33:53.575 | INFO | BAZWRITER : 63.8%, 4.8, 155.6ms, 0 | 20260506 21:33:53.575 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 21:33:53.575 | INFO | Uploaded 36GiB to GPU in 2.0434s (17.6177GiB/s) | 20260506 21:33:53.575 | INFO | Downloaded 38GiB to GPU in 2.19531s (17.3097GiB/s) | 20260506 21:33:58.711 | INFO | Analyzing chunk frames = [4616704,4617216) | 20260506 21:33:58.711 | INFO | MemInfo: { | 20260506 21:33:58.711 | INFO | "gpu" : " 17 GB", | 20260506 21:33:58.711 | INFO | "huge" : "", | 20260506 21:33:58.711 | INFO | "huge_shared" : " 27 GB", | 20260506 21:33:58.711 | INFO | "malloc" : " 14 GB", | 20260506 21:33:58.711 | INFO | "pinned" : " 29 GB" | 20260506 21:33:58.711 | INFO | } | 20260506 21:33:58.711 | INFO | Perf report for the last 5.13579 seconds: | 20260506 21:33:58.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:33:58.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:58.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:58.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:58.711 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 21:33:58.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:58.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:58.711 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 21:33:58.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:58.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:33:58.711 | INFO | Uploaded 36GiB to GPU in 1.99264s (18.0665GiB/s) | 20260506 21:33:58.711 | INFO | Downloaded 34GiB to GPU in 1.92777s (17.6369GiB/s) | 20260506 21:34:03.847 | INFO | Analyzing chunk frames = [4617216,4617728) | 20260506 21:34:03.847 | INFO | MemInfo: { | 20260506 21:34:03.847 | INFO | "gpu" : " 17 GB", | 20260506 21:34:03.847 | INFO | "huge" : "", | 20260506 21:34:03.847 | INFO | "huge_shared" : " 27 GB", | 20260506 21:34:03.847 | INFO | "malloc" : " 14 GB", | 20260506 21:34:03.847 | INFO | "pinned" : " 29 GB" | 20260506 21:34:03.847 | INFO | } | 20260506 21:34:03.847 | INFO | Perf report for the last 5.13662 seconds: | 20260506 21:34:03.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:34:03.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:03.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:03.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:03.848 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 21:34:03.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:03.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:03.848 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:34:03.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:03.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:03.848 | INFO | Uploaded 36GiB to GPU in 1.95145s (18.4479GiB/s) | 20260506 21:34:03.848 | INFO | Downloaded 34GiB to GPU in 1.92189s (17.6909GiB/s) | 20260506 21:34:08.981 | INFO | Analyzing chunk frames = [4617728,4618240) | 20260506 21:34:08.981 | INFO | MemInfo: { | 20260506 21:34:08.981 | INFO | "gpu" : " 17 GB", | 20260506 21:34:08.981 | INFO | "huge" : "", | 20260506 21:34:08.981 | INFO | "huge_shared" : " 27 GB", | 20260506 21:34:08.981 | INFO | "malloc" : " 14 GB", | 20260506 21:34:08.981 | INFO | "pinned" : " 29 GB" | 20260506 21:34:08.981 | INFO | } | 20260506 21:34:08.981 | INFO | Perf report for the last 5.13372 seconds: | 20260506 21:34:08.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:34:08.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:08.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:08.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:08.981 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 21:34:08.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:08.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:08.981 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 21:34:08.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:08.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:08.981 | INFO | Uploaded 36GiB to GPU in 1.95972s (18.3699GiB/s) | 20260506 21:34:08.981 | INFO | Downloaded 34GiB to GPU in 1.87678s (18.1162GiB/s) | 20260506 21:34:14.125 | INFO | Analyzing chunk frames = [4618240,4618752) | 20260506 21:34:14.125 | INFO | MemInfo: { | 20260506 21:34:14.125 | INFO | "gpu" : " 17 GB", | 20260506 21:34:14.125 | INFO | "huge" : "", | 20260506 21:34:14.125 | INFO | "huge_shared" : " 27 GB", | 20260506 21:34:14.125 | INFO | "malloc" : " 14 GB", | 20260506 21:34:14.125 | INFO | "pinned" : " 29 GB" | 20260506 21:34:14.125 | INFO | } | 20260506 21:34:14.125 | INFO | Perf report for the last 5.14363 seconds: | 20260506 21:34:14.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:34:14.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:14.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:14.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:14.125 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 21:34:14.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:14.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:14.125 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:34:14.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:14.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:14.125 | INFO | Uploaded 36GiB to GPU in 2.01047s (17.9063GiB/s) | 20260506 21:34:14.125 | INFO | Downloaded 34GiB to GPU in 1.91967s (17.7114GiB/s) | 20260506 21:34:19.270 | INFO | Analyzing chunk frames = [4618752,4619264) | 20260506 21:34:19.270 | INFO | MemInfo: { | 20260506 21:34:19.270 | INFO | "gpu" : " 17 GB", | 20260506 21:34:19.270 | INFO | "huge" : "", | 20260506 21:34:19.270 | INFO | "huge_shared" : " 27 GB", | 20260506 21:34:19.270 | INFO | "malloc" : " 14 GB", | 20260506 21:34:19.270 | INFO | "pinned" : " 29 GB" | 20260506 21:34:19.270 | INFO | } | 20260506 21:34:19.270 | INFO | Perf report for the last 5.14557 seconds: | 20260506 21:34:19.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:34:19.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:19.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:19.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:19.271 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 21:34:19.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:19.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:19.271 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 21:34:19.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:19.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:19.271 | INFO | Uploaded 36GiB to GPU in 1.99107s (18.0807GiB/s) | 20260506 21:34:19.271 | INFO | Downloaded 34GiB to GPU in 1.95175s (17.4202GiB/s) | 20260506 21:34:24.407 | INFO | Analyzing chunk frames = [4619264,4619776) | 20260506 21:34:24.407 | INFO | MemInfo: { | 20260506 21:34:24.407 | INFO | "gpu" : " 17 GB", | 20260506 21:34:24.407 | INFO | "huge" : "", | 20260506 21:34:24.407 | INFO | "huge_shared" : " 27 GB", | 20260506 21:34:24.407 | INFO | "malloc" : " 14 GB", | 20260506 21:34:24.407 | INFO | "pinned" : " 29 GB" | 20260506 21:34:24.407 | INFO | } | 20260506 21:34:24.407 | INFO | Perf report for the last 5.13662 seconds: | 20260506 21:34:24.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:34:24.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:24.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:24.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:24.407 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:34:24.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:24.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:24.407 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 21:34:24.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:24.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:24.407 | INFO | Uploaded 36GiB to GPU in 1.95768s (18.3891GiB/s) | 20260506 21:34:24.407 | INFO | Downloaded 34GiB to GPU in 1.99177s (17.0703GiB/s) | 20260506 21:34:29.547 | INFO | Analyzing chunk frames = [4619776,4620288) | 20260506 21:34:29.547 | INFO | MemInfo: { | 20260506 21:34:29.547 | INFO | "gpu" : " 17 GB", | 20260506 21:34:29.547 | INFO | "huge" : "", | 20260506 21:34:29.547 | INFO | "huge_shared" : " 27 GB", | 20260506 21:34:29.547 | INFO | "malloc" : " 14 GB", | 20260506 21:34:29.547 | INFO | "pinned" : " 29 GB" | 20260506 21:34:29.547 | INFO | } | 20260506 21:34:29.547 | INFO | Perf report for the last 5.14012 seconds: | 20260506 21:34:29.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:34:29.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:29.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:29.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:29.547 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 21:34:29.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:29.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:29.547 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 21:34:29.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:29.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:29.548 | INFO | Uploaded 36GiB to GPU in 2.03526s (17.6882GiB/s) | 20260506 21:34:29.548 | INFO | Downloaded 34GiB to GPU in 2.01056s (16.9107GiB/s) | 20260506 21:34:29.548 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:34:29.548 | INFO | Total mem usage: 18 GiB | 20260506 21:34:29.548 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:34:29.548 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.2%) | 20260506 21:34:29.548 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:34:29.548 | INFO | High Water Breakdown: | 20260506 21:34:29.548 | INFO | 61.8% : 9.83 GiB ( 86.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:34:29.548 | INFO | 38.1% : 6.05 GiB ( 90.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:34:29.548 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:34:29.548 | INFO | ------------------------------------------ | 20260506 21:34:29.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:34:29.548 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:34:29.548 | INFO | Total mem usage: 26.5 GiB | 20260506 21:34:29.548 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:34:29.548 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:34:29.548 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:34:29.548 | INFO | High Water Breakdown: | 20260506 21:34:29.548 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:34:29.548 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:34:29.548 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:34:29.548 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:34:29.548 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:34:29.548 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:34:29.548 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:34:29.548 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:34:29.548 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:34:29.548 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:34:29.548 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:34:29.548 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:34:29.548 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:34:29.548 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:34:29.548 | INFO | ------------------------------------------ | 20260506 21:34:29.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:34:29.548 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:34:29.548 | INFO | Total mem usage: 25.6 GiB | 20260506 21:34:29.548 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:34:29.548 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:34:29.548 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:34:29.548 | INFO | High Water Breakdown: | 20260506 21:34:29.548 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:34:29.548 | INFO | ------------------------------------------ | 20260506 21:34:29.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:34:29.548 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:34:29.548 | INFO | Total mem usage: 15.7 GiB | 20260506 21:34:29.548 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:34:29.548 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:34:29.548 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:34:29.548 | INFO | High Water Breakdown: | 20260506 21:34:29.548 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:34:29.548 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:34:29.548 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:34:29.548 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:34:29.548 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:34:29.548 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:34:29.548 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:34:29.548 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:34:29.548 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:34:29.548 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:34:29.548 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:34:29.548 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:34:29.548 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:34:29.548 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:34:29.548 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:34:29.548 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:34:29.548 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:34:29.548 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:34:29.549 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:34:29.549 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:34:29.549 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:34:29.549 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:34:29.549 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:34:29.549 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:34:29.549 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:34:29.549 | INFO | ------------------------------------------ | 20260506 21:34:29.549 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:34:29.549 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:34:29.549 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:34:29.549 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:34:29.549 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.57e+01, 1.36e+01 / 2.38e+00 | 20260506 21:34:29.549 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.02e-01, 1.10e-02 / 9.14e-03 | 20260506 21:34:29.549 | INFO | PerfReport: 11.64%, Baseline -- 3.62e+00 / 4.35e+00, 5.43e-01 / 3.76e-01 | 20260506 21:34:29.549 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.44e+00, 1.81e-01 / 1.47e-01 | 20260506 21:34:29.549 | INFO | PerfReport: 16.02%, DME -- 3.99e+01 / 4.44e+01, 4.09e+00 / 2.67e+00 | 20260506 21:34:29.549 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.46e+00 / 1.86e+00 | 20260506 21:34:29.549 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.21e+00 / 2.75e+00, 3.73e-01 / 2.64e-01 | 20260506 21:34:29.549 | INFO | PerfReport: 3.25%, Metrics -- 1.01e+00 / 1.35e+00, 2.63e-01 / 1.85e-01 | 20260506 21:34:29.549 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.47e-01 / 3.93e-01, 5.32e-02 / 2.13e-02 | 20260506 21:34:29.549 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:34:29.549 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:34:29.549 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:34:29.549 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 21:34:29.549 | INFO | PerfReport: 21.58%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 21:34:29.549 | INFO | PerfReport: 21.25%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 21:34:29.549 | INFO | PerfReport: 32.18%, Compute -- 3.11e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 21:34:29.549 | INFO | PerfReport: 1.30%, LockUpload -- 1.26e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260506 21:34:29.549 | INFO | PerfReport: 1.19%, LockDownload -- 1.15e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 21:34:29.549 | INFO | PerfReport: 22.49%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:34:29.549 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260506 21:34:29.662 | INFO | Starting a round of Baz IO | 20260506 21:34:32.782 | INFO | Populating RTMetrics report | 20260506 21:34:32.783 | INFO | RTMetrics report populated | 20260506 21:34:32.783 | INFO | Opening tmp RTMetrics json file | 20260506 21:34:32.837 | INFO | Serializing RTMetrics to json | 20260506 21:34:32.940 | INFO | Serialization done, swapping tmp file now... | 20260506 21:34:32.945 | INFO | RTMetrics done | 20260506 21:34:32.981 | INFO | Finished a round of Baz IO: Wrote 1.942 GiB in 3.319 seconds | 20260506 21:34:32.982 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.90% / 0.00% / 66.10% | 20260506 21:34:32.982 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.599054% / 7.94729e-06% / 8.74201e-05% | 20260506 21:34:32.982 | INFO | Baz IO overall: | 20260506 21:34:32.982 | INFO | BytesWritten : 2433622561840 | 20260506 21:34:32.982 | INFO | NumEvents : 1174918696685 | 20260506 21:34:34.689 | INFO | Analyzing chunk frames = [4620288,4620800) | 20260506 21:34:34.689 | INFO | MemInfo: { | 20260506 21:34:34.689 | INFO | "gpu" : " 17 GB", | 20260506 21:34:34.689 | INFO | "huge" : "", | 20260506 21:34:34.689 | INFO | "huge_shared" : " 27 GB", | 20260506 21:34:34.689 | INFO | "malloc" : " 14 GB", | 20260506 21:34:34.689 | INFO | "pinned" : " 29 GB" | 20260506 21:34:34.689 | INFO | } | 20260506 21:34:34.689 | INFO | Perf report for the last 5.14158 seconds: | 20260506 21:34:34.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:34:34.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:34.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:34.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:34.689 | INFO | ANALYSIS : 62.4%, 2.9, 95.7ms, 0 | 20260506 21:34:34.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:34.689 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260506 21:34:34.689 | INFO | PRE_HQ : 59.8%, 1.8, 57.1ms, 0 | 20260506 21:34:34.689 | INFO | BAZWRITER : 64.3%, 4.4, 144.4ms, 0 | 20260506 21:34:34.689 | INFO | BAZ_SAMPLING : 3.1%, 1.0, 1.6ms, 0 | 20260506 21:34:34.689 | INFO | Uploaded 36GiB to GPU in 2.05424s (17.5247GiB/s) | 20260506 21:34:34.689 | INFO | Downloaded 38GiB to GPU in 2.20767s (17.2127GiB/s) | 20260506 21:34:39.824 | INFO | Analyzing chunk frames = [4620800,4621312) | 20260506 21:34:39.824 | INFO | MemInfo: { | 20260506 21:34:39.824 | INFO | "gpu" : " 17 GB", | 20260506 21:34:39.824 | INFO | "huge" : "", | 20260506 21:34:39.824 | INFO | "huge_shared" : " 27 GB", | 20260506 21:34:39.824 | INFO | "malloc" : " 14 GB", | 20260506 21:34:39.824 | INFO | "pinned" : " 29 GB" | 20260506 21:34:39.824 | INFO | } | 20260506 21:34:39.824 | INFO | Perf report for the last 5.13545 seconds: | 20260506 21:34:39.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:34:39.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:39.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:39.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:39.824 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 21:34:39.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:39.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:39.825 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 21:34:39.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:39.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:39.825 | INFO | Uploaded 36GiB to GPU in 1.9882s (18.1069GiB/s) | 20260506 21:34:39.825 | INFO | Downloaded 34GiB to GPU in 1.93068s (17.6104GiB/s) | 20260506 21:34:44.962 | INFO | Analyzing chunk frames = [4621312,4621824) | 20260506 21:34:44.962 | INFO | MemInfo: { | 20260506 21:34:44.962 | INFO | "gpu" : " 17 GB", | 20260506 21:34:44.962 | INFO | "huge" : "", | 20260506 21:34:44.962 | INFO | "huge_shared" : " 27 GB", | 20260506 21:34:44.962 | INFO | "malloc" : " 14 GB", | 20260506 21:34:44.962 | INFO | "pinned" : " 29 GB" | 20260506 21:34:44.962 | INFO | } | 20260506 21:34:44.962 | INFO | Perf report for the last 5.13812 seconds: | 20260506 21:34:44.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:34:44.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:44.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:44.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:44.962 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 21:34:44.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:44.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:44.962 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 21:34:44.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:44.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:44.963 | INFO | Uploaded 36GiB to GPU in 1.97872s (18.1936GiB/s) | 20260506 21:34:44.963 | INFO | Downloaded 34GiB to GPU in 1.92743s (17.64GiB/s) | 20260506 21:34:50.102 | INFO | Analyzing chunk frames = [4621824,4622336) | 20260506 21:34:50.102 | INFO | MemInfo: { | 20260506 21:34:50.102 | INFO | "gpu" : " 17 GB", | 20260506 21:34:50.102 | INFO | "huge" : "", | 20260506 21:34:50.102 | INFO | "huge_shared" : " 27 GB", | 20260506 21:34:50.102 | INFO | "malloc" : " 14 GB", | 20260506 21:34:50.102 | INFO | "pinned" : " 29 GB" | 20260506 21:34:50.102 | INFO | } | 20260506 21:34:50.102 | INFO | Perf report for the last 5.13949 seconds: | 20260506 21:34:50.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:34:50.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:50.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:50.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:50.102 | INFO | ANALYSIS : 60.3%, 3.0, 93.6ms, 0 | 20260506 21:34:50.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:50.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:50.102 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:34:50.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:50.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:50.102 | INFO | Uploaded 36GiB to GPU in 2.00695s (17.9377GiB/s) | 20260506 21:34:50.102 | INFO | Downloaded 34GiB to GPU in 1.90941s (17.8065GiB/s) | 20260506 21:34:55.242 | INFO | Analyzing chunk frames = [4622336,4622848) | 20260506 21:34:55.242 | INFO | MemInfo: { | 20260506 21:34:55.242 | INFO | "gpu" : " 17 GB", | 20260506 21:34:55.242 | INFO | "huge" : "", | 20260506 21:34:55.242 | INFO | "huge_shared" : " 27 GB", | 20260506 21:34:55.242 | INFO | "malloc" : " 14 GB", | 20260506 21:34:55.242 | INFO | "pinned" : " 29 GB" | 20260506 21:34:55.242 | INFO | } | 20260506 21:34:55.242 | INFO | Perf report for the last 5.14034 seconds: | 20260506 21:34:55.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:34:55.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:55.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:55.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:55.242 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260506 21:34:55.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:55.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:55.242 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:34:55.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:55.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:34:55.243 | INFO | Uploaded 36GiB to GPU in 1.97029s (18.2714GiB/s) | 20260506 21:34:55.243 | INFO | Downloaded 34GiB to GPU in 1.93228s (17.5958GiB/s) | 20260506 21:35:00.387 | INFO | Analyzing chunk frames = [4622848,4623360) | 20260506 21:35:00.387 | INFO | MemInfo: { | 20260506 21:35:00.387 | INFO | "gpu" : " 17 GB", | 20260506 21:35:00.387 | INFO | "huge" : "", | 20260506 21:35:00.387 | INFO | "huge_shared" : " 27 GB", | 20260506 21:35:00.387 | INFO | "malloc" : " 14 GB", | 20260506 21:35:00.387 | INFO | "pinned" : " 29 GB" | 20260506 21:35:00.387 | INFO | } | 20260506 21:35:00.387 | INFO | Perf report for the last 5.14456 seconds: | 20260506 21:35:00.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:35:00.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:00.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:00.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:00.387 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:35:00.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:00.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:00.387 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:35:00.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:00.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:00.387 | INFO | Uploaded 36GiB to GPU in 2.00963s (17.9137GiB/s) | 20260506 21:35:00.387 | INFO | Downloaded 34GiB to GPU in 1.94462s (17.4841GiB/s) | 20260506 21:35:05.526 | INFO | Analyzing chunk frames = [4623360,4623872) | 20260506 21:35:05.526 | INFO | MemInfo: { | 20260506 21:35:05.526 | INFO | "gpu" : " 17 GB", | 20260506 21:35:05.526 | INFO | "huge" : "", | 20260506 21:35:05.526 | INFO | "huge_shared" : " 27 GB", | 20260506 21:35:05.526 | INFO | "malloc" : " 14 GB", | 20260506 21:35:05.526 | INFO | "pinned" : " 29 GB" | 20260506 21:35:05.526 | INFO | } | 20260506 21:35:05.526 | INFO | Perf report for the last 5.13918 seconds: | 20260506 21:35:05.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:35:05.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:05.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:05.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:05.526 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 21:35:05.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:05.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:05.526 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 21:35:05.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:05.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:05.526 | INFO | Uploaded 36GiB to GPU in 1.95774s (18.3885GiB/s) | 20260506 21:35:05.527 | INFO | Downloaded 34GiB to GPU in 1.98633s (17.117GiB/s) | 20260506 21:35:10.661 | INFO | Analyzing chunk frames = [4623872,4624384) | 20260506 21:35:10.661 | INFO | MemInfo: { | 20260506 21:35:10.661 | INFO | "gpu" : " 17 GB", | 20260506 21:35:10.661 | INFO | "huge" : "", | 20260506 21:35:10.661 | INFO | "huge_shared" : " 27 GB", | 20260506 21:35:10.661 | INFO | "malloc" : " 14 GB", | 20260506 21:35:10.661 | INFO | "pinned" : " 29 GB" | 20260506 21:35:10.661 | INFO | } | 20260506 21:35:10.661 | INFO | Perf report for the last 5.13496 seconds: | 20260506 21:35:10.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:35:10.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:10.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:10.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:10.661 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 21:35:10.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:10.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:10.661 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:35:10.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:10.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:10.661 | INFO | Uploaded 36GiB to GPU in 2.0192s (17.8289GiB/s) | 20260506 21:35:10.661 | INFO | Downloaded 34GiB to GPU in 1.98227s (17.152GiB/s) | 20260506 21:35:10.765 | INFO | Starting a round of Baz IO | 20260506 21:35:13.905 | INFO | Populating RTMetrics report | 20260506 21:35:13.906 | INFO | RTMetrics report populated | 20260506 21:35:13.906 | INFO | Opening tmp RTMetrics json file | 20260506 21:35:13.925 | INFO | Serializing RTMetrics to json | 20260506 21:35:13.930 | INFO | Serialization done, swapping tmp file now... | 20260506 21:35:13.934 | INFO | RTMetrics done | 20260506 21:35:14.137 | INFO | Finished a round of Baz IO: Wrote 1.944 GiB in 3.371 seconds | 20260506 21:35:14.137 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.90% / 0.00% / 66.10% | 20260506 21:35:14.137 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.598248% / 0% / 5.16574e-05% | 20260506 21:35:14.137 | INFO | Baz IO overall: | 20260506 21:35:14.137 | INFO | BytesWritten : 2435709577629 | 20260506 21:35:14.137 | INFO | NumEvents : 1175918145060 | 20260506 21:35:15.800 | INFO | Analyzing chunk frames = [4624384,4624896) | 20260506 21:35:15.800 | INFO | MemInfo: { | 20260506 21:35:15.800 | INFO | "gpu" : " 17 GB", | 20260506 21:35:15.800 | INFO | "huge" : "", | 20260506 21:35:15.800 | INFO | "huge_shared" : " 27 GB", | 20260506 21:35:15.800 | INFO | "malloc" : " 14 GB", | 20260506 21:35:15.800 | INFO | "pinned" : " 29 GB" | 20260506 21:35:15.800 | INFO | } | 20260506 21:35:15.800 | INFO | Perf report for the last 5.13924 seconds: | 20260506 21:35:15.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:35:15.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:15.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:15.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:15.800 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 21:35:15.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:15.800 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 21:35:15.800 | INFO | PRE_HQ : 60.0%, 1.8, 56.7ms, 0 | 20260506 21:35:15.801 | INFO | BAZWRITER : 65.3%, 4.8, 160.6ms, 0 | 20260506 21:35:15.801 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 21:35:15.801 | INFO | Uploaded 36GiB to GPU in 2.04235s (17.6268GiB/s) | 20260506 21:35:15.801 | INFO | Downloaded 38GiB to GPU in 2.19922s (17.2789GiB/s) | 20260506 21:35:20.936 | INFO | Analyzing chunk frames = [4624896,4625408) | 20260506 21:35:20.936 | INFO | MemInfo: { | 20260506 21:35:20.936 | INFO | "gpu" : " 17 GB", | 20260506 21:35:20.936 | INFO | "huge" : "", | 20260506 21:35:20.936 | INFO | "huge_shared" : " 27 GB", | 20260506 21:35:20.936 | INFO | "malloc" : " 14 GB", | 20260506 21:35:20.936 | INFO | "pinned" : " 29 GB" | 20260506 21:35:20.936 | INFO | } | 20260506 21:35:20.936 | INFO | Perf report for the last 5.13624 seconds: | 20260506 21:35:20.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:35:20.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:20.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:20.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:20.937 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:35:20.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:20.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:20.937 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:35:20.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:20.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:20.937 | INFO | Uploaded 36GiB to GPU in 2.01672s (17.8508GiB/s) | 20260506 21:35:20.937 | INFO | Downloaded 34GiB to GPU in 1.92742s (17.6401GiB/s) | 20260506 21:35:26.076 | INFO | Analyzing chunk frames = [4625408,4625920) | 20260506 21:35:26.076 | INFO | MemInfo: { | 20260506 21:35:26.076 | INFO | "gpu" : " 17 GB", | 20260506 21:35:26.076 | INFO | "huge" : "", | 20260506 21:35:26.076 | INFO | "huge_shared" : " 27 GB", | 20260506 21:35:26.076 | INFO | "malloc" : " 14 GB", | 20260506 21:35:26.076 | INFO | "pinned" : " 29 GB" | 20260506 21:35:26.076 | INFO | } | 20260506 21:35:26.076 | INFO | Perf report for the last 5.13998 seconds: | 20260506 21:35:26.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:35:26.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:26.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:26.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:26.077 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 21:35:26.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:26.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:26.077 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 21:35:26.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:26.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:26.077 | INFO | Uploaded 36GiB to GPU in 1.97476s (18.23GiB/s) | 20260506 21:35:26.077 | INFO | Downloaded 34GiB to GPU in 1.92s (17.7083GiB/s) | 20260506 21:35:31.215 | INFO | Analyzing chunk frames = [4625920,4626432) | 20260506 21:35:31.215 | INFO | MemInfo: { | 20260506 21:35:31.215 | INFO | "gpu" : " 17 GB", | 20260506 21:35:31.215 | INFO | "huge" : "", | 20260506 21:35:31.216 | INFO | "huge_shared" : " 27 GB", | 20260506 21:35:31.216 | INFO | "malloc" : " 14 GB", | 20260506 21:35:31.216 | INFO | "pinned" : " 29 GB" | 20260506 21:35:31.216 | INFO | } | 20260506 21:35:31.216 | INFO | Perf report for the last 5.13914 seconds: | 20260506 21:35:31.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:35:31.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:31.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:31.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:31.216 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 21:35:31.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:31.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:31.216 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:35:31.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:31.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:31.216 | INFO | Uploaded 36GiB to GPU in 1.95804s (18.3857GiB/s) | 20260506 21:35:31.216 | INFO | Downloaded 34GiB to GPU in 1.87546s (18.1288GiB/s) | 20260506 21:35:36.358 | INFO | Analyzing chunk frames = [4626432,4626944) | 20260506 21:35:36.358 | INFO | MemInfo: { | 20260506 21:35:36.358 | INFO | "gpu" : " 17 GB", | 20260506 21:35:36.358 | INFO | "huge" : "", | 20260506 21:35:36.358 | INFO | "huge_shared" : " 27 GB", | 20260506 21:35:36.358 | INFO | "malloc" : " 14 GB", | 20260506 21:35:36.359 | INFO | "pinned" : " 29 GB" | 20260506 21:35:36.359 | INFO | } | 20260506 21:35:36.359 | INFO | Perf report for the last 5.14294 seconds: | 20260506 21:35:36.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:35:36.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:36.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:36.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:36.359 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260506 21:35:36.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:36.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:36.359 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:35:36.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:36.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:36.359 | INFO | Uploaded 36GiB to GPU in 2.016s (17.8571GiB/s) | 20260506 21:35:36.359 | INFO | Downloaded 34GiB to GPU in 1.90787s (17.8209GiB/s) | 20260506 21:35:41.498 | INFO | Analyzing chunk frames = [4626944,4627456) | 20260506 21:35:41.498 | INFO | MemInfo: { | 20260506 21:35:41.498 | INFO | "gpu" : " 17 GB", | 20260506 21:35:41.498 | INFO | "huge" : "", | 20260506 21:35:41.498 | INFO | "huge_shared" : " 27 GB", | 20260506 21:35:41.498 | INFO | "malloc" : " 14 GB", | 20260506 21:35:41.498 | INFO | "pinned" : " 29 GB" | 20260506 21:35:41.498 | INFO | } | 20260506 21:35:41.498 | INFO | Perf report for the last 5.13988 seconds: | 20260506 21:35:41.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:35:41.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:41.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:41.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:41.498 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:35:41.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:41.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:41.499 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 21:35:41.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:41.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:41.499 | INFO | Uploaded 36GiB to GPU in 2.01084s (17.903GiB/s) | 20260506 21:35:41.499 | INFO | Downloaded 34GiB to GPU in 1.95127s (17.4245GiB/s) | 20260506 21:35:46.634 | INFO | Analyzing chunk frames = [4627456,4627968) | 20260506 21:35:46.634 | INFO | MemInfo: { | 20260506 21:35:46.634 | INFO | "gpu" : " 17 GB", | 20260506 21:35:46.634 | INFO | "huge" : "", | 20260506 21:35:46.634 | INFO | "huge_shared" : " 27 GB", | 20260506 21:35:46.634 | INFO | "malloc" : " 14 GB", | 20260506 21:35:46.634 | INFO | "pinned" : " 29 GB" | 20260506 21:35:46.634 | INFO | } | 20260506 21:35:46.634 | INFO | Perf report for the last 5.13575 seconds: | 20260506 21:35:46.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:35:46.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:46.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:46.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:46.634 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:35:46.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:46.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:46.634 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 21:35:46.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:46.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:46.634 | INFO | Uploaded 36GiB to GPU in 1.95971s (18.37GiB/s) | 20260506 21:35:46.634 | INFO | Downloaded 34GiB to GPU in 2.00935s (16.9209GiB/s) | 20260506 21:35:51.775 | INFO | Analyzing chunk frames = [4627968,4628480) | 20260506 21:35:51.775 | INFO | MemInfo: { | 20260506 21:35:51.775 | INFO | "gpu" : " 17 GB", | 20260506 21:35:51.775 | INFO | "huge" : "", | 20260506 21:35:51.775 | INFO | "huge_shared" : " 27 GB", | 20260506 21:35:51.775 | INFO | "malloc" : " 14 GB", | 20260506 21:35:51.775 | INFO | "pinned" : " 29 GB" | 20260506 21:35:51.775 | INFO | } | 20260506 21:35:51.775 | INFO | Perf report for the last 5.14051 seconds: | 20260506 21:35:51.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:35:51.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:51.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:51.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:51.775 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 21:35:51.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:51.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:51.775 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 21:35:51.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:51.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:51.775 | INFO | Uploaded 36GiB to GPU in 1.97755s (18.2043GiB/s) | 20260506 21:35:51.775 | INFO | Downloaded 34GiB to GPU in 2.00168s (16.9857GiB/s) | 20260506 21:35:51.775 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:35:51.775 | INFO | Total mem usage: 18 GiB | 20260506 21:35:51.775 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:35:51.775 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.3%) | 20260506 21:35:51.775 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:35:51.775 | INFO | High Water Breakdown: | 20260506 21:35:51.775 | INFO | 61.8% : 9.82 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:35:51.775 | INFO | 38.1% : 6.05 GiB ( 90.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:35:51.775 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:35:51.775 | INFO | ------------------------------------------ | 20260506 21:35:51.775 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:35:51.775 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:35:51.775 | INFO | Total mem usage: 26.5 GiB | 20260506 21:35:51.775 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:35:51.775 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:35:51.775 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:35:51.775 | INFO | High Water Breakdown: | 20260506 21:35:51.775 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:35:51.775 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:35:51.776 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:35:51.776 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:35:51.776 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:35:51.776 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:35:51.776 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:35:51.776 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:35:51.776 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:35:51.776 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:35:51.776 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:35:51.776 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:35:51.776 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:35:51.776 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:35:51.776 | INFO | ------------------------------------------ | 20260506 21:35:51.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:35:51.776 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:35:51.776 | INFO | Total mem usage: 25.6 GiB | 20260506 21:35:51.776 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:35:51.776 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:35:51.776 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:35:51.776 | INFO | High Water Breakdown: | 20260506 21:35:51.776 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:35:51.776 | INFO | ------------------------------------------ | 20260506 21:35:51.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:35:51.776 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:35:51.776 | INFO | Total mem usage: 15.7 GiB | 20260506 21:35:51.776 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:35:51.776 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:35:51.776 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:35:51.776 | INFO | High Water Breakdown: | 20260506 21:35:51.776 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:35:51.776 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:35:51.776 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:35:51.776 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:35:51.776 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:35:51.776 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:35:51.776 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:35:51.776 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:35:51.776 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:35:51.776 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:35:51.776 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:35:51.776 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:35:51.776 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:35:51.776 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:35:51.776 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:35:51.776 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:35:51.776 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:35:51.776 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:35:51.776 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:35:51.776 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:35:51.776 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:35:51.776 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:35:51.776 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:35:51.776 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:35:51.776 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:35:51.776 | INFO | ------------------------------------------ | 20260506 21:35:51.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:35:51.776 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:35:51.776 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:35:51.776 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:35:51.776 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.61e+01, 1.35e+01 / 2.52e+00 | 20260506 21:35:51.776 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 9.96e-02, 9.64e-03 / 6.38e-03 | 20260506 21:35:51.776 | INFO | PerfReport: 11.64%, Baseline -- 3.62e+00 / 4.38e+00, 5.50e-01 / 3.87e-01 | 20260506 21:35:51.776 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.45e+00, 1.82e-01 / 1.43e-01 | 20260506 21:35:51.776 | INFO | PerfReport: 16.07%, DME -- 4.00e+01 / 4.49e+01, 4.31e+00 / 3.05e+00 | 20260506 21:35:51.776 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.37e+00 / 2.05e+00 | 20260506 21:35:51.776 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.22e+00 / 2.82e+00, 3.86e-01 / 2.48e-01 | 20260506 21:35:51.776 | INFO | PerfReport: 3.26%, Metrics -- 1.01e+00 / 1.36e+00, 2.68e-01 / 1.51e-01 | 20260506 21:35:51.776 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.47e-01 / 3.96e-01, 5.37e-02 / 2.49e-02 | 20260506 21:35:51.776 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:35:51.776 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:35:51.776 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:35:51.776 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 21:35:51.776 | INFO | PerfReport: 21.50%, Upload -- 2.08e+01 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260506 21:35:51.776 | INFO | PerfReport: 21.28%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 21:35:51.776 | INFO | PerfReport: 32.16%, Compute -- 3.11e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 21:35:51.776 | INFO | PerfReport: 1.30%, LockUpload -- 1.26e+00 / 0.00e+00, 4.96e+00 / 0.00e+00 | 20260506 21:35:51.776 | INFO | PerfReport: 1.25%, LockDownload -- 1.21e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260506 21:35:51.776 | INFO | PerfReport: 22.50%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:35:51.776 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260506 21:35:51.874 | INFO | Starting a round of Baz IO | 20260506 21:35:55.012 | INFO | Populating RTMetrics report | 20260506 21:35:55.012 | INFO | RTMetrics report populated | 20260506 21:35:55.012 | INFO | Opening tmp RTMetrics json file | 20260506 21:35:55.201 | INFO | Serializing RTMetrics to json | 20260506 21:35:55.204 | INFO | Serialization done, swapping tmp file now... | 20260506 21:35:55.205 | INFO | RTMetrics done | 20260506 21:35:55.795 | INFO | Finished a round of Baz IO: Wrote 1.942 GiB in 3.921 seconds | 20260506 21:35:55.795 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.89% / 0.00% / 66.10% | 20260506 21:35:55.795 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.597843% / 7.94729e-06% / 5.96046e-05% | 20260506 21:35:55.795 | INFO | Baz IO overall: | 20260506 21:35:55.795 | INFO | BytesWritten : 2437794633011 | 20260506 21:35:55.795 | INFO | NumEvents : 1176916004122 | 20260506 21:35:56.912 | INFO | Analyzing chunk frames = [4628480,4628992) | 20260506 21:35:56.912 | INFO | MemInfo: { | 20260506 21:35:56.912 | INFO | "gpu" : " 17 GB", | 20260506 21:35:56.912 | INFO | "huge" : "", | 20260506 21:35:56.912 | INFO | "huge_shared" : " 27 GB", | 20260506 21:35:56.912 | INFO | "malloc" : " 14 GB", | 20260506 21:35:56.912 | INFO | "pinned" : " 29 GB" | 20260506 21:35:56.912 | INFO | } | 20260506 21:35:56.912 | INFO | Perf report for the last 5.13741 seconds: | 20260506 21:35:56.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:35:56.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:56.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:56.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:56.912 | INFO | ANALYSIS : 62.4%, 2.9, 95.0ms, 0 | 20260506 21:35:56.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:35:56.912 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260506 21:35:56.912 | INFO | PRE_HQ : 60.0%, 1.8, 57.3ms, 0 | 20260506 21:35:56.912 | INFO | BAZWRITER : 75.9%, 4.8, 187.8ms, 0 | 20260506 21:35:56.912 | INFO | BAZ_SAMPLING : 3.6%, 1.0, 1.9ms, 0 | 20260506 21:35:56.912 | INFO | Uploaded 36GiB to GPU in 2.03017s (17.7325GiB/s) | 20260506 21:35:56.912 | INFO | Downloaded 38GiB to GPU in 2.19238s (17.3327GiB/s) | 20260506 21:36:02.051 | INFO | Analyzing chunk frames = [4628992,4629504) | 20260506 21:36:02.051 | INFO | MemInfo: { | 20260506 21:36:02.051 | INFO | "gpu" : " 17 GB", | 20260506 21:36:02.051 | INFO | "huge" : "", | 20260506 21:36:02.051 | INFO | "huge_shared" : " 27 GB", | 20260506 21:36:02.051 | INFO | "malloc" : " 14 GB", | 20260506 21:36:02.051 | INFO | "pinned" : " 29 GB" | 20260506 21:36:02.051 | INFO | } | 20260506 21:36:02.051 | INFO | Perf report for the last 5.13891 seconds: | 20260506 21:36:02.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:36:02.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:02.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:02.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:02.051 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 21:36:02.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:02.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:02.051 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:36:02.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:02.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:02.051 | INFO | Uploaded 36GiB to GPU in 1.98238s (18.16GiB/s) | 20260506 21:36:02.051 | INFO | Downloaded 34GiB to GPU in 1.92864s (17.629GiB/s) | 20260506 21:36:07.192 | INFO | Analyzing chunk frames = [4629504,4630016) | 20260506 21:36:07.192 | INFO | MemInfo: { | 20260506 21:36:07.192 | INFO | "gpu" : " 17 GB", | 20260506 21:36:07.192 | INFO | "huge" : "", | 20260506 21:36:07.192 | INFO | "huge_shared" : " 27 GB", | 20260506 21:36:07.192 | INFO | "malloc" : " 14 GB", | 20260506 21:36:07.192 | INFO | "pinned" : " 29 GB" | 20260506 21:36:07.192 | INFO | } | 20260506 21:36:07.192 | INFO | Perf report for the last 5.14073 seconds: | 20260506 21:36:07.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:36:07.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:07.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:07.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:07.192 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 21:36:07.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:07.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:07.192 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:36:07.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:07.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:07.192 | INFO | Uploaded 36GiB to GPU in 1.97165s (18.2588GiB/s) | 20260506 21:36:07.192 | INFO | Downloaded 34GiB to GPU in 1.92344s (17.6767GiB/s) | 20260506 21:36:12.334 | INFO | Analyzing chunk frames = [4630016,4630528) | 20260506 21:36:12.334 | INFO | MemInfo: { | 20260506 21:36:12.334 | INFO | "gpu" : " 17 GB", | 20260506 21:36:12.334 | INFO | "huge" : "", | 20260506 21:36:12.334 | INFO | "huge_shared" : " 27 GB", | 20260506 21:36:12.334 | INFO | "malloc" : " 14 GB", | 20260506 21:36:12.334 | INFO | "pinned" : " 29 GB" | 20260506 21:36:12.334 | INFO | } | 20260506 21:36:12.334 | INFO | Perf report for the last 5.14234 seconds: | 20260506 21:36:12.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:36:12.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:12.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:12.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:12.334 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 21:36:12.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:12.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:12.334 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:36:12.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:12.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:12.334 | INFO | Uploaded 36GiB to GPU in 1.92998s (18.653GiB/s) | 20260506 21:36:12.334 | INFO | Downloaded 34GiB to GPU in 1.87518s (18.1316GiB/s) | 20260506 21:36:17.476 | INFO | Analyzing chunk frames = [4630528,4631040) | 20260506 21:36:17.476 | INFO | MemInfo: { | 20260506 21:36:17.476 | INFO | "gpu" : " 17 GB", | 20260506 21:36:17.476 | INFO | "huge" : "", | 20260506 21:36:17.476 | INFO | "huge_shared" : " 27 GB", | 20260506 21:36:17.476 | INFO | "malloc" : " 14 GB", | 20260506 21:36:17.476 | INFO | "pinned" : " 29 GB" | 20260506 21:36:17.476 | INFO | } | 20260506 21:36:17.476 | INFO | Perf report for the last 5.14179 seconds: | 20260506 21:36:17.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:36:17.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:17.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:17.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:17.476 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 21:36:17.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:17.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:17.476 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:36:17.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:17.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:17.476 | INFO | Uploaded 36GiB to GPU in 1.98527s (18.1336GiB/s) | 20260506 21:36:17.476 | INFO | Downloaded 34GiB to GPU in 1.92344s (17.6767GiB/s) | 20260506 21:36:22.614 | INFO | Analyzing chunk frames = [4631040,4631552) | 20260506 21:36:22.614 | INFO | MemInfo: { | 20260506 21:36:22.614 | INFO | "gpu" : " 17 GB", | 20260506 21:36:22.614 | INFO | "huge" : "", | 20260506 21:36:22.614 | INFO | "huge_shared" : " 27 GB", | 20260506 21:36:22.614 | INFO | "malloc" : " 14 GB", | 20260506 21:36:22.614 | INFO | "pinned" : " 29 GB" | 20260506 21:36:22.614 | INFO | } | 20260506 21:36:22.614 | INFO | Perf report for the last 5.13816 seconds: | 20260506 21:36:22.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:36:22.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:22.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:22.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:22.614 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 21:36:22.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:22.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:22.614 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:36:22.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:22.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:22.614 | INFO | Uploaded 36GiB to GPU in 2.01501s (17.8659GiB/s) | 20260506 21:36:22.614 | INFO | Downloaded 34GiB to GPU in 1.9369s (17.5538GiB/s) | 20260506 21:36:27.752 | INFO | Analyzing chunk frames = [4631552,4632064) | 20260506 21:36:27.752 | INFO | MemInfo: { | 20260506 21:36:27.752 | INFO | "gpu" : " 17 GB", | 20260506 21:36:27.752 | INFO | "huge" : "", | 20260506 21:36:27.752 | INFO | "huge_shared" : " 27 GB", | 20260506 21:36:27.752 | INFO | "malloc" : " 14 GB", | 20260506 21:36:27.752 | INFO | "pinned" : " 29 GB" | 20260506 21:36:27.752 | INFO | } | 20260506 21:36:27.752 | INFO | Perf report for the last 5.13766 seconds: | 20260506 21:36:27.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:36:27.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:27.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:27.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:27.752 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:36:27.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:27.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:27.752 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:36:27.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:27.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:27.752 | INFO | Uploaded 36GiB to GPU in 1.9788s (18.1928GiB/s) | 20260506 21:36:27.752 | INFO | Downloaded 34GiB to GPU in 2.00422s (16.9642GiB/s) | 20260506 21:36:32.891 | INFO | Analyzing chunk frames = [4632064,4632576) | 20260506 21:36:32.891 | INFO | MemInfo: { | 20260506 21:36:32.891 | INFO | "gpu" : " 17 GB", | 20260506 21:36:32.892 | INFO | "huge" : "", | 20260506 21:36:32.892 | INFO | "huge_shared" : " 27 GB", | 20260506 21:36:32.892 | INFO | "malloc" : " 14 GB", | 20260506 21:36:32.892 | INFO | "pinned" : " 29 GB" | 20260506 21:36:32.892 | INFO | } | 20260506 21:36:32.892 | INFO | Perf report for the last 5.13976 seconds: | 20260506 21:36:32.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:36:32.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:32.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:32.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:32.892 | INFO | ANALYSIS : 61.8%, 3.0, 96.5ms, 0 | 20260506 21:36:32.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:32.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:32.892 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 21:36:32.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:32.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:32.892 | INFO | Uploaded 36GiB to GPU in 2.01314s (17.8825GiB/s) | 20260506 21:36:32.892 | INFO | Downloaded 34GiB to GPU in 1.99392s (17.0519GiB/s) | 20260506 21:36:32.992 | INFO | Starting a round of Baz IO | 20260506 21:36:36.118 | INFO | Populating RTMetrics report | 20260506 21:36:36.118 | INFO | RTMetrics report populated | 20260506 21:36:36.118 | INFO | Opening tmp RTMetrics json file | 20260506 21:36:36.169 | INFO | Serializing RTMetrics to json | 20260506 21:36:36.225 | INFO | Serialization done, swapping tmp file now... | 20260506 21:36:36.232 | INFO | RTMetrics done | 20260506 21:36:36.535 | INFO | Finished a round of Baz IO: Wrote 1.941 GiB in 3.543 seconds | 20260506 21:36:36.535 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.89% / 0.00% / 66.11% | 20260506 21:36:36.535 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.59433% / 1.19209e-05% / 5.96046e-05% | 20260506 21:36:36.536 | INFO | Baz IO overall: | 20260506 21:36:36.536 | INFO | BytesWritten : 2439878459173 | 20260506 21:36:36.536 | INFO | NumEvents : 1177913101925 | 20260506 21:36:38.028 | INFO | Analyzing chunk frames = [4632576,4633088) | 20260506 21:36:38.029 | INFO | MemInfo: { | 20260506 21:36:38.029 | INFO | "gpu" : " 17 GB", | 20260506 21:36:38.029 | INFO | "huge" : "", | 20260506 21:36:38.029 | INFO | "huge_shared" : " 27 GB", | 20260506 21:36:38.029 | INFO | "malloc" : " 14 GB", | 20260506 21:36:38.029 | INFO | "pinned" : " 29 GB" | 20260506 21:36:38.029 | INFO | } | 20260506 21:36:38.029 | INFO | Perf report for the last 5.13702 seconds: | 20260506 21:36:38.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:36:38.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:38.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:38.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:38.029 | INFO | ANALYSIS : 62.3%, 2.9, 95.2ms, 0 | 20260506 21:36:38.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:38.029 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 21:36:38.029 | INFO | PRE_HQ : 59.7%, 1.8, 56.7ms, 0 | 20260506 21:36:38.029 | INFO | BAZWRITER : 68.6%, 4.8, 168.5ms, 0 | 20260506 21:36:38.029 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 21:36:38.029 | INFO | Uploaded 36GiB to GPU in 2.06684s (17.4179GiB/s) | 20260506 21:36:38.029 | INFO | Downloaded 38GiB to GPU in 2.19815s (17.2873GiB/s) | 20260506 21:36:43.164 | INFO | Analyzing chunk frames = [4633088,4633600) | 20260506 21:36:43.164 | INFO | MemInfo: { | 20260506 21:36:43.164 | INFO | "gpu" : " 17 GB", | 20260506 21:36:43.164 | INFO | "huge" : "", | 20260506 21:36:43.164 | INFO | "huge_shared" : " 27 GB", | 20260506 21:36:43.164 | INFO | "malloc" : " 14 GB", | 20260506 21:36:43.164 | INFO | "pinned" : " 29 GB" | 20260506 21:36:43.164 | INFO | } | 20260506 21:36:43.164 | INFO | Perf report for the last 5.13535 seconds: | 20260506 21:36:43.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:36:43.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:43.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:43.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:43.164 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:36:43.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:43.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:43.164 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:36:43.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:43.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:43.164 | INFO | Uploaded 36GiB to GPU in 2.01339s (17.8803GiB/s) | 20260506 21:36:43.164 | INFO | Downloaded 34GiB to GPU in 1.92813s (17.6337GiB/s) | 20260506 21:36:48.302 | INFO | Analyzing chunk frames = [4633600,4634112) | 20260506 21:36:48.302 | INFO | MemInfo: { | 20260506 21:36:48.302 | INFO | "gpu" : " 17 GB", | 20260506 21:36:48.302 | INFO | "huge" : "", | 20260506 21:36:48.302 | INFO | "huge_shared" : " 27 GB", | 20260506 21:36:48.302 | INFO | "malloc" : " 14 GB", | 20260506 21:36:48.303 | INFO | "pinned" : " 29 GB" | 20260506 21:36:48.303 | INFO | } | 20260506 21:36:48.303 | INFO | Perf report for the last 5.13855 seconds: | 20260506 21:36:48.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:36:48.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:48.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:48.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:48.303 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260506 21:36:48.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:48.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:48.303 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:36:48.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:48.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:48.303 | INFO | Uploaded 36GiB to GPU in 1.98409s (18.1443GiB/s) | 20260506 21:36:48.303 | INFO | Downloaded 34GiB to GPU in 1.91906s (17.717GiB/s) | 20260506 21:36:53.444 | INFO | Analyzing chunk frames = [4634112,4634624) | 20260506 21:36:53.444 | INFO | MemInfo: { | 20260506 21:36:53.444 | INFO | "gpu" : " 17 GB", | 20260506 21:36:53.444 | INFO | "huge" : "", | 20260506 21:36:53.444 | INFO | "huge_shared" : " 27 GB", | 20260506 21:36:53.444 | INFO | "malloc" : " 14 GB", | 20260506 21:36:53.444 | INFO | "pinned" : " 29 GB" | 20260506 21:36:53.444 | INFO | } | 20260506 21:36:53.444 | INFO | Perf report for the last 5.14169 seconds: | 20260506 21:36:53.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:36:53.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:53.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:53.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:53.444 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 21:36:53.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:53.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:53.444 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:36:53.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:53.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:53.445 | INFO | Uploaded 36GiB to GPU in 1.93031s (18.6499GiB/s) | 20260506 21:36:53.445 | INFO | Downloaded 34GiB to GPU in 1.86995s (18.1823GiB/s) | 20260506 21:36:58.589 | INFO | Analyzing chunk frames = [4634624,4635136) | 20260506 21:36:58.589 | INFO | MemInfo: { | 20260506 21:36:58.589 | INFO | "gpu" : " 17 GB", | 20260506 21:36:58.589 | INFO | "huge" : "", | 20260506 21:36:58.589 | INFO | "huge_shared" : " 27 GB", | 20260506 21:36:58.589 | INFO | "malloc" : " 14 GB", | 20260506 21:36:58.589 | INFO | "pinned" : " 29 GB" | 20260506 21:36:58.589 | INFO | } | 20260506 21:36:58.589 | INFO | Perf report for the last 5.14465 seconds: | 20260506 21:36:58.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:36:58.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:58.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:58.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:58.589 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 21:36:58.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:58.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:58.589 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:36:58.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:58.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:36:58.590 | INFO | Uploaded 36GiB to GPU in 2.02257s (17.7992GiB/s) | 20260506 21:36:58.590 | INFO | Downloaded 34GiB to GPU in 1.9139s (17.7648GiB/s) | 20260506 21:37:03.729 | INFO | Analyzing chunk frames = [4635136,4635648) | 20260506 21:37:03.729 | INFO | MemInfo: { | 20260506 21:37:03.729 | INFO | "gpu" : " 17 GB", | 20260506 21:37:03.729 | INFO | "huge" : "", | 20260506 21:37:03.729 | INFO | "huge_shared" : " 27 GB", | 20260506 21:37:03.729 | INFO | "malloc" : " 14 GB", | 20260506 21:37:03.729 | INFO | "pinned" : " 29 GB" | 20260506 21:37:03.729 | INFO | } | 20260506 21:37:03.729 | INFO | Perf report for the last 5.14054 seconds: | 20260506 21:37:03.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:37:03.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:03.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:03.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:03.730 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:37:03.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:03.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:03.730 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:37:03.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:03.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:03.730 | INFO | Uploaded 36GiB to GPU in 2.01753s (17.8436GiB/s) | 20260506 21:37:03.730 | INFO | Downloaded 34GiB to GPU in 1.94611s (17.4707GiB/s) | 20260506 21:37:08.869 | INFO | Analyzing chunk frames = [4635648,4636160) | 20260506 21:37:08.869 | INFO | MemInfo: { | 20260506 21:37:08.869 | INFO | "gpu" : " 17 GB", | 20260506 21:37:08.869 | INFO | "huge" : "", | 20260506 21:37:08.869 | INFO | "huge_shared" : " 27 GB", | 20260506 21:37:08.869 | INFO | "malloc" : " 14 GB", | 20260506 21:37:08.869 | INFO | "pinned" : " 29 GB" | 20260506 21:37:08.869 | INFO | } | 20260506 21:37:08.869 | INFO | Perf report for the last 5.13988 seconds: | 20260506 21:37:08.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:37:08.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:08.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:08.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:08.869 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:37:08.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:08.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:08.869 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 21:37:08.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:08.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:08.870 | INFO | Uploaded 36GiB to GPU in 1.98048s (18.1774GiB/s) | 20260506 21:37:08.870 | INFO | Downloaded 34GiB to GPU in 2.00293s (16.9751GiB/s) | 20260506 21:37:14.005 | INFO | Analyzing chunk frames = [4636160,4636672) | 20260506 21:37:14.005 | INFO | MemInfo: { | 20260506 21:37:14.005 | INFO | "gpu" : " 17 GB", | 20260506 21:37:14.005 | INFO | "huge" : "", | 20260506 21:37:14.005 | INFO | "huge_shared" : " 27 GB", | 20260506 21:37:14.005 | INFO | "malloc" : " 14 GB", | 20260506 21:37:14.005 | INFO | "pinned" : " 29 GB" | 20260506 21:37:14.005 | INFO | } | 20260506 21:37:14.005 | INFO | Perf report for the last 5.13543 seconds: | 20260506 21:37:14.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:37:14.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:14.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:14.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:14.005 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260506 21:37:14.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:14.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:14.005 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 21:37:14.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:14.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:14.005 | INFO | Uploaded 36GiB to GPU in 2.03243s (17.7128GiB/s) | 20260506 21:37:14.005 | INFO | Downloaded 34GiB to GPU in 1.98614s (17.1186GiB/s) | 20260506 21:37:14.005 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:37:14.005 | INFO | Total mem usage: 18 GiB | 20260506 21:37:14.005 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:37:14.005 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -16.9%) | 20260506 21:37:14.005 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:37:14.005 | INFO | High Water Breakdown: | 20260506 21:37:14.005 | INFO | 61.4% : 9.45 GiB ( 82.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:37:14.005 | INFO | 38.5% : 5.93 GiB ( 88.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:37:14.005 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:37:14.005 | INFO | ------------------------------------------ | 20260506 21:37:14.005 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:37:14.005 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:37:14.005 | INFO | Total mem usage: 26.5 GiB | 20260506 21:37:14.005 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:37:14.005 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:37:14.005 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:37:14.005 | INFO | High Water Breakdown: | 20260506 21:37:14.005 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:37:14.005 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:37:14.005 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:37:14.005 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:37:14.005 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:37:14.005 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:37:14.006 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:37:14.006 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:37:14.006 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:37:14.006 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:37:14.006 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:37:14.006 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:37:14.006 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:37:14.006 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:37:14.006 | INFO | ------------------------------------------ | 20260506 21:37:14.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:37:14.006 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:37:14.006 | INFO | Total mem usage: 25.6 GiB | 20260506 21:37:14.006 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:37:14.006 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:37:14.006 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:37:14.006 | INFO | High Water Breakdown: | 20260506 21:37:14.006 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:37:14.006 | INFO | ------------------------------------------ | 20260506 21:37:14.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:37:14.006 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:37:14.006 | INFO | Total mem usage: 15.7 GiB | 20260506 21:37:14.006 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:37:14.006 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:37:14.006 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:37:14.006 | INFO | High Water Breakdown: | 20260506 21:37:14.006 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:37:14.006 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:37:14.006 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:37:14.006 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:37:14.006 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:37:14.006 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:37:14.006 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:37:14.006 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:37:14.006 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:37:14.006 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:37:14.006 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:37:14.006 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:37:14.006 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:37:14.006 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:37:14.006 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:37:14.006 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:37:14.006 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:37:14.006 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:37:14.006 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:37:14.006 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:37:14.006 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:37:14.006 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:37:14.006 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:37:14.006 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:37:14.006 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:37:14.006 | INFO | ------------------------------------------ | 20260506 21:37:14.006 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:37:14.006 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:37:14.006 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:37:14.006 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:37:14.006 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.59e+01, 1.35e+01 / 2.39e+00 | 20260506 21:37:14.006 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 9.77e-03 / 6.46e-03 | 20260506 21:37:14.006 | INFO | PerfReport: 11.65%, Baseline -- 3.63e+00 / 4.40e+00, 5.55e-01 / 3.68e-01 | 20260506 21:37:14.006 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.45e+00, 1.81e-01 / 1.42e-01 | 20260506 21:37:14.006 | INFO | PerfReport: 16.08%, DME -- 4.01e+01 / 4.49e+01, 4.32e+00 / 2.72e+00 | 20260506 21:37:14.006 | INFO | PerfReport: 57.24%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.39e+00 / 1.53e+00 | 20260506 21:37:14.006 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.21e+00 / 2.80e+00, 3.75e-01 / 1.56e-01 | 20260506 21:37:14.006 | INFO | PerfReport: 3.26%, Metrics -- 1.01e+00 / 1.34e+00, 2.67e-01 / 1.20e-01 | 20260506 21:37:14.006 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.48e-01 / 3.95e-01, 5.35e-02 / 2.33e-02 | 20260506 21:37:14.006 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:37:14.006 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:37:14.006 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:37:14.006 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 21:37:14.006 | INFO | PerfReport: 21.51%, Upload -- 2.08e+01 / 0.00e+00, 6.92e+00 / 0.00e+00 | 20260506 21:37:14.006 | INFO | PerfReport: 21.23%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 21:37:14.006 | INFO | PerfReport: 32.19%, Compute -- 3.11e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 21:37:14.006 | INFO | PerfReport: 1.34%, LockUpload -- 1.30e+00 / 0.00e+00, 5.16e+00 / 0.00e+00 | 20260506 21:37:14.006 | INFO | PerfReport: 1.20%, LockDownload -- 1.16e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260506 21:37:14.006 | INFO | PerfReport: 22.52%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:37:14.006 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260506 21:37:14.121 | INFO | Starting a round of Baz IO | 20260506 21:37:17.243 | INFO | Populating RTMetrics report | 20260506 21:37:17.244 | INFO | RTMetrics report populated | 20260506 21:37:17.244 | INFO | Opening tmp RTMetrics json file | 20260506 21:37:17.288 | INFO | Serializing RTMetrics to json | 20260506 21:37:17.316 | INFO | Serialization done, swapping tmp file now... | 20260506 21:37:17.330 | INFO | RTMetrics done | 20260506 21:37:17.696 | INFO | Finished a round of Baz IO: Wrote 1.941 GiB in 3.575 seconds | 20260506 21:37:17.696 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.89% / 0.00% / 66.11% | 20260506 21:37:17.696 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.59338% / 7.94729e-06% / 5.5631e-05% | 20260506 21:37:17.696 | INFO | Baz IO overall: | 20260506 21:37:17.696 | INFO | BytesWritten : 2441962383753 | 20260506 21:37:17.696 | INFO | NumEvents : 1178910145446 | 20260506 21:37:19.145 | INFO | Analyzing chunk frames = [4636672,4637184) | 20260506 21:37:19.145 | INFO | MemInfo: { | 20260506 21:37:19.145 | INFO | "gpu" : " 17 GB", | 20260506 21:37:19.145 | INFO | "huge" : "", | 20260506 21:37:19.145 | INFO | "huge_shared" : " 27 GB", | 20260506 21:37:19.145 | INFO | "malloc" : " 14 GB", | 20260506 21:37:19.145 | INFO | "pinned" : " 29 GB" | 20260506 21:37:19.145 | INFO | } | 20260506 21:37:19.145 | INFO | Perf report for the last 5.14043 seconds: | 20260506 21:37:19.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:37:19.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:19.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:19.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:19.145 | INFO | ANALYSIS : 62.4%, 2.9, 95.3ms, 0 | 20260506 21:37:19.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:19.145 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260506 21:37:19.145 | INFO | PRE_HQ : 59.9%, 1.8, 57.1ms, 0 | 20260506 21:37:19.145 | INFO | BAZWRITER : 69.3%, 4.8, 171.0ms, 0 | 20260506 21:37:19.145 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 21:37:19.145 | INFO | Uploaded 36GiB to GPU in 2.05192s (17.5446GiB/s) | 20260506 21:37:19.145 | INFO | Downloaded 38GiB to GPU in 2.20058s (17.2682GiB/s) | 20260506 21:37:24.285 | INFO | Analyzing chunk frames = [4637184,4637696) | 20260506 21:37:24.285 | INFO | MemInfo: { | 20260506 21:37:24.285 | INFO | "gpu" : " 17 GB", | 20260506 21:37:24.285 | INFO | "huge" : "", | 20260506 21:37:24.285 | INFO | "huge_shared" : " 27 GB", | 20260506 21:37:24.285 | INFO | "malloc" : " 14 GB", | 20260506 21:37:24.285 | INFO | "pinned" : " 29 GB" | 20260506 21:37:24.285 | INFO | } | 20260506 21:37:24.285 | INFO | Perf report for the last 5.14011 seconds: | 20260506 21:37:24.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:37:24.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:24.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:24.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:24.285 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:37:24.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:24.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:24.285 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:37:24.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:24.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:24.286 | INFO | Uploaded 36GiB to GPU in 2.0166s (17.8519GiB/s) | 20260506 21:37:24.286 | INFO | Downloaded 34GiB to GPU in 1.92369s (17.6744GiB/s) | 20260506 21:37:29.420 | INFO | Analyzing chunk frames = [4637696,4638208) | 20260506 21:37:29.420 | INFO | MemInfo: { | 20260506 21:37:29.420 | INFO | "gpu" : " 17 GB", | 20260506 21:37:29.420 | INFO | "huge" : "", | 20260506 21:37:29.420 | INFO | "huge_shared" : " 27 GB", | 20260506 21:37:29.420 | INFO | "malloc" : " 14 GB", | 20260506 21:37:29.420 | INFO | "pinned" : " 29 GB" | 20260506 21:37:29.420 | INFO | } | 20260506 21:37:29.420 | INFO | Perf report for the last 5.13493 seconds: | 20260506 21:37:29.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:37:29.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:29.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:29.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:29.420 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 21:37:29.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:29.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:29.420 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:37:29.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:29.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:29.421 | INFO | Uploaded 36GiB to GPU in 1.97714s (18.2081GiB/s) | 20260506 21:37:29.421 | INFO | Downloaded 34GiB to GPU in 1.91584s (17.7468GiB/s) | 20260506 21:37:34.567 | INFO | Analyzing chunk frames = [4638208,4638720) | 20260506 21:37:34.567 | INFO | MemInfo: { | 20260506 21:37:34.567 | INFO | "gpu" : " 17 GB", | 20260506 21:37:34.567 | INFO | "huge" : "", | 20260506 21:37:34.567 | INFO | "huge_shared" : " 27 GB", | 20260506 21:37:34.567 | INFO | "malloc" : " 14 GB", | 20260506 21:37:34.567 | INFO | "pinned" : " 29 GB" | 20260506 21:37:34.567 | INFO | } | 20260506 21:37:34.567 | INFO | Perf report for the last 5.14685 seconds: | 20260506 21:37:34.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:37:34.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:34.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:34.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:34.567 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 21:37:34.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:34.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:34.567 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:37:34.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:34.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:34.567 | INFO | Uploaded 36GiB to GPU in 1.96067s (18.3611GiB/s) | 20260506 21:37:34.568 | INFO | Downloaded 34GiB to GPU in 1.871s (18.1721GiB/s) | 20260506 21:37:39.704 | INFO | Analyzing chunk frames = [4638720,4639232) | 20260506 21:37:39.704 | INFO | MemInfo: { | 20260506 21:37:39.704 | INFO | "gpu" : " 17 GB", | 20260506 21:37:39.704 | INFO | "huge" : "", | 20260506 21:37:39.704 | INFO | "huge_shared" : " 27 GB", | 20260506 21:37:39.704 | INFO | "malloc" : " 14 GB", | 20260506 21:37:39.704 | INFO | "pinned" : " 29 GB" | 20260506 21:37:39.704 | INFO | } | 20260506 21:37:39.704 | INFO | Perf report for the last 5.13673 seconds: | 20260506 21:37:39.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:37:39.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:39.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:39.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:39.704 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 21:37:39.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:39.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:39.704 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:37:39.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:39.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:39.704 | INFO | Uploaded 36GiB to GPU in 1.98732s (18.1148GiB/s) | 20260506 21:37:39.704 | INFO | Downloaded 34GiB to GPU in 1.92678s (17.646GiB/s) | 20260506 21:37:44.841 | INFO | Analyzing chunk frames = [4639232,4639744) | 20260506 21:37:44.841 | INFO | MemInfo: { | 20260506 21:37:44.841 | INFO | "gpu" : " 17 GB", | 20260506 21:37:44.841 | INFO | "huge" : "", | 20260506 21:37:44.841 | INFO | "huge_shared" : " 27 GB", | 20260506 21:37:44.841 | INFO | "malloc" : " 14 GB", | 20260506 21:37:44.841 | INFO | "pinned" : " 29 GB" | 20260506 21:37:44.841 | INFO | } | 20260506 21:37:44.841 | INFO | Perf report for the last 5.13687 seconds: | 20260506 21:37:44.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:37:44.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:44.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:44.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:44.841 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 21:37:44.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:44.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:44.841 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:37:44.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:44.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:44.841 | INFO | Uploaded 36GiB to GPU in 1.99972s (18.0025GiB/s) | 20260506 21:37:44.841 | INFO | Downloaded 34GiB to GPU in 1.9366s (17.5565GiB/s) | 20260506 21:37:49.983 | INFO | Analyzing chunk frames = [4639744,4640256) | 20260506 21:37:49.983 | INFO | MemInfo: { | 20260506 21:37:49.983 | INFO | "gpu" : " 17 GB", | 20260506 21:37:49.983 | INFO | "huge" : "", | 20260506 21:37:49.983 | INFO | "huge_shared" : " 27 GB", | 20260506 21:37:49.983 | INFO | "malloc" : " 14 GB", | 20260506 21:37:49.983 | INFO | "pinned" : " 29 GB" | 20260506 21:37:49.983 | INFO | } | 20260506 21:37:49.983 | INFO | Perf report for the last 5.14224 seconds: | 20260506 21:37:49.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:37:49.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:49.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:49.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:49.983 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 21:37:49.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:49.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:49.983 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 21:37:49.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:49.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:49.983 | INFO | Uploaded 36GiB to GPU in 1.97833s (18.1972GiB/s) | 20260506 21:37:49.983 | INFO | Downloaded 34GiB to GPU in 2.00444s (16.9624GiB/s) | 20260506 21:37:55.119 | INFO | Analyzing chunk frames = [4640256,4640768) | 20260506 21:37:55.119 | INFO | MemInfo: { | 20260506 21:37:55.119 | INFO | "gpu" : " 17 GB", | 20260506 21:37:55.119 | INFO | "huge" : "", | 20260506 21:37:55.119 | INFO | "huge_shared" : " 27 GB", | 20260506 21:37:55.119 | INFO | "malloc" : " 14 GB", | 20260506 21:37:55.119 | INFO | "pinned" : " 29 GB" | 20260506 21:37:55.119 | INFO | } | 20260506 21:37:55.119 | INFO | Perf report for the last 5.13623 seconds: | 20260506 21:37:55.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:37:55.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:55.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:55.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:55.119 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 21:37:55.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:55.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:55.119 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 21:37:55.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:55.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:37:55.120 | INFO | Uploaded 36GiB to GPU in 1.99995s (18.0004GiB/s) | 20260506 21:37:55.120 | INFO | Downloaded 34GiB to GPU in 2.00422s (16.9642GiB/s) | 20260506 21:37:55.224 | INFO | Starting a round of Baz IO | 20260506 21:37:58.357 | INFO | Populating RTMetrics report | 20260506 21:37:58.357 | INFO | RTMetrics report populated | 20260506 21:37:58.357 | INFO | Opening tmp RTMetrics json file | 20260506 21:37:58.373 | INFO | Serializing RTMetrics to json | 20260506 21:37:58.376 | INFO | Serialization done, swapping tmp file now... | 20260506 21:37:58.381 | INFO | RTMetrics done | 20260506 21:37:58.558 | INFO | Finished a round of Baz IO: Wrote 1.939 GiB in 3.333 seconds | 20260506 21:37:58.558 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.89% / 0.00% / 66.11% | 20260506 21:37:58.558 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.591675% / 3.97364e-06% / 6.75519e-05% | 20260506 21:37:58.558 | INFO | Baz IO overall: | 20260506 21:37:58.558 | INFO | BytesWritten : 2444044284506 | 20260506 21:37:58.558 | INFO | NumEvents : 1179905531859 | 20260506 21:38:00.256 | INFO | Analyzing chunk frames = [4640768,4641280) | 20260506 21:38:00.256 | INFO | MemInfo: { | 20260506 21:38:00.256 | INFO | "gpu" : " 17 GB", | 20260506 21:38:00.256 | INFO | "huge" : "", | 20260506 21:38:00.256 | INFO | "huge_shared" : " 27 GB", | 20260506 21:38:00.256 | INFO | "malloc" : " 14 GB", | 20260506 21:38:00.256 | INFO | "pinned" : " 29 GB" | 20260506 21:38:00.256 | INFO | } | 20260506 21:38:00.256 | INFO | Perf report for the last 5.13683 seconds: | 20260506 21:38:00.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:38:00.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:00.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:00.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:00.256 | INFO | ANALYSIS : 62.5%, 2.9, 95.3ms, 0 | 20260506 21:38:00.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:00.256 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 21:38:00.256 | INFO | PRE_HQ : 60.0%, 1.8, 57.1ms, 0 | 20260506 21:38:00.256 | INFO | BAZWRITER : 64.5%, 4.8, 158.8ms, 0 | 20260506 21:38:00.256 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 21:38:00.256 | INFO | Uploaded 36GiB to GPU in 2.02904s (17.7424GiB/s) | 20260506 21:38:00.256 | INFO | Downloaded 38GiB to GPU in 2.2083s (17.2078GiB/s) | 20260506 21:38:05.393 | INFO | Analyzing chunk frames = [4641280,4641792) | 20260506 21:38:05.393 | INFO | MemInfo: { | 20260506 21:38:05.393 | INFO | "gpu" : " 17 GB", | 20260506 21:38:05.393 | INFO | "huge" : "", | 20260506 21:38:05.393 | INFO | "huge_shared" : " 27 GB", | 20260506 21:38:05.393 | INFO | "malloc" : " 14 GB", | 20260506 21:38:05.393 | INFO | "pinned" : " 29 GB" | 20260506 21:38:05.393 | INFO | } | 20260506 21:38:05.393 | INFO | Perf report for the last 5.13666 seconds: | 20260506 21:38:05.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:38:05.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:05.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:05.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:05.393 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:38:05.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:05.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:05.393 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:38:05.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:05.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:05.393 | INFO | Uploaded 36GiB to GPU in 1.99861s (18.0126GiB/s) | 20260506 21:38:05.393 | INFO | Downloaded 34GiB to GPU in 1.92561s (17.6568GiB/s) | 20260506 21:38:10.538 | INFO | Analyzing chunk frames = [4641792,4642304) | 20260506 21:38:10.538 | INFO | MemInfo: { | 20260506 21:38:10.538 | INFO | "gpu" : " 17 GB", | 20260506 21:38:10.539 | INFO | "huge" : "", | 20260506 21:38:10.539 | INFO | "huge_shared" : " 27 GB", | 20260506 21:38:10.539 | INFO | "malloc" : " 14 GB", | 20260506 21:38:10.539 | INFO | "pinned" : " 29 GB" | 20260506 21:38:10.539 | INFO | } | 20260506 21:38:10.539 | INFO | Perf report for the last 5.14577 seconds: | 20260506 21:38:10.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:38:10.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:10.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:10.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:10.539 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 21:38:10.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:10.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:10.539 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:38:10.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:10.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:10.539 | INFO | Uploaded 36GiB to GPU in 1.96482s (18.3223GiB/s) | 20260506 21:38:10.539 | INFO | Downloaded 34GiB to GPU in 1.9236s (17.6752GiB/s) | 20260506 21:38:15.679 | INFO | Analyzing chunk frames = [4642304,4642816) | 20260506 21:38:15.679 | INFO | MemInfo: { | 20260506 21:38:15.679 | INFO | "gpu" : " 17 GB", | 20260506 21:38:15.679 | INFO | "huge" : "", | 20260506 21:38:15.679 | INFO | "huge_shared" : " 27 GB", | 20260506 21:38:15.679 | INFO | "malloc" : " 14 GB", | 20260506 21:38:15.679 | INFO | "pinned" : " 29 GB" | 20260506 21:38:15.679 | INFO | } | 20260506 21:38:15.679 | INFO | Perf report for the last 5.14079 seconds: | 20260506 21:38:15.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:38:15.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:15.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:15.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:15.679 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 21:38:15.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:15.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:15.679 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:38:15.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:15.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:15.680 | INFO | Uploaded 36GiB to GPU in 1.94292s (18.5288GiB/s) | 20260506 21:38:15.680 | INFO | Downloaded 34GiB to GPU in 1.8741s (18.142GiB/s) | 20260506 21:38:20.824 | INFO | Analyzing chunk frames = [4642816,4643328) | 20260506 21:38:20.824 | INFO | MemInfo: { | 20260506 21:38:20.824 | INFO | "gpu" : " 17 GB", | 20260506 21:38:20.824 | INFO | "huge" : "", | 20260506 21:38:20.824 | INFO | "huge_shared" : " 27 GB", | 20260506 21:38:20.824 | INFO | "malloc" : " 14 GB", | 20260506 21:38:20.824 | INFO | "pinned" : " 29 GB" | 20260506 21:38:20.824 | INFO | } | 20260506 21:38:20.824 | INFO | Perf report for the last 5.14463 seconds: | 20260506 21:38:20.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:38:20.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:20.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:20.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:20.824 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 21:38:20.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:20.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:20.824 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:38:20.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:20.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:20.824 | INFO | Uploaded 36GiB to GPU in 1.97794s (18.2007GiB/s) | 20260506 21:38:20.824 | INFO | Downloaded 34GiB to GPU in 1.92859s (17.6294GiB/s) | 20260506 21:38:25.960 | INFO | Analyzing chunk frames = [4643328,4643840) | 20260506 21:38:25.960 | INFO | MemInfo: { | 20260506 21:38:25.960 | INFO | "gpu" : " 17 GB", | 20260506 21:38:25.960 | INFO | "huge" : "", | 20260506 21:38:25.960 | INFO | "huge_shared" : " 27 GB", | 20260506 21:38:25.960 | INFO | "malloc" : " 14 GB", | 20260506 21:38:25.960 | INFO | "pinned" : " 29 GB" | 20260506 21:38:25.960 | INFO | } | 20260506 21:38:25.960 | INFO | Perf report for the last 5.13631 seconds: | 20260506 21:38:25.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:38:25.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:25.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:25.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:25.960 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 21:38:25.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:25.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:25.960 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:38:25.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:25.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:25.961 | INFO | Uploaded 36GiB to GPU in 2.01279s (17.8857GiB/s) | 20260506 21:38:25.961 | INFO | Downloaded 34GiB to GPU in 1.94134s (17.5137GiB/s) | 20260506 21:38:31.096 | INFO | Analyzing chunk frames = [4643840,4644352) | 20260506 21:38:31.096 | INFO | MemInfo: { | 20260506 21:38:31.096 | INFO | "gpu" : " 17 GB", | 20260506 21:38:31.096 | INFO | "huge" : "", | 20260506 21:38:31.096 | INFO | "huge_shared" : " 27 GB", | 20260506 21:38:31.096 | INFO | "malloc" : " 14 GB", | 20260506 21:38:31.096 | INFO | "pinned" : " 29 GB" | 20260506 21:38:31.096 | INFO | } | 20260506 21:38:31.096 | INFO | Perf report for the last 5.13598 seconds: | 20260506 21:38:31.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:38:31.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:31.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:31.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:31.096 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 21:38:31.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:31.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:31.097 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 21:38:31.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:31.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:31.097 | INFO | Uploaded 36GiB to GPU in 1.94993s (18.4622GiB/s) | 20260506 21:38:31.097 | INFO | Downloaded 34GiB to GPU in 1.99127s (17.0745GiB/s) | 20260506 21:38:36.236 | INFO | Analyzing chunk frames = [4644352,4644864) | 20260506 21:38:36.236 | INFO | MemInfo: { | 20260506 21:38:36.236 | INFO | "gpu" : " 17 GB", | 20260506 21:38:36.236 | INFO | "huge" : "", | 20260506 21:38:36.236 | INFO | "huge_shared" : " 27 GB", | 20260506 21:38:36.236 | INFO | "malloc" : " 14 GB", | 20260506 21:38:36.236 | INFO | "pinned" : " 29 GB" | 20260506 21:38:36.236 | INFO | } | 20260506 21:38:36.236 | INFO | Perf report for the last 5.13951 seconds: | 20260506 21:38:36.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:38:36.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:36.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:36.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:36.236 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 21:38:36.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:36.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:36.236 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 21:38:36.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:36.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:36.236 | INFO | Uploaded 36GiB to GPU in 1.98104s (18.1723GiB/s) | 20260506 21:38:36.236 | INFO | Downloaded 34GiB to GPU in 1.99979s (17.0018GiB/s) | 20260506 21:38:36.236 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:38:36.236 | INFO | Total mem usage: 18 GiB | 20260506 21:38:36.236 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:38:36.236 | INFO | Recent High Water Marks Sum: 16.6 GiB (Savings: -8.75%) | 20260506 21:38:36.236 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:38:36.236 | INFO | High Water Breakdown: | 20260506 21:38:36.236 | INFO | 62.4% : 10.3 GiB ( 90.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:38:36.236 | INFO | 37.5% : 6.21 GiB ( 92.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:38:36.236 | INFO | 0.12% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:38:36.236 | INFO | ------------------------------------------ | 20260506 21:38:36.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:38:36.236 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:38:36.236 | INFO | Total mem usage: 26.5 GiB | 20260506 21:38:36.236 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:38:36.236 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:38:36.236 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:38:36.236 | INFO | High Water Breakdown: | 20260506 21:38:36.236 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:38:36.236 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:38:36.236 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:38:36.236 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:38:36.236 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:38:36.236 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:38:36.236 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:38:36.236 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:38:36.236 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:38:36.236 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:38:36.236 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:38:36.236 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:38:36.236 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:38:36.236 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:38:36.236 | INFO | ------------------------------------------ | 20260506 21:38:36.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:38:36.237 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:38:36.237 | INFO | Total mem usage: 25.6 GiB | 20260506 21:38:36.237 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:38:36.237 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:38:36.237 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:38:36.237 | INFO | High Water Breakdown: | 20260506 21:38:36.237 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:38:36.237 | INFO | ------------------------------------------ | 20260506 21:38:36.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:38:36.237 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:38:36.237 | INFO | Total mem usage: 15.7 GiB | 20260506 21:38:36.237 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:38:36.237 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:38:36.237 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:38:36.237 | INFO | High Water Breakdown: | 20260506 21:38:36.237 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:38:36.237 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:38:36.237 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:38:36.237 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:38:36.237 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:38:36.237 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:38:36.237 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:38:36.237 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:38:36.237 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:38:36.237 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:38:36.237 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:38:36.237 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:38:36.237 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:38:36.237 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:38:36.237 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:38:36.237 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:38:36.237 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:38:36.237 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:38:36.237 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:38:36.237 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:38:36.237 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:38:36.237 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:38:36.237 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:38:36.237 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:38:36.237 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:38:36.237 | INFO | ------------------------------------------ | 20260506 21:38:36.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:38:36.237 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:38:36.237 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:38:36.237 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:38:36.237 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.58e+01, 1.35e+01 / 2.47e+00 | 20260506 21:38:36.237 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 9.70e-03 / 6.59e-03 | 20260506 21:38:36.237 | INFO | PerfReport: 11.66%, Baseline -- 3.63e+00 / 4.38e+00, 5.53e-01 / 3.61e-01 | 20260506 21:38:36.237 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.45e+00, 1.82e-01 / 1.42e-01 | 20260506 21:38:36.237 | INFO | PerfReport: 16.05%, DME -- 3.99e+01 / 4.46e+01, 4.27e+00 / 2.89e+00 | 20260506 21:38:36.237 | INFO | PerfReport: 57.22%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.36e+00 / 1.83e+00 | 20260506 21:38:36.237 | INFO | PerfReport: 7.13%, PulseAccumulating -- 2.22e+00 / 2.77e+00, 3.83e-01 / 2.02e-01 | 20260506 21:38:36.237 | INFO | PerfReport: 3.26%, Metrics -- 1.01e+00 / 1.36e+00, 2.68e-01 / 1.76e-01 | 20260506 21:38:36.237 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.47e-01 / 3.95e-01, 5.39e-02 / 1.69e-02 | 20260506 21:38:36.237 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:38:36.237 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:38:36.237 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:38:36.237 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 21:38:36.237 | INFO | PerfReport: 21.45%, Upload -- 2.07e+01 / 0.00e+00, 6.84e+00 / 0.00e+00 | 20260506 21:38:36.237 | INFO | PerfReport: 21.28%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 21:38:36.237 | INFO | PerfReport: 32.20%, Compute -- 3.11e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 21:38:36.237 | INFO | PerfReport: 1.23%, LockUpload -- 1.19e+00 / 0.00e+00, 4.75e+00 / 0.00e+00 | 20260506 21:38:36.237 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260506 21:38:36.237 | INFO | PerfReport: 22.55%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 21:38:36.237 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260506 21:38:36.335 | INFO | Starting a round of Baz IO | 20260506 21:38:39.471 | INFO | Populating RTMetrics report | 20260506 21:38:39.472 | INFO | RTMetrics report populated | 20260506 21:38:39.472 | INFO | Opening tmp RTMetrics json file | 20260506 21:38:39.683 | INFO | Serializing RTMetrics to json | 20260506 21:38:39.721 | INFO | Serialization done, swapping tmp file now... | 20260506 21:38:39.725 | INFO | RTMetrics done | 20260506 21:38:39.944 | INFO | Finished a round of Baz IO: Wrote 1.939 GiB in 3.608 seconds | 20260506 21:38:39.944 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.88% / 0.00% / 66.11% | 20260506 21:38:39.944 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.589112% / 3.97364e-06% / 5.5631e-05% | 20260506 21:38:39.944 | INFO | Baz IO overall: | 20260506 21:38:39.944 | INFO | BytesWritten : 2446126314366 | 20260506 21:38:39.944 | INFO | NumEvents : 1180901450682 | 20260506 21:38:41.369 | INFO | Analyzing chunk frames = [4644864,4645376) | 20260506 21:38:41.369 | INFO | MemInfo: { | 20260506 21:38:41.369 | INFO | "gpu" : " 17 GB", | 20260506 21:38:41.369 | INFO | "huge" : "", | 20260506 21:38:41.369 | INFO | "huge_shared" : " 27 GB", | 20260506 21:38:41.369 | INFO | "malloc" : " 14 GB", | 20260506 21:38:41.369 | INFO | "pinned" : " 29 GB" | 20260506 21:38:41.369 | INFO | } | 20260506 21:38:41.369 | INFO | Perf report for the last 5.13281 seconds: | 20260506 21:38:41.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:38:41.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:41.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:41.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:41.369 | INFO | ANALYSIS : 62.5%, 2.9, 95.5ms, 0 | 20260506 21:38:41.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:41.369 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260506 21:38:41.369 | INFO | PRE_HQ : 60.0%, 1.8, 57.0ms, 0 | 20260506 21:38:41.369 | INFO | BAZWRITER : 69.9%, 4.7, 168.3ms, 0 | 20260506 21:38:41.369 | INFO | BAZ_SAMPLING : 3.1%, 1.0, 1.6ms, 0 | 20260506 21:38:41.369 | INFO | Uploaded 36GiB to GPU in 2.02657s (17.764GiB/s) | 20260506 21:38:41.369 | INFO | Downloaded 38GiB to GPU in 2.19894s (17.281GiB/s) | 20260506 21:38:46.511 | INFO | Analyzing chunk frames = [4645376,4645888) | 20260506 21:38:46.511 | INFO | MemInfo: { | 20260506 21:38:46.511 | INFO | "gpu" : " 17 GB", | 20260506 21:38:46.511 | INFO | "huge" : "", | 20260506 21:38:46.511 | INFO | "huge_shared" : " 27 GB", | 20260506 21:38:46.511 | INFO | "malloc" : " 14 GB", | 20260506 21:38:46.511 | INFO | "pinned" : " 29 GB" | 20260506 21:38:46.511 | INFO | } | 20260506 21:38:46.511 | INFO | Perf report for the last 5.14231 seconds: | 20260506 21:38:46.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:38:46.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:46.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:46.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:46.511 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 21:38:46.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:46.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:46.511 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:38:46.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:46.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:46.511 | INFO | Uploaded 36GiB to GPU in 2.00962s (17.9139GiB/s) | 20260506 21:38:46.511 | INFO | Downloaded 34GiB to GPU in 1.93349s (17.5848GiB/s) | 20260506 21:38:51.659 | INFO | Analyzing chunk frames = [4645888,4646400) | 20260506 21:38:51.659 | INFO | MemInfo: { | 20260506 21:38:51.659 | INFO | "gpu" : " 17 GB", | 20260506 21:38:51.659 | INFO | "huge" : "", | 20260506 21:38:51.659 | INFO | "huge_shared" : " 27 GB", | 20260506 21:38:51.659 | INFO | "malloc" : " 14 GB", | 20260506 21:38:51.659 | INFO | "pinned" : " 29 GB" | 20260506 21:38:51.659 | INFO | } | 20260506 21:38:51.659 | INFO | Perf report for the last 5.14793 seconds: | 20260506 21:38:51.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:38:51.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:51.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:51.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:51.659 | INFO | ANALYSIS : 60.5%, 3.0, 93.9ms, 0 | 20260506 21:38:51.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:51.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:51.659 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:38:51.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:51.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:51.659 | INFO | Uploaded 36GiB to GPU in 1.97661s (18.213GiB/s) | 20260506 21:38:51.659 | INFO | Downloaded 34GiB to GPU in 1.92355s (17.6756GiB/s) | 20260506 21:38:56.794 | INFO | Analyzing chunk frames = [4646400,4646912) | 20260506 21:38:56.794 | INFO | MemInfo: { | 20260506 21:38:56.794 | INFO | "gpu" : " 17 GB", | 20260506 21:38:56.794 | INFO | "huge" : "", | 20260506 21:38:56.794 | INFO | "huge_shared" : " 27 GB", | 20260506 21:38:56.794 | INFO | "malloc" : " 14 GB", | 20260506 21:38:56.794 | INFO | "pinned" : " 29 GB" | 20260506 21:38:56.794 | INFO | } | 20260506 21:38:56.794 | INFO | Perf report for the last 5.13512 seconds: | 20260506 21:38:56.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:38:56.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:56.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:56.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:56.794 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260506 21:38:56.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:56.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:56.794 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:38:56.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:56.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:38:56.795 | INFO | Uploaded 36GiB to GPU in 2.01508s (17.8653GiB/s) | 20260506 21:38:56.795 | INFO | Downloaded 34GiB to GPU in 1.90909s (17.8095GiB/s) | 20260506 21:39:01.936 | INFO | Analyzing chunk frames = [4646912,4647424) | 20260506 21:39:01.936 | INFO | MemInfo: { | 20260506 21:39:01.936 | INFO | "gpu" : " 17 GB", | 20260506 21:39:01.936 | INFO | "huge" : "", | 20260506 21:39:01.936 | INFO | "huge_shared" : " 27 GB", | 20260506 21:39:01.936 | INFO | "malloc" : " 14 GB", | 20260506 21:39:01.936 | INFO | "pinned" : " 29 GB" | 20260506 21:39:01.936 | INFO | } | 20260506 21:39:01.936 | INFO | Perf report for the last 5.14217 seconds: | 20260506 21:39:01.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:39:01.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:01.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:01.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:01.936 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 21:39:01.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:01.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:01.936 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:39:01.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:01.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:01.937 | INFO | Uploaded 36GiB to GPU in 1.97543s (18.2239GiB/s) | 20260506 21:39:01.937 | INFO | Downloaded 34GiB to GPU in 1.92462s (17.6658GiB/s) | 20260506 21:39:07.071 | INFO | Analyzing chunk frames = [4647424,4647936) | 20260506 21:39:07.071 | INFO | MemInfo: { | 20260506 21:39:07.071 | INFO | "gpu" : " 17 GB", | 20260506 21:39:07.071 | INFO | "huge" : "", | 20260506 21:39:07.071 | INFO | "huge_shared" : " 27 GB", | 20260506 21:39:07.071 | INFO | "malloc" : " 14 GB", | 20260506 21:39:07.071 | INFO | "pinned" : " 29 GB" | 20260506 21:39:07.071 | INFO | } | 20260506 21:39:07.071 | INFO | Perf report for the last 5.13461 seconds: | 20260506 21:39:07.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:39:07.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:07.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:07.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:07.071 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 21:39:07.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:07.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:07.071 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 21:39:07.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:07.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:07.071 | INFO | Uploaded 36GiB to GPU in 2.00873s (17.9218GiB/s) | 20260506 21:39:07.071 | INFO | Downloaded 34GiB to GPU in 1.9364s (17.5583GiB/s) | 20260506 21:39:12.208 | INFO | Analyzing chunk frames = [4647936,4648448) | 20260506 21:39:12.208 | INFO | MemInfo: { | 20260506 21:39:12.208 | INFO | "gpu" : " 17 GB", | 20260506 21:39:12.208 | INFO | "huge" : "", | 20260506 21:39:12.208 | INFO | "huge_shared" : " 27 GB", | 20260506 21:39:12.208 | INFO | "malloc" : " 14 GB", | 20260506 21:39:12.208 | INFO | "pinned" : " 29 GB" | 20260506 21:39:12.208 | INFO | } | 20260506 21:39:12.208 | INFO | Perf report for the last 5.13739 seconds: | 20260506 21:39:12.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:39:12.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:12.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:12.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:12.208 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:39:12.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:12.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:12.208 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 21:39:12.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:12.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:12.209 | INFO | Uploaded 36GiB to GPU in 1.98269s (18.1572GiB/s) | 20260506 21:39:12.209 | INFO | Downloaded 34GiB to GPU in 2.01007s (16.9148GiB/s) | 20260506 21:39:17.346 | INFO | Analyzing chunk frames = [4648448,4648960) | 20260506 21:39:17.346 | INFO | MemInfo: { | 20260506 21:39:17.346 | INFO | "gpu" : " 17 GB", | 20260506 21:39:17.346 | INFO | "huge" : "", | 20260506 21:39:17.346 | INFO | "huge_shared" : " 27 GB", | 20260506 21:39:17.346 | INFO | "malloc" : " 14 GB", | 20260506 21:39:17.346 | INFO | "pinned" : " 29 GB" | 20260506 21:39:17.346 | INFO | } | 20260506 21:39:17.346 | INFO | Perf report for the last 5.138 seconds: | 20260506 21:39:17.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:39:17.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:17.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:17.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:17.346 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 21:39:17.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:17.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:17.346 | INFO | PRE_HQ : 31.5%, 1.0, 16.4ms, 0 | 20260506 21:39:17.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:17.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:17.347 | INFO | Uploaded 36GiB to GPU in 2.00186s (17.9832GiB/s) | 20260506 21:39:17.347 | INFO | Downloaded 34GiB to GPU in 1.99189s (17.0692GiB/s) | 20260506 21:39:17.452 | INFO | Starting a round of Baz IO | 20260506 21:39:20.607 | INFO | Populating RTMetrics report | 20260506 21:39:20.607 | INFO | RTMetrics report populated | 20260506 21:39:20.607 | INFO | Opening tmp RTMetrics json file | 20260506 21:39:20.627 | INFO | Serializing RTMetrics to json | 20260506 21:39:20.688 | INFO | Serialization done, swapping tmp file now... | 20260506 21:39:20.739 | INFO | RTMetrics done | 20260506 21:39:21.188 | INFO | Finished a round of Baz IO: Wrote 1.939 GiB in 3.736 seconds | 20260506 21:39:21.188 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.88% / 0.00% / 66.12% | 20260506 21:39:21.189 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.589355% / 0% / 7.94729e-05% | 20260506 21:39:21.189 | INFO | Baz IO overall: | 20260506 21:39:21.189 | INFO | BytesWritten : 2448208166524 | 20260506 21:39:21.189 | INFO | NumEvents : 1181897220368 | 20260506 21:39:22.484 | INFO | Analyzing chunk frames = [4648960,4649472) | 20260506 21:39:22.484 | INFO | MemInfo: { | 20260506 21:39:22.484 | INFO | "gpu" : " 17 GB", | 20260506 21:39:22.484 | INFO | "huge" : "", | 20260506 21:39:22.484 | INFO | "huge_shared" : " 27 GB", | 20260506 21:39:22.484 | INFO | "malloc" : " 14 GB", | 20260506 21:39:22.484 | INFO | "pinned" : " 29 GB" | 20260506 21:39:22.484 | INFO | } | 20260506 21:39:22.484 | INFO | Perf report for the last 5.13747 seconds: | 20260506 21:39:22.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:39:22.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:22.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:22.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:22.484 | INFO | ANALYSIS : 62.3%, 2.9, 94.8ms, 0 | 20260506 21:39:22.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:22.484 | INFO | RT_METRICS : 47.8%, 1.0, 25.1ms, 0 | 20260506 21:39:22.484 | INFO | PRE_HQ : 60.7%, 1.9, 60.6ms, 0 | 20260506 21:39:22.484 | INFO | BAZWRITER : 72.1%, 4.8, 177.8ms, 0 | 20260506 21:39:22.484 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 21:39:22.484 | INFO | Uploaded 36GiB to GPU in 2.05647s (17.5057GiB/s) | 20260506 21:39:22.484 | INFO | Downloaded 38GiB to GPU in 2.1971s (17.2955GiB/s) | 20260506 21:39:27.632 | INFO | Analyzing chunk frames = [4649472,4649984) | 20260506 21:39:27.632 | INFO | MemInfo: { | 20260506 21:39:27.632 | INFO | "gpu" : " 17 GB", | 20260506 21:39:27.632 | INFO | "huge" : "", | 20260506 21:39:27.632 | INFO | "huge_shared" : " 27 GB", | 20260506 21:39:27.632 | INFO | "malloc" : " 14 GB", | 20260506 21:39:27.632 | INFO | "pinned" : " 29 GB" | 20260506 21:39:27.632 | INFO | } | 20260506 21:39:27.632 | INFO | Perf report for the last 5.14804 seconds: | 20260506 21:39:27.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:39:27.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:27.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:27.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:27.632 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 21:39:27.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:27.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:27.632 | INFO | PRE_HQ : 28.5%, 1.0, 14.9ms, 0 | 20260506 21:39:27.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:27.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:27.632 | INFO | Uploaded 36GiB to GPU in 2.02542s (17.7741GiB/s) | 20260506 21:39:27.632 | INFO | Downloaded 34GiB to GPU in 1.93444s (17.5762GiB/s) | 20260506 21:39:32.773 | INFO | Analyzing chunk frames = [4649984,4650496) | 20260506 21:39:32.773 | INFO | MemInfo: { | 20260506 21:39:32.773 | INFO | "gpu" : " 17 GB", | 20260506 21:39:32.773 | INFO | "huge" : "", | 20260506 21:39:32.773 | INFO | "huge_shared" : " 27 GB", | 20260506 21:39:32.773 | INFO | "malloc" : " 14 GB", | 20260506 21:39:32.773 | INFO | "pinned" : " 29 GB" | 20260506 21:39:32.773 | INFO | } | 20260506 21:39:32.773 | INFO | Perf report for the last 5.1416 seconds: | 20260506 21:39:32.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:39:32.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:32.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:32.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:32.774 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 21:39:32.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:32.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:32.774 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:39:32.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:32.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:32.774 | INFO | Uploaded 36GiB to GPU in 1.99543s (18.0412GiB/s) | 20260506 21:39:32.774 | INFO | Downloaded 34GiB to GPU in 1.92759s (17.6386GiB/s) | 20260506 21:39:37.914 | INFO | Analyzing chunk frames = [4650496,4651008) | 20260506 21:39:37.914 | INFO | MemInfo: { | 20260506 21:39:37.914 | INFO | "gpu" : " 17 GB", | 20260506 21:39:37.914 | INFO | "huge" : "", | 20260506 21:39:37.914 | INFO | "huge_shared" : " 27 GB", | 20260506 21:39:37.914 | INFO | "malloc" : " 14 GB", | 20260506 21:39:37.914 | INFO | "pinned" : " 29 GB" | 20260506 21:39:37.914 | INFO | } | 20260506 21:39:37.914 | INFO | Perf report for the last 5.14041 seconds: | 20260506 21:39:37.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:39:37.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:37.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:37.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:37.914 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260506 21:39:37.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:37.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:37.914 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:39:37.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:37.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:37.914 | INFO | Uploaded 36GiB to GPU in 2.01439s (17.8714GiB/s) | 20260506 21:39:37.914 | INFO | Downloaded 34GiB to GPU in 1.90766s (17.8229GiB/s) | 20260506 21:39:43.049 | INFO | Analyzing chunk frames = [4651008,4651520) | 20260506 21:39:43.049 | INFO | MemInfo: { | 20260506 21:39:43.049 | INFO | "gpu" : " 17 GB", | 20260506 21:39:43.049 | INFO | "huge" : "", | 20260506 21:39:43.049 | INFO | "huge_shared" : " 27 GB", | 20260506 21:39:43.049 | INFO | "malloc" : " 14 GB", | 20260506 21:39:43.050 | INFO | "pinned" : " 29 GB" | 20260506 21:39:43.050 | INFO | } | 20260506 21:39:43.050 | INFO | Perf report for the last 5.13569 seconds: | 20260506 21:39:43.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:39:43.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:43.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:43.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:43.050 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 21:39:43.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:43.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:43.050 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:39:43.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:43.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:43.050 | INFO | Uploaded 36GiB to GPU in 1.99916s (18.0075GiB/s) | 20260506 21:39:43.050 | INFO | Downloaded 34GiB to GPU in 1.92186s (17.6912GiB/s) | 20260506 21:39:48.185 | INFO | Analyzing chunk frames = [4651520,4652032) | 20260506 21:39:48.185 | INFO | MemInfo: { | 20260506 21:39:48.185 | INFO | "gpu" : " 17 GB", | 20260506 21:39:48.185 | INFO | "huge" : "", | 20260506 21:39:48.185 | INFO | "huge_shared" : " 27 GB", | 20260506 21:39:48.185 | INFO | "malloc" : " 14 GB", | 20260506 21:39:48.185 | INFO | "pinned" : " 29 GB" | 20260506 21:39:48.185 | INFO | } | 20260506 21:39:48.185 | INFO | Perf report for the last 5.1355 seconds: | 20260506 21:39:48.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:39:48.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:48.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:48.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:48.185 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 21:39:48.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:48.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:48.185 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:39:48.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:48.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:48.185 | INFO | Uploaded 36GiB to GPU in 2.01531s (17.8633GiB/s) | 20260506 21:39:48.185 | INFO | Downloaded 34GiB to GPU in 1.9564s (17.3788GiB/s) | 20260506 21:39:53.323 | INFO | Analyzing chunk frames = [4652032,4652544) | 20260506 21:39:53.323 | INFO | MemInfo: { | 20260506 21:39:53.323 | INFO | "gpu" : " 17 GB", | 20260506 21:39:53.323 | INFO | "huge" : "", | 20260506 21:39:53.323 | INFO | "huge_shared" : " 27 GB", | 20260506 21:39:53.323 | INFO | "malloc" : " 14 GB", | 20260506 21:39:53.323 | INFO | "pinned" : " 29 GB" | 20260506 21:39:53.323 | INFO | } | 20260506 21:39:53.323 | INFO | Perf report for the last 5.13822 seconds: | 20260506 21:39:53.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:39:53.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:53.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:53.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:53.323 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 21:39:53.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:53.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:53.323 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 21:39:53.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:53.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:53.324 | INFO | Uploaded 36GiB to GPU in 1.96875s (18.2857GiB/s) | 20260506 21:39:53.324 | INFO | Downloaded 34GiB to GPU in 2.00924s (16.9218GiB/s) | 20260506 21:39:58.462 | INFO | Analyzing chunk frames = [4652544,4653056) | 20260506 21:39:58.462 | INFO | MemInfo: { | 20260506 21:39:58.462 | INFO | "gpu" : " 17 GB", | 20260506 21:39:58.462 | INFO | "huge" : "", | 20260506 21:39:58.463 | INFO | "huge_shared" : " 27 GB", | 20260506 21:39:58.463 | INFO | "malloc" : " 14 GB", | 20260506 21:39:58.463 | INFO | "pinned" : " 29 GB" | 20260506 21:39:58.463 | INFO | } | 20260506 21:39:58.463 | INFO | Perf report for the last 5.13921 seconds: | 20260506 21:39:58.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:39:58.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:58.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:58.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:58.463 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 21:39:58.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:58.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:58.463 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:39:58.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:58.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:39:58.463 | INFO | Uploaded 36GiB to GPU in 2.02781s (17.7532GiB/s) | 20260506 21:39:58.463 | INFO | Downloaded 34GiB to GPU in 1.99805s (17.0166GiB/s) | 20260506 21:39:58.463 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:39:58.463 | INFO | Total mem usage: 18 GiB | 20260506 21:39:58.463 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:39:58.463 | INFO | Recent High Water Marks Sum: 16.8 GiB (Savings: -7.24%) | 20260506 21:39:58.463 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:39:58.463 | INFO | High Water Breakdown: | 20260506 21:39:58.463 | INFO | 62.6% : 10.5 GiB ( 92.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:39:58.463 | INFO | 37.3% : 6.26 GiB ( 93.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:39:58.463 | INFO | 0.119% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:39:58.463 | INFO | ------------------------------------------ | 20260506 21:39:58.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:39:58.463 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:39:58.463 | INFO | Total mem usage: 26.5 GiB | 20260506 21:39:58.463 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:39:58.463 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:39:58.463 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:39:58.463 | INFO | High Water Breakdown: | 20260506 21:39:58.463 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:39:58.463 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:39:58.463 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:39:58.463 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:39:58.463 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:39:58.463 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:39:58.463 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:39:58.463 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:39:58.463 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:39:58.463 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:39:58.463 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:39:58.463 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:39:58.463 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:39:58.463 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:39:58.463 | INFO | ------------------------------------------ | 20260506 21:39:58.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:39:58.463 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:39:58.463 | INFO | Total mem usage: 25.6 GiB | 20260506 21:39:58.463 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:39:58.463 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:39:58.463 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:39:58.463 | INFO | High Water Breakdown: | 20260506 21:39:58.463 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:39:58.463 | INFO | ------------------------------------------ | 20260506 21:39:58.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:39:58.463 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:39:58.463 | INFO | Total mem usage: 15.7 GiB | 20260506 21:39:58.463 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:39:58.463 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:39:58.463 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:39:58.463 | INFO | High Water Breakdown: | 20260506 21:39:58.463 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:39:58.463 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:39:58.463 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:39:58.463 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:39:58.463 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:39:58.463 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:39:58.464 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:39:58.464 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:39:58.464 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:39:58.464 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:39:58.464 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:39:58.464 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:39:58.464 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:39:58.464 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:39:58.464 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:39:58.464 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:39:58.464 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:39:58.464 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:39:58.464 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:39:58.464 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:39:58.464 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:39:58.464 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:39:58.464 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:39:58.464 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:39:58.464 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:39:58.464 | INFO | ------------------------------------------ | 20260506 21:39:58.464 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:39:58.464 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:39:58.464 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:39:58.464 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:39:58.464 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.62e+01, 1.34e+01 / 2.17e+00 | 20260506 21:39:58.464 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 9.97e-02, 9.79e-03 / 6.36e-03 | 20260506 21:39:58.464 | INFO | PerfReport: 11.65%, Baseline -- 3.62e+00 / 4.36e+00, 5.50e-01 / 3.81e-01 | 20260506 21:39:58.464 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.44e+00, 1.80e-01 / 1.38e-01 | 20260506 21:39:58.464 | INFO | PerfReport: 16.03%, DME -- 3.99e+01 / 4.46e+01, 4.15e+00 / 1.87e+00 | 20260506 21:39:58.464 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.78e+01 / 2.11e+01, 1.38e+00 / 1.82e+00 | 20260506 21:39:58.464 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.21e+00 / 2.75e+00, 3.78e-01 / 2.10e-01 | 20260506 21:39:58.464 | INFO | PerfReport: 3.26%, Metrics -- 1.01e+00 / 1.35e+00, 2.69e-01 / 1.25e-01 | 20260506 21:39:58.464 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.47e-01 / 3.97e-01, 5.40e-02 / 2.73e-02 | 20260506 21:39:58.464 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:39:58.464 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:39:58.464 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:39:58.464 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 21:39:58.464 | INFO | PerfReport: 21.60%, Upload -- 2.09e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260506 21:39:58.464 | INFO | PerfReport: 21.32%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 21:39:58.464 | INFO | PerfReport: 32.16%, Compute -- 3.11e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 21:39:58.464 | INFO | PerfReport: 1.28%, LockUpload -- 1.24e+00 / 0.00e+00, 4.76e+00 / 0.00e+00 | 20260506 21:39:58.464 | INFO | PerfReport: 1.26%, LockDownload -- 1.22e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260506 21:39:58.464 | INFO | PerfReport: 22.36%, LockCompute -- 2.16e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 21:39:58.464 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260506 21:39:58.572 | INFO | Starting a round of Baz IO | 20260506 21:40:01.704 | INFO | Populating RTMetrics report | 20260506 21:40:01.704 | INFO | RTMetrics report populated | 20260506 21:40:01.704 | INFO | Opening tmp RTMetrics json file | 20260506 21:40:01.860 | INFO | Serializing RTMetrics to json | 20260506 21:40:01.929 | INFO | Serialization done, swapping tmp file now... | 20260506 21:40:01.963 | INFO | RTMetrics done | 20260506 21:40:02.094 | INFO | Finished a round of Baz IO: Wrote 1.938 GiB in 3.522 seconds | 20260506 21:40:02.095 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.88% / 0.00% / 66.12% | 20260506 21:40:02.095 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.588% / 3.97364e-06% / 5.16574e-05% | 20260506 21:40:02.095 | INFO | Baz IO overall: | 20260506 21:40:02.095 | INFO | BytesWritten : 2450289047077 | 20260506 21:40:02.095 | INFO | NumEvents : 1182892565720 | 20260506 21:40:03.602 | INFO | Analyzing chunk frames = [4653056,4653568) | 20260506 21:40:03.602 | INFO | MemInfo: { | 20260506 21:40:03.602 | INFO | "gpu" : " 17 GB", | 20260506 21:40:03.602 | INFO | "huge" : "", | 20260506 21:40:03.602 | INFO | "huge_shared" : " 27 GB", | 20260506 21:40:03.602 | INFO | "malloc" : " 14 GB", | 20260506 21:40:03.602 | INFO | "pinned" : " 29 GB" | 20260506 21:40:03.602 | INFO | } | 20260506 21:40:03.602 | INFO | Perf report for the last 5.13922 seconds: | 20260506 21:40:03.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:40:03.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:03.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:03.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:03.602 | INFO | ANALYSIS : 62.5%, 2.9, 95.2ms, 0 | 20260506 21:40:03.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:03.602 | INFO | RT_METRICS : 48.0%, 1.0, 25.2ms, 0 | 20260506 21:40:03.602 | INFO | PRE_HQ : 59.9%, 1.8, 57.5ms, 0 | 20260506 21:40:03.602 | INFO | BAZWRITER : 68.2%, 4.8, 166.8ms, 0 | 20260506 21:40:03.602 | INFO | BAZ_SAMPLING : 2.9%, 1.0, 1.5ms, 0 | 20260506 21:40:03.602 | INFO | Uploaded 36GiB to GPU in 2.03877s (17.6577GiB/s) | 20260506 21:40:03.602 | INFO | Downloaded 38GiB to GPU in 2.20193s (17.2576GiB/s) | 20260506 21:40:08.742 | INFO | Analyzing chunk frames = [4653568,4654080) | 20260506 21:40:08.742 | INFO | MemInfo: { | 20260506 21:40:08.742 | INFO | "gpu" : " 17 GB", | 20260506 21:40:08.742 | INFO | "huge" : "", | 20260506 21:40:08.742 | INFO | "huge_shared" : " 27 GB", | 20260506 21:40:08.743 | INFO | "malloc" : " 14 GB", | 20260506 21:40:08.743 | INFO | "pinned" : " 29 GB" | 20260506 21:40:08.743 | INFO | } | 20260506 21:40:08.743 | INFO | Perf report for the last 5.14078 seconds: | 20260506 21:40:08.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:40:08.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:08.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:08.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:08.743 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:40:08.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:08.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:08.743 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 21:40:08.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:08.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:08.743 | INFO | Uploaded 36GiB to GPU in 1.99898s (18.0092GiB/s) | 20260506 21:40:08.743 | INFO | Downloaded 34GiB to GPU in 1.92931s (17.6229GiB/s) | 20260506 21:40:13.886 | INFO | Analyzing chunk frames = [4654080,4654592) | 20260506 21:40:13.886 | INFO | MemInfo: { | 20260506 21:40:13.886 | INFO | "gpu" : " 17 GB", | 20260506 21:40:13.886 | INFO | "huge" : "", | 20260506 21:40:13.886 | INFO | "huge_shared" : " 27 GB", | 20260506 21:40:13.886 | INFO | "malloc" : " 14 GB", | 20260506 21:40:13.886 | INFO | "pinned" : " 29 GB" | 20260506 21:40:13.886 | INFO | } | 20260506 21:40:13.886 | INFO | Perf report for the last 5.14344 seconds: | 20260506 21:40:13.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:40:13.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:13.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:13.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:13.886 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 21:40:13.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:13.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:13.886 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:40:13.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:13.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:13.886 | INFO | Uploaded 36GiB to GPU in 1.96008s (18.3666GiB/s) | 20260506 21:40:13.886 | INFO | Downloaded 34GiB to GPU in 1.9259s (17.6541GiB/s) | 20260506 21:40:19.023 | INFO | Analyzing chunk frames = [4654592,4655104) | 20260506 21:40:19.023 | INFO | MemInfo: { | 20260506 21:40:19.023 | INFO | "gpu" : " 17 GB", | 20260506 21:40:19.023 | INFO | "huge" : "", | 20260506 21:40:19.023 | INFO | "huge_shared" : " 27 GB", | 20260506 21:40:19.023 | INFO | "malloc" : " 14 GB", | 20260506 21:40:19.023 | INFO | "pinned" : " 29 GB" | 20260506 21:40:19.023 | INFO | } | 20260506 21:40:19.023 | INFO | Perf report for the last 5.1369 seconds: | 20260506 21:40:19.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:40:19.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:19.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:19.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:19.023 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260506 21:40:19.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:19.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:19.023 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:40:19.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:19.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:19.023 | INFO | Uploaded 36GiB to GPU in 1.96524s (18.3184GiB/s) | 20260506 21:40:19.023 | INFO | Downloaded 34GiB to GPU in 1.89587s (17.9337GiB/s) | 20260506 21:40:24.161 | INFO | Analyzing chunk frames = [4655104,4655616) | 20260506 21:40:24.161 | INFO | MemInfo: { | 20260506 21:40:24.161 | INFO | "gpu" : " 17 GB", | 20260506 21:40:24.161 | INFO | "huge" : "", | 20260506 21:40:24.161 | INFO | "huge_shared" : " 27 GB", | 20260506 21:40:24.161 | INFO | "malloc" : " 14 GB", | 20260506 21:40:24.161 | INFO | "pinned" : " 29 GB" | 20260506 21:40:24.161 | INFO | } | 20260506 21:40:24.161 | INFO | Perf report for the last 5.13782 seconds: | 20260506 21:40:24.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:40:24.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:24.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:24.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:24.161 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 21:40:24.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:24.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:24.161 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 21:40:24.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:24.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:24.161 | INFO | Uploaded 36GiB to GPU in 2.00052s (17.9954GiB/s) | 20260506 21:40:24.161 | INFO | Downloaded 34GiB to GPU in 1.91997s (17.7086GiB/s) | 20260506 21:40:29.300 | INFO | Analyzing chunk frames = [4655616,4656128) | 20260506 21:40:29.300 | INFO | MemInfo: { | 20260506 21:40:29.300 | INFO | "gpu" : " 17 GB", | 20260506 21:40:29.300 | INFO | "huge" : "", | 20260506 21:40:29.300 | INFO | "huge_shared" : " 27 GB", | 20260506 21:40:29.300 | INFO | "malloc" : " 14 GB", | 20260506 21:40:29.300 | INFO | "pinned" : " 29 GB" | 20260506 21:40:29.300 | INFO | } | 20260506 21:40:29.300 | INFO | Perf report for the last 5.13931 seconds: | 20260506 21:40:29.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:40:29.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:29.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:29.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:29.300 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:40:29.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:29.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:29.300 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 21:40:29.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:29.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:29.300 | INFO | Uploaded 36GiB to GPU in 2.0077s (17.9309GiB/s) | 20260506 21:40:29.300 | INFO | Downloaded 34GiB to GPU in 1.95285s (17.4105GiB/s) | 20260506 21:40:34.436 | INFO | Analyzing chunk frames = [4656128,4656640) | 20260506 21:40:34.436 | INFO | MemInfo: { | 20260506 21:40:34.436 | INFO | "gpu" : " 17 GB", | 20260506 21:40:34.436 | INFO | "huge" : "", | 20260506 21:40:34.436 | INFO | "huge_shared" : " 27 GB", | 20260506 21:40:34.436 | INFO | "malloc" : " 14 GB", | 20260506 21:40:34.436 | INFO | "pinned" : " 29 GB" | 20260506 21:40:34.436 | INFO | } | 20260506 21:40:34.437 | INFO | Perf report for the last 5.13637 seconds: | 20260506 21:40:34.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:40:34.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:34.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:34.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:34.437 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:40:34.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:34.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:34.437 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:40:34.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:34.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:34.437 | INFO | Uploaded 36GiB to GPU in 1.96407s (18.3293GiB/s) | 20260506 21:40:34.437 | INFO | Downloaded 34GiB to GPU in 2.00326s (16.9723GiB/s) | 20260506 21:40:39.579 | INFO | Analyzing chunk frames = [4656640,4657152) | 20260506 21:40:39.579 | INFO | MemInfo: { | 20260506 21:40:39.579 | INFO | "gpu" : " 17 GB", | 20260506 21:40:39.579 | INFO | "huge" : "", | 20260506 21:40:39.579 | INFO | "huge_shared" : " 27 GB", | 20260506 21:40:39.579 | INFO | "malloc" : " 14 GB", | 20260506 21:40:39.579 | INFO | "pinned" : " 29 GB" | 20260506 21:40:39.579 | INFO | } | 20260506 21:40:39.579 | INFO | Perf report for the last 5.14249 seconds: | 20260506 21:40:39.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:40:39.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:39.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:39.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:39.579 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 21:40:39.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:39.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:39.579 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 21:40:39.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:39.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:39.579 | INFO | Uploaded 36GiB to GPU in 1.98677s (18.1198GiB/s) | 20260506 21:40:39.579 | INFO | Downloaded 34GiB to GPU in 1.98591s (17.1206GiB/s) | 20260506 21:40:39.678 | INFO | Starting a round of Baz IO | 20260506 21:40:42.811 | INFO | Populating RTMetrics report | 20260506 21:40:42.812 | INFO | RTMetrics report populated | 20260506 21:40:42.812 | INFO | Opening tmp RTMetrics json file | 20260506 21:40:42.813 | INFO | Serializing RTMetrics to json | 20260506 21:40:42.815 | INFO | Serialization done, swapping tmp file now... | 20260506 21:40:42.815 | INFO | RTMetrics done | 20260506 21:40:42.958 | INFO | Finished a round of Baz IO: Wrote 1.936 GiB in 3.28 seconds | 20260506 21:40:42.958 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.87% / 0.00% / 66.12% | 20260506 21:40:42.958 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.584726% / 3.97364e-06% / 5.96046e-05% | 20260506 21:40:42.958 | INFO | Baz IO overall: | 20260506 21:40:42.958 | INFO | BytesWritten : 2452368162174 | 20260506 21:40:42.958 | INFO | NumEvents : 1183886380549 | 20260506 21:40:44.719 | INFO | Analyzing chunk frames = [4657152,4657664) | 20260506 21:40:44.719 | INFO | MemInfo: { | 20260506 21:40:44.719 | INFO | "gpu" : " 17 GB", | 20260506 21:40:44.719 | INFO | "huge" : "", | 20260506 21:40:44.720 | INFO | "huge_shared" : " 27 GB", | 20260506 21:40:44.720 | INFO | "malloc" : " 14 GB", | 20260506 21:40:44.720 | INFO | "pinned" : " 29 GB" | 20260506 21:40:44.720 | INFO | } | 20260506 21:40:44.720 | INFO | Perf report for the last 5.14053 seconds: | 20260506 21:40:44.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:40:44.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:44.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:44.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:44.720 | INFO | ANALYSIS : 62.4%, 2.9, 95.2ms, 0 | 20260506 21:40:44.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:44.720 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 21:40:44.720 | INFO | PRE_HQ : 60.0%, 1.8, 57.1ms, 0 | 20260506 21:40:44.720 | INFO | BAZWRITER : 63.4%, 4.5, 146.1ms, 0 | 20260506 21:40:44.720 | INFO | BAZ_SAMPLING : 1.8%, 1.0, 0.9ms, 0 | 20260506 21:40:44.720 | INFO | Uploaded 36GiB to GPU in 2.05174s (17.5461GiB/s) | 20260506 21:40:44.720 | INFO | Downloaded 38GiB to GPU in 2.19332s (17.3253GiB/s) | 20260506 21:40:49.859 | INFO | Analyzing chunk frames = [4657664,4658176) | 20260506 21:40:49.859 | INFO | MemInfo: { | 20260506 21:40:49.859 | INFO | "gpu" : " 17 GB", | 20260506 21:40:49.859 | INFO | "huge" : "", | 20260506 21:40:49.859 | INFO | "huge_shared" : " 27 GB", | 20260506 21:40:49.859 | INFO | "malloc" : " 14 GB", | 20260506 21:40:49.859 | INFO | "pinned" : " 29 GB" | 20260506 21:40:49.859 | INFO | } | 20260506 21:40:49.859 | INFO | Perf report for the last 5.13988 seconds: | 20260506 21:40:49.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:40:49.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:49.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:49.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:49.859 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:40:49.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:49.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:49.859 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:40:49.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:49.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:49.860 | INFO | Uploaded 36GiB to GPU in 2.02046s (17.8177GiB/s) | 20260506 21:40:49.860 | INFO | Downloaded 34GiB to GPU in 1.92833s (17.6318GiB/s) | 20260506 21:40:55.003 | INFO | Analyzing chunk frames = [4658176,4658688) | 20260506 21:40:55.003 | INFO | MemInfo: { | 20260506 21:40:55.003 | INFO | "gpu" : " 17 GB", | 20260506 21:40:55.003 | INFO | "huge" : "", | 20260506 21:40:55.003 | INFO | "huge_shared" : " 27 GB", | 20260506 21:40:55.003 | INFO | "malloc" : " 14 GB", | 20260506 21:40:55.003 | INFO | "pinned" : " 29 GB" | 20260506 21:40:55.003 | INFO | } | 20260506 21:40:55.003 | INFO | Perf report for the last 5.14329 seconds: | 20260506 21:40:55.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:40:55.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:55.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:55.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:55.003 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 21:40:55.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:55.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:55.003 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:40:55.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:55.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:40:55.003 | INFO | Uploaded 36GiB to GPU in 1.9971s (18.0262GiB/s) | 20260506 21:40:55.003 | INFO | Downloaded 34GiB to GPU in 1.92169s (17.6928GiB/s) | 20260506 21:41:00.137 | INFO | Analyzing chunk frames = [4658688,4659200) | 20260506 21:41:00.137 | INFO | MemInfo: { | 20260506 21:41:00.137 | INFO | "gpu" : " 17 GB", | 20260506 21:41:00.137 | INFO | "huge" : "", | 20260506 21:41:00.137 | INFO | "huge_shared" : " 27 GB", | 20260506 21:41:00.137 | INFO | "malloc" : " 14 GB", | 20260506 21:41:00.137 | INFO | "pinned" : " 29 GB" | 20260506 21:41:00.137 | INFO | } | 20260506 21:41:00.137 | INFO | Perf report for the last 5.13465 seconds: | 20260506 21:41:00.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:41:00.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:00.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:00.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:00.137 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260506 21:41:00.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:00.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:00.137 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:41:00.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:00.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:00.138 | INFO | Uploaded 36GiB to GPU in 2.01289s (17.8847GiB/s) | 20260506 21:41:00.138 | INFO | Downloaded 34GiB to GPU in 1.90643s (17.8344GiB/s) | 20260506 21:41:05.274 | INFO | Analyzing chunk frames = [4659200,4659712) | 20260506 21:41:05.274 | INFO | MemInfo: { | 20260506 21:41:05.274 | INFO | "gpu" : " 17 GB", | 20260506 21:41:05.274 | INFO | "huge" : "", | 20260506 21:41:05.274 | INFO | "huge_shared" : " 27 GB", | 20260506 21:41:05.274 | INFO | "malloc" : " 14 GB", | 20260506 21:41:05.275 | INFO | "pinned" : " 29 GB" | 20260506 21:41:05.275 | INFO | } | 20260506 21:41:05.275 | INFO | Perf report for the last 5.13715 seconds: | 20260506 21:41:05.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:41:05.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:05.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:05.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:05.275 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 21:41:05.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:05.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:05.275 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:41:05.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:05.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:05.275 | INFO | Uploaded 36GiB to GPU in 2.00519s (17.9534GiB/s) | 20260506 21:41:05.275 | INFO | Downloaded 34GiB to GPU in 1.91583s (17.7469GiB/s) | 20260506 21:41:10.415 | INFO | Analyzing chunk frames = [4659712,4660224) | 20260506 21:41:10.415 | INFO | MemInfo: { | 20260506 21:41:10.415 | INFO | "gpu" : " 17 GB", | 20260506 21:41:10.415 | INFO | "huge" : "", | 20260506 21:41:10.415 | INFO | "huge_shared" : " 27 GB", | 20260506 21:41:10.415 | INFO | "malloc" : " 14 GB", | 20260506 21:41:10.415 | INFO | "pinned" : " 29 GB" | 20260506 21:41:10.415 | INFO | } | 20260506 21:41:10.415 | INFO | Perf report for the last 5.1406 seconds: | 20260506 21:41:10.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:41:10.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:10.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:10.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:10.415 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:41:10.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:10.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:10.415 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 21:41:10.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:10.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:10.415 | INFO | Uploaded 36GiB to GPU in 2.01225s (17.8904GiB/s) | 20260506 21:41:10.415 | INFO | Downloaded 34GiB to GPU in 1.93537s (17.5677GiB/s) | 20260506 21:41:15.558 | INFO | Analyzing chunk frames = [4660224,4660736) | 20260506 21:41:15.558 | INFO | MemInfo: { | 20260506 21:41:15.558 | INFO | "gpu" : " 17 GB", | 20260506 21:41:15.558 | INFO | "huge" : "", | 20260506 21:41:15.558 | INFO | "huge_shared" : " 27 GB", | 20260506 21:41:15.558 | INFO | "malloc" : " 14 GB", | 20260506 21:41:15.558 | INFO | "pinned" : " 29 GB" | 20260506 21:41:15.558 | INFO | } | 20260506 21:41:15.558 | INFO | Perf report for the last 5.14262 seconds: | 20260506 21:41:15.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:41:15.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:15.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:15.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:15.558 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:41:15.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:15.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:15.558 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:41:15.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:15.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:15.558 | INFO | Uploaded 36GiB to GPU in 1.97989s (18.1828GiB/s) | 20260506 21:41:15.558 | INFO | Downloaded 34GiB to GPU in 2.00466s (16.9605GiB/s) | 20260506 21:41:20.690 | INFO | Analyzing chunk frames = [4660736,4661248) | 20260506 21:41:20.690 | INFO | MemInfo: { | 20260506 21:41:20.690 | INFO | "gpu" : " 17 GB", | 20260506 21:41:20.690 | INFO | "huge" : "", | 20260506 21:41:20.690 | INFO | "huge_shared" : " 27 GB", | 20260506 21:41:20.690 | INFO | "malloc" : " 14 GB", | 20260506 21:41:20.690 | INFO | "pinned" : " 29 GB" | 20260506 21:41:20.690 | INFO | } | 20260506 21:41:20.690 | INFO | Perf report for the last 5.13204 seconds: | 20260506 21:41:20.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:41:20.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:20.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:20.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:20.690 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 21:41:20.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:20.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:20.690 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 21:41:20.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:20.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:20.690 | INFO | Uploaded 36GiB to GPU in 1.97529s (18.2252GiB/s) | 20260506 21:41:20.690 | INFO | Downloaded 34GiB to GPU in 1.99s (17.0855GiB/s) | 20260506 21:41:20.690 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:41:20.690 | INFO | Total mem usage: 18 GiB | 20260506 21:41:20.690 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:41:20.690 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -19%) | 20260506 21:41:20.690 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:41:20.690 | INFO | High Water Breakdown: | 20260506 21:41:20.690 | INFO | 61% : 9.23 GiB ( 80.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:41:20.690 | INFO | 38.8% : 5.88 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:41:20.690 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:41:20.690 | INFO | ------------------------------------------ | 20260506 21:41:20.690 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:41:20.690 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:41:20.690 | INFO | Total mem usage: 26.5 GiB | 20260506 21:41:20.690 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:41:20.690 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:41:20.690 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:41:20.690 | INFO | High Water Breakdown: | 20260506 21:41:20.690 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:41:20.690 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:41:20.690 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:41:20.690 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:41:20.690 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:41:20.690 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:41:20.690 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:41:20.690 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:41:20.690 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:41:20.690 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:41:20.690 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:41:20.690 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:41:20.690 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:41:20.690 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:41:20.690 | INFO | ------------------------------------------ | 20260506 21:41:20.690 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:41:20.690 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:41:20.690 | INFO | Total mem usage: 25.6 GiB | 20260506 21:41:20.691 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:41:20.691 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 21:41:20.691 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 21:41:20.691 | INFO | High Water Breakdown: | 20260506 21:41:20.691 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 21:41:20.691 | INFO | ------------------------------------------ | 20260506 21:41:20.691 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:41:20.691 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:41:20.691 | INFO | Total mem usage: 15.7 GiB | 20260506 21:41:20.691 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:41:20.691 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:41:20.691 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:41:20.691 | INFO | High Water Breakdown: | 20260506 21:41:20.691 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:41:20.691 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:41:20.691 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:41:20.691 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:41:20.691 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:41:20.691 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:41:20.691 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:41:20.691 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:41:20.691 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:41:20.691 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:41:20.691 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:41:20.691 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:41:20.691 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:41:20.691 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:41:20.691 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:41:20.691 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:41:20.691 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:41:20.691 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:41:20.691 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:41:20.691 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:41:20.691 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:41:20.691 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:41:20.691 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:41:20.691 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:41:20.691 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:41:20.691 | INFO | ------------------------------------------ | 20260506 21:41:20.691 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:41:20.691 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:41:20.691 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:41:20.691 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:41:20.691 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.59e+01, 1.33e+01 / 2.28e+00 | 20260506 21:41:20.691 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.02e-01, 1.02e-02 / 5.92e-03 | 20260506 21:41:20.691 | INFO | PerfReport: 11.65%, Baseline -- 3.62e+00 / 4.38e+00, 5.51e-01 / 3.69e-01 | 20260506 21:41:20.691 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.45e+00, 1.80e-01 / 1.42e-01 | 20260506 21:41:20.691 | INFO | PerfReport: 16.06%, DME -- 3.99e+01 / 4.48e+01, 4.40e+00 / 2.95e+00 | 20260506 21:41:20.691 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.38e+00 / 1.97e+00 | 20260506 21:41:20.691 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.21e+00 / 2.76e+00, 3.77e-01 / 2.35e-01 | 20260506 21:41:20.691 | INFO | PerfReport: 3.26%, Metrics -- 1.01e+00 / 1.34e+00, 2.70e-01 / 1.52e-01 | 20260506 21:41:20.691 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.47e-01 / 3.95e-01, 5.41e-02 / 2.16e-02 | 20260506 21:41:20.691 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:41:20.691 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:41:20.691 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:41:20.691 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 21:41:20.691 | INFO | PerfReport: 21.54%, Upload -- 2.08e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 21:41:20.691 | INFO | PerfReport: 21.29%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 21:41:20.691 | INFO | PerfReport: 32.18%, Compute -- 3.11e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 21:41:20.691 | INFO | PerfReport: 1.29%, LockUpload -- 1.24e+00 / 0.00e+00, 4.65e+00 / 0.00e+00 | 20260506 21:41:20.691 | INFO | PerfReport: 1.23%, LockDownload -- 1.19e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260506 21:41:20.691 | INFO | PerfReport: 22.46%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:41:20.691 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.40e-02 / 0.00e+00 | 20260506 21:41:20.788 | INFO | Starting a round of Baz IO | 20260506 21:41:23.921 | INFO | Populating RTMetrics report | 20260506 21:41:23.921 | INFO | RTMetrics report populated | 20260506 21:41:23.921 | INFO | Opening tmp RTMetrics json file | 20260506 21:41:23.987 | INFO | Serializing RTMetrics to json | 20260506 21:41:24.027 | INFO | Serialization done, swapping tmp file now... | 20260506 21:41:24.057 | INFO | RTMetrics done | 20260506 21:41:24.230 | INFO | Finished a round of Baz IO: Wrote 1.937 GiB in 3.442 seconds | 20260506 21:41:24.230 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.87% / 0.00% / 66.13% | 20260506 21:41:24.230 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.583959% / 3.97364e-06% / 8.34465e-05% | 20260506 21:41:24.230 | INFO | Baz IO overall: | 20260506 21:41:24.230 | INFO | BytesWritten : 2454447546887 | 20260506 21:41:24.230 | INFO | NumEvents : 1184880543249 | 20260506 21:41:25.832 | INFO | Analyzing chunk frames = [4661248,4661760) | 20260506 21:41:25.832 | INFO | MemInfo: { | 20260506 21:41:25.832 | INFO | "gpu" : " 17 GB", | 20260506 21:41:25.832 | INFO | "huge" : "", | 20260506 21:41:25.832 | INFO | "huge_shared" : " 27 GB", | 20260506 21:41:25.832 | INFO | "malloc" : " 14 GB", | 20260506 21:41:25.832 | INFO | "pinned" : " 29 GB" | 20260506 21:41:25.832 | INFO | } | 20260506 21:41:25.832 | INFO | Perf report for the last 5.14187 seconds: | 20260506 21:41:25.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:41:25.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:25.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:25.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:25.832 | INFO | ANALYSIS : 62.3%, 2.9, 95.3ms, 0 | 20260506 21:41:25.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:25.832 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 21:41:25.832 | INFO | PRE_HQ : 59.7%, 1.8, 57.0ms, 0 | 20260506 21:41:25.832 | INFO | BAZWRITER : 66.6%, 4.8, 163.4ms, 0 | 20260506 21:41:25.832 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 21:41:25.832 | INFO | Uploaded 36GiB to GPU in 2.00336s (17.9698GiB/s) | 20260506 21:41:25.832 | INFO | Downloaded 38GiB to GPU in 2.19516s (17.3108GiB/s) | 20260506 21:41:30.973 | INFO | Analyzing chunk frames = [4661760,4662272) | 20260506 21:41:30.973 | INFO | MemInfo: { | 20260506 21:41:30.973 | INFO | "gpu" : " 17 GB", | 20260506 21:41:30.973 | INFO | "huge" : "", | 20260506 21:41:30.974 | INFO | "huge_shared" : " 27 GB", | 20260506 21:41:30.974 | INFO | "malloc" : " 14 GB", | 20260506 21:41:30.974 | INFO | "pinned" : " 29 GB" | 20260506 21:41:30.974 | INFO | } | 20260506 21:41:30.974 | INFO | Perf report for the last 5.14183 seconds: | 20260506 21:41:30.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:41:30.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:30.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:30.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:30.974 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:41:30.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:30.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:30.974 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 21:41:30.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:30.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:30.974 | INFO | Uploaded 36GiB to GPU in 1.99297s (18.0635GiB/s) | 20260506 21:41:30.974 | INFO | Downloaded 34GiB to GPU in 1.9262s (17.6513GiB/s) | 20260506 21:41:36.111 | INFO | Analyzing chunk frames = [4662272,4662784) | 20260506 21:41:36.111 | INFO | MemInfo: { | 20260506 21:41:36.111 | INFO | "gpu" : " 17 GB", | 20260506 21:41:36.111 | INFO | "huge" : "", | 20260506 21:41:36.111 | INFO | "huge_shared" : " 27 GB", | 20260506 21:41:36.111 | INFO | "malloc" : " 14 GB", | 20260506 21:41:36.111 | INFO | "pinned" : " 29 GB" | 20260506 21:41:36.111 | INFO | } | 20260506 21:41:36.111 | INFO | Perf report for the last 5.13728 seconds: | 20260506 21:41:36.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:41:36.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:36.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:36.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:36.111 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 21:41:36.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:36.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:36.111 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:41:36.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:36.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:36.111 | INFO | Uploaded 36GiB to GPU in 1.96864s (18.2867GiB/s) | 20260506 21:41:36.111 | INFO | Downloaded 34GiB to GPU in 1.92107s (17.6985GiB/s) | 20260506 21:41:41.256 | INFO | Analyzing chunk frames = [4662784,4663296) | 20260506 21:41:41.256 | INFO | MemInfo: { | 20260506 21:41:41.256 | INFO | "gpu" : " 17 GB", | 20260506 21:41:41.256 | INFO | "huge" : "", | 20260506 21:41:41.256 | INFO | "huge_shared" : " 27 GB", | 20260506 21:41:41.256 | INFO | "malloc" : " 14 GB", | 20260506 21:41:41.256 | INFO | "pinned" : " 29 GB" | 20260506 21:41:41.256 | INFO | } | 20260506 21:41:41.256 | INFO | Perf report for the last 5.14524 seconds: | 20260506 21:41:41.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:41:41.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:41.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:41.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:41.256 | INFO | ANALYSIS : 60.2%, 3.0, 93.5ms, 0 | 20260506 21:41:41.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:41.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:41.256 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:41:41.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:41.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:41.256 | INFO | Uploaded 36GiB to GPU in 2.01279s (17.8857GiB/s) | 20260506 21:41:41.256 | INFO | Downloaded 34GiB to GPU in 1.91031s (17.7981GiB/s) | 20260506 21:41:46.394 | INFO | Analyzing chunk frames = [4663296,4663808) | 20260506 21:41:46.394 | INFO | MemInfo: { | 20260506 21:41:46.394 | INFO | "gpu" : " 17 GB", | 20260506 21:41:46.394 | INFO | "huge" : "", | 20260506 21:41:46.394 | INFO | "huge_shared" : " 27 GB", | 20260506 21:41:46.394 | INFO | "malloc" : " 14 GB", | 20260506 21:41:46.394 | INFO | "pinned" : " 29 GB" | 20260506 21:41:46.394 | INFO | } | 20260506 21:41:46.394 | INFO | Perf report for the last 5.13757 seconds: | 20260506 21:41:46.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:41:46.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:46.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:46.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:46.394 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 21:41:46.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:46.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:46.394 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:41:46.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:46.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:46.394 | INFO | Uploaded 36GiB to GPU in 1.985s (18.136GiB/s) | 20260506 21:41:46.394 | INFO | Downloaded 34GiB to GPU in 1.93277s (17.5913GiB/s) | 20260506 21:41:51.531 | INFO | Analyzing chunk frames = [4663808,4664320) | 20260506 21:41:51.531 | INFO | MemInfo: { | 20260506 21:41:51.531 | INFO | "gpu" : " 17 GB", | 20260506 21:41:51.531 | INFO | "huge" : "", | 20260506 21:41:51.531 | INFO | "huge_shared" : " 27 GB", | 20260506 21:41:51.532 | INFO | "malloc" : " 14 GB", | 20260506 21:41:51.532 | INFO | "pinned" : " 29 GB" | 20260506 21:41:51.532 | INFO | } | 20260506 21:41:51.532 | INFO | Perf report for the last 5.13785 seconds: | 20260506 21:41:51.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:41:51.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:51.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:51.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:51.532 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:41:51.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:51.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:51.532 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 21:41:51.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:51.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:51.532 | INFO | Uploaded 36GiB to GPU in 2.02403s (17.7863GiB/s) | 20260506 21:41:51.532 | INFO | Downloaded 34GiB to GPU in 1.95164s (17.4212GiB/s) | 20260506 21:41:56.669 | INFO | Analyzing chunk frames = [4664320,4664832) | 20260506 21:41:56.669 | INFO | MemInfo: { | 20260506 21:41:56.669 | INFO | "gpu" : " 17 GB", | 20260506 21:41:56.669 | INFO | "huge" : "", | 20260506 21:41:56.669 | INFO | "huge_shared" : " 27 GB", | 20260506 21:41:56.669 | INFO | "malloc" : " 14 GB", | 20260506 21:41:56.669 | INFO | "pinned" : " 29 GB" | 20260506 21:41:56.669 | INFO | } | 20260506 21:41:56.669 | INFO | Perf report for the last 5.13782 seconds: | 20260506 21:41:56.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:41:56.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:56.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:56.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:56.669 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:41:56.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:56.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:56.669 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:41:56.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:56.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:41:56.670 | INFO | Uploaded 36GiB to GPU in 1.96155s (18.3528GiB/s) | 20260506 21:41:56.670 | INFO | Downloaded 34GiB to GPU in 2.00373s (16.9684GiB/s) | 20260506 21:42:01.802 | INFO | Analyzing chunk frames = [4664832,4665344) | 20260506 21:42:01.802 | INFO | MemInfo: { | 20260506 21:42:01.802 | INFO | "gpu" : " 17 GB", | 20260506 21:42:01.802 | INFO | "huge" : "", | 20260506 21:42:01.802 | INFO | "huge_shared" : " 27 GB", | 20260506 21:42:01.802 | INFO | "malloc" : " 14 GB", | 20260506 21:42:01.802 | INFO | "pinned" : " 29 GB" | 20260506 21:42:01.802 | INFO | } | 20260506 21:42:01.802 | INFO | Perf report for the last 5.13296 seconds: | 20260506 21:42:01.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:42:01.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:01.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:01.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:01.802 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:42:01.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:01.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:01.802 | INFO | PRE_HQ : 30.2%, 1.0, 15.6ms, 0 | 20260506 21:42:01.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:01.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:01.803 | INFO | Uploaded 36GiB to GPU in 2.03953s (17.6511GiB/s) | 20260506 21:42:01.803 | INFO | Downloaded 34GiB to GPU in 1.99591s (17.0348GiB/s) | 20260506 21:42:01.920 | INFO | Starting a round of Baz IO | 20260506 21:42:05.038 | INFO | Populating RTMetrics report | 20260506 21:42:05.038 | INFO | RTMetrics report populated | 20260506 21:42:05.038 | INFO | Opening tmp RTMetrics json file | 20260506 21:42:05.058 | INFO | Serializing RTMetrics to json | 20260506 21:42:05.084 | INFO | Serialization done, swapping tmp file now... | 20260506 21:42:05.084 | INFO | RTMetrics done | 20260506 21:42:05.161 | INFO | Finished a round of Baz IO: Wrote 1.936 GiB in 3.241 seconds | 20260506 21:42:05.161 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.87% / 0.00% / 66.13% | 20260506 21:42:05.161 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.584094% / 3.97364e-06% / 5.16574e-05% | 20260506 21:42:05.161 | INFO | Baz IO overall: | 20260506 21:42:05.161 | INFO | BytesWritten : 2456526574403 | 20260506 21:42:05.161 | INFO | NumEvents : 1185874519089 | 20260506 21:42:06.947 | INFO | Analyzing chunk frames = [4665344,4665856) | 20260506 21:42:06.947 | INFO | MemInfo: { | 20260506 21:42:06.947 | INFO | "gpu" : " 17 GB", | 20260506 21:42:06.947 | INFO | "huge" : "", | 20260506 21:42:06.947 | INFO | "huge_shared" : " 27 GB", | 20260506 21:42:06.947 | INFO | "malloc" : " 14 GB", | 20260506 21:42:06.947 | INFO | "pinned" : " 29 GB" | 20260506 21:42:06.947 | INFO | } | 20260506 21:42:06.947 | INFO | Perf report for the last 5.14509 seconds: | 20260506 21:42:06.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:42:06.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:06.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:06.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:06.948 | INFO | ANALYSIS : 62.3%, 2.9, 95.3ms, 0 | 20260506 21:42:06.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:06.948 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 21:42:06.948 | INFO | PRE_HQ : 59.5%, 1.8, 56.8ms, 0 | 20260506 21:42:06.948 | INFO | BAZWRITER : 62.7%, 4.8, 154.9ms, 0 | 20260506 21:42:06.948 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 21:42:06.948 | INFO | Uploaded 36GiB to GPU in 2.05888s (17.4852GiB/s) | 20260506 21:42:06.948 | INFO | Downloaded 38GiB to GPU in 2.21463s (17.1586GiB/s) | 20260506 21:42:12.092 | INFO | Analyzing chunk frames = [4665856,4666368) | 20260506 21:42:12.092 | INFO | MemInfo: { | 20260506 21:42:12.092 | INFO | "gpu" : " 17 GB", | 20260506 21:42:12.092 | INFO | "huge" : "", | 20260506 21:42:12.092 | INFO | "huge_shared" : " 27 GB", | 20260506 21:42:12.092 | INFO | "malloc" : " 14 GB", | 20260506 21:42:12.092 | INFO | "pinned" : " 29 GB" | 20260506 21:42:12.092 | INFO | } | 20260506 21:42:12.092 | INFO | Perf report for the last 5.14485 seconds: | 20260506 21:42:12.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:42:12.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:12.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:12.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:12.092 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 21:42:12.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:12.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:12.092 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 21:42:12.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:12.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:12.093 | INFO | Uploaded 36GiB to GPU in 2.00459s (17.9588GiB/s) | 20260506 21:42:12.093 | INFO | Downloaded 34GiB to GPU in 1.93221s (17.5964GiB/s) | 20260506 21:42:17.228 | INFO | Analyzing chunk frames = [4666368,4666880) | 20260506 21:42:17.228 | INFO | MemInfo: { | 20260506 21:42:17.228 | INFO | "gpu" : " 17 GB", | 20260506 21:42:17.228 | INFO | "huge" : "", | 20260506 21:42:17.228 | INFO | "huge_shared" : " 27 GB", | 20260506 21:42:17.228 | INFO | "malloc" : " 14 GB", | 20260506 21:42:17.228 | INFO | "pinned" : " 29 GB" | 20260506 21:42:17.228 | INFO | } | 20260506 21:42:17.228 | INFO | Perf report for the last 5.13539 seconds: | 20260506 21:42:17.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:42:17.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:17.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:17.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:17.228 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 21:42:17.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:17.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:17.228 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:42:17.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:17.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:17.228 | INFO | Uploaded 36GiB to GPU in 1.95044s (18.4574GiB/s) | 20260506 21:42:17.228 | INFO | Downloaded 34GiB to GPU in 1.93217s (17.5968GiB/s) | 20260506 21:42:22.368 | INFO | Analyzing chunk frames = [4666880,4667392) | 20260506 21:42:22.368 | INFO | MemInfo: { | 20260506 21:42:22.368 | INFO | "gpu" : " 17 GB", | 20260506 21:42:22.368 | INFO | "huge" : "", | 20260506 21:42:22.368 | INFO | "huge_shared" : " 27 GB", | 20260506 21:42:22.368 | INFO | "malloc" : " 14 GB", | 20260506 21:42:22.368 | INFO | "pinned" : " 29 GB" | 20260506 21:42:22.368 | INFO | } | 20260506 21:42:22.368 | INFO | Perf report for the last 5.14017 seconds: | 20260506 21:42:22.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:42:22.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:22.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:22.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:22.368 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 21:42:22.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:22.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:22.368 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:42:22.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:22.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:22.368 | INFO | Uploaded 36GiB to GPU in 1.93519s (18.6028GiB/s) | 20260506 21:42:22.368 | INFO | Downloaded 34GiB to GPU in 1.87806s (18.1038GiB/s) | 20260506 21:42:27.503 | INFO | Analyzing chunk frames = [4667392,4667904) | 20260506 21:42:27.503 | INFO | MemInfo: { | 20260506 21:42:27.503 | INFO | "gpu" : " 17 GB", | 20260506 21:42:27.503 | INFO | "huge" : "", | 20260506 21:42:27.503 | INFO | "huge_shared" : " 27 GB", | 20260506 21:42:27.503 | INFO | "malloc" : " 14 GB", | 20260506 21:42:27.503 | INFO | "pinned" : " 29 GB" | 20260506 21:42:27.503 | INFO | } | 20260506 21:42:27.503 | INFO | Perf report for the last 5.1355 seconds: | 20260506 21:42:27.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:42:27.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:27.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:27.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:27.503 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 21:42:27.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:27.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:27.503 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:42:27.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:27.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:27.504 | INFO | Uploaded 36GiB to GPU in 2.00145s (17.9869GiB/s) | 20260506 21:42:27.504 | INFO | Downloaded 34GiB to GPU in 1.91939s (17.714GiB/s) | 20260506 21:42:32.641 | INFO | Analyzing chunk frames = [4667904,4668416) | 20260506 21:42:32.641 | INFO | MemInfo: { | 20260506 21:42:32.641 | INFO | "gpu" : " 17 GB", | 20260506 21:42:32.641 | INFO | "huge" : "", | 20260506 21:42:32.641 | INFO | "huge_shared" : " 27 GB", | 20260506 21:42:32.641 | INFO | "malloc" : " 14 GB", | 20260506 21:42:32.641 | INFO | "pinned" : " 29 GB" | 20260506 21:42:32.641 | INFO | } | 20260506 21:42:32.641 | INFO | Perf report for the last 5.13784 seconds: | 20260506 21:42:32.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:42:32.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:32.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:32.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:32.641 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 21:42:32.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:32.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:32.641 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 21:42:32.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:32.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:32.641 | INFO | Uploaded 36GiB to GPU in 1.9828s (18.1561GiB/s) | 20260506 21:42:32.642 | INFO | Downloaded 34GiB to GPU in 1.95785s (17.366GiB/s) | 20260506 21:42:37.779 | INFO | Analyzing chunk frames = [4668416,4668928) | 20260506 21:42:37.779 | INFO | MemInfo: { | 20260506 21:42:37.779 | INFO | "gpu" : " 17 GB", | 20260506 21:42:37.779 | INFO | "huge" : "", | 20260506 21:42:37.780 | INFO | "huge_shared" : " 27 GB", | 20260506 21:42:37.780 | INFO | "malloc" : " 14 GB", | 20260506 21:42:37.780 | INFO | "pinned" : " 29 GB" | 20260506 21:42:37.780 | INFO | } | 20260506 21:42:37.780 | INFO | Perf report for the last 5.1383 seconds: | 20260506 21:42:37.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:42:37.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:37.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:37.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:37.780 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:42:37.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:37.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:37.780 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 21:42:37.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:37.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:37.780 | INFO | Uploaded 36GiB to GPU in 1.94879s (18.473GiB/s) | 20260506 21:42:37.780 | INFO | Downloaded 34GiB to GPU in 1.98982s (17.087GiB/s) | 20260506 21:42:42.921 | INFO | Analyzing chunk frames = [4668928,4669440) | 20260506 21:42:42.921 | INFO | MemInfo: { | 20260506 21:42:42.921 | INFO | "gpu" : " 17 GB", | 20260506 21:42:42.921 | INFO | "huge" : "", | 20260506 21:42:42.921 | INFO | "huge_shared" : " 27 GB", | 20260506 21:42:42.921 | INFO | "malloc" : " 14 GB", | 20260506 21:42:42.921 | INFO | "pinned" : " 29 GB" | 20260506 21:42:42.921 | INFO | } | 20260506 21:42:42.921 | INFO | Perf report for the last 5.14116 seconds: | 20260506 21:42:42.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:42:42.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:42.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:42.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:42.921 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 21:42:42.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:42.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:42.921 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 21:42:42.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:42.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:42.921 | INFO | Uploaded 36GiB to GPU in 2.05694s (17.5017GiB/s) | 20260506 21:42:42.921 | INFO | Downloaded 34GiB to GPU in 2.00112s (16.9905GiB/s) | 20260506 21:42:42.921 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:42:42.921 | INFO | Total mem usage: 18 GiB | 20260506 21:42:42.921 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:42:42.921 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.5%) | 20260506 21:42:42.921 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:42:42.921 | INFO | High Water Breakdown: | 20260506 21:42:42.921 | INFO | 61.4% : 9.49 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:42:42.921 | INFO | 38.5% : 5.95 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:42:42.921 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:42:42.921 | INFO | ------------------------------------------ | 20260506 21:42:42.921 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:42:42.921 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:42:42.921 | INFO | Total mem usage: 26.5 GiB | 20260506 21:42:42.921 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:42:42.921 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:42:42.921 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:42:42.921 | INFO | High Water Breakdown: | 20260506 21:42:42.921 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:42:42.921 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:42:42.921 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:42:42.921 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:42:42.921 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:42:42.921 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:42:42.921 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:42:42.921 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:42:42.921 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:42:42.921 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:42:42.921 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:42:42.921 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:42:42.921 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:42:42.921 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:42:42.921 | INFO | ------------------------------------------ | 20260506 21:42:42.921 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:42:42.921 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:42:42.921 | INFO | Total mem usage: 25.6 GiB | 20260506 21:42:42.921 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:42:42.921 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 21:42:42.921 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 21:42:42.921 | INFO | High Water Breakdown: | 20260506 21:42:42.921 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 21:42:42.921 | INFO | ------------------------------------------ | 20260506 21:42:42.921 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:42:42.922 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:42:42.922 | INFO | Total mem usage: 15.7 GiB | 20260506 21:42:42.922 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:42:42.922 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:42:42.922 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:42:42.922 | INFO | High Water Breakdown: | 20260506 21:42:42.922 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:42:42.922 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:42:42.922 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:42:42.922 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:42:42.922 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:42:42.922 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:42:42.922 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:42:42.922 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:42:42.922 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:42:42.922 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:42:42.922 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:42:42.922 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:42:42.922 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:42:42.922 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:42:42.922 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:42:42.922 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:42:42.922 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:42:42.922 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:42:42.922 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:42:42.922 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:42:42.922 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:42:42.922 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:42:42.922 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:42:42.922 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:42:42.922 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:42:42.922 | INFO | ------------------------------------------ | 20260506 21:42:42.922 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:42:42.922 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:42:42.922 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:42:42.922 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:42:42.922 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.62e+01, 1.35e+01 / 2.51e+00 | 20260506 21:42:42.922 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.02e-01, 1.06e-02 / 7.32e-03 | 20260506 21:42:42.922 | INFO | PerfReport: 11.64%, Baseline -- 3.62e+00 / 4.35e+00, 5.49e-01 / 3.87e-01 | 20260506 21:42:42.922 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.41e+00, 1.80e-01 / 1.45e-01 | 20260506 21:42:42.922 | INFO | PerfReport: 16.07%, DME -- 4.00e+01 / 4.51e+01, 4.21e+00 / 2.91e+00 | 20260506 21:42:42.922 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.35e+00 / 1.85e+00 | 20260506 21:42:42.922 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.20e+00 / 2.75e+00, 3.74e-01 / 2.25e-01 | 20260506 21:42:42.922 | INFO | PerfReport: 3.25%, Metrics -- 1.01e+00 / 1.35e+00, 2.68e-01 / 1.42e-01 | 20260506 21:42:42.922 | INFO | PerfReport: 1.12%, PulseSerializing -- 3.47e-01 / 3.99e-01, 5.47e-02 / 2.43e-02 | 20260506 21:42:42.922 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:42:42.922 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:42:42.922 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:42:42.922 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 21:42:42.922 | INFO | PerfReport: 21.50%, Upload -- 2.08e+01 / 0.00e+00, 6.91e+00 / 0.00e+00 | 20260506 21:42:42.922 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 21:42:42.922 | INFO | PerfReport: 32.16%, Compute -- 3.11e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 21:42:42.922 | INFO | PerfReport: 1.20%, LockUpload -- 1.16e+00 / 0.00e+00, 4.57e+00 / 0.00e+00 | 20260506 21:42:42.922 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.16e+00 / 0.00e+00 | 20260506 21:42:42.922 | INFO | PerfReport: 22.49%, LockCompute -- 2.17e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 21:42:42.922 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260506 21:42:43.034 | INFO | Starting a round of Baz IO | 20260506 21:42:46.163 | INFO | Populating RTMetrics report | 20260506 21:42:46.163 | INFO | RTMetrics report populated | 20260506 21:42:46.163 | INFO | Opening tmp RTMetrics json file | 20260506 21:42:46.178 | INFO | Serializing RTMetrics to json | 20260506 21:42:46.184 | INFO | Serialization done, swapping tmp file now... | 20260506 21:42:46.188 | INFO | RTMetrics done | 20260506 21:42:46.601 | INFO | Finished a round of Baz IO: Wrote 1.936 GiB in 3.567 seconds | 20260506 21:42:46.601 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.86% / 0.00% / 66.13% | 20260506 21:42:46.601 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.581984% / 0% / 5.16574e-05% | 20260506 21:42:46.601 | INFO | Baz IO overall: | 20260506 21:42:46.601 | INFO | BytesWritten : 2458605509861 | 20260506 21:42:46.601 | INFO | NumEvents : 1186868727546 | 20260506 21:42:48.065 | INFO | Analyzing chunk frames = [4669440,4669952) | 20260506 21:42:48.065 | INFO | MemInfo: { | 20260506 21:42:48.065 | INFO | "gpu" : " 17 GB", | 20260506 21:42:48.065 | INFO | "huge" : "", | 20260506 21:42:48.065 | INFO | "huge_shared" : " 27 GB", | 20260506 21:42:48.065 | INFO | "malloc" : " 14 GB", | 20260506 21:42:48.065 | INFO | "pinned" : " 29 GB" | 20260506 21:42:48.065 | INFO | } | 20260506 21:42:48.065 | INFO | Perf report for the last 5.1446 seconds: | 20260506 21:42:48.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:42:48.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:48.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:48.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:48.065 | INFO | ANALYSIS : 62.5%, 2.9, 95.9ms, 0 | 20260506 21:42:48.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:48.065 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 21:42:48.065 | INFO | PRE_HQ : 59.7%, 1.8, 56.6ms, 0 | 20260506 21:42:48.065 | INFO | BAZWRITER : 69.0%, 4.8, 169.5ms, 0 | 20260506 21:42:48.065 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 21:42:48.066 | INFO | Uploaded 36GiB to GPU in 2.0579s (17.4936GiB/s) | 20260506 21:42:48.066 | INFO | Downloaded 38GiB to GPU in 2.20633s (17.2232GiB/s) | 20260506 21:42:53.208 | INFO | Analyzing chunk frames = [4669952,4670464) | 20260506 21:42:53.208 | INFO | MemInfo: { | 20260506 21:42:53.208 | INFO | "gpu" : " 17 GB", | 20260506 21:42:53.208 | INFO | "huge" : "", | 20260506 21:42:53.208 | INFO | "huge_shared" : " 27 GB", | 20260506 21:42:53.208 | INFO | "malloc" : " 14 GB", | 20260506 21:42:53.208 | INFO | "pinned" : " 29 GB" | 20260506 21:42:53.208 | INFO | } | 20260506 21:42:53.208 | INFO | Perf report for the last 5.14227 seconds: | 20260506 21:42:53.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:42:53.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:53.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:53.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:53.208 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 21:42:53.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:53.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:53.208 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 21:42:53.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:53.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:53.208 | INFO | Uploaded 36GiB to GPU in 2.01327s (17.8813GiB/s) | 20260506 21:42:53.208 | INFO | Downloaded 34GiB to GPU in 1.92625s (17.6509GiB/s) | 20260506 21:42:58.352 | INFO | Analyzing chunk frames = [4670464,4670976) | 20260506 21:42:58.352 | INFO | MemInfo: { | 20260506 21:42:58.352 | INFO | "gpu" : " 17 GB", | 20260506 21:42:58.352 | INFO | "huge" : "", | 20260506 21:42:58.352 | INFO | "huge_shared" : " 27 GB", | 20260506 21:42:58.352 | INFO | "malloc" : " 14 GB", | 20260506 21:42:58.352 | INFO | "pinned" : " 29 GB" | 20260506 21:42:58.352 | INFO | } | 20260506 21:42:58.352 | INFO | Perf report for the last 5.14412 seconds: | 20260506 21:42:58.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:42:58.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:58.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:58.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:58.352 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 21:42:58.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:58.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:58.352 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:42:58.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:58.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:42:58.352 | INFO | Uploaded 36GiB to GPU in 1.98787s (18.1098GiB/s) | 20260506 21:42:58.352 | INFO | Downloaded 34GiB to GPU in 1.93018s (17.6149GiB/s) | 20260506 21:43:03.481 | INFO | Analyzing chunk frames = [4670976,4671488) | 20260506 21:43:03.481 | INFO | MemInfo: { | 20260506 21:43:03.481 | INFO | "gpu" : " 17 GB", | 20260506 21:43:03.481 | INFO | "huge" : "", | 20260506 21:43:03.481 | INFO | "huge_shared" : " 27 GB", | 20260506 21:43:03.481 | INFO | "malloc" : " 14 GB", | 20260506 21:43:03.481 | INFO | "pinned" : " 29 GB" | 20260506 21:43:03.481 | INFO | } | 20260506 21:43:03.481 | INFO | Perf report for the last 5.12964 seconds: | 20260506 21:43:03.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:43:03.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:03.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:03.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:03.482 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260506 21:43:03.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:03.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:03.482 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:43:03.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:03.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:03.482 | INFO | Uploaded 36GiB to GPU in 1.99809s (18.0172GiB/s) | 20260506 21:43:03.482 | INFO | Downloaded 34GiB to GPU in 1.91025s (17.7987GiB/s) | 20260506 21:43:08.618 | INFO | Analyzing chunk frames = [4671488,4672000) | 20260506 21:43:08.618 | INFO | MemInfo: { | 20260506 21:43:08.618 | INFO | "gpu" : " 17 GB", | 20260506 21:43:08.618 | INFO | "huge" : "", | 20260506 21:43:08.618 | INFO | "huge_shared" : " 27 GB", | 20260506 21:43:08.618 | INFO | "malloc" : " 14 GB", | 20260506 21:43:08.618 | INFO | "pinned" : " 29 GB" | 20260506 21:43:08.618 | INFO | } | 20260506 21:43:08.618 | INFO | Perf report for the last 5.1364 seconds: | 20260506 21:43:08.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:43:08.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:08.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:08.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:08.618 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 21:43:08.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:08.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:08.618 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:43:08.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:08.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:08.618 | INFO | Uploaded 36GiB to GPU in 2.01342s (17.88GiB/s) | 20260506 21:43:08.618 | INFO | Downloaded 34GiB to GPU in 1.90622s (17.8363GiB/s) | 20260506 21:43:13.756 | INFO | Analyzing chunk frames = [4672000,4672512) | 20260506 21:43:13.756 | INFO | MemInfo: { | 20260506 21:43:13.756 | INFO | "gpu" : " 17 GB", | 20260506 21:43:13.757 | INFO | "huge" : "", | 20260506 21:43:13.757 | INFO | "huge_shared" : " 27 GB", | 20260506 21:43:13.757 | INFO | "malloc" : " 14 GB", | 20260506 21:43:13.757 | INFO | "pinned" : " 29 GB" | 20260506 21:43:13.757 | INFO | } | 20260506 21:43:13.757 | INFO | Perf report for the last 5.13868 seconds: | 20260506 21:43:13.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:43:13.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:13.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:13.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:13.757 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 21:43:13.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:13.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:13.757 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:43:13.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:13.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:13.757 | INFO | Uploaded 36GiB to GPU in 2.01178s (17.8946GiB/s) | 20260506 21:43:13.757 | INFO | Downloaded 34GiB to GPU in 1.9525s (17.4136GiB/s) | 20260506 21:43:18.896 | INFO | Analyzing chunk frames = [4672512,4673024) | 20260506 21:43:18.896 | INFO | MemInfo: { | 20260506 21:43:18.896 | INFO | "gpu" : " 17 GB", | 20260506 21:43:18.896 | INFO | "huge" : "", | 20260506 21:43:18.896 | INFO | "huge_shared" : " 27 GB", | 20260506 21:43:18.896 | INFO | "malloc" : " 14 GB", | 20260506 21:43:18.896 | INFO | "pinned" : " 29 GB" | 20260506 21:43:18.896 | INFO | } | 20260506 21:43:18.897 | INFO | Perf report for the last 5.13995 seconds: | 20260506 21:43:18.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:43:18.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:18.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:18.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:18.897 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:43:18.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:18.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:18.897 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 21:43:18.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:18.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:18.897 | INFO | Uploaded 36GiB to GPU in 1.98047s (18.1775GiB/s) | 20260506 21:43:18.897 | INFO | Downloaded 34GiB to GPU in 2.00797s (16.9325GiB/s) | 20260506 21:43:24.038 | INFO | Analyzing chunk frames = [4673024,4673536) | 20260506 21:43:24.038 | INFO | MemInfo: { | 20260506 21:43:24.038 | INFO | "gpu" : " 17 GB", | 20260506 21:43:24.038 | INFO | "huge" : "", | 20260506 21:43:24.038 | INFO | "huge_shared" : " 27 GB", | 20260506 21:43:24.038 | INFO | "malloc" : " 14 GB", | 20260506 21:43:24.038 | INFO | "pinned" : " 29 GB" | 20260506 21:43:24.038 | INFO | } | 20260506 21:43:24.038 | INFO | Perf report for the last 5.14168 seconds: | 20260506 21:43:24.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:43:24.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:24.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:24.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:24.038 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:43:24.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:24.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:24.038 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 21:43:24.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:24.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:24.039 | INFO | Uploaded 36GiB to GPU in 2.05001s (17.5609GiB/s) | 20260506 21:43:24.039 | INFO | Downloaded 34GiB to GPU in 1.991s (17.0768GiB/s) | 20260506 21:43:24.151 | INFO | Starting a round of Baz IO | 20260506 21:43:27.274 | INFO | Populating RTMetrics report | 20260506 21:43:27.274 | INFO | RTMetrics report populated | 20260506 21:43:27.274 | INFO | Opening tmp RTMetrics json file | 20260506 21:43:27.303 | INFO | Serializing RTMetrics to json | 20260506 21:43:27.377 | INFO | Serialization done, swapping tmp file now... | 20260506 21:43:27.389 | INFO | RTMetrics done | 20260506 21:43:27.669 | INFO | Finished a round of Baz IO: Wrote 1.937 GiB in 3.518 seconds | 20260506 21:43:27.669 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.86% / 0.00% / 66.14% | 20260506 21:43:27.669 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.582238% / 0% / 7.94729e-05% | 20260506 21:43:27.669 | INFO | Baz IO overall: | 20260506 21:43:27.669 | INFO | BytesWritten : 2460685354817 | 20260506 21:43:27.669 | INFO | NumEvents : 1187863689633 | 20260506 21:43:29.176 | INFO | Analyzing chunk frames = [4673536,4674048) | 20260506 21:43:29.176 | INFO | MemInfo: { | 20260506 21:43:29.176 | INFO | "gpu" : " 17 GB", | 20260506 21:43:29.176 | INFO | "huge" : "", | 20260506 21:43:29.176 | INFO | "huge_shared" : " 27 GB", | 20260506 21:43:29.176 | INFO | "malloc" : " 14 GB", | 20260506 21:43:29.176 | INFO | "pinned" : " 29 GB" | 20260506 21:43:29.176 | INFO | } | 20260506 21:43:29.176 | INFO | Perf report for the last 5.13784 seconds: | 20260506 21:43:29.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:43:29.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:29.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:29.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:29.176 | INFO | ANALYSIS : 62.4%, 2.9, 95.4ms, 0 | 20260506 21:43:29.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:29.176 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260506 21:43:29.176 | INFO | PRE_HQ : 59.5%, 1.8, 56.9ms, 0 | 20260506 21:43:29.176 | INFO | BAZWRITER : 68.1%, 4.8, 167.5ms, 0 | 20260506 21:43:29.176 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 21:43:29.176 | INFO | Uploaded 36GiB to GPU in 2.06358s (17.4454GiB/s) | 20260506 21:43:29.176 | INFO | Downloaded 38GiB to GPU in 2.20167s (17.2596GiB/s) | 20260506 21:43:34.318 | INFO | Analyzing chunk frames = [4674048,4674560) | 20260506 21:43:34.318 | INFO | MemInfo: { | 20260506 21:43:34.318 | INFO | "gpu" : " 17 GB", | 20260506 21:43:34.318 | INFO | "huge" : "", | 20260506 21:43:34.318 | INFO | "huge_shared" : " 27 GB", | 20260506 21:43:34.318 | INFO | "malloc" : " 14 GB", | 20260506 21:43:34.318 | INFO | "pinned" : " 29 GB" | 20260506 21:43:34.318 | INFO | } | 20260506 21:43:34.318 | INFO | Perf report for the last 5.14235 seconds: | 20260506 21:43:34.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:43:34.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:34.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:34.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:34.319 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:43:34.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:34.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:34.319 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:43:34.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:34.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:34.319 | INFO | Uploaded 36GiB to GPU in 2.00394s (17.9646GiB/s) | 20260506 21:43:34.319 | INFO | Downloaded 34GiB to GPU in 1.92112s (17.698GiB/s) | 20260506 21:43:39.460 | INFO | Analyzing chunk frames = [4674560,4675072) | 20260506 21:43:39.460 | INFO | MemInfo: { | 20260506 21:43:39.460 | INFO | "gpu" : " 17 GB", | 20260506 21:43:39.460 | INFO | "huge" : "", | 20260506 21:43:39.460 | INFO | "huge_shared" : " 27 GB", | 20260506 21:43:39.460 | INFO | "malloc" : " 14 GB", | 20260506 21:43:39.460 | INFO | "pinned" : " 29 GB" | 20260506 21:43:39.460 | INFO | } | 20260506 21:43:39.460 | INFO | Perf report for the last 5.14156 seconds: | 20260506 21:43:39.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:43:39.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:39.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:39.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:39.460 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 21:43:39.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:39.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:39.460 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:43:39.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:39.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:39.460 | INFO | Uploaded 36GiB to GPU in 1.98109s (18.1718GiB/s) | 20260506 21:43:39.460 | INFO | Downloaded 34GiB to GPU in 1.91506s (17.754GiB/s) | 20260506 21:43:44.597 | INFO | Analyzing chunk frames = [4675072,4675584) | 20260506 21:43:44.597 | INFO | MemInfo: { | 20260506 21:43:44.597 | INFO | "gpu" : " 17 GB", | 20260506 21:43:44.597 | INFO | "huge" : "", | 20260506 21:43:44.597 | INFO | "huge_shared" : " 27 GB", | 20260506 21:43:44.597 | INFO | "malloc" : " 14 GB", | 20260506 21:43:44.597 | INFO | "pinned" : " 29 GB" | 20260506 21:43:44.597 | INFO | } | 20260506 21:43:44.597 | INFO | Perf report for the last 5.13743 seconds: | 20260506 21:43:44.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:43:44.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:44.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:44.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:44.597 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 21:43:44.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:44.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:44.598 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:43:44.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:44.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:44.598 | INFO | Uploaded 36GiB to GPU in 2.02078s (17.8149GiB/s) | 20260506 21:43:44.598 | INFO | Downloaded 34GiB to GPU in 1.90595s (17.8389GiB/s) | 20260506 21:43:49.736 | INFO | Analyzing chunk frames = [4675584,4676096) | 20260506 21:43:49.736 | INFO | MemInfo: { | 20260506 21:43:49.736 | INFO | "gpu" : " 17 GB", | 20260506 21:43:49.736 | INFO | "huge" : "", | 20260506 21:43:49.736 | INFO | "huge_shared" : " 27 GB", | 20260506 21:43:49.736 | INFO | "malloc" : " 14 GB", | 20260506 21:43:49.736 | INFO | "pinned" : " 29 GB" | 20260506 21:43:49.736 | INFO | } | 20260506 21:43:49.736 | INFO | Perf report for the last 5.13844 seconds: | 20260506 21:43:49.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:43:49.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:49.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:49.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:49.736 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 21:43:49.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:49.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:49.736 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:43:49.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:49.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:49.736 | INFO | Uploaded 36GiB to GPU in 2.01699s (17.8484GiB/s) | 20260506 21:43:49.736 | INFO | Downloaded 34GiB to GPU in 1.91841s (17.7231GiB/s) | 20260506 21:43:54.870 | INFO | Analyzing chunk frames = [4676096,4676608) | 20260506 21:43:54.870 | INFO | MemInfo: { | 20260506 21:43:54.870 | INFO | "gpu" : " 17 GB", | 20260506 21:43:54.870 | INFO | "huge" : "", | 20260506 21:43:54.870 | INFO | "huge_shared" : " 27 GB", | 20260506 21:43:54.870 | INFO | "malloc" : " 14 GB", | 20260506 21:43:54.870 | INFO | "pinned" : " 29 GB" | 20260506 21:43:54.870 | INFO | } | 20260506 21:43:54.870 | INFO | Perf report for the last 5.13407 seconds: | 20260506 21:43:54.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:43:54.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:54.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:54.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:54.870 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 21:43:54.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:54.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:54.870 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 21:43:54.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:54.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:43:54.870 | INFO | Uploaded 36GiB to GPU in 2.02789s (17.7524GiB/s) | 20260506 21:43:54.870 | INFO | Downloaded 34GiB to GPU in 1.94322s (17.4967GiB/s) | 20260506 21:44:00.011 | INFO | Analyzing chunk frames = [4676608,4677120) | 20260506 21:44:00.011 | INFO | MemInfo: { | 20260506 21:44:00.011 | INFO | "gpu" : " 17 GB", | 20260506 21:44:00.011 | INFO | "huge" : "", | 20260506 21:44:00.011 | INFO | "huge_shared" : " 27 GB", | 20260506 21:44:00.011 | INFO | "malloc" : " 14 GB", | 20260506 21:44:00.011 | INFO | "pinned" : " 29 GB" | 20260506 21:44:00.011 | INFO | } | 20260506 21:44:00.011 | INFO | Perf report for the last 5.1412 seconds: | 20260506 21:44:00.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:44:00.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:00.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:00.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:00.011 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 21:44:00.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:00.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:00.011 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:44:00.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:00.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:00.011 | INFO | Uploaded 36GiB to GPU in 1.94656s (18.4942GiB/s) | 20260506 21:44:00.012 | INFO | Downloaded 34GiB to GPU in 1.99488s (17.0436GiB/s) | 20260506 21:44:05.157 | INFO | Analyzing chunk frames = [4677120,4677632) | 20260506 21:44:05.157 | INFO | MemInfo: { | 20260506 21:44:05.157 | INFO | "gpu" : " 17 GB", | 20260506 21:44:05.157 | INFO | "huge" : "", | 20260506 21:44:05.157 | INFO | "huge_shared" : " 27 GB", | 20260506 21:44:05.157 | INFO | "malloc" : " 14 GB", | 20260506 21:44:05.157 | INFO | "pinned" : " 29 GB" | 20260506 21:44:05.157 | INFO | } | 20260506 21:44:05.157 | INFO | Perf report for the last 5.14616 seconds: | 20260506 21:44:05.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:44:05.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:05.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:05.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:05.157 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 21:44:05.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:05.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:05.157 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:44:05.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:05.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:05.158 | INFO | Uploaded 36GiB to GPU in 2.02914s (17.7415GiB/s) | 20260506 21:44:05.158 | INFO | Downloaded 34GiB to GPU in 1.98931s (17.0913GiB/s) | 20260506 21:44:05.158 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:44:05.158 | INFO | Total mem usage: 18 GiB | 20260506 21:44:05.158 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:44:05.158 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -14%) | 20260506 21:44:05.158 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:44:05.158 | INFO | High Water Breakdown: | 20260506 21:44:05.158 | INFO | 61.7% : 9.74 GiB ( 85.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:44:05.158 | INFO | 38.2% : 6.03 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:44:05.158 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:44:05.158 | INFO | ------------------------------------------ | 20260506 21:44:05.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:44:05.158 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:44:05.158 | INFO | Total mem usage: 26.5 GiB | 20260506 21:44:05.158 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:44:05.158 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:44:05.158 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:44:05.158 | INFO | High Water Breakdown: | 20260506 21:44:05.158 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:44:05.158 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:44:05.158 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:44:05.158 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:44:05.158 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:44:05.158 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:44:05.158 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:44:05.158 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:44:05.158 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:44:05.158 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:44:05.158 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:44:05.158 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:44:05.158 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:44:05.158 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:44:05.158 | INFO | ------------------------------------------ | 20260506 21:44:05.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:44:05.158 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:44:05.158 | INFO | Total mem usage: 25.6 GiB | 20260506 21:44:05.158 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:44:05.158 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 21:44:05.158 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 21:44:05.158 | INFO | High Water Breakdown: | 20260506 21:44:05.158 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 21:44:05.158 | INFO | ------------------------------------------ | 20260506 21:44:05.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:44:05.158 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:44:05.158 | INFO | Total mem usage: 15.7 GiB | 20260506 21:44:05.158 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:44:05.158 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:44:05.158 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:44:05.158 | INFO | High Water Breakdown: | 20260506 21:44:05.158 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:44:05.158 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:44:05.158 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:44:05.158 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:44:05.158 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:44:05.158 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:44:05.158 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:44:05.158 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:44:05.158 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:44:05.158 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:44:05.158 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:44:05.158 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:44:05.158 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:44:05.158 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:44:05.158 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:44:05.158 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:44:05.158 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:44:05.158 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:44:05.158 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:44:05.158 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:44:05.158 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:44:05.158 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:44:05.158 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:44:05.158 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:44:05.158 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:44:05.158 | INFO | ------------------------------------------ | 20260506 21:44:05.158 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:44:05.158 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:44:05.158 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:44:05.158 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:44:05.158 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.60e+01, 1.34e+01 / 2.21e+00 | 20260506 21:44:05.158 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.02e-01, 1.10e-02 / 4.83e-03 | 20260506 21:44:05.158 | INFO | PerfReport: 11.66%, Baseline -- 3.63e+00 / 4.35e+00, 5.50e-01 / 3.00e-01 | 20260506 21:44:05.159 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.46e+00, 1.80e-01 / 1.10e-01 | 20260506 21:44:05.159 | INFO | PerfReport: 15.99%, DME -- 3.98e+01 / 4.43e+01, 4.06e+00 / 2.35e+00 | 20260506 21:44:05.159 | INFO | PerfReport: 57.32%, FrameLabeling -- 1.78e+01 / 2.13e+01, 1.42e+00 / 1.94e+00 | 20260506 21:44:05.159 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.21e+00 / 2.74e+00, 3.70e-01 / 2.30e-01 | 20260506 21:44:05.159 | INFO | PerfReport: 3.26%, Metrics -- 1.01e+00 / 1.33e+00, 2.67e-01 / 1.84e-01 | 20260506 21:44:05.159 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.47e-01 / 3.92e-01, 5.42e-02 / 2.03e-02 | 20260506 21:44:05.159 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:44:05.159 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:44:05.159 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:44:05.159 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 21:44:05.159 | INFO | PerfReport: 21.66%, Upload -- 2.10e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260506 21:44:05.159 | INFO | PerfReport: 21.27%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 21:44:05.159 | INFO | PerfReport: 32.14%, Compute -- 3.11e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 21:44:05.159 | INFO | PerfReport: 1.39%, LockUpload -- 1.35e+00 / 0.00e+00, 4.94e+00 / 0.00e+00 | 20260506 21:44:05.159 | INFO | PerfReport: 1.11%, LockDownload -- 1.08e+00 / 0.00e+00, 2.74e+00 / 0.00e+00 | 20260506 21:44:05.159 | INFO | PerfReport: 22.40%, LockCompute -- 2.17e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 21:44:05.159 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260506 21:44:05.258 | INFO | Starting a round of Baz IO | 20260506 21:44:08.391 | INFO | Populating RTMetrics report | 20260506 21:44:08.391 | INFO | RTMetrics report populated | 20260506 21:44:08.391 | INFO | Opening tmp RTMetrics json file | 20260506 21:44:08.545 | INFO | Serializing RTMetrics to json | 20260506 21:44:08.550 | INFO | Serialization done, swapping tmp file now... | 20260506 21:44:08.555 | INFO | RTMetrics done | 20260506 21:44:09.048 | INFO | Finished a round of Baz IO: Wrote 1.936 GiB in 3.789 seconds | 20260506 21:44:09.048 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.86% / 0.00% / 66.14% | 20260506 21:44:09.048 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.580005% / 7.94729e-06% / 8.34465e-05% | 20260506 21:44:09.048 | INFO | Baz IO overall: | 20260506 21:44:09.048 | INFO | BytesWritten : 2462763929521 | 20260506 21:44:09.048 | INFO | NumEvents : 1188858083600 | 20260506 21:44:10.293 | INFO | Analyzing chunk frames = [4677632,4678144) | 20260506 21:44:10.293 | INFO | MemInfo: { | 20260506 21:44:10.293 | INFO | "gpu" : " 17 GB", | 20260506 21:44:10.293 | INFO | "huge" : "", | 20260506 21:44:10.293 | INFO | "huge_shared" : " 27 GB", | 20260506 21:44:10.293 | INFO | "malloc" : " 14 GB", | 20260506 21:44:10.293 | INFO | "pinned" : " 29 GB" | 20260506 21:44:10.293 | INFO | } | 20260506 21:44:10.293 | INFO | Perf report for the last 5.13578 seconds: | 20260506 21:44:10.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:44:10.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:10.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:10.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:10.293 | INFO | ANALYSIS : 62.4%, 2.9, 95.3ms, 0 | 20260506 21:44:10.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:10.293 | INFO | RT_METRICS : 45.9%, 1.0, 24.0ms, 0 | 20260506 21:44:10.293 | INFO | PRE_HQ : 60.0%, 1.8, 56.6ms, 0 | 20260506 21:44:10.293 | INFO | BAZWRITER : 73.5%, 4.8, 181.5ms, 0 | 20260506 21:44:10.293 | INFO | BAZ_SAMPLING : 3.1%, 1.0, 1.6ms, 0 | 20260506 21:44:10.293 | INFO | Uploaded 36GiB to GPU in 2.06485s (17.4346GiB/s) | 20260506 21:44:10.294 | INFO | Downloaded 38GiB to GPU in 2.19877s (17.2824GiB/s) | 20260506 21:44:15.435 | INFO | Analyzing chunk frames = [4678144,4678656) | 20260506 21:44:15.435 | INFO | MemInfo: { | 20260506 21:44:15.435 | INFO | "gpu" : " 17 GB", | 20260506 21:44:15.435 | INFO | "huge" : "", | 20260506 21:44:15.435 | INFO | "huge_shared" : " 27 GB", | 20260506 21:44:15.435 | INFO | "malloc" : " 14 GB", | 20260506 21:44:15.435 | INFO | "pinned" : " 29 GB" | 20260506 21:44:15.435 | INFO | } | 20260506 21:44:15.435 | INFO | Perf report for the last 5.14151 seconds: | 20260506 21:44:15.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:44:15.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:15.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:15.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:15.435 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:44:15.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:15.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:15.435 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:44:15.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:15.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:15.435 | INFO | Uploaded 36GiB to GPU in 2.01058s (17.9052GiB/s) | 20260506 21:44:15.435 | INFO | Downloaded 34GiB to GPU in 1.93319s (17.5875GiB/s) | 20260506 21:44:20.572 | INFO | Analyzing chunk frames = [4678656,4679168) | 20260506 21:44:20.572 | INFO | MemInfo: { | 20260506 21:44:20.572 | INFO | "gpu" : " 17 GB", | 20260506 21:44:20.572 | INFO | "huge" : "", | 20260506 21:44:20.572 | INFO | "huge_shared" : " 27 GB", | 20260506 21:44:20.572 | INFO | "malloc" : " 14 GB", | 20260506 21:44:20.572 | INFO | "pinned" : " 29 GB" | 20260506 21:44:20.572 | INFO | } | 20260506 21:44:20.572 | INFO | Perf report for the last 5.13765 seconds: | 20260506 21:44:20.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:44:20.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:20.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:20.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:20.572 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 21:44:20.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:20.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:20.572 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:44:20.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:20.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:20.573 | INFO | Uploaded 36GiB to GPU in 1.9966s (18.0307GiB/s) | 20260506 21:44:20.573 | INFO | Downloaded 34GiB to GPU in 1.92797s (17.6351GiB/s) | 20260506 21:44:25.709 | INFO | Analyzing chunk frames = [4679168,4679680) | 20260506 21:44:25.709 | INFO | MemInfo: { | 20260506 21:44:25.709 | INFO | "gpu" : " 17 GB", | 20260506 21:44:25.709 | INFO | "huge" : "", | 20260506 21:44:25.709 | INFO | "huge_shared" : " 27 GB", | 20260506 21:44:25.709 | INFO | "malloc" : " 14 GB", | 20260506 21:44:25.709 | INFO | "pinned" : " 29 GB" | 20260506 21:44:25.709 | INFO | } | 20260506 21:44:25.709 | INFO | Perf report for the last 5.13635 seconds: | 20260506 21:44:25.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:44:25.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:25.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:25.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:25.709 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 21:44:25.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:25.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:25.709 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:44:25.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:25.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:25.709 | INFO | Uploaded 36GiB to GPU in 2.02008s (17.8211GiB/s) | 20260506 21:44:25.709 | INFO | Downloaded 34GiB to GPU in 1.90708s (17.8283GiB/s) | 20260506 21:44:30.853 | INFO | Analyzing chunk frames = [4679680,4680192) | 20260506 21:44:30.853 | INFO | MemInfo: { | 20260506 21:44:30.853 | INFO | "gpu" : " 17 GB", | 20260506 21:44:30.853 | INFO | "huge" : "", | 20260506 21:44:30.853 | INFO | "huge_shared" : " 27 GB", | 20260506 21:44:30.853 | INFO | "malloc" : " 14 GB", | 20260506 21:44:30.853 | INFO | "pinned" : " 29 GB" | 20260506 21:44:30.853 | INFO | } | 20260506 21:44:30.853 | INFO | Perf report for the last 5.1441 seconds: | 20260506 21:44:30.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:44:30.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:30.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:30.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:30.853 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 21:44:30.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:30.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:30.853 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:44:30.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:30.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:30.853 | INFO | Uploaded 36GiB to GPU in 2.00461s (17.9586GiB/s) | 20260506 21:44:30.853 | INFO | Downloaded 34GiB to GPU in 1.92032s (17.7054GiB/s) | 20260506 21:44:35.988 | INFO | Analyzing chunk frames = [4680192,4680704) | 20260506 21:44:35.988 | INFO | MemInfo: { | 20260506 21:44:35.988 | INFO | "gpu" : " 17 GB", | 20260506 21:44:35.988 | INFO | "huge" : "", | 20260506 21:44:35.988 | INFO | "huge_shared" : " 27 GB", | 20260506 21:44:35.988 | INFO | "malloc" : " 14 GB", | 20260506 21:44:35.988 | INFO | "pinned" : " 29 GB" | 20260506 21:44:35.988 | INFO | } | 20260506 21:44:35.988 | INFO | Perf report for the last 5.13553 seconds: | 20260506 21:44:35.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:44:35.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:35.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:35.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:35.988 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 21:44:35.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:35.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:35.988 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 21:44:35.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:35.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:35.989 | INFO | Uploaded 36GiB to GPU in 2.0179s (17.8404GiB/s) | 20260506 21:44:35.989 | INFO | Downloaded 34GiB to GPU in 1.95173s (17.4204GiB/s) | 20260506 21:44:41.125 | INFO | Analyzing chunk frames = [4680704,4681216) | 20260506 21:44:41.125 | INFO | MemInfo: { | 20260506 21:44:41.125 | INFO | "gpu" : " 17 GB", | 20260506 21:44:41.125 | INFO | "huge" : "", | 20260506 21:44:41.125 | INFO | "huge_shared" : " 27 GB", | 20260506 21:44:41.125 | INFO | "malloc" : " 14 GB", | 20260506 21:44:41.125 | INFO | "pinned" : " 29 GB" | 20260506 21:44:41.125 | INFO | } | 20260506 21:44:41.126 | INFO | Perf report for the last 5.13711 seconds: | 20260506 21:44:41.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:44:41.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:41.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:41.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:41.126 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 21:44:41.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:41.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:41.126 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 21:44:41.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:41.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:41.126 | INFO | Uploaded 36GiB to GPU in 1.95596s (18.4052GiB/s) | 20260506 21:44:41.126 | INFO | Downloaded 34GiB to GPU in 1.99049s (17.0813GiB/s) | 20260506 21:44:46.263 | INFO | Analyzing chunk frames = [4681216,4681728) | 20260506 21:44:46.263 | INFO | MemInfo: { | 20260506 21:44:46.263 | INFO | "gpu" : " 17 GB", | 20260506 21:44:46.263 | INFO | "huge" : "", | 20260506 21:44:46.263 | INFO | "huge_shared" : " 27 GB", | 20260506 21:44:46.263 | INFO | "malloc" : " 14 GB", | 20260506 21:44:46.263 | INFO | "pinned" : " 29 GB" | 20260506 21:44:46.263 | INFO | } | 20260506 21:44:46.263 | INFO | Perf report for the last 5.13758 seconds: | 20260506 21:44:46.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:44:46.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:46.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:46.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:46.263 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 21:44:46.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:46.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:46.263 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 21:44:46.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:46.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:46.263 | INFO | Uploaded 36GiB to GPU in 2.04427s (17.6102GiB/s) | 20260506 21:44:46.263 | INFO | Downloaded 34GiB to GPU in 2.00134s (16.9886GiB/s) | 20260506 21:44:46.377 | INFO | Starting a round of Baz IO | 20260506 21:44:49.496 | INFO | Populating RTMetrics report | 20260506 21:44:49.496 | INFO | RTMetrics report populated | 20260506 21:44:49.496 | INFO | Opening tmp RTMetrics json file | 20260506 21:44:49.538 | INFO | Serializing RTMetrics to json | 20260506 21:44:49.642 | INFO | Serialization done, swapping tmp file now... | 20260506 21:44:49.706 | INFO | RTMetrics done | 20260506 21:44:49.823 | INFO | Finished a round of Baz IO: Wrote 1.938 GiB in 3.446 seconds | 20260506 21:44:49.823 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.85% / 0.00% / 66.14% | 20260506 21:44:49.823 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.58049% / 7.94729e-06% / 6.75519e-05% | 20260506 21:44:49.823 | INFO | Baz IO overall: | 20260506 21:44:49.823 | INFO | BytesWritten : 2464844365485 | 20260506 21:44:49.823 | INFO | NumEvents : 1189853870060 | 20260506 21:44:51.409 | INFO | Analyzing chunk frames = [4681728,4682240) | 20260506 21:44:51.409 | INFO | MemInfo: { | 20260506 21:44:51.409 | INFO | "gpu" : " 17 GB", | 20260506 21:44:51.409 | INFO | "huge" : "", | 20260506 21:44:51.409 | INFO | "huge_shared" : " 27 GB", | 20260506 21:44:51.409 | INFO | "malloc" : " 14 GB", | 20260506 21:44:51.409 | INFO | "pinned" : " 29 GB" | 20260506 21:44:51.409 | INFO | } | 20260506 21:44:51.409 | INFO | Perf report for the last 5.146 seconds: | 20260506 21:44:51.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:44:51.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:51.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:51.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:51.409 | INFO | ANALYSIS : 62.3%, 2.9, 95.4ms, 0 | 20260506 21:44:51.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:51.409 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260506 21:44:51.409 | INFO | PRE_HQ : 59.5%, 1.8, 56.8ms, 0 | 20260506 21:44:51.409 | INFO | BAZWRITER : 66.7%, 4.8, 163.2ms, 0 | 20260506 21:44:51.409 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 21:44:51.409 | INFO | Uploaded 36GiB to GPU in 2.03234s (17.7136GiB/s) | 20260506 21:44:51.409 | INFO | Downloaded 38GiB to GPU in 2.20161s (17.2601GiB/s) | 20260506 21:44:56.550 | INFO | Analyzing chunk frames = [4682240,4682752) | 20260506 21:44:56.550 | INFO | MemInfo: { | 20260506 21:44:56.550 | INFO | "gpu" : " 17 GB", | 20260506 21:44:56.550 | INFO | "huge" : "", | 20260506 21:44:56.550 | INFO | "huge_shared" : " 27 GB", | 20260506 21:44:56.550 | INFO | "malloc" : " 14 GB", | 20260506 21:44:56.550 | INFO | "pinned" : " 29 GB" | 20260506 21:44:56.550 | INFO | } | 20260506 21:44:56.550 | INFO | Perf report for the last 5.14067 seconds: | 20260506 21:44:56.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:44:56.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:56.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:56.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:56.550 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 21:44:56.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:56.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:56.550 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 21:44:56.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:56.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:44:56.550 | INFO | Uploaded 36GiB to GPU in 2.02341s (17.7918GiB/s) | 20260506 21:44:56.550 | INFO | Downloaded 34GiB to GPU in 1.92751s (17.6393GiB/s) | 20260506 21:45:01.688 | INFO | Analyzing chunk frames = [4682752,4683264) | 20260506 21:45:01.688 | INFO | MemInfo: { | 20260506 21:45:01.688 | INFO | "gpu" : " 17 GB", | 20260506 21:45:01.688 | INFO | "huge" : "", | 20260506 21:45:01.688 | INFO | "huge_shared" : " 27 GB", | 20260506 21:45:01.688 | INFO | "malloc" : " 14 GB", | 20260506 21:45:01.688 | INFO | "pinned" : " 29 GB" | 20260506 21:45:01.688 | INFO | } | 20260506 21:45:01.688 | INFO | Perf report for the last 5.13837 seconds: | 20260506 21:45:01.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:45:01.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:01.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:01.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:01.688 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 21:45:01.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:01.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:01.688 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:45:01.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:01.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:01.688 | INFO | Uploaded 36GiB to GPU in 1.97377s (18.2392GiB/s) | 20260506 21:45:01.689 | INFO | Downloaded 34GiB to GPU in 1.92171s (17.6926GiB/s) | 20260506 21:45:06.828 | INFO | Analyzing chunk frames = [4683264,4683776) | 20260506 21:45:06.828 | INFO | MemInfo: { | 20260506 21:45:06.828 | INFO | "gpu" : " 17 GB", | 20260506 21:45:06.828 | INFO | "huge" : "", | 20260506 21:45:06.828 | INFO | "huge_shared" : " 27 GB", | 20260506 21:45:06.828 | INFO | "malloc" : " 14 GB", | 20260506 21:45:06.828 | INFO | "pinned" : " 29 GB" | 20260506 21:45:06.828 | INFO | } | 20260506 21:45:06.828 | INFO | Perf report for the last 5.13967 seconds: | 20260506 21:45:06.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:45:06.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:06.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:06.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:06.828 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260506 21:45:06.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:06.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:06.828 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 21:45:06.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:06.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:06.828 | INFO | Uploaded 36GiB to GPU in 2.00478s (17.9571GiB/s) | 20260506 21:45:06.828 | INFO | Downloaded 34GiB to GPU in 1.90675s (17.8314GiB/s) | 20260506 21:45:11.965 | INFO | Analyzing chunk frames = [4683776,4684288) | 20260506 21:45:11.965 | INFO | MemInfo: { | 20260506 21:45:11.965 | INFO | "gpu" : " 17 GB", | 20260506 21:45:11.965 | INFO | "huge" : "", | 20260506 21:45:11.965 | INFO | "huge_shared" : " 27 GB", | 20260506 21:45:11.965 | INFO | "malloc" : " 14 GB", | 20260506 21:45:11.965 | INFO | "pinned" : " 29 GB" | 20260506 21:45:11.965 | INFO | } | 20260506 21:45:11.965 | INFO | Perf report for the last 5.13719 seconds: | 20260506 21:45:11.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:45:11.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:11.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:11.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:11.965 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 21:45:11.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:11.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:11.965 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:45:11.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:11.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:11.965 | INFO | Uploaded 36GiB to GPU in 2.00215s (17.9807GiB/s) | 20260506 21:45:11.965 | INFO | Downloaded 34GiB to GPU in 1.92473s (17.6648GiB/s) | 20260506 21:45:17.106 | INFO | Analyzing chunk frames = [4684288,4684800) | 20260506 21:45:17.106 | INFO | MemInfo: { | 20260506 21:45:17.106 | INFO | "gpu" : " 17 GB", | 20260506 21:45:17.106 | INFO | "huge" : "", | 20260506 21:45:17.106 | INFO | "huge_shared" : " 27 GB", | 20260506 21:45:17.106 | INFO | "malloc" : " 14 GB", | 20260506 21:45:17.106 | INFO | "pinned" : " 29 GB" | 20260506 21:45:17.106 | INFO | } | 20260506 21:45:17.106 | INFO | Perf report for the last 5.14092 seconds: | 20260506 21:45:17.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:45:17.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:17.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:17.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:17.106 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 21:45:17.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:17.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:17.106 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:45:17.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:17.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:17.106 | INFO | Uploaded 36GiB to GPU in 2.00986s (17.9117GiB/s) | 20260506 21:45:17.106 | INFO | Downloaded 34GiB to GPU in 1.94924s (17.4427GiB/s) | 20260506 21:45:22.240 | INFO | Analyzing chunk frames = [4684800,4685312) | 20260506 21:45:22.240 | INFO | MemInfo: { | 20260506 21:45:22.240 | INFO | "gpu" : " 17 GB", | 20260506 21:45:22.240 | INFO | "huge" : "", | 20260506 21:45:22.240 | INFO | "huge_shared" : " 27 GB", | 20260506 21:45:22.240 | INFO | "malloc" : " 14 GB", | 20260506 21:45:22.240 | INFO | "pinned" : " 29 GB" | 20260506 21:45:22.240 | INFO | } | 20260506 21:45:22.240 | INFO | Perf report for the last 5.13422 seconds: | 20260506 21:45:22.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:45:22.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:22.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:22.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:22.240 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 21:45:22.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:22.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:22.240 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 21:45:22.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:22.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:22.240 | INFO | Uploaded 36GiB to GPU in 1.96086s (18.3593GiB/s) | 20260506 21:45:22.240 | INFO | Downloaded 34GiB to GPU in 1.99272s (17.0621GiB/s) | 20260506 21:45:27.383 | INFO | Analyzing chunk frames = [4685312,4685824) | 20260506 21:45:27.383 | INFO | MemInfo: { | 20260506 21:45:27.383 | INFO | "gpu" : " 17 GB", | 20260506 21:45:27.383 | INFO | "huge" : "", | 20260506 21:45:27.383 | INFO | "huge_shared" : " 27 GB", | 20260506 21:45:27.383 | INFO | "malloc" : " 14 GB", | 20260506 21:45:27.383 | INFO | "pinned" : " 29 GB" | 20260506 21:45:27.383 | INFO | } | 20260506 21:45:27.383 | INFO | Perf report for the last 5.14288 seconds: | 20260506 21:45:27.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:45:27.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:27.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:27.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:27.383 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 21:45:27.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:27.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:27.383 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:45:27.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:27.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:27.383 | INFO | Uploaded 36GiB to GPU in 1.99596s (18.0364GiB/s) | 20260506 21:45:27.383 | INFO | Downloaded 34GiB to GPU in 1.99642s (17.0305GiB/s) | 20260506 21:45:27.383 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:45:27.383 | INFO | Total mem usage: 18 GiB | 20260506 21:45:27.383 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:45:27.383 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -15.9%) | 20260506 21:45:27.383 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:45:27.383 | INFO | High Water Breakdown: | 20260506 21:45:27.383 | INFO | 61.4% : 9.53 GiB ( 83.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:45:27.384 | INFO | 38.5% : 5.98 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:45:27.384 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:45:27.384 | INFO | ------------------------------------------ | 20260506 21:45:27.384 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:45:27.384 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:45:27.384 | INFO | Total mem usage: 26.5 GiB | 20260506 21:45:27.384 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:45:27.384 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:45:27.384 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:45:27.384 | INFO | High Water Breakdown: | 20260506 21:45:27.384 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:45:27.384 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:45:27.384 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:45:27.384 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:45:27.384 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:45:27.384 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:45:27.384 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:45:27.384 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:45:27.384 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:45:27.384 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:45:27.384 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:45:27.384 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:45:27.384 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:45:27.384 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:45:27.384 | INFO | ------------------------------------------ | 20260506 21:45:27.384 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:45:27.384 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:45:27.384 | INFO | Total mem usage: 25.6 GiB | 20260506 21:45:27.384 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:45:27.384 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:45:27.384 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:45:27.384 | INFO | High Water Breakdown: | 20260506 21:45:27.384 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:45:27.384 | INFO | ------------------------------------------ | 20260506 21:45:27.384 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:45:27.384 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:45:27.384 | INFO | Total mem usage: 15.7 GiB | 20260506 21:45:27.384 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:45:27.384 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:45:27.384 | INFO | Unused Cache: 866 MiB (Waste: 5.4%) | 20260506 21:45:27.384 | INFO | High Water Breakdown: | 20260506 21:45:27.384 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:45:27.384 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:45:27.384 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:45:27.384 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:45:27.384 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:45:27.384 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:45:27.384 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:45:27.384 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:45:27.384 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:45:27.384 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:45:27.384 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:45:27.384 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:45:27.384 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:45:27.384 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:45:27.384 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:45:27.384 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:45:27.384 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:45:27.384 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:45:27.384 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:45:27.384 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:45:27.384 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:45:27.384 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:45:27.384 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:45:27.384 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:45:27.384 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:45:27.384 | INFO | ------------------------------------------ | 20260506 21:45:27.384 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:45:27.384 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:45:27.384 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:45:27.384 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:45:27.384 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.57e+01, 1.35e+01 / 2.38e+00 | 20260506 21:45:27.384 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 9.81e-03 / 5.57e-03 | 20260506 21:45:27.384 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.33e+00, 5.52e-01 / 3.49e-01 | 20260506 21:45:27.384 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.42e+00, 1.80e-01 / 1.31e-01 | 20260506 21:45:27.384 | INFO | PerfReport: 15.99%, DME -- 3.98e+01 / 4.41e+01, 4.07e+00 / 2.74e+00 | 20260506 21:45:27.384 | INFO | PerfReport: 57.33%, FrameLabeling -- 1.78e+01 / 2.13e+01, 1.41e+00 / 2.02e+00 | 20260506 21:45:27.384 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.21e+00 / 2.75e+00, 3.78e-01 / 2.53e-01 | 20260506 21:45:27.384 | INFO | PerfReport: 3.25%, Metrics -- 1.01e+00 / 1.32e+00, 2.67e-01 / 1.65e-01 | 20260506 21:45:27.384 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.46e-01 / 3.92e-01, 5.45e-02 / 2.51e-02 | 20260506 21:45:27.384 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:45:27.384 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:45:27.384 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:45:27.384 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 21:45:27.384 | INFO | PerfReport: 21.62%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 21:45:27.384 | INFO | PerfReport: 21.31%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 21:45:27.384 | INFO | PerfReport: 32.16%, Compute -- 3.11e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 21:45:27.384 | INFO | PerfReport: 1.33%, LockUpload -- 1.29e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260506 21:45:27.384 | INFO | PerfReport: 1.14%, LockDownload -- 1.11e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 21:45:27.384 | INFO | PerfReport: 22.42%, LockCompute -- 2.17e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 21:45:27.384 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260506 21:45:27.483 | INFO | Starting a round of Baz IO | 20260506 21:45:30.624 | INFO | Populating RTMetrics report | 20260506 21:45:30.624 | INFO | RTMetrics report populated | 20260506 21:45:30.624 | INFO | Opening tmp RTMetrics json file | 20260506 21:45:30.682 | INFO | Serializing RTMetrics to json | 20260506 21:45:30.738 | INFO | Serialization done, swapping tmp file now... | 20260506 21:45:30.741 | INFO | RTMetrics done | 20260506 21:45:30.781 | INFO | Finished a round of Baz IO: Wrote 1.936 GiB in 3.297 seconds | 20260506 21:45:30.781 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.85% / 0.00% / 66.15% | 20260506 21:45:30.781 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.579723% / 1.19209e-05% / 4.76837e-05% | 20260506 21:45:30.781 | INFO | Baz IO overall: | 20260506 21:45:30.781 | INFO | BytesWritten : 2466923188173 | 20260506 21:45:30.781 | INFO | NumEvents : 1190848320595 | 20260506 21:45:32.531 | INFO | Analyzing chunk frames = [4685824,4686336) | 20260506 21:45:32.531 | INFO | MemInfo: { | 20260506 21:45:32.531 | INFO | "gpu" : " 17 GB", | 20260506 21:45:32.531 | INFO | "huge" : "", | 20260506 21:45:32.531 | INFO | "huge_shared" : " 27 GB", | 20260506 21:45:32.531 | INFO | "malloc" : " 14 GB", | 20260506 21:45:32.531 | INFO | "pinned" : " 29 GB" | 20260506 21:45:32.531 | INFO | } | 20260506 21:45:32.531 | INFO | Perf report for the last 5.14758 seconds: | 20260506 21:45:32.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:45:32.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:32.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:32.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:32.531 | INFO | ANALYSIS : 62.4%, 2.9, 95.3ms, 0 | 20260506 21:45:32.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:32.531 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260506 21:45:32.531 | INFO | PRE_HQ : 59.8%, 1.8, 57.3ms, 0 | 20260506 21:45:32.531 | INFO | BAZWRITER : 63.7%, 4.6, 149.6ms, 0 | 20260506 21:45:32.531 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 21:45:32.531 | INFO | Uploaded 36GiB to GPU in 2.07263s (17.3693GiB/s) | 20260506 21:45:32.531 | INFO | Downloaded 38GiB to GPU in 2.19703s (17.2961GiB/s) | 20260506 21:45:37.665 | INFO | Analyzing chunk frames = [4686336,4686848) | 20260506 21:45:37.665 | INFO | MemInfo: { | 20260506 21:45:37.665 | INFO | "gpu" : " 17 GB", | 20260506 21:45:37.665 | INFO | "huge" : "", | 20260506 21:45:37.665 | INFO | "huge_shared" : " 27 GB", | 20260506 21:45:37.665 | INFO | "malloc" : " 14 GB", | 20260506 21:45:37.665 | INFO | "pinned" : " 29 GB" | 20260506 21:45:37.665 | INFO | } | 20260506 21:45:37.665 | INFO | Perf report for the last 5.13473 seconds: | 20260506 21:45:37.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:45:37.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:37.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:37.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:37.666 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 21:45:37.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:37.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:37.666 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 21:45:37.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:37.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:37.666 | INFO | Uploaded 36GiB to GPU in 2.0022s (17.9802GiB/s) | 20260506 21:45:37.666 | INFO | Downloaded 34GiB to GPU in 1.9218s (17.6918GiB/s) | 20260506 21:45:42.802 | INFO | Analyzing chunk frames = [4686848,4687360) | 20260506 21:45:42.802 | INFO | MemInfo: { | 20260506 21:45:42.802 | INFO | "gpu" : " 17 GB", | 20260506 21:45:42.802 | INFO | "huge" : "", | 20260506 21:45:42.803 | INFO | "huge_shared" : " 27 GB", | 20260506 21:45:42.803 | INFO | "malloc" : " 14 GB", | 20260506 21:45:42.803 | INFO | "pinned" : " 29 GB" | 20260506 21:45:42.803 | INFO | } | 20260506 21:45:42.803 | INFO | Perf report for the last 5.13713 seconds: | 20260506 21:45:42.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:45:42.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:42.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:42.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:42.803 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 21:45:42.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:42.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:42.803 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:45:42.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:42.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:42.803 | INFO | Uploaded 36GiB to GPU in 2.01194s (17.8932GiB/s) | 20260506 21:45:42.803 | INFO | Downloaded 34GiB to GPU in 1.9336s (17.5838GiB/s) | 20260506 21:45:47.939 | INFO | Analyzing chunk frames = [4687360,4687872) | 20260506 21:45:47.939 | INFO | MemInfo: { | 20260506 21:45:47.939 | INFO | "gpu" : " 17 GB", | 20260506 21:45:47.939 | INFO | "huge" : "", | 20260506 21:45:47.939 | INFO | "huge_shared" : " 27 GB", | 20260506 21:45:47.939 | INFO | "malloc" : " 14 GB", | 20260506 21:45:47.939 | INFO | "pinned" : " 29 GB" | 20260506 21:45:47.939 | INFO | } | 20260506 21:45:47.939 | INFO | Perf report for the last 5.13643 seconds: | 20260506 21:45:47.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:45:47.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:47.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:47.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:47.939 | INFO | ANALYSIS : 59.8%, 3.0, 92.5ms, 0 | 20260506 21:45:47.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:47.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:47.939 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:45:47.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:47.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:47.939 | INFO | Uploaded 36GiB to GPU in 1.96985s (18.2755GiB/s) | 20260506 21:45:47.939 | INFO | Downloaded 34GiB to GPU in 1.87895s (18.0952GiB/s) | 20260506 21:45:53.075 | INFO | Analyzing chunk frames = [4687872,4688384) | 20260506 21:45:53.075 | INFO | MemInfo: { | 20260506 21:45:53.075 | INFO | "gpu" : " 17 GB", | 20260506 21:45:53.075 | INFO | "huge" : "", | 20260506 21:45:53.075 | INFO | "huge_shared" : " 27 GB", | 20260506 21:45:53.075 | INFO | "malloc" : " 14 GB", | 20260506 21:45:53.075 | INFO | "pinned" : " 29 GB" | 20260506 21:45:53.075 | INFO | } | 20260506 21:45:53.075 | INFO | Perf report for the last 5.13617 seconds: | 20260506 21:45:53.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:45:53.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:53.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:53.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:53.075 | INFO | ANALYSIS : 60.5%, 3.0, 94.0ms, 0 | 20260506 21:45:53.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:53.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:53.075 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:45:53.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:53.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:53.075 | INFO | Uploaded 36GiB to GPU in 2.011s (17.9015GiB/s) | 20260506 21:45:53.076 | INFO | Downloaded 34GiB to GPU in 1.91424s (17.7616GiB/s) | 20260506 21:45:58.213 | INFO | Analyzing chunk frames = [4688384,4688896) | 20260506 21:45:58.213 | INFO | MemInfo: { | 20260506 21:45:58.213 | INFO | "gpu" : " 17 GB", | 20260506 21:45:58.214 | INFO | "huge" : "", | 20260506 21:45:58.214 | INFO | "huge_shared" : " 27 GB", | 20260506 21:45:58.214 | INFO | "malloc" : " 14 GB", | 20260506 21:45:58.214 | INFO | "pinned" : " 29 GB" | 20260506 21:45:58.214 | INFO | } | 20260506 21:45:58.214 | INFO | Perf report for the last 5.13834 seconds: | 20260506 21:45:58.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:45:58.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:58.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:58.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:58.214 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 21:45:58.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:58.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:58.214 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 21:45:58.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:58.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:45:58.214 | INFO | Uploaded 36GiB to GPU in 2.02553s (17.7731GiB/s) | 20260506 21:45:58.214 | INFO | Downloaded 34GiB to GPU in 1.96611s (17.293GiB/s) | 20260506 21:46:03.356 | INFO | Analyzing chunk frames = [4688896,4689408) | 20260506 21:46:03.356 | INFO | MemInfo: { | 20260506 21:46:03.356 | INFO | "gpu" : " 17 GB", | 20260506 21:46:03.356 | INFO | "huge" : "", | 20260506 21:46:03.356 | INFO | "huge_shared" : " 27 GB", | 20260506 21:46:03.356 | INFO | "malloc" : " 14 GB", | 20260506 21:46:03.356 | INFO | "pinned" : " 29 GB" | 20260506 21:46:03.356 | INFO | } | 20260506 21:46:03.356 | INFO | Perf report for the last 5.14225 seconds: | 20260506 21:46:03.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:46:03.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:03.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:03.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:03.356 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 21:46:03.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:03.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:03.356 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:46:03.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:03.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:03.356 | INFO | Uploaded 36GiB to GPU in 1.95796s (18.3865GiB/s) | 20260506 21:46:03.356 | INFO | Downloaded 34GiB to GPU in 1.99385s (17.0525GiB/s) | 20260506 21:46:08.500 | INFO | Analyzing chunk frames = [4689408,4689920) | 20260506 21:46:08.500 | INFO | MemInfo: { | 20260506 21:46:08.500 | INFO | "gpu" : " 17 GB", | 20260506 21:46:08.500 | INFO | "huge" : "", | 20260506 21:46:08.500 | INFO | "huge_shared" : " 27 GB", | 20260506 21:46:08.500 | INFO | "malloc" : " 14 GB", | 20260506 21:46:08.500 | INFO | "pinned" : " 29 GB" | 20260506 21:46:08.500 | INFO | } | 20260506 21:46:08.500 | INFO | Perf report for the last 5.14379 seconds: | 20260506 21:46:08.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:46:08.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:08.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:08.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:08.500 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 21:46:08.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:08.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:08.500 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 21:46:08.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:08.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:08.500 | INFO | Uploaded 36GiB to GPU in 1.9876s (18.1123GiB/s) | 20260506 21:46:08.500 | INFO | Downloaded 34GiB to GPU in 1.98041s (17.1682GiB/s) | 20260506 21:46:08.597 | INFO | Starting a round of Baz IO | 20260506 21:46:11.736 | INFO | Populating RTMetrics report | 20260506 21:46:11.736 | INFO | RTMetrics report populated | 20260506 21:46:11.736 | INFO | Opening tmp RTMetrics json file | 20260506 21:46:11.766 | INFO | Serializing RTMetrics to json | 20260506 21:46:11.814 | INFO | Serialization done, swapping tmp file now... | 20260506 21:46:11.840 | INFO | RTMetrics done | 20260506 21:46:12.204 | INFO | Finished a round of Baz IO: Wrote 1.934 GiB in 3.606 seconds | 20260506 21:46:12.204 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.85% / 0.00% / 66.15% | 20260506 21:46:12.204 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.574982% / 3.97364e-06% / 7.94729e-05% | 20260506 21:46:12.204 | INFO | Baz IO overall: | 20260506 21:46:12.204 | INFO | BytesWritten : 2469000015814 | 20260506 21:46:12.204 | INFO | NumEvents : 1191841456633 | 20260506 21:46:13.642 | INFO | Analyzing chunk frames = [4689920,4690432) | 20260506 21:46:13.642 | INFO | MemInfo: { | 20260506 21:46:13.642 | INFO | "gpu" : " 17 GB", | 20260506 21:46:13.642 | INFO | "huge" : "", | 20260506 21:46:13.642 | INFO | "huge_shared" : " 27 GB", | 20260506 21:46:13.642 | INFO | "malloc" : " 14 GB", | 20260506 21:46:13.642 | INFO | "pinned" : " 29 GB" | 20260506 21:46:13.642 | INFO | } | 20260506 21:46:13.643 | INFO | Perf report for the last 5.14284 seconds: | 20260506 21:46:13.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:46:13.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:13.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:13.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:13.643 | INFO | ANALYSIS : 62.4%, 2.9, 95.3ms, 0 | 20260506 21:46:13.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:13.643 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 21:46:13.643 | INFO | PRE_HQ : 59.8%, 1.8, 56.7ms, 0 | 20260506 21:46:13.643 | INFO | BAZWRITER : 69.8%, 4.8, 173.2ms, 0 | 20260506 21:46:13.643 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 21:46:13.643 | INFO | Uploaded 36GiB to GPU in 2.05942s (17.4807GiB/s) | 20260506 21:46:13.643 | INFO | Downloaded 38GiB to GPU in 2.19207s (17.3352GiB/s) | 20260506 21:46:18.783 | INFO | Analyzing chunk frames = [4690432,4690944) | 20260506 21:46:18.783 | INFO | MemInfo: { | 20260506 21:46:18.783 | INFO | "gpu" : " 17 GB", | 20260506 21:46:18.783 | INFO | "huge" : "", | 20260506 21:46:18.783 | INFO | "huge_shared" : " 27 GB", | 20260506 21:46:18.783 | INFO | "malloc" : " 14 GB", | 20260506 21:46:18.783 | INFO | "pinned" : " 29 GB" | 20260506 21:46:18.783 | INFO | } | 20260506 21:46:18.783 | INFO | Perf report for the last 5.14038 seconds: | 20260506 21:46:18.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:46:18.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:18.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:18.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:18.783 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 21:46:18.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:18.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:18.783 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:46:18.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:18.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:18.783 | INFO | Uploaded 36GiB to GPU in 1.99605s (18.0356GiB/s) | 20260506 21:46:18.783 | INFO | Downloaded 34GiB to GPU in 1.92011s (17.7074GiB/s) | 20260506 21:46:23.920 | INFO | Analyzing chunk frames = [4690944,4691456) | 20260506 21:46:23.920 | INFO | MemInfo: { | 20260506 21:46:23.920 | INFO | "gpu" : " 17 GB", | 20260506 21:46:23.920 | INFO | "huge" : "", | 20260506 21:46:23.920 | INFO | "huge_shared" : " 27 GB", | 20260506 21:46:23.920 | INFO | "malloc" : " 14 GB", | 20260506 21:46:23.920 | INFO | "pinned" : " 29 GB" | 20260506 21:46:23.920 | INFO | } | 20260506 21:46:23.920 | INFO | Perf report for the last 5.13708 seconds: | 20260506 21:46:23.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:46:23.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:23.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:23.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:23.920 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 21:46:23.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:23.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:23.920 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 21:46:23.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:23.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:23.920 | INFO | Uploaded 36GiB to GPU in 1.98838s (18.1052GiB/s) | 20260506 21:46:23.920 | INFO | Downloaded 34GiB to GPU in 1.92583s (17.6547GiB/s) | 20260506 21:46:29.054 | INFO | Analyzing chunk frames = [4691456,4691968) | 20260506 21:46:29.054 | INFO | MemInfo: { | 20260506 21:46:29.054 | INFO | "gpu" : " 17 GB", | 20260506 21:46:29.054 | INFO | "huge" : "", | 20260506 21:46:29.054 | INFO | "huge_shared" : " 27 GB", | 20260506 21:46:29.054 | INFO | "malloc" : " 14 GB", | 20260506 21:46:29.054 | INFO | "pinned" : " 29 GB" | 20260506 21:46:29.054 | INFO | } | 20260506 21:46:29.054 | INFO | Perf report for the last 5.13367 seconds: | 20260506 21:46:29.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:46:29.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:29.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:29.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:29.054 | INFO | ANALYSIS : 59.7%, 3.0, 92.4ms, 0 | 20260506 21:46:29.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:29.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:29.054 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:46:29.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:29.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:29.054 | INFO | Uploaded 36GiB to GPU in 1.96002s (18.3671GiB/s) | 20260506 21:46:29.054 | INFO | Downloaded 34GiB to GPU in 1.87371s (18.1459GiB/s) | 20260506 21:46:34.191 | INFO | Analyzing chunk frames = [4691968,4692480) | 20260506 21:46:34.191 | INFO | MemInfo: { | 20260506 21:46:34.191 | INFO | "gpu" : " 17 GB", | 20260506 21:46:34.192 | INFO | "huge" : "", | 20260506 21:46:34.192 | INFO | "huge_shared" : " 27 GB", | 20260506 21:46:34.192 | INFO | "malloc" : " 14 GB", | 20260506 21:46:34.192 | INFO | "pinned" : " 29 GB" | 20260506 21:46:34.192 | INFO | } | 20260506 21:46:34.192 | INFO | Perf report for the last 5.13789 seconds: | 20260506 21:46:34.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:46:34.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:34.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:34.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:34.192 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 21:46:34.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:34.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:34.192 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:46:34.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:34.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:34.192 | INFO | Uploaded 36GiB to GPU in 2.00873s (17.9218GiB/s) | 20260506 21:46:34.192 | INFO | Downloaded 34GiB to GPU in 1.91964s (17.7116GiB/s) | 20260506 21:46:39.333 | INFO | Analyzing chunk frames = [4692480,4692992) | 20260506 21:46:39.333 | INFO | MemInfo: { | 20260506 21:46:39.333 | INFO | "gpu" : " 17 GB", | 20260506 21:46:39.333 | INFO | "huge" : "", | 20260506 21:46:39.334 | INFO | "huge_shared" : " 27 GB", | 20260506 21:46:39.334 | INFO | "malloc" : " 14 GB", | 20260506 21:46:39.334 | INFO | "pinned" : " 29 GB" | 20260506 21:46:39.334 | INFO | } | 20260506 21:46:39.334 | INFO | Perf report for the last 5.14199 seconds: | 20260506 21:46:39.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:46:39.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:39.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:39.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:39.334 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:46:39.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:39.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:39.334 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 21:46:39.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:39.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:39.334 | INFO | Uploaded 36GiB to GPU in 2.02644s (17.7652GiB/s) | 20260506 21:46:39.334 | INFO | Downloaded 34GiB to GPU in 1.93022s (17.6145GiB/s) | 20260506 21:46:44.478 | INFO | Analyzing chunk frames = [4692992,4693504) | 20260506 21:46:44.478 | INFO | MemInfo: { | 20260506 21:46:44.478 | INFO | "gpu" : " 17 GB", | 20260506 21:46:44.478 | INFO | "huge" : "", | 20260506 21:46:44.478 | INFO | "huge_shared" : " 27 GB", | 20260506 21:46:44.478 | INFO | "malloc" : " 14 GB", | 20260506 21:46:44.478 | INFO | "pinned" : " 29 GB" | 20260506 21:46:44.478 | INFO | } | 20260506 21:46:44.478 | INFO | Perf report for the last 5.14451 seconds: | 20260506 21:46:44.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:46:44.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:44.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:44.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:44.478 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 21:46:44.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:44.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:44.478 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 21:46:44.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:44.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:44.478 | INFO | Uploaded 36GiB to GPU in 1.95994s (18.3679GiB/s) | 20260506 21:46:44.478 | INFO | Downloaded 34GiB to GPU in 1.98654s (17.1152GiB/s) | 20260506 21:46:49.617 | INFO | Analyzing chunk frames = [4693504,4694016) | 20260506 21:46:49.617 | INFO | MemInfo: { | 20260506 21:46:49.617 | INFO | "gpu" : " 17 GB", | 20260506 21:46:49.617 | INFO | "huge" : "", | 20260506 21:46:49.617 | INFO | "huge_shared" : " 27 GB", | 20260506 21:46:49.617 | INFO | "malloc" : " 14 GB", | 20260506 21:46:49.617 | INFO | "pinned" : " 29 GB" | 20260506 21:46:49.617 | INFO | } | 20260506 21:46:49.617 | INFO | Perf report for the last 5.13912 seconds: | 20260506 21:46:49.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:46:49.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:49.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:49.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:49.617 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:46:49.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:49.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:49.617 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 21:46:49.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:49.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:49.618 | INFO | Uploaded 36GiB to GPU in 2.0483s (17.5756GiB/s) | 20260506 21:46:49.618 | INFO | Downloaded 34GiB to GPU in 1.99544s (17.0389GiB/s) | 20260506 21:46:49.618 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:46:49.618 | INFO | Total mem usage: 18 GiB | 20260506 21:46:49.618 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:46:49.618 | INFO | Recent High Water Marks Sum: 16.7 GiB (Savings: -8.11%) | 20260506 21:46:49.618 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:46:49.618 | INFO | High Water Breakdown: | 20260506 21:46:49.618 | INFO | 62.5% : 10.4 GiB ( 91.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:46:49.618 | INFO | 37.4% : 6.23 GiB ( 93.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:46:49.618 | INFO | 0.119% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:46:49.618 | INFO | ------------------------------------------ | 20260506 21:46:49.618 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:46:49.618 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:46:49.618 | INFO | Total mem usage: 26.5 GiB | 20260506 21:46:49.618 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:46:49.618 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:46:49.618 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:46:49.618 | INFO | High Water Breakdown: | 20260506 21:46:49.618 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:46:49.618 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:46:49.618 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:46:49.618 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:46:49.618 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:46:49.618 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:46:49.618 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:46:49.618 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:46:49.618 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:46:49.618 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:46:49.618 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:46:49.618 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:46:49.618 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:46:49.618 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:46:49.618 | INFO | ------------------------------------------ | 20260506 21:46:49.618 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:46:49.618 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:46:49.618 | INFO | Total mem usage: 25.6 GiB | 20260506 21:46:49.618 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:46:49.618 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 21:46:49.618 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 21:46:49.618 | INFO | High Water Breakdown: | 20260506 21:46:49.618 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 21:46:49.618 | INFO | ------------------------------------------ | 20260506 21:46:49.618 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:46:49.618 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:46:49.618 | INFO | Total mem usage: 15.7 GiB | 20260506 21:46:49.618 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:46:49.618 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:46:49.618 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:46:49.618 | INFO | High Water Breakdown: | 20260506 21:46:49.618 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:46:49.618 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:46:49.618 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:46:49.618 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:46:49.618 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:46:49.618 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:46:49.618 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:46:49.618 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:46:49.618 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:46:49.618 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:46:49.618 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:46:49.618 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:46:49.618 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:46:49.618 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:46:49.618 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:46:49.618 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:46:49.618 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:46:49.618 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:46:49.618 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:46:49.618 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:46:49.618 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:46:49.618 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:46:49.618 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:46:49.618 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:46:49.618 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:46:49.618 | INFO | ------------------------------------------ | 20260506 21:46:49.618 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:46:49.618 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:46:49.618 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:46:49.618 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:46:49.618 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.59e+01, 1.33e+01 / 2.39e+00 | 20260506 21:46:49.618 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.03e-01, 1.02e-02 / 6.31e-03 | 20260506 21:46:49.618 | INFO | PerfReport: 11.65%, Baseline -- 3.62e+00 / 4.43e+00, 5.47e-01 / 2.67e-01 | 20260506 21:46:49.618 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.47e+00, 1.77e-01 / 1.28e-01 | 20260506 21:46:49.618 | INFO | PerfReport: 16.02%, DME -- 3.98e+01 / 4.48e+01, 4.12e+00 / 2.92e+00 | 20260506 21:46:49.618 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.39e+00 / 1.81e+00 | 20260506 21:46:49.618 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.21e+00 / 2.76e+00, 3.82e-01 / 2.19e-01 | 20260506 21:46:49.618 | INFO | PerfReport: 3.25%, Metrics -- 1.01e+00 / 1.35e+00, 2.66e-01 / 1.44e-01 | 20260506 21:46:49.618 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.46e-01 / 3.92e-01, 5.38e-02 / 1.80e-02 | 20260506 21:46:49.618 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:46:49.618 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:46:49.618 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:46:49.618 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 21:46:49.618 | INFO | PerfReport: 21.63%, Upload -- 2.09e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 21:46:49.619 | INFO | PerfReport: 21.26%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 21:46:49.619 | INFO | PerfReport: 32.19%, Compute -- 3.11e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 21:46:49.619 | INFO | PerfReport: 1.33%, LockUpload -- 1.28e+00 / 0.00e+00, 4.72e+00 / 0.00e+00 | 20260506 21:46:49.619 | INFO | PerfReport: 1.13%, LockDownload -- 1.09e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260506 21:46:49.619 | INFO | PerfReport: 22.45%, LockCompute -- 2.17e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 21:46:49.619 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.30e-02 / 0.00e+00 | 20260506 21:46:49.728 | INFO | Starting a round of Baz IO | 20260506 21:46:52.855 | INFO | Populating RTMetrics report | 20260506 21:46:52.856 | INFO | RTMetrics report populated | 20260506 21:46:52.856 | INFO | Opening tmp RTMetrics json file | 20260506 21:46:52.857 | INFO | Serializing RTMetrics to json | 20260506 21:46:52.870 | INFO | Serialization done, swapping tmp file now... | 20260506 21:46:52.882 | INFO | RTMetrics done | 20260506 21:46:52.987 | INFO | Finished a round of Baz IO: Wrote 1.934 GiB in 3.259 seconds | 20260506 21:46:52.987 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.85% / 0.00% / 66.15% | 20260506 21:46:52.987 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.575503% / 0% / 8.34465e-05% | 20260506 21:46:52.987 | INFO | Baz IO overall: | 20260506 21:46:52.987 | INFO | BytesWritten : 2471076854826 | 20260506 21:46:52.987 | INFO | NumEvents : 1192834808478 | 20260506 21:46:54.752 | INFO | Analyzing chunk frames = [4694016,4694528) | 20260506 21:46:54.752 | INFO | MemInfo: { | 20260506 21:46:54.752 | INFO | "gpu" : " 17 GB", | 20260506 21:46:54.752 | INFO | "huge" : "", | 20260506 21:46:54.752 | INFO | "huge_shared" : " 27 GB", | 20260506 21:46:54.752 | INFO | "malloc" : " 14 GB", | 20260506 21:46:54.752 | INFO | "pinned" : " 29 GB" | 20260506 21:46:54.752 | INFO | } | 20260506 21:46:54.752 | INFO | Perf report for the last 5.13514 seconds: | 20260506 21:46:54.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:46:54.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:54.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:54.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:54.752 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 21:46:54.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:54.752 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 21:46:54.752 | INFO | PRE_HQ : 60.2%, 1.8, 56.9ms, 0 | 20260506 21:46:54.752 | INFO | BAZWRITER : 63.1%, 4.6, 150.7ms, 0 | 20260506 21:46:54.752 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 21:46:54.753 | INFO | Uploaded 36GiB to GPU in 2.06182s (17.4603GiB/s) | 20260506 21:46:54.753 | INFO | Downloaded 38GiB to GPU in 2.20749s (17.2141GiB/s) | 20260506 21:46:59.890 | INFO | Analyzing chunk frames = [4694528,4695040) | 20260506 21:46:59.890 | INFO | MemInfo: { | 20260506 21:46:59.890 | INFO | "gpu" : " 17 GB", | 20260506 21:46:59.890 | INFO | "huge" : "", | 20260506 21:46:59.890 | INFO | "huge_shared" : " 27 GB", | 20260506 21:46:59.890 | INFO | "malloc" : " 14 GB", | 20260506 21:46:59.890 | INFO | "pinned" : " 29 GB" | 20260506 21:46:59.890 | INFO | } | 20260506 21:46:59.890 | INFO | Perf report for the last 5.13791 seconds: | 20260506 21:46:59.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:46:59.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:59.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:59.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:59.890 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:46:59.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:59.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:59.890 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 21:46:59.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:59.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:46:59.891 | INFO | Uploaded 36GiB to GPU in 2.00858s (17.9231GiB/s) | 20260506 21:46:59.891 | INFO | Downloaded 34GiB to GPU in 1.92287s (17.6819GiB/s) | 20260506 21:47:05.028 | INFO | Analyzing chunk frames = [4695040,4695552) | 20260506 21:47:05.028 | INFO | MemInfo: { | 20260506 21:47:05.028 | INFO | "gpu" : " 17 GB", | 20260506 21:47:05.028 | INFO | "huge" : "", | 20260506 21:47:05.028 | INFO | "huge_shared" : " 27 GB", | 20260506 21:47:05.028 | INFO | "malloc" : " 14 GB", | 20260506 21:47:05.028 | INFO | "pinned" : " 29 GB" | 20260506 21:47:05.028 | INFO | } | 20260506 21:47:05.028 | INFO | Perf report for the last 5.13772 seconds: | 20260506 21:47:05.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:47:05.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:05.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:05.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:05.028 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 21:47:05.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:05.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:05.028 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:47:05.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:05.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:05.028 | INFO | Uploaded 36GiB to GPU in 1.98894s (18.1001GiB/s) | 20260506 21:47:05.028 | INFO | Downloaded 34GiB to GPU in 1.92065s (17.7023GiB/s) | 20260506 21:47:10.168 | INFO | Analyzing chunk frames = [4695552,4696064) | 20260506 21:47:10.168 | INFO | MemInfo: { | 20260506 21:47:10.168 | INFO | "gpu" : " 17 GB", | 20260506 21:47:10.168 | INFO | "huge" : "", | 20260506 21:47:10.168 | INFO | "huge_shared" : " 27 GB", | 20260506 21:47:10.168 | INFO | "malloc" : " 14 GB", | 20260506 21:47:10.168 | INFO | "pinned" : " 29 GB" | 20260506 21:47:10.168 | INFO | } | 20260506 21:47:10.168 | INFO | Perf report for the last 5.14038 seconds: | 20260506 21:47:10.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:47:10.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:10.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:10.168 | INFO | SAVE_TRACE : 0.2%, 1.0, 0.1ms, 0 | 20260506 21:47:10.168 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:47:10.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:10.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:10.168 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:47:10.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:10.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:10.169 | INFO | Uploaded 36GiB to GPU in 1.9629s (18.3402GiB/s) | 20260506 21:47:10.169 | INFO | Downloaded 34GiB to GPU in 1.87439s (18.1393GiB/s) | 20260506 21:47:15.304 | INFO | Analyzing chunk frames = [4696064,4696576) | 20260506 21:47:15.304 | INFO | MemInfo: { | 20260506 21:47:15.304 | INFO | "gpu" : " 17 GB", | 20260506 21:47:15.304 | INFO | "huge" : "", | 20260506 21:47:15.304 | INFO | "huge_shared" : " 27 GB", | 20260506 21:47:15.304 | INFO | "malloc" : " 14 GB", | 20260506 21:47:15.304 | INFO | "pinned" : " 29 GB" | 20260506 21:47:15.304 | INFO | } | 20260506 21:47:15.304 | INFO | Perf report for the last 5.136 seconds: | 20260506 21:47:15.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:47:15.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:15.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:15.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:15.304 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 21:47:15.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:15.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:15.305 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:47:15.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:15.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:15.305 | INFO | Uploaded 36GiB to GPU in 2.01301s (17.8837GiB/s) | 20260506 21:47:15.305 | INFO | Downloaded 34GiB to GPU in 1.91721s (17.7341GiB/s) | 20260506 21:47:20.449 | INFO | Analyzing chunk frames = [4696576,4697088) | 20260506 21:47:20.449 | INFO | MemInfo: { | 20260506 21:47:20.449 | INFO | "gpu" : " 17 GB", | 20260506 21:47:20.449 | INFO | "huge" : "", | 20260506 21:47:20.449 | INFO | "huge_shared" : " 27 GB", | 20260506 21:47:20.449 | INFO | "malloc" : " 14 GB", | 20260506 21:47:20.449 | INFO | "pinned" : " 29 GB" | 20260506 21:47:20.449 | INFO | } | 20260506 21:47:20.450 | INFO | Perf report for the last 5.14515 seconds: | 20260506 21:47:20.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:47:20.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:20.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:20.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:20.450 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 21:47:20.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:20.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:20.450 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 21:47:20.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:20.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:20.450 | INFO | Uploaded 36GiB to GPU in 2.02093s (17.8136GiB/s) | 20260506 21:47:20.450 | INFO | Downloaded 34GiB to GPU in 1.93281s (17.591GiB/s) | 20260506 21:47:25.595 | INFO | Analyzing chunk frames = [4697088,4697600) | 20260506 21:47:25.595 | INFO | MemInfo: { | 20260506 21:47:25.595 | INFO | "gpu" : " 17 GB", | 20260506 21:47:25.595 | INFO | "huge" : "", | 20260506 21:47:25.595 | INFO | "huge_shared" : " 27 GB", | 20260506 21:47:25.595 | INFO | "malloc" : " 14 GB", | 20260506 21:47:25.595 | INFO | "pinned" : " 29 GB" | 20260506 21:47:25.595 | INFO | } | 20260506 21:47:25.595 | INFO | Perf report for the last 5.14506 seconds: | 20260506 21:47:25.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:47:25.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:25.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:25.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:25.595 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:47:25.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:25.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:25.595 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:47:25.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:25.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:25.595 | INFO | Uploaded 36GiB to GPU in 1.95883s (18.3783GiB/s) | 20260506 21:47:25.595 | INFO | Downloaded 34GiB to GPU in 1.98646s (17.1159GiB/s) | 20260506 21:47:30.733 | INFO | Analyzing chunk frames = [4697600,4698112) | 20260506 21:47:30.733 | INFO | MemInfo: { | 20260506 21:47:30.733 | INFO | "gpu" : " 17 GB", | 20260506 21:47:30.733 | INFO | "huge" : "", | 20260506 21:47:30.733 | INFO | "huge_shared" : " 27 GB", | 20260506 21:47:30.733 | INFO | "malloc" : " 14 GB", | 20260506 21:47:30.733 | INFO | "pinned" : " 29 GB" | 20260506 21:47:30.733 | INFO | } | 20260506 21:47:30.733 | INFO | Perf report for the last 5.13804 seconds: | 20260506 21:47:30.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:47:30.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:30.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:30.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:30.733 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 21:47:30.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:30.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:30.733 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 21:47:30.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:30.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:30.733 | INFO | Uploaded 36GiB to GPU in 1.98741s (18.114GiB/s) | 20260506 21:47:30.733 | INFO | Downloaded 34GiB to GPU in 1.97278s (17.2346GiB/s) | 20260506 21:47:30.831 | INFO | Starting a round of Baz IO | 20260506 21:47:33.970 | INFO | Populating RTMetrics report | 20260506 21:47:33.970 | INFO | RTMetrics report populated | 20260506 21:47:33.970 | INFO | Opening tmp RTMetrics json file | 20260506 21:47:34.005 | INFO | Serializing RTMetrics to json | 20260506 21:47:34.073 | INFO | Serialization done, swapping tmp file now... | 20260506 21:47:34.075 | INFO | RTMetrics done | 20260506 21:47:34.147 | INFO | Finished a round of Baz IO: Wrote 1.933 GiB in 3.315 seconds | 20260506 21:47:34.147 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.84% / 0.00% / 66.15% | 20260506 21:47:34.147 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.572566% / 1.19209e-05% / 5.16574e-05% | 20260506 21:47:34.147 | INFO | Baz IO overall: | 20260506 21:47:34.147 | INFO | BytesWritten : 2473152911852 | 20260506 21:47:34.147 | INFO | NumEvents : 1193827510626 | 20260506 21:47:35.874 | INFO | Analyzing chunk frames = [4698112,4698624) | 20260506 21:47:35.874 | INFO | MemInfo: { | 20260506 21:47:35.874 | INFO | "gpu" : " 17 GB", | 20260506 21:47:35.874 | INFO | "huge" : "", | 20260506 21:47:35.874 | INFO | "huge_shared" : " 27 GB", | 20260506 21:47:35.874 | INFO | "malloc" : " 14 GB", | 20260506 21:47:35.874 | INFO | "pinned" : " 29 GB" | 20260506 21:47:35.874 | INFO | } | 20260506 21:47:35.874 | INFO | Perf report for the last 5.1412 seconds: | 20260506 21:47:35.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:47:35.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:35.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:35.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:35.874 | INFO | ANALYSIS : 62.4%, 2.9, 95.4ms, 0 | 20260506 21:47:35.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:35.874 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 21:47:35.874 | INFO | PRE_HQ : 60.1%, 1.8, 57.2ms, 0 | 20260506 21:47:35.874 | INFO | BAZWRITER : 64.1%, 4.8, 156.9ms, 0 | 20260506 21:47:35.874 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 21:47:35.874 | INFO | Uploaded 36GiB to GPU in 2.06732s (17.4138GiB/s) | 20260506 21:47:35.874 | INFO | Downloaded 38GiB to GPU in 2.20179s (17.2587GiB/s) | 20260506 21:47:41.003 | INFO | Analyzing chunk frames = [4698624,4699136) | 20260506 21:47:41.003 | INFO | MemInfo: { | 20260506 21:47:41.003 | INFO | "gpu" : " 17 GB", | 20260506 21:47:41.003 | INFO | "huge" : "", | 20260506 21:47:41.003 | INFO | "huge_shared" : " 27 GB", | 20260506 21:47:41.003 | INFO | "malloc" : " 14 GB", | 20260506 21:47:41.003 | INFO | "pinned" : " 29 GB" | 20260506 21:47:41.003 | INFO | } | 20260506 21:47:41.003 | INFO | Perf report for the last 5.12945 seconds: | 20260506 21:47:41.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:47:41.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:41.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:41.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:41.003 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 21:47:41.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:41.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:41.003 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:47:41.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:41.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:41.004 | INFO | Uploaded 36GiB to GPU in 2.00397s (17.9643GiB/s) | 20260506 21:47:41.004 | INFO | Downloaded 34GiB to GPU in 1.92129s (17.6965GiB/s) | 20260506 21:47:46.142 | INFO | Analyzing chunk frames = [4699136,4699648) | 20260506 21:47:46.142 | INFO | MemInfo: { | 20260506 21:47:46.142 | INFO | "gpu" : " 17 GB", | 20260506 21:47:46.142 | INFO | "huge" : "", | 20260506 21:47:46.142 | INFO | "huge_shared" : " 27 GB", | 20260506 21:47:46.142 | INFO | "malloc" : " 14 GB", | 20260506 21:47:46.142 | INFO | "pinned" : " 29 GB" | 20260506 21:47:46.142 | INFO | } | 20260506 21:47:46.142 | INFO | Perf report for the last 5.13871 seconds: | 20260506 21:47:46.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:47:46.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:46.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:46.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:46.142 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 21:47:46.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:46.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:46.142 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:47:46.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:46.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:46.142 | INFO | Uploaded 36GiB to GPU in 1.97872s (18.1936GiB/s) | 20260506 21:47:46.142 | INFO | Downloaded 34GiB to GPU in 1.92062s (17.7026GiB/s) | 20260506 21:47:51.285 | INFO | Analyzing chunk frames = [4699648,4700160) | 20260506 21:47:51.285 | INFO | MemInfo: { | 20260506 21:47:51.285 | INFO | "gpu" : " 17 GB", | 20260506 21:47:51.285 | INFO | "huge" : "", | 20260506 21:47:51.285 | INFO | "huge_shared" : " 27 GB", | 20260506 21:47:51.286 | INFO | "malloc" : " 14 GB", | 20260506 21:47:51.286 | INFO | "pinned" : " 29 GB" | 20260506 21:47:51.286 | INFO | } | 20260506 21:47:51.286 | INFO | Perf report for the last 5.14345 seconds: | 20260506 21:47:51.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:47:51.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:51.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:51.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:51.286 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 21:47:51.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:51.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:51.286 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:47:51.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:51.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:51.286 | INFO | Uploaded 36GiB to GPU in 1.96504s (18.3202GiB/s) | 20260506 21:47:51.286 | INFO | Downloaded 34GiB to GPU in 1.87423s (18.1408GiB/s) | 20260506 21:47:56.423 | INFO | Analyzing chunk frames = [4700160,4700672) | 20260506 21:47:56.423 | INFO | MemInfo: { | 20260506 21:47:56.423 | INFO | "gpu" : " 17 GB", | 20260506 21:47:56.423 | INFO | "huge" : "", | 20260506 21:47:56.423 | INFO | "huge_shared" : " 27 GB", | 20260506 21:47:56.423 | INFO | "malloc" : " 14 GB", | 20260506 21:47:56.423 | INFO | "pinned" : " 29 GB" | 20260506 21:47:56.423 | INFO | } | 20260506 21:47:56.423 | INFO | Perf report for the last 5.13744 seconds: | 20260506 21:47:56.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:47:56.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:56.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:56.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:56.423 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 21:47:56.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:56.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:56.423 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:47:56.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:56.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:47:56.423 | INFO | Uploaded 36GiB to GPU in 2.00242s (17.9783GiB/s) | 20260506 21:47:56.423 | INFO | Downloaded 34GiB to GPU in 1.90527s (17.8452GiB/s) | 20260506 21:48:01.563 | INFO | Analyzing chunk frames = [4700672,4701184) | 20260506 21:48:01.563 | INFO | MemInfo: { | 20260506 21:48:01.563 | INFO | "gpu" : " 17 GB", | 20260506 21:48:01.563 | INFO | "huge" : "", | 20260506 21:48:01.563 | INFO | "huge_shared" : " 27 GB", | 20260506 21:48:01.563 | INFO | "malloc" : " 14 GB", | 20260506 21:48:01.563 | INFO | "pinned" : " 29 GB" | 20260506 21:48:01.563 | INFO | } | 20260506 21:48:01.563 | INFO | Perf report for the last 5.14023 seconds: | 20260506 21:48:01.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:48:01.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:01.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:01.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:01.563 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 21:48:01.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:01.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:01.563 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 21:48:01.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:01.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:01.564 | INFO | Uploaded 36GiB to GPU in 2.01981s (17.8234GiB/s) | 20260506 21:48:01.564 | INFO | Downloaded 34GiB to GPU in 1.9475s (17.4583GiB/s) | 20260506 21:48:06.706 | INFO | Analyzing chunk frames = [4701184,4701696) | 20260506 21:48:06.706 | INFO | MemInfo: { | 20260506 21:48:06.706 | INFO | "gpu" : " 17 GB", | 20260506 21:48:06.706 | INFO | "huge" : "", | 20260506 21:48:06.706 | INFO | "huge_shared" : " 27 GB", | 20260506 21:48:06.706 | INFO | "malloc" : " 14 GB", | 20260506 21:48:06.706 | INFO | "pinned" : " 29 GB" | 20260506 21:48:06.706 | INFO | } | 20260506 21:48:06.706 | INFO | Perf report for the last 5.14261 seconds: | 20260506 21:48:06.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:48:06.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:06.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:06.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:06.706 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:48:06.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:06.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:06.706 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:48:06.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:06.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:06.706 | INFO | Uploaded 36GiB to GPU in 1.94601s (18.4993GiB/s) | 20260506 21:48:06.706 | INFO | Downloaded 34GiB to GPU in 1.98497s (17.1287GiB/s) | 20260506 21:48:11.845 | INFO | Analyzing chunk frames = [4701696,4702208) | 20260506 21:48:11.845 | INFO | MemInfo: { | 20260506 21:48:11.845 | INFO | "gpu" : " 17 GB", | 20260506 21:48:11.845 | INFO | "huge" : "", | 20260506 21:48:11.845 | INFO | "huge_shared" : " 27 GB", | 20260506 21:48:11.845 | INFO | "malloc" : " 14 GB", | 20260506 21:48:11.845 | INFO | "pinned" : " 29 GB" | 20260506 21:48:11.845 | INFO | } | 20260506 21:48:11.845 | INFO | Perf report for the last 5.13895 seconds: | 20260506 21:48:11.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:48:11.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:11.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:11.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:11.845 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 21:48:11.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:11.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:11.845 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 21:48:11.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:11.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:11.845 | INFO | Uploaded 36GiB to GPU in 2.02487s (17.7789GiB/s) | 20260506 21:48:11.845 | INFO | Downloaded 34GiB to GPU in 1.99844s (17.0133GiB/s) | 20260506 21:48:11.845 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:48:11.845 | INFO | Total mem usage: 18 GiB | 20260506 21:48:11.845 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:48:11.845 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -21.6%) | 20260506 21:48:11.845 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:48:11.845 | INFO | High Water Breakdown: | 20260506 21:48:11.845 | INFO | 60.7% : 8.99 GiB ( 78.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:48:11.845 | INFO | 39.2% : 5.8 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:48:11.845 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:48:11.845 | INFO | ------------------------------------------ | 20260506 21:48:11.845 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:48:11.845 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:48:11.845 | INFO | Total mem usage: 26.5 GiB | 20260506 21:48:11.846 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:48:11.846 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:48:11.846 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:48:11.846 | INFO | High Water Breakdown: | 20260506 21:48:11.846 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:48:11.846 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:48:11.846 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:48:11.846 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:48:11.846 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:48:11.846 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:48:11.846 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:48:11.846 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:48:11.846 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:48:11.846 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:48:11.846 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:48:11.846 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:48:11.846 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:48:11.846 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:48:11.846 | INFO | ------------------------------------------ | 20260506 21:48:11.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:48:11.846 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:48:11.846 | INFO | Total mem usage: 25.6 GiB | 20260506 21:48:11.846 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:48:11.846 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:48:11.846 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:48:11.846 | INFO | High Water Breakdown: | 20260506 21:48:11.846 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:48:11.846 | INFO | ------------------------------------------ | 20260506 21:48:11.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:48:11.846 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:48:11.846 | INFO | Total mem usage: 15.7 GiB | 20260506 21:48:11.846 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:48:11.846 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:48:11.846 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:48:11.846 | INFO | High Water Breakdown: | 20260506 21:48:11.846 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:48:11.846 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:48:11.846 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:48:11.846 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:48:11.846 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:48:11.846 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:48:11.846 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:48:11.846 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:48:11.846 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:48:11.846 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:48:11.846 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:48:11.846 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:48:11.846 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:48:11.846 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:48:11.846 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:48:11.846 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:48:11.846 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:48:11.846 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:48:11.846 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:48:11.846 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:48:11.846 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:48:11.846 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:48:11.846 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:48:11.846 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:48:11.846 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:48:11.846 | INFO | ------------------------------------------ | 20260506 21:48:11.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:48:11.846 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:48:11.846 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:48:11.846 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:48:11.846 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.58e+01, 1.36e+01 / 2.49e+00 | 20260506 21:48:11.846 | INFO | PerfReport: 0.30%, AggregateStats -- 9.19e-02 / 1.01e-01, 1.01e-02 / 5.50e-03 | 20260506 21:48:11.846 | INFO | PerfReport: 11.67%, Baseline -- 3.63e+00 / 4.40e+00, 5.54e-01 / 3.56e-01 | 20260506 21:48:11.846 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.47e+00, 1.83e-01 / 1.36e-01 | 20260506 21:48:11.846 | INFO | PerfReport: 16.04%, DME -- 3.99e+01 / 4.47e+01, 4.10e+00 / 2.98e+00 | 20260506 21:48:11.846 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.39e+00 / 1.98e+00 | 20260506 21:48:11.846 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.21e+00 / 2.72e+00, 3.78e-01 / 2.23e-01 | 20260506 21:48:11.846 | INFO | PerfReport: 3.26%, Metrics -- 1.01e+00 / 1.38e+00, 2.67e-01 / 1.70e-01 | 20260506 21:48:11.846 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.46e-01 / 3.94e-01, 5.35e-02 / 2.43e-02 | 20260506 21:48:11.846 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:48:11.846 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:48:11.846 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:48:11.846 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 21:48:11.846 | INFO | PerfReport: 21.58%, Upload -- 2.08e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 21:48:11.846 | INFO | PerfReport: 21.22%, Download -- 2.05e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260506 21:48:11.846 | INFO | PerfReport: 32.21%, Compute -- 3.11e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 21:48:11.846 | INFO | PerfReport: 1.33%, LockUpload -- 1.29e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260506 21:48:11.846 | INFO | PerfReport: 1.13%, LockDownload -- 1.09e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260506 21:48:11.846 | INFO | PerfReport: 22.52%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:48:11.846 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260506 21:48:11.950 | INFO | Starting a round of Baz IO | 20260506 21:48:15.132 | INFO | Populating RTMetrics report | 20260506 21:48:15.133 | INFO | RTMetrics report populated | 20260506 21:48:15.133 | INFO | Opening tmp RTMetrics json file | 20260506 21:48:15.155 | INFO | Serializing RTMetrics to json | 20260506 21:48:15.156 | INFO | Serialization done, swapping tmp file now... | 20260506 21:48:15.157 | INFO | RTMetrics done | 20260506 21:48:15.749 | INFO | Finished a round of Baz IO: Wrote 1.935 GiB in 3.798 seconds | 20260506 21:48:15.749 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.84% / 0.00% / 66.16% | 20260506 21:48:15.749 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.571593% / 0% / 7.54992e-05% | 20260506 21:48:15.749 | INFO | Baz IO overall: | 20260506 21:48:15.749 | INFO | BytesWritten : 2475230075685 | 20260506 21:48:15.749 | INFO | NumEvents : 1194820831348 | 20260506 21:48:16.980 | INFO | Analyzing chunk frames = [4702208,4702720) | 20260506 21:48:16.980 | INFO | MemInfo: { | 20260506 21:48:16.980 | INFO | "gpu" : " 17 GB", | 20260506 21:48:16.980 | INFO | "huge" : "", | 20260506 21:48:16.980 | INFO | "huge_shared" : " 27 GB", | 20260506 21:48:16.980 | INFO | "malloc" : " 14 GB", | 20260506 21:48:16.980 | INFO | "pinned" : " 29 GB" | 20260506 21:48:16.980 | INFO | } | 20260506 21:48:16.980 | INFO | Perf report for the last 5.13486 seconds: | 20260506 21:48:16.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:48:16.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:16.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:16.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:16.980 | INFO | ANALYSIS : 62.9%, 2.8, 93.7ms, 0 | 20260506 21:48:16.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:16.980 | INFO | RT_METRICS : 47.5%, 1.0, 24.9ms, 0 | 20260506 21:48:16.980 | INFO | PRE_HQ : 61.0%, 2.0, 62.7ms, 0 | 20260506 21:48:16.980 | INFO | BAZWRITER : 73.6%, 4.8, 181.9ms, 0 | 20260506 21:48:16.980 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.4ms, 0 | 20260506 21:48:16.980 | INFO | Uploaded 36GiB to GPU in 2.05839s (17.4894GiB/s) | 20260506 21:48:16.980 | INFO | Downloaded 38GiB to GPU in 2.17048s (17.5076GiB/s) | 20260506 21:48:22.121 | INFO | Analyzing chunk frames = [4702720,4703232) | 20260506 21:48:22.121 | INFO | MemInfo: { | 20260506 21:48:22.121 | INFO | "gpu" : " 17 GB", | 20260506 21:48:22.121 | INFO | "huge" : "", | 20260506 21:48:22.121 | INFO | "huge_shared" : " 27 GB", | 20260506 21:48:22.121 | INFO | "malloc" : " 14 GB", | 20260506 21:48:22.121 | INFO | "pinned" : " 29 GB" | 20260506 21:48:22.121 | INFO | } | 20260506 21:48:22.121 | INFO | Perf report for the last 5.14088 seconds: | 20260506 21:48:22.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:48:22.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:22.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:22.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:22.121 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:48:22.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:22.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:22.121 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 21:48:22.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:22.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:22.121 | INFO | Uploaded 36GiB to GPU in 1.99592s (18.0368GiB/s) | 20260506 21:48:22.121 | INFO | Downloaded 34GiB to GPU in 1.93323s (17.5872GiB/s) | 20260506 21:48:27.258 | INFO | Analyzing chunk frames = [4703232,4703744) | 20260506 21:48:27.258 | INFO | MemInfo: { | 20260506 21:48:27.258 | INFO | "gpu" : " 17 GB", | 20260506 21:48:27.258 | INFO | "huge" : "", | 20260506 21:48:27.258 | INFO | "huge_shared" : " 27 GB", | 20260506 21:48:27.258 | INFO | "malloc" : " 14 GB", | 20260506 21:48:27.258 | INFO | "pinned" : " 29 GB" | 20260506 21:48:27.258 | INFO | } | 20260506 21:48:27.258 | INFO | Perf report for the last 5.13765 seconds: | 20260506 21:48:27.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:48:27.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:27.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:27.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:27.258 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 21:48:27.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:27.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:27.258 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:48:27.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:27.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:27.259 | INFO | Uploaded 36GiB to GPU in 1.96792s (18.2934GiB/s) | 20260506 21:48:27.259 | INFO | Downloaded 34GiB to GPU in 1.92025s (17.706GiB/s) | 20260506 21:48:32.399 | INFO | Analyzing chunk frames = [4703744,4704256) | 20260506 21:48:32.399 | INFO | MemInfo: { | 20260506 21:48:32.399 | INFO | "gpu" : " 17 GB", | 20260506 21:48:32.399 | INFO | "huge" : "", | 20260506 21:48:32.399 | INFO | "huge_shared" : " 27 GB", | 20260506 21:48:32.399 | INFO | "malloc" : " 14 GB", | 20260506 21:48:32.399 | INFO | "pinned" : " 29 GB" | 20260506 21:48:32.399 | INFO | } | 20260506 21:48:32.399 | INFO | Perf report for the last 5.1406 seconds: | 20260506 21:48:32.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:48:32.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:32.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:32.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:32.399 | INFO | ANALYSIS : 60.2%, 3.0, 93.5ms, 0 | 20260506 21:48:32.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:32.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:32.399 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:48:32.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:32.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:32.399 | INFO | Uploaded 36GiB to GPU in 2.00418s (17.9624GiB/s) | 20260506 21:48:32.399 | INFO | Downloaded 34GiB to GPU in 1.90965s (17.8043GiB/s) | 20260506 21:48:37.550 | INFO | Analyzing chunk frames = [4704256,4704768) | 20260506 21:48:37.550 | INFO | MemInfo: { | 20260506 21:48:37.550 | INFO | "gpu" : " 17 GB", | 20260506 21:48:37.550 | INFO | "huge" : "", | 20260506 21:48:37.550 | INFO | "huge_shared" : " 27 GB", | 20260506 21:48:37.550 | INFO | "malloc" : " 14 GB", | 20260506 21:48:37.550 | INFO | "pinned" : " 29 GB" | 20260506 21:48:37.550 | INFO | } | 20260506 21:48:37.550 | INFO | Perf report for the last 5.15143 seconds: | 20260506 21:48:37.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:48:37.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:37.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:37.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:37.550 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260506 21:48:37.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:37.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:37.550 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:48:37.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:37.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:37.551 | INFO | Uploaded 36GiB to GPU in 1.99978s (18.002GiB/s) | 20260506 21:48:37.551 | INFO | Downloaded 34GiB to GPU in 1.92254s (17.6849GiB/s) | 20260506 21:48:42.681 | INFO | Analyzing chunk frames = [4704768,4705280) | 20260506 21:48:42.681 | INFO | MemInfo: { | 20260506 21:48:42.681 | INFO | "gpu" : " 17 GB", | 20260506 21:48:42.681 | INFO | "huge" : "", | 20260506 21:48:42.681 | INFO | "huge_shared" : " 27 GB", | 20260506 21:48:42.681 | INFO | "malloc" : " 14 GB", | 20260506 21:48:42.682 | INFO | "pinned" : " 29 GB" | 20260506 21:48:42.682 | INFO | } | 20260506 21:48:42.682 | INFO | Perf report for the last 5.13149 seconds: | 20260506 21:48:42.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:48:42.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:42.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:42.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:42.682 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 21:48:42.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:42.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:42.682 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:48:42.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:42.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:42.682 | INFO | Uploaded 36GiB to GPU in 2.01115s (17.9002GiB/s) | 20260506 21:48:42.682 | INFO | Downloaded 34GiB to GPU in 1.93457s (17.575GiB/s) | 20260506 21:48:47.819 | INFO | Analyzing chunk frames = [4705280,4705792) | 20260506 21:48:47.819 | INFO | MemInfo: { | 20260506 21:48:47.819 | INFO | "gpu" : " 17 GB", | 20260506 21:48:47.819 | INFO | "huge" : "", | 20260506 21:48:47.820 | INFO | "huge_shared" : " 27 GB", | 20260506 21:48:47.820 | INFO | "malloc" : " 14 GB", | 20260506 21:48:47.820 | INFO | "pinned" : " 29 GB" | 20260506 21:48:47.820 | INFO | } | 20260506 21:48:47.820 | INFO | Perf report for the last 5.13766 seconds: | 20260506 21:48:47.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:48:47.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:47.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:47.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:47.820 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:48:47.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:47.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:47.820 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:48:47.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:47.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:47.820 | INFO | Uploaded 36GiB to GPU in 1.95748s (18.391GiB/s) | 20260506 21:48:47.820 | INFO | Downloaded 34GiB to GPU in 1.99992s (17.0007GiB/s) | 20260506 21:48:52.959 | INFO | Analyzing chunk frames = [4705792,4706304) | 20260506 21:48:52.959 | INFO | MemInfo: { | 20260506 21:48:52.959 | INFO | "gpu" : " 17 GB", | 20260506 21:48:52.959 | INFO | "huge" : "", | 20260506 21:48:52.959 | INFO | "huge_shared" : " 27 GB", | 20260506 21:48:52.959 | INFO | "malloc" : " 14 GB", | 20260506 21:48:52.959 | INFO | "pinned" : " 29 GB" | 20260506 21:48:52.959 | INFO | } | 20260506 21:48:52.959 | INFO | Perf report for the last 5.13921 seconds: | 20260506 21:48:52.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:48:52.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:52.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:52.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:52.959 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 21:48:52.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:52.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:52.959 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 21:48:52.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:52.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:52.959 | INFO | Uploaded 36GiB to GPU in 2.02454s (17.7818GiB/s) | 20260506 21:48:52.959 | INFO | Downloaded 34GiB to GPU in 1.99771s (17.0195GiB/s) | 20260506 21:48:53.064 | INFO | Starting a round of Baz IO | 20260506 21:48:56.197 | INFO | Populating RTMetrics report | 20260506 21:48:56.197 | INFO | RTMetrics report populated | 20260506 21:48:56.197 | INFO | Opening tmp RTMetrics json file | 20260506 21:48:56.233 | INFO | Serializing RTMetrics to json | 20260506 21:48:56.339 | INFO | Serialization done, swapping tmp file now... | 20260506 21:48:56.368 | INFO | RTMetrics done | 20260506 21:48:56.411 | INFO | Finished a round of Baz IO: Wrote 1.933 GiB in 3.346 seconds | 20260506 21:48:56.411 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.84% / 0.00% / 66.16% | 20260506 21:48:56.411 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.569391% / 0% / 8.34465e-05% | 20260506 21:48:56.411 | INFO | Baz IO overall: | 20260506 21:48:56.411 | INFO | BytesWritten : 2477305697197 | 20260506 21:48:56.411 | INFO | NumEvents : 1195812934699 | 20260506 21:48:58.094 | INFO | Analyzing chunk frames = [4706304,4706816) | 20260506 21:48:58.094 | INFO | MemInfo: { | 20260506 21:48:58.094 | INFO | "gpu" : " 17 GB", | 20260506 21:48:58.094 | INFO | "huge" : "", | 20260506 21:48:58.094 | INFO | "huge_shared" : " 27 GB", | 20260506 21:48:58.094 | INFO | "malloc" : " 14 GB", | 20260506 21:48:58.094 | INFO | "pinned" : " 29 GB" | 20260506 21:48:58.094 | INFO | } | 20260506 21:48:58.094 | INFO | Perf report for the last 5.13526 seconds: | 20260506 21:48:58.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:48:58.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:58.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:58.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:58.094 | INFO | ANALYSIS : 62.5%, 2.9, 95.6ms, 0 | 20260506 21:48:58.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:48:58.094 | INFO | RT_METRICS : 45.9%, 1.0, 24.0ms, 0 | 20260506 21:48:58.094 | INFO | PRE_HQ : 59.7%, 1.8, 56.6ms, 0 | 20260506 21:48:58.094 | INFO | BAZWRITER : 64.9%, 4.7, 158.0ms, 0 | 20260506 21:48:58.094 | INFO | BAZ_SAMPLING : 3.1%, 1.0, 1.6ms, 0 | 20260506 21:48:58.094 | INFO | Uploaded 36GiB to GPU in 2.03457s (17.6942GiB/s) | 20260506 21:48:58.094 | INFO | Downloaded 38GiB to GPU in 2.20742s (17.2146GiB/s) | 20260506 21:49:03.237 | INFO | Analyzing chunk frames = [4706816,4707328) | 20260506 21:49:03.237 | INFO | MemInfo: { | 20260506 21:49:03.237 | INFO | "gpu" : " 17 GB", | 20260506 21:49:03.237 | INFO | "huge" : "", | 20260506 21:49:03.237 | INFO | "huge_shared" : " 27 GB", | 20260506 21:49:03.237 | INFO | "malloc" : " 14 GB", | 20260506 21:49:03.237 | INFO | "pinned" : " 29 GB" | 20260506 21:49:03.237 | INFO | } | 20260506 21:49:03.237 | INFO | Perf report for the last 5.14289 seconds: | 20260506 21:49:03.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:49:03.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:03.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:03.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:03.237 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 21:49:03.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:03.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:03.237 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:49:03.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:03.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:03.237 | INFO | Uploaded 36GiB to GPU in 2.00267s (17.976GiB/s) | 20260506 21:49:03.237 | INFO | Downloaded 34GiB to GPU in 1.93101s (17.6074GiB/s) | 20260506 21:49:08.380 | INFO | Analyzing chunk frames = [4707328,4707840) | 20260506 21:49:08.380 | INFO | MemInfo: { | 20260506 21:49:08.380 | INFO | "gpu" : " 17 GB", | 20260506 21:49:08.380 | INFO | "huge" : "", | 20260506 21:49:08.380 | INFO | "huge_shared" : " 27 GB", | 20260506 21:49:08.380 | INFO | "malloc" : " 14 GB", | 20260506 21:49:08.380 | INFO | "pinned" : " 29 GB" | 20260506 21:49:08.380 | INFO | } | 20260506 21:49:08.380 | INFO | Perf report for the last 5.1428 seconds: | 20260506 21:49:08.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:49:08.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:08.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:08.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:08.380 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 21:49:08.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:08.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:08.380 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:49:08.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:08.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:08.380 | INFO | Uploaded 36GiB to GPU in 1.94898s (18.4712GiB/s) | 20260506 21:49:08.380 | INFO | Downloaded 34GiB to GPU in 1.93715s (17.5515GiB/s) | 20260506 21:49:13.511 | INFO | Analyzing chunk frames = [4707840,4708352) | 20260506 21:49:13.511 | INFO | MemInfo: { | 20260506 21:49:13.511 | INFO | "gpu" : " 17 GB", | 20260506 21:49:13.511 | INFO | "huge" : "", | 20260506 21:49:13.511 | INFO | "huge_shared" : " 27 GB", | 20260506 21:49:13.511 | INFO | "malloc" : " 14 GB", | 20260506 21:49:13.511 | INFO | "pinned" : " 29 GB" | 20260506 21:49:13.511 | INFO | } | 20260506 21:49:13.511 | INFO | Perf report for the last 5.1311 seconds: | 20260506 21:49:13.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:49:13.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:13.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:13.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:13.511 | INFO | ANALYSIS : 59.7%, 3.0, 92.3ms, 0 | 20260506 21:49:13.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:13.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:13.511 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:49:13.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:13.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:13.511 | INFO | Uploaded 36GiB to GPU in 1.93633s (18.5919GiB/s) | 20260506 21:49:13.511 | INFO | Downloaded 34GiB to GPU in 1.87705s (18.1135GiB/s) | 20260506 21:49:18.655 | INFO | Analyzing chunk frames = [4708352,4708864) | 20260506 21:49:18.655 | INFO | MemInfo: { | 20260506 21:49:18.655 | INFO | "gpu" : " 17 GB", | 20260506 21:49:18.655 | INFO | "huge" : "", | 20260506 21:49:18.655 | INFO | "huge_shared" : " 27 GB", | 20260506 21:49:18.655 | INFO | "malloc" : " 14 GB", | 20260506 21:49:18.655 | INFO | "pinned" : " 29 GB" | 20260506 21:49:18.655 | INFO | } | 20260506 21:49:18.655 | INFO | Perf report for the last 5.14444 seconds: | 20260506 21:49:18.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:49:18.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:18.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:18.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:18.655 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260506 21:49:18.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:18.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:18.655 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:49:18.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:18.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:18.656 | INFO | Uploaded 36GiB to GPU in 1.99788s (18.0191GiB/s) | 20260506 21:49:18.657 | INFO | Downloaded 34GiB to GPU in 1.91197s (17.7827GiB/s) | 20260506 21:49:23.796 | INFO | Analyzing chunk frames = [4708864,4709376) | 20260506 21:49:23.796 | INFO | MemInfo: { | 20260506 21:49:23.796 | INFO | "gpu" : " 17 GB", | 20260506 21:49:23.796 | INFO | "huge" : "", | 20260506 21:49:23.796 | INFO | "huge_shared" : " 27 GB", | 20260506 21:49:23.796 | INFO | "malloc" : " 14 GB", | 20260506 21:49:23.796 | INFO | "pinned" : " 29 GB" | 20260506 21:49:23.796 | INFO | } | 20260506 21:49:23.796 | INFO | Perf report for the last 5.14067 seconds: | 20260506 21:49:23.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:49:23.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:23.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:23.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:23.796 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260506 21:49:23.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:23.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:23.796 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 21:49:23.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:23.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:23.796 | INFO | Uploaded 36GiB to GPU in 1.98584s (18.1283GiB/s) | 20260506 21:49:23.796 | INFO | Downloaded 34GiB to GPU in 1.94897s (17.4451GiB/s) | 20260506 21:49:28.933 | INFO | Analyzing chunk frames = [4709376,4709888) | 20260506 21:49:28.933 | INFO | MemInfo: { | 20260506 21:49:28.933 | INFO | "gpu" : " 17 GB", | 20260506 21:49:28.933 | INFO | "huge" : "", | 20260506 21:49:28.933 | INFO | "huge_shared" : " 27 GB", | 20260506 21:49:28.933 | INFO | "malloc" : " 14 GB", | 20260506 21:49:28.933 | INFO | "pinned" : " 29 GB" | 20260506 21:49:28.933 | INFO | } | 20260506 21:49:28.933 | INFO | Perf report for the last 5.13699 seconds: | 20260506 21:49:28.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:49:28.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:28.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:28.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:28.933 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 21:49:28.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:28.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:28.933 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 21:49:28.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:28.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:28.933 | INFO | Uploaded 36GiB to GPU in 2.00486s (17.9564GiB/s) | 20260506 21:49:28.933 | INFO | Downloaded 34GiB to GPU in 2.001s (16.9915GiB/s) | 20260506 21:49:34.069 | INFO | Analyzing chunk frames = [4709888,4710400) | 20260506 21:49:34.069 | INFO | MemInfo: { | 20260506 21:49:34.069 | INFO | "gpu" : " 17 GB", | 20260506 21:49:34.069 | INFO | "huge" : "", | 20260506 21:49:34.069 | INFO | "huge_shared" : " 27 GB", | 20260506 21:49:34.069 | INFO | "malloc" : " 14 GB", | 20260506 21:49:34.069 | INFO | "pinned" : " 29 GB" | 20260506 21:49:34.069 | INFO | } | 20260506 21:49:34.069 | INFO | Perf report for the last 5.13623 seconds: | 20260506 21:49:34.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:49:34.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:34.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:34.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:34.069 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260506 21:49:34.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:34.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:34.070 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 21:49:34.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:34.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:34.070 | INFO | Uploaded 36GiB to GPU in 2.03376s (17.7012GiB/s) | 20260506 21:49:34.070 | INFO | Downloaded 34GiB to GPU in 1.98093s (17.1637GiB/s) | 20260506 21:49:34.070 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:49:34.070 | INFO | Total mem usage: 18 GiB | 20260506 21:49:34.070 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:49:34.070 | INFO | Recent High Water Marks Sum: 16.9 GiB (Savings: -6.6%) | 20260506 21:49:34.070 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:49:34.070 | INFO | High Water Breakdown: | 20260506 21:49:34.070 | INFO | 62.7% : 10.6 GiB ( 92.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:49:34.070 | INFO | 37.2% : 6.28 GiB ( 93.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:49:34.070 | INFO | 0.118% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:49:34.070 | INFO | ------------------------------------------ | 20260506 21:49:34.070 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:49:34.070 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:49:34.070 | INFO | Total mem usage: 26.5 GiB | 20260506 21:49:34.070 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:49:34.070 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:49:34.070 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:49:34.070 | INFO | High Water Breakdown: | 20260506 21:49:34.070 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:49:34.070 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:49:34.070 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:49:34.070 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:49:34.070 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:49:34.070 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:49:34.070 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:49:34.070 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:49:34.070 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:49:34.070 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:49:34.070 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:49:34.070 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:49:34.070 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:49:34.070 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:49:34.070 | INFO | ------------------------------------------ | 20260506 21:49:34.070 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:49:34.070 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:49:34.070 | INFO | Total mem usage: 25.6 GiB | 20260506 21:49:34.070 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:49:34.070 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:49:34.070 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:49:34.070 | INFO | High Water Breakdown: | 20260506 21:49:34.070 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:49:34.070 | INFO | ------------------------------------------ | 20260506 21:49:34.070 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:49:34.070 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:49:34.070 | INFO | Total mem usage: 15.7 GiB | 20260506 21:49:34.070 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:49:34.070 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:49:34.070 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 21:49:34.070 | INFO | High Water Breakdown: | 20260506 21:49:34.070 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:49:34.070 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:49:34.070 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:49:34.070 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:49:34.070 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:49:34.071 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:49:34.071 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:49:34.071 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:49:34.071 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:49:34.071 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:49:34.071 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:49:34.071 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:49:34.071 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:49:34.071 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:49:34.071 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:49:34.071 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:49:34.071 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:49:34.071 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:49:34.071 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:49:34.071 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:49:34.071 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:49:34.071 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:49:34.071 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:49:34.071 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:49:34.071 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:49:34.071 | INFO | ------------------------------------------ | 20260506 21:49:34.071 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:49:34.071 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:49:34.071 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:49:34.071 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:49:34.071 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.59e+01, 1.34e+01 / 2.42e+00 | 20260506 21:49:34.071 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 1.00e-02 / 6.55e-03 | 20260506 21:49:34.071 | INFO | PerfReport: 11.64%, Baseline -- 3.63e+00 / 4.37e+00, 5.54e-01 / 3.98e-01 | 20260506 21:49:34.071 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.46e+00, 1.86e-01 / 1.49e-01 | 20260506 21:49:34.071 | INFO | PerfReport: 16.02%, DME -- 3.99e+01 / 4.46e+01, 4.19e+00 / 2.59e+00 | 20260506 21:49:34.071 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.79e+01 / 2.10e+01, 1.47e+00 / 2.01e+00 | 20260506 21:49:34.071 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.20e+00 / 2.69e+00, 3.73e-01 / 2.56e-01 | 20260506 21:49:34.071 | INFO | PerfReport: 3.24%, Metrics -- 1.01e+00 / 1.32e+00, 2.66e-01 / 1.79e-01 | 20260506 21:49:34.071 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.46e-01 / 3.93e-01, 5.34e-02 / 3.19e-02 | 20260506 21:49:34.071 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:49:34.071 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:49:34.071 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:49:34.071 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 21:49:34.071 | INFO | PerfReport: 21.54%, Upload -- 2.08e+01 / 0.00e+00, 6.92e+00 / 0.00e+00 | 20260506 21:49:34.071 | INFO | PerfReport: 21.28%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 21:49:34.071 | INFO | PerfReport: 32.24%, Compute -- 3.11e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 21:49:34.071 | INFO | PerfReport: 1.28%, LockUpload -- 1.24e+00 / 0.00e+00, 5.07e+00 / 0.00e+00 | 20260506 21:49:34.071 | INFO | PerfReport: 1.26%, LockDownload -- 1.22e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260506 21:49:34.071 | INFO | PerfReport: 22.39%, LockCompute -- 2.16e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 21:49:34.071 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260506 21:49:34.210 | INFO | Starting a round of Baz IO | 20260506 21:49:37.309 | INFO | Populating RTMetrics report | 20260506 21:49:37.309 | INFO | RTMetrics report populated | 20260506 21:49:37.309 | INFO | Opening tmp RTMetrics json file | 20260506 21:49:37.310 | INFO | Serializing RTMetrics to json | 20260506 21:49:37.312 | INFO | Serialization done, swapping tmp file now... | 20260506 21:49:37.312 | INFO | RTMetrics done | 20260506 21:49:37.472 | INFO | Finished a round of Baz IO: Wrote 1.933 GiB in 3.261 seconds | 20260506 21:49:37.472 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.83% / 0.00% / 66.16% | 20260506 21:49:37.472 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.568187% / 0% / 6.75519e-05% | 20260506 21:49:37.472 | INFO | Baz IO overall: | 20260506 21:49:37.472 | INFO | BytesWritten : 2479381434614 | 20260506 21:49:37.472 | INFO | NumEvents : 1196805562763 | 20260506 21:49:39.212 | INFO | Analyzing chunk frames = [4710400,4710912) | 20260506 21:49:39.212 | INFO | MemInfo: { | 20260506 21:49:39.212 | INFO | "gpu" : " 17 GB", | 20260506 21:49:39.212 | INFO | "huge" : "", | 20260506 21:49:39.212 | INFO | "huge_shared" : " 27 GB", | 20260506 21:49:39.212 | INFO | "malloc" : " 14 GB", | 20260506 21:49:39.212 | INFO | "pinned" : " 29 GB" | 20260506 21:49:39.212 | INFO | } | 20260506 21:49:39.212 | INFO | Perf report for the last 5.14282 seconds: | 20260506 21:49:39.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:49:39.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:39.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:39.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:39.212 | INFO | ANALYSIS : 62.4%, 2.9, 95.4ms, 0 | 20260506 21:49:39.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:39.212 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 21:49:39.212 | INFO | PRE_HQ : 60.1%, 1.8, 57.2ms, 0 | 20260506 21:49:39.212 | INFO | BAZWRITER : 63.4%, 4.8, 157.4ms, 0 | 20260506 21:49:39.212 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 21:49:39.212 | INFO | Uploaded 36GiB to GPU in 2.04776s (17.5801GiB/s) | 20260506 21:49:39.212 | INFO | Downloaded 38GiB to GPU in 2.19371s (17.3223GiB/s) | 20260506 21:49:44.352 | INFO | Analyzing chunk frames = [4710912,4711424) | 20260506 21:49:44.352 | INFO | MemInfo: { | 20260506 21:49:44.352 | INFO | "gpu" : " 17 GB", | 20260506 21:49:44.352 | INFO | "huge" : "", | 20260506 21:49:44.352 | INFO | "huge_shared" : " 27 GB", | 20260506 21:49:44.352 | INFO | "malloc" : " 14 GB", | 20260506 21:49:44.352 | INFO | "pinned" : " 29 GB" | 20260506 21:49:44.352 | INFO | } | 20260506 21:49:44.352 | INFO | Perf report for the last 5.14034 seconds: | 20260506 21:49:44.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:49:44.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:44.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:44.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:44.353 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:49:44.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:44.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:44.353 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:49:44.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:44.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:44.353 | INFO | Uploaded 36GiB to GPU in 2.01483s (17.8675GiB/s) | 20260506 21:49:44.353 | INFO | Downloaded 34GiB to GPU in 1.92264s (17.684GiB/s) | 20260506 21:49:49.490 | INFO | Analyzing chunk frames = [4711424,4711936) | 20260506 21:49:49.490 | INFO | MemInfo: { | 20260506 21:49:49.490 | INFO | "gpu" : " 17 GB", | 20260506 21:49:49.490 | INFO | "huge" : "", | 20260506 21:49:49.490 | INFO | "huge_shared" : " 27 GB", | 20260506 21:49:49.490 | INFO | "malloc" : " 14 GB", | 20260506 21:49:49.490 | INFO | "pinned" : " 29 GB" | 20260506 21:49:49.490 | INFO | } | 20260506 21:49:49.490 | INFO | Perf report for the last 5.13743 seconds: | 20260506 21:49:49.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:49:49.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:49.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:49.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:49.490 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 21:49:49.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:49.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:49.490 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:49:49.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:49.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:49.491 | INFO | Uploaded 36GiB to GPU in 1.98786s (18.1099GiB/s) | 20260506 21:49:49.491 | INFO | Downloaded 34GiB to GPU in 1.92661s (17.6476GiB/s) | 20260506 21:49:54.631 | INFO | Analyzing chunk frames = [4711936,4712448) | 20260506 21:49:54.631 | INFO | MemInfo: { | 20260506 21:49:54.631 | INFO | "gpu" : " 17 GB", | 20260506 21:49:54.631 | INFO | "huge" : "", | 20260506 21:49:54.631 | INFO | "huge_shared" : " 27 GB", | 20260506 21:49:54.631 | INFO | "malloc" : " 14 GB", | 20260506 21:49:54.631 | INFO | "pinned" : " 29 GB" | 20260506 21:49:54.631 | INFO | } | 20260506 21:49:54.631 | INFO | Perf report for the last 5.14146 seconds: | 20260506 21:49:54.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:49:54.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:54.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:54.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:54.631 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 21:49:54.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:54.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:54.631 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:49:54.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:54.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:54.632 | INFO | Uploaded 36GiB to GPU in 1.93912s (18.5651GiB/s) | 20260506 21:49:54.632 | INFO | Downloaded 34GiB to GPU in 1.87261s (18.1564GiB/s) | 20260506 21:49:59.770 | INFO | Analyzing chunk frames = [4712448,4712960) | 20260506 21:49:59.770 | INFO | MemInfo: { | 20260506 21:49:59.770 | INFO | "gpu" : " 17 GB", | 20260506 21:49:59.770 | INFO | "huge" : "", | 20260506 21:49:59.770 | INFO | "huge_shared" : " 27 GB", | 20260506 21:49:59.770 | INFO | "malloc" : " 14 GB", | 20260506 21:49:59.770 | INFO | "pinned" : " 29 GB" | 20260506 21:49:59.770 | INFO | } | 20260506 21:49:59.770 | INFO | Perf report for the last 5.1388 seconds: | 20260506 21:49:59.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:49:59.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:59.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:59.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:59.770 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 21:49:59.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:59.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:59.770 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:49:59.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:59.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:49:59.770 | INFO | Uploaded 36GiB to GPU in 1.9997s (18.0027GiB/s) | 20260506 21:49:59.771 | INFO | Downloaded 34GiB to GPU in 1.91961s (17.7119GiB/s) | 20260506 21:50:04.911 | INFO | Analyzing chunk frames = [4712960,4713472) | 20260506 21:50:04.911 | INFO | MemInfo: { | 20260506 21:50:04.911 | INFO | "gpu" : " 17 GB", | 20260506 21:50:04.911 | INFO | "huge" : "", | 20260506 21:50:04.911 | INFO | "huge_shared" : " 27 GB", | 20260506 21:50:04.911 | INFO | "malloc" : " 14 GB", | 20260506 21:50:04.911 | INFO | "pinned" : " 29 GB" | 20260506 21:50:04.911 | INFO | } | 20260506 21:50:04.911 | INFO | Perf report for the last 5.14052 seconds: | 20260506 21:50:04.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:50:04.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:04.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:04.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:04.911 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:50:04.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:04.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:04.911 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:50:04.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:04.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:04.911 | INFO | Uploaded 36GiB to GPU in 2.02494s (17.7783GiB/s) | 20260506 21:50:04.911 | INFO | Downloaded 34GiB to GPU in 1.92442s (17.6677GiB/s) | 20260506 21:50:10.051 | INFO | Analyzing chunk frames = [4713472,4713984) | 20260506 21:50:10.051 | INFO | MemInfo: { | 20260506 21:50:10.051 | INFO | "gpu" : " 17 GB", | 20260506 21:50:10.051 | INFO | "huge" : "", | 20260506 21:50:10.051 | INFO | "huge_shared" : " 27 GB", | 20260506 21:50:10.051 | INFO | "malloc" : " 14 GB", | 20260506 21:50:10.051 | INFO | "pinned" : " 29 GB" | 20260506 21:50:10.051 | INFO | } | 20260506 21:50:10.051 | INFO | Perf report for the last 5.14025 seconds: | 20260506 21:50:10.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:50:10.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:10.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:10.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:10.051 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:50:10.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:10.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:10.051 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 21:50:10.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:10.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:10.051 | INFO | Uploaded 36GiB to GPU in 1.98008s (18.1811GiB/s) | 20260506 21:50:10.051 | INFO | Downloaded 34GiB to GPU in 1.99875s (17.0106GiB/s) | 20260506 21:50:15.188 | INFO | Analyzing chunk frames = [4713984,4714496) | 20260506 21:50:15.188 | INFO | MemInfo: { | 20260506 21:50:15.188 | INFO | "gpu" : " 17 GB", | 20260506 21:50:15.188 | INFO | "huge" : "", | 20260506 21:50:15.188 | INFO | "huge_shared" : " 27 GB", | 20260506 21:50:15.188 | INFO | "malloc" : " 14 GB", | 20260506 21:50:15.188 | INFO | "pinned" : " 29 GB" | 20260506 21:50:15.188 | INFO | } | 20260506 21:50:15.188 | INFO | Perf report for the last 5.13707 seconds: | 20260506 21:50:15.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:50:15.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:15.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:15.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:15.188 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 21:50:15.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:15.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:15.188 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 21:50:15.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:15.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:15.188 | INFO | Uploaded 36GiB to GPU in 2.02704s (17.7599GiB/s) | 20260506 21:50:15.188 | INFO | Downloaded 34GiB to GPU in 1.98117s (17.1616GiB/s) | 20260506 21:50:15.316 | INFO | Starting a round of Baz IO | 20260506 21:50:18.425 | INFO | Populating RTMetrics report | 20260506 21:50:18.425 | INFO | RTMetrics report populated | 20260506 21:50:18.425 | INFO | Opening tmp RTMetrics json file | 20260506 21:50:18.448 | INFO | Serializing RTMetrics to json | 20260506 21:50:18.566 | INFO | Serialization done, swapping tmp file now... | 20260506 21:50:18.580 | INFO | RTMetrics done | 20260506 21:50:18.606 | INFO | Finished a round of Baz IO: Wrote 1.933 GiB in 3.289 seconds | 20260506 21:50:18.606 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.83% / 0.00% / 66.17% | 20260506 21:50:18.606 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.567619% / 3.97364e-06% / 6.35783e-05% | 20260506 21:50:18.606 | INFO | Baz IO overall: | 20260506 21:50:18.606 | INFO | BytesWritten : 2481457123087 | 20260506 21:50:18.606 | INFO | NumEvents : 1197797901478 | 20260506 21:50:20.324 | INFO | Analyzing chunk frames = [4714496,4715008) | 20260506 21:50:20.324 | INFO | MemInfo: { | 20260506 21:50:20.324 | INFO | "gpu" : " 17 GB", | 20260506 21:50:20.324 | INFO | "huge" : "", | 20260506 21:50:20.324 | INFO | "huge_shared" : " 27 GB", | 20260506 21:50:20.324 | INFO | "malloc" : " 14 GB", | 20260506 21:50:20.324 | INFO | "pinned" : " 29 GB" | 20260506 21:50:20.324 | INFO | } | 20260506 21:50:20.324 | INFO | Perf report for the last 5.13568 seconds: | 20260506 21:50:20.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:50:20.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:20.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:20.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:20.324 | INFO | ANALYSIS : 62.4%, 2.9, 95.4ms, 0 | 20260506 21:50:20.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:20.324 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260506 21:50:20.324 | INFO | PRE_HQ : 60.0%, 1.9, 57.7ms, 0 | 20260506 21:50:20.324 | INFO | BAZWRITER : 63.8%, 4.8, 157.1ms, 0 | 20260506 21:50:20.324 | INFO | BAZ_SAMPLING : 2.6%, 1.0, 1.3ms, 0 | 20260506 21:50:20.324 | INFO | Uploaded 36GiB to GPU in 2.06067s (17.47GiB/s) | 20260506 21:50:20.324 | INFO | Downloaded 38GiB to GPU in 2.2061s (17.225GiB/s) | 20260506 21:50:25.461 | INFO | Analyzing chunk frames = [4715008,4715520) | 20260506 21:50:25.461 | INFO | MemInfo: { | 20260506 21:50:25.461 | INFO | "gpu" : " 17 GB", | 20260506 21:50:25.461 | INFO | "huge" : "", | 20260506 21:50:25.461 | INFO | "huge_shared" : " 27 GB", | 20260506 21:50:25.461 | INFO | "malloc" : " 14 GB", | 20260506 21:50:25.461 | INFO | "pinned" : " 29 GB" | 20260506 21:50:25.461 | INFO | } | 20260506 21:50:25.461 | INFO | Perf report for the last 5.13718 seconds: | 20260506 21:50:25.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:50:25.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:25.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:25.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:25.461 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:50:25.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:25.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:25.461 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 21:50:25.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:25.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:25.461 | INFO | Uploaded 36GiB to GPU in 2.01616s (17.8557GiB/s) | 20260506 21:50:25.461 | INFO | Downloaded 34GiB to GPU in 1.92659s (17.6478GiB/s) | 20260506 21:50:30.599 | INFO | Analyzing chunk frames = [4715520,4716032) | 20260506 21:50:30.599 | INFO | MemInfo: { | 20260506 21:50:30.599 | INFO | "gpu" : " 17 GB", | 20260506 21:50:30.599 | INFO | "huge" : "", | 20260506 21:50:30.599 | INFO | "huge_shared" : " 27 GB", | 20260506 21:50:30.599 | INFO | "malloc" : " 14 GB", | 20260506 21:50:30.599 | INFO | "pinned" : " 29 GB" | 20260506 21:50:30.599 | INFO | } | 20260506 21:50:30.599 | INFO | Perf report for the last 5.13833 seconds: | 20260506 21:50:30.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:50:30.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:30.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:30.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:30.599 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 21:50:30.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:30.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:30.599 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:50:30.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:30.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:30.599 | INFO | Uploaded 36GiB to GPU in 1.97387s (18.2383GiB/s) | 20260506 21:50:30.600 | INFO | Downloaded 34GiB to GPU in 1.92165s (17.6931GiB/s) | 20260506 21:50:35.741 | INFO | Analyzing chunk frames = [4716032,4716544) | 20260506 21:50:35.741 | INFO | MemInfo: { | 20260506 21:50:35.741 | INFO | "gpu" : " 17 GB", | 20260506 21:50:35.741 | INFO | "huge" : "", | 20260506 21:50:35.741 | INFO | "huge_shared" : " 27 GB", | 20260506 21:50:35.741 | INFO | "malloc" : " 14 GB", | 20260506 21:50:35.741 | INFO | "pinned" : " 29 GB" | 20260506 21:50:35.741 | INFO | } | 20260506 21:50:35.741 | INFO | Perf report for the last 5.142 seconds: | 20260506 21:50:35.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:50:35.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:35.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:35.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:35.741 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 21:50:35.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:35.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:35.741 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:50:35.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:35.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:35.742 | INFO | Uploaded 36GiB to GPU in 1.93816s (18.5743GiB/s) | 20260506 21:50:35.742 | INFO | Downloaded 34GiB to GPU in 1.86921s (18.1895GiB/s) | 20260506 21:50:40.887 | INFO | Analyzing chunk frames = [4716544,4717056) | 20260506 21:50:40.887 | INFO | MemInfo: { | 20260506 21:50:40.887 | INFO | "gpu" : " 17 GB", | 20260506 21:50:40.888 | INFO | "huge" : "", | 20260506 21:50:40.888 | INFO | "huge_shared" : " 27 GB", | 20260506 21:50:40.888 | INFO | "malloc" : " 14 GB", | 20260506 21:50:40.888 | INFO | "pinned" : " 29 GB" | 20260506 21:50:40.888 | INFO | } | 20260506 21:50:40.888 | INFO | Perf report for the last 5.14631 seconds: | 20260506 21:50:40.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:50:40.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:40.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:40.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:40.888 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260506 21:50:40.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:40.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:40.888 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:50:40.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:40.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:40.888 | INFO | Uploaded 36GiB to GPU in 2.01015s (17.9091GiB/s) | 20260506 21:50:40.888 | INFO | Downloaded 34GiB to GPU in 1.91526s (17.7521GiB/s) | 20260506 21:50:46.028 | INFO | Analyzing chunk frames = [4717056,4717568) | 20260506 21:50:46.028 | INFO | MemInfo: { | 20260506 21:50:46.028 | INFO | "gpu" : " 17 GB", | 20260506 21:50:46.028 | INFO | "huge" : "", | 20260506 21:50:46.028 | INFO | "huge_shared" : " 27 GB", | 20260506 21:50:46.028 | INFO | "malloc" : " 14 GB", | 20260506 21:50:46.028 | INFO | "pinned" : " 29 GB" | 20260506 21:50:46.028 | INFO | } | 20260506 21:50:46.028 | INFO | Perf report for the last 5.14038 seconds: | 20260506 21:50:46.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:50:46.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:46.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:46.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:46.028 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 21:50:46.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:46.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:46.028 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:50:46.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:46.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:46.028 | INFO | Uploaded 36GiB to GPU in 2.02062s (17.8164GiB/s) | 20260506 21:50:46.028 | INFO | Downloaded 34GiB to GPU in 1.94992s (17.4367GiB/s) | 20260506 21:50:51.168 | INFO | Analyzing chunk frames = [4717568,4718080) | 20260506 21:50:51.168 | INFO | MemInfo: { | 20260506 21:50:51.168 | INFO | "gpu" : " 17 GB", | 20260506 21:50:51.168 | INFO | "huge" : "", | 20260506 21:50:51.168 | INFO | "huge_shared" : " 27 GB", | 20260506 21:50:51.168 | INFO | "malloc" : " 14 GB", | 20260506 21:50:51.168 | INFO | "pinned" : " 29 GB" | 20260506 21:50:51.168 | INFO | } | 20260506 21:50:51.168 | INFO | Perf report for the last 5.1398 seconds: | 20260506 21:50:51.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:50:51.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:51.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:51.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:51.168 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:50:51.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:51.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:51.168 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:50:51.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:51.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:51.168 | INFO | Uploaded 36GiB to GPU in 2.00004s (17.9996GiB/s) | 20260506 21:50:51.168 | INFO | Downloaded 34GiB to GPU in 1.98667s (17.1141GiB/s) | 20260506 21:50:56.307 | INFO | Analyzing chunk frames = [4718080,4718592) | 20260506 21:50:56.307 | INFO | MemInfo: { | 20260506 21:50:56.307 | INFO | "gpu" : " 17 GB", | 20260506 21:50:56.308 | INFO | "huge" : "", | 20260506 21:50:56.308 | INFO | "huge_shared" : " 27 GB", | 20260506 21:50:56.308 | INFO | "malloc" : " 14 GB", | 20260506 21:50:56.308 | INFO | "pinned" : " 29 GB" | 20260506 21:50:56.308 | INFO | } | 20260506 21:50:56.308 | INFO | Perf report for the last 5.13993 seconds: | 20260506 21:50:56.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:50:56.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:56.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:56.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:56.308 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 21:50:56.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:56.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:56.308 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260506 21:50:56.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:56.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:50:56.308 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:50:56.308 | INFO | Total mem usage: 18 GiB | 20260506 21:50:56.308 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:50:56.308 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.7%) | 20260506 21:50:56.308 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:50:56.308 | INFO | High Water Breakdown: | 20260506 21:50:56.308 | INFO | 61.5% : 9.57 GiB ( 83.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:50:56.308 | INFO | 38.4% : 5.98 GiB ( 89.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:50:56.308 | INFO | 0.128% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:50:56.308 | INFO | ------------------------------------------ | 20260506 21:50:56.308 | INFO | Uploaded 36GiB to GPU in 2.0073s (17.9345GiB/s) | 20260506 21:50:56.308 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:50:56.308 | INFO | Downloaded 34GiB to GPU in 1.99339s (17.0564GiB/s) | 20260506 21:50:56.308 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:50:56.308 | INFO | Total mem usage: 26.5 GiB | 20260506 21:50:56.309 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:50:56.309 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:50:56.309 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:50:56.309 | INFO | High Water Breakdown: | 20260506 21:50:56.309 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:50:56.309 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:50:56.309 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:50:56.309 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:50:56.309 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:50:56.309 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:50:56.309 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:50:56.309 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:50:56.309 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:50:56.309 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:50:56.309 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:50:56.309 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:50:56.309 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:50:56.309 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:50:56.309 | INFO | ------------------------------------------ | 20260506 21:50:56.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:50:56.309 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:50:56.309 | INFO | Total mem usage: 25.6 GiB | 20260506 21:50:56.309 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:50:56.309 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:50:56.309 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:50:56.309 | INFO | High Water Breakdown: | 20260506 21:50:56.309 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:50:56.309 | INFO | ------------------------------------------ | 20260506 21:50:56.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:50:56.309 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:50:56.309 | INFO | Total mem usage: 15.7 GiB | 20260506 21:50:56.309 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:50:56.309 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:50:56.309 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:50:56.309 | INFO | High Water Breakdown: | 20260506 21:50:56.309 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:50:56.309 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:50:56.309 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:50:56.309 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:50:56.309 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:50:56.309 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:50:56.309 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:50:56.309 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:50:56.309 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:50:56.309 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:50:56.309 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:50:56.309 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:50:56.309 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:50:56.309 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:50:56.309 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:50:56.309 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:50:56.309 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:50:56.309 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:50:56.309 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:50:56.309 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:50:56.309 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:50:56.309 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:50:56.309 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:50:56.309 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:50:56.309 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:50:56.309 | INFO | ------------------------------------------ | 20260506 21:50:56.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:50:56.309 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:50:56.309 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:50:56.309 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:50:56.309 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.57e+01, 1.35e+01 / 2.45e+00 | 20260506 21:50:56.309 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 9.69e-03 / 5.75e-03 | 20260506 21:50:56.309 | INFO | PerfReport: 11.65%, Baseline -- 3.63e+00 / 4.43e+00, 5.55e-01 / 3.30e-01 | 20260506 21:50:56.309 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.49e+00, 1.84e-01 / 1.26e-01 | 20260506 21:50:56.309 | INFO | PerfReport: 16.07%, DME -- 4.00e+01 / 4.48e+01, 4.27e+00 / 2.57e+00 | 20260506 21:50:56.309 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.39e+00 / 1.76e+00 | 20260506 21:50:56.309 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.21e+00 / 2.74e+00, 3.85e-01 / 2.38e-01 | 20260506 21:50:56.309 | INFO | PerfReport: 3.25%, Metrics -- 1.01e+00 / 1.37e+00, 2.69e-01 / 2.03e-01 | 20260506 21:50:56.309 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.46e-01 / 3.93e-01, 5.37e-02 / 2.37e-02 | 20260506 21:50:56.310 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:50:56.310 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:50:56.310 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:50:56.310 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 21:50:56.310 | INFO | PerfReport: 21.59%, Upload -- 2.09e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260506 21:50:56.310 | INFO | PerfReport: 21.22%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 21:50:56.310 | INFO | PerfReport: 32.19%, Compute -- 3.11e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 21:50:56.310 | INFO | PerfReport: 1.35%, LockUpload -- 1.30e+00 / 0.00e+00, 5.00e+00 / 0.00e+00 | 20260506 21:50:56.310 | INFO | PerfReport: 1.13%, LockDownload -- 1.10e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260506 21:50:56.310 | INFO | PerfReport: 22.50%, LockCompute -- 2.17e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 21:50:56.310 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260506 21:50:56.425 | INFO | Starting a round of Baz IO | 20260506 21:50:59.546 | INFO | Populating RTMetrics report | 20260506 21:50:59.546 | INFO | RTMetrics report populated | 20260506 21:50:59.546 | INFO | Opening tmp RTMetrics json file | 20260506 21:50:59.628 | INFO | Serializing RTMetrics to json | 20260506 21:50:59.633 | INFO | Serialization done, swapping tmp file now... | 20260506 21:50:59.636 | INFO | RTMetrics done | 20260506 21:50:59.818 | INFO | Finished a round of Baz IO: Wrote 1.932 GiB in 3.393 seconds | 20260506 21:50:59.818 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.83% / 0.00% / 66.17% | 20260506 21:50:59.818 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.564738% / 0% / 8.34465e-05% | 20260506 21:50:59.818 | INFO | Baz IO overall: | 20260506 21:50:59.818 | INFO | BytesWritten : 2483531317404 | 20260506 21:50:59.818 | INFO | NumEvents : 1198789408911 | 20260506 21:51:01.439 | INFO | Analyzing chunk frames = [4718592,4719104) | 20260506 21:51:01.439 | INFO | MemInfo: { | 20260506 21:51:01.439 | INFO | "gpu" : " 17 GB", | 20260506 21:51:01.439 | INFO | "huge" : "", | 20260506 21:51:01.439 | INFO | "huge_shared" : " 27 GB", | 20260506 21:51:01.439 | INFO | "malloc" : " 14 GB", | 20260506 21:51:01.439 | INFO | "pinned" : " 29 GB" | 20260506 21:51:01.439 | INFO | } | 20260506 21:51:01.439 | INFO | Perf report for the last 5.13116 seconds: | 20260506 21:51:01.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:51:01.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:01.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:01.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:01.439 | INFO | ANALYSIS : 62.5%, 2.9, 95.4ms, 0 | 20260506 21:51:01.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:01.439 | INFO | RT_METRICS : 44.6%, 1.0, 23.3ms, 0 | 20260506 21:51:01.439 | INFO | PRE_HQ : 60.1%, 1.8, 57.5ms, 0 | 20260506 21:51:01.439 | INFO | BAZWRITER : 66.1%, 4.7, 160.8ms, 0 | 20260506 21:51:01.439 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 21:51:01.439 | INFO | Uploaded 36GiB to GPU in 2.04616s (17.5939GiB/s) | 20260506 21:51:01.439 | INFO | Downloaded 38GiB to GPU in 2.20936s (17.1996GiB/s) | 20260506 21:51:06.577 | INFO | Analyzing chunk frames = [4719104,4719616) | 20260506 21:51:06.577 | INFO | MemInfo: { | 20260506 21:51:06.577 | INFO | "gpu" : " 17 GB", | 20260506 21:51:06.577 | INFO | "huge" : "", | 20260506 21:51:06.577 | INFO | "huge_shared" : " 27 GB", | 20260506 21:51:06.577 | INFO | "malloc" : " 14 GB", | 20260506 21:51:06.577 | INFO | "pinned" : " 29 GB" | 20260506 21:51:06.577 | INFO | } | 20260506 21:51:06.577 | INFO | Perf report for the last 5.13808 seconds: | 20260506 21:51:06.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:51:06.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:06.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:06.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:06.577 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:51:06.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:06.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:06.577 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:51:06.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:06.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:06.577 | INFO | Uploaded 36GiB to GPU in 2.01884s (17.832GiB/s) | 20260506 21:51:06.577 | INFO | Downloaded 34GiB to GPU in 1.92621s (17.6512GiB/s) | 20260506 21:51:11.719 | INFO | Analyzing chunk frames = [4719616,4720128) | 20260506 21:51:11.719 | INFO | MemInfo: { | 20260506 21:51:11.719 | INFO | "gpu" : " 17 GB", | 20260506 21:51:11.719 | INFO | "huge" : "", | 20260506 21:51:11.719 | INFO | "huge_shared" : " 27 GB", | 20260506 21:51:11.719 | INFO | "malloc" : " 14 GB", | 20260506 21:51:11.719 | INFO | "pinned" : " 29 GB" | 20260506 21:51:11.719 | INFO | } | 20260506 21:51:11.719 | INFO | Perf report for the last 5.14215 seconds: | 20260506 21:51:11.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:51:11.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:11.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:11.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:11.719 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 21:51:11.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:11.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:11.719 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:51:11.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:11.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:11.719 | INFO | Uploaded 36GiB to GPU in 1.99452s (18.0495GiB/s) | 20260506 21:51:11.719 | INFO | Downloaded 34GiB to GPU in 1.91569s (17.7482GiB/s) | 20260506 21:51:16.860 | INFO | Analyzing chunk frames = [4720128,4720640) | 20260506 21:51:16.860 | INFO | MemInfo: { | 20260506 21:51:16.860 | INFO | "gpu" : " 17 GB", | 20260506 21:51:16.860 | INFO | "huge" : "", | 20260506 21:51:16.861 | INFO | "huge_shared" : " 27 GB", | 20260506 21:51:16.861 | INFO | "malloc" : " 14 GB", | 20260506 21:51:16.861 | INFO | "pinned" : " 29 GB" | 20260506 21:51:16.861 | INFO | } | 20260506 21:51:16.861 | INFO | Perf report for the last 5.14135 seconds: | 20260506 21:51:16.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:51:16.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:16.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:16.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:16.861 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 21:51:16.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:16.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:16.861 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:51:16.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:16.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:16.861 | INFO | Uploaded 36GiB to GPU in 1.94743s (18.4859GiB/s) | 20260506 21:51:16.861 | INFO | Downloaded 34GiB to GPU in 1.87233s (18.1592GiB/s) | 20260506 21:51:21.999 | INFO | Analyzing chunk frames = [4720640,4721152) | 20260506 21:51:21.999 | INFO | MemInfo: { | 20260506 21:51:21.999 | INFO | "gpu" : " 17 GB", | 20260506 21:51:21.999 | INFO | "huge" : "", | 20260506 21:51:21.999 | INFO | "huge_shared" : " 27 GB", | 20260506 21:51:21.999 | INFO | "malloc" : " 14 GB", | 20260506 21:51:21.999 | INFO | "pinned" : " 29 GB" | 20260506 21:51:21.999 | INFO | } | 20260506 21:51:21.999 | INFO | Perf report for the last 5.13851 seconds: | 20260506 21:51:21.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:51:21.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:21.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:21.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:21.999 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 21:51:21.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:21.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:21.999 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 21:51:21.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:21.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:21.999 | INFO | Uploaded 36GiB to GPU in 2.01127s (17.8992GiB/s) | 20260506 21:51:21.999 | INFO | Downloaded 34GiB to GPU in 1.91455s (17.7588GiB/s) | 20260506 21:51:27.139 | INFO | Analyzing chunk frames = [4721152,4721664) | 20260506 21:51:27.139 | INFO | MemInfo: { | 20260506 21:51:27.139 | INFO | "gpu" : " 17 GB", | 20260506 21:51:27.139 | INFO | "huge" : "", | 20260506 21:51:27.140 | INFO | "huge_shared" : " 27 GB", | 20260506 21:51:27.140 | INFO | "malloc" : " 14 GB", | 20260506 21:51:27.140 | INFO | "pinned" : " 29 GB" | 20260506 21:51:27.140 | INFO | } | 20260506 21:51:27.140 | INFO | Perf report for the last 5.14047 seconds: | 20260506 21:51:27.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:51:27.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:27.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:27.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:27.140 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 21:51:27.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:27.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:27.140 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 21:51:27.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:27.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:27.140 | INFO | Uploaded 36GiB to GPU in 2.00645s (17.9422GiB/s) | 20260506 21:51:27.140 | INFO | Downloaded 34GiB to GPU in 1.94477s (17.4828GiB/s) | 20260506 21:51:32.280 | INFO | Analyzing chunk frames = [4721664,4722176) | 20260506 21:51:32.280 | INFO | MemInfo: { | 20260506 21:51:32.280 | INFO | "gpu" : " 17 GB", | 20260506 21:51:32.280 | INFO | "huge" : "", | 20260506 21:51:32.280 | INFO | "huge_shared" : " 27 GB", | 20260506 21:51:32.280 | INFO | "malloc" : " 14 GB", | 20260506 21:51:32.280 | INFO | "pinned" : " 29 GB" | 20260506 21:51:32.280 | INFO | } | 20260506 21:51:32.280 | INFO | Perf report for the last 5.14012 seconds: | 20260506 21:51:32.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:51:32.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:32.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:32.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:32.280 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:51:32.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:32.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:32.280 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 21:51:32.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:32.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:32.280 | INFO | Uploaded 36GiB to GPU in 1.95037s (18.458GiB/s) | 20260506 21:51:32.280 | INFO | Downloaded 34GiB to GPU in 1.97719s (17.1961GiB/s) | 20260506 21:51:37.415 | INFO | Analyzing chunk frames = [4722176,4722688) | 20260506 21:51:37.415 | INFO | MemInfo: { | 20260506 21:51:37.415 | INFO | "gpu" : " 17 GB", | 20260506 21:51:37.415 | INFO | "huge" : "", | 20260506 21:51:37.415 | INFO | "huge_shared" : " 27 GB", | 20260506 21:51:37.415 | INFO | "malloc" : " 14 GB", | 20260506 21:51:37.415 | INFO | "pinned" : " 29 GB" | 20260506 21:51:37.415 | INFO | } | 20260506 21:51:37.415 | INFO | Perf report for the last 5.13535 seconds: | 20260506 21:51:37.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:51:37.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:37.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:37.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:37.415 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 21:51:37.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:37.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:37.415 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 21:51:37.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:37.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:37.415 | INFO | Uploaded 36GiB to GPU in 2.03003s (17.7337GiB/s) | 20260506 21:51:37.415 | INFO | Downloaded 34GiB to GPU in 1.98581s (17.1214GiB/s) | 20260506 21:51:37.515 | INFO | Starting a round of Baz IO | 20260506 21:51:40.645 | INFO | Populating RTMetrics report | 20260506 21:51:40.645 | INFO | RTMetrics report populated | 20260506 21:51:40.645 | INFO | Opening tmp RTMetrics json file | 20260506 21:51:40.719 | INFO | Serializing RTMetrics to json | 20260506 21:51:40.723 | INFO | Serialization done, swapping tmp file now... | 20260506 21:51:40.729 | INFO | RTMetrics done | 20260506 21:51:41.074 | INFO | Finished a round of Baz IO: Wrote 1.932 GiB in 3.558 seconds | 20260506 21:51:41.074 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.82% / 0.00% / 66.17% | 20260506 21:51:41.074 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.564818% / 3.97364e-06% / 6.75519e-05% | 20260506 21:51:41.074 | INFO | Baz IO overall: | 20260506 21:51:41.074 | INFO | BytesWritten : 2485606319382 | 20260506 21:51:41.074 | INFO | NumEvents : 1199781538797 | 20260506 21:51:42.557 | INFO | Analyzing chunk frames = [4722688,4723200) | 20260506 21:51:42.557 | INFO | MemInfo: { | 20260506 21:51:42.557 | INFO | "gpu" : " 17 GB", | 20260506 21:51:42.557 | INFO | "huge" : "", | 20260506 21:51:42.557 | INFO | "huge_shared" : " 27 GB", | 20260506 21:51:42.557 | INFO | "malloc" : " 14 GB", | 20260506 21:51:42.557 | INFO | "pinned" : " 29 GB" | 20260506 21:51:42.557 | INFO | } | 20260506 21:51:42.557 | INFO | Perf report for the last 5.14193 seconds: | 20260506 21:51:42.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:51:42.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:42.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:42.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:42.557 | INFO | ANALYSIS : 62.2%, 2.9, 95.0ms, 0 | 20260506 21:51:42.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:42.557 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 21:51:42.557 | INFO | PRE_HQ : 59.8%, 1.8, 56.8ms, 0 | 20260506 21:51:42.557 | INFO | BAZWRITER : 68.8%, 4.8, 169.9ms, 0 | 20260506 21:51:42.557 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 21:51:42.557 | INFO | Uploaded 36GiB to GPU in 2.04152s (17.6339GiB/s) | 20260506 21:51:42.557 | INFO | Downloaded 38GiB to GPU in 2.19482s (17.3135GiB/s) | 20260506 21:51:47.693 | INFO | Analyzing chunk frames = [4723200,4723712) | 20260506 21:51:47.693 | INFO | MemInfo: { | 20260506 21:51:47.693 | INFO | "gpu" : " 17 GB", | 20260506 21:51:47.693 | INFO | "huge" : "", | 20260506 21:51:47.693 | INFO | "huge_shared" : " 27 GB", | 20260506 21:51:47.693 | INFO | "malloc" : " 14 GB", | 20260506 21:51:47.693 | INFO | "pinned" : " 29 GB" | 20260506 21:51:47.693 | INFO | } | 20260506 21:51:47.693 | INFO | Perf report for the last 5.13595 seconds: | 20260506 21:51:47.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:51:47.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:47.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:47.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:47.693 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 21:51:47.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:47.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:47.693 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:51:47.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:47.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:47.693 | INFO | Uploaded 36GiB to GPU in 2.00014s (17.9987GiB/s) | 20260506 21:51:47.693 | INFO | Downloaded 34GiB to GPU in 1.92261s (17.6843GiB/s) | 20260506 21:51:52.830 | INFO | Analyzing chunk frames = [4723712,4724224) | 20260506 21:51:52.830 | INFO | MemInfo: { | 20260506 21:51:52.830 | INFO | "gpu" : " 17 GB", | 20260506 21:51:52.830 | INFO | "huge" : "", | 20260506 21:51:52.830 | INFO | "huge_shared" : " 27 GB", | 20260506 21:51:52.830 | INFO | "malloc" : " 14 GB", | 20260506 21:51:52.830 | INFO | "pinned" : " 29 GB" | 20260506 21:51:52.830 | INFO | } | 20260506 21:51:52.830 | INFO | Perf report for the last 5.13728 seconds: | 20260506 21:51:52.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:51:52.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:52.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:52.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:52.830 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 21:51:52.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:52.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:52.830 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:51:52.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:52.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:52.830 | INFO | Uploaded 36GiB to GPU in 1.97996s (18.1822GiB/s) | 20260506 21:51:52.831 | INFO | Downloaded 34GiB to GPU in 1.91705s (17.7356GiB/s) | 20260506 21:51:57.979 | INFO | Analyzing chunk frames = [4724224,4724736) | 20260506 21:51:57.979 | INFO | MemInfo: { | 20260506 21:51:57.979 | INFO | "gpu" : " 17 GB", | 20260506 21:51:57.979 | INFO | "huge" : "", | 20260506 21:51:57.979 | INFO | "huge_shared" : " 27 GB", | 20260506 21:51:57.979 | INFO | "malloc" : " 14 GB", | 20260506 21:51:57.979 | INFO | "pinned" : " 29 GB" | 20260506 21:51:57.979 | INFO | } | 20260506 21:51:57.979 | INFO | Perf report for the last 5.14859 seconds: | 20260506 21:51:57.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:51:57.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:57.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:57.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:57.979 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260506 21:51:57.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:57.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:57.979 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:51:57.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:57.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:51:57.979 | INFO | Uploaded 36GiB to GPU in 1.94493s (18.5097GiB/s) | 20260506 21:51:57.979 | INFO | Downloaded 34GiB to GPU in 1.87047s (18.1772GiB/s) | 20260506 21:52:03.116 | INFO | Analyzing chunk frames = [4724736,4725248) | 20260506 21:52:03.116 | INFO | MemInfo: { | 20260506 21:52:03.116 | INFO | "gpu" : " 17 GB", | 20260506 21:52:03.116 | INFO | "huge" : "", | 20260506 21:52:03.116 | INFO | "huge_shared" : " 27 GB", | 20260506 21:52:03.116 | INFO | "malloc" : " 14 GB", | 20260506 21:52:03.116 | INFO | "pinned" : " 29 GB" | 20260506 21:52:03.116 | INFO | } | 20260506 21:52:03.116 | INFO | Perf report for the last 5.13729 seconds: | 20260506 21:52:03.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:52:03.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:03.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:03.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:03.116 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 21:52:03.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:03.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:03.116 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 21:52:03.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:03.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:03.116 | INFO | Uploaded 36GiB to GPU in 2.00749s (17.9329GiB/s) | 20260506 21:52:03.116 | INFO | Downloaded 34GiB to GPU in 1.92255s (17.6849GiB/s) | 20260506 21:52:08.255 | INFO | Analyzing chunk frames = [4725248,4725760) | 20260506 21:52:08.255 | INFO | MemInfo: { | 20260506 21:52:08.255 | INFO | "gpu" : " 17 GB", | 20260506 21:52:08.255 | INFO | "huge" : "", | 20260506 21:52:08.255 | INFO | "huge_shared" : " 27 GB", | 20260506 21:52:08.255 | INFO | "malloc" : " 14 GB", | 20260506 21:52:08.255 | INFO | "pinned" : " 29 GB" | 20260506 21:52:08.255 | INFO | } | 20260506 21:52:08.255 | INFO | Perf report for the last 5.13878 seconds: | 20260506 21:52:08.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:52:08.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:08.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:08.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:08.255 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 21:52:08.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:08.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:08.255 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 21:52:08.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:08.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:08.255 | INFO | Uploaded 36GiB to GPU in 2.03542s (17.6868GiB/s) | 20260506 21:52:08.255 | INFO | Downloaded 34GiB to GPU in 1.93982s (17.5274GiB/s) | 20260506 21:52:13.394 | INFO | Analyzing chunk frames = [4725760,4726272) | 20260506 21:52:13.394 | INFO | MemInfo: { | 20260506 21:52:13.394 | INFO | "gpu" : " 17 GB", | 20260506 21:52:13.394 | INFO | "huge" : "", | 20260506 21:52:13.394 | INFO | "huge_shared" : " 27 GB", | 20260506 21:52:13.394 | INFO | "malloc" : " 14 GB", | 20260506 21:52:13.394 | INFO | "pinned" : " 29 GB" | 20260506 21:52:13.394 | INFO | } | 20260506 21:52:13.394 | INFO | Perf report for the last 5.13909 seconds: | 20260506 21:52:13.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:52:13.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:13.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:13.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:13.394 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:52:13.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:13.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:13.394 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 21:52:13.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:13.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:13.395 | INFO | Uploaded 36GiB to GPU in 1.9805s (18.1772GiB/s) | 20260506 21:52:13.395 | INFO | Downloaded 34GiB to GPU in 2.00231s (16.9804GiB/s) | 20260506 21:52:18.531 | INFO | Analyzing chunk frames = [4726272,4726784) | 20260506 21:52:18.531 | INFO | MemInfo: { | 20260506 21:52:18.531 | INFO | "gpu" : " 17 GB", | 20260506 21:52:18.531 | INFO | "huge" : "", | 20260506 21:52:18.531 | INFO | "huge_shared" : " 27 GB", | 20260506 21:52:18.531 | INFO | "malloc" : " 14 GB", | 20260506 21:52:18.531 | INFO | "pinned" : " 29 GB" | 20260506 21:52:18.531 | INFO | } | 20260506 21:52:18.531 | INFO | Perf report for the last 5.13684 seconds: | 20260506 21:52:18.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:52:18.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:18.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:18.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:18.531 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:52:18.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:18.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:18.531 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 21:52:18.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:18.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:18.531 | INFO | Uploaded 36GiB to GPU in 2.0528s (17.537GiB/s) | 20260506 21:52:18.531 | INFO | Downloaded 34GiB to GPU in 2.00217s (16.9816GiB/s) | 20260506 21:52:18.531 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:52:18.531 | INFO | Total mem usage: 18 GiB | 20260506 21:52:18.531 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:52:18.531 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.5%) | 20260506 21:52:18.531 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:52:18.531 | INFO | High Water Breakdown: | 20260506 21:52:18.531 | INFO | 61.4% : 9.48 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:52:18.531 | INFO | 38.5% : 5.95 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:52:18.531 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:52:18.531 | INFO | ------------------------------------------ | 20260506 21:52:18.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:52:18.532 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:52:18.532 | INFO | Total mem usage: 26.5 GiB | 20260506 21:52:18.532 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:52:18.532 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:52:18.532 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:52:18.532 | INFO | High Water Breakdown: | 20260506 21:52:18.532 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:52:18.532 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:52:18.532 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:52:18.532 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:52:18.532 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:52:18.532 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:52:18.532 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:52:18.532 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:52:18.532 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:52:18.532 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:52:18.532 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:52:18.532 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:52:18.532 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:52:18.532 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:52:18.532 | INFO | ------------------------------------------ | 20260506 21:52:18.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:52:18.532 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:52:18.532 | INFO | Total mem usage: 25.6 GiB | 20260506 21:52:18.532 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:52:18.532 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:52:18.532 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:52:18.532 | INFO | High Water Breakdown: | 20260506 21:52:18.532 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:52:18.532 | INFO | ------------------------------------------ | 20260506 21:52:18.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:52:18.532 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:52:18.532 | INFO | Total mem usage: 15.7 GiB | 20260506 21:52:18.532 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:52:18.532 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:52:18.532 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:52:18.532 | INFO | High Water Breakdown: | 20260506 21:52:18.532 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:52:18.532 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:52:18.532 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:52:18.532 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:52:18.532 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:52:18.532 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:52:18.532 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:52:18.532 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:52:18.532 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:52:18.532 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:52:18.532 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:52:18.532 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:52:18.532 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:52:18.532 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:52:18.532 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:52:18.532 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:52:18.532 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:52:18.532 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:52:18.532 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:52:18.532 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:52:18.532 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:52:18.532 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:52:18.532 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:52:18.532 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:52:18.532 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:52:18.532 | INFO | ------------------------------------------ | 20260506 21:52:18.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:52:18.532 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:52:18.532 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:52:18.532 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:52:18.532 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.59e+01, 1.36e+01 / 2.49e+00 | 20260506 21:52:18.532 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 1.10e-02 / 5.84e-03 | 20260506 21:52:18.532 | INFO | PerfReport: 11.67%, Baseline -- 3.63e+00 / 4.42e+00, 5.55e-01 / 3.59e-01 | 20260506 21:52:18.532 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.48e+00, 1.83e-01 / 1.28e-01 | 20260506 21:52:18.532 | INFO | PerfReport: 16.06%, DME -- 3.99e+01 / 4.48e+01, 4.18e+00 / 2.61e+00 | 20260506 21:52:18.532 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.37e+00 / 1.72e+00 | 20260506 21:52:18.532 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.20e+00 / 2.77e+00, 3.75e-01 / 1.77e-01 | 20260506 21:52:18.532 | INFO | PerfReport: 3.25%, Metrics -- 1.01e+00 / 1.38e+00, 2.70e-01 / 1.78e-01 | 20260506 21:52:18.532 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.46e-01 / 3.97e-01, 5.58e-02 / 2.35e-02 | 20260506 21:52:18.532 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:52:18.532 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:52:18.532 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:52:18.532 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 21:52:18.532 | INFO | PerfReport: 21.60%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 21:52:18.532 | INFO | PerfReport: 21.25%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 21:52:18.532 | INFO | PerfReport: 32.17%, Compute -- 3.11e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 21:52:18.532 | INFO | PerfReport: 1.31%, LockUpload -- 1.27e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260506 21:52:18.532 | INFO | PerfReport: 1.17%, LockDownload -- 1.13e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260506 21:52:18.532 | INFO | PerfReport: 22.48%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:52:18.532 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260506 21:52:18.646 | INFO | Starting a round of Baz IO | 20260506 21:52:21.768 | INFO | Populating RTMetrics report | 20260506 21:52:21.768 | INFO | RTMetrics report populated | 20260506 21:52:21.768 | INFO | Opening tmp RTMetrics json file | 20260506 21:52:21.861 | INFO | Serializing RTMetrics to json | 20260506 21:52:21.892 | INFO | Serialization done, swapping tmp file now... | 20260506 21:52:21.898 | INFO | RTMetrics done | 20260506 21:52:22.291 | INFO | Finished a round of Baz IO: Wrote 1.931 GiB in 3.645 seconds | 20260506 21:52:22.291 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.82% / 0.00% / 66.18% | 20260506 21:52:22.291 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.563939% / 0% / 7.94729e-05% | 20260506 21:52:22.291 | INFO | Baz IO overall: | 20260506 21:52:22.291 | INFO | BytesWritten : 2487679568465 | 20260506 21:52:22.291 | INFO | NumEvents : 1200772656602 | 20260506 21:52:23.670 | INFO | Analyzing chunk frames = [4726784,4727296) | 20260506 21:52:23.670 | INFO | MemInfo: { | 20260506 21:52:23.670 | INFO | "gpu" : " 17 GB", | 20260506 21:52:23.670 | INFO | "huge" : "", | 20260506 21:52:23.670 | INFO | "huge_shared" : " 27 GB", | 20260506 21:52:23.670 | INFO | "malloc" : " 14 GB", | 20260506 21:52:23.670 | INFO | "pinned" : " 29 GB" | 20260506 21:52:23.670 | INFO | } | 20260506 21:52:23.670 | INFO | Perf report for the last 5.13882 seconds: | 20260506 21:52:23.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:52:23.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:23.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:23.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:23.670 | INFO | ANALYSIS : 62.4%, 2.9, 95.5ms, 0 | 20260506 21:52:23.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:23.670 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260506 21:52:23.670 | INFO | PRE_HQ : 59.7%, 1.8, 56.5ms, 0 | 20260506 21:52:23.670 | INFO | BAZWRITER : 70.6%, 4.8, 175.2ms, 0 | 20260506 21:52:23.670 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.2ms, 0 | 20260506 21:52:23.670 | INFO | Uploaded 36GiB to GPU in 2.07989s (17.3086GiB/s) | 20260506 21:52:23.670 | INFO | Downloaded 38GiB to GPU in 2.21305s (17.1709GiB/s) | 20260506 21:52:28.810 | INFO | Analyzing chunk frames = [4727296,4727808) | 20260506 21:52:28.811 | INFO | MemInfo: { | 20260506 21:52:28.811 | INFO | "gpu" : " 17 GB", | 20260506 21:52:28.811 | INFO | "huge" : "", | 20260506 21:52:28.811 | INFO | "huge_shared" : " 27 GB", | 20260506 21:52:28.811 | INFO | "malloc" : " 14 GB", | 20260506 21:52:28.811 | INFO | "pinned" : " 29 GB" | 20260506 21:52:28.811 | INFO | } | 20260506 21:52:28.811 | INFO | Perf report for the last 5.14088 seconds: | 20260506 21:52:28.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:52:28.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:28.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:28.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:28.811 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:52:28.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:28.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:28.811 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 21:52:28.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:28.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:28.811 | INFO | Uploaded 36GiB to GPU in 2.01961s (17.8252GiB/s) | 20260506 21:52:28.811 | INFO | Downloaded 34GiB to GPU in 1.9275s (17.6395GiB/s) | 20260506 21:52:33.947 | INFO | Analyzing chunk frames = [4727808,4728320) | 20260506 21:52:33.947 | INFO | MemInfo: { | 20260506 21:52:33.947 | INFO | "gpu" : " 17 GB", | 20260506 21:52:33.947 | INFO | "huge" : "", | 20260506 21:52:33.947 | INFO | "huge_shared" : " 27 GB", | 20260506 21:52:33.947 | INFO | "malloc" : " 14 GB", | 20260506 21:52:33.947 | INFO | "pinned" : " 29 GB" | 20260506 21:52:33.947 | INFO | } | 20260506 21:52:33.947 | INFO | Perf report for the last 5.13646 seconds: | 20260506 21:52:33.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:52:33.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:33.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:33.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:33.947 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 21:52:33.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:33.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:33.947 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 21:52:33.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:33.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:33.948 | INFO | Uploaded 36GiB to GPU in 1.98083s (18.1742GiB/s) | 20260506 21:52:33.948 | INFO | Downloaded 34GiB to GPU in 1.91959s (17.7121GiB/s) | 20260506 21:52:39.086 | INFO | Analyzing chunk frames = [4728320,4728832) | 20260506 21:52:39.086 | INFO | MemInfo: { | 20260506 21:52:39.086 | INFO | "gpu" : " 17 GB", | 20260506 21:52:39.086 | INFO | "huge" : "", | 20260506 21:52:39.086 | INFO | "huge_shared" : " 27 GB", | 20260506 21:52:39.086 | INFO | "malloc" : " 14 GB", | 20260506 21:52:39.087 | INFO | "pinned" : " 29 GB" | 20260506 21:52:39.087 | INFO | } | 20260506 21:52:39.087 | INFO | Perf report for the last 5.13945 seconds: | 20260506 21:52:39.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:52:39.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:39.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:39.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:39.087 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:52:39.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:39.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:39.087 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:52:39.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:39.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:39.087 | INFO | Uploaded 36GiB to GPU in 1.96171s (18.3514GiB/s) | 20260506 21:52:39.087 | INFO | Downloaded 34GiB to GPU in 1.8752s (18.1314GiB/s) | 20260506 21:52:44.230 | INFO | Analyzing chunk frames = [4728832,4729344) | 20260506 21:52:44.230 | INFO | MemInfo: { | 20260506 21:52:44.230 | INFO | "gpu" : " 17 GB", | 20260506 21:52:44.230 | INFO | "huge" : "", | 20260506 21:52:44.230 | INFO | "huge_shared" : " 27 GB", | 20260506 21:52:44.230 | INFO | "malloc" : " 14 GB", | 20260506 21:52:44.231 | INFO | "pinned" : " 29 GB" | 20260506 21:52:44.231 | INFO | } | 20260506 21:52:44.231 | INFO | Perf report for the last 5.14401 seconds: | 20260506 21:52:44.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:52:44.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:44.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:44.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:44.231 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 21:52:44.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:44.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:44.231 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:52:44.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:44.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:44.231 | INFO | Uploaded 36GiB to GPU in 2.02054s (17.817GiB/s) | 20260506 21:52:44.231 | INFO | Downloaded 34GiB to GPU in 1.9124s (17.7787GiB/s) | 20260506 21:52:49.373 | INFO | Analyzing chunk frames = [4729344,4729856) | 20260506 21:52:49.373 | INFO | MemInfo: { | 20260506 21:52:49.373 | INFO | "gpu" : " 17 GB", | 20260506 21:52:49.373 | INFO | "huge" : "", | 20260506 21:52:49.373 | INFO | "huge_shared" : " 27 GB", | 20260506 21:52:49.373 | INFO | "malloc" : " 14 GB", | 20260506 21:52:49.373 | INFO | "pinned" : " 29 GB" | 20260506 21:52:49.373 | INFO | } | 20260506 21:52:49.373 | INFO | Perf report for the last 5.14275 seconds: | 20260506 21:52:49.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:52:49.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:49.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:49.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:49.373 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 21:52:49.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:49.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:49.374 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 21:52:49.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:49.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:49.374 | INFO | Uploaded 36GiB to GPU in 2.01612s (17.8561GiB/s) | 20260506 21:52:49.374 | INFO | Downloaded 34GiB to GPU in 1.94062s (17.5202GiB/s) | 20260506 21:52:54.509 | INFO | Analyzing chunk frames = [4729856,4730368) | 20260506 21:52:54.509 | INFO | MemInfo: { | 20260506 21:52:54.509 | INFO | "gpu" : " 17 GB", | 20260506 21:52:54.509 | INFO | "huge" : "", | 20260506 21:52:54.509 | INFO | "huge_shared" : " 27 GB", | 20260506 21:52:54.509 | INFO | "malloc" : " 14 GB", | 20260506 21:52:54.509 | INFO | "pinned" : " 29 GB" | 20260506 21:52:54.509 | INFO | } | 20260506 21:52:54.509 | INFO | Perf report for the last 5.13549 seconds: | 20260506 21:52:54.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:52:54.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:54.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:54.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:54.509 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 21:52:54.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:54.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:54.509 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 21:52:54.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:54.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:54.509 | INFO | Uploaded 36GiB to GPU in 1.9997s (18.0027GiB/s) | 20260506 21:52:54.509 | INFO | Downloaded 34GiB to GPU in 2.00247s (16.979GiB/s) | 20260506 21:52:59.646 | INFO | Analyzing chunk frames = [4730368,4730880) | 20260506 21:52:59.646 | INFO | MemInfo: { | 20260506 21:52:59.646 | INFO | "gpu" : " 17 GB", | 20260506 21:52:59.646 | INFO | "huge" : "", | 20260506 21:52:59.646 | INFO | "huge_shared" : " 27 GB", | 20260506 21:52:59.646 | INFO | "malloc" : " 14 GB", | 20260506 21:52:59.646 | INFO | "pinned" : " 29 GB" | 20260506 21:52:59.646 | INFO | } | 20260506 21:52:59.646 | INFO | Perf report for the last 5.13752 seconds: | 20260506 21:52:59.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:52:59.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:59.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:59.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:59.646 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 21:52:59.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:59.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:59.646 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:52:59.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:59.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:52:59.647 | INFO | Uploaded 36GiB to GPU in 1.97562s (18.2222GiB/s) | 20260506 21:52:59.647 | INFO | Downloaded 34GiB to GPU in 1.9833s (17.1432GiB/s) | 20260506 21:52:59.757 | INFO | Starting a round of Baz IO | 20260506 21:53:02.885 | INFO | Populating RTMetrics report | 20260506 21:53:02.885 | INFO | RTMetrics report populated | 20260506 21:53:02.885 | INFO | Opening tmp RTMetrics json file | 20260506 21:53:02.939 | INFO | Serializing RTMetrics to json | 20260506 21:53:03.041 | INFO | Serialization done, swapping tmp file now... | 20260506 21:53:03.044 | INFO | RTMetrics done | 20260506 21:53:03.069 | INFO | Finished a round of Baz IO: Wrote 1.93 GiB in 3.312 seconds | 20260506 21:53:03.069 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.82% / 0.00% / 66.18% | 20260506 21:53:03.069 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.562334% / 0% / 9.13938e-05% | 20260506 21:53:03.069 | INFO | Baz IO overall: | 20260506 21:53:03.069 | INFO | BytesWritten : 2489752146291 | 20260506 21:53:03.069 | INFO | NumEvents : 1201763628363 | 20260506 21:53:04.783 | INFO | Analyzing chunk frames = [4730880,4731392) | 20260506 21:53:04.783 | INFO | MemInfo: { | 20260506 21:53:04.783 | INFO | "gpu" : " 17 GB", | 20260506 21:53:04.783 | INFO | "huge" : "", | 20260506 21:53:04.783 | INFO | "huge_shared" : " 27 GB", | 20260506 21:53:04.783 | INFO | "malloc" : " 14 GB", | 20260506 21:53:04.783 | INFO | "pinned" : " 29 GB" | 20260506 21:53:04.783 | INFO | } | 20260506 21:53:04.783 | INFO | Perf report for the last 5.13698 seconds: | 20260506 21:53:04.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:53:04.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:04.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:04.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:04.783 | INFO | ANALYSIS : 62.5%, 2.9, 95.9ms, 0 | 20260506 21:53:04.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:04.783 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260506 21:53:04.783 | INFO | PRE_HQ : 60.0%, 1.8, 57.2ms, 0 | 20260506 21:53:04.783 | INFO | BAZWRITER : 64.3%, 4.6, 151.2ms, 0 | 20260506 21:53:04.783 | INFO | BAZ_SAMPLING : 2.9%, 1.0, 1.5ms, 0 | 20260506 21:53:04.784 | INFO | Uploaded 36GiB to GPU in 2.05335s (17.5323GiB/s) | 20260506 21:53:04.784 | INFO | Downloaded 38GiB to GPU in 2.1945s (17.316GiB/s) | 20260506 21:53:09.922 | INFO | Analyzing chunk frames = [4731392,4731904) | 20260506 21:53:09.922 | INFO | MemInfo: { | 20260506 21:53:09.922 | INFO | "gpu" : " 17 GB", | 20260506 21:53:09.922 | INFO | "huge" : "", | 20260506 21:53:09.922 | INFO | "huge_shared" : " 27 GB", | 20260506 21:53:09.923 | INFO | "malloc" : " 14 GB", | 20260506 21:53:09.923 | INFO | "pinned" : " 29 GB" | 20260506 21:53:09.923 | INFO | } | 20260506 21:53:09.923 | INFO | Perf report for the last 5.13915 seconds: | 20260506 21:53:09.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:53:09.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:09.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:09.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:09.923 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:53:09.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:09.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:09.923 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:53:09.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:09.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:09.923 | INFO | Uploaded 36GiB to GPU in 2.02036s (17.8186GiB/s) | 20260506 21:53:09.923 | INFO | Downloaded 34GiB to GPU in 1.92787s (17.636GiB/s) | 20260506 21:53:15.066 | INFO | Analyzing chunk frames = [4731904,4732416) | 20260506 21:53:15.066 | INFO | MemInfo: { | 20260506 21:53:15.066 | INFO | "gpu" : " 17 GB", | 20260506 21:53:15.066 | INFO | "huge" : "", | 20260506 21:53:15.066 | INFO | "huge_shared" : " 27 GB", | 20260506 21:53:15.066 | INFO | "malloc" : " 14 GB", | 20260506 21:53:15.066 | INFO | "pinned" : " 29 GB" | 20260506 21:53:15.066 | INFO | } | 20260506 21:53:15.066 | INFO | Perf report for the last 5.14357 seconds: | 20260506 21:53:15.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:53:15.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:15.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:15.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:15.066 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 21:53:15.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:15.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:15.066 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:53:15.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:15.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:15.066 | INFO | Uploaded 36GiB to GPU in 1.96557s (18.3153GiB/s) | 20260506 21:53:15.067 | INFO | Downloaded 34GiB to GPU in 1.91665s (17.7392GiB/s) | 20260506 21:53:20.200 | INFO | Analyzing chunk frames = [4732416,4732928) | 20260506 21:53:20.200 | INFO | MemInfo: { | 20260506 21:53:20.200 | INFO | "gpu" : " 17 GB", | 20260506 21:53:20.200 | INFO | "huge" : "", | 20260506 21:53:20.200 | INFO | "huge_shared" : " 27 GB", | 20260506 21:53:20.200 | INFO | "malloc" : " 14 GB", | 20260506 21:53:20.200 | INFO | "pinned" : " 29 GB" | 20260506 21:53:20.200 | INFO | } | 20260506 21:53:20.200 | INFO | Perf report for the last 5.13407 seconds: | 20260506 21:53:20.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:53:20.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:20.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:20.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:20.200 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:53:20.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:20.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:20.200 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:53:20.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:20.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:20.201 | INFO | Uploaded 36GiB to GPU in 1.97013s (18.2729GiB/s) | 20260506 21:53:20.201 | INFO | Downloaded 34GiB to GPU in 1.8773s (18.1111GiB/s) | 20260506 21:53:25.345 | INFO | Analyzing chunk frames = [4732928,4733440) | 20260506 21:53:25.345 | INFO | MemInfo: { | 20260506 21:53:25.345 | INFO | "gpu" : " 17 GB", | 20260506 21:53:25.345 | INFO | "huge" : "", | 20260506 21:53:25.345 | INFO | "huge_shared" : " 27 GB", | 20260506 21:53:25.345 | INFO | "malloc" : " 14 GB", | 20260506 21:53:25.345 | INFO | "pinned" : " 29 GB" | 20260506 21:53:25.345 | INFO | } | 20260506 21:53:25.345 | INFO | Perf report for the last 5.14473 seconds: | 20260506 21:53:25.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:53:25.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:25.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:25.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:25.345 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 21:53:25.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:25.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:25.345 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:53:25.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:25.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:25.345 | INFO | Uploaded 36GiB to GPU in 2.00828s (17.9258GiB/s) | 20260506 21:53:25.345 | INFO | Downloaded 34GiB to GPU in 1.91783s (17.7283GiB/s) | 20260506 21:53:30.485 | INFO | Analyzing chunk frames = [4733440,4733952) | 20260506 21:53:30.485 | INFO | MemInfo: { | 20260506 21:53:30.485 | INFO | "gpu" : " 17 GB", | 20260506 21:53:30.485 | INFO | "huge" : "", | 20260506 21:53:30.485 | INFO | "huge_shared" : " 27 GB", | 20260506 21:53:30.485 | INFO | "malloc" : " 14 GB", | 20260506 21:53:30.485 | INFO | "pinned" : " 29 GB" | 20260506 21:53:30.485 | INFO | } | 20260506 21:53:30.485 | INFO | Perf report for the last 5.14037 seconds: | 20260506 21:53:30.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:53:30.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:30.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:30.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:30.485 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 21:53:30.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:30.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:30.485 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 21:53:30.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:30.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:30.486 | INFO | Uploaded 36GiB to GPU in 2.0133s (17.8811GiB/s) | 20260506 21:53:30.486 | INFO | Downloaded 34GiB to GPU in 1.9292s (17.6239GiB/s) | 20260506 21:53:35.624 | INFO | Analyzing chunk frames = [4733952,4734464) | 20260506 21:53:35.624 | INFO | MemInfo: { | 20260506 21:53:35.624 | INFO | "gpu" : " 17 GB", | 20260506 21:53:35.624 | INFO | "huge" : "", | 20260506 21:53:35.625 | INFO | "huge_shared" : " 27 GB", | 20260506 21:53:35.625 | INFO | "malloc" : " 14 GB", | 20260506 21:53:35.625 | INFO | "pinned" : " 29 GB" | 20260506 21:53:35.625 | INFO | } | 20260506 21:53:35.625 | INFO | Perf report for the last 5.13925 seconds: | 20260506 21:53:35.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:53:35.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:35.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:35.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:35.625 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:53:35.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:35.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:35.625 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 21:53:35.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:35.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:35.625 | INFO | Uploaded 36GiB to GPU in 1.99501s (18.045GiB/s) | 20260506 21:53:35.625 | INFO | Downloaded 34GiB to GPU in 1.99786s (17.0182GiB/s) | 20260506 21:53:40.760 | INFO | Analyzing chunk frames = [4734464,4734976) | 20260506 21:53:40.760 | INFO | MemInfo: { | 20260506 21:53:40.760 | INFO | "gpu" : " 17 GB", | 20260506 21:53:40.760 | INFO | "huge" : "", | 20260506 21:53:40.760 | INFO | "huge_shared" : " 27 GB", | 20260506 21:53:40.760 | INFO | "malloc" : " 14 GB", | 20260506 21:53:40.760 | INFO | "pinned" : " 29 GB" | 20260506 21:53:40.760 | INFO | } | 20260506 21:53:40.760 | INFO | Perf report for the last 5.1353 seconds: | 20260506 21:53:40.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:53:40.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:40.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:40.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:40.760 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 21:53:40.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:40.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:40.760 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 21:53:40.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:40.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:40.760 | INFO | Uploaded 36GiB to GPU in 2.02616s (17.7676GiB/s) | 20260506 21:53:40.760 | INFO | Downloaded 34GiB to GPU in 1.98875s (17.0961GiB/s) | 20260506 21:53:40.760 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:53:40.760 | INFO | Total mem usage: 18 GiB | 20260506 21:53:40.760 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:53:40.760 | INFO | Recent High Water Marks Sum: 16.8 GiB (Savings: -7.3%) | 20260506 21:53:40.760 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:53:40.760 | INFO | High Water Breakdown: | 20260506 21:53:40.760 | INFO | 62.6% : 10.5 GiB ( 92% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:53:40.760 | INFO | 37.3% : 6.26 GiB ( 93.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:53:40.760 | INFO | 0.119% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:53:40.760 | INFO | ------------------------------------------ | 20260506 21:53:40.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:53:40.760 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:53:40.760 | INFO | Total mem usage: 26.5 GiB | 20260506 21:53:40.760 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:53:40.760 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:53:40.760 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:53:40.760 | INFO | High Water Breakdown: | 20260506 21:53:40.760 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:53:40.760 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:53:40.760 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:53:40.761 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:53:40.761 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:53:40.761 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:53:40.761 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:53:40.761 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:53:40.761 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:53:40.761 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:53:40.761 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:53:40.761 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:53:40.761 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:53:40.761 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:53:40.761 | INFO | ------------------------------------------ | 20260506 21:53:40.761 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:53:40.761 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:53:40.761 | INFO | Total mem usage: 25.6 GiB | 20260506 21:53:40.761 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:53:40.761 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:53:40.761 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:53:40.761 | INFO | High Water Breakdown: | 20260506 21:53:40.761 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:53:40.761 | INFO | ------------------------------------------ | 20260506 21:53:40.761 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:53:40.761 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:53:40.761 | INFO | Total mem usage: 15.7 GiB | 20260506 21:53:40.761 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:53:40.761 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:53:40.761 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:53:40.761 | INFO | High Water Breakdown: | 20260506 21:53:40.761 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:53:40.761 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:53:40.761 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:53:40.761 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:53:40.761 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:53:40.761 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:53:40.761 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:53:40.761 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:53:40.761 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:53:40.761 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:53:40.761 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:53:40.761 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:53:40.761 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:53:40.761 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:53:40.761 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:53:40.761 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:53:40.761 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:53:40.761 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:53:40.761 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:53:40.761 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:53:40.761 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:53:40.761 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:53:40.761 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:53:40.761 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:53:40.761 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:53:40.761 | INFO | ------------------------------------------ | 20260506 21:53:40.761 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:53:40.761 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:53:40.761 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:53:40.761 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:53:40.761 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.61e+01, 1.36e+01 / 2.63e+00 | 20260506 21:53:40.761 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 1.07e-02 / 6.76e-03 | 20260506 21:53:40.761 | INFO | PerfReport: 11.65%, Baseline -- 3.62e+00 / 4.43e+00, 5.50e-01 / 3.15e-01 | 20260506 21:53:40.761 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.82e-01 / 1.23e-01 | 20260506 21:53:40.761 | INFO | PerfReport: 16.08%, DME -- 4.00e+01 / 4.50e+01, 4.27e+00 / 3.17e+00 | 20260506 21:53:40.761 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.41e+00 / 1.83e+00 | 20260506 21:53:40.761 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.20e+00 / 2.76e+00, 3.76e-01 / 2.66e-01 | 20260506 21:53:40.761 | INFO | PerfReport: 3.24%, Metrics -- 1.01e+00 / 1.33e+00, 2.65e-01 / 1.12e-01 | 20260506 21:53:40.761 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.46e-01 / 3.93e-01, 5.57e-02 / 2.11e-02 | 20260506 21:53:40.761 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:53:40.761 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:53:40.761 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:53:40.761 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 21:53:40.761 | INFO | PerfReport: 21.61%, Upload -- 2.09e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 21:53:40.761 | INFO | PerfReport: 21.22%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 21:53:40.761 | INFO | PerfReport: 32.16%, Compute -- 3.11e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 21:53:40.761 | INFO | PerfReport: 1.38%, LockUpload -- 1.34e+00 / 0.00e+00, 5.02e+00 / 0.00e+00 | 20260506 21:53:40.761 | INFO | PerfReport: 1.11%, LockDownload -- 1.07e+00 / 0.00e+00, 2.72e+00 / 0.00e+00 | 20260506 21:53:40.761 | INFO | PerfReport: 22.50%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 21:53:40.761 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260506 21:53:40.864 | INFO | Starting a round of Baz IO | 20260506 21:53:43.999 | INFO | Populating RTMetrics report | 20260506 21:53:44.000 | INFO | RTMetrics report populated | 20260506 21:53:44.000 | INFO | Opening tmp RTMetrics json file | 20260506 21:53:44.177 | INFO | Serializing RTMetrics to json | 20260506 21:53:44.188 | INFO | Serialization done, swapping tmp file now... | 20260506 21:53:44.203 | INFO | RTMetrics done | 20260506 21:53:44.464 | INFO | Finished a round of Baz IO: Wrote 1.932 GiB in 3.599 seconds | 20260506 21:53:44.464 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.82% / 0.00% / 66.18% | 20260506 21:53:44.464 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.563542% / 1.19209e-05% / 9.53674e-05% | 20260506 21:53:44.464 | INFO | Baz IO overall: | 20260506 21:53:44.464 | INFO | BytesWritten : 2491826981948 | 20260506 21:53:44.464 | INFO | NumEvents : 1202756675174 | 20260506 21:53:45.896 | INFO | Analyzing chunk frames = [4734976,4735488) | 20260506 21:53:45.896 | INFO | MemInfo: { | 20260506 21:53:45.896 | INFO | "gpu" : " 17 GB", | 20260506 21:53:45.896 | INFO | "huge" : "", | 20260506 21:53:45.896 | INFO | "huge_shared" : " 27 GB", | 20260506 21:53:45.896 | INFO | "malloc" : " 14 GB", | 20260506 21:53:45.896 | INFO | "pinned" : " 29 GB" | 20260506 21:53:45.896 | INFO | } | 20260506 21:53:45.896 | INFO | Perf report for the last 5.13623 seconds: | 20260506 21:53:45.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:53:45.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:45.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:45.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:45.896 | INFO | ANALYSIS : 62.5%, 2.9, 95.5ms, 0 | 20260506 21:53:45.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:45.896 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260506 21:53:45.896 | INFO | PRE_HQ : 59.6%, 1.8, 56.2ms, 0 | 20260506 21:53:45.896 | INFO | BAZWRITER : 69.8%, 4.8, 172.5ms, 0 | 20260506 21:53:45.896 | INFO | BAZ_SAMPLING : 3.8%, 1.0, 2.0ms, 0 | 20260506 21:53:45.896 | INFO | Uploaded 36GiB to GPU in 2.02316s (17.794GiB/s) | 20260506 21:53:45.896 | INFO | Downloaded 38GiB to GPU in 2.19058s (17.347GiB/s) | 20260506 21:53:51.036 | INFO | Analyzing chunk frames = [4735488,4736000) | 20260506 21:53:51.036 | INFO | MemInfo: { | 20260506 21:53:51.036 | INFO | "gpu" : " 17 GB", | 20260506 21:53:51.036 | INFO | "huge" : "", | 20260506 21:53:51.036 | INFO | "huge_shared" : " 27 GB", | 20260506 21:53:51.036 | INFO | "malloc" : " 14 GB", | 20260506 21:53:51.036 | INFO | "pinned" : " 29 GB" | 20260506 21:53:51.036 | INFO | } | 20260506 21:53:51.036 | INFO | Perf report for the last 5.14007 seconds: | 20260506 21:53:51.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:53:51.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:51.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:51.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:51.036 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:53:51.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:51.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:51.036 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 21:53:51.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:51.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:51.037 | INFO | Uploaded 36GiB to GPU in 2.01512s (17.8649GiB/s) | 20260506 21:53:51.037 | INFO | Downloaded 34GiB to GPU in 1.92585s (17.6545GiB/s) | 20260506 21:53:56.174 | INFO | Analyzing chunk frames = [4736000,4736512) | 20260506 21:53:56.174 | INFO | MemInfo: { | 20260506 21:53:56.174 | INFO | "gpu" : " 17 GB", | 20260506 21:53:56.174 | INFO | "huge" : "", | 20260506 21:53:56.174 | INFO | "huge_shared" : " 27 GB", | 20260506 21:53:56.174 | INFO | "malloc" : " 14 GB", | 20260506 21:53:56.174 | INFO | "pinned" : " 29 GB" | 20260506 21:53:56.174 | INFO | } | 20260506 21:53:56.174 | INFO | Perf report for the last 5.13813 seconds: | 20260506 21:53:56.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:53:56.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:56.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:56.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:56.174 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 21:53:56.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:56.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:56.174 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:53:56.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:56.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:53:56.175 | INFO | Uploaded 36GiB to GPU in 1.9896s (18.0941GiB/s) | 20260506 21:53:56.175 | INFO | Downloaded 34GiB to GPU in 1.91755s (17.7309GiB/s) | 20260506 21:54:01.325 | INFO | Analyzing chunk frames = [4736512,4737024) | 20260506 21:54:01.325 | INFO | MemInfo: { | 20260506 21:54:01.325 | INFO | "gpu" : " 17 GB", | 20260506 21:54:01.325 | INFO | "huge" : "", | 20260506 21:54:01.326 | INFO | "huge_shared" : " 27 GB", | 20260506 21:54:01.326 | INFO | "malloc" : " 14 GB", | 20260506 21:54:01.326 | INFO | "pinned" : " 29 GB" | 20260506 21:54:01.326 | INFO | } | 20260506 21:54:01.326 | INFO | Perf report for the last 5.15131 seconds: | 20260506 21:54:01.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:54:01.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:01.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:01.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:01.326 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 21:54:01.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:01.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:01.326 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:54:01.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:01.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:01.326 | INFO | Uploaded 36GiB to GPU in 1.94982s (18.4632GiB/s) | 20260506 21:54:01.326 | INFO | Downloaded 34GiB to GPU in 1.87691s (18.1148GiB/s) | 20260506 21:54:06.463 | INFO | Analyzing chunk frames = [4737024,4737536) | 20260506 21:54:06.463 | INFO | MemInfo: { | 20260506 21:54:06.463 | INFO | "gpu" : " 17 GB", | 20260506 21:54:06.463 | INFO | "huge" : "", | 20260506 21:54:06.463 | INFO | "huge_shared" : " 27 GB", | 20260506 21:54:06.463 | INFO | "malloc" : " 14 GB", | 20260506 21:54:06.463 | INFO | "pinned" : " 29 GB" | 20260506 21:54:06.463 | INFO | } | 20260506 21:54:06.463 | INFO | Perf report for the last 5.13742 seconds: | 20260506 21:54:06.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:54:06.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:06.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:06.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:06.463 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 21:54:06.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:06.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:06.463 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:54:06.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:06.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:06.463 | INFO | Uploaded 36GiB to GPU in 1.99765s (18.0212GiB/s) | 20260506 21:54:06.463 | INFO | Downloaded 34GiB to GPU in 1.92485s (17.6637GiB/s) | 20260506 21:54:11.602 | INFO | Analyzing chunk frames = [4737536,4738048) | 20260506 21:54:11.602 | INFO | MemInfo: { | 20260506 21:54:11.602 | INFO | "gpu" : " 17 GB", | 20260506 21:54:11.602 | INFO | "huge" : "", | 20260506 21:54:11.602 | INFO | "huge_shared" : " 27 GB", | 20260506 21:54:11.602 | INFO | "malloc" : " 14 GB", | 20260506 21:54:11.602 | INFO | "pinned" : " 29 GB" | 20260506 21:54:11.602 | INFO | } | 20260506 21:54:11.602 | INFO | Perf report for the last 5.13918 seconds: | 20260506 21:54:11.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:54:11.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:11.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:11.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:11.602 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 21:54:11.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:11.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:11.602 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 21:54:11.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:11.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:11.603 | INFO | Uploaded 36GiB to GPU in 2.02542s (17.7741GiB/s) | 20260506 21:54:11.603 | INFO | Downloaded 34GiB to GPU in 1.93635s (17.5588GiB/s) | 20260506 21:54:16.741 | INFO | Analyzing chunk frames = [4738048,4738560) | 20260506 21:54:16.741 | INFO | MemInfo: { | 20260506 21:54:16.741 | INFO | "gpu" : " 17 GB", | 20260506 21:54:16.741 | INFO | "huge" : "", | 20260506 21:54:16.741 | INFO | "huge_shared" : " 27 GB", | 20260506 21:54:16.741 | INFO | "malloc" : " 14 GB", | 20260506 21:54:16.741 | INFO | "pinned" : " 29 GB" | 20260506 21:54:16.741 | INFO | } | 20260506 21:54:16.741 | INFO | Perf report for the last 5.13859 seconds: | 20260506 21:54:16.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:54:16.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:16.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:16.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:16.741 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:54:16.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:16.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:16.741 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 21:54:16.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:16.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:16.741 | INFO | Uploaded 36GiB to GPU in 1.96767s (18.2957GiB/s) | 20260506 21:54:16.741 | INFO | Downloaded 34GiB to GPU in 2.00118s (16.99GiB/s) | 20260506 21:54:21.875 | INFO | Analyzing chunk frames = [4738560,4739072) | 20260506 21:54:21.875 | INFO | MemInfo: { | 20260506 21:54:21.875 | INFO | "gpu" : " 17 GB", | 20260506 21:54:21.875 | INFO | "huge" : "", | 20260506 21:54:21.875 | INFO | "huge_shared" : " 27 GB", | 20260506 21:54:21.875 | INFO | "malloc" : " 14 GB", | 20260506 21:54:21.875 | INFO | "pinned" : " 29 GB" | 20260506 21:54:21.875 | INFO | } | 20260506 21:54:21.875 | INFO | Perf report for the last 5.13435 seconds: | 20260506 21:54:21.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:54:21.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:21.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:21.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:21.875 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 21:54:21.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:21.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:21.875 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 21:54:21.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:21.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:21.876 | INFO | Uploaded 36GiB to GPU in 2.05017s (17.5595GiB/s) | 20260506 21:54:21.876 | INFO | Downloaded 34GiB to GPU in 2.00306s (16.974GiB/s) | 20260506 21:54:21.989 | INFO | Starting a round of Baz IO | 20260506 21:54:25.108 | INFO | Populating RTMetrics report | 20260506 21:54:25.108 | INFO | RTMetrics report populated | 20260506 21:54:25.108 | INFO | Opening tmp RTMetrics json file | 20260506 21:54:25.110 | INFO | Serializing RTMetrics to json | 20260506 21:54:25.119 | INFO | Serialization done, swapping tmp file now... | 20260506 21:54:25.143 | INFO | RTMetrics done | 20260506 21:54:25.255 | INFO | Finished a round of Baz IO: Wrote 1.93 GiB in 3.266 seconds | 20260506 21:54:25.255 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.81% / 0.00% / 66.18% | 20260506 21:54:25.255 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.560574% / 0% / 6.75519e-05% | 20260506 21:54:25.255 | INFO | Baz IO overall: | 20260506 21:54:25.255 | INFO | BytesWritten : 2493898959083 | 20260506 21:54:25.255 | INFO | NumEvents : 1203747512257 | 20260506 21:54:27.014 | INFO | Analyzing chunk frames = [4739072,4739584) | 20260506 21:54:27.014 | INFO | MemInfo: { | 20260506 21:54:27.014 | INFO | "gpu" : " 17 GB", | 20260506 21:54:27.014 | INFO | "huge" : "", | 20260506 21:54:27.014 | INFO | "huge_shared" : " 27 GB", | 20260506 21:54:27.014 | INFO | "malloc" : " 14 GB", | 20260506 21:54:27.014 | INFO | "pinned" : " 29 GB" | 20260506 21:54:27.014 | INFO | } | 20260506 21:54:27.014 | INFO | Perf report for the last 5.13898 seconds: | 20260506 21:54:27.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:54:27.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:27.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:27.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:27.014 | INFO | ANALYSIS : 62.3%, 2.9, 95.4ms, 0 | 20260506 21:54:27.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:27.014 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260506 21:54:27.014 | INFO | PRE_HQ : 59.4%, 1.8, 56.3ms, 0 | 20260506 21:54:27.014 | INFO | BAZWRITER : 63.3%, 4.7, 154.1ms, 0 | 20260506 21:54:27.014 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 21:54:27.015 | INFO | Uploaded 36GiB to GPU in 2.07216s (17.3732GiB/s) | 20260506 21:54:27.015 | INFO | Downloaded 38GiB to GPU in 2.21083s (17.1881GiB/s) | 20260506 21:54:32.152 | INFO | Analyzing chunk frames = [4739584,4740096) | 20260506 21:54:32.152 | INFO | MemInfo: { | 20260506 21:54:32.152 | INFO | "gpu" : " 17 GB", | 20260506 21:54:32.152 | INFO | "huge" : "", | 20260506 21:54:32.152 | INFO | "huge_shared" : " 27 GB", | 20260506 21:54:32.152 | INFO | "malloc" : " 14 GB", | 20260506 21:54:32.152 | INFO | "pinned" : " 29 GB" | 20260506 21:54:32.152 | INFO | } | 20260506 21:54:32.152 | INFO | Perf report for the last 5.13779 seconds: | 20260506 21:54:32.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:54:32.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:32.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:32.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:32.152 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:54:32.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:32.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:32.152 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 21:54:32.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:32.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:32.152 | INFO | Uploaded 36GiB to GPU in 2.00416s (17.9627GiB/s) | 20260506 21:54:32.153 | INFO | Downloaded 34GiB to GPU in 1.923s (17.6807GiB/s) | 20260506 21:54:37.296 | INFO | Analyzing chunk frames = [4740096,4740608) | 20260506 21:54:37.296 | INFO | MemInfo: { | 20260506 21:54:37.296 | INFO | "gpu" : " 17 GB", | 20260506 21:54:37.296 | INFO | "huge" : "", | 20260506 21:54:37.296 | INFO | "huge_shared" : " 27 GB", | 20260506 21:54:37.296 | INFO | "malloc" : " 14 GB", | 20260506 21:54:37.296 | INFO | "pinned" : " 29 GB" | 20260506 21:54:37.296 | INFO | } | 20260506 21:54:37.296 | INFO | Perf report for the last 5.14409 seconds: | 20260506 21:54:37.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:54:37.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:37.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:37.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:37.296 | INFO | ANALYSIS : 60.5%, 3.0, 93.9ms, 0 | 20260506 21:54:37.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:37.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:37.296 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:54:37.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:37.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:37.296 | INFO | Uploaded 36GiB to GPU in 1.99391s (18.055GiB/s) | 20260506 21:54:37.297 | INFO | Downloaded 34GiB to GPU in 1.92094s (17.6996GiB/s) | 20260506 21:54:42.442 | INFO | Analyzing chunk frames = [4740608,4741120) | 20260506 21:54:42.442 | INFO | MemInfo: { | 20260506 21:54:42.442 | INFO | "gpu" : " 17 GB", | 20260506 21:54:42.442 | INFO | "huge" : "", | 20260506 21:54:42.442 | INFO | "huge_shared" : " 27 GB", | 20260506 21:54:42.442 | INFO | "malloc" : " 14 GB", | 20260506 21:54:42.442 | INFO | "pinned" : " 29 GB" | 20260506 21:54:42.442 | INFO | } | 20260506 21:54:42.442 | INFO | Perf report for the last 5.14616 seconds: | 20260506 21:54:42.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:54:42.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:42.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:42.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:42.442 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260506 21:54:42.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:42.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:42.442 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 21:54:42.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:42.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:42.443 | INFO | Uploaded 36GiB to GPU in 2.01999s (17.8219GiB/s) | 20260506 21:54:42.443 | INFO | Downloaded 34GiB to GPU in 1.90687s (17.8303GiB/s) | 20260506 21:54:47.577 | INFO | Analyzing chunk frames = [4741120,4741632) | 20260506 21:54:47.577 | INFO | MemInfo: { | 20260506 21:54:47.577 | INFO | "gpu" : " 17 GB", | 20260506 21:54:47.577 | INFO | "huge" : "", | 20260506 21:54:47.577 | INFO | "huge_shared" : " 27 GB", | 20260506 21:54:47.577 | INFO | "malloc" : " 14 GB", | 20260506 21:54:47.577 | INFO | "pinned" : " 29 GB" | 20260506 21:54:47.577 | INFO | } | 20260506 21:54:47.577 | INFO | Perf report for the last 5.1347 seconds: | 20260506 21:54:47.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:54:47.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:47.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:47.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:47.577 | INFO | ANALYSIS : 60.3%, 3.0, 93.7ms, 0 | 20260506 21:54:47.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:47.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:47.577 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:54:47.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:47.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:47.577 | INFO | Uploaded 36GiB to GPU in 1.99839s (18.0145GiB/s) | 20260506 21:54:47.577 | INFO | Downloaded 34GiB to GPU in 1.92741s (17.6402GiB/s) | 20260506 21:54:52.712 | INFO | Analyzing chunk frames = [4741632,4742144) | 20260506 21:54:52.712 | INFO | MemInfo: { | 20260506 21:54:52.712 | INFO | "gpu" : " 17 GB", | 20260506 21:54:52.712 | INFO | "huge" : "", | 20260506 21:54:52.712 | INFO | "huge_shared" : " 27 GB", | 20260506 21:54:52.712 | INFO | "malloc" : " 14 GB", | 20260506 21:54:52.712 | INFO | "pinned" : " 29 GB" | 20260506 21:54:52.712 | INFO | } | 20260506 21:54:52.712 | INFO | Perf report for the last 5.13509 seconds: | 20260506 21:54:52.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:54:52.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:52.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:52.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:52.712 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 21:54:52.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:52.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:52.712 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 21:54:52.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:52.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:52.713 | INFO | Uploaded 36GiB to GPU in 2.01821s (17.8376GiB/s) | 20260506 21:54:52.713 | INFO | Downloaded 34GiB to GPU in 1.94143s (17.5129GiB/s) | 20260506 21:54:57.847 | INFO | Analyzing chunk frames = [4742144,4742656) | 20260506 21:54:57.847 | INFO | MemInfo: { | 20260506 21:54:57.847 | INFO | "gpu" : " 17 GB", | 20260506 21:54:57.847 | INFO | "huge" : "", | 20260506 21:54:57.847 | INFO | "huge_shared" : " 27 GB", | 20260506 21:54:57.847 | INFO | "malloc" : " 14 GB", | 20260506 21:54:57.847 | INFO | "pinned" : " 29 GB" | 20260506 21:54:57.847 | INFO | } | 20260506 21:54:57.847 | INFO | Perf report for the last 5.13468 seconds: | 20260506 21:54:57.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:54:57.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:57.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:57.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:57.847 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 21:54:57.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:57.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:57.847 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:54:57.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:57.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:54:57.847 | INFO | Uploaded 36GiB to GPU in 1.9746s (18.2315GiB/s) | 20260506 21:54:57.847 | INFO | Downloaded 34GiB to GPU in 2.00361s (16.9694GiB/s) | 20260506 21:55:02.988 | INFO | Analyzing chunk frames = [4742656,4743168) | 20260506 21:55:02.988 | INFO | MemInfo: { | 20260506 21:55:02.988 | INFO | "gpu" : " 17 GB", | 20260506 21:55:02.988 | INFO | "huge" : "", | 20260506 21:55:02.988 | INFO | "huge_shared" : " 27 GB", | 20260506 21:55:02.988 | INFO | "malloc" : " 14 GB", | 20260506 21:55:02.988 | INFO | "pinned" : " 29 GB" | 20260506 21:55:02.988 | INFO | } | 20260506 21:55:02.988 | INFO | Perf report for the last 5.14134 seconds: | 20260506 21:55:02.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:55:02.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:02.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:02.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:02.988 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:55:02.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:02.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:02.988 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 21:55:02.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:02.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:02.989 | INFO | Uploaded 36GiB to GPU in 1.99677s (18.0291GiB/s) | 20260506 21:55:02.989 | INFO | Downloaded 34GiB to GPU in 1.97917s (17.179GiB/s) | 20260506 21:55:02.989 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:55:02.989 | INFO | Total mem usage: 18 GiB | 20260506 21:55:02.989 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:55:02.989 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -14.1%) | 20260506 21:55:02.989 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:55:02.989 | INFO | High Water Breakdown: | 20260506 21:55:02.989 | INFO | 61.7% : 9.73 GiB ( 85.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:55:02.989 | INFO | 38.2% : 6.03 GiB ( 90.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:55:02.989 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:55:02.989 | INFO | ------------------------------------------ | 20260506 21:55:02.989 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:55:02.989 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:55:02.989 | INFO | Total mem usage: 26.5 GiB | 20260506 21:55:02.989 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:55:02.989 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:55:02.989 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:55:02.989 | INFO | High Water Breakdown: | 20260506 21:55:02.989 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:55:02.989 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:55:02.989 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:55:02.989 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:55:02.989 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:55:02.989 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:55:02.989 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:55:02.989 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:55:02.989 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:55:02.989 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:55:02.989 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:55:02.989 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:55:02.989 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:55:02.989 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:55:02.989 | INFO | ------------------------------------------ | 20260506 21:55:02.989 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:55:02.989 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:55:02.989 | INFO | Total mem usage: 25.6 GiB | 20260506 21:55:02.989 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:55:02.989 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 21:55:02.989 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 21:55:02.989 | INFO | High Water Breakdown: | 20260506 21:55:02.989 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 21:55:02.989 | INFO | ------------------------------------------ | 20260506 21:55:02.989 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:55:02.989 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:55:02.989 | INFO | Total mem usage: 15.7 GiB | 20260506 21:55:02.989 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:55:02.989 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:55:02.989 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 21:55:02.989 | INFO | High Water Breakdown: | 20260506 21:55:02.989 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:55:02.989 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:55:02.989 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:55:02.989 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:55:02.989 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:55:02.989 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:55:02.989 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:55:02.989 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:55:02.989 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:55:02.989 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:55:02.989 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:55:02.989 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:55:02.989 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:55:02.989 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:55:02.989 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:55:02.989 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:55:02.989 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:55:02.989 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:55:02.989 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:55:02.989 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:55:02.989 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:55:02.989 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:55:02.989 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:55:02.989 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:55:02.989 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:55:02.989 | INFO | ------------------------------------------ | 20260506 21:55:02.989 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:55:02.989 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:55:02.989 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:55:02.989 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:55:02.989 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.61e+01, 1.36e+01 / 2.19e+00 | 20260506 21:55:02.989 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 9.92e-02, 9.75e-03 / 5.52e-03 | 20260506 21:55:02.989 | INFO | PerfReport: 11.65%, Baseline -- 3.62e+00 / 4.36e+00, 5.45e-01 / 3.65e-01 | 20260506 21:55:02.990 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.40e+00, 1.77e-01 / 1.40e-01 | 20260506 21:55:02.990 | INFO | PerfReport: 16.07%, DME -- 3.99e+01 / 4.50e+01, 4.20e+00 / 2.75e+00 | 20260506 21:55:02.990 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.36e+00 / 1.77e+00 | 20260506 21:55:02.990 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.21e+00 / 2.80e+00, 3.77e-01 / 1.95e-01 | 20260506 21:55:02.990 | INFO | PerfReport: 3.25%, Metrics -- 1.01e+00 / 1.35e+00, 2.71e-01 / 1.57e-01 | 20260506 21:55:02.990 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.45e-01 / 3.93e-01, 5.58e-02 / 1.84e-02 | 20260506 21:55:02.990 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:55:02.990 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:55:02.990 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:55:02.990 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 21:55:02.990 | INFO | PerfReport: 21.61%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 21:55:02.990 | INFO | PerfReport: 21.27%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 21:55:02.990 | INFO | PerfReport: 32.14%, Compute -- 3.11e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 21:55:02.990 | INFO | PerfReport: 1.38%, LockUpload -- 1.34e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260506 21:55:02.990 | INFO | PerfReport: 1.15%, LockDownload -- 1.11e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260506 21:55:02.990 | INFO | PerfReport: 22.44%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:55:02.990 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260506 21:55:03.086 | INFO | Starting a round of Baz IO | 20260506 21:55:06.220 | INFO | Populating RTMetrics report | 20260506 21:55:06.220 | INFO | RTMetrics report populated | 20260506 21:55:06.220 | INFO | Opening tmp RTMetrics json file | 20260506 21:55:06.224 | INFO | Serializing RTMetrics to json | 20260506 21:55:06.227 | INFO | Serialization done, swapping tmp file now... | 20260506 21:55:06.231 | INFO | RTMetrics done | 20260506 21:55:06.830 | INFO | Finished a round of Baz IO: Wrote 1.932 GiB in 3.743 seconds | 20260506 21:55:06.830 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.81% / 0.00% / 66.19% | 20260506 21:55:06.830 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.561833% / 7.94729e-06% / 7.15256e-05% | 20260506 21:55:06.830 | INFO | Baz IO overall: | 20260506 21:55:06.831 | INFO | BytesWritten : 2495973291477 | 20260506 21:55:06.831 | INFO | NumEvents : 1204740168446 | 20260506 21:55:08.129 | INFO | Analyzing chunk frames = [4743168,4743680) | 20260506 21:55:08.129 | INFO | MemInfo: { | 20260506 21:55:08.129 | INFO | "gpu" : " 17 GB", | 20260506 21:55:08.129 | INFO | "huge" : "", | 20260506 21:55:08.129 | INFO | "huge_shared" : " 27 GB", | 20260506 21:55:08.129 | INFO | "malloc" : " 14 GB", | 20260506 21:55:08.129 | INFO | "pinned" : " 29 GB" | 20260506 21:55:08.129 | INFO | } | 20260506 21:55:08.129 | INFO | Perf report for the last 5.14051 seconds: | 20260506 21:55:08.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:55:08.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:08.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:08.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:08.129 | INFO | ANALYSIS : 62.3%, 2.9, 95.3ms, 0 | 20260506 21:55:08.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:08.129 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 21:55:08.129 | INFO | PRE_HQ : 59.9%, 1.8, 56.5ms, 0 | 20260506 21:55:08.129 | INFO | BAZWRITER : 72.4%, 4.8, 179.2ms, 0 | 20260506 21:55:08.129 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 21:55:08.129 | INFO | Uploaded 36GiB to GPU in 2.03978s (17.649GiB/s) | 20260506 21:55:08.129 | INFO | Downloaded 38GiB to GPU in 2.19454s (17.3157GiB/s) | 20260506 21:55:13.268 | INFO | Analyzing chunk frames = [4743680,4744192) | 20260506 21:55:13.268 | INFO | MemInfo: { | 20260506 21:55:13.268 | INFO | "gpu" : " 17 GB", | 20260506 21:55:13.268 | INFO | "huge" : "", | 20260506 21:55:13.268 | INFO | "huge_shared" : " 27 GB", | 20260506 21:55:13.268 | INFO | "malloc" : " 14 GB", | 20260506 21:55:13.268 | INFO | "pinned" : " 29 GB" | 20260506 21:55:13.268 | INFO | } | 20260506 21:55:13.268 | INFO | Perf report for the last 5.13896 seconds: | 20260506 21:55:13.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:55:13.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:13.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:13.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:13.268 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 21:55:13.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:13.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:13.268 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:55:13.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:13.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:13.268 | INFO | Uploaded 36GiB to GPU in 2.01011s (17.9095GiB/s) | 20260506 21:55:13.268 | INFO | Downloaded 34GiB to GPU in 1.92779s (17.6368GiB/s) | 20260506 21:55:18.418 | INFO | Analyzing chunk frames = [4744192,4744704) | 20260506 21:55:18.418 | INFO | MemInfo: { | 20260506 21:55:18.418 | INFO | "gpu" : " 17 GB", | 20260506 21:55:18.418 | INFO | "huge" : "", | 20260506 21:55:18.418 | INFO | "huge_shared" : " 27 GB", | 20260506 21:55:18.418 | INFO | "malloc" : " 14 GB", | 20260506 21:55:18.418 | INFO | "pinned" : " 29 GB" | 20260506 21:55:18.418 | INFO | } | 20260506 21:55:18.418 | INFO | Perf report for the last 5.15016 seconds: | 20260506 21:55:18.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:55:18.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:18.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:18.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:18.418 | INFO | ANALYSIS : 60.6%, 3.0, 94.0ms, 0 | 20260506 21:55:18.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:18.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:18.418 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:55:18.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:18.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:18.418 | INFO | Uploaded 36GiB to GPU in 1.99112s (18.0803GiB/s) | 20260506 21:55:18.418 | INFO | Downloaded 34GiB to GPU in 1.92345s (17.6765GiB/s) | 20260506 21:55:23.553 | INFO | Analyzing chunk frames = [4744704,4745216) | 20260506 21:55:23.553 | INFO | MemInfo: { | 20260506 21:55:23.553 | INFO | "gpu" : " 17 GB", | 20260506 21:55:23.553 | INFO | "huge" : "", | 20260506 21:55:23.553 | INFO | "huge_shared" : " 27 GB", | 20260506 21:55:23.553 | INFO | "malloc" : " 14 GB", | 20260506 21:55:23.553 | INFO | "pinned" : " 29 GB" | 20260506 21:55:23.553 | INFO | } | 20260506 21:55:23.553 | INFO | Perf report for the last 5.13502 seconds: | 20260506 21:55:23.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:55:23.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:23.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:23.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:23.553 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 21:55:23.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:23.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:23.553 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:55:23.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:23.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:23.553 | INFO | Uploaded 36GiB to GPU in 2.02256s (17.7993GiB/s) | 20260506 21:55:23.553 | INFO | Downloaded 34GiB to GPU in 1.9072s (17.8272GiB/s) | 20260506 21:55:28.692 | INFO | Analyzing chunk frames = [4745216,4745728) | 20260506 21:55:28.692 | INFO | MemInfo: { | 20260506 21:55:28.692 | INFO | "gpu" : " 17 GB", | 20260506 21:55:28.692 | INFO | "huge" : "", | 20260506 21:55:28.692 | INFO | "huge_shared" : " 27 GB", | 20260506 21:55:28.692 | INFO | "malloc" : " 14 GB", | 20260506 21:55:28.692 | INFO | "pinned" : " 29 GB" | 20260506 21:55:28.692 | INFO | } | 20260506 21:55:28.692 | INFO | Perf report for the last 5.13913 seconds: | 20260506 21:55:28.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:55:28.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:28.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:28.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:28.692 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 21:55:28.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:28.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:28.692 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:55:28.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:28.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:28.692 | INFO | Uploaded 36GiB to GPU in 2.00707s (17.9366GiB/s) | 20260506 21:55:28.692 | INFO | Downloaded 34GiB to GPU in 1.92396s (17.6719GiB/s) | 20260506 21:55:33.829 | INFO | Analyzing chunk frames = [4745728,4746240) | 20260506 21:55:33.829 | INFO | MemInfo: { | 20260506 21:55:33.829 | INFO | "gpu" : " 17 GB", | 20260506 21:55:33.829 | INFO | "huge" : "", | 20260506 21:55:33.829 | INFO | "huge_shared" : " 27 GB", | 20260506 21:55:33.829 | INFO | "malloc" : " 14 GB", | 20260506 21:55:33.829 | INFO | "pinned" : " 29 GB" | 20260506 21:55:33.829 | INFO | } | 20260506 21:55:33.829 | INFO | Perf report for the last 5.13716 seconds: | 20260506 21:55:33.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:55:33.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:33.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:33.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:33.829 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 21:55:33.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:33.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:33.829 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 21:55:33.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:33.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:33.830 | INFO | Uploaded 36GiB to GPU in 2.03589s (17.6826GiB/s) | 20260506 21:55:33.830 | INFO | Downloaded 34GiB to GPU in 1.94126s (17.5144GiB/s) | 20260506 21:55:38.963 | INFO | Analyzing chunk frames = [4746240,4746752) | 20260506 21:55:38.963 | INFO | MemInfo: { | 20260506 21:55:38.963 | INFO | "gpu" : " 17 GB", | 20260506 21:55:38.963 | INFO | "huge" : "", | 20260506 21:55:38.963 | INFO | "huge_shared" : " 27 GB", | 20260506 21:55:38.963 | INFO | "malloc" : " 14 GB", | 20260506 21:55:38.963 | INFO | "pinned" : " 29 GB" | 20260506 21:55:38.963 | INFO | } | 20260506 21:55:38.963 | INFO | Perf report for the last 5.13364 seconds: | 20260506 21:55:38.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:55:38.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:38.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:38.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:38.963 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 21:55:38.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:38.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:38.963 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 21:55:38.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:38.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:38.963 | INFO | Uploaded 36GiB to GPU in 1.95587s (18.4062GiB/s) | 20260506 21:55:38.963 | INFO | Downloaded 34GiB to GPU in 1.99222s (17.0664GiB/s) | 20260506 21:55:44.100 | INFO | Analyzing chunk frames = [4746752,4747264) | 20260506 21:55:44.100 | INFO | MemInfo: { | 20260506 21:55:44.100 | INFO | "gpu" : " 17 GB", | 20260506 21:55:44.100 | INFO | "huge" : "", | 20260506 21:55:44.100 | INFO | "huge_shared" : " 27 GB", | 20260506 21:55:44.100 | INFO | "malloc" : " 14 GB", | 20260506 21:55:44.100 | INFO | "pinned" : " 29 GB" | 20260506 21:55:44.100 | INFO | } | 20260506 21:55:44.101 | INFO | Perf report for the last 5.13759 seconds: | 20260506 21:55:44.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:55:44.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:44.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:44.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:44.101 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:55:44.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:44.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:44.101 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 21:55:44.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:44.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:44.101 | INFO | Uploaded 36GiB to GPU in 1.98807s (18.108GiB/s) | 20260506 21:55:44.101 | INFO | Downloaded 34GiB to GPU in 1.9833s (17.1432GiB/s) | 20260506 21:55:44.199 | INFO | Starting a round of Baz IO | 20260506 21:55:47.336 | INFO | Populating RTMetrics report | 20260506 21:55:47.336 | INFO | RTMetrics report populated | 20260506 21:55:47.336 | INFO | Opening tmp RTMetrics json file | 20260506 21:55:47.337 | INFO | Serializing RTMetrics to json | 20260506 21:55:47.338 | INFO | Serialization done, swapping tmp file now... | 20260506 21:55:47.339 | INFO | RTMetrics done | 20260506 21:55:48.138 | INFO | Finished a round of Baz IO: Wrote 1.93 GiB in 3.939 seconds | 20260506 21:55:48.138 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.81% / 0.00% / 66.19% | 20260506 21:55:48.138 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.560025% / 7.94729e-06% / 7.15256e-05% | 20260506 21:55:48.138 | INFO | Baz IO overall: | 20260506 21:55:48.138 | INFO | BytesWritten : 2498045545872 | 20260506 21:55:48.138 | INFO | NumEvents : 1205731142133 | 20260506 21:55:49.246 | INFO | Analyzing chunk frames = [4747264,4747776) | 20260506 21:55:49.246 | INFO | MemInfo: { | 20260506 21:55:49.246 | INFO | "gpu" : " 17 GB", | 20260506 21:55:49.246 | INFO | "huge" : "", | 20260506 21:55:49.246 | INFO | "huge_shared" : " 27 GB", | 20260506 21:55:49.246 | INFO | "malloc" : " 14 GB", | 20260506 21:55:49.246 | INFO | "pinned" : " 29 GB" | 20260506 21:55:49.246 | INFO | } | 20260506 21:55:49.246 | INFO | Perf report for the last 5.14592 seconds: | 20260506 21:55:49.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:55:49.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:49.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:49.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:49.247 | INFO | ANALYSIS : 62.3%, 2.9, 95.5ms, 0 | 20260506 21:55:49.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:49.247 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 21:55:49.247 | INFO | PRE_HQ : 59.6%, 1.8, 56.4ms, 0 | 20260506 21:55:49.247 | INFO | BAZWRITER : 76.2%, 4.8, 187.2ms, 0 | 20260506 21:55:49.247 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 21:55:49.247 | INFO | Uploaded 36GiB to GPU in 2.02468s (17.7806GiB/s) | 20260506 21:55:49.247 | INFO | Downloaded 38GiB to GPU in 2.19759s (17.2916GiB/s) | 20260506 21:55:54.381 | INFO | Analyzing chunk frames = [4747776,4748288) | 20260506 21:55:54.381 | INFO | MemInfo: { | 20260506 21:55:54.381 | INFO | "gpu" : " 17 GB", | 20260506 21:55:54.381 | INFO | "huge" : "", | 20260506 21:55:54.381 | INFO | "huge_shared" : " 27 GB", | 20260506 21:55:54.381 | INFO | "malloc" : " 14 GB", | 20260506 21:55:54.381 | INFO | "pinned" : " 29 GB" | 20260506 21:55:54.381 | INFO | } | 20260506 21:55:54.381 | INFO | Perf report for the last 5.13465 seconds: | 20260506 21:55:54.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:55:54.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:54.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:54.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:54.381 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 21:55:54.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:54.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:54.381 | INFO | PRE_HQ : 26.4%, 1.0, 13.9ms, 0 | 20260506 21:55:54.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:54.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:54.381 | INFO | Uploaded 36GiB to GPU in 2.0194s (17.8271GiB/s) | 20260506 21:55:54.382 | INFO | Downloaded 34GiB to GPU in 1.93126s (17.6051GiB/s) | 20260506 21:55:59.520 | INFO | Analyzing chunk frames = [4748288,4748800) | 20260506 21:55:59.520 | INFO | MemInfo: { | 20260506 21:55:59.520 | INFO | "gpu" : " 17 GB", | 20260506 21:55:59.520 | INFO | "huge" : "", | 20260506 21:55:59.520 | INFO | "huge_shared" : " 27 GB", | 20260506 21:55:59.520 | INFO | "malloc" : " 14 GB", | 20260506 21:55:59.520 | INFO | "pinned" : " 29 GB" | 20260506 21:55:59.520 | INFO | } | 20260506 21:55:59.520 | INFO | Perf report for the last 5.13913 seconds: | 20260506 21:55:59.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:55:59.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:59.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:59.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:59.520 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 21:55:59.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:59.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:59.520 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:55:59.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:59.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:55:59.521 | INFO | Uploaded 36GiB to GPU in 1.9819s (18.1644GiB/s) | 20260506 21:55:59.521 | INFO | Downloaded 34GiB to GPU in 1.92099s (17.6992GiB/s) | 20260506 21:56:04.665 | INFO | Analyzing chunk frames = [4748800,4749312) | 20260506 21:56:04.665 | INFO | MemInfo: { | 20260506 21:56:04.665 | INFO | "gpu" : " 17 GB", | 20260506 21:56:04.665 | INFO | "huge" : "", | 20260506 21:56:04.665 | INFO | "huge_shared" : " 27 GB", | 20260506 21:56:04.665 | INFO | "malloc" : " 14 GB", | 20260506 21:56:04.665 | INFO | "pinned" : " 29 GB" | 20260506 21:56:04.665 | INFO | } | 20260506 21:56:04.665 | INFO | Perf report for the last 5.14498 seconds: | 20260506 21:56:04.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:56:04.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:04.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:04.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:04.665 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 21:56:04.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:04.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:04.665 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 21:56:04.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:04.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:04.665 | INFO | Uploaded 36GiB to GPU in 1.93425s (18.6119GiB/s) | 20260506 21:56:04.665 | INFO | Downloaded 34GiB to GPU in 1.8679s (18.2023GiB/s) | 20260506 21:56:09.805 | INFO | Analyzing chunk frames = [4749312,4749824) | 20260506 21:56:09.805 | INFO | MemInfo: { | 20260506 21:56:09.805 | INFO | "gpu" : " 17 GB", | 20260506 21:56:09.805 | INFO | "huge" : "", | 20260506 21:56:09.805 | INFO | "huge_shared" : " 27 GB", | 20260506 21:56:09.805 | INFO | "malloc" : " 14 GB", | 20260506 21:56:09.805 | INFO | "pinned" : " 29 GB" | 20260506 21:56:09.805 | INFO | } | 20260506 21:56:09.805 | INFO | Perf report for the last 5.13962 seconds: | 20260506 21:56:09.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:56:09.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:09.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:09.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:09.805 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 21:56:09.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:09.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:09.805 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:56:09.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:09.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:09.805 | INFO | Uploaded 36GiB to GPU in 1.99684s (18.0285GiB/s) | 20260506 21:56:09.805 | INFO | Downloaded 34GiB to GPU in 1.92172s (17.6924GiB/s) | 20260506 21:56:14.941 | INFO | Analyzing chunk frames = [4749824,4750336) | 20260506 21:56:14.941 | INFO | MemInfo: { | 20260506 21:56:14.941 | INFO | "gpu" : " 17 GB", | 20260506 21:56:14.941 | INFO | "huge" : "", | 20260506 21:56:14.941 | INFO | "huge_shared" : " 27 GB", | 20260506 21:56:14.941 | INFO | "malloc" : " 14 GB", | 20260506 21:56:14.941 | INFO | "pinned" : " 29 GB" | 20260506 21:56:14.941 | INFO | } | 20260506 21:56:14.941 | INFO | Perf report for the last 5.13652 seconds: | 20260506 21:56:14.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:56:14.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:14.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:14.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:14.942 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 21:56:14.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:14.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:14.942 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 21:56:14.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:14.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:14.942 | INFO | Uploaded 36GiB to GPU in 2.0358s (17.6834GiB/s) | 20260506 21:56:14.942 | INFO | Downloaded 34GiB to GPU in 1.93893s (17.5355GiB/s) | 20260506 21:56:20.083 | INFO | Analyzing chunk frames = [4750336,4750848) | 20260506 21:56:20.083 | INFO | MemInfo: { | 20260506 21:56:20.083 | INFO | "gpu" : " 17 GB", | 20260506 21:56:20.083 | INFO | "huge" : "", | 20260506 21:56:20.083 | INFO | "huge_shared" : " 27 GB", | 20260506 21:56:20.083 | INFO | "malloc" : " 14 GB", | 20260506 21:56:20.083 | INFO | "pinned" : " 29 GB" | 20260506 21:56:20.083 | INFO | } | 20260506 21:56:20.083 | INFO | Perf report for the last 5.14167 seconds: | 20260506 21:56:20.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:56:20.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:20.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:20.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:20.083 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:56:20.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:20.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:20.083 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 21:56:20.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:20.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:20.083 | INFO | Uploaded 36GiB to GPU in 1.97668s (18.2124GiB/s) | 20260506 21:56:20.083 | INFO | Downloaded 34GiB to GPU in 1.99902s (17.0084GiB/s) | 20260506 21:56:25.220 | INFO | Analyzing chunk frames = [4750848,4751360) | 20260506 21:56:25.220 | INFO | MemInfo: { | 20260506 21:56:25.220 | INFO | "gpu" : " 17 GB", | 20260506 21:56:25.220 | INFO | "huge" : "", | 20260506 21:56:25.220 | INFO | "huge_shared" : " 27 GB", | 20260506 21:56:25.220 | INFO | "malloc" : " 14 GB", | 20260506 21:56:25.220 | INFO | "pinned" : " 29 GB" | 20260506 21:56:25.220 | INFO | } | 20260506 21:56:25.220 | INFO | Perf report for the last 5.13688 seconds: | 20260506 21:56:25.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:56:25.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:25.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:25.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:25.220 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260506 21:56:25.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:25.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:25.220 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 21:56:25.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:25.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:25.220 | INFO | Uploaded 36GiB to GPU in 2.01628s (17.8546GiB/s) | 20260506 21:56:25.220 | INFO | Downloaded 34GiB to GPU in 1.98313s (17.1446GiB/s) | 20260506 21:56:25.220 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:56:25.220 | INFO | Total mem usage: 18 GiB | 20260506 21:56:25.220 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:56:25.220 | INFO | Recent High Water Marks Sum: 17.5 GiB (Savings: -2.61%) | 20260506 21:56:25.220 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:56:25.220 | INFO | High Water Breakdown: | 20260506 21:56:25.220 | INFO | 63.2% : 11.1 GiB ( 97.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:56:25.220 | INFO | 36.7% : 6.43 GiB ( 96.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:56:25.220 | INFO | 0.113% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:56:25.220 | INFO | ------------------------------------------ | 20260506 21:56:25.220 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:56:25.221 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:56:25.221 | INFO | Total mem usage: 26.5 GiB | 20260506 21:56:25.221 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:56:25.221 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:56:25.221 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:56:25.221 | INFO | High Water Breakdown: | 20260506 21:56:25.221 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:56:25.221 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:56:25.221 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:56:25.221 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:56:25.221 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:56:25.221 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:56:25.221 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:56:25.221 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:56:25.221 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:56:25.221 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:56:25.221 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:56:25.221 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:56:25.221 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:56:25.221 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:56:25.221 | INFO | ------------------------------------------ | 20260506 21:56:25.221 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:56:25.221 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:56:25.221 | INFO | Total mem usage: 25.6 GiB | 20260506 21:56:25.221 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:56:25.221 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:56:25.221 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:56:25.221 | INFO | High Water Breakdown: | 20260506 21:56:25.221 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:56:25.221 | INFO | ------------------------------------------ | 20260506 21:56:25.221 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:56:25.221 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:56:25.221 | INFO | Total mem usage: 15.7 GiB | 20260506 21:56:25.221 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:56:25.221 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:56:25.221 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:56:25.221 | INFO | High Water Breakdown: | 20260506 21:56:25.221 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:56:25.221 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:56:25.221 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:56:25.221 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:56:25.221 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:56:25.221 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:56:25.221 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:56:25.221 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:56:25.221 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:56:25.221 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:56:25.221 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:56:25.221 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:56:25.221 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:56:25.221 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:56:25.221 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:56:25.221 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:56:25.221 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:56:25.221 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:56:25.221 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:56:25.221 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:56:25.221 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:56:25.221 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:56:25.221 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:56:25.221 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:56:25.221 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:56:25.221 | INFO | ------------------------------------------ | 20260506 21:56:25.221 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:56:25.221 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:56:25.221 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:56:25.221 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:56:25.221 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.58e+01, 1.35e+01 / 2.05e+00 | 20260506 21:56:25.221 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.00e-01, 9.64e-03 / 6.04e-03 | 20260506 21:56:25.221 | INFO | PerfReport: 11.65%, Baseline -- 3.62e+00 / 4.36e+00, 5.56e-01 / 3.87e-01 | 20260506 21:56:25.221 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.44e+00, 1.83e-01 / 1.44e-01 | 20260506 21:56:25.221 | INFO | PerfReport: 16.03%, DME -- 3.99e+01 / 4.44e+01, 4.11e+00 / 2.58e+00 | 20260506 21:56:25.221 | INFO | PerfReport: 57.31%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.41e+00 / 1.86e+00 | 20260506 21:56:25.221 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.20e+00 / 2.74e+00, 3.74e-01 / 1.94e-01 | 20260506 21:56:25.221 | INFO | PerfReport: 3.25%, Metrics -- 1.01e+00 / 1.34e+00, 2.67e-01 / 1.36e-01 | 20260506 21:56:25.221 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.46e-01 / 3.93e-01, 5.61e-02 / 2.28e-02 | 20260506 21:56:25.221 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:56:25.221 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:56:25.221 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:56:25.221 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 21:56:25.221 | INFO | PerfReport: 21.56%, Upload -- 2.09e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260506 21:56:25.221 | INFO | PerfReport: 21.24%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 21:56:25.221 | INFO | PerfReport: 32.16%, Compute -- 3.11e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 21:56:25.221 | INFO | PerfReport: 1.40%, LockUpload -- 1.35e+00 / 0.00e+00, 5.01e+00 / 0.00e+00 | 20260506 21:56:25.221 | INFO | PerfReport: 1.13%, LockDownload -- 1.10e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260506 21:56:25.221 | INFO | PerfReport: 22.49%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:56:25.221 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260506 21:56:25.319 | INFO | Starting a round of Baz IO | 20260506 21:56:28.454 | INFO | Populating RTMetrics report | 20260506 21:56:28.454 | INFO | RTMetrics report populated | 20260506 21:56:28.454 | INFO | Opening tmp RTMetrics json file | 20260506 21:56:28.620 | INFO | Serializing RTMetrics to json | 20260506 21:56:28.779 | INFO | Serialization done, swapping tmp file now... | 20260506 21:56:28.822 | INFO | RTMetrics done | 20260506 21:56:28.858 | INFO | Finished a round of Baz IO: Wrote 1.93 GiB in 3.538 seconds | 20260506 21:56:28.858 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.80% / 0.00% / 66.19% | 20260506 21:56:28.858 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.557892% / 0% / 7.54992e-05% | 20260506 21:56:28.858 | INFO | Baz IO overall: | 20260506 21:56:28.858 | INFO | BytesWritten : 2500117630753 | 20260506 21:56:28.858 | INFO | NumEvents : 1206721898453 | 20260506 21:56:30.356 | INFO | Analyzing chunk frames = [4751360,4751872) | 20260506 21:56:30.357 | INFO | MemInfo: { | 20260506 21:56:30.357 | INFO | "gpu" : " 17 GB", | 20260506 21:56:30.357 | INFO | "huge" : "", | 20260506 21:56:30.357 | INFO | "huge_shared" : " 27 GB", | 20260506 21:56:30.357 | INFO | "malloc" : " 14 GB", | 20260506 21:56:30.357 | INFO | "pinned" : " 29 GB" | 20260506 21:56:30.357 | INFO | } | 20260506 21:56:30.357 | INFO | Perf report for the last 5.13655 seconds: | 20260506 21:56:30.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:56:30.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:30.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:30.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:30.357 | INFO | ANALYSIS : 62.4%, 2.9, 95.4ms, 0 | 20260506 21:56:30.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:30.357 | INFO | RT_METRICS : 49.6%, 1.0, 26.0ms, 0 | 20260506 21:56:30.357 | INFO | PRE_HQ : 59.9%, 1.8, 56.3ms, 0 | 20260506 21:56:30.357 | INFO | BAZWRITER : 68.5%, 4.8, 169.9ms, 0 | 20260506 21:56:30.357 | INFO | BAZ_SAMPLING : 3.5%, 1.0, 1.8ms, 0 | 20260506 21:56:30.357 | INFO | Uploaded 36GiB to GPU in 2.04257s (17.6248GiB/s) | 20260506 21:56:30.357 | INFO | Downloaded 38GiB to GPU in 2.20675s (17.2199GiB/s) | 20260506 21:56:35.506 | INFO | Analyzing chunk frames = [4751872,4752384) | 20260506 21:56:35.506 | INFO | MemInfo: { | 20260506 21:56:35.506 | INFO | "gpu" : " 17 GB", | 20260506 21:56:35.506 | INFO | "huge" : "", | 20260506 21:56:35.506 | INFO | "huge_shared" : " 27 GB", | 20260506 21:56:35.506 | INFO | "malloc" : " 14 GB", | 20260506 21:56:35.506 | INFO | "pinned" : " 29 GB" | 20260506 21:56:35.506 | INFO | } | 20260506 21:56:35.506 | INFO | Perf report for the last 5.14942 seconds: | 20260506 21:56:35.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:56:35.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:35.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:35.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:35.506 | INFO | ANALYSIS : 61.3%, 3.0, 95.7ms, 0 | 20260506 21:56:35.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:35.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:35.506 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 21:56:35.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:35.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:35.506 | INFO | Uploaded 36GiB to GPU in 2.01971s (17.8243GiB/s) | 20260506 21:56:35.506 | INFO | Downloaded 34GiB to GPU in 1.92915s (17.6244GiB/s) | 20260506 21:56:40.640 | INFO | Analyzing chunk frames = [4752384,4752896) | 20260506 21:56:40.640 | INFO | MemInfo: { | 20260506 21:56:40.640 | INFO | "gpu" : " 17 GB", | 20260506 21:56:40.640 | INFO | "huge" : "", | 20260506 21:56:40.640 | INFO | "huge_shared" : " 27 GB", | 20260506 21:56:40.640 | INFO | "malloc" : " 14 GB", | 20260506 21:56:40.640 | INFO | "pinned" : " 29 GB" | 20260506 21:56:40.640 | INFO | } | 20260506 21:56:40.640 | INFO | Perf report for the last 5.13406 seconds: | 20260506 21:56:40.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:56:40.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:40.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:40.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:40.640 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 21:56:40.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:40.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:40.640 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:56:40.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:40.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:40.640 | INFO | Uploaded 36GiB to GPU in 1.99279s (18.0651GiB/s) | 20260506 21:56:40.640 | INFO | Downloaded 34GiB to GPU in 1.92838s (17.6314GiB/s) | 20260506 21:56:45.778 | INFO | Analyzing chunk frames = [4752896,4753408) | 20260506 21:56:45.778 | INFO | MemInfo: { | 20260506 21:56:45.778 | INFO | "gpu" : " 17 GB", | 20260506 21:56:45.778 | INFO | "huge" : "", | 20260506 21:56:45.778 | INFO | "huge_shared" : " 27 GB", | 20260506 21:56:45.778 | INFO | "malloc" : " 14 GB", | 20260506 21:56:45.778 | INFO | "pinned" : " 29 GB" | 20260506 21:56:45.778 | INFO | } | 20260506 21:56:45.778 | INFO | Perf report for the last 5.13799 seconds: | 20260506 21:56:45.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:56:45.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:45.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:45.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:45.778 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 21:56:45.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:45.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:45.778 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:56:45.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:45.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:45.778 | INFO | Uploaded 36GiB to GPU in 2.02077s (17.815GiB/s) | 20260506 21:56:45.779 | INFO | Downloaded 34GiB to GPU in 1.90696s (17.8294GiB/s) | 20260506 21:56:50.919 | INFO | Analyzing chunk frames = [4753408,4753920) | 20260506 21:56:50.919 | INFO | MemInfo: { | 20260506 21:56:50.919 | INFO | "gpu" : " 17 GB", | 20260506 21:56:50.920 | INFO | "huge" : "", | 20260506 21:56:50.920 | INFO | "huge_shared" : " 27 GB", | 20260506 21:56:50.920 | INFO | "malloc" : " 14 GB", | 20260506 21:56:50.920 | INFO | "pinned" : " 29 GB" | 20260506 21:56:50.920 | INFO | } | 20260506 21:56:50.920 | INFO | Perf report for the last 5.14152 seconds: | 20260506 21:56:50.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:56:50.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:50.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:50.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:50.920 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 21:56:50.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:50.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:50.920 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:56:50.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:50.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:50.920 | INFO | Uploaded 36GiB to GPU in 1.9983s (18.0153GiB/s) | 20260506 21:56:50.920 | INFO | Downloaded 34GiB to GPU in 1.91982s (17.71GiB/s) | 20260506 21:56:56.055 | INFO | Analyzing chunk frames = [4753920,4754432) | 20260506 21:56:56.055 | INFO | MemInfo: { | 20260506 21:56:56.055 | INFO | "gpu" : " 17 GB", | 20260506 21:56:56.055 | INFO | "huge" : "", | 20260506 21:56:56.055 | INFO | "huge_shared" : " 27 GB", | 20260506 21:56:56.055 | INFO | "malloc" : " 14 GB", | 20260506 21:56:56.055 | INFO | "pinned" : " 29 GB" | 20260506 21:56:56.055 | INFO | } | 20260506 21:56:56.055 | INFO | Perf report for the last 5.13507 seconds: | 20260506 21:56:56.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:56:56.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:56.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:56.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:56.055 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 21:56:56.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:56.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:56.055 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 21:56:56.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:56.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:56:56.055 | INFO | Uploaded 36GiB to GPU in 2.01172s (17.8951GiB/s) | 20260506 21:56:56.055 | INFO | Downloaded 34GiB to GPU in 1.94068s (17.5196GiB/s) | 20260506 21:57:01.193 | INFO | Analyzing chunk frames = [4754432,4754944) | 20260506 21:57:01.193 | INFO | MemInfo: { | 20260506 21:57:01.193 | INFO | "gpu" : " 17 GB", | 20260506 21:57:01.193 | INFO | "huge" : "", | 20260506 21:57:01.193 | INFO | "huge_shared" : " 27 GB", | 20260506 21:57:01.194 | INFO | "malloc" : " 14 GB", | 20260506 21:57:01.194 | INFO | "pinned" : " 29 GB" | 20260506 21:57:01.194 | INFO | } | 20260506 21:57:01.194 | INFO | Perf report for the last 5.13901 seconds: | 20260506 21:57:01.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:57:01.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:01.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:01.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:01.194 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 21:57:01.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:01.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:01.194 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 21:57:01.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:01.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:01.194 | INFO | Uploaded 36GiB to GPU in 2.02204s (17.8038GiB/s) | 20260506 21:57:01.194 | INFO | Downloaded 34GiB to GPU in 1.99565s (17.0371GiB/s) | 20260506 21:57:06.334 | INFO | Analyzing chunk frames = [4754944,4755456) | 20260506 21:57:06.334 | INFO | MemInfo: { | 20260506 21:57:06.334 | INFO | "gpu" : " 17 GB", | 20260506 21:57:06.334 | INFO | "huge" : "", | 20260506 21:57:06.334 | INFO | "huge_shared" : " 27 GB", | 20260506 21:57:06.334 | INFO | "malloc" : " 14 GB", | 20260506 21:57:06.334 | INFO | "pinned" : " 29 GB" | 20260506 21:57:06.334 | INFO | } | 20260506 21:57:06.334 | INFO | Perf report for the last 5.14054 seconds: | 20260506 21:57:06.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:57:06.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:06.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:06.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:06.334 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:57:06.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:06.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:06.334 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 21:57:06.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:06.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:06.335 | INFO | Uploaded 36GiB to GPU in 1.9942s (18.0524GiB/s) | 20260506 21:57:06.335 | INFO | Downloaded 34GiB to GPU in 1.98019s (17.1701GiB/s) | 20260506 21:57:06.432 | INFO | Starting a round of Baz IO | 20260506 21:57:09.567 | INFO | Populating RTMetrics report | 20260506 21:57:09.567 | INFO | RTMetrics report populated | 20260506 21:57:09.567 | INFO | Opening tmp RTMetrics json file | 20260506 21:57:09.570 | INFO | Serializing RTMetrics to json | 20260506 21:57:09.576 | INFO | Serialization done, swapping tmp file now... | 20260506 21:57:09.580 | INFO | RTMetrics done | 20260506 21:57:10.375 | INFO | Finished a round of Baz IO: Wrote 1.93 GiB in 3.942 seconds | 20260506 21:57:10.375 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.80% / 0.00% / 66.20% | 20260506 21:57:10.375 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.557077% / 3.97364e-06% / 6.75519e-05% | 20260506 21:57:10.375 | INFO | Baz IO overall: | 20260506 21:57:10.375 | INFO | BytesWritten : 2502190157913 | 20260506 21:57:10.375 | INFO | NumEvents : 1207713320281 | 20260506 21:57:11.471 | INFO | Analyzing chunk frames = [4755456,4755968) | 20260506 21:57:11.471 | INFO | MemInfo: { | 20260506 21:57:11.471 | INFO | "gpu" : " 17 GB", | 20260506 21:57:11.471 | INFO | "huge" : "", | 20260506 21:57:11.471 | INFO | "huge_shared" : " 27 GB", | 20260506 21:57:11.471 | INFO | "malloc" : " 14 GB", | 20260506 21:57:11.471 | INFO | "pinned" : " 29 GB" | 20260506 21:57:11.471 | INFO | } | 20260506 21:57:11.471 | INFO | Perf report for the last 5.13666 seconds: | 20260506 21:57:11.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:57:11.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:11.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:11.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:11.471 | INFO | ANALYSIS : 62.4%, 2.9, 95.5ms, 0 | 20260506 21:57:11.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:11.471 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 21:57:11.471 | INFO | PRE_HQ : 59.8%, 1.8, 56.4ms, 0 | 20260506 21:57:11.471 | INFO | BAZWRITER : 76.4%, 4.8, 188.1ms, 0 | 20260506 21:57:11.471 | INFO | BAZ_SAMPLING : 7.0%, 1.0, 3.6ms, 0 | 20260506 21:57:11.471 | INFO | Uploaded 36GiB to GPU in 2.03639s (17.6784GiB/s) | 20260506 21:57:11.471 | INFO | Downloaded 38GiB to GPU in 2.19452s (17.3158GiB/s) | 20260506 21:57:16.611 | INFO | Analyzing chunk frames = [4755968,4756480) | 20260506 21:57:16.611 | INFO | MemInfo: { | 20260506 21:57:16.611 | INFO | "gpu" : " 17 GB", | 20260506 21:57:16.611 | INFO | "huge" : "", | 20260506 21:57:16.611 | INFO | "huge_shared" : " 27 GB", | 20260506 21:57:16.611 | INFO | "malloc" : " 14 GB", | 20260506 21:57:16.611 | INFO | "pinned" : " 29 GB" | 20260506 21:57:16.611 | INFO | } | 20260506 21:57:16.611 | INFO | Perf report for the last 5.14009 seconds: | 20260506 21:57:16.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:57:16.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:16.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:16.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:16.611 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:57:16.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:16.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:16.611 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:57:16.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:16.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:16.611 | INFO | Uploaded 36GiB to GPU in 2.01535s (17.8629GiB/s) | 20260506 21:57:16.611 | INFO | Downloaded 34GiB to GPU in 1.92465s (17.6656GiB/s) | 20260506 21:57:21.757 | INFO | Analyzing chunk frames = [4756480,4756992) | 20260506 21:57:21.757 | INFO | MemInfo: { | 20260506 21:57:21.757 | INFO | "gpu" : " 17 GB", | 20260506 21:57:21.757 | INFO | "huge" : "", | 20260506 21:57:21.757 | INFO | "huge_shared" : " 27 GB", | 20260506 21:57:21.757 | INFO | "malloc" : " 14 GB", | 20260506 21:57:21.757 | INFO | "pinned" : " 29 GB" | 20260506 21:57:21.757 | INFO | } | 20260506 21:57:21.757 | INFO | Perf report for the last 5.1462 seconds: | 20260506 21:57:21.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:57:21.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:21.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:21.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:21.757 | INFO | ANALYSIS : 60.4%, 3.0, 93.8ms, 0 | 20260506 21:57:21.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:21.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:21.757 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 21:57:21.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:21.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:21.758 | INFO | Uploaded 36GiB to GPU in 1.97613s (18.2175GiB/s) | 20260506 21:57:21.758 | INFO | Downloaded 34GiB to GPU in 1.91894s (17.7181GiB/s) | 20260506 21:57:26.896 | INFO | Analyzing chunk frames = [4756992,4757504) | 20260506 21:57:26.896 | INFO | MemInfo: { | 20260506 21:57:26.896 | INFO | "gpu" : " 17 GB", | 20260506 21:57:26.896 | INFO | "huge" : "", | 20260506 21:57:26.896 | INFO | "huge_shared" : " 27 GB", | 20260506 21:57:26.896 | INFO | "malloc" : " 14 GB", | 20260506 21:57:26.896 | INFO | "pinned" : " 29 GB" | 20260506 21:57:26.896 | INFO | } | 20260506 21:57:26.896 | INFO | Perf report for the last 5.13905 seconds: | 20260506 21:57:26.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:57:26.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:26.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:26.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:26.896 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:57:26.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:26.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:26.896 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 21:57:26.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:26.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:26.897 | INFO | Uploaded 36GiB to GPU in 1.97014s (18.2728GiB/s) | 20260506 21:57:26.897 | INFO | Downloaded 34GiB to GPU in 1.87629s (18.1209GiB/s) | 20260506 21:57:32.035 | INFO | Analyzing chunk frames = [4757504,4758016) | 20260506 21:57:32.035 | INFO | MemInfo: { | 20260506 21:57:32.035 | INFO | "gpu" : " 17 GB", | 20260506 21:57:32.035 | INFO | "huge" : "", | 20260506 21:57:32.035 | INFO | "huge_shared" : " 27 GB", | 20260506 21:57:32.035 | INFO | "malloc" : " 14 GB", | 20260506 21:57:32.035 | INFO | "pinned" : " 29 GB" | 20260506 21:57:32.035 | INFO | } | 20260506 21:57:32.035 | INFO | Perf report for the last 5.13898 seconds: | 20260506 21:57:32.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:57:32.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:32.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:32.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:32.035 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 21:57:32.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:32.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:32.035 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:57:32.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:32.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:32.036 | INFO | Uploaded 36GiB to GPU in 1.99578s (18.0381GiB/s) | 20260506 21:57:32.036 | INFO | Downloaded 34GiB to GPU in 1.93609s (17.5612GiB/s) | 20260506 21:57:37.172 | INFO | Analyzing chunk frames = [4758016,4758528) | 20260506 21:57:37.172 | INFO | MemInfo: { | 20260506 21:57:37.172 | INFO | "gpu" : " 17 GB", | 20260506 21:57:37.172 | INFO | "huge" : "", | 20260506 21:57:37.172 | INFO | "huge_shared" : " 27 GB", | 20260506 21:57:37.172 | INFO | "malloc" : " 14 GB", | 20260506 21:57:37.172 | INFO | "pinned" : " 29 GB" | 20260506 21:57:37.172 | INFO | } | 20260506 21:57:37.172 | INFO | Perf report for the last 5.13675 seconds: | 20260506 21:57:37.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:57:37.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:37.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:37.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:37.172 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 21:57:37.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:37.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:37.172 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 21:57:37.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:37.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:37.173 | INFO | Uploaded 36GiB to GPU in 2.02462s (17.7811GiB/s) | 20260506 21:57:37.173 | INFO | Downloaded 34GiB to GPU in 1.93695s (17.5534GiB/s) | 20260506 21:57:42.310 | INFO | Analyzing chunk frames = [4758528,4759040) | 20260506 21:57:42.310 | INFO | MemInfo: { | 20260506 21:57:42.310 | INFO | "gpu" : " 17 GB", | 20260506 21:57:42.310 | INFO | "huge" : "", | 20260506 21:57:42.310 | INFO | "huge_shared" : " 27 GB", | 20260506 21:57:42.310 | INFO | "malloc" : " 14 GB", | 20260506 21:57:42.310 | INFO | "pinned" : " 29 GB" | 20260506 21:57:42.310 | INFO | } | 20260506 21:57:42.310 | INFO | Perf report for the last 5.13775 seconds: | 20260506 21:57:42.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:57:42.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:42.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:42.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:42.310 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:57:42.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:42.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:42.310 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 21:57:42.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:42.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:42.310 | INFO | Uploaded 36GiB to GPU in 1.96255s (18.3435GiB/s) | 20260506 21:57:42.310 | INFO | Downloaded 34GiB to GPU in 1.98992s (17.0861GiB/s) | 20260506 21:57:47.448 | INFO | Analyzing chunk frames = [4759040,4759552) | 20260506 21:57:47.448 | INFO | MemInfo: { | 20260506 21:57:47.448 | INFO | "gpu" : " 17 GB", | 20260506 21:57:47.448 | INFO | "huge" : "", | 20260506 21:57:47.448 | INFO | "huge_shared" : " 27 GB", | 20260506 21:57:47.448 | INFO | "malloc" : " 14 GB", | 20260506 21:57:47.448 | INFO | "pinned" : " 29 GB" | 20260506 21:57:47.448 | INFO | } | 20260506 21:57:47.448 | INFO | Perf report for the last 5.1379 seconds: | 20260506 21:57:47.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:57:47.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:47.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:47.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:47.448 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:57:47.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:47.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:47.448 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 21:57:47.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:47.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:47.448 | INFO | Uploaded 36GiB to GPU in 2.04756s (17.5819GiB/s) | 20260506 21:57:47.448 | INFO | Downloaded 34GiB to GPU in 1.99428s (17.0488GiB/s) | 20260506 21:57:47.448 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:57:47.448 | INFO | Total mem usage: 18 GiB | 20260506 21:57:47.448 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:57:47.448 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.6%) | 20260506 21:57:47.448 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:57:47.448 | INFO | High Water Breakdown: | 20260506 21:57:47.448 | INFO | 61.9% : 9.9 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:57:47.448 | INFO | 38% : 6.08 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:57:47.448 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:57:47.448 | INFO | ------------------------------------------ | 20260506 21:57:47.448 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:57:47.448 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:57:47.448 | INFO | Total mem usage: 26.5 GiB | 20260506 21:57:47.448 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:57:47.448 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:57:47.448 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:57:47.449 | INFO | High Water Breakdown: | 20260506 21:57:47.449 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:57:47.449 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:57:47.449 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:57:47.449 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:57:47.449 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:57:47.449 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:57:47.449 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:57:47.449 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:57:47.449 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:57:47.449 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:57:47.449 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:57:47.449 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:57:47.449 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:57:47.449 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:57:47.449 | INFO | ------------------------------------------ | 20260506 21:57:47.449 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:57:47.449 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:57:47.449 | INFO | Total mem usage: 25.6 GiB | 20260506 21:57:47.449 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:57:47.449 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 21:57:47.449 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 21:57:47.449 | INFO | High Water Breakdown: | 20260506 21:57:47.449 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 21:57:47.449 | INFO | ------------------------------------------ | 20260506 21:57:47.449 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:57:47.449 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:57:47.449 | INFO | Total mem usage: 15.7 GiB | 20260506 21:57:47.449 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:57:47.449 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:57:47.449 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:57:47.449 | INFO | High Water Breakdown: | 20260506 21:57:47.449 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:57:47.449 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:57:47.449 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:57:47.449 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:57:47.449 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:57:47.449 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:57:47.449 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:57:47.449 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:57:47.449 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:57:47.449 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:57:47.449 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:57:47.449 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:57:47.449 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:57:47.449 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:57:47.449 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:57:47.449 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:57:47.449 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:57:47.449 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:57:47.449 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:57:47.449 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:57:47.449 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:57:47.449 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:57:47.449 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:57:47.449 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:57:47.449 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:57:47.449 | INFO | ------------------------------------------ | 20260506 21:57:47.449 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:57:47.449 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:57:47.449 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:57:47.449 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:57:47.449 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.60e+01, 1.34e+01 / 2.13e+00 | 20260506 21:57:47.449 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 1.04e-02 / 6.44e-03 | 20260506 21:57:47.449 | INFO | PerfReport: 11.65%, Baseline -- 3.62e+00 / 4.36e+00, 5.51e-01 / 3.34e-01 | 20260506 21:57:47.449 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.45e+00, 1.81e-01 / 1.30e-01 | 20260506 21:57:47.449 | INFO | PerfReport: 16.05%, DME -- 3.99e+01 / 4.47e+01, 4.21e+00 / 2.63e+00 | 20260506 21:57:47.449 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.39e+00 / 1.95e+00 | 20260506 21:57:47.449 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.21e+00 / 2.77e+00, 3.82e-01 / 2.09e-01 | 20260506 21:57:47.449 | INFO | PerfReport: 3.25%, Metrics -- 1.01e+00 / 1.38e+00, 2.70e-01 / 1.27e-01 | 20260506 21:57:47.449 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.45e-01 / 3.92e-01, 5.58e-02 / 1.95e-02 | 20260506 21:57:47.450 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:57:47.450 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:57:47.450 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:57:47.450 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 21:57:47.450 | INFO | PerfReport: 21.63%, Upload -- 2.09e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 21:57:47.450 | INFO | PerfReport: 21.26%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 21:57:47.450 | INFO | PerfReport: 32.15%, Compute -- 3.11e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 21:57:47.450 | INFO | PerfReport: 1.35%, LockUpload -- 1.31e+00 / 0.00e+00, 4.76e+00 / 0.00e+00 | 20260506 21:57:47.450 | INFO | PerfReport: 1.14%, LockDownload -- 1.10e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 21:57:47.450 | INFO | PerfReport: 22.45%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 21:57:47.450 | INFO | PerfReport: 0.02%, Misc -- 1.56e-02 / 0.00e+00, 3.12e-02 / 0.00e+00 | 20260506 21:57:47.563 | INFO | Starting a round of Baz IO | 20260506 21:57:50.686 | INFO | Populating RTMetrics report | 20260506 21:57:50.686 | INFO | RTMetrics report populated | 20260506 21:57:50.686 | INFO | Opening tmp RTMetrics json file | 20260506 21:57:50.689 | INFO | Serializing RTMetrics to json | 20260506 21:57:50.722 | INFO | Serialization done, swapping tmp file now... | 20260506 21:57:50.733 | INFO | RTMetrics done | 20260506 21:57:50.859 | INFO | Finished a round of Baz IO: Wrote 1.93 GiB in 3.296 seconds | 20260506 21:57:50.859 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.80% / 0.00% / 66.20% | 20260506 21:57:50.859 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.556179% / 3.97364e-06% / 5.16574e-05% | 20260506 21:57:50.859 | INFO | Baz IO overall: | 20260506 21:57:50.859 | INFO | BytesWritten : 2504262342913 | 20260506 21:57:50.859 | INFO | NumEvents : 1208704264449 | 20260506 21:57:52.589 | INFO | Analyzing chunk frames = [4759552,4760064) | 20260506 21:57:52.589 | INFO | MemInfo: { | 20260506 21:57:52.589 | INFO | "gpu" : " 17 GB", | 20260506 21:57:52.589 | INFO | "huge" : "", | 20260506 21:57:52.589 | INFO | "huge_shared" : " 27 GB", | 20260506 21:57:52.589 | INFO | "malloc" : " 14 GB", | 20260506 21:57:52.589 | INFO | "pinned" : " 29 GB" | 20260506 21:57:52.589 | INFO | } | 20260506 21:57:52.589 | INFO | Perf report for the last 5.14148 seconds: | 20260506 21:57:52.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:57:52.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:52.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:52.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:52.589 | INFO | ANALYSIS : 62.4%, 2.9, 95.5ms, 0 | 20260506 21:57:52.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:52.589 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260506 21:57:52.589 | INFO | PRE_HQ : 59.5%, 1.8, 56.6ms, 0 | 20260506 21:57:52.589 | INFO | BAZWRITER : 63.8%, 4.7, 153.5ms, 0 | 20260506 21:57:52.589 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 21:57:52.590 | INFO | Uploaded 36GiB to GPU in 2.06753s (17.4121GiB/s) | 20260506 21:57:52.590 | INFO | Downloaded 38GiB to GPU in 2.21569s (17.1504GiB/s) | 20260506 21:57:57.732 | INFO | Analyzing chunk frames = [4760064,4760576) | 20260506 21:57:57.732 | INFO | MemInfo: { | 20260506 21:57:57.732 | INFO | "gpu" : " 17 GB", | 20260506 21:57:57.732 | INFO | "huge" : "", | 20260506 21:57:57.732 | INFO | "huge_shared" : " 27 GB", | 20260506 21:57:57.732 | INFO | "malloc" : " 14 GB", | 20260506 21:57:57.732 | INFO | "pinned" : " 29 GB" | 20260506 21:57:57.732 | INFO | } | 20260506 21:57:57.732 | INFO | Perf report for the last 5.14266 seconds: | 20260506 21:57:57.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:57:57.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:57.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:57.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:57.732 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 21:57:57.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:57.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:57.732 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 21:57:57.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:57.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:57:57.732 | INFO | Uploaded 36GiB to GPU in 2.01421s (17.873GiB/s) | 20260506 21:57:57.732 | INFO | Downloaded 34GiB to GPU in 1.92455s (17.6665GiB/s) | 20260506 21:58:02.871 | INFO | Analyzing chunk frames = [4760576,4761088) | 20260506 21:58:02.871 | INFO | MemInfo: { | 20260506 21:58:02.871 | INFO | "gpu" : " 17 GB", | 20260506 21:58:02.871 | INFO | "huge" : "", | 20260506 21:58:02.871 | INFO | "huge_shared" : " 27 GB", | 20260506 21:58:02.871 | INFO | "malloc" : " 14 GB", | 20260506 21:58:02.871 | INFO | "pinned" : " 29 GB" | 20260506 21:58:02.871 | INFO | } | 20260506 21:58:02.871 | INFO | Perf report for the last 5.13952 seconds: | 20260506 21:58:02.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:58:02.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:02.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:02.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:02.871 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 21:58:02.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:02.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:02.871 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:58:02.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:02.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:02.872 | INFO | Uploaded 36GiB to GPU in 1.99711s (18.026GiB/s) | 20260506 21:58:02.872 | INFO | Downloaded 34GiB to GPU in 1.9293s (17.623GiB/s) | 20260506 21:58:08.011 | INFO | Analyzing chunk frames = [4761088,4761600) | 20260506 21:58:08.011 | INFO | MemInfo: { | 20260506 21:58:08.011 | INFO | "gpu" : " 17 GB", | 20260506 21:58:08.011 | INFO | "huge" : "", | 20260506 21:58:08.011 | INFO | "huge_shared" : " 27 GB", | 20260506 21:58:08.011 | INFO | "malloc" : " 14 GB", | 20260506 21:58:08.011 | INFO | "pinned" : " 29 GB" | 20260506 21:58:08.011 | INFO | } | 20260506 21:58:08.011 | INFO | Perf report for the last 5.13984 seconds: | 20260506 21:58:08.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:58:08.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:08.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:08.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:08.011 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260506 21:58:08.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:08.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:08.011 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 21:58:08.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:08.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:08.012 | INFO | Uploaded 36GiB to GPU in 2.01892s (17.8313GiB/s) | 20260506 21:58:08.012 | INFO | Downloaded 34GiB to GPU in 1.90538s (17.8442GiB/s) | 20260506 21:58:13.150 | INFO | Analyzing chunk frames = [4761600,4762112) | 20260506 21:58:13.150 | INFO | MemInfo: { | 20260506 21:58:13.150 | INFO | "gpu" : " 17 GB", | 20260506 21:58:13.150 | INFO | "huge" : "", | 20260506 21:58:13.150 | INFO | "huge_shared" : " 27 GB", | 20260506 21:58:13.150 | INFO | "malloc" : " 14 GB", | 20260506 21:58:13.150 | INFO | "pinned" : " 29 GB" | 20260506 21:58:13.150 | INFO | } | 20260506 21:58:13.150 | INFO | Perf report for the last 5.13896 seconds: | 20260506 21:58:13.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:58:13.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:13.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:13.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:13.150 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 21:58:13.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:13.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:13.151 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:58:13.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:13.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:13.151 | INFO | Uploaded 36GiB to GPU in 2.01426s (17.8726GiB/s) | 20260506 21:58:13.151 | INFO | Downloaded 34GiB to GPU in 1.91499s (17.7547GiB/s) | 20260506 21:58:18.283 | INFO | Analyzing chunk frames = [4762112,4762624) | 20260506 21:58:18.283 | INFO | MemInfo: { | 20260506 21:58:18.283 | INFO | "gpu" : " 17 GB", | 20260506 21:58:18.283 | INFO | "huge" : "", | 20260506 21:58:18.283 | INFO | "huge_shared" : " 27 GB", | 20260506 21:58:18.283 | INFO | "malloc" : " 14 GB", | 20260506 21:58:18.283 | INFO | "pinned" : " 29 GB" | 20260506 21:58:18.283 | INFO | } | 20260506 21:58:18.283 | INFO | Perf report for the last 5.1329 seconds: | 20260506 21:58:18.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:58:18.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:18.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:18.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:18.283 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 21:58:18.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:18.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:18.283 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 21:58:18.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:18.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:18.283 | INFO | Uploaded 36GiB to GPU in 2.02135s (17.8099GiB/s) | 20260506 21:58:18.283 | INFO | Downloaded 34GiB to GPU in 1.92977s (17.6187GiB/s) | 20260506 21:58:23.421 | INFO | Analyzing chunk frames = [4762624,4763136) | 20260506 21:58:23.421 | INFO | MemInfo: { | 20260506 21:58:23.421 | INFO | "gpu" : " 17 GB", | 20260506 21:58:23.421 | INFO | "huge" : "", | 20260506 21:58:23.421 | INFO | "huge_shared" : " 27 GB", | 20260506 21:58:23.421 | INFO | "malloc" : " 14 GB", | 20260506 21:58:23.421 | INFO | "pinned" : " 29 GB" | 20260506 21:58:23.421 | INFO | } | 20260506 21:58:23.421 | INFO | Perf report for the last 5.13793 seconds: | 20260506 21:58:23.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:58:23.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:23.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:23.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:23.421 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 21:58:23.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:23.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:23.421 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 21:58:23.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:23.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:23.421 | INFO | Uploaded 36GiB to GPU in 1.96275s (18.3416GiB/s) | 20260506 21:58:23.422 | INFO | Downloaded 34GiB to GPU in 1.98937s (17.0908GiB/s) | 20260506 21:58:28.562 | INFO | Analyzing chunk frames = [4763136,4763648) | 20260506 21:58:28.562 | INFO | MemInfo: { | 20260506 21:58:28.562 | INFO | "gpu" : " 17 GB", | 20260506 21:58:28.562 | INFO | "huge" : "", | 20260506 21:58:28.562 | INFO | "huge_shared" : " 27 GB", | 20260506 21:58:28.562 | INFO | "malloc" : " 14 GB", | 20260506 21:58:28.562 | INFO | "pinned" : " 29 GB" | 20260506 21:58:28.562 | INFO | } | 20260506 21:58:28.562 | INFO | Perf report for the last 5.14054 seconds: | 20260506 21:58:28.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:58:28.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:28.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:28.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:28.562 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 21:58:28.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:28.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:28.562 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 21:58:28.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:28.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:28.562 | INFO | Uploaded 36GiB to GPU in 2.0138s (17.8767GiB/s) | 20260506 21:58:28.562 | INFO | Downloaded 34GiB to GPU in 1.99493s (17.0432GiB/s) | 20260506 21:58:28.666 | INFO | Starting a round of Baz IO | 20260506 21:58:31.816 | INFO | Populating RTMetrics report | 20260506 21:58:31.816 | INFO | RTMetrics report populated | 20260506 21:58:31.816 | INFO | Opening tmp RTMetrics json file | 20260506 21:58:32.026 | INFO | Serializing RTMetrics to json | 20260506 21:58:32.031 | INFO | Serialization done, swapping tmp file now... | 20260506 21:58:32.036 | INFO | RTMetrics done | 20260506 21:58:32.327 | INFO | Finished a round of Baz IO: Wrote 1.928 GiB in 3.661 seconds | 20260506 21:58:32.327 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.80% / 0.00% / 66.20% | 20260506 21:58:32.327 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.554526% / 7.94729e-06% / 4.37101e-05% | 20260506 21:58:32.327 | INFO | Baz IO overall: | 20260506 21:58:32.327 | INFO | BytesWritten : 2506332532869 | 20260506 21:58:32.327 | INFO | NumEvents : 1209693771643 | 20260506 21:58:33.704 | INFO | Analyzing chunk frames = [4763648,4764160) | 20260506 21:58:33.704 | INFO | MemInfo: { | 20260506 21:58:33.704 | INFO | "gpu" : " 17 GB", | 20260506 21:58:33.704 | INFO | "huge" : "", | 20260506 21:58:33.704 | INFO | "huge_shared" : " 27 GB", | 20260506 21:58:33.704 | INFO | "malloc" : " 14 GB", | 20260506 21:58:33.704 | INFO | "pinned" : " 29 GB" | 20260506 21:58:33.704 | INFO | } | 20260506 21:58:33.704 | INFO | Perf report for the last 5.14222 seconds: | 20260506 21:58:33.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:58:33.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:33.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:33.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:33.704 | INFO | ANALYSIS : 62.6%, 2.9, 94.7ms, 0 | 20260506 21:58:33.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:33.704 | INFO | RT_METRICS : 49.1%, 1.0, 25.8ms, 0 | 20260506 21:58:33.704 | INFO | PRE_HQ : 60.4%, 1.9, 60.1ms, 0 | 20260506 21:58:33.704 | INFO | BAZWRITER : 70.9%, 4.8, 173.9ms, 0 | 20260506 21:58:33.704 | INFO | BAZ_SAMPLING : 4.3%, 1.0, 2.2ms, 0 | 20260506 21:58:33.704 | INFO | Uploaded 36GiB to GPU in 2.05166s (17.5468GiB/s) | 20260506 21:58:33.704 | INFO | Downloaded 38GiB to GPU in 2.19724s (17.2944GiB/s) | 20260506 21:58:38.849 | INFO | Analyzing chunk frames = [4764160,4764672) | 20260506 21:58:38.849 | INFO | MemInfo: { | 20260506 21:58:38.849 | INFO | "gpu" : " 17 GB", | 20260506 21:58:38.849 | INFO | "huge" : "", | 20260506 21:58:38.849 | INFO | "huge_shared" : " 27 GB", | 20260506 21:58:38.849 | INFO | "malloc" : " 14 GB", | 20260506 21:58:38.849 | INFO | "pinned" : " 29 GB" | 20260506 21:58:38.849 | INFO | } | 20260506 21:58:38.849 | INFO | Perf report for the last 5.14479 seconds: | 20260506 21:58:38.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:58:38.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:38.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:38.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:38.849 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 21:58:38.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:38.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:38.849 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 21:58:38.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:38.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:38.849 | INFO | Uploaded 36GiB to GPU in 2s (18GiB/s) | 20260506 21:58:38.849 | INFO | Downloaded 34GiB to GPU in 1.92472s (17.6649GiB/s) | 20260506 21:58:43.987 | INFO | Analyzing chunk frames = [4764672,4765184) | 20260506 21:58:43.987 | INFO | MemInfo: { | 20260506 21:58:43.987 | INFO | "gpu" : " 17 GB", | 20260506 21:58:43.987 | INFO | "huge" : "", | 20260506 21:58:43.987 | INFO | "huge_shared" : " 27 GB", | 20260506 21:58:43.987 | INFO | "malloc" : " 14 GB", | 20260506 21:58:43.987 | INFO | "pinned" : " 29 GB" | 20260506 21:58:43.987 | INFO | } | 20260506 21:58:43.987 | INFO | Perf report for the last 5.13834 seconds: | 20260506 21:58:43.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:58:43.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:43.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:43.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:43.987 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260506 21:58:43.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:43.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:43.987 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 21:58:43.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:43.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:43.987 | INFO | Uploaded 36GiB to GPU in 1.97288s (18.2474GiB/s) | 20260506 21:58:43.987 | INFO | Downloaded 34GiB to GPU in 1.92858s (17.6296GiB/s) | 20260506 21:58:49.126 | INFO | Analyzing chunk frames = [4765184,4765696) | 20260506 21:58:49.126 | INFO | MemInfo: { | 20260506 21:58:49.126 | INFO | "gpu" : " 17 GB", | 20260506 21:58:49.126 | INFO | "huge" : "", | 20260506 21:58:49.126 | INFO | "huge_shared" : " 27 GB", | 20260506 21:58:49.126 | INFO | "malloc" : " 14 GB", | 20260506 21:58:49.126 | INFO | "pinned" : " 29 GB" | 20260506 21:58:49.126 | INFO | } | 20260506 21:58:49.126 | INFO | Perf report for the last 5.1394 seconds: | 20260506 21:58:49.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:58:49.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:49.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:49.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:49.127 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 21:58:49.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:49.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:49.127 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 21:58:49.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:49.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:49.127 | INFO | Uploaded 36GiB to GPU in 1.97349s (18.2418GiB/s) | 20260506 21:58:49.127 | INFO | Downloaded 34GiB to GPU in 1.87292s (18.1535GiB/s) | 20260506 21:58:54.262 | INFO | Analyzing chunk frames = [4765696,4766208) | 20260506 21:58:54.262 | INFO | MemInfo: { | 20260506 21:58:54.262 | INFO | "gpu" : " 17 GB", | 20260506 21:58:54.262 | INFO | "huge" : "", | 20260506 21:58:54.262 | INFO | "huge_shared" : " 27 GB", | 20260506 21:58:54.262 | INFO | "malloc" : " 14 GB", | 20260506 21:58:54.262 | INFO | "pinned" : " 29 GB" | 20260506 21:58:54.262 | INFO | } | 20260506 21:58:54.262 | INFO | Perf report for the last 5.13546 seconds: | 20260506 21:58:54.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:58:54.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:54.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:54.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:54.262 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 21:58:54.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:54.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:54.262 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 21:58:54.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:54.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:54.262 | INFO | Uploaded 36GiB to GPU in 2.00848s (17.924GiB/s) | 20260506 21:58:54.262 | INFO | Downloaded 34GiB to GPU in 1.92175s (17.6922GiB/s) | 20260506 21:58:59.399 | INFO | Analyzing chunk frames = [4766208,4766720) | 20260506 21:58:59.399 | INFO | MemInfo: { | 20260506 21:58:59.399 | INFO | "gpu" : " 17 GB", | 20260506 21:58:59.399 | INFO | "huge" : "", | 20260506 21:58:59.399 | INFO | "huge_shared" : " 27 GB", | 20260506 21:58:59.399 | INFO | "malloc" : " 14 GB", | 20260506 21:58:59.399 | INFO | "pinned" : " 29 GB" | 20260506 21:58:59.399 | INFO | } | 20260506 21:58:59.399 | INFO | Perf report for the last 5.13743 seconds: | 20260506 21:58:59.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:58:59.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:59.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:59.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:59.399 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:58:59.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:59.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:59.399 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 21:58:59.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:59.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:58:59.400 | INFO | Uploaded 36GiB to GPU in 2.00879s (17.9212GiB/s) | 20260506 21:58:59.400 | INFO | Downloaded 34GiB to GPU in 1.94725s (17.4606GiB/s) | 20260506 21:59:04.539 | INFO | Analyzing chunk frames = [4766720,4767232) | 20260506 21:59:04.539 | INFO | MemInfo: { | 20260506 21:59:04.539 | INFO | "gpu" : " 17 GB", | 20260506 21:59:04.539 | INFO | "huge" : "", | 20260506 21:59:04.539 | INFO | "huge_shared" : " 27 GB", | 20260506 21:59:04.539 | INFO | "malloc" : " 14 GB", | 20260506 21:59:04.539 | INFO | "pinned" : " 29 GB" | 20260506 21:59:04.539 | INFO | } | 20260506 21:59:04.539 | INFO | Perf report for the last 5.13935 seconds: | 20260506 21:59:04.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:59:04.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:04.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:04.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:04.539 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:59:04.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:04.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:04.539 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 21:59:04.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:04.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:04.539 | INFO | Uploaded 36GiB to GPU in 1.96753s (18.2971GiB/s) | 20260506 21:59:04.539 | INFO | Downloaded 34GiB to GPU in 1.98964s (17.0885GiB/s) | 20260506 21:59:09.682 | INFO | Analyzing chunk frames = [4767232,4767744) | 20260506 21:59:09.682 | INFO | MemInfo: { | 20260506 21:59:09.682 | INFO | "gpu" : " 17 GB", | 20260506 21:59:09.682 | INFO | "huge" : "", | 20260506 21:59:09.682 | INFO | "huge_shared" : " 27 GB", | 20260506 21:59:09.682 | INFO | "malloc" : " 14 GB", | 20260506 21:59:09.682 | INFO | "pinned" : " 29 GB" | 20260506 21:59:09.682 | INFO | } | 20260506 21:59:09.682 | INFO | Perf report for the last 5.14302 seconds: | 20260506 21:59:09.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:59:09.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:09.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:09.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:09.682 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 21:59:09.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:09.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:09.682 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 21:59:09.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:09.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:09.682 | INFO | Uploaded 36GiB to GPU in 1.99469s (18.0479GiB/s) | 20260506 21:59:09.682 | INFO | Downloaded 34GiB to GPU in 1.98423s (17.1351GiB/s) | 20260506 21:59:09.682 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 21:59:09.682 | INFO | Total mem usage: 18 GiB | 20260506 21:59:09.682 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 21:59:09.682 | INFO | Recent High Water Marks Sum: 15.5 GiB (Savings: -16.5%) | 20260506 21:59:09.682 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 21:59:09.682 | INFO | High Water Breakdown: | 20260506 21:59:09.682 | INFO | 61.4% : 9.49 GiB ( 83.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 21:59:09.682 | INFO | 38.5% : 5.95 GiB ( 88.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 21:59:09.682 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 21:59:09.682 | INFO | ------------------------------------------ | 20260506 21:59:09.682 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:59:09.682 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 21:59:09.682 | INFO | Total mem usage: 26.5 GiB | 20260506 21:59:09.682 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 21:59:09.682 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 21:59:09.682 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 21:59:09.682 | INFO | High Water Breakdown: | 20260506 21:59:09.682 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:59:09.682 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:59:09.682 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:59:09.682 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:59:09.682 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:59:09.682 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:59:09.682 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 21:59:09.682 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 21:59:09.682 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:59:09.682 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 21:59:09.682 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 21:59:09.682 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 21:59:09.682 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 21:59:09.682 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 21:59:09.682 | INFO | ------------------------------------------ | 20260506 21:59:09.682 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:59:09.682 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 21:59:09.683 | INFO | Total mem usage: 25.6 GiB | 20260506 21:59:09.683 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 21:59:09.683 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 21:59:09.683 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 21:59:09.683 | INFO | High Water Breakdown: | 20260506 21:59:09.683 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 21:59:09.683 | INFO | ------------------------------------------ | 20260506 21:59:09.683 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:59:09.683 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 21:59:09.683 | INFO | Total mem usage: 15.7 GiB | 20260506 21:59:09.683 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 21:59:09.683 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 21:59:09.683 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 21:59:09.683 | INFO | High Water Breakdown: | 20260506 21:59:09.683 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 21:59:09.683 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 21:59:09.683 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 21:59:09.683 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 21:59:09.683 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 21:59:09.683 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 21:59:09.683 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 21:59:09.683 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 21:59:09.683 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 21:59:09.683 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 21:59:09.683 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 21:59:09.683 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 21:59:09.683 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 21:59:09.683 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 21:59:09.683 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 21:59:09.683 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 21:59:09.683 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 21:59:09.683 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 21:59:09.683 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 21:59:09.683 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 21:59:09.683 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 21:59:09.683 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 21:59:09.683 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 21:59:09.683 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 21:59:09.683 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 21:59:09.683 | INFO | ------------------------------------------ | 20260506 21:59:09.683 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 21:59:09.683 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 21:59:09.683 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:59:09.683 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:59:09.683 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.59e+01, 1.36e+01 / 2.17e+00 | 20260506 21:59:09.683 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.01e-01, 9.93e-03 / 5.52e-03 | 20260506 21:59:09.683 | INFO | PerfReport: 11.68%, Baseline -- 3.63e+00 / 4.37e+00, 5.63e-01 / 3.82e-01 | 20260506 21:59:09.683 | INFO | PerfReport: 3.28%, Histogram -- 1.02e+00 / 1.46e+00, 2.09e-01 / 1.29e-01 | 20260506 21:59:09.683 | INFO | PerfReport: 16.05%, DME -- 3.99e+01 / 4.46e+01, 4.27e+00 / 2.04e+00 | 20260506 21:59:09.683 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.39e+00 / 1.51e+00 | 20260506 21:59:09.683 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.20e+00 / 2.73e+00, 3.79e-01 / 2.51e-01 | 20260506 21:59:09.683 | INFO | PerfReport: 3.24%, Metrics -- 1.01e+00 / 1.34e+00, 2.72e-01 / 1.54e-01 | 20260506 21:59:09.683 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.45e-01 / 3.88e-01, 5.60e-02 / 2.04e-02 | 20260506 21:59:09.683 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 21:59:09.683 | INFO | PerfReport: Gathered 1536 samples. | 20260506 21:59:09.683 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 21:59:09.683 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 21:59:09.683 | INFO | PerfReport: 21.62%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 21:59:09.683 | INFO | PerfReport: 21.27%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 21:59:09.683 | INFO | PerfReport: 32.20%, Compute -- 3.11e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 21:59:09.683 | INFO | PerfReport: 1.34%, LockUpload -- 1.29e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260506 21:59:09.683 | INFO | PerfReport: 1.17%, LockDownload -- 1.13e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 21:59:09.683 | INFO | PerfReport: 22.38%, LockCompute -- 2.16e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 21:59:09.683 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.33e-02 / 0.00e+00 | 20260506 21:59:09.780 | INFO | Starting a round of Baz IO | 20260506 21:59:12.915 | INFO | Populating RTMetrics report | 20260506 21:59:12.916 | INFO | RTMetrics report populated | 20260506 21:59:12.916 | INFO | Opening tmp RTMetrics json file | 20260506 21:59:13.245 | INFO | Serializing RTMetrics to json | 20260506 21:59:13.300 | INFO | Serialization done, swapping tmp file now... | 20260506 21:59:13.302 | INFO | RTMetrics done | 20260506 21:59:13.601 | INFO | Finished a round of Baz IO: Wrote 1.929 GiB in 3.82 seconds | 20260506 21:59:13.601 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.79% / 0.00% / 66.20% | 20260506 21:59:13.601 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.553878% / 0% / 3.57628e-05% | 20260506 21:59:13.601 | INFO | Baz IO overall: | 20260506 21:59:13.601 | INFO | BytesWritten : 2508403495471 | 20260506 21:59:13.601 | INFO | NumEvents : 1210683951354 | 20260506 21:59:14.819 | INFO | Analyzing chunk frames = [4767744,4768256) | 20260506 21:59:14.819 | INFO | MemInfo: { | 20260506 21:59:14.819 | INFO | "gpu" : " 17 GB", | 20260506 21:59:14.819 | INFO | "huge" : "", | 20260506 21:59:14.819 | INFO | "huge_shared" : " 27 GB", | 20260506 21:59:14.819 | INFO | "malloc" : " 14 GB", | 20260506 21:59:14.819 | INFO | "pinned" : " 29 GB" | 20260506 21:59:14.819 | INFO | } | 20260506 21:59:14.819 | INFO | Perf report for the last 5.13725 seconds: | 20260506 21:59:14.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:59:14.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:14.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:14.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:14.819 | INFO | ANALYSIS : 62.4%, 2.9, 95.6ms, 0 | 20260506 21:59:14.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:14.819 | INFO | RT_METRICS : 50.0%, 1.0, 26.2ms, 0 | 20260506 21:59:14.819 | INFO | PRE_HQ : 59.8%, 1.8, 56.3ms, 0 | 20260506 21:59:14.819 | INFO | BAZWRITER : 74.0%, 4.8, 181.6ms, 0 | 20260506 21:59:14.819 | INFO | BAZ_SAMPLING : 5.6%, 1.0, 2.9ms, 0 | 20260506 21:59:14.819 | INFO | Uploaded 36GiB to GPU in 2.05708s (17.5005GiB/s) | 20260506 21:59:14.819 | INFO | Downloaded 38GiB to GPU in 2.19925s (17.2786GiB/s) | 20260506 21:59:19.956 | INFO | Analyzing chunk frames = [4768256,4768768) | 20260506 21:59:19.956 | INFO | MemInfo: { | 20260506 21:59:19.956 | INFO | "gpu" : " 17 GB", | 20260506 21:59:19.956 | INFO | "huge" : "", | 20260506 21:59:19.956 | INFO | "huge_shared" : " 27 GB", | 20260506 21:59:19.956 | INFO | "malloc" : " 14 GB", | 20260506 21:59:19.956 | INFO | "pinned" : " 29 GB" | 20260506 21:59:19.956 | INFO | } | 20260506 21:59:19.956 | INFO | Perf report for the last 5.13743 seconds: | 20260506 21:59:19.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:59:19.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:19.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:19.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:19.957 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 21:59:19.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:19.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:19.957 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 21:59:19.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:19.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:19.957 | INFO | Uploaded 36GiB to GPU in 2.02366s (17.7896GiB/s) | 20260506 21:59:19.957 | INFO | Downloaded 34GiB to GPU in 1.92394s (17.6721GiB/s) | 20260506 21:59:25.098 | INFO | Analyzing chunk frames = [4768768,4769280) | 20260506 21:59:25.098 | INFO | MemInfo: { | 20260506 21:59:25.098 | INFO | "gpu" : " 17 GB", | 20260506 21:59:25.098 | INFO | "huge" : "", | 20260506 21:59:25.098 | INFO | "huge_shared" : " 27 GB", | 20260506 21:59:25.098 | INFO | "malloc" : " 14 GB", | 20260506 21:59:25.098 | INFO | "pinned" : " 29 GB" | 20260506 21:59:25.098 | INFO | } | 20260506 21:59:25.098 | INFO | Perf report for the last 5.14155 seconds: | 20260506 21:59:25.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:59:25.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:25.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:25.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:25.098 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 21:59:25.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:25.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:25.098 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 21:59:25.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:25.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:25.098 | INFO | Uploaded 36GiB to GPU in 1.98729s (18.1151GiB/s) | 20260506 21:59:25.098 | INFO | Downloaded 34GiB to GPU in 1.91642s (17.7414GiB/s) | 20260506 21:59:30.237 | INFO | Analyzing chunk frames = [4769280,4769792) | 20260506 21:59:30.237 | INFO | MemInfo: { | 20260506 21:59:30.237 | INFO | "gpu" : " 17 GB", | 20260506 21:59:30.237 | INFO | "huge" : "", | 20260506 21:59:30.237 | INFO | "huge_shared" : " 27 GB", | 20260506 21:59:30.237 | INFO | "malloc" : " 14 GB", | 20260506 21:59:30.237 | INFO | "pinned" : " 29 GB" | 20260506 21:59:30.237 | INFO | } | 20260506 21:59:30.237 | INFO | Perf report for the last 5.13872 seconds: | 20260506 21:59:30.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:59:30.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:30.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:30.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:30.237 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 21:59:30.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:30.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:30.237 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 21:59:30.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:30.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:30.237 | INFO | Uploaded 36GiB to GPU in 1.95783s (18.3877GiB/s) | 20260506 21:59:30.237 | INFO | Downloaded 34GiB to GPU in 1.87436s (18.1395GiB/s) | 20260506 21:59:35.374 | INFO | Analyzing chunk frames = [4769792,4770304) | 20260506 21:59:35.374 | INFO | MemInfo: { | 20260506 21:59:35.374 | INFO | "gpu" : " 17 GB", | 20260506 21:59:35.374 | INFO | "huge" : "", | 20260506 21:59:35.374 | INFO | "huge_shared" : " 27 GB", | 20260506 21:59:35.375 | INFO | "malloc" : " 14 GB", | 20260506 21:59:35.375 | INFO | "pinned" : " 29 GB" | 20260506 21:59:35.375 | INFO | } | 20260506 21:59:35.375 | INFO | Perf report for the last 5.13778 seconds: | 20260506 21:59:35.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:59:35.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:35.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:35.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:35.375 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 21:59:35.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:35.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:35.375 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 21:59:35.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:35.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:35.375 | INFO | Uploaded 36GiB to GPU in 2.00462s (17.9585GiB/s) | 20260506 21:59:35.375 | INFO | Downloaded 34GiB to GPU in 1.92108s (17.6984GiB/s) | 20260506 21:59:40.516 | INFO | Analyzing chunk frames = [4770304,4770816) | 20260506 21:59:40.516 | INFO | MemInfo: { | 20260506 21:59:40.516 | INFO | "gpu" : " 17 GB", | 20260506 21:59:40.516 | INFO | "huge" : "", | 20260506 21:59:40.516 | INFO | "huge_shared" : " 27 GB", | 20260506 21:59:40.516 | INFO | "malloc" : " 14 GB", | 20260506 21:59:40.516 | INFO | "pinned" : " 29 GB" | 20260506 21:59:40.516 | INFO | } | 20260506 21:59:40.516 | INFO | Perf report for the last 5.14157 seconds: | 20260506 21:59:40.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:59:40.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:40.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:40.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:40.516 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 21:59:40.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:40.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:40.516 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 21:59:40.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:40.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:40.516 | INFO | Uploaded 36GiB to GPU in 2.01196s (17.893GiB/s) | 20260506 21:59:40.516 | INFO | Downloaded 34GiB to GPU in 1.93756s (17.5478GiB/s) | 20260506 21:59:45.651 | INFO | Analyzing chunk frames = [4770816,4771328) | 20260506 21:59:45.651 | INFO | MemInfo: { | 20260506 21:59:45.651 | INFO | "gpu" : " 17 GB", | 20260506 21:59:45.651 | INFO | "huge" : "", | 20260506 21:59:45.651 | INFO | "huge_shared" : " 27 GB", | 20260506 21:59:45.651 | INFO | "malloc" : " 14 GB", | 20260506 21:59:45.651 | INFO | "pinned" : " 29 GB" | 20260506 21:59:45.651 | INFO | } | 20260506 21:59:45.651 | INFO | Perf report for the last 5.13506 seconds: | 20260506 21:59:45.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:59:45.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:45.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:45.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:45.651 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 21:59:45.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:45.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:45.651 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 21:59:45.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:45.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:45.652 | INFO | Uploaded 36GiB to GPU in 1.96521s (18.3186GiB/s) | 20260506 21:59:45.652 | INFO | Downloaded 34GiB to GPU in 1.99558s (17.0377GiB/s) | 20260506 21:59:50.798 | INFO | Analyzing chunk frames = [4771328,4771840) | 20260506 21:59:50.798 | INFO | MemInfo: { | 20260506 21:59:50.798 | INFO | "gpu" : " 17 GB", | 20260506 21:59:50.798 | INFO | "huge" : "", | 20260506 21:59:50.798 | INFO | "huge_shared" : " 27 GB", | 20260506 21:59:50.798 | INFO | "malloc" : " 14 GB", | 20260506 21:59:50.798 | INFO | "pinned" : " 29 GB" | 20260506 21:59:50.798 | INFO | } | 20260506 21:59:50.798 | INFO | Perf report for the last 5.14705 seconds: | 20260506 21:59:50.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:59:50.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:50.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:50.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:50.798 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 21:59:50.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:50.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:50.799 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 21:59:50.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:50.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:50.799 | INFO | Uploaded 36GiB to GPU in 2.04992s (17.5616GiB/s) | 20260506 21:59:50.799 | INFO | Downloaded 34GiB to GPU in 1.98558s (17.1235GiB/s) | 20260506 21:59:50.904 | INFO | Starting a round of Baz IO | 20260506 21:59:54.033 | INFO | Populating RTMetrics report | 20260506 21:59:54.033 | INFO | RTMetrics report populated | 20260506 21:59:54.033 | INFO | Opening tmp RTMetrics json file | 20260506 21:59:54.034 | INFO | Serializing RTMetrics to json | 20260506 21:59:54.035 | INFO | Serialization done, swapping tmp file now... | 20260506 21:59:54.035 | INFO | RTMetrics done | 20260506 21:59:54.150 | INFO | Finished a round of Baz IO: Wrote 1.927 GiB in 3.245 seconds | 20260506 21:59:54.150 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.79% / 0.00% / 66.21% | 20260506 21:59:54.150 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.551295% / 1.19209e-05% / 5.5631e-05% | 20260506 21:59:54.150 | INFO | Baz IO overall: | 20260506 21:59:54.150 | INFO | BytesWritten : 2510472996029 | 20260506 21:59:54.150 | INFO | NumEvents : 1211673175484 | 20260506 21:59:55.940 | INFO | Analyzing chunk frames = [4771840,4772352) | 20260506 21:59:55.940 | INFO | MemInfo: { | 20260506 21:59:55.940 | INFO | "gpu" : " 17 GB", | 20260506 21:59:55.940 | INFO | "huge" : "", | 20260506 21:59:55.940 | INFO | "huge_shared" : " 27 GB", | 20260506 21:59:55.940 | INFO | "malloc" : " 14 GB", | 20260506 21:59:55.940 | INFO | "pinned" : " 29 GB" | 20260506 21:59:55.940 | INFO | } | 20260506 21:59:55.941 | INFO | Perf report for the last 5.14226 seconds: | 20260506 21:59:55.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 21:59:55.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:55.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:55.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:55.941 | INFO | ANALYSIS : 62.3%, 2.9, 95.4ms, 0 | 20260506 21:59:55.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 21:59:55.941 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 21:59:55.941 | INFO | PRE_HQ : 59.9%, 1.8, 56.8ms, 0 | 20260506 21:59:55.941 | INFO | BAZWRITER : 62.8%, 4.7, 150.6ms, 0 | 20260506 21:59:55.941 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 21:59:55.941 | INFO | Uploaded 36GiB to GPU in 2.05335s (17.5323GiB/s) | 20260506 21:59:55.941 | INFO | Downloaded 38GiB to GPU in 2.20572s (17.2279GiB/s) | 20260506 22:00:01.074 | INFO | Analyzing chunk frames = [4772352,4772864) | 20260506 22:00:01.074 | INFO | MemInfo: { | 20260506 22:00:01.074 | INFO | "gpu" : " 17 GB", | 20260506 22:00:01.074 | INFO | "huge" : "", | 20260506 22:00:01.074 | INFO | "huge_shared" : " 27 GB", | 20260506 22:00:01.074 | INFO | "malloc" : " 14 GB", | 20260506 22:00:01.074 | INFO | "pinned" : " 29 GB" | 20260506 22:00:01.074 | INFO | } | 20260506 22:00:01.074 | INFO | Perf report for the last 5.1333 seconds: | 20260506 22:00:01.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:00:01.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:01.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:01.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:01.074 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:00:01.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:01.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:01.074 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:00:01.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:01.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:01.074 | INFO | Uploaded 36GiB to GPU in 2.00682s (17.9389GiB/s) | 20260506 22:00:01.074 | INFO | Downloaded 34GiB to GPU in 1.92808s (17.6341GiB/s) | 20260506 22:00:06.214 | INFO | Analyzing chunk frames = [4772864,4773376) | 20260506 22:00:06.214 | INFO | MemInfo: { | 20260506 22:00:06.214 | INFO | "gpu" : " 17 GB", | 20260506 22:00:06.214 | INFO | "huge" : "", | 20260506 22:00:06.214 | INFO | "huge_shared" : " 27 GB", | 20260506 22:00:06.214 | INFO | "malloc" : " 14 GB", | 20260506 22:00:06.214 | INFO | "pinned" : " 29 GB" | 20260506 22:00:06.214 | INFO | } | 20260506 22:00:06.214 | INFO | Perf report for the last 5.13995 seconds: | 20260506 22:00:06.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:00:06.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:06.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:06.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:06.214 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 22:00:06.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:06.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:06.214 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:00:06.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:06.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:06.214 | INFO | Uploaded 36GiB to GPU in 1.99548s (18.0408GiB/s) | 20260506 22:00:06.214 | INFO | Downloaded 34GiB to GPU in 1.92139s (17.6955GiB/s) | 20260506 22:00:11.351 | INFO | Analyzing chunk frames = [4773376,4773888) | 20260506 22:00:11.351 | INFO | MemInfo: { | 20260506 22:00:11.351 | INFO | "gpu" : " 17 GB", | 20260506 22:00:11.351 | INFO | "huge" : "", | 20260506 22:00:11.351 | INFO | "huge_shared" : " 27 GB", | 20260506 22:00:11.351 | INFO | "malloc" : " 14 GB", | 20260506 22:00:11.351 | INFO | "pinned" : " 29 GB" | 20260506 22:00:11.351 | INFO | } | 20260506 22:00:11.351 | INFO | Perf report for the last 5.13676 seconds: | 20260506 22:00:11.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:00:11.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:11.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:11.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:11.351 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 22:00:11.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:11.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:11.351 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:00:11.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:11.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:11.351 | INFO | Uploaded 36GiB to GPU in 1.99797s (18.0183GiB/s) | 20260506 22:00:11.352 | INFO | Downloaded 34GiB to GPU in 1.90231s (17.873GiB/s) | 20260506 22:00:16.488 | INFO | Analyzing chunk frames = [4773888,4774400) | 20260506 22:00:16.488 | INFO | MemInfo: { | 20260506 22:00:16.488 | INFO | "gpu" : " 17 GB", | 20260506 22:00:16.488 | INFO | "huge" : "", | 20260506 22:00:16.488 | INFO | "huge_shared" : " 27 GB", | 20260506 22:00:16.488 | INFO | "malloc" : " 14 GB", | 20260506 22:00:16.488 | INFO | "pinned" : " 29 GB" | 20260506 22:00:16.488 | INFO | } | 20260506 22:00:16.488 | INFO | Perf report for the last 5.13703 seconds: | 20260506 22:00:16.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:00:16.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:16.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:16.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:16.488 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 22:00:16.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:16.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:16.488 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:00:16.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:16.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:16.488 | INFO | Uploaded 36GiB to GPU in 1.98824s (18.1065GiB/s) | 20260506 22:00:16.488 | INFO | Downloaded 34GiB to GPU in 1.92183s (17.6915GiB/s) | 20260506 22:00:21.632 | INFO | Analyzing chunk frames = [4774400,4774912) | 20260506 22:00:21.632 | INFO | MemInfo: { | 20260506 22:00:21.632 | INFO | "gpu" : " 17 GB", | 20260506 22:00:21.632 | INFO | "huge" : "", | 20260506 22:00:21.632 | INFO | "huge_shared" : " 27 GB", | 20260506 22:00:21.632 | INFO | "malloc" : " 14 GB", | 20260506 22:00:21.632 | INFO | "pinned" : " 29 GB" | 20260506 22:00:21.632 | INFO | } | 20260506 22:00:21.632 | INFO | Perf report for the last 5.14414 seconds: | 20260506 22:00:21.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:00:21.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:21.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:21.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:21.632 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 22:00:21.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:21.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:21.632 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 22:00:21.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:21.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:21.632 | INFO | Uploaded 36GiB to GPU in 2.01816s (17.8381GiB/s) | 20260506 22:00:21.633 | INFO | Downloaded 34GiB to GPU in 1.95347s (17.4049GiB/s) | 20260506 22:00:26.766 | INFO | Analyzing chunk frames = [4774912,4775424) | 20260506 22:00:26.766 | INFO | MemInfo: { | 20260506 22:00:26.766 | INFO | "gpu" : " 17 GB", | 20260506 22:00:26.766 | INFO | "huge" : "", | 20260506 22:00:26.766 | INFO | "huge_shared" : " 27 GB", | 20260506 22:00:26.766 | INFO | "malloc" : " 14 GB", | 20260506 22:00:26.766 | INFO | "pinned" : " 29 GB" | 20260506 22:00:26.766 | INFO | } | 20260506 22:00:26.766 | INFO | Perf report for the last 5.13402 seconds: | 20260506 22:00:26.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:00:26.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:26.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:26.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:26.766 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 22:00:26.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:26.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:26.766 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 22:00:26.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:26.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:26.766 | INFO | Uploaded 36GiB to GPU in 1.9496s (18.4653GiB/s) | 20260506 22:00:26.766 | INFO | Downloaded 34GiB to GPU in 1.98179s (17.1562GiB/s) | 20260506 22:00:31.909 | INFO | Analyzing chunk frames = [4775424,4775936) | 20260506 22:00:31.909 | INFO | MemInfo: { | 20260506 22:00:31.909 | INFO | "gpu" : " 17 GB", | 20260506 22:00:31.909 | INFO | "huge" : "", | 20260506 22:00:31.909 | INFO | "huge_shared" : " 27 GB", | 20260506 22:00:31.909 | INFO | "malloc" : " 14 GB", | 20260506 22:00:31.909 | INFO | "pinned" : " 29 GB" | 20260506 22:00:31.909 | INFO | } | 20260506 22:00:31.909 | INFO | Perf report for the last 5.14362 seconds: | 20260506 22:00:31.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:00:31.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:31.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:31.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:31.910 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 22:00:31.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:31.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:31.910 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 22:00:31.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:31.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:31.910 | INFO | Uploaded 36GiB to GPU in 2.00912s (17.9183GiB/s) | 20260506 22:00:31.910 | INFO | Downloaded 34GiB to GPU in 1.99793s (17.0176GiB/s) | 20260506 22:00:31.910 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:00:31.910 | INFO | Total mem usage: 18 GiB | 20260506 22:00:31.910 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:00:31.910 | INFO | Recent High Water Marks Sum: 16.3 GiB (Savings: -10.3%) | 20260506 22:00:31.910 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:00:31.910 | INFO | High Water Breakdown: | 20260506 22:00:31.910 | INFO | 62.2% : 10.2 GiB ( 88.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:00:31.910 | INFO | 37.7% : 6.15 GiB ( 92% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:00:31.910 | INFO | 0.122% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:00:31.910 | INFO | ------------------------------------------ | 20260506 22:00:31.910 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:00:31.910 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:00:31.910 | INFO | Total mem usage: 26.5 GiB | 20260506 22:00:31.910 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:00:31.910 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:00:31.910 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:00:31.910 | INFO | High Water Breakdown: | 20260506 22:00:31.910 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:00:31.910 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:00:31.910 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:00:31.910 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:00:31.910 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:00:31.910 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:00:31.910 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:00:31.910 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:00:31.910 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:00:31.910 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:00:31.910 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:00:31.910 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:00:31.910 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:00:31.910 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:00:31.910 | INFO | ------------------------------------------ | 20260506 22:00:31.910 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:00:31.910 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:00:31.911 | INFO | Total mem usage: 25.6 GiB | 20260506 22:00:31.911 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:00:31.911 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:00:31.911 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:00:31.911 | INFO | High Water Breakdown: | 20260506 22:00:31.911 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:00:31.911 | INFO | ------------------------------------------ | 20260506 22:00:31.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:00:31.911 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:00:31.911 | INFO | Total mem usage: 15.7 GiB | 20260506 22:00:31.911 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:00:31.911 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:00:31.911 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:00:31.911 | INFO | High Water Breakdown: | 20260506 22:00:31.911 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:00:31.911 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:00:31.911 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:00:31.911 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:00:31.911 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:00:31.911 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:00:31.911 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:00:31.911 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:00:31.911 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:00:31.911 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:00:31.911 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:00:31.911 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:00:31.911 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:00:31.911 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:00:31.911 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:00:31.911 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:00:31.911 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:00:31.911 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:00:31.911 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:00:31.911 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:00:31.911 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:00:31.911 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:00:31.911 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:00:31.911 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:00:31.911 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:00:31.911 | INFO | ------------------------------------------ | 20260506 22:00:31.911 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:00:31.911 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:00:31.911 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:00:31.911 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:00:31.911 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.60e+01, 1.34e+01 / 2.63e+00 | 20260506 22:00:31.911 | INFO | PerfReport: 0.29%, AggregateStats -- 9.17e-02 / 1.02e-01, 1.01e-02 / 8.58e-03 | 20260506 22:00:31.911 | INFO | PerfReport: 11.67%, Baseline -- 3.63e+00 / 4.36e+00, 5.54e-01 / 3.68e-01 | 20260506 22:00:31.911 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.44e+00, 1.82e-01 / 1.36e-01 | 20260506 22:00:31.911 | INFO | PerfReport: 16.07%, DME -- 4.00e+01 / 4.50e+01, 4.33e+00 / 2.74e+00 | 20260506 22:00:31.911 | INFO | PerfReport: 57.26%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.40e+00 / 1.89e+00 | 20260506 22:00:31.911 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.21e+00 / 2.73e+00, 3.76e-01 / 2.26e-01 | 20260506 22:00:31.911 | INFO | PerfReport: 3.24%, Metrics -- 1.01e+00 / 1.32e+00, 2.66e-01 / 1.52e-01 | 20260506 22:00:31.911 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.45e-01 / 3.93e-01, 5.62e-02 / 1.74e-02 | 20260506 22:00:31.911 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:00:31.911 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:00:31.911 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:00:31.911 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 22:00:31.911 | INFO | PerfReport: 21.60%, Upload -- 2.09e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260506 22:00:31.912 | INFO | PerfReport: 21.26%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 22:00:31.912 | INFO | PerfReport: 32.17%, Compute -- 3.11e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 22:00:31.912 | INFO | PerfReport: 1.33%, LockUpload -- 1.29e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260506 22:00:31.912 | INFO | PerfReport: 1.16%, LockDownload -- 1.12e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 22:00:31.912 | INFO | PerfReport: 22.46%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 22:00:31.912 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.68e-02 / 0.00e+00 | 20260506 22:00:32.028 | INFO | Starting a round of Baz IO | 20260506 22:00:35.155 | INFO | Populating RTMetrics report | 20260506 22:00:35.155 | INFO | RTMetrics report populated | 20260506 22:00:35.155 | INFO | Opening tmp RTMetrics json file | 20260506 22:00:35.232 | INFO | Serializing RTMetrics to json | 20260506 22:00:35.253 | INFO | Serialization done, swapping tmp file now... | 20260506 22:00:35.263 | INFO | RTMetrics done | 20260506 22:00:35.840 | INFO | Finished a round of Baz IO: Wrote 1.927 GiB in 3.812 seconds | 20260506 22:00:35.840 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.79% / 0.00% / 66.21% | 20260506 22:00:35.840 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.5482% / 1.98682e-05% / 8.74201e-05% | 20260506 22:00:35.840 | INFO | Baz IO overall: | 20260506 22:00:35.840 | INFO | BytesWritten : 2512541746508 | 20260506 22:00:35.840 | INFO | NumEvents : 1212661706909 | 20260506 22:00:37.048 | INFO | Analyzing chunk frames = [4775936,4776448) | 20260506 22:00:37.048 | INFO | MemInfo: { | 20260506 22:00:37.048 | INFO | "gpu" : " 17 GB", | 20260506 22:00:37.048 | INFO | "huge" : "", | 20260506 22:00:37.048 | INFO | "huge_shared" : " 27 GB", | 20260506 22:00:37.048 | INFO | "malloc" : " 14 GB", | 20260506 22:00:37.048 | INFO | "pinned" : " 29 GB" | 20260506 22:00:37.048 | INFO | } | 20260506 22:00:37.048 | INFO | Perf report for the last 5.13866 seconds: | 20260506 22:00:37.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:00:37.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:37.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:37.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:37.048 | INFO | ANALYSIS : 62.5%, 2.9, 94.5ms, 0 | 20260506 22:00:37.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:37.048 | INFO | RT_METRICS : 47.1%, 1.0, 24.7ms, 0 | 20260506 22:00:37.048 | INFO | PRE_HQ : 60.7%, 1.9, 60.1ms, 0 | 20260506 22:00:37.048 | INFO | BAZWRITER : 73.9%, 4.8, 181.0ms, 0 | 20260506 22:00:37.048 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 22:00:37.049 | INFO | Uploaded 36GiB to GPU in 2.08223s (17.2892GiB/s) | 20260506 22:00:37.049 | INFO | Downloaded 38GiB to GPU in 2.19842s (17.2851GiB/s) | 20260506 22:00:42.191 | INFO | Analyzing chunk frames = [4776448,4776960) | 20260506 22:00:42.191 | INFO | MemInfo: { | 20260506 22:00:42.191 | INFO | "gpu" : " 17 GB", | 20260506 22:00:42.191 | INFO | "huge" : "", | 20260506 22:00:42.191 | INFO | "huge_shared" : " 27 GB", | 20260506 22:00:42.191 | INFO | "malloc" : " 14 GB", | 20260506 22:00:42.191 | INFO | "pinned" : " 29 GB" | 20260506 22:00:42.191 | INFO | } | 20260506 22:00:42.192 | INFO | Perf report for the last 5.14342 seconds: | 20260506 22:00:42.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:00:42.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:42.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:42.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:42.192 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 22:00:42.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:42.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:42.192 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:00:42.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:42.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:42.192 | INFO | Uploaded 36GiB to GPU in 2.00599s (17.9462GiB/s) | 20260506 22:00:42.192 | INFO | Downloaded 34GiB to GPU in 1.93588s (17.5631GiB/s) | 20260506 22:00:47.328 | INFO | Analyzing chunk frames = [4776960,4777472) | 20260506 22:00:47.328 | INFO | MemInfo: { | 20260506 22:00:47.328 | INFO | "gpu" : " 17 GB", | 20260506 22:00:47.328 | INFO | "huge" : "", | 20260506 22:00:47.328 | INFO | "huge_shared" : " 27 GB", | 20260506 22:00:47.328 | INFO | "malloc" : " 14 GB", | 20260506 22:00:47.328 | INFO | "pinned" : " 29 GB" | 20260506 22:00:47.328 | INFO | } | 20260506 22:00:47.328 | INFO | Perf report for the last 5.13661 seconds: | 20260506 22:00:47.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:00:47.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:47.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:47.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:47.328 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 22:00:47.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:47.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:47.328 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:00:47.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:47.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:47.329 | INFO | Uploaded 36GiB to GPU in 1.99428s (18.0517GiB/s) | 20260506 22:00:47.329 | INFO | Downloaded 34GiB to GPU in 1.92987s (17.6177GiB/s) | 20260506 22:00:52.467 | INFO | Analyzing chunk frames = [4777472,4777984) | 20260506 22:00:52.467 | INFO | MemInfo: { | 20260506 22:00:52.467 | INFO | "gpu" : " 17 GB", | 20260506 22:00:52.467 | INFO | "huge" : "", | 20260506 22:00:52.467 | INFO | "huge_shared" : " 27 GB", | 20260506 22:00:52.467 | INFO | "malloc" : " 14 GB", | 20260506 22:00:52.467 | INFO | "pinned" : " 29 GB" | 20260506 22:00:52.467 | INFO | } | 20260506 22:00:52.467 | INFO | Perf report for the last 5.13871 seconds: | 20260506 22:00:52.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:00:52.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:52.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:52.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:52.467 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 22:00:52.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:52.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:52.467 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 22:00:52.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:52.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:52.467 | INFO | Uploaded 36GiB to GPU in 2.0232s (17.7936GiB/s) | 20260506 22:00:52.467 | INFO | Downloaded 34GiB to GPU in 1.90837s (17.8162GiB/s) | 20260506 22:00:57.606 | INFO | Analyzing chunk frames = [4777984,4778496) | 20260506 22:00:57.606 | INFO | MemInfo: { | 20260506 22:00:57.606 | INFO | "gpu" : " 17 GB", | 20260506 22:00:57.606 | INFO | "huge" : "", | 20260506 22:00:57.606 | INFO | "huge_shared" : " 27 GB", | 20260506 22:00:57.606 | INFO | "malloc" : " 14 GB", | 20260506 22:00:57.606 | INFO | "pinned" : " 29 GB" | 20260506 22:00:57.606 | INFO | } | 20260506 22:00:57.607 | INFO | Perf report for the last 5.13979 seconds: | 20260506 22:00:57.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:00:57.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:57.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:57.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:57.607 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 22:00:57.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:57.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:57.607 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:00:57.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:57.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:00:57.607 | INFO | Uploaded 36GiB to GPU in 1.99021s (18.0885GiB/s) | 20260506 22:00:57.607 | INFO | Downloaded 34GiB to GPU in 1.92348s (17.6763GiB/s) | 20260506 22:01:02.746 | INFO | Analyzing chunk frames = [4778496,4779008) | 20260506 22:01:02.746 | INFO | MemInfo: { | 20260506 22:01:02.746 | INFO | "gpu" : " 17 GB", | 20260506 22:01:02.746 | INFO | "huge" : "", | 20260506 22:01:02.746 | INFO | "huge_shared" : " 27 GB", | 20260506 22:01:02.746 | INFO | "malloc" : " 14 GB", | 20260506 22:01:02.746 | INFO | "pinned" : " 29 GB" | 20260506 22:01:02.746 | INFO | } | 20260506 22:01:02.746 | INFO | Perf report for the last 5.13903 seconds: | 20260506 22:01:02.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:01:02.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:02.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:02.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:02.746 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 22:01:02.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:02.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:02.746 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:01:02.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:02.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:02.746 | INFO | Uploaded 36GiB to GPU in 2.0118s (17.8944GiB/s) | 20260506 22:01:02.746 | INFO | Downloaded 34GiB to GPU in 1.93941s (17.5311GiB/s) | 20260506 22:01:07.887 | INFO | Analyzing chunk frames = [4779008,4779520) | 20260506 22:01:07.887 | INFO | MemInfo: { | 20260506 22:01:07.887 | INFO | "gpu" : " 17 GB", | 20260506 22:01:07.887 | INFO | "huge" : "", | 20260506 22:01:07.887 | INFO | "huge_shared" : " 27 GB", | 20260506 22:01:07.887 | INFO | "malloc" : " 14 GB", | 20260506 22:01:07.887 | INFO | "pinned" : " 29 GB" | 20260506 22:01:07.887 | INFO | } | 20260506 22:01:07.887 | INFO | Perf report for the last 5.14126 seconds: | 20260506 22:01:07.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:01:07.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:07.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:07.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:07.887 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 22:01:07.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:07.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:07.887 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:01:07.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:07.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:07.888 | INFO | Uploaded 36GiB to GPU in 1.98898s (18.0997GiB/s) | 20260506 22:01:07.888 | INFO | Downloaded 34GiB to GPU in 1.99216s (17.0669GiB/s) | 20260506 22:01:13.029 | INFO | Analyzing chunk frames = [4779520,4780032) | 20260506 22:01:13.029 | INFO | MemInfo: { | 20260506 22:01:13.029 | INFO | "gpu" : " 17 GB", | 20260506 22:01:13.029 | INFO | "huge" : "", | 20260506 22:01:13.029 | INFO | "huge_shared" : " 27 GB", | 20260506 22:01:13.029 | INFO | "malloc" : " 14 GB", | 20260506 22:01:13.029 | INFO | "pinned" : " 29 GB" | 20260506 22:01:13.029 | INFO | } | 20260506 22:01:13.029 | INFO | Perf report for the last 5.14186 seconds: | 20260506 22:01:13.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:01:13.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:13.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:13.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:13.029 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 22:01:13.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:13.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:13.029 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 22:01:13.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:13.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:13.029 | INFO | Uploaded 36GiB to GPU in 2.05073s (17.5547GiB/s) | 20260506 22:01:13.029 | INFO | Downloaded 34GiB to GPU in 1.99916s (17.0071GiB/s) | 20260506 22:01:13.145 | INFO | Starting a round of Baz IO | 20260506 22:01:16.263 | INFO | Populating RTMetrics report | 20260506 22:01:16.263 | INFO | RTMetrics report populated | 20260506 22:01:16.263 | INFO | Opening tmp RTMetrics json file | 20260506 22:01:16.366 | INFO | Serializing RTMetrics to json | 20260506 22:01:16.372 | INFO | Serialization done, swapping tmp file now... | 20260506 22:01:16.376 | INFO | RTMetrics done | 20260506 22:01:16.843 | INFO | Finished a round of Baz IO: Wrote 1.933 GiB in 3.698 seconds | 20260506 22:01:16.843 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.78% / 0.00% / 66.21% | 20260506 22:01:16.843 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.554649% / 0% / 6.75519e-05% | 20260506 22:01:16.843 | INFO | Baz IO overall: | 20260506 22:01:16.843 | INFO | BytesWritten : 2514617478121 | 20260506 22:01:16.843 | INFO | NumEvents : 1213654460871 | 20260506 22:01:18.162 | INFO | Analyzing chunk frames = [4780032,4780544) | 20260506 22:01:18.162 | INFO | MemInfo: { | 20260506 22:01:18.162 | INFO | "gpu" : " 17 GB", | 20260506 22:01:18.162 | INFO | "huge" : "", | 20260506 22:01:18.162 | INFO | "huge_shared" : " 27 GB", | 20260506 22:01:18.162 | INFO | "malloc" : " 14 GB", | 20260506 22:01:18.162 | INFO | "pinned" : " 29 GB" | 20260506 22:01:18.162 | INFO | } | 20260506 22:01:18.162 | INFO | Perf report for the last 5.13323 seconds: | 20260506 22:01:18.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:01:18.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:18.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:18.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:18.162 | INFO | ANALYSIS : 62.4%, 2.9, 95.2ms, 0 | 20260506 22:01:18.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:18.162 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260506 22:01:18.162 | INFO | PRE_HQ : 59.6%, 1.8, 57.0ms, 0 | 20260506 22:01:18.162 | INFO | BAZWRITER : 71.7%, 4.8, 177.0ms, 0 | 20260506 22:01:18.162 | INFO | BAZ_SAMPLING : 2.2%, 1.0, 1.1ms, 0 | 20260506 22:01:18.162 | INFO | Uploaded 36GiB to GPU in 2.04332s (17.6183GiB/s) | 20260506 22:01:18.163 | INFO | Downloaded 38GiB to GPU in 2.21897s (17.125GiB/s) | 20260506 22:01:23.303 | INFO | Analyzing chunk frames = [4780544,4781056) | 20260506 22:01:23.303 | INFO | MemInfo: { | 20260506 22:01:23.303 | INFO | "gpu" : " 17 GB", | 20260506 22:01:23.303 | INFO | "huge" : "", | 20260506 22:01:23.303 | INFO | "huge_shared" : " 27 GB", | 20260506 22:01:23.303 | INFO | "malloc" : " 14 GB", | 20260506 22:01:23.303 | INFO | "pinned" : " 29 GB" | 20260506 22:01:23.303 | INFO | } | 20260506 22:01:23.303 | INFO | Perf report for the last 5.14088 seconds: | 20260506 22:01:23.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:01:23.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:23.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:23.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:23.303 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:01:23.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:23.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:23.303 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:01:23.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:23.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:23.303 | INFO | Uploaded 36GiB to GPU in 2.00138s (17.9876GiB/s) | 20260506 22:01:23.303 | INFO | Downloaded 34GiB to GPU in 1.92536s (17.6591GiB/s) | 20260506 22:01:28.440 | INFO | Analyzing chunk frames = [4781056,4781568) | 20260506 22:01:28.440 | INFO | MemInfo: { | 20260506 22:01:28.440 | INFO | "gpu" : " 17 GB", | 20260506 22:01:28.440 | INFO | "huge" : "", | 20260506 22:01:28.440 | INFO | "huge_shared" : " 27 GB", | 20260506 22:01:28.440 | INFO | "malloc" : " 14 GB", | 20260506 22:01:28.440 | INFO | "pinned" : " 29 GB" | 20260506 22:01:28.440 | INFO | } | 20260506 22:01:28.440 | INFO | Perf report for the last 5.13687 seconds: | 20260506 22:01:28.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:01:28.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:28.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:28.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:28.440 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 22:01:28.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:28.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:28.440 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:01:28.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:28.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:28.440 | INFO | Uploaded 36GiB to GPU in 1.97711s (18.2084GiB/s) | 20260506 22:01:28.440 | INFO | Downloaded 34GiB to GPU in 1.92335s (17.6775GiB/s) | 20260506 22:01:33.575 | INFO | Analyzing chunk frames = [4781568,4782080) | 20260506 22:01:33.575 | INFO | MemInfo: { | 20260506 22:01:33.575 | INFO | "gpu" : " 17 GB", | 20260506 22:01:33.575 | INFO | "huge" : "", | 20260506 22:01:33.575 | INFO | "huge_shared" : " 27 GB", | 20260506 22:01:33.575 | INFO | "malloc" : " 14 GB", | 20260506 22:01:33.575 | INFO | "pinned" : " 29 GB" | 20260506 22:01:33.575 | INFO | } | 20260506 22:01:33.575 | INFO | Perf report for the last 5.13512 seconds: | 20260506 22:01:33.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:01:33.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:33.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:33.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:33.575 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 22:01:33.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:33.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:33.575 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 22:01:33.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:33.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:33.575 | INFO | Uploaded 36GiB to GPU in 1.94493s (18.5097GiB/s) | 20260506 22:01:33.576 | INFO | Downloaded 34GiB to GPU in 1.87309s (18.1518GiB/s) | 20260506 22:01:38.722 | INFO | Analyzing chunk frames = [4782080,4782592) | 20260506 22:01:38.722 | INFO | MemInfo: { | 20260506 22:01:38.722 | INFO | "gpu" : " 17 GB", | 20260506 22:01:38.722 | INFO | "huge" : "", | 20260506 22:01:38.722 | INFO | "huge_shared" : " 27 GB", | 20260506 22:01:38.722 | INFO | "malloc" : " 14 GB", | 20260506 22:01:38.722 | INFO | "pinned" : " 29 GB" | 20260506 22:01:38.722 | INFO | } | 20260506 22:01:38.722 | INFO | Perf report for the last 5.14736 seconds: | 20260506 22:01:38.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:01:38.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:38.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:38.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:38.722 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260506 22:01:38.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:38.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:38.723 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:01:38.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:38.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:38.723 | INFO | Uploaded 36GiB to GPU in 2.01977s (17.8238GiB/s) | 20260506 22:01:38.723 | INFO | Downloaded 34GiB to GPU in 1.91061s (17.7953GiB/s) | 20260506 22:01:43.863 | INFO | Analyzing chunk frames = [4782592,4783104) | 20260506 22:01:43.864 | INFO | MemInfo: { | 20260506 22:01:43.864 | INFO | "gpu" : " 17 GB", | 20260506 22:01:43.864 | INFO | "huge" : "", | 20260506 22:01:43.864 | INFO | "huge_shared" : " 27 GB", | 20260506 22:01:43.864 | INFO | "malloc" : " 14 GB", | 20260506 22:01:43.864 | INFO | "pinned" : " 29 GB" | 20260506 22:01:43.864 | INFO | } | 20260506 22:01:43.864 | INFO | Perf report for the last 5.14119 seconds: | 20260506 22:01:43.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:01:43.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:43.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:43.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:43.864 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 22:01:43.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:43.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:43.864 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 22:01:43.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:43.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:43.864 | INFO | Uploaded 36GiB to GPU in 2.00617s (17.9446GiB/s) | 20260506 22:01:43.864 | INFO | Downloaded 34GiB to GPU in 1.94935s (17.4417GiB/s) | 20260506 22:01:49.002 | INFO | Analyzing chunk frames = [4783104,4783616) | 20260506 22:01:49.002 | INFO | MemInfo: { | 20260506 22:01:49.002 | INFO | "gpu" : " 17 GB", | 20260506 22:01:49.002 | INFO | "huge" : "", | 20260506 22:01:49.003 | INFO | "huge_shared" : " 27 GB", | 20260506 22:01:49.003 | INFO | "malloc" : " 14 GB", | 20260506 22:01:49.003 | INFO | "pinned" : " 29 GB" | 20260506 22:01:49.003 | INFO | } | 20260506 22:01:49.003 | INFO | Perf report for the last 5.13896 seconds: | 20260506 22:01:49.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:01:49.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:49.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:49.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:49.003 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 22:01:49.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:49.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:49.003 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 22:01:49.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:49.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:49.003 | INFO | Uploaded 36GiB to GPU in 1.95326s (18.4307GiB/s) | 20260506 22:01:49.003 | INFO | Downloaded 34GiB to GPU in 1.97688s (17.1988GiB/s) | 20260506 22:01:54.136 | INFO | Analyzing chunk frames = [4783616,4784128) | 20260506 22:01:54.136 | INFO | MemInfo: { | 20260506 22:01:54.136 | INFO | "gpu" : " 17 GB", | 20260506 22:01:54.136 | INFO | "huge" : "", | 20260506 22:01:54.136 | INFO | "huge_shared" : " 27 GB", | 20260506 22:01:54.136 | INFO | "malloc" : " 14 GB", | 20260506 22:01:54.136 | INFO | "pinned" : " 29 GB" | 20260506 22:01:54.136 | INFO | } | 20260506 22:01:54.136 | INFO | Perf report for the last 5.13332 seconds: | 20260506 22:01:54.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:01:54.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:54.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:54.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:54.136 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:01:54.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:54.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:54.136 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 22:01:54.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:54.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:54.136 | INFO | Uploaded 36GiB to GPU in 1.98438s (18.1417GiB/s) | 20260506 22:01:54.136 | INFO | Downloaded 34GiB to GPU in 1.97613s (17.2053GiB/s) | 20260506 22:01:54.136 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:01:54.136 | INFO | Total mem usage: 18 GiB | 20260506 22:01:54.136 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:01:54.136 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.6%) | 20260506 22:01:54.136 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:01:54.136 | INFO | High Water Breakdown: | 20260506 22:01:54.136 | INFO | 61.9% : 9.9 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:01:54.136 | INFO | 38% : 6.08 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:01:54.136 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:01:54.136 | INFO | ------------------------------------------ | 20260506 22:01:54.136 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:01:54.136 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:01:54.136 | INFO | Total mem usage: 26.5 GiB | 20260506 22:01:54.136 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:01:54.136 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 22:01:54.136 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 22:01:54.136 | INFO | High Water Breakdown: | 20260506 22:01:54.136 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:01:54.136 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:01:54.136 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:01:54.136 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:01:54.136 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:01:54.136 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:01:54.136 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:01:54.136 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:01:54.136 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:01:54.136 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:01:54.136 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:01:54.136 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:01:54.137 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:01:54.137 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:01:54.137 | INFO | ------------------------------------------ | 20260506 22:01:54.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:01:54.137 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:01:54.137 | INFO | Total mem usage: 25.6 GiB | 20260506 22:01:54.137 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:01:54.137 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:01:54.137 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:01:54.137 | INFO | High Water Breakdown: | 20260506 22:01:54.137 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:01:54.137 | INFO | ------------------------------------------ | 20260506 22:01:54.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:01:54.137 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:01:54.137 | INFO | Total mem usage: 15.7 GiB | 20260506 22:01:54.137 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:01:54.137 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:01:54.137 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260506 22:01:54.137 | INFO | High Water Breakdown: | 20260506 22:01:54.137 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:01:54.137 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:01:54.137 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:01:54.137 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:01:54.137 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:01:54.137 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:01:54.137 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:01:54.137 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:01:54.137 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:01:54.137 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:01:54.137 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:01:54.137 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:01:54.137 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:01:54.137 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:01:54.137 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:01:54.137 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:01:54.137 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:01:54.137 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:01:54.137 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:01:54.137 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:01:54.137 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:01:54.137 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:01:54.137 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:01:54.137 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:01:54.137 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:01:54.137 | INFO | ------------------------------------------ | 20260506 22:01:54.137 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:01:54.137 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:01:54.137 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:01:54.137 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:01:54.137 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.58e+01, 1.34e+01 / 2.75e+00 | 20260506 22:01:54.137 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 9.94e-02, 9.63e-03 / 6.18e-03 | 20260506 22:01:54.137 | INFO | PerfReport: 11.63%, Baseline -- 3.62e+00 / 4.36e+00, 5.47e-01 / 3.63e-01 | 20260506 22:01:54.137 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.45e+00, 1.83e-01 / 1.42e-01 | 20260506 22:01:54.137 | INFO | PerfReport: 16.01%, DME -- 3.98e+01 / 4.42e+01, 4.14e+00 / 3.20e+00 | 20260506 22:01:54.137 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.78e+01 / 2.12e+01, 1.43e+00 / 1.94e+00 | 20260506 22:01:54.137 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.20e+00 / 2.79e+00, 3.76e-01 / 2.09e-01 | 20260506 22:01:54.137 | INFO | PerfReport: 3.24%, Metrics -- 1.01e+00 / 1.32e+00, 2.67e-01 / 1.44e-01 | 20260506 22:01:54.137 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.46e-01 / 3.93e-01, 5.55e-02 / 2.87e-02 | 20260506 22:01:54.137 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:01:54.137 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:01:54.137 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:01:54.137 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 22:01:54.137 | INFO | PerfReport: 21.63%, Upload -- 2.09e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260506 22:01:54.137 | INFO | PerfReport: 21.29%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 22:01:54.137 | INFO | PerfReport: 32.22%, Compute -- 3.11e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 22:01:54.137 | INFO | PerfReport: 1.32%, LockUpload -- 1.27e+00 / 0.00e+00, 4.63e+00 / 0.00e+00 | 20260506 22:01:54.137 | INFO | PerfReport: 1.13%, LockDownload -- 1.10e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260506 22:01:54.137 | INFO | PerfReport: 22.39%, LockCompute -- 2.16e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 22:01:54.137 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260506 22:01:54.236 | INFO | Starting a round of Baz IO | 20260506 22:01:57.363 | INFO | Populating RTMetrics report | 20260506 22:01:57.364 | INFO | RTMetrics report populated | 20260506 22:01:57.364 | INFO | Opening tmp RTMetrics json file | 20260506 22:01:57.368 | INFO | Serializing RTMetrics to json | 20260506 22:01:57.400 | INFO | Serialization done, swapping tmp file now... | 20260506 22:01:57.411 | INFO | RTMetrics done | 20260506 22:01:57.741 | INFO | Finished a round of Baz IO: Wrote 1.933 GiB in 3.505 seconds | 20260506 22:01:57.741 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.78% / 0.00% / 66.22% | 20260506 22:01:57.741 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.554403% / 7.94729e-06% / 8.34465e-05% | 20260506 22:01:57.741 | INFO | Baz IO overall: | 20260506 22:01:57.741 | INFO | BytesWritten : 2516692903532 | 20260506 22:01:57.741 | INFO | NumEvents : 1214647001801 | 20260506 22:01:59.275 | INFO | Analyzing chunk frames = [4784128,4784640) | 20260506 22:01:59.275 | INFO | MemInfo: { | 20260506 22:01:59.275 | INFO | "gpu" : " 17 GB", | 20260506 22:01:59.275 | INFO | "huge" : "", | 20260506 22:01:59.275 | INFO | "huge_shared" : " 27 GB", | 20260506 22:01:59.275 | INFO | "malloc" : " 14 GB", | 20260506 22:01:59.275 | INFO | "pinned" : " 29 GB" | 20260506 22:01:59.275 | INFO | } | 20260506 22:01:59.275 | INFO | Perf report for the last 5.1395 seconds: | 20260506 22:01:59.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:01:59.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:59.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:59.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:59.276 | INFO | ANALYSIS : 62.2%, 2.9, 95.1ms, 0 | 20260506 22:01:59.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:01:59.276 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260506 22:01:59.276 | INFO | PRE_HQ : 59.9%, 1.8, 56.8ms, 0 | 20260506 22:01:59.276 | INFO | BAZWRITER : 67.8%, 4.8, 167.0ms, 0 | 20260506 22:01:59.276 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 22:01:59.276 | INFO | Uploaded 36GiB to GPU in 2.0649s (17.4342GiB/s) | 20260506 22:01:59.276 | INFO | Downloaded 38GiB to GPU in 2.19885s (17.2818GiB/s) | 20260506 22:02:04.415 | INFO | Analyzing chunk frames = [4784640,4785152) | 20260506 22:02:04.415 | INFO | MemInfo: { | 20260506 22:02:04.415 | INFO | "gpu" : " 17 GB", | 20260506 22:02:04.415 | INFO | "huge" : "", | 20260506 22:02:04.415 | INFO | "huge_shared" : " 27 GB", | 20260506 22:02:04.415 | INFO | "malloc" : " 14 GB", | 20260506 22:02:04.415 | INFO | "pinned" : " 29 GB" | 20260506 22:02:04.415 | INFO | } | 20260506 22:02:04.415 | INFO | Perf report for the last 5.13994 seconds: | 20260506 22:02:04.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:02:04.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:04.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:04.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:04.415 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 22:02:04.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:04.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:04.415 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:02:04.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:04.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:04.416 | INFO | Uploaded 36GiB to GPU in 2.02167s (17.8071GiB/s) | 20260506 22:02:04.416 | INFO | Downloaded 34GiB to GPU in 1.92869s (17.6286GiB/s) | 20260506 22:02:09.556 | INFO | Analyzing chunk frames = [4785152,4785664) | 20260506 22:02:09.556 | INFO | MemInfo: { | 20260506 22:02:09.556 | INFO | "gpu" : " 17 GB", | 20260506 22:02:09.556 | INFO | "huge" : "", | 20260506 22:02:09.556 | INFO | "huge_shared" : " 27 GB", | 20260506 22:02:09.556 | INFO | "malloc" : " 14 GB", | 20260506 22:02:09.556 | INFO | "pinned" : " 29 GB" | 20260506 22:02:09.556 | INFO | } | 20260506 22:02:09.556 | INFO | Perf report for the last 5.14072 seconds: | 20260506 22:02:09.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:02:09.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:09.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:09.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:09.556 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 22:02:09.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:09.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:09.556 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 22:02:09.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:09.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:09.556 | INFO | Uploaded 36GiB to GPU in 1.98839s (18.1051GiB/s) | 20260506 22:02:09.556 | INFO | Downloaded 34GiB to GPU in 1.92544s (17.6583GiB/s) | 20260506 22:02:14.700 | INFO | Analyzing chunk frames = [4785664,4786176) | 20260506 22:02:14.700 | INFO | MemInfo: { | 20260506 22:02:14.700 | INFO | "gpu" : " 17 GB", | 20260506 22:02:14.700 | INFO | "huge" : "", | 20260506 22:02:14.700 | INFO | "huge_shared" : " 27 GB", | 20260506 22:02:14.700 | INFO | "malloc" : " 14 GB", | 20260506 22:02:14.700 | INFO | "pinned" : " 29 GB" | 20260506 22:02:14.700 | INFO | } | 20260506 22:02:14.701 | INFO | Perf report for the last 5.14438 seconds: | 20260506 22:02:14.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:02:14.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:14.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:14.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:14.701 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260506 22:02:14.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:14.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:14.701 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 22:02:14.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:14.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:14.701 | INFO | Uploaded 36GiB to GPU in 1.97906s (18.1905GiB/s) | 20260506 22:02:14.701 | INFO | Downloaded 34GiB to GPU in 1.87534s (18.13GiB/s) | 20260506 22:02:19.835 | INFO | Analyzing chunk frames = [4786176,4786688) | 20260506 22:02:19.835 | INFO | MemInfo: { | 20260506 22:02:19.835 | INFO | "gpu" : " 17 GB", | 20260506 22:02:19.835 | INFO | "huge" : "", | 20260506 22:02:19.835 | INFO | "huge_shared" : " 27 GB", | 20260506 22:02:19.835 | INFO | "malloc" : " 14 GB", | 20260506 22:02:19.835 | INFO | "pinned" : " 29 GB" | 20260506 22:02:19.835 | INFO | } | 20260506 22:02:19.835 | INFO | Perf report for the last 5.13485 seconds: | 20260506 22:02:19.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:02:19.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:19.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:19.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:19.835 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 22:02:19.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:19.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:19.835 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:02:19.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:19.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:19.836 | INFO | Uploaded 36GiB to GPU in 2.01376s (17.877GiB/s) | 20260506 22:02:19.836 | INFO | Downloaded 34GiB to GPU in 1.91865s (17.7208GiB/s) | 20260506 22:02:24.979 | INFO | Analyzing chunk frames = [4786688,4787200) | 20260506 22:02:24.979 | INFO | MemInfo: { | 20260506 22:02:24.979 | INFO | "gpu" : " 17 GB", | 20260506 22:02:24.979 | INFO | "huge" : "", | 20260506 22:02:24.979 | INFO | "huge_shared" : " 27 GB", | 20260506 22:02:24.979 | INFO | "malloc" : " 14 GB", | 20260506 22:02:24.979 | INFO | "pinned" : " 29 GB" | 20260506 22:02:24.979 | INFO | } | 20260506 22:02:24.979 | INFO | Perf report for the last 5.14344 seconds: | 20260506 22:02:24.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:02:24.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:24.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:24.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:24.979 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 22:02:24.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:24.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:24.979 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 22:02:24.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:24.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:24.979 | INFO | Uploaded 36GiB to GPU in 2.00939s (17.9159GiB/s) | 20260506 22:02:24.979 | INFO | Downloaded 34GiB to GPU in 1.9495s (17.4403GiB/s) | 20260506 22:02:30.115 | INFO | Analyzing chunk frames = [4787200,4787712) | 20260506 22:02:30.115 | INFO | MemInfo: { | 20260506 22:02:30.115 | INFO | "gpu" : " 17 GB", | 20260506 22:02:30.115 | INFO | "huge" : "", | 20260506 22:02:30.115 | INFO | "huge_shared" : " 27 GB", | 20260506 22:02:30.115 | INFO | "malloc" : " 14 GB", | 20260506 22:02:30.115 | INFO | "pinned" : " 29 GB" | 20260506 22:02:30.115 | INFO | } | 20260506 22:02:30.115 | INFO | Perf report for the last 5.13588 seconds: | 20260506 22:02:30.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:02:30.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:30.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:30.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:30.115 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 22:02:30.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:30.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:30.115 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 22:02:30.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:30.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:30.115 | INFO | Uploaded 36GiB to GPU in 1.98138s (18.1692GiB/s) | 20260506 22:02:30.115 | INFO | Downloaded 34GiB to GPU in 1.98941s (17.0905GiB/s) | 20260506 22:02:35.250 | INFO | Analyzing chunk frames = [4787712,4788224) | 20260506 22:02:35.250 | INFO | MemInfo: { | 20260506 22:02:35.250 | INFO | "gpu" : " 17 GB", | 20260506 22:02:35.250 | INFO | "huge" : "", | 20260506 22:02:35.250 | INFO | "huge_shared" : " 27 GB", | 20260506 22:02:35.250 | INFO | "malloc" : " 14 GB", | 20260506 22:02:35.250 | INFO | "pinned" : " 29 GB" | 20260506 22:02:35.250 | INFO | } | 20260506 22:02:35.250 | INFO | Perf report for the last 5.13547 seconds: | 20260506 22:02:35.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:02:35.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:35.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:35.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:35.250 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:02:35.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:35.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:35.250 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 22:02:35.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:35.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:35.251 | INFO | Uploaded 36GiB to GPU in 1.98238s (18.16GiB/s) | 20260506 22:02:35.251 | INFO | Downloaded 34GiB to GPU in 1.98407s (17.1365GiB/s) | 20260506 22:02:35.349 | INFO | Starting a round of Baz IO | 20260506 22:02:38.483 | INFO | Populating RTMetrics report | 20260506 22:02:38.484 | INFO | RTMetrics report populated | 20260506 22:02:38.484 | INFO | Opening tmp RTMetrics json file | 20260506 22:02:38.535 | INFO | Serializing RTMetrics to json | 20260506 22:02:38.567 | INFO | Serialization done, swapping tmp file now... | 20260506 22:02:38.574 | INFO | RTMetrics done | 20260506 22:02:38.762 | INFO | Finished a round of Baz IO: Wrote 1.929 GiB in 3.412 seconds | 20260506 22:02:38.762 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.78% / 0.00% / 66.22% | 20260506 22:02:38.762 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.55031% / 3.97364e-06% / 6.35783e-05% | 20260506 22:02:38.762 | INFO | Baz IO overall: | 20260506 22:02:38.762 | INFO | BytesWritten : 2518764112837 | 20260506 22:02:38.762 | INFO | NumEvents : 1215636958888 | 20260506 22:02:40.388 | INFO | Analyzing chunk frames = [4788224,4788736) | 20260506 22:02:40.388 | INFO | MemInfo: { | 20260506 22:02:40.388 | INFO | "gpu" : " 17 GB", | 20260506 22:02:40.388 | INFO | "huge" : "", | 20260506 22:02:40.388 | INFO | "huge_shared" : " 27 GB", | 20260506 22:02:40.388 | INFO | "malloc" : " 14 GB", | 20260506 22:02:40.388 | INFO | "pinned" : " 29 GB" | 20260506 22:02:40.388 | INFO | } | 20260506 22:02:40.388 | INFO | Perf report for the last 5.13801 seconds: | 20260506 22:02:40.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:02:40.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:40.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:40.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:40.388 | INFO | ANALYSIS : 62.4%, 2.9, 95.0ms, 0 | 20260506 22:02:40.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:40.388 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260506 22:02:40.388 | INFO | PRE_HQ : 60.0%, 1.8, 57.3ms, 0 | 20260506 22:02:40.388 | INFO | BAZWRITER : 66.0%, 4.6, 155.3ms, 0 | 20260506 22:02:40.388 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 22:02:40.388 | INFO | Uploaded 36GiB to GPU in 2.04045s (17.6431GiB/s) | 20260506 22:02:40.388 | INFO | Downloaded 38GiB to GPU in 2.18814s (17.3663GiB/s) | 20260506 22:02:45.534 | INFO | Analyzing chunk frames = [4788736,4789248) | 20260506 22:02:45.534 | INFO | MemInfo: { | 20260506 22:02:45.534 | INFO | "gpu" : " 17 GB", | 20260506 22:02:45.534 | INFO | "huge" : "", | 20260506 22:02:45.534 | INFO | "huge_shared" : " 27 GB", | 20260506 22:02:45.534 | INFO | "malloc" : " 14 GB", | 20260506 22:02:45.534 | INFO | "pinned" : " 29 GB" | 20260506 22:02:45.534 | INFO | } | 20260506 22:02:45.534 | INFO | Perf report for the last 5.14562 seconds: | 20260506 22:02:45.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:02:45.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:45.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:45.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:45.534 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 22:02:45.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:45.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:45.534 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:02:45.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:45.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:45.534 | INFO | Uploaded 36GiB to GPU in 2.01683s (17.8498GiB/s) | 20260506 22:02:45.534 | INFO | Downloaded 34GiB to GPU in 1.92485s (17.6637GiB/s) | 20260506 22:02:50.676 | INFO | Analyzing chunk frames = [4789248,4789760) | 20260506 22:02:50.676 | INFO | MemInfo: { | 20260506 22:02:50.676 | INFO | "gpu" : " 17 GB", | 20260506 22:02:50.676 | INFO | "huge" : "", | 20260506 22:02:50.676 | INFO | "huge_shared" : " 27 GB", | 20260506 22:02:50.676 | INFO | "malloc" : " 14 GB", | 20260506 22:02:50.676 | INFO | "pinned" : " 29 GB" | 20260506 22:02:50.676 | INFO | } | 20260506 22:02:50.676 | INFO | Perf report for the last 5.14215 seconds: | 20260506 22:02:50.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:02:50.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:50.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:50.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:50.676 | INFO | ANALYSIS : 60.5%, 3.0, 93.9ms, 0 | 20260506 22:02:50.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:50.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:50.676 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:02:50.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:50.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:50.676 | INFO | Uploaded 36GiB to GPU in 1.99759s (18.0217GiB/s) | 20260506 22:02:50.676 | INFO | Downloaded 34GiB to GPU in 1.92281s (17.6825GiB/s) | 20260506 22:02:55.814 | INFO | Analyzing chunk frames = [4789760,4790272) | 20260506 22:02:55.814 | INFO | MemInfo: { | 20260506 22:02:55.814 | INFO | "gpu" : " 17 GB", | 20260506 22:02:55.814 | INFO | "huge" : "", | 20260506 22:02:55.814 | INFO | "huge_shared" : " 27 GB", | 20260506 22:02:55.814 | INFO | "malloc" : " 14 GB", | 20260506 22:02:55.814 | INFO | "pinned" : " 29 GB" | 20260506 22:02:55.814 | INFO | } | 20260506 22:02:55.814 | INFO | Perf report for the last 5.13829 seconds: | 20260506 22:02:55.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:02:55.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:55.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:55.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:55.814 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260506 22:02:55.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:55.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:55.814 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:02:55.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:55.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:02:55.815 | INFO | Uploaded 36GiB to GPU in 2.02062s (17.8163GiB/s) | 20260506 22:02:55.815 | INFO | Downloaded 34GiB to GPU in 1.90867s (17.8135GiB/s) | 20260506 22:03:00.948 | INFO | Analyzing chunk frames = [4790272,4790784) | 20260506 22:03:00.948 | INFO | MemInfo: { | 20260506 22:03:00.948 | INFO | "gpu" : " 17 GB", | 20260506 22:03:00.948 | INFO | "huge" : "", | 20260506 22:03:00.948 | INFO | "huge_shared" : " 27 GB", | 20260506 22:03:00.948 | INFO | "malloc" : " 14 GB", | 20260506 22:03:00.948 | INFO | "pinned" : " 29 GB" | 20260506 22:03:00.948 | INFO | } | 20260506 22:03:00.948 | INFO | Perf report for the last 5.13379 seconds: | 20260506 22:03:00.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:03:00.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:00.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:00.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:00.948 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 22:03:00.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:00.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:00.948 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 22:03:00.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:00.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:00.948 | INFO | Uploaded 36GiB to GPU in 2.01313s (17.8826GiB/s) | 20260506 22:03:00.948 | INFO | Downloaded 34GiB to GPU in 1.9253s (17.6596GiB/s) | 20260506 22:03:06.085 | INFO | Analyzing chunk frames = [4790784,4791296) | 20260506 22:03:06.085 | INFO | MemInfo: { | 20260506 22:03:06.085 | INFO | "gpu" : " 17 GB", | 20260506 22:03:06.085 | INFO | "huge" : "", | 20260506 22:03:06.085 | INFO | "huge_shared" : " 27 GB", | 20260506 22:03:06.085 | INFO | "malloc" : " 14 GB", | 20260506 22:03:06.085 | INFO | "pinned" : " 29 GB" | 20260506 22:03:06.085 | INFO | } | 20260506 22:03:06.085 | INFO | Perf report for the last 5.13663 seconds: | 20260506 22:03:06.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:03:06.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:06.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:06.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:06.085 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 22:03:06.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:06.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:06.085 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 22:03:06.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:06.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:06.085 | INFO | Uploaded 36GiB to GPU in 2.01864s (17.8338GiB/s) | 20260506 22:03:06.085 | INFO | Downloaded 34GiB to GPU in 1.96014s (17.3457GiB/s) | 20260506 22:03:11.226 | INFO | Analyzing chunk frames = [4791296,4791808) | 20260506 22:03:11.227 | INFO | MemInfo: { | 20260506 22:03:11.227 | INFO | "gpu" : " 17 GB", | 20260506 22:03:11.227 | INFO | "huge" : "", | 20260506 22:03:11.227 | INFO | "huge_shared" : " 27 GB", | 20260506 22:03:11.227 | INFO | "malloc" : " 14 GB", | 20260506 22:03:11.227 | INFO | "pinned" : " 29 GB" | 20260506 22:03:11.227 | INFO | } | 20260506 22:03:11.227 | INFO | Perf report for the last 5.14189 seconds: | 20260506 22:03:11.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:03:11.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:11.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:11.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:11.227 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 22:03:11.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:11.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:11.227 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 22:03:11.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:11.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:11.227 | INFO | Uploaded 36GiB to GPU in 1.95805s (18.3856GiB/s) | 20260506 22:03:11.227 | INFO | Downloaded 34GiB to GPU in 1.99091s (17.0776GiB/s) | 20260506 22:03:16.365 | INFO | Analyzing chunk frames = [4791808,4792320) | 20260506 22:03:16.365 | INFO | MemInfo: { | 20260506 22:03:16.365 | INFO | "gpu" : " 17 GB", | 20260506 22:03:16.365 | INFO | "huge" : "", | 20260506 22:03:16.365 | INFO | "huge_shared" : " 27 GB", | 20260506 22:03:16.365 | INFO | "malloc" : " 14 GB", | 20260506 22:03:16.365 | INFO | "pinned" : " 29 GB" | 20260506 22:03:16.365 | INFO | } | 20260506 22:03:16.365 | INFO | Perf report for the last 5.13855 seconds: | 20260506 22:03:16.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:03:16.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:16.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:16.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:16.365 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 22:03:16.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:16.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:16.365 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 22:03:16.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:16.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:16.366 | INFO | Uploaded 36GiB to GPU in 2.00829s (17.9257GiB/s) | 20260506 22:03:16.366 | INFO | Downloaded 34GiB to GPU in 2.0025s (16.9788GiB/s) | 20260506 22:03:16.366 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:03:16.366 | INFO | Total mem usage: 18 GiB | 20260506 22:03:16.366 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:03:16.366 | INFO | Recent High Water Marks Sum: 15.9 GiB (Savings: -13.3%) | 20260506 22:03:16.366 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:03:16.366 | INFO | High Water Breakdown: | 20260506 22:03:16.366 | INFO | 61.8% : 9.81 GiB ( 86% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:03:16.366 | INFO | 38.1% : 6.05 GiB ( 90.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:03:16.366 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:03:16.366 | INFO | ------------------------------------------ | 20260506 22:03:16.366 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:03:16.366 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:03:16.366 | INFO | Total mem usage: 26.5 GiB | 20260506 22:03:16.366 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:03:16.366 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:03:16.366 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:03:16.366 | INFO | High Water Breakdown: | 20260506 22:03:16.366 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:03:16.366 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:03:16.366 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:03:16.366 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:03:16.366 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:03:16.366 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:03:16.366 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:03:16.366 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:03:16.366 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:03:16.366 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:03:16.366 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:03:16.366 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:03:16.366 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:03:16.366 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:03:16.366 | INFO | ------------------------------------------ | 20260506 22:03:16.366 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:03:16.366 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:03:16.366 | INFO | Total mem usage: 25.6 GiB | 20260506 22:03:16.366 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:03:16.366 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:03:16.366 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:03:16.366 | INFO | High Water Breakdown: | 20260506 22:03:16.366 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:03:16.366 | INFO | ------------------------------------------ | 20260506 22:03:16.366 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:03:16.366 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:03:16.366 | INFO | Total mem usage: 15.7 GiB | 20260506 22:03:16.366 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:03:16.366 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:03:16.366 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:03:16.366 | INFO | High Water Breakdown: | 20260506 22:03:16.366 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:03:16.366 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:03:16.366 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:03:16.366 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:03:16.366 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:03:16.366 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:03:16.366 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:03:16.366 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:03:16.366 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:03:16.366 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:03:16.366 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:03:16.366 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:03:16.366 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:03:16.366 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:03:16.366 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:03:16.366 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:03:16.366 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:03:16.366 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:03:16.366 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:03:16.366 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:03:16.366 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:03:16.366 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:03:16.366 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:03:16.367 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:03:16.367 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:03:16.367 | INFO | ------------------------------------------ | 20260506 22:03:16.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:03:16.367 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:03:16.367 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:03:16.367 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:03:16.367 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.61e+01, 1.33e+01 / 2.39e+00 | 20260506 22:03:16.367 | INFO | PerfReport: 0.30%, AggregateStats -- 9.18e-02 / 1.03e-01, 1.06e-02 / 7.15e-03 | 20260506 22:03:16.367 | INFO | PerfReport: 11.65%, Baseline -- 3.62e+00 / 4.35e+00, 5.52e-01 / 3.66e-01 | 20260506 22:03:16.367 | INFO | PerfReport: 3.26%, Histogram -- 1.02e+00 / 1.44e+00, 1.82e-01 / 1.46e-01 | 20260506 22:03:16.367 | INFO | PerfReport: 16.03%, DME -- 3.99e+01 / 4.47e+01, 4.20e+00 / 2.88e+00 | 20260506 22:03:16.367 | INFO | PerfReport: 57.31%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.39e+00 / 2.06e+00 | 20260506 22:03:16.367 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.20e+00 / 2.75e+00, 3.77e-01 / 2.19e-01 | 20260506 22:03:16.367 | INFO | PerfReport: 3.25%, Metrics -- 1.01e+00 / 1.32e+00, 2.68e-01 / 1.47e-01 | 20260506 22:03:16.367 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.46e-01 / 3.91e-01, 5.47e-02 / 2.23e-02 | 20260506 22:03:16.367 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:03:16.367 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:03:16.367 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:03:16.367 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 22:03:16.367 | INFO | PerfReport: 21.63%, Upload -- 2.09e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260506 22:03:16.367 | INFO | PerfReport: 21.28%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 22:03:16.367 | INFO | PerfReport: 32.17%, Compute -- 3.11e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 22:03:16.367 | INFO | PerfReport: 1.35%, LockUpload -- 1.30e+00 / 0.00e+00, 4.78e+00 / 0.00e+00 | 20260506 22:03:16.367 | INFO | PerfReport: 1.11%, LockDownload -- 1.07e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260506 22:03:16.367 | INFO | PerfReport: 22.45%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 22:03:16.367 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.46e-02 / 0.00e+00 | 20260506 22:03:16.487 | INFO | Starting a round of Baz IO | 20260506 22:03:19.613 | INFO | Populating RTMetrics report | 20260506 22:03:19.613 | INFO | RTMetrics report populated | 20260506 22:03:19.613 | INFO | Opening tmp RTMetrics json file | 20260506 22:03:19.674 | INFO | Serializing RTMetrics to json | 20260506 22:03:19.677 | INFO | Serialization done, swapping tmp file now... | 20260506 22:03:19.681 | INFO | RTMetrics done | 20260506 22:03:19.777 | INFO | Finished a round of Baz IO: Wrote 1.926 GiB in 3.289 seconds | 20260506 22:03:19.777 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.77% / 0.00% / 66.22% | 20260506 22:03:19.777 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.543876% / 3.97364e-06% / 8.34465e-05% | 20260506 22:03:19.777 | INFO | Baz IO overall: | 20260506 22:03:19.777 | INFO | BytesWritten : 2520831726101 | 20260506 22:03:19.777 | INFO | NumEvents : 1216624761227 | 20260506 22:03:21.510 | INFO | Analyzing chunk frames = [4792320,4792832) | 20260506 22:03:21.510 | INFO | MemInfo: { | 20260506 22:03:21.510 | INFO | "gpu" : " 17 GB", | 20260506 22:03:21.510 | INFO | "huge" : "", | 20260506 22:03:21.510 | INFO | "huge_shared" : " 27 GB", | 20260506 22:03:21.510 | INFO | "malloc" : " 14 GB", | 20260506 22:03:21.510 | INFO | "pinned" : " 29 GB" | 20260506 22:03:21.510 | INFO | } | 20260506 22:03:21.510 | INFO | Perf report for the last 5.14505 seconds: | 20260506 22:03:21.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:03:21.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:21.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:21.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:21.510 | INFO | ANALYSIS : 62.6%, 2.9, 95.3ms, 0 | 20260506 22:03:21.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:21.510 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260506 22:03:21.510 | INFO | PRE_HQ : 60.3%, 1.9, 58.2ms, 0 | 20260506 22:03:21.510 | INFO | BAZWRITER : 63.9%, 4.8, 156.4ms, 0 | 20260506 22:03:21.510 | INFO | BAZ_SAMPLING : 1.1%, 1.0, 0.6ms, 0 | 20260506 22:03:21.510 | INFO | Uploaded 36GiB to GPU in 2.08526s (17.264GiB/s) | 20260506 22:03:21.510 | INFO | Downloaded 38GiB to GPU in 2.21624s (17.1462GiB/s) | 20260506 22:03:26.654 | INFO | Analyzing chunk frames = [4792832,4793344) | 20260506 22:03:26.654 | INFO | MemInfo: { | 20260506 22:03:26.654 | INFO | "gpu" : " 17 GB", | 20260506 22:03:26.654 | INFO | "huge" : "", | 20260506 22:03:26.654 | INFO | "huge_shared" : " 27 GB", | 20260506 22:03:26.654 | INFO | "malloc" : " 14 GB", | 20260506 22:03:26.654 | INFO | "pinned" : " 29 GB" | 20260506 22:03:26.654 | INFO | } | 20260506 22:03:26.654 | INFO | Perf report for the last 5.14335 seconds: | 20260506 22:03:26.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:03:26.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:26.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:26.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:26.654 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 22:03:26.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:26.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:26.654 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:03:26.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:26.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:26.654 | INFO | Uploaded 36GiB to GPU in 2.01763s (17.8427GiB/s) | 20260506 22:03:26.654 | INFO | Downloaded 34GiB to GPU in 1.92864s (17.629GiB/s) | 20260506 22:03:31.791 | INFO | Analyzing chunk frames = [4793344,4793856) | 20260506 22:03:31.791 | INFO | MemInfo: { | 20260506 22:03:31.791 | INFO | "gpu" : " 17 GB", | 20260506 22:03:31.791 | INFO | "huge" : "", | 20260506 22:03:31.791 | INFO | "huge_shared" : " 27 GB", | 20260506 22:03:31.791 | INFO | "malloc" : " 14 GB", | 20260506 22:03:31.791 | INFO | "pinned" : " 29 GB" | 20260506 22:03:31.791 | INFO | } | 20260506 22:03:31.791 | INFO | Perf report for the last 5.13761 seconds: | 20260506 22:03:31.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:03:31.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:31.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:31.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:31.791 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 22:03:31.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:31.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:31.791 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:03:31.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:31.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:31.791 | INFO | Uploaded 36GiB to GPU in 1.99957s (18.0039GiB/s) | 20260506 22:03:31.792 | INFO | Downloaded 34GiB to GPU in 1.93076s (17.6096GiB/s) | 20260506 22:03:36.931 | INFO | Analyzing chunk frames = [4793856,4794368) | 20260506 22:03:36.931 | INFO | MemInfo: { | 20260506 22:03:36.931 | INFO | "gpu" : " 17 GB", | 20260506 22:03:36.932 | INFO | "huge" : "", | 20260506 22:03:36.932 | INFO | "huge_shared" : " 27 GB", | 20260506 22:03:36.932 | INFO | "malloc" : " 14 GB", | 20260506 22:03:36.932 | INFO | "pinned" : " 29 GB" | 20260506 22:03:36.932 | INFO | } | 20260506 22:03:36.932 | INFO | Perf report for the last 5.14029 seconds: | 20260506 22:03:36.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:03:36.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:36.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:36.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:36.932 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 22:03:36.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:36.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:36.932 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 22:03:36.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:36.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:36.932 | INFO | Uploaded 36GiB to GPU in 2.01811s (17.8385GiB/s) | 20260506 22:03:36.932 | INFO | Downloaded 34GiB to GPU in 1.90738s (17.8255GiB/s) | 20260506 22:03:42.064 | INFO | Analyzing chunk frames = [4794368,4794880) | 20260506 22:03:42.064 | INFO | MemInfo: { | 20260506 22:03:42.064 | INFO | "gpu" : " 17 GB", | 20260506 22:03:42.064 | INFO | "huge" : "", | 20260506 22:03:42.064 | INFO | "huge_shared" : " 27 GB", | 20260506 22:03:42.064 | INFO | "malloc" : " 14 GB", | 20260506 22:03:42.064 | INFO | "pinned" : " 29 GB" | 20260506 22:03:42.064 | INFO | } | 20260506 22:03:42.064 | INFO | Perf report for the last 5.13284 seconds: | 20260506 22:03:42.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:03:42.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:42.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:42.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:42.065 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 22:03:42.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:42.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:42.065 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 22:03:42.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:42.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:42.065 | INFO | Uploaded 36GiB to GPU in 2.0001s (17.9991GiB/s) | 20260506 22:03:42.065 | INFO | Downloaded 34GiB to GPU in 1.9331s (17.5883GiB/s) | 20260506 22:03:47.202 | INFO | Analyzing chunk frames = [4794880,4795392) | 20260506 22:03:47.202 | INFO | MemInfo: { | 20260506 22:03:47.202 | INFO | "gpu" : " 17 GB", | 20260506 22:03:47.202 | INFO | "huge" : "", | 20260506 22:03:47.202 | INFO | "huge_shared" : " 27 GB", | 20260506 22:03:47.202 | INFO | "malloc" : " 14 GB", | 20260506 22:03:47.202 | INFO | "pinned" : " 29 GB" | 20260506 22:03:47.202 | INFO | } | 20260506 22:03:47.202 | INFO | Perf report for the last 5.13728 seconds: | 20260506 22:03:47.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:03:47.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:47.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:47.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:47.202 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:03:47.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:47.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:47.202 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:03:47.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:47.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:47.202 | INFO | Uploaded 36GiB to GPU in 2.01184s (17.8941GiB/s) | 20260506 22:03:47.202 | INFO | Downloaded 34GiB to GPU in 1.93473s (17.5735GiB/s) | 20260506 22:03:52.343 | INFO | Analyzing chunk frames = [4795392,4795904) | 20260506 22:03:52.343 | INFO | MemInfo: { | 20260506 22:03:52.343 | INFO | "gpu" : " 17 GB", | 20260506 22:03:52.343 | INFO | "huge" : "", | 20260506 22:03:52.343 | INFO | "huge_shared" : " 27 GB", | 20260506 22:03:52.343 | INFO | "malloc" : " 14 GB", | 20260506 22:03:52.343 | INFO | "pinned" : " 29 GB" | 20260506 22:03:52.343 | INFO | } | 20260506 22:03:52.343 | INFO | Perf report for the last 5.14137 seconds: | 20260506 22:03:52.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:03:52.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:52.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:52.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:52.343 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 22:03:52.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:52.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:52.343 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 22:03:52.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:52.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:52.343 | INFO | Uploaded 36GiB to GPU in 1.95149s (18.4474GiB/s) | 20260506 22:03:52.343 | INFO | Downloaded 34GiB to GPU in 1.97468s (17.218GiB/s) | 20260506 22:03:57.485 | INFO | Analyzing chunk frames = [4795904,4796416) | 20260506 22:03:57.485 | INFO | MemInfo: { | 20260506 22:03:57.485 | INFO | "gpu" : " 17 GB", | 20260506 22:03:57.485 | INFO | "huge" : "", | 20260506 22:03:57.485 | INFO | "huge_shared" : " 27 GB", | 20260506 22:03:57.485 | INFO | "malloc" : " 14 GB", | 20260506 22:03:57.485 | INFO | "pinned" : " 29 GB" | 20260506 22:03:57.485 | INFO | } | 20260506 22:03:57.485 | INFO | Perf report for the last 5.1423 seconds: | 20260506 22:03:57.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:03:57.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:57.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:57.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:57.486 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 22:03:57.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:57.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:57.486 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 22:03:57.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:57.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:03:57.486 | INFO | Uploaded 36GiB to GPU in 2.00359s (17.9678GiB/s) | 20260506 22:03:57.486 | INFO | Downloaded 34GiB to GPU in 1.99506s (17.0421GiB/s) | 20260506 22:03:57.585 | INFO | Starting a round of Baz IO | 20260506 22:04:00.724 | INFO | Populating RTMetrics report | 20260506 22:04:00.724 | INFO | RTMetrics report populated | 20260506 22:04:00.724 | INFO | Opening tmp RTMetrics json file | 20260506 22:04:00.741 | INFO | Serializing RTMetrics to json | 20260506 22:04:00.847 | INFO | Serialization done, swapping tmp file now... | 20260506 22:04:00.860 | INFO | RTMetrics done | 20260506 22:04:00.885 | INFO | Finished a round of Baz IO: Wrote 1.925 GiB in 3.3 seconds | 20260506 22:04:00.886 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.77% / 0.00% / 66.23% | 20260506 22:04:00.886 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.543058% / 7.94729e-06% / 7.15256e-05% | 20260506 22:04:00.886 | INFO | Baz IO overall: | 20260506 22:04:00.886 | INFO | BytesWritten : 2522898175279 | 20260506 22:04:00.886 | INFO | NumEvents : 1217611969773 | 20260506 22:04:02.627 | INFO | Analyzing chunk frames = [4796416,4796928) | 20260506 22:04:02.627 | INFO | MemInfo: { | 20260506 22:04:02.627 | INFO | "gpu" : " 17 GB", | 20260506 22:04:02.627 | INFO | "huge" : "", | 20260506 22:04:02.627 | INFO | "huge_shared" : " 27 GB", | 20260506 22:04:02.627 | INFO | "malloc" : " 14 GB", | 20260506 22:04:02.627 | INFO | "pinned" : " 29 GB" | 20260506 22:04:02.627 | INFO | } | 20260506 22:04:02.627 | INFO | Perf report for the last 5.14139 seconds: | 20260506 22:04:02.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:04:02.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:02.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:02.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:02.627 | INFO | ANALYSIS : 62.4%, 2.9, 95.5ms, 0 | 20260506 22:04:02.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:02.627 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260506 22:04:02.627 | INFO | PRE_HQ : 60.3%, 1.8, 57.5ms, 0 | 20260506 22:04:02.627 | INFO | BAZWRITER : 63.8%, 4.5, 146.4ms, 0 | 20260506 22:04:02.627 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 22:04:02.627 | INFO | Uploaded 36GiB to GPU in 2.04308s (17.6205GiB/s) | 20260506 22:04:02.627 | INFO | Downloaded 38GiB to GPU in 2.18721s (17.3737GiB/s) | 20260506 22:04:07.766 | INFO | Analyzing chunk frames = [4796928,4797440) | 20260506 22:04:07.766 | INFO | MemInfo: { | 20260506 22:04:07.766 | INFO | "gpu" : " 17 GB", | 20260506 22:04:07.766 | INFO | "huge" : "", | 20260506 22:04:07.766 | INFO | "huge_shared" : " 27 GB", | 20260506 22:04:07.766 | INFO | "malloc" : " 14 GB", | 20260506 22:04:07.766 | INFO | "pinned" : " 29 GB" | 20260506 22:04:07.766 | INFO | } | 20260506 22:04:07.766 | INFO | Perf report for the last 5.13917 seconds: | 20260506 22:04:07.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:04:07.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:07.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:07.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:07.766 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 22:04:07.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:07.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:07.766 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:04:07.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:07.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:07.766 | INFO | Uploaded 36GiB to GPU in 2.01824s (17.8374GiB/s) | 20260506 22:04:07.766 | INFO | Downloaded 34GiB to GPU in 1.92692s (17.6447GiB/s) | 20260506 22:04:12.903 | INFO | Analyzing chunk frames = [4797440,4797952) | 20260506 22:04:12.903 | INFO | MemInfo: { | 20260506 22:04:12.903 | INFO | "gpu" : " 17 GB", | 20260506 22:04:12.903 | INFO | "huge" : "", | 20260506 22:04:12.903 | INFO | "huge_shared" : " 27 GB", | 20260506 22:04:12.903 | INFO | "malloc" : " 14 GB", | 20260506 22:04:12.903 | INFO | "pinned" : " 29 GB" | 20260506 22:04:12.903 | INFO | } | 20260506 22:04:12.903 | INFO | Perf report for the last 5.13746 seconds: | 20260506 22:04:12.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:04:12.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:12.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:12.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:12.904 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 22:04:12.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:12.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:12.904 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:04:12.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:12.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:12.904 | INFO | Uploaded 36GiB to GPU in 1.98802s (18.1084GiB/s) | 20260506 22:04:12.904 | INFO | Downloaded 34GiB to GPU in 1.92859s (17.6294GiB/s) | 20260506 22:04:18.039 | INFO | Analyzing chunk frames = [4797952,4798464) | 20260506 22:04:18.039 | INFO | MemInfo: { | 20260506 22:04:18.039 | INFO | "gpu" : " 17 GB", | 20260506 22:04:18.039 | INFO | "huge" : "", | 20260506 22:04:18.039 | INFO | "huge_shared" : " 27 GB", | 20260506 22:04:18.039 | INFO | "malloc" : " 14 GB", | 20260506 22:04:18.039 | INFO | "pinned" : " 29 GB" | 20260506 22:04:18.039 | INFO | } | 20260506 22:04:18.039 | INFO | Perf report for the last 5.13567 seconds: | 20260506 22:04:18.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:04:18.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:18.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:18.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:18.039 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 22:04:18.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:18.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:18.039 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:04:18.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:18.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:18.039 | INFO | Uploaded 36GiB to GPU in 1.95818s (18.3844GiB/s) | 20260506 22:04:18.039 | INFO | Downloaded 34GiB to GPU in 1.87443s (18.1388GiB/s) | 20260506 22:04:23.177 | INFO | Analyzing chunk frames = [4798464,4798976) | 20260506 22:04:23.177 | INFO | MemInfo: { | 20260506 22:04:23.177 | INFO | "gpu" : " 17 GB", | 20260506 22:04:23.177 | INFO | "huge" : "", | 20260506 22:04:23.177 | INFO | "huge_shared" : " 27 GB", | 20260506 22:04:23.177 | INFO | "malloc" : " 14 GB", | 20260506 22:04:23.177 | INFO | "pinned" : " 29 GB" | 20260506 22:04:23.177 | INFO | } | 20260506 22:04:23.177 | INFO | Perf report for the last 5.13777 seconds: | 20260506 22:04:23.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:04:23.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:23.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:23.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:23.177 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 22:04:23.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:23.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:23.177 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:04:23.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:23.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:23.177 | INFO | Uploaded 36GiB to GPU in 2.00085s (17.9923GiB/s) | 20260506 22:04:23.177 | INFO | Downloaded 34GiB to GPU in 1.92289s (17.6817GiB/s) | 20260506 22:04:28.315 | INFO | Analyzing chunk frames = [4798976,4799488) | 20260506 22:04:28.315 | INFO | MemInfo: { | 20260506 22:04:28.315 | INFO | "gpu" : " 17 GB", | 20260506 22:04:28.316 | INFO | "huge" : "", | 20260506 22:04:28.316 | INFO | "huge_shared" : " 27 GB", | 20260506 22:04:28.316 | INFO | "malloc" : " 14 GB", | 20260506 22:04:28.316 | INFO | "pinned" : " 29 GB" | 20260506 22:04:28.316 | INFO | } | 20260506 22:04:28.316 | INFO | Perf report for the last 5.13861 seconds: | 20260506 22:04:28.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:04:28.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:28.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:28.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:28.316 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 22:04:28.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:28.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:28.316 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 22:04:28.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:28.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:28.316 | INFO | Uploaded 36GiB to GPU in 2.02442s (17.7829GiB/s) | 20260506 22:04:28.316 | INFO | Downloaded 34GiB to GPU in 1.94835s (17.4507GiB/s) | 20260506 22:04:33.457 | INFO | Analyzing chunk frames = [4799488,4800000) | 20260506 22:04:33.457 | INFO | MemInfo: { | 20260506 22:04:33.457 | INFO | "gpu" : " 17 GB", | 20260506 22:04:33.457 | INFO | "huge" : "", | 20260506 22:04:33.457 | INFO | "huge_shared" : " 27 GB", | 20260506 22:04:33.457 | INFO | "malloc" : " 14 GB", | 20260506 22:04:33.457 | INFO | "pinned" : " 29 GB" | 20260506 22:04:33.457 | INFO | } | 20260506 22:04:33.457 | INFO | Perf report for the last 5.14188 seconds: | 20260506 22:04:33.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:04:33.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:33.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:33.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:33.458 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 22:04:33.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:33.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:33.458 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 22:04:33.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:33.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:33.458 | INFO | Uploaded 36GiB to GPU in 1.98235s (18.1603GiB/s) | 20260506 22:04:33.458 | INFO | Downloaded 34GiB to GPU in 2.00454s (16.9615GiB/s) | 20260506 22:04:38.604 | INFO | Analyzing chunk frames = [4800000,4800512) | 20260506 22:04:38.604 | INFO | MemInfo: { | 20260506 22:04:38.604 | INFO | "gpu" : " 17 GB", | 20260506 22:04:38.604 | INFO | "huge" : "", | 20260506 22:04:38.604 | INFO | "huge_shared" : " 27 GB", | 20260506 22:04:38.604 | INFO | "malloc" : " 14 GB", | 20260506 22:04:38.604 | INFO | "pinned" : " 29 GB" | 20260506 22:04:38.604 | INFO | } | 20260506 22:04:38.604 | INFO | Perf report for the last 5.14632 seconds: | 20260506 22:04:38.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:04:38.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:38.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:38.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:38.604 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 22:04:38.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:38.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:38.604 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:04:38.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:38.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:38.604 | INFO | Uploaded 36GiB to GPU in 2.05069s (17.5551GiB/s) | 20260506 22:04:38.604 | INFO | Downloaded 34GiB to GPU in 1.99912s (17.0075GiB/s) | 20260506 22:04:38.604 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:04:38.604 | INFO | Total mem usage: 18 GiB | 20260506 22:04:38.604 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:04:38.604 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -17.3%) | 20260506 22:04:38.604 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:04:38.604 | INFO | High Water Breakdown: | 20260506 22:04:38.604 | INFO | 61.3% : 9.4 GiB ( 82.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:04:38.604 | INFO | 38.6% : 5.93 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:04:38.604 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:04:38.604 | INFO | ------------------------------------------ | 20260506 22:04:38.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:04:38.604 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:04:38.604 | INFO | Total mem usage: 26.5 GiB | 20260506 22:04:38.604 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:04:38.604 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:04:38.605 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:04:38.605 | INFO | High Water Breakdown: | 20260506 22:04:38.605 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:04:38.605 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:04:38.605 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:04:38.605 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:04:38.605 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:04:38.605 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:04:38.605 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:04:38.605 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:04:38.605 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:04:38.605 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:04:38.605 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:04:38.605 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:04:38.605 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:04:38.605 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:04:38.605 | INFO | ------------------------------------------ | 20260506 22:04:38.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:04:38.605 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:04:38.605 | INFO | Total mem usage: 25.6 GiB | 20260506 22:04:38.605 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:04:38.605 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:04:38.605 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:04:38.605 | INFO | High Water Breakdown: | 20260506 22:04:38.605 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:04:38.605 | INFO | ------------------------------------------ | 20260506 22:04:38.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:04:38.605 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:04:38.605 | INFO | Total mem usage: 15.7 GiB | 20260506 22:04:38.605 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:04:38.605 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:04:38.605 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:04:38.605 | INFO | High Water Breakdown: | 20260506 22:04:38.605 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:04:38.605 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:04:38.605 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:04:38.605 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:04:38.605 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:04:38.605 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:04:38.605 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:04:38.605 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:04:38.605 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:04:38.605 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:04:38.605 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:04:38.605 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:04:38.605 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:04:38.605 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:04:38.605 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:04:38.605 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:04:38.605 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:04:38.605 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:04:38.605 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:04:38.605 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:04:38.605 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:04:38.605 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:04:38.605 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:04:38.605 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:04:38.605 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:04:38.605 | INFO | ------------------------------------------ | 20260506 22:04:38.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:04:38.605 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:04:38.605 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:04:38.605 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:04:38.605 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.60e+01, 1.36e+01 / 2.88e+00 | 20260506 22:04:38.605 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.02e-01, 9.86e-03 / 6.77e-03 | 20260506 22:04:38.605 | INFO | PerfReport: 11.64%, Baseline -- 3.62e+00 / 4.40e+00, 5.43e-01 / 3.56e-01 | 20260506 22:04:38.605 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.48e+00, 1.80e-01 / 1.40e-01 | 20260506 22:04:38.605 | INFO | PerfReport: 16.09%, DME -- 4.00e+01 / 4.47e+01, 4.25e+00 / 3.20e+00 | 20260506 22:04:38.605 | INFO | PerfReport: 57.27%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.36e+00 / 1.79e+00 | 20260506 22:04:38.605 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.21e+00 / 2.80e+00, 3.86e-01 / 2.12e-01 | 20260506 22:04:38.605 | INFO | PerfReport: 3.24%, Metrics -- 1.01e+00 / 1.35e+00, 2.68e-01 / 1.83e-01 | 20260506 22:04:38.605 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.44e-01 / 3.91e-01, 5.35e-02 / 2.85e-02 | 20260506 22:04:38.605 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:04:38.605 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:04:38.605 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:04:38.605 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 22:04:38.605 | INFO | PerfReport: 21.65%, Upload -- 2.09e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260506 22:04:38.606 | INFO | PerfReport: 21.29%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 22:04:38.606 | INFO | PerfReport: 32.14%, Compute -- 3.11e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 22:04:38.606 | INFO | PerfReport: 1.37%, LockUpload -- 1.32e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260506 22:04:38.606 | INFO | PerfReport: 1.16%, LockDownload -- 1.13e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260506 22:04:38.606 | INFO | PerfReport: 22.38%, LockCompute -- 2.16e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 22:04:38.606 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260506 22:04:38.715 | INFO | Starting a round of Baz IO | 20260506 22:04:41.841 | INFO | Populating RTMetrics report | 20260506 22:04:41.842 | INFO | RTMetrics report populated | 20260506 22:04:41.842 | INFO | Opening tmp RTMetrics json file | 20260506 22:04:41.883 | INFO | Serializing RTMetrics to json | 20260506 22:04:41.889 | INFO | Serialization done, swapping tmp file now... | 20260506 22:04:41.897 | INFO | RTMetrics done | 20260506 22:04:42.408 | INFO | Finished a round of Baz IO: Wrote 1.924 GiB in 3.693 seconds | 20260506 22:04:42.409 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.77% / 0.00% / 66.23% | 20260506 22:04:42.409 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.541238% / 3.97364e-06% / 7.15256e-05% | 20260506 22:04:42.409 | INFO | Baz IO overall: | 20260506 22:04:42.409 | INFO | BytesWritten : 2524964011881 | 20260506 22:04:42.409 | INFO | NumEvents : 1218599257558 | 20260506 22:04:43.744 | INFO | Analyzing chunk frames = [4800512,4801024) | 20260506 22:04:43.744 | INFO | MemInfo: { | 20260506 22:04:43.744 | INFO | "gpu" : " 17 GB", | 20260506 22:04:43.744 | INFO | "huge" : "", | 20260506 22:04:43.744 | INFO | "huge_shared" : " 27 GB", | 20260506 22:04:43.744 | INFO | "malloc" : " 14 GB", | 20260506 22:04:43.744 | INFO | "pinned" : " 29 GB" | 20260506 22:04:43.744 | INFO | } | 20260506 22:04:43.745 | INFO | Perf report for the last 5.14066 seconds: | 20260506 22:04:43.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:04:43.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:43.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:43.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:43.745 | INFO | ANALYSIS : 62.4%, 2.9, 95.6ms, 0 | 20260506 22:04:43.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:43.745 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260506 22:04:43.745 | INFO | PRE_HQ : 59.9%, 1.8, 56.6ms, 0 | 20260506 22:04:43.745 | INFO | BAZWRITER : 71.4%, 4.8, 175.2ms, 0 | 20260506 22:04:43.745 | INFO | BAZ_SAMPLING : 1.7%, 1.0, 0.9ms, 0 | 20260506 22:04:43.745 | INFO | Uploaded 36GiB to GPU in 2.07201s (17.3744GiB/s) | 20260506 22:04:43.745 | INFO | Downloaded 38GiB to GPU in 2.20476s (17.2354GiB/s) | 20260506 22:04:48.878 | INFO | Analyzing chunk frames = [4801024,4801536) | 20260506 22:04:48.878 | INFO | MemInfo: { | 20260506 22:04:48.878 | INFO | "gpu" : " 17 GB", | 20260506 22:04:48.878 | INFO | "huge" : "", | 20260506 22:04:48.878 | INFO | "huge_shared" : " 27 GB", | 20260506 22:04:48.878 | INFO | "malloc" : " 14 GB", | 20260506 22:04:48.878 | INFO | "pinned" : " 29 GB" | 20260506 22:04:48.878 | INFO | } | 20260506 22:04:48.878 | INFO | Perf report for the last 5.13362 seconds: | 20260506 22:04:48.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:04:48.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:48.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:48.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:48.878 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:04:48.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:48.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:48.878 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:04:48.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:48.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:48.878 | INFO | Uploaded 36GiB to GPU in 2.00796s (17.9286GiB/s) | 20260506 22:04:48.878 | INFO | Downloaded 34GiB to GPU in 1.92318s (17.6791GiB/s) | 20260506 22:04:54.015 | INFO | Analyzing chunk frames = [4801536,4802048) | 20260506 22:04:54.015 | INFO | MemInfo: { | 20260506 22:04:54.015 | INFO | "gpu" : " 17 GB", | 20260506 22:04:54.016 | INFO | "huge" : "", | 20260506 22:04:54.016 | INFO | "huge_shared" : " 27 GB", | 20260506 22:04:54.016 | INFO | "malloc" : " 14 GB", | 20260506 22:04:54.016 | INFO | "pinned" : " 29 GB" | 20260506 22:04:54.016 | INFO | } | 20260506 22:04:54.016 | INFO | Perf report for the last 5.1375 seconds: | 20260506 22:04:54.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:04:54.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:54.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:54.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:54.016 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 22:04:54.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:54.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:54.016 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:04:54.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:54.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:54.016 | INFO | Uploaded 36GiB to GPU in 1.99558s (18.0399GiB/s) | 20260506 22:04:54.016 | INFO | Downloaded 34GiB to GPU in 1.92372s (17.6741GiB/s) | 20260506 22:04:59.153 | INFO | Analyzing chunk frames = [4802048,4802560) | 20260506 22:04:59.153 | INFO | MemInfo: { | 20260506 22:04:59.153 | INFO | "gpu" : " 17 GB", | 20260506 22:04:59.153 | INFO | "huge" : "", | 20260506 22:04:59.153 | INFO | "huge_shared" : " 27 GB", | 20260506 22:04:59.153 | INFO | "malloc" : " 14 GB", | 20260506 22:04:59.153 | INFO | "pinned" : " 29 GB" | 20260506 22:04:59.153 | INFO | } | 20260506 22:04:59.153 | INFO | Perf report for the last 5.13774 seconds: | 20260506 22:04:59.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:04:59.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:59.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:59.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:59.153 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 22:04:59.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:59.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:59.153 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:04:59.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:59.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:04:59.154 | INFO | Uploaded 36GiB to GPU in 1.97063s (18.2683GiB/s) | 20260506 22:04:59.154 | INFO | Downloaded 34GiB to GPU in 1.87532s (18.1302GiB/s) | 20260506 22:05:04.290 | INFO | Analyzing chunk frames = [4802560,4803072) | 20260506 22:05:04.290 | INFO | MemInfo: { | 20260506 22:05:04.290 | INFO | "gpu" : " 17 GB", | 20260506 22:05:04.290 | INFO | "huge" : "", | 20260506 22:05:04.290 | INFO | "huge_shared" : " 27 GB", | 20260506 22:05:04.290 | INFO | "malloc" : " 14 GB", | 20260506 22:05:04.291 | INFO | "pinned" : " 29 GB" | 20260506 22:05:04.291 | INFO | } | 20260506 22:05:04.291 | INFO | Perf report for the last 5.13716 seconds: | 20260506 22:05:04.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:05:04.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:04.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:04.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:04.291 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260506 22:05:04.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:04.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:04.291 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 22:05:04.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:04.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:04.291 | INFO | Uploaded 36GiB to GPU in 2.01249s (17.8883GiB/s) | 20260506 22:05:04.291 | INFO | Downloaded 34GiB to GPU in 1.91749s (17.7315GiB/s) | 20260506 22:05:09.432 | INFO | Analyzing chunk frames = [4803072,4803584) | 20260506 22:05:09.432 | INFO | MemInfo: { | 20260506 22:05:09.432 | INFO | "gpu" : " 17 GB", | 20260506 22:05:09.432 | INFO | "huge" : "", | 20260506 22:05:09.432 | INFO | "huge_shared" : " 27 GB", | 20260506 22:05:09.432 | INFO | "malloc" : " 14 GB", | 20260506 22:05:09.432 | INFO | "pinned" : " 29 GB" | 20260506 22:05:09.432 | INFO | } | 20260506 22:05:09.432 | INFO | Perf report for the last 5.14138 seconds: | 20260506 22:05:09.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:05:09.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:09.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:09.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:09.432 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 22:05:09.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:09.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:09.432 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 22:05:09.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:09.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:09.432 | INFO | Uploaded 36GiB to GPU in 2.0232s (17.7936GiB/s) | 20260506 22:05:09.432 | INFO | Downloaded 34GiB to GPU in 1.95786s (17.3659GiB/s) | 20260506 22:05:14.574 | INFO | Analyzing chunk frames = [4803584,4804096) | 20260506 22:05:14.574 | INFO | MemInfo: { | 20260506 22:05:14.574 | INFO | "gpu" : " 17 GB", | 20260506 22:05:14.574 | INFO | "huge" : "", | 20260506 22:05:14.574 | INFO | "huge_shared" : " 27 GB", | 20260506 22:05:14.574 | INFO | "malloc" : " 14 GB", | 20260506 22:05:14.574 | INFO | "pinned" : " 29 GB" | 20260506 22:05:14.574 | INFO | } | 20260506 22:05:14.574 | INFO | Perf report for the last 5.14201 seconds: | 20260506 22:05:14.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:05:14.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:14.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:14.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:14.574 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:05:14.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:14.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:14.574 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:05:14.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:14.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:14.574 | INFO | Uploaded 36GiB to GPU in 1.9662s (18.3094GiB/s) | 20260506 22:05:14.574 | INFO | Downloaded 34GiB to GPU in 1.99248s (17.0642GiB/s) | 20260506 22:05:19.717 | INFO | Analyzing chunk frames = [4804096,4804608) | 20260506 22:05:19.717 | INFO | MemInfo: { | 20260506 22:05:19.717 | INFO | "gpu" : " 17 GB", | 20260506 22:05:19.717 | INFO | "huge" : "", | 20260506 22:05:19.717 | INFO | "huge_shared" : " 27 GB", | 20260506 22:05:19.717 | INFO | "malloc" : " 14 GB", | 20260506 22:05:19.717 | INFO | "pinned" : " 29 GB" | 20260506 22:05:19.717 | INFO | } | 20260506 22:05:19.717 | INFO | Perf report for the last 5.14275 seconds: | 20260506 22:05:19.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:05:19.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:19.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:19.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:19.717 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 22:05:19.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:19.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:19.717 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 22:05:19.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:19.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:19.717 | INFO | Uploaded 36GiB to GPU in 1.99654s (18.0312GiB/s) | 20260506 22:05:19.717 | INFO | Downloaded 34GiB to GPU in 1.98739s (17.1078GiB/s) | 20260506 22:05:19.815 | INFO | Starting a round of Baz IO | 20260506 22:05:22.961 | INFO | Populating RTMetrics report | 20260506 22:05:22.961 | INFO | RTMetrics report populated | 20260506 22:05:22.961 | INFO | Opening tmp RTMetrics json file | 20260506 22:05:22.964 | INFO | Serializing RTMetrics to json | 20260506 22:05:22.967 | INFO | Serialization done, swapping tmp file now... | 20260506 22:05:22.969 | INFO | RTMetrics done | 20260506 22:05:23.163 | INFO | Finished a round of Baz IO: Wrote 1.925 GiB in 3.348 seconds | 20260506 22:05:23.163 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.77% / 0.00% / 66.23% | 20260506 22:05:23.163 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.541619% / 1.19209e-05% / 8.74201e-05% | 20260506 22:05:23.163 | INFO | Baz IO overall: | 20260506 22:05:23.163 | INFO | BytesWritten : 2527031498418 | 20260506 22:05:23.163 | INFO | NumEvents : 1219588347281 | 20260506 22:05:24.853 | INFO | Analyzing chunk frames = [4804608,4805120) | 20260506 22:05:24.853 | INFO | MemInfo: { | 20260506 22:05:24.853 | INFO | "gpu" : " 17 GB", | 20260506 22:05:24.853 | INFO | "huge" : "", | 20260506 22:05:24.853 | INFO | "huge_shared" : " 27 GB", | 20260506 22:05:24.853 | INFO | "malloc" : " 14 GB", | 20260506 22:05:24.853 | INFO | "pinned" : " 29 GB" | 20260506 22:05:24.853 | INFO | } | 20260506 22:05:24.853 | INFO | Perf report for the last 5.13622 seconds: | 20260506 22:05:24.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:05:24.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:24.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:24.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:24.853 | INFO | ANALYSIS : 62.2%, 2.9, 94.8ms, 0 | 20260506 22:05:24.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:24.853 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 22:05:24.853 | INFO | PRE_HQ : 60.3%, 1.9, 58.5ms, 0 | 20260506 22:05:24.853 | INFO | BAZWRITER : 64.8%, 4.8, 159.6ms, 0 | 20260506 22:05:24.853 | INFO | BAZ_SAMPLING : 3.3%, 1.0, 1.7ms, 0 | 20260506 22:05:24.853 | INFO | Uploaded 36GiB to GPU in 2.03823s (17.6624GiB/s) | 20260506 22:05:24.853 | INFO | Downloaded 38GiB to GPU in 2.1931s (17.327GiB/s) | 20260506 22:05:29.995 | INFO | Analyzing chunk frames = [4805120,4805632) | 20260506 22:05:29.995 | INFO | MemInfo: { | 20260506 22:05:29.995 | INFO | "gpu" : " 17 GB", | 20260506 22:05:29.995 | INFO | "huge" : "", | 20260506 22:05:29.995 | INFO | "huge_shared" : " 27 GB", | 20260506 22:05:29.995 | INFO | "malloc" : " 14 GB", | 20260506 22:05:29.995 | INFO | "pinned" : " 29 GB" | 20260506 22:05:29.995 | INFO | } | 20260506 22:05:29.995 | INFO | Perf report for the last 5.14224 seconds: | 20260506 22:05:29.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:05:29.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:29.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:29.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:29.995 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:05:29.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:29.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:29.995 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 22:05:29.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:29.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:29.995 | INFO | Uploaded 36GiB to GPU in 2.00823s (17.9262GiB/s) | 20260506 22:05:29.995 | INFO | Downloaded 34GiB to GPU in 1.92366s (17.6746GiB/s) | 20260506 22:05:35.131 | INFO | Analyzing chunk frames = [4805632,4806144) | 20260506 22:05:35.131 | INFO | MemInfo: { | 20260506 22:05:35.131 | INFO | "gpu" : " 17 GB", | 20260506 22:05:35.131 | INFO | "huge" : "", | 20260506 22:05:35.131 | INFO | "huge_shared" : " 27 GB", | 20260506 22:05:35.131 | INFO | "malloc" : " 14 GB", | 20260506 22:05:35.131 | INFO | "pinned" : " 29 GB" | 20260506 22:05:35.131 | INFO | } | 20260506 22:05:35.131 | INFO | Perf report for the last 5.13577 seconds: | 20260506 22:05:35.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:05:35.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:35.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:35.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:35.131 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 22:05:35.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:35.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:35.131 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:05:35.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:35.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:35.131 | INFO | Uploaded 36GiB to GPU in 1.98942s (18.0958GiB/s) | 20260506 22:05:35.131 | INFO | Downloaded 34GiB to GPU in 1.92147s (17.6948GiB/s) | 20260506 22:05:40.266 | INFO | Analyzing chunk frames = [4806144,4806656) | 20260506 22:05:40.266 | INFO | MemInfo: { | 20260506 22:05:40.266 | INFO | "gpu" : " 17 GB", | 20260506 22:05:40.266 | INFO | "huge" : "", | 20260506 22:05:40.266 | INFO | "huge_shared" : " 27 GB", | 20260506 22:05:40.266 | INFO | "malloc" : " 14 GB", | 20260506 22:05:40.266 | INFO | "pinned" : " 29 GB" | 20260506 22:05:40.266 | INFO | } | 20260506 22:05:40.266 | INFO | Perf report for the last 5.13509 seconds: | 20260506 22:05:40.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:05:40.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:40.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:40.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:40.266 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260506 22:05:40.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:40.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:40.266 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:05:40.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:40.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:40.266 | INFO | Uploaded 36GiB to GPU in 2.00946s (17.9153GiB/s) | 20260506 22:05:40.266 | INFO | Downloaded 34GiB to GPU in 1.90822s (17.8177GiB/s) | 20260506 22:05:45.412 | INFO | Analyzing chunk frames = [4806656,4807168) | 20260506 22:05:45.412 | INFO | MemInfo: { | 20260506 22:05:45.412 | INFO | "gpu" : " 17 GB", | 20260506 22:05:45.412 | INFO | "huge" : "", | 20260506 22:05:45.412 | INFO | "huge_shared" : " 27 GB", | 20260506 22:05:45.412 | INFO | "malloc" : " 14 GB", | 20260506 22:05:45.412 | INFO | "pinned" : " 29 GB" | 20260506 22:05:45.412 | INFO | } | 20260506 22:05:45.412 | INFO | Perf report for the last 5.14619 seconds: | 20260506 22:05:45.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:05:45.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:45.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:45.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:45.412 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 22:05:45.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:45.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:45.412 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:05:45.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:45.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:45.413 | INFO | Uploaded 36GiB to GPU in 2.01827s (17.8371GiB/s) | 20260506 22:05:45.413 | INFO | Downloaded 34GiB to GPU in 1.91846s (17.7226GiB/s) | 20260506 22:05:50.551 | INFO | Analyzing chunk frames = [4807168,4807680) | 20260506 22:05:50.551 | INFO | MemInfo: { | 20260506 22:05:50.551 | INFO | "gpu" : " 17 GB", | 20260506 22:05:50.551 | INFO | "huge" : "", | 20260506 22:05:50.551 | INFO | "huge_shared" : " 27 GB", | 20260506 22:05:50.551 | INFO | "malloc" : " 14 GB", | 20260506 22:05:50.551 | INFO | "pinned" : " 29 GB" | 20260506 22:05:50.551 | INFO | } | 20260506 22:05:50.551 | INFO | Perf report for the last 5.139 seconds: | 20260506 22:05:50.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:05:50.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:50.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:50.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:50.551 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 22:05:50.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:50.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:50.551 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:05:50.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:50.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:50.552 | INFO | Uploaded 36GiB to GPU in 2.01724s (17.8462GiB/s) | 20260506 22:05:50.552 | INFO | Downloaded 34GiB to GPU in 1.94983s (17.4375GiB/s) | 20260506 22:05:55.690 | INFO | Analyzing chunk frames = [4807680,4808192) | 20260506 22:05:55.690 | INFO | MemInfo: { | 20260506 22:05:55.690 | INFO | "gpu" : " 17 GB", | 20260506 22:05:55.690 | INFO | "huge" : "", | 20260506 22:05:55.690 | INFO | "huge_shared" : " 27 GB", | 20260506 22:05:55.690 | INFO | "malloc" : " 14 GB", | 20260506 22:05:55.690 | INFO | "pinned" : " 29 GB" | 20260506 22:05:55.690 | INFO | } | 20260506 22:05:55.690 | INFO | Perf report for the last 5.13869 seconds: | 20260506 22:05:55.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:05:55.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:55.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:55.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:55.690 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 22:05:55.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:55.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:55.690 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 22:05:55.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:55.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:05:55.690 | INFO | Uploaded 36GiB to GPU in 1.93772s (18.5786GiB/s) | 20260506 22:05:55.690 | INFO | Downloaded 34GiB to GPU in 1.97806s (17.1886GiB/s) | 20260506 22:06:00.831 | INFO | Analyzing chunk frames = [4808192,4808704) | 20260506 22:06:00.831 | INFO | MemInfo: { | 20260506 22:06:00.831 | INFO | "gpu" : " 17 GB", | 20260506 22:06:00.831 | INFO | "huge" : "", | 20260506 22:06:00.831 | INFO | "huge_shared" : " 27 GB", | 20260506 22:06:00.831 | INFO | "malloc" : " 14 GB", | 20260506 22:06:00.831 | INFO | "pinned" : " 29 GB" | 20260506 22:06:00.831 | INFO | } | 20260506 22:06:00.831 | INFO | Perf report for the last 5.14143 seconds: | 20260506 22:06:00.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:06:00.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:00.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:00.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:00.832 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 22:06:00.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:00.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:00.832 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:06:00.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:00.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:00.832 | INFO | Uploaded 36GiB to GPU in 2.05016s (17.5596GiB/s) | 20260506 22:06:00.832 | INFO | Downloaded 34GiB to GPU in 1.99715s (17.0243GiB/s) | 20260506 22:06:00.832 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:06:00.832 | INFO | Total mem usage: 18 GiB | 20260506 22:06:00.832 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:06:00.832 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.5%) | 20260506 22:06:00.832 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:06:00.832 | INFO | High Water Breakdown: | 20260506 22:06:00.832 | INFO | 61.9% : 9.9 GiB ( 86.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:06:00.832 | INFO | 38% : 6.08 GiB ( 90.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:06:00.832 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:06:00.832 | INFO | ------------------------------------------ | 20260506 22:06:00.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:06:00.832 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:06:00.832 | INFO | Total mem usage: 26.5 GiB | 20260506 22:06:00.832 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:06:00.832 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:06:00.832 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:06:00.832 | INFO | High Water Breakdown: | 20260506 22:06:00.832 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:06:00.832 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:06:00.832 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:06:00.832 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:06:00.832 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:06:00.832 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:06:00.832 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:06:00.832 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:06:00.832 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:06:00.832 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:06:00.832 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:06:00.832 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:06:00.832 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:06:00.832 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:06:00.832 | INFO | ------------------------------------------ | 20260506 22:06:00.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:06:00.832 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:06:00.832 | INFO | Total mem usage: 25.6 GiB | 20260506 22:06:00.832 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:06:00.832 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:06:00.832 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:06:00.832 | INFO | High Water Breakdown: | 20260506 22:06:00.832 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:06:00.832 | INFO | ------------------------------------------ | 20260506 22:06:00.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:06:00.832 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:06:00.832 | INFO | Total mem usage: 15.7 GiB | 20260506 22:06:00.832 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:06:00.832 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:06:00.832 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:06:00.832 | INFO | High Water Breakdown: | 20260506 22:06:00.832 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:06:00.832 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:06:00.832 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:06:00.832 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:06:00.832 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:06:00.832 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:06:00.832 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:06:00.832 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:06:00.832 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:06:00.832 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:06:00.832 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:06:00.832 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:06:00.832 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:06:00.832 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:06:00.832 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:06:00.832 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:06:00.832 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:06:00.832 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:06:00.832 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:06:00.832 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:06:00.832 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:06:00.832 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:06:00.832 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:06:00.832 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:06:00.832 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:06:00.832 | INFO | ------------------------------------------ | 20260506 22:06:00.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:06:00.833 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:06:00.833 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:06:00.833 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:06:00.833 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.62e+01, 1.36e+01 / 2.57e+00 | 20260506 22:06:00.833 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.02e-01, 1.01e-02 / 5.67e-03 | 20260506 22:06:00.833 | INFO | PerfReport: 11.64%, Baseline -- 3.62e+00 / 4.42e+00, 5.44e-01 / 3.00e-01 | 20260506 22:06:00.833 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.81e-01 / 1.23e-01 | 20260506 22:06:00.833 | INFO | PerfReport: 16.08%, DME -- 4.00e+01 / 4.48e+01, 4.32e+00 / 3.23e+00 | 20260506 22:06:00.833 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.38e+00 / 1.91e+00 | 20260506 22:06:00.833 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.21e+00 / 2.75e+00, 3.79e-01 / 2.36e-01 | 20260506 22:06:00.833 | INFO | PerfReport: 3.24%, Metrics -- 1.01e+00 / 1.32e+00, 2.66e-01 / 1.33e-01 | 20260506 22:06:00.833 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.44e-01 / 3.94e-01, 5.38e-02 / 2.29e-02 | 20260506 22:06:00.833 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:06:00.833 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:06:00.833 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:06:00.833 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 22:06:00.833 | INFO | PerfReport: 21.63%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260506 22:06:00.833 | INFO | PerfReport: 21.27%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260506 22:06:00.833 | INFO | PerfReport: 32.18%, Compute -- 3.11e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 22:06:00.833 | INFO | PerfReport: 1.32%, LockUpload -- 1.28e+00 / 0.00e+00, 4.71e+00 / 0.00e+00 | 20260506 22:06:00.833 | INFO | PerfReport: 1.13%, LockDownload -- 1.10e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260506 22:06:00.833 | INFO | PerfReport: 22.45%, LockCompute -- 2.17e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 22:06:00.833 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260506 22:06:00.946 | INFO | Starting a round of Baz IO | 20260506 22:06:04.069 | INFO | Populating RTMetrics report | 20260506 22:06:04.069 | INFO | RTMetrics report populated | 20260506 22:06:04.069 | INFO | Opening tmp RTMetrics json file | 20260506 22:06:04.070 | INFO | Serializing RTMetrics to json | 20260506 22:06:04.071 | INFO | Serialization done, swapping tmp file now... | 20260506 22:06:04.072 | INFO | RTMetrics done | 20260506 22:06:04.166 | INFO | Finished a round of Baz IO: Wrote 1.925 GiB in 3.219 seconds | 20260506 22:06:04.166 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.76% / 0.00% / 66.23% | 20260506 22:06:04.166 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.540658% / 0% / 7.54992e-05% | 20260506 22:06:04.166 | INFO | Baz IO overall: | 20260506 22:06:04.166 | INFO | BytesWritten : 2529098161361 | 20260506 22:06:04.166 | INFO | NumEvents : 1220576428702 | 20260506 22:06:05.965 | INFO | Analyzing chunk frames = [4808704,4809216) | 20260506 22:06:05.965 | INFO | MemInfo: { | 20260506 22:06:05.965 | INFO | "gpu" : " 17 GB", | 20260506 22:06:05.966 | INFO | "huge" : "", | 20260506 22:06:05.966 | INFO | "huge_shared" : " 27 GB", | 20260506 22:06:05.966 | INFO | "malloc" : " 14 GB", | 20260506 22:06:05.966 | INFO | "pinned" : " 29 GB" | 20260506 22:06:05.966 | INFO | } | 20260506 22:06:05.966 | INFO | Perf report for the last 5.13409 seconds: | 20260506 22:06:05.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:06:05.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:05.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:05.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:05.966 | INFO | ANALYSIS : 62.5%, 3.0, 97.0ms, 0 | 20260506 22:06:05.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:05.966 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 22:06:05.966 | INFO | PRE_HQ : 59.9%, 1.9, 57.6ms, 0 | 20260506 22:06:05.966 | INFO | BAZWRITER : 62.3%, 3.5, 113.5ms, 0 | 20260506 22:06:05.966 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:06:05.966 | INFO | Uploaded 36GiB to GPU in 2.04014s (17.6459GiB/s) | 20260506 22:06:05.966 | INFO | Downloaded 38GiB to GPU in 2.19055s (17.3472GiB/s) | 20260506 22:06:11.107 | INFO | Analyzing chunk frames = [4809216,4809728) | 20260506 22:06:11.107 | INFO | MemInfo: { | 20260506 22:06:11.107 | INFO | "gpu" : " 17 GB", | 20260506 22:06:11.107 | INFO | "huge" : "", | 20260506 22:06:11.107 | INFO | "huge_shared" : " 27 GB", | 20260506 22:06:11.107 | INFO | "malloc" : " 14 GB", | 20260506 22:06:11.107 | INFO | "pinned" : " 29 GB" | 20260506 22:06:11.107 | INFO | } | 20260506 22:06:11.107 | INFO | Perf report for the last 5.14149 seconds: | 20260506 22:06:11.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:06:11.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:11.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:11.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:11.107 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 22:06:11.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:11.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:11.107 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:06:11.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:11.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:11.107 | INFO | Uploaded 36GiB to GPU in 2.0088s (17.9212GiB/s) | 20260506 22:06:11.108 | INFO | Downloaded 34GiB to GPU in 1.92411s (17.6705GiB/s) | 20260506 22:06:16.245 | INFO | Analyzing chunk frames = [4809728,4810240) | 20260506 22:06:16.245 | INFO | MemInfo: { | 20260506 22:06:16.245 | INFO | "gpu" : " 17 GB", | 20260506 22:06:16.245 | INFO | "huge" : "", | 20260506 22:06:16.245 | INFO | "huge_shared" : " 27 GB", | 20260506 22:06:16.245 | INFO | "malloc" : " 14 GB", | 20260506 22:06:16.245 | INFO | "pinned" : " 29 GB" | 20260506 22:06:16.245 | INFO | } | 20260506 22:06:16.245 | INFO | Perf report for the last 5.13817 seconds: | 20260506 22:06:16.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:06:16.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:16.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:16.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:16.245 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 22:06:16.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:16.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:16.245 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:06:16.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:16.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:16.245 | INFO | Uploaded 36GiB to GPU in 2.00066s (17.9941GiB/s) | 20260506 22:06:16.246 | INFO | Downloaded 34GiB to GPU in 1.92729s (17.6414GiB/s) | 20260506 22:06:21.382 | INFO | Analyzing chunk frames = [4810240,4810752) | 20260506 22:06:21.382 | INFO | MemInfo: { | 20260506 22:06:21.382 | INFO | "gpu" : " 17 GB", | 20260506 22:06:21.382 | INFO | "huge" : "", | 20260506 22:06:21.382 | INFO | "huge_shared" : " 27 GB", | 20260506 22:06:21.382 | INFO | "malloc" : " 14 GB", | 20260506 22:06:21.382 | INFO | "pinned" : " 29 GB" | 20260506 22:06:21.382 | INFO | } | 20260506 22:06:21.382 | INFO | Perf report for the last 5.13672 seconds: | 20260506 22:06:21.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:06:21.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:21.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:21.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:21.382 | INFO | ANALYSIS : 59.6%, 3.0, 92.3ms, 0 | 20260506 22:06:21.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:21.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:21.382 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:06:21.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:21.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:21.382 | INFO | Uploaded 36GiB to GPU in 1.95673s (18.398GiB/s) | 20260506 22:06:21.382 | INFO | Downloaded 34GiB to GPU in 1.87631s (18.1207GiB/s) | 20260506 22:06:26.521 | INFO | Analyzing chunk frames = [4810752,4811264) | 20260506 22:06:26.521 | INFO | MemInfo: { | 20260506 22:06:26.522 | INFO | "gpu" : " 17 GB", | 20260506 22:06:26.522 | INFO | "huge" : "", | 20260506 22:06:26.522 | INFO | "huge_shared" : " 27 GB", | 20260506 22:06:26.522 | INFO | "malloc" : " 14 GB", | 20260506 22:06:26.522 | INFO | "pinned" : " 29 GB" | 20260506 22:06:26.522 | INFO | } | 20260506 22:06:26.522 | INFO | Perf report for the last 5.13964 seconds: | 20260506 22:06:26.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:06:26.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:26.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:26.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:26.522 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 22:06:26.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:26.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:26.522 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:06:26.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:26.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:26.522 | INFO | Uploaded 36GiB to GPU in 1.98852s (18.1039GiB/s) | 20260506 22:06:26.522 | INFO | Downloaded 34GiB to GPU in 1.91725s (17.7338GiB/s) | 20260506 22:06:31.663 | INFO | Analyzing chunk frames = [4811264,4811776) | 20260506 22:06:31.663 | INFO | MemInfo: { | 20260506 22:06:31.663 | INFO | "gpu" : " 17 GB", | 20260506 22:06:31.663 | INFO | "huge" : "", | 20260506 22:06:31.663 | INFO | "huge_shared" : " 27 GB", | 20260506 22:06:31.663 | INFO | "malloc" : " 14 GB", | 20260506 22:06:31.663 | INFO | "pinned" : " 29 GB" | 20260506 22:06:31.663 | INFO | } | 20260506 22:06:31.663 | INFO | Perf report for the last 5.1415 seconds: | 20260506 22:06:31.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:06:31.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:31.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:31.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:31.663 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260506 22:06:31.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:31.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:31.663 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260506 22:06:31.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:31.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:31.663 | INFO | Uploaded 36GiB to GPU in 2.00686s (17.9385GiB/s) | 20260506 22:06:31.663 | INFO | Downloaded 34GiB to GPU in 1.92741s (17.6403GiB/s) | 20260506 22:06:36.809 | INFO | Analyzing chunk frames = [4811776,4812288) | 20260506 22:06:36.809 | INFO | MemInfo: { | 20260506 22:06:36.809 | INFO | "gpu" : " 17 GB", | 20260506 22:06:36.809 | INFO | "huge" : "", | 20260506 22:06:36.809 | INFO | "huge_shared" : " 27 GB", | 20260506 22:06:36.809 | INFO | "malloc" : " 14 GB", | 20260506 22:06:36.809 | INFO | "pinned" : " 29 GB" | 20260506 22:06:36.809 | INFO | } | 20260506 22:06:36.809 | INFO | Perf report for the last 5.14557 seconds: | 20260506 22:06:36.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:06:36.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:36.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:36.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:36.809 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 22:06:36.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:36.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:36.809 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 22:06:36.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:36.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:36.809 | INFO | Uploaded 36GiB to GPU in 1.96066s (18.3611GiB/s) | 20260506 22:06:36.809 | INFO | Downloaded 34GiB to GPU in 1.99147s (17.0728GiB/s) | 20260506 22:06:41.944 | INFO | Analyzing chunk frames = [4812288,4812800) | 20260506 22:06:41.944 | INFO | MemInfo: { | 20260506 22:06:41.944 | INFO | "gpu" : " 17 GB", | 20260506 22:06:41.944 | INFO | "huge" : "", | 20260506 22:06:41.944 | INFO | "huge_shared" : " 27 GB", | 20260506 22:06:41.944 | INFO | "malloc" : " 14 GB", | 20260506 22:06:41.944 | INFO | "pinned" : " 29 GB" | 20260506 22:06:41.944 | INFO | } | 20260506 22:06:41.944 | INFO | Perf report for the last 5.13523 seconds: | 20260506 22:06:41.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:06:41.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:41.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:41.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:41.944 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 22:06:41.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:41.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:41.944 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:06:41.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:41.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:41.944 | INFO | Uploaded 36GiB to GPU in 1.98008s (18.1811GiB/s) | 20260506 22:06:41.944 | INFO | Downloaded 34GiB to GPU in 1.9787s (17.183GiB/s) | 20260506 22:06:42.043 | INFO | Starting a round of Baz IO | 20260506 22:06:45.174 | INFO | Populating RTMetrics report | 20260506 22:06:45.174 | INFO | RTMetrics report populated | 20260506 22:06:45.174 | INFO | Opening tmp RTMetrics json file | 20260506 22:06:45.175 | INFO | Serializing RTMetrics to json | 20260506 22:06:45.176 | INFO | Serialization done, swapping tmp file now... | 20260506 22:06:45.177 | INFO | RTMetrics done | 20260506 22:06:45.279 | INFO | Finished a round of Baz IO: Wrote 1.923 GiB in 3.236 seconds | 20260506 22:06:45.279 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.76% / 0.00% / 66.24% | 20260506 22:06:45.279 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.537268% / 3.97364e-06% / 5.5631e-05% | 20260506 22:06:45.279 | INFO | Baz IO overall: | 20260506 22:06:45.279 | INFO | BytesWritten : 2531162637699 | 20260506 22:06:45.279 | INFO | NumEvents : 1221563168610 | 20260506 22:06:47.085 | INFO | Analyzing chunk frames = [4812800,4813312) | 20260506 22:06:47.085 | INFO | MemInfo: { | 20260506 22:06:47.085 | INFO | "gpu" : " 17 GB", | 20260506 22:06:47.085 | INFO | "huge" : "", | 20260506 22:06:47.085 | INFO | "huge_shared" : " 27 GB", | 20260506 22:06:47.085 | INFO | "malloc" : " 14 GB", | 20260506 22:06:47.085 | INFO | "pinned" : " 29 GB" | 20260506 22:06:47.085 | INFO | } | 20260506 22:06:47.085 | INFO | Perf report for the last 5.14138 seconds: | 20260506 22:06:47.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:06:47.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:47.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:47.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:47.085 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 22:06:47.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:47.085 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 22:06:47.085 | INFO | PRE_HQ : 59.8%, 1.8, 57.2ms, 0 | 20260506 22:06:47.085 | INFO | BAZWRITER : 62.5%, 3.6, 115.5ms, 0 | 20260506 22:06:47.085 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 22:06:47.086 | INFO | Uploaded 36GiB to GPU in 2.01268s (17.8866GiB/s) | 20260506 22:06:47.086 | INFO | Downloaded 38GiB to GPU in 2.16652s (17.5396GiB/s) | 20260506 22:06:52.222 | INFO | Analyzing chunk frames = [4813312,4813824) | 20260506 22:06:52.222 | INFO | MemInfo: { | 20260506 22:06:52.222 | INFO | "gpu" : " 17 GB", | 20260506 22:06:52.222 | INFO | "huge" : "", | 20260506 22:06:52.222 | INFO | "huge_shared" : " 27 GB", | 20260506 22:06:52.222 | INFO | "malloc" : " 14 GB", | 20260506 22:06:52.222 | INFO | "pinned" : " 29 GB" | 20260506 22:06:52.222 | INFO | } | 20260506 22:06:52.222 | INFO | Perf report for the last 5.13705 seconds: | 20260506 22:06:52.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:06:52.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:52.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:52.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:52.223 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:06:52.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:52.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:52.223 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:06:52.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:52.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:52.223 | INFO | Uploaded 36GiB to GPU in 2.0127s (17.8864GiB/s) | 20260506 22:06:52.223 | INFO | Downloaded 34GiB to GPU in 1.92479s (17.6642GiB/s) | 20260506 22:06:57.360 | INFO | Analyzing chunk frames = [4813824,4814336) | 20260506 22:06:57.360 | INFO | MemInfo: { | 20260506 22:06:57.360 | INFO | "gpu" : " 17 GB", | 20260506 22:06:57.360 | INFO | "huge" : "", | 20260506 22:06:57.360 | INFO | "huge_shared" : " 27 GB", | 20260506 22:06:57.360 | INFO | "malloc" : " 14 GB", | 20260506 22:06:57.360 | INFO | "pinned" : " 29 GB" | 20260506 22:06:57.360 | INFO | } | 20260506 22:06:57.360 | INFO | Perf report for the last 5.13736 seconds: | 20260506 22:06:57.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:06:57.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:57.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:57.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:57.360 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 22:06:57.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:57.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:57.360 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:06:57.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:57.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:06:57.360 | INFO | Uploaded 36GiB to GPU in 1.99119s (18.0796GiB/s) | 20260506 22:06:57.360 | INFO | Downloaded 34GiB to GPU in 1.92923s (17.6236GiB/s) | 20260506 22:07:02.501 | INFO | Analyzing chunk frames = [4814336,4814848) | 20260506 22:07:02.501 | INFO | MemInfo: { | 20260506 22:07:02.501 | INFO | "gpu" : " 17 GB", | 20260506 22:07:02.501 | INFO | "huge" : "", | 20260506 22:07:02.501 | INFO | "huge_shared" : " 27 GB", | 20260506 22:07:02.501 | INFO | "malloc" : " 14 GB", | 20260506 22:07:02.501 | INFO | "pinned" : " 29 GB" | 20260506 22:07:02.501 | INFO | } | 20260506 22:07:02.501 | INFO | Perf report for the last 5.14142 seconds: | 20260506 22:07:02.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:07:02.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:02.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:02.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:02.501 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 22:07:02.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:02.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:02.501 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 22:07:02.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:02.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:02.501 | INFO | Uploaded 36GiB to GPU in 1.94382s (18.5202GiB/s) | 20260506 22:07:02.502 | INFO | Downloaded 34GiB to GPU in 1.87141s (18.1681GiB/s) | 20260506 22:07:07.637 | INFO | Analyzing chunk frames = [4814848,4815360) | 20260506 22:07:07.637 | INFO | MemInfo: { | 20260506 22:07:07.638 | INFO | "gpu" : " 17 GB", | 20260506 22:07:07.638 | INFO | "huge" : "", | 20260506 22:07:07.638 | INFO | "huge_shared" : " 27 GB", | 20260506 22:07:07.638 | INFO | "malloc" : " 14 GB", | 20260506 22:07:07.638 | INFO | "pinned" : " 29 GB" | 20260506 22:07:07.638 | INFO | } | 20260506 22:07:07.638 | INFO | Perf report for the last 5.13657 seconds: | 20260506 22:07:07.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:07:07.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:07.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:07.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:07.638 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 22:07:07.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:07.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:07.638 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:07:07.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:07.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:07.638 | INFO | Uploaded 36GiB to GPU in 2.0138s (17.8766GiB/s) | 20260506 22:07:07.638 | INFO | Downloaded 34GiB to GPU in 1.91428s (17.7613GiB/s) | 20260506 22:07:12.778 | INFO | Analyzing chunk frames = [4815360,4815872) | 20260506 22:07:12.778 | INFO | MemInfo: { | 20260506 22:07:12.778 | INFO | "gpu" : " 17 GB", | 20260506 22:07:12.778 | INFO | "huge" : "", | 20260506 22:07:12.778 | INFO | "huge_shared" : " 27 GB", | 20260506 22:07:12.778 | INFO | "malloc" : " 14 GB", | 20260506 22:07:12.778 | INFO | "pinned" : " 29 GB" | 20260506 22:07:12.778 | INFO | } | 20260506 22:07:12.778 | INFO | Perf report for the last 5.14004 seconds: | 20260506 22:07:12.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:07:12.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:12.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:12.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:12.778 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 22:07:12.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:12.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:12.778 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 22:07:12.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:12.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:12.778 | INFO | Uploaded 36GiB to GPU in 2.01589s (17.8581GiB/s) | 20260506 22:07:12.778 | INFO | Downloaded 34GiB to GPU in 1.92841s (17.6311GiB/s) | 20260506 22:07:17.919 | INFO | Analyzing chunk frames = [4815872,4816384) | 20260506 22:07:17.919 | INFO | MemInfo: { | 20260506 22:07:17.919 | INFO | "gpu" : " 17 GB", | 20260506 22:07:17.919 | INFO | "huge" : "", | 20260506 22:07:17.919 | INFO | "huge_shared" : " 27 GB", | 20260506 22:07:17.919 | INFO | "malloc" : " 14 GB", | 20260506 22:07:17.919 | INFO | "pinned" : " 29 GB" | 20260506 22:07:17.919 | INFO | } | 20260506 22:07:17.919 | INFO | Perf report for the last 5.14082 seconds: | 20260506 22:07:17.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:07:17.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:17.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:17.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:17.919 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 22:07:17.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:17.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:17.919 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:07:17.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:17.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:17.919 | INFO | Uploaded 36GiB to GPU in 1.96078s (18.36GiB/s) | 20260506 22:07:17.919 | INFO | Downloaded 34GiB to GPU in 1.99011s (17.0845GiB/s) | 20260506 22:07:23.060 | INFO | Analyzing chunk frames = [4816384,4816896) | 20260506 22:07:23.061 | INFO | MemInfo: { | 20260506 22:07:23.061 | INFO | "gpu" : " 17 GB", | 20260506 22:07:23.061 | INFO | "huge" : "", | 20260506 22:07:23.061 | INFO | "huge_shared" : " 27 GB", | 20260506 22:07:23.061 | INFO | "malloc" : " 14 GB", | 20260506 22:07:23.061 | INFO | "pinned" : " 29 GB" | 20260506 22:07:23.061 | INFO | } | 20260506 22:07:23.061 | INFO | Perf report for the last 5.14196 seconds: | 20260506 22:07:23.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:07:23.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:23.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:23.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:23.061 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 22:07:23.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:23.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:23.061 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:07:23.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:23.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:23.061 | INFO | Uploaded 36GiB to GPU in 2.06158s (17.4623GiB/s) | 20260506 22:07:23.061 | INFO | Downloaded 34GiB to GPU in 2.01037s (16.9123GiB/s) | 20260506 22:07:23.061 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:07:23.061 | INFO | Total mem usage: 18 GiB | 20260506 22:07:23.061 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:07:23.061 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -25.3%) | 20260506 22:07:23.061 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:07:23.061 | INFO | High Water Breakdown: | 20260506 22:07:23.061 | INFO | 60.2% : 8.65 GiB ( 75.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:07:23.061 | INFO | 39.7% : 5.7 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:07:23.061 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:07:23.061 | INFO | ------------------------------------------ | 20260506 22:07:23.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:07:23.061 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:07:23.061 | INFO | Total mem usage: 26.5 GiB | 20260506 22:07:23.061 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:07:23.061 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:07:23.061 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:07:23.061 | INFO | High Water Breakdown: | 20260506 22:07:23.061 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:07:23.061 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:07:23.061 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:07:23.061 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:07:23.061 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:07:23.061 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:07:23.061 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:07:23.061 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:07:23.061 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:07:23.061 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:07:23.061 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:07:23.061 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:07:23.061 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:07:23.061 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:07:23.061 | INFO | ------------------------------------------ | 20260506 22:07:23.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:07:23.061 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:07:23.061 | INFO | Total mem usage: 25.6 GiB | 20260506 22:07:23.061 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:07:23.061 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:07:23.061 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:07:23.061 | INFO | High Water Breakdown: | 20260506 22:07:23.061 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:07:23.061 | INFO | ------------------------------------------ | 20260506 22:07:23.062 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:07:23.062 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:07:23.062 | INFO | Total mem usage: 15.7 GiB | 20260506 22:07:23.062 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:07:23.062 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:07:23.062 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:07:23.062 | INFO | High Water Breakdown: | 20260506 22:07:23.062 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:07:23.062 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:07:23.062 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:07:23.062 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:07:23.062 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:07:23.062 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:07:23.062 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:07:23.062 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:07:23.062 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:07:23.062 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:07:23.062 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:07:23.062 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:07:23.062 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:07:23.062 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:07:23.062 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:07:23.062 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:07:23.062 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:07:23.062 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:07:23.062 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:07:23.062 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:07:23.062 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:07:23.062 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:07:23.062 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:07:23.062 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:07:23.062 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:07:23.062 | INFO | ------------------------------------------ | 20260506 22:07:23.062 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:07:23.062 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:07:23.062 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:07:23.062 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:07:23.062 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.56e+01, 1.33e+01 / 2.69e+00 | 20260506 22:07:23.062 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 9.70e-03 / 5.48e-03 | 20260506 22:07:23.062 | INFO | PerfReport: 11.65%, Baseline -- 3.62e+00 / 4.37e+00, 5.47e-01 / 3.62e-01 | 20260506 22:07:23.062 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.80e-01 / 1.42e-01 | 20260506 22:07:23.062 | INFO | PerfReport: 16.02%, DME -- 3.98e+01 / 4.44e+01, 4.01e+00 / 2.89e+00 | 20260506 22:07:23.062 | INFO | PerfReport: 57.32%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.38e+00 / 1.54e+00 | 20260506 22:07:23.062 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.21e+00 / 2.78e+00, 3.79e-01 / 1.79e-01 | 20260506 22:07:23.062 | INFO | PerfReport: 3.24%, Metrics -- 1.01e+00 / 1.34e+00, 2.67e-01 / 1.50e-01 | 20260506 22:07:23.062 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.44e-01 / 3.89e-01, 5.42e-02 / 1.85e-02 | 20260506 22:07:23.062 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:07:23.062 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:07:23.062 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:07:23.062 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 22:07:23.062 | INFO | PerfReport: 21.51%, Upload -- 2.08e+01 / 0.00e+00, 7.04e+00 / 0.00e+00 | 20260506 22:07:23.062 | INFO | PerfReport: 21.19%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 22:07:23.062 | INFO | PerfReport: 32.13%, Compute -- 3.11e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 22:07:23.062 | INFO | PerfReport: 1.31%, LockUpload -- 1.26e+00 / 0.00e+00, 4.69e+00 / 0.00e+00 | 20260506 22:07:23.062 | INFO | PerfReport: 1.15%, LockDownload -- 1.11e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260506 22:07:23.062 | INFO | PerfReport: 22.70%, LockCompute -- 2.20e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 22:07:23.062 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260506 22:07:23.172 | INFO | Starting a round of Baz IO | 20260506 22:07:26.296 | INFO | Populating RTMetrics report | 20260506 22:07:26.297 | INFO | RTMetrics report populated | 20260506 22:07:26.297 | INFO | Opening tmp RTMetrics json file | 20260506 22:07:26.298 | INFO | Serializing RTMetrics to json | 20260506 22:07:26.310 | INFO | Serialization done, swapping tmp file now... | 20260506 22:07:26.318 | INFO | RTMetrics done | 20260506 22:07:26.393 | INFO | Finished a round of Baz IO: Wrote 1.924 GiB in 3.22 seconds | 20260506 22:07:26.393 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.76% / 0.00% / 66.24% | 20260506 22:07:26.393 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.537415% / 3.97364e-06% / 6.35783e-05% | 20260506 22:07:26.393 | INFO | Baz IO overall: | 20260506 22:07:26.393 | INFO | BytesWritten : 2533228331963 | 20260506 22:07:26.393 | INFO | NumEvents : 1222550978633 | 20260506 22:07:28.198 | INFO | Analyzing chunk frames = [4816896,4817408) | 20260506 22:07:28.198 | INFO | MemInfo: { | 20260506 22:07:28.198 | INFO | "gpu" : " 17 GB", | 20260506 22:07:28.198 | INFO | "huge" : "", | 20260506 22:07:28.198 | INFO | "huge_shared" : " 27 GB", | 20260506 22:07:28.198 | INFO | "malloc" : " 14 GB", | 20260506 22:07:28.198 | INFO | "pinned" : " 29 GB" | 20260506 22:07:28.198 | INFO | } | 20260506 22:07:28.198 | INFO | Perf report for the last 5.13696 seconds: | 20260506 22:07:28.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:07:28.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:28.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:28.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:28.198 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260506 22:07:28.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:28.198 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 22:07:28.198 | INFO | PRE_HQ : 59.7%, 1.8, 56.9ms, 0 | 20260506 22:07:28.198 | INFO | BAZWRITER : 62.4%, 3.5, 111.8ms, 0 | 20260506 22:07:28.198 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:07:28.198 | INFO | Uploaded 36GiB to GPU in 2.0598s (17.4774GiB/s) | 20260506 22:07:28.198 | INFO | Downloaded 38GiB to GPU in 2.18154s (17.4188GiB/s) | 20260506 22:07:33.337 | INFO | Analyzing chunk frames = [4817408,4817920) | 20260506 22:07:33.337 | INFO | MemInfo: { | 20260506 22:07:33.337 | INFO | "gpu" : " 17 GB", | 20260506 22:07:33.337 | INFO | "huge" : "", | 20260506 22:07:33.337 | INFO | "huge_shared" : " 27 GB", | 20260506 22:07:33.337 | INFO | "malloc" : " 14 GB", | 20260506 22:07:33.337 | INFO | "pinned" : " 29 GB" | 20260506 22:07:33.337 | INFO | } | 20260506 22:07:33.337 | INFO | Perf report for the last 5.13933 seconds: | 20260506 22:07:33.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:07:33.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:33.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:33.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:33.337 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:07:33.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:33.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:33.337 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:07:33.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:33.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:33.337 | INFO | Uploaded 36GiB to GPU in 2.00256s (17.977GiB/s) | 20260506 22:07:33.337 | INFO | Downloaded 34GiB to GPU in 1.92408s (17.6708GiB/s) | 20260506 22:07:38.471 | INFO | Analyzing chunk frames = [4817920,4818432) | 20260506 22:07:38.471 | INFO | MemInfo: { | 20260506 22:07:38.471 | INFO | "gpu" : " 17 GB", | 20260506 22:07:38.471 | INFO | "huge" : "", | 20260506 22:07:38.471 | INFO | "huge_shared" : " 27 GB", | 20260506 22:07:38.471 | INFO | "malloc" : " 14 GB", | 20260506 22:07:38.471 | INFO | "pinned" : " 29 GB" | 20260506 22:07:38.471 | INFO | } | 20260506 22:07:38.471 | INFO | Perf report for the last 5.13441 seconds: | 20260506 22:07:38.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:07:38.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:38.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:38.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:38.471 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 22:07:38.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:38.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:38.472 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:07:38.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:38.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:38.472 | INFO | Uploaded 36GiB to GPU in 1.97992s (18.1825GiB/s) | 20260506 22:07:38.472 | INFO | Downloaded 34GiB to GPU in 1.91688s (17.7372GiB/s) | 20260506 22:07:43.610 | INFO | Analyzing chunk frames = [4818432,4818944) | 20260506 22:07:43.610 | INFO | MemInfo: { | 20260506 22:07:43.610 | INFO | "gpu" : " 17 GB", | 20260506 22:07:43.610 | INFO | "huge" : "", | 20260506 22:07:43.610 | INFO | "huge_shared" : " 27 GB", | 20260506 22:07:43.610 | INFO | "malloc" : " 14 GB", | 20260506 22:07:43.610 | INFO | "pinned" : " 29 GB" | 20260506 22:07:43.610 | INFO | } | 20260506 22:07:43.610 | INFO | Perf report for the last 5.13856 seconds: | 20260506 22:07:43.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:07:43.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:43.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:43.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:43.610 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 22:07:43.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:43.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:43.610 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:07:43.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:43.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:43.610 | INFO | Uploaded 36GiB to GPU in 1.95057s (18.4561GiB/s) | 20260506 22:07:43.610 | INFO | Downloaded 34GiB to GPU in 1.8778s (18.1063GiB/s) | 20260506 22:07:48.755 | INFO | Analyzing chunk frames = [4818944,4819456) | 20260506 22:07:48.755 | INFO | MemInfo: { | 20260506 22:07:48.755 | INFO | "gpu" : " 17 GB", | 20260506 22:07:48.755 | INFO | "huge" : "", | 20260506 22:07:48.755 | INFO | "huge_shared" : " 27 GB", | 20260506 22:07:48.755 | INFO | "malloc" : " 14 GB", | 20260506 22:07:48.755 | INFO | "pinned" : " 29 GB" | 20260506 22:07:48.755 | INFO | } | 20260506 22:07:48.755 | INFO | Perf report for the last 5.14506 seconds: | 20260506 22:07:48.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:07:48.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:48.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:48.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:48.755 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260506 22:07:48.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:48.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:48.755 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:07:48.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:48.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:48.755 | INFO | Uploaded 36GiB to GPU in 2.02034s (17.8188GiB/s) | 20260506 22:07:48.756 | INFO | Downloaded 34GiB to GPU in 1.90992s (17.8018GiB/s) | 20260506 22:07:53.896 | INFO | Analyzing chunk frames = [4819456,4819968) | 20260506 22:07:53.896 | INFO | MemInfo: { | 20260506 22:07:53.896 | INFO | "gpu" : " 17 GB", | 20260506 22:07:53.896 | INFO | "huge" : "", | 20260506 22:07:53.896 | INFO | "huge_shared" : " 27 GB", | 20260506 22:07:53.896 | INFO | "malloc" : " 14 GB", | 20260506 22:07:53.896 | INFO | "pinned" : " 29 GB" | 20260506 22:07:53.896 | INFO | } | 20260506 22:07:53.896 | INFO | Perf report for the last 5.14095 seconds: | 20260506 22:07:53.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:07:53.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:53.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:53.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:53.896 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 22:07:53.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:53.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:53.896 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 22:07:53.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:53.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:53.896 | INFO | Uploaded 36GiB to GPU in 2.00666s (17.9402GiB/s) | 20260506 22:07:53.896 | INFO | Downloaded 34GiB to GPU in 1.95301s (17.409GiB/s) | 20260506 22:07:59.035 | INFO | Analyzing chunk frames = [4819968,4820480) | 20260506 22:07:59.035 | INFO | MemInfo: { | 20260506 22:07:59.035 | INFO | "gpu" : " 17 GB", | 20260506 22:07:59.035 | INFO | "huge" : "", | 20260506 22:07:59.035 | INFO | "huge_shared" : " 27 GB", | 20260506 22:07:59.035 | INFO | "malloc" : " 14 GB", | 20260506 22:07:59.035 | INFO | "pinned" : " 29 GB" | 20260506 22:07:59.035 | INFO | } | 20260506 22:07:59.035 | INFO | Perf report for the last 5.13896 seconds: | 20260506 22:07:59.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:07:59.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:59.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:59.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:59.035 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 22:07:59.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:59.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:59.035 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 22:07:59.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:59.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:07:59.035 | INFO | Uploaded 36GiB to GPU in 2.00585s (17.9475GiB/s) | 20260506 22:07:59.035 | INFO | Downloaded 34GiB to GPU in 1.98146s (17.1591GiB/s) | 20260506 22:08:04.179 | INFO | Analyzing chunk frames = [4820480,4820992) | 20260506 22:08:04.179 | INFO | MemInfo: { | 20260506 22:08:04.179 | INFO | "gpu" : " 17 GB", | 20260506 22:08:04.179 | INFO | "huge" : "", | 20260506 22:08:04.179 | INFO | "huge_shared" : " 27 GB", | 20260506 22:08:04.179 | INFO | "malloc" : " 14 GB", | 20260506 22:08:04.179 | INFO | "pinned" : " 29 GB" | 20260506 22:08:04.179 | INFO | } | 20260506 22:08:04.179 | INFO | Perf report for the last 5.14397 seconds: | 20260506 22:08:04.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:08:04.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:04.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:04.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:04.179 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 22:08:04.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:04.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:04.179 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 22:08:04.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:04.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:04.179 | INFO | Uploaded 36GiB to GPU in 2.04355s (17.6164GiB/s) | 20260506 22:08:04.179 | INFO | Downloaded 34GiB to GPU in 1.9919s (17.0691GiB/s) | 20260506 22:08:04.295 | INFO | Starting a round of Baz IO | 20260506 22:08:07.418 | INFO | Populating RTMetrics report | 20260506 22:08:07.418 | INFO | RTMetrics report populated | 20260506 22:08:07.418 | INFO | Opening tmp RTMetrics json file | 20260506 22:08:07.419 | INFO | Serializing RTMetrics to json | 20260506 22:08:07.420 | INFO | Serialization done, swapping tmp file now... | 20260506 22:08:07.421 | INFO | RTMetrics done | 20260506 22:08:07.514 | INFO | Finished a round of Baz IO: Wrote 1.923 GiB in 3.219 seconds | 20260506 22:08:07.514 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.76% / 0.00% / 66.24% | 20260506 22:08:07.514 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.537133% / 1.19209e-05% / 6.75519e-05% | 20260506 22:08:07.514 | INFO | Baz IO overall: | 20260506 22:08:07.514 | INFO | BytesWritten : 2535293569447 | 20260506 22:08:07.514 | INFO | NumEvents : 1223538265265 | 20260506 22:08:09.314 | INFO | Analyzing chunk frames = [4820992,4821504) | 20260506 22:08:09.314 | INFO | MemInfo: { | 20260506 22:08:09.314 | INFO | "gpu" : " 17 GB", | 20260506 22:08:09.314 | INFO | "huge" : "", | 20260506 22:08:09.314 | INFO | "huge_shared" : " 27 GB", | 20260506 22:08:09.314 | INFO | "malloc" : " 14 GB", | 20260506 22:08:09.314 | INFO | "pinned" : " 29 GB" | 20260506 22:08:09.314 | INFO | } | 20260506 22:08:09.314 | INFO | Perf report for the last 5.13536 seconds: | 20260506 22:08:09.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:08:09.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:09.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:09.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:09.314 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260506 22:08:09.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:09.314 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 22:08:09.314 | INFO | PRE_HQ : 59.8%, 1.8, 57.1ms, 0 | 20260506 22:08:09.314 | INFO | BAZWRITER : 62.4%, 3.5, 110.9ms, 0 | 20260506 22:08:09.314 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:08:09.315 | INFO | Uploaded 36GiB to GPU in 2.04093s (17.639GiB/s) | 20260506 22:08:09.315 | INFO | Downloaded 38GiB to GPU in 2.18674s (17.3774GiB/s) | 20260506 22:08:14.451 | INFO | Analyzing chunk frames = [4821504,4822016) | 20260506 22:08:14.451 | INFO | MemInfo: { | 20260506 22:08:14.451 | INFO | "gpu" : " 17 GB", | 20260506 22:08:14.451 | INFO | "huge" : "", | 20260506 22:08:14.451 | INFO | "huge_shared" : " 27 GB", | 20260506 22:08:14.451 | INFO | "malloc" : " 14 GB", | 20260506 22:08:14.451 | INFO | "pinned" : " 29 GB" | 20260506 22:08:14.451 | INFO | } | 20260506 22:08:14.452 | INFO | Perf report for the last 5.13717 seconds: | 20260506 22:08:14.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:08:14.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:14.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:14.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:14.452 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:08:14.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:14.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:14.452 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:08:14.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:14.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:14.452 | INFO | Uploaded 36GiB to GPU in 2.0139s (17.8757GiB/s) | 20260506 22:08:14.452 | INFO | Downloaded 34GiB to GPU in 1.92367s (17.6745GiB/s) | 20260506 22:08:19.589 | INFO | Analyzing chunk frames = [4822016,4822528) | 20260506 22:08:19.589 | INFO | MemInfo: { | 20260506 22:08:19.589 | INFO | "gpu" : " 17 GB", | 20260506 22:08:19.589 | INFO | "huge" : "", | 20260506 22:08:19.589 | INFO | "huge_shared" : " 27 GB", | 20260506 22:08:19.589 | INFO | "malloc" : " 14 GB", | 20260506 22:08:19.589 | INFO | "pinned" : " 29 GB" | 20260506 22:08:19.589 | INFO | } | 20260506 22:08:19.589 | INFO | Perf report for the last 5.13743 seconds: | 20260506 22:08:19.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:08:19.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:19.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:19.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:19.589 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 22:08:19.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:19.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:19.589 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:08:19.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:19.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:19.589 | INFO | Uploaded 36GiB to GPU in 1.99137s (18.078GiB/s) | 20260506 22:08:19.589 | INFO | Downloaded 34GiB to GPU in 1.91984s (17.7098GiB/s) | 20260506 22:08:24.728 | INFO | Analyzing chunk frames = [4822528,4823040) | 20260506 22:08:24.728 | INFO | MemInfo: { | 20260506 22:08:24.728 | INFO | "gpu" : " 17 GB", | 20260506 22:08:24.728 | INFO | "huge" : "", | 20260506 22:08:24.728 | INFO | "huge_shared" : " 27 GB", | 20260506 22:08:24.728 | INFO | "malloc" : " 14 GB", | 20260506 22:08:24.728 | INFO | "pinned" : " 29 GB" | 20260506 22:08:24.729 | INFO | } | 20260506 22:08:24.729 | INFO | Perf report for the last 5.13968 seconds: | 20260506 22:08:24.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:08:24.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:24.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:24.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:24.729 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 22:08:24.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:24.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:24.729 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 22:08:24.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:24.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:24.729 | INFO | Uploaded 36GiB to GPU in 1.96701s (18.3018GiB/s) | 20260506 22:08:24.729 | INFO | Downloaded 34GiB to GPU in 1.87497s (18.1336GiB/s) | 20260506 22:08:29.873 | INFO | Analyzing chunk frames = [4823040,4823552) | 20260506 22:08:29.873 | INFO | MemInfo: { | 20260506 22:08:29.873 | INFO | "gpu" : " 17 GB", | 20260506 22:08:29.873 | INFO | "huge" : "", | 20260506 22:08:29.873 | INFO | "huge_shared" : " 27 GB", | 20260506 22:08:29.873 | INFO | "malloc" : " 14 GB", | 20260506 22:08:29.873 | INFO | "pinned" : " 29 GB" | 20260506 22:08:29.873 | INFO | } | 20260506 22:08:29.874 | INFO | Perf report for the last 5.14481 seconds: | 20260506 22:08:29.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:08:29.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:29.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:29.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:29.874 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260506 22:08:29.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:29.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:29.874 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 22:08:29.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:29.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:29.874 | INFO | Uploaded 36GiB to GPU in 2.00895s (17.9198GiB/s) | 20260506 22:08:29.874 | INFO | Downloaded 34GiB to GPU in 1.91839s (17.7232GiB/s) | 20260506 22:08:35.015 | INFO | Analyzing chunk frames = [4823552,4824064) | 20260506 22:08:35.015 | INFO | MemInfo: { | 20260506 22:08:35.015 | INFO | "gpu" : " 17 GB", | 20260506 22:08:35.015 | INFO | "huge" : "", | 20260506 22:08:35.015 | INFO | "huge_shared" : " 27 GB", | 20260506 22:08:35.015 | INFO | "malloc" : " 14 GB", | 20260506 22:08:35.015 | INFO | "pinned" : " 29 GB" | 20260506 22:08:35.015 | INFO | } | 20260506 22:08:35.015 | INFO | Perf report for the last 5.14156 seconds: | 20260506 22:08:35.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:08:35.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:35.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:35.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:35.015 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 22:08:35.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:35.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:35.015 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:08:35.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:35.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:35.016 | INFO | Uploaded 36GiB to GPU in 2.02384s (17.7879GiB/s) | 20260506 22:08:35.016 | INFO | Downloaded 34GiB to GPU in 1.93747s (17.5487GiB/s) | 20260506 22:08:40.148 | INFO | Analyzing chunk frames = [4824064,4824576) | 20260506 22:08:40.148 | INFO | MemInfo: { | 20260506 22:08:40.148 | INFO | "gpu" : " 17 GB", | 20260506 22:08:40.148 | INFO | "huge" : "", | 20260506 22:08:40.148 | INFO | "huge_shared" : " 27 GB", | 20260506 22:08:40.148 | INFO | "malloc" : " 14 GB", | 20260506 22:08:40.148 | INFO | "pinned" : " 29 GB" | 20260506 22:08:40.148 | INFO | } | 20260506 22:08:40.148 | INFO | Perf report for the last 5.13297 seconds: | 20260506 22:08:40.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:08:40.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:40.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:40.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:40.148 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:08:40.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:40.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:40.148 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260506 22:08:40.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:40.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:40.148 | INFO | Uploaded 36GiB to GPU in 1.97909s (18.1902GiB/s) | 20260506 22:08:40.148 | INFO | Downloaded 34GiB to GPU in 1.99294s (17.0602GiB/s) | 20260506 22:08:45.284 | INFO | Analyzing chunk frames = [4824576,4825088) | 20260506 22:08:45.284 | INFO | MemInfo: { | 20260506 22:08:45.284 | INFO | "gpu" : " 17 GB", | 20260506 22:08:45.284 | INFO | "huge" : "", | 20260506 22:08:45.284 | INFO | "huge_shared" : " 27 GB", | 20260506 22:08:45.284 | INFO | "malloc" : " 14 GB", | 20260506 22:08:45.284 | INFO | "pinned" : " 29 GB" | 20260506 22:08:45.284 | INFO | } | 20260506 22:08:45.284 | INFO | Perf report for the last 5.13602 seconds: | 20260506 22:08:45.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:08:45.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:45.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:45.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:45.284 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:08:45.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:45.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:45.284 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 22:08:45.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:45.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:45.284 | INFO | Uploaded 36GiB to GPU in 1.99005s (18.09GiB/s) | 20260506 22:08:45.284 | INFO | Downloaded 34GiB to GPU in 1.98487s (17.1296GiB/s) | 20260506 22:08:45.285 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:08:45.285 | INFO | Total mem usage: 18 GiB | 20260506 22:08:45.285 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:08:45.285 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 22:08:45.285 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:08:45.285 | INFO | High Water Breakdown: | 20260506 22:08:45.285 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:08:45.285 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:08:45.285 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:08:45.285 | INFO | ------------------------------------------ | 20260506 22:08:45.285 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:08:45.285 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:08:45.285 | INFO | Total mem usage: 26.5 GiB | 20260506 22:08:45.285 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:08:45.285 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:08:45.285 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:08:45.285 | INFO | High Water Breakdown: | 20260506 22:08:45.285 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:08:45.285 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:08:45.285 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:08:45.285 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:08:45.285 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:08:45.285 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:08:45.285 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:08:45.285 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:08:45.285 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:08:45.285 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:08:45.285 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:08:45.285 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:08:45.285 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:08:45.285 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:08:45.285 | INFO | ------------------------------------------ | 20260506 22:08:45.285 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:08:45.285 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:08:45.285 | INFO | Total mem usage: 25.6 GiB | 20260506 22:08:45.285 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:08:45.285 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:08:45.285 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:08:45.285 | INFO | High Water Breakdown: | 20260506 22:08:45.285 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:08:45.285 | INFO | ------------------------------------------ | 20260506 22:08:45.285 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:08:45.285 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:08:45.285 | INFO | Total mem usage: 15.7 GiB | 20260506 22:08:45.285 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:08:45.285 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:08:45.285 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:08:45.285 | INFO | High Water Breakdown: | 20260506 22:08:45.285 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:08:45.285 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:08:45.285 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:08:45.285 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:08:45.285 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:08:45.285 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:08:45.285 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:08:45.285 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:08:45.285 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:08:45.285 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:08:45.285 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:08:45.285 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:08:45.285 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:08:45.285 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:08:45.285 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:08:45.285 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:08:45.285 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:08:45.285 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:08:45.285 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:08:45.285 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:08:45.285 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:08:45.285 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:08:45.285 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:08:45.285 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:08:45.285 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:08:45.285 | INFO | ------------------------------------------ | 20260506 22:08:45.285 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:08:45.286 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:08:45.286 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:08:45.286 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:08:45.286 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.59e+01, 1.34e+01 / 2.53e+00 | 20260506 22:08:45.286 | INFO | PerfReport: 0.29%, AggregateStats -- 9.16e-02 / 1.02e-01, 1.08e-02 / 6.35e-03 | 20260506 22:08:45.286 | INFO | PerfReport: 11.66%, Baseline -- 3.63e+00 / 4.47e+00, 5.51e-01 / 3.14e-01 | 20260506 22:08:45.286 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.49e+00, 1.82e-01 / 1.14e-01 | 20260506 22:08:45.286 | INFO | PerfReport: 16.07%, DME -- 4.00e+01 / 4.49e+01, 4.16e+00 / 3.00e+00 | 20260506 22:08:45.286 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.45e+00 / 1.85e+00 | 20260506 22:08:45.286 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.19e+00 / 2.76e+00, 3.71e-01 / 2.34e-01 | 20260506 22:08:45.286 | INFO | PerfReport: 3.22%, Metrics -- 1.00e+00 / 1.33e+00, 2.62e-01 / 1.70e-01 | 20260506 22:08:45.286 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.44e-01 / 3.88e-01, 5.47e-02 / 2.06e-02 | 20260506 22:08:45.286 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:08:45.286 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:08:45.286 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:08:45.286 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 22:08:45.286 | INFO | PerfReport: 21.56%, Upload -- 2.09e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 22:08:45.286 | INFO | PerfReport: 21.15%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 22:08:45.286 | INFO | PerfReport: 32.12%, Compute -- 3.11e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 22:08:45.286 | INFO | PerfReport: 1.36%, LockUpload -- 1.32e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260506 22:08:45.286 | INFO | PerfReport: 1.10%, LockDownload -- 1.07e+00 / 0.00e+00, 2.70e+00 / 0.00e+00 | 20260506 22:08:45.286 | INFO | PerfReport: 22.69%, LockCompute -- 2.20e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 22:08:45.286 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260506 22:08:45.383 | INFO | Starting a round of Baz IO | 20260506 22:08:48.519 | INFO | Populating RTMetrics report | 20260506 22:08:48.519 | INFO | RTMetrics report populated | 20260506 22:08:48.519 | INFO | Opening tmp RTMetrics json file | 20260506 22:08:48.520 | INFO | Serializing RTMetrics to json | 20260506 22:08:48.528 | INFO | Serialization done, swapping tmp file now... | 20260506 22:08:48.541 | INFO | RTMetrics done | 20260506 22:08:48.613 | INFO | Finished a round of Baz IO: Wrote 1.922 GiB in 3.23 seconds | 20260506 22:08:48.613 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.75% / 0.00% / 66.24% | 20260506 22:08:48.613 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.536323% / 3.97364e-06% / 9.53674e-05% | 20260506 22:08:48.613 | INFO | Baz IO overall: | 20260506 22:08:48.613 | INFO | BytesWritten : 2537357011522 | 20260506 22:08:48.613 | INFO | NumEvents : 1224524242442 | 20260506 22:08:50.423 | INFO | Analyzing chunk frames = [4825088,4825600) | 20260506 22:08:50.423 | INFO | MemInfo: { | 20260506 22:08:50.423 | INFO | "gpu" : " 17 GB", | 20260506 22:08:50.423 | INFO | "huge" : "", | 20260506 22:08:50.423 | INFO | "huge_shared" : " 27 GB", | 20260506 22:08:50.423 | INFO | "malloc" : " 14 GB", | 20260506 22:08:50.423 | INFO | "pinned" : " 29 GB" | 20260506 22:08:50.423 | INFO | } | 20260506 22:08:50.423 | INFO | Perf report for the last 5.13882 seconds: | 20260506 22:08:50.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:08:50.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:50.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:50.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:50.423 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260506 22:08:50.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:50.423 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 22:08:50.423 | INFO | PRE_HQ : 59.7%, 1.8, 57.0ms, 0 | 20260506 22:08:50.423 | INFO | BAZWRITER : 62.4%, 3.4, 110.1ms, 0 | 20260506 22:08:50.423 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:08:50.423 | INFO | Uploaded 36GiB to GPU in 2.01417s (17.8733GiB/s) | 20260506 22:08:50.423 | INFO | Downloaded 38GiB to GPU in 2.1743s (17.4769GiB/s) | 20260506 22:08:55.562 | INFO | Analyzing chunk frames = [4825600,4826112) | 20260506 22:08:55.562 | INFO | MemInfo: { | 20260506 22:08:55.562 | INFO | "gpu" : " 17 GB", | 20260506 22:08:55.562 | INFO | "huge" : "", | 20260506 22:08:55.562 | INFO | "huge_shared" : " 27 GB", | 20260506 22:08:55.562 | INFO | "malloc" : " 14 GB", | 20260506 22:08:55.562 | INFO | "pinned" : " 29 GB" | 20260506 22:08:55.562 | INFO | } | 20260506 22:08:55.562 | INFO | Perf report for the last 5.13952 seconds: | 20260506 22:08:55.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:08:55.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:55.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:55.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:55.563 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:08:55.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:55.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:55.563 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:08:55.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:55.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:08:55.563 | INFO | Uploaded 36GiB to GPU in 2.01883s (17.8321GiB/s) | 20260506 22:08:55.563 | INFO | Downloaded 34GiB to GPU in 1.92274s (17.6831GiB/s) | 20260506 22:09:00.702 | INFO | Analyzing chunk frames = [4826112,4826624) | 20260506 22:09:00.702 | INFO | MemInfo: { | 20260506 22:09:00.702 | INFO | "gpu" : " 17 GB", | 20260506 22:09:00.702 | INFO | "huge" : "", | 20260506 22:09:00.702 | INFO | "huge_shared" : " 27 GB", | 20260506 22:09:00.702 | INFO | "malloc" : " 14 GB", | 20260506 22:09:00.702 | INFO | "pinned" : " 29 GB" | 20260506 22:09:00.702 | INFO | } | 20260506 22:09:00.702 | INFO | Perf report for the last 5.13938 seconds: | 20260506 22:09:00.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:09:00.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:00.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:00.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:00.702 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 22:09:00.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:00.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:00.702 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:09:00.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:00.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:00.702 | INFO | Uploaded 36GiB to GPU in 1.99114s (18.0801GiB/s) | 20260506 22:09:00.702 | INFO | Downloaded 34GiB to GPU in 1.91925s (17.7153GiB/s) | 20260506 22:09:05.847 | INFO | Analyzing chunk frames = [4826624,4827136) | 20260506 22:09:05.847 | INFO | MemInfo: { | 20260506 22:09:05.847 | INFO | "gpu" : " 17 GB", | 20260506 22:09:05.847 | INFO | "huge" : "", | 20260506 22:09:05.847 | INFO | "huge_shared" : " 27 GB", | 20260506 22:09:05.847 | INFO | "malloc" : " 14 GB", | 20260506 22:09:05.847 | INFO | "pinned" : " 29 GB" | 20260506 22:09:05.847 | INFO | } | 20260506 22:09:05.847 | INFO | Perf report for the last 5.14536 seconds: | 20260506 22:09:05.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:09:05.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:05.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:05.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:05.847 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 22:09:05.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:05.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:05.847 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:09:05.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:05.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:05.848 | INFO | Uploaded 36GiB to GPU in 1.96804s (18.2923GiB/s) | 20260506 22:09:05.848 | INFO | Downloaded 34GiB to GPU in 1.87637s (18.1201GiB/s) | 20260506 22:09:10.990 | INFO | Analyzing chunk frames = [4827136,4827648) | 20260506 22:09:10.990 | INFO | MemInfo: { | 20260506 22:09:10.990 | INFO | "gpu" : " 17 GB", | 20260506 22:09:10.990 | INFO | "huge" : "", | 20260506 22:09:10.990 | INFO | "huge_shared" : " 27 GB", | 20260506 22:09:10.990 | INFO | "malloc" : " 14 GB", | 20260506 22:09:10.990 | INFO | "pinned" : " 29 GB" | 20260506 22:09:10.990 | INFO | } | 20260506 22:09:10.990 | INFO | Perf report for the last 5.14303 seconds: | 20260506 22:09:10.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:09:10.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:10.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:10.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:10.990 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 22:09:10.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:10.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:10.990 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:09:10.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:10.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:10.991 | INFO | Uploaded 36GiB to GPU in 2.0093s (17.9167GiB/s) | 20260506 22:09:10.991 | INFO | Downloaded 34GiB to GPU in 1.91919s (17.7158GiB/s) | 20260506 22:09:16.130 | INFO | Analyzing chunk frames = [4827648,4828160) | 20260506 22:09:16.130 | INFO | MemInfo: { | 20260506 22:09:16.130 | INFO | "gpu" : " 17 GB", | 20260506 22:09:16.130 | INFO | "huge" : "", | 20260506 22:09:16.130 | INFO | "huge_shared" : " 27 GB", | 20260506 22:09:16.130 | INFO | "malloc" : " 14 GB", | 20260506 22:09:16.130 | INFO | "pinned" : " 29 GB" | 20260506 22:09:16.130 | INFO | } | 20260506 22:09:16.130 | INFO | Perf report for the last 5.13972 seconds: | 20260506 22:09:16.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:09:16.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:16.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:16.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:16.130 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 22:09:16.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:16.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:16.130 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 22:09:16.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:16.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:16.130 | INFO | Uploaded 36GiB to GPU in 2.05364s (17.5299GiB/s) | 20260506 22:09:16.130 | INFO | Downloaded 34GiB to GPU in 1.94874s (17.4471GiB/s) | 20260506 22:09:21.270 | INFO | Analyzing chunk frames = [4828160,4828672) | 20260506 22:09:21.270 | INFO | MemInfo: { | 20260506 22:09:21.270 | INFO | "gpu" : " 17 GB", | 20260506 22:09:21.270 | INFO | "huge" : "", | 20260506 22:09:21.270 | INFO | "huge_shared" : " 27 GB", | 20260506 22:09:21.270 | INFO | "malloc" : " 14 GB", | 20260506 22:09:21.270 | INFO | "pinned" : " 29 GB" | 20260506 22:09:21.270 | INFO | } | 20260506 22:09:21.270 | INFO | Perf report for the last 5.14005 seconds: | 20260506 22:09:21.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:09:21.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:21.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:21.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:21.270 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:09:21.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:21.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:21.270 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:09:21.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:21.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:21.271 | INFO | Uploaded 36GiB to GPU in 2s (18GiB/s) | 20260506 22:09:21.271 | INFO | Downloaded 34GiB to GPU in 2.00604s (16.9488GiB/s) | 20260506 22:09:26.399 | INFO | Analyzing chunk frames = [4828672,4829184) | 20260506 22:09:26.399 | INFO | MemInfo: { | 20260506 22:09:26.399 | INFO | "gpu" : " 17 GB", | 20260506 22:09:26.399 | INFO | "huge" : "", | 20260506 22:09:26.399 | INFO | "huge_shared" : " 27 GB", | 20260506 22:09:26.399 | INFO | "malloc" : " 14 GB", | 20260506 22:09:26.400 | INFO | "pinned" : " 29 GB" | 20260506 22:09:26.400 | INFO | } | 20260506 22:09:26.400 | INFO | Perf report for the last 5.12944 seconds: | 20260506 22:09:26.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:09:26.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:26.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:26.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:26.400 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:09:26.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:26.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:26.400 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:09:26.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:26.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:26.400 | INFO | Uploaded 36GiB to GPU in 1.98975s (18.0927GiB/s) | 20260506 22:09:26.400 | INFO | Downloaded 34GiB to GPU in 1.98166s (17.1573GiB/s) | 20260506 22:09:26.498 | INFO | Starting a round of Baz IO | 20260506 22:09:29.635 | INFO | Populating RTMetrics report | 20260506 22:09:29.635 | INFO | RTMetrics report populated | 20260506 22:09:29.635 | INFO | Opening tmp RTMetrics json file | 20260506 22:09:29.637 | INFO | Serializing RTMetrics to json | 20260506 22:09:29.656 | INFO | Serialization done, swapping tmp file now... | 20260506 22:09:29.656 | INFO | RTMetrics done | 20260506 22:09:29.729 | INFO | Finished a round of Baz IO: Wrote 1.922 GiB in 3.231 seconds | 20260506 22:09:29.729 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.75% / 0.00% / 66.25% | 20260506 22:09:29.729 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.533748% / 0% / 7.15256e-05% | 20260506 22:09:29.729 | INFO | Baz IO overall: | 20260506 22:09:29.729 | INFO | BytesWritten : 2539420378346 | 20260506 22:09:29.729 | INFO | NumEvents : 1225510088405 | 20260506 22:09:31.536 | INFO | Analyzing chunk frames = [4829184,4829696) | 20260506 22:09:31.536 | INFO | MemInfo: { | 20260506 22:09:31.536 | INFO | "gpu" : " 17 GB", | 20260506 22:09:31.536 | INFO | "huge" : "", | 20260506 22:09:31.536 | INFO | "huge_shared" : " 27 GB", | 20260506 22:09:31.536 | INFO | "malloc" : " 14 GB", | 20260506 22:09:31.536 | INFO | "pinned" : " 29 GB" | 20260506 22:09:31.536 | INFO | } | 20260506 22:09:31.536 | INFO | Perf report for the last 5.13687 seconds: | 20260506 22:09:31.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:09:31.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:31.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:31.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:31.536 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260506 22:09:31.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:31.537 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 22:09:31.537 | INFO | PRE_HQ : 59.9%, 1.8, 57.4ms, 0 | 20260506 22:09:31.537 | INFO | BAZWRITER : 62.5%, 3.5, 111.7ms, 0 | 20260506 22:09:31.537 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:09:31.537 | INFO | Uploaded 36GiB to GPU in 2.04566s (17.5982GiB/s) | 20260506 22:09:31.537 | INFO | Downloaded 38GiB to GPU in 2.17394s (17.4798GiB/s) | 20260506 22:09:36.685 | INFO | Analyzing chunk frames = [4829696,4830208) | 20260506 22:09:36.685 | INFO | MemInfo: { | 20260506 22:09:36.685 | INFO | "gpu" : " 17 GB", | 20260506 22:09:36.685 | INFO | "huge" : "", | 20260506 22:09:36.685 | INFO | "huge_shared" : " 27 GB", | 20260506 22:09:36.685 | INFO | "malloc" : " 14 GB", | 20260506 22:09:36.685 | INFO | "pinned" : " 29 GB" | 20260506 22:09:36.685 | INFO | } | 20260506 22:09:36.685 | INFO | Perf report for the last 5.14854 seconds: | 20260506 22:09:36.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:09:36.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:36.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:36.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:36.685 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 22:09:36.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:36.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:36.685 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:09:36.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:36.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:36.685 | INFO | Uploaded 36GiB to GPU in 2.02305s (17.7949GiB/s) | 20260506 22:09:36.686 | INFO | Downloaded 34GiB to GPU in 1.92578s (17.6552GiB/s) | 20260506 22:09:41.824 | INFO | Analyzing chunk frames = [4830208,4830720) | 20260506 22:09:41.824 | INFO | MemInfo: { | 20260506 22:09:41.824 | INFO | "gpu" : " 17 GB", | 20260506 22:09:41.824 | INFO | "huge" : "", | 20260506 22:09:41.824 | INFO | "huge_shared" : " 27 GB", | 20260506 22:09:41.824 | INFO | "malloc" : " 14 GB", | 20260506 22:09:41.824 | INFO | "pinned" : " 29 GB" | 20260506 22:09:41.824 | INFO | } | 20260506 22:09:41.824 | INFO | Perf report for the last 5.1389 seconds: | 20260506 22:09:41.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:09:41.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:41.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:41.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:41.824 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 22:09:41.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:41.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:41.824 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:09:41.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:41.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:41.824 | INFO | Uploaded 36GiB to GPU in 1.97988s (18.183GiB/s) | 20260506 22:09:41.824 | INFO | Downloaded 34GiB to GPU in 1.92877s (17.6278GiB/s) | 20260506 22:09:46.963 | INFO | Analyzing chunk frames = [4830720,4831232) | 20260506 22:09:46.963 | INFO | MemInfo: { | 20260506 22:09:46.963 | INFO | "gpu" : " 17 GB", | 20260506 22:09:46.963 | INFO | "huge" : "", | 20260506 22:09:46.963 | INFO | "huge_shared" : " 27 GB", | 20260506 22:09:46.963 | INFO | "malloc" : " 14 GB", | 20260506 22:09:46.963 | INFO | "pinned" : " 29 GB" | 20260506 22:09:46.963 | INFO | } | 20260506 22:09:46.963 | INFO | Perf report for the last 5.13924 seconds: | 20260506 22:09:46.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:09:46.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:46.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:46.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:46.963 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260506 22:09:46.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:46.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:46.963 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:09:46.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:46.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:46.963 | INFO | Uploaded 36GiB to GPU in 2.02202s (17.8039GiB/s) | 20260506 22:09:46.963 | INFO | Downloaded 34GiB to GPU in 1.9058s (17.8403GiB/s) | 20260506 22:09:52.103 | INFO | Analyzing chunk frames = [4831232,4831744) | 20260506 22:09:52.103 | INFO | MemInfo: { | 20260506 22:09:52.103 | INFO | "gpu" : " 17 GB", | 20260506 22:09:52.103 | INFO | "huge" : "", | 20260506 22:09:52.103 | INFO | "huge_shared" : " 27 GB", | 20260506 22:09:52.103 | INFO | "malloc" : " 14 GB", | 20260506 22:09:52.103 | INFO | "pinned" : " 29 GB" | 20260506 22:09:52.103 | INFO | } | 20260506 22:09:52.103 | INFO | Perf report for the last 5.13973 seconds: | 20260506 22:09:52.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:09:52.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:52.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:52.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:52.103 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 22:09:52.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:52.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:52.103 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:09:52.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:52.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:52.104 | INFO | Uploaded 36GiB to GPU in 1.9916s (18.076GiB/s) | 20260506 22:09:52.104 | INFO | Downloaded 34GiB to GPU in 1.92778s (17.6368GiB/s) | 20260506 22:09:57.239 | INFO | Analyzing chunk frames = [4831744,4832256) | 20260506 22:09:57.239 | INFO | MemInfo: { | 20260506 22:09:57.239 | INFO | "gpu" : " 17 GB", | 20260506 22:09:57.239 | INFO | "huge" : "", | 20260506 22:09:57.239 | INFO | "huge_shared" : " 27 GB", | 20260506 22:09:57.239 | INFO | "malloc" : " 14 GB", | 20260506 22:09:57.239 | INFO | "pinned" : " 29 GB" | 20260506 22:09:57.239 | INFO | } | 20260506 22:09:57.239 | INFO | Perf report for the last 5.13634 seconds: | 20260506 22:09:57.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:09:57.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:57.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:57.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:57.239 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:09:57.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:57.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:57.239 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 22:09:57.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:57.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:09:57.240 | INFO | Uploaded 36GiB to GPU in 2.02522s (17.7758GiB/s) | 20260506 22:09:57.240 | INFO | Downloaded 34GiB to GPU in 1.93781s (17.5456GiB/s) | 20260506 22:10:02.374 | INFO | Analyzing chunk frames = [4832256,4832768) | 20260506 22:10:02.374 | INFO | MemInfo: { | 20260506 22:10:02.374 | INFO | "gpu" : " 17 GB", | 20260506 22:10:02.374 | INFO | "huge" : "", | 20260506 22:10:02.374 | INFO | "huge_shared" : " 27 GB", | 20260506 22:10:02.374 | INFO | "malloc" : " 14 GB", | 20260506 22:10:02.374 | INFO | "pinned" : " 29 GB" | 20260506 22:10:02.374 | INFO | } | 20260506 22:10:02.374 | INFO | Perf report for the last 5.13506 seconds: | 20260506 22:10:02.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:10:02.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:02.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:02.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:02.375 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:10:02.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:02.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:02.375 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 22:10:02.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:02.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:02.375 | INFO | Uploaded 36GiB to GPU in 1.97789s (18.2012GiB/s) | 20260506 22:10:02.375 | INFO | Downloaded 34GiB to GPU in 1.99847s (17.013GiB/s) | 20260506 22:10:07.515 | INFO | Analyzing chunk frames = [4832768,4833280) | 20260506 22:10:07.515 | INFO | MemInfo: { | 20260506 22:10:07.515 | INFO | "gpu" : " 17 GB", | 20260506 22:10:07.515 | INFO | "huge" : "", | 20260506 22:10:07.515 | INFO | "huge_shared" : " 27 GB", | 20260506 22:10:07.515 | INFO | "malloc" : " 14 GB", | 20260506 22:10:07.515 | INFO | "pinned" : " 29 GB" | 20260506 22:10:07.515 | INFO | } | 20260506 22:10:07.515 | INFO | Perf report for the last 5.14065 seconds: | 20260506 22:10:07.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:10:07.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:07.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:07.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:07.515 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 22:10:07.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:07.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:07.515 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:10:07.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:07.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:07.515 | INFO | Uploaded 36GiB to GPU in 2.05051s (17.5566GiB/s) | 20260506 22:10:07.515 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:10:07.516 | INFO | Total mem usage: 18 GiB | 20260506 22:10:07.516 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:10:07.516 | INFO | Downloaded 34GiB to GPU in 1.99688s (17.0266GiB/s) | 20260506 22:10:07.516 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 22:10:07.516 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:10:07.516 | INFO | High Water Breakdown: | 20260506 22:10:07.516 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:10:07.516 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:10:07.516 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:10:07.516 | INFO | ------------------------------------------ | 20260506 22:10:07.516 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:10:07.516 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:10:07.516 | INFO | Total mem usage: 26.5 GiB | 20260506 22:10:07.516 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:10:07.516 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:10:07.516 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:10:07.516 | INFO | High Water Breakdown: | 20260506 22:10:07.516 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:10:07.516 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:10:07.516 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:10:07.516 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:10:07.516 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:10:07.516 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:10:07.516 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:10:07.516 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:10:07.516 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:10:07.516 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:10:07.516 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:10:07.516 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:10:07.516 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:10:07.516 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:10:07.516 | INFO | ------------------------------------------ | 20260506 22:10:07.516 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:10:07.516 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:10:07.516 | INFO | Total mem usage: 25.6 GiB | 20260506 22:10:07.516 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:10:07.516 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:10:07.516 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:10:07.516 | INFO | High Water Breakdown: | 20260506 22:10:07.516 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:10:07.516 | INFO | ------------------------------------------ | 20260506 22:10:07.516 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:10:07.516 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:10:07.516 | INFO | Total mem usage: 15.7 GiB | 20260506 22:10:07.516 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:10:07.516 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:10:07.516 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:10:07.516 | INFO | High Water Breakdown: | 20260506 22:10:07.516 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:10:07.516 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:10:07.516 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:10:07.516 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:10:07.516 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:10:07.516 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:10:07.516 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:10:07.516 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:10:07.516 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:10:07.516 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:10:07.516 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:10:07.516 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:10:07.516 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:10:07.516 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:10:07.516 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:10:07.516 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:10:07.516 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:10:07.516 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:10:07.516 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:10:07.516 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:10:07.516 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:10:07.516 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:10:07.516 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:10:07.516 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:10:07.516 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:10:07.516 | INFO | ------------------------------------------ | 20260506 22:10:07.516 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:10:07.517 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:10:07.517 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:10:07.517 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:10:07.517 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.59e+01, 1.32e+01 / 2.36e+00 | 20260506 22:10:07.517 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.00e-01, 9.48e-03 / 6.07e-03 | 20260506 22:10:07.517 | INFO | PerfReport: 11.64%, Baseline -- 3.61e+00 / 4.38e+00, 5.43e-01 / 3.74e-01 | 20260506 22:10:07.517 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.46e+00, 1.76e-01 / 1.27e-01 | 20260506 22:10:07.517 | INFO | PerfReport: 16.07%, DME -- 3.99e+01 / 4.45e+01, 4.15e+00 / 3.11e+00 | 20260506 22:10:07.517 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.78e+01 / 2.11e+01, 1.34e+00 / 2.17e+00 | 20260506 22:10:07.517 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.20e+00 / 2.72e+00, 3.77e-01 / 2.68e-01 | 20260506 22:10:07.517 | INFO | PerfReport: 3.24%, Metrics -- 1.01e+00 / 1.32e+00, 2.69e-01 / 2.08e-01 | 20260506 22:10:07.517 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.44e-01 / 3.90e-01, 5.48e-02 / 2.48e-02 | 20260506 22:10:07.517 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:10:07.517 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:10:07.517 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:10:07.517 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 22:10:07.517 | INFO | PerfReport: 21.62%, Upload -- 2.09e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260506 22:10:07.517 | INFO | PerfReport: 21.20%, Download -- 2.05e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 22:10:07.517 | INFO | PerfReport: 32.04%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 22:10:07.517 | INFO | PerfReport: 1.39%, LockUpload -- 1.35e+00 / 0.00e+00, 4.78e+00 / 0.00e+00 | 20260506 22:10:07.517 | INFO | PerfReport: 1.12%, LockDownload -- 1.09e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260506 22:10:07.517 | INFO | PerfReport: 22.60%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 22:10:07.517 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.56e-02 / 0.00e+00 | 20260506 22:10:07.626 | INFO | Starting a round of Baz IO | 20260506 22:10:10.752 | INFO | Populating RTMetrics report | 20260506 22:10:10.753 | INFO | RTMetrics report populated | 20260506 22:10:10.753 | INFO | Opening tmp RTMetrics json file | 20260506 22:10:10.753 | INFO | Serializing RTMetrics to json | 20260506 22:10:10.756 | INFO | Serialization done, swapping tmp file now... | 20260506 22:10:10.778 | INFO | RTMetrics done | 20260506 22:10:10.855 | INFO | Finished a round of Baz IO: Wrote 1.922 GiB in 3.228 seconds | 20260506 22:10:10.855 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.75% / 0.00% / 66.25% | 20260506 22:10:10.855 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.532305% / 1.19209e-05% / 7.15256e-05% | 20260506 22:10:10.855 | INFO | Baz IO overall: | 20260506 22:10:10.855 | INFO | BytesWritten : 2541483648160 | 20260506 22:10:10.855 | INFO | NumEvents : 1226495375693 | 20260506 22:10:12.654 | INFO | Analyzing chunk frames = [4833280,4833792) | 20260506 22:10:12.654 | INFO | MemInfo: { | 20260506 22:10:12.654 | INFO | "gpu" : " 17 GB", | 20260506 22:10:12.654 | INFO | "huge" : "", | 20260506 22:10:12.654 | INFO | "huge_shared" : " 27 GB", | 20260506 22:10:12.654 | INFO | "malloc" : " 14 GB", | 20260506 22:10:12.654 | INFO | "pinned" : " 29 GB" | 20260506 22:10:12.654 | INFO | } | 20260506 22:10:12.654 | INFO | Perf report for the last 5.13884 seconds: | 20260506 22:10:12.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:10:12.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:12.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:12.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:12.654 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260506 22:10:12.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:12.654 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 22:10:12.654 | INFO | PRE_HQ : 59.6%, 1.8, 57.0ms, 0 | 20260506 22:10:12.654 | INFO | BAZWRITER : 62.5%, 3.5, 111.1ms, 0 | 20260506 22:10:12.654 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:10:12.654 | INFO | Uploaded 36GiB to GPU in 2.02405s (17.7861GiB/s) | 20260506 22:10:12.654 | INFO | Downloaded 38GiB to GPU in 2.19035s (17.3488GiB/s) | 20260506 22:10:17.796 | INFO | Analyzing chunk frames = [4833792,4834304) | 20260506 22:10:17.796 | INFO | MemInfo: { | 20260506 22:10:17.796 | INFO | "gpu" : " 17 GB", | 20260506 22:10:17.796 | INFO | "huge" : "", | 20260506 22:10:17.796 | INFO | "huge_shared" : " 27 GB", | 20260506 22:10:17.796 | INFO | "malloc" : " 14 GB", | 20260506 22:10:17.796 | INFO | "pinned" : " 29 GB" | 20260506 22:10:17.796 | INFO | } | 20260506 22:10:17.796 | INFO | Perf report for the last 5.14214 seconds: | 20260506 22:10:17.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:10:17.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:17.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:17.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:17.796 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 22:10:17.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:17.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:17.796 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:10:17.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:17.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:17.796 | INFO | Uploaded 36GiB to GPU in 2.01547s (17.8619GiB/s) | 20260506 22:10:17.797 | INFO | Downloaded 34GiB to GPU in 1.92747s (17.6397GiB/s) | 20260506 22:10:22.947 | INFO | Analyzing chunk frames = [4834304,4834816) | 20260506 22:10:22.947 | INFO | MemInfo: { | 20260506 22:10:22.947 | INFO | "gpu" : " 17 GB", | 20260506 22:10:22.947 | INFO | "huge" : "", | 20260506 22:10:22.947 | INFO | "huge_shared" : " 27 GB", | 20260506 22:10:22.947 | INFO | "malloc" : " 14 GB", | 20260506 22:10:22.947 | INFO | "pinned" : " 29 GB" | 20260506 22:10:22.947 | INFO | } | 20260506 22:10:22.948 | INFO | Perf report for the last 5.1515 seconds: | 20260506 22:10:22.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:10:22.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:22.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:22.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:22.948 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 22:10:22.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:22.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:22.948 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:10:22.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:22.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:22.948 | INFO | Uploaded 36GiB to GPU in 1.98244s (18.1594GiB/s) | 20260506 22:10:22.948 | INFO | Downloaded 34GiB to GPU in 1.92749s (17.6396GiB/s) | 20260506 22:10:28.079 | INFO | Analyzing chunk frames = [4834816,4835328) | 20260506 22:10:28.079 | INFO | MemInfo: { | 20260506 22:10:28.079 | INFO | "gpu" : " 17 GB", | 20260506 22:10:28.079 | INFO | "huge" : "", | 20260506 22:10:28.079 | INFO | "huge_shared" : " 27 GB", | 20260506 22:10:28.079 | INFO | "malloc" : " 14 GB", | 20260506 22:10:28.079 | INFO | "pinned" : " 29 GB" | 20260506 22:10:28.079 | INFO | } | 20260506 22:10:28.079 | INFO | Perf report for the last 5.13162 seconds: | 20260506 22:10:28.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:10:28.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:28.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:28.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:28.079 | INFO | ANALYSIS : 60.2%, 3.0, 93.1ms, 0 | 20260506 22:10:28.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:28.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:28.079 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:10:28.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:28.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:28.079 | INFO | Uploaded 36GiB to GPU in 1.97411s (18.2361GiB/s) | 20260506 22:10:28.079 | INFO | Downloaded 34GiB to GPU in 1.90432s (17.8542GiB/s) | 20260506 22:10:33.217 | INFO | Analyzing chunk frames = [4835328,4835840) | 20260506 22:10:33.217 | INFO | MemInfo: { | 20260506 22:10:33.217 | INFO | "gpu" : " 17 GB", | 20260506 22:10:33.217 | INFO | "huge" : "", | 20260506 22:10:33.217 | INFO | "huge_shared" : " 27 GB", | 20260506 22:10:33.217 | INFO | "malloc" : " 14 GB", | 20260506 22:10:33.217 | INFO | "pinned" : " 29 GB" | 20260506 22:10:33.217 | INFO | } | 20260506 22:10:33.217 | INFO | Perf report for the last 5.13773 seconds: | 20260506 22:10:33.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:10:33.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:33.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:33.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:33.217 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 22:10:33.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:33.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:33.217 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:10:33.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:33.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:33.217 | INFO | Uploaded 36GiB to GPU in 2.0033s (17.9703GiB/s) | 20260506 22:10:33.217 | INFO | Downloaded 34GiB to GPU in 1.91975s (17.7107GiB/s) | 20260506 22:10:38.352 | INFO | Analyzing chunk frames = [4835840,4836352) | 20260506 22:10:38.352 | INFO | MemInfo: { | 20260506 22:10:38.352 | INFO | "gpu" : " 17 GB", | 20260506 22:10:38.352 | INFO | "huge" : "", | 20260506 22:10:38.352 | INFO | "huge_shared" : " 27 GB", | 20260506 22:10:38.352 | INFO | "malloc" : " 14 GB", | 20260506 22:10:38.352 | INFO | "pinned" : " 29 GB" | 20260506 22:10:38.352 | INFO | } | 20260506 22:10:38.352 | INFO | Perf report for the last 5.135 seconds: | 20260506 22:10:38.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:10:38.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:38.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:38.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:38.352 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:10:38.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:38.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:38.352 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:10:38.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:38.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:38.352 | INFO | Uploaded 36GiB to GPU in 1.9992s (18.0072GiB/s) | 20260506 22:10:38.352 | INFO | Downloaded 34GiB to GPU in 1.94158s (17.5115GiB/s) | 20260506 22:10:43.487 | INFO | Analyzing chunk frames = [4836352,4836864) | 20260506 22:10:43.487 | INFO | MemInfo: { | 20260506 22:10:43.487 | INFO | "gpu" : " 17 GB", | 20260506 22:10:43.487 | INFO | "huge" : "", | 20260506 22:10:43.487 | INFO | "huge_shared" : " 27 GB", | 20260506 22:10:43.487 | INFO | "malloc" : " 14 GB", | 20260506 22:10:43.487 | INFO | "pinned" : " 29 GB" | 20260506 22:10:43.487 | INFO | } | 20260506 22:10:43.487 | INFO | Perf report for the last 5.13536 seconds: | 20260506 22:10:43.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:10:43.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:43.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:43.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:43.487 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 22:10:43.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:43.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:43.487 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:10:43.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:43.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:43.488 | INFO | Uploaded 36GiB to GPU in 1.96378s (18.332GiB/s) | 20260506 22:10:43.488 | INFO | Downloaded 34GiB to GPU in 1.99914s (17.0073GiB/s) | 20260506 22:10:48.629 | INFO | Analyzing chunk frames = [4836864,4837376) | 20260506 22:10:48.629 | INFO | MemInfo: { | 20260506 22:10:48.629 | INFO | "gpu" : " 17 GB", | 20260506 22:10:48.629 | INFO | "huge" : "", | 20260506 22:10:48.630 | INFO | "huge_shared" : " 27 GB", | 20260506 22:10:48.630 | INFO | "malloc" : " 14 GB", | 20260506 22:10:48.630 | INFO | "pinned" : " 29 GB" | 20260506 22:10:48.630 | INFO | } | 20260506 22:10:48.630 | INFO | Perf report for the last 5.14243 seconds: | 20260506 22:10:48.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:10:48.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:48.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:48.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:48.630 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:10:48.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:48.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:48.630 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 22:10:48.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:48.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:48.630 | INFO | Uploaded 36GiB to GPU in 2.04603s (17.5951GiB/s) | 20260506 22:10:48.630 | INFO | Downloaded 34GiB to GPU in 1.99272s (17.0621GiB/s) | 20260506 22:10:48.740 | INFO | Starting a round of Baz IO | 20260506 22:10:51.868 | INFO | Populating RTMetrics report | 20260506 22:10:51.868 | INFO | RTMetrics report populated | 20260506 22:10:51.868 | INFO | Opening tmp RTMetrics json file | 20260506 22:10:51.869 | INFO | Serializing RTMetrics to json | 20260506 22:10:51.870 | INFO | Serialization done, swapping tmp file now... | 20260506 22:10:51.870 | INFO | RTMetrics done | 20260506 22:10:51.964 | INFO | Finished a round of Baz IO: Wrote 1.923 GiB in 3.223 seconds | 20260506 22:10:51.964 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.74% / 0.00% / 66.25% | 20260506 22:10:51.964 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.531467% / 3.97364e-06% / 6.35783e-05% | 20260506 22:10:51.964 | INFO | Baz IO overall: | 20260506 22:10:51.964 | INFO | BytesWritten : 2543548406939 | 20260506 22:10:51.964 | INFO | NumEvents : 1227481847426 | 20260506 22:10:53.775 | INFO | Analyzing chunk frames = [4837376,4837888) | 20260506 22:10:53.775 | INFO | MemInfo: { | 20260506 22:10:53.775 | INFO | "gpu" : " 17 GB", | 20260506 22:10:53.775 | INFO | "huge" : "", | 20260506 22:10:53.775 | INFO | "huge_shared" : " 27 GB", | 20260506 22:10:53.775 | INFO | "malloc" : " 14 GB", | 20260506 22:10:53.775 | INFO | "pinned" : " 29 GB" | 20260506 22:10:53.775 | INFO | } | 20260506 22:10:53.775 | INFO | Perf report for the last 5.14575 seconds: | 20260506 22:10:53.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:10:53.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:53.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:53.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:53.776 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 22:10:53.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:53.776 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 22:10:53.776 | INFO | PRE_HQ : 59.7%, 1.8, 57.1ms, 0 | 20260506 22:10:53.776 | INFO | BAZWRITER : 62.3%, 3.5, 111.6ms, 0 | 20260506 22:10:53.776 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 22:10:53.776 | INFO | Uploaded 36GiB to GPU in 2.02672s (17.7627GiB/s) | 20260506 22:10:53.776 | INFO | Downloaded 38GiB to GPU in 2.18756s (17.3709GiB/s) | 20260506 22:10:58.911 | INFO | Analyzing chunk frames = [4837888,4838400) | 20260506 22:10:58.911 | INFO | MemInfo: { | 20260506 22:10:58.911 | INFO | "gpu" : " 17 GB", | 20260506 22:10:58.911 | INFO | "huge" : "", | 20260506 22:10:58.911 | INFO | "huge_shared" : " 27 GB", | 20260506 22:10:58.911 | INFO | "malloc" : " 14 GB", | 20260506 22:10:58.911 | INFO | "pinned" : " 29 GB" | 20260506 22:10:58.911 | INFO | } | 20260506 22:10:58.911 | INFO | Perf report for the last 5.13569 seconds: | 20260506 22:10:58.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:10:58.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:58.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:58.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:58.911 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:10:58.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:58.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:58.911 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:10:58.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:58.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:10:58.912 | INFO | Uploaded 36GiB to GPU in 2.002s (17.982GiB/s) | 20260506 22:10:58.912 | INFO | Downloaded 34GiB to GPU in 1.92749s (17.6395GiB/s) | 20260506 22:11:04.051 | INFO | Analyzing chunk frames = [4838400,4838912) | 20260506 22:11:04.051 | INFO | MemInfo: { | 20260506 22:11:04.051 | INFO | "gpu" : " 17 GB", | 20260506 22:11:04.051 | INFO | "huge" : "", | 20260506 22:11:04.051 | INFO | "huge_shared" : " 27 GB", | 20260506 22:11:04.051 | INFO | "malloc" : " 14 GB", | 20260506 22:11:04.051 | INFO | "pinned" : " 29 GB" | 20260506 22:11:04.051 | INFO | } | 20260506 22:11:04.051 | INFO | Perf report for the last 5.13972 seconds: | 20260506 22:11:04.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:11:04.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:04.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:04.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:04.051 | INFO | ANALYSIS : 60.8%, 3.0, 94.1ms, 0 | 20260506 22:11:04.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:04.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:04.051 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 22:11:04.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:04.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:04.051 | INFO | Uploaded 36GiB to GPU in 2.00346s (17.9689GiB/s) | 20260506 22:11:04.051 | INFO | Downloaded 34GiB to GPU in 1.93174s (17.6007GiB/s) | 20260506 22:11:09.191 | INFO | Analyzing chunk frames = [4838912,4839424) | 20260506 22:11:09.191 | INFO | MemInfo: { | 20260506 22:11:09.191 | INFO | "gpu" : " 17 GB", | 20260506 22:11:09.191 | INFO | "huge" : "", | 20260506 22:11:09.191 | INFO | "huge_shared" : " 27 GB", | 20260506 22:11:09.191 | INFO | "malloc" : " 14 GB", | 20260506 22:11:09.191 | INFO | "pinned" : " 29 GB" | 20260506 22:11:09.191 | INFO | } | 20260506 22:11:09.191 | INFO | Perf report for the last 5.14058 seconds: | 20260506 22:11:09.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:11:09.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:09.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:09.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:09.192 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 22:11:09.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:09.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:09.192 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:11:09.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:09.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:09.192 | INFO | Uploaded 36GiB to GPU in 1.96019s (18.3656GiB/s) | 20260506 22:11:09.192 | INFO | Downloaded 34GiB to GPU in 1.87395s (18.1435GiB/s) | 20260506 22:11:14.328 | INFO | Analyzing chunk frames = [4839424,4839936) | 20260506 22:11:14.328 | INFO | MemInfo: { | 20260506 22:11:14.328 | INFO | "gpu" : " 17 GB", | 20260506 22:11:14.328 | INFO | "huge" : "", | 20260506 22:11:14.328 | INFO | "huge_shared" : " 27 GB", | 20260506 22:11:14.328 | INFO | "malloc" : " 14 GB", | 20260506 22:11:14.328 | INFO | "pinned" : " 29 GB" | 20260506 22:11:14.328 | INFO | } | 20260506 22:11:14.328 | INFO | Perf report for the last 5.13665 seconds: | 20260506 22:11:14.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:11:14.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:14.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:14.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:14.328 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 22:11:14.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:14.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:14.328 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:11:14.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:14.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:14.328 | INFO | Uploaded 36GiB to GPU in 2.00665s (17.9403GiB/s) | 20260506 22:11:14.328 | INFO | Downloaded 34GiB to GPU in 1.91598s (17.7455GiB/s) | 20260506 22:11:19.466 | INFO | Analyzing chunk frames = [4839936,4840448) | 20260506 22:11:19.466 | INFO | MemInfo: { | 20260506 22:11:19.466 | INFO | "gpu" : " 17 GB", | 20260506 22:11:19.466 | INFO | "huge" : "", | 20260506 22:11:19.466 | INFO | "huge_shared" : " 27 GB", | 20260506 22:11:19.466 | INFO | "malloc" : " 14 GB", | 20260506 22:11:19.466 | INFO | "pinned" : " 29 GB" | 20260506 22:11:19.466 | INFO | } | 20260506 22:11:19.466 | INFO | Perf report for the last 5.13823 seconds: | 20260506 22:11:19.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:11:19.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:19.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:19.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:19.466 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 22:11:19.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:19.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:19.466 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:11:19.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:19.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:19.467 | INFO | Uploaded 36GiB to GPU in 2.00613s (17.945GiB/s) | 20260506 22:11:19.467 | INFO | Downloaded 34GiB to GPU in 1.93729s (17.5503GiB/s) | 20260506 22:11:24.604 | INFO | Analyzing chunk frames = [4840448,4840960) | 20260506 22:11:24.604 | INFO | MemInfo: { | 20260506 22:11:24.604 | INFO | "gpu" : " 17 GB", | 20260506 22:11:24.604 | INFO | "huge" : "", | 20260506 22:11:24.605 | INFO | "huge_shared" : " 27 GB", | 20260506 22:11:24.605 | INFO | "malloc" : " 14 GB", | 20260506 22:11:24.605 | INFO | "pinned" : " 29 GB" | 20260506 22:11:24.605 | INFO | } | 20260506 22:11:24.605 | INFO | Perf report for the last 5.13844 seconds: | 20260506 22:11:24.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:11:24.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:24.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:24.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:24.605 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:11:24.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:24.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:24.605 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:11:24.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:24.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:24.605 | INFO | Uploaded 36GiB to GPU in 1.96333s (18.3362GiB/s) | 20260506 22:11:24.605 | INFO | Downloaded 34GiB to GPU in 1.99456s (17.0464GiB/s) | 20260506 22:11:29.743 | INFO | Analyzing chunk frames = [4840960,4841472) | 20260506 22:11:29.743 | INFO | MemInfo: { | 20260506 22:11:29.743 | INFO | "gpu" : " 17 GB", | 20260506 22:11:29.743 | INFO | "huge" : "", | 20260506 22:11:29.743 | INFO | "huge_shared" : " 27 GB", | 20260506 22:11:29.743 | INFO | "malloc" : " 14 GB", | 20260506 22:11:29.743 | INFO | "pinned" : " 29 GB" | 20260506 22:11:29.743 | INFO | } | 20260506 22:11:29.743 | INFO | Perf report for the last 5.13838 seconds: | 20260506 22:11:29.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:11:29.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:29.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:29.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:29.743 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 22:11:29.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:29.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:29.743 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 22:11:29.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:29.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:29.744 | INFO | Uploaded 36GiB to GPU in 1.99516s (18.0436GiB/s) | 20260506 22:11:29.744 | INFO | Downloaded 34GiB to GPU in 1.99744s (17.0218GiB/s) | 20260506 22:11:29.744 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:11:29.744 | INFO | Total mem usage: 18 GiB | 20260506 22:11:29.744 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:11:29.744 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 22:11:29.744 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:11:29.744 | INFO | High Water Breakdown: | 20260506 22:11:29.744 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:11:29.744 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:11:29.744 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:11:29.744 | INFO | ------------------------------------------ | 20260506 22:11:29.744 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:11:29.744 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:11:29.744 | INFO | Total mem usage: 26.5 GiB | 20260506 22:11:29.744 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:11:29.744 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:11:29.744 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:11:29.744 | INFO | High Water Breakdown: | 20260506 22:11:29.744 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:11:29.744 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:11:29.744 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:11:29.744 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:11:29.744 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:11:29.744 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:11:29.744 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:11:29.744 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:11:29.744 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:11:29.744 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:11:29.744 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:11:29.744 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:11:29.744 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:11:29.744 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:11:29.744 | INFO | ------------------------------------------ | 20260506 22:11:29.744 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:11:29.744 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:11:29.744 | INFO | Total mem usage: 25.6 GiB | 20260506 22:11:29.744 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:11:29.744 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:11:29.744 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:11:29.744 | INFO | High Water Breakdown: | 20260506 22:11:29.744 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:11:29.744 | INFO | ------------------------------------------ | 20260506 22:11:29.744 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:11:29.744 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:11:29.744 | INFO | Total mem usage: 15.7 GiB | 20260506 22:11:29.744 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:11:29.744 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:11:29.744 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:11:29.744 | INFO | High Water Breakdown: | 20260506 22:11:29.744 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:11:29.744 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:11:29.744 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:11:29.744 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:11:29.744 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:11:29.744 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:11:29.744 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:11:29.744 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:11:29.744 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:11:29.744 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:11:29.744 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:11:29.744 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:11:29.744 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:11:29.744 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:11:29.744 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:11:29.744 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:11:29.744 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:11:29.745 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:11:29.745 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:11:29.745 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:11:29.745 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:11:29.745 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:11:29.745 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:11:29.745 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:11:29.745 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:11:29.745 | INFO | ------------------------------------------ | 20260506 22:11:29.745 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:11:29.745 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:11:29.745 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:11:29.745 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:11:29.745 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.64e+01, 1.35e+01 / 2.80e+00 | 20260506 22:11:29.745 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.02e-01, 9.94e-03 / 4.99e-03 | 20260506 22:11:29.745 | INFO | PerfReport: 11.67%, Baseline -- 3.63e+00 / 4.44e+00, 5.54e-01 / 2.95e-01 | 20260506 22:11:29.745 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.49e+00, 1.85e-01 / 1.23e-01 | 20260506 22:11:29.745 | INFO | PerfReport: 16.09%, DME -- 4.00e+01 / 4.51e+01, 4.38e+00 / 3.12e+00 | 20260506 22:11:29.745 | INFO | PerfReport: 57.23%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.36e+00 / 2.03e+00 | 20260506 22:11:29.745 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.20e+00 / 2.76e+00, 3.80e-01 / 2.26e-01 | 20260506 22:11:29.745 | INFO | PerfReport: 3.24%, Metrics -- 1.01e+00 / 1.36e+00, 2.67e-01 / 1.76e-01 | 20260506 22:11:29.745 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.44e-01 / 3.94e-01, 5.40e-02 / 2.47e-02 | 20260506 22:11:29.745 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:11:29.745 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:11:29.745 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:11:29.745 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 22:11:29.745 | INFO | PerfReport: 21.49%, Upload -- 2.08e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 22:11:29.745 | INFO | PerfReport: 21.22%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 22:11:29.745 | INFO | PerfReport: 32.10%, Compute -- 3.11e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 22:11:29.745 | INFO | PerfReport: 1.32%, LockUpload -- 1.28e+00 / 0.00e+00, 4.77e+00 / 0.00e+00 | 20260506 22:11:29.745 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260506 22:11:29.745 | INFO | PerfReport: 22.65%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 22:11:29.745 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.47e-02 / 0.00e+00 | 20260506 22:11:29.860 | INFO | Starting a round of Baz IO | 20260506 22:11:32.982 | INFO | Populating RTMetrics report | 20260506 22:11:32.982 | INFO | RTMetrics report populated | 20260506 22:11:32.982 | INFO | Opening tmp RTMetrics json file | 20260506 22:11:33.002 | INFO | Serializing RTMetrics to json | 20260506 22:11:33.003 | INFO | Serialization done, swapping tmp file now... | 20260506 22:11:33.004 | INFO | RTMetrics done | 20260506 22:11:33.079 | INFO | Finished a round of Baz IO: Wrote 1.921 GiB in 3.218 seconds | 20260506 22:11:33.079 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.74% / 0.00% / 66.26% | 20260506 22:11:33.079 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.530485% / 0% / 9.13938e-05% | 20260506 22:11:33.079 | INFO | Baz IO overall: | 20260506 22:11:33.079 | INFO | BytesWritten : 2545610957963 | 20260506 22:11:33.079 | INFO | NumEvents : 1228466667492 | 20260506 22:11:34.890 | INFO | Analyzing chunk frames = [4841472,4841984) | 20260506 22:11:34.890 | INFO | MemInfo: { | 20260506 22:11:34.890 | INFO | "gpu" : " 17 GB", | 20260506 22:11:34.890 | INFO | "huge" : "", | 20260506 22:11:34.890 | INFO | "huge_shared" : " 27 GB", | 20260506 22:11:34.890 | INFO | "malloc" : " 14 GB", | 20260506 22:11:34.890 | INFO | "pinned" : " 29 GB" | 20260506 22:11:34.890 | INFO | } | 20260506 22:11:34.890 | INFO | Perf report for the last 5.14697 seconds: | 20260506 22:11:34.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:11:34.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:34.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:34.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:34.890 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 22:11:34.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:34.890 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260506 22:11:34.890 | INFO | PRE_HQ : 59.7%, 1.9, 57.5ms, 0 | 20260506 22:11:34.890 | INFO | BAZWRITER : 62.1%, 3.5, 110.7ms, 0 | 20260506 22:11:34.890 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 22:11:34.890 | INFO | Uploaded 36GiB to GPU in 2.04418s (17.611GiB/s) | 20260506 22:11:34.890 | INFO | Downloaded 38GiB to GPU in 2.18233s (17.4126GiB/s) | 20260506 22:11:40.030 | INFO | Analyzing chunk frames = [4841984,4842496) | 20260506 22:11:40.030 | INFO | MemInfo: { | 20260506 22:11:40.030 | INFO | "gpu" : " 17 GB", | 20260506 22:11:40.030 | INFO | "huge" : "", | 20260506 22:11:40.030 | INFO | "huge_shared" : " 27 GB", | 20260506 22:11:40.030 | INFO | "malloc" : " 14 GB", | 20260506 22:11:40.030 | INFO | "pinned" : " 29 GB" | 20260506 22:11:40.030 | INFO | } | 20260506 22:11:40.030 | INFO | Perf report for the last 5.13982 seconds: | 20260506 22:11:40.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:11:40.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:40.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:40.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:40.030 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 22:11:40.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:40.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:40.030 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 22:11:40.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:40.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:40.030 | INFO | Uploaded 36GiB to GPU in 1.99576s (18.0382GiB/s) | 20260506 22:11:40.030 | INFO | Downloaded 34GiB to GPU in 1.92344s (17.6766GiB/s) | 20260506 22:11:45.167 | INFO | Analyzing chunk frames = [4842496,4843008) | 20260506 22:11:45.167 | INFO | MemInfo: { | 20260506 22:11:45.167 | INFO | "gpu" : " 17 GB", | 20260506 22:11:45.167 | INFO | "huge" : "", | 20260506 22:11:45.167 | INFO | "huge_shared" : " 27 GB", | 20260506 22:11:45.167 | INFO | "malloc" : " 14 GB", | 20260506 22:11:45.167 | INFO | "pinned" : " 29 GB" | 20260506 22:11:45.167 | INFO | } | 20260506 22:11:45.167 | INFO | Perf report for the last 5.1374 seconds: | 20260506 22:11:45.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:11:45.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:45.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:45.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:45.168 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 22:11:45.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:45.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:45.168 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:11:45.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:45.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:45.168 | INFO | Uploaded 36GiB to GPU in 1.97998s (18.182GiB/s) | 20260506 22:11:45.168 | INFO | Downloaded 34GiB to GPU in 1.91964s (17.7117GiB/s) | 20260506 22:11:50.305 | INFO | Analyzing chunk frames = [4843008,4843520) | 20260506 22:11:50.305 | INFO | MemInfo: { | 20260506 22:11:50.305 | INFO | "gpu" : " 17 GB", | 20260506 22:11:50.305 | INFO | "huge" : "", | 20260506 22:11:50.305 | INFO | "huge_shared" : " 27 GB", | 20260506 22:11:50.305 | INFO | "malloc" : " 14 GB", | 20260506 22:11:50.305 | INFO | "pinned" : " 29 GB" | 20260506 22:11:50.305 | INFO | } | 20260506 22:11:50.305 | INFO | Perf report for the last 5.13789 seconds: | 20260506 22:11:50.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:11:50.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:50.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:50.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:50.305 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 22:11:50.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:50.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:50.305 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:11:50.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:50.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:50.306 | INFO | Uploaded 36GiB to GPU in 2.02009s (17.821GiB/s) | 20260506 22:11:50.306 | INFO | Downloaded 34GiB to GPU in 1.90681s (17.8308GiB/s) | 20260506 22:11:55.442 | INFO | Analyzing chunk frames = [4843520,4844032) | 20260506 22:11:55.442 | INFO | MemInfo: { | 20260506 22:11:55.442 | INFO | "gpu" : " 17 GB", | 20260506 22:11:55.442 | INFO | "huge" : "", | 20260506 22:11:55.442 | INFO | "huge_shared" : " 27 GB", | 20260506 22:11:55.442 | INFO | "malloc" : " 14 GB", | 20260506 22:11:55.442 | INFO | "pinned" : " 29 GB" | 20260506 22:11:55.442 | INFO | } | 20260506 22:11:55.442 | INFO | Perf report for the last 5.13698 seconds: | 20260506 22:11:55.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:11:55.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:55.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:55.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:55.442 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 22:11:55.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:55.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:55.442 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:11:55.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:55.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:11:55.443 | INFO | Uploaded 36GiB to GPU in 2.00772s (17.9308GiB/s) | 20260506 22:11:55.443 | INFO | Downloaded 34GiB to GPU in 1.92396s (17.6719GiB/s) | 20260506 22:12:00.580 | INFO | Analyzing chunk frames = [4844032,4844544) | 20260506 22:12:00.580 | INFO | MemInfo: { | 20260506 22:12:00.580 | INFO | "gpu" : " 17 GB", | 20260506 22:12:00.580 | INFO | "huge" : "", | 20260506 22:12:00.580 | INFO | "huge_shared" : " 27 GB", | 20260506 22:12:00.580 | INFO | "malloc" : " 14 GB", | 20260506 22:12:00.580 | INFO | "pinned" : " 29 GB" | 20260506 22:12:00.580 | INFO | } | 20260506 22:12:00.580 | INFO | Perf report for the last 5.13771 seconds: | 20260506 22:12:00.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:12:00.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:00.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:00.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:00.580 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 22:12:00.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:00.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:00.580 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:12:00.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:00.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:00.580 | INFO | Uploaded 36GiB to GPU in 2.00876s (17.9215GiB/s) | 20260506 22:12:00.580 | INFO | Downloaded 34GiB to GPU in 1.94167s (17.5107GiB/s) | 20260506 22:12:05.723 | INFO | Analyzing chunk frames = [4844544,4845056) | 20260506 22:12:05.723 | INFO | MemInfo: { | 20260506 22:12:05.723 | INFO | "gpu" : " 17 GB", | 20260506 22:12:05.723 | INFO | "huge" : "", | 20260506 22:12:05.723 | INFO | "huge_shared" : " 27 GB", | 20260506 22:12:05.723 | INFO | "malloc" : " 14 GB", | 20260506 22:12:05.723 | INFO | "pinned" : " 29 GB" | 20260506 22:12:05.723 | INFO | } | 20260506 22:12:05.723 | INFO | Perf report for the last 5.14287 seconds: | 20260506 22:12:05.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:12:05.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:05.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:05.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:05.723 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 22:12:05.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:05.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:05.723 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 22:12:05.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:05.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:05.723 | INFO | Uploaded 36GiB to GPU in 1.95695s (18.396GiB/s) | 20260506 22:12:05.723 | INFO | Downloaded 34GiB to GPU in 1.9904s (17.082GiB/s) | 20260506 22:12:10.865 | INFO | Analyzing chunk frames = [4845056,4845568) | 20260506 22:12:10.865 | INFO | MemInfo: { | 20260506 22:12:10.865 | INFO | "gpu" : " 17 GB", | 20260506 22:12:10.865 | INFO | "huge" : "", | 20260506 22:12:10.865 | INFO | "huge_shared" : " 27 GB", | 20260506 22:12:10.865 | INFO | "malloc" : " 14 GB", | 20260506 22:12:10.865 | INFO | "pinned" : " 29 GB" | 20260506 22:12:10.865 | INFO | } | 20260506 22:12:10.865 | INFO | Perf report for the last 5.14239 seconds: | 20260506 22:12:10.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:12:10.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:10.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:10.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:10.865 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:12:10.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:10.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:10.865 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 22:12:10.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:10.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:10.866 | INFO | Uploaded 36GiB to GPU in 1.99611s (18.0351GiB/s) | 20260506 22:12:10.866 | INFO | Downloaded 34GiB to GPU in 1.98243s (17.1506GiB/s) | 20260506 22:12:10.971 | INFO | Starting a round of Baz IO | 20260506 22:12:14.102 | INFO | Populating RTMetrics report | 20260506 22:12:14.102 | INFO | RTMetrics report populated | 20260506 22:12:14.102 | INFO | Opening tmp RTMetrics json file | 20260506 22:12:14.110 | INFO | Serializing RTMetrics to json | 20260506 22:12:14.125 | INFO | Serialization done, swapping tmp file now... | 20260506 22:12:14.125 | INFO | RTMetrics done | 20260506 22:12:14.206 | INFO | Finished a round of Baz IO: Wrote 1.921 GiB in 3.234 seconds | 20260506 22:12:14.206 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.74% / 0.00% / 66.26% | 20260506 22:12:14.206 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.530326% / 3.97364e-06% / 5.5631e-05% | 20260506 22:12:14.206 | INFO | Baz IO overall: | 20260506 22:12:14.206 | INFO | BytesWritten : 2547674027386 | 20260506 22:12:14.206 | INFO | NumEvents : 1229451997430 | 20260506 22:12:16.003 | INFO | Analyzing chunk frames = [4845568,4846080) | 20260506 22:12:16.003 | INFO | MemInfo: { | 20260506 22:12:16.003 | INFO | "gpu" : " 17 GB", | 20260506 22:12:16.003 | INFO | "huge" : "", | 20260506 22:12:16.003 | INFO | "huge_shared" : " 27 GB", | 20260506 22:12:16.003 | INFO | "malloc" : " 14 GB", | 20260506 22:12:16.003 | INFO | "pinned" : " 29 GB" | 20260506 22:12:16.003 | INFO | } | 20260506 22:12:16.003 | INFO | Perf report for the last 5.13788 seconds: | 20260506 22:12:16.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:12:16.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:16.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:16.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:16.003 | INFO | ANALYSIS : 62.4%, 3.0, 96.7ms, 0 | 20260506 22:12:16.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:16.003 | INFO | RT_METRICS : 47.0%, 1.0, 24.7ms, 0 | 20260506 22:12:16.003 | INFO | PRE_HQ : 60.5%, 2.0, 62.4ms, 0 | 20260506 22:12:16.003 | INFO | BAZWRITER : 62.4%, 3.7, 118.0ms, 0 | 20260506 22:12:16.003 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:12:16.003 | INFO | Uploaded 36GiB to GPU in 2.04383s (17.614GiB/s) | 20260506 22:12:16.003 | INFO | Downloaded 38GiB to GPU in 2.1775s (17.4512GiB/s) | 20260506 22:12:21.142 | INFO | Analyzing chunk frames = [4846080,4846592) | 20260506 22:12:21.143 | INFO | MemInfo: { | 20260506 22:12:21.143 | INFO | "gpu" : " 17 GB", | 20260506 22:12:21.143 | INFO | "huge" : "", | 20260506 22:12:21.143 | INFO | "huge_shared" : " 27 GB", | 20260506 22:12:21.143 | INFO | "malloc" : " 14 GB", | 20260506 22:12:21.143 | INFO | "pinned" : " 29 GB" | 20260506 22:12:21.143 | INFO | } | 20260506 22:12:21.143 | INFO | Perf report for the last 5.13943 seconds: | 20260506 22:12:21.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:12:21.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:21.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:21.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:21.143 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:12:21.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:21.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:21.143 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:12:21.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:21.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:21.143 | INFO | Uploaded 36GiB to GPU in 2.00923s (17.9173GiB/s) | 20260506 22:12:21.143 | INFO | Downloaded 34GiB to GPU in 1.92768s (17.6378GiB/s) | 20260506 22:12:26.281 | INFO | Analyzing chunk frames = [4846592,4847104) | 20260506 22:12:26.281 | INFO | MemInfo: { | 20260506 22:12:26.281 | INFO | "gpu" : " 17 GB", | 20260506 22:12:26.281 | INFO | "huge" : "", | 20260506 22:12:26.281 | INFO | "huge_shared" : " 27 GB", | 20260506 22:12:26.281 | INFO | "malloc" : " 14 GB", | 20260506 22:12:26.281 | INFO | "pinned" : " 29 GB" | 20260506 22:12:26.281 | INFO | } | 20260506 22:12:26.281 | INFO | Perf report for the last 5.13833 seconds: | 20260506 22:12:26.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:12:26.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:26.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:26.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:26.281 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260506 22:12:26.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:26.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:26.281 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:12:26.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:26.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:26.281 | INFO | Uploaded 36GiB to GPU in 1.97743s (18.2054GiB/s) | 20260506 22:12:26.281 | INFO | Downloaded 34GiB to GPU in 1.91918s (17.7159GiB/s) | 20260506 22:12:31.417 | INFO | Analyzing chunk frames = [4847104,4847616) | 20260506 22:12:31.417 | INFO | MemInfo: { | 20260506 22:12:31.417 | INFO | "gpu" : " 17 GB", | 20260506 22:12:31.417 | INFO | "huge" : "", | 20260506 22:12:31.417 | INFO | "huge_shared" : " 27 GB", | 20260506 22:12:31.417 | INFO | "malloc" : " 14 GB", | 20260506 22:12:31.417 | INFO | "pinned" : " 29 GB" | 20260506 22:12:31.417 | INFO | } | 20260506 22:12:31.417 | INFO | Perf report for the last 5.13648 seconds: | 20260506 22:12:31.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:12:31.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:31.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:31.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:31.418 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260506 22:12:31.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:31.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:31.418 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 22:12:31.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:31.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:31.418 | INFO | Uploaded 36GiB to GPU in 1.99383s (18.0557GiB/s) | 20260506 22:12:31.418 | INFO | Downloaded 34GiB to GPU in 1.89524s (17.9397GiB/s) | 20260506 22:12:36.559 | INFO | Analyzing chunk frames = [4847616,4848128) | 20260506 22:12:36.559 | INFO | MemInfo: { | 20260506 22:12:36.559 | INFO | "gpu" : " 17 GB", | 20260506 22:12:36.559 | INFO | "huge" : "", | 20260506 22:12:36.559 | INFO | "huge_shared" : " 27 GB", | 20260506 22:12:36.559 | INFO | "malloc" : " 14 GB", | 20260506 22:12:36.559 | INFO | "pinned" : " 29 GB" | 20260506 22:12:36.559 | INFO | } | 20260506 22:12:36.559 | INFO | Perf report for the last 5.14156 seconds: | 20260506 22:12:36.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:12:36.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:36.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:36.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:36.559 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 22:12:36.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:36.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:36.559 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:12:36.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:36.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:36.559 | INFO | Uploaded 36GiB to GPU in 2.0036s (17.9677GiB/s) | 20260506 22:12:36.559 | INFO | Downloaded 34GiB to GPU in 1.92141s (17.6954GiB/s) | 20260506 22:12:41.699 | INFO | Analyzing chunk frames = [4848128,4848640) | 20260506 22:12:41.699 | INFO | MemInfo: { | 20260506 22:12:41.699 | INFO | "gpu" : " 17 GB", | 20260506 22:12:41.699 | INFO | "huge" : "", | 20260506 22:12:41.699 | INFO | "huge_shared" : " 27 GB", | 20260506 22:12:41.699 | INFO | "malloc" : " 14 GB", | 20260506 22:12:41.699 | INFO | "pinned" : " 29 GB" | 20260506 22:12:41.699 | INFO | } | 20260506 22:12:41.699 | INFO | Perf report for the last 5.13993 seconds: | 20260506 22:12:41.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:12:41.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:41.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:41.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:41.699 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 22:12:41.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:41.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:41.699 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:12:41.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:41.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:41.699 | INFO | Uploaded 36GiB to GPU in 2.02476s (17.7799GiB/s) | 20260506 22:12:41.699 | INFO | Downloaded 34GiB to GPU in 1.93479s (17.573GiB/s) | 20260506 22:12:46.838 | INFO | Analyzing chunk frames = [4848640,4849152) | 20260506 22:12:46.838 | INFO | MemInfo: { | 20260506 22:12:46.838 | INFO | "gpu" : " 17 GB", | 20260506 22:12:46.838 | INFO | "huge" : "", | 20260506 22:12:46.838 | INFO | "huge_shared" : " 27 GB", | 20260506 22:12:46.838 | INFO | "malloc" : " 14 GB", | 20260506 22:12:46.838 | INFO | "pinned" : " 29 GB" | 20260506 22:12:46.838 | INFO | } | 20260506 22:12:46.838 | INFO | Perf report for the last 5.13921 seconds: | 20260506 22:12:46.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:12:46.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:46.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:46.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:46.838 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:12:46.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:46.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:46.838 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 22:12:46.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:46.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:46.839 | INFO | Uploaded 36GiB to GPU in 1.96486s (18.3219GiB/s) | 20260506 22:12:46.839 | INFO | Downloaded 34GiB to GPU in 1.99592s (17.0348GiB/s) | 20260506 22:12:51.986 | INFO | Analyzing chunk frames = [4849152,4849664) | 20260506 22:12:51.986 | INFO | MemInfo: { | 20260506 22:12:51.986 | INFO | "gpu" : " 17 GB", | 20260506 22:12:51.986 | INFO | "huge" : "", | 20260506 22:12:51.986 | INFO | "huge_shared" : " 27 GB", | 20260506 22:12:51.986 | INFO | "malloc" : " 14 GB", | 20260506 22:12:51.986 | INFO | "pinned" : " 29 GB" | 20260506 22:12:51.986 | INFO | } | 20260506 22:12:51.986 | INFO | Perf report for the last 5.14757 seconds: | 20260506 22:12:51.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:12:51.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:51.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:51.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:51.986 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 22:12:51.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:51.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:51.986 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 22:12:51.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:51.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:51.986 | INFO | Uploaded 36GiB to GPU in 1.98912s (18.0984GiB/s) | 20260506 22:12:51.986 | INFO | Downloaded 34GiB to GPU in 1.97689s (17.1987GiB/s) | 20260506 22:12:51.986 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:12:51.986 | INFO | Total mem usage: 18 GiB | 20260506 22:12:51.986 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:12:51.986 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -25.1%) | 20260506 22:12:51.986 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:12:51.986 | INFO | High Water Breakdown: | 20260506 22:12:51.986 | INFO | 60.3% : 8.68 GiB ( 76% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:12:51.986 | INFO | 39.6% : 5.7 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:12:51.986 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:12:51.986 | INFO | ------------------------------------------ | 20260506 22:12:51.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:12:51.986 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:12:51.986 | INFO | Total mem usage: 26.5 GiB | 20260506 22:12:51.986 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:12:51.986 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:12:51.986 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:12:51.986 | INFO | High Water Breakdown: | 20260506 22:12:51.986 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:12:51.986 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:12:51.986 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:12:51.986 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:12:51.986 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:12:51.986 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:12:51.986 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:12:51.986 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:12:51.986 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:12:51.986 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:12:51.986 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:12:51.986 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:12:51.986 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:12:51.986 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:12:51.986 | INFO | ------------------------------------------ | 20260506 22:12:51.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:12:51.986 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:12:51.986 | INFO | Total mem usage: 25.6 GiB | 20260506 22:12:51.986 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:12:51.986 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:12:51.986 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:12:51.986 | INFO | High Water Breakdown: | 20260506 22:12:51.986 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:12:51.986 | INFO | ------------------------------------------ | 20260506 22:12:51.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:12:51.987 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:12:51.987 | INFO | Total mem usage: 15.7 GiB | 20260506 22:12:51.987 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:12:51.987 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:12:51.987 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:12:51.987 | INFO | High Water Breakdown: | 20260506 22:12:51.987 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:12:51.987 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:12:51.987 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:12:51.987 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:12:51.987 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:12:51.987 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:12:51.987 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:12:51.987 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:12:51.987 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:12:51.987 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:12:51.987 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:12:51.987 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:12:51.987 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:12:51.987 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:12:51.987 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:12:51.987 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:12:51.987 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:12:51.987 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:12:51.987 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:12:51.987 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:12:51.987 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:12:51.987 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:12:51.987 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:12:51.987 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:12:51.987 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:12:51.987 | INFO | ------------------------------------------ | 20260506 22:12:51.987 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:12:51.987 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:12:51.987 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:12:51.987 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:12:51.987 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.61e+01, 1.33e+01 / 2.39e+00 | 20260506 22:12:51.987 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.02e-01, 9.66e-03 / 6.64e-03 | 20260506 22:12:51.987 | INFO | PerfReport: 11.67%, Baseline -- 3.63e+00 / 4.42e+00, 5.47e-01 / 3.22e-01 | 20260506 22:12:51.987 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.44e+00, 1.76e-01 / 1.25e-01 | 20260506 22:12:51.987 | INFO | PerfReport: 16.06%, DME -- 3.99e+01 / 4.47e+01, 4.06e+00 / 2.84e+00 | 20260506 22:12:51.987 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.39e+00 / 2.00e+00 | 20260506 22:12:51.987 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.20e+00 / 2.76e+00, 3.74e-01 / 2.06e-01 | 20260506 22:12:51.987 | INFO | PerfReport: 3.22%, Metrics -- 1.00e+00 / 1.30e+00, 2.61e-01 / 1.56e-01 | 20260506 22:12:51.987 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.44e-01 / 3.91e-01, 5.40e-02 / 2.64e-02 | 20260506 22:12:51.987 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:12:51.987 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:12:51.987 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:12:51.987 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 22:12:51.987 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 22:12:51.987 | INFO | PerfReport: 21.19%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 22:12:51.987 | INFO | PerfReport: 32.10%, Compute -- 3.11e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 22:12:51.987 | INFO | PerfReport: 1.35%, LockUpload -- 1.30e+00 / 0.00e+00, 4.79e+00 / 0.00e+00 | 20260506 22:12:51.987 | INFO | PerfReport: 1.18%, LockDownload -- 1.14e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 22:12:51.987 | INFO | PerfReport: 22.64%, LockCompute -- 2.19e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 22:12:51.987 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.89e-02 / 0.00e+00 | 20260506 22:12:52.085 | INFO | Starting a round of Baz IO | 20260506 22:12:55.216 | INFO | Populating RTMetrics report | 20260506 22:12:55.217 | INFO | RTMetrics report populated | 20260506 22:12:55.217 | INFO | Opening tmp RTMetrics json file | 20260506 22:12:55.217 | INFO | Serializing RTMetrics to json | 20260506 22:12:55.219 | INFO | Serialization done, swapping tmp file now... | 20260506 22:12:55.221 | INFO | RTMetrics done | 20260506 22:12:55.315 | INFO | Finished a round of Baz IO: Wrote 1.925 GiB in 3.229 seconds | 20260506 22:12:55.315 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.74% / 0.00% / 66.26% | 20260506 22:12:55.315 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.527211% / 3.97364e-06% / 5.16574e-05% | 20260506 22:12:55.315 | INFO | Baz IO overall: | 20260506 22:12:55.315 | INFO | BytesWritten : 2549740530686 | 20260506 22:12:55.315 | INFO | NumEvents : 1230437909249 | 20260506 22:12:57.117 | INFO | Analyzing chunk frames = [4849664,4850176) | 20260506 22:12:57.117 | INFO | MemInfo: { | 20260506 22:12:57.117 | INFO | "gpu" : " 17 GB", | 20260506 22:12:57.117 | INFO | "huge" : "", | 20260506 22:12:57.117 | INFO | "huge_shared" : " 27 GB", | 20260506 22:12:57.117 | INFO | "malloc" : " 14 GB", | 20260506 22:12:57.117 | INFO | "pinned" : " 29 GB" | 20260506 22:12:57.117 | INFO | } | 20260506 22:12:57.117 | INFO | Perf report for the last 5.13142 seconds: | 20260506 22:12:57.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:12:57.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:57.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:57.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:57.117 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260506 22:12:57.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:12:57.117 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 22:12:57.117 | INFO | PRE_HQ : 60.1%, 1.8, 57.1ms, 0 | 20260506 22:12:57.117 | INFO | BAZWRITER : 62.5%, 3.5, 111.3ms, 0 | 20260506 22:12:57.117 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:12:57.118 | INFO | Uploaded 36GiB to GPU in 2.0274s (17.7567GiB/s) | 20260506 22:12:57.118 | INFO | Downloaded 38GiB to GPU in 2.17293s (17.4879GiB/s) | 20260506 22:13:02.256 | INFO | Analyzing chunk frames = [4850176,4850688) | 20260506 22:13:02.256 | INFO | MemInfo: { | 20260506 22:13:02.256 | INFO | "gpu" : " 17 GB", | 20260506 22:13:02.256 | INFO | "huge" : "", | 20260506 22:13:02.256 | INFO | "huge_shared" : " 27 GB", | 20260506 22:13:02.256 | INFO | "malloc" : " 14 GB", | 20260506 22:13:02.256 | INFO | "pinned" : " 29 GB" | 20260506 22:13:02.256 | INFO | } | 20260506 22:13:02.256 | INFO | Perf report for the last 5.13872 seconds: | 20260506 22:13:02.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:13:02.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:02.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:02.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:02.256 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:13:02.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:02.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:02.256 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:13:02.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:02.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:02.256 | INFO | Uploaded 36GiB to GPU in 2.00277s (17.9751GiB/s) | 20260506 22:13:02.256 | INFO | Downloaded 34GiB to GPU in 1.92397s (17.6718GiB/s) | 20260506 22:13:07.394 | INFO | Analyzing chunk frames = [4850688,4851200) | 20260506 22:13:07.394 | INFO | MemInfo: { | 20260506 22:13:07.395 | INFO | "gpu" : " 17 GB", | 20260506 22:13:07.395 | INFO | "huge" : "", | 20260506 22:13:07.395 | INFO | "huge_shared" : " 27 GB", | 20260506 22:13:07.395 | INFO | "malloc" : " 14 GB", | 20260506 22:13:07.395 | INFO | "pinned" : " 29 GB" | 20260506 22:13:07.395 | INFO | } | 20260506 22:13:07.395 | INFO | Perf report for the last 5.13863 seconds: | 20260506 22:13:07.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:13:07.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:07.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:07.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:07.395 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 22:13:07.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:07.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:07.395 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:13:07.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:07.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:07.395 | INFO | Uploaded 36GiB to GPU in 1.99117s (18.0798GiB/s) | 20260506 22:13:07.395 | INFO | Downloaded 34GiB to GPU in 1.9243s (17.6688GiB/s) | 20260506 22:13:12.531 | INFO | Analyzing chunk frames = [4851200,4851712) | 20260506 22:13:12.531 | INFO | MemInfo: { | 20260506 22:13:12.531 | INFO | "gpu" : " 17 GB", | 20260506 22:13:12.531 | INFO | "huge" : "", | 20260506 22:13:12.531 | INFO | "huge_shared" : " 27 GB", | 20260506 22:13:12.531 | INFO | "malloc" : " 14 GB", | 20260506 22:13:12.531 | INFO | "pinned" : " 29 GB" | 20260506 22:13:12.531 | INFO | } | 20260506 22:13:12.531 | INFO | Perf report for the last 5.13644 seconds: | 20260506 22:13:12.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:13:12.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:12.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:12.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:12.531 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260506 22:13:12.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:12.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:12.531 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:13:12.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:12.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:12.531 | INFO | Uploaded 36GiB to GPU in 1.99347s (18.059GiB/s) | 20260506 22:13:12.531 | INFO | Downloaded 34GiB to GPU in 1.90136s (17.8819GiB/s) | 20260506 22:13:17.672 | INFO | Analyzing chunk frames = [4851712,4852224) | 20260506 22:13:17.672 | INFO | MemInfo: { | 20260506 22:13:17.672 | INFO | "gpu" : " 17 GB", | 20260506 22:13:17.672 | INFO | "huge" : "", | 20260506 22:13:17.672 | INFO | "huge_shared" : " 27 GB", | 20260506 22:13:17.672 | INFO | "malloc" : " 14 GB", | 20260506 22:13:17.672 | INFO | "pinned" : " 29 GB" | 20260506 22:13:17.672 | INFO | } | 20260506 22:13:17.672 | INFO | Perf report for the last 5.14085 seconds: | 20260506 22:13:17.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:13:17.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:17.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:17.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:17.672 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 22:13:17.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:17.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:17.672 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 22:13:17.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:17.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:17.672 | INFO | Uploaded 36GiB to GPU in 2.00996s (17.9108GiB/s) | 20260506 22:13:17.672 | INFO | Downloaded 34GiB to GPU in 1.92234s (17.6867GiB/s) | 20260506 22:13:22.807 | INFO | Analyzing chunk frames = [4852224,4852736) | 20260506 22:13:22.807 | INFO | MemInfo: { | 20260506 22:13:22.807 | INFO | "gpu" : " 17 GB", | 20260506 22:13:22.807 | INFO | "huge" : "", | 20260506 22:13:22.807 | INFO | "huge_shared" : " 27 GB", | 20260506 22:13:22.807 | INFO | "malloc" : " 14 GB", | 20260506 22:13:22.807 | INFO | "pinned" : " 29 GB" | 20260506 22:13:22.807 | INFO | } | 20260506 22:13:22.807 | INFO | Perf report for the last 5.1351 seconds: | 20260506 22:13:22.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:13:22.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:22.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:22.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:22.807 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 22:13:22.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:22.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:22.807 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 22:13:22.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:22.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:22.807 | INFO | Uploaded 36GiB to GPU in 2.0275s (17.7558GiB/s) | 20260506 22:13:22.807 | INFO | Downloaded 34GiB to GPU in 1.94616s (17.4703GiB/s) | 20260506 22:13:27.951 | INFO | Analyzing chunk frames = [4852736,4853248) | 20260506 22:13:27.951 | INFO | MemInfo: { | 20260506 22:13:27.951 | INFO | "gpu" : " 17 GB", | 20260506 22:13:27.951 | INFO | "huge" : "", | 20260506 22:13:27.951 | INFO | "huge_shared" : " 27 GB", | 20260506 22:13:27.951 | INFO | "malloc" : " 14 GB", | 20260506 22:13:27.951 | INFO | "pinned" : " 29 GB" | 20260506 22:13:27.951 | INFO | } | 20260506 22:13:27.951 | INFO | Perf report for the last 5.14397 seconds: | 20260506 22:13:27.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:13:27.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:27.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:27.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:27.951 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 22:13:27.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:27.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:27.951 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 22:13:27.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:27.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:27.951 | INFO | Uploaded 36GiB to GPU in 1.94287s (18.5293GiB/s) | 20260506 22:13:27.951 | INFO | Downloaded 34GiB to GPU in 1.98227s (17.152GiB/s) | 20260506 22:13:33.093 | INFO | Analyzing chunk frames = [4853248,4853760) | 20260506 22:13:33.093 | INFO | MemInfo: { | 20260506 22:13:33.093 | INFO | "gpu" : " 17 GB", | 20260506 22:13:33.094 | INFO | "huge" : "", | 20260506 22:13:33.094 | INFO | "huge_shared" : " 27 GB", | 20260506 22:13:33.094 | INFO | "malloc" : " 14 GB", | 20260506 22:13:33.094 | INFO | "pinned" : " 29 GB" | 20260506 22:13:33.094 | INFO | } | 20260506 22:13:33.094 | INFO | Perf report for the last 5.14256 seconds: | 20260506 22:13:33.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:13:33.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:33.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:33.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:33.094 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 22:13:33.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:33.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:33.094 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 22:13:33.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:33.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:33.094 | INFO | Uploaded 36GiB to GPU in 2.04309s (17.6203GiB/s) | 20260506 22:13:33.094 | INFO | Downloaded 34GiB to GPU in 1.9676s (17.2799GiB/s) | 20260506 22:13:33.207 | INFO | Starting a round of Baz IO | 20260506 22:13:36.332 | INFO | Populating RTMetrics report | 20260506 22:13:36.333 | INFO | RTMetrics report populated | 20260506 22:13:36.333 | INFO | Opening tmp RTMetrics json file | 20260506 22:13:36.333 | INFO | Serializing RTMetrics to json | 20260506 22:13:36.335 | INFO | Serialization done, swapping tmp file now... | 20260506 22:13:36.335 | INFO | RTMetrics done | 20260506 22:13:36.430 | INFO | Finished a round of Baz IO: Wrote 1.923 GiB in 3.222 seconds | 20260506 22:13:36.430 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.73% / 0.00% / 66.26% | 20260506 22:13:36.430 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.523702% / 7.94729e-06% / 7.54992e-05% | 20260506 22:13:36.430 | INFO | Baz IO overall: | 20260506 22:13:36.430 | INFO | BytesWritten : 2551805217237 | 20260506 22:13:36.431 | INFO | NumEvents : 1231422543119 | 20260506 22:13:38.235 | INFO | Analyzing chunk frames = [4853760,4854272) | 20260506 22:13:38.235 | INFO | MemInfo: { | 20260506 22:13:38.235 | INFO | "gpu" : " 17 GB", | 20260506 22:13:38.235 | INFO | "huge" : "", | 20260506 22:13:38.235 | INFO | "huge_shared" : " 27 GB", | 20260506 22:13:38.235 | INFO | "malloc" : " 14 GB", | 20260506 22:13:38.235 | INFO | "pinned" : " 29 GB" | 20260506 22:13:38.235 | INFO | } | 20260506 22:13:38.235 | INFO | Perf report for the last 5.14111 seconds: | 20260506 22:13:38.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:13:38.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:38.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:38.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:38.235 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 22:13:38.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:38.235 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 22:13:38.235 | INFO | PRE_HQ : 59.8%, 1.8, 57.3ms, 0 | 20260506 22:13:38.235 | INFO | BAZWRITER : 62.4%, 3.5, 111.6ms, 0 | 20260506 22:13:38.235 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:13:38.235 | INFO | Uploaded 36GiB to GPU in 2.03933s (17.6529GiB/s) | 20260506 22:13:38.235 | INFO | Downloaded 38GiB to GPU in 2.18752s (17.3713GiB/s) | 20260506 22:13:43.373 | INFO | Analyzing chunk frames = [4854272,4854784) | 20260506 22:13:43.373 | INFO | MemInfo: { | 20260506 22:13:43.373 | INFO | "gpu" : " 17 GB", | 20260506 22:13:43.373 | INFO | "huge" : "", | 20260506 22:13:43.373 | INFO | "huge_shared" : " 27 GB", | 20260506 22:13:43.373 | INFO | "malloc" : " 14 GB", | 20260506 22:13:43.373 | INFO | "pinned" : " 29 GB" | 20260506 22:13:43.373 | INFO | } | 20260506 22:13:43.373 | INFO | Perf report for the last 5.13842 seconds: | 20260506 22:13:43.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:13:43.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:43.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:43.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:43.373 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:13:43.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:43.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:43.373 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:13:43.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:43.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:43.373 | INFO | Uploaded 36GiB to GPU in 1.99915s (18.0077GiB/s) | 20260506 22:13:43.374 | INFO | Downloaded 34GiB to GPU in 1.92363s (17.6749GiB/s) | 20260506 22:13:48.511 | INFO | Analyzing chunk frames = [4854784,4855296) | 20260506 22:13:48.511 | INFO | MemInfo: { | 20260506 22:13:48.511 | INFO | "gpu" : " 17 GB", | 20260506 22:13:48.511 | INFO | "huge" : "", | 20260506 22:13:48.511 | INFO | "huge_shared" : " 27 GB", | 20260506 22:13:48.511 | INFO | "malloc" : " 14 GB", | 20260506 22:13:48.511 | INFO | "pinned" : " 29 GB" | 20260506 22:13:48.511 | INFO | } | 20260506 22:13:48.511 | INFO | Perf report for the last 5.1381 seconds: | 20260506 22:13:48.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:13:48.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:48.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:48.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:48.511 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 22:13:48.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:48.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:48.511 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:13:48.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:48.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:48.511 | INFO | Uploaded 36GiB to GPU in 2.00615s (17.9448GiB/s) | 20260506 22:13:48.512 | INFO | Downloaded 34GiB to GPU in 1.93309s (17.5884GiB/s) | 20260506 22:13:53.650 | INFO | Analyzing chunk frames = [4855296,4855808) | 20260506 22:13:53.650 | INFO | MemInfo: { | 20260506 22:13:53.650 | INFO | "gpu" : " 17 GB", | 20260506 22:13:53.650 | INFO | "huge" : "", | 20260506 22:13:53.650 | INFO | "huge_shared" : " 27 GB", | 20260506 22:13:53.650 | INFO | "malloc" : " 14 GB", | 20260506 22:13:53.650 | INFO | "pinned" : " 29 GB" | 20260506 22:13:53.650 | INFO | } | 20260506 22:13:53.650 | INFO | Perf report for the last 5.13845 seconds: | 20260506 22:13:53.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:13:53.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:53.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:53.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:53.650 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260506 22:13:53.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:53.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:53.650 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:13:53.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:53.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:53.650 | INFO | Uploaded 36GiB to GPU in 2.02198s (17.8044GiB/s) | 20260506 22:13:53.650 | INFO | Downloaded 34GiB to GPU in 1.90799s (17.8198GiB/s) | 20260506 22:13:58.786 | INFO | Analyzing chunk frames = [4855808,4856320) | 20260506 22:13:58.786 | INFO | MemInfo: { | 20260506 22:13:58.786 | INFO | "gpu" : " 17 GB", | 20260506 22:13:58.786 | INFO | "huge" : "", | 20260506 22:13:58.786 | INFO | "huge_shared" : " 27 GB", | 20260506 22:13:58.786 | INFO | "malloc" : " 14 GB", | 20260506 22:13:58.787 | INFO | "pinned" : " 29 GB" | 20260506 22:13:58.787 | INFO | } | 20260506 22:13:58.787 | INFO | Perf report for the last 5.13689 seconds: | 20260506 22:13:58.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:13:58.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:58.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:58.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:58.787 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260506 22:13:58.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:58.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:58.787 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 22:13:58.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:58.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:13:58.787 | INFO | Uploaded 36GiB to GPU in 2.00425s (17.9618GiB/s) | 20260506 22:13:58.787 | INFO | Downloaded 34GiB to GPU in 1.92587s (17.6543GiB/s) | 20260506 22:14:03.926 | INFO | Analyzing chunk frames = [4856320,4856832) | 20260506 22:14:03.926 | INFO | MemInfo: { | 20260506 22:14:03.926 | INFO | "gpu" : " 17 GB", | 20260506 22:14:03.926 | INFO | "huge" : "", | 20260506 22:14:03.926 | INFO | "huge_shared" : " 27 GB", | 20260506 22:14:03.926 | INFO | "malloc" : " 14 GB", | 20260506 22:14:03.926 | INFO | "pinned" : " 29 GB" | 20260506 22:14:03.926 | INFO | } | 20260506 22:14:03.926 | INFO | Perf report for the last 5.13965 seconds: | 20260506 22:14:03.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:14:03.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:03.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:03.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:03.926 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 22:14:03.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:03.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:03.926 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:14:03.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:03.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:03.926 | INFO | Uploaded 36GiB to GPU in 1.99581s (18.0378GiB/s) | 20260506 22:14:03.927 | INFO | Downloaded 34GiB to GPU in 1.94752s (17.4581GiB/s) | 20260506 22:14:09.072 | INFO | Analyzing chunk frames = [4856832,4857344) | 20260506 22:14:09.072 | INFO | MemInfo: { | 20260506 22:14:09.072 | INFO | "gpu" : " 17 GB", | 20260506 22:14:09.072 | INFO | "huge" : "", | 20260506 22:14:09.072 | INFO | "huge_shared" : " 27 GB", | 20260506 22:14:09.072 | INFO | "malloc" : " 14 GB", | 20260506 22:14:09.072 | INFO | "pinned" : " 29 GB" | 20260506 22:14:09.072 | INFO | } | 20260506 22:14:09.072 | INFO | Perf report for the last 5.14595 seconds: | 20260506 22:14:09.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:14:09.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:09.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:09.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:09.072 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260506 22:14:09.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:09.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:09.072 | INFO | PRE_HQ : 27.0%, 1.0, 14.1ms, 0 | 20260506 22:14:09.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:09.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:09.073 | INFO | Uploaded 36GiB to GPU in 1.9605s (18.3627GiB/s) | 20260506 22:14:09.073 | INFO | Downloaded 34GiB to GPU in 1.99055s (17.0807GiB/s) | 20260506 22:14:14.205 | INFO | Analyzing chunk frames = [4857344,4857856) | 20260506 22:14:14.205 | INFO | MemInfo: { | 20260506 22:14:14.206 | INFO | "gpu" : " 17 GB", | 20260506 22:14:14.206 | INFO | "huge" : "", | 20260506 22:14:14.206 | INFO | "huge_shared" : " 27 GB", | 20260506 22:14:14.206 | INFO | "malloc" : " 14 GB", | 20260506 22:14:14.206 | INFO | "pinned" : " 29 GB" | 20260506 22:14:14.206 | INFO | } | 20260506 22:14:14.206 | INFO | Perf report for the last 5.1334 seconds: | 20260506 22:14:14.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:14:14.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:14.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:14.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:14.206 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 22:14:14.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:14.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:14.206 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:14:14.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:14.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:14.206 | INFO | Uploaded 36GiB to GPU in 1.98279s (18.1562GiB/s) | 20260506 22:14:14.206 | INFO | Downloaded 34GiB to GPU in 1.97272s (17.2351GiB/s) | 20260506 22:14:14.206 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:14:14.206 | INFO | Total mem usage: 18 GiB | 20260506 22:14:14.206 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:14:14.206 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 22:14:14.206 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:14:14.206 | INFO | High Water Breakdown: | 20260506 22:14:14.206 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:14:14.206 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:14:14.206 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:14:14.206 | INFO | ------------------------------------------ | 20260506 22:14:14.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:14:14.206 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:14:14.206 | INFO | Total mem usage: 26.5 GiB | 20260506 22:14:14.206 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:14:14.206 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:14:14.206 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:14:14.206 | INFO | High Water Breakdown: | 20260506 22:14:14.206 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:14:14.206 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:14:14.206 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:14:14.206 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:14:14.206 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:14:14.206 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:14:14.206 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:14:14.206 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:14:14.206 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:14:14.206 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:14:14.206 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:14:14.206 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:14:14.206 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:14:14.206 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:14:14.206 | INFO | ------------------------------------------ | 20260506 22:14:14.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:14:14.207 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:14:14.207 | INFO | Total mem usage: 25.6 GiB | 20260506 22:14:14.207 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:14:14.207 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:14:14.207 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:14:14.207 | INFO | High Water Breakdown: | 20260506 22:14:14.207 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:14:14.207 | INFO | ------------------------------------------ | 20260506 22:14:14.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:14:14.207 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:14:14.207 | INFO | Total mem usage: 15.7 GiB | 20260506 22:14:14.207 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:14:14.207 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:14:14.207 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:14:14.207 | INFO | High Water Breakdown: | 20260506 22:14:14.207 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:14:14.207 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:14:14.207 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:14:14.207 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:14:14.207 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:14:14.207 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:14:14.207 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:14:14.207 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:14:14.207 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:14:14.207 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:14:14.207 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:14:14.207 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:14:14.207 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:14:14.207 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:14:14.207 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:14:14.207 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:14:14.207 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:14:14.207 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:14:14.207 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:14:14.207 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:14:14.207 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:14:14.207 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:14:14.207 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:14:14.207 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:14:14.207 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:14:14.207 | INFO | ------------------------------------------ | 20260506 22:14:14.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:14:14.207 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:14:14.207 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:14:14.207 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:14:14.207 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.55e+01, 1.36e+01 / 2.23e+00 | 20260506 22:14:14.207 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.48e-03 / 7.01e-03 | 20260506 22:14:14.207 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.35e+00, 5.50e-01 / 3.82e-01 | 20260506 22:14:14.207 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.44e+00, 1.83e-01 / 1.51e-01 | 20260506 22:14:14.207 | INFO | PerfReport: 15.94%, DME -- 3.96e+01 / 4.36e+01, 3.76e+00 / 2.26e+00 | 20260506 22:14:14.207 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.79e+01 / 2.16e+01, 1.47e+00 / 1.87e+00 | 20260506 22:14:14.207 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.19e+00 / 2.69e+00, 3.64e-01 / 2.40e-01 | 20260506 22:14:14.207 | INFO | PerfReport: 3.23%, Metrics -- 1.00e+00 / 1.33e+00, 2.66e-01 / 1.58e-01 | 20260506 22:14:14.207 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.43e-01 / 3.92e-01, 5.38e-02 / 2.07e-02 | 20260506 22:14:14.207 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:14:14.207 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:14:14.207 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:14:14.207 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.30e+01 / 0.00e+00 | 20260506 22:14:14.207 | INFO | PerfReport: 21.57%, Upload -- 2.09e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260506 22:14:14.207 | INFO | PerfReport: 21.22%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 22:14:14.207 | INFO | PerfReport: 32.14%, Compute -- 3.11e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 22:14:14.207 | INFO | PerfReport: 1.33%, LockUpload -- 1.28e+00 / 0.00e+00, 4.62e+00 / 0.00e+00 | 20260506 22:14:14.207 | INFO | PerfReport: 1.11%, LockDownload -- 1.08e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260506 22:14:14.207 | INFO | PerfReport: 22.62%, LockCompute -- 2.19e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 22:14:14.207 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260506 22:14:14.304 | INFO | Starting a round of Baz IO | 20260506 22:14:17.442 | INFO | Populating RTMetrics report | 20260506 22:14:17.442 | INFO | RTMetrics report populated | 20260506 22:14:17.443 | INFO | Opening tmp RTMetrics json file | 20260506 22:14:17.445 | INFO | Serializing RTMetrics to json | 20260506 22:14:17.463 | INFO | Serialization done, swapping tmp file now... | 20260506 22:14:17.464 | INFO | RTMetrics done | 20260506 22:14:17.538 | INFO | Finished a round of Baz IO: Wrote 1.921 GiB in 3.234 seconds | 20260506 22:14:17.538 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.73% / 0.00% / 66.27% | 20260506 22:14:17.538 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.521974% / 0% / 3.97364e-05% | 20260506 22:14:17.538 | INFO | Baz IO overall: | 20260506 22:14:17.538 | INFO | BytesWritten : 2553867988664 | 20260506 22:14:17.538 | INFO | NumEvents : 1232406715996 | 20260506 22:14:19.348 | INFO | Analyzing chunk frames = [4857856,4858368) | 20260506 22:14:19.348 | INFO | MemInfo: { | 20260506 22:14:19.348 | INFO | "gpu" : " 17 GB", | 20260506 22:14:19.348 | INFO | "huge" : "", | 20260506 22:14:19.348 | INFO | "huge_shared" : " 27 GB", | 20260506 22:14:19.348 | INFO | "malloc" : " 14 GB", | 20260506 22:14:19.348 | INFO | "pinned" : " 29 GB" | 20260506 22:14:19.348 | INFO | } | 20260506 22:14:19.348 | INFO | Perf report for the last 5.14223 seconds: | 20260506 22:14:19.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:14:19.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:19.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:19.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:19.348 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260506 22:14:19.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:19.348 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 22:14:19.348 | INFO | PRE_HQ : 59.8%, 1.8, 57.3ms, 0 | 20260506 22:14:19.348 | INFO | BAZWRITER : 62.4%, 3.5, 111.8ms, 0 | 20260506 22:14:19.348 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 22:14:19.348 | INFO | Uploaded 36GiB to GPU in 2.03913s (17.6546GiB/s) | 20260506 22:14:19.348 | INFO | Downloaded 38GiB to GPU in 2.18011s (17.4303GiB/s) | 20260506 22:14:24.491 | INFO | Analyzing chunk frames = [4858368,4858880) | 20260506 22:14:24.491 | INFO | MemInfo: { | 20260506 22:14:24.491 | INFO | "gpu" : " 17 GB", | 20260506 22:14:24.491 | INFO | "huge" : "", | 20260506 22:14:24.491 | INFO | "huge_shared" : " 27 GB", | 20260506 22:14:24.491 | INFO | "malloc" : " 14 GB", | 20260506 22:14:24.491 | INFO | "pinned" : " 29 GB" | 20260506 22:14:24.491 | INFO | } | 20260506 22:14:24.491 | INFO | Perf report for the last 5.14313 seconds: | 20260506 22:14:24.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:14:24.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:24.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:24.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:24.491 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 22:14:24.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:24.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:24.491 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:14:24.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:24.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:24.491 | INFO | Uploaded 36GiB to GPU in 2.00478s (17.9571GiB/s) | 20260506 22:14:24.491 | INFO | Downloaded 34GiB to GPU in 1.92669s (17.6468GiB/s) | 20260506 22:14:29.622 | INFO | Analyzing chunk frames = [4858880,4859392) | 20260506 22:14:29.622 | INFO | MemInfo: { | 20260506 22:14:29.622 | INFO | "gpu" : " 17 GB", | 20260506 22:14:29.622 | INFO | "huge" : "", | 20260506 22:14:29.622 | INFO | "huge_shared" : " 27 GB", | 20260506 22:14:29.622 | INFO | "malloc" : " 14 GB", | 20260506 22:14:29.622 | INFO | "pinned" : " 29 GB" | 20260506 22:14:29.622 | INFO | } | 20260506 22:14:29.623 | INFO | Perf report for the last 5.13144 seconds: | 20260506 22:14:29.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:14:29.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:29.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:29.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:29.623 | INFO | ANALYSIS : 60.7%, 3.0, 93.8ms, 0 | 20260506 22:14:29.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:29.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:29.623 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:14:29.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:29.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:29.623 | INFO | Uploaded 36GiB to GPU in 1.99622s (18.0341GiB/s) | 20260506 22:14:29.623 | INFO | Downloaded 34GiB to GPU in 1.92943s (17.6218GiB/s) | 20260506 22:14:34.759 | INFO | Analyzing chunk frames = [4859392,4859904) | 20260506 22:14:34.759 | INFO | MemInfo: { | 20260506 22:14:34.759 | INFO | "gpu" : " 17 GB", | 20260506 22:14:34.759 | INFO | "huge" : "", | 20260506 22:14:34.759 | INFO | "huge_shared" : " 27 GB", | 20260506 22:14:34.759 | INFO | "malloc" : " 14 GB", | 20260506 22:14:34.759 | INFO | "pinned" : " 29 GB" | 20260506 22:14:34.759 | INFO | } | 20260506 22:14:34.759 | INFO | Perf report for the last 5.1366 seconds: | 20260506 22:14:34.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:14:34.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:34.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:34.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:34.759 | INFO | ANALYSIS : 59.7%, 3.0, 92.4ms, 0 | 20260506 22:14:34.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:34.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:34.759 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:14:34.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:34.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:34.759 | INFO | Uploaded 36GiB to GPU in 1.93116s (18.6416GiB/s) | 20260506 22:14:34.759 | INFO | Downloaded 34GiB to GPU in 1.87403s (18.1427GiB/s) | 20260506 22:14:39.898 | INFO | Analyzing chunk frames = [4859904,4860416) | 20260506 22:14:39.898 | INFO | MemInfo: { | 20260506 22:14:39.898 | INFO | "gpu" : " 17 GB", | 20260506 22:14:39.898 | INFO | "huge" : "", | 20260506 22:14:39.898 | INFO | "huge_shared" : " 27 GB", | 20260506 22:14:39.898 | INFO | "malloc" : " 14 GB", | 20260506 22:14:39.899 | INFO | "pinned" : " 29 GB" | 20260506 22:14:39.899 | INFO | } | 20260506 22:14:39.899 | INFO | Perf report for the last 5.13947 seconds: | 20260506 22:14:39.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:14:39.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:39.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:39.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:39.899 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260506 22:14:39.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:39.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:39.899 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:14:39.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:39.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:39.899 | INFO | Uploaded 36GiB to GPU in 2.02173s (17.8065GiB/s) | 20260506 22:14:39.899 | INFO | Downloaded 34GiB to GPU in 1.91487s (17.7558GiB/s) | 20260506 22:14:45.043 | INFO | Analyzing chunk frames = [4860416,4860928) | 20260506 22:14:45.043 | INFO | MemInfo: { | 20260506 22:14:45.043 | INFO | "gpu" : " 17 GB", | 20260506 22:14:45.043 | INFO | "huge" : "", | 20260506 22:14:45.043 | INFO | "huge_shared" : " 27 GB", | 20260506 22:14:45.043 | INFO | "malloc" : " 14 GB", | 20260506 22:14:45.043 | INFO | "pinned" : " 29 GB" | 20260506 22:14:45.043 | INFO | } | 20260506 22:14:45.044 | INFO | Perf report for the last 5.14492 seconds: | 20260506 22:14:45.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:14:45.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:45.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:45.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:45.044 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 22:14:45.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:45.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:45.044 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260506 22:14:45.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:45.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:45.044 | INFO | Uploaded 36GiB to GPU in 1.99478s (18.0471GiB/s) | 20260506 22:14:45.044 | INFO | Downloaded 34GiB to GPU in 1.94148s (17.5124GiB/s) | 20260506 22:14:50.183 | INFO | Analyzing chunk frames = [4860928,4861440) | 20260506 22:14:50.183 | INFO | MemInfo: { | 20260506 22:14:50.183 | INFO | "gpu" : " 17 GB", | 20260506 22:14:50.183 | INFO | "huge" : "", | 20260506 22:14:50.183 | INFO | "huge_shared" : " 27 GB", | 20260506 22:14:50.183 | INFO | "malloc" : " 14 GB", | 20260506 22:14:50.183 | INFO | "pinned" : " 29 GB" | 20260506 22:14:50.183 | INFO | } | 20260506 22:14:50.183 | INFO | Perf report for the last 5.13915 seconds: | 20260506 22:14:50.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:14:50.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:50.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:50.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:50.183 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 22:14:50.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:50.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:50.183 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 22:14:50.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:50.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:50.183 | INFO | Uploaded 36GiB to GPU in 2.01288s (17.8848GiB/s) | 20260506 22:14:50.183 | INFO | Downloaded 34GiB to GPU in 1.99323s (17.0577GiB/s) | 20260506 22:14:55.325 | INFO | Analyzing chunk frames = [4861440,4861952) | 20260506 22:14:55.325 | INFO | MemInfo: { | 20260506 22:14:55.325 | INFO | "gpu" : " 17 GB", | 20260506 22:14:55.325 | INFO | "huge" : "", | 20260506 22:14:55.325 | INFO | "huge_shared" : " 27 GB", | 20260506 22:14:55.325 | INFO | "malloc" : " 14 GB", | 20260506 22:14:55.325 | INFO | "pinned" : " 29 GB" | 20260506 22:14:55.325 | INFO | } | 20260506 22:14:55.325 | INFO | Perf report for the last 5.14249 seconds: | 20260506 22:14:55.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:14:55.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:55.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:55.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:55.325 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260506 22:14:55.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:55.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:55.325 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 22:14:55.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:55.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:14:55.325 | INFO | Uploaded 36GiB to GPU in 2.05153s (17.5479GiB/s) | 20260506 22:14:55.326 | INFO | Downloaded 34GiB to GPU in 2.00867s (16.9266GiB/s) | 20260506 22:14:55.440 | INFO | Starting a round of Baz IO | 20260506 22:14:58.562 | INFO | Populating RTMetrics report | 20260506 22:14:58.562 | INFO | RTMetrics report populated | 20260506 22:14:58.562 | INFO | Opening tmp RTMetrics json file | 20260506 22:14:58.562 | INFO | Serializing RTMetrics to json | 20260506 22:14:58.563 | INFO | Serialization done, swapping tmp file now... | 20260506 22:14:58.564 | INFO | RTMetrics done | 20260506 22:14:58.655 | INFO | Finished a round of Baz IO: Wrote 1.919 GiB in 3.214 seconds | 20260506 22:14:58.655 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.73% / 0.00% / 66.27% | 20260506 22:14:58.655 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.522848% / 7.94729e-06% / 3.97364e-05% | 20260506 22:14:58.655 | INFO | Baz IO overall: | 20260506 22:14:58.655 | INFO | BytesWritten : 2555928631346 | 20260506 22:14:58.655 | INFO | NumEvents : 1233389975747 | 20260506 22:15:00.463 | INFO | Analyzing chunk frames = [4861952,4862464) | 20260506 22:15:00.463 | INFO | MemInfo: { | 20260506 22:15:00.463 | INFO | "gpu" : " 17 GB", | 20260506 22:15:00.463 | INFO | "huge" : "", | 20260506 22:15:00.463 | INFO | "huge_shared" : " 27 GB", | 20260506 22:15:00.463 | INFO | "malloc" : " 14 GB", | 20260506 22:15:00.463 | INFO | "pinned" : " 29 GB" | 20260506 22:15:00.463 | INFO | } | 20260506 22:15:00.463 | INFO | Perf report for the last 5.13761 seconds: | 20260506 22:15:00.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:15:00.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:00.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:00.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:00.463 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260506 22:15:00.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:00.463 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 22:15:00.463 | INFO | PRE_HQ : 59.8%, 1.8, 57.2ms, 0 | 20260506 22:15:00.463 | INFO | BAZWRITER : 62.2%, 3.4, 110.0ms, 0 | 20260506 22:15:00.463 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:15:00.463 | INFO | Uploaded 36GiB to GPU in 2.04247s (17.6257GiB/s) | 20260506 22:15:00.463 | INFO | Downloaded 38GiB to GPU in 2.18377s (17.4011GiB/s) | 20260506 22:15:05.598 | INFO | Analyzing chunk frames = [4862464,4862976) | 20260506 22:15:05.598 | INFO | MemInfo: { | 20260506 22:15:05.598 | INFO | "gpu" : " 17 GB", | 20260506 22:15:05.598 | INFO | "huge" : "", | 20260506 22:15:05.598 | INFO | "huge_shared" : " 27 GB", | 20260506 22:15:05.598 | INFO | "malloc" : " 14 GB", | 20260506 22:15:05.598 | INFO | "pinned" : " 29 GB" | 20260506 22:15:05.598 | INFO | } | 20260506 22:15:05.598 | INFO | Perf report for the last 5.13559 seconds: | 20260506 22:15:05.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:15:05.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:05.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:05.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:05.598 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:15:05.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:05.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:05.598 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:15:05.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:05.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:05.599 | INFO | Uploaded 36GiB to GPU in 2.00231s (17.9792GiB/s) | 20260506 22:15:05.599 | INFO | Downloaded 34GiB to GPU in 1.92401s (17.6714GiB/s) | 20260506 22:15:10.739 | INFO | Analyzing chunk frames = [4862976,4863488) | 20260506 22:15:10.739 | INFO | MemInfo: { | 20260506 22:15:10.739 | INFO | "gpu" : " 17 GB", | 20260506 22:15:10.739 | INFO | "huge" : "", | 20260506 22:15:10.739 | INFO | "huge_shared" : " 27 GB", | 20260506 22:15:10.739 | INFO | "malloc" : " 14 GB", | 20260506 22:15:10.739 | INFO | "pinned" : " 29 GB" | 20260506 22:15:10.739 | INFO | } | 20260506 22:15:10.739 | INFO | Perf report for the last 5.14052 seconds: | 20260506 22:15:10.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:15:10.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:10.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:10.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:10.739 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 22:15:10.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:10.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:10.739 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:15:10.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:10.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:10.739 | INFO | Uploaded 36GiB to GPU in 1.99077s (18.0835GiB/s) | 20260506 22:15:10.739 | INFO | Downloaded 34GiB to GPU in 1.91688s (17.7371GiB/s) | 20260506 22:15:15.874 | INFO | Analyzing chunk frames = [4863488,4864000) | 20260506 22:15:15.874 | INFO | MemInfo: { | 20260506 22:15:15.874 | INFO | "gpu" : " 17 GB", | 20260506 22:15:15.874 | INFO | "huge" : "", | 20260506 22:15:15.874 | INFO | "huge_shared" : " 27 GB", | 20260506 22:15:15.874 | INFO | "malloc" : " 14 GB", | 20260506 22:15:15.874 | INFO | "pinned" : " 29 GB" | 20260506 22:15:15.874 | INFO | } | 20260506 22:15:15.874 | INFO | Perf report for the last 5.13545 seconds: | 20260506 22:15:15.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:15:15.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:15.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:15.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:15.875 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 22:15:15.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:15.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:15.875 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:15:15.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:15.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:15.875 | INFO | Uploaded 36GiB to GPU in 1.94481s (18.5108GiB/s) | 20260506 22:15:15.875 | INFO | Downloaded 34GiB to GPU in 1.87312s (18.1515GiB/s) | 20260506 22:15:21.015 | INFO | Analyzing chunk frames = [4864000,4864512) | 20260506 22:15:21.015 | INFO | MemInfo: { | 20260506 22:15:21.015 | INFO | "gpu" : " 17 GB", | 20260506 22:15:21.022 | INFO | "huge" : "", | 20260506 22:15:21.022 | INFO | "huge_shared" : " 27 GB", | 20260506 22:15:21.022 | INFO | "malloc" : " 14 GB", | 20260506 22:15:21.022 | INFO | "pinned" : " 29 GB" | 20260506 22:15:21.022 | INFO | } | 20260506 22:15:21.022 | INFO | Perf report for the last 5.14805 seconds: | 20260506 22:15:21.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:15:21.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:21.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:21.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:21.023 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 22:15:21.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:21.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:21.023 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 22:15:21.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:21.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:21.023 | INFO | Uploaded 36GiB to GPU in 1.99993s (18.0006GiB/s) | 20260506 22:15:21.023 | INFO | Downloaded 34GiB to GPU in 1.91572s (17.7479GiB/s) | 20260506 22:15:26.163 | INFO | Analyzing chunk frames = [4864512,4865024) | 20260506 22:15:26.163 | INFO | MemInfo: { | 20260506 22:15:26.163 | INFO | "gpu" : " 17 GB", | 20260506 22:15:26.163 | INFO | "huge" : "", | 20260506 22:15:26.163 | INFO | "huge_shared" : " 27 GB", | 20260506 22:15:26.163 | INFO | "malloc" : " 14 GB", | 20260506 22:15:26.163 | INFO | "pinned" : " 29 GB" | 20260506 22:15:26.163 | INFO | } | 20260506 22:15:26.164 | INFO | Perf report for the last 5.14104 seconds: | 20260506 22:15:26.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:15:26.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:26.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:26.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:26.164 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:15:26.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:26.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:26.164 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:15:26.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:26.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:26.164 | INFO | Uploaded 36GiB to GPU in 2.01752s (17.8437GiB/s) | 20260506 22:15:26.164 | INFO | Downloaded 34GiB to GPU in 1.94038s (17.5223GiB/s) | 20260506 22:15:31.298 | INFO | Analyzing chunk frames = [4865024,4865536) | 20260506 22:15:31.298 | INFO | MemInfo: { | 20260506 22:15:31.298 | INFO | "gpu" : " 17 GB", | 20260506 22:15:31.298 | INFO | "huge" : "", | 20260506 22:15:31.299 | INFO | "huge_shared" : " 27 GB", | 20260506 22:15:31.299 | INFO | "malloc" : " 14 GB", | 20260506 22:15:31.299 | INFO | "pinned" : " 29 GB" | 20260506 22:15:31.299 | INFO | } | 20260506 22:15:31.299 | INFO | Perf report for the last 5.13508 seconds: | 20260506 22:15:31.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:15:31.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:31.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:31.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:31.299 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:15:31.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:31.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:31.299 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 22:15:31.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:31.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:31.299 | INFO | Uploaded 36GiB to GPU in 1.98099s (18.1727GiB/s) | 20260506 22:15:31.299 | INFO | Downloaded 34GiB to GPU in 1.99534s (17.0397GiB/s) | 20260506 22:15:36.438 | INFO | Analyzing chunk frames = [4865536,4866048) | 20260506 22:15:36.438 | INFO | MemInfo: { | 20260506 22:15:36.438 | INFO | "gpu" : " 17 GB", | 20260506 22:15:36.438 | INFO | "huge" : "", | 20260506 22:15:36.438 | INFO | "huge_shared" : " 27 GB", | 20260506 22:15:36.438 | INFO | "malloc" : " 14 GB", | 20260506 22:15:36.438 | INFO | "pinned" : " 29 GB" | 20260506 22:15:36.438 | INFO | } | 20260506 22:15:36.438 | INFO | Perf report for the last 5.13936 seconds: | 20260506 22:15:36.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:15:36.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:36.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:36.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:36.438 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:15:36.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:36.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:36.438 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:15:36.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:36.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:36.438 | INFO | Uploaded 36GiB to GPU in 1.98979s (18.0924GiB/s) | 20260506 22:15:36.438 | INFO | Downloaded 34GiB to GPU in 1.97932s (17.1776GiB/s) | 20260506 22:15:36.438 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:15:36.438 | INFO | Total mem usage: 18 GiB | 20260506 22:15:36.438 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:15:36.438 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 22:15:36.438 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:15:36.438 | INFO | High Water Breakdown: | 20260506 22:15:36.438 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:15:36.438 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:15:36.438 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:15:36.438 | INFO | ------------------------------------------ | 20260506 22:15:36.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:15:36.438 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:15:36.438 | INFO | Total mem usage: 26.5 GiB | 20260506 22:15:36.438 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:15:36.438 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:15:36.438 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:15:36.438 | INFO | High Water Breakdown: | 20260506 22:15:36.439 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:15:36.439 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:15:36.439 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:15:36.439 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:15:36.439 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:15:36.439 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:15:36.439 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:15:36.439 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:15:36.439 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:15:36.439 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:15:36.439 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:15:36.439 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:15:36.439 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:15:36.439 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:15:36.439 | INFO | ------------------------------------------ | 20260506 22:15:36.439 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:15:36.439 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:15:36.439 | INFO | Total mem usage: 25.6 GiB | 20260506 22:15:36.439 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:15:36.439 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:15:36.439 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:15:36.439 | INFO | High Water Breakdown: | 20260506 22:15:36.439 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:15:36.439 | INFO | ------------------------------------------ | 20260506 22:15:36.439 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:15:36.439 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:15:36.439 | INFO | Total mem usage: 15.7 GiB | 20260506 22:15:36.439 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:15:36.439 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:15:36.439 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:15:36.439 | INFO | High Water Breakdown: | 20260506 22:15:36.439 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:15:36.439 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:15:36.439 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:15:36.439 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:15:36.439 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:15:36.439 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:15:36.439 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:15:36.439 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:15:36.439 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:15:36.439 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:15:36.439 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:15:36.439 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:15:36.439 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:15:36.439 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:15:36.440 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:15:36.440 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:15:36.440 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:15:36.440 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:15:36.440 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:15:36.440 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:15:36.440 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:15:36.440 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:15:36.440 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:15:36.440 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:15:36.440 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:15:36.440 | INFO | ------------------------------------------ | 20260506 22:15:36.440 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:15:36.440 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:15:36.440 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:15:36.440 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:15:36.440 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.56e+01, 1.35e+01 / 2.53e+00 | 20260506 22:15:36.440 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.02e-01, 9.59e-03 / 5.14e-03 | 20260506 22:15:36.440 | INFO | PerfReport: 11.65%, Baseline -- 3.62e+00 / 4.44e+00, 5.50e-01 / 3.31e-01 | 20260506 22:15:36.440 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.47e+00, 1.80e-01 / 1.22e-01 | 20260506 22:15:36.440 | INFO | PerfReport: 16.03%, DME -- 3.99e+01 / 4.42e+01, 4.11e+00 / 2.56e+00 | 20260506 22:15:36.440 | INFO | PerfReport: 57.40%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.44e+00 / 1.59e+00 | 20260506 22:15:36.440 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.19e+00 / 2.72e+00, 3.68e-01 / 2.06e-01 | 20260506 22:15:36.440 | INFO | PerfReport: 3.22%, Metrics -- 1.00e+00 / 1.33e+00, 2.65e-01 / 1.80e-01 | 20260506 22:15:36.440 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.43e-01 / 3.88e-01, 5.46e-02 / 2.28e-02 | 20260506 22:15:36.440 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:15:36.440 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:15:36.440 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:15:36.440 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 22:15:36.440 | INFO | PerfReport: 21.53%, Upload -- 2.08e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260506 22:15:36.440 | INFO | PerfReport: 21.18%, Download -- 2.05e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 22:15:36.440 | INFO | PerfReport: 32.11%, Compute -- 3.11e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 22:15:36.440 | INFO | PerfReport: 1.34%, LockUpload -- 1.30e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260506 22:15:36.440 | INFO | PerfReport: 1.11%, LockDownload -- 1.08e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260506 22:15:36.440 | INFO | PerfReport: 22.72%, LockCompute -- 2.20e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 22:15:36.440 | INFO | PerfReport: 0.02%, Misc -- 1.45e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260506 22:15:36.538 | INFO | Starting a round of Baz IO | 20260506 22:15:39.671 | INFO | Populating RTMetrics report | 20260506 22:15:39.672 | INFO | RTMetrics report populated | 20260506 22:15:39.672 | INFO | Opening tmp RTMetrics json file | 20260506 22:15:39.694 | INFO | Serializing RTMetrics to json | 20260506 22:15:39.695 | INFO | Serialization done, swapping tmp file now... | 20260506 22:15:39.696 | INFO | RTMetrics done | 20260506 22:15:39.765 | INFO | Finished a round of Baz IO: Wrote 1.919 GiB in 3.226 seconds | 20260506 22:15:39.765 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.72% / 0.00% / 66.27% | 20260506 22:15:39.765 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.523047% / 3.97364e-06% / 5.16574e-05% | 20260506 22:15:39.765 | INFO | Baz IO overall: | 20260506 22:15:39.765 | INFO | BytesWritten : 2557988879419 | 20260506 22:15:39.765 | INFO | NumEvents : 1234373723249 | 20260506 22:15:41.575 | INFO | Analyzing chunk frames = [4866048,4866560) | 20260506 22:15:41.575 | INFO | MemInfo: { | 20260506 22:15:41.575 | INFO | "gpu" : " 17 GB", | 20260506 22:15:41.575 | INFO | "huge" : "", | 20260506 22:15:41.575 | INFO | "huge_shared" : " 27 GB", | 20260506 22:15:41.575 | INFO | "malloc" : " 14 GB", | 20260506 22:15:41.575 | INFO | "pinned" : " 29 GB" | 20260506 22:15:41.576 | INFO | } | 20260506 22:15:41.576 | INFO | Perf report for the last 5.13752 seconds: | 20260506 22:15:41.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:15:41.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:41.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:41.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:41.576 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260506 22:15:41.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:41.576 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 22:15:41.576 | INFO | PRE_HQ : 59.7%, 1.8, 57.1ms, 0 | 20260506 22:15:41.576 | INFO | BAZWRITER : 62.4%, 3.4, 108.2ms, 0 | 20260506 22:15:41.576 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:15:41.576 | INFO | Uploaded 36GiB to GPU in 2.0344s (17.6956GiB/s) | 20260506 22:15:41.576 | INFO | Downloaded 38GiB to GPU in 2.17781s (17.4487GiB/s) | 20260506 22:15:46.714 | INFO | Analyzing chunk frames = [4866560,4867072) | 20260506 22:15:46.714 | INFO | MemInfo: { | 20260506 22:15:46.714 | INFO | "gpu" : " 17 GB", | 20260506 22:15:46.714 | INFO | "huge" : "", | 20260506 22:15:46.714 | INFO | "huge_shared" : " 27 GB", | 20260506 22:15:46.714 | INFO | "malloc" : " 14 GB", | 20260506 22:15:46.714 | INFO | "pinned" : " 29 GB" | 20260506 22:15:46.714 | INFO | } | 20260506 22:15:46.714 | INFO | Perf report for the last 5.13837 seconds: | 20260506 22:15:46.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:15:46.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:46.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:46.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:46.714 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 22:15:46.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:46.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:46.714 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:15:46.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:46.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:46.714 | INFO | Uploaded 36GiB to GPU in 2.01867s (17.8335GiB/s) | 20260506 22:15:46.714 | INFO | Downloaded 34GiB to GPU in 1.93064s (17.6108GiB/s) | 20260506 22:15:51.849 | INFO | Analyzing chunk frames = [4867072,4867584) | 20260506 22:15:51.849 | INFO | MemInfo: { | 20260506 22:15:51.849 | INFO | "gpu" : " 17 GB", | 20260506 22:15:51.849 | INFO | "huge" : "", | 20260506 22:15:51.849 | INFO | "huge_shared" : " 27 GB", | 20260506 22:15:51.849 | INFO | "malloc" : " 14 GB", | 20260506 22:15:51.849 | INFO | "pinned" : " 29 GB" | 20260506 22:15:51.849 | INFO | } | 20260506 22:15:51.849 | INFO | Perf report for the last 5.13516 seconds: | 20260506 22:15:51.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:15:51.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:51.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:51.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:51.849 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 22:15:51.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:51.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:51.849 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:15:51.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:51.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:51.849 | INFO | Uploaded 36GiB to GPU in 1.9758s (18.2205GiB/s) | 20260506 22:15:51.849 | INFO | Downloaded 34GiB to GPU in 1.91928s (17.7149GiB/s) | 20260506 22:15:56.995 | INFO | Analyzing chunk frames = [4867584,4868096) | 20260506 22:15:56.995 | INFO | MemInfo: { | 20260506 22:15:56.995 | INFO | "gpu" : " 17 GB", | 20260506 22:15:56.995 | INFO | "huge" : "", | 20260506 22:15:56.995 | INFO | "huge_shared" : " 27 GB", | 20260506 22:15:56.995 | INFO | "malloc" : " 14 GB", | 20260506 22:15:56.995 | INFO | "pinned" : " 29 GB" | 20260506 22:15:56.995 | INFO | } | 20260506 22:15:56.995 | INFO | Perf report for the last 5.14576 seconds: | 20260506 22:15:56.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:15:56.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:56.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:56.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:56.995 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 22:15:56.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:56.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:56.995 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:15:56.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:56.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:15:56.995 | INFO | Uploaded 36GiB to GPU in 1.95048s (18.457GiB/s) | 20260506 22:15:56.995 | INFO | Downloaded 34GiB to GPU in 1.86858s (18.1957GiB/s) | 20260506 22:16:02.134 | INFO | Analyzing chunk frames = [4868096,4868608) | 20260506 22:16:02.134 | INFO | MemInfo: { | 20260506 22:16:02.134 | INFO | "gpu" : " 17 GB", | 20260506 22:16:02.134 | INFO | "huge" : "", | 20260506 22:16:02.134 | INFO | "huge_shared" : " 27 GB", | 20260506 22:16:02.134 | INFO | "malloc" : " 14 GB", | 20260506 22:16:02.134 | INFO | "pinned" : " 29 GB" | 20260506 22:16:02.134 | INFO | } | 20260506 22:16:02.134 | INFO | Perf report for the last 5.13874 seconds: | 20260506 22:16:02.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:16:02.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:02.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:02.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:02.134 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 22:16:02.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:02.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:02.134 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:16:02.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:02.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:02.134 | INFO | Uploaded 36GiB to GPU in 2.00842s (17.9245GiB/s) | 20260506 22:16:02.134 | INFO | Downloaded 34GiB to GPU in 1.91792s (17.7276GiB/s) | 20260506 22:16:07.271 | INFO | Analyzing chunk frames = [4868608,4869120) | 20260506 22:16:07.271 | INFO | MemInfo: { | 20260506 22:16:07.271 | INFO | "gpu" : " 17 GB", | 20260506 22:16:07.271 | INFO | "huge" : "", | 20260506 22:16:07.271 | INFO | "huge_shared" : " 27 GB", | 20260506 22:16:07.271 | INFO | "malloc" : " 14 GB", | 20260506 22:16:07.271 | INFO | "pinned" : " 29 GB" | 20260506 22:16:07.271 | INFO | } | 20260506 22:16:07.271 | INFO | Perf report for the last 5.13738 seconds: | 20260506 22:16:07.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:16:07.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:07.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:07.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:07.271 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 22:16:07.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:07.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:07.271 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 22:16:07.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:07.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:07.271 | INFO | Uploaded 36GiB to GPU in 2.03158s (17.7202GiB/s) | 20260506 22:16:07.271 | INFO | Downloaded 34GiB to GPU in 1.9378s (17.5457GiB/s) | 20260506 22:16:12.415 | INFO | Analyzing chunk frames = [4869120,4869632) | 20260506 22:16:12.415 | INFO | MemInfo: { | 20260506 22:16:12.415 | INFO | "gpu" : " 17 GB", | 20260506 22:16:12.415 | INFO | "huge" : "", | 20260506 22:16:12.415 | INFO | "huge_shared" : " 27 GB", | 20260506 22:16:12.415 | INFO | "malloc" : " 14 GB", | 20260506 22:16:12.415 | INFO | "pinned" : " 29 GB" | 20260506 22:16:12.415 | INFO | } | 20260506 22:16:12.415 | INFO | Perf report for the last 5.14408 seconds: | 20260506 22:16:12.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:16:12.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:12.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:12.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:12.415 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 22:16:12.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:12.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:12.415 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 22:16:12.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:12.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:12.415 | INFO | Uploaded 36GiB to GPU in 1.96548s (18.3161GiB/s) | 20260506 22:16:12.415 | INFO | Downloaded 34GiB to GPU in 1.98707s (17.1106GiB/s) | 20260506 22:16:17.552 | INFO | Analyzing chunk frames = [4869632,4870144) | 20260506 22:16:17.552 | INFO | MemInfo: { | 20260506 22:16:17.552 | INFO | "gpu" : " 17 GB", | 20260506 22:16:17.552 | INFO | "huge" : "", | 20260506 22:16:17.552 | INFO | "huge_shared" : " 27 GB", | 20260506 22:16:17.552 | INFO | "malloc" : " 14 GB", | 20260506 22:16:17.552 | INFO | "pinned" : " 29 GB" | 20260506 22:16:17.552 | INFO | } | 20260506 22:16:17.553 | INFO | Perf report for the last 5.13756 seconds: | 20260506 22:16:17.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:16:17.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:17.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:17.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:17.553 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 22:16:17.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:17.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:17.553 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 22:16:17.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:17.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:17.553 | INFO | Uploaded 36GiB to GPU in 2.02675s (17.7624GiB/s) | 20260506 22:16:17.553 | INFO | Downloaded 34GiB to GPU in 1.98991s (17.0862GiB/s) | 20260506 22:16:17.659 | INFO | Starting a round of Baz IO | 20260506 22:16:20.790 | INFO | Populating RTMetrics report | 20260506 22:16:20.791 | INFO | RTMetrics report populated | 20260506 22:16:20.791 | INFO | Opening tmp RTMetrics json file | 20260506 22:16:20.799 | INFO | Serializing RTMetrics to json | 20260506 22:16:20.809 | INFO | Serialization done, swapping tmp file now... | 20260506 22:16:20.809 | INFO | RTMetrics done | 20260506 22:16:20.887 | INFO | Finished a round of Baz IO: Wrote 1.917 GiB in 3.228 seconds | 20260506 22:16:20.887 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.72% / 0.00% / 66.28% | 20260506 22:16:20.887 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.521851% / 7.94729e-06% / 3.57628e-05% | 20260506 22:16:20.887 | INFO | Baz IO overall: | 20260506 22:16:20.887 | INFO | BytesWritten : 2560047670674 | 20260506 22:16:20.887 | INFO | NumEvents : 1235357119961 | 20260506 22:16:22.688 | INFO | Analyzing chunk frames = [4870144,4870656) | 20260506 22:16:22.688 | INFO | MemInfo: { | 20260506 22:16:22.688 | INFO | "gpu" : " 17 GB", | 20260506 22:16:22.688 | INFO | "huge" : "", | 20260506 22:16:22.688 | INFO | "huge_shared" : " 27 GB", | 20260506 22:16:22.688 | INFO | "malloc" : " 14 GB", | 20260506 22:16:22.688 | INFO | "pinned" : " 29 GB" | 20260506 22:16:22.688 | INFO | } | 20260506 22:16:22.688 | INFO | Perf report for the last 5.13521 seconds: | 20260506 22:16:22.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:16:22.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:22.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:22.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:22.688 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260506 22:16:22.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:22.688 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 22:16:22.688 | INFO | PRE_HQ : 59.7%, 1.8, 56.8ms, 0 | 20260506 22:16:22.688 | INFO | BAZWRITER : 62.5%, 3.4, 110.3ms, 0 | 20260506 22:16:22.688 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 22:16:22.688 | INFO | Uploaded 36GiB to GPU in 2.0498s (17.5627GiB/s) | 20260506 22:16:22.688 | INFO | Downloaded 38GiB to GPU in 2.18677s (17.3772GiB/s) | 20260506 22:16:27.826 | INFO | Analyzing chunk frames = [4870656,4871168) | 20260506 22:16:27.826 | INFO | MemInfo: { | 20260506 22:16:27.826 | INFO | "gpu" : " 17 GB", | 20260506 22:16:27.826 | INFO | "huge" : "", | 20260506 22:16:27.826 | INFO | "huge_shared" : " 27 GB", | 20260506 22:16:27.826 | INFO | "malloc" : " 14 GB", | 20260506 22:16:27.826 | INFO | "pinned" : " 29 GB" | 20260506 22:16:27.826 | INFO | } | 20260506 22:16:27.826 | INFO | Perf report for the last 5.13804 seconds: | 20260506 22:16:27.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:16:27.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:27.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:27.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:27.826 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 22:16:27.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:27.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:27.826 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:16:27.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:27.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:27.826 | INFO | Uploaded 36GiB to GPU in 2.01759s (17.8431GiB/s) | 20260506 22:16:27.826 | INFO | Downloaded 34GiB to GPU in 1.92849s (17.6304GiB/s) | 20260506 22:16:32.969 | INFO | Analyzing chunk frames = [4871168,4871680) | 20260506 22:16:32.969 | INFO | MemInfo: { | 20260506 22:16:32.969 | INFO | "gpu" : " 17 GB", | 20260506 22:16:32.969 | INFO | "huge" : "", | 20260506 22:16:32.969 | INFO | "huge_shared" : " 27 GB", | 20260506 22:16:32.969 | INFO | "malloc" : " 14 GB", | 20260506 22:16:32.969 | INFO | "pinned" : " 29 GB" | 20260506 22:16:32.969 | INFO | } | 20260506 22:16:32.969 | INFO | Perf report for the last 5.14342 seconds: | 20260506 22:16:32.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:16:32.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:32.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:32.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:32.969 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 22:16:32.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:32.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:32.970 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 22:16:32.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:32.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:32.970 | INFO | Uploaded 36GiB to GPU in 1.98542s (18.1321GiB/s) | 20260506 22:16:32.970 | INFO | Downloaded 34GiB to GPU in 1.91849s (17.7223GiB/s) | 20260506 22:16:38.112 | INFO | Analyzing chunk frames = [4871680,4872192) | 20260506 22:16:38.112 | INFO | MemInfo: { | 20260506 22:16:38.112 | INFO | "gpu" : " 17 GB", | 20260506 22:16:38.112 | INFO | "huge" : "", | 20260506 22:16:38.112 | INFO | "huge_shared" : " 27 GB", | 20260506 22:16:38.112 | INFO | "malloc" : " 14 GB", | 20260506 22:16:38.112 | INFO | "pinned" : " 29 GB" | 20260506 22:16:38.112 | INFO | } | 20260506 22:16:38.112 | INFO | Perf report for the last 5.14269 seconds: | 20260506 22:16:38.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:16:38.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:38.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:38.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:38.112 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 22:16:38.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:38.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:38.112 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 22:16:38.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:38.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:38.112 | INFO | Uploaded 36GiB to GPU in 1.97863s (18.1945GiB/s) | 20260506 22:16:38.112 | INFO | Downloaded 34GiB to GPU in 1.87812s (18.1032GiB/s) | 20260506 22:16:43.252 | INFO | Analyzing chunk frames = [4872192,4872704) | 20260506 22:16:43.252 | INFO | MemInfo: { | 20260506 22:16:43.252 | INFO | "gpu" : " 17 GB", | 20260506 22:16:43.252 | INFO | "huge" : "", | 20260506 22:16:43.252 | INFO | "huge_shared" : " 27 GB", | 20260506 22:16:43.252 | INFO | "malloc" : " 14 GB", | 20260506 22:16:43.252 | INFO | "pinned" : " 29 GB" | 20260506 22:16:43.252 | INFO | } | 20260506 22:16:43.252 | INFO | Perf report for the last 5.1399 seconds: | 20260506 22:16:43.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:16:43.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:43.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:43.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:43.252 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 22:16:43.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:43.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:43.252 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:16:43.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:43.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:43.252 | INFO | Uploaded 36GiB to GPU in 2.00645s (17.9422GiB/s) | 20260506 22:16:43.252 | INFO | Downloaded 34GiB to GPU in 1.92584s (17.6546GiB/s) | 20260506 22:16:48.389 | INFO | Analyzing chunk frames = [4872704,4873216) | 20260506 22:16:48.389 | INFO | MemInfo: { | 20260506 22:16:48.389 | INFO | "gpu" : " 17 GB", | 20260506 22:16:48.389 | INFO | "huge" : "", | 20260506 22:16:48.389 | INFO | "huge_shared" : " 27 GB", | 20260506 22:16:48.389 | INFO | "malloc" : " 14 GB", | 20260506 22:16:48.389 | INFO | "pinned" : " 29 GB" | 20260506 22:16:48.389 | INFO | } | 20260506 22:16:48.389 | INFO | Perf report for the last 5.13689 seconds: | 20260506 22:16:48.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:16:48.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:48.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:48.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:48.389 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 22:16:48.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:48.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:48.389 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 22:16:48.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:48.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:48.389 | INFO | Uploaded 36GiB to GPU in 2.02145s (17.809GiB/s) | 20260506 22:16:48.389 | INFO | Downloaded 34GiB to GPU in 1.92973s (17.619GiB/s) | 20260506 22:16:53.524 | INFO | Analyzing chunk frames = [4873216,4873728) | 20260506 22:16:53.524 | INFO | MemInfo: { | 20260506 22:16:53.524 | INFO | "gpu" : " 17 GB", | 20260506 22:16:53.524 | INFO | "huge" : "", | 20260506 22:16:53.524 | INFO | "huge_shared" : " 27 GB", | 20260506 22:16:53.524 | INFO | "malloc" : " 14 GB", | 20260506 22:16:53.524 | INFO | "pinned" : " 29 GB" | 20260506 22:16:53.524 | INFO | } | 20260506 22:16:53.524 | INFO | Perf report for the last 5.13512 seconds: | 20260506 22:16:53.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:16:53.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:53.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:53.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:53.524 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 22:16:53.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:53.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:53.524 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:16:53.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:53.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:53.524 | INFO | Uploaded 36GiB to GPU in 1.97135s (18.2616GiB/s) | 20260506 22:16:53.524 | INFO | Downloaded 34GiB to GPU in 1.99328s (17.0573GiB/s) | 20260506 22:16:58.664 | INFO | Analyzing chunk frames = [4873728,4874240) | 20260506 22:16:58.664 | INFO | MemInfo: { | 20260506 22:16:58.664 | INFO | "gpu" : " 17 GB", | 20260506 22:16:58.664 | INFO | "huge" : "", | 20260506 22:16:58.664 | INFO | "huge_shared" : " 27 GB", | 20260506 22:16:58.664 | INFO | "malloc" : " 14 GB", | 20260506 22:16:58.664 | INFO | "pinned" : " 29 GB" | 20260506 22:16:58.664 | INFO | } | 20260506 22:16:58.664 | INFO | Perf report for the last 5.14027 seconds: | 20260506 22:16:58.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:16:58.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:58.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:58.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:58.664 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 22:16:58.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:58.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:58.664 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 22:16:58.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:58.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:16:58.665 | INFO | Uploaded 36GiB to GPU in 2.04923s (17.5676GiB/s) | 20260506 22:16:58.665 | INFO | Downloaded 34GiB to GPU in 1.99904s (17.0082GiB/s) | 20260506 22:16:58.665 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:16:58.665 | INFO | Total mem usage: 18 GiB | 20260506 22:16:58.665 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:16:58.665 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:16:58.665 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:16:58.665 | INFO | High Water Breakdown: | 20260506 22:16:58.665 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:16:58.665 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:16:58.665 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:16:58.665 | INFO | ------------------------------------------ | 20260506 22:16:58.665 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:16:58.665 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:16:58.665 | INFO | Total mem usage: 26.5 GiB | 20260506 22:16:58.665 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:16:58.665 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:16:58.665 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:16:58.665 | INFO | High Water Breakdown: | 20260506 22:16:58.665 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:16:58.665 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:16:58.665 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:16:58.665 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:16:58.665 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:16:58.665 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:16:58.665 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:16:58.665 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:16:58.665 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:16:58.665 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:16:58.665 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:16:58.665 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:16:58.665 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:16:58.665 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:16:58.665 | INFO | ------------------------------------------ | 20260506 22:16:58.665 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:16:58.665 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:16:58.665 | INFO | Total mem usage: 25.6 GiB | 20260506 22:16:58.665 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:16:58.665 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:16:58.665 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:16:58.666 | INFO | High Water Breakdown: | 20260506 22:16:58.666 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:16:58.666 | INFO | ------------------------------------------ | 20260506 22:16:58.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:16:58.666 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:16:58.666 | INFO | Total mem usage: 15.7 GiB | 20260506 22:16:58.666 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:16:58.666 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:16:58.666 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:16:58.666 | INFO | High Water Breakdown: | 20260506 22:16:58.666 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:16:58.666 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:16:58.666 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:16:58.666 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:16:58.666 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:16:58.666 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:16:58.666 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:16:58.666 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:16:58.666 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:16:58.666 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:16:58.666 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:16:58.666 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:16:58.666 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:16:58.666 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:16:58.666 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:16:58.666 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:16:58.666 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:16:58.666 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:16:58.666 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:16:58.666 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:16:58.666 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:16:58.666 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:16:58.666 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:16:58.666 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:16:58.666 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:16:58.666 | INFO | ------------------------------------------ | 20260506 22:16:58.666 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:16:58.666 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:16:58.666 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:16:58.666 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:16:58.666 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.53e+01, 1.35e+01 / 2.11e+00 | 20260506 22:16:58.666 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.02e-01, 9.74e-03 / 8.73e-03 | 20260506 22:16:58.666 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.38e+00, 5.47e-01 / 3.57e-01 | 20260506 22:16:58.666 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.81e-01 / 1.27e-01 | 20260506 22:16:58.666 | INFO | PerfReport: 16.03%, DME -- 3.98e+01 / 4.43e+01, 3.97e+00 / 2.07e+00 | 20260506 22:16:58.666 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.36e+00 / 1.53e+00 | 20260506 22:16:58.666 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.21e+00 / 2.71e+00, 3.81e-01 / 2.18e-01 | 20260506 22:16:58.666 | INFO | PerfReport: 3.24%, Metrics -- 1.01e+00 / 1.33e+00, 2.69e-01 / 1.76e-01 | 20260506 22:16:58.666 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.44e-01 / 3.94e-01, 5.46e-02 / 1.94e-02 | 20260506 22:16:58.666 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:16:58.666 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:16:58.666 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:16:58.666 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 22:16:58.666 | INFO | PerfReport: 21.58%, Upload -- 2.09e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260506 22:16:58.666 | INFO | PerfReport: 21.17%, Download -- 2.05e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 22:16:58.666 | INFO | PerfReport: 32.06%, Compute -- 3.10e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 22:16:58.666 | INFO | PerfReport: 1.39%, LockUpload -- 1.34e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260506 22:16:58.666 | INFO | PerfReport: 1.13%, LockDownload -- 1.10e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260506 22:16:58.666 | INFO | PerfReport: 22.66%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 22:16:58.666 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260506 22:16:58.779 | INFO | Starting a round of Baz IO | 20260506 22:17:01.903 | INFO | Populating RTMetrics report | 20260506 22:17:01.903 | INFO | RTMetrics report populated | 20260506 22:17:01.903 | INFO | Opening tmp RTMetrics json file | 20260506 22:17:01.904 | INFO | Serializing RTMetrics to json | 20260506 22:17:01.905 | INFO | Serialization done, swapping tmp file now... | 20260506 22:17:01.907 | INFO | RTMetrics done | 20260506 22:17:01.998 | INFO | Finished a round of Baz IO: Wrote 1.916 GiB in 3.219 seconds | 20260506 22:17:01.998 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.72% / 0.00% / 66.28% | 20260506 22:17:01.998 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.519367% / 7.94729e-06% / 6.35783e-05% | 20260506 22:17:01.998 | INFO | Baz IO overall: | 20260506 22:17:01.998 | INFO | BytesWritten : 2562104913510 | 20260506 22:17:01.998 | INFO | NumEvents : 1236339222816 | 20260506 22:17:03.799 | INFO | Analyzing chunk frames = [4874240,4874752) | 20260506 22:17:03.799 | INFO | MemInfo: { | 20260506 22:17:03.799 | INFO | "gpu" : " 17 GB", | 20260506 22:17:03.799 | INFO | "huge" : "", | 20260506 22:17:03.799 | INFO | "huge_shared" : " 27 GB", | 20260506 22:17:03.799 | INFO | "malloc" : " 14 GB", | 20260506 22:17:03.799 | INFO | "pinned" : " 29 GB" | 20260506 22:17:03.799 | INFO | } | 20260506 22:17:03.799 | INFO | Perf report for the last 5.13498 seconds: | 20260506 22:17:03.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:17:03.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:03.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:03.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:03.799 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260506 22:17:03.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:03.800 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 22:17:03.800 | INFO | PRE_HQ : 59.7%, 1.8, 57.3ms, 0 | 20260506 22:17:03.800 | INFO | BAZWRITER : 62.3%, 3.4, 109.1ms, 0 | 20260506 22:17:03.800 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:17:03.800 | INFO | Uploaded 36GiB to GPU in 2.02654s (17.7643GiB/s) | 20260506 22:17:03.800 | INFO | Downloaded 38GiB to GPU in 2.19432s (17.3174GiB/s) | 20260506 22:17:08.940 | INFO | Analyzing chunk frames = [4874752,4875264) | 20260506 22:17:08.940 | INFO | MemInfo: { | 20260506 22:17:08.940 | INFO | "gpu" : " 17 GB", | 20260506 22:17:08.940 | INFO | "huge" : "", | 20260506 22:17:08.940 | INFO | "huge_shared" : " 27 GB", | 20260506 22:17:08.940 | INFO | "malloc" : " 14 GB", | 20260506 22:17:08.940 | INFO | "pinned" : " 29 GB" | 20260506 22:17:08.940 | INFO | } | 20260506 22:17:08.940 | INFO | Perf report for the last 5.14097 seconds: | 20260506 22:17:08.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:17:08.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:08.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:08.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:08.940 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:17:08.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:08.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:08.940 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:17:08.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:08.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:08.941 | INFO | Uploaded 36GiB to GPU in 2.00025s (17.9978GiB/s) | 20260506 22:17:08.941 | INFO | Downloaded 34GiB to GPU in 1.92523s (17.6602GiB/s) | 20260506 22:17:14.085 | INFO | Analyzing chunk frames = [4875264,4875776) | 20260506 22:17:14.085 | INFO | MemInfo: { | 20260506 22:17:14.085 | INFO | "gpu" : " 17 GB", | 20260506 22:17:14.085 | INFO | "huge" : "", | 20260506 22:17:14.085 | INFO | "huge_shared" : " 27 GB", | 20260506 22:17:14.085 | INFO | "malloc" : " 14 GB", | 20260506 22:17:14.086 | INFO | "pinned" : " 29 GB" | 20260506 22:17:14.086 | INFO | } | 20260506 22:17:14.086 | INFO | Perf report for the last 5.14527 seconds: | 20260506 22:17:14.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:17:14.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:14.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:14.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:14.086 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 22:17:14.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:14.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:14.086 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:17:14.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:14.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:14.086 | INFO | Uploaded 36GiB to GPU in 1.98719s (18.116GiB/s) | 20260506 22:17:14.086 | INFO | Downloaded 34GiB to GPU in 1.91601s (17.7452GiB/s) | 20260506 22:17:19.229 | INFO | Analyzing chunk frames = [4875776,4876288) | 20260506 22:17:19.229 | INFO | MemInfo: { | 20260506 22:17:19.229 | INFO | "gpu" : " 17 GB", | 20260506 22:17:19.229 | INFO | "huge" : "", | 20260506 22:17:19.229 | INFO | "huge_shared" : " 27 GB", | 20260506 22:17:19.229 | INFO | "malloc" : " 14 GB", | 20260506 22:17:19.229 | INFO | "pinned" : " 29 GB" | 20260506 22:17:19.229 | INFO | } | 20260506 22:17:19.229 | INFO | Perf report for the last 5.14388 seconds: | 20260506 22:17:19.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:17:19.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:19.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:19.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:19.230 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 22:17:19.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:19.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:19.230 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:17:19.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:19.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:19.230 | INFO | Uploaded 36GiB to GPU in 1.96471s (18.3233GiB/s) | 20260506 22:17:19.230 | INFO | Downloaded 34GiB to GPU in 1.87675s (18.1164GiB/s) | 20260506 22:17:24.369 | INFO | Analyzing chunk frames = [4876288,4876800) | 20260506 22:17:24.369 | INFO | MemInfo: { | 20260506 22:17:24.369 | INFO | "gpu" : " 17 GB", | 20260506 22:17:24.369 | INFO | "huge" : "", | 20260506 22:17:24.369 | INFO | "huge_shared" : " 27 GB", | 20260506 22:17:24.369 | INFO | "malloc" : " 14 GB", | 20260506 22:17:24.369 | INFO | "pinned" : " 29 GB" | 20260506 22:17:24.369 | INFO | } | 20260506 22:17:24.369 | INFO | Perf report for the last 5.13974 seconds: | 20260506 22:17:24.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:17:24.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:24.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:24.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:24.369 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260506 22:17:24.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:24.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:24.369 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:17:24.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:24.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:24.370 | INFO | Uploaded 36GiB to GPU in 2.00812s (17.9272GiB/s) | 20260506 22:17:24.370 | INFO | Downloaded 34GiB to GPU in 1.93986s (17.527GiB/s) | 20260506 22:17:29.505 | INFO | Analyzing chunk frames = [4876800,4877312) | 20260506 22:17:29.505 | INFO | MemInfo: { | 20260506 22:17:29.505 | INFO | "gpu" : " 17 GB", | 20260506 22:17:29.505 | INFO | "huge" : "", | 20260506 22:17:29.505 | INFO | "huge_shared" : " 27 GB", | 20260506 22:17:29.505 | INFO | "malloc" : " 14 GB", | 20260506 22:17:29.505 | INFO | "pinned" : " 29 GB" | 20260506 22:17:29.505 | INFO | } | 20260506 22:17:29.505 | INFO | Perf report for the last 5.13586 seconds: | 20260506 22:17:29.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:17:29.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:29.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:29.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:29.505 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:17:29.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:29.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:29.505 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 22:17:29.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:29.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:29.505 | INFO | Uploaded 36GiB to GPU in 2.02978s (17.736GiB/s) | 20260506 22:17:29.505 | INFO | Downloaded 34GiB to GPU in 1.93728s (17.5503GiB/s) | 20260506 22:17:34.638 | INFO | Analyzing chunk frames = [4877312,4877824) | 20260506 22:17:34.638 | INFO | MemInfo: { | 20260506 22:17:34.638 | INFO | "gpu" : " 17 GB", | 20260506 22:17:34.638 | INFO | "huge" : "", | 20260506 22:17:34.638 | INFO | "huge_shared" : " 27 GB", | 20260506 22:17:34.638 | INFO | "malloc" : " 14 GB", | 20260506 22:17:34.638 | INFO | "pinned" : " 29 GB" | 20260506 22:17:34.638 | INFO | } | 20260506 22:17:34.638 | INFO | Perf report for the last 5.13307 seconds: | 20260506 22:17:34.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:17:34.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:34.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:34.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:34.638 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 22:17:34.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:34.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:34.638 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 22:17:34.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:34.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:34.638 | INFO | Uploaded 36GiB to GPU in 1.9658s (18.3132GiB/s) | 20260506 22:17:34.638 | INFO | Downloaded 34GiB to GPU in 1.98983s (17.0869GiB/s) | 20260506 22:17:39.776 | INFO | Analyzing chunk frames = [4877824,4878336) | 20260506 22:17:39.776 | INFO | MemInfo: { | 20260506 22:17:39.776 | INFO | "gpu" : " 17 GB", | 20260506 22:17:39.776 | INFO | "huge" : "", | 20260506 22:17:39.776 | INFO | "huge_shared" : " 27 GB", | 20260506 22:17:39.776 | INFO | "malloc" : " 14 GB", | 20260506 22:17:39.776 | INFO | "pinned" : " 29 GB" | 20260506 22:17:39.776 | INFO | } | 20260506 22:17:39.776 | INFO | Perf report for the last 5.13795 seconds: | 20260506 22:17:39.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:17:39.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:39.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:39.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:39.776 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 22:17:39.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:39.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:39.776 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 22:17:39.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:39.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:39.776 | INFO | Uploaded 36GiB to GPU in 1.98154s (18.1677GiB/s) | 20260506 22:17:39.776 | INFO | Downloaded 34GiB to GPU in 1.99348s (17.0556GiB/s) | 20260506 22:17:39.876 | INFO | Starting a round of Baz IO | 20260506 22:17:43.011 | INFO | Populating RTMetrics report | 20260506 22:17:43.012 | INFO | RTMetrics report populated | 20260506 22:17:43.012 | INFO | Opening tmp RTMetrics json file | 20260506 22:17:43.017 | INFO | Serializing RTMetrics to json | 20260506 22:17:43.031 | INFO | Serialization done, swapping tmp file now... | 20260506 22:17:43.031 | INFO | RTMetrics done | 20260506 22:17:43.104 | INFO | Finished a round of Baz IO: Wrote 1.917 GiB in 3.228 seconds | 20260506 22:17:43.104 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.72% / 0.00% / 66.28% | 20260506 22:17:43.104 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.519351% / 0% / 5.16574e-05% | 20260506 22:17:43.105 | INFO | Baz IO overall: | 20260506 22:17:43.105 | INFO | BytesWritten : 2564163534197 | 20260506 22:17:43.105 | INFO | NumEvents : 1237322558426 | 20260506 22:17:44.923 | INFO | Analyzing chunk frames = [4878336,4878848) | 20260506 22:17:44.923 | INFO | MemInfo: { | 20260506 22:17:44.923 | INFO | "gpu" : " 17 GB", | 20260506 22:17:44.923 | INFO | "huge" : "", | 20260506 22:17:44.923 | INFO | "huge_shared" : " 27 GB", | 20260506 22:17:44.923 | INFO | "malloc" : " 14 GB", | 20260506 22:17:44.923 | INFO | "pinned" : " 29 GB" | 20260506 22:17:44.923 | INFO | } | 20260506 22:17:44.923 | INFO | Perf report for the last 5.14688 seconds: | 20260506 22:17:44.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:17:44.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:44.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:44.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:44.923 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 22:17:44.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:44.923 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 22:17:44.923 | INFO | PRE_HQ : 59.6%, 1.8, 57.0ms, 0 | 20260506 22:17:44.923 | INFO | BAZWRITER : 62.3%, 3.4, 108.7ms, 0 | 20260506 22:17:44.923 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:17:44.923 | INFO | Uploaded 36GiB to GPU in 2.03314s (17.7066GiB/s) | 20260506 22:17:44.923 | INFO | Downloaded 38GiB to GPU in 2.17822s (17.4454GiB/s) | 20260506 22:17:50.061 | INFO | Analyzing chunk frames = [4878848,4879360) | 20260506 22:17:50.061 | INFO | MemInfo: { | 20260506 22:17:50.061 | INFO | "gpu" : " 17 GB", | 20260506 22:17:50.061 | INFO | "huge" : "", | 20260506 22:17:50.061 | INFO | "huge_shared" : " 27 GB", | 20260506 22:17:50.061 | INFO | "malloc" : " 14 GB", | 20260506 22:17:50.061 | INFO | "pinned" : " 29 GB" | 20260506 22:17:50.061 | INFO | } | 20260506 22:17:50.062 | INFO | Perf report for the last 5.13848 seconds: | 20260506 22:17:50.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:17:50.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:50.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:50.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:50.062 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:17:50.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:50.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:50.062 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:17:50.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:50.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:50.062 | INFO | Uploaded 36GiB to GPU in 2.00741s (17.9336GiB/s) | 20260506 22:17:50.062 | INFO | Downloaded 34GiB to GPU in 1.92699s (17.6441GiB/s) | 20260506 22:17:55.201 | INFO | Analyzing chunk frames = [4879360,4879872) | 20260506 22:17:55.201 | INFO | MemInfo: { | 20260506 22:17:55.201 | INFO | "gpu" : " 17 GB", | 20260506 22:17:55.201 | INFO | "huge" : "", | 20260506 22:17:55.201 | INFO | "huge_shared" : " 27 GB", | 20260506 22:17:55.201 | INFO | "malloc" : " 14 GB", | 20260506 22:17:55.201 | INFO | "pinned" : " 29 GB" | 20260506 22:17:55.201 | INFO | } | 20260506 22:17:55.201 | INFO | Perf report for the last 5.13949 seconds: | 20260506 22:17:55.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:17:55.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:55.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:55.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:55.201 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 22:17:55.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:55.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:55.201 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:17:55.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:55.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:17:55.201 | INFO | Uploaded 36GiB to GPU in 1.98906s (18.099GiB/s) | 20260506 22:17:55.201 | INFO | Downloaded 34GiB to GPU in 1.93233s (17.5954GiB/s) | 20260506 22:18:00.343 | INFO | Analyzing chunk frames = [4879872,4880384) | 20260506 22:18:00.343 | INFO | MemInfo: { | 20260506 22:18:00.343 | INFO | "gpu" : " 17 GB", | 20260506 22:18:00.344 | INFO | "huge" : "", | 20260506 22:18:00.344 | INFO | "huge_shared" : " 27 GB", | 20260506 22:18:00.344 | INFO | "malloc" : " 14 GB", | 20260506 22:18:00.344 | INFO | "pinned" : " 29 GB" | 20260506 22:18:00.344 | INFO | } | 20260506 22:18:00.344 | INFO | Perf report for the last 5.14252 seconds: | 20260506 22:18:00.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:18:00.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:00.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:00.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:00.344 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260506 22:18:00.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:00.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:00.344 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:18:00.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:00.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:00.344 | INFO | Uploaded 36GiB to GPU in 2.01659s (17.8519GiB/s) | 20260506 22:18:00.344 | INFO | Downloaded 34GiB to GPU in 1.90794s (17.8203GiB/s) | 20260506 22:18:05.481 | INFO | Analyzing chunk frames = [4880384,4880896) | 20260506 22:18:05.481 | INFO | MemInfo: { | 20260506 22:18:05.481 | INFO | "gpu" : " 17 GB", | 20260506 22:18:05.481 | INFO | "huge" : "", | 20260506 22:18:05.481 | INFO | "huge_shared" : " 27 GB", | 20260506 22:18:05.481 | INFO | "malloc" : " 14 GB", | 20260506 22:18:05.481 | INFO | "pinned" : " 29 GB" | 20260506 22:18:05.481 | INFO | } | 20260506 22:18:05.481 | INFO | Perf report for the last 5.13708 seconds: | 20260506 22:18:05.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:18:05.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:05.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:05.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:05.481 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 22:18:05.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:05.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:05.481 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:18:05.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:05.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:05.481 | INFO | Uploaded 36GiB to GPU in 1.99434s (18.0511GiB/s) | 20260506 22:18:05.481 | INFO | Downloaded 34GiB to GPU in 1.92449s (17.667GiB/s) | 20260506 22:18:10.613 | INFO | Analyzing chunk frames = [4880896,4881408) | 20260506 22:18:10.613 | INFO | MemInfo: { | 20260506 22:18:10.613 | INFO | "gpu" : " 17 GB", | 20260506 22:18:10.613 | INFO | "huge" : "", | 20260506 22:18:10.613 | INFO | "huge_shared" : " 27 GB", | 20260506 22:18:10.613 | INFO | "malloc" : " 14 GB", | 20260506 22:18:10.613 | INFO | "pinned" : " 29 GB" | 20260506 22:18:10.613 | INFO | } | 20260506 22:18:10.613 | INFO | Perf report for the last 5.13257 seconds: | 20260506 22:18:10.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:18:10.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:10.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:10.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:10.613 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:18:10.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:10.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:10.613 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 22:18:10.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:10.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:10.613 | INFO | Uploaded 36GiB to GPU in 2.01219s (17.891GiB/s) | 20260506 22:18:10.613 | INFO | Downloaded 34GiB to GPU in 1.93072s (17.61GiB/s) | 20260506 22:18:15.752 | INFO | Analyzing chunk frames = [4881408,4881920) | 20260506 22:18:15.753 | INFO | MemInfo: { | 20260506 22:18:15.753 | INFO | "gpu" : " 17 GB", | 20260506 22:18:15.753 | INFO | "huge" : "", | 20260506 22:18:15.753 | INFO | "huge_shared" : " 27 GB", | 20260506 22:18:15.753 | INFO | "malloc" : " 14 GB", | 20260506 22:18:15.753 | INFO | "pinned" : " 29 GB" | 20260506 22:18:15.753 | INFO | } | 20260506 22:18:15.753 | INFO | Perf report for the last 5.13949 seconds: | 20260506 22:18:15.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:18:15.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:15.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:15.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:15.753 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 22:18:15.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:15.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:15.753 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 22:18:15.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:15.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:15.753 | INFO | Uploaded 36GiB to GPU in 1.98388s (18.1463GiB/s) | 20260506 22:18:15.753 | INFO | Downloaded 34GiB to GPU in 2.00683s (16.9421GiB/s) | 20260506 22:18:20.892 | INFO | Analyzing chunk frames = [4881920,4882432) | 20260506 22:18:20.892 | INFO | MemInfo: { | 20260506 22:18:20.892 | INFO | "gpu" : " 17 GB", | 20260506 22:18:20.892 | INFO | "huge" : "", | 20260506 22:18:20.892 | INFO | "huge_shared" : " 27 GB", | 20260506 22:18:20.892 | INFO | "malloc" : " 14 GB", | 20260506 22:18:20.892 | INFO | "pinned" : " 29 GB" | 20260506 22:18:20.892 | INFO | } | 20260506 22:18:20.892 | INFO | Perf report for the last 5.13971 seconds: | 20260506 22:18:20.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:18:20.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:20.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:20.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:20.893 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:18:20.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:20.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:20.893 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 22:18:20.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:20.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:20.893 | INFO | Uploaded 36GiB to GPU in 2.04791s (17.5789GiB/s) | 20260506 22:18:20.893 | INFO | Downloaded 34GiB to GPU in 1.99042s (17.0819GiB/s) | 20260506 22:18:20.893 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:18:20.893 | INFO | Total mem usage: 18 GiB | 20260506 22:18:20.893 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:18:20.893 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:18:20.893 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:18:20.893 | INFO | High Water Breakdown: | 20260506 22:18:20.893 | INFO | 60.1% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:18:20.893 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:18:20.893 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:18:20.893 | INFO | ------------------------------------------ | 20260506 22:18:20.893 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:18:20.893 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:18:20.893 | INFO | Total mem usage: 26.5 GiB | 20260506 22:18:20.893 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:18:20.893 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:18:20.893 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:18:20.893 | INFO | High Water Breakdown: | 20260506 22:18:20.893 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:18:20.893 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:18:20.893 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:18:20.893 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:18:20.893 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:18:20.893 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:18:20.893 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:18:20.893 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:18:20.893 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:18:20.893 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:18:20.893 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:18:20.893 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:18:20.893 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:18:20.893 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:18:20.893 | INFO | ------------------------------------------ | 20260506 22:18:20.893 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:18:20.893 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:18:20.893 | INFO | Total mem usage: 25.6 GiB | 20260506 22:18:20.893 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:18:20.893 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:18:20.893 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:18:20.893 | INFO | High Water Breakdown: | 20260506 22:18:20.893 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:18:20.893 | INFO | ------------------------------------------ | 20260506 22:18:20.893 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:18:20.893 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:18:20.893 | INFO | Total mem usage: 15.7 GiB | 20260506 22:18:20.893 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:18:20.893 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:18:20.893 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:18:20.893 | INFO | High Water Breakdown: | 20260506 22:18:20.893 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:18:20.893 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:18:20.893 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:18:20.893 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:18:20.893 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:18:20.893 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:18:20.893 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:18:20.894 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:18:20.894 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:18:20.894 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:18:20.894 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:18:20.894 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:18:20.894 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:18:20.894 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:18:20.894 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:18:20.894 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:18:20.894 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:18:20.894 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:18:20.894 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:18:20.894 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:18:20.894 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:18:20.894 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:18:20.894 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:18:20.894 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:18:20.894 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:18:20.894 | INFO | ------------------------------------------ | 20260506 22:18:20.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:18:20.894 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:18:20.894 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:18:20.894 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:18:20.894 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.60e+01, 1.36e+01 / 2.36e+00 | 20260506 22:18:20.894 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.02e-01, 9.74e-03 / 1.01e-02 | 20260506 22:18:20.894 | INFO | PerfReport: 11.65%, Baseline -- 3.62e+00 / 4.34e+00, 5.43e-01 / 3.71e-01 | 20260506 22:18:20.894 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.43e+00, 1.77e-01 / 1.39e-01 | 20260506 22:18:20.894 | INFO | PerfReport: 16.09%, DME -- 4.00e+01 / 4.51e+01, 4.16e+00 / 2.60e+00 | 20260506 22:18:20.894 | INFO | PerfReport: 57.29%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.34e+00 / 1.50e+00 | 20260506 22:18:20.894 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.20e+00 / 2.75e+00, 3.76e-01 / 2.17e-01 | 20260506 22:18:20.894 | INFO | PerfReport: 3.23%, Metrics -- 1.00e+00 / 1.36e+00, 2.65e-01 / 1.56e-01 | 20260506 22:18:20.894 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.44e-01 / 3.95e-01, 5.46e-02 / 2.17e-02 | 20260506 22:18:20.894 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:18:20.894 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:18:20.894 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:18:20.894 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 22:18:20.894 | INFO | PerfReport: 21.54%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 22:18:20.894 | INFO | PerfReport: 21.22%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 22:18:20.894 | INFO | PerfReport: 32.06%, Compute -- 3.11e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 22:18:20.894 | INFO | PerfReport: 1.39%, LockUpload -- 1.35e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260506 22:18:20.894 | INFO | PerfReport: 1.19%, LockDownload -- 1.15e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260506 22:18:20.894 | INFO | PerfReport: 22.59%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 22:18:20.894 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260506 22:18:21.002 | INFO | Starting a round of Baz IO | 20260506 22:18:24.131 | INFO | Populating RTMetrics report | 20260506 22:18:24.131 | INFO | RTMetrics report populated | 20260506 22:18:24.131 | INFO | Opening tmp RTMetrics json file | 20260506 22:18:24.137 | INFO | Serializing RTMetrics to json | 20260506 22:18:24.152 | INFO | Serialization done, swapping tmp file now... | 20260506 22:18:24.153 | INFO | RTMetrics done | 20260506 22:18:24.225 | INFO | Finished a round of Baz IO: Wrote 1.917 GiB in 3.223 seconds | 20260506 22:18:24.225 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.71% / 0.00% / 66.28% | 20260506 22:18:24.226 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.519689% / 1.19209e-05% / 6.35783e-05% | 20260506 22:18:24.226 | INFO | Baz IO overall: | 20260506 22:18:24.226 | INFO | BytesWritten : 2566222299502 | 20260506 22:18:24.226 | INFO | NumEvents : 1238306086844 | 20260506 22:18:26.037 | INFO | Analyzing chunk frames = [4882432,4882944) | 20260506 22:18:26.038 | INFO | MemInfo: { | 20260506 22:18:26.038 | INFO | "gpu" : " 17 GB", | 20260506 22:18:26.038 | INFO | "huge" : "", | 20260506 22:18:26.038 | INFO | "huge_shared" : " 27 GB", | 20260506 22:18:26.038 | INFO | "malloc" : " 14 GB", | 20260506 22:18:26.038 | INFO | "pinned" : " 29 GB" | 20260506 22:18:26.038 | INFO | } | 20260506 22:18:26.038 | INFO | Perf report for the last 5.14516 seconds: | 20260506 22:18:26.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:18:26.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:26.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:26.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:26.038 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 22:18:26.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:26.038 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 22:18:26.038 | INFO | PRE_HQ : 59.7%, 1.8, 56.8ms, 0 | 20260506 22:18:26.038 | INFO | BAZWRITER : 62.3%, 3.4, 109.8ms, 0 | 20260506 22:18:26.038 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:18:26.038 | INFO | Uploaded 36GiB to GPU in 2.02875s (17.7449GiB/s) | 20260506 22:18:26.038 | INFO | Downloaded 38GiB to GPU in 2.18678s (17.3771GiB/s) | 20260506 22:18:31.182 | INFO | Analyzing chunk frames = [4882944,4883456) | 20260506 22:18:31.182 | INFO | MemInfo: { | 20260506 22:18:31.182 | INFO | "gpu" : " 17 GB", | 20260506 22:18:31.182 | INFO | "huge" : "", | 20260506 22:18:31.182 | INFO | "huge_shared" : " 27 GB", | 20260506 22:18:31.182 | INFO | "malloc" : " 14 GB", | 20260506 22:18:31.182 | INFO | "pinned" : " 29 GB" | 20260506 22:18:31.182 | INFO | } | 20260506 22:18:31.182 | INFO | Perf report for the last 5.14478 seconds: | 20260506 22:18:31.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:18:31.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:31.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:31.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:31.182 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 22:18:31.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:31.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:31.183 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:18:31.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:31.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:31.183 | INFO | Uploaded 36GiB to GPU in 2.00927s (17.9169GiB/s) | 20260506 22:18:31.183 | INFO | Downloaded 34GiB to GPU in 1.92644s (17.6491GiB/s) | 20260506 22:18:36.316 | INFO | Analyzing chunk frames = [4883456,4883968) | 20260506 22:18:36.316 | INFO | MemInfo: { | 20260506 22:18:36.316 | INFO | "gpu" : " 17 GB", | 20260506 22:18:36.316 | INFO | "huge" : "", | 20260506 22:18:36.316 | INFO | "huge_shared" : " 27 GB", | 20260506 22:18:36.316 | INFO | "malloc" : " 14 GB", | 20260506 22:18:36.316 | INFO | "pinned" : " 29 GB" | 20260506 22:18:36.316 | INFO | } | 20260506 22:18:36.316 | INFO | Perf report for the last 5.13401 seconds: | 20260506 22:18:36.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:18:36.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:36.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:36.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:36.317 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260506 22:18:36.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:36.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:36.317 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:18:36.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:36.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:36.317 | INFO | Uploaded 36GiB to GPU in 2.00864s (17.9226GiB/s) | 20260506 22:18:36.317 | INFO | Downloaded 34GiB to GPU in 1.9337s (17.5828GiB/s) | 20260506 22:18:41.453 | INFO | Analyzing chunk frames = [4883968,4884480) | 20260506 22:18:41.453 | INFO | MemInfo: { | 20260506 22:18:41.453 | INFO | "gpu" : " 17 GB", | 20260506 22:18:41.453 | INFO | "huge" : "", | 20260506 22:18:41.453 | INFO | "huge_shared" : " 27 GB", | 20260506 22:18:41.453 | INFO | "malloc" : " 14 GB", | 20260506 22:18:41.453 | INFO | "pinned" : " 29 GB" | 20260506 22:18:41.453 | INFO | } | 20260506 22:18:41.453 | INFO | Perf report for the last 5.13682 seconds: | 20260506 22:18:41.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:18:41.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:41.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:41.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:41.453 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260506 22:18:41.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:41.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:41.453 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:18:41.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:41.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:41.453 | INFO | Uploaded 36GiB to GPU in 2.01808s (17.8388GiB/s) | 20260506 22:18:41.453 | INFO | Downloaded 34GiB to GPU in 1.90669s (17.832GiB/s) | 20260506 22:18:46.588 | INFO | Analyzing chunk frames = [4884480,4884992) | 20260506 22:18:46.588 | INFO | MemInfo: { | 20260506 22:18:46.588 | INFO | "gpu" : " 17 GB", | 20260506 22:18:46.588 | INFO | "huge" : "", | 20260506 22:18:46.588 | INFO | "huge_shared" : " 27 GB", | 20260506 22:18:46.588 | INFO | "malloc" : " 14 GB", | 20260506 22:18:46.588 | INFO | "pinned" : " 29 GB" | 20260506 22:18:46.588 | INFO | } | 20260506 22:18:46.588 | INFO | Perf report for the last 5.13464 seconds: | 20260506 22:18:46.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:18:46.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:46.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:46.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:46.588 | INFO | ANALYSIS : 60.5%, 3.0, 94.0ms, 0 | 20260506 22:18:46.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:46.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:46.588 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:18:46.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:46.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:46.588 | INFO | Uploaded 36GiB to GPU in 2.02041s (17.8182GiB/s) | 20260506 22:18:46.588 | INFO | Downloaded 34GiB to GPU in 1.91294s (17.7737GiB/s) | 20260506 22:18:51.726 | INFO | Analyzing chunk frames = [4884992,4885504) | 20260506 22:18:51.726 | INFO | MemInfo: { | 20260506 22:18:51.726 | INFO | "gpu" : " 17 GB", | 20260506 22:18:51.726 | INFO | "huge" : "", | 20260506 22:18:51.726 | INFO | "huge_shared" : " 27 GB", | 20260506 22:18:51.726 | INFO | "malloc" : " 14 GB", | 20260506 22:18:51.726 | INFO | "pinned" : " 29 GB" | 20260506 22:18:51.726 | INFO | } | 20260506 22:18:51.726 | INFO | Perf report for the last 5.13826 seconds: | 20260506 22:18:51.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:18:51.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:51.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:51.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:51.726 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 22:18:51.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:51.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:51.726 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 22:18:51.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:51.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:51.726 | INFO | Uploaded 36GiB to GPU in 2.00786s (17.9295GiB/s) | 20260506 22:18:51.726 | INFO | Downloaded 34GiB to GPU in 1.94851s (17.4493GiB/s) | 20260506 22:18:56.867 | INFO | Analyzing chunk frames = [4885504,4886016) | 20260506 22:18:56.867 | INFO | MemInfo: { | 20260506 22:18:56.867 | INFO | "gpu" : " 17 GB", | 20260506 22:18:56.867 | INFO | "huge" : "", | 20260506 22:18:56.867 | INFO | "huge_shared" : " 27 GB", | 20260506 22:18:56.867 | INFO | "malloc" : " 14 GB", | 20260506 22:18:56.867 | INFO | "pinned" : " 29 GB" | 20260506 22:18:56.868 | INFO | } | 20260506 22:18:56.868 | INFO | Perf report for the last 5.14133 seconds: | 20260506 22:18:56.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:18:56.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:56.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:56.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:56.868 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 22:18:56.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:56.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:56.868 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:18:56.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:56.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:18:56.868 | INFO | Uploaded 36GiB to GPU in 2.00065s (17.9941GiB/s) | 20260506 22:18:56.868 | INFO | Downloaded 34GiB to GPU in 2.00969s (16.918GiB/s) | 20260506 22:19:02.013 | INFO | Analyzing chunk frames = [4886016,4886528) | 20260506 22:19:02.013 | INFO | MemInfo: { | 20260506 22:19:02.013 | INFO | "gpu" : " 17 GB", | 20260506 22:19:02.013 | INFO | "huge" : "", | 20260506 22:19:02.013 | INFO | "huge_shared" : " 27 GB", | 20260506 22:19:02.013 | INFO | "malloc" : " 14 GB", | 20260506 22:19:02.013 | INFO | "pinned" : " 29 GB" | 20260506 22:19:02.013 | INFO | } | 20260506 22:19:02.013 | INFO | Perf report for the last 5.14521 seconds: | 20260506 22:19:02.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:19:02.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:02.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:02.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:02.013 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 22:19:02.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:02.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:02.013 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 22:19:02.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:02.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:02.013 | INFO | Uploaded 36GiB to GPU in 2.05688s (17.5022GiB/s) | 20260506 22:19:02.013 | INFO | Downloaded 34GiB to GPU in 2.00202s (16.9828GiB/s) | 20260506 22:19:02.124 | INFO | Starting a round of Baz IO | 20260506 22:19:05.243 | INFO | Populating RTMetrics report | 20260506 22:19:05.243 | INFO | RTMetrics report populated | 20260506 22:19:05.243 | INFO | Opening tmp RTMetrics json file | 20260506 22:19:05.243 | INFO | Serializing RTMetrics to json | 20260506 22:19:05.245 | INFO | Serialization done, swapping tmp file now... | 20260506 22:19:05.245 | INFO | RTMetrics done | 20260506 22:19:05.340 | INFO | Finished a round of Baz IO: Wrote 1.915 GiB in 3.216 seconds | 20260506 22:19:05.341 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.71% / 0.00% / 66.29% | 20260506 22:19:05.341 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.516268% / 3.97364e-06% / 6.75519e-05% | 20260506 22:19:05.341 | INFO | Baz IO overall: | 20260506 22:19:05.341 | INFO | BytesWritten : 2568279006488 | 20260506 22:19:05.341 | INFO | NumEvents : 1239288184283 | 20260506 22:19:07.154 | INFO | Analyzing chunk frames = [4886528,4887040) | 20260506 22:19:07.154 | INFO | MemInfo: { | 20260506 22:19:07.154 | INFO | "gpu" : " 17 GB", | 20260506 22:19:07.154 | INFO | "huge" : "", | 20260506 22:19:07.154 | INFO | "huge_shared" : " 27 GB", | 20260506 22:19:07.154 | INFO | "malloc" : " 14 GB", | 20260506 22:19:07.154 | INFO | "pinned" : " 29 GB" | 20260506 22:19:07.154 | INFO | } | 20260506 22:19:07.154 | INFO | Perf report for the last 5.14127 seconds: | 20260506 22:19:07.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:19:07.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:07.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:07.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:07.154 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 22:19:07.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:07.154 | INFO | RT_METRICS : 42.4%, 1.0, 22.3ms, 0 | 20260506 22:19:07.154 | INFO | PRE_HQ : 59.7%, 1.8, 57.1ms, 0 | 20260506 22:19:07.154 | INFO | BAZWRITER : 62.1%, 3.5, 111.2ms, 0 | 20260506 22:19:07.154 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:19:07.154 | INFO | Uploaded 36GiB to GPU in 2.04794s (17.5786GiB/s) | 20260506 22:19:07.154 | INFO | Downloaded 38GiB to GPU in 2.18348s (17.4034GiB/s) | 20260506 22:19:12.294 | INFO | Analyzing chunk frames = [4887040,4887552) | 20260506 22:19:12.294 | INFO | MemInfo: { | 20260506 22:19:12.294 | INFO | "gpu" : " 17 GB", | 20260506 22:19:12.294 | INFO | "huge" : "", | 20260506 22:19:12.294 | INFO | "huge_shared" : " 27 GB", | 20260506 22:19:12.294 | INFO | "malloc" : " 14 GB", | 20260506 22:19:12.294 | INFO | "pinned" : " 29 GB" | 20260506 22:19:12.294 | INFO | } | 20260506 22:19:12.294 | INFO | Perf report for the last 5.13965 seconds: | 20260506 22:19:12.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:19:12.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:12.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:12.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:12.294 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:19:12.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:12.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:12.294 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:19:12.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:12.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:12.294 | INFO | Uploaded 36GiB to GPU in 2.00168s (17.9849GiB/s) | 20260506 22:19:12.294 | INFO | Downloaded 34GiB to GPU in 1.92485s (17.6637GiB/s) | 20260506 22:19:17.430 | INFO | Analyzing chunk frames = [4887552,4888064) | 20260506 22:19:17.430 | INFO | MemInfo: { | 20260506 22:19:17.430 | INFO | "gpu" : " 17 GB", | 20260506 22:19:17.430 | INFO | "huge" : "", | 20260506 22:19:17.430 | INFO | "huge_shared" : " 27 GB", | 20260506 22:19:17.430 | INFO | "malloc" : " 14 GB", | 20260506 22:19:17.430 | INFO | "pinned" : " 29 GB" | 20260506 22:19:17.430 | INFO | } | 20260506 22:19:17.430 | INFO | Perf report for the last 5.13668 seconds: | 20260506 22:19:17.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:19:17.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:17.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:17.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:17.430 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 22:19:17.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:17.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:17.430 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:19:17.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:17.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:17.431 | INFO | Uploaded 36GiB to GPU in 1.98096s (18.173GiB/s) | 20260506 22:19:17.431 | INFO | Downloaded 34GiB to GPU in 1.92511s (17.6613GiB/s) | 20260506 22:19:22.565 | INFO | Analyzing chunk frames = [4888064,4888576) | 20260506 22:19:22.565 | INFO | MemInfo: { | 20260506 22:19:22.565 | INFO | "gpu" : " 17 GB", | 20260506 22:19:22.565 | INFO | "huge" : "", | 20260506 22:19:22.565 | INFO | "huge_shared" : " 27 GB", | 20260506 22:19:22.565 | INFO | "malloc" : " 14 GB", | 20260506 22:19:22.565 | INFO | "pinned" : " 29 GB" | 20260506 22:19:22.565 | INFO | } | 20260506 22:19:22.565 | INFO | Perf report for the last 5.13477 seconds: | 20260506 22:19:22.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:19:22.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:22.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:22.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:22.565 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 22:19:22.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:22.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:22.565 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:19:22.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:22.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:22.565 | INFO | Uploaded 36GiB to GPU in 1.96615s (18.3099GiB/s) | 20260506 22:19:22.565 | INFO | Downloaded 34GiB to GPU in 1.87475s (18.1358GiB/s) | 20260506 22:19:27.702 | INFO | Analyzing chunk frames = [4888576,4889088) | 20260506 22:19:27.702 | INFO | MemInfo: { | 20260506 22:19:27.702 | INFO | "gpu" : " 17 GB", | 20260506 22:19:27.702 | INFO | "huge" : "", | 20260506 22:19:27.702 | INFO | "huge_shared" : " 27 GB", | 20260506 22:19:27.702 | INFO | "malloc" : " 14 GB", | 20260506 22:19:27.702 | INFO | "pinned" : " 29 GB" | 20260506 22:19:27.702 | INFO | } | 20260506 22:19:27.702 | INFO | Perf report for the last 5.13668 seconds: | 20260506 22:19:27.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:19:27.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:27.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:27.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:27.702 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 22:19:27.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:27.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:27.702 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:19:27.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:27.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:27.702 | INFO | Uploaded 36GiB to GPU in 2.005s (17.9551GiB/s) | 20260506 22:19:27.702 | INFO | Downloaded 34GiB to GPU in 1.91567s (17.7483GiB/s) | 20260506 22:19:32.845 | INFO | Analyzing chunk frames = [4889088,4889600) | 20260506 22:19:32.845 | INFO | MemInfo: { | 20260506 22:19:32.845 | INFO | "gpu" : " 17 GB", | 20260506 22:19:32.845 | INFO | "huge" : "", | 20260506 22:19:32.845 | INFO | "huge_shared" : " 27 GB", | 20260506 22:19:32.845 | INFO | "malloc" : " 14 GB", | 20260506 22:19:32.845 | INFO | "pinned" : " 29 GB" | 20260506 22:19:32.845 | INFO | } | 20260506 22:19:32.845 | INFO | Perf report for the last 5.14311 seconds: | 20260506 22:19:32.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:19:32.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:32.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:32.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:32.845 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 22:19:32.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:32.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:32.845 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260506 22:19:32.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:32.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:32.845 | INFO | Uploaded 36GiB to GPU in 2.02099s (17.813GiB/s) | 20260506 22:19:32.845 | INFO | Downloaded 34GiB to GPU in 1.94774s (17.4562GiB/s) | 20260506 22:19:37.986 | INFO | Analyzing chunk frames = [4889600,4890112) | 20260506 22:19:37.986 | INFO | MemInfo: { | 20260506 22:19:37.986 | INFO | "gpu" : " 17 GB", | 20260506 22:19:37.986 | INFO | "huge" : "", | 20260506 22:19:37.986 | INFO | "huge_shared" : " 27 GB", | 20260506 22:19:37.986 | INFO | "malloc" : " 14 GB", | 20260506 22:19:37.986 | INFO | "pinned" : " 29 GB" | 20260506 22:19:37.986 | INFO | } | 20260506 22:19:37.986 | INFO | Perf report for the last 5.14126 seconds: | 20260506 22:19:37.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:19:37.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:37.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:37.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:37.986 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:19:37.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:37.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:37.986 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 22:19:37.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:37.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:37.987 | INFO | Uploaded 36GiB to GPU in 1.96625s (18.309GiB/s) | 20260506 22:19:37.987 | INFO | Downloaded 34GiB to GPU in 1.98704s (17.1109GiB/s) | 20260506 22:19:43.130 | INFO | Analyzing chunk frames = [4890112,4890624) | 20260506 22:19:43.130 | INFO | MemInfo: { | 20260506 22:19:43.130 | INFO | "gpu" : " 17 GB", | 20260506 22:19:43.130 | INFO | "huge" : "", | 20260506 22:19:43.130 | INFO | "huge_shared" : " 27 GB", | 20260506 22:19:43.130 | INFO | "malloc" : " 14 GB", | 20260506 22:19:43.130 | INFO | "pinned" : " 29 GB" | 20260506 22:19:43.130 | INFO | } | 20260506 22:19:43.130 | INFO | Perf report for the last 5.14395 seconds: | 20260506 22:19:43.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:19:43.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:43.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:43.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:43.130 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 22:19:43.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:43.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:43.130 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 22:19:43.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:43.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:43.131 | INFO | Uploaded 36GiB to GPU in 1.99063s (18.0848GiB/s) | 20260506 22:19:43.131 | INFO | Downloaded 34GiB to GPU in 1.98911s (17.093GiB/s) | 20260506 22:19:43.131 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:19:43.131 | INFO | Total mem usage: 18 GiB | 20260506 22:19:43.131 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:19:43.131 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:19:43.131 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:19:43.131 | INFO | High Water Breakdown: | 20260506 22:19:43.131 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:19:43.131 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:19:43.131 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:19:43.131 | INFO | ------------------------------------------ | 20260506 22:19:43.131 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:19:43.131 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:19:43.131 | INFO | Total mem usage: 26.5 GiB | 20260506 22:19:43.131 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:19:43.131 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:19:43.131 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:19:43.131 | INFO | High Water Breakdown: | 20260506 22:19:43.131 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:19:43.131 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:19:43.131 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:19:43.131 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:19:43.131 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:19:43.131 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:19:43.131 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:19:43.131 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:19:43.131 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:19:43.131 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:19:43.131 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:19:43.131 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:19:43.131 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:19:43.131 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:19:43.131 | INFO | ------------------------------------------ | 20260506 22:19:43.131 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:19:43.131 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:19:43.131 | INFO | Total mem usage: 25.6 GiB | 20260506 22:19:43.131 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:19:43.131 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:19:43.131 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:19:43.131 | INFO | High Water Breakdown: | 20260506 22:19:43.131 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:19:43.131 | INFO | ------------------------------------------ | 20260506 22:19:43.131 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:19:43.131 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:19:43.131 | INFO | Total mem usage: 15.7 GiB | 20260506 22:19:43.131 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:19:43.131 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:19:43.131 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:19:43.131 | INFO | High Water Breakdown: | 20260506 22:19:43.131 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:19:43.131 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:19:43.131 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:19:43.131 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:19:43.131 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:19:43.131 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:19:43.131 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:19:43.131 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:19:43.131 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:19:43.131 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:19:43.131 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:19:43.131 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:19:43.131 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:19:43.131 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:19:43.131 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:19:43.131 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:19:43.131 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:19:43.131 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:19:43.131 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:19:43.131 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:19:43.131 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:19:43.131 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:19:43.131 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:19:43.131 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:19:43.131 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:19:43.132 | INFO | ------------------------------------------ | 20260506 22:19:43.132 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:19:43.132 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:19:43.132 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:19:43.132 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:19:43.132 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.64e+01, 1.33e+01 / 2.71e+00 | 20260506 22:19:43.132 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.02e-01, 1.01e-02 / 6.57e-03 | 20260506 22:19:43.132 | INFO | PerfReport: 11.64%, Baseline -- 3.62e+00 / 4.39e+00, 5.43e-01 / 3.31e-01 | 20260506 22:19:43.132 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.45e+00, 1.78e-01 / 1.22e-01 | 20260506 22:19:43.132 | INFO | PerfReport: 16.04%, DME -- 3.99e+01 / 4.47e+01, 4.16e+00 / 2.82e+00 | 20260506 22:19:43.132 | INFO | PerfReport: 57.38%, FrameLabeling -- 1.78e+01 / 2.12e+01, 1.40e+00 / 1.66e+00 | 20260506 22:19:43.132 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.19e+00 / 2.77e+00, 3.68e-01 / 1.75e-01 | 20260506 22:19:43.132 | INFO | PerfReport: 3.24%, Metrics -- 1.01e+00 / 1.33e+00, 2.71e-01 / 1.67e-01 | 20260506 22:19:43.132 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.44e-01 / 3.91e-01, 5.38e-02 / 2.33e-02 | 20260506 22:19:43.132 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:19:43.132 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:19:43.132 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:19:43.132 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 22:19:43.132 | INFO | PerfReport: 21.60%, Upload -- 2.09e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260506 22:19:43.132 | INFO | PerfReport: 21.23%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 22:19:43.132 | INFO | PerfReport: 32.08%, Compute -- 3.11e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 22:19:43.132 | INFO | PerfReport: 1.34%, LockUpload -- 1.29e+00 / 0.00e+00, 4.74e+00 / 0.00e+00 | 20260506 22:19:43.132 | INFO | PerfReport: 1.13%, LockDownload -- 1.10e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260506 22:19:43.132 | INFO | PerfReport: 22.61%, LockCompute -- 2.19e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 22:19:43.132 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260506 22:19:43.229 | INFO | Starting a round of Baz IO | 20260506 22:19:46.362 | INFO | Populating RTMetrics report | 20260506 22:19:46.362 | INFO | RTMetrics report populated | 20260506 22:19:46.362 | INFO | Opening tmp RTMetrics json file | 20260506 22:19:46.363 | INFO | Serializing RTMetrics to json | 20260506 22:19:46.364 | INFO | Serialization done, swapping tmp file now... | 20260506 22:19:46.364 | INFO | RTMetrics done | 20260506 22:19:46.456 | INFO | Finished a round of Baz IO: Wrote 1.917 GiB in 3.226 seconds | 20260506 22:19:46.456 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.71% / 0.00% / 66.29% | 20260506 22:19:46.456 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.516029% / 0% / 7.94729e-05% | 20260506 22:19:46.456 | INFO | Baz IO overall: | 20260506 22:19:46.456 | INFO | BytesWritten : 2570337102208 | 20260506 22:19:46.456 | INFO | NumEvents : 1240271317326 | 20260506 22:19:48.271 | INFO | Analyzing chunk frames = [4890624,4891136) | 20260506 22:19:48.271 | INFO | MemInfo: { | 20260506 22:19:48.271 | INFO | "gpu" : " 17 GB", | 20260506 22:19:48.271 | INFO | "huge" : "", | 20260506 22:19:48.271 | INFO | "huge_shared" : " 27 GB", | 20260506 22:19:48.271 | INFO | "malloc" : " 14 GB", | 20260506 22:19:48.271 | INFO | "pinned" : " 29 GB" | 20260506 22:19:48.271 | INFO | } | 20260506 22:19:48.271 | INFO | Perf report for the last 5.14048 seconds: | 20260506 22:19:48.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:19:48.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:48.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:48.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:48.271 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 22:19:48.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:48.271 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 22:19:48.271 | INFO | PRE_HQ : 59.8%, 1.8, 57.0ms, 0 | 20260506 22:19:48.271 | INFO | BAZWRITER : 62.3%, 3.5, 113.0ms, 0 | 20260506 22:19:48.271 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 22:19:48.271 | INFO | Uploaded 36GiB to GPU in 2.01132s (17.8987GiB/s) | 20260506 22:19:48.271 | INFO | Downloaded 38GiB to GPU in 2.17509s (17.4706GiB/s) | 20260506 22:19:53.407 | INFO | Analyzing chunk frames = [4891136,4891648) | 20260506 22:19:53.407 | INFO | MemInfo: { | 20260506 22:19:53.407 | INFO | "gpu" : " 17 GB", | 20260506 22:19:53.407 | INFO | "huge" : "", | 20260506 22:19:53.407 | INFO | "huge_shared" : " 27 GB", | 20260506 22:19:53.407 | INFO | "malloc" : " 14 GB", | 20260506 22:19:53.407 | INFO | "pinned" : " 29 GB" | 20260506 22:19:53.407 | INFO | } | 20260506 22:19:53.407 | INFO | Perf report for the last 5.13616 seconds: | 20260506 22:19:53.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:19:53.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:53.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:53.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:53.407 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:19:53.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:53.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:53.407 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:19:53.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:53.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:53.407 | INFO | Uploaded 36GiB to GPU in 2.00977s (17.9125GiB/s) | 20260506 22:19:53.407 | INFO | Downloaded 34GiB to GPU in 1.93017s (17.615GiB/s) | 20260506 22:19:58.547 | INFO | Analyzing chunk frames = [4891648,4892160) | 20260506 22:19:58.547 | INFO | MemInfo: { | 20260506 22:19:58.547 | INFO | "gpu" : " 17 GB", | 20260506 22:19:58.547 | INFO | "huge" : "", | 20260506 22:19:58.547 | INFO | "huge_shared" : " 27 GB", | 20260506 22:19:58.547 | INFO | "malloc" : " 14 GB", | 20260506 22:19:58.547 | INFO | "pinned" : " 29 GB" | 20260506 22:19:58.547 | INFO | } | 20260506 22:19:58.547 | INFO | Perf report for the last 5.13991 seconds: | 20260506 22:19:58.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:19:58.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:58.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:58.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:58.547 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 22:19:58.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:58.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:58.547 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:19:58.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:58.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:19:58.547 | INFO | Uploaded 36GiB to GPU in 1.97932s (18.188GiB/s) | 20260506 22:19:58.547 | INFO | Downloaded 34GiB to GPU in 1.91835s (17.7235GiB/s) | 20260506 22:20:03.679 | INFO | Analyzing chunk frames = [4892160,4892672) | 20260506 22:20:03.679 | INFO | MemInfo: { | 20260506 22:20:03.679 | INFO | "gpu" : " 17 GB", | 20260506 22:20:03.679 | INFO | "huge" : "", | 20260506 22:20:03.679 | INFO | "huge_shared" : " 27 GB", | 20260506 22:20:03.679 | INFO | "malloc" : " 14 GB", | 20260506 22:20:03.679 | INFO | "pinned" : " 29 GB" | 20260506 22:20:03.679 | INFO | } | 20260506 22:20:03.679 | INFO | Perf report for the last 5.13208 seconds: | 20260506 22:20:03.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:20:03.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:03.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:03.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:03.679 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 22:20:03.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:03.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:03.679 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:20:03.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:03.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:03.679 | INFO | Uploaded 36GiB to GPU in 1.97136s (18.2615GiB/s) | 20260506 22:20:03.679 | INFO | Downloaded 34GiB to GPU in 1.88007s (18.0845GiB/s) | 20260506 22:20:08.819 | INFO | Analyzing chunk frames = [4892672,4893184) | 20260506 22:20:08.819 | INFO | MemInfo: { | 20260506 22:20:08.819 | INFO | "gpu" : " 17 GB", | 20260506 22:20:08.819 | INFO | "huge" : "", | 20260506 22:20:08.819 | INFO | "huge_shared" : " 27 GB", | 20260506 22:20:08.819 | INFO | "malloc" : " 14 GB", | 20260506 22:20:08.819 | INFO | "pinned" : " 29 GB" | 20260506 22:20:08.819 | INFO | } | 20260506 22:20:08.819 | INFO | Perf report for the last 5.14043 seconds: | 20260506 22:20:08.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:20:08.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:08.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:08.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:08.820 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 22:20:08.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:08.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:08.820 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:20:08.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:08.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:08.820 | INFO | Uploaded 36GiB to GPU in 2.00238s (17.9786GiB/s) | 20260506 22:20:08.820 | INFO | Downloaded 34GiB to GPU in 1.91755s (17.731GiB/s) | 20260506 22:20:13.971 | INFO | Analyzing chunk frames = [4893184,4893696) | 20260506 22:20:13.971 | INFO | MemInfo: { | 20260506 22:20:13.971 | INFO | "gpu" : " 17 GB", | 20260506 22:20:13.971 | INFO | "huge" : "", | 20260506 22:20:13.971 | INFO | "huge_shared" : " 27 GB", | 20260506 22:20:13.971 | INFO | "malloc" : " 14 GB", | 20260506 22:20:13.971 | INFO | "pinned" : " 29 GB" | 20260506 22:20:13.971 | INFO | } | 20260506 22:20:13.971 | INFO | Perf report for the last 5.15192 seconds: | 20260506 22:20:13.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:20:13.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:13.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:13.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:13.971 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 22:20:13.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:13.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:13.971 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 22:20:13.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:13.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:13.971 | INFO | Uploaded 36GiB to GPU in 2.01585s (17.8585GiB/s) | 20260506 22:20:13.972 | INFO | Downloaded 34GiB to GPU in 1.95349s (17.4047GiB/s) | 20260506 22:20:19.101 | INFO | Analyzing chunk frames = [4893696,4894208) | 20260506 22:20:19.101 | INFO | MemInfo: { | 20260506 22:20:19.101 | INFO | "gpu" : " 17 GB", | 20260506 22:20:19.101 | INFO | "huge" : "", | 20260506 22:20:19.101 | INFO | "huge_shared" : " 27 GB", | 20260506 22:20:19.101 | INFO | "malloc" : " 14 GB", | 20260506 22:20:19.101 | INFO | "pinned" : " 29 GB" | 20260506 22:20:19.101 | INFO | } | 20260506 22:20:19.101 | INFO | Perf report for the last 5.1298 seconds: | 20260506 22:20:19.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:20:19.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:19.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:19.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:19.101 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 22:20:19.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:19.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:19.101 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:20:19.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:19.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:19.101 | INFO | Uploaded 36GiB to GPU in 1.9528s (18.4351GiB/s) | 20260506 22:20:19.102 | INFO | Downloaded 34GiB to GPU in 1.99292s (17.0604GiB/s) | 20260506 22:20:24.242 | INFO | Analyzing chunk frames = [4894208,4894720) | 20260506 22:20:24.242 | INFO | MemInfo: { | 20260506 22:20:24.242 | INFO | "gpu" : " 17 GB", | 20260506 22:20:24.242 | INFO | "huge" : "", | 20260506 22:20:24.242 | INFO | "huge_shared" : " 27 GB", | 20260506 22:20:24.242 | INFO | "malloc" : " 14 GB", | 20260506 22:20:24.242 | INFO | "pinned" : " 29 GB" | 20260506 22:20:24.242 | INFO | } | 20260506 22:20:24.242 | INFO | Perf report for the last 5.14079 seconds: | 20260506 22:20:24.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:20:24.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:24.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:24.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:24.242 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 22:20:24.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:24.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:24.242 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:20:24.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:24.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:24.242 | INFO | Uploaded 36GiB to GPU in 2.01416s (17.8735GiB/s) | 20260506 22:20:24.242 | INFO | Downloaded 34GiB to GPU in 1.9939s (17.052GiB/s) | 20260506 22:20:24.341 | INFO | Starting a round of Baz IO | 20260506 22:20:27.473 | INFO | Populating RTMetrics report | 20260506 22:20:27.474 | INFO | RTMetrics report populated | 20260506 22:20:27.474 | INFO | Opening tmp RTMetrics json file | 20260506 22:20:27.474 | INFO | Serializing RTMetrics to json | 20260506 22:20:27.475 | INFO | Serialization done, swapping tmp file now... | 20260506 22:20:27.476 | INFO | RTMetrics done | 20260506 22:20:27.569 | INFO | Finished a round of Baz IO: Wrote 1.917 GiB in 3.228 seconds | 20260506 22:20:27.569 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.71% / 0.00% / 66.29% | 20260506 22:20:27.569 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.516387% / 0% / 5.16574e-05% | 20260506 22:20:27.569 | INFO | Baz IO overall: | 20260506 22:20:27.569 | INFO | BytesWritten : 2572395114565 | 20260506 22:20:27.569 | INFO | NumEvents : 1241254470699 | 20260506 22:20:29.379 | INFO | Analyzing chunk frames = [4894720,4895232) | 20260506 22:20:29.379 | INFO | MemInfo: { | 20260506 22:20:29.379 | INFO | "gpu" : " 17 GB", | 20260506 22:20:29.379 | INFO | "huge" : "", | 20260506 22:20:29.379 | INFO | "huge_shared" : " 27 GB", | 20260506 22:20:29.379 | INFO | "malloc" : " 14 GB", | 20260506 22:20:29.379 | INFO | "pinned" : " 29 GB" | 20260506 22:20:29.379 | INFO | } | 20260506 22:20:29.379 | INFO | Perf report for the last 5.13743 seconds: | 20260506 22:20:29.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:20:29.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:29.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:29.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:29.379 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 22:20:29.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:29.379 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 22:20:29.379 | INFO | PRE_HQ : 59.9%, 1.8, 57.1ms, 0 | 20260506 22:20:29.379 | INFO | BAZWRITER : 62.4%, 3.4, 110.6ms, 0 | 20260506 22:20:29.379 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:20:29.380 | INFO | Uploaded 36GiB to GPU in 2.05407s (17.5262GiB/s) | 20260506 22:20:29.380 | INFO | Downloaded 38GiB to GPU in 2.17422s (17.4775GiB/s) | 20260506 22:20:34.517 | INFO | Analyzing chunk frames = [4895232,4895744) | 20260506 22:20:34.517 | INFO | MemInfo: { | 20260506 22:20:34.517 | INFO | "gpu" : " 17 GB", | 20260506 22:20:34.517 | INFO | "huge" : "", | 20260506 22:20:34.517 | INFO | "huge_shared" : " 27 GB", | 20260506 22:20:34.517 | INFO | "malloc" : " 14 GB", | 20260506 22:20:34.517 | INFO | "pinned" : " 29 GB" | 20260506 22:20:34.517 | INFO | } | 20260506 22:20:34.517 | INFO | Perf report for the last 5.13807 seconds: | 20260506 22:20:34.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:20:34.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:34.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:34.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:34.518 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 22:20:34.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:34.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:34.518 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:20:34.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:34.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:34.518 | INFO | Uploaded 36GiB to GPU in 2.01721s (17.8464GiB/s) | 20260506 22:20:34.518 | INFO | Downloaded 34GiB to GPU in 1.93099s (17.6076GiB/s) | 20260506 22:20:39.653 | INFO | Analyzing chunk frames = [4895744,4896256) | 20260506 22:20:39.653 | INFO | MemInfo: { | 20260506 22:20:39.653 | INFO | "gpu" : " 17 GB", | 20260506 22:20:39.653 | INFO | "huge" : "", | 20260506 22:20:39.653 | INFO | "huge_shared" : " 27 GB", | 20260506 22:20:39.653 | INFO | "malloc" : " 14 GB", | 20260506 22:20:39.653 | INFO | "pinned" : " 29 GB" | 20260506 22:20:39.653 | INFO | } | 20260506 22:20:39.653 | INFO | Perf report for the last 5.13598 seconds: | 20260506 22:20:39.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:20:39.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:39.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:39.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:39.653 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260506 22:20:39.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:39.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:39.654 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:20:39.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:39.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:39.654 | INFO | Uploaded 36GiB to GPU in 1.99181s (18.074GiB/s) | 20260506 22:20:39.654 | INFO | Downloaded 34GiB to GPU in 1.92745s (17.6399GiB/s) | 20260506 22:20:44.795 | INFO | Analyzing chunk frames = [4896256,4896768) | 20260506 22:20:44.795 | INFO | MemInfo: { | 20260506 22:20:44.795 | INFO | "gpu" : " 17 GB", | 20260506 22:20:44.795 | INFO | "huge" : "", | 20260506 22:20:44.795 | INFO | "huge_shared" : " 27 GB", | 20260506 22:20:44.795 | INFO | "malloc" : " 14 GB", | 20260506 22:20:44.795 | INFO | "pinned" : " 29 GB" | 20260506 22:20:44.795 | INFO | } | 20260506 22:20:44.795 | INFO | Perf report for the last 5.14158 seconds: | 20260506 22:20:44.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:20:44.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:44.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:44.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:44.795 | INFO | ANALYSIS : 59.6%, 3.0, 92.3ms, 0 | 20260506 22:20:44.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:44.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:44.795 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 22:20:44.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:44.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:44.795 | INFO | Uploaded 36GiB to GPU in 1.94979s (18.4636GiB/s) | 20260506 22:20:44.795 | INFO | Downloaded 34GiB to GPU in 1.87694s (18.1146GiB/s) | 20260506 22:20:49.936 | INFO | Analyzing chunk frames = [4896768,4897280) | 20260506 22:20:49.936 | INFO | MemInfo: { | 20260506 22:20:49.936 | INFO | "gpu" : " 17 GB", | 20260506 22:20:49.936 | INFO | "huge" : "", | 20260506 22:20:49.936 | INFO | "huge_shared" : " 27 GB", | 20260506 22:20:49.936 | INFO | "malloc" : " 14 GB", | 20260506 22:20:49.936 | INFO | "pinned" : " 29 GB" | 20260506 22:20:49.936 | INFO | } | 20260506 22:20:49.936 | INFO | Perf report for the last 5.14116 seconds: | 20260506 22:20:49.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:20:49.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:49.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:49.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:49.936 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260506 22:20:49.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:49.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:49.936 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:20:49.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:49.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:49.936 | INFO | Uploaded 36GiB to GPU in 1.97314s (18.245GiB/s) | 20260506 22:20:49.936 | INFO | Downloaded 34GiB to GPU in 1.92376s (17.6738GiB/s) | 20260506 22:20:55.081 | INFO | Analyzing chunk frames = [4897280,4897792) | 20260506 22:20:55.081 | INFO | MemInfo: { | 20260506 22:20:55.081 | INFO | "gpu" : " 17 GB", | 20260506 22:20:55.081 | INFO | "huge" : "", | 20260506 22:20:55.081 | INFO | "huge_shared" : " 27 GB", | 20260506 22:20:55.081 | INFO | "malloc" : " 14 GB", | 20260506 22:20:55.081 | INFO | "pinned" : " 29 GB" | 20260506 22:20:55.081 | INFO | } | 20260506 22:20:55.081 | INFO | Perf report for the last 5.14512 seconds: | 20260506 22:20:55.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:20:55.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:55.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:55.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:55.081 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 22:20:55.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:55.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:55.081 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 22:20:55.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:55.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:20:55.082 | INFO | Uploaded 36GiB to GPU in 2.02163s (17.8074GiB/s) | 20260506 22:20:55.082 | INFO | Downloaded 34GiB to GPU in 1.93792s (17.5445GiB/s) | 20260506 22:21:00.228 | INFO | Analyzing chunk frames = [4897792,4898304) | 20260506 22:21:00.229 | INFO | MemInfo: { | 20260506 22:21:00.229 | INFO | "gpu" : " 17 GB", | 20260506 22:21:00.229 | INFO | "huge" : "", | 20260506 22:21:00.229 | INFO | "huge_shared" : " 27 GB", | 20260506 22:21:00.229 | INFO | "malloc" : " 14 GB", | 20260506 22:21:00.229 | INFO | "pinned" : " 29 GB" | 20260506 22:21:00.229 | INFO | } | 20260506 22:21:00.229 | INFO | Perf report for the last 5.14735 seconds: | 20260506 22:21:00.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:21:00.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:00.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:00.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:00.229 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 22:21:00.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:00.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:00.229 | INFO | PRE_HQ : 27.2%, 1.0, 14.2ms, 0 | 20260506 22:21:00.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:00.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:00.229 | INFO | Uploaded 36GiB to GPU in 1.97501s (18.2277GiB/s) | 20260506 22:21:00.229 | INFO | Downloaded 34GiB to GPU in 1.99442s (17.0476GiB/s) | 20260506 22:21:05.354 | INFO | Analyzing chunk frames = [4898304,4898816) | 20260506 22:21:05.354 | INFO | MemInfo: { | 20260506 22:21:05.354 | INFO | "gpu" : " 17 GB", | 20260506 22:21:05.354 | INFO | "huge" : "", | 20260506 22:21:05.354 | INFO | "huge_shared" : " 27 GB", | 20260506 22:21:05.354 | INFO | "malloc" : " 14 GB", | 20260506 22:21:05.355 | INFO | "pinned" : " 29 GB" | 20260506 22:21:05.355 | INFO | } | 20260506 22:21:05.355 | INFO | Perf report for the last 5.12589 seconds: | 20260506 22:21:05.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:21:05.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:05.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:05.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:05.355 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260506 22:21:05.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:05.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:05.355 | INFO | PRE_HQ : 30.4%, 1.0, 15.7ms, 0 | 20260506 22:21:05.355 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:05.355 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:05.355 | INFO | Uploaded 36GiB to GPU in 2.00067s (17.9939GiB/s) | 20260506 22:21:05.355 | INFO | Downloaded 34GiB to GPU in 1.99079s (17.0786GiB/s) | 20260506 22:21:05.355 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:21:05.355 | INFO | Total mem usage: 18 GiB | 20260506 22:21:05.355 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:21:05.355 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:21:05.355 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:21:05.355 | INFO | High Water Breakdown: | 20260506 22:21:05.355 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:21:05.355 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:21:05.355 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:21:05.355 | INFO | ------------------------------------------ | 20260506 22:21:05.355 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:21:05.355 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:21:05.355 | INFO | Total mem usage: 26.5 GiB | 20260506 22:21:05.355 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:21:05.355 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:21:05.355 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:21:05.355 | INFO | High Water Breakdown: | 20260506 22:21:05.355 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:21:05.355 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:21:05.355 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:21:05.355 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:21:05.355 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:21:05.355 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:21:05.355 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:21:05.355 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:21:05.355 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:21:05.355 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:21:05.355 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:21:05.355 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:21:05.355 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:21:05.355 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:21:05.355 | INFO | ------------------------------------------ | 20260506 22:21:05.355 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:21:05.355 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:21:05.355 | INFO | Total mem usage: 25.6 GiB | 20260506 22:21:05.355 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:21:05.355 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:21:05.355 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:21:05.355 | INFO | High Water Breakdown: | 20260506 22:21:05.355 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:21:05.355 | INFO | ------------------------------------------ | 20260506 22:21:05.355 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:21:05.355 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:21:05.355 | INFO | Total mem usage: 15.7 GiB | 20260506 22:21:05.355 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:21:05.355 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:21:05.355 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:21:05.355 | INFO | High Water Breakdown: | 20260506 22:21:05.355 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:21:05.355 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:21:05.355 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:21:05.355 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:21:05.355 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:21:05.356 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:21:05.356 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:21:05.356 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:21:05.356 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:21:05.356 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:21:05.356 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:21:05.356 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:21:05.356 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:21:05.356 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:21:05.356 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:21:05.356 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:21:05.356 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:21:05.356 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:21:05.356 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:21:05.356 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:21:05.356 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:21:05.356 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:21:05.356 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:21:05.356 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:21:05.356 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:21:05.356 | INFO | ------------------------------------------ | 20260506 22:21:05.356 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:21:05.356 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:21:05.356 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:21:05.356 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:21:05.356 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.59e+01, 1.35e+01 / 2.57e+00 | 20260506 22:21:05.356 | INFO | PerfReport: 0.30%, AggregateStats -- 9.16e-02 / 1.01e-01, 1.01e-02 / 5.24e-03 | 20260506 22:21:05.356 | INFO | PerfReport: 11.67%, Baseline -- 3.62e+00 / 4.40e+00, 5.44e-01 / 3.55e-01 | 20260506 22:21:05.356 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.79e-01 / 1.46e-01 | 20260506 22:21:05.356 | INFO | PerfReport: 16.11%, DME -- 4.00e+01 / 4.50e+01, 4.19e+00 / 2.83e+00 | 20260506 22:21:05.356 | INFO | PerfReport: 57.21%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.28e+00 / 1.49e+00 | 20260506 22:21:05.356 | INFO | PerfReport: 7.12%, PulseAccumulating -- 2.21e+00 / 2.79e+00, 3.83e-01 / 1.91e-01 | 20260506 22:21:05.356 | INFO | PerfReport: 3.23%, Metrics -- 1.00e+00 / 1.35e+00, 2.61e-01 / 1.44e-01 | 20260506 22:21:05.356 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.43e-01 / 3.91e-01, 5.28e-02 / 1.96e-02 | 20260506 22:21:05.356 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:21:05.356 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:21:05.356 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:21:05.356 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 22:21:05.356 | INFO | PerfReport: 21.49%, Upload -- 2.08e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260506 22:21:05.356 | INFO | PerfReport: 21.20%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 22:21:05.356 | INFO | PerfReport: 32.07%, Compute -- 3.10e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 22:21:05.356 | INFO | PerfReport: 1.34%, LockUpload -- 1.30e+00 / 0.00e+00, 4.98e+00 / 0.00e+00 | 20260506 22:21:05.356 | INFO | PerfReport: 1.24%, LockDownload -- 1.20e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260506 22:21:05.356 | INFO | PerfReport: 22.65%, LockCompute -- 2.19e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 22:21:05.356 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260506 22:21:05.459 | INFO | Starting a round of Baz IO | 20260506 22:21:08.598 | INFO | Populating RTMetrics report | 20260506 22:21:08.599 | INFO | RTMetrics report populated | 20260506 22:21:08.599 | INFO | Opening tmp RTMetrics json file | 20260506 22:21:08.599 | INFO | Serializing RTMetrics to json | 20260506 22:21:08.600 | INFO | Serialization done, swapping tmp file now... | 20260506 22:21:08.601 | INFO | RTMetrics done | 20260506 22:21:08.695 | INFO | Finished a round of Baz IO: Wrote 1.915 GiB in 3.236 seconds | 20260506 22:21:08.695 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.70% / 0.00% / 66.29% | 20260506 22:21:08.695 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.513625% / 0% / 6.75519e-05% | 20260506 22:21:08.695 | INFO | Baz IO overall: | 20260506 22:21:08.696 | INFO | BytesWritten : 2574451119874 | 20260506 22:21:08.696 | INFO | NumEvents : 1242236120974 | 20260506 22:21:10.493 | INFO | Analyzing chunk frames = [4898816,4899328) | 20260506 22:21:10.493 | INFO | MemInfo: { | 20260506 22:21:10.493 | INFO | "gpu" : " 17 GB", | 20260506 22:21:10.493 | INFO | "huge" : "", | 20260506 22:21:10.493 | INFO | "huge_shared" : " 27 GB", | 20260506 22:21:10.493 | INFO | "malloc" : " 14 GB", | 20260506 22:21:10.493 | INFO | "pinned" : " 29 GB" | 20260506 22:21:10.493 | INFO | } | 20260506 22:21:10.493 | INFO | Perf report for the last 5.13822 seconds: | 20260506 22:21:10.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:21:10.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:10.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:10.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:10.493 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260506 22:21:10.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:10.493 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 22:21:10.493 | INFO | PRE_HQ : 60.5%, 1.9, 60.1ms, 0 | 20260506 22:21:10.493 | INFO | BAZWRITER : 62.6%, 3.6, 115.7ms, 0 | 20260506 22:21:10.493 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 22:21:10.493 | INFO | Uploaded 36GiB to GPU in 2.02313s (17.7942GiB/s) | 20260506 22:21:10.493 | INFO | Downloaded 38GiB to GPU in 2.17972s (17.4334GiB/s) | 20260506 22:21:15.631 | INFO | Analyzing chunk frames = [4899328,4899840) | 20260506 22:21:15.631 | INFO | MemInfo: { | 20260506 22:21:15.631 | INFO | "gpu" : " 17 GB", | 20260506 22:21:15.631 | INFO | "huge" : "", | 20260506 22:21:15.631 | INFO | "huge_shared" : " 27 GB", | 20260506 22:21:15.631 | INFO | "malloc" : " 14 GB", | 20260506 22:21:15.631 | INFO | "pinned" : " 29 GB" | 20260506 22:21:15.631 | INFO | } | 20260506 22:21:15.631 | INFO | Perf report for the last 5.13802 seconds: | 20260506 22:21:15.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:21:15.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:15.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:15.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:15.631 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:21:15.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:15.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:15.631 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 22:21:15.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:15.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:15.631 | INFO | Uploaded 36GiB to GPU in 2.01003s (17.9102GiB/s) | 20260506 22:21:15.631 | INFO | Downloaded 34GiB to GPU in 1.92824s (17.6327GiB/s) | 20260506 22:21:20.769 | INFO | Analyzing chunk frames = [4899840,4900352) | 20260506 22:21:20.770 | INFO | MemInfo: { | 20260506 22:21:20.770 | INFO | "gpu" : " 17 GB", | 20260506 22:21:20.770 | INFO | "huge" : "", | 20260506 22:21:20.770 | INFO | "huge_shared" : " 27 GB", | 20260506 22:21:20.770 | INFO | "malloc" : " 14 GB", | 20260506 22:21:20.770 | INFO | "pinned" : " 29 GB" | 20260506 22:21:20.770 | INFO | } | 20260506 22:21:20.770 | INFO | Perf report for the last 5.13878 seconds: | 20260506 22:21:20.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:21:20.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:20.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:20.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:20.770 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 22:21:20.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:20.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:20.770 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:21:20.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:20.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:20.770 | INFO | Uploaded 36GiB to GPU in 1.98891s (18.1003GiB/s) | 20260506 22:21:20.770 | INFO | Downloaded 34GiB to GPU in 1.92288s (17.6818GiB/s) | 20260506 22:21:25.911 | INFO | Analyzing chunk frames = [4900352,4900864) | 20260506 22:21:25.911 | INFO | MemInfo: { | 20260506 22:21:25.911 | INFO | "gpu" : " 17 GB", | 20260506 22:21:25.911 | INFO | "huge" : "", | 20260506 22:21:25.911 | INFO | "huge_shared" : " 27 GB", | 20260506 22:21:25.911 | INFO | "malloc" : " 14 GB", | 20260506 22:21:25.911 | INFO | "pinned" : " 29 GB" | 20260506 22:21:25.911 | INFO | } | 20260506 22:21:25.911 | INFO | Perf report for the last 5.14177 seconds: | 20260506 22:21:25.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:21:25.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:25.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:25.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:25.911 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 22:21:25.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:25.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:25.911 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:21:25.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:25.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:25.912 | INFO | Uploaded 36GiB to GPU in 1.94426s (18.5161GiB/s) | 20260506 22:21:25.912 | INFO | Downloaded 34GiB to GPU in 1.87413s (18.1417GiB/s) | 20260506 22:21:31.052 | INFO | Analyzing chunk frames = [4900864,4901376) | 20260506 22:21:31.052 | INFO | MemInfo: { | 20260506 22:21:31.052 | INFO | "gpu" : " 17 GB", | 20260506 22:21:31.052 | INFO | "huge" : "", | 20260506 22:21:31.052 | INFO | "huge_shared" : " 27 GB", | 20260506 22:21:31.052 | INFO | "malloc" : " 14 GB", | 20260506 22:21:31.052 | INFO | "pinned" : " 29 GB" | 20260506 22:21:31.052 | INFO | } | 20260506 22:21:31.053 | INFO | Perf report for the last 5.14108 seconds: | 20260506 22:21:31.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:21:31.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:31.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:31.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:31.053 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 22:21:31.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:31.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:31.053 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:21:31.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:31.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:31.053 | INFO | Uploaded 36GiB to GPU in 2.00481s (17.9568GiB/s) | 20260506 22:21:31.053 | INFO | Downloaded 34GiB to GPU in 1.91807s (17.7262GiB/s) | 20260506 22:21:36.190 | INFO | Analyzing chunk frames = [4901376,4901888) | 20260506 22:21:36.190 | INFO | MemInfo: { | 20260506 22:21:36.190 | INFO | "gpu" : " 17 GB", | 20260506 22:21:36.190 | INFO | "huge" : "", | 20260506 22:21:36.190 | INFO | "huge_shared" : " 27 GB", | 20260506 22:21:36.190 | INFO | "malloc" : " 14 GB", | 20260506 22:21:36.190 | INFO | "pinned" : " 29 GB" | 20260506 22:21:36.190 | INFO | } | 20260506 22:21:36.190 | INFO | Perf report for the last 5.13792 seconds: | 20260506 22:21:36.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:21:36.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:36.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:36.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:36.190 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260506 22:21:36.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:36.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:36.191 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:21:36.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:36.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:36.191 | INFO | Uploaded 36GiB to GPU in 2.02843s (17.7477GiB/s) | 20260506 22:21:36.191 | INFO | Downloaded 34GiB to GPU in 1.94534s (17.4777GiB/s) | 20260506 22:21:41.331 | INFO | Analyzing chunk frames = [4901888,4902400) | 20260506 22:21:41.331 | INFO | MemInfo: { | 20260506 22:21:41.331 | INFO | "gpu" : " 17 GB", | 20260506 22:21:41.331 | INFO | "huge" : "", | 20260506 22:21:41.331 | INFO | "huge_shared" : " 27 GB", | 20260506 22:21:41.331 | INFO | "malloc" : " 14 GB", | 20260506 22:21:41.331 | INFO | "pinned" : " 29 GB" | 20260506 22:21:41.331 | INFO | } | 20260506 22:21:41.331 | INFO | Perf report for the last 5.14027 seconds: | 20260506 22:21:41.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:21:41.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:41.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:41.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:41.331 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 22:21:41.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:41.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:41.331 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:21:41.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:41.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:41.331 | INFO | Uploaded 36GiB to GPU in 1.98136s (18.1694GiB/s) | 20260506 22:21:41.331 | INFO | Downloaded 34GiB to GPU in 2.00522s (16.9557GiB/s) | 20260506 22:21:46.469 | INFO | Analyzing chunk frames = [4902400,4902912) | 20260506 22:21:46.469 | INFO | MemInfo: { | 20260506 22:21:46.469 | INFO | "gpu" : " 17 GB", | 20260506 22:21:46.469 | INFO | "huge" : "", | 20260506 22:21:46.469 | INFO | "huge_shared" : " 27 GB", | 20260506 22:21:46.469 | INFO | "malloc" : " 14 GB", | 20260506 22:21:46.469 | INFO | "pinned" : " 29 GB" | 20260506 22:21:46.469 | INFO | } | 20260506 22:21:46.469 | INFO | Perf report for the last 5.13871 seconds: | 20260506 22:21:46.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:21:46.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:46.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:46.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:46.469 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 22:21:46.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:46.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:46.470 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 22:21:46.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:46.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:46.470 | INFO | Uploaded 36GiB to GPU in 2.02233s (17.8013GiB/s) | 20260506 22:21:46.470 | INFO | Downloaded 34GiB to GPU in 1.99348s (17.0556GiB/s) | 20260506 22:21:46.574 | INFO | Starting a round of Baz IO | 20260506 22:21:49.708 | INFO | Populating RTMetrics report | 20260506 22:21:49.708 | INFO | RTMetrics report populated | 20260506 22:21:49.708 | INFO | Opening tmp RTMetrics json file | 20260506 22:21:49.708 | INFO | Serializing RTMetrics to json | 20260506 22:21:49.710 | INFO | Serialization done, swapping tmp file now... | 20260506 22:21:49.710 | INFO | RTMetrics done | 20260506 22:21:49.805 | INFO | Finished a round of Baz IO: Wrote 1.914 GiB in 3.23 seconds | 20260506 22:21:49.805 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.70% / 0.00% / 66.30% | 20260506 22:21:49.805 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.511579% / 7.94729e-06% / 6.35783e-05% | 20260506 22:21:49.805 | INFO | Baz IO overall: | 20260506 22:21:49.805 | INFO | BytesWritten : 2576506568856 | 20260506 22:21:49.805 | INFO | NumEvents : 1243217222166 | 20260506 22:21:51.609 | INFO | Analyzing chunk frames = [4902912,4903424) | 20260506 22:21:51.609 | INFO | MemInfo: { | 20260506 22:21:51.609 | INFO | "gpu" : " 17 GB", | 20260506 22:21:51.609 | INFO | "huge" : "", | 20260506 22:21:51.609 | INFO | "huge_shared" : " 27 GB", | 20260506 22:21:51.609 | INFO | "malloc" : " 14 GB", | 20260506 22:21:51.609 | INFO | "pinned" : " 29 GB" | 20260506 22:21:51.609 | INFO | } | 20260506 22:21:51.609 | INFO | Perf report for the last 5.13928 seconds: | 20260506 22:21:51.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:21:51.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:51.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:51.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:51.609 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260506 22:21:51.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:51.609 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260506 22:21:51.609 | INFO | PRE_HQ : 59.9%, 1.9, 58.6ms, 0 | 20260506 22:21:51.609 | INFO | BAZWRITER : 62.5%, 3.5, 111.0ms, 0 | 20260506 22:21:51.609 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:21:51.609 | INFO | Uploaded 36GiB to GPU in 2.03103s (17.725GiB/s) | 20260506 22:21:51.609 | INFO | Downloaded 38GiB to GPU in 2.18063s (17.4262GiB/s) | 20260506 22:21:56.745 | INFO | Analyzing chunk frames = [4903424,4903936) | 20260506 22:21:56.745 | INFO | MemInfo: { | 20260506 22:21:56.745 | INFO | "gpu" : " 17 GB", | 20260506 22:21:56.745 | INFO | "huge" : "", | 20260506 22:21:56.745 | INFO | "huge_shared" : " 27 GB", | 20260506 22:21:56.745 | INFO | "malloc" : " 14 GB", | 20260506 22:21:56.745 | INFO | "pinned" : " 29 GB" | 20260506 22:21:56.745 | INFO | } | 20260506 22:21:56.745 | INFO | Perf report for the last 5.13599 seconds: | 20260506 22:21:56.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:21:56.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:56.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:56.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:56.745 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:21:56.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:56.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:56.745 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:21:56.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:56.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:21:56.745 | INFO | Uploaded 36GiB to GPU in 2.00447s (17.9599GiB/s) | 20260506 22:21:56.745 | INFO | Downloaded 34GiB to GPU in 1.9282s (17.633GiB/s) | 20260506 22:22:01.887 | INFO | Analyzing chunk frames = [4903936,4904448) | 20260506 22:22:01.887 | INFO | MemInfo: { | 20260506 22:22:01.887 | INFO | "gpu" : " 17 GB", | 20260506 22:22:01.887 | INFO | "huge" : "", | 20260506 22:22:01.887 | INFO | "huge_shared" : " 27 GB", | 20260506 22:22:01.887 | INFO | "malloc" : " 14 GB", | 20260506 22:22:01.887 | INFO | "pinned" : " 29 GB" | 20260506 22:22:01.887 | INFO | } | 20260506 22:22:01.887 | INFO | Perf report for the last 5.14215 seconds: | 20260506 22:22:01.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:22:01.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:01.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:01.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:01.887 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 22:22:01.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:01.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:01.887 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:22:01.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:01.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:01.887 | INFO | Uploaded 36GiB to GPU in 1.99347s (18.059GiB/s) | 20260506 22:22:01.887 | INFO | Downloaded 34GiB to GPU in 1.9259s (17.6541GiB/s) | 20260506 22:22:07.031 | INFO | Analyzing chunk frames = [4904448,4904960) | 20260506 22:22:07.031 | INFO | MemInfo: { | 20260506 22:22:07.031 | INFO | "gpu" : " 17 GB", | 20260506 22:22:07.032 | INFO | "huge" : "", | 20260506 22:22:07.032 | INFO | "huge_shared" : " 27 GB", | 20260506 22:22:07.032 | INFO | "malloc" : " 14 GB", | 20260506 22:22:07.032 | INFO | "pinned" : " 29 GB" | 20260506 22:22:07.032 | INFO | } | 20260506 22:22:07.032 | INFO | Perf report for the last 5.14471 seconds: | 20260506 22:22:07.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:22:07.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:07.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:07.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:07.032 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 22:22:07.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:07.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:07.032 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:22:07.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:07.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:07.032 | INFO | Uploaded 36GiB to GPU in 1.96255s (18.3434GiB/s) | 20260506 22:22:07.032 | INFO | Downloaded 34GiB to GPU in 1.87761s (18.1081GiB/s) | 20260506 22:22:12.169 | INFO | Analyzing chunk frames = [4904960,4905472) | 20260506 22:22:12.169 | INFO | MemInfo: { | 20260506 22:22:12.169 | INFO | "gpu" : " 17 GB", | 20260506 22:22:12.169 | INFO | "huge" : "", | 20260506 22:22:12.169 | INFO | "huge_shared" : " 27 GB", | 20260506 22:22:12.169 | INFO | "malloc" : " 14 GB", | 20260506 22:22:12.169 | INFO | "pinned" : " 29 GB" | 20260506 22:22:12.169 | INFO | } | 20260506 22:22:12.169 | INFO | Perf report for the last 5.13708 seconds: | 20260506 22:22:12.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:22:12.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:12.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:12.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:12.169 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 22:22:12.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:12.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:12.169 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:22:12.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:12.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:12.169 | INFO | Uploaded 36GiB to GPU in 2.01182s (17.8942GiB/s) | 20260506 22:22:12.169 | INFO | Downloaded 34GiB to GPU in 1.93305s (17.5887GiB/s) | 20260506 22:22:17.312 | INFO | Analyzing chunk frames = [4905472,4905984) | 20260506 22:22:17.312 | INFO | MemInfo: { | 20260506 22:22:17.312 | INFO | "gpu" : " 17 GB", | 20260506 22:22:17.312 | INFO | "huge" : "", | 20260506 22:22:17.312 | INFO | "huge_shared" : " 27 GB", | 20260506 22:22:17.312 | INFO | "malloc" : " 14 GB", | 20260506 22:22:17.312 | INFO | "pinned" : " 29 GB" | 20260506 22:22:17.312 | INFO | } | 20260506 22:22:17.312 | INFO | Perf report for the last 5.14299 seconds: | 20260506 22:22:17.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:22:17.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:17.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:17.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:17.312 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 22:22:17.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:17.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:17.312 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:22:17.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:17.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:17.312 | INFO | Uploaded 36GiB to GPU in 2.03258s (17.7115GiB/s) | 20260506 22:22:17.312 | INFO | Downloaded 34GiB to GPU in 1.95441s (17.3965GiB/s) | 20260506 22:22:22.445 | INFO | Analyzing chunk frames = [4905984,4906496) | 20260506 22:22:22.445 | INFO | MemInfo: { | 20260506 22:22:22.445 | INFO | "gpu" : " 17 GB", | 20260506 22:22:22.445 | INFO | "huge" : "", | 20260506 22:22:22.445 | INFO | "huge_shared" : " 27 GB", | 20260506 22:22:22.445 | INFO | "malloc" : " 14 GB", | 20260506 22:22:22.445 | INFO | "pinned" : " 29 GB" | 20260506 22:22:22.445 | INFO | } | 20260506 22:22:22.445 | INFO | Perf report for the last 5.13334 seconds: | 20260506 22:22:22.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:22:22.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:22.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:22.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:22.445 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 22:22:22.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:22.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:22.445 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 22:22:22.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:22.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:22.445 | INFO | Uploaded 36GiB to GPU in 1.96098s (18.3581GiB/s) | 20260506 22:22:22.445 | INFO | Downloaded 34GiB to GPU in 1.98517s (17.127GiB/s) | 20260506 22:22:27.583 | INFO | Analyzing chunk frames = [4906496,4907008) | 20260506 22:22:27.583 | INFO | MemInfo: { | 20260506 22:22:27.583 | INFO | "gpu" : " 17 GB", | 20260506 22:22:27.583 | INFO | "huge" : "", | 20260506 22:22:27.584 | INFO | "huge_shared" : " 27 GB", | 20260506 22:22:27.584 | INFO | "malloc" : " 14 GB", | 20260506 22:22:27.584 | INFO | "pinned" : " 29 GB" | 20260506 22:22:27.584 | INFO | } | 20260506 22:22:27.584 | INFO | Perf report for the last 5.13849 seconds: | 20260506 22:22:27.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:22:27.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:27.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:27.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:27.584 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 22:22:27.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:27.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:27.584 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 22:22:27.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:27.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:27.584 | INFO | Uploaded 36GiB to GPU in 2.03222s (17.7146GiB/s) | 20260506 22:22:27.584 | INFO | Downloaded 34GiB to GPU in 2.00088s (16.9926GiB/s) | 20260506 22:22:27.584 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:22:27.584 | INFO | Total mem usage: 18 GiB | 20260506 22:22:27.584 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:22:27.584 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:22:27.584 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:22:27.584 | INFO | High Water Breakdown: | 20260506 22:22:27.584 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:22:27.584 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:22:27.584 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:22:27.584 | INFO | ------------------------------------------ | 20260506 22:22:27.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:22:27.584 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:22:27.584 | INFO | Total mem usage: 26.5 GiB | 20260506 22:22:27.584 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:22:27.584 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:22:27.584 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:22:27.584 | INFO | High Water Breakdown: | 20260506 22:22:27.584 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:22:27.584 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:22:27.584 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:22:27.584 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:22:27.584 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:22:27.584 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:22:27.584 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:22:27.584 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:22:27.584 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:22:27.584 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:22:27.584 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:22:27.584 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:22:27.584 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:22:27.584 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:22:27.584 | INFO | ------------------------------------------ | 20260506 22:22:27.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:22:27.584 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:22:27.584 | INFO | Total mem usage: 25.6 GiB | 20260506 22:22:27.584 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:22:27.584 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:22:27.584 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:22:27.584 | INFO | High Water Breakdown: | 20260506 22:22:27.584 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:22:27.584 | INFO | ------------------------------------------ | 20260506 22:22:27.584 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:22:27.584 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:22:27.584 | INFO | Total mem usage: 15.7 GiB | 20260506 22:22:27.584 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:22:27.584 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:22:27.584 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:22:27.584 | INFO | High Water Breakdown: | 20260506 22:22:27.584 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:22:27.584 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:22:27.584 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:22:27.584 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:22:27.584 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:22:27.584 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:22:27.584 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:22:27.584 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:22:27.584 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:22:27.584 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:22:27.584 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:22:27.584 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:22:27.584 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:22:27.584 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:22:27.584 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:22:27.584 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:22:27.584 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:22:27.585 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:22:27.585 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:22:27.585 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:22:27.585 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:22:27.585 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:22:27.585 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:22:27.585 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:22:27.585 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:22:27.585 | INFO | ------------------------------------------ | 20260506 22:22:27.585 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:22:27.585 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:22:27.585 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:22:27.585 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:22:27.585 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.55e+01, 1.34e+01 / 2.51e+00 | 20260506 22:22:27.585 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 9.94e-02, 1.05e-02 / 6.22e-03 | 20260506 22:22:27.585 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.35e+00, 5.43e-01 / 3.96e-01 | 20260506 22:22:27.585 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.44e+00, 1.77e-01 / 1.46e-01 | 20260506 22:22:27.585 | INFO | PerfReport: 16.05%, DME -- 3.99e+01 / 4.45e+01, 4.21e+00 / 3.01e+00 | 20260506 22:22:27.585 | INFO | PerfReport: 57.31%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.36e+00 / 1.88e+00 | 20260506 22:22:27.585 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.20e+00 / 2.73e+00, 3.80e-01 / 2.61e-01 | 20260506 22:22:27.585 | INFO | PerfReport: 3.23%, Metrics -- 1.00e+00 / 1.36e+00, 2.65e-01 / 1.52e-01 | 20260506 22:22:27.585 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.43e-01 / 3.93e-01, 5.28e-02 / 1.94e-02 | 20260506 22:22:27.585 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:22:27.585 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:22:27.585 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:22:27.585 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 22:22:27.585 | INFO | PerfReport: 21.55%, Upload -- 2.09e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 22:22:27.585 | INFO | PerfReport: 21.23%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 22:22:27.585 | INFO | PerfReport: 32.08%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 22:22:27.585 | INFO | PerfReport: 1.33%, LockUpload -- 1.29e+00 / 0.00e+00, 5.02e+00 / 0.00e+00 | 20260506 22:22:27.585 | INFO | PerfReport: 1.19%, LockDownload -- 1.15e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 22:22:27.585 | INFO | PerfReport: 22.60%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 22:22:27.585 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260506 22:22:27.686 | INFO | Starting a round of Baz IO | 20260506 22:22:30.820 | INFO | Populating RTMetrics report | 20260506 22:22:30.820 | INFO | RTMetrics report populated | 20260506 22:22:30.820 | INFO | Opening tmp RTMetrics json file | 20260506 22:22:30.841 | INFO | Serializing RTMetrics to json | 20260506 22:22:30.842 | INFO | Serialization done, swapping tmp file now... | 20260506 22:22:30.842 | INFO | RTMetrics done | 20260506 22:22:30.919 | INFO | Finished a round of Baz IO: Wrote 1.916 GiB in 3.233 seconds | 20260506 22:22:30.920 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.70% / 0.00% / 66.30% | 20260506 22:22:30.920 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.511591% / 7.94729e-06% / 6.75519e-05% | 20260506 22:22:30.920 | INFO | Baz IO overall: | 20260506 22:22:30.920 | INFO | BytesWritten : 2578563482735 | 20260506 22:22:30.920 | INFO | NumEvents : 1244199586136 | 20260506 22:22:32.727 | INFO | Analyzing chunk frames = [4907008,4907520) | 20260506 22:22:32.727 | INFO | MemInfo: { | 20260506 22:22:32.727 | INFO | "gpu" : " 17 GB", | 20260506 22:22:32.727 | INFO | "huge" : "", | 20260506 22:22:32.727 | INFO | "huge_shared" : " 27 GB", | 20260506 22:22:32.727 | INFO | "malloc" : " 14 GB", | 20260506 22:22:32.727 | INFO | "pinned" : " 29 GB" | 20260506 22:22:32.727 | INFO | } | 20260506 22:22:32.727 | INFO | Perf report for the last 5.14331 seconds: | 20260506 22:22:32.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:22:32.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:32.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:32.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:32.727 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260506 22:22:32.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:32.727 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 22:22:32.727 | INFO | PRE_HQ : 59.6%, 1.8, 57.2ms, 0 | 20260506 22:22:32.727 | INFO | BAZWRITER : 62.5%, 3.5, 112.0ms, 0 | 20260506 22:22:32.727 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:22:32.728 | INFO | Uploaded 36GiB to GPU in 2.03818s (17.6629GiB/s) | 20260506 22:22:32.728 | INFO | Downloaded 38GiB to GPU in 2.1795s (17.4352GiB/s) | 20260506 22:22:37.859 | INFO | Analyzing chunk frames = [4907520,4908032) | 20260506 22:22:37.859 | INFO | MemInfo: { | 20260506 22:22:37.859 | INFO | "gpu" : " 17 GB", | 20260506 22:22:37.859 | INFO | "huge" : "", | 20260506 22:22:37.859 | INFO | "huge_shared" : " 27 GB", | 20260506 22:22:37.859 | INFO | "malloc" : " 14 GB", | 20260506 22:22:37.859 | INFO | "pinned" : " 29 GB" | 20260506 22:22:37.859 | INFO | } | 20260506 22:22:37.859 | INFO | Perf report for the last 5.1322 seconds: | 20260506 22:22:37.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:22:37.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:37.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:37.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:37.859 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:22:37.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:37.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:37.859 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:22:37.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:37.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:37.859 | INFO | Uploaded 36GiB to GPU in 2.02192s (17.8048GiB/s) | 20260506 22:22:37.860 | INFO | Downloaded 34GiB to GPU in 1.92716s (17.6425GiB/s) | 20260506 22:22:43.002 | INFO | Analyzing chunk frames = [4908032,4908544) | 20260506 22:22:43.002 | INFO | MemInfo: { | 20260506 22:22:43.002 | INFO | "gpu" : " 17 GB", | 20260506 22:22:43.002 | INFO | "huge" : "", | 20260506 22:22:43.002 | INFO | "huge_shared" : " 27 GB", | 20260506 22:22:43.002 | INFO | "malloc" : " 14 GB", | 20260506 22:22:43.002 | INFO | "pinned" : " 29 GB" | 20260506 22:22:43.002 | INFO | } | 20260506 22:22:43.002 | INFO | Perf report for the last 5.1429 seconds: | 20260506 22:22:43.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:22:43.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:43.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:43.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:43.002 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 22:22:43.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:43.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:43.002 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:22:43.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:43.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:43.002 | INFO | Uploaded 36GiB to GPU in 1.9813s (18.1699GiB/s) | 20260506 22:22:43.002 | INFO | Downloaded 34GiB to GPU in 1.92112s (17.6981GiB/s) | 20260506 22:22:48.143 | INFO | Analyzing chunk frames = [4908544,4909056) | 20260506 22:22:48.143 | INFO | MemInfo: { | 20260506 22:22:48.143 | INFO | "gpu" : " 17 GB", | 20260506 22:22:48.143 | INFO | "huge" : "", | 20260506 22:22:48.143 | INFO | "huge_shared" : " 27 GB", | 20260506 22:22:48.143 | INFO | "malloc" : " 14 GB", | 20260506 22:22:48.143 | INFO | "pinned" : " 29 GB" | 20260506 22:22:48.143 | INFO | } | 20260506 22:22:48.143 | INFO | Perf report for the last 5.1407 seconds: | 20260506 22:22:48.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:22:48.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:48.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:48.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:48.143 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 22:22:48.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:48.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:48.143 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:22:48.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:48.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:48.143 | INFO | Uploaded 36GiB to GPU in 1.93139s (18.6395GiB/s) | 20260506 22:22:48.143 | INFO | Downloaded 34GiB to GPU in 1.87616s (18.1221GiB/s) | 20260506 22:22:53.284 | INFO | Analyzing chunk frames = [4909056,4909568) | 20260506 22:22:53.284 | INFO | MemInfo: { | 20260506 22:22:53.284 | INFO | "gpu" : " 17 GB", | 20260506 22:22:53.284 | INFO | "huge" : "", | 20260506 22:22:53.284 | INFO | "huge_shared" : " 27 GB", | 20260506 22:22:53.284 | INFO | "malloc" : " 14 GB", | 20260506 22:22:53.284 | INFO | "pinned" : " 29 GB" | 20260506 22:22:53.284 | INFO | } | 20260506 22:22:53.284 | INFO | Perf report for the last 5.14097 seconds: | 20260506 22:22:53.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:22:53.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:53.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:53.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:53.284 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 22:22:53.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:53.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:53.284 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:22:53.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:53.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:53.284 | INFO | Uploaded 36GiB to GPU in 2.00164s (17.9853GiB/s) | 20260506 22:22:53.284 | INFO | Downloaded 34GiB to GPU in 1.90611s (17.8374GiB/s) | 20260506 22:22:58.419 | INFO | Analyzing chunk frames = [4909568,4910080) | 20260506 22:22:58.419 | INFO | MemInfo: { | 20260506 22:22:58.419 | INFO | "gpu" : " 17 GB", | 20260506 22:22:58.420 | INFO | "huge" : "", | 20260506 22:22:58.420 | INFO | "huge_shared" : " 27 GB", | 20260506 22:22:58.420 | INFO | "malloc" : " 14 GB", | 20260506 22:22:58.420 | INFO | "pinned" : " 29 GB" | 20260506 22:22:58.420 | INFO | } | 20260506 22:22:58.420 | INFO | Perf report for the last 5.13583 seconds: | 20260506 22:22:58.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:22:58.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:58.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:58.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:58.420 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:22:58.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:58.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:58.420 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:22:58.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:58.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:22:58.420 | INFO | Uploaded 36GiB to GPU in 2.0355s (17.6861GiB/s) | 20260506 22:22:58.420 | INFO | Downloaded 34GiB to GPU in 1.93927s (17.5324GiB/s) | 20260506 22:23:03.559 | INFO | Analyzing chunk frames = [4910080,4910592) | 20260506 22:23:03.559 | INFO | MemInfo: { | 20260506 22:23:03.559 | INFO | "gpu" : " 17 GB", | 20260506 22:23:03.559 | INFO | "huge" : "", | 20260506 22:23:03.559 | INFO | "huge_shared" : " 27 GB", | 20260506 22:23:03.559 | INFO | "malloc" : " 14 GB", | 20260506 22:23:03.559 | INFO | "pinned" : " 29 GB" | 20260506 22:23:03.559 | INFO | } | 20260506 22:23:03.559 | INFO | Perf report for the last 5.13937 seconds: | 20260506 22:23:03.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:23:03.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:03.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:03.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:03.559 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 22:23:03.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:03.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:03.559 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260506 22:23:03.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:03.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:03.559 | INFO | Uploaded 36GiB to GPU in 1.95223s (18.4404GiB/s) | 20260506 22:23:03.559 | INFO | Downloaded 34GiB to GPU in 1.98004s (17.1714GiB/s) | 20260506 22:23:08.698 | INFO | Analyzing chunk frames = [4910592,4911104) | 20260506 22:23:08.698 | INFO | MemInfo: { | 20260506 22:23:08.698 | INFO | "gpu" : " 17 GB", | 20260506 22:23:08.698 | INFO | "huge" : "", | 20260506 22:23:08.699 | INFO | "huge_shared" : " 27 GB", | 20260506 22:23:08.699 | INFO | "malloc" : " 14 GB", | 20260506 22:23:08.699 | INFO | "pinned" : " 29 GB" | 20260506 22:23:08.699 | INFO | } | 20260506 22:23:08.699 | INFO | Perf report for the last 5.13956 seconds: | 20260506 22:23:08.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:23:08.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:08.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:08.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:08.699 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 22:23:08.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:08.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:08.699 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 22:23:08.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:08.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:08.699 | INFO | Uploaded 36GiB to GPU in 2.00696s (17.9375GiB/s) | 20260506 22:23:08.699 | INFO | Downloaded 34GiB to GPU in 1.9926s (17.0631GiB/s) | 20260506 22:23:08.797 | INFO | Starting a round of Baz IO | 20260506 22:23:11.931 | INFO | Populating RTMetrics report | 20260506 22:23:11.931 | INFO | RTMetrics report populated | 20260506 22:23:11.931 | INFO | Opening tmp RTMetrics json file | 20260506 22:23:11.932 | INFO | Serializing RTMetrics to json | 20260506 22:23:11.933 | INFO | Serialization done, swapping tmp file now... | 20260506 22:23:11.933 | INFO | RTMetrics done | 20260506 22:23:12.024 | INFO | Finished a round of Baz IO: Wrote 1.915 GiB in 3.226 seconds | 20260506 22:23:12.024 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.69% / 0.00% / 66.30% | 20260506 22:23:12.024 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.510812% / 0% / 7.15256e-05% | 20260506 22:23:12.024 | INFO | Baz IO overall: | 20260506 22:23:12.024 | INFO | BytesWritten : 2580619504973 | 20260506 22:23:12.024 | INFO | NumEvents : 1245181234598 | 20260506 22:23:13.838 | INFO | Analyzing chunk frames = [4911104,4911616) | 20260506 22:23:13.838 | INFO | MemInfo: { | 20260506 22:23:13.838 | INFO | "gpu" : " 17 GB", | 20260506 22:23:13.838 | INFO | "huge" : "", | 20260506 22:23:13.838 | INFO | "huge_shared" : " 27 GB", | 20260506 22:23:13.838 | INFO | "malloc" : " 14 GB", | 20260506 22:23:13.838 | INFO | "pinned" : " 29 GB" | 20260506 22:23:13.838 | INFO | } | 20260506 22:23:13.838 | INFO | Perf report for the last 5.13961 seconds: | 20260506 22:23:13.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:23:13.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:13.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:13.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:13.838 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 22:23:13.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:13.838 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260506 22:23:13.838 | INFO | PRE_HQ : 59.6%, 1.8, 56.7ms, 0 | 20260506 22:23:13.838 | INFO | BAZWRITER : 62.4%, 3.5, 111.1ms, 0 | 20260506 22:23:13.838 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:23:13.838 | INFO | Uploaded 36GiB to GPU in 2.03225s (17.7144GiB/s) | 20260506 22:23:13.838 | INFO | Downloaded 38GiB to GPU in 2.18488s (17.3923GiB/s) | 20260506 22:23:18.987 | INFO | Analyzing chunk frames = [4911616,4912128) | 20260506 22:23:18.987 | INFO | MemInfo: { | 20260506 22:23:18.987 | INFO | "gpu" : " 17 GB", | 20260506 22:23:18.987 | INFO | "huge" : "", | 20260506 22:23:18.988 | INFO | "huge_shared" : " 27 GB", | 20260506 22:23:18.988 | INFO | "malloc" : " 14 GB", | 20260506 22:23:18.988 | INFO | "pinned" : " 29 GB" | 20260506 22:23:18.988 | INFO | } | 20260506 22:23:18.988 | INFO | Perf report for the last 5.14944 seconds: | 20260506 22:23:18.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:23:18.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:18.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:18.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:18.988 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 22:23:18.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:18.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:18.988 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:23:18.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:18.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:18.988 | INFO | Uploaded 36GiB to GPU in 2.00117s (17.9895GiB/s) | 20260506 22:23:18.988 | INFO | Downloaded 34GiB to GPU in 1.92261s (17.6843GiB/s) | 20260506 22:23:24.121 | INFO | Analyzing chunk frames = [4912128,4912640) | 20260506 22:23:24.121 | INFO | MemInfo: { | 20260506 22:23:24.121 | INFO | "gpu" : " 17 GB", | 20260506 22:23:24.122 | INFO | "huge" : "", | 20260506 22:23:24.122 | INFO | "huge_shared" : " 27 GB", | 20260506 22:23:24.122 | INFO | "malloc" : " 14 GB", | 20260506 22:23:24.122 | INFO | "pinned" : " 29 GB" | 20260506 22:23:24.122 | INFO | } | 20260506 22:23:24.122 | INFO | Perf report for the last 5.13398 seconds: | 20260506 22:23:24.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:23:24.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:24.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:24.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:24.122 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260506 22:23:24.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:24.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:24.122 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:23:24.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:24.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:24.122 | INFO | Uploaded 36GiB to GPU in 1.99441s (18.0505GiB/s) | 20260506 22:23:24.122 | INFO | Downloaded 34GiB to GPU in 1.92793s (17.6355GiB/s) | 20260506 22:23:29.259 | INFO | Analyzing chunk frames = [4912640,4913152) | 20260506 22:23:29.259 | INFO | MemInfo: { | 20260506 22:23:29.259 | INFO | "gpu" : " 17 GB", | 20260506 22:23:29.259 | INFO | "huge" : "", | 20260506 22:23:29.259 | INFO | "huge_shared" : " 27 GB", | 20260506 22:23:29.259 | INFO | "malloc" : " 14 GB", | 20260506 22:23:29.259 | INFO | "pinned" : " 29 GB" | 20260506 22:23:29.259 | INFO | } | 20260506 22:23:29.259 | INFO | Perf report for the last 5.13708 seconds: | 20260506 22:23:29.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:23:29.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:29.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:29.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:29.259 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260506 22:23:29.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:29.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:29.259 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:23:29.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:29.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:29.259 | INFO | Uploaded 36GiB to GPU in 2.01871s (17.8332GiB/s) | 20260506 22:23:29.259 | INFO | Downloaded 34GiB to GPU in 1.90716s (17.8275GiB/s) | 20260506 22:23:34.401 | INFO | Analyzing chunk frames = [4913152,4913664) | 20260506 22:23:34.401 | INFO | MemInfo: { | 20260506 22:23:34.401 | INFO | "gpu" : " 17 GB", | 20260506 22:23:34.401 | INFO | "huge" : "", | 20260506 22:23:34.401 | INFO | "huge_shared" : " 27 GB", | 20260506 22:23:34.401 | INFO | "malloc" : " 14 GB", | 20260506 22:23:34.401 | INFO | "pinned" : " 29 GB" | 20260506 22:23:34.401 | INFO | } | 20260506 22:23:34.401 | INFO | Perf report for the last 5.14258 seconds: | 20260506 22:23:34.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:23:34.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:34.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:34.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:34.401 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 22:23:34.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:34.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:34.401 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 22:23:34.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:34.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:34.402 | INFO | Uploaded 36GiB to GPU in 2.00285s (17.9744GiB/s) | 20260506 22:23:34.402 | INFO | Downloaded 34GiB to GPU in 1.9207s (17.7019GiB/s) | 20260506 22:23:39.536 | INFO | Analyzing chunk frames = [4913664,4914176) | 20260506 22:23:39.536 | INFO | MemInfo: { | 20260506 22:23:39.536 | INFO | "gpu" : " 17 GB", | 20260506 22:23:39.536 | INFO | "huge" : "", | 20260506 22:23:39.536 | INFO | "huge_shared" : " 27 GB", | 20260506 22:23:39.536 | INFO | "malloc" : " 14 GB", | 20260506 22:23:39.536 | INFO | "pinned" : " 29 GB" | 20260506 22:23:39.536 | INFO | } | 20260506 22:23:39.536 | INFO | Perf report for the last 5.13479 seconds: | 20260506 22:23:39.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:23:39.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:39.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:39.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:39.536 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:23:39.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:39.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:39.536 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 22:23:39.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:39.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:39.536 | INFO | Uploaded 36GiB to GPU in 2.03873s (17.658GiB/s) | 20260506 22:23:39.536 | INFO | Downloaded 34GiB to GPU in 1.94171s (17.5103GiB/s) | 20260506 22:23:44.674 | INFO | Analyzing chunk frames = [4914176,4914688) | 20260506 22:23:44.674 | INFO | MemInfo: { | 20260506 22:23:44.674 | INFO | "gpu" : " 17 GB", | 20260506 22:23:44.674 | INFO | "huge" : "", | 20260506 22:23:44.674 | INFO | "huge_shared" : " 27 GB", | 20260506 22:23:44.674 | INFO | "malloc" : " 14 GB", | 20260506 22:23:44.674 | INFO | "pinned" : " 29 GB" | 20260506 22:23:44.674 | INFO | } | 20260506 22:23:44.674 | INFO | Perf report for the last 5.13801 seconds: | 20260506 22:23:44.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:23:44.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:44.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:44.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:44.674 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 22:23:44.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:44.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:44.674 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 22:23:44.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:44.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:44.674 | INFO | Uploaded 36GiB to GPU in 1.9695s (18.2788GiB/s) | 20260506 22:23:44.674 | INFO | Downloaded 34GiB to GPU in 1.99736s (17.0225GiB/s) | 20260506 22:23:49.814 | INFO | Analyzing chunk frames = [4914688,4915200) | 20260506 22:23:49.814 | INFO | MemInfo: { | 20260506 22:23:49.814 | INFO | "gpu" : " 17 GB", | 20260506 22:23:49.814 | INFO | "huge" : "", | 20260506 22:23:49.814 | INFO | "huge_shared" : " 27 GB", | 20260506 22:23:49.814 | INFO | "malloc" : " 14 GB", | 20260506 22:23:49.814 | INFO | "pinned" : " 29 GB" | 20260506 22:23:49.814 | INFO | } | 20260506 22:23:49.814 | INFO | Perf report for the last 5.13995 seconds: | 20260506 22:23:49.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:23:49.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:49.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:49.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:49.814 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260506 22:23:49.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:49.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:49.814 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:23:49.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:49.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:49.814 | INFO | Uploaded 36GiB to GPU in 2.02604s (17.7687GiB/s) | 20260506 22:23:49.814 | INFO | Downloaded 34GiB to GPU in 1.98644s (17.116GiB/s) | 20260506 22:23:49.814 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:23:49.814 | INFO | Total mem usage: 18 GiB | 20260506 22:23:49.814 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:23:49.814 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -25.4%) | 20260506 22:23:49.814 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:23:49.814 | INFO | High Water Breakdown: | 20260506 22:23:49.814 | INFO | 60.2% : 8.64 GiB ( 75.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:23:49.814 | INFO | 39.7% : 5.7 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:23:49.814 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:23:49.814 | INFO | ------------------------------------------ | 20260506 22:23:49.814 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:23:49.815 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:23:49.815 | INFO | Total mem usage: 26.5 GiB | 20260506 22:23:49.815 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:23:49.815 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:23:49.815 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:23:49.815 | INFO | High Water Breakdown: | 20260506 22:23:49.815 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:23:49.815 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:23:49.815 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:23:49.815 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:23:49.815 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:23:49.815 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:23:49.815 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:23:49.815 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:23:49.815 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:23:49.815 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:23:49.815 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:23:49.815 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:23:49.815 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:23:49.815 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:23:49.815 | INFO | ------------------------------------------ | 20260506 22:23:49.815 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:23:49.815 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:23:49.815 | INFO | Total mem usage: 25.6 GiB | 20260506 22:23:49.815 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:23:49.815 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:23:49.815 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:23:49.815 | INFO | High Water Breakdown: | 20260506 22:23:49.815 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:23:49.815 | INFO | ------------------------------------------ | 20260506 22:23:49.815 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:23:49.815 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:23:49.815 | INFO | Total mem usage: 15.7 GiB | 20260506 22:23:49.815 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:23:49.815 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:23:49.815 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:23:49.815 | INFO | High Water Breakdown: | 20260506 22:23:49.815 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:23:49.815 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:23:49.815 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:23:49.815 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:23:49.815 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:23:49.815 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:23:49.815 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:23:49.815 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:23:49.815 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:23:49.815 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:23:49.815 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:23:49.815 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:23:49.815 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:23:49.815 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:23:49.815 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:23:49.815 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:23:49.815 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:23:49.815 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:23:49.815 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:23:49.815 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:23:49.815 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:23:49.815 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:23:49.815 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:23:49.815 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:23:49.815 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:23:49.815 | INFO | ------------------------------------------ | 20260506 22:23:49.815 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:23:49.815 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:23:49.815 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:23:49.815 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:23:49.815 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.56e+01, 1.32e+01 / 2.28e+00 | 20260506 22:23:49.815 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.02e-01, 9.75e-03 / 6.93e-03 | 20260506 22:23:49.815 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.35e+00, 5.46e-01 / 4.05e-01 | 20260506 22:23:49.815 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.45e+00, 1.78e-01 / 1.31e-01 | 20260506 22:23:49.815 | INFO | PerfReport: 16.03%, DME -- 3.98e+01 / 4.44e+01, 3.91e+00 / 2.31e+00 | 20260506 22:23:49.815 | INFO | PerfReport: 57.33%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.37e+00 / 2.01e+00 | 20260506 22:23:49.815 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.20e+00 / 2.71e+00, 3.76e-01 / 1.91e-01 | 20260506 22:23:49.815 | INFO | PerfReport: 3.24%, Metrics -- 1.01e+00 / 1.34e+00, 2.67e-01 / 1.73e-01 | 20260506 22:23:49.815 | INFO | PerfReport: 1.11%, PulseSerializing -- 3.43e-01 / 3.91e-01, 5.40e-02 / 2.34e-02 | 20260506 22:23:49.815 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:23:49.815 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:23:49.815 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:23:49.815 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 22:23:49.815 | INFO | PerfReport: 21.56%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 22:23:49.815 | INFO | PerfReport: 21.19%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 22:23:49.815 | INFO | PerfReport: 32.07%, Compute -- 3.10e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 22:23:49.815 | INFO | PerfReport: 1.39%, LockUpload -- 1.35e+00 / 0.00e+00, 5.07e+00 / 0.00e+00 | 20260506 22:23:49.815 | INFO | PerfReport: 1.16%, LockDownload -- 1.12e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 22:23:49.815 | INFO | PerfReport: 22.62%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 22:23:49.815 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260506 22:23:49.913 | INFO | Starting a round of Baz IO | 20260506 22:23:53.042 | INFO | Populating RTMetrics report | 20260506 22:23:53.042 | INFO | RTMetrics report populated | 20260506 22:23:53.042 | INFO | Opening tmp RTMetrics json file | 20260506 22:23:53.043 | INFO | Serializing RTMetrics to json | 20260506 22:23:53.051 | INFO | Serialization done, swapping tmp file now... | 20260506 22:23:53.065 | INFO | RTMetrics done | 20260506 22:23:53.134 | INFO | Finished a round of Baz IO: Wrote 1.914 GiB in 3.22 seconds | 20260506 22:23:53.134 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.69% / 0.00% / 66.31% | 20260506 22:23:53.134 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.510327% / 3.97364e-06% / 7.94729e-05% | 20260506 22:23:53.134 | INFO | Baz IO overall: | 20260506 22:23:53.134 | INFO | BytesWritten : 2582675159312 | 20260506 22:23:53.134 | INFO | NumEvents : 1246162784471 | 20260506 22:23:54.952 | INFO | Analyzing chunk frames = [4915200,4915712) | 20260506 22:23:54.952 | INFO | MemInfo: { | 20260506 22:23:54.952 | INFO | "gpu" : " 17 GB", | 20260506 22:23:54.952 | INFO | "huge" : "", | 20260506 22:23:54.952 | INFO | "huge_shared" : " 27 GB", | 20260506 22:23:54.952 | INFO | "malloc" : " 14 GB", | 20260506 22:23:54.952 | INFO | "pinned" : " 29 GB" | 20260506 22:23:54.952 | INFO | } | 20260506 22:23:54.952 | INFO | Perf report for the last 5.13783 seconds: | 20260506 22:23:54.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:23:54.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:54.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:54.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:54.952 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260506 22:23:54.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:23:54.952 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 22:23:54.952 | INFO | PRE_HQ : 59.8%, 1.8, 56.8ms, 0 | 20260506 22:23:54.952 | INFO | BAZWRITER : 62.3%, 3.4, 110.3ms, 0 | 20260506 22:23:54.952 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:23:54.952 | INFO | Uploaded 36GiB to GPU in 2.04247s (17.6257GiB/s) | 20260506 22:23:54.952 | INFO | Downloaded 38GiB to GPU in 2.16588s (17.5448GiB/s) | 20260506 22:24:00.090 | INFO | Analyzing chunk frames = [4915712,4916224) | 20260506 22:24:00.090 | INFO | MemInfo: { | 20260506 22:24:00.090 | INFO | "gpu" : " 17 GB", | 20260506 22:24:00.090 | INFO | "huge" : "", | 20260506 22:24:00.090 | INFO | "huge_shared" : " 27 GB", | 20260506 22:24:00.090 | INFO | "malloc" : " 14 GB", | 20260506 22:24:00.090 | INFO | "pinned" : " 29 GB" | 20260506 22:24:00.090 | INFO | } | 20260506 22:24:00.090 | INFO | Perf report for the last 5.1385 seconds: | 20260506 22:24:00.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:24:00.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:00.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:00.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:00.090 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:24:00.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:00.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:00.091 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:24:00.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:00.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:00.091 | INFO | Uploaded 36GiB to GPU in 2.00448s (17.9598GiB/s) | 20260506 22:24:00.091 | INFO | Downloaded 34GiB to GPU in 1.92264s (17.684GiB/s) | 20260506 22:24:05.235 | INFO | Analyzing chunk frames = [4916224,4916736) | 20260506 22:24:05.235 | INFO | MemInfo: { | 20260506 22:24:05.235 | INFO | "gpu" : " 17 GB", | 20260506 22:24:05.235 | INFO | "huge" : "", | 20260506 22:24:05.235 | INFO | "huge_shared" : " 27 GB", | 20260506 22:24:05.235 | INFO | "malloc" : " 14 GB", | 20260506 22:24:05.235 | INFO | "pinned" : " 29 GB" | 20260506 22:24:05.235 | INFO | } | 20260506 22:24:05.235 | INFO | Perf report for the last 5.14426 seconds: | 20260506 22:24:05.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:24:05.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:05.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:05.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:05.235 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 22:24:05.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:05.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:05.235 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:24:05.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:05.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:05.235 | INFO | Uploaded 36GiB to GPU in 1.97444s (18.233GiB/s) | 20260506 22:24:05.235 | INFO | Downloaded 34GiB to GPU in 1.9159s (17.7462GiB/s) | 20260506 22:24:10.372 | INFO | Analyzing chunk frames = [4916736,4917248) | 20260506 22:24:10.372 | INFO | MemInfo: { | 20260506 22:24:10.372 | INFO | "gpu" : " 17 GB", | 20260506 22:24:10.372 | INFO | "huge" : "", | 20260506 22:24:10.372 | INFO | "huge_shared" : " 27 GB", | 20260506 22:24:10.372 | INFO | "malloc" : " 14 GB", | 20260506 22:24:10.372 | INFO | "pinned" : " 29 GB" | 20260506 22:24:10.372 | INFO | } | 20260506 22:24:10.372 | INFO | Perf report for the last 5.13707 seconds: | 20260506 22:24:10.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:24:10.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:10.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:10.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:10.372 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 22:24:10.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:10.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:10.372 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:24:10.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:10.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:10.372 | INFO | Uploaded 36GiB to GPU in 1.97368s (18.24GiB/s) | 20260506 22:24:10.372 | INFO | Downloaded 34GiB to GPU in 1.8754s (18.1294GiB/s) | 20260506 22:24:15.509 | INFO | Analyzing chunk frames = [4917248,4917760) | 20260506 22:24:15.509 | INFO | MemInfo: { | 20260506 22:24:15.509 | INFO | "gpu" : " 17 GB", | 20260506 22:24:15.509 | INFO | "huge" : "", | 20260506 22:24:15.510 | INFO | "huge_shared" : " 27 GB", | 20260506 22:24:15.510 | INFO | "malloc" : " 14 GB", | 20260506 22:24:15.510 | INFO | "pinned" : " 29 GB" | 20260506 22:24:15.510 | INFO | } | 20260506 22:24:15.510 | INFO | Perf report for the last 5.1378 seconds: | 20260506 22:24:15.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:24:15.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:15.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:15.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:15.510 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 22:24:15.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:15.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:15.510 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:24:15.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:15.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:15.510 | INFO | Uploaded 36GiB to GPU in 1.99825s (18.0157GiB/s) | 20260506 22:24:15.510 | INFO | Downloaded 34GiB to GPU in 1.92365s (17.6747GiB/s) | 20260506 22:24:20.654 | INFO | Analyzing chunk frames = [4917760,4918272) | 20260506 22:24:20.654 | INFO | MemInfo: { | 20260506 22:24:20.654 | INFO | "gpu" : " 17 GB", | 20260506 22:24:20.654 | INFO | "huge" : "", | 20260506 22:24:20.654 | INFO | "huge_shared" : " 27 GB", | 20260506 22:24:20.654 | INFO | "malloc" : " 14 GB", | 20260506 22:24:20.654 | INFO | "pinned" : " 29 GB" | 20260506 22:24:20.654 | INFO | } | 20260506 22:24:20.654 | INFO | Perf report for the last 5.14472 seconds: | 20260506 22:24:20.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:24:20.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:20.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:20.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:20.654 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 22:24:20.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:20.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:20.654 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:24:20.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:20.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:20.655 | INFO | Uploaded 36GiB to GPU in 2.00829s (17.9257GiB/s) | 20260506 22:24:20.655 | INFO | Downloaded 34GiB to GPU in 1.93963s (17.5291GiB/s) | 20260506 22:24:25.790 | INFO | Analyzing chunk frames = [4918272,4918784) | 20260506 22:24:25.790 | INFO | MemInfo: { | 20260506 22:24:25.790 | INFO | "gpu" : " 17 GB", | 20260506 22:24:25.790 | INFO | "huge" : "", | 20260506 22:24:25.790 | INFO | "huge_shared" : " 27 GB", | 20260506 22:24:25.790 | INFO | "malloc" : " 14 GB", | 20260506 22:24:25.790 | INFO | "pinned" : " 29 GB" | 20260506 22:24:25.790 | INFO | } | 20260506 22:24:25.790 | INFO | Perf report for the last 5.13577 seconds: | 20260506 22:24:25.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:24:25.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:25.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:25.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:25.790 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 22:24:25.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:25.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:25.790 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 22:24:25.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:25.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:25.790 | INFO | Uploaded 36GiB to GPU in 1.97463s (18.2313GiB/s) | 20260506 22:24:25.790 | INFO | Downloaded 34GiB to GPU in 1.99878s (17.0103GiB/s) | 20260506 22:24:30.927 | INFO | Analyzing chunk frames = [4918784,4919296) | 20260506 22:24:30.927 | INFO | MemInfo: { | 20260506 22:24:30.927 | INFO | "gpu" : " 17 GB", | 20260506 22:24:30.927 | INFO | "huge" : "", | 20260506 22:24:30.927 | INFO | "huge_shared" : " 27 GB", | 20260506 22:24:30.927 | INFO | "malloc" : " 14 GB", | 20260506 22:24:30.927 | INFO | "pinned" : " 29 GB" | 20260506 22:24:30.927 | INFO | } | 20260506 22:24:30.927 | INFO | Perf report for the last 5.13666 seconds: | 20260506 22:24:30.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:24:30.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:30.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:30.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:30.927 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:24:30.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:30.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:30.927 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 22:24:30.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:30.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:30.927 | INFO | Uploaded 36GiB to GPU in 1.99653s (18.0313GiB/s) | 20260506 22:24:30.927 | INFO | Downloaded 34GiB to GPU in 1.98599s (17.1199GiB/s) | 20260506 22:24:31.025 | INFO | Starting a round of Baz IO | 20260506 22:24:34.164 | INFO | Populating RTMetrics report | 20260506 22:24:34.164 | INFO | RTMetrics report populated | 20260506 22:24:34.164 | INFO | Opening tmp RTMetrics json file | 20260506 22:24:34.167 | INFO | Serializing RTMetrics to json | 20260506 22:24:34.184 | INFO | Serialization done, swapping tmp file now... | 20260506 22:24:34.185 | INFO | RTMetrics done | 20260506 22:24:34.260 | INFO | Finished a round of Baz IO: Wrote 1.913 GiB in 3.234 seconds | 20260506 22:24:34.260 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.69% / 0.00% / 66.31% | 20260506 22:24:34.260 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.507331% / 7.94729e-06% / 5.96046e-05% | 20260506 22:24:34.260 | INFO | Baz IO overall: | 20260506 22:24:34.260 | INFO | BytesWritten : 2584729299094 | 20260506 22:24:34.260 | INFO | NumEvents : 1247143158591 | 20260506 22:24:36.071 | INFO | Analyzing chunk frames = [4919296,4919808) | 20260506 22:24:36.071 | INFO | MemInfo: { | 20260506 22:24:36.071 | INFO | "gpu" : " 17 GB", | 20260506 22:24:36.071 | INFO | "huge" : "", | 20260506 22:24:36.071 | INFO | "huge_shared" : " 27 GB", | 20260506 22:24:36.071 | INFO | "malloc" : " 14 GB", | 20260506 22:24:36.071 | INFO | "pinned" : " 29 GB" | 20260506 22:24:36.071 | INFO | } | 20260506 22:24:36.071 | INFO | Perf report for the last 5.14393 seconds: | 20260506 22:24:36.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:24:36.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:36.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:36.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:36.071 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260506 22:24:36.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:36.071 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 22:24:36.071 | INFO | PRE_HQ : 59.7%, 1.8, 57.1ms, 0 | 20260506 22:24:36.071 | INFO | BAZWRITER : 62.5%, 3.4, 110.8ms, 0 | 20260506 22:24:36.071 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 22:24:36.071 | INFO | Uploaded 36GiB to GPU in 2.04183s (17.6313GiB/s) | 20260506 22:24:36.071 | INFO | Downloaded 38GiB to GPU in 2.17882s (17.4407GiB/s) | 20260506 22:24:41.211 | INFO | Analyzing chunk frames = [4919808,4920320) | 20260506 22:24:41.211 | INFO | MemInfo: { | 20260506 22:24:41.211 | INFO | "gpu" : " 17 GB", | 20260506 22:24:41.211 | INFO | "huge" : "", | 20260506 22:24:41.211 | INFO | "huge_shared" : " 27 GB", | 20260506 22:24:41.211 | INFO | "malloc" : " 14 GB", | 20260506 22:24:41.211 | INFO | "pinned" : " 29 GB" | 20260506 22:24:41.211 | INFO | } | 20260506 22:24:41.211 | INFO | Perf report for the last 5.14043 seconds: | 20260506 22:24:41.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:24:41.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:41.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:41.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:41.211 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:24:41.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:41.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:41.211 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 22:24:41.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:41.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:41.211 | INFO | Uploaded 36GiB to GPU in 2.0173s (17.8456GiB/s) | 20260506 22:24:41.211 | INFO | Downloaded 34GiB to GPU in 1.92828s (17.6323GiB/s) | 20260506 22:24:46.349 | INFO | Analyzing chunk frames = [4920320,4920832) | 20260506 22:24:46.349 | INFO | MemInfo: { | 20260506 22:24:46.349 | INFO | "gpu" : " 17 GB", | 20260506 22:24:46.349 | INFO | "huge" : "", | 20260506 22:24:46.349 | INFO | "huge_shared" : " 27 GB", | 20260506 22:24:46.349 | INFO | "malloc" : " 14 GB", | 20260506 22:24:46.349 | INFO | "pinned" : " 29 GB" | 20260506 22:24:46.349 | INFO | } | 20260506 22:24:46.349 | INFO | Perf report for the last 5.13767 seconds: | 20260506 22:24:46.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:24:46.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:46.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:46.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:46.349 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 22:24:46.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:46.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:46.349 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:24:46.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:46.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:46.349 | INFO | Uploaded 36GiB to GPU in 2.00133s (17.9881GiB/s) | 20260506 22:24:46.349 | INFO | Downloaded 34GiB to GPU in 1.93242s (17.5945GiB/s) | 20260506 22:24:51.485 | INFO | Analyzing chunk frames = [4920832,4921344) | 20260506 22:24:51.485 | INFO | MemInfo: { | 20260506 22:24:51.485 | INFO | "gpu" : " 17 GB", | 20260506 22:24:51.485 | INFO | "huge" : "", | 20260506 22:24:51.485 | INFO | "huge_shared" : " 27 GB", | 20260506 22:24:51.485 | INFO | "malloc" : " 14 GB", | 20260506 22:24:51.485 | INFO | "pinned" : " 29 GB" | 20260506 22:24:51.485 | INFO | } | 20260506 22:24:51.485 | INFO | Perf report for the last 5.13617 seconds: | 20260506 22:24:51.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:24:51.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:51.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:51.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:51.485 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260506 22:24:51.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:51.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:51.485 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:24:51.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:51.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:51.485 | INFO | Uploaded 36GiB to GPU in 2.01112s (17.9004GiB/s) | 20260506 22:24:51.485 | INFO | Downloaded 34GiB to GPU in 1.90634s (17.8352GiB/s) | 20260506 22:24:56.622 | INFO | Analyzing chunk frames = [4921344,4921856) | 20260506 22:24:56.622 | INFO | MemInfo: { | 20260506 22:24:56.622 | INFO | "gpu" : " 17 GB", | 20260506 22:24:56.622 | INFO | "huge" : "", | 20260506 22:24:56.622 | INFO | "huge_shared" : " 27 GB", | 20260506 22:24:56.622 | INFO | "malloc" : " 14 GB", | 20260506 22:24:56.622 | INFO | "pinned" : " 29 GB" | 20260506 22:24:56.622 | INFO | } | 20260506 22:24:56.622 | INFO | Perf report for the last 5.13677 seconds: | 20260506 22:24:56.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:24:56.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:56.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:56.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:56.622 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 22:24:56.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:56.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:56.622 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:24:56.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:56.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:24:56.622 | INFO | Uploaded 36GiB to GPU in 2.01152s (17.8969GiB/s) | 20260506 22:24:56.622 | INFO | Downloaded 34GiB to GPU in 1.92345s (17.6766GiB/s) | 20260506 22:25:01.763 | INFO | Analyzing chunk frames = [4921856,4922368) | 20260506 22:25:01.763 | INFO | MemInfo: { | 20260506 22:25:01.763 | INFO | "gpu" : " 17 GB", | 20260506 22:25:01.763 | INFO | "huge" : "", | 20260506 22:25:01.763 | INFO | "huge_shared" : " 27 GB", | 20260506 22:25:01.763 | INFO | "malloc" : " 14 GB", | 20260506 22:25:01.763 | INFO | "pinned" : " 29 GB" | 20260506 22:25:01.763 | INFO | } | 20260506 22:25:01.763 | INFO | Perf report for the last 5.14126 seconds: | 20260506 22:25:01.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:25:01.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:01.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:01.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:01.763 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 22:25:01.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:01.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:01.763 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 22:25:01.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:01.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:01.763 | INFO | Uploaded 36GiB to GPU in 1.99801s (18.0179GiB/s) | 20260506 22:25:01.763 | INFO | Downloaded 34GiB to GPU in 1.93643s (17.5581GiB/s) | 20260506 22:25:06.902 | INFO | Analyzing chunk frames = [4922368,4922880) | 20260506 22:25:06.902 | INFO | MemInfo: { | 20260506 22:25:06.902 | INFO | "gpu" : " 17 GB", | 20260506 22:25:06.902 | INFO | "huge" : "", | 20260506 22:25:06.902 | INFO | "huge_shared" : " 27 GB", | 20260506 22:25:06.902 | INFO | "malloc" : " 14 GB", | 20260506 22:25:06.902 | INFO | "pinned" : " 29 GB" | 20260506 22:25:06.902 | INFO | } | 20260506 22:25:06.902 | INFO | Perf report for the last 5.13912 seconds: | 20260506 22:25:06.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:25:06.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:06.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:06.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:06.902 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 22:25:06.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:06.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:06.902 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:25:06.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:06.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:06.902 | INFO | Uploaded 36GiB to GPU in 1.96595s (18.3118GiB/s) | 20260506 22:25:06.902 | INFO | Downloaded 34GiB to GPU in 1.98997s (17.0857GiB/s) | 20260506 22:25:12.040 | INFO | Analyzing chunk frames = [4922880,4923392) | 20260506 22:25:12.040 | INFO | MemInfo: { | 20260506 22:25:12.040 | INFO | "gpu" : " 17 GB", | 20260506 22:25:12.040 | INFO | "huge" : "", | 20260506 22:25:12.040 | INFO | "huge_shared" : " 27 GB", | 20260506 22:25:12.040 | INFO | "malloc" : " 14 GB", | 20260506 22:25:12.040 | INFO | "pinned" : " 29 GB" | 20260506 22:25:12.040 | INFO | } | 20260506 22:25:12.040 | INFO | Perf report for the last 5.13792 seconds: | 20260506 22:25:12.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:25:12.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:12.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:12.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:12.040 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 22:25:12.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:12.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:12.040 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 22:25:12.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:12.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:12.040 | INFO | Uploaded 36GiB to GPU in 2.01397s (17.8752GiB/s) | 20260506 22:25:12.041 | INFO | Downloaded 34GiB to GPU in 2.00166s (16.9859GiB/s) | 20260506 22:25:12.041 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:25:12.041 | INFO | Total mem usage: 18 GiB | 20260506 22:25:12.041 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:25:12.041 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:25:12.041 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:25:12.041 | INFO | High Water Breakdown: | 20260506 22:25:12.041 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:25:12.041 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:25:12.041 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:25:12.041 | INFO | ------------------------------------------ | 20260506 22:25:12.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:25:12.041 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:25:12.041 | INFO | Total mem usage: 26.5 GiB | 20260506 22:25:12.041 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:25:12.041 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:25:12.041 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:25:12.041 | INFO | High Water Breakdown: | 20260506 22:25:12.041 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:25:12.041 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:25:12.041 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:25:12.041 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:25:12.041 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:25:12.041 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:25:12.041 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:25:12.041 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:25:12.041 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:25:12.041 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:25:12.041 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:25:12.041 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:25:12.041 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:25:12.041 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:25:12.041 | INFO | ------------------------------------------ | 20260506 22:25:12.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:25:12.041 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:25:12.041 | INFO | Total mem usage: 25.6 GiB | 20260506 22:25:12.041 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:25:12.041 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:25:12.041 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:25:12.041 | INFO | High Water Breakdown: | 20260506 22:25:12.041 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:25:12.041 | INFO | ------------------------------------------ | 20260506 22:25:12.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:25:12.041 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:25:12.041 | INFO | Total mem usage: 15.7 GiB | 20260506 22:25:12.041 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:25:12.041 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:25:12.041 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:25:12.041 | INFO | High Water Breakdown: | 20260506 22:25:12.041 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:25:12.041 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:25:12.041 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:25:12.041 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:25:12.041 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:25:12.041 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:25:12.041 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:25:12.041 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:25:12.041 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:25:12.041 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:25:12.041 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:25:12.041 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:25:12.041 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:25:12.041 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:25:12.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:25:12.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:25:12.041 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:25:12.041 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:25:12.041 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:25:12.041 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:25:12.041 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:25:12.041 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:25:12.041 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:25:12.041 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:25:12.041 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:25:12.041 | INFO | ------------------------------------------ | 20260506 22:25:12.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:25:12.042 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:25:12.042 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:25:12.042 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:25:12.042 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.56e+01, 1.35e+01 / 2.45e+00 | 20260506 22:25:12.042 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.03e-01, 9.93e-03 / 7.73e-03 | 20260506 22:25:12.042 | INFO | PerfReport: 11.65%, Baseline -- 3.62e+00 / 4.33e+00, 5.48e-01 / 3.92e-01 | 20260506 22:25:12.042 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.46e+00, 1.83e-01 / 1.38e-01 | 20260506 22:25:12.042 | INFO | PerfReport: 16.01%, DME -- 3.98e+01 / 4.42e+01, 3.96e+00 / 3.10e+00 | 20260506 22:25:12.042 | INFO | PerfReport: 57.36%, FrameLabeling -- 1.78e+01 / 2.11e+01, 1.40e+00 / 1.78e+00 | 20260506 22:25:12.042 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.19e+00 / 2.67e+00, 3.73e-01 / 2.39e-01 | 20260506 22:25:12.042 | INFO | PerfReport: 3.24%, Metrics -- 1.01e+00 / 1.32e+00, 2.72e-01 / 1.67e-01 | 20260506 22:25:12.042 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.43e-01 / 3.90e-01, 5.42e-02 / 1.96e-02 | 20260506 22:25:12.042 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:25:12.042 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:25:12.042 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:25:12.042 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 22:25:12.042 | INFO | PerfReport: 21.55%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 22:25:12.042 | INFO | PerfReport: 21.21%, Download -- 2.05e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 22:25:12.042 | INFO | PerfReport: 32.10%, Compute -- 3.11e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 22:25:12.042 | INFO | PerfReport: 1.30%, LockUpload -- 1.25e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260506 22:25:12.042 | INFO | PerfReport: 1.18%, LockDownload -- 1.14e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 22:25:12.042 | INFO | PerfReport: 22.65%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 22:25:12.042 | INFO | PerfReport: 0.02%, Misc -- 1.52e-02 / 0.00e+00, 3.78e-02 / 0.00e+00 | 20260506 22:25:12.149 | INFO | Starting a round of Baz IO | 20260506 22:25:15.284 | INFO | Populating RTMetrics report | 20260506 22:25:15.284 | INFO | RTMetrics report populated | 20260506 22:25:15.284 | INFO | Opening tmp RTMetrics json file | 20260506 22:25:15.285 | INFO | Serializing RTMetrics to json | 20260506 22:25:15.286 | INFO | Serialization done, swapping tmp file now... | 20260506 22:25:15.286 | INFO | RTMetrics done | 20260506 22:25:15.393 | INFO | Finished a round of Baz IO: Wrote 1.915 GiB in 3.243 seconds | 20260506 22:25:15.393 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.69% / 0.00% / 66.31% | 20260506 22:25:15.393 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.509195% / 3.97364e-06% / 4.76837e-05% | 20260506 22:25:15.393 | INFO | Baz IO overall: | 20260506 22:25:15.393 | INFO | BytesWritten : 2586785102761 | 20260506 22:25:15.393 | INFO | NumEvents : 1248124958076 | 20260506 22:25:17.181 | INFO | Analyzing chunk frames = [4923392,4923904) | 20260506 22:25:17.181 | INFO | MemInfo: { | 20260506 22:25:17.181 | INFO | "gpu" : " 17 GB", | 20260506 22:25:17.181 | INFO | "huge" : "", | 20260506 22:25:17.181 | INFO | "huge_shared" : " 27 GB", | 20260506 22:25:17.181 | INFO | "malloc" : " 14 GB", | 20260506 22:25:17.181 | INFO | "pinned" : " 29 GB" | 20260506 22:25:17.181 | INFO | } | 20260506 22:25:17.181 | INFO | Perf report for the last 5.14094 seconds: | 20260506 22:25:17.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:25:17.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:17.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:17.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:17.181 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260506 22:25:17.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:17.181 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 22:25:17.181 | INFO | PRE_HQ : 60.2%, 1.9, 60.7ms, 0 | 20260506 22:25:17.181 | INFO | BAZWRITER : 62.8%, 3.6, 116.2ms, 0 | 20260506 22:25:17.181 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 22:25:17.181 | INFO | Uploaded 36GiB to GPU in 2.01132s (17.8987GiB/s) | 20260506 22:25:17.182 | INFO | Downloaded 38GiB to GPU in 2.18914s (17.3584GiB/s) | 20260506 22:25:22.328 | INFO | Analyzing chunk frames = [4923904,4924416) | 20260506 22:25:22.328 | INFO | MemInfo: { | 20260506 22:25:22.328 | INFO | "gpu" : " 17 GB", | 20260506 22:25:22.328 | INFO | "huge" : "", | 20260506 22:25:22.328 | INFO | "huge_shared" : " 27 GB", | 20260506 22:25:22.328 | INFO | "malloc" : " 14 GB", | 20260506 22:25:22.328 | INFO | "pinned" : " 29 GB" | 20260506 22:25:22.328 | INFO | } | 20260506 22:25:22.328 | INFO | Perf report for the last 5.14723 seconds: | 20260506 22:25:22.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:25:22.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:22.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:22.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:22.328 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 22:25:22.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:22.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:22.329 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:25:22.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:22.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:22.329 | INFO | Uploaded 36GiB to GPU in 1.99962s (18.0034GiB/s) | 20260506 22:25:22.329 | INFO | Downloaded 34GiB to GPU in 1.92777s (17.637GiB/s) | 20260506 22:25:27.467 | INFO | Analyzing chunk frames = [4924416,4924928) | 20260506 22:25:27.467 | INFO | MemInfo: { | 20260506 22:25:27.467 | INFO | "gpu" : " 17 GB", | 20260506 22:25:27.467 | INFO | "huge" : "", | 20260506 22:25:27.467 | INFO | "huge_shared" : " 27 GB", | 20260506 22:25:27.467 | INFO | "malloc" : " 14 GB", | 20260506 22:25:27.467 | INFO | "pinned" : " 29 GB" | 20260506 22:25:27.467 | INFO | } | 20260506 22:25:27.467 | INFO | Perf report for the last 5.13888 seconds: | 20260506 22:25:27.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:25:27.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:27.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:27.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:27.467 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 22:25:27.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:27.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:27.467 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:25:27.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:27.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:27.468 | INFO | Uploaded 36GiB to GPU in 1.99115s (18.08GiB/s) | 20260506 22:25:27.468 | INFO | Downloaded 34GiB to GPU in 1.93661s (17.5564GiB/s) | 20260506 22:25:32.603 | INFO | Analyzing chunk frames = [4924928,4925440) | 20260506 22:25:32.603 | INFO | MemInfo: { | 20260506 22:25:32.603 | INFO | "gpu" : " 17 GB", | 20260506 22:25:32.603 | INFO | "huge" : "", | 20260506 22:25:32.603 | INFO | "huge_shared" : " 27 GB", | 20260506 22:25:32.603 | INFO | "malloc" : " 14 GB", | 20260506 22:25:32.603 | INFO | "pinned" : " 29 GB" | 20260506 22:25:32.603 | INFO | } | 20260506 22:25:32.603 | INFO | Perf report for the last 5.13586 seconds: | 20260506 22:25:32.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:25:32.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:32.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:32.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:32.603 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 22:25:32.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:32.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:32.603 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:25:32.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:32.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:32.603 | INFO | Uploaded 36GiB to GPU in 1.96521s (18.3186GiB/s) | 20260506 22:25:32.603 | INFO | Downloaded 34GiB to GPU in 1.9105s (17.7964GiB/s) | 20260506 22:25:37.740 | INFO | Analyzing chunk frames = [4925440,4925952) | 20260506 22:25:37.740 | INFO | MemInfo: { | 20260506 22:25:37.740 | INFO | "gpu" : " 17 GB", | 20260506 22:25:37.740 | INFO | "huge" : "", | 20260506 22:25:37.740 | INFO | "huge_shared" : " 27 GB", | 20260506 22:25:37.740 | INFO | "malloc" : " 14 GB", | 20260506 22:25:37.740 | INFO | "pinned" : " 29 GB" | 20260506 22:25:37.740 | INFO | } | 20260506 22:25:37.740 | INFO | Perf report for the last 5.13694 seconds: | 20260506 22:25:37.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:25:37.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:37.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:37.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:37.740 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 22:25:37.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:37.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:37.740 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:25:37.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:37.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:37.740 | INFO | Uploaded 36GiB to GPU in 1.97834s (18.1971GiB/s) | 20260506 22:25:37.740 | INFO | Downloaded 34GiB to GPU in 1.92882s (17.6274GiB/s) | 20260506 22:25:42.877 | INFO | Analyzing chunk frames = [4925952,4926464) | 20260506 22:25:42.877 | INFO | MemInfo: { | 20260506 22:25:42.877 | INFO | "gpu" : " 17 GB", | 20260506 22:25:42.878 | INFO | "huge" : "", | 20260506 22:25:42.878 | INFO | "huge_shared" : " 27 GB", | 20260506 22:25:42.878 | INFO | "malloc" : " 14 GB", | 20260506 22:25:42.878 | INFO | "pinned" : " 29 GB" | 20260506 22:25:42.878 | INFO | } | 20260506 22:25:42.878 | INFO | Perf report for the last 5.1375 seconds: | 20260506 22:25:42.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:25:42.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:42.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:42.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:42.878 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 22:25:42.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:42.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:42.878 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:25:42.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:42.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:42.878 | INFO | Uploaded 36GiB to GPU in 1.99659s (18.0307GiB/s) | 20260506 22:25:42.878 | INFO | Downloaded 34GiB to GPU in 1.93314s (17.588GiB/s) | 20260506 22:25:48.019 | INFO | Analyzing chunk frames = [4926464,4926976) | 20260506 22:25:48.019 | INFO | MemInfo: { | 20260506 22:25:48.019 | INFO | "gpu" : " 17 GB", | 20260506 22:25:48.019 | INFO | "huge" : "", | 20260506 22:25:48.019 | INFO | "huge_shared" : " 27 GB", | 20260506 22:25:48.019 | INFO | "malloc" : " 14 GB", | 20260506 22:25:48.019 | INFO | "pinned" : " 29 GB" | 20260506 22:25:48.019 | INFO | } | 20260506 22:25:48.019 | INFO | Perf report for the last 5.14149 seconds: | 20260506 22:25:48.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:25:48.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:48.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:48.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:48.019 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 22:25:48.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:48.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:48.019 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 22:25:48.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:48.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:48.019 | INFO | Uploaded 36GiB to GPU in 1.9746s (18.2316GiB/s) | 20260506 22:25:48.019 | INFO | Downloaded 34GiB to GPU in 2.00791s (16.933GiB/s) | 20260506 22:25:53.157 | INFO | Analyzing chunk frames = [4926976,4927488) | 20260506 22:25:53.157 | INFO | MemInfo: { | 20260506 22:25:53.157 | INFO | "gpu" : " 17 GB", | 20260506 22:25:53.157 | INFO | "huge" : "", | 20260506 22:25:53.157 | INFO | "huge_shared" : " 27 GB", | 20260506 22:25:53.157 | INFO | "malloc" : " 14 GB", | 20260506 22:25:53.157 | INFO | "pinned" : " 29 GB" | 20260506 22:25:53.157 | INFO | } | 20260506 22:25:53.157 | INFO | Perf report for the last 5.13794 seconds: | 20260506 22:25:53.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:25:53.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:53.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:53.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:53.157 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:25:53.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:53.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:53.157 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:25:53.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:53.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:53.157 | INFO | Uploaded 36GiB to GPU in 1.96818s (18.291GiB/s) | 20260506 22:25:53.157 | INFO | Downloaded 34GiB to GPU in 1.99352s (17.0553GiB/s) | 20260506 22:25:53.255 | INFO | Starting a round of Baz IO | 20260506 22:25:56.388 | INFO | Populating RTMetrics report | 20260506 22:25:56.389 | INFO | RTMetrics report populated | 20260506 22:25:56.389 | INFO | Opening tmp RTMetrics json file | 20260506 22:25:56.404 | INFO | Serializing RTMetrics to json | 20260506 22:25:56.409 | INFO | Serialization done, swapping tmp file now... | 20260506 22:25:56.410 | INFO | RTMetrics done | 20260506 22:25:56.484 | INFO | Finished a round of Baz IO: Wrote 1.912 GiB in 3.229 seconds | 20260506 22:25:56.484 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.68% / 0.00% / 66.31% | 20260506 22:25:56.484 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.506083% / 1.19209e-05% / 5.16574e-05% | 20260506 22:25:56.484 | INFO | Baz IO overall: | 20260506 22:25:56.484 | INFO | BytesWritten : 2588838297642 | 20260506 22:25:56.484 | INFO | NumEvents : 1249105150313 | 20260506 22:25:58.299 | INFO | Analyzing chunk frames = [4927488,4928000) | 20260506 22:25:58.299 | INFO | MemInfo: { | 20260506 22:25:58.299 | INFO | "gpu" : " 17 GB", | 20260506 22:25:58.299 | INFO | "huge" : "", | 20260506 22:25:58.299 | INFO | "huge_shared" : " 27 GB", | 20260506 22:25:58.299 | INFO | "malloc" : " 14 GB", | 20260506 22:25:58.299 | INFO | "pinned" : " 29 GB" | 20260506 22:25:58.299 | INFO | } | 20260506 22:25:58.299 | INFO | Perf report for the last 5.14205 seconds: | 20260506 22:25:58.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:25:58.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:58.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:58.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:58.299 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 22:25:58.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:25:58.299 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 22:25:58.299 | INFO | PRE_HQ : 59.6%, 1.8, 56.7ms, 0 | 20260506 22:25:58.299 | INFO | BAZWRITER : 62.4%, 3.4, 110.3ms, 0 | 20260506 22:25:58.299 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 22:25:58.299 | INFO | Uploaded 36GiB to GPU in 2.03204s (17.7162GiB/s) | 20260506 22:25:58.299 | INFO | Downloaded 38GiB to GPU in 2.18102s (17.423GiB/s) | 20260506 22:26:03.443 | INFO | Analyzing chunk frames = [4928000,4928512) | 20260506 22:26:03.443 | INFO | MemInfo: { | 20260506 22:26:03.443 | INFO | "gpu" : " 17 GB", | 20260506 22:26:03.443 | INFO | "huge" : "", | 20260506 22:26:03.443 | INFO | "huge_shared" : " 27 GB", | 20260506 22:26:03.443 | INFO | "malloc" : " 14 GB", | 20260506 22:26:03.443 | INFO | "pinned" : " 29 GB" | 20260506 22:26:03.443 | INFO | } | 20260506 22:26:03.443 | INFO | Perf report for the last 5.14421 seconds: | 20260506 22:26:03.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:26:03.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:03.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:03.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:03.443 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 22:26:03.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:03.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:03.443 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:26:03.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:03.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:03.444 | INFO | Uploaded 36GiB to GPU in 2.00273s (17.9755GiB/s) | 20260506 22:26:03.444 | INFO | Downloaded 34GiB to GPU in 1.93399s (17.5802GiB/s) | 20260506 22:26:08.580 | INFO | Analyzing chunk frames = [4928512,4929024) | 20260506 22:26:08.580 | INFO | MemInfo: { | 20260506 22:26:08.580 | INFO | "gpu" : " 17 GB", | 20260506 22:26:08.580 | INFO | "huge" : "", | 20260506 22:26:08.580 | INFO | "huge_shared" : " 27 GB", | 20260506 22:26:08.580 | INFO | "malloc" : " 14 GB", | 20260506 22:26:08.581 | INFO | "pinned" : " 29 GB" | 20260506 22:26:08.581 | INFO | } | 20260506 22:26:08.581 | INFO | Perf report for the last 5.13724 seconds: | 20260506 22:26:08.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:26:08.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:08.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:08.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:08.581 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 22:26:08.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:08.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:08.581 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:26:08.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:08.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:08.581 | INFO | Uploaded 36GiB to GPU in 1.98375s (18.1474GiB/s) | 20260506 22:26:08.581 | INFO | Downloaded 34GiB to GPU in 1.93804s (17.5435GiB/s) | 20260506 22:26:13.716 | INFO | Analyzing chunk frames = [4929024,4929536) | 20260506 22:26:13.716 | INFO | MemInfo: { | 20260506 22:26:13.716 | INFO | "gpu" : " 17 GB", | 20260506 22:26:13.716 | INFO | "huge" : "", | 20260506 22:26:13.716 | INFO | "huge_shared" : " 27 GB", | 20260506 22:26:13.716 | INFO | "malloc" : " 14 GB", | 20260506 22:26:13.716 | INFO | "pinned" : " 29 GB" | 20260506 22:26:13.716 | INFO | } | 20260506 22:26:13.716 | INFO | Perf report for the last 5.13539 seconds: | 20260506 22:26:13.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:26:13.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:13.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:13.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:13.716 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 22:26:13.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:13.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:13.716 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:26:13.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:13.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:13.716 | INFO | Uploaded 36GiB to GPU in 1.97811s (18.1992GiB/s) | 20260506 22:26:13.716 | INFO | Downloaded 34GiB to GPU in 1.91149s (17.7872GiB/s) | 20260506 22:26:18.852 | INFO | Analyzing chunk frames = [4929536,4930048) | 20260506 22:26:18.852 | INFO | MemInfo: { | 20260506 22:26:18.852 | INFO | "gpu" : " 17 GB", | 20260506 22:26:18.852 | INFO | "huge" : "", | 20260506 22:26:18.852 | INFO | "huge_shared" : " 27 GB", | 20260506 22:26:18.852 | INFO | "malloc" : " 14 GB", | 20260506 22:26:18.852 | INFO | "pinned" : " 29 GB" | 20260506 22:26:18.852 | INFO | } | 20260506 22:26:18.852 | INFO | Perf report for the last 5.13603 seconds: | 20260506 22:26:18.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:26:18.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:18.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:18.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:18.852 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 22:26:18.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:18.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:18.852 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:26:18.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:18.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:18.852 | INFO | Uploaded 36GiB to GPU in 1.97105s (18.2644GiB/s) | 20260506 22:26:18.852 | INFO | Downloaded 34GiB to GPU in 1.93073s (17.6099GiB/s) | 20260506 22:26:23.990 | INFO | Analyzing chunk frames = [4930048,4930560) | 20260506 22:26:23.990 | INFO | MemInfo: { | 20260506 22:26:23.990 | INFO | "gpu" : " 17 GB", | 20260506 22:26:23.990 | INFO | "huge" : "", | 20260506 22:26:23.990 | INFO | "huge_shared" : " 27 GB", | 20260506 22:26:23.991 | INFO | "malloc" : " 14 GB", | 20260506 22:26:23.991 | INFO | "pinned" : " 29 GB" | 20260506 22:26:23.991 | INFO | } | 20260506 22:26:23.991 | INFO | Perf report for the last 5.13872 seconds: | 20260506 22:26:23.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:26:23.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:23.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:23.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:23.991 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 22:26:23.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:23.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:23.991 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 22:26:23.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:23.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:23.991 | INFO | Uploaded 36GiB to GPU in 1.98789s (18.1097GiB/s) | 20260506 22:26:23.991 | INFO | Downloaded 34GiB to GPU in 1.95527s (17.3889GiB/s) | 20260506 22:26:29.134 | INFO | Analyzing chunk frames = [4930560,4931072) | 20260506 22:26:29.134 | INFO | MemInfo: { | 20260506 22:26:29.134 | INFO | "gpu" : " 17 GB", | 20260506 22:26:29.134 | INFO | "huge" : "", | 20260506 22:26:29.134 | INFO | "huge_shared" : " 27 GB", | 20260506 22:26:29.134 | INFO | "malloc" : " 14 GB", | 20260506 22:26:29.134 | INFO | "pinned" : " 29 GB" | 20260506 22:26:29.134 | INFO | } | 20260506 22:26:29.134 | INFO | Perf report for the last 5.14353 seconds: | 20260506 22:26:29.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:26:29.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:29.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:29.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:29.134 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 22:26:29.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:29.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:29.134 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 22:26:29.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:29.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:29.135 | INFO | Uploaded 36GiB to GPU in 1.95765s (18.3894GiB/s) | 20260506 22:26:29.135 | INFO | Downloaded 34GiB to GPU in 1.97896s (17.1807GiB/s) | 20260506 22:26:34.276 | INFO | Analyzing chunk frames = [4931072,4931584) | 20260506 22:26:34.276 | INFO | MemInfo: { | 20260506 22:26:34.276 | INFO | "gpu" : " 17 GB", | 20260506 22:26:34.276 | INFO | "huge" : "", | 20260506 22:26:34.276 | INFO | "huge_shared" : " 27 GB", | 20260506 22:26:34.276 | INFO | "malloc" : " 14 GB", | 20260506 22:26:34.276 | INFO | "pinned" : " 29 GB" | 20260506 22:26:34.276 | INFO | } | 20260506 22:26:34.276 | INFO | Perf report for the last 5.14149 seconds: | 20260506 22:26:34.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:26:34.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:34.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:34.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:34.276 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 22:26:34.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:34.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:34.276 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 22:26:34.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:34.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:34.276 | INFO | Uploaded 36GiB to GPU in 2.04981s (17.5626GiB/s) | 20260506 22:26:34.276 | INFO | Downloaded 34GiB to GPU in 2.00102s (16.9913GiB/s) | 20260506 22:26:34.276 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:26:34.276 | INFO | Total mem usage: 18 GiB | 20260506 22:26:34.276 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:26:34.276 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 22:26:34.276 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:26:34.276 | INFO | High Water Breakdown: | 20260506 22:26:34.276 | INFO | 60.1% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:26:34.276 | INFO | 39.7% : 5.68 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:26:34.276 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:26:34.276 | INFO | ------------------------------------------ | 20260506 22:26:34.276 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:26:34.276 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:26:34.276 | INFO | Total mem usage: 26.5 GiB | 20260506 22:26:34.276 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:26:34.276 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:26:34.276 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:26:34.276 | INFO | High Water Breakdown: | 20260506 22:26:34.276 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:26:34.276 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:26:34.276 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:26:34.276 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:26:34.276 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:26:34.276 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:26:34.277 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:26:34.277 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:26:34.277 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:26:34.277 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:26:34.277 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:26:34.277 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:26:34.277 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:26:34.277 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:26:34.277 | INFO | ------------------------------------------ | 20260506 22:26:34.277 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:26:34.277 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:26:34.277 | INFO | Total mem usage: 25.6 GiB | 20260506 22:26:34.277 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:26:34.277 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:26:34.277 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:26:34.277 | INFO | High Water Breakdown: | 20260506 22:26:34.277 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:26:34.277 | INFO | ------------------------------------------ | 20260506 22:26:34.277 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:26:34.277 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:26:34.277 | INFO | Total mem usage: 15.7 GiB | 20260506 22:26:34.277 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:26:34.277 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:26:34.277 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:26:34.277 | INFO | High Water Breakdown: | 20260506 22:26:34.277 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:26:34.277 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:26:34.277 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:26:34.277 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:26:34.277 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:26:34.277 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:26:34.277 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:26:34.277 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:26:34.277 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:26:34.277 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:26:34.277 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:26:34.277 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:26:34.277 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:26:34.277 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:26:34.277 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:26:34.277 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:26:34.277 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:26:34.277 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:26:34.277 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:26:34.277 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:26:34.277 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:26:34.277 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:26:34.277 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:26:34.277 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:26:34.277 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:26:34.277 | INFO | ------------------------------------------ | 20260506 22:26:34.277 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:26:34.277 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:26:34.277 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:26:34.277 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:26:34.277 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.60e+01, 1.33e+01 / 2.65e+00 | 20260506 22:26:34.278 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.02e-01, 9.36e-03 / 7.41e-03 | 20260506 22:26:34.278 | INFO | PerfReport: 11.65%, Baseline -- 3.62e+00 / 4.35e+00, 5.46e-01 / 3.47e-01 | 20260506 22:26:34.278 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.42e+00, 1.78e-01 / 1.48e-01 | 20260506 22:26:34.278 | INFO | PerfReport: 16.03%, DME -- 3.98e+01 / 4.45e+01, 4.25e+00 / 3.28e+00 | 20260506 22:26:34.278 | INFO | PerfReport: 57.35%, FrameLabeling -- 1.78e+01 / 2.13e+01, 1.39e+00 / 1.98e+00 | 20260506 22:26:34.278 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.20e+00 / 2.69e+00, 3.73e-01 / 2.47e-01 | 20260506 22:26:34.278 | INFO | PerfReport: 3.24%, Metrics -- 1.00e+00 / 1.34e+00, 2.67e-01 / 1.56e-01 | 20260506 22:26:34.278 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.43e-01 / 3.91e-01, 5.33e-02 / 2.09e-02 | 20260506 22:26:34.278 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:26:34.278 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:26:34.278 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:26:34.278 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 22:26:34.278 | INFO | PerfReport: 21.44%, Upload -- 2.07e+01 / 0.00e+00, 6.85e+00 / 0.00e+00 | 20260506 22:26:34.278 | INFO | PerfReport: 21.31%, Download -- 2.06e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 22:26:34.278 | INFO | PerfReport: 32.11%, Compute -- 3.11e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 22:26:34.278 | INFO | PerfReport: 1.17%, LockUpload -- 1.13e+00 / 0.00e+00, 4.48e+00 / 0.00e+00 | 20260506 22:26:34.278 | INFO | PerfReport: 1.35%, LockDownload -- 1.31e+00 / 0.00e+00, 3.23e+00 / 0.00e+00 | 20260506 22:26:34.278 | INFO | PerfReport: 22.61%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 22:26:34.278 | INFO | PerfReport: 0.02%, Misc -- 1.45e-02 / 0.00e+00, 2.38e-02 / 0.00e+00 | 20260506 22:26:34.386 | INFO | Starting a round of Baz IO | 20260506 22:26:37.510 | INFO | Populating RTMetrics report | 20260506 22:26:37.511 | INFO | RTMetrics report populated | 20260506 22:26:37.511 | INFO | Opening tmp RTMetrics json file | 20260506 22:26:37.511 | INFO | Serializing RTMetrics to json | 20260506 22:26:37.520 | INFO | Serialization done, swapping tmp file now... | 20260506 22:26:37.532 | INFO | RTMetrics done | 20260506 22:26:37.603 | INFO | Finished a round of Baz IO: Wrote 1.912 GiB in 3.217 seconds | 20260506 22:26:37.603 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.68% / 0.00% / 66.32% | 20260506 22:26:37.603 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.504994% / 3.97364e-06% / 5.5631e-05% | 20260506 22:26:37.603 | INFO | Baz IO overall: | 20260506 22:26:37.603 | INFO | BytesWritten : 2590891357557 | 20260506 22:26:37.603 | INFO | NumEvents : 1250085502291 | 20260506 22:26:39.418 | INFO | Analyzing chunk frames = [4931584,4932096) | 20260506 22:26:39.418 | INFO | MemInfo: { | 20260506 22:26:39.418 | INFO | "gpu" : " 17 GB", | 20260506 22:26:39.418 | INFO | "huge" : "", | 20260506 22:26:39.418 | INFO | "huge_shared" : " 27 GB", | 20260506 22:26:39.418 | INFO | "malloc" : " 14 GB", | 20260506 22:26:39.418 | INFO | "pinned" : " 29 GB" | 20260506 22:26:39.418 | INFO | } | 20260506 22:26:39.418 | INFO | Perf report for the last 5.14243 seconds: | 20260506 22:26:39.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:26:39.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:39.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:39.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:39.418 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 22:26:39.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:39.418 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 22:26:39.418 | INFO | PRE_HQ : 59.6%, 1.8, 56.9ms, 0 | 20260506 22:26:39.418 | INFO | BAZWRITER : 62.2%, 3.5, 111.0ms, 0 | 20260506 22:26:39.418 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:26:39.419 | INFO | Uploaded 36GiB to GPU in 2.02789s (17.7525GiB/s) | 20260506 22:26:39.419 | INFO | Downloaded 38GiB to GPU in 2.17951s (17.4352GiB/s) | 20260506 22:26:44.556 | INFO | Analyzing chunk frames = [4932096,4932608) | 20260506 22:26:44.556 | INFO | MemInfo: { | 20260506 22:26:44.556 | INFO | "gpu" : " 17 GB", | 20260506 22:26:44.556 | INFO | "huge" : "", | 20260506 22:26:44.556 | INFO | "huge_shared" : " 27 GB", | 20260506 22:26:44.556 | INFO | "malloc" : " 14 GB", | 20260506 22:26:44.556 | INFO | "pinned" : " 29 GB" | 20260506 22:26:44.556 | INFO | } | 20260506 22:26:44.556 | INFO | Perf report for the last 5.13807 seconds: | 20260506 22:26:44.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:26:44.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:44.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:44.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:44.556 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:26:44.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:44.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:44.556 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:26:44.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:44.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:44.557 | INFO | Uploaded 36GiB to GPU in 2.01402s (17.8747GiB/s) | 20260506 22:26:44.557 | INFO | Downloaded 34GiB to GPU in 1.92929s (17.6231GiB/s) | 20260506 22:26:49.696 | INFO | Analyzing chunk frames = [4932608,4933120) | 20260506 22:26:49.696 | INFO | MemInfo: { | 20260506 22:26:49.696 | INFO | "gpu" : " 17 GB", | 20260506 22:26:49.696 | INFO | "huge" : "", | 20260506 22:26:49.696 | INFO | "huge_shared" : " 27 GB", | 20260506 22:26:49.696 | INFO | "malloc" : " 14 GB", | 20260506 22:26:49.697 | INFO | "pinned" : " 29 GB" | 20260506 22:26:49.697 | INFO | } | 20260506 22:26:49.697 | INFO | Perf report for the last 5.14022 seconds: | 20260506 22:26:49.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:26:49.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:49.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:49.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:49.697 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 22:26:49.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:49.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:49.697 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:26:49.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:49.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:49.697 | INFO | Uploaded 36GiB to GPU in 1.9739s (18.238GiB/s) | 20260506 22:26:49.697 | INFO | Downloaded 34GiB to GPU in 1.92971s (17.6192GiB/s) | 20260506 22:26:54.830 | INFO | Analyzing chunk frames = [4933120,4933632) | 20260506 22:26:54.830 | INFO | MemInfo: { | 20260506 22:26:54.830 | INFO | "gpu" : " 17 GB", | 20260506 22:26:54.830 | INFO | "huge" : "", | 20260506 22:26:54.830 | INFO | "huge_shared" : " 27 GB", | 20260506 22:26:54.830 | INFO | "malloc" : " 14 GB", | 20260506 22:26:54.830 | INFO | "pinned" : " 29 GB" | 20260506 22:26:54.830 | INFO | } | 20260506 22:26:54.830 | INFO | Perf report for the last 5.13328 seconds: | 20260506 22:26:54.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:26:54.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:54.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:54.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:54.830 | INFO | ANALYSIS : 59.7%, 3.0, 92.4ms, 0 | 20260506 22:26:54.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:54.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:54.830 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:26:54.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:54.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:54.830 | INFO | Uploaded 36GiB to GPU in 1.94819s (18.4787GiB/s) | 20260506 22:26:54.830 | INFO | Downloaded 34GiB to GPU in 1.87394s (18.1436GiB/s) | 20260506 22:26:59.965 | INFO | Analyzing chunk frames = [4933632,4934144) | 20260506 22:26:59.965 | INFO | MemInfo: { | 20260506 22:26:59.965 | INFO | "gpu" : " 17 GB", | 20260506 22:26:59.965 | INFO | "huge" : "", | 20260506 22:26:59.965 | INFO | "huge_shared" : " 27 GB", | 20260506 22:26:59.965 | INFO | "malloc" : " 14 GB", | 20260506 22:26:59.965 | INFO | "pinned" : " 29 GB" | 20260506 22:26:59.966 | INFO | } | 20260506 22:26:59.966 | INFO | Perf report for the last 5.13577 seconds: | 20260506 22:26:59.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:26:59.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:59.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:59.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:59.966 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 22:26:59.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:59.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:59.966 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:26:59.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:59.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:26:59.966 | INFO | Uploaded 36GiB to GPU in 1.98903s (18.0993GiB/s) | 20260506 22:26:59.966 | INFO | Downloaded 34GiB to GPU in 1.92889s (17.6267GiB/s) | 20260506 22:27:05.111 | INFO | Analyzing chunk frames = [4934144,4934656) | 20260506 22:27:05.112 | INFO | MemInfo: { | 20260506 22:27:05.112 | INFO | "gpu" : " 17 GB", | 20260506 22:27:05.112 | INFO | "huge" : "", | 20260506 22:27:05.112 | INFO | "huge_shared" : " 27 GB", | 20260506 22:27:05.112 | INFO | "malloc" : " 14 GB", | 20260506 22:27:05.112 | INFO | "pinned" : " 29 GB" | 20260506 22:27:05.112 | INFO | } | 20260506 22:27:05.112 | INFO | Perf report for the last 5.14609 seconds: | 20260506 22:27:05.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:27:05.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:05.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:05.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:05.112 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 22:27:05.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:05.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:05.112 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 22:27:05.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:05.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:05.112 | INFO | Uploaded 36GiB to GPU in 2.01924s (17.8285GiB/s) | 20260506 22:27:05.112 | INFO | Downloaded 34GiB to GPU in 1.94801s (17.4537GiB/s) | 20260506 22:27:10.255 | INFO | Analyzing chunk frames = [4934656,4935168) | 20260506 22:27:10.255 | INFO | MemInfo: { | 20260506 22:27:10.255 | INFO | "gpu" : " 17 GB", | 20260506 22:27:10.255 | INFO | "huge" : "", | 20260506 22:27:10.255 | INFO | "huge_shared" : " 27 GB", | 20260506 22:27:10.255 | INFO | "malloc" : " 14 GB", | 20260506 22:27:10.255 | INFO | "pinned" : " 29 GB" | 20260506 22:27:10.255 | INFO | } | 20260506 22:27:10.255 | INFO | Perf report for the last 5.14366 seconds: | 20260506 22:27:10.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:27:10.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:10.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:10.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:10.256 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 22:27:10.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:10.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:10.256 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 22:27:10.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:10.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:10.256 | INFO | Uploaded 36GiB to GPU in 1.95326s (18.4307GiB/s) | 20260506 22:27:10.257 | INFO | Downloaded 34GiB to GPU in 1.99483s (17.044GiB/s) | 20260506 22:27:15.393 | INFO | Analyzing chunk frames = [4935168,4935680) | 20260506 22:27:15.393 | INFO | MemInfo: { | 20260506 22:27:15.393 | INFO | "gpu" : " 17 GB", | 20260506 22:27:15.393 | INFO | "huge" : "", | 20260506 22:27:15.393 | INFO | "huge_shared" : " 27 GB", | 20260506 22:27:15.393 | INFO | "malloc" : " 14 GB", | 20260506 22:27:15.393 | INFO | "pinned" : " 29 GB" | 20260506 22:27:15.393 | INFO | } | 20260506 22:27:15.393 | INFO | Perf report for the last 5.1377 seconds: | 20260506 22:27:15.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:27:15.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:15.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:15.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:15.393 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 22:27:15.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:15.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:15.393 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:27:15.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:15.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:15.393 | INFO | Uploaded 36GiB to GPU in 1.9743s (18.2343GiB/s) | 20260506 22:27:15.393 | INFO | Downloaded 34GiB to GPU in 1.99527s (17.0403GiB/s) | 20260506 22:27:15.492 | INFO | Starting a round of Baz IO | 20260506 22:27:18.623 | INFO | Populating RTMetrics report | 20260506 22:27:18.624 | INFO | RTMetrics report populated | 20260506 22:27:18.624 | INFO | Opening tmp RTMetrics json file | 20260506 22:27:18.624 | INFO | Serializing RTMetrics to json | 20260506 22:27:18.646 | INFO | Serialization done, swapping tmp file now... | 20260506 22:27:18.646 | INFO | RTMetrics done | 20260506 22:27:18.718 | INFO | Finished a round of Baz IO: Wrote 1.911 GiB in 3.225 seconds | 20260506 22:27:18.718 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.68% / 0.00% / 66.32% | 20260506 22:27:18.718 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.504045% / 0% / 3.97364e-05% | 20260506 22:27:18.718 | INFO | Baz IO overall: | 20260506 22:27:18.718 | INFO | BytesWritten : 2592943759663 | 20260506 22:27:18.718 | INFO | NumEvents : 1251065537545 | 20260506 22:27:20.529 | INFO | Analyzing chunk frames = [4935680,4936192) | 20260506 22:27:20.529 | INFO | MemInfo: { | 20260506 22:27:20.529 | INFO | "gpu" : " 17 GB", | 20260506 22:27:20.529 | INFO | "huge" : "", | 20260506 22:27:20.529 | INFO | "huge_shared" : " 27 GB", | 20260506 22:27:20.529 | INFO | "malloc" : " 14 GB", | 20260506 22:27:20.529 | INFO | "pinned" : " 29 GB" | 20260506 22:27:20.529 | INFO | } | 20260506 22:27:20.529 | INFO | Perf report for the last 5.13613 seconds: | 20260506 22:27:20.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:27:20.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:20.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:20.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:20.529 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 22:27:20.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:20.529 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 22:27:20.529 | INFO | PRE_HQ : 60.0%, 1.8, 57.0ms, 0 | 20260506 22:27:20.529 | INFO | BAZWRITER : 62.4%, 3.5, 112.1ms, 0 | 20260506 22:27:20.529 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:27:20.530 | INFO | Uploaded 36GiB to GPU in 2.02251s (17.7997GiB/s) | 20260506 22:27:20.530 | INFO | Downloaded 38GiB to GPU in 2.1759s (17.464GiB/s) | 20260506 22:27:25.666 | INFO | Analyzing chunk frames = [4936192,4936704) | 20260506 22:27:25.666 | INFO | MemInfo: { | 20260506 22:27:25.666 | INFO | "gpu" : " 17 GB", | 20260506 22:27:25.666 | INFO | "huge" : "", | 20260506 22:27:25.666 | INFO | "huge_shared" : " 27 GB", | 20260506 22:27:25.666 | INFO | "malloc" : " 14 GB", | 20260506 22:27:25.666 | INFO | "pinned" : " 29 GB" | 20260506 22:27:25.666 | INFO | } | 20260506 22:27:25.666 | INFO | Perf report for the last 5.13719 seconds: | 20260506 22:27:25.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:27:25.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:25.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:25.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:25.666 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:27:25.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:25.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:25.666 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 22:27:25.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:25.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:25.667 | INFO | Uploaded 36GiB to GPU in 2.00996s (17.9108GiB/s) | 20260506 22:27:25.667 | INFO | Downloaded 34GiB to GPU in 1.93008s (17.6158GiB/s) | 20260506 22:27:30.803 | INFO | Analyzing chunk frames = [4936704,4937216) | 20260506 22:27:30.803 | INFO | MemInfo: { | 20260506 22:27:30.803 | INFO | "gpu" : " 17 GB", | 20260506 22:27:30.803 | INFO | "huge" : "", | 20260506 22:27:30.803 | INFO | "huge_shared" : " 27 GB", | 20260506 22:27:30.803 | INFO | "malloc" : " 14 GB", | 20260506 22:27:30.803 | INFO | "pinned" : " 29 GB" | 20260506 22:27:30.803 | INFO | } | 20260506 22:27:30.803 | INFO | Perf report for the last 5.13673 seconds: | 20260506 22:27:30.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:27:30.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:30.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:30.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:30.803 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260506 22:27:30.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:30.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:30.803 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:27:30.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:30.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:30.804 | INFO | Uploaded 36GiB to GPU in 1.96834s (18.2896GiB/s) | 20260506 22:27:30.804 | INFO | Downloaded 34GiB to GPU in 1.93016s (17.6151GiB/s) | 20260506 22:27:35.940 | INFO | Analyzing chunk frames = [4937216,4937728) | 20260506 22:27:35.940 | INFO | MemInfo: { | 20260506 22:27:35.940 | INFO | "gpu" : " 17 GB", | 20260506 22:27:35.940 | INFO | "huge" : "", | 20260506 22:27:35.940 | INFO | "huge_shared" : " 27 GB", | 20260506 22:27:35.940 | INFO | "malloc" : " 14 GB", | 20260506 22:27:35.940 | INFO | "pinned" : " 29 GB" | 20260506 22:27:35.940 | INFO | } | 20260506 22:27:35.940 | INFO | Perf report for the last 5.13722 seconds: | 20260506 22:27:35.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:27:35.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:35.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:35.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:35.941 | INFO | ANALYSIS : 59.6%, 3.0, 92.3ms, 0 | 20260506 22:27:35.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:35.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:35.941 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:27:35.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:35.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:35.941 | INFO | Uploaded 36GiB to GPU in 1.9432s (18.5262GiB/s) | 20260506 22:27:35.941 | INFO | Downloaded 34GiB to GPU in 1.87838s (18.1007GiB/s) | 20260506 22:27:41.082 | INFO | Analyzing chunk frames = [4937728,4938240) | 20260506 22:27:41.082 | INFO | MemInfo: { | 20260506 22:27:41.082 | INFO | "gpu" : " 17 GB", | 20260506 22:27:41.082 | INFO | "huge" : "", | 20260506 22:27:41.082 | INFO | "huge_shared" : " 27 GB", | 20260506 22:27:41.082 | INFO | "malloc" : " 14 GB", | 20260506 22:27:41.082 | INFO | "pinned" : " 29 GB" | 20260506 22:27:41.082 | INFO | } | 20260506 22:27:41.082 | INFO | Perf report for the last 5.1419 seconds: | 20260506 22:27:41.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:27:41.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:41.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:41.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:41.082 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 22:27:41.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:41.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:41.082 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:27:41.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:41.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:41.083 | INFO | Uploaded 36GiB to GPU in 1.98759s (18.1124GiB/s) | 20260506 22:27:41.083 | INFO | Downloaded 34GiB to GPU in 1.91717s (17.7345GiB/s) | 20260506 22:27:46.225 | INFO | Analyzing chunk frames = [4938240,4938752) | 20260506 22:27:46.225 | INFO | MemInfo: { | 20260506 22:27:46.225 | INFO | "gpu" : " 17 GB", | 20260506 22:27:46.225 | INFO | "huge" : "", | 20260506 22:27:46.225 | INFO | "huge_shared" : " 27 GB", | 20260506 22:27:46.225 | INFO | "malloc" : " 14 GB", | 20260506 22:27:46.225 | INFO | "pinned" : " 29 GB" | 20260506 22:27:46.225 | INFO | } | 20260506 22:27:46.225 | INFO | Perf report for the last 5.1424 seconds: | 20260506 22:27:46.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:27:46.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:46.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:46.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:46.225 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 22:27:46.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:46.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:46.225 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 22:27:46.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:46.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:46.225 | INFO | Uploaded 36GiB to GPU in 1.98918s (18.0979GiB/s) | 20260506 22:27:46.225 | INFO | Downloaded 34GiB to GPU in 1.95582s (17.384GiB/s) | 20260506 22:27:51.370 | INFO | Analyzing chunk frames = [4938752,4939264) | 20260506 22:27:51.370 | INFO | MemInfo: { | 20260506 22:27:51.370 | INFO | "gpu" : " 17 GB", | 20260506 22:27:51.370 | INFO | "huge" : "", | 20260506 22:27:51.370 | INFO | "huge_shared" : " 27 GB", | 20260506 22:27:51.370 | INFO | "malloc" : " 14 GB", | 20260506 22:27:51.370 | INFO | "pinned" : " 29 GB" | 20260506 22:27:51.370 | INFO | } | 20260506 22:27:51.370 | INFO | Perf report for the last 5.14492 seconds: | 20260506 22:27:51.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:27:51.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:51.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:51.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:51.370 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 22:27:51.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:51.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:51.370 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 22:27:51.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:51.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:51.370 | INFO | Uploaded 36GiB to GPU in 1.95813s (18.3849GiB/s) | 20260506 22:27:51.370 | INFO | Downloaded 34GiB to GPU in 1.98786s (17.1038GiB/s) | 20260506 22:27:56.509 | INFO | Analyzing chunk frames = [4939264,4939776) | 20260506 22:27:56.509 | INFO | MemInfo: { | 20260506 22:27:56.509 | INFO | "gpu" : " 17 GB", | 20260506 22:27:56.509 | INFO | "huge" : "", | 20260506 22:27:56.509 | INFO | "huge_shared" : " 27 GB", | 20260506 22:27:56.509 | INFO | "malloc" : " 14 GB", | 20260506 22:27:56.509 | INFO | "pinned" : " 29 GB" | 20260506 22:27:56.509 | INFO | } | 20260506 22:27:56.509 | INFO | Perf report for the last 5.13921 seconds: | 20260506 22:27:56.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:27:56.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:56.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:56.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:56.509 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:27:56.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:56.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:56.509 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:27:56.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:56.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:27:56.509 | INFO | Uploaded 36GiB to GPU in 1.97716s (18.2079GiB/s) | 20260506 22:27:56.509 | INFO | Downloaded 34GiB to GPU in 1.98407s (17.1365GiB/s) | 20260506 22:27:56.509 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:27:56.510 | INFO | Total mem usage: 18 GiB | 20260506 22:27:56.510 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:27:56.510 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:27:56.510 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:27:56.510 | INFO | High Water Breakdown: | 20260506 22:27:56.510 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:27:56.510 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:27:56.510 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:27:56.510 | INFO | ------------------------------------------ | 20260506 22:27:56.510 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:27:56.510 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:27:56.510 | INFO | Total mem usage: 26.5 GiB | 20260506 22:27:56.510 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:27:56.510 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:27:56.510 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:27:56.510 | INFO | High Water Breakdown: | 20260506 22:27:56.510 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:27:56.510 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:27:56.510 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:27:56.510 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:27:56.510 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:27:56.510 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:27:56.510 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:27:56.510 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:27:56.510 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:27:56.510 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:27:56.510 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:27:56.510 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:27:56.510 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:27:56.510 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:27:56.510 | INFO | ------------------------------------------ | 20260506 22:27:56.510 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:27:56.510 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:27:56.510 | INFO | Total mem usage: 25.6 GiB | 20260506 22:27:56.510 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:27:56.510 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:27:56.510 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:27:56.510 | INFO | High Water Breakdown: | 20260506 22:27:56.510 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:27:56.510 | INFO | ------------------------------------------ | 20260506 22:27:56.510 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:27:56.510 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:27:56.510 | INFO | Total mem usage: 15.7 GiB | 20260506 22:27:56.510 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:27:56.510 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:27:56.510 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:27:56.510 | INFO | High Water Breakdown: | 20260506 22:27:56.510 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:27:56.510 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:27:56.510 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:27:56.510 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:27:56.510 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:27:56.510 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:27:56.510 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:27:56.510 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:27:56.510 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:27:56.510 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:27:56.510 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:27:56.510 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:27:56.510 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:27:56.510 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:27:56.510 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:27:56.510 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:27:56.510 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:27:56.510 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:27:56.510 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:27:56.510 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:27:56.510 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:27:56.511 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:27:56.511 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:27:56.511 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:27:56.511 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:27:56.511 | INFO | ------------------------------------------ | 20260506 22:27:56.511 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:27:56.511 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:27:56.511 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:27:56.511 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:27:56.511 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.55e+01, 1.34e+01 / 2.14e+00 | 20260506 22:27:56.511 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.31e-03 / 5.80e-03 | 20260506 22:27:56.511 | INFO | PerfReport: 11.65%, Baseline -- 3.62e+00 / 4.39e+00, 5.45e-01 / 3.49e-01 | 20260506 22:27:56.511 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.47e+00, 1.81e-01 / 1.13e-01 | 20260506 22:27:56.511 | INFO | PerfReport: 16.01%, DME -- 3.97e+01 / 4.42e+01, 3.95e+00 / 2.38e+00 | 20260506 22:27:56.511 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.37e+00 / 1.37e+00 | 20260506 22:27:56.511 | INFO | PerfReport: 7.11%, PulseAccumulating -- 2.20e+00 / 2.80e+00, 3.83e-01 / 1.95e-01 | 20260506 22:27:56.511 | INFO | PerfReport: 3.23%, Metrics -- 1.00e+00 / 1.35e+00, 2.63e-01 / 1.27e-01 | 20260506 22:27:56.511 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.42e-01 / 3.97e-01, 5.31e-02 / 2.62e-02 | 20260506 22:27:56.511 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:27:56.511 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:27:56.511 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:27:56.511 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 22:27:56.511 | INFO | PerfReport: 21.41%, Upload -- 2.07e+01 / 0.00e+00, 6.85e+00 / 0.00e+00 | 20260506 22:27:56.511 | INFO | PerfReport: 21.26%, Download -- 2.05e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 22:27:56.511 | INFO | PerfReport: 32.14%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 22:27:56.511 | INFO | PerfReport: 1.19%, LockUpload -- 1.15e+00 / 0.00e+00, 4.56e+00 / 0.00e+00 | 20260506 22:27:56.511 | INFO | PerfReport: 1.29%, LockDownload -- 1.25e+00 / 0.00e+00, 3.18e+00 / 0.00e+00 | 20260506 22:27:56.511 | INFO | PerfReport: 22.69%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 22:27:56.511 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.46e-02 / 0.00e+00 | 20260506 22:27:56.608 | INFO | Starting a round of Baz IO | 20260506 22:27:59.748 | INFO | Populating RTMetrics report | 20260506 22:27:59.748 | INFO | RTMetrics report populated | 20260506 22:27:59.748 | INFO | Opening tmp RTMetrics json file | 20260506 22:27:59.749 | INFO | Serializing RTMetrics to json | 20260506 22:27:59.756 | INFO | Serialization done, swapping tmp file now... | 20260506 22:27:59.770 | INFO | RTMetrics done | 20260506 22:27:59.842 | INFO | Finished a round of Baz IO: Wrote 1.913 GiB in 3.233 seconds | 20260506 22:27:59.842 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.68% / 0.00% / 66.32% | 20260506 22:27:59.842 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.502296% / 3.97364e-06% / 4.37101e-05% | 20260506 22:27:59.842 | INFO | Baz IO overall: | 20260506 22:27:59.842 | INFO | BytesWritten : 2594997779251 | 20260506 22:27:59.842 | INFO | NumEvents : 1252046396395 | 20260506 22:28:01.647 | INFO | Analyzing chunk frames = [4939776,4940288) | 20260506 22:28:01.647 | INFO | MemInfo: { | 20260506 22:28:01.647 | INFO | "gpu" : " 17 GB", | 20260506 22:28:01.647 | INFO | "huge" : "", | 20260506 22:28:01.647 | INFO | "huge_shared" : " 27 GB", | 20260506 22:28:01.647 | INFO | "malloc" : " 14 GB", | 20260506 22:28:01.647 | INFO | "pinned" : " 29 GB" | 20260506 22:28:01.647 | INFO | } | 20260506 22:28:01.647 | INFO | Perf report for the last 5.13764 seconds: | 20260506 22:28:01.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:28:01.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:01.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:01.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:01.647 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260506 22:28:01.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:01.647 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 22:28:01.647 | INFO | PRE_HQ : 60.1%, 1.8, 57.0ms, 0 | 20260506 22:28:01.647 | INFO | BAZWRITER : 62.5%, 3.4, 110.3ms, 0 | 20260506 22:28:01.647 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:28:01.647 | INFO | Uploaded 36GiB to GPU in 2.04631s (17.5927GiB/s) | 20260506 22:28:01.647 | INFO | Downloaded 38GiB to GPU in 2.17999s (17.4313GiB/s) | 20260506 22:28:06.784 | INFO | Analyzing chunk frames = [4940288,4940800) | 20260506 22:28:06.784 | INFO | MemInfo: { | 20260506 22:28:06.784 | INFO | "gpu" : " 17 GB", | 20260506 22:28:06.784 | INFO | "huge" : "", | 20260506 22:28:06.784 | INFO | "huge_shared" : " 27 GB", | 20260506 22:28:06.784 | INFO | "malloc" : " 14 GB", | 20260506 22:28:06.784 | INFO | "pinned" : " 29 GB" | 20260506 22:28:06.784 | INFO | } | 20260506 22:28:06.784 | INFO | Perf report for the last 5.13726 seconds: | 20260506 22:28:06.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:28:06.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:06.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:06.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:06.784 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:28:06.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:06.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:06.784 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:28:06.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:06.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:06.784 | INFO | Uploaded 36GiB to GPU in 2.01644s (17.8533GiB/s) | 20260506 22:28:06.784 | INFO | Downloaded 34GiB to GPU in 1.93201s (17.5982GiB/s) | 20260506 22:28:11.919 | INFO | Analyzing chunk frames = [4940800,4941312) | 20260506 22:28:11.919 | INFO | MemInfo: { | 20260506 22:28:11.919 | INFO | "gpu" : " 17 GB", | 20260506 22:28:11.919 | INFO | "huge" : "", | 20260506 22:28:11.919 | INFO | "huge_shared" : " 27 GB", | 20260506 22:28:11.919 | INFO | "malloc" : " 14 GB", | 20260506 22:28:11.919 | INFO | "pinned" : " 29 GB" | 20260506 22:28:11.919 | INFO | } | 20260506 22:28:11.919 | INFO | Perf report for the last 5.13511 seconds: | 20260506 22:28:11.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:28:11.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:11.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:11.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:11.919 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260506 22:28:11.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:11.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:11.919 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:28:11.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:11.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:11.919 | INFO | Uploaded 36GiB to GPU in 1.9795s (18.1864GiB/s) | 20260506 22:28:11.919 | INFO | Downloaded 34GiB to GPU in 1.92769s (17.6377GiB/s) | 20260506 22:28:17.059 | INFO | Analyzing chunk frames = [4941312,4941824) | 20260506 22:28:17.059 | INFO | MemInfo: { | 20260506 22:28:17.059 | INFO | "gpu" : " 17 GB", | 20260506 22:28:17.059 | INFO | "huge" : "", | 20260506 22:28:17.059 | INFO | "huge_shared" : " 27 GB", | 20260506 22:28:17.059 | INFO | "malloc" : " 14 GB", | 20260506 22:28:17.059 | INFO | "pinned" : " 29 GB" | 20260506 22:28:17.059 | INFO | } | 20260506 22:28:17.059 | INFO | Perf report for the last 5.14003 seconds: | 20260506 22:28:17.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:28:17.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:17.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:17.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:17.059 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 22:28:17.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:17.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:17.059 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:28:17.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:17.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:17.059 | INFO | Uploaded 36GiB to GPU in 1.97354s (18.2414GiB/s) | 20260506 22:28:17.059 | INFO | Downloaded 34GiB to GPU in 1.87979s (18.0872GiB/s) | 20260506 22:28:22.207 | INFO | Analyzing chunk frames = [4941824,4942336) | 20260506 22:28:22.207 | INFO | MemInfo: { | 20260506 22:28:22.207 | INFO | "gpu" : " 17 GB", | 20260506 22:28:22.207 | INFO | "huge" : "", | 20260506 22:28:22.207 | INFO | "huge_shared" : " 27 GB", | 20260506 22:28:22.207 | INFO | "malloc" : " 14 GB", | 20260506 22:28:22.208 | INFO | "pinned" : " 29 GB" | 20260506 22:28:22.208 | INFO | } | 20260506 22:28:22.208 | INFO | Perf report for the last 5.14861 seconds: | 20260506 22:28:22.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:28:22.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:22.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:22.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:22.208 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260506 22:28:22.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:22.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:22.208 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:28:22.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:22.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:22.208 | INFO | Uploaded 36GiB to GPU in 2.00369s (17.9669GiB/s) | 20260506 22:28:22.208 | INFO | Downloaded 34GiB to GPU in 1.9186s (17.7213GiB/s) | 20260506 22:28:27.339 | INFO | Analyzing chunk frames = [4942336,4942848) | 20260506 22:28:27.339 | INFO | MemInfo: { | 20260506 22:28:27.339 | INFO | "gpu" : " 17 GB", | 20260506 22:28:27.339 | INFO | "huge" : "", | 20260506 22:28:27.339 | INFO | "huge_shared" : " 27 GB", | 20260506 22:28:27.339 | INFO | "malloc" : " 14 GB", | 20260506 22:28:27.339 | INFO | "pinned" : " 29 GB" | 20260506 22:28:27.339 | INFO | } | 20260506 22:28:27.339 | INFO | Perf report for the last 5.13147 seconds: | 20260506 22:28:27.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:28:27.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:27.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:27.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:27.339 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260506 22:28:27.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:27.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:27.339 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 22:28:27.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:27.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:27.340 | INFO | Uploaded 36GiB to GPU in 2.04465s (17.6069GiB/s) | 20260506 22:28:27.340 | INFO | Downloaded 34GiB to GPU in 1.94216s (17.5063GiB/s) | 20260506 22:28:32.478 | INFO | Analyzing chunk frames = [4942848,4943360) | 20260506 22:28:32.478 | INFO | MemInfo: { | 20260506 22:28:32.478 | INFO | "gpu" : " 17 GB", | 20260506 22:28:32.478 | INFO | "huge" : "", | 20260506 22:28:32.478 | INFO | "huge_shared" : " 27 GB", | 20260506 22:28:32.478 | INFO | "malloc" : " 14 GB", | 20260506 22:28:32.478 | INFO | "pinned" : " 29 GB" | 20260506 22:28:32.478 | INFO | } | 20260506 22:28:32.478 | INFO | Perf report for the last 5.1391 seconds: | 20260506 22:28:32.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:28:32.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:32.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:32.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:32.478 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 22:28:32.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:32.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:32.479 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 22:28:32.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:32.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:32.479 | INFO | Uploaded 36GiB to GPU in 1.97655s (18.2136GiB/s) | 20260506 22:28:32.479 | INFO | Downloaded 34GiB to GPU in 2.01001s (16.9153GiB/s) | 20260506 22:28:37.618 | INFO | Analyzing chunk frames = [4943360,4943872) | 20260506 22:28:37.618 | INFO | MemInfo: { | 20260506 22:28:37.618 | INFO | "gpu" : " 17 GB", | 20260506 22:28:37.618 | INFO | "huge" : "", | 20260506 22:28:37.618 | INFO | "huge_shared" : " 27 GB", | 20260506 22:28:37.618 | INFO | "malloc" : " 14 GB", | 20260506 22:28:37.618 | INFO | "pinned" : " 29 GB" | 20260506 22:28:37.618 | INFO | } | 20260506 22:28:37.618 | INFO | Perf report for the last 5.1402 seconds: | 20260506 22:28:37.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:28:37.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:37.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:37.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:37.619 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:28:37.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:37.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:37.619 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:28:37.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:37.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:37.619 | INFO | Uploaded 36GiB to GPU in 1.99473s (18.0475GiB/s) | 20260506 22:28:37.619 | INFO | Downloaded 34GiB to GPU in 1.98837s (17.0995GiB/s) | 20260506 22:28:37.717 | INFO | Starting a round of Baz IO | 20260506 22:28:40.851 | INFO | Populating RTMetrics report | 20260506 22:28:40.852 | INFO | RTMetrics report populated | 20260506 22:28:40.852 | INFO | Opening tmp RTMetrics json file | 20260506 22:28:40.852 | INFO | Serializing RTMetrics to json | 20260506 22:28:40.855 | INFO | Serialization done, swapping tmp file now... | 20260506 22:28:40.861 | INFO | RTMetrics done | 20260506 22:28:40.946 | INFO | Finished a round of Baz IO: Wrote 1.91 GiB in 3.228 seconds | 20260506 22:28:40.946 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.67% / 0.00% / 66.32% | 20260506 22:28:40.946 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.501076% / 1.19209e-05% / 7.15256e-05% | 20260506 22:28:40.946 | INFO | Baz IO overall: | 20260506 22:28:40.946 | INFO | BytesWritten : 2597049038688 | 20260506 22:28:40.946 | INFO | NumEvents : 1253025540608 | 20260506 22:28:42.756 | INFO | Analyzing chunk frames = [4943872,4944384) | 20260506 22:28:42.756 | INFO | MemInfo: { | 20260506 22:28:42.756 | INFO | "gpu" : " 17 GB", | 20260506 22:28:42.756 | INFO | "huge" : "", | 20260506 22:28:42.756 | INFO | "huge_shared" : " 27 GB", | 20260506 22:28:42.756 | INFO | "malloc" : " 14 GB", | 20260506 22:28:42.756 | INFO | "pinned" : " 29 GB" | 20260506 22:28:42.756 | INFO | } | 20260506 22:28:42.757 | INFO | Perf report for the last 5.13808 seconds: | 20260506 22:28:42.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:28:42.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:42.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:42.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:42.757 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 22:28:42.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:42.757 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 22:28:42.757 | INFO | PRE_HQ : 59.6%, 1.8, 56.8ms, 0 | 20260506 22:28:42.757 | INFO | BAZWRITER : 62.4%, 3.4, 110.7ms, 0 | 20260506 22:28:42.757 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:28:42.757 | INFO | Uploaded 36GiB to GPU in 2.01716s (17.8469GiB/s) | 20260506 22:28:42.757 | INFO | Downloaded 38GiB to GPU in 2.16606s (17.5434GiB/s) | 20260506 22:28:47.894 | INFO | Analyzing chunk frames = [4944384,4944896) | 20260506 22:28:47.894 | INFO | MemInfo: { | 20260506 22:28:47.894 | INFO | "gpu" : " 17 GB", | 20260506 22:28:47.894 | INFO | "huge" : "", | 20260506 22:28:47.894 | INFO | "huge_shared" : " 27 GB", | 20260506 22:28:47.894 | INFO | "malloc" : " 14 GB", | 20260506 22:28:47.894 | INFO | "pinned" : " 29 GB" | 20260506 22:28:47.894 | INFO | } | 20260506 22:28:47.894 | INFO | Perf report for the last 5.13776 seconds: | 20260506 22:28:47.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:28:47.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:47.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:47.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:47.895 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:28:47.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:47.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:47.895 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:28:47.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:47.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:47.895 | INFO | Uploaded 36GiB to GPU in 2.01762s (17.8428GiB/s) | 20260506 22:28:47.895 | INFO | Downloaded 34GiB to GPU in 1.92684s (17.6455GiB/s) | 20260506 22:28:53.034 | INFO | Analyzing chunk frames = [4944896,4945408) | 20260506 22:28:53.034 | INFO | MemInfo: { | 20260506 22:28:53.034 | INFO | "gpu" : " 17 GB", | 20260506 22:28:53.034 | INFO | "huge" : "", | 20260506 22:28:53.034 | INFO | "huge_shared" : " 27 GB", | 20260506 22:28:53.034 | INFO | "malloc" : " 14 GB", | 20260506 22:28:53.034 | INFO | "pinned" : " 29 GB" | 20260506 22:28:53.034 | INFO | } | 20260506 22:28:53.034 | INFO | Perf report for the last 5.13982 seconds: | 20260506 22:28:53.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:28:53.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:53.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:53.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:53.034 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 22:28:53.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:53.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:53.034 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:28:53.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:53.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:53.035 | INFO | Uploaded 36GiB to GPU in 1.98468s (18.139GiB/s) | 20260506 22:28:53.035 | INFO | Downloaded 34GiB to GPU in 1.92709s (17.6432GiB/s) | 20260506 22:28:58.175 | INFO | Analyzing chunk frames = [4945408,4945920) | 20260506 22:28:58.176 | INFO | MemInfo: { | 20260506 22:28:58.176 | INFO | "gpu" : " 17 GB", | 20260506 22:28:58.176 | INFO | "huge" : "", | 20260506 22:28:58.176 | INFO | "huge_shared" : " 27 GB", | 20260506 22:28:58.176 | INFO | "malloc" : " 14 GB", | 20260506 22:28:58.176 | INFO | "pinned" : " 29 GB" | 20260506 22:28:58.176 | INFO | } | 20260506 22:28:58.176 | INFO | Perf report for the last 5.1414 seconds: | 20260506 22:28:58.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:28:58.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:58.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:58.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:58.176 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 22:28:58.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:58.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:58.176 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:28:58.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:58.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:28:58.176 | INFO | Uploaded 36GiB to GPU in 1.96262s (18.3428GiB/s) | 20260506 22:28:58.176 | INFO | Downloaded 34GiB to GPU in 1.87771s (18.1071GiB/s) | 20260506 22:29:03.320 | INFO | Analyzing chunk frames = [4945920,4946432) | 20260506 22:29:03.320 | INFO | MemInfo: { | 20260506 22:29:03.320 | INFO | "gpu" : " 17 GB", | 20260506 22:29:03.320 | INFO | "huge" : "", | 20260506 22:29:03.320 | INFO | "huge_shared" : " 27 GB", | 20260506 22:29:03.320 | INFO | "malloc" : " 14 GB", | 20260506 22:29:03.320 | INFO | "pinned" : " 29 GB" | 20260506 22:29:03.320 | INFO | } | 20260506 22:29:03.320 | INFO | Perf report for the last 5.14426 seconds: | 20260506 22:29:03.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:29:03.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:03.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:03.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:03.320 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 22:29:03.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:03.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:03.320 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:29:03.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:03.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:03.320 | INFO | Uploaded 36GiB to GPU in 2.0027s (17.9758GiB/s) | 20260506 22:29:03.320 | INFO | Downloaded 34GiB to GPU in 1.92153s (17.6943GiB/s) | 20260506 22:29:08.463 | INFO | Analyzing chunk frames = [4946432,4946944) | 20260506 22:29:08.463 | INFO | MemInfo: { | 20260506 22:29:08.463 | INFO | "gpu" : " 17 GB", | 20260506 22:29:08.463 | INFO | "huge" : "", | 20260506 22:29:08.463 | INFO | "huge_shared" : " 27 GB", | 20260506 22:29:08.463 | INFO | "malloc" : " 14 GB", | 20260506 22:29:08.463 | INFO | "pinned" : " 29 GB" | 20260506 22:29:08.463 | INFO | } | 20260506 22:29:08.463 | INFO | Perf report for the last 5.14323 seconds: | 20260506 22:29:08.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:29:08.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:08.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:08.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:08.463 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 22:29:08.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:08.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:08.463 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 22:29:08.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:08.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:08.464 | INFO | Uploaded 36GiB to GPU in 2.02693s (17.7609GiB/s) | 20260506 22:29:08.464 | INFO | Downloaded 34GiB to GPU in 1.9301s (17.6157GiB/s) | 20260506 22:29:13.594 | INFO | Analyzing chunk frames = [4946944,4947456) | 20260506 22:29:13.594 | INFO | MemInfo: { | 20260506 22:29:13.594 | INFO | "gpu" : " 17 GB", | 20260506 22:29:13.594 | INFO | "huge" : "", | 20260506 22:29:13.594 | INFO | "huge_shared" : " 27 GB", | 20260506 22:29:13.594 | INFO | "malloc" : " 14 GB", | 20260506 22:29:13.594 | INFO | "pinned" : " 29 GB" | 20260506 22:29:13.594 | INFO | } | 20260506 22:29:13.594 | INFO | Perf report for the last 5.13064 seconds: | 20260506 22:29:13.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:29:13.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:13.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:13.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:13.594 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:29:13.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:13.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:13.594 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 22:29:13.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:13.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:13.594 | INFO | Uploaded 36GiB to GPU in 1.94516s (18.5074GiB/s) | 20260506 22:29:13.594 | INFO | Downloaded 34GiB to GPU in 1.97412s (17.2229GiB/s) | 20260506 22:29:18.731 | INFO | Analyzing chunk frames = [4947456,4947968) | 20260506 22:29:18.731 | INFO | MemInfo: { | 20260506 22:29:18.731 | INFO | "gpu" : " 17 GB", | 20260506 22:29:18.731 | INFO | "huge" : "", | 20260506 22:29:18.731 | INFO | "huge_shared" : " 27 GB", | 20260506 22:29:18.731 | INFO | "malloc" : " 14 GB", | 20260506 22:29:18.731 | INFO | "pinned" : " 29 GB" | 20260506 22:29:18.731 | INFO | } | 20260506 22:29:18.731 | INFO | Perf report for the last 5.13711 seconds: | 20260506 22:29:18.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:29:18.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:18.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:18.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:18.731 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 22:29:18.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:18.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:18.731 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 22:29:18.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:18.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:18.731 | INFO | Uploaded 36GiB to GPU in 2.01045s (17.9064GiB/s) | 20260506 22:29:18.731 | INFO | Downloaded 34GiB to GPU in 2.00044s (16.9963GiB/s) | 20260506 22:29:18.731 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:29:18.731 | INFO | Total mem usage: 18 GiB | 20260506 22:29:18.732 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:29:18.732 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:29:18.732 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:29:18.732 | INFO | High Water Breakdown: | 20260506 22:29:18.732 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:29:18.732 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:29:18.732 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:29:18.732 | INFO | ------------------------------------------ | 20260506 22:29:18.732 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:29:18.732 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:29:18.732 | INFO | Total mem usage: 26.5 GiB | 20260506 22:29:18.732 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:29:18.732 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:29:18.732 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:29:18.732 | INFO | High Water Breakdown: | 20260506 22:29:18.732 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:29:18.732 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:29:18.732 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:29:18.732 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:29:18.732 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:29:18.732 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:29:18.732 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:29:18.732 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:29:18.732 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:29:18.732 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:29:18.732 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:29:18.732 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:29:18.732 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:29:18.732 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:29:18.732 | INFO | ------------------------------------------ | 20260506 22:29:18.732 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:29:18.732 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:29:18.732 | INFO | Total mem usage: 25.6 GiB | 20260506 22:29:18.732 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:29:18.732 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:29:18.732 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:29:18.732 | INFO | High Water Breakdown: | 20260506 22:29:18.732 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:29:18.732 | INFO | ------------------------------------------ | 20260506 22:29:18.732 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:29:18.732 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:29:18.732 | INFO | Total mem usage: 15.7 GiB | 20260506 22:29:18.732 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:29:18.732 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:29:18.732 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:29:18.732 | INFO | High Water Breakdown: | 20260506 22:29:18.732 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:29:18.732 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:29:18.732 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:29:18.732 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:29:18.732 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:29:18.732 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:29:18.732 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:29:18.732 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:29:18.732 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:29:18.732 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:29:18.732 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:29:18.732 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:29:18.732 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:29:18.732 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:29:18.732 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:29:18.732 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:29:18.732 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:29:18.732 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:29:18.732 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:29:18.732 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:29:18.732 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:29:18.732 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:29:18.732 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:29:18.732 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:29:18.733 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:29:18.733 | INFO | ------------------------------------------ | 20260506 22:29:18.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:29:18.733 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:29:18.733 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:29:18.733 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:29:18.733 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.57e+01, 1.34e+01 / 2.17e+00 | 20260506 22:29:18.733 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 1.02e-02 / 5.70e-03 | 20260506 22:29:18.733 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.39e+00, 5.46e-01 / 3.14e-01 | 20260506 22:29:18.733 | INFO | PerfReport: 3.27%, Histogram -- 1.01e+00 / 1.48e+00, 1.82e-01 / 1.14e-01 | 20260506 22:29:18.733 | INFO | PerfReport: 16.07%, DME -- 3.99e+01 / 4.48e+01, 4.08e+00 / 2.36e+00 | 20260506 22:29:18.733 | INFO | PerfReport: 57.31%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.37e+00 / 1.72e+00 | 20260506 22:29:18.733 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.20e+00 / 2.77e+00, 3.78e-01 / 2.22e-01 | 20260506 22:29:18.733 | INFO | PerfReport: 3.22%, Metrics -- 9.99e-01 / 1.35e+00, 2.61e-01 / 1.81e-01 | 20260506 22:29:18.733 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.42e-01 / 3.94e-01, 5.21e-02 / 2.08e-02 | 20260506 22:29:18.733 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:29:18.733 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:29:18.733 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:29:18.733 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 22:29:18.733 | INFO | PerfReport: 21.53%, Upload -- 2.08e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260506 22:29:18.733 | INFO | PerfReport: 21.20%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 22:29:18.733 | INFO | PerfReport: 32.08%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 22:29:18.733 | INFO | PerfReport: 1.35%, LockUpload -- 1.30e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260506 22:29:18.733 | INFO | PerfReport: 1.20%, LockDownload -- 1.16e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260506 22:29:18.733 | INFO | PerfReport: 22.62%, LockCompute -- 2.19e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 22:29:18.733 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260506 22:29:18.839 | INFO | Starting a round of Baz IO | 20260506 22:29:21.964 | INFO | Populating RTMetrics report | 20260506 22:29:21.965 | INFO | RTMetrics report populated | 20260506 22:29:21.965 | INFO | Opening tmp RTMetrics json file | 20260506 22:29:21.966 | INFO | Serializing RTMetrics to json | 20260506 22:29:21.967 | INFO | Serialization done, swapping tmp file now... | 20260506 22:29:21.967 | INFO | RTMetrics done | 20260506 22:29:22.058 | INFO | Finished a round of Baz IO: Wrote 1.91 GiB in 3.219 seconds | 20260506 22:29:22.058 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.67% / 0.00% / 66.33% | 20260506 22:29:22.058 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.500814% / 1.19209e-05% / 6.75519e-05% | 20260506 22:29:22.058 | INFO | Baz IO overall: | 20260506 22:29:22.058 | INFO | BytesWritten : 2599100338785 | 20260506 22:29:22.058 | INFO | NumEvents : 1254004778232 | 20260506 22:29:23.871 | INFO | Analyzing chunk frames = [4947968,4948480) | 20260506 22:29:23.871 | INFO | MemInfo: { | 20260506 22:29:23.871 | INFO | "gpu" : " 17 GB", | 20260506 22:29:23.871 | INFO | "huge" : "", | 20260506 22:29:23.871 | INFO | "huge_shared" : " 27 GB", | 20260506 22:29:23.871 | INFO | "malloc" : " 14 GB", | 20260506 22:29:23.871 | INFO | "pinned" : " 29 GB" | 20260506 22:29:23.871 | INFO | } | 20260506 22:29:23.871 | INFO | Perf report for the last 5.14029 seconds: | 20260506 22:29:23.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:29:23.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:23.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:23.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:23.871 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 22:29:23.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:23.871 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 22:29:23.871 | INFO | PRE_HQ : 59.7%, 1.8, 57.3ms, 0 | 20260506 22:29:23.871 | INFO | BAZWRITER : 62.1%, 3.5, 111.0ms, 0 | 20260506 22:29:23.871 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:29:23.871 | INFO | Uploaded 36GiB to GPU in 2.01482s (17.8676GiB/s) | 20260506 22:29:23.871 | INFO | Downloaded 38GiB to GPU in 2.17716s (17.4539GiB/s) | 20260506 22:29:29.011 | INFO | Analyzing chunk frames = [4948480,4948992) | 20260506 22:29:29.011 | INFO | MemInfo: { | 20260506 22:29:29.011 | INFO | "gpu" : " 17 GB", | 20260506 22:29:29.011 | INFO | "huge" : "", | 20260506 22:29:29.011 | INFO | "huge_shared" : " 27 GB", | 20260506 22:29:29.011 | INFO | "malloc" : " 14 GB", | 20260506 22:29:29.011 | INFO | "pinned" : " 29 GB" | 20260506 22:29:29.011 | INFO | } | 20260506 22:29:29.011 | INFO | Perf report for the last 5.1402 seconds: | 20260506 22:29:29.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:29:29.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:29.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:29.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:29.011 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:29:29.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:29.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:29.011 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:29:29.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:29.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:29.012 | INFO | Uploaded 36GiB to GPU in 1.99443s (18.0503GiB/s) | 20260506 22:29:29.012 | INFO | Downloaded 34GiB to GPU in 1.92996s (17.6169GiB/s) | 20260506 22:29:34.149 | INFO | Analyzing chunk frames = [4948992,4949504) | 20260506 22:29:34.149 | INFO | MemInfo: { | 20260506 22:29:34.149 | INFO | "gpu" : " 17 GB", | 20260506 22:29:34.149 | INFO | "huge" : "", | 20260506 22:29:34.149 | INFO | "huge_shared" : " 27 GB", | 20260506 22:29:34.149 | INFO | "malloc" : " 14 GB", | 20260506 22:29:34.149 | INFO | "pinned" : " 29 GB" | 20260506 22:29:34.149 | INFO | } | 20260506 22:29:34.149 | INFO | Perf report for the last 5.13784 seconds: | 20260506 22:29:34.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:29:34.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:34.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:34.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:34.149 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260506 22:29:34.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:34.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:34.149 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:29:34.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:34.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:34.150 | INFO | Uploaded 36GiB to GPU in 1.97425s (18.2348GiB/s) | 20260506 22:29:34.150 | INFO | Downloaded 34GiB to GPU in 1.9219s (17.6908GiB/s) | 20260506 22:29:39.288 | INFO | Analyzing chunk frames = [4949504,4950016) | 20260506 22:29:39.288 | INFO | MemInfo: { | 20260506 22:29:39.288 | INFO | "gpu" : " 17 GB", | 20260506 22:29:39.288 | INFO | "huge" : "", | 20260506 22:29:39.288 | INFO | "huge_shared" : " 27 GB", | 20260506 22:29:39.288 | INFO | "malloc" : " 14 GB", | 20260506 22:29:39.288 | INFO | "pinned" : " 29 GB" | 20260506 22:29:39.288 | INFO | } | 20260506 22:29:39.289 | INFO | Perf report for the last 5.13929 seconds: | 20260506 22:29:39.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:29:39.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:39.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:39.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:39.289 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 22:29:39.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:39.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:39.289 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:29:39.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:39.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:39.289 | INFO | Uploaded 36GiB to GPU in 1.94447s (18.514GiB/s) | 20260506 22:29:39.289 | INFO | Downloaded 34GiB to GPU in 1.87398s (18.1432GiB/s) | 20260506 22:29:44.430 | INFO | Analyzing chunk frames = [4950016,4950528) | 20260506 22:29:44.430 | INFO | MemInfo: { | 20260506 22:29:44.430 | INFO | "gpu" : " 17 GB", | 20260506 22:29:44.430 | INFO | "huge" : "", | 20260506 22:29:44.430 | INFO | "huge_shared" : " 27 GB", | 20260506 22:29:44.430 | INFO | "malloc" : " 14 GB", | 20260506 22:29:44.430 | INFO | "pinned" : " 29 GB" | 20260506 22:29:44.430 | INFO | } | 20260506 22:29:44.430 | INFO | Perf report for the last 5.14145 seconds: | 20260506 22:29:44.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:29:44.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:44.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:44.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:44.430 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 22:29:44.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:44.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:44.430 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:29:44.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:44.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:44.430 | INFO | Uploaded 36GiB to GPU in 1.96281s (18.3411GiB/s) | 20260506 22:29:44.430 | INFO | Downloaded 34GiB to GPU in 1.92507s (17.6617GiB/s) | 20260506 22:29:49.573 | INFO | Analyzing chunk frames = [4950528,4951040) | 20260506 22:29:49.573 | INFO | MemInfo: { | 20260506 22:29:49.573 | INFO | "gpu" : " 17 GB", | 20260506 22:29:49.573 | INFO | "huge" : "", | 20260506 22:29:49.573 | INFO | "huge_shared" : " 27 GB", | 20260506 22:29:49.573 | INFO | "malloc" : " 14 GB", | 20260506 22:29:49.573 | INFO | "pinned" : " 29 GB" | 20260506 22:29:49.573 | INFO | } | 20260506 22:29:49.573 | INFO | Perf report for the last 5.14278 seconds: | 20260506 22:29:49.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:29:49.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:49.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:49.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:49.573 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 22:29:49.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:49.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:49.573 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:29:49.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:49.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:49.573 | INFO | Uploaded 36GiB to GPU in 1.9974s (18.0234GiB/s) | 20260506 22:29:49.573 | INFO | Downloaded 34GiB to GPU in 1.95345s (17.4051GiB/s) | 20260506 22:29:54.712 | INFO | Analyzing chunk frames = [4951040,4951552) | 20260506 22:29:54.712 | INFO | MemInfo: { | 20260506 22:29:54.712 | INFO | "gpu" : " 17 GB", | 20260506 22:29:54.712 | INFO | "huge" : "", | 20260506 22:29:54.712 | INFO | "huge_shared" : " 27 GB", | 20260506 22:29:54.712 | INFO | "malloc" : " 14 GB", | 20260506 22:29:54.712 | INFO | "pinned" : " 29 GB" | 20260506 22:29:54.712 | INFO | } | 20260506 22:29:54.712 | INFO | Perf report for the last 5.13927 seconds: | 20260506 22:29:54.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:29:54.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:54.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:54.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:54.712 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:29:54.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:54.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:54.712 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 22:29:54.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:54.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:54.713 | INFO | Uploaded 36GiB to GPU in 1.95661s (18.3991GiB/s) | 20260506 22:29:54.713 | INFO | Downloaded 34GiB to GPU in 1.99282s (17.0612GiB/s) | 20260506 22:29:59.851 | INFO | Analyzing chunk frames = [4951552,4952064) | 20260506 22:29:59.851 | INFO | MemInfo: { | 20260506 22:29:59.851 | INFO | "gpu" : " 17 GB", | 20260506 22:29:59.851 | INFO | "huge" : "", | 20260506 22:29:59.851 | INFO | "huge_shared" : " 27 GB", | 20260506 22:29:59.851 | INFO | "malloc" : " 14 GB", | 20260506 22:29:59.851 | INFO | "pinned" : " 29 GB" | 20260506 22:29:59.851 | INFO | } | 20260506 22:29:59.851 | INFO | Perf report for the last 5.1387 seconds: | 20260506 22:29:59.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:29:59.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:59.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:59.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:59.851 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 22:29:59.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:59.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:59.851 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 22:29:59.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:59.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:29:59.851 | INFO | Uploaded 36GiB to GPU in 2.0013s (17.9883GiB/s) | 20260506 22:29:59.851 | INFO | Downloaded 34GiB to GPU in 1.99314s (17.0586GiB/s) | 20260506 22:29:59.949 | INFO | Starting a round of Baz IO | 20260506 22:30:03.078 | INFO | Populating RTMetrics report | 20260506 22:30:03.079 | INFO | RTMetrics report populated | 20260506 22:30:03.079 | INFO | Opening tmp RTMetrics json file | 20260506 22:30:03.079 | INFO | Serializing RTMetrics to json | 20260506 22:30:03.080 | INFO | Serialization done, swapping tmp file now... | 20260506 22:30:03.081 | INFO | RTMetrics done | 20260506 22:30:03.177 | INFO | Finished a round of Baz IO: Wrote 1.911 GiB in 3.228 seconds | 20260506 22:30:03.177 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.67% / 0.00% / 66.33% | 20260506 22:30:03.177 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.499547% / 0% / 7.94729e-05% | 20260506 22:30:03.177 | INFO | Baz IO overall: | 20260506 22:30:03.177 | INFO | BytesWritten : 2601151853683 | 20260506 22:30:03.177 | INFO | NumEvents : 1254984228032 | 20260506 22:30:04.986 | INFO | Analyzing chunk frames = [4952064,4952576) | 20260506 22:30:04.986 | INFO | MemInfo: { | 20260506 22:30:04.986 | INFO | "gpu" : " 17 GB", | 20260506 22:30:04.986 | INFO | "huge" : "", | 20260506 22:30:04.986 | INFO | "huge_shared" : " 27 GB", | 20260506 22:30:04.986 | INFO | "malloc" : " 14 GB", | 20260506 22:30:04.986 | INFO | "pinned" : " 29 GB" | 20260506 22:30:04.986 | INFO | } | 20260506 22:30:04.986 | INFO | Perf report for the last 5.13512 seconds: | 20260506 22:30:04.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:30:04.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:04.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:04.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:04.986 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 22:30:04.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:04.986 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 22:30:04.986 | INFO | PRE_HQ : 59.9%, 1.8, 56.9ms, 0 | 20260506 22:30:04.986 | INFO | BAZWRITER : 62.5%, 3.4, 110.1ms, 0 | 20260506 22:30:04.986 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:30:04.986 | INFO | Uploaded 36GiB to GPU in 2.0253s (17.7752GiB/s) | 20260506 22:30:04.986 | INFO | Downloaded 38GiB to GPU in 2.16627s (17.5417GiB/s) | 20260506 22:30:10.125 | INFO | Analyzing chunk frames = [4952576,4953088) | 20260506 22:30:10.125 | INFO | MemInfo: { | 20260506 22:30:10.125 | INFO | "gpu" : " 17 GB", | 20260506 22:30:10.125 | INFO | "huge" : "", | 20260506 22:30:10.125 | INFO | "huge_shared" : " 27 GB", | 20260506 22:30:10.125 | INFO | "malloc" : " 14 GB", | 20260506 22:30:10.126 | INFO | "pinned" : " 29 GB" | 20260506 22:30:10.126 | INFO | } | 20260506 22:30:10.126 | INFO | Perf report for the last 5.13961 seconds: | 20260506 22:30:10.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:30:10.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:10.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:10.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:10.126 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:30:10.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:10.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:10.126 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:30:10.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:10.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:10.126 | INFO | Uploaded 36GiB to GPU in 1.99278s (18.0652GiB/s) | 20260506 22:30:10.126 | INFO | Downloaded 34GiB to GPU in 1.93579s (17.5639GiB/s) | 20260506 22:30:15.267 | INFO | Analyzing chunk frames = [4953088,4953600) | 20260506 22:30:15.267 | INFO | MemInfo: { | 20260506 22:30:15.267 | INFO | "gpu" : " 17 GB", | 20260506 22:30:15.267 | INFO | "huge" : "", | 20260506 22:30:15.267 | INFO | "huge_shared" : " 27 GB", | 20260506 22:30:15.267 | INFO | "malloc" : " 14 GB", | 20260506 22:30:15.267 | INFO | "pinned" : " 29 GB" | 20260506 22:30:15.267 | INFO | } | 20260506 22:30:15.267 | INFO | Perf report for the last 5.1414 seconds: | 20260506 22:30:15.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:30:15.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:15.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:15.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:15.267 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 22:30:15.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:15.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:15.267 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:30:15.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:15.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:15.267 | INFO | Uploaded 36GiB to GPU in 1.97223s (18.2534GiB/s) | 20260506 22:30:15.267 | INFO | Downloaded 34GiB to GPU in 1.92489s (17.6633GiB/s) | 20260506 22:30:20.405 | INFO | Analyzing chunk frames = [4953600,4954112) | 20260506 22:30:20.405 | INFO | MemInfo: { | 20260506 22:30:20.405 | INFO | "gpu" : " 17 GB", | 20260506 22:30:20.405 | INFO | "huge" : "", | 20260506 22:30:20.405 | INFO | "huge_shared" : " 27 GB", | 20260506 22:30:20.405 | INFO | "malloc" : " 14 GB", | 20260506 22:30:20.405 | INFO | "pinned" : " 29 GB" | 20260506 22:30:20.405 | INFO | } | 20260506 22:30:20.405 | INFO | Perf report for the last 5.13842 seconds: | 20260506 22:30:20.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:30:20.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:20.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:20.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:20.406 | INFO | ANALYSIS : 59.6%, 3.0, 92.3ms, 0 | 20260506 22:30:20.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:20.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:20.406 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:30:20.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:20.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:20.406 | INFO | Uploaded 36GiB to GPU in 1.92884s (18.6641GiB/s) | 20260506 22:30:20.406 | INFO | Downloaded 34GiB to GPU in 1.87603s (18.1234GiB/s) | 20260506 22:30:25.549 | INFO | Analyzing chunk frames = [4954112,4954624) | 20260506 22:30:25.549 | INFO | MemInfo: { | 20260506 22:30:25.549 | INFO | "gpu" : " 17 GB", | 20260506 22:30:25.549 | INFO | "huge" : "", | 20260506 22:30:25.549 | INFO | "huge_shared" : " 27 GB", | 20260506 22:30:25.549 | INFO | "malloc" : " 14 GB", | 20260506 22:30:25.549 | INFO | "pinned" : " 29 GB" | 20260506 22:30:25.549 | INFO | } | 20260506 22:30:25.549 | INFO | Perf report for the last 5.14324 seconds: | 20260506 22:30:25.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:30:25.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:25.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:25.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:25.549 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260506 22:30:25.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:25.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:25.549 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:30:25.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:25.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:25.549 | INFO | Uploaded 36GiB to GPU in 1.99106s (18.0808GiB/s) | 20260506 22:30:25.549 | INFO | Downloaded 34GiB to GPU in 1.91761s (17.7304GiB/s) | 20260506 22:30:30.681 | INFO | Analyzing chunk frames = [4954624,4955136) | 20260506 22:30:30.681 | INFO | MemInfo: { | 20260506 22:30:30.681 | INFO | "gpu" : " 17 GB", | 20260506 22:30:30.681 | INFO | "huge" : "", | 20260506 22:30:30.681 | INFO | "huge_shared" : " 27 GB", | 20260506 22:30:30.681 | INFO | "malloc" : " 14 GB", | 20260506 22:30:30.681 | INFO | "pinned" : " 29 GB" | 20260506 22:30:30.681 | INFO | } | 20260506 22:30:30.681 | INFO | Perf report for the last 5.13248 seconds: | 20260506 22:30:30.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:30:30.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:30.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:30.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:30.681 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 22:30:30.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:30.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:30.681 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 22:30:30.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:30.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:30.682 | INFO | Uploaded 36GiB to GPU in 2.00766s (17.9313GiB/s) | 20260506 22:30:30.682 | INFO | Downloaded 34GiB to GPU in 1.9414s (17.5131GiB/s) | 20260506 22:30:35.825 | INFO | Analyzing chunk frames = [4955136,4955648) | 20260506 22:30:35.825 | INFO | MemInfo: { | 20260506 22:30:35.825 | INFO | "gpu" : " 17 GB", | 20260506 22:30:35.825 | INFO | "huge" : "", | 20260506 22:30:35.825 | INFO | "huge_shared" : " 27 GB", | 20260506 22:30:35.825 | INFO | "malloc" : " 14 GB", | 20260506 22:30:35.825 | INFO | "pinned" : " 29 GB" | 20260506 22:30:35.825 | INFO | } | 20260506 22:30:35.825 | INFO | Perf report for the last 5.14406 seconds: | 20260506 22:30:35.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:30:35.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:35.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:35.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:35.825 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:30:35.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:35.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:35.825 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 22:30:35.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:35.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:35.825 | INFO | Uploaded 36GiB to GPU in 1.95967s (18.3704GiB/s) | 20260506 22:30:35.825 | INFO | Downloaded 34GiB to GPU in 2.0036s (16.9695GiB/s) | 20260506 22:30:40.964 | INFO | Analyzing chunk frames = [4955648,4956160) | 20260506 22:30:40.964 | INFO | MemInfo: { | 20260506 22:30:40.964 | INFO | "gpu" : " 17 GB", | 20260506 22:30:40.964 | INFO | "huge" : "", | 20260506 22:30:40.964 | INFO | "huge_shared" : " 27 GB", | 20260506 22:30:40.964 | INFO | "malloc" : " 14 GB", | 20260506 22:30:40.964 | INFO | "pinned" : " 29 GB" | 20260506 22:30:40.964 | INFO | } | 20260506 22:30:40.964 | INFO | Perf report for the last 5.13887 seconds: | 20260506 22:30:40.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:30:40.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:40.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:40.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:40.964 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:30:40.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:40.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:40.964 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:30:40.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:40.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:40.964 | INFO | Uploaded 36GiB to GPU in 1.98418s (18.1435GiB/s) | 20260506 22:30:40.964 | INFO | Downloaded 34GiB to GPU in 1.99198s (17.0684GiB/s) | 20260506 22:30:40.965 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:30:40.965 | INFO | Total mem usage: 18 GiB | 20260506 22:30:40.965 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:30:40.965 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:30:40.965 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:30:40.965 | INFO | High Water Breakdown: | 20260506 22:30:40.965 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:30:40.965 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:30:40.965 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:30:40.965 | INFO | ------------------------------------------ | 20260506 22:30:40.965 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:30:40.965 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:30:40.965 | INFO | Total mem usage: 26.5 GiB | 20260506 22:30:40.965 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:30:40.965 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:30:40.965 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:30:40.965 | INFO | High Water Breakdown: | 20260506 22:30:40.965 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:30:40.965 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:30:40.965 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:30:40.965 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:30:40.965 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:30:40.965 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:30:40.965 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:30:40.965 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:30:40.965 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:30:40.965 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:30:40.965 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:30:40.965 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:30:40.965 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:30:40.965 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:30:40.965 | INFO | ------------------------------------------ | 20260506 22:30:40.965 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:30:40.965 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:30:40.965 | INFO | Total mem usage: 25.6 GiB | 20260506 22:30:40.965 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:30:40.965 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:30:40.965 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:30:40.965 | INFO | High Water Breakdown: | 20260506 22:30:40.965 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:30:40.965 | INFO | ------------------------------------------ | 20260506 22:30:40.965 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:30:40.965 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:30:40.965 | INFO | Total mem usage: 15.7 GiB | 20260506 22:30:40.965 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:30:40.965 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:30:40.965 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:30:40.965 | INFO | High Water Breakdown: | 20260506 22:30:40.965 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:30:40.965 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:30:40.965 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:30:40.965 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:30:40.965 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:30:40.965 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:30:40.965 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:30:40.965 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:30:40.965 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:30:40.965 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:30:40.965 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:30:40.965 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:30:40.965 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:30:40.965 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:30:40.965 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:30:40.965 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:30:40.965 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:30:40.965 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:30:40.965 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:30:40.965 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:30:40.965 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:30:40.965 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:30:40.965 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:30:40.965 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:30:40.965 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:30:40.965 | INFO | ------------------------------------------ | 20260506 22:30:40.965 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:30:40.965 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:30:40.965 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:30:40.965 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:30:40.965 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.62e+01, 1.34e+01 / 2.91e+00 | 20260506 22:30:40.965 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.02e-01, 1.01e-02 / 6.58e-03 | 20260506 22:30:40.965 | INFO | PerfReport: 11.67%, Baseline -- 3.62e+00 / 4.41e+00, 5.51e-01 / 3.49e-01 | 20260506 22:30:40.965 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.47e+00, 1.81e-01 / 1.33e-01 | 20260506 22:30:40.965 | INFO | PerfReport: 16.11%, DME -- 4.00e+01 / 4.55e+01, 4.29e+00 / 3.21e+00 | 20260506 22:30:40.965 | INFO | PerfReport: 57.26%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.34e+00 / 2.07e+00 | 20260506 22:30:40.965 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.20e+00 / 2.78e+00, 3.81e-01 / 2.09e-01 | 20260506 22:30:40.965 | INFO | PerfReport: 3.23%, Metrics -- 1.00e+00 / 1.34e+00, 2.67e-01 / 1.73e-01 | 20260506 22:30:40.965 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.42e-01 / 3.92e-01, 5.32e-02 / 1.86e-02 | 20260506 22:30:40.966 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:30:40.966 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:30:40.966 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:30:40.966 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 22:30:40.966 | INFO | PerfReport: 21.35%, Upload -- 2.06e+01 / 0.00e+00, 6.78e+00 / 0.00e+00 | 20260506 22:30:40.966 | INFO | PerfReport: 21.23%, Download -- 2.05e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 22:30:40.966 | INFO | PerfReport: 32.13%, Compute -- 3.11e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 22:30:40.966 | INFO | PerfReport: 1.19%, LockUpload -- 1.15e+00 / 0.00e+00, 4.73e+00 / 0.00e+00 | 20260506 22:30:40.966 | INFO | PerfReport: 1.33%, LockDownload -- 1.29e+00 / 0.00e+00, 3.26e+00 / 0.00e+00 | 20260506 22:30:40.966 | INFO | PerfReport: 22.75%, LockCompute -- 2.20e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 22:30:40.966 | INFO | PerfReport: 0.02%, Misc -- 1.50e-02 / 0.00e+00, 2.62e-02 / 0.00e+00 | 20260506 22:30:41.063 | INFO | Starting a round of Baz IO | 20260506 22:30:44.197 | INFO | Populating RTMetrics report | 20260506 22:30:44.197 | INFO | RTMetrics report populated | 20260506 22:30:44.197 | INFO | Opening tmp RTMetrics json file | 20260506 22:30:44.198 | INFO | Serializing RTMetrics to json | 20260506 22:30:44.199 | INFO | Serialization done, swapping tmp file now... | 20260506 22:30:44.199 | INFO | RTMetrics done | 20260506 22:30:44.293 | INFO | Finished a round of Baz IO: Wrote 1.91 GiB in 3.23 seconds | 20260506 22:30:44.293 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.67% / 0.00% / 66.33% | 20260506 22:30:44.293 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.498513% / 0% / 6.35783e-05% | 20260506 22:30:44.293 | INFO | Baz IO overall: | 20260506 22:30:44.293 | INFO | BytesWritten : 2603203065963 | 20260506 22:30:44.293 | INFO | NumEvents : 1255963365585 | 20260506 22:30:46.102 | INFO | Analyzing chunk frames = [4956160,4956672) | 20260506 22:30:46.102 | INFO | MemInfo: { | 20260506 22:30:46.102 | INFO | "gpu" : " 17 GB", | 20260506 22:30:46.102 | INFO | "huge" : "", | 20260506 22:30:46.102 | INFO | "huge_shared" : " 27 GB", | 20260506 22:30:46.102 | INFO | "malloc" : " 14 GB", | 20260506 22:30:46.102 | INFO | "pinned" : " 29 GB" | 20260506 22:30:46.102 | INFO | } | 20260506 22:30:46.102 | INFO | Perf report for the last 5.13778 seconds: | 20260506 22:30:46.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:30:46.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:46.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:46.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:46.102 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 22:30:46.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:46.102 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 22:30:46.102 | INFO | PRE_HQ : 60.0%, 1.8, 57.1ms, 0 | 20260506 22:30:46.102 | INFO | BAZWRITER : 62.5%, 3.4, 110.3ms, 0 | 20260506 22:30:46.102 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:30:46.102 | INFO | Uploaded 36GiB to GPU in 2.0394s (17.6522GiB/s) | 20260506 22:30:46.102 | INFO | Downloaded 38GiB to GPU in 2.17508s (17.4707GiB/s) | 20260506 22:30:51.237 | INFO | Analyzing chunk frames = [4956672,4957184) | 20260506 22:30:51.237 | INFO | MemInfo: { | 20260506 22:30:51.237 | INFO | "gpu" : " 17 GB", | 20260506 22:30:51.237 | INFO | "huge" : "", | 20260506 22:30:51.237 | INFO | "huge_shared" : " 27 GB", | 20260506 22:30:51.237 | INFO | "malloc" : " 14 GB", | 20260506 22:30:51.237 | INFO | "pinned" : " 29 GB" | 20260506 22:30:51.237 | INFO | } | 20260506 22:30:51.238 | INFO | Perf report for the last 5.13565 seconds: | 20260506 22:30:51.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:30:51.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:51.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:51.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:51.238 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:30:51.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:51.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:51.238 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:30:51.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:51.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:51.238 | INFO | Uploaded 36GiB to GPU in 2.01489s (17.867GiB/s) | 20260506 22:30:51.238 | INFO | Downloaded 34GiB to GPU in 1.92572s (17.6557GiB/s) | 20260506 22:30:56.378 | INFO | Analyzing chunk frames = [4957184,4957696) | 20260506 22:30:56.378 | INFO | MemInfo: { | 20260506 22:30:56.378 | INFO | "gpu" : " 17 GB", | 20260506 22:30:56.378 | INFO | "huge" : "", | 20260506 22:30:56.378 | INFO | "huge_shared" : " 27 GB", | 20260506 22:30:56.378 | INFO | "malloc" : " 14 GB", | 20260506 22:30:56.378 | INFO | "pinned" : " 29 GB" | 20260506 22:30:56.378 | INFO | } | 20260506 22:30:56.378 | INFO | Perf report for the last 5.14023 seconds: | 20260506 22:30:56.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:30:56.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:56.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:56.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:56.378 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260506 22:30:56.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:56.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:56.378 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:30:56.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:56.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:30:56.378 | INFO | Uploaded 36GiB to GPU in 1.98664s (18.1211GiB/s) | 20260506 22:30:56.378 | INFO | Downloaded 34GiB to GPU in 1.92133s (17.6961GiB/s) | 20260506 22:31:01.521 | INFO | Analyzing chunk frames = [4957696,4958208) | 20260506 22:31:01.521 | INFO | MemInfo: { | 20260506 22:31:01.521 | INFO | "gpu" : " 17 GB", | 20260506 22:31:01.521 | INFO | "huge" : "", | 20260506 22:31:01.521 | INFO | "huge_shared" : " 27 GB", | 20260506 22:31:01.521 | INFO | "malloc" : " 14 GB", | 20260506 22:31:01.521 | INFO | "pinned" : " 29 GB" | 20260506 22:31:01.521 | INFO | } | 20260506 22:31:01.521 | INFO | Perf report for the last 5.14336 seconds: | 20260506 22:31:01.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:31:01.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:01.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:01.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:01.521 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 22:31:01.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:01.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:01.521 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:31:01.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:01.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:01.521 | INFO | Uploaded 36GiB to GPU in 1.95687s (18.3967GiB/s) | 20260506 22:31:01.521 | INFO | Downloaded 34GiB to GPU in 1.87775s (18.1068GiB/s) | 20260506 22:31:06.660 | INFO | Analyzing chunk frames = [4958208,4958720) | 20260506 22:31:06.660 | INFO | MemInfo: { | 20260506 22:31:06.660 | INFO | "gpu" : " 17 GB", | 20260506 22:31:06.660 | INFO | "huge" : "", | 20260506 22:31:06.660 | INFO | "huge_shared" : " 27 GB", | 20260506 22:31:06.660 | INFO | "malloc" : " 14 GB", | 20260506 22:31:06.660 | INFO | "pinned" : " 29 GB" | 20260506 22:31:06.660 | INFO | } | 20260506 22:31:06.660 | INFO | Perf report for the last 5.13846 seconds: | 20260506 22:31:06.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:31:06.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:06.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:06.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:06.660 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 22:31:06.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:06.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:06.660 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:31:06.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:06.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:06.660 | INFO | Uploaded 36GiB to GPU in 1.98864s (18.1028GiB/s) | 20260506 22:31:06.660 | INFO | Downloaded 34GiB to GPU in 1.91725s (17.7338GiB/s) | 20260506 22:31:11.804 | INFO | Analyzing chunk frames = [4958720,4959232) | 20260506 22:31:11.804 | INFO | MemInfo: { | 20260506 22:31:11.804 | INFO | "gpu" : " 17 GB", | 20260506 22:31:11.804 | INFO | "huge" : "", | 20260506 22:31:11.804 | INFO | "huge_shared" : " 27 GB", | 20260506 22:31:11.804 | INFO | "malloc" : " 14 GB", | 20260506 22:31:11.804 | INFO | "pinned" : " 29 GB" | 20260506 22:31:11.804 | INFO | } | 20260506 22:31:11.804 | INFO | Perf report for the last 5.14411 seconds: | 20260506 22:31:11.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:31:11.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:11.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:11.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:11.804 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 22:31:11.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:11.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:11.804 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 22:31:11.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:11.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:11.804 | INFO | Uploaded 36GiB to GPU in 2.01151s (17.897GiB/s) | 20260506 22:31:11.804 | INFO | Downloaded 34GiB to GPU in 1.93521s (17.5692GiB/s) | 20260506 22:31:16.939 | INFO | Analyzing chunk frames = [4959232,4959744) | 20260506 22:31:16.939 | INFO | MemInfo: { | 20260506 22:31:16.939 | INFO | "gpu" : " 17 GB", | 20260506 22:31:16.939 | INFO | "huge" : "", | 20260506 22:31:16.939 | INFO | "huge_shared" : " 27 GB", | 20260506 22:31:16.939 | INFO | "malloc" : " 14 GB", | 20260506 22:31:16.939 | INFO | "pinned" : " 29 GB" | 20260506 22:31:16.939 | INFO | } | 20260506 22:31:16.939 | INFO | Perf report for the last 5.13535 seconds: | 20260506 22:31:16.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:31:16.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:16.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:16.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:16.939 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:31:16.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:16.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:16.939 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260506 22:31:16.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:16.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:16.939 | INFO | Uploaded 36GiB to GPU in 1.95729s (18.3927GiB/s) | 20260506 22:31:16.939 | INFO | Downloaded 34GiB to GPU in 1.98544s (17.1246GiB/s) | 20260506 22:31:22.077 | INFO | Analyzing chunk frames = [4959744,4960256) | 20260506 22:31:22.077 | INFO | MemInfo: { | 20260506 22:31:22.077 | INFO | "gpu" : " 17 GB", | 20260506 22:31:22.077 | INFO | "huge" : "", | 20260506 22:31:22.077 | INFO | "huge_shared" : " 27 GB", | 20260506 22:31:22.077 | INFO | "malloc" : " 14 GB", | 20260506 22:31:22.077 | INFO | "pinned" : " 29 GB" | 20260506 22:31:22.077 | INFO | } | 20260506 22:31:22.077 | INFO | Perf report for the last 5.13791 seconds: | 20260506 22:31:22.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:31:22.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:22.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:22.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:22.077 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 22:31:22.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:22.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:22.077 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 22:31:22.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:22.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:22.077 | INFO | Uploaded 36GiB to GPU in 1.9657s (18.3141GiB/s) | 20260506 22:31:22.077 | INFO | Downloaded 34GiB to GPU in 1.98948s (17.0899GiB/s) | 20260506 22:31:22.175 | INFO | Starting a round of Baz IO | 20260506 22:31:25.311 | INFO | Populating RTMetrics report | 20260506 22:31:25.311 | INFO | RTMetrics report populated | 20260506 22:31:25.311 | INFO | Opening tmp RTMetrics json file | 20260506 22:31:25.313 | INFO | Serializing RTMetrics to json | 20260506 22:31:25.332 | INFO | Serialization done, swapping tmp file now... | 20260506 22:31:25.333 | INFO | RTMetrics done | 20260506 22:31:25.409 | INFO | Finished a round of Baz IO: Wrote 1.911 GiB in 3.233 seconds | 20260506 22:31:25.409 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.66% / 0.00% / 66.33% | 20260506 22:31:25.409 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.498164% / 0% / 5.96046e-05% | 20260506 22:31:25.409 | INFO | Baz IO overall: | 20260506 22:31:25.409 | INFO | BytesWritten : 2605255017213 | 20260506 22:31:25.409 | INFO | NumEvents : 1256943157482 | 20260506 22:31:27.217 | INFO | Analyzing chunk frames = [4960256,4960768) | 20260506 22:31:27.217 | INFO | MemInfo: { | 20260506 22:31:27.217 | INFO | "gpu" : " 17 GB", | 20260506 22:31:27.217 | INFO | "huge" : "", | 20260506 22:31:27.217 | INFO | "huge_shared" : " 27 GB", | 20260506 22:31:27.217 | INFO | "malloc" : " 14 GB", | 20260506 22:31:27.217 | INFO | "pinned" : " 29 GB" | 20260506 22:31:27.217 | INFO | } | 20260506 22:31:27.217 | INFO | Perf report for the last 5.14018 seconds: | 20260506 22:31:27.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:31:27.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:27.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:27.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:27.217 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260506 22:31:27.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:27.217 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 22:31:27.217 | INFO | PRE_HQ : 60.0%, 1.8, 57.3ms, 0 | 20260506 22:31:27.217 | INFO | BAZWRITER : 62.4%, 3.5, 111.6ms, 0 | 20260506 22:31:27.217 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:31:27.217 | INFO | Uploaded 36GiB to GPU in 2.03807s (17.6637GiB/s) | 20260506 22:31:27.217 | INFO | Downloaded 38GiB to GPU in 2.18279s (17.4089GiB/s) | 20260506 22:31:32.352 | INFO | Analyzing chunk frames = [4960768,4961280) | 20260506 22:31:32.352 | INFO | MemInfo: { | 20260506 22:31:32.352 | INFO | "gpu" : " 17 GB", | 20260506 22:31:32.352 | INFO | "huge" : "", | 20260506 22:31:32.352 | INFO | "huge_shared" : " 27 GB", | 20260506 22:31:32.352 | INFO | "malloc" : " 14 GB", | 20260506 22:31:32.352 | INFO | "pinned" : " 29 GB" | 20260506 22:31:32.352 | INFO | } | 20260506 22:31:32.352 | INFO | Perf report for the last 5.13527 seconds: | 20260506 22:31:32.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:31:32.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:32.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:32.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:32.353 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:31:32.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:32.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:32.353 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:31:32.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:32.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:32.353 | INFO | Uploaded 36GiB to GPU in 1.99338s (18.0598GiB/s) | 20260506 22:31:32.353 | INFO | Downloaded 34GiB to GPU in 1.92829s (17.6322GiB/s) | 20260506 22:31:37.494 | INFO | Analyzing chunk frames = [4961280,4961792) | 20260506 22:31:37.494 | INFO | MemInfo: { | 20260506 22:31:37.494 | INFO | "gpu" : " 17 GB", | 20260506 22:31:37.494 | INFO | "huge" : "", | 20260506 22:31:37.494 | INFO | "huge_shared" : " 27 GB", | 20260506 22:31:37.494 | INFO | "malloc" : " 14 GB", | 20260506 22:31:37.494 | INFO | "pinned" : " 29 GB" | 20260506 22:31:37.494 | INFO | } | 20260506 22:31:37.494 | INFO | Perf report for the last 5.14176 seconds: | 20260506 22:31:37.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:31:37.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:37.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:37.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:37.494 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 22:31:37.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:37.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:37.494 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:31:37.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:37.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:37.494 | INFO | Uploaded 36GiB to GPU in 1.96801s (18.2926GiB/s) | 20260506 22:31:37.494 | INFO | Downloaded 34GiB to GPU in 1.92604s (17.6528GiB/s) | 20260506 22:31:42.639 | INFO | Analyzing chunk frames = [4961792,4962304) | 20260506 22:31:42.639 | INFO | MemInfo: { | 20260506 22:31:42.639 | INFO | "gpu" : " 17 GB", | 20260506 22:31:42.639 | INFO | "huge" : "", | 20260506 22:31:42.639 | INFO | "huge_shared" : " 27 GB", | 20260506 22:31:42.639 | INFO | "malloc" : " 14 GB", | 20260506 22:31:42.639 | INFO | "pinned" : " 29 GB" | 20260506 22:31:42.639 | INFO | } | 20260506 22:31:42.639 | INFO | Perf report for the last 5.14525 seconds: | 20260506 22:31:42.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:31:42.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:42.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:42.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:42.640 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260506 22:31:42.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:42.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:42.640 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:31:42.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:42.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:42.640 | INFO | Uploaded 36GiB to GPU in 1.95939s (18.3731GiB/s) | 20260506 22:31:42.640 | INFO | Downloaded 34GiB to GPU in 1.88198s (18.0661GiB/s) | 20260506 22:31:47.779 | INFO | Analyzing chunk frames = [4962304,4962816) | 20260506 22:31:47.779 | INFO | MemInfo: { | 20260506 22:31:47.779 | INFO | "gpu" : " 17 GB", | 20260506 22:31:47.779 | INFO | "huge" : "", | 20260506 22:31:47.779 | INFO | "huge_shared" : " 27 GB", | 20260506 22:31:47.779 | INFO | "malloc" : " 14 GB", | 20260506 22:31:47.779 | INFO | "pinned" : " 29 GB" | 20260506 22:31:47.779 | INFO | } | 20260506 22:31:47.779 | INFO | Perf report for the last 5.13998 seconds: | 20260506 22:31:47.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:31:47.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:47.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:47.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:47.780 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 22:31:47.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:47.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:47.780 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:31:47.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:47.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:47.780 | INFO | Uploaded 36GiB to GPU in 1.98179s (18.1654GiB/s) | 20260506 22:31:47.780 | INFO | Downloaded 34GiB to GPU in 1.93733s (17.5499GiB/s) | 20260506 22:31:52.916 | INFO | Analyzing chunk frames = [4962816,4963328) | 20260506 22:31:52.916 | INFO | MemInfo: { | 20260506 22:31:52.916 | INFO | "gpu" : " 17 GB", | 20260506 22:31:52.916 | INFO | "huge" : "", | 20260506 22:31:52.916 | INFO | "huge_shared" : " 27 GB", | 20260506 22:31:52.916 | INFO | "malloc" : " 14 GB", | 20260506 22:31:52.916 | INFO | "pinned" : " 29 GB" | 20260506 22:31:52.916 | INFO | } | 20260506 22:31:52.916 | INFO | Perf report for the last 5.13644 seconds: | 20260506 22:31:52.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:31:52.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:52.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:52.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:52.916 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 22:31:52.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:52.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:52.916 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 22:31:52.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:52.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:52.916 | INFO | Uploaded 36GiB to GPU in 2.02086s (17.8142GiB/s) | 20260506 22:31:52.916 | INFO | Downloaded 34GiB to GPU in 1.95491s (17.3921GiB/s) | 20260506 22:31:58.053 | INFO | Analyzing chunk frames = [4963328,4963840) | 20260506 22:31:58.054 | INFO | MemInfo: { | 20260506 22:31:58.054 | INFO | "gpu" : " 17 GB", | 20260506 22:31:58.054 | INFO | "huge" : "", | 20260506 22:31:58.054 | INFO | "huge_shared" : " 27 GB", | 20260506 22:31:58.054 | INFO | "malloc" : " 14 GB", | 20260506 22:31:58.054 | INFO | "pinned" : " 29 GB" | 20260506 22:31:58.054 | INFO | } | 20260506 22:31:58.054 | INFO | Perf report for the last 5.1379 seconds: | 20260506 22:31:58.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:31:58.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:58.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:58.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:58.054 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:31:58.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:58.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:58.054 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:31:58.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:58.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:31:58.054 | INFO | Uploaded 36GiB to GPU in 1.95712s (18.3944GiB/s) | 20260506 22:31:58.054 | INFO | Downloaded 34GiB to GPU in 1.99535s (17.0396GiB/s) | 20260506 22:32:03.191 | INFO | Analyzing chunk frames = [4963840,4964352) | 20260506 22:32:03.191 | INFO | MemInfo: { | 20260506 22:32:03.191 | INFO | "gpu" : " 17 GB", | 20260506 22:32:03.191 | INFO | "huge" : "", | 20260506 22:32:03.191 | INFO | "huge_shared" : " 27 GB", | 20260506 22:32:03.191 | INFO | "malloc" : " 14 GB", | 20260506 22:32:03.191 | INFO | "pinned" : " 29 GB" | 20260506 22:32:03.191 | INFO | } | 20260506 22:32:03.191 | INFO | Perf report for the last 5.13755 seconds: | 20260506 22:32:03.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:32:03.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:03.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:03.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:03.192 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 22:32:03.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:03.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:03.192 | INFO | PRE_HQ : 31.0%, 1.0, 16.1ms, 0 | 20260506 22:32:03.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:03.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:03.192 | INFO | Uploaded 36GiB to GPU in 2.00208s (17.9813GiB/s) | 20260506 22:32:03.192 | INFO | Downloaded 34GiB to GPU in 1.99663s (17.0287GiB/s) | 20260506 22:32:03.192 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:32:03.192 | INFO | Total mem usage: 18 GiB | 20260506 22:32:03.192 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:32:03.192 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:32:03.192 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:32:03.192 | INFO | High Water Breakdown: | 20260506 22:32:03.192 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:32:03.192 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:32:03.192 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:32:03.192 | INFO | ------------------------------------------ | 20260506 22:32:03.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:32:03.192 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:32:03.192 | INFO | Total mem usage: 26.5 GiB | 20260506 22:32:03.192 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:32:03.192 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:32:03.192 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:32:03.192 | INFO | High Water Breakdown: | 20260506 22:32:03.192 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:32:03.192 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:32:03.192 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:32:03.192 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:32:03.192 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:32:03.192 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:32:03.192 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:32:03.192 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:32:03.192 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:32:03.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:32:03.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:32:03.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:32:03.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:32:03.192 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:32:03.192 | INFO | ------------------------------------------ | 20260506 22:32:03.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:32:03.192 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:32:03.192 | INFO | Total mem usage: 25.6 GiB | 20260506 22:32:03.192 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:32:03.192 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:32:03.192 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:32:03.192 | INFO | High Water Breakdown: | 20260506 22:32:03.192 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:32:03.192 | INFO | ------------------------------------------ | 20260506 22:32:03.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:32:03.192 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:32:03.192 | INFO | Total mem usage: 15.7 GiB | 20260506 22:32:03.192 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:32:03.192 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:32:03.192 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:32:03.192 | INFO | High Water Breakdown: | 20260506 22:32:03.192 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:32:03.192 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:32:03.192 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:32:03.192 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:32:03.192 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:32:03.192 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:32:03.192 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:32:03.192 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:32:03.192 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:32:03.192 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:32:03.193 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:32:03.193 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:32:03.193 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:32:03.193 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:32:03.193 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:32:03.193 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:32:03.193 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:32:03.193 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:32:03.193 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:32:03.193 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:32:03.193 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:32:03.193 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:32:03.193 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:32:03.193 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:32:03.193 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:32:03.193 | INFO | ------------------------------------------ | 20260506 22:32:03.193 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:32:03.193 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:32:03.193 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:32:03.193 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:32:03.193 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.59e+01, 1.33e+01 / 2.49e+00 | 20260506 22:32:03.193 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.02e-01, 9.67e-03 / 7.46e-03 | 20260506 22:32:03.193 | INFO | PerfReport: 11.64%, Baseline -- 3.61e+00 / 4.39e+00, 5.41e-01 / 3.85e-01 | 20260506 22:32:03.193 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.45e+00, 1.81e-01 / 1.47e-01 | 20260506 22:32:03.193 | INFO | PerfReport: 16.06%, DME -- 3.99e+01 / 4.49e+01, 4.21e+00 / 2.74e+00 | 20260506 22:32:03.193 | INFO | PerfReport: 57.33%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.36e+00 / 1.92e+00 | 20260506 22:32:03.193 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.20e+00 / 2.76e+00, 3.82e-01 / 2.17e-01 | 20260506 22:32:03.193 | INFO | PerfReport: 3.22%, Metrics -- 9.99e-01 / 1.33e+00, 2.64e-01 / 1.50e-01 | 20260506 22:32:03.193 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.43e-01 / 3.89e-01, 5.38e-02 / 1.63e-02 | 20260506 22:32:03.193 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:32:03.193 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:32:03.193 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:32:03.193 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 22:32:03.193 | INFO | PerfReport: 21.44%, Upload -- 2.07e+01 / 0.00e+00, 6.90e+00 / 0.00e+00 | 20260506 22:32:03.193 | INFO | PerfReport: 21.23%, Download -- 2.05e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 22:32:03.193 | INFO | PerfReport: 32.09%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 22:32:03.193 | INFO | PerfReport: 1.28%, LockUpload -- 1.24e+00 / 0.00e+00, 4.76e+00 / 0.00e+00 | 20260506 22:32:03.193 | INFO | PerfReport: 1.26%, LockDownload -- 1.22e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260506 22:32:03.193 | INFO | PerfReport: 22.68%, LockCompute -- 2.19e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 22:32:03.193 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260506 22:32:03.291 | INFO | Starting a round of Baz IO | 20260506 22:32:06.422 | INFO | Populating RTMetrics report | 20260506 22:32:06.422 | INFO | RTMetrics report populated | 20260506 22:32:06.422 | INFO | Opening tmp RTMetrics json file | 20260506 22:32:06.423 | INFO | Serializing RTMetrics to json | 20260506 22:32:06.424 | INFO | Serialization done, swapping tmp file now... | 20260506 22:32:06.425 | INFO | RTMetrics done | 20260506 22:32:06.525 | INFO | Finished a round of Baz IO: Wrote 1.91 GiB in 3.233 seconds | 20260506 22:32:06.525 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.66% / 0.00% / 66.34% | 20260506 22:32:06.525 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.497731% / 1.19209e-05% / 7.15256e-05% | 20260506 22:32:06.525 | INFO | Baz IO overall: | 20260506 22:32:06.525 | INFO | BytesWritten : 2607306043301 | 20260506 22:32:06.525 | INFO | NumEvents : 1257922504706 | 20260506 22:32:08.325 | INFO | Analyzing chunk frames = [4964352,4964864) | 20260506 22:32:08.325 | INFO | MemInfo: { | 20260506 22:32:08.325 | INFO | "gpu" : " 17 GB", | 20260506 22:32:08.325 | INFO | "huge" : "", | 20260506 22:32:08.325 | INFO | "huge_shared" : " 27 GB", | 20260506 22:32:08.325 | INFO | "malloc" : " 14 GB", | 20260506 22:32:08.325 | INFO | "pinned" : " 29 GB" | 20260506 22:32:08.325 | INFO | } | 20260506 22:32:08.325 | INFO | Perf report for the last 5.13388 seconds: | 20260506 22:32:08.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:32:08.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:08.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:08.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:08.326 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260506 22:32:08.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:08.326 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 22:32:08.326 | INFO | PRE_HQ : 59.7%, 1.8, 57.2ms, 0 | 20260506 22:32:08.326 | INFO | BAZWRITER : 62.4%, 3.5, 112.7ms, 0 | 20260506 22:32:08.326 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:32:08.326 | INFO | Uploaded 36GiB to GPU in 2.01338s (17.8804GiB/s) | 20260506 22:32:08.326 | INFO | Downloaded 38GiB to GPU in 2.16579s (17.5456GiB/s) | 20260506 22:32:13.472 | INFO | Analyzing chunk frames = [4964864,4965376) | 20260506 22:32:13.472 | INFO | MemInfo: { | 20260506 22:32:13.472 | INFO | "gpu" : " 17 GB", | 20260506 22:32:13.472 | INFO | "huge" : "", | 20260506 22:32:13.472 | INFO | "huge_shared" : " 27 GB", | 20260506 22:32:13.472 | INFO | "malloc" : " 14 GB", | 20260506 22:32:13.472 | INFO | "pinned" : " 29 GB" | 20260506 22:32:13.472 | INFO | } | 20260506 22:32:13.472 | INFO | Perf report for the last 5.14652 seconds: | 20260506 22:32:13.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:32:13.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:13.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:13.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:13.472 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 22:32:13.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:13.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:13.472 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:32:13.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:13.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:13.472 | INFO | Uploaded 36GiB to GPU in 2.00051s (17.9954GiB/s) | 20260506 22:32:13.472 | INFO | Downloaded 34GiB to GPU in 1.92501s (17.6622GiB/s) | 20260506 22:32:18.614 | INFO | Analyzing chunk frames = [4965376,4965888) | 20260506 22:32:18.614 | INFO | MemInfo: { | 20260506 22:32:18.614 | INFO | "gpu" : " 17 GB", | 20260506 22:32:18.614 | INFO | "huge" : "", | 20260506 22:32:18.614 | INFO | "huge_shared" : " 27 GB", | 20260506 22:32:18.614 | INFO | "malloc" : " 14 GB", | 20260506 22:32:18.614 | INFO | "pinned" : " 29 GB" | 20260506 22:32:18.614 | INFO | } | 20260506 22:32:18.614 | INFO | Perf report for the last 5.14254 seconds: | 20260506 22:32:18.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:32:18.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:18.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:18.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:18.614 | INFO | ANALYSIS : 60.4%, 3.0, 93.8ms, 0 | 20260506 22:32:18.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:18.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:18.615 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:32:18.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:18.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:18.615 | INFO | Uploaded 36GiB to GPU in 1.98414s (18.1439GiB/s) | 20260506 22:32:18.615 | INFO | Downloaded 34GiB to GPU in 1.91751s (17.7313GiB/s) | 20260506 22:32:23.755 | INFO | Analyzing chunk frames = [4965888,4966400) | 20260506 22:32:23.756 | INFO | MemInfo: { | 20260506 22:32:23.756 | INFO | "gpu" : " 17 GB", | 20260506 22:32:23.756 | INFO | "huge" : "", | 20260506 22:32:23.756 | INFO | "huge_shared" : " 27 GB", | 20260506 22:32:23.756 | INFO | "malloc" : " 14 GB", | 20260506 22:32:23.756 | INFO | "pinned" : " 29 GB" | 20260506 22:32:23.756 | INFO | } | 20260506 22:32:23.756 | INFO | Perf report for the last 5.14134 seconds: | 20260506 22:32:23.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:32:23.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:23.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:23.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:23.756 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260506 22:32:23.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:23.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:23.756 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:32:23.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:23.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:23.756 | INFO | Uploaded 36GiB to GPU in 2.01304s (17.8834GiB/s) | 20260506 22:32:23.756 | INFO | Downloaded 34GiB to GPU in 1.90867s (17.8134GiB/s) | 20260506 22:32:28.899 | INFO | Analyzing chunk frames = [4966400,4966912) | 20260506 22:32:28.899 | INFO | MemInfo: { | 20260506 22:32:28.899 | INFO | "gpu" : " 17 GB", | 20260506 22:32:28.899 | INFO | "huge" : "", | 20260506 22:32:28.899 | INFO | "huge_shared" : " 27 GB", | 20260506 22:32:28.899 | INFO | "malloc" : " 14 GB", | 20260506 22:32:28.899 | INFO | "pinned" : " 29 GB" | 20260506 22:32:28.899 | INFO | } | 20260506 22:32:28.899 | INFO | Perf report for the last 5.14335 seconds: | 20260506 22:32:28.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:32:28.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:28.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:28.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:28.899 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260506 22:32:28.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:28.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:28.899 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:32:28.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:28.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:28.899 | INFO | Uploaded 36GiB to GPU in 2.01915s (17.8293GiB/s) | 20260506 22:32:28.899 | INFO | Downloaded 34GiB to GPU in 1.94216s (17.5063GiB/s) | 20260506 22:32:34.030 | INFO | Analyzing chunk frames = [4966912,4967424) | 20260506 22:32:34.030 | INFO | MemInfo: { | 20260506 22:32:34.030 | INFO | "gpu" : " 17 GB", | 20260506 22:32:34.030 | INFO | "huge" : "", | 20260506 22:32:34.030 | INFO | "huge_shared" : " 27 GB", | 20260506 22:32:34.030 | INFO | "malloc" : " 14 GB", | 20260506 22:32:34.030 | INFO | "pinned" : " 29 GB" | 20260506 22:32:34.030 | INFO | } | 20260506 22:32:34.030 | INFO | Perf report for the last 5.13121 seconds: | 20260506 22:32:34.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:32:34.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:34.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:34.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:34.030 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 22:32:34.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:34.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:34.030 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:32:34.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:34.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:34.031 | INFO | Uploaded 36GiB to GPU in 2.02454s (17.7818GiB/s) | 20260506 22:32:34.031 | INFO | Downloaded 34GiB to GPU in 1.93535s (17.5679GiB/s) | 20260506 22:32:39.168 | INFO | Analyzing chunk frames = [4967424,4967936) | 20260506 22:32:39.168 | INFO | MemInfo: { | 20260506 22:32:39.168 | INFO | "gpu" : " 17 GB", | 20260506 22:32:39.168 | INFO | "huge" : "", | 20260506 22:32:39.168 | INFO | "huge_shared" : " 27 GB", | 20260506 22:32:39.168 | INFO | "malloc" : " 14 GB", | 20260506 22:32:39.168 | INFO | "pinned" : " 29 GB" | 20260506 22:32:39.168 | INFO | } | 20260506 22:32:39.168 | INFO | Perf report for the last 5.13768 seconds: | 20260506 22:32:39.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:32:39.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:39.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:39.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:39.168 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:32:39.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:39.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:39.168 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 22:32:39.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:39.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:39.168 | INFO | Uploaded 36GiB to GPU in 1.97983s (18.1834GiB/s) | 20260506 22:32:39.168 | INFO | Downloaded 34GiB to GPU in 2.00021s (16.9983GiB/s) | 20260506 22:32:44.305 | INFO | Analyzing chunk frames = [4967936,4968448) | 20260506 22:32:44.305 | INFO | MemInfo: { | 20260506 22:32:44.305 | INFO | "gpu" : " 17 GB", | 20260506 22:32:44.305 | INFO | "huge" : "", | 20260506 22:32:44.305 | INFO | "huge_shared" : " 27 GB", | 20260506 22:32:44.305 | INFO | "malloc" : " 14 GB", | 20260506 22:32:44.305 | INFO | "pinned" : " 29 GB" | 20260506 22:32:44.305 | INFO | } | 20260506 22:32:44.306 | INFO | Perf report for the last 5.13752 seconds: | 20260506 22:32:44.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:32:44.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:44.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:44.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:44.306 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:32:44.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:44.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:44.306 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:32:44.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:44.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:44.306 | INFO | Uploaded 36GiB to GPU in 1.99224s (18.0701GiB/s) | 20260506 22:32:44.306 | INFO | Downloaded 34GiB to GPU in 1.98137s (17.1599GiB/s) | 20260506 22:32:44.404 | INFO | Starting a round of Baz IO | 20260506 22:32:47.530 | INFO | Populating RTMetrics report | 20260506 22:32:47.530 | INFO | RTMetrics report populated | 20260506 22:32:47.530 | INFO | Opening tmp RTMetrics json file | 20260506 22:32:47.531 | INFO | Serializing RTMetrics to json | 20260506 22:32:47.532 | INFO | Serialization done, swapping tmp file now... | 20260506 22:32:47.532 | INFO | RTMetrics done | 20260506 22:32:47.626 | INFO | Finished a round of Baz IO: Wrote 1.909 GiB in 3.222 seconds | 20260506 22:32:47.626 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.66% / 0.00% / 66.34% | 20260506 22:32:47.626 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.495529% / 3.97364e-06% / 5.16574e-05% | 20260506 22:32:47.626 | INFO | Baz IO overall: | 20260506 22:32:47.626 | INFO | BytesWritten : 2609356003093 | 20260506 22:32:47.626 | INFO | NumEvents : 1258900806245 | 20260506 22:32:49.442 | INFO | Analyzing chunk frames = [4968448,4968960) | 20260506 22:32:49.442 | INFO | MemInfo: { | 20260506 22:32:49.442 | INFO | "gpu" : " 17 GB", | 20260506 22:32:49.442 | INFO | "huge" : "", | 20260506 22:32:49.442 | INFO | "huge_shared" : " 27 GB", | 20260506 22:32:49.443 | INFO | "malloc" : " 14 GB", | 20260506 22:32:49.443 | INFO | "pinned" : " 29 GB" | 20260506 22:32:49.443 | INFO | } | 20260506 22:32:49.443 | INFO | Perf report for the last 5.13699 seconds: | 20260506 22:32:49.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:32:49.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:49.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:49.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:49.443 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260506 22:32:49.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:49.443 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 22:32:49.443 | INFO | PRE_HQ : 59.8%, 1.8, 56.9ms, 0 | 20260506 22:32:49.443 | INFO | BAZWRITER : 62.3%, 3.5, 112.1ms, 0 | 20260506 22:32:49.443 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:32:49.443 | INFO | Uploaded 36GiB to GPU in 2.04676s (17.5887GiB/s) | 20260506 22:32:49.443 | INFO | Downloaded 38GiB to GPU in 2.174s (17.4793GiB/s) | 20260506 22:32:54.592 | INFO | Analyzing chunk frames = [4968960,4969472) | 20260506 22:32:54.592 | INFO | MemInfo: { | 20260506 22:32:54.592 | INFO | "gpu" : " 17 GB", | 20260506 22:32:54.592 | INFO | "huge" : "", | 20260506 22:32:54.592 | INFO | "huge_shared" : " 27 GB", | 20260506 22:32:54.593 | INFO | "malloc" : " 14 GB", | 20260506 22:32:54.593 | INFO | "pinned" : " 29 GB" | 20260506 22:32:54.593 | INFO | } | 20260506 22:32:54.593 | INFO | Perf report for the last 5.15003 seconds: | 20260506 22:32:54.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:32:54.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:54.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:54.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:54.593 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 22:32:54.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:54.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:54.593 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:32:54.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:54.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:54.593 | INFO | Uploaded 36GiB to GPU in 2.01433s (17.872GiB/s) | 20260506 22:32:54.593 | INFO | Downloaded 34GiB to GPU in 1.92718s (17.6424GiB/s) | 20260506 22:32:59.728 | INFO | Analyzing chunk frames = [4969472,4969984) | 20260506 22:32:59.728 | INFO | MemInfo: { | 20260506 22:32:59.728 | INFO | "gpu" : " 17 GB", | 20260506 22:32:59.728 | INFO | "huge" : "", | 20260506 22:32:59.728 | INFO | "huge_shared" : " 27 GB", | 20260506 22:32:59.728 | INFO | "malloc" : " 14 GB", | 20260506 22:32:59.728 | INFO | "pinned" : " 29 GB" | 20260506 22:32:59.728 | INFO | } | 20260506 22:32:59.728 | INFO | Perf report for the last 5.13574 seconds: | 20260506 22:32:59.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:32:59.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:59.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:59.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:59.728 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 22:32:59.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:59.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:59.728 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:32:59.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:59.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:32:59.729 | INFO | Uploaded 36GiB to GPU in 1.99672s (18.0296GiB/s) | 20260506 22:32:59.729 | INFO | Downloaded 34GiB to GPU in 1.92639s (17.6496GiB/s) | 20260506 22:33:04.864 | INFO | Analyzing chunk frames = [4969984,4970496) | 20260506 22:33:04.864 | INFO | MemInfo: { | 20260506 22:33:04.864 | INFO | "gpu" : " 17 GB", | 20260506 22:33:04.864 | INFO | "huge" : "", | 20260506 22:33:04.864 | INFO | "huge_shared" : " 27 GB", | 20260506 22:33:04.864 | INFO | "malloc" : " 14 GB", | 20260506 22:33:04.864 | INFO | "pinned" : " 29 GB" | 20260506 22:33:04.864 | INFO | } | 20260506 22:33:04.865 | INFO | Perf report for the last 5.13618 seconds: | 20260506 22:33:04.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:33:04.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:04.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:04.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:04.865 | INFO | ANALYSIS : 59.7%, 3.0, 92.4ms, 0 | 20260506 22:33:04.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:04.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:04.865 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:33:04.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:04.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:04.865 | INFO | Uploaded 36GiB to GPU in 1.96172s (18.3512GiB/s) | 20260506 22:33:04.865 | INFO | Downloaded 34GiB to GPU in 1.87679s (18.116GiB/s) | 20260506 22:33:10.003 | INFO | Analyzing chunk frames = [4970496,4971008) | 20260506 22:33:10.003 | INFO | MemInfo: { | 20260506 22:33:10.003 | INFO | "gpu" : " 17 GB", | 20260506 22:33:10.003 | INFO | "huge" : "", | 20260506 22:33:10.003 | INFO | "huge_shared" : " 27 GB", | 20260506 22:33:10.003 | INFO | "malloc" : " 14 GB", | 20260506 22:33:10.003 | INFO | "pinned" : " 29 GB" | 20260506 22:33:10.003 | INFO | } | 20260506 22:33:10.003 | INFO | Perf report for the last 5.13862 seconds: | 20260506 22:33:10.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:33:10.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:10.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:10.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:10.003 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 22:33:10.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:10.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:10.003 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:33:10.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:10.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:10.003 | INFO | Uploaded 36GiB to GPU in 2.02074s (17.8153GiB/s) | 20260506 22:33:10.003 | INFO | Downloaded 34GiB to GPU in 1.91716s (17.7345GiB/s) | 20260506 22:33:15.141 | INFO | Analyzing chunk frames = [4971008,4971520) | 20260506 22:33:15.141 | INFO | MemInfo: { | 20260506 22:33:15.141 | INFO | "gpu" : " 17 GB", | 20260506 22:33:15.141 | INFO | "huge" : "", | 20260506 22:33:15.141 | INFO | "huge_shared" : " 27 GB", | 20260506 22:33:15.141 | INFO | "malloc" : " 14 GB", | 20260506 22:33:15.141 | INFO | "pinned" : " 29 GB" | 20260506 22:33:15.141 | INFO | } | 20260506 22:33:15.142 | INFO | Perf report for the last 5.13832 seconds: | 20260506 22:33:15.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:33:15.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:15.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:15.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:15.142 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 22:33:15.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:15.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:15.142 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 22:33:15.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:15.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:15.142 | INFO | Uploaded 36GiB to GPU in 2.01461s (17.8695GiB/s) | 20260506 22:33:15.142 | INFO | Downloaded 34GiB to GPU in 1.93326s (17.5868GiB/s) | 20260506 22:33:20.280 | INFO | Analyzing chunk frames = [4971520,4972032) | 20260506 22:33:20.280 | INFO | MemInfo: { | 20260506 22:33:20.280 | INFO | "gpu" : " 17 GB", | 20260506 22:33:20.280 | INFO | "huge" : "", | 20260506 22:33:20.280 | INFO | "huge_shared" : " 27 GB", | 20260506 22:33:20.280 | INFO | "malloc" : " 14 GB", | 20260506 22:33:20.280 | INFO | "pinned" : " 29 GB" | 20260506 22:33:20.280 | INFO | } | 20260506 22:33:20.280 | INFO | Perf report for the last 5.13855 seconds: | 20260506 22:33:20.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:33:20.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:20.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:20.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:20.280 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 22:33:20.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:20.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:20.280 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 22:33:20.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:20.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:20.280 | INFO | Uploaded 36GiB to GPU in 1.96503s (18.3203GiB/s) | 20260506 22:33:20.280 | INFO | Downloaded 34GiB to GPU in 1.98467s (17.1313GiB/s) | 20260506 22:33:25.421 | INFO | Analyzing chunk frames = [4972032,4972544) | 20260506 22:33:25.421 | INFO | MemInfo: { | 20260506 22:33:25.421 | INFO | "gpu" : " 17 GB", | 20260506 22:33:25.421 | INFO | "huge" : "", | 20260506 22:33:25.421 | INFO | "huge_shared" : " 27 GB", | 20260506 22:33:25.421 | INFO | "malloc" : " 14 GB", | 20260506 22:33:25.421 | INFO | "pinned" : " 29 GB" | 20260506 22:33:25.421 | INFO | } | 20260506 22:33:25.421 | INFO | Perf report for the last 5.14095 seconds: | 20260506 22:33:25.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:33:25.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:25.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:25.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:25.421 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 22:33:25.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:25.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:25.421 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 22:33:25.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:25.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:25.421 | INFO | Uploaded 36GiB to GPU in 2.01637s (17.8539GiB/s) | 20260506 22:33:25.422 | INFO | Downloaded 34GiB to GPU in 1.98802s (17.1025GiB/s) | 20260506 22:33:25.422 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:33:25.422 | INFO | Total mem usage: 18 GiB | 20260506 22:33:25.422 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:33:25.422 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.9%) | 20260506 22:33:25.422 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:33:25.422 | INFO | High Water Breakdown: | 20260506 22:33:25.422 | INFO | 60.2% : 8.6 GiB ( 75.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:33:25.422 | INFO | 39.7% : 5.68 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:33:25.422 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:33:25.422 | INFO | ------------------------------------------ | 20260506 22:33:25.422 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:33:25.422 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:33:25.422 | INFO | Total mem usage: 26.5 GiB | 20260506 22:33:25.422 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:33:25.422 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:33:25.422 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:33:25.422 | INFO | High Water Breakdown: | 20260506 22:33:25.422 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:33:25.422 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:33:25.422 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:33:25.422 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:33:25.422 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:33:25.422 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:33:25.422 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:33:25.422 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:33:25.422 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:33:25.422 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:33:25.422 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:33:25.422 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:33:25.422 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:33:25.422 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:33:25.422 | INFO | ------------------------------------------ | 20260506 22:33:25.422 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:33:25.422 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:33:25.422 | INFO | Total mem usage: 25.6 GiB | 20260506 22:33:25.422 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:33:25.422 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:33:25.422 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:33:25.422 | INFO | High Water Breakdown: | 20260506 22:33:25.422 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:33:25.422 | INFO | ------------------------------------------ | 20260506 22:33:25.422 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:33:25.422 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:33:25.422 | INFO | Total mem usage: 15.7 GiB | 20260506 22:33:25.422 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:33:25.422 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:33:25.422 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:33:25.422 | INFO | High Water Breakdown: | 20260506 22:33:25.422 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:33:25.422 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:33:25.422 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:33:25.422 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:33:25.422 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:33:25.422 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:33:25.422 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:33:25.422 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:33:25.422 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:33:25.422 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:33:25.422 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:33:25.422 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:33:25.422 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:33:25.422 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:33:25.422 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:33:25.422 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:33:25.422 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:33:25.422 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:33:25.422 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:33:25.422 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:33:25.422 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:33:25.422 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:33:25.422 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:33:25.422 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:33:25.422 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:33:25.422 | INFO | ------------------------------------------ | 20260506 22:33:25.422 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:33:25.423 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:33:25.423 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:33:25.423 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:33:25.423 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.62e+01, 1.33e+01 / 2.48e+00 | 20260506 22:33:25.423 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 1.05e-02 / 5.21e-03 | 20260506 22:33:25.423 | INFO | PerfReport: 11.67%, Baseline -- 3.62e+00 / 4.35e+00, 5.51e-01 / 3.52e-01 | 20260506 22:33:25.423 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.44e+00, 1.83e-01 / 1.40e-01 | 20260506 22:33:25.423 | INFO | PerfReport: 16.11%, DME -- 4.00e+01 / 4.54e+01, 4.36e+00 / 2.75e+00 | 20260506 22:33:25.423 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.35e+00 / 1.46e+00 | 20260506 22:33:25.423 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.20e+00 / 2.74e+00, 3.78e-01 / 1.58e-01 | 20260506 22:33:25.423 | INFO | PerfReport: 3.22%, Metrics -- 1.00e+00 / 1.34e+00, 2.63e-01 / 1.23e-01 | 20260506 22:33:25.423 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.41e-01 / 3.85e-01, 5.30e-02 / 1.21e-02 | 20260506 22:33:25.423 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:33:25.423 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:33:25.423 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:33:25.423 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 22:33:25.423 | INFO | PerfReport: 21.57%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 22:33:25.423 | INFO | PerfReport: 21.19%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 22:33:25.423 | INFO | PerfReport: 32.08%, Compute -- 3.11e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 22:33:25.423 | INFO | PerfReport: 1.37%, LockUpload -- 1.33e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260506 22:33:25.423 | INFO | PerfReport: 1.17%, LockDownload -- 1.13e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260506 22:33:25.423 | INFO | PerfReport: 22.60%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 22:33:25.423 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260506 22:33:25.519 | INFO | Starting a round of Baz IO | 20260506 22:33:28.655 | INFO | Populating RTMetrics report | 20260506 22:33:28.655 | INFO | RTMetrics report populated | 20260506 22:33:28.655 | INFO | Opening tmp RTMetrics json file | 20260506 22:33:28.656 | INFO | Serializing RTMetrics to json | 20260506 22:33:28.657 | INFO | Serialization done, swapping tmp file now... | 20260506 22:33:28.657 | INFO | RTMetrics done | 20260506 22:33:28.749 | INFO | Finished a round of Baz IO: Wrote 1.908 GiB in 3.23 seconds | 20260506 22:33:28.749 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.65% / 0.00% / 66.34% | 20260506 22:33:28.749 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.492907% / 3.97364e-06% / 5.16574e-05% | 20260506 22:33:28.749 | INFO | Baz IO overall: | 20260506 22:33:28.750 | INFO | BytesWritten : 2611404521449 | 20260506 22:33:28.750 | INFO | NumEvents : 1259878221255 | 20260506 22:33:30.562 | INFO | Analyzing chunk frames = [4972544,4973056) | 20260506 22:33:30.562 | INFO | MemInfo: { | 20260506 22:33:30.562 | INFO | "gpu" : " 17 GB", | 20260506 22:33:30.562 | INFO | "huge" : "", | 20260506 22:33:30.562 | INFO | "huge_shared" : " 27 GB", | 20260506 22:33:30.562 | INFO | "malloc" : " 14 GB", | 20260506 22:33:30.562 | INFO | "pinned" : " 29 GB" | 20260506 22:33:30.562 | INFO | } | 20260506 22:33:30.562 | INFO | Perf report for the last 5.14112 seconds: | 20260506 22:33:30.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:33:30.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:30.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:30.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:30.562 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 22:33:30.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:30.562 | INFO | RT_METRICS : 42.4%, 1.0, 22.3ms, 0 | 20260506 22:33:30.562 | INFO | PRE_HQ : 59.6%, 1.8, 56.9ms, 0 | 20260506 22:33:30.562 | INFO | BAZWRITER : 62.4%, 3.5, 111.9ms, 0 | 20260506 22:33:30.562 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:33:30.563 | INFO | Uploaded 36GiB to GPU in 2.02303s (17.7951GiB/s) | 20260506 22:33:30.563 | INFO | Downloaded 38GiB to GPU in 2.16735s (17.5329GiB/s) | 20260506 22:33:35.704 | INFO | Analyzing chunk frames = [4973056,4973568) | 20260506 22:33:35.704 | INFO | MemInfo: { | 20260506 22:33:35.704 | INFO | "gpu" : " 17 GB", | 20260506 22:33:35.704 | INFO | "huge" : "", | 20260506 22:33:35.704 | INFO | "huge_shared" : " 27 GB", | 20260506 22:33:35.704 | INFO | "malloc" : " 14 GB", | 20260506 22:33:35.704 | INFO | "pinned" : " 29 GB" | 20260506 22:33:35.704 | INFO | } | 20260506 22:33:35.704 | INFO | Perf report for the last 5.14186 seconds: | 20260506 22:33:35.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:33:35.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:35.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:35.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:35.704 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:33:35.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:35.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:35.704 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:33:35.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:35.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:35.704 | INFO | Uploaded 36GiB to GPU in 2.01118s (17.8999GiB/s) | 20260506 22:33:35.704 | INFO | Downloaded 34GiB to GPU in 1.92666s (17.6471GiB/s) | 20260506 22:33:40.849 | INFO | Analyzing chunk frames = [4973568,4974080) | 20260506 22:33:40.849 | INFO | MemInfo: { | 20260506 22:33:40.849 | INFO | "gpu" : " 17 GB", | 20260506 22:33:40.849 | INFO | "huge" : "", | 20260506 22:33:40.849 | INFO | "huge_shared" : " 27 GB", | 20260506 22:33:40.849 | INFO | "malloc" : " 14 GB", | 20260506 22:33:40.849 | INFO | "pinned" : " 29 GB" | 20260506 22:33:40.849 | INFO | } | 20260506 22:33:40.849 | INFO | Perf report for the last 5.14473 seconds: | 20260506 22:33:40.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:33:40.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:40.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:40.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:40.849 | INFO | ANALYSIS : 60.3%, 3.0, 93.6ms, 0 | 20260506 22:33:40.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:40.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:40.849 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:33:40.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:40.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:40.849 | INFO | Uploaded 36GiB to GPU in 1.98847s (18.1044GiB/s) | 20260506 22:33:40.849 | INFO | Downloaded 34GiB to GPU in 1.92934s (17.6226GiB/s) | 20260506 22:33:45.987 | INFO | Analyzing chunk frames = [4974080,4974592) | 20260506 22:33:45.987 | INFO | MemInfo: { | 20260506 22:33:45.987 | INFO | "gpu" : " 17 GB", | 20260506 22:33:45.987 | INFO | "huge" : "", | 20260506 22:33:45.987 | INFO | "huge_shared" : " 27 GB", | 20260506 22:33:45.987 | INFO | "malloc" : " 14 GB", | 20260506 22:33:45.987 | INFO | "pinned" : " 29 GB" | 20260506 22:33:45.987 | INFO | } | 20260506 22:33:45.987 | INFO | Perf report for the last 5.13858 seconds: | 20260506 22:33:45.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:33:45.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:45.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:45.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:45.987 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260506 22:33:45.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:45.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:45.987 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:33:45.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:45.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:45.988 | INFO | Uploaded 36GiB to GPU in 2.00753s (17.9325GiB/s) | 20260506 22:33:45.988 | INFO | Downloaded 34GiB to GPU in 1.90746s (17.8247GiB/s) | 20260506 22:33:51.117 | INFO | Analyzing chunk frames = [4974592,4975104) | 20260506 22:33:51.117 | INFO | MemInfo: { | 20260506 22:33:51.117 | INFO | "gpu" : " 17 GB", | 20260506 22:33:51.117 | INFO | "huge" : "", | 20260506 22:33:51.117 | INFO | "huge_shared" : " 27 GB", | 20260506 22:33:51.117 | INFO | "malloc" : " 14 GB", | 20260506 22:33:51.117 | INFO | "pinned" : " 29 GB" | 20260506 22:33:51.117 | INFO | } | 20260506 22:33:51.117 | INFO | Perf report for the last 5.12974 seconds: | 20260506 22:33:51.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:33:51.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:51.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:51.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:51.117 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 22:33:51.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:51.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:51.117 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:33:51.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:51.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:51.117 | INFO | Uploaded 36GiB to GPU in 1.99505s (18.0447GiB/s) | 20260506 22:33:51.117 | INFO | Downloaded 34GiB to GPU in 1.92386s (17.6728GiB/s) | 20260506 22:33:56.256 | INFO | Analyzing chunk frames = [4975104,4975616) | 20260506 22:33:56.256 | INFO | MemInfo: { | 20260506 22:33:56.256 | INFO | "gpu" : " 17 GB", | 20260506 22:33:56.256 | INFO | "huge" : "", | 20260506 22:33:56.256 | INFO | "huge_shared" : " 27 GB", | 20260506 22:33:56.256 | INFO | "malloc" : " 14 GB", | 20260506 22:33:56.256 | INFO | "pinned" : " 29 GB" | 20260506 22:33:56.256 | INFO | } | 20260506 22:33:56.256 | INFO | Perf report for the last 5.13867 seconds: | 20260506 22:33:56.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:33:56.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:56.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:56.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:56.256 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:33:56.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:56.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:56.256 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260506 22:33:56.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:56.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:33:56.256 | INFO | Uploaded 36GiB to GPU in 2.027s (17.7602GiB/s) | 20260506 22:33:56.256 | INFO | Downloaded 34GiB to GPU in 1.94136s (17.5135GiB/s) | 20260506 22:34:01.399 | INFO | Analyzing chunk frames = [4975616,4976128) | 20260506 22:34:01.399 | INFO | MemInfo: { | 20260506 22:34:01.399 | INFO | "gpu" : " 17 GB", | 20260506 22:34:01.399 | INFO | "huge" : "", | 20260506 22:34:01.399 | INFO | "huge_shared" : " 27 GB", | 20260506 22:34:01.399 | INFO | "malloc" : " 14 GB", | 20260506 22:34:01.399 | INFO | "pinned" : " 29 GB" | 20260506 22:34:01.399 | INFO | } | 20260506 22:34:01.399 | INFO | Perf report for the last 5.14315 seconds: | 20260506 22:34:01.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:34:01.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:01.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:01.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:01.399 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 22:34:01.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:01.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:01.399 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 22:34:01.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:01.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:01.399 | INFO | Uploaded 36GiB to GPU in 1.96563s (18.3147GiB/s) | 20260506 22:34:01.399 | INFO | Downloaded 34GiB to GPU in 1.99404s (17.0508GiB/s) | 20260506 22:34:06.536 | INFO | Analyzing chunk frames = [4976128,4976640) | 20260506 22:34:06.536 | INFO | MemInfo: { | 20260506 22:34:06.536 | INFO | "gpu" : " 17 GB", | 20260506 22:34:06.536 | INFO | "huge" : "", | 20260506 22:34:06.536 | INFO | "huge_shared" : " 27 GB", | 20260506 22:34:06.536 | INFO | "malloc" : " 14 GB", | 20260506 22:34:06.536 | INFO | "pinned" : " 29 GB" | 20260506 22:34:06.536 | INFO | } | 20260506 22:34:06.536 | INFO | Perf report for the last 5.13683 seconds: | 20260506 22:34:06.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:34:06.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:06.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:06.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:06.536 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 22:34:06.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:06.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:06.536 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 22:34:06.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:06.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:06.536 | INFO | Uploaded 36GiB to GPU in 2.02122s (17.811GiB/s) | 20260506 22:34:06.536 | INFO | Downloaded 34GiB to GPU in 1.99734s (17.0226GiB/s) | 20260506 22:34:06.642 | INFO | Starting a round of Baz IO | 20260506 22:34:09.770 | INFO | Populating RTMetrics report | 20260506 22:34:09.770 | INFO | RTMetrics report populated | 20260506 22:34:09.770 | INFO | Opening tmp RTMetrics json file | 20260506 22:34:09.771 | INFO | Serializing RTMetrics to json | 20260506 22:34:09.772 | INFO | Serialization done, swapping tmp file now... | 20260506 22:34:09.772 | INFO | RTMetrics done | 20260506 22:34:09.869 | INFO | Finished a round of Baz IO: Wrote 1.908 GiB in 3.227 seconds | 20260506 22:34:09.869 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.65% / 0.00% / 66.35% | 20260506 22:34:09.869 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.493487% / 0% / 6.75519e-05% | 20260506 22:34:09.869 | INFO | Baz IO overall: | 20260506 22:34:09.869 | INFO | BytesWritten : 2613453620987 | 20260506 22:34:09.869 | INFO | NumEvents : 1260856180685 | 20260506 22:34:11.675 | INFO | Analyzing chunk frames = [4976640,4977152) | 20260506 22:34:11.675 | INFO | MemInfo: { | 20260506 22:34:11.675 | INFO | "gpu" : " 17 GB", | 20260506 22:34:11.675 | INFO | "huge" : "", | 20260506 22:34:11.675 | INFO | "huge_shared" : " 27 GB", | 20260506 22:34:11.676 | INFO | "malloc" : " 14 GB", | 20260506 22:34:11.676 | INFO | "pinned" : " 29 GB" | 20260506 22:34:11.676 | INFO | } | 20260506 22:34:11.676 | INFO | Perf report for the last 5.13979 seconds: | 20260506 22:34:11.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:34:11.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:11.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:11.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:11.676 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 22:34:11.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:11.676 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 22:34:11.676 | INFO | PRE_HQ : 59.6%, 1.8, 57.1ms, 0 | 20260506 22:34:11.676 | INFO | BAZWRITER : 62.4%, 3.5, 112.3ms, 0 | 20260506 22:34:11.676 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:34:11.676 | INFO | Uploaded 36GiB to GPU in 2.01011s (17.9095GiB/s) | 20260506 22:34:11.676 | INFO | Downloaded 38GiB to GPU in 2.1815s (17.4192GiB/s) | 20260506 22:34:16.815 | INFO | Analyzing chunk frames = [4977152,4977664) | 20260506 22:34:16.815 | INFO | MemInfo: { | 20260506 22:34:16.815 | INFO | "gpu" : " 17 GB", | 20260506 22:34:16.815 | INFO | "huge" : "", | 20260506 22:34:16.815 | INFO | "huge_shared" : " 27 GB", | 20260506 22:34:16.815 | INFO | "malloc" : " 14 GB", | 20260506 22:34:16.815 | INFO | "pinned" : " 29 GB" | 20260506 22:34:16.815 | INFO | } | 20260506 22:34:16.815 | INFO | Perf report for the last 5.13937 seconds: | 20260506 22:34:16.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:34:16.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:16.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:16.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:16.815 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:34:16.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:16.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:16.815 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:34:16.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:16.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:16.815 | INFO | Uploaded 36GiB to GPU in 1.99016s (18.089GiB/s) | 20260506 22:34:16.815 | INFO | Downloaded 34GiB to GPU in 1.92591s (17.6539GiB/s) | 20260506 22:34:21.958 | INFO | Analyzing chunk frames = [4977664,4978176) | 20260506 22:34:21.958 | INFO | MemInfo: { | 20260506 22:34:21.958 | INFO | "gpu" : " 17 GB", | 20260506 22:34:21.958 | INFO | "huge" : "", | 20260506 22:34:21.958 | INFO | "huge_shared" : " 27 GB", | 20260506 22:34:21.958 | INFO | "malloc" : " 14 GB", | 20260506 22:34:21.958 | INFO | "pinned" : " 29 GB" | 20260506 22:34:21.958 | INFO | } | 20260506 22:34:21.958 | INFO | Perf report for the last 5.14273 seconds: | 20260506 22:34:21.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:34:21.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:21.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:21.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:21.958 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 22:34:21.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:21.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:21.958 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:34:21.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:21.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:21.958 | INFO | Uploaded 36GiB to GPU in 1.96027s (18.3648GiB/s) | 20260506 22:34:21.958 | INFO | Downloaded 34GiB to GPU in 1.92595s (17.6536GiB/s) | 20260506 22:34:27.095 | INFO | Analyzing chunk frames = [4978176,4978688) | 20260506 22:34:27.095 | INFO | MemInfo: { | 20260506 22:34:27.095 | INFO | "gpu" : " 17 GB", | 20260506 22:34:27.095 | INFO | "huge" : "", | 20260506 22:34:27.095 | INFO | "huge_shared" : " 27 GB", | 20260506 22:34:27.095 | INFO | "malloc" : " 14 GB", | 20260506 22:34:27.095 | INFO | "pinned" : " 29 GB" | 20260506 22:34:27.095 | INFO | } | 20260506 22:34:27.095 | INFO | Perf report for the last 5.137 seconds: | 20260506 22:34:27.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:34:27.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:27.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:27.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:27.095 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260506 22:34:27.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:27.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:27.095 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:34:27.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:27.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:27.095 | INFO | Uploaded 36GiB to GPU in 2.01188s (17.8937GiB/s) | 20260506 22:34:27.095 | INFO | Downloaded 34GiB to GPU in 1.90864s (17.8137GiB/s) | 20260506 22:34:32.234 | INFO | Analyzing chunk frames = [4978688,4979200) | 20260506 22:34:32.234 | INFO | MemInfo: { | 20260506 22:34:32.234 | INFO | "gpu" : " 17 GB", | 20260506 22:34:32.234 | INFO | "huge" : "", | 20260506 22:34:32.234 | INFO | "huge_shared" : " 27 GB", | 20260506 22:34:32.234 | INFO | "malloc" : " 14 GB", | 20260506 22:34:32.234 | INFO | "pinned" : " 29 GB" | 20260506 22:34:32.234 | INFO | } | 20260506 22:34:32.234 | INFO | Perf report for the last 5.1393 seconds: | 20260506 22:34:32.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:34:32.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:32.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:32.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:32.234 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 22:34:32.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:32.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:32.234 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:34:32.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:32.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:32.234 | INFO | Uploaded 36GiB to GPU in 1.967s (18.302GiB/s) | 20260506 22:34:32.234 | INFO | Downloaded 34GiB to GPU in 1.93086s (17.6088GiB/s) | 20260506 22:34:37.372 | INFO | Analyzing chunk frames = [4979200,4979712) | 20260506 22:34:37.372 | INFO | MemInfo: { | 20260506 22:34:37.372 | INFO | "gpu" : " 17 GB", | 20260506 22:34:37.372 | INFO | "huge" : "", | 20260506 22:34:37.372 | INFO | "huge_shared" : " 27 GB", | 20260506 22:34:37.372 | INFO | "malloc" : " 14 GB", | 20260506 22:34:37.372 | INFO | "pinned" : " 29 GB" | 20260506 22:34:37.372 | INFO | } | 20260506 22:34:37.373 | INFO | Perf report for the last 5.1386 seconds: | 20260506 22:34:37.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:34:37.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:37.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:37.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:37.373 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 22:34:37.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:37.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:37.373 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:34:37.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:37.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:37.373 | INFO | Uploaded 36GiB to GPU in 1.99967s (18.003GiB/s) | 20260506 22:34:37.373 | INFO | Downloaded 34GiB to GPU in 1.94841s (17.4502GiB/s) | 20260506 22:34:42.509 | INFO | Analyzing chunk frames = [4979712,4980224) | 20260506 22:34:42.509 | INFO | MemInfo: { | 20260506 22:34:42.509 | INFO | "gpu" : " 17 GB", | 20260506 22:34:42.509 | INFO | "huge" : "", | 20260506 22:34:42.509 | INFO | "huge_shared" : " 27 GB", | 20260506 22:34:42.509 | INFO | "malloc" : " 14 GB", | 20260506 22:34:42.509 | INFO | "pinned" : " 29 GB" | 20260506 22:34:42.509 | INFO | } | 20260506 22:34:42.509 | INFO | Perf report for the last 5.13618 seconds: | 20260506 22:34:42.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:34:42.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:42.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:42.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:42.509 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 22:34:42.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:42.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:42.509 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 22:34:42.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:42.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:42.509 | INFO | Uploaded 36GiB to GPU in 1.98278s (18.1563GiB/s) | 20260506 22:34:42.509 | INFO | Downloaded 34GiB to GPU in 1.99874s (17.0107GiB/s) | 20260506 22:34:47.661 | INFO | Analyzing chunk frames = [4980224,4980736) | 20260506 22:34:47.661 | INFO | MemInfo: { | 20260506 22:34:47.661 | INFO | "gpu" : " 17 GB", | 20260506 22:34:47.661 | INFO | "huge" : "", | 20260506 22:34:47.661 | INFO | "huge_shared" : " 27 GB", | 20260506 22:34:47.661 | INFO | "malloc" : " 14 GB", | 20260506 22:34:47.661 | INFO | "pinned" : " 29 GB" | 20260506 22:34:47.661 | INFO | } | 20260506 22:34:47.661 | INFO | Perf report for the last 5.15181 seconds: | 20260506 22:34:47.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:34:47.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:47.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:47.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:47.661 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 22:34:47.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:47.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:47.661 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260506 22:34:47.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:47.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:47.661 | INFO | Uploaded 36GiB to GPU in 1.988s (18.1086GiB/s) | 20260506 22:34:47.661 | INFO | Downloaded 34GiB to GPU in 1.98773s (17.105GiB/s) | 20260506 22:34:47.661 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:34:47.661 | INFO | Total mem usage: 18 GiB | 20260506 22:34:47.661 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:34:47.661 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 22:34:47.661 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:34:47.661 | INFO | High Water Breakdown: | 20260506 22:34:47.661 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:34:47.661 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:34:47.661 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:34:47.661 | INFO | ------------------------------------------ | 20260506 22:34:47.661 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:34:47.661 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:34:47.661 | INFO | Total mem usage: 26.5 GiB | 20260506 22:34:47.661 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:34:47.661 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:34:47.661 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:34:47.661 | INFO | High Water Breakdown: | 20260506 22:34:47.661 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:34:47.662 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:34:47.662 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:34:47.662 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:34:47.662 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:34:47.662 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:34:47.662 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:34:47.662 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:34:47.662 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:34:47.662 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:34:47.662 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:34:47.662 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:34:47.662 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:34:47.662 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:34:47.662 | INFO | ------------------------------------------ | 20260506 22:34:47.662 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:34:47.662 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:34:47.662 | INFO | Total mem usage: 25.6 GiB | 20260506 22:34:47.662 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:34:47.662 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:34:47.662 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:34:47.662 | INFO | High Water Breakdown: | 20260506 22:34:47.662 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:34:47.662 | INFO | ------------------------------------------ | 20260506 22:34:47.662 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:34:47.662 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:34:47.662 | INFO | Total mem usage: 15.7 GiB | 20260506 22:34:47.662 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:34:47.662 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:34:47.662 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:34:47.662 | INFO | High Water Breakdown: | 20260506 22:34:47.662 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:34:47.662 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:34:47.662 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:34:47.662 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:34:47.662 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:34:47.662 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:34:47.662 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:34:47.662 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:34:47.662 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:34:47.662 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:34:47.662 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:34:47.662 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:34:47.662 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:34:47.662 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:34:47.662 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:34:47.662 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:34:47.662 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:34:47.662 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:34:47.662 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:34:47.662 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:34:47.662 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:34:47.662 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:34:47.662 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:34:47.662 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:34:47.662 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:34:47.662 | INFO | ------------------------------------------ | 20260506 22:34:47.662 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:34:47.662 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:34:47.662 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:34:47.662 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:34:47.662 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.61e+01, 1.36e+01 / 2.23e+00 | 20260506 22:34:47.662 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 9.92e-03 / 6.96e-03 | 20260506 22:34:47.662 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.34e+00, 5.51e-01 / 3.73e-01 | 20260506 22:34:47.662 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.43e+00, 1.81e-01 / 1.42e-01 | 20260506 22:34:47.662 | INFO | PerfReport: 16.05%, DME -- 3.99e+01 / 4.45e+01, 4.25e+00 / 2.92e+00 | 20260506 22:34:47.662 | INFO | PerfReport: 57.31%, FrameLabeling -- 1.78e+01 / 2.12e+01, 1.38e+00 / 1.80e+00 | 20260506 22:34:47.662 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.20e+00 / 2.75e+00, 3.78e-01 / 2.09e-01 | 20260506 22:34:47.662 | INFO | PerfReport: 3.23%, Metrics -- 1.00e+00 / 1.34e+00, 2.68e-01 / 1.36e-01 | 20260506 22:34:47.662 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.43e-01 / 3.95e-01, 5.45e-02 / 2.46e-02 | 20260506 22:34:47.662 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:34:47.662 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:34:47.662 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:34:47.662 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 22:34:47.662 | INFO | PerfReport: 21.48%, Upload -- 2.08e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 22:34:47.662 | INFO | PerfReport: 21.25%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 22:34:47.662 | INFO | PerfReport: 32.08%, Compute -- 3.11e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 22:34:47.662 | INFO | PerfReport: 1.29%, LockUpload -- 1.25e+00 / 0.00e+00, 4.94e+00 / 0.00e+00 | 20260506 22:34:47.662 | INFO | PerfReport: 1.29%, LockDownload -- 1.25e+00 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260506 22:34:47.662 | INFO | PerfReport: 22.60%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 22:34:47.662 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260506 22:34:47.759 | INFO | Starting a round of Baz IO | 20260506 22:34:50.893 | INFO | Populating RTMetrics report | 20260506 22:34:50.893 | INFO | RTMetrics report populated | 20260506 22:34:50.893 | INFO | Opening tmp RTMetrics json file | 20260506 22:34:50.894 | INFO | Serializing RTMetrics to json | 20260506 22:34:50.905 | INFO | Serialization done, swapping tmp file now... | 20260506 22:34:50.915 | INFO | RTMetrics done | 20260506 22:34:50.990 | INFO | Finished a round of Baz IO: Wrote 1.909 GiB in 3.231 seconds | 20260506 22:34:50.990 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.65% / 0.00% / 66.35% | 20260506 22:34:50.990 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.493081% / 3.97364e-06% / 5.96046e-05% | 20260506 22:34:50.990 | INFO | Baz IO overall: | 20260506 22:34:50.990 | INFO | BytesWritten : 2615503415043 | 20260506 22:34:50.990 | INFO | NumEvents : 1261834531399 | 20260506 22:34:52.792 | INFO | Analyzing chunk frames = [4980736,4981248) | 20260506 22:34:52.792 | INFO | MemInfo: { | 20260506 22:34:52.792 | INFO | "gpu" : " 17 GB", | 20260506 22:34:52.792 | INFO | "huge" : "", | 20260506 22:34:52.792 | INFO | "huge_shared" : " 27 GB", | 20260506 22:34:52.792 | INFO | "malloc" : " 14 GB", | 20260506 22:34:52.792 | INFO | "pinned" : " 29 GB" | 20260506 22:34:52.792 | INFO | } | 20260506 22:34:52.792 | INFO | Perf report for the last 5.13159 seconds: | 20260506 22:34:52.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:34:52.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:52.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:52.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:52.793 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260506 22:34:52.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:52.793 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 22:34:52.793 | INFO | PRE_HQ : 60.0%, 1.8, 57.1ms, 0 | 20260506 22:34:52.793 | INFO | BAZWRITER : 62.5%, 3.4, 110.7ms, 0 | 20260506 22:34:52.793 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:34:52.793 | INFO | Uploaded 36GiB to GPU in 2.01095s (17.9019GiB/s) | 20260506 22:34:52.793 | INFO | Downloaded 38GiB to GPU in 2.17079s (17.5052GiB/s) | 20260506 22:34:57.933 | INFO | Analyzing chunk frames = [4981248,4981760) | 20260506 22:34:57.933 | INFO | MemInfo: { | 20260506 22:34:57.933 | INFO | "gpu" : " 17 GB", | 20260506 22:34:57.933 | INFO | "huge" : "", | 20260506 22:34:57.933 | INFO | "huge_shared" : " 27 GB", | 20260506 22:34:57.933 | INFO | "malloc" : " 14 GB", | 20260506 22:34:57.933 | INFO | "pinned" : " 29 GB" | 20260506 22:34:57.933 | INFO | } | 20260506 22:34:57.934 | INFO | Perf report for the last 5.1412 seconds: | 20260506 22:34:57.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:34:57.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:57.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:57.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:57.934 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 22:34:57.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:57.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:57.934 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 22:34:57.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:57.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:34:57.934 | INFO | Uploaded 36GiB to GPU in 2.00993s (17.9111GiB/s) | 20260506 22:34:57.934 | INFO | Downloaded 34GiB to GPU in 1.93146s (17.6033GiB/s) | 20260506 22:35:03.070 | INFO | Analyzing chunk frames = [4981760,4982272) | 20260506 22:35:03.070 | INFO | MemInfo: { | 20260506 22:35:03.070 | INFO | "gpu" : " 17 GB", | 20260506 22:35:03.070 | INFO | "huge" : "", | 20260506 22:35:03.070 | INFO | "huge_shared" : " 27 GB", | 20260506 22:35:03.070 | INFO | "malloc" : " 14 GB", | 20260506 22:35:03.070 | INFO | "pinned" : " 29 GB" | 20260506 22:35:03.070 | INFO | } | 20260506 22:35:03.070 | INFO | Perf report for the last 5.13689 seconds: | 20260506 22:35:03.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:35:03.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:03.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:03.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:03.071 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 22:35:03.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:03.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:03.071 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:35:03.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:03.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:03.071 | INFO | Uploaded 36GiB to GPU in 1.98531s (18.1332GiB/s) | 20260506 22:35:03.071 | INFO | Downloaded 34GiB to GPU in 1.93588s (17.5631GiB/s) | 20260506 22:35:08.207 | INFO | Analyzing chunk frames = [4982272,4982784) | 20260506 22:35:08.207 | INFO | MemInfo: { | 20260506 22:35:08.207 | INFO | "gpu" : " 17 GB", | 20260506 22:35:08.207 | INFO | "huge" : "", | 20260506 22:35:08.207 | INFO | "huge_shared" : " 27 GB", | 20260506 22:35:08.207 | INFO | "malloc" : " 14 GB", | 20260506 22:35:08.207 | INFO | "pinned" : " 29 GB" | 20260506 22:35:08.207 | INFO | } | 20260506 22:35:08.207 | INFO | Perf report for the last 5.13616 seconds: | 20260506 22:35:08.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:35:08.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:08.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:08.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:08.207 | INFO | ANALYSIS : 59.6%, 3.0, 92.3ms, 0 | 20260506 22:35:08.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:08.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:08.207 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:35:08.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:08.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:08.207 | INFO | Uploaded 36GiB to GPU in 1.94961s (18.4653GiB/s) | 20260506 22:35:08.207 | INFO | Downloaded 34GiB to GPU in 1.88401s (18.0466GiB/s) | 20260506 22:35:13.347 | INFO | Analyzing chunk frames = [4982784,4983296) | 20260506 22:35:13.347 | INFO | MemInfo: { | 20260506 22:35:13.347 | INFO | "gpu" : " 17 GB", | 20260506 22:35:13.347 | INFO | "huge" : "", | 20260506 22:35:13.347 | INFO | "huge_shared" : " 27 GB", | 20260506 22:35:13.347 | INFO | "malloc" : " 14 GB", | 20260506 22:35:13.347 | INFO | "pinned" : " 29 GB" | 20260506 22:35:13.348 | INFO | } | 20260506 22:35:13.348 | INFO | Perf report for the last 5.14097 seconds: | 20260506 22:35:13.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:35:13.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:13.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:13.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:13.348 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 22:35:13.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:13.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:13.348 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:35:13.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:13.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:13.348 | INFO | Uploaded 36GiB to GPU in 1.97547s (18.2235GiB/s) | 20260506 22:35:13.348 | INFO | Downloaded 34GiB to GPU in 1.93721s (17.551GiB/s) | 20260506 22:35:18.487 | INFO | Analyzing chunk frames = [4983296,4983808) | 20260506 22:35:18.487 | INFO | MemInfo: { | 20260506 22:35:18.487 | INFO | "gpu" : " 17 GB", | 20260506 22:35:18.487 | INFO | "huge" : "", | 20260506 22:35:18.487 | INFO | "huge_shared" : " 27 GB", | 20260506 22:35:18.487 | INFO | "malloc" : " 14 GB", | 20260506 22:35:18.487 | INFO | "pinned" : " 29 GB" | 20260506 22:35:18.487 | INFO | } | 20260506 22:35:18.487 | INFO | Perf report for the last 5.13984 seconds: | 20260506 22:35:18.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:35:18.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:18.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:18.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:18.488 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 22:35:18.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:18.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:18.488 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260506 22:35:18.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:18.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:18.488 | INFO | Uploaded 36GiB to GPU in 1.97409s (18.2363GiB/s) | 20260506 22:35:18.488 | INFO | Downloaded 34GiB to GPU in 1.9436s (17.4933GiB/s) | 20260506 22:35:23.624 | INFO | Analyzing chunk frames = [4983808,4984320) | 20260506 22:35:23.624 | INFO | MemInfo: { | 20260506 22:35:23.624 | INFO | "gpu" : " 17 GB", | 20260506 22:35:23.624 | INFO | "huge" : "", | 20260506 22:35:23.624 | INFO | "huge_shared" : " 27 GB", | 20260506 22:35:23.624 | INFO | "malloc" : " 14 GB", | 20260506 22:35:23.624 | INFO | "pinned" : " 29 GB" | 20260506 22:35:23.624 | INFO | } | 20260506 22:35:23.624 | INFO | Perf report for the last 5.13696 seconds: | 20260506 22:35:23.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:35:23.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:23.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:23.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:23.625 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:35:23.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:23.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:23.625 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 22:35:23.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:23.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:23.625 | INFO | Uploaded 36GiB to GPU in 1.96355s (18.3341GiB/s) | 20260506 22:35:23.625 | INFO | Downloaded 34GiB to GPU in 1.99771s (17.0195GiB/s) | 20260506 22:35:28.763 | INFO | Analyzing chunk frames = [4984320,4984832) | 20260506 22:35:28.763 | INFO | MemInfo: { | 20260506 22:35:28.763 | INFO | "gpu" : " 17 GB", | 20260506 22:35:28.763 | INFO | "huge" : "", | 20260506 22:35:28.763 | INFO | "huge_shared" : " 27 GB", | 20260506 22:35:28.763 | INFO | "malloc" : " 14 GB", | 20260506 22:35:28.763 | INFO | "pinned" : " 29 GB" | 20260506 22:35:28.763 | INFO | } | 20260506 22:35:28.763 | INFO | Perf report for the last 5.13906 seconds: | 20260506 22:35:28.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:35:28.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:28.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:28.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:28.764 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 22:35:28.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:28.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:28.764 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:35:28.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:28.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:28.764 | INFO | Uploaded 36GiB to GPU in 2.05337s (17.5321GiB/s) | 20260506 22:35:28.764 | INFO | Downloaded 34GiB to GPU in 2.00337s (16.9714GiB/s) | 20260506 22:35:28.886 | INFO | Starting a round of Baz IO | 20260506 22:35:32.004 | INFO | Populating RTMetrics report | 20260506 22:35:32.004 | INFO | RTMetrics report populated | 20260506 22:35:32.004 | INFO | Opening tmp RTMetrics json file | 20260506 22:35:32.012 | INFO | Serializing RTMetrics to json | 20260506 22:35:32.022 | INFO | Serialization done, swapping tmp file now... | 20260506 22:35:32.023 | INFO | RTMetrics done | 20260506 22:35:32.102 | INFO | Finished a round of Baz IO: Wrote 1.907 GiB in 3.216 seconds | 20260506 22:35:32.102 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.65% / 0.00% / 66.35% | 20260506 22:35:32.102 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.491492% / 0% / 5.16574e-05% | 20260506 22:35:32.102 | INFO | Baz IO overall: | 20260506 22:35:32.102 | INFO | BytesWritten : 2617551451639 | 20260506 22:35:32.103 | INFO | NumEvents : 1262811634707 | 20260506 22:35:33.911 | INFO | Analyzing chunk frames = [4984832,4985344) | 20260506 22:35:33.911 | INFO | MemInfo: { | 20260506 22:35:33.911 | INFO | "gpu" : " 17 GB", | 20260506 22:35:33.911 | INFO | "huge" : "", | 20260506 22:35:33.911 | INFO | "huge_shared" : " 27 GB", | 20260506 22:35:33.911 | INFO | "malloc" : " 14 GB", | 20260506 22:35:33.911 | INFO | "pinned" : " 29 GB" | 20260506 22:35:33.911 | INFO | } | 20260506 22:35:33.911 | INFO | Perf report for the last 5.14762 seconds: | 20260506 22:35:33.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:35:33.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:33.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:33.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:33.911 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 22:35:33.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:33.911 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 22:35:33.911 | INFO | PRE_HQ : 59.5%, 1.9, 57.3ms, 0 | 20260506 22:35:33.911 | INFO | BAZWRITER : 62.3%, 3.5, 111.7ms, 0 | 20260506 22:35:33.911 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:35:33.911 | INFO | Uploaded 36GiB to GPU in 2.04206s (17.6293GiB/s) | 20260506 22:35:33.911 | INFO | Downloaded 38GiB to GPU in 2.19408s (17.3194GiB/s) | 20260506 22:35:39.048 | INFO | Analyzing chunk frames = [4985344,4985856) | 20260506 22:35:39.048 | INFO | MemInfo: { | 20260506 22:35:39.048 | INFO | "gpu" : " 17 GB", | 20260506 22:35:39.048 | INFO | "huge" : "", | 20260506 22:35:39.048 | INFO | "huge_shared" : " 27 GB", | 20260506 22:35:39.048 | INFO | "malloc" : " 14 GB", | 20260506 22:35:39.048 | INFO | "pinned" : " 29 GB" | 20260506 22:35:39.048 | INFO | } | 20260506 22:35:39.048 | INFO | Perf report for the last 5.13695 seconds: | 20260506 22:35:39.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:35:39.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:39.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:39.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:39.048 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:35:39.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:39.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:39.048 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 22:35:39.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:39.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:39.048 | INFO | Uploaded 36GiB to GPU in 1.99872s (18.0115GiB/s) | 20260506 22:35:39.048 | INFO | Downloaded 34GiB to GPU in 1.92378s (17.6735GiB/s) | 20260506 22:35:44.183 | INFO | Analyzing chunk frames = [4985856,4986368) | 20260506 22:35:44.183 | INFO | MemInfo: { | 20260506 22:35:44.183 | INFO | "gpu" : " 17 GB", | 20260506 22:35:44.183 | INFO | "huge" : "", | 20260506 22:35:44.183 | INFO | "huge_shared" : " 27 GB", | 20260506 22:35:44.183 | INFO | "malloc" : " 14 GB", | 20260506 22:35:44.183 | INFO | "pinned" : " 29 GB" | 20260506 22:35:44.183 | INFO | } | 20260506 22:35:44.184 | INFO | Perf report for the last 5.13541 seconds: | 20260506 22:35:44.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:35:44.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:44.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:44.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:44.184 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260506 22:35:44.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:44.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:44.184 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:35:44.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:44.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:44.184 | INFO | Uploaded 36GiB to GPU in 1.98902s (18.0994GiB/s) | 20260506 22:35:44.184 | INFO | Downloaded 34GiB to GPU in 1.92858s (17.6296GiB/s) | 20260506 22:35:49.327 | INFO | Analyzing chunk frames = [4986368,4986880) | 20260506 22:35:49.327 | INFO | MemInfo: { | 20260506 22:35:49.327 | INFO | "gpu" : " 17 GB", | 20260506 22:35:49.328 | INFO | "huge" : "", | 20260506 22:35:49.328 | INFO | "huge_shared" : " 27 GB", | 20260506 22:35:49.328 | INFO | "malloc" : " 14 GB", | 20260506 22:35:49.328 | INFO | "pinned" : " 29 GB" | 20260506 22:35:49.328 | INFO | } | 20260506 22:35:49.328 | INFO | Perf report for the last 5.14428 seconds: | 20260506 22:35:49.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:35:49.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:49.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:49.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:49.328 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260506 22:35:49.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:49.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:49.328 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:35:49.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:49.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:49.328 | INFO | Uploaded 36GiB to GPU in 2.01708s (17.8476GiB/s) | 20260506 22:35:49.328 | INFO | Downloaded 34GiB to GPU in 1.90968s (17.804GiB/s) | 20260506 22:35:54.461 | INFO | Analyzing chunk frames = [4986880,4987392) | 20260506 22:35:54.461 | INFO | MemInfo: { | 20260506 22:35:54.461 | INFO | "gpu" : " 17 GB", | 20260506 22:35:54.461 | INFO | "huge" : "", | 20260506 22:35:54.461 | INFO | "huge_shared" : " 27 GB", | 20260506 22:35:54.462 | INFO | "malloc" : " 14 GB", | 20260506 22:35:54.462 | INFO | "pinned" : " 29 GB" | 20260506 22:35:54.462 | INFO | } | 20260506 22:35:54.462 | INFO | Perf report for the last 5.13393 seconds: | 20260506 22:35:54.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:35:54.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:54.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:54.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:54.462 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 22:35:54.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:54.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:54.462 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:35:54.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:54.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:54.462 | INFO | Uploaded 36GiB to GPU in 1.98726s (18.1154GiB/s) | 20260506 22:35:54.462 | INFO | Downloaded 34GiB to GPU in 1.93216s (17.5969GiB/s) | 20260506 22:35:59.605 | INFO | Analyzing chunk frames = [4987392,4987904) | 20260506 22:35:59.605 | INFO | MemInfo: { | 20260506 22:35:59.605 | INFO | "gpu" : " 17 GB", | 20260506 22:35:59.605 | INFO | "huge" : "", | 20260506 22:35:59.605 | INFO | "huge_shared" : " 27 GB", | 20260506 22:35:59.605 | INFO | "malloc" : " 14 GB", | 20260506 22:35:59.605 | INFO | "pinned" : " 29 GB" | 20260506 22:35:59.605 | INFO | } | 20260506 22:35:59.605 | INFO | Perf report for the last 5.14328 seconds: | 20260506 22:35:59.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:35:59.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:59.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:59.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:59.605 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 22:35:59.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:59.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:59.605 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 22:35:59.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:59.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:35:59.605 | INFO | Uploaded 36GiB to GPU in 2.01611s (17.8562GiB/s) | 20260506 22:35:59.605 | INFO | Downloaded 34GiB to GPU in 1.93943s (17.5309GiB/s) | 20260506 22:36:04.748 | INFO | Analyzing chunk frames = [4987904,4988416) | 20260506 22:36:04.748 | INFO | MemInfo: { | 20260506 22:36:04.748 | INFO | "gpu" : " 17 GB", | 20260506 22:36:04.748 | INFO | "huge" : "", | 20260506 22:36:04.748 | INFO | "huge_shared" : " 27 GB", | 20260506 22:36:04.748 | INFO | "malloc" : " 14 GB", | 20260506 22:36:04.748 | INFO | "pinned" : " 29 GB" | 20260506 22:36:04.748 | INFO | } | 20260506 22:36:04.748 | INFO | Perf report for the last 5.14294 seconds: | 20260506 22:36:04.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:36:04.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:04.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:04.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:04.748 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:36:04.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:04.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:04.748 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 22:36:04.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:04.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:04.748 | INFO | Uploaded 36GiB to GPU in 1.97427s (18.2346GiB/s) | 20260506 22:36:04.748 | INFO | Downloaded 34GiB to GPU in 1.9789s (17.1813GiB/s) | 20260506 22:36:09.884 | INFO | Analyzing chunk frames = [4988416,4988928) | 20260506 22:36:09.884 | INFO | MemInfo: { | 20260506 22:36:09.884 | INFO | "gpu" : " 17 GB", | 20260506 22:36:09.884 | INFO | "huge" : "", | 20260506 22:36:09.884 | INFO | "huge_shared" : " 27 GB", | 20260506 22:36:09.884 | INFO | "malloc" : " 14 GB", | 20260506 22:36:09.884 | INFO | "pinned" : " 29 GB" | 20260506 22:36:09.884 | INFO | } | 20260506 22:36:09.884 | INFO | Perf report for the last 5.13611 seconds: | 20260506 22:36:09.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:36:09.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:09.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:09.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:09.884 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:36:09.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:09.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:09.884 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 22:36:09.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:09.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:09.884 | INFO | Uploaded 36GiB to GPU in 1.98985s (18.0918GiB/s) | 20260506 22:36:09.884 | INFO | Downloaded 34GiB to GPU in 1.98586s (17.121GiB/s) | 20260506 22:36:09.885 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:36:09.885 | INFO | Total mem usage: 18 GiB | 20260506 22:36:09.885 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:36:09.885 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:36:09.885 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:36:09.885 | INFO | High Water Breakdown: | 20260506 22:36:09.885 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:36:09.885 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:36:09.885 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:36:09.885 | INFO | ------------------------------------------ | 20260506 22:36:09.885 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:36:09.885 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:36:09.885 | INFO | Total mem usage: 26.5 GiB | 20260506 22:36:09.885 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:36:09.885 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:36:09.885 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:36:09.885 | INFO | High Water Breakdown: | 20260506 22:36:09.885 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:36:09.885 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:36:09.885 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:36:09.885 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:36:09.885 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:36:09.885 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:36:09.885 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:36:09.885 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:36:09.885 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:36:09.885 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:36:09.885 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:36:09.885 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:36:09.885 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:36:09.885 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:36:09.885 | INFO | ------------------------------------------ | 20260506 22:36:09.885 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:36:09.885 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:36:09.885 | INFO | Total mem usage: 25.6 GiB | 20260506 22:36:09.885 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:36:09.885 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:36:09.885 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:36:09.885 | INFO | High Water Breakdown: | 20260506 22:36:09.885 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:36:09.885 | INFO | ------------------------------------------ | 20260506 22:36:09.885 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:36:09.885 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:36:09.885 | INFO | Total mem usage: 15.7 GiB | 20260506 22:36:09.885 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:36:09.885 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:36:09.885 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:36:09.885 | INFO | High Water Breakdown: | 20260506 22:36:09.885 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:36:09.885 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:36:09.885 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:36:09.885 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:36:09.885 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:36:09.885 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:36:09.885 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:36:09.885 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:36:09.885 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:36:09.885 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:36:09.885 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:36:09.885 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:36:09.885 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:36:09.885 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:36:09.885 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:36:09.885 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:36:09.885 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:36:09.885 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:36:09.885 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:36:09.885 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:36:09.885 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:36:09.885 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:36:09.885 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:36:09.885 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:36:09.885 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:36:09.885 | INFO | ------------------------------------------ | 20260506 22:36:09.885 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:36:09.886 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:36:09.886 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:36:09.886 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:36:09.886 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.62e+01, 1.33e+01 / 2.58e+00 | 20260506 22:36:09.886 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.75e-03 / 7.35e-03 | 20260506 22:36:09.886 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.37e+00, 5.41e-01 / 3.62e-01 | 20260506 22:36:09.886 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.44e+00, 1.77e-01 / 1.29e-01 | 20260506 22:36:09.886 | INFO | PerfReport: 16.09%, DME -- 4.00e+01 / 4.50e+01, 4.40e+00 / 3.31e+00 | 20260506 22:36:09.886 | INFO | PerfReport: 57.31%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.36e+00 / 1.98e+00 | 20260506 22:36:09.886 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.19e+00 / 2.79e+00, 3.74e-01 / 2.16e-01 | 20260506 22:36:09.886 | INFO | PerfReport: 3.22%, Metrics -- 1.00e+00 / 1.34e+00, 2.65e-01 / 1.55e-01 | 20260506 22:36:09.886 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.42e-01 / 3.92e-01, 5.36e-02 / 2.42e-02 | 20260506 22:36:09.886 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:36:09.886 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:36:09.886 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:36:09.886 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 22:36:09.886 | INFO | PerfReport: 21.49%, Upload -- 2.08e+01 / 0.00e+00, 6.88e+00 / 0.00e+00 | 20260506 22:36:09.886 | INFO | PerfReport: 21.26%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 22:36:09.886 | INFO | PerfReport: 32.09%, Compute -- 3.11e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 22:36:09.886 | INFO | PerfReport: 1.27%, LockUpload -- 1.23e+00 / 0.00e+00, 4.68e+00 / 0.00e+00 | 20260506 22:36:09.886 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260506 22:36:09.886 | INFO | PerfReport: 22.60%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 22:36:09.886 | INFO | PerfReport: 0.02%, Misc -- 1.62e-02 / 0.00e+00, 3.34e-02 / 0.00e+00 | 20260506 22:36:09.982 | INFO | Starting a round of Baz IO | 20260506 22:36:13.113 | INFO | Populating RTMetrics report | 20260506 22:36:13.114 | INFO | RTMetrics report populated | 20260506 22:36:13.114 | INFO | Opening tmp RTMetrics json file | 20260506 22:36:13.115 | INFO | Serializing RTMetrics to json | 20260506 22:36:13.124 | INFO | Serialization done, swapping tmp file now... | 20260506 22:36:13.135 | INFO | RTMetrics done | 20260506 22:36:13.206 | INFO | Finished a round of Baz IO: Wrote 1.908 GiB in 3.224 seconds | 20260506 22:36:13.207 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.64% / 0.00% / 66.35% | 20260506 22:36:13.207 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.492108% / 0% / 7.94729e-05% | 20260506 22:36:13.207 | INFO | Baz IO overall: | 20260506 22:36:13.207 | INFO | BytesWritten : 2619600061693 | 20260506 22:36:13.207 | INFO | NumEvents : 1263789575370 | 20260506 22:36:15.024 | INFO | Analyzing chunk frames = [4988928,4989440) | 20260506 22:36:15.024 | INFO | MemInfo: { | 20260506 22:36:15.024 | INFO | "gpu" : " 17 GB", | 20260506 22:36:15.024 | INFO | "huge" : "", | 20260506 22:36:15.024 | INFO | "huge_shared" : " 27 GB", | 20260506 22:36:15.024 | INFO | "malloc" : " 14 GB", | 20260506 22:36:15.024 | INFO | "pinned" : " 29 GB" | 20260506 22:36:15.024 | INFO | } | 20260506 22:36:15.024 | INFO | Perf report for the last 5.13993 seconds: | 20260506 22:36:15.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:36:15.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:15.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:15.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:15.024 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 22:36:15.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:15.024 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 22:36:15.024 | INFO | PRE_HQ : 59.6%, 1.8, 56.6ms, 0 | 20260506 22:36:15.024 | INFO | BAZWRITER : 62.3%, 3.4, 109.8ms, 0 | 20260506 22:36:15.024 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:36:15.024 | INFO | Uploaded 36GiB to GPU in 2.00298s (17.9732GiB/s) | 20260506 22:36:15.024 | INFO | Downloaded 38GiB to GPU in 2.1728s (17.489GiB/s) | 20260506 22:36:20.162 | INFO | Analyzing chunk frames = [4989440,4989952) | 20260506 22:36:20.162 | INFO | MemInfo: { | 20260506 22:36:20.162 | INFO | "gpu" : " 17 GB", | 20260506 22:36:20.162 | INFO | "huge" : "", | 20260506 22:36:20.162 | INFO | "huge_shared" : " 27 GB", | 20260506 22:36:20.162 | INFO | "malloc" : " 14 GB", | 20260506 22:36:20.162 | INFO | "pinned" : " 29 GB" | 20260506 22:36:20.162 | INFO | } | 20260506 22:36:20.162 | INFO | Perf report for the last 5.13813 seconds: | 20260506 22:36:20.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:36:20.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:20.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:20.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:20.162 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 22:36:20.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:20.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:20.162 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 22:36:20.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:20.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:20.163 | INFO | Uploaded 36GiB to GPU in 1.98781s (18.1103GiB/s) | 20260506 22:36:20.163 | INFO | Downloaded 34GiB to GPU in 1.92836s (17.6316GiB/s) | 20260506 22:36:25.298 | INFO | Analyzing chunk frames = [4989952,4990464) | 20260506 22:36:25.298 | INFO | MemInfo: { | 20260506 22:36:25.298 | INFO | "gpu" : " 17 GB", | 20260506 22:36:25.298 | INFO | "huge" : "", | 20260506 22:36:25.298 | INFO | "huge_shared" : " 27 GB", | 20260506 22:36:25.298 | INFO | "malloc" : " 14 GB", | 20260506 22:36:25.298 | INFO | "pinned" : " 29 GB" | 20260506 22:36:25.298 | INFO | } | 20260506 22:36:25.298 | INFO | Perf report for the last 5.13609 seconds: | 20260506 22:36:25.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:36:25.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:25.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:25.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:25.298 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 22:36:25.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:25.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:25.298 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 22:36:25.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:25.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:25.299 | INFO | Uploaded 36GiB to GPU in 1.97432s (18.2341GiB/s) | 20260506 22:36:25.299 | INFO | Downloaded 34GiB to GPU in 1.9306s (17.6111GiB/s) | 20260506 22:36:30.434 | INFO | Analyzing chunk frames = [4990464,4990976) | 20260506 22:36:30.434 | INFO | MemInfo: { | 20260506 22:36:30.434 | INFO | "gpu" : " 17 GB", | 20260506 22:36:30.435 | INFO | "huge" : "", | 20260506 22:36:30.435 | INFO | "huge_shared" : " 27 GB", | 20260506 22:36:30.435 | INFO | "malloc" : " 14 GB", | 20260506 22:36:30.435 | INFO | "pinned" : " 29 GB" | 20260506 22:36:30.435 | INFO | } | 20260506 22:36:30.435 | INFO | Perf report for the last 5.13645 seconds: | 20260506 22:36:30.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:36:30.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:30.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:30.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:30.435 | INFO | ANALYSIS : 59.6%, 3.0, 92.3ms, 0 | 20260506 22:36:30.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:30.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:30.435 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:36:30.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:30.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:30.435 | INFO | Uploaded 36GiB to GPU in 1.92839s (18.6685GiB/s) | 20260506 22:36:30.435 | INFO | Downloaded 34GiB to GPU in 1.87621s (18.1217GiB/s) | 20260506 22:36:35.575 | INFO | Analyzing chunk frames = [4990976,4991488) | 20260506 22:36:35.575 | INFO | MemInfo: { | 20260506 22:36:35.575 | INFO | "gpu" : " 17 GB", | 20260506 22:36:35.575 | INFO | "huge" : "", | 20260506 22:36:35.575 | INFO | "huge_shared" : " 27 GB", | 20260506 22:36:35.575 | INFO | "malloc" : " 14 GB", | 20260506 22:36:35.575 | INFO | "pinned" : " 29 GB" | 20260506 22:36:35.575 | INFO | } | 20260506 22:36:35.575 | INFO | Perf report for the last 5.14036 seconds: | 20260506 22:36:35.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:36:35.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:35.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:35.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:35.575 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 22:36:35.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:35.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:35.575 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:36:35.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:35.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:35.575 | INFO | Uploaded 36GiB to GPU in 1.99064s (18.0847GiB/s) | 20260506 22:36:35.576 | INFO | Downloaded 34GiB to GPU in 1.91076s (17.794GiB/s) | 20260506 22:36:40.719 | INFO | Analyzing chunk frames = [4991488,4992000) | 20260506 22:36:40.719 | INFO | MemInfo: { | 20260506 22:36:40.719 | INFO | "gpu" : " 17 GB", | 20260506 22:36:40.719 | INFO | "huge" : "", | 20260506 22:36:40.719 | INFO | "huge_shared" : " 27 GB", | 20260506 22:36:40.719 | INFO | "malloc" : " 14 GB", | 20260506 22:36:40.719 | INFO | "pinned" : " 29 GB" | 20260506 22:36:40.719 | INFO | } | 20260506 22:36:40.720 | INFO | Perf report for the last 5.14459 seconds: | 20260506 22:36:40.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:36:40.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:40.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:40.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:40.720 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260506 22:36:40.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:40.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:40.720 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 22:36:40.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:40.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:40.720 | INFO | Uploaded 36GiB to GPU in 1.97313s (18.2451GiB/s) | 20260506 22:36:40.720 | INFO | Downloaded 34GiB to GPU in 1.93177s (17.6005GiB/s) | 20260506 22:36:45.857 | INFO | Analyzing chunk frames = [4992000,4992512) | 20260506 22:36:45.857 | INFO | MemInfo: { | 20260506 22:36:45.857 | INFO | "gpu" : " 17 GB", | 20260506 22:36:45.857 | INFO | "huge" : "", | 20260506 22:36:45.857 | INFO | "huge_shared" : " 27 GB", | 20260506 22:36:45.857 | INFO | "malloc" : " 14 GB", | 20260506 22:36:45.857 | INFO | "pinned" : " 29 GB" | 20260506 22:36:45.857 | INFO | } | 20260506 22:36:45.857 | INFO | Perf report for the last 5.13714 seconds: | 20260506 22:36:45.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:36:45.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:45.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:45.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:45.857 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:36:45.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:45.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:45.857 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:36:45.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:45.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:45.857 | INFO | Uploaded 36GiB to GPU in 1.97259s (18.2501GiB/s) | 20260506 22:36:45.857 | INFO | Downloaded 34GiB to GPU in 2.00137s (16.9883GiB/s) | 20260506 22:36:50.995 | INFO | Analyzing chunk frames = [4992512,4993024) | 20260506 22:36:50.995 | INFO | MemInfo: { | 20260506 22:36:50.995 | INFO | "gpu" : " 17 GB", | 20260506 22:36:50.995 | INFO | "huge" : "", | 20260506 22:36:50.995 | INFO | "huge_shared" : " 27 GB", | 20260506 22:36:50.995 | INFO | "malloc" : " 14 GB", | 20260506 22:36:50.995 | INFO | "pinned" : " 29 GB" | 20260506 22:36:50.995 | INFO | } | 20260506 22:36:50.995 | INFO | Perf report for the last 5.13838 seconds: | 20260506 22:36:50.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:36:50.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:50.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:50.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:50.995 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:36:50.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:50.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:50.995 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:36:50.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:50.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:50.996 | INFO | Uploaded 36GiB to GPU in 1.98474s (18.1384GiB/s) | 20260506 22:36:50.996 | INFO | Downloaded 34GiB to GPU in 1.98517s (17.127GiB/s) | 20260506 22:36:51.094 | INFO | Starting a round of Baz IO | 20260506 22:36:54.223 | INFO | Populating RTMetrics report | 20260506 22:36:54.224 | INFO | RTMetrics report populated | 20260506 22:36:54.224 | INFO | Opening tmp RTMetrics json file | 20260506 22:36:54.225 | INFO | Serializing RTMetrics to json | 20260506 22:36:54.231 | INFO | Serialization done, swapping tmp file now... | 20260506 22:36:54.245 | INFO | RTMetrics done | 20260506 22:36:54.317 | INFO | Finished a round of Baz IO: Wrote 1.91 GiB in 3.222 seconds | 20260506 22:36:54.317 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.64% / 0.00% / 66.36% | 20260506 22:36:54.317 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.493077% / 0% / 5.5631e-05% | 20260506 22:36:54.317 | INFO | Baz IO overall: | 20260506 22:36:54.317 | INFO | BytesWritten : 2621651416350 | 20260506 22:36:54.317 | INFO | NumEvents : 1264769855859 | 20260506 22:36:56.135 | INFO | Analyzing chunk frames = [4993024,4993536) | 20260506 22:36:56.135 | INFO | MemInfo: { | 20260506 22:36:56.135 | INFO | "gpu" : " 17 GB", | 20260506 22:36:56.135 | INFO | "huge" : "", | 20260506 22:36:56.135 | INFO | "huge_shared" : " 27 GB", | 20260506 22:36:56.135 | INFO | "malloc" : " 14 GB", | 20260506 22:36:56.135 | INFO | "pinned" : " 29 GB" | 20260506 22:36:56.135 | INFO | } | 20260506 22:36:56.135 | INFO | Perf report for the last 5.13946 seconds: | 20260506 22:36:56.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:36:56.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:56.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:56.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:56.135 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260506 22:36:56.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:36:56.135 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 22:36:56.135 | INFO | PRE_HQ : 59.8%, 1.8, 56.8ms, 0 | 20260506 22:36:56.135 | INFO | BAZWRITER : 62.3%, 3.5, 111.3ms, 0 | 20260506 22:36:56.135 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:36:56.135 | INFO | Uploaded 36GiB to GPU in 2.06054s (17.4711GiB/s) | 20260506 22:36:56.135 | INFO | Downloaded 38GiB to GPU in 2.17598s (17.4634GiB/s) | 20260506 22:37:01.275 | INFO | Analyzing chunk frames = [4993536,4994048) | 20260506 22:37:01.275 | INFO | MemInfo: { | 20260506 22:37:01.275 | INFO | "gpu" : " 17 GB", | 20260506 22:37:01.276 | INFO | "huge" : "", | 20260506 22:37:01.276 | INFO | "huge_shared" : " 27 GB", | 20260506 22:37:01.276 | INFO | "malloc" : " 14 GB", | 20260506 22:37:01.276 | INFO | "pinned" : " 29 GB" | 20260506 22:37:01.276 | INFO | } | 20260506 22:37:01.276 | INFO | Perf report for the last 5.14086 seconds: | 20260506 22:37:01.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:37:01.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:01.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:01.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:01.276 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:37:01.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:01.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:01.276 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 22:37:01.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:01.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:01.276 | INFO | Uploaded 36GiB to GPU in 1.99933s (18.006GiB/s) | 20260506 22:37:01.276 | INFO | Downloaded 34GiB to GPU in 1.92282s (17.6824GiB/s) | 20260506 22:37:06.415 | INFO | Analyzing chunk frames = [4994048,4994560) | 20260506 22:37:06.415 | INFO | MemInfo: { | 20260506 22:37:06.415 | INFO | "gpu" : " 17 GB", | 20260506 22:37:06.415 | INFO | "huge" : "", | 20260506 22:37:06.415 | INFO | "huge_shared" : " 27 GB", | 20260506 22:37:06.415 | INFO | "malloc" : " 14 GB", | 20260506 22:37:06.415 | INFO | "pinned" : " 29 GB" | 20260506 22:37:06.415 | INFO | } | 20260506 22:37:06.415 | INFO | Perf report for the last 5.1397 seconds: | 20260506 22:37:06.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:37:06.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:06.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:06.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:06.416 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260506 22:37:06.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:06.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:06.416 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:37:06.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:06.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:06.416 | INFO | Uploaded 36GiB to GPU in 1.99016s (18.089GiB/s) | 20260506 22:37:06.416 | INFO | Downloaded 34GiB to GPU in 1.91776s (17.729GiB/s) | 20260506 22:37:11.554 | INFO | Analyzing chunk frames = [4994560,4995072) | 20260506 22:37:11.554 | INFO | MemInfo: { | 20260506 22:37:11.554 | INFO | "gpu" : " 17 GB", | 20260506 22:37:11.554 | INFO | "huge" : "", | 20260506 22:37:11.554 | INFO | "huge_shared" : " 27 GB", | 20260506 22:37:11.554 | INFO | "malloc" : " 14 GB", | 20260506 22:37:11.554 | INFO | "pinned" : " 29 GB" | 20260506 22:37:11.554 | INFO | } | 20260506 22:37:11.554 | INFO | Perf report for the last 5.13877 seconds: | 20260506 22:37:11.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:37:11.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:11.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:11.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:11.554 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 22:37:11.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:11.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:11.554 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:37:11.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:11.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:11.554 | INFO | Uploaded 36GiB to GPU in 1.96145s (18.3537GiB/s) | 20260506 22:37:11.554 | INFO | Downloaded 34GiB to GPU in 1.87316s (18.1512GiB/s) | 20260506 22:37:16.699 | INFO | Analyzing chunk frames = [4995072,4995584) | 20260506 22:37:16.699 | INFO | MemInfo: { | 20260506 22:37:16.699 | INFO | "gpu" : " 17 GB", | 20260506 22:37:16.699 | INFO | "huge" : "", | 20260506 22:37:16.699 | INFO | "huge_shared" : " 27 GB", | 20260506 22:37:16.699 | INFO | "malloc" : " 14 GB", | 20260506 22:37:16.699 | INFO | "pinned" : " 29 GB" | 20260506 22:37:16.699 | INFO | } | 20260506 22:37:16.699 | INFO | Perf report for the last 5.14524 seconds: | 20260506 22:37:16.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:37:16.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:16.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:16.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:16.699 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260506 22:37:16.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:16.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:16.699 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:37:16.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:16.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:16.700 | INFO | Uploaded 36GiB to GPU in 2.02042s (17.8181GiB/s) | 20260506 22:37:16.700 | INFO | Downloaded 34GiB to GPU in 1.91748s (17.7316GiB/s) | 20260506 22:37:21.835 | INFO | Analyzing chunk frames = [4995584,4996096) | 20260506 22:37:21.835 | INFO | MemInfo: { | 20260506 22:37:21.835 | INFO | "gpu" : " 17 GB", | 20260506 22:37:21.835 | INFO | "huge" : "", | 20260506 22:37:21.835 | INFO | "huge_shared" : " 27 GB", | 20260506 22:37:21.835 | INFO | "malloc" : " 14 GB", | 20260506 22:37:21.836 | INFO | "pinned" : " 29 GB" | 20260506 22:37:21.836 | INFO | } | 20260506 22:37:21.836 | INFO | Perf report for the last 5.1365 seconds: | 20260506 22:37:21.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:37:21.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:21.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:21.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:21.836 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:37:21.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:21.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:21.836 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 22:37:21.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:21.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:21.836 | INFO | Uploaded 36GiB to GPU in 2.0334s (17.7043GiB/s) | 20260506 22:37:21.836 | INFO | Downloaded 34GiB to GPU in 1.94085s (17.5181GiB/s) | 20260506 22:37:26.974 | INFO | Analyzing chunk frames = [4996096,4996608) | 20260506 22:37:26.974 | INFO | MemInfo: { | 20260506 22:37:26.974 | INFO | "gpu" : " 17 GB", | 20260506 22:37:26.974 | INFO | "huge" : "", | 20260506 22:37:26.974 | INFO | "huge_shared" : " 27 GB", | 20260506 22:37:26.974 | INFO | "malloc" : " 14 GB", | 20260506 22:37:26.974 | INFO | "pinned" : " 29 GB" | 20260506 22:37:26.975 | INFO | } | 20260506 22:37:26.975 | INFO | Perf report for the last 5.13883 seconds: | 20260506 22:37:26.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:37:26.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:26.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:26.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:26.975 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:37:26.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:26.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:26.975 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 22:37:26.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:26.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:26.975 | INFO | Uploaded 36GiB to GPU in 1.96815s (18.2913GiB/s) | 20260506 22:37:26.975 | INFO | Downloaded 34GiB to GPU in 1.99995s (17.0004GiB/s) | 20260506 22:37:32.111 | INFO | Analyzing chunk frames = [4996608,4997120) | 20260506 22:37:32.111 | INFO | MemInfo: { | 20260506 22:37:32.111 | INFO | "gpu" : " 17 GB", | 20260506 22:37:32.111 | INFO | "huge" : "", | 20260506 22:37:32.112 | INFO | "huge_shared" : " 27 GB", | 20260506 22:37:32.112 | INFO | "malloc" : " 14 GB", | 20260506 22:37:32.112 | INFO | "pinned" : " 29 GB" | 20260506 22:37:32.112 | INFO | } | 20260506 22:37:32.112 | INFO | Perf report for the last 5.13696 seconds: | 20260506 22:37:32.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:37:32.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:32.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:32.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:32.112 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:37:32.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:32.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:32.112 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 22:37:32.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:32.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:32.112 | INFO | Uploaded 36GiB to GPU in 1.97269s (18.2492GiB/s) | 20260506 22:37:32.112 | INFO | Downloaded 34GiB to GPU in 1.99649s (17.0299GiB/s) | 20260506 22:37:32.112 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:37:32.112 | INFO | Total mem usage: 18 GiB | 20260506 22:37:32.112 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:37:32.112 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:37:32.112 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:37:32.112 | INFO | High Water Breakdown: | 20260506 22:37:32.112 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:37:32.112 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:37:32.112 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:37:32.112 | INFO | ------------------------------------------ | 20260506 22:37:32.112 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:37:32.112 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:37:32.112 | INFO | Total mem usage: 26.5 GiB | 20260506 22:37:32.113 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:37:32.113 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:37:32.113 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:37:32.113 | INFO | High Water Breakdown: | 20260506 22:37:32.113 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:37:32.113 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:37:32.113 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:37:32.113 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:37:32.113 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:37:32.113 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:37:32.113 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:37:32.113 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:37:32.113 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:37:32.113 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:37:32.113 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:37:32.113 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:37:32.113 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:37:32.113 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:37:32.113 | INFO | ------------------------------------------ | 20260506 22:37:32.113 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:37:32.113 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:37:32.113 | INFO | Total mem usage: 25.6 GiB | 20260506 22:37:32.113 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:37:32.113 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:37:32.113 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:37:32.113 | INFO | High Water Breakdown: | 20260506 22:37:32.113 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:37:32.113 | INFO | ------------------------------------------ | 20260506 22:37:32.113 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:37:32.113 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:37:32.113 | INFO | Total mem usage: 15.7 GiB | 20260506 22:37:32.113 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:37:32.113 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:37:32.113 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:37:32.113 | INFO | High Water Breakdown: | 20260506 22:37:32.113 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:37:32.113 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:37:32.113 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:37:32.113 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:37:32.113 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:37:32.113 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:37:32.113 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:37:32.113 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:37:32.113 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:37:32.113 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:37:32.113 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:37:32.113 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:37:32.113 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:37:32.113 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:37:32.113 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:37:32.113 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:37:32.113 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:37:32.113 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:37:32.113 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:37:32.113 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:37:32.113 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:37:32.113 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:37:32.113 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:37:32.113 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:37:32.113 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:37:32.113 | INFO | ------------------------------------------ | 20260506 22:37:32.113 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:37:32.114 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:37:32.114 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:37:32.114 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:37:32.114 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.58e+01, 1.34e+01 / 2.46e+00 | 20260506 22:37:32.114 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.00e-01, 9.57e-03 / 5.17e-03 | 20260506 22:37:32.114 | INFO | PerfReport: 11.65%, Baseline -- 3.62e+00 / 4.38e+00, 5.51e-01 / 3.47e-01 | 20260506 22:37:32.114 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.44e+00, 1.79e-01 / 1.33e-01 | 20260506 22:37:32.114 | INFO | PerfReport: 16.05%, DME -- 3.98e+01 / 4.45e+01, 4.11e+00 / 2.64e+00 | 20260506 22:37:32.114 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.39e+00 / 1.52e+00 | 20260506 22:37:32.114 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.19e+00 / 2.77e+00, 3.72e-01 / 1.94e-01 | 20260506 22:37:32.114 | INFO | PerfReport: 3.22%, Metrics -- 1.00e+00 / 1.33e+00, 2.66e-01 / 1.46e-01 | 20260506 22:37:32.114 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.41e-01 / 3.86e-01, 5.36e-02 / 1.76e-02 | 20260506 22:37:32.114 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:37:32.114 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:37:32.114 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:37:32.114 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 22:37:32.114 | INFO | PerfReport: 21.44%, Upload -- 2.07e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260506 22:37:32.114 | INFO | PerfReport: 21.21%, Download -- 2.05e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 22:37:32.114 | INFO | PerfReport: 32.12%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 22:37:32.114 | INFO | PerfReport: 1.22%, LockUpload -- 1.18e+00 / 0.00e+00, 4.64e+00 / 0.00e+00 | 20260506 22:37:32.114 | INFO | PerfReport: 1.25%, LockDownload -- 1.20e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260506 22:37:32.114 | INFO | PerfReport: 22.74%, LockCompute -- 2.20e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 22:37:32.114 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260506 22:37:32.210 | INFO | Starting a round of Baz IO | 20260506 22:37:35.340 | INFO | Populating RTMetrics report | 20260506 22:37:35.340 | INFO | RTMetrics report populated | 20260506 22:37:35.340 | INFO | Opening tmp RTMetrics json file | 20260506 22:37:35.341 | INFO | Serializing RTMetrics to json | 20260506 22:37:35.349 | INFO | Serialization done, swapping tmp file now... | 20260506 22:37:35.362 | INFO | RTMetrics done | 20260506 22:37:35.433 | INFO | Finished a round of Baz IO: Wrote 1.908 GiB in 3.222 seconds | 20260506 22:37:35.433 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.64% / 0.00% / 66.36% | 20260506 22:37:35.433 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.489958% / 3.97364e-06% / 5.96046e-05% | 20260506 22:37:35.433 | INFO | Baz IO overall: | 20260506 22:37:35.433 | INFO | BytesWritten : 2623700159572 | 20260506 22:37:35.433 | INFO | NumEvents : 1265748229059 | 20260506 22:37:37.249 | INFO | Analyzing chunk frames = [4997120,4997632) | 20260506 22:37:37.249 | INFO | MemInfo: { | 20260506 22:37:37.249 | INFO | "gpu" : " 17 GB", | 20260506 22:37:37.249 | INFO | "huge" : "", | 20260506 22:37:37.249 | INFO | "huge_shared" : " 27 GB", | 20260506 22:37:37.249 | INFO | "malloc" : " 14 GB", | 20260506 22:37:37.249 | INFO | "pinned" : " 29 GB" | 20260506 22:37:37.249 | INFO | } | 20260506 22:37:37.249 | INFO | Perf report for the last 5.13732 seconds: | 20260506 22:37:37.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:37:37.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:37.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:37.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:37.249 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260506 22:37:37.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:37.249 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 22:37:37.249 | INFO | PRE_HQ : 59.6%, 1.8, 56.8ms, 0 | 20260506 22:37:37.249 | INFO | BAZWRITER : 62.3%, 3.5, 110.8ms, 0 | 20260506 22:37:37.249 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:37:37.249 | INFO | Uploaded 36GiB to GPU in 2.00575s (17.9484GiB/s) | 20260506 22:37:37.249 | INFO | Downloaded 38GiB to GPU in 2.17445s (17.4757GiB/s) | 20260506 22:37:42.389 | INFO | Analyzing chunk frames = [4997632,4998144) | 20260506 22:37:42.389 | INFO | MemInfo: { | 20260506 22:37:42.389 | INFO | "gpu" : " 17 GB", | 20260506 22:37:42.389 | INFO | "huge" : "", | 20260506 22:37:42.389 | INFO | "huge_shared" : " 27 GB", | 20260506 22:37:42.389 | INFO | "malloc" : " 14 GB", | 20260506 22:37:42.389 | INFO | "pinned" : " 29 GB" | 20260506 22:37:42.389 | INFO | } | 20260506 22:37:42.389 | INFO | Perf report for the last 5.14003 seconds: | 20260506 22:37:42.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:37:42.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:42.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:42.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:42.389 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:37:42.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:42.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:42.389 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 22:37:42.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:42.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:42.389 | INFO | Uploaded 36GiB to GPU in 2.01492s (17.8667GiB/s) | 20260506 22:37:42.389 | INFO | Downloaded 34GiB to GPU in 1.93406s (17.5796GiB/s) | 20260506 22:37:47.529 | INFO | Analyzing chunk frames = [4998144,4998656) | 20260506 22:37:47.529 | INFO | MemInfo: { | 20260506 22:37:47.529 | INFO | "gpu" : " 17 GB", | 20260506 22:37:47.529 | INFO | "huge" : "", | 20260506 22:37:47.529 | INFO | "huge_shared" : " 27 GB", | 20260506 22:37:47.529 | INFO | "malloc" : " 14 GB", | 20260506 22:37:47.530 | INFO | "pinned" : " 29 GB" | 20260506 22:37:47.530 | INFO | } | 20260506 22:37:47.530 | INFO | Perf report for the last 5.14055 seconds: | 20260506 22:37:47.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:37:47.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:47.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:47.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:47.530 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 22:37:47.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:47.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:47.530 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:37:47.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:47.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:47.530 | INFO | Uploaded 36GiB to GPU in 1.97411s (18.236GiB/s) | 20260506 22:37:47.530 | INFO | Downloaded 34GiB to GPU in 1.91986s (17.7096GiB/s) | 20260506 22:37:52.670 | INFO | Analyzing chunk frames = [4998656,4999168) | 20260506 22:37:52.670 | INFO | MemInfo: { | 20260506 22:37:52.670 | INFO | "gpu" : " 17 GB", | 20260506 22:37:52.670 | INFO | "huge" : "", | 20260506 22:37:52.670 | INFO | "huge_shared" : " 27 GB", | 20260506 22:37:52.670 | INFO | "malloc" : " 14 GB", | 20260506 22:37:52.670 | INFO | "pinned" : " 29 GB" | 20260506 22:37:52.670 | INFO | } | 20260506 22:37:52.670 | INFO | Perf report for the last 5.1407 seconds: | 20260506 22:37:52.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:37:52.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:52.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:52.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:52.670 | INFO | ANALYSIS : 59.6%, 3.0, 92.3ms, 0 | 20260506 22:37:52.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:52.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:52.670 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:37:52.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:52.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:52.671 | INFO | Uploaded 36GiB to GPU in 1.95439s (18.4201GiB/s) | 20260506 22:37:52.671 | INFO | Downloaded 34GiB to GPU in 1.88087s (18.0768GiB/s) | 20260506 22:37:57.810 | INFO | Analyzing chunk frames = [4999168,4999680) | 20260506 22:37:57.810 | INFO | MemInfo: { | 20260506 22:37:57.810 | INFO | "gpu" : " 17 GB", | 20260506 22:37:57.810 | INFO | "huge" : "", | 20260506 22:37:57.810 | INFO | "huge_shared" : " 27 GB", | 20260506 22:37:57.810 | INFO | "malloc" : " 14 GB", | 20260506 22:37:57.810 | INFO | "pinned" : " 29 GB" | 20260506 22:37:57.810 | INFO | } | 20260506 22:37:57.810 | INFO | Perf report for the last 5.1397 seconds: | 20260506 22:37:57.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:37:57.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:57.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:57.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:57.810 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 22:37:57.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:57.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:57.810 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:37:57.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:57.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:37:57.810 | INFO | Uploaded 36GiB to GPU in 1.98597s (18.1272GiB/s) | 20260506 22:37:57.811 | INFO | Downloaded 34GiB to GPU in 1.9214s (17.6955GiB/s) | 20260506 22:38:02.952 | INFO | Analyzing chunk frames = [4999680,5000192) | 20260506 22:38:02.952 | INFO | MemInfo: { | 20260506 22:38:02.952 | INFO | "gpu" : " 17 GB", | 20260506 22:38:02.952 | INFO | "huge" : "", | 20260506 22:38:02.952 | INFO | "huge_shared" : " 27 GB", | 20260506 22:38:02.952 | INFO | "malloc" : " 14 GB", | 20260506 22:38:02.952 | INFO | "pinned" : " 29 GB" | 20260506 22:38:02.952 | INFO | } | 20260506 22:38:02.952 | INFO | Perf report for the last 5.14231 seconds: | 20260506 22:38:02.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:38:02.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:02.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:02.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:02.952 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 22:38:02.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:02.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:02.953 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 22:38:02.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:02.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:02.953 | INFO | Uploaded 36GiB to GPU in 1.9932s (18.0614GiB/s) | 20260506 22:38:02.953 | INFO | Downloaded 34GiB to GPU in 1.95999s (17.347GiB/s) | 20260506 22:38:08.089 | INFO | Analyzing chunk frames = [5000192,5000704) | 20260506 22:38:08.089 | INFO | MemInfo: { | 20260506 22:38:08.089 | INFO | "gpu" : " 17 GB", | 20260506 22:38:08.089 | INFO | "huge" : "", | 20260506 22:38:08.089 | INFO | "huge_shared" : " 27 GB", | 20260506 22:38:08.089 | INFO | "malloc" : " 14 GB", | 20260506 22:38:08.089 | INFO | "pinned" : " 29 GB" | 20260506 22:38:08.089 | INFO | } | 20260506 22:38:08.090 | INFO | Perf report for the last 5.13714 seconds: | 20260506 22:38:08.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:38:08.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:08.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:08.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:08.090 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 22:38:08.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:08.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:08.090 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:38:08.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:08.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:08.090 | INFO | Uploaded 36GiB to GPU in 1.96605s (18.3109GiB/s) | 20260506 22:38:08.090 | INFO | Downloaded 34GiB to GPU in 1.99648s (17.0299GiB/s) | 20260506 22:38:13.224 | INFO | Analyzing chunk frames = [5000704,5001216) | 20260506 22:38:13.224 | INFO | MemInfo: { | 20260506 22:38:13.224 | INFO | "gpu" : " 17 GB", | 20260506 22:38:13.224 | INFO | "huge" : "", | 20260506 22:38:13.224 | INFO | "huge_shared" : " 27 GB", | 20260506 22:38:13.224 | INFO | "malloc" : " 14 GB", | 20260506 22:38:13.224 | INFO | "pinned" : " 29 GB" | 20260506 22:38:13.224 | INFO | } | 20260506 22:38:13.224 | INFO | Perf report for the last 5.13427 seconds: | 20260506 22:38:13.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:38:13.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:13.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:13.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:13.224 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 22:38:13.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:13.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:13.224 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 22:38:13.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:13.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:13.224 | INFO | Uploaded 36GiB to GPU in 1.9985s (18.0135GiB/s) | 20260506 22:38:13.224 | INFO | Downloaded 34GiB to GPU in 1.99945s (17.0047GiB/s) | 20260506 22:38:13.322 | INFO | Starting a round of Baz IO | 20260506 22:38:16.453 | INFO | Populating RTMetrics report | 20260506 22:38:16.453 | INFO | RTMetrics report populated | 20260506 22:38:16.453 | INFO | Opening tmp RTMetrics json file | 20260506 22:38:16.454 | INFO | Serializing RTMetrics to json | 20260506 22:38:16.455 | INFO | Serialization done, swapping tmp file now... | 20260506 22:38:16.455 | INFO | RTMetrics done | 20260506 22:38:16.546 | INFO | Finished a round of Baz IO: Wrote 1.906 GiB in 3.224 seconds | 20260506 22:38:16.546 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.64% / 0.00% / 66.36% | 20260506 22:38:16.546 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.487725% / 3.97364e-06% / 7.54992e-05% | 20260506 22:38:16.546 | INFO | Baz IO overall: | 20260506 22:38:16.546 | INFO | BytesWritten : 2625747101897 | 20260506 22:38:16.546 | INFO | NumEvents : 1266725398415 | 20260506 22:38:18.362 | INFO | Analyzing chunk frames = [5001216,5001728) | 20260506 22:38:18.362 | INFO | MemInfo: { | 20260506 22:38:18.362 | INFO | "gpu" : " 17 GB", | 20260506 22:38:18.362 | INFO | "huge" : "", | 20260506 22:38:18.362 | INFO | "huge_shared" : " 27 GB", | 20260506 22:38:18.362 | INFO | "malloc" : " 14 GB", | 20260506 22:38:18.362 | INFO | "pinned" : " 29 GB" | 20260506 22:38:18.362 | INFO | } | 20260506 22:38:18.362 | INFO | Perf report for the last 5.13825 seconds: | 20260506 22:38:18.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:38:18.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:18.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:18.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:18.362 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 22:38:18.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:18.362 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 22:38:18.362 | INFO | PRE_HQ : 59.8%, 1.8, 57.1ms, 0 | 20260506 22:38:18.362 | INFO | BAZWRITER : 62.2%, 3.5, 111.0ms, 0 | 20260506 22:38:18.362 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:38:18.362 | INFO | Uploaded 36GiB to GPU in 2.01414s (17.8736GiB/s) | 20260506 22:38:18.362 | INFO | Downloaded 38GiB to GPU in 2.17767s (17.4498GiB/s) | 20260506 22:38:23.506 | INFO | Analyzing chunk frames = [5001728,5002240) | 20260506 22:38:23.506 | INFO | MemInfo: { | 20260506 22:38:23.506 | INFO | "gpu" : " 17 GB", | 20260506 22:38:23.506 | INFO | "huge" : "", | 20260506 22:38:23.506 | INFO | "huge_shared" : " 27 GB", | 20260506 22:38:23.506 | INFO | "malloc" : " 14 GB", | 20260506 22:38:23.506 | INFO | "pinned" : " 29 GB" | 20260506 22:38:23.506 | INFO | } | 20260506 22:38:23.507 | INFO | Perf report for the last 5.14446 seconds: | 20260506 22:38:23.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:38:23.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:23.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:23.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:23.507 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 22:38:23.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:23.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:23.507 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:38:23.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:23.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:23.507 | INFO | Uploaded 36GiB to GPU in 1.98391s (18.146GiB/s) | 20260506 22:38:23.507 | INFO | Downloaded 34GiB to GPU in 1.9276s (17.6385GiB/s) | 20260506 22:38:28.647 | INFO | Analyzing chunk frames = [5002240,5002752) | 20260506 22:38:28.647 | INFO | MemInfo: { | 20260506 22:38:28.647 | INFO | "gpu" : " 17 GB", | 20260506 22:38:28.647 | INFO | "huge" : "", | 20260506 22:38:28.648 | INFO | "huge_shared" : " 27 GB", | 20260506 22:38:28.648 | INFO | "malloc" : " 14 GB", | 20260506 22:38:28.648 | INFO | "pinned" : " 29 GB" | 20260506 22:38:28.648 | INFO | } | 20260506 22:38:28.648 | INFO | Perf report for the last 5.14119 seconds: | 20260506 22:38:28.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:38:28.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:28.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:28.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:28.648 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 22:38:28.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:28.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:28.648 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:38:28.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:28.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:28.648 | INFO | Uploaded 36GiB to GPU in 1.96385s (18.3314GiB/s) | 20260506 22:38:28.648 | INFO | Downloaded 34GiB to GPU in 1.92017s (17.7068GiB/s) | 20260506 22:38:33.787 | INFO | Analyzing chunk frames = [5002752,5003264) | 20260506 22:38:33.787 | INFO | MemInfo: { | 20260506 22:38:33.787 | INFO | "gpu" : " 17 GB", | 20260506 22:38:33.787 | INFO | "huge" : "", | 20260506 22:38:33.788 | INFO | "huge_shared" : " 27 GB", | 20260506 22:38:33.788 | INFO | "malloc" : " 14 GB", | 20260506 22:38:33.788 | INFO | "pinned" : " 29 GB" | 20260506 22:38:33.788 | INFO | } | 20260506 22:38:33.788 | INFO | Perf report for the last 5.13993 seconds: | 20260506 22:38:33.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:38:33.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:33.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:33.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:33.788 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 22:38:33.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:33.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:33.788 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:38:33.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:33.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:33.788 | INFO | Uploaded 36GiB to GPU in 1.94498s (18.5092GiB/s) | 20260506 22:38:33.788 | INFO | Downloaded 34GiB to GPU in 1.88094s (18.0761GiB/s) | 20260506 22:38:38.922 | INFO | Analyzing chunk frames = [5003264,5003776) | 20260506 22:38:38.922 | INFO | MemInfo: { | 20260506 22:38:38.922 | INFO | "gpu" : " 17 GB", | 20260506 22:38:38.922 | INFO | "huge" : "", | 20260506 22:38:38.922 | INFO | "huge_shared" : " 27 GB", | 20260506 22:38:38.922 | INFO | "malloc" : " 14 GB", | 20260506 22:38:38.922 | INFO | "pinned" : " 29 GB" | 20260506 22:38:38.922 | INFO | } | 20260506 22:38:38.922 | INFO | Perf report for the last 5.1346 seconds: | 20260506 22:38:38.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:38:38.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:38.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:38.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:38.922 | INFO | ANALYSIS : 60.3%, 3.0, 93.7ms, 0 | 20260506 22:38:38.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:38.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:38.922 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:38:38.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:38.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:38.923 | INFO | Uploaded 36GiB to GPU in 2.00068s (17.9939GiB/s) | 20260506 22:38:38.923 | INFO | Downloaded 34GiB to GPU in 1.93488s (17.5721GiB/s) | 20260506 22:38:44.061 | INFO | Analyzing chunk frames = [5003776,5004288) | 20260506 22:38:44.061 | INFO | MemInfo: { | 20260506 22:38:44.061 | INFO | "gpu" : " 17 GB", | 20260506 22:38:44.061 | INFO | "huge" : "", | 20260506 22:38:44.062 | INFO | "huge_shared" : " 27 GB", | 20260506 22:38:44.062 | INFO | "malloc" : " 14 GB", | 20260506 22:38:44.062 | INFO | "pinned" : " 29 GB" | 20260506 22:38:44.062 | INFO | } | 20260506 22:38:44.062 | INFO | Perf report for the last 5.13946 seconds: | 20260506 22:38:44.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:38:44.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:44.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:44.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:44.062 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 22:38:44.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:44.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:44.062 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 22:38:44.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:44.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:44.062 | INFO | Uploaded 36GiB to GPU in 1.99406s (18.0536GiB/s) | 20260506 22:38:44.062 | INFO | Downloaded 34GiB to GPU in 1.94602s (17.4716GiB/s) | 20260506 22:38:49.202 | INFO | Analyzing chunk frames = [5004288,5004800) | 20260506 22:38:49.202 | INFO | MemInfo: { | 20260506 22:38:49.202 | INFO | "gpu" : " 17 GB", | 20260506 22:38:49.203 | INFO | "huge" : "", | 20260506 22:38:49.203 | INFO | "huge_shared" : " 27 GB", | 20260506 22:38:49.203 | INFO | "malloc" : " 14 GB", | 20260506 22:38:49.203 | INFO | "pinned" : " 29 GB" | 20260506 22:38:49.203 | INFO | } | 20260506 22:38:49.203 | INFO | Perf report for the last 5.14095 seconds: | 20260506 22:38:49.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:38:49.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:49.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:49.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:49.203 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 22:38:49.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:49.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:49.203 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 22:38:49.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:49.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:49.203 | INFO | Uploaded 36GiB to GPU in 1.96935s (18.2801GiB/s) | 20260506 22:38:49.203 | INFO | Downloaded 34GiB to GPU in 1.97581s (17.2081GiB/s) | 20260506 22:38:54.337 | INFO | Analyzing chunk frames = [5004800,5005312) | 20260506 22:38:54.337 | INFO | MemInfo: { | 20260506 22:38:54.337 | INFO | "gpu" : " 17 GB", | 20260506 22:38:54.337 | INFO | "huge" : "", | 20260506 22:38:54.337 | INFO | "huge_shared" : " 27 GB", | 20260506 22:38:54.337 | INFO | "malloc" : " 14 GB", | 20260506 22:38:54.337 | INFO | "pinned" : " 29 GB" | 20260506 22:38:54.337 | INFO | } | 20260506 22:38:54.337 | INFO | Perf report for the last 5.13435 seconds: | 20260506 22:38:54.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:38:54.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:54.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:54.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:54.337 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 22:38:54.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:54.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:54.337 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 22:38:54.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:54.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:54.337 | INFO | Uploaded 36GiB to GPU in 2.05493s (17.5189GiB/s) | 20260506 22:38:54.337 | INFO | Downloaded 34GiB to GPU in 2.00521s (16.9558GiB/s) | 20260506 22:38:54.338 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:38:54.338 | INFO | Total mem usage: 18 GiB | 20260506 22:38:54.338 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:38:54.338 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 22:38:54.338 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:38:54.338 | INFO | High Water Breakdown: | 20260506 22:38:54.338 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:38:54.338 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:38:54.338 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:38:54.338 | INFO | ------------------------------------------ | 20260506 22:38:54.338 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:38:54.338 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:38:54.338 | INFO | Total mem usage: 26.5 GiB | 20260506 22:38:54.338 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:38:54.338 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:38:54.338 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:38:54.338 | INFO | High Water Breakdown: | 20260506 22:38:54.338 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:38:54.338 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:38:54.338 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:38:54.338 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:38:54.338 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:38:54.338 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:38:54.338 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:38:54.338 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:38:54.338 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:38:54.338 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:38:54.338 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:38:54.338 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:38:54.338 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:38:54.338 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:38:54.338 | INFO | ------------------------------------------ | 20260506 22:38:54.338 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:38:54.338 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:38:54.338 | INFO | Total mem usage: 25.6 GiB | 20260506 22:38:54.338 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:38:54.338 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:38:54.338 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:38:54.338 | INFO | High Water Breakdown: | 20260506 22:38:54.338 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:38:54.338 | INFO | ------------------------------------------ | 20260506 22:38:54.338 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:38:54.338 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:38:54.338 | INFO | Total mem usage: 15.7 GiB | 20260506 22:38:54.338 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:38:54.338 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:38:54.338 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:38:54.338 | INFO | High Water Breakdown: | 20260506 22:38:54.338 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:38:54.338 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:38:54.338 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:38:54.338 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:38:54.338 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:38:54.338 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:38:54.338 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:38:54.338 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:38:54.338 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:38:54.338 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:38:54.338 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:38:54.338 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:38:54.338 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:38:54.338 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:38:54.338 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:38:54.338 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:38:54.338 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:38:54.338 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:38:54.338 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:38:54.338 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:38:54.338 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:38:54.338 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:38:54.338 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:38:54.338 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:38:54.338 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:38:54.338 | INFO | ------------------------------------------ | 20260506 22:38:54.338 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:38:54.338 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:38:54.338 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:38:54.338 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:38:54.338 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.59e+01, 1.33e+01 / 2.88e+00 | 20260506 22:38:54.338 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 1.02e-02 / 6.76e-03 | 20260506 22:38:54.338 | INFO | PerfReport: 11.63%, Baseline -- 3.61e+00 / 4.33e+00, 5.40e-01 / 4.01e-01 | 20260506 22:38:54.338 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.42e+00, 1.75e-01 / 1.36e-01 | 20260506 22:38:54.338 | INFO | PerfReport: 16.09%, DME -- 3.99e+01 / 4.48e+01, 4.26e+00 / 3.44e+00 | 20260506 22:38:54.338 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.36e+00 / 1.74e+00 | 20260506 22:38:54.338 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.19e+00 / 2.74e+00, 3.77e-01 / 2.13e-01 | 20260506 22:38:54.338 | INFO | PerfReport: 3.22%, Metrics -- 9.98e-01 / 1.33e+00, 2.63e-01 / 1.62e-01 | 20260506 22:38:54.338 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.41e-01 / 3.89e-01, 5.29e-02 / 2.40e-02 | 20260506 22:38:54.338 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:38:54.338 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:38:54.338 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:38:54.338 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 22:38:54.338 | INFO | PerfReport: 21.43%, Upload -- 2.07e+01 / 0.00e+00, 6.90e+00 / 0.00e+00 | 20260506 22:38:54.339 | INFO | PerfReport: 21.25%, Download -- 2.05e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 22:38:54.339 | INFO | PerfReport: 32.09%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 22:38:54.339 | INFO | PerfReport: 1.20%, LockUpload -- 1.16e+00 / 0.00e+00, 4.61e+00 / 0.00e+00 | 20260506 22:38:54.339 | INFO | PerfReport: 1.33%, LockDownload -- 1.28e+00 / 0.00e+00, 3.25e+00 / 0.00e+00 | 20260506 22:38:54.339 | INFO | PerfReport: 22.69%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 22:38:54.339 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260506 22:38:54.452 | INFO | Starting a round of Baz IO | 20260506 22:38:57.580 | INFO | Populating RTMetrics report | 20260506 22:38:57.580 | INFO | RTMetrics report populated | 20260506 22:38:57.580 | INFO | Opening tmp RTMetrics json file | 20260506 22:38:57.597 | INFO | Serializing RTMetrics to json | 20260506 22:38:57.598 | INFO | Serialization done, swapping tmp file now... | 20260506 22:38:57.599 | INFO | RTMetrics done | 20260506 22:38:57.672 | INFO | Finished a round of Baz IO: Wrote 1.907 GiB in 3.22 seconds | 20260506 22:38:57.672 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.63% / 0.00% / 66.36% | 20260506 22:38:57.672 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.486453% / 7.94729e-06% / 5.16574e-05% | 20260506 22:38:57.672 | INFO | Baz IO overall: | 20260506 22:38:57.672 | INFO | BytesWritten : 2627794874001 | 20260506 22:38:57.672 | INFO | NumEvents : 1267703189914 | 20260506 22:38:59.477 | INFO | Analyzing chunk frames = [5005312,5005824) | 20260506 22:38:59.477 | INFO | MemInfo: { | 20260506 22:38:59.477 | INFO | "gpu" : " 17 GB", | 20260506 22:38:59.477 | INFO | "huge" : "", | 20260506 22:38:59.477 | INFO | "huge_shared" : " 27 GB", | 20260506 22:38:59.477 | INFO | "malloc" : " 14 GB", | 20260506 22:38:59.477 | INFO | "pinned" : " 29 GB" | 20260506 22:38:59.477 | INFO | } | 20260506 22:38:59.477 | INFO | Perf report for the last 5.13983 seconds: | 20260506 22:38:59.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:38:59.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:59.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:59.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:59.477 | INFO | ANALYSIS : 62.5%, 3.0, 97.1ms, 0 | 20260506 22:38:59.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:38:59.477 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260506 22:38:59.477 | INFO | PRE_HQ : 59.6%, 1.8, 57.1ms, 0 | 20260506 22:38:59.477 | INFO | BAZWRITER : 62.2%, 3.5, 112.3ms, 0 | 20260506 22:38:59.477 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:38:59.477 | INFO | Uploaded 36GiB to GPU in 2.05176s (17.5459GiB/s) | 20260506 22:38:59.477 | INFO | Downloaded 38GiB to GPU in 2.18945s (17.356GiB/s) | 20260506 22:39:04.620 | INFO | Analyzing chunk frames = [5005824,5006336) | 20260506 22:39:04.620 | INFO | MemInfo: { | 20260506 22:39:04.620 | INFO | "gpu" : " 17 GB", | 20260506 22:39:04.620 | INFO | "huge" : "", | 20260506 22:39:04.620 | INFO | "huge_shared" : " 27 GB", | 20260506 22:39:04.620 | INFO | "malloc" : " 14 GB", | 20260506 22:39:04.620 | INFO | "pinned" : " 29 GB" | 20260506 22:39:04.621 | INFO | } | 20260506 22:39:04.621 | INFO | Perf report for the last 5.1436 seconds: | 20260506 22:39:04.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:39:04.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:04.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:04.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:04.621 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 22:39:04.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:04.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:04.621 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:39:04.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:04.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:04.621 | INFO | Uploaded 36GiB to GPU in 1.99843s (18.0142GiB/s) | 20260506 22:39:04.621 | INFO | Downloaded 34GiB to GPU in 1.92215s (17.6885GiB/s) | 20260506 22:39:09.762 | INFO | Analyzing chunk frames = [5006336,5006848) | 20260506 22:39:09.762 | INFO | MemInfo: { | 20260506 22:39:09.762 | INFO | "gpu" : " 17 GB", | 20260506 22:39:09.762 | INFO | "huge" : "", | 20260506 22:39:09.762 | INFO | "huge_shared" : " 27 GB", | 20260506 22:39:09.762 | INFO | "malloc" : " 14 GB", | 20260506 22:39:09.762 | INFO | "pinned" : " 29 GB" | 20260506 22:39:09.762 | INFO | } | 20260506 22:39:09.763 | INFO | Perf report for the last 5.14197 seconds: | 20260506 22:39:09.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:39:09.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:09.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:09.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:09.763 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 22:39:09.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:09.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:09.763 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:39:09.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:09.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:09.763 | INFO | Uploaded 36GiB to GPU in 1.98536s (18.1327GiB/s) | 20260506 22:39:09.763 | INFO | Downloaded 34GiB to GPU in 1.91607s (17.7447GiB/s) | 20260506 22:39:14.900 | INFO | Analyzing chunk frames = [5006848,5007360) | 20260506 22:39:14.900 | INFO | MemInfo: { | 20260506 22:39:14.900 | INFO | "gpu" : " 17 GB", | 20260506 22:39:14.900 | INFO | "huge" : "", | 20260506 22:39:14.900 | INFO | "huge_shared" : " 27 GB", | 20260506 22:39:14.900 | INFO | "malloc" : " 14 GB", | 20260506 22:39:14.900 | INFO | "pinned" : " 29 GB" | 20260506 22:39:14.900 | INFO | } | 20260506 22:39:14.900 | INFO | Perf report for the last 5.13748 seconds: | 20260506 22:39:14.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:39:14.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:14.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:14.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:14.900 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 22:39:14.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:14.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:14.900 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:39:14.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:14.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:14.900 | INFO | Uploaded 36GiB to GPU in 2.01693s (17.8489GiB/s) | 20260506 22:39:14.900 | INFO | Downloaded 34GiB to GPU in 1.90874s (17.8128GiB/s) | 20260506 22:39:20.041 | INFO | Analyzing chunk frames = [5007360,5007872) | 20260506 22:39:20.041 | INFO | MemInfo: { | 20260506 22:39:20.041 | INFO | "gpu" : " 17 GB", | 20260506 22:39:20.041 | INFO | "huge" : "", | 20260506 22:39:20.041 | INFO | "huge_shared" : " 27 GB", | 20260506 22:39:20.041 | INFO | "malloc" : " 14 GB", | 20260506 22:39:20.041 | INFO | "pinned" : " 29 GB" | 20260506 22:39:20.041 | INFO | } | 20260506 22:39:20.041 | INFO | Perf report for the last 5.14097 seconds: | 20260506 22:39:20.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:39:20.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:20.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:20.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:20.041 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260506 22:39:20.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:20.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:20.041 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 22:39:20.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:20.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:20.041 | INFO | Uploaded 36GiB to GPU in 2.0276s (17.755GiB/s) | 20260506 22:39:20.041 | INFO | Downloaded 34GiB to GPU in 1.9415s (17.5122GiB/s) | 20260506 22:39:25.178 | INFO | Analyzing chunk frames = [5007872,5008384) | 20260506 22:39:25.178 | INFO | MemInfo: { | 20260506 22:39:25.178 | INFO | "gpu" : " 17 GB", | 20260506 22:39:25.178 | INFO | "huge" : "", | 20260506 22:39:25.178 | INFO | "huge_shared" : " 27 GB", | 20260506 22:39:25.178 | INFO | "malloc" : " 14 GB", | 20260506 22:39:25.178 | INFO | "pinned" : " 29 GB" | 20260506 22:39:25.178 | INFO | } | 20260506 22:39:25.179 | INFO | Perf report for the last 5.1375 seconds: | 20260506 22:39:25.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:39:25.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:25.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:25.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:25.179 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 22:39:25.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:25.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:25.179 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:39:25.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:25.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:25.179 | INFO | Uploaded 36GiB to GPU in 2.02223s (17.8021GiB/s) | 20260506 22:39:25.179 | INFO | Downloaded 34GiB to GPU in 1.94068s (17.5196GiB/s) | 20260506 22:39:30.314 | INFO | Analyzing chunk frames = [5008384,5008896) | 20260506 22:39:30.314 | INFO | MemInfo: { | 20260506 22:39:30.314 | INFO | "gpu" : " 17 GB", | 20260506 22:39:30.314 | INFO | "huge" : "", | 20260506 22:39:30.314 | INFO | "huge_shared" : " 27 GB", | 20260506 22:39:30.314 | INFO | "malloc" : " 14 GB", | 20260506 22:39:30.314 | INFO | "pinned" : " 29 GB" | 20260506 22:39:30.314 | INFO | } | 20260506 22:39:30.314 | INFO | Perf report for the last 5.13533 seconds: | 20260506 22:39:30.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:39:30.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:30.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:30.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:30.314 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:39:30.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:30.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:30.314 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:39:30.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:30.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:30.314 | INFO | Uploaded 36GiB to GPU in 1.98243s (18.1595GiB/s) | 20260506 22:39:30.314 | INFO | Downloaded 34GiB to GPU in 1.99526s (17.0404GiB/s) | 20260506 22:39:35.457 | INFO | Analyzing chunk frames = [5008896,5009408) | 20260506 22:39:35.457 | INFO | MemInfo: { | 20260506 22:39:35.457 | INFO | "gpu" : " 17 GB", | 20260506 22:39:35.457 | INFO | "huge" : "", | 20260506 22:39:35.457 | INFO | "huge_shared" : " 27 GB", | 20260506 22:39:35.457 | INFO | "malloc" : " 14 GB", | 20260506 22:39:35.457 | INFO | "pinned" : " 29 GB" | 20260506 22:39:35.457 | INFO | } | 20260506 22:39:35.457 | INFO | Perf report for the last 5.14338 seconds: | 20260506 22:39:35.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:39:35.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:35.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:35.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:35.457 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 22:39:35.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:35.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:35.457 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:39:35.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:35.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:35.458 | INFO | Uploaded 36GiB to GPU in 2.04832s (17.5754GiB/s) | 20260506 22:39:35.458 | INFO | Downloaded 34GiB to GPU in 2.00293s (16.9751GiB/s) | 20260506 22:39:35.573 | INFO | Starting a round of Baz IO | 20260506 22:39:38.693 | INFO | Populating RTMetrics report | 20260506 22:39:38.693 | INFO | RTMetrics report populated | 20260506 22:39:38.693 | INFO | Opening tmp RTMetrics json file | 20260506 22:39:38.694 | INFO | Serializing RTMetrics to json | 20260506 22:39:38.695 | INFO | Serialization done, swapping tmp file now... | 20260506 22:39:38.695 | INFO | RTMetrics done | 20260506 22:39:38.787 | INFO | Finished a round of Baz IO: Wrote 1.907 GiB in 3.213 seconds | 20260506 22:39:38.787 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.63% / 0.00% / 66.37% | 20260506 22:39:38.787 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.487403% / 0% / 5.16574e-05% | 20260506 22:39:38.787 | INFO | Baz IO overall: | 20260506 22:39:38.787 | INFO | BytesWritten : 2629842093804 | 20260506 22:39:38.787 | INFO | NumEvents : 1268680658605 | 20260506 22:39:40.593 | INFO | Analyzing chunk frames = [5009408,5009920) | 20260506 22:39:40.593 | INFO | MemInfo: { | 20260506 22:39:40.593 | INFO | "gpu" : " 17 GB", | 20260506 22:39:40.593 | INFO | "huge" : "", | 20260506 22:39:40.593 | INFO | "huge_shared" : " 27 GB", | 20260506 22:39:40.593 | INFO | "malloc" : " 14 GB", | 20260506 22:39:40.593 | INFO | "pinned" : " 29 GB" | 20260506 22:39:40.593 | INFO | } | 20260506 22:39:40.593 | INFO | Perf report for the last 5.13558 seconds: | 20260506 22:39:40.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:39:40.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:40.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:40.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:40.593 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260506 22:39:40.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:40.593 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 22:39:40.593 | INFO | PRE_HQ : 59.6%, 1.8, 57.0ms, 0 | 20260506 22:39:40.593 | INFO | BAZWRITER : 62.3%, 3.5, 111.7ms, 0 | 20260506 22:39:40.593 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:39:40.593 | INFO | Uploaded 36GiB to GPU in 2.0074s (17.9337GiB/s) | 20260506 22:39:40.593 | INFO | Downloaded 38GiB to GPU in 2.19117s (17.3424GiB/s) | 20260506 22:39:45.733 | INFO | Analyzing chunk frames = [5009920,5010432) | 20260506 22:39:45.733 | INFO | MemInfo: { | 20260506 22:39:45.733 | INFO | "gpu" : " 17 GB", | 20260506 22:39:45.733 | INFO | "huge" : "", | 20260506 22:39:45.733 | INFO | "huge_shared" : " 27 GB", | 20260506 22:39:45.733 | INFO | "malloc" : " 14 GB", | 20260506 22:39:45.733 | INFO | "pinned" : " 29 GB" | 20260506 22:39:45.733 | INFO | } | 20260506 22:39:45.734 | INFO | Perf report for the last 5.14067 seconds: | 20260506 22:39:45.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:39:45.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:45.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:45.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:45.734 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:39:45.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:45.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:45.734 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:39:45.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:45.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:45.734 | INFO | Uploaded 36GiB to GPU in 2.00493s (17.9558GiB/s) | 20260506 22:39:45.734 | INFO | Downloaded 34GiB to GPU in 1.93024s (17.6144GiB/s) | 20260506 22:39:50.877 | INFO | Analyzing chunk frames = [5010432,5010944) | 20260506 22:39:50.877 | INFO | MemInfo: { | 20260506 22:39:50.877 | INFO | "gpu" : " 17 GB", | 20260506 22:39:50.877 | INFO | "huge" : "", | 20260506 22:39:50.877 | INFO | "huge_shared" : " 27 GB", | 20260506 22:39:50.877 | INFO | "malloc" : " 14 GB", | 20260506 22:39:50.877 | INFO | "pinned" : " 29 GB" | 20260506 22:39:50.877 | INFO | } | 20260506 22:39:50.877 | INFO | Perf report for the last 5.14374 seconds: | 20260506 22:39:50.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:39:50.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:50.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:50.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:50.877 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 22:39:50.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:50.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:50.877 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:39:50.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:50.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:50.878 | INFO | Uploaded 36GiB to GPU in 1.95199s (18.4427GiB/s) | 20260506 22:39:50.878 | INFO | Downloaded 34GiB to GPU in 1.93023s (17.6145GiB/s) | 20260506 22:39:56.012 | INFO | Analyzing chunk frames = [5010944,5011456) | 20260506 22:39:56.012 | INFO | MemInfo: { | 20260506 22:39:56.012 | INFO | "gpu" : " 17 GB", | 20260506 22:39:56.012 | INFO | "huge" : "", | 20260506 22:39:56.012 | INFO | "huge_shared" : " 27 GB", | 20260506 22:39:56.012 | INFO | "malloc" : " 14 GB", | 20260506 22:39:56.012 | INFO | "pinned" : " 29 GB" | 20260506 22:39:56.012 | INFO | } | 20260506 22:39:56.012 | INFO | Perf report for the last 5.13486 seconds: | 20260506 22:39:56.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:39:56.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:56.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:56.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:56.012 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 22:39:56.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:56.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:56.012 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 22:39:56.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:56.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:39:56.013 | INFO | Uploaded 36GiB to GPU in 1.9706s (18.2685GiB/s) | 20260506 22:39:56.013 | INFO | Downloaded 34GiB to GPU in 1.90807s (17.819GiB/s) | 20260506 22:40:01.153 | INFO | Analyzing chunk frames = [5011456,5011968) | 20260506 22:40:01.153 | INFO | MemInfo: { | 20260506 22:40:01.153 | INFO | "gpu" : " 17 GB", | 20260506 22:40:01.153 | INFO | "huge" : "", | 20260506 22:40:01.153 | INFO | "huge_shared" : " 27 GB", | 20260506 22:40:01.153 | INFO | "malloc" : " 14 GB", | 20260506 22:40:01.153 | INFO | "pinned" : " 29 GB" | 20260506 22:40:01.153 | INFO | } | 20260506 22:40:01.153 | INFO | Perf report for the last 5.14122 seconds: | 20260506 22:40:01.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:40:01.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:01.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:01.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:01.153 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 22:40:01.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:01.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:01.154 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 22:40:01.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:01.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:01.154 | INFO | Uploaded 36GiB to GPU in 1.98214s (18.1622GiB/s) | 20260506 22:40:01.154 | INFO | Downloaded 34GiB to GPU in 1.92403s (17.6712GiB/s) | 20260506 22:40:06.291 | INFO | Analyzing chunk frames = [5011968,5012480) | 20260506 22:40:06.291 | INFO | MemInfo: { | 20260506 22:40:06.291 | INFO | "gpu" : " 17 GB", | 20260506 22:40:06.291 | INFO | "huge" : "", | 20260506 22:40:06.291 | INFO | "huge_shared" : " 27 GB", | 20260506 22:40:06.291 | INFO | "malloc" : " 14 GB", | 20260506 22:40:06.291 | INFO | "pinned" : " 29 GB" | 20260506 22:40:06.291 | INFO | } | 20260506 22:40:06.291 | INFO | Perf report for the last 5.13793 seconds: | 20260506 22:40:06.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:40:06.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:06.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:06.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:06.291 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 22:40:06.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:06.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:06.291 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:40:06.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:06.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:06.292 | INFO | Uploaded 36GiB to GPU in 2.00399s (17.9642GiB/s) | 20260506 22:40:06.292 | INFO | Downloaded 34GiB to GPU in 1.94002s (17.5256GiB/s) | 20260506 22:40:11.428 | INFO | Analyzing chunk frames = [5012480,5012992) | 20260506 22:40:11.428 | INFO | MemInfo: { | 20260506 22:40:11.428 | INFO | "gpu" : " 17 GB", | 20260506 22:40:11.428 | INFO | "huge" : "", | 20260506 22:40:11.428 | INFO | "huge_shared" : " 27 GB", | 20260506 22:40:11.428 | INFO | "malloc" : " 14 GB", | 20260506 22:40:11.428 | INFO | "pinned" : " 29 GB" | 20260506 22:40:11.428 | INFO | } | 20260506 22:40:11.428 | INFO | Perf report for the last 5.13694 seconds: | 20260506 22:40:11.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:40:11.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:11.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:11.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:11.428 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:40:11.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:11.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:11.428 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 22:40:11.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:11.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:11.429 | INFO | Uploaded 36GiB to GPU in 1.95366s (18.427GiB/s) | 20260506 22:40:11.429 | INFO | Downloaded 34GiB to GPU in 1.98192s (17.1551GiB/s) | 20260506 22:40:16.569 | INFO | Analyzing chunk frames = [5012992,5013504) | 20260506 22:40:16.569 | INFO | MemInfo: { | 20260506 22:40:16.569 | INFO | "gpu" : " 17 GB", | 20260506 22:40:16.569 | INFO | "huge" : "", | 20260506 22:40:16.569 | INFO | "huge_shared" : " 27 GB", | 20260506 22:40:16.569 | INFO | "malloc" : " 14 GB", | 20260506 22:40:16.569 | INFO | "pinned" : " 29 GB" | 20260506 22:40:16.569 | INFO | } | 20260506 22:40:16.569 | INFO | Perf report for the last 5.14106 seconds: | 20260506 22:40:16.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:40:16.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:16.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:16.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:16.570 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 22:40:16.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:16.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:16.570 | INFO | PRE_HQ : 30.9%, 1.0, 16.0ms, 0 | 20260506 22:40:16.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:16.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:16.570 | INFO | Uploaded 36GiB to GPU in 2.00477s (17.9572GiB/s) | 20260506 22:40:16.570 | INFO | Downloaded 34GiB to GPU in 1.99796s (17.0173GiB/s) | 20260506 22:40:16.570 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:40:16.570 | INFO | Total mem usage: 18 GiB | 20260506 22:40:16.570 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:40:16.570 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:40:16.570 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:40:16.570 | INFO | High Water Breakdown: | 20260506 22:40:16.570 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:40:16.570 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:40:16.570 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:40:16.570 | INFO | ------------------------------------------ | 20260506 22:40:16.570 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:40:16.570 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:40:16.570 | INFO | Total mem usage: 26.5 GiB | 20260506 22:40:16.570 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:40:16.570 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:40:16.570 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:40:16.570 | INFO | High Water Breakdown: | 20260506 22:40:16.570 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:40:16.570 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:40:16.570 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:40:16.570 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:40:16.570 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:40:16.570 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:40:16.570 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:40:16.570 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:40:16.570 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:40:16.570 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:40:16.570 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:40:16.570 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:40:16.570 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:40:16.570 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:40:16.570 | INFO | ------------------------------------------ | 20260506 22:40:16.571 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:40:16.571 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:40:16.571 | INFO | Total mem usage: 25.6 GiB | 20260506 22:40:16.571 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:40:16.571 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:40:16.571 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:40:16.571 | INFO | High Water Breakdown: | 20260506 22:40:16.571 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:40:16.571 | INFO | ------------------------------------------ | 20260506 22:40:16.571 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:40:16.571 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:40:16.571 | INFO | Total mem usage: 15.7 GiB | 20260506 22:40:16.571 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:40:16.571 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:40:16.571 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:40:16.571 | INFO | High Water Breakdown: | 20260506 22:40:16.571 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:40:16.571 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:40:16.571 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:40:16.571 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:40:16.571 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:40:16.571 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:40:16.571 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:40:16.571 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:40:16.571 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:40:16.571 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:40:16.571 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:40:16.571 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:40:16.571 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:40:16.571 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:40:16.571 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:40:16.571 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:40:16.571 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:40:16.571 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:40:16.571 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:40:16.571 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:40:16.571 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:40:16.571 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:40:16.571 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:40:16.571 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:40:16.571 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:40:16.571 | INFO | ------------------------------------------ | 20260506 22:40:16.571 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:40:16.571 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:40:16.571 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:40:16.571 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:40:16.571 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.64e+01, 1.34e+01 / 2.15e+00 | 20260506 22:40:16.571 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.00e-01, 1.01e-02 / 5.81e-03 | 20260506 22:40:16.571 | INFO | PerfReport: 11.65%, Baseline -- 3.62e+00 / 4.35e+00, 5.46e-01 / 3.50e-01 | 20260506 22:40:16.571 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.44e+00, 1.78e-01 / 1.29e-01 | 20260506 22:40:16.571 | INFO | PerfReport: 16.10%, DME -- 4.00e+01 / 4.50e+01, 4.37e+00 / 2.53e+00 | 20260506 22:40:16.571 | INFO | PerfReport: 57.32%, FrameLabeling -- 1.78e+01 / 2.11e+01, 1.37e+00 / 1.94e+00 | 20260506 22:40:16.571 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.19e+00 / 2.73e+00, 3.77e-01 / 2.57e-01 | 20260506 22:40:16.571 | INFO | PerfReport: 3.22%, Metrics -- 9.99e-01 / 1.33e+00, 2.63e-01 / 1.57e-01 | 20260506 22:40:16.571 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.43e-01 / 3.91e-01, 5.50e-02 / 2.61e-02 | 20260506 22:40:16.571 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:40:16.571 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:40:16.571 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:40:16.571 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260506 22:40:16.571 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 7.04e+00 / 0.00e+00 | 20260506 22:40:16.571 | INFO | PerfReport: 21.25%, Download -- 2.06e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 22:40:16.572 | INFO | PerfReport: 32.06%, Compute -- 3.11e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 22:40:16.572 | INFO | PerfReport: 1.34%, LockUpload -- 1.30e+00 / 0.00e+00, 4.94e+00 / 0.00e+00 | 20260506 22:40:16.572 | INFO | PerfReport: 1.27%, LockDownload -- 1.23e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260506 22:40:16.572 | INFO | PerfReport: 22.55%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 22:40:16.572 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260506 22:40:16.686 | INFO | Starting a round of Baz IO | 20260506 22:40:19.805 | INFO | Populating RTMetrics report | 20260506 22:40:19.806 | INFO | RTMetrics report populated | 20260506 22:40:19.806 | INFO | Opening tmp RTMetrics json file | 20260506 22:40:19.826 | INFO | Serializing RTMetrics to json | 20260506 22:40:19.827 | INFO | Serialization done, swapping tmp file now... | 20260506 22:40:19.828 | INFO | RTMetrics done | 20260506 22:40:19.901 | INFO | Finished a round of Baz IO: Wrote 1.907 GiB in 3.214 seconds | 20260506 22:40:19.901 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.63% / 0.00% / 66.37% | 20260506 22:40:19.901 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.485396% / 0% / 6.35783e-05% | 20260506 22:40:19.901 | INFO | Baz IO overall: | 20260506 22:40:19.901 | INFO | BytesWritten : 2631889522239 | 20260506 22:40:19.901 | INFO | NumEvents : 1269658304254 | 20260506 22:40:21.713 | INFO | Analyzing chunk frames = [5013504,5014016) | 20260506 22:40:21.713 | INFO | MemInfo: { | 20260506 22:40:21.713 | INFO | "gpu" : " 17 GB", | 20260506 22:40:21.713 | INFO | "huge" : "", | 20260506 22:40:21.713 | INFO | "huge_shared" : " 27 GB", | 20260506 22:40:21.713 | INFO | "malloc" : " 14 GB", | 20260506 22:40:21.713 | INFO | "pinned" : " 29 GB" | 20260506 22:40:21.713 | INFO | } | 20260506 22:40:21.713 | INFO | Perf report for the last 5.14408 seconds: | 20260506 22:40:21.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:40:21.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:21.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:21.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:21.714 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 22:40:21.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:21.714 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 22:40:21.714 | INFO | PRE_HQ : 59.8%, 1.9, 58.3ms, 0 | 20260506 22:40:21.714 | INFO | BAZWRITER : 62.5%, 3.5, 111.0ms, 0 | 20260506 22:40:21.714 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:40:21.714 | INFO | Uploaded 36GiB to GPU in 2.04209s (17.629GiB/s) | 20260506 22:40:21.714 | INFO | Downloaded 38GiB to GPU in 2.18069s (17.4256GiB/s) | 20260506 22:40:26.851 | INFO | Analyzing chunk frames = [5014016,5014528) | 20260506 22:40:26.851 | INFO | MemInfo: { | 20260506 22:40:26.851 | INFO | "gpu" : " 17 GB", | 20260506 22:40:26.851 | INFO | "huge" : "", | 20260506 22:40:26.851 | INFO | "huge_shared" : " 27 GB", | 20260506 22:40:26.851 | INFO | "malloc" : " 14 GB", | 20260506 22:40:26.851 | INFO | "pinned" : " 29 GB" | 20260506 22:40:26.851 | INFO | } | 20260506 22:40:26.851 | INFO | Perf report for the last 5.13787 seconds: | 20260506 22:40:26.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:40:26.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:26.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:26.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:26.851 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:40:26.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:26.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:26.851 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 22:40:26.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:26.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:26.852 | INFO | Uploaded 36GiB to GPU in 2.00243s (17.9782GiB/s) | 20260506 22:40:26.852 | INFO | Downloaded 34GiB to GPU in 1.92557s (17.6571GiB/s) | 20260506 22:40:31.990 | INFO | Analyzing chunk frames = [5014528,5015040) | 20260506 22:40:31.990 | INFO | MemInfo: { | 20260506 22:40:31.990 | INFO | "gpu" : " 17 GB", | 20260506 22:40:31.990 | INFO | "huge" : "", | 20260506 22:40:31.990 | INFO | "huge_shared" : " 27 GB", | 20260506 22:40:31.990 | INFO | "malloc" : " 14 GB", | 20260506 22:40:31.990 | INFO | "pinned" : " 29 GB" | 20260506 22:40:31.990 | INFO | } | 20260506 22:40:31.990 | INFO | Perf report for the last 5.13862 seconds: | 20260506 22:40:31.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:40:31.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:31.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:31.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:31.990 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 22:40:31.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:31.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:31.990 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:40:31.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:31.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:31.990 | INFO | Uploaded 36GiB to GPU in 1.99134s (18.0783GiB/s) | 20260506 22:40:31.990 | INFO | Downloaded 34GiB to GPU in 1.93124s (17.6052GiB/s) | 20260506 22:40:37.129 | INFO | Analyzing chunk frames = [5015040,5015552) | 20260506 22:40:37.129 | INFO | MemInfo: { | 20260506 22:40:37.129 | INFO | "gpu" : " 17 GB", | 20260506 22:40:37.129 | INFO | "huge" : "", | 20260506 22:40:37.129 | INFO | "huge_shared" : " 27 GB", | 20260506 22:40:37.129 | INFO | "malloc" : " 14 GB", | 20260506 22:40:37.129 | INFO | "pinned" : " 29 GB" | 20260506 22:40:37.129 | INFO | } | 20260506 22:40:37.129 | INFO | Perf report for the last 5.13891 seconds: | 20260506 22:40:37.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:40:37.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:37.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:37.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:37.129 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 22:40:37.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:37.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:37.129 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:40:37.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:37.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:37.129 | INFO | Uploaded 36GiB to GPU in 1.98208s (18.1628GiB/s) | 20260506 22:40:37.129 | INFO | Downloaded 34GiB to GPU in 1.90324s (17.8643GiB/s) | 20260506 22:40:42.270 | INFO | Analyzing chunk frames = [5015552,5016064) | 20260506 22:40:42.270 | INFO | MemInfo: { | 20260506 22:40:42.270 | INFO | "gpu" : " 17 GB", | 20260506 22:40:42.270 | INFO | "huge" : "", | 20260506 22:40:42.270 | INFO | "huge_shared" : " 27 GB", | 20260506 22:40:42.270 | INFO | "malloc" : " 14 GB", | 20260506 22:40:42.270 | INFO | "pinned" : " 29 GB" | 20260506 22:40:42.270 | INFO | } | 20260506 22:40:42.270 | INFO | Perf report for the last 5.14109 seconds: | 20260506 22:40:42.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:40:42.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:42.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:42.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:42.270 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 22:40:42.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:42.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:42.270 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 22:40:42.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:42.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:42.270 | INFO | Uploaded 36GiB to GPU in 2.01604s (17.8568GiB/s) | 20260506 22:40:42.270 | INFO | Downloaded 34GiB to GPU in 1.91621s (17.7434GiB/s) | 20260506 22:40:47.408 | INFO | Analyzing chunk frames = [5016064,5016576) | 20260506 22:40:47.408 | INFO | MemInfo: { | 20260506 22:40:47.408 | INFO | "gpu" : " 17 GB", | 20260506 22:40:47.408 | INFO | "huge" : "", | 20260506 22:40:47.408 | INFO | "huge_shared" : " 27 GB", | 20260506 22:40:47.408 | INFO | "malloc" : " 14 GB", | 20260506 22:40:47.408 | INFO | "pinned" : " 29 GB" | 20260506 22:40:47.408 | INFO | } | 20260506 22:40:47.408 | INFO | Perf report for the last 5.13837 seconds: | 20260506 22:40:47.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:40:47.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:47.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:47.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:47.408 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:40:47.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:47.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:47.409 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:40:47.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:47.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:47.409 | INFO | Uploaded 36GiB to GPU in 2.01741s (17.8446GiB/s) | 20260506 22:40:47.409 | INFO | Downloaded 34GiB to GPU in 1.94847s (17.4496GiB/s) | 20260506 22:40:52.547 | INFO | Analyzing chunk frames = [5016576,5017088) | 20260506 22:40:52.547 | INFO | MemInfo: { | 20260506 22:40:52.547 | INFO | "gpu" : " 17 GB", | 20260506 22:40:52.547 | INFO | "huge" : "", | 20260506 22:40:52.547 | INFO | "huge_shared" : " 27 GB", | 20260506 22:40:52.547 | INFO | "malloc" : " 14 GB", | 20260506 22:40:52.547 | INFO | "pinned" : " 29 GB" | 20260506 22:40:52.547 | INFO | } | 20260506 22:40:52.547 | INFO | Perf report for the last 5.13842 seconds: | 20260506 22:40:52.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:40:52.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:52.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:52.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:52.547 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:40:52.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:52.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:52.547 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:40:52.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:52.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:52.547 | INFO | Uploaded 36GiB to GPU in 1.96983s (18.2757GiB/s) | 20260506 22:40:52.547 | INFO | Downloaded 34GiB to GPU in 1.99022s (17.0835GiB/s) | 20260506 22:40:57.685 | INFO | Analyzing chunk frames = [5017088,5017600) | 20260506 22:40:57.685 | INFO | MemInfo: { | 20260506 22:40:57.685 | INFO | "gpu" : " 17 GB", | 20260506 22:40:57.685 | INFO | "huge" : "", | 20260506 22:40:57.685 | INFO | "huge_shared" : " 27 GB", | 20260506 22:40:57.685 | INFO | "malloc" : " 14 GB", | 20260506 22:40:57.685 | INFO | "pinned" : " 29 GB" | 20260506 22:40:57.685 | INFO | } | 20260506 22:40:57.686 | INFO | Perf report for the last 5.13863 seconds: | 20260506 22:40:57.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:40:57.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:57.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:57.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:57.686 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:40:57.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:57.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:57.686 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 22:40:57.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:57.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:40:57.686 | INFO | Uploaded 36GiB to GPU in 1.98815s (18.1073GiB/s) | 20260506 22:40:57.686 | INFO | Downloaded 34GiB to GPU in 1.98186s (17.1556GiB/s) | 20260506 22:40:57.788 | INFO | Starting a round of Baz IO | 20260506 22:41:00.915 | INFO | Populating RTMetrics report | 20260506 22:41:00.915 | INFO | RTMetrics report populated | 20260506 22:41:00.915 | INFO | Opening tmp RTMetrics json file | 20260506 22:41:00.918 | INFO | Serializing RTMetrics to json | 20260506 22:41:00.935 | INFO | Serialization done, swapping tmp file now... | 20260506 22:41:00.935 | INFO | RTMetrics done | 20260506 22:41:01.012 | INFO | Finished a round of Baz IO: Wrote 1.905 GiB in 3.224 seconds | 20260506 22:41:01.012 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.63% / 0.00% / 66.37% | 20260506 22:41:01.012 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.483537% / 3.97364e-06% / 5.16574e-05% | 20260506 22:41:01.012 | INFO | Baz IO overall: | 20260506 22:41:01.012 | INFO | BytesWritten : 2633935349276 | 20260506 22:41:01.012 | INFO | NumEvents : 1270634940727 | 20260506 22:41:02.830 | INFO | Analyzing chunk frames = [5017600,5018112) | 20260506 22:41:02.830 | INFO | MemInfo: { | 20260506 22:41:02.830 | INFO | "gpu" : " 17 GB", | 20260506 22:41:02.830 | INFO | "huge" : "", | 20260506 22:41:02.830 | INFO | "huge_shared" : " 27 GB", | 20260506 22:41:02.830 | INFO | "malloc" : " 14 GB", | 20260506 22:41:02.830 | INFO | "pinned" : " 29 GB" | 20260506 22:41:02.830 | INFO | } | 20260506 22:41:02.830 | INFO | Perf report for the last 5.1447 seconds: | 20260506 22:41:02.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:41:02.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:02.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:02.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:02.830 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 22:41:02.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:02.830 | INFO | RT_METRICS : 43.1%, 1.0, 22.7ms, 0 | 20260506 22:41:02.830 | INFO | PRE_HQ : 59.6%, 1.8, 57.2ms, 0 | 20260506 22:41:02.830 | INFO | BAZWRITER : 62.3%, 3.5, 111.6ms, 0 | 20260506 22:41:02.830 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:41:02.831 | INFO | Uploaded 36GiB to GPU in 2.0376s (17.6678GiB/s) | 20260506 22:41:02.831 | INFO | Downloaded 38GiB to GPU in 2.17536s (17.4684GiB/s) | 20260506 22:41:07.967 | INFO | Analyzing chunk frames = [5018112,5018624) | 20260506 22:41:07.967 | INFO | MemInfo: { | 20260506 22:41:07.967 | INFO | "gpu" : " 17 GB", | 20260506 22:41:07.967 | INFO | "huge" : "", | 20260506 22:41:07.967 | INFO | "huge_shared" : " 27 GB", | 20260506 22:41:07.967 | INFO | "malloc" : " 14 GB", | 20260506 22:41:07.967 | INFO | "pinned" : " 29 GB" | 20260506 22:41:07.967 | INFO | } | 20260506 22:41:07.967 | INFO | Perf report for the last 5.13664 seconds: | 20260506 22:41:07.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:41:07.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:07.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:07.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:07.967 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:41:07.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:07.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:07.967 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:41:07.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:07.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:07.967 | INFO | Uploaded 36GiB to GPU in 2.01708s (17.8476GiB/s) | 20260506 22:41:07.967 | INFO | Downloaded 34GiB to GPU in 1.93161s (17.6019GiB/s) | 20260506 22:41:13.104 | INFO | Analyzing chunk frames = [5018624,5019136) | 20260506 22:41:13.104 | INFO | MemInfo: { | 20260506 22:41:13.104 | INFO | "gpu" : " 17 GB", | 20260506 22:41:13.104 | INFO | "huge" : "", | 20260506 22:41:13.104 | INFO | "huge_shared" : " 27 GB", | 20260506 22:41:13.104 | INFO | "malloc" : " 14 GB", | 20260506 22:41:13.104 | INFO | "pinned" : " 29 GB" | 20260506 22:41:13.104 | INFO | } | 20260506 22:41:13.104 | INFO | Perf report for the last 5.13755 seconds: | 20260506 22:41:13.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:41:13.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:13.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:13.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:13.105 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260506 22:41:13.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:13.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:13.105 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:41:13.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:13.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:13.105 | INFO | Uploaded 36GiB to GPU in 1.99953s (18.0042GiB/s) | 20260506 22:41:13.105 | INFO | Downloaded 34GiB to GPU in 1.93466s (17.5741GiB/s) | 20260506 22:41:18.239 | INFO | Analyzing chunk frames = [5019136,5019648) | 20260506 22:41:18.239 | INFO | MemInfo: { | 20260506 22:41:18.239 | INFO | "gpu" : " 17 GB", | 20260506 22:41:18.239 | INFO | "huge" : "", | 20260506 22:41:18.239 | INFO | "huge_shared" : " 27 GB", | 20260506 22:41:18.239 | INFO | "malloc" : " 14 GB", | 20260506 22:41:18.239 | INFO | "pinned" : " 29 GB" | 20260506 22:41:18.239 | INFO | } | 20260506 22:41:18.239 | INFO | Perf report for the last 5.13489 seconds: | 20260506 22:41:18.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:41:18.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:18.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:18.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:18.239 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260506 22:41:18.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:18.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:18.239 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:41:18.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:18.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:18.240 | INFO | Uploaded 36GiB to GPU in 2.01819s (17.8377GiB/s) | 20260506 22:41:18.240 | INFO | Downloaded 34GiB to GPU in 1.90915s (17.809GiB/s) | 20260506 22:41:23.377 | INFO | Analyzing chunk frames = [5019648,5020160) | 20260506 22:41:23.377 | INFO | MemInfo: { | 20260506 22:41:23.377 | INFO | "gpu" : " 17 GB", | 20260506 22:41:23.377 | INFO | "huge" : "", | 20260506 22:41:23.377 | INFO | "huge_shared" : " 27 GB", | 20260506 22:41:23.377 | INFO | "malloc" : " 14 GB", | 20260506 22:41:23.377 | INFO | "pinned" : " 29 GB" | 20260506 22:41:23.377 | INFO | } | 20260506 22:41:23.377 | INFO | Perf report for the last 5.13785 seconds: | 20260506 22:41:23.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:41:23.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:23.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:23.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:23.377 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 22:41:23.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:23.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:23.377 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:41:23.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:23.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:23.377 | INFO | Uploaded 36GiB to GPU in 2.01089s (17.9025GiB/s) | 20260506 22:41:23.377 | INFO | Downloaded 34GiB to GPU in 1.91796s (17.7272GiB/s) | 20260506 22:41:28.517 | INFO | Analyzing chunk frames = [5020160,5020672) | 20260506 22:41:28.517 | INFO | MemInfo: { | 20260506 22:41:28.517 | INFO | "gpu" : " 17 GB", | 20260506 22:41:28.517 | INFO | "huge" : "", | 20260506 22:41:28.517 | INFO | "huge_shared" : " 27 GB", | 20260506 22:41:28.517 | INFO | "malloc" : " 14 GB", | 20260506 22:41:28.517 | INFO | "pinned" : " 29 GB" | 20260506 22:41:28.517 | INFO | } | 20260506 22:41:28.517 | INFO | Perf report for the last 5.14019 seconds: | 20260506 22:41:28.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:41:28.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:28.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:28.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:28.517 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260506 22:41:28.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:28.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:28.518 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:41:28.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:28.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:28.518 | INFO | Uploaded 36GiB to GPU in 2.01664s (17.8515GiB/s) | 20260506 22:41:28.518 | INFO | Downloaded 34GiB to GPU in 1.9447s (17.4834GiB/s) | 20260506 22:41:33.659 | INFO | Analyzing chunk frames = [5020672,5021184) | 20260506 22:41:33.659 | INFO | MemInfo: { | 20260506 22:41:33.659 | INFO | "gpu" : " 17 GB", | 20260506 22:41:33.659 | INFO | "huge" : "", | 20260506 22:41:33.659 | INFO | "huge_shared" : " 27 GB", | 20260506 22:41:33.659 | INFO | "malloc" : " 14 GB", | 20260506 22:41:33.659 | INFO | "pinned" : " 29 GB" | 20260506 22:41:33.659 | INFO | } | 20260506 22:41:33.659 | INFO | Perf report for the last 5.14173 seconds: | 20260506 22:41:33.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:41:33.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:33.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:33.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:33.659 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 22:41:33.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:33.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:33.659 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 22:41:33.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:33.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:33.659 | INFO | Uploaded 36GiB to GPU in 1.95872s (18.3793GiB/s) | 20260506 22:41:33.660 | INFO | Downloaded 34GiB to GPU in 1.9853s (17.1258GiB/s) | 20260506 22:41:38.800 | INFO | Analyzing chunk frames = [5021184,5021696) | 20260506 22:41:38.800 | INFO | MemInfo: { | 20260506 22:41:38.800 | INFO | "gpu" : " 17 GB", | 20260506 22:41:38.800 | INFO | "huge" : "", | 20260506 22:41:38.800 | INFO | "huge_shared" : " 27 GB", | 20260506 22:41:38.800 | INFO | "malloc" : " 14 GB", | 20260506 22:41:38.800 | INFO | "pinned" : " 29 GB" | 20260506 22:41:38.800 | INFO | } | 20260506 22:41:38.800 | INFO | Perf report for the last 5.14119 seconds: | 20260506 22:41:38.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:41:38.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:38.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:38.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:38.800 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 22:41:38.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:38.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:38.800 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:41:38.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:38.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:38.801 | INFO | Uploaded 36GiB to GPU in 2.04127s (17.6361GiB/s) | 20260506 22:41:38.801 | INFO | Downloaded 34GiB to GPU in 2.00442s (16.9625GiB/s) | 20260506 22:41:38.801 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:41:38.801 | INFO | Total mem usage: 18 GiB | 20260506 22:41:38.801 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:41:38.801 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -25.1%) | 20260506 22:41:38.801 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:41:38.801 | INFO | High Water Breakdown: | 20260506 22:41:38.801 | INFO | 60.3% : 8.68 GiB ( 76% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:41:38.801 | INFO | 39.6% : 5.69 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:41:38.801 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:41:38.801 | INFO | ------------------------------------------ | 20260506 22:41:38.801 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:41:38.801 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:41:38.801 | INFO | Total mem usage: 26.5 GiB | 20260506 22:41:38.801 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:41:38.801 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:41:38.801 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:41:38.801 | INFO | High Water Breakdown: | 20260506 22:41:38.801 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:41:38.801 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:41:38.801 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:41:38.801 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:41:38.801 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:41:38.801 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:41:38.801 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:41:38.801 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:41:38.801 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:41:38.801 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:41:38.801 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:41:38.801 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:41:38.801 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:41:38.801 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:41:38.801 | INFO | ------------------------------------------ | 20260506 22:41:38.801 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:41:38.801 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:41:38.801 | INFO | Total mem usage: 25.6 GiB | 20260506 22:41:38.801 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:41:38.801 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:41:38.801 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:41:38.801 | INFO | High Water Breakdown: | 20260506 22:41:38.801 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:41:38.801 | INFO | ------------------------------------------ | 20260506 22:41:38.801 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:41:38.801 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:41:38.801 | INFO | Total mem usage: 15.7 GiB | 20260506 22:41:38.801 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:41:38.801 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:41:38.801 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:41:38.801 | INFO | High Water Breakdown: | 20260506 22:41:38.801 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:41:38.801 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:41:38.801 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:41:38.801 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:41:38.801 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:41:38.802 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:41:38.802 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:41:38.802 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:41:38.802 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:41:38.802 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:41:38.802 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:41:38.802 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:41:38.802 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:41:38.802 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:41:38.802 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:41:38.802 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:41:38.802 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:41:38.802 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:41:38.802 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:41:38.802 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:41:38.802 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:41:38.802 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:41:38.802 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:41:38.802 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:41:38.802 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:41:38.802 | INFO | ------------------------------------------ | 20260506 22:41:38.802 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:41:38.802 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:41:38.802 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:41:38.802 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:41:38.802 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.64e+01, 1.33e+01 / 2.60e+00 | 20260506 22:41:38.802 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.02e-01, 1.04e-02 / 7.75e-03 | 20260506 22:41:38.802 | INFO | PerfReport: 11.64%, Baseline -- 3.61e+00 / 4.40e+00, 5.37e-01 / 2.86e-01 | 20260506 22:41:38.802 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.42e+00, 1.77e-01 / 1.29e-01 | 20260506 22:41:38.802 | INFO | PerfReport: 16.02%, DME -- 3.98e+01 / 4.46e+01, 4.18e+00 / 2.97e+00 | 20260506 22:41:38.802 | INFO | PerfReport: 57.42%, FrameLabeling -- 1.78e+01 / 2.14e+01, 1.42e+00 / 2.02e+00 | 20260506 22:41:38.802 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.19e+00 / 2.69e+00, 3.65e-01 / 2.43e-01 | 20260506 22:41:38.802 | INFO | PerfReport: 3.21%, Metrics -- 9.97e-01 / 1.33e+00, 2.61e-01 / 1.57e-01 | 20260506 22:41:38.802 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.41e-01 / 3.94e-01, 5.38e-02 / 2.25e-02 | 20260506 22:41:38.802 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:41:38.802 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:41:38.802 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:41:38.802 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 22:41:38.802 | INFO | PerfReport: 21.60%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 22:41:38.802 | INFO | PerfReport: 21.25%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 22:41:38.802 | INFO | PerfReport: 32.08%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 22:41:38.802 | INFO | PerfReport: 1.30%, LockUpload -- 1.26e+00 / 0.00e+00, 4.62e+00 / 0.00e+00 | 20260506 22:41:38.802 | INFO | PerfReport: 1.17%, LockDownload -- 1.13e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260506 22:41:38.802 | INFO | PerfReport: 22.58%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 22:41:38.802 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260506 22:41:38.914 | INFO | Starting a round of Baz IO | 20260506 22:41:42.033 | INFO | Populating RTMetrics report | 20260506 22:41:42.033 | INFO | RTMetrics report populated | 20260506 22:41:42.034 | INFO | Opening tmp RTMetrics json file | 20260506 22:41:42.034 | INFO | Serializing RTMetrics to json | 20260506 22:41:42.036 | INFO | Serialization done, swapping tmp file now... | 20260506 22:41:42.039 | INFO | RTMetrics done | 20260506 22:41:42.129 | INFO | Finished a round of Baz IO: Wrote 1.905 GiB in 3.214 seconds | 20260506 22:41:42.129 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.62% / 0.00% / 66.37% | 20260506 22:41:42.129 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.48358% / 0% / 4.37101e-05% | 20260506 22:41:42.129 | INFO | Baz IO overall: | 20260506 22:41:42.129 | INFO | BytesWritten : 2635981046785 | 20260506 22:41:42.129 | INFO | NumEvents : 1271611334129 | 20260506 22:41:43.944 | INFO | Analyzing chunk frames = [5021696,5022208) | 20260506 22:41:43.944 | INFO | MemInfo: { | 20260506 22:41:43.944 | INFO | "gpu" : " 17 GB", | 20260506 22:41:43.944 | INFO | "huge" : "", | 20260506 22:41:43.944 | INFO | "huge_shared" : " 27 GB", | 20260506 22:41:43.944 | INFO | "malloc" : " 14 GB", | 20260506 22:41:43.944 | INFO | "pinned" : " 29 GB" | 20260506 22:41:43.944 | INFO | } | 20260506 22:41:43.944 | INFO | Perf report for the last 5.14364 seconds: | 20260506 22:41:43.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:41:43.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:43.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:43.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:43.944 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 22:41:43.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:43.944 | INFO | RT_METRICS : 42.4%, 1.0, 22.3ms, 0 | 20260506 22:41:43.944 | INFO | PRE_HQ : 59.7%, 1.8, 56.8ms, 0 | 20260506 22:41:43.944 | INFO | BAZWRITER : 62.2%, 3.5, 111.9ms, 0 | 20260506 22:41:43.944 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:41:43.944 | INFO | Uploaded 36GiB to GPU in 2.00362s (17.9675GiB/s) | 20260506 22:41:43.944 | INFO | Downloaded 38GiB to GPU in 2.16847s (17.5239GiB/s) | 20260506 22:41:49.080 | INFO | Analyzing chunk frames = [5022208,5022720) | 20260506 22:41:49.080 | INFO | MemInfo: { | 20260506 22:41:49.080 | INFO | "gpu" : " 17 GB", | 20260506 22:41:49.080 | INFO | "huge" : "", | 20260506 22:41:49.080 | INFO | "huge_shared" : " 27 GB", | 20260506 22:41:49.080 | INFO | "malloc" : " 14 GB", | 20260506 22:41:49.080 | INFO | "pinned" : " 29 GB" | 20260506 22:41:49.080 | INFO | } | 20260506 22:41:49.080 | INFO | Perf report for the last 5.13626 seconds: | 20260506 22:41:49.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:41:49.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:49.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:49.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:49.081 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:41:49.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:49.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:49.081 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:41:49.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:49.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:49.081 | INFO | Uploaded 36GiB to GPU in 2.00833s (17.9253GiB/s) | 20260506 22:41:49.081 | INFO | Downloaded 34GiB to GPU in 1.93161s (17.6019GiB/s) | 20260506 22:41:54.223 | INFO | Analyzing chunk frames = [5022720,5023232) | 20260506 22:41:54.223 | INFO | MemInfo: { | 20260506 22:41:54.223 | INFO | "gpu" : " 17 GB", | 20260506 22:41:54.223 | INFO | "huge" : "", | 20260506 22:41:54.223 | INFO | "huge_shared" : " 27 GB", | 20260506 22:41:54.223 | INFO | "malloc" : " 14 GB", | 20260506 22:41:54.223 | INFO | "pinned" : " 29 GB" | 20260506 22:41:54.223 | INFO | } | 20260506 22:41:54.223 | INFO | Perf report for the last 5.14286 seconds: | 20260506 22:41:54.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:41:54.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:54.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:54.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:54.223 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 22:41:54.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:54.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:54.223 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:41:54.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:54.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:54.223 | INFO | Uploaded 36GiB to GPU in 1.98698s (18.1179GiB/s) | 20260506 22:41:54.223 | INFO | Downloaded 34GiB to GPU in 1.93456s (17.5751GiB/s) | 20260506 22:41:59.358 | INFO | Analyzing chunk frames = [5023232,5023744) | 20260506 22:41:59.358 | INFO | MemInfo: { | 20260506 22:41:59.358 | INFO | "gpu" : " 17 GB", | 20260506 22:41:59.358 | INFO | "huge" : "", | 20260506 22:41:59.358 | INFO | "huge_shared" : " 27 GB", | 20260506 22:41:59.358 | INFO | "malloc" : " 14 GB", | 20260506 22:41:59.358 | INFO | "pinned" : " 29 GB" | 20260506 22:41:59.358 | INFO | } | 20260506 22:41:59.358 | INFO | Perf report for the last 5.13501 seconds: | 20260506 22:41:59.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:41:59.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:59.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:59.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:59.358 | INFO | ANALYSIS : 59.6%, 3.0, 92.3ms, 0 | 20260506 22:41:59.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:59.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:59.358 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:41:59.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:59.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:41:59.358 | INFO | Uploaded 36GiB to GPU in 1.9688s (18.2853GiB/s) | 20260506 22:41:59.359 | INFO | Downloaded 34GiB to GPU in 1.87908s (18.094GiB/s) | 20260506 22:42:04.494 | INFO | Analyzing chunk frames = [5023744,5024256) | 20260506 22:42:04.494 | INFO | MemInfo: { | 20260506 22:42:04.494 | INFO | "gpu" : " 17 GB", | 20260506 22:42:04.494 | INFO | "huge" : "", | 20260506 22:42:04.494 | INFO | "huge_shared" : " 27 GB", | 20260506 22:42:04.494 | INFO | "malloc" : " 14 GB", | 20260506 22:42:04.494 | INFO | "pinned" : " 29 GB" | 20260506 22:42:04.494 | INFO | } | 20260506 22:42:04.494 | INFO | Perf report for the last 5.13557 seconds: | 20260506 22:42:04.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:42:04.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:04.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:04.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:04.494 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 22:42:04.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:04.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:04.494 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:42:04.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:04.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:04.494 | INFO | Uploaded 36GiB to GPU in 1.99015s (18.089GiB/s) | 20260506 22:42:04.494 | INFO | Downloaded 34GiB to GPU in 1.92591s (17.654GiB/s) | 20260506 22:42:09.641 | INFO | Analyzing chunk frames = [5024256,5024768) | 20260506 22:42:09.641 | INFO | MemInfo: { | 20260506 22:42:09.641 | INFO | "gpu" : " 17 GB", | 20260506 22:42:09.641 | INFO | "huge" : "", | 20260506 22:42:09.641 | INFO | "huge_shared" : " 27 GB", | 20260506 22:42:09.641 | INFO | "malloc" : " 14 GB", | 20260506 22:42:09.641 | INFO | "pinned" : " 29 GB" | 20260506 22:42:09.641 | INFO | } | 20260506 22:42:09.641 | INFO | Perf report for the last 5.14773 seconds: | 20260506 22:42:09.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:42:09.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:09.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:09.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:09.642 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 22:42:09.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:09.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:09.642 | INFO | PRE_HQ : 30.1%, 1.0, 15.7ms, 0 | 20260506 22:42:09.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:09.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:09.642 | INFO | Uploaded 36GiB to GPU in 2.01094s (17.9021GiB/s) | 20260506 22:42:09.642 | INFO | Downloaded 34GiB to GPU in 1.95502s (17.3912GiB/s) | 20260506 22:42:14.778 | INFO | Analyzing chunk frames = [5024768,5025280) | 20260506 22:42:14.778 | INFO | MemInfo: { | 20260506 22:42:14.778 | INFO | "gpu" : " 17 GB", | 20260506 22:42:14.778 | INFO | "huge" : "", | 20260506 22:42:14.778 | INFO | "huge_shared" : " 27 GB", | 20260506 22:42:14.778 | INFO | "malloc" : " 14 GB", | 20260506 22:42:14.778 | INFO | "pinned" : " 29 GB" | 20260506 22:42:14.778 | INFO | } | 20260506 22:42:14.778 | INFO | Perf report for the last 5.13683 seconds: | 20260506 22:42:14.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:42:14.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:14.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:14.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:14.778 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:42:14.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:14.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:14.778 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 22:42:14.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:14.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:14.779 | INFO | Uploaded 36GiB to GPU in 1.982s (18.1635GiB/s) | 20260506 22:42:14.779 | INFO | Downloaded 34GiB to GPU in 1.99956s (17.0038GiB/s) | 20260506 22:42:19.919 | INFO | Analyzing chunk frames = [5025280,5025792) | 20260506 22:42:19.919 | INFO | MemInfo: { | 20260506 22:42:19.919 | INFO | "gpu" : " 17 GB", | 20260506 22:42:19.919 | INFO | "huge" : "", | 20260506 22:42:19.919 | INFO | "huge_shared" : " 27 GB", | 20260506 22:42:19.919 | INFO | "malloc" : " 14 GB", | 20260506 22:42:19.919 | INFO | "pinned" : " 29 GB" | 20260506 22:42:19.919 | INFO | } | 20260506 22:42:19.919 | INFO | Perf report for the last 5.14081 seconds: | 20260506 22:42:19.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:42:19.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:19.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:19.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:19.919 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 22:42:19.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:19.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:19.919 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 22:42:19.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:19.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:19.919 | INFO | Uploaded 36GiB to GPU in 2.00413s (17.9629GiB/s) | 20260506 22:42:19.919 | INFO | Downloaded 34GiB to GPU in 2.00261s (16.9779GiB/s) | 20260506 22:42:20.018 | INFO | Starting a round of Baz IO | 20260506 22:42:23.169 | INFO | Populating RTMetrics report | 20260506 22:42:23.170 | INFO | RTMetrics report populated | 20260506 22:42:23.170 | INFO | Opening tmp RTMetrics json file | 20260506 22:42:23.170 | INFO | Serializing RTMetrics to json | 20260506 22:42:23.171 | INFO | Serialization done, swapping tmp file now... | 20260506 22:42:23.172 | INFO | RTMetrics done | 20260506 22:42:23.265 | INFO | Finished a round of Baz IO: Wrote 1.907 GiB in 3.247 seconds | 20260506 22:42:23.265 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.62% / 0.00% / 66.38% | 20260506 22:42:23.265 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.483255% / 0% / 5.5631e-05% | 20260506 22:42:23.265 | INFO | Baz IO overall: | 20260506 22:42:23.265 | INFO | BytesWritten : 2638028348650 | 20260506 22:42:23.265 | INFO | NumEvents : 1272589064039 | 20260506 22:42:25.057 | INFO | Analyzing chunk frames = [5025792,5026304) | 20260506 22:42:25.057 | INFO | MemInfo: { | 20260506 22:42:25.057 | INFO | "gpu" : " 17 GB", | 20260506 22:42:25.057 | INFO | "huge" : "", | 20260506 22:42:25.057 | INFO | "huge_shared" : " 27 GB", | 20260506 22:42:25.057 | INFO | "malloc" : " 14 GB", | 20260506 22:42:25.057 | INFO | "pinned" : " 29 GB" | 20260506 22:42:25.057 | INFO | } | 20260506 22:42:25.057 | INFO | Perf report for the last 5.13821 seconds: | 20260506 22:42:25.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:42:25.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:25.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:25.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:25.057 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260506 22:42:25.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:25.057 | INFO | RT_METRICS : 46.1%, 1.0, 24.2ms, 0 | 20260506 22:42:25.057 | INFO | PRE_HQ : 60.6%, 2.0, 61.7ms, 0 | 20260506 22:42:25.058 | INFO | BAZWRITER : 62.7%, 3.6, 116.0ms, 0 | 20260506 22:42:25.058 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:42:25.058 | INFO | Uploaded 36GiB to GPU in 2.02256s (17.7992GiB/s) | 20260506 22:42:25.058 | INFO | Downloaded 38GiB to GPU in 2.18446s (17.3956GiB/s) | 20260506 22:42:30.191 | INFO | Analyzing chunk frames = [5026304,5026816) | 20260506 22:42:30.191 | INFO | MemInfo: { | 20260506 22:42:30.191 | INFO | "gpu" : " 17 GB", | 20260506 22:42:30.191 | INFO | "huge" : "", | 20260506 22:42:30.191 | INFO | "huge_shared" : " 27 GB", | 20260506 22:42:30.191 | INFO | "malloc" : " 14 GB", | 20260506 22:42:30.191 | INFO | "pinned" : " 29 GB" | 20260506 22:42:30.191 | INFO | } | 20260506 22:42:30.191 | INFO | Perf report for the last 5.1336 seconds: | 20260506 22:42:30.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:42:30.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:30.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:30.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:30.191 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 22:42:30.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:30.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:30.191 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 22:42:30.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:30.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:30.191 | INFO | Uploaded 36GiB to GPU in 1.98336s (18.151GiB/s) | 20260506 22:42:30.191 | INFO | Downloaded 34GiB to GPU in 1.93043s (17.6127GiB/s) | 20260506 22:42:35.329 | INFO | Analyzing chunk frames = [5026816,5027328) | 20260506 22:42:35.329 | INFO | MemInfo: { | 20260506 22:42:35.329 | INFO | "gpu" : " 17 GB", | 20260506 22:42:35.329 | INFO | "huge" : "", | 20260506 22:42:35.329 | INFO | "huge_shared" : " 27 GB", | 20260506 22:42:35.329 | INFO | "malloc" : " 14 GB", | 20260506 22:42:35.329 | INFO | "pinned" : " 29 GB" | 20260506 22:42:35.329 | INFO | } | 20260506 22:42:35.329 | INFO | Perf report for the last 5.1384 seconds: | 20260506 22:42:35.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:42:35.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:35.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:35.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:35.329 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 22:42:35.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:35.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:35.330 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:42:35.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:35.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:35.330 | INFO | Uploaded 36GiB to GPU in 1.9646s (18.3243GiB/s) | 20260506 22:42:35.330 | INFO | Downloaded 34GiB to GPU in 1.92382s (17.6732GiB/s) | 20260506 22:42:40.470 | INFO | Analyzing chunk frames = [5027328,5027840) | 20260506 22:42:40.470 | INFO | MemInfo: { | 20260506 22:42:40.470 | INFO | "gpu" : " 17 GB", | 20260506 22:42:40.470 | INFO | "huge" : "", | 20260506 22:42:40.471 | INFO | "huge_shared" : " 27 GB", | 20260506 22:42:40.471 | INFO | "malloc" : " 14 GB", | 20260506 22:42:40.471 | INFO | "pinned" : " 29 GB" | 20260506 22:42:40.471 | INFO | } | 20260506 22:42:40.471 | INFO | Perf report for the last 5.14131 seconds: | 20260506 22:42:40.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:42:40.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:40.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:40.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:40.471 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 22:42:40.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:40.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:40.471 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:42:40.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:40.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:40.471 | INFO | Uploaded 36GiB to GPU in 1.91842s (18.7655GiB/s) | 20260506 22:42:40.471 | INFO | Downloaded 34GiB to GPU in 1.87432s (18.1399GiB/s) | 20260506 22:42:45.615 | INFO | Analyzing chunk frames = [5027840,5028352) | 20260506 22:42:45.615 | INFO | MemInfo: { | 20260506 22:42:45.615 | INFO | "gpu" : " 17 GB", | 20260506 22:42:45.615 | INFO | "huge" : "", | 20260506 22:42:45.615 | INFO | "huge_shared" : " 27 GB", | 20260506 22:42:45.615 | INFO | "malloc" : " 14 GB", | 20260506 22:42:45.615 | INFO | "pinned" : " 29 GB" | 20260506 22:42:45.615 | INFO | } | 20260506 22:42:45.615 | INFO | Perf report for the last 5.14407 seconds: | 20260506 22:42:45.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:42:45.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:45.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:45.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:45.615 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 22:42:45.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:45.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:45.615 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:42:45.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:45.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:45.615 | INFO | Uploaded 36GiB to GPU in 1.98896s (18.0999GiB/s) | 20260506 22:42:45.615 | INFO | Downloaded 34GiB to GPU in 1.92287s (17.6819GiB/s) | 20260506 22:42:50.753 | INFO | Analyzing chunk frames = [5028352,5028864) | 20260506 22:42:50.753 | INFO | MemInfo: { | 20260506 22:42:50.753 | INFO | "gpu" : " 17 GB", | 20260506 22:42:50.753 | INFO | "huge" : "", | 20260506 22:42:50.753 | INFO | "huge_shared" : " 27 GB", | 20260506 22:42:50.754 | INFO | "malloc" : " 14 GB", | 20260506 22:42:50.754 | INFO | "pinned" : " 29 GB" | 20260506 22:42:50.754 | INFO | } | 20260506 22:42:50.754 | INFO | Perf report for the last 5.13879 seconds: | 20260506 22:42:50.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:42:50.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:50.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:50.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:50.754 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 22:42:50.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:50.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:50.754 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:42:50.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:50.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:50.754 | INFO | Uploaded 36GiB to GPU in 2.00825s (17.9261GiB/s) | 20260506 22:42:50.754 | INFO | Downloaded 34GiB to GPU in 1.94281s (17.5005GiB/s) | 20260506 22:42:55.902 | INFO | Analyzing chunk frames = [5028864,5029376) | 20260506 22:42:55.902 | INFO | MemInfo: { | 20260506 22:42:55.902 | INFO | "gpu" : " 17 GB", | 20260506 22:42:55.902 | INFO | "huge" : "", | 20260506 22:42:55.902 | INFO | "huge_shared" : " 27 GB", | 20260506 22:42:55.902 | INFO | "malloc" : " 14 GB", | 20260506 22:42:55.902 | INFO | "pinned" : " 29 GB" | 20260506 22:42:55.902 | INFO | } | 20260506 22:42:55.902 | INFO | Perf report for the last 5.1484 seconds: | 20260506 22:42:55.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:42:55.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:55.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:55.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:55.902 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 22:42:55.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:55.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:55.902 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:42:55.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:55.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:42:55.902 | INFO | Uploaded 36GiB to GPU in 1.97065s (18.2681GiB/s) | 20260506 22:42:55.902 | INFO | Downloaded 34GiB to GPU in 1.98696s (17.1115GiB/s) | 20260506 22:43:01.035 | INFO | Analyzing chunk frames = [5029376,5029888) | 20260506 22:43:01.035 | INFO | MemInfo: { | 20260506 22:43:01.035 | INFO | "gpu" : " 17 GB", | 20260506 22:43:01.035 | INFO | "huge" : "", | 20260506 22:43:01.035 | INFO | "huge_shared" : " 27 GB", | 20260506 22:43:01.035 | INFO | "malloc" : " 14 GB", | 20260506 22:43:01.035 | INFO | "pinned" : " 29 GB" | 20260506 22:43:01.035 | INFO | } | 20260506 22:43:01.035 | INFO | Perf report for the last 5.13342 seconds: | 20260506 22:43:01.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:43:01.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:01.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:01.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:01.036 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260506 22:43:01.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:01.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:01.036 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 22:43:01.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:01.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:01.036 | INFO | Uploaded 36GiB to GPU in 2.011s (17.9015GiB/s) | 20260506 22:43:01.036 | INFO | Downloaded 34GiB to GPU in 1.98354s (17.1411GiB/s) | 20260506 22:43:01.036 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:43:01.036 | INFO | Total mem usage: 18 GiB | 20260506 22:43:01.036 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:43:01.036 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:43:01.036 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:43:01.036 | INFO | High Water Breakdown: | 20260506 22:43:01.036 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:43:01.036 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:43:01.036 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:43:01.036 | INFO | ------------------------------------------ | 20260506 22:43:01.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:43:01.036 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:43:01.036 | INFO | Total mem usage: 26.5 GiB | 20260506 22:43:01.036 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:43:01.036 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:43:01.036 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:43:01.036 | INFO | High Water Breakdown: | 20260506 22:43:01.036 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:43:01.036 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:43:01.036 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:43:01.036 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:43:01.036 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:43:01.036 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:43:01.036 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:43:01.036 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:43:01.036 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:43:01.036 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:43:01.037 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:43:01.037 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:43:01.037 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:43:01.037 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:43:01.037 | INFO | ------------------------------------------ | 20260506 22:43:01.037 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:43:01.037 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:43:01.037 | INFO | Total mem usage: 25.6 GiB | 20260506 22:43:01.037 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:43:01.037 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:43:01.037 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:43:01.037 | INFO | High Water Breakdown: | 20260506 22:43:01.037 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:43:01.037 | INFO | ------------------------------------------ | 20260506 22:43:01.037 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:43:01.037 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:43:01.037 | INFO | Total mem usage: 15.7 GiB | 20260506 22:43:01.037 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:43:01.037 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:43:01.037 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:43:01.037 | INFO | High Water Breakdown: | 20260506 22:43:01.037 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:43:01.037 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:43:01.037 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:43:01.037 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:43:01.037 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:43:01.037 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:43:01.037 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:43:01.037 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:43:01.037 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:43:01.037 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:43:01.037 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:43:01.037 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:43:01.037 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:43:01.037 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:43:01.037 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:43:01.037 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:43:01.037 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:43:01.037 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:43:01.037 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:43:01.037 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:43:01.037 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:43:01.037 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:43:01.037 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:43:01.037 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:43:01.037 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:43:01.037 | INFO | ------------------------------------------ | 20260506 22:43:01.037 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:43:01.037 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:43:01.037 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:43:01.037 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:43:01.037 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.61e+01, 1.34e+01 / 2.48e+00 | 20260506 22:43:01.037 | INFO | PerfReport: 0.30%, AggregateStats -- 9.16e-02 / 1.02e-01, 9.82e-03 / 3.88e-03 | 20260506 22:43:01.037 | INFO | PerfReport: 11.65%, Baseline -- 3.62e+00 / 4.48e+00, 5.43e-01 / 2.32e-01 | 20260506 22:43:01.037 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.50e+00, 1.80e-01 / 8.96e-02 | 20260506 22:43:01.037 | INFO | PerfReport: 16.09%, DME -- 4.00e+01 / 4.51e+01, 4.12e+00 / 2.59e+00 | 20260506 22:43:01.037 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.34e+00 / 1.45e+00 | 20260506 22:43:01.037 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.20e+00 / 2.77e+00, 3.81e-01 / 2.09e-01 | 20260506 22:43:01.037 | INFO | PerfReport: 3.23%, Metrics -- 1.00e+00 / 1.36e+00, 2.64e-01 / 1.25e-01 | 20260506 22:43:01.037 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.42e-01 / 3.96e-01, 5.27e-02 / 1.33e-02 | 20260506 22:43:01.037 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:43:01.037 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:43:01.037 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:43:01.037 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 22:43:01.037 | INFO | PerfReport: 21.43%, Upload -- 2.07e+01 / 0.00e+00, 6.85e+00 / 0.00e+00 | 20260506 22:43:01.037 | INFO | PerfReport: 21.25%, Download -- 2.05e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 22:43:01.037 | INFO | PerfReport: 32.12%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 22:43:01.037 | INFO | PerfReport: 1.20%, LockUpload -- 1.16e+00 / 0.00e+00, 4.72e+00 / 0.00e+00 | 20260506 22:43:01.037 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.14e+00 / 0.00e+00 | 20260506 22:43:01.037 | INFO | PerfReport: 22.67%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 22:43:01.037 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.13e-02 / 0.00e+00 | 20260506 22:43:01.143 | INFO | Starting a round of Baz IO | 20260506 22:43:04.268 | INFO | Populating RTMetrics report | 20260506 22:43:04.268 | INFO | RTMetrics report populated | 20260506 22:43:04.268 | INFO | Opening tmp RTMetrics json file | 20260506 22:43:04.287 | INFO | Serializing RTMetrics to json | 20260506 22:43:04.289 | INFO | Serialization done, swapping tmp file now... | 20260506 22:43:04.290 | INFO | RTMetrics done | 20260506 22:43:04.362 | INFO | Finished a round of Baz IO: Wrote 1.906 GiB in 3.218 seconds | 20260506 22:43:04.362 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.62% / 0.00% / 66.38% | 20260506 22:43:04.362 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.481089% / 3.97364e-06% / 9.93411e-05% | 20260506 22:43:04.362 | INFO | Baz IO overall: | 20260506 22:43:04.362 | INFO | BytesWritten : 2640074662179 | 20260506 22:43:04.362 | INFO | NumEvents : 1273566166726 | 20260506 22:43:06.174 | INFO | Analyzing chunk frames = [5029888,5030400) | 20260506 22:43:06.174 | INFO | MemInfo: { | 20260506 22:43:06.174 | INFO | "gpu" : " 17 GB", | 20260506 22:43:06.174 | INFO | "huge" : "", | 20260506 22:43:06.174 | INFO | "huge_shared" : " 27 GB", | 20260506 22:43:06.174 | INFO | "malloc" : " 14 GB", | 20260506 22:43:06.174 | INFO | "pinned" : " 29 GB" | 20260506 22:43:06.174 | INFO | } | 20260506 22:43:06.174 | INFO | Perf report for the last 5.13887 seconds: | 20260506 22:43:06.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:43:06.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:06.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:06.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:06.175 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 22:43:06.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:06.175 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 22:43:06.175 | INFO | PRE_HQ : 60.1%, 1.9, 58.1ms, 0 | 20260506 22:43:06.175 | INFO | BAZWRITER : 62.1%, 3.4, 109.8ms, 0 | 20260506 22:43:06.175 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 22:43:06.175 | INFO | Uploaded 36GiB to GPU in 2.04864s (17.5726GiB/s) | 20260506 22:43:06.175 | INFO | Downloaded 38GiB to GPU in 2.18508s (17.3907GiB/s) | 20260506 22:43:11.308 | INFO | Analyzing chunk frames = [5030400,5030912) | 20260506 22:43:11.308 | INFO | MemInfo: { | 20260506 22:43:11.308 | INFO | "gpu" : " 17 GB", | 20260506 22:43:11.308 | INFO | "huge" : "", | 20260506 22:43:11.309 | INFO | "huge_shared" : " 27 GB", | 20260506 22:43:11.309 | INFO | "malloc" : " 14 GB", | 20260506 22:43:11.309 | INFO | "pinned" : " 29 GB" | 20260506 22:43:11.309 | INFO | } | 20260506 22:43:11.309 | INFO | Perf report for the last 5.1343 seconds: | 20260506 22:43:11.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:43:11.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:11.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:11.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:11.309 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 22:43:11.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:11.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:11.309 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:43:11.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:11.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:11.309 | INFO | Uploaded 36GiB to GPU in 2.00005s (17.9995GiB/s) | 20260506 22:43:11.309 | INFO | Downloaded 34GiB to GPU in 1.93077s (17.6096GiB/s) | 20260506 22:43:16.443 | INFO | Analyzing chunk frames = [5030912,5031424) | 20260506 22:43:16.443 | INFO | MemInfo: { | 20260506 22:43:16.443 | INFO | "gpu" : " 17 GB", | 20260506 22:43:16.443 | INFO | "huge" : "", | 20260506 22:43:16.443 | INFO | "huge_shared" : " 27 GB", | 20260506 22:43:16.443 | INFO | "malloc" : " 14 GB", | 20260506 22:43:16.443 | INFO | "pinned" : " 29 GB" | 20260506 22:43:16.443 | INFO | } | 20260506 22:43:16.443 | INFO | Perf report for the last 5.13472 seconds: | 20260506 22:43:16.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:43:16.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:16.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:16.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:16.443 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 22:43:16.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:16.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:16.444 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:43:16.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:16.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:16.444 | INFO | Uploaded 36GiB to GPU in 1.97399s (18.2372GiB/s) | 20260506 22:43:16.444 | INFO | Downloaded 34GiB to GPU in 1.92851s (17.6302GiB/s) | 20260506 22:43:21.587 | INFO | Analyzing chunk frames = [5031424,5031936) | 20260506 22:43:21.587 | INFO | MemInfo: { | 20260506 22:43:21.587 | INFO | "gpu" : " 17 GB", | 20260506 22:43:21.587 | INFO | "huge" : "", | 20260506 22:43:21.587 | INFO | "huge_shared" : " 27 GB", | 20260506 22:43:21.587 | INFO | "malloc" : " 14 GB", | 20260506 22:43:21.587 | INFO | "pinned" : " 29 GB" | 20260506 22:43:21.587 | INFO | } | 20260506 22:43:21.587 | INFO | Perf report for the last 5.14402 seconds: | 20260506 22:43:21.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:43:21.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:21.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:21.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:21.587 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 22:43:21.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:21.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:21.588 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:43:21.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:21.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:21.588 | INFO | Uploaded 36GiB to GPU in 1.93619s (18.5933GiB/s) | 20260506 22:43:21.588 | INFO | Downloaded 34GiB to GPU in 1.8735s (18.1478GiB/s) | 20260506 22:43:26.732 | INFO | Analyzing chunk frames = [5031936,5032448) | 20260506 22:43:26.732 | INFO | MemInfo: { | 20260506 22:43:26.732 | INFO | "gpu" : " 17 GB", | 20260506 22:43:26.732 | INFO | "huge" : "", | 20260506 22:43:26.732 | INFO | "huge_shared" : " 27 GB", | 20260506 22:43:26.732 | INFO | "malloc" : " 14 GB", | 20260506 22:43:26.732 | INFO | "pinned" : " 29 GB" | 20260506 22:43:26.732 | INFO | } | 20260506 22:43:26.732 | INFO | Perf report for the last 5.14454 seconds: | 20260506 22:43:26.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:43:26.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:26.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:26.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:26.732 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 22:43:26.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:26.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:26.732 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:43:26.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:26.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:26.732 | INFO | Uploaded 36GiB to GPU in 1.99286s (18.0645GiB/s) | 20260506 22:43:26.732 | INFO | Downloaded 34GiB to GPU in 1.91904s (17.7172GiB/s) | 20260506 22:43:31.867 | INFO | Analyzing chunk frames = [5032448,5032960) | 20260506 22:43:31.867 | INFO | MemInfo: { | 20260506 22:43:31.867 | INFO | "gpu" : " 17 GB", | 20260506 22:43:31.867 | INFO | "huge" : "", | 20260506 22:43:31.867 | INFO | "huge_shared" : " 27 GB", | 20260506 22:43:31.867 | INFO | "malloc" : " 14 GB", | 20260506 22:43:31.867 | INFO | "pinned" : " 29 GB" | 20260506 22:43:31.867 | INFO | } | 20260506 22:43:31.867 | INFO | Perf report for the last 5.13478 seconds: | 20260506 22:43:31.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:43:31.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:31.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:31.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:31.867 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 22:43:31.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:31.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:31.867 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:43:31.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:31.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:31.867 | INFO | Uploaded 36GiB to GPU in 2.01562s (17.8605GiB/s) | 20260506 22:43:31.867 | INFO | Downloaded 34GiB to GPU in 1.94676s (17.4649GiB/s) | 20260506 22:43:37.007 | INFO | Analyzing chunk frames = [5032960,5033472) | 20260506 22:43:37.007 | INFO | MemInfo: { | 20260506 22:43:37.007 | INFO | "gpu" : " 17 GB", | 20260506 22:43:37.008 | INFO | "huge" : "", | 20260506 22:43:37.008 | INFO | "huge_shared" : " 27 GB", | 20260506 22:43:37.008 | INFO | "malloc" : " 14 GB", | 20260506 22:43:37.008 | INFO | "pinned" : " 29 GB" | 20260506 22:43:37.008 | INFO | } | 20260506 22:43:37.008 | INFO | Perf report for the last 5.14096 seconds: | 20260506 22:43:37.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:43:37.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:37.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:37.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:37.008 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:43:37.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:37.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:37.008 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:43:37.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:37.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:37.008 | INFO | Uploaded 36GiB to GPU in 1.98023s (18.1797GiB/s) | 20260506 22:43:37.008 | INFO | Downloaded 34GiB to GPU in 2.00293s (16.9752GiB/s) | 20260506 22:43:42.148 | INFO | Analyzing chunk frames = [5033472,5033984) | 20260506 22:43:42.148 | INFO | MemInfo: { | 20260506 22:43:42.148 | INFO | "gpu" : " 17 GB", | 20260506 22:43:42.148 | INFO | "huge" : "", | 20260506 22:43:42.148 | INFO | "huge_shared" : " 27 GB", | 20260506 22:43:42.148 | INFO | "malloc" : " 14 GB", | 20260506 22:43:42.148 | INFO | "pinned" : " 29 GB" | 20260506 22:43:42.148 | INFO | } | 20260506 22:43:42.148 | INFO | Perf report for the last 5.14044 seconds: | 20260506 22:43:42.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:43:42.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:42.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:42.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:42.148 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 22:43:42.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:42.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:42.148 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 22:43:42.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:42.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:42.149 | INFO | Uploaded 36GiB to GPU in 2.02555s (17.7729GiB/s) | 20260506 22:43:42.149 | INFO | Downloaded 34GiB to GPU in 1.98695s (17.1116GiB/s) | 20260506 22:43:42.247 | INFO | Starting a round of Baz IO | 20260506 22:43:45.385 | INFO | Populating RTMetrics report | 20260506 22:43:45.386 | INFO | RTMetrics report populated | 20260506 22:43:45.386 | INFO | Opening tmp RTMetrics json file | 20260506 22:43:45.386 | INFO | Serializing RTMetrics to json | 20260506 22:43:45.387 | INFO | Serialization done, swapping tmp file now... | 20260506 22:43:45.388 | INFO | RTMetrics done | 20260506 22:43:45.476 | INFO | Finished a round of Baz IO: Wrote 1.904 GiB in 3.228 seconds | 20260506 22:43:45.476 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.62% / 0.00% / 66.38% | 20260506 22:43:45.476 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.479273% / 3.97364e-06% / 5.5631e-05% | 20260506 22:43:45.476 | INFO | Baz IO overall: | 20260506 22:43:45.476 | INFO | BytesWritten : 2642118902262 | 20260506 22:43:45.476 | INFO | NumEvents : 1274541599392 | 20260506 22:43:47.283 | INFO | Analyzing chunk frames = [5033984,5034496) | 20260506 22:43:47.283 | INFO | MemInfo: { | 20260506 22:43:47.283 | INFO | "gpu" : " 17 GB", | 20260506 22:43:47.283 | INFO | "huge" : "", | 20260506 22:43:47.283 | INFO | "huge_shared" : " 27 GB", | 20260506 22:43:47.283 | INFO | "malloc" : " 14 GB", | 20260506 22:43:47.283 | INFO | "pinned" : " 29 GB" | 20260506 22:43:47.283 | INFO | } | 20260506 22:43:47.283 | INFO | Perf report for the last 5.13509 seconds: | 20260506 22:43:47.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:43:47.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:47.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:47.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:47.283 | INFO | ANALYSIS : 62.5%, 2.9, 96.5ms, 0 | 20260506 22:43:47.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:47.283 | INFO | RT_METRICS : 44.2%, 1.0, 23.1ms, 0 | 20260506 22:43:47.283 | INFO | PRE_HQ : 60.4%, 1.9, 59.1ms, 0 | 20260506 22:43:47.283 | INFO | BAZWRITER : 62.5%, 3.5, 112.1ms, 0 | 20260506 22:43:47.283 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 22:43:47.284 | INFO | Uploaded 36GiB to GPU in 2.041s (17.6384GiB/s) | 20260506 22:43:47.284 | INFO | Downloaded 38GiB to GPU in 2.17218s (17.4939GiB/s) | 20260506 22:43:52.418 | INFO | Analyzing chunk frames = [5034496,5035008) | 20260506 22:43:52.418 | INFO | MemInfo: { | 20260506 22:43:52.418 | INFO | "gpu" : " 17 GB", | 20260506 22:43:52.418 | INFO | "huge" : "", | 20260506 22:43:52.418 | INFO | "huge_shared" : " 27 GB", | 20260506 22:43:52.418 | INFO | "malloc" : " 14 GB", | 20260506 22:43:52.418 | INFO | "pinned" : " 29 GB" | 20260506 22:43:52.418 | INFO | } | 20260506 22:43:52.418 | INFO | Perf report for the last 5.13496 seconds: | 20260506 22:43:52.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:43:52.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:52.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:52.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:52.418 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 22:43:52.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:52.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:52.418 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 22:43:52.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:52.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:52.419 | INFO | Uploaded 36GiB to GPU in 2.0207s (17.8156GiB/s) | 20260506 22:43:52.419 | INFO | Downloaded 34GiB to GPU in 1.93408s (17.5794GiB/s) | 20260506 22:43:57.562 | INFO | Analyzing chunk frames = [5035008,5035520) | 20260506 22:43:57.562 | INFO | MemInfo: { | 20260506 22:43:57.562 | INFO | "gpu" : " 17 GB", | 20260506 22:43:57.562 | INFO | "huge" : "", | 20260506 22:43:57.562 | INFO | "huge_shared" : " 27 GB", | 20260506 22:43:57.562 | INFO | "malloc" : " 14 GB", | 20260506 22:43:57.562 | INFO | "pinned" : " 29 GB" | 20260506 22:43:57.562 | INFO | } | 20260506 22:43:57.562 | INFO | Perf report for the last 5.14355 seconds: | 20260506 22:43:57.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:43:57.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:57.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:57.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:57.562 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 22:43:57.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:57.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:57.562 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:43:57.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:57.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:43:57.562 | INFO | Uploaded 36GiB to GPU in 1.97608s (18.2179GiB/s) | 20260506 22:43:57.562 | INFO | Downloaded 34GiB to GPU in 1.91637s (17.7419GiB/s) | 20260506 22:44:02.710 | INFO | Analyzing chunk frames = [5035520,5036032) | 20260506 22:44:02.710 | INFO | MemInfo: { | 20260506 22:44:02.710 | INFO | "gpu" : " 17 GB", | 20260506 22:44:02.710 | INFO | "huge" : "", | 20260506 22:44:02.710 | INFO | "huge_shared" : " 27 GB", | 20260506 22:44:02.710 | INFO | "malloc" : " 14 GB", | 20260506 22:44:02.710 | INFO | "pinned" : " 29 GB" | 20260506 22:44:02.710 | INFO | } | 20260506 22:44:02.710 | INFO | Perf report for the last 5.14835 seconds: | 20260506 22:44:02.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:44:02.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:02.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:02.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:02.710 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 22:44:02.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:02.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:02.710 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:44:02.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:02.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:02.710 | INFO | Uploaded 36GiB to GPU in 1.96158s (18.3526GiB/s) | 20260506 22:44:02.711 | INFO | Downloaded 34GiB to GPU in 1.87778s (18.1065GiB/s) | 20260506 22:44:07.847 | INFO | Analyzing chunk frames = [5036032,5036544) | 20260506 22:44:07.847 | INFO | MemInfo: { | 20260506 22:44:07.847 | INFO | "gpu" : " 17 GB", | 20260506 22:44:07.847 | INFO | "huge" : "", | 20260506 22:44:07.847 | INFO | "huge_shared" : " 27 GB", | 20260506 22:44:07.847 | INFO | "malloc" : " 14 GB", | 20260506 22:44:07.847 | INFO | "pinned" : " 29 GB" | 20260506 22:44:07.847 | INFO | } | 20260506 22:44:07.847 | INFO | Perf report for the last 5.1366 seconds: | 20260506 22:44:07.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:44:07.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:07.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:07.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:07.847 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 22:44:07.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:07.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:07.847 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:44:07.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:07.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:07.847 | INFO | Uploaded 36GiB to GPU in 2.00034s (17.9969GiB/s) | 20260506 22:44:07.847 | INFO | Downloaded 34GiB to GPU in 1.93235s (17.5952GiB/s) | 20260506 22:44:12.990 | INFO | Analyzing chunk frames = [5036544,5037056) | 20260506 22:44:12.990 | INFO | MemInfo: { | 20260506 22:44:12.990 | INFO | "gpu" : " 17 GB", | 20260506 22:44:12.990 | INFO | "huge" : "", | 20260506 22:44:12.990 | INFO | "huge_shared" : " 27 GB", | 20260506 22:44:12.990 | INFO | "malloc" : " 14 GB", | 20260506 22:44:12.990 | INFO | "pinned" : " 29 GB" | 20260506 22:44:12.990 | INFO | } | 20260506 22:44:12.990 | INFO | Perf report for the last 5.14308 seconds: | 20260506 22:44:12.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:44:12.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:12.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:12.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:12.990 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 22:44:12.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:12.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:12.990 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:44:12.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:12.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:12.990 | INFO | Uploaded 36GiB to GPU in 2.02934s (17.7397GiB/s) | 20260506 22:44:12.990 | INFO | Downloaded 34GiB to GPU in 1.9416s (17.5113GiB/s) | 20260506 22:44:18.121 | INFO | Analyzing chunk frames = [5037056,5037568) | 20260506 22:44:18.121 | INFO | MemInfo: { | 20260506 22:44:18.121 | INFO | "gpu" : " 17 GB", | 20260506 22:44:18.121 | INFO | "huge" : "", | 20260506 22:44:18.121 | INFO | "huge_shared" : " 27 GB", | 20260506 22:44:18.121 | INFO | "malloc" : " 14 GB", | 20260506 22:44:18.121 | INFO | "pinned" : " 29 GB" | 20260506 22:44:18.121 | INFO | } | 20260506 22:44:18.121 | INFO | Perf report for the last 5.13085 seconds: | 20260506 22:44:18.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:44:18.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:18.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:18.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:18.121 | INFO | ANALYSIS : 61.4%, 3.0, 95.3ms, 0 | 20260506 22:44:18.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:18.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:18.121 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:44:18.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:18.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:18.121 | INFO | Uploaded 36GiB to GPU in 1.97218s (18.2539GiB/s) | 20260506 22:44:18.121 | INFO | Downloaded 34GiB to GPU in 1.99964s (17.003GiB/s) | 20260506 22:44:23.258 | INFO | Analyzing chunk frames = [5037568,5038080) | 20260506 22:44:23.258 | INFO | MemInfo: { | 20260506 22:44:23.258 | INFO | "gpu" : " 17 GB", | 20260506 22:44:23.258 | INFO | "huge" : "", | 20260506 22:44:23.258 | INFO | "huge_shared" : " 27 GB", | 20260506 22:44:23.258 | INFO | "malloc" : " 14 GB", | 20260506 22:44:23.258 | INFO | "pinned" : " 29 GB" | 20260506 22:44:23.258 | INFO | } | 20260506 22:44:23.258 | INFO | Perf report for the last 5.13746 seconds: | 20260506 22:44:23.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:44:23.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:23.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:23.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:23.258 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:44:23.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:23.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:23.258 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 22:44:23.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:23.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:23.259 | INFO | Uploaded 36GiB to GPU in 1.9761s (18.2177GiB/s) | 20260506 22:44:23.259 | INFO | Downloaded 34GiB to GPU in 1.98606s (17.1193GiB/s) | 20260506 22:44:23.259 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:44:23.259 | INFO | Total mem usage: 18 GiB | 20260506 22:44:23.259 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:44:23.259 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -25.1%) | 20260506 22:44:23.259 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:44:23.259 | INFO | High Water Breakdown: | 20260506 22:44:23.259 | INFO | 60.3% : 8.68 GiB ( 76% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:44:23.259 | INFO | 39.6% : 5.69 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:44:23.259 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:44:23.259 | INFO | ------------------------------------------ | 20260506 22:44:23.259 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:44:23.259 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:44:23.259 | INFO | Total mem usage: 26.5 GiB | 20260506 22:44:23.259 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:44:23.259 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 22:44:23.259 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 22:44:23.259 | INFO | High Water Breakdown: | 20260506 22:44:23.259 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:44:23.259 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:44:23.259 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:44:23.259 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:44:23.259 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:44:23.259 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:44:23.259 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:44:23.259 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:44:23.259 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:44:23.259 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:44:23.259 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:44:23.259 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:44:23.259 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:44:23.259 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:44:23.259 | INFO | ------------------------------------------ | 20260506 22:44:23.259 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:44:23.259 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:44:23.259 | INFO | Total mem usage: 25.6 GiB | 20260506 22:44:23.259 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:44:23.259 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:44:23.259 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:44:23.259 | INFO | High Water Breakdown: | 20260506 22:44:23.259 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:44:23.259 | INFO | ------------------------------------------ | 20260506 22:44:23.259 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:44:23.259 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:44:23.259 | INFO | Total mem usage: 15.7 GiB | 20260506 22:44:23.259 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:44:23.259 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:44:23.259 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:44:23.259 | INFO | High Water Breakdown: | 20260506 22:44:23.259 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:44:23.259 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:44:23.259 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:44:23.259 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:44:23.259 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:44:23.259 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:44:23.259 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:44:23.259 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:44:23.259 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:44:23.259 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:44:23.259 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:44:23.259 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:44:23.259 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:44:23.259 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:44:23.259 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:44:23.259 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:44:23.259 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:44:23.259 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:44:23.259 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:44:23.260 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:44:23.260 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:44:23.260 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:44:23.260 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:44:23.260 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:44:23.260 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:44:23.260 | INFO | ------------------------------------------ | 20260506 22:44:23.260 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:44:23.260 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:44:23.260 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:44:23.260 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:44:23.260 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.60e+01, 1.32e+01 / 2.17e+00 | 20260506 22:44:23.260 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 9.94e-02, 9.71e-03 / 5.45e-03 | 20260506 22:44:23.260 | INFO | PerfReport: 11.65%, Baseline -- 3.61e+00 / 4.42e+00, 5.41e-01 / 3.57e-01 | 20260506 22:44:23.260 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.48e+00, 1.79e-01 / 1.27e-01 | 20260506 22:44:23.260 | INFO | PerfReport: 16.12%, DME -- 4.00e+01 / 4.51e+01, 4.24e+00 / 2.17e+00 | 20260506 22:44:23.260 | INFO | PerfReport: 57.25%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.29e+00 / 1.50e+00 | 20260506 22:44:23.260 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.20e+00 / 2.78e+00, 3.82e-01 / 1.94e-01 | 20260506 22:44:23.260 | INFO | PerfReport: 3.22%, Metrics -- 9.99e-01 / 1.35e+00, 2.60e-01 / 1.37e-01 | 20260506 22:44:23.260 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.41e-01 / 4.00e-01, 5.29e-02 / 2.87e-02 | 20260506 22:44:23.260 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:44:23.260 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:44:23.260 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:44:23.260 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 22:44:23.260 | INFO | PerfReport: 21.51%, Upload -- 2.08e+01 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260506 22:44:23.260 | INFO | PerfReport: 21.23%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 22:44:23.260 | INFO | PerfReport: 32.07%, Compute -- 3.10e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 22:44:23.260 | INFO | PerfReport: 1.33%, LockUpload -- 1.29e+00 / 0.00e+00, 4.87e+00 / 0.00e+00 | 20260506 22:44:23.260 | INFO | PerfReport: 1.25%, LockDownload -- 1.21e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260506 22:44:23.260 | INFO | PerfReport: 22.58%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 22:44:23.260 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260506 22:44:23.356 | INFO | Starting a round of Baz IO | 20260506 22:44:26.488 | INFO | Populating RTMetrics report | 20260506 22:44:26.489 | INFO | RTMetrics report populated | 20260506 22:44:26.489 | INFO | Opening tmp RTMetrics json file | 20260506 22:44:26.490 | INFO | Serializing RTMetrics to json | 20260506 22:44:26.491 | INFO | Serialization done, swapping tmp file now... | 20260506 22:44:26.491 | INFO | RTMetrics done | 20260506 22:44:26.585 | INFO | Finished a round of Baz IO: Wrote 1.905 GiB in 3.228 seconds | 20260506 22:44:26.585 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.61% / 0.00% / 66.38% | 20260506 22:44:26.585 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.479925% / 3.97364e-06% / 5.5631e-05% | 20260506 22:44:26.585 | INFO | Baz IO overall: | 20260506 22:44:26.585 | INFO | BytesWritten : 2644164560360 | 20260506 22:44:26.585 | INFO | NumEvents : 1275518320571 | 20260506 22:44:28.399 | INFO | Analyzing chunk frames = [5038080,5038592) | 20260506 22:44:28.399 | INFO | MemInfo: { | 20260506 22:44:28.399 | INFO | "gpu" : " 17 GB", | 20260506 22:44:28.399 | INFO | "huge" : "", | 20260506 22:44:28.399 | INFO | "huge_shared" : " 27 GB", | 20260506 22:44:28.399 | INFO | "malloc" : " 14 GB", | 20260506 22:44:28.399 | INFO | "pinned" : " 29 GB" | 20260506 22:44:28.399 | INFO | } | 20260506 22:44:28.399 | INFO | Perf report for the last 5.14123 seconds: | 20260506 22:44:28.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:44:28.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:28.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:28.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:28.400 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 22:44:28.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:28.400 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 22:44:28.400 | INFO | PRE_HQ : 59.5%, 1.8, 56.8ms, 0 | 20260506 22:44:28.400 | INFO | BAZWRITER : 62.3%, 3.5, 111.7ms, 0 | 20260506 22:44:28.400 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:44:28.400 | INFO | Uploaded 36GiB to GPU in 2.01618s (17.8555GiB/s) | 20260506 22:44:28.400 | INFO | Downloaded 38GiB to GPU in 2.18248s (17.4114GiB/s) | 20260506 22:44:33.536 | INFO | Analyzing chunk frames = [5038592,5039104) | 20260506 22:44:33.536 | INFO | MemInfo: { | 20260506 22:44:33.536 | INFO | "gpu" : " 17 GB", | 20260506 22:44:33.536 | INFO | "huge" : "", | 20260506 22:44:33.536 | INFO | "huge_shared" : " 27 GB", | 20260506 22:44:33.536 | INFO | "malloc" : " 14 GB", | 20260506 22:44:33.536 | INFO | "pinned" : " 29 GB" | 20260506 22:44:33.536 | INFO | } | 20260506 22:44:33.536 | INFO | Perf report for the last 5.13647 seconds: | 20260506 22:44:33.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:44:33.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:33.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:33.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:33.536 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:44:33.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:33.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:33.536 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:44:33.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:33.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:33.536 | INFO | Uploaded 36GiB to GPU in 1.99573s (18.0385GiB/s) | 20260506 22:44:33.536 | INFO | Downloaded 34GiB to GPU in 1.93319s (17.5875GiB/s) | 20260506 22:44:38.679 | INFO | Analyzing chunk frames = [5039104,5039616) | 20260506 22:44:38.679 | INFO | MemInfo: { | 20260506 22:44:38.679 | INFO | "gpu" : " 17 GB", | 20260506 22:44:38.679 | INFO | "huge" : "", | 20260506 22:44:38.679 | INFO | "huge_shared" : " 27 GB", | 20260506 22:44:38.679 | INFO | "malloc" : " 14 GB", | 20260506 22:44:38.679 | INFO | "pinned" : " 29 GB" | 20260506 22:44:38.679 | INFO | } | 20260506 22:44:38.679 | INFO | Perf report for the last 5.14285 seconds: | 20260506 22:44:38.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:44:38.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:38.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:38.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:38.679 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 22:44:38.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:38.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:38.679 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:44:38.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:38.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:38.679 | INFO | Uploaded 36GiB to GPU in 1.97582s (18.2202GiB/s) | 20260506 22:44:38.679 | INFO | Downloaded 34GiB to GPU in 1.92717s (17.6424GiB/s) | 20260506 22:44:43.820 | INFO | Analyzing chunk frames = [5039616,5040128) | 20260506 22:44:43.820 | INFO | MemInfo: { | 20260506 22:44:43.820 | INFO | "gpu" : " 17 GB", | 20260506 22:44:43.820 | INFO | "huge" : "", | 20260506 22:44:43.820 | INFO | "huge_shared" : " 27 GB", | 20260506 22:44:43.820 | INFO | "malloc" : " 14 GB", | 20260506 22:44:43.820 | INFO | "pinned" : " 29 GB" | 20260506 22:44:43.820 | INFO | } | 20260506 22:44:43.820 | INFO | Perf report for the last 5.14135 seconds: | 20260506 22:44:43.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:44:43.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:43.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:43.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:43.820 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 22:44:43.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:43.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:43.820 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:44:43.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:43.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:43.820 | INFO | Uploaded 36GiB to GPU in 1.93833s (18.5727GiB/s) | 20260506 22:44:43.821 | INFO | Downloaded 34GiB to GPU in 1.88111s (18.0744GiB/s) | 20260506 22:44:48.963 | INFO | Analyzing chunk frames = [5040128,5040640) | 20260506 22:44:48.963 | INFO | MemInfo: { | 20260506 22:44:48.963 | INFO | "gpu" : " 17 GB", | 20260506 22:44:48.963 | INFO | "huge" : "", | 20260506 22:44:48.963 | INFO | "huge_shared" : " 27 GB", | 20260506 22:44:48.963 | INFO | "malloc" : " 14 GB", | 20260506 22:44:48.963 | INFO | "pinned" : " 29 GB" | 20260506 22:44:48.963 | INFO | } | 20260506 22:44:48.963 | INFO | Perf report for the last 5.14261 seconds: | 20260506 22:44:48.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:44:48.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:48.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:48.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:48.963 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 22:44:48.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:48.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:48.963 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:44:48.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:48.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:48.963 | INFO | Uploaded 36GiB to GPU in 2.00064s (17.9942GiB/s) | 20260506 22:44:48.963 | INFO | Downloaded 34GiB to GPU in 1.93366s (17.5832GiB/s) | 20260506 22:44:54.096 | INFO | Analyzing chunk frames = [5040640,5041152) | 20260506 22:44:54.096 | INFO | MemInfo: { | 20260506 22:44:54.096 | INFO | "gpu" : " 17 GB", | 20260506 22:44:54.096 | INFO | "huge" : "", | 20260506 22:44:54.096 | INFO | "huge_shared" : " 27 GB", | 20260506 22:44:54.096 | INFO | "malloc" : " 14 GB", | 20260506 22:44:54.096 | INFO | "pinned" : " 29 GB" | 20260506 22:44:54.096 | INFO | } | 20260506 22:44:54.096 | INFO | Perf report for the last 5.13344 seconds: | 20260506 22:44:54.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:44:54.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:54.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:54.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:54.096 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 22:44:54.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:54.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:54.096 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:44:54.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:54.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:54.097 | INFO | Uploaded 36GiB to GPU in 2.01022s (17.9085GiB/s) | 20260506 22:44:54.097 | INFO | Downloaded 34GiB to GPU in 1.9454s (17.4771GiB/s) | 20260506 22:44:59.233 | INFO | Analyzing chunk frames = [5041152,5041664) | 20260506 22:44:59.233 | INFO | MemInfo: { | 20260506 22:44:59.233 | INFO | "gpu" : " 17 GB", | 20260506 22:44:59.233 | INFO | "huge" : "", | 20260506 22:44:59.233 | INFO | "huge_shared" : " 27 GB", | 20260506 22:44:59.233 | INFO | "malloc" : " 14 GB", | 20260506 22:44:59.233 | INFO | "pinned" : " 29 GB" | 20260506 22:44:59.233 | INFO | } | 20260506 22:44:59.233 | INFO | Perf report for the last 5.13675 seconds: | 20260506 22:44:59.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:44:59.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:59.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:59.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:59.233 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:44:59.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:59.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:59.233 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 22:44:59.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:59.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:44:59.233 | INFO | Uploaded 36GiB to GPU in 1.95879s (18.3787GiB/s) | 20260506 22:44:59.233 | INFO | Downloaded 34GiB to GPU in 1.9769s (17.1987GiB/s) | 20260506 22:45:04.372 | INFO | Analyzing chunk frames = [5041664,5042176) | 20260506 22:45:04.372 | INFO | MemInfo: { | 20260506 22:45:04.372 | INFO | "gpu" : " 17 GB", | 20260506 22:45:04.372 | INFO | "huge" : "", | 20260506 22:45:04.372 | INFO | "huge_shared" : " 27 GB", | 20260506 22:45:04.373 | INFO | "malloc" : " 14 GB", | 20260506 22:45:04.373 | INFO | "pinned" : " 29 GB" | 20260506 22:45:04.373 | INFO | } | 20260506 22:45:04.373 | INFO | Perf report for the last 5.13964 seconds: | 20260506 22:45:04.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:45:04.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:04.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:04.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:04.373 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 22:45:04.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:04.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:04.373 | INFO | PRE_HQ : 29.7%, 1.0, 15.5ms, 0 | 20260506 22:45:04.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:04.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:04.373 | INFO | Uploaded 36GiB to GPU in 2.0111s (17.9007GiB/s) | 20260506 22:45:04.373 | INFO | Downloaded 34GiB to GPU in 2.00163s (16.9861GiB/s) | 20260506 22:45:04.471 | INFO | Starting a round of Baz IO | 20260506 22:45:07.603 | INFO | Populating RTMetrics report | 20260506 22:45:07.603 | INFO | RTMetrics report populated | 20260506 22:45:07.603 | INFO | Opening tmp RTMetrics json file | 20260506 22:45:07.604 | INFO | Serializing RTMetrics to json | 20260506 22:45:07.605 | INFO | Serialization done, swapping tmp file now... | 20260506 22:45:07.606 | INFO | RTMetrics done | 20260506 22:45:07.699 | INFO | Finished a round of Baz IO: Wrote 1.904 GiB in 3.228 seconds | 20260506 22:45:07.700 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.61% / 0.00% / 66.39% | 20260506 22:45:07.700 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.477175% / 3.97364e-06% / 4.76837e-05% | 20260506 22:45:07.700 | INFO | Baz IO overall: | 20260506 22:45:07.700 | INFO | BytesWritten : 2646209378728 | 20260506 22:45:07.700 | INFO | NumEvents : 1276494236132 | 20260506 22:45:09.509 | INFO | Analyzing chunk frames = [5042176,5042688) | 20260506 22:45:09.509 | INFO | MemInfo: { | 20260506 22:45:09.509 | INFO | "gpu" : " 17 GB", | 20260506 22:45:09.509 | INFO | "huge" : "", | 20260506 22:45:09.509 | INFO | "huge_shared" : " 27 GB", | 20260506 22:45:09.509 | INFO | "malloc" : " 14 GB", | 20260506 22:45:09.510 | INFO | "pinned" : " 29 GB" | 20260506 22:45:09.510 | INFO | } | 20260506 22:45:09.510 | INFO | Perf report for the last 5.13687 seconds: | 20260506 22:45:09.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:45:09.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:09.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:09.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:09.510 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 22:45:09.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:09.510 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 22:45:09.510 | INFO | PRE_HQ : 59.7%, 1.8, 57.0ms, 0 | 20260506 22:45:09.510 | INFO | BAZWRITER : 62.4%, 3.5, 111.6ms, 0 | 20260506 22:45:09.510 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:45:09.510 | INFO | Uploaded 36GiB to GPU in 2.02293s (17.796GiB/s) | 20260506 22:45:09.510 | INFO | Downloaded 38GiB to GPU in 2.1829s (17.408GiB/s) | 20260506 22:45:14.662 | INFO | Analyzing chunk frames = [5042688,5043200) | 20260506 22:45:14.662 | INFO | MemInfo: { | 20260506 22:45:14.662 | INFO | "gpu" : " 17 GB", | 20260506 22:45:14.662 | INFO | "huge" : "", | 20260506 22:45:14.662 | INFO | "huge_shared" : " 27 GB", | 20260506 22:45:14.662 | INFO | "malloc" : " 14 GB", | 20260506 22:45:14.662 | INFO | "pinned" : " 29 GB" | 20260506 22:45:14.662 | INFO | } | 20260506 22:45:14.662 | INFO | Perf report for the last 5.15229 seconds: | 20260506 22:45:14.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:45:14.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:14.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:14.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:14.662 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 22:45:14.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:14.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:14.662 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 22:45:14.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:14.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:14.662 | INFO | Uploaded 36GiB to GPU in 2.01272s (17.8863GiB/s) | 20260506 22:45:14.662 | INFO | Downloaded 34GiB to GPU in 1.92662s (17.6475GiB/s) | 20260506 22:45:19.799 | INFO | Analyzing chunk frames = [5043200,5043712) | 20260506 22:45:19.799 | INFO | MemInfo: { | 20260506 22:45:19.799 | INFO | "gpu" : " 17 GB", | 20260506 22:45:19.799 | INFO | "huge" : "", | 20260506 22:45:19.799 | INFO | "huge_shared" : " 27 GB", | 20260506 22:45:19.799 | INFO | "malloc" : " 14 GB", | 20260506 22:45:19.799 | INFO | "pinned" : " 29 GB" | 20260506 22:45:19.799 | INFO | } | 20260506 22:45:19.799 | INFO | Perf report for the last 5.13709 seconds: | 20260506 22:45:19.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:45:19.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:19.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:19.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:19.799 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260506 22:45:19.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:19.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:19.799 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:45:19.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:19.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:19.799 | INFO | Uploaded 36GiB to GPU in 1.99868s (18.0119GiB/s) | 20260506 22:45:19.799 | INFO | Downloaded 34GiB to GPU in 1.9306s (17.6111GiB/s) | 20260506 22:45:24.939 | INFO | Analyzing chunk frames = [5043712,5044224) | 20260506 22:45:24.939 | INFO | MemInfo: { | 20260506 22:45:24.939 | INFO | "gpu" : " 17 GB", | 20260506 22:45:24.939 | INFO | "huge" : "", | 20260506 22:45:24.939 | INFO | "huge_shared" : " 27 GB", | 20260506 22:45:24.939 | INFO | "malloc" : " 14 GB", | 20260506 22:45:24.939 | INFO | "pinned" : " 29 GB" | 20260506 22:45:24.939 | INFO | } | 20260506 22:45:24.939 | INFO | Perf report for the last 5.13999 seconds: | 20260506 22:45:24.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:45:24.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:24.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:24.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:24.939 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 22:45:24.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:24.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:24.939 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:45:24.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:24.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:24.939 | INFO | Uploaded 36GiB to GPU in 1.97926s (18.1886GiB/s) | 20260506 22:45:24.939 | INFO | Downloaded 34GiB to GPU in 1.90545s (17.8435GiB/s) | 20260506 22:45:30.075 | INFO | Analyzing chunk frames = [5044224,5044736) | 20260506 22:45:30.075 | INFO | MemInfo: { | 20260506 22:45:30.075 | INFO | "gpu" : " 17 GB", | 20260506 22:45:30.075 | INFO | "huge" : "", | 20260506 22:45:30.075 | INFO | "huge_shared" : " 27 GB", | 20260506 22:45:30.075 | INFO | "malloc" : " 14 GB", | 20260506 22:45:30.075 | INFO | "pinned" : " 29 GB" | 20260506 22:45:30.075 | INFO | } | 20260506 22:45:30.075 | INFO | Perf report for the last 5.13647 seconds: | 20260506 22:45:30.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:45:30.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:30.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:30.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:30.075 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 22:45:30.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:30.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:30.076 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:45:30.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:30.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:30.076 | INFO | Uploaded 36GiB to GPU in 1.99355s (18.0583GiB/s) | 20260506 22:45:30.076 | INFO | Downloaded 34GiB to GPU in 1.93995s (17.5262GiB/s) | 20260506 22:45:35.208 | INFO | Analyzing chunk frames = [5044736,5045248) | 20260506 22:45:35.208 | INFO | MemInfo: { | 20260506 22:45:35.208 | INFO | "gpu" : " 17 GB", | 20260506 22:45:35.208 | INFO | "huge" : "", | 20260506 22:45:35.209 | INFO | "huge_shared" : " 27 GB", | 20260506 22:45:35.209 | INFO | "malloc" : " 14 GB", | 20260506 22:45:35.209 | INFO | "pinned" : " 29 GB" | 20260506 22:45:35.209 | INFO | } | 20260506 22:45:35.209 | INFO | Perf report for the last 5.13323 seconds: | 20260506 22:45:35.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:45:35.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:35.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:35.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:35.209 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 22:45:35.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:35.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:35.209 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:45:35.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:35.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:35.209 | INFO | Uploaded 36GiB to GPU in 2.00236s (17.9788GiB/s) | 20260506 22:45:35.209 | INFO | Downloaded 34GiB to GPU in 1.94165s (17.5108GiB/s) | 20260506 22:45:40.347 | INFO | Analyzing chunk frames = [5045248,5045760) | 20260506 22:45:40.347 | INFO | MemInfo: { | 20260506 22:45:40.347 | INFO | "gpu" : " 17 GB", | 20260506 22:45:40.348 | INFO | "huge" : "", | 20260506 22:45:40.348 | INFO | "huge_shared" : " 27 GB", | 20260506 22:45:40.348 | INFO | "malloc" : " 14 GB", | 20260506 22:45:40.348 | INFO | "pinned" : " 29 GB" | 20260506 22:45:40.348 | INFO | } | 20260506 22:45:40.348 | INFO | Perf report for the last 5.13927 seconds: | 20260506 22:45:40.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:45:40.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:40.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:40.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:40.348 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 22:45:40.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:40.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:40.348 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 22:45:40.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:40.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:40.348 | INFO | Uploaded 36GiB to GPU in 1.95674s (18.398GiB/s) | 20260506 22:45:40.348 | INFO | Downloaded 34GiB to GPU in 1.98524s (17.1264GiB/s) | 20260506 22:45:45.491 | INFO | Analyzing chunk frames = [5045760,5046272) | 20260506 22:45:45.491 | INFO | MemInfo: { | 20260506 22:45:45.491 | INFO | "gpu" : " 17 GB", | 20260506 22:45:45.491 | INFO | "huge" : "", | 20260506 22:45:45.491 | INFO | "huge_shared" : " 27 GB", | 20260506 22:45:45.491 | INFO | "malloc" : " 14 GB", | 20260506 22:45:45.491 | INFO | "pinned" : " 29 GB" | 20260506 22:45:45.491 | INFO | } | 20260506 22:45:45.491 | INFO | Perf report for the last 5.14285 seconds: | 20260506 22:45:45.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:45:45.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:45.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:45.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:45.491 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 22:45:45.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:45.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:45.491 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 22:45:45.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:45.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:45.491 | INFO | Uploaded 36GiB to GPU in 2.00842s (17.9245GiB/s) | 20260506 22:45:45.491 | INFO | Downloaded 34GiB to GPU in 2.0005s (16.9958GiB/s) | 20260506 22:45:45.491 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:45:45.491 | INFO | Total mem usage: 18 GiB | 20260506 22:45:45.491 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:45:45.491 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 22:45:45.491 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:45:45.491 | INFO | High Water Breakdown: | 20260506 22:45:45.491 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:45:45.491 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:45:45.491 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:45:45.491 | INFO | ------------------------------------------ | 20260506 22:45:45.491 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:45:45.491 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:45:45.491 | INFO | Total mem usage: 26.5 GiB | 20260506 22:45:45.491 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:45:45.491 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:45:45.491 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:45:45.491 | INFO | High Water Breakdown: | 20260506 22:45:45.491 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:45:45.491 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:45:45.492 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:45:45.492 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:45:45.492 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:45:45.492 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:45:45.492 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:45:45.492 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:45:45.492 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:45:45.492 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:45:45.492 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:45:45.492 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:45:45.492 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:45:45.492 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:45:45.492 | INFO | ------------------------------------------ | 20260506 22:45:45.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:45:45.492 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:45:45.492 | INFO | Total mem usage: 25.6 GiB | 20260506 22:45:45.492 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:45:45.492 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:45:45.492 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:45:45.492 | INFO | High Water Breakdown: | 20260506 22:45:45.492 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:45:45.492 | INFO | ------------------------------------------ | 20260506 22:45:45.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:45:45.492 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:45:45.492 | INFO | Total mem usage: 15.7 GiB | 20260506 22:45:45.492 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:45:45.492 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:45:45.492 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:45:45.492 | INFO | High Water Breakdown: | 20260506 22:45:45.492 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:45:45.492 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:45:45.492 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:45:45.492 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:45:45.492 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:45:45.492 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:45:45.492 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:45:45.492 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:45:45.492 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:45:45.492 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:45:45.492 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:45:45.492 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:45:45.492 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:45:45.492 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:45:45.492 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:45:45.492 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:45:45.492 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:45:45.492 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:45:45.492 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:45:45.492 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:45:45.492 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:45:45.492 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:45:45.492 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:45:45.492 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:45:45.492 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:45:45.492 | INFO | ------------------------------------------ | 20260506 22:45:45.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:45:45.492 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:45:45.492 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:45:45.492 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:45:45.492 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.56e+01, 1.34e+01 / 2.70e+00 | 20260506 22:45:45.492 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 9.69e-03 / 8.03e-03 | 20260506 22:45:45.492 | INFO | PerfReport: 11.67%, Baseline -- 3.62e+00 / 4.35e+00, 5.49e-01 / 3.88e-01 | 20260506 22:45:45.492 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.42e+00, 1.79e-01 / 1.58e-01 | 20260506 22:45:45.492 | INFO | PerfReport: 16.06%, DME -- 3.99e+01 / 4.45e+01, 4.10e+00 / 2.80e+00 | 20260506 22:45:45.492 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.33e+00 / 1.28e+00 | 20260506 22:45:45.492 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.20e+00 / 2.76e+00, 3.84e-01 / 2.00e-01 | 20260506 22:45:45.492 | INFO | PerfReport: 3.22%, Metrics -- 1.00e+00 / 1.34e+00, 2.66e-01 / 1.75e-01 | 20260506 22:45:45.492 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.41e-01 / 3.85e-01, 5.30e-02 / 1.68e-02 | 20260506 22:45:45.492 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:45:45.492 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:45:45.492 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:45:45.492 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 22:45:45.492 | INFO | PerfReport: 21.47%, Upload -- 2.08e+01 / 0.00e+00, 6.93e+00 / 0.00e+00 | 20260506 22:45:45.492 | INFO | PerfReport: 21.28%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 22:45:45.492 | INFO | PerfReport: 32.09%, Compute -- 3.10e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 22:45:45.492 | INFO | PerfReport: 1.25%, LockUpload -- 1.21e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260506 22:45:45.492 | INFO | PerfReport: 1.29%, LockDownload -- 1.25e+00 / 0.00e+00, 3.15e+00 / 0.00e+00 | 20260506 22:45:45.492 | INFO | PerfReport: 22.61%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 22:45:45.492 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260506 22:45:45.590 | INFO | Starting a round of Baz IO | 20260506 22:45:48.733 | INFO | Populating RTMetrics report | 20260506 22:45:48.733 | INFO | RTMetrics report populated | 20260506 22:45:48.733 | INFO | Opening tmp RTMetrics json file | 20260506 22:45:48.734 | INFO | Serializing RTMetrics to json | 20260506 22:45:48.735 | INFO | Serialization done, swapping tmp file now... | 20260506 22:45:48.736 | INFO | RTMetrics done | 20260506 22:45:48.827 | INFO | Finished a round of Baz IO: Wrote 1.902 GiB in 3.237 seconds | 20260506 22:45:48.827 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.61% / 0.00% / 66.39% | 20260506 22:45:48.827 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.475732% / 7.94729e-06% / 6.35783e-05% | 20260506 22:45:48.827 | INFO | Baz IO overall: | 20260506 22:45:48.827 | INFO | BytesWritten : 2648251652850 | 20260506 22:45:48.827 | INFO | NumEvents : 1277468510046 | 20260506 22:45:50.628 | INFO | Analyzing chunk frames = [5046272,5046784) | 20260506 22:45:50.628 | INFO | MemInfo: { | 20260506 22:45:50.628 | INFO | "gpu" : " 17 GB", | 20260506 22:45:50.628 | INFO | "huge" : "", | 20260506 22:45:50.628 | INFO | "huge_shared" : " 27 GB", | 20260506 22:45:50.628 | INFO | "malloc" : " 14 GB", | 20260506 22:45:50.628 | INFO | "pinned" : " 29 GB" | 20260506 22:45:50.628 | INFO | } | 20260506 22:45:50.628 | INFO | Perf report for the last 5.13715 seconds: | 20260506 22:45:50.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:45:50.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:50.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:50.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:50.628 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 22:45:50.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:50.628 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 22:45:50.628 | INFO | PRE_HQ : 60.4%, 1.9, 59.6ms, 0 | 20260506 22:45:50.628 | INFO | BAZWRITER : 62.6%, 3.6, 114.7ms, 0 | 20260506 22:45:50.628 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260506 22:45:50.628 | INFO | Uploaded 36GiB to GPU in 2.02847s (17.7474GiB/s) | 20260506 22:45:50.629 | INFO | Downloaded 38GiB to GPU in 2.17917s (17.4379GiB/s) | 20260506 22:45:55.768 | INFO | Analyzing chunk frames = [5046784,5047296) | 20260506 22:45:55.769 | INFO | MemInfo: { | 20260506 22:45:55.769 | INFO | "gpu" : " 17 GB", | 20260506 22:45:55.769 | INFO | "huge" : "", | 20260506 22:45:55.769 | INFO | "huge_shared" : " 27 GB", | 20260506 22:45:55.769 | INFO | "malloc" : " 14 GB", | 20260506 22:45:55.769 | INFO | "pinned" : " 29 GB" | 20260506 22:45:55.769 | INFO | } | 20260506 22:45:55.769 | INFO | Perf report for the last 5.14075 seconds: | 20260506 22:45:55.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:45:55.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:55.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:55.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:55.769 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:45:55.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:55.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:55.769 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:45:55.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:55.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:45:55.769 | INFO | Uploaded 36GiB to GPU in 2.00043s (17.9962GiB/s) | 20260506 22:45:55.769 | INFO | Downloaded 34GiB to GPU in 1.93358s (17.584GiB/s) | 20260506 22:46:00.913 | INFO | Analyzing chunk frames = [5047296,5047808) | 20260506 22:46:00.913 | INFO | MemInfo: { | 20260506 22:46:00.913 | INFO | "gpu" : " 17 GB", | 20260506 22:46:00.913 | INFO | "huge" : "", | 20260506 22:46:00.913 | INFO | "huge_shared" : " 27 GB", | 20260506 22:46:00.913 | INFO | "malloc" : " 14 GB", | 20260506 22:46:00.913 | INFO | "pinned" : " 29 GB" | 20260506 22:46:00.914 | INFO | } | 20260506 22:46:00.914 | INFO | Perf report for the last 5.1448 seconds: | 20260506 22:46:00.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:46:00.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:00.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:00.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:00.914 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 22:46:00.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:00.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:00.914 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:46:00.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:00.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:00.914 | INFO | Uploaded 36GiB to GPU in 1.95648s (18.4004GiB/s) | 20260506 22:46:00.914 | INFO | Downloaded 34GiB to GPU in 1.9291s (17.6248GiB/s) | 20260506 22:46:06.050 | INFO | Analyzing chunk frames = [5047808,5048320) | 20260506 22:46:06.050 | INFO | MemInfo: { | 20260506 22:46:06.050 | INFO | "gpu" : " 17 GB", | 20260506 22:46:06.050 | INFO | "huge" : "", | 20260506 22:46:06.050 | INFO | "huge_shared" : " 27 GB", | 20260506 22:46:06.050 | INFO | "malloc" : " 14 GB", | 20260506 22:46:06.050 | INFO | "pinned" : " 29 GB" | 20260506 22:46:06.050 | INFO | } | 20260506 22:46:06.050 | INFO | Perf report for the last 5.13659 seconds: | 20260506 22:46:06.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:46:06.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:06.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:06.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:06.050 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 22:46:06.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:06.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:06.050 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:46:06.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:06.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:06.051 | INFO | Uploaded 36GiB to GPU in 2.0057s (17.9489GiB/s) | 20260506 22:46:06.051 | INFO | Downloaded 34GiB to GPU in 1.90779s (17.8217GiB/s) | 20260506 22:46:11.185 | INFO | Analyzing chunk frames = [5048320,5048832) | 20260506 22:46:11.185 | INFO | MemInfo: { | 20260506 22:46:11.185 | INFO | "gpu" : " 17 GB", | 20260506 22:46:11.185 | INFO | "huge" : "", | 20260506 22:46:11.185 | INFO | "huge_shared" : " 27 GB", | 20260506 22:46:11.185 | INFO | "malloc" : " 14 GB", | 20260506 22:46:11.185 | INFO | "pinned" : " 29 GB" | 20260506 22:46:11.185 | INFO | } | 20260506 22:46:11.185 | INFO | Perf report for the last 5.13495 seconds: | 20260506 22:46:11.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:46:11.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:11.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:11.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:11.185 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 22:46:11.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:11.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:11.185 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:46:11.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:11.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:11.186 | INFO | Uploaded 36GiB to GPU in 2.0024s (17.9784GiB/s) | 20260506 22:46:11.186 | INFO | Downloaded 34GiB to GPU in 1.93628s (17.5594GiB/s) | 20260506 22:46:16.323 | INFO | Analyzing chunk frames = [5048832,5049344) | 20260506 22:46:16.323 | INFO | MemInfo: { | 20260506 22:46:16.323 | INFO | "gpu" : " 17 GB", | 20260506 22:46:16.323 | INFO | "huge" : "", | 20260506 22:46:16.323 | INFO | "huge_shared" : " 27 GB", | 20260506 22:46:16.323 | INFO | "malloc" : " 14 GB", | 20260506 22:46:16.323 | INFO | "pinned" : " 29 GB" | 20260506 22:46:16.323 | INFO | } | 20260506 22:46:16.323 | INFO | Perf report for the last 5.13836 seconds: | 20260506 22:46:16.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:46:16.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:16.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:16.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:16.324 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 22:46:16.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:16.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:16.324 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:46:16.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:16.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:16.324 | INFO | Uploaded 36GiB to GPU in 1.98195s (18.1639GiB/s) | 20260506 22:46:16.324 | INFO | Downloaded 34GiB to GPU in 1.95407s (17.3996GiB/s) | 20260506 22:46:21.463 | INFO | Analyzing chunk frames = [5049344,5049856) | 20260506 22:46:21.463 | INFO | MemInfo: { | 20260506 22:46:21.463 | INFO | "gpu" : " 17 GB", | 20260506 22:46:21.463 | INFO | "huge" : "", | 20260506 22:46:21.463 | INFO | "huge_shared" : " 27 GB", | 20260506 22:46:21.463 | INFO | "malloc" : " 14 GB", | 20260506 22:46:21.463 | INFO | "pinned" : " 29 GB" | 20260506 22:46:21.463 | INFO | } | 20260506 22:46:21.463 | INFO | Perf report for the last 5.13951 seconds: | 20260506 22:46:21.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:46:21.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:21.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:21.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:21.463 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:46:21.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:21.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:21.463 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 22:46:21.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:21.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:21.463 | INFO | Uploaded 36GiB to GPU in 1.95342s (18.4292GiB/s) | 20260506 22:46:21.463 | INFO | Downloaded 34GiB to GPU in 1.97472s (17.2176GiB/s) | 20260506 22:46:26.602 | INFO | Analyzing chunk frames = [5049856,5050368) | 20260506 22:46:26.602 | INFO | MemInfo: { | 20260506 22:46:26.602 | INFO | "gpu" : " 17 GB", | 20260506 22:46:26.603 | INFO | "huge" : "", | 20260506 22:46:26.603 | INFO | "huge_shared" : " 27 GB", | 20260506 22:46:26.603 | INFO | "malloc" : " 14 GB", | 20260506 22:46:26.603 | INFO | "pinned" : " 29 GB" | 20260506 22:46:26.603 | INFO | } | 20260506 22:46:26.603 | INFO | Perf report for the last 5.1397 seconds: | 20260506 22:46:26.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:46:26.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:26.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:26.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:26.603 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 22:46:26.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:26.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:26.603 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 22:46:26.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:26.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:26.603 | INFO | Uploaded 36GiB to GPU in 2.00772s (17.9308GiB/s) | 20260506 22:46:26.603 | INFO | Downloaded 34GiB to GPU in 1.99953s (17.004GiB/s) | 20260506 22:46:26.701 | INFO | Starting a round of Baz IO | 20260506 22:46:29.840 | INFO | Populating RTMetrics report | 20260506 22:46:29.841 | INFO | RTMetrics report populated | 20260506 22:46:29.841 | INFO | Opening tmp RTMetrics json file | 20260506 22:46:29.841 | INFO | Serializing RTMetrics to json | 20260506 22:46:29.845 | INFO | Serialization done, swapping tmp file now... | 20260506 22:46:29.853 | INFO | RTMetrics done | 20260506 22:46:29.936 | INFO | Finished a round of Baz IO: Wrote 1.902 GiB in 3.234 seconds | 20260506 22:46:29.936 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.61% / 0.00% / 66.39% | 20260506 22:46:29.936 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.475379% / 3.97364e-06% / 5.96046e-05% | 20260506 22:46:29.936 | INFO | Baz IO overall: | 20260506 22:46:29.937 | INFO | BytesWritten : 2650293867497 | 20260506 22:46:29.937 | INFO | NumEvents : 1278442844568 | 20260506 22:46:31.750 | INFO | Analyzing chunk frames = [5050368,5050880) | 20260506 22:46:31.750 | INFO | MemInfo: { | 20260506 22:46:31.750 | INFO | "gpu" : " 17 GB", | 20260506 22:46:31.750 | INFO | "huge" : "", | 20260506 22:46:31.750 | INFO | "huge_shared" : " 27 GB", | 20260506 22:46:31.750 | INFO | "malloc" : " 14 GB", | 20260506 22:46:31.750 | INFO | "pinned" : " 29 GB" | 20260506 22:46:31.751 | INFO | } | 20260506 22:46:31.751 | INFO | Perf report for the last 5.14782 seconds: | 20260506 22:46:31.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:46:31.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:31.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:31.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:31.751 | INFO | ANALYSIS : 62.3%, 3.0, 97.1ms, 0 | 20260506 22:46:31.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:31.751 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 22:46:31.751 | INFO | PRE_HQ : 59.7%, 1.8, 57.1ms, 0 | 20260506 22:46:31.751 | INFO | BAZWRITER : 62.4%, 3.5, 112.5ms, 0 | 20260506 22:46:31.751 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:46:31.751 | INFO | Uploaded 36GiB to GPU in 2.02843s (17.7477GiB/s) | 20260506 22:46:31.751 | INFO | Downloaded 38GiB to GPU in 2.17188s (17.4964GiB/s) | 20260506 22:46:36.886 | INFO | Analyzing chunk frames = [5050880,5051392) | 20260506 22:46:36.886 | INFO | MemInfo: { | 20260506 22:46:36.886 | INFO | "gpu" : " 17 GB", | 20260506 22:46:36.886 | INFO | "huge" : "", | 20260506 22:46:36.886 | INFO | "huge_shared" : " 27 GB", | 20260506 22:46:36.886 | INFO | "malloc" : " 14 GB", | 20260506 22:46:36.886 | INFO | "pinned" : " 29 GB" | 20260506 22:46:36.886 | INFO | } | 20260506 22:46:36.886 | INFO | Perf report for the last 5.13528 seconds: | 20260506 22:46:36.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:46:36.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:36.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:36.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:36.886 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:46:36.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:36.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:36.886 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 22:46:36.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:36.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:36.886 | INFO | Uploaded 36GiB to GPU in 1.99259s (18.0669GiB/s) | 20260506 22:46:36.886 | INFO | Downloaded 34GiB to GPU in 1.92837s (17.6315GiB/s) | 20260506 22:46:42.028 | INFO | Analyzing chunk frames = [5051392,5051904) | 20260506 22:46:42.028 | INFO | MemInfo: { | 20260506 22:46:42.028 | INFO | "gpu" : " 17 GB", | 20260506 22:46:42.028 | INFO | "huge" : "", | 20260506 22:46:42.028 | INFO | "huge_shared" : " 27 GB", | 20260506 22:46:42.028 | INFO | "malloc" : " 14 GB", | 20260506 22:46:42.028 | INFO | "pinned" : " 29 GB" | 20260506 22:46:42.028 | INFO | } | 20260506 22:46:42.028 | INFO | Perf report for the last 5.14244 seconds: | 20260506 22:46:42.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:46:42.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:42.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:42.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:42.028 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 22:46:42.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:42.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:42.028 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:46:42.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:42.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:42.029 | INFO | Uploaded 36GiB to GPU in 1.97498s (18.228GiB/s) | 20260506 22:46:42.029 | INFO | Downloaded 34GiB to GPU in 1.92107s (17.6985GiB/s) | 20260506 22:46:47.161 | INFO | Analyzing chunk frames = [5051904,5052416) | 20260506 22:46:47.161 | INFO | MemInfo: { | 20260506 22:46:47.161 | INFO | "gpu" : " 17 GB", | 20260506 22:46:47.161 | INFO | "huge" : "", | 20260506 22:46:47.161 | INFO | "huge_shared" : " 27 GB", | 20260506 22:46:47.161 | INFO | "malloc" : " 14 GB", | 20260506 22:46:47.161 | INFO | "pinned" : " 29 GB" | 20260506 22:46:47.161 | INFO | } | 20260506 22:46:47.161 | INFO | Perf report for the last 5.13243 seconds: | 20260506 22:46:47.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:46:47.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:47.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:47.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:47.161 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 22:46:47.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:47.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:47.161 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:46:47.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:47.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:47.161 | INFO | Uploaded 36GiB to GPU in 2.00088s (17.9921GiB/s) | 20260506 22:46:47.161 | INFO | Downloaded 34GiB to GPU in 1.91018s (17.7994GiB/s) | 20260506 22:46:52.299 | INFO | Analyzing chunk frames = [5052416,5052928) | 20260506 22:46:52.299 | INFO | MemInfo: { | 20260506 22:46:52.299 | INFO | "gpu" : " 17 GB", | 20260506 22:46:52.299 | INFO | "huge" : "", | 20260506 22:46:52.299 | INFO | "huge_shared" : " 27 GB", | 20260506 22:46:52.299 | INFO | "malloc" : " 14 GB", | 20260506 22:46:52.299 | INFO | "pinned" : " 29 GB" | 20260506 22:46:52.299 | INFO | } | 20260506 22:46:52.300 | INFO | Perf report for the last 5.13876 seconds: | 20260506 22:46:52.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:46:52.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:52.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:52.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:52.300 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 22:46:52.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:52.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:52.300 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:46:52.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:52.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:52.300 | INFO | Uploaded 36GiB to GPU in 2.00194s (17.9826GiB/s) | 20260506 22:46:52.300 | INFO | Downloaded 34GiB to GPU in 1.92146s (17.6949GiB/s) | 20260506 22:46:57.440 | INFO | Analyzing chunk frames = [5052928,5053440) | 20260506 22:46:57.440 | INFO | MemInfo: { | 20260506 22:46:57.440 | INFO | "gpu" : " 17 GB", | 20260506 22:46:57.440 | INFO | "huge" : "", | 20260506 22:46:57.440 | INFO | "huge_shared" : " 27 GB", | 20260506 22:46:57.440 | INFO | "malloc" : " 14 GB", | 20260506 22:46:57.440 | INFO | "pinned" : " 29 GB" | 20260506 22:46:57.440 | INFO | } | 20260506 22:46:57.440 | INFO | Perf report for the last 5.14083 seconds: | 20260506 22:46:57.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:46:57.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:57.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:57.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:57.440 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 22:46:57.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:57.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:57.440 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:46:57.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:57.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:46:57.441 | INFO | Uploaded 36GiB to GPU in 2.01757s (17.8432GiB/s) | 20260506 22:46:57.441 | INFO | Downloaded 34GiB to GPU in 1.95661s (17.377GiB/s) | 20260506 22:47:02.580 | INFO | Analyzing chunk frames = [5053440,5053952) | 20260506 22:47:02.580 | INFO | MemInfo: { | 20260506 22:47:02.580 | INFO | "gpu" : " 17 GB", | 20260506 22:47:02.580 | INFO | "huge" : "", | 20260506 22:47:02.580 | INFO | "huge_shared" : " 27 GB", | 20260506 22:47:02.580 | INFO | "malloc" : " 14 GB", | 20260506 22:47:02.580 | INFO | "pinned" : " 29 GB" | 20260506 22:47:02.580 | INFO | } | 20260506 22:47:02.580 | INFO | Perf report for the last 5.13956 seconds: | 20260506 22:47:02.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:47:02.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:02.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:02.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:02.580 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:47:02.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:02.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:02.580 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 22:47:02.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:02.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:02.580 | INFO | Uploaded 36GiB to GPU in 1.95775s (18.3885GiB/s) | 20260506 22:47:02.580 | INFO | Downloaded 34GiB to GPU in 1.98049s (17.1675GiB/s) | 20260506 22:47:07.722 | INFO | Analyzing chunk frames = [5053952,5054464) | 20260506 22:47:07.722 | INFO | MemInfo: { | 20260506 22:47:07.722 | INFO | "gpu" : " 17 GB", | 20260506 22:47:07.722 | INFO | "huge" : "", | 20260506 22:47:07.722 | INFO | "huge_shared" : " 27 GB", | 20260506 22:47:07.722 | INFO | "malloc" : " 14 GB", | 20260506 22:47:07.722 | INFO | "pinned" : " 29 GB" | 20260506 22:47:07.722 | INFO | } | 20260506 22:47:07.723 | INFO | Perf report for the last 5.14264 seconds: | 20260506 22:47:07.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:47:07.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:07.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:07.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:07.723 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 22:47:07.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:07.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:07.723 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:47:07.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:07.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:07.723 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:47:07.723 | INFO | Total mem usage: 18 GiB | 20260506 22:47:07.723 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:47:07.723 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 22:47:07.723 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:47:07.723 | INFO | High Water Breakdown: | 20260506 22:47:07.723 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:47:07.723 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:47:07.723 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:47:07.723 | INFO | ------------------------------------------ | 20260506 22:47:07.723 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:47:07.723 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:47:07.723 | INFO | Total mem usage: 26.5 GiB | 20260506 22:47:07.723 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:47:07.723 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:47:07.723 | INFO | Uploaded 36GiB to GPU in 1.96859s (18.2872GiB/s) | 20260506 22:47:07.723 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:47:07.723 | INFO | Downloaded 34GiB to GPU in 1.98714s (17.11GiB/s) | 20260506 22:47:07.723 | INFO | High Water Breakdown: | 20260506 22:47:07.723 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:47:07.723 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:47:07.723 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:47:07.723 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:47:07.723 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:47:07.723 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:47:07.723 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:47:07.723 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:47:07.723 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:47:07.723 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:47:07.723 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:47:07.723 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:47:07.723 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:47:07.723 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:47:07.723 | INFO | ------------------------------------------ | 20260506 22:47:07.723 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:47:07.723 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:47:07.723 | INFO | Total mem usage: 25.6 GiB | 20260506 22:47:07.723 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:47:07.723 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:47:07.723 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:47:07.723 | INFO | High Water Breakdown: | 20260506 22:47:07.723 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:47:07.723 | INFO | ------------------------------------------ | 20260506 22:47:07.723 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:47:07.723 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:47:07.724 | INFO | Total mem usage: 15.7 GiB | 20260506 22:47:07.724 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:47:07.724 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:47:07.724 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:47:07.724 | INFO | High Water Breakdown: | 20260506 22:47:07.724 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:47:07.724 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:47:07.724 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:47:07.724 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:47:07.724 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:47:07.724 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:47:07.724 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:47:07.724 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:47:07.724 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:47:07.724 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:47:07.724 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:47:07.724 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:47:07.724 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:47:07.724 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:47:07.724 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:47:07.724 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:47:07.724 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:47:07.724 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:47:07.724 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:47:07.724 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:47:07.724 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:47:07.724 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:47:07.724 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:47:07.724 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:47:07.724 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:47:07.724 | INFO | ------------------------------------------ | 20260506 22:47:07.724 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:47:07.724 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:47:07.724 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:47:07.724 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:47:07.724 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.60e+01, 1.34e+01 / 2.41e+00 | 20260506 22:47:07.724 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 9.69e-03 / 6.47e-03 | 20260506 22:47:07.724 | INFO | PerfReport: 11.67%, Baseline -- 3.63e+00 / 4.30e+00, 5.51e-01 / 3.88e-01 | 20260506 22:47:07.724 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.44e+00, 1.82e-01 / 1.31e-01 | 20260506 22:47:07.724 | INFO | PerfReport: 16.05%, DME -- 3.99e+01 / 4.46e+01, 4.32e+00 / 3.21e+00 | 20260506 22:47:07.724 | INFO | PerfReport: 57.36%, FrameLabeling -- 1.78e+01 / 2.11e+01, 1.42e+00 / 1.80e+00 | 20260506 22:47:07.724 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.19e+00 / 2.73e+00, 3.70e-01 / 2.09e-01 | 20260506 22:47:07.724 | INFO | PerfReport: 3.21%, Metrics -- 9.98e-01 / 1.31e+00, 2.66e-01 / 1.43e-01 | 20260506 22:47:07.724 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.41e-01 / 3.87e-01, 5.30e-02 / 1.94e-02 | 20260506 22:47:07.724 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:47:07.724 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:47:07.724 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:47:07.724 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 22:47:07.724 | INFO | PerfReport: 21.45%, Upload -- 2.08e+01 / 0.00e+00, 6.92e+00 / 0.00e+00 | 20260506 22:47:07.724 | INFO | PerfReport: 21.26%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 22:47:07.724 | INFO | PerfReport: 32.12%, Compute -- 3.11e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 22:47:07.724 | INFO | PerfReport: 1.20%, LockUpload -- 1.16e+00 / 0.00e+00, 4.66e+00 / 0.00e+00 | 20260506 22:47:07.724 | INFO | PerfReport: 1.31%, LockDownload -- 1.27e+00 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260506 22:47:07.724 | INFO | PerfReport: 22.64%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 22:47:07.724 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260506 22:47:07.821 | INFO | Starting a round of Baz IO | 20260506 22:47:10.950 | INFO | Populating RTMetrics report | 20260506 22:47:10.950 | INFO | RTMetrics report populated | 20260506 22:47:10.950 | INFO | Opening tmp RTMetrics json file | 20260506 22:47:10.971 | INFO | Serializing RTMetrics to json | 20260506 22:47:10.972 | INFO | Serialization done, swapping tmp file now... | 20260506 22:47:10.972 | INFO | RTMetrics done | 20260506 22:47:11.043 | INFO | Finished a round of Baz IO: Wrote 1.904 GiB in 3.222 seconds | 20260506 22:47:11.043 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.60% / 0.00% / 66.39% | 20260506 22:47:11.043 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.475649% / 3.97364e-06% / 4.76837e-05% | 20260506 22:47:11.043 | INFO | Baz IO overall: | 20260506 22:47:11.043 | INFO | BytesWritten : 2652337937536 | 20260506 22:47:11.043 | INFO | NumEvents : 1279418736798 | 20260506 22:47:12.858 | INFO | Analyzing chunk frames = [5054464,5054976) | 20260506 22:47:12.858 | INFO | MemInfo: { | 20260506 22:47:12.858 | INFO | "gpu" : " 17 GB", | 20260506 22:47:12.858 | INFO | "huge" : "", | 20260506 22:47:12.858 | INFO | "huge_shared" : " 27 GB", | 20260506 22:47:12.858 | INFO | "malloc" : " 14 GB", | 20260506 22:47:12.858 | INFO | "pinned" : " 29 GB" | 20260506 22:47:12.858 | INFO | } | 20260506 22:47:12.858 | INFO | Perf report for the last 5.13551 seconds: | 20260506 22:47:12.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:47:12.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:12.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:12.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:12.858 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260506 22:47:12.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:12.858 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 22:47:12.858 | INFO | PRE_HQ : 59.6%, 1.8, 56.7ms, 0 | 20260506 22:47:12.858 | INFO | BAZWRITER : 62.3%, 3.5, 112.8ms, 0 | 20260506 22:47:12.858 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:47:12.859 | INFO | Uploaded 36GiB to GPU in 1.99985s (18.0014GiB/s) | 20260506 22:47:12.859 | INFO | Downloaded 38GiB to GPU in 2.16584s (17.5452GiB/s) | 20260506 22:47:17.999 | INFO | Analyzing chunk frames = [5054976,5055488) | 20260506 22:47:17.999 | INFO | MemInfo: { | 20260506 22:47:17.999 | INFO | "gpu" : " 17 GB", | 20260506 22:47:17.999 | INFO | "huge" : "", | 20260506 22:47:17.999 | INFO | "huge_shared" : " 27 GB", | 20260506 22:47:18.000 | INFO | "malloc" : " 14 GB", | 20260506 22:47:18.000 | INFO | "pinned" : " 29 GB" | 20260506 22:47:18.000 | INFO | } | 20260506 22:47:18.000 | INFO | Perf report for the last 5.1416 seconds: | 20260506 22:47:18.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:47:18.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:18.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:18.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:18.000 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:47:18.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:18.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:18.000 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:47:18.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:18.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:18.000 | INFO | Uploaded 36GiB to GPU in 2.00049s (17.9956GiB/s) | 20260506 22:47:18.000 | INFO | Downloaded 34GiB to GPU in 1.93236s (17.5951GiB/s) | 20260506 22:47:23.138 | INFO | Analyzing chunk frames = [5055488,5056000) | 20260506 22:47:23.138 | INFO | MemInfo: { | 20260506 22:47:23.138 | INFO | "gpu" : " 17 GB", | 20260506 22:47:23.138 | INFO | "huge" : "", | 20260506 22:47:23.138 | INFO | "huge_shared" : " 27 GB", | 20260506 22:47:23.138 | INFO | "malloc" : " 14 GB", | 20260506 22:47:23.138 | INFO | "pinned" : " 29 GB" | 20260506 22:47:23.138 | INFO | } | 20260506 22:47:23.138 | INFO | Perf report for the last 5.13812 seconds: | 20260506 22:47:23.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:47:23.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:23.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:23.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:23.138 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 22:47:23.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:23.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:23.138 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:47:23.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:23.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:23.138 | INFO | Uploaded 36GiB to GPU in 1.97383s (18.2386GiB/s) | 20260506 22:47:23.138 | INFO | Downloaded 34GiB to GPU in 1.93097s (17.6077GiB/s) | 20260506 22:47:28.273 | INFO | Analyzing chunk frames = [5056000,5056512) | 20260506 22:47:28.273 | INFO | MemInfo: { | 20260506 22:47:28.273 | INFO | "gpu" : " 17 GB", | 20260506 22:47:28.273 | INFO | "huge" : "", | 20260506 22:47:28.273 | INFO | "huge_shared" : " 27 GB", | 20260506 22:47:28.273 | INFO | "malloc" : " 14 GB", | 20260506 22:47:28.273 | INFO | "pinned" : " 29 GB" | 20260506 22:47:28.273 | INFO | } | 20260506 22:47:28.273 | INFO | Perf report for the last 5.13533 seconds: | 20260506 22:47:28.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:47:28.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:28.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:28.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:28.273 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 22:47:28.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:28.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:28.273 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:47:28.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:28.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:28.273 | INFO | Uploaded 36GiB to GPU in 2.00961s (17.914GiB/s) | 20260506 22:47:28.274 | INFO | Downloaded 34GiB to GPU in 1.9078s (17.8216GiB/s) | 20260506 22:47:33.418 | INFO | Analyzing chunk frames = [5056512,5057024) | 20260506 22:47:33.418 | INFO | MemInfo: { | 20260506 22:47:33.418 | INFO | "gpu" : " 17 GB", | 20260506 22:47:33.419 | INFO | "huge" : "", | 20260506 22:47:33.419 | INFO | "huge_shared" : " 27 GB", | 20260506 22:47:33.419 | INFO | "malloc" : " 14 GB", | 20260506 22:47:33.419 | INFO | "pinned" : " 29 GB" | 20260506 22:47:33.419 | INFO | } | 20260506 22:47:33.419 | INFO | Perf report for the last 5.14555 seconds: | 20260506 22:47:33.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:47:33.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:33.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:33.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:33.419 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260506 22:47:33.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:33.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:33.419 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:47:33.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:33.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:33.419 | INFO | Uploaded 36GiB to GPU in 1.96005s (18.3669GiB/s) | 20260506 22:47:33.419 | INFO | Downloaded 34GiB to GPU in 1.93316s (17.5878GiB/s) | 20260506 22:47:38.555 | INFO | Analyzing chunk frames = [5057024,5057536) | 20260506 22:47:38.555 | INFO | MemInfo: { | 20260506 22:47:38.555 | INFO | "gpu" : " 17 GB", | 20260506 22:47:38.555 | INFO | "huge" : "", | 20260506 22:47:38.555 | INFO | "huge_shared" : " 27 GB", | 20260506 22:47:38.555 | INFO | "malloc" : " 14 GB", | 20260506 22:47:38.555 | INFO | "pinned" : " 29 GB" | 20260506 22:47:38.555 | INFO | } | 20260506 22:47:38.555 | INFO | Perf report for the last 5.13661 seconds: | 20260506 22:47:38.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:47:38.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:38.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:38.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:38.555 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 22:47:38.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:38.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:38.556 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 22:47:38.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:38.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:38.556 | INFO | Uploaded 36GiB to GPU in 2.00747s (17.933GiB/s) | 20260506 22:47:38.556 | INFO | Downloaded 34GiB to GPU in 1.94089s (17.5177GiB/s) | 20260506 22:47:43.702 | INFO | Analyzing chunk frames = [5057536,5058048) | 20260506 22:47:43.702 | INFO | MemInfo: { | 20260506 22:47:43.702 | INFO | "gpu" : " 17 GB", | 20260506 22:47:43.702 | INFO | "huge" : "", | 20260506 22:47:43.702 | INFO | "huge_shared" : " 27 GB", | 20260506 22:47:43.702 | INFO | "malloc" : " 14 GB", | 20260506 22:47:43.702 | INFO | "pinned" : " 29 GB" | 20260506 22:47:43.703 | INFO | } | 20260506 22:47:43.703 | INFO | Perf report for the last 5.14718 seconds: | 20260506 22:47:43.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:47:43.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:43.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:43.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:43.703 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 22:47:43.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:43.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:43.703 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 22:47:43.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:43.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:43.703 | INFO | Uploaded 36GiB to GPU in 1.95457s (18.4183GiB/s) | 20260506 22:47:43.703 | INFO | Downloaded 34GiB to GPU in 1.98227s (17.152GiB/s) | 20260506 22:47:48.844 | INFO | Analyzing chunk frames = [5058048,5058560) | 20260506 22:47:48.844 | INFO | MemInfo: { | 20260506 22:47:48.844 | INFO | "gpu" : " 17 GB", | 20260506 22:47:48.844 | INFO | "huge" : "", | 20260506 22:47:48.844 | INFO | "huge_shared" : " 27 GB", | 20260506 22:47:48.844 | INFO | "malloc" : " 14 GB", | 20260506 22:47:48.844 | INFO | "pinned" : " 29 GB" | 20260506 22:47:48.844 | INFO | } | 20260506 22:47:48.844 | INFO | Perf report for the last 5.14161 seconds: | 20260506 22:47:48.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:47:48.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:48.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:48.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:48.844 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:47:48.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:48.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:48.844 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:47:48.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:48.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:48.844 | INFO | Uploaded 36GiB to GPU in 1.98949s (18.0951GiB/s) | 20260506 22:47:48.845 | INFO | Downloaded 34GiB to GPU in 2.00032s (16.9973GiB/s) | 20260506 22:47:48.943 | INFO | Starting a round of Baz IO | 20260506 22:47:52.075 | INFO | Populating RTMetrics report | 20260506 22:47:52.075 | INFO | RTMetrics report populated | 20260506 22:47:52.075 | INFO | Opening tmp RTMetrics json file | 20260506 22:47:52.076 | INFO | Serializing RTMetrics to json | 20260506 22:47:52.077 | INFO | Serialization done, swapping tmp file now... | 20260506 22:47:52.077 | INFO | RTMetrics done | 20260506 22:47:52.169 | INFO | Finished a round of Baz IO: Wrote 1.902 GiB in 3.225 seconds | 20260506 22:47:52.169 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.60% / 0.00% / 66.40% | 20260506 22:47:52.169 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.47464% / 0% / 3.57628e-05% | 20260506 22:47:52.169 | INFO | Baz IO overall: | 20260506 22:47:52.169 | INFO | BytesWritten : 2654380408276 | 20260506 22:47:52.169 | INFO | NumEvents : 1280393417093 | 20260506 22:47:53.978 | INFO | Analyzing chunk frames = [5058560,5059072) | 20260506 22:47:53.978 | INFO | MemInfo: { | 20260506 22:47:53.978 | INFO | "gpu" : " 17 GB", | 20260506 22:47:53.978 | INFO | "huge" : "", | 20260506 22:47:53.978 | INFO | "huge_shared" : " 27 GB", | 20260506 22:47:53.978 | INFO | "malloc" : " 14 GB", | 20260506 22:47:53.978 | INFO | "pinned" : " 29 GB" | 20260506 22:47:53.978 | INFO | } | 20260506 22:47:53.978 | INFO | Perf report for the last 5.13425 seconds: | 20260506 22:47:53.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:47:53.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:53.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:53.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:53.979 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260506 22:47:53.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:53.979 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260506 22:47:53.979 | INFO | PRE_HQ : 59.9%, 1.8, 56.8ms, 0 | 20260506 22:47:53.979 | INFO | BAZWRITER : 62.4%, 3.4, 110.0ms, 0 | 20260506 22:47:53.979 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:47:53.979 | INFO | Uploaded 36GiB to GPU in 2.02492s (17.7785GiB/s) | 20260506 22:47:53.979 | INFO | Downloaded 38GiB to GPU in 2.17783s (17.4486GiB/s) | 20260506 22:47:59.116 | INFO | Analyzing chunk frames = [5059072,5059584) | 20260506 22:47:59.116 | INFO | MemInfo: { | 20260506 22:47:59.116 | INFO | "gpu" : " 17 GB", | 20260506 22:47:59.116 | INFO | "huge" : "", | 20260506 22:47:59.116 | INFO | "huge_shared" : " 27 GB", | 20260506 22:47:59.116 | INFO | "malloc" : " 14 GB", | 20260506 22:47:59.116 | INFO | "pinned" : " 29 GB" | 20260506 22:47:59.116 | INFO | } | 20260506 22:47:59.116 | INFO | Perf report for the last 5.13754 seconds: | 20260506 22:47:59.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:47:59.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:59.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:59.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:59.116 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:47:59.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:59.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:59.116 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:47:59.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:59.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:47:59.116 | INFO | Uploaded 36GiB to GPU in 2.00465s (17.9582GiB/s) | 20260506 22:47:59.116 | INFO | Downloaded 34GiB to GPU in 1.92675s (17.6463GiB/s) | 20260506 22:48:04.249 | INFO | Analyzing chunk frames = [5059584,5060096) | 20260506 22:48:04.249 | INFO | MemInfo: { | 20260506 22:48:04.249 | INFO | "gpu" : " 17 GB", | 20260506 22:48:04.249 | INFO | "huge" : "", | 20260506 22:48:04.249 | INFO | "huge_shared" : " 27 GB", | 20260506 22:48:04.249 | INFO | "malloc" : " 14 GB", | 20260506 22:48:04.249 | INFO | "pinned" : " 29 GB" | 20260506 22:48:04.249 | INFO | } | 20260506 22:48:04.249 | INFO | Perf report for the last 5.13345 seconds: | 20260506 22:48:04.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:48:04.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:04.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:04.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:04.250 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260506 22:48:04.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:04.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:04.250 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:48:04.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:04.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:04.250 | INFO | Uploaded 36GiB to GPU in 1.99367s (18.0572GiB/s) | 20260506 22:48:04.250 | INFO | Downloaded 34GiB to GPU in 1.93038s (17.6131GiB/s) | 20260506 22:48:09.388 | INFO | Analyzing chunk frames = [5060096,5060608) | 20260506 22:48:09.388 | INFO | MemInfo: { | 20260506 22:48:09.388 | INFO | "gpu" : " 17 GB", | 20260506 22:48:09.388 | INFO | "huge" : "", | 20260506 22:48:09.388 | INFO | "huge_shared" : " 27 GB", | 20260506 22:48:09.388 | INFO | "malloc" : " 14 GB", | 20260506 22:48:09.388 | INFO | "pinned" : " 29 GB" | 20260506 22:48:09.388 | INFO | } | 20260506 22:48:09.388 | INFO | Perf report for the last 5.1388 seconds: | 20260506 22:48:09.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:48:09.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:09.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:09.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:09.388 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 22:48:09.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:09.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:09.388 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:48:09.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:09.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:09.389 | INFO | Uploaded 36GiB to GPU in 1.97244s (18.2515GiB/s) | 20260506 22:48:09.389 | INFO | Downloaded 34GiB to GPU in 1.88009s (18.0842GiB/s) | 20260506 22:48:14.539 | INFO | Analyzing chunk frames = [5060608,5061120) | 20260506 22:48:14.539 | INFO | MemInfo: { | 20260506 22:48:14.539 | INFO | "gpu" : " 17 GB", | 20260506 22:48:14.539 | INFO | "huge" : "", | 20260506 22:48:14.539 | INFO | "huge_shared" : " 27 GB", | 20260506 22:48:14.539 | INFO | "malloc" : " 14 GB", | 20260506 22:48:14.539 | INFO | "pinned" : " 29 GB" | 20260506 22:48:14.539 | INFO | } | 20260506 22:48:14.539 | INFO | Perf report for the last 5.15085 seconds: | 20260506 22:48:14.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:48:14.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:14.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:14.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:14.539 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260506 22:48:14.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:14.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:14.539 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:48:14.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:14.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:14.539 | INFO | Uploaded 36GiB to GPU in 1.99206s (18.0718GiB/s) | 20260506 22:48:14.540 | INFO | Downloaded 34GiB to GPU in 1.91876s (17.7198GiB/s) | 20260506 22:48:19.670 | INFO | Analyzing chunk frames = [5061120,5061632) | 20260506 22:48:19.670 | INFO | MemInfo: { | 20260506 22:48:19.670 | INFO | "gpu" : " 17 GB", | 20260506 22:48:19.670 | INFO | "huge" : "", | 20260506 22:48:19.671 | INFO | "huge_shared" : " 27 GB", | 20260506 22:48:19.671 | INFO | "malloc" : " 14 GB", | 20260506 22:48:19.671 | INFO | "pinned" : " 29 GB" | 20260506 22:48:19.671 | INFO | } | 20260506 22:48:19.671 | INFO | Perf report for the last 5.13149 seconds: | 20260506 22:48:19.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:48:19.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:19.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:19.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:19.671 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 22:48:19.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:19.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:19.671 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 22:48:19.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:19.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:19.671 | INFO | Uploaded 36GiB to GPU in 2.02511s (17.7768GiB/s) | 20260506 22:48:19.671 | INFO | Downloaded 34GiB to GPU in 1.93681s (17.5546GiB/s) | 20260506 22:48:24.810 | INFO | Analyzing chunk frames = [5061632,5062144) | 20260506 22:48:24.810 | INFO | MemInfo: { | 20260506 22:48:24.810 | INFO | "gpu" : " 17 GB", | 20260506 22:48:24.810 | INFO | "huge" : "", | 20260506 22:48:24.810 | INFO | "huge_shared" : " 27 GB", | 20260506 22:48:24.810 | INFO | "malloc" : " 14 GB", | 20260506 22:48:24.810 | INFO | "pinned" : " 29 GB" | 20260506 22:48:24.810 | INFO | } | 20260506 22:48:24.810 | INFO | Perf report for the last 5.13981 seconds: | 20260506 22:48:24.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:48:24.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:24.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:24.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:24.811 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 22:48:24.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:24.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:24.811 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 22:48:24.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:24.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:24.811 | INFO | Uploaded 36GiB to GPU in 1.95617s (18.4033GiB/s) | 20260506 22:48:24.811 | INFO | Downloaded 34GiB to GPU in 1.99319s (17.0581GiB/s) | 20260506 22:48:29.952 | INFO | Analyzing chunk frames = [5062144,5062656) | 20260506 22:48:29.952 | INFO | MemInfo: { | 20260506 22:48:29.952 | INFO | "gpu" : " 17 GB", | 20260506 22:48:29.952 | INFO | "huge" : "", | 20260506 22:48:29.952 | INFO | "huge_shared" : " 27 GB", | 20260506 22:48:29.952 | INFO | "malloc" : " 14 GB", | 20260506 22:48:29.952 | INFO | "pinned" : " 29 GB" | 20260506 22:48:29.952 | INFO | } | 20260506 22:48:29.952 | INFO | Perf report for the last 5.14175 seconds: | 20260506 22:48:29.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:48:29.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:29.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:29.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:29.952 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:48:29.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:29.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:29.952 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:48:29.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:29.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:29.953 | INFO | Uploaded 36GiB to GPU in 1.9927s (18.066GiB/s) | 20260506 22:48:29.953 | INFO | Downloaded 34GiB to GPU in 1.99374s (17.0534GiB/s) | 20260506 22:48:29.953 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:48:29.953 | INFO | Total mem usage: 18 GiB | 20260506 22:48:29.953 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:48:29.953 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 22:48:29.953 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:48:29.953 | INFO | High Water Breakdown: | 20260506 22:48:29.953 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:48:29.953 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:48:29.953 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:48:29.953 | INFO | ------------------------------------------ | 20260506 22:48:29.953 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:48:29.953 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:48:29.953 | INFO | Total mem usage: 26.5 GiB | 20260506 22:48:29.953 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:48:29.953 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:48:29.953 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:48:29.953 | INFO | High Water Breakdown: | 20260506 22:48:29.953 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:48:29.953 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:48:29.953 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:48:29.953 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:48:29.953 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:48:29.953 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:48:29.953 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:48:29.953 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:48:29.953 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:48:29.953 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:48:29.953 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:48:29.953 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:48:29.953 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:48:29.953 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:48:29.953 | INFO | ------------------------------------------ | 20260506 22:48:29.953 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:48:29.953 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:48:29.953 | INFO | Total mem usage: 25.6 GiB | 20260506 22:48:29.953 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:48:29.953 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:48:29.953 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:48:29.953 | INFO | High Water Breakdown: | 20260506 22:48:29.953 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:48:29.953 | INFO | ------------------------------------------ | 20260506 22:48:29.953 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:48:29.953 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:48:29.953 | INFO | Total mem usage: 15.7 GiB | 20260506 22:48:29.953 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:48:29.953 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:48:29.953 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:48:29.953 | INFO | High Water Breakdown: | 20260506 22:48:29.953 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:48:29.954 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:48:29.954 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:48:29.954 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:48:29.954 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:48:29.954 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:48:29.954 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:48:29.954 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:48:29.954 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:48:29.954 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:48:29.954 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:48:29.954 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:48:29.954 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:48:29.954 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:48:29.954 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:48:29.954 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:48:29.954 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:48:29.954 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:48:29.954 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:48:29.954 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:48:29.954 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:48:29.954 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:48:29.954 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:48:29.954 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:48:29.954 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:48:29.954 | INFO | ------------------------------------------ | 20260506 22:48:29.954 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:48:29.954 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:48:29.954 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:48:29.954 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:48:29.954 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.56e+01, 1.34e+01 / 2.17e+00 | 20260506 22:48:29.954 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.01e-01, 9.88e-03 / 6.28e-03 | 20260506 22:48:29.954 | INFO | PerfReport: 11.67%, Baseline -- 3.62e+00 / 4.38e+00, 5.42e-01 / 3.57e-01 | 20260506 22:48:29.954 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.48e+00, 1.79e-01 / 1.33e-01 | 20260506 22:48:29.954 | INFO | PerfReport: 16.05%, DME -- 3.98e+01 / 4.44e+01, 4.11e+00 / 2.65e+00 | 20260506 22:48:29.954 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.34e+00 / 1.98e+00 | 20260506 22:48:29.954 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.19e+00 / 2.77e+00, 3.76e-01 / 1.93e-01 | 20260506 22:48:29.954 | INFO | PerfReport: 3.22%, Metrics -- 9.98e-01 / 1.34e+00, 2.63e-01 / 1.80e-01 | 20260506 22:48:29.954 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.40e-01 / 3.89e-01, 5.24e-02 / 2.41e-02 | 20260506 22:48:29.954 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:48:29.954 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:48:29.954 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:48:29.954 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 22:48:29.954 | INFO | PerfReport: 21.46%, Upload -- 2.07e+01 / 0.00e+00, 6.88e+00 / 0.00e+00 | 20260506 22:48:29.954 | INFO | PerfReport: 21.25%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 22:48:29.954 | INFO | PerfReport: 32.09%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 22:48:29.954 | INFO | PerfReport: 1.29%, LockUpload -- 1.25e+00 / 0.00e+00, 4.71e+00 / 0.00e+00 | 20260506 22:48:29.954 | INFO | PerfReport: 1.27%, LockDownload -- 1.23e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260506 22:48:29.954 | INFO | PerfReport: 22.63%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 22:48:29.954 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.63e-02 / 0.00e+00 | 20260506 22:48:30.051 | INFO | Starting a round of Baz IO | 20260506 22:48:33.185 | INFO | Populating RTMetrics report | 20260506 22:48:33.185 | INFO | RTMetrics report populated | 20260506 22:48:33.185 | INFO | Opening tmp RTMetrics json file | 20260506 22:48:33.186 | INFO | Serializing RTMetrics to json | 20260506 22:48:33.194 | INFO | Serialization done, swapping tmp file now... | 20260506 22:48:33.207 | INFO | RTMetrics done | 20260506 22:48:33.280 | INFO | Finished a round of Baz IO: Wrote 1.899 GiB in 3.229 seconds | 20260506 22:48:33.280 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.60% / 0.00% / 66.40% | 20260506 22:48:33.280 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.470944% / 3.97364e-06% / 5.16574e-05% | 20260506 22:48:33.280 | INFO | Baz IO overall: | 20260506 22:48:33.280 | INFO | BytesWritten : 2656419943548 | 20260506 22:48:33.280 | INFO | NumEvents : 1281366086995 | 20260506 22:48:35.092 | INFO | Analyzing chunk frames = [5062656,5063168) | 20260506 22:48:35.092 | INFO | MemInfo: { | 20260506 22:48:35.092 | INFO | "gpu" : " 17 GB", | 20260506 22:48:35.092 | INFO | "huge" : "", | 20260506 22:48:35.092 | INFO | "huge_shared" : " 27 GB", | 20260506 22:48:35.092 | INFO | "malloc" : " 14 GB", | 20260506 22:48:35.092 | INFO | "pinned" : " 29 GB" | 20260506 22:48:35.092 | INFO | } | 20260506 22:48:35.092 | INFO | Perf report for the last 5.13987 seconds: | 20260506 22:48:35.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:48:35.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:35.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:35.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:35.092 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 22:48:35.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:35.092 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 22:48:35.092 | INFO | PRE_HQ : 59.6%, 1.8, 56.6ms, 0 | 20260506 22:48:35.092 | INFO | BAZWRITER : 62.4%, 3.5, 111.9ms, 0 | 20260506 22:48:35.092 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:48:35.092 | INFO | Uploaded 36GiB to GPU in 2.02473s (17.7801GiB/s) | 20260506 22:48:35.092 | INFO | Downloaded 38GiB to GPU in 2.17848s (17.4434GiB/s) | 20260506 22:48:40.229 | INFO | Analyzing chunk frames = [5063168,5063680) | 20260506 22:48:40.229 | INFO | MemInfo: { | 20260506 22:48:40.229 | INFO | "gpu" : " 17 GB", | 20260506 22:48:40.229 | INFO | "huge" : "", | 20260506 22:48:40.230 | INFO | "huge_shared" : " 27 GB", | 20260506 22:48:40.230 | INFO | "malloc" : " 14 GB", | 20260506 22:48:40.230 | INFO | "pinned" : " 29 GB" | 20260506 22:48:40.230 | INFO | } | 20260506 22:48:40.230 | INFO | Perf report for the last 5.1376 seconds: | 20260506 22:48:40.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:48:40.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:40.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:40.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:40.230 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:48:40.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:40.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:40.230 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 22:48:40.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:40.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:40.230 | INFO | Uploaded 36GiB to GPU in 2.00683s (17.9387GiB/s) | 20260506 22:48:40.230 | INFO | Downloaded 34GiB to GPU in 1.92663s (17.6474GiB/s) | 20260506 22:48:45.367 | INFO | Analyzing chunk frames = [5063680,5064192) | 20260506 22:48:45.367 | INFO | MemInfo: { | 20260506 22:48:45.367 | INFO | "gpu" : " 17 GB", | 20260506 22:48:45.367 | INFO | "huge" : "", | 20260506 22:48:45.367 | INFO | "huge_shared" : " 27 GB", | 20260506 22:48:45.368 | INFO | "malloc" : " 14 GB", | 20260506 22:48:45.368 | INFO | "pinned" : " 29 GB" | 20260506 22:48:45.368 | INFO | } | 20260506 22:48:45.368 | INFO | Perf report for the last 5.1379 seconds: | 20260506 22:48:45.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:48:45.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:45.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:45.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:45.368 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 22:48:45.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:45.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:45.368 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:48:45.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:45.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:45.368 | INFO | Uploaded 36GiB to GPU in 1.99067s (18.0844GiB/s) | 20260506 22:48:45.368 | INFO | Downloaded 34GiB to GPU in 1.92802s (17.6347GiB/s) | 20260506 22:48:50.506 | INFO | Analyzing chunk frames = [5064192,5064704) | 20260506 22:48:50.506 | INFO | MemInfo: { | 20260506 22:48:50.506 | INFO | "gpu" : " 17 GB", | 20260506 22:48:50.506 | INFO | "huge" : "", | 20260506 22:48:50.506 | INFO | "huge_shared" : " 27 GB", | 20260506 22:48:50.506 | INFO | "malloc" : " 14 GB", | 20260506 22:48:50.506 | INFO | "pinned" : " 29 GB" | 20260506 22:48:50.506 | INFO | } | 20260506 22:48:50.506 | INFO | Perf report for the last 5.13881 seconds: | 20260506 22:48:50.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:48:50.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:50.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:50.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:50.507 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260506 22:48:50.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:50.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:50.507 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:48:50.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:50.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:50.507 | INFO | Uploaded 36GiB to GPU in 2.01719s (17.8466GiB/s) | 20260506 22:48:50.507 | INFO | Downloaded 34GiB to GPU in 1.90815s (17.8183GiB/s) | 20260506 22:48:55.646 | INFO | Analyzing chunk frames = [5064704,5065216) | 20260506 22:48:55.646 | INFO | MemInfo: { | 20260506 22:48:55.646 | INFO | "gpu" : " 17 GB", | 20260506 22:48:55.646 | INFO | "huge" : "", | 20260506 22:48:55.646 | INFO | "huge_shared" : " 27 GB", | 20260506 22:48:55.646 | INFO | "malloc" : " 14 GB", | 20260506 22:48:55.646 | INFO | "pinned" : " 29 GB" | 20260506 22:48:55.646 | INFO | } | 20260506 22:48:55.647 | INFO | Perf report for the last 5.14002 seconds: | 20260506 22:48:55.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:48:55.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:55.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:55.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:55.647 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 22:48:55.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:55.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:55.647 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:48:55.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:55.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:48:55.647 | INFO | Uploaded 36GiB to GPU in 1.98812s (18.1076GiB/s) | 20260506 22:48:55.647 | INFO | Downloaded 34GiB to GPU in 1.92219s (17.6882GiB/s) | 20260506 22:49:00.795 | INFO | Analyzing chunk frames = [5065216,5065728) | 20260506 22:49:00.795 | INFO | MemInfo: { | 20260506 22:49:00.795 | INFO | "gpu" : " 17 GB", | 20260506 22:49:00.795 | INFO | "huge" : "", | 20260506 22:49:00.795 | INFO | "huge_shared" : " 27 GB", | 20260506 22:49:00.795 | INFO | "malloc" : " 14 GB", | 20260506 22:49:00.795 | INFO | "pinned" : " 29 GB" | 20260506 22:49:00.795 | INFO | } | 20260506 22:49:00.795 | INFO | Perf report for the last 5.14842 seconds: | 20260506 22:49:00.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:49:00.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:00.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:00.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:00.795 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 22:49:00.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:00.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:00.795 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 22:49:00.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:00.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:00.795 | INFO | Uploaded 36GiB to GPU in 2.01593s (17.8578GiB/s) | 20260506 22:49:00.795 | INFO | Downloaded 34GiB to GPU in 1.95138s (17.4236GiB/s) | 20260506 22:49:05.926 | INFO | Analyzing chunk frames = [5065728,5066240) | 20260506 22:49:05.926 | INFO | MemInfo: { | 20260506 22:49:05.926 | INFO | "gpu" : " 17 GB", | 20260506 22:49:05.926 | INFO | "huge" : "", | 20260506 22:49:05.926 | INFO | "huge_shared" : " 27 GB", | 20260506 22:49:05.926 | INFO | "malloc" : " 14 GB", | 20260506 22:49:05.926 | INFO | "pinned" : " 29 GB" | 20260506 22:49:05.926 | INFO | } | 20260506 22:49:05.926 | INFO | Perf report for the last 5.13145 seconds: | 20260506 22:49:05.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:49:05.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:05.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:05.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:05.926 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 22:49:05.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:05.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:05.927 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 22:49:05.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:05.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:05.927 | INFO | Uploaded 36GiB to GPU in 1.95292s (18.4339GiB/s) | 20260506 22:49:05.927 | INFO | Downloaded 34GiB to GPU in 1.9866s (17.1147GiB/s) | 20260506 22:49:11.065 | INFO | Analyzing chunk frames = [5066240,5066752) | 20260506 22:49:11.065 | INFO | MemInfo: { | 20260506 22:49:11.065 | INFO | "gpu" : " 17 GB", | 20260506 22:49:11.065 | INFO | "huge" : "", | 20260506 22:49:11.065 | INFO | "huge_shared" : " 27 GB", | 20260506 22:49:11.065 | INFO | "malloc" : " 14 GB", | 20260506 22:49:11.065 | INFO | "pinned" : " 29 GB" | 20260506 22:49:11.065 | INFO | } | 20260506 22:49:11.065 | INFO | Perf report for the last 5.13894 seconds: | 20260506 22:49:11.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:49:11.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:11.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:11.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:11.065 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 22:49:11.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:11.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:11.065 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:49:11.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:11.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:11.066 | INFO | Uploaded 36GiB to GPU in 1.96596s (18.3117GiB/s) | 20260506 22:49:11.066 | INFO | Downloaded 34GiB to GPU in 1.98803s (17.1023GiB/s) | 20260506 22:49:11.165 | INFO | Starting a round of Baz IO | 20260506 22:49:14.295 | INFO | Populating RTMetrics report | 20260506 22:49:14.295 | INFO | RTMetrics report populated | 20260506 22:49:14.295 | INFO | Opening tmp RTMetrics json file | 20260506 22:49:14.296 | INFO | Serializing RTMetrics to json | 20260506 22:49:14.297 | INFO | Serialization done, swapping tmp file now... | 20260506 22:49:14.298 | INFO | RTMetrics done | 20260506 22:49:14.388 | INFO | Finished a round of Baz IO: Wrote 1.901 GiB in 3.222 seconds | 20260506 22:49:14.388 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.60% / 0.00% / 66.40% | 20260506 22:49:14.388 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.47133% / 3.97364e-06% / 6.75519e-05% | 20260506 22:49:14.389 | INFO | Baz IO overall: | 20260506 22:49:14.389 | INFO | BytesWritten : 2658460984825 | 20260506 22:49:14.389 | INFO | NumEvents : 1282339830120 | 20260506 22:49:16.201 | INFO | Analyzing chunk frames = [5066752,5067264) | 20260506 22:49:16.201 | INFO | MemInfo: { | 20260506 22:49:16.201 | INFO | "gpu" : " 17 GB", | 20260506 22:49:16.201 | INFO | "huge" : "", | 20260506 22:49:16.201 | INFO | "huge_shared" : " 27 GB", | 20260506 22:49:16.201 | INFO | "malloc" : " 14 GB", | 20260506 22:49:16.201 | INFO | "pinned" : " 29 GB" | 20260506 22:49:16.201 | INFO | } | 20260506 22:49:16.201 | INFO | Perf report for the last 5.13579 seconds: | 20260506 22:49:16.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:49:16.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:16.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:16.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:16.201 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 22:49:16.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:16.201 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 22:49:16.201 | INFO | PRE_HQ : 59.9%, 1.8, 56.5ms, 0 | 20260506 22:49:16.201 | INFO | BAZWRITER : 62.4%, 3.4, 109.8ms, 0 | 20260506 22:49:16.201 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:49:16.201 | INFO | Uploaded 36GiB to GPU in 2.01633s (17.8542GiB/s) | 20260506 22:49:16.201 | INFO | Downloaded 38GiB to GPU in 2.17449s (17.4753GiB/s) | 20260506 22:49:21.348 | INFO | Analyzing chunk frames = [5067264,5067776) | 20260506 22:49:21.348 | INFO | MemInfo: { | 20260506 22:49:21.348 | INFO | "gpu" : " 17 GB", | 20260506 22:49:21.348 | INFO | "huge" : "", | 20260506 22:49:21.348 | INFO | "huge_shared" : " 27 GB", | 20260506 22:49:21.348 | INFO | "malloc" : " 14 GB", | 20260506 22:49:21.348 | INFO | "pinned" : " 29 GB" | 20260506 22:49:21.348 | INFO | } | 20260506 22:49:21.348 | INFO | Perf report for the last 5.14696 seconds: | 20260506 22:49:21.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:49:21.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:21.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:21.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:21.348 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 22:49:21.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:21.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:21.348 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:49:21.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:21.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:21.348 | INFO | Uploaded 36GiB to GPU in 2.00141s (17.9873GiB/s) | 20260506 22:49:21.349 | INFO | Downloaded 34GiB to GPU in 1.93306s (17.5887GiB/s) | 20260506 22:49:26.482 | INFO | Analyzing chunk frames = [5067776,5068288) | 20260506 22:49:26.482 | INFO | MemInfo: { | 20260506 22:49:26.482 | INFO | "gpu" : " 17 GB", | 20260506 22:49:26.482 | INFO | "huge" : "", | 20260506 22:49:26.482 | INFO | "huge_shared" : " 27 GB", | 20260506 22:49:26.482 | INFO | "malloc" : " 14 GB", | 20260506 22:49:26.482 | INFO | "pinned" : " 29 GB" | 20260506 22:49:26.482 | INFO | } | 20260506 22:49:26.482 | INFO | Perf report for the last 5.13416 seconds: | 20260506 22:49:26.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:49:26.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:26.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:26.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:26.482 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 22:49:26.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:26.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:26.482 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:49:26.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:26.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:26.483 | INFO | Uploaded 36GiB to GPU in 1.9882s (18.1068GiB/s) | 20260506 22:49:26.483 | INFO | Downloaded 34GiB to GPU in 1.93505s (17.5706GiB/s) | 20260506 22:49:31.616 | INFO | Analyzing chunk frames = [5068288,5068800) | 20260506 22:49:31.617 | INFO | MemInfo: { | 20260506 22:49:31.617 | INFO | "gpu" : " 17 GB", | 20260506 22:49:31.617 | INFO | "huge" : "", | 20260506 22:49:31.617 | INFO | "huge_shared" : " 27 GB", | 20260506 22:49:31.617 | INFO | "malloc" : " 14 GB", | 20260506 22:49:31.617 | INFO | "pinned" : " 29 GB" | 20260506 22:49:31.617 | INFO | } | 20260506 22:49:31.617 | INFO | Perf report for the last 5.13443 seconds: | 20260506 22:49:31.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:49:31.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:31.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:31.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:31.617 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260506 22:49:31.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:31.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:31.617 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:49:31.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:31.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:31.617 | INFO | Uploaded 36GiB to GPU in 1.97116s (18.2634GiB/s) | 20260506 22:49:31.617 | INFO | Downloaded 34GiB to GPU in 1.88126s (18.073GiB/s) | 20260506 22:49:36.764 | INFO | Analyzing chunk frames = [5068800,5069312) | 20260506 22:49:36.764 | INFO | MemInfo: { | 20260506 22:49:36.764 | INFO | "gpu" : " 17 GB", | 20260506 22:49:36.764 | INFO | "huge" : "", | 20260506 22:49:36.764 | INFO | "huge_shared" : " 27 GB", | 20260506 22:49:36.764 | INFO | "malloc" : " 14 GB", | 20260506 22:49:36.764 | INFO | "pinned" : " 29 GB" | 20260506 22:49:36.765 | INFO | } | 20260506 22:49:36.765 | INFO | Perf report for the last 5.14787 seconds: | 20260506 22:49:36.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:49:36.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:36.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:36.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:36.765 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 22:49:36.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:36.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:36.765 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:49:36.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:36.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:36.765 | INFO | Uploaded 36GiB to GPU in 1.99427s (18.0517GiB/s) | 20260506 22:49:36.765 | INFO | Downloaded 34GiB to GPU in 1.92062s (17.7026GiB/s) | 20260506 22:49:41.903 | INFO | Analyzing chunk frames = [5069312,5069824) | 20260506 22:49:41.903 | INFO | MemInfo: { | 20260506 22:49:41.903 | INFO | "gpu" : " 17 GB", | 20260506 22:49:41.903 | INFO | "huge" : "", | 20260506 22:49:41.903 | INFO | "huge_shared" : " 27 GB", | 20260506 22:49:41.903 | INFO | "malloc" : " 14 GB", | 20260506 22:49:41.903 | INFO | "pinned" : " 29 GB" | 20260506 22:49:41.903 | INFO | } | 20260506 22:49:41.903 | INFO | Perf report for the last 5.13826 seconds: | 20260506 22:49:41.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:49:41.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:41.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:41.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:41.903 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 22:49:41.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:41.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:41.903 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 22:49:41.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:41.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:41.903 | INFO | Uploaded 36GiB to GPU in 1.9966s (18.0307GiB/s) | 20260506 22:49:41.903 | INFO | Downloaded 34GiB to GPU in 1.96085s (17.3394GiB/s) | 20260506 22:49:47.042 | INFO | Analyzing chunk frames = [5069824,5070336) | 20260506 22:49:47.042 | INFO | MemInfo: { | 20260506 22:49:47.042 | INFO | "gpu" : " 17 GB", | 20260506 22:49:47.042 | INFO | "huge" : "", | 20260506 22:49:47.042 | INFO | "huge_shared" : " 27 GB", | 20260506 22:49:47.043 | INFO | "malloc" : " 14 GB", | 20260506 22:49:47.043 | INFO | "pinned" : " 29 GB" | 20260506 22:49:47.043 | INFO | } | 20260506 22:49:47.043 | INFO | Perf report for the last 5.13971 seconds: | 20260506 22:49:47.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:49:47.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:47.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:47.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:47.043 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 22:49:47.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:47.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:47.043 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:49:47.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:47.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:47.043 | INFO | Uploaded 36GiB to GPU in 1.97717s (18.2079GiB/s) | 20260506 22:49:47.043 | INFO | Downloaded 34GiB to GPU in 1.99976s (17.002GiB/s) | 20260506 22:49:52.182 | INFO | Analyzing chunk frames = [5070336,5070848) | 20260506 22:49:52.182 | INFO | MemInfo: { | 20260506 22:49:52.182 | INFO | "gpu" : " 17 GB", | 20260506 22:49:52.182 | INFO | "huge" : "", | 20260506 22:49:52.182 | INFO | "huge_shared" : " 27 GB", | 20260506 22:49:52.182 | INFO | "malloc" : " 14 GB", | 20260506 22:49:52.182 | INFO | "pinned" : " 29 GB" | 20260506 22:49:52.182 | INFO | } | 20260506 22:49:52.183 | INFO | Perf report for the last 5.13989 seconds: | 20260506 22:49:52.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:49:52.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:52.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:52.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:52.183 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:49:52.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:52.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:52.183 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 22:49:52.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:52.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:52.183 | INFO | Uploaded 36GiB to GPU in 1.97363s (18.2405GiB/s) | 20260506 22:49:52.183 | INFO | Downloaded 34GiB to GPU in 1.99147s (17.0728GiB/s) | 20260506 22:49:52.183 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:49:52.183 | INFO | Total mem usage: 18 GiB | 20260506 22:49:52.183 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:49:52.183 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:49:52.183 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:49:52.183 | INFO | High Water Breakdown: | 20260506 22:49:52.183 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:49:52.183 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:49:52.183 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:49:52.183 | INFO | ------------------------------------------ | 20260506 22:49:52.183 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:49:52.183 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:49:52.183 | INFO | Total mem usage: 26.5 GiB | 20260506 22:49:52.183 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:49:52.183 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:49:52.183 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:49:52.183 | INFO | High Water Breakdown: | 20260506 22:49:52.183 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:49:52.183 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:49:52.183 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:49:52.183 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:49:52.183 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:49:52.183 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:49:52.183 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:49:52.183 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:49:52.183 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:49:52.183 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:49:52.183 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:49:52.183 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:49:52.183 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:49:52.183 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:49:52.183 | INFO | ------------------------------------------ | 20260506 22:49:52.183 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:49:52.183 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:49:52.183 | INFO | Total mem usage: 25.6 GiB | 20260506 22:49:52.183 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:49:52.183 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:49:52.183 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:49:52.183 | INFO | High Water Breakdown: | 20260506 22:49:52.183 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:49:52.183 | INFO | ------------------------------------------ | 20260506 22:49:52.183 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:49:52.183 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:49:52.183 | INFO | Total mem usage: 15.7 GiB | 20260506 22:49:52.183 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:49:52.184 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:49:52.184 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:49:52.184 | INFO | High Water Breakdown: | 20260506 22:49:52.184 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:49:52.184 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:49:52.184 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:49:52.184 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:49:52.184 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:49:52.184 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:49:52.184 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:49:52.184 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:49:52.184 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:49:52.184 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:49:52.184 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:49:52.184 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:49:52.184 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:49:52.184 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:49:52.184 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:49:52.184 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:49:52.184 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:49:52.184 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:49:52.184 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:49:52.184 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:49:52.184 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:49:52.184 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:49:52.184 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:49:52.184 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:49:52.184 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:49:52.184 | INFO | ------------------------------------------ | 20260506 22:49:52.184 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:49:52.184 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:49:52.184 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:49:52.184 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:49:52.184 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.63e+01, 1.35e+01 / 2.52e+00 | 20260506 22:49:52.184 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.05e-01, 1.04e-02 / 4.19e-03 | 20260506 22:49:52.184 | INFO | PerfReport: 11.64%, Baseline -- 3.61e+00 / 4.41e+00, 5.40e-01 / 3.27e-01 | 20260506 22:49:52.184 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.48e+00, 1.80e-01 / 1.08e-01 | 20260506 22:49:52.184 | INFO | PerfReport: 16.07%, DME -- 3.99e+01 / 4.52e+01, 4.24e+00 / 3.09e+00 | 20260506 22:49:52.184 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.39e+00 / 2.12e+00 | 20260506 22:49:52.184 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.19e+00 / 2.77e+00, 3.78e-01 / 2.23e-01 | 20260506 22:49:52.184 | INFO | PerfReport: 3.21%, Metrics -- 9.95e-01 / 1.34e+00, 2.62e-01 / 1.51e-01 | 20260506 22:49:52.184 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.40e-01 / 3.93e-01, 5.28e-02 / 2.31e-02 | 20260506 22:49:52.184 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:49:52.184 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:49:52.184 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:49:52.184 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 22:49:52.184 | INFO | PerfReport: 21.47%, Upload -- 2.08e+01 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260506 22:49:52.184 | INFO | PerfReport: 21.27%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 22:49:52.184 | INFO | PerfReport: 32.10%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 22:49:52.184 | INFO | PerfReport: 1.24%, LockUpload -- 1.20e+00 / 0.00e+00, 4.58e+00 / 0.00e+00 | 20260506 22:49:52.184 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.16e+00 / 0.00e+00 | 20260506 22:49:52.184 | INFO | PerfReport: 22.61%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 22:49:52.184 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260506 22:49:52.280 | INFO | Starting a round of Baz IO | 20260506 22:49:55.413 | INFO | Populating RTMetrics report | 20260506 22:49:55.413 | INFO | RTMetrics report populated | 20260506 22:49:55.413 | INFO | Opening tmp RTMetrics json file | 20260506 22:49:55.414 | INFO | Serializing RTMetrics to json | 20260506 22:49:55.415 | INFO | Serialization done, swapping tmp file now... | 20260506 22:49:55.416 | INFO | RTMetrics done | 20260506 22:49:55.507 | INFO | Finished a round of Baz IO: Wrote 1.9 GiB in 3.226 seconds | 20260506 22:49:55.507 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.59% / 0.00% / 66.40% | 20260506 22:49:55.507 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.46974% / 3.97364e-06% / 3.17891e-05% | 20260506 22:49:55.507 | INFO | Baz IO overall: | 20260506 22:49:55.507 | INFO | BytesWritten : 2660501576150 | 20260506 22:49:55.507 | INFO | NumEvents : 1283313241363 | 20260506 22:49:57.318 | INFO | Analyzing chunk frames = [5070848,5071360) | 20260506 22:49:57.318 | INFO | MemInfo: { | 20260506 22:49:57.318 | INFO | "gpu" : " 17 GB", | 20260506 22:49:57.318 | INFO | "huge" : "", | 20260506 22:49:57.318 | INFO | "huge_shared" : " 27 GB", | 20260506 22:49:57.318 | INFO | "malloc" : " 14 GB", | 20260506 22:49:57.318 | INFO | "pinned" : " 29 GB" | 20260506 22:49:57.318 | INFO | } | 20260506 22:49:57.318 | INFO | Perf report for the last 5.13553 seconds: | 20260506 22:49:57.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:49:57.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:57.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:57.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:57.318 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 22:49:57.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:49:57.318 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260506 22:49:57.318 | INFO | PRE_HQ : 59.7%, 1.8, 56.7ms, 0 | 20260506 22:49:57.318 | INFO | BAZWRITER : 62.4%, 3.5, 111.9ms, 0 | 20260506 22:49:57.318 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:49:57.318 | INFO | Uploaded 36GiB to GPU in 2.02398s (17.7867GiB/s) | 20260506 22:49:57.318 | INFO | Downloaded 38GiB to GPU in 2.16033s (17.5899GiB/s) | 20260506 22:50:02.455 | INFO | Analyzing chunk frames = [5071360,5071872) | 20260506 22:50:02.455 | INFO | MemInfo: { | 20260506 22:50:02.455 | INFO | "gpu" : " 17 GB", | 20260506 22:50:02.455 | INFO | "huge" : "", | 20260506 22:50:02.455 | INFO | "huge_shared" : " 27 GB", | 20260506 22:50:02.456 | INFO | "malloc" : " 14 GB", | 20260506 22:50:02.456 | INFO | "pinned" : " 29 GB" | 20260506 22:50:02.456 | INFO | } | 20260506 22:50:02.456 | INFO | Perf report for the last 5.13763 seconds: | 20260506 22:50:02.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:50:02.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:02.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:02.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:02.456 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:50:02.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:02.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:02.456 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:50:02.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:02.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:02.456 | INFO | Uploaded 36GiB to GPU in 1.99738s (18.0236GiB/s) | 20260506 22:50:02.456 | INFO | Downloaded 34GiB to GPU in 1.92527s (17.6599GiB/s) | 20260506 22:50:07.599 | INFO | Analyzing chunk frames = [5071872,5072384) | 20260506 22:50:07.599 | INFO | MemInfo: { | 20260506 22:50:07.599 | INFO | "gpu" : " 17 GB", | 20260506 22:50:07.599 | INFO | "huge" : "", | 20260506 22:50:07.599 | INFO | "huge_shared" : " 27 GB", | 20260506 22:50:07.599 | INFO | "malloc" : " 14 GB", | 20260506 22:50:07.599 | INFO | "pinned" : " 29 GB" | 20260506 22:50:07.599 | INFO | } | 20260506 22:50:07.599 | INFO | Perf report for the last 5.14371 seconds: | 20260506 22:50:07.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:50:07.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:07.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:07.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:07.600 | INFO | ANALYSIS : 60.3%, 3.0, 93.6ms, 0 | 20260506 22:50:07.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:07.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:07.600 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:50:07.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:07.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:07.600 | INFO | Uploaded 36GiB to GPU in 1.9776s (18.2039GiB/s) | 20260506 22:50:07.600 | INFO | Downloaded 34GiB to GPU in 1.92207s (17.6893GiB/s) | 20260506 22:50:12.739 | INFO | Analyzing chunk frames = [5072384,5072896) | 20260506 22:50:12.739 | INFO | MemInfo: { | 20260506 22:50:12.739 | INFO | "gpu" : " 17 GB", | 20260506 22:50:12.739 | INFO | "huge" : "", | 20260506 22:50:12.739 | INFO | "huge_shared" : " 27 GB", | 20260506 22:50:12.739 | INFO | "malloc" : " 14 GB", | 20260506 22:50:12.739 | INFO | "pinned" : " 29 GB" | 20260506 22:50:12.739 | INFO | } | 20260506 22:50:12.739 | INFO | Perf report for the last 5.13938 seconds: | 20260506 22:50:12.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:50:12.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:12.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:12.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:12.739 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 22:50:12.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:12.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:12.739 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:50:12.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:12.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:12.739 | INFO | Uploaded 36GiB to GPU in 1.95989s (18.3684GiB/s) | 20260506 22:50:12.739 | INFO | Downloaded 34GiB to GPU in 1.87768s (18.1074GiB/s) | 20260506 22:50:17.879 | INFO | Analyzing chunk frames = [5072896,5073408) | 20260506 22:50:17.879 | INFO | MemInfo: { | 20260506 22:50:17.879 | INFO | "gpu" : " 17 GB", | 20260506 22:50:17.879 | INFO | "huge" : "", | 20260506 22:50:17.880 | INFO | "huge_shared" : " 27 GB", | 20260506 22:50:17.880 | INFO | "malloc" : " 14 GB", | 20260506 22:50:17.880 | INFO | "pinned" : " 29 GB" | 20260506 22:50:17.880 | INFO | } | 20260506 22:50:17.880 | INFO | Perf report for the last 5.14081 seconds: | 20260506 22:50:17.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:50:17.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:17.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:17.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:17.880 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 22:50:17.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:17.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:17.880 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:50:17.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:17.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:17.880 | INFO | Uploaded 36GiB to GPU in 2.02295s (17.7958GiB/s) | 20260506 22:50:17.880 | INFO | Downloaded 34GiB to GPU in 1.91918s (17.7159GiB/s) | 20260506 22:50:23.014 | INFO | Analyzing chunk frames = [5073408,5073920) | 20260506 22:50:23.014 | INFO | MemInfo: { | 20260506 22:50:23.014 | INFO | "gpu" : " 17 GB", | 20260506 22:50:23.014 | INFO | "huge" : "", | 20260506 22:50:23.014 | INFO | "huge_shared" : " 27 GB", | 20260506 22:50:23.014 | INFO | "malloc" : " 14 GB", | 20260506 22:50:23.015 | INFO | "pinned" : " 29 GB" | 20260506 22:50:23.015 | INFO | } | 20260506 22:50:23.015 | INFO | Perf report for the last 5.13494 seconds: | 20260506 22:50:23.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:50:23.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:23.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:23.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:23.015 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:50:23.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:23.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:23.015 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260506 22:50:23.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:23.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:23.015 | INFO | Uploaded 36GiB to GPU in 2.02678s (17.7621GiB/s) | 20260506 22:50:23.015 | INFO | Downloaded 34GiB to GPU in 1.94489s (17.4817GiB/s) | 20260506 22:50:28.153 | INFO | Analyzing chunk frames = [5073920,5074432) | 20260506 22:50:28.153 | INFO | MemInfo: { | 20260506 22:50:28.153 | INFO | "gpu" : " 17 GB", | 20260506 22:50:28.153 | INFO | "huge" : "", | 20260506 22:50:28.153 | INFO | "huge_shared" : " 27 GB", | 20260506 22:50:28.153 | INFO | "malloc" : " 14 GB", | 20260506 22:50:28.153 | INFO | "pinned" : " 29 GB" | 20260506 22:50:28.153 | INFO | } | 20260506 22:50:28.153 | INFO | Perf report for the last 5.1387 seconds: | 20260506 22:50:28.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:50:28.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:28.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:28.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:28.153 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 22:50:28.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:28.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:28.153 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:50:28.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:28.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:28.154 | INFO | Uploaded 36GiB to GPU in 1.96154s (18.3529GiB/s) | 20260506 22:50:28.154 | INFO | Downloaded 34GiB to GPU in 1.98188s (17.1554GiB/s) | 20260506 22:50:33.297 | INFO | Analyzing chunk frames = [5074432,5074944) | 20260506 22:50:33.297 | INFO | MemInfo: { | 20260506 22:50:33.297 | INFO | "gpu" : " 17 GB", | 20260506 22:50:33.297 | INFO | "huge" : "", | 20260506 22:50:33.297 | INFO | "huge_shared" : " 27 GB", | 20260506 22:50:33.297 | INFO | "malloc" : " 14 GB", | 20260506 22:50:33.297 | INFO | "pinned" : " 29 GB" | 20260506 22:50:33.297 | INFO | } | 20260506 22:50:33.297 | INFO | Perf report for the last 5.14359 seconds: | 20260506 22:50:33.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:50:33.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:33.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:33.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:33.297 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 22:50:33.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:33.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:33.297 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:50:33.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:33.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:33.297 | INFO | Uploaded 36GiB to GPU in 2.03138s (17.7219GiB/s) | 20260506 22:50:33.297 | INFO | Downloaded 34GiB to GPU in 1.99717s (17.0241GiB/s) | 20260506 22:50:33.395 | INFO | Starting a round of Baz IO | 20260506 22:50:36.524 | INFO | Populating RTMetrics report | 20260506 22:50:36.524 | INFO | RTMetrics report populated | 20260506 22:50:36.524 | INFO | Opening tmp RTMetrics json file | 20260506 22:50:36.525 | INFO | Serializing RTMetrics to json | 20260506 22:50:36.538 | INFO | Serialization done, swapping tmp file now... | 20260506 22:50:36.545 | INFO | RTMetrics done | 20260506 22:50:36.619 | INFO | Finished a round of Baz IO: Wrote 1.9 GiB in 3.224 seconds | 20260506 22:50:36.619 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.59% / 0.00% / 66.41% | 20260506 22:50:36.619 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.467813% / 1.19209e-05% / 6.35783e-05% | 20260506 22:50:36.619 | INFO | Baz IO overall: | 20260506 22:50:36.619 | INFO | BytesWritten : 2662541354243 | 20260506 22:50:36.619 | INFO | NumEvents : 1284285957595 | 20260506 22:50:38.432 | INFO | Analyzing chunk frames = [5074944,5075456) | 20260506 22:50:38.432 | INFO | MemInfo: { | 20260506 22:50:38.432 | INFO | "gpu" : " 17 GB", | 20260506 22:50:38.432 | INFO | "huge" : "", | 20260506 22:50:38.432 | INFO | "huge_shared" : " 27 GB", | 20260506 22:50:38.432 | INFO | "malloc" : " 14 GB", | 20260506 22:50:38.432 | INFO | "pinned" : " 29 GB" | 20260506 22:50:38.432 | INFO | } | 20260506 22:50:38.432 | INFO | Perf report for the last 5.13534 seconds: | 20260506 22:50:38.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:50:38.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:38.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:38.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:38.432 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 22:50:38.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:38.432 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 22:50:38.432 | INFO | PRE_HQ : 59.9%, 1.8, 56.8ms, 0 | 20260506 22:50:38.432 | INFO | BAZWRITER : 62.4%, 3.5, 112.3ms, 0 | 20260506 22:50:38.432 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:50:38.433 | INFO | Uploaded 36GiB to GPU in 2.00972s (17.9129GiB/s) | 20260506 22:50:38.433 | INFO | Downloaded 38GiB to GPU in 2.18029s (17.4288GiB/s) | 20260506 22:50:43.570 | INFO | Analyzing chunk frames = [5075456,5075968) | 20260506 22:50:43.570 | INFO | MemInfo: { | 20260506 22:50:43.570 | INFO | "gpu" : " 17 GB", | 20260506 22:50:43.570 | INFO | "huge" : "", | 20260506 22:50:43.570 | INFO | "huge_shared" : " 27 GB", | 20260506 22:50:43.571 | INFO | "malloc" : " 14 GB", | 20260506 22:50:43.571 | INFO | "pinned" : " 29 GB" | 20260506 22:50:43.571 | INFO | } | 20260506 22:50:43.571 | INFO | Perf report for the last 5.13836 seconds: | 20260506 22:50:43.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:50:43.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:43.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:43.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:43.571 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:50:43.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:43.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:43.571 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 22:50:43.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:43.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:43.571 | INFO | Uploaded 36GiB to GPU in 2.0028s (17.9749GiB/s) | 20260506 22:50:43.571 | INFO | Downloaded 34GiB to GPU in 1.93139s (17.6039GiB/s) | 20260506 22:50:48.709 | INFO | Analyzing chunk frames = [5075968,5076480) | 20260506 22:50:48.709 | INFO | MemInfo: { | 20260506 22:50:48.709 | INFO | "gpu" : " 17 GB", | 20260506 22:50:48.709 | INFO | "huge" : "", | 20260506 22:50:48.709 | INFO | "huge_shared" : " 27 GB", | 20260506 22:50:48.709 | INFO | "malloc" : " 14 GB", | 20260506 22:50:48.709 | INFO | "pinned" : " 29 GB" | 20260506 22:50:48.709 | INFO | } | 20260506 22:50:48.709 | INFO | Perf report for the last 5.13865 seconds: | 20260506 22:50:48.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:50:48.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:48.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:48.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:48.709 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 22:50:48.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:48.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:48.709 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:50:48.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:48.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:48.710 | INFO | Uploaded 36GiB to GPU in 1.96317s (18.3377GiB/s) | 20260506 22:50:48.710 | INFO | Downloaded 34GiB to GPU in 1.92181s (17.6916GiB/s) | 20260506 22:50:53.855 | INFO | Analyzing chunk frames = [5076480,5076992) | 20260506 22:50:53.855 | INFO | MemInfo: { | 20260506 22:50:53.855 | INFO | "gpu" : " 17 GB", | 20260506 22:50:53.855 | INFO | "huge" : "", | 20260506 22:50:53.855 | INFO | "huge_shared" : " 27 GB", | 20260506 22:50:53.855 | INFO | "malloc" : " 14 GB", | 20260506 22:50:53.855 | INFO | "pinned" : " 29 GB" | 20260506 22:50:53.855 | INFO | } | 20260506 22:50:53.855 | INFO | Perf report for the last 5.14574 seconds: | 20260506 22:50:53.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:50:53.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:53.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:53.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:53.855 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 22:50:53.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:53.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:53.855 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:50:53.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:53.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:53.855 | INFO | Uploaded 36GiB to GPU in 1.92828s (18.6695GiB/s) | 20260506 22:50:53.855 | INFO | Downloaded 34GiB to GPU in 1.87823s (18.1021GiB/s) | 20260506 22:50:58.995 | INFO | Analyzing chunk frames = [5076992,5077504) | 20260506 22:50:58.995 | INFO | MemInfo: { | 20260506 22:50:58.995 | INFO | "gpu" : " 17 GB", | 20260506 22:50:58.995 | INFO | "huge" : "", | 20260506 22:50:58.995 | INFO | "huge_shared" : " 27 GB", | 20260506 22:50:58.995 | INFO | "malloc" : " 14 GB", | 20260506 22:50:58.995 | INFO | "pinned" : " 29 GB" | 20260506 22:50:58.995 | INFO | } | 20260506 22:50:58.995 | INFO | Perf report for the last 5.13976 seconds: | 20260506 22:50:58.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:50:58.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:58.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:58.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:58.995 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 22:50:58.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:58.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:58.995 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 22:50:58.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:58.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:50:58.995 | INFO | Uploaded 36GiB to GPU in 1.98122s (18.1706GiB/s) | 20260506 22:50:58.995 | INFO | Downloaded 34GiB to GPU in 1.92453s (17.6666GiB/s) | 20260506 22:51:04.133 | INFO | Analyzing chunk frames = [5077504,5078016) | 20260506 22:51:04.133 | INFO | MemInfo: { | 20260506 22:51:04.133 | INFO | "gpu" : " 17 GB", | 20260506 22:51:04.133 | INFO | "huge" : "", | 20260506 22:51:04.134 | INFO | "huge_shared" : " 27 GB", | 20260506 22:51:04.134 | INFO | "malloc" : " 14 GB", | 20260506 22:51:04.134 | INFO | "pinned" : " 29 GB" | 20260506 22:51:04.134 | INFO | } | 20260506 22:51:04.134 | INFO | Perf report for the last 5.13879 seconds: | 20260506 22:51:04.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:51:04.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:04.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:04.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:04.134 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 22:51:04.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:04.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:04.134 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 22:51:04.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:04.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:04.134 | INFO | Uploaded 36GiB to GPU in 2.02392s (17.7873GiB/s) | 20260506 22:51:04.134 | INFO | Downloaded 34GiB to GPU in 1.95304s (17.4088GiB/s) | 20260506 22:51:09.273 | INFO | Analyzing chunk frames = [5078016,5078528) | 20260506 22:51:09.273 | INFO | MemInfo: { | 20260506 22:51:09.273 | INFO | "gpu" : " 17 GB", | 20260506 22:51:09.273 | INFO | "huge" : "", | 20260506 22:51:09.273 | INFO | "huge_shared" : " 27 GB", | 20260506 22:51:09.273 | INFO | "malloc" : " 14 GB", | 20260506 22:51:09.273 | INFO | "pinned" : " 29 GB" | 20260506 22:51:09.273 | INFO | } | 20260506 22:51:09.273 | INFO | Perf report for the last 5.13919 seconds: | 20260506 22:51:09.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:51:09.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:09.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:09.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:09.273 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:51:09.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:09.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:09.273 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 22:51:09.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:09.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:09.273 | INFO | Uploaded 36GiB to GPU in 1.99267s (18.0662GiB/s) | 20260506 22:51:09.273 | INFO | Downloaded 34GiB to GPU in 1.99546s (17.0386GiB/s) | 20260506 22:51:14.409 | INFO | Analyzing chunk frames = [5078528,5079040) | 20260506 22:51:14.409 | INFO | MemInfo: { | 20260506 22:51:14.409 | INFO | "gpu" : " 17 GB", | 20260506 22:51:14.409 | INFO | "huge" : "", | 20260506 22:51:14.409 | INFO | "huge_shared" : " 27 GB", | 20260506 22:51:14.409 | INFO | "malloc" : " 14 GB", | 20260506 22:51:14.409 | INFO | "pinned" : " 29 GB" | 20260506 22:51:14.409 | INFO | } | 20260506 22:51:14.409 | INFO | Perf report for the last 5.13657 seconds: | 20260506 22:51:14.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:51:14.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:14.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:14.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:14.409 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 22:51:14.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:14.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:14.409 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:51:14.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:14.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:14.410 | INFO | Uploaded 36GiB to GPU in 2.0424s (17.6263GiB/s) | 20260506 22:51:14.410 | INFO | Downloaded 34GiB to GPU in 2.00541s (16.9542GiB/s) | 20260506 22:51:14.410 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:51:14.410 | INFO | Total mem usage: 18 GiB | 20260506 22:51:14.410 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:51:14.410 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 22:51:14.410 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:51:14.410 | INFO | High Water Breakdown: | 20260506 22:51:14.410 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:51:14.410 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:51:14.410 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:51:14.410 | INFO | ------------------------------------------ | 20260506 22:51:14.410 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:51:14.410 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:51:14.410 | INFO | Total mem usage: 26.5 GiB | 20260506 22:51:14.410 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:51:14.410 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:51:14.410 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:51:14.410 | INFO | High Water Breakdown: | 20260506 22:51:14.410 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:51:14.410 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:51:14.410 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:51:14.410 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:51:14.410 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:51:14.410 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:51:14.410 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:51:14.410 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:51:14.410 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:51:14.410 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:51:14.410 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:51:14.410 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:51:14.410 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:51:14.410 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:51:14.410 | INFO | ------------------------------------------ | 20260506 22:51:14.410 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:51:14.410 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:51:14.410 | INFO | Total mem usage: 25.6 GiB | 20260506 22:51:14.410 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:51:14.410 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:51:14.410 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:51:14.410 | INFO | High Water Breakdown: | 20260506 22:51:14.410 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:51:14.410 | INFO | ------------------------------------------ | 20260506 22:51:14.410 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:51:14.410 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:51:14.410 | INFO | Total mem usage: 15.7 GiB | 20260506 22:51:14.410 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:51:14.410 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:51:14.410 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:51:14.410 | INFO | High Water Breakdown: | 20260506 22:51:14.410 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:51:14.410 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:51:14.410 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:51:14.410 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:51:14.410 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:51:14.410 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:51:14.410 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:51:14.410 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:51:14.410 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:51:14.410 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:51:14.410 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:51:14.411 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:51:14.411 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:51:14.411 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:51:14.411 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:51:14.411 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:51:14.411 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:51:14.411 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:51:14.411 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:51:14.411 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:51:14.411 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:51:14.411 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:51:14.411 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:51:14.411 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:51:14.411 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:51:14.411 | INFO | ------------------------------------------ | 20260506 22:51:14.411 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:51:14.411 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:51:14.411 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:51:14.411 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:51:14.411 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.56e+01, 1.32e+01 / 2.06e+00 | 20260506 22:51:14.411 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.97e-03 / 4.35e-03 | 20260506 22:51:14.411 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.44e+00, 5.43e-01 / 3.01e-01 | 20260506 22:51:14.411 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.50e+00, 1.81e-01 / 1.10e-01 | 20260506 22:51:14.411 | INFO | PerfReport: 16.06%, DME -- 3.98e+01 / 4.45e+01, 3.89e+00 / 2.27e+00 | 20260506 22:51:14.411 | INFO | PerfReport: 57.32%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.36e+00 / 1.59e+00 | 20260506 22:51:14.411 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.20e+00 / 2.72e+00, 3.78e-01 / 2.30e-01 | 20260506 22:51:14.411 | INFO | PerfReport: 3.21%, Metrics -- 9.97e-01 / 1.35e+00, 2.66e-01 / 1.48e-01 | 20260506 22:51:14.411 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.40e-01 / 3.87e-01, 5.25e-02 / 2.15e-02 | 20260506 22:51:14.411 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:51:14.411 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:51:14.411 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:51:14.411 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 22:51:14.411 | INFO | PerfReport: 21.50%, Upload -- 2.08e+01 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260506 22:51:14.411 | INFO | PerfReport: 21.22%, Download -- 2.05e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 22:51:14.411 | INFO | PerfReport: 32.07%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 22:51:14.411 | INFO | PerfReport: 1.30%, LockUpload -- 1.25e+00 / 0.00e+00, 5.10e+00 / 0.00e+00 | 20260506 22:51:14.411 | INFO | PerfReport: 1.25%, LockDownload -- 1.21e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260506 22:51:14.411 | INFO | PerfReport: 22.65%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 22:51:14.411 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260506 22:51:14.516 | INFO | Starting a round of Baz IO | 20260506 22:51:17.642 | INFO | Populating RTMetrics report | 20260506 22:51:17.643 | INFO | RTMetrics report populated | 20260506 22:51:17.643 | INFO | Opening tmp RTMetrics json file | 20260506 22:51:17.644 | INFO | Serializing RTMetrics to json | 20260506 22:51:17.650 | INFO | Serialization done, swapping tmp file now... | 20260506 22:51:17.663 | INFO | RTMetrics done | 20260506 22:51:17.741 | INFO | Finished a round of Baz IO: Wrote 1.9 GiB in 3.225 seconds | 20260506 22:51:17.741 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.59% / 0.00% / 66.41% | 20260506 22:51:17.741 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.467261% / 0% / 5.96046e-05% | 20260506 22:51:17.741 | INFO | Baz IO overall: | 20260506 22:51:17.741 | INFO | BytesWritten : 2664581797751 | 20260506 22:51:17.741 | INFO | NumEvents : 1285259190925 | 20260506 22:51:19.551 | INFO | Analyzing chunk frames = [5079040,5079552) | 20260506 22:51:19.551 | INFO | MemInfo: { | 20260506 22:51:19.551 | INFO | "gpu" : " 17 GB", | 20260506 22:51:19.551 | INFO | "huge" : "", | 20260506 22:51:19.551 | INFO | "huge_shared" : " 27 GB", | 20260506 22:51:19.551 | INFO | "malloc" : " 14 GB", | 20260506 22:51:19.551 | INFO | "pinned" : " 29 GB" | 20260506 22:51:19.551 | INFO | } | 20260506 22:51:19.551 | INFO | Perf report for the last 5.14195 seconds: | 20260506 22:51:19.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:51:19.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:19.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:19.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:19.551 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 22:51:19.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:19.552 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 22:51:19.552 | INFO | PRE_HQ : 59.5%, 1.8, 56.7ms, 0 | 20260506 22:51:19.552 | INFO | BAZWRITER : 62.4%, 3.5, 110.8ms, 0 | 20260506 22:51:19.552 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:51:19.552 | INFO | Uploaded 36GiB to GPU in 2.02893s (17.7433GiB/s) | 20260506 22:51:19.552 | INFO | Downloaded 38GiB to GPU in 2.17386s (17.4804GiB/s) | 20260506 22:51:24.684 | INFO | Analyzing chunk frames = [5079552,5080064) | 20260506 22:51:24.684 | INFO | MemInfo: { | 20260506 22:51:24.684 | INFO | "gpu" : " 17 GB", | 20260506 22:51:24.684 | INFO | "huge" : "", | 20260506 22:51:24.685 | INFO | "huge_shared" : " 27 GB", | 20260506 22:51:24.685 | INFO | "malloc" : " 14 GB", | 20260506 22:51:24.685 | INFO | "pinned" : " 29 GB" | 20260506 22:51:24.685 | INFO | } | 20260506 22:51:24.685 | INFO | Perf report for the last 5.13336 seconds: | 20260506 22:51:24.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:51:24.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:24.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:24.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:24.685 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:51:24.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:24.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:24.685 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 22:51:24.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:24.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:24.685 | INFO | Uploaded 36GiB to GPU in 2.01094s (17.9021GiB/s) | 20260506 22:51:24.685 | INFO | Downloaded 34GiB to GPU in 1.92806s (17.6343GiB/s) | 20260506 22:51:29.827 | INFO | Analyzing chunk frames = [5080064,5080576) | 20260506 22:51:29.827 | INFO | MemInfo: { | 20260506 22:51:29.827 | INFO | "gpu" : " 17 GB", | 20260506 22:51:29.827 | INFO | "huge" : "", | 20260506 22:51:29.827 | INFO | "huge_shared" : " 27 GB", | 20260506 22:51:29.827 | INFO | "malloc" : " 14 GB", | 20260506 22:51:29.827 | INFO | "pinned" : " 29 GB" | 20260506 22:51:29.827 | INFO | } | 20260506 22:51:29.827 | INFO | Perf report for the last 5.14224 seconds: | 20260506 22:51:29.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:51:29.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:29.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:29.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:29.827 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 22:51:29.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:29.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:29.827 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:51:29.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:29.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:29.827 | INFO | Uploaded 36GiB to GPU in 1.98403s (18.1449GiB/s) | 20260506 22:51:29.827 | INFO | Downloaded 34GiB to GPU in 1.91585s (17.7467GiB/s) | 20260506 22:51:34.969 | INFO | Analyzing chunk frames = [5080576,5081088) | 20260506 22:51:34.969 | INFO | MemInfo: { | 20260506 22:51:34.969 | INFO | "gpu" : " 17 GB", | 20260506 22:51:34.969 | INFO | "huge" : "", | 20260506 22:51:34.969 | INFO | "huge_shared" : " 27 GB", | 20260506 22:51:34.969 | INFO | "malloc" : " 14 GB", | 20260506 22:51:34.969 | INFO | "pinned" : " 29 GB" | 20260506 22:51:34.969 | INFO | } | 20260506 22:51:34.969 | INFO | Perf report for the last 5.1423 seconds: | 20260506 22:51:34.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:51:34.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:34.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:34.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:34.969 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 22:51:34.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:34.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:34.969 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:51:34.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:34.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:34.970 | INFO | Uploaded 36GiB to GPU in 1.97118s (18.2632GiB/s) | 20260506 22:51:34.970 | INFO | Downloaded 34GiB to GPU in 1.87603s (18.1234GiB/s) | 20260506 22:51:40.110 | INFO | Analyzing chunk frames = [5081088,5081600) | 20260506 22:51:40.110 | INFO | MemInfo: { | 20260506 22:51:40.110 | INFO | "gpu" : " 17 GB", | 20260506 22:51:40.110 | INFO | "huge" : "", | 20260506 22:51:40.110 | INFO | "huge_shared" : " 27 GB", | 20260506 22:51:40.110 | INFO | "malloc" : " 14 GB", | 20260506 22:51:40.110 | INFO | "pinned" : " 29 GB" | 20260506 22:51:40.110 | INFO | } | 20260506 22:51:40.110 | INFO | Perf report for the last 5.14054 seconds: | 20260506 22:51:40.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:51:40.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:40.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:40.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:40.110 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 22:51:40.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:40.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:40.110 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:51:40.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:40.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:40.110 | INFO | Uploaded 36GiB to GPU in 1.99854s (18.0132GiB/s) | 20260506 22:51:40.110 | INFO | Downloaded 34GiB to GPU in 1.94105s (17.5162GiB/s) | 20260506 22:51:45.246 | INFO | Analyzing chunk frames = [5081600,5082112) | 20260506 22:51:45.246 | INFO | MemInfo: { | 20260506 22:51:45.246 | INFO | "gpu" : " 17 GB", | 20260506 22:51:45.246 | INFO | "huge" : "", | 20260506 22:51:45.246 | INFO | "huge_shared" : " 27 GB", | 20260506 22:51:45.246 | INFO | "malloc" : " 14 GB", | 20260506 22:51:45.246 | INFO | "pinned" : " 29 GB" | 20260506 22:51:45.246 | INFO | } | 20260506 22:51:45.246 | INFO | Perf report for the last 5.1366 seconds: | 20260506 22:51:45.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:51:45.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:45.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:45.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:45.247 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 22:51:45.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:45.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:45.247 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 22:51:45.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:45.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:45.247 | INFO | Uploaded 36GiB to GPU in 2.04238s (17.6265GiB/s) | 20260506 22:51:45.247 | INFO | Downloaded 34GiB to GPU in 1.94193s (17.5084GiB/s) | 20260506 22:51:50.385 | INFO | Analyzing chunk frames = [5082112,5082624) | 20260506 22:51:50.385 | INFO | MemInfo: { | 20260506 22:51:50.385 | INFO | "gpu" : " 17 GB", | 20260506 22:51:50.385 | INFO | "huge" : "", | 20260506 22:51:50.385 | INFO | "huge_shared" : " 27 GB", | 20260506 22:51:50.385 | INFO | "malloc" : " 14 GB", | 20260506 22:51:50.385 | INFO | "pinned" : " 29 GB" | 20260506 22:51:50.385 | INFO | } | 20260506 22:51:50.385 | INFO | Perf report for the last 5.13834 seconds: | 20260506 22:51:50.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:51:50.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:50.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:50.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:50.385 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:51:50.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:50.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:50.385 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 22:51:50.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:50.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:50.385 | INFO | Uploaded 36GiB to GPU in 1.98196s (18.1638GiB/s) | 20260506 22:51:50.385 | INFO | Downloaded 34GiB to GPU in 2.00444s (16.9624GiB/s) | 20260506 22:51:55.522 | INFO | Analyzing chunk frames = [5082624,5083136) | 20260506 22:51:55.522 | INFO | MemInfo: { | 20260506 22:51:55.522 | INFO | "gpu" : " 17 GB", | 20260506 22:51:55.522 | INFO | "huge" : "", | 20260506 22:51:55.522 | INFO | "huge_shared" : " 27 GB", | 20260506 22:51:55.523 | INFO | "malloc" : " 14 GB", | 20260506 22:51:55.523 | INFO | "pinned" : " 29 GB" | 20260506 22:51:55.523 | INFO | } | 20260506 22:51:55.523 | INFO | Perf report for the last 5.1379 seconds: | 20260506 22:51:55.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:51:55.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:55.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:55.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:55.523 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 22:51:55.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:55.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:55.523 | INFO | PRE_HQ : 31.6%, 1.0, 16.4ms, 0 | 20260506 22:51:55.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:55.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:51:55.523 | INFO | Uploaded 36GiB to GPU in 2.00847s (17.9241GiB/s) | 20260506 22:51:55.523 | INFO | Downloaded 34GiB to GPU in 2.00308s (16.9739GiB/s) | 20260506 22:51:55.621 | INFO | Starting a round of Baz IO | 20260506 22:51:58.755 | INFO | Populating RTMetrics report | 20260506 22:51:58.756 | INFO | RTMetrics report populated | 20260506 22:51:58.756 | INFO | Opening tmp RTMetrics json file | 20260506 22:51:58.758 | INFO | Serializing RTMetrics to json | 20260506 22:51:58.775 | INFO | Serialization done, swapping tmp file now... | 20260506 22:51:58.776 | INFO | RTMetrics done | 20260506 22:51:58.850 | INFO | Finished a round of Baz IO: Wrote 1.9 GiB in 3.229 seconds | 20260506 22:51:58.850 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.59% / 0.00% / 66.41% | 20260506 22:51:58.850 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.465643% / 3.97364e-06% / 7.94729e-05% | 20260506 22:51:58.850 | INFO | Baz IO overall: | 20260506 22:51:58.850 | INFO | BytesWritten : 2666621627075 | 20260506 22:51:58.850 | INFO | NumEvents : 1286232320271 | 20260506 22:52:00.661 | INFO | Analyzing chunk frames = [5083136,5083648) | 20260506 22:52:00.661 | INFO | MemInfo: { | 20260506 22:52:00.661 | INFO | "gpu" : " 17 GB", | 20260506 22:52:00.661 | INFO | "huge" : "", | 20260506 22:52:00.661 | INFO | "huge_shared" : " 27 GB", | 20260506 22:52:00.661 | INFO | "malloc" : " 14 GB", | 20260506 22:52:00.661 | INFO | "pinned" : " 29 GB" | 20260506 22:52:00.661 | INFO | } | 20260506 22:52:00.661 | INFO | Perf report for the last 5.13867 seconds: | 20260506 22:52:00.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:52:00.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:00.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:00.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:00.661 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260506 22:52:00.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:00.662 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 22:52:00.662 | INFO | PRE_HQ : 59.8%, 1.9, 57.6ms, 0 | 20260506 22:52:00.662 | INFO | BAZWRITER : 62.2%, 3.5, 111.4ms, 0 | 20260506 22:52:00.662 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:52:00.662 | INFO | Uploaded 36GiB to GPU in 2.03097s (17.7255GiB/s) | 20260506 22:52:00.662 | INFO | Downloaded 38GiB to GPU in 2.17042s (17.5081GiB/s) | 20260506 22:52:05.806 | INFO | Analyzing chunk frames = [5083648,5084160) | 20260506 22:52:05.806 | INFO | MemInfo: { | 20260506 22:52:05.806 | INFO | "gpu" : " 17 GB", | 20260506 22:52:05.806 | INFO | "huge" : "", | 20260506 22:52:05.806 | INFO | "huge_shared" : " 27 GB", | 20260506 22:52:05.806 | INFO | "malloc" : " 14 GB", | 20260506 22:52:05.806 | INFO | "pinned" : " 29 GB" | 20260506 22:52:05.806 | INFO | } | 20260506 22:52:05.806 | INFO | Perf report for the last 5.145 seconds: | 20260506 22:52:05.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:52:05.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:05.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:05.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:05.807 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 22:52:05.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:05.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:05.807 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:52:05.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:05.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:05.807 | INFO | Uploaded 36GiB to GPU in 1.99561s (18.0396GiB/s) | 20260506 22:52:05.807 | INFO | Downloaded 34GiB to GPU in 1.93279s (17.5912GiB/s) | 20260506 22:52:10.947 | INFO | Analyzing chunk frames = [5084160,5084672) | 20260506 22:52:10.947 | INFO | MemInfo: { | 20260506 22:52:10.947 | INFO | "gpu" : " 17 GB", | 20260506 22:52:10.947 | INFO | "huge" : "", | 20260506 22:52:10.947 | INFO | "huge_shared" : " 27 GB", | 20260506 22:52:10.947 | INFO | "malloc" : " 14 GB", | 20260506 22:52:10.947 | INFO | "pinned" : " 29 GB" | 20260506 22:52:10.947 | INFO | } | 20260506 22:52:10.947 | INFO | Perf report for the last 5.14066 seconds: | 20260506 22:52:10.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:52:10.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:10.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:10.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:10.947 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 22:52:10.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:10.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:10.947 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:52:10.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:10.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:10.947 | INFO | Uploaded 36GiB to GPU in 1.95217s (18.441GiB/s) | 20260506 22:52:10.947 | INFO | Downloaded 34GiB to GPU in 1.93027s (17.6141GiB/s) | 20260506 22:52:16.083 | INFO | Analyzing chunk frames = [5084672,5085184) | 20260506 22:52:16.083 | INFO | MemInfo: { | 20260506 22:52:16.083 | INFO | "gpu" : " 17 GB", | 20260506 22:52:16.083 | INFO | "huge" : "", | 20260506 22:52:16.083 | INFO | "huge_shared" : " 27 GB", | 20260506 22:52:16.083 | INFO | "malloc" : " 14 GB", | 20260506 22:52:16.083 | INFO | "pinned" : " 29 GB" | 20260506 22:52:16.083 | INFO | } | 20260506 22:52:16.083 | INFO | Perf report for the last 5.13624 seconds: | 20260506 22:52:16.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:52:16.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:16.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:16.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:16.083 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 22:52:16.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:16.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:16.083 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:52:16.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:16.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:16.084 | INFO | Uploaded 36GiB to GPU in 2.004s (17.9641GiB/s) | 20260506 22:52:16.084 | INFO | Downloaded 34GiB to GPU in 1.91139s (17.7881GiB/s) | 20260506 22:52:21.223 | INFO | Analyzing chunk frames = [5085184,5085696) | 20260506 22:52:21.223 | INFO | MemInfo: { | 20260506 22:52:21.223 | INFO | "gpu" : " 17 GB", | 20260506 22:52:21.223 | INFO | "huge" : "", | 20260506 22:52:21.223 | INFO | "huge_shared" : " 27 GB", | 20260506 22:52:21.223 | INFO | "malloc" : " 14 GB", | 20260506 22:52:21.223 | INFO | "pinned" : " 29 GB" | 20260506 22:52:21.223 | INFO | } | 20260506 22:52:21.223 | INFO | Perf report for the last 5.13984 seconds: | 20260506 22:52:21.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:52:21.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:21.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:21.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:21.223 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 22:52:21.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:21.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:21.223 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:52:21.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:21.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:21.224 | INFO | Uploaded 36GiB to GPU in 2.00316s (17.9716GiB/s) | 20260506 22:52:21.224 | INFO | Downloaded 34GiB to GPU in 1.94513s (17.4795GiB/s) | 20260506 22:52:26.359 | INFO | Analyzing chunk frames = [5085696,5086208) | 20260506 22:52:26.359 | INFO | MemInfo: { | 20260506 22:52:26.359 | INFO | "gpu" : " 17 GB", | 20260506 22:52:26.359 | INFO | "huge" : "", | 20260506 22:52:26.359 | INFO | "huge_shared" : " 27 GB", | 20260506 22:52:26.359 | INFO | "malloc" : " 14 GB", | 20260506 22:52:26.359 | INFO | "pinned" : " 29 GB" | 20260506 22:52:26.359 | INFO | } | 20260506 22:52:26.359 | INFO | Perf report for the last 5.13583 seconds: | 20260506 22:52:26.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:52:26.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:26.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:26.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:26.359 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 22:52:26.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:26.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:26.359 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:52:26.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:26.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:26.359 | INFO | Uploaded 36GiB to GPU in 2.01125s (17.8993GiB/s) | 20260506 22:52:26.359 | INFO | Downloaded 34GiB to GPU in 1.94617s (17.4702GiB/s) | 20260506 22:52:31.496 | INFO | Analyzing chunk frames = [5086208,5086720) | 20260506 22:52:31.496 | INFO | MemInfo: { | 20260506 22:52:31.496 | INFO | "gpu" : " 17 GB", | 20260506 22:52:31.496 | INFO | "huge" : "", | 20260506 22:52:31.496 | INFO | "huge_shared" : " 27 GB", | 20260506 22:52:31.496 | INFO | "malloc" : " 14 GB", | 20260506 22:52:31.497 | INFO | "pinned" : " 29 GB" | 20260506 22:52:31.497 | INFO | } | 20260506 22:52:31.497 | INFO | Perf report for the last 5.13758 seconds: | 20260506 22:52:31.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:52:31.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:31.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:31.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:31.497 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:52:31.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:31.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:31.497 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 22:52:31.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:31.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:31.497 | INFO | Uploaded 36GiB to GPU in 1.93402s (18.6141GiB/s) | 20260506 22:52:31.497 | INFO | Downloaded 34GiB to GPU in 1.97963s (17.1749GiB/s) | 20260506 22:52:36.640 | INFO | Analyzing chunk frames = [5086720,5087232) | 20260506 22:52:36.640 | INFO | MemInfo: { | 20260506 22:52:36.640 | INFO | "gpu" : " 17 GB", | 20260506 22:52:36.640 | INFO | "huge" : "", | 20260506 22:52:36.640 | INFO | "huge_shared" : " 27 GB", | 20260506 22:52:36.640 | INFO | "malloc" : " 14 GB", | 20260506 22:52:36.640 | INFO | "pinned" : " 29 GB" | 20260506 22:52:36.640 | INFO | } | 20260506 22:52:36.640 | INFO | Perf report for the last 5.14378 seconds: | 20260506 22:52:36.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:52:36.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:36.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:36.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:36.640 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:52:36.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:36.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:36.640 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:52:36.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:36.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:36.641 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:52:36.641 | INFO | Total mem usage: 18 GiB | 20260506 22:52:36.641 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:52:36.641 | INFO | Uploaded 36GiB to GPU in 1.97871s (18.1936GiB/s) | 20260506 22:52:36.641 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 22:52:36.641 | INFO | Downloaded 34GiB to GPU in 1.98203s (17.1541GiB/s) | 20260506 22:52:36.641 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:52:36.641 | INFO | High Water Breakdown: | 20260506 22:52:36.641 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:52:36.641 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:52:36.641 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:52:36.641 | INFO | ------------------------------------------ | 20260506 22:52:36.641 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:52:36.641 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:52:36.641 | INFO | Total mem usage: 26.5 GiB | 20260506 22:52:36.641 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:52:36.641 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:52:36.641 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:52:36.641 | INFO | High Water Breakdown: | 20260506 22:52:36.641 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:52:36.641 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:52:36.641 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:52:36.641 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:52:36.641 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:52:36.641 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:52:36.641 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:52:36.641 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:52:36.641 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:52:36.641 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:52:36.641 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:52:36.641 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:52:36.642 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:52:36.642 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:52:36.642 | INFO | ------------------------------------------ | 20260506 22:52:36.642 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:52:36.642 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:52:36.642 | INFO | Total mem usage: 25.6 GiB | 20260506 22:52:36.642 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:52:36.642 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:52:36.642 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:52:36.642 | INFO | High Water Breakdown: | 20260506 22:52:36.642 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:52:36.642 | INFO | ------------------------------------------ | 20260506 22:52:36.642 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:52:36.642 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:52:36.642 | INFO | Total mem usage: 15.7 GiB | 20260506 22:52:36.642 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:52:36.642 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:52:36.642 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:52:36.642 | INFO | High Water Breakdown: | 20260506 22:52:36.642 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:52:36.642 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:52:36.642 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:52:36.642 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:52:36.642 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:52:36.642 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:52:36.642 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:52:36.642 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:52:36.642 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:52:36.642 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:52:36.642 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:52:36.642 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:52:36.642 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:52:36.642 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:52:36.642 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:52:36.642 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:52:36.642 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:52:36.642 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:52:36.642 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:52:36.642 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:52:36.642 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:52:36.642 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:52:36.642 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:52:36.642 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:52:36.642 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:52:36.642 | INFO | ------------------------------------------ | 20260506 22:52:36.642 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:52:36.642 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:52:36.642 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:52:36.642 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:52:36.642 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.59e+01, 1.35e+01 / 2.42e+00 | 20260506 22:52:36.642 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 1.01e-01, 1.05e-02 / 6.48e-03 | 20260506 22:52:36.642 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.30e+00, 5.46e-01 / 3.86e-01 | 20260506 22:52:36.642 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.44e+00, 1.79e-01 / 1.37e-01 | 20260506 22:52:36.642 | INFO | PerfReport: 16.06%, DME -- 3.99e+01 / 4.48e+01, 4.17e+00 / 2.68e+00 | 20260506 22:52:36.642 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.33e+00 / 1.79e+00 | 20260506 22:52:36.642 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.20e+00 / 2.75e+00, 3.84e-01 / 1.99e-01 | 20260506 22:52:36.642 | INFO | PerfReport: 3.22%, Metrics -- 9.99e-01 / 1.35e+00, 2.63e-01 / 1.42e-01 | 20260506 22:52:36.642 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.40e-01 / 3.90e-01, 5.30e-02 / 2.29e-02 | 20260506 22:52:36.642 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:52:36.642 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:52:36.642 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:52:36.642 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 22:52:36.642 | INFO | PerfReport: 21.49%, Upload -- 2.08e+01 / 0.00e+00, 6.93e+00 / 0.00e+00 | 20260506 22:52:36.642 | INFO | PerfReport: 21.26%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 22:52:36.642 | INFO | PerfReport: 32.08%, Compute -- 3.10e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 22:52:36.642 | INFO | PerfReport: 1.27%, LockUpload -- 1.23e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260506 22:52:36.642 | INFO | PerfReport: 1.27%, LockDownload -- 1.23e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260506 22:52:36.642 | INFO | PerfReport: 22.62%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 22:52:36.642 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260506 22:52:36.739 | INFO | Starting a round of Baz IO | 20260506 22:52:39.871 | INFO | Populating RTMetrics report | 20260506 22:52:39.871 | INFO | RTMetrics report populated | 20260506 22:52:39.871 | INFO | Opening tmp RTMetrics json file | 20260506 22:52:39.872 | INFO | Serializing RTMetrics to json | 20260506 22:52:39.873 | INFO | Serialization done, swapping tmp file now... | 20260506 22:52:39.875 | INFO | RTMetrics done | 20260506 22:52:39.964 | INFO | Finished a round of Baz IO: Wrote 1.898 GiB in 3.225 seconds | 20260506 22:52:39.964 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.58% / 0.00% / 66.41% | 20260506 22:52:39.964 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.464948% / 7.94729e-06% / 7.54992e-05% | 20260506 22:52:39.964 | INFO | Baz IO overall: | 20260506 22:52:39.964 | INFO | BytesWritten : 2668659743621 | 20260506 22:52:39.964 | INFO | NumEvents : 1287204210798 | 20260506 22:52:41.780 | INFO | Analyzing chunk frames = [5087232,5087744) | 20260506 22:52:41.780 | INFO | MemInfo: { | 20260506 22:52:41.780 | INFO | "gpu" : " 17 GB", | 20260506 22:52:41.780 | INFO | "huge" : "", | 20260506 22:52:41.780 | INFO | "huge_shared" : " 27 GB", | 20260506 22:52:41.780 | INFO | "malloc" : " 14 GB", | 20260506 22:52:41.780 | INFO | "pinned" : " 29 GB" | 20260506 22:52:41.780 | INFO | } | 20260506 22:52:41.780 | INFO | Perf report for the last 5.13961 seconds: | 20260506 22:52:41.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:52:41.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:41.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:41.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:41.780 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 22:52:41.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:41.780 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260506 22:52:41.780 | INFO | PRE_HQ : 59.6%, 1.8, 56.5ms, 0 | 20260506 22:52:41.780 | INFO | BAZWRITER : 62.2%, 3.5, 111.2ms, 0 | 20260506 22:52:41.780 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:52:41.780 | INFO | Uploaded 36GiB to GPU in 2.0208s (17.8147GiB/s) | 20260506 22:52:41.781 | INFO | Downloaded 38GiB to GPU in 2.1814s (17.42GiB/s) | 20260506 22:52:46.922 | INFO | Analyzing chunk frames = [5087744,5088256) | 20260506 22:52:46.922 | INFO | MemInfo: { | 20260506 22:52:46.922 | INFO | "gpu" : " 17 GB", | 20260506 22:52:46.922 | INFO | "huge" : "", | 20260506 22:52:46.922 | INFO | "huge_shared" : " 27 GB", | 20260506 22:52:46.922 | INFO | "malloc" : " 14 GB", | 20260506 22:52:46.922 | INFO | "pinned" : " 29 GB" | 20260506 22:52:46.922 | INFO | } | 20260506 22:52:46.922 | INFO | Perf report for the last 5.14206 seconds: | 20260506 22:52:46.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:52:46.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:46.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:46.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:46.922 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:52:46.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:46.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:46.922 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 22:52:46.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:46.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:46.922 | INFO | Uploaded 36GiB to GPU in 2.00577s (17.9482GiB/s) | 20260506 22:52:46.922 | INFO | Downloaded 34GiB to GPU in 1.92703s (17.6437GiB/s) | 20260506 22:52:52.064 | INFO | Analyzing chunk frames = [5088256,5088768) | 20260506 22:52:52.064 | INFO | MemInfo: { | 20260506 22:52:52.064 | INFO | "gpu" : " 17 GB", | 20260506 22:52:52.064 | INFO | "huge" : "", | 20260506 22:52:52.064 | INFO | "huge_shared" : " 27 GB", | 20260506 22:52:52.064 | INFO | "malloc" : " 14 GB", | 20260506 22:52:52.064 | INFO | "pinned" : " 29 GB" | 20260506 22:52:52.064 | INFO | } | 20260506 22:52:52.064 | INFO | Perf report for the last 5.14184 seconds: | 20260506 22:52:52.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:52:52.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:52.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:52.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:52.064 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 22:52:52.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:52.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:52.064 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:52:52.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:52.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:52.064 | INFO | Uploaded 36GiB to GPU in 1.99721s (18.0252GiB/s) | 20260506 22:52:52.064 | INFO | Downloaded 34GiB to GPU in 1.9371s (17.552GiB/s) | 20260506 22:52:57.201 | INFO | Analyzing chunk frames = [5088768,5089280) | 20260506 22:52:57.201 | INFO | MemInfo: { | 20260506 22:52:57.201 | INFO | "gpu" : " 17 GB", | 20260506 22:52:57.201 | INFO | "huge" : "", | 20260506 22:52:57.201 | INFO | "huge_shared" : " 27 GB", | 20260506 22:52:57.202 | INFO | "malloc" : " 14 GB", | 20260506 22:52:57.202 | INFO | "pinned" : " 29 GB" | 20260506 22:52:57.202 | INFO | } | 20260506 22:52:57.202 | INFO | Perf report for the last 5.13762 seconds: | 20260506 22:52:57.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:52:57.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:57.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:57.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:57.202 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 22:52:57.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:57.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:57.202 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:52:57.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:57.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:52:57.202 | INFO | Uploaded 36GiB to GPU in 1.973s (18.2463GiB/s) | 20260506 22:52:57.202 | INFO | Downloaded 34GiB to GPU in 1.90857s (17.8144GiB/s) | 20260506 22:53:02.339 | INFO | Analyzing chunk frames = [5089280,5089792) | 20260506 22:53:02.339 | INFO | MemInfo: { | 20260506 22:53:02.339 | INFO | "gpu" : " 17 GB", | 20260506 22:53:02.339 | INFO | "huge" : "", | 20260506 22:53:02.339 | INFO | "huge_shared" : " 27 GB", | 20260506 22:53:02.339 | INFO | "malloc" : " 14 GB", | 20260506 22:53:02.339 | INFO | "pinned" : " 29 GB" | 20260506 22:53:02.339 | INFO | } | 20260506 22:53:02.339 | INFO | Perf report for the last 5.13767 seconds: | 20260506 22:53:02.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:53:02.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:02.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:02.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:02.339 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 22:53:02.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:02.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:02.339 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:53:02.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:02.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:02.340 | INFO | Uploaded 36GiB to GPU in 1.97763s (18.2036GiB/s) | 20260506 22:53:02.340 | INFO | Downloaded 34GiB to GPU in 1.92977s (17.6187GiB/s) | 20260506 22:53:07.471 | INFO | Analyzing chunk frames = [5089792,5090304) | 20260506 22:53:07.471 | INFO | MemInfo: { | 20260506 22:53:07.471 | INFO | "gpu" : " 17 GB", | 20260506 22:53:07.471 | INFO | "huge" : "", | 20260506 22:53:07.471 | INFO | "huge_shared" : " 27 GB", | 20260506 22:53:07.471 | INFO | "malloc" : " 14 GB", | 20260506 22:53:07.472 | INFO | "pinned" : " 29 GB" | 20260506 22:53:07.472 | INFO | } | 20260506 22:53:07.472 | INFO | Perf report for the last 5.13232 seconds: | 20260506 22:53:07.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:53:07.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:07.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:07.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:07.472 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 22:53:07.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:07.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:07.472 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 22:53:07.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:07.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:07.472 | INFO | Uploaded 36GiB to GPU in 2.00256s (17.977GiB/s) | 20260506 22:53:07.472 | INFO | Downloaded 34GiB to GPU in 1.93996s (17.5261GiB/s) | 20260506 22:53:12.612 | INFO | Analyzing chunk frames = [5090304,5090816) | 20260506 22:53:12.612 | INFO | MemInfo: { | 20260506 22:53:12.612 | INFO | "gpu" : " 17 GB", | 20260506 22:53:12.612 | INFO | "huge" : "", | 20260506 22:53:12.612 | INFO | "huge_shared" : " 27 GB", | 20260506 22:53:12.612 | INFO | "malloc" : " 14 GB", | 20260506 22:53:12.612 | INFO | "pinned" : " 29 GB" | 20260506 22:53:12.612 | INFO | } | 20260506 22:53:12.612 | INFO | Perf report for the last 5.14072 seconds: | 20260506 22:53:12.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:53:12.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:12.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:12.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:12.612 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 22:53:12.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:12.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:12.613 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 22:53:12.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:12.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:12.613 | INFO | Uploaded 36GiB to GPU in 1.9627s (18.3421GiB/s) | 20260506 22:53:12.613 | INFO | Downloaded 34GiB to GPU in 1.99692s (17.0262GiB/s) | 20260506 22:53:17.754 | INFO | Analyzing chunk frames = [5090816,5091328) | 20260506 22:53:17.754 | INFO | MemInfo: { | 20260506 22:53:17.754 | INFO | "gpu" : " 17 GB", | 20260506 22:53:17.754 | INFO | "huge" : "", | 20260506 22:53:17.754 | INFO | "huge_shared" : " 27 GB", | 20260506 22:53:17.755 | INFO | "malloc" : " 14 GB", | 20260506 22:53:17.755 | INFO | "pinned" : " 29 GB" | 20260506 22:53:17.755 | INFO | } | 20260506 22:53:17.755 | INFO | Perf report for the last 5.14224 seconds: | 20260506 22:53:17.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:53:17.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:17.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:17.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:17.755 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 22:53:17.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:17.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:17.755 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 22:53:17.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:17.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:17.755 | INFO | Uploaded 36GiB to GPU in 2.02497s (17.778GiB/s) | 20260506 22:53:17.755 | INFO | Downloaded 34GiB to GPU in 1.96847s (17.2723GiB/s) | 20260506 22:53:17.859 | INFO | Starting a round of Baz IO | 20260506 22:53:20.989 | INFO | Populating RTMetrics report | 20260506 22:53:20.989 | INFO | RTMetrics report populated | 20260506 22:53:20.989 | INFO | Opening tmp RTMetrics json file | 20260506 22:53:20.990 | INFO | Serializing RTMetrics to json | 20260506 22:53:20.991 | INFO | Serialization done, swapping tmp file now... | 20260506 22:53:20.991 | INFO | RTMetrics done | 20260506 22:53:21.087 | INFO | Finished a round of Baz IO: Wrote 1.899 GiB in 3.227 seconds | 20260506 22:53:21.087 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.58% / 0.00% / 66.42% | 20260506 22:53:21.087 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.464992% / 3.97364e-06% / 4.76837e-05% | 20260506 22:53:21.087 | INFO | Baz IO overall: | 20260506 22:53:21.087 | INFO | BytesWritten : 2670698376728 | 20260506 22:53:21.087 | INFO | NumEvents : 1288176607574 | 20260506 22:53:22.894 | INFO | Analyzing chunk frames = [5091328,5091840) | 20260506 22:53:22.894 | INFO | MemInfo: { | 20260506 22:53:22.894 | INFO | "gpu" : " 17 GB", | 20260506 22:53:22.894 | INFO | "huge" : "", | 20260506 22:53:22.894 | INFO | "huge_shared" : " 27 GB", | 20260506 22:53:22.894 | INFO | "malloc" : " 14 GB", | 20260506 22:53:22.894 | INFO | "pinned" : " 29 GB" | 20260506 22:53:22.894 | INFO | } | 20260506 22:53:22.894 | INFO | Perf report for the last 5.13921 seconds: | 20260506 22:53:22.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:53:22.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:22.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:22.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:22.894 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260506 22:53:22.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:22.894 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260506 22:53:22.894 | INFO | PRE_HQ : 59.7%, 1.8, 56.9ms, 0 | 20260506 22:53:22.894 | INFO | BAZWRITER : 62.5%, 3.5, 110.9ms, 0 | 20260506 22:53:22.894 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:53:22.894 | INFO | Uploaded 36GiB to GPU in 2.03319s (17.7062GiB/s) | 20260506 22:53:22.894 | INFO | Downloaded 38GiB to GPU in 2.18761s (17.3705GiB/s) | 20260506 22:53:28.040 | INFO | Analyzing chunk frames = [5091840,5092352) | 20260506 22:53:28.040 | INFO | MemInfo: { | 20260506 22:53:28.040 | INFO | "gpu" : " 17 GB", | 20260506 22:53:28.040 | INFO | "huge" : "", | 20260506 22:53:28.040 | INFO | "huge_shared" : " 27 GB", | 20260506 22:53:28.040 | INFO | "malloc" : " 14 GB", | 20260506 22:53:28.040 | INFO | "pinned" : " 29 GB" | 20260506 22:53:28.040 | INFO | } | 20260506 22:53:28.040 | INFO | Perf report for the last 5.14606 seconds: | 20260506 22:53:28.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:53:28.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:28.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:28.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:28.040 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 22:53:28.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:28.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:28.040 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 22:53:28.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:28.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:28.040 | INFO | Uploaded 36GiB to GPU in 2.00609s (17.9453GiB/s) | 20260506 22:53:28.040 | INFO | Downloaded 34GiB to GPU in 1.92371s (17.6742GiB/s) | 20260506 22:53:33.176 | INFO | Analyzing chunk frames = [5092352,5092864) | 20260506 22:53:33.176 | INFO | MemInfo: { | 20260506 22:53:33.176 | INFO | "gpu" : " 17 GB", | 20260506 22:53:33.176 | INFO | "huge" : "", | 20260506 22:53:33.176 | INFO | "huge_shared" : " 27 GB", | 20260506 22:53:33.176 | INFO | "malloc" : " 14 GB", | 20260506 22:53:33.176 | INFO | "pinned" : " 29 GB" | 20260506 22:53:33.176 | INFO | } | 20260506 22:53:33.176 | INFO | Perf report for the last 5.13599 seconds: | 20260506 22:53:33.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:53:33.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:33.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:33.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:33.176 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 22:53:33.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:33.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:33.176 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:53:33.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:33.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:33.176 | INFO | Uploaded 36GiB to GPU in 1.99885s (18.0104GiB/s) | 20260506 22:53:33.176 | INFO | Downloaded 34GiB to GPU in 1.93677s (17.555GiB/s) | 20260506 22:53:38.315 | INFO | Analyzing chunk frames = [5092864,5093376) | 20260506 22:53:38.315 | INFO | MemInfo: { | 20260506 22:53:38.315 | INFO | "gpu" : " 17 GB", | 20260506 22:53:38.315 | INFO | "huge" : "", | 20260506 22:53:38.315 | INFO | "huge_shared" : " 27 GB", | 20260506 22:53:38.315 | INFO | "malloc" : " 14 GB", | 20260506 22:53:38.315 | INFO | "pinned" : " 29 GB" | 20260506 22:53:38.315 | INFO | } | 20260506 22:53:38.315 | INFO | Perf report for the last 5.13939 seconds: | 20260506 22:53:38.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:53:38.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:38.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:38.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:38.315 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 22:53:38.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:38.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:38.315 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:53:38.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:38.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:38.316 | INFO | Uploaded 36GiB to GPU in 1.98507s (18.1354GiB/s) | 20260506 22:53:38.316 | INFO | Downloaded 34GiB to GPU in 1.90341s (17.8627GiB/s) | 20260506 22:53:43.452 | INFO | Analyzing chunk frames = [5093376,5093888) | 20260506 22:53:43.452 | INFO | MemInfo: { | 20260506 22:53:43.452 | INFO | "gpu" : " 17 GB", | 20260506 22:53:43.452 | INFO | "huge" : "", | 20260506 22:53:43.452 | INFO | "huge_shared" : " 27 GB", | 20260506 22:53:43.452 | INFO | "malloc" : " 14 GB", | 20260506 22:53:43.452 | INFO | "pinned" : " 29 GB" | 20260506 22:53:43.452 | INFO | } | 20260506 22:53:43.452 | INFO | Perf report for the last 5.1366 seconds: | 20260506 22:53:43.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:53:43.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:43.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:43.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:43.452 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 22:53:43.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:43.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:43.452 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:53:43.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:43.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:43.452 | INFO | Uploaded 36GiB to GPU in 1.99431s (18.0514GiB/s) | 20260506 22:53:43.452 | INFO | Downloaded 34GiB to GPU in 1.92399s (17.6716GiB/s) | 20260506 22:53:48.588 | INFO | Analyzing chunk frames = [5093888,5094400) | 20260506 22:53:48.588 | INFO | MemInfo: { | 20260506 22:53:48.588 | INFO | "gpu" : " 17 GB", | 20260506 22:53:48.588 | INFO | "huge" : "", | 20260506 22:53:48.588 | INFO | "huge_shared" : " 27 GB", | 20260506 22:53:48.588 | INFO | "malloc" : " 14 GB", | 20260506 22:53:48.588 | INFO | "pinned" : " 29 GB" | 20260506 22:53:48.588 | INFO | } | 20260506 22:53:48.588 | INFO | Perf report for the last 5.13634 seconds: | 20260506 22:53:48.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:53:48.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:48.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:48.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:48.589 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260506 22:53:48.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:48.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:48.589 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 22:53:48.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:48.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:48.589 | INFO | Uploaded 36GiB to GPU in 1.99876s (18.0111GiB/s) | 20260506 22:53:48.589 | INFO | Downloaded 34GiB to GPU in 1.93565s (17.5651GiB/s) | 20260506 22:53:53.732 | INFO | Analyzing chunk frames = [5094400,5094912) | 20260506 22:53:53.732 | INFO | MemInfo: { | 20260506 22:53:53.732 | INFO | "gpu" : " 17 GB", | 20260506 22:53:53.732 | INFO | "huge" : "", | 20260506 22:53:53.732 | INFO | "huge_shared" : " 27 GB", | 20260506 22:53:53.732 | INFO | "malloc" : " 14 GB", | 20260506 22:53:53.732 | INFO | "pinned" : " 29 GB" | 20260506 22:53:53.732 | INFO | } | 20260506 22:53:53.732 | INFO | Perf report for the last 5.14359 seconds: | 20260506 22:53:53.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:53:53.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:53.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:53.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:53.732 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 22:53:53.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:53.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:53.732 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260506 22:53:53.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:53.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:53.732 | INFO | Uploaded 36GiB to GPU in 1.99874s (18.0113GiB/s) | 20260506 22:53:53.732 | INFO | Downloaded 34GiB to GPU in 2.00202s (16.9829GiB/s) | 20260506 22:53:58.873 | INFO | Analyzing chunk frames = [5094912,5095424) | 20260506 22:53:58.873 | INFO | MemInfo: { | 20260506 22:53:58.873 | INFO | "gpu" : " 17 GB", | 20260506 22:53:58.873 | INFO | "huge" : "", | 20260506 22:53:58.873 | INFO | "huge_shared" : " 27 GB", | 20260506 22:53:58.873 | INFO | "malloc" : " 14 GB", | 20260506 22:53:58.874 | INFO | "pinned" : " 29 GB" | 20260506 22:53:58.874 | INFO | } | 20260506 22:53:58.874 | INFO | Perf report for the last 5.14165 seconds: | 20260506 22:53:58.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:53:58.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:58.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:58.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:58.874 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:53:58.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:58.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:58.874 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 22:53:58.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:58.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:53:58.874 | INFO | Uploaded 36GiB to GPU in 1.98497s (18.1363GiB/s) | 20260506 22:53:58.874 | INFO | Downloaded 34GiB to GPU in 2.00088s (16.9925GiB/s) | 20260506 22:53:58.874 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:53:58.874 | INFO | Total mem usage: 18 GiB | 20260506 22:53:58.874 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:53:58.874 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 22:53:58.874 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:53:58.874 | INFO | High Water Breakdown: | 20260506 22:53:58.874 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:53:58.874 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:53:58.874 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:53:58.874 | INFO | ------------------------------------------ | 20260506 22:53:58.874 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:53:58.874 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:53:58.874 | INFO | Total mem usage: 26.5 GiB | 20260506 22:53:58.874 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:53:58.874 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:53:58.874 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:53:58.874 | INFO | High Water Breakdown: | 20260506 22:53:58.874 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:53:58.874 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:53:58.874 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:53:58.874 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:53:58.874 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:53:58.874 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:53:58.874 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:53:58.874 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:53:58.874 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:53:58.874 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:53:58.874 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:53:58.874 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:53:58.874 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:53:58.874 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:53:58.874 | INFO | ------------------------------------------ | 20260506 22:53:58.874 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:53:58.874 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:53:58.874 | INFO | Total mem usage: 25.6 GiB | 20260506 22:53:58.874 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:53:58.874 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:53:58.874 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:53:58.874 | INFO | High Water Breakdown: | 20260506 22:53:58.874 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:53:58.874 | INFO | ------------------------------------------ | 20260506 22:53:58.874 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:53:58.874 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:53:58.874 | INFO | Total mem usage: 15.7 GiB | 20260506 22:53:58.874 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:53:58.874 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:53:58.874 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:53:58.874 | INFO | High Water Breakdown: | 20260506 22:53:58.875 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:53:58.875 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:53:58.875 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:53:58.875 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:53:58.875 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:53:58.875 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:53:58.875 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:53:58.875 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:53:58.875 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:53:58.875 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:53:58.875 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:53:58.875 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:53:58.875 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:53:58.875 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:53:58.875 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:53:58.875 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:53:58.875 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:53:58.875 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:53:58.875 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:53:58.875 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:53:58.875 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:53:58.875 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:53:58.875 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:53:58.875 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:53:58.875 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:53:58.875 | INFO | ------------------------------------------ | 20260506 22:53:58.875 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:53:58.875 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:53:58.875 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:53:58.875 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:53:58.875 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.60e+01, 1.36e+01 / 1.96e+00 | 20260506 22:53:58.875 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.65e-03 / 5.12e-03 | 20260506 22:53:58.875 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.45e+00, 5.49e-01 / 2.47e-01 | 20260506 22:53:58.875 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.45e+00, 1.77e-01 / 1.19e-01 | 20260506 22:53:58.875 | INFO | PerfReport: 16.01%, DME -- 3.97e+01 / 4.41e+01, 4.06e+00 / 2.55e+00 | 20260506 22:53:58.875 | INFO | PerfReport: 57.39%, FrameLabeling -- 1.78e+01 / 2.14e+01, 1.37e+00 / 1.68e+00 | 20260506 22:53:58.875 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.20e+00 / 2.71e+00, 3.79e-01 / 2.41e-01 | 20260506 22:53:58.875 | INFO | PerfReport: 3.21%, Metrics -- 9.96e-01 / 1.33e+00, 2.62e-01 / 1.51e-01 | 20260506 22:53:58.875 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.40e-01 / 3.89e-01, 5.25e-02 / 2.27e-02 | 20260506 22:53:58.875 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:53:58.875 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:53:58.875 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:53:58.875 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 22:53:58.875 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 22:53:58.875 | INFO | PerfReport: 21.28%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 22:53:58.875 | INFO | PerfReport: 32.07%, Compute -- 3.10e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 22:53:58.875 | INFO | PerfReport: 1.28%, LockUpload -- 1.24e+00 / 0.00e+00, 4.73e+00 / 0.00e+00 | 20260506 22:53:58.875 | INFO | PerfReport: 1.27%, LockDownload -- 1.23e+00 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260506 22:53:58.875 | INFO | PerfReport: 22.56%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 22:53:58.875 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.47e-02 / 0.00e+00 | 20260506 22:53:58.972 | INFO | Starting a round of Baz IO | 20260506 22:54:02.100 | INFO | Populating RTMetrics report | 20260506 22:54:02.101 | INFO | RTMetrics report populated | 20260506 22:54:02.101 | INFO | Opening tmp RTMetrics json file | 20260506 22:54:02.102 | INFO | Serializing RTMetrics to json | 20260506 22:54:02.118 | INFO | Serialization done, swapping tmp file now... | 20260506 22:54:02.121 | INFO | RTMetrics done | 20260506 22:54:02.195 | INFO | Finished a round of Baz IO: Wrote 1.899 GiB in 3.222 seconds | 20260506 22:54:02.195 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.58% / 0.00% / 66.42% | 20260506 22:54:02.195 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.463752% / 1.19209e-05% / 5.5631e-05% | 20260506 22:54:02.195 | INFO | Baz IO overall: | 20260506 22:54:02.195 | INFO | BytesWritten : 2672737474486 | 20260506 22:54:02.195 | INFO | NumEvents : 1289149484637 | 20260506 22:54:04.012 | INFO | Analyzing chunk frames = [5095424,5095936) | 20260506 22:54:04.012 | INFO | MemInfo: { | 20260506 22:54:04.012 | INFO | "gpu" : " 17 GB", | 20260506 22:54:04.012 | INFO | "huge" : "", | 20260506 22:54:04.012 | INFO | "huge_shared" : " 27 GB", | 20260506 22:54:04.012 | INFO | "malloc" : " 14 GB", | 20260506 22:54:04.012 | INFO | "pinned" : " 29 GB" | 20260506 22:54:04.012 | INFO | } | 20260506 22:54:04.012 | INFO | Perf report for the last 5.13823 seconds: | 20260506 22:54:04.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:54:04.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:04.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:04.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:04.012 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 22:54:04.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:04.012 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 22:54:04.012 | INFO | PRE_HQ : 59.7%, 1.8, 56.6ms, 0 | 20260506 22:54:04.012 | INFO | BAZWRITER : 62.3%, 3.5, 110.4ms, 0 | 20260506 22:54:04.012 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:54:04.012 | INFO | Uploaded 36GiB to GPU in 2.01848s (17.8352GiB/s) | 20260506 22:54:04.012 | INFO | Downloaded 38GiB to GPU in 2.17895s (17.4396GiB/s) | 20260506 22:54:09.150 | INFO | Analyzing chunk frames = [5095936,5096448) | 20260506 22:54:09.150 | INFO | MemInfo: { | 20260506 22:54:09.150 | INFO | "gpu" : " 17 GB", | 20260506 22:54:09.150 | INFO | "huge" : "", | 20260506 22:54:09.150 | INFO | "huge_shared" : " 27 GB", | 20260506 22:54:09.150 | INFO | "malloc" : " 14 GB", | 20260506 22:54:09.150 | INFO | "pinned" : " 29 GB" | 20260506 22:54:09.150 | INFO | } | 20260506 22:54:09.150 | INFO | Perf report for the last 5.13857 seconds: | 20260506 22:54:09.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:54:09.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:09.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:09.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:09.150 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 22:54:09.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:09.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:09.150 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 22:54:09.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:09.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:09.151 | INFO | Uploaded 36GiB to GPU in 1.98783s (18.1102GiB/s) | 20260506 22:54:09.151 | INFO | Downloaded 34GiB to GPU in 1.926s (17.6532GiB/s) | 20260506 22:54:14.286 | INFO | Analyzing chunk frames = [5096448,5096960) | 20260506 22:54:14.286 | INFO | MemInfo: { | 20260506 22:54:14.286 | INFO | "gpu" : " 17 GB", | 20260506 22:54:14.286 | INFO | "huge" : "", | 20260506 22:54:14.286 | INFO | "huge_shared" : " 27 GB", | 20260506 22:54:14.286 | INFO | "malloc" : " 14 GB", | 20260506 22:54:14.287 | INFO | "pinned" : " 29 GB" | 20260506 22:54:14.287 | INFO | } | 20260506 22:54:14.287 | INFO | Perf report for the last 5.1363 seconds: | 20260506 22:54:14.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:54:14.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:14.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:14.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:14.287 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 22:54:14.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:14.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:14.287 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:54:14.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:14.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:14.287 | INFO | Uploaded 36GiB to GPU in 1.98609s (18.1261GiB/s) | 20260506 22:54:14.287 | INFO | Downloaded 34GiB to GPU in 1.93488s (17.5721GiB/s) | 20260506 22:54:19.421 | INFO | Analyzing chunk frames = [5096960,5097472) | 20260506 22:54:19.421 | INFO | MemInfo: { | 20260506 22:54:19.421 | INFO | "gpu" : " 17 GB", | 20260506 22:54:19.421 | INFO | "huge" : "", | 20260506 22:54:19.421 | INFO | "huge_shared" : " 27 GB", | 20260506 22:54:19.421 | INFO | "malloc" : " 14 GB", | 20260506 22:54:19.421 | INFO | "pinned" : " 29 GB" | 20260506 22:54:19.421 | INFO | } | 20260506 22:54:19.421 | INFO | Perf report for the last 5.13465 seconds: | 20260506 22:54:19.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:54:19.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:19.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:19.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:19.422 | INFO | ANALYSIS : 59.7%, 3.0, 92.4ms, 0 | 20260506 22:54:19.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:19.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:19.422 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:54:19.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:19.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:19.422 | INFO | Uploaded 36GiB to GPU in 1.97158s (18.2595GiB/s) | 20260506 22:54:19.422 | INFO | Downloaded 34GiB to GPU in 1.87951s (18.0899GiB/s) | 20260506 22:54:24.560 | INFO | Analyzing chunk frames = [5097472,5097984) | 20260506 22:54:24.560 | INFO | MemInfo: { | 20260506 22:54:24.560 | INFO | "gpu" : " 17 GB", | 20260506 22:54:24.560 | INFO | "huge" : "", | 20260506 22:54:24.560 | INFO | "huge_shared" : " 27 GB", | 20260506 22:54:24.560 | INFO | "malloc" : " 14 GB", | 20260506 22:54:24.560 | INFO | "pinned" : " 29 GB" | 20260506 22:54:24.560 | INFO | } | 20260506 22:54:24.560 | INFO | Perf report for the last 5.13887 seconds: | 20260506 22:54:24.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:54:24.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:24.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:24.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:24.560 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 22:54:24.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:24.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:24.560 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:54:24.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:24.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:24.561 | INFO | Uploaded 36GiB to GPU in 2.00003s (17.9997GiB/s) | 20260506 22:54:24.561 | INFO | Downloaded 34GiB to GPU in 1.90943s (17.8064GiB/s) | 20260506 22:54:29.705 | INFO | Analyzing chunk frames = [5097984,5098496) | 20260506 22:54:29.705 | INFO | MemInfo: { | 20260506 22:54:29.705 | INFO | "gpu" : " 17 GB", | 20260506 22:54:29.705 | INFO | "huge" : "", | 20260506 22:54:29.705 | INFO | "huge_shared" : " 27 GB", | 20260506 22:54:29.705 | INFO | "malloc" : " 14 GB", | 20260506 22:54:29.705 | INFO | "pinned" : " 29 GB" | 20260506 22:54:29.705 | INFO | } | 20260506 22:54:29.706 | INFO | Perf report for the last 5.1454 seconds: | 20260506 22:54:29.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:54:29.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:29.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:29.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:29.706 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 22:54:29.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:29.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:29.706 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 22:54:29.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:29.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:29.706 | INFO | Uploaded 36GiB to GPU in 1.99481s (18.0468GiB/s) | 20260506 22:54:29.706 | INFO | Downloaded 34GiB to GPU in 1.95672s (17.376GiB/s) | 20260506 22:54:34.846 | INFO | Analyzing chunk frames = [5098496,5099008) | 20260506 22:54:34.846 | INFO | MemInfo: { | 20260506 22:54:34.846 | INFO | "gpu" : " 17 GB", | 20260506 22:54:34.846 | INFO | "huge" : "", | 20260506 22:54:34.846 | INFO | "huge_shared" : " 27 GB", | 20260506 22:54:34.846 | INFO | "malloc" : " 14 GB", | 20260506 22:54:34.846 | INFO | "pinned" : " 29 GB" | 20260506 22:54:34.846 | INFO | } | 20260506 22:54:34.846 | INFO | Perf report for the last 5.14071 seconds: | 20260506 22:54:34.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:54:34.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:34.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:34.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:34.847 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 22:54:34.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:34.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:34.847 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 22:54:34.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:34.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:34.847 | INFO | Uploaded 36GiB to GPU in 1.95303s (18.4329GiB/s) | 20260506 22:54:34.847 | INFO | Downloaded 34GiB to GPU in 1.97387s (17.2251GiB/s) | 20260506 22:54:39.992 | INFO | Analyzing chunk frames = [5099008,5099520) | 20260506 22:54:39.992 | INFO | MemInfo: { | 20260506 22:54:39.992 | INFO | "gpu" : " 17 GB", | 20260506 22:54:39.992 | INFO | "huge" : "", | 20260506 22:54:39.992 | INFO | "huge_shared" : " 27 GB", | 20260506 22:54:39.992 | INFO | "malloc" : " 14 GB", | 20260506 22:54:39.992 | INFO | "pinned" : " 29 GB" | 20260506 22:54:39.992 | INFO | } | 20260506 22:54:39.992 | INFO | Perf report for the last 5.14584 seconds: | 20260506 22:54:39.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:54:39.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:39.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:39.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:39.992 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 22:54:39.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:39.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:39.992 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 22:54:39.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:39.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:39.993 | INFO | Uploaded 36GiB to GPU in 2.02114s (17.8117GiB/s) | 20260506 22:54:39.993 | INFO | Downloaded 34GiB to GPU in 1.99946s (17.0046GiB/s) | 20260506 22:54:40.105 | INFO | Starting a round of Baz IO | 20260506 22:54:43.212 | INFO | Populating RTMetrics report | 20260506 22:54:43.212 | INFO | RTMetrics report populated | 20260506 22:54:43.212 | INFO | Opening tmp RTMetrics json file | 20260506 22:54:43.213 | INFO | Serializing RTMetrics to json | 20260506 22:54:43.216 | INFO | Serialization done, swapping tmp file now... | 20260506 22:54:43.223 | INFO | RTMetrics done | 20260506 22:54:43.308 | INFO | Finished a round of Baz IO: Wrote 1.898 GiB in 3.203 seconds | 20260506 22:54:43.308 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.58% / 0.00% / 66.42% | 20260506 22:54:43.308 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.461539% / 3.97364e-06% / 6.35783e-05% | 20260506 22:54:43.308 | INFO | Baz IO overall: | 20260506 22:54:43.308 | INFO | BytesWritten : 2674775787205 | 20260506 22:54:43.308 | INFO | NumEvents : 1290121552011 | 20260506 22:54:45.132 | INFO | Analyzing chunk frames = [5099520,5100032) | 20260506 22:54:45.132 | INFO | MemInfo: { | 20260506 22:54:45.132 | INFO | "gpu" : " 17 GB", | 20260506 22:54:45.132 | INFO | "huge" : "", | 20260506 22:54:45.132 | INFO | "huge_shared" : " 27 GB", | 20260506 22:54:45.132 | INFO | "malloc" : " 14 GB", | 20260506 22:54:45.132 | INFO | "pinned" : " 29 GB" | 20260506 22:54:45.132 | INFO | } | 20260506 22:54:45.132 | INFO | Perf report for the last 5.13955 seconds: | 20260506 22:54:45.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:54:45.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:45.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:45.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:45.132 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260506 22:54:45.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:45.132 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 22:54:45.132 | INFO | PRE_HQ : 59.8%, 1.8, 57.3ms, 0 | 20260506 22:54:45.132 | INFO | BAZWRITER : 61.9%, 3.5, 110.2ms, 0 | 20260506 22:54:45.132 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:54:45.132 | INFO | Uploaded 36GiB to GPU in 2.04637s (17.5921GiB/s) | 20260506 22:54:45.132 | INFO | Downloaded 38GiB to GPU in 2.18234s (17.4125GiB/s) | 20260506 22:54:50.265 | INFO | Analyzing chunk frames = [5100032,5100544) | 20260506 22:54:50.265 | INFO | MemInfo: { | 20260506 22:54:50.265 | INFO | "gpu" : " 17 GB", | 20260506 22:54:50.265 | INFO | "huge" : "", | 20260506 22:54:50.265 | INFO | "huge_shared" : " 27 GB", | 20260506 22:54:50.265 | INFO | "malloc" : " 14 GB", | 20260506 22:54:50.265 | INFO | "pinned" : " 29 GB" | 20260506 22:54:50.265 | INFO | } | 20260506 22:54:50.265 | INFO | Perf report for the last 5.13291 seconds: | 20260506 22:54:50.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:54:50.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:50.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:50.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:50.265 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 22:54:50.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:50.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:50.265 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:54:50.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:50.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:50.265 | INFO | Uploaded 36GiB to GPU in 2.01941s (17.827GiB/s) | 20260506 22:54:50.265 | INFO | Downloaded 34GiB to GPU in 1.92831s (17.632GiB/s) | 20260506 22:54:55.403 | INFO | Analyzing chunk frames = [5100544,5101056) | 20260506 22:54:55.403 | INFO | MemInfo: { | 20260506 22:54:55.403 | INFO | "gpu" : " 17 GB", | 20260506 22:54:55.403 | INFO | "huge" : "", | 20260506 22:54:55.403 | INFO | "huge_shared" : " 27 GB", | 20260506 22:54:55.403 | INFO | "malloc" : " 14 GB", | 20260506 22:54:55.403 | INFO | "pinned" : " 29 GB" | 20260506 22:54:55.403 | INFO | } | 20260506 22:54:55.403 | INFO | Perf report for the last 5.1386 seconds: | 20260506 22:54:55.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:54:55.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:55.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:55.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:55.404 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 22:54:55.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:55.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:55.404 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:54:55.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:55.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:54:55.404 | INFO | Uploaded 36GiB to GPU in 1.97944s (18.187GiB/s) | 20260506 22:54:55.404 | INFO | Downloaded 34GiB to GPU in 1.92522s (17.6603GiB/s) | 20260506 22:55:00.541 | INFO | Analyzing chunk frames = [5101056,5101568) | 20260506 22:55:00.541 | INFO | MemInfo: { | 20260506 22:55:00.541 | INFO | "gpu" : " 17 GB", | 20260506 22:55:00.541 | INFO | "huge" : "", | 20260506 22:55:00.541 | INFO | "huge_shared" : " 27 GB", | 20260506 22:55:00.541 | INFO | "malloc" : " 14 GB", | 20260506 22:55:00.541 | INFO | "pinned" : " 29 GB" | 20260506 22:55:00.541 | INFO | } | 20260506 22:55:00.542 | INFO | Perf report for the last 5.13814 seconds: | 20260506 22:55:00.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:55:00.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:00.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:00.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:00.542 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 22:55:00.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:00.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:00.542 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:55:00.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:00.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:00.542 | INFO | Uploaded 36GiB to GPU in 1.96713s (18.3007GiB/s) | 20260506 22:55:00.542 | INFO | Downloaded 34GiB to GPU in 1.87765s (18.1078GiB/s) | 20260506 22:55:05.680 | INFO | Analyzing chunk frames = [5101568,5102080) | 20260506 22:55:05.680 | INFO | MemInfo: { | 20260506 22:55:05.680 | INFO | "gpu" : " 17 GB", | 20260506 22:55:05.680 | INFO | "huge" : "", | 20260506 22:55:05.680 | INFO | "huge_shared" : " 27 GB", | 20260506 22:55:05.680 | INFO | "malloc" : " 14 GB", | 20260506 22:55:05.680 | INFO | "pinned" : " 29 GB" | 20260506 22:55:05.680 | INFO | } | 20260506 22:55:05.680 | INFO | Perf report for the last 5.1388 seconds: | 20260506 22:55:05.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:55:05.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:05.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:05.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:05.681 | INFO | ANALYSIS : 60.2%, 3.0, 92.7ms, 0 | 20260506 22:55:05.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:05.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:05.681 | INFO | PRE_HQ : 25.8%, 1.0, 13.3ms, 0 | 20260506 22:55:05.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:05.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:05.681 | INFO | Uploaded 36GiB to GPU in 1.98883s (18.1011GiB/s) | 20260506 22:55:05.681 | INFO | Downloaded 34GiB to GPU in 1.92323s (17.6786GiB/s) | 20260506 22:55:10.826 | INFO | Analyzing chunk frames = [5102080,5102592) | 20260506 22:55:10.826 | INFO | MemInfo: { | 20260506 22:55:10.826 | INFO | "gpu" : " 17 GB", | 20260506 22:55:10.826 | INFO | "huge" : "", | 20260506 22:55:10.826 | INFO | "huge_shared" : " 27 GB", | 20260506 22:55:10.826 | INFO | "malloc" : " 14 GB", | 20260506 22:55:10.826 | INFO | "pinned" : " 29 GB" | 20260506 22:55:10.826 | INFO | } | 20260506 22:55:10.826 | INFO | Perf report for the last 5.14564 seconds: | 20260506 22:55:10.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:55:10.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:10.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:10.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:10.826 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 22:55:10.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:10.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:10.826 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:55:10.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:10.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:10.826 | INFO | Uploaded 36GiB to GPU in 2.02191s (17.805GiB/s) | 20260506 22:55:10.826 | INFO | Downloaded 34GiB to GPU in 1.95007s (17.4352GiB/s) | 20260506 22:55:15.966 | INFO | Analyzing chunk frames = [5102592,5103104) | 20260506 22:55:15.966 | INFO | MemInfo: { | 20260506 22:55:15.966 | INFO | "gpu" : " 17 GB", | 20260506 22:55:15.966 | INFO | "huge" : "", | 20260506 22:55:15.966 | INFO | "huge_shared" : " 27 GB", | 20260506 22:55:15.966 | INFO | "malloc" : " 14 GB", | 20260506 22:55:15.966 | INFO | "pinned" : " 29 GB" | 20260506 22:55:15.966 | INFO | } | 20260506 22:55:15.966 | INFO | Perf report for the last 5.13989 seconds: | 20260506 22:55:15.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:55:15.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:15.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:15.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:15.966 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:55:15.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:15.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:15.966 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 22:55:15.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:15.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:15.966 | INFO | Uploaded 36GiB to GPU in 1.99467s (18.0481GiB/s) | 20260506 22:55:15.966 | INFO | Downloaded 34GiB to GPU in 1.99605s (17.0336GiB/s) | 20260506 22:55:21.103 | INFO | Analyzing chunk frames = [5103104,5103616) | 20260506 22:55:21.103 | INFO | MemInfo: { | 20260506 22:55:21.103 | INFO | "gpu" : " 17 GB", | 20260506 22:55:21.103 | INFO | "huge" : "", | 20260506 22:55:21.103 | INFO | "huge_shared" : " 27 GB", | 20260506 22:55:21.103 | INFO | "malloc" : " 14 GB", | 20260506 22:55:21.103 | INFO | "pinned" : " 29 GB" | 20260506 22:55:21.103 | INFO | } | 20260506 22:55:21.103 | INFO | Perf report for the last 5.13705 seconds: | 20260506 22:55:21.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:55:21.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:21.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:21.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:21.103 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 22:55:21.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:21.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:21.103 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 22:55:21.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:21.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:21.103 | INFO | Uploaded 36GiB to GPU in 1.9789s (18.192GiB/s) | 20260506 22:55:21.103 | INFO | Downloaded 34GiB to GPU in 1.9913s (17.0743GiB/s) | 20260506 22:55:21.103 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:55:21.103 | INFO | Total mem usage: 18 GiB | 20260506 22:55:21.103 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:55:21.103 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:55:21.103 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:55:21.103 | INFO | High Water Breakdown: | 20260506 22:55:21.103 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:55:21.103 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:55:21.103 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:55:21.103 | INFO | ------------------------------------------ | 20260506 22:55:21.103 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:55:21.103 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:55:21.103 | INFO | Total mem usage: 26.5 GiB | 20260506 22:55:21.103 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:55:21.103 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:55:21.103 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:55:21.104 | INFO | High Water Breakdown: | 20260506 22:55:21.104 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:55:21.104 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:55:21.104 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:55:21.104 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:55:21.104 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:55:21.104 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:55:21.104 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:55:21.104 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:55:21.104 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:55:21.104 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:55:21.104 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:55:21.104 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:55:21.104 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:55:21.104 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:55:21.104 | INFO | ------------------------------------------ | 20260506 22:55:21.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:55:21.104 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:55:21.104 | INFO | Total mem usage: 25.6 GiB | 20260506 22:55:21.104 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:55:21.104 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:55:21.104 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:55:21.104 | INFO | High Water Breakdown: | 20260506 22:55:21.104 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:55:21.104 | INFO | ------------------------------------------ | 20260506 22:55:21.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:55:21.104 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:55:21.104 | INFO | Total mem usage: 15.7 GiB | 20260506 22:55:21.104 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:55:21.104 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:55:21.104 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:55:21.104 | INFO | High Water Breakdown: | 20260506 22:55:21.104 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:55:21.104 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:55:21.104 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:55:21.104 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:55:21.104 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:55:21.104 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:55:21.104 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:55:21.104 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:55:21.104 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:55:21.104 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:55:21.104 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:55:21.104 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:55:21.104 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:55:21.104 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:55:21.104 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:55:21.104 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:55:21.104 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:55:21.104 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:55:21.104 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:55:21.104 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:55:21.104 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:55:21.104 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:55:21.104 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:55:21.104 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:55:21.104 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:55:21.104 | INFO | ------------------------------------------ | 20260506 22:55:21.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:55:21.104 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:55:21.104 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:55:21.104 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:55:21.104 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.60e+01, 1.35e+01 / 2.82e+00 | 20260506 22:55:21.104 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 1.01e-02 / 4.87e-03 | 20260506 22:55:21.104 | INFO | PerfReport: 11.65%, Baseline -- 3.61e+00 / 4.40e+00, 5.42e-01 / 3.20e-01 | 20260506 22:55:21.104 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.47e+00, 1.79e-01 / 1.30e-01 | 20260506 22:55:21.104 | INFO | PerfReport: 16.10%, DME -- 3.99e+01 / 4.51e+01, 4.24e+00 / 3.07e+00 | 20260506 22:55:21.104 | INFO | PerfReport: 57.31%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.33e+00 / 1.24e+00 | 20260506 22:55:21.104 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.20e+00 / 2.78e+00, 3.82e-01 / 1.65e-01 | 20260506 22:55:21.104 | INFO | PerfReport: 3.21%, Metrics -- 9.96e-01 / 1.35e+00, 2.66e-01 / 1.44e-01 | 20260506 22:55:21.104 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.40e-01 / 3.92e-01, 5.26e-02 / 2.20e-02 | 20260506 22:55:21.104 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:55:21.104 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:55:21.104 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:55:21.104 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 22:55:21.104 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260506 22:55:21.104 | INFO | PerfReport: 21.25%, Download -- 2.05e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 22:55:21.104 | INFO | PerfReport: 32.11%, Compute -- 3.10e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 22:55:21.104 | INFO | PerfReport: 1.25%, LockUpload -- 1.20e+00 / 0.00e+00, 4.69e+00 / 0.00e+00 | 20260506 22:55:21.104 | INFO | PerfReport: 1.26%, LockDownload -- 1.21e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260506 22:55:21.104 | INFO | PerfReport: 22.60%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 22:55:21.104 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260506 22:55:21.201 | INFO | Starting a round of Baz IO | 20260506 22:55:24.335 | INFO | Populating RTMetrics report | 20260506 22:55:24.335 | INFO | RTMetrics report populated | 20260506 22:55:24.335 | INFO | Opening tmp RTMetrics json file | 20260506 22:55:24.336 | INFO | Serializing RTMetrics to json | 20260506 22:55:24.343 | INFO | Serialization done, swapping tmp file now... | 20260506 22:55:24.357 | INFO | RTMetrics done | 20260506 22:55:24.429 | INFO | Finished a round of Baz IO: Wrote 1.897 GiB in 3.228 seconds | 20260506 22:55:24.429 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.57% / 0.00% / 66.42% | 20260506 22:55:24.429 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.461074% / 0% / 5.5631e-05% | 20260506 22:55:24.429 | INFO | Baz IO overall: | 20260506 22:55:24.429 | INFO | BytesWritten : 2676812897106 | 20260506 22:55:24.429 | INFO | NumEvents : 1291092895282 | 20260506 22:55:26.236 | INFO | Analyzing chunk frames = [5103616,5104128) | 20260506 22:55:26.236 | INFO | MemInfo: { | 20260506 22:55:26.236 | INFO | "gpu" : " 17 GB", | 20260506 22:55:26.236 | INFO | "huge" : "", | 20260506 22:55:26.236 | INFO | "huge_shared" : " 27 GB", | 20260506 22:55:26.236 | INFO | "malloc" : " 14 GB", | 20260506 22:55:26.236 | INFO | "pinned" : " 29 GB" | 20260506 22:55:26.236 | INFO | } | 20260506 22:55:26.236 | INFO | Perf report for the last 5.13267 seconds: | 20260506 22:55:26.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:55:26.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:26.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:26.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:26.236 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260506 22:55:26.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:26.236 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 22:55:26.236 | INFO | PRE_HQ : 59.9%, 1.8, 56.6ms, 0 | 20260506 22:55:26.236 | INFO | BAZWRITER : 62.5%, 3.4, 110.0ms, 0 | 20260506 22:55:26.236 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:55:26.236 | INFO | Uploaded 36GiB to GPU in 2.02563s (17.7723GiB/s) | 20260506 22:55:26.236 | INFO | Downloaded 38GiB to GPU in 2.17497s (17.4715GiB/s) | 20260506 22:55:31.375 | INFO | Analyzing chunk frames = [5104128,5104640) | 20260506 22:55:31.375 | INFO | MemInfo: { | 20260506 22:55:31.375 | INFO | "gpu" : " 17 GB", | 20260506 22:55:31.375 | INFO | "huge" : "", | 20260506 22:55:31.375 | INFO | "huge_shared" : " 27 GB", | 20260506 22:55:31.375 | INFO | "malloc" : " 14 GB", | 20260506 22:55:31.375 | INFO | "pinned" : " 29 GB" | 20260506 22:55:31.375 | INFO | } | 20260506 22:55:31.375 | INFO | Perf report for the last 5.13937 seconds: | 20260506 22:55:31.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:55:31.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:31.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:31.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:31.375 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:55:31.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:31.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:31.375 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 22:55:31.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:31.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:31.375 | INFO | Uploaded 36GiB to GPU in 2.00694s (17.9377GiB/s) | 20260506 22:55:31.375 | INFO | Downloaded 34GiB to GPU in 1.93091s (17.6083GiB/s) | 20260506 22:55:36.510 | INFO | Analyzing chunk frames = [5104640,5105152) | 20260506 22:55:36.510 | INFO | MemInfo: { | 20260506 22:55:36.510 | INFO | "gpu" : " 17 GB", | 20260506 22:55:36.510 | INFO | "huge" : "", | 20260506 22:55:36.510 | INFO | "huge_shared" : " 27 GB", | 20260506 22:55:36.510 | INFO | "malloc" : " 14 GB", | 20260506 22:55:36.510 | INFO | "pinned" : " 29 GB" | 20260506 22:55:36.510 | INFO | } | 20260506 22:55:36.510 | INFO | Perf report for the last 5.13491 seconds: | 20260506 22:55:36.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:55:36.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:36.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:36.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:36.510 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 22:55:36.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:36.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:36.510 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:55:36.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:36.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:36.510 | INFO | Uploaded 36GiB to GPU in 1.96574s (18.3137GiB/s) | 20260506 22:55:36.510 | INFO | Downloaded 34GiB to GPU in 1.92565s (17.6563GiB/s) | 20260506 22:55:41.654 | INFO | Analyzing chunk frames = [5105152,5105664) | 20260506 22:55:41.654 | INFO | MemInfo: { | 20260506 22:55:41.654 | INFO | "gpu" : " 17 GB", | 20260506 22:55:41.654 | INFO | "huge" : "", | 20260506 22:55:41.654 | INFO | "huge_shared" : " 27 GB", | 20260506 22:55:41.654 | INFO | "malloc" : " 14 GB", | 20260506 22:55:41.654 | INFO | "pinned" : " 29 GB" | 20260506 22:55:41.654 | INFO | } | 20260506 22:55:41.654 | INFO | Perf report for the last 5.14383 seconds: | 20260506 22:55:41.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:55:41.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:41.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:41.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:41.654 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 22:55:41.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:41.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:41.654 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:55:41.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:41.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:41.654 | INFO | Uploaded 36GiB to GPU in 1.92818s (18.6704GiB/s) | 20260506 22:55:41.654 | INFO | Downloaded 34GiB to GPU in 1.87381s (18.1448GiB/s) | 20260506 22:55:46.801 | INFO | Analyzing chunk frames = [5105664,5106176) | 20260506 22:55:46.801 | INFO | MemInfo: { | 20260506 22:55:46.801 | INFO | "gpu" : " 17 GB", | 20260506 22:55:46.801 | INFO | "huge" : "", | 20260506 22:55:46.801 | INFO | "huge_shared" : " 27 GB", | 20260506 22:55:46.801 | INFO | "malloc" : " 14 GB", | 20260506 22:55:46.801 | INFO | "pinned" : " 29 GB" | 20260506 22:55:46.801 | INFO | } | 20260506 22:55:46.801 | INFO | Perf report for the last 5.14735 seconds: | 20260506 22:55:46.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:55:46.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:46.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:46.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:46.801 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 22:55:46.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:46.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:46.801 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:55:46.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:46.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:46.801 | INFO | Uploaded 36GiB to GPU in 1.98267s (18.1574GiB/s) | 20260506 22:55:46.801 | INFO | Downloaded 34GiB to GPU in 1.92324s (17.6785GiB/s) | 20260506 22:55:51.940 | INFO | Analyzing chunk frames = [5106176,5106688) | 20260506 22:55:51.940 | INFO | MemInfo: { | 20260506 22:55:51.940 | INFO | "gpu" : " 17 GB", | 20260506 22:55:51.940 | INFO | "huge" : "", | 20260506 22:55:51.940 | INFO | "huge_shared" : " 27 GB", | 20260506 22:55:51.940 | INFO | "malloc" : " 14 GB", | 20260506 22:55:51.940 | INFO | "pinned" : " 29 GB" | 20260506 22:55:51.940 | INFO | } | 20260506 22:55:51.940 | INFO | Perf report for the last 5.13927 seconds: | 20260506 22:55:51.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:55:51.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:51.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:51.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:51.940 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 22:55:51.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:51.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:51.941 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 22:55:51.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:51.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:51.941 | INFO | Uploaded 36GiB to GPU in 2.01572s (17.8596GiB/s) | 20260506 22:55:51.941 | INFO | Downloaded 34GiB to GPU in 1.9528s (17.4109GiB/s) | 20260506 22:55:57.083 | INFO | Analyzing chunk frames = [5106688,5107200) | 20260506 22:55:57.083 | INFO | MemInfo: { | 20260506 22:55:57.083 | INFO | "gpu" : " 17 GB", | 20260506 22:55:57.083 | INFO | "huge" : "", | 20260506 22:55:57.083 | INFO | "huge_shared" : " 27 GB", | 20260506 22:55:57.083 | INFO | "malloc" : " 14 GB", | 20260506 22:55:57.083 | INFO | "pinned" : " 29 GB" | 20260506 22:55:57.083 | INFO | } | 20260506 22:55:57.083 | INFO | Perf report for the last 5.1424 seconds: | 20260506 22:55:57.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:55:57.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:57.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:57.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:57.083 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:55:57.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:57.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:57.083 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:55:57.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:57.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:55:57.083 | INFO | Uploaded 36GiB to GPU in 1.99022s (18.0885GiB/s) | 20260506 22:55:57.083 | INFO | Downloaded 34GiB to GPU in 1.98039s (17.1683GiB/s) | 20260506 22:56:02.219 | INFO | Analyzing chunk frames = [5107200,5107712) | 20260506 22:56:02.219 | INFO | MemInfo: { | 20260506 22:56:02.219 | INFO | "gpu" : " 17 GB", | 20260506 22:56:02.219 | INFO | "huge" : "", | 20260506 22:56:02.219 | INFO | "huge_shared" : " 27 GB", | 20260506 22:56:02.219 | INFO | "malloc" : " 14 GB", | 20260506 22:56:02.219 | INFO | "pinned" : " 29 GB" | 20260506 22:56:02.219 | INFO | } | 20260506 22:56:02.219 | INFO | Perf report for the last 5.13584 seconds: | 20260506 22:56:02.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:56:02.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:02.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:02.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:02.219 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 22:56:02.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:02.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:02.219 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 22:56:02.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:02.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:02.219 | INFO | Uploaded 36GiB to GPU in 1.97633s (18.2156GiB/s) | 20260506 22:56:02.219 | INFO | Downloaded 34GiB to GPU in 1.97619s (17.2048GiB/s) | 20260506 22:56:02.317 | INFO | Starting a round of Baz IO | 20260506 22:56:05.453 | INFO | Populating RTMetrics report | 20260506 22:56:05.454 | INFO | RTMetrics report populated | 20260506 22:56:05.454 | INFO | Opening tmp RTMetrics json file | 20260506 22:56:05.456 | INFO | Serializing RTMetrics to json | 20260506 22:56:05.473 | INFO | Serialization done, swapping tmp file now... | 20260506 22:56:05.474 | INFO | RTMetrics done | 20260506 22:56:05.548 | INFO | Finished a round of Baz IO: Wrote 1.896 GiB in 3.231 seconds | 20260506 22:56:05.548 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.57% / 0.00% / 66.43% | 20260506 22:56:05.548 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.459985% / 1.19209e-05% / 5.5631e-05% | 20260506 22:56:05.548 | INFO | Baz IO overall: | 20260506 22:56:05.548 | INFO | BytesWritten : 2678848857099 | 20260506 22:56:05.548 | INFO | NumEvents : 1292063556537 | 20260506 22:56:07.353 | INFO | Analyzing chunk frames = [5107712,5108224) | 20260506 22:56:07.353 | INFO | MemInfo: { | 20260506 22:56:07.353 | INFO | "gpu" : " 17 GB", | 20260506 22:56:07.353 | INFO | "huge" : "", | 20260506 22:56:07.353 | INFO | "huge_shared" : " 27 GB", | 20260506 22:56:07.353 | INFO | "malloc" : " 14 GB", | 20260506 22:56:07.353 | INFO | "pinned" : " 29 GB" | 20260506 22:56:07.353 | INFO | } | 20260506 22:56:07.353 | INFO | Perf report for the last 5.13468 seconds: | 20260506 22:56:07.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:56:07.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:07.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:07.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:07.353 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260506 22:56:07.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:07.353 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 22:56:07.353 | INFO | PRE_HQ : 59.7%, 1.8, 56.6ms, 0 | 20260506 22:56:07.354 | INFO | BAZWRITER : 62.5%, 3.4, 110.4ms, 0 | 20260506 22:56:07.354 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:56:07.354 | INFO | Uploaded 36GiB to GPU in 2.04012s (17.646GiB/s) | 20260506 22:56:07.354 | INFO | Downloaded 38GiB to GPU in 2.16986s (17.5126GiB/s) | 20260506 22:56:12.489 | INFO | Analyzing chunk frames = [5108224,5108736) | 20260506 22:56:12.489 | INFO | MemInfo: { | 20260506 22:56:12.489 | INFO | "gpu" : " 17 GB", | 20260506 22:56:12.489 | INFO | "huge" : "", | 20260506 22:56:12.490 | INFO | "huge_shared" : " 27 GB", | 20260506 22:56:12.490 | INFO | "malloc" : " 14 GB", | 20260506 22:56:12.490 | INFO | "pinned" : " 29 GB" | 20260506 22:56:12.490 | INFO | } | 20260506 22:56:12.490 | INFO | Perf report for the last 5.1362 seconds: | 20260506 22:56:12.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:56:12.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:12.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:12.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:12.490 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:56:12.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:12.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:12.490 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:56:12.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:12.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:12.490 | INFO | Uploaded 36GiB to GPU in 2.01915s (17.8293GiB/s) | 20260506 22:56:12.490 | INFO | Downloaded 34GiB to GPU in 1.92697s (17.6443GiB/s) | 20260506 22:56:17.629 | INFO | Analyzing chunk frames = [5108736,5109248) | 20260506 22:56:17.629 | INFO | MemInfo: { | 20260506 22:56:17.629 | INFO | "gpu" : " 17 GB", | 20260506 22:56:17.629 | INFO | "huge" : "", | 20260506 22:56:17.629 | INFO | "huge_shared" : " 27 GB", | 20260506 22:56:17.629 | INFO | "malloc" : " 14 GB", | 20260506 22:56:17.629 | INFO | "pinned" : " 29 GB" | 20260506 22:56:17.629 | INFO | } | 20260506 22:56:17.629 | INFO | Perf report for the last 5.13925 seconds: | 20260506 22:56:17.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:56:17.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:17.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:17.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:17.629 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 22:56:17.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:17.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:17.629 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:56:17.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:17.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:17.629 | INFO | Uploaded 36GiB to GPU in 1.99776s (18.0202GiB/s) | 20260506 22:56:17.629 | INFO | Downloaded 34GiB to GPU in 1.93113s (17.6063GiB/s) | 20260506 22:56:22.772 | INFO | Analyzing chunk frames = [5109248,5109760) | 20260506 22:56:22.772 | INFO | MemInfo: { | 20260506 22:56:22.772 | INFO | "gpu" : " 17 GB", | 20260506 22:56:22.772 | INFO | "huge" : "", | 20260506 22:56:22.772 | INFO | "huge_shared" : " 27 GB", | 20260506 22:56:22.772 | INFO | "malloc" : " 14 GB", | 20260506 22:56:22.772 | INFO | "pinned" : " 29 GB" | 20260506 22:56:22.772 | INFO | } | 20260506 22:56:22.772 | INFO | Perf report for the last 5.14357 seconds: | 20260506 22:56:22.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:56:22.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:22.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:22.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:22.772 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 22:56:22.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:22.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:22.772 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:56:22.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:22.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:22.773 | INFO | Uploaded 36GiB to GPU in 1.975s (18.2278GiB/s) | 20260506 22:56:22.773 | INFO | Downloaded 34GiB to GPU in 1.87745s (18.1097GiB/s) | 20260506 22:56:27.914 | INFO | Analyzing chunk frames = [5109760,5110272) | 20260506 22:56:27.914 | INFO | MemInfo: { | 20260506 22:56:27.914 | INFO | "gpu" : " 17 GB", | 20260506 22:56:27.914 | INFO | "huge" : "", | 20260506 22:56:27.914 | INFO | "huge_shared" : " 27 GB", | 20260506 22:56:27.914 | INFO | "malloc" : " 14 GB", | 20260506 22:56:27.914 | INFO | "pinned" : " 29 GB" | 20260506 22:56:27.914 | INFO | } | 20260506 22:56:27.914 | INFO | Perf report for the last 5.1419 seconds: | 20260506 22:56:27.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:56:27.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:27.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:27.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:27.914 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260506 22:56:27.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:27.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:27.914 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:56:27.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:27.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:27.915 | INFO | Uploaded 36GiB to GPU in 2.00119s (17.9893GiB/s) | 20260506 22:56:27.915 | INFO | Downloaded 34GiB to GPU in 1.92174s (17.6923GiB/s) | 20260506 22:56:33.051 | INFO | Analyzing chunk frames = [5110272,5110784) | 20260506 22:56:33.051 | INFO | MemInfo: { | 20260506 22:56:33.051 | INFO | "gpu" : " 17 GB", | 20260506 22:56:33.052 | INFO | "huge" : "", | 20260506 22:56:33.052 | INFO | "huge_shared" : " 27 GB", | 20260506 22:56:33.052 | INFO | "malloc" : " 14 GB", | 20260506 22:56:33.052 | INFO | "pinned" : " 29 GB" | 20260506 22:56:33.052 | INFO | } | 20260506 22:56:33.052 | INFO | Perf report for the last 5.13727 seconds: | 20260506 22:56:33.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:56:33.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:33.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:33.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:33.052 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:56:33.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:33.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:33.052 | INFO | PRE_HQ : 30.2%, 1.0, 15.6ms, 0 | 20260506 22:56:33.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:33.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:33.052 | INFO | Uploaded 36GiB to GPU in 2.01956s (17.8257GiB/s) | 20260506 22:56:33.052 | INFO | Downloaded 34GiB to GPU in 1.93443s (17.5762GiB/s) | 20260506 22:56:38.190 | INFO | Analyzing chunk frames = [5110784,5111296) | 20260506 22:56:38.190 | INFO | MemInfo: { | 20260506 22:56:38.190 | INFO | "gpu" : " 17 GB", | 20260506 22:56:38.191 | INFO | "huge" : "", | 20260506 22:56:38.191 | INFO | "huge_shared" : " 27 GB", | 20260506 22:56:38.191 | INFO | "malloc" : " 14 GB", | 20260506 22:56:38.191 | INFO | "pinned" : " 29 GB" | 20260506 22:56:38.191 | INFO | } | 20260506 22:56:38.191 | INFO | Perf report for the last 5.13897 seconds: | 20260506 22:56:38.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:56:38.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:38.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:38.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:38.191 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:56:38.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:38.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:38.191 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 22:56:38.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:38.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:38.191 | INFO | Uploaded 36GiB to GPU in 1.97911s (18.19GiB/s) | 20260506 22:56:38.191 | INFO | Downloaded 34GiB to GPU in 1.99801s (17.0169GiB/s) | 20260506 22:56:43.326 | INFO | Analyzing chunk frames = [5111296,5111808) | 20260506 22:56:43.326 | INFO | MemInfo: { | 20260506 22:56:43.326 | INFO | "gpu" : " 17 GB", | 20260506 22:56:43.326 | INFO | "huge" : "", | 20260506 22:56:43.326 | INFO | "huge_shared" : " 27 GB", | 20260506 22:56:43.326 | INFO | "malloc" : " 14 GB", | 20260506 22:56:43.326 | INFO | "pinned" : " 29 GB" | 20260506 22:56:43.326 | INFO | } | 20260506 22:56:43.326 | INFO | Perf report for the last 5.13513 seconds: | 20260506 22:56:43.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:56:43.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:43.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:43.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:43.326 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 22:56:43.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:43.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:43.326 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260506 22:56:43.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:43.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:43.326 | INFO | Uploaded 36GiB to GPU in 2.00216s (17.9805GiB/s) | 20260506 22:56:43.326 | INFO | Downloaded 34GiB to GPU in 2.00949s (16.9197GiB/s) | 20260506 22:56:43.326 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:56:43.326 | INFO | Total mem usage: 18 GiB | 20260506 22:56:43.326 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:56:43.326 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:56:43.326 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:56:43.326 | INFO | High Water Breakdown: | 20260506 22:56:43.326 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:56:43.326 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:56:43.326 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:56:43.326 | INFO | ------------------------------------------ | 20260506 22:56:43.326 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:56:43.326 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:56:43.326 | INFO | Total mem usage: 26.5 GiB | 20260506 22:56:43.326 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:56:43.326 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:56:43.326 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:56:43.326 | INFO | High Water Breakdown: | 20260506 22:56:43.326 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:56:43.327 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:56:43.327 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:56:43.327 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:56:43.327 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:56:43.327 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:56:43.327 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:56:43.327 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:56:43.327 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:56:43.327 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:56:43.327 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:56:43.327 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:56:43.327 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:56:43.327 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:56:43.327 | INFO | ------------------------------------------ | 20260506 22:56:43.327 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:56:43.327 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:56:43.327 | INFO | Total mem usage: 25.6 GiB | 20260506 22:56:43.327 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:56:43.327 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:56:43.327 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:56:43.327 | INFO | High Water Breakdown: | 20260506 22:56:43.327 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:56:43.327 | INFO | ------------------------------------------ | 20260506 22:56:43.327 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:56:43.327 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:56:43.327 | INFO | Total mem usage: 15.7 GiB | 20260506 22:56:43.327 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:56:43.327 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:56:43.327 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:56:43.327 | INFO | High Water Breakdown: | 20260506 22:56:43.327 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:56:43.327 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:56:43.327 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:56:43.327 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:56:43.327 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:56:43.327 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:56:43.327 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:56:43.327 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:56:43.327 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:56:43.327 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:56:43.327 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:56:43.327 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:56:43.327 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:56:43.327 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:56:43.327 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:56:43.327 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:56:43.327 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:56:43.327 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:56:43.327 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:56:43.327 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:56:43.327 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:56:43.327 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:56:43.327 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:56:43.327 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:56:43.327 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:56:43.327 | INFO | ------------------------------------------ | 20260506 22:56:43.327 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:56:43.327 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:56:43.327 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:56:43.327 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:56:43.327 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.58e+01, 1.34e+01 / 1.99e+00 | 20260506 22:56:43.327 | INFO | PerfReport: 0.29%, AggregateStats -- 9.15e-02 / 9.98e-02, 1.03e-02 / 5.83e-03 | 20260506 22:56:43.327 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.42e+00, 5.43e-01 / 3.22e-01 | 20260506 22:56:43.327 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.48e+00, 1.81e-01 / 1.10e-01 | 20260506 22:56:43.327 | INFO | PerfReport: 16.07%, DME -- 3.99e+01 / 4.46e+01, 4.04e+00 / 2.55e+00 | 20260506 22:56:43.327 | INFO | PerfReport: 57.31%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.37e+00 / 1.73e+00 | 20260506 22:56:43.327 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.20e+00 / 2.77e+00, 3.79e-01 / 1.53e-01 | 20260506 22:56:43.327 | INFO | PerfReport: 3.21%, Metrics -- 9.96e-01 / 1.34e+00, 2.62e-01 / 1.32e-01 | 20260506 22:56:43.327 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.40e-01 / 3.89e-01, 5.27e-02 / 2.33e-02 | 20260506 22:56:43.327 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:56:43.327 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:56:43.327 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:56:43.327 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 22:56:43.327 | INFO | PerfReport: 21.50%, Upload -- 2.08e+01 / 0.00e+00, 6.91e+00 / 0.00e+00 | 20260506 22:56:43.327 | INFO | PerfReport: 21.22%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 22:56:43.327 | INFO | PerfReport: 32.09%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 22:56:43.327 | INFO | PerfReport: 1.27%, LockUpload -- 1.22e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260506 22:56:43.327 | INFO | PerfReport: 1.26%, LockDownload -- 1.22e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260506 22:56:43.327 | INFO | PerfReport: 22.65%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 22:56:43.327 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260506 22:56:43.426 | INFO | Starting a round of Baz IO | 20260506 22:56:46.566 | INFO | Populating RTMetrics report | 20260506 22:56:46.567 | INFO | RTMetrics report populated | 20260506 22:56:46.567 | INFO | Opening tmp RTMetrics json file | 20260506 22:56:46.586 | INFO | Serializing RTMetrics to json | 20260506 22:56:46.587 | INFO | Serialization done, swapping tmp file now... | 20260506 22:56:46.587 | INFO | RTMetrics done | 20260506 22:56:46.663 | INFO | Finished a round of Baz IO: Wrote 1.899 GiB in 3.237 seconds | 20260506 22:56:46.663 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.57% / 0.00% / 66.43% | 20260506 22:56:46.663 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.461145% / 3.97364e-06% / 6.75519e-05% | 20260506 22:56:46.663 | INFO | Baz IO overall: | 20260506 22:56:46.663 | INFO | BytesWritten : 2680887415440 | 20260506 22:56:46.663 | INFO | NumEvents : 1293036153517 | 20260506 22:56:48.467 | INFO | Analyzing chunk frames = [5111808,5112320) | 20260506 22:56:48.467 | INFO | MemInfo: { | 20260506 22:56:48.467 | INFO | "gpu" : " 17 GB", | 20260506 22:56:48.467 | INFO | "huge" : "", | 20260506 22:56:48.467 | INFO | "huge_shared" : " 27 GB", | 20260506 22:56:48.467 | INFO | "malloc" : " 14 GB", | 20260506 22:56:48.467 | INFO | "pinned" : " 29 GB" | 20260506 22:56:48.467 | INFO | } | 20260506 22:56:48.467 | INFO | Perf report for the last 5.14119 seconds: | 20260506 22:56:48.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:56:48.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:48.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:48.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:48.467 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 22:56:48.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:48.467 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260506 22:56:48.467 | INFO | PRE_HQ : 60.3%, 1.9, 59.6ms, 0 | 20260506 22:56:48.467 | INFO | BAZWRITER : 62.5%, 3.5, 112.9ms, 0 | 20260506 22:56:48.467 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 22:56:48.467 | INFO | Uploaded 36GiB to GPU in 2.03759s (17.668GiB/s) | 20260506 22:56:48.467 | INFO | Downloaded 38GiB to GPU in 2.1799s (17.432GiB/s) | 20260506 22:56:53.607 | INFO | Analyzing chunk frames = [5112320,5112832) | 20260506 22:56:53.608 | INFO | MemInfo: { | 20260506 22:56:53.608 | INFO | "gpu" : " 17 GB", | 20260506 22:56:53.608 | INFO | "huge" : "", | 20260506 22:56:53.608 | INFO | "huge_shared" : " 27 GB", | 20260506 22:56:53.608 | INFO | "malloc" : " 14 GB", | 20260506 22:56:53.608 | INFO | "pinned" : " 29 GB" | 20260506 22:56:53.608 | INFO | } | 20260506 22:56:53.608 | INFO | Perf report for the last 5.14071 seconds: | 20260506 22:56:53.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:56:53.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:53.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:53.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:53.608 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 22:56:53.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:53.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:53.608 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:56:53.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:53.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:53.608 | INFO | Uploaded 36GiB to GPU in 1.99299s (18.0633GiB/s) | 20260506 22:56:53.608 | INFO | Downloaded 34GiB to GPU in 1.92743s (17.6401GiB/s) | 20260506 22:56:58.744 | INFO | Analyzing chunk frames = [5112832,5113344) | 20260506 22:56:58.744 | INFO | MemInfo: { | 20260506 22:56:58.744 | INFO | "gpu" : " 17 GB", | 20260506 22:56:58.744 | INFO | "huge" : "", | 20260506 22:56:58.744 | INFO | "huge_shared" : " 27 GB", | 20260506 22:56:58.744 | INFO | "malloc" : " 14 GB", | 20260506 22:56:58.744 | INFO | "pinned" : " 29 GB" | 20260506 22:56:58.744 | INFO | } | 20260506 22:56:58.744 | INFO | Perf report for the last 5.13625 seconds: | 20260506 22:56:58.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:56:58.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:58.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:58.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:58.744 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260506 22:56:58.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:58.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:58.744 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 22:56:58.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:58.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:56:58.744 | INFO | Uploaded 36GiB to GPU in 1.96372s (18.3325GiB/s) | 20260506 22:56:58.744 | INFO | Downloaded 34GiB to GPU in 1.93066s (17.6105GiB/s) | 20260506 22:57:03.896 | INFO | Analyzing chunk frames = [5113344,5113856) | 20260506 22:57:03.896 | INFO | MemInfo: { | 20260506 22:57:03.896 | INFO | "gpu" : " 17 GB", | 20260506 22:57:03.896 | INFO | "huge" : "", | 20260506 22:57:03.896 | INFO | "huge_shared" : " 27 GB", | 20260506 22:57:03.896 | INFO | "malloc" : " 14 GB", | 20260506 22:57:03.896 | INFO | "pinned" : " 29 GB" | 20260506 22:57:03.896 | INFO | } | 20260506 22:57:03.896 | INFO | Perf report for the last 5.15209 seconds: | 20260506 22:57:03.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:57:03.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:03.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:03.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:03.896 | INFO | ANALYSIS : 59.4%, 3.0, 92.4ms, 0 | 20260506 22:57:03.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:03.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:03.896 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:57:03.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:03.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:03.896 | INFO | Uploaded 36GiB to GPU in 1.96873s (18.2859GiB/s) | 20260506 22:57:03.896 | INFO | Downloaded 34GiB to GPU in 1.87907s (18.0941GiB/s) | 20260506 22:57:09.030 | INFO | Analyzing chunk frames = [5113856,5114368) | 20260506 22:57:09.030 | INFO | MemInfo: { | 20260506 22:57:09.030 | INFO | "gpu" : " 17 GB", | 20260506 22:57:09.030 | INFO | "huge" : "", | 20260506 22:57:09.030 | INFO | "huge_shared" : " 27 GB", | 20260506 22:57:09.030 | INFO | "malloc" : " 14 GB", | 20260506 22:57:09.030 | INFO | "pinned" : " 29 GB" | 20260506 22:57:09.030 | INFO | } | 20260506 22:57:09.030 | INFO | Perf report for the last 5.13432 seconds: | 20260506 22:57:09.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:57:09.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:09.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:09.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:09.030 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 22:57:09.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:09.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:09.030 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 22:57:09.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:09.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:09.031 | INFO | Uploaded 36GiB to GPU in 1.97657s (18.2134GiB/s) | 20260506 22:57:09.031 | INFO | Downloaded 34GiB to GPU in 1.93792s (17.5446GiB/s) | 20260506 22:57:14.169 | INFO | Analyzing chunk frames = [5114368,5114880) | 20260506 22:57:14.169 | INFO | MemInfo: { | 20260506 22:57:14.169 | INFO | "gpu" : " 17 GB", | 20260506 22:57:14.169 | INFO | "huge" : "", | 20260506 22:57:14.169 | INFO | "huge_shared" : " 27 GB", | 20260506 22:57:14.169 | INFO | "malloc" : " 14 GB", | 20260506 22:57:14.169 | INFO | "pinned" : " 29 GB" | 20260506 22:57:14.169 | INFO | } | 20260506 22:57:14.169 | INFO | Perf report for the last 5.13839 seconds: | 20260506 22:57:14.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:57:14.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:14.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:14.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:14.169 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260506 22:57:14.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:14.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:14.169 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:57:14.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:14.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:14.169 | INFO | Uploaded 36GiB to GPU in 1.96821s (18.2907GiB/s) | 20260506 22:57:14.169 | INFO | Downloaded 34GiB to GPU in 1.92991s (17.6174GiB/s) | 20260506 22:57:19.304 | INFO | Analyzing chunk frames = [5114880,5115392) | 20260506 22:57:19.304 | INFO | MemInfo: { | 20260506 22:57:19.304 | INFO | "gpu" : " 17 GB", | 20260506 22:57:19.304 | INFO | "huge" : "", | 20260506 22:57:19.304 | INFO | "huge_shared" : " 27 GB", | 20260506 22:57:19.304 | INFO | "malloc" : " 14 GB", | 20260506 22:57:19.304 | INFO | "pinned" : " 29 GB" | 20260506 22:57:19.304 | INFO | } | 20260506 22:57:19.304 | INFO | Perf report for the last 5.13546 seconds: | 20260506 22:57:19.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:57:19.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:19.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:19.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:19.304 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:57:19.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:19.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:19.304 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 22:57:19.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:19.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:19.305 | INFO | Uploaded 36GiB to GPU in 1.95035s (18.4583GiB/s) | 20260506 22:57:19.305 | INFO | Downloaded 34GiB to GPU in 1.98923s (17.092GiB/s) | 20260506 22:57:24.440 | INFO | Analyzing chunk frames = [5115392,5115904) | 20260506 22:57:24.440 | INFO | MemInfo: { | 20260506 22:57:24.440 | INFO | "gpu" : " 17 GB", | 20260506 22:57:24.440 | INFO | "huge" : "", | 20260506 22:57:24.440 | INFO | "huge_shared" : " 27 GB", | 20260506 22:57:24.440 | INFO | "malloc" : " 14 GB", | 20260506 22:57:24.440 | INFO | "pinned" : " 29 GB" | 20260506 22:57:24.440 | INFO | } | 20260506 22:57:24.440 | INFO | Perf report for the last 5.13548 seconds: | 20260506 22:57:24.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:57:24.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:24.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:24.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:24.440 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 22:57:24.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:24.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:24.440 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 22:57:24.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:24.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:24.440 | INFO | Uploaded 36GiB to GPU in 2.00223s (17.98GiB/s) | 20260506 22:57:24.440 | INFO | Downloaded 34GiB to GPU in 1.98963s (17.0886GiB/s) | 20260506 22:57:24.539 | INFO | Starting a round of Baz IO | 20260506 22:57:27.672 | INFO | Populating RTMetrics report | 20260506 22:57:27.673 | INFO | RTMetrics report populated | 20260506 22:57:27.673 | INFO | Opening tmp RTMetrics json file | 20260506 22:57:27.679 | INFO | Serializing RTMetrics to json | 20260506 22:57:27.691 | INFO | Serialization done, swapping tmp file now... | 20260506 22:57:27.692 | INFO | RTMetrics done | 20260506 22:57:27.764 | INFO | Finished a round of Baz IO: Wrote 1.896 GiB in 3.225 seconds | 20260506 22:57:27.764 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.57% / 0.00% / 66.43% | 20260506 22:57:27.764 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.457982% / 1.19209e-05% / 6.35783e-05% | 20260506 22:57:27.764 | INFO | Baz IO overall: | 20260506 22:57:27.764 | INFO | BytesWritten : 2682922883567 | 20260506 22:57:27.765 | INFO | NumEvents : 1294006404243 | 20260506 22:57:29.583 | INFO | Analyzing chunk frames = [5115904,5116416) | 20260506 22:57:29.583 | INFO | MemInfo: { | 20260506 22:57:29.583 | INFO | "gpu" : " 17 GB", | 20260506 22:57:29.583 | INFO | "huge" : "", | 20260506 22:57:29.583 | INFO | "huge_shared" : " 27 GB", | 20260506 22:57:29.583 | INFO | "malloc" : " 14 GB", | 20260506 22:57:29.583 | INFO | "pinned" : " 29 GB" | 20260506 22:57:29.583 | INFO | } | 20260506 22:57:29.583 | INFO | Perf report for the last 5.14358 seconds: | 20260506 22:57:29.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:57:29.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:29.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:29.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:29.583 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 22:57:29.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:29.583 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 22:57:29.583 | INFO | PRE_HQ : 59.7%, 1.8, 57.0ms, 0 | 20260506 22:57:29.584 | INFO | BAZWRITER : 62.3%, 3.5, 111.5ms, 0 | 20260506 22:57:29.584 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 22:57:29.584 | INFO | Uploaded 36GiB to GPU in 2.05152s (17.548GiB/s) | 20260506 22:57:29.584 | INFO | Downloaded 38GiB to GPU in 2.17794s (17.4477GiB/s) | 20260506 22:57:34.722 | INFO | Analyzing chunk frames = [5116416,5116928) | 20260506 22:57:34.722 | INFO | MemInfo: { | 20260506 22:57:34.722 | INFO | "gpu" : " 17 GB", | 20260506 22:57:34.722 | INFO | "huge" : "", | 20260506 22:57:34.722 | INFO | "huge_shared" : " 27 GB", | 20260506 22:57:34.722 | INFO | "malloc" : " 14 GB", | 20260506 22:57:34.722 | INFO | "pinned" : " 29 GB" | 20260506 22:57:34.722 | INFO | } | 20260506 22:57:34.722 | INFO | Perf report for the last 5.13884 seconds: | 20260506 22:57:34.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:57:34.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:34.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:34.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:34.722 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 22:57:34.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:34.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:34.722 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 22:57:34.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:34.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:34.723 | INFO | Uploaded 36GiB to GPU in 2.00997s (17.9107GiB/s) | 20260506 22:57:34.723 | INFO | Downloaded 34GiB to GPU in 1.92799s (17.6349GiB/s) | 20260506 22:57:39.862 | INFO | Analyzing chunk frames = [5116928,5117440) | 20260506 22:57:39.862 | INFO | MemInfo: { | 20260506 22:57:39.862 | INFO | "gpu" : " 17 GB", | 20260506 22:57:39.862 | INFO | "huge" : "", | 20260506 22:57:39.862 | INFO | "huge_shared" : " 27 GB", | 20260506 22:57:39.862 | INFO | "malloc" : " 14 GB", | 20260506 22:57:39.862 | INFO | "pinned" : " 29 GB" | 20260506 22:57:39.862 | INFO | } | 20260506 22:57:39.862 | INFO | Perf report for the last 5.13985 seconds: | 20260506 22:57:39.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:57:39.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:39.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:39.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:39.862 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 22:57:39.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:39.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:39.862 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:57:39.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:39.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:39.862 | INFO | Uploaded 36GiB to GPU in 1.97787s (18.2014GiB/s) | 20260506 22:57:39.862 | INFO | Downloaded 34GiB to GPU in 1.91925s (17.7152GiB/s) | 20260506 22:57:44.999 | INFO | Analyzing chunk frames = [5117440,5117952) | 20260506 22:57:44.999 | INFO | MemInfo: { | 20260506 22:57:44.999 | INFO | "gpu" : " 17 GB", | 20260506 22:57:44.999 | INFO | "huge" : "", | 20260506 22:57:44.999 | INFO | "huge_shared" : " 27 GB", | 20260506 22:57:44.999 | INFO | "malloc" : " 14 GB", | 20260506 22:57:44.999 | INFO | "pinned" : " 29 GB" | 20260506 22:57:44.999 | INFO | } | 20260506 22:57:45.000 | INFO | Perf report for the last 5.13751 seconds: | 20260506 22:57:45.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:57:45.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:45.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:45.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:45.000 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 22:57:45.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:45.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:45.000 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:57:45.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:45.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:45.000 | INFO | Uploaded 36GiB to GPU in 1.96196s (18.349GiB/s) | 20260506 22:57:45.000 | INFO | Downloaded 34GiB to GPU in 1.87804s (18.104GiB/s) | 20260506 22:57:50.147 | INFO | Analyzing chunk frames = [5117952,5118464) | 20260506 22:57:50.147 | INFO | MemInfo: { | 20260506 22:57:50.147 | INFO | "gpu" : " 17 GB", | 20260506 22:57:50.147 | INFO | "huge" : "", | 20260506 22:57:50.147 | INFO | "huge_shared" : " 27 GB", | 20260506 22:57:50.147 | INFO | "malloc" : " 14 GB", | 20260506 22:57:50.147 | INFO | "pinned" : " 29 GB" | 20260506 22:57:50.147 | INFO | } | 20260506 22:57:50.147 | INFO | Perf report for the last 5.14783 seconds: | 20260506 22:57:50.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:57:50.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:50.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:50.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:50.147 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260506 22:57:50.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:50.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:50.148 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:57:50.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:50.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:50.148 | INFO | Uploaded 36GiB to GPU in 2.0096s (17.914GiB/s) | 20260506 22:57:50.148 | INFO | Downloaded 34GiB to GPU in 1.91756s (17.7309GiB/s) | 20260506 22:57:55.279 | INFO | Analyzing chunk frames = [5118464,5118976) | 20260506 22:57:55.279 | INFO | MemInfo: { | 20260506 22:57:55.279 | INFO | "gpu" : " 17 GB", | 20260506 22:57:55.279 | INFO | "huge" : "", | 20260506 22:57:55.279 | INFO | "huge_shared" : " 27 GB", | 20260506 22:57:55.279 | INFO | "malloc" : " 14 GB", | 20260506 22:57:55.279 | INFO | "pinned" : " 29 GB" | 20260506 22:57:55.279 | INFO | } | 20260506 22:57:55.279 | INFO | Perf report for the last 5.13204 seconds: | 20260506 22:57:55.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:57:55.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:55.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:55.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:55.280 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:57:55.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:55.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:55.280 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:57:55.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:55.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:57:55.280 | INFO | Uploaded 36GiB to GPU in 2.04681s (17.5884GiB/s) | 20260506 22:57:55.280 | INFO | Downloaded 34GiB to GPU in 1.94204s (17.5073GiB/s) | 20260506 22:58:00.421 | INFO | Analyzing chunk frames = [5118976,5119488) | 20260506 22:58:00.421 | INFO | MemInfo: { | 20260506 22:58:00.421 | INFO | "gpu" : " 17 GB", | 20260506 22:58:00.421 | INFO | "huge" : "", | 20260506 22:58:00.422 | INFO | "huge_shared" : " 27 GB", | 20260506 22:58:00.422 | INFO | "malloc" : " 14 GB", | 20260506 22:58:00.422 | INFO | "pinned" : " 29 GB" | 20260506 22:58:00.422 | INFO | } | 20260506 22:58:00.422 | INFO | Perf report for the last 5.14221 seconds: | 20260506 22:58:00.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:58:00.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:00.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:00.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:00.422 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 22:58:00.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:00.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:00.422 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:58:00.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:00.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:00.422 | INFO | Uploaded 36GiB to GPU in 1.97582s (18.2203GiB/s) | 20260506 22:58:00.422 | INFO | Downloaded 34GiB to GPU in 2.00314s (16.9734GiB/s) | 20260506 22:58:05.559 | INFO | Analyzing chunk frames = [5119488,5120000) | 20260506 22:58:05.559 | INFO | MemInfo: { | 20260506 22:58:05.559 | INFO | "gpu" : " 17 GB", | 20260506 22:58:05.559 | INFO | "huge" : "", | 20260506 22:58:05.559 | INFO | "huge_shared" : " 27 GB", | 20260506 22:58:05.559 | INFO | "malloc" : " 14 GB", | 20260506 22:58:05.559 | INFO | "pinned" : " 29 GB" | 20260506 22:58:05.559 | INFO | } | 20260506 22:58:05.559 | INFO | Perf report for the last 5.13724 seconds: | 20260506 22:58:05.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:58:05.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:05.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:05.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:05.559 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:58:05.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:05.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:05.559 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:58:05.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:05.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:05.559 | INFO | Uploaded 36GiB to GPU in 2.00072s (17.9935GiB/s) | 20260506 22:58:05.559 | INFO | Downloaded 34GiB to GPU in 1.98504s (17.1282GiB/s) | 20260506 22:58:05.559 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:58:05.559 | INFO | Total mem usage: 18 GiB | 20260506 22:58:05.559 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:58:05.559 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.9%) | 20260506 22:58:05.559 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:58:05.559 | INFO | High Water Breakdown: | 20260506 22:58:05.559 | INFO | 60.1% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:58:05.559 | INFO | 39.8% : 5.68 GiB ( 84.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:58:05.559 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:58:05.559 | INFO | ------------------------------------------ | 20260506 22:58:05.560 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:58:05.560 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:58:05.560 | INFO | Total mem usage: 26.5 GiB | 20260506 22:58:05.560 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:58:05.560 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:58:05.560 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:58:05.560 | INFO | High Water Breakdown: | 20260506 22:58:05.560 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:58:05.560 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:58:05.560 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:58:05.560 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:58:05.560 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:58:05.560 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:58:05.560 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:58:05.560 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:58:05.560 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:58:05.560 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:58:05.560 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:58:05.560 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:58:05.560 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:58:05.560 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:58:05.560 | INFO | ------------------------------------------ | 20260506 22:58:05.560 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:58:05.560 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:58:05.560 | INFO | Total mem usage: 25.6 GiB | 20260506 22:58:05.560 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:58:05.560 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 22:58:05.560 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 22:58:05.560 | INFO | High Water Breakdown: | 20260506 22:58:05.560 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 22:58:05.560 | INFO | ------------------------------------------ | 20260506 22:58:05.560 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:58:05.560 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:58:05.560 | INFO | Total mem usage: 15.7 GiB | 20260506 22:58:05.560 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:58:05.560 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:58:05.560 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 22:58:05.560 | INFO | High Water Breakdown: | 20260506 22:58:05.560 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:58:05.560 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:58:05.560 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:58:05.560 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:58:05.560 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:58:05.560 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:58:05.560 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:58:05.560 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:58:05.560 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:58:05.560 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:58:05.560 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:58:05.560 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:58:05.560 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:58:05.560 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:58:05.560 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:58:05.560 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:58:05.561 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:58:05.561 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:58:05.561 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:58:05.561 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:58:05.561 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:58:05.561 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:58:05.561 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:58:05.561 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:58:05.561 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:58:05.561 | INFO | ------------------------------------------ | 20260506 22:58:05.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:58:05.561 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:58:05.561 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:58:05.561 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:58:05.561 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.56e+01, 1.35e+01 / 2.37e+00 | 20260506 22:58:05.561 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.04e-01, 1.04e-02 / 6.78e-03 | 20260506 22:58:05.561 | INFO | PerfReport: 11.68%, Baseline -- 3.62e+00 / 4.37e+00, 5.46e-01 / 2.99e-01 | 20260506 22:58:05.561 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.45e+00, 1.79e-01 / 1.20e-01 | 20260506 22:58:05.561 | INFO | PerfReport: 16.06%, DME -- 3.98e+01 / 4.47e+01, 4.06e+00 / 2.75e+00 | 20260506 22:58:05.561 | INFO | PerfReport: 57.31%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.34e+00 / 1.82e+00 | 20260506 22:58:05.561 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.20e+00 / 2.76e+00, 3.81e-01 / 2.05e-01 | 20260506 22:58:05.561 | INFO | PerfReport: 3.22%, Metrics -- 9.98e-01 / 1.34e+00, 2.64e-01 / 1.40e-01 | 20260506 22:58:05.561 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.40e-01 / 3.88e-01, 5.35e-02 / 1.67e-02 | 20260506 22:58:05.561 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:58:05.561 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:58:05.561 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:58:05.561 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 22:58:05.561 | INFO | PerfReport: 21.49%, Upload -- 2.08e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 22:58:05.561 | INFO | PerfReport: 21.24%, Download -- 2.05e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 22:58:05.561 | INFO | PerfReport: 32.09%, Compute -- 3.10e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 22:58:05.561 | INFO | PerfReport: 1.27%, LockUpload -- 1.22e+00 / 0.00e+00, 4.70e+00 / 0.00e+00 | 20260506 22:58:05.561 | INFO | PerfReport: 1.26%, LockDownload -- 1.22e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260506 22:58:05.561 | INFO | PerfReport: 22.64%, LockCompute -- 2.19e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 22:58:05.561 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260506 22:58:05.658 | INFO | Starting a round of Baz IO | 20260506 22:58:08.786 | INFO | Populating RTMetrics report | 20260506 22:58:08.786 | INFO | RTMetrics report populated | 20260506 22:58:08.786 | INFO | Opening tmp RTMetrics json file | 20260506 22:58:08.787 | INFO | Serializing RTMetrics to json | 20260506 22:58:08.788 | INFO | Serialization done, swapping tmp file now... | 20260506 22:58:08.788 | INFO | RTMetrics done | 20260506 22:58:08.881 | INFO | Finished a round of Baz IO: Wrote 1.897 GiB in 3.223 seconds | 20260506 22:58:08.881 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.56% / 0.00% / 66.43% | 20260506 22:58:08.881 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.457513% / 3.97364e-06% / 4.37101e-05% | 20260506 22:58:08.881 | INFO | Baz IO overall: | 20260506 22:58:08.881 | INFO | BytesWritten : 2684959729635 | 20260506 22:58:08.881 | INFO | NumEvents : 1294978052002 | 20260506 22:58:10.696 | INFO | Analyzing chunk frames = [5120000,5120512) | 20260506 22:58:10.696 | INFO | MemInfo: { | 20260506 22:58:10.696 | INFO | "gpu" : " 17 GB", | 20260506 22:58:10.696 | INFO | "huge" : "", | 20260506 22:58:10.696 | INFO | "huge_shared" : " 27 GB", | 20260506 22:58:10.696 | INFO | "malloc" : " 14 GB", | 20260506 22:58:10.696 | INFO | "pinned" : " 29 GB" | 20260506 22:58:10.696 | INFO | } | 20260506 22:58:10.696 | INFO | Perf report for the last 5.1368 seconds: | 20260506 22:58:10.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:58:10.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:10.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:10.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:10.696 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 22:58:10.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:10.696 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260506 22:58:10.696 | INFO | PRE_HQ : 59.7%, 1.8, 56.6ms, 0 | 20260506 22:58:10.696 | INFO | BAZWRITER : 62.3%, 3.5, 110.6ms, 0 | 20260506 22:58:10.696 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:58:10.696 | INFO | Uploaded 36GiB to GPU in 2.05037s (17.5578GiB/s) | 20260506 22:58:10.696 | INFO | Downloaded 38GiB to GPU in 2.17756s (17.4507GiB/s) | 20260506 22:58:15.843 | INFO | Analyzing chunk frames = [5120512,5121024) | 20260506 22:58:15.843 | INFO | MemInfo: { | 20260506 22:58:15.843 | INFO | "gpu" : " 17 GB", | 20260506 22:58:15.843 | INFO | "huge" : "", | 20260506 22:58:15.844 | INFO | "huge_shared" : " 27 GB", | 20260506 22:58:15.844 | INFO | "malloc" : " 14 GB", | 20260506 22:58:15.844 | INFO | "pinned" : " 29 GB" | 20260506 22:58:15.844 | INFO | } | 20260506 22:58:15.844 | INFO | Perf report for the last 5.14794 seconds: | 20260506 22:58:15.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:58:15.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:15.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:15.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:15.844 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 22:58:15.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:15.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:15.844 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 22:58:15.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:15.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:15.844 | INFO | Uploaded 36GiB to GPU in 2.01271s (17.8863GiB/s) | 20260506 22:58:15.844 | INFO | Downloaded 34GiB to GPU in 1.93277s (17.5914GiB/s) | 20260506 22:58:20.988 | INFO | Analyzing chunk frames = [5121024,5121536) | 20260506 22:58:20.988 | INFO | MemInfo: { | 20260506 22:58:20.988 | INFO | "gpu" : " 17 GB", | 20260506 22:58:20.988 | INFO | "huge" : "", | 20260506 22:58:20.988 | INFO | "huge_shared" : " 27 GB", | 20260506 22:58:20.988 | INFO | "malloc" : " 14 GB", | 20260506 22:58:20.988 | INFO | "pinned" : " 29 GB" | 20260506 22:58:20.988 | INFO | } | 20260506 22:58:20.988 | INFO | Perf report for the last 5.14433 seconds: | 20260506 22:58:20.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:58:20.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:20.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:20.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:20.988 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 22:58:20.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:20.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:20.988 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:58:20.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:20.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:20.988 | INFO | Uploaded 36GiB to GPU in 1.99306s (18.0626GiB/s) | 20260506 22:58:20.988 | INFO | Downloaded 34GiB to GPU in 1.93028s (17.6141GiB/s) | 20260506 22:58:26.120 | INFO | Analyzing chunk frames = [5121536,5122048) | 20260506 22:58:26.120 | INFO | MemInfo: { | 20260506 22:58:26.120 | INFO | "gpu" : " 17 GB", | 20260506 22:58:26.120 | INFO | "huge" : "", | 20260506 22:58:26.120 | INFO | "huge_shared" : " 27 GB", | 20260506 22:58:26.120 | INFO | "malloc" : " 14 GB", | 20260506 22:58:26.120 | INFO | "pinned" : " 29 GB" | 20260506 22:58:26.120 | INFO | } | 20260506 22:58:26.120 | INFO | Perf report for the last 5.13197 seconds: | 20260506 22:58:26.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:58:26.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:26.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:26.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:26.120 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 22:58:26.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:26.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:26.120 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:58:26.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:26.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:26.120 | INFO | Uploaded 36GiB to GPU in 1.99247s (18.068GiB/s) | 20260506 22:58:26.120 | INFO | Downloaded 34GiB to GPU in 1.90823s (17.8176GiB/s) | 20260506 22:58:31.256 | INFO | Analyzing chunk frames = [5122048,5122560) | 20260506 22:58:31.257 | INFO | MemInfo: { | 20260506 22:58:31.257 | INFO | "gpu" : " 17 GB", | 20260506 22:58:31.257 | INFO | "huge" : "", | 20260506 22:58:31.257 | INFO | "huge_shared" : " 27 GB", | 20260506 22:58:31.257 | INFO | "malloc" : " 14 GB", | 20260506 22:58:31.257 | INFO | "pinned" : " 29 GB" | 20260506 22:58:31.257 | INFO | } | 20260506 22:58:31.257 | INFO | Perf report for the last 5.13673 seconds: | 20260506 22:58:31.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:58:31.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:31.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:31.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:31.257 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 22:58:31.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:31.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:31.257 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 22:58:31.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:31.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:31.257 | INFO | Uploaded 36GiB to GPU in 1.99776s (18.0202GiB/s) | 20260506 22:58:31.257 | INFO | Downloaded 34GiB to GPU in 1.92617s (17.6516GiB/s) | 20260506 22:58:36.391 | INFO | Analyzing chunk frames = [5122560,5123072) | 20260506 22:58:36.391 | INFO | MemInfo: { | 20260506 22:58:36.391 | INFO | "gpu" : " 17 GB", | 20260506 22:58:36.391 | INFO | "huge" : "", | 20260506 22:58:36.391 | INFO | "huge_shared" : " 27 GB", | 20260506 22:58:36.391 | INFO | "malloc" : " 14 GB", | 20260506 22:58:36.391 | INFO | "pinned" : " 29 GB" | 20260506 22:58:36.391 | INFO | } | 20260506 22:58:36.391 | INFO | Perf report for the last 5.13457 seconds: | 20260506 22:58:36.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:58:36.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:36.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:36.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:36.391 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:58:36.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:36.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:36.391 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260506 22:58:36.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:36.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:36.392 | INFO | Uploaded 36GiB to GPU in 2.03323s (17.7058GiB/s) | 20260506 22:58:36.392 | INFO | Downloaded 34GiB to GPU in 1.94451s (17.4851GiB/s) | 20260506 22:58:41.530 | INFO | Analyzing chunk frames = [5123072,5123584) | 20260506 22:58:41.530 | INFO | MemInfo: { | 20260506 22:58:41.530 | INFO | "gpu" : " 17 GB", | 20260506 22:58:41.530 | INFO | "huge" : "", | 20260506 22:58:41.530 | INFO | "huge_shared" : " 27 GB", | 20260506 22:58:41.530 | INFO | "malloc" : " 14 GB", | 20260506 22:58:41.530 | INFO | "pinned" : " 29 GB" | 20260506 22:58:41.530 | INFO | } | 20260506 22:58:41.531 | INFO | Perf report for the last 5.13933 seconds: | 20260506 22:58:41.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:58:41.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:41.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:41.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:41.531 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 22:58:41.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:41.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:41.531 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 22:58:41.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:41.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:41.531 | INFO | Uploaded 36GiB to GPU in 1.97713s (18.2082GiB/s) | 20260506 22:58:41.531 | INFO | Downloaded 34GiB to GPU in 2.0084s (16.9289GiB/s) | 20260506 22:58:46.676 | INFO | Analyzing chunk frames = [5123584,5124096) | 20260506 22:58:46.676 | INFO | MemInfo: { | 20260506 22:58:46.676 | INFO | "gpu" : " 17 GB", | 20260506 22:58:46.676 | INFO | "huge" : "", | 20260506 22:58:46.676 | INFO | "huge_shared" : " 27 GB", | 20260506 22:58:46.676 | INFO | "malloc" : " 14 GB", | 20260506 22:58:46.676 | INFO | "pinned" : " 29 GB" | 20260506 22:58:46.676 | INFO | } | 20260506 22:58:46.676 | INFO | Perf report for the last 5.14568 seconds: | 20260506 22:58:46.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:58:46.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:46.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:46.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:46.677 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 22:58:46.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:46.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:46.677 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 22:58:46.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:46.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:46.677 | INFO | Uploaded 36GiB to GPU in 2.04457s (17.6076GiB/s) | 20260506 22:58:46.677 | INFO | Downloaded 34GiB to GPU in 2.00761s (16.9356GiB/s) | 20260506 22:58:46.784 | INFO | Starting a round of Baz IO | 20260506 22:58:49.910 | INFO | Populating RTMetrics report | 20260506 22:58:49.911 | INFO | RTMetrics report populated | 20260506 22:58:49.911 | INFO | Opening tmp RTMetrics json file | 20260506 22:58:49.912 | INFO | Serializing RTMetrics to json | 20260506 22:58:49.931 | INFO | Serialization done, swapping tmp file now... | 20260506 22:58:49.931 | INFO | RTMetrics done | 20260506 22:58:50.009 | INFO | Finished a round of Baz IO: Wrote 1.897 GiB in 3.224 seconds | 20260506 22:58:50.009 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.56% / 0.00% / 66.44% | 20260506 22:58:50.009 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.457712% / 7.94729e-06% / 5.96046e-05% | 20260506 22:58:50.009 | INFO | Baz IO overall: | 20260506 22:58:50.009 | INFO | BytesWritten : 2686996221055 | 20260506 22:58:50.009 | INFO | NumEvents : 1295949592043 | 20260506 22:58:51.811 | INFO | Analyzing chunk frames = [5124096,5124608) | 20260506 22:58:51.811 | INFO | MemInfo: { | 20260506 22:58:51.811 | INFO | "gpu" : " 17 GB", | 20260506 22:58:51.811 | INFO | "huge" : "", | 20260506 22:58:51.811 | INFO | "huge_shared" : " 27 GB", | 20260506 22:58:51.811 | INFO | "malloc" : " 14 GB", | 20260506 22:58:51.811 | INFO | "pinned" : " 29 GB" | 20260506 22:58:51.812 | INFO | } | 20260506 22:58:51.812 | INFO | Perf report for the last 5.13515 seconds: | 20260506 22:58:51.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:58:51.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:51.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:51.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:51.812 | INFO | ANALYSIS : 62.4%, 3.0, 96.9ms, 0 | 20260506 22:58:51.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:51.812 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 22:58:51.812 | INFO | PRE_HQ : 59.7%, 1.8, 56.7ms, 0 | 20260506 22:58:51.812 | INFO | BAZWRITER : 62.5%, 3.5, 110.8ms, 0 | 20260506 22:58:51.812 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:58:51.812 | INFO | Uploaded 36GiB to GPU in 2.03342s (17.7042GiB/s) | 20260506 22:58:51.812 | INFO | Downloaded 38GiB to GPU in 2.17066s (17.5062GiB/s) | 20260506 22:58:56.953 | INFO | Analyzing chunk frames = [5124608,5125120) | 20260506 22:58:56.953 | INFO | MemInfo: { | 20260506 22:58:56.953 | INFO | "gpu" : " 17 GB", | 20260506 22:58:56.953 | INFO | "huge" : "", | 20260506 22:58:56.953 | INFO | "huge_shared" : " 27 GB", | 20260506 22:58:56.953 | INFO | "malloc" : " 14 GB", | 20260506 22:58:56.953 | INFO | "pinned" : " 29 GB" | 20260506 22:58:56.953 | INFO | } | 20260506 22:58:56.953 | INFO | Perf report for the last 5.14162 seconds: | 20260506 22:58:56.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:58:56.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:56.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:56.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:56.953 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 22:58:56.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:56.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:56.953 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 22:58:56.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:56.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:58:56.953 | INFO | Uploaded 36GiB to GPU in 2.01128s (17.899GiB/s) | 20260506 22:58:56.954 | INFO | Downloaded 34GiB to GPU in 1.92315s (17.6794GiB/s) | 20260506 22:59:02.092 | INFO | Analyzing chunk frames = [5125120,5125632) | 20260506 22:59:02.092 | INFO | MemInfo: { | 20260506 22:59:02.092 | INFO | "gpu" : " 17 GB", | 20260506 22:59:02.092 | INFO | "huge" : "", | 20260506 22:59:02.092 | INFO | "huge_shared" : " 27 GB", | 20260506 22:59:02.092 | INFO | "malloc" : " 14 GB", | 20260506 22:59:02.092 | INFO | "pinned" : " 29 GB" | 20260506 22:59:02.092 | INFO | } | 20260506 22:59:02.092 | INFO | Perf report for the last 5.13851 seconds: | 20260506 22:59:02.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:59:02.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:02.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:02.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:02.092 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 22:59:02.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:02.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:02.092 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 22:59:02.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:02.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:02.092 | INFO | Uploaded 36GiB to GPU in 1.99488s (18.0462GiB/s) | 20260506 22:59:02.092 | INFO | Downloaded 34GiB to GPU in 1.92555s (17.6573GiB/s) | 20260506 22:59:07.233 | INFO | Analyzing chunk frames = [5125632,5126144) | 20260506 22:59:07.233 | INFO | MemInfo: { | 20260506 22:59:07.233 | INFO | "gpu" : " 17 GB", | 20260506 22:59:07.233 | INFO | "huge" : "", | 20260506 22:59:07.233 | INFO | "huge_shared" : " 27 GB", | 20260506 22:59:07.233 | INFO | "malloc" : " 14 GB", | 20260506 22:59:07.233 | INFO | "pinned" : " 29 GB" | 20260506 22:59:07.233 | INFO | } | 20260506 22:59:07.234 | INFO | Perf report for the last 5.14197 seconds: | 20260506 22:59:07.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:59:07.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:07.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:07.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:07.234 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 22:59:07.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:07.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:07.234 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 22:59:07.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:07.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:07.234 | INFO | Uploaded 36GiB to GPU in 1.96984s (18.2756GiB/s) | 20260506 22:59:07.234 | INFO | Downloaded 34GiB to GPU in 1.87733s (18.1108GiB/s) | 20260506 22:59:12.371 | INFO | Analyzing chunk frames = [5126144,5126656) | 20260506 22:59:12.371 | INFO | MemInfo: { | 20260506 22:59:12.371 | INFO | "gpu" : " 17 GB", | 20260506 22:59:12.371 | INFO | "huge" : "", | 20260506 22:59:12.371 | INFO | "huge_shared" : " 27 GB", | 20260506 22:59:12.371 | INFO | "malloc" : " 14 GB", | 20260506 22:59:12.371 | INFO | "pinned" : " 29 GB" | 20260506 22:59:12.371 | INFO | } | 20260506 22:59:12.371 | INFO | Perf report for the last 5.13769 seconds: | 20260506 22:59:12.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:59:12.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:12.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:12.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:12.371 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 22:59:12.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:12.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:12.372 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:59:12.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:12.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:12.372 | INFO | Uploaded 36GiB to GPU in 1.99107s (18.0807GiB/s) | 20260506 22:59:12.372 | INFO | Downloaded 34GiB to GPU in 1.92616s (17.6517GiB/s) | 20260506 22:59:17.509 | INFO | Analyzing chunk frames = [5126656,5127168) | 20260506 22:59:17.509 | INFO | MemInfo: { | 20260506 22:59:17.509 | INFO | "gpu" : " 17 GB", | 20260506 22:59:17.509 | INFO | "huge" : "", | 20260506 22:59:17.509 | INFO | "huge_shared" : " 27 GB", | 20260506 22:59:17.509 | INFO | "malloc" : " 14 GB", | 20260506 22:59:17.509 | INFO | "pinned" : " 29 GB" | 20260506 22:59:17.509 | INFO | } | 20260506 22:59:17.509 | INFO | Perf report for the last 5.13724 seconds: | 20260506 22:59:17.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:59:17.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:17.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:17.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:17.509 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 22:59:17.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:17.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:17.509 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 22:59:17.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:17.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:17.509 | INFO | Uploaded 36GiB to GPU in 2.03457s (17.6941GiB/s) | 20260506 22:59:17.509 | INFO | Downloaded 34GiB to GPU in 1.94006s (17.5252GiB/s) | 20260506 22:59:22.652 | INFO | Analyzing chunk frames = [5127168,5127680) | 20260506 22:59:22.652 | INFO | MemInfo: { | 20260506 22:59:22.652 | INFO | "gpu" : " 17 GB", | 20260506 22:59:22.652 | INFO | "huge" : "", | 20260506 22:59:22.652 | INFO | "huge_shared" : " 27 GB", | 20260506 22:59:22.652 | INFO | "malloc" : " 14 GB", | 20260506 22:59:22.652 | INFO | "pinned" : " 29 GB" | 20260506 22:59:22.652 | INFO | } | 20260506 22:59:22.652 | INFO | Perf report for the last 5.14381 seconds: | 20260506 22:59:22.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:59:22.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:22.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:22.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:22.653 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 22:59:22.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:22.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:22.653 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 22:59:22.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:22.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:22.653 | INFO | Uploaded 36GiB to GPU in 1.95757s (18.3901GiB/s) | 20260506 22:59:22.653 | INFO | Downloaded 34GiB to GPU in 1.99327s (17.0574GiB/s) | 20260506 22:59:27.793 | INFO | Analyzing chunk frames = [5127680,5128192) | 20260506 22:59:27.793 | INFO | MemInfo: { | 20260506 22:59:27.793 | INFO | "gpu" : " 17 GB", | 20260506 22:59:27.793 | INFO | "huge" : "", | 20260506 22:59:27.793 | INFO | "huge_shared" : " 27 GB", | 20260506 22:59:27.793 | INFO | "malloc" : " 14 GB", | 20260506 22:59:27.793 | INFO | "pinned" : " 29 GB" | 20260506 22:59:27.793 | INFO | } | 20260506 22:59:27.793 | INFO | Perf report for the last 5.14052 seconds: | 20260506 22:59:27.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:59:27.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:27.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:27.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:27.793 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 22:59:27.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:27.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:27.793 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 22:59:27.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:27.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:27.793 | INFO | Uploaded 36GiB to GPU in 2.00297s (17.9733GiB/s) | 20260506 22:59:27.793 | INFO | Downloaded 34GiB to GPU in 2.00651s (16.9449GiB/s) | 20260506 22:59:27.794 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 22:59:27.794 | INFO | Total mem usage: 18 GiB | 20260506 22:59:27.794 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 22:59:27.794 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 22:59:27.794 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 22:59:27.794 | INFO | High Water Breakdown: | 20260506 22:59:27.794 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 22:59:27.794 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 22:59:27.794 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 22:59:27.794 | INFO | ------------------------------------------ | 20260506 22:59:27.794 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:59:27.794 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 22:59:27.794 | INFO | Total mem usage: 26.5 GiB | 20260506 22:59:27.794 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 22:59:27.794 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 22:59:27.794 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 22:59:27.794 | INFO | High Water Breakdown: | 20260506 22:59:27.794 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:59:27.794 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:59:27.794 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:59:27.794 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:59:27.794 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:59:27.794 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:59:27.794 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 22:59:27.794 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 22:59:27.794 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:59:27.794 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 22:59:27.794 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 22:59:27.794 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 22:59:27.794 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 22:59:27.794 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 22:59:27.794 | INFO | ------------------------------------------ | 20260506 22:59:27.794 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:59:27.794 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 22:59:27.794 | INFO | Total mem usage: 25.6 GiB | 20260506 22:59:27.794 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 22:59:27.794 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 22:59:27.794 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 22:59:27.794 | INFO | High Water Breakdown: | 20260506 22:59:27.794 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 22:59:27.794 | INFO | ------------------------------------------ | 20260506 22:59:27.794 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:59:27.794 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 22:59:27.794 | INFO | Total mem usage: 15.7 GiB | 20260506 22:59:27.794 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 22:59:27.794 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 22:59:27.794 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 22:59:27.794 | INFO | High Water Breakdown: | 20260506 22:59:27.794 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 22:59:27.794 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 22:59:27.794 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 22:59:27.794 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 22:59:27.794 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 22:59:27.794 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 22:59:27.794 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 22:59:27.794 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 22:59:27.794 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 22:59:27.794 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 22:59:27.794 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 22:59:27.794 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 22:59:27.794 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 22:59:27.794 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 22:59:27.794 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 22:59:27.794 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 22:59:27.794 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 22:59:27.794 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 22:59:27.794 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 22:59:27.794 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 22:59:27.794 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 22:59:27.794 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 22:59:27.794 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 22:59:27.795 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 22:59:27.795 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 22:59:27.795 | INFO | ------------------------------------------ | 20260506 22:59:27.795 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 22:59:27.795 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 22:59:27.795 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:59:27.795 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:59:27.795 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.56e+01, 1.33e+01 / 2.38e+00 | 20260506 22:59:27.795 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 9.89e-02, 1.02e-02 / 5.75e-03 | 20260506 22:59:27.795 | INFO | PerfReport: 11.65%, Baseline -- 3.61e+00 / 4.34e+00, 5.42e-01 / 3.87e-01 | 20260506 22:59:27.795 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.44e+00, 1.76e-01 / 1.44e-01 | 20260506 22:59:27.795 | INFO | PerfReport: 16.07%, DME -- 3.98e+01 / 4.45e+01, 4.08e+00 / 2.77e+00 | 20260506 22:59:27.795 | INFO | PerfReport: 57.32%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.32e+00 / 1.81e+00 | 20260506 22:59:27.795 | INFO | PerfReport: 7.10%, PulseAccumulating -- 2.20e+00 / 2.74e+00, 3.84e-01 / 2.28e-01 | 20260506 22:59:27.795 | INFO | PerfReport: 3.22%, Metrics -- 9.96e-01 / 1.33e+00, 2.62e-01 / 1.09e-01 | 20260506 22:59:27.795 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.40e-01 / 3.88e-01, 5.31e-02 / 1.54e-02 | 20260506 22:59:27.795 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 22:59:27.795 | INFO | PerfReport: Gathered 1536 samples. | 20260506 22:59:27.795 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 22:59:27.795 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 22:59:27.795 | INFO | PerfReport: 21.60%, Upload -- 2.09e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260506 22:59:27.795 | INFO | PerfReport: 21.27%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 22:59:27.795 | INFO | PerfReport: 32.02%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 22:59:27.795 | INFO | PerfReport: 1.35%, LockUpload -- 1.31e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260506 22:59:27.795 | INFO | PerfReport: 1.23%, LockDownload -- 1.19e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260506 22:59:27.795 | INFO | PerfReport: 22.52%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 22:59:27.795 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.38e-02 / 0.00e+00 | 20260506 22:59:27.891 | INFO | Starting a round of Baz IO | 20260506 22:59:31.020 | INFO | Populating RTMetrics report | 20260506 22:59:31.020 | INFO | RTMetrics report populated | 20260506 22:59:31.020 | INFO | Opening tmp RTMetrics json file | 20260506 22:59:31.023 | INFO | Serializing RTMetrics to json | 20260506 22:59:31.041 | INFO | Serialization done, swapping tmp file now... | 20260506 22:59:31.043 | INFO | RTMetrics done | 20260506 22:59:31.118 | INFO | Finished a round of Baz IO: Wrote 1.897 GiB in 3.226 seconds | 20260506 22:59:31.118 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.56% / 0.00% / 66.44% | 20260506 22:59:31.118 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.456047% / 3.97364e-06% / 5.96046e-05% | 20260506 22:59:31.118 | INFO | Baz IO overall: | 20260506 22:59:31.118 | INFO | BytesWritten : 2689033028358 | 20260506 22:59:31.118 | INFO | NumEvents : 1296921811168 | 20260506 22:59:32.934 | INFO | Analyzing chunk frames = [5128192,5128704) | 20260506 22:59:32.934 | INFO | MemInfo: { | 20260506 22:59:32.934 | INFO | "gpu" : " 17 GB", | 20260506 22:59:32.934 | INFO | "huge" : "", | 20260506 22:59:32.934 | INFO | "huge_shared" : " 27 GB", | 20260506 22:59:32.934 | INFO | "malloc" : " 14 GB", | 20260506 22:59:32.934 | INFO | "pinned" : " 29 GB" | 20260506 22:59:32.934 | INFO | } | 20260506 22:59:32.934 | INFO | Perf report for the last 5.14107 seconds: | 20260506 22:59:32.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:59:32.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:32.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:32.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:32.934 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 22:59:32.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:32.934 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260506 22:59:32.934 | INFO | PRE_HQ : 59.8%, 1.8, 57.3ms, 0 | 20260506 22:59:32.934 | INFO | BAZWRITER : 62.4%, 3.5, 111.4ms, 0 | 20260506 22:59:32.934 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 22:59:32.934 | INFO | Uploaded 36GiB to GPU in 2.0134s (17.8802GiB/s) | 20260506 22:59:32.934 | INFO | Downloaded 38GiB to GPU in 2.16659s (17.5391GiB/s) | 20260506 22:59:38.070 | INFO | Analyzing chunk frames = [5128704,5129216) | 20260506 22:59:38.070 | INFO | MemInfo: { | 20260506 22:59:38.070 | INFO | "gpu" : " 17 GB", | 20260506 22:59:38.070 | INFO | "huge" : "", | 20260506 22:59:38.070 | INFO | "huge_shared" : " 27 GB", | 20260506 22:59:38.070 | INFO | "malloc" : " 14 GB", | 20260506 22:59:38.070 | INFO | "pinned" : " 29 GB" | 20260506 22:59:38.070 | INFO | } | 20260506 22:59:38.070 | INFO | Perf report for the last 5.13632 seconds: | 20260506 22:59:38.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:59:38.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:38.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:38.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:38.070 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 22:59:38.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:38.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:38.070 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 22:59:38.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:38.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:38.071 | INFO | Uploaded 36GiB to GPU in 1.98434s (18.142GiB/s) | 20260506 22:59:38.071 | INFO | Downloaded 34GiB to GPU in 1.92632s (17.6502GiB/s) | 20260506 22:59:43.202 | INFO | Analyzing chunk frames = [5129216,5129728) | 20260506 22:59:43.202 | INFO | MemInfo: { | 20260506 22:59:43.202 | INFO | "gpu" : " 17 GB", | 20260506 22:59:43.202 | INFO | "huge" : "", | 20260506 22:59:43.202 | INFO | "huge_shared" : " 27 GB", | 20260506 22:59:43.202 | INFO | "malloc" : " 14 GB", | 20260506 22:59:43.202 | INFO | "pinned" : " 29 GB" | 20260506 22:59:43.202 | INFO | } | 20260506 22:59:43.202 | INFO | Perf report for the last 5.13191 seconds: | 20260506 22:59:43.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:59:43.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:43.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:43.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:43.202 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 22:59:43.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:43.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:43.202 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 22:59:43.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:43.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:43.203 | INFO | Uploaded 36GiB to GPU in 1.95325s (18.4308GiB/s) | 20260506 22:59:43.203 | INFO | Downloaded 34GiB to GPU in 1.93101s (17.6073GiB/s) | 20260506 22:59:48.341 | INFO | Analyzing chunk frames = [5129728,5130240) | 20260506 22:59:48.341 | INFO | MemInfo: { | 20260506 22:59:48.341 | INFO | "gpu" : " 17 GB", | 20260506 22:59:48.341 | INFO | "huge" : "", | 20260506 22:59:48.341 | INFO | "huge_shared" : " 27 GB", | 20260506 22:59:48.341 | INFO | "malloc" : " 14 GB", | 20260506 22:59:48.341 | INFO | "pinned" : " 29 GB" | 20260506 22:59:48.341 | INFO | } | 20260506 22:59:48.341 | INFO | Perf report for the last 5.13908 seconds: | 20260506 22:59:48.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:59:48.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:48.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:48.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:48.341 | INFO | ANALYSIS : 59.5%, 3.0, 92.2ms, 0 | 20260506 22:59:48.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:48.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:48.342 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 22:59:48.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:48.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:48.342 | INFO | Uploaded 36GiB to GPU in 1.94448s (18.514GiB/s) | 20260506 22:59:48.342 | INFO | Downloaded 34GiB to GPU in 1.88121s (18.0735GiB/s) | 20260506 22:59:53.486 | INFO | Analyzing chunk frames = [5130240,5130752) | 20260506 22:59:53.486 | INFO | MemInfo: { | 20260506 22:59:53.486 | INFO | "gpu" : " 17 GB", | 20260506 22:59:53.486 | INFO | "huge" : "", | 20260506 22:59:53.486 | INFO | "huge_shared" : " 27 GB", | 20260506 22:59:53.486 | INFO | "malloc" : " 14 GB", | 20260506 22:59:53.486 | INFO | "pinned" : " 29 GB" | 20260506 22:59:53.486 | INFO | } | 20260506 22:59:53.486 | INFO | Perf report for the last 5.14486 seconds: | 20260506 22:59:53.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:59:53.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:53.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:53.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:53.487 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260506 22:59:53.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:53.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:53.487 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 22:59:53.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:53.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:53.487 | INFO | Uploaded 36GiB to GPU in 1.99595s (18.0365GiB/s) | 20260506 22:59:53.487 | INFO | Downloaded 34GiB to GPU in 1.92397s (17.6718GiB/s) | 20260506 22:59:58.624 | INFO | Analyzing chunk frames = [5130752,5131264) | 20260506 22:59:58.624 | INFO | MemInfo: { | 20260506 22:59:58.624 | INFO | "gpu" : " 17 GB", | 20260506 22:59:58.624 | INFO | "huge" : "", | 20260506 22:59:58.624 | INFO | "huge_shared" : " 27 GB", | 20260506 22:59:58.624 | INFO | "malloc" : " 14 GB", | 20260506 22:59:58.624 | INFO | "pinned" : " 29 GB" | 20260506 22:59:58.624 | INFO | } | 20260506 22:59:58.624 | INFO | Perf report for the last 5.13773 seconds: | 20260506 22:59:58.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 22:59:58.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:58.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:58.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:58.624 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 22:59:58.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:58.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:58.624 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 22:59:58.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:58.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 22:59:58.624 | INFO | Uploaded 36GiB to GPU in 1.99144s (18.0774GiB/s) | 20260506 22:59:58.624 | INFO | Downloaded 34GiB to GPU in 1.95292s (17.4098GiB/s) | 20260506 23:00:03.766 | INFO | Analyzing chunk frames = [5131264,5131776) | 20260506 23:00:03.766 | INFO | MemInfo: { | 20260506 23:00:03.766 | INFO | "gpu" : " 17 GB", | 20260506 23:00:03.766 | INFO | "huge" : "", | 20260506 23:00:03.766 | INFO | "huge_shared" : " 27 GB", | 20260506 23:00:03.766 | INFO | "malloc" : " 14 GB", | 20260506 23:00:03.766 | INFO | "pinned" : " 29 GB" | 20260506 23:00:03.766 | INFO | } | 20260506 23:00:03.766 | INFO | Perf report for the last 5.14245 seconds: | 20260506 23:00:03.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:00:03.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:03.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:03.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:03.767 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 23:00:03.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:03.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:03.767 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 23:00:03.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:03.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:03.767 | INFO | Uploaded 36GiB to GPU in 1.95471s (18.4171GiB/s) | 20260506 23:00:03.767 | INFO | Downloaded 34GiB to GPU in 1.9821s (17.1535GiB/s) | 20260506 23:00:08.907 | INFO | Analyzing chunk frames = [5131776,5132288) | 20260506 23:00:08.907 | INFO | MemInfo: { | 20260506 23:00:08.907 | INFO | "gpu" : " 17 GB", | 20260506 23:00:08.907 | INFO | "huge" : "", | 20260506 23:00:08.907 | INFO | "huge_shared" : " 27 GB", | 20260506 23:00:08.907 | INFO | "malloc" : " 14 GB", | 20260506 23:00:08.907 | INFO | "pinned" : " 29 GB" | 20260506 23:00:08.907 | INFO | } | 20260506 23:00:08.907 | INFO | Perf report for the last 5.14013 seconds: | 20260506 23:00:08.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:00:08.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:08.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:08.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:08.907 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 23:00:08.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:08.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:08.907 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 23:00:08.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:08.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:08.907 | INFO | Uploaded 36GiB to GPU in 2.03552s (17.6859GiB/s) | 20260506 23:00:08.907 | INFO | Downloaded 34GiB to GPU in 1.99202s (17.0681GiB/s) | 20260506 23:00:09.012 | INFO | Starting a round of Baz IO | 20260506 23:00:12.145 | INFO | Populating RTMetrics report | 20260506 23:00:12.145 | INFO | RTMetrics report populated | 20260506 23:00:12.145 | INFO | Opening tmp RTMetrics json file | 20260506 23:00:12.146 | INFO | Serializing RTMetrics to json | 20260506 23:00:12.150 | INFO | Serialization done, swapping tmp file now... | 20260506 23:00:12.158 | INFO | RTMetrics done | 20260506 23:00:12.246 | INFO | Finished a round of Baz IO: Wrote 1.896 GiB in 3.233 seconds | 20260506 23:00:12.246 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.56% / 0.00% / 66.44% | 20260506 23:00:12.246 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.455153% / 3.97364e-06% / 7.15256e-05% | 20260506 23:00:12.246 | INFO | Baz IO overall: | 20260506 23:00:12.246 | INFO | BytesWritten : 2691068600790 | 20260506 23:00:12.246 | INFO | NumEvents : 1297892986480 | 20260506 23:00:14.045 | INFO | Analyzing chunk frames = [5132288,5132800) | 20260506 23:00:14.045 | INFO | MemInfo: { | 20260506 23:00:14.045 | INFO | "gpu" : " 17 GB", | 20260506 23:00:14.045 | INFO | "huge" : "", | 20260506 23:00:14.045 | INFO | "huge_shared" : " 27 GB", | 20260506 23:00:14.045 | INFO | "malloc" : " 14 GB", | 20260506 23:00:14.046 | INFO | "pinned" : " 29 GB" | 20260506 23:00:14.046 | INFO | } | 20260506 23:00:14.046 | INFO | Perf report for the last 5.13891 seconds: | 20260506 23:00:14.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:00:14.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:14.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:14.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:14.046 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260506 23:00:14.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:14.046 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 23:00:14.046 | INFO | PRE_HQ : 60.5%, 1.9, 59.9ms, 0 | 20260506 23:00:14.046 | INFO | BAZWRITER : 62.5%, 3.6, 114.7ms, 0 | 20260506 23:00:14.046 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:00:14.046 | INFO | Uploaded 36GiB to GPU in 2.04178s (17.6317GiB/s) | 20260506 23:00:14.046 | INFO | Downloaded 38GiB to GPU in 2.18495s (17.3917GiB/s) | 20260506 23:00:19.182 | INFO | Analyzing chunk frames = [5132800,5133312) | 20260506 23:00:19.182 | INFO | MemInfo: { | 20260506 23:00:19.182 | INFO | "gpu" : " 17 GB", | 20260506 23:00:19.182 | INFO | "huge" : "", | 20260506 23:00:19.182 | INFO | "huge_shared" : " 27 GB", | 20260506 23:00:19.182 | INFO | "malloc" : " 14 GB", | 20260506 23:00:19.182 | INFO | "pinned" : " 29 GB" | 20260506 23:00:19.182 | INFO | } | 20260506 23:00:19.182 | INFO | Perf report for the last 5.13655 seconds: | 20260506 23:00:19.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:00:19.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:19.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:19.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:19.182 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:00:19.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:19.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:19.182 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 23:00:19.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:19.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:19.182 | INFO | Uploaded 36GiB to GPU in 2.01113s (17.9003GiB/s) | 20260506 23:00:19.183 | INFO | Downloaded 34GiB to GPU in 1.92858s (17.6295GiB/s) | 20260506 23:00:24.320 | INFO | Analyzing chunk frames = [5133312,5133824) | 20260506 23:00:24.320 | INFO | MemInfo: { | 20260506 23:00:24.320 | INFO | "gpu" : " 17 GB", | 20260506 23:00:24.320 | INFO | "huge" : "", | 20260506 23:00:24.320 | INFO | "huge_shared" : " 27 GB", | 20260506 23:00:24.320 | INFO | "malloc" : " 14 GB", | 20260506 23:00:24.320 | INFO | "pinned" : " 29 GB" | 20260506 23:00:24.320 | INFO | } | 20260506 23:00:24.320 | INFO | Perf report for the last 5.13807 seconds: | 20260506 23:00:24.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:00:24.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:24.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:24.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:24.320 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:00:24.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:24.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:24.320 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:00:24.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:24.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:24.321 | INFO | Uploaded 36GiB to GPU in 1.98898s (18.0997GiB/s) | 20260506 23:00:24.321 | INFO | Downloaded 34GiB to GPU in 1.92879s (17.6277GiB/s) | 20260506 23:00:29.459 | INFO | Analyzing chunk frames = [5133824,5134336) | 20260506 23:00:29.459 | INFO | MemInfo: { | 20260506 23:00:29.459 | INFO | "gpu" : " 17 GB", | 20260506 23:00:29.459 | INFO | "huge" : "", | 20260506 23:00:29.459 | INFO | "huge_shared" : " 27 GB", | 20260506 23:00:29.459 | INFO | "malloc" : " 14 GB", | 20260506 23:00:29.459 | INFO | "pinned" : " 29 GB" | 20260506 23:00:29.459 | INFO | } | 20260506 23:00:29.459 | INFO | Perf report for the last 5.13875 seconds: | 20260506 23:00:29.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:00:29.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:29.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:29.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:29.459 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 23:00:29.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:29.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:29.459 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:00:29.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:29.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:29.459 | INFO | Uploaded 36GiB to GPU in 1.97106s (18.2643GiB/s) | 20260506 23:00:29.459 | INFO | Downloaded 34GiB to GPU in 1.87552s (18.1283GiB/s) | 20260506 23:00:34.597 | INFO | Analyzing chunk frames = [5134336,5134848) | 20260506 23:00:34.597 | INFO | MemInfo: { | 20260506 23:00:34.597 | INFO | "gpu" : " 17 GB", | 20260506 23:00:34.597 | INFO | "huge" : "", | 20260506 23:00:34.597 | INFO | "huge_shared" : " 27 GB", | 20260506 23:00:34.597 | INFO | "malloc" : " 14 GB", | 20260506 23:00:34.597 | INFO | "pinned" : " 29 GB" | 20260506 23:00:34.597 | INFO | } | 20260506 23:00:34.597 | INFO | Perf report for the last 5.13813 seconds: | 20260506 23:00:34.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:00:34.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:34.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:34.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:34.597 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 23:00:34.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:34.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:34.597 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:00:34.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:34.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:34.597 | INFO | Uploaded 36GiB to GPU in 2.01357s (17.8787GiB/s) | 20260506 23:00:34.597 | INFO | Downloaded 34GiB to GPU in 1.9224s (17.6862GiB/s) | 20260506 23:00:39.751 | INFO | Analyzing chunk frames = [5134848,5135360) | 20260506 23:00:39.751 | INFO | MemInfo: { | 20260506 23:00:39.751 | INFO | "gpu" : " 17 GB", | 20260506 23:00:39.751 | INFO | "huge" : "", | 20260506 23:00:39.751 | INFO | "huge_shared" : " 27 GB", | 20260506 23:00:39.751 | INFO | "malloc" : " 14 GB", | 20260506 23:00:39.751 | INFO | "pinned" : " 29 GB" | 20260506 23:00:39.751 | INFO | } | 20260506 23:00:39.751 | INFO | Perf report for the last 5.15408 seconds: | 20260506 23:00:39.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:00:39.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:39.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:39.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:39.751 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 23:00:39.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:39.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:39.751 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:00:39.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:39.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:39.751 | INFO | Uploaded 36GiB to GPU in 2.03887s (17.6568GiB/s) | 20260506 23:00:39.752 | INFO | Downloaded 34GiB to GPU in 1.9534s (17.4056GiB/s) | 20260506 23:00:44.883 | INFO | Analyzing chunk frames = [5135360,5135872) | 20260506 23:00:44.883 | INFO | MemInfo: { | 20260506 23:00:44.883 | INFO | "gpu" : " 17 GB", | 20260506 23:00:44.883 | INFO | "huge" : "", | 20260506 23:00:44.883 | INFO | "huge_shared" : " 27 GB", | 20260506 23:00:44.883 | INFO | "malloc" : " 14 GB", | 20260506 23:00:44.883 | INFO | "pinned" : " 29 GB" | 20260506 23:00:44.883 | INFO | } | 20260506 23:00:44.883 | INFO | Perf report for the last 5.13204 seconds: | 20260506 23:00:44.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:00:44.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:44.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:44.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:44.883 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:00:44.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:44.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:44.883 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:00:44.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:44.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:44.884 | INFO | Uploaded 36GiB to GPU in 1.98464s (18.1394GiB/s) | 20260506 23:00:44.884 | INFO | Downloaded 34GiB to GPU in 1.99554s (17.038GiB/s) | 20260506 23:00:50.022 | INFO | Analyzing chunk frames = [5135872,5136384) | 20260506 23:00:50.022 | INFO | MemInfo: { | 20260506 23:00:50.022 | INFO | "gpu" : " 17 GB", | 20260506 23:00:50.022 | INFO | "huge" : "", | 20260506 23:00:50.022 | INFO | "huge_shared" : " 27 GB", | 20260506 23:00:50.022 | INFO | "malloc" : " 14 GB", | 20260506 23:00:50.022 | INFO | "pinned" : " 29 GB" | 20260506 23:00:50.022 | INFO | } | 20260506 23:00:50.022 | INFO | Perf report for the last 5.13843 seconds: | 20260506 23:00:50.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:00:50.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:50.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:50.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:50.022 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:00:50.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:50.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:50.022 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:00:50.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:50.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:50.022 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:00:50.022 | INFO | Total mem usage: 18 GiB | 20260506 23:00:50.022 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:00:50.022 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 23:00:50.022 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:00:50.022 | INFO | High Water Breakdown: | 20260506 23:00:50.022 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:00:50.022 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:00:50.022 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:00:50.022 | INFO | ------------------------------------------ | 20260506 23:00:50.022 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:00:50.022 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:00:50.022 | INFO | Total mem usage: 26.5 GiB | 20260506 23:00:50.022 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:00:50.022 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:00:50.022 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:00:50.022 | INFO | High Water Breakdown: | 20260506 23:00:50.022 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:00:50.022 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:00:50.022 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:00:50.022 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:00:50.022 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:00:50.022 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:00:50.022 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:00:50.022 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:00:50.022 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:00:50.022 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:00:50.022 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:00:50.022 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:00:50.022 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:00:50.022 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:00:50.022 | INFO | ------------------------------------------ | 20260506 23:00:50.022 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:00:50.022 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:00:50.022 | INFO | Total mem usage: 25.6 GiB | 20260506 23:00:50.022 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:00:50.022 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:00:50.022 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:00:50.022 | INFO | High Water Breakdown: | 20260506 23:00:50.022 | INFO | Uploaded 36GiB to GPU in 1.97804s (18.1998GiB/s) | 20260506 23:00:50.023 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:00:50.023 | INFO | Downloaded 34GiB to GPU in 1.99731s (17.0229GiB/s) | 20260506 23:00:50.023 | INFO | ------------------------------------------ | 20260506 23:00:50.023 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:00:50.023 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:00:50.023 | INFO | Total mem usage: 15.7 GiB | 20260506 23:00:50.023 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:00:50.023 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:00:50.023 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:00:50.023 | INFO | High Water Breakdown: | 20260506 23:00:50.023 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:00:50.023 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:00:50.023 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:00:50.023 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:00:50.023 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:00:50.023 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:00:50.023 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:00:50.023 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:00:50.023 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:00:50.023 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:00:50.023 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:00:50.023 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:00:50.023 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:00:50.023 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:00:50.023 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:00:50.023 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:00:50.023 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:00:50.023 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:00:50.023 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:00:50.023 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:00:50.023 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:00:50.023 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:00:50.023 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:00:50.023 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:00:50.023 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:00:50.023 | INFO | ------------------------------------------ | 20260506 23:00:50.023 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:00:50.023 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:00:50.023 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:00:50.023 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:00:50.023 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.60e+01, 1.34e+01 / 2.52e+00 | 20260506 23:00:50.023 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.02e-01, 1.02e-02 / 3.86e-03 | 20260506 23:00:50.023 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.44e+00, 5.47e-01 / 2.64e-01 | 20260506 23:00:50.023 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.49e+00, 1.81e-01 / 9.73e-02 | 20260506 23:00:50.023 | INFO | PerfReport: 16.10%, DME -- 4.00e+01 / 4.50e+01, 4.14e+00 / 2.61e+00 | 20260506 23:00:50.023 | INFO | PerfReport: 57.33%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.38e+00 / 1.33e+00 | 20260506 23:00:50.023 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.19e+00 / 2.76e+00, 3.80e-01 / 1.62e-01 | 20260506 23:00:50.023 | INFO | PerfReport: 3.20%, Metrics -- 9.94e-01 / 1.34e+00, 2.63e-01 / 1.38e-01 | 20260506 23:00:50.023 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.39e-01 / 3.91e-01, 5.29e-02 / 2.38e-02 | 20260506 23:00:50.023 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:00:50.023 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:00:50.024 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:00:50.024 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 23:00:50.024 | INFO | PerfReport: 21.49%, Upload -- 2.08e+01 / 0.00e+00, 6.94e+00 / 0.00e+00 | 20260506 23:00:50.024 | INFO | PerfReport: 21.25%, Download -- 2.05e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 23:00:50.024 | INFO | PerfReport: 32.12%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 23:00:50.024 | INFO | PerfReport: 1.24%, LockUpload -- 1.20e+00 / 0.00e+00, 4.74e+00 / 0.00e+00 | 20260506 23:00:50.024 | INFO | PerfReport: 1.27%, LockDownload -- 1.23e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260506 23:00:50.024 | INFO | PerfReport: 22.62%, LockCompute -- 2.19e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 23:00:50.024 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260506 23:00:50.121 | INFO | Starting a round of Baz IO | 20260506 23:00:53.248 | INFO | Populating RTMetrics report | 20260506 23:00:53.248 | INFO | RTMetrics report populated | 20260506 23:00:53.248 | INFO | Opening tmp RTMetrics json file | 20260506 23:00:53.267 | INFO | Serializing RTMetrics to json | 20260506 23:00:53.269 | INFO | Serialization done, swapping tmp file now... | 20260506 23:00:53.269 | INFO | RTMetrics done | 20260506 23:00:53.347 | INFO | Finished a round of Baz IO: Wrote 1.895 GiB in 3.226 seconds | 20260506 23:00:53.347 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.56% / 0.00% / 66.44% | 20260506 23:00:53.347 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.454001% / 0% / 6.35783e-05% | 20260506 23:00:53.347 | INFO | Baz IO overall: | 20260506 23:00:53.347 | INFO | BytesWritten : 2693103249835 | 20260506 23:00:53.348 | INFO | NumEvents : 1298863452015 | 20260506 23:00:55.158 | INFO | Analyzing chunk frames = [5136384,5136896) | 20260506 23:00:55.158 | INFO | MemInfo: { | 20260506 23:00:55.158 | INFO | "gpu" : " 17 GB", | 20260506 23:00:55.158 | INFO | "huge" : "", | 20260506 23:00:55.158 | INFO | "huge_shared" : " 27 GB", | 20260506 23:00:55.158 | INFO | "malloc" : " 14 GB", | 20260506 23:00:55.158 | INFO | "pinned" : " 29 GB" | 20260506 23:00:55.158 | INFO | } | 20260506 23:00:55.158 | INFO | Perf report for the last 5.13654 seconds: | 20260506 23:00:55.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:00:55.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:55.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:55.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:55.158 | INFO | ANALYSIS : 62.2%, 2.9, 95.9ms, 0 | 20260506 23:00:55.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:00:55.158 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 23:00:55.158 | INFO | PRE_HQ : 59.6%, 1.8, 56.5ms, 0 | 20260506 23:00:55.159 | INFO | BAZWRITER : 62.4%, 3.8, 122.8ms, 0 | 20260506 23:00:55.159 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:00:55.159 | INFO | Uploaded 36GiB to GPU in 2.02262s (17.7987GiB/s) | 20260506 23:00:55.159 | INFO | Downloaded 38GiB to GPU in 2.19126s (17.3416GiB/s) | 20260506 23:01:00.291 | INFO | Analyzing chunk frames = [5136896,5137408) | 20260506 23:01:00.291 | INFO | MemInfo: { | 20260506 23:01:00.291 | INFO | "gpu" : " 17 GB", | 20260506 23:01:00.291 | INFO | "huge" : "", | 20260506 23:01:00.291 | INFO | "huge_shared" : " 27 GB", | 20260506 23:01:00.291 | INFO | "malloc" : " 14 GB", | 20260506 23:01:00.292 | INFO | "pinned" : " 29 GB" | 20260506 23:01:00.292 | INFO | } | 20260506 23:01:00.292 | INFO | Perf report for the last 5.13332 seconds: | 20260506 23:01:00.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:01:00.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:00.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:00.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:00.292 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:01:00.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:00.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:00.292 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:01:00.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:00.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:00.292 | INFO | Uploaded 36GiB to GPU in 1.99148s (18.077GiB/s) | 20260506 23:01:00.292 | INFO | Downloaded 34GiB to GPU in 1.93206s (17.5978GiB/s) | 20260506 23:01:05.434 | INFO | Analyzing chunk frames = [5137408,5137920) | 20260506 23:01:05.434 | INFO | MemInfo: { | 20260506 23:01:05.434 | INFO | "gpu" : " 17 GB", | 20260506 23:01:05.434 | INFO | "huge" : "", | 20260506 23:01:05.434 | INFO | "huge_shared" : " 27 GB", | 20260506 23:01:05.434 | INFO | "malloc" : " 14 GB", | 20260506 23:01:05.434 | INFO | "pinned" : " 29 GB" | 20260506 23:01:05.434 | INFO | } | 20260506 23:01:05.434 | INFO | Perf report for the last 5.14222 seconds: | 20260506 23:01:05.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:01:05.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:05.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:05.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:05.434 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260506 23:01:05.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:05.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:05.434 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:01:05.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:05.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:05.434 | INFO | Uploaded 36GiB to GPU in 1.96262s (18.3428GiB/s) | 20260506 23:01:05.434 | INFO | Downloaded 34GiB to GPU in 1.92427s (17.669GiB/s) | 20260506 23:01:10.580 | INFO | Analyzing chunk frames = [5137920,5138432) | 20260506 23:01:10.580 | INFO | MemInfo: { | 20260506 23:01:10.580 | INFO | "gpu" : " 17 GB", | 20260506 23:01:10.580 | INFO | "huge" : "", | 20260506 23:01:10.580 | INFO | "huge_shared" : " 27 GB", | 20260506 23:01:10.580 | INFO | "malloc" : " 14 GB", | 20260506 23:01:10.580 | INFO | "pinned" : " 29 GB" | 20260506 23:01:10.580 | INFO | } | 20260506 23:01:10.581 | INFO | Perf report for the last 5.1467 seconds: | 20260506 23:01:10.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:01:10.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:10.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:10.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:10.581 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 23:01:10.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:10.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:10.581 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:01:10.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:10.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:10.581 | INFO | Uploaded 36GiB to GPU in 1.94531s (18.5061GiB/s) | 20260506 23:01:10.581 | INFO | Downloaded 34GiB to GPU in 1.8819s (18.0668GiB/s) | 20260506 23:01:15.715 | INFO | Analyzing chunk frames = [5138432,5138944) | 20260506 23:01:15.715 | INFO | MemInfo: { | 20260506 23:01:15.715 | INFO | "gpu" : " 17 GB", | 20260506 23:01:15.715 | INFO | "huge" : "", | 20260506 23:01:15.715 | INFO | "huge_shared" : " 27 GB", | 20260506 23:01:15.715 | INFO | "malloc" : " 14 GB", | 20260506 23:01:15.715 | INFO | "pinned" : " 29 GB" | 20260506 23:01:15.715 | INFO | } | 20260506 23:01:15.715 | INFO | Perf report for the last 5.1343 seconds: | 20260506 23:01:15.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:01:15.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:15.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:15.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:15.715 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260506 23:01:15.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:15.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:15.715 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:01:15.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:15.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:15.716 | INFO | Uploaded 36GiB to GPU in 1.97705s (18.209GiB/s) | 20260506 23:01:15.716 | INFO | Downloaded 34GiB to GPU in 1.93069s (17.6102GiB/s) | 20260506 23:01:20.854 | INFO | Analyzing chunk frames = [5138944,5139456) | 20260506 23:01:20.854 | INFO | MemInfo: { | 20260506 23:01:20.854 | INFO | "gpu" : " 17 GB", | 20260506 23:01:20.854 | INFO | "huge" : "", | 20260506 23:01:20.854 | INFO | "huge_shared" : " 27 GB", | 20260506 23:01:20.854 | INFO | "malloc" : " 14 GB", | 20260506 23:01:20.854 | INFO | "pinned" : " 29 GB" | 20260506 23:01:20.854 | INFO | } | 20260506 23:01:20.854 | INFO | Perf report for the last 5.13954 seconds: | 20260506 23:01:20.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:01:20.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:20.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:20.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:20.854 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 23:01:20.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:20.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:20.854 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:01:20.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:20.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:20.855 | INFO | Uploaded 36GiB to GPU in 2.01232s (17.8898GiB/s) | 20260506 23:01:20.855 | INFO | Downloaded 34GiB to GPU in 1.93409s (17.5793GiB/s) | 20260506 23:01:25.997 | INFO | Analyzing chunk frames = [5139456,5139968) | 20260506 23:01:25.997 | INFO | MemInfo: { | 20260506 23:01:25.997 | INFO | "gpu" : " 17 GB", | 20260506 23:01:25.997 | INFO | "huge" : "", | 20260506 23:01:25.997 | INFO | "huge_shared" : " 27 GB", | 20260506 23:01:25.997 | INFO | "malloc" : " 14 GB", | 20260506 23:01:25.997 | INFO | "pinned" : " 29 GB" | 20260506 23:01:25.997 | INFO | } | 20260506 23:01:25.997 | INFO | Perf report for the last 5.14259 seconds: | 20260506 23:01:25.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:01:25.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:25.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:25.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:25.997 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 23:01:25.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:25.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:25.997 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 23:01:25.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:25.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:25.997 | INFO | Uploaded 36GiB to GPU in 1.95439s (18.4201GiB/s) | 20260506 23:01:25.998 | INFO | Downloaded 34GiB to GPU in 1.97968s (17.1745GiB/s) | 20260506 23:01:31.133 | INFO | Analyzing chunk frames = [5139968,5140480) | 20260506 23:01:31.133 | INFO | MemInfo: { | 20260506 23:01:31.133 | INFO | "gpu" : " 17 GB", | 20260506 23:01:31.133 | INFO | "huge" : "", | 20260506 23:01:31.133 | INFO | "huge_shared" : " 27 GB", | 20260506 23:01:31.133 | INFO | "malloc" : " 14 GB", | 20260506 23:01:31.134 | INFO | "pinned" : " 29 GB" | 20260506 23:01:31.134 | INFO | } | 20260506 23:01:31.134 | INFO | Perf report for the last 5.13674 seconds: | 20260506 23:01:31.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:01:31.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:31.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:31.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:31.134 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260506 23:01:31.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:31.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:31.134 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:01:31.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:31.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:31.134 | INFO | Uploaded 36GiB to GPU in 2.02234s (17.8012GiB/s) | 20260506 23:01:31.134 | INFO | Downloaded 34GiB to GPU in 2.0065s (16.9449GiB/s) | 20260506 23:01:31.234 | INFO | Starting a round of Baz IO | 20260506 23:01:34.366 | INFO | Populating RTMetrics report | 20260506 23:01:34.367 | INFO | RTMetrics report populated | 20260506 23:01:34.367 | INFO | Opening tmp RTMetrics json file | 20260506 23:01:34.368 | INFO | Serializing RTMetrics to json | 20260506 23:01:34.373 | INFO | Serialization done, swapping tmp file now... | 20260506 23:01:34.389 | INFO | RTMetrics done | 20260506 23:01:34.461 | INFO | Finished a round of Baz IO: Wrote 1.895 GiB in 3.227 seconds | 20260506 23:01:34.461 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.55% / 0.00% / 66.44% | 20260506 23:01:34.461 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.45317% / 7.94729e-06% / 5.96046e-05% | 20260506 23:01:34.461 | INFO | Baz IO overall: | 20260506 23:01:34.461 | INFO | BytesWritten : 2695138360245 | 20260506 23:01:34.461 | INFO | NumEvents : 1299834405802 | 20260506 23:01:36.271 | INFO | Analyzing chunk frames = [5140480,5140992) | 20260506 23:01:36.271 | INFO | MemInfo: { | 20260506 23:01:36.271 | INFO | "gpu" : " 17 GB", | 20260506 23:01:36.271 | INFO | "huge" : "", | 20260506 23:01:36.271 | INFO | "huge_shared" : " 27 GB", | 20260506 23:01:36.271 | INFO | "malloc" : " 14 GB", | 20260506 23:01:36.271 | INFO | "pinned" : " 29 GB" | 20260506 23:01:36.271 | INFO | } | 20260506 23:01:36.271 | INFO | Perf report for the last 5.13731 seconds: | 20260506 23:01:36.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:01:36.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:36.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:36.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:36.271 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 23:01:36.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:36.271 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 23:01:36.271 | INFO | PRE_HQ : 59.7%, 1.8, 56.5ms, 0 | 20260506 23:01:36.271 | INFO | BAZWRITER : 62.4%, 3.4, 109.8ms, 0 | 20260506 23:01:36.271 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:01:36.271 | INFO | Uploaded 36GiB to GPU in 2.032s (17.7165GiB/s) | 20260506 23:01:36.271 | INFO | Downloaded 38GiB to GPU in 2.16875s (17.5216GiB/s) | 20260506 23:01:41.407 | INFO | Analyzing chunk frames = [5140992,5141504) | 20260506 23:01:41.407 | INFO | MemInfo: { | 20260506 23:01:41.407 | INFO | "gpu" : " 17 GB", | 20260506 23:01:41.407 | INFO | "huge" : "", | 20260506 23:01:41.407 | INFO | "huge_shared" : " 27 GB", | 20260506 23:01:41.407 | INFO | "malloc" : " 14 GB", | 20260506 23:01:41.407 | INFO | "pinned" : " 29 GB" | 20260506 23:01:41.407 | INFO | } | 20260506 23:01:41.408 | INFO | Perf report for the last 5.13648 seconds: | 20260506 23:01:41.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:01:41.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:41.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:41.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:41.408 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:01:41.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:41.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:41.408 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260506 23:01:41.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:41.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:41.408 | INFO | Uploaded 36GiB to GPU in 1.99854s (18.0131GiB/s) | 20260506 23:01:41.408 | INFO | Downloaded 34GiB to GPU in 1.9253s (17.6596GiB/s) | 20260506 23:01:46.546 | INFO | Analyzing chunk frames = [5141504,5142016) | 20260506 23:01:46.546 | INFO | MemInfo: { | 20260506 23:01:46.546 | INFO | "gpu" : " 17 GB", | 20260506 23:01:46.546 | INFO | "huge" : "", | 20260506 23:01:46.546 | INFO | "huge_shared" : " 27 GB", | 20260506 23:01:46.546 | INFO | "malloc" : " 14 GB", | 20260506 23:01:46.546 | INFO | "pinned" : " 29 GB" | 20260506 23:01:46.546 | INFO | } | 20260506 23:01:46.547 | INFO | Perf report for the last 5.13892 seconds: | 20260506 23:01:46.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:01:46.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:46.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:46.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:46.547 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260506 23:01:46.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:46.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:46.547 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:01:46.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:46.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:46.547 | INFO | Uploaded 36GiB to GPU in 1.97431s (18.2342GiB/s) | 20260506 23:01:46.547 | INFO | Downloaded 34GiB to GPU in 1.9204s (17.7046GiB/s) | 20260506 23:01:51.695 | INFO | Analyzing chunk frames = [5142016,5142528) | 20260506 23:01:51.695 | INFO | MemInfo: { | 20260506 23:01:51.695 | INFO | "gpu" : " 17 GB", | 20260506 23:01:51.695 | INFO | "huge" : "", | 20260506 23:01:51.695 | INFO | "huge_shared" : " 27 GB", | 20260506 23:01:51.695 | INFO | "malloc" : " 14 GB", | 20260506 23:01:51.695 | INFO | "pinned" : " 29 GB" | 20260506 23:01:51.695 | INFO | } | 20260506 23:01:51.695 | INFO | Perf report for the last 5.14869 seconds: | 20260506 23:01:51.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:01:51.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:51.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:51.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:51.695 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260506 23:01:51.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:51.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:51.695 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:01:51.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:51.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:51.696 | INFO | Uploaded 36GiB to GPU in 1.96129s (18.3553GiB/s) | 20260506 23:01:51.696 | INFO | Downloaded 34GiB to GPU in 1.87784s (18.1059GiB/s) | 20260506 23:01:56.837 | INFO | Analyzing chunk frames = [5142528,5143040) | 20260506 23:01:56.837 | INFO | MemInfo: { | 20260506 23:01:56.838 | INFO | "gpu" : " 17 GB", | 20260506 23:01:56.838 | INFO | "huge" : "", | 20260506 23:01:56.838 | INFO | "huge_shared" : " 27 GB", | 20260506 23:01:56.838 | INFO | "malloc" : " 14 GB", | 20260506 23:01:56.838 | INFO | "pinned" : " 29 GB" | 20260506 23:01:56.838 | INFO | } | 20260506 23:01:56.838 | INFO | Perf report for the last 5.14236 seconds: | 20260506 23:01:56.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:01:56.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:56.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:56.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:56.838 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 23:01:56.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:56.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:56.838 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:01:56.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:56.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:01:56.838 | INFO | Uploaded 36GiB to GPU in 1.99525s (18.0428GiB/s) | 20260506 23:01:56.838 | INFO | Downloaded 34GiB to GPU in 1.92288s (17.6818GiB/s) | 20260506 23:02:01.969 | INFO | Analyzing chunk frames = [5143040,5143552) | 20260506 23:02:01.969 | INFO | MemInfo: { | 20260506 23:02:01.969 | INFO | "gpu" : " 17 GB", | 20260506 23:02:01.969 | INFO | "huge" : "", | 20260506 23:02:01.969 | INFO | "huge_shared" : " 27 GB", | 20260506 23:02:01.969 | INFO | "malloc" : " 14 GB", | 20260506 23:02:01.969 | INFO | "pinned" : " 29 GB" | 20260506 23:02:01.969 | INFO | } | 20260506 23:02:01.969 | INFO | Perf report for the last 5.1316 seconds: | 20260506 23:02:01.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:02:01.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:01.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:01.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:01.969 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 23:02:01.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:01.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:01.969 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 23:02:01.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:01.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:01.969 | INFO | Uploaded 36GiB to GPU in 2.02968s (17.7367GiB/s) | 20260506 23:02:01.970 | INFO | Downloaded 34GiB to GPU in 1.92472s (17.6649GiB/s) | 20260506 23:02:07.107 | INFO | Analyzing chunk frames = [5143552,5144064) | 20260506 23:02:07.107 | INFO | MemInfo: { | 20260506 23:02:07.107 | INFO | "gpu" : " 17 GB", | 20260506 23:02:07.107 | INFO | "huge" : "", | 20260506 23:02:07.107 | INFO | "huge_shared" : " 27 GB", | 20260506 23:02:07.107 | INFO | "malloc" : " 14 GB", | 20260506 23:02:07.107 | INFO | "pinned" : " 29 GB" | 20260506 23:02:07.107 | INFO | } | 20260506 23:02:07.107 | INFO | Perf report for the last 5.13818 seconds: | 20260506 23:02:07.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:02:07.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:07.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:07.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:07.108 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 23:02:07.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:07.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:07.108 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 23:02:07.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:07.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:07.108 | INFO | Uploaded 36GiB to GPU in 1.96594s (18.3119GiB/s) | 20260506 23:02:07.108 | INFO | Downloaded 34GiB to GPU in 2.00189s (16.9839GiB/s) | 20260506 23:02:12.245 | INFO | Analyzing chunk frames = [5144064,5144576) | 20260506 23:02:12.245 | INFO | MemInfo: { | 20260506 23:02:12.245 | INFO | "gpu" : " 17 GB", | 20260506 23:02:12.245 | INFO | "huge" : "", | 20260506 23:02:12.245 | INFO | "huge_shared" : " 27 GB", | 20260506 23:02:12.245 | INFO | "malloc" : " 14 GB", | 20260506 23:02:12.245 | INFO | "pinned" : " 29 GB" | 20260506 23:02:12.245 | INFO | } | 20260506 23:02:12.246 | INFO | Perf report for the last 5.13809 seconds: | 20260506 23:02:12.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:02:12.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:12.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:12.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:12.246 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 23:02:12.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:12.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:12.246 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:02:12.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:12.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:12.246 | INFO | Uploaded 36GiB to GPU in 2.04067s (17.6413GiB/s) | 20260506 23:02:12.246 | INFO | Downloaded 34GiB to GPU in 2.00286s (16.9757GiB/s) | 20260506 23:02:12.246 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:02:12.246 | INFO | Total mem usage: 18 GiB | 20260506 23:02:12.246 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:02:12.246 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -19.8%) | 20260506 23:02:12.246 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:02:12.246 | INFO | High Water Breakdown: | 20260506 23:02:12.246 | INFO | 61% : 9.16 GiB ( 80.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:02:12.246 | INFO | 38.9% : 5.85 GiB ( 87.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:02:12.246 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:02:12.246 | INFO | ------------------------------------------ | 20260506 23:02:12.246 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:02:12.246 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:02:12.246 | INFO | Total mem usage: 26.5 GiB | 20260506 23:02:12.246 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:02:12.246 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:02:12.246 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:02:12.246 | INFO | High Water Breakdown: | 20260506 23:02:12.246 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:02:12.246 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:02:12.246 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:02:12.246 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:02:12.246 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:02:12.246 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:02:12.246 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:02:12.246 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:02:12.246 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:02:12.246 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:02:12.246 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:02:12.246 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:02:12.246 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:02:12.246 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:02:12.246 | INFO | ------------------------------------------ | 20260506 23:02:12.246 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:02:12.246 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:02:12.246 | INFO | Total mem usage: 25.6 GiB | 20260506 23:02:12.246 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:02:12.246 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:02:12.246 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:02:12.246 | INFO | High Water Breakdown: | 20260506 23:02:12.246 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:02:12.246 | INFO | ------------------------------------------ | 20260506 23:02:12.246 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:02:12.246 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:02:12.246 | INFO | Total mem usage: 15.7 GiB | 20260506 23:02:12.246 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:02:12.246 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:02:12.246 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:02:12.246 | INFO | High Water Breakdown: | 20260506 23:02:12.246 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:02:12.246 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:02:12.246 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:02:12.246 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:02:12.246 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:02:12.246 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:02:12.247 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:02:12.247 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:02:12.247 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:02:12.247 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:02:12.247 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:02:12.247 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:02:12.247 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:02:12.247 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:02:12.247 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:02:12.247 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:02:12.247 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:02:12.247 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:02:12.247 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:02:12.247 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:02:12.247 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:02:12.247 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:02:12.247 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:02:12.247 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:02:12.247 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:02:12.247 | INFO | ------------------------------------------ | 20260506 23:02:12.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:02:12.247 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:02:12.247 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:02:12.247 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:02:12.247 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.58e+01, 1.32e+01 / 2.45e+00 | 20260506 23:02:12.247 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 1.01e-01, 9.55e-03 / 4.91e-03 | 20260506 23:02:12.247 | INFO | PerfReport: 11.66%, Baseline -- 3.61e+00 / 4.44e+00, 5.45e-01 / 2.46e-01 | 20260506 23:02:12.247 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.46e+00, 1.77e-01 / 1.15e-01 | 20260506 23:02:12.247 | INFO | PerfReport: 16.09%, DME -- 3.99e+01 / 4.48e+01, 4.21e+00 / 3.04e+00 | 20260506 23:02:12.247 | INFO | PerfReport: 57.33%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.33e+00 / 1.73e+00 | 20260506 23:02:12.247 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.19e+00 / 2.77e+00, 3.78e-01 / 1.88e-01 | 20260506 23:02:12.247 | INFO | PerfReport: 3.21%, Metrics -- 9.96e-01 / 1.34e+00, 2.65e-01 / 1.67e-01 | 20260506 23:02:12.247 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.40e-01 / 3.91e-01, 5.33e-02 / 2.68e-02 | 20260506 23:02:12.247 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:02:12.247 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:02:12.247 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:02:12.247 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 23:02:12.247 | INFO | PerfReport: 21.49%, Upload -- 2.08e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260506 23:02:12.247 | INFO | PerfReport: 21.25%, Download -- 2.05e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 23:02:12.247 | INFO | PerfReport: 32.09%, Compute -- 3.10e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260506 23:02:12.247 | INFO | PerfReport: 1.28%, LockUpload -- 1.23e+00 / 0.00e+00, 4.79e+00 / 0.00e+00 | 20260506 23:02:12.247 | INFO | PerfReport: 1.30%, LockDownload -- 1.26e+00 / 0.00e+00, 3.16e+00 / 0.00e+00 | 20260506 23:02:12.247 | INFO | PerfReport: 22.57%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 23:02:12.247 | INFO | PerfReport: 0.02%, Misc -- 1.55e-02 / 0.00e+00, 3.88e-02 / 0.00e+00 | 20260506 23:02:12.355 | INFO | Starting a round of Baz IO | 20260506 23:02:15.482 | INFO | Populating RTMetrics report | 20260506 23:02:15.482 | INFO | RTMetrics report populated | 20260506 23:02:15.482 | INFO | Opening tmp RTMetrics json file | 20260506 23:02:15.484 | INFO | Serializing RTMetrics to json | 20260506 23:02:15.503 | INFO | Serialization done, swapping tmp file now... | 20260506 23:02:15.503 | INFO | RTMetrics done | 20260506 23:02:15.576 | INFO | Finished a round of Baz IO: Wrote 1.892 GiB in 3.22 seconds | 20260506 23:02:15.576 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.55% / 0.00% / 66.45% | 20260506 23:02:15.576 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.451052% / 0% / 7.54992e-05% | 20260506 23:02:15.576 | INFO | Baz IO overall: | 20260506 23:02:15.576 | INFO | BytesWritten : 2697169834070 | 20260506 23:02:15.576 | INFO | NumEvents : 1300803083197 | 20260506 23:02:17.385 | INFO | Analyzing chunk frames = [5144576,5145088) | 20260506 23:02:17.385 | INFO | MemInfo: { | 20260506 23:02:17.385 | INFO | "gpu" : " 17 GB", | 20260506 23:02:17.385 | INFO | "huge" : "", | 20260506 23:02:17.385 | INFO | "huge_shared" : " 27 GB", | 20260506 23:02:17.385 | INFO | "malloc" : " 14 GB", | 20260506 23:02:17.385 | INFO | "pinned" : " 29 GB" | 20260506 23:02:17.385 | INFO | } | 20260506 23:02:17.385 | INFO | Perf report for the last 5.13944 seconds: | 20260506 23:02:17.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:02:17.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:17.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:17.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:17.385 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 23:02:17.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:17.385 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260506 23:02:17.385 | INFO | PRE_HQ : 59.7%, 1.8, 56.6ms, 0 | 20260506 23:02:17.385 | INFO | BAZWRITER : 62.3%, 3.5, 111.2ms, 0 | 20260506 23:02:17.385 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:02:17.385 | INFO | Uploaded 36GiB to GPU in 2.02871s (17.7453GiB/s) | 20260506 23:02:17.385 | INFO | Downloaded 38GiB to GPU in 2.18722s (17.3736GiB/s) | 20260506 23:02:22.529 | INFO | Analyzing chunk frames = [5145088,5145600) | 20260506 23:02:22.529 | INFO | MemInfo: { | 20260506 23:02:22.529 | INFO | "gpu" : " 17 GB", | 20260506 23:02:22.530 | INFO | "huge" : "", | 20260506 23:02:22.530 | INFO | "huge_shared" : " 27 GB", | 20260506 23:02:22.530 | INFO | "malloc" : " 14 GB", | 20260506 23:02:22.530 | INFO | "pinned" : " 29 GB" | 20260506 23:02:22.530 | INFO | } | 20260506 23:02:22.530 | INFO | Perf report for the last 5.14469 seconds: | 20260506 23:02:22.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:02:22.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:22.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:22.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:22.530 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 23:02:22.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:22.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:22.530 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:02:22.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:22.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:22.530 | INFO | Uploaded 36GiB to GPU in 2.00229s (17.9794GiB/s) | 20260506 23:02:22.530 | INFO | Downloaded 34GiB to GPU in 1.92974s (17.6189GiB/s) | 20260506 23:02:27.669 | INFO | Analyzing chunk frames = [5145600,5146112) | 20260506 23:02:27.669 | INFO | MemInfo: { | 20260506 23:02:27.669 | INFO | "gpu" : " 17 GB", | 20260506 23:02:27.669 | INFO | "huge" : "", | 20260506 23:02:27.669 | INFO | "huge_shared" : " 27 GB", | 20260506 23:02:27.669 | INFO | "malloc" : " 14 GB", | 20260506 23:02:27.669 | INFO | "pinned" : " 29 GB" | 20260506 23:02:27.669 | INFO | } | 20260506 23:02:27.669 | INFO | Perf report for the last 5.13968 seconds: | 20260506 23:02:27.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:02:27.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:27.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:27.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:27.669 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 23:02:27.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:27.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:27.669 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:02:27.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:27.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:27.670 | INFO | Uploaded 36GiB to GPU in 1.98489s (18.1371GiB/s) | 20260506 23:02:27.670 | INFO | Downloaded 34GiB to GPU in 1.93249s (17.5939GiB/s) | 20260506 23:02:32.811 | INFO | Analyzing chunk frames = [5146112,5146624) | 20260506 23:02:32.811 | INFO | MemInfo: { | 20260506 23:02:32.811 | INFO | "gpu" : " 17 GB", | 20260506 23:02:32.811 | INFO | "huge" : "", | 20260506 23:02:32.812 | INFO | "huge_shared" : " 27 GB", | 20260506 23:02:32.812 | INFO | "malloc" : " 14 GB", | 20260506 23:02:32.812 | INFO | "pinned" : " 29 GB" | 20260506 23:02:32.812 | INFO | } | 20260506 23:02:32.812 | INFO | Perf report for the last 5.1424 seconds: | 20260506 23:02:32.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:02:32.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:32.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:32.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:32.812 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 23:02:32.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:32.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:32.812 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:02:32.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:32.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:32.812 | INFO | Uploaded 36GiB to GPU in 1.96973s (18.2766GiB/s) | 20260506 23:02:32.812 | INFO | Downloaded 34GiB to GPU in 1.88049s (18.0804GiB/s) | 20260506 23:02:37.945 | INFO | Analyzing chunk frames = [5146624,5147136) | 20260506 23:02:37.945 | INFO | MemInfo: { | 20260506 23:02:37.945 | INFO | "gpu" : " 17 GB", | 20260506 23:02:37.945 | INFO | "huge" : "", | 20260506 23:02:37.945 | INFO | "huge_shared" : " 27 GB", | 20260506 23:02:37.945 | INFO | "malloc" : " 14 GB", | 20260506 23:02:37.945 | INFO | "pinned" : " 29 GB" | 20260506 23:02:37.946 | INFO | } | 20260506 23:02:37.946 | INFO | Perf report for the last 5.13385 seconds: | 20260506 23:02:37.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:02:37.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:37.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:37.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:37.946 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 23:02:37.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:37.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:37.946 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:02:37.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:37.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:37.946 | INFO | Uploaded 36GiB to GPU in 1.99143s (18.0775GiB/s) | 20260506 23:02:37.946 | INFO | Downloaded 34GiB to GPU in 1.9257s (17.6559GiB/s) | 20260506 23:02:43.084 | INFO | Analyzing chunk frames = [5147136,5147648) | 20260506 23:02:43.084 | INFO | MemInfo: { | 20260506 23:02:43.084 | INFO | "gpu" : " 17 GB", | 20260506 23:02:43.084 | INFO | "huge" : "", | 20260506 23:02:43.084 | INFO | "huge_shared" : " 27 GB", | 20260506 23:02:43.084 | INFO | "malloc" : " 14 GB", | 20260506 23:02:43.084 | INFO | "pinned" : " 29 GB" | 20260506 23:02:43.084 | INFO | } | 20260506 23:02:43.084 | INFO | Perf report for the last 5.13836 seconds: | 20260506 23:02:43.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:02:43.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:43.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:43.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:43.084 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 23:02:43.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:43.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:43.084 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 23:02:43.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:43.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:43.084 | INFO | Uploaded 36GiB to GPU in 2.00811s (17.9273GiB/s) | 20260506 23:02:43.084 | INFO | Downloaded 34GiB to GPU in 1.94038s (17.5223GiB/s) | 20260506 23:02:48.222 | INFO | Analyzing chunk frames = [5147648,5148160) | 20260506 23:02:48.222 | INFO | MemInfo: { | 20260506 23:02:48.222 | INFO | "gpu" : " 17 GB", | 20260506 23:02:48.222 | INFO | "huge" : "", | 20260506 23:02:48.222 | INFO | "huge_shared" : " 27 GB", | 20260506 23:02:48.222 | INFO | "malloc" : " 14 GB", | 20260506 23:02:48.222 | INFO | "pinned" : " 29 GB" | 20260506 23:02:48.222 | INFO | } | 20260506 23:02:48.222 | INFO | Perf report for the last 5.13775 seconds: | 20260506 23:02:48.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:02:48.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:48.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:48.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:48.222 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:02:48.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:48.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:48.222 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 23:02:48.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:48.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:48.222 | INFO | Uploaded 36GiB to GPU in 1.9667s (18.3047GiB/s) | 20260506 23:02:48.222 | INFO | Downloaded 34GiB to GPU in 1.99665s (17.0285GiB/s) | 20260506 23:02:53.357 | INFO | Analyzing chunk frames = [5148160,5148672) | 20260506 23:02:53.357 | INFO | MemInfo: { | 20260506 23:02:53.357 | INFO | "gpu" : " 17 GB", | 20260506 23:02:53.357 | INFO | "huge" : "", | 20260506 23:02:53.357 | INFO | "huge_shared" : " 27 GB", | 20260506 23:02:53.357 | INFO | "malloc" : " 14 GB", | 20260506 23:02:53.357 | INFO | "pinned" : " 29 GB" | 20260506 23:02:53.357 | INFO | } | 20260506 23:02:53.357 | INFO | Perf report for the last 5.13538 seconds: | 20260506 23:02:53.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:02:53.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:53.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:53.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:53.357 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:02:53.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:53.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:53.357 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 23:02:53.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:53.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:53.357 | INFO | Uploaded 36GiB to GPU in 1.9935s (18.0586GiB/s) | 20260506 23:02:53.357 | INFO | Downloaded 34GiB to GPU in 1.98395s (17.1375GiB/s) | 20260506 23:02:53.455 | INFO | Starting a round of Baz IO | 20260506 23:02:56.582 | INFO | Populating RTMetrics report | 20260506 23:02:56.583 | INFO | RTMetrics report populated | 20260506 23:02:56.583 | INFO | Opening tmp RTMetrics json file | 20260506 23:02:56.583 | INFO | Serializing RTMetrics to json | 20260506 23:02:56.584 | INFO | Serialization done, swapping tmp file now... | 20260506 23:02:56.585 | INFO | RTMetrics done | 20260506 23:02:56.681 | INFO | Finished a round of Baz IO: Wrote 1.893 GiB in 3.225 seconds | 20260506 23:02:56.681 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.55% / 0.00% / 66.45% | 20260506 23:02:56.681 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.451024% / 0% / 7.15256e-05% | 20260506 23:02:56.681 | INFO | Baz IO overall: | 20260506 23:02:56.681 | INFO | BytesWritten : 2699202886309 | 20260506 23:02:56.681 | INFO | NumEvents : 1301772776214 | 20260506 23:02:58.506 | INFO | Analyzing chunk frames = [5148672,5149184) | 20260506 23:02:58.506 | INFO | MemInfo: { | 20260506 23:02:58.506 | INFO | "gpu" : " 17 GB", | 20260506 23:02:58.506 | INFO | "huge" : "", | 20260506 23:02:58.506 | INFO | "huge_shared" : " 27 GB", | 20260506 23:02:58.506 | INFO | "malloc" : " 14 GB", | 20260506 23:02:58.506 | INFO | "pinned" : " 29 GB" | 20260506 23:02:58.506 | INFO | } | 20260506 23:02:58.506 | INFO | Perf report for the last 5.14905 seconds: | 20260506 23:02:58.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:02:58.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:58.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:58.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:58.506 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 23:02:58.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:02:58.506 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260506 23:02:58.506 | INFO | PRE_HQ : 59.6%, 1.8, 56.7ms, 0 | 20260506 23:02:58.506 | INFO | BAZWRITER : 62.1%, 3.5, 111.8ms, 0 | 20260506 23:02:58.506 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:02:58.506 | INFO | Uploaded 36GiB to GPU in 2.04535s (17.6009GiB/s) | 20260506 23:02:58.506 | INFO | Downloaded 38GiB to GPU in 2.17285s (17.4886GiB/s) | 20260506 23:03:03.645 | INFO | Analyzing chunk frames = [5149184,5149696) | 20260506 23:03:03.645 | INFO | MemInfo: { | 20260506 23:03:03.645 | INFO | "gpu" : " 17 GB", | 20260506 23:03:03.645 | INFO | "huge" : "", | 20260506 23:03:03.645 | INFO | "huge_shared" : " 27 GB", | 20260506 23:03:03.645 | INFO | "malloc" : " 14 GB", | 20260506 23:03:03.645 | INFO | "pinned" : " 29 GB" | 20260506 23:03:03.645 | INFO | } | 20260506 23:03:03.645 | INFO | Perf report for the last 5.1391 seconds: | 20260506 23:03:03.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:03:03.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:03.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:03.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:03.645 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:03:03.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:03.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:03.645 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:03:03.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:03.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:03.646 | INFO | Uploaded 36GiB to GPU in 2.01237s (17.8893GiB/s) | 20260506 23:03:03.646 | INFO | Downloaded 34GiB to GPU in 1.92957s (17.6205GiB/s) | 20260506 23:03:08.786 | INFO | Analyzing chunk frames = [5149696,5150208) | 20260506 23:03:08.786 | INFO | MemInfo: { | 20260506 23:03:08.786 | INFO | "gpu" : " 17 GB", | 20260506 23:03:08.786 | INFO | "huge" : "", | 20260506 23:03:08.786 | INFO | "huge_shared" : " 27 GB", | 20260506 23:03:08.786 | INFO | "malloc" : " 14 GB", | 20260506 23:03:08.786 | INFO | "pinned" : " 29 GB" | 20260506 23:03:08.786 | INFO | } | 20260506 23:03:08.786 | INFO | Perf report for the last 5.14082 seconds: | 20260506 23:03:08.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:03:08.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:08.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:08.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:08.786 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:03:08.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:08.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:08.787 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:03:08.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:08.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:08.787 | INFO | Uploaded 36GiB to GPU in 1.99529s (18.0425GiB/s) | 20260506 23:03:08.787 | INFO | Downloaded 34GiB to GPU in 1.93006s (17.616GiB/s) | 20260506 23:03:13.926 | INFO | Analyzing chunk frames = [5150208,5150720) | 20260506 23:03:13.926 | INFO | MemInfo: { | 20260506 23:03:13.926 | INFO | "gpu" : " 17 GB", | 20260506 23:03:13.926 | INFO | "huge" : "", | 20260506 23:03:13.926 | INFO | "huge_shared" : " 27 GB", | 20260506 23:03:13.926 | INFO | "malloc" : " 14 GB", | 20260506 23:03:13.926 | INFO | "pinned" : " 29 GB" | 20260506 23:03:13.926 | INFO | } | 20260506 23:03:13.926 | INFO | Perf report for the last 5.14009 seconds: | 20260506 23:03:13.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:03:13.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:13.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:13.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:13.926 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 23:03:13.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:13.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:13.926 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:03:13.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:13.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:13.926 | INFO | Uploaded 36GiB to GPU in 2.01796s (17.8398GiB/s) | 20260506 23:03:13.926 | INFO | Downloaded 34GiB to GPU in 1.90721s (17.8271GiB/s) | 20260506 23:03:19.060 | INFO | Analyzing chunk frames = [5150720,5151232) | 20260506 23:03:19.060 | INFO | MemInfo: { | 20260506 23:03:19.060 | INFO | "gpu" : " 17 GB", | 20260506 23:03:19.060 | INFO | "huge" : "", | 20260506 23:03:19.060 | INFO | "huge_shared" : " 27 GB", | 20260506 23:03:19.060 | INFO | "malloc" : " 14 GB", | 20260506 23:03:19.060 | INFO | "pinned" : " 29 GB" | 20260506 23:03:19.060 | INFO | } | 20260506 23:03:19.060 | INFO | Perf report for the last 5.13336 seconds: | 20260506 23:03:19.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:03:19.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:19.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:19.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:19.060 | INFO | ANALYSIS : 60.3%, 3.0, 93.7ms, 0 | 20260506 23:03:19.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:19.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:19.060 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:03:19.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:19.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:19.060 | INFO | Uploaded 36GiB to GPU in 1.99822s (18.016GiB/s) | 20260506 23:03:19.060 | INFO | Downloaded 34GiB to GPU in 1.92404s (17.6712GiB/s) | 20260506 23:03:24.201 | INFO | Analyzing chunk frames = [5151232,5151744) | 20260506 23:03:24.201 | INFO | MemInfo: { | 20260506 23:03:24.201 | INFO | "gpu" : " 17 GB", | 20260506 23:03:24.201 | INFO | "huge" : "", | 20260506 23:03:24.201 | INFO | "huge_shared" : " 27 GB", | 20260506 23:03:24.201 | INFO | "malloc" : " 14 GB", | 20260506 23:03:24.201 | INFO | "pinned" : " 29 GB" | 20260506 23:03:24.201 | INFO | } | 20260506 23:03:24.201 | INFO | Perf report for the last 5.14156 seconds: | 20260506 23:03:24.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:03:24.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:24.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:24.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:24.201 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 23:03:24.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:24.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:24.201 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 23:03:24.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:24.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:24.201 | INFO | Uploaded 36GiB to GPU in 2.01694s (17.8488GiB/s) | 20260506 23:03:24.202 | INFO | Downloaded 34GiB to GPU in 1.93386s (17.5815GiB/s) | 20260506 23:03:29.340 | INFO | Analyzing chunk frames = [5151744,5152256) | 20260506 23:03:29.340 | INFO | MemInfo: { | 20260506 23:03:29.340 | INFO | "gpu" : " 17 GB", | 20260506 23:03:29.340 | INFO | "huge" : "", | 20260506 23:03:29.340 | INFO | "huge_shared" : " 27 GB", | 20260506 23:03:29.340 | INFO | "malloc" : " 14 GB", | 20260506 23:03:29.340 | INFO | "pinned" : " 29 GB" | 20260506 23:03:29.340 | INFO | } | 20260506 23:03:29.340 | INFO | Perf report for the last 5.13849 seconds: | 20260506 23:03:29.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:03:29.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:29.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:29.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:29.340 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 23:03:29.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:29.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:29.340 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 23:03:29.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:29.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:29.340 | INFO | Uploaded 36GiB to GPU in 1.98227s (18.161GiB/s) | 20260506 23:03:29.340 | INFO | Downloaded 34GiB to GPU in 1.99837s (17.0138GiB/s) | 20260506 23:03:34.478 | INFO | Analyzing chunk frames = [5152256,5152768) | 20260506 23:03:34.478 | INFO | MemInfo: { | 20260506 23:03:34.478 | INFO | "gpu" : " 17 GB", | 20260506 23:03:34.478 | INFO | "huge" : "", | 20260506 23:03:34.478 | INFO | "huge_shared" : " 27 GB", | 20260506 23:03:34.478 | INFO | "malloc" : " 14 GB", | 20260506 23:03:34.478 | INFO | "pinned" : " 29 GB" | 20260506 23:03:34.478 | INFO | } | 20260506 23:03:34.479 | INFO | Perf report for the last 5.13894 seconds: | 20260506 23:03:34.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:03:34.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:34.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:34.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:34.479 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:03:34.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:34.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:34.479 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:03:34.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:34.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:34.479 | INFO | Uploaded 36GiB to GPU in 1.97266s (18.2495GiB/s) | 20260506 23:03:34.479 | INFO | Downloaded 34GiB to GPU in 1.99226s (17.0661GiB/s) | 20260506 23:03:34.479 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:03:34.479 | INFO | Total mem usage: 18 GiB | 20260506 23:03:34.479 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:03:34.479 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 23:03:34.479 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:03:34.479 | INFO | High Water Breakdown: | 20260506 23:03:34.479 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:03:34.479 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:03:34.479 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:03:34.479 | INFO | ------------------------------------------ | 20260506 23:03:34.479 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:03:34.479 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:03:34.479 | INFO | Total mem usage: 26.5 GiB | 20260506 23:03:34.479 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:03:34.479 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:03:34.479 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:03:34.479 | INFO | High Water Breakdown: | 20260506 23:03:34.479 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:03:34.479 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:03:34.479 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:03:34.479 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:03:34.480 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:03:34.480 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:03:34.480 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:03:34.480 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:03:34.480 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:03:34.480 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:03:34.480 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:03:34.480 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:03:34.480 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:03:34.480 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:03:34.480 | INFO | ------------------------------------------ | 20260506 23:03:34.480 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:03:34.480 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:03:34.480 | INFO | Total mem usage: 25.6 GiB | 20260506 23:03:34.480 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:03:34.480 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:03:34.480 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:03:34.480 | INFO | High Water Breakdown: | 20260506 23:03:34.480 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:03:34.480 | INFO | ------------------------------------------ | 20260506 23:03:34.480 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:03:34.480 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:03:34.480 | INFO | Total mem usage: 15.7 GiB | 20260506 23:03:34.480 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:03:34.480 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:03:34.480 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:03:34.480 | INFO | High Water Breakdown: | 20260506 23:03:34.480 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:03:34.480 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:03:34.480 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:03:34.480 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:03:34.480 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:03:34.480 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:03:34.480 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:03:34.480 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:03:34.480 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:03:34.480 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:03:34.480 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:03:34.480 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:03:34.480 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:03:34.480 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:03:34.480 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:03:34.480 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:03:34.480 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:03:34.480 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:03:34.480 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:03:34.480 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:03:34.480 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:03:34.480 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:03:34.480 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:03:34.480 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:03:34.480 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:03:34.480 | INFO | ------------------------------------------ | 20260506 23:03:34.480 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:03:34.480 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:03:34.480 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:03:34.480 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:03:34.480 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.61e+01, 1.33e+01 / 2.19e+00 | 20260506 23:03:34.480 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.88e-03 / 5.50e-03 | 20260506 23:03:34.480 | INFO | PerfReport: 11.65%, Baseline -- 3.61e+00 / 4.42e+00, 5.43e-01 / 3.21e-01 | 20260506 23:03:34.480 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.47e+00, 1.79e-01 / 1.25e-01 | 20260506 23:03:34.480 | INFO | PerfReport: 16.08%, DME -- 3.99e+01 / 4.49e+01, 4.20e+00 / 2.39e+00 | 20260506 23:03:34.480 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.35e+00 / 1.38e+00 | 20260506 23:03:34.480 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.19e+00 / 2.78e+00, 3.80e-01 / 2.07e-01 | 20260506 23:03:34.480 | INFO | PerfReport: 3.21%, Metrics -- 9.97e-01 / 1.33e+00, 2.64e-01 / 1.64e-01 | 20260506 23:03:34.480 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.39e-01 / 3.89e-01, 5.28e-02 / 2.85e-02 | 20260506 23:03:34.480 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:03:34.480 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:03:34.480 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:03:34.480 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 23:03:34.480 | INFO | PerfReport: 21.54%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 23:03:34.480 | INFO | PerfReport: 21.26%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 23:03:34.480 | INFO | PerfReport: 32.09%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 23:03:34.480 | INFO | PerfReport: 1.29%, LockUpload -- 1.25e+00 / 0.00e+00, 4.74e+00 / 0.00e+00 | 20260506 23:03:34.480 | INFO | PerfReport: 1.27%, LockDownload -- 1.22e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260506 23:03:34.480 | INFO | PerfReport: 22.55%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:03:34.480 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260506 23:03:34.577 | INFO | Starting a round of Baz IO | 20260506 23:03:37.709 | INFO | Populating RTMetrics report | 20260506 23:03:37.709 | INFO | RTMetrics report populated | 20260506 23:03:37.709 | INFO | Opening tmp RTMetrics json file | 20260506 23:03:37.714 | INFO | Serializing RTMetrics to json | 20260506 23:03:37.729 | INFO | Serialization done, swapping tmp file now... | 20260506 23:03:37.730 | INFO | RTMetrics done | 20260506 23:03:37.804 | INFO | Finished a round of Baz IO: Wrote 1.893 GiB in 3.226 seconds | 20260506 23:03:37.804 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.55% / 0.00% / 66.45% | 20260506 23:03:37.804 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.450357% / 7.94729e-06% / 5.96046e-05% | 20260506 23:03:37.804 | INFO | Baz IO overall: | 20260506 23:03:37.804 | INFO | BytesWritten : 2701235650987 | 20260506 23:03:37.804 | INFO | NumEvents : 1302742374194 | 20260506 23:03:39.620 | INFO | Analyzing chunk frames = [5152768,5153280) | 20260506 23:03:39.620 | INFO | MemInfo: { | 20260506 23:03:39.620 | INFO | "gpu" : " 17 GB", | 20260506 23:03:39.620 | INFO | "huge" : "", | 20260506 23:03:39.620 | INFO | "huge_shared" : " 27 GB", | 20260506 23:03:39.620 | INFO | "malloc" : " 14 GB", | 20260506 23:03:39.620 | INFO | "pinned" : " 29 GB" | 20260506 23:03:39.620 | INFO | } | 20260506 23:03:39.620 | INFO | Perf report for the last 5.1411 seconds: | 20260506 23:03:39.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:03:39.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:39.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:39.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:39.620 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 23:03:39.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:39.620 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 23:03:39.620 | INFO | PRE_HQ : 59.7%, 1.8, 56.6ms, 0 | 20260506 23:03:39.620 | INFO | BAZWRITER : 62.3%, 3.4, 109.7ms, 0 | 20260506 23:03:39.620 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:03:39.620 | INFO | Uploaded 36GiB to GPU in 2.02399s (17.7867GiB/s) | 20260506 23:03:39.620 | INFO | Downloaded 38GiB to GPU in 2.18593s (17.3839GiB/s) | 20260506 23:03:44.764 | INFO | Analyzing chunk frames = [5153280,5153792) | 20260506 23:03:44.764 | INFO | MemInfo: { | 20260506 23:03:44.764 | INFO | "gpu" : " 17 GB", | 20260506 23:03:44.764 | INFO | "huge" : "", | 20260506 23:03:44.764 | INFO | "huge_shared" : " 27 GB", | 20260506 23:03:44.764 | INFO | "malloc" : " 14 GB", | 20260506 23:03:44.764 | INFO | "pinned" : " 29 GB" | 20260506 23:03:44.764 | INFO | } | 20260506 23:03:44.764 | INFO | Perf report for the last 5.14408 seconds: | 20260506 23:03:44.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:03:44.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:44.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:44.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:44.764 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 23:03:44.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:44.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:44.764 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:03:44.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:44.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:44.764 | INFO | Uploaded 36GiB to GPU in 1.9941s (18.0533GiB/s) | 20260506 23:03:44.764 | INFO | Downloaded 34GiB to GPU in 1.92868s (17.6287GiB/s) | 20260506 23:03:49.902 | INFO | Analyzing chunk frames = [5153792,5154304) | 20260506 23:03:49.902 | INFO | MemInfo: { | 20260506 23:03:49.902 | INFO | "gpu" : " 17 GB", | 20260506 23:03:49.902 | INFO | "huge" : "", | 20260506 23:03:49.902 | INFO | "huge_shared" : " 27 GB", | 20260506 23:03:49.902 | INFO | "malloc" : " 14 GB", | 20260506 23:03:49.902 | INFO | "pinned" : " 29 GB" | 20260506 23:03:49.902 | INFO | } | 20260506 23:03:49.902 | INFO | Perf report for the last 5.13802 seconds: | 20260506 23:03:49.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:03:49.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:49.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:49.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:49.902 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 23:03:49.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:49.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:49.902 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:03:49.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:49.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:49.902 | INFO | Uploaded 36GiB to GPU in 1.96645s (18.3071GiB/s) | 20260506 23:03:49.902 | INFO | Downloaded 34GiB to GPU in 1.92915s (17.6243GiB/s) | 20260506 23:03:55.034 | INFO | Analyzing chunk frames = [5154304,5154816) | 20260506 23:03:55.034 | INFO | MemInfo: { | 20260506 23:03:55.034 | INFO | "gpu" : " 17 GB", | 20260506 23:03:55.034 | INFO | "huge" : "", | 20260506 23:03:55.034 | INFO | "huge_shared" : " 27 GB", | 20260506 23:03:55.034 | INFO | "malloc" : " 14 GB", | 20260506 23:03:55.034 | INFO | "pinned" : " 29 GB" | 20260506 23:03:55.034 | INFO | } | 20260506 23:03:55.034 | INFO | Perf report for the last 5.13201 seconds: | 20260506 23:03:55.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:03:55.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:55.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:55.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:55.034 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 23:03:55.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:55.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:55.034 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:03:55.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:55.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:03:55.034 | INFO | Uploaded 36GiB to GPU in 2.00111s (17.99GiB/s) | 20260506 23:03:55.034 | INFO | Downloaded 34GiB to GPU in 1.90966s (17.8043GiB/s) | 20260506 23:04:00.170 | INFO | Analyzing chunk frames = [5154816,5155328) | 20260506 23:04:00.170 | INFO | MemInfo: { | 20260506 23:04:00.170 | INFO | "gpu" : " 17 GB", | 20260506 23:04:00.170 | INFO | "huge" : "", | 20260506 23:04:00.170 | INFO | "huge_shared" : " 27 GB", | 20260506 23:04:00.170 | INFO | "malloc" : " 14 GB", | 20260506 23:04:00.170 | INFO | "pinned" : " 29 GB" | 20260506 23:04:00.170 | INFO | } | 20260506 23:04:00.171 | INFO | Perf report for the last 5.13657 seconds: | 20260506 23:04:00.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:04:00.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:00.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:00.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:00.171 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 23:04:00.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:00.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:00.171 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:04:00.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:00.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:00.171 | INFO | Uploaded 36GiB to GPU in 1.97829s (18.1975GiB/s) | 20260506 23:04:00.171 | INFO | Downloaded 34GiB to GPU in 1.92627s (17.6507GiB/s) | 20260506 23:04:05.311 | INFO | Analyzing chunk frames = [5155328,5155840) | 20260506 23:04:05.311 | INFO | MemInfo: { | 20260506 23:04:05.311 | INFO | "gpu" : " 17 GB", | 20260506 23:04:05.311 | INFO | "huge" : "", | 20260506 23:04:05.311 | INFO | "huge_shared" : " 27 GB", | 20260506 23:04:05.312 | INFO | "malloc" : " 14 GB", | 20260506 23:04:05.312 | INFO | "pinned" : " 29 GB" | 20260506 23:04:05.312 | INFO | } | 20260506 23:04:05.312 | INFO | Perf report for the last 5.14119 seconds: | 20260506 23:04:05.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:04:05.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:05.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:05.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:05.312 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 23:04:05.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:05.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:05.312 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 23:04:05.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:05.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:05.312 | INFO | Uploaded 36GiB to GPU in 1.99914s (18.0077GiB/s) | 20260506 23:04:05.312 | INFO | Downloaded 34GiB to GPU in 1.93665s (17.556GiB/s) | 20260506 23:04:10.451 | INFO | Analyzing chunk frames = [5155840,5156352) | 20260506 23:04:10.451 | INFO | MemInfo: { | 20260506 23:04:10.451 | INFO | "gpu" : " 17 GB", | 20260506 23:04:10.451 | INFO | "huge" : "", | 20260506 23:04:10.451 | INFO | "huge_shared" : " 27 GB", | 20260506 23:04:10.451 | INFO | "malloc" : " 14 GB", | 20260506 23:04:10.451 | INFO | "pinned" : " 29 GB" | 20260506 23:04:10.451 | INFO | } | 20260506 23:04:10.451 | INFO | Perf report for the last 5.13965 seconds: | 20260506 23:04:10.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:04:10.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:10.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:10.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:10.452 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 23:04:10.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:10.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:10.452 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 23:04:10.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:10.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:10.452 | INFO | Uploaded 36GiB to GPU in 1.96754s (18.2969GiB/s) | 20260506 23:04:10.452 | INFO | Downloaded 34GiB to GPU in 1.97768s (17.1919GiB/s) | 20260506 23:04:15.600 | INFO | Analyzing chunk frames = [5156352,5156864) | 20260506 23:04:15.600 | INFO | MemInfo: { | 20260506 23:04:15.600 | INFO | "gpu" : " 17 GB", | 20260506 23:04:15.600 | INFO | "huge" : "", | 20260506 23:04:15.600 | INFO | "huge_shared" : " 27 GB", | 20260506 23:04:15.600 | INFO | "malloc" : " 14 GB", | 20260506 23:04:15.600 | INFO | "pinned" : " 29 GB" | 20260506 23:04:15.600 | INFO | } | 20260506 23:04:15.600 | INFO | Perf report for the last 5.14843 seconds: | 20260506 23:04:15.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:04:15.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:15.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:15.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:15.600 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 23:04:15.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:15.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:15.600 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 23:04:15.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:15.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:15.600 | INFO | Uploaded 36GiB to GPU in 1.99171s (18.0749GiB/s) | 20260506 23:04:15.600 | INFO | Downloaded 34GiB to GPU in 1.98625s (17.1177GiB/s) | 20260506 23:04:15.698 | INFO | Starting a round of Baz IO | 20260506 23:04:18.826 | INFO | Populating RTMetrics report | 20260506 23:04:18.826 | INFO | RTMetrics report populated | 20260506 23:04:18.826 | INFO | Opening tmp RTMetrics json file | 20260506 23:04:18.828 | INFO | Serializing RTMetrics to json | 20260506 23:04:18.847 | INFO | Serialization done, swapping tmp file now... | 20260506 23:04:18.847 | INFO | RTMetrics done | 20260506 23:04:18.920 | INFO | Finished a round of Baz IO: Wrote 1.892 GiB in 3.222 seconds | 20260506 23:04:18.920 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.54% / 0.00% / 66.45% | 20260506 23:04:18.920 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.448191% / 3.97364e-06% / 6.75519e-05% | 20260506 23:04:18.920 | INFO | Baz IO overall: | 20260506 23:04:18.920 | INFO | BytesWritten : 2703267535518 | 20260506 23:04:18.920 | INFO | NumEvents : 1303711148132 | 20260506 23:04:20.739 | INFO | Analyzing chunk frames = [5156864,5157376) | 20260506 23:04:20.739 | INFO | MemInfo: { | 20260506 23:04:20.739 | INFO | "gpu" : " 17 GB", | 20260506 23:04:20.739 | INFO | "huge" : "", | 20260506 23:04:20.739 | INFO | "huge_shared" : " 27 GB", | 20260506 23:04:20.739 | INFO | "malloc" : " 14 GB", | 20260506 23:04:20.739 | INFO | "pinned" : " 29 GB" | 20260506 23:04:20.739 | INFO | } | 20260506 23:04:20.739 | INFO | Perf report for the last 5.13925 seconds: | 20260506 23:04:20.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:04:20.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:20.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:20.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:20.739 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 23:04:20.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:20.739 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 23:04:20.739 | INFO | PRE_HQ : 59.8%, 1.8, 56.7ms, 0 | 20260506 23:04:20.739 | INFO | BAZWRITER : 62.3%, 3.5, 111.5ms, 0 | 20260506 23:04:20.739 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:04:20.739 | INFO | Uploaded 36GiB to GPU in 2.02873s (17.7451GiB/s) | 20260506 23:04:20.739 | INFO | Downloaded 38GiB to GPU in 2.17175s (17.4974GiB/s) | 20260506 23:04:25.876 | INFO | Analyzing chunk frames = [5157376,5157888) | 20260506 23:04:25.876 | INFO | MemInfo: { | 20260506 23:04:25.876 | INFO | "gpu" : " 17 GB", | 20260506 23:04:25.876 | INFO | "huge" : "", | 20260506 23:04:25.876 | INFO | "huge_shared" : " 27 GB", | 20260506 23:04:25.876 | INFO | "malloc" : " 14 GB", | 20260506 23:04:25.876 | INFO | "pinned" : " 29 GB" | 20260506 23:04:25.876 | INFO | } | 20260506 23:04:25.876 | INFO | Perf report for the last 5.13729 seconds: | 20260506 23:04:25.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:04:25.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:25.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:25.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:25.876 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:04:25.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:25.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:25.876 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:04:25.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:25.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:25.877 | INFO | Uploaded 36GiB to GPU in 2.015s (17.866GiB/s) | 20260506 23:04:25.877 | INFO | Downloaded 34GiB to GPU in 1.92951s (17.621GiB/s) | 20260506 23:04:31.013 | INFO | Analyzing chunk frames = [5157888,5158400) | 20260506 23:04:31.013 | INFO | MemInfo: { | 20260506 23:04:31.013 | INFO | "gpu" : " 17 GB", | 20260506 23:04:31.013 | INFO | "huge" : "", | 20260506 23:04:31.013 | INFO | "huge_shared" : " 27 GB", | 20260506 23:04:31.013 | INFO | "malloc" : " 14 GB", | 20260506 23:04:31.013 | INFO | "pinned" : " 29 GB" | 20260506 23:04:31.013 | INFO | } | 20260506 23:04:31.013 | INFO | Perf report for the last 5.1366 seconds: | 20260506 23:04:31.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:04:31.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:31.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:31.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:31.013 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 23:04:31.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:31.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:31.013 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:04:31.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:31.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:31.013 | INFO | Uploaded 36GiB to GPU in 1.98333s (18.1513GiB/s) | 20260506 23:04:31.013 | INFO | Downloaded 34GiB to GPU in 1.92652s (17.6484GiB/s) | 20260506 23:04:36.149 | INFO | Analyzing chunk frames = [5158400,5158912) | 20260506 23:04:36.149 | INFO | MemInfo: { | 20260506 23:04:36.149 | INFO | "gpu" : " 17 GB", | 20260506 23:04:36.149 | INFO | "huge" : "", | 20260506 23:04:36.149 | INFO | "huge_shared" : " 27 GB", | 20260506 23:04:36.149 | INFO | "malloc" : " 14 GB", | 20260506 23:04:36.149 | INFO | "pinned" : " 29 GB" | 20260506 23:04:36.149 | INFO | } | 20260506 23:04:36.149 | INFO | Perf report for the last 5.13579 seconds: | 20260506 23:04:36.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:04:36.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:36.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:36.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:36.149 | INFO | ANALYSIS : 59.7%, 3.0, 92.4ms, 0 | 20260506 23:04:36.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:36.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:36.149 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:04:36.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:36.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:36.149 | INFO | Uploaded 36GiB to GPU in 1.97214s (18.2543GiB/s) | 20260506 23:04:36.149 | INFO | Downloaded 34GiB to GPU in 1.87478s (18.1355GiB/s) | 20260506 23:04:41.288 | INFO | Analyzing chunk frames = [5158912,5159424) | 20260506 23:04:41.288 | INFO | MemInfo: { | 20260506 23:04:41.288 | INFO | "gpu" : " 17 GB", | 20260506 23:04:41.288 | INFO | "huge" : "", | 20260506 23:04:41.288 | INFO | "huge_shared" : " 27 GB", | 20260506 23:04:41.288 | INFO | "malloc" : " 14 GB", | 20260506 23:04:41.288 | INFO | "pinned" : " 29 GB" | 20260506 23:04:41.288 | INFO | } | 20260506 23:04:41.288 | INFO | Perf report for the last 5.13907 seconds: | 20260506 23:04:41.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:04:41.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:41.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:41.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:41.288 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 23:04:41.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:41.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:41.288 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:04:41.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:41.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:41.288 | INFO | Uploaded 36GiB to GPU in 1.98712s (18.1167GiB/s) | 20260506 23:04:41.288 | INFO | Downloaded 34GiB to GPU in 1.92416s (17.67GiB/s) | 20260506 23:04:46.433 | INFO | Analyzing chunk frames = [5159424,5159936) | 20260506 23:04:46.433 | INFO | MemInfo: { | 20260506 23:04:46.433 | INFO | "gpu" : " 17 GB", | 20260506 23:04:46.433 | INFO | "huge" : "", | 20260506 23:04:46.433 | INFO | "huge_shared" : " 27 GB", | 20260506 23:04:46.433 | INFO | "malloc" : " 14 GB", | 20260506 23:04:46.433 | INFO | "pinned" : " 29 GB" | 20260506 23:04:46.433 | INFO | } | 20260506 23:04:46.433 | INFO | Perf report for the last 5.14556 seconds: | 20260506 23:04:46.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:04:46.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:46.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:46.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:46.434 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 23:04:46.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:46.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:46.434 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260506 23:04:46.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:46.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:46.434 | INFO | Uploaded 36GiB to GPU in 2.0334s (17.7043GiB/s) | 20260506 23:04:46.434 | INFO | Downloaded 34GiB to GPU in 1.94206s (17.5072GiB/s) | 20260506 23:04:51.573 | INFO | Analyzing chunk frames = [5159936,5160448) | 20260506 23:04:51.573 | INFO | MemInfo: { | 20260506 23:04:51.573 | INFO | "gpu" : " 17 GB", | 20260506 23:04:51.573 | INFO | "huge" : "", | 20260506 23:04:51.573 | INFO | "huge_shared" : " 27 GB", | 20260506 23:04:51.573 | INFO | "malloc" : " 14 GB", | 20260506 23:04:51.573 | INFO | "pinned" : " 29 GB" | 20260506 23:04:51.573 | INFO | } | 20260506 23:04:51.574 | INFO | Perf report for the last 5.14005 seconds: | 20260506 23:04:51.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:04:51.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:51.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:51.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:51.574 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 23:04:51.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:51.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:51.574 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 23:04:51.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:51.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:51.574 | INFO | Uploaded 36GiB to GPU in 1.98551s (18.1314GiB/s) | 20260506 23:04:51.574 | INFO | Downloaded 34GiB to GPU in 1.99573s (17.0364GiB/s) | 20260506 23:04:56.709 | INFO | Analyzing chunk frames = [5160448,5160960) | 20260506 23:04:56.709 | INFO | MemInfo: { | 20260506 23:04:56.709 | INFO | "gpu" : " 17 GB", | 20260506 23:04:56.709 | INFO | "huge" : "", | 20260506 23:04:56.709 | INFO | "huge_shared" : " 27 GB", | 20260506 23:04:56.709 | INFO | "malloc" : " 14 GB", | 20260506 23:04:56.709 | INFO | "pinned" : " 29 GB" | 20260506 23:04:56.709 | INFO | } | 20260506 23:04:56.709 | INFO | Perf report for the last 5.13577 seconds: | 20260506 23:04:56.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:04:56.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:56.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:56.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:56.709 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:04:56.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:56.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:56.709 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 23:04:56.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:56.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:04:56.710 | INFO | Uploaded 36GiB to GPU in 1.99142s (18.0775GiB/s) | 20260506 23:04:56.710 | INFO | Downloaded 34GiB to GPU in 1.9984s (17.0136GiB/s) | 20260506 23:04:56.710 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:04:56.710 | INFO | Total mem usage: 18 GiB | 20260506 23:04:56.710 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:04:56.710 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 23:04:56.710 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:04:56.710 | INFO | High Water Breakdown: | 20260506 23:04:56.710 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:04:56.710 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:04:56.710 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:04:56.710 | INFO | ------------------------------------------ | 20260506 23:04:56.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:04:56.710 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:04:56.710 | INFO | Total mem usage: 26.5 GiB | 20260506 23:04:56.710 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:04:56.710 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:04:56.710 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:04:56.710 | INFO | High Water Breakdown: | 20260506 23:04:56.710 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:04:56.710 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:04:56.710 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:04:56.710 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:04:56.710 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:04:56.710 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:04:56.710 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:04:56.710 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:04:56.710 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:04:56.710 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:04:56.710 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:04:56.710 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:04:56.710 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:04:56.710 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:04:56.710 | INFO | ------------------------------------------ | 20260506 23:04:56.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:04:56.710 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:04:56.710 | INFO | Total mem usage: 25.6 GiB | 20260506 23:04:56.710 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:04:56.710 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:04:56.710 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:04:56.710 | INFO | High Water Breakdown: | 20260506 23:04:56.710 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:04:56.710 | INFO | ------------------------------------------ | 20260506 23:04:56.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:04:56.710 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:04:56.710 | INFO | Total mem usage: 15.7 GiB | 20260506 23:04:56.710 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:04:56.710 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:04:56.710 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:04:56.710 | INFO | High Water Breakdown: | 20260506 23:04:56.710 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:04:56.710 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:04:56.710 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:04:56.710 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:04:56.711 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:04:56.711 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:04:56.711 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:04:56.711 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:04:56.711 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:04:56.711 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:04:56.711 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:04:56.711 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:04:56.711 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:04:56.711 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:04:56.711 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:04:56.711 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:04:56.711 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:04:56.711 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:04:56.711 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:04:56.711 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:04:56.711 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:04:56.711 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:04:56.711 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:04:56.711 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:04:56.711 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:04:56.711 | INFO | ------------------------------------------ | 20260506 23:04:56.711 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:04:56.711 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:04:56.711 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:04:56.711 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:04:56.711 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.59e+01, 1.35e+01 / 2.20e+00 | 20260506 23:04:56.711 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 9.62e-03 / 4.93e-03 | 20260506 23:04:56.711 | INFO | PerfReport: 11.68%, Baseline -- 3.62e+00 / 4.39e+00, 5.47e-01 / 3.17e-01 | 20260506 23:04:56.711 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.45e+00, 1.78e-01 / 1.21e-01 | 20260506 23:04:56.711 | INFO | PerfReport: 16.07%, DME -- 3.99e+01 / 4.48e+01, 4.15e+00 / 2.56e+00 | 20260506 23:04:56.711 | INFO | PerfReport: 57.32%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.36e+00 / 1.53e+00 | 20260506 23:04:56.711 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.19e+00 / 2.75e+00, 3.77e-01 / 2.06e-01 | 20260506 23:04:56.711 | INFO | PerfReport: 3.21%, Metrics -- 9.97e-01 / 1.33e+00, 2.64e-01 / 1.44e-01 | 20260506 23:04:56.711 | INFO | PerfReport: 1.10%, PulseSerializing -- 3.40e-01 / 3.93e-01, 5.38e-02 / 2.58e-02 | 20260506 23:04:56.711 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:04:56.711 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:04:56.711 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:04:56.711 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 23:04:56.711 | INFO | PerfReport: 21.50%, Upload -- 2.08e+01 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260506 23:04:56.711 | INFO | PerfReport: 21.25%, Download -- 2.05e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 23:04:56.711 | INFO | PerfReport: 32.10%, Compute -- 3.10e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 23:04:56.711 | INFO | PerfReport: 1.25%, LockUpload -- 1.21e+00 / 0.00e+00, 4.67e+00 / 0.00e+00 | 20260506 23:04:56.711 | INFO | PerfReport: 1.27%, LockDownload -- 1.23e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260506 23:04:56.711 | INFO | PerfReport: 22.60%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:04:56.711 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260506 23:04:56.807 | INFO | Starting a round of Baz IO | 20260506 23:04:59.944 | INFO | Populating RTMetrics report | 20260506 23:04:59.944 | INFO | RTMetrics report populated | 20260506 23:04:59.944 | INFO | Opening tmp RTMetrics json file | 20260506 23:04:59.964 | INFO | Serializing RTMetrics to json | 20260506 23:04:59.967 | INFO | Serialization done, swapping tmp file now... | 20260506 23:04:59.967 | INFO | RTMetrics done | 20260506 23:05:00.037 | INFO | Finished a round of Baz IO: Wrote 1.892 GiB in 3.229 seconds | 20260506 23:05:00.037 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.54% / 0.00% / 66.46% | 20260506 23:05:00.037 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.447567% / 7.94729e-06% / 5.5631e-05% | 20260506 23:05:00.037 | INFO | Baz IO overall: | 20260506 23:05:00.037 | INFO | BytesWritten : 2705299432702 | 20260506 23:05:00.037 | INFO | NumEvents : 1304680100511 | 20260506 23:05:01.846 | INFO | Analyzing chunk frames = [5160960,5161472) | 20260506 23:05:01.846 | INFO | MemInfo: { | 20260506 23:05:01.846 | INFO | "gpu" : " 17 GB", | 20260506 23:05:01.846 | INFO | "huge" : "", | 20260506 23:05:01.846 | INFO | "huge_shared" : " 27 GB", | 20260506 23:05:01.846 | INFO | "malloc" : " 14 GB", | 20260506 23:05:01.846 | INFO | "pinned" : " 29 GB" | 20260506 23:05:01.846 | INFO | } | 20260506 23:05:01.846 | INFO | Perf report for the last 5.13653 seconds: | 20260506 23:05:01.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:05:01.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:01.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:01.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:01.846 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 23:05:01.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:01.846 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260506 23:05:01.846 | INFO | PRE_HQ : 60.4%, 1.9, 59.8ms, 0 | 20260506 23:05:01.846 | INFO | BAZWRITER : 62.5%, 3.5, 113.5ms, 0 | 20260506 23:05:01.846 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:05:01.846 | INFO | Uploaded 36GiB to GPU in 2.02844s (17.7476GiB/s) | 20260506 23:05:01.846 | INFO | Downloaded 38GiB to GPU in 2.17119s (17.5019GiB/s) | 20260506 23:05:06.980 | INFO | Analyzing chunk frames = [5161472,5161984) | 20260506 23:05:06.980 | INFO | MemInfo: { | 20260506 23:05:06.980 | INFO | "gpu" : " 17 GB", | 20260506 23:05:06.980 | INFO | "huge" : "", | 20260506 23:05:06.980 | INFO | "huge_shared" : " 27 GB", | 20260506 23:05:06.980 | INFO | "malloc" : " 14 GB", | 20260506 23:05:06.980 | INFO | "pinned" : " 29 GB" | 20260506 23:05:06.980 | INFO | } | 20260506 23:05:06.980 | INFO | Perf report for the last 5.1346 seconds: | 20260506 23:05:06.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:05:06.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:06.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:06.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:06.981 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:05:06.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:06.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:06.981 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:05:06.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:06.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:06.981 | INFO | Uploaded 36GiB to GPU in 1.99342s (18.0595GiB/s) | 20260506 23:05:06.981 | INFO | Downloaded 34GiB to GPU in 1.92976s (17.6187GiB/s) | 20260506 23:05:12.120 | INFO | Analyzing chunk frames = [5161984,5162496) | 20260506 23:05:12.120 | INFO | MemInfo: { | 20260506 23:05:12.120 | INFO | "gpu" : " 17 GB", | 20260506 23:05:12.120 | INFO | "huge" : "", | 20260506 23:05:12.121 | INFO | "huge_shared" : " 27 GB", | 20260506 23:05:12.121 | INFO | "malloc" : " 14 GB", | 20260506 23:05:12.121 | INFO | "pinned" : " 29 GB" | 20260506 23:05:12.121 | INFO | } | 20260506 23:05:12.121 | INFO | Perf report for the last 5.14018 seconds: | 20260506 23:05:12.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:05:12.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:12.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:12.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:12.121 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 23:05:12.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:12.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:12.121 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:05:12.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:12.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:12.121 | INFO | Uploaded 36GiB to GPU in 1.96979s (18.276GiB/s) | 20260506 23:05:12.121 | INFO | Downloaded 34GiB to GPU in 1.92538s (17.6588GiB/s) | 20260506 23:05:17.264 | INFO | Analyzing chunk frames = [5162496,5163008) | 20260506 23:05:17.264 | INFO | MemInfo: { | 20260506 23:05:17.264 | INFO | "gpu" : " 17 GB", | 20260506 23:05:17.264 | INFO | "huge" : "", | 20260506 23:05:17.264 | INFO | "huge_shared" : " 27 GB", | 20260506 23:05:17.264 | INFO | "malloc" : " 14 GB", | 20260506 23:05:17.264 | INFO | "pinned" : " 29 GB" | 20260506 23:05:17.264 | INFO | } | 20260506 23:05:17.264 | INFO | Perf report for the last 5.14348 seconds: | 20260506 23:05:17.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:05:17.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:17.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:17.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:17.264 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 23:05:17.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:17.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:17.264 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:05:17.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:17.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:17.264 | INFO | Uploaded 36GiB to GPU in 1.93646s (18.5906GiB/s) | 20260506 23:05:17.264 | INFO | Downloaded 34GiB to GPU in 1.87762s (18.108GiB/s) | 20260506 23:05:22.404 | INFO | Analyzing chunk frames = [5163008,5163520) | 20260506 23:05:22.404 | INFO | MemInfo: { | 20260506 23:05:22.404 | INFO | "gpu" : " 17 GB", | 20260506 23:05:22.404 | INFO | "huge" : "", | 20260506 23:05:22.404 | INFO | "huge_shared" : " 27 GB", | 20260506 23:05:22.405 | INFO | "malloc" : " 14 GB", | 20260506 23:05:22.405 | INFO | "pinned" : " 29 GB" | 20260506 23:05:22.405 | INFO | } | 20260506 23:05:22.405 | INFO | Perf report for the last 5.1405 seconds: | 20260506 23:05:22.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:05:22.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:22.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:22.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:22.405 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 23:05:22.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:22.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:22.405 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:05:22.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:22.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:22.405 | INFO | Uploaded 36GiB to GPU in 2.0088s (17.9211GiB/s) | 20260506 23:05:22.405 | INFO | Downloaded 34GiB to GPU in 1.92151s (17.6945GiB/s) | 20260506 23:05:27.550 | INFO | Analyzing chunk frames = [5163520,5164032) | 20260506 23:05:27.550 | INFO | MemInfo: { | 20260506 23:05:27.550 | INFO | "gpu" : " 17 GB", | 20260506 23:05:27.550 | INFO | "huge" : "", | 20260506 23:05:27.550 | INFO | "huge_shared" : " 27 GB", | 20260506 23:05:27.550 | INFO | "malloc" : " 14 GB", | 20260506 23:05:27.550 | INFO | "pinned" : " 29 GB" | 20260506 23:05:27.550 | INFO | } | 20260506 23:05:27.550 | INFO | Perf report for the last 5.14528 seconds: | 20260506 23:05:27.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:05:27.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:27.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:27.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:27.550 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 23:05:27.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:27.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:27.550 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:05:27.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:27.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:27.550 | INFO | Uploaded 36GiB to GPU in 2.01532s (17.8631GiB/s) | 20260506 23:05:27.550 | INFO | Downloaded 34GiB to GPU in 1.93431s (17.5773GiB/s) | 20260506 23:05:32.692 | INFO | Analyzing chunk frames = [5164032,5164544) | 20260506 23:05:32.692 | INFO | MemInfo: { | 20260506 23:05:32.692 | INFO | "gpu" : " 17 GB", | 20260506 23:05:32.692 | INFO | "huge" : "", | 20260506 23:05:32.693 | INFO | "huge_shared" : " 27 GB", | 20260506 23:05:32.693 | INFO | "malloc" : " 14 GB", | 20260506 23:05:32.693 | INFO | "pinned" : " 29 GB" | 20260506 23:05:32.693 | INFO | } | 20260506 23:05:32.693 | INFO | Perf report for the last 5.14274 seconds: | 20260506 23:05:32.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:05:32.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:32.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:32.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:32.693 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:05:32.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:32.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:32.693 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 23:05:32.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:32.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:32.693 | INFO | Uploaded 36GiB to GPU in 1.9901s (18.0895GiB/s) | 20260506 23:05:32.693 | INFO | Downloaded 34GiB to GPU in 1.98416s (17.1357GiB/s) | 20260506 23:05:37.827 | INFO | Analyzing chunk frames = [5164544,5165056) | 20260506 23:05:37.827 | INFO | MemInfo: { | 20260506 23:05:37.827 | INFO | "gpu" : " 17 GB", | 20260506 23:05:37.827 | INFO | "huge" : "", | 20260506 23:05:37.827 | INFO | "huge_shared" : " 27 GB", | 20260506 23:05:37.827 | INFO | "malloc" : " 14 GB", | 20260506 23:05:37.827 | INFO | "pinned" : " 29 GB" | 20260506 23:05:37.827 | INFO | } | 20260506 23:05:37.827 | INFO | Perf report for the last 5.13411 seconds: | 20260506 23:05:37.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:05:37.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:37.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:37.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:37.827 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:05:37.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:37.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:37.827 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:05:37.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:37.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:37.827 | INFO | Uploaded 36GiB to GPU in 1.99082s (18.083GiB/s) | 20260506 23:05:37.827 | INFO | Downloaded 34GiB to GPU in 1.98121s (17.1612GiB/s) | 20260506 23:05:37.928 | INFO | Starting a round of Baz IO | 20260506 23:05:41.060 | INFO | Populating RTMetrics report | 20260506 23:05:41.060 | INFO | RTMetrics report populated | 20260506 23:05:41.060 | INFO | Opening tmp RTMetrics json file | 20260506 23:05:41.061 | INFO | Serializing RTMetrics to json | 20260506 23:05:41.071 | INFO | Serialization done, swapping tmp file now... | 20260506 23:05:41.082 | INFO | RTMetrics done | 20260506 23:05:41.153 | INFO | Finished a round of Baz IO: Wrote 1.891 GiB in 3.225 seconds | 20260506 23:05:41.153 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.54% / 0.00% / 66.46% | 20260506 23:05:41.153 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.446816% / 0% / 4.37101e-05% | 20260506 23:05:41.153 | INFO | Baz IO overall: | 20260506 23:05:41.153 | INFO | BytesWritten : 2707330151543 | 20260506 23:05:41.153 | INFO | NumEvents : 1305648052639 | 20260506 23:05:42.964 | INFO | Analyzing chunk frames = [5165056,5165568) | 20260506 23:05:42.964 | INFO | MemInfo: { | 20260506 23:05:42.964 | INFO | "gpu" : " 17 GB", | 20260506 23:05:42.964 | INFO | "huge" : "", | 20260506 23:05:42.964 | INFO | "huge_shared" : " 27 GB", | 20260506 23:05:42.964 | INFO | "malloc" : " 14 GB", | 20260506 23:05:42.964 | INFO | "pinned" : " 29 GB" | 20260506 23:05:42.964 | INFO | } | 20260506 23:05:42.964 | INFO | Perf report for the last 5.13736 seconds: | 20260506 23:05:42.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:05:42.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:42.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:42.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:42.964 | INFO | ANALYSIS : 62.4%, 3.0, 97.0ms, 0 | 20260506 23:05:42.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:42.964 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 23:05:42.964 | INFO | PRE_HQ : 60.1%, 1.8, 56.9ms, 0 | 20260506 23:05:42.964 | INFO | BAZWRITER : 62.4%, 3.4, 108.2ms, 0 | 20260506 23:05:42.964 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:05:42.965 | INFO | Uploaded 36GiB to GPU in 2.0288s (17.7444GiB/s) | 20260506 23:05:42.965 | INFO | Downloaded 38GiB to GPU in 2.17963s (17.4342GiB/s) | 20260506 23:05:48.096 | INFO | Analyzing chunk frames = [5165568,5166080) | 20260506 23:05:48.096 | INFO | MemInfo: { | 20260506 23:05:48.096 | INFO | "gpu" : " 17 GB", | 20260506 23:05:48.096 | INFO | "huge" : "", | 20260506 23:05:48.096 | INFO | "huge_shared" : " 27 GB", | 20260506 23:05:48.096 | INFO | "malloc" : " 14 GB", | 20260506 23:05:48.096 | INFO | "pinned" : " 29 GB" | 20260506 23:05:48.096 | INFO | } | 20260506 23:05:48.096 | INFO | Perf report for the last 5.13198 seconds: | 20260506 23:05:48.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:05:48.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:48.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:48.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:48.096 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:05:48.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:48.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:48.096 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:05:48.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:48.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:48.096 | INFO | Uploaded 36GiB to GPU in 1.99226s (18.07GiB/s) | 20260506 23:05:48.097 | INFO | Downloaded 34GiB to GPU in 1.924s (17.6715GiB/s) | 20260506 23:05:53.242 | INFO | Analyzing chunk frames = [5166080,5166592) | 20260506 23:05:53.242 | INFO | MemInfo: { | 20260506 23:05:53.242 | INFO | "gpu" : " 17 GB", | 20260506 23:05:53.242 | INFO | "huge" : "", | 20260506 23:05:53.242 | INFO | "huge_shared" : " 27 GB", | 20260506 23:05:53.242 | INFO | "malloc" : " 14 GB", | 20260506 23:05:53.242 | INFO | "pinned" : " 29 GB" | 20260506 23:05:53.242 | INFO | } | 20260506 23:05:53.242 | INFO | Perf report for the last 5.14557 seconds: | 20260506 23:05:53.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:05:53.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:53.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:53.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:53.242 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 23:05:53.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:53.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:53.242 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:05:53.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:53.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:53.242 | INFO | Uploaded 36GiB to GPU in 1.97779s (18.2022GiB/s) | 20260506 23:05:53.242 | INFO | Downloaded 34GiB to GPU in 1.91735s (17.7328GiB/s) | 20260506 23:05:58.387 | INFO | Analyzing chunk frames = [5166592,5167104) | 20260506 23:05:58.387 | INFO | MemInfo: { | 20260506 23:05:58.387 | INFO | "gpu" : " 17 GB", | 20260506 23:05:58.387 | INFO | "huge" : "", | 20260506 23:05:58.387 | INFO | "huge_shared" : " 27 GB", | 20260506 23:05:58.387 | INFO | "malloc" : " 14 GB", | 20260506 23:05:58.387 | INFO | "pinned" : " 29 GB" | 20260506 23:05:58.387 | INFO | } | 20260506 23:05:58.387 | INFO | Perf report for the last 5.14557 seconds: | 20260506 23:05:58.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:05:58.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:58.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:58.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:58.387 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260506 23:05:58.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:58.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:58.387 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:05:58.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:58.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:05:58.388 | INFO | Uploaded 36GiB to GPU in 2.01869s (17.8334GiB/s) | 20260506 23:05:58.388 | INFO | Downloaded 34GiB to GPU in 1.90934s (17.8072GiB/s) | 20260506 23:06:03.522 | INFO | Analyzing chunk frames = [5167104,5167616) | 20260506 23:06:03.522 | INFO | MemInfo: { | 20260506 23:06:03.522 | INFO | "gpu" : " 17 GB", | 20260506 23:06:03.522 | INFO | "huge" : "", | 20260506 23:06:03.522 | INFO | "huge_shared" : " 27 GB", | 20260506 23:06:03.522 | INFO | "malloc" : " 14 GB", | 20260506 23:06:03.522 | INFO | "pinned" : " 29 GB" | 20260506 23:06:03.522 | INFO | } | 20260506 23:06:03.522 | INFO | Perf report for the last 5.13475 seconds: | 20260506 23:06:03.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:06:03.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:03.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:03.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:03.522 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 23:06:03.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:03.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:03.522 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:06:03.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:03.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:03.522 | INFO | Uploaded 36GiB to GPU in 2.01479s (17.8679GiB/s) | 20260506 23:06:03.522 | INFO | Downloaded 34GiB to GPU in 1.94814s (17.4525GiB/s) | 20260506 23:06:08.661 | INFO | Analyzing chunk frames = [5167616,5168128) | 20260506 23:06:08.661 | INFO | MemInfo: { | 20260506 23:06:08.661 | INFO | "gpu" : " 17 GB", | 20260506 23:06:08.661 | INFO | "huge" : "", | 20260506 23:06:08.661 | INFO | "huge_shared" : " 27 GB", | 20260506 23:06:08.661 | INFO | "malloc" : " 14 GB", | 20260506 23:06:08.661 | INFO | "pinned" : " 29 GB" | 20260506 23:06:08.661 | INFO | } | 20260506 23:06:08.661 | INFO | Perf report for the last 5.1393 seconds: | 20260506 23:06:08.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:06:08.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:08.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:08.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:08.661 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 23:06:08.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:08.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:08.661 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:06:08.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:08.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:08.662 | INFO | Uploaded 36GiB to GPU in 2.02603s (17.7687GiB/s) | 20260506 23:06:08.662 | INFO | Downloaded 34GiB to GPU in 1.9345s (17.5756GiB/s) | 20260506 23:06:13.794 | INFO | Analyzing chunk frames = [5168128,5168640) | 20260506 23:06:13.794 | INFO | MemInfo: { | 20260506 23:06:13.794 | INFO | "gpu" : " 17 GB", | 20260506 23:06:13.794 | INFO | "huge" : "", | 20260506 23:06:13.795 | INFO | "huge_shared" : " 27 GB", | 20260506 23:06:13.795 | INFO | "malloc" : " 14 GB", | 20260506 23:06:13.795 | INFO | "pinned" : " 29 GB" | 20260506 23:06:13.795 | INFO | } | 20260506 23:06:13.795 | INFO | Perf report for the last 5.13324 seconds: | 20260506 23:06:13.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:06:13.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:13.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:13.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:13.795 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 23:06:13.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:13.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:13.795 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 23:06:13.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:13.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:13.795 | INFO | Uploaded 36GiB to GPU in 1.95148s (18.4475GiB/s) | 20260506 23:06:13.795 | INFO | Downloaded 34GiB to GPU in 1.98904s (17.0937GiB/s) | 20260506 23:06:18.931 | INFO | Analyzing chunk frames = [5168640,5169152) | 20260506 23:06:18.931 | INFO | MemInfo: { | 20260506 23:06:18.931 | INFO | "gpu" : " 17 GB", | 20260506 23:06:18.932 | INFO | "huge" : "", | 20260506 23:06:18.932 | INFO | "huge_shared" : " 27 GB", | 20260506 23:06:18.932 | INFO | "malloc" : " 14 GB", | 20260506 23:06:18.932 | INFO | "pinned" : " 29 GB" | 20260506 23:06:18.932 | INFO | } | 20260506 23:06:18.932 | INFO | Perf report for the last 5.13704 seconds: | 20260506 23:06:18.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:06:18.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:18.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:18.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:18.932 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 23:06:18.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:18.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:18.932 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 23:06:18.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:18.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:18.932 | INFO | Uploaded 36GiB to GPU in 1.98542s (18.1322GiB/s) | 20260506 23:06:18.932 | INFO | Downloaded 34GiB to GPU in 1.98982s (17.087GiB/s) | 20260506 23:06:18.932 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:06:18.932 | INFO | Total mem usage: 18 GiB | 20260506 23:06:18.932 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:06:18.932 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 23:06:18.932 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:06:18.932 | INFO | High Water Breakdown: | 20260506 23:06:18.932 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:06:18.932 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:06:18.932 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:06:18.932 | INFO | ------------------------------------------ | 20260506 23:06:18.932 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:06:18.932 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:06:18.932 | INFO | Total mem usage: 26.5 GiB | 20260506 23:06:18.932 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:06:18.932 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:06:18.932 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:06:18.932 | INFO | High Water Breakdown: | 20260506 23:06:18.932 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:06:18.933 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:06:18.933 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:06:18.933 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:06:18.933 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:06:18.933 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:06:18.933 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:06:18.933 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:06:18.933 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:06:18.933 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:06:18.933 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:06:18.933 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:06:18.933 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:06:18.933 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:06:18.933 | INFO | ------------------------------------------ | 20260506 23:06:18.933 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:06:18.933 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:06:18.933 | INFO | Total mem usage: 25.6 GiB | 20260506 23:06:18.933 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:06:18.933 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:06:18.933 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:06:18.933 | INFO | High Water Breakdown: | 20260506 23:06:18.933 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:06:18.933 | INFO | ------------------------------------------ | 20260506 23:06:18.933 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:06:18.933 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:06:18.933 | INFO | Total mem usage: 15.7 GiB | 20260506 23:06:18.933 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:06:18.933 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:06:18.933 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:06:18.933 | INFO | High Water Breakdown: | 20260506 23:06:18.933 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:06:18.933 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:06:18.933 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:06:18.933 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:06:18.933 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:06:18.933 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:06:18.933 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:06:18.933 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:06:18.933 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:06:18.933 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:06:18.933 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:06:18.933 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:06:18.933 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:06:18.933 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:06:18.933 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:06:18.933 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:06:18.933 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:06:18.933 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:06:18.933 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:06:18.933 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:06:18.933 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:06:18.933 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:06:18.933 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:06:18.933 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:06:18.933 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:06:18.933 | INFO | ------------------------------------------ | 20260506 23:06:18.933 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:06:18.933 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:06:18.933 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:06:18.933 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:06:18.933 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.58e+01, 1.35e+01 / 2.27e+00 | 20260506 23:06:18.933 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 9.68e-03 / 5.07e-03 | 20260506 23:06:18.933 | INFO | PerfReport: 11.65%, Baseline -- 3.61e+00 / 4.43e+00, 5.44e-01 / 3.47e-01 | 20260506 23:06:18.933 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.48e+00, 1.80e-01 / 1.20e-01 | 20260506 23:06:18.933 | INFO | PerfReport: 16.06%, DME -- 3.99e+01 / 4.47e+01, 4.08e+00 / 2.39e+00 | 20260506 23:06:18.933 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.39e+00 / 1.92e+00 | 20260506 23:06:18.933 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.19e+00 / 2.79e+00, 3.79e-01 / 1.84e-01 | 20260506 23:06:18.933 | INFO | PerfReport: 3.21%, Metrics -- 9.96e-01 / 1.33e+00, 2.64e-01 / 1.87e-01 | 20260506 23:06:18.933 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.39e-01 / 3.92e-01, 5.32e-02 / 2.86e-02 | 20260506 23:06:18.934 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:06:18.934 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:06:18.934 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:06:18.934 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 23:06:18.934 | INFO | PerfReport: 21.50%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 23:06:18.934 | INFO | PerfReport: 21.22%, Download -- 2.05e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 23:06:18.934 | INFO | PerfReport: 32.10%, Compute -- 3.10e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 23:06:18.934 | INFO | PerfReport: 1.29%, LockUpload -- 1.25e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260506 23:06:18.934 | INFO | PerfReport: 1.25%, LockDownload -- 1.21e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260506 23:06:18.934 | INFO | PerfReport: 22.62%, LockCompute -- 2.19e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:06:18.934 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260506 23:06:19.030 | INFO | Starting a round of Baz IO | 20260506 23:06:22.162 | INFO | Populating RTMetrics report | 20260506 23:06:22.163 | INFO | RTMetrics report populated | 20260506 23:06:22.163 | INFO | Opening tmp RTMetrics json file | 20260506 23:06:22.164 | INFO | Serializing RTMetrics to json | 20260506 23:06:22.165 | INFO | Serialization done, swapping tmp file now... | 20260506 23:06:22.165 | INFO | RTMetrics done | 20260506 23:06:22.257 | INFO | Finished a round of Baz IO: Wrote 1.893 GiB in 3.226 seconds | 20260506 23:06:22.257 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.54% / 0.00% / 66.46% | 20260506 23:06:22.257 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.445394% / 1.98682e-05% / 5.96046e-05% | 20260506 23:06:22.257 | INFO | Baz IO overall: | 20260506 23:06:22.257 | INFO | BytesWritten : 2709362550897 | 20260506 23:06:22.257 | INFO | NumEvents : 1306617401297 | 20260506 23:06:24.071 | INFO | Analyzing chunk frames = [5169152,5169664) | 20260506 23:06:24.071 | INFO | MemInfo: { | 20260506 23:06:24.071 | INFO | "gpu" : " 17 GB", | 20260506 23:06:24.071 | INFO | "huge" : "", | 20260506 23:06:24.071 | INFO | "huge_shared" : " 27 GB", | 20260506 23:06:24.071 | INFO | "malloc" : " 14 GB", | 20260506 23:06:24.071 | INFO | "pinned" : " 29 GB" | 20260506 23:06:24.071 | INFO | } | 20260506 23:06:24.071 | INFO | Perf report for the last 5.13973 seconds: | 20260506 23:06:24.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:06:24.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:24.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:24.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:24.072 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 23:06:24.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:24.072 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 23:06:24.072 | INFO | PRE_HQ : 59.6%, 1.9, 57.4ms, 0 | 20260506 23:06:24.072 | INFO | BAZWRITER : 62.3%, 3.4, 108.9ms, 0 | 20260506 23:06:24.072 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:06:24.072 | INFO | Uploaded 36GiB to GPU in 2.04043s (17.6433GiB/s) | 20260506 23:06:24.072 | INFO | Downloaded 38GiB to GPU in 2.17705s (17.4548GiB/s) | 20260506 23:06:29.212 | INFO | Analyzing chunk frames = [5169664,5170176) | 20260506 23:06:29.213 | INFO | MemInfo: { | 20260506 23:06:29.213 | INFO | "gpu" : " 17 GB", | 20260506 23:06:29.213 | INFO | "huge" : "", | 20260506 23:06:29.213 | INFO | "huge_shared" : " 27 GB", | 20260506 23:06:29.213 | INFO | "malloc" : " 14 GB", | 20260506 23:06:29.213 | INFO | "pinned" : " 29 GB" | 20260506 23:06:29.213 | INFO | } | 20260506 23:06:29.213 | INFO | Perf report for the last 5.14128 seconds: | 20260506 23:06:29.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:06:29.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:29.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:29.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:29.213 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 23:06:29.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:29.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:29.213 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:06:29.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:29.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:29.213 | INFO | Uploaded 36GiB to GPU in 2.01235s (17.8895GiB/s) | 20260506 23:06:29.213 | INFO | Downloaded 34GiB to GPU in 1.92535s (17.6591GiB/s) | 20260506 23:06:34.359 | INFO | Analyzing chunk frames = [5170176,5170688) | 20260506 23:06:34.359 | INFO | MemInfo: { | 20260506 23:06:34.359 | INFO | "gpu" : " 17 GB", | 20260506 23:06:34.359 | INFO | "huge" : "", | 20260506 23:06:34.359 | INFO | "huge_shared" : " 27 GB", | 20260506 23:06:34.359 | INFO | "malloc" : " 14 GB", | 20260506 23:06:34.359 | INFO | "pinned" : " 29 GB" | 20260506 23:06:34.359 | INFO | } | 20260506 23:06:34.360 | INFO | Perf report for the last 5.14676 seconds: | 20260506 23:06:34.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:06:34.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:34.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:34.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:34.360 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260506 23:06:34.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:34.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:34.360 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:06:34.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:34.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:34.360 | INFO | Uploaded 36GiB to GPU in 1.99202s (18.0721GiB/s) | 20260506 23:06:34.360 | INFO | Downloaded 34GiB to GPU in 1.9229s (17.6816GiB/s) | 20260506 23:06:39.501 | INFO | Analyzing chunk frames = [5170688,5171200) | 20260506 23:06:39.501 | INFO | MemInfo: { | 20260506 23:06:39.501 | INFO | "gpu" : " 17 GB", | 20260506 23:06:39.501 | INFO | "huge" : "", | 20260506 23:06:39.501 | INFO | "huge_shared" : " 27 GB", | 20260506 23:06:39.501 | INFO | "malloc" : " 14 GB", | 20260506 23:06:39.501 | INFO | "pinned" : " 29 GB" | 20260506 23:06:39.501 | INFO | } | 20260506 23:06:39.501 | INFO | Perf report for the last 5.14187 seconds: | 20260506 23:06:39.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:06:39.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:39.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:39.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:39.501 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260506 23:06:39.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:39.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:39.502 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:06:39.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:39.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:39.502 | INFO | Uploaded 36GiB to GPU in 2.0116s (17.8962GiB/s) | 20260506 23:06:39.502 | INFO | Downloaded 34GiB to GPU in 1.90994s (17.8016GiB/s) | 20260506 23:06:44.639 | INFO | Analyzing chunk frames = [5171200,5171712) | 20260506 23:06:44.639 | INFO | MemInfo: { | 20260506 23:06:44.639 | INFO | "gpu" : " 17 GB", | 20260506 23:06:44.639 | INFO | "huge" : "", | 20260506 23:06:44.639 | INFO | "huge_shared" : " 27 GB", | 20260506 23:06:44.639 | INFO | "malloc" : " 14 GB", | 20260506 23:06:44.639 | INFO | "pinned" : " 29 GB" | 20260506 23:06:44.639 | INFO | } | 20260506 23:06:44.639 | INFO | Perf report for the last 5.13748 seconds: | 20260506 23:06:44.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:06:44.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:44.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:44.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:44.639 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260506 23:06:44.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:44.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:44.639 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:06:44.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:44.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:44.639 | INFO | Uploaded 36GiB to GPU in 2.01645s (17.8532GiB/s) | 20260506 23:06:44.639 | INFO | Downloaded 34GiB to GPU in 1.93616s (17.5606GiB/s) | 20260506 23:06:49.777 | INFO | Analyzing chunk frames = [5171712,5172224) | 20260506 23:06:49.777 | INFO | MemInfo: { | 20260506 23:06:49.777 | INFO | "gpu" : " 17 GB", | 20260506 23:06:49.777 | INFO | "huge" : "", | 20260506 23:06:49.777 | INFO | "huge_shared" : " 27 GB", | 20260506 23:06:49.777 | INFO | "malloc" : " 14 GB", | 20260506 23:06:49.777 | INFO | "pinned" : " 29 GB" | 20260506 23:06:49.777 | INFO | } | 20260506 23:06:49.777 | INFO | Perf report for the last 5.13835 seconds: | 20260506 23:06:49.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:06:49.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:49.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:49.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:49.777 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 23:06:49.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:49.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:49.777 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:06:49.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:49.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:49.778 | INFO | Uploaded 36GiB to GPU in 2.01573s (17.8595GiB/s) | 20260506 23:06:49.778 | INFO | Downloaded 34GiB to GPU in 1.92812s (17.6338GiB/s) | 20260506 23:06:54.912 | INFO | Analyzing chunk frames = [5172224,5172736) | 20260506 23:06:54.912 | INFO | MemInfo: { | 20260506 23:06:54.912 | INFO | "gpu" : " 17 GB", | 20260506 23:06:54.912 | INFO | "huge" : "", | 20260506 23:06:54.912 | INFO | "huge_shared" : " 27 GB", | 20260506 23:06:54.912 | INFO | "malloc" : " 14 GB", | 20260506 23:06:54.912 | INFO | "pinned" : " 29 GB" | 20260506 23:06:54.912 | INFO | } | 20260506 23:06:54.912 | INFO | Perf report for the last 5.13487 seconds: | 20260506 23:06:54.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:06:54.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:54.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:54.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:54.912 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 23:06:54.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:54.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:54.912 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 23:06:54.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:54.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:06:54.913 | INFO | Uploaded 36GiB to GPU in 1.97443s (18.2331GiB/s) | 20260506 23:06:54.913 | INFO | Downloaded 34GiB to GPU in 1.99755s (17.0208GiB/s) | 20260506 23:07:00.049 | INFO | Analyzing chunk frames = [5172736,5173248) | 20260506 23:07:00.049 | INFO | MemInfo: { | 20260506 23:07:00.049 | INFO | "gpu" : " 17 GB", | 20260506 23:07:00.049 | INFO | "huge" : "", | 20260506 23:07:00.049 | INFO | "huge_shared" : " 27 GB", | 20260506 23:07:00.049 | INFO | "malloc" : " 14 GB", | 20260506 23:07:00.049 | INFO | "pinned" : " 29 GB" | 20260506 23:07:00.049 | INFO | } | 20260506 23:07:00.049 | INFO | Perf report for the last 5.1368 seconds: | 20260506 23:07:00.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:07:00.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:00.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:00.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:00.049 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 23:07:00.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:00.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:00.049 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 23:07:00.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:00.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:00.049 | INFO | Uploaded 36GiB to GPU in 2.0017s (17.9847GiB/s) | 20260506 23:07:00.049 | INFO | Downloaded 34GiB to GPU in 1.99883s (17.01GiB/s) | 20260506 23:07:00.165 | INFO | Starting a round of Baz IO | 20260506 23:07:03.282 | INFO | Populating RTMetrics report | 20260506 23:07:03.282 | INFO | RTMetrics report populated | 20260506 23:07:03.282 | INFO | Opening tmp RTMetrics json file | 20260506 23:07:03.283 | INFO | Serializing RTMetrics to json | 20260506 23:07:03.284 | INFO | Serialization done, swapping tmp file now... | 20260506 23:07:03.285 | INFO | RTMetrics done | 20260506 23:07:03.379 | INFO | Finished a round of Baz IO: Wrote 1.89 GiB in 3.214 seconds | 20260506 23:07:03.379 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.53% / 0.00% / 66.46% | 20260506 23:07:03.379 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.444674% / 1.19209e-05% / 4.37101e-05% | 20260506 23:07:03.379 | INFO | Baz IO overall: | 20260506 23:07:03.379 | INFO | BytesWritten : 2711392344987 | 20260506 23:07:03.379 | INFO | NumEvents : 1307584674577 | 20260506 23:07:05.191 | INFO | Analyzing chunk frames = [5173248,5173760) | 20260506 23:07:05.191 | INFO | MemInfo: { | 20260506 23:07:05.191 | INFO | "gpu" : " 17 GB", | 20260506 23:07:05.191 | INFO | "huge" : "", | 20260506 23:07:05.192 | INFO | "huge_shared" : " 27 GB", | 20260506 23:07:05.192 | INFO | "malloc" : " 14 GB", | 20260506 23:07:05.192 | INFO | "pinned" : " 29 GB" | 20260506 23:07:05.192 | INFO | } | 20260506 23:07:05.192 | INFO | Perf report for the last 5.14264 seconds: | 20260506 23:07:05.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:07:05.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:05.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:05.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:05.192 | INFO | ANALYSIS : 62.3%, 3.0, 97.0ms, 0 | 20260506 23:07:05.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:05.192 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 23:07:05.192 | INFO | PRE_HQ : 59.9%, 1.9, 57.7ms, 0 | 20260506 23:07:05.192 | INFO | BAZWRITER : 61.9%, 3.5, 111.6ms, 0 | 20260506 23:07:05.192 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:07:05.192 | INFO | Uploaded 36GiB to GPU in 2.02309s (17.7945GiB/s) | 20260506 23:07:05.192 | INFO | Downloaded 38GiB to GPU in 2.18507s (17.3908GiB/s) | 20260506 23:07:10.334 | INFO | Analyzing chunk frames = [5173760,5174272) | 20260506 23:07:10.334 | INFO | MemInfo: { | 20260506 23:07:10.334 | INFO | "gpu" : " 17 GB", | 20260506 23:07:10.334 | INFO | "huge" : "", | 20260506 23:07:10.334 | INFO | "huge_shared" : " 27 GB", | 20260506 23:07:10.334 | INFO | "malloc" : " 14 GB", | 20260506 23:07:10.334 | INFO | "pinned" : " 29 GB" | 20260506 23:07:10.334 | INFO | } | 20260506 23:07:10.334 | INFO | Perf report for the last 5.14259 seconds: | 20260506 23:07:10.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:07:10.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:10.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:10.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:10.334 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:07:10.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:10.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:10.334 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 23:07:10.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:10.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:10.335 | INFO | Uploaded 36GiB to GPU in 2.01224s (17.8905GiB/s) | 20260506 23:07:10.335 | INFO | Downloaded 34GiB to GPU in 1.93005s (17.6161GiB/s) | 20260506 23:07:15.471 | INFO | Analyzing chunk frames = [5174272,5174784) | 20260506 23:07:15.472 | INFO | MemInfo: { | 20260506 23:07:15.472 | INFO | "gpu" : " 17 GB", | 20260506 23:07:15.472 | INFO | "huge" : "", | 20260506 23:07:15.472 | INFO | "huge_shared" : " 27 GB", | 20260506 23:07:15.472 | INFO | "malloc" : " 14 GB", | 20260506 23:07:15.472 | INFO | "pinned" : " 29 GB" | 20260506 23:07:15.472 | INFO | } | 20260506 23:07:15.472 | INFO | Perf report for the last 5.1375 seconds: | 20260506 23:07:15.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:07:15.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:15.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:15.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:15.472 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 23:07:15.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:15.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:15.472 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:07:15.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:15.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:15.472 | INFO | Uploaded 36GiB to GPU in 1.99331s (18.0604GiB/s) | 20260506 23:07:15.472 | INFO | Downloaded 34GiB to GPU in 1.92926s (17.6233GiB/s) | 20260506 23:07:20.610 | INFO | Analyzing chunk frames = [5174784,5175296) | 20260506 23:07:20.610 | INFO | MemInfo: { | 20260506 23:07:20.610 | INFO | "gpu" : " 17 GB", | 20260506 23:07:20.610 | INFO | "huge" : "", | 20260506 23:07:20.610 | INFO | "huge_shared" : " 27 GB", | 20260506 23:07:20.610 | INFO | "malloc" : " 14 GB", | 20260506 23:07:20.610 | INFO | "pinned" : " 29 GB" | 20260506 23:07:20.610 | INFO | } | 20260506 23:07:20.610 | INFO | Perf report for the last 5.13867 seconds: | 20260506 23:07:20.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:07:20.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:20.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:20.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:20.610 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 23:07:20.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:20.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:20.611 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:07:20.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:20.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:20.611 | INFO | Uploaded 36GiB to GPU in 2.00644s (17.9422GiB/s) | 20260506 23:07:20.611 | INFO | Downloaded 34GiB to GPU in 1.90851s (17.8149GiB/s) | 20260506 23:07:25.746 | INFO | Analyzing chunk frames = [5175296,5175808) | 20260506 23:07:25.746 | INFO | MemInfo: { | 20260506 23:07:25.746 | INFO | "gpu" : " 17 GB", | 20260506 23:07:25.746 | INFO | "huge" : "", | 20260506 23:07:25.746 | INFO | "huge_shared" : " 27 GB", | 20260506 23:07:25.747 | INFO | "malloc" : " 14 GB", | 20260506 23:07:25.747 | INFO | "pinned" : " 29 GB" | 20260506 23:07:25.747 | INFO | } | 20260506 23:07:25.747 | INFO | Perf report for the last 5.13618 seconds: | 20260506 23:07:25.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:07:25.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:25.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:25.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:25.747 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 23:07:25.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:25.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:25.747 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:07:25.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:25.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:25.747 | INFO | Uploaded 36GiB to GPU in 1.99953s (18.0042GiB/s) | 20260506 23:07:25.747 | INFO | Downloaded 34GiB to GPU in 1.92577s (17.6553GiB/s) | 20260506 23:07:30.886 | INFO | Analyzing chunk frames = [5175808,5176320) | 20260506 23:07:30.886 | INFO | MemInfo: { | 20260506 23:07:30.886 | INFO | "gpu" : " 17 GB", | 20260506 23:07:30.886 | INFO | "huge" : "", | 20260506 23:07:30.886 | INFO | "huge_shared" : " 27 GB", | 20260506 23:07:30.886 | INFO | "malloc" : " 14 GB", | 20260506 23:07:30.886 | INFO | "pinned" : " 29 GB" | 20260506 23:07:30.886 | INFO | } | 20260506 23:07:30.887 | INFO | Perf report for the last 5.13989 seconds: | 20260506 23:07:30.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:07:30.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:30.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:30.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:30.887 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 23:07:30.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:30.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:30.887 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 23:07:30.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:30.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:30.887 | INFO | Uploaded 36GiB to GPU in 2.02301s (17.7952GiB/s) | 20260506 23:07:30.887 | INFO | Downloaded 34GiB to GPU in 1.94985s (17.4373GiB/s) | 20260506 23:07:36.025 | INFO | Analyzing chunk frames = [5176320,5176832) | 20260506 23:07:36.025 | INFO | MemInfo: { | 20260506 23:07:36.025 | INFO | "gpu" : " 17 GB", | 20260506 23:07:36.025 | INFO | "huge" : "", | 20260506 23:07:36.025 | INFO | "huge_shared" : " 27 GB", | 20260506 23:07:36.025 | INFO | "malloc" : " 14 GB", | 20260506 23:07:36.025 | INFO | "pinned" : " 29 GB" | 20260506 23:07:36.025 | INFO | } | 20260506 23:07:36.025 | INFO | Perf report for the last 5.13832 seconds: | 20260506 23:07:36.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:07:36.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:36.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:36.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:36.025 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 23:07:36.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:36.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:36.025 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 23:07:36.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:36.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:36.025 | INFO | Uploaded 36GiB to GPU in 1.97539s (18.2242GiB/s) | 20260506 23:07:36.025 | INFO | Downloaded 34GiB to GPU in 1.98936s (17.0909GiB/s) | 20260506 23:07:41.167 | INFO | Analyzing chunk frames = [5176832,5177344) | 20260506 23:07:41.167 | INFO | MemInfo: { | 20260506 23:07:41.167 | INFO | "gpu" : " 17 GB", | 20260506 23:07:41.167 | INFO | "huge" : "", | 20260506 23:07:41.167 | INFO | "huge_shared" : " 27 GB", | 20260506 23:07:41.167 | INFO | "malloc" : " 14 GB", | 20260506 23:07:41.167 | INFO | "pinned" : " 29 GB" | 20260506 23:07:41.167 | INFO | } | 20260506 23:07:41.168 | INFO | Perf report for the last 5.14274 seconds: | 20260506 23:07:41.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:07:41.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:41.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:41.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:41.168 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 23:07:41.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:41.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:41.168 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 23:07:41.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:41.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:41.168 | INFO | Uploaded 36GiB to GPU in 1.99688s (18.0281GiB/s) | 20260506 23:07:41.168 | INFO | Downloaded 34GiB to GPU in 2.00175s (16.9851GiB/s) | 20260506 23:07:41.168 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:07:41.168 | INFO | Total mem usage: 18 GiB | 20260506 23:07:41.168 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:07:41.168 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 23:07:41.168 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:07:41.168 | INFO | High Water Breakdown: | 20260506 23:07:41.168 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:07:41.168 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:07:41.168 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:07:41.168 | INFO | ------------------------------------------ | 20260506 23:07:41.168 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:07:41.168 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:07:41.168 | INFO | Total mem usage: 26.5 GiB | 20260506 23:07:41.168 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:07:41.168 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:07:41.168 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:07:41.168 | INFO | High Water Breakdown: | 20260506 23:07:41.168 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:07:41.168 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:07:41.168 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:07:41.168 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:07:41.168 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:07:41.168 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:07:41.168 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:07:41.168 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:07:41.168 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:07:41.168 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:07:41.168 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:07:41.168 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:07:41.168 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:07:41.168 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:07:41.168 | INFO | ------------------------------------------ | 20260506 23:07:41.168 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:07:41.168 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:07:41.168 | INFO | Total mem usage: 25.6 GiB | 20260506 23:07:41.168 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:07:41.169 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:07:41.169 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:07:41.169 | INFO | High Water Breakdown: | 20260506 23:07:41.169 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:07:41.169 | INFO | ------------------------------------------ | 20260506 23:07:41.169 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:07:41.169 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:07:41.169 | INFO | Total mem usage: 15.7 GiB | 20260506 23:07:41.169 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:07:41.169 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:07:41.169 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 23:07:41.169 | INFO | High Water Breakdown: | 20260506 23:07:41.169 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:07:41.169 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:07:41.169 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:07:41.169 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:07:41.169 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:07:41.169 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:07:41.169 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:07:41.169 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:07:41.169 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:07:41.169 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:07:41.169 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:07:41.169 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:07:41.169 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:07:41.169 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:07:41.169 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:07:41.169 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:07:41.169 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:07:41.169 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:07:41.169 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:07:41.169 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:07:41.169 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:07:41.169 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:07:41.169 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:07:41.169 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:07:41.169 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:07:41.169 | INFO | ------------------------------------------ | 20260506 23:07:41.169 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:07:41.169 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:07:41.169 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:07:41.169 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:07:41.169 | INFO | PerfReport: Full Execution -- 3.11e+01 / 7.65e+01, 1.36e+01 / 2.76e+00 | 20260506 23:07:41.169 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 1.01e-02 / 5.91e-03 | 20260506 23:07:41.169 | INFO | PerfReport: 11.67%, Baseline -- 3.62e+00 / 4.35e+00, 5.46e-01 / 3.59e-01 | 20260506 23:07:41.169 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.43e+00, 1.80e-01 / 1.25e-01 | 20260506 23:07:41.169 | INFO | PerfReport: 16.14%, DME -- 4.01e+01 / 4.54e+01, 4.54e+00 / 3.64e+00 | 20260506 23:07:41.169 | INFO | PerfReport: 57.28%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.37e+00 / 1.94e+00 | 20260506 23:07:41.169 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.19e+00 / 2.73e+00, 3.77e-01 / 2.04e-01 | 20260506 23:07:41.169 | INFO | PerfReport: 3.20%, Metrics -- 9.95e-01 / 1.33e+00, 2.64e-01 / 1.41e-01 | 20260506 23:07:41.169 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.39e-01 / 3.87e-01, 5.27e-02 / 1.82e-02 | 20260506 23:07:41.169 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:07:41.169 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:07:41.169 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:07:41.169 | INFO | PerfReport: Full Execution -- 9.69e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260506 23:07:41.169 | INFO | PerfReport: 21.56%, Upload -- 2.09e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260506 23:07:41.169 | INFO | PerfReport: 21.24%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 23:07:41.169 | INFO | PerfReport: 32.05%, Compute -- 3.11e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 23:07:41.169 | INFO | PerfReport: 1.36%, LockUpload -- 1.32e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260506 23:07:41.169 | INFO | PerfReport: 1.25%, LockDownload -- 1.21e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260506 23:07:41.169 | INFO | PerfReport: 22.52%, LockCompute -- 2.18e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260506 23:07:41.169 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260506 23:07:41.267 | INFO | Starting a round of Baz IO | 20260506 23:07:44.399 | INFO | Populating RTMetrics report | 20260506 23:07:44.400 | INFO | RTMetrics report populated | 20260506 23:07:44.400 | INFO | Opening tmp RTMetrics json file | 20260506 23:07:44.419 | INFO | Serializing RTMetrics to json | 20260506 23:07:44.420 | INFO | Serialization done, swapping tmp file now... | 20260506 23:07:44.421 | INFO | RTMetrics done | 20260506 23:07:44.494 | INFO | Finished a round of Baz IO: Wrote 1.891 GiB in 3.227 seconds | 20260506 23:07:44.494 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.53% / 0.00% / 66.47% | 20260506 23:07:44.494 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.443641% / 3.97364e-06% / 3.97364e-05% | 20260506 23:07:44.494 | INFO | Baz IO overall: | 20260506 23:07:44.494 | INFO | BytesWritten : 2713422920187 | 20260506 23:07:44.494 | INFO | NumEvents : 1308552683683 | 20260506 23:07:46.311 | INFO | Analyzing chunk frames = [5177344,5177856) | 20260506 23:07:46.311 | INFO | MemInfo: { | 20260506 23:07:46.311 | INFO | "gpu" : " 17 GB", | 20260506 23:07:46.311 | INFO | "huge" : "", | 20260506 23:07:46.311 | INFO | "huge_shared" : " 27 GB", | 20260506 23:07:46.311 | INFO | "malloc" : " 14 GB", | 20260506 23:07:46.311 | INFO | "pinned" : " 29 GB" | 20260506 23:07:46.311 | INFO | } | 20260506 23:07:46.311 | INFO | Perf report for the last 5.14371 seconds: | 20260506 23:07:46.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:07:46.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:46.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:46.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:46.311 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 23:07:46.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:46.312 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260506 23:07:46.312 | INFO | PRE_HQ : 60.2%, 1.9, 60.1ms, 0 | 20260506 23:07:46.312 | INFO | BAZWRITER : 62.3%, 3.6, 115.2ms, 0 | 20260506 23:07:46.312 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:07:46.312 | INFO | Uploaded 36GiB to GPU in 2.00325s (17.9708GiB/s) | 20260506 23:07:46.312 | INFO | Downloaded 38GiB to GPU in 2.17752s (17.4511GiB/s) | 20260506 23:07:51.453 | INFO | Analyzing chunk frames = [5177856,5178368) | 20260506 23:07:51.453 | INFO | MemInfo: { | 20260506 23:07:51.453 | INFO | "gpu" : " 17 GB", | 20260506 23:07:51.453 | INFO | "huge" : "", | 20260506 23:07:51.453 | INFO | "huge_shared" : " 27 GB", | 20260506 23:07:51.453 | INFO | "malloc" : " 14 GB", | 20260506 23:07:51.453 | INFO | "pinned" : " 29 GB" | 20260506 23:07:51.453 | INFO | } | 20260506 23:07:51.453 | INFO | Perf report for the last 5.14165 seconds: | 20260506 23:07:51.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:07:51.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:51.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:51.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:51.453 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:07:51.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:51.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:51.453 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 23:07:51.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:51.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:51.453 | INFO | Uploaded 36GiB to GPU in 2.01953s (17.8259GiB/s) | 20260506 23:07:51.453 | INFO | Downloaded 34GiB to GPU in 1.93649s (17.5575GiB/s) | 20260506 23:07:56.588 | INFO | Analyzing chunk frames = [5178368,5178880) | 20260506 23:07:56.588 | INFO | MemInfo: { | 20260506 23:07:56.588 | INFO | "gpu" : " 17 GB", | 20260506 23:07:56.588 | INFO | "huge" : "", | 20260506 23:07:56.588 | INFO | "huge_shared" : " 27 GB", | 20260506 23:07:56.588 | INFO | "malloc" : " 14 GB", | 20260506 23:07:56.588 | INFO | "pinned" : " 29 GB" | 20260506 23:07:56.588 | INFO | } | 20260506 23:07:56.588 | INFO | Perf report for the last 5.13519 seconds: | 20260506 23:07:56.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:07:56.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:56.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:56.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:56.588 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:07:56.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:56.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:56.588 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:07:56.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:56.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:07:56.588 | INFO | Uploaded 36GiB to GPU in 2.00442s (17.9603GiB/s) | 20260506 23:07:56.588 | INFO | Downloaded 34GiB to GPU in 1.93756s (17.5478GiB/s) | 20260506 23:08:01.729 | INFO | Analyzing chunk frames = [5178880,5179392) | 20260506 23:08:01.729 | INFO | MemInfo: { | 20260506 23:08:01.729 | INFO | "gpu" : " 17 GB", | 20260506 23:08:01.729 | INFO | "huge" : "", | 20260506 23:08:01.729 | INFO | "huge_shared" : " 27 GB", | 20260506 23:08:01.729 | INFO | "malloc" : " 14 GB", | 20260506 23:08:01.729 | INFO | "pinned" : " 29 GB" | 20260506 23:08:01.729 | INFO | } | 20260506 23:08:01.730 | INFO | Perf report for the last 5.14137 seconds: | 20260506 23:08:01.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:08:01.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:01.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:01.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:01.730 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260506 23:08:01.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:01.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:01.730 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:08:01.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:01.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:01.730 | INFO | Uploaded 36GiB to GPU in 1.98434s (18.142GiB/s) | 20260506 23:08:01.730 | INFO | Downloaded 34GiB to GPU in 1.91234s (17.7793GiB/s) | 20260506 23:08:06.861 | INFO | Analyzing chunk frames = [5179392,5179904) | 20260506 23:08:06.861 | INFO | MemInfo: { | 20260506 23:08:06.861 | INFO | "gpu" : " 17 GB", | 20260506 23:08:06.861 | INFO | "huge" : "", | 20260506 23:08:06.861 | INFO | "huge_shared" : " 27 GB", | 20260506 23:08:06.861 | INFO | "malloc" : " 14 GB", | 20260506 23:08:06.861 | INFO | "pinned" : " 29 GB" | 20260506 23:08:06.861 | INFO | } | 20260506 23:08:06.861 | INFO | Perf report for the last 5.1315 seconds: | 20260506 23:08:06.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:08:06.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:06.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:06.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:06.861 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 23:08:06.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:06.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:06.861 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:08:06.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:06.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:06.861 | INFO | Uploaded 36GiB to GPU in 1.9823s (18.1607GiB/s) | 20260506 23:08:06.861 | INFO | Downloaded 34GiB to GPU in 1.92841s (17.6311GiB/s) | 20260506 23:08:12.000 | INFO | Analyzing chunk frames = [5179904,5180416) | 20260506 23:08:12.000 | INFO | MemInfo: { | 20260506 23:08:12.000 | INFO | "gpu" : " 17 GB", | 20260506 23:08:12.000 | INFO | "huge" : "", | 20260506 23:08:12.000 | INFO | "huge_shared" : " 27 GB", | 20260506 23:08:12.000 | INFO | "malloc" : " 14 GB", | 20260506 23:08:12.000 | INFO | "pinned" : " 29 GB" | 20260506 23:08:12.000 | INFO | } | 20260506 23:08:12.000 | INFO | Perf report for the last 5.13867 seconds: | 20260506 23:08:12.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:08:12.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:12.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:12.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:12.000 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 23:08:12.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:12.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:12.000 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:08:12.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:12.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:12.000 | INFO | Uploaded 36GiB to GPU in 1.99398s (18.0544GiB/s) | 20260506 23:08:12.000 | INFO | Downloaded 34GiB to GPU in 1.95535s (17.3882GiB/s) | 20260506 23:08:17.147 | INFO | Analyzing chunk frames = [5180416,5180928) | 20260506 23:08:17.147 | INFO | MemInfo: { | 20260506 23:08:17.147 | INFO | "gpu" : " 17 GB", | 20260506 23:08:17.147 | INFO | "huge" : "", | 20260506 23:08:17.147 | INFO | "huge_shared" : " 27 GB", | 20260506 23:08:17.147 | INFO | "malloc" : " 14 GB", | 20260506 23:08:17.147 | INFO | "pinned" : " 29 GB" | 20260506 23:08:17.147 | INFO | } | 20260506 23:08:17.147 | INFO | Perf report for the last 5.1477 seconds: | 20260506 23:08:17.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:08:17.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:17.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:17.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:17.147 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260506 23:08:17.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:17.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:17.148 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 23:08:17.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:17.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:17.148 | INFO | Uploaded 36GiB to GPU in 1.95604s (18.4046GiB/s) | 20260506 23:08:17.148 | INFO | Downloaded 34GiB to GPU in 1.97714s (17.1966GiB/s) | 20260506 23:08:22.281 | INFO | Analyzing chunk frames = [5180928,5181440) | 20260506 23:08:22.281 | INFO | MemInfo: { | 20260506 23:08:22.281 | INFO | "gpu" : " 17 GB", | 20260506 23:08:22.281 | INFO | "huge" : "", | 20260506 23:08:22.281 | INFO | "huge_shared" : " 27 GB", | 20260506 23:08:22.281 | INFO | "malloc" : " 14 GB", | 20260506 23:08:22.281 | INFO | "pinned" : " 29 GB" | 20260506 23:08:22.281 | INFO | } | 20260506 23:08:22.281 | INFO | Perf report for the last 5.13382 seconds: | 20260506 23:08:22.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:08:22.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:22.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:22.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:22.281 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:08:22.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:22.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:22.281 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 23:08:22.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:22.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:22.282 | INFO | Uploaded 36GiB to GPU in 1.99364s (18.0574GiB/s) | 20260506 23:08:22.282 | INFO | Downloaded 34GiB to GPU in 1.98944s (17.0902GiB/s) | 20260506 23:08:22.380 | INFO | Starting a round of Baz IO | 20260506 23:08:25.510 | INFO | Populating RTMetrics report | 20260506 23:08:25.511 | INFO | RTMetrics report populated | 20260506 23:08:25.511 | INFO | Opening tmp RTMetrics json file | 20260506 23:08:25.511 | INFO | Serializing RTMetrics to json | 20260506 23:08:25.513 | INFO | Serialization done, swapping tmp file now... | 20260506 23:08:25.513 | INFO | RTMetrics done | 20260506 23:08:25.604 | INFO | Finished a round of Baz IO: Wrote 1.891 GiB in 3.224 seconds | 20260506 23:08:25.604 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.53% / 0.00% / 66.47% | 20260506 23:08:25.604 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.442215% / 0% / 7.15256e-05% | 20260506 23:08:25.604 | INFO | Baz IO overall: | 20260506 23:08:25.604 | INFO | BytesWritten : 2715453157253 | 20260506 23:08:25.604 | INFO | NumEvents : 1309520557697 | 20260506 23:08:27.426 | INFO | Analyzing chunk frames = [5181440,5181952) | 20260506 23:08:27.426 | INFO | MemInfo: { | 20260506 23:08:27.426 | INFO | "gpu" : " 17 GB", | 20260506 23:08:27.426 | INFO | "huge" : "", | 20260506 23:08:27.427 | INFO | "huge_shared" : " 27 GB", | 20260506 23:08:27.427 | INFO | "malloc" : " 14 GB", | 20260506 23:08:27.427 | INFO | "pinned" : " 29 GB" | 20260506 23:08:27.427 | INFO | } | 20260506 23:08:27.427 | INFO | Perf report for the last 5.1454 seconds: | 20260506 23:08:27.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:08:27.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:27.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:27.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:27.427 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 23:08:27.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:27.427 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260506 23:08:27.427 | INFO | PRE_HQ : 59.7%, 1.8, 56.7ms, 0 | 20260506 23:08:27.427 | INFO | BAZWRITER : 62.2%, 3.5, 111.1ms, 0 | 20260506 23:08:27.427 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:08:27.427 | INFO | Uploaded 36GiB to GPU in 2.02614s (17.7678GiB/s) | 20260506 23:08:27.427 | INFO | Downloaded 38GiB to GPU in 2.1829s (17.408GiB/s) | 20260506 23:08:32.569 | INFO | Analyzing chunk frames = [5181952,5182464) | 20260506 23:08:32.569 | INFO | MemInfo: { | 20260506 23:08:32.569 | INFO | "gpu" : " 17 GB", | 20260506 23:08:32.569 | INFO | "huge" : "", | 20260506 23:08:32.569 | INFO | "huge_shared" : " 27 GB", | 20260506 23:08:32.569 | INFO | "malloc" : " 14 GB", | 20260506 23:08:32.569 | INFO | "pinned" : " 29 GB" | 20260506 23:08:32.569 | INFO | } | 20260506 23:08:32.569 | INFO | Perf report for the last 5.14226 seconds: | 20260506 23:08:32.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:08:32.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:32.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:32.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:32.569 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 23:08:32.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:32.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:32.569 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:08:32.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:32.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:32.569 | INFO | Uploaded 36GiB to GPU in 2.02376s (17.7887GiB/s) | 20260506 23:08:32.569 | INFO | Downloaded 34GiB to GPU in 1.9382s (17.542GiB/s) | 20260506 23:08:37.700 | INFO | Analyzing chunk frames = [5182464,5182976) | 20260506 23:08:37.700 | INFO | MemInfo: { | 20260506 23:08:37.700 | INFO | "gpu" : " 17 GB", | 20260506 23:08:37.700 | INFO | "huge" : "", | 20260506 23:08:37.700 | INFO | "huge_shared" : " 27 GB", | 20260506 23:08:37.700 | INFO | "malloc" : " 14 GB", | 20260506 23:08:37.700 | INFO | "pinned" : " 29 GB" | 20260506 23:08:37.700 | INFO | } | 20260506 23:08:37.700 | INFO | Perf report for the last 5.13135 seconds: | 20260506 23:08:37.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:08:37.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:37.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:37.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:37.700 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260506 23:08:37.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:37.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:37.700 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:08:37.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:37.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:37.701 | INFO | Uploaded 36GiB to GPU in 2.00654s (17.9413GiB/s) | 20260506 23:08:37.701 | INFO | Downloaded 34GiB to GPU in 1.93425s (17.5779GiB/s) | 20260506 23:08:42.836 | INFO | Analyzing chunk frames = [5182976,5183488) | 20260506 23:08:42.836 | INFO | MemInfo: { | 20260506 23:08:42.836 | INFO | "gpu" : " 17 GB", | 20260506 23:08:42.836 | INFO | "huge" : "", | 20260506 23:08:42.836 | INFO | "huge_shared" : " 27 GB", | 20260506 23:08:42.836 | INFO | "malloc" : " 14 GB", | 20260506 23:08:42.836 | INFO | "pinned" : " 29 GB" | 20260506 23:08:42.836 | INFO | } | 20260506 23:08:42.836 | INFO | Perf report for the last 5.13596 seconds: | 20260506 23:08:42.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:08:42.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:42.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:42.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:42.836 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260506 23:08:42.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:42.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:42.836 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:08:42.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:42.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:42.837 | INFO | Uploaded 36GiB to GPU in 2.01328s (17.8813GiB/s) | 20260506 23:08:42.837 | INFO | Downloaded 34GiB to GPU in 1.90756s (17.8238GiB/s) | 20260506 23:08:47.974 | INFO | Analyzing chunk frames = [5183488,5184000) | 20260506 23:08:47.974 | INFO | MemInfo: { | 20260506 23:08:47.974 | INFO | "gpu" : " 17 GB", | 20260506 23:08:47.975 | INFO | "huge" : "", | 20260506 23:08:47.975 | INFO | "huge_shared" : " 27 GB", | 20260506 23:08:47.975 | INFO | "malloc" : " 14 GB", | 20260506 23:08:47.975 | INFO | "pinned" : " 29 GB" | 20260506 23:08:47.975 | INFO | } | 20260506 23:08:47.975 | INFO | Perf report for the last 5.13873 seconds: | 20260506 23:08:47.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:08:47.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:47.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:47.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:47.975 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260506 23:08:47.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:47.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:47.975 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:08:47.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:47.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:47.975 | INFO | Uploaded 36GiB to GPU in 2.00875s (17.9216GiB/s) | 20260506 23:08:47.975 | INFO | Downloaded 34GiB to GPU in 1.91786s (17.7281GiB/s) | 20260506 23:08:53.122 | INFO | Analyzing chunk frames = [5184000,5184512) | 20260506 23:08:53.122 | INFO | MemInfo: { | 20260506 23:08:53.122 | INFO | "gpu" : " 17 GB", | 20260506 23:08:53.122 | INFO | "huge" : "", | 20260506 23:08:53.122 | INFO | "huge_shared" : " 27 GB", | 20260506 23:08:53.122 | INFO | "malloc" : " 14 GB", | 20260506 23:08:53.122 | INFO | "pinned" : " 29 GB" | 20260506 23:08:53.122 | INFO | } | 20260506 23:08:53.122 | INFO | Perf report for the last 5.14699 seconds: | 20260506 23:08:53.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:08:53.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:53.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:53.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:53.122 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260506 23:08:53.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:53.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:53.122 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:08:53.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:53.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:53.122 | INFO | Uploaded 36GiB to GPU in 2.01303s (17.8835GiB/s) | 20260506 23:08:53.122 | INFO | Downloaded 34GiB to GPU in 1.94452s (17.485GiB/s) | 20260506 23:08:58.261 | INFO | Analyzing chunk frames = [5184512,5185024) | 20260506 23:08:58.261 | INFO | MemInfo: { | 20260506 23:08:58.261 | INFO | "gpu" : " 17 GB", | 20260506 23:08:58.262 | INFO | "huge" : "", | 20260506 23:08:58.262 | INFO | "huge_shared" : " 27 GB", | 20260506 23:08:58.262 | INFO | "malloc" : " 14 GB", | 20260506 23:08:58.262 | INFO | "pinned" : " 29 GB" | 20260506 23:08:58.262 | INFO | } | 20260506 23:08:58.262 | INFO | Perf report for the last 5.13969 seconds: | 20260506 23:08:58.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:08:58.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:58.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:58.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:58.262 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:08:58.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:58.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:58.262 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:08:58.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:58.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:08:58.262 | INFO | Uploaded 36GiB to GPU in 1.97891s (18.1919GiB/s) | 20260506 23:08:58.262 | INFO | Downloaded 34GiB to GPU in 2.00406s (16.9656GiB/s) | 20260506 23:09:03.403 | INFO | Analyzing chunk frames = [5185024,5185536) | 20260506 23:09:03.403 | INFO | MemInfo: { | 20260506 23:09:03.403 | INFO | "gpu" : " 17 GB", | 20260506 23:09:03.403 | INFO | "huge" : "", | 20260506 23:09:03.403 | INFO | "huge_shared" : " 27 GB", | 20260506 23:09:03.403 | INFO | "malloc" : " 14 GB", | 20260506 23:09:03.403 | INFO | "pinned" : " 29 GB" | 20260506 23:09:03.403 | INFO | } | 20260506 23:09:03.403 | INFO | Perf report for the last 5.14146 seconds: | 20260506 23:09:03.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:09:03.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:03.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:03.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:03.403 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:09:03.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:03.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:03.403 | INFO | PRE_HQ : 30.6%, 1.0, 15.9ms, 0 | 20260506 23:09:03.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:03.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:03.403 | INFO | Uploaded 36GiB to GPU in 1.97746s (18.2051GiB/s) | 20260506 23:09:03.404 | INFO | Downloaded 34GiB to GPU in 2.00414s (16.9649GiB/s) | 20260506 23:09:03.404 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:09:03.404 | INFO | Total mem usage: 18 GiB | 20260506 23:09:03.404 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:09:03.404 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -25.3%) | 20260506 23:09:03.404 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:09:03.404 | INFO | High Water Breakdown: | 20260506 23:09:03.404 | INFO | 60.2% : 8.66 GiB ( 75.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:09:03.404 | INFO | 39.6% : 5.69 GiB ( 85.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:09:03.404 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:09:03.404 | INFO | ------------------------------------------ | 20260506 23:09:03.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:09:03.404 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:09:03.404 | INFO | Total mem usage: 26.5 GiB | 20260506 23:09:03.404 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:09:03.404 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:09:03.404 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:09:03.404 | INFO | High Water Breakdown: | 20260506 23:09:03.404 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:09:03.404 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:09:03.404 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:09:03.404 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:09:03.404 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:09:03.404 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:09:03.404 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:09:03.404 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:09:03.404 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:09:03.404 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:09:03.404 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:09:03.404 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:09:03.404 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:09:03.404 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:09:03.404 | INFO | ------------------------------------------ | 20260506 23:09:03.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:09:03.404 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:09:03.404 | INFO | Total mem usage: 25.6 GiB | 20260506 23:09:03.404 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:09:03.404 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:09:03.404 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:09:03.404 | INFO | High Water Breakdown: | 20260506 23:09:03.404 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:09:03.404 | INFO | ------------------------------------------ | 20260506 23:09:03.404 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:09:03.404 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:09:03.404 | INFO | Total mem usage: 15.7 GiB | 20260506 23:09:03.404 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:09:03.404 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:09:03.404 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:09:03.404 | INFO | High Water Breakdown: | 20260506 23:09:03.404 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:09:03.404 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:09:03.404 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:09:03.404 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:09:03.404 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:09:03.404 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:09:03.404 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:09:03.404 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:09:03.404 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:09:03.404 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:09:03.404 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:09:03.404 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:09:03.404 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:09:03.404 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:09:03.404 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:09:03.404 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:09:03.404 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:09:03.404 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:09:03.404 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:09:03.404 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:09:03.404 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:09:03.405 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:09:03.405 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:09:03.405 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:09:03.405 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:09:03.405 | INFO | ------------------------------------------ | 20260506 23:09:03.405 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:09:03.405 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:09:03.405 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:09:03.405 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:09:03.405 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.64e+01, 1.36e+01 / 2.44e+00 | 20260506 23:09:03.405 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.04e-01, 9.76e-03 / 5.78e-03 | 20260506 23:09:03.405 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.43e+00, 5.44e-01 / 2.65e-01 | 20260506 23:09:03.405 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.47e+00, 1.79e-01 / 1.11e-01 | 20260506 23:09:03.405 | INFO | PerfReport: 16.04%, DME -- 3.98e+01 / 4.46e+01, 3.94e+00 / 2.54e+00 | 20260506 23:09:03.405 | INFO | PerfReport: 57.36%, FrameLabeling -- 1.78e+01 / 2.14e+01, 1.37e+00 / 1.50e+00 | 20260506 23:09:03.405 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.20e+00 / 2.69e+00, 3.80e-01 / 2.10e-01 | 20260506 23:09:03.405 | INFO | PerfReport: 3.21%, Metrics -- 9.95e-01 / 1.33e+00, 2.63e-01 / 1.75e-01 | 20260506 23:09:03.405 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.39e-01 / 3.89e-01, 5.24e-02 / 1.97e-02 | 20260506 23:09:03.405 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:09:03.405 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:09:03.405 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:09:03.405 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 23:09:03.405 | INFO | PerfReport: 21.54%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 23:09:03.405 | INFO | PerfReport: 21.31%, Download -- 2.06e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260506 23:09:03.405 | INFO | PerfReport: 32.08%, Compute -- 3.10e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 23:09:03.405 | INFO | PerfReport: 1.25%, LockUpload -- 1.21e+00 / 0.00e+00, 4.54e+00 / 0.00e+00 | 20260506 23:09:03.405 | INFO | PerfReport: 1.26%, LockDownload -- 1.22e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260506 23:09:03.405 | INFO | PerfReport: 22.55%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:09:03.405 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260506 23:09:03.502 | INFO | Starting a round of Baz IO | 20260506 23:09:06.626 | INFO | Populating RTMetrics report | 20260506 23:09:06.627 | INFO | RTMetrics report populated | 20260506 23:09:06.627 | INFO | Opening tmp RTMetrics json file | 20260506 23:09:06.628 | INFO | Serializing RTMetrics to json | 20260506 23:09:06.629 | INFO | Serialization done, swapping tmp file now... | 20260506 23:09:06.629 | INFO | RTMetrics done | 20260506 23:09:06.719 | INFO | Finished a round of Baz IO: Wrote 1.892 GiB in 3.216 seconds | 20260506 23:09:06.719 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.53% / 0.00% / 66.47% | 20260506 23:09:06.719 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.443188% / 7.94729e-06% / 5.16574e-05% | 20260506 23:09:06.719 | INFO | Baz IO overall: | 20260506 23:09:06.719 | INFO | BytesWritten : 2717484889705 | 20260506 23:09:06.719 | INFO | NumEvents : 1310489662825 | 20260506 23:09:08.543 | INFO | Analyzing chunk frames = [5185536,5186048) | 20260506 23:09:08.543 | INFO | MemInfo: { | 20260506 23:09:08.543 | INFO | "gpu" : " 17 GB", | 20260506 23:09:08.543 | INFO | "huge" : "", | 20260506 23:09:08.543 | INFO | "huge_shared" : " 27 GB", | 20260506 23:09:08.543 | INFO | "malloc" : " 14 GB", | 20260506 23:09:08.543 | INFO | "pinned" : " 29 GB" | 20260506 23:09:08.543 | INFO | } | 20260506 23:09:08.543 | INFO | Perf report for the last 5.13978 seconds: | 20260506 23:09:08.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:09:08.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:08.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:08.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:08.543 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 23:09:08.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:08.543 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 23:09:08.543 | INFO | PRE_HQ : 59.9%, 1.8, 57.1ms, 0 | 20260506 23:09:08.543 | INFO | BAZWRITER : 62.2%, 3.5, 111.0ms, 0 | 20260506 23:09:08.543 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:09:08.543 | INFO | Uploaded 36GiB to GPU in 2.04517s (17.6025GiB/s) | 20260506 23:09:08.543 | INFO | Downloaded 38GiB to GPU in 2.18331s (17.4048GiB/s) | 20260506 23:09:13.678 | INFO | Analyzing chunk frames = [5186048,5186560) | 20260506 23:09:13.678 | INFO | MemInfo: { | 20260506 23:09:13.678 | INFO | "gpu" : " 17 GB", | 20260506 23:09:13.678 | INFO | "huge" : "", | 20260506 23:09:13.678 | INFO | "huge_shared" : " 27 GB", | 20260506 23:09:13.678 | INFO | "malloc" : " 14 GB", | 20260506 23:09:13.678 | INFO | "pinned" : " 29 GB" | 20260506 23:09:13.678 | INFO | } | 20260506 23:09:13.678 | INFO | Perf report for the last 5.13501 seconds: | 20260506 23:09:13.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:09:13.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:13.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:13.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:13.678 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:09:13.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:13.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:13.678 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:09:13.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:13.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:13.678 | INFO | Uploaded 36GiB to GPU in 2.00342s (17.9693GiB/s) | 20260506 23:09:13.678 | INFO | Downloaded 34GiB to GPU in 1.92468s (17.6652GiB/s) | 20260506 23:09:18.816 | INFO | Analyzing chunk frames = [5186560,5187072) | 20260506 23:09:18.816 | INFO | MemInfo: { | 20260506 23:09:18.816 | INFO | "gpu" : " 17 GB", | 20260506 23:09:18.816 | INFO | "huge" : "", | 20260506 23:09:18.816 | INFO | "huge_shared" : " 27 GB", | 20260506 23:09:18.816 | INFO | "malloc" : " 14 GB", | 20260506 23:09:18.816 | INFO | "pinned" : " 29 GB" | 20260506 23:09:18.816 | INFO | } | 20260506 23:09:18.816 | INFO | Perf report for the last 5.13781 seconds: | 20260506 23:09:18.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:09:18.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:18.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:18.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:18.816 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 23:09:18.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:18.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:18.816 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:09:18.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:18.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:18.816 | INFO | Uploaded 36GiB to GPU in 1.99038s (18.087GiB/s) | 20260506 23:09:18.816 | INFO | Downloaded 34GiB to GPU in 1.9309s (17.6084GiB/s) | 20260506 23:09:23.950 | INFO | Analyzing chunk frames = [5187072,5187584) | 20260506 23:09:23.950 | INFO | MemInfo: { | 20260506 23:09:23.950 | INFO | "gpu" : " 17 GB", | 20260506 23:09:23.950 | INFO | "huge" : "", | 20260506 23:09:23.950 | INFO | "huge_shared" : " 27 GB", | 20260506 23:09:23.950 | INFO | "malloc" : " 14 GB", | 20260506 23:09:23.950 | INFO | "pinned" : " 29 GB" | 20260506 23:09:23.951 | INFO | } | 20260506 23:09:23.951 | INFO | Perf report for the last 5.13472 seconds: | 20260506 23:09:23.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:09:23.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:23.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:23.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:23.951 | INFO | ANALYSIS : 59.7%, 3.0, 92.4ms, 0 | 20260506 23:09:23.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:23.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:23.951 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:09:23.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:23.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:23.951 | INFO | Uploaded 36GiB to GPU in 1.9775s (18.2048GiB/s) | 20260506 23:09:23.951 | INFO | Downloaded 34GiB to GPU in 1.87786s (18.1057GiB/s) | 20260506 23:09:29.092 | INFO | Analyzing chunk frames = [5187584,5188096) | 20260506 23:09:29.092 | INFO | MemInfo: { | 20260506 23:09:29.092 | INFO | "gpu" : " 17 GB", | 20260506 23:09:29.092 | INFO | "huge" : "", | 20260506 23:09:29.092 | INFO | "huge_shared" : " 27 GB", | 20260506 23:09:29.092 | INFO | "malloc" : " 14 GB", | 20260506 23:09:29.092 | INFO | "pinned" : " 29 GB" | 20260506 23:09:29.092 | INFO | } | 20260506 23:09:29.092 | INFO | Perf report for the last 5.1416 seconds: | 20260506 23:09:29.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:09:29.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:29.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:29.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:29.092 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 23:09:29.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:29.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:29.092 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:09:29.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:29.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:29.092 | INFO | Uploaded 36GiB to GPU in 2.00278s (17.975GiB/s) | 20260506 23:09:29.092 | INFO | Downloaded 34GiB to GPU in 1.93212s (17.5973GiB/s) | 20260506 23:09:34.238 | INFO | Analyzing chunk frames = [5188096,5188608) | 20260506 23:09:34.238 | INFO | MemInfo: { | 20260506 23:09:34.238 | INFO | "gpu" : " 17 GB", | 20260506 23:09:34.238 | INFO | "huge" : "", | 20260506 23:09:34.238 | INFO | "huge_shared" : " 27 GB", | 20260506 23:09:34.239 | INFO | "malloc" : " 14 GB", | 20260506 23:09:34.239 | INFO | "pinned" : " 29 GB" | 20260506 23:09:34.239 | INFO | } | 20260506 23:09:34.239 | INFO | Perf report for the last 5.14651 seconds: | 20260506 23:09:34.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:09:34.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:34.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:34.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:34.239 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260506 23:09:34.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:34.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:34.239 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:09:34.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:34.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:34.239 | INFO | Uploaded 36GiB to GPU in 2.0138s (17.8766GiB/s) | 20260506 23:09:34.239 | INFO | Downloaded 34GiB to GPU in 1.94279s (17.5006GiB/s) | 20260506 23:09:39.375 | INFO | Analyzing chunk frames = [5188608,5189120) | 20260506 23:09:39.375 | INFO | MemInfo: { | 20260506 23:09:39.375 | INFO | "gpu" : " 17 GB", | 20260506 23:09:39.375 | INFO | "huge" : "", | 20260506 23:09:39.375 | INFO | "huge_shared" : " 27 GB", | 20260506 23:09:39.375 | INFO | "malloc" : " 14 GB", | 20260506 23:09:39.375 | INFO | "pinned" : " 29 GB" | 20260506 23:09:39.375 | INFO | } | 20260506 23:09:39.375 | INFO | Perf report for the last 5.13607 seconds: | 20260506 23:09:39.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:09:39.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:39.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:39.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:39.375 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 23:09:39.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:39.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:39.375 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 23:09:39.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:39.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:39.375 | INFO | Uploaded 36GiB to GPU in 1.99657s (18.0309GiB/s) | 20260506 23:09:39.375 | INFO | Downloaded 34GiB to GPU in 1.99377s (17.0531GiB/s) | 20260506 23:09:44.516 | INFO | Analyzing chunk frames = [5189120,5189632) | 20260506 23:09:44.516 | INFO | MemInfo: { | 20260506 23:09:44.516 | INFO | "gpu" : " 17 GB", | 20260506 23:09:44.516 | INFO | "huge" : "", | 20260506 23:09:44.516 | INFO | "huge_shared" : " 27 GB", | 20260506 23:09:44.516 | INFO | "malloc" : " 14 GB", | 20260506 23:09:44.516 | INFO | "pinned" : " 29 GB" | 20260506 23:09:44.516 | INFO | } | 20260506 23:09:44.516 | INFO | Perf report for the last 5.1414 seconds: | 20260506 23:09:44.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:09:44.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:44.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:44.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:44.516 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 23:09:44.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:44.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:44.516 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:09:44.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:44.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:44.517 | INFO | Uploaded 36GiB to GPU in 1.97857s (18.195GiB/s) | 20260506 23:09:44.517 | INFO | Downloaded 34GiB to GPU in 1.99457s (17.0463GiB/s) | 20260506 23:09:44.614 | INFO | Starting a round of Baz IO | 20260506 23:09:47.742 | INFO | Populating RTMetrics report | 20260506 23:09:47.742 | INFO | RTMetrics report populated | 20260506 23:09:47.742 | INFO | Opening tmp RTMetrics json file | 20260506 23:09:47.743 | INFO | Serializing RTMetrics to json | 20260506 23:09:47.744 | INFO | Serialization done, swapping tmp file now... | 20260506 23:09:47.744 | INFO | RTMetrics done | 20260506 23:09:47.836 | INFO | Finished a round of Baz IO: Wrote 1.889 GiB in 3.221 seconds | 20260506 23:09:47.836 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.52% / 0.00% / 66.47% | 20260506 23:09:47.836 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.439147% / 0% / 6.35783e-05% | 20260506 23:09:47.836 | INFO | Baz IO overall: | 20260506 23:09:47.836 | INFO | BytesWritten : 2719513350080 | 20260506 23:09:47.836 | INFO | NumEvents : 1311456684358 | 20260506 23:09:49.651 | INFO | Analyzing chunk frames = [5189632,5190144) | 20260506 23:09:49.651 | INFO | MemInfo: { | 20260506 23:09:49.651 | INFO | "gpu" : " 17 GB", | 20260506 23:09:49.651 | INFO | "huge" : "", | 20260506 23:09:49.651 | INFO | "huge_shared" : " 27 GB", | 20260506 23:09:49.651 | INFO | "malloc" : " 14 GB", | 20260506 23:09:49.651 | INFO | "pinned" : " 29 GB" | 20260506 23:09:49.651 | INFO | } | 20260506 23:09:49.651 | INFO | Perf report for the last 5.1351 seconds: | 20260506 23:09:49.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:09:49.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:49.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:49.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:49.651 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260506 23:09:49.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:49.651 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260506 23:09:49.651 | INFO | PRE_HQ : 59.8%, 1.8, 56.7ms, 0 | 20260506 23:09:49.651 | INFO | BAZWRITER : 62.3%, 3.4, 109.7ms, 0 | 20260506 23:09:49.652 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:09:49.652 | INFO | Uploaded 36GiB to GPU in 2.02159s (17.8078GiB/s) | 20260506 23:09:49.652 | INFO | Downloaded 38GiB to GPU in 2.1717s (17.4978GiB/s) | 20260506 23:09:54.789 | INFO | Analyzing chunk frames = [5190144,5190656) | 20260506 23:09:54.789 | INFO | MemInfo: { | 20260506 23:09:54.789 | INFO | "gpu" : " 17 GB", | 20260506 23:09:54.789 | INFO | "huge" : "", | 20260506 23:09:54.789 | INFO | "huge_shared" : " 27 GB", | 20260506 23:09:54.789 | INFO | "malloc" : " 14 GB", | 20260506 23:09:54.789 | INFO | "pinned" : " 29 GB" | 20260506 23:09:54.789 | INFO | } | 20260506 23:09:54.790 | INFO | Perf report for the last 5.13816 seconds: | 20260506 23:09:54.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:09:54.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:54.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:54.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:54.790 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:09:54.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:54.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:54.790 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:09:54.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:54.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:54.790 | INFO | Uploaded 36GiB to GPU in 1.99446s (18.05GiB/s) | 20260506 23:09:54.790 | INFO | Downloaded 34GiB to GPU in 1.93406s (17.5796GiB/s) | 20260506 23:09:59.926 | INFO | Analyzing chunk frames = [5190656,5191168) | 20260506 23:09:59.926 | INFO | MemInfo: { | 20260506 23:09:59.926 | INFO | "gpu" : " 17 GB", | 20260506 23:09:59.926 | INFO | "huge" : "", | 20260506 23:09:59.926 | INFO | "huge_shared" : " 27 GB", | 20260506 23:09:59.926 | INFO | "malloc" : " 14 GB", | 20260506 23:09:59.926 | INFO | "pinned" : " 29 GB" | 20260506 23:09:59.926 | INFO | } | 20260506 23:09:59.926 | INFO | Perf report for the last 5.13652 seconds: | 20260506 23:09:59.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:09:59.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:59.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:59.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:59.926 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260506 23:09:59.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:59.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:59.926 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:09:59.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:59.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:09:59.926 | INFO | Uploaded 36GiB to GPU in 1.97532s (18.2249GiB/s) | 20260506 23:09:59.926 | INFO | Downloaded 34GiB to GPU in 1.92352s (17.676GiB/s) | 20260506 23:10:05.072 | INFO | Analyzing chunk frames = [5191168,5191680) | 20260506 23:10:05.072 | INFO | MemInfo: { | 20260506 23:10:05.072 | INFO | "gpu" : " 17 GB", | 20260506 23:10:05.072 | INFO | "huge" : "", | 20260506 23:10:05.072 | INFO | "huge_shared" : " 27 GB", | 20260506 23:10:05.072 | INFO | "malloc" : " 14 GB", | 20260506 23:10:05.072 | INFO | "pinned" : " 29 GB" | 20260506 23:10:05.072 | INFO | } | 20260506 23:10:05.072 | INFO | Perf report for the last 5.14595 seconds: | 20260506 23:10:05.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:10:05.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:05.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:05.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:05.072 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 23:10:05.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:05.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:05.072 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 23:10:05.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:05.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:05.072 | INFO | Uploaded 36GiB to GPU in 1.93764s (18.5793GiB/s) | 20260506 23:10:05.072 | INFO | Downloaded 34GiB to GPU in 1.87604s (18.1233GiB/s) | 20260506 23:10:10.213 | INFO | Analyzing chunk frames = [5191680,5192192) | 20260506 23:10:10.213 | INFO | MemInfo: { | 20260506 23:10:10.213 | INFO | "gpu" : " 17 GB", | 20260506 23:10:10.213 | INFO | "huge" : "", | 20260506 23:10:10.213 | INFO | "huge_shared" : " 27 GB", | 20260506 23:10:10.213 | INFO | "malloc" : " 14 GB", | 20260506 23:10:10.213 | INFO | "pinned" : " 29 GB" | 20260506 23:10:10.213 | INFO | } | 20260506 23:10:10.213 | INFO | Perf report for the last 5.14133 seconds: | 20260506 23:10:10.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:10:10.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:10.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:10.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:10.213 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260506 23:10:10.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:10.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:10.213 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 23:10:10.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:10.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:10.214 | INFO | Uploaded 36GiB to GPU in 1.9696s (18.2778GiB/s) | 20260506 23:10:10.214 | INFO | Downloaded 34GiB to GPU in 1.92858s (17.6296GiB/s) | 20260506 23:10:15.349 | INFO | Analyzing chunk frames = [5192192,5192704) | 20260506 23:10:15.349 | INFO | MemInfo: { | 20260506 23:10:15.349 | INFO | "gpu" : " 17 GB", | 20260506 23:10:15.349 | INFO | "huge" : "", | 20260506 23:10:15.349 | INFO | "huge_shared" : " 27 GB", | 20260506 23:10:15.349 | INFO | "malloc" : " 14 GB", | 20260506 23:10:15.349 | INFO | "pinned" : " 29 GB" | 20260506 23:10:15.349 | INFO | } | 20260506 23:10:15.349 | INFO | Perf report for the last 5.13579 seconds: | 20260506 23:10:15.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:10:15.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:15.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:15.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:15.349 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260506 23:10:15.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:15.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:15.349 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 23:10:15.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:15.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:15.349 | INFO | Uploaded 36GiB to GPU in 1.99694s (18.0276GiB/s) | 20260506 23:10:15.349 | INFO | Downloaded 34GiB to GPU in 1.93542s (17.5673GiB/s) | 20260506 23:10:20.492 | INFO | Analyzing chunk frames = [5192704,5193216) | 20260506 23:10:20.492 | INFO | MemInfo: { | 20260506 23:10:20.492 | INFO | "gpu" : " 17 GB", | 20260506 23:10:20.492 | INFO | "huge" : "", | 20260506 23:10:20.492 | INFO | "huge_shared" : " 27 GB", | 20260506 23:10:20.492 | INFO | "malloc" : " 14 GB", | 20260506 23:10:20.492 | INFO | "pinned" : " 29 GB" | 20260506 23:10:20.493 | INFO | } | 20260506 23:10:20.493 | INFO | Perf report for the last 5.14358 seconds: | 20260506 23:10:20.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:10:20.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:20.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:20.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:20.493 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 23:10:20.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:20.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:20.493 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 23:10:20.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:20.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:20.493 | INFO | Uploaded 36GiB to GPU in 1.9635s (18.3346GiB/s) | 20260506 23:10:20.493 | INFO | Downloaded 34GiB to GPU in 1.99119s (17.0752GiB/s) | 20260506 23:10:25.629 | INFO | Analyzing chunk frames = [5193216,5193728) | 20260506 23:10:25.629 | INFO | MemInfo: { | 20260506 23:10:25.629 | INFO | "gpu" : " 17 GB", | 20260506 23:10:25.629 | INFO | "huge" : "", | 20260506 23:10:25.629 | INFO | "huge_shared" : " 27 GB", | 20260506 23:10:25.629 | INFO | "malloc" : " 14 GB", | 20260506 23:10:25.629 | INFO | "pinned" : " 29 GB" | 20260506 23:10:25.629 | INFO | } | 20260506 23:10:25.630 | INFO | Perf report for the last 5.13681 seconds: | 20260506 23:10:25.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:10:25.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:25.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:25.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:25.630 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 23:10:25.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:25.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:25.630 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 23:10:25.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:25.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:25.630 | INFO | Uploaded 36GiB to GPU in 1.95827s (18.3836GiB/s) | 20260506 23:10:25.630 | INFO | Downloaded 34GiB to GPU in 1.99607s (17.0335GiB/s) | 20260506 23:10:25.630 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:10:25.630 | INFO | Total mem usage: 18 GiB | 20260506 23:10:25.630 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:10:25.630 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 23:10:25.630 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:10:25.630 | INFO | High Water Breakdown: | 20260506 23:10:25.630 | INFO | 60.1% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:10:25.630 | INFO | 39.7% : 5.67 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:10:25.630 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:10:25.630 | INFO | ------------------------------------------ | 20260506 23:10:25.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:10:25.630 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:10:25.630 | INFO | Total mem usage: 26.5 GiB | 20260506 23:10:25.630 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:10:25.630 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:10:25.630 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:10:25.630 | INFO | High Water Breakdown: | 20260506 23:10:25.630 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:10:25.630 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:10:25.630 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:10:25.630 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:10:25.630 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:10:25.630 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:10:25.630 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:10:25.630 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:10:25.630 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:10:25.631 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:10:25.631 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:10:25.631 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:10:25.631 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:10:25.631 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:10:25.631 | INFO | ------------------------------------------ | 20260506 23:10:25.631 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:10:25.631 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:10:25.631 | INFO | Total mem usage: 25.6 GiB | 20260506 23:10:25.631 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:10:25.631 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:10:25.631 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:10:25.631 | INFO | High Water Breakdown: | 20260506 23:10:25.631 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:10:25.631 | INFO | ------------------------------------------ | 20260506 23:10:25.631 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:10:25.631 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:10:25.631 | INFO | Total mem usage: 15.7 GiB | 20260506 23:10:25.631 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:10:25.631 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:10:25.631 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:10:25.631 | INFO | High Water Breakdown: | 20260506 23:10:25.631 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:10:25.631 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:10:25.631 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:10:25.631 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:10:25.631 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:10:25.631 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:10:25.631 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:10:25.631 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:10:25.631 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:10:25.631 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:10:25.631 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:10:25.631 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:10:25.631 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:10:25.631 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:10:25.631 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:10:25.631 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:10:25.631 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:10:25.631 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:10:25.631 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:10:25.631 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:10:25.631 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:10:25.631 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:10:25.631 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:10:25.631 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:10:25.631 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:10:25.631 | INFO | ------------------------------------------ | 20260506 23:10:25.631 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:10:25.631 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:10:25.631 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:10:25.631 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:10:25.631 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.52e+01, 1.34e+01 / 2.24e+00 | 20260506 23:10:25.631 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.65e-03 / 5.46e-03 | 20260506 23:10:25.631 | INFO | PerfReport: 11.65%, Baseline -- 3.61e+00 / 4.41e+00, 5.42e-01 / 3.10e-01 | 20260506 23:10:25.631 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.45e+00, 1.78e-01 / 1.36e-01 | 20260506 23:10:25.631 | INFO | PerfReport: 16.04%, DME -- 3.98e+01 / 4.43e+01, 3.87e+00 / 2.44e+00 | 20260506 23:10:25.631 | INFO | PerfReport: 57.38%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.34e+00 / 1.70e+00 | 20260506 23:10:25.631 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.20e+00 / 2.77e+00, 3.86e-01 / 1.90e-01 | 20260506 23:10:25.631 | INFO | PerfReport: 3.20%, Metrics -- 9.92e-01 / 1.35e+00, 2.62e-01 / 1.59e-01 | 20260506 23:10:25.631 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.39e-01 / 3.97e-01, 5.26e-02 / 2.95e-02 | 20260506 23:10:25.631 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:10:25.631 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:10:25.631 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:10:25.631 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 23:10:25.631 | INFO | PerfReport: 21.47%, Upload -- 2.07e+01 / 0.00e+00, 6.90e+00 / 0.00e+00 | 20260506 23:10:25.631 | INFO | PerfReport: 21.28%, Download -- 2.05e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 23:10:25.631 | INFO | PerfReport: 32.11%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 23:10:25.631 | INFO | PerfReport: 1.22%, LockUpload -- 1.17e+00 / 0.00e+00, 4.65e+00 / 0.00e+00 | 20260506 23:10:25.631 | INFO | PerfReport: 1.29%, LockDownload -- 1.25e+00 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260506 23:10:25.631 | INFO | PerfReport: 22.62%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:10:25.631 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260506 23:10:25.728 | INFO | Starting a round of Baz IO | 20260506 23:10:28.862 | INFO | Populating RTMetrics report | 20260506 23:10:28.862 | INFO | RTMetrics report populated | 20260506 23:10:28.862 | INFO | Opening tmp RTMetrics json file | 20260506 23:10:28.863 | INFO | Serializing RTMetrics to json | 20260506 23:10:28.864 | INFO | Serialization done, swapping tmp file now... | 20260506 23:10:28.864 | INFO | RTMetrics done | 20260506 23:10:28.957 | INFO | Finished a round of Baz IO: Wrote 1.89 GiB in 3.228 seconds | 20260506 23:10:28.957 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.52% / 0.00% / 66.48% | 20260506 23:10:28.957 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.438948% / 7.94729e-06% / 3.57628e-05% | 20260506 23:10:28.957 | INFO | Baz IO overall: | 20260506 23:10:28.957 | INFO | BytesWritten : 2721543038996 | 20260506 23:10:28.957 | INFO | NumEvents : 1312424791553 | 20260506 23:10:30.761 | INFO | Analyzing chunk frames = [5193728,5194240) | 20260506 23:10:30.761 | INFO | MemInfo: { | 20260506 23:10:30.761 | INFO | "gpu" : " 17 GB", | 20260506 23:10:30.761 | INFO | "huge" : "", | 20260506 23:10:30.761 | INFO | "huge_shared" : " 27 GB", | 20260506 23:10:30.761 | INFO | "malloc" : " 14 GB", | 20260506 23:10:30.761 | INFO | "pinned" : " 29 GB" | 20260506 23:10:30.761 | INFO | } | 20260506 23:10:30.761 | INFO | Perf report for the last 5.13171 seconds: | 20260506 23:10:30.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:10:30.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:30.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:30.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:30.761 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260506 23:10:30.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:30.761 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 23:10:30.761 | INFO | PRE_HQ : 60.0%, 1.8, 57.2ms, 0 | 20260506 23:10:30.761 | INFO | BAZWRITER : 62.5%, 3.4, 109.1ms, 0 | 20260506 23:10:30.761 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:10:30.762 | INFO | Uploaded 36GiB to GPU in 2.01308s (17.8831GiB/s) | 20260506 23:10:30.762 | INFO | Downloaded 38GiB to GPU in 2.18778s (17.3692GiB/s) | 20260506 23:10:35.903 | INFO | Analyzing chunk frames = [5194240,5194752) | 20260506 23:10:35.903 | INFO | MemInfo: { | 20260506 23:10:35.903 | INFO | "gpu" : " 17 GB", | 20260506 23:10:35.903 | INFO | "huge" : "", | 20260506 23:10:35.903 | INFO | "huge_shared" : " 27 GB", | 20260506 23:10:35.904 | INFO | "malloc" : " 14 GB", | 20260506 23:10:35.904 | INFO | "pinned" : " 29 GB" | 20260506 23:10:35.904 | INFO | } | 20260506 23:10:35.904 | INFO | Perf report for the last 5.14245 seconds: | 20260506 23:10:35.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:10:35.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:35.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:35.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:35.904 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:10:35.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:35.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:35.904 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:10:35.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:35.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:35.904 | INFO | Uploaded 36GiB to GPU in 1.98814s (18.1074GiB/s) | 20260506 23:10:35.904 | INFO | Downloaded 34GiB to GPU in 1.92815s (17.6335GiB/s) | 20260506 23:10:41.042 | INFO | Analyzing chunk frames = [5194752,5195264) | 20260506 23:10:41.042 | INFO | MemInfo: { | 20260506 23:10:41.042 | INFO | "gpu" : " 17 GB", | 20260506 23:10:41.042 | INFO | "huge" : "", | 20260506 23:10:41.042 | INFO | "huge_shared" : " 27 GB", | 20260506 23:10:41.042 | INFO | "malloc" : " 14 GB", | 20260506 23:10:41.042 | INFO | "pinned" : " 29 GB" | 20260506 23:10:41.042 | INFO | } | 20260506 23:10:41.042 | INFO | Perf report for the last 5.1384 seconds: | 20260506 23:10:41.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:10:41.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:41.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:41.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:41.042 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 23:10:41.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:41.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:41.042 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:10:41.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:41.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:41.042 | INFO | Uploaded 36GiB to GPU in 1.9688s (18.2852GiB/s) | 20260506 23:10:41.042 | INFO | Downloaded 34GiB to GPU in 1.92726s (17.6417GiB/s) | 20260506 23:10:46.186 | INFO | Analyzing chunk frames = [5195264,5195776) | 20260506 23:10:46.186 | INFO | MemInfo: { | 20260506 23:10:46.186 | INFO | "gpu" : " 17 GB", | 20260506 23:10:46.186 | INFO | "huge" : "", | 20260506 23:10:46.186 | INFO | "huge_shared" : " 27 GB", | 20260506 23:10:46.186 | INFO | "malloc" : " 14 GB", | 20260506 23:10:46.186 | INFO | "pinned" : " 29 GB" | 20260506 23:10:46.186 | INFO | } | 20260506 23:10:46.186 | INFO | Perf report for the last 5.14434 seconds: | 20260506 23:10:46.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:10:46.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:46.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:46.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:46.187 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 23:10:46.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:46.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:46.187 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:10:46.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:46.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:46.187 | INFO | Uploaded 36GiB to GPU in 1.93793s (18.5765GiB/s) | 20260506 23:10:46.187 | INFO | Downloaded 34GiB to GPU in 1.88074s (18.078GiB/s) | 20260506 23:10:51.334 | INFO | Analyzing chunk frames = [5195776,5196288) | 20260506 23:10:51.334 | INFO | MemInfo: { | 20260506 23:10:51.334 | INFO | "gpu" : " 17 GB", | 20260506 23:10:51.334 | INFO | "huge" : "", | 20260506 23:10:51.334 | INFO | "huge_shared" : " 27 GB", | 20260506 23:10:51.334 | INFO | "malloc" : " 14 GB", | 20260506 23:10:51.334 | INFO | "pinned" : " 29 GB" | 20260506 23:10:51.334 | INFO | } | 20260506 23:10:51.334 | INFO | Perf report for the last 5.14789 seconds: | 20260506 23:10:51.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:10:51.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:51.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:51.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:51.334 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 23:10:51.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:51.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:51.334 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:10:51.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:51.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:51.335 | INFO | Uploaded 36GiB to GPU in 1.98519s (18.1343GiB/s) | 20260506 23:10:51.335 | INFO | Downloaded 34GiB to GPU in 1.92712s (17.6429GiB/s) | 20260506 23:10:56.467 | INFO | Analyzing chunk frames = [5196288,5196800) | 20260506 23:10:56.467 | INFO | MemInfo: { | 20260506 23:10:56.467 | INFO | "gpu" : " 17 GB", | 20260506 23:10:56.467 | INFO | "huge" : "", | 20260506 23:10:56.467 | INFO | "huge_shared" : " 27 GB", | 20260506 23:10:56.467 | INFO | "malloc" : " 14 GB", | 20260506 23:10:56.467 | INFO | "pinned" : " 29 GB" | 20260506 23:10:56.467 | INFO | } | 20260506 23:10:56.467 | INFO | Perf report for the last 5.13302 seconds: | 20260506 23:10:56.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:10:56.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:56.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:56.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:56.467 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 23:10:56.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:56.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:56.468 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 23:10:56.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:56.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:10:56.468 | INFO | Uploaded 36GiB to GPU in 2.05823s (17.4907GiB/s) | 20260506 23:10:56.468 | INFO | Downloaded 34GiB to GPU in 1.96484s (17.3042GiB/s) | 20260506 23:11:01.602 | INFO | Analyzing chunk frames = [5196800,5197312) | 20260506 23:11:01.602 | INFO | MemInfo: { | 20260506 23:11:01.602 | INFO | "gpu" : " 17 GB", | 20260506 23:11:01.602 | INFO | "huge" : "", | 20260506 23:11:01.602 | INFO | "huge_shared" : " 27 GB", | 20260506 23:11:01.602 | INFO | "malloc" : " 14 GB", | 20260506 23:11:01.602 | INFO | "pinned" : " 29 GB" | 20260506 23:11:01.602 | INFO | } | 20260506 23:11:01.602 | INFO | Perf report for the last 5.13493 seconds: | 20260506 23:11:01.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:11:01.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:01.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:01.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:01.602 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260506 23:11:01.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:01.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:01.602 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 23:11:01.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:01.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:01.603 | INFO | Uploaded 36GiB to GPU in 1.99887s (18.0102GiB/s) | 20260506 23:11:01.603 | INFO | Downloaded 34GiB to GPU in 2.0024s (16.9797GiB/s) | 20260506 23:11:06.738 | INFO | Analyzing chunk frames = [5197312,5197824) | 20260506 23:11:06.738 | INFO | MemInfo: { | 20260506 23:11:06.738 | INFO | "gpu" : " 17 GB", | 20260506 23:11:06.738 | INFO | "huge" : "", | 20260506 23:11:06.738 | INFO | "huge_shared" : " 27 GB", | 20260506 23:11:06.738 | INFO | "malloc" : " 14 GB", | 20260506 23:11:06.738 | INFO | "pinned" : " 29 GB" | 20260506 23:11:06.738 | INFO | } | 20260506 23:11:06.738 | INFO | Perf report for the last 5.13567 seconds: | 20260506 23:11:06.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:11:06.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:06.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:06.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:06.738 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 23:11:06.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:06.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:06.738 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260506 23:11:06.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:06.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:06.738 | INFO | Uploaded 36GiB to GPU in 2.01524s (17.8638GiB/s) | 20260506 23:11:06.738 | INFO | Downloaded 34GiB to GPU in 1.99105s (17.0764GiB/s) | 20260506 23:11:06.838 | INFO | Starting a round of Baz IO | 20260506 23:11:09.974 | INFO | Populating RTMetrics report | 20260506 23:11:09.974 | INFO | RTMetrics report populated | 20260506 23:11:09.974 | INFO | Opening tmp RTMetrics json file | 20260506 23:11:09.975 | INFO | Serializing RTMetrics to json | 20260506 23:11:09.976 | INFO | Serialization done, swapping tmp file now... | 20260506 23:11:09.977 | INFO | RTMetrics done | 20260506 23:11:10.071 | INFO | Finished a round of Baz IO: Wrote 1.89 GiB in 3.232 seconds | 20260506 23:11:10.071 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.52% / 0.00% / 66.48% | 20260506 23:11:10.071 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.438531% / 0% / 4.37101e-05% | 20260506 23:11:10.071 | INFO | Baz IO overall: | 20260506 23:11:10.071 | INFO | BytesWritten : 2723571973653 | 20260506 23:11:10.071 | INFO | NumEvents : 1313392626931 | 20260506 23:11:11.876 | INFO | Analyzing chunk frames = [5197824,5198336) | 20260506 23:11:11.876 | INFO | MemInfo: { | 20260506 23:11:11.876 | INFO | "gpu" : " 17 GB", | 20260506 23:11:11.876 | INFO | "huge" : "", | 20260506 23:11:11.877 | INFO | "huge_shared" : " 27 GB", | 20260506 23:11:11.877 | INFO | "malloc" : " 14 GB", | 20260506 23:11:11.877 | INFO | "pinned" : " 29 GB" | 20260506 23:11:11.877 | INFO | } | 20260506 23:11:11.877 | INFO | Perf report for the last 5.13865 seconds: | 20260506 23:11:11.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:11:11.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:11.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:11.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:11.877 | INFO | ANALYSIS : 62.4%, 3.0, 97.1ms, 0 | 20260506 23:11:11.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:11.877 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260506 23:11:11.877 | INFO | PRE_HQ : 60.0%, 1.8, 57.1ms, 0 | 20260506 23:11:11.877 | INFO | BAZWRITER : 62.5%, 3.4, 109.7ms, 0 | 20260506 23:11:11.877 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:11:11.877 | INFO | Uploaded 36GiB to GPU in 2.03556s (17.6855GiB/s) | 20260506 23:11:11.877 | INFO | Downloaded 38GiB to GPU in 2.18042s (17.4279GiB/s) | 20260506 23:11:17.026 | INFO | Analyzing chunk frames = [5198336,5198848) | 20260506 23:11:17.026 | INFO | MemInfo: { | 20260506 23:11:17.026 | INFO | "gpu" : " 17 GB", | 20260506 23:11:17.026 | INFO | "huge" : "", | 20260506 23:11:17.026 | INFO | "huge_shared" : " 27 GB", | 20260506 23:11:17.026 | INFO | "malloc" : " 14 GB", | 20260506 23:11:17.026 | INFO | "pinned" : " 29 GB" | 20260506 23:11:17.026 | INFO | } | 20260506 23:11:17.026 | INFO | Perf report for the last 5.14965 seconds: | 20260506 23:11:17.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:11:17.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:17.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:17.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:17.026 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 23:11:17.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:17.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:17.026 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:11:17.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:17.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:17.027 | INFO | Uploaded 36GiB to GPU in 2.02431s (17.7838GiB/s) | 20260506 23:11:17.027 | INFO | Downloaded 34GiB to GPU in 1.93046s (17.6124GiB/s) | 20260506 23:11:22.162 | INFO | Analyzing chunk frames = [5198848,5199360) | 20260506 23:11:22.162 | INFO | MemInfo: { | 20260506 23:11:22.162 | INFO | "gpu" : " 17 GB", | 20260506 23:11:22.162 | INFO | "huge" : "", | 20260506 23:11:22.162 | INFO | "huge_shared" : " 27 GB", | 20260506 23:11:22.162 | INFO | "malloc" : " 14 GB", | 20260506 23:11:22.162 | INFO | "pinned" : " 29 GB" | 20260506 23:11:22.162 | INFO | } | 20260506 23:11:22.163 | INFO | Perf report for the last 5.13618 seconds: | 20260506 23:11:22.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:11:22.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:22.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:22.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:22.163 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 23:11:22.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:22.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:22.163 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:11:22.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:22.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:22.163 | INFO | Uploaded 36GiB to GPU in 1.99464s (18.0483GiB/s) | 20260506 23:11:22.163 | INFO | Downloaded 34GiB to GPU in 1.92894s (17.6262GiB/s) | 20260506 23:11:27.304 | INFO | Analyzing chunk frames = [5199360,5199872) | 20260506 23:11:27.304 | INFO | MemInfo: { | 20260506 23:11:27.304 | INFO | "gpu" : " 17 GB", | 20260506 23:11:27.304 | INFO | "huge" : "", | 20260506 23:11:27.304 | INFO | "huge_shared" : " 27 GB", | 20260506 23:11:27.304 | INFO | "malloc" : " 14 GB", | 20260506 23:11:27.304 | INFO | "pinned" : " 29 GB" | 20260506 23:11:27.304 | INFO | } | 20260506 23:11:27.305 | INFO | Perf report for the last 5.14212 seconds: | 20260506 23:11:27.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:11:27.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:27.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:27.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:27.305 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 23:11:27.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:27.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:27.305 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:11:27.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:27.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:27.305 | INFO | Uploaded 36GiB to GPU in 1.99623s (18.034GiB/s) | 20260506 23:11:27.305 | INFO | Downloaded 34GiB to GPU in 1.9074s (17.8253GiB/s) | 20260506 23:11:32.442 | INFO | Analyzing chunk frames = [5199872,5200384) | 20260506 23:11:32.443 | INFO | MemInfo: { | 20260506 23:11:32.443 | INFO | "gpu" : " 17 GB", | 20260506 23:11:32.443 | INFO | "huge" : "", | 20260506 23:11:32.443 | INFO | "huge_shared" : " 27 GB", | 20260506 23:11:32.443 | INFO | "malloc" : " 14 GB", | 20260506 23:11:32.443 | INFO | "pinned" : " 29 GB" | 20260506 23:11:32.443 | INFO | } | 20260506 23:11:32.443 | INFO | Perf report for the last 5.13807 seconds: | 20260506 23:11:32.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:11:32.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:32.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:32.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:32.443 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 23:11:32.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:32.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:32.443 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:11:32.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:32.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:32.443 | INFO | Uploaded 36GiB to GPU in 1.99928s (18.0065GiB/s) | 20260506 23:11:32.443 | INFO | Downloaded 34GiB to GPU in 1.93803s (17.5436GiB/s) | 20260506 23:11:37.582 | INFO | Analyzing chunk frames = [5200384,5200896) | 20260506 23:11:37.582 | INFO | MemInfo: { | 20260506 23:11:37.582 | INFO | "gpu" : " 17 GB", | 20260506 23:11:37.582 | INFO | "huge" : "", | 20260506 23:11:37.582 | INFO | "huge_shared" : " 27 GB", | 20260506 23:11:37.582 | INFO | "malloc" : " 14 GB", | 20260506 23:11:37.582 | INFO | "pinned" : " 29 GB" | 20260506 23:11:37.582 | INFO | } | 20260506 23:11:37.582 | INFO | Perf report for the last 5.13956 seconds: | 20260506 23:11:37.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:11:37.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:37.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:37.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:37.583 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 23:11:37.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:37.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:37.583 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 23:11:37.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:37.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:37.583 | INFO | Uploaded 36GiB to GPU in 2.0269s (17.7611GiB/s) | 20260506 23:11:37.583 | INFO | Downloaded 34GiB to GPU in 1.93708s (17.5522GiB/s) | 20260506 23:11:42.715 | INFO | Analyzing chunk frames = [5200896,5201408) | 20260506 23:11:42.715 | INFO | MemInfo: { | 20260506 23:11:42.715 | INFO | "gpu" : " 17 GB", | 20260506 23:11:42.715 | INFO | "huge" : "", | 20260506 23:11:42.715 | INFO | "huge_shared" : " 27 GB", | 20260506 23:11:42.715 | INFO | "malloc" : " 14 GB", | 20260506 23:11:42.715 | INFO | "pinned" : " 29 GB" | 20260506 23:11:42.715 | INFO | } | 20260506 23:11:42.715 | INFO | Perf report for the last 5.13287 seconds: | 20260506 23:11:42.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:11:42.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:42.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:42.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:42.715 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 23:11:42.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:42.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:42.715 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 23:11:42.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:42.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:42.715 | INFO | Uploaded 36GiB to GPU in 1.95327s (18.4306GiB/s) | 20260506 23:11:42.715 | INFO | Downloaded 34GiB to GPU in 1.98196s (17.1548GiB/s) | 20260506 23:11:47.853 | INFO | Analyzing chunk frames = [5201408,5201920) | 20260506 23:11:47.853 | INFO | MemInfo: { | 20260506 23:11:47.853 | INFO | "gpu" : " 17 GB", | 20260506 23:11:47.853 | INFO | "huge" : "", | 20260506 23:11:47.853 | INFO | "huge_shared" : " 27 GB", | 20260506 23:11:47.853 | INFO | "malloc" : " 14 GB", | 20260506 23:11:47.853 | INFO | "pinned" : " 29 GB" | 20260506 23:11:47.853 | INFO | } | 20260506 23:11:47.853 | INFO | Perf report for the last 5.13765 seconds: | 20260506 23:11:47.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:11:47.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:47.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:47.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:47.853 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 23:11:47.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:47.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:47.853 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 23:11:47.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:47.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:47.853 | INFO | Uploaded 36GiB to GPU in 2.00532s (17.9523GiB/s) | 20260506 23:11:47.853 | INFO | Downloaded 34GiB to GPU in 2.00033s (16.9972GiB/s) | 20260506 23:11:47.853 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:11:47.853 | INFO | Total mem usage: 18 GiB | 20260506 23:11:47.853 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:11:47.853 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 23:11:47.853 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:11:47.853 | INFO | High Water Breakdown: | 20260506 23:11:47.853 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:11:47.853 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:11:47.853 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:11:47.853 | INFO | ------------------------------------------ | 20260506 23:11:47.854 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:11:47.854 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:11:47.854 | INFO | Total mem usage: 26.5 GiB | 20260506 23:11:47.854 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:11:47.854 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:11:47.854 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:11:47.854 | INFO | High Water Breakdown: | 20260506 23:11:47.854 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:11:47.854 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:11:47.854 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:11:47.854 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:11:47.854 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:11:47.854 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:11:47.854 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:11:47.854 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:11:47.854 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:11:47.854 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:11:47.854 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:11:47.854 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:11:47.854 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:11:47.854 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:11:47.854 | INFO | ------------------------------------------ | 20260506 23:11:47.854 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:11:47.854 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:11:47.854 | INFO | Total mem usage: 25.6 GiB | 20260506 23:11:47.854 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:11:47.854 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:11:47.854 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:11:47.854 | INFO | High Water Breakdown: | 20260506 23:11:47.854 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:11:47.854 | INFO | ------------------------------------------ | 20260506 23:11:47.854 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:11:47.854 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:11:47.854 | INFO | Total mem usage: 15.7 GiB | 20260506 23:11:47.854 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:11:47.854 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:11:47.854 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:11:47.854 | INFO | High Water Breakdown: | 20260506 23:11:47.854 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:11:47.854 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:11:47.854 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:11:47.854 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:11:47.854 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:11:47.854 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:11:47.854 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:11:47.854 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:11:47.854 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:11:47.854 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:11:47.854 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:11:47.854 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:11:47.854 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:11:47.854 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:11:47.854 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:11:47.854 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:11:47.854 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:11:47.854 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:11:47.854 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:11:47.854 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:11:47.854 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:11:47.854 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:11:47.854 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:11:47.854 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:11:47.854 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:11:47.854 | INFO | ------------------------------------------ | 20260506 23:11:47.855 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:11:47.855 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:11:47.855 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:11:47.855 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:11:47.855 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.57e+01, 1.34e+01 / 2.49e+00 | 20260506 23:11:47.855 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 9.68e-03 / 6.99e-03 | 20260506 23:11:47.855 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.38e+00, 5.46e-01 / 3.46e-01 | 20260506 23:11:47.855 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.79e-01 / 1.27e-01 | 20260506 23:11:47.855 | INFO | PerfReport: 16.05%, DME -- 3.98e+01 / 4.44e+01, 4.01e+00 / 2.68e+00 | 20260506 23:11:47.855 | INFO | PerfReport: 57.35%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.35e+00 / 1.40e+00 | 20260506 23:11:47.855 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.19e+00 / 2.69e+00, 3.82e-01 / 2.26e-01 | 20260506 23:11:47.855 | INFO | PerfReport: 3.21%, Metrics -- 9.95e-01 / 1.35e+00, 2.64e-01 / 1.36e-01 | 20260506 23:11:47.855 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.39e-01 / 3.89e-01, 5.33e-02 / 2.06e-02 | 20260506 23:11:47.855 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:11:47.855 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:11:47.855 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:11:47.855 | INFO | PerfReport: Full Execution -- 9.68e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 23:11:47.855 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 23:11:47.855 | INFO | PerfReport: 21.26%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 23:11:47.855 | INFO | PerfReport: 32.03%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 23:11:47.855 | INFO | PerfReport: 1.35%, LockUpload -- 1.31e+00 / 0.00e+00, 4.96e+00 / 0.00e+00 | 20260506 23:11:47.855 | INFO | PerfReport: 1.27%, LockDownload -- 1.23e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260506 23:11:47.855 | INFO | PerfReport: 22.55%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:11:47.855 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.39e-02 / 0.00e+00 | 20260506 23:11:47.965 | INFO | Starting a round of Baz IO | 20260506 23:11:51.082 | INFO | Populating RTMetrics report | 20260506 23:11:51.082 | INFO | RTMetrics report populated | 20260506 23:11:51.082 | INFO | Opening tmp RTMetrics json file | 20260506 23:11:51.083 | INFO | Serializing RTMetrics to json | 20260506 23:11:51.084 | INFO | Serialization done, swapping tmp file now... | 20260506 23:11:51.085 | INFO | RTMetrics done | 20260506 23:11:51.179 | INFO | Finished a round of Baz IO: Wrote 1.89 GiB in 3.213 seconds | 20260506 23:11:51.179 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.52% / 0.00% / 66.48% | 20260506 23:11:51.179 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.438527% / 7.94729e-06% / 3.17891e-05% | 20260506 23:11:51.179 | INFO | Baz IO overall: | 20260506 23:11:51.179 | INFO | BytesWritten : 2725601776724 | 20260506 23:11:51.179 | INFO | NumEvents : 1314361074080 | 20260506 23:11:52.997 | INFO | Analyzing chunk frames = [5201920,5202432) | 20260506 23:11:52.997 | INFO | MemInfo: { | 20260506 23:11:52.997 | INFO | "gpu" : " 17 GB", | 20260506 23:11:52.997 | INFO | "huge" : "", | 20260506 23:11:52.997 | INFO | "huge_shared" : " 27 GB", | 20260506 23:11:52.997 | INFO | "malloc" : " 14 GB", | 20260506 23:11:52.997 | INFO | "pinned" : " 29 GB" | 20260506 23:11:52.997 | INFO | } | 20260506 23:11:52.997 | INFO | Perf report for the last 5.1446 seconds: | 20260506 23:11:52.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:11:52.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:52.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:52.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:52.998 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 23:11:52.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:52.998 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 23:11:52.998 | INFO | PRE_HQ : 59.7%, 1.9, 57.4ms, 0 | 20260506 23:11:52.998 | INFO | BAZWRITER : 62.3%, 3.4, 109.9ms, 0 | 20260506 23:11:52.998 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:11:52.998 | INFO | Uploaded 36GiB to GPU in 2.01606s (17.8566GiB/s) | 20260506 23:11:52.998 | INFO | Downloaded 38GiB to GPU in 2.17759s (17.4505GiB/s) | 20260506 23:11:58.139 | INFO | Analyzing chunk frames = [5202432,5202944) | 20260506 23:11:58.139 | INFO | MemInfo: { | 20260506 23:11:58.139 | INFO | "gpu" : " 17 GB", | 20260506 23:11:58.139 | INFO | "huge" : "", | 20260506 23:11:58.139 | INFO | "huge_shared" : " 27 GB", | 20260506 23:11:58.139 | INFO | "malloc" : " 14 GB", | 20260506 23:11:58.139 | INFO | "pinned" : " 29 GB" | 20260506 23:11:58.139 | INFO | } | 20260506 23:11:58.139 | INFO | Perf report for the last 5.1416 seconds: | 20260506 23:11:58.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:11:58.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:58.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:58.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:58.139 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 23:11:58.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:58.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:58.139 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:11:58.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:58.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:11:58.139 | INFO | Uploaded 36GiB to GPU in 1.98725s (18.1155GiB/s) | 20260506 23:11:58.139 | INFO | Downloaded 34GiB to GPU in 1.93145s (17.6033GiB/s) | 20260506 23:12:03.280 | INFO | Analyzing chunk frames = [5202944,5203456) | 20260506 23:12:03.280 | INFO | MemInfo: { | 20260506 23:12:03.280 | INFO | "gpu" : " 17 GB", | 20260506 23:12:03.280 | INFO | "huge" : "", | 20260506 23:12:03.280 | INFO | "huge_shared" : " 27 GB", | 20260506 23:12:03.280 | INFO | "malloc" : " 14 GB", | 20260506 23:12:03.280 | INFO | "pinned" : " 29 GB" | 20260506 23:12:03.280 | INFO | } | 20260506 23:12:03.280 | INFO | Perf report for the last 5.14117 seconds: | 20260506 23:12:03.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:12:03.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:03.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:03.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:03.280 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260506 23:12:03.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:03.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:03.280 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 23:12:03.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:03.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:03.281 | INFO | Uploaded 36GiB to GPU in 1.96207s (18.348GiB/s) | 20260506 23:12:03.281 | INFO | Downloaded 34GiB to GPU in 1.93026s (17.6142GiB/s) | 20260506 23:12:08.421 | INFO | Analyzing chunk frames = [5203456,5203968) | 20260506 23:12:08.421 | INFO | MemInfo: { | 20260506 23:12:08.421 | INFO | "gpu" : " 17 GB", | 20260506 23:12:08.421 | INFO | "huge" : "", | 20260506 23:12:08.421 | INFO | "huge_shared" : " 27 GB", | 20260506 23:12:08.421 | INFO | "malloc" : " 14 GB", | 20260506 23:12:08.421 | INFO | "pinned" : " 29 GB" | 20260506 23:12:08.421 | INFO | } | 20260506 23:12:08.421 | INFO | Perf report for the last 5.14095 seconds: | 20260506 23:12:08.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:12:08.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:08.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:08.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:08.421 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 23:12:08.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:08.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:08.421 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:12:08.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:08.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:08.422 | INFO | Uploaded 36GiB to GPU in 1.96911s (18.2823GiB/s) | 20260506 23:12:08.422 | INFO | Downloaded 34GiB to GPU in 1.90825s (17.8173GiB/s) | 20260506 23:12:13.557 | INFO | Analyzing chunk frames = [5203968,5204480) | 20260506 23:12:13.557 | INFO | MemInfo: { | 20260506 23:12:13.557 | INFO | "gpu" : " 17 GB", | 20260506 23:12:13.557 | INFO | "huge" : "", | 20260506 23:12:13.557 | INFO | "huge_shared" : " 27 GB", | 20260506 23:12:13.557 | INFO | "malloc" : " 14 GB", | 20260506 23:12:13.557 | INFO | "pinned" : " 29 GB" | 20260506 23:12:13.557 | INFO | } | 20260506 23:12:13.557 | INFO | Perf report for the last 5.13595 seconds: | 20260506 23:12:13.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:12:13.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:13.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:13.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:13.557 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260506 23:12:13.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:13.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:13.557 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:12:13.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:13.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:13.557 | INFO | Uploaded 36GiB to GPU in 1.97218s (18.2539GiB/s) | 20260506 23:12:13.558 | INFO | Downloaded 34GiB to GPU in 1.93217s (17.5968GiB/s) | 20260506 23:12:18.691 | INFO | Analyzing chunk frames = [5204480,5204992) | 20260506 23:12:18.691 | INFO | MemInfo: { | 20260506 23:12:18.691 | INFO | "gpu" : " 17 GB", | 20260506 23:12:18.691 | INFO | "huge" : "", | 20260506 23:12:18.691 | INFO | "huge_shared" : " 27 GB", | 20260506 23:12:18.692 | INFO | "malloc" : " 14 GB", | 20260506 23:12:18.692 | INFO | "pinned" : " 29 GB" | 20260506 23:12:18.692 | INFO | } | 20260506 23:12:18.692 | INFO | Perf report for the last 5.13446 seconds: | 20260506 23:12:18.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:12:18.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:18.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:18.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:18.692 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 23:12:18.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:18.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:18.692 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 23:12:18.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:18.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:18.692 | INFO | Uploaded 36GiB to GPU in 2.01897s (17.8308GiB/s) | 20260506 23:12:18.692 | INFO | Downloaded 34GiB to GPU in 1.9361s (17.5611GiB/s) | 20260506 23:12:23.826 | INFO | Analyzing chunk frames = [5204992,5205504) | 20260506 23:12:23.826 | INFO | MemInfo: { | 20260506 23:12:23.827 | INFO | "gpu" : " 17 GB", | 20260506 23:12:23.827 | INFO | "huge" : "", | 20260506 23:12:23.827 | INFO | "huge_shared" : " 27 GB", | 20260506 23:12:23.827 | INFO | "malloc" : " 14 GB", | 20260506 23:12:23.827 | INFO | "pinned" : " 29 GB" | 20260506 23:12:23.827 | INFO | } | 20260506 23:12:23.827 | INFO | Perf report for the last 5.13511 seconds: | 20260506 23:12:23.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:12:23.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:23.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:23.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:23.827 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 23:12:23.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:23.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:23.827 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 23:12:23.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:23.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:23.827 | INFO | Uploaded 36GiB to GPU in 1.96827s (18.2902GiB/s) | 20260506 23:12:23.827 | INFO | Downloaded 34GiB to GPU in 2.00204s (16.9827GiB/s) | 20260506 23:12:28.970 | INFO | Analyzing chunk frames = [5205504,5206016) | 20260506 23:12:28.971 | INFO | MemInfo: { | 20260506 23:12:28.971 | INFO | "gpu" : " 17 GB", | 20260506 23:12:28.971 | INFO | "huge" : "", | 20260506 23:12:28.971 | INFO | "huge_shared" : " 27 GB", | 20260506 23:12:28.971 | INFO | "malloc" : " 14 GB", | 20260506 23:12:28.971 | INFO | "pinned" : " 29 GB" | 20260506 23:12:28.971 | INFO | } | 20260506 23:12:28.971 | INFO | Perf report for the last 5.14389 seconds: | 20260506 23:12:28.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:12:28.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:28.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:28.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:28.971 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:12:28.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:28.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:28.971 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:12:28.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:28.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:28.971 | INFO | Uploaded 36GiB to GPU in 1.97944s (18.187GiB/s) | 20260506 23:12:28.971 | INFO | Downloaded 34GiB to GPU in 1.99551s (17.0383GiB/s) | 20260506 23:12:29.068 | INFO | Starting a round of Baz IO | 20260506 23:12:32.199 | INFO | Populating RTMetrics report | 20260506 23:12:32.199 | INFO | RTMetrics report populated | 20260506 23:12:32.199 | INFO | Opening tmp RTMetrics json file | 20260506 23:12:32.200 | INFO | Serializing RTMetrics to json | 20260506 23:12:32.201 | INFO | Serialization done, swapping tmp file now... | 20260506 23:12:32.202 | INFO | RTMetrics done | 20260506 23:12:32.295 | INFO | Finished a round of Baz IO: Wrote 1.89 GiB in 3.226 seconds | 20260506 23:12:32.295 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.51% / 0.00% / 66.48% | 20260506 23:12:32.295 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.433524% / 0% / 5.96046e-05% | 20260506 23:12:32.295 | INFO | Baz IO overall: | 20260506 23:12:32.295 | INFO | BytesWritten : 2727631003652 | 20260506 23:12:32.295 | INFO | NumEvents : 1315326898787 | 20260506 23:12:34.109 | INFO | Analyzing chunk frames = [5206016,5206528) | 20260506 23:12:34.109 | INFO | MemInfo: { | 20260506 23:12:34.109 | INFO | "gpu" : " 17 GB", | 20260506 23:12:34.109 | INFO | "huge" : "", | 20260506 23:12:34.110 | INFO | "huge_shared" : " 27 GB", | 20260506 23:12:34.110 | INFO | "malloc" : " 14 GB", | 20260506 23:12:34.110 | INFO | "pinned" : " 29 GB" | 20260506 23:12:34.110 | INFO | } | 20260506 23:12:34.110 | INFO | Perf report for the last 5.13897 seconds: | 20260506 23:12:34.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:12:34.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:34.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:34.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:34.110 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 23:12:34.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:34.110 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260506 23:12:34.110 | INFO | PRE_HQ : 59.7%, 1.8, 56.8ms, 0 | 20260506 23:12:34.110 | INFO | BAZWRITER : 62.2%, 3.4, 110.1ms, 0 | 20260506 23:12:34.110 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:12:34.110 | INFO | Uploaded 36GiB to GPU in 2.00498s (17.9553GiB/s) | 20260506 23:12:34.110 | INFO | Downloaded 38GiB to GPU in 2.18477s (17.3932GiB/s) | 20260506 23:12:39.255 | INFO | Analyzing chunk frames = [5206528,5207040) | 20260506 23:12:39.255 | INFO | MemInfo: { | 20260506 23:12:39.255 | INFO | "gpu" : " 17 GB", | 20260506 23:12:39.255 | INFO | "huge" : "", | 20260506 23:12:39.255 | INFO | "huge_shared" : " 27 GB", | 20260506 23:12:39.255 | INFO | "malloc" : " 14 GB", | 20260506 23:12:39.255 | INFO | "pinned" : " 29 GB" | 20260506 23:12:39.255 | INFO | } | 20260506 23:12:39.255 | INFO | Perf report for the last 5.14561 seconds: | 20260506 23:12:39.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:12:39.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:39.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:39.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:39.255 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:12:39.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:39.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:39.255 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260506 23:12:39.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:39.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:39.256 | INFO | Uploaded 36GiB to GPU in 1.99332s (18.0603GiB/s) | 20260506 23:12:39.256 | INFO | Downloaded 34GiB to GPU in 1.93116s (17.606GiB/s) | 20260506 23:12:44.393 | INFO | Analyzing chunk frames = [5207040,5207552) | 20260506 23:12:44.393 | INFO | MemInfo: { | 20260506 23:12:44.393 | INFO | "gpu" : " 17 GB", | 20260506 23:12:44.393 | INFO | "huge" : "", | 20260506 23:12:44.394 | INFO | "huge_shared" : " 27 GB", | 20260506 23:12:44.394 | INFO | "malloc" : " 14 GB", | 20260506 23:12:44.394 | INFO | "pinned" : " 29 GB" | 20260506 23:12:44.394 | INFO | } | 20260506 23:12:44.394 | INFO | Perf report for the last 5.13834 seconds: | 20260506 23:12:44.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:12:44.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:44.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:44.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:44.394 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 23:12:44.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:44.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:44.394 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:12:44.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:44.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:44.394 | INFO | Uploaded 36GiB to GPU in 1.97444s (18.233GiB/s) | 20260506 23:12:44.394 | INFO | Downloaded 34GiB to GPU in 1.93753s (17.5481GiB/s) | 20260506 23:12:49.531 | INFO | Analyzing chunk frames = [5207552,5208064) | 20260506 23:12:49.531 | INFO | MemInfo: { | 20260506 23:12:49.531 | INFO | "gpu" : " 17 GB", | 20260506 23:12:49.531 | INFO | "huge" : "", | 20260506 23:12:49.531 | INFO | "huge_shared" : " 27 GB", | 20260506 23:12:49.531 | INFO | "malloc" : " 14 GB", | 20260506 23:12:49.531 | INFO | "pinned" : " 29 GB" | 20260506 23:12:49.531 | INFO | } | 20260506 23:12:49.531 | INFO | Perf report for the last 5.13774 seconds: | 20260506 23:12:49.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:12:49.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:49.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:49.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:49.531 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260506 23:12:49.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:49.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:49.532 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:12:49.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:49.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:49.532 | INFO | Uploaded 36GiB to GPU in 2.00826s (17.926GiB/s) | 20260506 23:12:49.532 | INFO | Downloaded 34GiB to GPU in 1.90904s (17.81GiB/s) | 20260506 23:12:54.668 | INFO | Analyzing chunk frames = [5208064,5208576) | 20260506 23:12:54.668 | INFO | MemInfo: { | 20260506 23:12:54.668 | INFO | "gpu" : " 17 GB", | 20260506 23:12:54.668 | INFO | "huge" : "", | 20260506 23:12:54.668 | INFO | "huge_shared" : " 27 GB", | 20260506 23:12:54.668 | INFO | "malloc" : " 14 GB", | 20260506 23:12:54.668 | INFO | "pinned" : " 29 GB" | 20260506 23:12:54.668 | INFO | } | 20260506 23:12:54.669 | INFO | Perf report for the last 5.13713 seconds: | 20260506 23:12:54.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:12:54.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:54.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:54.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:54.669 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 23:12:54.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:54.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:54.669 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:12:54.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:54.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:54.669 | INFO | Uploaded 36GiB to GPU in 1.98707s (18.1171GiB/s) | 20260506 23:12:54.669 | INFO | Downloaded 34GiB to GPU in 1.92575s (17.6554GiB/s) | 20260506 23:12:59.806 | INFO | Analyzing chunk frames = [5208576,5209088) | 20260506 23:12:59.806 | INFO | MemInfo: { | 20260506 23:12:59.806 | INFO | "gpu" : " 17 GB", | 20260506 23:12:59.806 | INFO | "huge" : "", | 20260506 23:12:59.806 | INFO | "huge_shared" : " 27 GB", | 20260506 23:12:59.806 | INFO | "malloc" : " 14 GB", | 20260506 23:12:59.806 | INFO | "pinned" : " 29 GB" | 20260506 23:12:59.806 | INFO | } | 20260506 23:12:59.806 | INFO | Perf report for the last 5.13761 seconds: | 20260506 23:12:59.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:12:59.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:59.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:59.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:59.806 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 23:12:59.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:59.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:59.806 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 23:12:59.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:59.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:12:59.807 | INFO | Uploaded 36GiB to GPU in 2.01292s (17.8845GiB/s) | 20260506 23:12:59.807 | INFO | Downloaded 34GiB to GPU in 1.95522s (17.3894GiB/s) | 20260506 23:13:04.947 | INFO | Analyzing chunk frames = [5209088,5209600) | 20260506 23:13:04.947 | INFO | MemInfo: { | 20260506 23:13:04.947 | INFO | "gpu" : " 17 GB", | 20260506 23:13:04.947 | INFO | "huge" : "", | 20260506 23:13:04.947 | INFO | "huge_shared" : " 27 GB", | 20260506 23:13:04.947 | INFO | "malloc" : " 14 GB", | 20260506 23:13:04.947 | INFO | "pinned" : " 29 GB" | 20260506 23:13:04.947 | INFO | } | 20260506 23:13:04.947 | INFO | Perf report for the last 5.14109 seconds: | 20260506 23:13:04.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:13:04.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:04.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:04.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:04.947 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:13:04.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:04.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:04.947 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:13:04.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:04.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:04.948 | INFO | Uploaded 36GiB to GPU in 1.95594s (18.4055GiB/s) | 20260506 23:13:04.948 | INFO | Downloaded 34GiB to GPU in 1.9925s (17.0639GiB/s) | 20260506 23:13:10.089 | INFO | Analyzing chunk frames = [5209600,5210112) | 20260506 23:13:10.089 | INFO | MemInfo: { | 20260506 23:13:10.089 | INFO | "gpu" : " 17 GB", | 20260506 23:13:10.089 | INFO | "huge" : "", | 20260506 23:13:10.089 | INFO | "huge_shared" : " 27 GB", | 20260506 23:13:10.089 | INFO | "malloc" : " 14 GB", | 20260506 23:13:10.089 | INFO | "pinned" : " 29 GB" | 20260506 23:13:10.089 | INFO | } | 20260506 23:13:10.089 | INFO | Perf report for the last 5.14212 seconds: | 20260506 23:13:10.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:13:10.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:10.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:10.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:10.089 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 23:13:10.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:10.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:10.090 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 23:13:10.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:10.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:10.090 | INFO | Uploaded 36GiB to GPU in 1.99781s (18.0197GiB/s) | 20260506 23:13:10.090 | INFO | Downloaded 34GiB to GPU in 2.00197s (16.9832GiB/s) | 20260506 23:13:10.090 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:13:10.090 | INFO | Total mem usage: 18 GiB | 20260506 23:13:10.090 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:13:10.090 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 23:13:10.090 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:13:10.090 | INFO | High Water Breakdown: | 20260506 23:13:10.090 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:13:10.090 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:13:10.090 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:13:10.090 | INFO | ------------------------------------------ | 20260506 23:13:10.090 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:13:10.090 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:13:10.090 | INFO | Total mem usage: 26.5 GiB | 20260506 23:13:10.090 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:13:10.090 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:13:10.090 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:13:10.090 | INFO | High Water Breakdown: | 20260506 23:13:10.090 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:13:10.090 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:13:10.090 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:13:10.090 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:13:10.090 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:13:10.090 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:13:10.090 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:13:10.090 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:13:10.090 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:13:10.090 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:13:10.090 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:13:10.090 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:13:10.090 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:13:10.090 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:13:10.090 | INFO | ------------------------------------------ | 20260506 23:13:10.090 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:13:10.090 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:13:10.090 | INFO | Total mem usage: 25.6 GiB | 20260506 23:13:10.090 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:13:10.090 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:13:10.090 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:13:10.090 | INFO | High Water Breakdown: | 20260506 23:13:10.090 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:13:10.090 | INFO | ------------------------------------------ | 20260506 23:13:10.090 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:13:10.090 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:13:10.090 | INFO | Total mem usage: 15.7 GiB | 20260506 23:13:10.091 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:13:10.091 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:13:10.091 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:13:10.091 | INFO | High Water Breakdown: | 20260506 23:13:10.091 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:13:10.091 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:13:10.091 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:13:10.091 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:13:10.091 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:13:10.091 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:13:10.091 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:13:10.091 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:13:10.091 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:13:10.091 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:13:10.091 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:13:10.091 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:13:10.091 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:13:10.091 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:13:10.091 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:13:10.091 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:13:10.091 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:13:10.091 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:13:10.091 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:13:10.091 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:13:10.091 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:13:10.091 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:13:10.091 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:13:10.091 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:13:10.091 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:13:10.091 | INFO | ------------------------------------------ | 20260506 23:13:10.091 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:13:10.091 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:13:10.091 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:13:10.091 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:13:10.091 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.61e+01, 1.35e+01 / 2.42e+00 | 20260506 23:13:10.091 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.59e-03 / 5.42e-03 | 20260506 23:13:10.091 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.36e+00, 5.47e-01 / 3.55e-01 | 20260506 23:13:10.091 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.44e+00, 1.79e-01 / 1.42e-01 | 20260506 23:13:10.091 | INFO | PerfReport: 16.09%, DME -- 3.99e+01 / 4.47e+01, 4.31e+00 / 2.87e+00 | 20260506 23:13:10.091 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.78e+01 / 2.11e+01, 1.39e+00 / 1.52e+00 | 20260506 23:13:10.091 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.19e+00 / 2.74e+00, 3.77e-01 / 2.30e-01 | 20260506 23:13:10.091 | INFO | PerfReport: 3.20%, Metrics -- 9.94e-01 / 1.32e+00, 2.67e-01 / 1.39e-01 | 20260506 23:13:10.091 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.38e-01 / 3.89e-01, 5.39e-02 / 3.07e-02 | 20260506 23:13:10.091 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:13:10.091 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:13:10.091 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:13:10.091 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 23:13:10.091 | INFO | PerfReport: 21.41%, Upload -- 2.07e+01 / 0.00e+00, 6.82e+00 / 0.00e+00 | 20260506 23:13:10.091 | INFO | PerfReport: 21.30%, Download -- 2.06e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 23:13:10.091 | INFO | PerfReport: 32.09%, Compute -- 3.10e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 23:13:10.091 | INFO | PerfReport: 1.20%, LockUpload -- 1.16e+00 / 0.00e+00, 4.61e+00 / 0.00e+00 | 20260506 23:13:10.091 | INFO | PerfReport: 1.40%, LockDownload -- 1.36e+00 / 0.00e+00, 3.38e+00 / 0.00e+00 | 20260506 23:13:10.091 | INFO | PerfReport: 22.58%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:13:10.091 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260506 23:13:10.187 | INFO | Starting a round of Baz IO | 20260506 23:13:13.327 | INFO | Populating RTMetrics report | 20260506 23:13:13.327 | INFO | RTMetrics report populated | 20260506 23:13:13.327 | INFO | Opening tmp RTMetrics json file | 20260506 23:13:13.334 | INFO | Serializing RTMetrics to json | 20260506 23:13:13.337 | INFO | Serialization done, swapping tmp file now... | 20260506 23:13:13.337 | INFO | RTMetrics done | 20260506 23:13:13.453 | INFO | Finished a round of Baz IO: Wrote 1.892 GiB in 3.265 seconds | 20260506 23:13:13.453 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.51% / 0.00% / 66.49% | 20260506 23:13:13.453 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.429678% / 3.97364e-06% / 8.34465e-05% | 20260506 23:13:13.453 | INFO | Baz IO overall: | 20260506 23:13:13.453 | INFO | BytesWritten : 2729662719527 | 20260506 23:13:13.453 | INFO | NumEvents : 1316292681887 | 20260506 23:13:15.233 | INFO | Analyzing chunk frames = [5210112,5210624) | 20260506 23:13:15.233 | INFO | MemInfo: { | 20260506 23:13:15.233 | INFO | "gpu" : " 17 GB", | 20260506 23:13:15.233 | INFO | "huge" : "", | 20260506 23:13:15.233 | INFO | "huge_shared" : " 27 GB", | 20260506 23:13:15.233 | INFO | "malloc" : " 14 GB", | 20260506 23:13:15.233 | INFO | "pinned" : " 29 GB" | 20260506 23:13:15.233 | INFO | } | 20260506 23:13:15.233 | INFO | Perf report for the last 5.14368 seconds: | 20260506 23:13:15.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:13:15.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:15.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:15.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:15.233 | INFO | ANALYSIS : 62.2%, 2.9, 95.4ms, 0 | 20260506 23:13:15.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:15.233 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 23:13:15.233 | INFO | PRE_HQ : 60.4%, 1.9, 61.2ms, 0 | 20260506 23:13:15.233 | INFO | BAZWRITER : 63.1%, 3.8, 122.4ms, 0 | 20260506 23:13:15.233 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 23:13:15.233 | INFO | Uploaded 36GiB to GPU in 2.01441s (17.8712GiB/s) | 20260506 23:13:15.233 | INFO | Downloaded 38GiB to GPU in 2.17089s (17.5043GiB/s) | 20260506 23:13:20.371 | INFO | Analyzing chunk frames = [5210624,5211136) | 20260506 23:13:20.371 | INFO | MemInfo: { | 20260506 23:13:20.371 | INFO | "gpu" : " 17 GB", | 20260506 23:13:20.371 | INFO | "huge" : "", | 20260506 23:13:20.371 | INFO | "huge_shared" : " 27 GB", | 20260506 23:13:20.371 | INFO | "malloc" : " 14 GB", | 20260506 23:13:20.372 | INFO | "pinned" : " 29 GB" | 20260506 23:13:20.372 | INFO | } | 20260506 23:13:20.372 | INFO | Perf report for the last 5.13853 seconds: | 20260506 23:13:20.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:13:20.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:20.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:20.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:20.372 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:13:20.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:20.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:20.372 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:13:20.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:20.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:20.372 | INFO | Uploaded 36GiB to GPU in 1.98391s (18.146GiB/s) | 20260506 23:13:20.372 | INFO | Downloaded 34GiB to GPU in 1.92896s (17.6261GiB/s) | 20260506 23:13:25.505 | INFO | Analyzing chunk frames = [5211136,5211648) | 20260506 23:13:25.505 | INFO | MemInfo: { | 20260506 23:13:25.505 | INFO | "gpu" : " 17 GB", | 20260506 23:13:25.505 | INFO | "huge" : "", | 20260506 23:13:25.505 | INFO | "huge_shared" : " 27 GB", | 20260506 23:13:25.505 | INFO | "malloc" : " 14 GB", | 20260506 23:13:25.505 | INFO | "pinned" : " 29 GB" | 20260506 23:13:25.505 | INFO | } | 20260506 23:13:25.505 | INFO | Perf report for the last 5.13369 seconds: | 20260506 23:13:25.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:13:25.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:25.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:25.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:25.505 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260506 23:13:25.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:25.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:25.505 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:13:25.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:25.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:25.506 | INFO | Uploaded 36GiB to GPU in 1.98276s (18.1565GiB/s) | 20260506 23:13:25.506 | INFO | Downloaded 34GiB to GPU in 1.93549s (17.5666GiB/s) | 20260506 23:13:30.643 | INFO | Analyzing chunk frames = [5211648,5212160) | 20260506 23:13:30.643 | INFO | MemInfo: { | 20260506 23:13:30.643 | INFO | "gpu" : " 17 GB", | 20260506 23:13:30.643 | INFO | "huge" : "", | 20260506 23:13:30.643 | INFO | "huge_shared" : " 27 GB", | 20260506 23:13:30.643 | INFO | "malloc" : " 14 GB", | 20260506 23:13:30.643 | INFO | "pinned" : " 29 GB" | 20260506 23:13:30.643 | INFO | } | 20260506 23:13:30.643 | INFO | Perf report for the last 5.13773 seconds: | 20260506 23:13:30.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:13:30.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:30.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:30.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:30.643 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 23:13:30.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:30.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:30.643 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:13:30.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:30.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:30.643 | INFO | Uploaded 36GiB to GPU in 2.01343s (17.8799GiB/s) | 20260506 23:13:30.643 | INFO | Downloaded 34GiB to GPU in 1.90928s (17.8078GiB/s) | 20260506 23:13:35.779 | INFO | Analyzing chunk frames = [5212160,5212672) | 20260506 23:13:35.779 | INFO | MemInfo: { | 20260506 23:13:35.779 | INFO | "gpu" : " 17 GB", | 20260506 23:13:35.779 | INFO | "huge" : "", | 20260506 23:13:35.779 | INFO | "huge_shared" : " 27 GB", | 20260506 23:13:35.779 | INFO | "malloc" : " 14 GB", | 20260506 23:13:35.779 | INFO | "pinned" : " 29 GB" | 20260506 23:13:35.779 | INFO | } | 20260506 23:13:35.780 | INFO | Perf report for the last 5.13645 seconds: | 20260506 23:13:35.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:13:35.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:35.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:35.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:35.780 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 23:13:35.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:35.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:35.780 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:13:35.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:35.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:35.780 | INFO | Uploaded 36GiB to GPU in 1.9746s (18.2315GiB/s) | 20260506 23:13:35.780 | INFO | Downloaded 34GiB to GPU in 1.92962s (17.6201GiB/s) | 20260506 23:13:40.918 | INFO | Analyzing chunk frames = [5212672,5213184) | 20260506 23:13:40.918 | INFO | MemInfo: { | 20260506 23:13:40.918 | INFO | "gpu" : " 17 GB", | 20260506 23:13:40.918 | INFO | "huge" : "", | 20260506 23:13:40.918 | INFO | "huge_shared" : " 27 GB", | 20260506 23:13:40.918 | INFO | "malloc" : " 14 GB", | 20260506 23:13:40.918 | INFO | "pinned" : " 29 GB" | 20260506 23:13:40.918 | INFO | } | 20260506 23:13:40.918 | INFO | Perf report for the last 5.1383 seconds: | 20260506 23:13:40.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:13:40.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:40.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:40.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:40.918 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260506 23:13:40.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:40.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:40.918 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:13:40.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:40.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:40.918 | INFO | Uploaded 36GiB to GPU in 1.98378s (18.1472GiB/s) | 20260506 23:13:40.918 | INFO | Downloaded 34GiB to GPU in 1.93987s (17.527GiB/s) | 20260506 23:13:46.064 | INFO | Analyzing chunk frames = [5213184,5213696) | 20260506 23:13:46.065 | INFO | MemInfo: { | 20260506 23:13:46.065 | INFO | "gpu" : " 17 GB", | 20260506 23:13:46.065 | INFO | "huge" : "", | 20260506 23:13:46.065 | INFO | "huge_shared" : " 27 GB", | 20260506 23:13:46.065 | INFO | "malloc" : " 14 GB", | 20260506 23:13:46.065 | INFO | "pinned" : " 29 GB" | 20260506 23:13:46.065 | INFO | } | 20260506 23:13:46.065 | INFO | Perf report for the last 5.14688 seconds: | 20260506 23:13:46.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:13:46.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:46.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:46.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:46.065 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:13:46.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:46.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:46.065 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:13:46.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:46.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:46.065 | INFO | Uploaded 36GiB to GPU in 1.97066s (18.268GiB/s) | 20260506 23:13:46.065 | INFO | Downloaded 34GiB to GPU in 2.00261s (16.9779GiB/s) | 20260506 23:13:51.209 | INFO | Analyzing chunk frames = [5213696,5214208) | 20260506 23:13:51.209 | INFO | MemInfo: { | 20260506 23:13:51.209 | INFO | "gpu" : " 17 GB", | 20260506 23:13:51.209 | INFO | "huge" : "", | 20260506 23:13:51.209 | INFO | "huge_shared" : " 27 GB", | 20260506 23:13:51.209 | INFO | "malloc" : " 14 GB", | 20260506 23:13:51.209 | INFO | "pinned" : " 29 GB" | 20260506 23:13:51.210 | INFO | } | 20260506 23:13:51.210 | INFO | Perf report for the last 5.1448 seconds: | 20260506 23:13:51.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:13:51.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:51.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:51.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:51.210 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 23:13:51.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:51.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:51.210 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 23:13:51.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:51.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:51.210 | INFO | Uploaded 36GiB to GPU in 1.98651s (18.1223GiB/s) | 20260506 23:13:51.210 | INFO | Downloaded 34GiB to GPU in 1.97436s (17.2208GiB/s) | 20260506 23:13:51.308 | INFO | Starting a round of Baz IO | 20260506 23:13:54.433 | INFO | Populating RTMetrics report | 20260506 23:13:54.433 | INFO | RTMetrics report populated | 20260506 23:13:54.433 | INFO | Opening tmp RTMetrics json file | 20260506 23:13:54.434 | INFO | Serializing RTMetrics to json | 20260506 23:13:54.444 | INFO | Serialization done, swapping tmp file now... | 20260506 23:13:54.455 | INFO | RTMetrics done | 20260506 23:13:54.595 | INFO | Finished a round of Baz IO: Wrote 1.889 GiB in 3.287 seconds | 20260506 23:13:54.595 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.51% / 0.00% / 66.49% | 20260506 23:13:54.595 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.428569% / 3.97364e-06% / 7.15256e-05% | 20260506 23:13:54.595 | INFO | Baz IO overall: | 20260506 23:13:54.595 | INFO | BytesWritten : 2731691382507 | 20260506 23:13:54.595 | INFO | NumEvents : 1317257030836 | 20260506 23:13:56.350 | INFO | Analyzing chunk frames = [5214208,5214720) | 20260506 23:13:56.350 | INFO | MemInfo: { | 20260506 23:13:56.350 | INFO | "gpu" : " 17 GB", | 20260506 23:13:56.350 | INFO | "huge" : "", | 20260506 23:13:56.350 | INFO | "huge_shared" : " 27 GB", | 20260506 23:13:56.350 | INFO | "malloc" : " 14 GB", | 20260506 23:13:56.350 | INFO | "pinned" : " 29 GB" | 20260506 23:13:56.350 | INFO | } | 20260506 23:13:56.350 | INFO | Perf report for the last 5.14052 seconds: | 20260506 23:13:56.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:13:56.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:56.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:56.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:56.350 | INFO | ANALYSIS : 62.2%, 2.9, 95.5ms, 0 | 20260506 23:13:56.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:13:56.350 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 23:13:56.350 | INFO | PRE_HQ : 60.0%, 1.8, 56.6ms, 0 | 20260506 23:13:56.350 | INFO | BAZWRITER : 63.5%, 4.0, 131.1ms, 0 | 20260506 23:13:56.350 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:13:56.350 | INFO | Uploaded 36GiB to GPU in 2.0417s (17.6324GiB/s) | 20260506 23:13:56.350 | INFO | Downloaded 38GiB to GPU in 2.19736s (17.2935GiB/s) | 20260506 23:14:01.485 | INFO | Analyzing chunk frames = [5214720,5215232) | 20260506 23:14:01.485 | INFO | MemInfo: { | 20260506 23:14:01.485 | INFO | "gpu" : " 17 GB", | 20260506 23:14:01.485 | INFO | "huge" : "", | 20260506 23:14:01.485 | INFO | "huge_shared" : " 27 GB", | 20260506 23:14:01.485 | INFO | "malloc" : " 14 GB", | 20260506 23:14:01.486 | INFO | "pinned" : " 29 GB" | 20260506 23:14:01.486 | INFO | } | 20260506 23:14:01.486 | INFO | Perf report for the last 5.13599 seconds: | 20260506 23:14:01.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:14:01.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:01.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:01.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:01.486 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:14:01.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:01.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:01.486 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:14:01.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:01.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:01.486 | INFO | Uploaded 36GiB to GPU in 2.01819s (17.8378GiB/s) | 20260506 23:14:01.486 | INFO | Downloaded 34GiB to GPU in 1.9308s (17.6093GiB/s) | 20260506 23:14:06.621 | INFO | Analyzing chunk frames = [5215232,5215744) | 20260506 23:14:06.621 | INFO | MemInfo: { | 20260506 23:14:06.621 | INFO | "gpu" : " 17 GB", | 20260506 23:14:06.621 | INFO | "huge" : "", | 20260506 23:14:06.621 | INFO | "huge_shared" : " 27 GB", | 20260506 23:14:06.621 | INFO | "malloc" : " 14 GB", | 20260506 23:14:06.621 | INFO | "pinned" : " 29 GB" | 20260506 23:14:06.621 | INFO | } | 20260506 23:14:06.622 | INFO | Perf report for the last 5.13544 seconds: | 20260506 23:14:06.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:14:06.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:06.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:06.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:06.622 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260506 23:14:06.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:06.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:06.622 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:14:06.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:06.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:06.622 | INFO | Uploaded 36GiB to GPU in 1.98768s (18.1116GiB/s) | 20260506 23:14:06.622 | INFO | Downloaded 34GiB to GPU in 1.9272s (17.6422GiB/s) | 20260506 23:14:11.759 | INFO | Analyzing chunk frames = [5215744,5216256) | 20260506 23:14:11.759 | INFO | MemInfo: { | 20260506 23:14:11.759 | INFO | "gpu" : " 17 GB", | 20260506 23:14:11.759 | INFO | "huge" : "", | 20260506 23:14:11.759 | INFO | "huge_shared" : " 27 GB", | 20260506 23:14:11.759 | INFO | "malloc" : " 14 GB", | 20260506 23:14:11.759 | INFO | "pinned" : " 29 GB" | 20260506 23:14:11.759 | INFO | } | 20260506 23:14:11.759 | INFO | Perf report for the last 5.13727 seconds: | 20260506 23:14:11.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:14:11.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:11.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:11.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:11.759 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260506 23:14:11.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:11.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:11.759 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:14:11.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:11.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:11.759 | INFO | Uploaded 36GiB to GPU in 2.01089s (17.9025GiB/s) | 20260506 23:14:11.759 | INFO | Downloaded 34GiB to GPU in 1.90883s (17.812GiB/s) | 20260506 23:14:16.896 | INFO | Analyzing chunk frames = [5216256,5216768) | 20260506 23:14:16.896 | INFO | MemInfo: { | 20260506 23:14:16.896 | INFO | "gpu" : " 17 GB", | 20260506 23:14:16.897 | INFO | "huge" : "", | 20260506 23:14:16.897 | INFO | "huge_shared" : " 27 GB", | 20260506 23:14:16.897 | INFO | "malloc" : " 14 GB", | 20260506 23:14:16.897 | INFO | "pinned" : " 29 GB" | 20260506 23:14:16.897 | INFO | } | 20260506 23:14:16.897 | INFO | Perf report for the last 5.13788 seconds: | 20260506 23:14:16.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:14:16.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:16.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:16.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:16.897 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 23:14:16.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:16.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:16.897 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:14:16.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:16.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:16.897 | INFO | Uploaded 36GiB to GPU in 2.00647s (17.942GiB/s) | 20260506 23:14:16.897 | INFO | Downloaded 34GiB to GPU in 1.93576s (17.5641GiB/s) | 20260506 23:14:22.038 | INFO | Analyzing chunk frames = [5216768,5217280) | 20260506 23:14:22.038 | INFO | MemInfo: { | 20260506 23:14:22.038 | INFO | "gpu" : " 17 GB", | 20260506 23:14:22.038 | INFO | "huge" : "", | 20260506 23:14:22.039 | INFO | "huge_shared" : " 27 GB", | 20260506 23:14:22.039 | INFO | "malloc" : " 14 GB", | 20260506 23:14:22.039 | INFO | "pinned" : " 29 GB" | 20260506 23:14:22.039 | INFO | } | 20260506 23:14:22.039 | INFO | Perf report for the last 5.14193 seconds: | 20260506 23:14:22.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:14:22.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:22.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:22.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:22.039 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 23:14:22.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:22.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:22.039 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 23:14:22.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:22.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:22.039 | INFO | Uploaded 36GiB to GPU in 2.01308s (17.8831GiB/s) | 20260506 23:14:22.039 | INFO | Downloaded 34GiB to GPU in 1.9427s (17.5015GiB/s) | 20260506 23:14:27.182 | INFO | Analyzing chunk frames = [5217280,5217792) | 20260506 23:14:27.182 | INFO | MemInfo: { | 20260506 23:14:27.182 | INFO | "gpu" : " 17 GB", | 20260506 23:14:27.182 | INFO | "huge" : "", | 20260506 23:14:27.182 | INFO | "huge_shared" : " 27 GB", | 20260506 23:14:27.182 | INFO | "malloc" : " 14 GB", | 20260506 23:14:27.182 | INFO | "pinned" : " 29 GB" | 20260506 23:14:27.182 | INFO | } | 20260506 23:14:27.182 | INFO | Perf report for the last 5.14374 seconds: | 20260506 23:14:27.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:14:27.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:27.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:27.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:27.182 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 23:14:27.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:27.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:27.182 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:14:27.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:27.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:27.183 | INFO | Uploaded 36GiB to GPU in 1.95926s (18.3743GiB/s) | 20260506 23:14:27.183 | INFO | Downloaded 34GiB to GPU in 1.97748s (17.1936GiB/s) | 20260506 23:14:32.321 | INFO | Analyzing chunk frames = [5217792,5218304) | 20260506 23:14:32.321 | INFO | MemInfo: { | 20260506 23:14:32.321 | INFO | "gpu" : " 17 GB", | 20260506 23:14:32.321 | INFO | "huge" : "", | 20260506 23:14:32.321 | INFO | "huge_shared" : " 27 GB", | 20260506 23:14:32.321 | INFO | "malloc" : " 14 GB", | 20260506 23:14:32.321 | INFO | "pinned" : " 29 GB" | 20260506 23:14:32.321 | INFO | } | 20260506 23:14:32.321 | INFO | Perf report for the last 5.13825 seconds: | 20260506 23:14:32.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:14:32.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:32.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:32.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:32.321 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:14:32.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:32.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:32.321 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 23:14:32.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:32.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:32.321 | INFO | Uploaded 36GiB to GPU in 1.96822s (18.2907GiB/s) | 20260506 23:14:32.321 | INFO | Downloaded 34GiB to GPU in 1.98684s (17.1126GiB/s) | 20260506 23:14:32.321 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:14:32.321 | INFO | Total mem usage: 18 GiB | 20260506 23:14:32.321 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:14:32.321 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -24.5%) | 20260506 23:14:32.321 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:14:32.321 | INFO | High Water Breakdown: | 20260506 23:14:32.321 | INFO | 60.3% : 8.72 GiB ( 76.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:14:32.321 | INFO | 39.6% : 5.72 GiB ( 85.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:14:32.321 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:14:32.321 | INFO | ------------------------------------------ | 20260506 23:14:32.321 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:14:32.321 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:14:32.321 | INFO | Total mem usage: 26.5 GiB | 20260506 23:14:32.321 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:14:32.321 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260506 23:14:32.321 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260506 23:14:32.321 | INFO | High Water Breakdown: | 20260506 23:14:32.321 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:14:32.321 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:14:32.321 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:14:32.321 | INFO | 1.99% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:14:32.322 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:14:32.322 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:14:32.322 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:14:32.322 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:14:32.322 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:14:32.322 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:14:32.322 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:14:32.322 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:14:32.322 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:14:32.322 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:14:32.322 | INFO | ------------------------------------------ | 20260506 23:14:32.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:14:32.322 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:14:32.322 | INFO | Total mem usage: 25.6 GiB | 20260506 23:14:32.322 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:14:32.322 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:14:32.322 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:14:32.322 | INFO | High Water Breakdown: | 20260506 23:14:32.322 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:14:32.322 | INFO | ------------------------------------------ | 20260506 23:14:32.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:14:32.322 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:14:32.322 | INFO | Total mem usage: 15.7 GiB | 20260506 23:14:32.322 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:14:32.322 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:14:32.322 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:14:32.322 | INFO | High Water Breakdown: | 20260506 23:14:32.322 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:14:32.322 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:14:32.322 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:14:32.322 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:14:32.323 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:14:32.323 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:14:32.323 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:14:32.323 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:14:32.323 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:14:32.323 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:14:32.323 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:14:32.323 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:14:32.323 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:14:32.323 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:14:32.323 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:14:32.323 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:14:32.323 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:14:32.323 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:14:32.323 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:14:32.323 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:14:32.323 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:14:32.323 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:14:32.323 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:14:32.323 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:14:32.323 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:14:32.323 | INFO | ------------------------------------------ | 20260506 23:14:32.323 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:14:32.323 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:14:32.323 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:14:32.323 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:14:32.323 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.57e+01, 1.36e+01 / 2.12e+00 | 20260506 23:14:32.323 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.02e-01, 9.81e-03 / 8.80e-03 | 20260506 23:14:32.323 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.35e+00, 5.49e-01 / 4.04e-01 | 20260506 23:14:32.323 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.42e+00, 1.80e-01 / 1.37e-01 | 20260506 23:14:32.323 | INFO | PerfReport: 16.04%, DME -- 3.98e+01 / 4.40e+01, 4.12e+00 / 2.41e+00 | 20260506 23:14:32.323 | INFO | PerfReport: 57.43%, FrameLabeling -- 1.78e+01 / 2.14e+01, 1.42e+00 / 2.11e+00 | 20260506 23:14:32.323 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.18e+00 / 2.70e+00, 3.88e-01 / 2.51e-01 | 20260506 23:14:32.323 | INFO | PerfReport: 3.19%, Metrics -- 9.89e-01 / 1.30e+00, 2.65e-01 / 1.81e-01 | 20260506 23:14:32.323 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.37e-01 / 3.83e-01, 5.35e-02 / 1.99e-02 | 20260506 23:14:32.323 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:14:32.323 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:14:32.323 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:14:32.323 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 23:14:32.323 | INFO | PerfReport: 21.54%, Upload -- 2.08e+01 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260506 23:14:32.323 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 23:14:32.323 | INFO | PerfReport: 32.16%, Compute -- 3.10e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 23:14:32.323 | INFO | PerfReport: 1.26%, LockUpload -- 1.21e+00 / 0.00e+00, 4.72e+00 / 0.00e+00 | 20260506 23:14:32.323 | INFO | PerfReport: 1.26%, LockDownload -- 1.21e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260506 23:14:32.323 | INFO | PerfReport: 22.44%, LockCompute -- 2.16e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 23:14:32.323 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.26e-02 / 0.00e+00 | 20260506 23:14:32.419 | INFO | Starting a round of Baz IO | 20260506 23:14:35.545 | INFO | Populating RTMetrics report | 20260506 23:14:35.546 | INFO | RTMetrics report populated | 20260506 23:14:35.546 | INFO | Opening tmp RTMetrics json file | 20260506 23:14:35.579 | INFO | Serializing RTMetrics to json | 20260506 23:14:35.587 | INFO | Serialization done, swapping tmp file now... | 20260506 23:14:35.611 | INFO | RTMetrics done | 20260506 23:14:35.639 | INFO | Finished a round of Baz IO: Wrote 1.887 GiB in 3.219 seconds | 20260506 23:14:35.639 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.51% / 0.00% / 66.49% | 20260506 23:14:35.639 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.428891% / 3.97364e-06% / 7.94729e-05% | 20260506 23:14:35.639 | INFO | Baz IO overall: | 20260506 23:14:35.639 | INFO | BytesWritten : 2733717383078 | 20260506 23:14:35.639 | INFO | NumEvents : 1318221751522 | 20260506 23:14:37.459 | INFO | Analyzing chunk frames = [5218304,5218816) | 20260506 23:14:37.459 | INFO | MemInfo: { | 20260506 23:14:37.459 | INFO | "gpu" : " 17 GB", | 20260506 23:14:37.459 | INFO | "huge" : "", | 20260506 23:14:37.459 | INFO | "huge_shared" : " 27 GB", | 20260506 23:14:37.459 | INFO | "malloc" : " 14 GB", | 20260506 23:14:37.459 | INFO | "pinned" : " 29 GB" | 20260506 23:14:37.459 | INFO | } | 20260506 23:14:37.459 | INFO | Perf report for the last 5.13847 seconds: | 20260506 23:14:37.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:14:37.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:37.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:37.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:37.459 | INFO | ANALYSIS : 62.2%, 2.9, 95.8ms, 0 | 20260506 23:14:37.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:37.459 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260506 23:14:37.459 | INFO | PRE_HQ : 59.6%, 1.8, 56.4ms, 0 | 20260506 23:14:37.459 | INFO | BAZWRITER : 62.4%, 4.2, 134.2ms, 0 | 20260506 23:14:37.459 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 23:14:37.460 | INFO | Uploaded 36GiB to GPU in 2.0244s (17.783GiB/s) | 20260506 23:14:37.460 | INFO | Downloaded 38GiB to GPU in 2.18556s (17.3868GiB/s) | 20260506 23:14:42.593 | INFO | Analyzing chunk frames = [5218816,5219328) | 20260506 23:14:42.593 | INFO | MemInfo: { | 20260506 23:14:42.593 | INFO | "gpu" : " 17 GB", | 20260506 23:14:42.593 | INFO | "huge" : "", | 20260506 23:14:42.593 | INFO | "huge_shared" : " 27 GB", | 20260506 23:14:42.593 | INFO | "malloc" : " 14 GB", | 20260506 23:14:42.593 | INFO | "pinned" : " 29 GB" | 20260506 23:14:42.593 | INFO | } | 20260506 23:14:42.594 | INFO | Perf report for the last 5.13437 seconds: | 20260506 23:14:42.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:14:42.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:42.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:42.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:42.594 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 23:14:42.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:42.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:42.594 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:14:42.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:42.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:42.594 | INFO | Uploaded 36GiB to GPU in 1.9818s (18.1653GiB/s) | 20260506 23:14:42.594 | INFO | Downloaded 34GiB to GPU in 1.92877s (17.6278GiB/s) | 20260506 23:14:47.732 | INFO | Analyzing chunk frames = [5219328,5219840) | 20260506 23:14:47.732 | INFO | MemInfo: { | 20260506 23:14:47.732 | INFO | "gpu" : " 17 GB", | 20260506 23:14:47.732 | INFO | "huge" : "", | 20260506 23:14:47.732 | INFO | "huge_shared" : " 27 GB", | 20260506 23:14:47.732 | INFO | "malloc" : " 14 GB", | 20260506 23:14:47.732 | INFO | "pinned" : " 29 GB" | 20260506 23:14:47.732 | INFO | } | 20260506 23:14:47.732 | INFO | Perf report for the last 5.13895 seconds: | 20260506 23:14:47.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:14:47.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:47.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:47.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:47.733 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 23:14:47.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:47.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:47.733 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:14:47.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:47.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:47.733 | INFO | Uploaded 36GiB to GPU in 1.98281s (18.156GiB/s) | 20260506 23:14:47.733 | INFO | Downloaded 34GiB to GPU in 1.92808s (17.6341GiB/s) | 20260506 23:14:52.871 | INFO | Analyzing chunk frames = [5219840,5220352) | 20260506 23:14:52.871 | INFO | MemInfo: { | 20260506 23:14:52.871 | INFO | "gpu" : " 17 GB", | 20260506 23:14:52.871 | INFO | "huge" : "", | 20260506 23:14:52.871 | INFO | "huge_shared" : " 27 GB", | 20260506 23:14:52.871 | INFO | "malloc" : " 14 GB", | 20260506 23:14:52.871 | INFO | "pinned" : " 29 GB" | 20260506 23:14:52.871 | INFO | } | 20260506 23:14:52.871 | INFO | Perf report for the last 5.13843 seconds: | 20260506 23:14:52.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:14:52.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:52.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:52.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:52.871 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 23:14:52.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:52.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:52.871 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:14:52.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:52.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:52.871 | INFO | Uploaded 36GiB to GPU in 1.95058s (18.4561GiB/s) | 20260506 23:14:52.871 | INFO | Downloaded 34GiB to GPU in 1.88087s (18.0767GiB/s) | 20260506 23:14:58.014 | INFO | Analyzing chunk frames = [5220352,5220864) | 20260506 23:14:58.014 | INFO | MemInfo: { | 20260506 23:14:58.014 | INFO | "gpu" : " 17 GB", | 20260506 23:14:58.014 | INFO | "huge" : "", | 20260506 23:14:58.014 | INFO | "huge_shared" : " 27 GB", | 20260506 23:14:58.014 | INFO | "malloc" : " 14 GB", | 20260506 23:14:58.014 | INFO | "pinned" : " 29 GB" | 20260506 23:14:58.014 | INFO | } | 20260506 23:14:58.014 | INFO | Perf report for the last 5.14286 seconds: | 20260506 23:14:58.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:14:58.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:58.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:58.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:58.014 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 23:14:58.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:58.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:58.014 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:14:58.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:58.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:14:58.014 | INFO | Uploaded 36GiB to GPU in 1.99706s (18.0265GiB/s) | 20260506 23:14:58.014 | INFO | Downloaded 34GiB to GPU in 1.91962s (17.7118GiB/s) | 20260506 23:15:03.158 | INFO | Analyzing chunk frames = [5220864,5221376) | 20260506 23:15:03.158 | INFO | MemInfo: { | 20260506 23:15:03.158 | INFO | "gpu" : " 17 GB", | 20260506 23:15:03.158 | INFO | "huge" : "", | 20260506 23:15:03.158 | INFO | "huge_shared" : " 27 GB", | 20260506 23:15:03.158 | INFO | "malloc" : " 14 GB", | 20260506 23:15:03.158 | INFO | "pinned" : " 29 GB" | 20260506 23:15:03.159 | INFO | } | 20260506 23:15:03.159 | INFO | Perf report for the last 5.1449 seconds: | 20260506 23:15:03.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:15:03.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:03.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:03.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:03.159 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 23:15:03.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:03.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:03.159 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 23:15:03.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:03.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:03.159 | INFO | Uploaded 36GiB to GPU in 2.01575s (17.8594GiB/s) | 20260506 23:15:03.159 | INFO | Downloaded 34GiB to GPU in 1.94355s (17.4938GiB/s) | 20260506 23:15:08.297 | INFO | Analyzing chunk frames = [5221376,5221888) | 20260506 23:15:08.297 | INFO | MemInfo: { | 20260506 23:15:08.297 | INFO | "gpu" : " 17 GB", | 20260506 23:15:08.297 | INFO | "huge" : "", | 20260506 23:15:08.297 | INFO | "huge_shared" : " 27 GB", | 20260506 23:15:08.297 | INFO | "malloc" : " 14 GB", | 20260506 23:15:08.297 | INFO | "pinned" : " 29 GB" | 20260506 23:15:08.297 | INFO | } | 20260506 23:15:08.297 | INFO | Perf report for the last 5.13823 seconds: | 20260506 23:15:08.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:15:08.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:08.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:08.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:08.297 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 23:15:08.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:08.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:08.297 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 23:15:08.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:08.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:08.297 | INFO | Uploaded 36GiB to GPU in 1.94776s (18.4828GiB/s) | 20260506 23:15:08.297 | INFO | Downloaded 34GiB to GPU in 1.98539s (17.1251GiB/s) | 20260506 23:15:13.439 | INFO | Analyzing chunk frames = [5221888,5222400) | 20260506 23:15:13.439 | INFO | MemInfo: { | 20260506 23:15:13.439 | INFO | "gpu" : " 17 GB", | 20260506 23:15:13.439 | INFO | "huge" : "", | 20260506 23:15:13.439 | INFO | "huge_shared" : " 27 GB", | 20260506 23:15:13.439 | INFO | "malloc" : " 14 GB", | 20260506 23:15:13.439 | INFO | "pinned" : " 29 GB" | 20260506 23:15:13.439 | INFO | } | 20260506 23:15:13.439 | INFO | Perf report for the last 5.14218 seconds: | 20260506 23:15:13.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:15:13.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:13.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:13.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:13.439 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 23:15:13.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:13.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:13.439 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 23:15:13.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:13.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:13.439 | INFO | Uploaded 36GiB to GPU in 2.01666s (17.8513GiB/s) | 20260506 23:15:13.439 | INFO | Downloaded 34GiB to GPU in 1.99516s (17.0412GiB/s) | 20260506 23:15:13.537 | INFO | Starting a round of Baz IO | 20260506 23:15:16.663 | INFO | Populating RTMetrics report | 20260506 23:15:16.663 | INFO | RTMetrics report populated | 20260506 23:15:16.663 | INFO | Opening tmp RTMetrics json file | 20260506 23:15:16.719 | INFO | Serializing RTMetrics to json | 20260506 23:15:16.772 | INFO | Serialization done, swapping tmp file now... | 20260506 23:15:16.810 | INFO | RTMetrics done | 20260506 23:15:16.919 | INFO | Finished a round of Baz IO: Wrote 1.888 GiB in 3.381 seconds | 20260506 23:15:16.919 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.51% / 0.00% / 66.49% | 20260506 23:15:16.919 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.426126% / 7.94729e-06% / 4.37101e-05% | 20260506 23:15:16.919 | INFO | Baz IO overall: | 20260506 23:15:16.919 | INFO | BytesWritten : 2735744207349 | 20260506 23:15:16.919 | INFO | NumEvents : 1319186464024 | 20260506 23:15:18.573 | INFO | Analyzing chunk frames = [5222400,5222912) | 20260506 23:15:18.573 | INFO | MemInfo: { | 20260506 23:15:18.573 | INFO | "gpu" : " 17 GB", | 20260506 23:15:18.573 | INFO | "huge" : "", | 20260506 23:15:18.573 | INFO | "huge_shared" : " 27 GB", | 20260506 23:15:18.573 | INFO | "malloc" : " 14 GB", | 20260506 23:15:18.573 | INFO | "pinned" : " 29 GB" | 20260506 23:15:18.573 | INFO | } | 20260506 23:15:18.574 | INFO | Perf report for the last 5.13433 seconds: | 20260506 23:15:18.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:15:18.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:18.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:18.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:18.574 | INFO | ANALYSIS : 62.2%, 2.9, 95.0ms, 0 | 20260506 23:15:18.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:18.574 | INFO | RT_METRICS : 45.2%, 1.0, 23.7ms, 0 | 20260506 23:15:18.574 | INFO | PRE_HQ : 59.9%, 1.8, 56.4ms, 0 | 20260506 23:15:18.574 | INFO | BAZWRITER : 65.5%, 4.8, 159.9ms, 0 | 20260506 23:15:18.574 | INFO | BAZ_SAMPLING : 2.0%, 1.0, 1.0ms, 0 | 20260506 23:15:18.574 | INFO | Uploaded 36GiB to GPU in 2.03505s (17.69GiB/s) | 20260506 23:15:18.574 | INFO | Downloaded 38GiB to GPU in 2.20399s (17.2415GiB/s) | 20260506 23:15:23.709 | INFO | Analyzing chunk frames = [5222912,5223424) | 20260506 23:15:23.709 | INFO | MemInfo: { | 20260506 23:15:23.709 | INFO | "gpu" : " 17 GB", | 20260506 23:15:23.709 | INFO | "huge" : "", | 20260506 23:15:23.709 | INFO | "huge_shared" : " 27 GB", | 20260506 23:15:23.709 | INFO | "malloc" : " 14 GB", | 20260506 23:15:23.709 | INFO | "pinned" : " 29 GB" | 20260506 23:15:23.709 | INFO | } | 20260506 23:15:23.709 | INFO | Perf report for the last 5.13574 seconds: | 20260506 23:15:23.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:15:23.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:23.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:23.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:23.709 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:15:23.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:23.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:23.709 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:15:23.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:23.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:23.710 | INFO | Uploaded 36GiB to GPU in 1.99466s (18.0482GiB/s) | 20260506 23:15:23.710 | INFO | Downloaded 34GiB to GPU in 1.92673s (17.6464GiB/s) | 20260506 23:15:28.846 | INFO | Analyzing chunk frames = [5223424,5223936) | 20260506 23:15:28.846 | INFO | MemInfo: { | 20260506 23:15:28.846 | INFO | "gpu" : " 17 GB", | 20260506 23:15:28.846 | INFO | "huge" : "", | 20260506 23:15:28.846 | INFO | "huge_shared" : " 27 GB", | 20260506 23:15:28.846 | INFO | "malloc" : " 14 GB", | 20260506 23:15:28.846 | INFO | "pinned" : " 29 GB" | 20260506 23:15:28.846 | INFO | } | 20260506 23:15:28.846 | INFO | Perf report for the last 5.1372 seconds: | 20260506 23:15:28.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:15:28.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:28.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:28.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:28.847 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 23:15:28.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:28.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:28.847 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:15:28.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:28.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:28.847 | INFO | Uploaded 36GiB to GPU in 1.9915s (18.0768GiB/s) | 20260506 23:15:28.847 | INFO | Downloaded 34GiB to GPU in 1.92445s (17.6674GiB/s) | 20260506 23:15:33.990 | INFO | Analyzing chunk frames = [5223936,5224448) | 20260506 23:15:33.990 | INFO | MemInfo: { | 20260506 23:15:33.990 | INFO | "gpu" : " 17 GB", | 20260506 23:15:33.990 | INFO | "huge" : "", | 20260506 23:15:33.990 | INFO | "huge_shared" : " 27 GB", | 20260506 23:15:33.990 | INFO | "malloc" : " 14 GB", | 20260506 23:15:33.990 | INFO | "pinned" : " 29 GB" | 20260506 23:15:33.990 | INFO | } | 20260506 23:15:33.990 | INFO | Perf report for the last 5.14357 seconds: | 20260506 23:15:33.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:15:33.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:33.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:33.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:33.990 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260506 23:15:33.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:33.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:33.990 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:15:33.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:33.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:33.990 | INFO | Uploaded 36GiB to GPU in 1.96766s (18.2959GiB/s) | 20260506 23:15:33.990 | INFO | Downloaded 34GiB to GPU in 1.89059s (17.9838GiB/s) | 20260506 23:15:39.129 | INFO | Analyzing chunk frames = [5224448,5224960) | 20260506 23:15:39.129 | INFO | MemInfo: { | 20260506 23:15:39.129 | INFO | "gpu" : " 17 GB", | 20260506 23:15:39.129 | INFO | "huge" : "", | 20260506 23:15:39.129 | INFO | "huge_shared" : " 27 GB", | 20260506 23:15:39.129 | INFO | "malloc" : " 14 GB", | 20260506 23:15:39.129 | INFO | "pinned" : " 29 GB" | 20260506 23:15:39.129 | INFO | } | 20260506 23:15:39.129 | INFO | Perf report for the last 5.13928 seconds: | 20260506 23:15:39.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:15:39.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:39.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:39.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:39.129 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 23:15:39.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:39.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:39.129 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:15:39.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:39.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:39.130 | INFO | Uploaded 36GiB to GPU in 2.01187s (17.8938GiB/s) | 20260506 23:15:39.130 | INFO | Downloaded 34GiB to GPU in 1.92102s (17.6989GiB/s) | 20260506 23:15:44.268 | INFO | Analyzing chunk frames = [5224960,5225472) | 20260506 23:15:44.268 | INFO | MemInfo: { | 20260506 23:15:44.268 | INFO | "gpu" : " 17 GB", | 20260506 23:15:44.268 | INFO | "huge" : "", | 20260506 23:15:44.269 | INFO | "huge_shared" : " 27 GB", | 20260506 23:15:44.269 | INFO | "malloc" : " 14 GB", | 20260506 23:15:44.269 | INFO | "pinned" : " 29 GB" | 20260506 23:15:44.269 | INFO | } | 20260506 23:15:44.269 | INFO | Perf report for the last 5.13934 seconds: | 20260506 23:15:44.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:15:44.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:44.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:44.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:44.269 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260506 23:15:44.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:44.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:44.269 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:15:44.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:44.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:44.269 | INFO | Uploaded 36GiB to GPU in 1.99405s (18.0537GiB/s) | 20260506 23:15:44.269 | INFO | Downloaded 34GiB to GPU in 1.94396s (17.4901GiB/s) | 20260506 23:15:49.409 | INFO | Analyzing chunk frames = [5225472,5225984) | 20260506 23:15:49.409 | INFO | MemInfo: { | 20260506 23:15:49.409 | INFO | "gpu" : " 17 GB", | 20260506 23:15:49.409 | INFO | "huge" : "", | 20260506 23:15:49.409 | INFO | "huge_shared" : " 27 GB", | 20260506 23:15:49.409 | INFO | "malloc" : " 14 GB", | 20260506 23:15:49.409 | INFO | "pinned" : " 29 GB" | 20260506 23:15:49.409 | INFO | } | 20260506 23:15:49.410 | INFO | Perf report for the last 5.14077 seconds: | 20260506 23:15:49.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:15:49.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:49.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:49.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:49.410 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 23:15:49.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:49.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:49.410 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 23:15:49.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:49.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:49.410 | INFO | Uploaded 36GiB to GPU in 1.9572s (18.3936GiB/s) | 20260506 23:15:49.410 | INFO | Downloaded 34GiB to GPU in 1.98459s (17.132GiB/s) | 20260506 23:15:54.546 | INFO | Analyzing chunk frames = [5225984,5226496) | 20260506 23:15:54.546 | INFO | MemInfo: { | 20260506 23:15:54.546 | INFO | "gpu" : " 17 GB", | 20260506 23:15:54.546 | INFO | "huge" : "", | 20260506 23:15:54.546 | INFO | "huge_shared" : " 27 GB", | 20260506 23:15:54.546 | INFO | "malloc" : " 14 GB", | 20260506 23:15:54.546 | INFO | "pinned" : " 29 GB" | 20260506 23:15:54.546 | INFO | } | 20260506 23:15:54.546 | INFO | Perf report for the last 5.13671 seconds: | 20260506 23:15:54.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:15:54.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:54.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:54.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:54.546 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 23:15:54.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:54.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:54.546 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260506 23:15:54.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:54.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:54.546 | INFO | Uploaded 36GiB to GPU in 1.99509s (18.0443GiB/s) | 20260506 23:15:54.547 | INFO | Downloaded 34GiB to GPU in 1.99514s (17.0414GiB/s) | 20260506 23:15:54.547 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:15:54.547 | INFO | Total mem usage: 18 GiB | 20260506 23:15:54.547 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:15:54.547 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -18.4%) | 20260506 23:15:54.547 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:15:54.547 | INFO | High Water Breakdown: | 20260506 23:15:54.547 | INFO | 61.1% : 9.29 GiB ( 81.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:15:54.547 | INFO | 38.8% : 5.9 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:15:54.547 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:15:54.547 | INFO | ------------------------------------------ | 20260506 23:15:54.547 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:15:54.547 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:15:54.547 | INFO | Total mem usage: 26.5 GiB | 20260506 23:15:54.547 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:15:54.547 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:15:54.547 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:15:54.547 | INFO | High Water Breakdown: | 20260506 23:15:54.547 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:15:54.547 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:15:54.547 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:15:54.547 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:15:54.547 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:15:54.547 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:15:54.547 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:15:54.547 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:15:54.547 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:15:54.547 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:15:54.547 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:15:54.547 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:15:54.547 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:15:54.547 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:15:54.547 | INFO | ------------------------------------------ | 20260506 23:15:54.547 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:15:54.547 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:15:54.547 | INFO | Total mem usage: 25.6 GiB | 20260506 23:15:54.547 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:15:54.547 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:15:54.547 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:15:54.547 | INFO | High Water Breakdown: | 20260506 23:15:54.547 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:15:54.547 | INFO | ------------------------------------------ | 20260506 23:15:54.547 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:15:54.547 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:15:54.547 | INFO | Total mem usage: 15.7 GiB | 20260506 23:15:54.547 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:15:54.547 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:15:54.547 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:15:54.547 | INFO | High Water Breakdown: | 20260506 23:15:54.547 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:15:54.547 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:15:54.547 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:15:54.547 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:15:54.547 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:15:54.547 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:15:54.547 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:15:54.547 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:15:54.547 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:15:54.547 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:15:54.547 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:15:54.547 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:15:54.547 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:15:54.547 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:15:54.548 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:15:54.548 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:15:54.548 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:15:54.548 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:15:54.548 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:15:54.548 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:15:54.548 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:15:54.548 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:15:54.548 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:15:54.548 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:15:54.548 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:15:54.548 | INFO | ------------------------------------------ | 20260506 23:15:54.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:15:54.548 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:15:54.548 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:15:54.548 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:15:54.548 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.54e+01, 1.31e+01 / 2.16e+00 | 20260506 23:15:54.548 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 9.99e-03 / 5.87e-03 | 20260506 23:15:54.548 | INFO | PerfReport: 11.67%, Baseline -- 3.62e+00 / 4.38e+00, 5.44e-01 / 3.44e-01 | 20260506 23:15:54.548 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.45e+00, 1.78e-01 / 1.49e-01 | 20260506 23:15:54.548 | INFO | PerfReport: 16.01%, DME -- 3.97e+01 / 4.41e+01, 3.86e+00 / 2.40e+00 | 20260506 23:15:54.548 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.35e+00 / 1.97e+00 | 20260506 23:15:54.548 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.19e+00 / 2.69e+00, 3.77e-01 / 2.35e-01 | 20260506 23:15:54.548 | INFO | PerfReport: 3.20%, Metrics -- 9.90e-01 / 1.32e+00, 2.62e-01 / 1.24e-01 | 20260506 23:15:54.548 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.38e-01 / 3.88e-01, 5.42e-02 / 2.66e-02 | 20260506 23:15:54.548 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:15:54.548 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:15:54.548 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:15:54.548 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.31e+01 / 0.00e+00 | 20260506 23:15:54.548 | INFO | PerfReport: 21.53%, Upload -- 2.07e+01 / 0.00e+00, 6.89e+00 / 0.00e+00 | 20260506 23:15:54.548 | INFO | PerfReport: 21.32%, Download -- 2.05e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 23:15:54.548 | INFO | PerfReport: 32.15%, Compute -- 3.10e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 23:15:54.548 | INFO | PerfReport: 1.23%, LockUpload -- 1.19e+00 / 0.00e+00, 4.68e+00 / 0.00e+00 | 20260506 23:15:54.548 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 3.16e+00 / 0.00e+00 | 20260506 23:15:54.548 | INFO | PerfReport: 22.47%, LockCompute -- 2.16e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 23:15:54.548 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.40e-02 / 0.00e+00 | 20260506 23:15:54.644 | INFO | Starting a round of Baz IO | 20260506 23:15:57.776 | INFO | Populating RTMetrics report | 20260506 23:15:57.777 | INFO | RTMetrics report populated | 20260506 23:15:57.777 | INFO | Opening tmp RTMetrics json file | 20260506 23:15:57.802 | INFO | Serializing RTMetrics to json | 20260506 23:15:57.811 | INFO | Serialization done, swapping tmp file now... | 20260506 23:15:57.811 | INFO | RTMetrics done | 20260506 23:15:57.905 | INFO | Finished a round of Baz IO: Wrote 1.888 GiB in 3.26 seconds | 20260506 23:15:57.905 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.50% / 0.00% / 66.49% | 20260506 23:15:57.905 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.426753% / 3.97364e-06% / 5.5631e-05% | 20260506 23:15:57.905 | INFO | Baz IO overall: | 20260506 23:15:57.905 | INFO | BytesWritten : 2737770944051 | 20260506 23:15:57.905 | INFO | NumEvents : 1320150876074 | 20260506 23:15:59.683 | INFO | Analyzing chunk frames = [5226496,5227008) | 20260506 23:15:59.683 | INFO | MemInfo: { | 20260506 23:15:59.683 | INFO | "gpu" : " 17 GB", | 20260506 23:15:59.683 | INFO | "huge" : "", | 20260506 23:15:59.683 | INFO | "huge_shared" : " 27 GB", | 20260506 23:15:59.683 | INFO | "malloc" : " 14 GB", | 20260506 23:15:59.683 | INFO | "pinned" : " 29 GB" | 20260506 23:15:59.683 | INFO | } | 20260506 23:15:59.683 | INFO | Perf report for the last 5.13668 seconds: | 20260506 23:15:59.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:15:59.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:59.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:59.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:59.683 | INFO | ANALYSIS : 62.3%, 2.9, 95.7ms, 0 | 20260506 23:15:59.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:15:59.683 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 23:15:59.683 | INFO | PRE_HQ : 59.9%, 1.8, 56.8ms, 0 | 20260506 23:15:59.683 | INFO | BAZWRITER : 63.1%, 4.5, 144.7ms, 0 | 20260506 23:15:59.683 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 23:15:59.683 | INFO | Uploaded 36GiB to GPU in 2.0341s (17.6982GiB/s) | 20260506 23:15:59.683 | INFO | Downloaded 38GiB to GPU in 2.1922s (17.3342GiB/s) | 20260506 23:16:04.829 | INFO | Analyzing chunk frames = [5227008,5227520) | 20260506 23:16:04.829 | INFO | MemInfo: { | 20260506 23:16:04.829 | INFO | "gpu" : " 17 GB", | 20260506 23:16:04.829 | INFO | "huge" : "", | 20260506 23:16:04.829 | INFO | "huge_shared" : " 27 GB", | 20260506 23:16:04.829 | INFO | "malloc" : " 14 GB", | 20260506 23:16:04.829 | INFO | "pinned" : " 29 GB" | 20260506 23:16:04.829 | INFO | } | 20260506 23:16:04.829 | INFO | Perf report for the last 5.14638 seconds: | 20260506 23:16:04.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:16:04.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:04.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:04.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:04.829 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:16:04.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:04.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:04.829 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:16:04.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:04.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:04.830 | INFO | Uploaded 36GiB to GPU in 2.00852s (17.9236GiB/s) | 20260506 23:16:04.830 | INFO | Downloaded 34GiB to GPU in 1.9355s (17.5666GiB/s) | 20260506 23:16:09.966 | INFO | Analyzing chunk frames = [5227520,5228032) | 20260506 23:16:09.966 | INFO | MemInfo: { | 20260506 23:16:09.966 | INFO | "gpu" : " 17 GB", | 20260506 23:16:09.966 | INFO | "huge" : "", | 20260506 23:16:09.966 | INFO | "huge_shared" : " 27 GB", | 20260506 23:16:09.967 | INFO | "malloc" : " 14 GB", | 20260506 23:16:09.967 | INFO | "pinned" : " 29 GB" | 20260506 23:16:09.967 | INFO | } | 20260506 23:16:09.967 | INFO | Perf report for the last 5.1373 seconds: | 20260506 23:16:09.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:16:09.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:09.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:09.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:09.967 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:16:09.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:09.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:09.967 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:16:09.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:09.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:09.967 | INFO | Uploaded 36GiB to GPU in 1.97972s (18.1844GiB/s) | 20260506 23:16:09.967 | INFO | Downloaded 34GiB to GPU in 1.93828s (17.5413GiB/s) | 20260506 23:16:15.106 | INFO | Analyzing chunk frames = [5228032,5228544) | 20260506 23:16:15.106 | INFO | MemInfo: { | 20260506 23:16:15.106 | INFO | "gpu" : " 17 GB", | 20260506 23:16:15.106 | INFO | "huge" : "", | 20260506 23:16:15.106 | INFO | "huge_shared" : " 27 GB", | 20260506 23:16:15.106 | INFO | "malloc" : " 14 GB", | 20260506 23:16:15.107 | INFO | "pinned" : " 29 GB" | 20260506 23:16:15.107 | INFO | } | 20260506 23:16:15.107 | INFO | Perf report for the last 5.13996 seconds: | 20260506 23:16:15.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:16:15.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:15.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:15.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:15.107 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 23:16:15.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:15.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:15.107 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:16:15.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:15.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:15.107 | INFO | Uploaded 36GiB to GPU in 1.98645s (18.1228GiB/s) | 20260506 23:16:15.107 | INFO | Downloaded 34GiB to GPU in 1.91215s (17.781GiB/s) | 20260506 23:16:20.251 | INFO | Analyzing chunk frames = [5228544,5229056) | 20260506 23:16:20.251 | INFO | MemInfo: { | 20260506 23:16:20.251 | INFO | "gpu" : " 17 GB", | 20260506 23:16:20.251 | INFO | "huge" : "", | 20260506 23:16:20.251 | INFO | "huge_shared" : " 27 GB", | 20260506 23:16:20.251 | INFO | "malloc" : " 14 GB", | 20260506 23:16:20.251 | INFO | "pinned" : " 29 GB" | 20260506 23:16:20.251 | INFO | } | 20260506 23:16:20.251 | INFO | Perf report for the last 5.14482 seconds: | 20260506 23:16:20.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:16:20.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:20.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:20.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:20.251 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 23:16:20.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:20.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:20.252 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:16:20.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:20.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:20.252 | INFO | Uploaded 36GiB to GPU in 1.96255s (18.3434GiB/s) | 20260506 23:16:20.252 | INFO | Downloaded 34GiB to GPU in 1.9288s (17.6275GiB/s) | 20260506 23:16:25.386 | INFO | Analyzing chunk frames = [5229056,5229568) | 20260506 23:16:25.386 | INFO | MemInfo: { | 20260506 23:16:25.386 | INFO | "gpu" : " 17 GB", | 20260506 23:16:25.386 | INFO | "huge" : "", | 20260506 23:16:25.386 | INFO | "huge_shared" : " 27 GB", | 20260506 23:16:25.386 | INFO | "malloc" : " 14 GB", | 20260506 23:16:25.386 | INFO | "pinned" : " 29 GB" | 20260506 23:16:25.386 | INFO | } | 20260506 23:16:25.386 | INFO | Perf report for the last 5.13423 seconds: | 20260506 23:16:25.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:16:25.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:25.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:25.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:25.386 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 23:16:25.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:25.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:25.386 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:16:25.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:25.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:25.386 | INFO | Uploaded 36GiB to GPU in 2.02251s (17.7996GiB/s) | 20260506 23:16:25.386 | INFO | Downloaded 34GiB to GPU in 1.93171s (17.601GiB/s) | 20260506 23:16:30.526 | INFO | Analyzing chunk frames = [5229568,5230080) | 20260506 23:16:30.526 | INFO | MemInfo: { | 20260506 23:16:30.526 | INFO | "gpu" : " 17 GB", | 20260506 23:16:30.527 | INFO | "huge" : "", | 20260506 23:16:30.527 | INFO | "huge_shared" : " 27 GB", | 20260506 23:16:30.527 | INFO | "malloc" : " 14 GB", | 20260506 23:16:30.527 | INFO | "pinned" : " 29 GB" | 20260506 23:16:30.527 | INFO | } | 20260506 23:16:30.527 | INFO | Perf report for the last 5.14105 seconds: | 20260506 23:16:30.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:16:30.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:30.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:30.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:30.527 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 23:16:30.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:30.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:30.527 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 23:16:30.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:30.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:30.527 | INFO | Uploaded 36GiB to GPU in 1.97528s (18.2253GiB/s) | 20260506 23:16:30.527 | INFO | Downloaded 34GiB to GPU in 1.99653s (17.0295GiB/s) | 20260506 23:16:35.662 | INFO | Analyzing chunk frames = [5230080,5230592) | 20260506 23:16:35.662 | INFO | MemInfo: { | 20260506 23:16:35.662 | INFO | "gpu" : " 17 GB", | 20260506 23:16:35.662 | INFO | "huge" : "", | 20260506 23:16:35.662 | INFO | "huge_shared" : " 27 GB", | 20260506 23:16:35.663 | INFO | "malloc" : " 14 GB", | 20260506 23:16:35.663 | INFO | "pinned" : " 29 GB" | 20260506 23:16:35.663 | INFO | } | 20260506 23:16:35.663 | INFO | Perf report for the last 5.13592 seconds: | 20260506 23:16:35.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:16:35.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:35.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:35.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:35.663 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 23:16:35.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:35.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:35.663 | INFO | PRE_HQ : 30.5%, 1.0, 15.8ms, 0 | 20260506 23:16:35.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:35.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:35.663 | INFO | Uploaded 36GiB to GPU in 2.02151s (17.8085GiB/s) | 20260506 23:16:35.663 | INFO | Downloaded 34GiB to GPU in 1.97719s (17.1962GiB/s) | 20260506 23:16:35.764 | INFO | Starting a round of Baz IO | 20260506 23:16:38.926 | INFO | Populating RTMetrics report | 20260506 23:16:38.927 | INFO | RTMetrics report populated | 20260506 23:16:38.927 | INFO | Opening tmp RTMetrics json file | 20260506 23:16:38.938 | INFO | Serializing RTMetrics to json | 20260506 23:16:38.976 | INFO | Serialization done, swapping tmp file now... | 20260506 23:16:38.995 | INFO | RTMetrics done | 20260506 23:16:39.236 | INFO | Finished a round of Baz IO: Wrote 1.885 GiB in 3.471 seconds | 20260506 23:16:39.236 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.50% / 0.00% / 66.50% | 20260506 23:16:39.236 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.424858% / 7.94729e-06% / 7.15256e-05% | 20260506 23:16:39.236 | INFO | Baz IO overall: | 20260506 23:16:39.236 | INFO | BytesWritten : 2739794447785 | 20260506 23:16:39.236 | INFO | NumEvents : 1321113512025 | 20260506 23:16:40.799 | INFO | Analyzing chunk frames = [5230592,5231104) | 20260506 23:16:40.799 | INFO | MemInfo: { | 20260506 23:16:40.799 | INFO | "gpu" : " 17 GB", | 20260506 23:16:40.799 | INFO | "huge" : "", | 20260506 23:16:40.799 | INFO | "huge_shared" : " 27 GB", | 20260506 23:16:40.799 | INFO | "malloc" : " 14 GB", | 20260506 23:16:40.799 | INFO | "pinned" : " 29 GB" | 20260506 23:16:40.799 | INFO | } | 20260506 23:16:40.799 | INFO | Perf report for the last 5.13648 seconds: | 20260506 23:16:40.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:16:40.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:40.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:40.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:40.799 | INFO | ANALYSIS : 62.7%, 2.9, 94.0ms, 0 | 20260506 23:16:40.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:40.799 | INFO | RT_METRICS : 47.8%, 1.0, 25.0ms, 0 | 20260506 23:16:40.799 | INFO | PRE_HQ : 61.3%, 2.0, 62.3ms, 0 | 20260506 23:16:40.799 | INFO | BAZWRITER : 67.1%, 4.8, 164.1ms, 0 | 20260506 23:16:40.799 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 23:16:40.799 | INFO | Uploaded 36GiB to GPU in 2.07643s (17.3375GiB/s) | 20260506 23:16:40.799 | INFO | Downloaded 38GiB to GPU in 2.19927s (17.2785GiB/s) | 20260506 23:16:45.937 | INFO | Analyzing chunk frames = [5231104,5231616) | 20260506 23:16:45.937 | INFO | MemInfo: { | 20260506 23:16:45.937 | INFO | "gpu" : " 17 GB", | 20260506 23:16:45.937 | INFO | "huge" : "", | 20260506 23:16:45.937 | INFO | "huge_shared" : " 27 GB", | 20260506 23:16:45.937 | INFO | "malloc" : " 14 GB", | 20260506 23:16:45.937 | INFO | "pinned" : " 29 GB" | 20260506 23:16:45.937 | INFO | } | 20260506 23:16:45.937 | INFO | Perf report for the last 5.13765 seconds: | 20260506 23:16:45.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:16:45.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:45.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:45.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:45.937 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:16:45.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:45.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:45.937 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:16:45.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:45.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:45.937 | INFO | Uploaded 36GiB to GPU in 2.01134s (17.8985GiB/s) | 20260506 23:16:45.937 | INFO | Downloaded 34GiB to GPU in 1.9337s (17.5829GiB/s) | 20260506 23:16:51.077 | INFO | Analyzing chunk frames = [5231616,5232128) | 20260506 23:16:51.077 | INFO | MemInfo: { | 20260506 23:16:51.077 | INFO | "gpu" : " 17 GB", | 20260506 23:16:51.077 | INFO | "huge" : "", | 20260506 23:16:51.077 | INFO | "huge_shared" : " 27 GB", | 20260506 23:16:51.077 | INFO | "malloc" : " 14 GB", | 20260506 23:16:51.077 | INFO | "pinned" : " 29 GB" | 20260506 23:16:51.077 | INFO | } | 20260506 23:16:51.077 | INFO | Perf report for the last 5.14003 seconds: | 20260506 23:16:51.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:16:51.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:51.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:51.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:51.077 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260506 23:16:51.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:51.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:51.077 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:16:51.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:51.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:51.077 | INFO | Uploaded 36GiB to GPU in 1.98018s (18.1802GiB/s) | 20260506 23:16:51.077 | INFO | Downloaded 34GiB to GPU in 1.91505s (17.7541GiB/s) | 20260506 23:16:56.219 | INFO | Analyzing chunk frames = [5232128,5232640) | 20260506 23:16:56.219 | INFO | MemInfo: { | 20260506 23:16:56.219 | INFO | "gpu" : " 17 GB", | 20260506 23:16:56.219 | INFO | "huge" : "", | 20260506 23:16:56.219 | INFO | "huge_shared" : " 27 GB", | 20260506 23:16:56.219 | INFO | "malloc" : " 14 GB", | 20260506 23:16:56.219 | INFO | "pinned" : " 29 GB" | 20260506 23:16:56.219 | INFO | } | 20260506 23:16:56.219 | INFO | Perf report for the last 5.14206 seconds: | 20260506 23:16:56.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:16:56.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:56.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:56.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:56.219 | INFO | ANALYSIS : 59.6%, 3.0, 92.3ms, 0 | 20260506 23:16:56.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:56.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:56.219 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:16:56.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:56.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:16:56.219 | INFO | Uploaded 36GiB to GPU in 1.94625s (18.4971GiB/s) | 20260506 23:16:56.219 | INFO | Downloaded 34GiB to GPU in 1.87336s (18.1492GiB/s) | 20260506 23:17:01.357 | INFO | Analyzing chunk frames = [5232640,5233152) | 20260506 23:17:01.357 | INFO | MemInfo: { | 20260506 23:17:01.357 | INFO | "gpu" : " 17 GB", | 20260506 23:17:01.357 | INFO | "huge" : "", | 20260506 23:17:01.357 | INFO | "huge_shared" : " 27 GB", | 20260506 23:17:01.358 | INFO | "malloc" : " 14 GB", | 20260506 23:17:01.358 | INFO | "pinned" : " 29 GB" | 20260506 23:17:01.358 | INFO | } | 20260506 23:17:01.358 | INFO | Perf report for the last 5.13869 seconds: | 20260506 23:17:01.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:17:01.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:01.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:01.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:01.358 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 23:17:01.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:01.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:01.358 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:17:01.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:01.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:01.358 | INFO | Uploaded 36GiB to GPU in 2.00093s (17.9916GiB/s) | 20260506 23:17:01.358 | INFO | Downloaded 34GiB to GPU in 1.92342s (17.6769GiB/s) | 20260506 23:17:06.499 | INFO | Analyzing chunk frames = [5233152,5233664) | 20260506 23:17:06.499 | INFO | MemInfo: { | 20260506 23:17:06.499 | INFO | "gpu" : " 17 GB", | 20260506 23:17:06.499 | INFO | "huge" : "", | 20260506 23:17:06.499 | INFO | "huge_shared" : " 27 GB", | 20260506 23:17:06.499 | INFO | "malloc" : " 14 GB", | 20260506 23:17:06.499 | INFO | "pinned" : " 29 GB" | 20260506 23:17:06.499 | INFO | } | 20260506 23:17:06.499 | INFO | Perf report for the last 5.14174 seconds: | 20260506 23:17:06.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:17:06.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:06.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:06.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:06.499 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 23:17:06.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:06.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:06.499 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:17:06.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:06.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:06.500 | INFO | Uploaded 36GiB to GPU in 2.02147s (17.8089GiB/s) | 20260506 23:17:06.500 | INFO | Downloaded 34GiB to GPU in 1.95598s (17.3826GiB/s) | 20260506 23:17:11.643 | INFO | Analyzing chunk frames = [5233664,5234176) | 20260506 23:17:11.643 | INFO | MemInfo: { | 20260506 23:17:11.643 | INFO | "gpu" : " 17 GB", | 20260506 23:17:11.643 | INFO | "huge" : "", | 20260506 23:17:11.643 | INFO | "huge_shared" : " 27 GB", | 20260506 23:17:11.643 | INFO | "malloc" : " 14 GB", | 20260506 23:17:11.643 | INFO | "pinned" : " 29 GB" | 20260506 23:17:11.643 | INFO | } | 20260506 23:17:11.643 | INFO | Perf report for the last 5.14398 seconds: | 20260506 23:17:11.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:17:11.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:11.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:11.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:11.643 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:17:11.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:11.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:11.643 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 23:17:11.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:11.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:11.644 | INFO | Uploaded 36GiB to GPU in 1.96384s (18.3314GiB/s) | 20260506 23:17:11.644 | INFO | Downloaded 34GiB to GPU in 1.99003s (17.0851GiB/s) | 20260506 23:17:16.780 | INFO | Analyzing chunk frames = [5234176,5234688) | 20260506 23:17:16.780 | INFO | MemInfo: { | 20260506 23:17:16.780 | INFO | "gpu" : " 17 GB", | 20260506 23:17:16.780 | INFO | "huge" : "", | 20260506 23:17:16.780 | INFO | "huge_shared" : " 27 GB", | 20260506 23:17:16.780 | INFO | "malloc" : " 14 GB", | 20260506 23:17:16.780 | INFO | "pinned" : " 29 GB" | 20260506 23:17:16.780 | INFO | } | 20260506 23:17:16.780 | INFO | Perf report for the last 5.13623 seconds: | 20260506 23:17:16.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:17:16.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:16.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:16.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:16.780 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:17:16.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:16.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:16.780 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:17:16.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:16.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:16.780 | INFO | Uploaded 36GiB to GPU in 2.00326s (17.9707GiB/s) | 20260506 23:17:16.780 | INFO | Downloaded 34GiB to GPU in 1.9734s (17.2291GiB/s) | 20260506 23:17:16.780 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:17:16.780 | INFO | Total mem usage: 18 GiB | 20260506 23:17:16.780 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:17:16.780 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -18.8%) | 20260506 23:17:16.780 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:17:16.780 | INFO | High Water Breakdown: | 20260506 23:17:16.780 | INFO | 61.1% : 9.25 GiB ( 81.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:17:16.780 | INFO | 38.8% : 5.88 GiB ( 87.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:17:16.780 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:17:16.780 | INFO | ------------------------------------------ | 20260506 23:17:16.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:17:16.780 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:17:16.780 | INFO | Total mem usage: 26.5 GiB | 20260506 23:17:16.780 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:17:16.780 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:17:16.780 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:17:16.780 | INFO | High Water Breakdown: | 20260506 23:17:16.780 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:17:16.780 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:17:16.780 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:17:16.780 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:17:16.780 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:17:16.781 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:17:16.781 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:17:16.781 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:17:16.781 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:17:16.781 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:17:16.781 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:17:16.781 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:17:16.781 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:17:16.781 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:17:16.781 | INFO | ------------------------------------------ | 20260506 23:17:16.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:17:16.781 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:17:16.781 | INFO | Total mem usage: 25.6 GiB | 20260506 23:17:16.781 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:17:16.781 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:17:16.781 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:17:16.781 | INFO | High Water Breakdown: | 20260506 23:17:16.781 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:17:16.781 | INFO | ------------------------------------------ | 20260506 23:17:16.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:17:16.781 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:17:16.781 | INFO | Total mem usage: 15.7 GiB | 20260506 23:17:16.781 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:17:16.781 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:17:16.781 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:17:16.781 | INFO | High Water Breakdown: | 20260506 23:17:16.781 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:17:16.781 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:17:16.781 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:17:16.781 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:17:16.781 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:17:16.781 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:17:16.781 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:17:16.781 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:17:16.781 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:17:16.781 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:17:16.781 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:17:16.781 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:17:16.781 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:17:16.781 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:17:16.781 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:17:16.781 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:17:16.781 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:17:16.781 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:17:16.781 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:17:16.781 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:17:16.781 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:17:16.781 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:17:16.781 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:17:16.781 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:17:16.781 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:17:16.781 | INFO | ------------------------------------------ | 20260506 23:17:16.781 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:17:16.782 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:17:16.782 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:17:16.782 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:17:16.782 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.59e+01, 1.35e+01 / 2.31e+00 | 20260506 23:17:16.782 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.02e-01, 1.03e-02 / 7.17e-03 | 20260506 23:17:16.782 | INFO | PerfReport: 11.65%, Baseline -- 3.61e+00 / 4.34e+00, 5.43e-01 / 3.50e-01 | 20260506 23:17:16.782 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.46e+00, 1.79e-01 / 1.29e-01 | 20260506 23:17:16.782 | INFO | PerfReport: 16.10%, DME -- 3.99e+01 / 4.48e+01, 4.26e+00 / 2.54e+00 | 20260506 23:17:16.782 | INFO | PerfReport: 57.36%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.35e+00 / 1.86e+00 | 20260506 23:17:16.782 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.19e+00 / 2.72e+00, 3.82e-01 / 2.38e-01 | 20260506 23:17:16.782 | INFO | PerfReport: 3.19%, Metrics -- 9.89e-01 / 1.30e+00, 2.63e-01 / 1.45e-01 | 20260506 23:17:16.782 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.38e-01 / 3.93e-01, 5.33e-02 / 2.93e-02 | 20260506 23:17:16.782 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:17:16.782 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:17:16.782 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:17:16.782 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 23:17:16.782 | INFO | PerfReport: 21.60%, Upload -- 2.08e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 23:17:16.782 | INFO | PerfReport: 21.32%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 23:17:16.782 | INFO | PerfReport: 32.16%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 23:17:16.782 | INFO | PerfReport: 1.30%, LockUpload -- 1.25e+00 / 0.00e+00, 4.96e+00 / 0.00e+00 | 20260506 23:17:16.782 | INFO | PerfReport: 1.27%, LockDownload -- 1.23e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260506 23:17:16.782 | INFO | PerfReport: 22.34%, LockCompute -- 2.15e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 23:17:16.782 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260506 23:17:16.878 | INFO | Starting a round of Baz IO | 20260506 23:17:20.002 | INFO | Populating RTMetrics report | 20260506 23:17:20.002 | INFO | RTMetrics report populated | 20260506 23:17:20.002 | INFO | Opening tmp RTMetrics json file | 20260506 23:17:20.007 | INFO | Serializing RTMetrics to json | 20260506 23:17:20.029 | INFO | Serialization done, swapping tmp file now... | 20260506 23:17:20.033 | INFO | RTMetrics done | 20260506 23:17:20.167 | INFO | Finished a round of Baz IO: Wrote 1.886 GiB in 3.288 seconds | 20260506 23:17:20.167 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.50% / 0.00% / 66.50% | 20260506 23:17:20.167 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.423261% / 0% / 6.35783e-05% | 20260506 23:17:20.167 | INFO | Baz IO overall: | 20260506 23:17:20.167 | INFO | BytesWritten : 2741819133991 | 20260506 23:17:20.167 | INFO | NumEvents : 1322077416331 | 20260506 23:17:21.921 | INFO | Analyzing chunk frames = [5234688,5235200) | 20260506 23:17:21.922 | INFO | MemInfo: { | 20260506 23:17:21.922 | INFO | "gpu" : " 17 GB", | 20260506 23:17:21.922 | INFO | "huge" : "", | 20260506 23:17:21.922 | INFO | "huge_shared" : " 27 GB", | 20260506 23:17:21.922 | INFO | "malloc" : " 14 GB", | 20260506 23:17:21.922 | INFO | "pinned" : " 29 GB" | 20260506 23:17:21.922 | INFO | } | 20260506 23:17:21.922 | INFO | Perf report for the last 5.14196 seconds: | 20260506 23:17:21.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:17:21.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:21.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:21.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:21.922 | INFO | ANALYSIS : 62.1%, 2.9, 95.1ms, 0 | 20260506 23:17:21.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:21.922 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 23:17:21.922 | INFO | PRE_HQ : 59.7%, 1.8, 56.2ms, 0 | 20260506 23:17:21.922 | INFO | BAZWRITER : 63.6%, 4.7, 153.8ms, 0 | 20260506 23:17:21.922 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 23:17:21.922 | INFO | Uploaded 36GiB to GPU in 2.05901s (17.4841GiB/s) | 20260506 23:17:21.922 | INFO | Downloaded 38GiB to GPU in 2.20053s (17.2686GiB/s) | 20260506 23:17:27.055 | INFO | Analyzing chunk frames = [5235200,5235712) | 20260506 23:17:27.055 | INFO | MemInfo: { | 20260506 23:17:27.055 | INFO | "gpu" : " 17 GB", | 20260506 23:17:27.055 | INFO | "huge" : "", | 20260506 23:17:27.055 | INFO | "huge_shared" : " 27 GB", | 20260506 23:17:27.055 | INFO | "malloc" : " 14 GB", | 20260506 23:17:27.055 | INFO | "pinned" : " 29 GB" | 20260506 23:17:27.055 | INFO | } | 20260506 23:17:27.055 | INFO | Perf report for the last 5.13394 seconds: | 20260506 23:17:27.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:17:27.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:27.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:27.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:27.056 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:17:27.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:27.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:27.056 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260506 23:17:27.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:27.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:27.056 | INFO | Uploaded 36GiB to GPU in 1.99821s (18.0161GiB/s) | 20260506 23:17:27.056 | INFO | Downloaded 34GiB to GPU in 1.9246s (17.666GiB/s) | 20260506 23:17:32.189 | INFO | Analyzing chunk frames = [5235712,5236224) | 20260506 23:17:32.189 | INFO | MemInfo: { | 20260506 23:17:32.189 | INFO | "gpu" : " 17 GB", | 20260506 23:17:32.189 | INFO | "huge" : "", | 20260506 23:17:32.189 | INFO | "huge_shared" : " 27 GB", | 20260506 23:17:32.189 | INFO | "malloc" : " 14 GB", | 20260506 23:17:32.189 | INFO | "pinned" : " 29 GB" | 20260506 23:17:32.189 | INFO | } | 20260506 23:17:32.189 | INFO | Perf report for the last 5.13377 seconds: | 20260506 23:17:32.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:17:32.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:32.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:32.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:32.189 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260506 23:17:32.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:32.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:32.189 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:17:32.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:32.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:32.190 | INFO | Uploaded 36GiB to GPU in 1.98274s (18.1567GiB/s) | 20260506 23:17:32.190 | INFO | Downloaded 34GiB to GPU in 1.92589s (17.6541GiB/s) | 20260506 23:17:37.341 | INFO | Analyzing chunk frames = [5236224,5236736) | 20260506 23:17:37.341 | INFO | MemInfo: { | 20260506 23:17:37.341 | INFO | "gpu" : " 17 GB", | 20260506 23:17:37.341 | INFO | "huge" : "", | 20260506 23:17:37.341 | INFO | "huge_shared" : " 27 GB", | 20260506 23:17:37.341 | INFO | "malloc" : " 14 GB", | 20260506 23:17:37.341 | INFO | "pinned" : " 29 GB" | 20260506 23:17:37.341 | INFO | } | 20260506 23:17:37.341 | INFO | Perf report for the last 5.15207 seconds: | 20260506 23:17:37.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:17:37.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:37.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:37.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:37.341 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 23:17:37.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:37.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:37.341 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:17:37.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:37.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:37.342 | INFO | Uploaded 36GiB to GPU in 1.95695s (18.396GiB/s) | 20260506 23:17:37.342 | INFO | Downloaded 34GiB to GPU in 1.87233s (18.1592GiB/s) | 20260506 23:17:42.478 | INFO | Analyzing chunk frames = [5236736,5237248) | 20260506 23:17:42.478 | INFO | MemInfo: { | 20260506 23:17:42.478 | INFO | "gpu" : " 17 GB", | 20260506 23:17:42.478 | INFO | "huge" : "", | 20260506 23:17:42.478 | INFO | "huge_shared" : " 27 GB", | 20260506 23:17:42.478 | INFO | "malloc" : " 14 GB", | 20260506 23:17:42.478 | INFO | "pinned" : " 29 GB" | 20260506 23:17:42.478 | INFO | } | 20260506 23:17:42.478 | INFO | Perf report for the last 5.13684 seconds: | 20260506 23:17:42.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:17:42.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:42.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:42.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:42.478 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 23:17:42.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:42.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:42.478 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:17:42.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:42.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:42.478 | INFO | Uploaded 36GiB to GPU in 2.01937s (17.8274GiB/s) | 20260506 23:17:42.479 | INFO | Downloaded 34GiB to GPU in 1.9448s (17.4825GiB/s) | 20260506 23:17:47.616 | INFO | Analyzing chunk frames = [5237248,5237760) | 20260506 23:17:47.616 | INFO | MemInfo: { | 20260506 23:17:47.616 | INFO | "gpu" : " 17 GB", | 20260506 23:17:47.616 | INFO | "huge" : "", | 20260506 23:17:47.616 | INFO | "huge_shared" : " 27 GB", | 20260506 23:17:47.616 | INFO | "malloc" : " 14 GB", | 20260506 23:17:47.616 | INFO | "pinned" : " 29 GB" | 20260506 23:17:47.616 | INFO | } | 20260506 23:17:47.616 | INFO | Perf report for the last 5.13817 seconds: | 20260506 23:17:47.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:17:47.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:47.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:47.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:47.617 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 23:17:47.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:47.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:47.617 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 23:17:47.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:47.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:47.617 | INFO | Uploaded 36GiB to GPU in 2.06396s (17.4422GiB/s) | 20260506 23:17:47.617 | INFO | Downloaded 34GiB to GPU in 1.95441s (17.3966GiB/s) | 20260506 23:17:52.755 | INFO | Analyzing chunk frames = [5237760,5238272) | 20260506 23:17:52.755 | INFO | MemInfo: { | 20260506 23:17:52.755 | INFO | "gpu" : " 17 GB", | 20260506 23:17:52.755 | INFO | "huge" : "", | 20260506 23:17:52.755 | INFO | "huge_shared" : " 27 GB", | 20260506 23:17:52.755 | INFO | "malloc" : " 14 GB", | 20260506 23:17:52.755 | INFO | "pinned" : " 29 GB" | 20260506 23:17:52.755 | INFO | } | 20260506 23:17:52.755 | INFO | Perf report for the last 5.13867 seconds: | 20260506 23:17:52.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:17:52.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:52.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:52.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:52.755 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 23:17:52.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:52.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:52.755 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 23:17:52.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:52.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:52.756 | INFO | Uploaded 36GiB to GPU in 2.03496s (17.6908GiB/s) | 20260506 23:17:52.756 | INFO | Downloaded 34GiB to GPU in 1.97872s (17.1828GiB/s) | 20260506 23:17:57.893 | INFO | Analyzing chunk frames = [5238272,5238784) | 20260506 23:17:57.893 | INFO | MemInfo: { | 20260506 23:17:57.893 | INFO | "gpu" : " 17 GB", | 20260506 23:17:57.893 | INFO | "huge" : "", | 20260506 23:17:57.893 | INFO | "huge_shared" : " 27 GB", | 20260506 23:17:57.893 | INFO | "malloc" : " 14 GB", | 20260506 23:17:57.893 | INFO | "pinned" : " 29 GB" | 20260506 23:17:57.893 | INFO | } | 20260506 23:17:57.893 | INFO | Perf report for the last 5.13747 seconds: | 20260506 23:17:57.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:17:57.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:57.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:57.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:57.893 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:17:57.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:57.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:57.893 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 23:17:57.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:57.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:17:57.893 | INFO | Uploaded 36GiB to GPU in 1.98131s (18.1698GiB/s) | 20260506 23:17:57.893 | INFO | Downloaded 34GiB to GPU in 1.98098s (17.1632GiB/s) | 20260506 23:17:57.994 | INFO | Starting a round of Baz IO | 20260506 23:18:01.122 | INFO | Populating RTMetrics report | 20260506 23:18:01.122 | INFO | RTMetrics report populated | 20260506 23:18:01.122 | INFO | Opening tmp RTMetrics json file | 20260506 23:18:01.122 | INFO | Serializing RTMetrics to json | 20260506 23:18:01.124 | INFO | Serialization done, swapping tmp file now... | 20260506 23:18:01.126 | INFO | RTMetrics done | 20260506 23:18:01.239 | INFO | Finished a round of Baz IO: Wrote 1.886 GiB in 3.244 seconds | 20260506 23:18:01.239 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.50% / 0.00% / 66.50% | 20260506 23:18:01.239 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.425406% / 3.97364e-06% / 4.37101e-05% | 20260506 23:18:01.239 | INFO | Baz IO overall: | 20260506 23:18:01.239 | INFO | BytesWritten : 2743844711625 | 20260506 23:18:01.239 | INFO | NumEvents : 1323041794286 | 20260506 23:18:03.028 | INFO | Analyzing chunk frames = [5238784,5239296) | 20260506 23:18:03.028 | INFO | MemInfo: { | 20260506 23:18:03.028 | INFO | "gpu" : " 17 GB", | 20260506 23:18:03.028 | INFO | "huge" : "", | 20260506 23:18:03.028 | INFO | "huge_shared" : " 27 GB", | 20260506 23:18:03.028 | INFO | "malloc" : " 14 GB", | 20260506 23:18:03.028 | INFO | "pinned" : " 29 GB" | 20260506 23:18:03.028 | INFO | } | 20260506 23:18:03.028 | INFO | Perf report for the last 5.1358 seconds: | 20260506 23:18:03.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:18:03.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:03.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:03.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:03.028 | INFO | ANALYSIS : 62.3%, 2.9, 95.2ms, 0 | 20260506 23:18:03.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:03.028 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 23:18:03.028 | INFO | PRE_HQ : 59.9%, 1.8, 56.9ms, 0 | 20260506 23:18:03.029 | INFO | BAZWRITER : 62.9%, 4.3, 138.7ms, 0 | 20260506 23:18:03.029 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 23:18:03.029 | INFO | Uploaded 36GiB to GPU in 2.0321s (17.7156GiB/s) | 20260506 23:18:03.029 | INFO | Downloaded 38GiB to GPU in 2.19041s (17.3484GiB/s) | 20260506 23:18:08.168 | INFO | Analyzing chunk frames = [5239296,5239808) | 20260506 23:18:08.168 | INFO | MemInfo: { | 20260506 23:18:08.168 | INFO | "gpu" : " 17 GB", | 20260506 23:18:08.168 | INFO | "huge" : "", | 20260506 23:18:08.168 | INFO | "huge_shared" : " 27 GB", | 20260506 23:18:08.168 | INFO | "malloc" : " 14 GB", | 20260506 23:18:08.168 | INFO | "pinned" : " 29 GB" | 20260506 23:18:08.168 | INFO | } | 20260506 23:18:08.168 | INFO | Perf report for the last 5.13944 seconds: | 20260506 23:18:08.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:18:08.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:08.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:08.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:08.168 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:18:08.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:08.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:08.168 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:18:08.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:08.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:08.168 | INFO | Uploaded 36GiB to GPU in 2.0133s (17.8811GiB/s) | 20260506 23:18:08.168 | INFO | Downloaded 34GiB to GPU in 1.92722s (17.642GiB/s) | 20260506 23:18:13.311 | INFO | Analyzing chunk frames = [5239808,5240320) | 20260506 23:18:13.311 | INFO | MemInfo: { | 20260506 23:18:13.311 | INFO | "gpu" : " 17 GB", | 20260506 23:18:13.311 | INFO | "huge" : "", | 20260506 23:18:13.311 | INFO | "huge_shared" : " 27 GB", | 20260506 23:18:13.311 | INFO | "malloc" : " 14 GB", | 20260506 23:18:13.311 | INFO | "pinned" : " 29 GB" | 20260506 23:18:13.311 | INFO | } | 20260506 23:18:13.311 | INFO | Perf report for the last 5.14319 seconds: | 20260506 23:18:13.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:18:13.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:13.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:13.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:13.311 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260506 23:18:13.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:13.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:13.311 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:18:13.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:13.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:13.311 | INFO | Uploaded 36GiB to GPU in 1.98633s (18.1239GiB/s) | 20260506 23:18:13.311 | INFO | Downloaded 34GiB to GPU in 1.91938s (17.7141GiB/s) | 20260506 23:18:18.448 | INFO | Analyzing chunk frames = [5240320,5240832) | 20260506 23:18:18.448 | INFO | MemInfo: { | 20260506 23:18:18.448 | INFO | "gpu" : " 17 GB", | 20260506 23:18:18.448 | INFO | "huge" : "", | 20260506 23:18:18.448 | INFO | "huge_shared" : " 27 GB", | 20260506 23:18:18.448 | INFO | "malloc" : " 14 GB", | 20260506 23:18:18.448 | INFO | "pinned" : " 29 GB" | 20260506 23:18:18.449 | INFO | } | 20260506 23:18:18.449 | INFO | Perf report for the last 5.13761 seconds: | 20260506 23:18:18.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:18:18.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:18.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:18.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:18.449 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 23:18:18.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:18.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:18.449 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:18:18.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:18.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:18.449 | INFO | Uploaded 36GiB to GPU in 1.99405s (18.0537GiB/s) | 20260506 23:18:18.449 | INFO | Downloaded 34GiB to GPU in 1.90977s (17.8032GiB/s) | 20260506 23:18:23.598 | INFO | Analyzing chunk frames = [5240832,5241344) | 20260506 23:18:23.598 | INFO | MemInfo: { | 20260506 23:18:23.598 | INFO | "gpu" : " 17 GB", | 20260506 23:18:23.598 | INFO | "huge" : "", | 20260506 23:18:23.599 | INFO | "huge_shared" : " 27 GB", | 20260506 23:18:23.599 | INFO | "malloc" : " 14 GB", | 20260506 23:18:23.599 | INFO | "pinned" : " 29 GB" | 20260506 23:18:23.599 | INFO | } | 20260506 23:18:23.599 | INFO | Perf report for the last 5.1499 seconds: | 20260506 23:18:23.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:18:23.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:23.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:23.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:23.599 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260506 23:18:23.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:23.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:23.599 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 23:18:23.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:23.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:23.599 | INFO | Uploaded 36GiB to GPU in 1.99818s (18.0164GiB/s) | 20260506 23:18:23.599 | INFO | Downloaded 34GiB to GPU in 1.92464s (17.6657GiB/s) | 20260506 23:18:28.735 | INFO | Analyzing chunk frames = [5241344,5241856) | 20260506 23:18:28.735 | INFO | MemInfo: { | 20260506 23:18:28.735 | INFO | "gpu" : " 17 GB", | 20260506 23:18:28.735 | INFO | "huge" : "", | 20260506 23:18:28.735 | INFO | "huge_shared" : " 27 GB", | 20260506 23:18:28.735 | INFO | "malloc" : " 14 GB", | 20260506 23:18:28.735 | INFO | "pinned" : " 29 GB" | 20260506 23:18:28.735 | INFO | } | 20260506 23:18:28.735 | INFO | Perf report for the last 5.13627 seconds: | 20260506 23:18:28.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:18:28.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:28.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:28.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:28.735 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:18:28.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:28.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:28.735 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 23:18:28.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:28.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:28.735 | INFO | Uploaded 36GiB to GPU in 2.06379s (17.4437GiB/s) | 20260506 23:18:28.735 | INFO | Downloaded 34GiB to GPU in 1.96269s (17.3232GiB/s) | 20260506 23:18:33.873 | INFO | Analyzing chunk frames = [5241856,5242368) | 20260506 23:18:33.873 | INFO | MemInfo: { | 20260506 23:18:33.873 | INFO | "gpu" : " 17 GB", | 20260506 23:18:33.873 | INFO | "huge" : "", | 20260506 23:18:33.873 | INFO | "huge_shared" : " 27 GB", | 20260506 23:18:33.873 | INFO | "malloc" : " 14 GB", | 20260506 23:18:33.873 | INFO | "pinned" : " 29 GB" | 20260506 23:18:33.873 | INFO | } | 20260506 23:18:33.873 | INFO | Perf report for the last 5.13852 seconds: | 20260506 23:18:33.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:18:33.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:33.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:33.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:33.874 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:18:33.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:33.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:33.874 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 23:18:33.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:33.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:33.874 | INFO | Uploaded 36GiB to GPU in 1.99352s (18.0585GiB/s) | 20260506 23:18:33.874 | INFO | Downloaded 34GiB to GPU in 1.95487s (17.3924GiB/s) | 20260506 23:18:39.006 | INFO | Analyzing chunk frames = [5242368,5242880) | 20260506 23:18:39.006 | INFO | MemInfo: { | 20260506 23:18:39.006 | INFO | "gpu" : " 17 GB", | 20260506 23:18:39.006 | INFO | "huge" : "", | 20260506 23:18:39.006 | INFO | "huge_shared" : " 27 GB", | 20260506 23:18:39.006 | INFO | "malloc" : " 14 GB", | 20260506 23:18:39.006 | INFO | "pinned" : " 29 GB" | 20260506 23:18:39.006 | INFO | } | 20260506 23:18:39.006 | INFO | Perf report for the last 5.13287 seconds: | 20260506 23:18:39.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:18:39.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:39.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:39.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:39.006 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:18:39.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:39.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:39.007 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 23:18:39.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:39.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:39.007 | INFO | Uploaded 36GiB to GPU in 1.98529s (18.1334GiB/s) | 20260506 23:18:39.007 | INFO | Downloaded 34GiB to GPU in 1.98558s (17.1235GiB/s) | 20260506 23:18:39.007 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:18:39.007 | INFO | Total mem usage: 18 GiB | 20260506 23:18:39.007 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:18:39.007 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -19.1%) | 20260506 23:18:39.007 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:18:39.007 | INFO | High Water Breakdown: | 20260506 23:18:39.007 | INFO | 61% : 9.22 GiB ( 80.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:18:39.007 | INFO | 38.8% : 5.87 GiB ( 87.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:18:39.007 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:18:39.007 | INFO | ------------------------------------------ | 20260506 23:18:39.007 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:18:39.007 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:18:39.007 | INFO | Total mem usage: 26.5 GiB | 20260506 23:18:39.007 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:18:39.007 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:18:39.007 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:18:39.008 | INFO | High Water Breakdown: | 20260506 23:18:39.008 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:18:39.008 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:18:39.008 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:18:39.008 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:18:39.008 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:18:39.008 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:18:39.008 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:18:39.008 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:18:39.008 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:18:39.008 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:18:39.008 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:18:39.008 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:18:39.008 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:18:39.008 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:18:39.008 | INFO | ------------------------------------------ | 20260506 23:18:39.008 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:18:39.008 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:18:39.008 | INFO | Total mem usage: 25.6 GiB | 20260506 23:18:39.008 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:18:39.008 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:18:39.008 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:18:39.008 | INFO | High Water Breakdown: | 20260506 23:18:39.008 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:18:39.008 | INFO | ------------------------------------------ | 20260506 23:18:39.008 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:18:39.008 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:18:39.008 | INFO | Total mem usage: 15.7 GiB | 20260506 23:18:39.008 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:18:39.008 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:18:39.008 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 23:18:39.008 | INFO | High Water Breakdown: | 20260506 23:18:39.008 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:18:39.008 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:18:39.008 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:18:39.008 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:18:39.008 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:18:39.008 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:18:39.008 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:18:39.008 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:18:39.008 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:18:39.008 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:18:39.008 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:18:39.008 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:18:39.008 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:18:39.008 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:18:39.008 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:18:39.008 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:18:39.008 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:18:39.008 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:18:39.008 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:18:39.008 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:18:39.008 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:18:39.008 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:18:39.008 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:18:39.008 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:18:39.008 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:18:39.008 | INFO | ------------------------------------------ | 20260506 23:18:39.008 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:18:39.008 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:18:39.008 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:18:39.008 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:18:39.008 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.51e+01, 1.33e+01 / 1.36e+00 | 20260506 23:18:39.008 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.00e-01, 9.52e-03 / 6.67e-03 | 20260506 23:18:39.008 | INFO | PerfReport: 11.65%, Baseline -- 3.61e+00 / 4.36e+00, 5.38e-01 / 3.85e-01 | 20260506 23:18:39.008 | INFO | PerfReport: 3.24%, Histogram -- 1.01e+00 / 1.46e+00, 1.76e-01 / 1.34e-01 | 20260506 23:18:39.008 | INFO | PerfReport: 16.00%, DME -- 3.96e+01 / 4.38e+01, 3.61e+00 / 1.38e+00 | 20260506 23:18:39.008 | INFO | PerfReport: 57.46%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.41e+00 / 1.50e+00 | 20260506 23:18:39.008 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.19e+00 / 2.70e+00, 3.78e-01 / 2.06e-01 | 20260506 23:18:39.008 | INFO | PerfReport: 3.20%, Metrics -- 9.90e-01 / 1.33e+00, 2.61e-01 / 1.64e-01 | 20260506 23:18:39.008 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.38e-01 / 3.89e-01, 5.34e-02 / 2.28e-02 | 20260506 23:18:39.008 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:18:39.008 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:18:39.008 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:18:39.008 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 23:18:39.008 | INFO | PerfReport: 21.69%, Upload -- 2.09e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260506 23:18:39.008 | INFO | PerfReport: 21.28%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 23:18:39.008 | INFO | PerfReport: 32.10%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 23:18:39.008 | INFO | PerfReport: 1.48%, LockUpload -- 1.42e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260506 23:18:39.008 | INFO | PerfReport: 1.11%, LockDownload -- 1.07e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260506 23:18:39.008 | INFO | PerfReport: 22.32%, LockCompute -- 2.15e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 23:18:39.008 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260506 23:18:39.105 | INFO | Starting a round of Baz IO | 20260506 23:18:42.238 | INFO | Populating RTMetrics report | 20260506 23:18:42.239 | INFO | RTMetrics report populated | 20260506 23:18:42.239 | INFO | Opening tmp RTMetrics json file | 20260506 23:18:42.242 | INFO | Serializing RTMetrics to json | 20260506 23:18:42.265 | INFO | Serialization done, swapping tmp file now... | 20260506 23:18:42.267 | INFO | RTMetrics done | 20260506 23:18:42.360 | INFO | Finished a round of Baz IO: Wrote 1.885 GiB in 3.255 seconds | 20260506 23:18:42.360 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.49% / 0.00% / 66.50% | 20260506 23:18:42.360 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.424298% / 3.97364e-06% / 7.15256e-05% | 20260506 23:18:42.360 | INFO | Baz IO overall: | 20260506 23:18:42.360 | INFO | BytesWritten : 2745868382154 | 20260506 23:18:42.360 | INFO | NumEvents : 1324005165518 | 20260506 23:18:44.140 | INFO | Analyzing chunk frames = [5242880,5243392) | 20260506 23:18:44.140 | INFO | MemInfo: { | 20260506 23:18:44.140 | INFO | "gpu" : " 17 GB", | 20260506 23:18:44.140 | INFO | "huge" : "", | 20260506 23:18:44.140 | INFO | "huge_shared" : " 27 GB", | 20260506 23:18:44.140 | INFO | "malloc" : " 14 GB", | 20260506 23:18:44.140 | INFO | "pinned" : " 29 GB" | 20260506 23:18:44.140 | INFO | } | 20260506 23:18:44.140 | INFO | Perf report for the last 5.134 seconds: | 20260506 23:18:44.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:18:44.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:44.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:44.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:44.140 | INFO | ANALYSIS : 62.4%, 2.9, 95.5ms, 0 | 20260506 23:18:44.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:44.141 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 23:18:44.141 | INFO | PRE_HQ : 59.8%, 1.8, 56.4ms, 0 | 20260506 23:18:44.141 | INFO | BAZWRITER : 63.0%, 4.5, 144.2ms, 0 | 20260506 23:18:44.141 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 23:18:44.141 | INFO | Uploaded 36GiB to GPU in 2.04803s (17.5779GiB/s) | 20260506 23:18:44.141 | INFO | Downloaded 38GiB to GPU in 2.20012s (17.2718GiB/s) | 20260506 23:18:49.285 | INFO | Analyzing chunk frames = [5243392,5243904) | 20260506 23:18:49.285 | INFO | MemInfo: { | 20260506 23:18:49.285 | INFO | "gpu" : " 17 GB", | 20260506 23:18:49.285 | INFO | "huge" : "", | 20260506 23:18:49.285 | INFO | "huge_shared" : " 27 GB", | 20260506 23:18:49.285 | INFO | "malloc" : " 14 GB", | 20260506 23:18:49.285 | INFO | "pinned" : " 29 GB" | 20260506 23:18:49.285 | INFO | } | 20260506 23:18:49.285 | INFO | Perf report for the last 5.14458 seconds: | 20260506 23:18:49.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:18:49.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:49.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:49.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:49.285 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:18:49.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:49.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:49.285 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:18:49.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:49.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:49.285 | INFO | Uploaded 36GiB to GPU in 2.02388s (17.7876GiB/s) | 20260506 23:18:49.285 | INFO | Downloaded 34GiB to GPU in 1.92719s (17.6423GiB/s) | 20260506 23:18:54.436 | INFO | Analyzing chunk frames = [5243904,5244416) | 20260506 23:18:54.436 | INFO | MemInfo: { | 20260506 23:18:54.436 | INFO | "gpu" : " 17 GB", | 20260506 23:18:54.436 | INFO | "huge" : "", | 20260506 23:18:54.436 | INFO | "huge_shared" : " 27 GB", | 20260506 23:18:54.436 | INFO | "malloc" : " 14 GB", | 20260506 23:18:54.436 | INFO | "pinned" : " 29 GB" | 20260506 23:18:54.436 | INFO | } | 20260506 23:18:54.436 | INFO | Perf report for the last 5.15135 seconds: | 20260506 23:18:54.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:18:54.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:54.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:54.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:54.436 | INFO | ANALYSIS : 60.3%, 3.0, 93.6ms, 0 | 20260506 23:18:54.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:54.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:54.436 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:18:54.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:54.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:54.436 | INFO | Uploaded 36GiB to GPU in 1.97917s (18.1894GiB/s) | 20260506 23:18:54.436 | INFO | Downloaded 34GiB to GPU in 1.92391s (17.6723GiB/s) | 20260506 23:18:59.565 | INFO | Analyzing chunk frames = [5244416,5244928) | 20260506 23:18:59.565 | INFO | MemInfo: { | 20260506 23:18:59.565 | INFO | "gpu" : " 17 GB", | 20260506 23:18:59.565 | INFO | "huge" : "", | 20260506 23:18:59.565 | INFO | "huge_shared" : " 27 GB", | 20260506 23:18:59.565 | INFO | "malloc" : " 14 GB", | 20260506 23:18:59.565 | INFO | "pinned" : " 29 GB" | 20260506 23:18:59.565 | INFO | } | 20260506 23:18:59.565 | INFO | Perf report for the last 5.12904 seconds: | 20260506 23:18:59.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:18:59.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:59.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:59.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:59.565 | INFO | ANALYSIS : 60.1%, 3.0, 93.0ms, 0 | 20260506 23:18:59.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:59.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:59.565 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:18:59.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:59.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:18:59.565 | INFO | Uploaded 36GiB to GPU in 1.97882s (18.1927GiB/s) | 20260506 23:18:59.566 | INFO | Downloaded 34GiB to GPU in 1.90047s (17.8903GiB/s) | 20260506 23:19:04.705 | INFO | Analyzing chunk frames = [5244928,5245440) | 20260506 23:19:04.705 | INFO | MemInfo: { | 20260506 23:19:04.705 | INFO | "gpu" : " 17 GB", | 20260506 23:19:04.705 | INFO | "huge" : "", | 20260506 23:19:04.705 | INFO | "huge_shared" : " 27 GB", | 20260506 23:19:04.705 | INFO | "malloc" : " 14 GB", | 20260506 23:19:04.705 | INFO | "pinned" : " 29 GB" | 20260506 23:19:04.705 | INFO | } | 20260506 23:19:04.705 | INFO | Perf report for the last 5.1395 seconds: | 20260506 23:19:04.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:19:04.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:04.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:04.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:04.705 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 23:19:04.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:04.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:04.705 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:19:04.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:04.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:04.705 | INFO | Uploaded 36GiB to GPU in 1.99891s (18.0098GiB/s) | 20260506 23:19:04.705 | INFO | Downloaded 34GiB to GPU in 1.92253s (17.685GiB/s) | 20260506 23:19:09.846 | INFO | Analyzing chunk frames = [5245440,5245952) | 20260506 23:19:09.846 | INFO | MemInfo: { | 20260506 23:19:09.846 | INFO | "gpu" : " 17 GB", | 20260506 23:19:09.846 | INFO | "huge" : "", | 20260506 23:19:09.846 | INFO | "huge_shared" : " 27 GB", | 20260506 23:19:09.846 | INFO | "malloc" : " 14 GB", | 20260506 23:19:09.846 | INFO | "pinned" : " 29 GB" | 20260506 23:19:09.846 | INFO | } | 20260506 23:19:09.846 | INFO | Perf report for the last 5.1415 seconds: | 20260506 23:19:09.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:19:09.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:09.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:09.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:09.846 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 23:19:09.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:09.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:09.846 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 23:19:09.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:09.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:09.847 | INFO | Uploaded 36GiB to GPU in 2.02791s (17.7522GiB/s) | 20260506 23:19:09.847 | INFO | Downloaded 34GiB to GPU in 1.93698s (17.5531GiB/s) | 20260506 23:19:14.984 | INFO | Analyzing chunk frames = [5245952,5246464) | 20260506 23:19:14.984 | INFO | MemInfo: { | 20260506 23:19:14.984 | INFO | "gpu" : " 17 GB", | 20260506 23:19:14.984 | INFO | "huge" : "", | 20260506 23:19:14.984 | INFO | "huge_shared" : " 27 GB", | 20260506 23:19:14.984 | INFO | "malloc" : " 14 GB", | 20260506 23:19:14.984 | INFO | "pinned" : " 29 GB" | 20260506 23:19:14.984 | INFO | } | 20260506 23:19:14.984 | INFO | Perf report for the last 5.13805 seconds: | 20260506 23:19:14.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:19:14.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:14.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:14.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:14.984 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 23:19:14.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:14.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:14.984 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 23:19:14.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:14.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:14.985 | INFO | Uploaded 36GiB to GPU in 2.02422s (17.7847GiB/s) | 20260506 23:19:14.985 | INFO | Downloaded 34GiB to GPU in 1.98123s (17.1611GiB/s) | 20260506 23:19:20.119 | INFO | Analyzing chunk frames = [5246464,5246976) | 20260506 23:19:20.119 | INFO | MemInfo: { | 20260506 23:19:20.119 | INFO | "gpu" : " 17 GB", | 20260506 23:19:20.119 | INFO | "huge" : "", | 20260506 23:19:20.119 | INFO | "huge_shared" : " 27 GB", | 20260506 23:19:20.119 | INFO | "malloc" : " 14 GB", | 20260506 23:19:20.119 | INFO | "pinned" : " 29 GB" | 20260506 23:19:20.119 | INFO | } | 20260506 23:19:20.119 | INFO | Perf report for the last 5.13451 seconds: | 20260506 23:19:20.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:19:20.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:20.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:20.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:20.119 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 23:19:20.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:20.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:20.119 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 23:19:20.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:20.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:20.119 | INFO | Uploaded 36GiB to GPU in 2.01192s (17.8933GiB/s) | 20260506 23:19:20.119 | INFO | Downloaded 34GiB to GPU in 1.99176s (17.0703GiB/s) | 20260506 23:19:20.217 | INFO | Starting a round of Baz IO | 20260506 23:19:23.344 | INFO | Populating RTMetrics report | 20260506 23:19:23.344 | INFO | RTMetrics report populated | 20260506 23:19:23.344 | INFO | Opening tmp RTMetrics json file | 20260506 23:19:23.345 | INFO | Serializing RTMetrics to json | 20260506 23:19:23.348 | INFO | Serialization done, swapping tmp file now... | 20260506 23:19:23.356 | INFO | RTMetrics done | 20260506 23:19:23.530 | INFO | Finished a round of Baz IO: Wrote 1.883 GiB in 3.312 seconds | 20260506 23:19:23.530 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.49% / 0.00% / 66.51% | 20260506 23:19:23.530 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.422267% / 0% / 5.5631e-05% | 20260506 23:19:23.530 | INFO | Baz IO overall: | 20260506 23:19:23.530 | INFO | BytesWritten : 2747890630071 | 20260506 23:19:23.530 | INFO | NumEvents : 1324968159152 | 20260506 23:19:25.257 | INFO | Analyzing chunk frames = [5246976,5247488) | 20260506 23:19:25.257 | INFO | MemInfo: { | 20260506 23:19:25.257 | INFO | "gpu" : " 17 GB", | 20260506 23:19:25.257 | INFO | "huge" : "", | 20260506 23:19:25.257 | INFO | "huge_shared" : " 27 GB", | 20260506 23:19:25.257 | INFO | "malloc" : " 14 GB", | 20260506 23:19:25.257 | INFO | "pinned" : " 29 GB" | 20260506 23:19:25.257 | INFO | } | 20260506 23:19:25.257 | INFO | Perf report for the last 5.13831 seconds: | 20260506 23:19:25.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:19:25.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:25.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:25.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:25.257 | INFO | ANALYSIS : 62.2%, 2.9, 95.7ms, 0 | 20260506 23:19:25.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:25.257 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 23:19:25.257 | INFO | PRE_HQ : 59.8%, 1.8, 56.7ms, 0 | 20260506 23:19:25.257 | INFO | BAZWRITER : 64.2%, 4.0, 131.3ms, 0 | 20260506 23:19:25.257 | INFO | BAZ_SAMPLING : 3.2%, 1.0, 1.7ms, 0 | 20260506 23:19:25.257 | INFO | Uploaded 36GiB to GPU in 2.05892s (17.4849GiB/s) | 20260506 23:19:25.257 | INFO | Downloaded 38GiB to GPU in 2.18309s (17.4065GiB/s) | 20260506 23:19:30.403 | INFO | Analyzing chunk frames = [5247488,5248000) | 20260506 23:19:30.403 | INFO | MemInfo: { | 20260506 23:19:30.403 | INFO | "gpu" : " 17 GB", | 20260506 23:19:30.403 | INFO | "huge" : "", | 20260506 23:19:30.403 | INFO | "huge_shared" : " 27 GB", | 20260506 23:19:30.403 | INFO | "malloc" : " 14 GB", | 20260506 23:19:30.403 | INFO | "pinned" : " 29 GB" | 20260506 23:19:30.403 | INFO | } | 20260506 23:19:30.403 | INFO | Perf report for the last 5.14573 seconds: | 20260506 23:19:30.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:19:30.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:30.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:30.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:30.403 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:19:30.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:30.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:30.403 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:19:30.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:30.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:30.403 | INFO | Uploaded 36GiB to GPU in 2.01734s (17.8453GiB/s) | 20260506 23:19:30.403 | INFO | Downloaded 34GiB to GPU in 1.92537s (17.6589GiB/s) | 20260506 23:19:35.547 | INFO | Analyzing chunk frames = [5248000,5248512) | 20260506 23:19:35.547 | INFO | MemInfo: { | 20260506 23:19:35.547 | INFO | "gpu" : " 17 GB", | 20260506 23:19:35.548 | INFO | "huge" : "", | 20260506 23:19:35.548 | INFO | "huge_shared" : " 27 GB", | 20260506 23:19:35.548 | INFO | "malloc" : " 14 GB", | 20260506 23:19:35.548 | INFO | "pinned" : " 29 GB" | 20260506 23:19:35.548 | INFO | } | 20260506 23:19:35.548 | INFO | Perf report for the last 5.14468 seconds: | 20260506 23:19:35.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:19:35.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:35.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:35.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:35.548 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260506 23:19:35.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:35.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:35.548 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:19:35.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:35.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:35.548 | INFO | Uploaded 36GiB to GPU in 1.98656s (18.1218GiB/s) | 20260506 23:19:35.548 | INFO | Downloaded 34GiB to GPU in 1.92945s (17.6216GiB/s) | 20260506 23:19:40.693 | INFO | Analyzing chunk frames = [5248512,5249024) | 20260506 23:19:40.693 | INFO | MemInfo: { | 20260506 23:19:40.693 | INFO | "gpu" : " 17 GB", | 20260506 23:19:40.693 | INFO | "huge" : "", | 20260506 23:19:40.693 | INFO | "huge_shared" : " 27 GB", | 20260506 23:19:40.693 | INFO | "malloc" : " 14 GB", | 20260506 23:19:40.693 | INFO | "pinned" : " 29 GB" | 20260506 23:19:40.693 | INFO | } | 20260506 23:19:40.693 | INFO | Perf report for the last 5.14512 seconds: | 20260506 23:19:40.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:19:40.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:40.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:40.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:40.693 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 23:19:40.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:40.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:40.693 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:19:40.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:40.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:40.693 | INFO | Uploaded 36GiB to GPU in 1.9877s (18.1114GiB/s) | 20260506 23:19:40.693 | INFO | Downloaded 34GiB to GPU in 1.90762s (17.8232GiB/s) | 20260506 23:19:45.819 | INFO | Analyzing chunk frames = [5249024,5249536) | 20260506 23:19:45.819 | INFO | MemInfo: { | 20260506 23:19:45.819 | INFO | "gpu" : " 17 GB", | 20260506 23:19:45.819 | INFO | "huge" : "", | 20260506 23:19:45.819 | INFO | "huge_shared" : " 27 GB", | 20260506 23:19:45.819 | INFO | "malloc" : " 14 GB", | 20260506 23:19:45.819 | INFO | "pinned" : " 29 GB" | 20260506 23:19:45.819 | INFO | } | 20260506 23:19:45.819 | INFO | Perf report for the last 5.12615 seconds: | 20260506 23:19:45.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:19:45.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:45.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:45.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:45.819 | INFO | ANALYSIS : 60.3%, 3.0, 93.6ms, 0 | 20260506 23:19:45.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:45.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:45.819 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:19:45.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:45.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:45.819 | INFO | Uploaded 36GiB to GPU in 2.01027s (17.908GiB/s) | 20260506 23:19:45.819 | INFO | Downloaded 34GiB to GPU in 1.94648s (17.4674GiB/s) | 20260506 23:19:50.956 | INFO | Analyzing chunk frames = [5249536,5250048) | 20260506 23:19:50.956 | INFO | MemInfo: { | 20260506 23:19:50.956 | INFO | "gpu" : " 17 GB", | 20260506 23:19:50.956 | INFO | "huge" : "", | 20260506 23:19:50.956 | INFO | "huge_shared" : " 27 GB", | 20260506 23:19:50.956 | INFO | "malloc" : " 14 GB", | 20260506 23:19:50.956 | INFO | "pinned" : " 29 GB" | 20260506 23:19:50.956 | INFO | } | 20260506 23:19:50.956 | INFO | Perf report for the last 5.13728 seconds: | 20260506 23:19:50.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:19:50.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:50.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:50.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:50.956 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 23:19:50.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:50.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:50.956 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:19:50.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:50.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:50.957 | INFO | Uploaded 36GiB to GPU in 2.01214s (17.8914GiB/s) | 20260506 23:19:50.957 | INFO | Downloaded 34GiB to GPU in 1.94275s (17.501GiB/s) | 20260506 23:19:56.095 | INFO | Analyzing chunk frames = [5250048,5250560) | 20260506 23:19:56.096 | INFO | MemInfo: { | 20260506 23:19:56.096 | INFO | "gpu" : " 17 GB", | 20260506 23:19:56.096 | INFO | "huge" : "", | 20260506 23:19:56.096 | INFO | "huge_shared" : " 27 GB", | 20260506 23:19:56.096 | INFO | "malloc" : " 14 GB", | 20260506 23:19:56.096 | INFO | "pinned" : " 29 GB" | 20260506 23:19:56.096 | INFO | } | 20260506 23:19:56.096 | INFO | Perf report for the last 5.13941 seconds: | 20260506 23:19:56.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:19:56.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:56.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:56.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:56.096 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:19:56.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:56.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:56.096 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260506 23:19:56.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:56.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:19:56.096 | INFO | Uploaded 36GiB to GPU in 1.96711s (18.3009GiB/s) | 20260506 23:19:56.096 | INFO | Downloaded 34GiB to GPU in 1.99566s (17.037GiB/s) | 20260506 23:20:01.234 | INFO | Analyzing chunk frames = [5250560,5251072) | 20260506 23:20:01.234 | INFO | MemInfo: { | 20260506 23:20:01.234 | INFO | "gpu" : " 17 GB", | 20260506 23:20:01.234 | INFO | "huge" : "", | 20260506 23:20:01.234 | INFO | "huge_shared" : " 27 GB", | 20260506 23:20:01.234 | INFO | "malloc" : " 14 GB", | 20260506 23:20:01.234 | INFO | "pinned" : " 29 GB" | 20260506 23:20:01.234 | INFO | } | 20260506 23:20:01.234 | INFO | Perf report for the last 5.13824 seconds: | 20260506 23:20:01.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:20:01.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:01.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:01.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:01.234 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:20:01.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:01.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:01.234 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 23:20:01.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:01.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:01.234 | INFO | Uploaded 36GiB to GPU in 1.98048s (18.1774GiB/s) | 20260506 23:20:01.234 | INFO | Downloaded 34GiB to GPU in 1.9862s (17.1181GiB/s) | 20260506 23:20:01.234 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:20:01.234 | INFO | Total mem usage: 18 GiB | 20260506 23:20:01.234 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:20:01.234 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -21.4%) | 20260506 23:20:01.234 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:20:01.234 | INFO | High Water Breakdown: | 20260506 23:20:01.234 | INFO | 60.8% : 9.01 GiB ( 78.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:20:01.234 | INFO | 39.1% : 5.8 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:20:01.234 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:20:01.234 | INFO | ------------------------------------------ | 20260506 23:20:01.234 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:20:01.234 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:20:01.234 | INFO | Total mem usage: 26.5 GiB | 20260506 23:20:01.234 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:20:01.234 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:20:01.234 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:20:01.234 | INFO | High Water Breakdown: | 20260506 23:20:01.234 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:20:01.234 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:20:01.234 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:20:01.234 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:20:01.234 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:20:01.234 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:20:01.234 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:20:01.234 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:20:01.235 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:20:01.235 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:20:01.235 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:20:01.235 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:20:01.235 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:20:01.235 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:20:01.235 | INFO | ------------------------------------------ | 20260506 23:20:01.235 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:20:01.235 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:20:01.235 | INFO | Total mem usage: 25.6 GiB | 20260506 23:20:01.235 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:20:01.235 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:20:01.235 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:20:01.235 | INFO | High Water Breakdown: | 20260506 23:20:01.235 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:20:01.235 | INFO | ------------------------------------------ | 20260506 23:20:01.235 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:20:01.235 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:20:01.235 | INFO | Total mem usage: 15.7 GiB | 20260506 23:20:01.235 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:20:01.235 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:20:01.235 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:20:01.235 | INFO | High Water Breakdown: | 20260506 23:20:01.235 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:20:01.235 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:20:01.235 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:20:01.235 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:20:01.235 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:20:01.235 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:20:01.235 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:20:01.235 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:20:01.235 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:20:01.235 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:20:01.235 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:20:01.235 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:20:01.235 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:20:01.235 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:20:01.235 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:20:01.235 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:20:01.235 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:20:01.235 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:20:01.235 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:20:01.235 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:20:01.235 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:20:01.235 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:20:01.235 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:20:01.235 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:20:01.235 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:20:01.235 | INFO | ------------------------------------------ | 20260506 23:20:01.235 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:20:01.235 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:20:01.235 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:20:01.235 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:20:01.235 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.62e+01, 1.36e+01 / 2.67e+00 | 20260506 23:20:01.235 | INFO | PerfReport: 0.30%, AggregateStats -- 9.16e-02 / 1.01e-01, 1.07e-02 / 6.84e-03 | 20260506 23:20:01.235 | INFO | PerfReport: 11.68%, Baseline -- 3.63e+00 / 4.39e+00, 5.56e-01 / 3.49e-01 | 20260506 23:20:01.235 | INFO | PerfReport: 3.27%, Histogram -- 1.02e+00 / 1.46e+00, 1.84e-01 / 1.40e-01 | 20260506 23:20:01.235 | INFO | PerfReport: 16.07%, DME -- 3.99e+01 / 4.49e+01, 4.24e+00 / 2.82e+00 | 20260506 23:20:01.235 | INFO | PerfReport: 57.33%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.36e+00 / 1.52e+00 | 20260506 23:20:01.235 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.19e+00 / 2.69e+00, 3.84e-01 / 2.34e-01 | 20260506 23:20:01.235 | INFO | PerfReport: 3.20%, Metrics -- 9.92e-01 / 1.35e+00, 2.68e-01 / 1.71e-01 | 20260506 23:20:01.235 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.38e-01 / 3.85e-01, 5.39e-02 / 1.99e-02 | 20260506 23:20:01.235 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:20:01.235 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:20:01.235 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:20:01.235 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 23:20:01.235 | INFO | PerfReport: 21.64%, Upload -- 2.09e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260506 23:20:01.235 | INFO | PerfReport: 21.30%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 23:20:01.235 | INFO | PerfReport: 32.12%, Compute -- 3.10e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 23:20:01.235 | INFO | PerfReport: 1.35%, LockUpload -- 1.30e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260506 23:20:01.235 | INFO | PerfReport: 1.19%, LockDownload -- 1.15e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 23:20:01.235 | INFO | PerfReport: 22.39%, LockCompute -- 2.16e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 23:20:01.235 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260506 23:20:01.333 | INFO | Starting a round of Baz IO | 20260506 23:20:04.465 | INFO | Populating RTMetrics report | 20260506 23:20:04.465 | INFO | RTMetrics report populated | 20260506 23:20:04.465 | INFO | Opening tmp RTMetrics json file | 20260506 23:20:04.467 | INFO | Serializing RTMetrics to json | 20260506 23:20:04.487 | INFO | Serialization done, swapping tmp file now... | 20260506 23:20:04.488 | INFO | RTMetrics done | 20260506 23:20:04.592 | INFO | Finished a round of Baz IO: Wrote 1.884 GiB in 3.258 seconds | 20260506 23:20:04.592 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.49% / 0.00% / 66.51% | 20260506 23:20:04.592 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.421838% / 7.94729e-06% / 4.76837e-05% | 20260506 23:20:04.592 | INFO | Baz IO overall: | 20260506 23:20:04.592 | INFO | BytesWritten : 2749913363245 | 20260506 23:20:04.592 | INFO | NumEvents : 1325931043504 | 20260506 23:20:06.374 | INFO | Analyzing chunk frames = [5251072,5251584) | 20260506 23:20:06.374 | INFO | MemInfo: { | 20260506 23:20:06.374 | INFO | "gpu" : " 17 GB", | 20260506 23:20:06.374 | INFO | "huge" : "", | 20260506 23:20:06.374 | INFO | "huge_shared" : " 27 GB", | 20260506 23:20:06.374 | INFO | "malloc" : " 14 GB", | 20260506 23:20:06.374 | INFO | "pinned" : " 29 GB" | 20260506 23:20:06.374 | INFO | } | 20260506 23:20:06.374 | INFO | Perf report for the last 5.14039 seconds: | 20260506 23:20:06.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:20:06.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:06.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:06.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:06.374 | INFO | ANALYSIS : 62.3%, 2.9, 95.6ms, 0 | 20260506 23:20:06.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:06.374 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 23:20:06.374 | INFO | PRE_HQ : 59.6%, 1.8, 56.5ms, 0 | 20260506 23:20:06.374 | INFO | BAZWRITER : 63.0%, 4.2, 135.0ms, 0 | 20260506 23:20:06.374 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:20:06.374 | INFO | Uploaded 36GiB to GPU in 2.03471s (17.6929GiB/s) | 20260506 23:20:06.374 | INFO | Downloaded 38GiB to GPU in 2.19879s (17.2822GiB/s) | 20260506 23:20:11.514 | INFO | Analyzing chunk frames = [5251584,5252096) | 20260506 23:20:11.514 | INFO | MemInfo: { | 20260506 23:20:11.514 | INFO | "gpu" : " 17 GB", | 20260506 23:20:11.514 | INFO | "huge" : "", | 20260506 23:20:11.514 | INFO | "huge_shared" : " 27 GB", | 20260506 23:20:11.514 | INFO | "malloc" : " 14 GB", | 20260506 23:20:11.514 | INFO | "pinned" : " 29 GB" | 20260506 23:20:11.514 | INFO | } | 20260506 23:20:11.514 | INFO | Perf report for the last 5.14017 seconds: | 20260506 23:20:11.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:20:11.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:11.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:11.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:11.515 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:20:11.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:11.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:11.515 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:20:11.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:11.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:11.515 | INFO | Uploaded 36GiB to GPU in 2.00463s (17.9584GiB/s) | 20260506 23:20:11.515 | INFO | Downloaded 34GiB to GPU in 1.93119s (17.6058GiB/s) | 20260506 23:20:16.656 | INFO | Analyzing chunk frames = [5252096,5252608) | 20260506 23:20:16.656 | INFO | MemInfo: { | 20260506 23:20:16.656 | INFO | "gpu" : " 17 GB", | 20260506 23:20:16.656 | INFO | "huge" : "", | 20260506 23:20:16.656 | INFO | "huge_shared" : " 27 GB", | 20260506 23:20:16.656 | INFO | "malloc" : " 14 GB", | 20260506 23:20:16.656 | INFO | "pinned" : " 29 GB" | 20260506 23:20:16.656 | INFO | } | 20260506 23:20:16.656 | INFO | Perf report for the last 5.14162 seconds: | 20260506 23:20:16.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:20:16.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:16.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:16.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:16.656 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 23:20:16.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:16.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:16.656 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:20:16.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:16.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:16.656 | INFO | Uploaded 36GiB to GPU in 1.96138s (18.3544GiB/s) | 20260506 23:20:16.656 | INFO | Downloaded 34GiB to GPU in 1.92948s (17.6213GiB/s) | 20260506 23:20:21.797 | INFO | Analyzing chunk frames = [5252608,5253120) | 20260506 23:20:21.797 | INFO | MemInfo: { | 20260506 23:20:21.797 | INFO | "gpu" : " 17 GB", | 20260506 23:20:21.797 | INFO | "huge" : "", | 20260506 23:20:21.797 | INFO | "huge_shared" : " 27 GB", | 20260506 23:20:21.797 | INFO | "malloc" : " 14 GB", | 20260506 23:20:21.798 | INFO | "pinned" : " 29 GB" | 20260506 23:20:21.798 | INFO | } | 20260506 23:20:21.798 | INFO | Perf report for the last 5.14142 seconds: | 20260506 23:20:21.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:20:21.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:21.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:21.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:21.798 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 23:20:21.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:21.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:21.798 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:20:21.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:21.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:21.798 | INFO | Uploaded 36GiB to GPU in 1.9487s (18.4739GiB/s) | 20260506 23:20:21.798 | INFO | Downloaded 34GiB to GPU in 1.8814s (18.0717GiB/s) | 20260506 23:20:26.935 | INFO | Analyzing chunk frames = [5253120,5253632) | 20260506 23:20:26.935 | INFO | MemInfo: { | 20260506 23:20:26.935 | INFO | "gpu" : " 17 GB", | 20260506 23:20:26.935 | INFO | "huge" : "", | 20260506 23:20:26.935 | INFO | "huge_shared" : " 27 GB", | 20260506 23:20:26.935 | INFO | "malloc" : " 14 GB", | 20260506 23:20:26.935 | INFO | "pinned" : " 29 GB" | 20260506 23:20:26.936 | INFO | } | 20260506 23:20:26.936 | INFO | Perf report for the last 5.13805 seconds: | 20260506 23:20:26.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:20:26.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:26.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:26.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:26.936 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 23:20:26.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:26.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:26.936 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:20:26.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:26.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:26.936 | INFO | Uploaded 36GiB to GPU in 1.98235s (18.1602GiB/s) | 20260506 23:20:26.936 | INFO | Downloaded 34GiB to GPU in 1.93472s (17.5736GiB/s) | 20260506 23:20:32.073 | INFO | Analyzing chunk frames = [5253632,5254144) | 20260506 23:20:32.073 | INFO | MemInfo: { | 20260506 23:20:32.073 | INFO | "gpu" : " 17 GB", | 20260506 23:20:32.073 | INFO | "huge" : "", | 20260506 23:20:32.073 | INFO | "huge_shared" : " 27 GB", | 20260506 23:20:32.073 | INFO | "malloc" : " 14 GB", | 20260506 23:20:32.073 | INFO | "pinned" : " 29 GB" | 20260506 23:20:32.073 | INFO | } | 20260506 23:20:32.073 | INFO | Perf report for the last 5.13765 seconds: | 20260506 23:20:32.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:20:32.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:32.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:32.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:32.073 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 23:20:32.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:32.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:32.073 | INFO | PRE_HQ : 30.8%, 1.0, 16.0ms, 0 | 20260506 23:20:32.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:32.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:32.073 | INFO | Uploaded 36GiB to GPU in 1.98918s (18.0979GiB/s) | 20260506 23:20:32.074 | INFO | Downloaded 34GiB to GPU in 1.94677s (17.4648GiB/s) | 20260506 23:20:37.209 | INFO | Analyzing chunk frames = [5254144,5254656) | 20260506 23:20:37.209 | INFO | MemInfo: { | 20260506 23:20:37.209 | INFO | "gpu" : " 17 GB", | 20260506 23:20:37.209 | INFO | "huge" : "", | 20260506 23:20:37.209 | INFO | "huge_shared" : " 27 GB", | 20260506 23:20:37.209 | INFO | "malloc" : " 14 GB", | 20260506 23:20:37.209 | INFO | "pinned" : " 29 GB" | 20260506 23:20:37.209 | INFO | } | 20260506 23:20:37.209 | INFO | Perf report for the last 5.13564 seconds: | 20260506 23:20:37.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:20:37.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:37.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:37.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:37.209 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:20:37.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:37.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:37.209 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260506 23:20:37.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:37.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:37.209 | INFO | Uploaded 36GiB to GPU in 1.98108s (18.1719GiB/s) | 20260506 23:20:37.209 | INFO | Downloaded 34GiB to GPU in 1.99977s (17.002GiB/s) | 20260506 23:20:42.354 | INFO | Analyzing chunk frames = [5254656,5255168) | 20260506 23:20:42.354 | INFO | MemInfo: { | 20260506 23:20:42.354 | INFO | "gpu" : " 17 GB", | 20260506 23:20:42.354 | INFO | "huge" : "", | 20260506 23:20:42.354 | INFO | "huge_shared" : " 27 GB", | 20260506 23:20:42.354 | INFO | "malloc" : " 14 GB", | 20260506 23:20:42.354 | INFO | "pinned" : " 29 GB" | 20260506 23:20:42.354 | INFO | } | 20260506 23:20:42.354 | INFO | Perf report for the last 5.14522 seconds: | 20260506 23:20:42.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:20:42.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:42.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:42.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:42.354 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 23:20:42.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:42.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:42.354 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:20:42.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:42.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:42.355 | INFO | Uploaded 36GiB to GPU in 1.98512s (18.1349GiB/s) | 20260506 23:20:42.355 | INFO | Downloaded 34GiB to GPU in 1.98835s (17.0996GiB/s) | 20260506 23:20:42.453 | INFO | Starting a round of Baz IO | 20260506 23:20:45.586 | INFO | Populating RTMetrics report | 20260506 23:20:45.586 | INFO | RTMetrics report populated | 20260506 23:20:45.586 | INFO | Opening tmp RTMetrics json file | 20260506 23:20:45.588 | INFO | Serializing RTMetrics to json | 20260506 23:20:45.630 | INFO | Serialization done, swapping tmp file now... | 20260506 23:20:45.689 | INFO | RTMetrics done | 20260506 23:20:45.759 | INFO | Finished a round of Baz IO: Wrote 1.885 GiB in 3.306 seconds | 20260506 23:20:45.759 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.49% / 0.00% / 66.51% | 20260506 23:20:45.759 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.421456% / 3.97364e-06% / 5.16574e-05% | 20260506 23:20:45.760 | INFO | Baz IO overall: | 20260506 23:20:45.760 | INFO | BytesWritten : 2751937359108 | 20260506 23:20:45.760 | INFO | NumEvents : 1326894604605 | 20260506 23:20:47.490 | INFO | Analyzing chunk frames = [5255168,5255680) | 20260506 23:20:47.490 | INFO | MemInfo: { | 20260506 23:20:47.490 | INFO | "gpu" : " 17 GB", | 20260506 23:20:47.491 | INFO | "huge" : "", | 20260506 23:20:47.491 | INFO | "huge_shared" : " 27 GB", | 20260506 23:20:47.491 | INFO | "malloc" : " 14 GB", | 20260506 23:20:47.491 | INFO | "pinned" : " 29 GB" | 20260506 23:20:47.491 | INFO | } | 20260506 23:20:47.491 | INFO | Perf report for the last 5.13642 seconds: | 20260506 23:20:47.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:20:47.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:47.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:47.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:47.491 | INFO | ANALYSIS : 62.3%, 2.9, 95.3ms, 0 | 20260506 23:20:47.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:47.491 | INFO | RT_METRICS : 44.2%, 1.0, 23.1ms, 0 | 20260506 23:20:47.491 | INFO | PRE_HQ : 59.7%, 1.8, 56.5ms, 0 | 20260506 23:20:47.491 | INFO | BAZWRITER : 64.0%, 4.4, 143.7ms, 0 | 20260506 23:20:47.491 | INFO | BAZ_SAMPLING : 2.7%, 1.0, 1.4ms, 0 | 20260506 23:20:47.491 | INFO | Uploaded 36GiB to GPU in 2.04351s (17.6167GiB/s) | 20260506 23:20:47.491 | INFO | Downloaded 38GiB to GPU in 2.20349s (17.2454GiB/s) | 20260506 23:20:52.632 | INFO | Analyzing chunk frames = [5255680,5256192) | 20260506 23:20:52.632 | INFO | MemInfo: { | 20260506 23:20:52.632 | INFO | "gpu" : " 17 GB", | 20260506 23:20:52.632 | INFO | "huge" : "", | 20260506 23:20:52.632 | INFO | "huge_shared" : " 27 GB", | 20260506 23:20:52.632 | INFO | "malloc" : " 14 GB", | 20260506 23:20:52.632 | INFO | "pinned" : " 29 GB" | 20260506 23:20:52.632 | INFO | } | 20260506 23:20:52.632 | INFO | Perf report for the last 5.14119 seconds: | 20260506 23:20:52.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:20:52.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:52.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:52.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:52.632 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:20:52.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:52.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:52.632 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:20:52.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:52.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:52.632 | INFO | Uploaded 36GiB to GPU in 2.00944s (17.9155GiB/s) | 20260506 23:20:52.632 | INFO | Downloaded 34GiB to GPU in 1.93236s (17.5951GiB/s) | 20260506 23:20:57.773 | INFO | Analyzing chunk frames = [5256192,5256704) | 20260506 23:20:57.773 | INFO | MemInfo: { | 20260506 23:20:57.773 | INFO | "gpu" : " 17 GB", | 20260506 23:20:57.773 | INFO | "huge" : "", | 20260506 23:20:57.773 | INFO | "huge_shared" : " 27 GB", | 20260506 23:20:57.773 | INFO | "malloc" : " 14 GB", | 20260506 23:20:57.773 | INFO | "pinned" : " 29 GB" | 20260506 23:20:57.773 | INFO | } | 20260506 23:20:57.773 | INFO | Perf report for the last 5.14098 seconds: | 20260506 23:20:57.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:20:57.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:57.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:57.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:57.773 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:20:57.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:57.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:57.773 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:20:57.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:57.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:20:57.773 | INFO | Uploaded 36GiB to GPU in 1.96357s (18.3339GiB/s) | 20260506 23:20:57.773 | INFO | Downloaded 34GiB to GPU in 1.9318s (17.6002GiB/s) | 20260506 23:21:02.911 | INFO | Analyzing chunk frames = [5256704,5257216) | 20260506 23:21:02.911 | INFO | MemInfo: { | 20260506 23:21:02.911 | INFO | "gpu" : " 17 GB", | 20260506 23:21:02.911 | INFO | "huge" : "", | 20260506 23:21:02.911 | INFO | "huge_shared" : " 27 GB", | 20260506 23:21:02.911 | INFO | "malloc" : " 14 GB", | 20260506 23:21:02.911 | INFO | "pinned" : " 29 GB" | 20260506 23:21:02.911 | INFO | } | 20260506 23:21:02.911 | INFO | Perf report for the last 5.13865 seconds: | 20260506 23:21:02.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:21:02.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:02.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:02.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:02.912 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 23:21:02.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:02.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:02.912 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:21:02.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:02.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:02.912 | INFO | Uploaded 36GiB to GPU in 2.01279s (17.8856GiB/s) | 20260506 23:21:02.912 | INFO | Downloaded 34GiB to GPU in 1.91419s (17.7621GiB/s) | 20260506 23:21:08.051 | INFO | Analyzing chunk frames = [5257216,5257728) | 20260506 23:21:08.051 | INFO | MemInfo: { | 20260506 23:21:08.051 | INFO | "gpu" : " 17 GB", | 20260506 23:21:08.051 | INFO | "huge" : "", | 20260506 23:21:08.051 | INFO | "huge_shared" : " 27 GB", | 20260506 23:21:08.051 | INFO | "malloc" : " 14 GB", | 20260506 23:21:08.051 | INFO | "pinned" : " 29 GB" | 20260506 23:21:08.051 | INFO | } | 20260506 23:21:08.051 | INFO | Perf report for the last 5.13925 seconds: | 20260506 23:21:08.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:21:08.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:08.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:08.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:08.051 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 23:21:08.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:08.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:08.051 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:21:08.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:08.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:08.051 | INFO | Uploaded 36GiB to GPU in 1.98212s (18.1624GiB/s) | 20260506 23:21:08.051 | INFO | Downloaded 34GiB to GPU in 1.939s (17.5348GiB/s) | 20260506 23:21:13.186 | INFO | Analyzing chunk frames = [5257728,5258240) | 20260506 23:21:13.186 | INFO | MemInfo: { | 20260506 23:21:13.186 | INFO | "gpu" : " 17 GB", | 20260506 23:21:13.186 | INFO | "huge" : "", | 20260506 23:21:13.187 | INFO | "huge_shared" : " 27 GB", | 20260506 23:21:13.187 | INFO | "malloc" : " 14 GB", | 20260506 23:21:13.187 | INFO | "pinned" : " 29 GB" | 20260506 23:21:13.187 | INFO | } | 20260506 23:21:13.187 | INFO | Perf report for the last 5.1359 seconds: | 20260506 23:21:13.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:21:13.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:13.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:13.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:13.187 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 23:21:13.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:13.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:13.187 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 23:21:13.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:13.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:13.187 | INFO | Uploaded 36GiB to GPU in 2.04746s (17.5828GiB/s) | 20260506 23:21:13.187 | INFO | Downloaded 34GiB to GPU in 1.95009s (17.4351GiB/s) | 20260506 23:21:18.322 | INFO | Analyzing chunk frames = [5258240,5258752) | 20260506 23:21:18.322 | INFO | MemInfo: { | 20260506 23:21:18.322 | INFO | "gpu" : " 17 GB", | 20260506 23:21:18.322 | INFO | "huge" : "", | 20260506 23:21:18.322 | INFO | "huge_shared" : " 27 GB", | 20260506 23:21:18.322 | INFO | "malloc" : " 14 GB", | 20260506 23:21:18.322 | INFO | "pinned" : " 29 GB" | 20260506 23:21:18.322 | INFO | } | 20260506 23:21:18.322 | INFO | Perf report for the last 5.1356 seconds: | 20260506 23:21:18.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:21:18.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:18.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:18.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:18.322 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 23:21:18.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:18.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:18.322 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 23:21:18.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:18.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:18.323 | INFO | Uploaded 36GiB to GPU in 1.96492s (18.3214GiB/s) | 20260506 23:21:18.323 | INFO | Downloaded 34GiB to GPU in 1.98862s (17.0973GiB/s) | 20260506 23:21:23.462 | INFO | Analyzing chunk frames = [5258752,5259264) | 20260506 23:21:23.462 | INFO | MemInfo: { | 20260506 23:21:23.462 | INFO | "gpu" : " 17 GB", | 20260506 23:21:23.462 | INFO | "huge" : "", | 20260506 23:21:23.462 | INFO | "huge_shared" : " 27 GB", | 20260506 23:21:23.462 | INFO | "malloc" : " 14 GB", | 20260506 23:21:23.462 | INFO | "pinned" : " 29 GB" | 20260506 23:21:23.462 | INFO | } | 20260506 23:21:23.462 | INFO | Perf report for the last 5.13961 seconds: | 20260506 23:21:23.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:21:23.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:23.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:23.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:23.462 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260506 23:21:23.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:23.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:23.462 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 23:21:23.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:23.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:23.462 | INFO | Uploaded 36GiB to GPU in 2.03925s (17.6536GiB/s) | 20260506 23:21:23.462 | INFO | Downloaded 34GiB to GPU in 1.9741s (17.223GiB/s) | 20260506 23:21:23.462 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:21:23.462 | INFO | Total mem usage: 18 GiB | 20260506 23:21:23.462 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:21:23.462 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -21.9%) | 20260506 23:21:23.462 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:21:23.462 | INFO | High Water Breakdown: | 20260506 23:21:23.462 | INFO | 60.6% : 8.96 GiB ( 78.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:21:23.462 | INFO | 39.2% : 5.79 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:21:23.462 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:21:23.462 | INFO | ------------------------------------------ | 20260506 23:21:23.462 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:21:23.462 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:21:23.462 | INFO | Total mem usage: 26.5 GiB | 20260506 23:21:23.462 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:21:23.462 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:21:23.462 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:21:23.462 | INFO | High Water Breakdown: | 20260506 23:21:23.462 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:21:23.462 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:21:23.462 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:21:23.462 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:21:23.462 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:21:23.462 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:21:23.462 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:21:23.462 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:21:23.462 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:21:23.462 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:21:23.462 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:21:23.462 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:21:23.462 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:21:23.462 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:21:23.463 | INFO | ------------------------------------------ | 20260506 23:21:23.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:21:23.463 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:21:23.463 | INFO | Total mem usage: 25.6 GiB | 20260506 23:21:23.463 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:21:23.463 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:21:23.463 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:21:23.463 | INFO | High Water Breakdown: | 20260506 23:21:23.463 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:21:23.463 | INFO | ------------------------------------------ | 20260506 23:21:23.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:21:23.463 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:21:23.463 | INFO | Total mem usage: 15.7 GiB | 20260506 23:21:23.463 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:21:23.463 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:21:23.463 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:21:23.463 | INFO | High Water Breakdown: | 20260506 23:21:23.463 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:21:23.463 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:21:23.463 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:21:23.463 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:21:23.463 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:21:23.463 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:21:23.463 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:21:23.463 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:21:23.463 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:21:23.463 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:21:23.463 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:21:23.463 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:21:23.463 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:21:23.463 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:21:23.463 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:21:23.463 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:21:23.463 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:21:23.463 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:21:23.463 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:21:23.463 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:21:23.463 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:21:23.463 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:21:23.463 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:21:23.463 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:21:23.463 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:21:23.463 | INFO | ------------------------------------------ | 20260506 23:21:23.463 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:21:23.463 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:21:23.464 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:21:23.464 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:21:23.464 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.58e+01, 1.34e+01 / 2.58e+00 | 20260506 23:21:23.464 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 9.98e-02, 9.52e-03 / 6.89e-03 | 20260506 23:21:23.464 | INFO | PerfReport: 11.66%, Baseline -- 3.62e+00 / 4.32e+00, 5.47e-01 / 4.04e-01 | 20260506 23:21:23.464 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.43e+00, 1.77e-01 / 1.43e-01 | 20260506 23:21:23.464 | INFO | PerfReport: 16.07%, DME -- 3.99e+01 / 4.47e+01, 4.21e+00 / 2.54e+00 | 20260506 23:21:23.464 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.36e+00 / 1.44e+00 | 20260506 23:21:23.464 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.19e+00 / 2.74e+00, 3.77e-01 / 1.89e-01 | 20260506 23:21:23.464 | INFO | PerfReport: 3.20%, Metrics -- 9.92e-01 / 1.35e+00, 2.63e-01 / 1.59e-01 | 20260506 23:21:23.464 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.38e-01 / 3.88e-01, 5.38e-02 / 2.15e-02 | 20260506 23:21:23.464 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:21:23.464 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:21:23.464 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:21:23.464 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 23:21:23.464 | INFO | PerfReport: 21.55%, Upload -- 2.08e+01 / 0.00e+00, 6.89e+00 / 0.00e+00 | 20260506 23:21:23.464 | INFO | PerfReport: 21.34%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 23:21:23.464 | INFO | PerfReport: 32.14%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 23:21:23.464 | INFO | PerfReport: 1.28%, LockUpload -- 1.23e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260506 23:21:23.464 | INFO | PerfReport: 1.31%, LockDownload -- 1.27e+00 / 0.00e+00, 3.18e+00 / 0.00e+00 | 20260506 23:21:23.464 | INFO | PerfReport: 22.37%, LockCompute -- 2.16e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:21:23.464 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260506 23:21:23.570 | INFO | Starting a round of Baz IO | 20260506 23:21:26.726 | INFO | Populating RTMetrics report | 20260506 23:21:26.726 | INFO | RTMetrics report populated | 20260506 23:21:26.726 | INFO | Opening tmp RTMetrics json file | 20260506 23:21:26.731 | INFO | Serializing RTMetrics to json | 20260506 23:21:26.748 | INFO | Serialization done, swapping tmp file now... | 20260506 23:21:26.829 | INFO | RTMetrics done | 20260506 23:21:26.951 | INFO | Finished a round of Baz IO: Wrote 1.883 GiB in 3.38 seconds | 20260506 23:21:26.951 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.48% / 0.00% / 66.51% | 20260506 23:21:26.951 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.421703% / 0% / 5.96046e-05% | 20260506 23:21:26.951 | INFO | Baz IO overall: | 20260506 23:21:26.951 | INFO | BytesWritten : 2753959592415 | 20260506 23:21:26.951 | INFO | NumEvents : 1327857395886 | 20260506 23:21:28.605 | INFO | Analyzing chunk frames = [5259264,5259776) | 20260506 23:21:28.605 | INFO | MemInfo: { | 20260506 23:21:28.605 | INFO | "gpu" : " 17 GB", | 20260506 23:21:28.605 | INFO | "huge" : "", | 20260506 23:21:28.605 | INFO | "huge_shared" : " 27 GB", | 20260506 23:21:28.605 | INFO | "malloc" : " 14 GB", | 20260506 23:21:28.606 | INFO | "pinned" : " 29 GB" | 20260506 23:21:28.606 | INFO | } | 20260506 23:21:28.606 | INFO | Perf report for the last 5.14374 seconds: | 20260506 23:21:28.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:21:28.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:28.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:28.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:28.606 | INFO | ANALYSIS : 62.4%, 2.9, 93.7ms, 0 | 20260506 23:21:28.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:28.606 | INFO | RT_METRICS : 47.8%, 1.0, 25.1ms, 0 | 20260506 23:21:28.606 | INFO | PRE_HQ : 60.3%, 2.0, 61.6ms, 0 | 20260506 23:21:28.606 | INFO | BAZWRITER : 65.5%, 4.6, 154.5ms, 0 | 20260506 23:21:28.606 | INFO | BAZ_SAMPLING : 3.9%, 1.0, 2.0ms, 0 | 20260506 23:21:28.606 | INFO | Uploaded 36GiB to GPU in 2.03657s (17.6768GiB/s) | 20260506 23:21:28.606 | INFO | Downloaded 38GiB to GPU in 2.2014s (17.2617GiB/s) | 20260506 23:21:33.750 | INFO | Analyzing chunk frames = [5259776,5260288) | 20260506 23:21:33.750 | INFO | MemInfo: { | 20260506 23:21:33.750 | INFO | "gpu" : " 17 GB", | 20260506 23:21:33.750 | INFO | "huge" : "", | 20260506 23:21:33.750 | INFO | "huge_shared" : " 27 GB", | 20260506 23:21:33.750 | INFO | "malloc" : " 14 GB", | 20260506 23:21:33.750 | INFO | "pinned" : " 29 GB" | 20260506 23:21:33.750 | INFO | } | 20260506 23:21:33.750 | INFO | Perf report for the last 5.14411 seconds: | 20260506 23:21:33.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:21:33.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:33.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:33.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:33.750 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:21:33.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:33.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:33.750 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:21:33.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:33.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:33.750 | INFO | Uploaded 36GiB to GPU in 2.01892s (17.8314GiB/s) | 20260506 23:21:33.750 | INFO | Downloaded 34GiB to GPU in 1.92652s (17.6484GiB/s) | 20260506 23:21:38.886 | INFO | Analyzing chunk frames = [5260288,5260800) | 20260506 23:21:38.886 | INFO | MemInfo: { | 20260506 23:21:38.886 | INFO | "gpu" : " 17 GB", | 20260506 23:21:38.886 | INFO | "huge" : "", | 20260506 23:21:38.886 | INFO | "huge_shared" : " 27 GB", | 20260506 23:21:38.886 | INFO | "malloc" : " 14 GB", | 20260506 23:21:38.886 | INFO | "pinned" : " 29 GB" | 20260506 23:21:38.886 | INFO | } | 20260506 23:21:38.886 | INFO | Perf report for the last 5.13663 seconds: | 20260506 23:21:38.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:21:38.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:38.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:38.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:38.886 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 23:21:38.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:38.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:38.887 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:21:38.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:38.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:38.887 | INFO | Uploaded 36GiB to GPU in 2.00061s (17.9946GiB/s) | 20260506 23:21:38.887 | INFO | Downloaded 34GiB to GPU in 1.93219s (17.5966GiB/s) | 20260506 23:21:44.026 | INFO | Analyzing chunk frames = [5260800,5261312) | 20260506 23:21:44.026 | INFO | MemInfo: { | 20260506 23:21:44.026 | INFO | "gpu" : " 17 GB", | 20260506 23:21:44.026 | INFO | "huge" : "", | 20260506 23:21:44.026 | INFO | "huge_shared" : " 27 GB", | 20260506 23:21:44.026 | INFO | "malloc" : " 14 GB", | 20260506 23:21:44.026 | INFO | "pinned" : " 29 GB" | 20260506 23:21:44.026 | INFO | } | 20260506 23:21:44.026 | INFO | Perf report for the last 5.13976 seconds: | 20260506 23:21:44.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:21:44.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:44.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:44.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:44.026 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 23:21:44.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:44.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:44.026 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:21:44.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:44.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:44.026 | INFO | Uploaded 36GiB to GPU in 1.97481s (18.2296GiB/s) | 20260506 23:21:44.026 | INFO | Downloaded 34GiB to GPU in 1.90436s (17.8537GiB/s) | 20260506 23:21:49.165 | INFO | Analyzing chunk frames = [5261312,5261824) | 20260506 23:21:49.165 | INFO | MemInfo: { | 20260506 23:21:49.165 | INFO | "gpu" : " 17 GB", | 20260506 23:21:49.165 | INFO | "huge" : "", | 20260506 23:21:49.166 | INFO | "huge_shared" : " 27 GB", | 20260506 23:21:49.166 | INFO | "malloc" : " 14 GB", | 20260506 23:21:49.166 | INFO | "pinned" : " 29 GB" | 20260506 23:21:49.166 | INFO | } | 20260506 23:21:49.166 | INFO | Perf report for the last 5.13943 seconds: | 20260506 23:21:49.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:21:49.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:49.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:49.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:49.166 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 23:21:49.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:49.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:49.166 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:21:49.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:49.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:49.166 | INFO | Uploaded 36GiB to GPU in 2.00101s (17.9909GiB/s) | 20260506 23:21:49.166 | INFO | Downloaded 34GiB to GPU in 1.92346s (17.6764GiB/s) | 20260506 23:21:54.305 | INFO | Analyzing chunk frames = [5261824,5262336) | 20260506 23:21:54.305 | INFO | MemInfo: { | 20260506 23:21:54.305 | INFO | "gpu" : " 17 GB", | 20260506 23:21:54.305 | INFO | "huge" : "", | 20260506 23:21:54.305 | INFO | "huge_shared" : " 27 GB", | 20260506 23:21:54.305 | INFO | "malloc" : " 14 GB", | 20260506 23:21:54.305 | INFO | "pinned" : " 29 GB" | 20260506 23:21:54.305 | INFO | } | 20260506 23:21:54.305 | INFO | Perf report for the last 5.13921 seconds: | 20260506 23:21:54.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:21:54.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:54.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:54.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:54.305 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 23:21:54.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:54.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:54.305 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 23:21:54.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:54.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:54.305 | INFO | Uploaded 36GiB to GPU in 2.02834s (17.7485GiB/s) | 20260506 23:21:54.305 | INFO | Downloaded 34GiB to GPU in 1.93613s (17.5608GiB/s) | 20260506 23:21:59.441 | INFO | Analyzing chunk frames = [5262336,5262848) | 20260506 23:21:59.441 | INFO | MemInfo: { | 20260506 23:21:59.441 | INFO | "gpu" : " 17 GB", | 20260506 23:21:59.441 | INFO | "huge" : "", | 20260506 23:21:59.441 | INFO | "huge_shared" : " 27 GB", | 20260506 23:21:59.441 | INFO | "malloc" : " 14 GB", | 20260506 23:21:59.441 | INFO | "pinned" : " 29 GB" | 20260506 23:21:59.441 | INFO | } | 20260506 23:21:59.441 | INFO | Perf report for the last 5.13596 seconds: | 20260506 23:21:59.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:21:59.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:59.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:59.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:59.441 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 23:21:59.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:59.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:59.441 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 23:21:59.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:59.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:21:59.441 | INFO | Uploaded 36GiB to GPU in 1.95591s (18.4057GiB/s) | 20260506 23:21:59.441 | INFO | Downloaded 34GiB to GPU in 1.98344s (17.1419GiB/s) | 20260506 23:22:04.575 | INFO | Analyzing chunk frames = [5262848,5263360) | 20260506 23:22:04.575 | INFO | MemInfo: { | 20260506 23:22:04.575 | INFO | "gpu" : " 17 GB", | 20260506 23:22:04.575 | INFO | "huge" : "", | 20260506 23:22:04.575 | INFO | "huge_shared" : " 27 GB", | 20260506 23:22:04.575 | INFO | "malloc" : " 14 GB", | 20260506 23:22:04.575 | INFO | "pinned" : " 29 GB" | 20260506 23:22:04.575 | INFO | } | 20260506 23:22:04.575 | INFO | Perf report for the last 5.13417 seconds: | 20260506 23:22:04.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:22:04.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:04.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:04.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:04.575 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 23:22:04.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:04.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:04.575 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 23:22:04.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:04.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:04.575 | INFO | Uploaded 36GiB to GPU in 2.07099s (17.383GiB/s) | 20260506 23:22:04.575 | INFO | Downloaded 34GiB to GPU in 1.99784s (17.0184GiB/s) | 20260506 23:22:04.689 | INFO | Starting a round of Baz IO | 20260506 23:22:07.811 | INFO | Populating RTMetrics report | 20260506 23:22:07.812 | INFO | RTMetrics report populated | 20260506 23:22:07.812 | INFO | Opening tmp RTMetrics json file | 20260506 23:22:07.821 | INFO | Serializing RTMetrics to json | 20260506 23:22:07.823 | INFO | Serialization done, swapping tmp file now... | 20260506 23:22:07.827 | INFO | RTMetrics done | 20260506 23:22:07.988 | INFO | Finished a round of Baz IO: Wrote 1.882 GiB in 3.299 seconds | 20260506 23:22:07.988 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.48% / 0.00% / 66.52% | 20260506 23:22:07.988 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.419033% / 7.94729e-06% / 4.76837e-05% | 20260506 23:22:07.988 | INFO | Baz IO overall: | 20260506 23:22:07.988 | INFO | BytesWritten : 2755980179399 | 20260506 23:22:07.988 | INFO | NumEvents : 1328819510734 | 20260506 23:22:09.721 | INFO | Analyzing chunk frames = [5263360,5263872) | 20260506 23:22:09.721 | INFO | MemInfo: { | 20260506 23:22:09.721 | INFO | "gpu" : " 17 GB", | 20260506 23:22:09.721 | INFO | "huge" : "", | 20260506 23:22:09.721 | INFO | "huge_shared" : " 27 GB", | 20260506 23:22:09.721 | INFO | "malloc" : " 14 GB", | 20260506 23:22:09.721 | INFO | "pinned" : " 29 GB" | 20260506 23:22:09.721 | INFO | } | 20260506 23:22:09.721 | INFO | Perf report for the last 5.14599 seconds: | 20260506 23:22:09.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:22:09.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:09.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:09.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:09.721 | INFO | ANALYSIS : 62.4%, 2.9, 95.2ms, 0 | 20260506 23:22:09.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:09.721 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 23:22:09.721 | INFO | PRE_HQ : 59.4%, 1.8, 56.8ms, 0 | 20260506 23:22:09.721 | INFO | BAZWRITER : 63.7%, 4.8, 157.1ms, 0 | 20260506 23:22:09.721 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 23:22:09.721 | INFO | Uploaded 36GiB to GPU in 2.07622s (17.3392GiB/s) | 20260506 23:22:09.721 | INFO | Downloaded 38GiB to GPU in 2.22206s (17.1013GiB/s) | 20260506 23:22:14.863 | INFO | Analyzing chunk frames = [5263872,5264384) | 20260506 23:22:14.863 | INFO | MemInfo: { | 20260506 23:22:14.863 | INFO | "gpu" : " 17 GB", | 20260506 23:22:14.863 | INFO | "huge" : "", | 20260506 23:22:14.863 | INFO | "huge_shared" : " 27 GB", | 20260506 23:22:14.863 | INFO | "malloc" : " 14 GB", | 20260506 23:22:14.863 | INFO | "pinned" : " 29 GB" | 20260506 23:22:14.863 | INFO | } | 20260506 23:22:14.863 | INFO | Perf report for the last 5.14211 seconds: | 20260506 23:22:14.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:22:14.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:14.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:14.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:14.863 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:22:14.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:14.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:14.863 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:22:14.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:14.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:14.863 | INFO | Uploaded 36GiB to GPU in 2.00273s (17.9755GiB/s) | 20260506 23:22:14.863 | INFO | Downloaded 34GiB to GPU in 1.92855s (17.6298GiB/s) | 20260506 23:22:20.004 | INFO | Analyzing chunk frames = [5264384,5264896) | 20260506 23:22:20.004 | INFO | MemInfo: { | 20260506 23:22:20.004 | INFO | "gpu" : " 17 GB", | 20260506 23:22:20.004 | INFO | "huge" : "", | 20260506 23:22:20.004 | INFO | "huge_shared" : " 27 GB", | 20260506 23:22:20.004 | INFO | "malloc" : " 14 GB", | 20260506 23:22:20.004 | INFO | "pinned" : " 29 GB" | 20260506 23:22:20.004 | INFO | } | 20260506 23:22:20.004 | INFO | Perf report for the last 5.1411 seconds: | 20260506 23:22:20.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:22:20.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:20.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:20.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:20.004 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 23:22:20.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:20.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:20.004 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:22:20.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:20.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:20.004 | INFO | Uploaded 36GiB to GPU in 2.00121s (17.9891GiB/s) | 20260506 23:22:20.004 | INFO | Downloaded 34GiB to GPU in 1.93314s (17.5879GiB/s) | 20260506 23:22:25.143 | INFO | Analyzing chunk frames = [5264896,5265408) | 20260506 23:22:25.143 | INFO | MemInfo: { | 20260506 23:22:25.143 | INFO | "gpu" : " 17 GB", | 20260506 23:22:25.143 | INFO | "huge" : "", | 20260506 23:22:25.143 | INFO | "huge_shared" : " 27 GB", | 20260506 23:22:25.143 | INFO | "malloc" : " 14 GB", | 20260506 23:22:25.143 | INFO | "pinned" : " 29 GB" | 20260506 23:22:25.143 | INFO | } | 20260506 23:22:25.143 | INFO | Perf report for the last 5.13902 seconds: | 20260506 23:22:25.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:22:25.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:25.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:25.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:25.143 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 23:22:25.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:25.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:25.143 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:22:25.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:25.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:25.143 | INFO | Uploaded 36GiB to GPU in 1.98085s (18.174GiB/s) | 20260506 23:22:25.143 | INFO | Downloaded 34GiB to GPU in 1.90532s (17.8447GiB/s) | 20260506 23:22:30.276 | INFO | Analyzing chunk frames = [5265408,5265920) | 20260506 23:22:30.276 | INFO | MemInfo: { | 20260506 23:22:30.276 | INFO | "gpu" : " 17 GB", | 20260506 23:22:30.276 | INFO | "huge" : "", | 20260506 23:22:30.276 | INFO | "huge_shared" : " 27 GB", | 20260506 23:22:30.276 | INFO | "malloc" : " 14 GB", | 20260506 23:22:30.276 | INFO | "pinned" : " 29 GB" | 20260506 23:22:30.276 | INFO | } | 20260506 23:22:30.276 | INFO | Perf report for the last 5.13281 seconds: | 20260506 23:22:30.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:22:30.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:30.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:30.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:30.276 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 23:22:30.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:30.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:30.276 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:22:30.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:30.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:30.276 | INFO | Uploaded 36GiB to GPU in 2.00685s (17.9385GiB/s) | 20260506 23:22:30.276 | INFO | Downloaded 34GiB to GPU in 1.94105s (17.5163GiB/s) | 20260506 23:22:35.416 | INFO | Analyzing chunk frames = [5265920,5266432) | 20260506 23:22:35.416 | INFO | MemInfo: { | 20260506 23:22:35.416 | INFO | "gpu" : " 17 GB", | 20260506 23:22:35.416 | INFO | "huge" : "", | 20260506 23:22:35.416 | INFO | "huge_shared" : " 27 GB", | 20260506 23:22:35.416 | INFO | "malloc" : " 14 GB", | 20260506 23:22:35.416 | INFO | "pinned" : " 29 GB" | 20260506 23:22:35.417 | INFO | } | 20260506 23:22:35.417 | INFO | Perf report for the last 5.14048 seconds: | 20260506 23:22:35.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:22:35.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:35.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:35.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:35.417 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 23:22:35.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:35.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:35.417 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 23:22:35.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:35.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:35.417 | INFO | Uploaded 36GiB to GPU in 2.02763s (17.7547GiB/s) | 20260506 23:22:35.417 | INFO | Downloaded 34GiB to GPU in 1.93923s (17.5327GiB/s) | 20260506 23:22:40.550 | INFO | Analyzing chunk frames = [5266432,5266944) | 20260506 23:22:40.550 | INFO | MemInfo: { | 20260506 23:22:40.550 | INFO | "gpu" : " 17 GB", | 20260506 23:22:40.550 | INFO | "huge" : "", | 20260506 23:22:40.550 | INFO | "huge_shared" : " 27 GB", | 20260506 23:22:40.550 | INFO | "malloc" : " 14 GB", | 20260506 23:22:40.550 | INFO | "pinned" : " 29 GB" | 20260506 23:22:40.550 | INFO | } | 20260506 23:22:40.550 | INFO | Perf report for the last 5.13377 seconds: | 20260506 23:22:40.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:22:40.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:40.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:40.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:40.550 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 23:22:40.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:40.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:40.550 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 23:22:40.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:40.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:40.551 | INFO | Uploaded 36GiB to GPU in 1.97523s (18.2257GiB/s) | 20260506 23:22:40.551 | INFO | Downloaded 34GiB to GPU in 1.99917s (17.007GiB/s) | 20260506 23:22:45.696 | INFO | Analyzing chunk frames = [5266944,5267456) | 20260506 23:22:45.696 | INFO | MemInfo: { | 20260506 23:22:45.696 | INFO | "gpu" : " 17 GB", | 20260506 23:22:45.696 | INFO | "huge" : "", | 20260506 23:22:45.696 | INFO | "huge_shared" : " 27 GB", | 20260506 23:22:45.696 | INFO | "malloc" : " 14 GB", | 20260506 23:22:45.696 | INFO | "pinned" : " 29 GB" | 20260506 23:22:45.696 | INFO | } | 20260506 23:22:45.696 | INFO | Perf report for the last 5.14596 seconds: | 20260506 23:22:45.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:22:45.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:45.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:45.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:45.696 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 23:22:45.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:45.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:45.696 | INFO | PRE_HQ : 30.5%, 1.0, 15.9ms, 0 | 20260506 23:22:45.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:45.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:45.696 | INFO | Uploaded 36GiB to GPU in 1.99133s (18.0784GiB/s) | 20260506 23:22:45.696 | INFO | Downloaded 34GiB to GPU in 1.9871s (17.1103GiB/s) | 20260506 23:22:45.697 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:22:45.697 | INFO | Total mem usage: 18 GiB | 20260506 23:22:45.697 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:22:45.697 | INFO | Recent High Water Marks Sum: 15.3 GiB (Savings: -17.5%) | 20260506 23:22:45.697 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:22:45.697 | INFO | High Water Breakdown: | 20260506 23:22:45.697 | INFO | 61.2% : 9.39 GiB ( 82.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:22:45.697 | INFO | 38.6% : 5.92 GiB ( 88.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:22:45.697 | INFO | 0.13% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:22:45.697 | INFO | ------------------------------------------ | 20260506 23:22:45.697 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:22:45.697 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:22:45.697 | INFO | Total mem usage: 26.5 GiB | 20260506 23:22:45.697 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:22:45.697 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 23:22:45.697 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 23:22:45.697 | INFO | High Water Breakdown: | 20260506 23:22:45.697 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:22:45.697 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:22:45.697 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:22:45.697 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:22:45.697 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:22:45.697 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:22:45.697 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:22:45.697 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:22:45.697 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:22:45.697 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:22:45.697 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:22:45.697 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:22:45.697 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:22:45.697 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:22:45.697 | INFO | ------------------------------------------ | 20260506 23:22:45.697 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:22:45.697 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:22:45.697 | INFO | Total mem usage: 25.6 GiB | 20260506 23:22:45.697 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:22:45.697 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:22:45.697 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:22:45.697 | INFO | High Water Breakdown: | 20260506 23:22:45.697 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:22:45.697 | INFO | ------------------------------------------ | 20260506 23:22:45.697 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:22:45.697 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:22:45.697 | INFO | Total mem usage: 15.7 GiB | 20260506 23:22:45.697 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:22:45.697 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:22:45.697 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 23:22:45.697 | INFO | High Water Breakdown: | 20260506 23:22:45.697 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:22:45.697 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:22:45.697 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:22:45.697 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:22:45.697 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:22:45.697 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:22:45.697 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:22:45.697 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:22:45.697 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:22:45.697 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:22:45.697 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:22:45.697 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:22:45.697 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:22:45.697 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:22:45.697 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:22:45.697 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:22:45.697 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:22:45.697 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:22:45.697 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:22:45.697 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:22:45.697 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:22:45.697 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:22:45.697 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:22:45.697 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:22:45.697 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:22:45.697 | INFO | ------------------------------------------ | 20260506 23:22:45.697 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:22:45.697 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:22:45.697 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:22:45.697 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:22:45.697 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.59e+01, 1.36e+01 / 2.17e+00 | 20260506 23:22:45.697 | INFO | PerfReport: 0.30%, AggregateStats -- 9.17e-02 / 1.03e-01, 1.07e-02 / 6.50e-03 | 20260506 23:22:45.697 | INFO | PerfReport: 11.67%, Baseline -- 3.62e+00 / 4.42e+00, 5.45e-01 / 2.65e-01 | 20260506 23:22:45.697 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.45e+00, 1.77e-01 / 1.13e-01 | 20260506 23:22:45.697 | INFO | PerfReport: 16.07%, DME -- 3.99e+01 / 4.46e+01, 4.13e+00 / 2.66e+00 | 20260506 23:22:45.697 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.33e+00 / 1.81e+00 | 20260506 23:22:45.697 | INFO | PerfReport: 7.09%, PulseAccumulating -- 2.20e+00 / 2.72e+00, 3.85e-01 / 2.55e-01 | 20260506 23:22:45.697 | INFO | PerfReport: 3.20%, Metrics -- 9.91e-01 / 1.31e+00, 2.61e-01 / 1.56e-01 | 20260506 23:22:45.697 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.38e-01 / 3.97e-01, 5.28e-02 / 3.61e-02 | 20260506 23:22:45.697 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:22:45.697 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:22:45.697 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:22:45.697 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 23:22:45.697 | INFO | PerfReport: 21.69%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 23:22:45.698 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 23:22:45.698 | INFO | PerfReport: 32.14%, Compute -- 3.10e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 23:22:45.698 | INFO | PerfReport: 1.42%, LockUpload -- 1.37e+00 / 0.00e+00, 5.07e+00 / 0.00e+00 | 20260506 23:22:45.698 | INFO | PerfReport: 1.23%, LockDownload -- 1.18e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260506 23:22:45.698 | INFO | PerfReport: 22.15%, LockCompute -- 2.14e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:22:45.698 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260506 23:22:45.802 | INFO | Starting a round of Baz IO | 20260506 23:22:48.934 | INFO | Populating RTMetrics report | 20260506 23:22:48.935 | INFO | RTMetrics report populated | 20260506 23:22:48.935 | INFO | Opening tmp RTMetrics json file | 20260506 23:22:48.978 | INFO | Serializing RTMetrics to json | 20260506 23:22:48.992 | INFO | Serialization done, swapping tmp file now... | 20260506 23:22:48.997 | INFO | RTMetrics done | 20260506 23:22:49.347 | INFO | Finished a round of Baz IO: Wrote 1.882 GiB in 3.545 seconds | 20260506 23:22:49.347 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.48% / 0.00% / 66.52% | 20260506 23:22:49.347 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.419203% / 0% / 6.35783e-05% | 20260506 23:22:49.347 | INFO | Baz IO overall: | 20260506 23:22:49.347 | INFO | BytesWritten : 2758001359471 | 20260506 23:22:49.347 | INFO | NumEvents : 1329782036250 | 20260506 23:22:50.836 | INFO | Analyzing chunk frames = [5267456,5267968) | 20260506 23:22:50.836 | INFO | MemInfo: { | 20260506 23:22:50.836 | INFO | "gpu" : " 17 GB", | 20260506 23:22:50.836 | INFO | "huge" : "", | 20260506 23:22:50.836 | INFO | "huge_shared" : " 27 GB", | 20260506 23:22:50.836 | INFO | "malloc" : " 14 GB", | 20260506 23:22:50.836 | INFO | "pinned" : " 29 GB" | 20260506 23:22:50.836 | INFO | } | 20260506 23:22:50.836 | INFO | Perf report for the last 5.14 seconds: | 20260506 23:22:50.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:22:50.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:50.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:50.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:50.836 | INFO | ANALYSIS : 62.4%, 2.9, 94.2ms, 0 | 20260506 23:22:50.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:50.836 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260506 23:22:50.836 | INFO | PRE_HQ : 60.6%, 1.9, 59.7ms, 0 | 20260506 23:22:50.836 | INFO | BAZWRITER : 68.6%, 4.7, 167.5ms, 0 | 20260506 23:22:50.836 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 23:22:50.837 | INFO | Uploaded 36GiB to GPU in 2.06524s (17.4314GiB/s) | 20260506 23:22:50.837 | INFO | Downloaded 38GiB to GPU in 2.18969s (17.3541GiB/s) | 20260506 23:22:55.978 | INFO | Analyzing chunk frames = [5267968,5268480) | 20260506 23:22:55.978 | INFO | MemInfo: { | 20260506 23:22:55.978 | INFO | "gpu" : " 17 GB", | 20260506 23:22:55.978 | INFO | "huge" : "", | 20260506 23:22:55.978 | INFO | "huge_shared" : " 27 GB", | 20260506 23:22:55.978 | INFO | "malloc" : " 14 GB", | 20260506 23:22:55.978 | INFO | "pinned" : " 29 GB" | 20260506 23:22:55.978 | INFO | } | 20260506 23:22:55.978 | INFO | Perf report for the last 5.14144 seconds: | 20260506 23:22:55.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:22:55.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:55.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:55.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:55.978 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:22:55.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:55.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:55.978 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:22:55.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:55.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:22:55.978 | INFO | Uploaded 36GiB to GPU in 1.99891s (18.0098GiB/s) | 20260506 23:22:55.978 | INFO | Downloaded 34GiB to GPU in 1.93069s (17.6103GiB/s) | 20260506 23:23:01.126 | INFO | Analyzing chunk frames = [5268480,5268992) | 20260506 23:23:01.126 | INFO | MemInfo: { | 20260506 23:23:01.126 | INFO | "gpu" : " 17 GB", | 20260506 23:23:01.126 | INFO | "huge" : "", | 20260506 23:23:01.126 | INFO | "huge_shared" : " 27 GB", | 20260506 23:23:01.126 | INFO | "malloc" : " 14 GB", | 20260506 23:23:01.126 | INFO | "pinned" : " 29 GB" | 20260506 23:23:01.126 | INFO | } | 20260506 23:23:01.127 | INFO | Perf report for the last 5.14873 seconds: | 20260506 23:23:01.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:23:01.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:01.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:01.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:01.127 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260506 23:23:01.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:01.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:01.127 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:23:01.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:01.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:01.127 | INFO | Uploaded 36GiB to GPU in 2.00108s (17.9903GiB/s) | 20260506 23:23:01.127 | INFO | Downloaded 34GiB to GPU in 1.93608s (17.5612GiB/s) | 20260506 23:23:06.256 | INFO | Analyzing chunk frames = [5268992,5269504) | 20260506 23:23:06.256 | INFO | MemInfo: { | 20260506 23:23:06.256 | INFO | "gpu" : " 17 GB", | 20260506 23:23:06.256 | INFO | "huge" : "", | 20260506 23:23:06.256 | INFO | "huge_shared" : " 27 GB", | 20260506 23:23:06.256 | INFO | "malloc" : " 14 GB", | 20260506 23:23:06.256 | INFO | "pinned" : " 29 GB" | 20260506 23:23:06.256 | INFO | } | 20260506 23:23:06.256 | INFO | Perf report for the last 5.12944 seconds: | 20260506 23:23:06.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:23:06.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:06.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:06.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:06.256 | INFO | ANALYSIS : 60.2%, 3.0, 93.1ms, 0 | 20260506 23:23:06.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:06.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:06.256 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:23:06.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:06.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:06.256 | INFO | Uploaded 36GiB to GPU in 1.97988s (18.1829GiB/s) | 20260506 23:23:06.256 | INFO | Downloaded 34GiB to GPU in 1.90556s (17.8425GiB/s) | 20260506 23:23:11.392 | INFO | Analyzing chunk frames = [5269504,5270016) | 20260506 23:23:11.392 | INFO | MemInfo: { | 20260506 23:23:11.392 | INFO | "gpu" : " 17 GB", | 20260506 23:23:11.392 | INFO | "huge" : "", | 20260506 23:23:11.392 | INFO | "huge_shared" : " 27 GB", | 20260506 23:23:11.392 | INFO | "malloc" : " 14 GB", | 20260506 23:23:11.392 | INFO | "pinned" : " 29 GB" | 20260506 23:23:11.392 | INFO | } | 20260506 23:23:11.392 | INFO | Perf report for the last 5.13585 seconds: | 20260506 23:23:11.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:23:11.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:11.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:11.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:11.392 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 23:23:11.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:11.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:11.392 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:23:11.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:11.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:11.392 | INFO | Uploaded 36GiB to GPU in 1.98495s (18.1365GiB/s) | 20260506 23:23:11.392 | INFO | Downloaded 34GiB to GPU in 1.92766s (17.638GiB/s) | 20260506 23:23:16.527 | INFO | Analyzing chunk frames = [5270016,5270528) | 20260506 23:23:16.527 | INFO | MemInfo: { | 20260506 23:23:16.527 | INFO | "gpu" : " 17 GB", | 20260506 23:23:16.527 | INFO | "huge" : "", | 20260506 23:23:16.527 | INFO | "huge_shared" : " 27 GB", | 20260506 23:23:16.527 | INFO | "malloc" : " 14 GB", | 20260506 23:23:16.527 | INFO | "pinned" : " 29 GB" | 20260506 23:23:16.527 | INFO | } | 20260506 23:23:16.527 | INFO | Perf report for the last 5.13558 seconds: | 20260506 23:23:16.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:23:16.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:16.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:16.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:16.527 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 23:23:16.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:16.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:16.528 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:23:16.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:16.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:16.528 | INFO | Uploaded 36GiB to GPU in 2.03829s (17.6619GiB/s) | 20260506 23:23:16.528 | INFO | Downloaded 34GiB to GPU in 1.95516s (17.3899GiB/s) | 20260506 23:23:21.667 | INFO | Analyzing chunk frames = [5270528,5271040) | 20260506 23:23:21.667 | INFO | MemInfo: { | 20260506 23:23:21.667 | INFO | "gpu" : " 17 GB", | 20260506 23:23:21.667 | INFO | "huge" : "", | 20260506 23:23:21.667 | INFO | "huge_shared" : " 27 GB", | 20260506 23:23:21.667 | INFO | "malloc" : " 14 GB", | 20260506 23:23:21.667 | INFO | "pinned" : " 29 GB" | 20260506 23:23:21.667 | INFO | } | 20260506 23:23:21.667 | INFO | Perf report for the last 5.13996 seconds: | 20260506 23:23:21.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:23:21.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:21.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:21.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:21.667 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 23:23:21.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:21.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:21.667 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:23:21.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:21.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:21.668 | INFO | Uploaded 36GiB to GPU in 1.97539s (18.2242GiB/s) | 20260506 23:23:21.668 | INFO | Downloaded 34GiB to GPU in 1.99182s (17.0698GiB/s) | 20260506 23:23:26.817 | INFO | Analyzing chunk frames = [5271040,5271552) | 20260506 23:23:26.817 | INFO | MemInfo: { | 20260506 23:23:26.817 | INFO | "gpu" : " 17 GB", | 20260506 23:23:26.817 | INFO | "huge" : "", | 20260506 23:23:26.817 | INFO | "huge_shared" : " 27 GB", | 20260506 23:23:26.817 | INFO | "malloc" : " 14 GB", | 20260506 23:23:26.817 | INFO | "pinned" : " 29 GB" | 20260506 23:23:26.817 | INFO | } | 20260506 23:23:26.817 | INFO | Perf report for the last 5.14986 seconds: | 20260506 23:23:26.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:23:26.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:26.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:26.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:26.817 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 23:23:26.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:26.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:26.817 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260506 23:23:26.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:26.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:26.818 | INFO | Uploaded 36GiB to GPU in 1.98443s (18.1413GiB/s) | 20260506 23:23:26.818 | INFO | Downloaded 34GiB to GPU in 2.00449s (16.9619GiB/s) | 20260506 23:23:26.920 | INFO | Starting a round of Baz IO | 20260506 23:23:30.056 | INFO | Populating RTMetrics report | 20260506 23:23:30.057 | INFO | RTMetrics report populated | 20260506 23:23:30.057 | INFO | Opening tmp RTMetrics json file | 20260506 23:23:30.064 | INFO | Serializing RTMetrics to json | 20260506 23:23:30.076 | INFO | Serialization done, swapping tmp file now... | 20260506 23:23:30.131 | INFO | RTMetrics done | 20260506 23:23:30.346 | INFO | Finished a round of Baz IO: Wrote 1.882 GiB in 3.425 seconds | 20260506 23:23:30.346 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.48% / 0.00% / 66.52% | 20260506 23:23:30.346 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.418142% / 0% / 7.54992e-05% | 20260506 23:23:30.346 | INFO | Baz IO overall: | 20260506 23:23:30.346 | INFO | BytesWritten : 2760021707821 | 20260506 23:23:30.346 | INFO | NumEvents : 1330743923516 | 20260506 23:23:31.952 | INFO | Analyzing chunk frames = [5271552,5272064) | 20260506 23:23:31.952 | INFO | MemInfo: { | 20260506 23:23:31.952 | INFO | "gpu" : " 17 GB", | 20260506 23:23:31.952 | INFO | "huge" : "", | 20260506 23:23:31.952 | INFO | "huge_shared" : " 27 GB", | 20260506 23:23:31.952 | INFO | "malloc" : " 14 GB", | 20260506 23:23:31.952 | INFO | "pinned" : " 29 GB" | 20260506 23:23:31.952 | INFO | } | 20260506 23:23:31.952 | INFO | Perf report for the last 5.13493 seconds: | 20260506 23:23:31.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:23:31.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:31.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:31.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:31.952 | INFO | ANALYSIS : 62.2%, 2.9, 94.1ms, 0 | 20260506 23:23:31.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:31.952 | INFO | RT_METRICS : 46.8%, 1.0, 24.5ms, 0 | 20260506 23:23:31.952 | INFO | PRE_HQ : 60.1%, 1.9, 60.7ms, 0 | 20260506 23:23:31.952 | INFO | BAZWRITER : 66.4%, 4.8, 163.9ms, 0 | 20260506 23:23:31.952 | INFO | BAZ_SAMPLING : 2.1%, 1.0, 1.1ms, 0 | 20260506 23:23:31.952 | INFO | Uploaded 36GiB to GPU in 2.01488s (17.8671GiB/s) | 20260506 23:23:31.952 | INFO | Downloaded 38GiB to GPU in 2.19479s (17.3137GiB/s) | 20260506 23:23:37.093 | INFO | Analyzing chunk frames = [5272064,5272576) | 20260506 23:23:37.093 | INFO | MemInfo: { | 20260506 23:23:37.093 | INFO | "gpu" : " 17 GB", | 20260506 23:23:37.093 | INFO | "huge" : "", | 20260506 23:23:37.093 | INFO | "huge_shared" : " 27 GB", | 20260506 23:23:37.093 | INFO | "malloc" : " 14 GB", | 20260506 23:23:37.093 | INFO | "pinned" : " 29 GB" | 20260506 23:23:37.093 | INFO | } | 20260506 23:23:37.093 | INFO | Perf report for the last 5.14062 seconds: | 20260506 23:23:37.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:23:37.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:37.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:37.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:37.093 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:23:37.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:37.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:37.093 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260506 23:23:37.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:37.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:37.093 | INFO | Uploaded 36GiB to GPU in 2.00067s (17.994GiB/s) | 20260506 23:23:37.093 | INFO | Downloaded 34GiB to GPU in 1.93797s (17.5441GiB/s) | 20260506 23:23:42.227 | INFO | Analyzing chunk frames = [5272576,5273088) | 20260506 23:23:42.227 | INFO | MemInfo: { | 20260506 23:23:42.227 | INFO | "gpu" : " 17 GB", | 20260506 23:23:42.227 | INFO | "huge" : "", | 20260506 23:23:42.227 | INFO | "huge_shared" : " 27 GB", | 20260506 23:23:42.227 | INFO | "malloc" : " 14 GB", | 20260506 23:23:42.227 | INFO | "pinned" : " 29 GB" | 20260506 23:23:42.227 | INFO | } | 20260506 23:23:42.227 | INFO | Perf report for the last 5.13441 seconds: | 20260506 23:23:42.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:23:42.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:42.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:42.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:42.227 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260506 23:23:42.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:42.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:42.227 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:23:42.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:42.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:42.227 | INFO | Uploaded 36GiB to GPU in 1.96272s (18.3419GiB/s) | 20260506 23:23:42.228 | INFO | Downloaded 34GiB to GPU in 1.93303s (17.5889GiB/s) | 20260506 23:23:47.368 | INFO | Analyzing chunk frames = [5273088,5273600) | 20260506 23:23:47.368 | INFO | MemInfo: { | 20260506 23:23:47.368 | INFO | "gpu" : " 17 GB", | 20260506 23:23:47.368 | INFO | "huge" : "", | 20260506 23:23:47.368 | INFO | "huge_shared" : " 27 GB", | 20260506 23:23:47.368 | INFO | "malloc" : " 14 GB", | 20260506 23:23:47.368 | INFO | "pinned" : " 29 GB" | 20260506 23:23:47.368 | INFO | } | 20260506 23:23:47.368 | INFO | Perf report for the last 5.14085 seconds: | 20260506 23:23:47.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:23:47.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:47.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:47.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:47.368 | INFO | ANALYSIS : 59.5%, 3.0, 92.2ms, 0 | 20260506 23:23:47.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:47.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:47.368 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:23:47.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:47.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:47.368 | INFO | Uploaded 36GiB to GPU in 1.93731s (18.5825GiB/s) | 20260506 23:23:47.369 | INFO | Downloaded 34GiB to GPU in 1.87816s (18.1028GiB/s) | 20260506 23:23:52.506 | INFO | Analyzing chunk frames = [5273600,5274112) | 20260506 23:23:52.506 | INFO | MemInfo: { | 20260506 23:23:52.506 | INFO | "gpu" : " 17 GB", | 20260506 23:23:52.506 | INFO | "huge" : "", | 20260506 23:23:52.507 | INFO | "huge_shared" : " 27 GB", | 20260506 23:23:52.507 | INFO | "malloc" : " 14 GB", | 20260506 23:23:52.507 | INFO | "pinned" : " 29 GB" | 20260506 23:23:52.507 | INFO | } | 20260506 23:23:52.507 | INFO | Perf report for the last 5.13855 seconds: | 20260506 23:23:52.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:23:52.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:52.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:52.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:52.507 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 23:23:52.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:52.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:52.507 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:23:52.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:52.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:52.507 | INFO | Uploaded 36GiB to GPU in 1.97741s (18.2057GiB/s) | 20260506 23:23:52.507 | INFO | Downloaded 34GiB to GPU in 1.92583s (17.6547GiB/s) | 20260506 23:23:57.643 | INFO | Analyzing chunk frames = [5274112,5274624) | 20260506 23:23:57.643 | INFO | MemInfo: { | 20260506 23:23:57.643 | INFO | "gpu" : " 17 GB", | 20260506 23:23:57.644 | INFO | "huge" : "", | 20260506 23:23:57.644 | INFO | "huge_shared" : " 27 GB", | 20260506 23:23:57.644 | INFO | "malloc" : " 14 GB", | 20260506 23:23:57.644 | INFO | "pinned" : " 29 GB" | 20260506 23:23:57.644 | INFO | } | 20260506 23:23:57.644 | INFO | Perf report for the last 5.13695 seconds: | 20260506 23:23:57.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:23:57.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:57.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:57.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:57.644 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 23:23:57.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:57.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:57.644 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:23:57.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:57.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:23:57.644 | INFO | Uploaded 36GiB to GPU in 2.00553s (17.9503GiB/s) | 20260506 23:23:57.644 | INFO | Downloaded 34GiB to GPU in 1.94956s (17.4398GiB/s) | 20260506 23:24:02.788 | INFO | Analyzing chunk frames = [5274624,5275136) | 20260506 23:24:02.788 | INFO | MemInfo: { | 20260506 23:24:02.788 | INFO | "gpu" : " 17 GB", | 20260506 23:24:02.788 | INFO | "huge" : "", | 20260506 23:24:02.788 | INFO | "huge_shared" : " 27 GB", | 20260506 23:24:02.788 | INFO | "malloc" : " 14 GB", | 20260506 23:24:02.788 | INFO | "pinned" : " 29 GB" | 20260506 23:24:02.788 | INFO | } | 20260506 23:24:02.788 | INFO | Perf report for the last 5.1445 seconds: | 20260506 23:24:02.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:24:02.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:02.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:02.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:02.788 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 23:24:02.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:02.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:02.788 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:24:02.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:02.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:02.788 | INFO | Uploaded 36GiB to GPU in 1.95425s (18.4213GiB/s) | 20260506 23:24:02.789 | INFO | Downloaded 34GiB to GPU in 1.98188s (17.1554GiB/s) | 20260506 23:24:07.928 | INFO | Analyzing chunk frames = [5275136,5275648) | 20260506 23:24:07.928 | INFO | MemInfo: { | 20260506 23:24:07.928 | INFO | "gpu" : " 17 GB", | 20260506 23:24:07.929 | INFO | "huge" : "", | 20260506 23:24:07.929 | INFO | "huge_shared" : " 27 GB", | 20260506 23:24:07.929 | INFO | "malloc" : " 14 GB", | 20260506 23:24:07.929 | INFO | "pinned" : " 29 GB" | 20260506 23:24:07.929 | INFO | } | 20260506 23:24:07.929 | INFO | Perf report for the last 5.14049 seconds: | 20260506 23:24:07.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:24:07.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:07.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:07.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:07.929 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 23:24:07.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:07.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:07.929 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:24:07.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:07.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:07.929 | INFO | Uploaded 36GiB to GPU in 1.97113s (18.2637GiB/s) | 20260506 23:24:07.929 | INFO | Downloaded 34GiB to GPU in 1.98312s (17.1447GiB/s) | 20260506 23:24:07.929 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:24:07.929 | INFO | Total mem usage: 18 GiB | 20260506 23:24:07.929 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:24:07.929 | INFO | Recent High Water Marks Sum: 15.7 GiB (Savings: -15%) | 20260506 23:24:07.929 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:24:07.929 | INFO | High Water Breakdown: | 20260506 23:24:07.929 | INFO | 61.6% : 9.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:24:07.929 | INFO | 38.3% : 6 GiB ( 89.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:24:07.929 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:24:07.929 | INFO | ------------------------------------------ | 20260506 23:24:07.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:24:07.929 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:24:07.929 | INFO | Total mem usage: 26.5 GiB | 20260506 23:24:07.929 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:24:07.929 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:24:07.929 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:24:07.929 | INFO | High Water Breakdown: | 20260506 23:24:07.929 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:24:07.929 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:24:07.929 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:24:07.929 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:24:07.929 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:24:07.929 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:24:07.929 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:24:07.929 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:24:07.929 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:24:07.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:24:07.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:24:07.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:24:07.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:24:07.929 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:24:07.929 | INFO | ------------------------------------------ | 20260506 23:24:07.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:24:07.929 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:24:07.929 | INFO | Total mem usage: 25.6 GiB | 20260506 23:24:07.929 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:24:07.929 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:24:07.929 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:24:07.929 | INFO | High Water Breakdown: | 20260506 23:24:07.929 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:24:07.930 | INFO | ------------------------------------------ | 20260506 23:24:07.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:24:07.930 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:24:07.930 | INFO | Total mem usage: 15.7 GiB | 20260506 23:24:07.930 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:24:07.930 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:24:07.930 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 23:24:07.930 | INFO | High Water Breakdown: | 20260506 23:24:07.930 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:24:07.930 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:24:07.930 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:24:07.930 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:24:07.930 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:24:07.930 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:24:07.930 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:24:07.930 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:24:07.930 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:24:07.930 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:24:07.930 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:24:07.930 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:24:07.930 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:24:07.930 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:24:07.930 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:24:07.930 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:24:07.930 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:24:07.930 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:24:07.930 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:24:07.930 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:24:07.930 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:24:07.930 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:24:07.930 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:24:07.930 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:24:07.930 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:24:07.930 | INFO | ------------------------------------------ | 20260506 23:24:07.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:24:07.930 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:24:07.930 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:24:07.930 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:24:07.930 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.61e+01, 1.34e+01 / 2.30e+00 | 20260506 23:24:07.930 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.02e-01, 1.00e-02 / 5.93e-03 | 20260506 23:24:07.930 | INFO | PerfReport: 11.67%, Baseline -- 3.62e+00 / 4.39e+00, 5.48e-01 / 1.79e-01 | 20260506 23:24:07.930 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.43e+00, 1.77e-01 / 1.20e-01 | 20260506 23:24:07.930 | INFO | PerfReport: 16.09%, DME -- 3.99e+01 / 4.50e+01, 4.29e+00 / 2.91e+00 | 20260506 23:24:07.930 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.37e+00 / 1.77e+00 | 20260506 23:24:07.930 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.19e+00 / 2.75e+00, 3.82e-01 / 2.17e-01 | 20260506 23:24:07.930 | INFO | PerfReport: 3.19%, Metrics -- 9.88e-01 / 1.33e+00, 2.61e-01 / 1.44e-01 | 20260506 23:24:07.930 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.37e-01 / 3.92e-01, 5.29e-02 / 2.48e-02 | 20260506 23:24:07.930 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:24:07.930 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:24:07.930 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:24:07.930 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 23:24:07.930 | INFO | PerfReport: 21.53%, Upload -- 2.07e+01 / 0.00e+00, 6.87e+00 / 0.00e+00 | 20260506 23:24:07.930 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 23:24:07.930 | INFO | PerfReport: 32.21%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 23:24:07.930 | INFO | PerfReport: 1.24%, LockUpload -- 1.20e+00 / 0.00e+00, 4.68e+00 / 0.00e+00 | 20260506 23:24:07.930 | INFO | PerfReport: 1.33%, LockDownload -- 1.28e+00 / 0.00e+00, 3.19e+00 / 0.00e+00 | 20260506 23:24:07.930 | INFO | PerfReport: 22.28%, LockCompute -- 2.15e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 23:24:07.930 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260506 23:24:08.027 | INFO | Starting a round of Baz IO | 20260506 23:24:11.155 | INFO | Populating RTMetrics report | 20260506 23:24:11.155 | INFO | RTMetrics report populated | 20260506 23:24:11.155 | INFO | Opening tmp RTMetrics json file | 20260506 23:24:11.234 | INFO | Serializing RTMetrics to json | 20260506 23:24:11.243 | INFO | Serialization done, swapping tmp file now... | 20260506 23:24:11.245 | INFO | RTMetrics done | 20260506 23:24:11.289 | INFO | Finished a round of Baz IO: Wrote 1.881 GiB in 3.261 seconds | 20260506 23:24:11.289 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.48% / 0.00% / 66.52% | 20260506 23:24:11.289 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.418349% / 7.94729e-06% / 2.38419e-05% | 20260506 23:24:11.289 | INFO | Baz IO overall: | 20260506 23:24:11.289 | INFO | BytesWritten : 2762041797530 | 20260506 23:24:11.289 | INFO | NumEvents : 1331706494734 | 20260506 23:24:13.075 | INFO | Analyzing chunk frames = [5275648,5276160) | 20260506 23:24:13.075 | INFO | MemInfo: { | 20260506 23:24:13.075 | INFO | "gpu" : " 17 GB", | 20260506 23:24:13.075 | INFO | "huge" : "", | 20260506 23:24:13.075 | INFO | "huge_shared" : " 27 GB", | 20260506 23:24:13.075 | INFO | "malloc" : " 14 GB", | 20260506 23:24:13.075 | INFO | "pinned" : " 29 GB" | 20260506 23:24:13.075 | INFO | } | 20260506 23:24:13.075 | INFO | Perf report for the last 5.1463 seconds: | 20260506 23:24:13.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:24:13.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:13.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:13.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:13.075 | INFO | ANALYSIS : 62.1%, 2.9, 95.3ms, 0 | 20260506 23:24:13.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:13.075 | INFO | RT_METRICS : 43.9%, 1.0, 23.1ms, 0 | 20260506 23:24:13.075 | INFO | PRE_HQ : 59.5%, 1.8, 56.2ms, 0 | 20260506 23:24:13.075 | INFO | BAZWRITER : 63.1%, 4.7, 153.3ms, 0 | 20260506 23:24:13.075 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.7ms, 0 | 20260506 23:24:13.075 | INFO | Uploaded 36GiB to GPU in 2.03566s (17.6847GiB/s) | 20260506 23:24:13.075 | INFO | Downloaded 38GiB to GPU in 2.19853s (17.2843GiB/s) | 20260506 23:24:18.211 | INFO | Analyzing chunk frames = [5276160,5276672) | 20260506 23:24:18.211 | INFO | MemInfo: { | 20260506 23:24:18.211 | INFO | "gpu" : " 17 GB", | 20260506 23:24:18.211 | INFO | "huge" : "", | 20260506 23:24:18.211 | INFO | "huge_shared" : " 27 GB", | 20260506 23:24:18.211 | INFO | "malloc" : " 14 GB", | 20260506 23:24:18.211 | INFO | "pinned" : " 29 GB" | 20260506 23:24:18.211 | INFO | } | 20260506 23:24:18.211 | INFO | Perf report for the last 5.13596 seconds: | 20260506 23:24:18.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:24:18.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:18.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:18.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:18.211 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 23:24:18.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:18.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:18.211 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:24:18.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:18.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:18.211 | INFO | Uploaded 36GiB to GPU in 2.00903s (17.9191GiB/s) | 20260506 23:24:18.211 | INFO | Downloaded 34GiB to GPU in 1.93562s (17.5654GiB/s) | 20260506 23:24:23.343 | INFO | Analyzing chunk frames = [5276672,5277184) | 20260506 23:24:23.343 | INFO | MemInfo: { | 20260506 23:24:23.343 | INFO | "gpu" : " 17 GB", | 20260506 23:24:23.343 | INFO | "huge" : "", | 20260506 23:24:23.343 | INFO | "huge_shared" : " 27 GB", | 20260506 23:24:23.343 | INFO | "malloc" : " 14 GB", | 20260506 23:24:23.343 | INFO | "pinned" : " 29 GB" | 20260506 23:24:23.343 | INFO | } | 20260506 23:24:23.343 | INFO | Perf report for the last 5.1322 seconds: | 20260506 23:24:23.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:24:23.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:23.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:23.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:23.343 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:24:23.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:23.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:23.343 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:24:23.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:23.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:23.344 | INFO | Uploaded 36GiB to GPU in 1.98397s (18.1454GiB/s) | 20260506 23:24:23.344 | INFO | Downloaded 34GiB to GPU in 1.93561s (17.5655GiB/s) | 20260506 23:24:28.481 | INFO | Analyzing chunk frames = [5277184,5277696) | 20260506 23:24:28.481 | INFO | MemInfo: { | 20260506 23:24:28.481 | INFO | "gpu" : " 17 GB", | 20260506 23:24:28.481 | INFO | "huge" : "", | 20260506 23:24:28.481 | INFO | "huge_shared" : " 27 GB", | 20260506 23:24:28.481 | INFO | "malloc" : " 14 GB", | 20260506 23:24:28.481 | INFO | "pinned" : " 29 GB" | 20260506 23:24:28.481 | INFO | } | 20260506 23:24:28.481 | INFO | Perf report for the last 5.13804 seconds: | 20260506 23:24:28.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:24:28.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:28.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:28.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:28.481 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 23:24:28.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:28.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:28.481 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:24:28.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:28.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:28.481 | INFO | Uploaded 36GiB to GPU in 2.01124s (17.8994GiB/s) | 20260506 23:24:28.481 | INFO | Downloaded 34GiB to GPU in 1.91043s (17.7971GiB/s) | 20260506 23:24:33.625 | INFO | Analyzing chunk frames = [5277696,5278208) | 20260506 23:24:33.625 | INFO | MemInfo: { | 20260506 23:24:33.625 | INFO | "gpu" : " 17 GB", | 20260506 23:24:33.625 | INFO | "huge" : "", | 20260506 23:24:33.625 | INFO | "huge_shared" : " 27 GB", | 20260506 23:24:33.625 | INFO | "malloc" : " 14 GB", | 20260506 23:24:33.625 | INFO | "pinned" : " 29 GB" | 20260506 23:24:33.625 | INFO | } | 20260506 23:24:33.625 | INFO | Perf report for the last 5.14415 seconds: | 20260506 23:24:33.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:24:33.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:33.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:33.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:33.625 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 23:24:33.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:33.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:33.625 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:24:33.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:33.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:33.626 | INFO | Uploaded 36GiB to GPU in 1.9633s (18.3365GiB/s) | 20260506 23:24:33.626 | INFO | Downloaded 34GiB to GPU in 1.92982s (17.6182GiB/s) | 20260506 23:24:38.761 | INFO | Analyzing chunk frames = [5278208,5278720) | 20260506 23:24:38.761 | INFO | MemInfo: { | 20260506 23:24:38.761 | INFO | "gpu" : " 17 GB", | 20260506 23:24:38.761 | INFO | "huge" : "", | 20260506 23:24:38.761 | INFO | "huge_shared" : " 27 GB", | 20260506 23:24:38.761 | INFO | "malloc" : " 14 GB", | 20260506 23:24:38.761 | INFO | "pinned" : " 29 GB" | 20260506 23:24:38.762 | INFO | } | 20260506 23:24:38.762 | INFO | Perf report for the last 5.13614 seconds: | 20260506 23:24:38.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:24:38.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:38.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:38.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:38.762 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 23:24:38.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:38.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:38.762 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 23:24:38.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:38.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:38.762 | INFO | Uploaded 36GiB to GPU in 2.044s (17.6125GiB/s) | 20260506 23:24:38.762 | INFO | Downloaded 34GiB to GPU in 1.95137s (17.4237GiB/s) | 20260506 23:24:43.902 | INFO | Analyzing chunk frames = [5278720,5279232) | 20260506 23:24:43.902 | INFO | MemInfo: { | 20260506 23:24:43.902 | INFO | "gpu" : " 17 GB", | 20260506 23:24:43.902 | INFO | "huge" : "", | 20260506 23:24:43.902 | INFO | "huge_shared" : " 27 GB", | 20260506 23:24:43.902 | INFO | "malloc" : " 14 GB", | 20260506 23:24:43.902 | INFO | "pinned" : " 29 GB" | 20260506 23:24:43.903 | INFO | } | 20260506 23:24:43.904 | INFO | Perf report for the last 5.14195 seconds: | 20260506 23:24:43.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:24:43.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:43.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:43.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:43.904 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:24:43.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:43.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:43.904 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:24:43.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:43.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:43.904 | INFO | Uploaded 36GiB to GPU in 1.96191s (18.3495GiB/s) | 20260506 23:24:43.904 | INFO | Downloaded 34GiB to GPU in 1.99469s (17.0453GiB/s) | 20260506 23:24:49.043 | INFO | Analyzing chunk frames = [5279232,5279744) | 20260506 23:24:49.043 | INFO | MemInfo: { | 20260506 23:24:49.043 | INFO | "gpu" : " 17 GB", | 20260506 23:24:49.043 | INFO | "huge" : "", | 20260506 23:24:49.043 | INFO | "huge_shared" : " 27 GB", | 20260506 23:24:49.043 | INFO | "malloc" : " 14 GB", | 20260506 23:24:49.043 | INFO | "pinned" : " 29 GB" | 20260506 23:24:49.043 | INFO | } | 20260506 23:24:49.043 | INFO | Perf report for the last 5.13989 seconds: | 20260506 23:24:49.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:24:49.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:49.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:49.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:49.044 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 23:24:49.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:49.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:49.044 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260506 23:24:49.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:49.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:49.044 | INFO | Uploaded 36GiB to GPU in 1.99798s (18.0182GiB/s) | 20260506 23:24:49.044 | INFO | Downloaded 34GiB to GPU in 1.99155s (17.0721GiB/s) | 20260506 23:24:49.141 | INFO | Starting a round of Baz IO | 20260506 23:24:52.286 | INFO | Populating RTMetrics report | 20260506 23:24:52.287 | INFO | RTMetrics report populated | 20260506 23:24:52.287 | INFO | Opening tmp RTMetrics json file | 20260506 23:24:52.287 | INFO | Serializing RTMetrics to json | 20260506 23:24:52.288 | INFO | Serialization done, swapping tmp file now... | 20260506 23:24:52.289 | INFO | RTMetrics done | 20260506 23:24:52.918 | INFO | Finished a round of Baz IO: Wrote 1.879 GiB in 3.777 seconds | 20260506 23:24:52.918 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.47% / 0.00% / 66.52% | 20260506 23:24:52.918 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.416978% / 7.94729e-06% / 4.37101e-05% | 20260506 23:24:52.918 | INFO | Baz IO overall: | 20260506 23:24:52.918 | INFO | BytesWritten : 2764059785139 | 20260506 23:24:52.918 | INFO | NumEvents : 1332667465500 | 20260506 23:24:54.185 | INFO | Analyzing chunk frames = [5279744,5280256) | 20260506 23:24:54.185 | INFO | MemInfo: { | 20260506 23:24:54.185 | INFO | "gpu" : " 17 GB", | 20260506 23:24:54.185 | INFO | "huge" : "", | 20260506 23:24:54.185 | INFO | "huge_shared" : " 27 GB", | 20260506 23:24:54.185 | INFO | "malloc" : " 14 GB", | 20260506 23:24:54.185 | INFO | "pinned" : " 29 GB" | 20260506 23:24:54.185 | INFO | } | 20260506 23:24:54.185 | INFO | Perf report for the last 5.14134 seconds: | 20260506 23:24:54.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:24:54.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:54.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:54.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:54.185 | INFO | ANALYSIS : 62.4%, 2.9, 93.8ms, 0 | 20260506 23:24:54.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:54.185 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260506 23:24:54.185 | INFO | PRE_HQ : 61.0%, 1.9, 60.0ms, 0 | 20260506 23:24:54.185 | INFO | BAZWRITER : 73.1%, 4.8, 180.2ms, 0 | 20260506 23:24:54.185 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 23:24:54.185 | INFO | Uploaded 36GiB to GPU in 2.02859s (17.7463GiB/s) | 20260506 23:24:54.185 | INFO | Downloaded 38GiB to GPU in 2.19306s (17.3274GiB/s) | 20260506 23:24:59.324 | INFO | Analyzing chunk frames = [5280256,5280768) | 20260506 23:24:59.324 | INFO | MemInfo: { | 20260506 23:24:59.324 | INFO | "gpu" : " 17 GB", | 20260506 23:24:59.324 | INFO | "huge" : "", | 20260506 23:24:59.324 | INFO | "huge_shared" : " 27 GB", | 20260506 23:24:59.324 | INFO | "malloc" : " 14 GB", | 20260506 23:24:59.324 | INFO | "pinned" : " 29 GB" | 20260506 23:24:59.324 | INFO | } | 20260506 23:24:59.324 | INFO | Perf report for the last 5.13928 seconds: | 20260506 23:24:59.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:24:59.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:59.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:59.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:59.324 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:24:59.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:59.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:59.324 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 23:24:59.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:59.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:24:59.324 | INFO | Uploaded 36GiB to GPU in 2.01491s (17.8668GiB/s) | 20260506 23:24:59.324 | INFO | Downloaded 34GiB to GPU in 1.93452s (17.5754GiB/s) | 20260506 23:25:04.458 | INFO | Analyzing chunk frames = [5280768,5281280) | 20260506 23:25:04.458 | INFO | MemInfo: { | 20260506 23:25:04.458 | INFO | "gpu" : " 17 GB", | 20260506 23:25:04.458 | INFO | "huge" : "", | 20260506 23:25:04.458 | INFO | "huge_shared" : " 27 GB", | 20260506 23:25:04.458 | INFO | "malloc" : " 14 GB", | 20260506 23:25:04.458 | INFO | "pinned" : " 29 GB" | 20260506 23:25:04.458 | INFO | } | 20260506 23:25:04.458 | INFO | Perf report for the last 5.13431 seconds: | 20260506 23:25:04.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:25:04.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:04.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:04.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:04.458 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260506 23:25:04.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:04.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:04.458 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:25:04.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:04.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:04.459 | INFO | Uploaded 36GiB to GPU in 1.98746s (18.1136GiB/s) | 20260506 23:25:04.459 | INFO | Downloaded 34GiB to GPU in 1.93772s (17.5464GiB/s) | 20260506 23:25:09.594 | INFO | Analyzing chunk frames = [5281280,5281792) | 20260506 23:25:09.594 | INFO | MemInfo: { | 20260506 23:25:09.594 | INFO | "gpu" : " 17 GB", | 20260506 23:25:09.594 | INFO | "huge" : "", | 20260506 23:25:09.594 | INFO | "huge_shared" : " 27 GB", | 20260506 23:25:09.594 | INFO | "malloc" : " 14 GB", | 20260506 23:25:09.594 | INFO | "pinned" : " 29 GB" | 20260506 23:25:09.594 | INFO | } | 20260506 23:25:09.594 | INFO | Perf report for the last 5.13526 seconds: | 20260506 23:25:09.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:25:09.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:09.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:09.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:09.594 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 23:25:09.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:09.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:09.594 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:25:09.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:09.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:09.594 | INFO | Uploaded 36GiB to GPU in 1.96606s (18.3107GiB/s) | 20260506 23:25:09.594 | INFO | Downloaded 34GiB to GPU in 1.90531s (17.8449GiB/s) | 20260506 23:25:14.735 | INFO | Analyzing chunk frames = [5281792,5282304) | 20260506 23:25:14.735 | INFO | MemInfo: { | 20260506 23:25:14.735 | INFO | "gpu" : " 17 GB", | 20260506 23:25:14.735 | INFO | "huge" : "", | 20260506 23:25:14.735 | INFO | "huge_shared" : " 27 GB", | 20260506 23:25:14.735 | INFO | "malloc" : " 14 GB", | 20260506 23:25:14.735 | INFO | "pinned" : " 29 GB" | 20260506 23:25:14.735 | INFO | } | 20260506 23:25:14.735 | INFO | Perf report for the last 5.14104 seconds: | 20260506 23:25:14.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:25:14.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:14.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:14.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:14.735 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260506 23:25:14.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:14.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:14.735 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:25:14.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:14.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:14.735 | INFO | Uploaded 36GiB to GPU in 2.00899s (17.9194GiB/s) | 20260506 23:25:14.735 | INFO | Downloaded 34GiB to GPU in 1.92039s (17.7047GiB/s) | 20260506 23:25:19.884 | INFO | Analyzing chunk frames = [5282304,5282816) | 20260506 23:25:19.884 | INFO | MemInfo: { | 20260506 23:25:19.884 | INFO | "gpu" : " 17 GB", | 20260506 23:25:19.884 | INFO | "huge" : "", | 20260506 23:25:19.884 | INFO | "huge_shared" : " 27 GB", | 20260506 23:25:19.884 | INFO | "malloc" : " 14 GB", | 20260506 23:25:19.884 | INFO | "pinned" : " 29 GB" | 20260506 23:25:19.884 | INFO | } | 20260506 23:25:19.884 | INFO | Perf report for the last 5.1493 seconds: | 20260506 23:25:19.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:25:19.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:19.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:19.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:19.884 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 23:25:19.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:19.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:19.884 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:25:19.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:19.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:19.884 | INFO | Uploaded 36GiB to GPU in 1.98757s (18.1125GiB/s) | 20260506 23:25:19.884 | INFO | Downloaded 34GiB to GPU in 1.95428s (17.3977GiB/s) | 20260506 23:25:25.015 | INFO | Analyzing chunk frames = [5282816,5283328) | 20260506 23:25:25.015 | INFO | MemInfo: { | 20260506 23:25:25.015 | INFO | "gpu" : " 17 GB", | 20260506 23:25:25.015 | INFO | "huge" : "", | 20260506 23:25:25.015 | INFO | "huge_shared" : " 27 GB", | 20260506 23:25:25.015 | INFO | "malloc" : " 14 GB", | 20260506 23:25:25.015 | INFO | "pinned" : " 29 GB" | 20260506 23:25:25.015 | INFO | } | 20260506 23:25:25.016 | INFO | Perf report for the last 5.13149 seconds: | 20260506 23:25:25.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:25:25.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:25.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:25.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:25.016 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260506 23:25:25.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:25.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:25.016 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:25:25.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:25.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:25.016 | INFO | Uploaded 36GiB to GPU in 1.99265s (18.0664GiB/s) | 20260506 23:25:25.016 | INFO | Downloaded 34GiB to GPU in 1.98974s (17.0876GiB/s) | 20260506 23:25:30.158 | INFO | Analyzing chunk frames = [5283328,5283840) | 20260506 23:25:30.158 | INFO | MemInfo: { | 20260506 23:25:30.158 | INFO | "gpu" : " 17 GB", | 20260506 23:25:30.158 | INFO | "huge" : "", | 20260506 23:25:30.158 | INFO | "huge_shared" : " 27 GB", | 20260506 23:25:30.158 | INFO | "malloc" : " 14 GB", | 20260506 23:25:30.158 | INFO | "pinned" : " 29 GB" | 20260506 23:25:30.158 | INFO | } | 20260506 23:25:30.158 | INFO | Perf report for the last 5.14267 seconds: | 20260506 23:25:30.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:25:30.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:30.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:30.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:30.158 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:25:30.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:30.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:30.158 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:25:30.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:30.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:30.159 | INFO | Uploaded 36GiB to GPU in 1.99216s (18.0708GiB/s) | 20260506 23:25:30.159 | INFO | Downloaded 34GiB to GPU in 1.9903s (17.0829GiB/s) | 20260506 23:25:30.159 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:25:30.159 | INFO | Total mem usage: 18 GiB | 20260506 23:25:30.159 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:25:30.159 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -11.9%) | 20260506 23:25:30.159 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:25:30.159 | INFO | High Water Breakdown: | 20260506 23:25:30.159 | INFO | 62% : 9.97 GiB ( 87.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:25:30.159 | INFO | 37.9% : 6.1 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:25:30.159 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:25:30.159 | INFO | ------------------------------------------ | 20260506 23:25:30.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:25:30.159 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:25:30.159 | INFO | Total mem usage: 26.5 GiB | 20260506 23:25:30.159 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:25:30.159 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:25:30.159 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:25:30.159 | INFO | High Water Breakdown: | 20260506 23:25:30.159 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:25:30.159 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:25:30.159 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:25:30.159 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:25:30.159 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:25:30.159 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:25:30.159 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:25:30.159 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:25:30.159 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:25:30.159 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:25:30.159 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:25:30.159 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:25:30.159 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:25:30.159 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:25:30.159 | INFO | ------------------------------------------ | 20260506 23:25:30.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:25:30.159 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:25:30.159 | INFO | Total mem usage: 25.6 GiB | 20260506 23:25:30.159 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:25:30.159 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:25:30.159 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:25:30.159 | INFO | High Water Breakdown: | 20260506 23:25:30.159 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:25:30.159 | INFO | ------------------------------------------ | 20260506 23:25:30.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:25:30.159 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:25:30.159 | INFO | Total mem usage: 15.7 GiB | 20260506 23:25:30.159 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:25:30.159 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:25:30.159 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:25:30.159 | INFO | High Water Breakdown: | 20260506 23:25:30.159 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:25:30.159 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:25:30.159 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:25:30.159 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:25:30.159 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:25:30.159 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:25:30.159 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:25:30.159 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:25:30.159 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:25:30.159 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:25:30.159 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:25:30.159 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:25:30.159 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:25:30.159 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:25:30.159 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:25:30.160 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:25:30.160 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:25:30.160 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:25:30.160 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:25:30.160 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:25:30.160 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:25:30.160 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:25:30.160 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:25:30.160 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:25:30.160 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:25:30.160 | INFO | ------------------------------------------ | 20260506 23:25:30.160 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:25:30.160 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:25:30.160 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:25:30.160 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:25:30.160 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.62e+01, 1.33e+01 / 2.20e+00 | 20260506 23:25:30.160 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 9.67e-03 / 5.07e-03 | 20260506 23:25:30.160 | INFO | PerfReport: 11.65%, Baseline -- 3.61e+00 / 4.44e+00, 5.41e-01 / 3.15e-01 | 20260506 23:25:30.160 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.48e+00, 1.79e-01 / 1.14e-01 | 20260506 23:25:30.160 | INFO | PerfReport: 16.05%, DME -- 3.98e+01 / 4.48e+01, 4.17e+00 / 2.51e+00 | 20260506 23:25:30.160 | INFO | PerfReport: 57.40%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.37e+00 / 1.81e+00 | 20260506 23:25:30.160 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.19e+00 / 2.73e+00, 3.78e-01 / 2.05e-01 | 20260506 23:25:30.160 | INFO | PerfReport: 3.20%, Metrics -- 9.92e-01 / 1.32e+00, 2.83e-01 / 1.38e-01 | 20260506 23:25:30.160 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.37e-01 / 3.91e-01, 5.31e-02 / 2.76e-02 | 20260506 23:25:30.160 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:25:30.160 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:25:30.160 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:25:30.160 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 23:25:30.160 | INFO | PerfReport: 21.60%, Upload -- 2.08e+01 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260506 23:25:30.160 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260506 23:25:30.160 | INFO | PerfReport: 32.16%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 23:25:30.160 | INFO | PerfReport: 1.25%, LockUpload -- 1.20e+00 / 0.00e+00, 4.67e+00 / 0.00e+00 | 20260506 23:25:30.160 | INFO | PerfReport: 1.34%, LockDownload -- 1.29e+00 / 0.00e+00, 3.23e+00 / 0.00e+00 | 20260506 23:25:30.160 | INFO | PerfReport: 22.25%, LockCompute -- 2.15e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 23:25:30.160 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260506 23:25:30.257 | INFO | Starting a round of Baz IO | 20260506 23:25:33.385 | INFO | Populating RTMetrics report | 20260506 23:25:33.386 | INFO | RTMetrics report populated | 20260506 23:25:33.386 | INFO | Opening tmp RTMetrics json file | 20260506 23:25:33.514 | INFO | Serializing RTMetrics to json | 20260506 23:25:33.710 | INFO | Serialization done, swapping tmp file now... | 20260506 23:25:33.783 | INFO | RTMetrics done | 20260506 23:25:34.030 | INFO | Finished a round of Baz IO: Wrote 1.879 GiB in 3.773 seconds | 20260506 23:25:34.030 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.47% / 0.00% / 66.53% | 20260506 23:25:34.030 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.415806% / 0% / 5.16574e-05% | 20260506 23:25:34.030 | INFO | Baz IO overall: | 20260506 23:25:34.030 | INFO | BytesWritten : 2766077241430 | 20260506 23:25:34.030 | INFO | NumEvents : 1333628421001 | 20260506 23:25:35.295 | INFO | Analyzing chunk frames = [5283840,5284352) | 20260506 23:25:35.295 | INFO | MemInfo: { | 20260506 23:25:35.295 | INFO | "gpu" : " 17 GB", | 20260506 23:25:35.295 | INFO | "huge" : "", | 20260506 23:25:35.295 | INFO | "huge_shared" : " 27 GB", | 20260506 23:25:35.295 | INFO | "malloc" : " 14 GB", | 20260506 23:25:35.295 | INFO | "pinned" : " 29 GB" | 20260506 23:25:35.295 | INFO | } | 20260506 23:25:35.295 | INFO | Perf report for the last 5.13706 seconds: | 20260506 23:25:35.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:25:35.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:35.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:35.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:35.295 | INFO | ANALYSIS : 62.3%, 2.9, 95.4ms, 0 | 20260506 23:25:35.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:35.295 | INFO | RT_METRICS : 50.1%, 1.0, 26.3ms, 0 | 20260506 23:25:35.296 | INFO | PRE_HQ : 59.7%, 1.8, 56.0ms, 0 | 20260506 23:25:35.296 | INFO | BAZWRITER : 73.0%, 4.8, 180.9ms, 0 | 20260506 23:25:35.296 | INFO | BAZ_SAMPLING : 2.5%, 1.0, 1.3ms, 0 | 20260506 23:25:35.296 | INFO | Uploaded 36GiB to GPU in 2.04776s (17.5802GiB/s) | 20260506 23:25:35.296 | INFO | Downloaded 38GiB to GPU in 2.19778s (17.2902GiB/s) | 20260506 23:25:40.437 | INFO | Analyzing chunk frames = [5284352,5284864) | 20260506 23:25:40.437 | INFO | MemInfo: { | 20260506 23:25:40.437 | INFO | "gpu" : " 17 GB", | 20260506 23:25:40.437 | INFO | "huge" : "", | 20260506 23:25:40.437 | INFO | "huge_shared" : " 27 GB", | 20260506 23:25:40.437 | INFO | "malloc" : " 14 GB", | 20260506 23:25:40.437 | INFO | "pinned" : " 29 GB" | 20260506 23:25:40.437 | INFO | } | 20260506 23:25:40.437 | INFO | Perf report for the last 5.1417 seconds: | 20260506 23:25:40.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:25:40.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:40.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:40.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:40.437 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:25:40.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:40.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:40.437 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:25:40.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:40.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:40.437 | INFO | Uploaded 36GiB to GPU in 2.00309s (17.9722GiB/s) | 20260506 23:25:40.438 | INFO | Downloaded 34GiB to GPU in 1.92757s (17.6388GiB/s) | 20260506 23:25:45.575 | INFO | Analyzing chunk frames = [5284864,5285376) | 20260506 23:25:45.575 | INFO | MemInfo: { | 20260506 23:25:45.575 | INFO | "gpu" : " 17 GB", | 20260506 23:25:45.575 | INFO | "huge" : "", | 20260506 23:25:45.575 | INFO | "huge_shared" : " 27 GB", | 20260506 23:25:45.575 | INFO | "malloc" : " 14 GB", | 20260506 23:25:45.575 | INFO | "pinned" : " 29 GB" | 20260506 23:25:45.575 | INFO | } | 20260506 23:25:45.575 | INFO | Perf report for the last 5.13811 seconds: | 20260506 23:25:45.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:25:45.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:45.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:45.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:45.575 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:25:45.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:45.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:45.575 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:25:45.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:45.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:45.575 | INFO | Uploaded 36GiB to GPU in 2.00194s (17.9826GiB/s) | 20260506 23:25:45.575 | INFO | Downloaded 34GiB to GPU in 1.93095s (17.6079GiB/s) | 20260506 23:25:50.712 | INFO | Analyzing chunk frames = [5285376,5285888) | 20260506 23:25:50.712 | INFO | MemInfo: { | 20260506 23:25:50.712 | INFO | "gpu" : " 17 GB", | 20260506 23:25:50.712 | INFO | "huge" : "", | 20260506 23:25:50.712 | INFO | "huge_shared" : " 27 GB", | 20260506 23:25:50.712 | INFO | "malloc" : " 14 GB", | 20260506 23:25:50.712 | INFO | "pinned" : " 29 GB" | 20260506 23:25:50.712 | INFO | } | 20260506 23:25:50.713 | INFO | Perf report for the last 5.13742 seconds: | 20260506 23:25:50.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:25:50.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:50.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:50.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:50.713 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260506 23:25:50.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:50.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:50.713 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:25:50.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:50.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:50.713 | INFO | Uploaded 36GiB to GPU in 2.01315s (17.8824GiB/s) | 20260506 23:25:50.713 | INFO | Downloaded 34GiB to GPU in 1.90931s (17.8075GiB/s) | 20260506 23:25:55.848 | INFO | Analyzing chunk frames = [5285888,5286400) | 20260506 23:25:55.848 | INFO | MemInfo: { | 20260506 23:25:55.848 | INFO | "gpu" : " 17 GB", | 20260506 23:25:55.848 | INFO | "huge" : "", | 20260506 23:25:55.848 | INFO | "huge_shared" : " 27 GB", | 20260506 23:25:55.848 | INFO | "malloc" : " 14 GB", | 20260506 23:25:55.848 | INFO | "pinned" : " 29 GB" | 20260506 23:25:55.848 | INFO | } | 20260506 23:25:55.848 | INFO | Perf report for the last 5.1359 seconds: | 20260506 23:25:55.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:25:55.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:55.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:55.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:55.848 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 23:25:55.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:55.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:55.849 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:25:55.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:55.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:25:55.849 | INFO | Uploaded 36GiB to GPU in 1.99534s (18.042GiB/s) | 20260506 23:25:55.849 | INFO | Downloaded 34GiB to GPU in 1.9267s (17.6467GiB/s) | 20260506 23:26:00.988 | INFO | Analyzing chunk frames = [5286400,5286912) | 20260506 23:26:00.988 | INFO | MemInfo: { | 20260506 23:26:00.988 | INFO | "gpu" : " 17 GB", | 20260506 23:26:00.988 | INFO | "huge" : "", | 20260506 23:26:00.988 | INFO | "huge_shared" : " 27 GB", | 20260506 23:26:00.988 | INFO | "malloc" : " 14 GB", | 20260506 23:26:00.988 | INFO | "pinned" : " 29 GB" | 20260506 23:26:00.988 | INFO | } | 20260506 23:26:00.988 | INFO | Perf report for the last 5.13981 seconds: | 20260506 23:26:00.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:26:00.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:00.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:00.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:00.988 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 23:26:00.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:00.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:00.988 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 23:26:00.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:00.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:00.989 | INFO | Uploaded 36GiB to GPU in 2.02352s (17.7908GiB/s) | 20260506 23:26:00.989 | INFO | Downloaded 34GiB to GPU in 1.95133s (17.424GiB/s) | 20260506 23:26:06.135 | INFO | Analyzing chunk frames = [5286912,5287424) | 20260506 23:26:06.135 | INFO | MemInfo: { | 20260506 23:26:06.135 | INFO | "gpu" : " 17 GB", | 20260506 23:26:06.135 | INFO | "huge" : "", | 20260506 23:26:06.135 | INFO | "huge_shared" : " 27 GB", | 20260506 23:26:06.136 | INFO | "malloc" : " 14 GB", | 20260506 23:26:06.136 | INFO | "pinned" : " 29 GB" | 20260506 23:26:06.136 | INFO | } | 20260506 23:26:06.136 | INFO | Perf report for the last 5.14727 seconds: | 20260506 23:26:06.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:26:06.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:06.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:06.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:06.136 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:26:06.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:06.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:06.136 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:26:06.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:06.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:06.136 | INFO | Uploaded 36GiB to GPU in 1.97098s (18.2651GiB/s) | 20260506 23:26:06.136 | INFO | Downloaded 34GiB to GPU in 1.99019s (17.0838GiB/s) | 20260506 23:26:11.276 | INFO | Analyzing chunk frames = [5287424,5287936) | 20260506 23:26:11.276 | INFO | MemInfo: { | 20260506 23:26:11.276 | INFO | "gpu" : " 17 GB", | 20260506 23:26:11.276 | INFO | "huge" : "", | 20260506 23:26:11.276 | INFO | "huge_shared" : " 27 GB", | 20260506 23:26:11.276 | INFO | "malloc" : " 14 GB", | 20260506 23:26:11.276 | INFO | "pinned" : " 29 GB" | 20260506 23:26:11.276 | INFO | } | 20260506 23:26:11.276 | INFO | Perf report for the last 5.14022 seconds: | 20260506 23:26:11.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:26:11.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:11.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:11.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:11.276 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:26:11.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:11.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:11.276 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:26:11.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:11.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:11.276 | INFO | Uploaded 36GiB to GPU in 1.98571s (18.1295GiB/s) | 20260506 23:26:11.276 | INFO | Downloaded 34GiB to GPU in 1.97397s (17.2242GiB/s) | 20260506 23:26:11.374 | INFO | Starting a round of Baz IO | 20260506 23:26:14.497 | INFO | Populating RTMetrics report | 20260506 23:26:14.497 | INFO | RTMetrics report populated | 20260506 23:26:14.497 | INFO | Opening tmp RTMetrics json file | 20260506 23:26:14.582 | INFO | Serializing RTMetrics to json | 20260506 23:26:14.626 | INFO | Serialization done, swapping tmp file now... | 20260506 23:26:14.627 | INFO | RTMetrics done | 20260506 23:26:14.769 | INFO | Finished a round of Baz IO: Wrote 1.878 GiB in 3.395 seconds | 20260506 23:26:14.769 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.47% / 0.00% / 66.53% | 20260506 23:26:14.769 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.415003% / 0% / 6.75519e-05% | 20260506 23:26:14.769 | INFO | Baz IO overall: | 20260506 23:26:14.769 | INFO | BytesWritten : 2768094071915 | 20260506 23:26:14.769 | INFO | NumEvents : 1334588773222 | 20260506 23:26:16.409 | INFO | Analyzing chunk frames = [5287936,5288448) | 20260506 23:26:16.409 | INFO | MemInfo: { | 20260506 23:26:16.409 | INFO | "gpu" : " 17 GB", | 20260506 23:26:16.410 | INFO | "huge" : "", | 20260506 23:26:16.410 | INFO | "huge_shared" : " 27 GB", | 20260506 23:26:16.410 | INFO | "malloc" : " 14 GB", | 20260506 23:26:16.410 | INFO | "pinned" : " 29 GB" | 20260506 23:26:16.410 | INFO | } | 20260506 23:26:16.410 | INFO | Perf report for the last 5.13387 seconds: | 20260506 23:26:16.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:26:16.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:16.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:16.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:16.410 | INFO | ANALYSIS : 62.2%, 2.9, 95.3ms, 0 | 20260506 23:26:16.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:16.410 | INFO | RT_METRICS : 44.8%, 1.0, 23.4ms, 0 | 20260506 23:26:16.410 | INFO | PRE_HQ : 59.7%, 1.8, 56.2ms, 0 | 20260506 23:26:16.410 | INFO | BAZWRITER : 65.8%, 4.8, 161.2ms, 0 | 20260506 23:26:16.410 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 23:26:16.410 | INFO | Uploaded 36GiB to GPU in 2.06798s (17.4083GiB/s) | 20260506 23:26:16.410 | INFO | Downloaded 38GiB to GPU in 2.1984s (17.2853GiB/s) | 20260506 23:26:21.552 | INFO | Analyzing chunk frames = [5288448,5288960) | 20260506 23:26:21.552 | INFO | MemInfo: { | 20260506 23:26:21.552 | INFO | "gpu" : " 17 GB", | 20260506 23:26:21.552 | INFO | "huge" : "", | 20260506 23:26:21.552 | INFO | "huge_shared" : " 27 GB", | 20260506 23:26:21.552 | INFO | "malloc" : " 14 GB", | 20260506 23:26:21.552 | INFO | "pinned" : " 29 GB" | 20260506 23:26:21.552 | INFO | } | 20260506 23:26:21.552 | INFO | Perf report for the last 5.14258 seconds: | 20260506 23:26:21.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:26:21.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:21.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:21.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:21.552 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:26:21.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:21.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:21.553 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:26:21.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:21.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:21.553 | INFO | Uploaded 36GiB to GPU in 2.00269s (17.9758GiB/s) | 20260506 23:26:21.553 | INFO | Downloaded 34GiB to GPU in 1.9274s (17.6403GiB/s) | 20260506 23:26:26.687 | INFO | Analyzing chunk frames = [5288960,5289472) | 20260506 23:26:26.687 | INFO | MemInfo: { | 20260506 23:26:26.687 | INFO | "gpu" : " 17 GB", | 20260506 23:26:26.687 | INFO | "huge" : "", | 20260506 23:26:26.687 | INFO | "huge_shared" : " 27 GB", | 20260506 23:26:26.687 | INFO | "malloc" : " 14 GB", | 20260506 23:26:26.687 | INFO | "pinned" : " 29 GB" | 20260506 23:26:26.687 | INFO | } | 20260506 23:26:26.687 | INFO | Perf report for the last 5.13503 seconds: | 20260506 23:26:26.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:26:26.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:26.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:26.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:26.687 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 23:26:26.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:26.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:26.687 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:26:26.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:26.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:26.688 | INFO | Uploaded 36GiB to GPU in 1.99924s (18.0069GiB/s) | 20260506 23:26:26.688 | INFO | Downloaded 34GiB to GPU in 1.9316s (17.602GiB/s) | 20260506 23:26:31.827 | INFO | Analyzing chunk frames = [5289472,5289984) | 20260506 23:26:31.827 | INFO | MemInfo: { | 20260506 23:26:31.827 | INFO | "gpu" : " 17 GB", | 20260506 23:26:31.827 | INFO | "huge" : "", | 20260506 23:26:31.827 | INFO | "huge_shared" : " 27 GB", | 20260506 23:26:31.827 | INFO | "malloc" : " 14 GB", | 20260506 23:26:31.827 | INFO | "pinned" : " 29 GB" | 20260506 23:26:31.827 | INFO | } | 20260506 23:26:31.827 | INFO | Perf report for the last 5.13995 seconds: | 20260506 23:26:31.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:26:31.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:31.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:31.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:31.827 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 23:26:31.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:31.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:31.827 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:26:31.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:31.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:31.828 | INFO | Uploaded 36GiB to GPU in 1.95551s (18.4096GiB/s) | 20260506 23:26:31.828 | INFO | Downloaded 34GiB to GPU in 1.87773s (18.107GiB/s) | 20260506 23:26:36.964 | INFO | Analyzing chunk frames = [5289984,5290496) | 20260506 23:26:36.964 | INFO | MemInfo: { | 20260506 23:26:36.964 | INFO | "gpu" : " 17 GB", | 20260506 23:26:36.964 | INFO | "huge" : "", | 20260506 23:26:36.964 | INFO | "huge_shared" : " 27 GB", | 20260506 23:26:36.964 | INFO | "malloc" : " 14 GB", | 20260506 23:26:36.964 | INFO | "pinned" : " 29 GB" | 20260506 23:26:36.964 | INFO | } | 20260506 23:26:36.965 | INFO | Perf report for the last 5.13725 seconds: | 20260506 23:26:36.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:26:36.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:36.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:36.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:36.965 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260506 23:26:36.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:36.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:36.965 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:26:36.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:36.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:36.965 | INFO | Uploaded 36GiB to GPU in 2.00533s (17.9522GiB/s) | 20260506 23:26:36.965 | INFO | Downloaded 34GiB to GPU in 1.91964s (17.7116GiB/s) | 20260506 23:26:42.107 | INFO | Analyzing chunk frames = [5290496,5291008) | 20260506 23:26:42.107 | INFO | MemInfo: { | 20260506 23:26:42.107 | INFO | "gpu" : " 17 GB", | 20260506 23:26:42.107 | INFO | "huge" : "", | 20260506 23:26:42.107 | INFO | "huge_shared" : " 27 GB", | 20260506 23:26:42.107 | INFO | "malloc" : " 14 GB", | 20260506 23:26:42.107 | INFO | "pinned" : " 29 GB" | 20260506 23:26:42.107 | INFO | } | 20260506 23:26:42.107 | INFO | Perf report for the last 5.14276 seconds: | 20260506 23:26:42.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:26:42.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:42.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:42.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:42.107 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 23:26:42.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:42.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:42.107 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 23:26:42.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:42.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:42.108 | INFO | Uploaded 36GiB to GPU in 2.00761s (17.9318GiB/s) | 20260506 23:26:42.108 | INFO | Downloaded 34GiB to GPU in 1.94789s (17.4548GiB/s) | 20260506 23:26:47.248 | INFO | Analyzing chunk frames = [5291008,5291520) | 20260506 23:26:47.248 | INFO | MemInfo: { | 20260506 23:26:47.248 | INFO | "gpu" : " 17 GB", | 20260506 23:26:47.248 | INFO | "huge" : "", | 20260506 23:26:47.248 | INFO | "huge_shared" : " 27 GB", | 20260506 23:26:47.248 | INFO | "malloc" : " 14 GB", | 20260506 23:26:47.248 | INFO | "pinned" : " 29 GB" | 20260506 23:26:47.248 | INFO | } | 20260506 23:26:47.248 | INFO | Perf report for the last 5.14032 seconds: | 20260506 23:26:47.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:26:47.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:47.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:47.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:47.248 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:26:47.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:47.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:47.248 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:26:47.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:47.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:47.248 | INFO | Uploaded 36GiB to GPU in 1.96711s (18.301GiB/s) | 20260506 23:26:47.248 | INFO | Downloaded 34GiB to GPU in 1.99077s (17.0789GiB/s) | 20260506 23:26:52.389 | INFO | Analyzing chunk frames = [5291520,5292032) | 20260506 23:26:52.390 | INFO | MemInfo: { | 20260506 23:26:52.390 | INFO | "gpu" : " 17 GB", | 20260506 23:26:52.390 | INFO | "huge" : "", | 20260506 23:26:52.390 | INFO | "huge_shared" : " 27 GB", | 20260506 23:26:52.390 | INFO | "malloc" : " 14 GB", | 20260506 23:26:52.390 | INFO | "pinned" : " 29 GB" | 20260506 23:26:52.390 | INFO | } | 20260506 23:26:52.390 | INFO | Perf report for the last 5.14214 seconds: | 20260506 23:26:52.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:26:52.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:52.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:52.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:52.390 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:26:52.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:52.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:52.390 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260506 23:26:52.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:52.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:52.390 | INFO | Uploaded 36GiB to GPU in 1.97397s (18.2373GiB/s) | 20260506 23:26:52.390 | INFO | Downloaded 34GiB to GPU in 1.9956s (17.0375GiB/s) | 20260506 23:26:52.390 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:26:52.390 | INFO | Total mem usage: 18 GiB | 20260506 23:26:52.390 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:26:52.390 | INFO | Recent High Water Marks Sum: 16 GiB (Savings: -12.7%) | 20260506 23:26:52.390 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:26:52.390 | INFO | High Water Breakdown: | 20260506 23:26:52.390 | INFO | 61.9% : 9.89 GiB ( 86.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:26:52.390 | INFO | 38% : 6.07 GiB ( 90.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:26:52.390 | INFO | 0.125% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:26:52.390 | INFO | ------------------------------------------ | 20260506 23:26:52.390 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:26:52.390 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:26:52.390 | INFO | Total mem usage: 26.5 GiB | 20260506 23:26:52.390 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:26:52.390 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:26:52.390 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:26:52.390 | INFO | High Water Breakdown: | 20260506 23:26:52.390 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:26:52.390 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:26:52.391 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:26:52.391 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:26:52.391 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:26:52.391 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:26:52.391 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:26:52.391 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:26:52.391 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:26:52.391 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:26:52.391 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:26:52.391 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:26:52.391 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:26:52.391 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:26:52.391 | INFO | ------------------------------------------ | 20260506 23:26:52.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:26:52.391 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:26:52.391 | INFO | Total mem usage: 25.6 GiB | 20260506 23:26:52.391 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:26:52.391 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:26:52.391 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:26:52.391 | INFO | High Water Breakdown: | 20260506 23:26:52.391 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:26:52.391 | INFO | ------------------------------------------ | 20260506 23:26:52.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:26:52.391 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:26:52.391 | INFO | Total mem usage: 15.7 GiB | 20260506 23:26:52.391 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:26:52.391 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:26:52.391 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 23:26:52.391 | INFO | High Water Breakdown: | 20260506 23:26:52.391 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:26:52.391 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:26:52.391 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:26:52.391 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:26:52.391 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:26:52.391 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:26:52.391 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:26:52.391 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:26:52.391 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:26:52.391 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:26:52.391 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:26:52.391 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:26:52.391 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:26:52.391 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:26:52.391 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:26:52.391 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:26:52.391 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:26:52.391 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:26:52.391 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:26:52.391 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:26:52.391 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:26:52.391 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:26:52.391 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:26:52.391 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:26:52.391 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:26:52.391 | INFO | ------------------------------------------ | 20260506 23:26:52.391 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:26:52.391 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:26:52.391 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:26:52.391 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:26:52.391 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.66e+01, 1.35e+01 / 2.33e+00 | 20260506 23:26:52.391 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.02e-01, 1.01e-02 / 6.10e-03 | 20260506 23:26:52.391 | INFO | PerfReport: 11.67%, Baseline -- 3.62e+00 / 4.40e+00, 5.46e-01 / 2.73e-01 | 20260506 23:26:52.391 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.79e-01 / 1.15e-01 | 20260506 23:26:52.391 | INFO | PerfReport: 16.09%, DME -- 3.99e+01 / 4.53e+01, 4.45e+00 / 2.55e+00 | 20260506 23:26:52.391 | INFO | PerfReport: 57.38%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.38e+00 / 1.85e+00 | 20260506 23:26:52.391 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.18e+00 / 2.72e+00, 3.78e-01 / 2.41e-01 | 20260506 23:26:52.391 | INFO | PerfReport: 3.19%, Metrics -- 9.88e-01 / 1.33e+00, 2.65e-01 / 1.63e-01 | 20260506 23:26:52.391 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.37e-01 / 3.86e-01, 5.30e-02 / 2.37e-02 | 20260506 23:26:52.391 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:26:52.391 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:26:52.391 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:26:52.391 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 23:26:52.391 | INFO | PerfReport: 21.61%, Upload -- 2.08e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260506 23:26:52.391 | INFO | PerfReport: 21.32%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 23:26:52.391 | INFO | PerfReport: 32.16%, Compute -- 3.10e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 23:26:52.391 | INFO | PerfReport: 1.30%, LockUpload -- 1.25e+00 / 0.00e+00, 4.67e+00 / 0.00e+00 | 20260506 23:26:52.391 | INFO | PerfReport: 1.21%, LockDownload -- 1.17e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260506 23:26:52.391 | INFO | PerfReport: 22.39%, LockCompute -- 2.16e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 23:26:52.391 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260506 23:26:52.489 | INFO | Starting a round of Baz IO | 20260506 23:26:55.608 | INFO | Populating RTMetrics report | 20260506 23:26:55.609 | INFO | RTMetrics report populated | 20260506 23:26:55.609 | INFO | Opening tmp RTMetrics json file | 20260506 23:26:55.887 | INFO | Serializing RTMetrics to json | 20260506 23:26:55.940 | INFO | Serialization done, swapping tmp file now... | 20260506 23:26:55.965 | INFO | RTMetrics done | 20260506 23:26:56.118 | INFO | Finished a round of Baz IO: Wrote 1.878 GiB in 3.629 seconds | 20260506 23:26:56.118 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.47% / 0.00% / 66.53% | 20260506 23:26:56.118 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.414801% / 7.94729e-06% / 5.16574e-05% | 20260506 23:26:56.118 | INFO | Baz IO overall: | 20260506 23:26:56.118 | INFO | BytesWritten : 2770110847375 | 20260506 23:26:56.118 | INFO | NumEvents : 1335549324211 | 20260506 23:26:57.528 | INFO | Analyzing chunk frames = [5292032,5292544) | 20260506 23:26:57.528 | INFO | MemInfo: { | 20260506 23:26:57.528 | INFO | "gpu" : " 17 GB", | 20260506 23:26:57.528 | INFO | "huge" : "", | 20260506 23:26:57.528 | INFO | "huge_shared" : " 27 GB", | 20260506 23:26:57.528 | INFO | "malloc" : " 14 GB", | 20260506 23:26:57.528 | INFO | "pinned" : " 29 GB" | 20260506 23:26:57.528 | INFO | } | 20260506 23:26:57.528 | INFO | Perf report for the last 5.13793 seconds: | 20260506 23:26:57.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:26:57.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:57.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:57.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:57.528 | INFO | ANALYSIS : 62.1%, 2.9, 94.9ms, 0 | 20260506 23:26:57.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:26:57.528 | INFO | RT_METRICS : 49.5%, 1.0, 26.0ms, 0 | 20260506 23:26:57.528 | INFO | PRE_HQ : 59.8%, 1.8, 56.3ms, 0 | 20260506 23:26:57.528 | INFO | BAZWRITER : 70.2%, 4.8, 172.7ms, 0 | 20260506 23:26:57.528 | INFO | BAZ_SAMPLING : 5.2%, 1.0, 2.7ms, 0 | 20260506 23:26:57.528 | INFO | Uploaded 36GiB to GPU in 2.05379s (17.5286GiB/s) | 20260506 23:26:57.528 | INFO | Downloaded 38GiB to GPU in 2.20447s (17.2377GiB/s) | 20260506 23:27:02.660 | INFO | Analyzing chunk frames = [5292544,5293056) | 20260506 23:27:02.660 | INFO | MemInfo: { | 20260506 23:27:02.660 | INFO | "gpu" : " 17 GB", | 20260506 23:27:02.660 | INFO | "huge" : "", | 20260506 23:27:02.660 | INFO | "huge_shared" : " 27 GB", | 20260506 23:27:02.660 | INFO | "malloc" : " 14 GB", | 20260506 23:27:02.660 | INFO | "pinned" : " 29 GB" | 20260506 23:27:02.660 | INFO | } | 20260506 23:27:02.660 | INFO | Perf report for the last 5.13243 seconds: | 20260506 23:27:02.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:27:02.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:02.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:02.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:02.660 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 23:27:02.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:02.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:02.660 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:27:02.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:02.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:02.661 | INFO | Uploaded 36GiB to GPU in 1.99204s (18.0719GiB/s) | 20260506 23:27:02.661 | INFO | Downloaded 34GiB to GPU in 1.93562s (17.5654GiB/s) | 20260506 23:27:07.797 | INFO | Analyzing chunk frames = [5293056,5293568) | 20260506 23:27:07.797 | INFO | MemInfo: { | 20260506 23:27:07.797 | INFO | "gpu" : " 17 GB", | 20260506 23:27:07.797 | INFO | "huge" : "", | 20260506 23:27:07.797 | INFO | "huge_shared" : " 27 GB", | 20260506 23:27:07.798 | INFO | "malloc" : " 14 GB", | 20260506 23:27:07.798 | INFO | "pinned" : " 29 GB" | 20260506 23:27:07.798 | INFO | } | 20260506 23:27:07.798 | INFO | Perf report for the last 5.13743 seconds: | 20260506 23:27:07.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:27:07.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:07.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:07.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:07.798 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 23:27:07.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:07.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:07.798 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:27:07.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:07.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:07.798 | INFO | Uploaded 36GiB to GPU in 1.95499s (18.4144GiB/s) | 20260506 23:27:07.798 | INFO | Downloaded 34GiB to GPU in 1.92217s (17.6884GiB/s) | 20260506 23:27:12.939 | INFO | Analyzing chunk frames = [5293568,5294080) | 20260506 23:27:12.939 | INFO | MemInfo: { | 20260506 23:27:12.939 | INFO | "gpu" : " 17 GB", | 20260506 23:27:12.940 | INFO | "huge" : "", | 20260506 23:27:12.940 | INFO | "huge_shared" : " 27 GB", | 20260506 23:27:12.940 | INFO | "malloc" : " 14 GB", | 20260506 23:27:12.940 | INFO | "pinned" : " 29 GB" | 20260506 23:27:12.940 | INFO | } | 20260506 23:27:12.940 | INFO | Perf report for the last 5.14214 seconds: | 20260506 23:27:12.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:27:12.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:12.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:12.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:12.940 | INFO | ANALYSIS : 59.5%, 3.0, 92.2ms, 0 | 20260506 23:27:12.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:12.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:12.940 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:27:12.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:12.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:12.940 | INFO | Uploaded 36GiB to GPU in 1.91265s (18.8221GiB/s) | 20260506 23:27:12.940 | INFO | Downloaded 34GiB to GPU in 1.87361s (18.1468GiB/s) | 20260506 23:27:18.081 | INFO | Analyzing chunk frames = [5294080,5294592) | 20260506 23:27:18.081 | INFO | MemInfo: { | 20260506 23:27:18.081 | INFO | "gpu" : " 17 GB", | 20260506 23:27:18.081 | INFO | "huge" : "", | 20260506 23:27:18.081 | INFO | "huge_shared" : " 27 GB", | 20260506 23:27:18.081 | INFO | "malloc" : " 14 GB", | 20260506 23:27:18.082 | INFO | "pinned" : " 29 GB" | 20260506 23:27:18.082 | INFO | } | 20260506 23:27:18.082 | INFO | Perf report for the last 5.14183 seconds: | 20260506 23:27:18.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:27:18.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:18.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:18.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:18.082 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 23:27:18.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:18.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:18.082 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:27:18.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:18.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:18.082 | INFO | Uploaded 36GiB to GPU in 2.00316s (17.9716GiB/s) | 20260506 23:27:18.082 | INFO | Downloaded 34GiB to GPU in 1.92016s (17.7069GiB/s) | 20260506 23:27:23.222 | INFO | Analyzing chunk frames = [5294592,5295104) | 20260506 23:27:23.222 | INFO | MemInfo: { | 20260506 23:27:23.222 | INFO | "gpu" : " 17 GB", | 20260506 23:27:23.222 | INFO | "huge" : "", | 20260506 23:27:23.222 | INFO | "huge_shared" : " 27 GB", | 20260506 23:27:23.222 | INFO | "malloc" : " 14 GB", | 20260506 23:27:23.222 | INFO | "pinned" : " 29 GB" | 20260506 23:27:23.222 | INFO | } | 20260506 23:27:23.222 | INFO | Perf report for the last 5.14067 seconds: | 20260506 23:27:23.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:27:23.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:23.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:23.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:23.222 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 23:27:23.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:23.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:23.222 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 23:27:23.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:23.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:23.223 | INFO | Uploaded 36GiB to GPU in 2.00809s (17.9275GiB/s) | 20260506 23:27:23.223 | INFO | Downloaded 34GiB to GPU in 1.94946s (17.4407GiB/s) | 20260506 23:27:28.366 | INFO | Analyzing chunk frames = [5295104,5295616) | 20260506 23:27:28.366 | INFO | MemInfo: { | 20260506 23:27:28.366 | INFO | "gpu" : " 17 GB", | 20260506 23:27:28.366 | INFO | "huge" : "", | 20260506 23:27:28.366 | INFO | "huge_shared" : " 27 GB", | 20260506 23:27:28.366 | INFO | "malloc" : " 14 GB", | 20260506 23:27:28.366 | INFO | "pinned" : " 29 GB" | 20260506 23:27:28.366 | INFO | } | 20260506 23:27:28.366 | INFO | Perf report for the last 5.14362 seconds: | 20260506 23:27:28.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:27:28.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:28.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:28.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:28.366 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:27:28.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:28.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:28.366 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 23:27:28.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:28.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:28.366 | INFO | Uploaded 36GiB to GPU in 1.95082s (18.4538GiB/s) | 20260506 23:27:28.366 | INFO | Downloaded 34GiB to GPU in 1.9966s (17.029GiB/s) | 20260506 23:27:33.505 | INFO | Analyzing chunk frames = [5295616,5296128) | 20260506 23:27:33.505 | INFO | MemInfo: { | 20260506 23:27:33.505 | INFO | "gpu" : " 17 GB", | 20260506 23:27:33.505 | INFO | "huge" : "", | 20260506 23:27:33.505 | INFO | "huge_shared" : " 27 GB", | 20260506 23:27:33.505 | INFO | "malloc" : " 14 GB", | 20260506 23:27:33.505 | INFO | "pinned" : " 29 GB" | 20260506 23:27:33.505 | INFO | } | 20260506 23:27:33.505 | INFO | Perf report for the last 5.1396 seconds: | 20260506 23:27:33.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:27:33.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:33.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:33.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:33.506 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:27:33.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:33.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:33.506 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 23:27:33.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:33.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:33.506 | INFO | Uploaded 36GiB to GPU in 1.9899s (18.0913GiB/s) | 20260506 23:27:33.506 | INFO | Downloaded 34GiB to GPU in 1.98673s (17.1135GiB/s) | 20260506 23:27:33.606 | INFO | Starting a round of Baz IO | 20260506 23:27:36.732 | INFO | Populating RTMetrics report | 20260506 23:27:36.733 | INFO | RTMetrics report populated | 20260506 23:27:36.733 | INFO | Opening tmp RTMetrics json file | 20260506 23:27:36.754 | INFO | Serializing RTMetrics to json | 20260506 23:27:36.761 | INFO | Serialization done, swapping tmp file now... | 20260506 23:27:36.764 | INFO | RTMetrics done | 20260506 23:27:37.003 | INFO | Finished a round of Baz IO: Wrote 1.881 GiB in 3.397 seconds | 20260506 23:27:37.003 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.46% / 0.00% / 66.53% | 20260506 23:27:37.003 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.414296% / 7.94729e-06% / 6.35783e-05% | 20260506 23:27:37.003 | INFO | Baz IO overall: | 20260506 23:27:37.003 | INFO | BytesWritten : 2772130825436 | 20260506 23:27:37.003 | INFO | NumEvents : 1336511054922 | 20260506 23:27:38.643 | INFO | Analyzing chunk frames = [5296128,5296640) | 20260506 23:27:38.643 | INFO | MemInfo: { | 20260506 23:27:38.643 | INFO | "gpu" : " 17 GB", | 20260506 23:27:38.643 | INFO | "huge" : "", | 20260506 23:27:38.643 | INFO | "huge_shared" : " 27 GB", | 20260506 23:27:38.643 | INFO | "malloc" : " 14 GB", | 20260506 23:27:38.643 | INFO | "pinned" : " 29 GB" | 20260506 23:27:38.643 | INFO | } | 20260506 23:27:38.643 | INFO | Perf report for the last 5.1376 seconds: | 20260506 23:27:38.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:27:38.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:38.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:38.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:38.643 | INFO | ANALYSIS : 62.2%, 2.9, 95.1ms, 0 | 20260506 23:27:38.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:38.643 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 23:27:38.643 | INFO | PRE_HQ : 59.8%, 1.8, 56.6ms, 0 | 20260506 23:27:38.643 | INFO | BAZWRITER : 65.7%, 4.8, 161.1ms, 0 | 20260506 23:27:38.643 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260506 23:27:38.643 | INFO | Uploaded 36GiB to GPU in 2.05899s (17.4843GiB/s) | 20260506 23:27:38.643 | INFO | Downloaded 38GiB to GPU in 2.19174s (17.3378GiB/s) | 20260506 23:27:43.779 | INFO | Analyzing chunk frames = [5296640,5297152) | 20260506 23:27:43.779 | INFO | MemInfo: { | 20260506 23:27:43.779 | INFO | "gpu" : " 17 GB", | 20260506 23:27:43.779 | INFO | "huge" : "", | 20260506 23:27:43.779 | INFO | "huge_shared" : " 27 GB", | 20260506 23:27:43.779 | INFO | "malloc" : " 14 GB", | 20260506 23:27:43.779 | INFO | "pinned" : " 29 GB" | 20260506 23:27:43.779 | INFO | } | 20260506 23:27:43.779 | INFO | Perf report for the last 5.1363 seconds: | 20260506 23:27:43.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:27:43.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:43.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:43.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:43.779 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:27:43.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:43.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:43.780 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:27:43.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:43.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:43.780 | INFO | Uploaded 36GiB to GPU in 2.01529s (17.8635GiB/s) | 20260506 23:27:43.780 | INFO | Downloaded 34GiB to GPU in 1.92642s (17.6493GiB/s) | 20260506 23:27:48.913 | INFO | Analyzing chunk frames = [5297152,5297664) | 20260506 23:27:48.914 | INFO | MemInfo: { | 20260506 23:27:48.914 | INFO | "gpu" : " 17 GB", | 20260506 23:27:48.914 | INFO | "huge" : "", | 20260506 23:27:48.914 | INFO | "huge_shared" : " 27 GB", | 20260506 23:27:48.914 | INFO | "malloc" : " 14 GB", | 20260506 23:27:48.914 | INFO | "pinned" : " 29 GB" | 20260506 23:27:48.914 | INFO | } | 20260506 23:27:48.914 | INFO | Perf report for the last 5.13431 seconds: | 20260506 23:27:48.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:27:48.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:48.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:48.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:48.914 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 23:27:48.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:48.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:48.914 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:27:48.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:48.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:48.914 | INFO | Uploaded 36GiB to GPU in 1.99044s (18.0864GiB/s) | 20260506 23:27:48.914 | INFO | Downloaded 34GiB to GPU in 1.92541s (17.6586GiB/s) | 20260506 23:27:54.057 | INFO | Analyzing chunk frames = [5297664,5298176) | 20260506 23:27:54.057 | INFO | MemInfo: { | 20260506 23:27:54.057 | INFO | "gpu" : " 17 GB", | 20260506 23:27:54.057 | INFO | "huge" : "", | 20260506 23:27:54.057 | INFO | "huge_shared" : " 27 GB", | 20260506 23:27:54.057 | INFO | "malloc" : " 14 GB", | 20260506 23:27:54.057 | INFO | "pinned" : " 29 GB" | 20260506 23:27:54.057 | INFO | } | 20260506 23:27:54.057 | INFO | Perf report for the last 5.1435 seconds: | 20260506 23:27:54.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:27:54.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:54.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:54.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:54.057 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260506 23:27:54.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:54.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:54.058 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:27:54.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:54.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:54.058 | INFO | Uploaded 36GiB to GPU in 1.97019s (18.2724GiB/s) | 20260506 23:27:54.058 | INFO | Downloaded 34GiB to GPU in 1.87751s (18.1091GiB/s) | 20260506 23:27:59.203 | INFO | Analyzing chunk frames = [5298176,5298688) | 20260506 23:27:59.203 | INFO | MemInfo: { | 20260506 23:27:59.203 | INFO | "gpu" : " 17 GB", | 20260506 23:27:59.203 | INFO | "huge" : "", | 20260506 23:27:59.203 | INFO | "huge_shared" : " 27 GB", | 20260506 23:27:59.203 | INFO | "malloc" : " 14 GB", | 20260506 23:27:59.204 | INFO | "pinned" : " 29 GB" | 20260506 23:27:59.204 | INFO | } | 20260506 23:27:59.204 | INFO | Perf report for the last 5.14637 seconds: | 20260506 23:27:59.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:27:59.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:59.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:59.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:59.204 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260506 23:27:59.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:59.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:59.204 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:27:59.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:59.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:27:59.204 | INFO | Uploaded 36GiB to GPU in 1.99669s (18.0299GiB/s) | 20260506 23:27:59.204 | INFO | Downloaded 34GiB to GPU in 1.92456s (17.6664GiB/s) | 20260506 23:28:04.343 | INFO | Analyzing chunk frames = [5298688,5299200) | 20260506 23:28:04.343 | INFO | MemInfo: { | 20260506 23:28:04.343 | INFO | "gpu" : " 17 GB", | 20260506 23:28:04.343 | INFO | "huge" : "", | 20260506 23:28:04.343 | INFO | "huge_shared" : " 27 GB", | 20260506 23:28:04.343 | INFO | "malloc" : " 14 GB", | 20260506 23:28:04.343 | INFO | "pinned" : " 29 GB" | 20260506 23:28:04.343 | INFO | } | 20260506 23:28:04.344 | INFO | Perf report for the last 5.14 seconds: | 20260506 23:28:04.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:28:04.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:04.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:04.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:04.344 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 23:28:04.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:04.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:04.344 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 23:28:04.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:04.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:04.344 | INFO | Uploaded 36GiB to GPU in 2.05115s (17.5511GiB/s) | 20260506 23:28:04.344 | INFO | Downloaded 34GiB to GPU in 1.94699s (17.4628GiB/s) | 20260506 23:28:09.478 | INFO | Analyzing chunk frames = [5299200,5299712) | 20260506 23:28:09.478 | INFO | MemInfo: { | 20260506 23:28:09.478 | INFO | "gpu" : " 17 GB", | 20260506 23:28:09.478 | INFO | "huge" : "", | 20260506 23:28:09.478 | INFO | "huge_shared" : " 27 GB", | 20260506 23:28:09.478 | INFO | "malloc" : " 14 GB", | 20260506 23:28:09.478 | INFO | "pinned" : " 29 GB" | 20260506 23:28:09.478 | INFO | } | 20260506 23:28:09.478 | INFO | Perf report for the last 5.13459 seconds: | 20260506 23:28:09.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:28:09.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:09.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:09.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:09.478 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 23:28:09.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:09.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:09.478 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:28:09.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:09.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:09.478 | INFO | Uploaded 36GiB to GPU in 2.03704s (17.6727GiB/s) | 20260506 23:28:09.479 | INFO | Downloaded 34GiB to GPU in 1.96366s (17.3146GiB/s) | 20260506 23:28:14.613 | INFO | Analyzing chunk frames = [5299712,5300224) | 20260506 23:28:14.613 | INFO | MemInfo: { | 20260506 23:28:14.613 | INFO | "gpu" : " 17 GB", | 20260506 23:28:14.613 | INFO | "huge" : "", | 20260506 23:28:14.613 | INFO | "huge_shared" : " 27 GB", | 20260506 23:28:14.613 | INFO | "malloc" : " 14 GB", | 20260506 23:28:14.614 | INFO | "pinned" : " 29 GB" | 20260506 23:28:14.614 | INFO | } | 20260506 23:28:14.614 | INFO | Perf report for the last 5.13526 seconds: | 20260506 23:28:14.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:28:14.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:14.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:14.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:14.614 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:28:14.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:14.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:14.614 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 23:28:14.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:14.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:14.614 | INFO | Uploaded 36GiB to GPU in 1.99021s (18.0885GiB/s) | 20260506 23:28:14.614 | INFO | Downloaded 34GiB to GPU in 1.98756s (17.1064GiB/s) | 20260506 23:28:14.614 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:28:14.614 | INFO | Total mem usage: 18 GiB | 20260506 23:28:14.614 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:28:14.614 | INFO | Recent High Water Marks Sum: 16.2 GiB (Savings: -11.1%) | 20260506 23:28:14.614 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:28:14.614 | INFO | High Water Breakdown: | 20260506 23:28:14.614 | INFO | 62.1% : 10.1 GiB ( 88.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:28:14.614 | INFO | 37.8% : 6.12 GiB ( 91.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:28:14.614 | INFO | 0.123% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:28:14.614 | INFO | ------------------------------------------ | 20260506 23:28:14.614 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:28:14.614 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:28:14.614 | INFO | Total mem usage: 26.5 GiB | 20260506 23:28:14.614 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:28:14.614 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:28:14.614 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:28:14.614 | INFO | High Water Breakdown: | 20260506 23:28:14.614 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:28:14.614 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:28:14.614 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:28:14.614 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:28:14.614 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:28:14.614 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:28:14.614 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:28:14.614 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:28:14.614 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:28:14.614 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:28:14.614 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:28:14.614 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:28:14.615 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:28:14.615 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:28:14.615 | INFO | ------------------------------------------ | 20260506 23:28:14.615 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:28:14.615 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:28:14.615 | INFO | Total mem usage: 25.6 GiB | 20260506 23:28:14.615 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:28:14.615 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:28:14.615 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:28:14.615 | INFO | High Water Breakdown: | 20260506 23:28:14.615 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:28:14.615 | INFO | ------------------------------------------ | 20260506 23:28:14.615 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:28:14.615 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:28:14.615 | INFO | Total mem usage: 15.7 GiB | 20260506 23:28:14.615 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:28:14.615 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:28:14.615 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:28:14.615 | INFO | High Water Breakdown: | 20260506 23:28:14.615 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:28:14.615 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:28:14.615 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:28:14.615 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:28:14.615 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:28:14.615 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:28:14.615 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:28:14.615 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:28:14.615 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:28:14.615 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:28:14.615 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:28:14.615 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:28:14.615 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:28:14.615 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:28:14.615 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:28:14.615 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:28:14.615 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:28:14.615 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:28:14.615 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:28:14.615 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:28:14.615 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:28:14.615 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:28:14.615 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:28:14.615 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:28:14.615 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:28:14.615 | INFO | ------------------------------------------ | 20260506 23:28:14.615 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:28:14.615 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:28:14.615 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:28:14.615 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:28:14.615 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.56e+01, 1.31e+01 / 2.11e+00 | 20260506 23:28:14.615 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.96e-03 / 5.07e-03 | 20260506 23:28:14.615 | INFO | PerfReport: 11.68%, Baseline -- 3.62e+00 / 4.45e+00, 5.50e-01 / 3.11e-01 | 20260506 23:28:14.615 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.47e+00, 1.76e-01 / 1.12e-01 | 20260506 23:28:14.615 | INFO | PerfReport: 16.05%, DME -- 3.98e+01 / 4.46e+01, 3.78e+00 / 1.80e+00 | 20260506 23:28:14.615 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.38e+00 / 1.85e+00 | 20260506 23:28:14.615 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.18e+00 / 2.75e+00, 3.76e-01 / 2.18e-01 | 20260506 23:28:14.615 | INFO | PerfReport: 3.18%, Metrics -- 9.86e-01 / 1.33e+00, 2.61e-01 / 1.85e-01 | 20260506 23:28:14.615 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.36e-01 / 3.89e-01, 5.25e-02 / 2.49e-02 | 20260506 23:28:14.615 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:28:14.615 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:28:14.615 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:28:14.615 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 23:28:14.615 | INFO | PerfReport: 21.60%, Upload -- 2.08e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260506 23:28:14.615 | INFO | PerfReport: 21.30%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260506 23:28:14.615 | INFO | PerfReport: 32.16%, Compute -- 3.10e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260506 23:28:14.615 | INFO | PerfReport: 1.30%, LockUpload -- 1.25e+00 / 0.00e+00, 4.77e+00 / 0.00e+00 | 20260506 23:28:14.615 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260506 23:28:14.615 | INFO | PerfReport: 22.40%, LockCompute -- 2.16e+01 / 0.00e+00, 1.77e+01 / 0.00e+00 | 20260506 23:28:14.615 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 3.77e-02 / 0.00e+00 | 20260506 23:28:14.712 | INFO | Starting a round of Baz IO | 20260506 23:28:17.838 | INFO | Populating RTMetrics report | 20260506 23:28:17.838 | INFO | RTMetrics report populated | 20260506 23:28:17.838 | INFO | Opening tmp RTMetrics json file | 20260506 23:28:18.129 | INFO | Serializing RTMetrics to json | 20260506 23:28:18.133 | INFO | Serialization done, swapping tmp file now... | 20260506 23:28:18.136 | INFO | RTMetrics done | 20260506 23:28:18.778 | INFO | Finished a round of Baz IO: Wrote 1.883 GiB in 4.066 seconds | 20260506 23:28:18.778 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.46% / 0.00% / 66.54% | 20260506 23:28:18.778 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.413219% / 3.97364e-06% / 3.97364e-05% | 20260506 23:28:18.778 | INFO | Baz IO overall: | 20260506 23:28:18.778 | INFO | BytesWritten : 2774152477960 | 20260506 23:28:18.779 | INFO | NumEvents : 1337473644841 | 20260506 23:28:19.753 | INFO | Analyzing chunk frames = [5300224,5300736) | 20260506 23:28:19.753 | INFO | MemInfo: { | 20260506 23:28:19.753 | INFO | "gpu" : " 17 GB", | 20260506 23:28:19.753 | INFO | "huge" : "", | 20260506 23:28:19.753 | INFO | "huge_shared" : " 27 GB", | 20260506 23:28:19.753 | INFO | "malloc" : " 14 GB", | 20260506 23:28:19.753 | INFO | "pinned" : " 29 GB" | 20260506 23:28:19.753 | INFO | } | 20260506 23:28:19.753 | INFO | Perf report for the last 5.13925 seconds: | 20260506 23:28:19.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:28:19.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:19.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:19.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:19.753 | INFO | ANALYSIS : 62.2%, 2.9, 95.1ms, 0 | 20260506 23:28:19.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:19.753 | INFO | RT_METRICS : 48.0%, 1.0, 25.2ms, 0 | 20260506 23:28:19.753 | INFO | PRE_HQ : 59.6%, 1.8, 56.2ms, 0 | 20260506 23:28:19.753 | INFO | BAZWRITER : 78.8%, 4.7, 190.9ms, 0 | 20260506 23:28:19.753 | INFO | BAZ_SAMPLING : 3.9%, 1.0, 2.0ms, 0 | 20260506 23:28:19.753 | INFO | Uploaded 36GiB to GPU in 2.0209s (17.8139GiB/s) | 20260506 23:28:19.753 | INFO | Downloaded 38GiB to GPU in 2.20122s (17.2631GiB/s) | 20260506 23:28:24.892 | INFO | Analyzing chunk frames = [5300736,5301248) | 20260506 23:28:24.892 | INFO | MemInfo: { | 20260506 23:28:24.892 | INFO | "gpu" : " 17 GB", | 20260506 23:28:24.892 | INFO | "huge" : "", | 20260506 23:28:24.892 | INFO | "huge_shared" : " 27 GB", | 20260506 23:28:24.892 | INFO | "malloc" : " 14 GB", | 20260506 23:28:24.892 | INFO | "pinned" : " 29 GB" | 20260506 23:28:24.892 | INFO | } | 20260506 23:28:24.892 | INFO | Perf report for the last 5.1389 seconds: | 20260506 23:28:24.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:28:24.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:24.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:24.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:24.892 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:28:24.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:24.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:24.892 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:28:24.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:24.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:24.892 | INFO | Uploaded 36GiB to GPU in 1.97718s (18.2077GiB/s) | 20260506 23:28:24.892 | INFO | Downloaded 34GiB to GPU in 1.93246s (17.5941GiB/s) | 20260506 23:28:30.031 | INFO | Analyzing chunk frames = [5301248,5301760) | 20260506 23:28:30.031 | INFO | MemInfo: { | 20260506 23:28:30.031 | INFO | "gpu" : " 17 GB", | 20260506 23:28:30.031 | INFO | "huge" : "", | 20260506 23:28:30.031 | INFO | "huge_shared" : " 27 GB", | 20260506 23:28:30.032 | INFO | "malloc" : " 14 GB", | 20260506 23:28:30.032 | INFO | "pinned" : " 29 GB" | 20260506 23:28:30.032 | INFO | } | 20260506 23:28:30.032 | INFO | Perf report for the last 5.13998 seconds: | 20260506 23:28:30.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:28:30.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:30.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:30.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:30.032 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 23:28:30.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:30.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:30.032 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:28:30.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:30.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:30.032 | INFO | Uploaded 36GiB to GPU in 1.96876s (18.2856GiB/s) | 20260506 23:28:30.032 | INFO | Downloaded 34GiB to GPU in 1.92589s (17.6542GiB/s) | 20260506 23:28:35.172 | INFO | Analyzing chunk frames = [5301760,5302272) | 20260506 23:28:35.172 | INFO | MemInfo: { | 20260506 23:28:35.172 | INFO | "gpu" : " 17 GB", | 20260506 23:28:35.172 | INFO | "huge" : "", | 20260506 23:28:35.172 | INFO | "huge_shared" : " 27 GB", | 20260506 23:28:35.172 | INFO | "malloc" : " 14 GB", | 20260506 23:28:35.172 | INFO | "pinned" : " 29 GB" | 20260506 23:28:35.172 | INFO | } | 20260506 23:28:35.172 | INFO | Perf report for the last 5.14061 seconds: | 20260506 23:28:35.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:28:35.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:35.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:35.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:35.172 | INFO | ANALYSIS : 59.6%, 3.0, 92.3ms, 0 | 20260506 23:28:35.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:35.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:35.172 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:28:35.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:35.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:35.173 | INFO | Uploaded 36GiB to GPU in 1.94879s (18.473GiB/s) | 20260506 23:28:35.173 | INFO | Downloaded 34GiB to GPU in 1.8795s (18.0899GiB/s) | 20260506 23:28:40.318 | INFO | Analyzing chunk frames = [5302272,5302784) | 20260506 23:28:40.318 | INFO | MemInfo: { | 20260506 23:28:40.318 | INFO | "gpu" : " 17 GB", | 20260506 23:28:40.318 | INFO | "huge" : "", | 20260506 23:28:40.318 | INFO | "huge_shared" : " 27 GB", | 20260506 23:28:40.319 | INFO | "malloc" : " 14 GB", | 20260506 23:28:40.319 | INFO | "pinned" : " 29 GB" | 20260506 23:28:40.319 | INFO | } | 20260506 23:28:40.319 | INFO | Perf report for the last 5.14627 seconds: | 20260506 23:28:40.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:28:40.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:40.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:40.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:40.319 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 23:28:40.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:40.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:40.319 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:28:40.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:40.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:40.319 | INFO | Uploaded 36GiB to GPU in 1.98449s (18.1407GiB/s) | 20260506 23:28:40.319 | INFO | Downloaded 34GiB to GPU in 1.93238s (17.5949GiB/s) | 20260506 23:28:45.457 | INFO | Analyzing chunk frames = [5302784,5303296) | 20260506 23:28:45.457 | INFO | MemInfo: { | 20260506 23:28:45.457 | INFO | "gpu" : " 17 GB", | 20260506 23:28:45.457 | INFO | "huge" : "", | 20260506 23:28:45.457 | INFO | "huge_shared" : " 27 GB", | 20260506 23:28:45.457 | INFO | "malloc" : " 14 GB", | 20260506 23:28:45.457 | INFO | "pinned" : " 29 GB" | 20260506 23:28:45.457 | INFO | } | 20260506 23:28:45.457 | INFO | Perf report for the last 5.13811 seconds: | 20260506 23:28:45.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:28:45.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:45.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:45.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:45.457 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 23:28:45.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:45.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:45.457 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 23:28:45.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:45.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:45.457 | INFO | Uploaded 36GiB to GPU in 2.02558s (17.7727GiB/s) | 20260506 23:28:45.457 | INFO | Downloaded 34GiB to GPU in 1.95715s (17.3722GiB/s) | 20260506 23:28:50.598 | INFO | Analyzing chunk frames = [5303296,5303808) | 20260506 23:28:50.598 | INFO | MemInfo: { | 20260506 23:28:50.598 | INFO | "gpu" : " 17 GB", | 20260506 23:28:50.598 | INFO | "huge" : "", | 20260506 23:28:50.598 | INFO | "huge_shared" : " 27 GB", | 20260506 23:28:50.598 | INFO | "malloc" : " 14 GB", | 20260506 23:28:50.598 | INFO | "pinned" : " 29 GB" | 20260506 23:28:50.598 | INFO | } | 20260506 23:28:50.598 | INFO | Perf report for the last 5.14169 seconds: | 20260506 23:28:50.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:28:50.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:50.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:50.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:50.599 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:28:50.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:50.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:50.599 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:28:50.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:50.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:50.599 | INFO | Uploaded 36GiB to GPU in 1.97382s (18.2387GiB/s) | 20260506 23:28:50.599 | INFO | Downloaded 34GiB to GPU in 1.99803s (17.0168GiB/s) | 20260506 23:28:55.725 | INFO | Analyzing chunk frames = [5303808,5304320) | 20260506 23:28:55.725 | INFO | MemInfo: { | 20260506 23:28:55.725 | INFO | "gpu" : " 17 GB", | 20260506 23:28:55.725 | INFO | "huge" : "", | 20260506 23:28:55.725 | INFO | "huge_shared" : " 27 GB", | 20260506 23:28:55.725 | INFO | "malloc" : " 14 GB", | 20260506 23:28:55.725 | INFO | "pinned" : " 29 GB" | 20260506 23:28:55.725 | INFO | } | 20260506 23:28:55.725 | INFO | Perf report for the last 5.12672 seconds: | 20260506 23:28:55.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:28:55.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:55.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:55.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:55.725 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260506 23:28:55.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:55.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:55.726 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 23:28:55.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:55.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:28:55.726 | INFO | Uploaded 36GiB to GPU in 2.01076s (17.9037GiB/s) | 20260506 23:28:55.726 | INFO | Downloaded 34GiB to GPU in 1.99935s (17.0055GiB/s) | 20260506 23:28:55.823 | INFO | Starting a round of Baz IO | 20260506 23:28:58.962 | INFO | Populating RTMetrics report | 20260506 23:28:58.963 | INFO | RTMetrics report populated | 20260506 23:28:58.963 | INFO | Opening tmp RTMetrics json file | 20260506 23:28:59.069 | INFO | Serializing RTMetrics to json | 20260506 23:28:59.108 | INFO | Serialization done, swapping tmp file now... | 20260506 23:28:59.111 | INFO | RTMetrics done | 20260506 23:28:59.316 | INFO | Finished a round of Baz IO: Wrote 1.882 GiB in 3.492 seconds | 20260506 23:28:59.316 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.46% / 0.00% / 66.54% | 20260506 23:28:59.316 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.412774% / 0% / 7.54992e-05% | 20260506 23:28:59.316 | INFO | Baz IO overall: | 20260506 23:28:59.316 | INFO | BytesWritten : 2776172889072 | 20260506 23:28:59.316 | INFO | NumEvents : 1338435714166 | 20260506 23:29:00.864 | INFO | Analyzing chunk frames = [5304320,5304832) | 20260506 23:29:00.864 | INFO | MemInfo: { | 20260506 23:29:00.864 | INFO | "gpu" : " 17 GB", | 20260506 23:29:00.864 | INFO | "huge" : "", | 20260506 23:29:00.864 | INFO | "huge_shared" : " 27 GB", | 20260506 23:29:00.864 | INFO | "malloc" : " 14 GB", | 20260506 23:29:00.864 | INFO | "pinned" : " 29 GB" | 20260506 23:29:00.864 | INFO | } | 20260506 23:29:00.864 | INFO | Perf report for the last 5.1391 seconds: | 20260506 23:29:00.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:29:00.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:00.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:00.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:00.864 | INFO | ANALYSIS : 62.4%, 2.9, 93.8ms, 0 | 20260506 23:29:00.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:00.864 | INFO | RT_METRICS : 48.0%, 1.0, 25.2ms, 0 | 20260506 23:29:00.864 | INFO | PRE_HQ : 60.5%, 1.9, 60.4ms, 0 | 20260506 23:29:00.864 | INFO | BAZWRITER : 67.6%, 4.8, 166.1ms, 0 | 20260506 23:29:00.864 | INFO | BAZ_SAMPLING : 2.9%, 1.0, 1.5ms, 0 | 20260506 23:29:00.864 | INFO | Uploaded 36GiB to GPU in 2.02421s (17.7847GiB/s) | 20260506 23:29:00.864 | INFO | Downloaded 38GiB to GPU in 2.18271s (17.4095GiB/s) | 20260506 23:29:06.009 | INFO | Analyzing chunk frames = [5304832,5305344) | 20260506 23:29:06.009 | INFO | MemInfo: { | 20260506 23:29:06.009 | INFO | "gpu" : " 17 GB", | 20260506 23:29:06.009 | INFO | "huge" : "", | 20260506 23:29:06.009 | INFO | "huge_shared" : " 27 GB", | 20260506 23:29:06.009 | INFO | "malloc" : " 14 GB", | 20260506 23:29:06.009 | INFO | "pinned" : " 29 GB" | 20260506 23:29:06.009 | INFO | } | 20260506 23:29:06.009 | INFO | Perf report for the last 5.14506 seconds: | 20260506 23:29:06.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:29:06.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:06.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:06.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:06.010 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:29:06.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:06.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:06.010 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260506 23:29:06.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:06.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:06.010 | INFO | Uploaded 36GiB to GPU in 2.00418s (17.9625GiB/s) | 20260506 23:29:06.010 | INFO | Downloaded 34GiB to GPU in 1.93418s (17.5785GiB/s) | 20260506 23:29:11.150 | INFO | Analyzing chunk frames = [5305344,5305856) | 20260506 23:29:11.150 | INFO | MemInfo: { | 20260506 23:29:11.150 | INFO | "gpu" : " 17 GB", | 20260506 23:29:11.150 | INFO | "huge" : "", | 20260506 23:29:11.150 | INFO | "huge_shared" : " 27 GB", | 20260506 23:29:11.150 | INFO | "malloc" : " 14 GB", | 20260506 23:29:11.150 | INFO | "pinned" : " 29 GB" | 20260506 23:29:11.150 | INFO | } | 20260506 23:29:11.150 | INFO | Perf report for the last 5.1409 seconds: | 20260506 23:29:11.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:29:11.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:11.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:11.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:11.150 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260506 23:29:11.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:11.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:11.150 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:29:11.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:11.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:11.151 | INFO | Uploaded 36GiB to GPU in 1.96358s (18.3339GiB/s) | 20260506 23:29:11.151 | INFO | Downloaded 34GiB to GPU in 1.93003s (17.6163GiB/s) | 20260506 23:29:16.289 | INFO | Analyzing chunk frames = [5305856,5306368) | 20260506 23:29:16.289 | INFO | MemInfo: { | 20260506 23:29:16.289 | INFO | "gpu" : " 17 GB", | 20260506 23:29:16.289 | INFO | "huge" : "", | 20260506 23:29:16.289 | INFO | "huge_shared" : " 27 GB", | 20260506 23:29:16.289 | INFO | "malloc" : " 14 GB", | 20260506 23:29:16.289 | INFO | "pinned" : " 29 GB" | 20260506 23:29:16.289 | INFO | } | 20260506 23:29:16.289 | INFO | Perf report for the last 5.1388 seconds: | 20260506 23:29:16.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:29:16.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:16.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:16.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:16.289 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 23:29:16.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:16.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:16.289 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:29:16.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:16.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:16.289 | INFO | Uploaded 36GiB to GPU in 1.97893s (18.1916GiB/s) | 20260506 23:29:16.289 | INFO | Downloaded 34GiB to GPU in 1.91332s (17.7702GiB/s) | 20260506 23:29:21.431 | INFO | Analyzing chunk frames = [5306368,5306880) | 20260506 23:29:21.431 | INFO | MemInfo: { | 20260506 23:29:21.431 | INFO | "gpu" : " 17 GB", | 20260506 23:29:21.431 | INFO | "huge" : "", | 20260506 23:29:21.431 | INFO | "huge_shared" : " 27 GB", | 20260506 23:29:21.431 | INFO | "malloc" : " 14 GB", | 20260506 23:29:21.431 | INFO | "pinned" : " 29 GB" | 20260506 23:29:21.431 | INFO | } | 20260506 23:29:21.431 | INFO | Perf report for the last 5.14178 seconds: | 20260506 23:29:21.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:29:21.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:21.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:21.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:21.431 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 23:29:21.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:21.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:21.431 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:29:21.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:21.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:21.431 | INFO | Uploaded 36GiB to GPU in 1.97933s (18.188GiB/s) | 20260506 23:29:21.431 | INFO | Downloaded 34GiB to GPU in 1.9277s (17.6376GiB/s) | 20260506 23:29:26.572 | INFO | Analyzing chunk frames = [5306880,5307392) | 20260506 23:29:26.572 | INFO | MemInfo: { | 20260506 23:29:26.572 | INFO | "gpu" : " 17 GB", | 20260506 23:29:26.572 | INFO | "huge" : "", | 20260506 23:29:26.572 | INFO | "huge_shared" : " 27 GB", | 20260506 23:29:26.572 | INFO | "malloc" : " 14 GB", | 20260506 23:29:26.572 | INFO | "pinned" : " 29 GB" | 20260506 23:29:26.573 | INFO | } | 20260506 23:29:26.573 | INFO | Perf report for the last 5.14177 seconds: | 20260506 23:29:26.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:29:26.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:26.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:26.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:26.573 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 23:29:26.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:26.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:26.573 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:29:26.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:26.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:26.573 | INFO | Uploaded 36GiB to GPU in 2.01786s (17.8407GiB/s) | 20260506 23:29:26.573 | INFO | Downloaded 34GiB to GPU in 1.93615s (17.5606GiB/s) | 20260506 23:29:31.705 | INFO | Analyzing chunk frames = [5307392,5307904) | 20260506 23:29:31.705 | INFO | MemInfo: { | 20260506 23:29:31.705 | INFO | "gpu" : " 17 GB", | 20260506 23:29:31.705 | INFO | "huge" : "", | 20260506 23:29:31.705 | INFO | "huge_shared" : " 27 GB", | 20260506 23:29:31.705 | INFO | "malloc" : " 14 GB", | 20260506 23:29:31.705 | INFO | "pinned" : " 29 GB" | 20260506 23:29:31.705 | INFO | } | 20260506 23:29:31.705 | INFO | Perf report for the last 5.13283 seconds: | 20260506 23:29:31.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:29:31.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:31.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:31.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:31.706 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 23:29:31.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:31.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:31.706 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 23:29:31.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:31.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:31.706 | INFO | Uploaded 36GiB to GPU in 1.96069s (18.3608GiB/s) | 20260506 23:29:31.706 | INFO | Downloaded 34GiB to GPU in 1.99713s (17.0244GiB/s) | 20260506 23:29:36.845 | INFO | Analyzing chunk frames = [5307904,5308416) | 20260506 23:29:36.845 | INFO | MemInfo: { | 20260506 23:29:36.845 | INFO | "gpu" : " 17 GB", | 20260506 23:29:36.845 | INFO | "huge" : "", | 20260506 23:29:36.845 | INFO | "huge_shared" : " 27 GB", | 20260506 23:29:36.845 | INFO | "malloc" : " 14 GB", | 20260506 23:29:36.845 | INFO | "pinned" : " 29 GB" | 20260506 23:29:36.845 | INFO | } | 20260506 23:29:36.845 | INFO | Perf report for the last 5.13957 seconds: | 20260506 23:29:36.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:29:36.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:36.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:36.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:36.845 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 23:29:36.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:36.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:36.845 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 23:29:36.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:36.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:36.845 | INFO | Uploaded 36GiB to GPU in 2.02662s (17.7636GiB/s) | 20260506 23:29:36.846 | INFO | Downloaded 34GiB to GPU in 1.99253s (17.0637GiB/s) | 20260506 23:29:36.846 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:29:36.846 | INFO | Total mem usage: 18 GiB | 20260506 23:29:36.846 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:29:36.846 | INFO | Recent High Water Marks Sum: 16.1 GiB (Savings: -11.9%) | 20260506 23:29:36.846 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:29:36.846 | INFO | High Water Breakdown: | 20260506 23:29:36.846 | INFO | 62% : 9.97 GiB ( 87.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:29:36.846 | INFO | 37.9% : 6.1 GiB ( 91.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:29:36.846 | INFO | 0.124% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:29:36.846 | INFO | ------------------------------------------ | 20260506 23:29:36.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:29:36.846 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:29:36.846 | INFO | Total mem usage: 26.5 GiB | 20260506 23:29:36.846 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:29:36.846 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:29:36.846 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:29:36.846 | INFO | High Water Breakdown: | 20260506 23:29:36.846 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:29:36.846 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:29:36.846 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:29:36.846 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:29:36.846 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:29:36.846 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:29:36.846 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:29:36.846 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:29:36.846 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:29:36.846 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:29:36.846 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:29:36.846 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:29:36.846 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:29:36.846 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:29:36.846 | INFO | ------------------------------------------ | 20260506 23:29:36.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:29:36.846 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:29:36.846 | INFO | Total mem usage: 25.6 GiB | 20260506 23:29:36.846 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:29:36.846 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:29:36.846 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:29:36.846 | INFO | High Water Breakdown: | 20260506 23:29:36.846 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:29:36.846 | INFO | ------------------------------------------ | 20260506 23:29:36.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:29:36.846 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:29:36.846 | INFO | Total mem usage: 15.7 GiB | 20260506 23:29:36.846 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:29:36.846 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:29:36.846 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:29:36.846 | INFO | High Water Breakdown: | 20260506 23:29:36.846 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:29:36.846 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:29:36.846 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:29:36.847 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:29:36.847 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:29:36.847 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:29:36.847 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:29:36.847 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:29:36.847 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:29:36.847 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:29:36.847 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:29:36.847 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:29:36.847 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:29:36.847 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:29:36.847 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:29:36.847 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:29:36.847 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:29:36.847 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:29:36.847 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:29:36.847 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:29:36.847 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:29:36.847 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:29:36.847 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:29:36.847 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:29:36.847 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:29:36.847 | INFO | ------------------------------------------ | 20260506 23:29:36.847 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:29:36.847 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:29:36.847 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:29:36.847 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:29:36.847 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.60e+01, 1.34e+01 / 1.87e+00 | 20260506 23:29:36.847 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.00e-01, 9.51e-03 / 6.74e-03 | 20260506 23:29:36.847 | INFO | PerfReport: 11.67%, Baseline -- 3.62e+00 / 4.42e+00, 5.49e-01 / 3.10e-01 | 20260506 23:29:36.847 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.80e-01 / 1.30e-01 | 20260506 23:29:36.847 | INFO | PerfReport: 16.09%, DME -- 3.99e+01 / 4.50e+01, 4.24e+00 / 2.17e+00 | 20260506 23:29:36.847 | INFO | PerfReport: 57.33%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.33e+00 / 1.51e+00 | 20260506 23:29:36.847 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.19e+00 / 2.73e+00, 3.82e-01 / 2.11e-01 | 20260506 23:29:36.847 | INFO | PerfReport: 3.19%, Metrics -- 9.89e-01 / 1.32e+00, 2.62e-01 / 1.16e-01 | 20260506 23:29:36.847 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.37e-01 / 3.88e-01, 5.22e-02 / 1.64e-02 | 20260506 23:29:36.847 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:29:36.847 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:29:36.847 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:29:36.847 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 23:29:36.847 | INFO | PerfReport: 21.53%, Upload -- 2.07e+01 / 0.00e+00, 6.80e+00 / 0.00e+00 | 20260506 23:29:36.847 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 23:29:36.847 | INFO | PerfReport: 32.17%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 23:29:36.847 | INFO | PerfReport: 1.23%, LockUpload -- 1.19e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260506 23:29:36.847 | INFO | PerfReport: 1.39%, LockDownload -- 1.34e+00 / 0.00e+00, 3.29e+00 / 0.00e+00 | 20260506 23:29:36.847 | INFO | PerfReport: 22.29%, LockCompute -- 2.15e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:29:36.847 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.29e-02 / 0.00e+00 | 20260506 23:29:36.959 | INFO | Starting a round of Baz IO | 20260506 23:29:40.068 | INFO | Populating RTMetrics report | 20260506 23:29:40.068 | INFO | RTMetrics report populated | 20260506 23:29:40.068 | INFO | Opening tmp RTMetrics json file | 20260506 23:29:40.098 | INFO | Serializing RTMetrics to json | 20260506 23:29:40.099 | INFO | Serialization done, swapping tmp file now... | 20260506 23:29:40.099 | INFO | RTMetrics done | 20260506 23:29:40.437 | INFO | Finished a round of Baz IO: Wrote 1.879 GiB in 3.478 seconds | 20260506 23:29:40.437 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.46% / 0.00% / 66.54% | 20260506 23:29:40.437 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.412202% / 3.97364e-06% / 4.76837e-05% | 20260506 23:29:40.437 | INFO | Baz IO overall: | 20260506 23:29:40.437 | INFO | BytesWritten : 2778190513565 | 20260506 23:29:40.437 | INFO | NumEvents : 1339396297779 | 20260506 23:29:41.984 | INFO | Analyzing chunk frames = [5308416,5308928) | 20260506 23:29:41.984 | INFO | MemInfo: { | 20260506 23:29:41.984 | INFO | "gpu" : " 17 GB", | 20260506 23:29:41.984 | INFO | "huge" : "", | 20260506 23:29:41.984 | INFO | "huge_shared" : " 27 GB", | 20260506 23:29:41.984 | INFO | "malloc" : " 14 GB", | 20260506 23:29:41.984 | INFO | "pinned" : " 29 GB" | 20260506 23:29:41.984 | INFO | } | 20260506 23:29:41.984 | INFO | Perf report for the last 5.13902 seconds: | 20260506 23:29:41.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:29:41.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:41.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:41.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:41.984 | INFO | ANALYSIS : 62.1%, 2.9, 94.9ms, 0 | 20260506 23:29:41.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:41.984 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260506 23:29:41.984 | INFO | PRE_HQ : 60.1%, 1.8, 57.6ms, 0 | 20260506 23:29:41.984 | INFO | BAZWRITER : 67.3%, 4.8, 165.5ms, 0 | 20260506 23:29:41.984 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 23:29:41.984 | INFO | Uploaded 36GiB to GPU in 2.06017s (17.4743GiB/s) | 20260506 23:29:41.985 | INFO | Downloaded 38GiB to GPU in 2.19321s (17.3262GiB/s) | 20260506 23:29:47.119 | INFO | Analyzing chunk frames = [5308928,5309440) | 20260506 23:29:47.119 | INFO | MemInfo: { | 20260506 23:29:47.119 | INFO | "gpu" : " 17 GB", | 20260506 23:29:47.119 | INFO | "huge" : "", | 20260506 23:29:47.119 | INFO | "huge_shared" : " 27 GB", | 20260506 23:29:47.119 | INFO | "malloc" : " 14 GB", | 20260506 23:29:47.119 | INFO | "pinned" : " 29 GB" | 20260506 23:29:47.119 | INFO | } | 20260506 23:29:47.119 | INFO | Perf report for the last 5.13523 seconds: | 20260506 23:29:47.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:29:47.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:47.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:47.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:47.119 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:29:47.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:47.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:47.119 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:29:47.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:47.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:47.120 | INFO | Uploaded 36GiB to GPU in 2.00645s (17.9421GiB/s) | 20260506 23:29:47.120 | INFO | Downloaded 34GiB to GPU in 1.92572s (17.6557GiB/s) | 20260506 23:29:52.270 | INFO | Analyzing chunk frames = [5309440,5309952) | 20260506 23:29:52.270 | INFO | MemInfo: { | 20260506 23:29:52.270 | INFO | "gpu" : " 17 GB", | 20260506 23:29:52.270 | INFO | "huge" : "", | 20260506 23:29:52.270 | INFO | "huge_shared" : " 27 GB", | 20260506 23:29:52.270 | INFO | "malloc" : " 14 GB", | 20260506 23:29:52.270 | INFO | "pinned" : " 29 GB" | 20260506 23:29:52.270 | INFO | } | 20260506 23:29:52.270 | INFO | Perf report for the last 5.15089 seconds: | 20260506 23:29:52.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:29:52.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:52.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:52.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:52.270 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260506 23:29:52.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:52.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:52.270 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:29:52.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:52.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:52.271 | INFO | Uploaded 36GiB to GPU in 1.98312s (18.1532GiB/s) | 20260506 23:29:52.271 | INFO | Downloaded 34GiB to GPU in 1.91839s (17.7232GiB/s) | 20260506 23:29:57.407 | INFO | Analyzing chunk frames = [5309952,5310464) | 20260506 23:29:57.407 | INFO | MemInfo: { | 20260506 23:29:57.407 | INFO | "gpu" : " 17 GB", | 20260506 23:29:57.407 | INFO | "huge" : "", | 20260506 23:29:57.407 | INFO | "huge_shared" : " 27 GB", | 20260506 23:29:57.407 | INFO | "malloc" : " 14 GB", | 20260506 23:29:57.407 | INFO | "pinned" : " 29 GB" | 20260506 23:29:57.407 | INFO | } | 20260506 23:29:57.407 | INFO | Perf report for the last 5.13694 seconds: | 20260506 23:29:57.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:29:57.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:57.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:57.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:57.407 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 23:29:57.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:57.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:57.407 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:29:57.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:57.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:29:57.407 | INFO | Uploaded 36GiB to GPU in 1.98648s (18.1225GiB/s) | 20260506 23:29:57.407 | INFO | Downloaded 34GiB to GPU in 1.90504s (17.8473GiB/s) | 20260506 23:30:02.546 | INFO | Analyzing chunk frames = [5310464,5310976) | 20260506 23:30:02.546 | INFO | MemInfo: { | 20260506 23:30:02.546 | INFO | "gpu" : " 17 GB", | 20260506 23:30:02.546 | INFO | "huge" : "", | 20260506 23:30:02.546 | INFO | "huge_shared" : " 27 GB", | 20260506 23:30:02.546 | INFO | "malloc" : " 14 GB", | 20260506 23:30:02.546 | INFO | "pinned" : " 29 GB" | 20260506 23:30:02.546 | INFO | } | 20260506 23:30:02.546 | INFO | Perf report for the last 5.13905 seconds: | 20260506 23:30:02.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:30:02.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:02.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:02.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:02.546 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 23:30:02.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:02.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:02.546 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:30:02.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:02.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:02.547 | INFO | Uploaded 36GiB to GPU in 1.98869s (18.1023GiB/s) | 20260506 23:30:02.547 | INFO | Downloaded 34GiB to GPU in 1.92528s (17.6597GiB/s) | 20260506 23:30:07.686 | INFO | Analyzing chunk frames = [5310976,5311488) | 20260506 23:30:07.686 | INFO | MemInfo: { | 20260506 23:30:07.686 | INFO | "gpu" : " 17 GB", | 20260506 23:30:07.686 | INFO | "huge" : "", | 20260506 23:30:07.686 | INFO | "huge_shared" : " 27 GB", | 20260506 23:30:07.686 | INFO | "malloc" : " 14 GB", | 20260506 23:30:07.686 | INFO | "pinned" : " 29 GB" | 20260506 23:30:07.686 | INFO | } | 20260506 23:30:07.687 | INFO | Perf report for the last 5.14019 seconds: | 20260506 23:30:07.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:30:07.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:07.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:07.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:07.687 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 23:30:07.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:07.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:07.687 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:30:07.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:07.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:07.687 | INFO | Uploaded 36GiB to GPU in 2.03196s (17.7169GiB/s) | 20260506 23:30:07.687 | INFO | Downloaded 34GiB to GPU in 1.93711s (17.5519GiB/s) | 20260506 23:30:12.821 | INFO | Analyzing chunk frames = [5311488,5312000) | 20260506 23:30:12.821 | INFO | MemInfo: { | 20260506 23:30:12.821 | INFO | "gpu" : " 17 GB", | 20260506 23:30:12.821 | INFO | "huge" : "", | 20260506 23:30:12.821 | INFO | "huge_shared" : " 27 GB", | 20260506 23:30:12.821 | INFO | "malloc" : " 14 GB", | 20260506 23:30:12.821 | INFO | "pinned" : " 29 GB" | 20260506 23:30:12.821 | INFO | } | 20260506 23:30:12.821 | INFO | Perf report for the last 5.13475 seconds: | 20260506 23:30:12.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:30:12.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:12.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:12.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:12.821 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260506 23:30:12.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:12.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:12.821 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 23:30:12.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:12.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:12.822 | INFO | Uploaded 36GiB to GPU in 2.01895s (17.831GiB/s) | 20260506 23:30:12.822 | INFO | Downloaded 34GiB to GPU in 1.98548s (17.1243GiB/s) | 20260506 23:30:17.960 | INFO | Analyzing chunk frames = [5312000,5312512) | 20260506 23:30:17.960 | INFO | MemInfo: { | 20260506 23:30:17.960 | INFO | "gpu" : " 17 GB", | 20260506 23:30:17.960 | INFO | "huge" : "", | 20260506 23:30:17.960 | INFO | "huge_shared" : " 27 GB", | 20260506 23:30:17.960 | INFO | "malloc" : " 14 GB", | 20260506 23:30:17.960 | INFO | "pinned" : " 29 GB" | 20260506 23:30:17.960 | INFO | } | 20260506 23:30:17.960 | INFO | Perf report for the last 5.13853 seconds: | 20260506 23:30:17.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:30:17.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:17.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:17.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:17.960 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 23:30:17.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:17.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:17.960 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 23:30:17.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:17.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:17.960 | INFO | Uploaded 36GiB to GPU in 2.01328s (17.8812GiB/s) | 20260506 23:30:17.960 | INFO | Downloaded 34GiB to GPU in 1.99151s (17.0725GiB/s) | 20260506 23:30:18.058 | INFO | Starting a round of Baz IO | 20260506 23:30:21.186 | INFO | Populating RTMetrics report | 20260506 23:30:21.186 | INFO | RTMetrics report populated | 20260506 23:30:21.186 | INFO | Opening tmp RTMetrics json file | 20260506 23:30:21.189 | INFO | Serializing RTMetrics to json | 20260506 23:30:21.191 | INFO | Serialization done, swapping tmp file now... | 20260506 23:30:21.192 | INFO | RTMetrics done | 20260506 23:30:21.391 | INFO | Finished a round of Baz IO: Wrote 1.878 GiB in 3.332 seconds | 20260506 23:30:21.391 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.46% / 0.00% / 66.54% | 20260506 23:30:21.391 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.411105% / 0% / 6.35783e-05% | 20260506 23:30:21.391 | INFO | Baz IO overall: | 20260506 23:30:21.391 | INFO | BytesWritten : 2780207076911 | 20260506 23:30:21.391 | INFO | NumEvents : 1340355874876 | 20260506 23:30:23.102 | INFO | Analyzing chunk frames = [5312512,5313024) | 20260506 23:30:23.102 | INFO | MemInfo: { | 20260506 23:30:23.102 | INFO | "gpu" : " 17 GB", | 20260506 23:30:23.102 | INFO | "huge" : "", | 20260506 23:30:23.102 | INFO | "huge_shared" : " 27 GB", | 20260506 23:30:23.102 | INFO | "malloc" : " 14 GB", | 20260506 23:30:23.102 | INFO | "pinned" : " 29 GB" | 20260506 23:30:23.102 | INFO | } | 20260506 23:30:23.102 | INFO | Perf report for the last 5.14252 seconds: | 20260506 23:30:23.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:30:23.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:23.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:23.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:23.102 | INFO | ANALYSIS : 62.2%, 2.9, 95.1ms, 0 | 20260506 23:30:23.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:23.102 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 23:30:23.102 | INFO | PRE_HQ : 59.5%, 1.8, 56.3ms, 0 | 20260506 23:30:23.102 | INFO | BAZWRITER : 64.4%, 4.8, 158.1ms, 0 | 20260506 23:30:23.102 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.9ms, 0 | 20260506 23:30:23.103 | INFO | Uploaded 36GiB to GPU in 2.04826s (17.5759GiB/s) | 20260506 23:30:23.103 | INFO | Downloaded 38GiB to GPU in 2.19069s (17.3461GiB/s) | 20260506 23:30:28.239 | INFO | Analyzing chunk frames = [5313024,5313536) | 20260506 23:30:28.239 | INFO | MemInfo: { | 20260506 23:30:28.239 | INFO | "gpu" : " 17 GB", | 20260506 23:30:28.239 | INFO | "huge" : "", | 20260506 23:30:28.239 | INFO | "huge_shared" : " 27 GB", | 20260506 23:30:28.240 | INFO | "malloc" : " 14 GB", | 20260506 23:30:28.240 | INFO | "pinned" : " 29 GB" | 20260506 23:30:28.240 | INFO | } | 20260506 23:30:28.240 | INFO | Perf report for the last 5.13719 seconds: | 20260506 23:30:28.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:30:28.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:28.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:28.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:28.240 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 23:30:28.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:28.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:28.240 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:30:28.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:28.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:28.240 | INFO | Uploaded 36GiB to GPU in 1.99688s (18.0282GiB/s) | 20260506 23:30:28.240 | INFO | Downloaded 34GiB to GPU in 1.92504s (17.662GiB/s) | 20260506 23:30:33.379 | INFO | Analyzing chunk frames = [5313536,5314048) | 20260506 23:30:33.379 | INFO | MemInfo: { | 20260506 23:30:33.379 | INFO | "gpu" : " 17 GB", | 20260506 23:30:33.379 | INFO | "huge" : "", | 20260506 23:30:33.379 | INFO | "huge_shared" : " 27 GB", | 20260506 23:30:33.379 | INFO | "malloc" : " 14 GB", | 20260506 23:30:33.379 | INFO | "pinned" : " 29 GB" | 20260506 23:30:33.379 | INFO | } | 20260506 23:30:33.379 | INFO | Perf report for the last 5.13974 seconds: | 20260506 23:30:33.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:30:33.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:33.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:33.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:33.379 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 23:30:33.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:33.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:33.379 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:30:33.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:33.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:33.380 | INFO | Uploaded 36GiB to GPU in 1.98223s (18.1614GiB/s) | 20260506 23:30:33.380 | INFO | Downloaded 34GiB to GPU in 1.92404s (17.6712GiB/s) | 20260506 23:30:38.518 | INFO | Analyzing chunk frames = [5314048,5314560) | 20260506 23:30:38.518 | INFO | MemInfo: { | 20260506 23:30:38.518 | INFO | "gpu" : " 17 GB", | 20260506 23:30:38.518 | INFO | "huge" : "", | 20260506 23:30:38.518 | INFO | "huge_shared" : " 27 GB", | 20260506 23:30:38.518 | INFO | "malloc" : " 14 GB", | 20260506 23:30:38.518 | INFO | "pinned" : " 29 GB" | 20260506 23:30:38.518 | INFO | } | 20260506 23:30:38.519 | INFO | Perf report for the last 5.13922 seconds: | 20260506 23:30:38.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:30:38.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:38.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:38.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:38.519 | INFO | ANALYSIS : 59.6%, 3.0, 92.3ms, 0 | 20260506 23:30:38.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:38.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:38.519 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:30:38.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:38.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:38.519 | INFO | Uploaded 36GiB to GPU in 1.93389s (18.6154GiB/s) | 20260506 23:30:38.519 | INFO | Downloaded 34GiB to GPU in 1.87033s (18.1787GiB/s) | 20260506 23:30:43.660 | INFO | Analyzing chunk frames = [5314560,5315072) | 20260506 23:30:43.660 | INFO | MemInfo: { | 20260506 23:30:43.660 | INFO | "gpu" : " 17 GB", | 20260506 23:30:43.661 | INFO | "huge" : "", | 20260506 23:30:43.661 | INFO | "huge_shared" : " 27 GB", | 20260506 23:30:43.661 | INFO | "malloc" : " 14 GB", | 20260506 23:30:43.661 | INFO | "pinned" : " 29 GB" | 20260506 23:30:43.661 | INFO | } | 20260506 23:30:43.661 | INFO | Perf report for the last 5.14201 seconds: | 20260506 23:30:43.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:30:43.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:43.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:43.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:43.661 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 23:30:43.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:43.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:43.661 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:30:43.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:43.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:43.661 | INFO | Uploaded 36GiB to GPU in 2.00158s (17.9858GiB/s) | 20260506 23:30:43.661 | INFO | Downloaded 34GiB to GPU in 1.93217s (17.5968GiB/s) | 20260506 23:30:48.796 | INFO | Analyzing chunk frames = [5315072,5315584) | 20260506 23:30:48.796 | INFO | MemInfo: { | 20260506 23:30:48.796 | INFO | "gpu" : " 17 GB", | 20260506 23:30:48.796 | INFO | "huge" : "", | 20260506 23:30:48.796 | INFO | "huge_shared" : " 27 GB", | 20260506 23:30:48.796 | INFO | "malloc" : " 14 GB", | 20260506 23:30:48.796 | INFO | "pinned" : " 29 GB" | 20260506 23:30:48.796 | INFO | } | 20260506 23:30:48.796 | INFO | Perf report for the last 5.13561 seconds: | 20260506 23:30:48.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:30:48.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:48.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:48.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:48.796 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 23:30:48.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:48.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:48.796 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:30:48.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:48.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:48.797 | INFO | Uploaded 36GiB to GPU in 2.03366s (17.7021GiB/s) | 20260506 23:30:48.797 | INFO | Downloaded 34GiB to GPU in 1.93565s (17.5652GiB/s) | 20260506 23:30:53.932 | INFO | Analyzing chunk frames = [5315584,5316096) | 20260506 23:30:53.932 | INFO | MemInfo: { | 20260506 23:30:53.932 | INFO | "gpu" : " 17 GB", | 20260506 23:30:53.932 | INFO | "huge" : "", | 20260506 23:30:53.933 | INFO | "huge_shared" : " 27 GB", | 20260506 23:30:53.933 | INFO | "malloc" : " 14 GB", | 20260506 23:30:53.933 | INFO | "pinned" : " 29 GB" | 20260506 23:30:53.933 | INFO | } | 20260506 23:30:53.933 | INFO | Perf report for the last 5.13684 seconds: | 20260506 23:30:53.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:30:53.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:53.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:53.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:53.933 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:30:53.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:53.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:53.933 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:30:53.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:53.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:53.933 | INFO | Uploaded 36GiB to GPU in 2.00273s (17.9755GiB/s) | 20260506 23:30:53.933 | INFO | Downloaded 34GiB to GPU in 1.98926s (17.0918GiB/s) | 20260506 23:30:59.074 | INFO | Analyzing chunk frames = [5316096,5316608) | 20260506 23:30:59.074 | INFO | MemInfo: { | 20260506 23:30:59.074 | INFO | "gpu" : " 17 GB", | 20260506 23:30:59.074 | INFO | "huge" : "", | 20260506 23:30:59.074 | INFO | "huge_shared" : " 27 GB", | 20260506 23:30:59.074 | INFO | "malloc" : " 14 GB", | 20260506 23:30:59.074 | INFO | "pinned" : " 29 GB" | 20260506 23:30:59.074 | INFO | } | 20260506 23:30:59.074 | INFO | Perf report for the last 5.14086 seconds: | 20260506 23:30:59.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:30:59.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:59.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:59.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:59.074 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:30:59.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:59.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:59.074 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260506 23:30:59.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:59.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:30:59.074 | INFO | Uploaded 36GiB to GPU in 1.99s (18.0905GiB/s) | 20260506 23:30:59.074 | INFO | Downloaded 34GiB to GPU in 1.99911s (17.0075GiB/s) | 20260506 23:30:59.074 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:30:59.074 | INFO | Total mem usage: 18 GiB | 20260506 23:30:59.074 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:30:59.074 | INFO | Recent High Water Marks Sum: 15.8 GiB (Savings: -14.2%) | 20260506 23:30:59.074 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:30:59.074 | INFO | High Water Breakdown: | 20260506 23:30:59.074 | INFO | 61.7% : 9.73 GiB ( 85.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:30:59.074 | INFO | 38.2% : 6.02 GiB ( 90% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:30:59.074 | INFO | 0.126% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:30:59.074 | INFO | ------------------------------------------ | 20260506 23:30:59.075 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:30:59.075 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:30:59.075 | INFO | Total mem usage: 26.5 GiB | 20260506 23:30:59.075 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:30:59.075 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:30:59.075 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:30:59.075 | INFO | High Water Breakdown: | 20260506 23:30:59.075 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:30:59.075 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:30:59.075 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:30:59.075 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:30:59.075 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:30:59.075 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:30:59.075 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:30:59.075 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:30:59.075 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:30:59.075 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:30:59.075 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:30:59.075 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:30:59.075 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:30:59.075 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:30:59.075 | INFO | ------------------------------------------ | 20260506 23:30:59.075 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:30:59.075 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:30:59.075 | INFO | Total mem usage: 25.6 GiB | 20260506 23:30:59.075 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:30:59.075 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:30:59.075 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:30:59.075 | INFO | High Water Breakdown: | 20260506 23:30:59.075 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:30:59.075 | INFO | ------------------------------------------ | 20260506 23:30:59.075 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:30:59.075 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:30:59.075 | INFO | Total mem usage: 15.7 GiB | 20260506 23:30:59.075 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:30:59.075 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:30:59.075 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 23:30:59.075 | INFO | High Water Breakdown: | 20260506 23:30:59.075 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:30:59.075 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:30:59.075 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:30:59.075 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:30:59.075 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:30:59.075 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:30:59.075 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:30:59.075 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:30:59.075 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:30:59.075 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:30:59.075 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:30:59.075 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:30:59.075 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:30:59.075 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:30:59.075 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:30:59.075 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:30:59.075 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:30:59.075 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:30:59.075 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:30:59.075 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:30:59.075 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:30:59.075 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:30:59.075 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:30:59.075 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:30:59.075 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:30:59.075 | INFO | ------------------------------------------ | 20260506 23:30:59.075 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:30:59.075 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:30:59.075 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:30:59.075 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:30:59.075 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.56e+01, 1.33e+01 / 2.12e+00 | 20260506 23:30:59.075 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.00e-01, 1.02e-02 / 6.15e-03 | 20260506 23:30:59.075 | INFO | PerfReport: 11.67%, Baseline -- 3.62e+00 / 4.40e+00, 5.49e-01 / 3.85e-01 | 20260506 23:30:59.075 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.45e+00, 1.77e-01 / 1.42e-01 | 20260506 23:30:59.075 | INFO | PerfReport: 16.05%, DME -- 3.98e+01 / 4.43e+01, 3.93e+00 / 2.56e+00 | 20260506 23:30:59.075 | INFO | PerfReport: 57.40%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.37e+00 / 1.94e+00 | 20260506 23:30:59.075 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.19e+00 / 2.71e+00, 3.80e-01 / 2.32e-01 | 20260506 23:30:59.075 | INFO | PerfReport: 3.19%, Metrics -- 9.89e-01 / 1.30e+00, 2.63e-01 / 1.69e-01 | 20260506 23:30:59.075 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.37e-01 / 3.81e-01, 5.33e-02 / 2.47e-02 | 20260506 23:30:59.075 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:30:59.075 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:30:59.075 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:30:59.075 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 23:30:59.075 | INFO | PerfReport: 21.65%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 23:30:59.075 | INFO | PerfReport: 21.29%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 23:30:59.075 | INFO | PerfReport: 32.14%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 23:30:59.075 | INFO | PerfReport: 1.38%, LockUpload -- 1.33e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260506 23:30:59.075 | INFO | PerfReport: 1.17%, LockDownload -- 1.13e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260506 23:30:59.075 | INFO | PerfReport: 22.34%, LockCompute -- 2.16e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 23:30:59.075 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.76e-02 / 0.00e+00 | 20260506 23:30:59.172 | INFO | Starting a round of Baz IO | 20260506 23:31:02.307 | INFO | Populating RTMetrics report | 20260506 23:31:02.307 | INFO | RTMetrics report populated | 20260506 23:31:02.307 | INFO | Opening tmp RTMetrics json file | 20260506 23:31:02.388 | INFO | Serializing RTMetrics to json | 20260506 23:31:02.391 | INFO | Serialization done, swapping tmp file now... | 20260506 23:31:02.395 | INFO | RTMetrics done | 20260506 23:31:02.530 | INFO | Finished a round of Baz IO: Wrote 1.878 GiB in 3.358 seconds | 20260506 23:31:02.530 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.45% / 0.00% / 66.54% | 20260506 23:31:02.530 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.40882% / 3.97364e-06% / 7.54992e-05% | 20260506 23:31:02.530 | INFO | Baz IO overall: | 20260506 23:31:02.530 | INFO | BytesWritten : 2782223443365 | 20260506 23:31:02.530 | INFO | NumEvents : 1341315072581 | 20260506 23:31:04.214 | INFO | Analyzing chunk frames = [5316608,5317120) | 20260506 23:31:04.215 | INFO | MemInfo: { | 20260506 23:31:04.215 | INFO | "gpu" : " 17 GB", | 20260506 23:31:04.215 | INFO | "huge" : "", | 20260506 23:31:04.215 | INFO | "huge_shared" : " 27 GB", | 20260506 23:31:04.215 | INFO | "malloc" : " 14 GB", | 20260506 23:31:04.215 | INFO | "pinned" : " 29 GB" | 20260506 23:31:04.215 | INFO | } | 20260506 23:31:04.215 | INFO | Perf report for the last 5.14066 seconds: | 20260506 23:31:04.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:31:04.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:04.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:04.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:04.215 | INFO | ANALYSIS : 62.3%, 2.9, 95.7ms, 0 | 20260506 23:31:04.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:04.215 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260506 23:31:04.215 | INFO | PRE_HQ : 59.7%, 1.8, 56.3ms, 0 | 20260506 23:31:04.215 | INFO | BAZWRITER : 65.1%, 4.7, 156.7ms, 0 | 20260506 23:31:04.215 | INFO | BAZ_SAMPLING : 1.5%, 1.0, 0.8ms, 0 | 20260506 23:31:04.215 | INFO | Uploaded 36GiB to GPU in 2.05099s (17.5525GiB/s) | 20260506 23:31:04.215 | INFO | Downloaded 38GiB to GPU in 2.19712s (17.2953GiB/s) | 20260506 23:31:09.357 | INFO | Analyzing chunk frames = [5317120,5317632) | 20260506 23:31:09.357 | INFO | MemInfo: { | 20260506 23:31:09.357 | INFO | "gpu" : " 17 GB", | 20260506 23:31:09.357 | INFO | "huge" : "", | 20260506 23:31:09.357 | INFO | "huge_shared" : " 27 GB", | 20260506 23:31:09.357 | INFO | "malloc" : " 14 GB", | 20260506 23:31:09.357 | INFO | "pinned" : " 29 GB" | 20260506 23:31:09.357 | INFO | } | 20260506 23:31:09.358 | INFO | Perf report for the last 5.14294 seconds: | 20260506 23:31:09.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:31:09.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:09.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:09.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:09.358 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:31:09.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:09.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:09.358 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260506 23:31:09.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:09.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:09.358 | INFO | Uploaded 36GiB to GPU in 2.01499s (17.8661GiB/s) | 20260506 23:31:09.358 | INFO | Downloaded 34GiB to GPU in 1.93756s (17.5479GiB/s) | 20260506 23:31:14.496 | INFO | Analyzing chunk frames = [5317632,5318144) | 20260506 23:31:14.496 | INFO | MemInfo: { | 20260506 23:31:14.496 | INFO | "gpu" : " 17 GB", | 20260506 23:31:14.496 | INFO | "huge" : "", | 20260506 23:31:14.496 | INFO | "huge_shared" : " 27 GB", | 20260506 23:31:14.496 | INFO | "malloc" : " 14 GB", | 20260506 23:31:14.496 | INFO | "pinned" : " 29 GB" | 20260506 23:31:14.496 | INFO | } | 20260506 23:31:14.496 | INFO | Perf report for the last 5.13857 seconds: | 20260506 23:31:14.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:31:14.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:14.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:14.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:14.496 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 23:31:14.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:14.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:14.496 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:31:14.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:14.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:14.496 | INFO | Uploaded 36GiB to GPU in 1.97874s (18.1934GiB/s) | 20260506 23:31:14.496 | INFO | Downloaded 34GiB to GPU in 1.93481s (17.5728GiB/s) | 20260506 23:31:19.636 | INFO | Analyzing chunk frames = [5318144,5318656) | 20260506 23:31:19.636 | INFO | MemInfo: { | 20260506 23:31:19.636 | INFO | "gpu" : " 17 GB", | 20260506 23:31:19.636 | INFO | "huge" : "", | 20260506 23:31:19.636 | INFO | "huge_shared" : " 27 GB", | 20260506 23:31:19.636 | INFO | "malloc" : " 14 GB", | 20260506 23:31:19.636 | INFO | "pinned" : " 29 GB" | 20260506 23:31:19.636 | INFO | } | 20260506 23:31:19.636 | INFO | Perf report for the last 5.13984 seconds: | 20260506 23:31:19.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:31:19.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:19.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:19.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:19.636 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 23:31:19.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:19.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:19.636 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:31:19.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:19.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:19.636 | INFO | Uploaded 36GiB to GPU in 2.01171s (17.8952GiB/s) | 20260506 23:31:19.636 | INFO | Downloaded 34GiB to GPU in 1.91338s (17.7696GiB/s) | 20260506 23:31:24.774 | INFO | Analyzing chunk frames = [5318656,5319168) | 20260506 23:31:24.774 | INFO | MemInfo: { | 20260506 23:31:24.774 | INFO | "gpu" : " 17 GB", | 20260506 23:31:24.774 | INFO | "huge" : "", | 20260506 23:31:24.774 | INFO | "huge_shared" : " 27 GB", | 20260506 23:31:24.774 | INFO | "malloc" : " 14 GB", | 20260506 23:31:24.774 | INFO | "pinned" : " 29 GB" | 20260506 23:31:24.774 | INFO | } | 20260506 23:31:24.774 | INFO | Perf report for the last 5.13832 seconds: | 20260506 23:31:24.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:31:24.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:24.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:24.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:24.774 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 23:31:24.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:24.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:24.774 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:31:24.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:24.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:24.775 | INFO | Uploaded 36GiB to GPU in 1.988s (18.1087GiB/s) | 20260506 23:31:24.775 | INFO | Downloaded 34GiB to GPU in 1.94235s (17.5046GiB/s) | 20260506 23:31:29.916 | INFO | Analyzing chunk frames = [5319168,5319680) | 20260506 23:31:29.916 | INFO | MemInfo: { | 20260506 23:31:29.916 | INFO | "gpu" : " 17 GB", | 20260506 23:31:29.916 | INFO | "huge" : "", | 20260506 23:31:29.916 | INFO | "huge_shared" : " 27 GB", | 20260506 23:31:29.916 | INFO | "malloc" : " 14 GB", | 20260506 23:31:29.916 | INFO | "pinned" : " 29 GB" | 20260506 23:31:29.916 | INFO | } | 20260506 23:31:29.916 | INFO | Perf report for the last 5.14148 seconds: | 20260506 23:31:29.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:31:29.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:29.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:29.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:29.916 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 23:31:29.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:29.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:29.916 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:31:29.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:29.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:29.916 | INFO | Uploaded 36GiB to GPU in 2.01897s (17.8309GiB/s) | 20260506 23:31:29.916 | INFO | Downloaded 34GiB to GPU in 1.94811s (17.4528GiB/s) | 20260506 23:31:35.051 | INFO | Analyzing chunk frames = [5319680,5320192) | 20260506 23:31:35.051 | INFO | MemInfo: { | 20260506 23:31:35.051 | INFO | "gpu" : " 17 GB", | 20260506 23:31:35.051 | INFO | "huge" : "", | 20260506 23:31:35.051 | INFO | "huge_shared" : " 27 GB", | 20260506 23:31:35.051 | INFO | "malloc" : " 14 GB", | 20260506 23:31:35.051 | INFO | "pinned" : " 29 GB" | 20260506 23:31:35.051 | INFO | } | 20260506 23:31:35.051 | INFO | Perf report for the last 5.13503 seconds: | 20260506 23:31:35.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:31:35.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:35.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:35.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:35.051 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:31:35.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:35.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:35.051 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 23:31:35.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:35.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:35.051 | INFO | Uploaded 36GiB to GPU in 1.97148s (18.2604GiB/s) | 20260506 23:31:35.051 | INFO | Downloaded 34GiB to GPU in 2.00194s (16.9836GiB/s) | 20260506 23:31:40.186 | INFO | Analyzing chunk frames = [5320192,5320704) | 20260506 23:31:40.186 | INFO | MemInfo: { | 20260506 23:31:40.186 | INFO | "gpu" : " 17 GB", | 20260506 23:31:40.186 | INFO | "huge" : "", | 20260506 23:31:40.186 | INFO | "huge_shared" : " 27 GB", | 20260506 23:31:40.186 | INFO | "malloc" : " 14 GB", | 20260506 23:31:40.186 | INFO | "pinned" : " 29 GB" | 20260506 23:31:40.186 | INFO | } | 20260506 23:31:40.186 | INFO | Perf report for the last 5.13529 seconds: | 20260506 23:31:40.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:31:40.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:40.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:40.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:40.186 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:31:40.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:40.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:40.186 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:31:40.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:40.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:40.186 | INFO | Uploaded 36GiB to GPU in 1.99003s (18.0902GiB/s) | 20260506 23:31:40.186 | INFO | Downloaded 34GiB to GPU in 1.98142s (17.1595GiB/s) | 20260506 23:31:40.285 | INFO | Starting a round of Baz IO | 20260506 23:31:43.416 | INFO | Populating RTMetrics report | 20260506 23:31:43.416 | INFO | RTMetrics report populated | 20260506 23:31:43.419 | INFO | Opening tmp RTMetrics json file | 20260506 23:31:43.442 | INFO | Serializing RTMetrics to json | 20260506 23:31:43.504 | INFO | Serialization done, swapping tmp file now... | 20260506 23:31:43.535 | INFO | RTMetrics done | 20260506 23:31:43.673 | INFO | Finished a round of Baz IO: Wrote 1.877 GiB in 3.387 seconds | 20260506 23:31:43.673 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.45% / 0.00% / 66.55% | 20260506 23:31:43.673 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.408343% / 0% / 9.53674e-05% | 20260506 23:31:43.673 | INFO | Baz IO overall: | 20260506 23:31:43.673 | INFO | BytesWritten : 2784238658320 | 20260506 23:31:43.673 | INFO | NumEvents : 1342273912217 | 20260506 23:31:45.327 | INFO | Analyzing chunk frames = [5320704,5321216) | 20260506 23:31:45.327 | INFO | MemInfo: { | 20260506 23:31:45.327 | INFO | "gpu" : " 17 GB", | 20260506 23:31:45.327 | INFO | "huge" : "", | 20260506 23:31:45.327 | INFO | "huge_shared" : " 27 GB", | 20260506 23:31:45.327 | INFO | "malloc" : " 14 GB", | 20260506 23:31:45.327 | INFO | "pinned" : " 29 GB" | 20260506 23:31:45.327 | INFO | } | 20260506 23:31:45.327 | INFO | Perf report for the last 5.14131 seconds: | 20260506 23:31:45.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:31:45.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:45.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:45.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:45.328 | INFO | ANALYSIS : 62.3%, 2.9, 95.3ms, 0 | 20260506 23:31:45.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:45.328 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 23:31:45.328 | INFO | PRE_HQ : 59.9%, 1.8, 56.5ms, 0 | 20260506 23:31:45.328 | INFO | BAZWRITER : 65.6%, 4.5, 151.8ms, 0 | 20260506 23:31:45.328 | INFO | BAZ_SAMPLING : 2.9%, 1.0, 1.5ms, 0 | 20260506 23:31:45.328 | INFO | Uploaded 36GiB to GPU in 2.05159s (17.5474GiB/s) | 20260506 23:31:45.328 | INFO | Downloaded 38GiB to GPU in 2.19366s (17.3226GiB/s) | 20260506 23:31:50.471 | INFO | Analyzing chunk frames = [5321216,5321728) | 20260506 23:31:50.471 | INFO | MemInfo: { | 20260506 23:31:50.471 | INFO | "gpu" : " 17 GB", | 20260506 23:31:50.471 | INFO | "huge" : "", | 20260506 23:31:50.471 | INFO | "huge_shared" : " 27 GB", | 20260506 23:31:50.471 | INFO | "malloc" : " 14 GB", | 20260506 23:31:50.471 | INFO | "pinned" : " 29 GB" | 20260506 23:31:50.471 | INFO | } | 20260506 23:31:50.471 | INFO | Perf report for the last 5.14383 seconds: | 20260506 23:31:50.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:31:50.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:50.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:50.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:50.471 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 23:31:50.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:50.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:50.471 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260506 23:31:50.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:50.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:50.471 | INFO | Uploaded 36GiB to GPU in 2.01808s (17.8388GiB/s) | 20260506 23:31:50.472 | INFO | Downloaded 34GiB to GPU in 1.92734s (17.6409GiB/s) | 20260506 23:31:55.609 | INFO | Analyzing chunk frames = [5321728,5322240) | 20260506 23:31:55.609 | INFO | MemInfo: { | 20260506 23:31:55.609 | INFO | "gpu" : " 17 GB", | 20260506 23:31:55.609 | INFO | "huge" : "", | 20260506 23:31:55.609 | INFO | "huge_shared" : " 27 GB", | 20260506 23:31:55.609 | INFO | "malloc" : " 14 GB", | 20260506 23:31:55.609 | INFO | "pinned" : " 29 GB" | 20260506 23:31:55.609 | INFO | } | 20260506 23:31:55.609 | INFO | Perf report for the last 5.13752 seconds: | 20260506 23:31:55.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:31:55.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:55.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:55.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:55.609 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:31:55.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:55.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:55.609 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:31:55.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:55.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:31:55.609 | INFO | Uploaded 36GiB to GPU in 1.99425s (18.0519GiB/s) | 20260506 23:31:55.609 | INFO | Downloaded 34GiB to GPU in 1.93152s (17.6027GiB/s) | 20260506 23:32:00.750 | INFO | Analyzing chunk frames = [5322240,5322752) | 20260506 23:32:00.750 | INFO | MemInfo: { | 20260506 23:32:00.750 | INFO | "gpu" : " 17 GB", | 20260506 23:32:00.750 | INFO | "huge" : "", | 20260506 23:32:00.750 | INFO | "huge_shared" : " 27 GB", | 20260506 23:32:00.750 | INFO | "malloc" : " 14 GB", | 20260506 23:32:00.750 | INFO | "pinned" : " 29 GB" | 20260506 23:32:00.750 | INFO | } | 20260506 23:32:00.751 | INFO | Perf report for the last 5.14166 seconds: | 20260506 23:32:00.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:32:00.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:00.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:00.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:00.751 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 23:32:00.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:00.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:00.751 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:32:00.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:00.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:00.751 | INFO | Uploaded 36GiB to GPU in 1.97255s (18.2505GiB/s) | 20260506 23:32:00.751 | INFO | Downloaded 34GiB to GPU in 1.90406s (17.8566GiB/s) | 20260506 23:32:05.885 | INFO | Analyzing chunk frames = [5322752,5323264) | 20260506 23:32:05.885 | INFO | MemInfo: { | 20260506 23:32:05.885 | INFO | "gpu" : " 17 GB", | 20260506 23:32:05.885 | INFO | "huge" : "", | 20260506 23:32:05.885 | INFO | "huge_shared" : " 27 GB", | 20260506 23:32:05.885 | INFO | "malloc" : " 14 GB", | 20260506 23:32:05.885 | INFO | "pinned" : " 29 GB" | 20260506 23:32:05.885 | INFO | } | 20260506 23:32:05.885 | INFO | Perf report for the last 5.13475 seconds: | 20260506 23:32:05.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:32:05.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:05.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:05.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:05.885 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 23:32:05.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:05.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:05.885 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:32:05.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:05.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:05.886 | INFO | Uploaded 36GiB to GPU in 2.00872s (17.9219GiB/s) | 20260506 23:32:05.886 | INFO | Downloaded 34GiB to GPU in 1.94736s (17.4595GiB/s) | 20260506 23:32:11.029 | INFO | Analyzing chunk frames = [5323264,5323776) | 20260506 23:32:11.029 | INFO | MemInfo: { | 20260506 23:32:11.029 | INFO | "gpu" : " 17 GB", | 20260506 23:32:11.029 | INFO | "huge" : "", | 20260506 23:32:11.029 | INFO | "huge_shared" : " 27 GB", | 20260506 23:32:11.029 | INFO | "malloc" : " 14 GB", | 20260506 23:32:11.029 | INFO | "pinned" : " 29 GB" | 20260506 23:32:11.029 | INFO | } | 20260506 23:32:11.030 | INFO | Perf report for the last 5.14423 seconds: | 20260506 23:32:11.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:32:11.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:11.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:11.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:11.030 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 23:32:11.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:11.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:11.030 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:32:11.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:11.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:11.030 | INFO | Uploaded 36GiB to GPU in 2.02283s (17.7968GiB/s) | 20260506 23:32:11.030 | INFO | Downloaded 34GiB to GPU in 1.93411s (17.5791GiB/s) | 20260506 23:32:16.167 | INFO | Analyzing chunk frames = [5323776,5324288) | 20260506 23:32:16.167 | INFO | MemInfo: { | 20260506 23:32:16.167 | INFO | "gpu" : " 17 GB", | 20260506 23:32:16.167 | INFO | "huge" : "", | 20260506 23:32:16.167 | INFO | "huge_shared" : " 27 GB", | 20260506 23:32:16.167 | INFO | "malloc" : " 14 GB", | 20260506 23:32:16.167 | INFO | "pinned" : " 29 GB" | 20260506 23:32:16.167 | INFO | } | 20260506 23:32:16.167 | INFO | Perf report for the last 5.13734 seconds: | 20260506 23:32:16.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:32:16.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:16.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:16.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:16.167 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:32:16.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:16.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:16.167 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:32:16.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:16.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:16.167 | INFO | Uploaded 36GiB to GPU in 1.96523s (18.3185GiB/s) | 20260506 23:32:16.167 | INFO | Downloaded 34GiB to GPU in 1.99761s (17.0203GiB/s) | 20260506 23:32:21.303 | INFO | Analyzing chunk frames = [5324288,5324800) | 20260506 23:32:21.303 | INFO | MemInfo: { | 20260506 23:32:21.303 | INFO | "gpu" : " 17 GB", | 20260506 23:32:21.303 | INFO | "huge" : "", | 20260506 23:32:21.303 | INFO | "huge_shared" : " 27 GB", | 20260506 23:32:21.303 | INFO | "malloc" : " 14 GB", | 20260506 23:32:21.303 | INFO | "pinned" : " 29 GB" | 20260506 23:32:21.303 | INFO | } | 20260506 23:32:21.303 | INFO | Perf report for the last 5.13661 seconds: | 20260506 23:32:21.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:32:21.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:21.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:21.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:21.304 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:32:21.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:21.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:21.304 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:32:21.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:21.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:21.304 | INFO | Uploaded 36GiB to GPU in 1.95811s (18.3851GiB/s) | 20260506 23:32:21.304 | INFO | Downloaded 34GiB to GPU in 1.98686s (17.1124GiB/s) | 20260506 23:32:21.304 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:32:21.304 | INFO | Total mem usage: 18 GiB | 20260506 23:32:21.304 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:32:21.304 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -18.3%) | 20260506 23:32:21.304 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:32:21.304 | INFO | High Water Breakdown: | 20260506 23:32:21.304 | INFO | 61.1% : 9.31 GiB ( 81.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:32:21.304 | INFO | 38.7% : 5.9 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:32:21.304 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:32:21.304 | INFO | ------------------------------------------ | 20260506 23:32:21.304 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:32:21.304 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:32:21.304 | INFO | Total mem usage: 26.5 GiB | 20260506 23:32:21.304 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:32:21.304 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:32:21.304 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:32:21.304 | INFO | High Water Breakdown: | 20260506 23:32:21.304 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:32:21.304 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:32:21.304 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:32:21.304 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:32:21.304 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:32:21.304 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:32:21.304 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:32:21.304 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:32:21.304 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:32:21.304 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:32:21.304 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:32:21.304 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:32:21.305 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:32:21.305 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:32:21.305 | INFO | ------------------------------------------ | 20260506 23:32:21.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:32:21.305 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:32:21.305 | INFO | Total mem usage: 25.6 GiB | 20260506 23:32:21.305 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:32:21.305 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:32:21.305 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:32:21.305 | INFO | High Water Breakdown: | 20260506 23:32:21.305 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:32:21.305 | INFO | ------------------------------------------ | 20260506 23:32:21.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:32:21.305 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:32:21.305 | INFO | Total mem usage: 15.7 GiB | 20260506 23:32:21.305 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:32:21.305 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:32:21.305 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 23:32:21.305 | INFO | High Water Breakdown: | 20260506 23:32:21.305 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:32:21.305 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:32:21.305 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:32:21.305 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:32:21.305 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:32:21.305 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:32:21.305 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:32:21.305 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:32:21.305 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:32:21.305 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:32:21.305 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:32:21.305 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:32:21.305 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:32:21.305 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:32:21.305 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:32:21.305 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:32:21.305 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:32:21.305 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:32:21.305 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:32:21.305 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:32:21.305 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:32:21.305 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:32:21.305 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:32:21.305 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:32:21.305 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:32:21.305 | INFO | ------------------------------------------ | 20260506 23:32:21.305 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:32:21.305 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:32:21.305 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:32:21.305 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:32:21.305 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.62e+01, 1.34e+01 / 2.48e+00 | 20260506 23:32:21.305 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.98e-03 / 5.39e-03 | 20260506 23:32:21.305 | INFO | PerfReport: 11.67%, Baseline -- 3.62e+00 / 4.40e+00, 5.44e-01 / 3.14e-01 | 20260506 23:32:21.305 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.45e+00, 1.77e-01 / 1.20e-01 | 20260506 23:32:21.305 | INFO | PerfReport: 16.08%, DME -- 3.99e+01 / 4.50e+01, 4.29e+00 / 2.93e+00 | 20260506 23:32:21.305 | INFO | PerfReport: 57.38%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.35e+00 / 2.05e+00 | 20260506 23:32:21.305 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.19e+00 / 2.72e+00, 3.77e-01 / 2.63e-01 | 20260506 23:32:21.305 | INFO | PerfReport: 3.18%, Metrics -- 9.87e-01 / 1.32e+00, 2.58e-01 / 9.28e-02 | 20260506 23:32:21.305 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.36e-01 / 3.88e-01, 5.38e-02 / 2.20e-02 | 20260506 23:32:21.305 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:32:21.305 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:32:21.305 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:32:21.305 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 23:32:21.305 | INFO | PerfReport: 21.59%, Upload -- 2.08e+01 / 0.00e+00, 6.92e+00 / 0.00e+00 | 20260506 23:32:21.305 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 23:32:21.305 | INFO | PerfReport: 32.12%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 23:32:21.305 | INFO | PerfReport: 1.29%, LockUpload -- 1.25e+00 / 0.00e+00, 4.72e+00 / 0.00e+00 | 20260506 23:32:21.305 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260506 23:32:21.305 | INFO | PerfReport: 22.32%, LockCompute -- 2.15e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:32:21.305 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260506 23:32:21.401 | INFO | Starting a round of Baz IO | 20260506 23:32:24.530 | INFO | Populating RTMetrics report | 20260506 23:32:24.530 | INFO | RTMetrics report populated | 20260506 23:32:24.530 | INFO | Opening tmp RTMetrics json file | 20260506 23:32:24.534 | INFO | Serializing RTMetrics to json | 20260506 23:32:24.557 | INFO | Serialization done, swapping tmp file now... | 20260506 23:32:24.557 | INFO | RTMetrics done | 20260506 23:32:24.655 | INFO | Finished a round of Baz IO: Wrote 1.878 GiB in 3.253 seconds | 20260506 23:32:24.655 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.45% / 0.00% / 66.55% | 20260506 23:32:24.655 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.409337% / 1.19209e-05% / 7.54992e-05% | 20260506 23:32:24.655 | INFO | Baz IO overall: | 20260506 23:32:24.655 | INFO | BytesWritten : 2786254962992 | 20260506 23:32:24.655 | INFO | NumEvents : 1343233996666 | 20260506 23:32:26.447 | INFO | Analyzing chunk frames = [5324800,5325312) | 20260506 23:32:26.447 | INFO | MemInfo: { | 20260506 23:32:26.447 | INFO | "gpu" : " 17 GB", | 20260506 23:32:26.447 | INFO | "huge" : "", | 20260506 23:32:26.447 | INFO | "huge_shared" : " 27 GB", | 20260506 23:32:26.447 | INFO | "malloc" : " 14 GB", | 20260506 23:32:26.447 | INFO | "pinned" : " 29 GB" | 20260506 23:32:26.447 | INFO | } | 20260506 23:32:26.447 | INFO | Perf report for the last 5.1436 seconds: | 20260506 23:32:26.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:32:26.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:26.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:26.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:26.447 | INFO | ANALYSIS : 62.2%, 2.9, 95.2ms, 0 | 20260506 23:32:26.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:26.447 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 23:32:26.447 | INFO | PRE_HQ : 59.7%, 1.8, 56.4ms, 0 | 20260506 23:32:26.447 | INFO | BAZWRITER : 62.8%, 4.6, 147.5ms, 0 | 20260506 23:32:26.447 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:32:26.447 | INFO | Uploaded 36GiB to GPU in 2.04588s (17.5963GiB/s) | 20260506 23:32:26.448 | INFO | Downloaded 38GiB to GPU in 2.1995s (17.2767GiB/s) | 20260506 23:32:31.585 | INFO | Analyzing chunk frames = [5325312,5325824) | 20260506 23:32:31.585 | INFO | MemInfo: { | 20260506 23:32:31.585 | INFO | "gpu" : " 17 GB", | 20260506 23:32:31.585 | INFO | "huge" : "", | 20260506 23:32:31.585 | INFO | "huge_shared" : " 27 GB", | 20260506 23:32:31.586 | INFO | "malloc" : " 14 GB", | 20260506 23:32:31.586 | INFO | "pinned" : " 29 GB" | 20260506 23:32:31.586 | INFO | } | 20260506 23:32:31.586 | INFO | Perf report for the last 5.13863 seconds: | 20260506 23:32:31.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:32:31.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:31.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:31.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:31.586 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 23:32:31.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:31.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:31.586 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:32:31.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:31.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:31.586 | INFO | Uploaded 36GiB to GPU in 1.99553s (18.0403GiB/s) | 20260506 23:32:31.586 | INFO | Downloaded 34GiB to GPU in 1.93373s (17.5826GiB/s) | 20260506 23:32:36.726 | INFO | Analyzing chunk frames = [5325824,5326336) | 20260506 23:32:36.726 | INFO | MemInfo: { | 20260506 23:32:36.726 | INFO | "gpu" : " 17 GB", | 20260506 23:32:36.726 | INFO | "huge" : "", | 20260506 23:32:36.727 | INFO | "huge_shared" : " 27 GB", | 20260506 23:32:36.727 | INFO | "malloc" : " 14 GB", | 20260506 23:32:36.727 | INFO | "pinned" : " 29 GB" | 20260506 23:32:36.727 | INFO | } | 20260506 23:32:36.727 | INFO | Perf report for the last 5.14094 seconds: | 20260506 23:32:36.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:32:36.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:36.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:36.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:36.727 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260506 23:32:36.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:36.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:36.727 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 23:32:36.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:36.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:36.727 | INFO | Uploaded 36GiB to GPU in 1.9636s (18.3336GiB/s) | 20260506 23:32:36.727 | INFO | Downloaded 34GiB to GPU in 1.93285s (17.5906GiB/s) | 20260506 23:32:41.865 | INFO | Analyzing chunk frames = [5326336,5326848) | 20260506 23:32:41.865 | INFO | MemInfo: { | 20260506 23:32:41.865 | INFO | "gpu" : " 17 GB", | 20260506 23:32:41.865 | INFO | "huge" : "", | 20260506 23:32:41.865 | INFO | "huge_shared" : " 27 GB", | 20260506 23:32:41.865 | INFO | "malloc" : " 14 GB", | 20260506 23:32:41.865 | INFO | "pinned" : " 29 GB" | 20260506 23:32:41.865 | INFO | } | 20260506 23:32:41.865 | INFO | Perf report for the last 5.13826 seconds: | 20260506 23:32:41.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:32:41.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:41.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:41.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:41.865 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 23:32:41.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:41.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:41.865 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:32:41.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:41.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:41.865 | INFO | Uploaded 36GiB to GPU in 1.96296s (18.3396GiB/s) | 20260506 23:32:41.865 | INFO | Downloaded 34GiB to GPU in 1.90611s (17.8373GiB/s) | 20260506 23:32:47.002 | INFO | Analyzing chunk frames = [5326848,5327360) | 20260506 23:32:47.002 | INFO | MemInfo: { | 20260506 23:32:47.002 | INFO | "gpu" : " 17 GB", | 20260506 23:32:47.002 | INFO | "huge" : "", | 20260506 23:32:47.002 | INFO | "huge_shared" : " 27 GB", | 20260506 23:32:47.002 | INFO | "malloc" : " 14 GB", | 20260506 23:32:47.002 | INFO | "pinned" : " 29 GB" | 20260506 23:32:47.002 | INFO | } | 20260506 23:32:47.002 | INFO | Perf report for the last 5.1375 seconds: | 20260506 23:32:47.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:32:47.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:47.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:47.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:47.003 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260506 23:32:47.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:47.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:47.003 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 23:32:47.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:47.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:47.003 | INFO | Uploaded 36GiB to GPU in 2.00118s (17.9894GiB/s) | 20260506 23:32:47.003 | INFO | Downloaded 34GiB to GPU in 1.94757s (17.4576GiB/s) | 20260506 23:32:52.134 | INFO | Analyzing chunk frames = [5327360,5327872) | 20260506 23:32:52.134 | INFO | MemInfo: { | 20260506 23:32:52.134 | INFO | "gpu" : " 17 GB", | 20260506 23:32:52.134 | INFO | "huge" : "", | 20260506 23:32:52.134 | INFO | "huge_shared" : " 27 GB", | 20260506 23:32:52.134 | INFO | "malloc" : " 14 GB", | 20260506 23:32:52.134 | INFO | "pinned" : " 29 GB" | 20260506 23:32:52.134 | INFO | } | 20260506 23:32:52.134 | INFO | Perf report for the last 5.13171 seconds: | 20260506 23:32:52.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:32:52.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:52.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:52.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:52.134 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260506 23:32:52.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:52.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:52.134 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:32:52.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:52.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:52.134 | INFO | Uploaded 36GiB to GPU in 2.01254s (17.8879GiB/s) | 20260506 23:32:52.134 | INFO | Downloaded 34GiB to GPU in 1.9403s (17.5231GiB/s) | 20260506 23:32:57.275 | INFO | Analyzing chunk frames = [5327872,5328384) | 20260506 23:32:57.275 | INFO | MemInfo: { | 20260506 23:32:57.275 | INFO | "gpu" : " 17 GB", | 20260506 23:32:57.275 | INFO | "huge" : "", | 20260506 23:32:57.275 | INFO | "huge_shared" : " 27 GB", | 20260506 23:32:57.275 | INFO | "malloc" : " 14 GB", | 20260506 23:32:57.275 | INFO | "pinned" : " 29 GB" | 20260506 23:32:57.276 | INFO | } | 20260506 23:32:57.276 | INFO | Perf report for the last 5.14142 seconds: | 20260506 23:32:57.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:32:57.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:57.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:57.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:57.276 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 23:32:57.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:57.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:57.276 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 23:32:57.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:57.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:32:57.276 | INFO | Uploaded 36GiB to GPU in 1.9521s (18.4417GiB/s) | 20260506 23:32:57.276 | INFO | Downloaded 34GiB to GPU in 1.9828s (17.1475GiB/s) | 20260506 23:33:02.421 | INFO | Analyzing chunk frames = [5328384,5328896) | 20260506 23:33:02.421 | INFO | MemInfo: { | 20260506 23:33:02.421 | INFO | "gpu" : " 17 GB", | 20260506 23:33:02.421 | INFO | "huge" : "", | 20260506 23:33:02.421 | INFO | "huge_shared" : " 27 GB", | 20260506 23:33:02.421 | INFO | "malloc" : " 14 GB", | 20260506 23:33:02.421 | INFO | "pinned" : " 29 GB" | 20260506 23:33:02.421 | INFO | } | 20260506 23:33:02.421 | INFO | Perf report for the last 5.14555 seconds: | 20260506 23:33:02.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:33:02.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:02.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:02.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:02.421 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260506 23:33:02.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:02.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:02.421 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260506 23:33:02.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:02.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:02.421 | INFO | Uploaded 36GiB to GPU in 2.03477s (17.6925GiB/s) | 20260506 23:33:02.421 | INFO | Downloaded 34GiB to GPU in 2.00162s (16.9862GiB/s) | 20260506 23:33:02.522 | INFO | Starting a round of Baz IO | 20260506 23:33:05.696 | INFO | Populating RTMetrics report | 20260506 23:33:05.696 | INFO | RTMetrics report populated | 20260506 23:33:05.696 | INFO | Opening tmp RTMetrics json file | 20260506 23:33:05.697 | INFO | Serializing RTMetrics to json | 20260506 23:33:05.698 | INFO | Serialization done, swapping tmp file now... | 20260506 23:33:05.699 | INFO | RTMetrics done | 20260506 23:33:05.824 | INFO | Finished a round of Baz IO: Wrote 1.877 GiB in 3.302 seconds | 20260506 23:33:05.824 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.45% / 0.00% / 66.55% | 20260506 23:33:05.824 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.407588% / 1.19209e-05% / 6.75519e-05% | 20260506 23:33:05.824 | INFO | Baz IO overall: | 20260506 23:33:05.825 | INFO | BytesWritten : 2788270605963 | 20260506 23:33:05.825 | INFO | NumEvents : 1344194173661 | 20260506 23:33:07.560 | INFO | Analyzing chunk frames = [5328896,5329408) | 20260506 23:33:07.560 | INFO | MemInfo: { | 20260506 23:33:07.560 | INFO | "gpu" : " 17 GB", | 20260506 23:33:07.560 | INFO | "huge" : "", | 20260506 23:33:07.560 | INFO | "huge_shared" : " 27 GB", | 20260506 23:33:07.560 | INFO | "malloc" : " 14 GB", | 20260506 23:33:07.560 | INFO | "pinned" : " 29 GB" | 20260506 23:33:07.561 | INFO | } | 20260506 23:33:07.561 | INFO | Perf report for the last 5.13938 seconds: | 20260506 23:33:07.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:33:07.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:07.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:07.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:07.561 | INFO | ANALYSIS : 62.6%, 2.9, 94.3ms, 0 | 20260506 23:33:07.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:07.561 | INFO | RT_METRICS : 46.5%, 1.0, 24.4ms, 0 | 20260506 23:33:07.561 | INFO | PRE_HQ : 61.1%, 2.0, 61.9ms, 0 | 20260506 23:33:07.561 | INFO | BAZWRITER : 64.0%, 4.3, 140.3ms, 0 | 20260506 23:33:07.561 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 23:33:07.561 | INFO | Uploaded 36GiB to GPU in 2.06978s (17.3932GiB/s) | 20260506 23:33:07.561 | INFO | Downloaded 38GiB to GPU in 2.18093s (17.4238GiB/s) | 20260506 23:33:12.703 | INFO | Analyzing chunk frames = [5329408,5329920) | 20260506 23:33:12.703 | INFO | MemInfo: { | 20260506 23:33:12.703 | INFO | "gpu" : " 17 GB", | 20260506 23:33:12.703 | INFO | "huge" : "", | 20260506 23:33:12.703 | INFO | "huge_shared" : " 27 GB", | 20260506 23:33:12.703 | INFO | "malloc" : " 14 GB", | 20260506 23:33:12.703 | INFO | "pinned" : " 29 GB" | 20260506 23:33:12.703 | INFO | } | 20260506 23:33:12.703 | INFO | Perf report for the last 5.14274 seconds: | 20260506 23:33:12.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:33:12.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:12.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:12.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:12.703 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:33:12.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:12.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:12.703 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:33:12.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:12.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:12.704 | INFO | Uploaded 36GiB to GPU in 1.99753s (18.0222GiB/s) | 20260506 23:33:12.704 | INFO | Downloaded 34GiB to GPU in 1.92312s (17.6796GiB/s) | 20260506 23:33:17.843 | INFO | Analyzing chunk frames = [5329920,5330432) | 20260506 23:33:17.843 | INFO | MemInfo: { | 20260506 23:33:17.843 | INFO | "gpu" : " 17 GB", | 20260506 23:33:17.843 | INFO | "huge" : "", | 20260506 23:33:17.843 | INFO | "huge_shared" : " 27 GB", | 20260506 23:33:17.843 | INFO | "malloc" : " 14 GB", | 20260506 23:33:17.843 | INFO | "pinned" : " 29 GB" | 20260506 23:33:17.843 | INFO | } | 20260506 23:33:17.843 | INFO | Perf report for the last 5.14015 seconds: | 20260506 23:33:17.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:33:17.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:17.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:17.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:17.844 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 23:33:17.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:17.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:17.844 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:33:17.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:17.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:17.844 | INFO | Uploaded 36GiB to GPU in 1.9974s (18.0234GiB/s) | 20260506 23:33:17.844 | INFO | Downloaded 34GiB to GPU in 1.93351s (17.5846GiB/s) | 20260506 23:33:22.980 | INFO | Analyzing chunk frames = [5330432,5330944) | 20260506 23:33:22.980 | INFO | MemInfo: { | 20260506 23:33:22.980 | INFO | "gpu" : " 17 GB", | 20260506 23:33:22.980 | INFO | "huge" : "", | 20260506 23:33:22.980 | INFO | "huge_shared" : " 27 GB", | 20260506 23:33:22.980 | INFO | "malloc" : " 14 GB", | 20260506 23:33:22.980 | INFO | "pinned" : " 29 GB" | 20260506 23:33:22.980 | INFO | } | 20260506 23:33:22.980 | INFO | Perf report for the last 5.13645 seconds: | 20260506 23:33:22.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:33:22.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:22.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:22.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:22.980 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260506 23:33:22.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:22.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:22.980 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 23:33:22.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:22.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:22.980 | INFO | Uploaded 36GiB to GPU in 1.98233s (18.1605GiB/s) | 20260506 23:33:22.980 | INFO | Downloaded 34GiB to GPU in 1.90431s (17.8542GiB/s) | 20260506 23:33:28.118 | INFO | Analyzing chunk frames = [5330944,5331456) | 20260506 23:33:28.118 | INFO | MemInfo: { | 20260506 23:33:28.118 | INFO | "gpu" : " 17 GB", | 20260506 23:33:28.118 | INFO | "huge" : "", | 20260506 23:33:28.118 | INFO | "huge_shared" : " 27 GB", | 20260506 23:33:28.118 | INFO | "malloc" : " 14 GB", | 20260506 23:33:28.118 | INFO | "pinned" : " 29 GB" | 20260506 23:33:28.118 | INFO | } | 20260506 23:33:28.118 | INFO | Perf report for the last 5.13821 seconds: | 20260506 23:33:28.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:33:28.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:28.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:28.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:28.118 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 23:33:28.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:28.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:28.118 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:33:28.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:28.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:28.118 | INFO | Uploaded 36GiB to GPU in 1.99746s (18.0229GiB/s) | 20260506 23:33:28.118 | INFO | Downloaded 34GiB to GPU in 1.93623s (17.5599GiB/s) | 20260506 23:33:33.254 | INFO | Analyzing chunk frames = [5331456,5331968) | 20260506 23:33:33.254 | INFO | MemInfo: { | 20260506 23:33:33.254 | INFO | "gpu" : " 17 GB", | 20260506 23:33:33.254 | INFO | "huge" : "", | 20260506 23:33:33.254 | INFO | "huge_shared" : " 27 GB", | 20260506 23:33:33.254 | INFO | "malloc" : " 14 GB", | 20260506 23:33:33.255 | INFO | "pinned" : " 29 GB" | 20260506 23:33:33.255 | INFO | } | 20260506 23:33:33.255 | INFO | Perf report for the last 5.13641 seconds: | 20260506 23:33:33.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:33:33.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:33.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:33.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:33.255 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 23:33:33.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:33.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:33.255 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 23:33:33.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:33.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:33.255 | INFO | Uploaded 36GiB to GPU in 2.01973s (17.8241GiB/s) | 20260506 23:33:33.255 | INFO | Downloaded 34GiB to GPU in 1.93541s (17.5673GiB/s) | 20260506 23:33:38.396 | INFO | Analyzing chunk frames = [5331968,5332480) | 20260506 23:33:38.396 | INFO | MemInfo: { | 20260506 23:33:38.396 | INFO | "gpu" : " 17 GB", | 20260506 23:33:38.396 | INFO | "huge" : "", | 20260506 23:33:38.396 | INFO | "huge_shared" : " 27 GB", | 20260506 23:33:38.396 | INFO | "malloc" : " 14 GB", | 20260506 23:33:38.396 | INFO | "pinned" : " 29 GB" | 20260506 23:33:38.396 | INFO | } | 20260506 23:33:38.396 | INFO | Perf report for the last 5.14124 seconds: | 20260506 23:33:38.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:33:38.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:38.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:38.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:38.396 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 23:33:38.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:38.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:38.396 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260506 23:33:38.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:38.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:38.396 | INFO | Uploaded 36GiB to GPU in 1.95651s (18.4001GiB/s) | 20260506 23:33:38.396 | INFO | Downloaded 34GiB to GPU in 1.98189s (17.1553GiB/s) | 20260506 23:33:43.538 | INFO | Analyzing chunk frames = [5332480,5332992) | 20260506 23:33:43.538 | INFO | MemInfo: { | 20260506 23:33:43.538 | INFO | "gpu" : " 17 GB", | 20260506 23:33:43.538 | INFO | "huge" : "", | 20260506 23:33:43.538 | INFO | "huge_shared" : " 27 GB", | 20260506 23:33:43.538 | INFO | "malloc" : " 14 GB", | 20260506 23:33:43.538 | INFO | "pinned" : " 29 GB" | 20260506 23:33:43.538 | INFO | } | 20260506 23:33:43.538 | INFO | Perf report for the last 5.14229 seconds: | 20260506 23:33:43.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:33:43.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:43.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:43.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:43.538 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 23:33:43.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:43.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:43.538 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 23:33:43.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:43.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:43.538 | INFO | Uploaded 36GiB to GPU in 2.00448s (17.9598GiB/s) | 20260506 23:33:43.539 | INFO | Downloaded 34GiB to GPU in 1.99867s (17.0113GiB/s) | 20260506 23:33:43.539 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:33:43.539 | INFO | Total mem usage: 18 GiB | 20260506 23:33:43.539 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:33:43.539 | INFO | Recent High Water Marks Sum: 15.2 GiB (Savings: -18.2%) | 20260506 23:33:43.539 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:33:43.539 | INFO | High Water Breakdown: | 20260506 23:33:43.539 | INFO | 61.2% : 9.31 GiB ( 81.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:33:43.539 | INFO | 38.7% : 5.9 GiB ( 88.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:33:43.539 | INFO | 0.131% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:33:43.539 | INFO | ------------------------------------------ | 20260506 23:33:43.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:33:43.539 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:33:43.539 | INFO | Total mem usage: 26.5 GiB | 20260506 23:33:43.539 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:33:43.539 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 23:33:43.539 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 23:33:43.539 | INFO | High Water Breakdown: | 20260506 23:33:43.539 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:33:43.539 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:33:43.539 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:33:43.539 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:33:43.539 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:33:43.539 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:33:43.539 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:33:43.539 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:33:43.539 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:33:43.539 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:33:43.539 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:33:43.539 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:33:43.539 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:33:43.539 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:33:43.539 | INFO | ------------------------------------------ | 20260506 23:33:43.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:33:43.539 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:33:43.539 | INFO | Total mem usage: 25.6 GiB | 20260506 23:33:43.539 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:33:43.539 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:33:43.539 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:33:43.539 | INFO | High Water Breakdown: | 20260506 23:33:43.539 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:33:43.539 | INFO | ------------------------------------------ | 20260506 23:33:43.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:33:43.539 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:33:43.539 | INFO | Total mem usage: 15.7 GiB | 20260506 23:33:43.539 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:33:43.539 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:33:43.539 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:33:43.539 | INFO | High Water Breakdown: | 20260506 23:33:43.539 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:33:43.539 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:33:43.539 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:33:43.539 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:33:43.539 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:33:43.539 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:33:43.539 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:33:43.539 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:33:43.539 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:33:43.539 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:33:43.539 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:33:43.539 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:33:43.539 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:33:43.539 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:33:43.539 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:33:43.539 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:33:43.539 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:33:43.539 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:33:43.539 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:33:43.539 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:33:43.539 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:33:43.539 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:33:43.539 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:33:43.539 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:33:43.539 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:33:43.539 | INFO | ------------------------------------------ | 20260506 23:33:43.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:33:43.539 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:33:43.539 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:33:43.539 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:33:43.539 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.61e+01, 1.34e+01 / 2.86e+00 | 20260506 23:33:43.539 | INFO | PerfReport: 0.30%, AggregateStats -- 9.15e-02 / 1.01e-01, 9.71e-03 / 4.38e-03 | 20260506 23:33:43.539 | INFO | PerfReport: 11.68%, Baseline -- 3.62e+00 / 4.43e+00, 5.42e-01 / 2.88e-01 | 20260506 23:33:43.539 | INFO | PerfReport: 3.27%, Histogram -- 1.01e+00 / 1.59e+00, 2.57e-01 / 9.60e-01 | 20260506 23:33:43.539 | INFO | PerfReport: 16.11%, DME -- 3.99e+01 / 4.50e+01, 4.26e+00 / 3.14e+00 | 20260506 23:33:43.539 | INFO | PerfReport: 57.30%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.29e+00 / 2.01e+00 | 20260506 23:33:43.539 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.19e+00 / 2.76e+00, 3.80e-01 / 2.34e-01 | 20260506 23:33:43.539 | INFO | PerfReport: 3.19%, Metrics -- 9.89e-01 / 1.32e+00, 2.65e-01 / 1.72e-01 | 20260506 23:33:43.539 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.37e-01 / 3.85e-01, 5.18e-02 / 2.09e-02 | 20260506 23:33:43.539 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:33:43.539 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:33:43.539 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:33:43.539 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 23:33:43.539 | INFO | PerfReport: 21.59%, Upload -- 2.08e+01 / 0.00e+00, 6.92e+00 / 0.00e+00 | 20260506 23:33:43.540 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 23:33:43.540 | INFO | PerfReport: 32.13%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 23:33:43.540 | INFO | PerfReport: 1.35%, LockUpload -- 1.30e+00 / 0.00e+00, 5.06e+00 / 0.00e+00 | 20260506 23:33:43.540 | INFO | PerfReport: 1.33%, LockDownload -- 1.29e+00 / 0.00e+00, 3.17e+00 / 0.00e+00 | 20260506 23:33:43.540 | INFO | PerfReport: 22.22%, LockCompute -- 2.14e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260506 23:33:43.540 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260506 23:33:43.637 | INFO | Starting a round of Baz IO | 20260506 23:33:46.799 | INFO | Populating RTMetrics report | 20260506 23:33:46.800 | INFO | RTMetrics report populated | 20260506 23:33:46.800 | INFO | Opening tmp RTMetrics json file | 20260506 23:33:46.801 | INFO | Serializing RTMetrics to json | 20260506 23:33:46.802 | INFO | Serialization done, swapping tmp file now... | 20260506 23:33:46.802 | INFO | RTMetrics done | 20260506 23:33:46.955 | INFO | Finished a round of Baz IO: Wrote 1.878 GiB in 3.318 seconds | 20260506 23:33:46.956 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.44% / 0.00% / 66.55% | 20260506 23:33:46.956 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.405502% / 0% / 8.74201e-05% | 20260506 23:33:46.956 | INFO | Baz IO overall: | 20260506 23:33:46.956 | INFO | BytesWritten : 2790287017881 | 20260506 23:33:46.956 | INFO | NumEvents : 1345154239933 | 20260506 23:33:48.679 | INFO | Analyzing chunk frames = [5332992,5333504) | 20260506 23:33:48.680 | INFO | MemInfo: { | 20260506 23:33:48.680 | INFO | "gpu" : " 17 GB", | 20260506 23:33:48.680 | INFO | "huge" : "", | 20260506 23:33:48.680 | INFO | "huge_shared" : " 27 GB", | 20260506 23:33:48.680 | INFO | "malloc" : " 14 GB", | 20260506 23:33:48.680 | INFO | "pinned" : " 29 GB" | 20260506 23:33:48.680 | INFO | } | 20260506 23:33:48.680 | INFO | Perf report for the last 5.14142 seconds: | 20260506 23:33:48.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:33:48.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:48.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:48.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:48.680 | INFO | ANALYSIS : 62.6%, 2.9, 94.1ms, 0 | 20260506 23:33:48.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:48.680 | INFO | RT_METRICS : 45.6%, 1.0, 23.9ms, 0 | 20260506 23:33:48.680 | INFO | PRE_HQ : 60.4%, 1.9, 61.1ms, 0 | 20260506 23:33:48.680 | INFO | BAZWRITER : 64.2%, 4.6, 150.9ms, 0 | 20260506 23:33:48.680 | INFO | BAZ_SAMPLING : 2.3%, 1.0, 1.2ms, 0 | 20260506 23:33:48.680 | INFO | Uploaded 36GiB to GPU in 2.04634s (17.5924GiB/s) | 20260506 23:33:48.680 | INFO | Downloaded 38GiB to GPU in 2.18118s (17.4218GiB/s) | 20260506 23:33:53.817 | INFO | Analyzing chunk frames = [5333504,5334016) | 20260506 23:33:53.817 | INFO | MemInfo: { | 20260506 23:33:53.817 | INFO | "gpu" : " 17 GB", | 20260506 23:33:53.817 | INFO | "huge" : "", | 20260506 23:33:53.817 | INFO | "huge_shared" : " 27 GB", | 20260506 23:33:53.817 | INFO | "malloc" : " 14 GB", | 20260506 23:33:53.817 | INFO | "pinned" : " 29 GB" | 20260506 23:33:53.817 | INFO | } | 20260506 23:33:53.817 | INFO | Perf report for the last 5.13733 seconds: | 20260506 23:33:53.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:33:53.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:53.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:53.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:53.817 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 23:33:53.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:53.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:53.817 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260506 23:33:53.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:53.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:53.817 | INFO | Uploaded 36GiB to GPU in 2.0124s (17.8891GiB/s) | 20260506 23:33:53.818 | INFO | Downloaded 34GiB to GPU in 1.93433s (17.5772GiB/s) | 20260506 23:33:58.949 | INFO | Analyzing chunk frames = [5334016,5334528) | 20260506 23:33:58.949 | INFO | MemInfo: { | 20260506 23:33:58.949 | INFO | "gpu" : " 17 GB", | 20260506 23:33:58.949 | INFO | "huge" : "", | 20260506 23:33:58.949 | INFO | "huge_shared" : " 27 GB", | 20260506 23:33:58.949 | INFO | "malloc" : " 14 GB", | 20260506 23:33:58.949 | INFO | "pinned" : " 29 GB" | 20260506 23:33:58.949 | INFO | } | 20260506 23:33:58.949 | INFO | Perf report for the last 5.13228 seconds: | 20260506 23:33:58.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:33:58.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:58.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:58.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:58.949 | INFO | ANALYSIS : 60.5%, 3.0, 93.5ms, 0 | 20260506 23:33:58.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:58.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:58.949 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:33:58.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:58.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:33:58.949 | INFO | Uploaded 36GiB to GPU in 1.97084s (18.2663GiB/s) | 20260506 23:33:58.949 | INFO | Downloaded 34GiB to GPU in 1.93424s (17.578GiB/s) | 20260506 23:34:04.089 | INFO | Analyzing chunk frames = [5334528,5335040) | 20260506 23:34:04.089 | INFO | MemInfo: { | 20260506 23:34:04.089 | INFO | "gpu" : " 17 GB", | 20260506 23:34:04.089 | INFO | "huge" : "", | 20260506 23:34:04.089 | INFO | "huge_shared" : " 27 GB", | 20260506 23:34:04.089 | INFO | "malloc" : " 14 GB", | 20260506 23:34:04.089 | INFO | "pinned" : " 29 GB" | 20260506 23:34:04.089 | INFO | } | 20260506 23:34:04.089 | INFO | Perf report for the last 5.13993 seconds: | 20260506 23:34:04.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:34:04.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:04.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:04.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:04.089 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 23:34:04.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:04.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:04.089 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:34:04.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:04.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:04.089 | INFO | Uploaded 36GiB to GPU in 1.93475s (18.6071GiB/s) | 20260506 23:34:04.089 | INFO | Downloaded 34GiB to GPU in 1.87691s (18.1149GiB/s) | 20260506 23:34:09.226 | INFO | Analyzing chunk frames = [5335040,5335552) | 20260506 23:34:09.226 | INFO | MemInfo: { | 20260506 23:34:09.226 | INFO | "gpu" : " 17 GB", | 20260506 23:34:09.226 | INFO | "huge" : "", | 20260506 23:34:09.226 | INFO | "huge_shared" : " 27 GB", | 20260506 23:34:09.226 | INFO | "malloc" : " 14 GB", | 20260506 23:34:09.226 | INFO | "pinned" : " 29 GB" | 20260506 23:34:09.226 | INFO | } | 20260506 23:34:09.226 | INFO | Perf report for the last 5.1368 seconds: | 20260506 23:34:09.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:34:09.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:09.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:09.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:09.226 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 23:34:09.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:09.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:09.226 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:34:09.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:09.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:09.226 | INFO | Uploaded 36GiB to GPU in 1.98137s (18.1692GiB/s) | 20260506 23:34:09.226 | INFO | Downloaded 34GiB to GPU in 1.92645s (17.6491GiB/s) | 20260506 23:34:14.366 | INFO | Analyzing chunk frames = [5335552,5336064) | 20260506 23:34:14.366 | INFO | MemInfo: { | 20260506 23:34:14.366 | INFO | "gpu" : " 17 GB", | 20260506 23:34:14.366 | INFO | "huge" : "", | 20260506 23:34:14.367 | INFO | "huge_shared" : " 27 GB", | 20260506 23:34:14.367 | INFO | "malloc" : " 14 GB", | 20260506 23:34:14.367 | INFO | "pinned" : " 29 GB" | 20260506 23:34:14.367 | INFO | } | 20260506 23:34:14.367 | INFO | Perf report for the last 5.14069 seconds: | 20260506 23:34:14.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:34:14.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:14.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:14.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:14.367 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260506 23:34:14.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:14.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:14.367 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:34:14.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:14.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:14.367 | INFO | Uploaded 36GiB to GPU in 1.9974s (18.0234GiB/s) | 20260506 23:34:14.367 | INFO | Downloaded 34GiB to GPU in 1.94823s (17.4517GiB/s) | 20260506 23:34:19.515 | INFO | Analyzing chunk frames = [5336064,5336576) | 20260506 23:34:19.515 | INFO | MemInfo: { | 20260506 23:34:19.515 | INFO | "gpu" : " 17 GB", | 20260506 23:34:19.515 | INFO | "huge" : "", | 20260506 23:34:19.515 | INFO | "huge_shared" : " 27 GB", | 20260506 23:34:19.515 | INFO | "malloc" : " 14 GB", | 20260506 23:34:19.515 | INFO | "pinned" : " 29 GB" | 20260506 23:34:19.515 | INFO | } | 20260506 23:34:19.515 | INFO | Perf report for the last 5.14826 seconds: | 20260506 23:34:19.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:34:19.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:19.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:19.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:19.515 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 23:34:19.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:19.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:19.515 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 23:34:19.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:19.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:19.515 | INFO | Uploaded 36GiB to GPU in 1.95323s (18.431GiB/s) | 20260506 23:34:19.515 | INFO | Downloaded 34GiB to GPU in 1.99498s (17.0428GiB/s) | 20260506 23:34:24.656 | INFO | Analyzing chunk frames = [5336576,5337088) | 20260506 23:34:24.656 | INFO | MemInfo: { | 20260506 23:34:24.656 | INFO | "gpu" : " 17 GB", | 20260506 23:34:24.656 | INFO | "huge" : "", | 20260506 23:34:24.656 | INFO | "huge_shared" : " 27 GB", | 20260506 23:34:24.656 | INFO | "malloc" : " 14 GB", | 20260506 23:34:24.656 | INFO | "pinned" : " 29 GB" | 20260506 23:34:24.656 | INFO | } | 20260506 23:34:24.656 | INFO | Perf report for the last 5.14105 seconds: | 20260506 23:34:24.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:34:24.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:24.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:24.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:24.656 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:34:24.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:24.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:24.656 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 23:34:24.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:24.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:24.656 | INFO | Uploaded 36GiB to GPU in 1.96433s (18.3269GiB/s) | 20260506 23:34:24.656 | INFO | Downloaded 34GiB to GPU in 1.97699s (17.1978GiB/s) | 20260506 23:34:24.754 | INFO | Starting a round of Baz IO | 20260506 23:34:27.875 | INFO | Populating RTMetrics report | 20260506 23:34:27.876 | INFO | RTMetrics report populated | 20260506 23:34:27.876 | INFO | Opening tmp RTMetrics json file | 20260506 23:34:27.879 | INFO | Serializing RTMetrics to json | 20260506 23:34:27.924 | INFO | Serialization done, swapping tmp file now... | 20260506 23:34:27.967 | INFO | RTMetrics done | 20260506 23:34:28.030 | INFO | Finished a round of Baz IO: Wrote 1.875 GiB in 3.275 seconds | 20260506 23:34:28.030 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.44% / 0.00% / 66.56% | 20260506 23:34:28.030 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.403404% / 3.97364e-06% / 4.37101e-05% | 20260506 23:34:28.030 | INFO | Baz IO overall: | 20260506 23:34:28.030 | INFO | BytesWritten : 2792300807388 | 20260506 23:34:28.030 | INFO | NumEvents : 1346112180206 | 20260506 23:34:29.797 | INFO | Analyzing chunk frames = [5337088,5337600) | 20260506 23:34:29.797 | INFO | MemInfo: { | 20260506 23:34:29.797 | INFO | "gpu" : " 17 GB", | 20260506 23:34:29.797 | INFO | "huge" : "", | 20260506 23:34:29.797 | INFO | "huge_shared" : " 27 GB", | 20260506 23:34:29.797 | INFO | "malloc" : " 14 GB", | 20260506 23:34:29.797 | INFO | "pinned" : " 29 GB" | 20260506 23:34:29.797 | INFO | } | 20260506 23:34:29.797 | INFO | Perf report for the last 5.14109 seconds: | 20260506 23:34:29.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:34:29.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:29.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:29.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:29.797 | INFO | ANALYSIS : 62.1%, 2.9, 95.1ms, 0 | 20260506 23:34:29.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:29.797 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260506 23:34:29.797 | INFO | PRE_HQ : 59.9%, 1.8, 56.1ms, 0 | 20260506 23:34:29.797 | INFO | BAZWRITER : 63.4%, 4.4, 144.3ms, 0 | 20260506 23:34:29.797 | INFO | BAZ_SAMPLING : 1.9%, 1.0, 1.0ms, 0 | 20260506 23:34:29.797 | INFO | Uploaded 36GiB to GPU in 2.058s (17.4927GiB/s) | 20260506 23:34:29.798 | INFO | Downloaded 38GiB to GPU in 2.20505s (17.2332GiB/s) | 20260506 23:34:34.934 | INFO | Analyzing chunk frames = [5337600,5338112) | 20260506 23:34:34.934 | INFO | MemInfo: { | 20260506 23:34:34.934 | INFO | "gpu" : " 17 GB", | 20260506 23:34:34.934 | INFO | "huge" : "", | 20260506 23:34:34.934 | INFO | "huge_shared" : " 27 GB", | 20260506 23:34:34.934 | INFO | "malloc" : " 14 GB", | 20260506 23:34:34.934 | INFO | "pinned" : " 29 GB" | 20260506 23:34:34.934 | INFO | } | 20260506 23:34:34.934 | INFO | Perf report for the last 5.13733 seconds: | 20260506 23:34:34.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:34:34.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:34.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:34.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:34.935 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 23:34:34.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:34.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:34.935 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260506 23:34:34.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:34.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:34.935 | INFO | Uploaded 36GiB to GPU in 1.99354s (18.0583GiB/s) | 20260506 23:34:34.935 | INFO | Downloaded 34GiB to GPU in 1.92985s (17.6179GiB/s) | 20260506 23:34:40.068 | INFO | Analyzing chunk frames = [5338112,5338624) | 20260506 23:34:40.068 | INFO | MemInfo: { | 20260506 23:34:40.068 | INFO | "gpu" : " 17 GB", | 20260506 23:34:40.068 | INFO | "huge" : "", | 20260506 23:34:40.068 | INFO | "huge_shared" : " 27 GB", | 20260506 23:34:40.068 | INFO | "malloc" : " 14 GB", | 20260506 23:34:40.068 | INFO | "pinned" : " 29 GB" | 20260506 23:34:40.068 | INFO | } | 20260506 23:34:40.068 | INFO | Perf report for the last 5.13317 seconds: | 20260506 23:34:40.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:34:40.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:40.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:40.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:40.068 | INFO | ANALYSIS : 60.5%, 3.0, 93.5ms, 0 | 20260506 23:34:40.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:40.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:40.068 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:34:40.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:40.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:40.068 | INFO | Uploaded 36GiB to GPU in 1.96255s (18.3435GiB/s) | 20260506 23:34:40.068 | INFO | Downloaded 34GiB to GPU in 1.93265s (17.5924GiB/s) | 20260506 23:34:45.202 | INFO | Analyzing chunk frames = [5338624,5339136) | 20260506 23:34:45.202 | INFO | MemInfo: { | 20260506 23:34:45.202 | INFO | "gpu" : " 17 GB", | 20260506 23:34:45.202 | INFO | "huge" : "", | 20260506 23:34:45.202 | INFO | "huge_shared" : " 27 GB", | 20260506 23:34:45.202 | INFO | "malloc" : " 14 GB", | 20260506 23:34:45.202 | INFO | "pinned" : " 29 GB" | 20260506 23:34:45.202 | INFO | } | 20260506 23:34:45.202 | INFO | Perf report for the last 5.13436 seconds: | 20260506 23:34:45.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:34:45.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:45.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:45.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:45.202 | INFO | ANALYSIS : 59.6%, 3.0, 92.2ms, 0 | 20260506 23:34:45.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:45.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:45.202 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:34:45.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:45.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:45.202 | INFO | Uploaded 36GiB to GPU in 1.95863s (18.3802GiB/s) | 20260506 23:34:45.202 | INFO | Downloaded 34GiB to GPU in 1.88218s (18.0642GiB/s) | 20260506 23:34:50.342 | INFO | Analyzing chunk frames = [5339136,5339648) | 20260506 23:34:50.342 | INFO | MemInfo: { | 20260506 23:34:50.342 | INFO | "gpu" : " 17 GB", | 20260506 23:34:50.342 | INFO | "huge" : "", | 20260506 23:34:50.342 | INFO | "huge_shared" : " 27 GB", | 20260506 23:34:50.342 | INFO | "malloc" : " 14 GB", | 20260506 23:34:50.342 | INFO | "pinned" : " 29 GB" | 20260506 23:34:50.342 | INFO | } | 20260506 23:34:50.342 | INFO | Perf report for the last 5.14019 seconds: | 20260506 23:34:50.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:34:50.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:50.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:50.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:50.342 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 23:34:50.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:50.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:50.342 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:34:50.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:50.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:50.342 | INFO | Uploaded 36GiB to GPU in 1.9669s (18.3029GiB/s) | 20260506 23:34:50.343 | INFO | Downloaded 34GiB to GPU in 1.93045s (17.6125GiB/s) | 20260506 23:34:55.490 | INFO | Analyzing chunk frames = [5339648,5340160) | 20260506 23:34:55.490 | INFO | MemInfo: { | 20260506 23:34:55.490 | INFO | "gpu" : " 17 GB", | 20260506 23:34:55.490 | INFO | "huge" : "", | 20260506 23:34:55.490 | INFO | "huge_shared" : " 27 GB", | 20260506 23:34:55.490 | INFO | "malloc" : " 14 GB", | 20260506 23:34:55.490 | INFO | "pinned" : " 29 GB" | 20260506 23:34:55.490 | INFO | } | 20260506 23:34:55.490 | INFO | Perf report for the last 5.14763 seconds: | 20260506 23:34:55.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:34:55.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:55.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:55.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:55.490 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260506 23:34:55.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:55.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:55.490 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:34:55.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:55.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:34:55.490 | INFO | Uploaded 36GiB to GPU in 1.99872s (18.0115GiB/s) | 20260506 23:34:55.490 | INFO | Downloaded 34GiB to GPU in 1.9529s (17.41GiB/s) | 20260506 23:35:00.634 | INFO | Analyzing chunk frames = [5340160,5340672) | 20260506 23:35:00.634 | INFO | MemInfo: { | 20260506 23:35:00.634 | INFO | "gpu" : " 17 GB", | 20260506 23:35:00.634 | INFO | "huge" : "", | 20260506 23:35:00.634 | INFO | "huge_shared" : " 27 GB", | 20260506 23:35:00.634 | INFO | "malloc" : " 14 GB", | 20260506 23:35:00.634 | INFO | "pinned" : " 29 GB" | 20260506 23:35:00.634 | INFO | } | 20260506 23:35:00.634 | INFO | Perf report for the last 5.144 seconds: | 20260506 23:35:00.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:35:00.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:00.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:00.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:00.634 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:35:00.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:00.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:00.634 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:35:00.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:00.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:00.634 | INFO | Uploaded 36GiB to GPU in 1.97068s (18.2678GiB/s) | 20260506 23:35:00.634 | INFO | Downloaded 34GiB to GPU in 2s (17GiB/s) | 20260506 23:35:05.767 | INFO | Analyzing chunk frames = [5340672,5341184) | 20260506 23:35:05.767 | INFO | MemInfo: { | 20260506 23:35:05.767 | INFO | "gpu" : " 17 GB", | 20260506 23:35:05.767 | INFO | "huge" : "", | 20260506 23:35:05.767 | INFO | "huge_shared" : " 27 GB", | 20260506 23:35:05.767 | INFO | "malloc" : " 14 GB", | 20260506 23:35:05.767 | INFO | "pinned" : " 29 GB" | 20260506 23:35:05.767 | INFO | } | 20260506 23:35:05.767 | INFO | Perf report for the last 5.13326 seconds: | 20260506 23:35:05.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:35:05.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:05.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:05.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:05.767 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:35:05.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:05.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:05.767 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:35:05.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:05.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:05.767 | INFO | Uploaded 36GiB to GPU in 1.99099s (18.0815GiB/s) | 20260506 23:35:05.767 | INFO | Downloaded 34GiB to GPU in 1.9827s (17.1484GiB/s) | 20260506 23:35:05.768 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:35:05.768 | INFO | Total mem usage: 18 GiB | 20260506 23:35:05.768 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:35:05.768 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -20.6%) | 20260506 23:35:05.768 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:35:05.768 | INFO | High Water Breakdown: | 20260506 23:35:05.768 | INFO | 60.9% : 9.09 GiB ( 79.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:35:05.768 | INFO | 39% : 5.82 GiB ( 86.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:35:05.768 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:35:05.768 | INFO | ------------------------------------------ | 20260506 23:35:05.768 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:35:05.768 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:35:05.768 | INFO | Total mem usage: 26.5 GiB | 20260506 23:35:05.768 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:35:05.768 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260506 23:35:05.768 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260506 23:35:05.768 | INFO | High Water Breakdown: | 20260506 23:35:05.768 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:35:05.768 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:35:05.768 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:35:05.768 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:35:05.768 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:35:05.768 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:35:05.768 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:35:05.768 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:35:05.768 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:35:05.768 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:35:05.768 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:35:05.768 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:35:05.768 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:35:05.768 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:35:05.768 | INFO | ------------------------------------------ | 20260506 23:35:05.768 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:35:05.768 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:35:05.768 | INFO | Total mem usage: 25.6 GiB | 20260506 23:35:05.768 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:35:05.768 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:35:05.768 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:35:05.768 | INFO | High Water Breakdown: | 20260506 23:35:05.768 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:35:05.768 | INFO | ------------------------------------------ | 20260506 23:35:05.768 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:35:05.768 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:35:05.768 | INFO | Total mem usage: 15.7 GiB | 20260506 23:35:05.768 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:35:05.768 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:35:05.768 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 23:35:05.768 | INFO | High Water Breakdown: | 20260506 23:35:05.768 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:35:05.768 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:35:05.768 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:35:05.768 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:35:05.768 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:35:05.768 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:35:05.768 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:35:05.768 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:35:05.768 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:35:05.768 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:35:05.768 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:35:05.768 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:35:05.768 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:35:05.768 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:35:05.768 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:35:05.768 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:35:05.768 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:35:05.768 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:35:05.768 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:35:05.768 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:35:05.768 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:35:05.768 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:35:05.768 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:35:05.768 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:35:05.768 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:35:05.768 | INFO | ------------------------------------------ | 20260506 23:35:05.768 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:35:05.768 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:35:05.768 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:35:05.768 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:35:05.768 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.54e+01, 1.33e+01 / 2.22e+00 | 20260506 23:35:05.768 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.02e-01, 1.03e-02 / 5.63e-03 | 20260506 23:35:05.769 | INFO | PerfReport: 11.65%, Baseline -- 3.61e+00 / 4.34e+00, 5.40e-01 / 2.94e-01 | 20260506 23:35:05.769 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.45e+00, 1.79e-01 / 9.13e-02 | 20260506 23:35:05.769 | INFO | PerfReport: 16.06%, DME -- 3.98e+01 / 4.43e+01, 4.13e+00 / 2.77e+00 | 20260506 23:35:05.769 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.35e+00 / 1.87e+00 | 20260506 23:35:05.769 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.19e+00 / 2.75e+00, 3.79e-01 / 2.32e-01 | 20260506 23:35:05.769 | INFO | PerfReport: 3.18%, Metrics -- 9.84e-01 / 1.31e+00, 2.58e-01 / 1.32e-01 | 20260506 23:35:05.769 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.36e-01 / 3.84e-01, 5.21e-02 / 2.32e-02 | 20260506 23:35:05.769 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:35:05.769 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:35:05.769 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:35:05.769 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260506 23:35:05.769 | INFO | PerfReport: 21.50%, Upload -- 2.07e+01 / 0.00e+00, 6.73e+00 / 0.00e+00 | 20260506 23:35:05.769 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 23:35:05.769 | INFO | PerfReport: 32.20%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 23:35:05.769 | INFO | PerfReport: 1.17%, LockUpload -- 1.13e+00 / 0.00e+00, 4.67e+00 / 0.00e+00 | 20260506 23:35:05.769 | INFO | PerfReport: 1.39%, LockDownload -- 1.33e+00 / 0.00e+00, 3.33e+00 / 0.00e+00 | 20260506 23:35:05.769 | INFO | PerfReport: 22.35%, LockCompute -- 2.15e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 23:35:05.769 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.74e-02 / 0.00e+00 | 20260506 23:35:05.866 | INFO | Starting a round of Baz IO | 20260506 23:35:08.996 | INFO | Populating RTMetrics report | 20260506 23:35:08.996 | INFO | RTMetrics report populated | 20260506 23:35:08.996 | INFO | Opening tmp RTMetrics json file | 20260506 23:35:08.997 | INFO | Serializing RTMetrics to json | 20260506 23:35:08.998 | INFO | Serialization done, swapping tmp file now... | 20260506 23:35:08.999 | INFO | RTMetrics done | 20260506 23:35:09.104 | INFO | Finished a round of Baz IO: Wrote 1.874 GiB in 3.237 seconds | 20260506 23:35:09.104 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.44% / 0.00% / 66.56% | 20260506 23:35:09.104 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.403035% / 0% / 5.96046e-05% | 20260506 23:35:09.104 | INFO | Baz IO overall: | 20260506 23:35:09.104 | INFO | BytesWritten : 2794313355639 | 20260506 23:35:09.104 | INFO | NumEvents : 1347069629905 | 20260506 23:35:10.901 | INFO | Analyzing chunk frames = [5341184,5341696) | 20260506 23:35:10.901 | INFO | MemInfo: { | 20260506 23:35:10.901 | INFO | "gpu" : " 17 GB", | 20260506 23:35:10.901 | INFO | "huge" : "", | 20260506 23:35:10.901 | INFO | "huge_shared" : " 27 GB", | 20260506 23:35:10.901 | INFO | "malloc" : " 14 GB", | 20260506 23:35:10.901 | INFO | "pinned" : " 29 GB" | 20260506 23:35:10.901 | INFO | } | 20260506 23:35:10.901 | INFO | Perf report for the last 5.13384 seconds: | 20260506 23:35:10.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:35:10.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:10.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:10.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:10.901 | INFO | ANALYSIS : 62.3%, 2.9, 95.6ms, 0 | 20260506 23:35:10.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:10.901 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260506 23:35:10.901 | INFO | PRE_HQ : 59.8%, 1.8, 56.0ms, 0 | 20260506 23:35:10.901 | INFO | BAZWRITER : 62.9%, 4.6, 150.0ms, 0 | 20260506 23:35:10.901 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 23:35:10.901 | INFO | Uploaded 36GiB to GPU in 2.05294s (17.5358GiB/s) | 20260506 23:35:10.901 | INFO | Downloaded 38GiB to GPU in 2.19061s (17.3467GiB/s) | 20260506 23:35:16.040 | INFO | Analyzing chunk frames = [5341696,5342208) | 20260506 23:35:16.040 | INFO | MemInfo: { | 20260506 23:35:16.040 | INFO | "gpu" : " 17 GB", | 20260506 23:35:16.040 | INFO | "huge" : "", | 20260506 23:35:16.040 | INFO | "huge_shared" : " 27 GB", | 20260506 23:35:16.040 | INFO | "malloc" : " 14 GB", | 20260506 23:35:16.040 | INFO | "pinned" : " 29 GB" | 20260506 23:35:16.040 | INFO | } | 20260506 23:35:16.040 | INFO | Perf report for the last 5.1391 seconds: | 20260506 23:35:16.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:35:16.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:16.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:16.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:16.040 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:35:16.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:16.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:16.040 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:35:16.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:16.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:16.040 | INFO | Uploaded 36GiB to GPU in 2.02278s (17.7973GiB/s) | 20260506 23:35:16.040 | INFO | Downloaded 34GiB to GPU in 1.93055s (17.6115GiB/s) | 20260506 23:35:21.180 | INFO | Analyzing chunk frames = [5342208,5342720) | 20260506 23:35:21.180 | INFO | MemInfo: { | 20260506 23:35:21.180 | INFO | "gpu" : " 17 GB", | 20260506 23:35:21.180 | INFO | "huge" : "", | 20260506 23:35:21.180 | INFO | "huge_shared" : " 27 GB", | 20260506 23:35:21.180 | INFO | "malloc" : " 14 GB", | 20260506 23:35:21.180 | INFO | "pinned" : " 29 GB" | 20260506 23:35:21.180 | INFO | } | 20260506 23:35:21.180 | INFO | Perf report for the last 5.14009 seconds: | 20260506 23:35:21.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:35:21.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:21.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:21.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:21.180 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 23:35:21.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:21.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:21.180 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:35:21.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:21.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:21.180 | INFO | Uploaded 36GiB to GPU in 1.97308s (18.2456GiB/s) | 20260506 23:35:21.180 | INFO | Downloaded 34GiB to GPU in 1.92117s (17.6976GiB/s) | 20260506 23:35:26.320 | INFO | Analyzing chunk frames = [5342720,5343232) | 20260506 23:35:26.320 | INFO | MemInfo: { | 20260506 23:35:26.320 | INFO | "gpu" : " 17 GB", | 20260506 23:35:26.320 | INFO | "huge" : "", | 20260506 23:35:26.320 | INFO | "huge_shared" : " 27 GB", | 20260506 23:35:26.320 | INFO | "malloc" : " 14 GB", | 20260506 23:35:26.321 | INFO | "pinned" : " 29 GB" | 20260506 23:35:26.321 | INFO | } | 20260506 23:35:26.321 | INFO | Perf report for the last 5.14051 seconds: | 20260506 23:35:26.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:35:26.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:26.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:26.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:26.321 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 23:35:26.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:26.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:26.321 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:35:26.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:26.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:26.321 | INFO | Uploaded 36GiB to GPU in 1.97707s (18.2088GiB/s) | 20260506 23:35:26.321 | INFO | Downloaded 34GiB to GPU in 1.8788s (18.0967GiB/s) | 20260506 23:35:31.463 | INFO | Analyzing chunk frames = [5343232,5343744) | 20260506 23:35:31.463 | INFO | MemInfo: { | 20260506 23:35:31.463 | INFO | "gpu" : " 17 GB", | 20260506 23:35:31.463 | INFO | "huge" : "", | 20260506 23:35:31.463 | INFO | "huge_shared" : " 27 GB", | 20260506 23:35:31.463 | INFO | "malloc" : " 14 GB", | 20260506 23:35:31.463 | INFO | "pinned" : " 29 GB" | 20260506 23:35:31.463 | INFO | } | 20260506 23:35:31.463 | INFO | Perf report for the last 5.14226 seconds: | 20260506 23:35:31.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:35:31.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:31.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:31.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:31.463 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 23:35:31.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:31.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:31.463 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:35:31.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:31.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:31.463 | INFO | Uploaded 36GiB to GPU in 2.00803s (17.928GiB/s) | 20260506 23:35:31.463 | INFO | Downloaded 34GiB to GPU in 1.9273s (17.6413GiB/s) | 20260506 23:35:36.608 | INFO | Analyzing chunk frames = [5343744,5344256) | 20260506 23:35:36.608 | INFO | MemInfo: { | 20260506 23:35:36.608 | INFO | "gpu" : " 17 GB", | 20260506 23:35:36.608 | INFO | "huge" : "", | 20260506 23:35:36.608 | INFO | "huge_shared" : " 27 GB", | 20260506 23:35:36.608 | INFO | "malloc" : " 14 GB", | 20260506 23:35:36.608 | INFO | "pinned" : " 29 GB" | 20260506 23:35:36.608 | INFO | } | 20260506 23:35:36.608 | INFO | Perf report for the last 5.1455 seconds: | 20260506 23:35:36.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:35:36.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:36.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:36.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:36.609 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 23:35:36.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:36.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:36.609 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 23:35:36.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:36.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:36.609 | INFO | Uploaded 36GiB to GPU in 2.03768s (17.6672GiB/s) | 20260506 23:35:36.609 | INFO | Downloaded 34GiB to GPU in 1.93976s (17.5279GiB/s) | 20260506 23:35:41.746 | INFO | Analyzing chunk frames = [5344256,5344768) | 20260506 23:35:41.746 | INFO | MemInfo: { | 20260506 23:35:41.746 | INFO | "gpu" : " 17 GB", | 20260506 23:35:41.746 | INFO | "huge" : "", | 20260506 23:35:41.746 | INFO | "huge_shared" : " 27 GB", | 20260506 23:35:41.746 | INFO | "malloc" : " 14 GB", | 20260506 23:35:41.746 | INFO | "pinned" : " 29 GB" | 20260506 23:35:41.746 | INFO | } | 20260506 23:35:41.746 | INFO | Perf report for the last 5.1379 seconds: | 20260506 23:35:41.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:35:41.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:41.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:41.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:41.746 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:35:41.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:41.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:41.747 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:35:41.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:41.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:41.747 | INFO | Uploaded 36GiB to GPU in 1.99026s (18.0881GiB/s) | 20260506 23:35:41.747 | INFO | Downloaded 34GiB to GPU in 1.99185s (17.0696GiB/s) | 20260506 23:35:46.884 | INFO | Analyzing chunk frames = [5344768,5345280) | 20260506 23:35:46.884 | INFO | MemInfo: { | 20260506 23:35:46.884 | INFO | "gpu" : " 17 GB", | 20260506 23:35:46.884 | INFO | "huge" : "", | 20260506 23:35:46.884 | INFO | "huge_shared" : " 27 GB", | 20260506 23:35:46.884 | INFO | "malloc" : " 14 GB", | 20260506 23:35:46.884 | INFO | "pinned" : " 29 GB" | 20260506 23:35:46.884 | INFO | } | 20260506 23:35:46.884 | INFO | Perf report for the last 5.13773 seconds: | 20260506 23:35:46.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:35:46.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:46.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:46.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:46.884 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:35:46.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:46.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:46.884 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 23:35:46.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:46.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:46.884 | INFO | Uploaded 36GiB to GPU in 1.98585s (18.1282GiB/s) | 20260506 23:35:46.884 | INFO | Downloaded 34GiB to GPU in 1.99652s (17.0296GiB/s) | 20260506 23:35:46.999 | INFO | Starting a round of Baz IO | 20260506 23:35:50.108 | INFO | Populating RTMetrics report | 20260506 23:35:50.108 | INFO | RTMetrics report populated | 20260506 23:35:50.108 | INFO | Opening tmp RTMetrics json file | 20260506 23:35:50.109 | INFO | Serializing RTMetrics to json | 20260506 23:35:50.118 | INFO | Serialization done, swapping tmp file now... | 20260506 23:35:50.132 | INFO | RTMetrics done | 20260506 23:35:50.230 | INFO | Finished a round of Baz IO: Wrote 1.875 GiB in 3.23 seconds | 20260506 23:35:50.230 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.44% / 0.00% / 66.56% | 20260506 23:35:50.230 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.402482% / 3.97364e-06% / 6.75519e-05% | 20260506 23:35:50.230 | INFO | Baz IO overall: | 20260506 23:35:50.230 | INFO | BytesWritten : 2796327137409 | 20260506 23:35:50.230 | INFO | NumEvents : 1348028791772 | 20260506 23:35:52.016 | INFO | Analyzing chunk frames = [5345280,5345792) | 20260506 23:35:52.016 | INFO | MemInfo: { | 20260506 23:35:52.016 | INFO | "gpu" : " 17 GB", | 20260506 23:35:52.016 | INFO | "huge" : "", | 20260506 23:35:52.016 | INFO | "huge_shared" : " 27 GB", | 20260506 23:35:52.016 | INFO | "malloc" : " 14 GB", | 20260506 23:35:52.016 | INFO | "pinned" : " 29 GB" | 20260506 23:35:52.016 | INFO | } | 20260506 23:35:52.016 | INFO | Perf report for the last 5.13207 seconds: | 20260506 23:35:52.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:35:52.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:52.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:52.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:52.016 | INFO | ANALYSIS : 62.3%, 2.9, 95.9ms, 0 | 20260506 23:35:52.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:52.016 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 23:35:52.016 | INFO | PRE_HQ : 60.0%, 1.8, 56.6ms, 0 | 20260506 23:35:52.016 | INFO | BAZWRITER : 63.0%, 4.0, 129.2ms, 0 | 20260506 23:35:52.016 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:35:52.016 | INFO | Uploaded 36GiB to GPU in 2.0242s (17.7848GiB/s) | 20260506 23:35:52.017 | INFO | Downloaded 38GiB to GPU in 2.18568s (17.3859GiB/s) | 20260506 23:35:57.152 | INFO | Analyzing chunk frames = [5345792,5346304) | 20260506 23:35:57.152 | INFO | MemInfo: { | 20260506 23:35:57.152 | INFO | "gpu" : " 17 GB", | 20260506 23:35:57.152 | INFO | "huge" : "", | 20260506 23:35:57.152 | INFO | "huge_shared" : " 27 GB", | 20260506 23:35:57.152 | INFO | "malloc" : " 14 GB", | 20260506 23:35:57.152 | INFO | "pinned" : " 29 GB" | 20260506 23:35:57.152 | INFO | } | 20260506 23:35:57.152 | INFO | Perf report for the last 5.13596 seconds: | 20260506 23:35:57.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:35:57.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:57.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:57.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:57.152 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:35:57.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:57.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:57.152 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:35:57.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:57.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:35:57.152 | INFO | Uploaded 36GiB to GPU in 2.00591s (17.947GiB/s) | 20260506 23:35:57.152 | INFO | Downloaded 34GiB to GPU in 1.93303s (17.5889GiB/s) | 20260506 23:36:02.297 | INFO | Analyzing chunk frames = [5346304,5346816) | 20260506 23:36:02.297 | INFO | MemInfo: { | 20260506 23:36:02.297 | INFO | "gpu" : " 17 GB", | 20260506 23:36:02.297 | INFO | "huge" : "", | 20260506 23:36:02.297 | INFO | "huge_shared" : " 27 GB", | 20260506 23:36:02.297 | INFO | "malloc" : " 14 GB", | 20260506 23:36:02.297 | INFO | "pinned" : " 29 GB" | 20260506 23:36:02.298 | INFO | } | 20260506 23:36:02.298 | INFO | Perf report for the last 5.14549 seconds: | 20260506 23:36:02.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:36:02.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:02.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:02.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:02.298 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260506 23:36:02.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:02.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:02.298 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:36:02.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:02.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:02.298 | INFO | Uploaded 36GiB to GPU in 1.95287s (18.4344GiB/s) | 20260506 23:36:02.298 | INFO | Downloaded 34GiB to GPU in 1.9062s (17.8365GiB/s) | 20260506 23:36:07.440 | INFO | Analyzing chunk frames = [5346816,5347328) | 20260506 23:36:07.440 | INFO | MemInfo: { | 20260506 23:36:07.440 | INFO | "gpu" : " 17 GB", | 20260506 23:36:07.440 | INFO | "huge" : "", | 20260506 23:36:07.440 | INFO | "huge_shared" : " 27 GB", | 20260506 23:36:07.440 | INFO | "malloc" : " 14 GB", | 20260506 23:36:07.440 | INFO | "pinned" : " 29 GB" | 20260506 23:36:07.440 | INFO | } | 20260506 23:36:07.440 | INFO | Perf report for the last 5.14246 seconds: | 20260506 23:36:07.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:36:07.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:07.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:07.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:07.440 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260506 23:36:07.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:07.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:07.440 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:36:07.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:07.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:07.440 | INFO | Uploaded 36GiB to GPU in 2.01323s (17.8817GiB/s) | 20260506 23:36:07.440 | INFO | Downloaded 34GiB to GPU in 1.91098s (17.792GiB/s) | 20260506 23:36:12.576 | INFO | Analyzing chunk frames = [5347328,5347840) | 20260506 23:36:12.576 | INFO | MemInfo: { | 20260506 23:36:12.576 | INFO | "gpu" : " 17 GB", | 20260506 23:36:12.576 | INFO | "huge" : "", | 20260506 23:36:12.576 | INFO | "huge_shared" : " 27 GB", | 20260506 23:36:12.576 | INFO | "malloc" : " 14 GB", | 20260506 23:36:12.576 | INFO | "pinned" : " 29 GB" | 20260506 23:36:12.576 | INFO | } | 20260506 23:36:12.576 | INFO | Perf report for the last 5.13616 seconds: | 20260506 23:36:12.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:36:12.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:12.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:12.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:12.576 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 23:36:12.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:12.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:12.576 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:36:12.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:12.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:12.577 | INFO | Uploaded 36GiB to GPU in 1.99154s (18.0764GiB/s) | 20260506 23:36:12.577 | INFO | Downloaded 34GiB to GPU in 1.93292s (17.59GiB/s) | 20260506 23:36:17.715 | INFO | Analyzing chunk frames = [5347840,5348352) | 20260506 23:36:17.715 | INFO | MemInfo: { | 20260506 23:36:17.715 | INFO | "gpu" : " 17 GB", | 20260506 23:36:17.715 | INFO | "huge" : "", | 20260506 23:36:17.715 | INFO | "huge_shared" : " 27 GB", | 20260506 23:36:17.715 | INFO | "malloc" : " 14 GB", | 20260506 23:36:17.715 | INFO | "pinned" : " 29 GB" | 20260506 23:36:17.715 | INFO | } | 20260506 23:36:17.716 | INFO | Perf report for the last 5.13915 seconds: | 20260506 23:36:17.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:36:17.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:17.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:17.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:17.716 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 23:36:17.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:17.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:17.716 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 23:36:17.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:17.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:17.716 | INFO | Uploaded 36GiB to GPU in 2.00624s (17.944GiB/s) | 20260506 23:36:17.716 | INFO | Downloaded 34GiB to GPU in 1.94224s (17.5055GiB/s) | 20260506 23:36:22.859 | INFO | Analyzing chunk frames = [5348352,5348864) | 20260506 23:36:22.859 | INFO | MemInfo: { | 20260506 23:36:22.859 | INFO | "gpu" : " 17 GB", | 20260506 23:36:22.859 | INFO | "huge" : "", | 20260506 23:36:22.859 | INFO | "huge_shared" : " 27 GB", | 20260506 23:36:22.859 | INFO | "malloc" : " 14 GB", | 20260506 23:36:22.859 | INFO | "pinned" : " 29 GB" | 20260506 23:36:22.859 | INFO | } | 20260506 23:36:22.859 | INFO | Perf report for the last 5.14369 seconds: | 20260506 23:36:22.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:36:22.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:22.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:22.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:22.859 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 23:36:22.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:22.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:22.859 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:36:22.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:22.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:22.859 | INFO | Uploaded 36GiB to GPU in 1.95875s (18.3791GiB/s) | 20260506 23:36:22.859 | INFO | Downloaded 34GiB to GPU in 1.97325s (17.2304GiB/s) | 20260506 23:36:27.993 | INFO | Analyzing chunk frames = [5348864,5349376) | 20260506 23:36:27.993 | INFO | MemInfo: { | 20260506 23:36:27.993 | INFO | "gpu" : " 17 GB", | 20260506 23:36:27.993 | INFO | "huge" : "", | 20260506 23:36:27.993 | INFO | "huge_shared" : " 27 GB", | 20260506 23:36:27.993 | INFO | "malloc" : " 14 GB", | 20260506 23:36:27.993 | INFO | "pinned" : " 29 GB" | 20260506 23:36:27.993 | INFO | } | 20260506 23:36:27.993 | INFO | Perf report for the last 5.13415 seconds: | 20260506 23:36:27.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:36:27.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:27.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:27.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:27.993 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:36:27.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:27.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:27.993 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260506 23:36:27.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:27.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:27.994 | INFO | Uploaded 36GiB to GPU in 1.97944s (18.187GiB/s) | 20260506 23:36:27.994 | INFO | Downloaded 34GiB to GPU in 1.99s (17.0854GiB/s) | 20260506 23:36:27.994 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:36:27.994 | INFO | Total mem usage: 18 GiB | 20260506 23:36:27.994 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:36:27.994 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -16.6%) | 20260506 23:36:27.994 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:36:27.994 | INFO | High Water Breakdown: | 20260506 23:36:27.994 | INFO | 61.4% : 9.47 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:36:27.994 | INFO | 38.5% : 5.95 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:36:27.994 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:36:27.994 | INFO | ------------------------------------------ | 20260506 23:36:27.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:36:27.994 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:36:27.994 | INFO | Total mem usage: 26.5 GiB | 20260506 23:36:27.994 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:36:27.994 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:36:27.994 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:36:27.994 | INFO | High Water Breakdown: | 20260506 23:36:27.994 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:36:27.994 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:36:27.994 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:36:27.994 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:36:27.994 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:36:27.994 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:36:27.994 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:36:27.994 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:36:27.994 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:36:27.994 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:36:27.994 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:36:27.994 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:36:27.994 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:36:27.994 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:36:27.994 | INFO | ------------------------------------------ | 20260506 23:36:27.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:36:27.994 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:36:27.994 | INFO | Total mem usage: 25.6 GiB | 20260506 23:36:27.994 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:36:27.994 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:36:27.994 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:36:27.994 | INFO | High Water Breakdown: | 20260506 23:36:27.994 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:36:27.994 | INFO | ------------------------------------------ | 20260506 23:36:27.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:36:27.994 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:36:27.994 | INFO | Total mem usage: 15.7 GiB | 20260506 23:36:27.994 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:36:27.994 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:36:27.994 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:36:27.994 | INFO | High Water Breakdown: | 20260506 23:36:27.994 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:36:27.994 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:36:27.994 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:36:27.994 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:36:27.994 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:36:27.994 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:36:27.995 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:36:27.995 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:36:27.995 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:36:27.995 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:36:27.995 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:36:27.995 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:36:27.995 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:36:27.995 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:36:27.995 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:36:27.995 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:36:27.995 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:36:27.995 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:36:27.995 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:36:27.995 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:36:27.995 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:36:27.995 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:36:27.995 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:36:27.995 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:36:27.995 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:36:27.995 | INFO | ------------------------------------------ | 20260506 23:36:27.995 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:36:27.995 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:36:27.995 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:36:27.995 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:36:27.995 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.60e+01, 1.34e+01 / 2.16e+00 | 20260506 23:36:27.995 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.00e-01, 9.60e-03 / 5.70e-03 | 20260506 23:36:27.995 | INFO | PerfReport: 11.67%, Baseline -- 3.62e+00 / 4.38e+00, 5.48e-01 / 3.67e-01 | 20260506 23:36:27.995 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.81e-01 / 1.38e-01 | 20260506 23:36:27.995 | INFO | PerfReport: 16.06%, DME -- 3.98e+01 / 4.47e+01, 4.21e+00 / 2.41e+00 | 20260506 23:36:27.995 | INFO | PerfReport: 57.39%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.38e+00 / 1.76e+00 | 20260506 23:36:27.995 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.18e+00 / 2.78e+00, 3.73e-01 / 1.81e-01 | 20260506 23:36:27.995 | INFO | PerfReport: 3.18%, Metrics -- 9.88e-01 / 1.32e+00, 2.62e-01 / 1.68e-01 | 20260506 23:36:27.995 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.36e-01 / 3.85e-01, 5.27e-02 / 2.03e-02 | 20260506 23:36:27.995 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:36:27.995 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:36:27.995 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:36:27.995 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 23:36:27.995 | INFO | PerfReport: 21.58%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 23:36:27.995 | INFO | PerfReport: 21.29%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260506 23:36:27.995 | INFO | PerfReport: 32.14%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 23:36:27.995 | INFO | PerfReport: 1.31%, LockUpload -- 1.26e+00 / 0.00e+00, 4.79e+00 / 0.00e+00 | 20260506 23:36:27.995 | INFO | PerfReport: 1.18%, LockDownload -- 1.13e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 23:36:27.995 | INFO | PerfReport: 22.49%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:36:27.995 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260506 23:36:28.092 | INFO | Starting a round of Baz IO | 20260506 23:36:31.253 | INFO | Populating RTMetrics report | 20260506 23:36:31.253 | INFO | RTMetrics report populated | 20260506 23:36:31.253 | INFO | Opening tmp RTMetrics json file | 20260506 23:36:31.301 | INFO | Serializing RTMetrics to json | 20260506 23:36:31.339 | INFO | Serialization done, swapping tmp file now... | 20260506 23:36:31.342 | INFO | RTMetrics done | 20260506 23:36:31.374 | INFO | Finished a round of Baz IO: Wrote 1.876 GiB in 3.281 seconds | 20260506 23:36:31.374 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.44% / 0.00% / 66.56% | 20260506 23:36:31.374 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.402864% / 3.97364e-06% / 5.5631e-05% | 20260506 23:36:31.374 | INFO | Baz IO overall: | 20260506 23:36:31.374 | INFO | BytesWritten : 2798341276603 | 20260506 23:36:31.374 | INFO | NumEvents : 1348988142793 | 20260506 23:36:33.134 | INFO | Analyzing chunk frames = [5349376,5349888) | 20260506 23:36:33.134 | INFO | MemInfo: { | 20260506 23:36:33.134 | INFO | "gpu" : " 17 GB", | 20260506 23:36:33.134 | INFO | "huge" : "", | 20260506 23:36:33.134 | INFO | "huge_shared" : " 27 GB", | 20260506 23:36:33.134 | INFO | "malloc" : " 14 GB", | 20260506 23:36:33.134 | INFO | "pinned" : " 29 GB" | 20260506 23:36:33.134 | INFO | } | 20260506 23:36:33.134 | INFO | Perf report for the last 5.14043 seconds: | 20260506 23:36:33.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:36:33.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:33.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:33.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:33.134 | INFO | ANALYSIS : 62.5%, 2.9, 94.1ms, 0 | 20260506 23:36:33.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:33.134 | INFO | RT_METRICS : 46.9%, 1.0, 24.6ms, 0 | 20260506 23:36:33.134 | INFO | PRE_HQ : 60.7%, 1.9, 60.7ms, 0 | 20260506 23:36:33.134 | INFO | BAZWRITER : 63.5%, 4.4, 142.8ms, 0 | 20260506 23:36:33.134 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260506 23:36:33.134 | INFO | Uploaded 36GiB to GPU in 2.04779s (17.5799GiB/s) | 20260506 23:36:33.134 | INFO | Downloaded 38GiB to GPU in 2.16099s (17.5845GiB/s) | 20260506 23:36:38.272 | INFO | Analyzing chunk frames = [5349888,5350400) | 20260506 23:36:38.272 | INFO | MemInfo: { | 20260506 23:36:38.272 | INFO | "gpu" : " 17 GB", | 20260506 23:36:38.272 | INFO | "huge" : "", | 20260506 23:36:38.272 | INFO | "huge_shared" : " 27 GB", | 20260506 23:36:38.273 | INFO | "malloc" : " 14 GB", | 20260506 23:36:38.273 | INFO | "pinned" : " 29 GB" | 20260506 23:36:38.273 | INFO | } | 20260506 23:36:38.273 | INFO | Perf report for the last 5.13891 seconds: | 20260506 23:36:38.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:36:38.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:38.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:38.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:38.273 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:36:38.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:38.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:38.273 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 23:36:38.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:38.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:38.273 | INFO | Uploaded 36GiB to GPU in 2.01598s (17.8574GiB/s) | 20260506 23:36:38.273 | INFO | Downloaded 34GiB to GPU in 1.92932s (17.6228GiB/s) | 20260506 23:36:43.410 | INFO | Analyzing chunk frames = [5350400,5350912) | 20260506 23:36:43.410 | INFO | MemInfo: { | 20260506 23:36:43.410 | INFO | "gpu" : " 17 GB", | 20260506 23:36:43.410 | INFO | "huge" : "", | 20260506 23:36:43.410 | INFO | "huge_shared" : " 27 GB", | 20260506 23:36:43.410 | INFO | "malloc" : " 14 GB", | 20260506 23:36:43.410 | INFO | "pinned" : " 29 GB" | 20260506 23:36:43.410 | INFO | } | 20260506 23:36:43.410 | INFO | Perf report for the last 5.13751 seconds: | 20260506 23:36:43.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:36:43.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:43.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:43.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:43.410 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:36:43.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:43.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:43.410 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:36:43.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:43.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:43.411 | INFO | Uploaded 36GiB to GPU in 1.98102s (18.1724GiB/s) | 20260506 23:36:43.411 | INFO | Downloaded 34GiB to GPU in 1.92709s (17.6432GiB/s) | 20260506 23:36:48.559 | INFO | Analyzing chunk frames = [5350912,5351424) | 20260506 23:36:48.559 | INFO | MemInfo: { | 20260506 23:36:48.559 | INFO | "gpu" : " 17 GB", | 20260506 23:36:48.559 | INFO | "huge" : "", | 20260506 23:36:48.559 | INFO | "huge_shared" : " 27 GB", | 20260506 23:36:48.559 | INFO | "malloc" : " 14 GB", | 20260506 23:36:48.559 | INFO | "pinned" : " 29 GB" | 20260506 23:36:48.559 | INFO | } | 20260506 23:36:48.559 | INFO | Perf report for the last 5.14869 seconds: | 20260506 23:36:48.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:36:48.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:48.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:48.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:48.559 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260506 23:36:48.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:48.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:48.559 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 23:36:48.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:48.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:48.559 | INFO | Uploaded 36GiB to GPU in 2.01429s (17.8723GiB/s) | 20260506 23:36:48.559 | INFO | Downloaded 34GiB to GPU in 1.90928s (17.8077GiB/s) | 20260506 23:36:53.703 | INFO | Analyzing chunk frames = [5351424,5351936) | 20260506 23:36:53.703 | INFO | MemInfo: { | 20260506 23:36:53.703 | INFO | "gpu" : " 17 GB", | 20260506 23:36:53.703 | INFO | "huge" : "", | 20260506 23:36:53.703 | INFO | "huge_shared" : " 27 GB", | 20260506 23:36:53.703 | INFO | "malloc" : " 14 GB", | 20260506 23:36:53.703 | INFO | "pinned" : " 29 GB" | 20260506 23:36:53.703 | INFO | } | 20260506 23:36:53.703 | INFO | Perf report for the last 5.14419 seconds: | 20260506 23:36:53.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:36:53.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:53.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:53.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:53.703 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 23:36:53.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:53.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:53.703 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 23:36:53.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:53.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:53.703 | INFO | Uploaded 36GiB to GPU in 2.00838s (17.9249GiB/s) | 20260506 23:36:53.704 | INFO | Downloaded 34GiB to GPU in 1.94305s (17.4983GiB/s) | 20260506 23:36:58.835 | INFO | Analyzing chunk frames = [5351936,5352448) | 20260506 23:36:58.835 | INFO | MemInfo: { | 20260506 23:36:58.835 | INFO | "gpu" : " 17 GB", | 20260506 23:36:58.835 | INFO | "huge" : "", | 20260506 23:36:58.835 | INFO | "huge_shared" : " 27 GB", | 20260506 23:36:58.835 | INFO | "malloc" : " 14 GB", | 20260506 23:36:58.835 | INFO | "pinned" : " 29 GB" | 20260506 23:36:58.835 | INFO | } | 20260506 23:36:58.835 | INFO | Perf report for the last 5.13171 seconds: | 20260506 23:36:58.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:36:58.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:58.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:58.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:58.835 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 23:36:58.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:58.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:58.835 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:36:58.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:58.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:36:58.835 | INFO | Uploaded 36GiB to GPU in 2.04195s (17.6302GiB/s) | 20260506 23:36:58.835 | INFO | Downloaded 34GiB to GPU in 1.9454s (17.4771GiB/s) | 20260506 23:37:03.971 | INFO | Analyzing chunk frames = [5352448,5352960) | 20260506 23:37:03.971 | INFO | MemInfo: { | 20260506 23:37:03.971 | INFO | "gpu" : " 17 GB", | 20260506 23:37:03.971 | INFO | "huge" : "", | 20260506 23:37:03.971 | INFO | "huge_shared" : " 27 GB", | 20260506 23:37:03.971 | INFO | "malloc" : " 14 GB", | 20260506 23:37:03.971 | INFO | "pinned" : " 29 GB" | 20260506 23:37:03.971 | INFO | } | 20260506 23:37:03.971 | INFO | Perf report for the last 5.13625 seconds: | 20260506 23:37:03.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:37:03.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:03.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:03.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:03.972 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:37:03.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:03.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:03.972 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 23:37:03.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:03.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:03.972 | INFO | Uploaded 36GiB to GPU in 1.97585s (18.22GiB/s) | 20260506 23:37:03.972 | INFO | Downloaded 34GiB to GPU in 2.00446s (16.9622GiB/s) | 20260506 23:37:09.107 | INFO | Analyzing chunk frames = [5352960,5353472) | 20260506 23:37:09.107 | INFO | MemInfo: { | 20260506 23:37:09.107 | INFO | "gpu" : " 17 GB", | 20260506 23:37:09.107 | INFO | "huge" : "", | 20260506 23:37:09.107 | INFO | "huge_shared" : " 27 GB", | 20260506 23:37:09.107 | INFO | "malloc" : " 14 GB", | 20260506 23:37:09.107 | INFO | "pinned" : " 29 GB" | 20260506 23:37:09.108 | INFO | } | 20260506 23:37:09.108 | INFO | Perf report for the last 5.13651 seconds: | 20260506 23:37:09.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:37:09.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:09.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:09.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:09.108 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260506 23:37:09.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:09.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:09.108 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 23:37:09.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:09.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:09.108 | INFO | Uploaded 36GiB to GPU in 2.03772s (17.6668GiB/s) | 20260506 23:37:09.108 | INFO | Downloaded 34GiB to GPU in 1.99385s (17.0524GiB/s) | 20260506 23:37:09.208 | INFO | Starting a round of Baz IO | 20260506 23:37:12.338 | INFO | Populating RTMetrics report | 20260506 23:37:12.339 | INFO | RTMetrics report populated | 20260506 23:37:12.339 | INFO | Opening tmp RTMetrics json file | 20260506 23:37:12.339 | INFO | Serializing RTMetrics to json | 20260506 23:37:12.340 | INFO | Serialization done, swapping tmp file now... | 20260506 23:37:12.341 | INFO | RTMetrics done | 20260506 23:37:12.430 | INFO | Finished a round of Baz IO: Wrote 1.878 GiB in 3.222 seconds | 20260506 23:37:12.430 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.43% / 0.00% / 66.56% | 20260506 23:37:12.430 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.404191% / 3.97364e-06% / 5.16574e-05% | 20260506 23:37:12.430 | INFO | Baz IO overall: | 20260506 23:37:12.430 | INFO | BytesWritten : 2800357296943 | 20260506 23:37:12.431 | INFO | NumEvents : 1349948517627 | 20260506 23:37:14.247 | INFO | Analyzing chunk frames = [5353472,5353984) | 20260506 23:37:14.247 | INFO | MemInfo: { | 20260506 23:37:14.247 | INFO | "gpu" : " 17 GB", | 20260506 23:37:14.247 | INFO | "huge" : "", | 20260506 23:37:14.247 | INFO | "huge_shared" : " 27 GB", | 20260506 23:37:14.247 | INFO | "malloc" : " 14 GB", | 20260506 23:37:14.247 | INFO | "pinned" : " 29 GB" | 20260506 23:37:14.247 | INFO | } | 20260506 23:37:14.247 | INFO | Perf report for the last 5.1396 seconds: | 20260506 23:37:14.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:37:14.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:14.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:14.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:14.247 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 23:37:14.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:14.247 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260506 23:37:14.247 | INFO | PRE_HQ : 59.8%, 1.8, 56.9ms, 0 | 20260506 23:37:14.247 | INFO | BAZWRITER : 62.3%, 3.4, 108.7ms, 0 | 20260506 23:37:14.247 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:37:14.248 | INFO | Uploaded 36GiB to GPU in 2.03761s (17.6677GiB/s) | 20260506 23:37:14.248 | INFO | Downloaded 38GiB to GPU in 2.16799s (17.5277GiB/s) | 20260506 23:37:19.390 | INFO | Analyzing chunk frames = [5353984,5354496) | 20260506 23:37:19.390 | INFO | MemInfo: { | 20260506 23:37:19.390 | INFO | "gpu" : " 17 GB", | 20260506 23:37:19.390 | INFO | "huge" : "", | 20260506 23:37:19.390 | INFO | "huge_shared" : " 27 GB", | 20260506 23:37:19.390 | INFO | "malloc" : " 14 GB", | 20260506 23:37:19.390 | INFO | "pinned" : " 29 GB" | 20260506 23:37:19.390 | INFO | } | 20260506 23:37:19.390 | INFO | Perf report for the last 5.14298 seconds: | 20260506 23:37:19.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:37:19.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:19.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:19.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:19.390 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 23:37:19.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:19.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:19.390 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:37:19.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:19.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:19.390 | INFO | Uploaded 36GiB to GPU in 2.01825s (17.8372GiB/s) | 20260506 23:37:19.391 | INFO | Downloaded 34GiB to GPU in 1.92739s (17.6405GiB/s) | 20260506 23:37:24.526 | INFO | Analyzing chunk frames = [5354496,5355008) | 20260506 23:37:24.526 | INFO | MemInfo: { | 20260506 23:37:24.526 | INFO | "gpu" : " 17 GB", | 20260506 23:37:24.526 | INFO | "huge" : "", | 20260506 23:37:24.526 | INFO | "huge_shared" : " 27 GB", | 20260506 23:37:24.526 | INFO | "malloc" : " 14 GB", | 20260506 23:37:24.526 | INFO | "pinned" : " 29 GB" | 20260506 23:37:24.526 | INFO | } | 20260506 23:37:24.526 | INFO | Perf report for the last 5.13561 seconds: | 20260506 23:37:24.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:37:24.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:24.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:24.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:24.526 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:37:24.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:24.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:24.526 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:37:24.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:24.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:24.526 | INFO | Uploaded 36GiB to GPU in 1.9927s (18.066GiB/s) | 20260506 23:37:24.526 | INFO | Downloaded 34GiB to GPU in 1.93061s (17.611GiB/s) | 20260506 23:37:29.666 | INFO | Analyzing chunk frames = [5355008,5355520) | 20260506 23:37:29.666 | INFO | MemInfo: { | 20260506 23:37:29.666 | INFO | "gpu" : " 17 GB", | 20260506 23:37:29.666 | INFO | "huge" : "", | 20260506 23:37:29.666 | INFO | "huge_shared" : " 27 GB", | 20260506 23:37:29.666 | INFO | "malloc" : " 14 GB", | 20260506 23:37:29.667 | INFO | "pinned" : " 29 GB" | 20260506 23:37:29.667 | INFO | } | 20260506 23:37:29.667 | INFO | Perf report for the last 5.14069 seconds: | 20260506 23:37:29.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:37:29.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:29.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:29.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:29.667 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 23:37:29.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:29.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:29.667 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:37:29.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:29.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:29.667 | INFO | Uploaded 36GiB to GPU in 1.95301s (18.4331GiB/s) | 20260506 23:37:29.667 | INFO | Downloaded 34GiB to GPU in 1.8744s (18.1392GiB/s) | 20260506 23:37:34.808 | INFO | Analyzing chunk frames = [5355520,5356032) | 20260506 23:37:34.808 | INFO | MemInfo: { | 20260506 23:37:34.808 | INFO | "gpu" : " 17 GB", | 20260506 23:37:34.808 | INFO | "huge" : "", | 20260506 23:37:34.808 | INFO | "huge_shared" : " 27 GB", | 20260506 23:37:34.808 | INFO | "malloc" : " 14 GB", | 20260506 23:37:34.808 | INFO | "pinned" : " 29 GB" | 20260506 23:37:34.808 | INFO | } | 20260506 23:37:34.808 | INFO | Perf report for the last 5.14184 seconds: | 20260506 23:37:34.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:37:34.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:34.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:34.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:34.808 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260506 23:37:34.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:34.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:34.809 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:37:34.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:34.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:34.809 | INFO | Uploaded 36GiB to GPU in 2.00203s (17.9817GiB/s) | 20260506 23:37:34.809 | INFO | Downloaded 34GiB to GPU in 1.91579s (17.7473GiB/s) | 20260506 23:37:39.950 | INFO | Analyzing chunk frames = [5356032,5356544) | 20260506 23:37:39.950 | INFO | MemInfo: { | 20260506 23:37:39.950 | INFO | "gpu" : " 17 GB", | 20260506 23:37:39.950 | INFO | "huge" : "", | 20260506 23:37:39.950 | INFO | "huge_shared" : " 27 GB", | 20260506 23:37:39.950 | INFO | "malloc" : " 14 GB", | 20260506 23:37:39.950 | INFO | "pinned" : " 29 GB" | 20260506 23:37:39.950 | INFO | } | 20260506 23:37:39.950 | INFO | Perf report for the last 5.14173 seconds: | 20260506 23:37:39.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:37:39.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:39.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:39.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:39.950 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 23:37:39.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:39.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:39.950 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:37:39.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:39.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:39.950 | INFO | Uploaded 36GiB to GPU in 2.02798s (17.7517GiB/s) | 20260506 23:37:39.950 | INFO | Downloaded 34GiB to GPU in 1.93363s (17.5835GiB/s) | 20260506 23:37:45.085 | INFO | Analyzing chunk frames = [5356544,5357056) | 20260506 23:37:45.085 | INFO | MemInfo: { | 20260506 23:37:45.085 | INFO | "gpu" : " 17 GB", | 20260506 23:37:45.085 | INFO | "huge" : "", | 20260506 23:37:45.085 | INFO | "huge_shared" : " 27 GB", | 20260506 23:37:45.085 | INFO | "malloc" : " 14 GB", | 20260506 23:37:45.085 | INFO | "pinned" : " 29 GB" | 20260506 23:37:45.085 | INFO | } | 20260506 23:37:45.085 | INFO | Perf report for the last 5.1349 seconds: | 20260506 23:37:45.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:37:45.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:45.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:45.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:45.085 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:37:45.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:45.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:45.085 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:37:45.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:45.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:45.085 | INFO | Uploaded 36GiB to GPU in 1.99158s (18.0761GiB/s) | 20260506 23:37:45.085 | INFO | Downloaded 34GiB to GPU in 1.99566s (17.0369GiB/s) | 20260506 23:37:50.221 | INFO | Analyzing chunk frames = [5357056,5357568) | 20260506 23:37:50.221 | INFO | MemInfo: { | 20260506 23:37:50.221 | INFO | "gpu" : " 17 GB", | 20260506 23:37:50.221 | INFO | "huge" : "", | 20260506 23:37:50.221 | INFO | "huge_shared" : " 27 GB", | 20260506 23:37:50.221 | INFO | "malloc" : " 14 GB", | 20260506 23:37:50.221 | INFO | "pinned" : " 29 GB" | 20260506 23:37:50.221 | INFO | } | 20260506 23:37:50.221 | INFO | Perf report for the last 5.13612 seconds: | 20260506 23:37:50.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:37:50.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:50.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:50.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:50.221 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:37:50.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:50.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:50.221 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:37:50.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:50.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:50.221 | INFO | Uploaded 36GiB to GPU in 1.98312s (18.1532GiB/s) | 20260506 23:37:50.221 | INFO | Downloaded 34GiB to GPU in 1.98751s (17.1068GiB/s) | 20260506 23:37:50.222 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:37:50.222 | INFO | Total mem usage: 18 GiB | 20260506 23:37:50.222 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:37:50.222 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -20.9%) | 20260506 23:37:50.222 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:37:50.222 | INFO | High Water Breakdown: | 20260506 23:37:50.222 | INFO | 60.8% : 9.06 GiB ( 79.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:37:50.222 | INFO | 39.1% : 5.82 GiB ( 86.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:37:50.222 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:37:50.222 | INFO | ------------------------------------------ | 20260506 23:37:50.222 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:37:50.222 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:37:50.222 | INFO | Total mem usage: 26.5 GiB | 20260506 23:37:50.222 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:37:50.222 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:37:50.222 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:37:50.222 | INFO | High Water Breakdown: | 20260506 23:37:50.222 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:37:50.222 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:37:50.222 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:37:50.222 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:37:50.222 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:37:50.222 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:37:50.222 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:37:50.222 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:37:50.222 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:37:50.222 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:37:50.222 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:37:50.222 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:37:50.222 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:37:50.222 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:37:50.222 | INFO | ------------------------------------------ | 20260506 23:37:50.222 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:37:50.222 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:37:50.222 | INFO | Total mem usage: 25.6 GiB | 20260506 23:37:50.222 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:37:50.222 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:37:50.222 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:37:50.222 | INFO | High Water Breakdown: | 20260506 23:37:50.222 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:37:50.222 | INFO | ------------------------------------------ | 20260506 23:37:50.222 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:37:50.222 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:37:50.222 | INFO | Total mem usage: 15.7 GiB | 20260506 23:37:50.222 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:37:50.222 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:37:50.222 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 23:37:50.222 | INFO | High Water Breakdown: | 20260506 23:37:50.222 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:37:50.222 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:37:50.222 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:37:50.222 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:37:50.222 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:37:50.222 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:37:50.222 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:37:50.222 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:37:50.222 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:37:50.222 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:37:50.222 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:37:50.222 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:37:50.222 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:37:50.222 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:37:50.222 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:37:50.222 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:37:50.223 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:37:50.223 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:37:50.223 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:37:50.223 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:37:50.223 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:37:50.223 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:37:50.223 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:37:50.223 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:37:50.223 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:37:50.223 | INFO | ------------------------------------------ | 20260506 23:37:50.223 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:37:50.223 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:37:50.223 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:37:50.223 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:37:50.223 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.58e+01, 1.35e+01 / 2.11e+00 | 20260506 23:37:50.223 | INFO | PerfReport: 0.30%, AggregateStats -- 9.16e-02 / 1.03e-01, 1.05e-02 / 8.76e-03 | 20260506 23:37:50.223 | INFO | PerfReport: 11.66%, Baseline -- 3.61e+00 / 4.38e+00, 5.40e-01 / 3.46e-01 | 20260506 23:37:50.223 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.79e-01 / 1.41e-01 | 20260506 23:37:50.223 | INFO | PerfReport: 16.08%, DME -- 3.99e+01 / 4.47e+01, 4.01e+00 / 2.43e+00 | 20260506 23:37:50.223 | INFO | PerfReport: 57.35%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.34e+00 / 1.70e+00 | 20260506 23:37:50.223 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.19e+00 / 2.75e+00, 3.81e-01 / 1.99e-01 | 20260506 23:37:50.223 | INFO | PerfReport: 3.20%, Metrics -- 9.90e-01 / 1.30e+00, 2.63e-01 / 1.59e-01 | 20260506 23:37:50.223 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.36e-01 / 3.82e-01, 5.34e-02 / 2.05e-02 | 20260506 23:37:50.223 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:37:50.223 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:37:50.223 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:37:50.223 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 23:37:50.223 | INFO | PerfReport: 21.67%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260506 23:37:50.223 | INFO | PerfReport: 21.28%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 23:37:50.223 | INFO | PerfReport: 32.10%, Compute -- 3.10e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 23:37:50.223 | INFO | PerfReport: 1.39%, LockUpload -- 1.35e+00 / 0.00e+00, 5.00e+00 / 0.00e+00 | 20260506 23:37:50.223 | INFO | PerfReport: 1.19%, LockDownload -- 1.15e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260506 23:37:50.223 | INFO | PerfReport: 22.34%, LockCompute -- 2.16e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 23:37:50.223 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260506 23:37:50.320 | INFO | Starting a round of Baz IO | 20260506 23:37:53.448 | INFO | Populating RTMetrics report | 20260506 23:37:53.448 | INFO | RTMetrics report populated | 20260506 23:37:53.448 | INFO | Opening tmp RTMetrics json file | 20260506 23:37:53.475 | INFO | Serializing RTMetrics to json | 20260506 23:37:53.476 | INFO | Serialization done, swapping tmp file now... | 20260506 23:37:53.476 | INFO | RTMetrics done | 20260506 23:37:53.561 | INFO | Finished a round of Baz IO: Wrote 1.878 GiB in 3.241 seconds | 20260506 23:37:53.561 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.43% / 0.00% / 66.57% | 20260506 23:37:53.561 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.402029% / 2.38419e-05% / 3.97364e-05% | 20260506 23:37:53.561 | INFO | Baz IO overall: | 20260506 23:37:53.561 | INFO | BytesWritten : 2802373866519 | 20260506 23:37:53.561 | INFO | NumEvents : 1350908360163 | 20260506 23:37:55.364 | INFO | Analyzing chunk frames = [5357568,5358080) | 20260506 23:37:55.364 | INFO | MemInfo: { | 20260506 23:37:55.364 | INFO | "gpu" : " 17 GB", | 20260506 23:37:55.364 | INFO | "huge" : "", | 20260506 23:37:55.364 | INFO | "huge_shared" : " 27 GB", | 20260506 23:37:55.364 | INFO | "malloc" : " 14 GB", | 20260506 23:37:55.364 | INFO | "pinned" : " 29 GB" | 20260506 23:37:55.364 | INFO | } | 20260506 23:37:55.365 | INFO | Perf report for the last 5.14333 seconds: | 20260506 23:37:55.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:37:55.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:55.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:55.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:55.365 | INFO | ANALYSIS : 62.2%, 2.9, 95.5ms, 0 | 20260506 23:37:55.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:37:55.365 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 23:37:55.365 | INFO | PRE_HQ : 59.7%, 1.8, 56.3ms, 0 | 20260506 23:37:55.365 | INFO | BAZWRITER : 62.6%, 4.6, 148.6ms, 0 | 20260506 23:37:55.365 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 23:37:55.365 | INFO | Uploaded 36GiB to GPU in 2.04529s (17.6014GiB/s) | 20260506 23:37:55.365 | INFO | Downloaded 38GiB to GPU in 2.19634s (17.3015GiB/s) | 20260506 23:38:00.505 | INFO | Analyzing chunk frames = [5358080,5358592) | 20260506 23:38:00.505 | INFO | MemInfo: { | 20260506 23:38:00.505 | INFO | "gpu" : " 17 GB", | 20260506 23:38:00.505 | INFO | "huge" : "", | 20260506 23:38:00.505 | INFO | "huge_shared" : " 27 GB", | 20260506 23:38:00.505 | INFO | "malloc" : " 14 GB", | 20260506 23:38:00.505 | INFO | "pinned" : " 29 GB" | 20260506 23:38:00.505 | INFO | } | 20260506 23:38:00.505 | INFO | Perf report for the last 5.14085 seconds: | 20260506 23:38:00.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:38:00.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:00.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:00.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:00.505 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:38:00.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:00.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:00.506 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:38:00.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:00.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:00.506 | INFO | Uploaded 36GiB to GPU in 2.00594s (17.9467GiB/s) | 20260506 23:38:00.506 | INFO | Downloaded 34GiB to GPU in 1.92809s (17.634GiB/s) | 20260506 23:38:05.645 | INFO | Analyzing chunk frames = [5358592,5359104) | 20260506 23:38:05.645 | INFO | MemInfo: { | 20260506 23:38:05.645 | INFO | "gpu" : " 17 GB", | 20260506 23:38:05.645 | INFO | "huge" : "", | 20260506 23:38:05.645 | INFO | "huge_shared" : " 27 GB", | 20260506 23:38:05.645 | INFO | "malloc" : " 14 GB", | 20260506 23:38:05.645 | INFO | "pinned" : " 29 GB" | 20260506 23:38:05.645 | INFO | } | 20260506 23:38:05.645 | INFO | Perf report for the last 5.13971 seconds: | 20260506 23:38:05.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:38:05.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:05.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:05.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:05.645 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 23:38:05.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:05.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:05.645 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:38:05.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:05.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:05.645 | INFO | Uploaded 36GiB to GPU in 1.9929s (18.0641GiB/s) | 20260506 23:38:05.645 | INFO | Downloaded 34GiB to GPU in 1.9294s (17.622GiB/s) | 20260506 23:38:10.783 | INFO | Analyzing chunk frames = [5359104,5359616) | 20260506 23:38:10.783 | INFO | MemInfo: { | 20260506 23:38:10.783 | INFO | "gpu" : " 17 GB", | 20260506 23:38:10.783 | INFO | "huge" : "", | 20260506 23:38:10.783 | INFO | "huge_shared" : " 27 GB", | 20260506 23:38:10.783 | INFO | "malloc" : " 14 GB", | 20260506 23:38:10.783 | INFO | "pinned" : " 29 GB" | 20260506 23:38:10.783 | INFO | } | 20260506 23:38:10.783 | INFO | Perf report for the last 5.13818 seconds: | 20260506 23:38:10.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:38:10.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:10.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:10.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:10.783 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 23:38:10.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:10.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:10.783 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:38:10.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:10.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:10.784 | INFO | Uploaded 36GiB to GPU in 2.00807s (17.9276GiB/s) | 20260506 23:38:10.784 | INFO | Downloaded 34GiB to GPU in 1.90279s (17.8685GiB/s) | 20260506 23:38:15.919 | INFO | Analyzing chunk frames = [5359616,5360128) | 20260506 23:38:15.919 | INFO | MemInfo: { | 20260506 23:38:15.919 | INFO | "gpu" : " 17 GB", | 20260506 23:38:15.919 | INFO | "huge" : "", | 20260506 23:38:15.919 | INFO | "huge_shared" : " 27 GB", | 20260506 23:38:15.919 | INFO | "malloc" : " 14 GB", | 20260506 23:38:15.919 | INFO | "pinned" : " 29 GB" | 20260506 23:38:15.919 | INFO | } | 20260506 23:38:15.919 | INFO | Perf report for the last 5.13578 seconds: | 20260506 23:38:15.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:38:15.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:15.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:15.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:15.919 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 23:38:15.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:15.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:15.919 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:38:15.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:15.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:15.919 | INFO | Uploaded 36GiB to GPU in 1.99775s (18.0203GiB/s) | 20260506 23:38:15.919 | INFO | Downloaded 34GiB to GPU in 1.93479s (17.5729GiB/s) | 20260506 23:38:21.060 | INFO | Analyzing chunk frames = [5360128,5360640) | 20260506 23:38:21.060 | INFO | MemInfo: { | 20260506 23:38:21.060 | INFO | "gpu" : " 17 GB", | 20260506 23:38:21.061 | INFO | "huge" : "", | 20260506 23:38:21.061 | INFO | "huge_shared" : " 27 GB", | 20260506 23:38:21.061 | INFO | "malloc" : " 14 GB", | 20260506 23:38:21.061 | INFO | "pinned" : " 29 GB" | 20260506 23:38:21.061 | INFO | } | 20260506 23:38:21.061 | INFO | Perf report for the last 5.14152 seconds: | 20260506 23:38:21.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:38:21.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:21.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:21.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:21.061 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 23:38:21.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:21.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:21.061 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 23:38:21.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:21.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:21.061 | INFO | Uploaded 36GiB to GPU in 2.02456s (17.7816GiB/s) | 20260506 23:38:21.061 | INFO | Downloaded 34GiB to GPU in 1.93486s (17.5723GiB/s) | 20260506 23:38:26.201 | INFO | Analyzing chunk frames = [5360640,5361152) | 20260506 23:38:26.201 | INFO | MemInfo: { | 20260506 23:38:26.201 | INFO | "gpu" : " 17 GB", | 20260506 23:38:26.201 | INFO | "huge" : "", | 20260506 23:38:26.201 | INFO | "huge_shared" : " 27 GB", | 20260506 23:38:26.201 | INFO | "malloc" : " 14 GB", | 20260506 23:38:26.201 | INFO | "pinned" : " 29 GB" | 20260506 23:38:26.201 | INFO | } | 20260506 23:38:26.201 | INFO | Perf report for the last 5.14057 seconds: | 20260506 23:38:26.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:38:26.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:26.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:26.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:26.201 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:38:26.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:26.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:26.201 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260506 23:38:26.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:26.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:26.201 | INFO | Uploaded 36GiB to GPU in 1.97679s (18.2113GiB/s) | 20260506 23:38:26.202 | INFO | Downloaded 34GiB to GPU in 2.00007s (16.9994GiB/s) | 20260506 23:38:31.335 | INFO | Analyzing chunk frames = [5361152,5361664) | 20260506 23:38:31.335 | INFO | MemInfo: { | 20260506 23:38:31.335 | INFO | "gpu" : " 17 GB", | 20260506 23:38:31.335 | INFO | "huge" : "", | 20260506 23:38:31.335 | INFO | "huge_shared" : " 27 GB", | 20260506 23:38:31.335 | INFO | "malloc" : " 14 GB", | 20260506 23:38:31.335 | INFO | "pinned" : " 29 GB" | 20260506 23:38:31.335 | INFO | } | 20260506 23:38:31.335 | INFO | Perf report for the last 5.1341 seconds: | 20260506 23:38:31.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:38:31.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:31.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:31.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:31.335 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 23:38:31.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:31.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:31.335 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:38:31.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:31.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:31.336 | INFO | Uploaded 36GiB to GPU in 1.91488s (18.8002GiB/s) | 20260506 23:38:31.336 | INFO | Downloaded 34GiB to GPU in 1.97115s (17.2489GiB/s) | 20260506 23:38:31.433 | INFO | Starting a round of Baz IO | 20260506 23:38:34.564 | INFO | Populating RTMetrics report | 20260506 23:38:34.565 | INFO | RTMetrics report populated | 20260506 23:38:34.565 | INFO | Opening tmp RTMetrics json file | 20260506 23:38:34.565 | INFO | Serializing RTMetrics to json | 20260506 23:38:34.567 | INFO | Serialization done, swapping tmp file now... | 20260506 23:38:34.568 | INFO | RTMetrics done | 20260506 23:38:34.685 | INFO | Finished a round of Baz IO: Wrote 1.877 GiB in 3.251 seconds | 20260506 23:38:34.685 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.43% / 0.00% / 66.57% | 20260506 23:38:34.685 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.400722% / 7.94729e-06% / 5.5631e-05% | 20260506 23:38:34.685 | INFO | Baz IO overall: | 20260506 23:38:34.685 | INFO | BytesWritten : 2804388827915 | 20260506 23:38:34.685 | INFO | NumEvents : 1351866982700 | 20260506 23:38:36.474 | INFO | Analyzing chunk frames = [5361664,5362176) | 20260506 23:38:36.474 | INFO | MemInfo: { | 20260506 23:38:36.474 | INFO | "gpu" : " 17 GB", | 20260506 23:38:36.474 | INFO | "huge" : "", | 20260506 23:38:36.474 | INFO | "huge_shared" : " 27 GB", | 20260506 23:38:36.474 | INFO | "malloc" : " 14 GB", | 20260506 23:38:36.474 | INFO | "pinned" : " 29 GB" | 20260506 23:38:36.474 | INFO | } | 20260506 23:38:36.474 | INFO | Perf report for the last 5.13869 seconds: | 20260506 23:38:36.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:38:36.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:36.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:36.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:36.474 | INFO | ANALYSIS : 62.3%, 2.9, 95.5ms, 0 | 20260506 23:38:36.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:36.474 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260506 23:38:36.474 | INFO | PRE_HQ : 59.8%, 1.8, 56.7ms, 0 | 20260506 23:38:36.474 | INFO | BAZWRITER : 62.8%, 4.3, 137.3ms, 0 | 20260506 23:38:36.474 | INFO | BAZ_SAMPLING : 0.9%, 1.0, 0.5ms, 0 | 20260506 23:38:36.474 | INFO | Uploaded 36GiB to GPU in 2.02729s (17.7577GiB/s) | 20260506 23:38:36.474 | INFO | Downloaded 38GiB to GPU in 2.20008s (17.2721GiB/s) | 20260506 23:38:41.618 | INFO | Analyzing chunk frames = [5362176,5362688) | 20260506 23:38:41.618 | INFO | MemInfo: { | 20260506 23:38:41.618 | INFO | "gpu" : " 17 GB", | 20260506 23:38:41.618 | INFO | "huge" : "", | 20260506 23:38:41.618 | INFO | "huge_shared" : " 27 GB", | 20260506 23:38:41.618 | INFO | "malloc" : " 14 GB", | 20260506 23:38:41.618 | INFO | "pinned" : " 29 GB" | 20260506 23:38:41.618 | INFO | } | 20260506 23:38:41.618 | INFO | Perf report for the last 5.14425 seconds: | 20260506 23:38:41.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:38:41.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:41.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:41.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:41.618 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:38:41.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:41.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:41.618 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260506 23:38:41.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:41.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:41.619 | INFO | Uploaded 36GiB to GPU in 2.00921s (17.9175GiB/s) | 20260506 23:38:41.619 | INFO | Downloaded 34GiB to GPU in 1.93044s (17.6125GiB/s) | 20260506 23:38:46.762 | INFO | Analyzing chunk frames = [5362688,5363200) | 20260506 23:38:46.762 | INFO | MemInfo: { | 20260506 23:38:46.762 | INFO | "gpu" : " 17 GB", | 20260506 23:38:46.762 | INFO | "huge" : "", | 20260506 23:38:46.762 | INFO | "huge_shared" : " 27 GB", | 20260506 23:38:46.762 | INFO | "malloc" : " 14 GB", | 20260506 23:38:46.762 | INFO | "pinned" : " 29 GB" | 20260506 23:38:46.762 | INFO | } | 20260506 23:38:46.762 | INFO | Perf report for the last 5.14415 seconds: | 20260506 23:38:46.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:38:46.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:46.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:46.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:46.763 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260506 23:38:46.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:46.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:46.763 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:38:46.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:46.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:46.763 | INFO | Uploaded 36GiB to GPU in 1.96665s (18.3052GiB/s) | 20260506 23:38:46.763 | INFO | Downloaded 34GiB to GPU in 1.92279s (17.6827GiB/s) | 20260506 23:38:51.899 | INFO | Analyzing chunk frames = [5363200,5363712) | 20260506 23:38:51.899 | INFO | MemInfo: { | 20260506 23:38:51.899 | INFO | "gpu" : " 17 GB", | 20260506 23:38:51.899 | INFO | "huge" : "", | 20260506 23:38:51.899 | INFO | "huge_shared" : " 27 GB", | 20260506 23:38:51.899 | INFO | "malloc" : " 14 GB", | 20260506 23:38:51.899 | INFO | "pinned" : " 29 GB" | 20260506 23:38:51.899 | INFO | } | 20260506 23:38:51.899 | INFO | Perf report for the last 5.13696 seconds: | 20260506 23:38:51.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:38:51.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:51.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:51.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:51.900 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 23:38:51.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:51.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:51.900 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:38:51.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:51.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:51.900 | INFO | Uploaded 36GiB to GPU in 1.95078s (18.4542GiB/s) | 20260506 23:38:51.900 | INFO | Downloaded 34GiB to GPU in 1.90835s (17.8164GiB/s) | 20260506 23:38:57.035 | INFO | Analyzing chunk frames = [5363712,5364224) | 20260506 23:38:57.035 | INFO | MemInfo: { | 20260506 23:38:57.035 | INFO | "gpu" : " 17 GB", | 20260506 23:38:57.035 | INFO | "huge" : "", | 20260506 23:38:57.035 | INFO | "huge_shared" : " 27 GB", | 20260506 23:38:57.035 | INFO | "malloc" : " 14 GB", | 20260506 23:38:57.035 | INFO | "pinned" : " 29 GB" | 20260506 23:38:57.035 | INFO | } | 20260506 23:38:57.035 | INFO | Perf report for the last 5.13603 seconds: | 20260506 23:38:57.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:38:57.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:57.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:57.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:57.036 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260506 23:38:57.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:57.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:57.036 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 23:38:57.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:57.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:38:57.036 | INFO | Uploaded 36GiB to GPU in 2.01573s (17.8595GiB/s) | 20260506 23:38:57.036 | INFO | Downloaded 34GiB to GPU in 1.94841s (17.4502GiB/s) | 20260506 23:39:02.172 | INFO | Analyzing chunk frames = [5364224,5364736) | 20260506 23:39:02.172 | INFO | MemInfo: { | 20260506 23:39:02.172 | INFO | "gpu" : " 17 GB", | 20260506 23:39:02.172 | INFO | "huge" : "", | 20260506 23:39:02.172 | INFO | "huge_shared" : " 27 GB", | 20260506 23:39:02.172 | INFO | "malloc" : " 14 GB", | 20260506 23:39:02.172 | INFO | "pinned" : " 29 GB" | 20260506 23:39:02.172 | INFO | } | 20260506 23:39:02.172 | INFO | Perf report for the last 5.13688 seconds: | 20260506 23:39:02.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:39:02.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:02.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:02.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:02.172 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 23:39:02.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:02.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:02.172 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 23:39:02.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:02.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:02.173 | INFO | Uploaded 36GiB to GPU in 2.03379s (17.7009GiB/s) | 20260506 23:39:02.173 | INFO | Downloaded 34GiB to GPU in 1.95552s (17.3866GiB/s) | 20260506 23:39:07.313 | INFO | Analyzing chunk frames = [5364736,5365248) | 20260506 23:39:07.313 | INFO | MemInfo: { | 20260506 23:39:07.313 | INFO | "gpu" : " 17 GB", | 20260506 23:39:07.313 | INFO | "huge" : "", | 20260506 23:39:07.313 | INFO | "huge_shared" : " 27 GB", | 20260506 23:39:07.313 | INFO | "malloc" : " 14 GB", | 20260506 23:39:07.313 | INFO | "pinned" : " 29 GB" | 20260506 23:39:07.313 | INFO | } | 20260506 23:39:07.313 | INFO | Perf report for the last 5.14087 seconds: | 20260506 23:39:07.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:39:07.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:07.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:07.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:07.313 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 23:39:07.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:07.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:07.313 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:39:07.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:07.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:07.314 | INFO | Uploaded 36GiB to GPU in 1.95901s (18.3766GiB/s) | 20260506 23:39:07.314 | INFO | Downloaded 34GiB to GPU in 1.98704s (17.1109GiB/s) | 20260506 23:39:12.458 | INFO | Analyzing chunk frames = [5365248,5365760) | 20260506 23:39:12.458 | INFO | MemInfo: { | 20260506 23:39:12.458 | INFO | "gpu" : " 17 GB", | 20260506 23:39:12.458 | INFO | "huge" : "", | 20260506 23:39:12.458 | INFO | "huge_shared" : " 27 GB", | 20260506 23:39:12.458 | INFO | "malloc" : " 14 GB", | 20260506 23:39:12.458 | INFO | "pinned" : " 29 GB" | 20260506 23:39:12.458 | INFO | } | 20260506 23:39:12.458 | INFO | Perf report for the last 5.1447 seconds: | 20260506 23:39:12.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:39:12.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:12.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:12.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:12.458 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260506 23:39:12.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:12.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:12.458 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 23:39:12.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:12.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:12.458 | INFO | Uploaded 36GiB to GPU in 2.00513s (17.9539GiB/s) | 20260506 23:39:12.458 | INFO | Downloaded 34GiB to GPU in 1.98954s (17.0894GiB/s) | 20260506 23:39:12.458 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:39:12.458 | INFO | Total mem usage: 18 GiB | 20260506 23:39:12.459 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:39:12.459 | INFO | Recent High Water Marks Sum: 15.4 GiB (Savings: -16.6%) | 20260506 23:39:12.459 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:39:12.459 | INFO | High Water Breakdown: | 20260506 23:39:12.459 | INFO | 61.4% : 9.48 GiB ( 83% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:39:12.459 | INFO | 38.5% : 5.95 GiB ( 88.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:39:12.459 | INFO | 0.129% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:39:12.459 | INFO | ------------------------------------------ | 20260506 23:39:12.459 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:39:12.459 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:39:12.459 | INFO | Total mem usage: 26.5 GiB | 20260506 23:39:12.459 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:39:12.459 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:39:12.459 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:39:12.459 | INFO | High Water Breakdown: | 20260506 23:39:12.459 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:39:12.459 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:39:12.459 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:39:12.459 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:39:12.459 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:39:12.459 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:39:12.459 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:39:12.459 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:39:12.459 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:39:12.459 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:39:12.459 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:39:12.459 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:39:12.459 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:39:12.459 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:39:12.459 | INFO | ------------------------------------------ | 20260506 23:39:12.459 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:39:12.459 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:39:12.459 | INFO | Total mem usage: 25.6 GiB | 20260506 23:39:12.459 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:39:12.459 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:39:12.459 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:39:12.459 | INFO | High Water Breakdown: | 20260506 23:39:12.459 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:39:12.459 | INFO | ------------------------------------------ | 20260506 23:39:12.459 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:39:12.459 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:39:12.459 | INFO | Total mem usage: 15.7 GiB | 20260506 23:39:12.459 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:39:12.459 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:39:12.459 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:39:12.459 | INFO | High Water Breakdown: | 20260506 23:39:12.459 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:39:12.459 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:39:12.459 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:39:12.459 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:39:12.459 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:39:12.459 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:39:12.459 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:39:12.459 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:39:12.459 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:39:12.459 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:39:12.459 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:39:12.459 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:39:12.459 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:39:12.459 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:39:12.459 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:39:12.459 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:39:12.459 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:39:12.459 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:39:12.459 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:39:12.459 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:39:12.459 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:39:12.459 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:39:12.459 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:39:12.459 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:39:12.459 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:39:12.459 | INFO | ------------------------------------------ | 20260506 23:39:12.459 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:39:12.459 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:39:12.459 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:39:12.459 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:39:12.459 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.59e+01, 1.36e+01 / 2.66e+00 | 20260506 23:39:12.459 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.00e-01, 9.76e-03 / 6.37e-03 | 20260506 23:39:12.459 | INFO | PerfReport: 11.66%, Baseline -- 3.61e+00 / 4.39e+00, 5.43e-01 / 3.51e-01 | 20260506 23:39:12.459 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.43e+00, 1.76e-01 / 1.39e-01 | 20260506 23:39:12.459 | INFO | PerfReport: 16.10%, DME -- 3.99e+01 / 4.47e+01, 4.31e+00 / 2.80e+00 | 20260506 23:39:12.459 | INFO | PerfReport: 57.36%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.35e+00 / 1.40e+00 | 20260506 23:39:12.459 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.19e+00 / 2.72e+00, 3.80e-01 / 1.91e-01 | 20260506 23:39:12.459 | INFO | PerfReport: 3.19%, Metrics -- 9.89e-01 / 1.32e+00, 2.65e-01 / 1.32e-01 | 20260506 23:39:12.459 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.36e-01 / 3.83e-01, 5.28e-02 / 2.21e-02 | 20260506 23:39:12.460 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:39:12.460 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:39:12.460 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:39:12.460 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 23:39:12.460 | INFO | PerfReport: 21.54%, Upload -- 2.08e+01 / 0.00e+00, 6.92e+00 / 0.00e+00 | 20260506 23:39:12.460 | INFO | PerfReport: 21.34%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 23:39:12.460 | INFO | PerfReport: 32.12%, Compute -- 3.10e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260506 23:39:12.460 | INFO | PerfReport: 1.31%, LockUpload -- 1.27e+00 / 0.00e+00, 4.79e+00 / 0.00e+00 | 20260506 23:39:12.460 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.20e+00 / 0.00e+00 | 20260506 23:39:12.460 | INFO | PerfReport: 22.35%, LockCompute -- 2.16e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 23:39:12.460 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260506 23:39:12.557 | INFO | Starting a round of Baz IO | 20260506 23:39:15.687 | INFO | Populating RTMetrics report | 20260506 23:39:15.688 | INFO | RTMetrics report populated | 20260506 23:39:15.688 | INFO | Opening tmp RTMetrics json file | 20260506 23:39:15.725 | INFO | Serializing RTMetrics to json | 20260506 23:39:15.764 | INFO | Serialization done, swapping tmp file now... | 20260506 23:39:15.776 | INFO | RTMetrics done | 20260506 23:39:15.811 | INFO | Finished a round of Baz IO: Wrote 1.877 GiB in 3.254 seconds | 20260506 23:39:15.811 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.43% / 0.00% / 66.57% | 20260506 23:39:15.811 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.401811% / 3.97364e-06% / 7.54992e-05% | 20260506 23:39:15.811 | INFO | Baz IO overall: | 20260506 23:39:15.811 | INFO | BytesWritten : 2806404095687 | 20260506 23:39:15.811 | INFO | NumEvents : 1352826810237 | 20260506 23:39:17.592 | INFO | Analyzing chunk frames = [5365760,5366272) | 20260506 23:39:17.592 | INFO | MemInfo: { | 20260506 23:39:17.592 | INFO | "gpu" : " 17 GB", | 20260506 23:39:17.592 | INFO | "huge" : "", | 20260506 23:39:17.592 | INFO | "huge_shared" : " 27 GB", | 20260506 23:39:17.592 | INFO | "malloc" : " 14 GB", | 20260506 23:39:17.592 | INFO | "pinned" : " 29 GB" | 20260506 23:39:17.592 | INFO | } | 20260506 23:39:17.592 | INFO | Perf report for the last 5.1336 seconds: | 20260506 23:39:17.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:39:17.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:17.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:17.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:17.592 | INFO | ANALYSIS : 62.4%, 2.9, 95.3ms, 0 | 20260506 23:39:17.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:17.592 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260506 23:39:17.592 | INFO | PRE_HQ : 59.7%, 1.8, 56.3ms, 0 | 20260506 23:39:17.592 | INFO | BAZWRITER : 63.0%, 4.3, 139.3ms, 0 | 20260506 23:39:17.592 | INFO | BAZ_SAMPLING : 1.6%, 1.0, 0.8ms, 0 | 20260506 23:39:17.592 | INFO | Uploaded 36GiB to GPU in 2.05338s (17.5321GiB/s) | 20260506 23:39:17.592 | INFO | Downloaded 38GiB to GPU in 2.19637s (17.3013GiB/s) | 20260506 23:39:22.733 | INFO | Analyzing chunk frames = [5366272,5366784) | 20260506 23:39:22.733 | INFO | MemInfo: { | 20260506 23:39:22.733 | INFO | "gpu" : " 17 GB", | 20260506 23:39:22.733 | INFO | "huge" : "", | 20260506 23:39:22.733 | INFO | "huge_shared" : " 27 GB", | 20260506 23:39:22.733 | INFO | "malloc" : " 14 GB", | 20260506 23:39:22.733 | INFO | "pinned" : " 29 GB" | 20260506 23:39:22.733 | INFO | } | 20260506 23:39:22.733 | INFO | Perf report for the last 5.14126 seconds: | 20260506 23:39:22.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:39:22.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:22.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:22.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:22.733 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 23:39:22.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:22.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:22.733 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260506 23:39:22.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:22.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:22.733 | INFO | Uploaded 36GiB to GPU in 2.01039s (17.907GiB/s) | 20260506 23:39:22.733 | INFO | Downloaded 34GiB to GPU in 1.93242s (17.5946GiB/s) | 20260506 23:39:27.872 | INFO | Analyzing chunk frames = [5366784,5367296) | 20260506 23:39:27.872 | INFO | MemInfo: { | 20260506 23:39:27.872 | INFO | "gpu" : " 17 GB", | 20260506 23:39:27.872 | INFO | "huge" : "", | 20260506 23:39:27.872 | INFO | "huge_shared" : " 27 GB", | 20260506 23:39:27.872 | INFO | "malloc" : " 14 GB", | 20260506 23:39:27.872 | INFO | "pinned" : " 29 GB" | 20260506 23:39:27.872 | INFO | } | 20260506 23:39:27.872 | INFO | Perf report for the last 5.13901 seconds: | 20260506 23:39:27.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:39:27.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:27.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:27.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:27.872 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260506 23:39:27.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:27.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:27.872 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:39:27.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:27.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:27.872 | INFO | Uploaded 36GiB to GPU in 1.98018s (18.1802GiB/s) | 20260506 23:39:27.872 | INFO | Downloaded 34GiB to GPU in 1.92795s (17.6353GiB/s) | 20260506 23:39:33.013 | INFO | Analyzing chunk frames = [5367296,5367808) | 20260506 23:39:33.013 | INFO | MemInfo: { | 20260506 23:39:33.013 | INFO | "gpu" : " 17 GB", | 20260506 23:39:33.013 | INFO | "huge" : "", | 20260506 23:39:33.013 | INFO | "huge_shared" : " 27 GB", | 20260506 23:39:33.013 | INFO | "malloc" : " 14 GB", | 20260506 23:39:33.013 | INFO | "pinned" : " 29 GB" | 20260506 23:39:33.013 | INFO | } | 20260506 23:39:33.013 | INFO | Perf report for the last 5.14145 seconds: | 20260506 23:39:33.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:39:33.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:33.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:33.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:33.013 | INFO | ANALYSIS : 60.0%, 3.0, 93.2ms, 0 | 20260506 23:39:33.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:33.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:33.013 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:39:33.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:33.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:33.014 | INFO | Uploaded 36GiB to GPU in 2.00681s (17.939GiB/s) | 20260506 23:39:33.014 | INFO | Downloaded 34GiB to GPU in 1.9097s (17.8039GiB/s) | 20260506 23:39:38.153 | INFO | Analyzing chunk frames = [5367808,5368320) | 20260506 23:39:38.153 | INFO | MemInfo: { | 20260506 23:39:38.154 | INFO | "gpu" : " 17 GB", | 20260506 23:39:38.154 | INFO | "huge" : "", | 20260506 23:39:38.154 | INFO | "huge_shared" : " 27 GB", | 20260506 23:39:38.154 | INFO | "malloc" : " 14 GB", | 20260506 23:39:38.154 | INFO | "pinned" : " 29 GB" | 20260506 23:39:38.154 | INFO | } | 20260506 23:39:38.154 | INFO | Perf report for the last 5.14039 seconds: | 20260506 23:39:38.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:39:38.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:38.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:38.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:38.154 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 23:39:38.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:38.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:38.154 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:39:38.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:38.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:38.154 | INFO | Uploaded 36GiB to GPU in 1.99435s (18.051GiB/s) | 20260506 23:39:38.154 | INFO | Downloaded 34GiB to GPU in 1.93986s (17.5271GiB/s) | 20260506 23:39:43.290 | INFO | Analyzing chunk frames = [5368320,5368832) | 20260506 23:39:43.290 | INFO | MemInfo: { | 20260506 23:39:43.290 | INFO | "gpu" : " 17 GB", | 20260506 23:39:43.290 | INFO | "huge" : "", | 20260506 23:39:43.290 | INFO | "huge_shared" : " 27 GB", | 20260506 23:39:43.290 | INFO | "malloc" : " 14 GB", | 20260506 23:39:43.291 | INFO | "pinned" : " 29 GB" | 20260506 23:39:43.291 | INFO | } | 20260506 23:39:43.291 | INFO | Perf report for the last 5.13694 seconds: | 20260506 23:39:43.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:39:43.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:43.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:43.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:43.291 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 23:39:43.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:43.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:43.291 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:39:43.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:43.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:43.291 | INFO | Uploaded 36GiB to GPU in 2.05309s (17.5346GiB/s) | 20260506 23:39:43.291 | INFO | Downloaded 34GiB to GPU in 1.94971s (17.4385GiB/s) | 20260506 23:39:48.425 | INFO | Analyzing chunk frames = [5368832,5369344) | 20260506 23:39:48.425 | INFO | MemInfo: { | 20260506 23:39:48.425 | INFO | "gpu" : " 17 GB", | 20260506 23:39:48.425 | INFO | "huge" : "", | 20260506 23:39:48.425 | INFO | "huge_shared" : " 27 GB", | 20260506 23:39:48.425 | INFO | "malloc" : " 14 GB", | 20260506 23:39:48.425 | INFO | "pinned" : " 29 GB" | 20260506 23:39:48.425 | INFO | } | 20260506 23:39:48.425 | INFO | Perf report for the last 5.13463 seconds: | 20260506 23:39:48.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:39:48.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:48.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:48.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:48.426 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:39:48.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:48.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:48.426 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 23:39:48.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:48.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:48.426 | INFO | Uploaded 36GiB to GPU in 1.98014s (18.1805GiB/s) | 20260506 23:39:48.426 | INFO | Downloaded 34GiB to GPU in 2.00081s (16.9931GiB/s) | 20260506 23:39:53.564 | INFO | Analyzing chunk frames = [5369344,5369856) | 20260506 23:39:53.564 | INFO | MemInfo: { | 20260506 23:39:53.564 | INFO | "gpu" : " 17 GB", | 20260506 23:39:53.564 | INFO | "huge" : "", | 20260506 23:39:53.564 | INFO | "huge_shared" : " 27 GB", | 20260506 23:39:53.564 | INFO | "malloc" : " 14 GB", | 20260506 23:39:53.564 | INFO | "pinned" : " 29 GB" | 20260506 23:39:53.564 | INFO | } | 20260506 23:39:53.564 | INFO | Perf report for the last 5.13852 seconds: | 20260506 23:39:53.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:39:53.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:53.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:53.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:53.564 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260506 23:39:53.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:53.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:53.564 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260506 23:39:53.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:53.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:53.564 | INFO | Uploaded 36GiB to GPU in 2.01206s (17.8921GiB/s) | 20260506 23:39:53.564 | INFO | Downloaded 34GiB to GPU in 2.00472s (16.9599GiB/s) | 20260506 23:39:53.662 | INFO | Starting a round of Baz IO | 20260506 23:39:56.794 | INFO | Populating RTMetrics report | 20260506 23:39:56.795 | INFO | RTMetrics report populated | 20260506 23:39:56.795 | INFO | Opening tmp RTMetrics json file | 20260506 23:39:56.795 | INFO | Serializing RTMetrics to json | 20260506 23:39:56.802 | INFO | Serialization done, swapping tmp file now... | 20260506 23:39:56.811 | INFO | RTMetrics done | 20260506 23:39:56.918 | INFO | Finished a round of Baz IO: Wrote 1.875 GiB in 3.256 seconds | 20260506 23:39:56.918 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.43% / 0.00% / 66.57% | 20260506 23:39:56.918 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.400043% / 3.97364e-06% / 5.96046e-05% | 20260506 23:39:56.918 | INFO | Baz IO overall: | 20260506 23:39:56.918 | INFO | BytesWritten : 2808417602766 | 20260506 23:39:56.918 | INFO | NumEvents : 1353785475952 | 20260506 23:39:58.713 | INFO | Analyzing chunk frames = [5369856,5370368) | 20260506 23:39:58.713 | INFO | MemInfo: { | 20260506 23:39:58.713 | INFO | "gpu" : " 17 GB", | 20260506 23:39:58.713 | INFO | "huge" : "", | 20260506 23:39:58.713 | INFO | "huge_shared" : " 27 GB", | 20260506 23:39:58.713 | INFO | "malloc" : " 14 GB", | 20260506 23:39:58.713 | INFO | "pinned" : " 29 GB" | 20260506 23:39:58.713 | INFO | } | 20260506 23:39:58.713 | INFO | Perf report for the last 5.14907 seconds: | 20260506 23:39:58.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:39:58.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:58.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:58.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:58.713 | INFO | ANALYSIS : 62.2%, 2.9, 95.6ms, 0 | 20260506 23:39:58.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:39:58.713 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 23:39:58.713 | INFO | PRE_HQ : 59.6%, 1.8, 56.7ms, 0 | 20260506 23:39:58.713 | INFO | BAZWRITER : 62.9%, 4.2, 134.7ms, 0 | 20260506 23:39:58.713 | INFO | BAZ_SAMPLING : 1.2%, 1.0, 0.6ms, 0 | 20260506 23:39:58.713 | INFO | Uploaded 36GiB to GPU in 2.02686s (17.7615GiB/s) | 20260506 23:39:58.713 | INFO | Downloaded 38GiB to GPU in 2.18825s (17.3655GiB/s) | 20260506 23:40:03.853 | INFO | Analyzing chunk frames = [5370368,5370880) | 20260506 23:40:03.853 | INFO | MemInfo: { | 20260506 23:40:03.853 | INFO | "gpu" : " 17 GB", | 20260506 23:40:03.853 | INFO | "huge" : "", | 20260506 23:40:03.853 | INFO | "huge_shared" : " 27 GB", | 20260506 23:40:03.853 | INFO | "malloc" : " 14 GB", | 20260506 23:40:03.853 | INFO | "pinned" : " 29 GB" | 20260506 23:40:03.853 | INFO | } | 20260506 23:40:03.853 | INFO | Perf report for the last 5.13994 seconds: | 20260506 23:40:03.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:40:03.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:03.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:03.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:03.853 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 23:40:03.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:03.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:03.853 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:40:03.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:03.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:03.853 | INFO | Uploaded 36GiB to GPU in 1.98182s (18.1652GiB/s) | 20260506 23:40:03.853 | INFO | Downloaded 34GiB to GPU in 1.92981s (17.6183GiB/s) | 20260506 23:40:08.988 | INFO | Analyzing chunk frames = [5370880,5371392) | 20260506 23:40:08.988 | INFO | MemInfo: { | 20260506 23:40:08.988 | INFO | "gpu" : " 17 GB", | 20260506 23:40:08.988 | INFO | "huge" : "", | 20260506 23:40:08.988 | INFO | "huge_shared" : " 27 GB", | 20260506 23:40:08.988 | INFO | "malloc" : " 14 GB", | 20260506 23:40:08.988 | INFO | "pinned" : " 29 GB" | 20260506 23:40:08.988 | INFO | } | 20260506 23:40:08.988 | INFO | Perf report for the last 5.13484 seconds: | 20260506 23:40:08.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:40:08.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:08.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:08.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:08.988 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260506 23:40:08.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:08.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:08.988 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:40:08.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:08.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:08.988 | INFO | Uploaded 36GiB to GPU in 1.98624s (18.1247GiB/s) | 20260506 23:40:08.988 | INFO | Downloaded 34GiB to GPU in 1.93732s (17.55GiB/s) | 20260506 23:40:14.132 | INFO | Analyzing chunk frames = [5371392,5371904) | 20260506 23:40:14.132 | INFO | MemInfo: { | 20260506 23:40:14.132 | INFO | "gpu" : " 17 GB", | 20260506 23:40:14.132 | INFO | "huge" : "", | 20260506 23:40:14.132 | INFO | "huge_shared" : " 27 GB", | 20260506 23:40:14.132 | INFO | "malloc" : " 14 GB", | 20260506 23:40:14.132 | INFO | "pinned" : " 29 GB" | 20260506 23:40:14.132 | INFO | } | 20260506 23:40:14.132 | INFO | Perf report for the last 5.14434 seconds: | 20260506 23:40:14.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:40:14.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:14.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:14.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:14.132 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260506 23:40:14.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:14.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:14.132 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:40:14.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:14.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:14.132 | INFO | Uploaded 36GiB to GPU in 1.96224s (18.3464GiB/s) | 20260506 23:40:14.132 | INFO | Downloaded 34GiB to GPU in 1.90674s (17.8315GiB/s) | 20260506 23:40:19.264 | INFO | Analyzing chunk frames = [5371904,5372416) | 20260506 23:40:19.264 | INFO | MemInfo: { | 20260506 23:40:19.264 | INFO | "gpu" : " 17 GB", | 20260506 23:40:19.264 | INFO | "huge" : "", | 20260506 23:40:19.264 | INFO | "huge_shared" : " 27 GB", | 20260506 23:40:19.264 | INFO | "malloc" : " 14 GB", | 20260506 23:40:19.264 | INFO | "pinned" : " 29 GB" | 20260506 23:40:19.264 | INFO | } | 20260506 23:40:19.264 | INFO | Perf report for the last 5.13192 seconds: | 20260506 23:40:19.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:40:19.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:19.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:19.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:19.264 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260506 23:40:19.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:19.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:19.264 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:40:19.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:19.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:19.264 | INFO | Uploaded 36GiB to GPU in 1.99128s (18.0788GiB/s) | 20260506 23:40:19.265 | INFO | Downloaded 34GiB to GPU in 1.9525s (17.4136GiB/s) | 20260506 23:40:24.403 | INFO | Analyzing chunk frames = [5372416,5372928) | 20260506 23:40:24.403 | INFO | MemInfo: { | 20260506 23:40:24.403 | INFO | "gpu" : " 17 GB", | 20260506 23:40:24.403 | INFO | "huge" : "", | 20260506 23:40:24.403 | INFO | "huge_shared" : " 27 GB", | 20260506 23:40:24.403 | INFO | "malloc" : " 14 GB", | 20260506 23:40:24.403 | INFO | "pinned" : " 29 GB" | 20260506 23:40:24.403 | INFO | } | 20260506 23:40:24.403 | INFO | Perf report for the last 5.13943 seconds: | 20260506 23:40:24.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:40:24.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:24.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:24.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:24.404 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 23:40:24.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:24.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:24.404 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 23:40:24.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:24.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:24.404 | INFO | Uploaded 36GiB to GPU in 2.00189s (17.983GiB/s) | 20260506 23:40:24.404 | INFO | Downloaded 34GiB to GPU in 1.94506s (17.4802GiB/s) | 20260506 23:40:29.540 | INFO | Analyzing chunk frames = [5372928,5373440) | 20260506 23:40:29.540 | INFO | MemInfo: { | 20260506 23:40:29.540 | INFO | "gpu" : " 17 GB", | 20260506 23:40:29.540 | INFO | "huge" : "", | 20260506 23:40:29.540 | INFO | "huge_shared" : " 27 GB", | 20260506 23:40:29.540 | INFO | "malloc" : " 14 GB", | 20260506 23:40:29.540 | INFO | "pinned" : " 29 GB" | 20260506 23:40:29.540 | INFO | } | 20260506 23:40:29.540 | INFO | Perf report for the last 5.13658 seconds: | 20260506 23:40:29.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:40:29.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:29.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:29.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:29.540 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 23:40:29.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:29.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:29.540 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260506 23:40:29.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:29.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:29.540 | INFO | Uploaded 36GiB to GPU in 1.9511s (18.4511GiB/s) | 20260506 23:40:29.541 | INFO | Downloaded 34GiB to GPU in 1.98663s (17.1144GiB/s) | 20260506 23:40:34.682 | INFO | Analyzing chunk frames = [5373440,5373952) | 20260506 23:40:34.682 | INFO | MemInfo: { | 20260506 23:40:34.682 | INFO | "gpu" : " 17 GB", | 20260506 23:40:34.682 | INFO | "huge" : "", | 20260506 23:40:34.682 | INFO | "huge_shared" : " 27 GB", | 20260506 23:40:34.682 | INFO | "malloc" : " 14 GB", | 20260506 23:40:34.682 | INFO | "pinned" : " 29 GB" | 20260506 23:40:34.682 | INFO | } | 20260506 23:40:34.682 | INFO | Perf report for the last 5.14182 seconds: | 20260506 23:40:34.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:40:34.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:34.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:34.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:34.682 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260506 23:40:34.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:34.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:34.682 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 23:40:34.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:34.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:34.682 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:40:34.682 | INFO | Uploaded 36GiB to GPU in 1.99688s (18.0281GiB/s) | 20260506 23:40:34.682 | INFO | Total mem usage: 18 GiB | 20260506 23:40:34.683 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:40:34.683 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -21.7%) | 20260506 23:40:34.683 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:40:34.683 | INFO | High Water Breakdown: | 20260506 23:40:34.683 | INFO | 60.7% : 8.98 GiB ( 78.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:40:34.683 | INFO | 39.2% : 5.79 GiB ( 86.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:40:34.683 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:40:34.683 | INFO | ------------------------------------------ | 20260506 23:40:34.683 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:40:34.683 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:40:34.683 | INFO | Total mem usage: 26.5 GiB | 20260506 23:40:34.683 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:40:34.683 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:40:34.683 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:40:34.683 | INFO | High Water Breakdown: | 20260506 23:40:34.683 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:40:34.683 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:40:34.683 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:40:34.683 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:40:34.683 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:40:34.683 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:40:34.683 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:40:34.683 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:40:34.683 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:40:34.683 | INFO | Downloaded 34GiB to GPU in 2.00535s (16.9546GiB/s) | 20260506 23:40:34.683 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:40:34.683 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:40:34.683 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:40:34.683 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:40:34.683 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:40:34.683 | INFO | ------------------------------------------ | 20260506 23:40:34.683 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:40:34.683 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:40:34.683 | INFO | Total mem usage: 25.6 GiB | 20260506 23:40:34.683 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:40:34.683 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:40:34.683 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:40:34.683 | INFO | High Water Breakdown: | 20260506 23:40:34.683 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:40:34.683 | INFO | ------------------------------------------ | 20260506 23:40:34.683 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:40:34.683 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:40:34.683 | INFO | Total mem usage: 15.7 GiB | 20260506 23:40:34.683 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:40:34.683 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:40:34.683 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:40:34.683 | INFO | High Water Breakdown: | 20260506 23:40:34.683 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:40:34.683 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:40:34.683 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:40:34.683 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:40:34.683 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:40:34.683 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:40:34.683 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:40:34.683 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:40:34.683 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:40:34.683 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:40:34.683 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:40:34.683 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:40:34.684 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:40:34.684 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:40:34.684 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:40:34.684 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:40:34.684 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:40:34.684 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:40:34.684 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:40:34.684 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:40:34.684 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:40:34.684 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:40:34.684 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:40:34.684 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:40:34.684 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:40:34.684 | INFO | ------------------------------------------ | 20260506 23:40:34.684 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:40:34.684 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:40:34.684 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:40:34.684 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:40:34.684 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.57e+01, 1.34e+01 / 2.34e+00 | 20260506 23:40:34.684 | INFO | PerfReport: 0.29%, AggregateStats -- 9.14e-02 / 1.01e-01, 1.08e-02 / 6.61e-03 | 20260506 23:40:34.684 | INFO | PerfReport: 11.66%, Baseline -- 3.61e+00 / 4.32e+00, 5.39e-01 / 3.40e-01 | 20260506 23:40:34.684 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.78e-01 / 1.18e-01 | 20260506 23:40:34.684 | INFO | PerfReport: 16.08%, DME -- 3.98e+01 / 4.43e+01, 4.10e+00 / 2.91e+00 | 20260506 23:40:34.684 | INFO | PerfReport: 57.35%, FrameLabeling -- 1.78e+01 / 2.11e+01, 1.32e+00 / 1.64e+00 | 20260506 23:40:34.684 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.19e+00 / 2.74e+00, 3.84e-01 / 2.39e-01 | 20260506 23:40:34.684 | INFO | PerfReport: 3.19%, Metrics -- 9.89e-01 / 1.32e+00, 2.64e-01 / 1.70e-01 | 20260506 23:40:34.684 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.36e-01 / 3.84e-01, 5.25e-02 / 2.25e-02 | 20260506 23:40:34.684 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:40:34.684 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:40:34.684 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:40:34.684 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 23:40:34.684 | INFO | PerfReport: 21.58%, Upload -- 2.08e+01 / 0.00e+00, 6.93e+00 / 0.00e+00 | 20260506 23:40:34.684 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260506 23:40:34.684 | INFO | PerfReport: 32.09%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 23:40:34.684 | INFO | PerfReport: 1.30%, LockUpload -- 1.26e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260506 23:40:34.684 | INFO | PerfReport: 1.34%, LockDownload -- 1.30e+00 / 0.00e+00, 3.18e+00 / 0.00e+00 | 20260506 23:40:34.684 | INFO | PerfReport: 22.28%, LockCompute -- 2.15e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:40:34.684 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260506 23:40:34.780 | INFO | Starting a round of Baz IO | 20260506 23:40:37.912 | INFO | Populating RTMetrics report | 20260506 23:40:37.912 | INFO | RTMetrics report populated | 20260506 23:40:37.912 | INFO | Opening tmp RTMetrics json file | 20260506 23:40:37.940 | INFO | Serializing RTMetrics to json | 20260506 23:40:37.942 | INFO | Serialization done, swapping tmp file now... | 20260506 23:40:37.942 | INFO | RTMetrics done | 20260506 23:40:38.023 | INFO | Finished a round of Baz IO: Wrote 1.875 GiB in 3.243 seconds | 20260506 23:40:38.023 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.42% / 0.00% / 66.57% | 20260506 23:40:38.023 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.398544% / 0% / 7.94729e-05% | 20260506 23:40:38.023 | INFO | Baz IO overall: | 20260506 23:40:38.023 | INFO | BytesWritten : 2810430525312 | 20260506 23:40:38.023 | INFO | NumEvents : 1354742907859 | 20260506 23:40:39.825 | INFO | Analyzing chunk frames = [5373952,5374464) | 20260506 23:40:39.825 | INFO | MemInfo: { | 20260506 23:40:39.825 | INFO | "gpu" : " 17 GB", | 20260506 23:40:39.825 | INFO | "huge" : "", | 20260506 23:40:39.825 | INFO | "huge_shared" : " 27 GB", | 20260506 23:40:39.825 | INFO | "malloc" : " 14 GB", | 20260506 23:40:39.825 | INFO | "pinned" : " 29 GB" | 20260506 23:40:39.825 | INFO | } | 20260506 23:40:39.825 | INFO | Perf report for the last 5.14311 seconds: | 20260506 23:40:39.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:40:39.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:39.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:39.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:39.825 | INFO | ANALYSIS : 62.2%, 2.9, 95.1ms, 0 | 20260506 23:40:39.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:39.825 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260506 23:40:39.825 | INFO | PRE_HQ : 59.7%, 1.8, 56.4ms, 0 | 20260506 23:40:39.825 | INFO | BAZWRITER : 62.7%, 4.4, 142.7ms, 0 | 20260506 23:40:39.825 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 23:40:39.825 | INFO | Uploaded 36GiB to GPU in 2.03327s (17.7055GiB/s) | 20260506 23:40:39.825 | INFO | Downloaded 38GiB to GPU in 2.19706s (17.2958GiB/s) | 20260506 23:40:44.970 | INFO | Analyzing chunk frames = [5374464,5374976) | 20260506 23:40:44.970 | INFO | MemInfo: { | 20260506 23:40:44.970 | INFO | "gpu" : " 17 GB", | 20260506 23:40:44.970 | INFO | "huge" : "", | 20260506 23:40:44.970 | INFO | "huge_shared" : " 27 GB", | 20260506 23:40:44.970 | INFO | "malloc" : " 14 GB", | 20260506 23:40:44.970 | INFO | "pinned" : " 29 GB" | 20260506 23:40:44.970 | INFO | } | 20260506 23:40:44.970 | INFO | Perf report for the last 5.14533 seconds: | 20260506 23:40:44.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:40:44.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:44.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:44.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:44.970 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 23:40:44.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:44.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:44.970 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260506 23:40:44.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:44.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:44.971 | INFO | Uploaded 36GiB to GPU in 1.99218s (18.0706GiB/s) | 20260506 23:40:44.971 | INFO | Downloaded 34GiB to GPU in 1.93048s (17.6122GiB/s) | 20260506 23:40:50.107 | INFO | Analyzing chunk frames = [5374976,5375488) | 20260506 23:40:50.107 | INFO | MemInfo: { | 20260506 23:40:50.107 | INFO | "gpu" : " 17 GB", | 20260506 23:40:50.107 | INFO | "huge" : "", | 20260506 23:40:50.107 | INFO | "huge_shared" : " 27 GB", | 20260506 23:40:50.107 | INFO | "malloc" : " 14 GB", | 20260506 23:40:50.107 | INFO | "pinned" : " 29 GB" | 20260506 23:40:50.107 | INFO | } | 20260506 23:40:50.107 | INFO | Perf report for the last 5.13688 seconds: | 20260506 23:40:50.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:40:50.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:50.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:50.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:50.107 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260506 23:40:50.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:50.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:50.107 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:40:50.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:50.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:50.107 | INFO | Uploaded 36GiB to GPU in 1.97875s (18.1933GiB/s) | 20260506 23:40:50.108 | INFO | Downloaded 34GiB to GPU in 1.93681s (17.5546GiB/s) | 20260506 23:40:55.239 | INFO | Analyzing chunk frames = [5375488,5376000) | 20260506 23:40:55.239 | INFO | MemInfo: { | 20260506 23:40:55.239 | INFO | "gpu" : " 17 GB", | 20260506 23:40:55.239 | INFO | "huge" : "", | 20260506 23:40:55.239 | INFO | "huge_shared" : " 27 GB", | 20260506 23:40:55.239 | INFO | "malloc" : " 14 GB", | 20260506 23:40:55.239 | INFO | "pinned" : " 29 GB" | 20260506 23:40:55.239 | INFO | } | 20260506 23:40:55.239 | INFO | Perf report for the last 5.13139 seconds: | 20260506 23:40:55.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:40:55.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:55.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:55.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:55.239 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260506 23:40:55.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:55.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:55.239 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:40:55.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:55.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:40:55.239 | INFO | Uploaded 36GiB to GPU in 1.96514s (18.3193GiB/s) | 20260506 23:40:55.239 | INFO | Downloaded 34GiB to GPU in 1.90769s (17.8226GiB/s) | 20260506 23:41:00.378 | INFO | Analyzing chunk frames = [5376000,5376512) | 20260506 23:41:00.378 | INFO | MemInfo: { | 20260506 23:41:00.378 | INFO | "gpu" : " 17 GB", | 20260506 23:41:00.378 | INFO | "huge" : "", | 20260506 23:41:00.378 | INFO | "huge_shared" : " 27 GB", | 20260506 23:41:00.378 | INFO | "malloc" : " 14 GB", | 20260506 23:41:00.378 | INFO | "pinned" : " 29 GB" | 20260506 23:41:00.378 | INFO | } | 20260506 23:41:00.379 | INFO | Perf report for the last 5.13986 seconds: | 20260506 23:41:00.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:41:00.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:00.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:00.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:00.379 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 23:41:00.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:00.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:00.379 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:41:00.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:00.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:00.379 | INFO | Uploaded 36GiB to GPU in 1.98524s (18.1339GiB/s) | 20260506 23:41:00.379 | INFO | Downloaded 34GiB to GPU in 1.92113s (17.6979GiB/s) | 20260506 23:41:05.517 | INFO | Analyzing chunk frames = [5376512,5377024) | 20260506 23:41:05.517 | INFO | MemInfo: { | 20260506 23:41:05.517 | INFO | "gpu" : " 17 GB", | 20260506 23:41:05.517 | INFO | "huge" : "", | 20260506 23:41:05.517 | INFO | "huge_shared" : " 27 GB", | 20260506 23:41:05.517 | INFO | "malloc" : " 14 GB", | 20260506 23:41:05.517 | INFO | "pinned" : " 29 GB" | 20260506 23:41:05.517 | INFO | } | 20260506 23:41:05.517 | INFO | Perf report for the last 5.13822 seconds: | 20260506 23:41:05.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:41:05.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:05.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:05.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:05.517 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 23:41:05.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:05.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:05.517 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:41:05.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:05.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:05.517 | INFO | Uploaded 36GiB to GPU in 1.98834s (18.1055GiB/s) | 20260506 23:41:05.517 | INFO | Downloaded 34GiB to GPU in 1.96281s (17.3221GiB/s) | 20260506 23:41:10.657 | INFO | Analyzing chunk frames = [5377024,5377536) | 20260506 23:41:10.657 | INFO | MemInfo: { | 20260506 23:41:10.657 | INFO | "gpu" : " 17 GB", | 20260506 23:41:10.657 | INFO | "huge" : "", | 20260506 23:41:10.657 | INFO | "huge_shared" : " 27 GB", | 20260506 23:41:10.657 | INFO | "malloc" : " 14 GB", | 20260506 23:41:10.657 | INFO | "pinned" : " 29 GB" | 20260506 23:41:10.657 | INFO | } | 20260506 23:41:10.657 | INFO | Perf report for the last 5.13984 seconds: | 20260506 23:41:10.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:41:10.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:10.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:10.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:10.657 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:41:10.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:10.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:10.657 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:41:10.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:10.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:10.657 | INFO | Uploaded 36GiB to GPU in 1.96073s (18.3605GiB/s) | 20260506 23:41:10.657 | INFO | Downloaded 34GiB to GPU in 1.99657s (17.0292GiB/s) | 20260506 23:41:15.803 | INFO | Analyzing chunk frames = [5377536,5378048) | 20260506 23:41:15.803 | INFO | MemInfo: { | 20260506 23:41:15.803 | INFO | "gpu" : " 17 GB", | 20260506 23:41:15.803 | INFO | "huge" : "", | 20260506 23:41:15.803 | INFO | "huge_shared" : " 27 GB", | 20260506 23:41:15.803 | INFO | "malloc" : " 14 GB", | 20260506 23:41:15.803 | INFO | "pinned" : " 29 GB" | 20260506 23:41:15.803 | INFO | } | 20260506 23:41:15.803 | INFO | Perf report for the last 5.14618 seconds: | 20260506 23:41:15.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:41:15.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:15.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:15.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:15.803 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:41:15.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:15.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:15.803 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:41:15.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:15.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:15.803 | INFO | Uploaded 36GiB to GPU in 1.98125s (18.1703GiB/s) | 20260506 23:41:15.803 | INFO | Downloaded 34GiB to GPU in 1.99638s (17.0308GiB/s) | 20260506 23:41:15.901 | INFO | Starting a round of Baz IO | 20260506 23:41:19.028 | INFO | Populating RTMetrics report | 20260506 23:41:19.028 | INFO | RTMetrics report populated | 20260506 23:41:19.028 | INFO | Opening tmp RTMetrics json file | 20260506 23:41:19.029 | INFO | Serializing RTMetrics to json | 20260506 23:41:19.030 | INFO | Serialization done, swapping tmp file now... | 20260506 23:41:19.031 | INFO | RTMetrics done | 20260506 23:41:19.122 | INFO | Finished a round of Baz IO: Wrote 1.874 GiB in 3.221 seconds | 20260506 23:41:19.122 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.42% / 0.00% / 66.58% | 20260506 23:41:19.122 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.39775% / 7.94729e-06% / 6.35783e-05% | 20260506 23:41:19.122 | INFO | Baz IO overall: | 20260506 23:41:19.122 | INFO | BytesWritten : 2812442678189 | 20260506 23:41:19.122 | INFO | NumEvents : 1355699840990 | 20260506 23:41:20.941 | INFO | Analyzing chunk frames = [5378048,5378560) | 20260506 23:41:20.941 | INFO | MemInfo: { | 20260506 23:41:20.941 | INFO | "gpu" : " 17 GB", | 20260506 23:41:20.941 | INFO | "huge" : "", | 20260506 23:41:20.941 | INFO | "huge_shared" : " 27 GB", | 20260506 23:41:20.941 | INFO | "malloc" : " 14 GB", | 20260506 23:41:20.941 | INFO | "pinned" : " 29 GB" | 20260506 23:41:20.941 | INFO | } | 20260506 23:41:20.941 | INFO | Perf report for the last 5.138 seconds: | 20260506 23:41:20.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:41:20.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:20.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:20.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:20.941 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 23:41:20.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:20.941 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 23:41:20.941 | INFO | PRE_HQ : 59.7%, 1.8, 56.6ms, 0 | 20260506 23:41:20.941 | INFO | BAZWRITER : 62.3%, 3.4, 109.7ms, 0 | 20260506 23:41:20.941 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:41:20.941 | INFO | Uploaded 36GiB to GPU in 2.00558s (17.9499GiB/s) | 20260506 23:41:20.941 | INFO | Downloaded 38GiB to GPU in 2.17476s (17.4732GiB/s) | 20260506 23:41:26.082 | INFO | Analyzing chunk frames = [5378560,5379072) | 20260506 23:41:26.082 | INFO | MemInfo: { | 20260506 23:41:26.082 | INFO | "gpu" : " 17 GB", | 20260506 23:41:26.082 | INFO | "huge" : "", | 20260506 23:41:26.082 | INFO | "huge_shared" : " 27 GB", | 20260506 23:41:26.082 | INFO | "malloc" : " 14 GB", | 20260506 23:41:26.082 | INFO | "pinned" : " 29 GB" | 20260506 23:41:26.082 | INFO | } | 20260506 23:41:26.082 | INFO | Perf report for the last 5.141 seconds: | 20260506 23:41:26.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:41:26.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:26.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:26.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:26.082 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 23:41:26.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:26.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:26.082 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260506 23:41:26.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:26.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:26.082 | INFO | Uploaded 36GiB to GPU in 2.00082s (17.9926GiB/s) | 20260506 23:41:26.082 | INFO | Downloaded 34GiB to GPU in 1.93015s (17.6152GiB/s) | 20260506 23:41:31.220 | INFO | Analyzing chunk frames = [5379072,5379584) | 20260506 23:41:31.220 | INFO | MemInfo: { | 20260506 23:41:31.220 | INFO | "gpu" : " 17 GB", | 20260506 23:41:31.220 | INFO | "huge" : "", | 20260506 23:41:31.220 | INFO | "huge_shared" : " 27 GB", | 20260506 23:41:31.220 | INFO | "malloc" : " 14 GB", | 20260506 23:41:31.220 | INFO | "pinned" : " 29 GB" | 20260506 23:41:31.220 | INFO | } | 20260506 23:41:31.220 | INFO | Perf report for the last 5.13782 seconds: | 20260506 23:41:31.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:41:31.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:31.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:31.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:31.220 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260506 23:41:31.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:31.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:31.220 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:41:31.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:31.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:31.220 | INFO | Uploaded 36GiB to GPU in 1.99912s (18.008GiB/s) | 20260506 23:41:31.220 | INFO | Downloaded 34GiB to GPU in 1.93798s (17.544GiB/s) | 20260506 23:41:36.356 | INFO | Analyzing chunk frames = [5379584,5380096) | 20260506 23:41:36.356 | INFO | MemInfo: { | 20260506 23:41:36.356 | INFO | "gpu" : " 17 GB", | 20260506 23:41:36.356 | INFO | "huge" : "", | 20260506 23:41:36.356 | INFO | "huge_shared" : " 27 GB", | 20260506 23:41:36.356 | INFO | "malloc" : " 14 GB", | 20260506 23:41:36.356 | INFO | "pinned" : " 29 GB" | 20260506 23:41:36.356 | INFO | } | 20260506 23:41:36.356 | INFO | Perf report for the last 5.13618 seconds: | 20260506 23:41:36.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:41:36.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:36.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:36.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:36.356 | INFO | ANALYSIS : 59.5%, 3.0, 92.2ms, 0 | 20260506 23:41:36.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:36.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:36.356 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:41:36.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:36.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:36.356 | INFO | Uploaded 36GiB to GPU in 1.95626s (18.4024GiB/s) | 20260506 23:41:36.356 | INFO | Downloaded 34GiB to GPU in 1.87861s (18.0985GiB/s) | 20260506 23:41:41.491 | INFO | Analyzing chunk frames = [5380096,5380608) | 20260506 23:41:41.491 | INFO | MemInfo: { | 20260506 23:41:41.491 | INFO | "gpu" : " 17 GB", | 20260506 23:41:41.491 | INFO | "huge" : "", | 20260506 23:41:41.491 | INFO | "huge_shared" : " 27 GB", | 20260506 23:41:41.491 | INFO | "malloc" : " 14 GB", | 20260506 23:41:41.491 | INFO | "pinned" : " 29 GB" | 20260506 23:41:41.491 | INFO | } | 20260506 23:41:41.491 | INFO | Perf report for the last 5.13528 seconds: | 20260506 23:41:41.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:41:41.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:41.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:41.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:41.491 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 23:41:41.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:41.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:41.491 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:41:41.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:41.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:41.492 | INFO | Uploaded 36GiB to GPU in 1.98719s (18.116GiB/s) | 20260506 23:41:41.492 | INFO | Downloaded 34GiB to GPU in 1.92444s (17.6675GiB/s) | 20260506 23:41:46.632 | INFO | Analyzing chunk frames = [5380608,5381120) | 20260506 23:41:46.632 | INFO | MemInfo: { | 20260506 23:41:46.632 | INFO | "gpu" : " 17 GB", | 20260506 23:41:46.632 | INFO | "huge" : "", | 20260506 23:41:46.632 | INFO | "huge_shared" : " 27 GB", | 20260506 23:41:46.632 | INFO | "malloc" : " 14 GB", | 20260506 23:41:46.633 | INFO | "pinned" : " 29 GB" | 20260506 23:41:46.633 | INFO | } | 20260506 23:41:46.633 | INFO | Perf report for the last 5.14153 seconds: | 20260506 23:41:46.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:41:46.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:46.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:46.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:46.633 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 23:41:46.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:46.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:46.633 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260506 23:41:46.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:46.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:46.633 | INFO | Uploaded 36GiB to GPU in 1.99799s (18.0181GiB/s) | 20260506 23:41:46.633 | INFO | Downloaded 34GiB to GPU in 1.96101s (17.338GiB/s) | 20260506 23:41:51.782 | INFO | Analyzing chunk frames = [5381120,5381632) | 20260506 23:41:51.782 | INFO | MemInfo: { | 20260506 23:41:51.782 | INFO | "gpu" : " 17 GB", | 20260506 23:41:51.782 | INFO | "huge" : "", | 20260506 23:41:51.782 | INFO | "huge_shared" : " 27 GB", | 20260506 23:41:51.782 | INFO | "malloc" : " 14 GB", | 20260506 23:41:51.782 | INFO | "pinned" : " 29 GB" | 20260506 23:41:51.782 | INFO | } | 20260506 23:41:51.782 | INFO | Perf report for the last 5.14914 seconds: | 20260506 23:41:51.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:41:51.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:51.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:51.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:51.782 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260506 23:41:51.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:51.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:51.782 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:41:51.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:51.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:51.782 | INFO | Uploaded 36GiB to GPU in 1.96006s (18.3668GiB/s) | 20260506 23:41:51.782 | INFO | Downloaded 34GiB to GPU in 1.98557s (17.1236GiB/s) | 20260506 23:41:56.920 | INFO | Analyzing chunk frames = [5381632,5382144) | 20260506 23:41:56.920 | INFO | MemInfo: { | 20260506 23:41:56.920 | INFO | "gpu" : " 17 GB", | 20260506 23:41:56.920 | INFO | "huge" : "", | 20260506 23:41:56.920 | INFO | "huge_shared" : " 27 GB", | 20260506 23:41:56.920 | INFO | "malloc" : " 14 GB", | 20260506 23:41:56.920 | INFO | "pinned" : " 29 GB" | 20260506 23:41:56.920 | INFO | } | 20260506 23:41:56.920 | INFO | Perf report for the last 5.13773 seconds: | 20260506 23:41:56.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:41:56.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:56.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:56.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:56.920 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:41:56.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:56.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:56.920 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:41:56.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:56.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:41:56.920 | INFO | Uploaded 36GiB to GPU in 2.00296s (17.9734GiB/s) | 20260506 23:41:56.920 | INFO | Downloaded 34GiB to GPU in 1.97506s (17.2146GiB/s) | 20260506 23:41:56.920 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:41:56.920 | INFO | Total mem usage: 18 GiB | 20260506 23:41:56.920 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:41:56.920 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -20.3%) | 20260506 23:41:56.920 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:41:56.920 | INFO | High Water Breakdown: | 20260506 23:41:56.920 | INFO | 61% : 9.13 GiB ( 80% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:41:56.920 | INFO | 38.9% : 5.82 GiB ( 87% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:41:56.920 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:41:56.920 | INFO | ------------------------------------------ | 20260506 23:41:56.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:41:56.920 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:41:56.920 | INFO | Total mem usage: 26.5 GiB | 20260506 23:41:56.920 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:41:56.920 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:41:56.920 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:41:56.920 | INFO | High Water Breakdown: | 20260506 23:41:56.920 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:41:56.920 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:41:56.920 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:41:56.920 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:41:56.920 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:41:56.920 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:41:56.920 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:41:56.920 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:41:56.920 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:41:56.920 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:41:56.920 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:41:56.920 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:41:56.920 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:41:56.920 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:41:56.920 | INFO | ------------------------------------------ | 20260506 23:41:56.920 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:41:56.920 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:41:56.921 | INFO | Total mem usage: 25.6 GiB | 20260506 23:41:56.921 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:41:56.921 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:41:56.921 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:41:56.921 | INFO | High Water Breakdown: | 20260506 23:41:56.921 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:41:56.921 | INFO | ------------------------------------------ | 20260506 23:41:56.921 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:41:56.921 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:41:56.921 | INFO | Total mem usage: 15.7 GiB | 20260506 23:41:56.921 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:41:56.921 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:41:56.921 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:41:56.921 | INFO | High Water Breakdown: | 20260506 23:41:56.921 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:41:56.921 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:41:56.921 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:41:56.921 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:41:56.921 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:41:56.921 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:41:56.921 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:41:56.921 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:41:56.921 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:41:56.921 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:41:56.921 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:41:56.921 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:41:56.921 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:41:56.921 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:41:56.921 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:41:56.921 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:41:56.921 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:41:56.921 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:41:56.921 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:41:56.921 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:41:56.921 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:41:56.921 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:41:56.921 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:41:56.921 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:41:56.921 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:41:56.921 | INFO | ------------------------------------------ | 20260506 23:41:56.921 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:41:56.921 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:41:56.921 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:41:56.921 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:41:56.921 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.65e+01, 1.34e+01 / 2.43e+00 | 20260506 23:41:56.921 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.01e-01, 1.03e-02 / 4.32e-03 | 20260506 23:41:56.921 | INFO | PerfReport: 11.68%, Baseline -- 3.62e+00 / 4.45e+00, 5.50e-01 / 2.10e-01 | 20260506 23:41:56.921 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.46e+00, 1.77e-01 / 1.04e-01 | 20260506 23:41:56.921 | INFO | PerfReport: 16.09%, DME -- 3.99e+01 / 4.52e+01, 4.38e+00 / 2.91e+00 | 20260506 23:41:56.921 | INFO | PerfReport: 57.38%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.35e+00 / 1.87e+00 | 20260506 23:41:56.921 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.18e+00 / 2.73e+00, 3.76e-01 / 2.15e-01 | 20260506 23:41:56.921 | INFO | PerfReport: 3.18%, Metrics -- 9.86e-01 / 1.33e+00, 2.65e-01 / 1.72e-01 | 20260506 23:41:56.921 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.35e-01 / 3.85e-01, 5.26e-02 / 1.74e-02 | 20260506 23:41:56.921 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:41:56.921 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:41:56.921 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:41:56.921 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 23:41:56.921 | INFO | PerfReport: 21.47%, Upload -- 2.07e+01 / 0.00e+00, 6.79e+00 / 0.00e+00 | 20260506 23:41:56.921 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260506 23:41:56.921 | INFO | PerfReport: 32.14%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 23:41:56.921 | INFO | PerfReport: 1.17%, LockUpload -- 1.13e+00 / 0.00e+00, 4.58e+00 / 0.00e+00 | 20260506 23:41:56.921 | INFO | PerfReport: 1.38%, LockDownload -- 1.34e+00 / 0.00e+00, 3.31e+00 / 0.00e+00 | 20260506 23:41:56.921 | INFO | PerfReport: 22.48%, LockCompute -- 2.17e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 23:41:56.921 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.61e-02 / 0.00e+00 | 20260506 23:41:57.018 | INFO | Starting a round of Baz IO | 20260506 23:42:00.140 | INFO | Populating RTMetrics report | 20260506 23:42:00.140 | INFO | RTMetrics report populated | 20260506 23:42:00.140 | INFO | Opening tmp RTMetrics json file | 20260506 23:42:00.144 | INFO | Serializing RTMetrics to json | 20260506 23:42:00.162 | INFO | Serialization done, swapping tmp file now... | 20260506 23:42:00.163 | INFO | RTMetrics done | 20260506 23:42:00.243 | INFO | Finished a round of Baz IO: Wrote 1.874 GiB in 3.224 seconds | 20260506 23:42:00.243 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.42% / 0.00% / 66.58% | 20260506 23:42:00.243 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.395866% / 3.97364e-06% / 4.76837e-05% | 20260506 23:42:00.243 | INFO | Baz IO overall: | 20260506 23:42:00.243 | INFO | BytesWritten : 2814454989380 | 20260506 23:42:00.243 | INFO | NumEvents : 1356656992367 | 20260506 23:42:02.057 | INFO | Analyzing chunk frames = [5382144,5382656) | 20260506 23:42:02.057 | INFO | MemInfo: { | 20260506 23:42:02.057 | INFO | "gpu" : " 17 GB", | 20260506 23:42:02.057 | INFO | "huge" : "", | 20260506 23:42:02.057 | INFO | "huge_shared" : " 27 GB", | 20260506 23:42:02.057 | INFO | "malloc" : " 14 GB", | 20260506 23:42:02.057 | INFO | "pinned" : " 29 GB" | 20260506 23:42:02.057 | INFO | } | 20260506 23:42:02.057 | INFO | Perf report for the last 5.13713 seconds: | 20260506 23:42:02.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:42:02.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:02.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:02.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:02.057 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260506 23:42:02.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:02.057 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260506 23:42:02.057 | INFO | PRE_HQ : 59.6%, 1.8, 56.7ms, 0 | 20260506 23:42:02.057 | INFO | BAZWRITER : 62.3%, 3.5, 111.2ms, 0 | 20260506 23:42:02.057 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:42:02.057 | INFO | Uploaded 36GiB to GPU in 2.0388s (17.6574GiB/s) | 20260506 23:42:02.057 | INFO | Downloaded 38GiB to GPU in 2.18873s (17.3617GiB/s) | 20260506 23:42:07.193 | INFO | Analyzing chunk frames = [5382656,5383168) | 20260506 23:42:07.193 | INFO | MemInfo: { | 20260506 23:42:07.193 | INFO | "gpu" : " 17 GB", | 20260506 23:42:07.193 | INFO | "huge" : "", | 20260506 23:42:07.193 | INFO | "huge_shared" : " 27 GB", | 20260506 23:42:07.193 | INFO | "malloc" : " 14 GB", | 20260506 23:42:07.193 | INFO | "pinned" : " 29 GB" | 20260506 23:42:07.193 | INFO | } | 20260506 23:42:07.193 | INFO | Perf report for the last 5.13596 seconds: | 20260506 23:42:07.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:42:07.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:07.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:07.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:07.193 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:42:07.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:07.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:07.193 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260506 23:42:07.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:07.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:07.193 | INFO | Uploaded 36GiB to GPU in 2.00101s (17.991GiB/s) | 20260506 23:42:07.193 | INFO | Downloaded 34GiB to GPU in 1.92467s (17.6654GiB/s) | 20260506 23:42:12.325 | INFO | Analyzing chunk frames = [5383168,5383680) | 20260506 23:42:12.325 | INFO | MemInfo: { | 20260506 23:42:12.325 | INFO | "gpu" : " 17 GB", | 20260506 23:42:12.325 | INFO | "huge" : "", | 20260506 23:42:12.325 | INFO | "huge_shared" : " 27 GB", | 20260506 23:42:12.325 | INFO | "malloc" : " 14 GB", | 20260506 23:42:12.325 | INFO | "pinned" : " 29 GB" | 20260506 23:42:12.325 | INFO | } | 20260506 23:42:12.325 | INFO | Perf report for the last 5.13207 seconds: | 20260506 23:42:12.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:42:12.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:12.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:12.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:12.325 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:42:12.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:12.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:12.325 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:42:12.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:12.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:12.325 | INFO | Uploaded 36GiB to GPU in 1.98817s (18.1071GiB/s) | 20260506 23:42:12.325 | INFO | Downloaded 34GiB to GPU in 1.91741s (17.7323GiB/s) | 20260506 23:42:17.469 | INFO | Analyzing chunk frames = [5383680,5384192) | 20260506 23:42:17.469 | INFO | MemInfo: { | 20260506 23:42:17.469 | INFO | "gpu" : " 17 GB", | 20260506 23:42:17.469 | INFO | "huge" : "", | 20260506 23:42:17.469 | INFO | "huge_shared" : " 27 GB", | 20260506 23:42:17.469 | INFO | "malloc" : " 14 GB", | 20260506 23:42:17.469 | INFO | "pinned" : " 29 GB" | 20260506 23:42:17.469 | INFO | } | 20260506 23:42:17.469 | INFO | Perf report for the last 5.14406 seconds: | 20260506 23:42:17.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:42:17.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:17.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:17.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:17.469 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260506 23:42:17.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:17.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:17.469 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:42:17.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:17.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:17.469 | INFO | Uploaded 36GiB to GPU in 1.95882s (18.3784GiB/s) | 20260506 23:42:17.469 | INFO | Downloaded 34GiB to GPU in 1.87565s (18.127GiB/s) | 20260506 23:42:22.612 | INFO | Analyzing chunk frames = [5384192,5384704) | 20260506 23:42:22.612 | INFO | MemInfo: { | 20260506 23:42:22.612 | INFO | "gpu" : " 17 GB", | 20260506 23:42:22.612 | INFO | "huge" : "", | 20260506 23:42:22.612 | INFO | "huge_shared" : " 27 GB", | 20260506 23:42:22.612 | INFO | "malloc" : " 14 GB", | 20260506 23:42:22.612 | INFO | "pinned" : " 29 GB" | 20260506 23:42:22.612 | INFO | } | 20260506 23:42:22.612 | INFO | Perf report for the last 5.14297 seconds: | 20260506 23:42:22.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:42:22.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:22.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:22.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:22.612 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 23:42:22.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:22.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:22.612 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:42:22.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:22.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:22.612 | INFO | Uploaded 36GiB to GPU in 2.00613s (17.945GiB/s) | 20260506 23:42:22.612 | INFO | Downloaded 34GiB to GPU in 1.91689s (17.737GiB/s) | 20260506 23:42:27.746 | INFO | Analyzing chunk frames = [5384704,5385216) | 20260506 23:42:27.746 | INFO | MemInfo: { | 20260506 23:42:27.746 | INFO | "gpu" : " 17 GB", | 20260506 23:42:27.746 | INFO | "huge" : "", | 20260506 23:42:27.746 | INFO | "huge_shared" : " 27 GB", | 20260506 23:42:27.746 | INFO | "malloc" : " 14 GB", | 20260506 23:42:27.746 | INFO | "pinned" : " 29 GB" | 20260506 23:42:27.746 | INFO | } | 20260506 23:42:27.746 | INFO | Perf report for the last 5.13425 seconds: | 20260506 23:42:27.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:42:27.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:27.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:27.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:27.746 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 23:42:27.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:27.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:27.746 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260506 23:42:27.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:27.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:27.746 | INFO | Uploaded 36GiB to GPU in 2.03259s (17.7114GiB/s) | 20260506 23:42:27.747 | INFO | Downloaded 34GiB to GPU in 1.94083s (17.5183GiB/s) | 20260506 23:42:32.894 | INFO | Analyzing chunk frames = [5385216,5385728) | 20260506 23:42:32.894 | INFO | MemInfo: { | 20260506 23:42:32.894 | INFO | "gpu" : " 17 GB", | 20260506 23:42:32.894 | INFO | "huge" : "", | 20260506 23:42:32.894 | INFO | "huge_shared" : " 27 GB", | 20260506 23:42:32.894 | INFO | "malloc" : " 14 GB", | 20260506 23:42:32.894 | INFO | "pinned" : " 29 GB" | 20260506 23:42:32.894 | INFO | } | 20260506 23:42:32.894 | INFO | Perf report for the last 5.14836 seconds: | 20260506 23:42:32.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:42:32.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:32.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:32.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:32.895 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 23:42:32.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:32.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:32.895 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:42:32.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:32.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:32.895 | INFO | Uploaded 36GiB to GPU in 1.97241s (18.2518GiB/s) | 20260506 23:42:32.895 | INFO | Downloaded 34GiB to GPU in 2.00206s (16.9825GiB/s) | 20260506 23:42:38.038 | INFO | Analyzing chunk frames = [5385728,5386240) | 20260506 23:42:38.038 | INFO | MemInfo: { | 20260506 23:42:38.038 | INFO | "gpu" : " 17 GB", | 20260506 23:42:38.038 | INFO | "huge" : "", | 20260506 23:42:38.038 | INFO | "huge_shared" : " 27 GB", | 20260506 23:42:38.038 | INFO | "malloc" : " 14 GB", | 20260506 23:42:38.038 | INFO | "pinned" : " 29 GB" | 20260506 23:42:38.038 | INFO | } | 20260506 23:42:38.038 | INFO | Perf report for the last 5.14354 seconds: | 20260506 23:42:38.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:42:38.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:38.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:38.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:38.038 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260506 23:42:38.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:38.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:38.038 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 23:42:38.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:38.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:38.038 | INFO | Uploaded 36GiB to GPU in 2.02706s (17.7597GiB/s) | 20260506 23:42:38.038 | INFO | Downloaded 34GiB to GPU in 1.99406s (17.0506GiB/s) | 20260506 23:42:38.136 | INFO | Starting a round of Baz IO | 20260506 23:42:41.259 | INFO | Populating RTMetrics report | 20260506 23:42:41.259 | INFO | RTMetrics report populated | 20260506 23:42:41.259 | INFO | Opening tmp RTMetrics json file | 20260506 23:42:41.260 | INFO | Serializing RTMetrics to json | 20260506 23:42:41.263 | INFO | Serialization done, swapping tmp file now... | 20260506 23:42:41.269 | INFO | RTMetrics done | 20260506 23:42:41.352 | INFO | Finished a round of Baz IO: Wrote 1.874 GiB in 3.215 seconds | 20260506 23:42:41.352 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.42% / 0.00% / 66.58% | 20260506 23:42:41.352 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.395099% / 0% / 7.54992e-05% | 20260506 23:42:41.352 | INFO | Baz IO overall: | 20260506 23:42:41.352 | INFO | BytesWritten : 2816467071794 | 20260506 23:42:41.352 | INFO | NumEvents : 1357614368361 | 20260506 23:42:43.174 | INFO | Analyzing chunk frames = [5386240,5386752) | 20260506 23:42:43.174 | INFO | MemInfo: { | 20260506 23:42:43.174 | INFO | "gpu" : " 17 GB", | 20260506 23:42:43.174 | INFO | "huge" : "", | 20260506 23:42:43.174 | INFO | "huge_shared" : " 27 GB", | 20260506 23:42:43.174 | INFO | "malloc" : " 14 GB", | 20260506 23:42:43.174 | INFO | "pinned" : " 29 GB" | 20260506 23:42:43.174 | INFO | } | 20260506 23:42:43.174 | INFO | Perf report for the last 5.13602 seconds: | 20260506 23:42:43.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:42:43.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:43.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:43.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:43.174 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260506 23:42:43.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:43.174 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260506 23:42:43.174 | INFO | PRE_HQ : 59.5%, 1.8, 56.3ms, 0 | 20260506 23:42:43.174 | INFO | BAZWRITER : 62.1%, 3.4, 109.6ms, 0 | 20260506 23:42:43.174 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:42:43.174 | INFO | Uploaded 36GiB to GPU in 2.04943s (17.5658GiB/s) | 20260506 23:42:43.174 | INFO | Downloaded 38GiB to GPU in 2.18802s (17.3673GiB/s) | 20260506 23:42:48.307 | INFO | Analyzing chunk frames = [5386752,5387264) | 20260506 23:42:48.307 | INFO | MemInfo: { | 20260506 23:42:48.307 | INFO | "gpu" : " 17 GB", | 20260506 23:42:48.307 | INFO | "huge" : "", | 20260506 23:42:48.307 | INFO | "huge_shared" : " 27 GB", | 20260506 23:42:48.307 | INFO | "malloc" : " 14 GB", | 20260506 23:42:48.307 | INFO | "pinned" : " 29 GB" | 20260506 23:42:48.307 | INFO | } | 20260506 23:42:48.307 | INFO | Perf report for the last 5.13261 seconds: | 20260506 23:42:48.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:42:48.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:48.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:48.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:48.307 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 23:42:48.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:48.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:48.307 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:42:48.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:48.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:48.307 | INFO | Uploaded 36GiB to GPU in 1.99769s (18.0208GiB/s) | 20260506 23:42:48.307 | INFO | Downloaded 34GiB to GPU in 1.92312s (17.6796GiB/s) | 20260506 23:42:53.443 | INFO | Analyzing chunk frames = [5387264,5387776) | 20260506 23:42:53.443 | INFO | MemInfo: { | 20260506 23:42:53.443 | INFO | "gpu" : " 17 GB", | 20260506 23:42:53.443 | INFO | "huge" : "", | 20260506 23:42:53.443 | INFO | "huge_shared" : " 27 GB", | 20260506 23:42:53.443 | INFO | "malloc" : " 14 GB", | 20260506 23:42:53.443 | INFO | "pinned" : " 29 GB" | 20260506 23:42:53.443 | INFO | } | 20260506 23:42:53.443 | INFO | Perf report for the last 5.13664 seconds: | 20260506 23:42:53.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:42:53.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:53.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:53.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:53.443 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260506 23:42:53.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:53.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:53.443 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:42:53.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:53.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:53.444 | INFO | Uploaded 36GiB to GPU in 1.98572s (18.1295GiB/s) | 20260506 23:42:53.444 | INFO | Downloaded 34GiB to GPU in 1.91904s (17.7172GiB/s) | 20260506 23:42:58.585 | INFO | Analyzing chunk frames = [5387776,5388288) | 20260506 23:42:58.585 | INFO | MemInfo: { | 20260506 23:42:58.585 | INFO | "gpu" : " 17 GB", | 20260506 23:42:58.585 | INFO | "huge" : "", | 20260506 23:42:58.585 | INFO | "huge_shared" : " 27 GB", | 20260506 23:42:58.585 | INFO | "malloc" : " 14 GB", | 20260506 23:42:58.585 | INFO | "pinned" : " 29 GB" | 20260506 23:42:58.585 | INFO | } | 20260506 23:42:58.585 | INFO | Perf report for the last 5.14173 seconds: | 20260506 23:42:58.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:42:58.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:58.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:58.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:58.585 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260506 23:42:58.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:58.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:58.585 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:42:58.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:58.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:42:58.586 | INFO | Uploaded 36GiB to GPU in 1.9753s (18.2251GiB/s) | 20260506 23:42:58.586 | INFO | Downloaded 34GiB to GPU in 1.87802s (18.1042GiB/s) | 20260506 23:43:03.732 | INFO | Analyzing chunk frames = [5388288,5388800) | 20260506 23:43:03.732 | INFO | MemInfo: { | 20260506 23:43:03.732 | INFO | "gpu" : " 17 GB", | 20260506 23:43:03.732 | INFO | "huge" : "", | 20260506 23:43:03.732 | INFO | "huge_shared" : " 27 GB", | 20260506 23:43:03.732 | INFO | "malloc" : " 14 GB", | 20260506 23:43:03.732 | INFO | "pinned" : " 29 GB" | 20260506 23:43:03.732 | INFO | } | 20260506 23:43:03.732 | INFO | Perf report for the last 5.14681 seconds: | 20260506 23:43:03.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:43:03.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:03.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:03.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:03.732 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260506 23:43:03.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:03.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:03.732 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:43:03.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:03.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:03.733 | INFO | Uploaded 36GiB to GPU in 1.99946s (18.0049GiB/s) | 20260506 23:43:03.733 | INFO | Downloaded 34GiB to GPU in 1.92657s (17.6479GiB/s) | 20260506 23:43:08.871 | INFO | Analyzing chunk frames = [5388800,5389312) | 20260506 23:43:08.872 | INFO | MemInfo: { | 20260506 23:43:08.872 | INFO | "gpu" : " 17 GB", | 20260506 23:43:08.872 | INFO | "huge" : "", | 20260506 23:43:08.872 | INFO | "huge_shared" : " 27 GB", | 20260506 23:43:08.872 | INFO | "malloc" : " 14 GB", | 20260506 23:43:08.872 | INFO | "pinned" : " 29 GB" | 20260506 23:43:08.872 | INFO | } | 20260506 23:43:08.872 | INFO | Perf report for the last 5.13976 seconds: | 20260506 23:43:08.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:43:08.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:08.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:08.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:08.872 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 23:43:08.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:08.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:08.872 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260506 23:43:08.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:08.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:08.872 | INFO | Uploaded 36GiB to GPU in 2.05284s (17.5367GiB/s) | 20260506 23:43:08.872 | INFO | Downloaded 34GiB to GPU in 1.95178s (17.42GiB/s) | 20260506 23:43:14.007 | INFO | Analyzing chunk frames = [5389312,5389824) | 20260506 23:43:14.007 | INFO | MemInfo: { | 20260506 23:43:14.007 | INFO | "gpu" : " 17 GB", | 20260506 23:43:14.007 | INFO | "huge" : "", | 20260506 23:43:14.007 | INFO | "huge_shared" : " 27 GB", | 20260506 23:43:14.007 | INFO | "malloc" : " 14 GB", | 20260506 23:43:14.007 | INFO | "pinned" : " 29 GB" | 20260506 23:43:14.007 | INFO | } | 20260506 23:43:14.007 | INFO | Perf report for the last 5.13547 seconds: | 20260506 23:43:14.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:43:14.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:14.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:14.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:14.007 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:43:14.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:14.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:14.007 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 23:43:14.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:14.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:14.007 | INFO | Uploaded 36GiB to GPU in 1.9799s (18.1827GiB/s) | 20260506 23:43:14.007 | INFO | Downloaded 34GiB to GPU in 1.96357s (17.3154GiB/s) | 20260506 23:43:19.143 | INFO | Analyzing chunk frames = [5389824,5390336) | 20260506 23:43:19.143 | INFO | MemInfo: { | 20260506 23:43:19.143 | INFO | "gpu" : " 17 GB", | 20260506 23:43:19.143 | INFO | "huge" : "", | 20260506 23:43:19.143 | INFO | "huge_shared" : " 27 GB", | 20260506 23:43:19.143 | INFO | "malloc" : " 14 GB", | 20260506 23:43:19.143 | INFO | "pinned" : " 29 GB" | 20260506 23:43:19.143 | INFO | } | 20260506 23:43:19.143 | INFO | Perf report for the last 5.13598 seconds: | 20260506 23:43:19.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:43:19.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:19.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:19.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:19.143 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:43:19.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:19.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:19.143 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:43:19.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:19.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:19.144 | INFO | Uploaded 36GiB to GPU in 1.98623s (18.1248GiB/s) | 20260506 23:43:19.144 | INFO | Downloaded 34GiB to GPU in 1.98701s (17.1112GiB/s) | 20260506 23:43:19.144 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:43:19.144 | INFO | Total mem usage: 18 GiB | 20260506 23:43:19.144 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:43:19.144 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 23:43:19.144 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:43:19.144 | INFO | High Water Breakdown: | 20260506 23:43:19.144 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:43:19.144 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:43:19.144 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:43:19.144 | INFO | ------------------------------------------ | 20260506 23:43:19.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:43:19.144 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:43:19.144 | INFO | Total mem usage: 26.5 GiB | 20260506 23:43:19.144 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:43:19.144 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:43:19.144 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:43:19.144 | INFO | High Water Breakdown: | 20260506 23:43:19.144 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:43:19.144 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:43:19.144 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:43:19.144 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:43:19.144 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:43:19.144 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:43:19.144 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:43:19.144 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:43:19.144 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:43:19.144 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:43:19.144 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:43:19.144 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:43:19.144 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:43:19.144 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:43:19.144 | INFO | ------------------------------------------ | 20260506 23:43:19.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:43:19.144 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:43:19.144 | INFO | Total mem usage: 25.6 GiB | 20260506 23:43:19.144 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:43:19.144 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:43:19.144 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:43:19.144 | INFO | High Water Breakdown: | 20260506 23:43:19.144 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:43:19.144 | INFO | ------------------------------------------ | 20260506 23:43:19.144 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:43:19.144 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:43:19.144 | INFO | Total mem usage: 15.7 GiB | 20260506 23:43:19.144 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:43:19.144 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:43:19.144 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:43:19.144 | INFO | High Water Breakdown: | 20260506 23:43:19.144 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:43:19.144 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:43:19.144 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:43:19.144 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:43:19.144 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:43:19.144 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:43:19.144 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:43:19.144 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:43:19.144 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:43:19.144 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:43:19.144 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:43:19.144 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:43:19.144 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:43:19.144 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:43:19.144 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:43:19.144 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:43:19.144 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:43:19.144 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:43:19.144 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:43:19.144 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:43:19.144 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:43:19.144 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:43:19.144 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:43:19.144 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:43:19.144 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:43:19.144 | INFO | ------------------------------------------ | 20260506 23:43:19.145 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:43:19.145 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:43:19.145 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:43:19.145 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:43:19.145 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.54e+01, 1.33e+01 / 2.03e+00 | 20260506 23:43:19.145 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.00e-01, 1.03e-02 / 5.13e-03 | 20260506 23:43:19.145 | INFO | PerfReport: 11.70%, Baseline -- 3.62e+00 / 4.40e+00, 5.50e-01 / 3.53e-01 | 20260506 23:43:19.145 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.46e+00, 1.77e-01 / 1.48e-01 | 20260506 23:43:19.145 | INFO | PerfReport: 16.08%, DME -- 3.98e+01 / 4.45e+01, 3.83e+00 / 2.14e+00 | 20260506 23:43:19.145 | INFO | PerfReport: 57.36%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.33e+00 / 1.62e+00 | 20260506 23:43:19.145 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.19e+00 / 2.72e+00, 3.78e-01 / 2.11e-01 | 20260506 23:43:19.145 | INFO | PerfReport: 3.18%, Metrics -- 9.86e-01 / 1.32e+00, 2.61e-01 / 1.86e-01 | 20260506 23:43:19.145 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.36e-01 / 3.92e-01, 5.26e-02 / 2.70e-02 | 20260506 23:43:19.145 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:43:19.145 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:43:19.145 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:43:19.145 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 23:43:19.145 | INFO | PerfReport: 21.61%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260506 23:43:19.145 | INFO | PerfReport: 21.23%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260506 23:43:19.145 | INFO | PerfReport: 32.09%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 23:43:19.145 | INFO | PerfReport: 1.35%, LockUpload -- 1.30e+00 / 0.00e+00, 4.75e+00 / 0.00e+00 | 20260506 23:43:19.145 | INFO | PerfReport: 1.17%, LockDownload -- 1.13e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260506 23:43:19.145 | INFO | PerfReport: 22.53%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:43:19.145 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260506 23:43:19.242 | INFO | Starting a round of Baz IO | 20260506 23:43:22.373 | INFO | Populating RTMetrics report | 20260506 23:43:22.373 | INFO | RTMetrics report populated | 20260506 23:43:22.373 | INFO | Opening tmp RTMetrics json file | 20260506 23:43:22.393 | INFO | Serializing RTMetrics to json | 20260506 23:43:22.394 | INFO | Serialization done, swapping tmp file now... | 20260506 23:43:22.395 | INFO | RTMetrics done | 20260506 23:43:22.467 | INFO | Finished a round of Baz IO: Wrote 1.873 GiB in 3.225 seconds | 20260506 23:43:22.467 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.42% / 0.00% / 66.58% | 20260506 23:43:22.467 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.394126% / 3.97364e-06% / 6.75519e-05% | 20260506 23:43:22.467 | INFO | Baz IO overall: | 20260506 23:43:22.467 | INFO | BytesWritten : 2818478175659 | 20260506 23:43:22.467 | INFO | NumEvents : 1358571420087 | 20260506 23:43:24.280 | INFO | Analyzing chunk frames = [5390336,5390848) | 20260506 23:43:24.280 | INFO | MemInfo: { | 20260506 23:43:24.280 | INFO | "gpu" : " 17 GB", | 20260506 23:43:24.280 | INFO | "huge" : "", | 20260506 23:43:24.280 | INFO | "huge_shared" : " 27 GB", | 20260506 23:43:24.280 | INFO | "malloc" : " 14 GB", | 20260506 23:43:24.280 | INFO | "pinned" : " 29 GB" | 20260506 23:43:24.280 | INFO | } | 20260506 23:43:24.280 | INFO | Perf report for the last 5.13676 seconds: | 20260506 23:43:24.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:43:24.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:24.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:24.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:24.280 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 23:43:24.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:24.280 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 23:43:24.280 | INFO | PRE_HQ : 59.6%, 1.8, 56.4ms, 0 | 20260506 23:43:24.280 | INFO | BAZWRITER : 62.4%, 3.5, 111.6ms, 0 | 20260506 23:43:24.280 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:43:24.280 | INFO | Uploaded 36GiB to GPU in 2.02687s (17.7614GiB/s) | 20260506 23:43:24.280 | INFO | Downloaded 38GiB to GPU in 2.17587s (17.4643GiB/s) | 20260506 23:43:29.416 | INFO | Analyzing chunk frames = [5390848,5391360) | 20260506 23:43:29.416 | INFO | MemInfo: { | 20260506 23:43:29.416 | INFO | "gpu" : " 17 GB", | 20260506 23:43:29.416 | INFO | "huge" : "", | 20260506 23:43:29.416 | INFO | "huge_shared" : " 27 GB", | 20260506 23:43:29.416 | INFO | "malloc" : " 14 GB", | 20260506 23:43:29.416 | INFO | "pinned" : " 29 GB" | 20260506 23:43:29.416 | INFO | } | 20260506 23:43:29.416 | INFO | Perf report for the last 5.13646 seconds: | 20260506 23:43:29.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:43:29.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:29.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:29.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:29.416 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:43:29.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:29.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:29.417 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:43:29.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:29.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:29.417 | INFO | Uploaded 36GiB to GPU in 1.99673s (18.0295GiB/s) | 20260506 23:43:29.417 | INFO | Downloaded 34GiB to GPU in 1.92443s (17.6676GiB/s) | 20260506 23:43:34.558 | INFO | Analyzing chunk frames = [5391360,5391872) | 20260506 23:43:34.558 | INFO | MemInfo: { | 20260506 23:43:34.558 | INFO | "gpu" : " 17 GB", | 20260506 23:43:34.558 | INFO | "huge" : "", | 20260506 23:43:34.559 | INFO | "huge_shared" : " 27 GB", | 20260506 23:43:34.559 | INFO | "malloc" : " 14 GB", | 20260506 23:43:34.559 | INFO | "pinned" : " 29 GB" | 20260506 23:43:34.559 | INFO | } | 20260506 23:43:34.559 | INFO | Perf report for the last 5.14221 seconds: | 20260506 23:43:34.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:43:34.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:34.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:34.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:34.559 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260506 23:43:34.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:34.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:34.559 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:43:34.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:34.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:34.559 | INFO | Uploaded 36GiB to GPU in 1.97641s (18.2148GiB/s) | 20260506 23:43:34.559 | INFO | Downloaded 34GiB to GPU in 1.91857s (17.7215GiB/s) | 20260506 23:43:39.704 | INFO | Analyzing chunk frames = [5391872,5392384) | 20260506 23:43:39.704 | INFO | MemInfo: { | 20260506 23:43:39.704 | INFO | "gpu" : " 17 GB", | 20260506 23:43:39.704 | INFO | "huge" : "", | 20260506 23:43:39.704 | INFO | "huge_shared" : " 27 GB", | 20260506 23:43:39.704 | INFO | "malloc" : " 14 GB", | 20260506 23:43:39.704 | INFO | "pinned" : " 29 GB" | 20260506 23:43:39.704 | INFO | } | 20260506 23:43:39.704 | INFO | Perf report for the last 5.14549 seconds: | 20260506 23:43:39.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:43:39.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:39.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:39.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:39.704 | INFO | ANALYSIS : 59.4%, 3.0, 92.2ms, 0 | 20260506 23:43:39.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:39.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:39.704 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:43:39.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:39.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:39.704 | INFO | Uploaded 36GiB to GPU in 1.94422s (18.5165GiB/s) | 20260506 23:43:39.704 | INFO | Downloaded 34GiB to GPU in 1.87911s (18.0936GiB/s) | 20260506 23:43:44.849 | INFO | Analyzing chunk frames = [5392384,5392896) | 20260506 23:43:44.849 | INFO | MemInfo: { | 20260506 23:43:44.849 | INFO | "gpu" : " 17 GB", | 20260506 23:43:44.849 | INFO | "huge" : "", | 20260506 23:43:44.849 | INFO | "huge_shared" : " 27 GB", | 20260506 23:43:44.849 | INFO | "malloc" : " 14 GB", | 20260506 23:43:44.849 | INFO | "pinned" : " 29 GB" | 20260506 23:43:44.849 | INFO | } | 20260506 23:43:44.849 | INFO | Perf report for the last 5.14534 seconds: | 20260506 23:43:44.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:43:44.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:44.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:44.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:44.850 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 23:43:44.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:44.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:44.850 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:43:44.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:44.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:44.850 | INFO | Uploaded 36GiB to GPU in 1.98704s (18.1174GiB/s) | 20260506 23:43:44.850 | INFO | Downloaded 34GiB to GPU in 1.9317s (17.6011GiB/s) | 20260506 23:43:49.992 | INFO | Analyzing chunk frames = [5392896,5393408) | 20260506 23:43:49.992 | INFO | MemInfo: { | 20260506 23:43:49.992 | INFO | "gpu" : " 17 GB", | 20260506 23:43:49.993 | INFO | "huge" : "", | 20260506 23:43:49.993 | INFO | "huge_shared" : " 27 GB", | 20260506 23:43:49.993 | INFO | "malloc" : " 14 GB", | 20260506 23:43:49.993 | INFO | "pinned" : " 29 GB" | 20260506 23:43:49.993 | INFO | } | 20260506 23:43:49.993 | INFO | Perf report for the last 5.14316 seconds: | 20260506 23:43:49.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:43:49.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:49.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:49.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:49.993 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 23:43:49.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:49.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:49.993 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:43:49.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:49.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:49.993 | INFO | Uploaded 36GiB to GPU in 2.05733s (17.4984GiB/s) | 20260506 23:43:49.993 | INFO | Downloaded 34GiB to GPU in 1.95589s (17.3834GiB/s) | 20260506 23:43:55.122 | INFO | Analyzing chunk frames = [5393408,5393920) | 20260506 23:43:55.122 | INFO | MemInfo: { | 20260506 23:43:55.122 | INFO | "gpu" : " 17 GB", | 20260506 23:43:55.122 | INFO | "huge" : "", | 20260506 23:43:55.122 | INFO | "huge_shared" : " 27 GB", | 20260506 23:43:55.122 | INFO | "malloc" : " 14 GB", | 20260506 23:43:55.122 | INFO | "pinned" : " 29 GB" | 20260506 23:43:55.122 | INFO | } | 20260506 23:43:55.122 | INFO | Perf report for the last 5.12965 seconds: | 20260506 23:43:55.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:43:55.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:55.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:55.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:55.122 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 23:43:55.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:55.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:55.122 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 23:43:55.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:55.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:43:55.123 | INFO | Uploaded 36GiB to GPU in 1.97001s (18.274GiB/s) | 20260506 23:43:55.123 | INFO | Downloaded 34GiB to GPU in 1.95513s (17.3902GiB/s) | 20260506 23:44:00.258 | INFO | Analyzing chunk frames = [5393920,5394432) | 20260506 23:44:00.258 | INFO | MemInfo: { | 20260506 23:44:00.258 | INFO | "gpu" : " 17 GB", | 20260506 23:44:00.258 | INFO | "huge" : "", | 20260506 23:44:00.258 | INFO | "huge_shared" : " 27 GB", | 20260506 23:44:00.258 | INFO | "malloc" : " 14 GB", | 20260506 23:44:00.258 | INFO | "pinned" : " 29 GB" | 20260506 23:44:00.258 | INFO | } | 20260506 23:44:00.258 | INFO | Perf report for the last 5.13605 seconds: | 20260506 23:44:00.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:44:00.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:00.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:00.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:00.258 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:44:00.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:00.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:00.258 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:44:00.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:00.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:00.259 | INFO | Uploaded 36GiB to GPU in 1.99665s (18.0302GiB/s) | 20260506 23:44:00.259 | INFO | Downloaded 34GiB to GPU in 1.97407s (17.2233GiB/s) | 20260506 23:44:00.356 | INFO | Starting a round of Baz IO | 20260506 23:44:03.488 | INFO | Populating RTMetrics report | 20260506 23:44:03.489 | INFO | RTMetrics report populated | 20260506 23:44:03.489 | INFO | Opening tmp RTMetrics json file | 20260506 23:44:03.492 | INFO | Serializing RTMetrics to json | 20260506 23:44:03.508 | INFO | Serialization done, swapping tmp file now... | 20260506 23:44:03.508 | INFO | RTMetrics done | 20260506 23:44:03.590 | INFO | Finished a round of Baz IO: Wrote 1.872 GiB in 3.233 seconds | 20260506 23:44:03.590 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.41% / 0.00% / 66.58% | 20260506 23:44:03.590 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.394913% / 3.97364e-06% / 6.35783e-05% | 20260506 23:44:03.590 | INFO | Baz IO overall: | 20260506 23:44:03.590 | INFO | BytesWritten : 2820488458682 | 20260506 23:44:03.590 | INFO | NumEvents : 1359528233944 | 20260506 23:44:05.396 | INFO | Analyzing chunk frames = [5394432,5394944) | 20260506 23:44:05.396 | INFO | MemInfo: { | 20260506 23:44:05.396 | INFO | "gpu" : " 17 GB", | 20260506 23:44:05.396 | INFO | "huge" : "", | 20260506 23:44:05.396 | INFO | "huge_shared" : " 27 GB", | 20260506 23:44:05.396 | INFO | "malloc" : " 14 GB", | 20260506 23:44:05.396 | INFO | "pinned" : " 29 GB" | 20260506 23:44:05.396 | INFO | } | 20260506 23:44:05.396 | INFO | Perf report for the last 5.13734 seconds: | 20260506 23:44:05.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:44:05.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:05.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:05.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:05.396 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 23:44:05.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:05.396 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 23:44:05.396 | INFO | PRE_HQ : 59.7%, 1.8, 56.5ms, 0 | 20260506 23:44:05.396 | INFO | BAZWRITER : 62.5%, 3.5, 111.3ms, 0 | 20260506 23:44:05.396 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:44:05.396 | INFO | Uploaded 36GiB to GPU in 2.03015s (17.7327GiB/s) | 20260506 23:44:05.396 | INFO | Downloaded 38GiB to GPU in 2.18685s (17.3766GiB/s) | 20260506 23:44:10.537 | INFO | Analyzing chunk frames = [5394944,5395456) | 20260506 23:44:10.537 | INFO | MemInfo: { | 20260506 23:44:10.537 | INFO | "gpu" : " 17 GB", | 20260506 23:44:10.537 | INFO | "huge" : "", | 20260506 23:44:10.537 | INFO | "huge_shared" : " 27 GB", | 20260506 23:44:10.537 | INFO | "malloc" : " 14 GB", | 20260506 23:44:10.537 | INFO | "pinned" : " 29 GB" | 20260506 23:44:10.537 | INFO | } | 20260506 23:44:10.537 | INFO | Perf report for the last 5.14117 seconds: | 20260506 23:44:10.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:44:10.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:10.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:10.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:10.537 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:44:10.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:10.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:10.537 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:44:10.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:10.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:10.537 | INFO | Uploaded 36GiB to GPU in 2.01891s (17.8314GiB/s) | 20260506 23:44:10.537 | INFO | Downloaded 34GiB to GPU in 1.92726s (17.6417GiB/s) | 20260506 23:44:15.683 | INFO | Analyzing chunk frames = [5395456,5395968) | 20260506 23:44:15.683 | INFO | MemInfo: { | 20260506 23:44:15.683 | INFO | "gpu" : " 17 GB", | 20260506 23:44:15.683 | INFO | "huge" : "", | 20260506 23:44:15.683 | INFO | "huge_shared" : " 27 GB", | 20260506 23:44:15.683 | INFO | "malloc" : " 14 GB", | 20260506 23:44:15.683 | INFO | "pinned" : " 29 GB" | 20260506 23:44:15.683 | INFO | } | 20260506 23:44:15.683 | INFO | Perf report for the last 5.14605 seconds: | 20260506 23:44:15.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:44:15.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:15.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:15.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:15.683 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 23:44:15.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:15.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:15.683 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:44:15.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:15.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:15.683 | INFO | Uploaded 36GiB to GPU in 1.98276s (18.1565GiB/s) | 20260506 23:44:15.683 | INFO | Downloaded 34GiB to GPU in 1.92777s (17.637GiB/s) | 20260506 23:44:20.819 | INFO | Analyzing chunk frames = [5395968,5396480) | 20260506 23:44:20.819 | INFO | MemInfo: { | 20260506 23:44:20.819 | INFO | "gpu" : " 17 GB", | 20260506 23:44:20.819 | INFO | "huge" : "", | 20260506 23:44:20.819 | INFO | "huge_shared" : " 27 GB", | 20260506 23:44:20.819 | INFO | "malloc" : " 14 GB", | 20260506 23:44:20.819 | INFO | "pinned" : " 29 GB" | 20260506 23:44:20.819 | INFO | } | 20260506 23:44:20.819 | INFO | Perf report for the last 5.13647 seconds: | 20260506 23:44:20.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:44:20.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:20.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:20.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:20.819 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 23:44:20.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:20.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:20.820 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:44:20.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:20.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:20.820 | INFO | Uploaded 36GiB to GPU in 2.00057s (17.9949GiB/s) | 20260506 23:44:20.820 | INFO | Downloaded 34GiB to GPU in 1.9087s (17.8132GiB/s) | 20260506 23:44:25.955 | INFO | Analyzing chunk frames = [5396480,5396992) | 20260506 23:44:25.955 | INFO | MemInfo: { | 20260506 23:44:25.955 | INFO | "gpu" : " 17 GB", | 20260506 23:44:25.955 | INFO | "huge" : "", | 20260506 23:44:25.955 | INFO | "huge_shared" : " 27 GB", | 20260506 23:44:25.955 | INFO | "malloc" : " 14 GB", | 20260506 23:44:25.955 | INFO | "pinned" : " 29 GB" | 20260506 23:44:25.955 | INFO | } | 20260506 23:44:25.955 | INFO | Perf report for the last 5.13584 seconds: | 20260506 23:44:25.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:44:25.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:25.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:25.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:25.955 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260506 23:44:25.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:25.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:25.955 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:44:25.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:25.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:25.956 | INFO | Uploaded 36GiB to GPU in 1.9987s (18.0117GiB/s) | 20260506 23:44:25.956 | INFO | Downloaded 34GiB to GPU in 1.92687s (17.6452GiB/s) | 20260506 23:44:31.092 | INFO | Analyzing chunk frames = [5396992,5397504) | 20260506 23:44:31.092 | INFO | MemInfo: { | 20260506 23:44:31.092 | INFO | "gpu" : " 17 GB", | 20260506 23:44:31.092 | INFO | "huge" : "", | 20260506 23:44:31.092 | INFO | "huge_shared" : " 27 GB", | 20260506 23:44:31.092 | INFO | "malloc" : " 14 GB", | 20260506 23:44:31.093 | INFO | "pinned" : " 29 GB" | 20260506 23:44:31.093 | INFO | } | 20260506 23:44:31.093 | INFO | Perf report for the last 5.13724 seconds: | 20260506 23:44:31.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:44:31.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:31.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:31.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:31.093 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 23:44:31.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:31.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:31.093 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260506 23:44:31.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:31.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:31.093 | INFO | Uploaded 36GiB to GPU in 2.02754s (17.7555GiB/s) | 20260506 23:44:31.093 | INFO | Downloaded 34GiB to GPU in 1.93449s (17.5757GiB/s) | 20260506 23:44:36.233 | INFO | Analyzing chunk frames = [5397504,5398016) | 20260506 23:44:36.233 | INFO | MemInfo: { | 20260506 23:44:36.233 | INFO | "gpu" : " 17 GB", | 20260506 23:44:36.233 | INFO | "huge" : "", | 20260506 23:44:36.233 | INFO | "huge_shared" : " 27 GB", | 20260506 23:44:36.233 | INFO | "malloc" : " 14 GB", | 20260506 23:44:36.233 | INFO | "pinned" : " 29 GB" | 20260506 23:44:36.233 | INFO | } | 20260506 23:44:36.233 | INFO | Perf report for the last 5.14023 seconds: | 20260506 23:44:36.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:44:36.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:36.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:36.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:36.233 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260506 23:44:36.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:36.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:36.233 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:44:36.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:36.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:36.233 | INFO | Uploaded 36GiB to GPU in 1.95063s (18.4556GiB/s) | 20260506 23:44:36.233 | INFO | Downloaded 34GiB to GPU in 1.98383s (17.1386GiB/s) | 20260506 23:44:41.368 | INFO | Analyzing chunk frames = [5398016,5398528) | 20260506 23:44:41.368 | INFO | MemInfo: { | 20260506 23:44:41.368 | INFO | "gpu" : " 17 GB", | 20260506 23:44:41.368 | INFO | "huge" : "", | 20260506 23:44:41.368 | INFO | "huge_shared" : " 27 GB", | 20260506 23:44:41.368 | INFO | "malloc" : " 14 GB", | 20260506 23:44:41.368 | INFO | "pinned" : " 29 GB" | 20260506 23:44:41.368 | INFO | } | 20260506 23:44:41.368 | INFO | Perf report for the last 5.1355 seconds: | 20260506 23:44:41.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:44:41.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:41.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:41.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:41.368 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:44:41.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:41.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:41.368 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:44:41.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:41.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:41.369 | INFO | Uploaded 36GiB to GPU in 1.96301s (18.3392GiB/s) | 20260506 23:44:41.369 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:44:41.369 | INFO | Total mem usage: 18 GiB | 20260506 23:44:41.369 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:44:41.369 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 23:44:41.369 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:44:41.369 | INFO | High Water Breakdown: | 20260506 23:44:41.369 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:44:41.369 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:44:41.369 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:44:41.369 | INFO | ------------------------------------------ | 20260506 23:44:41.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:44:41.369 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:44:41.369 | INFO | Total mem usage: 26.5 GiB | 20260506 23:44:41.369 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:44:41.369 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:44:41.369 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:44:41.369 | INFO | High Water Breakdown: | 20260506 23:44:41.369 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:44:41.369 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:44:41.369 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:44:41.369 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:44:41.369 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:44:41.369 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:44:41.369 | INFO | Downloaded 34GiB to GPU in 1.99348s (17.0556GiB/s) | 20260506 23:44:41.369 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:44:41.369 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:44:41.369 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:44:41.369 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:44:41.369 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:44:41.369 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:44:41.369 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:44:41.369 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:44:41.369 | INFO | ------------------------------------------ | 20260506 23:44:41.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:44:41.369 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:44:41.369 | INFO | Total mem usage: 25.6 GiB | 20260506 23:44:41.369 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:44:41.369 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:44:41.369 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:44:41.369 | INFO | High Water Breakdown: | 20260506 23:44:41.369 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:44:41.369 | INFO | ------------------------------------------ | 20260506 23:44:41.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:44:41.369 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:44:41.369 | INFO | Total mem usage: 15.7 GiB | 20260506 23:44:41.369 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:44:41.369 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:44:41.369 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:44:41.369 | INFO | High Water Breakdown: | 20260506 23:44:41.369 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:44:41.369 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:44:41.369 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:44:41.369 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:44:41.369 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:44:41.369 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:44:41.369 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:44:41.369 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:44:41.369 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:44:41.369 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:44:41.369 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:44:41.369 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:44:41.369 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:44:41.369 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:44:41.369 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:44:41.369 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:44:41.369 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:44:41.369 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:44:41.369 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:44:41.369 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:44:41.369 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:44:41.369 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:44:41.369 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:44:41.369 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:44:41.369 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:44:41.369 | INFO | ------------------------------------------ | 20260506 23:44:41.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:44:41.369 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:44:41.369 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:44:41.369 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:44:41.370 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.52e+01, 1.33e+01 / 1.84e+00 | 20260506 23:44:41.370 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.69e-03 / 8.30e-03 | 20260506 23:44:41.370 | INFO | PerfReport: 11.68%, Baseline -- 3.61e+00 / 4.37e+00, 5.46e-01 / 3.79e-01 | 20260506 23:44:41.370 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.46e+00, 1.77e-01 / 1.34e-01 | 20260506 23:44:41.370 | INFO | PerfReport: 16.01%, DME -- 3.97e+01 / 4.39e+01, 3.91e+00 / 1.82e+00 | 20260506 23:44:41.370 | INFO | PerfReport: 57.42%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.34e+00 / 1.64e+00 | 20260506 23:44:41.370 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.19e+00 / 2.74e+00, 3.81e-01 / 2.02e-01 | 20260506 23:44:41.370 | INFO | PerfReport: 3.19%, Metrics -- 9.87e-01 / 1.31e+00, 2.62e-01 / 1.92e-01 | 20260506 23:44:41.370 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.36e-01 / 3.89e-01, 5.26e-02 / 2.48e-02 | 20260506 23:44:41.370 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:44:41.370 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:44:41.370 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:44:41.370 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 23:44:41.370 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260506 23:44:41.370 | INFO | PerfReport: 21.24%, Download -- 2.05e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 23:44:41.370 | INFO | PerfReport: 32.05%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 23:44:41.370 | INFO | PerfReport: 1.36%, LockUpload -- 1.32e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260506 23:44:41.370 | INFO | PerfReport: 1.23%, LockDownload -- 1.19e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260506 23:44:41.370 | INFO | PerfReport: 22.58%, LockCompute -- 2.18e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 23:44:41.370 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260506 23:44:41.467 | INFO | Starting a round of Baz IO | 20260506 23:44:44.597 | INFO | Populating RTMetrics report | 20260506 23:44:44.598 | INFO | RTMetrics report populated | 20260506 23:44:44.598 | INFO | Opening tmp RTMetrics json file | 20260506 23:44:44.624 | INFO | Serializing RTMetrics to json | 20260506 23:44:44.628 | INFO | Serialization done, swapping tmp file now... | 20260506 23:44:44.628 | INFO | RTMetrics done | 20260506 23:44:44.696 | INFO | Finished a round of Baz IO: Wrote 1.873 GiB in 3.229 seconds | 20260506 23:44:44.696 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.41% / 0.00% / 66.59% | 20260506 23:44:44.696 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.393411% / 7.94729e-06% / 9.93411e-05% | 20260506 23:44:44.696 | INFO | Baz IO overall: | 20260506 23:44:44.696 | INFO | BytesWritten : 2822499657221 | 20260506 23:44:44.696 | INFO | NumEvents : 1360485954917 | 20260506 23:44:46.509 | INFO | Analyzing chunk frames = [5398528,5399040) | 20260506 23:44:46.509 | INFO | MemInfo: { | 20260506 23:44:46.509 | INFO | "gpu" : " 17 GB", | 20260506 23:44:46.509 | INFO | "huge" : "", | 20260506 23:44:46.509 | INFO | "huge_shared" : " 27 GB", | 20260506 23:44:46.509 | INFO | "malloc" : " 14 GB", | 20260506 23:44:46.509 | INFO | "pinned" : " 29 GB" | 20260506 23:44:46.509 | INFO | } | 20260506 23:44:46.509 | INFO | Perf report for the last 5.14076 seconds: | 20260506 23:44:46.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:44:46.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:46.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:46.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:46.509 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 23:44:46.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:46.509 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260506 23:44:46.509 | INFO | PRE_HQ : 59.6%, 1.8, 56.5ms, 0 | 20260506 23:44:46.509 | INFO | BAZWRITER : 62.4%, 3.5, 111.7ms, 0 | 20260506 23:44:46.509 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 23:44:46.509 | INFO | Uploaded 36GiB to GPU in 2.00736s (17.934GiB/s) | 20260506 23:44:46.509 | INFO | Downloaded 38GiB to GPU in 2.18454s (17.3949GiB/s) | 20260506 23:44:51.656 | INFO | Analyzing chunk frames = [5399040,5399552) | 20260506 23:44:51.656 | INFO | MemInfo: { | 20260506 23:44:51.656 | INFO | "gpu" : " 17 GB", | 20260506 23:44:51.656 | INFO | "huge" : "", | 20260506 23:44:51.656 | INFO | "huge_shared" : " 27 GB", | 20260506 23:44:51.656 | INFO | "malloc" : " 14 GB", | 20260506 23:44:51.656 | INFO | "pinned" : " 29 GB" | 20260506 23:44:51.656 | INFO | } | 20260506 23:44:51.656 | INFO | Perf report for the last 5.14693 seconds: | 20260506 23:44:51.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:44:51.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:51.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:51.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:51.656 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 23:44:51.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:51.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:51.656 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:44:51.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:51.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:51.656 | INFO | Uploaded 36GiB to GPU in 2.00284s (17.9745GiB/s) | 20260506 23:44:51.656 | INFO | Downloaded 34GiB to GPU in 1.93516s (17.5696GiB/s) | 20260506 23:44:56.801 | INFO | Analyzing chunk frames = [5399552,5400064) | 20260506 23:44:56.801 | INFO | MemInfo: { | 20260506 23:44:56.801 | INFO | "gpu" : " 17 GB", | 20260506 23:44:56.801 | INFO | "huge" : "", | 20260506 23:44:56.801 | INFO | "huge_shared" : " 27 GB", | 20260506 23:44:56.801 | INFO | "malloc" : " 14 GB", | 20260506 23:44:56.801 | INFO | "pinned" : " 29 GB" | 20260506 23:44:56.801 | INFO | } | 20260506 23:44:56.801 | INFO | Perf report for the last 5.14519 seconds: | 20260506 23:44:56.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:44:56.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:56.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:56.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:56.801 | INFO | ANALYSIS : 60.3%, 3.0, 93.6ms, 0 | 20260506 23:44:56.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:56.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:56.801 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:44:56.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:56.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:44:56.801 | INFO | Uploaded 36GiB to GPU in 1.9628s (18.3412GiB/s) | 20260506 23:44:56.801 | INFO | Downloaded 34GiB to GPU in 1.93492s (17.5718GiB/s) | 20260506 23:45:01.940 | INFO | Analyzing chunk frames = [5400064,5400576) | 20260506 23:45:01.940 | INFO | MemInfo: { | 20260506 23:45:01.940 | INFO | "gpu" : " 17 GB", | 20260506 23:45:01.940 | INFO | "huge" : "", | 20260506 23:45:01.940 | INFO | "huge_shared" : " 27 GB", | 20260506 23:45:01.940 | INFO | "malloc" : " 14 GB", | 20260506 23:45:01.940 | INFO | "pinned" : " 29 GB" | 20260506 23:45:01.940 | INFO | } | 20260506 23:45:01.940 | INFO | Perf report for the last 5.13875 seconds: | 20260506 23:45:01.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:45:01.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:01.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:01.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:01.940 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260506 23:45:01.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:01.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:01.940 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:45:01.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:01.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:01.940 | INFO | Uploaded 36GiB to GPU in 1.96197s (18.3489GiB/s) | 20260506 23:45:01.940 | INFO | Downloaded 34GiB to GPU in 1.91898s (17.7178GiB/s) | 20260506 23:45:07.076 | INFO | Analyzing chunk frames = [5400576,5401088) | 20260506 23:45:07.076 | INFO | MemInfo: { | 20260506 23:45:07.076 | INFO | "gpu" : " 17 GB", | 20260506 23:45:07.076 | INFO | "huge" : "", | 20260506 23:45:07.076 | INFO | "huge_shared" : " 27 GB", | 20260506 23:45:07.076 | INFO | "malloc" : " 14 GB", | 20260506 23:45:07.076 | INFO | "pinned" : " 29 GB" | 20260506 23:45:07.076 | INFO | } | 20260506 23:45:07.076 | INFO | Perf report for the last 5.13636 seconds: | 20260506 23:45:07.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:45:07.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:07.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:07.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:07.076 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260506 23:45:07.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:07.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:07.076 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:45:07.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:07.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:07.077 | INFO | Uploaded 36GiB to GPU in 1.97541s (18.2241GiB/s) | 20260506 23:45:07.077 | INFO | Downloaded 34GiB to GPU in 1.94536s (17.4775GiB/s) | 20260506 23:45:12.207 | INFO | Analyzing chunk frames = [5401088,5401600) | 20260506 23:45:12.207 | INFO | MemInfo: { | 20260506 23:45:12.207 | INFO | "gpu" : " 17 GB", | 20260506 23:45:12.207 | INFO | "huge" : "", | 20260506 23:45:12.207 | INFO | "huge_shared" : " 27 GB", | 20260506 23:45:12.207 | INFO | "malloc" : " 14 GB", | 20260506 23:45:12.207 | INFO | "pinned" : " 29 GB" | 20260506 23:45:12.207 | INFO | } | 20260506 23:45:12.207 | INFO | Perf report for the last 5.13063 seconds: | 20260506 23:45:12.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:45:12.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:12.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:12.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:12.207 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260506 23:45:12.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:12.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:12.207 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260506 23:45:12.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:12.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:12.207 | INFO | Uploaded 36GiB to GPU in 2.00281s (17.9747GiB/s) | 20260506 23:45:12.207 | INFO | Downloaded 34GiB to GPU in 1.93648s (17.5577GiB/s) | 20260506 23:45:17.345 | INFO | Analyzing chunk frames = [5401600,5402112) | 20260506 23:45:17.345 | INFO | MemInfo: { | 20260506 23:45:17.345 | INFO | "gpu" : " 17 GB", | 20260506 23:45:17.345 | INFO | "huge" : "", | 20260506 23:45:17.345 | INFO | "huge_shared" : " 27 GB", | 20260506 23:45:17.345 | INFO | "malloc" : " 14 GB", | 20260506 23:45:17.345 | INFO | "pinned" : " 29 GB" | 20260506 23:45:17.345 | INFO | } | 20260506 23:45:17.345 | INFO | Perf report for the last 5.13805 seconds: | 20260506 23:45:17.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:45:17.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:17.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:17.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:17.345 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 23:45:17.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:17.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:17.345 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260506 23:45:17.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:17.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:17.345 | INFO | Uploaded 36GiB to GPU in 1.95392s (18.4245GiB/s) | 20260506 23:45:17.345 | INFO | Downloaded 34GiB to GPU in 1.98872s (17.0964GiB/s) | 20260506 23:45:22.491 | INFO | Analyzing chunk frames = [5402112,5402624) | 20260506 23:45:22.491 | INFO | MemInfo: { | 20260506 23:45:22.491 | INFO | "gpu" : " 17 GB", | 20260506 23:45:22.491 | INFO | "huge" : "", | 20260506 23:45:22.491 | INFO | "huge_shared" : " 27 GB", | 20260506 23:45:22.491 | INFO | "malloc" : " 14 GB", | 20260506 23:45:22.491 | INFO | "pinned" : " 29 GB" | 20260506 23:45:22.491 | INFO | } | 20260506 23:45:22.491 | INFO | Perf report for the last 5.14618 seconds: | 20260506 23:45:22.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:45:22.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:22.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:22.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:22.491 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:45:22.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:22.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:22.491 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:45:22.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:22.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:22.491 | INFO | Uploaded 36GiB to GPU in 1.98732s (18.1149GiB/s) | 20260506 23:45:22.491 | INFO | Downloaded 34GiB to GPU in 1.97739s (17.1943GiB/s) | 20260506 23:45:22.590 | INFO | Starting a round of Baz IO | 20260506 23:45:25.718 | INFO | Populating RTMetrics report | 20260506 23:45:25.718 | INFO | RTMetrics report populated | 20260506 23:45:25.718 | INFO | Opening tmp RTMetrics json file | 20260506 23:45:25.720 | INFO | Serializing RTMetrics to json | 20260506 23:45:25.738 | INFO | Serialization done, swapping tmp file now... | 20260506 23:45:25.738 | INFO | RTMetrics done | 20260506 23:45:25.814 | INFO | Finished a round of Baz IO: Wrote 1.872 GiB in 3.223 seconds | 20260506 23:45:25.814 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.41% / 0.00% / 66.59% | 20260506 23:45:25.814 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.392008% / 3.97364e-06% / 6.75519e-05% | 20260506 23:45:25.814 | INFO | Baz IO overall: | 20260506 23:45:25.814 | INFO | BytesWritten : 2824509632220 | 20260506 23:45:25.814 | INFO | NumEvents : 1361442063912 | 20260506 23:45:27.631 | INFO | Analyzing chunk frames = [5402624,5403136) | 20260506 23:45:27.631 | INFO | MemInfo: { | 20260506 23:45:27.631 | INFO | "gpu" : " 17 GB", | 20260506 23:45:27.631 | INFO | "huge" : "", | 20260506 23:45:27.631 | INFO | "huge_shared" : " 27 GB", | 20260506 23:45:27.631 | INFO | "malloc" : " 14 GB", | 20260506 23:45:27.631 | INFO | "pinned" : " 29 GB" | 20260506 23:45:27.631 | INFO | } | 20260506 23:45:27.631 | INFO | Perf report for the last 5.13995 seconds: | 20260506 23:45:27.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:45:27.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:27.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:27.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:27.631 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 23:45:27.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:27.631 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 23:45:27.631 | INFO | PRE_HQ : 59.8%, 1.8, 56.5ms, 0 | 20260506 23:45:27.631 | INFO | BAZWRITER : 62.3%, 3.4, 108.7ms, 0 | 20260506 23:45:27.631 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:45:27.631 | INFO | Uploaded 36GiB to GPU in 2.04035s (17.644GiB/s) | 20260506 23:45:27.631 | INFO | Downloaded 38GiB to GPU in 2.17627s (17.4611GiB/s) | 20260506 23:45:32.768 | INFO | Analyzing chunk frames = [5403136,5403648) | 20260506 23:45:32.768 | INFO | MemInfo: { | 20260506 23:45:32.768 | INFO | "gpu" : " 17 GB", | 20260506 23:45:32.768 | INFO | "huge" : "", | 20260506 23:45:32.768 | INFO | "huge_shared" : " 27 GB", | 20260506 23:45:32.768 | INFO | "malloc" : " 14 GB", | 20260506 23:45:32.768 | INFO | "pinned" : " 29 GB" | 20260506 23:45:32.768 | INFO | } | 20260506 23:45:32.768 | INFO | Perf report for the last 5.13685 seconds: | 20260506 23:45:32.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:45:32.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:32.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:32.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:32.768 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:45:32.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:32.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:32.768 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:45:32.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:32.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:32.768 | INFO | Uploaded 36GiB to GPU in 2.00837s (17.925GiB/s) | 20260506 23:45:32.768 | INFO | Downloaded 34GiB to GPU in 1.93181s (17.6001GiB/s) | 20260506 23:45:37.906 | INFO | Analyzing chunk frames = [5403648,5404160) | 20260506 23:45:37.906 | INFO | MemInfo: { | 20260506 23:45:37.906 | INFO | "gpu" : " 17 GB", | 20260506 23:45:37.906 | INFO | "huge" : "", | 20260506 23:45:37.906 | INFO | "huge_shared" : " 27 GB", | 20260506 23:45:37.906 | INFO | "malloc" : " 14 GB", | 20260506 23:45:37.906 | INFO | "pinned" : " 29 GB" | 20260506 23:45:37.906 | INFO | } | 20260506 23:45:37.906 | INFO | Perf report for the last 5.13763 seconds: | 20260506 23:45:37.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:45:37.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:37.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:37.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:37.906 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:45:37.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:37.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:37.906 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:45:37.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:37.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:37.906 | INFO | Uploaded 36GiB to GPU in 1.97503s (18.2276GiB/s) | 20260506 23:45:37.906 | INFO | Downloaded 34GiB to GPU in 1.92498s (17.6625GiB/s) | 20260506 23:45:43.049 | INFO | Analyzing chunk frames = [5404160,5404672) | 20260506 23:45:43.049 | INFO | MemInfo: { | 20260506 23:45:43.049 | INFO | "gpu" : " 17 GB", | 20260506 23:45:43.049 | INFO | "huge" : "", | 20260506 23:45:43.049 | INFO | "huge_shared" : " 27 GB", | 20260506 23:45:43.049 | INFO | "malloc" : " 14 GB", | 20260506 23:45:43.049 | INFO | "pinned" : " 29 GB" | 20260506 23:45:43.049 | INFO | } | 20260506 23:45:43.049 | INFO | Perf report for the last 5.14345 seconds: | 20260506 23:45:43.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:45:43.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:43.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:43.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:43.049 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260506 23:45:43.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:43.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:43.049 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:45:43.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:43.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:43.049 | INFO | Uploaded 36GiB to GPU in 2.01677s (17.8503GiB/s) | 20260506 23:45:43.050 | INFO | Downloaded 34GiB to GPU in 1.90925s (17.8081GiB/s) | 20260506 23:45:48.183 | INFO | Analyzing chunk frames = [5404672,5405184) | 20260506 23:45:48.183 | INFO | MemInfo: { | 20260506 23:45:48.183 | INFO | "gpu" : " 17 GB", | 20260506 23:45:48.183 | INFO | "huge" : "", | 20260506 23:45:48.183 | INFO | "huge_shared" : " 27 GB", | 20260506 23:45:48.183 | INFO | "malloc" : " 14 GB", | 20260506 23:45:48.183 | INFO | "pinned" : " 29 GB" | 20260506 23:45:48.183 | INFO | } | 20260506 23:45:48.183 | INFO | Perf report for the last 5.13371 seconds: | 20260506 23:45:48.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:45:48.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:48.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:48.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:48.183 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 23:45:48.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:48.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:48.183 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:45:48.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:48.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:48.183 | INFO | Uploaded 36GiB to GPU in 2.00319s (17.9713GiB/s) | 20260506 23:45:48.183 | INFO | Downloaded 34GiB to GPU in 1.94605s (17.4713GiB/s) | 20260506 23:45:53.319 | INFO | Analyzing chunk frames = [5405184,5405696) | 20260506 23:45:53.319 | INFO | MemInfo: { | 20260506 23:45:53.319 | INFO | "gpu" : " 17 GB", | 20260506 23:45:53.319 | INFO | "huge" : "", | 20260506 23:45:53.319 | INFO | "huge_shared" : " 27 GB", | 20260506 23:45:53.319 | INFO | "malloc" : " 14 GB", | 20260506 23:45:53.319 | INFO | "pinned" : " 29 GB" | 20260506 23:45:53.319 | INFO | } | 20260506 23:45:53.319 | INFO | Perf report for the last 5.13655 seconds: | 20260506 23:45:53.320 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:45:53.320 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:53.320 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:53.320 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:53.320 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 23:45:53.320 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:53.320 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:53.320 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260506 23:45:53.320 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:53.320 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:53.320 | INFO | Uploaded 36GiB to GPU in 2.02419s (17.7849GiB/s) | 20260506 23:45:53.320 | INFO | Downloaded 34GiB to GPU in 1.9216s (17.6936GiB/s) | 20260506 23:45:58.463 | INFO | Analyzing chunk frames = [5405696,5406208) | 20260506 23:45:58.463 | INFO | MemInfo: { | 20260506 23:45:58.463 | INFO | "gpu" : " 17 GB", | 20260506 23:45:58.463 | INFO | "huge" : "", | 20260506 23:45:58.463 | INFO | "huge_shared" : " 27 GB", | 20260506 23:45:58.463 | INFO | "malloc" : " 14 GB", | 20260506 23:45:58.463 | INFO | "pinned" : " 29 GB" | 20260506 23:45:58.463 | INFO | } | 20260506 23:45:58.463 | INFO | Perf report for the last 5.14373 seconds: | 20260506 23:45:58.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:45:58.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:58.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:58.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:58.463 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 23:45:58.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:58.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:58.463 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 23:45:58.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:58.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:45:58.463 | INFO | Uploaded 36GiB to GPU in 1.95084s (18.4536GiB/s) | 20260506 23:45:58.463 | INFO | Downloaded 34GiB to GPU in 1.96996s (17.2593GiB/s) | 20260506 23:46:03.605 | INFO | Analyzing chunk frames = [5406208,5406720) | 20260506 23:46:03.605 | INFO | MemInfo: { | 20260506 23:46:03.605 | INFO | "gpu" : " 17 GB", | 20260506 23:46:03.605 | INFO | "huge" : "", | 20260506 23:46:03.605 | INFO | "huge_shared" : " 27 GB", | 20260506 23:46:03.605 | INFO | "malloc" : " 14 GB", | 20260506 23:46:03.605 | INFO | "pinned" : " 29 GB" | 20260506 23:46:03.605 | INFO | } | 20260506 23:46:03.605 | INFO | Perf report for the last 5.14211 seconds: | 20260506 23:46:03.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:46:03.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:03.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:03.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:03.605 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:46:03.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:03.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:03.605 | INFO | PRE_HQ : 31.2%, 1.0, 16.2ms, 0 | 20260506 23:46:03.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:03.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:03.606 | INFO | Uploaded 36GiB to GPU in 1.96438s (18.3264GiB/s) | 20260506 23:46:03.606 | INFO | Downloaded 34GiB to GPU in 1.98917s (17.0925GiB/s) | 20260506 23:46:03.606 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:46:03.606 | INFO | Total mem usage: 18 GiB | 20260506 23:46:03.606 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:46:03.606 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260506 23:46:03.606 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:46:03.606 | INFO | High Water Breakdown: | 20260506 23:46:03.606 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:46:03.606 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:46:03.606 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:46:03.606 | INFO | ------------------------------------------ | 20260506 23:46:03.606 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:46:03.606 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:46:03.606 | INFO | Total mem usage: 26.5 GiB | 20260506 23:46:03.606 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:46:03.606 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:46:03.606 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:46:03.606 | INFO | High Water Breakdown: | 20260506 23:46:03.606 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:46:03.606 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:46:03.606 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:46:03.606 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:46:03.606 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:46:03.606 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:46:03.606 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:46:03.606 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:46:03.606 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:46:03.606 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:46:03.606 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:46:03.606 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:46:03.606 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:46:03.606 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:46:03.606 | INFO | ------------------------------------------ | 20260506 23:46:03.606 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:46:03.606 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:46:03.606 | INFO | Total mem usage: 25.6 GiB | 20260506 23:46:03.606 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:46:03.606 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:46:03.606 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:46:03.606 | INFO | High Water Breakdown: | 20260506 23:46:03.606 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:46:03.606 | INFO | ------------------------------------------ | 20260506 23:46:03.606 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:46:03.606 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:46:03.606 | INFO | Total mem usage: 15.7 GiB | 20260506 23:46:03.606 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:46:03.606 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:46:03.606 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:46:03.606 | INFO | High Water Breakdown: | 20260506 23:46:03.606 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:46:03.606 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:46:03.606 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:46:03.606 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:46:03.606 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:46:03.606 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:46:03.606 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:46:03.606 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:46:03.606 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:46:03.606 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:46:03.606 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:46:03.606 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:46:03.606 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:46:03.606 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:46:03.606 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:46:03.606 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:46:03.606 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:46:03.606 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:46:03.606 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:46:03.606 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:46:03.607 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:46:03.607 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:46:03.607 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:46:03.607 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:46:03.607 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:46:03.607 | INFO | ------------------------------------------ | 20260506 23:46:03.607 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:46:03.607 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:46:03.607 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:46:03.607 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:46:03.607 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.57e+01, 1.33e+01 / 2.46e+00 | 20260506 23:46:03.607 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 9.98e-03 / 6.93e-03 | 20260506 23:46:03.607 | INFO | PerfReport: 11.68%, Baseline -- 3.62e+00 / 4.33e+00, 5.51e-01 / 3.93e-01 | 20260506 23:46:03.607 | INFO | PerfReport: 3.27%, Histogram -- 1.01e+00 / 1.46e+00, 1.83e-01 / 1.33e-01 | 20260506 23:46:03.607 | INFO | PerfReport: 16.09%, DME -- 3.99e+01 / 4.48e+01, 4.20e+00 / 2.61e+00 | 20260506 23:46:03.607 | INFO | PerfReport: 57.33%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.33e+00 / 1.81e+00 | 20260506 23:46:03.607 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.19e+00 / 2.79e+00, 3.83e-01 / 1.81e-01 | 20260506 23:46:03.607 | INFO | PerfReport: 3.19%, Metrics -- 9.88e-01 / 1.33e+00, 2.63e-01 / 1.58e-01 | 20260506 23:46:03.607 | INFO | PerfReport: 1.09%, PulseSerializing -- 3.36e-01 / 3.83e-01, 5.27e-02 / 2.05e-02 | 20260506 23:46:03.607 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:46:03.607 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:46:03.607 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:46:03.607 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260506 23:46:03.607 | INFO | PerfReport: 21.45%, Upload -- 2.07e+01 / 0.00e+00, 6.89e+00 / 0.00e+00 | 20260506 23:46:03.607 | INFO | PerfReport: 21.29%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260506 23:46:03.607 | INFO | PerfReport: 32.08%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 23:46:03.607 | INFO | PerfReport: 1.28%, LockUpload -- 1.24e+00 / 0.00e+00, 4.67e+00 / 0.00e+00 | 20260506 23:46:03.607 | INFO | PerfReport: 1.35%, LockDownload -- 1.30e+00 / 0.00e+00, 3.24e+00 / 0.00e+00 | 20260506 23:46:03.607 | INFO | PerfReport: 22.53%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:46:03.607 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260506 23:46:03.704 | INFO | Starting a round of Baz IO | 20260506 23:46:06.833 | INFO | Populating RTMetrics report | 20260506 23:46:06.834 | INFO | RTMetrics report populated | 20260506 23:46:06.834 | INFO | Opening tmp RTMetrics json file | 20260506 23:46:06.835 | INFO | Serializing RTMetrics to json | 20260506 23:46:06.848 | INFO | Serialization done, swapping tmp file now... | 20260506 23:46:06.855 | INFO | RTMetrics done | 20260506 23:46:06.928 | INFO | Finished a round of Baz IO: Wrote 1.872 GiB in 3.224 seconds | 20260506 23:46:06.928 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.41% / 0.00% / 66.59% | 20260506 23:46:06.928 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.391642% / 3.97364e-06% / 5.5631e-05% | 20260506 23:46:06.928 | INFO | Baz IO overall: | 20260506 23:46:06.928 | INFO | BytesWritten : 2826519701771 | 20260506 23:46:06.928 | INFO | NumEvents : 1362398224694 | 20260506 23:46:08.758 | INFO | Analyzing chunk frames = [5406720,5407232) | 20260506 23:46:08.758 | INFO | MemInfo: { | 20260506 23:46:08.758 | INFO | "gpu" : " 17 GB", | 20260506 23:46:08.758 | INFO | "huge" : "", | 20260506 23:46:08.758 | INFO | "huge_shared" : " 27 GB", | 20260506 23:46:08.758 | INFO | "malloc" : " 14 GB", | 20260506 23:46:08.758 | INFO | "pinned" : " 29 GB" | 20260506 23:46:08.758 | INFO | } | 20260506 23:46:08.758 | INFO | Perf report for the last 5.15222 seconds: | 20260506 23:46:08.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:46:08.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:08.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:08.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:08.758 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260506 23:46:08.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:08.758 | INFO | RT_METRICS : 45.2%, 1.0, 23.8ms, 0 | 20260506 23:46:08.758 | INFO | PRE_HQ : 60.4%, 1.9, 60.9ms, 0 | 20260506 23:46:08.758 | INFO | BAZWRITER : 62.1%, 3.6, 114.7ms, 0 | 20260506 23:46:08.758 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:46:08.758 | INFO | Uploaded 36GiB to GPU in 2.01895s (17.8311GiB/s) | 20260506 23:46:08.758 | INFO | Downloaded 38GiB to GPU in 2.17622s (17.4615GiB/s) | 20260506 23:46:13.889 | INFO | Analyzing chunk frames = [5407232,5407744) | 20260506 23:46:13.889 | INFO | MemInfo: { | 20260506 23:46:13.889 | INFO | "gpu" : " 17 GB", | 20260506 23:46:13.889 | INFO | "huge" : "", | 20260506 23:46:13.889 | INFO | "huge_shared" : " 27 GB", | 20260506 23:46:13.889 | INFO | "malloc" : " 14 GB", | 20260506 23:46:13.889 | INFO | "pinned" : " 29 GB" | 20260506 23:46:13.889 | INFO | } | 20260506 23:46:13.889 | INFO | Perf report for the last 5.13177 seconds: | 20260506 23:46:13.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:46:13.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:13.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:13.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:13.889 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 23:46:13.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:13.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:13.889 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:46:13.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:13.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:13.890 | INFO | Uploaded 36GiB to GPU in 2.04056s (17.6422GiB/s) | 20260506 23:46:13.890 | INFO | Downloaded 34GiB to GPU in 1.9376s (17.5475GiB/s) | 20260506 23:46:19.023 | INFO | Analyzing chunk frames = [5407744,5408256) | 20260506 23:46:19.023 | INFO | MemInfo: { | 20260506 23:46:19.023 | INFO | "gpu" : " 17 GB", | 20260506 23:46:19.023 | INFO | "huge" : "", | 20260506 23:46:19.023 | INFO | "huge_shared" : " 27 GB", | 20260506 23:46:19.023 | INFO | "malloc" : " 14 GB", | 20260506 23:46:19.023 | INFO | "pinned" : " 29 GB" | 20260506 23:46:19.023 | INFO | } | 20260506 23:46:19.023 | INFO | Perf report for the last 5.13383 seconds: | 20260506 23:46:19.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:46:19.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:19.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:19.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:19.023 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:46:19.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:19.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:19.023 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:46:19.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:19.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:19.023 | INFO | Uploaded 36GiB to GPU in 2.0031s (17.9722GiB/s) | 20260506 23:46:19.023 | INFO | Downloaded 34GiB to GPU in 1.93889s (17.5358GiB/s) | 20260506 23:46:24.160 | INFO | Analyzing chunk frames = [5408256,5408768) | 20260506 23:46:24.161 | INFO | MemInfo: { | 20260506 23:46:24.161 | INFO | "gpu" : " 17 GB", | 20260506 23:46:24.161 | INFO | "huge" : "", | 20260506 23:46:24.161 | INFO | "huge_shared" : " 27 GB", | 20260506 23:46:24.161 | INFO | "malloc" : " 14 GB", | 20260506 23:46:24.161 | INFO | "pinned" : " 29 GB" | 20260506 23:46:24.161 | INFO | } | 20260506 23:46:24.161 | INFO | Perf report for the last 5.13751 seconds: | 20260506 23:46:24.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:46:24.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:24.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:24.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:24.161 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 23:46:24.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:24.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:24.161 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:46:24.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:24.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:24.161 | INFO | Uploaded 36GiB to GPU in 2.00797s (17.9285GiB/s) | 20260506 23:46:24.161 | INFO | Downloaded 34GiB to GPU in 1.91049s (17.7964GiB/s) | 20260506 23:46:29.298 | INFO | Analyzing chunk frames = [5408768,5409280) | 20260506 23:46:29.298 | INFO | MemInfo: { | 20260506 23:46:29.298 | INFO | "gpu" : " 17 GB", | 20260506 23:46:29.299 | INFO | "huge" : "", | 20260506 23:46:29.299 | INFO | "huge_shared" : " 27 GB", | 20260506 23:46:29.299 | INFO | "malloc" : " 14 GB", | 20260506 23:46:29.299 | INFO | "pinned" : " 29 GB" | 20260506 23:46:29.299 | INFO | } | 20260506 23:46:29.299 | INFO | Perf report for the last 5.13792 seconds: | 20260506 23:46:29.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:46:29.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:29.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:29.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:29.299 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260506 23:46:29.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:29.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:29.299 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:46:29.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:29.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:29.299 | INFO | Uploaded 36GiB to GPU in 1.96837s (18.2892GiB/s) | 20260506 23:46:29.299 | INFO | Downloaded 34GiB to GPU in 1.93116s (17.606GiB/s) | 20260506 23:46:34.437 | INFO | Analyzing chunk frames = [5409280,5409792) | 20260506 23:46:34.437 | INFO | MemInfo: { | 20260506 23:46:34.437 | INFO | "gpu" : " 17 GB", | 20260506 23:46:34.437 | INFO | "huge" : "", | 20260506 23:46:34.437 | INFO | "huge_shared" : " 27 GB", | 20260506 23:46:34.437 | INFO | "malloc" : " 14 GB", | 20260506 23:46:34.437 | INFO | "pinned" : " 29 GB" | 20260506 23:46:34.437 | INFO | } | 20260506 23:46:34.437 | INFO | Perf report for the last 5.13817 seconds: | 20260506 23:46:34.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:46:34.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:34.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:34.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:34.437 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 23:46:34.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:34.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:34.437 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:46:34.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:34.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:34.437 | INFO | Uploaded 36GiB to GPU in 2.01534s (17.863GiB/s) | 20260506 23:46:34.437 | INFO | Downloaded 34GiB to GPU in 1.94026s (17.5234GiB/s) | 20260506 23:46:39.581 | INFO | Analyzing chunk frames = [5409792,5410304) | 20260506 23:46:39.582 | INFO | MemInfo: { | 20260506 23:46:39.582 | INFO | "gpu" : " 17 GB", | 20260506 23:46:39.582 | INFO | "huge" : "", | 20260506 23:46:39.582 | INFO | "huge_shared" : " 27 GB", | 20260506 23:46:39.582 | INFO | "malloc" : " 14 GB", | 20260506 23:46:39.582 | INFO | "pinned" : " 29 GB" | 20260506 23:46:39.582 | INFO | } | 20260506 23:46:39.582 | INFO | Perf report for the last 5.1448 seconds: | 20260506 23:46:39.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:46:39.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:39.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:39.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:39.582 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 23:46:39.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:39.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:39.582 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:46:39.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:39.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:39.582 | INFO | Uploaded 36GiB to GPU in 1.94851s (18.4756GiB/s) | 20260506 23:46:39.582 | INFO | Downloaded 34GiB to GPU in 1.98481s (17.1301GiB/s) | 20260506 23:46:44.717 | INFO | Analyzing chunk frames = [5410304,5410816) | 20260506 23:46:44.717 | INFO | MemInfo: { | 20260506 23:46:44.717 | INFO | "gpu" : " 17 GB", | 20260506 23:46:44.717 | INFO | "huge" : "", | 20260506 23:46:44.717 | INFO | "huge_shared" : " 27 GB", | 20260506 23:46:44.717 | INFO | "malloc" : " 14 GB", | 20260506 23:46:44.717 | INFO | "pinned" : " 29 GB" | 20260506 23:46:44.717 | INFO | } | 20260506 23:46:44.717 | INFO | Perf report for the last 5.135 seconds: | 20260506 23:46:44.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:46:44.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:44.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:44.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:44.717 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:46:44.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:44.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:44.717 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:46:44.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:44.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:44.717 | INFO | Uploaded 36GiB to GPU in 1.9902s (18.0886GiB/s) | 20260506 23:46:44.717 | INFO | Downloaded 34GiB to GPU in 1.97553s (17.2106GiB/s) | 20260506 23:46:44.815 | INFO | Starting a round of Baz IO | 20260506 23:46:47.945 | INFO | Populating RTMetrics report | 20260506 23:46:47.945 | INFO | RTMetrics report populated | 20260506 23:46:47.945 | INFO | Opening tmp RTMetrics json file | 20260506 23:46:47.950 | INFO | Serializing RTMetrics to json | 20260506 23:46:47.964 | INFO | Serialization done, swapping tmp file now... | 20260506 23:46:47.964 | INFO | RTMetrics done | 20260506 23:46:48.040 | INFO | Finished a round of Baz IO: Wrote 1.871 GiB in 3.224 seconds | 20260506 23:46:48.040 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.40% / 0.00% / 66.59% | 20260506 23:46:48.040 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.390506% / 3.97364e-06% / 5.5631e-05% | 20260506 23:46:48.040 | INFO | Baz IO overall: | 20260506 23:46:48.040 | INFO | BytesWritten : 2828529178271 | 20260506 23:46:48.040 | INFO | NumEvents : 1363354068421 | 20260506 23:46:49.860 | INFO | Analyzing chunk frames = [5410816,5411328) | 20260506 23:46:49.860 | INFO | MemInfo: { | 20260506 23:46:49.860 | INFO | "gpu" : " 17 GB", | 20260506 23:46:49.860 | INFO | "huge" : "", | 20260506 23:46:49.861 | INFO | "huge_shared" : " 27 GB", | 20260506 23:46:49.861 | INFO | "malloc" : " 14 GB", | 20260506 23:46:49.861 | INFO | "pinned" : " 29 GB" | 20260506 23:46:49.861 | INFO | } | 20260506 23:46:49.861 | INFO | Perf report for the last 5.14395 seconds: | 20260506 23:46:49.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:46:49.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:49.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:49.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:49.861 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 23:46:49.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:49.861 | INFO | RT_METRICS : 42.4%, 1.0, 22.3ms, 0 | 20260506 23:46:49.861 | INFO | PRE_HQ : 59.6%, 1.8, 56.4ms, 0 | 20260506 23:46:49.861 | INFO | BAZWRITER : 62.2%, 3.5, 111.6ms, 0 | 20260506 23:46:49.861 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:46:49.861 | INFO | Uploaded 36GiB to GPU in 2.05118s (17.5509GiB/s) | 20260506 23:46:49.861 | INFO | Downloaded 38GiB to GPU in 2.17922s (17.4374GiB/s) | 20260506 23:46:55.002 | INFO | Analyzing chunk frames = [5411328,5411840) | 20260506 23:46:55.002 | INFO | MemInfo: { | 20260506 23:46:55.002 | INFO | "gpu" : " 17 GB", | 20260506 23:46:55.002 | INFO | "huge" : "", | 20260506 23:46:55.002 | INFO | "huge_shared" : " 27 GB", | 20260506 23:46:55.002 | INFO | "malloc" : " 14 GB", | 20260506 23:46:55.002 | INFO | "pinned" : " 29 GB" | 20260506 23:46:55.002 | INFO | } | 20260506 23:46:55.002 | INFO | Perf report for the last 5.14181 seconds: | 20260506 23:46:55.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:46:55.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:55.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:55.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:55.003 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:46:55.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:55.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:55.003 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260506 23:46:55.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:55.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:46:55.003 | INFO | Uploaded 36GiB to GPU in 2.00327s (17.9706GiB/s) | 20260506 23:46:55.003 | INFO | Downloaded 34GiB to GPU in 1.92297s (17.681GiB/s) | 20260506 23:47:00.137 | INFO | Analyzing chunk frames = [5411840,5412352) | 20260506 23:47:00.137 | INFO | MemInfo: { | 20260506 23:47:00.137 | INFO | "gpu" : " 17 GB", | 20260506 23:47:00.137 | INFO | "huge" : "", | 20260506 23:47:00.137 | INFO | "huge_shared" : " 27 GB", | 20260506 23:47:00.137 | INFO | "malloc" : " 14 GB", | 20260506 23:47:00.137 | INFO | "pinned" : " 29 GB" | 20260506 23:47:00.137 | INFO | } | 20260506 23:47:00.137 | INFO | Perf report for the last 5.13467 seconds: | 20260506 23:47:00.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:47:00.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:00.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:00.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:00.137 | INFO | ANALYSIS : 60.5%, 3.0, 93.5ms, 0 | 20260506 23:47:00.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:00.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:00.137 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:47:00.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:00.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:00.137 | INFO | Uploaded 36GiB to GPU in 1.99623s (18.034GiB/s) | 20260506 23:47:00.137 | INFO | Downloaded 34GiB to GPU in 1.93302s (17.589GiB/s) | 20260506 23:47:05.274 | INFO | Analyzing chunk frames = [5412352,5412864) | 20260506 23:47:05.274 | INFO | MemInfo: { | 20260506 23:47:05.274 | INFO | "gpu" : " 17 GB", | 20260506 23:47:05.274 | INFO | "huge" : "", | 20260506 23:47:05.274 | INFO | "huge_shared" : " 27 GB", | 20260506 23:47:05.274 | INFO | "malloc" : " 14 GB", | 20260506 23:47:05.274 | INFO | "pinned" : " 29 GB" | 20260506 23:47:05.274 | INFO | } | 20260506 23:47:05.274 | INFO | Perf report for the last 5.13711 seconds: | 20260506 23:47:05.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:47:05.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:05.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:05.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:05.274 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 23:47:05.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:05.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:05.274 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:47:05.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:05.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:05.275 | INFO | Uploaded 36GiB to GPU in 1.99255s (18.0673GiB/s) | 20260506 23:47:05.275 | INFO | Downloaded 34GiB to GPU in 1.90914s (17.809GiB/s) | 20260506 23:47:10.413 | INFO | Analyzing chunk frames = [5412864,5413376) | 20260506 23:47:10.413 | INFO | MemInfo: { | 20260506 23:47:10.413 | INFO | "gpu" : " 17 GB", | 20260506 23:47:10.413 | INFO | "huge" : "", | 20260506 23:47:10.413 | INFO | "huge_shared" : " 27 GB", | 20260506 23:47:10.413 | INFO | "malloc" : " 14 GB", | 20260506 23:47:10.413 | INFO | "pinned" : " 29 GB" | 20260506 23:47:10.413 | INFO | } | 20260506 23:47:10.413 | INFO | Perf report for the last 5.13877 seconds: | 20260506 23:47:10.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:47:10.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:10.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:10.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:10.413 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 23:47:10.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:10.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:10.413 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:47:10.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:10.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:10.413 | INFO | Uploaded 36GiB to GPU in 1.99254s (18.0674GiB/s) | 20260506 23:47:10.413 | INFO | Downloaded 34GiB to GPU in 1.92732s (17.6411GiB/s) | 20260506 23:47:15.555 | INFO | Analyzing chunk frames = [5413376,5413888) | 20260506 23:47:15.555 | INFO | MemInfo: { | 20260506 23:47:15.555 | INFO | "gpu" : " 17 GB", | 20260506 23:47:15.555 | INFO | "huge" : "", | 20260506 23:47:15.555 | INFO | "huge_shared" : " 27 GB", | 20260506 23:47:15.555 | INFO | "malloc" : " 14 GB", | 20260506 23:47:15.555 | INFO | "pinned" : " 29 GB" | 20260506 23:47:15.555 | INFO | } | 20260506 23:47:15.555 | INFO | Perf report for the last 5.14161 seconds: | 20260506 23:47:15.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:47:15.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:15.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:15.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:15.555 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 23:47:15.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:15.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:15.555 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 23:47:15.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:15.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:15.555 | INFO | Uploaded 36GiB to GPU in 2.01306s (17.8832GiB/s) | 20260506 23:47:15.555 | INFO | Downloaded 34GiB to GPU in 1.92897s (17.626GiB/s) | 20260506 23:47:20.700 | INFO | Analyzing chunk frames = [5413888,5414400) | 20260506 23:47:20.700 | INFO | MemInfo: { | 20260506 23:47:20.700 | INFO | "gpu" : " 17 GB", | 20260506 23:47:20.700 | INFO | "huge" : "", | 20260506 23:47:20.700 | INFO | "huge_shared" : " 27 GB", | 20260506 23:47:20.700 | INFO | "malloc" : " 14 GB", | 20260506 23:47:20.700 | INFO | "pinned" : " 29 GB" | 20260506 23:47:20.700 | INFO | } | 20260506 23:47:20.700 | INFO | Perf report for the last 5.14524 seconds: | 20260506 23:47:20.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:47:20.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:20.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:20.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:20.700 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:47:20.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:20.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:20.700 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 23:47:20.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:20.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:20.700 | INFO | Uploaded 36GiB to GPU in 1.98205s (18.163GiB/s) | 20260506 23:47:20.700 | INFO | Downloaded 34GiB to GPU in 1.99613s (17.0329GiB/s) | 20260506 23:47:25.838 | INFO | Analyzing chunk frames = [5414400,5414912) | 20260506 23:47:25.838 | INFO | MemInfo: { | 20260506 23:47:25.838 | INFO | "gpu" : " 17 GB", | 20260506 23:47:25.838 | INFO | "huge" : "", | 20260506 23:47:25.838 | INFO | "huge_shared" : " 27 GB", | 20260506 23:47:25.838 | INFO | "malloc" : " 14 GB", | 20260506 23:47:25.838 | INFO | "pinned" : " 29 GB" | 20260506 23:47:25.838 | INFO | } | 20260506 23:47:25.838 | INFO | Perf report for the last 5.13795 seconds: | 20260506 23:47:25.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:47:25.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:25.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:25.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:25.838 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:47:25.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:25.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:25.838 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 23:47:25.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:25.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:25.838 | INFO | Uploaded 36GiB to GPU in 1.99171s (18.075GiB/s) | 20260506 23:47:25.838 | INFO | Downloaded 34GiB to GPU in 1.99306s (17.0592GiB/s) | 20260506 23:47:25.838 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:47:25.838 | INFO | Total mem usage: 18 GiB | 20260506 23:47:25.838 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:47:25.838 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.6%) | 20260506 23:47:25.838 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:47:25.838 | INFO | High Water Breakdown: | 20260506 23:47:25.838 | INFO | 60.3% : 8.65 GiB ( 75.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:47:25.838 | INFO | 39.5% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:47:25.838 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:47:25.838 | INFO | ------------------------------------------ | 20260506 23:47:25.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:47:25.839 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:47:25.839 | INFO | Total mem usage: 26.5 GiB | 20260506 23:47:25.839 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:47:25.839 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:47:25.839 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:47:25.839 | INFO | High Water Breakdown: | 20260506 23:47:25.839 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:47:25.839 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:47:25.839 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:47:25.839 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:47:25.839 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:47:25.839 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:47:25.839 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:47:25.839 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:47:25.839 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:47:25.839 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:47:25.839 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:47:25.839 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:47:25.839 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:47:25.839 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:47:25.839 | INFO | ------------------------------------------ | 20260506 23:47:25.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:47:25.839 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:47:25.839 | INFO | Total mem usage: 25.6 GiB | 20260506 23:47:25.839 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:47:25.839 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:47:25.839 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:47:25.839 | INFO | High Water Breakdown: | 20260506 23:47:25.839 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:47:25.839 | INFO | ------------------------------------------ | 20260506 23:47:25.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:47:25.839 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:47:25.839 | INFO | Total mem usage: 15.7 GiB | 20260506 23:47:25.839 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:47:25.839 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:47:25.839 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 23:47:25.839 | INFO | High Water Breakdown: | 20260506 23:47:25.839 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:47:25.839 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:47:25.839 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:47:25.839 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:47:25.839 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:47:25.839 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:47:25.839 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:47:25.839 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:47:25.839 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:47:25.839 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:47:25.839 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:47:25.839 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:47:25.839 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:47:25.839 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:47:25.839 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:47:25.839 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:47:25.839 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:47:25.839 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:47:25.839 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:47:25.839 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:47:25.839 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:47:25.839 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:47:25.839 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:47:25.839 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:47:25.839 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:47:25.839 | INFO | ------------------------------------------ | 20260506 23:47:25.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:47:25.839 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:47:25.839 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:47:25.839 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:47:25.839 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.62e+01, 1.33e+01 / 2.11e+00 | 20260506 23:47:25.839 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.00e-01, 9.69e-03 / 5.62e-03 | 20260506 23:47:25.839 | INFO | PerfReport: 11.65%, Baseline -- 3.61e+00 / 4.43e+00, 5.39e-01 / 2.71e-01 | 20260506 23:47:25.839 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.46e+00, 1.75e-01 / 1.22e-01 | 20260506 23:47:25.839 | INFO | PerfReport: 16.07%, DME -- 3.98e+01 / 4.48e+01, 4.08e+00 / 2.73e+00 | 20260506 23:47:25.839 | INFO | PerfReport: 57.43%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.36e+00 / 1.94e+00 | 20260506 23:47:25.839 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.18e+00 / 2.70e+00, 3.77e-01 / 2.32e-01 | 20260506 23:47:25.839 | INFO | PerfReport: 3.18%, Metrics -- 9.86e-01 / 1.31e+00, 2.64e-01 / 1.58e-01 | 20260506 23:47:25.839 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.35e-01 / 3.87e-01, 5.31e-02 / 2.08e-02 | 20260506 23:47:25.839 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:47:25.839 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:47:25.839 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:47:25.839 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 23:47:25.839 | INFO | PerfReport: 21.58%, Upload -- 2.08e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260506 23:47:25.839 | INFO | PerfReport: 21.29%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260506 23:47:25.839 | INFO | PerfReport: 32.06%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 23:47:25.839 | INFO | PerfReport: 1.35%, LockUpload -- 1.30e+00 / 0.00e+00, 4.78e+00 / 0.00e+00 | 20260506 23:47:25.839 | INFO | PerfReport: 1.24%, LockDownload -- 1.20e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260506 23:47:25.839 | INFO | PerfReport: 22.47%, LockCompute -- 2.17e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 23:47:25.839 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260506 23:47:25.935 | INFO | Starting a round of Baz IO | 20260506 23:47:29.068 | INFO | Populating RTMetrics report | 20260506 23:47:29.068 | INFO | RTMetrics report populated | 20260506 23:47:29.068 | INFO | Opening tmp RTMetrics json file | 20260506 23:47:29.087 | INFO | Serializing RTMetrics to json | 20260506 23:47:29.089 | INFO | Serialization done, swapping tmp file now... | 20260506 23:47:29.089 | INFO | RTMetrics done | 20260506 23:47:29.162 | INFO | Finished a round of Baz IO: Wrote 1.87 GiB in 3.226 seconds | 20260506 23:47:29.162 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.40% / 0.00% / 66.60% | 20260506 23:47:29.162 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.389687% / 1.19209e-05% / 5.5631e-05% | 20260506 23:47:29.162 | INFO | Baz IO overall: | 20260506 23:47:29.162 | INFO | BytesWritten : 2830537211358 | 20260506 23:47:29.162 | INFO | NumEvents : 1364309083773 | 20260506 23:47:30.972 | INFO | Analyzing chunk frames = [5414912,5415424) | 20260506 23:47:30.972 | INFO | MemInfo: { | 20260506 23:47:30.972 | INFO | "gpu" : " 17 GB", | 20260506 23:47:30.972 | INFO | "huge" : "", | 20260506 23:47:30.972 | INFO | "huge_shared" : " 27 GB", | 20260506 23:47:30.972 | INFO | "malloc" : " 14 GB", | 20260506 23:47:30.972 | INFO | "pinned" : " 29 GB" | 20260506 23:47:30.972 | INFO | } | 20260506 23:47:30.972 | INFO | Perf report for the last 5.1345 seconds: | 20260506 23:47:30.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:47:30.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:30.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:30.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:30.972 | INFO | ANALYSIS : 62.4%, 3.0, 96.8ms, 0 | 20260506 23:47:30.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:30.972 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 23:47:30.972 | INFO | PRE_HQ : 59.6%, 1.8, 56.5ms, 0 | 20260506 23:47:30.973 | INFO | BAZWRITER : 62.4%, 3.5, 112.0ms, 0 | 20260506 23:47:30.973 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 23:47:30.973 | INFO | Uploaded 36GiB to GPU in 2.03647s (17.6776GiB/s) | 20260506 23:47:30.973 | INFO | Downloaded 38GiB to GPU in 2.1765s (17.4592GiB/s) | 20260506 23:47:36.112 | INFO | Analyzing chunk frames = [5415424,5415936) | 20260506 23:47:36.112 | INFO | MemInfo: { | 20260506 23:47:36.112 | INFO | "gpu" : " 17 GB", | 20260506 23:47:36.112 | INFO | "huge" : "", | 20260506 23:47:36.112 | INFO | "huge_shared" : " 27 GB", | 20260506 23:47:36.112 | INFO | "malloc" : " 14 GB", | 20260506 23:47:36.112 | INFO | "pinned" : " 29 GB" | 20260506 23:47:36.112 | INFO | } | 20260506 23:47:36.112 | INFO | Perf report for the last 5.13964 seconds: | 20260506 23:47:36.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:47:36.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:36.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:36.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:36.112 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 23:47:36.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:36.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:36.112 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:47:36.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:36.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:36.112 | INFO | Uploaded 36GiB to GPU in 1.9962s (18.0342GiB/s) | 20260506 23:47:36.112 | INFO | Downloaded 34GiB to GPU in 1.9252s (17.6605GiB/s) | 20260506 23:47:41.249 | INFO | Analyzing chunk frames = [5415936,5416448) | 20260506 23:47:41.249 | INFO | MemInfo: { | 20260506 23:47:41.249 | INFO | "gpu" : " 17 GB", | 20260506 23:47:41.249 | INFO | "huge" : "", | 20260506 23:47:41.249 | INFO | "huge_shared" : " 27 GB", | 20260506 23:47:41.249 | INFO | "malloc" : " 14 GB", | 20260506 23:47:41.249 | INFO | "pinned" : " 29 GB" | 20260506 23:47:41.249 | INFO | } | 20260506 23:47:41.249 | INFO | Perf report for the last 5.13693 seconds: | 20260506 23:47:41.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:47:41.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:41.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:41.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:41.249 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260506 23:47:41.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:41.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:41.249 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:47:41.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:41.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:41.249 | INFO | Uploaded 36GiB to GPU in 2.00089s (17.9919GiB/s) | 20260506 23:47:41.249 | INFO | Downloaded 34GiB to GPU in 1.92916s (17.6242GiB/s) | 20260506 23:47:46.388 | INFO | Analyzing chunk frames = [5416448,5416960) | 20260506 23:47:46.388 | INFO | MemInfo: { | 20260506 23:47:46.388 | INFO | "gpu" : " 17 GB", | 20260506 23:47:46.388 | INFO | "huge" : "", | 20260506 23:47:46.388 | INFO | "huge_shared" : " 27 GB", | 20260506 23:47:46.388 | INFO | "malloc" : " 14 GB", | 20260506 23:47:46.388 | INFO | "pinned" : " 29 GB" | 20260506 23:47:46.388 | INFO | } | 20260506 23:47:46.388 | INFO | Perf report for the last 5.13926 seconds: | 20260506 23:47:46.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:47:46.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:46.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:46.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:46.388 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260506 23:47:46.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:46.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:46.388 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:47:46.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:46.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:46.389 | INFO | Uploaded 36GiB to GPU in 2.01867s (17.8335GiB/s) | 20260506 23:47:46.389 | INFO | Downloaded 34GiB to GPU in 1.91114s (17.7904GiB/s) | 20260506 23:47:51.526 | INFO | Analyzing chunk frames = [5416960,5417472) | 20260506 23:47:51.526 | INFO | MemInfo: { | 20260506 23:47:51.526 | INFO | "gpu" : " 17 GB", | 20260506 23:47:51.526 | INFO | "huge" : "", | 20260506 23:47:51.526 | INFO | "huge_shared" : " 27 GB", | 20260506 23:47:51.526 | INFO | "malloc" : " 14 GB", | 20260506 23:47:51.526 | INFO | "pinned" : " 29 GB" | 20260506 23:47:51.526 | INFO | } | 20260506 23:47:51.526 | INFO | Perf report for the last 5.13749 seconds: | 20260506 23:47:51.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:47:51.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:51.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:51.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:51.526 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 23:47:51.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:51.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:51.526 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:47:51.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:51.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:51.526 | INFO | Uploaded 36GiB to GPU in 1.98682s (18.1194GiB/s) | 20260506 23:47:51.526 | INFO | Downloaded 34GiB to GPU in 1.92525s (17.6601GiB/s) | 20260506 23:47:56.667 | INFO | Analyzing chunk frames = [5417472,5417984) | 20260506 23:47:56.667 | INFO | MemInfo: { | 20260506 23:47:56.667 | INFO | "gpu" : " 17 GB", | 20260506 23:47:56.667 | INFO | "huge" : "", | 20260506 23:47:56.667 | INFO | "huge_shared" : " 27 GB", | 20260506 23:47:56.667 | INFO | "malloc" : " 14 GB", | 20260506 23:47:56.667 | INFO | "pinned" : " 29 GB" | 20260506 23:47:56.667 | INFO | } | 20260506 23:47:56.667 | INFO | Perf report for the last 5.1414 seconds: | 20260506 23:47:56.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:47:56.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:56.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:56.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:56.667 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 23:47:56.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:56.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:56.667 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 23:47:56.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:56.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:47:56.667 | INFO | Uploaded 36GiB to GPU in 2.0092s (17.9175GiB/s) | 20260506 23:47:56.668 | INFO | Downloaded 34GiB to GPU in 1.94624s (17.4696GiB/s) | 20260506 23:48:01.817 | INFO | Analyzing chunk frames = [5417984,5418496) | 20260506 23:48:01.817 | INFO | MemInfo: { | 20260506 23:48:01.817 | INFO | "gpu" : " 17 GB", | 20260506 23:48:01.817 | INFO | "huge" : "", | 20260506 23:48:01.817 | INFO | "huge_shared" : " 27 GB", | 20260506 23:48:01.817 | INFO | "malloc" : " 14 GB", | 20260506 23:48:01.817 | INFO | "pinned" : " 29 GB" | 20260506 23:48:01.817 | INFO | } | 20260506 23:48:01.817 | INFO | Perf report for the last 5.14968 seconds: | 20260506 23:48:01.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:48:01.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:01.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:01.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:01.817 | INFO | ANALYSIS : 61.1%, 3.0, 95.4ms, 0 | 20260506 23:48:01.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:01.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:01.817 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 23:48:01.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:01.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:01.817 | INFO | Uploaded 36GiB to GPU in 1.97086s (18.2661GiB/s) | 20260506 23:48:01.817 | INFO | Downloaded 34GiB to GPU in 1.99053s (17.0809GiB/s) | 20260506 23:48:06.949 | INFO | Analyzing chunk frames = [5418496,5419008) | 20260506 23:48:06.949 | INFO | MemInfo: { | 20260506 23:48:06.949 | INFO | "gpu" : " 17 GB", | 20260506 23:48:06.949 | INFO | "huge" : "", | 20260506 23:48:06.949 | INFO | "huge_shared" : " 27 GB", | 20260506 23:48:06.949 | INFO | "malloc" : " 14 GB", | 20260506 23:48:06.949 | INFO | "pinned" : " 29 GB" | 20260506 23:48:06.949 | INFO | } | 20260506 23:48:06.949 | INFO | Perf report for the last 5.13225 seconds: | 20260506 23:48:06.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:48:06.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:06.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:06.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:06.949 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260506 23:48:06.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:06.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:06.949 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:48:06.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:06.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:06.949 | INFO | Uploaded 36GiB to GPU in 1.98405s (18.1447GiB/s) | 20260506 23:48:06.949 | INFO | Downloaded 34GiB to GPU in 1.98265s (17.1488GiB/s) | 20260506 23:48:07.048 | INFO | Starting a round of Baz IO | 20260506 23:48:10.172 | INFO | Populating RTMetrics report | 20260506 23:48:10.172 | INFO | RTMetrics report populated | 20260506 23:48:10.172 | INFO | Opening tmp RTMetrics json file | 20260506 23:48:10.178 | INFO | Serializing RTMetrics to json | 20260506 23:48:10.207 | INFO | Serialization done, swapping tmp file now... | 20260506 23:48:10.209 | INFO | RTMetrics done | 20260506 23:48:10.265 | INFO | Finished a round of Baz IO: Wrote 1.871 GiB in 3.217 seconds | 20260506 23:48:10.265 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.40% / 0.00% / 66.60% | 20260506 23:48:10.265 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.389973% / 3.97364e-06% / 4.37101e-05% | 20260506 23:48:10.265 | INFO | Baz IO overall: | 20260506 23:48:10.265 | INFO | BytesWritten : 2832546270636 | 20260506 23:48:10.265 | INFO | NumEvents : 1365264977304 | 20260506 23:48:12.085 | INFO | Analyzing chunk frames = [5419008,5419520) | 20260506 23:48:12.085 | INFO | MemInfo: { | 20260506 23:48:12.085 | INFO | "gpu" : " 17 GB", | 20260506 23:48:12.085 | INFO | "huge" : "", | 20260506 23:48:12.085 | INFO | "huge_shared" : " 27 GB", | 20260506 23:48:12.085 | INFO | "malloc" : " 14 GB", | 20260506 23:48:12.085 | INFO | "pinned" : " 29 GB" | 20260506 23:48:12.085 | INFO | } | 20260506 23:48:12.085 | INFO | Perf report for the last 5.13575 seconds: | 20260506 23:48:12.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:48:12.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:12.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:12.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:12.085 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260506 23:48:12.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:12.085 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260506 23:48:12.085 | INFO | PRE_HQ : 59.8%, 1.8, 56.5ms, 0 | 20260506 23:48:12.085 | INFO | BAZWRITER : 62.2%, 3.5, 110.3ms, 0 | 20260506 23:48:12.085 | INFO | BAZ_SAMPLING : 1.0%, 1.0, 0.5ms, 0 | 20260506 23:48:12.085 | INFO | Uploaded 36GiB to GPU in 2.0187s (17.8333GiB/s) | 20260506 23:48:12.085 | INFO | Downloaded 38GiB to GPU in 2.16865s (17.5224GiB/s) | 20260506 23:48:17.225 | INFO | Analyzing chunk frames = [5419520,5420032) | 20260506 23:48:17.225 | INFO | MemInfo: { | 20260506 23:48:17.225 | INFO | "gpu" : " 17 GB", | 20260506 23:48:17.226 | INFO | "huge" : "", | 20260506 23:48:17.226 | INFO | "huge_shared" : " 27 GB", | 20260506 23:48:17.226 | INFO | "malloc" : " 14 GB", | 20260506 23:48:17.226 | INFO | "pinned" : " 29 GB" | 20260506 23:48:17.226 | INFO | } | 20260506 23:48:17.226 | INFO | Perf report for the last 5.14073 seconds: | 20260506 23:48:17.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:48:17.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:17.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:17.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:17.226 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:48:17.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:17.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:17.226 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:48:17.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:17.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:17.226 | INFO | Uploaded 36GiB to GPU in 2.00716s (17.9358GiB/s) | 20260506 23:48:17.226 | INFO | Downloaded 34GiB to GPU in 1.93156s (17.6024GiB/s) | 20260506 23:48:22.362 | INFO | Analyzing chunk frames = [5420032,5420544) | 20260506 23:48:22.362 | INFO | MemInfo: { | 20260506 23:48:22.362 | INFO | "gpu" : " 17 GB", | 20260506 23:48:22.363 | INFO | "huge" : "", | 20260506 23:48:22.363 | INFO | "huge_shared" : " 27 GB", | 20260506 23:48:22.363 | INFO | "malloc" : " 14 GB", | 20260506 23:48:22.363 | INFO | "pinned" : " 29 GB" | 20260506 23:48:22.363 | INFO | } | 20260506 23:48:22.363 | INFO | Perf report for the last 5.13702 seconds: | 20260506 23:48:22.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:48:22.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:22.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:22.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:22.363 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260506 23:48:22.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:22.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:22.363 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:48:22.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:22.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:22.363 | INFO | Uploaded 36GiB to GPU in 1.96446s (18.3256GiB/s) | 20260506 23:48:22.363 | INFO | Downloaded 34GiB to GPU in 1.92421s (17.6696GiB/s) | 20260506 23:48:27.510 | INFO | Analyzing chunk frames = [5420544,5421056) | 20260506 23:48:27.510 | INFO | MemInfo: { | 20260506 23:48:27.510 | INFO | "gpu" : " 17 GB", | 20260506 23:48:27.510 | INFO | "huge" : "", | 20260506 23:48:27.510 | INFO | "huge_shared" : " 27 GB", | 20260506 23:48:27.510 | INFO | "malloc" : " 14 GB", | 20260506 23:48:27.510 | INFO | "pinned" : " 29 GB" | 20260506 23:48:27.510 | INFO | } | 20260506 23:48:27.510 | INFO | Perf report for the last 5.14742 seconds: | 20260506 23:48:27.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:48:27.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:27.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:27.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:27.510 | INFO | ANALYSIS : 60.0%, 3.0, 93.2ms, 0 | 20260506 23:48:27.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:27.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:27.510 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:48:27.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:27.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:27.510 | INFO | Uploaded 36GiB to GPU in 2.01436s (17.8717GiB/s) | 20260506 23:48:27.510 | INFO | Downloaded 34GiB to GPU in 1.90811s (17.8187GiB/s) | 20260506 23:48:32.648 | INFO | Analyzing chunk frames = [5421056,5421568) | 20260506 23:48:32.648 | INFO | MemInfo: { | 20260506 23:48:32.648 | INFO | "gpu" : " 17 GB", | 20260506 23:48:32.648 | INFO | "huge" : "", | 20260506 23:48:32.648 | INFO | "huge_shared" : " 27 GB", | 20260506 23:48:32.648 | INFO | "malloc" : " 14 GB", | 20260506 23:48:32.648 | INFO | "pinned" : " 29 GB" | 20260506 23:48:32.648 | INFO | } | 20260506 23:48:32.648 | INFO | Perf report for the last 5.13837 seconds: | 20260506 23:48:32.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:48:32.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:32.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:32.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:32.649 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 23:48:32.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:32.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:32.649 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:48:32.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:32.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:32.649 | INFO | Uploaded 36GiB to GPU in 2.00983s (17.9119GiB/s) | 20260506 23:48:32.649 | INFO | Downloaded 34GiB to GPU in 1.95216s (17.4166GiB/s) | 20260506 23:48:37.784 | INFO | Analyzing chunk frames = [5421568,5422080) | 20260506 23:48:37.784 | INFO | MemInfo: { | 20260506 23:48:37.784 | INFO | "gpu" : " 17 GB", | 20260506 23:48:37.784 | INFO | "huge" : "", | 20260506 23:48:37.785 | INFO | "huge_shared" : " 27 GB", | 20260506 23:48:37.785 | INFO | "malloc" : " 14 GB", | 20260506 23:48:37.785 | INFO | "pinned" : " 29 GB" | 20260506 23:48:37.785 | INFO | } | 20260506 23:48:37.785 | INFO | Perf report for the last 5.13615 seconds: | 20260506 23:48:37.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:48:37.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:37.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:37.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:37.785 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 23:48:37.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:37.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:37.785 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260506 23:48:37.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:37.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:37.785 | INFO | Uploaded 36GiB to GPU in 2.03602s (17.6816GiB/s) | 20260506 23:48:37.785 | INFO | Downloaded 34GiB to GPU in 1.96096s (17.3385GiB/s) | 20260506 23:48:42.926 | INFO | Analyzing chunk frames = [5422080,5422592) | 20260506 23:48:42.926 | INFO | MemInfo: { | 20260506 23:48:42.926 | INFO | "gpu" : " 17 GB", | 20260506 23:48:42.926 | INFO | "huge" : "", | 20260506 23:48:42.926 | INFO | "huge_shared" : " 27 GB", | 20260506 23:48:42.926 | INFO | "malloc" : " 14 GB", | 20260506 23:48:42.927 | INFO | "pinned" : " 29 GB" | 20260506 23:48:42.927 | INFO | } | 20260506 23:48:42.927 | INFO | Perf report for the last 5.14193 seconds: | 20260506 23:48:42.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:48:42.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:42.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:42.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:42.927 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:48:42.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:42.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:42.927 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:48:42.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:42.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:42.927 | INFO | Uploaded 36GiB to GPU in 1.96737s (18.2986GiB/s) | 20260506 23:48:42.927 | INFO | Downloaded 34GiB to GPU in 1.99303s (17.0595GiB/s) | 20260506 23:48:48.063 | INFO | Analyzing chunk frames = [5422592,5423104) | 20260506 23:48:48.063 | INFO | MemInfo: { | 20260506 23:48:48.063 | INFO | "gpu" : " 17 GB", | 20260506 23:48:48.063 | INFO | "huge" : "", | 20260506 23:48:48.063 | INFO | "huge_shared" : " 27 GB", | 20260506 23:48:48.063 | INFO | "malloc" : " 14 GB", | 20260506 23:48:48.063 | INFO | "pinned" : " 29 GB" | 20260506 23:48:48.063 | INFO | } | 20260506 23:48:48.063 | INFO | Perf report for the last 5.13659 seconds: | 20260506 23:48:48.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:48:48.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:48.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:48.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:48.063 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:48:48.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:48.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:48.063 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260506 23:48:48.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:48.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:48.063 | INFO | Uploaded 36GiB to GPU in 1.98291s (18.1551GiB/s) | 20260506 23:48:48.063 | INFO | Downloaded 34GiB to GPU in 1.97642s (17.2028GiB/s) | 20260506 23:48:48.063 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:48:48.063 | INFO | Total mem usage: 18 GiB | 20260506 23:48:48.063 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:48:48.064 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 23:48:48.064 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:48:48.064 | INFO | High Water Breakdown: | 20260506 23:48:48.064 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:48:48.064 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:48:48.064 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:48:48.064 | INFO | ------------------------------------------ | 20260506 23:48:48.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:48:48.064 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:48:48.064 | INFO | Total mem usage: 26.5 GiB | 20260506 23:48:48.064 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:48:48.064 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:48:48.064 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:48:48.064 | INFO | High Water Breakdown: | 20260506 23:48:48.064 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:48:48.064 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:48:48.064 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:48:48.064 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:48:48.064 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:48:48.064 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:48:48.064 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:48:48.064 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:48:48.064 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:48:48.064 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:48:48.064 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:48:48.064 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:48:48.064 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:48:48.064 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:48:48.064 | INFO | ------------------------------------------ | 20260506 23:48:48.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:48:48.064 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:48:48.064 | INFO | Total mem usage: 25.6 GiB | 20260506 23:48:48.064 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:48:48.064 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:48:48.064 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:48:48.064 | INFO | High Water Breakdown: | 20260506 23:48:48.064 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:48:48.064 | INFO | ------------------------------------------ | 20260506 23:48:48.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:48:48.064 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:48:48.064 | INFO | Total mem usage: 15.7 GiB | 20260506 23:48:48.064 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:48:48.064 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:48:48.064 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 23:48:48.064 | INFO | High Water Breakdown: | 20260506 23:48:48.064 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:48:48.064 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:48:48.064 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:48:48.064 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:48:48.064 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:48:48.064 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:48:48.064 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:48:48.064 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:48:48.064 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:48:48.064 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:48:48.064 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:48:48.064 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:48:48.064 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:48:48.064 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:48:48.064 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:48:48.064 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:48:48.064 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:48:48.064 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:48:48.064 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:48:48.064 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:48:48.064 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:48:48.064 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:48:48.064 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:48:48.064 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:48:48.064 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:48:48.064 | INFO | ------------------------------------------ | 20260506 23:48:48.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:48:48.064 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:48:48.064 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:48:48.064 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:48:48.064 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.60e+01, 1.34e+01 / 2.49e+00 | 20260506 23:48:48.064 | INFO | PerfReport: 0.30%, AggregateStats -- 9.15e-02 / 1.01e-01, 9.91e-03 / 8.93e-03 | 20260506 23:48:48.064 | INFO | PerfReport: 11.66%, Baseline -- 3.61e+00 / 4.35e+00, 5.39e-01 / 3.68e-01 | 20260506 23:48:48.064 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.44e+00, 1.78e-01 / 1.47e-01 | 20260506 23:48:48.064 | INFO | PerfReport: 16.09%, DME -- 3.99e+01 / 4.48e+01, 4.20e+00 / 2.88e+00 | 20260506 23:48:48.064 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.39e+00 / 1.77e+00 | 20260506 23:48:48.064 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.18e+00 / 2.73e+00, 3.75e-01 / 2.23e-01 | 20260506 23:48:48.064 | INFO | PerfReport: 3.17%, Metrics -- 9.84e-01 / 1.31e+00, 2.62e-01 / 1.64e-01 | 20260506 23:48:48.064 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.36e-01 / 3.80e-01, 5.32e-02 / 1.83e-02 | 20260506 23:48:48.064 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:48:48.064 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:48:48.064 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:48:48.064 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 23:48:48.064 | INFO | PerfReport: 21.55%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260506 23:48:48.064 | INFO | PerfReport: 21.28%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 23:48:48.064 | INFO | PerfReport: 32.08%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 23:48:48.064 | INFO | PerfReport: 1.29%, LockUpload -- 1.25e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260506 23:48:48.064 | INFO | PerfReport: 1.26%, LockDownload -- 1.22e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260506 23:48:48.064 | INFO | PerfReport: 22.53%, LockCompute -- 2.18e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:48:48.064 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260506 23:48:48.161 | INFO | Starting a round of Baz IO | 20260506 23:48:51.288 | INFO | Populating RTMetrics report | 20260506 23:48:51.288 | INFO | RTMetrics report populated | 20260506 23:48:51.288 | INFO | Opening tmp RTMetrics json file | 20260506 23:48:51.289 | INFO | Serializing RTMetrics to json | 20260506 23:48:51.290 | INFO | Serialization done, swapping tmp file now... | 20260506 23:48:51.291 | INFO | RTMetrics done | 20260506 23:48:51.382 | INFO | Finished a round of Baz IO: Wrote 1.87 GiB in 3.22 seconds | 20260506 23:48:51.382 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.40% / 0.00% / 66.60% | 20260506 23:48:51.382 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.388161% / 1.98682e-05% / 6.75519e-05% | 20260506 23:48:51.382 | INFO | Baz IO overall: | 20260506 23:48:51.382 | INFO | BytesWritten : 2834554409511 | 20260506 23:48:51.382 | INFO | NumEvents : 1366219875084 | 20260506 23:48:53.200 | INFO | Analyzing chunk frames = [5423104,5423616) | 20260506 23:48:53.200 | INFO | MemInfo: { | 20260506 23:48:53.200 | INFO | "gpu" : " 17 GB", | 20260506 23:48:53.200 | INFO | "huge" : "", | 20260506 23:48:53.200 | INFO | "huge_shared" : " 27 GB", | 20260506 23:48:53.200 | INFO | "malloc" : " 14 GB", | 20260506 23:48:53.200 | INFO | "pinned" : " 29 GB" | 20260506 23:48:53.200 | INFO | } | 20260506 23:48:53.200 | INFO | Perf report for the last 5.13718 seconds: | 20260506 23:48:53.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:48:53.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:53.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:53.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:53.200 | INFO | ANALYSIS : 62.2%, 3.0, 96.5ms, 0 | 20260506 23:48:53.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:53.200 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260506 23:48:53.200 | INFO | PRE_HQ : 60.8%, 2.0, 61.5ms, 0 | 20260506 23:48:53.200 | INFO | BAZWRITER : 62.3%, 3.6, 116.4ms, 0 | 20260506 23:48:53.200 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260506 23:48:53.201 | INFO | Uploaded 36GiB to GPU in 2.0449s (17.6048GiB/s) | 20260506 23:48:53.201 | INFO | Downloaded 38GiB to GPU in 2.18449s (17.3954GiB/s) | 20260506 23:48:58.342 | INFO | Analyzing chunk frames = [5423616,5424128) | 20260506 23:48:58.342 | INFO | MemInfo: { | 20260506 23:48:58.342 | INFO | "gpu" : " 17 GB", | 20260506 23:48:58.342 | INFO | "huge" : "", | 20260506 23:48:58.342 | INFO | "huge_shared" : " 27 GB", | 20260506 23:48:58.342 | INFO | "malloc" : " 14 GB", | 20260506 23:48:58.342 | INFO | "pinned" : " 29 GB" | 20260506 23:48:58.342 | INFO | } | 20260506 23:48:58.342 | INFO | Perf report for the last 5.14181 seconds: | 20260506 23:48:58.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:48:58.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:58.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:58.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:58.342 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 23:48:58.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:58.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:58.342 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260506 23:48:58.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:58.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:48:58.342 | INFO | Uploaded 36GiB to GPU in 1.99663s (18.0304GiB/s) | 20260506 23:48:58.343 | INFO | Downloaded 34GiB to GPU in 1.91687s (17.7373GiB/s) | 20260506 23:49:03.477 | INFO | Analyzing chunk frames = [5424128,5424640) | 20260506 23:49:03.477 | INFO | MemInfo: { | 20260506 23:49:03.477 | INFO | "gpu" : " 17 GB", | 20260506 23:49:03.477 | INFO | "huge" : "", | 20260506 23:49:03.477 | INFO | "huge_shared" : " 27 GB", | 20260506 23:49:03.477 | INFO | "malloc" : " 14 GB", | 20260506 23:49:03.477 | INFO | "pinned" : " 29 GB" | 20260506 23:49:03.477 | INFO | } | 20260506 23:49:03.477 | INFO | Perf report for the last 5.13524 seconds: | 20260506 23:49:03.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:49:03.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:03.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:03.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:03.477 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:49:03.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:03.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:03.478 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:49:03.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:03.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:03.478 | INFO | Uploaded 36GiB to GPU in 1.98793s (18.1093GiB/s) | 20260506 23:49:03.478 | INFO | Downloaded 34GiB to GPU in 1.92992s (17.6173GiB/s) | 20260506 23:49:08.619 | INFO | Analyzing chunk frames = [5424640,5425152) | 20260506 23:49:08.619 | INFO | MemInfo: { | 20260506 23:49:08.619 | INFO | "gpu" : " 17 GB", | 20260506 23:49:08.619 | INFO | "huge" : "", | 20260506 23:49:08.619 | INFO | "huge_shared" : " 27 GB", | 20260506 23:49:08.619 | INFO | "malloc" : " 14 GB", | 20260506 23:49:08.619 | INFO | "pinned" : " 29 GB" | 20260506 23:49:08.619 | INFO | } | 20260506 23:49:08.619 | INFO | Perf report for the last 5.14166 seconds: | 20260506 23:49:08.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:49:08.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:08.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:08.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:08.619 | INFO | ANALYSIS : 59.5%, 3.0, 92.2ms, 0 | 20260506 23:49:08.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:08.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:08.619 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:49:08.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:08.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:08.619 | INFO | Uploaded 36GiB to GPU in 1.96229s (18.3459GiB/s) | 20260506 23:49:08.619 | INFO | Downloaded 34GiB to GPU in 1.87781s (18.1062GiB/s) | 20260506 23:49:13.767 | INFO | Analyzing chunk frames = [5425152,5425664) | 20260506 23:49:13.767 | INFO | MemInfo: { | 20260506 23:49:13.767 | INFO | "gpu" : " 17 GB", | 20260506 23:49:13.768 | INFO | "huge" : "", | 20260506 23:49:13.768 | INFO | "huge_shared" : " 27 GB", | 20260506 23:49:13.768 | INFO | "malloc" : " 14 GB", | 20260506 23:49:13.768 | INFO | "pinned" : " 29 GB" | 20260506 23:49:13.768 | INFO | } | 20260506 23:49:13.768 | INFO | Perf report for the last 5.14853 seconds: | 20260506 23:49:13.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:49:13.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:13.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:13.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:13.768 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260506 23:49:13.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:13.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:13.768 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:49:13.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:13.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:13.768 | INFO | Uploaded 36GiB to GPU in 2.00289s (17.974GiB/s) | 20260506 23:49:13.768 | INFO | Downloaded 34GiB to GPU in 1.92092s (17.6998GiB/s) | 20260506 23:49:18.905 | INFO | Analyzing chunk frames = [5425664,5426176) | 20260506 23:49:18.905 | INFO | MemInfo: { | 20260506 23:49:18.905 | INFO | "gpu" : " 17 GB", | 20260506 23:49:18.905 | INFO | "huge" : "", | 20260506 23:49:18.905 | INFO | "huge_shared" : " 27 GB", | 20260506 23:49:18.905 | INFO | "malloc" : " 14 GB", | 20260506 23:49:18.905 | INFO | "pinned" : " 29 GB" | 20260506 23:49:18.905 | INFO | } | 20260506 23:49:18.906 | INFO | Perf report for the last 5.13788 seconds: | 20260506 23:49:18.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:49:18.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:18.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:18.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:18.906 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 23:49:18.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:18.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:18.906 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:49:18.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:18.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:18.906 | INFO | Uploaded 36GiB to GPU in 2.03101s (17.7252GiB/s) | 20260506 23:49:18.906 | INFO | Downloaded 34GiB to GPU in 1.93796s (17.5442GiB/s) | 20260506 23:49:24.036 | INFO | Analyzing chunk frames = [5426176,5426688) | 20260506 23:49:24.036 | INFO | MemInfo: { | 20260506 23:49:24.036 | INFO | "gpu" : " 17 GB", | 20260506 23:49:24.036 | INFO | "huge" : "", | 20260506 23:49:24.036 | INFO | "huge_shared" : " 27 GB", | 20260506 23:49:24.036 | INFO | "malloc" : " 14 GB", | 20260506 23:49:24.036 | INFO | "pinned" : " 29 GB" | 20260506 23:49:24.037 | INFO | } | 20260506 23:49:24.037 | INFO | Perf report for the last 5.131 seconds: | 20260506 23:49:24.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:49:24.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:24.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:24.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:24.037 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:49:24.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:24.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:24.037 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:49:24.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:24.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:24.037 | INFO | Uploaded 36GiB to GPU in 2.00231s (17.9792GiB/s) | 20260506 23:49:24.037 | INFO | Downloaded 34GiB to GPU in 1.99052s (17.081GiB/s) | 20260506 23:49:29.177 | INFO | Analyzing chunk frames = [5426688,5427200) | 20260506 23:49:29.177 | INFO | MemInfo: { | 20260506 23:49:29.177 | INFO | "gpu" : " 17 GB", | 20260506 23:49:29.177 | INFO | "huge" : "", | 20260506 23:49:29.177 | INFO | "huge_shared" : " 27 GB", | 20260506 23:49:29.177 | INFO | "malloc" : " 14 GB", | 20260506 23:49:29.177 | INFO | "pinned" : " 29 GB" | 20260506 23:49:29.177 | INFO | } | 20260506 23:49:29.177 | INFO | Perf report for the last 5.14055 seconds: | 20260506 23:49:29.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:49:29.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:29.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:29.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:29.177 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:49:29.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:29.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:29.177 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260506 23:49:29.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:29.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:29.177 | INFO | Uploaded 36GiB to GPU in 1.98679s (18.1196GiB/s) | 20260506 23:49:29.177 | INFO | Downloaded 34GiB to GPU in 2.00461s (16.9609GiB/s) | 20260506 23:49:29.275 | INFO | Starting a round of Baz IO | 20260506 23:49:32.411 | INFO | Populating RTMetrics report | 20260506 23:49:32.411 | INFO | RTMetrics report populated | 20260506 23:49:32.411 | INFO | Opening tmp RTMetrics json file | 20260506 23:49:32.412 | INFO | Serializing RTMetrics to json | 20260506 23:49:32.413 | INFO | Serialization done, swapping tmp file now... | 20260506 23:49:32.413 | INFO | RTMetrics done | 20260506 23:49:32.529 | INFO | Finished a round of Baz IO: Wrote 1.87 GiB in 3.254 seconds | 20260506 23:49:32.529 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.40% / 0.00% / 66.60% | 20260506 23:49:32.529 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.388722% / 3.97364e-06% / 5.5631e-05% | 20260506 23:49:32.529 | INFO | Baz IO overall: | 20260506 23:49:32.529 | INFO | BytesWritten : 2836562178873 | 20260506 23:49:32.529 | INFO | NumEvents : 1367175122505 | 20260506 23:49:34.316 | INFO | Analyzing chunk frames = [5427200,5427712) | 20260506 23:49:34.316 | INFO | MemInfo: { | 20260506 23:49:34.316 | INFO | "gpu" : " 17 GB", | 20260506 23:49:34.316 | INFO | "huge" : "", | 20260506 23:49:34.316 | INFO | "huge_shared" : " 27 GB", | 20260506 23:49:34.316 | INFO | "malloc" : " 14 GB", | 20260506 23:49:34.316 | INFO | "pinned" : " 29 GB" | 20260506 23:49:34.316 | INFO | } | 20260506 23:49:34.316 | INFO | Perf report for the last 5.13881 seconds: | 20260506 23:49:34.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:49:34.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:34.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:34.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:34.316 | INFO | ANALYSIS : 62.2%, 3.0, 96.5ms, 0 | 20260506 23:49:34.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:34.316 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260506 23:49:34.316 | INFO | PRE_HQ : 60.5%, 2.0, 61.5ms, 0 | 20260506 23:49:34.316 | INFO | BAZWRITER : 62.8%, 3.6, 116.8ms, 0 | 20260506 23:49:34.316 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260506 23:49:34.316 | INFO | Uploaded 36GiB to GPU in 2.03092s (17.7259GiB/s) | 20260506 23:49:34.316 | INFO | Downloaded 38GiB to GPU in 2.17574s (17.4654GiB/s) | 20260506 23:49:39.452 | INFO | Analyzing chunk frames = [5427712,5428224) | 20260506 23:49:39.452 | INFO | MemInfo: { | 20260506 23:49:39.452 | INFO | "gpu" : " 17 GB", | 20260506 23:49:39.452 | INFO | "huge" : "", | 20260506 23:49:39.452 | INFO | "huge_shared" : " 27 GB", | 20260506 23:49:39.452 | INFO | "malloc" : " 14 GB", | 20260506 23:49:39.452 | INFO | "pinned" : " 29 GB" | 20260506 23:49:39.452 | INFO | } | 20260506 23:49:39.452 | INFO | Perf report for the last 5.1362 seconds: | 20260506 23:49:39.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:49:39.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:39.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:39.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:39.452 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:49:39.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:39.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:39.452 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:49:39.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:39.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:39.452 | INFO | Uploaded 36GiB to GPU in 2.00806s (17.9277GiB/s) | 20260506 23:49:39.452 | INFO | Downloaded 34GiB to GPU in 1.93551s (17.5664GiB/s) | 20260506 23:49:44.597 | INFO | Analyzing chunk frames = [5428224,5428736) | 20260506 23:49:44.597 | INFO | MemInfo: { | 20260506 23:49:44.597 | INFO | "gpu" : " 17 GB", | 20260506 23:49:44.597 | INFO | "huge" : "", | 20260506 23:49:44.597 | INFO | "huge_shared" : " 27 GB", | 20260506 23:49:44.597 | INFO | "malloc" : " 14 GB", | 20260506 23:49:44.597 | INFO | "pinned" : " 29 GB" | 20260506 23:49:44.597 | INFO | } | 20260506 23:49:44.597 | INFO | Perf report for the last 5.14494 seconds: | 20260506 23:49:44.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:49:44.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:44.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:44.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:44.597 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260506 23:49:44.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:44.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:44.597 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260506 23:49:44.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:44.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:44.597 | INFO | Uploaded 36GiB to GPU in 1.97178s (18.2576GiB/s) | 20260506 23:49:44.597 | INFO | Downloaded 34GiB to GPU in 1.93148s (17.6031GiB/s) | 20260506 23:49:49.734 | INFO | Analyzing chunk frames = [5428736,5429248) | 20260506 23:49:49.734 | INFO | MemInfo: { | 20260506 23:49:49.734 | INFO | "gpu" : " 17 GB", | 20260506 23:49:49.734 | INFO | "huge" : "", | 20260506 23:49:49.734 | INFO | "huge_shared" : " 27 GB", | 20260506 23:49:49.734 | INFO | "malloc" : " 14 GB", | 20260506 23:49:49.734 | INFO | "pinned" : " 29 GB" | 20260506 23:49:49.734 | INFO | } | 20260506 23:49:49.735 | INFO | Perf report for the last 5.1374 seconds: | 20260506 23:49:49.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:49:49.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:49.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:49.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:49.735 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 23:49:49.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:49.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:49.735 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:49:49.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:49.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:49.735 | INFO | Uploaded 36GiB to GPU in 1.95437s (18.4203GiB/s) | 20260506 23:49:49.735 | INFO | Downloaded 34GiB to GPU in 1.8793s (18.0919GiB/s) | 20260506 23:49:54.877 | INFO | Analyzing chunk frames = [5429248,5429760) | 20260506 23:49:54.877 | INFO | MemInfo: { | 20260506 23:49:54.877 | INFO | "gpu" : " 17 GB", | 20260506 23:49:54.877 | INFO | "huge" : "", | 20260506 23:49:54.877 | INFO | "huge_shared" : " 27 GB", | 20260506 23:49:54.877 | INFO | "malloc" : " 14 GB", | 20260506 23:49:54.877 | INFO | "pinned" : " 29 GB" | 20260506 23:49:54.877 | INFO | } | 20260506 23:49:54.877 | INFO | Perf report for the last 5.14226 seconds: | 20260506 23:49:54.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:49:54.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:54.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:54.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:54.877 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 23:49:54.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:54.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:54.877 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:49:54.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:54.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:49:54.877 | INFO | Uploaded 36GiB to GPU in 1.98535s (18.1328GiB/s) | 20260506 23:49:54.877 | INFO | Downloaded 34GiB to GPU in 1.92146s (17.6949GiB/s) | 20260506 23:50:00.016 | INFO | Analyzing chunk frames = [5429760,5430272) | 20260506 23:50:00.016 | INFO | MemInfo: { | 20260506 23:50:00.016 | INFO | "gpu" : " 17 GB", | 20260506 23:50:00.016 | INFO | "huge" : "", | 20260506 23:50:00.016 | INFO | "huge_shared" : " 27 GB", | 20260506 23:50:00.016 | INFO | "malloc" : " 14 GB", | 20260506 23:50:00.016 | INFO | "pinned" : " 29 GB" | 20260506 23:50:00.017 | INFO | } | 20260506 23:50:00.017 | INFO | Perf report for the last 5.13976 seconds: | 20260506 23:50:00.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:50:00.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:00.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:00.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:00.017 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260506 23:50:00.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:00.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:00.017 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:50:00.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:00.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:00.017 | INFO | Uploaded 36GiB to GPU in 2.0343s (17.6965GiB/s) | 20260506 23:50:00.017 | INFO | Downloaded 34GiB to GPU in 1.9529s (17.41GiB/s) | 20260506 23:50:05.152 | INFO | Analyzing chunk frames = [5430272,5430784) | 20260506 23:50:05.152 | INFO | MemInfo: { | 20260506 23:50:05.152 | INFO | "gpu" : " 17 GB", | 20260506 23:50:05.153 | INFO | "huge" : "", | 20260506 23:50:05.153 | INFO | "huge_shared" : " 27 GB", | 20260506 23:50:05.153 | INFO | "malloc" : " 14 GB", | 20260506 23:50:05.153 | INFO | "pinned" : " 29 GB" | 20260506 23:50:05.153 | INFO | } | 20260506 23:50:05.153 | INFO | Perf report for the last 5.13608 seconds: | 20260506 23:50:05.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:50:05.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:05.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:05.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:05.153 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:50:05.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:05.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:05.153 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:50:05.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:05.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:05.153 | INFO | Uploaded 36GiB to GPU in 1.96453s (18.325GiB/s) | 20260506 23:50:05.153 | INFO | Downloaded 34GiB to GPU in 1.99993s (17.0006GiB/s) | 20260506 23:50:10.292 | INFO | Analyzing chunk frames = [5430784,5431296) | 20260506 23:50:10.292 | INFO | MemInfo: { | 20260506 23:50:10.292 | INFO | "gpu" : " 17 GB", | 20260506 23:50:10.292 | INFO | "huge" : "", | 20260506 23:50:10.292 | INFO | "huge_shared" : " 27 GB", | 20260506 23:50:10.292 | INFO | "malloc" : " 14 GB", | 20260506 23:50:10.292 | INFO | "pinned" : " 29 GB" | 20260506 23:50:10.292 | INFO | } | 20260506 23:50:10.293 | INFO | Perf report for the last 5.13986 seconds: | 20260506 23:50:10.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:50:10.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:10.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:10.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:10.293 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260506 23:50:10.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:10.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:10.293 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:50:10.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:10.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:10.293 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:50:10.293 | INFO | Total mem usage: 18 GiB | 20260506 23:50:10.293 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:50:10.293 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.6%) | 20260506 23:50:10.293 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:50:10.293 | INFO | High Water Breakdown: | 20260506 23:50:10.293 | INFO | 60.2% : 8.63 GiB ( 75.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:50:10.293 | INFO | Uploaded 36GiB to GPU in 1.99383s (18.0557GiB/s) | 20260506 23:50:10.293 | INFO | 39.7% : 5.69 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:50:10.293 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:50:10.293 | INFO | Downloaded 34GiB to GPU in 1.99675s (17.0277GiB/s) | 20260506 23:50:10.293 | INFO | ------------------------------------------ | 20260506 23:50:10.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:50:10.293 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:50:10.293 | INFO | Total mem usage: 26.5 GiB | 20260506 23:50:10.293 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:50:10.293 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:50:10.293 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:50:10.293 | INFO | High Water Breakdown: | 20260506 23:50:10.293 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:50:10.293 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:50:10.293 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:50:10.293 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:50:10.293 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:50:10.293 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:50:10.293 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:50:10.293 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:50:10.293 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:50:10.293 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:50:10.293 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:50:10.293 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:50:10.293 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:50:10.293 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:50:10.293 | INFO | ------------------------------------------ | 20260506 23:50:10.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:50:10.293 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:50:10.293 | INFO | Total mem usage: 25.6 GiB | 20260506 23:50:10.293 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:50:10.293 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:50:10.293 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:50:10.293 | INFO | High Water Breakdown: | 20260506 23:50:10.293 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:50:10.293 | INFO | ------------------------------------------ | 20260506 23:50:10.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:50:10.293 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:50:10.293 | INFO | Total mem usage: 15.7 GiB | 20260506 23:50:10.293 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:50:10.293 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:50:10.293 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 23:50:10.293 | INFO | High Water Breakdown: | 20260506 23:50:10.293 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:50:10.293 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:50:10.294 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:50:10.294 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:50:10.294 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:50:10.294 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:50:10.294 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:50:10.294 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:50:10.294 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:50:10.294 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:50:10.294 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:50:10.294 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:50:10.294 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:50:10.294 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:50:10.294 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:50:10.294 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:50:10.294 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:50:10.294 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:50:10.294 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:50:10.294 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:50:10.294 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:50:10.294 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:50:10.294 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:50:10.294 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:50:10.294 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:50:10.294 | INFO | ------------------------------------------ | 20260506 23:50:10.294 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:50:10.294 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:50:10.294 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:50:10.294 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:50:10.294 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.56e+01, 1.33e+01 / 2.48e+00 | 20260506 23:50:10.294 | INFO | PerfReport: 0.30%, AggregateStats -- 9.15e-02 / 1.02e-01, 1.04e-02 / 5.26e-03 | 20260506 23:50:10.294 | INFO | PerfReport: 11.68%, Baseline -- 3.62e+00 / 4.44e+00, 5.51e-01 / 2.91e-01 | 20260506 23:50:10.294 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.49e+00, 1.77e-01 / 1.03e-01 | 20260506 23:50:10.294 | INFO | PerfReport: 16.09%, DME -- 3.99e+01 / 4.46e+01, 3.98e+00 / 2.85e+00 | 20260506 23:50:10.294 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.32e+00 / 1.77e+00 | 20260506 23:50:10.294 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.19e+00 / 2.78e+00, 3.87e-01 / 1.87e-01 | 20260506 23:50:10.294 | INFO | PerfReport: 3.18%, Metrics -- 9.86e-01 / 1.32e+00, 2.61e-01 / 1.71e-01 | 20260506 23:50:10.294 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.35e-01 / 3.86e-01, 5.24e-02 / 2.49e-02 | 20260506 23:50:10.294 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:50:10.294 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:50:10.294 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:50:10.294 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 23:50:10.294 | INFO | PerfReport: 21.56%, Upload -- 2.08e+01 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260506 23:50:10.294 | INFO | PerfReport: 21.29%, Download -- 2.05e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260506 23:50:10.294 | INFO | PerfReport: 32.09%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 23:50:10.294 | INFO | PerfReport: 1.26%, LockUpload -- 1.22e+00 / 0.00e+00, 4.72e+00 / 0.00e+00 | 20260506 23:50:10.294 | INFO | PerfReport: 1.27%, LockDownload -- 1.23e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260506 23:50:10.294 | INFO | PerfReport: 22.51%, LockCompute -- 2.17e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 23:50:10.294 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260506 23:50:10.391 | INFO | Starting a round of Baz IO | 20260506 23:50:13.514 | INFO | Populating RTMetrics report | 20260506 23:50:13.514 | INFO | RTMetrics report populated | 20260506 23:50:13.514 | INFO | Opening tmp RTMetrics json file | 20260506 23:50:13.516 | INFO | Serializing RTMetrics to json | 20260506 23:50:13.535 | INFO | Serialization done, swapping tmp file now... | 20260506 23:50:13.535 | INFO | RTMetrics done | 20260506 23:50:13.611 | INFO | Finished a round of Baz IO: Wrote 1.868 GiB in 3.22 seconds | 20260506 23:50:13.611 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.39% / 0.00% / 66.60% | 20260506 23:50:13.611 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.387474% / 7.94729e-06% / 5.5631e-05% | 20260506 23:50:13.611 | INFO | Baz IO overall: | 20260506 23:50:13.611 | INFO | BytesWritten : 2838567474325 | 20260506 23:50:13.611 | INFO | NumEvents : 1368129579779 | 20260506 23:50:15.428 | INFO | Analyzing chunk frames = [5431296,5431808) | 20260506 23:50:15.428 | INFO | MemInfo: { | 20260506 23:50:15.428 | INFO | "gpu" : " 17 GB", | 20260506 23:50:15.428 | INFO | "huge" : "", | 20260506 23:50:15.428 | INFO | "huge_shared" : " 27 GB", | 20260506 23:50:15.428 | INFO | "malloc" : " 14 GB", | 20260506 23:50:15.428 | INFO | "pinned" : " 29 GB" | 20260506 23:50:15.428 | INFO | } | 20260506 23:50:15.428 | INFO | Perf report for the last 5.13565 seconds: | 20260506 23:50:15.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:50:15.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:15.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:15.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:15.428 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260506 23:50:15.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:15.428 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 23:50:15.428 | INFO | PRE_HQ : 59.6%, 1.8, 56.2ms, 0 | 20260506 23:50:15.428 | INFO | BAZWRITER : 62.3%, 3.5, 111.1ms, 0 | 20260506 23:50:15.428 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:50:15.428 | INFO | Uploaded 36GiB to GPU in 2.01885s (17.832GiB/s) | 20260506 23:50:15.429 | INFO | Downloaded 38GiB to GPU in 2.17873s (17.4413GiB/s) | 20260506 23:50:20.572 | INFO | Analyzing chunk frames = [5431808,5432320) | 20260506 23:50:20.572 | INFO | MemInfo: { | 20260506 23:50:20.572 | INFO | "gpu" : " 17 GB", | 20260506 23:50:20.572 | INFO | "huge" : "", | 20260506 23:50:20.572 | INFO | "huge_shared" : " 27 GB", | 20260506 23:50:20.572 | INFO | "malloc" : " 14 GB", | 20260506 23:50:20.572 | INFO | "pinned" : " 29 GB" | 20260506 23:50:20.572 | INFO | } | 20260506 23:50:20.572 | INFO | Perf report for the last 5.14395 seconds: | 20260506 23:50:20.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:50:20.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:20.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:20.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:20.572 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260506 23:50:20.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:20.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:20.572 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:50:20.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:20.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:20.572 | INFO | Uploaded 36GiB to GPU in 2.00909s (17.9186GiB/s) | 20260506 23:50:20.573 | INFO | Downloaded 34GiB to GPU in 1.92927s (17.6233GiB/s) | 20260506 23:50:25.709 | INFO | Analyzing chunk frames = [5432320,5432832) | 20260506 23:50:25.709 | INFO | MemInfo: { | 20260506 23:50:25.709 | INFO | "gpu" : " 17 GB", | 20260506 23:50:25.709 | INFO | "huge" : "", | 20260506 23:50:25.709 | INFO | "huge_shared" : " 27 GB", | 20260506 23:50:25.709 | INFO | "malloc" : " 14 GB", | 20260506 23:50:25.709 | INFO | "pinned" : " 29 GB" | 20260506 23:50:25.709 | INFO | } | 20260506 23:50:25.709 | INFO | Perf report for the last 5.13671 seconds: | 20260506 23:50:25.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:50:25.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:25.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:25.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:25.709 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260506 23:50:25.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:25.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:25.709 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:50:25.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:25.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:25.709 | INFO | Uploaded 36GiB to GPU in 1.94964s (18.4649GiB/s) | 20260506 23:50:25.709 | INFO | Downloaded 34GiB to GPU in 1.92763s (17.6383GiB/s) | 20260506 23:50:30.852 | INFO | Analyzing chunk frames = [5432832,5433344) | 20260506 23:50:30.852 | INFO | MemInfo: { | 20260506 23:50:30.852 | INFO | "gpu" : " 17 GB", | 20260506 23:50:30.852 | INFO | "huge" : "", | 20260506 23:50:30.852 | INFO | "huge_shared" : " 27 GB", | 20260506 23:50:30.852 | INFO | "malloc" : " 14 GB", | 20260506 23:50:30.852 | INFO | "pinned" : " 29 GB" | 20260506 23:50:30.852 | INFO | } | 20260506 23:50:30.852 | INFO | Perf report for the last 5.14291 seconds: | 20260506 23:50:30.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:50:30.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:30.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:30.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:30.852 | INFO | ANALYSIS : 59.5%, 3.0, 92.2ms, 0 | 20260506 23:50:30.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:30.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:30.852 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:50:30.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:30.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:30.852 | INFO | Uploaded 36GiB to GPU in 1.94327s (18.5255GiB/s) | 20260506 23:50:30.852 | INFO | Downloaded 34GiB to GPU in 1.87952s (18.0897GiB/s) | 20260506 23:50:35.989 | INFO | Analyzing chunk frames = [5433344,5433856) | 20260506 23:50:35.989 | INFO | MemInfo: { | 20260506 23:50:35.989 | INFO | "gpu" : " 17 GB", | 20260506 23:50:35.989 | INFO | "huge" : "", | 20260506 23:50:35.989 | INFO | "huge_shared" : " 27 GB", | 20260506 23:50:35.989 | INFO | "malloc" : " 14 GB", | 20260506 23:50:35.989 | INFO | "pinned" : " 29 GB" | 20260506 23:50:35.989 | INFO | } | 20260506 23:50:35.989 | INFO | Perf report for the last 5.1371 seconds: | 20260506 23:50:35.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:50:35.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:35.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:35.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:35.989 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 23:50:35.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:35.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:35.989 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:50:35.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:35.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:35.989 | INFO | Uploaded 36GiB to GPU in 1.98946s (18.0954GiB/s) | 20260506 23:50:35.989 | INFO | Downloaded 34GiB to GPU in 1.92312s (17.6796GiB/s) | 20260506 23:50:41.124 | INFO | Analyzing chunk frames = [5433856,5434368) | 20260506 23:50:41.124 | INFO | MemInfo: { | 20260506 23:50:41.124 | INFO | "gpu" : " 17 GB", | 20260506 23:50:41.124 | INFO | "huge" : "", | 20260506 23:50:41.124 | INFO | "huge_shared" : " 27 GB", | 20260506 23:50:41.124 | INFO | "malloc" : " 14 GB", | 20260506 23:50:41.124 | INFO | "pinned" : " 29 GB" | 20260506 23:50:41.124 | INFO | } | 20260506 23:50:41.124 | INFO | Perf report for the last 5.13532 seconds: | 20260506 23:50:41.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:50:41.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:41.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:41.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:41.124 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260506 23:50:41.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:41.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:41.124 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260506 23:50:41.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:41.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:41.125 | INFO | Uploaded 36GiB to GPU in 2.03245s (17.7126GiB/s) | 20260506 23:50:41.125 | INFO | Downloaded 34GiB to GPU in 1.95585s (17.3838GiB/s) | 20260506 23:50:46.270 | INFO | Analyzing chunk frames = [5434368,5434880) | 20260506 23:50:46.270 | INFO | MemInfo: { | 20260506 23:50:46.270 | INFO | "gpu" : " 17 GB", | 20260506 23:50:46.270 | INFO | "huge" : "", | 20260506 23:50:46.270 | INFO | "huge_shared" : " 27 GB", | 20260506 23:50:46.270 | INFO | "malloc" : " 14 GB", | 20260506 23:50:46.270 | INFO | "pinned" : " 29 GB" | 20260506 23:50:46.270 | INFO | } | 20260506 23:50:46.270 | INFO | Perf report for the last 5.14601 seconds: | 20260506 23:50:46.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:50:46.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:46.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:46.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:46.270 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:50:46.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:46.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:46.270 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 23:50:46.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:46.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:46.270 | INFO | Uploaded 36GiB to GPU in 1.97747s (18.2051GiB/s) | 20260506 23:50:46.270 | INFO | Downloaded 34GiB to GPU in 2.00611s (16.9482GiB/s) | 20260506 23:50:51.406 | INFO | Analyzing chunk frames = [5434880,5435392) | 20260506 23:50:51.406 | INFO | MemInfo: { | 20260506 23:50:51.406 | INFO | "gpu" : " 17 GB", | 20260506 23:50:51.406 | INFO | "huge" : "", | 20260506 23:50:51.406 | INFO | "huge_shared" : " 27 GB", | 20260506 23:50:51.406 | INFO | "malloc" : " 14 GB", | 20260506 23:50:51.406 | INFO | "pinned" : " 29 GB" | 20260506 23:50:51.406 | INFO | } | 20260506 23:50:51.406 | INFO | Perf report for the last 5.13566 seconds: | 20260506 23:50:51.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:50:51.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:51.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:51.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:51.406 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:50:51.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:51.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:51.406 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260506 23:50:51.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:51.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:51.406 | INFO | Uploaded 36GiB to GPU in 1.98789s (18.1097GiB/s) | 20260506 23:50:51.406 | INFO | Downloaded 34GiB to GPU in 1.99995s (17.0005GiB/s) | 20260506 23:50:51.513 | INFO | Starting a round of Baz IO | 20260506 23:50:54.626 | INFO | Populating RTMetrics report | 20260506 23:50:54.626 | INFO | RTMetrics report populated | 20260506 23:50:54.626 | INFO | Opening tmp RTMetrics json file | 20260506 23:50:54.627 | INFO | Serializing RTMetrics to json | 20260506 23:50:54.628 | INFO | Serialization done, swapping tmp file now... | 20260506 23:50:54.628 | INFO | RTMetrics done | 20260506 23:50:54.722 | INFO | Finished a round of Baz IO: Wrote 1.871 GiB in 3.208 seconds | 20260506 23:50:54.722 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.39% / 0.00% / 66.61% | 20260506 23:50:54.722 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.38751% / 3.97364e-06% / 5.96046e-05% | 20260506 23:50:54.722 | INFO | Baz IO overall: | 20260506 23:50:54.722 | INFO | BytesWritten : 2840576122756 | 20260506 23:50:54.722 | INFO | NumEvents : 1369085523031 | 20260506 23:50:56.547 | INFO | Analyzing chunk frames = [5435392,5435904) | 20260506 23:50:56.548 | INFO | MemInfo: { | 20260506 23:50:56.548 | INFO | "gpu" : " 17 GB", | 20260506 23:50:56.548 | INFO | "huge" : "", | 20260506 23:50:56.548 | INFO | "huge_shared" : " 27 GB", | 20260506 23:50:56.548 | INFO | "malloc" : " 14 GB", | 20260506 23:50:56.548 | INFO | "pinned" : " 29 GB" | 20260506 23:50:56.548 | INFO | } | 20260506 23:50:56.548 | INFO | Perf report for the last 5.14171 seconds: | 20260506 23:50:56.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:50:56.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:56.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:56.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:56.548 | INFO | ANALYSIS : 62.1%, 3.0, 96.4ms, 0 | 20260506 23:50:56.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:50:56.548 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260506 23:50:56.548 | INFO | PRE_HQ : 60.1%, 1.9, 60.1ms, 0 | 20260506 23:50:56.548 | INFO | BAZWRITER : 62.4%, 3.6, 115.5ms, 0 | 20260506 23:50:56.548 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:50:56.548 | INFO | Uploaded 36GiB to GPU in 2.01248s (17.8883GiB/s) | 20260506 23:50:56.548 | INFO | Downloaded 38GiB to GPU in 2.18356s (17.4028GiB/s) | 20260506 23:51:01.690 | INFO | Analyzing chunk frames = [5435904,5436416) | 20260506 23:51:01.690 | INFO | MemInfo: { | 20260506 23:51:01.690 | INFO | "gpu" : " 17 GB", | 20260506 23:51:01.690 | INFO | "huge" : "", | 20260506 23:51:01.690 | INFO | "huge_shared" : " 27 GB", | 20260506 23:51:01.690 | INFO | "malloc" : " 14 GB", | 20260506 23:51:01.690 | INFO | "pinned" : " 29 GB" | 20260506 23:51:01.690 | INFO | } | 20260506 23:51:01.690 | INFO | Perf report for the last 5.14278 seconds: | 20260506 23:51:01.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:51:01.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:01.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:01.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:01.691 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:51:01.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:01.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:01.691 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260506 23:51:01.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:01.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:01.691 | INFO | Uploaded 36GiB to GPU in 1.99387s (18.0553GiB/s) | 20260506 23:51:01.691 | INFO | Downloaded 34GiB to GPU in 1.93456s (17.5751GiB/s) | 20260506 23:51:06.829 | INFO | Analyzing chunk frames = [5436416,5436928) | 20260506 23:51:06.829 | INFO | MemInfo: { | 20260506 23:51:06.829 | INFO | "gpu" : " 17 GB", | 20260506 23:51:06.830 | INFO | "huge" : "", | 20260506 23:51:06.830 | INFO | "huge_shared" : " 27 GB", | 20260506 23:51:06.830 | INFO | "malloc" : " 14 GB", | 20260506 23:51:06.830 | INFO | "pinned" : " 29 GB" | 20260506 23:51:06.830 | INFO | } | 20260506 23:51:06.830 | INFO | Perf report for the last 5.13921 seconds: | 20260506 23:51:06.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:51:06.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:06.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:06.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:06.830 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260506 23:51:06.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:06.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:06.830 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:51:06.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:06.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:06.830 | INFO | Uploaded 36GiB to GPU in 1.97804s (18.1998GiB/s) | 20260506 23:51:06.830 | INFO | Downloaded 34GiB to GPU in 1.93256s (17.5933GiB/s) | 20260506 23:51:11.970 | INFO | Analyzing chunk frames = [5436928,5437440) | 20260506 23:51:11.970 | INFO | MemInfo: { | 20260506 23:51:11.970 | INFO | "gpu" : " 17 GB", | 20260506 23:51:11.970 | INFO | "huge" : "", | 20260506 23:51:11.970 | INFO | "huge_shared" : " 27 GB", | 20260506 23:51:11.970 | INFO | "malloc" : " 14 GB", | 20260506 23:51:11.970 | INFO | "pinned" : " 29 GB" | 20260506 23:51:11.970 | INFO | } | 20260506 23:51:11.970 | INFO | Perf report for the last 5.14006 seconds: | 20260506 23:51:11.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:51:11.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:11.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:11.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:11.970 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 23:51:11.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:11.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:11.970 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:51:11.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:11.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:11.970 | INFO | Uploaded 36GiB to GPU in 1.95519s (18.4126GiB/s) | 20260506 23:51:11.970 | INFO | Downloaded 34GiB to GPU in 1.91065s (17.795GiB/s) | 20260506 23:51:17.101 | INFO | Analyzing chunk frames = [5437440,5437952) | 20260506 23:51:17.101 | INFO | MemInfo: { | 20260506 23:51:17.101 | INFO | "gpu" : " 17 GB", | 20260506 23:51:17.101 | INFO | "huge" : "", | 20260506 23:51:17.101 | INFO | "huge_shared" : " 27 GB", | 20260506 23:51:17.101 | INFO | "malloc" : " 14 GB", | 20260506 23:51:17.101 | INFO | "pinned" : " 29 GB" | 20260506 23:51:17.101 | INFO | } | 20260506 23:51:17.101 | INFO | Perf report for the last 5.13115 seconds: | 20260506 23:51:17.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:51:17.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:17.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:17.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:17.101 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260506 23:51:17.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:17.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:17.101 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:51:17.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:17.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:17.101 | INFO | Uploaded 36GiB to GPU in 1.9891s (18.0986GiB/s) | 20260506 23:51:17.101 | INFO | Downloaded 34GiB to GPU in 1.9477s (17.4565GiB/s) | 20260506 23:51:22.240 | INFO | Analyzing chunk frames = [5437952,5438464) | 20260506 23:51:22.240 | INFO | MemInfo: { | 20260506 23:51:22.240 | INFO | "gpu" : " 17 GB", | 20260506 23:51:22.240 | INFO | "huge" : "", | 20260506 23:51:22.240 | INFO | "huge_shared" : " 27 GB", | 20260506 23:51:22.240 | INFO | "malloc" : " 14 GB", | 20260506 23:51:22.240 | INFO | "pinned" : " 29 GB" | 20260506 23:51:22.240 | INFO | } | 20260506 23:51:22.240 | INFO | Perf report for the last 5.13918 seconds: | 20260506 23:51:22.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:51:22.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:22.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:22.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:22.240 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 23:51:22.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:22.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:22.240 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:51:22.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:22.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:22.240 | INFO | Uploaded 36GiB to GPU in 1.9937s (18.0569GiB/s) | 20260506 23:51:22.240 | INFO | Downloaded 34GiB to GPU in 1.95705s (17.3731GiB/s) | 20260506 23:51:27.380 | INFO | Analyzing chunk frames = [5438464,5438976) | 20260506 23:51:27.380 | INFO | MemInfo: { | 20260506 23:51:27.380 | INFO | "gpu" : " 17 GB", | 20260506 23:51:27.380 | INFO | "huge" : "", | 20260506 23:51:27.381 | INFO | "huge_shared" : " 27 GB", | 20260506 23:51:27.381 | INFO | "malloc" : " 14 GB", | 20260506 23:51:27.381 | INFO | "pinned" : " 29 GB" | 20260506 23:51:27.381 | INFO | } | 20260506 23:51:27.381 | INFO | Perf report for the last 5.1405 seconds: | 20260506 23:51:27.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:51:27.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:27.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:27.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:27.381 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260506 23:51:27.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:27.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:27.381 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:51:27.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:27.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:27.381 | INFO | Uploaded 36GiB to GPU in 1.96025s (18.365GiB/s) | 20260506 23:51:27.381 | INFO | Downloaded 34GiB to GPU in 1.98208s (17.1537GiB/s) | 20260506 23:51:32.521 | INFO | Analyzing chunk frames = [5438976,5439488) | 20260506 23:51:32.521 | INFO | MemInfo: { | 20260506 23:51:32.521 | INFO | "gpu" : " 17 GB", | 20260506 23:51:32.521 | INFO | "huge" : "", | 20260506 23:51:32.521 | INFO | "huge_shared" : " 27 GB", | 20260506 23:51:32.521 | INFO | "malloc" : " 14 GB", | 20260506 23:51:32.521 | INFO | "pinned" : " 29 GB" | 20260506 23:51:32.521 | INFO | } | 20260506 23:51:32.521 | INFO | Perf report for the last 5.14012 seconds: | 20260506 23:51:32.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:51:32.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:32.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:32.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:32.521 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260506 23:51:32.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:32.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:32.521 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:51:32.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:32.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:32.521 | INFO | Uploaded 36GiB to GPU in 1.98297s (18.1546GiB/s) | 20260506 23:51:32.521 | INFO | Downloaded 34GiB to GPU in 2.00856s (16.9275GiB/s) | 20260506 23:51:32.521 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:51:32.521 | INFO | Total mem usage: 18 GiB | 20260506 23:51:32.521 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:51:32.521 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260506 23:51:32.521 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:51:32.521 | INFO | High Water Breakdown: | 20260506 23:51:32.521 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:51:32.521 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:51:32.521 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:51:32.521 | INFO | ------------------------------------------ | 20260506 23:51:32.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:51:32.521 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:51:32.521 | INFO | Total mem usage: 26.5 GiB | 20260506 23:51:32.521 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:51:32.521 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:51:32.521 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:51:32.521 | INFO | High Water Breakdown: | 20260506 23:51:32.521 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:51:32.521 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:51:32.521 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:51:32.521 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:51:32.521 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:51:32.521 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:51:32.521 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:51:32.521 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:51:32.521 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:51:32.521 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:51:32.521 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:51:32.521 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:51:32.521 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:51:32.521 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:51:32.521 | INFO | ------------------------------------------ | 20260506 23:51:32.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:51:32.521 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:51:32.521 | INFO | Total mem usage: 25.6 GiB | 20260506 23:51:32.521 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:51:32.522 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:51:32.522 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:51:32.522 | INFO | High Water Breakdown: | 20260506 23:51:32.522 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:51:32.522 | INFO | ------------------------------------------ | 20260506 23:51:32.522 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:51:32.522 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:51:32.522 | INFO | Total mem usage: 15.7 GiB | 20260506 23:51:32.522 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:51:32.522 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:51:32.522 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 23:51:32.522 | INFO | High Water Breakdown: | 20260506 23:51:32.522 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:51:32.522 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:51:32.522 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:51:32.522 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:51:32.522 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:51:32.522 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:51:32.522 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:51:32.522 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:51:32.522 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:51:32.522 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:51:32.522 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:51:32.522 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:51:32.522 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:51:32.522 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:51:32.522 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:51:32.522 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:51:32.522 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:51:32.522 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:51:32.522 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:51:32.522 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:51:32.522 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:51:32.522 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:51:32.522 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:51:32.522 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:51:32.522 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:51:32.522 | INFO | ------------------------------------------ | 20260506 23:51:32.522 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:51:32.522 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:51:32.522 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:51:32.522 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:51:32.522 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.61e+01, 1.33e+01 / 2.55e+00 | 20260506 23:51:32.522 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 9.45e-03 / 5.02e-03 | 20260506 23:51:32.522 | INFO | PerfReport: 11.66%, Baseline -- 3.61e+00 / 4.35e+00, 5.41e-01 / 2.92e-01 | 20260506 23:51:32.522 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.45e+00, 1.77e-01 / 1.22e-01 | 20260506 23:51:32.522 | INFO | PerfReport: 16.12%, DME -- 4.00e+01 / 4.53e+01, 4.46e+00 / 2.96e+00 | 20260506 23:51:32.522 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.34e+00 / 1.67e+00 | 20260506 23:51:32.522 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.18e+00 / 2.74e+00, 3.82e-01 / 2.26e-01 | 20260506 23:51:32.522 | INFO | PerfReport: 3.17%, Metrics -- 9.83e-01 / 1.32e+00, 2.61e-01 / 1.65e-01 | 20260506 23:51:32.522 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.35e-01 / 3.85e-01, 5.30e-02 / 2.78e-02 | 20260506 23:51:32.522 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:51:32.522 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:51:32.522 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:51:32.522 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260506 23:51:32.522 | INFO | PerfReport: 21.42%, Upload -- 2.07e+01 / 0.00e+00, 6.82e+00 / 0.00e+00 | 20260506 23:51:32.522 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 23:51:32.522 | INFO | PerfReport: 32.09%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 23:51:32.522 | INFO | PerfReport: 1.17%, LockUpload -- 1.13e+00 / 0.00e+00, 4.58e+00 / 0.00e+00 | 20260506 23:51:32.522 | INFO | PerfReport: 1.41%, LockDownload -- 1.37e+00 / 0.00e+00, 3.33e+00 / 0.00e+00 | 20260506 23:51:32.522 | INFO | PerfReport: 22.54%, LockCompute -- 2.18e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 23:51:32.522 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.31e-02 / 0.00e+00 | 20260506 23:51:32.619 | INFO | Starting a round of Baz IO | 20260506 23:51:35.751 | INFO | Populating RTMetrics report | 20260506 23:51:35.751 | INFO | RTMetrics report populated | 20260506 23:51:35.751 | INFO | Opening tmp RTMetrics json file | 20260506 23:51:35.770 | INFO | Serializing RTMetrics to json | 20260506 23:51:35.771 | INFO | Serialization done, swapping tmp file now... | 20260506 23:51:35.772 | INFO | RTMetrics done | 20260506 23:51:35.848 | INFO | Finished a round of Baz IO: Wrote 1.869 GiB in 3.228 seconds | 20260506 23:51:35.848 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.39% / 0.00% / 66.61% | 20260506 23:51:35.848 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.385479% / 3.97364e-06% / 5.96046e-05% | 20260506 23:51:35.848 | INFO | Baz IO overall: | 20260506 23:51:35.848 | INFO | BytesWritten : 2842583097311 | 20260506 23:51:35.848 | INFO | NumEvents : 1370040040888 | 20260506 23:51:37.660 | INFO | Analyzing chunk frames = [5439488,5440000) | 20260506 23:51:37.660 | INFO | MemInfo: { | 20260506 23:51:37.660 | INFO | "gpu" : " 17 GB", | 20260506 23:51:37.660 | INFO | "huge" : "", | 20260506 23:51:37.660 | INFO | "huge_shared" : " 27 GB", | 20260506 23:51:37.660 | INFO | "malloc" : " 14 GB", | 20260506 23:51:37.660 | INFO | "pinned" : " 29 GB" | 20260506 23:51:37.660 | INFO | } | 20260506 23:51:37.660 | INFO | Perf report for the last 5.13891 seconds: | 20260506 23:51:37.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:51:37.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:37.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:37.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:37.660 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260506 23:51:37.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:37.660 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 23:51:37.660 | INFO | PRE_HQ : 59.5%, 1.8, 56.2ms, 0 | 20260506 23:51:37.660 | INFO | BAZWRITER : 62.4%, 3.5, 111.9ms, 0 | 20260506 23:51:37.660 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:51:37.660 | INFO | Uploaded 36GiB to GPU in 2.02468s (17.7806GiB/s) | 20260506 23:51:37.660 | INFO | Downloaded 38GiB to GPU in 2.18101s (17.4231GiB/s) | 20260506 23:51:42.797 | INFO | Analyzing chunk frames = [5440000,5440512) | 20260506 23:51:42.797 | INFO | MemInfo: { | 20260506 23:51:42.797 | INFO | "gpu" : " 17 GB", | 20260506 23:51:42.797 | INFO | "huge" : "", | 20260506 23:51:42.797 | INFO | "huge_shared" : " 27 GB", | 20260506 23:51:42.797 | INFO | "malloc" : " 14 GB", | 20260506 23:51:42.797 | INFO | "pinned" : " 29 GB" | 20260506 23:51:42.797 | INFO | } | 20260506 23:51:42.798 | INFO | Perf report for the last 5.13782 seconds: | 20260506 23:51:42.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:51:42.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:42.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:42.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:42.798 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:51:42.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:42.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:42.798 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260506 23:51:42.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:42.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:42.798 | INFO | Uploaded 36GiB to GPU in 1.99308s (18.0625GiB/s) | 20260506 23:51:42.798 | INFO | Downloaded 34GiB to GPU in 1.92832s (17.632GiB/s) | 20260506 23:51:47.942 | INFO | Analyzing chunk frames = [5440512,5441024) | 20260506 23:51:47.942 | INFO | MemInfo: { | 20260506 23:51:47.943 | INFO | "gpu" : " 17 GB", | 20260506 23:51:47.943 | INFO | "huge" : "", | 20260506 23:51:47.943 | INFO | "huge_shared" : " 27 GB", | 20260506 23:51:47.943 | INFO | "malloc" : " 14 GB", | 20260506 23:51:47.943 | INFO | "pinned" : " 29 GB" | 20260506 23:51:47.943 | INFO | } | 20260506 23:51:47.943 | INFO | Perf report for the last 5.14521 seconds: | 20260506 23:51:47.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:51:47.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:47.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:47.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:47.943 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260506 23:51:47.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:47.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:47.943 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:51:47.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:47.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:47.943 | INFO | Uploaded 36GiB to GPU in 1.96817s (18.2911GiB/s) | 20260506 23:51:47.943 | INFO | Downloaded 34GiB to GPU in 1.92714s (17.6427GiB/s) | 20260506 23:51:53.081 | INFO | Analyzing chunk frames = [5441024,5441536) | 20260506 23:51:53.081 | INFO | MemInfo: { | 20260506 23:51:53.081 | INFO | "gpu" : " 17 GB", | 20260506 23:51:53.081 | INFO | "huge" : "", | 20260506 23:51:53.081 | INFO | "huge_shared" : " 27 GB", | 20260506 23:51:53.081 | INFO | "malloc" : " 14 GB", | 20260506 23:51:53.081 | INFO | "pinned" : " 29 GB" | 20260506 23:51:53.081 | INFO | } | 20260506 23:51:53.081 | INFO | Perf report for the last 5.138 seconds: | 20260506 23:51:53.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:51:53.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:53.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:53.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:53.081 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 23:51:53.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:53.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:53.081 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:51:53.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:53.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:53.081 | INFO | Uploaded 36GiB to GPU in 2.00149s (17.9866GiB/s) | 20260506 23:51:53.081 | INFO | Downloaded 34GiB to GPU in 1.91006s (17.8005GiB/s) | 20260506 23:51:58.220 | INFO | Analyzing chunk frames = [5441536,5442048) | 20260506 23:51:58.220 | INFO | MemInfo: { | 20260506 23:51:58.220 | INFO | "gpu" : " 17 GB", | 20260506 23:51:58.220 | INFO | "huge" : "", | 20260506 23:51:58.220 | INFO | "huge_shared" : " 27 GB", | 20260506 23:51:58.220 | INFO | "malloc" : " 14 GB", | 20260506 23:51:58.220 | INFO | "pinned" : " 29 GB" | 20260506 23:51:58.220 | INFO | } | 20260506 23:51:58.220 | INFO | Perf report for the last 5.13915 seconds: | 20260506 23:51:58.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:51:58.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:58.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:58.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:58.220 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 23:51:58.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:58.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:58.220 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:51:58.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:58.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:51:58.220 | INFO | Uploaded 36GiB to GPU in 2.01153s (17.8969GiB/s) | 20260506 23:51:58.220 | INFO | Downloaded 34GiB to GPU in 1.94887s (17.446GiB/s) | 20260506 23:52:03.357 | INFO | Analyzing chunk frames = [5442048,5442560) | 20260506 23:52:03.357 | INFO | MemInfo: { | 20260506 23:52:03.357 | INFO | "gpu" : " 17 GB", | 20260506 23:52:03.357 | INFO | "huge" : "", | 20260506 23:52:03.357 | INFO | "huge_shared" : " 27 GB", | 20260506 23:52:03.357 | INFO | "malloc" : " 14 GB", | 20260506 23:52:03.357 | INFO | "pinned" : " 29 GB" | 20260506 23:52:03.357 | INFO | } | 20260506 23:52:03.357 | INFO | Perf report for the last 5.13706 seconds: | 20260506 23:52:03.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:52:03.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:03.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:03.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:03.357 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 23:52:03.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:03.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:03.357 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260506 23:52:03.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:03.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:03.357 | INFO | Uploaded 36GiB to GPU in 2.03748s (17.6689GiB/s) | 20260506 23:52:03.357 | INFO | Downloaded 34GiB to GPU in 1.95601s (17.3823GiB/s) | 20260506 23:52:08.498 | INFO | Analyzing chunk frames = [5442560,5443072) | 20260506 23:52:08.498 | INFO | MemInfo: { | 20260506 23:52:08.498 | INFO | "gpu" : " 17 GB", | 20260506 23:52:08.498 | INFO | "huge" : "", | 20260506 23:52:08.498 | INFO | "huge_shared" : " 27 GB", | 20260506 23:52:08.498 | INFO | "malloc" : " 14 GB", | 20260506 23:52:08.498 | INFO | "pinned" : " 29 GB" | 20260506 23:52:08.498 | INFO | } | 20260506 23:52:08.498 | INFO | Perf report for the last 5.14102 seconds: | 20260506 23:52:08.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:52:08.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:08.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:08.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:08.498 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:52:08.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:08.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:08.498 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 23:52:08.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:08.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:08.498 | INFO | Uploaded 36GiB to GPU in 1.97687s (18.2106GiB/s) | 20260506 23:52:08.498 | INFO | Downloaded 34GiB to GPU in 1.99833s (17.0142GiB/s) | 20260506 23:52:13.642 | INFO | Analyzing chunk frames = [5443072,5443584) | 20260506 23:52:13.642 | INFO | MemInfo: { | 20260506 23:52:13.642 | INFO | "gpu" : " 17 GB", | 20260506 23:52:13.642 | INFO | "huge" : "", | 20260506 23:52:13.642 | INFO | "huge_shared" : " 27 GB", | 20260506 23:52:13.642 | INFO | "malloc" : " 14 GB", | 20260506 23:52:13.642 | INFO | "pinned" : " 29 GB" | 20260506 23:52:13.642 | INFO | } | 20260506 23:52:13.642 | INFO | Perf report for the last 5.14422 seconds: | 20260506 23:52:13.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:52:13.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:13.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:13.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:13.642 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:52:13.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:13.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:13.642 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260506 23:52:13.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:13.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:13.642 | INFO | Uploaded 36GiB to GPU in 1.9871s (18.1168GiB/s) | 20260506 23:52:13.642 | INFO | Downloaded 34GiB to GPU in 1.99694s (17.0261GiB/s) | 20260506 23:52:13.740 | INFO | Starting a round of Baz IO | 20260506 23:52:16.866 | INFO | Populating RTMetrics report | 20260506 23:52:16.867 | INFO | RTMetrics report populated | 20260506 23:52:16.867 | INFO | Opening tmp RTMetrics json file | 20260506 23:52:16.867 | INFO | Serializing RTMetrics to json | 20260506 23:52:16.868 | INFO | Serialization done, swapping tmp file now... | 20260506 23:52:16.869 | INFO | RTMetrics done | 20260506 23:52:16.964 | INFO | Finished a round of Baz IO: Wrote 1.87 GiB in 3.224 seconds | 20260506 23:52:16.964 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.39% / 0.00% / 66.61% | 20260506 23:52:16.964 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.385769% / 3.97364e-06% / 4.76837e-05% | 20260506 23:52:16.964 | INFO | Baz IO overall: | 20260506 23:52:16.964 | INFO | BytesWritten : 2844590507877 | 20260506 23:52:16.964 | INFO | NumEvents : 1370995672512 | 20260506 23:52:18.778 | INFO | Analyzing chunk frames = [5443584,5444096) | 20260506 23:52:18.778 | INFO | MemInfo: { | 20260506 23:52:18.778 | INFO | "gpu" : " 17 GB", | 20260506 23:52:18.778 | INFO | "huge" : "", | 20260506 23:52:18.778 | INFO | "huge_shared" : " 27 GB", | 20260506 23:52:18.778 | INFO | "malloc" : " 14 GB", | 20260506 23:52:18.778 | INFO | "pinned" : " 29 GB" | 20260506 23:52:18.778 | INFO | } | 20260506 23:52:18.778 | INFO | Perf report for the last 5.13587 seconds: | 20260506 23:52:18.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:52:18.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:18.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:18.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:18.778 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260506 23:52:18.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:18.778 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 23:52:18.778 | INFO | PRE_HQ : 59.7%, 1.8, 56.6ms, 0 | 20260506 23:52:18.778 | INFO | BAZWRITER : 62.4%, 3.5, 111.1ms, 0 | 20260506 23:52:18.778 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:52:18.778 | INFO | Uploaded 36GiB to GPU in 2.02134s (17.81GiB/s) | 20260506 23:52:18.778 | INFO | Downloaded 38GiB to GPU in 2.1722s (17.4937GiB/s) | 20260506 23:52:23.917 | INFO | Analyzing chunk frames = [5444096,5444608) | 20260506 23:52:23.917 | INFO | MemInfo: { | 20260506 23:52:23.918 | INFO | "gpu" : " 17 GB", | 20260506 23:52:23.918 | INFO | "huge" : "", | 20260506 23:52:23.918 | INFO | "huge_shared" : " 27 GB", | 20260506 23:52:23.918 | INFO | "malloc" : " 14 GB", | 20260506 23:52:23.918 | INFO | "pinned" : " 29 GB" | 20260506 23:52:23.918 | INFO | } | 20260506 23:52:23.918 | INFO | Perf report for the last 5.13957 seconds: | 20260506 23:52:23.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:52:23.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:23.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:23.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:23.918 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 23:52:23.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:23.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:23.918 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260506 23:52:23.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:23.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:23.918 | INFO | Uploaded 36GiB to GPU in 1.98386s (18.1464GiB/s) | 20260506 23:52:23.918 | INFO | Downloaded 34GiB to GPU in 1.92754s (17.6391GiB/s) | 20260506 23:52:29.054 | INFO | Analyzing chunk frames = [5444608,5445120) | 20260506 23:52:29.054 | INFO | MemInfo: { | 20260506 23:52:29.054 | INFO | "gpu" : " 17 GB", | 20260506 23:52:29.054 | INFO | "huge" : "", | 20260506 23:52:29.054 | INFO | "huge_shared" : " 27 GB", | 20260506 23:52:29.055 | INFO | "malloc" : " 14 GB", | 20260506 23:52:29.055 | INFO | "pinned" : " 29 GB" | 20260506 23:52:29.055 | INFO | } | 20260506 23:52:29.055 | INFO | Perf report for the last 5.13693 seconds: | 20260506 23:52:29.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:52:29.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:29.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:29.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:29.055 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:52:29.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:29.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:29.055 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:52:29.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:29.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:29.055 | INFO | Uploaded 36GiB to GPU in 1.99412s (18.0531GiB/s) | 20260506 23:52:29.055 | INFO | Downloaded 34GiB to GPU in 1.93534s (17.5679GiB/s) | 20260506 23:52:34.195 | INFO | Analyzing chunk frames = [5445120,5445632) | 20260506 23:52:34.195 | INFO | MemInfo: { | 20260506 23:52:34.195 | INFO | "gpu" : " 17 GB", | 20260506 23:52:34.195 | INFO | "huge" : "", | 20260506 23:52:34.195 | INFO | "huge_shared" : " 27 GB", | 20260506 23:52:34.195 | INFO | "malloc" : " 14 GB", | 20260506 23:52:34.195 | INFO | "pinned" : " 29 GB" | 20260506 23:52:34.195 | INFO | } | 20260506 23:52:34.195 | INFO | Perf report for the last 5.14036 seconds: | 20260506 23:52:34.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:52:34.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:34.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:34.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:34.195 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260506 23:52:34.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:34.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:34.195 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:52:34.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:34.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:34.195 | INFO | Uploaded 36GiB to GPU in 1.9876s (18.1123GiB/s) | 20260506 23:52:34.195 | INFO | Downloaded 34GiB to GPU in 1.9097s (17.8038GiB/s) | 20260506 23:52:39.333 | INFO | Analyzing chunk frames = [5445632,5446144) | 20260506 23:52:39.333 | INFO | MemInfo: { | 20260506 23:52:39.333 | INFO | "gpu" : " 17 GB", | 20260506 23:52:39.333 | INFO | "huge" : "", | 20260506 23:52:39.333 | INFO | "huge_shared" : " 27 GB", | 20260506 23:52:39.333 | INFO | "malloc" : " 14 GB", | 20260506 23:52:39.333 | INFO | "pinned" : " 29 GB" | 20260506 23:52:39.333 | INFO | } | 20260506 23:52:39.333 | INFO | Perf report for the last 5.13802 seconds: | 20260506 23:52:39.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:52:39.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:39.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:39.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:39.333 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 23:52:39.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:39.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:39.333 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:52:39.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:39.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:39.333 | INFO | Uploaded 36GiB to GPU in 1.9844s (18.1415GiB/s) | 20260506 23:52:39.333 | INFO | Downloaded 34GiB to GPU in 1.93336s (17.586GiB/s) | 20260506 23:52:44.470 | INFO | Analyzing chunk frames = [5446144,5446656) | 20260506 23:52:44.470 | INFO | MemInfo: { | 20260506 23:52:44.470 | INFO | "gpu" : " 17 GB", | 20260506 23:52:44.470 | INFO | "huge" : "", | 20260506 23:52:44.470 | INFO | "huge_shared" : " 27 GB", | 20260506 23:52:44.470 | INFO | "malloc" : " 14 GB", | 20260506 23:52:44.470 | INFO | "pinned" : " 29 GB" | 20260506 23:52:44.470 | INFO | } | 20260506 23:52:44.470 | INFO | Perf report for the last 5.13739 seconds: | 20260506 23:52:44.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:52:44.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:44.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:44.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:44.471 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 23:52:44.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:44.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:44.471 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:52:44.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:44.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:44.471 | INFO | Uploaded 36GiB to GPU in 2.00749s (17.9328GiB/s) | 20260506 23:52:44.471 | INFO | Downloaded 34GiB to GPU in 1.93267s (17.5923GiB/s) | 20260506 23:52:49.613 | INFO | Analyzing chunk frames = [5446656,5447168) | 20260506 23:52:49.613 | INFO | MemInfo: { | 20260506 23:52:49.613 | INFO | "gpu" : " 17 GB", | 20260506 23:52:49.613 | INFO | "huge" : "", | 20260506 23:52:49.613 | INFO | "huge_shared" : " 27 GB", | 20260506 23:52:49.613 | INFO | "malloc" : " 14 GB", | 20260506 23:52:49.613 | INFO | "pinned" : " 29 GB" | 20260506 23:52:49.613 | INFO | } | 20260506 23:52:49.613 | INFO | Perf report for the last 5.14258 seconds: | 20260506 23:52:49.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:52:49.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:49.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:49.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:49.613 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 23:52:49.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:49.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:49.613 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260506 23:52:49.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:49.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:49.613 | INFO | Uploaded 36GiB to GPU in 1.95842s (18.3822GiB/s) | 20260506 23:52:49.613 | INFO | Downloaded 34GiB to GPU in 1.97856s (17.1842GiB/s) | 20260506 23:52:54.754 | INFO | Analyzing chunk frames = [5447168,5447680) | 20260506 23:52:54.754 | INFO | MemInfo: { | 20260506 23:52:54.754 | INFO | "gpu" : " 17 GB", | 20260506 23:52:54.754 | INFO | "huge" : "", | 20260506 23:52:54.754 | INFO | "huge_shared" : " 27 GB", | 20260506 23:52:54.754 | INFO | "malloc" : " 14 GB", | 20260506 23:52:54.754 | INFO | "pinned" : " 29 GB" | 20260506 23:52:54.754 | INFO | } | 20260506 23:52:54.754 | INFO | Perf report for the last 5.14125 seconds: | 20260506 23:52:54.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:52:54.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:54.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:54.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:54.754 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260506 23:52:54.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:54.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:54.754 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:52:54.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:54.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:54.755 | INFO | Uploaded 36GiB to GPU in 2.01271s (17.8864GiB/s) | 20260506 23:52:54.755 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:52:54.755 | INFO | Total mem usage: 18 GiB | 20260506 23:52:54.755 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:52:54.755 | INFO | Downloaded 34GiB to GPU in 2.00036s (16.9969GiB/s) | 20260506 23:52:54.755 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 23:52:54.755 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:52:54.755 | INFO | High Water Breakdown: | 20260506 23:52:54.755 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:52:54.755 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:52:54.755 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:52:54.755 | INFO | ------------------------------------------ | 20260506 23:52:54.755 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:52:54.755 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:52:54.755 | INFO | Total mem usage: 26.5 GiB | 20260506 23:52:54.755 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:52:54.755 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:52:54.755 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:52:54.755 | INFO | High Water Breakdown: | 20260506 23:52:54.755 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:52:54.755 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:52:54.755 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:52:54.755 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:52:54.755 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:52:54.755 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:52:54.755 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:52:54.755 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:52:54.755 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:52:54.755 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:52:54.755 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:52:54.755 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:52:54.755 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:52:54.755 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:52:54.755 | INFO | ------------------------------------------ | 20260506 23:52:54.755 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:52:54.755 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:52:54.755 | INFO | Total mem usage: 25.6 GiB | 20260506 23:52:54.755 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:52:54.755 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:52:54.755 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:52:54.755 | INFO | High Water Breakdown: | 20260506 23:52:54.755 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:52:54.755 | INFO | ------------------------------------------ | 20260506 23:52:54.755 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:52:54.755 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:52:54.755 | INFO | Total mem usage: 15.7 GiB | 20260506 23:52:54.755 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:52:54.755 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:52:54.755 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 23:52:54.755 | INFO | High Water Breakdown: | 20260506 23:52:54.755 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:52:54.755 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:52:54.755 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:52:54.755 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:52:54.755 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:52:54.755 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:52:54.755 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:52:54.755 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:52:54.755 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:52:54.755 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:52:54.755 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:52:54.755 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:52:54.755 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:52:54.755 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:52:54.755 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:52:54.755 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:52:54.755 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:52:54.755 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:52:54.755 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:52:54.755 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:52:54.755 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:52:54.755 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:52:54.755 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:52:54.755 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:52:54.755 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:52:54.755 | INFO | ------------------------------------------ | 20260506 23:52:54.755 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:52:54.755 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:52:54.755 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:52:54.755 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:52:54.755 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.57e+01, 1.34e+01 / 2.25e+00 | 20260506 23:52:54.755 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.01e-01, 1.00e-02 / 5.03e-03 | 20260506 23:52:54.755 | INFO | PerfReport: 11.68%, Baseline -- 3.62e+00 / 4.34e+00, 5.49e-01 / 3.92e-01 | 20260506 23:52:54.755 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.44e+00, 1.78e-01 / 1.42e-01 | 20260506 23:52:54.755 | INFO | PerfReport: 16.06%, DME -- 3.98e+01 / 4.45e+01, 4.18e+00 / 2.88e+00 | 20260506 23:52:54.755 | INFO | PerfReport: 57.40%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.37e+00 / 1.90e+00 | 20260506 23:52:54.755 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.18e+00 / 2.68e+00, 3.77e-01 / 2.25e-01 | 20260506 23:52:54.755 | INFO | PerfReport: 3.18%, Metrics -- 9.84e-01 / 1.31e+00, 2.62e-01 / 1.43e-01 | 20260506 23:52:54.755 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.35e-01 / 3.85e-01, 5.30e-02 / 2.52e-02 | 20260506 23:52:54.756 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:52:54.756 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:52:54.756 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:52:54.756 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 23:52:54.756 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 6.94e+00 / 0.00e+00 | 20260506 23:52:54.756 | INFO | PerfReport: 21.31%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260506 23:52:54.756 | INFO | PerfReport: 32.08%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 23:52:54.756 | INFO | PerfReport: 1.25%, LockUpload -- 1.21e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260506 23:52:54.756 | INFO | PerfReport: 1.33%, LockDownload -- 1.28e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260506 23:52:54.756 | INFO | PerfReport: 22.49%, LockCompute -- 2.17e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260506 23:52:54.756 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260506 23:52:54.852 | INFO | Starting a round of Baz IO | 20260506 23:52:57.979 | INFO | Populating RTMetrics report | 20260506 23:52:57.980 | INFO | RTMetrics report populated | 20260506 23:52:57.980 | INFO | Opening tmp RTMetrics json file | 20260506 23:52:57.981 | INFO | Serializing RTMetrics to json | 20260506 23:52:57.989 | INFO | Serialization done, swapping tmp file now... | 20260506 23:52:58.006 | INFO | RTMetrics done | 20260506 23:52:58.073 | INFO | Finished a round of Baz IO: Wrote 1.87 GiB in 3.221 seconds | 20260506 23:52:58.073 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.39% / 0.00% / 66.61% | 20260506 23:52:58.073 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.384486% / 3.97364e-06% / 6.75519e-05% | 20260506 23:52:58.073 | INFO | Baz IO overall: | 20260506 23:52:58.073 | INFO | BytesWritten : 2846598324365 | 20260506 23:52:58.073 | INFO | NumEvents : 1371951142828 | 20260506 23:52:59.896 | INFO | Analyzing chunk frames = [5447680,5448192) | 20260506 23:52:59.896 | INFO | MemInfo: { | 20260506 23:52:59.896 | INFO | "gpu" : " 17 GB", | 20260506 23:52:59.896 | INFO | "huge" : "", | 20260506 23:52:59.896 | INFO | "huge_shared" : " 27 GB", | 20260506 23:52:59.896 | INFO | "malloc" : " 14 GB", | 20260506 23:52:59.896 | INFO | "pinned" : " 29 GB" | 20260506 23:52:59.896 | INFO | } | 20260506 23:52:59.896 | INFO | Perf report for the last 5.14213 seconds: | 20260506 23:52:59.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:52:59.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:59.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:59.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:59.897 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 23:52:59.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:52:59.897 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 23:52:59.897 | INFO | PRE_HQ : 59.6%, 1.8, 56.6ms, 0 | 20260506 23:52:59.897 | INFO | BAZWRITER : 62.2%, 3.5, 111.4ms, 0 | 20260506 23:52:59.897 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:52:59.897 | INFO | Uploaded 36GiB to GPU in 2.01537s (17.8627GiB/s) | 20260506 23:52:59.897 | INFO | Downloaded 38GiB to GPU in 2.17974s (17.4333GiB/s) | 20260506 23:53:05.037 | INFO | Analyzing chunk frames = [5448192,5448704) | 20260506 23:53:05.037 | INFO | MemInfo: { | 20260506 23:53:05.037 | INFO | "gpu" : " 17 GB", | 20260506 23:53:05.037 | INFO | "huge" : "", | 20260506 23:53:05.037 | INFO | "huge_shared" : " 27 GB", | 20260506 23:53:05.037 | INFO | "malloc" : " 14 GB", | 20260506 23:53:05.037 | INFO | "pinned" : " 29 GB" | 20260506 23:53:05.037 | INFO | } | 20260506 23:53:05.037 | INFO | Perf report for the last 5.14032 seconds: | 20260506 23:53:05.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:53:05.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:05.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:05.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:05.037 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:53:05.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:05.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:05.037 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:53:05.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:05.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:05.037 | INFO | Uploaded 36GiB to GPU in 2.02561s (17.7724GiB/s) | 20260506 23:53:05.037 | INFO | Downloaded 34GiB to GPU in 1.92828s (17.6323GiB/s) | 20260506 23:53:10.172 | INFO | Analyzing chunk frames = [5448704,5449216) | 20260506 23:53:10.172 | INFO | MemInfo: { | 20260506 23:53:10.172 | INFO | "gpu" : " 17 GB", | 20260506 23:53:10.172 | INFO | "huge" : "", | 20260506 23:53:10.172 | INFO | "huge_shared" : " 27 GB", | 20260506 23:53:10.172 | INFO | "malloc" : " 14 GB", | 20260506 23:53:10.172 | INFO | "pinned" : " 29 GB" | 20260506 23:53:10.172 | INFO | } | 20260506 23:53:10.172 | INFO | Perf report for the last 5.13513 seconds: | 20260506 23:53:10.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:53:10.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:10.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:10.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:10.172 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:53:10.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:10.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:10.172 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:53:10.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:10.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:10.172 | INFO | Uploaded 36GiB to GPU in 2.00531s (17.9523GiB/s) | 20260506 23:53:10.172 | INFO | Downloaded 34GiB to GPU in 1.93621s (17.5601GiB/s) | 20260506 23:53:15.309 | INFO | Analyzing chunk frames = [5449216,5449728) | 20260506 23:53:15.309 | INFO | MemInfo: { | 20260506 23:53:15.309 | INFO | "gpu" : " 17 GB", | 20260506 23:53:15.309 | INFO | "huge" : "", | 20260506 23:53:15.309 | INFO | "huge_shared" : " 27 GB", | 20260506 23:53:15.309 | INFO | "malloc" : " 14 GB", | 20260506 23:53:15.309 | INFO | "pinned" : " 29 GB" | 20260506 23:53:15.309 | INFO | } | 20260506 23:53:15.309 | INFO | Perf report for the last 5.13746 seconds: | 20260506 23:53:15.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:53:15.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:15.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:15.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:15.309 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 23:53:15.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:15.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:15.309 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:53:15.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:15.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:15.310 | INFO | Uploaded 36GiB to GPU in 1.99174s (18.0747GiB/s) | 20260506 23:53:15.310 | INFO | Downloaded 34GiB to GPU in 1.90607s (17.8378GiB/s) | 20260506 23:53:20.453 | INFO | Analyzing chunk frames = [5449728,5450240) | 20260506 23:53:20.453 | INFO | MemInfo: { | 20260506 23:53:20.453 | INFO | "gpu" : " 17 GB", | 20260506 23:53:20.453 | INFO | "huge" : "", | 20260506 23:53:20.453 | INFO | "huge_shared" : " 27 GB", | 20260506 23:53:20.453 | INFO | "malloc" : " 14 GB", | 20260506 23:53:20.453 | INFO | "pinned" : " 29 GB" | 20260506 23:53:20.453 | INFO | } | 20260506 23:53:20.453 | INFO | Perf report for the last 5.14383 seconds: | 20260506 23:53:20.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:53:20.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:20.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:20.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:20.453 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 23:53:20.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:20.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:20.453 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:53:20.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:20.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:20.453 | INFO | Uploaded 36GiB to GPU in 1.99201s (18.0722GiB/s) | 20260506 23:53:20.453 | INFO | Downloaded 34GiB to GPU in 1.92174s (17.6923GiB/s) | 20260506 23:53:25.587 | INFO | Analyzing chunk frames = [5450240,5450752) | 20260506 23:53:25.587 | INFO | MemInfo: { | 20260506 23:53:25.587 | INFO | "gpu" : " 17 GB", | 20260506 23:53:25.587 | INFO | "huge" : "", | 20260506 23:53:25.587 | INFO | "huge_shared" : " 27 GB", | 20260506 23:53:25.587 | INFO | "malloc" : " 14 GB", | 20260506 23:53:25.587 | INFO | "pinned" : " 29 GB" | 20260506 23:53:25.587 | INFO | } | 20260506 23:53:25.587 | INFO | Perf report for the last 5.1339 seconds: | 20260506 23:53:25.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:53:25.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:25.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:25.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:25.587 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 23:53:25.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:25.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:25.587 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260506 23:53:25.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:25.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:25.587 | INFO | Uploaded 36GiB to GPU in 2.02567s (17.7719GiB/s) | 20260506 23:53:25.587 | INFO | Downloaded 34GiB to GPU in 1.9389s (17.5357GiB/s) | 20260506 23:53:30.731 | INFO | Analyzing chunk frames = [5450752,5451264) | 20260506 23:53:30.731 | INFO | MemInfo: { | 20260506 23:53:30.731 | INFO | "gpu" : " 17 GB", | 20260506 23:53:30.731 | INFO | "huge" : "", | 20260506 23:53:30.731 | INFO | "huge_shared" : " 27 GB", | 20260506 23:53:30.731 | INFO | "malloc" : " 14 GB", | 20260506 23:53:30.731 | INFO | "pinned" : " 29 GB" | 20260506 23:53:30.731 | INFO | } | 20260506 23:53:30.731 | INFO | Perf report for the last 5.144 seconds: | 20260506 23:53:30.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:53:30.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:30.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:30.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:30.731 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:53:30.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:30.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:30.731 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 23:53:30.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:30.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:30.731 | INFO | Uploaded 36GiB to GPU in 1.98103s (18.1724GiB/s) | 20260506 23:53:30.731 | INFO | Downloaded 34GiB to GPU in 1.99755s (17.0209GiB/s) | 20260506 23:53:35.866 | INFO | Analyzing chunk frames = [5451264,5451776) | 20260506 23:53:35.866 | INFO | MemInfo: { | 20260506 23:53:35.866 | INFO | "gpu" : " 17 GB", | 20260506 23:53:35.866 | INFO | "huge" : "", | 20260506 23:53:35.866 | INFO | "huge_shared" : " 27 GB", | 20260506 23:53:35.866 | INFO | "malloc" : " 14 GB", | 20260506 23:53:35.866 | INFO | "pinned" : " 29 GB" | 20260506 23:53:35.866 | INFO | } | 20260506 23:53:35.866 | INFO | Perf report for the last 5.13483 seconds: | 20260506 23:53:35.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:53:35.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:35.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:35.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:35.866 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:53:35.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:35.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:35.866 | INFO | PRE_HQ : 30.2%, 1.0, 15.6ms, 0 | 20260506 23:53:35.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:35.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:35.866 | INFO | Uploaded 36GiB to GPU in 1.9776s (18.2038GiB/s) | 20260506 23:53:35.866 | INFO | Downloaded 34GiB to GPU in 2.00663s (16.9438GiB/s) | 20260506 23:53:35.965 | INFO | Starting a round of Baz IO | 20260506 23:53:39.088 | INFO | Populating RTMetrics report | 20260506 23:53:39.088 | INFO | RTMetrics report populated | 20260506 23:53:39.088 | INFO | Opening tmp RTMetrics json file | 20260506 23:53:39.091 | INFO | Serializing RTMetrics to json | 20260506 23:53:39.108 | INFO | Serialization done, swapping tmp file now... | 20260506 23:53:39.108 | INFO | RTMetrics done | 20260506 23:53:39.182 | INFO | Finished a round of Baz IO: Wrote 1.866 GiB in 3.217 seconds | 20260506 23:53:39.182 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.38% / 0.00% / 66.61% | 20260506 23:53:39.182 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.381966% / 7.94729e-06% / 4.76837e-05% | 20260506 23:53:39.182 | INFO | Baz IO overall: | 20260506 23:53:39.182 | INFO | BytesWritten : 2848602447176 | 20260506 23:53:39.182 | INFO | NumEvents : 1372904112806 | 20260506 23:53:41.006 | INFO | Analyzing chunk frames = [5451776,5452288) | 20260506 23:53:41.006 | INFO | MemInfo: { | 20260506 23:53:41.006 | INFO | "gpu" : " 17 GB", | 20260506 23:53:41.006 | INFO | "huge" : "", | 20260506 23:53:41.006 | INFO | "huge_shared" : " 27 GB", | 20260506 23:53:41.007 | INFO | "malloc" : " 14 GB", | 20260506 23:53:41.007 | INFO | "pinned" : " 29 GB" | 20260506 23:53:41.007 | INFO | } | 20260506 23:53:41.007 | INFO | Perf report for the last 5.14059 seconds: | 20260506 23:53:41.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:53:41.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:41.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:41.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:41.007 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260506 23:53:41.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:41.007 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 23:53:41.007 | INFO | PRE_HQ : 59.5%, 1.8, 56.2ms, 0 | 20260506 23:53:41.007 | INFO | BAZWRITER : 62.2%, 3.4, 110.1ms, 0 | 20260506 23:53:41.007 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:53:41.007 | INFO | Uploaded 36GiB to GPU in 1.99515s (18.0438GiB/s) | 20260506 23:53:41.007 | INFO | Downloaded 38GiB to GPU in 2.17477s (17.4731GiB/s) | 20260506 23:53:46.152 | INFO | Analyzing chunk frames = [5452288,5452800) | 20260506 23:53:46.152 | INFO | MemInfo: { | 20260506 23:53:46.152 | INFO | "gpu" : " 17 GB", | 20260506 23:53:46.152 | INFO | "huge" : "", | 20260506 23:53:46.152 | INFO | "huge_shared" : " 27 GB", | 20260506 23:53:46.152 | INFO | "malloc" : " 14 GB", | 20260506 23:53:46.152 | INFO | "pinned" : " 29 GB" | 20260506 23:53:46.152 | INFO | } | 20260506 23:53:46.152 | INFO | Perf report for the last 5.14573 seconds: | 20260506 23:53:46.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:53:46.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:46.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:46.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:46.152 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 23:53:46.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:46.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:46.152 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260506 23:53:46.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:46.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:46.153 | INFO | Uploaded 36GiB to GPU in 1.98414s (18.1439GiB/s) | 20260506 23:53:46.153 | INFO | Downloaded 34GiB to GPU in 1.92971s (17.6192GiB/s) | 20260506 23:53:51.291 | INFO | Analyzing chunk frames = [5452800,5453312) | 20260506 23:53:51.291 | INFO | MemInfo: { | 20260506 23:53:51.291 | INFO | "gpu" : " 17 GB", | 20260506 23:53:51.291 | INFO | "huge" : "", | 20260506 23:53:51.291 | INFO | "huge_shared" : " 27 GB", | 20260506 23:53:51.291 | INFO | "malloc" : " 14 GB", | 20260506 23:53:51.291 | INFO | "pinned" : " 29 GB" | 20260506 23:53:51.291 | INFO | } | 20260506 23:53:51.291 | INFO | Perf report for the last 5.1384 seconds: | 20260506 23:53:51.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:53:51.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:51.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:51.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:51.291 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260506 23:53:51.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:51.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:51.291 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:53:51.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:51.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:51.291 | INFO | Uploaded 36GiB to GPU in 1.99308s (18.0625GiB/s) | 20260506 23:53:51.291 | INFO | Downloaded 34GiB to GPU in 1.93854s (17.539GiB/s) | 20260506 23:53:56.417 | INFO | Analyzing chunk frames = [5453312,5453824) | 20260506 23:53:56.417 | INFO | MemInfo: { | 20260506 23:53:56.417 | INFO | "gpu" : " 17 GB", | 20260506 23:53:56.417 | INFO | "huge" : "", | 20260506 23:53:56.417 | INFO | "huge_shared" : " 27 GB", | 20260506 23:53:56.417 | INFO | "malloc" : " 14 GB", | 20260506 23:53:56.417 | INFO | "pinned" : " 29 GB" | 20260506 23:53:56.417 | INFO | } | 20260506 23:53:56.417 | INFO | Perf report for the last 5.12636 seconds: | 20260506 23:53:56.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:53:56.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:56.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:56.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:56.417 | INFO | ANALYSIS : 60.1%, 3.0, 93.0ms, 0 | 20260506 23:53:56.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:56.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:56.417 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:53:56.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:56.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:53:56.417 | INFO | Uploaded 36GiB to GPU in 1.97081s (18.2666GiB/s) | 20260506 23:53:56.417 | INFO | Downloaded 34GiB to GPU in 1.90772s (17.8223GiB/s) | 20260506 23:54:01.558 | INFO | Analyzing chunk frames = [5453824,5454336) | 20260506 23:54:01.558 | INFO | MemInfo: { | 20260506 23:54:01.558 | INFO | "gpu" : " 17 GB", | 20260506 23:54:01.558 | INFO | "huge" : "", | 20260506 23:54:01.558 | INFO | "huge_shared" : " 27 GB", | 20260506 23:54:01.558 | INFO | "malloc" : " 14 GB", | 20260506 23:54:01.558 | INFO | "pinned" : " 29 GB" | 20260506 23:54:01.558 | INFO | } | 20260506 23:54:01.558 | INFO | Perf report for the last 5.14121 seconds: | 20260506 23:54:01.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:54:01.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:01.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:01.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:01.558 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 23:54:01.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:01.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:01.558 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:54:01.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:01.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:01.559 | INFO | Uploaded 36GiB to GPU in 1.95698s (18.3957GiB/s) | 20260506 23:54:01.559 | INFO | Downloaded 34GiB to GPU in 1.93074s (17.6098GiB/s) | 20260506 23:54:06.705 | INFO | Analyzing chunk frames = [5454336,5454848) | 20260506 23:54:06.705 | INFO | MemInfo: { | 20260506 23:54:06.705 | INFO | "gpu" : " 17 GB", | 20260506 23:54:06.705 | INFO | "huge" : "", | 20260506 23:54:06.705 | INFO | "huge_shared" : " 27 GB", | 20260506 23:54:06.705 | INFO | "malloc" : " 14 GB", | 20260506 23:54:06.705 | INFO | "pinned" : " 29 GB" | 20260506 23:54:06.705 | INFO | } | 20260506 23:54:06.705 | INFO | Perf report for the last 5.14651 seconds: | 20260506 23:54:06.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:54:06.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:06.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:06.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:06.705 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260506 23:54:06.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:06.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:06.705 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:54:06.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:06.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:06.705 | INFO | Uploaded 36GiB to GPU in 1.98648s (18.1225GiB/s) | 20260506 23:54:06.705 | INFO | Downloaded 34GiB to GPU in 1.94881s (17.4465GiB/s) | 20260506 23:54:11.843 | INFO | Analyzing chunk frames = [5454848,5455360) | 20260506 23:54:11.843 | INFO | MemInfo: { | 20260506 23:54:11.843 | INFO | "gpu" : " 17 GB", | 20260506 23:54:11.844 | INFO | "huge" : "", | 20260506 23:54:11.844 | INFO | "huge_shared" : " 27 GB", | 20260506 23:54:11.844 | INFO | "malloc" : " 14 GB", | 20260506 23:54:11.844 | INFO | "pinned" : " 29 GB" | 20260506 23:54:11.844 | INFO | } | 20260506 23:54:11.844 | INFO | Perf report for the last 5.1387 seconds: | 20260506 23:54:11.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:54:11.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:11.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:11.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:11.844 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:54:11.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:11.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:11.844 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 23:54:11.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:11.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:11.844 | INFO | Uploaded 36GiB to GPU in 1.96925s (18.281GiB/s) | 20260506 23:54:11.844 | INFO | Downloaded 34GiB to GPU in 1.99859s (17.012GiB/s) | 20260506 23:54:16.986 | INFO | Analyzing chunk frames = [5455360,5455872) | 20260506 23:54:16.986 | INFO | MemInfo: { | 20260506 23:54:16.986 | INFO | "gpu" : " 17 GB", | 20260506 23:54:16.986 | INFO | "huge" : "", | 20260506 23:54:16.986 | INFO | "huge_shared" : " 27 GB", | 20260506 23:54:16.986 | INFO | "malloc" : " 14 GB", | 20260506 23:54:16.986 | INFO | "pinned" : " 29 GB" | 20260506 23:54:16.986 | INFO | } | 20260506 23:54:16.986 | INFO | Perf report for the last 5.14237 seconds: | 20260506 23:54:16.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:54:16.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:16.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:16.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:16.986 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:54:16.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:16.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:16.986 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260506 23:54:16.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:16.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:16.986 | INFO | Uploaded 36GiB to GPU in 1.99017s (18.0889GiB/s) | 20260506 23:54:16.986 | INFO | Downloaded 34GiB to GPU in 1.9888s (17.0958GiB/s) | 20260506 23:54:16.986 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:54:16.986 | INFO | Total mem usage: 18 GiB | 20260506 23:54:16.986 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:54:16.986 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 23:54:16.986 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:54:16.986 | INFO | High Water Breakdown: | 20260506 23:54:16.986 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:54:16.986 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:54:16.986 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:54:16.986 | INFO | ------------------------------------------ | 20260506 23:54:16.986 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:54:16.986 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:54:16.986 | INFO | Total mem usage: 26.5 GiB | 20260506 23:54:16.986 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:54:16.986 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:54:16.986 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:54:16.986 | INFO | High Water Breakdown: | 20260506 23:54:16.986 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:54:16.986 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:54:16.986 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:54:16.986 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:54:16.987 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:54:16.987 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:54:16.987 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:54:16.987 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:54:16.987 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:54:16.987 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:54:16.987 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:54:16.987 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:54:16.987 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:54:16.987 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:54:16.987 | INFO | ------------------------------------------ | 20260506 23:54:16.987 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:54:16.987 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:54:16.987 | INFO | Total mem usage: 25.6 GiB | 20260506 23:54:16.987 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:54:16.987 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:54:16.987 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:54:16.987 | INFO | High Water Breakdown: | 20260506 23:54:16.987 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:54:16.987 | INFO | ------------------------------------------ | 20260506 23:54:16.987 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:54:16.987 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:54:16.987 | INFO | Total mem usage: 15.7 GiB | 20260506 23:54:16.987 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:54:16.987 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:54:16.987 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:54:16.987 | INFO | High Water Breakdown: | 20260506 23:54:16.987 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:54:16.987 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:54:16.987 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:54:16.987 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:54:16.987 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:54:16.987 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:54:16.987 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:54:16.987 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:54:16.987 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:54:16.987 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:54:16.987 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:54:16.987 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:54:16.987 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:54:16.987 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:54:16.987 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:54:16.987 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:54:16.987 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:54:16.987 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:54:16.987 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:54:16.987 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:54:16.987 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:54:16.987 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:54:16.987 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:54:16.987 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:54:16.987 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:54:16.988 | INFO | ------------------------------------------ | 20260506 23:54:16.988 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:54:16.988 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:54:16.988 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:54:16.988 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:54:16.988 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.62e+01, 1.34e+01 / 2.55e+00 | 20260506 23:54:16.988 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.03e-01, 9.92e-03 / 5.13e-03 | 20260506 23:54:16.988 | INFO | PerfReport: 11.67%, Baseline -- 3.61e+00 / 4.45e+00, 5.44e-01 / 2.40e-01 | 20260506 23:54:16.988 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.49e+00, 1.78e-01 / 9.56e-02 | 20260506 23:54:16.988 | INFO | PerfReport: 16.07%, DME -- 3.98e+01 / 4.46e+01, 4.16e+00 / 2.88e+00 | 20260506 23:54:16.988 | INFO | PerfReport: 57.39%, FrameLabeling -- 1.78e+01 / 2.12e+01, 1.34e+00 / 2.12e+00 | 20260506 23:54:16.988 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.19e+00 / 2.71e+00, 3.84e-01 / 2.48e-01 | 20260506 23:54:16.988 | INFO | PerfReport: 3.18%, Metrics -- 9.85e-01 / 1.32e+00, 2.62e-01 / 1.74e-01 | 20260506 23:54:16.988 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.35e-01 / 3.85e-01, 5.32e-02 / 2.45e-02 | 20260506 23:54:16.988 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:54:16.988 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:54:16.988 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:54:16.988 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260506 23:54:16.988 | INFO | PerfReport: 21.48%, Upload -- 2.07e+01 / 0.00e+00, 6.87e+00 / 0.00e+00 | 20260506 23:54:16.988 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260506 23:54:16.988 | INFO | PerfReport: 32.08%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 23:54:16.988 | INFO | PerfReport: 1.24%, LockUpload -- 1.19e+00 / 0.00e+00, 4.65e+00 / 0.00e+00 | 20260506 23:54:16.988 | INFO | PerfReport: 1.33%, LockDownload -- 1.29e+00 / 0.00e+00, 3.23e+00 / 0.00e+00 | 20260506 23:54:16.988 | INFO | PerfReport: 22.52%, LockCompute -- 2.17e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260506 23:54:16.988 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260506 23:54:17.085 | INFO | Starting a round of Baz IO | 20260506 23:54:20.202 | INFO | Populating RTMetrics report | 20260506 23:54:20.202 | INFO | RTMetrics report populated | 20260506 23:54:20.202 | INFO | Opening tmp RTMetrics json file | 20260506 23:54:20.203 | INFO | Serializing RTMetrics to json | 20260506 23:54:20.210 | INFO | Serialization done, swapping tmp file now... | 20260506 23:54:20.224 | INFO | RTMetrics done | 20260506 23:54:20.295 | INFO | Finished a round of Baz IO: Wrote 1.865 GiB in 3.21 seconds | 20260506 23:54:20.295 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.38% / 0.00% / 66.62% | 20260506 23:54:20.295 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.381331% / 3.97364e-06% / 7.15256e-05% | 20260506 23:54:20.295 | INFO | Baz IO overall: | 20260506 23:54:20.295 | INFO | BytesWritten : 2850604730770 | 20260506 23:54:20.295 | INFO | NumEvents : 1373856744660 | 20260506 23:54:22.121 | INFO | Analyzing chunk frames = [5455872,5456384) | 20260506 23:54:22.121 | INFO | MemInfo: { | 20260506 23:54:22.121 | INFO | "gpu" : " 17 GB", | 20260506 23:54:22.121 | INFO | "huge" : "", | 20260506 23:54:22.121 | INFO | "huge_shared" : " 27 GB", | 20260506 23:54:22.121 | INFO | "malloc" : " 14 GB", | 20260506 23:54:22.121 | INFO | "pinned" : " 29 GB" | 20260506 23:54:22.121 | INFO | } | 20260506 23:54:22.121 | INFO | Perf report for the last 5.13521 seconds: | 20260506 23:54:22.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:54:22.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:22.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:22.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:22.121 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 23:54:22.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:22.121 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 23:54:22.121 | INFO | PRE_HQ : 59.5%, 1.8, 56.3ms, 0 | 20260506 23:54:22.121 | INFO | BAZWRITER : 62.1%, 3.5, 110.9ms, 0 | 20260506 23:54:22.121 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:54:22.122 | INFO | Uploaded 36GiB to GPU in 2.02709s (17.7595GiB/s) | 20260506 23:54:22.122 | INFO | Downloaded 38GiB to GPU in 2.18018s (17.4298GiB/s) | 20260506 23:54:27.263 | INFO | Analyzing chunk frames = [5456384,5456896) | 20260506 23:54:27.263 | INFO | MemInfo: { | 20260506 23:54:27.263 | INFO | "gpu" : " 17 GB", | 20260506 23:54:27.263 | INFO | "huge" : "", | 20260506 23:54:27.263 | INFO | "huge_shared" : " 27 GB", | 20260506 23:54:27.263 | INFO | "malloc" : " 14 GB", | 20260506 23:54:27.263 | INFO | "pinned" : " 29 GB" | 20260506 23:54:27.263 | INFO | } | 20260506 23:54:27.263 | INFO | Perf report for the last 5.14162 seconds: | 20260506 23:54:27.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:54:27.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:27.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:27.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:27.263 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260506 23:54:27.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:27.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:27.263 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260506 23:54:27.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:27.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:27.263 | INFO | Uploaded 36GiB to GPU in 1.99813s (18.0169GiB/s) | 20260506 23:54:27.263 | INFO | Downloaded 34GiB to GPU in 1.92442s (17.6677GiB/s) | 20260506 23:54:32.402 | INFO | Analyzing chunk frames = [5456896,5457408) | 20260506 23:54:32.402 | INFO | MemInfo: { | 20260506 23:54:32.402 | INFO | "gpu" : " 17 GB", | 20260506 23:54:32.402 | INFO | "huge" : "", | 20260506 23:54:32.402 | INFO | "huge_shared" : " 27 GB", | 20260506 23:54:32.402 | INFO | "malloc" : " 14 GB", | 20260506 23:54:32.402 | INFO | "pinned" : " 29 GB" | 20260506 23:54:32.402 | INFO | } | 20260506 23:54:32.402 | INFO | Perf report for the last 5.13948 seconds: | 20260506 23:54:32.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:54:32.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:32.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:32.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:32.402 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:54:32.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:32.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:32.402 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:54:32.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:32.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:32.403 | INFO | Uploaded 36GiB to GPU in 1.98995s (18.0909GiB/s) | 20260506 23:54:32.403 | INFO | Downloaded 34GiB to GPU in 1.92619s (17.6514GiB/s) | 20260506 23:54:37.537 | INFO | Analyzing chunk frames = [5457408,5457920) | 20260506 23:54:37.537 | INFO | MemInfo: { | 20260506 23:54:37.537 | INFO | "gpu" : " 17 GB", | 20260506 23:54:37.537 | INFO | "huge" : "", | 20260506 23:54:37.537 | INFO | "huge_shared" : " 27 GB", | 20260506 23:54:37.537 | INFO | "malloc" : " 14 GB", | 20260506 23:54:37.537 | INFO | "pinned" : " 29 GB" | 20260506 23:54:37.537 | INFO | } | 20260506 23:54:37.537 | INFO | Perf report for the last 5.13499 seconds: | 20260506 23:54:37.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:54:37.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:37.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:37.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:37.537 | INFO | ANALYSIS : 59.6%, 3.0, 92.3ms, 0 | 20260506 23:54:37.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:37.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:37.537 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:54:37.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:37.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:37.538 | INFO | Uploaded 36GiB to GPU in 1.96988s (18.2752GiB/s) | 20260506 23:54:37.538 | INFO | Downloaded 34GiB to GPU in 1.88031s (18.0821GiB/s) | 20260506 23:54:42.680 | INFO | Analyzing chunk frames = [5457920,5458432) | 20260506 23:54:42.680 | INFO | MemInfo: { | 20260506 23:54:42.680 | INFO | "gpu" : " 17 GB", | 20260506 23:54:42.680 | INFO | "huge" : "", | 20260506 23:54:42.680 | INFO | "huge_shared" : " 27 GB", | 20260506 23:54:42.680 | INFO | "malloc" : " 14 GB", | 20260506 23:54:42.680 | INFO | "pinned" : " 29 GB" | 20260506 23:54:42.680 | INFO | } | 20260506 23:54:42.680 | INFO | Perf report for the last 5.14257 seconds: | 20260506 23:54:42.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:54:42.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:42.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:42.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:42.680 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260506 23:54:42.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:42.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:42.680 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260506 23:54:42.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:42.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:42.680 | INFO | Uploaded 36GiB to GPU in 1.99253s (18.0675GiB/s) | 20260506 23:54:42.680 | INFO | Downloaded 34GiB to GPU in 1.92249s (17.6854GiB/s) | 20260506 23:54:47.823 | INFO | Analyzing chunk frames = [5458432,5458944) | 20260506 23:54:47.823 | INFO | MemInfo: { | 20260506 23:54:47.823 | INFO | "gpu" : " 17 GB", | 20260506 23:54:47.823 | INFO | "huge" : "", | 20260506 23:54:47.824 | INFO | "huge_shared" : " 27 GB", | 20260506 23:54:47.824 | INFO | "malloc" : " 14 GB", | 20260506 23:54:47.824 | INFO | "pinned" : " 29 GB" | 20260506 23:54:47.824 | INFO | } | 20260506 23:54:47.824 | INFO | Perf report for the last 5.14367 seconds: | 20260506 23:54:47.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:54:47.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:47.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:47.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:47.824 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260506 23:54:47.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:47.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:47.824 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:54:47.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:47.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:47.824 | INFO | Uploaded 36GiB to GPU in 2.01845s (17.8355GiB/s) | 20260506 23:54:47.824 | INFO | Downloaded 34GiB to GPU in 1.93219s (17.5966GiB/s) | 20260506 23:54:52.962 | INFO | Analyzing chunk frames = [5458944,5459456) | 20260506 23:54:52.962 | INFO | MemInfo: { | 20260506 23:54:52.962 | INFO | "gpu" : " 17 GB", | 20260506 23:54:52.962 | INFO | "huge" : "", | 20260506 23:54:52.962 | INFO | "huge_shared" : " 27 GB", | 20260506 23:54:52.962 | INFO | "malloc" : " 14 GB", | 20260506 23:54:52.962 | INFO | "pinned" : " 29 GB" | 20260506 23:54:52.962 | INFO | } | 20260506 23:54:52.962 | INFO | Perf report for the last 5.13812 seconds: | 20260506 23:54:52.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:54:52.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:52.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:52.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:52.962 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:54:52.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:52.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:52.962 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 23:54:52.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:52.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:52.962 | INFO | Uploaded 36GiB to GPU in 1.97364s (18.2404GiB/s) | 20260506 23:54:52.962 | INFO | Downloaded 34GiB to GPU in 1.99815s (17.0157GiB/s) | 20260506 23:54:58.100 | INFO | Analyzing chunk frames = [5459456,5459968) | 20260506 23:54:58.100 | INFO | MemInfo: { | 20260506 23:54:58.100 | INFO | "gpu" : " 17 GB", | 20260506 23:54:58.100 | INFO | "huge" : "", | 20260506 23:54:58.100 | INFO | "huge_shared" : " 27 GB", | 20260506 23:54:58.100 | INFO | "malloc" : " 14 GB", | 20260506 23:54:58.100 | INFO | "pinned" : " 29 GB" | 20260506 23:54:58.100 | INFO | } | 20260506 23:54:58.100 | INFO | Perf report for the last 5.13825 seconds: | 20260506 23:54:58.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:54:58.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:58.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:58.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:58.100 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:54:58.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:58.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:58.100 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:54:58.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:58.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:54:58.100 | INFO | Uploaded 36GiB to GPU in 1.98158s (18.1673GiB/s) | 20260506 23:54:58.100 | INFO | Downloaded 34GiB to GPU in 1.99293s (17.0603GiB/s) | 20260506 23:54:58.198 | INFO | Starting a round of Baz IO | 20260506 23:55:01.321 | INFO | Populating RTMetrics report | 20260506 23:55:01.321 | INFO | RTMetrics report populated | 20260506 23:55:01.321 | INFO | Opening tmp RTMetrics json file | 20260506 23:55:01.322 | INFO | Serializing RTMetrics to json | 20260506 23:55:01.323 | INFO | Serialization done, swapping tmp file now... | 20260506 23:55:01.324 | INFO | RTMetrics done | 20260506 23:55:01.417 | INFO | Finished a round of Baz IO: Wrote 1.866 GiB in 3.218 seconds | 20260506 23:55:01.417 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.38% / 0.00% / 66.62% | 20260506 23:55:01.417 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.381664% / 3.97364e-06% / 3.57628e-05% | 20260506 23:55:01.417 | INFO | Baz IO overall: | 20260506 23:55:01.417 | INFO | BytesWritten : 2852608485070 | 20260506 23:55:01.417 | INFO | NumEvents : 1374810527499 | 20260506 23:55:03.235 | INFO | Analyzing chunk frames = [5459968,5460480) | 20260506 23:55:03.235 | INFO | MemInfo: { | 20260506 23:55:03.235 | INFO | "gpu" : " 17 GB", | 20260506 23:55:03.235 | INFO | "huge" : "", | 20260506 23:55:03.235 | INFO | "huge_shared" : " 27 GB", | 20260506 23:55:03.235 | INFO | "malloc" : " 14 GB", | 20260506 23:55:03.235 | INFO | "pinned" : " 29 GB" | 20260506 23:55:03.235 | INFO | } | 20260506 23:55:03.235 | INFO | Perf report for the last 5.13505 seconds: | 20260506 23:55:03.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:55:03.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:03.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:03.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:03.235 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260506 23:55:03.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:03.235 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260506 23:55:03.235 | INFO | PRE_HQ : 59.8%, 1.8, 56.6ms, 0 | 20260506 23:55:03.235 | INFO | BAZWRITER : 62.3%, 3.4, 109.3ms, 0 | 20260506 23:55:03.235 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:55:03.235 | INFO | Uploaded 36GiB to GPU in 2.03665s (17.676GiB/s) | 20260506 23:55:03.235 | INFO | Downloaded 38GiB to GPU in 2.18753s (17.3712GiB/s) | 20260506 23:55:08.371 | INFO | Analyzing chunk frames = [5460480,5460992) | 20260506 23:55:08.371 | INFO | MemInfo: { | 20260506 23:55:08.371 | INFO | "gpu" : " 17 GB", | 20260506 23:55:08.371 | INFO | "huge" : "", | 20260506 23:55:08.371 | INFO | "huge_shared" : " 27 GB", | 20260506 23:55:08.371 | INFO | "malloc" : " 14 GB", | 20260506 23:55:08.371 | INFO | "pinned" : " 29 GB" | 20260506 23:55:08.371 | INFO | } | 20260506 23:55:08.371 | INFO | Perf report for the last 5.13603 seconds: | 20260506 23:55:08.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:55:08.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:08.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:08.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:08.371 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 23:55:08.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:08.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:08.371 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:55:08.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:08.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:08.371 | INFO | Uploaded 36GiB to GPU in 1.99228s (18.0697GiB/s) | 20260506 23:55:08.371 | INFO | Downloaded 34GiB to GPU in 1.93143s (17.6036GiB/s) | 20260506 23:55:13.512 | INFO | Analyzing chunk frames = [5460992,5461504) | 20260506 23:55:13.512 | INFO | MemInfo: { | 20260506 23:55:13.512 | INFO | "gpu" : " 17 GB", | 20260506 23:55:13.512 | INFO | "huge" : "", | 20260506 23:55:13.512 | INFO | "huge_shared" : " 27 GB", | 20260506 23:55:13.513 | INFO | "malloc" : " 14 GB", | 20260506 23:55:13.513 | INFO | "pinned" : " 29 GB" | 20260506 23:55:13.513 | INFO | } | 20260506 23:55:13.513 | INFO | Perf report for the last 5.14148 seconds: | 20260506 23:55:13.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:55:13.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:13.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:13.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:13.513 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260506 23:55:13.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:13.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:13.513 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:55:13.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:13.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:13.513 | INFO | Uploaded 36GiB to GPU in 1.96181s (18.3504GiB/s) | 20260506 23:55:13.513 | INFO | Downloaded 34GiB to GPU in 1.92786s (17.6362GiB/s) | 20260506 23:55:18.656 | INFO | Analyzing chunk frames = [5461504,5462016) | 20260506 23:55:18.656 | INFO | MemInfo: { | 20260506 23:55:18.656 | INFO | "gpu" : " 17 GB", | 20260506 23:55:18.657 | INFO | "huge" : "", | 20260506 23:55:18.657 | INFO | "huge_shared" : " 27 GB", | 20260506 23:55:18.657 | INFO | "malloc" : " 14 GB", | 20260506 23:55:18.657 | INFO | "pinned" : " 29 GB" | 20260506 23:55:18.657 | INFO | } | 20260506 23:55:18.657 | INFO | Perf report for the last 5.14396 seconds: | 20260506 23:55:18.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:55:18.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:18.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:18.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:18.657 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260506 23:55:18.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:18.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:18.657 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:55:18.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:18.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:18.657 | INFO | Uploaded 36GiB to GPU in 1.95084s (18.4536GiB/s) | 20260506 23:55:18.657 | INFO | Downloaded 34GiB to GPU in 1.87956s (18.0894GiB/s) | 20260506 23:55:23.797 | INFO | Analyzing chunk frames = [5462016,5462528) | 20260506 23:55:23.797 | INFO | MemInfo: { | 20260506 23:55:23.797 | INFO | "gpu" : " 17 GB", | 20260506 23:55:23.797 | INFO | "huge" : "", | 20260506 23:55:23.797 | INFO | "huge_shared" : " 27 GB", | 20260506 23:55:23.797 | INFO | "malloc" : " 14 GB", | 20260506 23:55:23.797 | INFO | "pinned" : " 29 GB" | 20260506 23:55:23.797 | INFO | } | 20260506 23:55:23.797 | INFO | Perf report for the last 5.14085 seconds: | 20260506 23:55:23.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:55:23.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:23.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:23.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:23.797 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 23:55:23.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:23.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:23.797 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:55:23.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:23.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:23.798 | INFO | Uploaded 36GiB to GPU in 1.96657s (18.306GiB/s) | 20260506 23:55:23.798 | INFO | Downloaded 34GiB to GPU in 1.93465s (17.5742GiB/s) | 20260506 23:55:28.937 | INFO | Analyzing chunk frames = [5462528,5463040) | 20260506 23:55:28.937 | INFO | MemInfo: { | 20260506 23:55:28.937 | INFO | "gpu" : " 17 GB", | 20260506 23:55:28.937 | INFO | "huge" : "", | 20260506 23:55:28.937 | INFO | "huge_shared" : " 27 GB", | 20260506 23:55:28.937 | INFO | "malloc" : " 14 GB", | 20260506 23:55:28.937 | INFO | "pinned" : " 29 GB" | 20260506 23:55:28.937 | INFO | } | 20260506 23:55:28.937 | INFO | Perf report for the last 5.14003 seconds: | 20260506 23:55:28.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:55:28.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:28.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:28.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:28.938 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 23:55:28.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:28.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:28.938 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260506 23:55:28.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:28.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:28.938 | INFO | Uploaded 36GiB to GPU in 2.04458s (17.6075GiB/s) | 20260506 23:55:28.938 | INFO | Downloaded 34GiB to GPU in 1.95695s (17.3739GiB/s) | 20260506 23:55:34.079 | INFO | Analyzing chunk frames = [5463040,5463552) | 20260506 23:55:34.079 | INFO | MemInfo: { | 20260506 23:55:34.079 | INFO | "gpu" : " 17 GB", | 20260506 23:55:34.079 | INFO | "huge" : "", | 20260506 23:55:34.079 | INFO | "huge_shared" : " 27 GB", | 20260506 23:55:34.079 | INFO | "malloc" : " 14 GB", | 20260506 23:55:34.079 | INFO | "pinned" : " 29 GB" | 20260506 23:55:34.079 | INFO | } | 20260506 23:55:34.079 | INFO | Perf report for the last 5.14165 seconds: | 20260506 23:55:34.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:55:34.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:34.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:34.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:34.079 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:55:34.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:34.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:34.079 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:55:34.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:34.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:34.079 | INFO | Uploaded 36GiB to GPU in 1.97025s (18.2718GiB/s) | 20260506 23:55:34.080 | INFO | Downloaded 34GiB to GPU in 1.98452s (17.1326GiB/s) | 20260506 23:55:39.215 | INFO | Analyzing chunk frames = [5463552,5464064) | 20260506 23:55:39.215 | INFO | MemInfo: { | 20260506 23:55:39.215 | INFO | "gpu" : " 17 GB", | 20260506 23:55:39.215 | INFO | "huge" : "", | 20260506 23:55:39.215 | INFO | "huge_shared" : " 27 GB", | 20260506 23:55:39.215 | INFO | "malloc" : " 14 GB", | 20260506 23:55:39.215 | INFO | "pinned" : " 29 GB" | 20260506 23:55:39.215 | INFO | } | 20260506 23:55:39.216 | INFO | Perf report for the last 5.13643 seconds: | 20260506 23:55:39.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:55:39.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:39.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:39.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:39.216 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260506 23:55:39.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:39.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:39.216 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:55:39.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:39.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:39.216 | INFO | Uploaded 36GiB to GPU in 1.97103s (18.2646GiB/s) | 20260506 23:55:39.216 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:55:39.216 | INFO | Total mem usage: 18 GiB | 20260506 23:55:39.216 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:55:39.216 | INFO | Downloaded 34GiB to GPU in 1.9996s (17.0034GiB/s) | 20260506 23:55:39.216 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260506 23:55:39.216 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:55:39.216 | INFO | High Water Breakdown: | 20260506 23:55:39.216 | INFO | 60.1% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:55:39.216 | INFO | 39.7% : 5.67 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:55:39.216 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:55:39.216 | INFO | ------------------------------------------ | 20260506 23:55:39.216 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:55:39.216 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:55:39.216 | INFO | Total mem usage: 26.5 GiB | 20260506 23:55:39.216 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:55:39.216 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:55:39.216 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:55:39.216 | INFO | High Water Breakdown: | 20260506 23:55:39.216 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:55:39.216 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:55:39.216 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:55:39.216 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:55:39.216 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:55:39.216 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:55:39.216 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:55:39.216 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:55:39.216 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:55:39.216 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:55:39.216 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:55:39.216 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:55:39.216 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:55:39.216 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:55:39.216 | INFO | ------------------------------------------ | 20260506 23:55:39.216 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:55:39.216 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:55:39.216 | INFO | Total mem usage: 25.6 GiB | 20260506 23:55:39.216 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:55:39.216 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:55:39.216 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:55:39.216 | INFO | High Water Breakdown: | 20260506 23:55:39.217 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:55:39.217 | INFO | ------------------------------------------ | 20260506 23:55:39.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:55:39.217 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:55:39.217 | INFO | Total mem usage: 15.7 GiB | 20260506 23:55:39.217 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:55:39.217 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:55:39.217 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 23:55:39.217 | INFO | High Water Breakdown: | 20260506 23:55:39.217 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:55:39.217 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:55:39.217 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:55:39.217 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:55:39.217 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:55:39.217 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:55:39.217 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:55:39.217 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:55:39.217 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:55:39.217 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:55:39.217 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:55:39.217 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:55:39.217 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:55:39.217 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:55:39.217 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:55:39.217 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:55:39.217 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:55:39.217 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:55:39.217 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:55:39.217 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:55:39.217 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:55:39.217 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:55:39.217 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:55:39.217 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:55:39.217 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:55:39.217 | INFO | ------------------------------------------ | 20260506 23:55:39.217 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:55:39.217 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:55:39.217 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:55:39.217 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:55:39.217 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.53e+01, 1.33e+01 / 2.33e+00 | 20260506 23:55:39.217 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.57e-03 / 6.10e-03 | 20260506 23:55:39.217 | INFO | PerfReport: 11.68%, Baseline -- 3.61e+00 / 4.35e+00, 5.41e-01 / 3.78e-01 | 20260506 23:55:39.217 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.45e+00, 1.75e-01 / 1.33e-01 | 20260506 23:55:39.217 | INFO | PerfReport: 16.07%, DME -- 3.98e+01 / 4.45e+01, 3.98e+00 / 2.73e+00 | 20260506 23:55:39.217 | INFO | PerfReport: 57.38%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.31e+00 / 1.56e+00 | 20260506 23:55:39.217 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.19e+00 / 2.75e+00, 3.83e-01 / 2.37e-01 | 20260506 23:55:39.217 | INFO | PerfReport: 3.18%, Metrics -- 9.83e-01 / 1.32e+00, 2.60e-01 / 1.36e-01 | 20260506 23:55:39.217 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.34e-01 / 3.82e-01, 5.22e-02 / 2.26e-02 | 20260506 23:55:39.217 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:55:39.217 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:55:39.217 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:55:39.217 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.31e+01 / 0.00e+00 | 20260506 23:55:39.217 | INFO | PerfReport: 21.49%, Upload -- 2.07e+01 / 0.00e+00, 6.89e+00 / 0.00e+00 | 20260506 23:55:39.217 | INFO | PerfReport: 21.30%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260506 23:55:39.217 | INFO | PerfReport: 32.09%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 23:55:39.217 | INFO | PerfReport: 1.26%, LockUpload -- 1.21e+00 / 0.00e+00, 4.68e+00 / 0.00e+00 | 20260506 23:55:39.217 | INFO | PerfReport: 1.30%, LockDownload -- 1.26e+00 / 0.00e+00, 3.15e+00 / 0.00e+00 | 20260506 23:55:39.217 | INFO | PerfReport: 22.55%, LockCompute -- 2.17e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 23:55:39.217 | INFO | PerfReport: 0.02%, Misc -- 1.52e-02 / 0.00e+00, 2.99e-02 / 0.00e+00 | 20260506 23:55:39.314 | INFO | Starting a round of Baz IO | 20260506 23:55:42.445 | INFO | Populating RTMetrics report | 20260506 23:55:42.445 | INFO | RTMetrics report populated | 20260506 23:55:42.445 | INFO | Opening tmp RTMetrics json file | 20260506 23:55:42.446 | INFO | Serializing RTMetrics to json | 20260506 23:55:42.464 | INFO | Serialization done, swapping tmp file now... | 20260506 23:55:42.466 | INFO | RTMetrics done | 20260506 23:55:42.542 | INFO | Finished a round of Baz IO: Wrote 1.867 GiB in 3.228 seconds | 20260506 23:55:42.542 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.38% / 0.00% / 66.62% | 20260506 23:55:42.542 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.382193% / 0% / 6.75519e-05% | 20260506 23:55:42.542 | INFO | Baz IO overall: | 20260506 23:55:42.542 | INFO | BytesWritten : 2854612709240 | 20260506 23:55:42.542 | INFO | NumEvents : 1375764839852 | 20260506 23:55:44.352 | INFO | Analyzing chunk frames = [5464064,5464576) | 20260506 23:55:44.352 | INFO | MemInfo: { | 20260506 23:55:44.352 | INFO | "gpu" : " 17 GB", | 20260506 23:55:44.352 | INFO | "huge" : "", | 20260506 23:55:44.352 | INFO | "huge_shared" : " 27 GB", | 20260506 23:55:44.352 | INFO | "malloc" : " 14 GB", | 20260506 23:55:44.352 | INFO | "pinned" : " 29 GB" | 20260506 23:55:44.352 | INFO | } | 20260506 23:55:44.352 | INFO | Perf report for the last 5.13649 seconds: | 20260506 23:55:44.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:55:44.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:44.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:44.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:44.352 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260506 23:55:44.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:44.352 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260506 23:55:44.352 | INFO | PRE_HQ : 59.8%, 1.8, 56.3ms, 0 | 20260506 23:55:44.352 | INFO | BAZWRITER : 62.4%, 3.4, 109.9ms, 0 | 20260506 23:55:44.352 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:55:44.352 | INFO | Uploaded 36GiB to GPU in 2.03145s (17.7214GiB/s) | 20260506 23:55:44.352 | INFO | Downloaded 38GiB to GPU in 2.16721s (17.534GiB/s) | 20260506 23:55:49.492 | INFO | Analyzing chunk frames = [5464576,5465088) | 20260506 23:55:49.492 | INFO | MemInfo: { | 20260506 23:55:49.492 | INFO | "gpu" : " 17 GB", | 20260506 23:55:49.492 | INFO | "huge" : "", | 20260506 23:55:49.492 | INFO | "huge_shared" : " 27 GB", | 20260506 23:55:49.492 | INFO | "malloc" : " 14 GB", | 20260506 23:55:49.492 | INFO | "pinned" : " 29 GB" | 20260506 23:55:49.492 | INFO | } | 20260506 23:55:49.492 | INFO | Perf report for the last 5.14038 seconds: | 20260506 23:55:49.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:55:49.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:49.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:49.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:49.493 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:55:49.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:49.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:49.493 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260506 23:55:49.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:49.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:49.493 | INFO | Uploaded 36GiB to GPU in 2.00015s (17.9987GiB/s) | 20260506 23:55:49.493 | INFO | Downloaded 34GiB to GPU in 1.92478s (17.6644GiB/s) | 20260506 23:55:54.629 | INFO | Analyzing chunk frames = [5465088,5465600) | 20260506 23:55:54.629 | INFO | MemInfo: { | 20260506 23:55:54.629 | INFO | "gpu" : " 17 GB", | 20260506 23:55:54.629 | INFO | "huge" : "", | 20260506 23:55:54.629 | INFO | "huge_shared" : " 27 GB", | 20260506 23:55:54.629 | INFO | "malloc" : " 14 GB", | 20260506 23:55:54.629 | INFO | "pinned" : " 29 GB" | 20260506 23:55:54.629 | INFO | } | 20260506 23:55:54.630 | INFO | Perf report for the last 5.13705 seconds: | 20260506 23:55:54.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:55:54.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:54.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:54.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:54.630 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:55:54.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:54.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:54.630 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260506 23:55:54.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:54.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:54.630 | INFO | Uploaded 36GiB to GPU in 2.00104s (17.9907GiB/s) | 20260506 23:55:54.630 | INFO | Downloaded 34GiB to GPU in 1.9327s (17.592GiB/s) | 20260506 23:55:59.770 | INFO | Analyzing chunk frames = [5465600,5466112) | 20260506 23:55:59.770 | INFO | MemInfo: { | 20260506 23:55:59.770 | INFO | "gpu" : " 17 GB", | 20260506 23:55:59.770 | INFO | "huge" : "", | 20260506 23:55:59.770 | INFO | "huge_shared" : " 27 GB", | 20260506 23:55:59.770 | INFO | "malloc" : " 14 GB", | 20260506 23:55:59.770 | INFO | "pinned" : " 29 GB" | 20260506 23:55:59.770 | INFO | } | 20260506 23:55:59.770 | INFO | Perf report for the last 5.14055 seconds: | 20260506 23:55:59.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:55:59.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:59.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:59.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:59.770 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260506 23:55:59.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:59.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:59.770 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:55:59.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:59.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:55:59.770 | INFO | Uploaded 36GiB to GPU in 2.01691s (17.8491GiB/s) | 20260506 23:55:59.770 | INFO | Downloaded 34GiB to GPU in 1.90877s (17.8125GiB/s) | 20260506 23:56:04.915 | INFO | Analyzing chunk frames = [5466112,5466624) | 20260506 23:56:04.915 | INFO | MemInfo: { | 20260506 23:56:04.915 | INFO | "gpu" : " 17 GB", | 20260506 23:56:04.915 | INFO | "huge" : "", | 20260506 23:56:04.915 | INFO | "huge_shared" : " 27 GB", | 20260506 23:56:04.915 | INFO | "malloc" : " 14 GB", | 20260506 23:56:04.915 | INFO | "pinned" : " 29 GB" | 20260506 23:56:04.915 | INFO | } | 20260506 23:56:04.915 | INFO | Perf report for the last 5.1452 seconds: | 20260506 23:56:04.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:56:04.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:04.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:04.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:04.915 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260506 23:56:04.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:04.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:04.915 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260506 23:56:04.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:04.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:04.915 | INFO | Uploaded 36GiB to GPU in 1.9925s (18.0678GiB/s) | 20260506 23:56:04.916 | INFO | Downloaded 34GiB to GPU in 1.93524s (17.5688GiB/s) | 20260506 23:56:10.050 | INFO | Analyzing chunk frames = [5466624,5467136) | 20260506 23:56:10.050 | INFO | MemInfo: { | 20260506 23:56:10.050 | INFO | "gpu" : " 17 GB", | 20260506 23:56:10.050 | INFO | "huge" : "", | 20260506 23:56:10.050 | INFO | "huge_shared" : " 27 GB", | 20260506 23:56:10.050 | INFO | "malloc" : " 14 GB", | 20260506 23:56:10.050 | INFO | "pinned" : " 29 GB" | 20260506 23:56:10.050 | INFO | } | 20260506 23:56:10.051 | INFO | Perf report for the last 5.13516 seconds: | 20260506 23:56:10.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:56:10.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:10.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:10.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:10.051 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260506 23:56:10.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:10.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:10.051 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 23:56:10.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:10.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:10.051 | INFO | Uploaded 36GiB to GPU in 2.05786s (17.4939GiB/s) | 20260506 23:56:10.051 | INFO | Downloaded 34GiB to GPU in 1.95714s (17.3722GiB/s) | 20260506 23:56:15.184 | INFO | Analyzing chunk frames = [5467136,5467648) | 20260506 23:56:15.184 | INFO | MemInfo: { | 20260506 23:56:15.184 | INFO | "gpu" : " 17 GB", | 20260506 23:56:15.184 | INFO | "huge" : "", | 20260506 23:56:15.184 | INFO | "huge_shared" : " 27 GB", | 20260506 23:56:15.184 | INFO | "malloc" : " 14 GB", | 20260506 23:56:15.184 | INFO | "pinned" : " 29 GB" | 20260506 23:56:15.184 | INFO | } | 20260506 23:56:15.184 | INFO | Perf report for the last 5.13398 seconds: | 20260506 23:56:15.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:56:15.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:15.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:15.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:15.185 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 23:56:15.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:15.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:15.185 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260506 23:56:15.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:15.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:15.185 | INFO | Uploaded 36GiB to GPU in 1.97368s (18.2401GiB/s) | 20260506 23:56:15.185 | INFO | Downloaded 34GiB to GPU in 1.97175s (17.2436GiB/s) | 20260506 23:56:20.325 | INFO | Analyzing chunk frames = [5467648,5468160) | 20260506 23:56:20.325 | INFO | MemInfo: { | 20260506 23:56:20.325 | INFO | "gpu" : " 17 GB", | 20260506 23:56:20.325 | INFO | "huge" : "", | 20260506 23:56:20.325 | INFO | "huge_shared" : " 27 GB", | 20260506 23:56:20.325 | INFO | "malloc" : " 14 GB", | 20260506 23:56:20.325 | INFO | "pinned" : " 29 GB" | 20260506 23:56:20.325 | INFO | } | 20260506 23:56:20.326 | INFO | Perf report for the last 5.14108 seconds: | 20260506 23:56:20.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:56:20.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:20.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:20.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:20.326 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:56:20.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:20.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:20.326 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260506 23:56:20.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:20.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:20.326 | INFO | Uploaded 36GiB to GPU in 1.97636s (18.2153GiB/s) | 20260506 23:56:20.326 | INFO | Downloaded 34GiB to GPU in 1.97226s (17.2391GiB/s) | 20260506 23:56:20.424 | INFO | Starting a round of Baz IO | 20260506 23:56:23.551 | INFO | Populating RTMetrics report | 20260506 23:56:23.551 | INFO | RTMetrics report populated | 20260506 23:56:23.551 | INFO | Opening tmp RTMetrics json file | 20260506 23:56:23.552 | INFO | Serializing RTMetrics to json | 20260506 23:56:23.566 | INFO | Serialization done, swapping tmp file now... | 20260506 23:56:23.572 | INFO | RTMetrics done | 20260506 23:56:23.643 | INFO | Finished a round of Baz IO: Wrote 1.867 GiB in 3.219 seconds | 20260506 23:56:23.643 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.38% / 0.00% / 66.62% | 20260506 23:56:23.643 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.379662% / 7.94729e-06% / 9.53674e-05% | 20260506 23:56:23.643 | INFO | Baz IO overall: | 20260506 23:56:23.643 | INFO | BytesWritten : 2856617161339 | 20260506 23:56:23.643 | INFO | NumEvents : 1376719064386 | 20260506 23:56:25.462 | INFO | Analyzing chunk frames = [5468160,5468672) | 20260506 23:56:25.462 | INFO | MemInfo: { | 20260506 23:56:25.462 | INFO | "gpu" : " 17 GB", | 20260506 23:56:25.462 | INFO | "huge" : "", | 20260506 23:56:25.462 | INFO | "huge_shared" : " 27 GB", | 20260506 23:56:25.462 | INFO | "malloc" : " 14 GB", | 20260506 23:56:25.462 | INFO | "pinned" : " 29 GB" | 20260506 23:56:25.462 | INFO | } | 20260506 23:56:25.462 | INFO | Perf report for the last 5.13627 seconds: | 20260506 23:56:25.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:56:25.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:25.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:25.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:25.462 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 23:56:25.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:25.462 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260506 23:56:25.462 | INFO | PRE_HQ : 59.6%, 1.8, 56.5ms, 0 | 20260506 23:56:25.462 | INFO | BAZWRITER : 62.2%, 3.4, 109.3ms, 0 | 20260506 23:56:25.462 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:56:25.462 | INFO | Uploaded 36GiB to GPU in 2.02352s (17.7908GiB/s) | 20260506 23:56:25.462 | INFO | Downloaded 38GiB to GPU in 2.17843s (17.4438GiB/s) | 20260506 23:56:30.604 | INFO | Analyzing chunk frames = [5468672,5469184) | 20260506 23:56:30.604 | INFO | MemInfo: { | 20260506 23:56:30.604 | INFO | "gpu" : " 17 GB", | 20260506 23:56:30.604 | INFO | "huge" : "", | 20260506 23:56:30.604 | INFO | "huge_shared" : " 27 GB", | 20260506 23:56:30.604 | INFO | "malloc" : " 14 GB", | 20260506 23:56:30.604 | INFO | "pinned" : " 29 GB" | 20260506 23:56:30.604 | INFO | } | 20260506 23:56:30.604 | INFO | Perf report for the last 5.14231 seconds: | 20260506 23:56:30.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:56:30.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:30.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:30.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:30.604 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:56:30.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:30.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:30.604 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260506 23:56:30.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:30.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:30.604 | INFO | Uploaded 36GiB to GPU in 1.99547s (18.0409GiB/s) | 20260506 23:56:30.604 | INFO | Downloaded 34GiB to GPU in 1.93561s (17.5656GiB/s) | 20260506 23:56:35.752 | INFO | Analyzing chunk frames = [5469184,5469696) | 20260506 23:56:35.752 | INFO | MemInfo: { | 20260506 23:56:35.752 | INFO | "gpu" : " 17 GB", | 20260506 23:56:35.752 | INFO | "huge" : "", | 20260506 23:56:35.752 | INFO | "huge_shared" : " 27 GB", | 20260506 23:56:35.752 | INFO | "malloc" : " 14 GB", | 20260506 23:56:35.753 | INFO | "pinned" : " 29 GB" | 20260506 23:56:35.753 | INFO | } | 20260506 23:56:35.753 | INFO | Perf report for the last 5.1484 seconds: | 20260506 23:56:35.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:56:35.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:35.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:35.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:35.753 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260506 23:56:35.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:35.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:35.753 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:56:35.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:35.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:35.753 | INFO | Uploaded 36GiB to GPU in 1.96573s (18.3138GiB/s) | 20260506 23:56:35.753 | INFO | Downloaded 34GiB to GPU in 1.93109s (17.6066GiB/s) | 20260506 23:56:40.891 | INFO | Analyzing chunk frames = [5469696,5470208) | 20260506 23:56:40.891 | INFO | MemInfo: { | 20260506 23:56:40.891 | INFO | "gpu" : " 17 GB", | 20260506 23:56:40.891 | INFO | "huge" : "", | 20260506 23:56:40.891 | INFO | "huge_shared" : " 27 GB", | 20260506 23:56:40.891 | INFO | "malloc" : " 14 GB", | 20260506 23:56:40.891 | INFO | "pinned" : " 29 GB" | 20260506 23:56:40.891 | INFO | } | 20260506 23:56:40.891 | INFO | Perf report for the last 5.13846 seconds: | 20260506 23:56:40.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:56:40.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:40.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:40.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:40.891 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260506 23:56:40.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:40.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:40.891 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260506 23:56:40.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:40.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:40.891 | INFO | Uploaded 36GiB to GPU in 1.97606s (18.2181GiB/s) | 20260506 23:56:40.891 | INFO | Downloaded 34GiB to GPU in 1.92672s (17.6465GiB/s) | 20260506 23:56:46.027 | INFO | Analyzing chunk frames = [5470208,5470720) | 20260506 23:56:46.027 | INFO | MemInfo: { | 20260506 23:56:46.027 | INFO | "gpu" : " 17 GB", | 20260506 23:56:46.027 | INFO | "huge" : "", | 20260506 23:56:46.027 | INFO | "huge_shared" : " 27 GB", | 20260506 23:56:46.027 | INFO | "malloc" : " 14 GB", | 20260506 23:56:46.027 | INFO | "pinned" : " 29 GB" | 20260506 23:56:46.027 | INFO | } | 20260506 23:56:46.027 | INFO | Perf report for the last 5.13592 seconds: | 20260506 23:56:46.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:56:46.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:46.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:46.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:46.027 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 23:56:46.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:46.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:46.027 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:56:46.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:46.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:46.027 | INFO | Uploaded 36GiB to GPU in 2.0093s (17.9166GiB/s) | 20260506 23:56:46.027 | INFO | Downloaded 34GiB to GPU in 1.95407s (17.3996GiB/s) | 20260506 23:56:51.167 | INFO | Analyzing chunk frames = [5470720,5471232) | 20260506 23:56:51.167 | INFO | MemInfo: { | 20260506 23:56:51.167 | INFO | "gpu" : " 17 GB", | 20260506 23:56:51.167 | INFO | "huge" : "", | 20260506 23:56:51.167 | INFO | "huge_shared" : " 27 GB", | 20260506 23:56:51.167 | INFO | "malloc" : " 14 GB", | 20260506 23:56:51.167 | INFO | "pinned" : " 29 GB" | 20260506 23:56:51.167 | INFO | } | 20260506 23:56:51.167 | INFO | Perf report for the last 5.13996 seconds: | 20260506 23:56:51.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:56:51.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:51.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:51.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:51.167 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 23:56:51.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:51.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:51.167 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260506 23:56:51.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:51.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:51.167 | INFO | Uploaded 36GiB to GPU in 2.04809s (17.5773GiB/s) | 20260506 23:56:51.167 | INFO | Downloaded 34GiB to GPU in 1.96052s (17.3423GiB/s) | 20260506 23:56:56.299 | INFO | Analyzing chunk frames = [5471232,5471744) | 20260506 23:56:56.299 | INFO | MemInfo: { | 20260506 23:56:56.299 | INFO | "gpu" : " 17 GB", | 20260506 23:56:56.299 | INFO | "huge" : "", | 20260506 23:56:56.299 | INFO | "huge_shared" : " 27 GB", | 20260506 23:56:56.299 | INFO | "malloc" : " 14 GB", | 20260506 23:56:56.299 | INFO | "pinned" : " 29 GB" | 20260506 23:56:56.299 | INFO | } | 20260506 23:56:56.299 | INFO | Perf report for the last 5.13233 seconds: | 20260506 23:56:56.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:56:56.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:56.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:56.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:56.299 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:56:56.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:56.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:56.299 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260506 23:56:56.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:56.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:56:56.300 | INFO | Uploaded 36GiB to GPU in 1.97121s (18.2629GiB/s) | 20260506 23:56:56.300 | INFO | Downloaded 34GiB to GPU in 2.00024s (16.9979GiB/s) | 20260506 23:57:01.439 | INFO | Analyzing chunk frames = [5471744,5472256) | 20260506 23:57:01.439 | INFO | MemInfo: { | 20260506 23:57:01.439 | INFO | "gpu" : " 17 GB", | 20260506 23:57:01.439 | INFO | "huge" : "", | 20260506 23:57:01.439 | INFO | "huge_shared" : " 27 GB", | 20260506 23:57:01.439 | INFO | "malloc" : " 14 GB", | 20260506 23:57:01.439 | INFO | "pinned" : " 29 GB" | 20260506 23:57:01.439 | INFO | } | 20260506 23:57:01.439 | INFO | Perf report for the last 5.13995 seconds: | 20260506 23:57:01.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:57:01.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:01.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:01.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:01.439 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260506 23:57:01.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:01.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:01.439 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:57:01.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:01.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:01.439 | INFO | Uploaded 36GiB to GPU in 1.99999s (18.0001GiB/s) | 20260506 23:57:01.440 | INFO | Downloaded 34GiB to GPU in 1.99463s (17.0457GiB/s) | 20260506 23:57:01.440 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:57:01.440 | INFO | Total mem usage: 18 GiB | 20260506 23:57:01.440 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:57:01.440 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260506 23:57:01.440 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:57:01.440 | INFO | High Water Breakdown: | 20260506 23:57:01.440 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:57:01.440 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:57:01.440 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:57:01.440 | INFO | ------------------------------------------ | 20260506 23:57:01.440 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:57:01.440 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:57:01.440 | INFO | Total mem usage: 26.5 GiB | 20260506 23:57:01.440 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:57:01.440 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:57:01.440 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:57:01.440 | INFO | High Water Breakdown: | 20260506 23:57:01.440 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:57:01.440 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:57:01.440 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:57:01.440 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:57:01.440 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:57:01.440 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:57:01.440 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:57:01.440 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:57:01.440 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:57:01.440 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:57:01.440 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:57:01.440 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:57:01.440 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:57:01.440 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:57:01.440 | INFO | ------------------------------------------ | 20260506 23:57:01.440 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:57:01.440 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:57:01.440 | INFO | Total mem usage: 25.6 GiB | 20260506 23:57:01.440 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:57:01.440 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:57:01.440 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:57:01.440 | INFO | High Water Breakdown: | 20260506 23:57:01.440 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:57:01.440 | INFO | ------------------------------------------ | 20260506 23:57:01.440 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:57:01.440 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:57:01.440 | INFO | Total mem usage: 15.7 GiB | 20260506 23:57:01.440 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:57:01.441 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:57:01.441 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260506 23:57:01.441 | INFO | High Water Breakdown: | 20260506 23:57:01.441 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:57:01.441 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:57:01.441 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:57:01.441 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:57:01.441 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:57:01.441 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:57:01.441 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:57:01.441 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:57:01.441 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:57:01.441 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:57:01.441 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:57:01.441 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:57:01.441 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:57:01.441 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:57:01.441 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:57:01.441 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:57:01.441 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:57:01.441 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:57:01.441 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:57:01.441 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:57:01.441 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:57:01.441 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:57:01.441 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:57:01.441 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:57:01.441 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:57:01.441 | INFO | ------------------------------------------ | 20260506 23:57:01.441 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:57:01.441 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:57:01.441 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:57:01.441 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:57:01.441 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.58e+01, 1.34e+01 / 2.04e+00 | 20260506 23:57:01.441 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.22e-03 / 8.30e-03 | 20260506 23:57:01.441 | INFO | PerfReport: 11.63%, Baseline -- 3.60e+00 / 4.38e+00, 5.24e-01 / 3.35e-01 | 20260506 23:57:01.441 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.45e+00, 1.74e-01 / 1.22e-01 | 20260506 23:57:01.441 | INFO | PerfReport: 16.06%, DME -- 3.98e+01 / 4.46e+01, 4.03e+00 / 2.73e+00 | 20260506 23:57:01.441 | INFO | PerfReport: 57.48%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.38e+00 / 1.81e+00 | 20260506 23:57:01.441 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.18e+00 / 2.76e+00, 3.76e-01 / 1.89e-01 | 20260506 23:57:01.441 | INFO | PerfReport: 3.17%, Metrics -- 9.83e-01 / 1.33e+00, 2.61e-01 / 1.52e-01 | 20260506 23:57:01.441 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.35e-01 / 3.88e-01, 5.34e-02 / 2.26e-02 | 20260506 23:57:01.441 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:57:01.441 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:57:01.441 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:57:01.441 | INFO | PerfReport: Full Execution -- 9.67e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260506 23:57:01.441 | INFO | PerfReport: 21.57%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260506 23:57:01.441 | INFO | PerfReport: 21.31%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260506 23:57:01.441 | INFO | PerfReport: 32.03%, Compute -- 3.10e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260506 23:57:01.441 | INFO | PerfReport: 1.35%, LockUpload -- 1.30e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260506 23:57:01.441 | INFO | PerfReport: 1.28%, LockDownload -- 1.24e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260506 23:57:01.441 | INFO | PerfReport: 22.44%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:57:01.441 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260506 23:57:01.538 | INFO | Starting a round of Baz IO | 20260506 23:57:04.666 | INFO | Populating RTMetrics report | 20260506 23:57:04.666 | INFO | RTMetrics report populated | 20260506 23:57:04.666 | INFO | Opening tmp RTMetrics json file | 20260506 23:57:04.685 | INFO | Serializing RTMetrics to json | 20260506 23:57:04.687 | INFO | Serialization done, swapping tmp file now... | 20260506 23:57:04.687 | INFO | RTMetrics done | 20260506 23:57:04.761 | INFO | Finished a round of Baz IO: Wrote 1.866 GiB in 3.223 seconds | 20260506 23:57:04.761 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.37% / 0.00% / 66.62% | 20260506 23:57:04.761 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.377937% / 0% / 4.37101e-05% | 20260506 23:57:04.761 | INFO | Baz IO overall: | 20260506 23:57:04.761 | INFO | BytesWritten : 2858620241368 | 20260506 23:57:04.761 | INFO | NumEvents : 1377671300971 | 20260506 23:57:06.580 | INFO | Analyzing chunk frames = [5472256,5472768) | 20260506 23:57:06.580 | INFO | MemInfo: { | 20260506 23:57:06.580 | INFO | "gpu" : " 17 GB", | 20260506 23:57:06.580 | INFO | "huge" : "", | 20260506 23:57:06.580 | INFO | "huge_shared" : " 27 GB", | 20260506 23:57:06.580 | INFO | "malloc" : " 14 GB", | 20260506 23:57:06.580 | INFO | "pinned" : " 29 GB" | 20260506 23:57:06.580 | INFO | } | 20260506 23:57:06.580 | INFO | Perf report for the last 5.1405 seconds: | 20260506 23:57:06.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:57:06.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:06.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:06.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:06.580 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260506 23:57:06.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:06.580 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 23:57:06.580 | INFO | PRE_HQ : 59.7%, 1.8, 56.8ms, 0 | 20260506 23:57:06.580 | INFO | BAZWRITER : 62.3%, 3.4, 109.5ms, 0 | 20260506 23:57:06.580 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:57:06.580 | INFO | Uploaded 36GiB to GPU in 2.03383s (17.7006GiB/s) | 20260506 23:57:06.580 | INFO | Downloaded 38GiB to GPU in 2.18518s (17.3898GiB/s) | 20260506 23:57:11.722 | INFO | Analyzing chunk frames = [5472768,5473280) | 20260506 23:57:11.722 | INFO | MemInfo: { | 20260506 23:57:11.722 | INFO | "gpu" : " 17 GB", | 20260506 23:57:11.722 | INFO | "huge" : "", | 20260506 23:57:11.722 | INFO | "huge_shared" : " 27 GB", | 20260506 23:57:11.722 | INFO | "malloc" : " 14 GB", | 20260506 23:57:11.722 | INFO | "pinned" : " 29 GB" | 20260506 23:57:11.722 | INFO | } | 20260506 23:57:11.722 | INFO | Perf report for the last 5.14231 seconds: | 20260506 23:57:11.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:57:11.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:11.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:11.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:11.722 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:57:11.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:11.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:11.722 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260506 23:57:11.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:11.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:11.722 | INFO | Uploaded 36GiB to GPU in 1.99719s (18.0253GiB/s) | 20260506 23:57:11.722 | INFO | Downloaded 34GiB to GPU in 1.93337s (17.5859GiB/s) | 20260506 23:57:16.862 | INFO | Analyzing chunk frames = [5473280,5473792) | 20260506 23:57:16.862 | INFO | MemInfo: { | 20260506 23:57:16.862 | INFO | "gpu" : " 17 GB", | 20260506 23:57:16.862 | INFO | "huge" : "", | 20260506 23:57:16.862 | INFO | "huge_shared" : " 27 GB", | 20260506 23:57:16.862 | INFO | "malloc" : " 14 GB", | 20260506 23:57:16.862 | INFO | "pinned" : " 29 GB" | 20260506 23:57:16.862 | INFO | } | 20260506 23:57:16.862 | INFO | Perf report for the last 5.14031 seconds: | 20260506 23:57:16.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:57:16.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:16.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:16.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:16.862 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260506 23:57:16.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:16.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:16.862 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:57:16.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:16.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:16.863 | INFO | Uploaded 36GiB to GPU in 1.98423s (18.1431GiB/s) | 20260506 23:57:16.863 | INFO | Downloaded 34GiB to GPU in 1.93728s (17.5504GiB/s) | 20260506 23:57:22.001 | INFO | Analyzing chunk frames = [5473792,5474304) | 20260506 23:57:22.001 | INFO | MemInfo: { | 20260506 23:57:22.001 | INFO | "gpu" : " 17 GB", | 20260506 23:57:22.001 | INFO | "huge" : "", | 20260506 23:57:22.001 | INFO | "huge_shared" : " 27 GB", | 20260506 23:57:22.001 | INFO | "malloc" : " 14 GB", | 20260506 23:57:22.001 | INFO | "pinned" : " 29 GB" | 20260506 23:57:22.001 | INFO | } | 20260506 23:57:22.001 | INFO | Perf report for the last 5.13904 seconds: | 20260506 23:57:22.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:57:22.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:22.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:22.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:22.001 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260506 23:57:22.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:22.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:22.002 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:57:22.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:22.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:22.002 | INFO | Uploaded 36GiB to GPU in 1.96537s (18.3172GiB/s) | 20260506 23:57:22.002 | INFO | Downloaded 34GiB to GPU in 1.90788s (17.8208GiB/s) | 20260506 23:57:27.140 | INFO | Analyzing chunk frames = [5474304,5474816) | 20260506 23:57:27.140 | INFO | MemInfo: { | 20260506 23:57:27.140 | INFO | "gpu" : " 17 GB", | 20260506 23:57:27.140 | INFO | "huge" : "", | 20260506 23:57:27.140 | INFO | "huge_shared" : " 27 GB", | 20260506 23:57:27.140 | INFO | "malloc" : " 14 GB", | 20260506 23:57:27.140 | INFO | "pinned" : " 29 GB" | 20260506 23:57:27.140 | INFO | } | 20260506 23:57:27.141 | INFO | Perf report for the last 5.13906 seconds: | 20260506 23:57:27.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:57:27.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:27.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:27.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:27.141 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260506 23:57:27.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:27.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:27.141 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260506 23:57:27.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:27.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:27.141 | INFO | Uploaded 36GiB to GPU in 1.96938s (18.2798GiB/s) | 20260506 23:57:27.141 | INFO | Downloaded 34GiB to GPU in 1.93998s (17.5259GiB/s) | 20260506 23:57:32.272 | INFO | Analyzing chunk frames = [5474816,5475328) | 20260506 23:57:32.272 | INFO | MemInfo: { | 20260506 23:57:32.272 | INFO | "gpu" : " 17 GB", | 20260506 23:57:32.272 | INFO | "huge" : "", | 20260506 23:57:32.273 | INFO | "huge_shared" : " 27 GB", | 20260506 23:57:32.273 | INFO | "malloc" : " 14 GB", | 20260506 23:57:32.273 | INFO | "pinned" : " 29 GB" | 20260506 23:57:32.273 | INFO | } | 20260506 23:57:32.273 | INFO | Perf report for the last 5.13208 seconds: | 20260506 23:57:32.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:57:32.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:32.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:32.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:32.273 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 23:57:32.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:32.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:32.273 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260506 23:57:32.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:32.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:32.273 | INFO | Uploaded 36GiB to GPU in 2.02386s (17.7878GiB/s) | 20260506 23:57:32.273 | INFO | Downloaded 34GiB to GPU in 1.93943s (17.5309GiB/s) | 20260506 23:57:37.417 | INFO | Analyzing chunk frames = [5475328,5475840) | 20260506 23:57:37.417 | INFO | MemInfo: { | 20260506 23:57:37.417 | INFO | "gpu" : " 17 GB", | 20260506 23:57:37.417 | INFO | "huge" : "", | 20260506 23:57:37.417 | INFO | "huge_shared" : " 27 GB", | 20260506 23:57:37.417 | INFO | "malloc" : " 14 GB", | 20260506 23:57:37.417 | INFO | "pinned" : " 29 GB" | 20260506 23:57:37.417 | INFO | } | 20260506 23:57:37.417 | INFO | Perf report for the last 5.14466 seconds: | 20260506 23:57:37.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:57:37.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:37.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:37.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:37.417 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 23:57:37.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:37.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:37.418 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:57:37.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:37.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:37.418 | INFO | Uploaded 36GiB to GPU in 1.95639s (18.4012GiB/s) | 20260506 23:57:37.418 | INFO | Downloaded 34GiB to GPU in 1.98127s (17.1607GiB/s) | 20260506 23:57:42.561 | INFO | Analyzing chunk frames = [5475840,5476352) | 20260506 23:57:42.561 | INFO | MemInfo: { | 20260506 23:57:42.561 | INFO | "gpu" : " 17 GB", | 20260506 23:57:42.561 | INFO | "huge" : "", | 20260506 23:57:42.561 | INFO | "huge_shared" : " 27 GB", | 20260506 23:57:42.561 | INFO | "malloc" : " 14 GB", | 20260506 23:57:42.561 | INFO | "pinned" : " 29 GB" | 20260506 23:57:42.561 | INFO | } | 20260506 23:57:42.561 | INFO | Perf report for the last 5.1441 seconds: | 20260506 23:57:42.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:57:42.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:42.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:42.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:42.562 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260506 23:57:42.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:42.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:42.562 | INFO | PRE_HQ : 30.7%, 1.0, 15.9ms, 0 | 20260506 23:57:42.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:42.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:42.562 | INFO | Uploaded 36GiB to GPU in 1.97202s (18.2554GiB/s) | 20260506 23:57:42.562 | INFO | Downloaded 34GiB to GPU in 1.99422s (17.0493GiB/s) | 20260506 23:57:42.668 | INFO | Starting a round of Baz IO | 20260506 23:57:45.781 | INFO | Populating RTMetrics report | 20260506 23:57:45.782 | INFO | RTMetrics report populated | 20260506 23:57:45.782 | INFO | Opening tmp RTMetrics json file | 20260506 23:57:45.783 | INFO | Serializing RTMetrics to json | 20260506 23:57:45.802 | INFO | Serialization done, swapping tmp file now... | 20260506 23:57:45.803 | INFO | RTMetrics done | 20260506 23:57:45.874 | INFO | Finished a round of Baz IO: Wrote 1.865 GiB in 3.206 seconds | 20260506 23:57:45.874 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.37% / 0.00% / 66.63% | 20260506 23:57:45.874 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.376868% / 0% / 6.35783e-05% | 20260506 23:57:45.874 | INFO | Baz IO overall: | 20260506 23:57:45.875 | INFO | BytesWritten : 2860622335230 | 20260506 23:57:45.875 | INFO | NumEvents : 1378623177657 | 20260506 23:57:47.698 | INFO | Analyzing chunk frames = [5476352,5476864) | 20260506 23:57:47.698 | INFO | MemInfo: { | 20260506 23:57:47.698 | INFO | "gpu" : " 17 GB", | 20260506 23:57:47.698 | INFO | "huge" : "", | 20260506 23:57:47.698 | INFO | "huge_shared" : " 27 GB", | 20260506 23:57:47.698 | INFO | "malloc" : " 14 GB", | 20260506 23:57:47.698 | INFO | "pinned" : " 29 GB" | 20260506 23:57:47.698 | INFO | } | 20260506 23:57:47.698 | INFO | Perf report for the last 5.1369 seconds: | 20260506 23:57:47.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:57:47.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:47.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:47.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:47.698 | INFO | ANALYSIS : 62.1%, 3.0, 96.4ms, 0 | 20260506 23:57:47.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:47.698 | INFO | RT_METRICS : 43.8%, 1.0, 22.9ms, 0 | 20260506 23:57:47.698 | INFO | PRE_HQ : 59.8%, 1.9, 58.4ms, 0 | 20260506 23:57:47.698 | INFO | BAZWRITER : 61.8%, 3.5, 110.5ms, 0 | 20260506 23:57:47.698 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260506 23:57:47.699 | INFO | Uploaded 36GiB to GPU in 2.03533s (17.6875GiB/s) | 20260506 23:57:47.699 | INFO | Downloaded 38GiB to GPU in 2.18s (17.4312GiB/s) | 20260506 23:57:52.839 | INFO | Analyzing chunk frames = [5476864,5477376) | 20260506 23:57:52.839 | INFO | MemInfo: { | 20260506 23:57:52.839 | INFO | "gpu" : " 17 GB", | 20260506 23:57:52.839 | INFO | "huge" : "", | 20260506 23:57:52.839 | INFO | "huge_shared" : " 27 GB", | 20260506 23:57:52.840 | INFO | "malloc" : " 14 GB", | 20260506 23:57:52.840 | INFO | "pinned" : " 29 GB" | 20260506 23:57:52.840 | INFO | } | 20260506 23:57:52.840 | INFO | Perf report for the last 5.14132 seconds: | 20260506 23:57:52.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:57:52.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:52.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:52.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:52.840 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:57:52.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:52.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:52.840 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:57:52.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:52.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:52.840 | INFO | Uploaded 36GiB to GPU in 1.99722s (18.0251GiB/s) | 20260506 23:57:52.840 | INFO | Downloaded 34GiB to GPU in 1.93161s (17.6019GiB/s) | 20260506 23:57:57.976 | INFO | Analyzing chunk frames = [5477376,5477888) | 20260506 23:57:57.976 | INFO | MemInfo: { | 20260506 23:57:57.976 | INFO | "gpu" : " 17 GB", | 20260506 23:57:57.976 | INFO | "huge" : "", | 20260506 23:57:57.976 | INFO | "huge_shared" : " 27 GB", | 20260506 23:57:57.976 | INFO | "malloc" : " 14 GB", | 20260506 23:57:57.976 | INFO | "pinned" : " 29 GB" | 20260506 23:57:57.976 | INFO | } | 20260506 23:57:57.976 | INFO | Perf report for the last 5.13626 seconds: | 20260506 23:57:57.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:57:57.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:57.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:57.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:57.976 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260506 23:57:57.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:57.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:57.976 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:57:57.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:57.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:57:57.976 | INFO | Uploaded 36GiB to GPU in 2.01272s (17.8862GiB/s) | 20260506 23:57:57.976 | INFO | Downloaded 34GiB to GPU in 1.93529s (17.5684GiB/s) | 20260506 23:58:03.118 | INFO | Analyzing chunk frames = [5477888,5478400) | 20260506 23:58:03.118 | INFO | MemInfo: { | 20260506 23:58:03.118 | INFO | "gpu" : " 17 GB", | 20260506 23:58:03.118 | INFO | "huge" : "", | 20260506 23:58:03.118 | INFO | "huge_shared" : " 27 GB", | 20260506 23:58:03.118 | INFO | "malloc" : " 14 GB", | 20260506 23:58:03.118 | INFO | "pinned" : " 29 GB" | 20260506 23:58:03.118 | INFO | } | 20260506 23:58:03.118 | INFO | Perf report for the last 5.14216 seconds: | 20260506 23:58:03.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:58:03.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:03.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:03.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:03.118 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260506 23:58:03.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:03.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:03.118 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:58:03.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:03.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:03.118 | INFO | Uploaded 36GiB to GPU in 1.98502s (18.1358GiB/s) | 20260506 23:58:03.118 | INFO | Downloaded 34GiB to GPU in 1.90554s (17.8428GiB/s) | 20260506 23:58:08.253 | INFO | Analyzing chunk frames = [5478400,5478912) | 20260506 23:58:08.253 | INFO | MemInfo: { | 20260506 23:58:08.253 | INFO | "gpu" : " 17 GB", | 20260506 23:58:08.253 | INFO | "huge" : "", | 20260506 23:58:08.253 | INFO | "huge_shared" : " 27 GB", | 20260506 23:58:08.254 | INFO | "malloc" : " 14 GB", | 20260506 23:58:08.254 | INFO | "pinned" : " 29 GB" | 20260506 23:58:08.254 | INFO | } | 20260506 23:58:08.254 | INFO | Perf report for the last 5.1354 seconds: | 20260506 23:58:08.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:58:08.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:08.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:08.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:08.254 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260506 23:58:08.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:08.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:08.254 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260506 23:58:08.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:08.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:08.254 | INFO | Uploaded 36GiB to GPU in 1.9912s (18.0795GiB/s) | 20260506 23:58:08.254 | INFO | Downloaded 34GiB to GPU in 1.93346s (17.585GiB/s) | 20260506 23:58:13.386 | INFO | Analyzing chunk frames = [5478912,5479424) | 20260506 23:58:13.386 | INFO | MemInfo: { | 20260506 23:58:13.386 | INFO | "gpu" : " 17 GB", | 20260506 23:58:13.386 | INFO | "huge" : "", | 20260506 23:58:13.386 | INFO | "huge_shared" : " 27 GB", | 20260506 23:58:13.386 | INFO | "malloc" : " 14 GB", | 20260506 23:58:13.386 | INFO | "pinned" : " 29 GB" | 20260506 23:58:13.386 | INFO | } | 20260506 23:58:13.386 | INFO | Perf report for the last 5.13281 seconds: | 20260506 23:58:13.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:58:13.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:13.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:13.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:13.386 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260506 23:58:13.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:13.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:13.386 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260506 23:58:13.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:13.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:13.387 | INFO | Uploaded 36GiB to GPU in 2.01117s (17.9GiB/s) | 20260506 23:58:13.387 | INFO | Downloaded 34GiB to GPU in 1.95074s (17.4292GiB/s) | 20260506 23:58:18.536 | INFO | Analyzing chunk frames = [5479424,5479936) | 20260506 23:58:18.536 | INFO | MemInfo: { | 20260506 23:58:18.536 | INFO | "gpu" : " 17 GB", | 20260506 23:58:18.536 | INFO | "huge" : "", | 20260506 23:58:18.536 | INFO | "huge_shared" : " 27 GB", | 20260506 23:58:18.536 | INFO | "malloc" : " 14 GB", | 20260506 23:58:18.536 | INFO | "pinned" : " 29 GB" | 20260506 23:58:18.536 | INFO | } | 20260506 23:58:18.536 | INFO | Perf report for the last 5.14939 seconds: | 20260506 23:58:18.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:58:18.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:18.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:18.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:18.536 | INFO | ANALYSIS : 61.2%, 3.0, 95.5ms, 0 | 20260506 23:58:18.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:18.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:18.536 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260506 23:58:18.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:18.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:18.536 | INFO | Uploaded 36GiB to GPU in 1.98295s (18.1548GiB/s) | 20260506 23:58:18.536 | INFO | Downloaded 34GiB to GPU in 1.98573s (17.1221GiB/s) | 20260506 23:58:23.675 | INFO | Analyzing chunk frames = [5479936,5480448) | 20260506 23:58:23.675 | INFO | MemInfo: { | 20260506 23:58:23.675 | INFO | "gpu" : " 17 GB", | 20260506 23:58:23.675 | INFO | "huge" : "", | 20260506 23:58:23.675 | INFO | "huge_shared" : " 27 GB", | 20260506 23:58:23.675 | INFO | "malloc" : " 14 GB", | 20260506 23:58:23.675 | INFO | "pinned" : " 29 GB" | 20260506 23:58:23.675 | INFO | } | 20260506 23:58:23.675 | INFO | Perf report for the last 5.13926 seconds: | 20260506 23:58:23.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:58:23.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:23.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:23.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:23.675 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:58:23.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:23.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:23.675 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260506 23:58:23.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:23.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:23.675 | INFO | Uploaded 36GiB to GPU in 1.98705s (18.1173GiB/s) | 20260506 23:58:23.675 | INFO | Downloaded 34GiB to GPU in 1.99791s (17.0178GiB/s) | 20260506 23:58:23.676 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:58:23.676 | INFO | Total mem usage: 18 GiB | 20260506 23:58:23.676 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:58:23.676 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -25.1%) | 20260506 23:58:23.676 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:58:23.676 | INFO | High Water Breakdown: | 20260506 23:58:23.676 | INFO | 60.3% : 8.68 GiB ( 76% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:58:23.676 | INFO | 39.5% : 5.69 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:58:23.676 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:58:23.676 | INFO | ------------------------------------------ | 20260506 23:58:23.676 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:58:23.676 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:58:23.676 | INFO | Total mem usage: 26.5 GiB | 20260506 23:58:23.676 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:58:23.676 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:58:23.676 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:58:23.676 | INFO | High Water Breakdown: | 20260506 23:58:23.676 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:58:23.676 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:58:23.676 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:58:23.676 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:58:23.676 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:58:23.676 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:58:23.676 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:58:23.676 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:58:23.676 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:58:23.676 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:58:23.676 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:58:23.676 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:58:23.676 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:58:23.676 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:58:23.676 | INFO | ------------------------------------------ | 20260506 23:58:23.676 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:58:23.676 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:58:23.676 | INFO | Total mem usage: 25.6 GiB | 20260506 23:58:23.676 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:58:23.676 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260506 23:58:23.676 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260506 23:58:23.676 | INFO | High Water Breakdown: | 20260506 23:58:23.676 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260506 23:58:23.676 | INFO | ------------------------------------------ | 20260506 23:58:23.676 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:58:23.676 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:58:23.676 | INFO | Total mem usage: 15.7 GiB | 20260506 23:58:23.676 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:58:23.676 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:58:23.676 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 23:58:23.676 | INFO | High Water Breakdown: | 20260506 23:58:23.676 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:58:23.676 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:58:23.676 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:58:23.676 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:58:23.676 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:58:23.676 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:58:23.676 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:58:23.676 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:58:23.676 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:58:23.676 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:58:23.676 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:58:23.676 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:58:23.676 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:58:23.676 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:58:23.676 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:58:23.676 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:58:23.676 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:58:23.676 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:58:23.676 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:58:23.676 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:58:23.676 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:58:23.676 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:58:23.676 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:58:23.676 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:58:23.676 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:58:23.676 | INFO | ------------------------------------------ | 20260506 23:58:23.676 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:58:23.677 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:58:23.677 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:58:23.677 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:58:23.677 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.60e+01, 1.35e+01 / 2.57e+00 | 20260506 23:58:23.677 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.01e-01, 9.49e-03 / 5.89e-03 | 20260506 23:58:23.677 | INFO | PerfReport: 11.68%, Baseline -- 3.62e+00 / 4.34e+00, 5.49e-01 / 3.54e-01 | 20260506 23:58:23.677 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.43e+00, 1.80e-01 / 1.43e-01 | 20260506 23:58:23.677 | INFO | PerfReport: 16.08%, DME -- 3.99e+01 / 4.48e+01, 4.23e+00 / 2.74e+00 | 20260506 23:58:23.677 | INFO | PerfReport: 57.38%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.35e+00 / 1.68e+00 | 20260506 23:58:23.677 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.18e+00 / 2.70e+00, 3.80e-01 / 2.48e-01 | 20260506 23:58:23.677 | INFO | PerfReport: 3.18%, Metrics -- 9.85e-01 / 1.32e+00, 2.63e-01 / 1.51e-01 | 20260506 23:58:23.677 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.35e-01 / 3.87e-01, 5.30e-02 / 2.65e-02 | 20260506 23:58:23.677 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:58:23.677 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:58:23.677 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:58:23.677 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260506 23:58:23.677 | INFO | PerfReport: 21.50%, Upload -- 2.08e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260506 23:58:23.677 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260506 23:58:23.677 | INFO | PerfReport: 32.09%, Compute -- 3.10e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260506 23:58:23.677 | INFO | PerfReport: 1.24%, LockUpload -- 1.20e+00 / 0.00e+00, 4.57e+00 / 0.00e+00 | 20260506 23:58:23.677 | INFO | PerfReport: 1.31%, LockDownload -- 1.27e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260506 23:58:23.677 | INFO | PerfReport: 22.52%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260506 23:58:23.677 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260506 23:58:23.773 | INFO | Starting a round of Baz IO | 20260506 23:58:26.893 | INFO | Populating RTMetrics report | 20260506 23:58:26.893 | INFO | RTMetrics report populated | 20260506 23:58:26.893 | INFO | Opening tmp RTMetrics json file | 20260506 23:58:26.894 | INFO | Serializing RTMetrics to json | 20260506 23:58:26.906 | INFO | Serialization done, swapping tmp file now... | 20260506 23:58:26.915 | INFO | RTMetrics done | 20260506 23:58:26.985 | INFO | Finished a round of Baz IO: Wrote 1.865 GiB in 3.211 seconds | 20260506 23:58:26.985 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.37% / 0.00% / 66.63% | 20260506 23:58:26.985 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.377846% / 0% / 6.75519e-05% | 20260506 23:58:26.985 | INFO | Baz IO overall: | 20260506 23:58:26.985 | INFO | BytesWritten : 2862625324647 | 20260506 23:58:26.985 | INFO | NumEvents : 1379575253599 | 20260506 23:58:28.813 | INFO | Analyzing chunk frames = [5480448,5480960) | 20260506 23:58:28.813 | INFO | MemInfo: { | 20260506 23:58:28.813 | INFO | "gpu" : " 17 GB", | 20260506 23:58:28.813 | INFO | "huge" : "", | 20260506 23:58:28.813 | INFO | "huge_shared" : " 27 GB", | 20260506 23:58:28.813 | INFO | "malloc" : " 14 GB", | 20260506 23:58:28.813 | INFO | "pinned" : " 29 GB" | 20260506 23:58:28.813 | INFO | } | 20260506 23:58:28.813 | INFO | Perf report for the last 5.13828 seconds: | 20260506 23:58:28.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:58:28.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:28.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:28.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:28.814 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260506 23:58:28.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:28.814 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260506 23:58:28.814 | INFO | PRE_HQ : 59.6%, 1.8, 56.4ms, 0 | 20260506 23:58:28.814 | INFO | BAZWRITER : 62.1%, 3.7, 116.9ms, 0 | 20260506 23:58:28.814 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:58:28.814 | INFO | Uploaded 36GiB to GPU in 2.04395s (17.6129GiB/s) | 20260506 23:58:28.814 | INFO | Downloaded 38GiB to GPU in 2.18577s (17.3852GiB/s) | 20260506 23:58:33.953 | INFO | Analyzing chunk frames = [5480960,5481472) | 20260506 23:58:33.953 | INFO | MemInfo: { | 20260506 23:58:33.953 | INFO | "gpu" : " 17 GB", | 20260506 23:58:33.953 | INFO | "huge" : "", | 20260506 23:58:33.953 | INFO | "huge_shared" : " 27 GB", | 20260506 23:58:33.953 | INFO | "malloc" : " 14 GB", | 20260506 23:58:33.953 | INFO | "pinned" : " 29 GB" | 20260506 23:58:33.953 | INFO | } | 20260506 23:58:33.953 | INFO | Perf report for the last 5.13943 seconds: | 20260506 23:58:33.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:58:33.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:33.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:33.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:33.953 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260506 23:58:33.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:33.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:33.953 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260506 23:58:33.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:33.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:33.953 | INFO | Uploaded 36GiB to GPU in 2.01465s (17.8691GiB/s) | 20260506 23:58:33.953 | INFO | Downloaded 34GiB to GPU in 1.93081s (17.6091GiB/s) | 20260506 23:58:39.088 | INFO | Analyzing chunk frames = [5481472,5481984) | 20260506 23:58:39.088 | INFO | MemInfo: { | 20260506 23:58:39.088 | INFO | "gpu" : " 17 GB", | 20260506 23:58:39.088 | INFO | "huge" : "", | 20260506 23:58:39.088 | INFO | "huge_shared" : " 27 GB", | 20260506 23:58:39.088 | INFO | "malloc" : " 14 GB", | 20260506 23:58:39.088 | INFO | "pinned" : " 29 GB" | 20260506 23:58:39.088 | INFO | } | 20260506 23:58:39.088 | INFO | Perf report for the last 5.13566 seconds: | 20260506 23:58:39.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:58:39.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:39.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:39.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:39.089 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260506 23:58:39.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:39.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:39.089 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:58:39.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:39.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:39.089 | INFO | Uploaded 36GiB to GPU in 1.984s (18.1452GiB/s) | 20260506 23:58:39.089 | INFO | Downloaded 34GiB to GPU in 1.93026s (17.6142GiB/s) | 20260506 23:58:44.223 | INFO | Analyzing chunk frames = [5481984,5482496) | 20260506 23:58:44.223 | INFO | MemInfo: { | 20260506 23:58:44.223 | INFO | "gpu" : " 17 GB", | 20260506 23:58:44.223 | INFO | "huge" : "", | 20260506 23:58:44.223 | INFO | "huge_shared" : " 27 GB", | 20260506 23:58:44.223 | INFO | "malloc" : " 14 GB", | 20260506 23:58:44.223 | INFO | "pinned" : " 29 GB" | 20260506 23:58:44.223 | INFO | } | 20260506 23:58:44.223 | INFO | Perf report for the last 5.13421 seconds: | 20260506 23:58:44.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:58:44.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:44.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:44.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:44.223 | INFO | ANALYSIS : 59.6%, 3.0, 92.2ms, 0 | 20260506 23:58:44.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:44.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:44.223 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:58:44.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:44.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:44.223 | INFO | Uploaded 36GiB to GPU in 1.96506s (18.32GiB/s) | 20260506 23:58:44.223 | INFO | Downloaded 34GiB to GPU in 1.88056s (18.0797GiB/s) | 20260506 23:58:49.363 | INFO | Analyzing chunk frames = [5482496,5483008) | 20260506 23:58:49.363 | INFO | MemInfo: { | 20260506 23:58:49.363 | INFO | "gpu" : " 17 GB", | 20260506 23:58:49.363 | INFO | "huge" : "", | 20260506 23:58:49.363 | INFO | "huge_shared" : " 27 GB", | 20260506 23:58:49.363 | INFO | "malloc" : " 14 GB", | 20260506 23:58:49.363 | INFO | "pinned" : " 29 GB" | 20260506 23:58:49.363 | INFO | } | 20260506 23:58:49.363 | INFO | Perf report for the last 5.14051 seconds: | 20260506 23:58:49.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:58:49.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:49.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:49.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:49.363 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260506 23:58:49.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:49.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:49.363 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:58:49.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:49.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:49.364 | INFO | Uploaded 36GiB to GPU in 2.01104s (17.9012GiB/s) | 20260506 23:58:49.364 | INFO | Downloaded 34GiB to GPU in 1.9202s (17.7065GiB/s) | 20260506 23:58:54.509 | INFO | Analyzing chunk frames = [5483008,5483520) | 20260506 23:58:54.509 | INFO | MemInfo: { | 20260506 23:58:54.509 | INFO | "gpu" : " 17 GB", | 20260506 23:58:54.509 | INFO | "huge" : "", | 20260506 23:58:54.509 | INFO | "huge_shared" : " 27 GB", | 20260506 23:58:54.509 | INFO | "malloc" : " 14 GB", | 20260506 23:58:54.509 | INFO | "pinned" : " 29 GB" | 20260506 23:58:54.509 | INFO | } | 20260506 23:58:54.509 | INFO | Perf report for the last 5.14574 seconds: | 20260506 23:58:54.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:58:54.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:54.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:54.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:54.509 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260506 23:58:54.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:54.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:54.509 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260506 23:58:54.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:54.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:54.509 | INFO | Uploaded 36GiB to GPU in 2.02163s (17.8074GiB/s) | 20260506 23:58:54.509 | INFO | Downloaded 34GiB to GPU in 1.9543s (17.3975GiB/s) | 20260506 23:58:59.647 | INFO | Analyzing chunk frames = [5483520,5484032) | 20260506 23:58:59.647 | INFO | MemInfo: { | 20260506 23:58:59.647 | INFO | "gpu" : " 17 GB", | 20260506 23:58:59.647 | INFO | "huge" : "", | 20260506 23:58:59.647 | INFO | "huge_shared" : " 27 GB", | 20260506 23:58:59.647 | INFO | "malloc" : " 14 GB", | 20260506 23:58:59.647 | INFO | "pinned" : " 29 GB" | 20260506 23:58:59.647 | INFO | } | 20260506 23:58:59.647 | INFO | Perf report for the last 5.13849 seconds: | 20260506 23:58:59.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:58:59.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:59.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:59.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:59.648 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260506 23:58:59.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:59.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:59.648 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260506 23:58:59.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:59.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:58:59.648 | INFO | Uploaded 36GiB to GPU in 1.95331s (18.4302GiB/s) | 20260506 23:58:59.648 | INFO | Downloaded 34GiB to GPU in 1.98761s (17.106GiB/s) | 20260506 23:59:04.792 | INFO | Analyzing chunk frames = [5484032,5484544) | 20260506 23:59:04.792 | INFO | MemInfo: { | 20260506 23:59:04.792 | INFO | "gpu" : " 17 GB", | 20260506 23:59:04.792 | INFO | "huge" : "", | 20260506 23:59:04.792 | INFO | "huge_shared" : " 27 GB", | 20260506 23:59:04.792 | INFO | "malloc" : " 14 GB", | 20260506 23:59:04.792 | INFO | "pinned" : " 29 GB" | 20260506 23:59:04.792 | INFO | } | 20260506 23:59:04.792 | INFO | Perf report for the last 5.14457 seconds: | 20260506 23:59:04.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:59:04.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:04.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:04.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:04.792 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260506 23:59:04.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:04.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:04.792 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260506 23:59:04.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:04.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:04.793 | INFO | Uploaded 36GiB to GPU in 1.97134s (18.2617GiB/s) | 20260506 23:59:04.793 | INFO | Downloaded 34GiB to GPU in 1.98238s (17.1511GiB/s) | 20260506 23:59:04.889 | INFO | Starting a round of Baz IO | 20260506 23:59:08.008 | INFO | Populating RTMetrics report | 20260506 23:59:08.008 | INFO | RTMetrics report populated | 20260506 23:59:08.008 | INFO | Opening tmp RTMetrics json file | 20260506 23:59:08.009 | INFO | Serializing RTMetrics to json | 20260506 23:59:08.010 | INFO | Serialization done, swapping tmp file now... | 20260506 23:59:08.015 | INFO | RTMetrics done | 20260506 23:59:08.102 | INFO | Finished a round of Baz IO: Wrote 1.866 GiB in 3.213 seconds | 20260506 23:59:08.102 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.37% / 0.00% / 66.63% | 20260506 23:59:08.102 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.377862% / 0% / 8.34465e-05% | 20260506 23:59:08.102 | INFO | Baz IO overall: | 20260506 23:59:08.102 | INFO | BytesWritten : 2864628876083 | 20260506 23:59:08.102 | INFO | NumEvents : 1380528343646 | 20260506 23:59:09.930 | INFO | Analyzing chunk frames = [5484544,5485056) | 20260506 23:59:09.930 | INFO | MemInfo: { | 20260506 23:59:09.930 | INFO | "gpu" : " 17 GB", | 20260506 23:59:09.930 | INFO | "huge" : "", | 20260506 23:59:09.930 | INFO | "huge_shared" : " 27 GB", | 20260506 23:59:09.930 | INFO | "malloc" : " 14 GB", | 20260506 23:59:09.930 | INFO | "pinned" : " 29 GB" | 20260506 23:59:09.930 | INFO | } | 20260506 23:59:09.930 | INFO | Perf report for the last 5.13839 seconds: | 20260506 23:59:09.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:59:09.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:09.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:09.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:09.930 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260506 23:59:09.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:09.931 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260506 23:59:09.931 | INFO | PRE_HQ : 59.5%, 1.8, 56.5ms, 0 | 20260506 23:59:09.931 | INFO | BAZWRITER : 62.1%, 3.5, 110.5ms, 0 | 20260506 23:59:09.931 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:59:09.931 | INFO | Uploaded 36GiB to GPU in 2.01922s (17.8287GiB/s) | 20260506 23:59:09.931 | INFO | Downloaded 38GiB to GPU in 2.18038s (17.4281GiB/s) | 20260506 23:59:15.064 | INFO | Analyzing chunk frames = [5485056,5485568) | 20260506 23:59:15.064 | INFO | MemInfo: { | 20260506 23:59:15.064 | INFO | "gpu" : " 17 GB", | 20260506 23:59:15.064 | INFO | "huge" : "", | 20260506 23:59:15.064 | INFO | "huge_shared" : " 27 GB", | 20260506 23:59:15.064 | INFO | "malloc" : " 14 GB", | 20260506 23:59:15.064 | INFO | "pinned" : " 29 GB" | 20260506 23:59:15.064 | INFO | } | 20260506 23:59:15.064 | INFO | Perf report for the last 5.13373 seconds: | 20260506 23:59:15.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:59:15.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:15.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:15.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:15.064 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260506 23:59:15.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:15.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:15.064 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260506 23:59:15.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:15.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:15.065 | INFO | Uploaded 36GiB to GPU in 2.01798s (17.8396GiB/s) | 20260506 23:59:15.065 | INFO | Downloaded 34GiB to GPU in 1.93536s (17.5678GiB/s) | 20260506 23:59:20.205 | INFO | Analyzing chunk frames = [5485568,5486080) | 20260506 23:59:20.205 | INFO | MemInfo: { | 20260506 23:59:20.205 | INFO | "gpu" : " 17 GB", | 20260506 23:59:20.205 | INFO | "huge" : "", | 20260506 23:59:20.205 | INFO | "huge_shared" : " 27 GB", | 20260506 23:59:20.205 | INFO | "malloc" : " 14 GB", | 20260506 23:59:20.205 | INFO | "pinned" : " 29 GB" | 20260506 23:59:20.205 | INFO | } | 20260506 23:59:20.205 | INFO | Perf report for the last 5.14048 seconds: | 20260506 23:59:20.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:59:20.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:20.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:20.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:20.205 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260506 23:59:20.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:20.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:20.205 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260506 23:59:20.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:20.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:20.205 | INFO | Uploaded 36GiB to GPU in 1.95333s (18.4301GiB/s) | 20260506 23:59:20.205 | INFO | Downloaded 34GiB to GPU in 1.92382s (17.6732GiB/s) | 20260506 23:59:25.340 | INFO | Analyzing chunk frames = [5486080,5486592) | 20260506 23:59:25.340 | INFO | MemInfo: { | 20260506 23:59:25.340 | INFO | "gpu" : " 17 GB", | 20260506 23:59:25.340 | INFO | "huge" : "", | 20260506 23:59:25.340 | INFO | "huge_shared" : " 27 GB", | 20260506 23:59:25.340 | INFO | "malloc" : " 14 GB", | 20260506 23:59:25.340 | INFO | "pinned" : " 29 GB" | 20260506 23:59:25.340 | INFO | } | 20260506 23:59:25.340 | INFO | Perf report for the last 5.13522 seconds: | 20260506 23:59:25.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:59:25.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:25.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:25.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:25.340 | INFO | ANALYSIS : 59.4%, 3.0, 92.0ms, 0 | 20260506 23:59:25.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:25.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:25.340 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260506 23:59:25.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:25.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:25.340 | INFO | Uploaded 36GiB to GPU in 1.92454s (18.7058GiB/s) | 20260506 23:59:25.340 | INFO | Downloaded 34GiB to GPU in 1.88192s (18.0667GiB/s) | 20260506 23:59:30.486 | INFO | Analyzing chunk frames = [5486592,5487104) | 20260506 23:59:30.486 | INFO | MemInfo: { | 20260506 23:59:30.486 | INFO | "gpu" : " 17 GB", | 20260506 23:59:30.486 | INFO | "huge" : "", | 20260506 23:59:30.486 | INFO | "huge_shared" : " 27 GB", | 20260506 23:59:30.486 | INFO | "malloc" : " 14 GB", | 20260506 23:59:30.486 | INFO | "pinned" : " 29 GB" | 20260506 23:59:30.486 | INFO | } | 20260506 23:59:30.486 | INFO | Perf report for the last 5.14576 seconds: | 20260506 23:59:30.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:59:30.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:30.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:30.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:30.486 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260506 23:59:30.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:30.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:30.486 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:59:30.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:30.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:30.486 | INFO | Uploaded 36GiB to GPU in 1.98264s (18.1576GiB/s) | 20260506 23:59:30.486 | INFO | Downloaded 34GiB to GPU in 1.92778s (17.6369GiB/s) | 20260506 23:59:35.628 | INFO | Analyzing chunk frames = [5487104,5487616) | 20260506 23:59:35.628 | INFO | MemInfo: { | 20260506 23:59:35.628 | INFO | "gpu" : " 17 GB", | 20260506 23:59:35.628 | INFO | "huge" : "", | 20260506 23:59:35.628 | INFO | "huge_shared" : " 27 GB", | 20260506 23:59:35.628 | INFO | "malloc" : " 14 GB", | 20260506 23:59:35.628 | INFO | "pinned" : " 29 GB" | 20260506 23:59:35.628 | INFO | } | 20260506 23:59:35.628 | INFO | Perf report for the last 5.14246 seconds: | 20260506 23:59:35.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:59:35.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:35.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:35.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:35.628 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260506 23:59:35.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:35.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:35.628 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260506 23:59:35.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:35.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:35.629 | INFO | Uploaded 36GiB to GPU in 2.00942s (17.9156GiB/s) | 20260506 23:59:35.629 | INFO | Downloaded 34GiB to GPU in 1.95325s (17.4069GiB/s) | 20260506 23:59:40.766 | INFO | Analyzing chunk frames = [5487616,5488128) | 20260506 23:59:40.766 | INFO | MemInfo: { | 20260506 23:59:40.766 | INFO | "gpu" : " 17 GB", | 20260506 23:59:40.766 | INFO | "huge" : "", | 20260506 23:59:40.766 | INFO | "huge_shared" : " 27 GB", | 20260506 23:59:40.766 | INFO | "malloc" : " 14 GB", | 20260506 23:59:40.766 | INFO | "pinned" : " 29 GB" | 20260506 23:59:40.766 | INFO | } | 20260506 23:59:40.767 | INFO | Perf report for the last 5.13835 seconds: | 20260506 23:59:40.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:59:40.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:40.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:40.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:40.767 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260506 23:59:40.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:40.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:40.767 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260506 23:59:40.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:40.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:40.767 | INFO | Uploaded 36GiB to GPU in 1.97009s (18.2732GiB/s) | 20260506 23:59:40.767 | INFO | Downloaded 34GiB to GPU in 1.99982s (17.0015GiB/s) | 20260506 23:59:45.901 | INFO | Analyzing chunk frames = [5488128,5488640) | 20260506 23:59:45.901 | INFO | MemInfo: { | 20260506 23:59:45.901 | INFO | "gpu" : " 17 GB", | 20260506 23:59:45.901 | INFO | "huge" : "", | 20260506 23:59:45.901 | INFO | "huge_shared" : " 27 GB", | 20260506 23:59:45.901 | INFO | "malloc" : " 14 GB", | 20260506 23:59:45.901 | INFO | "pinned" : " 29 GB" | 20260506 23:59:45.901 | INFO | } | 20260506 23:59:45.901 | INFO | Perf report for the last 5.13421 seconds: | 20260506 23:59:45.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:59:45.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:45.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:45.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:45.901 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260506 23:59:45.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:45.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:45.901 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260506 23:59:45.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:45.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:45.901 | INFO | Uploaded 36GiB to GPU in 2.0045s (17.9596GiB/s) | 20260506 23:59:45.901 | INFO | Downloaded 34GiB to GPU in 1.97958s (17.1754GiB/s) | 20260506 23:59:45.901 | INFO | ----Memory usage report for Standard Host Memory----- | 20260506 23:59:45.901 | INFO | Total mem usage: 18 GiB | 20260506 23:59:45.901 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260506 23:59:45.901 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -24.7%) | 20260506 23:59:45.901 | INFO | Unused Cache: 0B (Waste: 0%) | 20260506 23:59:45.901 | INFO | High Water Breakdown: | 20260506 23:59:45.901 | INFO | 60.3% : 8.7 GiB ( 76.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260506 23:59:45.901 | INFO | 39.6% : 5.72 GiB ( 85.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260506 23:59:45.901 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260506 23:59:45.901 | INFO | ------------------------------------------ | 20260506 23:59:45.901 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:59:45.901 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260506 23:59:45.901 | INFO | Total mem usage: 26.5 GiB | 20260506 23:59:45.901 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260506 23:59:45.901 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260506 23:59:45.901 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260506 23:59:45.901 | INFO | High Water Breakdown: | 20260506 23:59:45.901 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:59:45.901 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:59:45.901 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:59:45.901 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:59:45.901 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:59:45.901 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:59:45.901 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260506 23:59:45.901 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260506 23:59:45.901 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:59:45.901 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260506 23:59:45.901 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260506 23:59:45.901 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260506 23:59:45.901 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260506 23:59:45.901 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260506 23:59:45.901 | INFO | ------------------------------------------ | 20260506 23:59:45.901 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:59:45.901 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260506 23:59:45.902 | INFO | Total mem usage: 25.6 GiB | 20260506 23:59:45.902 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260506 23:59:45.902 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260506 23:59:45.902 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260506 23:59:45.902 | INFO | High Water Breakdown: | 20260506 23:59:45.902 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260506 23:59:45.902 | INFO | ------------------------------------------ | 20260506 23:59:45.902 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:59:45.902 | INFO | ----Memory usage report for Gpu Memory----- | 20260506 23:59:45.902 | INFO | Total mem usage: 15.7 GiB | 20260506 23:59:45.902 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260506 23:59:45.902 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260506 23:59:45.902 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260506 23:59:45.902 | INFO | High Water Breakdown: | 20260506 23:59:45.902 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260506 23:59:45.902 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260506 23:59:45.902 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260506 23:59:45.902 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260506 23:59:45.902 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260506 23:59:45.902 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260506 23:59:45.902 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260506 23:59:45.902 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260506 23:59:45.902 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260506 23:59:45.902 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260506 23:59:45.902 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260506 23:59:45.902 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260506 23:59:45.902 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260506 23:59:45.902 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260506 23:59:45.902 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260506 23:59:45.902 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260506 23:59:45.902 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260506 23:59:45.902 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260506 23:59:45.902 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260506 23:59:45.902 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260506 23:59:45.902 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260506 23:59:45.902 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260506 23:59:45.902 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260506 23:59:45.902 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260506 23:59:45.902 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260506 23:59:45.902 | INFO | ------------------------------------------ | 20260506 23:59:45.902 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260506 23:59:45.902 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260506 23:59:45.902 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:59:45.902 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:59:45.902 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.58e+01, 1.33e+01 / 2.03e+00 | 20260506 23:59:45.902 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.01e-01, 9.68e-03 / 4.95e-03 | 20260506 23:59:45.902 | INFO | PerfReport: 11.68%, Baseline -- 3.61e+00 / 4.44e+00, 5.42e-01 / 2.47e-01 | 20260506 23:59:45.902 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.45e+00, 1.74e-01 / 1.05e-01 | 20260506 23:59:45.902 | INFO | PerfReport: 16.09%, DME -- 3.98e+01 / 4.49e+01, 4.00e+00 / 2.56e+00 | 20260506 23:59:45.902 | INFO | PerfReport: 57.37%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.30e+00 / 1.94e+00 | 20260506 23:59:45.902 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.18e+00 / 2.72e+00, 3.81e-01 / 2.50e-01 | 20260506 23:59:45.902 | INFO | PerfReport: 3.17%, Metrics -- 9.81e-01 / 1.32e+00, 2.58e-01 / 1.47e-01 | 20260506 23:59:45.902 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.35e-01 / 3.91e-01, 5.25e-02 / 2.17e-02 | 20260506 23:59:45.902 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260506 23:59:45.902 | INFO | PerfReport: Gathered 1536 samples. | 20260506 23:59:45.902 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260506 23:59:45.902 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.30e+01 / 0.00e+00 | 20260506 23:59:45.902 | INFO | PerfReport: 21.50%, Upload -- 2.07e+01 / 0.00e+00, 6.86e+00 / 0.00e+00 | 20260506 23:59:45.902 | INFO | PerfReport: 21.30%, Download -- 2.05e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260506 23:59:45.902 | INFO | PerfReport: 32.09%, Compute -- 3.09e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260506 23:59:45.902 | INFO | PerfReport: 1.25%, LockUpload -- 1.21e+00 / 0.00e+00, 4.72e+00 / 0.00e+00 | 20260506 23:59:45.902 | INFO | PerfReport: 1.32%, LockDownload -- 1.28e+00 / 0.00e+00, 3.20e+00 / 0.00e+00 | 20260506 23:59:45.902 | INFO | PerfReport: 22.52%, LockCompute -- 2.17e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260506 23:59:45.902 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.44e-02 / 0.00e+00 | 20260506 23:59:46.001 | INFO | Starting a round of Baz IO | 20260506 23:59:49.124 | INFO | Populating RTMetrics report | 20260506 23:59:49.125 | INFO | RTMetrics report populated | 20260506 23:59:49.125 | INFO | Opening tmp RTMetrics json file | 20260506 23:59:49.141 | INFO | Serializing RTMetrics to json | 20260506 23:59:49.145 | INFO | Serialization done, swapping tmp file now... | 20260506 23:59:49.145 | INFO | RTMetrics done | 20260506 23:59:49.221 | INFO | Finished a round of Baz IO: Wrote 1.867 GiB in 3.219 seconds | 20260506 23:59:49.221 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.37% / 0.00% / 66.63% | 20260506 23:59:49.221 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.376983% / 0% / 8.74201e-05% | 20260506 23:59:49.221 | INFO | Baz IO overall: | 20260506 23:59:49.221 | INFO | BytesWritten : 2866633571767 | 20260506 23:59:49.221 | INFO | NumEvents : 1381482249217 | 20260506 23:59:51.041 | INFO | Analyzing chunk frames = [5488640,5489152) | 20260506 23:59:51.041 | INFO | MemInfo: { | 20260506 23:59:51.041 | INFO | "gpu" : " 17 GB", | 20260506 23:59:51.041 | INFO | "huge" : "", | 20260506 23:59:51.041 | INFO | "huge_shared" : " 27 GB", | 20260506 23:59:51.041 | INFO | "malloc" : " 14 GB", | 20260506 23:59:51.041 | INFO | "pinned" : " 29 GB" | 20260506 23:59:51.041 | INFO | } | 20260506 23:59:51.041 | INFO | Perf report for the last 5.14044 seconds: | 20260506 23:59:51.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:59:51.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:51.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:51.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:51.041 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260506 23:59:51.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:51.041 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260506 23:59:51.041 | INFO | PRE_HQ : 59.7%, 1.8, 56.8ms, 0 | 20260506 23:59:51.041 | INFO | BAZWRITER : 62.3%, 3.4, 108.9ms, 0 | 20260506 23:59:51.041 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260506 23:59:51.042 | INFO | Uploaded 36GiB to GPU in 2.03064s (17.7284GiB/s) | 20260506 23:59:51.042 | INFO | Downloaded 38GiB to GPU in 2.17581s (17.4647GiB/s) | 20260506 23:59:56.179 | INFO | Analyzing chunk frames = [5489152,5489664) | 20260506 23:59:56.179 | INFO | MemInfo: { | 20260506 23:59:56.179 | INFO | "gpu" : " 17 GB", | 20260506 23:59:56.179 | INFO | "huge" : "", | 20260506 23:59:56.179 | INFO | "huge_shared" : " 27 GB", | 20260506 23:59:56.179 | INFO | "malloc" : " 14 GB", | 20260506 23:59:56.179 | INFO | "pinned" : " 29 GB" | 20260506 23:59:56.179 | INFO | } | 20260506 23:59:56.180 | INFO | Perf report for the last 5.13851 seconds: | 20260506 23:59:56.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260506 23:59:56.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:56.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:56.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:56.180 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260506 23:59:56.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:56.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:56.180 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260506 23:59:56.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:56.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260506 23:59:56.180 | INFO | Uploaded 36GiB to GPU in 2.01151s (17.897GiB/s) | 20260506 23:59:56.180 | INFO | Downloaded 34GiB to GPU in 1.92962s (17.62GiB/s) | 20260507 00:00:01.316 | INFO | Analyzing chunk frames = [5489664,5490176) | 20260507 00:00:01.316 | INFO | MemInfo: { | 20260507 00:00:01.316 | INFO | "gpu" : " 17 GB", | 20260507 00:00:01.316 | INFO | "huge" : "", | 20260507 00:00:01.316 | INFO | "huge_shared" : " 27 GB", | 20260507 00:00:01.316 | INFO | "malloc" : " 14 GB", | 20260507 00:00:01.316 | INFO | "pinned" : " 29 GB" | 20260507 00:00:01.316 | INFO | } | 20260507 00:00:01.316 | INFO | Perf report for the last 5.13623 seconds: | 20260507 00:00:01.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:00:01.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:01.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:01.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:01.316 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:00:01.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:01.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:01.316 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:00:01.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:01.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:01.316 | INFO | Uploaded 36GiB to GPU in 1.97738s (18.2059GiB/s) | 20260507 00:00:01.316 | INFO | Downloaded 34GiB to GPU in 1.92035s (17.7051GiB/s) | 20260507 00:00:06.459 | INFO | Analyzing chunk frames = [5490176,5490688) | 20260507 00:00:06.459 | INFO | MemInfo: { | 20260507 00:00:06.459 | INFO | "gpu" : " 17 GB", | 20260507 00:00:06.459 | INFO | "huge" : "", | 20260507 00:00:06.459 | INFO | "huge_shared" : " 27 GB", | 20260507 00:00:06.459 | INFO | "malloc" : " 14 GB", | 20260507 00:00:06.459 | INFO | "pinned" : " 29 GB" | 20260507 00:00:06.459 | INFO | } | 20260507 00:00:06.459 | INFO | Perf report for the last 5.14336 seconds: | 20260507 00:00:06.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:00:06.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:06.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:06.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:06.460 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 00:00:06.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:06.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:06.460 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:00:06.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:06.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:06.460 | INFO | Uploaded 36GiB to GPU in 1.96776s (18.2949GiB/s) | 20260507 00:00:06.460 | INFO | Downloaded 34GiB to GPU in 1.87954s (18.0895GiB/s) | 20260507 00:00:11.600 | INFO | Analyzing chunk frames = [5490688,5491200) | 20260507 00:00:11.600 | INFO | MemInfo: { | 20260507 00:00:11.600 | INFO | "gpu" : " 17 GB", | 20260507 00:00:11.600 | INFO | "huge" : "", | 20260507 00:00:11.600 | INFO | "huge_shared" : " 27 GB", | 20260507 00:00:11.600 | INFO | "malloc" : " 14 GB", | 20260507 00:00:11.600 | INFO | "pinned" : " 29 GB" | 20260507 00:00:11.600 | INFO | } | 20260507 00:00:11.600 | INFO | Perf report for the last 5.14109 seconds: | 20260507 00:00:11.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:00:11.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:11.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:11.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:11.600 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 00:00:11.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:11.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:11.601 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:00:11.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:11.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:11.601 | INFO | Uploaded 36GiB to GPU in 2.0006s (17.9946GiB/s) | 20260507 00:00:11.601 | INFO | Downloaded 34GiB to GPU in 1.93103s (17.6072GiB/s) | 20260507 00:00:16.741 | INFO | Analyzing chunk frames = [5491200,5491712) | 20260507 00:00:16.741 | INFO | MemInfo: { | 20260507 00:00:16.741 | INFO | "gpu" : " 17 GB", | 20260507 00:00:16.741 | INFO | "huge" : "", | 20260507 00:00:16.741 | INFO | "huge_shared" : " 27 GB", | 20260507 00:00:16.741 | INFO | "malloc" : " 14 GB", | 20260507 00:00:16.741 | INFO | "pinned" : " 29 GB" | 20260507 00:00:16.741 | INFO | } | 20260507 00:00:16.741 | INFO | Perf report for the last 5.14048 seconds: | 20260507 00:00:16.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:00:16.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:16.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:16.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:16.741 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 00:00:16.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:16.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:16.741 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 00:00:16.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:16.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:16.741 | INFO | Uploaded 36GiB to GPU in 2.04491s (17.6047GiB/s) | 20260507 00:00:16.741 | INFO | Downloaded 34GiB to GPU in 1.94483s (17.4822GiB/s) | 20260507 00:00:21.884 | INFO | Analyzing chunk frames = [5491712,5492224) | 20260507 00:00:21.884 | INFO | MemInfo: { | 20260507 00:00:21.884 | INFO | "gpu" : " 17 GB", | 20260507 00:00:21.884 | INFO | "huge" : "", | 20260507 00:00:21.884 | INFO | "huge_shared" : " 27 GB", | 20260507 00:00:21.884 | INFO | "malloc" : " 14 GB", | 20260507 00:00:21.884 | INFO | "pinned" : " 29 GB" | 20260507 00:00:21.884 | INFO | } | 20260507 00:00:21.884 | INFO | Perf report for the last 5.14353 seconds: | 20260507 00:00:21.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:00:21.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:21.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:21.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:21.884 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 00:00:21.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:21.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:21.885 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:00:21.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:21.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:21.885 | INFO | Uploaded 36GiB to GPU in 2.00412s (17.963GiB/s) | 20260507 00:00:21.885 | INFO | Downloaded 34GiB to GPU in 1.97464s (17.2183GiB/s) | 20260507 00:00:27.018 | INFO | Analyzing chunk frames = [5492224,5492736) | 20260507 00:00:27.018 | INFO | MemInfo: { | 20260507 00:00:27.018 | INFO | "gpu" : " 17 GB", | 20260507 00:00:27.018 | INFO | "huge" : "", | 20260507 00:00:27.018 | INFO | "huge_shared" : " 27 GB", | 20260507 00:00:27.018 | INFO | "malloc" : " 14 GB", | 20260507 00:00:27.018 | INFO | "pinned" : " 29 GB" | 20260507 00:00:27.018 | INFO | } | 20260507 00:00:27.018 | INFO | Perf report for the last 5.13361 seconds: | 20260507 00:00:27.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:00:27.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:27.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:27.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:27.018 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:00:27.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:27.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:27.018 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:00:27.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:27.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:27.018 | INFO | Uploaded 36GiB to GPU in 1.99722s (18.0251GiB/s) | 20260507 00:00:27.018 | INFO | Downloaded 34GiB to GPU in 1.97288s (17.2337GiB/s) | 20260507 00:00:27.117 | INFO | Starting a round of Baz IO | 20260507 00:00:30.242 | INFO | Populating RTMetrics report | 20260507 00:00:30.242 | INFO | RTMetrics report populated | 20260507 00:00:30.242 | INFO | Opening tmp RTMetrics json file | 20260507 00:00:30.243 | INFO | Serializing RTMetrics to json | 20260507 00:00:30.244 | INFO | Serialization done, swapping tmp file now... | 20260507 00:00:30.244 | INFO | RTMetrics done | 20260507 00:00:30.336 | INFO | Finished a round of Baz IO: Wrote 1.863 GiB in 3.218 seconds | 20260507 00:00:30.336 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.36% / 0.00% / 66.63% | 20260507 00:00:30.336 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.373713% / 0% / 7.54992e-05% | 20260507 00:00:30.336 | INFO | Baz IO overall: | 20260507 00:00:30.336 | INFO | BytesWritten : 2868634038839 | 20260507 00:00:30.336 | INFO | NumEvents : 1382432848084 | 20260507 00:00:32.154 | INFO | Analyzing chunk frames = [5492736,5493248) | 20260507 00:00:32.154 | INFO | MemInfo: { | 20260507 00:00:32.154 | INFO | "gpu" : " 17 GB", | 20260507 00:00:32.154 | INFO | "huge" : "", | 20260507 00:00:32.154 | INFO | "huge_shared" : " 27 GB", | 20260507 00:00:32.154 | INFO | "malloc" : " 14 GB", | 20260507 00:00:32.154 | INFO | "pinned" : " 29 GB" | 20260507 00:00:32.154 | INFO | } | 20260507 00:00:32.154 | INFO | Perf report for the last 5.13578 seconds: | 20260507 00:00:32.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:00:32.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:32.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:32.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:32.154 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260507 00:00:32.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:32.154 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 00:00:32.154 | INFO | PRE_HQ : 59.9%, 1.8, 56.7ms, 0 | 20260507 00:00:32.154 | INFO | BAZWRITER : 62.3%, 3.4, 110.0ms, 0 | 20260507 00:00:32.154 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:00:32.154 | INFO | Uploaded 36GiB to GPU in 2.03259s (17.7114GiB/s) | 20260507 00:00:32.154 | INFO | Downloaded 38GiB to GPU in 2.18148s (17.4194GiB/s) | 20260507 00:00:37.298 | INFO | Analyzing chunk frames = [5493248,5493760) | 20260507 00:00:37.298 | INFO | MemInfo: { | 20260507 00:00:37.298 | INFO | "gpu" : " 17 GB", | 20260507 00:00:37.298 | INFO | "huge" : "", | 20260507 00:00:37.298 | INFO | "huge_shared" : " 27 GB", | 20260507 00:00:37.298 | INFO | "malloc" : " 14 GB", | 20260507 00:00:37.298 | INFO | "pinned" : " 29 GB" | 20260507 00:00:37.298 | INFO | } | 20260507 00:00:37.298 | INFO | Perf report for the last 5.14416 seconds: | 20260507 00:00:37.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:00:37.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:37.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:37.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:37.298 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:00:37.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:37.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:37.298 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:00:37.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:37.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:37.298 | INFO | Uploaded 36GiB to GPU in 1.99026s (18.0881GiB/s) | 20260507 00:00:37.298 | INFO | Downloaded 34GiB to GPU in 1.93267s (17.5922GiB/s) | 20260507 00:00:42.438 | INFO | Analyzing chunk frames = [5493760,5494272) | 20260507 00:00:42.438 | INFO | MemInfo: { | 20260507 00:00:42.438 | INFO | "gpu" : " 17 GB", | 20260507 00:00:42.438 | INFO | "huge" : "", | 20260507 00:00:42.438 | INFO | "huge_shared" : " 27 GB", | 20260507 00:00:42.438 | INFO | "malloc" : " 14 GB", | 20260507 00:00:42.438 | INFO | "pinned" : " 29 GB" | 20260507 00:00:42.438 | INFO | } | 20260507 00:00:42.438 | INFO | Perf report for the last 5.13968 seconds: | 20260507 00:00:42.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:00:42.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:42.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:42.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:42.438 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:00:42.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:42.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:42.438 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:00:42.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:42.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:42.438 | INFO | Uploaded 36GiB to GPU in 1.97073s (18.2673GiB/s) | 20260507 00:00:42.438 | INFO | Downloaded 34GiB to GPU in 1.93153s (17.6026GiB/s) | 20260507 00:00:47.576 | INFO | Analyzing chunk frames = [5494272,5494784) | 20260507 00:00:47.576 | INFO | MemInfo: { | 20260507 00:00:47.576 | INFO | "gpu" : " 17 GB", | 20260507 00:00:47.576 | INFO | "huge" : "", | 20260507 00:00:47.576 | INFO | "huge_shared" : " 27 GB", | 20260507 00:00:47.576 | INFO | "malloc" : " 14 GB", | 20260507 00:00:47.576 | INFO | "pinned" : " 29 GB" | 20260507 00:00:47.576 | INFO | } | 20260507 00:00:47.576 | INFO | Perf report for the last 5.13848 seconds: | 20260507 00:00:47.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:00:47.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:47.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:47.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:47.576 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:00:47.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:47.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:47.576 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:00:47.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:47.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:47.577 | INFO | Uploaded 36GiB to GPU in 1.96172s (18.3513GiB/s) | 20260507 00:00:47.577 | INFO | Downloaded 34GiB to GPU in 1.90894s (17.811GiB/s) | 20260507 00:00:52.718 | INFO | Analyzing chunk frames = [5494784,5495296) | 20260507 00:00:52.718 | INFO | MemInfo: { | 20260507 00:00:52.718 | INFO | "gpu" : " 17 GB", | 20260507 00:00:52.718 | INFO | "huge" : "", | 20260507 00:00:52.718 | INFO | "huge_shared" : " 27 GB", | 20260507 00:00:52.718 | INFO | "malloc" : " 14 GB", | 20260507 00:00:52.718 | INFO | "pinned" : " 29 GB" | 20260507 00:00:52.718 | INFO | } | 20260507 00:00:52.718 | INFO | Perf report for the last 5.14222 seconds: | 20260507 00:00:52.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:00:52.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:52.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:52.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:52.718 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 00:00:52.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:52.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:52.719 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:00:52.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:52.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:52.719 | INFO | Uploaded 36GiB to GPU in 1.98132s (18.1697GiB/s) | 20260507 00:00:52.719 | INFO | Downloaded 34GiB to GPU in 1.93886s (17.5361GiB/s) | 20260507 00:00:57.857 | INFO | Analyzing chunk frames = [5495296,5495808) | 20260507 00:00:57.857 | INFO | MemInfo: { | 20260507 00:00:57.857 | INFO | "gpu" : " 17 GB", | 20260507 00:00:57.857 | INFO | "huge" : "", | 20260507 00:00:57.857 | INFO | "huge_shared" : " 27 GB", | 20260507 00:00:57.857 | INFO | "malloc" : " 14 GB", | 20260507 00:00:57.857 | INFO | "pinned" : " 29 GB" | 20260507 00:00:57.857 | INFO | } | 20260507 00:00:57.857 | INFO | Perf report for the last 5.13886 seconds: | 20260507 00:00:57.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:00:57.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:57.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:57.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:57.857 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:00:57.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:57.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:57.857 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 00:00:57.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:57.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:00:57.858 | INFO | Uploaded 36GiB to GPU in 2.03485s (17.6917GiB/s) | 20260507 00:00:57.858 | INFO | Downloaded 34GiB to GPU in 1.95462s (17.3947GiB/s) | 20260507 00:01:02.994 | INFO | Analyzing chunk frames = [5495808,5496320) | 20260507 00:01:02.994 | INFO | MemInfo: { | 20260507 00:01:02.994 | INFO | "gpu" : " 17 GB", | 20260507 00:01:02.994 | INFO | "huge" : "", | 20260507 00:01:02.995 | INFO | "huge_shared" : " 27 GB", | 20260507 00:01:02.995 | INFO | "malloc" : " 14 GB", | 20260507 00:01:02.995 | INFO | "pinned" : " 29 GB" | 20260507 00:01:02.995 | INFO | } | 20260507 00:01:02.995 | INFO | Perf report for the last 5.13733 seconds: | 20260507 00:01:02.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:01:02.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:02.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:02.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:02.995 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:01:02.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:02.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:02.995 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:01:02.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:02.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:02.995 | INFO | Uploaded 36GiB to GPU in 1.95621s (18.4029GiB/s) | 20260507 00:01:02.995 | INFO | Downloaded 34GiB to GPU in 1.99787s (17.0181GiB/s) | 20260507 00:01:08.127 | INFO | Analyzing chunk frames = [5496320,5496832) | 20260507 00:01:08.127 | INFO | MemInfo: { | 20260507 00:01:08.127 | INFO | "gpu" : " 17 GB", | 20260507 00:01:08.127 | INFO | "huge" : "", | 20260507 00:01:08.127 | INFO | "huge_shared" : " 27 GB", | 20260507 00:01:08.127 | INFO | "malloc" : " 14 GB", | 20260507 00:01:08.128 | INFO | "pinned" : " 29 GB" | 20260507 00:01:08.128 | INFO | } | 20260507 00:01:08.128 | INFO | Perf report for the last 5.13288 seconds: | 20260507 00:01:08.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:01:08.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:08.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:08.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:08.128 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260507 00:01:08.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:08.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:08.128 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 00:01:08.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:08.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:08.128 | INFO | Uploaded 36GiB to GPU in 1.97271s (18.249GiB/s) | 20260507 00:01:08.128 | INFO | Downloaded 34GiB to GPU in 1.99783s (17.0185GiB/s) | 20260507 00:01:08.128 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:01:08.128 | INFO | Total mem usage: 18 GiB | 20260507 00:01:08.128 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:01:08.128 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:01:08.128 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:01:08.128 | INFO | High Water Breakdown: | 20260507 00:01:08.128 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:01:08.128 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:01:08.128 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:01:08.128 | INFO | ------------------------------------------ | 20260507 00:01:08.128 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:01:08.128 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:01:08.128 | INFO | Total mem usage: 26.5 GiB | 20260507 00:01:08.128 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:01:08.128 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:01:08.128 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:01:08.128 | INFO | High Water Breakdown: | 20260507 00:01:08.128 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:01:08.128 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:01:08.128 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:01:08.128 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:01:08.128 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:01:08.128 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:01:08.128 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:01:08.128 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:01:08.128 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:01:08.128 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:01:08.128 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:01:08.128 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:01:08.128 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:01:08.128 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:01:08.128 | INFO | ------------------------------------------ | 20260507 00:01:08.129 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:01:08.129 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:01:08.129 | INFO | Total mem usage: 25.6 GiB | 20260507 00:01:08.129 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:01:08.129 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:01:08.129 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:01:08.129 | INFO | High Water Breakdown: | 20260507 00:01:08.129 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:01:08.129 | INFO | ------------------------------------------ | 20260507 00:01:08.129 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:01:08.129 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:01:08.129 | INFO | Total mem usage: 15.7 GiB | 20260507 00:01:08.129 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:01:08.129 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:01:08.129 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:01:08.129 | INFO | High Water Breakdown: | 20260507 00:01:08.129 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:01:08.129 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:01:08.129 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:01:08.129 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:01:08.129 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:01:08.129 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:01:08.129 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:01:08.129 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:01:08.129 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:01:08.129 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:01:08.129 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:01:08.129 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:01:08.129 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:01:08.129 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:01:08.129 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:01:08.129 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:01:08.129 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:01:08.129 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:01:08.129 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:01:08.129 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:01:08.129 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:01:08.129 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:01:08.129 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:01:08.129 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:01:08.129 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:01:08.129 | INFO | ------------------------------------------ | 20260507 00:01:08.129 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:01:08.129 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:01:08.129 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:01:08.129 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:01:08.129 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.56e+01, 1.35e+01 / 1.90e+00 | 20260507 00:01:08.129 | INFO | PerfReport: 0.30%, AggregateStats -- 9.14e-02 / 1.00e-01, 9.94e-03 / 6.25e-03 | 20260507 00:01:08.129 | INFO | PerfReport: 11.67%, Baseline -- 3.61e+00 / 4.42e+00, 5.39e-01 / 3.48e-01 | 20260507 00:01:08.129 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.46e+00, 1.74e-01 / 1.40e-01 | 20260507 00:01:08.129 | INFO | PerfReport: 16.08%, DME -- 3.98e+01 / 4.45e+01, 4.09e+00 / 2.09e+00 | 20260507 00:01:08.129 | INFO | PerfReport: 57.38%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.34e+00 / 1.87e+00 | 20260507 00:01:08.129 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.19e+00 / 2.78e+00, 3.84e-01 / 1.93e-01 | 20260507 00:01:08.129 | INFO | PerfReport: 3.17%, Metrics -- 9.81e-01 / 1.33e+00, 2.58e-01 / 1.41e-01 | 20260507 00:01:08.129 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.35e-01 / 3.85e-01, 5.24e-02 / 2.44e-02 | 20260507 00:01:08.129 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:01:08.129 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:01:08.129 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:01:08.129 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 00:01:08.129 | INFO | PerfReport: 21.53%, Upload -- 2.08e+01 / 0.00e+00, 6.94e+00 / 0.00e+00 | 20260507 00:01:08.129 | INFO | PerfReport: 21.29%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260507 00:01:08.129 | INFO | PerfReport: 32.05%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:01:08.129 | INFO | PerfReport: 1.29%, LockUpload -- 1.25e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 00:01:08.129 | INFO | PerfReport: 1.30%, LockDownload -- 1.26e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260507 00:01:08.129 | INFO | PerfReport: 22.52%, LockCompute -- 2.17e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 00:01:08.129 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260507 00:01:08.227 | INFO | Starting a round of Baz IO | 20260507 00:01:11.350 | INFO | Populating RTMetrics report | 20260507 00:01:11.351 | INFO | RTMetrics report populated | 20260507 00:01:11.351 | INFO | Opening tmp RTMetrics json file | 20260507 00:01:11.353 | INFO | Serializing RTMetrics to json | 20260507 00:01:11.372 | INFO | Serialization done, swapping tmp file now... | 20260507 00:01:11.372 | INFO | RTMetrics done | 20260507 00:01:11.451 | INFO | Finished a round of Baz IO: Wrote 1.866 GiB in 3.224 seconds | 20260507 00:01:11.451 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.36% / 0.00% / 66.64% | 20260507 00:01:11.451 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.374504% / 3.97364e-06% / 5.16574e-05% | 20260507 00:01:11.451 | INFO | Baz IO overall: | 20260507 00:01:11.451 | INFO | BytesWritten : 2870637915757 | 20260507 00:01:11.451 | INFO | NumEvents : 1383385931533 | 20260507 00:01:13.268 | INFO | Analyzing chunk frames = [5496832,5497344) | 20260507 00:01:13.268 | INFO | MemInfo: { | 20260507 00:01:13.268 | INFO | "gpu" : " 17 GB", | 20260507 00:01:13.268 | INFO | "huge" : "", | 20260507 00:01:13.268 | INFO | "huge_shared" : " 27 GB", | 20260507 00:01:13.268 | INFO | "malloc" : " 14 GB", | 20260507 00:01:13.268 | INFO | "pinned" : " 29 GB" | 20260507 00:01:13.268 | INFO | } | 20260507 00:01:13.268 | INFO | Perf report for the last 5.14004 seconds: | 20260507 00:01:13.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:01:13.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:13.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:13.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:13.268 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260507 00:01:13.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:13.268 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 00:01:13.268 | INFO | PRE_HQ : 59.3%, 1.8, 56.3ms, 0 | 20260507 00:01:13.268 | INFO | BAZWRITER : 62.3%, 3.5, 111.0ms, 0 | 20260507 00:01:13.268 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:01:13.268 | INFO | Uploaded 36GiB to GPU in 1.99595s (18.0365GiB/s) | 20260507 00:01:13.268 | INFO | Downloaded 38GiB to GPU in 2.17342s (17.484GiB/s) | 20260507 00:01:18.407 | INFO | Analyzing chunk frames = [5497344,5497856) | 20260507 00:01:18.407 | INFO | MemInfo: { | 20260507 00:01:18.407 | INFO | "gpu" : " 17 GB", | 20260507 00:01:18.407 | INFO | "huge" : "", | 20260507 00:01:18.407 | INFO | "huge_shared" : " 27 GB", | 20260507 00:01:18.407 | INFO | "malloc" : " 14 GB", | 20260507 00:01:18.407 | INFO | "pinned" : " 29 GB" | 20260507 00:01:18.407 | INFO | } | 20260507 00:01:18.407 | INFO | Perf report for the last 5.13915 seconds: | 20260507 00:01:18.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:01:18.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:18.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:18.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:18.407 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:01:18.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:18.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:18.407 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:01:18.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:18.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:18.407 | INFO | Uploaded 36GiB to GPU in 1.98235s (18.1602GiB/s) | 20260507 00:01:18.407 | INFO | Downloaded 34GiB to GPU in 1.92896s (17.6261GiB/s) | 20260507 00:01:23.551 | INFO | Analyzing chunk frames = [5497856,5498368) | 20260507 00:01:23.551 | INFO | MemInfo: { | 20260507 00:01:23.551 | INFO | "gpu" : " 17 GB", | 20260507 00:01:23.551 | INFO | "huge" : "", | 20260507 00:01:23.551 | INFO | "huge_shared" : " 27 GB", | 20260507 00:01:23.551 | INFO | "malloc" : " 14 GB", | 20260507 00:01:23.551 | INFO | "pinned" : " 29 GB" | 20260507 00:01:23.551 | INFO | } | 20260507 00:01:23.551 | INFO | Perf report for the last 5.14466 seconds: | 20260507 00:01:23.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:01:23.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:23.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:23.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:23.552 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 00:01:23.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:23.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:23.552 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:01:23.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:23.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:23.552 | INFO | Uploaded 36GiB to GPU in 1.96988s (18.2752GiB/s) | 20260507 00:01:23.552 | INFO | Downloaded 34GiB to GPU in 1.92727s (17.6415GiB/s) | 20260507 00:01:28.696 | INFO | Analyzing chunk frames = [5498368,5498880) | 20260507 00:01:28.696 | INFO | MemInfo: { | 20260507 00:01:28.696 | INFO | "gpu" : " 17 GB", | 20260507 00:01:28.696 | INFO | "huge" : "", | 20260507 00:01:28.696 | INFO | "huge_shared" : " 27 GB", | 20260507 00:01:28.696 | INFO | "malloc" : " 14 GB", | 20260507 00:01:28.696 | INFO | "pinned" : " 29 GB" | 20260507 00:01:28.696 | INFO | } | 20260507 00:01:28.696 | INFO | Perf report for the last 5.14468 seconds: | 20260507 00:01:28.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:01:28.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:28.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:28.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:28.696 | INFO | ANALYSIS : 59.3%, 3.0, 92.0ms, 0 | 20260507 00:01:28.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:28.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:28.696 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:01:28.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:28.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:28.696 | INFO | Uploaded 36GiB to GPU in 1.94859s (18.4749GiB/s) | 20260507 00:01:28.696 | INFO | Downloaded 34GiB to GPU in 1.87728s (18.1113GiB/s) | 20260507 00:01:33.835 | INFO | Analyzing chunk frames = [5498880,5499392) | 20260507 00:01:33.835 | INFO | MemInfo: { | 20260507 00:01:33.835 | INFO | "gpu" : " 17 GB", | 20260507 00:01:33.835 | INFO | "huge" : "", | 20260507 00:01:33.835 | INFO | "huge_shared" : " 27 GB", | 20260507 00:01:33.835 | INFO | "malloc" : " 14 GB", | 20260507 00:01:33.835 | INFO | "pinned" : " 29 GB" | 20260507 00:01:33.835 | INFO | } | 20260507 00:01:33.836 | INFO | Perf report for the last 5.13937 seconds: | 20260507 00:01:33.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:01:33.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:33.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:33.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:33.836 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 00:01:33.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:33.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:33.836 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:01:33.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:33.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:33.836 | INFO | Uploaded 36GiB to GPU in 2.00917s (17.9178GiB/s) | 20260507 00:01:33.836 | INFO | Downloaded 34GiB to GPU in 1.93184s (17.5998GiB/s) | 20260507 00:01:38.971 | INFO | Analyzing chunk frames = [5499392,5499904) | 20260507 00:01:38.971 | INFO | MemInfo: { | 20260507 00:01:38.971 | INFO | "gpu" : " 17 GB", | 20260507 00:01:38.971 | INFO | "huge" : "", | 20260507 00:01:38.971 | INFO | "huge_shared" : " 27 GB", | 20260507 00:01:38.971 | INFO | "malloc" : " 14 GB", | 20260507 00:01:38.971 | INFO | "pinned" : " 29 GB" | 20260507 00:01:38.971 | INFO | } | 20260507 00:01:38.971 | INFO | Perf report for the last 5.13566 seconds: | 20260507 00:01:38.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:01:38.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:38.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:38.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:38.971 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 00:01:38.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:38.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:38.971 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 00:01:38.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:38.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:38.972 | INFO | Uploaded 36GiB to GPU in 2.04456s (17.6077GiB/s) | 20260507 00:01:38.972 | INFO | Downloaded 34GiB to GPU in 1.962s (17.3293GiB/s) | 20260507 00:01:44.109 | INFO | Analyzing chunk frames = [5499904,5500416) | 20260507 00:01:44.109 | INFO | MemInfo: { | 20260507 00:01:44.109 | INFO | "gpu" : " 17 GB", | 20260507 00:01:44.109 | INFO | "huge" : "", | 20260507 00:01:44.109 | INFO | "huge_shared" : " 27 GB", | 20260507 00:01:44.109 | INFO | "malloc" : " 14 GB", | 20260507 00:01:44.109 | INFO | "pinned" : " 29 GB" | 20260507 00:01:44.110 | INFO | } | 20260507 00:01:44.110 | INFO | Perf report for the last 5.13844 seconds: | 20260507 00:01:44.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:01:44.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:44.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:44.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:44.110 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260507 00:01:44.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:44.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:44.110 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:01:44.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:44.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:44.110 | INFO | Uploaded 36GiB to GPU in 1.97357s (18.2411GiB/s) | 20260507 00:01:44.110 | INFO | Downloaded 34GiB to GPU in 1.99695s (17.0259GiB/s) | 20260507 00:01:49.244 | INFO | Analyzing chunk frames = [5500416,5500928) | 20260507 00:01:49.244 | INFO | MemInfo: { | 20260507 00:01:49.244 | INFO | "gpu" : " 17 GB", | 20260507 00:01:49.244 | INFO | "huge" : "", | 20260507 00:01:49.244 | INFO | "huge_shared" : " 27 GB", | 20260507 00:01:49.244 | INFO | "malloc" : " 14 GB", | 20260507 00:01:49.244 | INFO | "pinned" : " 29 GB" | 20260507 00:01:49.244 | INFO | } | 20260507 00:01:49.244 | INFO | Perf report for the last 5.13429 seconds: | 20260507 00:01:49.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:01:49.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:49.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:49.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:49.244 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 00:01:49.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:49.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:49.244 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 00:01:49.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:49.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:49.244 | INFO | Uploaded 36GiB to GPU in 1.9921s (18.0714GiB/s) | 20260507 00:01:49.244 | INFO | Downloaded 34GiB to GPU in 1.98325s (17.1435GiB/s) | 20260507 00:01:49.342 | INFO | Starting a round of Baz IO | 20260507 00:01:52.470 | INFO | Populating RTMetrics report | 20260507 00:01:52.470 | INFO | RTMetrics report populated | 20260507 00:01:52.470 | INFO | Opening tmp RTMetrics json file | 20260507 00:01:52.471 | INFO | Serializing RTMetrics to json | 20260507 00:01:52.472 | INFO | Serialization done, swapping tmp file now... | 20260507 00:01:52.474 | INFO | RTMetrics done | 20260507 00:01:52.566 | INFO | Finished a round of Baz IO: Wrote 1.864 GiB in 3.223 seconds | 20260507 00:01:52.566 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.36% / 0.00% / 66.64% | 20260507 00:01:52.566 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.373745% / 3.97364e-06% / 6.35783e-05% | 20260507 00:01:52.566 | INFO | Baz IO overall: | 20260507 00:01:52.566 | INFO | BytesWritten : 2872639658363 | 20260507 00:01:52.566 | INFO | NumEvents : 1384337836766 | 20260507 00:01:54.386 | INFO | Analyzing chunk frames = [5500928,5501440) | 20260507 00:01:54.386 | INFO | MemInfo: { | 20260507 00:01:54.386 | INFO | "gpu" : " 17 GB", | 20260507 00:01:54.386 | INFO | "huge" : "", | 20260507 00:01:54.386 | INFO | "huge_shared" : " 27 GB", | 20260507 00:01:54.386 | INFO | "malloc" : " 14 GB", | 20260507 00:01:54.386 | INFO | "pinned" : " 29 GB" | 20260507 00:01:54.386 | INFO | } | 20260507 00:01:54.386 | INFO | Perf report for the last 5.14243 seconds: | 20260507 00:01:54.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:01:54.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:54.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:54.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:54.386 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 00:01:54.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:54.386 | INFO | RT_METRICS : 42.2%, 1.0, 22.2ms, 0 | 20260507 00:01:54.387 | INFO | PRE_HQ : 59.6%, 1.8, 56.8ms, 0 | 20260507 00:01:54.387 | INFO | BAZWRITER : 62.1%, 3.4, 109.5ms, 0 | 20260507 00:01:54.387 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:01:54.387 | INFO | Uploaded 36GiB to GPU in 2.02418s (17.7849GiB/s) | 20260507 00:01:54.387 | INFO | Downloaded 38GiB to GPU in 2.18154s (17.4189GiB/s) | 20260507 00:01:59.531 | INFO | Analyzing chunk frames = [5501440,5501952) | 20260507 00:01:59.531 | INFO | MemInfo: { | 20260507 00:01:59.531 | INFO | "gpu" : " 17 GB", | 20260507 00:01:59.531 | INFO | "huge" : "", | 20260507 00:01:59.531 | INFO | "huge_shared" : " 27 GB", | 20260507 00:01:59.531 | INFO | "malloc" : " 14 GB", | 20260507 00:01:59.531 | INFO | "pinned" : " 29 GB" | 20260507 00:01:59.531 | INFO | } | 20260507 00:01:59.531 | INFO | Perf report for the last 5.14471 seconds: | 20260507 00:01:59.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:01:59.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:59.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:59.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:59.531 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 00:01:59.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:59.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:59.531 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:01:59.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:59.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:01:59.531 | INFO | Uploaded 36GiB to GPU in 1.99012s (18.0894GiB/s) | 20260507 00:01:59.531 | INFO | Downloaded 34GiB to GPU in 1.92492s (17.663GiB/s) | 20260507 00:02:04.669 | INFO | Analyzing chunk frames = [5501952,5502464) | 20260507 00:02:04.669 | INFO | MemInfo: { | 20260507 00:02:04.669 | INFO | "gpu" : " 17 GB", | 20260507 00:02:04.669 | INFO | "huge" : "", | 20260507 00:02:04.669 | INFO | "huge_shared" : " 27 GB", | 20260507 00:02:04.669 | INFO | "malloc" : " 14 GB", | 20260507 00:02:04.669 | INFO | "pinned" : " 29 GB" | 20260507 00:02:04.669 | INFO | } | 20260507 00:02:04.669 | INFO | Perf report for the last 5.13802 seconds: | 20260507 00:02:04.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:02:04.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:04.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:04.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:04.669 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 00:02:04.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:04.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:04.669 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:02:04.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:04.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:04.669 | INFO | Uploaded 36GiB to GPU in 2.00154s (17.9861GiB/s) | 20260507 00:02:04.669 | INFO | Downloaded 34GiB to GPU in 1.93397s (17.5804GiB/s) | 20260507 00:02:09.806 | INFO | Analyzing chunk frames = [5502464,5502976) | 20260507 00:02:09.806 | INFO | MemInfo: { | 20260507 00:02:09.806 | INFO | "gpu" : " 17 GB", | 20260507 00:02:09.806 | INFO | "huge" : "", | 20260507 00:02:09.806 | INFO | "huge_shared" : " 27 GB", | 20260507 00:02:09.806 | INFO | "malloc" : " 14 GB", | 20260507 00:02:09.806 | INFO | "pinned" : " 29 GB" | 20260507 00:02:09.806 | INFO | } | 20260507 00:02:09.807 | INFO | Perf report for the last 5.13737 seconds: | 20260507 00:02:09.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:02:09.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:09.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:09.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:09.807 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260507 00:02:09.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:09.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:09.807 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:02:09.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:09.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:09.807 | INFO | Uploaded 36GiB to GPU in 1.98323s (18.1522GiB/s) | 20260507 00:02:09.807 | INFO | Downloaded 34GiB to GPU in 1.9058s (17.8403GiB/s) | 20260507 00:02:14.942 | INFO | Analyzing chunk frames = [5502976,5503488) | 20260507 00:02:14.942 | INFO | MemInfo: { | 20260507 00:02:14.942 | INFO | "gpu" : " 17 GB", | 20260507 00:02:14.942 | INFO | "huge" : "", | 20260507 00:02:14.942 | INFO | "huge_shared" : " 27 GB", | 20260507 00:02:14.942 | INFO | "malloc" : " 14 GB", | 20260507 00:02:14.942 | INFO | "pinned" : " 29 GB" | 20260507 00:02:14.942 | INFO | } | 20260507 00:02:14.942 | INFO | Perf report for the last 5.1357 seconds: | 20260507 00:02:14.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:02:14.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:14.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:14.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:14.942 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 00:02:14.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:14.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:14.942 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:02:14.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:14.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:14.943 | INFO | Uploaded 36GiB to GPU in 2.02639s (17.7656GiB/s) | 20260507 00:02:14.943 | INFO | Downloaded 34GiB to GPU in 1.94699s (17.4628GiB/s) | 20260507 00:02:20.078 | INFO | Analyzing chunk frames = [5503488,5504000) | 20260507 00:02:20.078 | INFO | MemInfo: { | 20260507 00:02:20.078 | INFO | "gpu" : " 17 GB", | 20260507 00:02:20.078 | INFO | "huge" : "", | 20260507 00:02:20.078 | INFO | "huge_shared" : " 27 GB", | 20260507 00:02:20.078 | INFO | "malloc" : " 14 GB", | 20260507 00:02:20.078 | INFO | "pinned" : " 29 GB" | 20260507 00:02:20.078 | INFO | } | 20260507 00:02:20.078 | INFO | Perf report for the last 5.13617 seconds: | 20260507 00:02:20.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:02:20.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:20.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:20.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:20.078 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:02:20.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:20.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:20.078 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 00:02:20.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:20.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:20.079 | INFO | Uploaded 36GiB to GPU in 2.02785s (17.7528GiB/s) | 20260507 00:02:20.079 | INFO | Downloaded 34GiB to GPU in 1.93861s (17.5383GiB/s) | 20260507 00:02:25.218 | INFO | Analyzing chunk frames = [5504000,5504512) | 20260507 00:02:25.218 | INFO | MemInfo: { | 20260507 00:02:25.218 | INFO | "gpu" : " 17 GB", | 20260507 00:02:25.218 | INFO | "huge" : "", | 20260507 00:02:25.218 | INFO | "huge_shared" : " 27 GB", | 20260507 00:02:25.218 | INFO | "malloc" : " 14 GB", | 20260507 00:02:25.218 | INFO | "pinned" : " 29 GB" | 20260507 00:02:25.218 | INFO | } | 20260507 00:02:25.218 | INFO | Perf report for the last 5.1394 seconds: | 20260507 00:02:25.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:02:25.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:25.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:25.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:25.218 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260507 00:02:25.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:25.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:25.218 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:02:25.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:25.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:25.218 | INFO | Uploaded 36GiB to GPU in 1.98822s (18.1066GiB/s) | 20260507 00:02:25.218 | INFO | Downloaded 34GiB to GPU in 1.99309s (17.0589GiB/s) | 20260507 00:02:30.361 | INFO | Analyzing chunk frames = [5504512,5505024) | 20260507 00:02:30.361 | INFO | MemInfo: { | 20260507 00:02:30.361 | INFO | "gpu" : " 17 GB", | 20260507 00:02:30.361 | INFO | "huge" : "", | 20260507 00:02:30.361 | INFO | "huge_shared" : " 27 GB", | 20260507 00:02:30.361 | INFO | "malloc" : " 14 GB", | 20260507 00:02:30.361 | INFO | "pinned" : " 29 GB" | 20260507 00:02:30.361 | INFO | } | 20260507 00:02:30.361 | INFO | Perf report for the last 5.14331 seconds: | 20260507 00:02:30.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:02:30.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:30.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:30.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:30.361 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260507 00:02:30.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:30.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:30.361 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 00:02:30.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:30.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:30.361 | INFO | Uploaded 36GiB to GPU in 2.05058s (17.556GiB/s) | 20260507 00:02:30.361 | INFO | Downloaded 34GiB to GPU in 1.99552s (17.0382GiB/s) | 20260507 00:02:30.362 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:02:30.362 | INFO | Total mem usage: 18 GiB | 20260507 00:02:30.362 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:02:30.362 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:02:30.362 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:02:30.362 | INFO | High Water Breakdown: | 20260507 00:02:30.362 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:02:30.362 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:02:30.362 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:02:30.362 | INFO | ------------------------------------------ | 20260507 00:02:30.362 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:02:30.362 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:02:30.362 | INFO | Total mem usage: 26.5 GiB | 20260507 00:02:30.362 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:02:30.362 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:02:30.362 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:02:30.362 | INFO | High Water Breakdown: | 20260507 00:02:30.362 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:02:30.362 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:02:30.362 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:02:30.362 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:02:30.362 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:02:30.362 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:02:30.362 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:02:30.362 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:02:30.362 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:02:30.362 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:02:30.362 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:02:30.362 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:02:30.362 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:02:30.362 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:02:30.362 | INFO | ------------------------------------------ | 20260507 00:02:30.362 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:02:30.362 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:02:30.362 | INFO | Total mem usage: 25.6 GiB | 20260507 00:02:30.362 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:02:30.362 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:02:30.362 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:02:30.362 | INFO | High Water Breakdown: | 20260507 00:02:30.362 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:02:30.362 | INFO | ------------------------------------------ | 20260507 00:02:30.362 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:02:30.362 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:02:30.362 | INFO | Total mem usage: 15.7 GiB | 20260507 00:02:30.362 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:02:30.362 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:02:30.362 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:02:30.362 | INFO | High Water Breakdown: | 20260507 00:02:30.362 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:02:30.362 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:02:30.362 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:02:30.362 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:02:30.362 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:02:30.363 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:02:30.363 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:02:30.363 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:02:30.363 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:02:30.363 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:02:30.363 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:02:30.363 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:02:30.363 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:02:30.363 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:02:30.363 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:02:30.363 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:02:30.363 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:02:30.363 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:02:30.363 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:02:30.363 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:02:30.363 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:02:30.363 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:02:30.363 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:02:30.363 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:02:30.363 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:02:30.363 | INFO | ------------------------------------------ | 20260507 00:02:30.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:02:30.363 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:02:30.363 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:02:30.363 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:02:30.363 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.60e+01, 1.35e+01 / 2.00e+00 | 20260507 00:02:30.363 | INFO | PerfReport: 0.30%, AggregateStats -- 9.13e-02 / 1.01e-01, 9.83e-03 / 5.83e-03 | 20260507 00:02:30.363 | INFO | PerfReport: 11.66%, Baseline -- 3.61e+00 / 4.42e+00, 5.33e-01 / 3.11e-01 | 20260507 00:02:30.363 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.46e+00, 1.72e-01 / 1.26e-01 | 20260507 00:02:30.363 | INFO | PerfReport: 16.12%, DME -- 3.99e+01 / 4.52e+01, 4.28e+00 / 2.24e+00 | 20260507 00:02:30.363 | INFO | PerfReport: 57.35%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.27e+00 / 1.53e+00 | 20260507 00:02:30.363 | INFO | PerfReport: 7.08%, PulseAccumulating -- 2.19e+00 / 2.77e+00, 3.86e-01 / 1.95e-01 | 20260507 00:02:30.363 | INFO | PerfReport: 3.17%, Metrics -- 9.82e-01 / 1.33e+00, 2.61e-01 / 1.59e-01 | 20260507 00:02:30.364 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.34e-01 / 3.88e-01, 5.25e-02 / 2.12e-02 | 20260507 00:02:30.364 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:02:30.364 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:02:30.364 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:02:30.364 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 00:02:30.364 | INFO | PerfReport: 21.57%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260507 00:02:30.364 | INFO | PerfReport: 21.30%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 00:02:30.364 | INFO | PerfReport: 32.01%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:02:30.364 | INFO | PerfReport: 1.35%, LockUpload -- 1.31e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260507 00:02:30.364 | INFO | PerfReport: 1.32%, LockDownload -- 1.28e+00 / 0.00e+00, 3.15e+00 / 0.00e+00 | 20260507 00:02:30.364 | INFO | PerfReport: 22.43%, LockCompute -- 2.17e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 00:02:30.364 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.84e-02 / 0.00e+00 | 20260507 00:02:30.463 | INFO | Starting a round of Baz IO | 20260507 00:02:33.588 | INFO | Populating RTMetrics report | 20260507 00:02:33.588 | INFO | RTMetrics report populated | 20260507 00:02:33.588 | INFO | Opening tmp RTMetrics json file | 20260507 00:02:33.589 | INFO | Serializing RTMetrics to json | 20260507 00:02:33.591 | INFO | Serialization done, swapping tmp file now... | 20260507 00:02:33.610 | INFO | RTMetrics done | 20260507 00:02:33.682 | INFO | Finished a round of Baz IO: Wrote 1.862 GiB in 3.219 seconds | 20260507 00:02:33.682 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.36% / 0.00% / 66.64% | 20260507 00:02:33.682 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.372577% / 1.19209e-05% / 5.5631e-05% | 20260507 00:02:33.682 | INFO | Baz IO overall: | 20260507 00:02:33.682 | INFO | BytesWritten : 2874639293293 | 20260507 00:02:33.682 | INFO | NumEvents : 1385288937983 | 20260507 00:02:35.503 | INFO | Analyzing chunk frames = [5505024,5505536) | 20260507 00:02:35.503 | INFO | MemInfo: { | 20260507 00:02:35.503 | INFO | "gpu" : " 17 GB", | 20260507 00:02:35.503 | INFO | "huge" : "", | 20260507 00:02:35.503 | INFO | "huge_shared" : " 27 GB", | 20260507 00:02:35.503 | INFO | "malloc" : " 14 GB", | 20260507 00:02:35.503 | INFO | "pinned" : " 29 GB" | 20260507 00:02:35.503 | INFO | } | 20260507 00:02:35.503 | INFO | Perf report for the last 5.14169 seconds: | 20260507 00:02:35.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:02:35.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:35.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:35.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:35.503 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 00:02:35.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:35.503 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 00:02:35.503 | INFO | PRE_HQ : 59.5%, 1.8, 56.6ms, 0 | 20260507 00:02:35.503 | INFO | BAZWRITER : 62.2%, 3.4, 108.6ms, 0 | 20260507 00:02:35.503 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:02:35.503 | INFO | Uploaded 36GiB to GPU in 2.01296s (17.8841GiB/s) | 20260507 00:02:35.503 | INFO | Downloaded 38GiB to GPU in 2.18806s (17.367GiB/s) | 20260507 00:02:40.653 | INFO | Analyzing chunk frames = [5505536,5506048) | 20260507 00:02:40.653 | INFO | MemInfo: { | 20260507 00:02:40.653 | INFO | "gpu" : " 17 GB", | 20260507 00:02:40.653 | INFO | "huge" : "", | 20260507 00:02:40.653 | INFO | "huge_shared" : " 27 GB", | 20260507 00:02:40.653 | INFO | "malloc" : " 14 GB", | 20260507 00:02:40.653 | INFO | "pinned" : " 29 GB" | 20260507 00:02:40.653 | INFO | } | 20260507 00:02:40.653 | INFO | Perf report for the last 5.15001 seconds: | 20260507 00:02:40.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:02:40.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:40.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:40.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:40.653 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260507 00:02:40.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:40.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:40.653 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 00:02:40.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:40.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:40.653 | INFO | Uploaded 36GiB to GPU in 2.00233s (17.979GiB/s) | 20260507 00:02:40.653 | INFO | Downloaded 34GiB to GPU in 1.93195s (17.5988GiB/s) | 20260507 00:02:45.787 | INFO | Analyzing chunk frames = [5506048,5506560) | 20260507 00:02:45.787 | INFO | MemInfo: { | 20260507 00:02:45.787 | INFO | "gpu" : " 17 GB", | 20260507 00:02:45.787 | INFO | "huge" : "", | 20260507 00:02:45.787 | INFO | "huge_shared" : " 27 GB", | 20260507 00:02:45.787 | INFO | "malloc" : " 14 GB", | 20260507 00:02:45.787 | INFO | "pinned" : " 29 GB" | 20260507 00:02:45.787 | INFO | } | 20260507 00:02:45.787 | INFO | Perf report for the last 5.13443 seconds: | 20260507 00:02:45.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:02:45.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:45.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:45.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:45.788 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:02:45.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:45.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:45.788 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:02:45.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:45.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:45.788 | INFO | Uploaded 36GiB to GPU in 1.99903s (18.0088GiB/s) | 20260507 00:02:45.788 | INFO | Downloaded 34GiB to GPU in 1.94118s (17.5151GiB/s) | 20260507 00:02:50.924 | INFO | Analyzing chunk frames = [5506560,5507072) | 20260507 00:02:50.924 | INFO | MemInfo: { | 20260507 00:02:50.924 | INFO | "gpu" : " 17 GB", | 20260507 00:02:50.924 | INFO | "huge" : "", | 20260507 00:02:50.924 | INFO | "huge_shared" : " 27 GB", | 20260507 00:02:50.924 | INFO | "malloc" : " 14 GB", | 20260507 00:02:50.924 | INFO | "pinned" : " 29 GB" | 20260507 00:02:50.924 | INFO | } | 20260507 00:02:50.924 | INFO | Perf report for the last 5.13685 seconds: | 20260507 00:02:50.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:02:50.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:50.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:50.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:50.924 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 00:02:50.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:50.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:50.924 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:02:50.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:50.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:50.924 | INFO | Uploaded 36GiB to GPU in 1.96665s (18.3053GiB/s) | 20260507 00:02:50.924 | INFO | Downloaded 34GiB to GPU in 1.90613s (17.8372GiB/s) | 20260507 00:02:56.057 | INFO | Analyzing chunk frames = [5507072,5507584) | 20260507 00:02:56.057 | INFO | MemInfo: { | 20260507 00:02:56.057 | INFO | "gpu" : " 17 GB", | 20260507 00:02:56.057 | INFO | "huge" : "", | 20260507 00:02:56.057 | INFO | "huge_shared" : " 27 GB", | 20260507 00:02:56.057 | INFO | "malloc" : " 14 GB", | 20260507 00:02:56.057 | INFO | "pinned" : " 29 GB" | 20260507 00:02:56.057 | INFO | } | 20260507 00:02:56.057 | INFO | Perf report for the last 5.13264 seconds: | 20260507 00:02:56.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:02:56.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:56.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:56.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:56.057 | INFO | ANALYSIS : 60.2%, 3.0, 93.5ms, 0 | 20260507 00:02:56.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:56.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:56.057 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:02:56.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:56.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:02:56.057 | INFO | Uploaded 36GiB to GPU in 1.98802s (18.1084GiB/s) | 20260507 00:02:56.057 | INFO | Downloaded 34GiB to GPU in 1.93608s (17.5612GiB/s) | 20260507 00:03:01.193 | INFO | Analyzing chunk frames = [5507584,5508096) | 20260507 00:03:01.193 | INFO | MemInfo: { | 20260507 00:03:01.193 | INFO | "gpu" : " 17 GB", | 20260507 00:03:01.193 | INFO | "huge" : "", | 20260507 00:03:01.193 | INFO | "huge_shared" : " 27 GB", | 20260507 00:03:01.193 | INFO | "malloc" : " 14 GB", | 20260507 00:03:01.193 | INFO | "pinned" : " 29 GB" | 20260507 00:03:01.194 | INFO | } | 20260507 00:03:01.194 | INFO | Perf report for the last 5.13688 seconds: | 20260507 00:03:01.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:03:01.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:01.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:01.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:01.194 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:03:01.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:01.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:01.194 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 00:03:01.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:01.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:01.194 | INFO | Uploaded 36GiB to GPU in 1.99169s (18.0751GiB/s) | 20260507 00:03:01.194 | INFO | Downloaded 34GiB to GPU in 1.95532s (17.3885GiB/s) | 20260507 00:03:06.340 | INFO | Analyzing chunk frames = [5508096,5508608) | 20260507 00:03:06.340 | INFO | MemInfo: { | 20260507 00:03:06.340 | INFO | "gpu" : " 17 GB", | 20260507 00:03:06.340 | INFO | "huge" : "", | 20260507 00:03:06.340 | INFO | "huge_shared" : " 27 GB", | 20260507 00:03:06.340 | INFO | "malloc" : " 14 GB", | 20260507 00:03:06.340 | INFO | "pinned" : " 29 GB" | 20260507 00:03:06.340 | INFO | } | 20260507 00:03:06.340 | INFO | Perf report for the last 5.14607 seconds: | 20260507 00:03:06.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:03:06.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:06.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:06.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:06.340 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260507 00:03:06.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:06.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:06.340 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 00:03:06.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:06.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:06.340 | INFO | Uploaded 36GiB to GPU in 1.97984s (18.1833GiB/s) | 20260507 00:03:06.340 | INFO | Downloaded 34GiB to GPU in 1.99715s (17.0243GiB/s) | 20260507 00:03:11.478 | INFO | Analyzing chunk frames = [5508608,5509120) | 20260507 00:03:11.478 | INFO | MemInfo: { | 20260507 00:03:11.478 | INFO | "gpu" : " 17 GB", | 20260507 00:03:11.478 | INFO | "huge" : "", | 20260507 00:03:11.478 | INFO | "huge_shared" : " 27 GB", | 20260507 00:03:11.478 | INFO | "malloc" : " 14 GB", | 20260507 00:03:11.478 | INFO | "pinned" : " 29 GB" | 20260507 00:03:11.478 | INFO | } | 20260507 00:03:11.478 | INFO | Perf report for the last 5.13792 seconds: | 20260507 00:03:11.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:03:11.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:11.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:11.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:11.478 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 00:03:11.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:11.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:11.478 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:03:11.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:11.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:11.478 | INFO | Uploaded 36GiB to GPU in 1.98498s (18.1362GiB/s) | 20260507 00:03:11.478 | INFO | Downloaded 34GiB to GPU in 2.00045s (16.9962GiB/s) | 20260507 00:03:11.576 | INFO | Starting a round of Baz IO | 20260507 00:03:14.708 | INFO | Populating RTMetrics report | 20260507 00:03:14.708 | INFO | RTMetrics report populated | 20260507 00:03:14.708 | INFO | Opening tmp RTMetrics json file | 20260507 00:03:14.715 | INFO | Serializing RTMetrics to json | 20260507 00:03:14.727 | INFO | Serialization done, swapping tmp file now... | 20260507 00:03:14.727 | INFO | RTMetrics done | 20260507 00:03:14.802 | INFO | Finished a round of Baz IO: Wrote 1.863 GiB in 3.226 seconds | 20260507 00:03:14.802 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.36% / 0.00% / 66.64% | 20260507 00:03:14.802 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.372545% / 0% / 7.54992e-05% | 20260507 00:03:14.802 | INFO | Baz IO overall: | 20260507 00:03:14.802 | INFO | BytesWritten : 2876639947342 | 20260507 00:03:14.802 | INFO | NumEvents : 1386240827895 | 20260507 00:03:16.617 | INFO | Analyzing chunk frames = [5509120,5509632) | 20260507 00:03:16.617 | INFO | MemInfo: { | 20260507 00:03:16.617 | INFO | "gpu" : " 17 GB", | 20260507 00:03:16.617 | INFO | "huge" : "", | 20260507 00:03:16.617 | INFO | "huge_shared" : " 27 GB", | 20260507 00:03:16.617 | INFO | "malloc" : " 14 GB", | 20260507 00:03:16.617 | INFO | "pinned" : " 29 GB" | 20260507 00:03:16.617 | INFO | } | 20260507 00:03:16.617 | INFO | Perf report for the last 5.13947 seconds: | 20260507 00:03:16.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:03:16.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:16.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:16.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:16.617 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260507 00:03:16.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:16.617 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260507 00:03:16.617 | INFO | PRE_HQ : 59.9%, 1.8, 56.6ms, 0 | 20260507 00:03:16.617 | INFO | BAZWRITER : 62.3%, 3.4, 108.7ms, 0 | 20260507 00:03:16.617 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 00:03:16.618 | INFO | Uploaded 36GiB to GPU in 2.04224s (17.6277GiB/s) | 20260507 00:03:16.618 | INFO | Downloaded 38GiB to GPU in 2.17994s (17.4317GiB/s) | 20260507 00:03:21.759 | INFO | Analyzing chunk frames = [5509632,5510144) | 20260507 00:03:21.759 | INFO | MemInfo: { | 20260507 00:03:21.759 | INFO | "gpu" : " 17 GB", | 20260507 00:03:21.759 | INFO | "huge" : "", | 20260507 00:03:21.759 | INFO | "huge_shared" : " 27 GB", | 20260507 00:03:21.759 | INFO | "malloc" : " 14 GB", | 20260507 00:03:21.759 | INFO | "pinned" : " 29 GB" | 20260507 00:03:21.759 | INFO | } | 20260507 00:03:21.759 | INFO | Perf report for the last 5.14201 seconds: | 20260507 00:03:21.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:03:21.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:21.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:21.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:21.759 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 00:03:21.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:21.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:21.759 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:03:21.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:21.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:21.760 | INFO | Uploaded 36GiB to GPU in 1.99178s (18.0743GiB/s) | 20260507 00:03:21.760 | INFO | Downloaded 34GiB to GPU in 1.9215s (17.6945GiB/s) | 20260507 00:03:26.894 | INFO | Analyzing chunk frames = [5510144,5510656) | 20260507 00:03:26.894 | INFO | MemInfo: { | 20260507 00:03:26.894 | INFO | "gpu" : " 17 GB", | 20260507 00:03:26.894 | INFO | "huge" : "", | 20260507 00:03:26.894 | INFO | "huge_shared" : " 27 GB", | 20260507 00:03:26.894 | INFO | "malloc" : " 14 GB", | 20260507 00:03:26.894 | INFO | "pinned" : " 29 GB" | 20260507 00:03:26.894 | INFO | } | 20260507 00:03:26.894 | INFO | Perf report for the last 5.13522 seconds: | 20260507 00:03:26.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:03:26.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:26.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:26.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:26.895 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:03:26.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:26.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:26.895 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:03:26.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:26.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:26.895 | INFO | Uploaded 36GiB to GPU in 1.98169s (18.1663GiB/s) | 20260507 00:03:26.895 | INFO | Downloaded 34GiB to GPU in 1.9313s (17.6047GiB/s) | 20260507 00:03:32.031 | INFO | Analyzing chunk frames = [5510656,5511168) | 20260507 00:03:32.031 | INFO | MemInfo: { | 20260507 00:03:32.031 | INFO | "gpu" : " 17 GB", | 20260507 00:03:32.031 | INFO | "huge" : "", | 20260507 00:03:32.031 | INFO | "huge_shared" : " 27 GB", | 20260507 00:03:32.031 | INFO | "malloc" : " 14 GB", | 20260507 00:03:32.031 | INFO | "pinned" : " 29 GB" | 20260507 00:03:32.031 | INFO | } | 20260507 00:03:32.031 | INFO | Perf report for the last 5.13645 seconds: | 20260507 00:03:32.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:03:32.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:32.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:32.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:32.031 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260507 00:03:32.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:32.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:32.031 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 00:03:32.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:32.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:32.031 | INFO | Uploaded 36GiB to GPU in 1.99491s (18.046GiB/s) | 20260507 00:03:32.031 | INFO | Downloaded 34GiB to GPU in 1.9043s (17.8544GiB/s) | 20260507 00:03:37.169 | INFO | Analyzing chunk frames = [5511168,5511680) | 20260507 00:03:37.169 | INFO | MemInfo: { | 20260507 00:03:37.169 | INFO | "gpu" : " 17 GB", | 20260507 00:03:37.169 | INFO | "huge" : "", | 20260507 00:03:37.169 | INFO | "huge_shared" : " 27 GB", | 20260507 00:03:37.169 | INFO | "malloc" : " 14 GB", | 20260507 00:03:37.169 | INFO | "pinned" : " 29 GB" | 20260507 00:03:37.169 | INFO | } | 20260507 00:03:37.169 | INFO | Perf report for the last 5.1381 seconds: | 20260507 00:03:37.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:03:37.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:37.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:37.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:37.169 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 00:03:37.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:37.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:37.169 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 00:03:37.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:37.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:37.169 | INFO | Uploaded 36GiB to GPU in 2.00831s (17.9255GiB/s) | 20260507 00:03:37.169 | INFO | Downloaded 34GiB to GPU in 1.91967s (17.7114GiB/s) | 20260507 00:03:42.314 | INFO | Analyzing chunk frames = [5511680,5512192) | 20260507 00:03:42.314 | INFO | MemInfo: { | 20260507 00:03:42.314 | INFO | "gpu" : " 17 GB", | 20260507 00:03:42.314 | INFO | "huge" : "", | 20260507 00:03:42.314 | INFO | "huge_shared" : " 27 GB", | 20260507 00:03:42.314 | INFO | "malloc" : " 14 GB", | 20260507 00:03:42.314 | INFO | "pinned" : " 29 GB" | 20260507 00:03:42.314 | INFO | } | 20260507 00:03:42.314 | INFO | Perf report for the last 5.14499 seconds: | 20260507 00:03:42.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:03:42.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:42.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:42.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:42.314 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:03:42.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:42.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:42.314 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 00:03:42.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:42.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:42.314 | INFO | Uploaded 36GiB to GPU in 2.021s (17.8129GiB/s) | 20260507 00:03:42.314 | INFO | Downloaded 34GiB to GPU in 1.95669s (17.3762GiB/s) | 20260507 00:03:47.456 | INFO | Analyzing chunk frames = [5512192,5512704) | 20260507 00:03:47.456 | INFO | MemInfo: { | 20260507 00:03:47.456 | INFO | "gpu" : " 17 GB", | 20260507 00:03:47.456 | INFO | "huge" : "", | 20260507 00:03:47.456 | INFO | "huge_shared" : " 27 GB", | 20260507 00:03:47.456 | INFO | "malloc" : " 14 GB", | 20260507 00:03:47.456 | INFO | "pinned" : " 29 GB" | 20260507 00:03:47.456 | INFO | } | 20260507 00:03:47.456 | INFO | Perf report for the last 5.1423 seconds: | 20260507 00:03:47.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:03:47.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:47.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:47.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:47.456 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:03:47.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:47.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:47.456 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 00:03:47.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:47.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:47.457 | INFO | Uploaded 36GiB to GPU in 1.96863s (18.2869GiB/s) | 20260507 00:03:47.457 | INFO | Downloaded 34GiB to GPU in 1.99304s (17.0594GiB/s) | 20260507 00:03:52.599 | INFO | Analyzing chunk frames = [5512704,5513216) | 20260507 00:03:52.599 | INFO | MemInfo: { | 20260507 00:03:52.600 | INFO | "gpu" : " 17 GB", | 20260507 00:03:52.600 | INFO | "huge" : "", | 20260507 00:03:52.600 | INFO | "huge_shared" : " 27 GB", | 20260507 00:03:52.600 | INFO | "malloc" : " 14 GB", | 20260507 00:03:52.600 | INFO | "pinned" : " 29 GB" | 20260507 00:03:52.600 | INFO | } | 20260507 00:03:52.600 | INFO | Perf report for the last 5.14321 seconds: | 20260507 00:03:52.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:03:52.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:52.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:52.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:52.600 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:03:52.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:52.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:52.600 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:03:52.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:52.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:52.600 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:03:52.600 | INFO | Uploaded 36GiB to GPU in 1.99207s (18.0716GiB/s) | 20260507 00:03:52.600 | INFO | Total mem usage: 18 GiB | 20260507 00:03:52.600 | INFO | Downloaded 34GiB to GPU in 1.99442s (17.0475GiB/s) | 20260507 00:03:52.600 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:03:52.600 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 00:03:52.600 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:03:52.600 | INFO | High Water Breakdown: | 20260507 00:03:52.600 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:03:52.600 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:03:52.600 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:03:52.600 | INFO | ------------------------------------------ | 20260507 00:03:52.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:03:52.600 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:03:52.600 | INFO | Total mem usage: 26.5 GiB | 20260507 00:03:52.600 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:03:52.600 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:03:52.600 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:03:52.600 | INFO | High Water Breakdown: | 20260507 00:03:52.600 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:03:52.600 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:03:52.600 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:03:52.600 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:03:52.600 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:03:52.600 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:03:52.600 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:03:52.600 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:03:52.600 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:03:52.600 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:03:52.600 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:03:52.600 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:03:52.600 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:03:52.600 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:03:52.600 | INFO | ------------------------------------------ | 20260507 00:03:52.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:03:52.600 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:03:52.600 | INFO | Total mem usage: 25.6 GiB | 20260507 00:03:52.600 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:03:52.600 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:03:52.600 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:03:52.600 | INFO | High Water Breakdown: | 20260507 00:03:52.600 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:03:52.600 | INFO | ------------------------------------------ | 20260507 00:03:52.600 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:03:52.600 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:03:52.600 | INFO | Total mem usage: 15.7 GiB | 20260507 00:03:52.601 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:03:52.601 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:03:52.601 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 00:03:52.601 | INFO | High Water Breakdown: | 20260507 00:03:52.601 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:03:52.601 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:03:52.601 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:03:52.601 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:03:52.601 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:03:52.601 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:03:52.601 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:03:52.601 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:03:52.601 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:03:52.601 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:03:52.601 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:03:52.601 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:03:52.601 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:03:52.601 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:03:52.601 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:03:52.601 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:03:52.601 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:03:52.601 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:03:52.601 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:03:52.601 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:03:52.601 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:03:52.601 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:03:52.601 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:03:52.601 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:03:52.601 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:03:52.601 | INFO | ------------------------------------------ | 20260507 00:03:52.601 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:03:52.601 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:03:52.601 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:03:52.601 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:03:52.601 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.68e+01, 1.35e+01 / 2.69e+00 | 20260507 00:03:52.601 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.02e-01, 1.01e-02 / 6.24e-03 | 20260507 00:03:52.601 | INFO | PerfReport: 11.67%, Baseline -- 3.61e+00 / 4.41e+00, 5.44e-01 / 3.24e-01 | 20260507 00:03:52.601 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.45e+00, 1.78e-01 / 1.42e-01 | 20260507 00:03:52.601 | INFO | PerfReport: 16.10%, DME -- 3.99e+01 / 4.54e+01, 4.39e+00 / 3.51e+00 | 20260507 00:03:52.601 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.36e+00 / 2.16e+00 | 20260507 00:03:52.601 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.18e+00 / 2.69e+00, 3.75e-01 / 2.64e-01 | 20260507 00:03:52.601 | INFO | PerfReport: 3.17%, Metrics -- 9.82e-01 / 1.31e+00, 2.64e-01 / 1.47e-01 | 20260507 00:03:52.601 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.34e-01 / 3.84e-01, 5.26e-02 / 1.64e-02 | 20260507 00:03:52.601 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:03:52.601 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:03:52.601 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:03:52.601 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 00:03:52.601 | INFO | PerfReport: 21.51%, Upload -- 2.08e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260507 00:03:52.601 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260507 00:03:52.601 | INFO | PerfReport: 32.07%, Compute -- 3.10e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:03:52.601 | INFO | PerfReport: 1.26%, LockUpload -- 1.22e+00 / 0.00e+00, 4.66e+00 / 0.00e+00 | 20260507 00:03:52.601 | INFO | PerfReport: 1.32%, LockDownload -- 1.28e+00 / 0.00e+00, 3.20e+00 / 0.00e+00 | 20260507 00:03:52.601 | INFO | PerfReport: 22.49%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 00:03:52.601 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260507 00:03:52.696 | INFO | Starting a round of Baz IO | 20260507 00:03:55.813 | INFO | Populating RTMetrics report | 20260507 00:03:55.813 | INFO | RTMetrics report populated | 20260507 00:03:55.813 | INFO | Opening tmp RTMetrics json file | 20260507 00:03:55.814 | INFO | Serializing RTMetrics to json | 20260507 00:03:55.821 | INFO | Serialization done, swapping tmp file now... | 20260507 00:03:55.835 | INFO | RTMetrics done | 20260507 00:03:55.906 | INFO | Finished a round of Baz IO: Wrote 1.863 GiB in 3.209 seconds | 20260507 00:03:55.906 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.35% / 0.00% / 66.64% | 20260507 00:03:55.906 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.36999% / 0% / 7.94729e-05% | 20260507 00:03:55.906 | INFO | Baz IO overall: | 20260507 00:03:55.906 | INFO | BytesWritten : 2878640052974 | 20260507 00:03:55.906 | INFO | NumEvents : 1387191479899 | 20260507 00:03:57.734 | INFO | Analyzing chunk frames = [5513216,5513728) | 20260507 00:03:57.734 | INFO | MemInfo: { | 20260507 00:03:57.734 | INFO | "gpu" : " 17 GB", | 20260507 00:03:57.734 | INFO | "huge" : "", | 20260507 00:03:57.734 | INFO | "huge_shared" : " 27 GB", | 20260507 00:03:57.734 | INFO | "malloc" : " 14 GB", | 20260507 00:03:57.734 | INFO | "pinned" : " 29 GB" | 20260507 00:03:57.734 | INFO | } | 20260507 00:03:57.734 | INFO | Perf report for the last 5.13449 seconds: | 20260507 00:03:57.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:03:57.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:57.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:57.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:57.734 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 00:03:57.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:03:57.734 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 00:03:57.734 | INFO | PRE_HQ : 59.8%, 1.8, 56.9ms, 0 | 20260507 00:03:57.734 | INFO | BAZWRITER : 62.1%, 3.4, 108.3ms, 0 | 20260507 00:03:57.734 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:03:57.734 | INFO | Uploaded 36GiB to GPU in 2.01779s (17.8413GiB/s) | 20260507 00:03:57.734 | INFO | Downloaded 38GiB to GPU in 2.17939s (17.4361GiB/s) | 20260507 00:04:02.875 | INFO | Analyzing chunk frames = [5513728,5514240) | 20260507 00:04:02.875 | INFO | MemInfo: { | 20260507 00:04:02.875 | INFO | "gpu" : " 17 GB", | 20260507 00:04:02.875 | INFO | "huge" : "", | 20260507 00:04:02.875 | INFO | "huge_shared" : " 27 GB", | 20260507 00:04:02.875 | INFO | "malloc" : " 14 GB", | 20260507 00:04:02.875 | INFO | "pinned" : " 29 GB" | 20260507 00:04:02.875 | INFO | } | 20260507 00:04:02.875 | INFO | Perf report for the last 5.14097 seconds: | 20260507 00:04:02.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:04:02.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:02.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:02.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:02.875 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:04:02.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:02.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:02.875 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:04:02.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:02.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:02.875 | INFO | Uploaded 36GiB to GPU in 1.98512s (18.1349GiB/s) | 20260507 00:04:02.875 | INFO | Downloaded 34GiB to GPU in 1.93103s (17.6072GiB/s) | 20260507 00:04:08.013 | INFO | Analyzing chunk frames = [5514240,5514752) | 20260507 00:04:08.013 | INFO | MemInfo: { | 20260507 00:04:08.013 | INFO | "gpu" : " 17 GB", | 20260507 00:04:08.013 | INFO | "huge" : "", | 20260507 00:04:08.013 | INFO | "huge_shared" : " 27 GB", | 20260507 00:04:08.013 | INFO | "malloc" : " 14 GB", | 20260507 00:04:08.013 | INFO | "pinned" : " 29 GB" | 20260507 00:04:08.013 | INFO | } | 20260507 00:04:08.013 | INFO | Perf report for the last 5.13811 seconds: | 20260507 00:04:08.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:04:08.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:08.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:08.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:08.013 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:04:08.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:08.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:08.013 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:04:08.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:08.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:08.013 | INFO | Uploaded 36GiB to GPU in 1.96747s (18.2976GiB/s) | 20260507 00:04:08.013 | INFO | Downloaded 34GiB to GPU in 1.93451s (17.5755GiB/s) | 20260507 00:04:13.144 | INFO | Analyzing chunk frames = [5514752,5515264) | 20260507 00:04:13.144 | INFO | MemInfo: { | 20260507 00:04:13.144 | INFO | "gpu" : " 17 GB", | 20260507 00:04:13.144 | INFO | "huge" : "", | 20260507 00:04:13.144 | INFO | "huge_shared" : " 27 GB", | 20260507 00:04:13.144 | INFO | "malloc" : " 14 GB", | 20260507 00:04:13.144 | INFO | "pinned" : " 29 GB" | 20260507 00:04:13.144 | INFO | } | 20260507 00:04:13.144 | INFO | Perf report for the last 5.13097 seconds: | 20260507 00:04:13.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:04:13.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:13.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:13.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:13.144 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 00:04:13.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:13.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:13.144 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:04:13.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:13.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:13.145 | INFO | Uploaded 36GiB to GPU in 1.97622s (18.2166GiB/s) | 20260507 00:04:13.145 | INFO | Downloaded 34GiB to GPU in 1.9078s (17.8216GiB/s) | 20260507 00:04:18.289 | INFO | Analyzing chunk frames = [5515264,5515776) | 20260507 00:04:18.289 | INFO | MemInfo: { | 20260507 00:04:18.289 | INFO | "gpu" : " 17 GB", | 20260507 00:04:18.289 | INFO | "huge" : "", | 20260507 00:04:18.289 | INFO | "huge_shared" : " 27 GB", | 20260507 00:04:18.289 | INFO | "malloc" : " 14 GB", | 20260507 00:04:18.289 | INFO | "pinned" : " 29 GB" | 20260507 00:04:18.289 | INFO | } | 20260507 00:04:18.289 | INFO | Perf report for the last 5.14526 seconds: | 20260507 00:04:18.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:04:18.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:18.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:18.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:18.289 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 00:04:18.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:18.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:18.290 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:04:18.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:18.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:18.290 | INFO | Uploaded 36GiB to GPU in 1.97347s (18.242GiB/s) | 20260507 00:04:18.290 | INFO | Downloaded 34GiB to GPU in 1.93026s (17.6142GiB/s) | 20260507 00:04:23.433 | INFO | Analyzing chunk frames = [5515776,5516288) | 20260507 00:04:23.433 | INFO | MemInfo: { | 20260507 00:04:23.433 | INFO | "gpu" : " 17 GB", | 20260507 00:04:23.433 | INFO | "huge" : "", | 20260507 00:04:23.433 | INFO | "huge_shared" : " 27 GB", | 20260507 00:04:23.434 | INFO | "malloc" : " 14 GB", | 20260507 00:04:23.434 | INFO | "pinned" : " 29 GB" | 20260507 00:04:23.434 | INFO | } | 20260507 00:04:23.434 | INFO | Perf report for the last 5.14416 seconds: | 20260507 00:04:23.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:04:23.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:23.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:23.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:23.434 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:04:23.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:23.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:23.434 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 00:04:23.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:23.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:23.434 | INFO | Uploaded 36GiB to GPU in 2.01041s (17.9068GiB/s) | 20260507 00:04:23.434 | INFO | Downloaded 34GiB to GPU in 1.94206s (17.5072GiB/s) | 20260507 00:04:28.569 | INFO | Analyzing chunk frames = [5516288,5516800) | 20260507 00:04:28.569 | INFO | MemInfo: { | 20260507 00:04:28.569 | INFO | "gpu" : " 17 GB", | 20260507 00:04:28.569 | INFO | "huge" : "", | 20260507 00:04:28.569 | INFO | "huge_shared" : " 27 GB", | 20260507 00:04:28.569 | INFO | "malloc" : " 14 GB", | 20260507 00:04:28.569 | INFO | "pinned" : " 29 GB" | 20260507 00:04:28.569 | INFO | } | 20260507 00:04:28.569 | INFO | Perf report for the last 5.13553 seconds: | 20260507 00:04:28.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:04:28.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:28.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:28.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:28.569 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:04:28.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:28.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:28.569 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 00:04:28.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:28.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:28.569 | INFO | Uploaded 36GiB to GPU in 1.98536s (18.1327GiB/s) | 20260507 00:04:28.570 | INFO | Downloaded 34GiB to GPU in 1.99952s (17.0041GiB/s) | 20260507 00:04:33.712 | INFO | Analyzing chunk frames = [5516800,5517312) | 20260507 00:04:33.712 | INFO | MemInfo: { | 20260507 00:04:33.712 | INFO | "gpu" : " 17 GB", | 20260507 00:04:33.712 | INFO | "huge" : "", | 20260507 00:04:33.712 | INFO | "huge_shared" : " 27 GB", | 20260507 00:04:33.712 | INFO | "malloc" : " 14 GB", | 20260507 00:04:33.712 | INFO | "pinned" : " 29 GB" | 20260507 00:04:33.712 | INFO | } | 20260507 00:04:33.712 | INFO | Perf report for the last 5.14282 seconds: | 20260507 00:04:33.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:04:33.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:33.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:33.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:33.712 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 00:04:33.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:33.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:33.712 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 00:04:33.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:33.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:33.712 | INFO | Uploaded 36GiB to GPU in 1.99151s (18.0768GiB/s) | 20260507 00:04:33.712 | INFO | Downloaded 34GiB to GPU in 1.99487s (17.0437GiB/s) | 20260507 00:04:33.809 | INFO | Starting a round of Baz IO | 20260507 00:04:36.934 | INFO | Populating RTMetrics report | 20260507 00:04:36.934 | INFO | RTMetrics report populated | 20260507 00:04:36.934 | INFO | Opening tmp RTMetrics json file | 20260507 00:04:36.935 | INFO | Serializing RTMetrics to json | 20260507 00:04:36.955 | INFO | Serialization done, swapping tmp file now... | 20260507 00:04:36.955 | INFO | RTMetrics done | 20260507 00:04:37.030 | INFO | Finished a round of Baz IO: Wrote 1.862 GiB in 3.221 seconds | 20260507 00:04:37.030 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.35% / 0.00% / 66.65% | 20260507 00:04:37.030 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.370034% / 0% / 5.5631e-05% | 20260507 00:04:37.031 | INFO | Baz IO overall: | 20260507 00:04:37.031 | INFO | BytesWritten : 2880639430781 | 20260507 00:04:37.031 | INFO | NumEvents : 1388141646758 | 20260507 00:04:38.851 | INFO | Analyzing chunk frames = [5517312,5517824) | 20260507 00:04:38.851 | INFO | MemInfo: { | 20260507 00:04:38.851 | INFO | "gpu" : " 17 GB", | 20260507 00:04:38.851 | INFO | "huge" : "", | 20260507 00:04:38.851 | INFO | "huge_shared" : " 27 GB", | 20260507 00:04:38.851 | INFO | "malloc" : " 14 GB", | 20260507 00:04:38.851 | INFO | "pinned" : " 29 GB" | 20260507 00:04:38.851 | INFO | } | 20260507 00:04:38.851 | INFO | Perf report for the last 5.1394 seconds: | 20260507 00:04:38.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:04:38.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:38.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:38.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:38.851 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260507 00:04:38.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:38.851 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 00:04:38.852 | INFO | PRE_HQ : 59.5%, 1.8, 56.4ms, 0 | 20260507 00:04:38.852 | INFO | BAZWRITER : 62.3%, 3.4, 108.4ms, 0 | 20260507 00:04:38.852 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:04:38.852 | INFO | Uploaded 36GiB to GPU in 2.03042s (17.7303GiB/s) | 20260507 00:04:38.852 | INFO | Downloaded 38GiB to GPU in 2.17369s (17.4818GiB/s) | 20260507 00:04:43.984 | INFO | Analyzing chunk frames = [5517824,5518336) | 20260507 00:04:43.984 | INFO | MemInfo: { | 20260507 00:04:43.984 | INFO | "gpu" : " 17 GB", | 20260507 00:04:43.984 | INFO | "huge" : "", | 20260507 00:04:43.984 | INFO | "huge_shared" : " 27 GB", | 20260507 00:04:43.984 | INFO | "malloc" : " 14 GB", | 20260507 00:04:43.984 | INFO | "pinned" : " 29 GB" | 20260507 00:04:43.984 | INFO | } | 20260507 00:04:43.984 | INFO | Perf report for the last 5.1328 seconds: | 20260507 00:04:43.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:04:43.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:43.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:43.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:43.984 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 00:04:43.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:43.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:43.984 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:04:43.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:43.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:43.985 | INFO | Uploaded 36GiB to GPU in 2.0135s (17.8793GiB/s) | 20260507 00:04:43.985 | INFO | Downloaded 34GiB to GPU in 1.92703s (17.6438GiB/s) | 20260507 00:04:49.124 | INFO | Analyzing chunk frames = [5518336,5518848) | 20260507 00:04:49.124 | INFO | MemInfo: { | 20260507 00:04:49.124 | INFO | "gpu" : " 17 GB", | 20260507 00:04:49.125 | INFO | "huge" : "", | 20260507 00:04:49.125 | INFO | "huge_shared" : " 27 GB", | 20260507 00:04:49.125 | INFO | "malloc" : " 14 GB", | 20260507 00:04:49.125 | INFO | "pinned" : " 29 GB" | 20260507 00:04:49.125 | INFO | } | 20260507 00:04:49.125 | INFO | Perf report for the last 5.14046 seconds: | 20260507 00:04:49.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:04:49.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:49.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:49.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:49.125 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 00:04:49.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:49.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:49.125 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 00:04:49.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:49.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:49.125 | INFO | Uploaded 36GiB to GPU in 1.97892s (18.1917GiB/s) | 20260507 00:04:49.125 | INFO | Downloaded 34GiB to GPU in 1.92385s (17.6729GiB/s) | 20260507 00:04:54.263 | INFO | Analyzing chunk frames = [5518848,5519360) | 20260507 00:04:54.263 | INFO | MemInfo: { | 20260507 00:04:54.263 | INFO | "gpu" : " 17 GB", | 20260507 00:04:54.263 | INFO | "huge" : "", | 20260507 00:04:54.263 | INFO | "huge_shared" : " 27 GB", | 20260507 00:04:54.263 | INFO | "malloc" : " 14 GB", | 20260507 00:04:54.263 | INFO | "pinned" : " 29 GB" | 20260507 00:04:54.263 | INFO | } | 20260507 00:04:54.263 | INFO | Perf report for the last 5.13843 seconds: | 20260507 00:04:54.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:04:54.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:54.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:54.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:54.263 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 00:04:54.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:54.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:54.263 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:04:54.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:54.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:54.263 | INFO | Uploaded 36GiB to GPU in 2.01816s (17.838GiB/s) | 20260507 00:04:54.264 | INFO | Downloaded 34GiB to GPU in 1.90859s (17.8142GiB/s) | 20260507 00:04:59.407 | INFO | Analyzing chunk frames = [5519360,5519872) | 20260507 00:04:59.407 | INFO | MemInfo: { | 20260507 00:04:59.407 | INFO | "gpu" : " 17 GB", | 20260507 00:04:59.407 | INFO | "huge" : "", | 20260507 00:04:59.407 | INFO | "huge_shared" : " 27 GB", | 20260507 00:04:59.407 | INFO | "malloc" : " 14 GB", | 20260507 00:04:59.407 | INFO | "pinned" : " 29 GB" | 20260507 00:04:59.407 | INFO | } | 20260507 00:04:59.408 | INFO | Perf report for the last 5.1444 seconds: | 20260507 00:04:59.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:04:59.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:59.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:59.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:59.408 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 00:04:59.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:59.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:59.408 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:04:59.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:59.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:04:59.408 | INFO | Uploaded 36GiB to GPU in 2.005s (17.9551GiB/s) | 20260507 00:04:59.408 | INFO | Downloaded 34GiB to GPU in 1.93328s (17.5867GiB/s) | 20260507 00:05:04.546 | INFO | Analyzing chunk frames = [5519872,5520384) | 20260507 00:05:04.546 | INFO | MemInfo: { | 20260507 00:05:04.546 | INFO | "gpu" : " 17 GB", | 20260507 00:05:04.546 | INFO | "huge" : "", | 20260507 00:05:04.546 | INFO | "huge_shared" : " 27 GB", | 20260507 00:05:04.546 | INFO | "malloc" : " 14 GB", | 20260507 00:05:04.546 | INFO | "pinned" : " 29 GB" | 20260507 00:05:04.546 | INFO | } | 20260507 00:05:04.546 | INFO | Perf report for the last 5.13861 seconds: | 20260507 00:05:04.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:05:04.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:04.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:04.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:04.546 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 00:05:04.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:04.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:04.546 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260507 00:05:04.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:04.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:04.546 | INFO | Uploaded 36GiB to GPU in 2.05908s (17.4835GiB/s) | 20260507 00:05:04.546 | INFO | Downloaded 34GiB to GPU in 1.95081s (17.4287GiB/s) | 20260507 00:05:09.690 | INFO | Analyzing chunk frames = [5520384,5520896) | 20260507 00:05:09.690 | INFO | MemInfo: { | 20260507 00:05:09.690 | INFO | "gpu" : " 17 GB", | 20260507 00:05:09.690 | INFO | "huge" : "", | 20260507 00:05:09.690 | INFO | "huge_shared" : " 27 GB", | 20260507 00:05:09.690 | INFO | "malloc" : " 14 GB", | 20260507 00:05:09.690 | INFO | "pinned" : " 29 GB" | 20260507 00:05:09.690 | INFO | } | 20260507 00:05:09.690 | INFO | Perf report for the last 5.14426 seconds: | 20260507 00:05:09.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:05:09.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:09.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:09.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:09.690 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 00:05:09.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:09.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:09.690 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 00:05:09.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:09.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:09.691 | INFO | Uploaded 36GiB to GPU in 1.96026s (18.3649GiB/s) | 20260507 00:05:09.691 | INFO | Downloaded 34GiB to GPU in 1.99544s (17.0388GiB/s) | 20260507 00:05:14.826 | INFO | Analyzing chunk frames = [5520896,5521408) | 20260507 00:05:14.826 | INFO | MemInfo: { | 20260507 00:05:14.826 | INFO | "gpu" : " 17 GB", | 20260507 00:05:14.826 | INFO | "huge" : "", | 20260507 00:05:14.826 | INFO | "huge_shared" : " 27 GB", | 20260507 00:05:14.826 | INFO | "malloc" : " 14 GB", | 20260507 00:05:14.826 | INFO | "pinned" : " 29 GB" | 20260507 00:05:14.826 | INFO | } | 20260507 00:05:14.826 | INFO | Perf report for the last 5.13561 seconds: | 20260507 00:05:14.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:05:14.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:14.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:14.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:14.826 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 00:05:14.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:14.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:14.826 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:05:14.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:14.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:14.826 | INFO | Uploaded 36GiB to GPU in 1.97426s (18.2347GiB/s) | 20260507 00:05:14.826 | INFO | Downloaded 34GiB to GPU in 1.99146s (17.0729GiB/s) | 20260507 00:05:14.826 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:05:14.826 | INFO | Total mem usage: 18 GiB | 20260507 00:05:14.826 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:05:14.826 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:05:14.826 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:05:14.826 | INFO | High Water Breakdown: | 20260507 00:05:14.826 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:05:14.826 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:05:14.826 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:05:14.826 | INFO | ------------------------------------------ | 20260507 00:05:14.826 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:05:14.826 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:05:14.827 | INFO | Total mem usage: 26.5 GiB | 20260507 00:05:14.827 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:05:14.827 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:05:14.827 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:05:14.827 | INFO | High Water Breakdown: | 20260507 00:05:14.827 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:05:14.827 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:05:14.827 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:05:14.827 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:05:14.827 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:05:14.827 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:05:14.827 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:05:14.827 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:05:14.827 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:05:14.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:05:14.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:05:14.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:05:14.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:05:14.827 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:05:14.827 | INFO | ------------------------------------------ | 20260507 00:05:14.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:05:14.827 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:05:14.827 | INFO | Total mem usage: 25.6 GiB | 20260507 00:05:14.827 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:05:14.827 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:05:14.827 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:05:14.827 | INFO | High Water Breakdown: | 20260507 00:05:14.827 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:05:14.827 | INFO | ------------------------------------------ | 20260507 00:05:14.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:05:14.827 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:05:14.827 | INFO | Total mem usage: 15.7 GiB | 20260507 00:05:14.827 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:05:14.827 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:05:14.827 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 00:05:14.827 | INFO | High Water Breakdown: | 20260507 00:05:14.827 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:05:14.827 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:05:14.827 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:05:14.827 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:05:14.827 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:05:14.827 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:05:14.827 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:05:14.827 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:05:14.827 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:05:14.827 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:05:14.827 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:05:14.827 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:05:14.827 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:05:14.827 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:05:14.827 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:05:14.827 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:05:14.827 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:05:14.827 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:05:14.827 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:05:14.827 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:05:14.827 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:05:14.827 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:05:14.827 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:05:14.827 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:05:14.827 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:05:14.827 | INFO | ------------------------------------------ | 20260507 00:05:14.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:05:14.827 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:05:14.827 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:05:14.827 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:05:14.827 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.60e+01, 1.34e+01 / 1.94e+00 | 20260507 00:05:14.827 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.01e-01, 1.00e-02 / 4.87e-03 | 20260507 00:05:14.827 | INFO | PerfReport: 11.66%, Baseline -- 3.61e+00 / 4.38e+00, 5.35e-01 / 3.25e-01 | 20260507 00:05:14.827 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.78e-01 / 1.31e-01 | 20260507 00:05:14.827 | INFO | PerfReport: 16.06%, DME -- 3.98e+01 / 4.45e+01, 4.11e+00 / 2.40e+00 | 20260507 00:05:14.827 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.78e+01 / 2.11e+01, 1.33e+00 / 1.96e+00 | 20260507 00:05:14.827 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.19e+00 / 2.72e+00, 3.80e-01 / 2.41e-01 | 20260507 00:05:14.827 | INFO | PerfReport: 3.17%, Metrics -- 9.82e-01 / 1.29e+00, 2.63e-01 / 1.39e-01 | 20260507 00:05:14.827 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.34e-01 / 3.84e-01, 5.35e-02 / 2.14e-02 | 20260507 00:05:14.827 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:05:14.827 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:05:14.827 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:05:14.827 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260507 00:05:14.827 | INFO | PerfReport: 21.54%, Upload -- 2.08e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260507 00:05:14.827 | INFO | PerfReport: 21.32%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 00:05:14.827 | INFO | PerfReport: 32.05%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 00:05:14.827 | INFO | PerfReport: 1.32%, LockUpload -- 1.27e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 00:05:14.827 | INFO | PerfReport: 1.27%, LockDownload -- 1.22e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260507 00:05:14.827 | INFO | PerfReport: 22.48%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 00:05:14.827 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.61e-02 / 0.00e+00 | 20260507 00:05:14.924 | INFO | Starting a round of Baz IO | 20260507 00:05:18.044 | INFO | Populating RTMetrics report | 20260507 00:05:18.044 | INFO | RTMetrics report populated | 20260507 00:05:18.044 | INFO | Opening tmp RTMetrics json file | 20260507 00:05:18.045 | INFO | Serializing RTMetrics to json | 20260507 00:05:18.048 | INFO | Serialization done, swapping tmp file now... | 20260507 00:05:18.066 | INFO | RTMetrics done | 20260507 00:05:18.138 | INFO | Finished a round of Baz IO: Wrote 1.861 GiB in 3.214 seconds | 20260507 00:05:18.138 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.35% / 0.00% / 66.65% | 20260507 00:05:18.138 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.368293% / 0% / 5.96046e-05% | 20260507 00:05:18.138 | INFO | Baz IO overall: | 20260507 00:05:18.138 | INFO | BytesWritten : 2882637364774 | 20260507 00:05:18.138 | INFO | NumEvents : 1389091259960 | 20260507 00:05:19.961 | INFO | Analyzing chunk frames = [5521408,5521920) | 20260507 00:05:19.961 | INFO | MemInfo: { | 20260507 00:05:19.961 | INFO | "gpu" : " 17 GB", | 20260507 00:05:19.961 | INFO | "huge" : "", | 20260507 00:05:19.961 | INFO | "huge_shared" : " 27 GB", | 20260507 00:05:19.961 | INFO | "malloc" : " 14 GB", | 20260507 00:05:19.961 | INFO | "pinned" : " 29 GB" | 20260507 00:05:19.961 | INFO | } | 20260507 00:05:19.961 | INFO | Perf report for the last 5.13538 seconds: | 20260507 00:05:19.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:05:19.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:19.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:19.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:19.962 | INFO | ANALYSIS : 62.1%, 3.0, 96.4ms, 0 | 20260507 00:05:19.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:19.962 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 00:05:19.962 | INFO | PRE_HQ : 59.6%, 1.8, 56.7ms, 0 | 20260507 00:05:19.962 | INFO | BAZWRITER : 62.2%, 3.4, 108.5ms, 0 | 20260507 00:05:19.962 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:05:19.962 | INFO | Uploaded 36GiB to GPU in 2.03484s (17.6918GiB/s) | 20260507 00:05:19.962 | INFO | Downloaded 38GiB to GPU in 2.18658s (17.3787GiB/s) | 20260507 00:05:25.098 | INFO | Analyzing chunk frames = [5521920,5522432) | 20260507 00:05:25.098 | INFO | MemInfo: { | 20260507 00:05:25.098 | INFO | "gpu" : " 17 GB", | 20260507 00:05:25.098 | INFO | "huge" : "", | 20260507 00:05:25.098 | INFO | "huge_shared" : " 27 GB", | 20260507 00:05:25.098 | INFO | "malloc" : " 14 GB", | 20260507 00:05:25.098 | INFO | "pinned" : " 29 GB" | 20260507 00:05:25.098 | INFO | } | 20260507 00:05:25.098 | INFO | Perf report for the last 5.13678 seconds: | 20260507 00:05:25.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:05:25.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:25.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:25.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:25.098 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:05:25.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:25.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:25.099 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:05:25.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:25.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:25.099 | INFO | Uploaded 36GiB to GPU in 1.99315s (18.0618GiB/s) | 20260507 00:05:25.099 | INFO | Downloaded 34GiB to GPU in 1.93197s (17.5986GiB/s) | 20260507 00:05:30.238 | INFO | Analyzing chunk frames = [5522432,5522944) | 20260507 00:05:30.238 | INFO | MemInfo: { | 20260507 00:05:30.238 | INFO | "gpu" : " 17 GB", | 20260507 00:05:30.238 | INFO | "huge" : "", | 20260507 00:05:30.238 | INFO | "huge_shared" : " 27 GB", | 20260507 00:05:30.238 | INFO | "malloc" : " 14 GB", | 20260507 00:05:30.238 | INFO | "pinned" : " 29 GB" | 20260507 00:05:30.238 | INFO | } | 20260507 00:05:30.238 | INFO | Perf report for the last 5.14015 seconds: | 20260507 00:05:30.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:05:30.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:30.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:30.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:30.238 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:05:30.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:30.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:30.239 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 00:05:30.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:30.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:30.239 | INFO | Uploaded 36GiB to GPU in 1.97806s (18.1997GiB/s) | 20260507 00:05:30.239 | INFO | Downloaded 34GiB to GPU in 1.9316s (17.6019GiB/s) | 20260507 00:05:35.383 | INFO | Analyzing chunk frames = [5522944,5523456) | 20260507 00:05:35.383 | INFO | MemInfo: { | 20260507 00:05:35.383 | INFO | "gpu" : " 17 GB", | 20260507 00:05:35.383 | INFO | "huge" : "", | 20260507 00:05:35.383 | INFO | "huge_shared" : " 27 GB", | 20260507 00:05:35.383 | INFO | "malloc" : " 14 GB", | 20260507 00:05:35.383 | INFO | "pinned" : " 29 GB" | 20260507 00:05:35.383 | INFO | } | 20260507 00:05:35.383 | INFO | Perf report for the last 5.14486 seconds: | 20260507 00:05:35.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:05:35.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:35.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:35.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:35.383 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260507 00:05:35.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:35.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:35.384 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 00:05:35.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:35.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:35.384 | INFO | Uploaded 36GiB to GPU in 1.99496s (18.0455GiB/s) | 20260507 00:05:35.384 | INFO | Downloaded 34GiB to GPU in 1.90922s (17.8084GiB/s) | 20260507 00:05:40.523 | INFO | Analyzing chunk frames = [5523456,5523968) | 20260507 00:05:40.523 | INFO | MemInfo: { | 20260507 00:05:40.523 | INFO | "gpu" : " 17 GB", | 20260507 00:05:40.523 | INFO | "huge" : "", | 20260507 00:05:40.523 | INFO | "huge_shared" : " 27 GB", | 20260507 00:05:40.523 | INFO | "malloc" : " 14 GB", | 20260507 00:05:40.523 | INFO | "pinned" : " 29 GB" | 20260507 00:05:40.523 | INFO | } | 20260507 00:05:40.523 | INFO | Perf report for the last 5.1399 seconds: | 20260507 00:05:40.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:05:40.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:40.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:40.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:40.523 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 00:05:40.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:40.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:40.523 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:05:40.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:40.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:40.523 | INFO | Uploaded 36GiB to GPU in 1.99369s (18.057GiB/s) | 20260507 00:05:40.523 | INFO | Downloaded 34GiB to GPU in 1.93767s (17.5468GiB/s) | 20260507 00:05:45.662 | INFO | Analyzing chunk frames = [5523968,5524480) | 20260507 00:05:45.662 | INFO | MemInfo: { | 20260507 00:05:45.662 | INFO | "gpu" : " 17 GB", | 20260507 00:05:45.662 | INFO | "huge" : "", | 20260507 00:05:45.662 | INFO | "huge_shared" : " 27 GB", | 20260507 00:05:45.662 | INFO | "malloc" : " 14 GB", | 20260507 00:05:45.662 | INFO | "pinned" : " 29 GB" | 20260507 00:05:45.662 | INFO | } | 20260507 00:05:45.662 | INFO | Perf report for the last 5.13901 seconds: | 20260507 00:05:45.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:05:45.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:45.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:45.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:45.662 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 00:05:45.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:45.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:45.662 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:05:45.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:45.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:45.662 | INFO | Uploaded 36GiB to GPU in 2.0274s (17.7568GiB/s) | 20260507 00:05:45.662 | INFO | Downloaded 34GiB to GPU in 1.94984s (17.4374GiB/s) | 20260507 00:05:50.797 | INFO | Analyzing chunk frames = [5524480,5524992) | 20260507 00:05:50.797 | INFO | MemInfo: { | 20260507 00:05:50.797 | INFO | "gpu" : " 17 GB", | 20260507 00:05:50.797 | INFO | "huge" : "", | 20260507 00:05:50.797 | INFO | "huge_shared" : " 27 GB", | 20260507 00:05:50.797 | INFO | "malloc" : " 14 GB", | 20260507 00:05:50.797 | INFO | "pinned" : " 29 GB" | 20260507 00:05:50.797 | INFO | } | 20260507 00:05:50.797 | INFO | Perf report for the last 5.13504 seconds: | 20260507 00:05:50.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:05:50.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:50.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:50.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:50.797 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:05:50.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:50.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:50.797 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:05:50.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:50.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:50.797 | INFO | Uploaded 36GiB to GPU in 1.9881s (18.1077GiB/s) | 20260507 00:05:50.798 | INFO | Downloaded 34GiB to GPU in 1.99397s (17.0514GiB/s) | 20260507 00:05:55.934 | INFO | Analyzing chunk frames = [5524992,5525504) | 20260507 00:05:55.934 | INFO | MemInfo: { | 20260507 00:05:55.934 | INFO | "gpu" : " 17 GB", | 20260507 00:05:55.934 | INFO | "huge" : "", | 20260507 00:05:55.934 | INFO | "huge_shared" : " 27 GB", | 20260507 00:05:55.934 | INFO | "malloc" : " 14 GB", | 20260507 00:05:55.934 | INFO | "pinned" : " 29 GB" | 20260507 00:05:55.934 | INFO | } | 20260507 00:05:55.934 | INFO | Perf report for the last 5.13695 seconds: | 20260507 00:05:55.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:05:55.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:55.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:55.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:55.934 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 00:05:55.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:55.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:55.934 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 00:05:55.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:55.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:05:55.934 | INFO | Uploaded 36GiB to GPU in 1.96963s (18.2775GiB/s) | 20260507 00:05:55.934 | INFO | Downloaded 34GiB to GPU in 1.99693s (17.0262GiB/s) | 20260507 00:05:56.033 | INFO | Starting a round of Baz IO | 20260507 00:05:59.162 | INFO | Populating RTMetrics report | 20260507 00:05:59.162 | INFO | RTMetrics report populated | 20260507 00:05:59.162 | INFO | Opening tmp RTMetrics json file | 20260507 00:05:59.163 | INFO | Serializing RTMetrics to json | 20260507 00:05:59.173 | INFO | Serialization done, swapping tmp file now... | 20260507 00:05:59.183 | INFO | RTMetrics done | 20260507 00:05:59.263 | INFO | Finished a round of Baz IO: Wrote 1.861 GiB in 3.23 seconds | 20260507 00:05:59.263 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.35% / 0.00% / 66.65% | 20260507 00:05:59.263 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.3679% / 3.97364e-06% / 5.16574e-05% | 20260507 00:05:59.263 | INFO | Baz IO overall: | 20260507 00:05:59.263 | INFO | BytesWritten : 2884635774212 | 20260507 00:05:59.263 | INFO | NumEvents : 1390041244387 | 20260507 00:06:01.075 | INFO | Analyzing chunk frames = [5525504,5526016) | 20260507 00:06:01.075 | INFO | MemInfo: { | 20260507 00:06:01.075 | INFO | "gpu" : " 17 GB", | 20260507 00:06:01.075 | INFO | "huge" : "", | 20260507 00:06:01.075 | INFO | "huge_shared" : " 27 GB", | 20260507 00:06:01.075 | INFO | "malloc" : " 14 GB", | 20260507 00:06:01.075 | INFO | "pinned" : " 29 GB" | 20260507 00:06:01.075 | INFO | } | 20260507 00:06:01.075 | INFO | Perf report for the last 5.14089 seconds: | 20260507 00:06:01.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:06:01.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:01.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:01.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:01.075 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 00:06:01.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:01.075 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260507 00:06:01.075 | INFO | PRE_HQ : 60.1%, 1.9, 59.5ms, 0 | 20260507 00:06:01.075 | INFO | BAZWRITER : 62.4%, 3.5, 112.0ms, 0 | 20260507 00:06:01.075 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:06:01.075 | INFO | Uploaded 36GiB to GPU in 2.00315s (17.9717GiB/s) | 20260507 00:06:01.075 | INFO | Downloaded 38GiB to GPU in 2.17589s (17.4642GiB/s) | 20260507 00:06:06.212 | INFO | Analyzing chunk frames = [5526016,5526528) | 20260507 00:06:06.212 | INFO | MemInfo: { | 20260507 00:06:06.212 | INFO | "gpu" : " 17 GB", | 20260507 00:06:06.212 | INFO | "huge" : "", | 20260507 00:06:06.212 | INFO | "huge_shared" : " 27 GB", | 20260507 00:06:06.212 | INFO | "malloc" : " 14 GB", | 20260507 00:06:06.212 | INFO | "pinned" : " 29 GB" | 20260507 00:06:06.212 | INFO | } | 20260507 00:06:06.212 | INFO | Perf report for the last 5.13674 seconds: | 20260507 00:06:06.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:06:06.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:06.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:06.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:06.212 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:06:06.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:06.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:06.212 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:06:06.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:06.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:06.212 | INFO | Uploaded 36GiB to GPU in 1.98586s (18.1282GiB/s) | 20260507 00:06:06.212 | INFO | Downloaded 34GiB to GPU in 1.92892s (17.6265GiB/s) | 20260507 00:06:11.356 | INFO | Analyzing chunk frames = [5526528,5527040) | 20260507 00:06:11.356 | INFO | MemInfo: { | 20260507 00:06:11.356 | INFO | "gpu" : " 17 GB", | 20260507 00:06:11.356 | INFO | "huge" : "", | 20260507 00:06:11.356 | INFO | "huge_shared" : " 27 GB", | 20260507 00:06:11.356 | INFO | "malloc" : " 14 GB", | 20260507 00:06:11.356 | INFO | "pinned" : " 29 GB" | 20260507 00:06:11.356 | INFO | } | 20260507 00:06:11.356 | INFO | Perf report for the last 5.14457 seconds: | 20260507 00:06:11.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:06:11.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:11.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:11.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:11.357 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 00:06:11.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:11.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:11.357 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:06:11.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:11.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:11.357 | INFO | Uploaded 36GiB to GPU in 1.97327s (18.2438GiB/s) | 20260507 00:06:11.357 | INFO | Downloaded 34GiB to GPU in 1.92953s (17.6209GiB/s) | 20260507 00:06:16.499 | INFO | Analyzing chunk frames = [5527040,5527552) | 20260507 00:06:16.499 | INFO | MemInfo: { | 20260507 00:06:16.499 | INFO | "gpu" : " 17 GB", | 20260507 00:06:16.499 | INFO | "huge" : "", | 20260507 00:06:16.499 | INFO | "huge_shared" : " 27 GB", | 20260507 00:06:16.499 | INFO | "malloc" : " 14 GB", | 20260507 00:06:16.499 | INFO | "pinned" : " 29 GB" | 20260507 00:06:16.499 | INFO | } | 20260507 00:06:16.499 | INFO | Perf report for the last 5.14282 seconds: | 20260507 00:06:16.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:06:16.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:16.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:16.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:16.499 | INFO | ANALYSIS : 60.0%, 3.0, 93.2ms, 0 | 20260507 00:06:16.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:16.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:16.499 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:06:16.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:16.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:16.499 | INFO | Uploaded 36GiB to GPU in 2.01046s (17.9064GiB/s) | 20260507 00:06:16.500 | INFO | Downloaded 34GiB to GPU in 1.90986s (17.8024GiB/s) | 20260507 00:06:21.641 | INFO | Analyzing chunk frames = [5527552,5528064) | 20260507 00:06:21.641 | INFO | MemInfo: { | 20260507 00:06:21.641 | INFO | "gpu" : " 17 GB", | 20260507 00:06:21.641 | INFO | "huge" : "", | 20260507 00:06:21.641 | INFO | "huge_shared" : " 27 GB", | 20260507 00:06:21.642 | INFO | "malloc" : " 14 GB", | 20260507 00:06:21.642 | INFO | "pinned" : " 29 GB" | 20260507 00:06:21.642 | INFO | } | 20260507 00:06:21.642 | INFO | Perf report for the last 5.14237 seconds: | 20260507 00:06:21.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:06:21.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:21.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:21.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:21.642 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 00:06:21.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:21.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:21.642 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 00:06:21.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:21.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:21.642 | INFO | Uploaded 36GiB to GPU in 1.98568s (18.1299GiB/s) | 20260507 00:06:21.642 | INFO | Downloaded 34GiB to GPU in 1.94898s (17.4451GiB/s) | 20260507 00:06:26.780 | INFO | Analyzing chunk frames = [5528064,5528576) | 20260507 00:06:26.780 | INFO | MemInfo: { | 20260507 00:06:26.780 | INFO | "gpu" : " 17 GB", | 20260507 00:06:26.780 | INFO | "huge" : "", | 20260507 00:06:26.780 | INFO | "huge_shared" : " 27 GB", | 20260507 00:06:26.780 | INFO | "malloc" : " 14 GB", | 20260507 00:06:26.780 | INFO | "pinned" : " 29 GB" | 20260507 00:06:26.780 | INFO | } | 20260507 00:06:26.780 | INFO | Perf report for the last 5.13805 seconds: | 20260507 00:06:26.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:06:26.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:26.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:26.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:26.780 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 00:06:26.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:26.780 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:26.780 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:06:26.780 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:26.780 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:26.780 | INFO | Uploaded 36GiB to GPU in 2.04046s (17.6431GiB/s) | 20260507 00:06:26.780 | INFO | Downloaded 34GiB to GPU in 1.95663s (17.3768GiB/s) | 20260507 00:06:31.910 | INFO | Analyzing chunk frames = [5528576,5529088) | 20260507 00:06:31.910 | INFO | MemInfo: { | 20260507 00:06:31.910 | INFO | "gpu" : " 17 GB", | 20260507 00:06:31.910 | INFO | "huge" : "", | 20260507 00:06:31.910 | INFO | "huge_shared" : " 27 GB", | 20260507 00:06:31.910 | INFO | "malloc" : " 14 GB", | 20260507 00:06:31.910 | INFO | "pinned" : " 29 GB" | 20260507 00:06:31.910 | INFO | } | 20260507 00:06:31.910 | INFO | Perf report for the last 5.13022 seconds: | 20260507 00:06:31.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:06:31.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:31.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:31.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:31.910 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 00:06:31.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:31.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:31.910 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 00:06:31.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:31.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:31.910 | INFO | Uploaded 36GiB to GPU in 1.96206s (18.348GiB/s) | 20260507 00:06:31.910 | INFO | Downloaded 34GiB to GPU in 1.99468s (17.0453GiB/s) | 20260507 00:06:37.044 | INFO | Analyzing chunk frames = [5529088,5529600) | 20260507 00:06:37.044 | INFO | MemInfo: { | 20260507 00:06:37.044 | INFO | "gpu" : " 17 GB", | 20260507 00:06:37.044 | INFO | "huge" : "", | 20260507 00:06:37.044 | INFO | "huge_shared" : " 27 GB", | 20260507 00:06:37.044 | INFO | "malloc" : " 14 GB", | 20260507 00:06:37.044 | INFO | "pinned" : " 29 GB" | 20260507 00:06:37.044 | INFO | } | 20260507 00:06:37.044 | INFO | Perf report for the last 5.13392 seconds: | 20260507 00:06:37.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:06:37.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:37.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:37.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:37.044 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260507 00:06:37.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:37.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:37.044 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:06:37.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:37.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:37.044 | INFO | Uploaded 36GiB to GPU in 1.97244s (18.2515GiB/s) | 20260507 00:06:37.044 | INFO | Downloaded 34GiB to GPU in 2.00536s (16.9545GiB/s) | 20260507 00:06:37.044 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:06:37.044 | INFO | Total mem usage: 18 GiB | 20260507 00:06:37.044 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:06:37.044 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:06:37.044 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:06:37.044 | INFO | High Water Breakdown: | 20260507 00:06:37.044 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:06:37.044 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:06:37.044 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:06:37.044 | INFO | ------------------------------------------ | 20260507 00:06:37.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:06:37.044 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:06:37.044 | INFO | Total mem usage: 26.5 GiB | 20260507 00:06:37.044 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:06:37.044 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:06:37.044 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:06:37.044 | INFO | High Water Breakdown: | 20260507 00:06:37.044 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:06:37.044 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:06:37.044 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:06:37.044 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:06:37.044 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:06:37.044 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:06:37.044 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:06:37.044 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:06:37.044 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:06:37.044 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:06:37.044 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:06:37.044 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:06:37.044 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:06:37.045 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:06:37.045 | INFO | ------------------------------------------ | 20260507 00:06:37.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:06:37.045 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:06:37.045 | INFO | Total mem usage: 25.6 GiB | 20260507 00:06:37.045 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:06:37.045 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:06:37.045 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:06:37.045 | INFO | High Water Breakdown: | 20260507 00:06:37.045 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:06:37.045 | INFO | ------------------------------------------ | 20260507 00:06:37.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:06:37.045 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:06:37.045 | INFO | Total mem usage: 15.7 GiB | 20260507 00:06:37.045 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:06:37.045 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:06:37.045 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 00:06:37.045 | INFO | High Water Breakdown: | 20260507 00:06:37.045 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:06:37.045 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:06:37.045 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:06:37.045 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:06:37.045 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:06:37.045 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:06:37.045 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:06:37.045 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:06:37.045 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:06:37.045 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:06:37.045 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:06:37.045 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:06:37.045 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:06:37.045 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:06:37.045 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:06:37.045 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:06:37.045 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:06:37.045 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:06:37.045 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:06:37.045 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:06:37.045 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:06:37.045 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:06:37.045 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:06:37.045 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:06:37.045 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:06:37.045 | INFO | ------------------------------------------ | 20260507 00:06:37.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:06:37.045 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:06:37.045 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:06:37.045 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:06:37.045 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.53e+01, 1.36e+01 / 2.30e+00 | 20260507 00:06:37.045 | INFO | PerfReport: 0.30%, AggregateStats -- 9.14e-02 / 1.00e-01, 1.01e-02 / 6.89e-03 | 20260507 00:06:37.045 | INFO | PerfReport: 11.66%, Baseline -- 3.61e+00 / 4.28e+00, 5.35e-01 / 3.89e-01 | 20260507 00:06:37.045 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.44e+00, 1.77e-01 / 1.28e-01 | 20260507 00:06:37.045 | INFO | PerfReport: 16.03%, DME -- 3.97e+01 / 4.40e+01, 3.97e+00 / 2.30e+00 | 20260507 00:06:37.045 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.36e+00 / 1.73e+00 | 20260507 00:06:37.045 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.18e+00 / 2.71e+00, 3.79e-01 / 1.99e-01 | 20260507 00:06:37.045 | INFO | PerfReport: 3.17%, Metrics -- 9.83e-01 / 1.30e+00, 2.64e-01 / 1.43e-01 | 20260507 00:06:37.045 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.35e-01 / 3.81e-01, 5.34e-02 / 1.71e-02 | 20260507 00:06:37.045 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:06:37.045 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:06:37.045 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:06:37.045 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 00:06:37.045 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 6.92e+00 / 0.00e+00 | 20260507 00:06:37.045 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260507 00:06:37.045 | INFO | PerfReport: 32.07%, Compute -- 3.10e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260507 00:06:37.045 | INFO | PerfReport: 1.22%, LockUpload -- 1.18e+00 / 0.00e+00, 4.51e+00 / 0.00e+00 | 20260507 00:06:37.045 | INFO | PerfReport: 1.36%, LockDownload -- 1.32e+00 / 0.00e+00, 3.24e+00 / 0.00e+00 | 20260507 00:06:37.045 | INFO | PerfReport: 22.46%, LockCompute -- 2.17e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 00:06:37.045 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260507 00:06:37.143 | INFO | Starting a round of Baz IO | 20260507 00:06:40.274 | INFO | Populating RTMetrics report | 20260507 00:06:40.274 | INFO | RTMetrics report populated | 20260507 00:06:40.274 | INFO | Opening tmp RTMetrics json file | 20260507 00:06:40.276 | INFO | Serializing RTMetrics to json | 20260507 00:06:40.294 | INFO | Serialization done, swapping tmp file now... | 20260507 00:06:40.295 | INFO | RTMetrics done | 20260507 00:06:40.373 | INFO | Finished a round of Baz IO: Wrote 1.861 GiB in 3.23 seconds | 20260507 00:06:40.373 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.35% / 0.00% / 66.65% | 20260507 00:06:40.373 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.369573% / 0% / 7.54992e-05% | 20260507 00:06:40.373 | INFO | Baz IO overall: | 20260507 00:06:40.373 | INFO | BytesWritten : 2886634273920 | 20260507 00:06:40.373 | INFO | NumEvents : 1390991820324 | 20260507 00:06:42.194 | INFO | Analyzing chunk frames = [5529600,5530112) | 20260507 00:06:42.194 | INFO | MemInfo: { | 20260507 00:06:42.194 | INFO | "gpu" : " 17 GB", | 20260507 00:06:42.194 | INFO | "huge" : "", | 20260507 00:06:42.194 | INFO | "huge_shared" : " 27 GB", | 20260507 00:06:42.194 | INFO | "malloc" : " 14 GB", | 20260507 00:06:42.194 | INFO | "pinned" : " 29 GB" | 20260507 00:06:42.194 | INFO | } | 20260507 00:06:42.194 | INFO | Perf report for the last 5.15043 seconds: | 20260507 00:06:42.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:06:42.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:42.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:42.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:42.194 | INFO | ANALYSIS : 62.2%, 3.0, 96.9ms, 0 | 20260507 00:06:42.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:42.195 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260507 00:06:42.195 | INFO | PRE_HQ : 59.4%, 1.8, 56.6ms, 0 | 20260507 00:06:42.195 | INFO | BAZWRITER : 62.3%, 3.4, 108.9ms, 0 | 20260507 00:06:42.195 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:06:42.195 | INFO | Uploaded 36GiB to GPU in 2.02998s (17.7342GiB/s) | 20260507 00:06:42.195 | INFO | Downloaded 38GiB to GPU in 2.18311s (17.4064GiB/s) | 20260507 00:06:47.333 | INFO | Analyzing chunk frames = [5530112,5530624) | 20260507 00:06:47.333 | INFO | MemInfo: { | 20260507 00:06:47.333 | INFO | "gpu" : " 17 GB", | 20260507 00:06:47.333 | INFO | "huge" : "", | 20260507 00:06:47.333 | INFO | "huge_shared" : " 27 GB", | 20260507 00:06:47.333 | INFO | "malloc" : " 14 GB", | 20260507 00:06:47.333 | INFO | "pinned" : " 29 GB" | 20260507 00:06:47.333 | INFO | } | 20260507 00:06:47.333 | INFO | Perf report for the last 5.13903 seconds: | 20260507 00:06:47.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:06:47.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:47.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:47.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:47.333 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:06:47.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:47.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:47.333 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:06:47.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:47.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:47.334 | INFO | Uploaded 36GiB to GPU in 1.99647s (18.0318GiB/s) | 20260507 00:06:47.334 | INFO | Downloaded 34GiB to GPU in 1.92405s (17.6711GiB/s) | 20260507 00:06:52.473 | INFO | Analyzing chunk frames = [5530624,5531136) | 20260507 00:06:52.473 | INFO | MemInfo: { | 20260507 00:06:52.473 | INFO | "gpu" : " 17 GB", | 20260507 00:06:52.473 | INFO | "huge" : "", | 20260507 00:06:52.473 | INFO | "huge_shared" : " 27 GB", | 20260507 00:06:52.473 | INFO | "malloc" : " 14 GB", | 20260507 00:06:52.473 | INFO | "pinned" : " 29 GB" | 20260507 00:06:52.473 | INFO | } | 20260507 00:06:52.473 | INFO | Perf report for the last 5.13954 seconds: | 20260507 00:06:52.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:06:52.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:52.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:52.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:52.473 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:06:52.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:52.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:52.473 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:06:52.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:52.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:52.473 | INFO | Uploaded 36GiB to GPU in 1.99988s (18.001GiB/s) | 20260507 00:06:52.473 | INFO | Downloaded 34GiB to GPU in 1.9331s (17.5883GiB/s) | 20260507 00:06:57.611 | INFO | Analyzing chunk frames = [5531136,5531648) | 20260507 00:06:57.611 | INFO | MemInfo: { | 20260507 00:06:57.611 | INFO | "gpu" : " 17 GB", | 20260507 00:06:57.611 | INFO | "huge" : "", | 20260507 00:06:57.611 | INFO | "huge_shared" : " 27 GB", | 20260507 00:06:57.611 | INFO | "malloc" : " 14 GB", | 20260507 00:06:57.611 | INFO | "pinned" : " 29 GB" | 20260507 00:06:57.611 | INFO | } | 20260507 00:06:57.611 | INFO | Perf report for the last 5.1378 seconds: | 20260507 00:06:57.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:06:57.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:57.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:57.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:57.611 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 00:06:57.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:57.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:57.611 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:06:57.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:57.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:06:57.611 | INFO | Uploaded 36GiB to GPU in 1.98492s (18.1368GiB/s) | 20260507 00:06:57.611 | INFO | Downloaded 34GiB to GPU in 1.90274s (17.8689GiB/s) | 20260507 00:07:02.748 | INFO | Analyzing chunk frames = [5531648,5532160) | 20260507 00:07:02.748 | INFO | MemInfo: { | 20260507 00:07:02.748 | INFO | "gpu" : " 17 GB", | 20260507 00:07:02.748 | INFO | "huge" : "", | 20260507 00:07:02.748 | INFO | "huge_shared" : " 27 GB", | 20260507 00:07:02.748 | INFO | "malloc" : " 14 GB", | 20260507 00:07:02.748 | INFO | "pinned" : " 29 GB" | 20260507 00:07:02.748 | INFO | } | 20260507 00:07:02.748 | INFO | Perf report for the last 5.13701 seconds: | 20260507 00:07:02.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:07:02.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:02.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:02.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:02.748 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 00:07:02.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:02.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:02.748 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 00:07:02.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:02.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:02.748 | INFO | Uploaded 36GiB to GPU in 1.98981s (18.0922GiB/s) | 20260507 00:07:02.748 | INFO | Downloaded 34GiB to GPU in 1.93081s (17.6092GiB/s) | 20260507 00:07:07.885 | INFO | Analyzing chunk frames = [5532160,5532672) | 20260507 00:07:07.885 | INFO | MemInfo: { | 20260507 00:07:07.885 | INFO | "gpu" : " 17 GB", | 20260507 00:07:07.885 | INFO | "huge" : "", | 20260507 00:07:07.885 | INFO | "huge_shared" : " 27 GB", | 20260507 00:07:07.885 | INFO | "malloc" : " 14 GB", | 20260507 00:07:07.885 | INFO | "pinned" : " 29 GB" | 20260507 00:07:07.885 | INFO | } | 20260507 00:07:07.885 | INFO | Perf report for the last 5.13769 seconds: | 20260507 00:07:07.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:07:07.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:07.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:07.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:07.886 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 00:07:07.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:07.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:07.886 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 00:07:07.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:07.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:07.886 | INFO | Uploaded 36GiB to GPU in 2.0257s (17.7717GiB/s) | 20260507 00:07:07.886 | INFO | Downloaded 34GiB to GPU in 1.93571s (17.5646GiB/s) | 20260507 00:07:13.022 | INFO | Analyzing chunk frames = [5532672,5533184) | 20260507 00:07:13.022 | INFO | MemInfo: { | 20260507 00:07:13.022 | INFO | "gpu" : " 17 GB", | 20260507 00:07:13.022 | INFO | "huge" : "", | 20260507 00:07:13.022 | INFO | "huge_shared" : " 27 GB", | 20260507 00:07:13.022 | INFO | "malloc" : " 14 GB", | 20260507 00:07:13.022 | INFO | "pinned" : " 29 GB" | 20260507 00:07:13.022 | INFO | } | 20260507 00:07:13.022 | INFO | Perf report for the last 5.1368 seconds: | 20260507 00:07:13.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:07:13.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:13.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:13.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:13.022 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260507 00:07:13.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:13.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:13.022 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:07:13.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:13.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:13.022 | INFO | Uploaded 36GiB to GPU in 1.98336s (18.151GiB/s) | 20260507 00:07:13.023 | INFO | Downloaded 34GiB to GPU in 1.99374s (17.0534GiB/s) | 20260507 00:07:18.163 | INFO | Analyzing chunk frames = [5533184,5533696) | 20260507 00:07:18.163 | INFO | MemInfo: { | 20260507 00:07:18.163 | INFO | "gpu" : " 17 GB", | 20260507 00:07:18.163 | INFO | "huge" : "", | 20260507 00:07:18.163 | INFO | "huge_shared" : " 27 GB", | 20260507 00:07:18.163 | INFO | "malloc" : " 14 GB", | 20260507 00:07:18.163 | INFO | "pinned" : " 29 GB" | 20260507 00:07:18.163 | INFO | } | 20260507 00:07:18.163 | INFO | Perf report for the last 5.14053 seconds: | 20260507 00:07:18.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:07:18.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:18.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:18.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:18.163 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:07:18.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:18.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:18.163 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:07:18.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:18.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:18.163 | INFO | Uploaded 36GiB to GPU in 1.97765s (18.2035GiB/s) | 20260507 00:07:18.163 | INFO | Downloaded 34GiB to GPU in 1.99109s (17.0761GiB/s) | 20260507 00:07:18.260 | INFO | Starting a round of Baz IO | 20260507 00:07:21.382 | INFO | Populating RTMetrics report | 20260507 00:07:21.382 | INFO | RTMetrics report populated | 20260507 00:07:21.382 | INFO | Opening tmp RTMetrics json file | 20260507 00:07:21.383 | INFO | Serializing RTMetrics to json | 20260507 00:07:21.384 | INFO | Serialization done, swapping tmp file now... | 20260507 00:07:21.384 | INFO | RTMetrics done | 20260507 00:07:21.473 | INFO | Finished a round of Baz IO: Wrote 1.858 GiB in 3.213 seconds | 20260507 00:07:21.473 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.34% / 0.00% / 66.65% | 20260507 00:07:21.473 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.367634% / 3.97364e-06% / 9.13938e-05% | 20260507 00:07:21.473 | INFO | Baz IO overall: | 20260507 00:07:21.473 | INFO | BytesWritten : 2888629116901 | 20260507 00:07:21.473 | INFO | NumEvents : 1391940289885 | 20260507 00:07:23.305 | INFO | Analyzing chunk frames = [5533696,5534208) | 20260507 00:07:23.305 | INFO | MemInfo: { | 20260507 00:07:23.305 | INFO | "gpu" : " 17 GB", | 20260507 00:07:23.305 | INFO | "huge" : "", | 20260507 00:07:23.305 | INFO | "huge_shared" : " 27 GB", | 20260507 00:07:23.305 | INFO | "malloc" : " 14 GB", | 20260507 00:07:23.305 | INFO | "pinned" : " 29 GB" | 20260507 00:07:23.305 | INFO | } | 20260507 00:07:23.305 | INFO | Perf report for the last 5.14253 seconds: | 20260507 00:07:23.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:07:23.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:23.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:23.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:23.305 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 00:07:23.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:23.305 | INFO | RT_METRICS : 42.2%, 1.0, 22.2ms, 0 | 20260507 00:07:23.305 | INFO | PRE_HQ : 59.6%, 1.8, 56.4ms, 0 | 20260507 00:07:23.305 | INFO | BAZWRITER : 62.0%, 3.4, 107.7ms, 0 | 20260507 00:07:23.305 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:07:23.306 | INFO | Uploaded 36GiB to GPU in 2.02949s (17.7385GiB/s) | 20260507 00:07:23.306 | INFO | Downloaded 38GiB to GPU in 2.17245s (17.4918GiB/s) | 20260507 00:07:28.451 | INFO | Analyzing chunk frames = [5534208,5534720) | 20260507 00:07:28.451 | INFO | MemInfo: { | 20260507 00:07:28.451 | INFO | "gpu" : " 17 GB", | 20260507 00:07:28.451 | INFO | "huge" : "", | 20260507 00:07:28.451 | INFO | "huge_shared" : " 27 GB", | 20260507 00:07:28.451 | INFO | "malloc" : " 14 GB", | 20260507 00:07:28.451 | INFO | "pinned" : " 29 GB" | 20260507 00:07:28.451 | INFO | } | 20260507 00:07:28.451 | INFO | Perf report for the last 5.14607 seconds: | 20260507 00:07:28.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:07:28.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:28.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:28.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:28.452 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:07:28.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:28.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:28.452 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:07:28.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:28.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:28.452 | INFO | Uploaded 36GiB to GPU in 2.00146s (17.9869GiB/s) | 20260507 00:07:28.452 | INFO | Downloaded 34GiB to GPU in 1.9339s (17.581GiB/s) | 20260507 00:07:33.591 | INFO | Analyzing chunk frames = [5534720,5535232) | 20260507 00:07:33.591 | INFO | MemInfo: { | 20260507 00:07:33.591 | INFO | "gpu" : " 17 GB", | 20260507 00:07:33.591 | INFO | "huge" : "", | 20260507 00:07:33.591 | INFO | "huge_shared" : " 27 GB", | 20260507 00:07:33.591 | INFO | "malloc" : " 14 GB", | 20260507 00:07:33.591 | INFO | "pinned" : " 29 GB" | 20260507 00:07:33.591 | INFO | } | 20260507 00:07:33.591 | INFO | Perf report for the last 5.14006 seconds: | 20260507 00:07:33.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:07:33.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:33.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:33.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:33.592 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:07:33.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:33.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:33.592 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:07:33.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:33.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:33.592 | INFO | Uploaded 36GiB to GPU in 2.00192s (17.9827GiB/s) | 20260507 00:07:33.592 | INFO | Downloaded 34GiB to GPU in 1.93919s (17.5331GiB/s) | 20260507 00:07:38.728 | INFO | Analyzing chunk frames = [5535232,5535744) | 20260507 00:07:38.728 | INFO | MemInfo: { | 20260507 00:07:38.728 | INFO | "gpu" : " 17 GB", | 20260507 00:07:38.728 | INFO | "huge" : "", | 20260507 00:07:38.728 | INFO | "huge_shared" : " 27 GB", | 20260507 00:07:38.728 | INFO | "malloc" : " 14 GB", | 20260507 00:07:38.728 | INFO | "pinned" : " 29 GB" | 20260507 00:07:38.728 | INFO | } | 20260507 00:07:38.728 | INFO | Perf report for the last 5.13654 seconds: | 20260507 00:07:38.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:07:38.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:38.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:38.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:38.728 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 00:07:38.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:38.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:38.728 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:07:38.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:38.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:38.728 | INFO | Uploaded 36GiB to GPU in 1.95082s (18.4538GiB/s) | 20260507 00:07:38.728 | INFO | Downloaded 34GiB to GPU in 1.89812s (17.9124GiB/s) | 20260507 00:07:43.860 | INFO | Analyzing chunk frames = [5535744,5536256) | 20260507 00:07:43.860 | INFO | MemInfo: { | 20260507 00:07:43.860 | INFO | "gpu" : " 17 GB", | 20260507 00:07:43.860 | INFO | "huge" : "", | 20260507 00:07:43.860 | INFO | "huge_shared" : " 27 GB", | 20260507 00:07:43.860 | INFO | "malloc" : " 14 GB", | 20260507 00:07:43.860 | INFO | "pinned" : " 29 GB" | 20260507 00:07:43.860 | INFO | } | 20260507 00:07:43.860 | INFO | Perf report for the last 5.13192 seconds: | 20260507 00:07:43.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:07:43.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:43.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:43.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:43.860 | INFO | ANALYSIS : 60.2%, 3.0, 93.5ms, 0 | 20260507 00:07:43.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:43.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:43.860 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:07:43.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:43.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:43.860 | INFO | Uploaded 36GiB to GPU in 1.99785s (18.0193GiB/s) | 20260507 00:07:43.860 | INFO | Downloaded 34GiB to GPU in 1.93541s (17.5673GiB/s) | 20260507 00:07:48.995 | INFO | Analyzing chunk frames = [5536256,5536768) | 20260507 00:07:48.995 | INFO | MemInfo: { | 20260507 00:07:48.995 | INFO | "gpu" : " 17 GB", | 20260507 00:07:48.995 | INFO | "huge" : "", | 20260507 00:07:48.995 | INFO | "huge_shared" : " 27 GB", | 20260507 00:07:48.995 | INFO | "malloc" : " 14 GB", | 20260507 00:07:48.995 | INFO | "pinned" : " 29 GB" | 20260507 00:07:48.995 | INFO | } | 20260507 00:07:48.995 | INFO | Perf report for the last 5.1347 seconds: | 20260507 00:07:48.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:07:48.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:48.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:48.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:48.995 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 00:07:48.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:48.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:48.995 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260507 00:07:48.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:48.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:48.995 | INFO | Uploaded 36GiB to GPU in 2.01095s (17.902GiB/s) | 20260507 00:07:48.995 | INFO | Downloaded 34GiB to GPU in 1.94372s (17.4923GiB/s) | 20260507 00:07:54.140 | INFO | Analyzing chunk frames = [5536768,5537280) | 20260507 00:07:54.140 | INFO | MemInfo: { | 20260507 00:07:54.140 | INFO | "gpu" : " 17 GB", | 20260507 00:07:54.140 | INFO | "huge" : "", | 20260507 00:07:54.140 | INFO | "huge_shared" : " 27 GB", | 20260507 00:07:54.140 | INFO | "malloc" : " 14 GB", | 20260507 00:07:54.140 | INFO | "pinned" : " 29 GB" | 20260507 00:07:54.140 | INFO | } | 20260507 00:07:54.140 | INFO | Perf report for the last 5.14565 seconds: | 20260507 00:07:54.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:07:54.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:54.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:54.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:54.140 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 00:07:54.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:54.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:54.140 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 00:07:54.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:54.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:54.141 | INFO | Uploaded 36GiB to GPU in 1.96393s (18.3306GiB/s) | 20260507 00:07:54.141 | INFO | Downloaded 34GiB to GPU in 1.98701s (17.1111GiB/s) | 20260507 00:07:59.287 | INFO | Analyzing chunk frames = [5537280,5537792) | 20260507 00:07:59.287 | INFO | MemInfo: { | 20260507 00:07:59.287 | INFO | "gpu" : " 17 GB", | 20260507 00:07:59.287 | INFO | "huge" : "", | 20260507 00:07:59.287 | INFO | "huge_shared" : " 27 GB", | 20260507 00:07:59.287 | INFO | "malloc" : " 14 GB", | 20260507 00:07:59.287 | INFO | "pinned" : " 29 GB" | 20260507 00:07:59.287 | INFO | } | 20260507 00:07:59.287 | INFO | Perf report for the last 5.14662 seconds: | 20260507 00:07:59.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:07:59.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:59.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:59.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:59.287 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:07:59.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:59.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:59.287 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:07:59.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:59.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:07:59.287 | INFO | Uploaded 36GiB to GPU in 1.98816s (18.1072GiB/s) | 20260507 00:07:59.287 | INFO | Downloaded 34GiB to GPU in 1.99837s (17.0138GiB/s) | 20260507 00:07:59.287 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:07:59.287 | INFO | Total mem usage: 18 GiB | 20260507 00:07:59.287 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:07:59.287 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:07:59.287 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:07:59.287 | INFO | High Water Breakdown: | 20260507 00:07:59.287 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:07:59.287 | INFO | 39.7% : 5.67 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:07:59.287 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:07:59.287 | INFO | ------------------------------------------ | 20260507 00:07:59.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:07:59.287 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:07:59.287 | INFO | Total mem usage: 26.5 GiB | 20260507 00:07:59.287 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:07:59.287 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:07:59.287 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:07:59.287 | INFO | High Water Breakdown: | 20260507 00:07:59.287 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:07:59.287 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:07:59.287 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:07:59.287 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:07:59.288 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:07:59.288 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:07:59.288 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:07:59.288 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:07:59.288 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:07:59.288 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:07:59.288 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:07:59.288 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:07:59.288 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:07:59.288 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:07:59.288 | INFO | ------------------------------------------ | 20260507 00:07:59.288 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:07:59.288 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:07:59.288 | INFO | Total mem usage: 25.6 GiB | 20260507 00:07:59.288 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:07:59.288 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:07:59.288 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:07:59.288 | INFO | High Water Breakdown: | 20260507 00:07:59.288 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:07:59.288 | INFO | ------------------------------------------ | 20260507 00:07:59.288 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:07:59.288 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:07:59.288 | INFO | Total mem usage: 15.7 GiB | 20260507 00:07:59.288 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:07:59.288 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:07:59.288 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:07:59.288 | INFO | High Water Breakdown: | 20260507 00:07:59.288 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:07:59.288 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:07:59.288 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:07:59.288 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:07:59.288 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:07:59.288 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:07:59.288 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:07:59.288 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:07:59.288 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:07:59.288 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:07:59.288 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:07:59.288 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:07:59.288 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:07:59.288 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:07:59.288 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:07:59.288 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:07:59.288 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:07:59.288 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:07:59.288 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:07:59.288 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:07:59.288 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:07:59.288 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:07:59.288 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:07:59.288 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:07:59.288 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:07:59.288 | INFO | ------------------------------------------ | 20260507 00:07:59.288 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:07:59.288 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:07:59.288 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:07:59.288 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:07:59.288 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.60e+01, 1.35e+01 / 2.38e+00 | 20260507 00:07:59.288 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.93e-03 / 4.48e-03 | 20260507 00:07:59.288 | INFO | PerfReport: 11.67%, Baseline -- 3.61e+00 / 4.41e+00, 5.40e-01 / 2.59e-01 | 20260507 00:07:59.288 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.45e+00, 1.75e-01 / 1.19e-01 | 20260507 00:07:59.288 | INFO | PerfReport: 16.07%, DME -- 3.98e+01 / 4.48e+01, 4.12e+00 / 2.73e+00 | 20260507 00:07:59.288 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.31e+00 / 1.85e+00 | 20260507 00:07:59.288 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.18e+00 / 2.73e+00, 3.81e-01 / 2.30e-01 | 20260507 00:07:59.288 | INFO | PerfReport: 3.18%, Metrics -- 9.83e-01 / 1.30e+00, 2.64e-01 / 1.63e-01 | 20260507 00:07:59.288 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.35e-01 / 3.92e-01, 5.39e-02 / 2.71e-02 | 20260507 00:07:59.288 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:07:59.288 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:07:59.288 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:07:59.288 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 00:07:59.288 | INFO | PerfReport: 21.53%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260507 00:07:59.288 | INFO | PerfReport: 21.31%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 00:07:59.288 | INFO | PerfReport: 32.05%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 00:07:59.288 | INFO | PerfReport: 1.29%, LockUpload -- 1.25e+00 / 0.00e+00, 4.79e+00 / 0.00e+00 | 20260507 00:07:59.288 | INFO | PerfReport: 1.30%, LockDownload -- 1.26e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260507 00:07:59.288 | INFO | PerfReport: 22.50%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 00:07:59.288 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.13e-02 / 0.00e+00 | 20260507 00:07:59.384 | INFO | Starting a round of Baz IO | 20260507 00:08:02.511 | INFO | Populating RTMetrics report | 20260507 00:08:02.511 | INFO | RTMetrics report populated | 20260507 00:08:02.511 | INFO | Opening tmp RTMetrics json file | 20260507 00:08:02.515 | INFO | Serializing RTMetrics to json | 20260507 00:08:02.531 | INFO | Serialization done, swapping tmp file now... | 20260507 00:08:02.532 | INFO | RTMetrics done | 20260507 00:08:02.603 | INFO | Finished a round of Baz IO: Wrote 1.859 GiB in 3.219 seconds | 20260507 00:08:02.603 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.34% / 0.00% / 66.66% | 20260507 00:08:02.603 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.367741% / 3.97364e-06% / 6.75519e-05% | 20260507 00:08:02.603 | INFO | Baz IO overall: | 20260507 00:08:02.603 | INFO | BytesWritten : 2890625362167 | 20260507 00:08:02.603 | INFO | NumEvents : 1392890128360 | 20260507 00:08:04.423 | INFO | Analyzing chunk frames = [5537792,5538304) | 20260507 00:08:04.423 | INFO | MemInfo: { | 20260507 00:08:04.423 | INFO | "gpu" : " 17 GB", | 20260507 00:08:04.423 | INFO | "huge" : "", | 20260507 00:08:04.423 | INFO | "huge_shared" : " 27 GB", | 20260507 00:08:04.423 | INFO | "malloc" : " 14 GB", | 20260507 00:08:04.423 | INFO | "pinned" : " 29 GB" | 20260507 00:08:04.423 | INFO | } | 20260507 00:08:04.423 | INFO | Perf report for the last 5.13642 seconds: | 20260507 00:08:04.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:08:04.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:04.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:04.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:04.424 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 00:08:04.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:04.424 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 00:08:04.424 | INFO | PRE_HQ : 59.7%, 1.8, 56.3ms, 0 | 20260507 00:08:04.424 | INFO | BAZWRITER : 62.1%, 3.4, 107.8ms, 0 | 20260507 00:08:04.424 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:08:04.424 | INFO | Uploaded 36GiB to GPU in 2.03826s (17.6622GiB/s) | 20260507 00:08:04.424 | INFO | Downloaded 38GiB to GPU in 2.17728s (17.453GiB/s) | 20260507 00:08:09.559 | INFO | Analyzing chunk frames = [5538304,5538816) | 20260507 00:08:09.559 | INFO | MemInfo: { | 20260507 00:08:09.559 | INFO | "gpu" : " 17 GB", | 20260507 00:08:09.559 | INFO | "huge" : "", | 20260507 00:08:09.559 | INFO | "huge_shared" : " 27 GB", | 20260507 00:08:09.559 | INFO | "malloc" : " 14 GB", | 20260507 00:08:09.559 | INFO | "pinned" : " 29 GB" | 20260507 00:08:09.559 | INFO | } | 20260507 00:08:09.559 | INFO | Perf report for the last 5.13599 seconds: | 20260507 00:08:09.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:08:09.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:09.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:09.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:09.559 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:08:09.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:09.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:09.559 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:08:09.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:09.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:09.560 | INFO | Uploaded 36GiB to GPU in 2.03583s (17.6832GiB/s) | 20260507 00:08:09.560 | INFO | Downloaded 34GiB to GPU in 1.93281s (17.591GiB/s) | 20260507 00:08:14.699 | INFO | Analyzing chunk frames = [5538816,5539328) | 20260507 00:08:14.699 | INFO | MemInfo: { | 20260507 00:08:14.699 | INFO | "gpu" : " 17 GB", | 20260507 00:08:14.699 | INFO | "huge" : "", | 20260507 00:08:14.699 | INFO | "huge_shared" : " 27 GB", | 20260507 00:08:14.699 | INFO | "malloc" : " 14 GB", | 20260507 00:08:14.699 | INFO | "pinned" : " 29 GB" | 20260507 00:08:14.699 | INFO | } | 20260507 00:08:14.699 | INFO | Perf report for the last 5.13947 seconds: | 20260507 00:08:14.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:08:14.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:14.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:14.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:14.699 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 00:08:14.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:14.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:14.699 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:08:14.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:14.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:14.699 | INFO | Uploaded 36GiB to GPU in 1.99002s (18.0903GiB/s) | 20260507 00:08:14.699 | INFO | Downloaded 34GiB to GPU in 1.9263s (17.6504GiB/s) | 20260507 00:08:19.837 | INFO | Analyzing chunk frames = [5539328,5539840) | 20260507 00:08:19.838 | INFO | MemInfo: { | 20260507 00:08:19.838 | INFO | "gpu" : " 17 GB", | 20260507 00:08:19.838 | INFO | "huge" : "", | 20260507 00:08:19.838 | INFO | "huge_shared" : " 27 GB", | 20260507 00:08:19.838 | INFO | "malloc" : " 14 GB", | 20260507 00:08:19.838 | INFO | "pinned" : " 29 GB" | 20260507 00:08:19.838 | INFO | } | 20260507 00:08:19.838 | INFO | Perf report for the last 5.13877 seconds: | 20260507 00:08:19.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:08:19.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:19.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:19.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:19.838 | INFO | ANALYSIS : 59.5%, 3.0, 92.2ms, 0 | 20260507 00:08:19.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:19.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:19.838 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:08:19.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:19.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:19.838 | INFO | Uploaded 36GiB to GPU in 1.96946s (18.2792GiB/s) | 20260507 00:08:19.838 | INFO | Downloaded 34GiB to GPU in 1.88012s (18.084GiB/s) | 20260507 00:08:24.972 | INFO | Analyzing chunk frames = [5539840,5540352) | 20260507 00:08:24.972 | INFO | MemInfo: { | 20260507 00:08:24.972 | INFO | "gpu" : " 17 GB", | 20260507 00:08:24.973 | INFO | "huge" : "", | 20260507 00:08:24.973 | INFO | "huge_shared" : " 27 GB", | 20260507 00:08:24.973 | INFO | "malloc" : " 14 GB", | 20260507 00:08:24.973 | INFO | "pinned" : " 29 GB" | 20260507 00:08:24.973 | INFO | } | 20260507 00:08:24.973 | INFO | Perf report for the last 5.13492 seconds: | 20260507 00:08:24.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:08:24.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:24.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:24.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:24.973 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 00:08:24.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:24.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:24.973 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:08:24.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:24.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:24.973 | INFO | Uploaded 36GiB to GPU in 1.99538s (18.0417GiB/s) | 20260507 00:08:24.973 | INFO | Downloaded 34GiB to GPU in 1.9252s (17.6605GiB/s) | 20260507 00:08:30.116 | INFO | Analyzing chunk frames = [5540352,5540864) | 20260507 00:08:30.116 | INFO | MemInfo: { | 20260507 00:08:30.116 | INFO | "gpu" : " 17 GB", | 20260507 00:08:30.116 | INFO | "huge" : "", | 20260507 00:08:30.116 | INFO | "huge_shared" : " 27 GB", | 20260507 00:08:30.116 | INFO | "malloc" : " 14 GB", | 20260507 00:08:30.116 | INFO | "pinned" : " 29 GB" | 20260507 00:08:30.116 | INFO | } | 20260507 00:08:30.116 | INFO | Perf report for the last 5.14318 seconds: | 20260507 00:08:30.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:08:30.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:30.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:30.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:30.116 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:08:30.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:30.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:30.116 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:08:30.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:30.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:30.116 | INFO | Uploaded 36GiB to GPU in 2.02692s (17.761GiB/s) | 20260507 00:08:30.116 | INFO | Downloaded 34GiB to GPU in 1.92976s (17.6187GiB/s) | 20260507 00:08:35.257 | INFO | Analyzing chunk frames = [5540864,5541376) | 20260507 00:08:35.257 | INFO | MemInfo: { | 20260507 00:08:35.257 | INFO | "gpu" : " 17 GB", | 20260507 00:08:35.257 | INFO | "huge" : "", | 20260507 00:08:35.257 | INFO | "huge_shared" : " 27 GB", | 20260507 00:08:35.257 | INFO | "malloc" : " 14 GB", | 20260507 00:08:35.257 | INFO | "pinned" : " 29 GB" | 20260507 00:08:35.257 | INFO | } | 20260507 00:08:35.257 | INFO | Perf report for the last 5.1415 seconds: | 20260507 00:08:35.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:08:35.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:35.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:35.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:35.257 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 00:08:35.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:35.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:35.257 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 00:08:35.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:35.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:35.258 | INFO | Uploaded 36GiB to GPU in 1.97871s (18.1937GiB/s) | 20260507 00:08:35.258 | INFO | Downloaded 34GiB to GPU in 2.00043s (16.9964GiB/s) | 20260507 00:08:40.401 | INFO | Analyzing chunk frames = [5541376,5541888) | 20260507 00:08:40.401 | INFO | MemInfo: { | 20260507 00:08:40.401 | INFO | "gpu" : " 17 GB", | 20260507 00:08:40.401 | INFO | "huge" : "", | 20260507 00:08:40.401 | INFO | "huge_shared" : " 27 GB", | 20260507 00:08:40.401 | INFO | "malloc" : " 14 GB", | 20260507 00:08:40.401 | INFO | "pinned" : " 29 GB" | 20260507 00:08:40.401 | INFO | } | 20260507 00:08:40.401 | INFO | Perf report for the last 5.14337 seconds: | 20260507 00:08:40.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:08:40.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:40.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:40.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:40.401 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:08:40.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:40.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:40.401 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 00:08:40.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:40.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:40.401 | INFO | Uploaded 36GiB to GPU in 1.98067s (18.1757GiB/s) | 20260507 00:08:40.401 | INFO | Downloaded 34GiB to GPU in 1.99186s (17.0695GiB/s) | 20260507 00:08:40.498 | INFO | Starting a round of Baz IO | 20260507 00:08:43.614 | INFO | Populating RTMetrics report | 20260507 00:08:43.614 | INFO | RTMetrics report populated | 20260507 00:08:43.614 | INFO | Opening tmp RTMetrics json file | 20260507 00:08:43.615 | INFO | Serializing RTMetrics to json | 20260507 00:08:43.616 | INFO | Serialization done, swapping tmp file now... | 20260507 00:08:43.616 | INFO | RTMetrics done | 20260507 00:08:43.707 | INFO | Finished a round of Baz IO: Wrote 1.859 GiB in 3.209 seconds | 20260507 00:08:43.707 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.34% / 0.00% / 66.66% | 20260507 00:08:43.707 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.363914% / 0% / 6.35783e-05% | 20260507 00:08:43.708 | INFO | Baz IO overall: | 20260507 00:08:43.708 | INFO | BytesWritten : 2892621879100 | 20260507 00:08:43.708 | INFO | NumEvents : 1393839213335 | 20260507 00:08:45.538 | INFO | Analyzing chunk frames = [5541888,5542400) | 20260507 00:08:45.538 | INFO | MemInfo: { | 20260507 00:08:45.538 | INFO | "gpu" : " 17 GB", | 20260507 00:08:45.538 | INFO | "huge" : "", | 20260507 00:08:45.538 | INFO | "huge_shared" : " 27 GB", | 20260507 00:08:45.538 | INFO | "malloc" : " 14 GB", | 20260507 00:08:45.538 | INFO | "pinned" : " 29 GB" | 20260507 00:08:45.538 | INFO | } | 20260507 00:08:45.538 | INFO | Perf report for the last 5.13728 seconds: | 20260507 00:08:45.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:08:45.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:45.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:45.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:45.538 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 00:08:45.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:45.538 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 00:08:45.538 | INFO | PRE_HQ : 59.7%, 1.8, 56.8ms, 0 | 20260507 00:08:45.538 | INFO | BAZWRITER : 62.1%, 3.3, 106.7ms, 0 | 20260507 00:08:45.538 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:08:45.538 | INFO | Uploaded 36GiB to GPU in 2.03039s (17.7306GiB/s) | 20260507 00:08:45.538 | INFO | Downloaded 38GiB to GPU in 2.17595s (17.4636GiB/s) | 20260507 00:08:50.671 | INFO | Analyzing chunk frames = [5542400,5542912) | 20260507 00:08:50.671 | INFO | MemInfo: { | 20260507 00:08:50.671 | INFO | "gpu" : " 17 GB", | 20260507 00:08:50.671 | INFO | "huge" : "", | 20260507 00:08:50.671 | INFO | "huge_shared" : " 27 GB", | 20260507 00:08:50.671 | INFO | "malloc" : " 14 GB", | 20260507 00:08:50.671 | INFO | "pinned" : " 29 GB" | 20260507 00:08:50.671 | INFO | } | 20260507 00:08:50.671 | INFO | Perf report for the last 5.13335 seconds: | 20260507 00:08:50.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:08:50.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:50.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:50.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:50.671 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:08:50.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:50.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:50.671 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:08:50.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:50.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:50.672 | INFO | Uploaded 36GiB to GPU in 2.00489s (17.9561GiB/s) | 20260507 00:08:50.672 | INFO | Downloaded 34GiB to GPU in 1.92813s (17.6336GiB/s) | 20260507 00:08:55.809 | INFO | Analyzing chunk frames = [5542912,5543424) | 20260507 00:08:55.809 | INFO | MemInfo: { | 20260507 00:08:55.809 | INFO | "gpu" : " 17 GB", | 20260507 00:08:55.809 | INFO | "huge" : "", | 20260507 00:08:55.809 | INFO | "huge_shared" : " 27 GB", | 20260507 00:08:55.809 | INFO | "malloc" : " 14 GB", | 20260507 00:08:55.809 | INFO | "pinned" : " 29 GB" | 20260507 00:08:55.809 | INFO | } | 20260507 00:08:55.809 | INFO | Perf report for the last 5.1379 seconds: | 20260507 00:08:55.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:08:55.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:55.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:55.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:55.809 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 00:08:55.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:55.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:55.809 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:08:55.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:55.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:08:55.809 | INFO | Uploaded 36GiB to GPU in 1.97953s (18.1861GiB/s) | 20260507 00:08:55.810 | INFO | Downloaded 34GiB to GPU in 1.92909s (17.6249GiB/s) | 20260507 00:09:00.947 | INFO | Analyzing chunk frames = [5543424,5543936) | 20260507 00:09:00.947 | INFO | MemInfo: { | 20260507 00:09:00.947 | INFO | "gpu" : " 17 GB", | 20260507 00:09:00.947 | INFO | "huge" : "", | 20260507 00:09:00.947 | INFO | "huge_shared" : " 27 GB", | 20260507 00:09:00.947 | INFO | "malloc" : " 14 GB", | 20260507 00:09:00.947 | INFO | "pinned" : " 29 GB" | 20260507 00:09:00.947 | INFO | } | 20260507 00:09:00.947 | INFO | Perf report for the last 5.13817 seconds: | 20260507 00:09:00.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:09:00.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:00.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:00.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:00.948 | INFO | ANALYSIS : 59.5%, 3.0, 92.2ms, 0 | 20260507 00:09:00.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:00.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:00.948 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:09:00.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:00.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:00.948 | INFO | Uploaded 36GiB to GPU in 1.94782s (18.4822GiB/s) | 20260507 00:09:00.948 | INFO | Downloaded 34GiB to GPU in 1.87538s (18.1296GiB/s) | 20260507 00:09:06.092 | INFO | Analyzing chunk frames = [5543936,5544448) | 20260507 00:09:06.092 | INFO | MemInfo: { | 20260507 00:09:06.092 | INFO | "gpu" : " 17 GB", | 20260507 00:09:06.092 | INFO | "huge" : "", | 20260507 00:09:06.092 | INFO | "huge_shared" : " 27 GB", | 20260507 00:09:06.092 | INFO | "malloc" : " 14 GB", | 20260507 00:09:06.092 | INFO | "pinned" : " 29 GB" | 20260507 00:09:06.092 | INFO | } | 20260507 00:09:06.092 | INFO | Perf report for the last 5.14446 seconds: | 20260507 00:09:06.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:09:06.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:06.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:06.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:06.092 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 00:09:06.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:06.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:06.092 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 00:09:06.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:06.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:06.092 | INFO | Uploaded 36GiB to GPU in 1.9968s (18.0289GiB/s) | 20260507 00:09:06.092 | INFO | Downloaded 34GiB to GPU in 1.92473s (17.6648GiB/s) | 20260507 00:09:11.234 | INFO | Analyzing chunk frames = [5544448,5544960) | 20260507 00:09:11.234 | INFO | MemInfo: { | 20260507 00:09:11.234 | INFO | "gpu" : " 17 GB", | 20260507 00:09:11.234 | INFO | "huge" : "", | 20260507 00:09:11.234 | INFO | "huge_shared" : " 27 GB", | 20260507 00:09:11.234 | INFO | "malloc" : " 14 GB", | 20260507 00:09:11.234 | INFO | "pinned" : " 29 GB" | 20260507 00:09:11.234 | INFO | } | 20260507 00:09:11.234 | INFO | Perf report for the last 5.142 seconds: | 20260507 00:09:11.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:09:11.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:11.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:11.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:11.234 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 00:09:11.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:11.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:11.234 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 00:09:11.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:11.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:11.234 | INFO | Uploaded 36GiB to GPU in 2.02508s (17.7771GiB/s) | 20260507 00:09:11.234 | INFO | Downloaded 34GiB to GPU in 1.93661s (17.5564GiB/s) | 20260507 00:09:16.375 | INFO | Analyzing chunk frames = [5544960,5545472) | 20260507 00:09:16.375 | INFO | MemInfo: { | 20260507 00:09:16.375 | INFO | "gpu" : " 17 GB", | 20260507 00:09:16.375 | INFO | "huge" : "", | 20260507 00:09:16.375 | INFO | "huge_shared" : " 27 GB", | 20260507 00:09:16.375 | INFO | "malloc" : " 14 GB", | 20260507 00:09:16.375 | INFO | "pinned" : " 29 GB" | 20260507 00:09:16.375 | INFO | } | 20260507 00:09:16.375 | INFO | Perf report for the last 5.14122 seconds: | 20260507 00:09:16.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:09:16.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:16.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:16.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:16.375 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:09:16.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:16.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:16.375 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 00:09:16.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:16.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:16.375 | INFO | Uploaded 36GiB to GPU in 1.98356s (18.1492GiB/s) | 20260507 00:09:16.375 | INFO | Downloaded 34GiB to GPU in 1.98548s (17.1243GiB/s) | 20260507 00:09:21.511 | INFO | Analyzing chunk frames = [5545472,5545984) | 20260507 00:09:21.511 | INFO | MemInfo: { | 20260507 00:09:21.511 | INFO | "gpu" : " 17 GB", | 20260507 00:09:21.511 | INFO | "huge" : "", | 20260507 00:09:21.511 | INFO | "huge_shared" : " 27 GB", | 20260507 00:09:21.511 | INFO | "malloc" : " 14 GB", | 20260507 00:09:21.511 | INFO | "pinned" : " 29 GB" | 20260507 00:09:21.511 | INFO | } | 20260507 00:09:21.511 | INFO | Perf report for the last 5.13628 seconds: | 20260507 00:09:21.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:09:21.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:21.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:21.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:21.511 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:09:21.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:21.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:21.512 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 00:09:21.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:21.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:21.512 | INFO | Uploaded 36GiB to GPU in 1.98304s (18.154GiB/s) | 20260507 00:09:21.512 | INFO | Downloaded 34GiB to GPU in 1.97694s (17.1983GiB/s) | 20260507 00:09:21.512 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:09:21.512 | INFO | Total mem usage: 18 GiB | 20260507 00:09:21.512 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:09:21.512 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:09:21.512 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:09:21.512 | INFO | High Water Breakdown: | 20260507 00:09:21.512 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:09:21.512 | INFO | 39.7% : 5.67 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:09:21.512 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:09:21.512 | INFO | ------------------------------------------ | 20260507 00:09:21.512 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:09:21.512 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:09:21.512 | INFO | Total mem usage: 26.5 GiB | 20260507 00:09:21.512 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:09:21.512 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:09:21.512 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:09:21.512 | INFO | High Water Breakdown: | 20260507 00:09:21.512 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:09:21.512 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:09:21.512 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:09:21.512 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:09:21.512 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:09:21.512 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:09:21.512 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:09:21.512 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:09:21.512 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:09:21.512 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:09:21.512 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:09:21.512 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:09:21.512 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:09:21.512 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:09:21.512 | INFO | ------------------------------------------ | 20260507 00:09:21.512 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:09:21.512 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:09:21.512 | INFO | Total mem usage: 25.6 GiB | 20260507 00:09:21.512 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:09:21.512 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:09:21.512 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:09:21.512 | INFO | High Water Breakdown: | 20260507 00:09:21.512 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:09:21.512 | INFO | ------------------------------------------ | 20260507 00:09:21.512 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:09:21.512 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:09:21.512 | INFO | Total mem usage: 15.7 GiB | 20260507 00:09:21.512 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:09:21.512 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:09:21.512 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 00:09:21.512 | INFO | High Water Breakdown: | 20260507 00:09:21.512 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:09:21.512 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:09:21.512 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:09:21.512 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:09:21.512 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:09:21.512 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:09:21.512 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:09:21.512 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:09:21.512 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:09:21.512 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:09:21.512 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:09:21.512 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:09:21.512 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:09:21.512 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:09:21.512 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:09:21.512 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:09:21.512 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:09:21.512 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:09:21.512 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:09:21.512 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:09:21.512 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:09:21.512 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:09:21.512 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:09:21.512 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:09:21.512 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:09:21.512 | INFO | ------------------------------------------ | 20260507 00:09:21.512 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:09:21.512 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:09:21.512 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:09:21.512 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:09:21.513 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.58e+01, 1.33e+01 / 2.03e+00 | 20260507 00:09:21.513 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 1.07e-02 / 4.79e-03 | 20260507 00:09:21.513 | INFO | PerfReport: 11.65%, Baseline -- 3.60e+00 / 4.45e+00, 5.36e-01 / 2.66e-01 | 20260507 00:09:21.513 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.49e+00, 1.76e-01 / 8.98e-02 | 20260507 00:09:21.513 | INFO | PerfReport: 16.08%, DME -- 3.98e+01 / 4.48e+01, 3.98e+00 / 2.07e+00 | 20260507 00:09:21.513 | INFO | PerfReport: 57.44%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.35e+00 / 1.59e+00 | 20260507 00:09:21.513 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.18e+00 / 2.74e+00, 3.81e-01 / 2.17e-01 | 20260507 00:09:21.513 | INFO | PerfReport: 3.16%, Metrics -- 9.78e-01 / 1.33e+00, 2.61e-01 / 1.31e-01 | 20260507 00:09:21.513 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.33e-01 / 3.85e-01, 5.28e-02 / 2.88e-02 | 20260507 00:09:21.513 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:09:21.513 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:09:21.513 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:09:21.513 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 00:09:21.513 | INFO | PerfReport: 21.57%, Upload -- 2.08e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260507 00:09:21.513 | INFO | PerfReport: 21.26%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260507 00:09:21.513 | INFO | PerfReport: 32.07%, Compute -- 3.09e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 00:09:21.513 | INFO | PerfReport: 1.33%, LockUpload -- 1.28e+00 / 0.00e+00, 4.86e+00 / 0.00e+00 | 20260507 00:09:21.513 | INFO | PerfReport: 1.23%, LockDownload -- 1.19e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260507 00:09:21.513 | INFO | PerfReport: 22.53%, LockCompute -- 2.17e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 00:09:21.513 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.55e-02 / 0.00e+00 | 20260507 00:09:21.610 | INFO | Starting a round of Baz IO | 20260507 00:09:24.739 | INFO | Populating RTMetrics report | 20260507 00:09:24.739 | INFO | RTMetrics report populated | 20260507 00:09:24.739 | INFO | Opening tmp RTMetrics json file | 20260507 00:09:24.740 | INFO | Serializing RTMetrics to json | 20260507 00:09:24.749 | INFO | Serialization done, swapping tmp file now... | 20260507 00:09:24.761 | INFO | RTMetrics done | 20260507 00:09:24.832 | INFO | Finished a round of Baz IO: Wrote 1.858 GiB in 3.221 seconds | 20260507 00:09:24.832 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.34% / 0.00% / 66.66% | 20260507 00:09:24.832 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.363918% / 0% / 7.54992e-05% | 20260507 00:09:24.832 | INFO | Baz IO overall: | 20260507 00:09:24.832 | INFO | BytesWritten : 2894617041286 | 20260507 00:09:24.832 | INFO | NumEvents : 1394787037501 | 20260507 00:09:26.650 | INFO | Analyzing chunk frames = [5545984,5546496) | 20260507 00:09:26.650 | INFO | MemInfo: { | 20260507 00:09:26.650 | INFO | "gpu" : " 17 GB", | 20260507 00:09:26.650 | INFO | "huge" : "", | 20260507 00:09:26.650 | INFO | "huge_shared" : " 27 GB", | 20260507 00:09:26.650 | INFO | "malloc" : " 14 GB", | 20260507 00:09:26.650 | INFO | "pinned" : " 29 GB" | 20260507 00:09:26.650 | INFO | } | 20260507 00:09:26.650 | INFO | Perf report for the last 5.13873 seconds: | 20260507 00:09:26.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:09:26.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:26.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:26.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:26.650 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260507 00:09:26.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:26.650 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 00:09:26.650 | INFO | PRE_HQ : 60.2%, 1.8, 56.8ms, 0 | 20260507 00:09:26.650 | INFO | BAZWRITER : 62.3%, 3.3, 107.1ms, 0 | 20260507 00:09:26.650 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:09:26.651 | INFO | Uploaded 36GiB to GPU in 2.03471s (17.6929GiB/s) | 20260507 00:09:26.651 | INFO | Downloaded 38GiB to GPU in 2.18286s (17.4083GiB/s) | 20260507 00:09:31.785 | INFO | Analyzing chunk frames = [5546496,5547008) | 20260507 00:09:31.785 | INFO | MemInfo: { | 20260507 00:09:31.785 | INFO | "gpu" : " 17 GB", | 20260507 00:09:31.785 | INFO | "huge" : "", | 20260507 00:09:31.785 | INFO | "huge_shared" : " 27 GB", | 20260507 00:09:31.785 | INFO | "malloc" : " 14 GB", | 20260507 00:09:31.785 | INFO | "pinned" : " 29 GB" | 20260507 00:09:31.785 | INFO | } | 20260507 00:09:31.785 | INFO | Perf report for the last 5.13495 seconds: | 20260507 00:09:31.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:09:31.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:31.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:31.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:31.785 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:09:31.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:31.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:31.785 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:09:31.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:31.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:31.785 | INFO | Uploaded 36GiB to GPU in 1.98931s (18.0967GiB/s) | 20260507 00:09:31.785 | INFO | Downloaded 34GiB to GPU in 1.92935s (17.6225GiB/s) | 20260507 00:09:36.923 | INFO | Analyzing chunk frames = [5547008,5547520) | 20260507 00:09:36.923 | INFO | MemInfo: { | 20260507 00:09:36.923 | INFO | "gpu" : " 17 GB", | 20260507 00:09:36.923 | INFO | "huge" : "", | 20260507 00:09:36.923 | INFO | "huge_shared" : " 27 GB", | 20260507 00:09:36.923 | INFO | "malloc" : " 14 GB", | 20260507 00:09:36.923 | INFO | "pinned" : " 29 GB" | 20260507 00:09:36.923 | INFO | } | 20260507 00:09:36.923 | INFO | Perf report for the last 5.13755 seconds: | 20260507 00:09:36.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:09:36.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:36.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:36.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:36.923 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 00:09:36.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:36.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:36.923 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:09:36.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:36.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:36.923 | INFO | Uploaded 36GiB to GPU in 1.98173s (18.1659GiB/s) | 20260507 00:09:36.923 | INFO | Downloaded 34GiB to GPU in 1.93064s (17.6107GiB/s) | 20260507 00:09:42.066 | INFO | Analyzing chunk frames = [5547520,5548032) | 20260507 00:09:42.066 | INFO | MemInfo: { | 20260507 00:09:42.066 | INFO | "gpu" : " 17 GB", | 20260507 00:09:42.066 | INFO | "huge" : "", | 20260507 00:09:42.066 | INFO | "huge_shared" : " 27 GB", | 20260507 00:09:42.066 | INFO | "malloc" : " 14 GB", | 20260507 00:09:42.066 | INFO | "pinned" : " 29 GB" | 20260507 00:09:42.066 | INFO | } | 20260507 00:09:42.066 | INFO | Perf report for the last 5.14343 seconds: | 20260507 00:09:42.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:09:42.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:42.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:42.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:42.066 | INFO | ANALYSIS : 59.4%, 3.0, 92.2ms, 0 | 20260507 00:09:42.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:42.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:42.066 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:09:42.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:42.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:42.066 | INFO | Uploaded 36GiB to GPU in 1.94045s (18.5524GiB/s) | 20260507 00:09:42.066 | INFO | Downloaded 34GiB to GPU in 1.87945s (18.0904GiB/s) | 20260507 00:09:47.207 | INFO | Analyzing chunk frames = [5548032,5548544) | 20260507 00:09:47.207 | INFO | MemInfo: { | 20260507 00:09:47.207 | INFO | "gpu" : " 17 GB", | 20260507 00:09:47.207 | INFO | "huge" : "", | 20260507 00:09:47.207 | INFO | "huge_shared" : " 27 GB", | 20260507 00:09:47.207 | INFO | "malloc" : " 14 GB", | 20260507 00:09:47.207 | INFO | "pinned" : " 29 GB" | 20260507 00:09:47.207 | INFO | } | 20260507 00:09:47.207 | INFO | Perf report for the last 5.14076 seconds: | 20260507 00:09:47.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:09:47.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:47.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:47.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:47.207 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 00:09:47.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:47.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:47.207 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:09:47.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:47.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:47.207 | INFO | Uploaded 36GiB to GPU in 1.97704s (18.2091GiB/s) | 20260507 00:09:47.207 | INFO | Downloaded 34GiB to GPU in 1.93268s (17.5922GiB/s) | 20260507 00:09:52.351 | INFO | Analyzing chunk frames = [5548544,5549056) | 20260507 00:09:52.351 | INFO | MemInfo: { | 20260507 00:09:52.351 | INFO | "gpu" : " 17 GB", | 20260507 00:09:52.351 | INFO | "huge" : "", | 20260507 00:09:52.351 | INFO | "huge_shared" : " 27 GB", | 20260507 00:09:52.351 | INFO | "malloc" : " 14 GB", | 20260507 00:09:52.352 | INFO | "pinned" : " 29 GB" | 20260507 00:09:52.352 | INFO | } | 20260507 00:09:52.352 | INFO | Perf report for the last 5.14464 seconds: | 20260507 00:09:52.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:09:52.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:52.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:52.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:52.352 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 00:09:52.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:52.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:52.352 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:09:52.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:52.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:52.352 | INFO | Uploaded 36GiB to GPU in 2.00943s (17.9156GiB/s) | 20260507 00:09:52.352 | INFO | Downloaded 34GiB to GPU in 1.94256s (17.5027GiB/s) | 20260507 00:09:57.490 | INFO | Analyzing chunk frames = [5549056,5549568) | 20260507 00:09:57.490 | INFO | MemInfo: { | 20260507 00:09:57.490 | INFO | "gpu" : " 17 GB", | 20260507 00:09:57.490 | INFO | "huge" : "", | 20260507 00:09:57.490 | INFO | "huge_shared" : " 27 GB", | 20260507 00:09:57.490 | INFO | "malloc" : " 14 GB", | 20260507 00:09:57.490 | INFO | "pinned" : " 29 GB" | 20260507 00:09:57.490 | INFO | } | 20260507 00:09:57.490 | INFO | Perf report for the last 5.13889 seconds: | 20260507 00:09:57.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:09:57.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:57.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:57.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:57.490 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:09:57.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:57.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:57.491 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 00:09:57.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:57.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:09:57.491 | INFO | Uploaded 36GiB to GPU in 1.97669s (18.2123GiB/s) | 20260507 00:09:57.491 | INFO | Downloaded 34GiB to GPU in 1.97918s (17.1788GiB/s) | 20260507 00:10:02.623 | INFO | Analyzing chunk frames = [5549568,5550080) | 20260507 00:10:02.623 | INFO | MemInfo: { | 20260507 00:10:02.623 | INFO | "gpu" : " 17 GB", | 20260507 00:10:02.623 | INFO | "huge" : "", | 20260507 00:10:02.623 | INFO | "huge_shared" : " 27 GB", | 20260507 00:10:02.623 | INFO | "malloc" : " 14 GB", | 20260507 00:10:02.623 | INFO | "pinned" : " 29 GB" | 20260507 00:10:02.624 | INFO | } | 20260507 00:10:02.624 | INFO | Perf report for the last 5.13312 seconds: | 20260507 00:10:02.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:10:02.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:02.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:02.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:02.624 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:10:02.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:02.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:02.624 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 00:10:02.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:02.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:02.624 | INFO | Uploaded 36GiB to GPU in 1.99544s (18.0411GiB/s) | 20260507 00:10:02.624 | INFO | Downloaded 34GiB to GPU in 1.97712s (17.1967GiB/s) | 20260507 00:10:02.723 | INFO | Starting a round of Baz IO | 20260507 00:10:05.848 | INFO | Populating RTMetrics report | 20260507 00:10:05.849 | INFO | RTMetrics report populated | 20260507 00:10:05.849 | INFO | Opening tmp RTMetrics json file | 20260507 00:10:05.859 | INFO | Serializing RTMetrics to json | 20260507 00:10:05.865 | INFO | Serialization done, swapping tmp file now... | 20260507 00:10:05.865 | INFO | RTMetrics done | 20260507 00:10:05.937 | INFO | Finished a round of Baz IO: Wrote 1.859 GiB in 3.214 seconds | 20260507 00:10:05.937 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.34% / 0.00% / 66.66% | 20260507 00:10:05.937 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.364121% / 0% / 7.94729e-05% | 20260507 00:10:05.937 | INFO | Baz IO overall: | 20260507 00:10:05.937 | INFO | BytesWritten : 2896613227533 | 20260507 00:10:05.937 | INFO | NumEvents : 1395735405752 | 20260507 00:10:07.761 | INFO | Analyzing chunk frames = [5550080,5550592) | 20260507 00:10:07.761 | INFO | MemInfo: { | 20260507 00:10:07.761 | INFO | "gpu" : " 17 GB", | 20260507 00:10:07.761 | INFO | "huge" : "", | 20260507 00:10:07.761 | INFO | "huge_shared" : " 27 GB", | 20260507 00:10:07.762 | INFO | "malloc" : " 14 GB", | 20260507 00:10:07.762 | INFO | "pinned" : " 29 GB" | 20260507 00:10:07.762 | INFO | } | 20260507 00:10:07.762 | INFO | Perf report for the last 5.13802 seconds: | 20260507 00:10:07.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:10:07.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:07.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:07.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:07.762 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 00:10:07.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:07.762 | INFO | RT_METRICS : 42.6%, 1.0, 22.4ms, 0 | 20260507 00:10:07.762 | INFO | PRE_HQ : 59.6%, 1.8, 56.9ms, 0 | 20260507 00:10:07.762 | INFO | BAZWRITER : 62.2%, 3.4, 107.2ms, 0 | 20260507 00:10:07.762 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 00:10:07.762 | INFO | Uploaded 36GiB to GPU in 2.0043s (17.9614GiB/s) | 20260507 00:10:07.762 | INFO | Downloaded 38GiB to GPU in 2.17616s (17.462GiB/s) | 20260507 00:10:12.903 | INFO | Analyzing chunk frames = [5550592,5551104) | 20260507 00:10:12.903 | INFO | MemInfo: { | 20260507 00:10:12.903 | INFO | "gpu" : " 17 GB", | 20260507 00:10:12.903 | INFO | "huge" : "", | 20260507 00:10:12.903 | INFO | "huge_shared" : " 27 GB", | 20260507 00:10:12.903 | INFO | "malloc" : " 14 GB", | 20260507 00:10:12.903 | INFO | "pinned" : " 29 GB" | 20260507 00:10:12.903 | INFO | } | 20260507 00:10:12.903 | INFO | Perf report for the last 5.14111 seconds: | 20260507 00:10:12.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:10:12.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:12.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:12.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:12.903 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 00:10:12.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:12.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:12.903 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:10:12.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:12.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:12.903 | INFO | Uploaded 36GiB to GPU in 2.00575s (17.9484GiB/s) | 20260507 00:10:12.903 | INFO | Downloaded 34GiB to GPU in 1.93457s (17.575GiB/s) | 20260507 00:10:18.044 | INFO | Analyzing chunk frames = [5551104,5551616) | 20260507 00:10:18.044 | INFO | MemInfo: { | 20260507 00:10:18.044 | INFO | "gpu" : " 17 GB", | 20260507 00:10:18.044 | INFO | "huge" : "", | 20260507 00:10:18.044 | INFO | "huge_shared" : " 27 GB", | 20260507 00:10:18.044 | INFO | "malloc" : " 14 GB", | 20260507 00:10:18.044 | INFO | "pinned" : " 29 GB" | 20260507 00:10:18.044 | INFO | } | 20260507 00:10:18.044 | INFO | Perf report for the last 5.14124 seconds: | 20260507 00:10:18.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:10:18.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:18.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:18.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:18.044 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:10:18.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:18.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:18.044 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:10:18.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:18.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:18.044 | INFO | Uploaded 36GiB to GPU in 1.97136s (18.2615GiB/s) | 20260507 00:10:18.044 | INFO | Downloaded 34GiB to GPU in 1.93179s (17.6002GiB/s) | 20260507 00:10:23.186 | INFO | Analyzing chunk frames = [5551616,5552128) | 20260507 00:10:23.186 | INFO | MemInfo: { | 20260507 00:10:23.186 | INFO | "gpu" : " 17 GB", | 20260507 00:10:23.186 | INFO | "huge" : "", | 20260507 00:10:23.186 | INFO | "huge_shared" : " 27 GB", | 20260507 00:10:23.186 | INFO | "malloc" : " 14 GB", | 20260507 00:10:23.186 | INFO | "pinned" : " 29 GB" | 20260507 00:10:23.186 | INFO | } | 20260507 00:10:23.186 | INFO | Perf report for the last 5.14213 seconds: | 20260507 00:10:23.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:10:23.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:23.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:23.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:23.186 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260507 00:10:23.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:23.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:23.186 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:10:23.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:23.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:23.186 | INFO | Uploaded 36GiB to GPU in 2.00058s (17.9948GiB/s) | 20260507 00:10:23.186 | INFO | Downloaded 34GiB to GPU in 1.9035s (17.8618GiB/s) | 20260507 00:10:28.327 | INFO | Analyzing chunk frames = [5552128,5552640) | 20260507 00:10:28.327 | INFO | MemInfo: { | 20260507 00:10:28.327 | INFO | "gpu" : " 17 GB", | 20260507 00:10:28.327 | INFO | "huge" : "", | 20260507 00:10:28.327 | INFO | "huge_shared" : " 27 GB", | 20260507 00:10:28.328 | INFO | "malloc" : " 14 GB", | 20260507 00:10:28.328 | INFO | "pinned" : " 29 GB" | 20260507 00:10:28.328 | INFO | } | 20260507 00:10:28.328 | INFO | Perf report for the last 5.14152 seconds: | 20260507 00:10:28.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:10:28.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:28.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:28.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:28.328 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 00:10:28.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:28.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:28.328 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 00:10:28.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:28.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:28.328 | INFO | Uploaded 36GiB to GPU in 1.98621s (18.125GiB/s) | 20260507 00:10:28.328 | INFO | Downloaded 34GiB to GPU in 1.9487s (17.4475GiB/s) | 20260507 00:10:33.465 | INFO | Analyzing chunk frames = [5552640,5553152) | 20260507 00:10:33.465 | INFO | MemInfo: { | 20260507 00:10:33.465 | INFO | "gpu" : " 17 GB", | 20260507 00:10:33.465 | INFO | "huge" : "", | 20260507 00:10:33.465 | INFO | "huge_shared" : " 27 GB", | 20260507 00:10:33.466 | INFO | "malloc" : " 14 GB", | 20260507 00:10:33.466 | INFO | "pinned" : " 29 GB" | 20260507 00:10:33.466 | INFO | } | 20260507 00:10:33.466 | INFO | Perf report for the last 5.13798 seconds: | 20260507 00:10:33.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:10:33.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:33.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:33.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:33.466 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:10:33.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:33.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:33.466 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:10:33.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:33.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:33.466 | INFO | Uploaded 36GiB to GPU in 2.04451s (17.6081GiB/s) | 20260507 00:10:33.466 | INFO | Downloaded 34GiB to GPU in 1.95931s (17.3531GiB/s) | 20260507 00:10:38.599 | INFO | Analyzing chunk frames = [5553152,5553664) | 20260507 00:10:38.599 | INFO | MemInfo: { | 20260507 00:10:38.599 | INFO | "gpu" : " 17 GB", | 20260507 00:10:38.599 | INFO | "huge" : "", | 20260507 00:10:38.599 | INFO | "huge_shared" : " 27 GB", | 20260507 00:10:38.599 | INFO | "malloc" : " 14 GB", | 20260507 00:10:38.599 | INFO | "pinned" : " 29 GB" | 20260507 00:10:38.599 | INFO | } | 20260507 00:10:38.599 | INFO | Perf report for the last 5.13366 seconds: | 20260507 00:10:38.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:10:38.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:38.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:38.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:38.599 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 00:10:38.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:38.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:38.599 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 00:10:38.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:38.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:38.600 | INFO | Uploaded 36GiB to GPU in 1.97252s (18.2508GiB/s) | 20260507 00:10:38.600 | INFO | Downloaded 34GiB to GPU in 1.98842s (17.099GiB/s) | 20260507 00:10:43.735 | INFO | Analyzing chunk frames = [5553664,5554176) | 20260507 00:10:43.735 | INFO | MemInfo: { | 20260507 00:10:43.735 | INFO | "gpu" : " 17 GB", | 20260507 00:10:43.735 | INFO | "huge" : "", | 20260507 00:10:43.735 | INFO | "huge_shared" : " 27 GB", | 20260507 00:10:43.735 | INFO | "malloc" : " 14 GB", | 20260507 00:10:43.735 | INFO | "pinned" : " 29 GB" | 20260507 00:10:43.735 | INFO | } | 20260507 00:10:43.735 | INFO | Perf report for the last 5.13603 seconds: | 20260507 00:10:43.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:10:43.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:43.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:43.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:43.735 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:10:43.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:43.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:43.735 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 00:10:43.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:43.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:43.736 | INFO | Uploaded 36GiB to GPU in 1.97183s (18.2571GiB/s) | 20260507 00:10:43.736 | INFO | Downloaded 34GiB to GPU in 1.99218s (17.0667GiB/s) | 20260507 00:10:43.736 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:10:43.736 | INFO | Total mem usage: 18 GiB | 20260507 00:10:43.736 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:10:43.736 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:10:43.736 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:10:43.736 | INFO | High Water Breakdown: | 20260507 00:10:43.736 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:10:43.736 | INFO | 39.7% : 5.67 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:10:43.736 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:10:43.736 | INFO | ------------------------------------------ | 20260507 00:10:43.736 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:10:43.736 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:10:43.736 | INFO | Total mem usage: 26.5 GiB | 20260507 00:10:43.736 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:10:43.736 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:10:43.736 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:10:43.736 | INFO | High Water Breakdown: | 20260507 00:10:43.736 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:10:43.736 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:10:43.736 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:10:43.736 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:10:43.736 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:10:43.736 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:10:43.736 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:10:43.736 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:10:43.736 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:10:43.736 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:10:43.736 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:10:43.736 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:10:43.736 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:10:43.736 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:10:43.736 | INFO | ------------------------------------------ | 20260507 00:10:43.736 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:10:43.736 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:10:43.736 | INFO | Total mem usage: 25.6 GiB | 20260507 00:10:43.736 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:10:43.736 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:10:43.736 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:10:43.736 | INFO | High Water Breakdown: | 20260507 00:10:43.736 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:10:43.736 | INFO | ------------------------------------------ | 20260507 00:10:43.736 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:10:43.736 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:10:43.736 | INFO | Total mem usage: 15.7 GiB | 20260507 00:10:43.736 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:10:43.736 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:10:43.736 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:10:43.736 | INFO | High Water Breakdown: | 20260507 00:10:43.736 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:10:43.737 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:10:43.737 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:10:43.737 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:10:43.737 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:10:43.737 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:10:43.737 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:10:43.737 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:10:43.737 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:10:43.737 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:10:43.737 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:10:43.737 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:10:43.737 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:10:43.737 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:10:43.737 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:10:43.737 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:10:43.737 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:10:43.737 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:10:43.737 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:10:43.737 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:10:43.737 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:10:43.737 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:10:43.737 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:10:43.737 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:10:43.737 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:10:43.737 | INFO | ------------------------------------------ | 20260507 00:10:43.737 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:10:43.737 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:10:43.737 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:10:43.737 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:10:43.737 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.54e+01, 1.33e+01 / 2.14e+00 | 20260507 00:10:43.737 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.01e-01, 9.58e-03 / 6.07e-03 | 20260507 00:10:43.737 | INFO | PerfReport: 11.65%, Baseline -- 3.61e+00 / 4.34e+00, 5.31e-01 / 3.40e-01 | 20260507 00:10:43.737 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.45e+00, 1.76e-01 / 1.20e-01 | 20260507 00:10:43.737 | INFO | PerfReport: 16.05%, DME -- 3.97e+01 / 4.44e+01, 3.77e+00 / 2.26e+00 | 20260507 00:10:43.737 | INFO | PerfReport: 57.47%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.35e+00 / 1.42e+00 | 20260507 00:10:43.737 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.18e+00 / 2.73e+00, 3.77e-01 / 2.22e-01 | 20260507 00:10:43.737 | INFO | PerfReport: 3.17%, Metrics -- 9.80e-01 / 1.31e+00, 2.61e-01 / 1.32e-01 | 20260507 00:10:43.737 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.33e-01 / 3.85e-01, 5.23e-02 / 2.67e-02 | 20260507 00:10:43.737 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:10:43.737 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:10:43.737 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:10:43.737 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 00:10:43.737 | INFO | PerfReport: 21.50%, Upload -- 2.07e+01 / 0.00e+00, 6.82e+00 / 0.00e+00 | 20260507 00:10:43.737 | INFO | PerfReport: 21.32%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 00:10:43.737 | INFO | PerfReport: 32.08%, Compute -- 3.09e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 00:10:43.737 | INFO | PerfReport: 1.23%, LockUpload -- 1.19e+00 / 0.00e+00, 4.68e+00 / 0.00e+00 | 20260507 00:10:43.737 | INFO | PerfReport: 1.35%, LockDownload -- 1.30e+00 / 0.00e+00, 3.22e+00 / 0.00e+00 | 20260507 00:10:43.737 | INFO | PerfReport: 22.51%, LockCompute -- 2.17e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 00:10:43.737 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260507 00:10:43.833 | INFO | Starting a round of Baz IO | 20260507 00:10:46.960 | INFO | Populating RTMetrics report | 20260507 00:10:46.960 | INFO | RTMetrics report populated | 20260507 00:10:46.960 | INFO | Opening tmp RTMetrics json file | 20260507 00:10:46.961 | INFO | Serializing RTMetrics to json | 20260507 00:10:46.978 | INFO | Serialization done, swapping tmp file now... | 20260507 00:10:46.981 | INFO | RTMetrics done | 20260507 00:10:47.053 | INFO | Finished a round of Baz IO: Wrote 1.858 GiB in 3.219 seconds | 20260507 00:10:47.053 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.33% / 0.00% / 66.66% | 20260507 00:10:47.053 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.363159% / 7.94729e-06% / 6.35783e-05% | 20260507 00:10:47.053 | INFO | Baz IO overall: | 20260507 00:10:47.053 | INFO | BytesWritten : 2898608183346 | 20260507 00:10:47.053 | INFO | NumEvents : 1396683339987 | 20260507 00:10:48.873 | INFO | Analyzing chunk frames = [5554176,5554688) | 20260507 00:10:48.873 | INFO | MemInfo: { | 20260507 00:10:48.873 | INFO | "gpu" : " 17 GB", | 20260507 00:10:48.873 | INFO | "huge" : "", | 20260507 00:10:48.873 | INFO | "huge_shared" : " 27 GB", | 20260507 00:10:48.873 | INFO | "malloc" : " 14 GB", | 20260507 00:10:48.873 | INFO | "pinned" : " 29 GB" | 20260507 00:10:48.873 | INFO | } | 20260507 00:10:48.874 | INFO | Perf report for the last 5.13816 seconds: | 20260507 00:10:48.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:10:48.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:48.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:48.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:48.874 | INFO | ANALYSIS : 62.3%, 3.0, 96.8ms, 0 | 20260507 00:10:48.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:48.874 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260507 00:10:48.874 | INFO | PRE_HQ : 59.7%, 1.8, 57.1ms, 0 | 20260507 00:10:48.874 | INFO | BAZWRITER : 62.0%, 3.3, 106.4ms, 0 | 20260507 00:10:48.874 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 00:10:48.874 | INFO | Uploaded 36GiB to GPU in 2.02588s (17.7701GiB/s) | 20260507 00:10:48.874 | INFO | Downloaded 38GiB to GPU in 2.17664s (17.4581GiB/s) | 20260507 00:10:54.014 | INFO | Analyzing chunk frames = [5554688,5555200) | 20260507 00:10:54.014 | INFO | MemInfo: { | 20260507 00:10:54.014 | INFO | "gpu" : " 17 GB", | 20260507 00:10:54.014 | INFO | "huge" : "", | 20260507 00:10:54.014 | INFO | "huge_shared" : " 27 GB", | 20260507 00:10:54.014 | INFO | "malloc" : " 14 GB", | 20260507 00:10:54.014 | INFO | "pinned" : " 29 GB" | 20260507 00:10:54.014 | INFO | } | 20260507 00:10:54.014 | INFO | Perf report for the last 5.14067 seconds: | 20260507 00:10:54.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:10:54.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:54.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:54.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:54.014 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 00:10:54.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:54.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:54.014 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:10:54.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:54.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:54.015 | INFO | Uploaded 36GiB to GPU in 2.00285s (17.9744GiB/s) | 20260507 00:10:54.015 | INFO | Downloaded 34GiB to GPU in 1.93336s (17.5859GiB/s) | 20260507 00:10:59.161 | INFO | Analyzing chunk frames = [5555200,5555712) | 20260507 00:10:59.161 | INFO | MemInfo: { | 20260507 00:10:59.161 | INFO | "gpu" : " 17 GB", | 20260507 00:10:59.161 | INFO | "huge" : "", | 20260507 00:10:59.161 | INFO | "huge_shared" : " 27 GB", | 20260507 00:10:59.161 | INFO | "malloc" : " 14 GB", | 20260507 00:10:59.161 | INFO | "pinned" : " 29 GB" | 20260507 00:10:59.161 | INFO | } | 20260507 00:10:59.161 | INFO | Perf report for the last 5.14658 seconds: | 20260507 00:10:59.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:10:59.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:59.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:59.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:59.161 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 00:10:59.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:59.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:59.161 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 00:10:59.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:59.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:10:59.161 | INFO | Uploaded 36GiB to GPU in 1.9666s (18.3057GiB/s) | 20260507 00:10:59.161 | INFO | Downloaded 34GiB to GPU in 1.9268s (17.6458GiB/s) | 20260507 00:11:04.307 | INFO | Analyzing chunk frames = [5555712,5556224) | 20260507 00:11:04.307 | INFO | MemInfo: { | 20260507 00:11:04.307 | INFO | "gpu" : " 17 GB", | 20260507 00:11:04.307 | INFO | "huge" : "", | 20260507 00:11:04.307 | INFO | "huge_shared" : " 27 GB", | 20260507 00:11:04.307 | INFO | "malloc" : " 14 GB", | 20260507 00:11:04.307 | INFO | "pinned" : " 29 GB" | 20260507 00:11:04.307 | INFO | } | 20260507 00:11:04.307 | INFO | Perf report for the last 5.14631 seconds: | 20260507 00:11:04.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:11:04.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:04.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:04.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:04.307 | INFO | ANALYSIS : 59.8%, 3.0, 93.0ms, 0 | 20260507 00:11:04.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:04.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:04.307 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:11:04.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:04.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:04.307 | INFO | Uploaded 36GiB to GPU in 1.96126s (18.3556GiB/s) | 20260507 00:11:04.307 | INFO | Downloaded 34GiB to GPU in 1.91079s (17.7937GiB/s) | 20260507 00:11:09.442 | INFO | Analyzing chunk frames = [5556224,5556736) | 20260507 00:11:09.442 | INFO | MemInfo: { | 20260507 00:11:09.442 | INFO | "gpu" : " 17 GB", | 20260507 00:11:09.442 | INFO | "huge" : "", | 20260507 00:11:09.442 | INFO | "huge_shared" : " 27 GB", | 20260507 00:11:09.442 | INFO | "malloc" : " 14 GB", | 20260507 00:11:09.442 | INFO | "pinned" : " 29 GB" | 20260507 00:11:09.442 | INFO | } | 20260507 00:11:09.442 | INFO | Perf report for the last 5.13503 seconds: | 20260507 00:11:09.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:11:09.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:09.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:09.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:09.442 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 00:11:09.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:09.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:09.442 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:11:09.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:09.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:09.442 | INFO | Uploaded 36GiB to GPU in 2.01549s (17.8617GiB/s) | 20260507 00:11:09.442 | INFO | Downloaded 34GiB to GPU in 1.94764s (17.4571GiB/s) | 20260507 00:11:14.581 | INFO | Analyzing chunk frames = [5556736,5557248) | 20260507 00:11:14.581 | INFO | MemInfo: { | 20260507 00:11:14.581 | INFO | "gpu" : " 17 GB", | 20260507 00:11:14.581 | INFO | "huge" : "", | 20260507 00:11:14.581 | INFO | "huge_shared" : " 27 GB", | 20260507 00:11:14.581 | INFO | "malloc" : " 14 GB", | 20260507 00:11:14.581 | INFO | "pinned" : " 29 GB" | 20260507 00:11:14.581 | INFO | } | 20260507 00:11:14.582 | INFO | Perf report for the last 5.13934 seconds: | 20260507 00:11:14.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:11:14.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:14.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:14.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:14.582 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:11:14.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:14.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:14.582 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 00:11:14.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:14.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:14.582 | INFO | Uploaded 36GiB to GPU in 2.04357s (17.6163GiB/s) | 20260507 00:11:14.582 | INFO | Downloaded 34GiB to GPU in 1.96076s (17.3403GiB/s) | 20260507 00:11:19.716 | INFO | Analyzing chunk frames = [5557248,5557760) | 20260507 00:11:19.716 | INFO | MemInfo: { | 20260507 00:11:19.716 | INFO | "gpu" : " 17 GB", | 20260507 00:11:19.716 | INFO | "huge" : "", | 20260507 00:11:19.716 | INFO | "huge_shared" : " 27 GB", | 20260507 00:11:19.716 | INFO | "malloc" : " 14 GB", | 20260507 00:11:19.716 | INFO | "pinned" : " 29 GB" | 20260507 00:11:19.716 | INFO | } | 20260507 00:11:19.716 | INFO | Perf report for the last 5.1348 seconds: | 20260507 00:11:19.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:11:19.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:19.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:19.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:19.716 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260507 00:11:19.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:19.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:19.716 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 00:11:19.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:19.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:19.717 | INFO | Uploaded 36GiB to GPU in 1.96966s (18.2772GiB/s) | 20260507 00:11:19.717 | INFO | Downloaded 34GiB to GPU in 1.99705s (17.0251GiB/s) | 20260507 00:11:24.853 | INFO | Analyzing chunk frames = [5557760,5558272) | 20260507 00:11:24.853 | INFO | MemInfo: { | 20260507 00:11:24.853 | INFO | "gpu" : " 17 GB", | 20260507 00:11:24.853 | INFO | "huge" : "", | 20260507 00:11:24.854 | INFO | "huge_shared" : " 27 GB", | 20260507 00:11:24.854 | INFO | "malloc" : " 14 GB", | 20260507 00:11:24.854 | INFO | "pinned" : " 29 GB" | 20260507 00:11:24.854 | INFO | } | 20260507 00:11:24.854 | INFO | Perf report for the last 5.13725 seconds: | 20260507 00:11:24.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:11:24.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:24.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:24.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:24.854 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:11:24.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:24.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:24.854 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:11:24.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:24.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:24.854 | INFO | Uploaded 36GiB to GPU in 1.97465s (18.2311GiB/s) | 20260507 00:11:24.854 | INFO | Downloaded 34GiB to GPU in 1.99071s (17.0794GiB/s) | 20260507 00:11:24.966 | INFO | Starting a round of Baz IO | 20260507 00:11:28.075 | INFO | Populating RTMetrics report | 20260507 00:11:28.075 | INFO | RTMetrics report populated | 20260507 00:11:28.075 | INFO | Opening tmp RTMetrics json file | 20260507 00:11:28.077 | INFO | Serializing RTMetrics to json | 20260507 00:11:28.094 | INFO | Serialization done, swapping tmp file now... | 20260507 00:11:28.095 | INFO | RTMetrics done | 20260507 00:11:28.167 | INFO | Finished a round of Baz IO: Wrote 1.859 GiB in 3.2 seconds | 20260507 00:11:28.167 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.33% / 0.00% / 66.67% | 20260507 00:11:28.167 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.362547% / 3.97364e-06% / 8.74201e-05% | 20260507 00:11:28.167 | INFO | Baz IO overall: | 20260507 00:11:28.167 | INFO | BytesWritten : 2900604512448 | 20260507 00:11:28.167 | INFO | NumEvents : 1397632858917 | 20260507 00:11:30.001 | INFO | Analyzing chunk frames = [5558272,5558784) | 20260507 00:11:30.001 | INFO | MemInfo: { | 20260507 00:11:30.001 | INFO | "gpu" : " 17 GB", | 20260507 00:11:30.001 | INFO | "huge" : "", | 20260507 00:11:30.001 | INFO | "huge_shared" : " 27 GB", | 20260507 00:11:30.001 | INFO | "malloc" : " 14 GB", | 20260507 00:11:30.001 | INFO | "pinned" : " 29 GB" | 20260507 00:11:30.001 | INFO | } | 20260507 00:11:30.001 | INFO | Perf report for the last 5.14706 seconds: | 20260507 00:11:30.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:11:30.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:30.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:30.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:30.001 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260507 00:11:30.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:30.001 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 00:11:30.001 | INFO | PRE_HQ : 59.5%, 1.8, 56.3ms, 0 | 20260507 00:11:30.001 | INFO | BAZWRITER : 62.2%, 3.3, 106.8ms, 0 | 20260507 00:11:30.001 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:11:30.001 | INFO | Uploaded 36GiB to GPU in 2.00235s (17.9789GiB/s) | 20260507 00:11:30.001 | INFO | Downloaded 38GiB to GPU in 2.18157s (17.4187GiB/s) | 20260507 00:11:35.134 | INFO | Analyzing chunk frames = [5558784,5559296) | 20260507 00:11:35.134 | INFO | MemInfo: { | 20260507 00:11:35.134 | INFO | "gpu" : " 17 GB", | 20260507 00:11:35.134 | INFO | "huge" : "", | 20260507 00:11:35.134 | INFO | "huge_shared" : " 27 GB", | 20260507 00:11:35.134 | INFO | "malloc" : " 14 GB", | 20260507 00:11:35.134 | INFO | "pinned" : " 29 GB" | 20260507 00:11:35.134 | INFO | } | 20260507 00:11:35.134 | INFO | Perf report for the last 5.13381 seconds: | 20260507 00:11:35.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:11:35.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:35.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:35.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:35.135 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:11:35.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:35.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:35.135 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:11:35.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:35.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:35.135 | INFO | Uploaded 36GiB to GPU in 2.02243s (17.8004GiB/s) | 20260507 00:11:35.135 | INFO | Downloaded 34GiB to GPU in 1.93772s (17.5464GiB/s) | 20260507 00:11:40.276 | INFO | Analyzing chunk frames = [5559296,5559808) | 20260507 00:11:40.276 | INFO | MemInfo: { | 20260507 00:11:40.276 | INFO | "gpu" : " 17 GB", | 20260507 00:11:40.276 | INFO | "huge" : "", | 20260507 00:11:40.276 | INFO | "huge_shared" : " 27 GB", | 20260507 00:11:40.276 | INFO | "malloc" : " 14 GB", | 20260507 00:11:40.276 | INFO | "pinned" : " 29 GB" | 20260507 00:11:40.276 | INFO | } | 20260507 00:11:40.276 | INFO | Perf report for the last 5.14148 seconds: | 20260507 00:11:40.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:11:40.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:40.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:40.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:40.276 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 00:11:40.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:40.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:40.276 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:11:40.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:40.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:40.276 | INFO | Uploaded 36GiB to GPU in 1.99073s (18.0838GiB/s) | 20260507 00:11:40.276 | INFO | Downloaded 34GiB to GPU in 1.93398s (17.5803GiB/s) | 20260507 00:11:45.417 | INFO | Analyzing chunk frames = [5559808,5560320) | 20260507 00:11:45.417 | INFO | MemInfo: { | 20260507 00:11:45.417 | INFO | "gpu" : " 17 GB", | 20260507 00:11:45.417 | INFO | "huge" : "", | 20260507 00:11:45.417 | INFO | "huge_shared" : " 27 GB", | 20260507 00:11:45.417 | INFO | "malloc" : " 14 GB", | 20260507 00:11:45.417 | INFO | "pinned" : " 29 GB" | 20260507 00:11:45.417 | INFO | } | 20260507 00:11:45.417 | INFO | Perf report for the last 5.14122 seconds: | 20260507 00:11:45.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:11:45.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:45.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:45.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:45.417 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260507 00:11:45.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:45.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:45.417 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:11:45.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:45.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:45.417 | INFO | Uploaded 36GiB to GPU in 1.95084s (18.4536GiB/s) | 20260507 00:11:45.417 | INFO | Downloaded 34GiB to GPU in 1.9083s (17.8169GiB/s) | 20260507 00:11:50.550 | INFO | Analyzing chunk frames = [5560320,5560832) | 20260507 00:11:50.551 | INFO | MemInfo: { | 20260507 00:11:50.551 | INFO | "gpu" : " 17 GB", | 20260507 00:11:50.551 | INFO | "huge" : "", | 20260507 00:11:50.551 | INFO | "huge_shared" : " 27 GB", | 20260507 00:11:50.551 | INFO | "malloc" : " 14 GB", | 20260507 00:11:50.551 | INFO | "pinned" : " 29 GB" | 20260507 00:11:50.551 | INFO | } | 20260507 00:11:50.551 | INFO | Perf report for the last 5.13347 seconds: | 20260507 00:11:50.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:11:50.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:50.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:50.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:50.551 | INFO | ANALYSIS : 60.2%, 3.0, 93.5ms, 0 | 20260507 00:11:50.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:50.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:50.551 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:11:50.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:50.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:50.551 | INFO | Uploaded 36GiB to GPU in 2.00089s (17.992GiB/s) | 20260507 00:11:50.551 | INFO | Downloaded 34GiB to GPU in 1.95256s (17.413GiB/s) | 20260507 00:11:55.688 | INFO | Analyzing chunk frames = [5560832,5561344) | 20260507 00:11:55.688 | INFO | MemInfo: { | 20260507 00:11:55.688 | INFO | "gpu" : " 17 GB", | 20260507 00:11:55.688 | INFO | "huge" : "", | 20260507 00:11:55.688 | INFO | "huge_shared" : " 27 GB", | 20260507 00:11:55.688 | INFO | "malloc" : " 14 GB", | 20260507 00:11:55.688 | INFO | "pinned" : " 29 GB" | 20260507 00:11:55.688 | INFO | } | 20260507 00:11:55.688 | INFO | Perf report for the last 5.13769 seconds: | 20260507 00:11:55.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:11:55.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:55.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:55.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:55.688 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 00:11:55.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:55.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:55.689 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 00:11:55.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:55.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:11:55.689 | INFO | Uploaded 36GiB to GPU in 2.00922s (17.9174GiB/s) | 20260507 00:11:55.689 | INFO | Downloaded 34GiB to GPU in 1.93818s (17.5422GiB/s) | 20260507 00:12:00.825 | INFO | Analyzing chunk frames = [5561344,5561856) | 20260507 00:12:00.825 | INFO | MemInfo: { | 20260507 00:12:00.825 | INFO | "gpu" : " 17 GB", | 20260507 00:12:00.825 | INFO | "huge" : "", | 20260507 00:12:00.825 | INFO | "huge_shared" : " 27 GB", | 20260507 00:12:00.825 | INFO | "malloc" : " 14 GB", | 20260507 00:12:00.825 | INFO | "pinned" : " 29 GB" | 20260507 00:12:00.825 | INFO | } | 20260507 00:12:00.826 | INFO | Perf report for the last 5.13713 seconds: | 20260507 00:12:00.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:12:00.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:00.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:00.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:00.826 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:12:00.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:00.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:00.826 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:12:00.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:00.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:00.826 | INFO | Uploaded 36GiB to GPU in 1.95759s (18.3899GiB/s) | 20260507 00:12:00.826 | INFO | Downloaded 34GiB to GPU in 1.98263s (17.149GiB/s) | 20260507 00:12:05.971 | INFO | Analyzing chunk frames = [5561856,5562368) | 20260507 00:12:05.971 | INFO | MemInfo: { | 20260507 00:12:05.971 | INFO | "gpu" : " 17 GB", | 20260507 00:12:05.971 | INFO | "huge" : "", | 20260507 00:12:05.971 | INFO | "huge_shared" : " 27 GB", | 20260507 00:12:05.971 | INFO | "malloc" : " 14 GB", | 20260507 00:12:05.971 | INFO | "pinned" : " 29 GB" | 20260507 00:12:05.971 | INFO | } | 20260507 00:12:05.971 | INFO | Perf report for the last 5.14536 seconds: | 20260507 00:12:05.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:12:05.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:05.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:05.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:05.971 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:12:05.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:05.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:05.971 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 00:12:05.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:05.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:05.971 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:12:05.971 | INFO | Total mem usage: 18 GiB | 20260507 00:12:05.971 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:12:05.971 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:12:05.971 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:12:05.971 | INFO | High Water Breakdown: | 20260507 00:12:05.971 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:12:05.971 | INFO | 39.7% : 5.67 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:12:05.971 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:12:05.971 | INFO | ------------------------------------------ | 20260507 00:12:05.971 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:12:05.971 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:12:05.971 | INFO | Total mem usage: 26.5 GiB | 20260507 00:12:05.971 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:12:05.971 | INFO | Uploaded 36GiB to GPU in 1.98786s (18.1099GiB/s) | 20260507 00:12:05.971 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:12:05.971 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:12:05.971 | INFO | High Water Breakdown: | 20260507 00:12:05.971 | INFO | Downloaded 34GiB to GPU in 1.98162s (17.1577GiB/s) | 20260507 00:12:05.971 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:12:05.972 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:12:05.972 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:12:05.972 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:12:05.972 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:12:05.972 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:12:05.972 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:12:05.972 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:12:05.972 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:12:05.972 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:12:05.972 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:12:05.972 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:12:05.972 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:12:05.972 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:12:05.972 | INFO | ------------------------------------------ | 20260507 00:12:05.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:12:05.972 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:12:05.972 | INFO | Total mem usage: 25.6 GiB | 20260507 00:12:05.972 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:12:05.972 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:12:05.972 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:12:05.972 | INFO | High Water Breakdown: | 20260507 00:12:05.972 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:12:05.972 | INFO | ------------------------------------------ | 20260507 00:12:05.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:12:05.972 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:12:05.972 | INFO | Total mem usage: 15.7 GiB | 20260507 00:12:05.972 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:12:05.972 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:12:05.972 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:12:05.972 | INFO | High Water Breakdown: | 20260507 00:12:05.972 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:12:05.972 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:12:05.972 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:12:05.972 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:12:05.972 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:12:05.972 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:12:05.972 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:12:05.972 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:12:05.972 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:12:05.972 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:12:05.972 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:12:05.972 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:12:05.972 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:12:05.972 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:12:05.972 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:12:05.972 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:12:05.972 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:12:05.972 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:12:05.972 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:12:05.972 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:12:05.972 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:12:05.972 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:12:05.972 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:12:05.972 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:12:05.972 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:12:05.972 | INFO | ------------------------------------------ | 20260507 00:12:05.972 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:12:05.972 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:12:05.972 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:12:05.972 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:12:05.972 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.59e+01, 1.31e+01 / 2.43e+00 | 20260507 00:12:05.972 | INFO | PerfReport: 0.30%, AggregateStats -- 9.14e-02 / 1.03e-01, 9.89e-03 / 6.00e-03 | 20260507 00:12:05.972 | INFO | PerfReport: 11.68%, Baseline -- 3.61e+00 / 4.42e+00, 5.38e-01 / 2.86e-01 | 20260507 00:12:05.972 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.45e+00, 1.74e-01 / 1.24e-01 | 20260507 00:12:05.972 | INFO | PerfReport: 16.13%, DME -- 3.99e+01 / 4.50e+01, 4.20e+00 / 2.96e+00 | 20260507 00:12:05.972 | INFO | PerfReport: 57.34%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.28e+00 / 1.69e+00 | 20260507 00:12:05.972 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.18e+00 / 2.74e+00, 3.82e-01 / 2.06e-01 | 20260507 00:12:05.972 | INFO | PerfReport: 3.17%, Metrics -- 9.81e-01 / 1.33e+00, 2.57e-01 / 1.37e-01 | 20260507 00:12:05.972 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.33e-01 / 3.86e-01, 5.16e-02 / 2.04e-02 | 20260507 00:12:05.972 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:12:05.972 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:12:05.972 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:12:05.972 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 00:12:05.972 | INFO | PerfReport: 21.49%, Upload -- 2.08e+01 / 0.00e+00, 6.91e+00 / 0.00e+00 | 20260507 00:12:05.972 | INFO | PerfReport: 21.34%, Download -- 2.06e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260507 00:12:05.972 | INFO | PerfReport: 32.03%, Compute -- 3.09e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 00:12:05.972 | INFO | PerfReport: 1.25%, LockUpload -- 1.21e+00 / 0.00e+00, 4.69e+00 / 0.00e+00 | 20260507 00:12:05.972 | INFO | PerfReport: 1.44%, LockDownload -- 1.39e+00 / 0.00e+00, 3.36e+00 / 0.00e+00 | 20260507 00:12:05.972 | INFO | PerfReport: 22.45%, LockCompute -- 2.17e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 00:12:05.972 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260507 00:12:06.068 | INFO | Starting a round of Baz IO | 20260507 00:12:09.192 | INFO | Populating RTMetrics report | 20260507 00:12:09.192 | INFO | RTMetrics report populated | 20260507 00:12:09.192 | INFO | Opening tmp RTMetrics json file | 20260507 00:12:09.213 | INFO | Serializing RTMetrics to json | 20260507 00:12:09.214 | INFO | Serialization done, swapping tmp file now... | 20260507 00:12:09.214 | INFO | RTMetrics done | 20260507 00:12:09.289 | INFO | Finished a round of Baz IO: Wrote 1.858 GiB in 3.22 seconds | 20260507 00:12:09.289 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.33% / 0.00% / 66.67% | 20260507 00:12:09.289 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.361435% / 0% / 9.13938e-05% | 20260507 00:12:09.289 | INFO | Baz IO overall: | 20260507 00:12:09.289 | INFO | BytesWritten : 2902599140797 | 20260507 00:12:09.289 | INFO | NumEvents : 1398581203929 | 20260507 00:12:11.113 | INFO | Analyzing chunk frames = [5562368,5562880) | 20260507 00:12:11.113 | INFO | MemInfo: { | 20260507 00:12:11.113 | INFO | "gpu" : " 17 GB", | 20260507 00:12:11.113 | INFO | "huge" : "", | 20260507 00:12:11.113 | INFO | "huge_shared" : " 27 GB", | 20260507 00:12:11.113 | INFO | "malloc" : " 14 GB", | 20260507 00:12:11.113 | INFO | "pinned" : " 29 GB" | 20260507 00:12:11.113 | INFO | } | 20260507 00:12:11.113 | INFO | Perf report for the last 5.14168 seconds: | 20260507 00:12:11.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:12:11.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:11.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:11.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:11.113 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 00:12:11.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:11.113 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 00:12:11.113 | INFO | PRE_HQ : 59.5%, 1.8, 56.2ms, 0 | 20260507 00:12:11.113 | INFO | BAZWRITER : 62.2%, 3.4, 107.2ms, 0 | 20260507 00:12:11.113 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:12:11.113 | INFO | Uploaded 36GiB to GPU in 2.01971s (17.8243GiB/s) | 20260507 00:12:11.113 | INFO | Downloaded 38GiB to GPU in 2.17703s (17.4549GiB/s) | 20260507 00:12:16.258 | INFO | Analyzing chunk frames = [5562880,5563392) | 20260507 00:12:16.258 | INFO | MemInfo: { | 20260507 00:12:16.258 | INFO | "gpu" : " 17 GB", | 20260507 00:12:16.258 | INFO | "huge" : "", | 20260507 00:12:16.258 | INFO | "huge_shared" : " 27 GB", | 20260507 00:12:16.258 | INFO | "malloc" : " 14 GB", | 20260507 00:12:16.258 | INFO | "pinned" : " 29 GB" | 20260507 00:12:16.258 | INFO | } | 20260507 00:12:16.258 | INFO | Perf report for the last 5.14573 seconds: | 20260507 00:12:16.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:12:16.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:16.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:16.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:16.258 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 00:12:16.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:16.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:16.258 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:12:16.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:16.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:16.259 | INFO | Uploaded 36GiB to GPU in 2.01485s (17.8673GiB/s) | 20260507 00:12:16.259 | INFO | Downloaded 34GiB to GPU in 1.93098s (17.6076GiB/s) | 20260507 00:12:21.392 | INFO | Analyzing chunk frames = [5563392,5563904) | 20260507 00:12:21.392 | INFO | MemInfo: { | 20260507 00:12:21.392 | INFO | "gpu" : " 17 GB", | 20260507 00:12:21.392 | INFO | "huge" : "", | 20260507 00:12:21.392 | INFO | "huge_shared" : " 27 GB", | 20260507 00:12:21.392 | INFO | "malloc" : " 14 GB", | 20260507 00:12:21.392 | INFO | "pinned" : " 29 GB" | 20260507 00:12:21.392 | INFO | } | 20260507 00:12:21.392 | INFO | Perf report for the last 5.13396 seconds: | 20260507 00:12:21.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:12:21.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:21.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:21.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:21.392 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 00:12:21.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:21.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:21.392 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:12:21.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:21.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:21.393 | INFO | Uploaded 36GiB to GPU in 2.0192s (17.8289GiB/s) | 20260507 00:12:21.393 | INFO | Downloaded 34GiB to GPU in 1.93611s (17.561GiB/s) | 20260507 00:12:26.530 | INFO | Analyzing chunk frames = [5563904,5564416) | 20260507 00:12:26.530 | INFO | MemInfo: { | 20260507 00:12:26.530 | INFO | "gpu" : " 17 GB", | 20260507 00:12:26.530 | INFO | "huge" : "", | 20260507 00:12:26.530 | INFO | "huge_shared" : " 27 GB", | 20260507 00:12:26.530 | INFO | "malloc" : " 14 GB", | 20260507 00:12:26.530 | INFO | "pinned" : " 29 GB" | 20260507 00:12:26.530 | INFO | } | 20260507 00:12:26.530 | INFO | Perf report for the last 5.13794 seconds: | 20260507 00:12:26.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:12:26.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:26.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:26.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:26.530 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:12:26.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:26.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:26.530 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:12:26.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:26.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:26.530 | INFO | Uploaded 36GiB to GPU in 1.97834s (18.1971GiB/s) | 20260507 00:12:26.531 | INFO | Downloaded 34GiB to GPU in 1.90764s (17.8231GiB/s) | 20260507 00:12:31.669 | INFO | Analyzing chunk frames = [5564416,5564928) | 20260507 00:12:31.669 | INFO | MemInfo: { | 20260507 00:12:31.669 | INFO | "gpu" : " 17 GB", | 20260507 00:12:31.669 | INFO | "huge" : "", | 20260507 00:12:31.669 | INFO | "huge_shared" : " 27 GB", | 20260507 00:12:31.669 | INFO | "malloc" : " 14 GB", | 20260507 00:12:31.669 | INFO | "pinned" : " 29 GB" | 20260507 00:12:31.669 | INFO | } | 20260507 00:12:31.669 | INFO | Perf report for the last 5.13857 seconds: | 20260507 00:12:31.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:12:31.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:31.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:31.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:31.669 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 00:12:31.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:31.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:31.669 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:12:31.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:31.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:31.669 | INFO | Uploaded 36GiB to GPU in 1.99691s (18.0279GiB/s) | 20260507 00:12:31.669 | INFO | Downloaded 34GiB to GPU in 1.92537s (17.659GiB/s) | 20260507 00:12:36.801 | INFO | Analyzing chunk frames = [5564928,5565440) | 20260507 00:12:36.801 | INFO | MemInfo: { | 20260507 00:12:36.801 | INFO | "gpu" : " 17 GB", | 20260507 00:12:36.801 | INFO | "huge" : "", | 20260507 00:12:36.801 | INFO | "huge_shared" : " 27 GB", | 20260507 00:12:36.801 | INFO | "malloc" : " 14 GB", | 20260507 00:12:36.801 | INFO | "pinned" : " 29 GB" | 20260507 00:12:36.801 | INFO | } | 20260507 00:12:36.801 | INFO | Perf report for the last 5.1322 seconds: | 20260507 00:12:36.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:12:36.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:36.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:36.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:36.801 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 00:12:36.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:36.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:36.801 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:12:36.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:36.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:36.801 | INFO | Uploaded 36GiB to GPU in 2.01561s (17.8606GiB/s) | 20260507 00:12:36.801 | INFO | Downloaded 34GiB to GPU in 1.93782s (17.5455GiB/s) | 20260507 00:12:41.946 | INFO | Analyzing chunk frames = [5565440,5565952) | 20260507 00:12:41.946 | INFO | MemInfo: { | 20260507 00:12:41.946 | INFO | "gpu" : " 17 GB", | 20260507 00:12:41.946 | INFO | "huge" : "", | 20260507 00:12:41.946 | INFO | "huge_shared" : " 27 GB", | 20260507 00:12:41.946 | INFO | "malloc" : " 14 GB", | 20260507 00:12:41.946 | INFO | "pinned" : " 29 GB" | 20260507 00:12:41.946 | INFO | } | 20260507 00:12:41.946 | INFO | Perf report for the last 5.14537 seconds: | 20260507 00:12:41.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:12:41.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:41.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:41.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:41.947 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 00:12:41.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:41.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:41.947 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 00:12:41.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:41.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:41.947 | INFO | Uploaded 36GiB to GPU in 1.9706s (18.2686GiB/s) | 20260507 00:12:41.947 | INFO | Downloaded 34GiB to GPU in 1.97866s (17.1833GiB/s) | 20260507 00:12:47.087 | INFO | Analyzing chunk frames = [5565952,5566464) | 20260507 00:12:47.087 | INFO | MemInfo: { | 20260507 00:12:47.087 | INFO | "gpu" : " 17 GB", | 20260507 00:12:47.087 | INFO | "huge" : "", | 20260507 00:12:47.087 | INFO | "huge_shared" : " 27 GB", | 20260507 00:12:47.087 | INFO | "malloc" : " 14 GB", | 20260507 00:12:47.087 | INFO | "pinned" : " 29 GB" | 20260507 00:12:47.087 | INFO | } | 20260507 00:12:47.087 | INFO | Perf report for the last 5.14068 seconds: | 20260507 00:12:47.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:12:47.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:47.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:47.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:47.087 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:12:47.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:47.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:47.087 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:12:47.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:47.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:47.087 | INFO | Uploaded 36GiB to GPU in 1.98079s (18.1745GiB/s) | 20260507 00:12:47.087 | INFO | Downloaded 34GiB to GPU in 1.9787s (17.183GiB/s) | 20260507 00:12:47.184 | INFO | Starting a round of Baz IO | 20260507 00:12:50.309 | INFO | Populating RTMetrics report | 20260507 00:12:50.309 | INFO | RTMetrics report populated | 20260507 00:12:50.309 | INFO | Opening tmp RTMetrics json file | 20260507 00:12:50.310 | INFO | Serializing RTMetrics to json | 20260507 00:12:50.311 | INFO | Serialization done, swapping tmp file now... | 20260507 00:12:50.311 | INFO | RTMetrics done | 20260507 00:12:50.407 | INFO | Finished a round of Baz IO: Wrote 1.861 GiB in 3.222 seconds | 20260507 00:12:50.407 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.33% / 0.00% / 66.67% | 20260507 00:12:50.407 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.356861% / 7.94729e-06% / 7.94729e-05% | 20260507 00:12:50.407 | INFO | Baz IO overall: | 20260507 00:12:50.407 | INFO | BytesWritten : 2904597198087 | 20260507 00:12:50.407 | INFO | NumEvents : 1399528433219 | 20260507 00:12:52.226 | INFO | Analyzing chunk frames = [5566464,5566976) | 20260507 00:12:52.226 | INFO | MemInfo: { | 20260507 00:12:52.226 | INFO | "gpu" : " 17 GB", | 20260507 00:12:52.226 | INFO | "huge" : "", | 20260507 00:12:52.226 | INFO | "huge_shared" : " 27 GB", | 20260507 00:12:52.226 | INFO | "malloc" : " 14 GB", | 20260507 00:12:52.226 | INFO | "pinned" : " 29 GB" | 20260507 00:12:52.226 | INFO | } | 20260507 00:12:52.226 | INFO | Perf report for the last 5.13882 seconds: | 20260507 00:12:52.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:12:52.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:52.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:52.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:52.226 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260507 00:12:52.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:52.226 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 00:12:52.226 | INFO | PRE_HQ : 59.9%, 1.8, 56.7ms, 0 | 20260507 00:12:52.226 | INFO | BAZWRITER : 62.3%, 3.4, 108.6ms, 0 | 20260507 00:12:52.226 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 00:12:52.226 | INFO | Uploaded 36GiB to GPU in 2.02808s (17.7508GiB/s) | 20260507 00:12:52.226 | INFO | Downloaded 38GiB to GPU in 2.17661s (17.4583GiB/s) | 20260507 00:12:57.365 | INFO | Analyzing chunk frames = [5566976,5567488) | 20260507 00:12:57.365 | INFO | MemInfo: { | 20260507 00:12:57.365 | INFO | "gpu" : " 17 GB", | 20260507 00:12:57.365 | INFO | "huge" : "", | 20260507 00:12:57.365 | INFO | "huge_shared" : " 27 GB", | 20260507 00:12:57.365 | INFO | "malloc" : " 14 GB", | 20260507 00:12:57.365 | INFO | "pinned" : " 29 GB" | 20260507 00:12:57.365 | INFO | } | 20260507 00:12:57.365 | INFO | Perf report for the last 5.13895 seconds: | 20260507 00:12:57.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:12:57.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:57.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:57.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:57.365 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:12:57.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:57.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:57.365 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:12:57.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:57.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:12:57.365 | INFO | Uploaded 36GiB to GPU in 2.00605s (17.9457GiB/s) | 20260507 00:12:57.365 | INFO | Downloaded 34GiB to GPU in 1.93352s (17.5845GiB/s) | 20260507 00:13:02.503 | INFO | Analyzing chunk frames = [5567488,5568000) | 20260507 00:13:02.503 | INFO | MemInfo: { | 20260507 00:13:02.503 | INFO | "gpu" : " 17 GB", | 20260507 00:13:02.503 | INFO | "huge" : "", | 20260507 00:13:02.503 | INFO | "huge_shared" : " 27 GB", | 20260507 00:13:02.503 | INFO | "malloc" : " 14 GB", | 20260507 00:13:02.503 | INFO | "pinned" : " 29 GB" | 20260507 00:13:02.503 | INFO | } | 20260507 00:13:02.504 | INFO | Perf report for the last 5.13858 seconds: | 20260507 00:13:02.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:13:02.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:02.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:02.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:02.504 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 00:13:02.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:02.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:02.504 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:13:02.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:02.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:02.504 | INFO | Uploaded 36GiB to GPU in 1.99425s (18.0519GiB/s) | 20260507 00:13:02.504 | INFO | Downloaded 34GiB to GPU in 1.9373s (17.5502GiB/s) | 20260507 00:13:07.637 | INFO | Analyzing chunk frames = [5568000,5568512) | 20260507 00:13:07.637 | INFO | MemInfo: { | 20260507 00:13:07.637 | INFO | "gpu" : " 17 GB", | 20260507 00:13:07.637 | INFO | "huge" : "", | 20260507 00:13:07.637 | INFO | "huge_shared" : " 27 GB", | 20260507 00:13:07.637 | INFO | "malloc" : " 14 GB", | 20260507 00:13:07.637 | INFO | "pinned" : " 29 GB" | 20260507 00:13:07.637 | INFO | } | 20260507 00:13:07.637 | INFO | Perf report for the last 5.1337 seconds: | 20260507 00:13:07.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:13:07.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:07.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:07.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:07.637 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 00:13:07.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:07.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:07.637 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:13:07.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:07.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:07.637 | INFO | Uploaded 36GiB to GPU in 1.99986s (18.0013GiB/s) | 20260507 00:13:07.638 | INFO | Downloaded 34GiB to GPU in 1.90894s (17.8109GiB/s) | 20260507 00:13:12.776 | INFO | Analyzing chunk frames = [5568512,5569024) | 20260507 00:13:12.776 | INFO | MemInfo: { | 20260507 00:13:12.776 | INFO | "gpu" : " 17 GB", | 20260507 00:13:12.776 | INFO | "huge" : "", | 20260507 00:13:12.776 | INFO | "huge_shared" : " 27 GB", | 20260507 00:13:12.777 | INFO | "malloc" : " 14 GB", | 20260507 00:13:12.777 | INFO | "pinned" : " 29 GB" | 20260507 00:13:12.777 | INFO | } | 20260507 00:13:12.777 | INFO | Perf report for the last 5.13939 seconds: | 20260507 00:13:12.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:13:12.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:12.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:12.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:12.777 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 00:13:12.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:12.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:12.777 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:13:12.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:12.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:12.777 | INFO | Uploaded 36GiB to GPU in 1.99295s (18.0636GiB/s) | 20260507 00:13:12.777 | INFO | Downloaded 34GiB to GPU in 1.91986s (17.7096GiB/s) | 20260507 00:13:17.923 | INFO | Analyzing chunk frames = [5569024,5569536) | 20260507 00:13:17.923 | INFO | MemInfo: { | 20260507 00:13:17.923 | INFO | "gpu" : " 17 GB", | 20260507 00:13:17.923 | INFO | "huge" : "", | 20260507 00:13:17.923 | INFO | "huge_shared" : " 27 GB", | 20260507 00:13:17.923 | INFO | "malloc" : " 14 GB", | 20260507 00:13:17.923 | INFO | "pinned" : " 29 GB" | 20260507 00:13:17.923 | INFO | } | 20260507 00:13:17.923 | INFO | Perf report for the last 5.14635 seconds: | 20260507 00:13:17.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:13:17.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:17.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:17.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:17.923 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 00:13:17.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:17.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:17.923 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 00:13:17.923 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:17.923 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:17.923 | INFO | Uploaded 36GiB to GPU in 2.00747s (17.933GiB/s) | 20260507 00:13:17.923 | INFO | Downloaded 34GiB to GPU in 1.95163s (17.4213GiB/s) | 20260507 00:13:23.071 | INFO | Analyzing chunk frames = [5569536,5570048) | 20260507 00:13:23.071 | INFO | MemInfo: { | 20260507 00:13:23.071 | INFO | "gpu" : " 17 GB", | 20260507 00:13:23.071 | INFO | "huge" : "", | 20260507 00:13:23.071 | INFO | "huge_shared" : " 27 GB", | 20260507 00:13:23.071 | INFO | "malloc" : " 14 GB", | 20260507 00:13:23.071 | INFO | "pinned" : " 29 GB" | 20260507 00:13:23.071 | INFO | } | 20260507 00:13:23.071 | INFO | Perf report for the last 5.14821 seconds: | 20260507 00:13:23.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:13:23.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:23.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:23.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:23.071 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260507 00:13:23.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:23.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:23.071 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:13:23.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:23.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:23.072 | INFO | Uploaded 36GiB to GPU in 1.95931s (18.3738GiB/s) | 20260507 00:13:23.072 | INFO | Downloaded 34GiB to GPU in 1.99539s (17.0393GiB/s) | 20260507 00:13:28.203 | INFO | Analyzing chunk frames = [5570048,5570560) | 20260507 00:13:28.203 | INFO | MemInfo: { | 20260507 00:13:28.203 | INFO | "gpu" : " 17 GB", | 20260507 00:13:28.203 | INFO | "huge" : "", | 20260507 00:13:28.203 | INFO | "huge_shared" : " 27 GB", | 20260507 00:13:28.203 | INFO | "malloc" : " 14 GB", | 20260507 00:13:28.203 | INFO | "pinned" : " 29 GB" | 20260507 00:13:28.203 | INFO | } | 20260507 00:13:28.203 | INFO | Perf report for the last 5.13189 seconds: | 20260507 00:13:28.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:13:28.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:28.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:28.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:28.203 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260507 00:13:28.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:28.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:28.203 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:13:28.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:28.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:28.204 | INFO | Uploaded 36GiB to GPU in 2.00003s (17.9997GiB/s) | 20260507 00:13:28.204 | INFO | Downloaded 34GiB to GPU in 1.98571s (17.1223GiB/s) | 20260507 00:13:28.204 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:13:28.204 | INFO | Total mem usage: 18 GiB | 20260507 00:13:28.204 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:13:28.204 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:13:28.204 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:13:28.204 | INFO | High Water Breakdown: | 20260507 00:13:28.204 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:13:28.204 | INFO | 39.7% : 5.67 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:13:28.204 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:13:28.204 | INFO | ------------------------------------------ | 20260507 00:13:28.204 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:13:28.204 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:13:28.204 | INFO | Total mem usage: 26.5 GiB | 20260507 00:13:28.204 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:13:28.204 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:13:28.204 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:13:28.204 | INFO | High Water Breakdown: | 20260507 00:13:28.204 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:13:28.204 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:13:28.204 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:13:28.204 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:13:28.204 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:13:28.204 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:13:28.204 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:13:28.204 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:13:28.204 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:13:28.204 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:13:28.204 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:13:28.204 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:13:28.204 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:13:28.204 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:13:28.204 | INFO | ------------------------------------------ | 20260507 00:13:28.204 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:13:28.204 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:13:28.204 | INFO | Total mem usage: 25.6 GiB | 20260507 00:13:28.204 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:13:28.204 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:13:28.204 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:13:28.204 | INFO | High Water Breakdown: | 20260507 00:13:28.204 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:13:28.204 | INFO | ------------------------------------------ | 20260507 00:13:28.204 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:13:28.204 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:13:28.204 | INFO | Total mem usage: 15.7 GiB | 20260507 00:13:28.204 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:13:28.204 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:13:28.204 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:13:28.204 | INFO | High Water Breakdown: | 20260507 00:13:28.204 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:13:28.204 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:13:28.204 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:13:28.205 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:13:28.205 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:13:28.205 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:13:28.205 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:13:28.205 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:13:28.205 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:13:28.205 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:13:28.205 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:13:28.205 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:13:28.205 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:13:28.205 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:13:28.205 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:13:28.205 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:13:28.205 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:13:28.205 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:13:28.205 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:13:28.205 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:13:28.205 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:13:28.205 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:13:28.205 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:13:28.205 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:13:28.205 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:13:28.205 | INFO | ------------------------------------------ | 20260507 00:13:28.205 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:13:28.205 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:13:28.205 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:13:28.205 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:13:28.205 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.56e+01, 1.35e+01 / 2.02e+00 | 20260507 00:13:28.205 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 1.02e-01, 1.03e-02 / 4.00e-03 | 20260507 00:13:28.205 | INFO | PerfReport: 11.66%, Baseline -- 3.61e+00 / 4.41e+00, 5.41e-01 / 2.92e-01 | 20260507 00:13:28.205 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.47e+00, 1.78e-01 / 1.10e-01 | 20260507 00:13:28.205 | INFO | PerfReport: 16.04%, DME -- 3.97e+01 / 4.40e+01, 3.92e+00 / 2.56e+00 | 20260507 00:13:28.205 | INFO | PerfReport: 57.48%, FrameLabeling -- 1.78e+01 / 2.13e+01, 1.37e+00 / 1.89e+00 | 20260507 00:13:28.205 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.18e+00 / 2.69e+00, 3.77e-01 / 2.52e-01 | 20260507 00:13:28.205 | INFO | PerfReport: 3.17%, Metrics -- 9.81e-01 / 1.30e+00, 2.63e-01 / 1.83e-01 | 20260507 00:13:28.205 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.33e-01 / 3.87e-01, 5.32e-02 / 3.15e-02 | 20260507 00:13:28.205 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:13:28.205 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:13:28.205 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:13:28.205 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.31e+01 / 0.00e+00 | 20260507 00:13:28.205 | INFO | PerfReport: 21.58%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260507 00:13:28.205 | INFO | PerfReport: 21.31%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 00:13:28.205 | INFO | PerfReport: 32.07%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:13:28.205 | INFO | PerfReport: 1.29%, LockUpload -- 1.24e+00 / 0.00e+00, 4.76e+00 / 0.00e+00 | 20260507 00:13:28.205 | INFO | PerfReport: 1.25%, LockDownload -- 1.21e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260507 00:13:28.205 | INFO | PerfReport: 22.48%, LockCompute -- 2.17e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 00:13:28.205 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260507 00:13:28.302 | INFO | Starting a round of Baz IO | 20260507 00:13:31.417 | INFO | Populating RTMetrics report | 20260507 00:13:31.417 | INFO | RTMetrics report populated | 20260507 00:13:31.417 | INFO | Opening tmp RTMetrics json file | 20260507 00:13:31.418 | INFO | Serializing RTMetrics to json | 20260507 00:13:31.420 | INFO | Serialization done, swapping tmp file now... | 20260507 00:13:31.420 | INFO | RTMetrics done | 20260507 00:13:31.514 | INFO | Finished a round of Baz IO: Wrote 1.861 GiB in 3.211 seconds | 20260507 00:13:31.514 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.33% / 0.00% / 66.67% | 20260507 00:13:31.514 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.356424% / 7.94729e-06% / 7.15256e-05% | 20260507 00:13:31.514 | INFO | Baz IO overall: | 20260507 00:13:31.514 | INFO | BytesWritten : 2906595188707 | 20260507 00:13:31.514 | INFO | NumEvents : 1400475633649 | 20260507 00:13:33.341 | INFO | Analyzing chunk frames = [5570560,5571072) | 20260507 00:13:33.341 | INFO | MemInfo: { | 20260507 00:13:33.341 | INFO | "gpu" : " 17 GB", | 20260507 00:13:33.341 | INFO | "huge" : "", | 20260507 00:13:33.341 | INFO | "huge_shared" : " 27 GB", | 20260507 00:13:33.341 | INFO | "malloc" : " 14 GB", | 20260507 00:13:33.341 | INFO | "pinned" : " 29 GB" | 20260507 00:13:33.341 | INFO | } | 20260507 00:13:33.341 | INFO | Perf report for the last 5.13772 seconds: | 20260507 00:13:33.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:13:33.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:33.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:33.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:33.341 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260507 00:13:33.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:33.341 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 00:13:33.341 | INFO | PRE_HQ : 59.6%, 1.8, 56.4ms, 0 | 20260507 00:13:33.341 | INFO | BAZWRITER : 62.1%, 3.4, 109.4ms, 0 | 20260507 00:13:33.341 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:13:33.341 | INFO | Uploaded 36GiB to GPU in 2.00663s (17.9405GiB/s) | 20260507 00:13:33.341 | INFO | Downloaded 38GiB to GPU in 2.1821s (17.4144GiB/s) | 20260507 00:13:38.479 | INFO | Analyzing chunk frames = [5571072,5571584) | 20260507 00:13:38.479 | INFO | MemInfo: { | 20260507 00:13:38.479 | INFO | "gpu" : " 17 GB", | 20260507 00:13:38.479 | INFO | "huge" : "", | 20260507 00:13:38.479 | INFO | "huge_shared" : " 27 GB", | 20260507 00:13:38.479 | INFO | "malloc" : " 14 GB", | 20260507 00:13:38.479 | INFO | "pinned" : " 29 GB" | 20260507 00:13:38.479 | INFO | } | 20260507 00:13:38.479 | INFO | Perf report for the last 5.13841 seconds: | 20260507 00:13:38.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:13:38.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:38.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:38.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:38.479 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:13:38.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:38.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:38.479 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:13:38.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:38.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:38.480 | INFO | Uploaded 36GiB to GPU in 1.99136s (18.0781GiB/s) | 20260507 00:13:38.480 | INFO | Downloaded 34GiB to GPU in 1.93003s (17.6163GiB/s) | 20260507 00:13:43.617 | INFO | Analyzing chunk frames = [5571584,5572096) | 20260507 00:13:43.617 | INFO | MemInfo: { | 20260507 00:13:43.617 | INFO | "gpu" : " 17 GB", | 20260507 00:13:43.617 | INFO | "huge" : "", | 20260507 00:13:43.617 | INFO | "huge_shared" : " 27 GB", | 20260507 00:13:43.618 | INFO | "malloc" : " 14 GB", | 20260507 00:13:43.618 | INFO | "pinned" : " 29 GB" | 20260507 00:13:43.618 | INFO | } | 20260507 00:13:43.618 | INFO | Perf report for the last 5.13874 seconds: | 20260507 00:13:43.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:13:43.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:43.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:43.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:43.618 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:13:43.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:43.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:43.618 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:13:43.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:43.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:43.618 | INFO | Uploaded 36GiB to GPU in 1.96796s (18.2931GiB/s) | 20260507 00:13:43.618 | INFO | Downloaded 34GiB to GPU in 1.92717s (17.6424GiB/s) | 20260507 00:13:48.761 | INFO | Analyzing chunk frames = [5572096,5572608) | 20260507 00:13:48.761 | INFO | MemInfo: { | 20260507 00:13:48.761 | INFO | "gpu" : " 17 GB", | 20260507 00:13:48.761 | INFO | "huge" : "", | 20260507 00:13:48.761 | INFO | "huge_shared" : " 27 GB", | 20260507 00:13:48.761 | INFO | "malloc" : " 14 GB", | 20260507 00:13:48.761 | INFO | "pinned" : " 29 GB" | 20260507 00:13:48.761 | INFO | } | 20260507 00:13:48.761 | INFO | Perf report for the last 5.14333 seconds: | 20260507 00:13:48.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:13:48.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:48.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:48.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:48.761 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260507 00:13:48.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:48.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:48.761 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:13:48.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:48.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:48.762 | INFO | Uploaded 36GiB to GPU in 1.95955s (18.3715GiB/s) | 20260507 00:13:48.762 | INFO | Downloaded 34GiB to GPU in 1.89535s (17.9386GiB/s) | 20260507 00:13:53.897 | INFO | Analyzing chunk frames = [5572608,5573120) | 20260507 00:13:53.897 | INFO | MemInfo: { | 20260507 00:13:53.897 | INFO | "gpu" : " 17 GB", | 20260507 00:13:53.897 | INFO | "huge" : "", | 20260507 00:13:53.897 | INFO | "huge_shared" : " 27 GB", | 20260507 00:13:53.897 | INFO | "malloc" : " 14 GB", | 20260507 00:13:53.897 | INFO | "pinned" : " 29 GB" | 20260507 00:13:53.897 | INFO | } | 20260507 00:13:53.897 | INFO | Perf report for the last 5.13599 seconds: | 20260507 00:13:53.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:13:53.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:53.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:53.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:53.897 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 00:13:53.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:53.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:53.897 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:13:53.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:53.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:53.898 | INFO | Uploaded 36GiB to GPU in 1.97718s (18.2077GiB/s) | 20260507 00:13:53.898 | INFO | Downloaded 34GiB to GPU in 1.93017s (17.615GiB/s) | 20260507 00:13:59.037 | INFO | Analyzing chunk frames = [5573120,5573632) | 20260507 00:13:59.037 | INFO | MemInfo: { | 20260507 00:13:59.037 | INFO | "gpu" : " 17 GB", | 20260507 00:13:59.037 | INFO | "huge" : "", | 20260507 00:13:59.037 | INFO | "huge_shared" : " 27 GB", | 20260507 00:13:59.037 | INFO | "malloc" : " 14 GB", | 20260507 00:13:59.037 | INFO | "pinned" : " 29 GB" | 20260507 00:13:59.037 | INFO | } | 20260507 00:13:59.037 | INFO | Perf report for the last 5.13953 seconds: | 20260507 00:13:59.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:13:59.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:59.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:59.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:59.037 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:13:59.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:59.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:59.037 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 00:13:59.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:59.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:13:59.037 | INFO | Uploaded 36GiB to GPU in 2.00835s (17.9251GiB/s) | 20260507 00:13:59.037 | INFO | Downloaded 34GiB to GPU in 1.94863s (17.4481GiB/s) | 20260507 00:14:04.178 | INFO | Analyzing chunk frames = [5573632,5574144) | 20260507 00:14:04.178 | INFO | MemInfo: { | 20260507 00:14:04.178 | INFO | "gpu" : " 17 GB", | 20260507 00:14:04.178 | INFO | "huge" : "", | 20260507 00:14:04.179 | INFO | "huge_shared" : " 27 GB", | 20260507 00:14:04.179 | INFO | "malloc" : " 14 GB", | 20260507 00:14:04.179 | INFO | "pinned" : " 29 GB" | 20260507 00:14:04.179 | INFO | } | 20260507 00:14:04.179 | INFO | Perf report for the last 5.14174 seconds: | 20260507 00:14:04.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:14:04.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:04.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:04.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:04.179 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:14:04.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:04.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:04.179 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:14:04.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:04.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:04.179 | INFO | Uploaded 36GiB to GPU in 1.94885s (18.4725GiB/s) | 20260507 00:14:04.179 | INFO | Downloaded 34GiB to GPU in 1.98255s (17.1496GiB/s) | 20260507 00:14:09.320 | INFO | Analyzing chunk frames = [5574144,5574656) | 20260507 00:14:09.320 | INFO | MemInfo: { | 20260507 00:14:09.320 | INFO | "gpu" : " 17 GB", | 20260507 00:14:09.320 | INFO | "huge" : "", | 20260507 00:14:09.320 | INFO | "huge_shared" : " 27 GB", | 20260507 00:14:09.320 | INFO | "malloc" : " 14 GB", | 20260507 00:14:09.320 | INFO | "pinned" : " 29 GB" | 20260507 00:14:09.320 | INFO | } | 20260507 00:14:09.321 | INFO | Perf report for the last 5.14184 seconds: | 20260507 00:14:09.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:14:09.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:09.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:09.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:09.321 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:14:09.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:09.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:09.321 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 00:14:09.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:09.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:09.321 | INFO | Uploaded 36GiB to GPU in 1.98177s (18.1656GiB/s) | 20260507 00:14:09.321 | INFO | Downloaded 34GiB to GPU in 1.99158s (17.0719GiB/s) | 20260507 00:14:09.418 | INFO | Starting a round of Baz IO | 20260507 00:14:12.538 | INFO | Populating RTMetrics report | 20260507 00:14:12.539 | INFO | RTMetrics report populated | 20260507 00:14:12.539 | INFO | Opening tmp RTMetrics json file | 20260507 00:14:12.539 | INFO | Serializing RTMetrics to json | 20260507 00:14:12.541 | INFO | Serialization done, swapping tmp file now... | 20260507 00:14:12.541 | INFO | RTMetrics done | 20260507 00:14:12.636 | INFO | Finished a round of Baz IO: Wrote 1.857 GiB in 3.218 seconds | 20260507 00:14:12.636 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.32% / 0.00% / 66.67% | 20260507 00:14:12.636 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.355069% / 7.94729e-06% / 7.15256e-05% | 20260507 00:14:12.636 | INFO | Baz IO overall: | 20260507 00:14:12.636 | INFO | BytesWritten : 2908589324436 | 20260507 00:14:12.636 | INFO | NumEvents : 1401421757026 | 20260507 00:14:14.455 | INFO | Analyzing chunk frames = [5574656,5575168) | 20260507 00:14:14.455 | INFO | MemInfo: { | 20260507 00:14:14.455 | INFO | "gpu" : " 17 GB", | 20260507 00:14:14.455 | INFO | "huge" : "", | 20260507 00:14:14.455 | INFO | "huge_shared" : " 27 GB", | 20260507 00:14:14.455 | INFO | "malloc" : " 14 GB", | 20260507 00:14:14.455 | INFO | "pinned" : " 29 GB" | 20260507 00:14:14.455 | INFO | } | 20260507 00:14:14.455 | INFO | Perf report for the last 5.13455 seconds: | 20260507 00:14:14.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:14:14.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:14.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:14.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:14.455 | INFO | ANALYSIS : 62.1%, 3.0, 96.4ms, 0 | 20260507 00:14:14.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:14.455 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 00:14:14.455 | INFO | PRE_HQ : 59.7%, 1.8, 56.3ms, 0 | 20260507 00:14:14.455 | INFO | BAZWRITER : 62.3%, 3.4, 107.7ms, 0 | 20260507 00:14:14.455 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:14:14.455 | INFO | Uploaded 36GiB to GPU in 2.05011s (17.56GiB/s) | 20260507 00:14:14.455 | INFO | Downloaded 38GiB to GPU in 2.18256s (17.4107GiB/s) | 20260507 00:14:19.588 | INFO | Analyzing chunk frames = [5575168,5575680) | 20260507 00:14:19.588 | INFO | MemInfo: { | 20260507 00:14:19.588 | INFO | "gpu" : " 17 GB", | 20260507 00:14:19.588 | INFO | "huge" : "", | 20260507 00:14:19.588 | INFO | "huge_shared" : " 27 GB", | 20260507 00:14:19.588 | INFO | "malloc" : " 14 GB", | 20260507 00:14:19.588 | INFO | "pinned" : " 29 GB" | 20260507 00:14:19.588 | INFO | } | 20260507 00:14:19.588 | INFO | Perf report for the last 5.13338 seconds: | 20260507 00:14:19.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:14:19.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:19.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:19.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:19.589 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:14:19.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:19.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:19.589 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:14:19.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:19.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:19.589 | INFO | Uploaded 36GiB to GPU in 1.99354s (18.0583GiB/s) | 20260507 00:14:19.589 | INFO | Downloaded 34GiB to GPU in 1.92609s (17.6523GiB/s) | 20260507 00:14:24.731 | INFO | Analyzing chunk frames = [5575680,5576192) | 20260507 00:14:24.731 | INFO | MemInfo: { | 20260507 00:14:24.731 | INFO | "gpu" : " 17 GB", | 20260507 00:14:24.731 | INFO | "huge" : "", | 20260507 00:14:24.731 | INFO | "huge_shared" : " 27 GB", | 20260507 00:14:24.731 | INFO | "malloc" : " 14 GB", | 20260507 00:14:24.731 | INFO | "pinned" : " 29 GB" | 20260507 00:14:24.731 | INFO | } | 20260507 00:14:24.731 | INFO | Perf report for the last 5.14297 seconds: | 20260507 00:14:24.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:14:24.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:24.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:24.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:24.732 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 00:14:24.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:24.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:24.732 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:14:24.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:24.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:24.732 | INFO | Uploaded 36GiB to GPU in 1.98123s (18.1705GiB/s) | 20260507 00:14:24.732 | INFO | Downloaded 34GiB to GPU in 1.91913s (17.7164GiB/s) | 20260507 00:14:29.878 | INFO | Analyzing chunk frames = [5576192,5576704) | 20260507 00:14:29.878 | INFO | MemInfo: { | 20260507 00:14:29.878 | INFO | "gpu" : " 17 GB", | 20260507 00:14:29.878 | INFO | "huge" : "", | 20260507 00:14:29.878 | INFO | "huge_shared" : " 27 GB", | 20260507 00:14:29.878 | INFO | "malloc" : " 14 GB", | 20260507 00:14:29.878 | INFO | "pinned" : " 29 GB" | 20260507 00:14:29.878 | INFO | } | 20260507 00:14:29.878 | INFO | Perf report for the last 5.14647 seconds: | 20260507 00:14:29.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:14:29.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:29.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:29.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:29.878 | INFO | ANALYSIS : 59.4%, 3.0, 92.2ms, 0 | 20260507 00:14:29.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:29.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:29.878 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:14:29.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:29.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:29.878 | INFO | Uploaded 36GiB to GPU in 1.96809s (18.2919GiB/s) | 20260507 00:14:29.878 | INFO | Downloaded 34GiB to GPU in 1.87683s (18.1156GiB/s) | 20260507 00:14:35.014 | INFO | Analyzing chunk frames = [5576704,5577216) | 20260507 00:14:35.014 | INFO | MemInfo: { | 20260507 00:14:35.014 | INFO | "gpu" : " 17 GB", | 20260507 00:14:35.014 | INFO | "huge" : "", | 20260507 00:14:35.014 | INFO | "huge_shared" : " 27 GB", | 20260507 00:14:35.015 | INFO | "malloc" : " 14 GB", | 20260507 00:14:35.015 | INFO | "pinned" : " 29 GB" | 20260507 00:14:35.015 | INFO | } | 20260507 00:14:35.015 | INFO | Perf report for the last 5.13668 seconds: | 20260507 00:14:35.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:14:35.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:35.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:35.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:35.015 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 00:14:35.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:35.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:35.015 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:14:35.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:35.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:35.015 | INFO | Uploaded 36GiB to GPU in 1.99427s (18.0517GiB/s) | 20260507 00:14:35.015 | INFO | Downloaded 34GiB to GPU in 1.9397s (17.5285GiB/s) | 20260507 00:14:40.156 | INFO | Analyzing chunk frames = [5577216,5577728) | 20260507 00:14:40.156 | INFO | MemInfo: { | 20260507 00:14:40.156 | INFO | "gpu" : " 17 GB", | 20260507 00:14:40.156 | INFO | "huge" : "", | 20260507 00:14:40.156 | INFO | "huge_shared" : " 27 GB", | 20260507 00:14:40.156 | INFO | "malloc" : " 14 GB", | 20260507 00:14:40.156 | INFO | "pinned" : " 29 GB" | 20260507 00:14:40.156 | INFO | } | 20260507 00:14:40.156 | INFO | Perf report for the last 5.14146 seconds: | 20260507 00:14:40.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:14:40.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:40.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:40.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:40.156 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 00:14:40.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:40.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:40.156 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 00:14:40.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:40.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:40.156 | INFO | Uploaded 36GiB to GPU in 2.05918s (17.4827GiB/s) | 20260507 00:14:40.156 | INFO | Downloaded 34GiB to GPU in 1.95542s (17.3876GiB/s) | 20260507 00:14:45.296 | INFO | Analyzing chunk frames = [5577728,5578240) | 20260507 00:14:45.296 | INFO | MemInfo: { | 20260507 00:14:45.296 | INFO | "gpu" : " 17 GB", | 20260507 00:14:45.296 | INFO | "huge" : "", | 20260507 00:14:45.296 | INFO | "huge_shared" : " 27 GB", | 20260507 00:14:45.297 | INFO | "malloc" : " 14 GB", | 20260507 00:14:45.297 | INFO | "pinned" : " 29 GB" | 20260507 00:14:45.297 | INFO | } | 20260507 00:14:45.297 | INFO | Perf report for the last 5.14063 seconds: | 20260507 00:14:45.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:14:45.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:45.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:45.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:45.297 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 00:14:45.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:45.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:45.297 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:14:45.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:45.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:45.297 | INFO | Uploaded 36GiB to GPU in 1.98503s (18.1357GiB/s) | 20260507 00:14:45.297 | INFO | Downloaded 34GiB to GPU in 1.97242s (17.2377GiB/s) | 20260507 00:14:50.433 | INFO | Analyzing chunk frames = [5578240,5578752) | 20260507 00:14:50.433 | INFO | MemInfo: { | 20260507 00:14:50.433 | INFO | "gpu" : " 17 GB", | 20260507 00:14:50.433 | INFO | "huge" : "", | 20260507 00:14:50.433 | INFO | "huge_shared" : " 27 GB", | 20260507 00:14:50.433 | INFO | "malloc" : " 14 GB", | 20260507 00:14:50.433 | INFO | "pinned" : " 29 GB" | 20260507 00:14:50.433 | INFO | } | 20260507 00:14:50.433 | INFO | Perf report for the last 5.1362 seconds: | 20260507 00:14:50.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:14:50.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:50.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:50.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:50.433 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260507 00:14:50.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:50.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:50.433 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 00:14:50.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:50.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:50.433 | INFO | Uploaded 36GiB to GPU in 2.00537s (17.9518GiB/s) | 20260507 00:14:50.433 | INFO | Downloaded 34GiB to GPU in 1.96979s (17.2607GiB/s) | 20260507 00:14:50.433 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:14:50.433 | INFO | Total mem usage: 18 GiB | 20260507 00:14:50.433 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:14:50.433 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 00:14:50.433 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:14:50.434 | INFO | High Water Breakdown: | 20260507 00:14:50.434 | INFO | 60.1% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:14:50.434 | INFO | 39.7% : 5.67 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:14:50.434 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:14:50.434 | INFO | ------------------------------------------ | 20260507 00:14:50.434 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:14:50.434 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:14:50.434 | INFO | Total mem usage: 26.5 GiB | 20260507 00:14:50.434 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:14:50.434 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:14:50.434 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:14:50.434 | INFO | High Water Breakdown: | 20260507 00:14:50.434 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:14:50.434 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:14:50.434 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:14:50.434 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:14:50.434 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:14:50.434 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:14:50.434 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:14:50.434 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:14:50.434 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:14:50.434 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:14:50.434 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:14:50.434 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:14:50.434 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:14:50.434 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:14:50.434 | INFO | ------------------------------------------ | 20260507 00:14:50.434 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:14:50.434 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:14:50.434 | INFO | Total mem usage: 25.6 GiB | 20260507 00:14:50.434 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:14:50.434 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:14:50.434 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:14:50.434 | INFO | High Water Breakdown: | 20260507 00:14:50.434 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:14:50.434 | INFO | ------------------------------------------ | 20260507 00:14:50.434 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:14:50.434 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:14:50.434 | INFO | Total mem usage: 15.7 GiB | 20260507 00:14:50.434 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:14:50.434 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:14:50.434 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 00:14:50.434 | INFO | High Water Breakdown: | 20260507 00:14:50.434 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:14:50.434 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:14:50.434 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:14:50.434 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:14:50.434 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:14:50.434 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:14:50.434 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:14:50.434 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:14:50.434 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:14:50.434 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:14:50.434 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:14:50.434 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:14:50.434 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:14:50.434 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:14:50.434 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:14:50.434 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:14:50.434 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:14:50.434 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:14:50.434 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:14:50.434 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:14:50.434 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:14:50.435 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:14:50.435 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:14:50.435 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:14:50.435 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:14:50.435 | INFO | ------------------------------------------ | 20260507 00:14:50.435 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:14:50.435 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:14:50.435 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:14:50.435 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:14:50.435 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.52e+01, 1.35e+01 / 2.13e+00 | 20260507 00:14:50.435 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 9.94e-02, 9.79e-03 / 5.86e-03 | 20260507 00:14:50.435 | INFO | PerfReport: 11.67%, Baseline -- 3.61e+00 / 4.38e+00, 5.34e-01 / 4.03e-01 | 20260507 00:14:50.435 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.47e+00, 1.77e-01 / 1.21e-01 | 20260507 00:14:50.435 | INFO | PerfReport: 16.07%, DME -- 3.97e+01 / 4.41e+01, 3.75e+00 / 2.29e+00 | 20260507 00:14:50.435 | INFO | PerfReport: 57.43%, FrameLabeling -- 1.77e+01 / 2.08e+01, 1.31e+00 / 1.80e+00 | 20260507 00:14:50.435 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.18e+00 / 2.70e+00, 3.80e-01 / 2.44e-01 | 20260507 00:14:50.435 | INFO | PerfReport: 3.16%, Metrics -- 9.76e-01 / 1.29e+00, 2.58e-01 / 1.54e-01 | 20260507 00:14:50.435 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.32e-01 / 3.76e-01, 5.21e-02 / 1.82e-02 | 20260507 00:14:50.435 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:14:50.435 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:14:50.435 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:14:50.435 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260507 00:14:50.435 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 6.92e+00 / 0.00e+00 | 20260507 00:14:50.435 | INFO | PerfReport: 21.29%, Download -- 2.05e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 00:14:50.435 | INFO | PerfReport: 32.05%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:14:50.435 | INFO | PerfReport: 1.29%, LockUpload -- 1.25e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 00:14:50.435 | INFO | PerfReport: 1.31%, LockDownload -- 1.27e+00 / 0.00e+00, 3.18e+00 / 0.00e+00 | 20260507 00:14:50.435 | INFO | PerfReport: 22.52%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 00:14:50.435 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260507 00:14:50.541 | INFO | Starting a round of Baz IO | 20260507 00:14:53.653 | INFO | Populating RTMetrics report | 20260507 00:14:53.654 | INFO | RTMetrics report populated | 20260507 00:14:53.654 | INFO | Opening tmp RTMetrics json file | 20260507 00:14:53.654 | INFO | Serializing RTMetrics to json | 20260507 00:14:53.656 | INFO | Serialization done, swapping tmp file now... | 20260507 00:14:53.656 | INFO | RTMetrics done | 20260507 00:14:53.746 | INFO | Finished a round of Baz IO: Wrote 1.855 GiB in 3.205 seconds | 20260507 00:14:53.746 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.32% / 0.00% / 66.68% | 20260507 00:14:53.746 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.35493% / 0% / 7.94729e-05% | 20260507 00:14:53.746 | INFO | Baz IO overall: | 20260507 00:14:53.746 | INFO | BytesWritten : 2910581561498 | 20260507 00:14:53.746 | INFO | NumEvents : 1402367644968 | 20260507 00:14:55.569 | INFO | Analyzing chunk frames = [5578752,5579264) | 20260507 00:14:55.569 | INFO | MemInfo: { | 20260507 00:14:55.569 | INFO | "gpu" : " 17 GB", | 20260507 00:14:55.569 | INFO | "huge" : "", | 20260507 00:14:55.569 | INFO | "huge_shared" : " 27 GB", | 20260507 00:14:55.569 | INFO | "malloc" : " 14 GB", | 20260507 00:14:55.569 | INFO | "pinned" : " 29 GB" | 20260507 00:14:55.569 | INFO | } | 20260507 00:14:55.569 | INFO | Perf report for the last 5.13602 seconds: | 20260507 00:14:55.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:14:55.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:55.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:55.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:55.569 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 00:14:55.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:14:55.569 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 00:14:55.569 | INFO | PRE_HQ : 59.6%, 1.8, 56.4ms, 0 | 20260507 00:14:55.569 | INFO | BAZWRITER : 62.1%, 3.3, 106.8ms, 0 | 20260507 00:14:55.569 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:14:55.569 | INFO | Uploaded 36GiB to GPU in 2.01872s (17.8331GiB/s) | 20260507 00:14:55.569 | INFO | Downloaded 38GiB to GPU in 2.17074s (17.5055GiB/s) | 20260507 00:15:00.713 | INFO | Analyzing chunk frames = [5579264,5579776) | 20260507 00:15:00.713 | INFO | MemInfo: { | 20260507 00:15:00.713 | INFO | "gpu" : " 17 GB", | 20260507 00:15:00.713 | INFO | "huge" : "", | 20260507 00:15:00.713 | INFO | "huge_shared" : " 27 GB", | 20260507 00:15:00.713 | INFO | "malloc" : " 14 GB", | 20260507 00:15:00.713 | INFO | "pinned" : " 29 GB" | 20260507 00:15:00.713 | INFO | } | 20260507 00:15:00.713 | INFO | Perf report for the last 5.14422 seconds: | 20260507 00:15:00.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:15:00.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:00.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:00.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:00.713 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260507 00:15:00.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:00.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:00.713 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:15:00.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:00.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:00.714 | INFO | Uploaded 36GiB to GPU in 2.00918s (17.9178GiB/s) | 20260507 00:15:00.714 | INFO | Downloaded 34GiB to GPU in 1.93308s (17.5885GiB/s) | 20260507 00:15:05.847 | INFO | Analyzing chunk frames = [5579776,5580288) | 20260507 00:15:05.847 | INFO | MemInfo: { | 20260507 00:15:05.847 | INFO | "gpu" : " 17 GB", | 20260507 00:15:05.847 | INFO | "huge" : "", | 20260507 00:15:05.847 | INFO | "huge_shared" : " 27 GB", | 20260507 00:15:05.847 | INFO | "malloc" : " 14 GB", | 20260507 00:15:05.847 | INFO | "pinned" : " 29 GB" | 20260507 00:15:05.847 | INFO | } | 20260507 00:15:05.847 | INFO | Perf report for the last 5.1338 seconds: | 20260507 00:15:05.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:15:05.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:05.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:05.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:05.847 | INFO | ANALYSIS : 60.5%, 3.0, 93.5ms, 0 | 20260507 00:15:05.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:05.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:05.847 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:15:05.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:05.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:05.847 | INFO | Uploaded 36GiB to GPU in 2.00038s (17.9966GiB/s) | 20260507 00:15:05.847 | INFO | Downloaded 34GiB to GPU in 1.93304s (17.5889GiB/s) | 20260507 00:15:10.990 | INFO | Analyzing chunk frames = [5580288,5580800) | 20260507 00:15:10.990 | INFO | MemInfo: { | 20260507 00:15:10.990 | INFO | "gpu" : " 17 GB", | 20260507 00:15:10.990 | INFO | "huge" : "", | 20260507 00:15:10.990 | INFO | "huge_shared" : " 27 GB", | 20260507 00:15:10.990 | INFO | "malloc" : " 14 GB", | 20260507 00:15:10.990 | INFO | "pinned" : " 29 GB" | 20260507 00:15:10.990 | INFO | } | 20260507 00:15:10.990 | INFO | Perf report for the last 5.14287 seconds: | 20260507 00:15:10.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:15:10.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:10.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:10.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:10.990 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260507 00:15:10.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:10.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:10.990 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:15:10.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:10.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:10.990 | INFO | Uploaded 36GiB to GPU in 1.98594s (18.1275GiB/s) | 20260507 00:15:10.990 | INFO | Downloaded 34GiB to GPU in 1.90692s (17.8298GiB/s) | 20260507 00:15:16.131 | INFO | Analyzing chunk frames = [5580800,5581312) | 20260507 00:15:16.131 | INFO | MemInfo: { | 20260507 00:15:16.131 | INFO | "gpu" : " 17 GB", | 20260507 00:15:16.131 | INFO | "huge" : "", | 20260507 00:15:16.131 | INFO | "huge_shared" : " 27 GB", | 20260507 00:15:16.131 | INFO | "malloc" : " 14 GB", | 20260507 00:15:16.131 | INFO | "pinned" : " 29 GB" | 20260507 00:15:16.131 | INFO | } | 20260507 00:15:16.131 | INFO | Perf report for the last 5.14149 seconds: | 20260507 00:15:16.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:15:16.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:16.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:16.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:16.131 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 00:15:16.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:16.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:16.131 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:15:16.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:16.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:16.132 | INFO | Uploaded 36GiB to GPU in 1.99554s (18.0402GiB/s) | 20260507 00:15:16.132 | INFO | Downloaded 34GiB to GPU in 1.92741s (17.6402GiB/s) | 20260507 00:15:21.270 | INFO | Analyzing chunk frames = [5581312,5581824) | 20260507 00:15:21.270 | INFO | MemInfo: { | 20260507 00:15:21.270 | INFO | "gpu" : " 17 GB", | 20260507 00:15:21.270 | INFO | "huge" : "", | 20260507 00:15:21.270 | INFO | "huge_shared" : " 27 GB", | 20260507 00:15:21.270 | INFO | "malloc" : " 14 GB", | 20260507 00:15:21.270 | INFO | "pinned" : " 29 GB" | 20260507 00:15:21.270 | INFO | } | 20260507 00:15:21.270 | INFO | Perf report for the last 5.13845 seconds: | 20260507 00:15:21.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:15:21.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:21.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:21.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:21.270 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:15:21.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:21.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:21.270 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 00:15:21.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:21.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:21.270 | INFO | Uploaded 36GiB to GPU in 2.05214s (17.5427GiB/s) | 20260507 00:15:21.270 | INFO | Downloaded 34GiB to GPU in 1.95138s (17.4235GiB/s) | 20260507 00:15:26.402 | INFO | Analyzing chunk frames = [5581824,5582336) | 20260507 00:15:26.402 | INFO | MemInfo: { | 20260507 00:15:26.402 | INFO | "gpu" : " 17 GB", | 20260507 00:15:26.402 | INFO | "huge" : "", | 20260507 00:15:26.402 | INFO | "huge_shared" : " 27 GB", | 20260507 00:15:26.402 | INFO | "malloc" : " 14 GB", | 20260507 00:15:26.402 | INFO | "pinned" : " 29 GB" | 20260507 00:15:26.402 | INFO | } | 20260507 00:15:26.402 | INFO | Perf report for the last 5.13184 seconds: | 20260507 00:15:26.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:15:26.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:26.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:26.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:26.402 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:15:26.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:26.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:26.402 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 00:15:26.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:26.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:26.402 | INFO | Uploaded 36GiB to GPU in 1.99548s (18.0408GiB/s) | 20260507 00:15:26.402 | INFO | Downloaded 34GiB to GPU in 1.98136s (17.16GiB/s) | 20260507 00:15:31.541 | INFO | Analyzing chunk frames = [5582336,5582848) | 20260507 00:15:31.541 | INFO | MemInfo: { | 20260507 00:15:31.541 | INFO | "gpu" : " 17 GB", | 20260507 00:15:31.541 | INFO | "huge" : "", | 20260507 00:15:31.541 | INFO | "huge_shared" : " 27 GB", | 20260507 00:15:31.541 | INFO | "malloc" : " 14 GB", | 20260507 00:15:31.541 | INFO | "pinned" : " 29 GB" | 20260507 00:15:31.541 | INFO | } | 20260507 00:15:31.541 | INFO | Perf report for the last 5.13939 seconds: | 20260507 00:15:31.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:15:31.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:31.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:31.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:31.541 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260507 00:15:31.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:31.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:31.541 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:15:31.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:31.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:31.541 | INFO | Uploaded 36GiB to GPU in 2.04876s (17.5716GiB/s) | 20260507 00:15:31.541 | INFO | Downloaded 34GiB to GPU in 2.00448s (16.962GiB/s) | 20260507 00:15:31.648 | INFO | Starting a round of Baz IO | 20260507 00:15:34.769 | INFO | Populating RTMetrics report | 20260507 00:15:34.769 | INFO | RTMetrics report populated | 20260507 00:15:34.769 | INFO | Opening tmp RTMetrics json file | 20260507 00:15:34.770 | INFO | Serializing RTMetrics to json | 20260507 00:15:34.771 | INFO | Serialization done, swapping tmp file now... | 20260507 00:15:34.772 | INFO | RTMetrics done | 20260507 00:15:34.863 | INFO | Finished a round of Baz IO: Wrote 1.853 GiB in 3.214 seconds | 20260507 00:15:34.863 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.32% / 0.00% / 66.68% | 20260507 00:15:34.863 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.354254% / 3.97364e-06% / 8.74201e-05% | 20260507 00:15:34.863 | INFO | Baz IO overall: | 20260507 00:15:34.863 | INFO | BytesWritten : 2912571249934 | 20260507 00:15:34.863 | INFO | NumEvents : 1403312482582 | 20260507 00:15:36.686 | INFO | Analyzing chunk frames = [5582848,5583360) | 20260507 00:15:36.686 | INFO | MemInfo: { | 20260507 00:15:36.686 | INFO | "gpu" : " 17 GB", | 20260507 00:15:36.686 | INFO | "huge" : "", | 20260507 00:15:36.686 | INFO | "huge_shared" : " 27 GB", | 20260507 00:15:36.686 | INFO | "malloc" : " 14 GB", | 20260507 00:15:36.686 | INFO | "pinned" : " 29 GB" | 20260507 00:15:36.686 | INFO | } | 20260507 00:15:36.686 | INFO | Perf report for the last 5.14463 seconds: | 20260507 00:15:36.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:15:36.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:36.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:36.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:36.686 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260507 00:15:36.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:36.686 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 00:15:36.686 | INFO | PRE_HQ : 59.5%, 1.8, 56.4ms, 0 | 20260507 00:15:36.686 | INFO | BAZWRITER : 62.2%, 3.4, 108.5ms, 0 | 20260507 00:15:36.686 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:15:36.686 | INFO | Uploaded 36GiB to GPU in 2.01369s (17.8777GiB/s) | 20260507 00:15:36.686 | INFO | Downloaded 38GiB to GPU in 2.18757s (17.3708GiB/s) | 20260507 00:15:41.829 | INFO | Analyzing chunk frames = [5583360,5583872) | 20260507 00:15:41.829 | INFO | MemInfo: { | 20260507 00:15:41.829 | INFO | "gpu" : " 17 GB", | 20260507 00:15:41.829 | INFO | "huge" : "", | 20260507 00:15:41.829 | INFO | "huge_shared" : " 27 GB", | 20260507 00:15:41.829 | INFO | "malloc" : " 14 GB", | 20260507 00:15:41.829 | INFO | "pinned" : " 29 GB" | 20260507 00:15:41.829 | INFO | } | 20260507 00:15:41.829 | INFO | Perf report for the last 5.1435 seconds: | 20260507 00:15:41.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:15:41.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:41.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:41.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:41.829 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:15:41.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:41.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:41.829 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 00:15:41.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:41.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:41.830 | INFO | Uploaded 36GiB to GPU in 1.99667s (18.03GiB/s) | 20260507 00:15:41.830 | INFO | Downloaded 34GiB to GPU in 1.93413s (17.579GiB/s) | 20260507 00:15:46.970 | INFO | Analyzing chunk frames = [5583872,5584384) | 20260507 00:15:46.970 | INFO | MemInfo: { | 20260507 00:15:46.970 | INFO | "gpu" : " 17 GB", | 20260507 00:15:46.970 | INFO | "huge" : "", | 20260507 00:15:46.970 | INFO | "huge_shared" : " 27 GB", | 20260507 00:15:46.970 | INFO | "malloc" : " 14 GB", | 20260507 00:15:46.970 | INFO | "pinned" : " 29 GB" | 20260507 00:15:46.970 | INFO | } | 20260507 00:15:46.970 | INFO | Perf report for the last 5.14081 seconds: | 20260507 00:15:46.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:15:46.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:46.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:46.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:46.970 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 00:15:46.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:46.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:46.970 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:15:46.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:46.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:46.970 | INFO | Uploaded 36GiB to GPU in 1.99398s (18.0543GiB/s) | 20260507 00:15:46.971 | INFO | Downloaded 34GiB to GPU in 1.93999s (17.5258GiB/s) | 20260507 00:15:52.102 | INFO | Analyzing chunk frames = [5584384,5584896) | 20260507 00:15:52.102 | INFO | MemInfo: { | 20260507 00:15:52.102 | INFO | "gpu" : " 17 GB", | 20260507 00:15:52.102 | INFO | "huge" : "", | 20260507 00:15:52.102 | INFO | "huge_shared" : " 27 GB", | 20260507 00:15:52.102 | INFO | "malloc" : " 14 GB", | 20260507 00:15:52.102 | INFO | "pinned" : " 29 GB" | 20260507 00:15:52.102 | INFO | } | 20260507 00:15:52.102 | INFO | Perf report for the last 5.13231 seconds: | 20260507 00:15:52.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:15:52.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:52.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:52.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:52.102 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:15:52.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:52.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:52.103 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:15:52.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:52.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:52.103 | INFO | Uploaded 36GiB to GPU in 1.96984s (18.2756GiB/s) | 20260507 00:15:52.103 | INFO | Downloaded 34GiB to GPU in 1.90737s (17.8256GiB/s) | 20260507 00:15:57.242 | INFO | Analyzing chunk frames = [5584896,5585408) | 20260507 00:15:57.242 | INFO | MemInfo: { | 20260507 00:15:57.242 | INFO | "gpu" : " 17 GB", | 20260507 00:15:57.242 | INFO | "huge" : "", | 20260507 00:15:57.243 | INFO | "huge_shared" : " 27 GB", | 20260507 00:15:57.243 | INFO | "malloc" : " 14 GB", | 20260507 00:15:57.243 | INFO | "pinned" : " 29 GB" | 20260507 00:15:57.243 | INFO | } | 20260507 00:15:57.243 | INFO | Perf report for the last 5.14022 seconds: | 20260507 00:15:57.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:15:57.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:57.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:57.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:57.243 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 00:15:57.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:57.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:57.243 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:15:57.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:57.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:15:57.243 | INFO | Uploaded 36GiB to GPU in 1.99163s (18.0756GiB/s) | 20260507 00:15:57.243 | INFO | Downloaded 34GiB to GPU in 1.94294s (17.4993GiB/s) | 20260507 00:16:02.382 | INFO | Analyzing chunk frames = [5585408,5585920) | 20260507 00:16:02.382 | INFO | MemInfo: { | 20260507 00:16:02.382 | INFO | "gpu" : " 17 GB", | 20260507 00:16:02.382 | INFO | "huge" : "", | 20260507 00:16:02.382 | INFO | "huge_shared" : " 27 GB", | 20260507 00:16:02.382 | INFO | "malloc" : " 14 GB", | 20260507 00:16:02.382 | INFO | "pinned" : " 29 GB" | 20260507 00:16:02.382 | INFO | } | 20260507 00:16:02.382 | INFO | Perf report for the last 5.13937 seconds: | 20260507 00:16:02.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:16:02.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:02.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:02.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:02.382 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 00:16:02.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:02.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:02.382 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:16:02.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:02.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:02.382 | INFO | Uploaded 36GiB to GPU in 2.03646s (17.6778GiB/s) | 20260507 00:16:02.382 | INFO | Downloaded 34GiB to GPU in 1.94888s (17.4459GiB/s) | 20260507 00:16:07.522 | INFO | Analyzing chunk frames = [5585920,5586432) | 20260507 00:16:07.522 | INFO | MemInfo: { | 20260507 00:16:07.522 | INFO | "gpu" : " 17 GB", | 20260507 00:16:07.522 | INFO | "huge" : "", | 20260507 00:16:07.522 | INFO | "huge_shared" : " 27 GB", | 20260507 00:16:07.522 | INFO | "malloc" : " 14 GB", | 20260507 00:16:07.522 | INFO | "pinned" : " 29 GB" | 20260507 00:16:07.522 | INFO | } | 20260507 00:16:07.522 | INFO | Perf report for the last 5.14001 seconds: | 20260507 00:16:07.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:16:07.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:07.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:07.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:07.522 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:16:07.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:07.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:07.522 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:16:07.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:07.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:07.522 | INFO | Uploaded 36GiB to GPU in 1.96897s (18.2836GiB/s) | 20260507 00:16:07.522 | INFO | Downloaded 34GiB to GPU in 2.00126s (16.9893GiB/s) | 20260507 00:16:12.662 | INFO | Analyzing chunk frames = [5586432,5586944) | 20260507 00:16:12.662 | INFO | MemInfo: { | 20260507 00:16:12.662 | INFO | "gpu" : " 17 GB", | 20260507 00:16:12.662 | INFO | "huge" : "", | 20260507 00:16:12.662 | INFO | "huge_shared" : " 27 GB", | 20260507 00:16:12.662 | INFO | "malloc" : " 14 GB", | 20260507 00:16:12.662 | INFO | "pinned" : " 29 GB" | 20260507 00:16:12.662 | INFO | } | 20260507 00:16:12.662 | INFO | Perf report for the last 5.14018 seconds: | 20260507 00:16:12.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:16:12.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:12.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:12.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:12.662 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260507 00:16:12.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:12.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:12.662 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 00:16:12.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:12.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:12.663 | INFO | Uploaded 36GiB to GPU in 2.00738s (17.9338GiB/s) | 20260507 00:16:12.663 | INFO | Downloaded 34GiB to GPU in 1.99497s (17.0429GiB/s) | 20260507 00:16:12.663 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:16:12.663 | INFO | Total mem usage: 18 GiB | 20260507 00:16:12.663 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:16:12.663 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 00:16:12.663 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:16:12.663 | INFO | High Water Breakdown: | 20260507 00:16:12.663 | INFO | 60.1% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:16:12.663 | INFO | 39.7% : 5.67 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:16:12.663 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:16:12.663 | INFO | ------------------------------------------ | 20260507 00:16:12.663 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:16:12.663 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:16:12.663 | INFO | Total mem usage: 26.5 GiB | 20260507 00:16:12.663 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:16:12.663 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:16:12.663 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:16:12.663 | INFO | High Water Breakdown: | 20260507 00:16:12.663 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:16:12.663 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:16:12.663 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:16:12.663 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:16:12.663 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:16:12.663 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:16:12.663 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:16:12.663 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:16:12.663 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:16:12.663 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:16:12.663 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:16:12.663 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:16:12.663 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:16:12.663 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:16:12.663 | INFO | ------------------------------------------ | 20260507 00:16:12.663 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:16:12.663 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:16:12.663 | INFO | Total mem usage: 25.6 GiB | 20260507 00:16:12.663 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:16:12.663 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:16:12.663 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:16:12.663 | INFO | High Water Breakdown: | 20260507 00:16:12.663 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:16:12.663 | INFO | ------------------------------------------ | 20260507 00:16:12.663 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:16:12.663 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:16:12.663 | INFO | Total mem usage: 15.7 GiB | 20260507 00:16:12.663 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:16:12.663 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:16:12.663 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:16:12.663 | INFO | High Water Breakdown: | 20260507 00:16:12.663 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:16:12.663 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:16:12.663 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:16:12.663 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:16:12.663 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:16:12.663 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:16:12.663 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:16:12.663 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:16:12.663 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:16:12.663 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:16:12.663 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:16:12.663 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:16:12.663 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:16:12.663 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:16:12.663 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:16:12.664 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:16:12.664 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:16:12.664 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:16:12.664 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:16:12.664 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:16:12.664 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:16:12.664 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:16:12.664 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:16:12.664 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:16:12.664 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:16:12.664 | INFO | ------------------------------------------ | 20260507 00:16:12.664 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:16:12.664 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:16:12.664 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:16:12.664 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:16:12.664 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.55e+01, 1.35e+01 / 1.99e+00 | 20260507 00:16:12.664 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 1.00e-01, 9.29e-03 / 5.79e-03 | 20260507 00:16:12.664 | INFO | PerfReport: 11.66%, Baseline -- 3.61e+00 / 4.41e+00, 5.36e-01 / 3.50e-01 | 20260507 00:16:12.664 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.45e+00, 1.72e-01 / 1.32e-01 | 20260507 00:16:12.664 | INFO | PerfReport: 16.08%, DME -- 3.98e+01 / 4.42e+01, 4.07e+00 / 2.87e+00 | 20260507 00:16:12.664 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.35e+00 / 1.94e+00 | 20260507 00:16:12.664 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.18e+00 / 2.71e+00, 3.78e-01 / 2.40e-01 | 20260507 00:16:12.664 | INFO | PerfReport: 3.16%, Metrics -- 9.77e-01 / 1.32e+00, 2.60e-01 / 1.51e-01 | 20260507 00:16:12.664 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.32e-01 / 3.87e-01, 5.25e-02 / 1.55e-02 | 20260507 00:16:12.664 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:16:12.664 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:16:12.664 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:16:12.664 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 00:16:12.664 | INFO | PerfReport: 21.62%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260507 00:16:12.664 | INFO | PerfReport: 21.34%, Download -- 2.06e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260507 00:16:12.664 | INFO | PerfReport: 32.01%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:16:12.664 | INFO | PerfReport: 1.32%, LockUpload -- 1.28e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 00:16:12.664 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260507 00:16:12.664 | INFO | PerfReport: 22.38%, LockCompute -- 2.16e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 00:16:12.664 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260507 00:16:12.760 | INFO | Starting a round of Baz IO | 20260507 00:16:15.888 | INFO | Populating RTMetrics report | 20260507 00:16:15.889 | INFO | RTMetrics report populated | 20260507 00:16:15.889 | INFO | Opening tmp RTMetrics json file | 20260507 00:16:15.907 | INFO | Serializing RTMetrics to json | 20260507 00:16:15.909 | INFO | Serialization done, swapping tmp file now... | 20260507 00:16:15.909 | INFO | RTMetrics done | 20260507 00:16:15.982 | INFO | Finished a round of Baz IO: Wrote 1.854 GiB in 3.221 seconds | 20260507 00:16:15.982 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.32% / 0.00% / 66.68% | 20260507 00:16:15.982 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.354397% / 0% / 8.74201e-05% | 20260507 00:16:15.982 | INFO | Baz IO overall: | 20260507 00:16:15.982 | INFO | BytesWritten : 2914562402249 | 20260507 00:16:15.982 | INFO | NumEvents : 1404259214066 | 20260507 00:16:17.804 | INFO | Analyzing chunk frames = [5586944,5587456) | 20260507 00:16:17.804 | INFO | MemInfo: { | 20260507 00:16:17.804 | INFO | "gpu" : " 17 GB", | 20260507 00:16:17.804 | INFO | "huge" : "", | 20260507 00:16:17.804 | INFO | "huge_shared" : " 27 GB", | 20260507 00:16:17.804 | INFO | "malloc" : " 14 GB", | 20260507 00:16:17.804 | INFO | "pinned" : " 29 GB" | 20260507 00:16:17.804 | INFO | } | 20260507 00:16:17.804 | INFO | Perf report for the last 5.1414 seconds: | 20260507 00:16:17.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:16:17.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:17.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:17.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:17.804 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 00:16:17.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:17.804 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260507 00:16:17.804 | INFO | PRE_HQ : 60.5%, 1.9, 60.0ms, 0 | 20260507 00:16:17.804 | INFO | BAZWRITER : 62.2%, 3.5, 110.5ms, 0 | 20260507 00:16:17.804 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:16:17.804 | INFO | Uploaded 36GiB to GPU in 2.03274s (17.7101GiB/s) | 20260507 00:16:17.804 | INFO | Downloaded 38GiB to GPU in 2.17511s (17.4704GiB/s) | 20260507 00:16:22.943 | INFO | Analyzing chunk frames = [5587456,5587968) | 20260507 00:16:22.943 | INFO | MemInfo: { | 20260507 00:16:22.943 | INFO | "gpu" : " 17 GB", | 20260507 00:16:22.943 | INFO | "huge" : "", | 20260507 00:16:22.943 | INFO | "huge_shared" : " 27 GB", | 20260507 00:16:22.943 | INFO | "malloc" : " 14 GB", | 20260507 00:16:22.943 | INFO | "pinned" : " 29 GB" | 20260507 00:16:22.943 | INFO | } | 20260507 00:16:22.943 | INFO | Perf report for the last 5.1391 seconds: | 20260507 00:16:22.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:16:22.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:22.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:22.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:22.943 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 00:16:22.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:22.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:22.943 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 00:16:22.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:22.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:22.943 | INFO | Uploaded 36GiB to GPU in 1.99259s (18.0669GiB/s) | 20260507 00:16:22.943 | INFO | Downloaded 34GiB to GPU in 1.92316s (17.6792GiB/s) | 20260507 00:16:28.080 | INFO | Analyzing chunk frames = [5587968,5588480) | 20260507 00:16:28.080 | INFO | MemInfo: { | 20260507 00:16:28.080 | INFO | "gpu" : " 17 GB", | 20260507 00:16:28.080 | INFO | "huge" : "", | 20260507 00:16:28.080 | INFO | "huge_shared" : " 27 GB", | 20260507 00:16:28.080 | INFO | "malloc" : " 14 GB", | 20260507 00:16:28.080 | INFO | "pinned" : " 29 GB" | 20260507 00:16:28.080 | INFO | } | 20260507 00:16:28.080 | INFO | Perf report for the last 5.13683 seconds: | 20260507 00:16:28.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:16:28.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:28.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:28.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:28.080 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 00:16:28.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:28.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:28.080 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:16:28.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:28.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:28.080 | INFO | Uploaded 36GiB to GPU in 2.00459s (17.9588GiB/s) | 20260507 00:16:28.080 | INFO | Downloaded 34GiB to GPU in 1.93321s (17.5873GiB/s) | 20260507 00:16:33.220 | INFO | Analyzing chunk frames = [5588480,5588992) | 20260507 00:16:33.220 | INFO | MemInfo: { | 20260507 00:16:33.220 | INFO | "gpu" : " 17 GB", | 20260507 00:16:33.220 | INFO | "huge" : "", | 20260507 00:16:33.220 | INFO | "huge_shared" : " 27 GB", | 20260507 00:16:33.220 | INFO | "malloc" : " 14 GB", | 20260507 00:16:33.221 | INFO | "pinned" : " 29 GB" | 20260507 00:16:33.221 | INFO | } | 20260507 00:16:33.221 | INFO | Perf report for the last 5.1409 seconds: | 20260507 00:16:33.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:16:33.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:33.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:33.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:33.221 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:16:33.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:33.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:33.221 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:16:33.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:33.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:33.221 | INFO | Uploaded 36GiB to GPU in 1.97425s (18.2347GiB/s) | 20260507 00:16:33.221 | INFO | Downloaded 34GiB to GPU in 1.90586s (17.8397GiB/s) | 20260507 00:16:38.352 | INFO | Analyzing chunk frames = [5588992,5589504) | 20260507 00:16:38.352 | INFO | MemInfo: { | 20260507 00:16:38.352 | INFO | "gpu" : " 17 GB", | 20260507 00:16:38.353 | INFO | "huge" : "", | 20260507 00:16:38.353 | INFO | "huge_shared" : " 27 GB", | 20260507 00:16:38.353 | INFO | "malloc" : " 14 GB", | 20260507 00:16:38.353 | INFO | "pinned" : " 29 GB" | 20260507 00:16:38.353 | INFO | } | 20260507 00:16:38.353 | INFO | Perf report for the last 5.13214 seconds: | 20260507 00:16:38.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:16:38.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:38.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:38.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:38.353 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 00:16:38.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:38.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:38.353 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 00:16:38.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:38.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:38.353 | INFO | Uploaded 36GiB to GPU in 2.00896s (17.9197GiB/s) | 20260507 00:16:38.353 | INFO | Downloaded 34GiB to GPU in 1.94148s (17.5124GiB/s) | 20260507 00:16:43.493 | INFO | Analyzing chunk frames = [5589504,5590016) | 20260507 00:16:43.493 | INFO | MemInfo: { | 20260507 00:16:43.493 | INFO | "gpu" : " 17 GB", | 20260507 00:16:43.493 | INFO | "huge" : "", | 20260507 00:16:43.493 | INFO | "huge_shared" : " 27 GB", | 20260507 00:16:43.493 | INFO | "malloc" : " 14 GB", | 20260507 00:16:43.493 | INFO | "pinned" : " 29 GB" | 20260507 00:16:43.493 | INFO | } | 20260507 00:16:43.493 | INFO | Perf report for the last 5.14015 seconds: | 20260507 00:16:43.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:16:43.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:43.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:43.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:43.493 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260507 00:16:43.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:43.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:43.493 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:16:43.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:43.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:43.493 | INFO | Uploaded 36GiB to GPU in 2.01587s (17.8583GiB/s) | 20260507 00:16:43.493 | INFO | Downloaded 34GiB to GPU in 1.94573s (17.4741GiB/s) | 20260507 00:16:48.643 | INFO | Analyzing chunk frames = [5590016,5590528) | 20260507 00:16:48.643 | INFO | MemInfo: { | 20260507 00:16:48.643 | INFO | "gpu" : " 17 GB", | 20260507 00:16:48.643 | INFO | "huge" : "", | 20260507 00:16:48.643 | INFO | "huge_shared" : " 27 GB", | 20260507 00:16:48.643 | INFO | "malloc" : " 14 GB", | 20260507 00:16:48.643 | INFO | "pinned" : " 29 GB" | 20260507 00:16:48.643 | INFO | } | 20260507 00:16:48.643 | INFO | Perf report for the last 5.14994 seconds: | 20260507 00:16:48.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:16:48.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:48.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:48.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:48.643 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260507 00:16:48.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:48.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:48.643 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 00:16:48.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:48.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:48.643 | INFO | Uploaded 36GiB to GPU in 1.96037s (18.3639GiB/s) | 20260507 00:16:48.643 | INFO | Downloaded 34GiB to GPU in 1.98304s (17.1454GiB/s) | 20260507 00:16:53.776 | INFO | Analyzing chunk frames = [5590528,5591040) | 20260507 00:16:53.776 | INFO | MemInfo: { | 20260507 00:16:53.776 | INFO | "gpu" : " 17 GB", | 20260507 00:16:53.776 | INFO | "huge" : "", | 20260507 00:16:53.776 | INFO | "huge_shared" : " 27 GB", | 20260507 00:16:53.776 | INFO | "malloc" : " 14 GB", | 20260507 00:16:53.776 | INFO | "pinned" : " 29 GB" | 20260507 00:16:53.776 | INFO | } | 20260507 00:16:53.776 | INFO | Perf report for the last 5.13293 seconds: | 20260507 00:16:53.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:16:53.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:53.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:53.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:53.776 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:16:53.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:53.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:53.776 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260507 00:16:53.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:53.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:53.776 | INFO | Uploaded 36GiB to GPU in 1.96776s (18.2949GiB/s) | 20260507 00:16:53.776 | INFO | Downloaded 34GiB to GPU in 1.98983s (17.0869GiB/s) | 20260507 00:16:53.875 | INFO | Starting a round of Baz IO | 20260507 00:16:56.991 | INFO | Populating RTMetrics report | 20260507 00:16:56.991 | INFO | RTMetrics report populated | 20260507 00:16:56.991 | INFO | Opening tmp RTMetrics json file | 20260507 00:16:56.992 | INFO | Serializing RTMetrics to json | 20260507 00:16:56.995 | INFO | Serialization done, swapping tmp file now... | 20260507 00:16:57.001 | INFO | RTMetrics done | 20260507 00:16:57.087 | INFO | Finished a round of Baz IO: Wrote 1.853 GiB in 3.212 seconds | 20260507 00:16:57.087 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.32% / 0.00% / 66.68% | 20260507 00:16:57.087 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.354016% / 3.97364e-06% / 8.34465e-05% | 20260507 00:16:57.087 | INFO | Baz IO overall: | 20260507 00:16:57.087 | INFO | BytesWritten : 2916551545968 | 20260507 00:16:57.087 | INFO | NumEvents : 1405204692584 | 20260507 00:16:58.912 | INFO | Analyzing chunk frames = [5591040,5591552) | 20260507 00:16:58.912 | INFO | MemInfo: { | 20260507 00:16:58.912 | INFO | "gpu" : " 17 GB", | 20260507 00:16:58.912 | INFO | "huge" : "", | 20260507 00:16:58.912 | INFO | "huge_shared" : " 27 GB", | 20260507 00:16:58.912 | INFO | "malloc" : " 14 GB", | 20260507 00:16:58.912 | INFO | "pinned" : " 29 GB" | 20260507 00:16:58.912 | INFO | } | 20260507 00:16:58.912 | INFO | Perf report for the last 5.13634 seconds: | 20260507 00:16:58.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:16:58.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:58.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:58.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:58.912 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 00:16:58.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:16:58.912 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260507 00:16:58.912 | INFO | PRE_HQ : 60.4%, 1.9, 60.1ms, 0 | 20260507 00:16:58.912 | INFO | BAZWRITER : 62.1%, 3.5, 111.5ms, 0 | 20260507 00:16:58.912 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 00:16:58.912 | INFO | Uploaded 36GiB to GPU in 2.01473s (17.8684GiB/s) | 20260507 00:16:58.912 | INFO | Downloaded 38GiB to GPU in 2.18581s (17.3848GiB/s) | 20260507 00:17:04.056 | INFO | Analyzing chunk frames = [5591552,5592064) | 20260507 00:17:04.056 | INFO | MemInfo: { | 20260507 00:17:04.056 | INFO | "gpu" : " 17 GB", | 20260507 00:17:04.056 | INFO | "huge" : "", | 20260507 00:17:04.056 | INFO | "huge_shared" : " 27 GB", | 20260507 00:17:04.056 | INFO | "malloc" : " 14 GB", | 20260507 00:17:04.056 | INFO | "pinned" : " 29 GB" | 20260507 00:17:04.057 | INFO | } | 20260507 00:17:04.057 | INFO | Perf report for the last 5.14444 seconds: | 20260507 00:17:04.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:17:04.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:04.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:04.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:04.057 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 00:17:04.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:04.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:04.057 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 00:17:04.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:04.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:04.057 | INFO | Uploaded 36GiB to GPU in 1.98401s (18.145GiB/s) | 20260507 00:17:04.057 | INFO | Downloaded 34GiB to GPU in 1.93017s (17.615GiB/s) | 20260507 00:17:09.191 | INFO | Analyzing chunk frames = [5592064,5592576) | 20260507 00:17:09.191 | INFO | MemInfo: { | 20260507 00:17:09.191 | INFO | "gpu" : " 17 GB", | 20260507 00:17:09.191 | INFO | "huge" : "", | 20260507 00:17:09.191 | INFO | "huge_shared" : " 27 GB", | 20260507 00:17:09.191 | INFO | "malloc" : " 14 GB", | 20260507 00:17:09.191 | INFO | "pinned" : " 29 GB" | 20260507 00:17:09.191 | INFO | } | 20260507 00:17:09.191 | INFO | Perf report for the last 5.13454 seconds: | 20260507 00:17:09.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:17:09.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:09.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:09.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:09.191 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:17:09.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:09.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:09.191 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:17:09.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:09.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:09.191 | INFO | Uploaded 36GiB to GPU in 1.99432s (18.0513GiB/s) | 20260507 00:17:09.191 | INFO | Downloaded 34GiB to GPU in 1.93835s (17.5407GiB/s) | 20260507 00:17:14.332 | INFO | Analyzing chunk frames = [5592576,5593088) | 20260507 00:17:14.332 | INFO | MemInfo: { | 20260507 00:17:14.333 | INFO | "gpu" : " 17 GB", | 20260507 00:17:14.333 | INFO | "huge" : "", | 20260507 00:17:14.333 | INFO | "huge_shared" : " 27 GB", | 20260507 00:17:14.333 | INFO | "malloc" : " 14 GB", | 20260507 00:17:14.333 | INFO | "pinned" : " 29 GB" | 20260507 00:17:14.333 | INFO | } | 20260507 00:17:14.333 | INFO | Perf report for the last 5.14166 seconds: | 20260507 00:17:14.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:17:14.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:14.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:14.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:14.333 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260507 00:17:14.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:14.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:14.333 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:17:14.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:14.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:14.333 | INFO | Uploaded 36GiB to GPU in 1.99974s (18.0024GiB/s) | 20260507 00:17:14.333 | INFO | Downloaded 34GiB to GPU in 1.90929s (17.8076GiB/s) | 20260507 00:17:19.474 | INFO | Analyzing chunk frames = [5593088,5593600) | 20260507 00:17:19.474 | INFO | MemInfo: { | 20260507 00:17:19.474 | INFO | "gpu" : " 17 GB", | 20260507 00:17:19.474 | INFO | "huge" : "", | 20260507 00:17:19.474 | INFO | "huge_shared" : " 27 GB", | 20260507 00:17:19.474 | INFO | "malloc" : " 14 GB", | 20260507 00:17:19.474 | INFO | "pinned" : " 29 GB" | 20260507 00:17:19.474 | INFO | } | 20260507 00:17:19.474 | INFO | Perf report for the last 5.14135 seconds: | 20260507 00:17:19.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:17:19.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:19.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:19.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:19.474 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 00:17:19.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:19.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:19.474 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:17:19.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:19.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:19.474 | INFO | Uploaded 36GiB to GPU in 1.97907s (18.1904GiB/s) | 20260507 00:17:19.474 | INFO | Downloaded 34GiB to GPU in 1.93448s (17.5758GiB/s) | 20260507 00:17:24.613 | INFO | Analyzing chunk frames = [5593600,5594112) | 20260507 00:17:24.613 | INFO | MemInfo: { | 20260507 00:17:24.613 | INFO | "gpu" : " 17 GB", | 20260507 00:17:24.613 | INFO | "huge" : "", | 20260507 00:17:24.613 | INFO | "huge_shared" : " 27 GB", | 20260507 00:17:24.613 | INFO | "malloc" : " 14 GB", | 20260507 00:17:24.613 | INFO | "pinned" : " 29 GB" | 20260507 00:17:24.613 | INFO | } | 20260507 00:17:24.613 | INFO | Perf report for the last 5.13933 seconds: | 20260507 00:17:24.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:17:24.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:24.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:24.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:24.614 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 00:17:24.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:24.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:24.614 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 00:17:24.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:24.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:24.614 | INFO | Uploaded 36GiB to GPU in 1.99144s (18.0774GiB/s) | 20260507 00:17:24.614 | INFO | Downloaded 34GiB to GPU in 1.94474s (17.4831GiB/s) | 20260507 00:17:29.756 | INFO | Analyzing chunk frames = [5594112,5594624) | 20260507 00:17:29.756 | INFO | MemInfo: { | 20260507 00:17:29.756 | INFO | "gpu" : " 17 GB", | 20260507 00:17:29.756 | INFO | "huge" : "", | 20260507 00:17:29.756 | INFO | "huge_shared" : " 27 GB", | 20260507 00:17:29.756 | INFO | "malloc" : " 14 GB", | 20260507 00:17:29.756 | INFO | "pinned" : " 29 GB" | 20260507 00:17:29.756 | INFO | } | 20260507 00:17:29.756 | INFO | Perf report for the last 5.14295 seconds: | 20260507 00:17:29.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:17:29.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:29.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:29.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:29.757 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 00:17:29.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:29.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:29.757 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 00:17:29.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:29.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:29.757 | INFO | Uploaded 36GiB to GPU in 1.97132s (18.2618GiB/s) | 20260507 00:17:29.757 | INFO | Downloaded 34GiB to GPU in 1.99639s (17.0307GiB/s) | 20260507 00:17:34.893 | INFO | Analyzing chunk frames = [5594624,5595136) | 20260507 00:17:34.893 | INFO | MemInfo: { | 20260507 00:17:34.893 | INFO | "gpu" : " 17 GB", | 20260507 00:17:34.893 | INFO | "huge" : "", | 20260507 00:17:34.893 | INFO | "huge_shared" : " 27 GB", | 20260507 00:17:34.893 | INFO | "malloc" : " 14 GB", | 20260507 00:17:34.893 | INFO | "pinned" : " 29 GB" | 20260507 00:17:34.893 | INFO | } | 20260507 00:17:34.893 | INFO | Perf report for the last 5.13644 seconds: | 20260507 00:17:34.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:17:34.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:34.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:34.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:34.893 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:17:34.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:34.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:34.893 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 00:17:34.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:34.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:34.893 | INFO | Uploaded 36GiB to GPU in 2.00217s (17.9805GiB/s) | 20260507 00:17:34.893 | INFO | Downloaded 34GiB to GPU in 1.99262s (17.0629GiB/s) | 20260507 00:17:34.893 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:17:34.893 | INFO | Total mem usage: 18 GiB | 20260507 00:17:34.893 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:17:34.893 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:17:34.893 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:17:34.894 | INFO | High Water Breakdown: | 20260507 00:17:34.894 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:17:34.894 | INFO | 39.7% : 5.67 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:17:34.894 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:17:34.894 | INFO | ------------------------------------------ | 20260507 00:17:34.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:17:34.894 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:17:34.894 | INFO | Total mem usage: 26.5 GiB | 20260507 00:17:34.894 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:17:34.894 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:17:34.894 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:17:34.894 | INFO | High Water Breakdown: | 20260507 00:17:34.894 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:17:34.894 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:17:34.894 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:17:34.894 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:17:34.894 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:17:34.894 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:17:34.894 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:17:34.894 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:17:34.894 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:17:34.894 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:17:34.894 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:17:34.894 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:17:34.894 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:17:34.894 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:17:34.894 | INFO | ------------------------------------------ | 20260507 00:17:34.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:17:34.894 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:17:34.894 | INFO | Total mem usage: 25.6 GiB | 20260507 00:17:34.894 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:17:34.894 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:17:34.894 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:17:34.894 | INFO | High Water Breakdown: | 20260507 00:17:34.894 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:17:34.894 | INFO | ------------------------------------------ | 20260507 00:17:34.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:17:34.894 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:17:34.894 | INFO | Total mem usage: 15.7 GiB | 20260507 00:17:34.894 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:17:34.894 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:17:34.894 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:17:34.894 | INFO | High Water Breakdown: | 20260507 00:17:34.894 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:17:34.894 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:17:34.894 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:17:34.894 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:17:34.894 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:17:34.894 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:17:34.894 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:17:34.894 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:17:34.894 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:17:34.894 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:17:34.894 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:17:34.894 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:17:34.894 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:17:34.894 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:17:34.894 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:17:34.894 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:17:34.894 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:17:34.894 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:17:34.894 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:17:34.894 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:17:34.894 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:17:34.894 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:17:34.894 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:17:34.894 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:17:34.894 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:17:34.894 | INFO | ------------------------------------------ | 20260507 00:17:34.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:17:34.894 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:17:34.894 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:17:34.894 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:17:34.894 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.59e+01, 1.35e+01 / 2.45e+00 | 20260507 00:17:34.895 | INFO | PerfReport: 0.30%, AggregateStats -- 9.17e-02 / 9.85e-02, 1.25e-02 / 6.31e-03 | 20260507 00:17:34.895 | INFO | PerfReport: 11.70%, Baseline -- 3.62e+00 / 4.33e+00, 5.51e-01 / 3.74e-01 | 20260507 00:17:34.895 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.42e+00, 1.79e-01 / 1.59e-01 | 20260507 00:17:34.895 | INFO | PerfReport: 16.06%, DME -- 3.98e+01 / 4.45e+01, 4.10e+00 / 2.41e+00 | 20260507 00:17:34.895 | INFO | PerfReport: 57.43%, FrameLabeling -- 1.78e+01 / 2.12e+01, 1.35e+00 / 1.47e+00 | 20260507 00:17:34.895 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.17e+00 / 2.69e+00, 3.74e-01 / 2.14e-01 | 20260507 00:17:34.895 | INFO | PerfReport: 3.16%, Metrics -- 9.78e-01 / 1.30e+00, 2.60e-01 / 1.59e-01 | 20260507 00:17:34.895 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.33e-01 / 3.76e-01, 5.20e-02 / 1.43e-02 | 20260507 00:17:34.895 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:17:34.895 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:17:34.895 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:17:34.895 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260507 00:17:34.895 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 6.93e+00 / 0.00e+00 | 20260507 00:17:34.895 | INFO | PerfReport: 21.34%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 00:17:34.895 | INFO | PerfReport: 32.10%, Compute -- 3.10e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:17:34.895 | INFO | PerfReport: 1.21%, LockUpload -- 1.17e+00 / 0.00e+00, 4.47e+00 / 0.00e+00 | 20260507 00:17:34.895 | INFO | PerfReport: 1.30%, LockDownload -- 1.26e+00 / 0.00e+00, 3.16e+00 / 0.00e+00 | 20260507 00:17:34.895 | INFO | PerfReport: 22.50%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 00:17:34.895 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.80e-02 / 0.00e+00 | 20260507 00:17:34.992 | INFO | Starting a round of Baz IO | 20260507 00:17:38.119 | INFO | Populating RTMetrics report | 20260507 00:17:38.119 | INFO | RTMetrics report populated | 20260507 00:17:38.119 | INFO | Opening tmp RTMetrics json file | 20260507 00:17:38.138 | INFO | Serializing RTMetrics to json | 20260507 00:17:38.140 | INFO | Serialization done, swapping tmp file now... | 20260507 00:17:38.140 | INFO | RTMetrics done | 20260507 00:17:38.218 | INFO | Finished a round of Baz IO: Wrote 1.852 GiB in 3.225 seconds | 20260507 00:17:38.218 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.31% / 0.00% / 66.68% | 20260507 00:17:38.218 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.353539% / 3.97364e-06% / 8.34465e-05% | 20260507 00:17:38.218 | INFO | Baz IO overall: | 20260507 00:17:38.218 | INFO | BytesWritten : 2918539618196 | 20260507 00:17:38.218 | INFO | NumEvents : 1406149849491 | 20260507 00:17:40.028 | INFO | Analyzing chunk frames = [5595136,5595648) | 20260507 00:17:40.028 | INFO | MemInfo: { | 20260507 00:17:40.028 | INFO | "gpu" : " 17 GB", | 20260507 00:17:40.028 | INFO | "huge" : "", | 20260507 00:17:40.028 | INFO | "huge_shared" : " 27 GB", | 20260507 00:17:40.028 | INFO | "malloc" : " 14 GB", | 20260507 00:17:40.028 | INFO | "pinned" : " 29 GB" | 20260507 00:17:40.028 | INFO | } | 20260507 00:17:40.028 | INFO | Perf report for the last 5.13488 seconds: | 20260507 00:17:40.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:17:40.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:40.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:40.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:40.028 | INFO | ANALYSIS : 62.3%, 3.0, 96.7ms, 0 | 20260507 00:17:40.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:40.028 | INFO | RT_METRICS : 42.7%, 1.0, 22.3ms, 0 | 20260507 00:17:40.028 | INFO | PRE_HQ : 59.7%, 1.8, 56.7ms, 0 | 20260507 00:17:40.028 | INFO | BAZWRITER : 62.4%, 3.4, 110.5ms, 0 | 20260507 00:17:40.028 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:17:40.028 | INFO | Uploaded 36GiB to GPU in 2.02481s (17.7795GiB/s) | 20260507 00:17:40.028 | INFO | Downloaded 38GiB to GPU in 2.1829s (17.4081GiB/s) | 20260507 00:17:45.169 | INFO | Analyzing chunk frames = [5595648,5596160) | 20260507 00:17:45.169 | INFO | MemInfo: { | 20260507 00:17:45.169 | INFO | "gpu" : " 17 GB", | 20260507 00:17:45.169 | INFO | "huge" : "", | 20260507 00:17:45.169 | INFO | "huge_shared" : " 27 GB", | 20260507 00:17:45.169 | INFO | "malloc" : " 14 GB", | 20260507 00:17:45.169 | INFO | "pinned" : " 29 GB" | 20260507 00:17:45.169 | INFO | } | 20260507 00:17:45.170 | INFO | Perf report for the last 5.14185 seconds: | 20260507 00:17:45.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:17:45.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:45.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:45.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:45.170 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:17:45.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:45.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:45.170 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:17:45.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:45.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:45.170 | INFO | Uploaded 36GiB to GPU in 2.00811s (17.9273GiB/s) | 20260507 00:17:45.170 | INFO | Downloaded 34GiB to GPU in 1.92678s (17.646GiB/s) | 20260507 00:17:50.305 | INFO | Analyzing chunk frames = [5596160,5596672) | 20260507 00:17:50.305 | INFO | MemInfo: { | 20260507 00:17:50.305 | INFO | "gpu" : " 17 GB", | 20260507 00:17:50.305 | INFO | "huge" : "", | 20260507 00:17:50.305 | INFO | "huge_shared" : " 27 GB", | 20260507 00:17:50.305 | INFO | "malloc" : " 14 GB", | 20260507 00:17:50.305 | INFO | "pinned" : " 29 GB" | 20260507 00:17:50.305 | INFO | } | 20260507 00:17:50.305 | INFO | Perf report for the last 5.13574 seconds: | 20260507 00:17:50.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:17:50.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:50.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:50.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:50.305 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 00:17:50.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:50.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:50.305 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:17:50.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:50.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:50.306 | INFO | Uploaded 36GiB to GPU in 1.98692s (18.1185GiB/s) | 20260507 00:17:50.306 | INFO | Downloaded 34GiB to GPU in 1.92891s (17.6265GiB/s) | 20260507 00:17:55.447 | INFO | Analyzing chunk frames = [5596672,5597184) | 20260507 00:17:55.447 | INFO | MemInfo: { | 20260507 00:17:55.447 | INFO | "gpu" : " 17 GB", | 20260507 00:17:55.447 | INFO | "huge" : "", | 20260507 00:17:55.447 | INFO | "huge_shared" : " 27 GB", | 20260507 00:17:55.447 | INFO | "malloc" : " 14 GB", | 20260507 00:17:55.447 | INFO | "pinned" : " 29 GB" | 20260507 00:17:55.447 | INFO | } | 20260507 00:17:55.447 | INFO | Perf report for the last 5.14171 seconds: | 20260507 00:17:55.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:17:55.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:55.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:55.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:55.447 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260507 00:17:55.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:55.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:55.447 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:17:55.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:55.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:17:55.447 | INFO | Uploaded 36GiB to GPU in 2.01635s (17.8541GiB/s) | 20260507 00:17:55.447 | INFO | Downloaded 34GiB to GPU in 1.90851s (17.815GiB/s) | 20260507 00:18:00.585 | INFO | Analyzing chunk frames = [5597184,5597696) | 20260507 00:18:00.585 | INFO | MemInfo: { | 20260507 00:18:00.585 | INFO | "gpu" : " 17 GB", | 20260507 00:18:00.585 | INFO | "huge" : "", | 20260507 00:18:00.585 | INFO | "huge_shared" : " 27 GB", | 20260507 00:18:00.585 | INFO | "malloc" : " 14 GB", | 20260507 00:18:00.585 | INFO | "pinned" : " 29 GB" | 20260507 00:18:00.585 | INFO | } | 20260507 00:18:00.585 | INFO | Perf report for the last 5.13772 seconds: | 20260507 00:18:00.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:18:00.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:00.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:00.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:00.585 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 00:18:00.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:00.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:00.585 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:18:00.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:00.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:00.585 | INFO | Uploaded 36GiB to GPU in 1.99392s (18.0549GiB/s) | 20260507 00:18:00.585 | INFO | Downloaded 34GiB to GPU in 1.92059s (17.7029GiB/s) | 20260507 00:18:05.727 | INFO | Analyzing chunk frames = [5597696,5598208) | 20260507 00:18:05.727 | INFO | MemInfo: { | 20260507 00:18:05.727 | INFO | "gpu" : " 17 GB", | 20260507 00:18:05.727 | INFO | "huge" : "", | 20260507 00:18:05.727 | INFO | "huge_shared" : " 27 GB", | 20260507 00:18:05.727 | INFO | "malloc" : " 14 GB", | 20260507 00:18:05.727 | INFO | "pinned" : " 29 GB" | 20260507 00:18:05.727 | INFO | } | 20260507 00:18:05.727 | INFO | Perf report for the last 5.14201 seconds: | 20260507 00:18:05.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:18:05.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:05.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:05.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:05.727 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 00:18:05.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:05.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:05.727 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:18:05.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:05.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:05.727 | INFO | Uploaded 36GiB to GPU in 2.00309s (17.9722GiB/s) | 20260507 00:18:05.727 | INFO | Downloaded 34GiB to GPU in 1.94239s (17.5043GiB/s) | 20260507 00:18:10.863 | INFO | Analyzing chunk frames = [5598208,5598720) | 20260507 00:18:10.863 | INFO | MemInfo: { | 20260507 00:18:10.863 | INFO | "gpu" : " 17 GB", | 20260507 00:18:10.863 | INFO | "huge" : "", | 20260507 00:18:10.863 | INFO | "huge_shared" : " 27 GB", | 20260507 00:18:10.863 | INFO | "malloc" : " 14 GB", | 20260507 00:18:10.864 | INFO | "pinned" : " 29 GB" | 20260507 00:18:10.864 | INFO | } | 20260507 00:18:10.864 | INFO | Perf report for the last 5.13682 seconds: | 20260507 00:18:10.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:18:10.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:10.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:10.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:10.864 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260507 00:18:10.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:10.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:10.864 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 00:18:10.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:10.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:10.864 | INFO | Uploaded 36GiB to GPU in 2.00363s (17.9674GiB/s) | 20260507 00:18:10.864 | INFO | Downloaded 34GiB to GPU in 1.99128s (17.0745GiB/s) | 20260507 00:18:16.008 | INFO | Analyzing chunk frames = [5598720,5599232) | 20260507 00:18:16.008 | INFO | MemInfo: { | 20260507 00:18:16.008 | INFO | "gpu" : " 17 GB", | 20260507 00:18:16.008 | INFO | "huge" : "", | 20260507 00:18:16.009 | INFO | "huge_shared" : " 27 GB", | 20260507 00:18:16.009 | INFO | "malloc" : " 14 GB", | 20260507 00:18:16.009 | INFO | "pinned" : " 29 GB" | 20260507 00:18:16.009 | INFO | } | 20260507 00:18:16.009 | INFO | Perf report for the last 5.14505 seconds: | 20260507 00:18:16.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:18:16.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:16.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:16.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:16.009 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 00:18:16.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:16.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:16.009 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 00:18:16.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:16.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:16.009 | INFO | Uploaded 36GiB to GPU in 1.96896s (18.2838GiB/s) | 20260507 00:18:16.009 | INFO | Downloaded 34GiB to GPU in 1.98829s (17.1001GiB/s) | 20260507 00:18:16.106 | INFO | Starting a round of Baz IO | 20260507 00:18:19.228 | INFO | Populating RTMetrics report | 20260507 00:18:19.228 | INFO | RTMetrics report populated | 20260507 00:18:19.228 | INFO | Opening tmp RTMetrics json file | 20260507 00:18:19.230 | INFO | Serializing RTMetrics to json | 20260507 00:18:19.249 | INFO | Serialization done, swapping tmp file now... | 20260507 00:18:19.249 | INFO | RTMetrics done | 20260507 00:18:19.322 | INFO | Finished a round of Baz IO: Wrote 1.85 GiB in 3.216 seconds | 20260507 00:18:19.322 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.31% / 0.00% / 66.69% | 20260507 00:18:19.322 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.353789% / 0% / 7.15256e-05% | 20260507 00:18:19.322 | INFO | Baz IO overall: | 20260507 00:18:19.322 | INFO | BytesWritten : 2920526494945 | 20260507 00:18:19.322 | INFO | NumEvents : 1407093996209 | 20260507 00:18:21.145 | INFO | Analyzing chunk frames = [5599232,5599744) | 20260507 00:18:21.145 | INFO | MemInfo: { | 20260507 00:18:21.145 | INFO | "gpu" : " 17 GB", | 20260507 00:18:21.145 | INFO | "huge" : "", | 20260507 00:18:21.145 | INFO | "huge_shared" : " 27 GB", | 20260507 00:18:21.145 | INFO | "malloc" : " 14 GB", | 20260507 00:18:21.145 | INFO | "pinned" : " 29 GB" | 20260507 00:18:21.145 | INFO | } | 20260507 00:18:21.145 | INFO | Perf report for the last 5.13632 seconds: | 20260507 00:18:21.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:18:21.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:21.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:21.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:21.145 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260507 00:18:21.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:21.145 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 00:18:21.145 | INFO | PRE_HQ : 59.6%, 1.8, 56.5ms, 0 | 20260507 00:18:21.145 | INFO | BAZWRITER : 62.2%, 3.4, 110.0ms, 0 | 20260507 00:18:21.145 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:18:21.145 | INFO | Uploaded 36GiB to GPU in 2.01905s (17.8302GiB/s) | 20260507 00:18:21.145 | INFO | Downloaded 38GiB to GPU in 2.17509s (17.4705GiB/s) | 20260507 00:18:26.282 | INFO | Analyzing chunk frames = [5599744,5600256) | 20260507 00:18:26.282 | INFO | MemInfo: { | 20260507 00:18:26.282 | INFO | "gpu" : " 17 GB", | 20260507 00:18:26.282 | INFO | "huge" : "", | 20260507 00:18:26.282 | INFO | "huge_shared" : " 27 GB", | 20260507 00:18:26.282 | INFO | "malloc" : " 14 GB", | 20260507 00:18:26.282 | INFO | "pinned" : " 29 GB" | 20260507 00:18:26.282 | INFO | } | 20260507 00:18:26.283 | INFO | Perf report for the last 5.13746 seconds: | 20260507 00:18:26.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:18:26.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:26.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:26.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:26.283 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:18:26.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:26.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:26.283 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:18:26.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:26.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:26.283 | INFO | Uploaded 36GiB to GPU in 2.00775s (17.9305GiB/s) | 20260507 00:18:26.283 | INFO | Downloaded 34GiB to GPU in 1.93514s (17.5698GiB/s) | 20260507 00:18:31.432 | INFO | Analyzing chunk frames = [5600256,5600768) | 20260507 00:18:31.432 | INFO | MemInfo: { | 20260507 00:18:31.432 | INFO | "gpu" : " 17 GB", | 20260507 00:18:31.432 | INFO | "huge" : "", | 20260507 00:18:31.432 | INFO | "huge_shared" : " 27 GB", | 20260507 00:18:31.432 | INFO | "malloc" : " 14 GB", | 20260507 00:18:31.432 | INFO | "pinned" : " 29 GB" | 20260507 00:18:31.432 | INFO | } | 20260507 00:18:31.433 | INFO | Perf report for the last 5.14998 seconds: | 20260507 00:18:31.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:18:31.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:31.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:31.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:31.433 | INFO | ANALYSIS : 60.2%, 3.0, 93.5ms, 0 | 20260507 00:18:31.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:31.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:31.433 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:18:31.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:31.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:31.433 | INFO | Uploaded 36GiB to GPU in 1.98393s (18.1458GiB/s) | 20260507 00:18:31.433 | INFO | Downloaded 34GiB to GPU in 1.93637s (17.5587GiB/s) | 20260507 00:18:36.561 | INFO | Analyzing chunk frames = [5600768,5601280) | 20260507 00:18:36.561 | INFO | MemInfo: { | 20260507 00:18:36.561 | INFO | "gpu" : " 17 GB", | 20260507 00:18:36.561 | INFO | "huge" : "", | 20260507 00:18:36.561 | INFO | "huge_shared" : " 27 GB", | 20260507 00:18:36.561 | INFO | "malloc" : " 14 GB", | 20260507 00:18:36.561 | INFO | "pinned" : " 29 GB" | 20260507 00:18:36.561 | INFO | } | 20260507 00:18:36.561 | INFO | Perf report for the last 5.12889 seconds: | 20260507 00:18:36.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:18:36.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:36.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:36.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:36.561 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260507 00:18:36.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:36.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:36.562 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:18:36.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:36.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:36.562 | INFO | Uploaded 36GiB to GPU in 1.97593s (18.2193GiB/s) | 20260507 00:18:36.562 | INFO | Downloaded 34GiB to GPU in 1.91206s (17.7819GiB/s) | 20260507 00:18:41.703 | INFO | Analyzing chunk frames = [5601280,5601792) | 20260507 00:18:41.703 | INFO | MemInfo: { | 20260507 00:18:41.703 | INFO | "gpu" : " 17 GB", | 20260507 00:18:41.703 | INFO | "huge" : "", | 20260507 00:18:41.703 | INFO | "huge_shared" : " 27 GB", | 20260507 00:18:41.703 | INFO | "malloc" : " 14 GB", | 20260507 00:18:41.703 | INFO | "pinned" : " 29 GB" | 20260507 00:18:41.703 | INFO | } | 20260507 00:18:41.703 | INFO | Perf report for the last 5.14192 seconds: | 20260507 00:18:41.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:18:41.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:41.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:41.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:41.703 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 00:18:41.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:41.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:41.703 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 00:18:41.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:41.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:41.704 | INFO | Uploaded 36GiB to GPU in 1.99308s (18.0625GiB/s) | 20260507 00:18:41.704 | INFO | Downloaded 34GiB to GPU in 1.91943s (17.7136GiB/s) | 20260507 00:18:46.842 | INFO | Analyzing chunk frames = [5601792,5602304) | 20260507 00:18:46.842 | INFO | MemInfo: { | 20260507 00:18:46.842 | INFO | "gpu" : " 17 GB", | 20260507 00:18:46.842 | INFO | "huge" : "", | 20260507 00:18:46.842 | INFO | "huge_shared" : " 27 GB", | 20260507 00:18:46.842 | INFO | "malloc" : " 14 GB", | 20260507 00:18:46.842 | INFO | "pinned" : " 29 GB" | 20260507 00:18:46.842 | INFO | } | 20260507 00:18:46.842 | INFO | Perf report for the last 5.13847 seconds: | 20260507 00:18:46.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:18:46.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:46.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:46.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:46.844 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:18:46.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:46.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:46.844 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 00:18:46.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:46.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:46.844 | INFO | Uploaded 36GiB to GPU in 2.01129s (17.899GiB/s) | 20260507 00:18:46.844 | INFO | Downloaded 34GiB to GPU in 1.95454s (17.3954GiB/s) | 20260507 00:18:51.976 | INFO | Analyzing chunk frames = [5602304,5602816) | 20260507 00:18:51.976 | INFO | MemInfo: { | 20260507 00:18:51.976 | INFO | "gpu" : " 17 GB", | 20260507 00:18:51.976 | INFO | "huge" : "", | 20260507 00:18:51.976 | INFO | "huge_shared" : " 27 GB", | 20260507 00:18:51.977 | INFO | "malloc" : " 14 GB", | 20260507 00:18:51.977 | INFO | "pinned" : " 29 GB" | 20260507 00:18:51.977 | INFO | } | 20260507 00:18:51.977 | INFO | Perf report for the last 5.1348 seconds: | 20260507 00:18:51.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:18:51.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:51.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:51.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:51.977 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:18:51.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:51.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:51.977 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 00:18:51.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:51.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:51.977 | INFO | Uploaded 36GiB to GPU in 1.97556s (18.2227GiB/s) | 20260507 00:18:51.977 | INFO | Downloaded 34GiB to GPU in 2.00238s (16.9798GiB/s) | 20260507 00:18:57.117 | INFO | Analyzing chunk frames = [5602816,5603328) | 20260507 00:18:57.117 | INFO | MemInfo: { | 20260507 00:18:57.117 | INFO | "gpu" : " 17 GB", | 20260507 00:18:57.121 | INFO | "huge" : "", | 20260507 00:18:57.121 | INFO | "huge_shared" : " 27 GB", | 20260507 00:18:57.121 | INFO | "malloc" : " 14 GB", | 20260507 00:18:57.121 | INFO | "pinned" : " 29 GB" | 20260507 00:18:57.121 | INFO | } | 20260507 00:18:57.121 | INFO | Perf report for the last 5.14463 seconds: | 20260507 00:18:57.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:18:57.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:57.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:57.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:57.121 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:18:57.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:57.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:57.121 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 00:18:57.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:57.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:18:57.122 | INFO | Uploaded 36GiB to GPU in 1.98811s (18.1076GiB/s) | 20260507 00:18:57.122 | INFO | Downloaded 34GiB to GPU in 1.98149s (17.1588GiB/s) | 20260507 00:18:57.122 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:18:57.122 | INFO | Total mem usage: 18 GiB | 20260507 00:18:57.122 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:18:57.122 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:18:57.122 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:18:57.122 | INFO | High Water Breakdown: | 20260507 00:18:57.122 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:18:57.122 | INFO | 39.7% : 5.67 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:18:57.122 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:18:57.122 | INFO | ------------------------------------------ | 20260507 00:18:57.122 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:18:57.122 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:18:57.122 | INFO | Total mem usage: 26.5 GiB | 20260507 00:18:57.122 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:18:57.122 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:18:57.122 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:18:57.122 | INFO | High Water Breakdown: | 20260507 00:18:57.122 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:18:57.122 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:18:57.122 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:18:57.122 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:18:57.122 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:18:57.122 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:18:57.122 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:18:57.122 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:18:57.122 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:18:57.122 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:18:57.122 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:18:57.122 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:18:57.122 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:18:57.122 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:18:57.122 | INFO | ------------------------------------------ | 20260507 00:18:57.122 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:18:57.122 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:18:57.122 | INFO | Total mem usage: 25.6 GiB | 20260507 00:18:57.122 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:18:57.122 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:18:57.122 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:18:57.122 | INFO | High Water Breakdown: | 20260507 00:18:57.122 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:18:57.122 | INFO | ------------------------------------------ | 20260507 00:18:57.122 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:18:57.122 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:18:57.122 | INFO | Total mem usage: 15.7 GiB | 20260507 00:18:57.122 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:18:57.122 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:18:57.122 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:18:57.122 | INFO | High Water Breakdown: | 20260507 00:18:57.122 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:18:57.122 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:18:57.122 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:18:57.122 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:18:57.122 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:18:57.122 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:18:57.122 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:18:57.122 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:18:57.122 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:18:57.122 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:18:57.122 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:18:57.122 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:18:57.122 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:18:57.122 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:18:57.122 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:18:57.122 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:18:57.122 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:18:57.122 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:18:57.122 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:18:57.122 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:18:57.123 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:18:57.123 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:18:57.123 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:18:57.123 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:18:57.123 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:18:57.123 | INFO | ------------------------------------------ | 20260507 00:18:57.123 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:18:57.123 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:18:57.123 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:18:57.123 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:18:57.123 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.65e+01, 1.35e+01 / 2.06e+00 | 20260507 00:18:57.123 | INFO | PerfReport: 0.29%, AggregateStats -- 9.13e-02 / 1.02e-01, 1.06e-02 / 4.96e-03 | 20260507 00:18:57.123 | INFO | PerfReport: 11.66%, Baseline -- 3.61e+00 / 4.46e+00, 5.42e-01 / 2.82e-01 | 20260507 00:18:57.123 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.49e+00, 1.80e-01 / 1.00e-01 | 20260507 00:18:57.123 | INFO | PerfReport: 16.09%, DME -- 3.98e+01 / 4.49e+01, 4.19e+00 / 2.79e+00 | 20260507 00:18:57.123 | INFO | PerfReport: 57.46%, FrameLabeling -- 1.78e+01 / 2.11e+01, 1.38e+00 / 2.16e+00 | 20260507 00:18:57.123 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.17e+00 / 2.72e+00, 3.76e-01 / 2.02e-01 | 20260507 00:18:57.123 | INFO | PerfReport: 3.15%, Metrics -- 9.77e-01 / 1.31e+00, 2.61e-01 / 1.51e-01 | 20260507 00:18:57.123 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.33e-01 / 3.85e-01, 5.33e-02 / 2.78e-02 | 20260507 00:18:57.123 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:18:57.123 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:18:57.123 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:18:57.123 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 00:18:57.123 | INFO | PerfReport: 21.55%, Upload -- 2.08e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260507 00:18:57.123 | INFO | PerfReport: 21.31%, Download -- 2.06e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260507 00:18:57.123 | INFO | PerfReport: 32.06%, Compute -- 3.10e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:18:57.123 | INFO | PerfReport: 1.28%, LockUpload -- 1.23e+00 / 0.00e+00, 4.74e+00 / 0.00e+00 | 20260507 00:18:57.123 | INFO | PerfReport: 1.30%, LockDownload -- 1.26e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260507 00:18:57.123 | INFO | PerfReport: 22.49%, LockCompute -- 2.17e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 00:18:57.123 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260507 00:18:57.218 | INFO | Starting a round of Baz IO | 20260507 00:19:00.350 | INFO | Populating RTMetrics report | 20260507 00:19:00.351 | INFO | RTMetrics report populated | 20260507 00:19:00.351 | INFO | Opening tmp RTMetrics json file | 20260507 00:19:00.374 | INFO | Serializing RTMetrics to json | 20260507 00:19:00.375 | INFO | Serialization done, swapping tmp file now... | 20260507 00:19:00.376 | INFO | RTMetrics done | 20260507 00:19:00.450 | INFO | Finished a round of Baz IO: Wrote 1.853 GiB in 3.231 seconds | 20260507 00:19:00.450 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.31% / 0.00% / 66.69% | 20260507 00:19:00.450 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.353678% / 7.94729e-06% / 6.35783e-05% | 20260507 00:19:00.450 | INFO | Baz IO overall: | 20260507 00:19:00.450 | INFO | BytesWritten : 2922515740683 | 20260507 00:19:00.450 | INFO | NumEvents : 1408039892661 | 20260507 00:19:02.259 | INFO | Analyzing chunk frames = [5603328,5603840) | 20260507 00:19:02.259 | INFO | MemInfo: { | 20260507 00:19:02.259 | INFO | "gpu" : " 17 GB", | 20260507 00:19:02.259 | INFO | "huge" : "", | 20260507 00:19:02.259 | INFO | "huge_shared" : " 27 GB", | 20260507 00:19:02.259 | INFO | "malloc" : " 14 GB", | 20260507 00:19:02.259 | INFO | "pinned" : " 29 GB" | 20260507 00:19:02.259 | INFO | } | 20260507 00:19:02.259 | INFO | Perf report for the last 5.13773 seconds: | 20260507 00:19:02.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:19:02.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:02.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:02.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:02.259 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 00:19:02.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:02.259 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260507 00:19:02.259 | INFO | PRE_HQ : 59.9%, 1.8, 56.9ms, 0 | 20260507 00:19:02.259 | INFO | BAZWRITER : 62.5%, 3.5, 111.2ms, 0 | 20260507 00:19:02.259 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:19:02.259 | INFO | Uploaded 36GiB to GPU in 2.02124s (17.8108GiB/s) | 20260507 00:19:02.259 | INFO | Downloaded 38GiB to GPU in 2.1844s (17.3961GiB/s) | 20260507 00:19:07.396 | INFO | Analyzing chunk frames = [5603840,5604352) | 20260507 00:19:07.396 | INFO | MemInfo: { | 20260507 00:19:07.396 | INFO | "gpu" : " 17 GB", | 20260507 00:19:07.396 | INFO | "huge" : "", | 20260507 00:19:07.396 | INFO | "huge_shared" : " 27 GB", | 20260507 00:19:07.396 | INFO | "malloc" : " 14 GB", | 20260507 00:19:07.396 | INFO | "pinned" : " 29 GB" | 20260507 00:19:07.396 | INFO | } | 20260507 00:19:07.396 | INFO | Perf report for the last 5.13705 seconds: | 20260507 00:19:07.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:19:07.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:07.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:07.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:07.396 | INFO | ANALYSIS : 61.3%, 3.0, 95.5ms, 0 | 20260507 00:19:07.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:07.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:07.396 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:19:07.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:07.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:07.396 | INFO | Uploaded 36GiB to GPU in 2.02501s (17.7777GiB/s) | 20260507 00:19:07.397 | INFO | Downloaded 34GiB to GPU in 1.9432s (17.4969GiB/s) | 20260507 00:19:12.539 | INFO | Analyzing chunk frames = [5604352,5604864) | 20260507 00:19:12.539 | INFO | MemInfo: { | 20260507 00:19:12.539 | INFO | "gpu" : " 17 GB", | 20260507 00:19:12.539 | INFO | "huge" : "", | 20260507 00:19:12.539 | INFO | "huge_shared" : " 27 GB", | 20260507 00:19:12.539 | INFO | "malloc" : " 14 GB", | 20260507 00:19:12.539 | INFO | "pinned" : " 29 GB" | 20260507 00:19:12.539 | INFO | } | 20260507 00:19:12.539 | INFO | Perf report for the last 5.14289 seconds: | 20260507 00:19:12.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:19:12.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:12.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:12.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:12.539 | INFO | ANALYSIS : 60.3%, 3.0, 93.6ms, 0 | 20260507 00:19:12.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:12.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:12.539 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:19:12.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:12.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:12.539 | INFO | Uploaded 36GiB to GPU in 1.99647s (18.0319GiB/s) | 20260507 00:19:12.539 | INFO | Downloaded 34GiB to GPU in 1.92889s (17.6268GiB/s) | 20260507 00:19:17.680 | INFO | Analyzing chunk frames = [5604864,5605376) | 20260507 00:19:17.680 | INFO | MemInfo: { | 20260507 00:19:17.680 | INFO | "gpu" : " 17 GB", | 20260507 00:19:17.680 | INFO | "huge" : "", | 20260507 00:19:17.680 | INFO | "huge_shared" : " 27 GB", | 20260507 00:19:17.680 | INFO | "malloc" : " 14 GB", | 20260507 00:19:17.680 | INFO | "pinned" : " 29 GB" | 20260507 00:19:17.680 | INFO | } | 20260507 00:19:17.681 | INFO | Perf report for the last 5.1415 seconds: | 20260507 00:19:17.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:19:17.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:17.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:17.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:17.681 | INFO | ANALYSIS : 59.5%, 3.0, 92.2ms, 0 | 20260507 00:19:17.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:17.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:17.681 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:19:17.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:17.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:17.681 | INFO | Uploaded 36GiB to GPU in 1.97476s (18.2301GiB/s) | 20260507 00:19:17.681 | INFO | Downloaded 34GiB to GPU in 1.88156s (18.0701GiB/s) | 20260507 00:19:22.819 | INFO | Analyzing chunk frames = [5605376,5605888) | 20260507 00:19:22.819 | INFO | MemInfo: { | 20260507 00:19:22.819 | INFO | "gpu" : " 17 GB", | 20260507 00:19:22.819 | INFO | "huge" : "", | 20260507 00:19:22.819 | INFO | "huge_shared" : " 27 GB", | 20260507 00:19:22.819 | INFO | "malloc" : " 14 GB", | 20260507 00:19:22.819 | INFO | "pinned" : " 29 GB" | 20260507 00:19:22.819 | INFO | } | 20260507 00:19:22.819 | INFO | Perf report for the last 5.13869 seconds: | 20260507 00:19:22.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:19:22.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:22.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:22.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:22.819 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 00:19:22.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:22.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:22.820 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:19:22.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:22.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:22.820 | INFO | Uploaded 36GiB to GPU in 2.01966s (17.8248GiB/s) | 20260507 00:19:22.820 | INFO | Downloaded 34GiB to GPU in 1.94384s (17.4912GiB/s) | 20260507 00:19:27.952 | INFO | Analyzing chunk frames = [5605888,5606400) | 20260507 00:19:27.952 | INFO | MemInfo: { | 20260507 00:19:27.952 | INFO | "gpu" : " 17 GB", | 20260507 00:19:27.952 | INFO | "huge" : "", | 20260507 00:19:27.952 | INFO | "huge_shared" : " 27 GB", | 20260507 00:19:27.952 | INFO | "malloc" : " 14 GB", | 20260507 00:19:27.952 | INFO | "pinned" : " 29 GB" | 20260507 00:19:27.952 | INFO | } | 20260507 00:19:27.952 | INFO | Perf report for the last 5.13299 seconds: | 20260507 00:19:27.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:19:27.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:27.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:27.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:27.952 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 00:19:27.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:27.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:27.952 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260507 00:19:27.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:27.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:27.953 | INFO | Uploaded 36GiB to GPU in 2.0586s (17.4876GiB/s) | 20260507 00:19:27.953 | INFO | Downloaded 34GiB to GPU in 1.95705s (17.3731GiB/s) | 20260507 00:19:33.096 | INFO | Analyzing chunk frames = [5606400,5606912) | 20260507 00:19:33.096 | INFO | MemInfo: { | 20260507 00:19:33.096 | INFO | "gpu" : " 17 GB", | 20260507 00:19:33.096 | INFO | "huge" : "", | 20260507 00:19:33.096 | INFO | "huge_shared" : " 27 GB", | 20260507 00:19:33.096 | INFO | "malloc" : " 14 GB", | 20260507 00:19:33.096 | INFO | "pinned" : " 29 GB" | 20260507 00:19:33.096 | INFO | } | 20260507 00:19:33.096 | INFO | Perf report for the last 5.14426 seconds: | 20260507 00:19:33.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:19:33.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:33.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:33.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:33.097 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 00:19:33.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:33.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:33.097 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 00:19:33.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:33.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:33.097 | INFO | Uploaded 36GiB to GPU in 1.98525s (18.1337GiB/s) | 20260507 00:19:33.097 | INFO | Downloaded 34GiB to GPU in 2.00576s (16.9512GiB/s) | 20260507 00:19:38.236 | INFO | Analyzing chunk frames = [5606912,5607424) | 20260507 00:19:38.236 | INFO | MemInfo: { | 20260507 00:19:38.236 | INFO | "gpu" : " 17 GB", | 20260507 00:19:38.236 | INFO | "huge" : "", | 20260507 00:19:38.236 | INFO | "huge_shared" : " 27 GB", | 20260507 00:19:38.236 | INFO | "malloc" : " 14 GB", | 20260507 00:19:38.236 | INFO | "pinned" : " 29 GB" | 20260507 00:19:38.236 | INFO | } | 20260507 00:19:38.237 | INFO | Perf report for the last 5.14002 seconds: | 20260507 00:19:38.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:19:38.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:38.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:38.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:38.237 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260507 00:19:38.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:38.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:38.237 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 00:19:38.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:38.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:38.237 | INFO | Uploaded 36GiB to GPU in 2.00979s (17.9123GiB/s) | 20260507 00:19:38.237 | INFO | Downloaded 34GiB to GPU in 1.99592s (17.0348GiB/s) | 20260507 00:19:38.334 | INFO | Starting a round of Baz IO | 20260507 00:19:41.462 | INFO | Populating RTMetrics report | 20260507 00:19:41.462 | INFO | RTMetrics report populated | 20260507 00:19:41.462 | INFO | Opening tmp RTMetrics json file | 20260507 00:19:41.619 | INFO | Serializing RTMetrics to json | 20260507 00:19:41.685 | INFO | Serialization done, swapping tmp file now... | 20260507 00:19:41.690 | INFO | RTMetrics done | 20260507 00:19:41.716 | INFO | Finished a round of Baz IO: Wrote 1.849 GiB in 3.382 seconds | 20260507 00:19:41.716 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.31% / 0.00% / 66.69% | 20260507 00:19:41.716 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.351111% / 0% / 6.35783e-05% | 20260507 00:19:41.716 | INFO | Baz IO overall: | 20260507 00:19:41.716 | INFO | BytesWritten : 2924501213916 | 20260507 00:19:41.716 | INFO | NumEvents : 1408982908215 | 20260507 00:19:43.375 | INFO | Analyzing chunk frames = [5607424,5607936) | 20260507 00:19:43.375 | INFO | MemInfo: { | 20260507 00:19:43.375 | INFO | "gpu" : " 17 GB", | 20260507 00:19:43.375 | INFO | "huge" : "", | 20260507 00:19:43.375 | INFO | "huge_shared" : " 27 GB", | 20260507 00:19:43.375 | INFO | "malloc" : " 14 GB", | 20260507 00:19:43.375 | INFO | "pinned" : " 29 GB" | 20260507 00:19:43.375 | INFO | } | 20260507 00:19:43.375 | INFO | Perf report for the last 5.13866 seconds: | 20260507 00:19:43.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:19:43.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:43.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:43.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:43.375 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260507 00:19:43.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:43.375 | INFO | RT_METRICS : 49.4%, 1.0, 25.9ms, 0 | 20260507 00:19:43.375 | INFO | PRE_HQ : 60.4%, 1.9, 60.2ms, 0 | 20260507 00:19:43.375 | INFO | BAZWRITER : 65.4%, 3.7, 124.0ms, 0 | 20260507 00:19:43.375 | INFO | BAZ_SAMPLING : 4.2%, 1.0, 2.2ms, 0 | 20260507 00:19:43.375 | INFO | Uploaded 36GiB to GPU in 2.03403s (17.6989GiB/s) | 20260507 00:19:43.375 | INFO | Downloaded 38GiB to GPU in 2.17704s (17.4549GiB/s) | 20260507 00:19:48.518 | INFO | Analyzing chunk frames = [5607936,5608448) | 20260507 00:19:48.518 | INFO | MemInfo: { | 20260507 00:19:48.518 | INFO | "gpu" : " 17 GB", | 20260507 00:19:48.518 | INFO | "huge" : "", | 20260507 00:19:48.518 | INFO | "huge_shared" : " 27 GB", | 20260507 00:19:48.518 | INFO | "malloc" : " 14 GB", | 20260507 00:19:48.518 | INFO | "pinned" : " 29 GB" | 20260507 00:19:48.518 | INFO | } | 20260507 00:19:48.518 | INFO | Perf report for the last 5.14276 seconds: | 20260507 00:19:48.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:19:48.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:48.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:48.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:48.518 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:19:48.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:48.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:48.518 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 00:19:48.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:48.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:48.518 | INFO | Uploaded 36GiB to GPU in 2.00988s (17.9115GiB/s) | 20260507 00:19:48.518 | INFO | Downloaded 34GiB to GPU in 1.93165s (17.6015GiB/s) | 20260507 00:19:53.657 | INFO | Analyzing chunk frames = [5608448,5608960) | 20260507 00:19:53.657 | INFO | MemInfo: { | 20260507 00:19:53.657 | INFO | "gpu" : " 17 GB", | 20260507 00:19:53.657 | INFO | "huge" : "", | 20260507 00:19:53.657 | INFO | "huge_shared" : " 27 GB", | 20260507 00:19:53.657 | INFO | "malloc" : " 14 GB", | 20260507 00:19:53.657 | INFO | "pinned" : " 29 GB" | 20260507 00:19:53.657 | INFO | } | 20260507 00:19:53.657 | INFO | Perf report for the last 5.13894 seconds: | 20260507 00:19:53.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:19:53.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:53.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:53.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:53.657 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:19:53.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:53.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:53.657 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:19:53.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:53.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:53.657 | INFO | Uploaded 36GiB to GPU in 1.97722s (18.2073GiB/s) | 20260507 00:19:53.657 | INFO | Downloaded 34GiB to GPU in 1.92065s (17.7024GiB/s) | 20260507 00:19:58.792 | INFO | Analyzing chunk frames = [5608960,5609472) | 20260507 00:19:58.792 | INFO | MemInfo: { | 20260507 00:19:58.792 | INFO | "gpu" : " 17 GB", | 20260507 00:19:58.792 | INFO | "huge" : "", | 20260507 00:19:58.793 | INFO | "huge_shared" : " 27 GB", | 20260507 00:19:58.793 | INFO | "malloc" : " 14 GB", | 20260507 00:19:58.793 | INFO | "pinned" : " 29 GB" | 20260507 00:19:58.793 | INFO | } | 20260507 00:19:58.793 | INFO | Perf report for the last 5.1358 seconds: | 20260507 00:19:58.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:19:58.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:58.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:58.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:58.793 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260507 00:19:58.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:58.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:58.793 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:19:58.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:58.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:19:58.793 | INFO | Uploaded 36GiB to GPU in 2.00364s (17.9673GiB/s) | 20260507 00:19:58.793 | INFO | Downloaded 34GiB to GPU in 1.90839s (17.8161GiB/s) | 20260507 00:20:03.927 | INFO | Analyzing chunk frames = [5609472,5609984) | 20260507 00:20:03.927 | INFO | MemInfo: { | 20260507 00:20:03.927 | INFO | "gpu" : " 17 GB", | 20260507 00:20:03.927 | INFO | "huge" : "", | 20260507 00:20:03.927 | INFO | "huge_shared" : " 27 GB", | 20260507 00:20:03.927 | INFO | "malloc" : " 14 GB", | 20260507 00:20:03.927 | INFO | "pinned" : " 29 GB" | 20260507 00:20:03.927 | INFO | } | 20260507 00:20:03.927 | INFO | Perf report for the last 5.13413 seconds: | 20260507 00:20:03.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:20:03.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:03.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:03.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:03.927 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 00:20:03.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:03.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:03.927 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:20:03.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:03.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:03.927 | INFO | Uploaded 36GiB to GPU in 2.01717s (17.8468GiB/s) | 20260507 00:20:03.927 | INFO | Downloaded 34GiB to GPU in 1.94789s (17.4548GiB/s) | 20260507 00:20:09.068 | INFO | Analyzing chunk frames = [5609984,5610496) | 20260507 00:20:09.068 | INFO | MemInfo: { | 20260507 00:20:09.068 | INFO | "gpu" : " 17 GB", | 20260507 00:20:09.068 | INFO | "huge" : "", | 20260507 00:20:09.068 | INFO | "huge_shared" : " 27 GB", | 20260507 00:20:09.068 | INFO | "malloc" : " 14 GB", | 20260507 00:20:09.068 | INFO | "pinned" : " 29 GB" | 20260507 00:20:09.068 | INFO | } | 20260507 00:20:09.068 | INFO | Perf report for the last 5.14133 seconds: | 20260507 00:20:09.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:20:09.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:09.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:09.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:09.068 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:20:09.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:09.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:09.068 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:20:09.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:09.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:09.069 | INFO | Uploaded 36GiB to GPU in 2.02151s (17.8084GiB/s) | 20260507 00:20:09.069 | INFO | Downloaded 34GiB to GPU in 1.94047s (17.5215GiB/s) | 20260507 00:20:14.217 | INFO | Analyzing chunk frames = [5610496,5611008) | 20260507 00:20:14.217 | INFO | MemInfo: { | 20260507 00:20:14.217 | INFO | "gpu" : " 17 GB", | 20260507 00:20:14.217 | INFO | "huge" : "", | 20260507 00:20:14.217 | INFO | "huge_shared" : " 27 GB", | 20260507 00:20:14.217 | INFO | "malloc" : " 14 GB", | 20260507 00:20:14.217 | INFO | "pinned" : " 29 GB" | 20260507 00:20:14.217 | INFO | } | 20260507 00:20:14.217 | INFO | Perf report for the last 5.14893 seconds: | 20260507 00:20:14.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:20:14.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:14.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:14.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:14.217 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 00:20:14.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:14.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:14.217 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 00:20:14.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:14.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:14.218 | INFO | Uploaded 36GiB to GPU in 1.95988s (18.3685GiB/s) | 20260507 00:20:14.218 | INFO | Downloaded 34GiB to GPU in 1.9901s (17.0845GiB/s) | 20260507 00:20:19.352 | INFO | Analyzing chunk frames = [5611008,5611520) | 20260507 00:20:19.352 | INFO | MemInfo: { | 20260507 00:20:19.352 | INFO | "gpu" : " 17 GB", | 20260507 00:20:19.352 | INFO | "huge" : "", | 20260507 00:20:19.352 | INFO | "huge_shared" : " 27 GB", | 20260507 00:20:19.352 | INFO | "malloc" : " 14 GB", | 20260507 00:20:19.352 | INFO | "pinned" : " 29 GB" | 20260507 00:20:19.352 | INFO | } | 20260507 00:20:19.352 | INFO | Perf report for the last 5.13503 seconds: | 20260507 00:20:19.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:20:19.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:19.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:19.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:19.352 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:20:19.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:19.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:19.352 | INFO | PRE_HQ : 30.4%, 1.0, 15.8ms, 0 | 20260507 00:20:19.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:19.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:19.353 | INFO | Uploaded 36GiB to GPU in 1.98778s (18.1106GiB/s) | 20260507 00:20:19.353 | INFO | Downloaded 34GiB to GPU in 1.99181s (17.0699GiB/s) | 20260507 00:20:19.353 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:20:19.353 | INFO | Total mem usage: 18 GiB | 20260507 00:20:19.353 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:20:19.353 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:20:19.353 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:20:19.353 | INFO | High Water Breakdown: | 20260507 00:20:19.353 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:20:19.353 | INFO | 39.7% : 5.67 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:20:19.353 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:20:19.353 | INFO | ------------------------------------------ | 20260507 00:20:19.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:20:19.353 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:20:19.353 | INFO | Total mem usage: 26.5 GiB | 20260507 00:20:19.353 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:20:19.353 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:20:19.353 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:20:19.353 | INFO | High Water Breakdown: | 20260507 00:20:19.353 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:20:19.353 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:20:19.353 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:20:19.353 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:20:19.353 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:20:19.353 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:20:19.353 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:20:19.353 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:20:19.353 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:20:19.353 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:20:19.353 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:20:19.353 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:20:19.353 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:20:19.353 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:20:19.353 | INFO | ------------------------------------------ | 20260507 00:20:19.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:20:19.353 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:20:19.353 | INFO | Total mem usage: 25.6 GiB | 20260507 00:20:19.353 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:20:19.353 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:20:19.353 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:20:19.353 | INFO | High Water Breakdown: | 20260507 00:20:19.353 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:20:19.353 | INFO | ------------------------------------------ | 20260507 00:20:19.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:20:19.353 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:20:19.353 | INFO | Total mem usage: 15.7 GiB | 20260507 00:20:19.353 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:20:19.353 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:20:19.353 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:20:19.354 | INFO | High Water Breakdown: | 20260507 00:20:19.354 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:20:19.354 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:20:19.354 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:20:19.354 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:20:19.354 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:20:19.354 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:20:19.354 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:20:19.354 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:20:19.354 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:20:19.354 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:20:19.354 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:20:19.354 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:20:19.354 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:20:19.354 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:20:19.354 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:20:19.354 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:20:19.354 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:20:19.354 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:20:19.354 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:20:19.354 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:20:19.354 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:20:19.354 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:20:19.354 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:20:19.354 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:20:19.354 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:20:19.354 | INFO | ------------------------------------------ | 20260507 00:20:19.354 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:20:19.354 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:20:19.354 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:20:19.354 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:20:19.354 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.56e+01, 1.34e+01 / 2.15e+00 | 20260507 00:20:19.354 | INFO | PerfReport: 0.30%, AggregateStats -- 9.14e-02 / 1.02e-01, 9.97e-03 / 6.20e-03 | 20260507 00:20:19.354 | INFO | PerfReport: 11.66%, Baseline -- 3.60e+00 / 4.35e+00, 5.31e-01 / 3.80e-01 | 20260507 00:20:19.354 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.45e+00, 1.75e-01 / 1.33e-01 | 20260507 00:20:19.354 | INFO | PerfReport: 16.09%, DME -- 3.98e+01 / 4.45e+01, 4.01e+00 / 2.39e+00 | 20260507 00:20:19.354 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.77e+01 / 2.07e+01, 1.30e+00 / 1.67e+00 | 20260507 00:20:19.354 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.18e+00 / 2.71e+00, 3.83e-01 / 2.31e-01 | 20260507 00:20:19.354 | INFO | PerfReport: 3.15%, Metrics -- 9.75e-01 / 1.30e+00, 2.58e-01 / 1.49e-01 | 20260507 00:20:19.354 | INFO | PerfReport: 1.08%, PulseSerializing -- 3.33e-01 / 3.85e-01, 5.24e-02 / 2.27e-02 | 20260507 00:20:19.354 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:20:19.354 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:20:19.354 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:20:19.354 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 00:20:19.354 | INFO | PerfReport: 21.64%, Upload -- 2.09e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260507 00:20:19.354 | INFO | PerfReport: 21.34%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 00:20:19.354 | INFO | PerfReport: 32.01%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 00:20:19.354 | INFO | PerfReport: 1.37%, LockUpload -- 1.33e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 00:20:19.354 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260507 00:20:19.354 | INFO | PerfReport: 22.40%, LockCompute -- 2.16e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 00:20:19.354 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.55e-02 / 0.00e+00 | 20260507 00:20:19.450 | INFO | Starting a round of Baz IO | 20260507 00:20:22.578 | INFO | Populating RTMetrics report | 20260507 00:20:22.578 | INFO | RTMetrics report populated | 20260507 00:20:22.578 | INFO | Opening tmp RTMetrics json file | 20260507 00:20:22.602 | INFO | Serializing RTMetrics to json | 20260507 00:20:22.605 | INFO | Serialization done, swapping tmp file now... | 20260507 00:20:22.605 | INFO | RTMetrics done | 20260507 00:20:22.680 | INFO | Finished a round of Baz IO: Wrote 1.848 GiB in 3.229 seconds | 20260507 00:20:22.680 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.31% / 0.00% / 66.69% | 20260507 00:20:22.680 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.349597% / 0% / 7.94729e-05% | 20260507 00:20:22.680 | INFO | Baz IO overall: | 20260507 00:20:22.680 | INFO | BytesWritten : 2926484960315 | 20260507 00:20:22.680 | INFO | NumEvents : 1409924560436 | 20260507 00:20:24.491 | INFO | Analyzing chunk frames = [5611520,5612032) | 20260507 00:20:24.491 | INFO | MemInfo: { | 20260507 00:20:24.491 | INFO | "gpu" : " 17 GB", | 20260507 00:20:24.491 | INFO | "huge" : "", | 20260507 00:20:24.491 | INFO | "huge_shared" : " 27 GB", | 20260507 00:20:24.491 | INFO | "malloc" : " 14 GB", | 20260507 00:20:24.491 | INFO | "pinned" : " 29 GB" | 20260507 00:20:24.491 | INFO | } | 20260507 00:20:24.491 | INFO | Perf report for the last 5.13842 seconds: | 20260507 00:20:24.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:20:24.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:24.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:24.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:24.491 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 00:20:24.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:24.491 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 00:20:24.491 | INFO | PRE_HQ : 59.7%, 1.8, 56.2ms, 0 | 20260507 00:20:24.491 | INFO | BAZWRITER : 62.4%, 3.5, 112.2ms, 0 | 20260507 00:20:24.491 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:20:24.491 | INFO | Uploaded 36GiB to GPU in 2.02938s (17.7394GiB/s) | 20260507 00:20:24.491 | INFO | Downloaded 38GiB to GPU in 2.18305s (17.4069GiB/s) | 20260507 00:20:29.626 | INFO | Analyzing chunk frames = [5612032,5612544) | 20260507 00:20:29.626 | INFO | MemInfo: { | 20260507 00:20:29.626 | INFO | "gpu" : " 17 GB", | 20260507 00:20:29.626 | INFO | "huge" : "", | 20260507 00:20:29.626 | INFO | "huge_shared" : " 27 GB", | 20260507 00:20:29.626 | INFO | "malloc" : " 14 GB", | 20260507 00:20:29.626 | INFO | "pinned" : " 29 GB" | 20260507 00:20:29.626 | INFO | } | 20260507 00:20:29.626 | INFO | Perf report for the last 5.13515 seconds: | 20260507 00:20:29.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:20:29.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:29.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:29.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:29.626 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 00:20:29.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:29.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:29.626 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:20:29.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:29.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:29.626 | INFO | Uploaded 36GiB to GPU in 1.98775s (18.1109GiB/s) | 20260507 00:20:29.626 | INFO | Downloaded 34GiB to GPU in 1.9284s (17.6312GiB/s) | 20260507 00:20:34.767 | INFO | Analyzing chunk frames = [5612544,5613056) | 20260507 00:20:34.767 | INFO | MemInfo: { | 20260507 00:20:34.767 | INFO | "gpu" : " 17 GB", | 20260507 00:20:34.767 | INFO | "huge" : "", | 20260507 00:20:34.767 | INFO | "huge_shared" : " 27 GB", | 20260507 00:20:34.767 | INFO | "malloc" : " 14 GB", | 20260507 00:20:34.767 | INFO | "pinned" : " 29 GB" | 20260507 00:20:34.767 | INFO | } | 20260507 00:20:34.767 | INFO | Perf report for the last 5.14146 seconds: | 20260507 00:20:34.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:20:34.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:34.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:34.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:34.767 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:20:34.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:34.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:34.767 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 00:20:34.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:34.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:34.768 | INFO | Uploaded 36GiB to GPU in 1.96607s (18.3106GiB/s) | 20260507 00:20:34.768 | INFO | Downloaded 34GiB to GPU in 1.92404s (17.6712GiB/s) | 20260507 00:20:39.905 | INFO | Analyzing chunk frames = [5613056,5613568) | 20260507 00:20:39.905 | INFO | MemInfo: { | 20260507 00:20:39.905 | INFO | "gpu" : " 17 GB", | 20260507 00:20:39.905 | INFO | "huge" : "", | 20260507 00:20:39.905 | INFO | "huge_shared" : " 27 GB", | 20260507 00:20:39.905 | INFO | "malloc" : " 14 GB", | 20260507 00:20:39.905 | INFO | "pinned" : " 29 GB" | 20260507 00:20:39.905 | INFO | } | 20260507 00:20:39.905 | INFO | Perf report for the last 5.13808 seconds: | 20260507 00:20:39.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:20:39.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:39.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:39.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:39.906 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:20:39.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:39.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:39.906 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:20:39.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:39.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:39.906 | INFO | Uploaded 36GiB to GPU in 1.98432s (18.1422GiB/s) | 20260507 00:20:39.906 | INFO | Downloaded 34GiB to GPU in 1.91021s (17.7991GiB/s) | 20260507 00:20:45.043 | INFO | Analyzing chunk frames = [5613568,5614080) | 20260507 00:20:45.043 | INFO | MemInfo: { | 20260507 00:20:45.043 | INFO | "gpu" : " 17 GB", | 20260507 00:20:45.043 | INFO | "huge" : "", | 20260507 00:20:45.043 | INFO | "huge_shared" : " 27 GB", | 20260507 00:20:45.043 | INFO | "malloc" : " 14 GB", | 20260507 00:20:45.043 | INFO | "pinned" : " 29 GB" | 20260507 00:20:45.043 | INFO | } | 20260507 00:20:45.043 | INFO | Perf report for the last 5.13801 seconds: | 20260507 00:20:45.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:20:45.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:45.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:45.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:45.043 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 00:20:45.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:45.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:45.043 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:20:45.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:45.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:45.044 | INFO | Uploaded 36GiB to GPU in 1.98745s (18.1137GiB/s) | 20260507 00:20:45.044 | INFO | Downloaded 34GiB to GPU in 1.93387s (17.5813GiB/s) | 20260507 00:20:50.186 | INFO | Analyzing chunk frames = [5614080,5614592) | 20260507 00:20:50.186 | INFO | MemInfo: { | 20260507 00:20:50.186 | INFO | "gpu" : " 17 GB", | 20260507 00:20:50.186 | INFO | "huge" : "", | 20260507 00:20:50.186 | INFO | "huge_shared" : " 27 GB", | 20260507 00:20:50.186 | INFO | "malloc" : " 14 GB", | 20260507 00:20:50.186 | INFO | "pinned" : " 29 GB" | 20260507 00:20:50.187 | INFO | } | 20260507 00:20:50.187 | INFO | Perf report for the last 5.14324 seconds: | 20260507 00:20:50.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:20:50.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:50.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:50.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:50.187 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 00:20:50.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:50.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:50.187 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:20:50.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:50.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:50.187 | INFO | Uploaded 36GiB to GPU in 2.01302s (17.8835GiB/s) | 20260507 00:20:50.187 | INFO | Downloaded 34GiB to GPU in 1.94817s (17.4523GiB/s) | 20260507 00:20:55.325 | INFO | Analyzing chunk frames = [5614592,5615104) | 20260507 00:20:55.325 | INFO | MemInfo: { | 20260507 00:20:55.325 | INFO | "gpu" : " 17 GB", | 20260507 00:20:55.325 | INFO | "huge" : "", | 20260507 00:20:55.325 | INFO | "huge_shared" : " 27 GB", | 20260507 00:20:55.325 | INFO | "malloc" : " 14 GB", | 20260507 00:20:55.325 | INFO | "pinned" : " 29 GB" | 20260507 00:20:55.325 | INFO | } | 20260507 00:20:55.325 | INFO | Perf report for the last 5.1388 seconds: | 20260507 00:20:55.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:20:55.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:55.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:55.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:55.326 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 00:20:55.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:55.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:55.326 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:20:55.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:55.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:20:55.326 | INFO | Uploaded 36GiB to GPU in 1.95417s (18.4221GiB/s) | 20260507 00:20:55.326 | INFO | Downloaded 34GiB to GPU in 1.98342s (17.1421GiB/s) | 20260507 00:21:00.475 | INFO | Analyzing chunk frames = [5615104,5615616) | 20260507 00:21:00.475 | INFO | MemInfo: { | 20260507 00:21:00.475 | INFO | "gpu" : " 17 GB", | 20260507 00:21:00.475 | INFO | "huge" : "", | 20260507 00:21:00.475 | INFO | "huge_shared" : " 27 GB", | 20260507 00:21:00.475 | INFO | "malloc" : " 14 GB", | 20260507 00:21:00.475 | INFO | "pinned" : " 29 GB" | 20260507 00:21:00.475 | INFO | } | 20260507 00:21:00.475 | INFO | Perf report for the last 5.14955 seconds: | 20260507 00:21:00.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:21:00.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:00.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:00.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:00.475 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260507 00:21:00.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:00.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:00.475 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260507 00:21:00.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:00.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:00.475 | INFO | Uploaded 36GiB to GPU in 1.98825s (18.1063GiB/s) | 20260507 00:21:00.475 | INFO | Downloaded 34GiB to GPU in 2.00032s (16.9973GiB/s) | 20260507 00:21:00.572 | INFO | Starting a round of Baz IO | 20260507 00:21:03.687 | INFO | Populating RTMetrics report | 20260507 00:21:03.688 | INFO | RTMetrics report populated | 20260507 00:21:03.688 | INFO | Opening tmp RTMetrics json file | 20260507 00:21:03.688 | INFO | Serializing RTMetrics to json | 20260507 00:21:03.689 | INFO | Serialization done, swapping tmp file now... | 20260507 00:21:03.690 | INFO | RTMetrics done | 20260507 00:21:03.786 | INFO | Finished a round of Baz IO: Wrote 1.847 GiB in 3.214 seconds | 20260507 00:21:03.786 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.30% / 0.00% / 66.69% | 20260507 00:21:03.786 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.349689% / 0% / 8.74201e-05% | 20260507 00:21:03.786 | INFO | Baz IO overall: | 20260507 00:21:03.787 | INFO | BytesWritten : 2928468668610 | 20260507 00:21:03.787 | INFO | NumEvents : 1410865928224 | 20260507 00:21:05.608 | INFO | Analyzing chunk frames = [5615616,5616128) | 20260507 00:21:05.608 | INFO | MemInfo: { | 20260507 00:21:05.608 | INFO | "gpu" : " 17 GB", | 20260507 00:21:05.608 | INFO | "huge" : "", | 20260507 00:21:05.609 | INFO | "huge_shared" : " 27 GB", | 20260507 00:21:05.609 | INFO | "malloc" : " 14 GB", | 20260507 00:21:05.609 | INFO | "pinned" : " 29 GB" | 20260507 00:21:05.609 | INFO | } | 20260507 00:21:05.609 | INFO | Perf report for the last 5.1336 seconds: | 20260507 00:21:05.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:21:05.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:05.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:05.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:05.609 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260507 00:21:05.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:05.609 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260507 00:21:05.609 | INFO | PRE_HQ : 59.8%, 1.9, 59.3ms, 0 | 20260507 00:21:05.609 | INFO | BAZWRITER : 62.2%, 3.5, 113.1ms, 0 | 20260507 00:21:05.609 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 00:21:05.609 | INFO | Uploaded 36GiB to GPU in 2.02535s (17.7747GiB/s) | 20260507 00:21:05.609 | INFO | Downloaded 38GiB to GPU in 2.17742s (17.4519GiB/s) | 20260507 00:21:10.741 | INFO | Analyzing chunk frames = [5616128,5616640) | 20260507 00:21:10.741 | INFO | MemInfo: { | 20260507 00:21:10.741 | INFO | "gpu" : " 17 GB", | 20260507 00:21:10.741 | INFO | "huge" : "", | 20260507 00:21:10.741 | INFO | "huge_shared" : " 27 GB", | 20260507 00:21:10.741 | INFO | "malloc" : " 14 GB", | 20260507 00:21:10.741 | INFO | "pinned" : " 29 GB" | 20260507 00:21:10.741 | INFO | } | 20260507 00:21:10.741 | INFO | Perf report for the last 5.13276 seconds: | 20260507 00:21:10.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:21:10.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:10.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:10.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:10.741 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:21:10.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:10.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:10.741 | INFO | PRE_HQ : 27.1%, 1.0, 14.0ms, 0 | 20260507 00:21:10.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:10.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:10.742 | INFO | Uploaded 36GiB to GPU in 2.00764s (17.9315GiB/s) | 20260507 00:21:10.742 | INFO | Downloaded 34GiB to GPU in 1.93156s (17.6023GiB/s) | 20260507 00:21:15.879 | INFO | Analyzing chunk frames = [5616640,5617152) | 20260507 00:21:15.879 | INFO | MemInfo: { | 20260507 00:21:15.879 | INFO | "gpu" : " 17 GB", | 20260507 00:21:15.879 | INFO | "huge" : "", | 20260507 00:21:15.879 | INFO | "huge_shared" : " 27 GB", | 20260507 00:21:15.879 | INFO | "malloc" : " 14 GB", | 20260507 00:21:15.879 | INFO | "pinned" : " 29 GB" | 20260507 00:21:15.879 | INFO | } | 20260507 00:21:15.879 | INFO | Perf report for the last 5.13769 seconds: | 20260507 00:21:15.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:21:15.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:15.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:15.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:15.879 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:21:15.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:15.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:15.879 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:21:15.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:15.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:15.879 | INFO | Uploaded 36GiB to GPU in 1.98967s (18.0935GiB/s) | 20260507 00:21:15.879 | INFO | Downloaded 34GiB to GPU in 1.92754s (17.6391GiB/s) | 20260507 00:21:21.019 | INFO | Analyzing chunk frames = [5617152,5617664) | 20260507 00:21:21.019 | INFO | MemInfo: { | 20260507 00:21:21.019 | INFO | "gpu" : " 17 GB", | 20260507 00:21:21.019 | INFO | "huge" : "", | 20260507 00:21:21.019 | INFO | "huge_shared" : " 27 GB", | 20260507 00:21:21.019 | INFO | "malloc" : " 14 GB", | 20260507 00:21:21.019 | INFO | "pinned" : " 29 GB" | 20260507 00:21:21.020 | INFO | } | 20260507 00:21:21.020 | INFO | Perf report for the last 5.14053 seconds: | 20260507 00:21:21.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:21:21.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:21.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:21.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:21.020 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 00:21:21.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:21.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:21.020 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:21:21.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:21.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:21.020 | INFO | Uploaded 36GiB to GPU in 2.01466s (17.869GiB/s) | 20260507 00:21:21.020 | INFO | Downloaded 34GiB to GPU in 1.90992s (17.8018GiB/s) | 20260507 00:21:26.163 | INFO | Analyzing chunk frames = [5617664,5618176) | 20260507 00:21:26.163 | INFO | MemInfo: { | 20260507 00:21:26.163 | INFO | "gpu" : " 17 GB", | 20260507 00:21:26.163 | INFO | "huge" : "", | 20260507 00:21:26.163 | INFO | "huge_shared" : " 27 GB", | 20260507 00:21:26.163 | INFO | "malloc" : " 14 GB", | 20260507 00:21:26.163 | INFO | "pinned" : " 29 GB" | 20260507 00:21:26.163 | INFO | } | 20260507 00:21:26.163 | INFO | Perf report for the last 5.14347 seconds: | 20260507 00:21:26.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:21:26.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:26.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:26.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:26.163 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 00:21:26.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:26.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:26.163 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 00:21:26.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:26.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:26.163 | INFO | Uploaded 36GiB to GPU in 1.98775s (18.1109GiB/s) | 20260507 00:21:26.163 | INFO | Downloaded 34GiB to GPU in 1.9244s (17.6678GiB/s) | 20260507 00:21:31.305 | INFO | Analyzing chunk frames = [5618176,5618688) | 20260507 00:21:31.305 | INFO | MemInfo: { | 20260507 00:21:31.305 | INFO | "gpu" : " 17 GB", | 20260507 00:21:31.305 | INFO | "huge" : "", | 20260507 00:21:31.305 | INFO | "huge_shared" : " 27 GB", | 20260507 00:21:31.305 | INFO | "malloc" : " 14 GB", | 20260507 00:21:31.305 | INFO | "pinned" : " 29 GB" | 20260507 00:21:31.305 | INFO | } | 20260507 00:21:31.305 | INFO | Perf report for the last 5.14182 seconds: | 20260507 00:21:31.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:21:31.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:31.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:31.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:31.305 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:21:31.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:31.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:31.305 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 00:21:31.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:31.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:31.305 | INFO | Uploaded 36GiB to GPU in 2.03854s (17.6597GiB/s) | 20260507 00:21:31.305 | INFO | Downloaded 34GiB to GPU in 1.94197s (17.508GiB/s) | 20260507 00:21:36.438 | INFO | Analyzing chunk frames = [5618688,5619200) | 20260507 00:21:36.438 | INFO | MemInfo: { | 20260507 00:21:36.438 | INFO | "gpu" : " 17 GB", | 20260507 00:21:36.438 | INFO | "huge" : "", | 20260507 00:21:36.438 | INFO | "huge_shared" : " 27 GB", | 20260507 00:21:36.438 | INFO | "malloc" : " 14 GB", | 20260507 00:21:36.438 | INFO | "pinned" : " 29 GB" | 20260507 00:21:36.438 | INFO | } | 20260507 00:21:36.439 | INFO | Perf report for the last 5.13366 seconds: | 20260507 00:21:36.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:21:36.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:36.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:36.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:36.439 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 00:21:36.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:36.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:36.439 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 00:21:36.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:36.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:36.439 | INFO | Uploaded 36GiB to GPU in 1.95402s (18.4235GiB/s) | 20260507 00:21:36.439 | INFO | Downloaded 34GiB to GPU in 1.99394s (17.0517GiB/s) | 20260507 00:21:41.578 | INFO | Analyzing chunk frames = [5619200,5619712) | 20260507 00:21:41.578 | INFO | MemInfo: { | 20260507 00:21:41.578 | INFO | "gpu" : " 17 GB", | 20260507 00:21:41.578 | INFO | "huge" : "", | 20260507 00:21:41.578 | INFO | "huge_shared" : " 27 GB", | 20260507 00:21:41.578 | INFO | "malloc" : " 14 GB", | 20260507 00:21:41.578 | INFO | "pinned" : " 29 GB" | 20260507 00:21:41.578 | INFO | } | 20260507 00:21:41.578 | INFO | Perf report for the last 5.13966 seconds: | 20260507 00:21:41.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:21:41.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:41.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:41.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:41.579 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 00:21:41.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:41.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:41.579 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:21:41.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:41.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:41.579 | INFO | Uploaded 36GiB to GPU in 2.00861s (17.9229GiB/s) | 20260507 00:21:41.579 | INFO | Downloaded 34GiB to GPU in 2s (17GiB/s) | 20260507 00:21:41.579 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:21:41.579 | INFO | Total mem usage: 18 GiB | 20260507 00:21:41.579 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:21:41.579 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26%) | 20260507 00:21:41.579 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:21:41.579 | INFO | High Water Breakdown: | 20260507 00:21:41.579 | INFO | 60.1% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:21:41.579 | INFO | 39.7% : 5.68 GiB ( 84.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:21:41.579 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:21:41.579 | INFO | ------------------------------------------ | 20260507 00:21:41.579 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:21:41.579 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:21:41.579 | INFO | Total mem usage: 26.5 GiB | 20260507 00:21:41.579 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:21:41.579 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:21:41.579 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:21:41.579 | INFO | High Water Breakdown: | 20260507 00:21:41.579 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:21:41.579 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:21:41.579 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:21:41.579 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:21:41.579 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:21:41.579 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:21:41.579 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:21:41.579 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:21:41.579 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:21:41.579 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:21:41.579 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:21:41.579 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:21:41.579 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:21:41.579 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:21:41.579 | INFO | ------------------------------------------ | 20260507 00:21:41.579 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:21:41.579 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:21:41.580 | INFO | Total mem usage: 25.6 GiB | 20260507 00:21:41.580 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:21:41.580 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:21:41.580 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:21:41.580 | INFO | High Water Breakdown: | 20260507 00:21:41.580 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:21:41.580 | INFO | ------------------------------------------ | 20260507 00:21:41.580 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:21:41.580 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:21:41.580 | INFO | Total mem usage: 15.7 GiB | 20260507 00:21:41.580 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:21:41.580 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:21:41.580 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 00:21:41.580 | INFO | High Water Breakdown: | 20260507 00:21:41.580 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:21:41.580 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:21:41.580 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:21:41.580 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:21:41.580 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:21:41.580 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:21:41.580 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:21:41.580 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:21:41.580 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:21:41.580 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:21:41.580 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:21:41.580 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:21:41.580 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:21:41.580 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:21:41.580 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:21:41.580 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:21:41.580 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:21:41.580 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:21:41.580 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:21:41.580 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:21:41.580 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:21:41.580 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:21:41.580 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:21:41.580 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:21:41.580 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:21:41.580 | INFO | ------------------------------------------ | 20260507 00:21:41.580 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:21:41.580 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:21:41.580 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:21:41.580 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:21:41.580 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.55e+01, 1.35e+01 / 2.16e+00 | 20260507 00:21:41.580 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.00e-01, 9.32e-03 / 6.05e-03 | 20260507 00:21:41.580 | INFO | PerfReport: 11.68%, Baseline -- 3.61e+00 / 4.31e+00, 5.40e-01 / 3.98e-01 | 20260507 00:21:41.580 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.81e-01 / 1.44e-01 | 20260507 00:21:41.580 | INFO | PerfReport: 16.06%, DME -- 3.97e+01 / 4.42e+01, 3.96e+00 / 2.41e+00 | 20260507 00:21:41.580 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.78e+01 / 2.12e+01, 1.34e+00 / 1.54e+00 | 20260507 00:21:41.580 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.17e+00 / 2.68e+00, 3.78e-01 / 1.97e-01 | 20260507 00:21:41.580 | INFO | PerfReport: 3.15%, Metrics -- 9.74e-01 / 1.28e+00, 2.58e-01 / 1.57e-01 | 20260507 00:21:41.580 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.32e-01 / 3.78e-01, 5.21e-02 / 1.87e-02 | 20260507 00:21:41.580 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:21:41.580 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:21:41.580 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:21:41.580 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 00:21:41.580 | INFO | PerfReport: 21.54%, Upload -- 2.08e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260507 00:21:41.580 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260507 00:21:41.580 | INFO | PerfReport: 32.05%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:21:41.580 | INFO | PerfReport: 1.29%, LockUpload -- 1.25e+00 / 0.00e+00, 4.71e+00 / 0.00e+00 | 20260507 00:21:41.580 | INFO | PerfReport: 1.29%, LockDownload -- 1.25e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260507 00:21:41.580 | INFO | PerfReport: 22.47%, LockCompute -- 2.17e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 00:21:41.580 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260507 00:21:41.677 | INFO | Starting a round of Baz IO | 20260507 00:21:44.799 | INFO | Populating RTMetrics report | 20260507 00:21:44.800 | INFO | RTMetrics report populated | 20260507 00:21:44.800 | INFO | Opening tmp RTMetrics json file | 20260507 00:21:44.801 | INFO | Serializing RTMetrics to json | 20260507 00:21:44.815 | INFO | Serialization done, swapping tmp file now... | 20260507 00:21:44.828 | INFO | RTMetrics done | 20260507 00:21:44.899 | INFO | Finished a round of Baz IO: Wrote 1.849 GiB in 3.222 seconds | 20260507 00:21:44.900 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.30% / 0.00% / 66.70% | 20260507 00:21:44.900 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.349851% / 3.97364e-06% / 6.75519e-05% | 20260507 00:21:44.900 | INFO | Baz IO overall: | 20260507 00:21:44.900 | INFO | BytesWritten : 2930453803900 | 20260507 00:21:44.900 | INFO | NumEvents : 1411808226371 | 20260507 00:21:46.718 | INFO | Analyzing chunk frames = [5619712,5620224) | 20260507 00:21:46.718 | INFO | MemInfo: { | 20260507 00:21:46.718 | INFO | "gpu" : " 17 GB", | 20260507 00:21:46.718 | INFO | "huge" : "", | 20260507 00:21:46.718 | INFO | "huge_shared" : " 27 GB", | 20260507 00:21:46.718 | INFO | "malloc" : " 14 GB", | 20260507 00:21:46.718 | INFO | "pinned" : " 29 GB" | 20260507 00:21:46.718 | INFO | } | 20260507 00:21:46.718 | INFO | Perf report for the last 5.13989 seconds: | 20260507 00:21:46.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:21:46.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:46.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:46.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:46.718 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 00:21:46.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:46.718 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260507 00:21:46.718 | INFO | PRE_HQ : 59.6%, 1.8, 57.2ms, 0 | 20260507 00:21:46.718 | INFO | BAZWRITER : 62.2%, 3.5, 110.3ms, 0 | 20260507 00:21:46.719 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 00:21:46.719 | INFO | Uploaded 36GiB to GPU in 2.01928s (17.8281GiB/s) | 20260507 00:21:46.719 | INFO | Downloaded 38GiB to GPU in 2.18303s (17.407GiB/s) | 20260507 00:21:51.858 | INFO | Analyzing chunk frames = [5620224,5620736) | 20260507 00:21:51.858 | INFO | MemInfo: { | 20260507 00:21:51.858 | INFO | "gpu" : " 17 GB", | 20260507 00:21:51.858 | INFO | "huge" : "", | 20260507 00:21:51.858 | INFO | "huge_shared" : " 27 GB", | 20260507 00:21:51.858 | INFO | "malloc" : " 14 GB", | 20260507 00:21:51.858 | INFO | "pinned" : " 29 GB" | 20260507 00:21:51.858 | INFO | } | 20260507 00:21:51.859 | INFO | Perf report for the last 5.14035 seconds: | 20260507 00:21:51.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:21:51.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:51.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:51.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:51.859 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:21:51.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:51.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:51.859 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:21:51.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:51.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:51.859 | INFO | Uploaded 36GiB to GPU in 2.00255s (17.9771GiB/s) | 20260507 00:21:51.859 | INFO | Downloaded 34GiB to GPU in 1.92624s (17.6509GiB/s) | 20260507 00:21:56.993 | INFO | Analyzing chunk frames = [5620736,5621248) | 20260507 00:21:56.993 | INFO | MemInfo: { | 20260507 00:21:56.993 | INFO | "gpu" : " 17 GB", | 20260507 00:21:56.993 | INFO | "huge" : "", | 20260507 00:21:56.993 | INFO | "huge_shared" : " 27 GB", | 20260507 00:21:56.993 | INFO | "malloc" : " 14 GB", | 20260507 00:21:56.993 | INFO | "pinned" : " 29 GB" | 20260507 00:21:56.993 | INFO | } | 20260507 00:21:56.993 | INFO | Perf report for the last 5.13459 seconds: | 20260507 00:21:56.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:21:56.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:56.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:56.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:56.993 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:21:56.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:56.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:56.993 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:21:56.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:56.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:21:56.993 | INFO | Uploaded 36GiB to GPU in 1.98814s (18.1074GiB/s) | 20260507 00:21:56.994 | INFO | Downloaded 34GiB to GPU in 1.92708s (17.6433GiB/s) | 20260507 00:22:02.137 | INFO | Analyzing chunk frames = [5621248,5621760) | 20260507 00:22:02.137 | INFO | MemInfo: { | 20260507 00:22:02.137 | INFO | "gpu" : " 17 GB", | 20260507 00:22:02.137 | INFO | "huge" : "", | 20260507 00:22:02.137 | INFO | "huge_shared" : " 27 GB", | 20260507 00:22:02.137 | INFO | "malloc" : " 14 GB", | 20260507 00:22:02.137 | INFO | "pinned" : " 29 GB" | 20260507 00:22:02.137 | INFO | } | 20260507 00:22:02.137 | INFO | Perf report for the last 5.14375 seconds: | 20260507 00:22:02.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:22:02.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:02.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:02.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:02.137 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 00:22:02.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:02.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:02.137 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:22:02.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:02.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:02.137 | INFO | Uploaded 36GiB to GPU in 2.01798s (17.8396GiB/s) | 20260507 00:22:02.137 | INFO | Downloaded 34GiB to GPU in 1.90928s (17.8077GiB/s) | 20260507 00:22:07.281 | INFO | Analyzing chunk frames = [5621760,5622272) | 20260507 00:22:07.281 | INFO | MemInfo: { | 20260507 00:22:07.281 | INFO | "gpu" : " 17 GB", | 20260507 00:22:07.281 | INFO | "huge" : "", | 20260507 00:22:07.281 | INFO | "huge_shared" : " 27 GB", | 20260507 00:22:07.281 | INFO | "malloc" : " 14 GB", | 20260507 00:22:07.281 | INFO | "pinned" : " 29 GB" | 20260507 00:22:07.281 | INFO | } | 20260507 00:22:07.281 | INFO | Perf report for the last 5.14431 seconds: | 20260507 00:22:07.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:22:07.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:07.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:07.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:07.281 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 00:22:07.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:07.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:07.281 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:22:07.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:07.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:07.282 | INFO | Uploaded 36GiB to GPU in 1.99369s (18.057GiB/s) | 20260507 00:22:07.282 | INFO | Downloaded 34GiB to GPU in 1.92493s (17.663GiB/s) | 20260507 00:22:12.421 | INFO | Analyzing chunk frames = [5622272,5622784) | 20260507 00:22:12.421 | INFO | MemInfo: { | 20260507 00:22:12.421 | INFO | "gpu" : " 17 GB", | 20260507 00:22:12.421 | INFO | "huge" : "", | 20260507 00:22:12.421 | INFO | "huge_shared" : " 27 GB", | 20260507 00:22:12.421 | INFO | "malloc" : " 14 GB", | 20260507 00:22:12.421 | INFO | "pinned" : " 29 GB" | 20260507 00:22:12.421 | INFO | } | 20260507 00:22:12.421 | INFO | Perf report for the last 5.1398 seconds: | 20260507 00:22:12.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:22:12.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:12.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:12.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:12.421 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:22:12.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:12.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:12.421 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:22:12.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:12.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:12.421 | INFO | Uploaded 36GiB to GPU in 2.04456s (17.6077GiB/s) | 20260507 00:22:12.421 | INFO | Downloaded 34GiB to GPU in 1.94257s (17.5026GiB/s) | 20260507 00:22:17.557 | INFO | Analyzing chunk frames = [5622784,5623296) | 20260507 00:22:17.557 | INFO | MemInfo: { | 20260507 00:22:17.557 | INFO | "gpu" : " 17 GB", | 20260507 00:22:17.557 | INFO | "huge" : "", | 20260507 00:22:17.557 | INFO | "huge_shared" : " 27 GB", | 20260507 00:22:17.557 | INFO | "malloc" : " 14 GB", | 20260507 00:22:17.557 | INFO | "pinned" : " 29 GB" | 20260507 00:22:17.557 | INFO | } | 20260507 00:22:17.557 | INFO | Perf report for the last 5.13629 seconds: | 20260507 00:22:17.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:22:17.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:17.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:17.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:17.557 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 00:22:17.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:17.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:17.557 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 00:22:17.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:17.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:17.558 | INFO | Uploaded 36GiB to GPU in 1.98484s (18.1375GiB/s) | 20260507 00:22:17.558 | INFO | Downloaded 34GiB to GPU in 1.98295s (17.1461GiB/s) | 20260507 00:22:22.691 | INFO | Analyzing chunk frames = [5623296,5623808) | 20260507 00:22:22.691 | INFO | MemInfo: { | 20260507 00:22:22.691 | INFO | "gpu" : " 17 GB", | 20260507 00:22:22.691 | INFO | "huge" : "", | 20260507 00:22:22.691 | INFO | "huge_shared" : " 27 GB", | 20260507 00:22:22.691 | INFO | "malloc" : " 14 GB", | 20260507 00:22:22.691 | INFO | "pinned" : " 29 GB" | 20260507 00:22:22.691 | INFO | } | 20260507 00:22:22.691 | INFO | Perf report for the last 5.13354 seconds: | 20260507 00:22:22.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:22:22.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:22.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:22.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:22.691 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:22:22.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:22.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:22.691 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 00:22:22.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:22.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:22.691 | INFO | Uploaded 36GiB to GPU in 1.98379s (18.1471GiB/s) | 20260507 00:22:22.691 | INFO | Downloaded 34GiB to GPU in 2.0012s (16.9898GiB/s) | 20260507 00:22:22.789 | INFO | Starting a round of Baz IO | 20260507 00:22:25.914 | INFO | Populating RTMetrics report | 20260507 00:22:25.914 | INFO | RTMetrics report populated | 20260507 00:22:25.914 | INFO | Opening tmp RTMetrics json file | 20260507 00:22:25.920 | INFO | Serializing RTMetrics to json | 20260507 00:22:25.938 | INFO | Serialization done, swapping tmp file now... | 20260507 00:22:25.965 | INFO | RTMetrics done | 20260507 00:22:26.029 | INFO | Finished a round of Baz IO: Wrote 1.849 GiB in 3.239 seconds | 20260507 00:22:26.029 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.30% / 0.00% / 66.70% | 20260507 00:22:26.029 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.348393% / 3.97364e-06% / 5.96046e-05% | 20260507 00:22:26.029 | INFO | Baz IO overall: | 20260507 00:22:26.029 | INFO | BytesWritten : 2932438928291 | 20260507 00:22:26.029 | INFO | NumEvents : 1412750630729 | 20260507 00:22:27.830 | INFO | Analyzing chunk frames = [5623808,5624320) | 20260507 00:22:27.830 | INFO | MemInfo: { | 20260507 00:22:27.830 | INFO | "gpu" : " 17 GB", | 20260507 00:22:27.830 | INFO | "huge" : "", | 20260507 00:22:27.830 | INFO | "huge_shared" : " 27 GB", | 20260507 00:22:27.830 | INFO | "malloc" : " 14 GB", | 20260507 00:22:27.830 | INFO | "pinned" : " 29 GB" | 20260507 00:22:27.830 | INFO | } | 20260507 00:22:27.830 | INFO | Perf report for the last 5.13887 seconds: | 20260507 00:22:27.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:22:27.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:27.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:27.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:27.830 | INFO | ANALYSIS : 62.2%, 3.0, 96.4ms, 0 | 20260507 00:22:27.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:27.830 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 00:22:27.830 | INFO | PRE_HQ : 59.4%, 1.8, 56.0ms, 0 | 20260507 00:22:27.830 | INFO | BAZWRITER : 62.7%, 4.4, 140.8ms, 0 | 20260507 00:22:27.830 | INFO | BAZ_SAMPLING : 1.4%, 1.0, 0.8ms, 0 | 20260507 00:22:27.830 | INFO | Uploaded 36GiB to GPU in 2.01396s (17.8753GiB/s) | 20260507 00:22:27.830 | INFO | Downloaded 38GiB to GPU in 2.18801s (17.3674GiB/s) | 20260507 00:22:32.975 | INFO | Analyzing chunk frames = [5624320,5624832) | 20260507 00:22:32.975 | INFO | MemInfo: { | 20260507 00:22:32.975 | INFO | "gpu" : " 17 GB", | 20260507 00:22:32.975 | INFO | "huge" : "", | 20260507 00:22:32.975 | INFO | "huge_shared" : " 27 GB", | 20260507 00:22:32.975 | INFO | "malloc" : " 14 GB", | 20260507 00:22:32.975 | INFO | "pinned" : " 29 GB" | 20260507 00:22:32.975 | INFO | } | 20260507 00:22:32.975 | INFO | Perf report for the last 5.14517 seconds: | 20260507 00:22:32.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:22:32.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:32.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:32.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:32.975 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260507 00:22:32.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:32.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:32.975 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:22:32.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:32.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:32.975 | INFO | Uploaded 36GiB to GPU in 2.00485s (17.9565GiB/s) | 20260507 00:22:32.975 | INFO | Downloaded 34GiB to GPU in 1.93803s (17.5436GiB/s) | 20260507 00:22:38.114 | INFO | Analyzing chunk frames = [5624832,5625344) | 20260507 00:22:38.114 | INFO | MemInfo: { | 20260507 00:22:38.114 | INFO | "gpu" : " 17 GB", | 20260507 00:22:38.114 | INFO | "huge" : "", | 20260507 00:22:38.114 | INFO | "huge_shared" : " 27 GB", | 20260507 00:22:38.114 | INFO | "malloc" : " 14 GB", | 20260507 00:22:38.114 | INFO | "pinned" : " 29 GB" | 20260507 00:22:38.114 | INFO | } | 20260507 00:22:38.114 | INFO | Perf report for the last 5.13876 seconds: | 20260507 00:22:38.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:22:38.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:38.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:38.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:38.114 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:22:38.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:38.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:38.114 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:22:38.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:38.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:38.114 | INFO | Uploaded 36GiB to GPU in 1.98062s (18.1761GiB/s) | 20260507 00:22:38.114 | INFO | Downloaded 34GiB to GPU in 1.93492s (17.5718GiB/s) | 20260507 00:22:43.254 | INFO | Analyzing chunk frames = [5625344,5625856) | 20260507 00:22:43.254 | INFO | MemInfo: { | 20260507 00:22:43.254 | INFO | "gpu" : " 17 GB", | 20260507 00:22:43.254 | INFO | "huge" : "", | 20260507 00:22:43.254 | INFO | "huge_shared" : " 27 GB", | 20260507 00:22:43.254 | INFO | "malloc" : " 14 GB", | 20260507 00:22:43.254 | INFO | "pinned" : " 29 GB" | 20260507 00:22:43.254 | INFO | } | 20260507 00:22:43.254 | INFO | Perf report for the last 5.14023 seconds: | 20260507 00:22:43.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:22:43.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:43.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:43.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:43.254 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:22:43.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:43.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:43.254 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:22:43.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:43.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:43.254 | INFO | Uploaded 36GiB to GPU in 1.96871s (18.2861GiB/s) | 20260507 00:22:43.254 | INFO | Downloaded 34GiB to GPU in 1.90825s (17.8174GiB/s) | 20260507 00:22:48.392 | INFO | Analyzing chunk frames = [5625856,5626368) | 20260507 00:22:48.392 | INFO | MemInfo: { | 20260507 00:22:48.392 | INFO | "gpu" : " 17 GB", | 20260507 00:22:48.392 | INFO | "huge" : "", | 20260507 00:22:48.392 | INFO | "huge_shared" : " 27 GB", | 20260507 00:22:48.392 | INFO | "malloc" : " 14 GB", | 20260507 00:22:48.392 | INFO | "pinned" : " 29 GB" | 20260507 00:22:48.392 | INFO | } | 20260507 00:22:48.392 | INFO | Perf report for the last 5.13836 seconds: | 20260507 00:22:48.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:22:48.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:48.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:48.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:48.393 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 00:22:48.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:48.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:48.393 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:22:48.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:48.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:48.393 | INFO | Uploaded 36GiB to GPU in 1.97958s (18.1857GiB/s) | 20260507 00:22:48.393 | INFO | Downloaded 34GiB to GPU in 1.94498s (17.4809GiB/s) | 20260507 00:22:53.532 | INFO | Analyzing chunk frames = [5626368,5626880) | 20260507 00:22:53.532 | INFO | MemInfo: { | 20260507 00:22:53.532 | INFO | "gpu" : " 17 GB", | 20260507 00:22:53.532 | INFO | "huge" : "", | 20260507 00:22:53.532 | INFO | "huge_shared" : " 27 GB", | 20260507 00:22:53.532 | INFO | "malloc" : " 14 GB", | 20260507 00:22:53.532 | INFO | "pinned" : " 29 GB" | 20260507 00:22:53.532 | INFO | } | 20260507 00:22:53.532 | INFO | Perf report for the last 5.13977 seconds: | 20260507 00:22:53.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:22:53.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:53.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:53.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:53.532 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:22:53.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:53.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:53.532 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:22:53.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:53.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:53.532 | INFO | Uploaded 36GiB to GPU in 2.00218s (17.9804GiB/s) | 20260507 00:22:53.532 | INFO | Downloaded 34GiB to GPU in 1.95294s (17.4097GiB/s) | 20260507 00:22:58.667 | INFO | Analyzing chunk frames = [5626880,5627392) | 20260507 00:22:58.667 | INFO | MemInfo: { | 20260507 00:22:58.667 | INFO | "gpu" : " 17 GB", | 20260507 00:22:58.667 | INFO | "huge" : "", | 20260507 00:22:58.667 | INFO | "huge_shared" : " 27 GB", | 20260507 00:22:58.667 | INFO | "malloc" : " 14 GB", | 20260507 00:22:58.667 | INFO | "pinned" : " 29 GB" | 20260507 00:22:58.667 | INFO | } | 20260507 00:22:58.667 | INFO | Perf report for the last 5.13462 seconds: | 20260507 00:22:58.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:22:58.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:58.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:58.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:58.667 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:22:58.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:58.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:58.667 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 00:22:58.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:58.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:22:58.667 | INFO | Uploaded 36GiB to GPU in 1.95355s (18.428GiB/s) | 20260507 00:22:58.667 | INFO | Downloaded 34GiB to GPU in 1.99673s (17.0279GiB/s) | 20260507 00:23:03.803 | INFO | Analyzing chunk frames = [5627392,5627904) | 20260507 00:23:03.803 | INFO | MemInfo: { | 20260507 00:23:03.803 | INFO | "gpu" : " 17 GB", | 20260507 00:23:03.803 | INFO | "huge" : "", | 20260507 00:23:03.803 | INFO | "huge_shared" : " 27 GB", | 20260507 00:23:03.803 | INFO | "malloc" : " 14 GB", | 20260507 00:23:03.804 | INFO | "pinned" : " 29 GB" | 20260507 00:23:03.804 | INFO | } | 20260507 00:23:03.804 | INFO | Perf report for the last 5.13686 seconds: | 20260507 00:23:03.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:23:03.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:03.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:03.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:03.804 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:23:03.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:03.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:03.804 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:23:03.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:03.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:03.804 | INFO | Uploaded 36GiB to GPU in 1.98583s (18.1285GiB/s) | 20260507 00:23:03.804 | INFO | Downloaded 34GiB to GPU in 1.98235s (17.1514GiB/s) | 20260507 00:23:03.804 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:23:03.804 | INFO | Total mem usage: 18 GiB | 20260507 00:23:03.804 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:23:03.804 | INFO | Recent High Water Marks Sum: 14.8 GiB (Savings: -21.9%) | 20260507 00:23:03.804 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:23:03.804 | INFO | High Water Breakdown: | 20260507 00:23:03.804 | INFO | 60.7% : 8.96 GiB ( 78.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:23:03.804 | INFO | 39.2% : 5.79 GiB ( 86.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:23:03.804 | INFO | 0.135% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:23:03.804 | INFO | ------------------------------------------ | 20260507 00:23:03.804 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:23:03.804 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:23:03.804 | INFO | Total mem usage: 26.5 GiB | 20260507 00:23:03.804 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:23:03.804 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:23:03.804 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:23:03.804 | INFO | High Water Breakdown: | 20260507 00:23:03.804 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:23:03.804 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:23:03.804 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:23:03.804 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:23:03.804 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:23:03.804 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:23:03.804 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:23:03.804 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:23:03.804 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:23:03.804 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:23:03.804 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:23:03.804 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:23:03.804 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:23:03.804 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:23:03.804 | INFO | ------------------------------------------ | 20260507 00:23:03.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:23:03.805 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:23:03.805 | INFO | Total mem usage: 25.6 GiB | 20260507 00:23:03.805 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:23:03.805 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:23:03.805 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:23:03.805 | INFO | High Water Breakdown: | 20260507 00:23:03.805 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:23:03.805 | INFO | ------------------------------------------ | 20260507 00:23:03.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:23:03.805 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:23:03.805 | INFO | Total mem usage: 15.7 GiB | 20260507 00:23:03.805 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:23:03.805 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:23:03.805 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:23:03.805 | INFO | High Water Breakdown: | 20260507 00:23:03.805 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:23:03.805 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:23:03.805 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:23:03.805 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:23:03.805 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:23:03.805 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:23:03.805 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:23:03.805 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:23:03.805 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:23:03.805 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:23:03.805 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:23:03.805 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:23:03.805 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:23:03.805 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:23:03.805 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:23:03.805 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:23:03.805 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:23:03.805 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:23:03.805 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:23:03.805 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:23:03.805 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:23:03.805 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:23:03.805 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:23:03.805 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:23:03.805 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:23:03.805 | INFO | ------------------------------------------ | 20260507 00:23:03.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:23:03.805 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:23:03.805 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:23:03.805 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:23:03.805 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.60e+01, 1.35e+01 / 2.47e+00 | 20260507 00:23:03.805 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.71e-03 / 5.78e-03 | 20260507 00:23:03.805 | INFO | PerfReport: 11.65%, Baseline -- 3.61e+00 / 4.37e+00, 5.39e-01 / 3.47e-01 | 20260507 00:23:03.805 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.43e+00, 1.76e-01 / 1.32e-01 | 20260507 00:23:03.805 | INFO | PerfReport: 16.08%, DME -- 3.98e+01 / 4.46e+01, 4.15e+00 / 2.84e+00 | 20260507 00:23:03.805 | INFO | PerfReport: 57.47%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.35e+00 / 1.90e+00 | 20260507 00:23:03.805 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.17e+00 / 2.72e+00, 3.79e-01 / 2.32e-01 | 20260507 00:23:03.805 | INFO | PerfReport: 3.15%, Metrics -- 9.76e-01 / 1.33e+00, 2.61e-01 / 1.59e-01 | 20260507 00:23:03.805 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.31e-01 / 3.83e-01, 5.21e-02 / 2.38e-02 | 20260507 00:23:03.805 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:23:03.805 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:23:03.805 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:23:03.805 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 00:23:03.805 | INFO | PerfReport: 21.53%, Upload -- 2.08e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260507 00:23:03.805 | INFO | PerfReport: 21.34%, Download -- 2.06e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260507 00:23:03.805 | INFO | PerfReport: 32.05%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:23:03.805 | INFO | PerfReport: 1.27%, LockUpload -- 1.22e+00 / 0.00e+00, 4.75e+00 / 0.00e+00 | 20260507 00:23:03.805 | INFO | PerfReport: 1.35%, LockDownload -- 1.30e+00 / 0.00e+00, 3.27e+00 / 0.00e+00 | 20260507 00:23:03.805 | INFO | PerfReport: 22.45%, LockCompute -- 2.17e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 00:23:03.805 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260507 00:23:03.902 | INFO | Starting a round of Baz IO | 20260507 00:23:07.033 | INFO | Populating RTMetrics report | 20260507 00:23:07.033 | INFO | RTMetrics report populated | 20260507 00:23:07.033 | INFO | Opening tmp RTMetrics json file | 20260507 00:23:07.060 | INFO | Serializing RTMetrics to json | 20260507 00:23:07.062 | INFO | Serialization done, swapping tmp file now... | 20260507 00:23:07.062 | INFO | RTMetrics done | 20260507 00:23:07.134 | INFO | Finished a round of Baz IO: Wrote 1.85 GiB in 3.232 seconds | 20260507 00:23:07.134 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.30% / 0.00% / 66.70% | 20260507 00:23:07.135 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.34856% / 1.19209e-05% / 7.15256e-05% | 20260507 00:23:07.135 | INFO | Baz IO overall: | 20260507 00:23:07.135 | INFO | BytesWritten : 2934425024373 | 20260507 00:23:07.135 | INFO | NumEvents : 1413693964438 | 20260507 00:23:08.946 | INFO | Analyzing chunk frames = [5627904,5628416) | 20260507 00:23:08.946 | INFO | MemInfo: { | 20260507 00:23:08.946 | INFO | "gpu" : " 17 GB", | 20260507 00:23:08.946 | INFO | "huge" : "", | 20260507 00:23:08.946 | INFO | "huge_shared" : " 27 GB", | 20260507 00:23:08.946 | INFO | "malloc" : " 14 GB", | 20260507 00:23:08.946 | INFO | "pinned" : " 29 GB" | 20260507 00:23:08.946 | INFO | } | 20260507 00:23:08.946 | INFO | Perf report for the last 5.14236 seconds: | 20260507 00:23:08.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:23:08.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:08.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:08.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:08.946 | INFO | ANALYSIS : 62.3%, 3.0, 96.9ms, 0 | 20260507 00:23:08.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:08.946 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 00:23:08.946 | INFO | PRE_HQ : 59.6%, 1.8, 56.3ms, 0 | 20260507 00:23:08.946 | INFO | BAZWRITER : 62.3%, 3.5, 112.1ms, 0 | 20260507 00:23:08.946 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 00:23:08.946 | INFO | Uploaded 36GiB to GPU in 2.04023s (17.6451GiB/s) | 20260507 00:23:08.946 | INFO | Downloaded 38GiB to GPU in 2.17776s (17.4491GiB/s) | 20260507 00:23:14.084 | INFO | Analyzing chunk frames = [5628416,5628928) | 20260507 00:23:14.084 | INFO | MemInfo: { | 20260507 00:23:14.084 | INFO | "gpu" : " 17 GB", | 20260507 00:23:14.084 | INFO | "huge" : "", | 20260507 00:23:14.084 | INFO | "huge_shared" : " 27 GB", | 20260507 00:23:14.084 | INFO | "malloc" : " 14 GB", | 20260507 00:23:14.084 | INFO | "pinned" : " 29 GB" | 20260507 00:23:14.084 | INFO | } | 20260507 00:23:14.085 | INFO | Perf report for the last 5.13847 seconds: | 20260507 00:23:14.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:23:14.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:14.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:14.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:14.085 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:23:14.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:14.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:14.085 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 00:23:14.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:14.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:14.085 | INFO | Uploaded 36GiB to GPU in 2.01347s (17.8796GiB/s) | 20260507 00:23:14.085 | INFO | Downloaded 34GiB to GPU in 1.92723s (17.6419GiB/s) | 20260507 00:23:19.236 | INFO | Analyzing chunk frames = [5628928,5629440) | 20260507 00:23:19.236 | INFO | MemInfo: { | 20260507 00:23:19.236 | INFO | "gpu" : " 17 GB", | 20260507 00:23:19.236 | INFO | "huge" : "", | 20260507 00:23:19.236 | INFO | "huge_shared" : " 27 GB", | 20260507 00:23:19.236 | INFO | "malloc" : " 14 GB", | 20260507 00:23:19.236 | INFO | "pinned" : " 29 GB" | 20260507 00:23:19.236 | INFO | } | 20260507 00:23:19.236 | INFO | Perf report for the last 5.1518 seconds: | 20260507 00:23:19.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:23:19.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:19.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:19.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:19.236 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260507 00:23:19.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:19.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:19.237 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 00:23:19.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:19.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:19.237 | INFO | Uploaded 36GiB to GPU in 1.98269s (18.1571GiB/s) | 20260507 00:23:19.237 | INFO | Downloaded 34GiB to GPU in 1.91825s (17.7245GiB/s) | 20260507 00:23:24.373 | INFO | Analyzing chunk frames = [5629440,5629952) | 20260507 00:23:24.373 | INFO | MemInfo: { | 20260507 00:23:24.373 | INFO | "gpu" : " 17 GB", | 20260507 00:23:24.373 | INFO | "huge" : "", | 20260507 00:23:24.373 | INFO | "huge_shared" : " 27 GB", | 20260507 00:23:24.373 | INFO | "malloc" : " 14 GB", | 20260507 00:23:24.373 | INFO | "pinned" : " 29 GB" | 20260507 00:23:24.373 | INFO | } | 20260507 00:23:24.374 | INFO | Perf report for the last 5.13729 seconds: | 20260507 00:23:24.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:23:24.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:24.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:24.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:24.374 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:23:24.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:24.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:24.374 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 00:23:24.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:24.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:24.374 | INFO | Uploaded 36GiB to GPU in 1.98117s (18.1711GiB/s) | 20260507 00:23:24.374 | INFO | Downloaded 34GiB to GPU in 1.90628s (17.8358GiB/s) | 20260507 00:23:29.510 | INFO | Analyzing chunk frames = [5629952,5630464) | 20260507 00:23:29.510 | INFO | MemInfo: { | 20260507 00:23:29.510 | INFO | "gpu" : " 17 GB", | 20260507 00:23:29.510 | INFO | "huge" : "", | 20260507 00:23:29.510 | INFO | "huge_shared" : " 27 GB", | 20260507 00:23:29.510 | INFO | "malloc" : " 14 GB", | 20260507 00:23:29.510 | INFO | "pinned" : " 29 GB" | 20260507 00:23:29.510 | INFO | } | 20260507 00:23:29.510 | INFO | Perf report for the last 5.13617 seconds: | 20260507 00:23:29.510 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:23:29.510 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:29.510 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:29.510 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:29.510 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 00:23:29.510 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:29.510 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:29.510 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:23:29.510 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:29.510 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:29.510 | INFO | Uploaded 36GiB to GPU in 2.02007s (17.8212GiB/s) | 20260507 00:23:29.510 | INFO | Downloaded 34GiB to GPU in 1.9439s (17.4906GiB/s) | 20260507 00:23:34.645 | INFO | Analyzing chunk frames = [5630464,5630976) | 20260507 00:23:34.645 | INFO | MemInfo: { | 20260507 00:23:34.645 | INFO | "gpu" : " 17 GB", | 20260507 00:23:34.645 | INFO | "huge" : "", | 20260507 00:23:34.645 | INFO | "huge_shared" : " 27 GB", | 20260507 00:23:34.645 | INFO | "malloc" : " 14 GB", | 20260507 00:23:34.646 | INFO | "pinned" : " 29 GB" | 20260507 00:23:34.646 | INFO | } | 20260507 00:23:34.646 | INFO | Perf report for the last 5.13584 seconds: | 20260507 00:23:34.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:23:34.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:34.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:34.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:34.646 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 00:23:34.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:34.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:34.646 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 00:23:34.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:34.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:34.646 | INFO | Uploaded 36GiB to GPU in 2.03253s (17.712GiB/s) | 20260507 00:23:34.646 | INFO | Downloaded 34GiB to GPU in 1.9449s (17.4816GiB/s) | 20260507 00:23:39.783 | INFO | Analyzing chunk frames = [5630976,5631488) | 20260507 00:23:39.783 | INFO | MemInfo: { | 20260507 00:23:39.783 | INFO | "gpu" : " 17 GB", | 20260507 00:23:39.783 | INFO | "huge" : "", | 20260507 00:23:39.783 | INFO | "huge_shared" : " 27 GB", | 20260507 00:23:39.783 | INFO | "malloc" : " 14 GB", | 20260507 00:23:39.783 | INFO | "pinned" : " 29 GB" | 20260507 00:23:39.783 | INFO | } | 20260507 00:23:39.783 | INFO | Perf report for the last 5.13732 seconds: | 20260507 00:23:39.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:23:39.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:39.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:39.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:39.783 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260507 00:23:39.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:39.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:39.783 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:23:39.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:39.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:39.783 | INFO | Uploaded 36GiB to GPU in 1.96751s (18.2973GiB/s) | 20260507 00:23:39.783 | INFO | Downloaded 34GiB to GPU in 1.99647s (17.0301GiB/s) | 20260507 00:23:44.927 | INFO | Analyzing chunk frames = [5631488,5632000) | 20260507 00:23:44.927 | INFO | MemInfo: { | 20260507 00:23:44.927 | INFO | "gpu" : " 17 GB", | 20260507 00:23:44.927 | INFO | "huge" : "", | 20260507 00:23:44.927 | INFO | "huge_shared" : " 27 GB", | 20260507 00:23:44.927 | INFO | "malloc" : " 14 GB", | 20260507 00:23:44.927 | INFO | "pinned" : " 29 GB" | 20260507 00:23:44.927 | INFO | } | 20260507 00:23:44.928 | INFO | Perf report for the last 5.14451 seconds: | 20260507 00:23:44.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:23:44.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:44.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:44.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:44.928 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260507 00:23:44.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:44.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:44.928 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:23:44.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:44.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:44.928 | INFO | Uploaded 36GiB to GPU in 2.01714s (17.8471GiB/s) | 20260507 00:23:44.928 | INFO | Downloaded 34GiB to GPU in 1.98923s (17.0921GiB/s) | 20260507 00:23:45.027 | INFO | Starting a round of Baz IO | 20260507 00:23:48.153 | INFO | Populating RTMetrics report | 20260507 00:23:48.153 | INFO | RTMetrics report populated | 20260507 00:23:48.153 | INFO | Opening tmp RTMetrics json file | 20260507 00:23:48.160 | INFO | Serializing RTMetrics to json | 20260507 00:23:48.177 | INFO | Serialization done, swapping tmp file now... | 20260507 00:23:48.178 | INFO | RTMetrics done | 20260507 00:23:48.252 | INFO | Finished a round of Baz IO: Wrote 1.847 GiB in 3.224 seconds | 20260507 00:23:48.252 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.30% / 0.00% / 66.70% | 20260507 00:23:48.252 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.34732% / 0% / 5.96046e-05% | 20260507 00:23:48.252 | INFO | Baz IO overall: | 20260507 00:23:48.252 | INFO | BytesWritten : 2936408533052 | 20260507 00:23:48.252 | INFO | NumEvents : 1414635555124 | 20260507 00:23:50.068 | INFO | Analyzing chunk frames = [5632000,5632512) | 20260507 00:23:50.068 | INFO | MemInfo: { | 20260507 00:23:50.068 | INFO | "gpu" : " 17 GB", | 20260507 00:23:50.068 | INFO | "huge" : "", | 20260507 00:23:50.068 | INFO | "huge_shared" : " 27 GB", | 20260507 00:23:50.068 | INFO | "malloc" : " 14 GB", | 20260507 00:23:50.068 | INFO | "pinned" : " 29 GB" | 20260507 00:23:50.068 | INFO | } | 20260507 00:23:50.068 | INFO | Perf report for the last 5.1407 seconds: | 20260507 00:23:50.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:23:50.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:50.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:50.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:50.068 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 00:23:50.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:50.068 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 00:23:50.068 | INFO | PRE_HQ : 59.7%, 1.8, 56.4ms, 0 | 20260507 00:23:50.068 | INFO | BAZWRITER : 62.3%, 3.4, 110.4ms, 0 | 20260507 00:23:50.068 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:23:50.068 | INFO | Uploaded 36GiB to GPU in 2.03288s (17.7089GiB/s) | 20260507 00:23:50.068 | INFO | Downloaded 38GiB to GPU in 2.17088s (17.5045GiB/s) | 20260507 00:23:55.205 | INFO | Analyzing chunk frames = [5632512,5633024) | 20260507 00:23:55.205 | INFO | MemInfo: { | 20260507 00:23:55.205 | INFO | "gpu" : " 17 GB", | 20260507 00:23:55.205 | INFO | "huge" : "", | 20260507 00:23:55.205 | INFO | "huge_shared" : " 27 GB", | 20260507 00:23:55.205 | INFO | "malloc" : " 14 GB", | 20260507 00:23:55.205 | INFO | "pinned" : " 29 GB" | 20260507 00:23:55.205 | INFO | } | 20260507 00:23:55.205 | INFO | Perf report for the last 5.1367 seconds: | 20260507 00:23:55.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:23:55.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:55.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:55.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:55.205 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:23:55.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:55.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:55.205 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:23:55.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:55.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:23:55.205 | INFO | Uploaded 36GiB to GPU in 2.00888s (17.9205GiB/s) | 20260507 00:23:55.205 | INFO | Downloaded 34GiB to GPU in 1.9248s (17.6642GiB/s) | 20260507 00:24:00.343 | INFO | Analyzing chunk frames = [5633024,5633536) | 20260507 00:24:00.343 | INFO | MemInfo: { | 20260507 00:24:00.343 | INFO | "gpu" : " 17 GB", | 20260507 00:24:00.343 | INFO | "huge" : "", | 20260507 00:24:00.343 | INFO | "huge_shared" : " 27 GB", | 20260507 00:24:00.343 | INFO | "malloc" : " 14 GB", | 20260507 00:24:00.343 | INFO | "pinned" : " 29 GB" | 20260507 00:24:00.343 | INFO | } | 20260507 00:24:00.343 | INFO | Perf report for the last 5.13823 seconds: | 20260507 00:24:00.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:24:00.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:00.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:00.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:00.343 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:24:00.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:00.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:00.343 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:24:00.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:00.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:00.343 | INFO | Uploaded 36GiB to GPU in 1.99116s (18.0799GiB/s) | 20260507 00:24:00.344 | INFO | Downloaded 34GiB to GPU in 1.92707s (17.6434GiB/s) | 20260507 00:24:05.485 | INFO | Analyzing chunk frames = [5633536,5634048) | 20260507 00:24:05.485 | INFO | MemInfo: { | 20260507 00:24:05.485 | INFO | "gpu" : " 17 GB", | 20260507 00:24:05.485 | INFO | "huge" : "", | 20260507 00:24:05.485 | INFO | "huge_shared" : " 27 GB", | 20260507 00:24:05.485 | INFO | "malloc" : " 14 GB", | 20260507 00:24:05.485 | INFO | "pinned" : " 29 GB" | 20260507 00:24:05.485 | INFO | } | 20260507 00:24:05.485 | INFO | Perf report for the last 5.14179 seconds: | 20260507 00:24:05.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:24:05.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:05.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:05.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:05.485 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260507 00:24:05.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:05.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:05.485 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:24:05.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:05.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:05.485 | INFO | Uploaded 36GiB to GPU in 1.99512s (18.044GiB/s) | 20260507 00:24:05.485 | INFO | Downloaded 34GiB to GPU in 1.90757s (17.8238GiB/s) | 20260507 00:24:10.619 | INFO | Analyzing chunk frames = [5634048,5634560) | 20260507 00:24:10.619 | INFO | MemInfo: { | 20260507 00:24:10.619 | INFO | "gpu" : " 17 GB", | 20260507 00:24:10.619 | INFO | "huge" : "", | 20260507 00:24:10.619 | INFO | "huge_shared" : " 27 GB", | 20260507 00:24:10.619 | INFO | "malloc" : " 14 GB", | 20260507 00:24:10.619 | INFO | "pinned" : " 29 GB" | 20260507 00:24:10.620 | INFO | } | 20260507 00:24:10.620 | INFO | Perf report for the last 5.13465 seconds: | 20260507 00:24:10.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:24:10.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:10.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:10.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:10.620 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260507 00:24:10.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:10.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:10.620 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:24:10.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:10.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:10.620 | INFO | Uploaded 36GiB to GPU in 1.98895s (18.1GiB/s) | 20260507 00:24:10.620 | INFO | Downloaded 34GiB to GPU in 1.93107s (17.6068GiB/s) | 20260507 00:24:15.754 | INFO | Analyzing chunk frames = [5634560,5635072) | 20260507 00:24:15.754 | INFO | MemInfo: { | 20260507 00:24:15.754 | INFO | "gpu" : " 17 GB", | 20260507 00:24:15.754 | INFO | "huge" : "", | 20260507 00:24:15.754 | INFO | "huge_shared" : " 27 GB", | 20260507 00:24:15.754 | INFO | "malloc" : " 14 GB", | 20260507 00:24:15.754 | INFO | "pinned" : " 29 GB" | 20260507 00:24:15.754 | INFO | } | 20260507 00:24:15.754 | INFO | Perf report for the last 5.13455 seconds: | 20260507 00:24:15.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:24:15.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:15.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:15.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:15.754 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 00:24:15.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:15.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:15.754 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:24:15.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:15.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:15.754 | INFO | Uploaded 36GiB to GPU in 2.02443s (17.7828GiB/s) | 20260507 00:24:15.754 | INFO | Downloaded 34GiB to GPU in 1.93441s (17.5764GiB/s) | 20260507 00:24:20.896 | INFO | Analyzing chunk frames = [5635072,5635584) | 20260507 00:24:20.896 | INFO | MemInfo: { | 20260507 00:24:20.896 | INFO | "gpu" : " 17 GB", | 20260507 00:24:20.896 | INFO | "huge" : "", | 20260507 00:24:20.896 | INFO | "huge_shared" : " 27 GB", | 20260507 00:24:20.896 | INFO | "malloc" : " 14 GB", | 20260507 00:24:20.896 | INFO | "pinned" : " 29 GB" | 20260507 00:24:20.896 | INFO | } | 20260507 00:24:20.896 | INFO | Perf report for the last 5.14211 seconds: | 20260507 00:24:20.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:24:20.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:20.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:20.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:20.897 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 00:24:20.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:20.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:20.897 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 00:24:20.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:20.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:20.897 | INFO | Uploaded 36GiB to GPU in 1.96353s (18.3344GiB/s) | 20260507 00:24:20.897 | INFO | Downloaded 34GiB to GPU in 1.98447s (17.133GiB/s) | 20260507 00:24:26.040 | INFO | Analyzing chunk frames = [5635584,5636096) | 20260507 00:24:26.040 | INFO | MemInfo: { | 20260507 00:24:26.040 | INFO | "gpu" : " 17 GB", | 20260507 00:24:26.040 | INFO | "huge" : "", | 20260507 00:24:26.040 | INFO | "huge_shared" : " 27 GB", | 20260507 00:24:26.040 | INFO | "malloc" : " 14 GB", | 20260507 00:24:26.040 | INFO | "pinned" : " 29 GB" | 20260507 00:24:26.040 | INFO | } | 20260507 00:24:26.040 | INFO | Perf report for the last 5.14348 seconds: | 20260507 00:24:26.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:24:26.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:26.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:26.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:26.040 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260507 00:24:26.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:26.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:26.040 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 00:24:26.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:26.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:26.040 | INFO | Uploaded 36GiB to GPU in 1.99331s (18.0604GiB/s) | 20260507 00:24:26.040 | INFO | Downloaded 34GiB to GPU in 1.98498s (17.1286GiB/s) | 20260507 00:24:26.040 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:24:26.040 | INFO | Total mem usage: 18 GiB | 20260507 00:24:26.040 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:24:26.040 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.6%) | 20260507 00:24:26.040 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:24:26.040 | INFO | High Water Breakdown: | 20260507 00:24:26.040 | INFO | 60.2% : 8.63 GiB ( 75.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:24:26.040 | INFO | 39.7% : 5.69 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:24:26.040 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:24:26.040 | INFO | ------------------------------------------ | 20260507 00:24:26.040 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:24:26.040 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:24:26.040 | INFO | Total mem usage: 26.5 GiB | 20260507 00:24:26.040 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:24:26.040 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:24:26.040 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:24:26.040 | INFO | High Water Breakdown: | 20260507 00:24:26.040 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:24:26.040 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:24:26.040 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:24:26.040 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:24:26.041 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:24:26.041 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:24:26.041 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:24:26.041 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:24:26.041 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:24:26.041 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:24:26.041 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:24:26.041 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:24:26.041 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:24:26.041 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:24:26.041 | INFO | ------------------------------------------ | 20260507 00:24:26.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:24:26.041 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:24:26.041 | INFO | Total mem usage: 25.6 GiB | 20260507 00:24:26.041 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:24:26.041 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:24:26.041 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:24:26.041 | INFO | High Water Breakdown: | 20260507 00:24:26.041 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:24:26.041 | INFO | ------------------------------------------ | 20260507 00:24:26.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:24:26.041 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:24:26.041 | INFO | Total mem usage: 15.7 GiB | 20260507 00:24:26.041 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:24:26.041 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:24:26.041 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:24:26.041 | INFO | High Water Breakdown: | 20260507 00:24:26.041 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:24:26.041 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:24:26.041 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:24:26.041 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:24:26.041 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:24:26.041 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:24:26.041 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:24:26.041 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:24:26.041 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:24:26.041 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:24:26.041 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:24:26.041 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:24:26.041 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:24:26.041 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:24:26.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:24:26.041 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:24:26.041 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:24:26.041 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:24:26.041 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:24:26.041 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:24:26.041 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:24:26.041 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:24:26.041 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:24:26.041 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:24:26.041 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:24:26.041 | INFO | ------------------------------------------ | 20260507 00:24:26.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:24:26.041 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:24:26.041 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:24:26.041 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:24:26.041 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.60e+01, 1.34e+01 / 2.28e+00 | 20260507 00:24:26.041 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.76e-03 / 5.48e-03 | 20260507 00:24:26.041 | INFO | PerfReport: 11.69%, Baseline -- 3.62e+00 / 4.40e+00, 5.38e-01 / 3.00e-01 | 20260507 00:24:26.041 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.46e+00, 1.78e-01 / 1.24e-01 | 20260507 00:24:26.041 | INFO | PerfReport: 16.14%, DME -- 4.00e+01 / 4.52e+01, 4.28e+00 / 2.66e+00 | 20260507 00:24:26.041 | INFO | PerfReport: 57.35%, FrameLabeling -- 1.77e+01 / 2.05e+01, 1.30e+00 / 1.97e+00 | 20260507 00:24:26.041 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.18e+00 / 2.74e+00, 3.78e-01 / 2.29e-01 | 20260507 00:24:26.041 | INFO | PerfReport: 3.15%, Metrics -- 9.76e-01 / 1.32e+00, 2.59e-01 / 1.28e-01 | 20260507 00:24:26.041 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.31e-01 / 3.82e-01, 5.17e-02 / 1.79e-02 | 20260507 00:24:26.041 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:24:26.041 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:24:26.041 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:24:26.041 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 00:24:26.041 | INFO | PerfReport: 21.60%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260507 00:24:26.041 | INFO | PerfReport: 21.28%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260507 00:24:26.041 | INFO | PerfReport: 32.03%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 00:24:26.041 | INFO | PerfReport: 1.36%, LockUpload -- 1.31e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 00:24:26.041 | INFO | PerfReport: 1.27%, LockDownload -- 1.23e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260507 00:24:26.041 | INFO | PerfReport: 22.45%, LockCompute -- 2.17e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 00:24:26.041 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.41e-02 / 0.00e+00 | 20260507 00:24:26.137 | INFO | Starting a round of Baz IO | 20260507 00:24:29.261 | INFO | Populating RTMetrics report | 20260507 00:24:29.261 | INFO | RTMetrics report populated | 20260507 00:24:29.261 | INFO | Opening tmp RTMetrics json file | 20260507 00:24:29.262 | INFO | Serializing RTMetrics to json | 20260507 00:24:29.263 | INFO | Serialization done, swapping tmp file now... | 20260507 00:24:29.264 | INFO | RTMetrics done | 20260507 00:24:29.354 | INFO | Finished a round of Baz IO: Wrote 1.846 GiB in 3.216 seconds | 20260507 00:24:29.354 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.30% / 0.00% / 66.70% | 20260507 00:24:29.354 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.346156% / 0% / 3.57628e-05% | 20260507 00:24:29.354 | INFO | Baz IO overall: | 20260507 00:24:29.354 | INFO | BytesWritten : 2938390944406 | 20260507 00:24:29.354 | INFO | NumEvents : 1415576813802 | 20260507 00:24:31.185 | INFO | Analyzing chunk frames = [5636096,5636608) | 20260507 00:24:31.185 | INFO | MemInfo: { | 20260507 00:24:31.185 | INFO | "gpu" : " 17 GB", | 20260507 00:24:31.185 | INFO | "huge" : "", | 20260507 00:24:31.185 | INFO | "huge_shared" : " 27 GB", | 20260507 00:24:31.185 | INFO | "malloc" : " 14 GB", | 20260507 00:24:31.185 | INFO | "pinned" : " 29 GB" | 20260507 00:24:31.185 | INFO | } | 20260507 00:24:31.185 | INFO | Perf report for the last 5.14492 seconds: | 20260507 00:24:31.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:24:31.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:31.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:31.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:31.185 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260507 00:24:31.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:31.185 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 00:24:31.185 | INFO | PRE_HQ : 59.6%, 1.8, 56.7ms, 0 | 20260507 00:24:31.185 | INFO | BAZWRITER : 62.1%, 3.4, 109.0ms, 0 | 20260507 00:24:31.185 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 00:24:31.185 | INFO | Uploaded 36GiB to GPU in 2.01526s (17.8637GiB/s) | 20260507 00:24:31.185 | INFO | Downloaded 38GiB to GPU in 2.17512s (17.4703GiB/s) | 20260507 00:24:36.326 | INFO | Analyzing chunk frames = [5636608,5637120) | 20260507 00:24:36.326 | INFO | MemInfo: { | 20260507 00:24:36.326 | INFO | "gpu" : " 17 GB", | 20260507 00:24:36.326 | INFO | "huge" : "", | 20260507 00:24:36.326 | INFO | "huge_shared" : " 27 GB", | 20260507 00:24:36.326 | INFO | "malloc" : " 14 GB", | 20260507 00:24:36.326 | INFO | "pinned" : " 29 GB" | 20260507 00:24:36.326 | INFO | } | 20260507 00:24:36.326 | INFO | Perf report for the last 5.14123 seconds: | 20260507 00:24:36.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:24:36.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:36.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:36.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:36.326 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 00:24:36.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:36.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:36.326 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:24:36.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:36.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:36.326 | INFO | Uploaded 36GiB to GPU in 2.04363s (17.6157GiB/s) | 20260507 00:24:36.326 | INFO | Downloaded 34GiB to GPU in 1.93353s (17.5844GiB/s) | 20260507 00:24:41.458 | INFO | Analyzing chunk frames = [5637120,5637632) | 20260507 00:24:41.458 | INFO | MemInfo: { | 20260507 00:24:41.458 | INFO | "gpu" : " 17 GB", | 20260507 00:24:41.458 | INFO | "huge" : "", | 20260507 00:24:41.458 | INFO | "huge_shared" : " 27 GB", | 20260507 00:24:41.458 | INFO | "malloc" : " 14 GB", | 20260507 00:24:41.458 | INFO | "pinned" : " 29 GB" | 20260507 00:24:41.458 | INFO | } | 20260507 00:24:41.458 | INFO | Perf report for the last 5.1324 seconds: | 20260507 00:24:41.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:24:41.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:41.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:41.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:41.459 | INFO | ANALYSIS : 60.5%, 3.0, 93.5ms, 0 | 20260507 00:24:41.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:41.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:41.459 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:24:41.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:41.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:41.459 | INFO | Uploaded 36GiB to GPU in 1.99693s (18.0276GiB/s) | 20260507 00:24:41.459 | INFO | Downloaded 34GiB to GPU in 1.93285s (17.5906GiB/s) | 20260507 00:24:46.596 | INFO | Analyzing chunk frames = [5637632,5638144) | 20260507 00:24:46.596 | INFO | MemInfo: { | 20260507 00:24:46.596 | INFO | "gpu" : " 17 GB", | 20260507 00:24:46.596 | INFO | "huge" : "", | 20260507 00:24:46.596 | INFO | "huge_shared" : " 27 GB", | 20260507 00:24:46.596 | INFO | "malloc" : " 14 GB", | 20260507 00:24:46.596 | INFO | "pinned" : " 29 GB" | 20260507 00:24:46.596 | INFO | } | 20260507 00:24:46.596 | INFO | Perf report for the last 5.13805 seconds: | 20260507 00:24:46.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:24:46.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:46.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:46.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:46.597 | INFO | ANALYSIS : 59.5%, 3.0, 92.2ms, 0 | 20260507 00:24:46.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:46.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:46.597 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:24:46.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:46.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:46.597 | INFO | Uploaded 36GiB to GPU in 1.97386s (18.2384GiB/s) | 20260507 00:24:46.597 | INFO | Downloaded 34GiB to GPU in 1.8775s (18.1092GiB/s) | 20260507 00:24:51.734 | INFO | Analyzing chunk frames = [5638144,5638656) | 20260507 00:24:51.734 | INFO | MemInfo: { | 20260507 00:24:51.734 | INFO | "gpu" : " 17 GB", | 20260507 00:24:51.734 | INFO | "huge" : "", | 20260507 00:24:51.734 | INFO | "huge_shared" : " 27 GB", | 20260507 00:24:51.734 | INFO | "malloc" : " 14 GB", | 20260507 00:24:51.734 | INFO | "pinned" : " 29 GB" | 20260507 00:24:51.734 | INFO | } | 20260507 00:24:51.734 | INFO | Perf report for the last 5.13767 seconds: | 20260507 00:24:51.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:24:51.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:51.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:51.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:51.734 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 00:24:51.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:51.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:51.734 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:24:51.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:51.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:51.734 | INFO | Uploaded 36GiB to GPU in 1.9863s (18.1242GiB/s) | 20260507 00:24:51.735 | INFO | Downloaded 34GiB to GPU in 1.92514s (17.6611GiB/s) | 20260507 00:24:56.874 | INFO | Analyzing chunk frames = [5638656,5639168) | 20260507 00:24:56.874 | INFO | MemInfo: { | 20260507 00:24:56.874 | INFO | "gpu" : " 17 GB", | 20260507 00:24:56.874 | INFO | "huge" : "", | 20260507 00:24:56.874 | INFO | "huge_shared" : " 27 GB", | 20260507 00:24:56.874 | INFO | "malloc" : " 14 GB", | 20260507 00:24:56.874 | INFO | "pinned" : " 29 GB" | 20260507 00:24:56.874 | INFO | } | 20260507 00:24:56.874 | INFO | Perf report for the last 5.13964 seconds: | 20260507 00:24:56.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:24:56.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:56.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:56.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:56.874 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:24:56.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:56.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:56.874 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 00:24:56.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:56.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:24:56.874 | INFO | Uploaded 36GiB to GPU in 2.03987s (17.6482GiB/s) | 20260507 00:24:56.874 | INFO | Downloaded 34GiB to GPU in 1.94306s (17.4982GiB/s) | 20260507 00:25:02.019 | INFO | Analyzing chunk frames = [5639168,5639680) | 20260507 00:25:02.019 | INFO | MemInfo: { | 20260507 00:25:02.019 | INFO | "gpu" : " 17 GB", | 20260507 00:25:02.019 | INFO | "huge" : "", | 20260507 00:25:02.019 | INFO | "huge_shared" : " 27 GB", | 20260507 00:25:02.019 | INFO | "malloc" : " 14 GB", | 20260507 00:25:02.019 | INFO | "pinned" : " 29 GB" | 20260507 00:25:02.019 | INFO | } | 20260507 00:25:02.019 | INFO | Perf report for the last 5.14545 seconds: | 20260507 00:25:02.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:25:02.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:02.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:02.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:02.019 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 00:25:02.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:02.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:02.020 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 00:25:02.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:02.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:02.020 | INFO | Uploaded 36GiB to GPU in 1.97861s (18.1946GiB/s) | 20260507 00:25:02.020 | INFO | Downloaded 34GiB to GPU in 1.99398s (17.0513GiB/s) | 20260507 00:25:07.150 | INFO | Analyzing chunk frames = [5639680,5640192) | 20260507 00:25:07.150 | INFO | MemInfo: { | 20260507 00:25:07.150 | INFO | "gpu" : " 17 GB", | 20260507 00:25:07.150 | INFO | "huge" : "", | 20260507 00:25:07.150 | INFO | "huge_shared" : " 27 GB", | 20260507 00:25:07.150 | INFO | "malloc" : " 14 GB", | 20260507 00:25:07.150 | INFO | "pinned" : " 29 GB" | 20260507 00:25:07.150 | INFO | } | 20260507 00:25:07.150 | INFO | Perf report for the last 5.13103 seconds: | 20260507 00:25:07.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:25:07.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:07.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:07.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:07.150 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260507 00:25:07.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:07.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:07.151 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:25:07.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:07.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:07.151 | INFO | Uploaded 36GiB to GPU in 1.98523s (18.1339GiB/s) | 20260507 00:25:07.151 | INFO | Downloaded 34GiB to GPU in 1.97373s (17.2262GiB/s) | 20260507 00:25:07.249 | INFO | Starting a round of Baz IO | 20260507 00:25:10.370 | INFO | Populating RTMetrics report | 20260507 00:25:10.370 | INFO | RTMetrics report populated | 20260507 00:25:10.370 | INFO | Opening tmp RTMetrics json file | 20260507 00:25:10.371 | INFO | Serializing RTMetrics to json | 20260507 00:25:10.388 | INFO | Serialization done, swapping tmp file now... | 20260507 00:25:10.391 | INFO | RTMetrics done | 20260507 00:25:10.468 | INFO | Finished a round of Baz IO: Wrote 1.848 GiB in 3.218 seconds | 20260507 00:25:10.468 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.29% / 0.00% / 66.70% | 20260507 00:25:10.468 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.34531% / 3.97364e-06% / 6.35783e-05% | 20260507 00:25:10.468 | INFO | Baz IO overall: | 20260507 00:25:10.468 | INFO | BytesWritten : 2940375058523 | 20260507 00:25:10.468 | INFO | NumEvents : 1416519117473 | 20260507 00:25:12.297 | INFO | Analyzing chunk frames = [5640192,5640704) | 20260507 00:25:12.297 | INFO | MemInfo: { | 20260507 00:25:12.297 | INFO | "gpu" : " 17 GB", | 20260507 00:25:12.297 | INFO | "huge" : "", | 20260507 00:25:12.297 | INFO | "huge_shared" : " 27 GB", | 20260507 00:25:12.297 | INFO | "malloc" : " 14 GB", | 20260507 00:25:12.297 | INFO | "pinned" : " 29 GB" | 20260507 00:25:12.297 | INFO | } | 20260507 00:25:12.297 | INFO | Perf report for the last 5.1465 seconds: | 20260507 00:25:12.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:25:12.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:12.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:12.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:12.297 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 00:25:12.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:12.297 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 00:25:12.297 | INFO | PRE_HQ : 59.2%, 1.8, 56.0ms, 0 | 20260507 00:25:12.297 | INFO | BAZWRITER : 62.1%, 3.4, 109.9ms, 0 | 20260507 00:25:12.297 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:25:12.297 | INFO | Uploaded 36GiB to GPU in 2.00332s (17.9702GiB/s) | 20260507 00:25:12.297 | INFO | Downloaded 38GiB to GPU in 2.18621s (17.3817GiB/s) | 20260507 00:25:17.432 | INFO | Analyzing chunk frames = [5640704,5641216) | 20260507 00:25:17.432 | INFO | MemInfo: { | 20260507 00:25:17.432 | INFO | "gpu" : " 17 GB", | 20260507 00:25:17.432 | INFO | "huge" : "", | 20260507 00:25:17.432 | INFO | "huge_shared" : " 27 GB", | 20260507 00:25:17.432 | INFO | "malloc" : " 14 GB", | 20260507 00:25:17.432 | INFO | "pinned" : " 29 GB" | 20260507 00:25:17.432 | INFO | } | 20260507 00:25:17.433 | INFO | Perf report for the last 5.13569 seconds: | 20260507 00:25:17.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:25:17.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:17.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:17.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:17.433 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 00:25:17.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:17.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:17.433 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:25:17.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:17.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:17.433 | INFO | Uploaded 36GiB to GPU in 1.99332s (18.0603GiB/s) | 20260507 00:25:17.433 | INFO | Downloaded 34GiB to GPU in 1.93441s (17.5764GiB/s) | 20260507 00:25:22.572 | INFO | Analyzing chunk frames = [5641216,5641728) | 20260507 00:25:22.572 | INFO | MemInfo: { | 20260507 00:25:22.572 | INFO | "gpu" : " 17 GB", | 20260507 00:25:22.572 | INFO | "huge" : "", | 20260507 00:25:22.572 | INFO | "huge_shared" : " 27 GB", | 20260507 00:25:22.572 | INFO | "malloc" : " 14 GB", | 20260507 00:25:22.572 | INFO | "pinned" : " 29 GB" | 20260507 00:25:22.572 | INFO | } | 20260507 00:25:22.572 | INFO | Perf report for the last 5.13978 seconds: | 20260507 00:25:22.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:25:22.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:22.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:22.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:22.572 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:25:22.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:22.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:22.572 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 00:25:22.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:22.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:22.573 | INFO | Uploaded 36GiB to GPU in 1.98425s (18.1429GiB/s) | 20260507 00:25:22.573 | INFO | Downloaded 34GiB to GPU in 1.93476s (17.5732GiB/s) | 20260507 00:25:27.707 | INFO | Analyzing chunk frames = [5641728,5642240) | 20260507 00:25:27.707 | INFO | MemInfo: { | 20260507 00:25:27.707 | INFO | "gpu" : " 17 GB", | 20260507 00:25:27.707 | INFO | "huge" : "", | 20260507 00:25:27.707 | INFO | "huge_shared" : " 27 GB", | 20260507 00:25:27.707 | INFO | "malloc" : " 14 GB", | 20260507 00:25:27.707 | INFO | "pinned" : " 29 GB" | 20260507 00:25:27.707 | INFO | } | 20260507 00:25:27.707 | INFO | Perf report for the last 5.13494 seconds: | 20260507 00:25:27.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:25:27.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:27.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:27.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:27.707 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:25:27.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:27.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:27.707 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:25:27.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:27.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:27.708 | INFO | Uploaded 36GiB to GPU in 1.97949s (18.1865GiB/s) | 20260507 00:25:27.708 | INFO | Downloaded 34GiB to GPU in 1.91282s (17.7748GiB/s) | 20260507 00:25:32.847 | INFO | Analyzing chunk frames = [5642240,5642752) | 20260507 00:25:32.847 | INFO | MemInfo: { | 20260507 00:25:32.847 | INFO | "gpu" : " 17 GB", | 20260507 00:25:32.847 | INFO | "huge" : "", | 20260507 00:25:32.847 | INFO | "huge_shared" : " 27 GB", | 20260507 00:25:32.847 | INFO | "malloc" : " 14 GB", | 20260507 00:25:32.847 | INFO | "pinned" : " 29 GB" | 20260507 00:25:32.847 | INFO | } | 20260507 00:25:32.847 | INFO | Perf report for the last 5.14001 seconds: | 20260507 00:25:32.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:25:32.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:32.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:32.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:32.847 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 00:25:32.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:32.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:32.847 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 00:25:32.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:32.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:32.848 | INFO | Uploaded 36GiB to GPU in 2.00265s (17.9762GiB/s) | 20260507 00:25:32.848 | INFO | Downloaded 34GiB to GPU in 1.92393s (17.6721GiB/s) | 20260507 00:25:37.993 | INFO | Analyzing chunk frames = [5642752,5643264) | 20260507 00:25:37.993 | INFO | MemInfo: { | 20260507 00:25:37.993 | INFO | "gpu" : " 17 GB", | 20260507 00:25:37.993 | INFO | "huge" : "", | 20260507 00:25:37.993 | INFO | "huge_shared" : " 27 GB", | 20260507 00:25:37.993 | INFO | "malloc" : " 14 GB", | 20260507 00:25:37.993 | INFO | "pinned" : " 29 GB" | 20260507 00:25:37.993 | INFO | } | 20260507 00:25:37.994 | INFO | Perf report for the last 5.14621 seconds: | 20260507 00:25:37.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:25:37.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:37.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:37.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:37.994 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 00:25:37.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:37.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:37.994 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 00:25:37.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:37.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:37.994 | INFO | Uploaded 36GiB to GPU in 2.01689s (17.8492GiB/s) | 20260507 00:25:37.994 | INFO | Downloaded 34GiB to GPU in 1.94784s (17.4552GiB/s) | 20260507 00:25:43.135 | INFO | Analyzing chunk frames = [5643264,5643776) | 20260507 00:25:43.135 | INFO | MemInfo: { | 20260507 00:25:43.135 | INFO | "gpu" : " 17 GB", | 20260507 00:25:43.135 | INFO | "huge" : "", | 20260507 00:25:43.135 | INFO | "huge_shared" : " 27 GB", | 20260507 00:25:43.135 | INFO | "malloc" : " 14 GB", | 20260507 00:25:43.135 | INFO | "pinned" : " 29 GB" | 20260507 00:25:43.135 | INFO | } | 20260507 00:25:43.135 | INFO | Perf report for the last 5.14135 seconds: | 20260507 00:25:43.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:25:43.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:43.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:43.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:43.135 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:25:43.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:43.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:43.135 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 00:25:43.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:43.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:43.135 | INFO | Uploaded 36GiB to GPU in 1.97355s (18.2413GiB/s) | 20260507 00:25:43.135 | INFO | Downloaded 34GiB to GPU in 1.98301s (17.1457GiB/s) | 20260507 00:25:48.278 | INFO | Analyzing chunk frames = [5643776,5644288) | 20260507 00:25:48.278 | INFO | MemInfo: { | 20260507 00:25:48.278 | INFO | "gpu" : " 17 GB", | 20260507 00:25:48.278 | INFO | "huge" : "", | 20260507 00:25:48.278 | INFO | "huge_shared" : " 27 GB", | 20260507 00:25:48.278 | INFO | "malloc" : " 14 GB", | 20260507 00:25:48.278 | INFO | "pinned" : " 29 GB" | 20260507 00:25:48.278 | INFO | } | 20260507 00:25:48.278 | INFO | Perf report for the last 5.14329 seconds: | 20260507 00:25:48.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:25:48.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:48.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:48.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:48.278 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:25:48.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:48.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:48.278 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 00:25:48.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:48.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:48.279 | INFO | Uploaded 36GiB to GPU in 1.9928s (18.065GiB/s) | 20260507 00:25:48.279 | INFO | Downloaded 34GiB to GPU in 1.97776s (17.1912GiB/s) | 20260507 00:25:48.279 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:25:48.279 | INFO | Total mem usage: 18 GiB | 20260507 00:25:48.279 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:25:48.279 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 00:25:48.279 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:25:48.279 | INFO | High Water Breakdown: | 20260507 00:25:48.279 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:25:48.279 | INFO | 39.7% : 5.67 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:25:48.279 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:25:48.279 | INFO | ------------------------------------------ | 20260507 00:25:48.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:25:48.279 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:25:48.279 | INFO | Total mem usage: 26.5 GiB | 20260507 00:25:48.279 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:25:48.279 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:25:48.279 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:25:48.279 | INFO | High Water Breakdown: | 20260507 00:25:48.279 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:25:48.279 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:25:48.279 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:25:48.279 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:25:48.279 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:25:48.279 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:25:48.279 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:25:48.279 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:25:48.279 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:25:48.279 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:25:48.279 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:25:48.279 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:25:48.279 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:25:48.279 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:25:48.279 | INFO | ------------------------------------------ | 20260507 00:25:48.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:25:48.279 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:25:48.279 | INFO | Total mem usage: 25.6 GiB | 20260507 00:25:48.279 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:25:48.279 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:25:48.279 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:25:48.279 | INFO | High Water Breakdown: | 20260507 00:25:48.279 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:25:48.279 | INFO | ------------------------------------------ | 20260507 00:25:48.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:25:48.279 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:25:48.279 | INFO | Total mem usage: 15.7 GiB | 20260507 00:25:48.279 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:25:48.279 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:25:48.279 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:25:48.279 | INFO | High Water Breakdown: | 20260507 00:25:48.279 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:25:48.279 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:25:48.279 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:25:48.279 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:25:48.279 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:25:48.279 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:25:48.279 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:25:48.279 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:25:48.279 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:25:48.279 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:25:48.279 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:25:48.280 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:25:48.280 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:25:48.280 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:25:48.280 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:25:48.280 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:25:48.280 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:25:48.280 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:25:48.280 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:25:48.280 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:25:48.280 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:25:48.280 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:25:48.280 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:25:48.280 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:25:48.280 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:25:48.280 | INFO | ------------------------------------------ | 20260507 00:25:48.280 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:25:48.280 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:25:48.280 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:25:48.280 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:25:48.280 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.59e+01, 1.33e+01 / 2.27e+00 | 20260507 00:25:48.280 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.03e-01, 1.03e-02 / 4.54e-03 | 20260507 00:25:48.280 | INFO | PerfReport: 11.67%, Baseline -- 3.61e+00 / 4.50e+00, 5.39e-01 / 2.21e-01 | 20260507 00:25:48.280 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.49e+00, 1.75e-01 / 1.03e-01 | 20260507 00:25:48.280 | INFO | PerfReport: 16.09%, DME -- 3.98e+01 / 4.46e+01, 4.08e+00 / 2.86e+00 | 20260507 00:25:48.280 | INFO | PerfReport: 57.43%, FrameLabeling -- 1.77e+01 / 2.08e+01, 1.33e+00 / 1.82e+00 | 20260507 00:25:48.280 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.18e+00 / 2.71e+00, 3.79e-01 / 1.97e-01 | 20260507 00:25:48.280 | INFO | PerfReport: 3.16%, Metrics -- 9.75e-01 / 1.30e+00, 2.60e-01 / 1.52e-01 | 20260507 00:25:48.280 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.31e-01 / 3.82e-01, 5.25e-02 / 2.34e-02 | 20260507 00:25:48.280 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:25:48.280 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:25:48.280 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:25:48.280 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 00:25:48.280 | INFO | PerfReport: 21.56%, Upload -- 2.08e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260507 00:25:48.280 | INFO | PerfReport: 21.29%, Download -- 2.05e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260507 00:25:48.280 | INFO | PerfReport: 32.03%, Compute -- 3.09e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 00:25:48.280 | INFO | PerfReport: 1.31%, LockUpload -- 1.26e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 00:25:48.280 | INFO | PerfReport: 1.33%, LockDownload -- 1.28e+00 / 0.00e+00, 3.17e+00 / 0.00e+00 | 20260507 00:25:48.280 | INFO | PerfReport: 22.47%, LockCompute -- 2.17e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 00:25:48.280 | INFO | PerfReport: 0.01%, Misc -- 1.45e-02 / 0.00e+00, 2.37e-02 / 0.00e+00 | 20260507 00:25:48.376 | INFO | Starting a round of Baz IO | 20260507 00:25:51.497 | INFO | Populating RTMetrics report | 20260507 00:25:51.497 | INFO | RTMetrics report populated | 20260507 00:25:51.497 | INFO | Opening tmp RTMetrics json file | 20260507 00:25:51.498 | INFO | Serializing RTMetrics to json | 20260507 00:25:51.505 | INFO | Serialization done, swapping tmp file now... | 20260507 00:25:51.518 | INFO | RTMetrics done | 20260507 00:25:51.591 | INFO | Finished a round of Baz IO: Wrote 1.847 GiB in 3.214 seconds | 20260507 00:25:51.591 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.29% / 0.00% / 66.71% | 20260507 00:25:51.591 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.345151% / 1.19209e-05% / 7.15256e-05% | 20260507 00:25:51.591 | INFO | Baz IO overall: | 20260507 00:25:51.591 | INFO | BytesWritten : 2942358066427 | 20260507 00:25:51.591 | INFO | NumEvents : 1417460717900 | 20260507 00:25:53.406 | INFO | Analyzing chunk frames = [5644288,5644800) | 20260507 00:25:53.406 | INFO | MemInfo: { | 20260507 00:25:53.406 | INFO | "gpu" : " 17 GB", | 20260507 00:25:53.406 | INFO | "huge" : "", | 20260507 00:25:53.406 | INFO | "huge_shared" : " 27 GB", | 20260507 00:25:53.406 | INFO | "malloc" : " 14 GB", | 20260507 00:25:53.406 | INFO | "pinned" : " 29 GB" | 20260507 00:25:53.406 | INFO | } | 20260507 00:25:53.406 | INFO | Perf report for the last 5.12785 seconds: | 20260507 00:25:53.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:25:53.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:53.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:53.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:53.406 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260507 00:25:53.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:53.406 | INFO | RT_METRICS : 42.5%, 1.0, 22.2ms, 0 | 20260507 00:25:53.406 | INFO | PRE_HQ : 59.8%, 1.8, 56.3ms, 0 | 20260507 00:25:53.406 | INFO | BAZWRITER : 62.3%, 3.4, 108.1ms, 0 | 20260507 00:25:53.406 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:25:53.406 | INFO | Uploaded 36GiB to GPU in 2.0331s (17.707GiB/s) | 20260507 00:25:53.407 | INFO | Downloaded 38GiB to GPU in 2.18163s (17.4181GiB/s) | 20260507 00:25:58.547 | INFO | Analyzing chunk frames = [5644800,5645312) | 20260507 00:25:58.547 | INFO | MemInfo: { | 20260507 00:25:58.547 | INFO | "gpu" : " 17 GB", | 20260507 00:25:58.547 | INFO | "huge" : "", | 20260507 00:25:58.547 | INFO | "huge_shared" : " 27 GB", | 20260507 00:25:58.547 | INFO | "malloc" : " 14 GB", | 20260507 00:25:58.547 | INFO | "pinned" : " 29 GB" | 20260507 00:25:58.547 | INFO | } | 20260507 00:25:58.547 | INFO | Perf report for the last 5.14074 seconds: | 20260507 00:25:58.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:25:58.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:58.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:58.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:58.547 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:25:58.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:58.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:58.547 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:25:58.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:58.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:25:58.547 | INFO | Uploaded 36GiB to GPU in 2.00222s (17.98GiB/s) | 20260507 00:25:58.547 | INFO | Downloaded 34GiB to GPU in 1.93317s (17.5877GiB/s) | 20260507 00:26:03.688 | INFO | Analyzing chunk frames = [5645312,5645824) | 20260507 00:26:03.688 | INFO | MemInfo: { | 20260507 00:26:03.688 | INFO | "gpu" : " 17 GB", | 20260507 00:26:03.688 | INFO | "huge" : "", | 20260507 00:26:03.688 | INFO | "huge_shared" : " 27 GB", | 20260507 00:26:03.688 | INFO | "malloc" : " 14 GB", | 20260507 00:26:03.688 | INFO | "pinned" : " 29 GB" | 20260507 00:26:03.688 | INFO | } | 20260507 00:26:03.688 | INFO | Perf report for the last 5.14092 seconds: | 20260507 00:26:03.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:26:03.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:03.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:03.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:03.688 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:26:03.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:03.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:03.688 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 00:26:03.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:03.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:03.688 | INFO | Uploaded 36GiB to GPU in 1.96293s (18.3399GiB/s) | 20260507 00:26:03.688 | INFO | Downloaded 34GiB to GPU in 1.93473s (17.5735GiB/s) | 20260507 00:26:08.826 | INFO | Analyzing chunk frames = [5645824,5646336) | 20260507 00:26:08.827 | INFO | MemInfo: { | 20260507 00:26:08.827 | INFO | "gpu" : " 17 GB", | 20260507 00:26:08.827 | INFO | "huge" : "", | 20260507 00:26:08.827 | INFO | "huge_shared" : " 27 GB", | 20260507 00:26:08.827 | INFO | "malloc" : " 14 GB", | 20260507 00:26:08.827 | INFO | "pinned" : " 29 GB" | 20260507 00:26:08.827 | INFO | } | 20260507 00:26:08.827 | INFO | Perf report for the last 5.13904 seconds: | 20260507 00:26:08.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:26:08.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:08.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:08.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:08.827 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 00:26:08.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:08.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:08.827 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:26:08.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:08.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:08.827 | INFO | Uploaded 36GiB to GPU in 1.98951s (18.0949GiB/s) | 20260507 00:26:08.827 | INFO | Downloaded 34GiB to GPU in 1.91338s (17.7696GiB/s) | 20260507 00:26:13.964 | INFO | Analyzing chunk frames = [5646336,5646848) | 20260507 00:26:13.964 | INFO | MemInfo: { | 20260507 00:26:13.964 | INFO | "gpu" : " 17 GB", | 20260507 00:26:13.964 | INFO | "huge" : "", | 20260507 00:26:13.964 | INFO | "huge_shared" : " 27 GB", | 20260507 00:26:13.964 | INFO | "malloc" : " 14 GB", | 20260507 00:26:13.964 | INFO | "pinned" : " 29 GB" | 20260507 00:26:13.964 | INFO | } | 20260507 00:26:13.964 | INFO | Perf report for the last 5.13764 seconds: | 20260507 00:26:13.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:26:13.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:13.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:13.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:13.965 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 00:26:13.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:13.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:13.965 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:26:13.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:13.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:13.965 | INFO | Uploaded 36GiB to GPU in 1.97686s (18.2107GiB/s) | 20260507 00:26:13.965 | INFO | Downloaded 34GiB to GPU in 1.93087s (17.6086GiB/s) | 20260507 00:26:19.106 | INFO | Analyzing chunk frames = [5646848,5647360) | 20260507 00:26:19.106 | INFO | MemInfo: { | 20260507 00:26:19.106 | INFO | "gpu" : " 17 GB", | 20260507 00:26:19.106 | INFO | "huge" : "", | 20260507 00:26:19.106 | INFO | "huge_shared" : " 27 GB", | 20260507 00:26:19.106 | INFO | "malloc" : " 14 GB", | 20260507 00:26:19.106 | INFO | "pinned" : " 29 GB" | 20260507 00:26:19.106 | INFO | } | 20260507 00:26:19.106 | INFO | Perf report for the last 5.14206 seconds: | 20260507 00:26:19.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:26:19.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:19.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:19.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:19.107 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:26:19.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:19.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:19.107 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 00:26:19.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:19.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:19.107 | INFO | Uploaded 36GiB to GPU in 2.00695s (17.9377GiB/s) | 20260507 00:26:19.107 | INFO | Downloaded 34GiB to GPU in 1.95401s (17.4001GiB/s) | 20260507 00:26:24.253 | INFO | Analyzing chunk frames = [5647360,5647872) | 20260507 00:26:24.253 | INFO | MemInfo: { | 20260507 00:26:24.253 | INFO | "gpu" : " 17 GB", | 20260507 00:26:24.253 | INFO | "huge" : "", | 20260507 00:26:24.253 | INFO | "huge_shared" : " 27 GB", | 20260507 00:26:24.253 | INFO | "malloc" : " 14 GB", | 20260507 00:26:24.253 | INFO | "pinned" : " 29 GB" | 20260507 00:26:24.253 | INFO | } | 20260507 00:26:24.253 | INFO | Perf report for the last 5.14613 seconds: | 20260507 00:26:24.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:26:24.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:24.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:24.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:24.253 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 00:26:24.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:24.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:24.253 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:26:24.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:24.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:24.253 | INFO | Uploaded 36GiB to GPU in 1.95927s (18.3742GiB/s) | 20260507 00:26:24.253 | INFO | Downloaded 34GiB to GPU in 1.99078s (17.0787GiB/s) | 20260507 00:26:29.387 | INFO | Analyzing chunk frames = [5647872,5648384) | 20260507 00:26:29.387 | INFO | MemInfo: { | 20260507 00:26:29.387 | INFO | "gpu" : " 17 GB", | 20260507 00:26:29.387 | INFO | "huge" : "", | 20260507 00:26:29.387 | INFO | "huge_shared" : " 27 GB", | 20260507 00:26:29.387 | INFO | "malloc" : " 14 GB", | 20260507 00:26:29.387 | INFO | "pinned" : " 29 GB" | 20260507 00:26:29.387 | INFO | } | 20260507 00:26:29.387 | INFO | Perf report for the last 5.13469 seconds: | 20260507 00:26:29.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:26:29.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:29.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:29.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:29.387 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260507 00:26:29.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:29.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:29.387 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:26:29.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:29.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:29.388 | INFO | Uploaded 36GiB to GPU in 2.00474s (17.9574GiB/s) | 20260507 00:26:29.388 | INFO | Downloaded 34GiB to GPU in 1.97937s (17.1771GiB/s) | 20260507 00:26:29.486 | INFO | Starting a round of Baz IO | 20260507 00:26:32.605 | INFO | Populating RTMetrics report | 20260507 00:26:32.606 | INFO | RTMetrics report populated | 20260507 00:26:32.606 | INFO | Opening tmp RTMetrics json file | 20260507 00:26:32.606 | INFO | Serializing RTMetrics to json | 20260507 00:26:32.608 | INFO | Serialization done, swapping tmp file now... | 20260507 00:26:32.608 | INFO | RTMetrics done | 20260507 00:26:32.695 | INFO | Finished a round of Baz IO: Wrote 1.845 GiB in 3.209 seconds | 20260507 00:26:32.695 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.29% / 0.00% / 66.71% | 20260507 00:26:32.695 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.344197% / 7.94729e-06% / 5.96046e-05% | 20260507 00:26:32.695 | INFO | Baz IO overall: | 20260507 00:26:32.695 | INFO | BytesWritten : 2944339032199 | 20260507 00:26:32.695 | INFO | NumEvents : 1418401277400 | 20260507 00:26:34.520 | INFO | Analyzing chunk frames = [5648384,5648896) | 20260507 00:26:34.520 | INFO | MemInfo: { | 20260507 00:26:34.520 | INFO | "gpu" : " 17 GB", | 20260507 00:26:34.520 | INFO | "huge" : "", | 20260507 00:26:34.521 | INFO | "huge_shared" : " 27 GB", | 20260507 00:26:34.521 | INFO | "malloc" : " 14 GB", | 20260507 00:26:34.521 | INFO | "pinned" : " 29 GB" | 20260507 00:26:34.521 | INFO | } | 20260507 00:26:34.521 | INFO | Perf report for the last 5.13325 seconds: | 20260507 00:26:34.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:26:34.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:34.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:34.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:34.521 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260507 00:26:34.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:34.521 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 00:26:34.521 | INFO | PRE_HQ : 59.3%, 1.8, 56.3ms, 0 | 20260507 00:26:34.521 | INFO | BAZWRITER : 62.1%, 3.4, 107.8ms, 0 | 20260507 00:26:34.521 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 00:26:34.521 | INFO | Uploaded 36GiB to GPU in 2.01989s (17.8228GiB/s) | 20260507 00:26:34.521 | INFO | Downloaded 38GiB to GPU in 2.16177s (17.5782GiB/s) | 20260507 00:26:39.663 | INFO | Analyzing chunk frames = [5648896,5649408) | 20260507 00:26:39.663 | INFO | MemInfo: { | 20260507 00:26:39.663 | INFO | "gpu" : " 17 GB", | 20260507 00:26:39.663 | INFO | "huge" : "", | 20260507 00:26:39.663 | INFO | "huge_shared" : " 27 GB", | 20260507 00:26:39.663 | INFO | "malloc" : " 14 GB", | 20260507 00:26:39.663 | INFO | "pinned" : " 29 GB" | 20260507 00:26:39.663 | INFO | } | 20260507 00:26:39.663 | INFO | Perf report for the last 5.14289 seconds: | 20260507 00:26:39.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:26:39.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:39.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:39.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:39.664 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 00:26:39.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:39.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:39.664 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 00:26:39.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:39.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:39.664 | INFO | Uploaded 36GiB to GPU in 1.99799s (18.0181GiB/s) | 20260507 00:26:39.664 | INFO | Downloaded 34GiB to GPU in 1.92489s (17.6633GiB/s) | 20260507 00:26:44.797 | INFO | Analyzing chunk frames = [5649408,5649920) | 20260507 00:26:44.797 | INFO | MemInfo: { | 20260507 00:26:44.797 | INFO | "gpu" : " 17 GB", | 20260507 00:26:44.797 | INFO | "huge" : "", | 20260507 00:26:44.797 | INFO | "huge_shared" : " 27 GB", | 20260507 00:26:44.797 | INFO | "malloc" : " 14 GB", | 20260507 00:26:44.798 | INFO | "pinned" : " 29 GB" | 20260507 00:26:44.798 | INFO | } | 20260507 00:26:44.798 | INFO | Perf report for the last 5.13405 seconds: | 20260507 00:26:44.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:26:44.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:44.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:44.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:44.798 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 00:26:44.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:44.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:44.798 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:26:44.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:44.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:44.798 | INFO | Uploaded 36GiB to GPU in 1.99367s (18.0572GiB/s) | 20260507 00:26:44.798 | INFO | Downloaded 34GiB to GPU in 1.93683s (17.5545GiB/s) | 20260507 00:26:49.947 | INFO | Analyzing chunk frames = [5649920,5650432) | 20260507 00:26:49.947 | INFO | MemInfo: { | 20260507 00:26:49.947 | INFO | "gpu" : " 17 GB", | 20260507 00:26:49.947 | INFO | "huge" : "", | 20260507 00:26:49.947 | INFO | "huge_shared" : " 27 GB", | 20260507 00:26:49.947 | INFO | "malloc" : " 14 GB", | 20260507 00:26:49.947 | INFO | "pinned" : " 29 GB" | 20260507 00:26:49.947 | INFO | } | 20260507 00:26:49.947 | INFO | Perf report for the last 5.14976 seconds: | 20260507 00:26:49.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:26:49.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:49.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:49.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:49.947 | INFO | ANALYSIS : 59.4%, 3.0, 92.3ms, 0 | 20260507 00:26:49.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:49.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:49.947 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:26:49.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:49.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:49.948 | INFO | Uploaded 36GiB to GPU in 1.97733s (18.2064GiB/s) | 20260507 00:26:49.948 | INFO | Downloaded 34GiB to GPU in 1.87975s (18.0875GiB/s) | 20260507 00:26:55.085 | INFO | Analyzing chunk frames = [5650432,5650944) | 20260507 00:26:55.085 | INFO | MemInfo: { | 20260507 00:26:55.085 | INFO | "gpu" : " 17 GB", | 20260507 00:26:55.085 | INFO | "huge" : "", | 20260507 00:26:55.085 | INFO | "huge_shared" : " 27 GB", | 20260507 00:26:55.085 | INFO | "malloc" : " 14 GB", | 20260507 00:26:55.085 | INFO | "pinned" : " 29 GB" | 20260507 00:26:55.085 | INFO | } | 20260507 00:26:55.085 | INFO | Perf report for the last 5.13807 seconds: | 20260507 00:26:55.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:26:55.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:55.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:55.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:55.085 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 00:26:55.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:55.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:55.086 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:26:55.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:55.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:26:55.086 | INFO | Uploaded 36GiB to GPU in 2.0094s (17.9158GiB/s) | 20260507 00:26:55.086 | INFO | Downloaded 34GiB to GPU in 1.94223s (17.5057GiB/s) | 20260507 00:27:00.221 | INFO | Analyzing chunk frames = [5650944,5651456) | 20260507 00:27:00.221 | INFO | MemInfo: { | 20260507 00:27:00.221 | INFO | "gpu" : " 17 GB", | 20260507 00:27:00.221 | INFO | "huge" : "", | 20260507 00:27:00.222 | INFO | "huge_shared" : " 27 GB", | 20260507 00:27:00.222 | INFO | "malloc" : " 14 GB", | 20260507 00:27:00.222 | INFO | "pinned" : " 29 GB" | 20260507 00:27:00.222 | INFO | } | 20260507 00:27:00.222 | INFO | Perf report for the last 5.13629 seconds: | 20260507 00:27:00.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:27:00.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:00.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:00.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:00.222 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 00:27:00.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:00.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:00.222 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 00:27:00.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:00.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:00.222 | INFO | Uploaded 36GiB to GPU in 2.05618s (17.5082GiB/s) | 20260507 00:27:00.222 | INFO | Downloaded 34GiB to GPU in 1.95405s (17.3998GiB/s) | 20260507 00:27:05.366 | INFO | Analyzing chunk frames = [5651456,5651968) | 20260507 00:27:05.366 | INFO | MemInfo: { | 20260507 00:27:05.366 | INFO | "gpu" : " 17 GB", | 20260507 00:27:05.366 | INFO | "huge" : "", | 20260507 00:27:05.366 | INFO | "huge_shared" : " 27 GB", | 20260507 00:27:05.366 | INFO | "malloc" : " 14 GB", | 20260507 00:27:05.366 | INFO | "pinned" : " 29 GB" | 20260507 00:27:05.366 | INFO | } | 20260507 00:27:05.366 | INFO | Perf report for the last 5.14475 seconds: | 20260507 00:27:05.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:27:05.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:05.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:05.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:05.367 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:27:05.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:05.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:05.367 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 00:27:05.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:05.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:05.367 | INFO | Uploaded 36GiB to GPU in 1.98496s (18.1364GiB/s) | 20260507 00:27:05.367 | INFO | Downloaded 34GiB to GPU in 2.00254s (16.9784GiB/s) | 20260507 00:27:10.500 | INFO | Analyzing chunk frames = [5651968,5652480) | 20260507 00:27:10.500 | INFO | MemInfo: { | 20260507 00:27:10.500 | INFO | "gpu" : " 17 GB", | 20260507 00:27:10.500 | INFO | "huge" : "", | 20260507 00:27:10.500 | INFO | "huge_shared" : " 27 GB", | 20260507 00:27:10.500 | INFO | "malloc" : " 14 GB", | 20260507 00:27:10.500 | INFO | "pinned" : " 29 GB" | 20260507 00:27:10.500 | INFO | } | 20260507 00:27:10.501 | INFO | Perf report for the last 5.13403 seconds: | 20260507 00:27:10.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:27:10.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:10.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:10.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:10.501 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:27:10.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:10.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:10.501 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:27:10.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:10.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:10.501 | INFO | Uploaded 36GiB to GPU in 1.98468s (18.1389GiB/s) | 20260507 00:27:10.501 | INFO | Downloaded 34GiB to GPU in 1.97841s (17.1855GiB/s) | 20260507 00:27:10.501 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:27:10.501 | INFO | Total mem usage: 18 GiB | 20260507 00:27:10.501 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:27:10.501 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 00:27:10.501 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:27:10.501 | INFO | High Water Breakdown: | 20260507 00:27:10.501 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:27:10.501 | INFO | 39.7% : 5.67 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:27:10.501 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:27:10.501 | INFO | ------------------------------------------ | 20260507 00:27:10.501 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:27:10.501 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:27:10.501 | INFO | Total mem usage: 26.5 GiB | 20260507 00:27:10.501 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:27:10.501 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:27:10.501 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:27:10.501 | INFO | High Water Breakdown: | 20260507 00:27:10.501 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:27:10.501 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:27:10.501 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:27:10.501 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:27:10.501 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:27:10.501 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:27:10.501 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:27:10.501 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:27:10.501 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:27:10.501 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:27:10.501 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:27:10.501 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:27:10.501 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:27:10.501 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:27:10.501 | INFO | ------------------------------------------ | 20260507 00:27:10.501 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:27:10.501 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:27:10.501 | INFO | Total mem usage: 25.6 GiB | 20260507 00:27:10.501 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:27:10.501 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:27:10.502 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:27:10.502 | INFO | High Water Breakdown: | 20260507 00:27:10.502 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:27:10.502 | INFO | ------------------------------------------ | 20260507 00:27:10.502 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:27:10.502 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:27:10.502 | INFO | Total mem usage: 15.7 GiB | 20260507 00:27:10.502 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:27:10.502 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:27:10.502 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 00:27:10.502 | INFO | High Water Breakdown: | 20260507 00:27:10.502 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:27:10.502 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:27:10.502 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:27:10.502 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:27:10.502 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:27:10.502 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:27:10.502 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:27:10.502 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:27:10.502 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:27:10.502 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:27:10.502 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:27:10.502 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:27:10.502 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:27:10.502 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:27:10.502 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:27:10.502 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:27:10.502 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:27:10.502 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:27:10.502 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:27:10.502 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:27:10.502 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:27:10.502 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:27:10.502 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:27:10.502 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:27:10.502 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:27:10.502 | INFO | ------------------------------------------ | 20260507 00:27:10.502 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:27:10.502 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:27:10.502 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:27:10.502 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:27:10.502 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.59e+01, 1.36e+01 / 2.58e+00 | 20260507 00:27:10.502 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 9.92e-02, 9.83e-03 / 5.06e-03 | 20260507 00:27:10.502 | INFO | PerfReport: 11.66%, Baseline -- 3.61e+00 / 4.32e+00, 5.35e-01 / 3.46e-01 | 20260507 00:27:10.502 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.43e+00, 1.73e-01 / 1.38e-01 | 20260507 00:27:10.502 | INFO | PerfReport: 16.14%, DME -- 3.99e+01 / 4.51e+01, 4.30e+00 / 2.86e+00 | 20260507 00:27:10.502 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.32e+00 / 1.59e+00 | 20260507 00:27:10.502 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.18e+00 / 2.69e+00, 3.83e-01 / 2.19e-01 | 20260507 00:27:10.502 | INFO | PerfReport: 3.15%, Metrics -- 9.73e-01 / 1.28e+00, 2.58e-01 / 1.30e-01 | 20260507 00:27:10.502 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.31e-01 / 3.75e-01, 5.17e-02 / 1.27e-02 | 20260507 00:27:10.502 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:27:10.502 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:27:10.502 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:27:10.502 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 00:27:10.502 | INFO | PerfReport: 21.56%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260507 00:27:10.502 | INFO | PerfReport: 21.32%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 00:27:10.502 | INFO | PerfReport: 32.05%, Compute -- 3.09e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260507 00:27:10.502 | INFO | PerfReport: 1.32%, LockUpload -- 1.27e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 00:27:10.502 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260507 00:27:10.502 | INFO | PerfReport: 22.44%, LockCompute -- 2.17e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 00:27:10.502 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.43e-02 / 0.00e+00 | 20260507 00:27:10.598 | INFO | Starting a round of Baz IO | 20260507 00:27:13.718 | INFO | Populating RTMetrics report | 20260507 00:27:13.718 | INFO | RTMetrics report populated | 20260507 00:27:13.718 | INFO | Opening tmp RTMetrics json file | 20260507 00:27:13.719 | INFO | Serializing RTMetrics to json | 20260507 00:27:13.720 | INFO | Serialization done, swapping tmp file now... | 20260507 00:27:13.721 | INFO | RTMetrics done | 20260507 00:27:13.814 | INFO | Finished a round of Baz IO: Wrote 1.848 GiB in 3.215 seconds | 20260507 00:27:13.814 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.29% / 0.00% / 66.71% | 20260507 00:27:13.814 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.344209% / 3.97364e-06% / 7.15256e-05% | 20260507 00:27:13.814 | INFO | Baz IO overall: | 20260507 00:27:13.814 | INFO | BytesWritten : 2946323260805 | 20260507 00:27:13.814 | INFO | NumEvents : 1419344559021 | 20260507 00:27:15.637 | INFO | Analyzing chunk frames = [5652480,5652992) | 20260507 00:27:15.637 | INFO | MemInfo: { | 20260507 00:27:15.637 | INFO | "gpu" : " 17 GB", | 20260507 00:27:15.637 | INFO | "huge" : "", | 20260507 00:27:15.637 | INFO | "huge_shared" : " 27 GB", | 20260507 00:27:15.637 | INFO | "malloc" : " 14 GB", | 20260507 00:27:15.637 | INFO | "pinned" : " 29 GB" | 20260507 00:27:15.637 | INFO | } | 20260507 00:27:15.637 | INFO | Perf report for the last 5.13628 seconds: | 20260507 00:27:15.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:27:15.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:15.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:15.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:15.637 | INFO | ANALYSIS : 62.1%, 3.0, 96.4ms, 0 | 20260507 00:27:15.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:15.637 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 00:27:15.637 | INFO | PRE_HQ : 59.4%, 1.8, 56.1ms, 0 | 20260507 00:27:15.637 | INFO | BAZWRITER : 62.2%, 3.4, 109.4ms, 0 | 20260507 00:27:15.637 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:27:15.637 | INFO | Uploaded 36GiB to GPU in 2.02419s (17.7849GiB/s) | 20260507 00:27:15.637 | INFO | Downloaded 38GiB to GPU in 2.18609s (17.3827GiB/s) | 20260507 00:27:20.777 | INFO | Analyzing chunk frames = [5652992,5653504) | 20260507 00:27:20.777 | INFO | MemInfo: { | 20260507 00:27:20.777 | INFO | "gpu" : " 17 GB", | 20260507 00:27:20.777 | INFO | "huge" : "", | 20260507 00:27:20.777 | INFO | "huge_shared" : " 27 GB", | 20260507 00:27:20.777 | INFO | "malloc" : " 14 GB", | 20260507 00:27:20.777 | INFO | "pinned" : " 29 GB" | 20260507 00:27:20.777 | INFO | } | 20260507 00:27:20.777 | INFO | Perf report for the last 5.1402 seconds: | 20260507 00:27:20.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:27:20.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:20.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:20.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:20.777 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 00:27:20.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:20.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:20.777 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:27:20.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:20.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:20.777 | INFO | Uploaded 36GiB to GPU in 1.98406s (18.1446GiB/s) | 20260507 00:27:20.777 | INFO | Downloaded 34GiB to GPU in 1.92881s (17.6275GiB/s) | 20260507 00:27:25.916 | INFO | Analyzing chunk frames = [5653504,5654016) | 20260507 00:27:25.916 | INFO | MemInfo: { | 20260507 00:27:25.916 | INFO | "gpu" : " 17 GB", | 20260507 00:27:25.916 | INFO | "huge" : "", | 20260507 00:27:25.916 | INFO | "huge_shared" : " 27 GB", | 20260507 00:27:25.916 | INFO | "malloc" : " 14 GB", | 20260507 00:27:25.916 | INFO | "pinned" : " 29 GB" | 20260507 00:27:25.916 | INFO | } | 20260507 00:27:25.916 | INFO | Perf report for the last 5.13923 seconds: | 20260507 00:27:25.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:27:25.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:25.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:25.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:25.916 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:27:25.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:25.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:25.916 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:27:25.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:25.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:25.917 | INFO | Uploaded 36GiB to GPU in 1.97337s (18.2429GiB/s) | 20260507 00:27:25.917 | INFO | Downloaded 34GiB to GPU in 1.93239s (17.5948GiB/s) | 20260507 00:27:31.063 | INFO | Analyzing chunk frames = [5654016,5654528) | 20260507 00:27:31.063 | INFO | MemInfo: { | 20260507 00:27:31.063 | INFO | "gpu" : " 17 GB", | 20260507 00:27:31.063 | INFO | "huge" : "", | 20260507 00:27:31.063 | INFO | "huge_shared" : " 27 GB", | 20260507 00:27:31.063 | INFO | "malloc" : " 14 GB", | 20260507 00:27:31.063 | INFO | "pinned" : " 29 GB" | 20260507 00:27:31.063 | INFO | } | 20260507 00:27:31.063 | INFO | Perf report for the last 5.14654 seconds: | 20260507 00:27:31.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:27:31.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:31.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:31.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:31.063 | INFO | ANALYSIS : 59.9%, 3.0, 93.1ms, 0 | 20260507 00:27:31.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:31.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:31.063 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:27:31.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:31.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:31.063 | INFO | Uploaded 36GiB to GPU in 1.99844s (18.0141GiB/s) | 20260507 00:27:31.063 | INFO | Downloaded 34GiB to GPU in 1.90975s (17.8034GiB/s) | 20260507 00:27:36.198 | INFO | Analyzing chunk frames = [5654528,5655040) | 20260507 00:27:36.198 | INFO | MemInfo: { | 20260507 00:27:36.198 | INFO | "gpu" : " 17 GB", | 20260507 00:27:36.198 | INFO | "huge" : "", | 20260507 00:27:36.199 | INFO | "huge_shared" : " 27 GB", | 20260507 00:27:36.199 | INFO | "malloc" : " 14 GB", | 20260507 00:27:36.199 | INFO | "pinned" : " 29 GB" | 20260507 00:27:36.199 | INFO | } | 20260507 00:27:36.199 | INFO | Perf report for the last 5.13597 seconds: | 20260507 00:27:36.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:27:36.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:36.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:36.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:36.199 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 00:27:36.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:36.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:36.199 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:27:36.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:36.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:36.199 | INFO | Uploaded 36GiB to GPU in 1.98716s (18.1163GiB/s) | 20260507 00:27:36.199 | INFO | Downloaded 34GiB to GPU in 1.95113s (17.4258GiB/s) | 20260507 00:27:41.338 | INFO | Analyzing chunk frames = [5655040,5655552) | 20260507 00:27:41.338 | INFO | MemInfo: { | 20260507 00:27:41.338 | INFO | "gpu" : " 17 GB", | 20260507 00:27:41.338 | INFO | "huge" : "", | 20260507 00:27:41.338 | INFO | "huge_shared" : " 27 GB", | 20260507 00:27:41.338 | INFO | "malloc" : " 14 GB", | 20260507 00:27:41.338 | INFO | "pinned" : " 29 GB" | 20260507 00:27:41.338 | INFO | } | 20260507 00:27:41.338 | INFO | Perf report for the last 5.13952 seconds: | 20260507 00:27:41.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:27:41.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:41.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:41.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:41.339 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 00:27:41.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:41.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:41.339 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 00:27:41.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:41.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:41.339 | INFO | Uploaded 36GiB to GPU in 2.04849s (17.5739GiB/s) | 20260507 00:27:41.339 | INFO | Downloaded 34GiB to GPU in 1.95929s (17.3532GiB/s) | 20260507 00:27:46.472 | INFO | Analyzing chunk frames = [5655552,5656064) | 20260507 00:27:46.472 | INFO | MemInfo: { | 20260507 00:27:46.472 | INFO | "gpu" : " 17 GB", | 20260507 00:27:46.472 | INFO | "huge" : "", | 20260507 00:27:46.472 | INFO | "huge_shared" : " 27 GB", | 20260507 00:27:46.472 | INFO | "malloc" : " 14 GB", | 20260507 00:27:46.472 | INFO | "pinned" : " 29 GB" | 20260507 00:27:46.472 | INFO | } | 20260507 00:27:46.472 | INFO | Perf report for the last 5.134 seconds: | 20260507 00:27:46.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:27:46.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:46.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:46.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:46.472 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:27:46.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:46.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:46.472 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 00:27:46.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:46.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:46.473 | INFO | Uploaded 36GiB to GPU in 1.98194s (18.164GiB/s) | 20260507 00:27:46.473 | INFO | Downloaded 34GiB to GPU in 1.98465s (17.1315GiB/s) | 20260507 00:27:51.615 | INFO | Analyzing chunk frames = [5656064,5656576) | 20260507 00:27:51.615 | INFO | MemInfo: { | 20260507 00:27:51.615 | INFO | "gpu" : " 17 GB", | 20260507 00:27:51.615 | INFO | "huge" : "", | 20260507 00:27:51.615 | INFO | "huge_shared" : " 27 GB", | 20260507 00:27:51.615 | INFO | "malloc" : " 14 GB", | 20260507 00:27:51.615 | INFO | "pinned" : " 29 GB" | 20260507 00:27:51.615 | INFO | } | 20260507 00:27:51.615 | INFO | Perf report for the last 5.14247 seconds: | 20260507 00:27:51.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:27:51.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:51.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:51.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:51.615 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:27:51.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:51.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:51.615 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:27:51.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:51.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:51.615 | INFO | Uploaded 36GiB to GPU in 1.9881s (18.1077GiB/s) | 20260507 00:27:51.615 | INFO | Downloaded 34GiB to GPU in 1.98691s (17.112GiB/s) | 20260507 00:27:51.712 | INFO | Starting a round of Baz IO | 20260507 00:27:54.833 | INFO | Populating RTMetrics report | 20260507 00:27:54.834 | INFO | RTMetrics report populated | 20260507 00:27:54.834 | INFO | Opening tmp RTMetrics json file | 20260507 00:27:54.835 | INFO | Serializing RTMetrics to json | 20260507 00:27:54.836 | INFO | Serialization done, swapping tmp file now... | 20260507 00:27:54.836 | INFO | RTMetrics done | 20260507 00:27:54.925 | INFO | Finished a round of Baz IO: Wrote 1.846 GiB in 3.213 seconds | 20260507 00:27:54.925 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.29% / 0.00% / 66.71% | 20260507 00:27:54.925 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.34368% / 7.94729e-06% / 7.15256e-05% | 20260507 00:27:54.925 | INFO | Baz IO overall: | 20260507 00:27:54.926 | INFO | BytesWritten : 2948305738778 | 20260507 00:27:54.926 | INFO | NumEvents : 1420286327649 | 20260507 00:27:56.751 | INFO | Analyzing chunk frames = [5656576,5657088) | 20260507 00:27:56.751 | INFO | MemInfo: { | 20260507 00:27:56.751 | INFO | "gpu" : " 17 GB", | 20260507 00:27:56.751 | INFO | "huge" : "", | 20260507 00:27:56.751 | INFO | "huge_shared" : " 27 GB", | 20260507 00:27:56.751 | INFO | "malloc" : " 14 GB", | 20260507 00:27:56.751 | INFO | "pinned" : " 29 GB" | 20260507 00:27:56.751 | INFO | } | 20260507 00:27:56.751 | INFO | Perf report for the last 5.13657 seconds: | 20260507 00:27:56.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:27:56.751 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:56.751 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:56.751 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:56.751 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260507 00:27:56.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:27:56.752 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 00:27:56.752 | INFO | PRE_HQ : 59.4%, 1.8, 56.1ms, 0 | 20260507 00:27:56.752 | INFO | BAZWRITER : 62.0%, 3.4, 109.8ms, 0 | 20260507 00:27:56.752 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:27:56.752 | INFO | Uploaded 36GiB to GPU in 2.01259s (17.8874GiB/s) | 20260507 00:27:56.752 | INFO | Downloaded 38GiB to GPU in 2.17584s (17.4646GiB/s) | 20260507 00:28:01.892 | INFO | Analyzing chunk frames = [5657088,5657600) | 20260507 00:28:01.892 | INFO | MemInfo: { | 20260507 00:28:01.892 | INFO | "gpu" : " 17 GB", | 20260507 00:28:01.892 | INFO | "huge" : "", | 20260507 00:28:01.892 | INFO | "huge_shared" : " 27 GB", | 20260507 00:28:01.892 | INFO | "malloc" : " 14 GB", | 20260507 00:28:01.892 | INFO | "pinned" : " 29 GB" | 20260507 00:28:01.892 | INFO | } | 20260507 00:28:01.892 | INFO | Perf report for the last 5.14079 seconds: | 20260507 00:28:01.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:28:01.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:01.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:01.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:01.892 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:28:01.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:01.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:01.892 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:28:01.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:01.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:01.893 | INFO | Uploaded 36GiB to GPU in 2.00798s (17.9285GiB/s) | 20260507 00:28:01.893 | INFO | Downloaded 34GiB to GPU in 1.92839s (17.6313GiB/s) | 20260507 00:28:07.041 | INFO | Analyzing chunk frames = [5657600,5658112) | 20260507 00:28:07.041 | INFO | MemInfo: { | 20260507 00:28:07.041 | INFO | "gpu" : " 17 GB", | 20260507 00:28:07.041 | INFO | "huge" : "", | 20260507 00:28:07.041 | INFO | "huge_shared" : " 27 GB", | 20260507 00:28:07.041 | INFO | "malloc" : " 14 GB", | 20260507 00:28:07.041 | INFO | "pinned" : " 29 GB" | 20260507 00:28:07.041 | INFO | } | 20260507 00:28:07.041 | INFO | Perf report for the last 5.14895 seconds: | 20260507 00:28:07.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:28:07.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:07.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:07.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:07.041 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 00:28:07.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:07.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:07.041 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:28:07.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:07.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:07.041 | INFO | Uploaded 36GiB to GPU in 1.99051s (18.0858GiB/s) | 20260507 00:28:07.041 | INFO | Downloaded 34GiB to GPU in 1.9295s (17.6212GiB/s) | 20260507 00:28:12.175 | INFO | Analyzing chunk frames = [5658112,5658624) | 20260507 00:28:12.175 | INFO | MemInfo: { | 20260507 00:28:12.175 | INFO | "gpu" : " 17 GB", | 20260507 00:28:12.175 | INFO | "huge" : "", | 20260507 00:28:12.175 | INFO | "huge_shared" : " 27 GB", | 20260507 00:28:12.175 | INFO | "malloc" : " 14 GB", | 20260507 00:28:12.175 | INFO | "pinned" : " 29 GB" | 20260507 00:28:12.175 | INFO | } | 20260507 00:28:12.175 | INFO | Perf report for the last 5.13414 seconds: | 20260507 00:28:12.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:28:12.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:12.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:12.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:12.175 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:28:12.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:12.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:12.175 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:28:12.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:12.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:12.176 | INFO | Uploaded 36GiB to GPU in 1.98042s (18.1779GiB/s) | 20260507 00:28:12.176 | INFO | Downloaded 34GiB to GPU in 1.91098s (17.792GiB/s) | 20260507 00:28:17.319 | INFO | Analyzing chunk frames = [5658624,5659136) | 20260507 00:28:17.319 | INFO | MemInfo: { | 20260507 00:28:17.319 | INFO | "gpu" : " 17 GB", | 20260507 00:28:17.319 | INFO | "huge" : "", | 20260507 00:28:17.319 | INFO | "huge_shared" : " 27 GB", | 20260507 00:28:17.319 | INFO | "malloc" : " 14 GB", | 20260507 00:28:17.319 | INFO | "pinned" : " 29 GB" | 20260507 00:28:17.319 | INFO | } | 20260507 00:28:17.319 | INFO | Perf report for the last 5.14346 seconds: | 20260507 00:28:17.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:28:17.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:17.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:17.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:17.319 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 00:28:17.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:17.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:17.319 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:28:17.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:17.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:17.319 | INFO | Uploaded 36GiB to GPU in 2.01025s (17.9082GiB/s) | 20260507 00:28:17.319 | INFO | Downloaded 34GiB to GPU in 1.94201s (17.5076GiB/s) | 20260507 00:28:22.455 | INFO | Analyzing chunk frames = [5659136,5659648) | 20260507 00:28:22.455 | INFO | MemInfo: { | 20260507 00:28:22.455 | INFO | "gpu" : " 17 GB", | 20260507 00:28:22.455 | INFO | "huge" : "", | 20260507 00:28:22.456 | INFO | "huge_shared" : " 27 GB", | 20260507 00:28:22.456 | INFO | "malloc" : " 14 GB", | 20260507 00:28:22.456 | INFO | "pinned" : " 29 GB" | 20260507 00:28:22.456 | INFO | } | 20260507 00:28:22.456 | INFO | Perf report for the last 5.13682 seconds: | 20260507 00:28:22.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:28:22.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:22.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:22.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:22.456 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 00:28:22.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:22.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:22.456 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 00:28:22.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:22.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:22.456 | INFO | Uploaded 36GiB to GPU in 2.05897s (17.4845GiB/s) | 20260507 00:28:22.456 | INFO | Downloaded 34GiB to GPU in 1.95787s (17.3658GiB/s) | 20260507 00:28:27.585 | INFO | Analyzing chunk frames = [5659648,5660160) | 20260507 00:28:27.585 | INFO | MemInfo: { | 20260507 00:28:27.585 | INFO | "gpu" : " 17 GB", | 20260507 00:28:27.585 | INFO | "huge" : "", | 20260507 00:28:27.585 | INFO | "huge_shared" : " 27 GB", | 20260507 00:28:27.585 | INFO | "malloc" : " 14 GB", | 20260507 00:28:27.585 | INFO | "pinned" : " 29 GB" | 20260507 00:28:27.585 | INFO | } | 20260507 00:28:27.585 | INFO | Perf report for the last 5.12961 seconds: | 20260507 00:28:27.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:28:27.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:27.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:27.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:27.585 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 00:28:27.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:27.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:27.585 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:28:27.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:27.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:27.585 | INFO | Uploaded 36GiB to GPU in 1.9989s (18.0099GiB/s) | 20260507 00:28:27.586 | INFO | Downloaded 34GiB to GPU in 1.9835s (17.1414GiB/s) | 20260507 00:28:32.726 | INFO | Analyzing chunk frames = [5660160,5660672) | 20260507 00:28:32.726 | INFO | MemInfo: { | 20260507 00:28:32.726 | INFO | "gpu" : " 17 GB", | 20260507 00:28:32.726 | INFO | "huge" : "", | 20260507 00:28:32.726 | INFO | "huge_shared" : " 27 GB", | 20260507 00:28:32.726 | INFO | "malloc" : " 14 GB", | 20260507 00:28:32.726 | INFO | "pinned" : " 29 GB" | 20260507 00:28:32.726 | INFO | } | 20260507 00:28:32.726 | INFO | Perf report for the last 5.14079 seconds: | 20260507 00:28:32.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:28:32.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:32.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:32.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:32.726 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:28:32.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:32.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:32.726 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:28:32.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:32.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:32.726 | INFO | Uploaded 36GiB to GPU in 1.98126s (18.1703GiB/s) | 20260507 00:28:32.726 | INFO | Downloaded 34GiB to GPU in 1.99239s (17.065GiB/s) | 20260507 00:28:32.727 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:28:32.727 | INFO | Total mem usage: 18 GiB | 20260507 00:28:32.727 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:28:32.727 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:28:32.727 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:28:32.727 | INFO | High Water Breakdown: | 20260507 00:28:32.727 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:28:32.727 | INFO | 39.7% : 5.67 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:28:32.727 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:28:32.727 | INFO | ------------------------------------------ | 20260507 00:28:32.727 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:28:32.727 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:28:32.727 | INFO | Total mem usage: 26.5 GiB | 20260507 00:28:32.727 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:28:32.727 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:28:32.727 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:28:32.727 | INFO | High Water Breakdown: | 20260507 00:28:32.727 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:28:32.727 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:28:32.727 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:28:32.727 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:28:32.727 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:28:32.727 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:28:32.727 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:28:32.727 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:28:32.727 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:28:32.727 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:28:32.727 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:28:32.727 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:28:32.727 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:28:32.727 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:28:32.727 | INFO | ------------------------------------------ | 20260507 00:28:32.727 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:28:32.727 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:28:32.727 | INFO | Total mem usage: 25.6 GiB | 20260507 00:28:32.727 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:28:32.727 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:28:32.727 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:28:32.727 | INFO | High Water Breakdown: | 20260507 00:28:32.727 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:28:32.727 | INFO | ------------------------------------------ | 20260507 00:28:32.727 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:28:32.727 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:28:32.727 | INFO | Total mem usage: 15.7 GiB | 20260507 00:28:32.727 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:28:32.727 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:28:32.727 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 00:28:32.727 | INFO | High Water Breakdown: | 20260507 00:28:32.727 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:28:32.727 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:28:32.727 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:28:32.727 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:28:32.727 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:28:32.727 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:28:32.727 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:28:32.727 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:28:32.727 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:28:32.727 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:28:32.727 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:28:32.727 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:28:32.727 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:28:32.727 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:28:32.727 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:28:32.727 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:28:32.727 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:28:32.727 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:28:32.727 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:28:32.727 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:28:32.727 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:28:32.727 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:28:32.728 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:28:32.728 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:28:32.728 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:28:32.728 | INFO | ------------------------------------------ | 20260507 00:28:32.728 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:28:32.728 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:28:32.728 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:28:32.728 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:28:32.728 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.54e+01, 1.33e+01 / 1.88e+00 | 20260507 00:28:32.728 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.97e-03 / 7.49e-03 | 20260507 00:28:32.728 | INFO | PerfReport: 11.64%, Baseline -- 3.60e+00 / 4.39e+00, 5.25e-01 / 3.45e-01 | 20260507 00:28:32.728 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.45e+00, 1.74e-01 / 1.35e-01 | 20260507 00:28:32.728 | INFO | PerfReport: 16.01%, DME -- 3.96e+01 / 4.39e+01, 3.90e+00 / 2.27e+00 | 20260507 00:28:32.728 | INFO | PerfReport: 57.55%, FrameLabeling -- 1.78e+01 / 2.11e+01, 1.40e+00 / 1.79e+00 | 20260507 00:28:32.728 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.17e+00 / 2.68e+00, 3.78e-01 / 2.58e-01 | 20260507 00:28:32.728 | INFO | PerfReport: 3.15%, Metrics -- 9.76e-01 / 1.31e+00, 2.59e-01 / 1.36e-01 | 20260507 00:28:32.728 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.32e-01 / 3.87e-01, 5.21e-02 / 2.82e-02 | 20260507 00:28:32.728 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:28:32.728 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:28:32.728 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:28:32.728 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 00:28:32.728 | INFO | PerfReport: 21.60%, Upload -- 2.09e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260507 00:28:32.728 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 00:28:32.728 | INFO | PerfReport: 32.05%, Compute -- 3.09e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 00:28:32.728 | INFO | PerfReport: 1.33%, LockUpload -- 1.28e+00 / 0.00e+00, 4.79e+00 / 0.00e+00 | 20260507 00:28:32.728 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260507 00:28:32.728 | INFO | PerfReport: 22.38%, LockCompute -- 2.16e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 00:28:32.728 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260507 00:28:32.823 | INFO | Starting a round of Baz IO | 20260507 00:28:35.947 | INFO | Populating RTMetrics report | 20260507 00:28:35.947 | INFO | RTMetrics report populated | 20260507 00:28:35.947 | INFO | Opening tmp RTMetrics json file | 20260507 00:28:35.966 | INFO | Serializing RTMetrics to json | 20260507 00:28:35.967 | INFO | Serialization done, swapping tmp file now... | 20260507 00:28:35.968 | INFO | RTMetrics done | 20260507 00:28:36.038 | INFO | Finished a round of Baz IO: Wrote 1.847 GiB in 3.215 seconds | 20260507 00:28:36.038 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.28% / 0.00% / 66.71% | 20260507 00:28:36.038 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.344396% / 7.94729e-06% / 5.96046e-05% | 20260507 00:28:36.039 | INFO | Baz IO overall: | 20260507 00:28:36.039 | INFO | BytesWritten : 2950289078041 | 20260507 00:28:36.039 | INFO | NumEvents : 1421229169805 | 20260507 00:28:37.874 | INFO | Analyzing chunk frames = [5660672,5661184) | 20260507 00:28:37.874 | INFO | MemInfo: { | 20260507 00:28:37.874 | INFO | "gpu" : " 17 GB", | 20260507 00:28:37.874 | INFO | "huge" : "", | 20260507 00:28:37.874 | INFO | "huge_shared" : " 27 GB", | 20260507 00:28:37.874 | INFO | "malloc" : " 14 GB", | 20260507 00:28:37.874 | INFO | "pinned" : " 29 GB" | 20260507 00:28:37.874 | INFO | } | 20260507 00:28:37.874 | INFO | Perf report for the last 5.14825 seconds: | 20260507 00:28:37.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:28:37.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:37.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:37.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:37.874 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260507 00:28:37.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:37.874 | INFO | RT_METRICS : 42.2%, 1.0, 22.2ms, 0 | 20260507 00:28:37.874 | INFO | PRE_HQ : 59.4%, 1.8, 55.8ms, 0 | 20260507 00:28:37.874 | INFO | BAZWRITER : 62.1%, 3.4, 108.3ms, 0 | 20260507 00:28:37.874 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:28:37.875 | INFO | Uploaded 36GiB to GPU in 2.00812s (17.9272GiB/s) | 20260507 00:28:37.875 | INFO | Downloaded 38GiB to GPU in 2.1867s (17.3778GiB/s) | 20260507 00:28:43.007 | INFO | Analyzing chunk frames = [5661184,5661696) | 20260507 00:28:43.007 | INFO | MemInfo: { | 20260507 00:28:43.007 | INFO | "gpu" : " 17 GB", | 20260507 00:28:43.008 | INFO | "huge" : "", | 20260507 00:28:43.008 | INFO | "huge_shared" : " 27 GB", | 20260507 00:28:43.008 | INFO | "malloc" : " 14 GB", | 20260507 00:28:43.008 | INFO | "pinned" : " 29 GB" | 20260507 00:28:43.008 | INFO | } | 20260507 00:28:43.008 | INFO | Perf report for the last 5.13359 seconds: | 20260507 00:28:43.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:28:43.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:43.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:43.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:43.008 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:28:43.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:43.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:43.008 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:28:43.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:43.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:43.008 | INFO | Uploaded 36GiB to GPU in 2.00264s (17.9762GiB/s) | 20260507 00:28:43.008 | INFO | Downloaded 34GiB to GPU in 1.93311s (17.5882GiB/s) | 20260507 00:28:48.147 | INFO | Analyzing chunk frames = [5661696,5662208) | 20260507 00:28:48.147 | INFO | MemInfo: { | 20260507 00:28:48.147 | INFO | "gpu" : " 17 GB", | 20260507 00:28:48.147 | INFO | "huge" : "", | 20260507 00:28:48.147 | INFO | "huge_shared" : " 27 GB", | 20260507 00:28:48.147 | INFO | "malloc" : " 14 GB", | 20260507 00:28:48.147 | INFO | "pinned" : " 29 GB" | 20260507 00:28:48.147 | INFO | } | 20260507 00:28:48.147 | INFO | Perf report for the last 5.13887 seconds: | 20260507 00:28:48.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:28:48.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:48.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:48.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:48.147 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:28:48.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:48.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:48.147 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:28:48.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:48.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:48.147 | INFO | Uploaded 36GiB to GPU in 1.98649s (18.1224GiB/s) | 20260507 00:28:48.147 | INFO | Downloaded 34GiB to GPU in 1.9369s (17.5538GiB/s) | 20260507 00:28:53.288 | INFO | Analyzing chunk frames = [5662208,5662720) | 20260507 00:28:53.288 | INFO | MemInfo: { | 20260507 00:28:53.288 | INFO | "gpu" : " 17 GB", | 20260507 00:28:53.288 | INFO | "huge" : "", | 20260507 00:28:53.288 | INFO | "huge_shared" : " 27 GB", | 20260507 00:28:53.288 | INFO | "malloc" : " 14 GB", | 20260507 00:28:53.288 | INFO | "pinned" : " 29 GB" | 20260507 00:28:53.288 | INFO | } | 20260507 00:28:53.288 | INFO | Perf report for the last 5.14157 seconds: | 20260507 00:28:53.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:28:53.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:53.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:53.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:53.288 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260507 00:28:53.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:53.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:53.288 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:28:53.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:53.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:53.289 | INFO | Uploaded 36GiB to GPU in 1.97368s (18.24GiB/s) | 20260507 00:28:53.289 | INFO | Downloaded 34GiB to GPU in 1.89909s (17.9033GiB/s) | 20260507 00:28:58.426 | INFO | Analyzing chunk frames = [5662720,5663232) | 20260507 00:28:58.426 | INFO | MemInfo: { | 20260507 00:28:58.426 | INFO | "gpu" : " 17 GB", | 20260507 00:28:58.426 | INFO | "huge" : "", | 20260507 00:28:58.426 | INFO | "huge_shared" : " 27 GB", | 20260507 00:28:58.426 | INFO | "malloc" : " 14 GB", | 20260507 00:28:58.426 | INFO | "pinned" : " 29 GB" | 20260507 00:28:58.426 | INFO | } | 20260507 00:28:58.426 | INFO | Perf report for the last 5.13738 seconds: | 20260507 00:28:58.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:28:58.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:58.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:58.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:58.426 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 00:28:58.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:58.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:58.426 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:28:58.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:58.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:28:58.426 | INFO | Uploaded 36GiB to GPU in 1.98324s (18.1521GiB/s) | 20260507 00:28:58.426 | INFO | Downloaded 34GiB to GPU in 1.93956s (17.5297GiB/s) | 20260507 00:29:03.563 | INFO | Analyzing chunk frames = [5663232,5663744) | 20260507 00:29:03.563 | INFO | MemInfo: { | 20260507 00:29:03.563 | INFO | "gpu" : " 17 GB", | 20260507 00:29:03.563 | INFO | "huge" : "", | 20260507 00:29:03.563 | INFO | "huge_shared" : " 27 GB", | 20260507 00:29:03.563 | INFO | "malloc" : " 14 GB", | 20260507 00:29:03.563 | INFO | "pinned" : " 29 GB" | 20260507 00:29:03.563 | INFO | } | 20260507 00:29:03.563 | INFO | Perf report for the last 5.13741 seconds: | 20260507 00:29:03.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:29:03.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:03.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:03.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:03.563 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 00:29:03.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:03.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:03.563 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260507 00:29:03.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:03.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:03.564 | INFO | Uploaded 36GiB to GPU in 2.05029s (17.5585GiB/s) | 20260507 00:29:03.564 | INFO | Downloaded 34GiB to GPU in 1.96476s (17.3049GiB/s) | 20260507 00:29:08.703 | INFO | Analyzing chunk frames = [5663744,5664256) | 20260507 00:29:08.703 | INFO | MemInfo: { | 20260507 00:29:08.703 | INFO | "gpu" : " 17 GB", | 20260507 00:29:08.703 | INFO | "huge" : "", | 20260507 00:29:08.703 | INFO | "huge_shared" : " 27 GB", | 20260507 00:29:08.703 | INFO | "malloc" : " 14 GB", | 20260507 00:29:08.703 | INFO | "pinned" : " 29 GB" | 20260507 00:29:08.703 | INFO | } | 20260507 00:29:08.703 | INFO | Perf report for the last 5.13994 seconds: | 20260507 00:29:08.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:29:08.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:08.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:08.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:08.703 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:29:08.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:08.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:08.703 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 00:29:08.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:08.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:08.703 | INFO | Uploaded 36GiB to GPU in 1.96289s (18.3403GiB/s) | 20260507 00:29:08.703 | INFO | Downloaded 34GiB to GPU in 1.98973s (17.0878GiB/s) | 20260507 00:29:13.848 | INFO | Analyzing chunk frames = [5664256,5664768) | 20260507 00:29:13.848 | INFO | MemInfo: { | 20260507 00:29:13.848 | INFO | "gpu" : " 17 GB", | 20260507 00:29:13.848 | INFO | "huge" : "", | 20260507 00:29:13.848 | INFO | "huge_shared" : " 27 GB", | 20260507 00:29:13.848 | INFO | "malloc" : " 14 GB", | 20260507 00:29:13.848 | INFO | "pinned" : " 29 GB" | 20260507 00:29:13.848 | INFO | } | 20260507 00:29:13.848 | INFO | Perf report for the last 5.14458 seconds: | 20260507 00:29:13.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:29:13.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:13.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:13.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:13.848 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 00:29:13.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:13.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:13.848 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 00:29:13.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:13.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:13.848 | INFO | Uploaded 36GiB to GPU in 1.99225s (18.07GiB/s) | 20260507 00:29:13.848 | INFO | Downloaded 34GiB to GPU in 1.98811s (17.1017GiB/s) | 20260507 00:29:13.946 | INFO | Starting a round of Baz IO | 20260507 00:29:17.063 | INFO | Populating RTMetrics report | 20260507 00:29:17.063 | INFO | RTMetrics report populated | 20260507 00:29:17.063 | INFO | Opening tmp RTMetrics json file | 20260507 00:29:17.065 | INFO | Serializing RTMetrics to json | 20260507 00:29:17.083 | INFO | Serialization done, swapping tmp file now... | 20260507 00:29:17.084 | INFO | RTMetrics done | 20260507 00:29:17.159 | INFO | Finished a round of Baz IO: Wrote 1.845 GiB in 3.213 seconds | 20260507 00:29:17.159 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.28% / 0.00% / 66.72% | 20260507 00:29:17.159 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.342127% / 7.94729e-06% / 7.15256e-05% | 20260507 00:29:17.159 | INFO | Baz IO overall: | 20260507 00:29:17.159 | INFO | BytesWritten : 2952270176143 | 20260507 00:29:17.159 | INFO | NumEvents : 1422170313020 | 20260507 00:29:18.986 | INFO | Analyzing chunk frames = [5664768,5665280) | 20260507 00:29:18.986 | INFO | MemInfo: { | 20260507 00:29:18.986 | INFO | "gpu" : " 17 GB", | 20260507 00:29:18.986 | INFO | "huge" : "", | 20260507 00:29:18.986 | INFO | "huge_shared" : " 27 GB", | 20260507 00:29:18.986 | INFO | "malloc" : " 14 GB", | 20260507 00:29:18.986 | INFO | "pinned" : " 29 GB" | 20260507 00:29:18.986 | INFO | } | 20260507 00:29:18.986 | INFO | Perf report for the last 5.13843 seconds: | 20260507 00:29:18.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:29:18.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:18.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:18.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:18.986 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 00:29:18.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:18.986 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 00:29:18.986 | INFO | PRE_HQ : 59.4%, 1.8, 56.2ms, 0 | 20260507 00:29:18.986 | INFO | BAZWRITER : 62.0%, 3.4, 109.2ms, 0 | 20260507 00:29:18.986 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:29:18.987 | INFO | Uploaded 36GiB to GPU in 2.02859s (17.7463GiB/s) | 20260507 00:29:18.987 | INFO | Downloaded 38GiB to GPU in 2.18621s (17.3817GiB/s) | 20260507 00:29:24.133 | INFO | Analyzing chunk frames = [5665280,5665792) | 20260507 00:29:24.133 | INFO | MemInfo: { | 20260507 00:29:24.133 | INFO | "gpu" : " 17 GB", | 20260507 00:29:24.133 | INFO | "huge" : "", | 20260507 00:29:24.133 | INFO | "huge_shared" : " 27 GB", | 20260507 00:29:24.133 | INFO | "malloc" : " 14 GB", | 20260507 00:29:24.133 | INFO | "pinned" : " 29 GB" | 20260507 00:29:24.133 | INFO | } | 20260507 00:29:24.133 | INFO | Perf report for the last 5.14682 seconds: | 20260507 00:29:24.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:29:24.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:24.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:24.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:24.133 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 00:29:24.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:24.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:24.133 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:29:24.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:24.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:24.133 | INFO | Uploaded 36GiB to GPU in 1.99732s (18.0241GiB/s) | 20260507 00:29:24.133 | INFO | Downloaded 34GiB to GPU in 1.92398s (17.6717GiB/s) | 20260507 00:29:29.264 | INFO | Analyzing chunk frames = [5665792,5666304) | 20260507 00:29:29.264 | INFO | MemInfo: { | 20260507 00:29:29.264 | INFO | "gpu" : " 17 GB", | 20260507 00:29:29.264 | INFO | "huge" : "", | 20260507 00:29:29.264 | INFO | "huge_shared" : " 27 GB", | 20260507 00:29:29.264 | INFO | "malloc" : " 14 GB", | 20260507 00:29:29.264 | INFO | "pinned" : " 29 GB" | 20260507 00:29:29.264 | INFO | } | 20260507 00:29:29.264 | INFO | Perf report for the last 5.13131 seconds: | 20260507 00:29:29.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:29:29.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:29.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:29.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:29.264 | INFO | ANALYSIS : 60.6%, 3.0, 93.6ms, 0 | 20260507 00:29:29.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:29.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:29.264 | INFO | PRE_HQ : 25.6%, 1.0, 13.2ms, 0 | 20260507 00:29:29.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:29.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:29.265 | INFO | Uploaded 36GiB to GPU in 2.0075s (17.9327GiB/s) | 20260507 00:29:29.265 | INFO | Downloaded 34GiB to GPU in 1.93288s (17.5904GiB/s) | 20260507 00:29:34.399 | INFO | Analyzing chunk frames = [5666304,5666816) | 20260507 00:29:34.399 | INFO | MemInfo: { | 20260507 00:29:34.399 | INFO | "gpu" : " 17 GB", | 20260507 00:29:34.399 | INFO | "huge" : "", | 20260507 00:29:34.399 | INFO | "huge_shared" : " 27 GB", | 20260507 00:29:34.399 | INFO | "malloc" : " 14 GB", | 20260507 00:29:34.399 | INFO | "pinned" : " 29 GB" | 20260507 00:29:34.399 | INFO | } | 20260507 00:29:34.399 | INFO | Perf report for the last 5.13463 seconds: | 20260507 00:29:34.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:29:34.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:34.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:34.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:34.399 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 00:29:34.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:34.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:34.399 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:29:34.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:34.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:34.399 | INFO | Uploaded 36GiB to GPU in 2.00762s (17.9317GiB/s) | 20260507 00:29:34.399 | INFO | Downloaded 34GiB to GPU in 1.91071s (17.7944GiB/s) | 20260507 00:29:39.538 | INFO | Analyzing chunk frames = [5666816,5667328) | 20260507 00:29:39.538 | INFO | MemInfo: { | 20260507 00:29:39.538 | INFO | "gpu" : " 17 GB", | 20260507 00:29:39.538 | INFO | "huge" : "", | 20260507 00:29:39.538 | INFO | "huge_shared" : " 27 GB", | 20260507 00:29:39.538 | INFO | "malloc" : " 14 GB", | 20260507 00:29:39.538 | INFO | "pinned" : " 29 GB" | 20260507 00:29:39.538 | INFO | } | 20260507 00:29:39.538 | INFO | Perf report for the last 5.13936 seconds: | 20260507 00:29:39.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:29:39.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:39.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:39.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:39.538 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 00:29:39.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:39.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:39.538 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:29:39.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:39.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:39.539 | INFO | Uploaded 36GiB to GPU in 1.99329s (18.0606GiB/s) | 20260507 00:29:39.539 | INFO | Downloaded 34GiB to GPU in 1.93406s (17.5796GiB/s) | 20260507 00:29:44.678 | INFO | Analyzing chunk frames = [5667328,5667840) | 20260507 00:29:44.678 | INFO | MemInfo: { | 20260507 00:29:44.678 | INFO | "gpu" : " 17 GB", | 20260507 00:29:44.678 | INFO | "huge" : "", | 20260507 00:29:44.678 | INFO | "huge_shared" : " 27 GB", | 20260507 00:29:44.678 | INFO | "malloc" : " 14 GB", | 20260507 00:29:44.678 | INFO | "pinned" : " 29 GB" | 20260507 00:29:44.678 | INFO | } | 20260507 00:29:44.678 | INFO | Perf report for the last 5.13969 seconds: | 20260507 00:29:44.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:29:44.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:44.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:44.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:44.678 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:29:44.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:44.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:44.678 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 00:29:44.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:44.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:44.678 | INFO | Uploaded 36GiB to GPU in 2.02548s (17.7736GiB/s) | 20260507 00:29:44.678 | INFO | Downloaded 34GiB to GPU in 1.94134s (17.5137GiB/s) | 20260507 00:29:49.819 | INFO | Analyzing chunk frames = [5667840,5668352) | 20260507 00:29:49.819 | INFO | MemInfo: { | 20260507 00:29:49.819 | INFO | "gpu" : " 17 GB", | 20260507 00:29:49.819 | INFO | "huge" : "", | 20260507 00:29:49.819 | INFO | "huge_shared" : " 27 GB", | 20260507 00:29:49.819 | INFO | "malloc" : " 14 GB", | 20260507 00:29:49.819 | INFO | "pinned" : " 29 GB" | 20260507 00:29:49.819 | INFO | } | 20260507 00:29:49.819 | INFO | Perf report for the last 5.14129 seconds: | 20260507 00:29:49.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:29:49.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:49.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:49.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:49.819 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 00:29:49.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:49.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:49.819 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:29:49.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:49.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:49.820 | INFO | Uploaded 36GiB to GPU in 1.97071s (18.2675GiB/s) | 20260507 00:29:49.820 | INFO | Downloaded 34GiB to GPU in 1.99747s (17.0215GiB/s) | 20260507 00:29:54.958 | INFO | Analyzing chunk frames = [5668352,5668864) | 20260507 00:29:54.958 | INFO | MemInfo: { | 20260507 00:29:54.958 | INFO | "gpu" : " 17 GB", | 20260507 00:29:54.958 | INFO | "huge" : "", | 20260507 00:29:54.958 | INFO | "huge_shared" : " 27 GB", | 20260507 00:29:54.958 | INFO | "malloc" : " 14 GB", | 20260507 00:29:54.958 | INFO | "pinned" : " 29 GB" | 20260507 00:29:54.958 | INFO | } | 20260507 00:29:54.958 | INFO | Perf report for the last 5.13859 seconds: | 20260507 00:29:54.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:29:54.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:54.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:54.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:54.958 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:29:54.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:54.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:54.958 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:29:54.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:54.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:29:54.958 | INFO | Uploaded 36GiB to GPU in 1.98324s (18.1521GiB/s) | 20260507 00:29:54.958 | INFO | Downloaded 34GiB to GPU in 1.98245s (17.1505GiB/s) | 20260507 00:29:54.959 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:29:54.959 | INFO | Total mem usage: 18 GiB | 20260507 00:29:54.959 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:29:54.959 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 00:29:54.959 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:29:54.959 | INFO | High Water Breakdown: | 20260507 00:29:54.959 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:29:54.959 | INFO | 39.7% : 5.67 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:29:54.959 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:29:54.959 | INFO | ------------------------------------------ | 20260507 00:29:54.959 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:29:54.959 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:29:54.959 | INFO | Total mem usage: 26.5 GiB | 20260507 00:29:54.959 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:29:54.959 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:29:54.959 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:29:54.959 | INFO | High Water Breakdown: | 20260507 00:29:54.959 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:29:54.959 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:29:54.959 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:29:54.959 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:29:54.959 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:29:54.959 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:29:54.959 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:29:54.959 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:29:54.959 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:29:54.959 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:29:54.959 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:29:54.959 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:29:54.959 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:29:54.959 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:29:54.959 | INFO | ------------------------------------------ | 20260507 00:29:54.959 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:29:54.959 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:29:54.959 | INFO | Total mem usage: 25.6 GiB | 20260507 00:29:54.959 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:29:54.959 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:29:54.959 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:29:54.959 | INFO | High Water Breakdown: | 20260507 00:29:54.959 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:29:54.959 | INFO | ------------------------------------------ | 20260507 00:29:54.959 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:29:54.959 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:29:54.959 | INFO | Total mem usage: 15.7 GiB | 20260507 00:29:54.959 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:29:54.959 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:29:54.959 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:29:54.959 | INFO | High Water Breakdown: | 20260507 00:29:54.959 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:29:54.959 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:29:54.959 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:29:54.959 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:29:54.959 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:29:54.959 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:29:54.959 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:29:54.959 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:29:54.959 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:29:54.959 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:29:54.959 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:29:54.959 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:29:54.960 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:29:54.960 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:29:54.960 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:29:54.960 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:29:54.960 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:29:54.960 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:29:54.960 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:29:54.960 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:29:54.960 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:29:54.960 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:29:54.960 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:29:54.960 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:29:54.960 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:29:54.960 | INFO | ------------------------------------------ | 20260507 00:29:54.960 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:29:54.960 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:29:54.960 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:29:54.960 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:29:54.960 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.55e+01, 1.33e+01 / 2.21e+00 | 20260507 00:29:54.960 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.01e-01, 9.14e-03 / 4.81e-03 | 20260507 00:29:54.960 | INFO | PerfReport: 11.66%, Baseline -- 3.60e+00 / 4.39e+00, 5.30e-01 / 2.97e-01 | 20260507 00:29:54.960 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.45e+00, 1.72e-01 / 1.17e-01 | 20260507 00:29:54.960 | INFO | PerfReport: 16.08%, DME -- 3.97e+01 / 4.45e+01, 3.96e+00 / 2.50e+00 | 20260507 00:29:54.960 | INFO | PerfReport: 57.42%, FrameLabeling -- 1.77e+01 / 2.06e+01, 1.26e+00 / 1.34e+00 | 20260507 00:29:54.960 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.18e+00 / 2.75e+00, 3.85e-01 / 1.90e-01 | 20260507 00:29:54.960 | INFO | PerfReport: 3.16%, Metrics -- 9.75e-01 / 1.31e+00, 2.58e-01 / 1.27e-01 | 20260507 00:29:54.960 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.31e-01 / 3.84e-01, 5.22e-02 / 1.44e-02 | 20260507 00:29:54.960 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:29:54.960 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:29:54.960 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:29:54.960 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260507 00:29:54.960 | INFO | PerfReport: 21.58%, Upload -- 2.08e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260507 00:29:54.960 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 00:29:54.960 | INFO | PerfReport: 32.01%, Compute -- 3.09e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 00:29:54.960 | INFO | PerfReport: 1.32%, LockUpload -- 1.28e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 00:29:54.960 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.15e+00 / 0.00e+00 | 20260507 00:29:54.960 | INFO | PerfReport: 22.38%, LockCompute -- 2.16e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 00:29:54.960 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.48e-02 / 0.00e+00 | 20260507 00:29:55.070 | INFO | Starting a round of Baz IO | 20260507 00:29:58.171 | INFO | Populating RTMetrics report | 20260507 00:29:58.172 | INFO | RTMetrics report populated | 20260507 00:29:58.172 | INFO | Opening tmp RTMetrics json file | 20260507 00:29:58.178 | INFO | Serializing RTMetrics to json | 20260507 00:29:58.192 | INFO | Serialization done, swapping tmp file now... | 20260507 00:29:58.192 | INFO | RTMetrics done | 20260507 00:29:58.267 | INFO | Finished a round of Baz IO: Wrote 1.846 GiB in 3.197 seconds | 20260507 00:29:58.267 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.28% / 0.00% / 66.72% | 20260507 00:29:58.267 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.342576% / 0% / 8.34465e-05% | 20260507 00:29:58.267 | INFO | Baz IO overall: | 20260507 00:29:58.267 | INFO | BytesWritten : 2954252211764 | 20260507 00:29:58.267 | INFO | NumEvents : 1423112159445 | 20260507 00:30:00.104 | INFO | Analyzing chunk frames = [5668864,5669376) | 20260507 00:30:00.104 | INFO | MemInfo: { | 20260507 00:30:00.104 | INFO | "gpu" : " 17 GB", | 20260507 00:30:00.104 | INFO | "huge" : "", | 20260507 00:30:00.104 | INFO | "huge_shared" : " 27 GB", | 20260507 00:30:00.104 | INFO | "malloc" : " 14 GB", | 20260507 00:30:00.104 | INFO | "pinned" : " 29 GB" | 20260507 00:30:00.104 | INFO | } | 20260507 00:30:00.104 | INFO | Perf report for the last 5.14563 seconds: | 20260507 00:30:00.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:30:00.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:00.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:00.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:00.104 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260507 00:30:00.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:00.104 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 00:30:00.104 | INFO | PRE_HQ : 59.5%, 1.8, 56.2ms, 0 | 20260507 00:30:00.104 | INFO | BAZWRITER : 62.1%, 3.4, 109.6ms, 0 | 20260507 00:30:00.104 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:30:00.104 | INFO | Uploaded 36GiB to GPU in 2.05038s (17.5577GiB/s) | 20260507 00:30:00.104 | INFO | Downloaded 38GiB to GPU in 2.17145s (17.4998GiB/s) | 20260507 00:30:05.237 | INFO | Analyzing chunk frames = [5669376,5669888) | 20260507 00:30:05.237 | INFO | MemInfo: { | 20260507 00:30:05.237 | INFO | "gpu" : " 17 GB", | 20260507 00:30:05.237 | INFO | "huge" : "", | 20260507 00:30:05.237 | INFO | "huge_shared" : " 27 GB", | 20260507 00:30:05.237 | INFO | "malloc" : " 14 GB", | 20260507 00:30:05.237 | INFO | "pinned" : " 29 GB" | 20260507 00:30:05.237 | INFO | } | 20260507 00:30:05.237 | INFO | Perf report for the last 5.13343 seconds: | 20260507 00:30:05.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:30:05.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:05.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:05.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:05.237 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:30:05.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:05.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:05.237 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:30:05.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:05.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:05.237 | INFO | Uploaded 36GiB to GPU in 2.02949s (17.7385GiB/s) | 20260507 00:30:05.237 | INFO | Downloaded 34GiB to GPU in 1.94257s (17.5026GiB/s) | 20260507 00:30:10.376 | INFO | Analyzing chunk frames = [5669888,5670400) | 20260507 00:30:10.376 | INFO | MemInfo: { | 20260507 00:30:10.376 | INFO | "gpu" : " 17 GB", | 20260507 00:30:10.376 | INFO | "huge" : "", | 20260507 00:30:10.376 | INFO | "huge_shared" : " 27 GB", | 20260507 00:30:10.376 | INFO | "malloc" : " 14 GB", | 20260507 00:30:10.376 | INFO | "pinned" : " 29 GB" | 20260507 00:30:10.376 | INFO | } | 20260507 00:30:10.376 | INFO | Perf report for the last 5.1391 seconds: | 20260507 00:30:10.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:30:10.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:10.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:10.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:10.376 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:30:10.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:10.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:10.376 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:30:10.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:10.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:10.376 | INFO | Uploaded 36GiB to GPU in 1.98065s (18.1758GiB/s) | 20260507 00:30:10.376 | INFO | Downloaded 34GiB to GPU in 1.93778s (17.5458GiB/s) | 20260507 00:30:15.515 | INFO | Analyzing chunk frames = [5670400,5670912) | 20260507 00:30:15.515 | INFO | MemInfo: { | 20260507 00:30:15.515 | INFO | "gpu" : " 17 GB", | 20260507 00:30:15.515 | INFO | "huge" : "", | 20260507 00:30:15.515 | INFO | "huge_shared" : " 27 GB", | 20260507 00:30:15.515 | INFO | "malloc" : " 14 GB", | 20260507 00:30:15.515 | INFO | "pinned" : " 29 GB" | 20260507 00:30:15.515 | INFO | } | 20260507 00:30:15.515 | INFO | Perf report for the last 5.13904 seconds: | 20260507 00:30:15.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:30:15.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:15.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:15.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:15.515 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 00:30:15.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:15.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:15.515 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 00:30:15.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:15.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:15.516 | INFO | Uploaded 36GiB to GPU in 2.00915s (17.9181GiB/s) | 20260507 00:30:15.516 | INFO | Downloaded 34GiB to GPU in 1.91228s (17.7799GiB/s) | 20260507 00:30:20.651 | INFO | Analyzing chunk frames = [5670912,5671424) | 20260507 00:30:20.651 | INFO | MemInfo: { | 20260507 00:30:20.651 | INFO | "gpu" : " 17 GB", | 20260507 00:30:20.651 | INFO | "huge" : "", | 20260507 00:30:20.651 | INFO | "huge_shared" : " 27 GB", | 20260507 00:30:20.651 | INFO | "malloc" : " 14 GB", | 20260507 00:30:20.651 | INFO | "pinned" : " 29 GB" | 20260507 00:30:20.651 | INFO | } | 20260507 00:30:20.651 | INFO | Perf report for the last 5.1359 seconds: | 20260507 00:30:20.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:30:20.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:20.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:20.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:20.651 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260507 00:30:20.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:20.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:20.651 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:30:20.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:20.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:20.651 | INFO | Uploaded 36GiB to GPU in 1.99s (18.0905GiB/s) | 20260507 00:30:20.651 | INFO | Downloaded 34GiB to GPU in 1.94399s (17.4898GiB/s) | 20260507 00:30:25.803 | INFO | Analyzing chunk frames = [5671424,5671936) | 20260507 00:30:25.803 | INFO | MemInfo: { | 20260507 00:30:25.803 | INFO | "gpu" : " 17 GB", | 20260507 00:30:25.803 | INFO | "huge" : "", | 20260507 00:30:25.803 | INFO | "huge_shared" : " 27 GB", | 20260507 00:30:25.803 | INFO | "malloc" : " 14 GB", | 20260507 00:30:25.803 | INFO | "pinned" : " 29 GB" | 20260507 00:30:25.803 | INFO | } | 20260507 00:30:25.803 | INFO | Perf report for the last 5.15176 seconds: | 20260507 00:30:25.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:30:25.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:25.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:25.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:25.803 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 00:30:25.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:25.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:25.803 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 00:30:25.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:25.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:25.803 | INFO | Uploaded 36GiB to GPU in 2.01031s (17.9076GiB/s) | 20260507 00:30:25.803 | INFO | Downloaded 34GiB to GPU in 1.93899s (17.5349GiB/s) | 20260507 00:30:30.938 | INFO | Analyzing chunk frames = [5671936,5672448) | 20260507 00:30:30.938 | INFO | MemInfo: { | 20260507 00:30:30.938 | INFO | "gpu" : " 17 GB", | 20260507 00:30:30.938 | INFO | "huge" : "", | 20260507 00:30:30.938 | INFO | "huge_shared" : " 27 GB", | 20260507 00:30:30.938 | INFO | "malloc" : " 14 GB", | 20260507 00:30:30.938 | INFO | "pinned" : " 29 GB" | 20260507 00:30:30.938 | INFO | } | 20260507 00:30:30.938 | INFO | Perf report for the last 5.13512 seconds: | 20260507 00:30:30.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:30:30.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:30.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:30.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:30.938 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 00:30:30.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:30.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:30.938 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:30:30.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:30.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:30.938 | INFO | Uploaded 36GiB to GPU in 1.99584s (18.0375GiB/s) | 20260507 00:30:30.938 | INFO | Downloaded 34GiB to GPU in 1.99069s (17.0795GiB/s) | 20260507 00:30:36.079 | INFO | Analyzing chunk frames = [5672448,5672960) | 20260507 00:30:36.079 | INFO | MemInfo: { | 20260507 00:30:36.079 | INFO | "gpu" : " 17 GB", | 20260507 00:30:36.079 | INFO | "huge" : "", | 20260507 00:30:36.079 | INFO | "huge_shared" : " 27 GB", | 20260507 00:30:36.079 | INFO | "malloc" : " 14 GB", | 20260507 00:30:36.079 | INFO | "pinned" : " 29 GB" | 20260507 00:30:36.079 | INFO | } | 20260507 00:30:36.079 | INFO | Perf report for the last 5.14133 seconds: | 20260507 00:30:36.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:30:36.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:36.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:36.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:36.079 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:30:36.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:36.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:36.079 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:30:36.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:36.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:36.080 | INFO | Uploaded 36GiB to GPU in 1.9914s (18.0777GiB/s) | 20260507 00:30:36.080 | INFO | Downloaded 34GiB to GPU in 1.98613s (17.1187GiB/s) | 20260507 00:30:36.177 | INFO | Starting a round of Baz IO | 20260507 00:30:39.294 | INFO | Populating RTMetrics report | 20260507 00:30:39.295 | INFO | RTMetrics report populated | 20260507 00:30:39.295 | INFO | Opening tmp RTMetrics json file | 20260507 00:30:39.295 | INFO | Serializing RTMetrics to json | 20260507 00:30:39.296 | INFO | Serialization done, swapping tmp file now... | 20260507 00:30:39.297 | INFO | RTMetrics done | 20260507 00:30:39.389 | INFO | Finished a round of Baz IO: Wrote 1.844 GiB in 3.211 seconds | 20260507 00:30:39.389 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.28% / 0.00% / 66.72% | 20260507 00:30:39.389 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.34039% / 1.58946e-05% / 7.54992e-05% | 20260507 00:30:39.389 | INFO | Baz IO overall: | 20260507 00:30:39.389 | INFO | BytesWritten : 2956232692763 | 20260507 00:30:39.389 | INFO | NumEvents : 1424052985074 | 20260507 00:30:41.215 | INFO | Analyzing chunk frames = [5672960,5673472) | 20260507 00:30:41.215 | INFO | MemInfo: { | 20260507 00:30:41.215 | INFO | "gpu" : " 17 GB", | 20260507 00:30:41.215 | INFO | "huge" : "", | 20260507 00:30:41.215 | INFO | "huge_shared" : " 27 GB", | 20260507 00:30:41.215 | INFO | "malloc" : " 14 GB", | 20260507 00:30:41.215 | INFO | "pinned" : " 29 GB" | 20260507 00:30:41.215 | INFO | } | 20260507 00:30:41.215 | INFO | Perf report for the last 5.13561 seconds: | 20260507 00:30:41.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:30:41.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:41.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:41.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:41.215 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 00:30:41.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:41.215 | INFO | RT_METRICS : 42.1%, 1.0, 22.0ms, 0 | 20260507 00:30:41.215 | INFO | PRE_HQ : 59.5%, 1.8, 56.2ms, 0 | 20260507 00:30:41.215 | INFO | BAZWRITER : 62.1%, 3.4, 109.8ms, 0 | 20260507 00:30:41.215 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:30:41.215 | INFO | Uploaded 36GiB to GPU in 2.03234s (17.7136GiB/s) | 20260507 00:30:41.215 | INFO | Downloaded 38GiB to GPU in 2.17587s (17.4643GiB/s) | 20260507 00:30:46.350 | INFO | Analyzing chunk frames = [5673472,5673984) | 20260507 00:30:46.350 | INFO | MemInfo: { | 20260507 00:30:46.350 | INFO | "gpu" : " 17 GB", | 20260507 00:30:46.350 | INFO | "huge" : "", | 20260507 00:30:46.351 | INFO | "huge_shared" : " 27 GB", | 20260507 00:30:46.351 | INFO | "malloc" : " 14 GB", | 20260507 00:30:46.351 | INFO | "pinned" : " 29 GB" | 20260507 00:30:46.351 | INFO | } | 20260507 00:30:46.351 | INFO | Perf report for the last 5.13573 seconds: | 20260507 00:30:46.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:30:46.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:46.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:46.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:46.351 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:30:46.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:46.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:46.351 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 00:30:46.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:46.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:46.351 | INFO | Uploaded 36GiB to GPU in 1.99989s (18.001GiB/s) | 20260507 00:30:46.351 | INFO | Downloaded 34GiB to GPU in 1.9265s (17.6486GiB/s) | 20260507 00:30:51.488 | INFO | Analyzing chunk frames = [5673984,5674496) | 20260507 00:30:51.488 | INFO | MemInfo: { | 20260507 00:30:51.488 | INFO | "gpu" : " 17 GB", | 20260507 00:30:51.488 | INFO | "huge" : "", | 20260507 00:30:51.488 | INFO | "huge_shared" : " 27 GB", | 20260507 00:30:51.488 | INFO | "malloc" : " 14 GB", | 20260507 00:30:51.488 | INFO | "pinned" : " 29 GB" | 20260507 00:30:51.488 | INFO | } | 20260507 00:30:51.488 | INFO | Perf report for the last 5.13708 seconds: | 20260507 00:30:51.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:30:51.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:51.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:51.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:51.488 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:30:51.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:51.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:51.488 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:30:51.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:51.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:51.488 | INFO | Uploaded 36GiB to GPU in 1.98199s (18.1636GiB/s) | 20260507 00:30:51.488 | INFO | Downloaded 34GiB to GPU in 1.93194s (17.5989GiB/s) | 20260507 00:30:56.629 | INFO | Analyzing chunk frames = [5674496,5675008) | 20260507 00:30:56.629 | INFO | MemInfo: { | 20260507 00:30:56.629 | INFO | "gpu" : " 17 GB", | 20260507 00:30:56.629 | INFO | "huge" : "", | 20260507 00:30:56.629 | INFO | "huge_shared" : " 27 GB", | 20260507 00:30:56.629 | INFO | "malloc" : " 14 GB", | 20260507 00:30:56.629 | INFO | "pinned" : " 29 GB" | 20260507 00:30:56.629 | INFO | } | 20260507 00:30:56.629 | INFO | Perf report for the last 5.14138 seconds: | 20260507 00:30:56.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:30:56.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:56.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:56.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:56.629 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 00:30:56.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:56.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:56.629 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:30:56.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:56.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:30:56.630 | INFO | Uploaded 36GiB to GPU in 2.01091s (17.9023GiB/s) | 20260507 00:30:56.630 | INFO | Downloaded 34GiB to GPU in 1.90729s (17.8263GiB/s) | 20260507 00:31:01.772 | INFO | Analyzing chunk frames = [5675008,5675520) | 20260507 00:31:01.772 | INFO | MemInfo: { | 20260507 00:31:01.772 | INFO | "gpu" : " 17 GB", | 20260507 00:31:01.772 | INFO | "huge" : "", | 20260507 00:31:01.772 | INFO | "huge_shared" : " 27 GB", | 20260507 00:31:01.772 | INFO | "malloc" : " 14 GB", | 20260507 00:31:01.773 | INFO | "pinned" : " 29 GB" | 20260507 00:31:01.773 | INFO | } | 20260507 00:31:01.773 | INFO | Perf report for the last 5.14333 seconds: | 20260507 00:31:01.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:31:01.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:01.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:01.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:01.773 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 00:31:01.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:01.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:01.773 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:31:01.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:01.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:01.773 | INFO | Uploaded 36GiB to GPU in 1.99927s (18.0066GiB/s) | 20260507 00:31:01.773 | INFO | Downloaded 34GiB to GPU in 1.92699s (17.6441GiB/s) | 20260507 00:31:06.911 | INFO | Analyzing chunk frames = [5675520,5676032) | 20260507 00:31:06.911 | INFO | MemInfo: { | 20260507 00:31:06.911 | INFO | "gpu" : " 17 GB", | 20260507 00:31:06.911 | INFO | "huge" : "", | 20260507 00:31:06.911 | INFO | "huge_shared" : " 27 GB", | 20260507 00:31:06.911 | INFO | "malloc" : " 14 GB", | 20260507 00:31:06.911 | INFO | "pinned" : " 29 GB" | 20260507 00:31:06.911 | INFO | } | 20260507 00:31:06.911 | INFO | Perf report for the last 5.1387 seconds: | 20260507 00:31:06.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:31:06.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:06.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:06.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:06.911 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:31:06.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:06.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:06.911 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:31:06.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:06.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:06.912 | INFO | Uploaded 36GiB to GPU in 2.0513s (17.5498GiB/s) | 20260507 00:31:06.912 | INFO | Downloaded 34GiB to GPU in 1.9469s (17.4636GiB/s) | 20260507 00:31:12.055 | INFO | Analyzing chunk frames = [5676032,5676544) | 20260507 00:31:12.055 | INFO | MemInfo: { | 20260507 00:31:12.055 | INFO | "gpu" : " 17 GB", | 20260507 00:31:12.055 | INFO | "huge" : "", | 20260507 00:31:12.055 | INFO | "huge_shared" : " 27 GB", | 20260507 00:31:12.055 | INFO | "malloc" : " 14 GB", | 20260507 00:31:12.055 | INFO | "pinned" : " 29 GB" | 20260507 00:31:12.055 | INFO | } | 20260507 00:31:12.055 | INFO | Perf report for the last 5.14367 seconds: | 20260507 00:31:12.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:31:12.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:12.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:12.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:12.055 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 00:31:12.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:12.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:12.055 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 00:31:12.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:12.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:12.055 | INFO | Uploaded 36GiB to GPU in 1.97364s (18.2404GiB/s) | 20260507 00:31:12.055 | INFO | Downloaded 34GiB to GPU in 2.00344s (16.9708GiB/s) | 20260507 00:31:17.191 | INFO | Analyzing chunk frames = [5676544,5677056) | 20260507 00:31:17.191 | INFO | MemInfo: { | 20260507 00:31:17.191 | INFO | "gpu" : " 17 GB", | 20260507 00:31:17.191 | INFO | "huge" : "", | 20260507 00:31:17.191 | INFO | "huge_shared" : " 27 GB", | 20260507 00:31:17.191 | INFO | "malloc" : " 14 GB", | 20260507 00:31:17.191 | INFO | "pinned" : " 29 GB" | 20260507 00:31:17.191 | INFO | } | 20260507 00:31:17.191 | INFO | Perf report for the last 5.13586 seconds: | 20260507 00:31:17.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:31:17.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:17.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:17.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:17.191 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:31:17.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:17.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:17.191 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 00:31:17.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:17.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:17.191 | INFO | Uploaded 36GiB to GPU in 1.99066s (18.0844GiB/s) | 20260507 00:31:17.191 | INFO | Downloaded 34GiB to GPU in 1.98665s (17.1142GiB/s) | 20260507 00:31:17.191 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:31:17.191 | INFO | Total mem usage: 18 GiB | 20260507 00:31:17.191 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:31:17.191 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:31:17.191 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:31:17.191 | INFO | High Water Breakdown: | 20260507 00:31:17.191 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:31:17.191 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:31:17.191 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:31:17.191 | INFO | ------------------------------------------ | 20260507 00:31:17.191 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:31:17.191 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:31:17.191 | INFO | Total mem usage: 26.5 GiB | 20260507 00:31:17.191 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:31:17.191 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:31:17.192 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:31:17.192 | INFO | High Water Breakdown: | 20260507 00:31:17.192 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:31:17.192 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:31:17.192 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:31:17.192 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:31:17.192 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:31:17.192 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:31:17.192 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:31:17.192 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:31:17.192 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:31:17.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:31:17.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:31:17.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:31:17.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:31:17.192 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:31:17.192 | INFO | ------------------------------------------ | 20260507 00:31:17.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:31:17.192 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:31:17.192 | INFO | Total mem usage: 25.6 GiB | 20260507 00:31:17.192 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:31:17.192 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:31:17.192 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:31:17.192 | INFO | High Water Breakdown: | 20260507 00:31:17.192 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:31:17.192 | INFO | ------------------------------------------ | 20260507 00:31:17.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:31:17.192 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:31:17.192 | INFO | Total mem usage: 15.7 GiB | 20260507 00:31:17.192 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:31:17.192 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:31:17.192 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 00:31:17.192 | INFO | High Water Breakdown: | 20260507 00:31:17.192 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:31:17.192 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:31:17.192 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:31:17.192 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:31:17.192 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:31:17.192 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:31:17.192 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:31:17.192 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:31:17.192 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:31:17.192 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:31:17.192 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:31:17.192 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:31:17.192 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:31:17.192 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:31:17.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:31:17.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:31:17.192 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:31:17.192 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:31:17.192 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:31:17.192 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:31:17.192 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:31:17.192 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:31:17.192 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:31:17.192 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:31:17.192 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:31:17.192 | INFO | ------------------------------------------ | 20260507 00:31:17.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:31:17.192 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:31:17.192 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:31:17.192 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:31:17.192 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.56e+01, 1.36e+01 / 2.12e+00 | 20260507 00:31:17.192 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 9.93e-02, 9.59e-03 / 6.29e-03 | 20260507 00:31:17.192 | INFO | PerfReport: 11.65%, Baseline -- 3.60e+00 / 4.37e+00, 5.30e-01 / 3.45e-01 | 20260507 00:31:17.192 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.45e+00, 1.74e-01 / 1.14e-01 | 20260507 00:31:17.192 | INFO | PerfReport: 16.06%, DME -- 3.97e+01 / 4.43e+01, 4.03e+00 / 2.10e+00 | 20260507 00:31:17.192 | INFO | PerfReport: 57.51%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.37e+00 / 1.45e+00 | 20260507 00:31:17.192 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.17e+00 / 2.70e+00, 3.78e-01 / 2.35e-01 | 20260507 00:31:17.192 | INFO | PerfReport: 3.15%, Metrics -- 9.75e-01 / 1.27e+00, 2.58e-01 / 1.48e-01 | 20260507 00:31:17.192 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.31e-01 / 3.74e-01, 5.23e-02 / 2.06e-02 | 20260507 00:31:17.192 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:31:17.192 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:31:17.192 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:31:17.192 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 00:31:17.192 | INFO | PerfReport: 21.65%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260507 00:31:17.192 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 00:31:17.192 | INFO | PerfReport: 32.03%, Compute -- 3.09e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260507 00:31:17.192 | INFO | PerfReport: 1.33%, LockUpload -- 1.28e+00 / 0.00e+00, 4.86e+00 / 0.00e+00 | 20260507 00:31:17.192 | INFO | PerfReport: 1.26%, LockDownload -- 1.22e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260507 00:31:17.192 | INFO | PerfReport: 22.38%, LockCompute -- 2.16e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 00:31:17.192 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260507 00:31:17.290 | INFO | Starting a round of Baz IO | 20260507 00:31:20.413 | INFO | Populating RTMetrics report | 20260507 00:31:20.413 | INFO | RTMetrics report populated | 20260507 00:31:20.413 | INFO | Opening tmp RTMetrics json file | 20260507 00:31:20.432 | INFO | Serializing RTMetrics to json | 20260507 00:31:20.433 | INFO | Serialization done, swapping tmp file now... | 20260507 00:31:20.434 | INFO | RTMetrics done | 20260507 00:31:20.506 | INFO | Finished a round of Baz IO: Wrote 1.844 GiB in 3.215 seconds | 20260507 00:31:20.506 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.28% / 0.00% / 66.72% | 20260507 00:31:20.506 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.339639% / 0% / 6.75519e-05% | 20260507 00:31:20.506 | INFO | Baz IO overall: | 20260507 00:31:20.506 | INFO | BytesWritten : 2958212283826 | 20260507 00:31:20.506 | INFO | NumEvents : 1424993061267 | 20260507 00:31:22.326 | INFO | Analyzing chunk frames = [5677056,5677568) | 20260507 00:31:22.326 | INFO | MemInfo: { | 20260507 00:31:22.326 | INFO | "gpu" : " 17 GB", | 20260507 00:31:22.326 | INFO | "huge" : "", | 20260507 00:31:22.326 | INFO | "huge_shared" : " 27 GB", | 20260507 00:31:22.326 | INFO | "malloc" : " 14 GB", | 20260507 00:31:22.326 | INFO | "pinned" : " 29 GB" | 20260507 00:31:22.326 | INFO | } | 20260507 00:31:22.327 | INFO | Perf report for the last 5.13567 seconds: | 20260507 00:31:22.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:31:22.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:22.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:22.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:22.327 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260507 00:31:22.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:22.327 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 00:31:22.327 | INFO | PRE_HQ : 59.8%, 1.8, 56.7ms, 0 | 20260507 00:31:22.327 | INFO | BAZWRITER : 62.2%, 3.4, 108.1ms, 0 | 20260507 00:31:22.327 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:31:22.327 | INFO | Uploaded 36GiB to GPU in 2.02874s (17.745GiB/s) | 20260507 00:31:22.327 | INFO | Downloaded 38GiB to GPU in 2.17472s (17.4735GiB/s) | 20260507 00:31:27.464 | INFO | Analyzing chunk frames = [5677568,5678080) | 20260507 00:31:27.464 | INFO | MemInfo: { | 20260507 00:31:27.464 | INFO | "gpu" : " 17 GB", | 20260507 00:31:27.464 | INFO | "huge" : "", | 20260507 00:31:27.464 | INFO | "huge_shared" : " 27 GB", | 20260507 00:31:27.464 | INFO | "malloc" : " 14 GB", | 20260507 00:31:27.464 | INFO | "pinned" : " 29 GB" | 20260507 00:31:27.464 | INFO | } | 20260507 00:31:27.465 | INFO | Perf report for the last 5.138 seconds: | 20260507 00:31:27.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:31:27.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:27.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:27.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:27.465 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:31:27.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:27.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:27.465 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:31:27.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:27.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:27.465 | INFO | Uploaded 36GiB to GPU in 1.99854s (18.0131GiB/s) | 20260507 00:31:27.465 | INFO | Downloaded 34GiB to GPU in 1.93264s (17.5925GiB/s) | 20260507 00:31:32.605 | INFO | Analyzing chunk frames = [5678080,5678592) | 20260507 00:31:32.605 | INFO | MemInfo: { | 20260507 00:31:32.605 | INFO | "gpu" : " 17 GB", | 20260507 00:31:32.605 | INFO | "huge" : "", | 20260507 00:31:32.605 | INFO | "huge_shared" : " 27 GB", | 20260507 00:31:32.605 | INFO | "malloc" : " 14 GB", | 20260507 00:31:32.605 | INFO | "pinned" : " 29 GB" | 20260507 00:31:32.605 | INFO | } | 20260507 00:31:32.605 | INFO | Perf report for the last 5.14048 seconds: | 20260507 00:31:32.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:31:32.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:32.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:32.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:32.605 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:31:32.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:32.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:32.605 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:31:32.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:32.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:32.605 | INFO | Uploaded 36GiB to GPU in 1.97692s (18.2101GiB/s) | 20260507 00:31:32.605 | INFO | Downloaded 34GiB to GPU in 1.93309s (17.5884GiB/s) | 20260507 00:31:37.743 | INFO | Analyzing chunk frames = [5678592,5679104) | 20260507 00:31:37.743 | INFO | MemInfo: { | 20260507 00:31:37.743 | INFO | "gpu" : " 17 GB", | 20260507 00:31:37.743 | INFO | "huge" : "", | 20260507 00:31:37.743 | INFO | "huge_shared" : " 27 GB", | 20260507 00:31:37.743 | INFO | "malloc" : " 14 GB", | 20260507 00:31:37.743 | INFO | "pinned" : " 29 GB" | 20260507 00:31:37.743 | INFO | } | 20260507 00:31:37.743 | INFO | Perf report for the last 5.13801 seconds: | 20260507 00:31:37.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:31:37.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:37.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:37.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:37.743 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 00:31:37.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:37.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:37.743 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:31:37.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:37.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:37.743 | INFO | Uploaded 36GiB to GPU in 2.00433s (17.9611GiB/s) | 20260507 00:31:37.743 | INFO | Downloaded 34GiB to GPU in 1.91159s (17.7863GiB/s) | 20260507 00:31:42.887 | INFO | Analyzing chunk frames = [5679104,5679616) | 20260507 00:31:42.887 | INFO | MemInfo: { | 20260507 00:31:42.887 | INFO | "gpu" : " 17 GB", | 20260507 00:31:42.887 | INFO | "huge" : "", | 20260507 00:31:42.887 | INFO | "huge_shared" : " 27 GB", | 20260507 00:31:42.887 | INFO | "malloc" : " 14 GB", | 20260507 00:31:42.887 | INFO | "pinned" : " 29 GB" | 20260507 00:31:42.887 | INFO | } | 20260507 00:31:42.887 | INFO | Perf report for the last 5.14432 seconds: | 20260507 00:31:42.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:31:42.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:42.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:42.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:42.887 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 00:31:42.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:42.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:42.888 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 00:31:42.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:42.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:42.888 | INFO | Uploaded 36GiB to GPU in 1.97718s (18.2077GiB/s) | 20260507 00:31:42.888 | INFO | Downloaded 34GiB to GPU in 1.92905s (17.6253GiB/s) | 20260507 00:31:48.025 | INFO | Analyzing chunk frames = [5679616,5680128) | 20260507 00:31:48.025 | INFO | MemInfo: { | 20260507 00:31:48.025 | INFO | "gpu" : " 17 GB", | 20260507 00:31:48.025 | INFO | "huge" : "", | 20260507 00:31:48.025 | INFO | "huge_shared" : " 27 GB", | 20260507 00:31:48.025 | INFO | "malloc" : " 14 GB", | 20260507 00:31:48.025 | INFO | "pinned" : " 29 GB" | 20260507 00:31:48.025 | INFO | } | 20260507 00:31:48.025 | INFO | Perf report for the last 5.13779 seconds: | 20260507 00:31:48.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:31:48.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:48.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:48.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:48.025 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:31:48.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:48.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:48.025 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 00:31:48.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:48.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:48.025 | INFO | Uploaded 36GiB to GPU in 2.04054s (17.6424GiB/s) | 20260507 00:31:48.026 | INFO | Downloaded 34GiB to GPU in 1.95664s (17.3767GiB/s) | 20260507 00:31:53.169 | INFO | Analyzing chunk frames = [5680128,5680640) | 20260507 00:31:53.169 | INFO | MemInfo: { | 20260507 00:31:53.169 | INFO | "gpu" : " 17 GB", | 20260507 00:31:53.169 | INFO | "huge" : "", | 20260507 00:31:53.169 | INFO | "huge_shared" : " 27 GB", | 20260507 00:31:53.169 | INFO | "malloc" : " 14 GB", | 20260507 00:31:53.169 | INFO | "pinned" : " 29 GB" | 20260507 00:31:53.169 | INFO | } | 20260507 00:31:53.169 | INFO | Perf report for the last 5.14395 seconds: | 20260507 00:31:53.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:31:53.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:53.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:53.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:53.169 | INFO | ANALYSIS : 61.1%, 3.0, 94.3ms, 0 | 20260507 00:31:53.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:53.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:53.169 | INFO | PRE_HQ : 26.1%, 1.0, 13.4ms, 0 | 20260507 00:31:53.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:53.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:53.170 | INFO | Uploaded 36GiB to GPU in 1.97638s (18.2152GiB/s) | 20260507 00:31:53.170 | INFO | Downloaded 34GiB to GPU in 2.00174s (16.9852GiB/s) | 20260507 00:31:58.302 | INFO | Analyzing chunk frames = [5680640,5681152) | 20260507 00:31:58.302 | INFO | MemInfo: { | 20260507 00:31:58.302 | INFO | "gpu" : " 17 GB", | 20260507 00:31:58.302 | INFO | "huge" : "", | 20260507 00:31:58.302 | INFO | "huge_shared" : " 27 GB", | 20260507 00:31:58.302 | INFO | "malloc" : " 14 GB", | 20260507 00:31:58.302 | INFO | "pinned" : " 29 GB" | 20260507 00:31:58.302 | INFO | } | 20260507 00:31:58.302 | INFO | Perf report for the last 5.13314 seconds: | 20260507 00:31:58.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:31:58.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:58.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:58.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:58.302 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260507 00:31:58.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:58.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:58.302 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 00:31:58.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:58.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:31:58.303 | INFO | Uploaded 36GiB to GPU in 2.01222s (17.8907GiB/s) | 20260507 00:31:58.303 | INFO | Downloaded 34GiB to GPU in 1.99744s (17.0218GiB/s) | 20260507 00:31:58.401 | INFO | Starting a round of Baz IO | 20260507 00:32:01.528 | INFO | Populating RTMetrics report | 20260507 00:32:01.528 | INFO | RTMetrics report populated | 20260507 00:32:01.528 | INFO | Opening tmp RTMetrics json file | 20260507 00:32:01.532 | INFO | Serializing RTMetrics to json | 20260507 00:32:01.547 | INFO | Serialization done, swapping tmp file now... | 20260507 00:32:01.548 | INFO | RTMetrics done | 20260507 00:32:01.620 | INFO | Finished a round of Baz IO: Wrote 1.843 GiB in 3.219 seconds | 20260507 00:32:01.620 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.27% / 0.00% / 66.72% | 20260507 00:32:01.620 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.339055% / 3.97364e-06% / 6.75519e-05% | 20260507 00:32:01.620 | INFO | Baz IO overall: | 20260507 00:32:01.620 | INFO | BytesWritten : 2960190799214 | 20260507 00:32:01.620 | INFO | NumEvents : 1425932484276 | 20260507 00:32:03.443 | INFO | Analyzing chunk frames = [5681152,5681664) | 20260507 00:32:03.443 | INFO | MemInfo: { | 20260507 00:32:03.443 | INFO | "gpu" : " 17 GB", | 20260507 00:32:03.443 | INFO | "huge" : "", | 20260507 00:32:03.443 | INFO | "huge_shared" : " 27 GB", | 20260507 00:32:03.443 | INFO | "malloc" : " 14 GB", | 20260507 00:32:03.443 | INFO | "pinned" : " 29 GB" | 20260507 00:32:03.443 | INFO | } | 20260507 00:32:03.443 | INFO | Perf report for the last 5.14094 seconds: | 20260507 00:32:03.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:32:03.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:03.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:03.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:03.443 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 00:32:03.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:03.443 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260507 00:32:03.443 | INFO | PRE_HQ : 59.5%, 1.8, 56.7ms, 0 | 20260507 00:32:03.443 | INFO | BAZWRITER : 62.2%, 3.4, 109.1ms, 0 | 20260507 00:32:03.443 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:32:03.444 | INFO | Uploaded 36GiB to GPU in 2.02087s (17.8141GiB/s) | 20260507 00:32:03.444 | INFO | Downloaded 38GiB to GPU in 2.1817s (17.4176GiB/s) | 20260507 00:32:08.580 | INFO | Analyzing chunk frames = [5681664,5682176) | 20260507 00:32:08.580 | INFO | MemInfo: { | 20260507 00:32:08.580 | INFO | "gpu" : " 17 GB", | 20260507 00:32:08.580 | INFO | "huge" : "", | 20260507 00:32:08.580 | INFO | "huge_shared" : " 27 GB", | 20260507 00:32:08.580 | INFO | "malloc" : " 14 GB", | 20260507 00:32:08.580 | INFO | "pinned" : " 29 GB" | 20260507 00:32:08.580 | INFO | } | 20260507 00:32:08.580 | INFO | Perf report for the last 5.13714 seconds: | 20260507 00:32:08.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:32:08.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:08.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:08.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:08.581 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:32:08.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:08.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:08.581 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:32:08.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:08.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:08.581 | INFO | Uploaded 36GiB to GPU in 1.99616s (18.0346GiB/s) | 20260507 00:32:08.581 | INFO | Downloaded 34GiB to GPU in 1.92581s (17.6549GiB/s) | 20260507 00:32:13.721 | INFO | Analyzing chunk frames = [5682176,5682688) | 20260507 00:32:13.721 | INFO | MemInfo: { | 20260507 00:32:13.722 | INFO | "gpu" : " 17 GB", | 20260507 00:32:13.722 | INFO | "huge" : "", | 20260507 00:32:13.722 | INFO | "huge_shared" : " 27 GB", | 20260507 00:32:13.722 | INFO | "malloc" : " 14 GB", | 20260507 00:32:13.722 | INFO | "pinned" : " 29 GB" | 20260507 00:32:13.722 | INFO | } | 20260507 00:32:13.722 | INFO | Perf report for the last 5.14131 seconds: | 20260507 00:32:13.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:32:13.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:13.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:13.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:13.722 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:32:13.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:13.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:13.722 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:32:13.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:13.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:13.722 | INFO | Uploaded 36GiB to GPU in 1.99653s (18.0313GiB/s) | 20260507 00:32:13.722 | INFO | Downloaded 34GiB to GPU in 1.93093s (17.6081GiB/s) | 20260507 00:32:18.862 | INFO | Analyzing chunk frames = [5682688,5683200) | 20260507 00:32:18.862 | INFO | MemInfo: { | 20260507 00:32:18.862 | INFO | "gpu" : " 17 GB", | 20260507 00:32:18.862 | INFO | "huge" : "", | 20260507 00:32:18.862 | INFO | "huge_shared" : " 27 GB", | 20260507 00:32:18.862 | INFO | "malloc" : " 14 GB", | 20260507 00:32:18.862 | INFO | "pinned" : " 29 GB" | 20260507 00:32:18.862 | INFO | } | 20260507 00:32:18.862 | INFO | Perf report for the last 5.14029 seconds: | 20260507 00:32:18.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:32:18.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:18.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:18.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:18.862 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:32:18.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:18.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:18.862 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:32:18.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:18.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:18.862 | INFO | Uploaded 36GiB to GPU in 1.97803s (18.2GiB/s) | 20260507 00:32:18.862 | INFO | Downloaded 34GiB to GPU in 1.90838s (17.8162GiB/s) | 20260507 00:32:23.998 | INFO | Analyzing chunk frames = [5683200,5683712) | 20260507 00:32:23.998 | INFO | MemInfo: { | 20260507 00:32:23.998 | INFO | "gpu" : " 17 GB", | 20260507 00:32:23.998 | INFO | "huge" : "", | 20260507 00:32:23.998 | INFO | "huge_shared" : " 27 GB", | 20260507 00:32:23.998 | INFO | "malloc" : " 14 GB", | 20260507 00:32:23.998 | INFO | "pinned" : " 29 GB" | 20260507 00:32:23.998 | INFO | } | 20260507 00:32:23.999 | INFO | Perf report for the last 5.1365 seconds: | 20260507 00:32:23.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:32:23.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:23.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:23.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:23.999 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 00:32:23.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:23.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:23.999 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:32:23.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:23.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:23.999 | INFO | Uploaded 36GiB to GPU in 1.98903s (18.0993GiB/s) | 20260507 00:32:23.999 | INFO | Downloaded 34GiB to GPU in 1.93452s (17.5754GiB/s) | 20260507 00:32:29.139 | INFO | Analyzing chunk frames = [5683712,5684224) | 20260507 00:32:29.139 | INFO | MemInfo: { | 20260507 00:32:29.139 | INFO | "gpu" : " 17 GB", | 20260507 00:32:29.139 | INFO | "huge" : "", | 20260507 00:32:29.139 | INFO | "huge_shared" : " 27 GB", | 20260507 00:32:29.139 | INFO | "malloc" : " 14 GB", | 20260507 00:32:29.140 | INFO | "pinned" : " 29 GB" | 20260507 00:32:29.140 | INFO | } | 20260507 00:32:29.140 | INFO | Perf report for the last 5.14104 seconds: | 20260507 00:32:29.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:32:29.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:29.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:29.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:29.140 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 00:32:29.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:29.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:29.140 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 00:32:29.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:29.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:29.140 | INFO | Uploaded 36GiB to GPU in 2.01865s (17.8337GiB/s) | 20260507 00:32:29.140 | INFO | Downloaded 34GiB to GPU in 1.92926s (17.6233GiB/s) | 20260507 00:32:34.277 | INFO | Analyzing chunk frames = [5684224,5684736) | 20260507 00:32:34.277 | INFO | MemInfo: { | 20260507 00:32:34.277 | INFO | "gpu" : " 17 GB", | 20260507 00:32:34.277 | INFO | "huge" : "", | 20260507 00:32:34.277 | INFO | "huge_shared" : " 27 GB", | 20260507 00:32:34.277 | INFO | "malloc" : " 14 GB", | 20260507 00:32:34.277 | INFO | "pinned" : " 29 GB" | 20260507 00:32:34.277 | INFO | } | 20260507 00:32:34.277 | INFO | Perf report for the last 5.13741 seconds: | 20260507 00:32:34.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:32:34.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:34.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:34.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:34.277 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:32:34.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:34.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:34.277 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 00:32:34.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:34.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:34.277 | INFO | Uploaded 36GiB to GPU in 1.98214s (18.1622GiB/s) | 20260507 00:32:34.277 | INFO | Downloaded 34GiB to GPU in 1.99629s (17.0316GiB/s) | 20260507 00:32:39.414 | INFO | Analyzing chunk frames = [5684736,5685248) | 20260507 00:32:39.414 | INFO | MemInfo: { | 20260507 00:32:39.414 | INFO | "gpu" : " 17 GB", | 20260507 00:32:39.414 | INFO | "huge" : "", | 20260507 00:32:39.414 | INFO | "huge_shared" : " 27 GB", | 20260507 00:32:39.415 | INFO | "malloc" : " 14 GB", | 20260507 00:32:39.415 | INFO | "pinned" : " 29 GB" | 20260507 00:32:39.415 | INFO | } | 20260507 00:32:39.415 | INFO | Perf report for the last 5.13757 seconds: | 20260507 00:32:39.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:32:39.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:39.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:39.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:39.415 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 00:32:39.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:39.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:39.415 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:32:39.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:39.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:39.415 | INFO | Uploaded 36GiB to GPU in 1.97941s (18.1872GiB/s) | 20260507 00:32:39.415 | INFO | Downloaded 34GiB to GPU in 1.97609s (17.2057GiB/s) | 20260507 00:32:39.415 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:32:39.415 | INFO | Total mem usage: 18 GiB | 20260507 00:32:39.415 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:32:39.415 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:32:39.415 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:32:39.415 | INFO | High Water Breakdown: | 20260507 00:32:39.415 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:32:39.415 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:32:39.415 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:32:39.415 | INFO | ------------------------------------------ | 20260507 00:32:39.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:32:39.415 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:32:39.415 | INFO | Total mem usage: 26.5 GiB | 20260507 00:32:39.415 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:32:39.415 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:32:39.415 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:32:39.415 | INFO | High Water Breakdown: | 20260507 00:32:39.415 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:32:39.415 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:32:39.415 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:32:39.415 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:32:39.415 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:32:39.415 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:32:39.415 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:32:39.415 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:32:39.415 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:32:39.416 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:32:39.416 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:32:39.416 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:32:39.416 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:32:39.416 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:32:39.416 | INFO | ------------------------------------------ | 20260507 00:32:39.416 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:32:39.416 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:32:39.416 | INFO | Total mem usage: 25.6 GiB | 20260507 00:32:39.416 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:32:39.416 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:32:39.416 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:32:39.416 | INFO | High Water Breakdown: | 20260507 00:32:39.416 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:32:39.416 | INFO | ------------------------------------------ | 20260507 00:32:39.416 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:32:39.416 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:32:39.416 | INFO | Total mem usage: 15.7 GiB | 20260507 00:32:39.416 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:32:39.416 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:32:39.416 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:32:39.416 | INFO | High Water Breakdown: | 20260507 00:32:39.416 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:32:39.416 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:32:39.416 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:32:39.416 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:32:39.416 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:32:39.416 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:32:39.416 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:32:39.416 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:32:39.416 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:32:39.416 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:32:39.416 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:32:39.416 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:32:39.416 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:32:39.416 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:32:39.416 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:32:39.416 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:32:39.416 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:32:39.416 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:32:39.416 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:32:39.416 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:32:39.416 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:32:39.416 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:32:39.416 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:32:39.416 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:32:39.416 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:32:39.416 | INFO | ------------------------------------------ | 20260507 00:32:39.416 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:32:39.416 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:32:39.416 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:32:39.416 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:32:39.416 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.61e+01, 1.35e+01 / 2.42e+00 | 20260507 00:32:39.416 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.00e-01, 9.47e-03 / 4.98e-03 | 20260507 00:32:39.416 | INFO | PerfReport: 11.66%, Baseline -- 3.61e+00 / 4.40e+00, 5.39e-01 / 3.06e-01 | 20260507 00:32:39.416 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.47e+00, 1.76e-01 / 1.15e-01 | 20260507 00:32:39.416 | INFO | PerfReport: 16.09%, DME -- 3.98e+01 / 4.47e+01, 4.26e+00 / 2.57e+00 | 20260507 00:32:39.416 | INFO | PerfReport: 57.44%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.34e+00 / 1.62e+00 | 20260507 00:32:39.416 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.18e+00 / 2.71e+00, 3.78e-01 / 2.20e-01 | 20260507 00:32:39.416 | INFO | PerfReport: 3.15%, Metrics -- 9.74e-01 / 1.29e+00, 2.59e-01 / 1.33e-01 | 20260507 00:32:39.416 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.31e-01 / 3.80e-01, 5.23e-02 / 2.32e-02 | 20260507 00:32:39.416 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:32:39.416 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:32:39.416 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:32:39.416 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 00:32:39.416 | INFO | PerfReport: 21.56%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260507 00:32:39.416 | INFO | PerfReport: 21.32%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260507 00:32:39.416 | INFO | PerfReport: 32.04%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:32:39.416 | INFO | PerfReport: 1.30%, LockUpload -- 1.26e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 00:32:39.416 | INFO | PerfReport: 1.30%, LockDownload -- 1.26e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260507 00:32:39.416 | INFO | PerfReport: 22.45%, LockCompute -- 2.17e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 00:32:39.416 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260507 00:32:39.512 | INFO | Starting a round of Baz IO | 20260507 00:32:42.638 | INFO | Populating RTMetrics report | 20260507 00:32:42.638 | INFO | RTMetrics report populated | 20260507 00:32:42.638 | INFO | Opening tmp RTMetrics json file | 20260507 00:32:42.657 | INFO | Serializing RTMetrics to json | 20260507 00:32:42.658 | INFO | Serialization done, swapping tmp file now... | 20260507 00:32:42.659 | INFO | RTMetrics done | 20260507 00:32:42.733 | INFO | Finished a round of Baz IO: Wrote 1.843 GiB in 3.221 seconds | 20260507 00:32:42.733 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.27% / 0.00% / 66.73% | 20260507 00:32:42.733 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.33855% / 7.94729e-06% / 5.96046e-05% | 20260507 00:32:42.733 | INFO | Baz IO overall: | 20260507 00:32:42.733 | INFO | BytesWritten : 2962169635037 | 20260507 00:32:42.733 | INFO | NumEvents : 1426872001777 | 20260507 00:32:44.554 | INFO | Analyzing chunk frames = [5685248,5685760) | 20260507 00:32:44.554 | INFO | MemInfo: { | 20260507 00:32:44.554 | INFO | "gpu" : " 17 GB", | 20260507 00:32:44.554 | INFO | "huge" : "", | 20260507 00:32:44.554 | INFO | "huge_shared" : " 27 GB", | 20260507 00:32:44.554 | INFO | "malloc" : " 14 GB", | 20260507 00:32:44.554 | INFO | "pinned" : " 29 GB" | 20260507 00:32:44.554 | INFO | } | 20260507 00:32:44.554 | INFO | Perf report for the last 5.13979 seconds: | 20260507 00:32:44.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:32:44.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:44.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:44.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:44.555 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260507 00:32:44.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:44.555 | INFO | RT_METRICS : 42.4%, 1.0, 22.3ms, 0 | 20260507 00:32:44.555 | INFO | PRE_HQ : 59.7%, 1.8, 56.3ms, 0 | 20260507 00:32:44.555 | INFO | BAZWRITER : 62.2%, 3.4, 109.1ms, 0 | 20260507 00:32:44.555 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:32:44.555 | INFO | Uploaded 36GiB to GPU in 2.03159s (17.7201GiB/s) | 20260507 00:32:44.555 | INFO | Downloaded 38GiB to GPU in 2.17102s (17.5033GiB/s) | 20260507 00:32:49.694 | INFO | Analyzing chunk frames = [5685760,5686272) | 20260507 00:32:49.694 | INFO | MemInfo: { | 20260507 00:32:49.694 | INFO | "gpu" : " 17 GB", | 20260507 00:32:49.694 | INFO | "huge" : "", | 20260507 00:32:49.694 | INFO | "huge_shared" : " 27 GB", | 20260507 00:32:49.694 | INFO | "malloc" : " 14 GB", | 20260507 00:32:49.694 | INFO | "pinned" : " 29 GB" | 20260507 00:32:49.694 | INFO | } | 20260507 00:32:49.694 | INFO | Perf report for the last 5.13973 seconds: | 20260507 00:32:49.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:32:49.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:49.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:49.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:49.694 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:32:49.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:49.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:49.694 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:32:49.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:49.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:49.695 | INFO | Uploaded 36GiB to GPU in 2.01012s (17.9094GiB/s) | 20260507 00:32:49.695 | INFO | Downloaded 34GiB to GPU in 1.93381s (17.5819GiB/s) | 20260507 00:32:54.844 | INFO | Analyzing chunk frames = [5686272,5686784) | 20260507 00:32:54.844 | INFO | MemInfo: { | 20260507 00:32:54.844 | INFO | "gpu" : " 17 GB", | 20260507 00:32:54.844 | INFO | "huge" : "", | 20260507 00:32:54.844 | INFO | "huge_shared" : " 27 GB", | 20260507 00:32:54.844 | INFO | "malloc" : " 14 GB", | 20260507 00:32:54.844 | INFO | "pinned" : " 29 GB" | 20260507 00:32:54.844 | INFO | } | 20260507 00:32:54.844 | INFO | Perf report for the last 5.1503 seconds: | 20260507 00:32:54.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:32:54.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:54.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:54.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:54.845 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260507 00:32:54.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:54.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:54.845 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 00:32:54.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:54.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:54.845 | INFO | Uploaded 36GiB to GPU in 1.95701s (18.3954GiB/s) | 20260507 00:32:54.845 | INFO | Downloaded 34GiB to GPU in 1.92069s (17.702GiB/s) | 20260507 00:32:59.981 | INFO | Analyzing chunk frames = [5686784,5687296) | 20260507 00:32:59.981 | INFO | MemInfo: { | 20260507 00:32:59.981 | INFO | "gpu" : " 17 GB", | 20260507 00:32:59.981 | INFO | "huge" : "", | 20260507 00:32:59.981 | INFO | "huge_shared" : " 27 GB", | 20260507 00:32:59.981 | INFO | "malloc" : " 14 GB", | 20260507 00:32:59.982 | INFO | "pinned" : " 29 GB" | 20260507 00:32:59.982 | INFO | } | 20260507 00:32:59.982 | INFO | Perf report for the last 5.13718 seconds: | 20260507 00:32:59.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:32:59.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:59.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:59.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:59.982 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 00:32:59.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:59.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:59.982 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:32:59.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:59.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:32:59.982 | INFO | Uploaded 36GiB to GPU in 1.96708s (18.3012GiB/s) | 20260507 00:32:59.982 | INFO | Downloaded 34GiB to GPU in 1.90717s (17.8275GiB/s) | 20260507 00:33:05.118 | INFO | Analyzing chunk frames = [5687296,5687808) | 20260507 00:33:05.118 | INFO | MemInfo: { | 20260507 00:33:05.118 | INFO | "gpu" : " 17 GB", | 20260507 00:33:05.118 | INFO | "huge" : "", | 20260507 00:33:05.118 | INFO | "huge_shared" : " 27 GB", | 20260507 00:33:05.118 | INFO | "malloc" : " 14 GB", | 20260507 00:33:05.118 | INFO | "pinned" : " 29 GB" | 20260507 00:33:05.118 | INFO | } | 20260507 00:33:05.119 | INFO | Perf report for the last 5.13679 seconds: | 20260507 00:33:05.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:33:05.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:05.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:05.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:05.119 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 00:33:05.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:05.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:05.119 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:33:05.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:05.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:05.119 | INFO | Uploaded 36GiB to GPU in 2.00671s (17.9398GiB/s) | 20260507 00:33:05.119 | INFO | Downloaded 34GiB to GPU in 1.94222s (17.5057GiB/s) | 20260507 00:33:10.256 | INFO | Analyzing chunk frames = [5687808,5688320) | 20260507 00:33:10.256 | INFO | MemInfo: { | 20260507 00:33:10.256 | INFO | "gpu" : " 17 GB", | 20260507 00:33:10.256 | INFO | "huge" : "", | 20260507 00:33:10.256 | INFO | "huge_shared" : " 27 GB", | 20260507 00:33:10.257 | INFO | "malloc" : " 14 GB", | 20260507 00:33:10.257 | INFO | "pinned" : " 29 GB" | 20260507 00:33:10.257 | INFO | } | 20260507 00:33:10.257 | INFO | Perf report for the last 5.1381 seconds: | 20260507 00:33:10.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:33:10.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:10.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:10.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:10.257 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:33:10.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:10.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:10.257 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 00:33:10.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:10.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:10.257 | INFO | Uploaded 36GiB to GPU in 2.02056s (17.8168GiB/s) | 20260507 00:33:10.257 | INFO | Downloaded 34GiB to GPU in 1.94276s (17.5008GiB/s) | 20260507 00:33:15.393 | INFO | Analyzing chunk frames = [5688320,5688832) | 20260507 00:33:15.393 | INFO | MemInfo: { | 20260507 00:33:15.393 | INFO | "gpu" : " 17 GB", | 20260507 00:33:15.393 | INFO | "huge" : "", | 20260507 00:33:15.393 | INFO | "huge_shared" : " 27 GB", | 20260507 00:33:15.393 | INFO | "malloc" : " 14 GB", | 20260507 00:33:15.393 | INFO | "pinned" : " 29 GB" | 20260507 00:33:15.393 | INFO | } | 20260507 00:33:15.393 | INFO | Perf report for the last 5.13644 seconds: | 20260507 00:33:15.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:33:15.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:15.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:15.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:15.393 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:33:15.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:15.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:15.393 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 00:33:15.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:15.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:15.394 | INFO | Uploaded 36GiB to GPU in 1.97306s (18.2457GiB/s) | 20260507 00:33:15.394 | INFO | Downloaded 34GiB to GPU in 1.99854s (17.0124GiB/s) | 20260507 00:33:20.533 | INFO | Analyzing chunk frames = [5688832,5689344) | 20260507 00:33:20.533 | INFO | MemInfo: { | 20260507 00:33:20.533 | INFO | "gpu" : " 17 GB", | 20260507 00:33:20.533 | INFO | "huge" : "", | 20260507 00:33:20.533 | INFO | "huge_shared" : " 27 GB", | 20260507 00:33:20.533 | INFO | "malloc" : " 14 GB", | 20260507 00:33:20.533 | INFO | "pinned" : " 29 GB" | 20260507 00:33:20.533 | INFO | } | 20260507 00:33:20.533 | INFO | Perf report for the last 5.14002 seconds: | 20260507 00:33:20.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:33:20.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:20.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:20.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:20.533 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 00:33:20.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:20.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:20.533 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 00:33:20.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:20.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:20.533 | INFO | Uploaded 36GiB to GPU in 2.02292s (17.796GiB/s) | 20260507 00:33:20.533 | INFO | Downloaded 34GiB to GPU in 2.01345s (16.8865GiB/s) | 20260507 00:33:20.634 | INFO | Starting a round of Baz IO | 20260507 00:33:23.754 | INFO | Populating RTMetrics report | 20260507 00:33:23.754 | INFO | RTMetrics report populated | 20260507 00:33:23.754 | INFO | Opening tmp RTMetrics json file | 20260507 00:33:23.755 | INFO | Serializing RTMetrics to json | 20260507 00:33:23.767 | INFO | Serialization done, swapping tmp file now... | 20260507 00:33:23.775 | INFO | RTMetrics done | 20260507 00:33:23.846 | INFO | Finished a round of Baz IO: Wrote 1.846 GiB in 3.212 seconds | 20260507 00:33:23.846 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.27% / 0.00% / 66.73% | 20260507 00:33:23.846 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.339337% / 3.97364e-06% / 5.16574e-05% | 20260507 00:33:23.846 | INFO | Baz IO overall: | 20260507 00:33:23.846 | INFO | BytesWritten : 2964151719463 | 20260507 00:33:23.846 | INFO | NumEvents : 1427814286606 | 20260507 00:33:25.681 | INFO | Analyzing chunk frames = [5689344,5689856) | 20260507 00:33:25.681 | INFO | MemInfo: { | 20260507 00:33:25.681 | INFO | "gpu" : " 17 GB", | 20260507 00:33:25.681 | INFO | "huge" : "", | 20260507 00:33:25.681 | INFO | "huge_shared" : " 27 GB", | 20260507 00:33:25.681 | INFO | "malloc" : " 14 GB", | 20260507 00:33:25.681 | INFO | "pinned" : " 29 GB" | 20260507 00:33:25.681 | INFO | } | 20260507 00:33:25.681 | INFO | Perf report for the last 5.14791 seconds: | 20260507 00:33:25.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:33:25.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:25.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:25.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:25.681 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260507 00:33:25.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:25.681 | INFO | RT_METRICS : 42.4%, 1.0, 22.3ms, 0 | 20260507 00:33:25.681 | INFO | PRE_HQ : 59.6%, 1.8, 56.3ms, 0 | 20260507 00:33:25.681 | INFO | BAZWRITER : 62.0%, 3.4, 108.1ms, 0 | 20260507 00:33:25.681 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:33:25.681 | INFO | Uploaded 36GiB to GPU in 2.03515s (17.6891GiB/s) | 20260507 00:33:25.681 | INFO | Downloaded 38GiB to GPU in 2.18907s (17.359GiB/s) | 20260507 00:33:30.809 | INFO | Analyzing chunk frames = [5689856,5690368) | 20260507 00:33:30.809 | INFO | MemInfo: { | 20260507 00:33:30.809 | INFO | "gpu" : " 17 GB", | 20260507 00:33:30.809 | INFO | "huge" : "", | 20260507 00:33:30.809 | INFO | "huge_shared" : " 27 GB", | 20260507 00:33:30.809 | INFO | "malloc" : " 14 GB", | 20260507 00:33:30.809 | INFO | "pinned" : " 29 GB" | 20260507 00:33:30.809 | INFO | } | 20260507 00:33:30.809 | INFO | Perf report for the last 5.12783 seconds: | 20260507 00:33:30.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:33:30.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:30.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:30.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:30.809 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260507 00:33:30.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:30.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:30.809 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 00:33:30.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:30.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:30.809 | INFO | Uploaded 36GiB to GPU in 2.02924s (17.7406GiB/s) | 20260507 00:33:30.809 | INFO | Downloaded 34GiB to GPU in 1.94011s (17.5248GiB/s) | 20260507 00:33:35.952 | INFO | Analyzing chunk frames = [5690368,5690880) | 20260507 00:33:35.952 | INFO | MemInfo: { | 20260507 00:33:35.952 | INFO | "gpu" : " 17 GB", | 20260507 00:33:35.952 | INFO | "huge" : "", | 20260507 00:33:35.952 | INFO | "huge_shared" : " 27 GB", | 20260507 00:33:35.952 | INFO | "malloc" : " 14 GB", | 20260507 00:33:35.952 | INFO | "pinned" : " 29 GB" | 20260507 00:33:35.952 | INFO | } | 20260507 00:33:35.952 | INFO | Perf report for the last 5.14301 seconds: | 20260507 00:33:35.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:33:35.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:35.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:35.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:35.952 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 00:33:35.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:35.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:35.952 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:33:35.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:35.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:35.952 | INFO | Uploaded 36GiB to GPU in 1.95951s (18.372GiB/s) | 20260507 00:33:35.952 | INFO | Downloaded 34GiB to GPU in 1.92994s (17.6171GiB/s) | 20260507 00:33:41.097 | INFO | Analyzing chunk frames = [5690880,5691392) | 20260507 00:33:41.097 | INFO | MemInfo: { | 20260507 00:33:41.097 | INFO | "gpu" : " 17 GB", | 20260507 00:33:41.097 | INFO | "huge" : "", | 20260507 00:33:41.097 | INFO | "huge_shared" : " 27 GB", | 20260507 00:33:41.097 | INFO | "malloc" : " 14 GB", | 20260507 00:33:41.097 | INFO | "pinned" : " 29 GB" | 20260507 00:33:41.097 | INFO | } | 20260507 00:33:41.097 | INFO | Perf report for the last 5.14478 seconds: | 20260507 00:33:41.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:33:41.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:41.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:41.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:41.097 | INFO | ANALYSIS : 59.4%, 3.0, 92.1ms, 0 | 20260507 00:33:41.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:41.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:41.097 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 00:33:41.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:41.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:41.097 | INFO | Uploaded 36GiB to GPU in 1.95113s (18.4509GiB/s) | 20260507 00:33:41.097 | INFO | Downloaded 34GiB to GPU in 1.88292s (18.057GiB/s) | 20260507 00:33:46.229 | INFO | Analyzing chunk frames = [5691392,5691904) | 20260507 00:33:46.229 | INFO | MemInfo: { | 20260507 00:33:46.229 | INFO | "gpu" : " 17 GB", | 20260507 00:33:46.229 | INFO | "huge" : "", | 20260507 00:33:46.229 | INFO | "huge_shared" : " 27 GB", | 20260507 00:33:46.229 | INFO | "malloc" : " 14 GB", | 20260507 00:33:46.229 | INFO | "pinned" : " 29 GB" | 20260507 00:33:46.229 | INFO | } | 20260507 00:33:46.229 | INFO | Perf report for the last 5.13221 seconds: | 20260507 00:33:46.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:33:46.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:46.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:46.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:46.229 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 00:33:46.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:46.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:46.229 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:33:46.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:46.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:46.229 | INFO | Uploaded 36GiB to GPU in 1.9973s (18.0243GiB/s) | 20260507 00:33:46.229 | INFO | Downloaded 34GiB to GPU in 1.94737s (17.4595GiB/s) | 20260507 00:33:51.362 | INFO | Analyzing chunk frames = [5691904,5692416) | 20260507 00:33:51.362 | INFO | MemInfo: { | 20260507 00:33:51.362 | INFO | "gpu" : " 17 GB", | 20260507 00:33:51.362 | INFO | "huge" : "", | 20260507 00:33:51.362 | INFO | "huge_shared" : " 27 GB", | 20260507 00:33:51.362 | INFO | "malloc" : " 14 GB", | 20260507 00:33:51.362 | INFO | "pinned" : " 29 GB" | 20260507 00:33:51.362 | INFO | } | 20260507 00:33:51.362 | INFO | Perf report for the last 5.13291 seconds: | 20260507 00:33:51.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:33:51.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:51.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:51.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:51.362 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 00:33:51.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:51.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:51.362 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:33:51.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:51.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:51.362 | INFO | Uploaded 36GiB to GPU in 2.02413s (17.7854GiB/s) | 20260507 00:33:51.362 | INFO | Downloaded 34GiB to GPU in 1.95016s (17.4345GiB/s) | 20260507 00:33:56.506 | INFO | Analyzing chunk frames = [5692416,5692928) | 20260507 00:33:56.506 | INFO | MemInfo: { | 20260507 00:33:56.506 | INFO | "gpu" : " 17 GB", | 20260507 00:33:56.506 | INFO | "huge" : "", | 20260507 00:33:56.506 | INFO | "huge_shared" : " 27 GB", | 20260507 00:33:56.506 | INFO | "malloc" : " 14 GB", | 20260507 00:33:56.506 | INFO | "pinned" : " 29 GB" | 20260507 00:33:56.506 | INFO | } | 20260507 00:33:56.506 | INFO | Perf report for the last 5.14443 seconds: | 20260507 00:33:56.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:33:56.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:56.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:56.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:56.506 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:33:56.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:56.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:56.506 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 00:33:56.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:56.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:33:56.507 | INFO | Uploaded 36GiB to GPU in 1.95715s (18.3941GiB/s) | 20260507 00:33:56.507 | INFO | Downloaded 34GiB to GPU in 1.98706s (17.1107GiB/s) | 20260507 00:34:01.653 | INFO | Analyzing chunk frames = [5692928,5693440) | 20260507 00:34:01.653 | INFO | MemInfo: { | 20260507 00:34:01.653 | INFO | "gpu" : " 17 GB", | 20260507 00:34:01.653 | INFO | "huge" : "", | 20260507 00:34:01.653 | INFO | "huge_shared" : " 27 GB", | 20260507 00:34:01.653 | INFO | "malloc" : " 14 GB", | 20260507 00:34:01.653 | INFO | "pinned" : " 29 GB" | 20260507 00:34:01.653 | INFO | } | 20260507 00:34:01.653 | INFO | Perf report for the last 5.14656 seconds: | 20260507 00:34:01.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:34:01.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:01.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:01.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:01.653 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260507 00:34:01.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:01.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:01.653 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260507 00:34:01.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:01.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:01.653 | INFO | Uploaded 36GiB to GPU in 1.99189s (18.0732GiB/s) | 20260507 00:34:01.653 | INFO | Downloaded 34GiB to GPU in 1.97296s (17.233GiB/s) | 20260507 00:34:01.653 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:34:01.653 | INFO | Total mem usage: 18 GiB | 20260507 00:34:01.653 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:34:01.653 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 00:34:01.653 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:34:01.653 | INFO | High Water Breakdown: | 20260507 00:34:01.653 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:34:01.653 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:34:01.653 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:34:01.654 | INFO | ------------------------------------------ | 20260507 00:34:01.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:34:01.654 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:34:01.654 | INFO | Total mem usage: 26.5 GiB | 20260507 00:34:01.654 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:34:01.654 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:34:01.654 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:34:01.654 | INFO | High Water Breakdown: | 20260507 00:34:01.654 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:34:01.654 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:34:01.654 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:34:01.654 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:34:01.654 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:34:01.654 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:34:01.654 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:34:01.654 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:34:01.654 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:34:01.654 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:34:01.654 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:34:01.654 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:34:01.654 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:34:01.654 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:34:01.654 | INFO | ------------------------------------------ | 20260507 00:34:01.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:34:01.654 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:34:01.654 | INFO | Total mem usage: 25.6 GiB | 20260507 00:34:01.654 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:34:01.654 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:34:01.654 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:34:01.654 | INFO | High Water Breakdown: | 20260507 00:34:01.654 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:34:01.654 | INFO | ------------------------------------------ | 20260507 00:34:01.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:34:01.654 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:34:01.654 | INFO | Total mem usage: 15.7 GiB | 20260507 00:34:01.654 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:34:01.654 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:34:01.654 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 00:34:01.654 | INFO | High Water Breakdown: | 20260507 00:34:01.654 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:34:01.654 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:34:01.654 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:34:01.654 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:34:01.654 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:34:01.654 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:34:01.654 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:34:01.654 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:34:01.654 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:34:01.654 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:34:01.654 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:34:01.654 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:34:01.654 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:34:01.654 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:34:01.654 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:34:01.654 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:34:01.654 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:34:01.654 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:34:01.654 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:34:01.654 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:34:01.654 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:34:01.654 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:34:01.654 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:34:01.654 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:34:01.654 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:34:01.654 | INFO | ------------------------------------------ | 20260507 00:34:01.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:34:01.655 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:34:01.655 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:34:01.655 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:34:01.655 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.58e+01, 1.36e+01 / 2.27e+00 | 20260507 00:34:01.655 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.68e-03 / 4.88e-03 | 20260507 00:34:01.655 | INFO | PerfReport: 11.67%, Baseline -- 3.61e+00 / 4.38e+00, 5.36e-01 / 2.77e-01 | 20260507 00:34:01.655 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.47e+00, 1.75e-01 / 1.17e-01 | 20260507 00:34:01.655 | INFO | PerfReport: 16.19%, DME -- 4.00e+01 / 4.55e+01, 4.44e+00 / 2.52e+00 | 20260507 00:34:01.655 | INFO | PerfReport: 57.31%, FrameLabeling -- 1.77e+01 / 1.99e+01, 1.24e+00 / 1.59e+00 | 20260507 00:34:01.655 | INFO | PerfReport: 7.07%, PulseAccumulating -- 2.18e+00 / 2.80e+00, 3.90e-01 / 1.78e-01 | 20260507 00:34:01.655 | INFO | PerfReport: 3.15%, Metrics -- 9.73e-01 / 1.33e+00, 2.58e-01 / 1.42e-01 | 20260507 00:34:01.655 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.31e-01 / 3.82e-01, 5.23e-02 / 2.10e-02 | 20260507 00:34:01.655 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:34:01.655 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:34:01.655 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:34:01.655 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 00:34:01.655 | INFO | PerfReport: 21.53%, Upload -- 2.08e+01 / 0.00e+00, 6.93e+00 / 0.00e+00 | 20260507 00:34:01.655 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 00:34:01.655 | INFO | PerfReport: 32.01%, Compute -- 3.09e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260507 00:34:01.655 | INFO | PerfReport: 1.30%, LockUpload -- 1.26e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 00:34:01.655 | INFO | PerfReport: 1.41%, LockDownload -- 1.36e+00 / 0.00e+00, 3.29e+00 / 0.00e+00 | 20260507 00:34:01.655 | INFO | PerfReport: 22.40%, LockCompute -- 2.16e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 00:34:01.655 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.53e-02 / 0.00e+00 | 20260507 00:34:01.750 | INFO | Starting a round of Baz IO | 20260507 00:34:04.868 | INFO | Populating RTMetrics report | 20260507 00:34:04.869 | INFO | RTMetrics report populated | 20260507 00:34:04.869 | INFO | Opening tmp RTMetrics json file | 20260507 00:34:04.870 | INFO | Serializing RTMetrics to json | 20260507 00:34:04.882 | INFO | Serialization done, swapping tmp file now... | 20260507 00:34:04.890 | INFO | RTMetrics done | 20260507 00:34:04.963 | INFO | Finished a round of Baz IO: Wrote 1.843 GiB in 3.213 seconds | 20260507 00:34:04.963 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.27% / 0.00% / 66.73% | 20260507 00:34:04.963 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.337096% / 0% / 4.76837e-05% | 20260507 00:34:04.963 | INFO | Baz IO overall: | 20260507 00:34:04.964 | INFO | BytesWritten : 2966131155127 | 20260507 00:34:04.964 | INFO | NumEvents : 1428754594059 | 20260507 00:34:06.792 | INFO | Analyzing chunk frames = [5693440,5693952) | 20260507 00:34:06.792 | INFO | MemInfo: { | 20260507 00:34:06.792 | INFO | "gpu" : " 17 GB", | 20260507 00:34:06.792 | INFO | "huge" : "", | 20260507 00:34:06.792 | INFO | "huge_shared" : " 27 GB", | 20260507 00:34:06.792 | INFO | "malloc" : " 14 GB", | 20260507 00:34:06.792 | INFO | "pinned" : " 29 GB" | 20260507 00:34:06.792 | INFO | } | 20260507 00:34:06.792 | INFO | Perf report for the last 5.13966 seconds: | 20260507 00:34:06.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:34:06.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:06.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:06.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:06.793 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 00:34:06.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:06.793 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 00:34:06.793 | INFO | PRE_HQ : 59.5%, 1.8, 56.2ms, 0 | 20260507 00:34:06.793 | INFO | BAZWRITER : 62.1%, 3.4, 108.9ms, 0 | 20260507 00:34:06.793 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:34:06.793 | INFO | Uploaded 36GiB to GPU in 2.04105s (17.638GiB/s) | 20260507 00:34:06.793 | INFO | Downloaded 38GiB to GPU in 2.18231s (17.4127GiB/s) | 20260507 00:34:11.937 | INFO | Analyzing chunk frames = [5693952,5694464) | 20260507 00:34:11.937 | INFO | MemInfo: { | 20260507 00:34:11.937 | INFO | "gpu" : " 17 GB", | 20260507 00:34:11.937 | INFO | "huge" : "", | 20260507 00:34:11.937 | INFO | "huge_shared" : " 27 GB", | 20260507 00:34:11.937 | INFO | "malloc" : " 14 GB", | 20260507 00:34:11.937 | INFO | "pinned" : " 29 GB" | 20260507 00:34:11.937 | INFO | } | 20260507 00:34:11.937 | INFO | Perf report for the last 5.14429 seconds: | 20260507 00:34:11.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:34:11.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:11.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:11.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:11.937 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 00:34:11.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:11.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:11.937 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:34:11.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:11.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:11.937 | INFO | Uploaded 36GiB to GPU in 2.0232s (17.7936GiB/s) | 20260507 00:34:11.937 | INFO | Downloaded 34GiB to GPU in 1.93019s (17.6148GiB/s) | 20260507 00:34:17.066 | INFO | Analyzing chunk frames = [5694464,5694976) | 20260507 00:34:17.066 | INFO | MemInfo: { | 20260507 00:34:17.066 | INFO | "gpu" : " 17 GB", | 20260507 00:34:17.066 | INFO | "huge" : "", | 20260507 00:34:17.066 | INFO | "huge_shared" : " 27 GB", | 20260507 00:34:17.066 | INFO | "malloc" : " 14 GB", | 20260507 00:34:17.066 | INFO | "pinned" : " 29 GB" | 20260507 00:34:17.067 | INFO | } | 20260507 00:34:17.067 | INFO | Perf report for the last 5.12979 seconds: | 20260507 00:34:17.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:34:17.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:17.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:17.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:17.067 | INFO | ANALYSIS : 60.5%, 3.0, 93.5ms, 0 | 20260507 00:34:17.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:17.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:17.067 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:34:17.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:17.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:17.067 | INFO | Uploaded 36GiB to GPU in 2.00112s (17.9899GiB/s) | 20260507 00:34:17.067 | INFO | Downloaded 34GiB to GPU in 1.93567s (17.565GiB/s) | 20260507 00:34:22.205 | INFO | Analyzing chunk frames = [5694976,5695488) | 20260507 00:34:22.205 | INFO | MemInfo: { | 20260507 00:34:22.205 | INFO | "gpu" : " 17 GB", | 20260507 00:34:22.205 | INFO | "huge" : "", | 20260507 00:34:22.205 | INFO | "huge_shared" : " 27 GB", | 20260507 00:34:22.205 | INFO | "malloc" : " 14 GB", | 20260507 00:34:22.205 | INFO | "pinned" : " 29 GB" | 20260507 00:34:22.205 | INFO | } | 20260507 00:34:22.206 | INFO | Perf report for the last 5.13899 seconds: | 20260507 00:34:22.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:34:22.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:22.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:22.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:22.206 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 00:34:22.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:22.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:22.206 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:34:22.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:22.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:22.206 | INFO | Uploaded 36GiB to GPU in 2.01657s (17.8521GiB/s) | 20260507 00:34:22.206 | INFO | Downloaded 34GiB to GPU in 1.90804s (17.8194GiB/s) | 20260507 00:34:27.344 | INFO | Analyzing chunk frames = [5695488,5696000) | 20260507 00:34:27.344 | INFO | MemInfo: { | 20260507 00:34:27.344 | INFO | "gpu" : " 17 GB", | 20260507 00:34:27.344 | INFO | "huge" : "", | 20260507 00:34:27.344 | INFO | "huge_shared" : " 27 GB", | 20260507 00:34:27.344 | INFO | "malloc" : " 14 GB", | 20260507 00:34:27.344 | INFO | "pinned" : " 29 GB" | 20260507 00:34:27.344 | INFO | } | 20260507 00:34:27.344 | INFO | Perf report for the last 5.1386 seconds: | 20260507 00:34:27.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:34:27.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:27.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:27.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:27.344 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 00:34:27.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:27.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:27.344 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:34:27.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:27.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:27.345 | INFO | Uploaded 36GiB to GPU in 1.98906s (18.099GiB/s) | 20260507 00:34:27.345 | INFO | Downloaded 34GiB to GPU in 1.92712s (17.6429GiB/s) | 20260507 00:34:32.481 | INFO | Analyzing chunk frames = [5696000,5696512) | 20260507 00:34:32.481 | INFO | MemInfo: { | 20260507 00:34:32.481 | INFO | "gpu" : " 17 GB", | 20260507 00:34:32.481 | INFO | "huge" : "", | 20260507 00:34:32.481 | INFO | "huge_shared" : " 27 GB", | 20260507 00:34:32.481 | INFO | "malloc" : " 14 GB", | 20260507 00:34:32.481 | INFO | "pinned" : " 29 GB" | 20260507 00:34:32.481 | INFO | } | 20260507 00:34:32.481 | INFO | Perf report for the last 5.13675 seconds: | 20260507 00:34:32.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:34:32.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:32.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:32.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:32.481 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 00:34:32.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:32.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:32.481 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:34:32.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:32.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:32.481 | INFO | Uploaded 36GiB to GPU in 2.02791s (17.7523GiB/s) | 20260507 00:34:32.481 | INFO | Downloaded 34GiB to GPU in 1.94218s (17.5061GiB/s) | 20260507 00:34:37.623 | INFO | Analyzing chunk frames = [5696512,5697024) | 20260507 00:34:37.623 | INFO | MemInfo: { | 20260507 00:34:37.623 | INFO | "gpu" : " 17 GB", | 20260507 00:34:37.623 | INFO | "huge" : "", | 20260507 00:34:37.623 | INFO | "huge_shared" : " 27 GB", | 20260507 00:34:37.623 | INFO | "malloc" : " 14 GB", | 20260507 00:34:37.623 | INFO | "pinned" : " 29 GB" | 20260507 00:34:37.623 | INFO | } | 20260507 00:34:37.623 | INFO | Perf report for the last 5.14183 seconds: | 20260507 00:34:37.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:34:37.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:37.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:37.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:37.623 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:34:37.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:37.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:37.623 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 00:34:37.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:37.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:37.623 | INFO | Uploaded 36GiB to GPU in 1.97139s (18.2612GiB/s) | 20260507 00:34:37.623 | INFO | Downloaded 34GiB to GPU in 1.99713s (17.0244GiB/s) | 20260507 00:34:42.764 | INFO | Analyzing chunk frames = [5697024,5697536) | 20260507 00:34:42.764 | INFO | MemInfo: { | 20260507 00:34:42.764 | INFO | "gpu" : " 17 GB", | 20260507 00:34:42.764 | INFO | "huge" : "", | 20260507 00:34:42.764 | INFO | "huge_shared" : " 27 GB", | 20260507 00:34:42.764 | INFO | "malloc" : " 14 GB", | 20260507 00:34:42.764 | INFO | "pinned" : " 29 GB" | 20260507 00:34:42.765 | INFO | } | 20260507 00:34:42.765 | INFO | Perf report for the last 5.14184 seconds: | 20260507 00:34:42.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:34:42.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:42.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:42.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:42.765 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:34:42.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:42.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:42.765 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 00:34:42.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:42.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:42.765 | INFO | Uploaded 36GiB to GPU in 1.96163s (18.3521GiB/s) | 20260507 00:34:42.765 | INFO | Downloaded 34GiB to GPU in 1.97005s (17.2585GiB/s) | 20260507 00:34:42.863 | INFO | Starting a round of Baz IO | 20260507 00:34:45.987 | INFO | Populating RTMetrics report | 20260507 00:34:45.988 | INFO | RTMetrics report populated | 20260507 00:34:45.988 | INFO | Opening tmp RTMetrics json file | 20260507 00:34:45.988 | INFO | Serializing RTMetrics to json | 20260507 00:34:45.990 | INFO | Serialization done, swapping tmp file now... | 20260507 00:34:45.992 | INFO | RTMetrics done | 20260507 00:34:46.084 | INFO | Finished a round of Baz IO: Wrote 1.842 GiB in 3.221 seconds | 20260507 00:34:46.084 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.27% / 0.00% / 66.73% | 20260507 00:34:46.084 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.335765% / 0% / 7.94729e-05% | 20260507 00:34:46.084 | INFO | Baz IO overall: | 20260507 00:34:46.084 | INFO | BytesWritten : 2968108979395 | 20260507 00:34:46.084 | INFO | NumEvents : 1429693580752 | 20260507 00:34:47.909 | INFO | Analyzing chunk frames = [5697536,5698048) | 20260507 00:34:47.909 | INFO | MemInfo: { | 20260507 00:34:47.909 | INFO | "gpu" : " 17 GB", | 20260507 00:34:47.909 | INFO | "huge" : "", | 20260507 00:34:47.909 | INFO | "huge_shared" : " 27 GB", | 20260507 00:34:47.909 | INFO | "malloc" : " 14 GB", | 20260507 00:34:47.909 | INFO | "pinned" : " 29 GB" | 20260507 00:34:47.909 | INFO | } | 20260507 00:34:47.909 | INFO | Perf report for the last 5.14421 seconds: | 20260507 00:34:47.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:34:47.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:47.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:47.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:47.909 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 00:34:47.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:47.909 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 00:34:47.909 | INFO | PRE_HQ : 59.9%, 1.8, 56.6ms, 0 | 20260507 00:34:47.909 | INFO | BAZWRITER : 62.2%, 3.4, 109.2ms, 0 | 20260507 00:34:47.909 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:34:47.909 | INFO | Uploaded 36GiB to GPU in 2.02322s (17.7934GiB/s) | 20260507 00:34:47.909 | INFO | Downloaded 38GiB to GPU in 2.18124s (17.4213GiB/s) | 20260507 00:34:53.042 | INFO | Analyzing chunk frames = [5698048,5698560) | 20260507 00:34:53.042 | INFO | MemInfo: { | 20260507 00:34:53.042 | INFO | "gpu" : " 17 GB", | 20260507 00:34:53.042 | INFO | "huge" : "", | 20260507 00:34:53.042 | INFO | "huge_shared" : " 27 GB", | 20260507 00:34:53.042 | INFO | "malloc" : " 14 GB", | 20260507 00:34:53.042 | INFO | "pinned" : " 29 GB" | 20260507 00:34:53.042 | INFO | } | 20260507 00:34:53.042 | INFO | Perf report for the last 5.13342 seconds: | 20260507 00:34:53.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:34:53.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:53.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:53.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:53.042 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:34:53.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:53.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:53.042 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 00:34:53.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:53.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:53.043 | INFO | Uploaded 36GiB to GPU in 1.99245s (18.0682GiB/s) | 20260507 00:34:53.043 | INFO | Downloaded 34GiB to GPU in 1.93604s (17.5617GiB/s) | 20260507 00:34:58.179 | INFO | Analyzing chunk frames = [5698560,5699072) | 20260507 00:34:58.179 | INFO | MemInfo: { | 20260507 00:34:58.179 | INFO | "gpu" : " 17 GB", | 20260507 00:34:58.179 | INFO | "huge" : "", | 20260507 00:34:58.179 | INFO | "huge_shared" : " 27 GB", | 20260507 00:34:58.179 | INFO | "malloc" : " 14 GB", | 20260507 00:34:58.179 | INFO | "pinned" : " 29 GB" | 20260507 00:34:58.179 | INFO | } | 20260507 00:34:58.179 | INFO | Perf report for the last 5.13713 seconds: | 20260507 00:34:58.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:34:58.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:58.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:58.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:58.180 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:34:58.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:58.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:58.180 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:34:58.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:58.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:34:58.180 | INFO | Uploaded 36GiB to GPU in 1.98775s (18.111GiB/s) | 20260507 00:34:58.180 | INFO | Downloaded 34GiB to GPU in 1.9384s (17.5402GiB/s) | 20260507 00:35:03.316 | INFO | Analyzing chunk frames = [5699072,5699584) | 20260507 00:35:03.316 | INFO | MemInfo: { | 20260507 00:35:03.316 | INFO | "gpu" : " 17 GB", | 20260507 00:35:03.316 | INFO | "huge" : "", | 20260507 00:35:03.316 | INFO | "huge_shared" : " 27 GB", | 20260507 00:35:03.316 | INFO | "malloc" : " 14 GB", | 20260507 00:35:03.316 | INFO | "pinned" : " 29 GB" | 20260507 00:35:03.316 | INFO | } | 20260507 00:35:03.316 | INFO | Perf report for the last 5.13651 seconds: | 20260507 00:35:03.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:35:03.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:03.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:03.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:03.316 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:35:03.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:03.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:03.316 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:35:03.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:03.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:03.316 | INFO | Uploaded 36GiB to GPU in 1.98406s (18.1446GiB/s) | 20260507 00:35:03.316 | INFO | Downloaded 34GiB to GPU in 1.91342s (17.7692GiB/s) | 20260507 00:35:08.457 | INFO | Analyzing chunk frames = [5699584,5700096) | 20260507 00:35:08.457 | INFO | MemInfo: { | 20260507 00:35:08.457 | INFO | "gpu" : " 17 GB", | 20260507 00:35:08.457 | INFO | "huge" : "", | 20260507 00:35:08.457 | INFO | "huge_shared" : " 27 GB", | 20260507 00:35:08.457 | INFO | "malloc" : " 14 GB", | 20260507 00:35:08.457 | INFO | "pinned" : " 29 GB" | 20260507 00:35:08.457 | INFO | } | 20260507 00:35:08.457 | INFO | Perf report for the last 5.14109 seconds: | 20260507 00:35:08.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:35:08.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:08.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:08.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:08.457 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 00:35:08.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:08.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:08.457 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:35:08.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:08.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:08.457 | INFO | Uploaded 36GiB to GPU in 1.96481s (18.3223GiB/s) | 20260507 00:35:08.457 | INFO | Downloaded 34GiB to GPU in 1.92475s (17.6646GiB/s) | 20260507 00:35:13.604 | INFO | Analyzing chunk frames = [5700096,5700608) | 20260507 00:35:13.604 | INFO | MemInfo: { | 20260507 00:35:13.604 | INFO | "gpu" : " 17 GB", | 20260507 00:35:13.604 | INFO | "huge" : "", | 20260507 00:35:13.604 | INFO | "huge_shared" : " 27 GB", | 20260507 00:35:13.604 | INFO | "malloc" : " 14 GB", | 20260507 00:35:13.604 | INFO | "pinned" : " 29 GB" | 20260507 00:35:13.604 | INFO | } | 20260507 00:35:13.604 | INFO | Perf report for the last 5.14687 seconds: | 20260507 00:35:13.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:35:13.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:13.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:13.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:13.604 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 00:35:13.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:13.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:13.604 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 00:35:13.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:13.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:13.604 | INFO | Uploaded 36GiB to GPU in 2.03845s (17.6605GiB/s) | 20260507 00:35:13.605 | INFO | Downloaded 34GiB to GPU in 1.94966s (17.4389GiB/s) | 20260507 00:35:18.743 | INFO | Analyzing chunk frames = [5700608,5701120) | 20260507 00:35:18.743 | INFO | MemInfo: { | 20260507 00:35:18.743 | INFO | "gpu" : " 17 GB", | 20260507 00:35:18.743 | INFO | "huge" : "", | 20260507 00:35:18.743 | INFO | "huge_shared" : " 27 GB", | 20260507 00:35:18.743 | INFO | "malloc" : " 14 GB", | 20260507 00:35:18.743 | INFO | "pinned" : " 29 GB" | 20260507 00:35:18.743 | INFO | } | 20260507 00:35:18.743 | INFO | Perf report for the last 5.13941 seconds: | 20260507 00:35:18.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:35:18.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:18.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:18.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:18.743 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:35:18.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:18.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:18.744 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 00:35:18.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:18.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:18.744 | INFO | Uploaded 36GiB to GPU in 1.98191s (18.1643GiB/s) | 20260507 00:35:18.744 | INFO | Downloaded 34GiB to GPU in 2.00599s (16.9492GiB/s) | 20260507 00:35:23.889 | INFO | Analyzing chunk frames = [5701120,5701632) | 20260507 00:35:23.889 | INFO | MemInfo: { | 20260507 00:35:23.889 | INFO | "gpu" : " 17 GB", | 20260507 00:35:23.889 | INFO | "huge" : "", | 20260507 00:35:23.889 | INFO | "huge_shared" : " 27 GB", | 20260507 00:35:23.889 | INFO | "malloc" : " 14 GB", | 20260507 00:35:23.889 | INFO | "pinned" : " 29 GB" | 20260507 00:35:23.889 | INFO | } | 20260507 00:35:23.889 | INFO | Perf report for the last 5.14569 seconds: | 20260507 00:35:23.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:35:23.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:23.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:23.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:23.889 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 00:35:23.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:23.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:23.889 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 00:35:23.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:23.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:23.889 | INFO | Uploaded 36GiB to GPU in 2.0042s (17.9623GiB/s) | 20260507 00:35:23.889 | INFO | Downloaded 34GiB to GPU in 1.99061s (17.0802GiB/s) | 20260507 00:35:23.890 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:35:23.890 | INFO | Total mem usage: 18 GiB | 20260507 00:35:23.890 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:35:23.890 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:35:23.890 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:35:23.890 | INFO | High Water Breakdown: | 20260507 00:35:23.890 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:35:23.890 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:35:23.890 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:35:23.890 | INFO | ------------------------------------------ | 20260507 00:35:23.890 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:35:23.890 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:35:23.890 | INFO | Total mem usage: 26.5 GiB | 20260507 00:35:23.890 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:35:23.890 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:35:23.890 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:35:23.890 | INFO | High Water Breakdown: | 20260507 00:35:23.890 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:35:23.890 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:35:23.890 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:35:23.890 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:35:23.890 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:35:23.890 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:35:23.890 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:35:23.890 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:35:23.890 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:35:23.890 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:35:23.890 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:35:23.890 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:35:23.890 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:35:23.890 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:35:23.890 | INFO | ------------------------------------------ | 20260507 00:35:23.890 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:35:23.890 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:35:23.890 | INFO | Total mem usage: 25.6 GiB | 20260507 00:35:23.890 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:35:23.890 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:35:23.890 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:35:23.890 | INFO | High Water Breakdown: | 20260507 00:35:23.890 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:35:23.890 | INFO | ------------------------------------------ | 20260507 00:35:23.890 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:35:23.890 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:35:23.890 | INFO | Total mem usage: 15.7 GiB | 20260507 00:35:23.890 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:35:23.890 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:35:23.890 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 00:35:23.890 | INFO | High Water Breakdown: | 20260507 00:35:23.890 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:35:23.890 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:35:23.890 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:35:23.890 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:35:23.890 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:35:23.890 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:35:23.890 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:35:23.890 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:35:23.890 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:35:23.890 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:35:23.890 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:35:23.890 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:35:23.890 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:35:23.890 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:35:23.890 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:35:23.890 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:35:23.890 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:35:23.890 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:35:23.890 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:35:23.890 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:35:23.890 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:35:23.890 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:35:23.890 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:35:23.890 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:35:23.890 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:35:23.890 | INFO | ------------------------------------------ | 20260507 00:35:23.890 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:35:23.891 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:35:23.891 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:35:23.891 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:35:23.891 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.63e+01, 1.35e+01 / 2.21e+00 | 20260507 00:35:23.891 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.79e-03 / 4.86e-03 | 20260507 00:35:23.891 | INFO | PerfReport: 11.65%, Baseline -- 3.60e+00 / 4.45e+00, 5.29e-01 / 2.57e-01 | 20260507 00:35:23.891 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.47e+00, 1.73e-01 / 1.12e-01 | 20260507 00:35:23.891 | INFO | PerfReport: 16.10%, DME -- 3.98e+01 / 4.48e+01, 4.15e+00 / 2.70e+00 | 20260507 00:35:23.891 | INFO | PerfReport: 57.46%, FrameLabeling -- 1.78e+01 / 2.11e+01, 1.34e+00 / 1.97e+00 | 20260507 00:35:23.891 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.17e+00 / 2.67e+00, 3.77e-01 / 2.48e-01 | 20260507 00:35:23.891 | INFO | PerfReport: 3.14%, Metrics -- 9.72e-01 / 1.29e+00, 2.58e-01 / 1.48e-01 | 20260507 00:35:23.891 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.30e-01 / 3.79e-01, 5.24e-02 / 2.29e-02 | 20260507 00:35:23.891 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:35:23.891 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:35:23.891 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:35:23.891 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 00:35:23.891 | INFO | PerfReport: 21.58%, Upload -- 2.08e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260507 00:35:23.891 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 00:35:23.891 | INFO | PerfReport: 32.03%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:35:23.891 | INFO | PerfReport: 1.32%, LockUpload -- 1.28e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 00:35:23.891 | INFO | PerfReport: 1.31%, LockDownload -- 1.27e+00 / 0.00e+00, 3.14e+00 / 0.00e+00 | 20260507 00:35:23.891 | INFO | PerfReport: 22.41%, LockCompute -- 2.16e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 00:35:23.891 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260507 00:35:23.986 | INFO | Starting a round of Baz IO | 20260507 00:35:27.113 | INFO | Populating RTMetrics report | 20260507 00:35:27.114 | INFO | RTMetrics report populated | 20260507 00:35:27.114 | INFO | Opening tmp RTMetrics json file | 20260507 00:35:27.114 | INFO | Serializing RTMetrics to json | 20260507 00:35:27.116 | INFO | Serialization done, swapping tmp file now... | 20260507 00:35:27.116 | INFO | RTMetrics done | 20260507 00:35:27.211 | INFO | Finished a round of Baz IO: Wrote 1.841 GiB in 3.224 seconds | 20260507 00:35:27.211 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.27% / 0.00% / 66.73% | 20260507 00:35:27.211 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.335677% / 0% / 9.53674e-05% | 20260507 00:35:27.211 | INFO | Baz IO overall: | 20260507 00:35:27.211 | INFO | BytesWritten : 2970086139549 | 20260507 00:35:27.211 | INFO | NumEvents : 1430632434482 | 20260507 00:35:29.022 | INFO | Analyzing chunk frames = [5701632,5702144) | 20260507 00:35:29.022 | INFO | MemInfo: { | 20260507 00:35:29.022 | INFO | "gpu" : " 17 GB", | 20260507 00:35:29.022 | INFO | "huge" : "", | 20260507 00:35:29.022 | INFO | "huge_shared" : " 27 GB", | 20260507 00:35:29.022 | INFO | "malloc" : " 14 GB", | 20260507 00:35:29.022 | INFO | "pinned" : " 29 GB" | 20260507 00:35:29.022 | INFO | } | 20260507 00:35:29.022 | INFO | Perf report for the last 5.13286 seconds: | 20260507 00:35:29.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:35:29.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:29.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:29.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:29.022 | INFO | ANALYSIS : 62.2%, 2.9, 95.8ms, 0 | 20260507 00:35:29.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:29.022 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260507 00:35:29.022 | INFO | PRE_HQ : 60.4%, 1.9, 60.3ms, 0 | 20260507 00:35:29.022 | INFO | BAZWRITER : 62.4%, 3.5, 113.5ms, 0 | 20260507 00:35:29.022 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:35:29.022 | INFO | Uploaded 36GiB to GPU in 2.03646s (17.6778GiB/s) | 20260507 00:35:29.022 | INFO | Downloaded 38GiB to GPU in 2.17318s (17.4859GiB/s) | 20260507 00:35:34.157 | INFO | Analyzing chunk frames = [5702144,5702656) | 20260507 00:35:34.157 | INFO | MemInfo: { | 20260507 00:35:34.157 | INFO | "gpu" : " 17 GB", | 20260507 00:35:34.157 | INFO | "huge" : "", | 20260507 00:35:34.157 | INFO | "huge_shared" : " 27 GB", | 20260507 00:35:34.157 | INFO | "malloc" : " 14 GB", | 20260507 00:35:34.157 | INFO | "pinned" : " 29 GB" | 20260507 00:35:34.157 | INFO | } | 20260507 00:35:34.157 | INFO | Perf report for the last 5.1351 seconds: | 20260507 00:35:34.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:35:34.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:34.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:34.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:34.157 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:35:34.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:34.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:34.157 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 00:35:34.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:34.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:34.157 | INFO | Uploaded 36GiB to GPU in 1.99025s (18.0882GiB/s) | 20260507 00:35:34.157 | INFO | Downloaded 34GiB to GPU in 1.92998s (17.6168GiB/s) | 20260507 00:35:39.295 | INFO | Analyzing chunk frames = [5702656,5703168) | 20260507 00:35:39.295 | INFO | MemInfo: { | 20260507 00:35:39.295 | INFO | "gpu" : " 17 GB", | 20260507 00:35:39.295 | INFO | "huge" : "", | 20260507 00:35:39.295 | INFO | "huge_shared" : " 27 GB", | 20260507 00:35:39.295 | INFO | "malloc" : " 14 GB", | 20260507 00:35:39.295 | INFO | "pinned" : " 29 GB" | 20260507 00:35:39.295 | INFO | } | 20260507 00:35:39.295 | INFO | Perf report for the last 5.13798 seconds: | 20260507 00:35:39.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:35:39.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:39.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:39.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:39.295 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 00:35:39.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:39.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:39.295 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 00:35:39.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:39.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:39.295 | INFO | Uploaded 36GiB to GPU in 1.97952s (18.1862GiB/s) | 20260507 00:35:39.295 | INFO | Downloaded 34GiB to GPU in 1.93279s (17.5912GiB/s) | 20260507 00:35:44.433 | INFO | Analyzing chunk frames = [5703168,5703680) | 20260507 00:35:44.433 | INFO | MemInfo: { | 20260507 00:35:44.433 | INFO | "gpu" : " 17 GB", | 20260507 00:35:44.433 | INFO | "huge" : "", | 20260507 00:35:44.433 | INFO | "huge_shared" : " 27 GB", | 20260507 00:35:44.433 | INFO | "malloc" : " 14 GB", | 20260507 00:35:44.433 | INFO | "pinned" : " 29 GB" | 20260507 00:35:44.433 | INFO | } | 20260507 00:35:44.433 | INFO | Perf report for the last 5.13828 seconds: | 20260507 00:35:44.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:35:44.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:44.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:44.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:44.433 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:35:44.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:44.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:44.433 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 00:35:44.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:44.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:44.434 | INFO | Uploaded 36GiB to GPU in 1.96653s (18.3064GiB/s) | 20260507 00:35:44.434 | INFO | Downloaded 34GiB to GPU in 1.90961s (17.8047GiB/s) | 20260507 00:35:49.575 | INFO | Analyzing chunk frames = [5703680,5704192) | 20260507 00:35:49.575 | INFO | MemInfo: { | 20260507 00:35:49.575 | INFO | "gpu" : " 17 GB", | 20260507 00:35:49.575 | INFO | "huge" : "", | 20260507 00:35:49.575 | INFO | "huge_shared" : " 27 GB", | 20260507 00:35:49.575 | INFO | "malloc" : " 14 GB", | 20260507 00:35:49.575 | INFO | "pinned" : " 29 GB" | 20260507 00:35:49.575 | INFO | } | 20260507 00:35:49.575 | INFO | Perf report for the last 5.14149 seconds: | 20260507 00:35:49.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:35:49.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:49.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:49.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:49.575 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 00:35:49.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:49.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:49.575 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 00:35:49.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:49.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:49.575 | INFO | Uploaded 36GiB to GPU in 1.96519s (18.3189GiB/s) | 20260507 00:35:49.575 | INFO | Downloaded 34GiB to GPU in 1.93083s (17.609GiB/s) | 20260507 00:35:54.714 | INFO | Analyzing chunk frames = [5704192,5704704) | 20260507 00:35:54.714 | INFO | MemInfo: { | 20260507 00:35:54.714 | INFO | "gpu" : " 17 GB", | 20260507 00:35:54.714 | INFO | "huge" : "", | 20260507 00:35:54.714 | INFO | "huge_shared" : " 27 GB", | 20260507 00:35:54.714 | INFO | "malloc" : " 14 GB", | 20260507 00:35:54.714 | INFO | "pinned" : " 29 GB" | 20260507 00:35:54.714 | INFO | } | 20260507 00:35:54.714 | INFO | Perf report for the last 5.13935 seconds: | 20260507 00:35:54.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:35:54.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:54.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:54.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:54.714 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 00:35:54.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:54.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:54.714 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:35:54.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:54.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:54.715 | INFO | Uploaded 36GiB to GPU in 2.00771s (17.9309GiB/s) | 20260507 00:35:54.715 | INFO | Downloaded 34GiB to GPU in 1.92574s (17.6556GiB/s) | 20260507 00:35:59.860 | INFO | Analyzing chunk frames = [5704704,5705216) | 20260507 00:35:59.860 | INFO | MemInfo: { | 20260507 00:35:59.860 | INFO | "gpu" : " 17 GB", | 20260507 00:35:59.860 | INFO | "huge" : "", | 20260507 00:35:59.860 | INFO | "huge_shared" : " 27 GB", | 20260507 00:35:59.860 | INFO | "malloc" : " 14 GB", | 20260507 00:35:59.860 | INFO | "pinned" : " 29 GB" | 20260507 00:35:59.860 | INFO | } | 20260507 00:35:59.860 | INFO | Perf report for the last 5.14559 seconds: | 20260507 00:35:59.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:35:59.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:59.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:59.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:59.860 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:35:59.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:59.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:59.860 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:35:59.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:59.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:35:59.860 | INFO | Uploaded 36GiB to GPU in 1.97479s (18.2298GiB/s) | 20260507 00:35:59.860 | INFO | Downloaded 34GiB to GPU in 2.00058s (16.995GiB/s) | 20260507 00:36:04.993 | INFO | Analyzing chunk frames = [5705216,5705728) | 20260507 00:36:04.993 | INFO | MemInfo: { | 20260507 00:36:04.993 | INFO | "gpu" : " 17 GB", | 20260507 00:36:04.993 | INFO | "huge" : "", | 20260507 00:36:04.993 | INFO | "huge_shared" : " 27 GB", | 20260507 00:36:04.993 | INFO | "malloc" : " 14 GB", | 20260507 00:36:04.993 | INFO | "pinned" : " 29 GB" | 20260507 00:36:04.994 | INFO | } | 20260507 00:36:04.994 | INFO | Perf report for the last 5.13372 seconds: | 20260507 00:36:04.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:36:04.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:04.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:04.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:04.994 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260507 00:36:04.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:04.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:04.994 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:36:04.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:04.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:04.994 | INFO | Uploaded 36GiB to GPU in 2.0134s (17.8802GiB/s) | 20260507 00:36:04.994 | INFO | Downloaded 34GiB to GPU in 1.98013s (17.1706GiB/s) | 20260507 00:36:05.093 | INFO | Starting a round of Baz IO | 20260507 00:36:08.220 | INFO | Populating RTMetrics report | 20260507 00:36:08.220 | INFO | RTMetrics report populated | 20260507 00:36:08.220 | INFO | Opening tmp RTMetrics json file | 20260507 00:36:08.221 | INFO | Serializing RTMetrics to json | 20260507 00:36:08.231 | INFO | Serialization done, swapping tmp file now... | 20260507 00:36:08.241 | INFO | RTMetrics done | 20260507 00:36:08.315 | INFO | Finished a round of Baz IO: Wrote 1.84 GiB in 3.222 seconds | 20260507 00:36:08.315 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.26% / 0.00% / 66.73% | 20260507 00:36:08.315 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.333778% / 3.97364e-06% / 8.74201e-05% | 20260507 00:36:08.315 | INFO | Baz IO overall: | 20260507 00:36:08.315 | INFO | BytesWritten : 2972061754874 | 20260507 00:36:08.315 | INFO | NumEvents : 1431569942063 | 20260507 00:36:10.132 | INFO | Analyzing chunk frames = [5705728,5706240) | 20260507 00:36:10.132 | INFO | MemInfo: { | 20260507 00:36:10.132 | INFO | "gpu" : " 17 GB", | 20260507 00:36:10.132 | INFO | "huge" : "", | 20260507 00:36:10.132 | INFO | "huge_shared" : " 27 GB", | 20260507 00:36:10.132 | INFO | "malloc" : " 14 GB", | 20260507 00:36:10.132 | INFO | "pinned" : " 29 GB" | 20260507 00:36:10.132 | INFO | } | 20260507 00:36:10.132 | INFO | Perf report for the last 5.13868 seconds: | 20260507 00:36:10.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:36:10.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:10.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:10.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:10.132 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 00:36:10.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:10.132 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 00:36:10.132 | INFO | PRE_HQ : 59.7%, 1.8, 56.1ms, 0 | 20260507 00:36:10.132 | INFO | BAZWRITER : 62.3%, 3.4, 110.0ms, 0 | 20260507 00:36:10.132 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:36:10.133 | INFO | Uploaded 36GiB to GPU in 2.0416s (17.6332GiB/s) | 20260507 00:36:10.133 | INFO | Downloaded 38GiB to GPU in 2.18179s (17.4169GiB/s) | 20260507 00:36:15.272 | INFO | Analyzing chunk frames = [5706240,5706752) | 20260507 00:36:15.272 | INFO | MemInfo: { | 20260507 00:36:15.272 | INFO | "gpu" : " 17 GB", | 20260507 00:36:15.272 | INFO | "huge" : "", | 20260507 00:36:15.272 | INFO | "huge_shared" : " 27 GB", | 20260507 00:36:15.272 | INFO | "malloc" : " 14 GB", | 20260507 00:36:15.272 | INFO | "pinned" : " 29 GB" | 20260507 00:36:15.272 | INFO | } | 20260507 00:36:15.272 | INFO | Perf report for the last 5.13956 seconds: | 20260507 00:36:15.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:36:15.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:15.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:15.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:15.272 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 00:36:15.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:15.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:15.272 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:36:15.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:15.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:15.272 | INFO | Uploaded 36GiB to GPU in 1.9901s (18.0896GiB/s) | 20260507 00:36:15.272 | INFO | Downloaded 34GiB to GPU in 1.92721s (17.642GiB/s) | 20260507 00:36:20.406 | INFO | Analyzing chunk frames = [5706752,5707264) | 20260507 00:36:20.406 | INFO | MemInfo: { | 20260507 00:36:20.406 | INFO | "gpu" : " 17 GB", | 20260507 00:36:20.406 | INFO | "huge" : "", | 20260507 00:36:20.406 | INFO | "huge_shared" : " 27 GB", | 20260507 00:36:20.406 | INFO | "malloc" : " 14 GB", | 20260507 00:36:20.406 | INFO | "pinned" : " 29 GB" | 20260507 00:36:20.406 | INFO | } | 20260507 00:36:20.406 | INFO | Perf report for the last 5.13422 seconds: | 20260507 00:36:20.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:36:20.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:20.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:20.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:20.406 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:36:20.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:20.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:20.406 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:36:20.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:20.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:20.406 | INFO | Uploaded 36GiB to GPU in 1.99995s (18.0005GiB/s) | 20260507 00:36:20.406 | INFO | Downloaded 34GiB to GPU in 1.93341s (17.5855GiB/s) | 20260507 00:36:25.558 | INFO | Analyzing chunk frames = [5707264,5707776) | 20260507 00:36:25.558 | INFO | MemInfo: { | 20260507 00:36:25.558 | INFO | "gpu" : " 17 GB", | 20260507 00:36:25.558 | INFO | "huge" : "", | 20260507 00:36:25.558 | INFO | "huge_shared" : " 27 GB", | 20260507 00:36:25.558 | INFO | "malloc" : " 14 GB", | 20260507 00:36:25.558 | INFO | "pinned" : " 29 GB" | 20260507 00:36:25.558 | INFO | } | 20260507 00:36:25.558 | INFO | Perf report for the last 5.15179 seconds: | 20260507 00:36:25.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:36:25.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:25.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:25.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:25.558 | INFO | ANALYSIS : 60.0%, 3.0, 93.2ms, 0 | 20260507 00:36:25.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:25.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:25.558 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:36:25.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:25.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:25.558 | INFO | Uploaded 36GiB to GPU in 2.01797s (17.8397GiB/s) | 20260507 00:36:25.558 | INFO | Downloaded 34GiB to GPU in 1.91044s (17.7969GiB/s) | 20260507 00:36:30.696 | INFO | Analyzing chunk frames = [5707776,5708288) | 20260507 00:36:30.696 | INFO | MemInfo: { | 20260507 00:36:30.696 | INFO | "gpu" : " 17 GB", | 20260507 00:36:30.696 | INFO | "huge" : "", | 20260507 00:36:30.696 | INFO | "huge_shared" : " 27 GB", | 20260507 00:36:30.696 | INFO | "malloc" : " 14 GB", | 20260507 00:36:30.696 | INFO | "pinned" : " 29 GB" | 20260507 00:36:30.696 | INFO | } | 20260507 00:36:30.696 | INFO | Perf report for the last 5.13849 seconds: | 20260507 00:36:30.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:36:30.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:30.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:30.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:30.696 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 00:36:30.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:30.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:30.697 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:36:30.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:30.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:30.697 | INFO | Uploaded 36GiB to GPU in 2.02423s (17.7846GiB/s) | 20260507 00:36:30.697 | INFO | Downloaded 34GiB to GPU in 1.9314s (17.6038GiB/s) | 20260507 00:36:35.831 | INFO | Analyzing chunk frames = [5708288,5708800) | 20260507 00:36:35.831 | INFO | MemInfo: { | 20260507 00:36:35.831 | INFO | "gpu" : " 17 GB", | 20260507 00:36:35.831 | INFO | "huge" : "", | 20260507 00:36:35.831 | INFO | "huge_shared" : " 27 GB", | 20260507 00:36:35.831 | INFO | "malloc" : " 14 GB", | 20260507 00:36:35.831 | INFO | "pinned" : " 29 GB" | 20260507 00:36:35.831 | INFO | } | 20260507 00:36:35.831 | INFO | Perf report for the last 5.13487 seconds: | 20260507 00:36:35.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:36:35.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:35.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:35.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:35.831 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 00:36:35.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:35.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:35.831 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260507 00:36:35.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:35.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:35.832 | INFO | Uploaded 36GiB to GPU in 2.05692s (17.5019GiB/s) | 20260507 00:36:35.832 | INFO | Downloaded 34GiB to GPU in 1.96008s (17.3462GiB/s) | 20260507 00:36:40.971 | INFO | Analyzing chunk frames = [5708800,5709312) | 20260507 00:36:40.971 | INFO | MemInfo: { | 20260507 00:36:40.971 | INFO | "gpu" : " 17 GB", | 20260507 00:36:40.971 | INFO | "huge" : "", | 20260507 00:36:40.971 | INFO | "huge_shared" : " 27 GB", | 20260507 00:36:40.971 | INFO | "malloc" : " 14 GB", | 20260507 00:36:40.971 | INFO | "pinned" : " 29 GB" | 20260507 00:36:40.971 | INFO | } | 20260507 00:36:40.971 | INFO | Perf report for the last 5.13972 seconds: | 20260507 00:36:40.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:36:40.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:40.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:40.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:40.971 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:36:40.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:40.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:40.971 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:36:40.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:40.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:40.971 | INFO | Uploaded 36GiB to GPU in 1.99192s (18.073GiB/s) | 20260507 00:36:40.971 | INFO | Downloaded 34GiB to GPU in 1.98027s (17.1694GiB/s) | 20260507 00:36:46.108 | INFO | Analyzing chunk frames = [5709312,5709824) | 20260507 00:36:46.108 | INFO | MemInfo: { | 20260507 00:36:46.108 | INFO | "gpu" : " 17 GB", | 20260507 00:36:46.108 | INFO | "huge" : "", | 20260507 00:36:46.108 | INFO | "huge_shared" : " 27 GB", | 20260507 00:36:46.108 | INFO | "malloc" : " 14 GB", | 20260507 00:36:46.108 | INFO | "pinned" : " 29 GB" | 20260507 00:36:46.108 | INFO | } | 20260507 00:36:46.108 | INFO | Perf report for the last 5.13716 seconds: | 20260507 00:36:46.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:36:46.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:46.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:46.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:46.108 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 00:36:46.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:46.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:46.108 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:36:46.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:46.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:46.109 | INFO | Uploaded 36GiB to GPU in 1.98273s (18.1567GiB/s) | 20260507 00:36:46.109 | INFO | Downloaded 34GiB to GPU in 1.98734s (17.1083GiB/s) | 20260507 00:36:46.109 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:36:46.109 | INFO | Total mem usage: 18 GiB | 20260507 00:36:46.109 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:36:46.109 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -25.5%) | 20260507 00:36:46.109 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:36:46.109 | INFO | High Water Breakdown: | 20260507 00:36:46.109 | INFO | 60.2% : 8.64 GiB ( 75.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:36:46.109 | INFO | 39.6% : 5.69 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:36:46.109 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:36:46.109 | INFO | ------------------------------------------ | 20260507 00:36:46.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:36:46.109 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:36:46.109 | INFO | Total mem usage: 26.5 GiB | 20260507 00:36:46.109 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:36:46.109 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:36:46.109 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:36:46.109 | INFO | High Water Breakdown: | 20260507 00:36:46.109 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:36:46.109 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:36:46.109 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:36:46.109 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:36:46.109 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:36:46.109 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:36:46.109 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:36:46.109 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:36:46.109 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:36:46.109 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:36:46.109 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:36:46.109 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:36:46.109 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:36:46.109 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:36:46.109 | INFO | ------------------------------------------ | 20260507 00:36:46.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:36:46.109 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:36:46.109 | INFO | Total mem usage: 25.6 GiB | 20260507 00:36:46.109 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:36:46.109 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:36:46.109 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:36:46.109 | INFO | High Water Breakdown: | 20260507 00:36:46.109 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:36:46.109 | INFO | ------------------------------------------ | 20260507 00:36:46.109 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:36:46.109 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:36:46.109 | INFO | Total mem usage: 15.7 GiB | 20260507 00:36:46.109 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:36:46.109 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:36:46.109 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 00:36:46.109 | INFO | High Water Breakdown: | 20260507 00:36:46.109 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:36:46.109 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:36:46.109 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:36:46.109 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:36:46.109 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:36:46.109 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:36:46.109 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:36:46.109 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:36:46.109 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:36:46.109 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:36:46.109 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:36:46.109 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:36:46.109 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:36:46.109 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:36:46.109 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:36:46.109 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:36:46.110 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:36:46.110 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:36:46.110 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:36:46.110 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:36:46.110 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:36:46.110 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:36:46.110 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:36:46.110 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:36:46.110 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:36:46.110 | INFO | ------------------------------------------ | 20260507 00:36:46.110 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:36:46.110 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:36:46.110 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:36:46.110 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:36:46.110 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.62e+01, 1.35e+01 / 1.29e+00 | 20260507 00:36:46.110 | INFO | PerfReport: 0.29%, AggregateStats -- 9.12e-02 / 1.02e-01, 9.82e-03 / 4.45e-03 | 20260507 00:36:46.110 | INFO | PerfReport: 11.65%, Baseline -- 3.60e+00 / 4.51e+00, 5.32e-01 / 1.96e-01 | 20260507 00:36:46.110 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.49e+00, 1.75e-01 / 9.00e-02 | 20260507 00:36:46.110 | INFO | PerfReport: 16.12%, DME -- 3.99e+01 / 4.50e+01, 4.16e+00 / 2.01e+00 | 20260507 00:36:46.110 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.35e+00 / 1.68e+00 | 20260507 00:36:46.110 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.17e+00 / 2.75e+00, 3.80e-01 / 1.78e-01 | 20260507 00:36:46.110 | INFO | PerfReport: 3.14%, Metrics -- 9.72e-01 / 1.32e+00, 2.60e-01 / 1.10e-01 | 20260507 00:36:46.110 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.31e-01 / 3.82e-01, 5.29e-02 / 2.23e-02 | 20260507 00:36:46.110 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:36:46.110 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:36:46.110 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:36:46.110 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 00:36:46.110 | INFO | PerfReport: 21.61%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260507 00:36:46.110 | INFO | PerfReport: 21.31%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260507 00:36:46.110 | INFO | PerfReport: 32.03%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:36:46.110 | INFO | PerfReport: 1.39%, LockUpload -- 1.34e+00 / 0.00e+00, 5.00e+00 / 0.00e+00 | 20260507 00:36:46.110 | INFO | PerfReport: 1.29%, LockDownload -- 1.25e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260507 00:36:46.110 | INFO | PerfReport: 22.36%, LockCompute -- 2.16e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 00:36:46.110 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260507 00:36:46.206 | INFO | Starting a round of Baz IO | 20260507 00:36:49.331 | INFO | Populating RTMetrics report | 20260507 00:36:49.332 | INFO | RTMetrics report populated | 20260507 00:36:49.332 | INFO | Opening tmp RTMetrics json file | 20260507 00:36:49.334 | INFO | Serializing RTMetrics to json | 20260507 00:36:49.352 | INFO | Serialization done, swapping tmp file now... | 20260507 00:36:49.353 | INFO | RTMetrics done | 20260507 00:36:49.426 | INFO | Finished a round of Baz IO: Wrote 1.84 GiB in 3.219 seconds | 20260507 00:36:49.426 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.26% / 0.00% / 66.74% | 20260507 00:36:49.426 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.334187% / 0% / 8.34465e-05% | 20260507 00:36:49.426 | INFO | Baz IO overall: | 20260507 00:36:49.426 | INFO | BytesWritten : 2974037660433 | 20260507 00:36:49.426 | INFO | NumEvents : 1432507723940 | 20260507 00:36:51.244 | INFO | Analyzing chunk frames = [5709824,5710336) | 20260507 00:36:51.244 | INFO | MemInfo: { | 20260507 00:36:51.244 | INFO | "gpu" : " 17 GB", | 20260507 00:36:51.244 | INFO | "huge" : "", | 20260507 00:36:51.244 | INFO | "huge_shared" : " 27 GB", | 20260507 00:36:51.244 | INFO | "malloc" : " 14 GB", | 20260507 00:36:51.244 | INFO | "pinned" : " 29 GB" | 20260507 00:36:51.244 | INFO | } | 20260507 00:36:51.244 | INFO | Perf report for the last 5.13606 seconds: | 20260507 00:36:51.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:36:51.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:51.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:51.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:51.244 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 00:36:51.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:51.244 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 00:36:51.244 | INFO | PRE_HQ : 59.8%, 1.8, 56.5ms, 0 | 20260507 00:36:51.244 | INFO | BAZWRITER : 62.3%, 3.4, 108.0ms, 0 | 20260507 00:36:51.244 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:36:51.245 | INFO | Uploaded 36GiB to GPU in 2.02648s (17.7648GiB/s) | 20260507 00:36:51.245 | INFO | Downloaded 38GiB to GPU in 2.17674s (17.4573GiB/s) | 20260507 00:36:56.386 | INFO | Analyzing chunk frames = [5710336,5710848) | 20260507 00:36:56.386 | INFO | MemInfo: { | 20260507 00:36:56.386 | INFO | "gpu" : " 17 GB", | 20260507 00:36:56.386 | INFO | "huge" : "", | 20260507 00:36:56.386 | INFO | "huge_shared" : " 27 GB", | 20260507 00:36:56.386 | INFO | "malloc" : " 14 GB", | 20260507 00:36:56.386 | INFO | "pinned" : " 29 GB" | 20260507 00:36:56.386 | INFO | } | 20260507 00:36:56.386 | INFO | Perf report for the last 5.14207 seconds: | 20260507 00:36:56.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:36:56.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:56.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:56.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:56.387 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 00:36:56.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:56.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:56.387 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 00:36:56.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:56.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:36:56.387 | INFO | Uploaded 36GiB to GPU in 1.98452s (18.1404GiB/s) | 20260507 00:36:56.387 | INFO | Downloaded 34GiB to GPU in 1.93047s (17.6123GiB/s) | 20260507 00:37:01.523 | INFO | Analyzing chunk frames = [5710848,5711360) | 20260507 00:37:01.523 | INFO | MemInfo: { | 20260507 00:37:01.523 | INFO | "gpu" : " 17 GB", | 20260507 00:37:01.523 | INFO | "huge" : "", | 20260507 00:37:01.523 | INFO | "huge_shared" : " 27 GB", | 20260507 00:37:01.523 | INFO | "malloc" : " 14 GB", | 20260507 00:37:01.523 | INFO | "pinned" : " 29 GB" | 20260507 00:37:01.523 | INFO | } | 20260507 00:37:01.524 | INFO | Perf report for the last 5.13724 seconds: | 20260507 00:37:01.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:37:01.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:01.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:01.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:01.524 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:37:01.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:01.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:01.524 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:37:01.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:01.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:01.524 | INFO | Uploaded 36GiB to GPU in 1.97919s (18.1893GiB/s) | 20260507 00:37:01.524 | INFO | Downloaded 34GiB to GPU in 1.93047s (17.6123GiB/s) | 20260507 00:37:06.670 | INFO | Analyzing chunk frames = [5711360,5711872) | 20260507 00:37:06.670 | INFO | MemInfo: { | 20260507 00:37:06.670 | INFO | "gpu" : " 17 GB", | 20260507 00:37:06.670 | INFO | "huge" : "", | 20260507 00:37:06.670 | INFO | "huge_shared" : " 27 GB", | 20260507 00:37:06.670 | INFO | "malloc" : " 14 GB", | 20260507 00:37:06.670 | INFO | "pinned" : " 29 GB" | 20260507 00:37:06.670 | INFO | } | 20260507 00:37:06.670 | INFO | Perf report for the last 5.14628 seconds: | 20260507 00:37:06.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:37:06.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:06.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:06.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:06.670 | INFO | ANALYSIS : 59.3%, 3.0, 92.1ms, 0 | 20260507 00:37:06.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:06.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:06.670 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:37:06.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:06.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:06.670 | INFO | Uploaded 36GiB to GPU in 1.94065s (18.5504GiB/s) | 20260507 00:37:06.670 | INFO | Downloaded 34GiB to GPU in 1.8839s (18.0477GiB/s) | 20260507 00:37:11.810 | INFO | Analyzing chunk frames = [5711872,5712384) | 20260507 00:37:11.810 | INFO | MemInfo: { | 20260507 00:37:11.810 | INFO | "gpu" : " 17 GB", | 20260507 00:37:11.810 | INFO | "huge" : "", | 20260507 00:37:11.810 | INFO | "huge_shared" : " 27 GB", | 20260507 00:37:11.810 | INFO | "malloc" : " 14 GB", | 20260507 00:37:11.810 | INFO | "pinned" : " 29 GB" | 20260507 00:37:11.810 | INFO | } | 20260507 00:37:11.810 | INFO | Perf report for the last 5.14059 seconds: | 20260507 00:37:11.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:37:11.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:11.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:11.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:11.811 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 00:37:11.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:11.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:11.811 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:37:11.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:11.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:11.811 | INFO | Uploaded 36GiB to GPU in 1.98995s (18.0909GiB/s) | 20260507 00:37:11.811 | INFO | Downloaded 34GiB to GPU in 1.95267s (17.4121GiB/s) | 20260507 00:37:16.945 | INFO | Analyzing chunk frames = [5712384,5712896) | 20260507 00:37:16.945 | INFO | MemInfo: { | 20260507 00:37:16.945 | INFO | "gpu" : " 17 GB", | 20260507 00:37:16.945 | INFO | "huge" : "", | 20260507 00:37:16.945 | INFO | "huge_shared" : " 27 GB", | 20260507 00:37:16.945 | INFO | "malloc" : " 14 GB", | 20260507 00:37:16.945 | INFO | "pinned" : " 29 GB" | 20260507 00:37:16.945 | INFO | } | 20260507 00:37:16.945 | INFO | Perf report for the last 5.1345 seconds: | 20260507 00:37:16.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:37:16.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:16.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:16.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:16.945 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:37:16.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:16.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:16.945 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 00:37:16.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:16.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:16.945 | INFO | Uploaded 36GiB to GPU in 2.04921s (17.5677GiB/s) | 20260507 00:37:16.945 | INFO | Downloaded 34GiB to GPU in 1.95782s (17.3663GiB/s) | 20260507 00:37:22.079 | INFO | Analyzing chunk frames = [5712896,5713408) | 20260507 00:37:22.079 | INFO | MemInfo: { | 20260507 00:37:22.079 | INFO | "gpu" : " 17 GB", | 20260507 00:37:22.079 | INFO | "huge" : "", | 20260507 00:37:22.079 | INFO | "huge_shared" : " 27 GB", | 20260507 00:37:22.079 | INFO | "malloc" : " 14 GB", | 20260507 00:37:22.079 | INFO | "pinned" : " 29 GB" | 20260507 00:37:22.079 | INFO | } | 20260507 00:37:22.079 | INFO | Perf report for the last 5.13392 seconds: | 20260507 00:37:22.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:37:22.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:22.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:22.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:22.079 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 00:37:22.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:22.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:22.079 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 00:37:22.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:22.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:22.079 | INFO | Uploaded 36GiB to GPU in 1.96393s (18.3306GiB/s) | 20260507 00:37:22.079 | INFO | Downloaded 34GiB to GPU in 1.99458s (17.0462GiB/s) | 20260507 00:37:27.221 | INFO | Analyzing chunk frames = [5713408,5713920) | 20260507 00:37:27.221 | INFO | MemInfo: { | 20260507 00:37:27.221 | INFO | "gpu" : " 17 GB", | 20260507 00:37:27.221 | INFO | "huge" : "", | 20260507 00:37:27.221 | INFO | "huge_shared" : " 27 GB", | 20260507 00:37:27.221 | INFO | "malloc" : " 14 GB", | 20260507 00:37:27.221 | INFO | "pinned" : " 29 GB" | 20260507 00:37:27.221 | INFO | } | 20260507 00:37:27.221 | INFO | Perf report for the last 5.14231 seconds: | 20260507 00:37:27.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:37:27.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:27.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:27.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:27.221 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:37:27.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:27.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:27.221 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:37:27.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:27.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:27.222 | INFO | Uploaded 36GiB to GPU in 1.9885s (18.1041GiB/s) | 20260507 00:37:27.222 | INFO | Downloaded 34GiB to GPU in 1.98772s (17.1051GiB/s) | 20260507 00:37:27.318 | INFO | Starting a round of Baz IO | 20260507 00:37:30.444 | INFO | Populating RTMetrics report | 20260507 00:37:30.444 | INFO | RTMetrics report populated | 20260507 00:37:30.444 | INFO | Opening tmp RTMetrics json file | 20260507 00:37:30.445 | INFO | Serializing RTMetrics to json | 20260507 00:37:30.446 | INFO | Serialization done, swapping tmp file now... | 20260507 00:37:30.447 | INFO | RTMetrics done | 20260507 00:37:30.541 | INFO | Finished a round of Baz IO: Wrote 1.838 GiB in 3.223 seconds | 20260507 00:37:30.541 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.26% / 0.00% / 66.74% | 20260507 00:37:30.541 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.332145% / 0% / 6.35783e-05% | 20260507 00:37:30.542 | INFO | Baz IO overall: | 20260507 00:37:30.542 | INFO | BytesWritten : 2976011668316 | 20260507 00:37:30.542 | INFO | NumEvents : 1433444076631 | 20260507 00:37:32.362 | INFO | Analyzing chunk frames = [5713920,5714432) | 20260507 00:37:32.362 | INFO | MemInfo: { | 20260507 00:37:32.362 | INFO | "gpu" : " 17 GB", | 20260507 00:37:32.362 | INFO | "huge" : "", | 20260507 00:37:32.362 | INFO | "huge_shared" : " 27 GB", | 20260507 00:37:32.362 | INFO | "malloc" : " 14 GB", | 20260507 00:37:32.362 | INFO | "pinned" : " 29 GB" | 20260507 00:37:32.362 | INFO | } | 20260507 00:37:32.362 | INFO | Perf report for the last 5.14102 seconds: | 20260507 00:37:32.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:37:32.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:32.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:32.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:32.362 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 00:37:32.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:32.362 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 00:37:32.362 | INFO | PRE_HQ : 59.7%, 1.8, 56.6ms, 0 | 20260507 00:37:32.362 | INFO | BAZWRITER : 62.3%, 3.4, 108.8ms, 0 | 20260507 00:37:32.362 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:37:32.362 | INFO | Uploaded 36GiB to GPU in 2.02925s (17.7406GiB/s) | 20260507 00:37:32.363 | INFO | Downloaded 38GiB to GPU in 2.17409s (17.4786GiB/s) | 20260507 00:37:37.501 | INFO | Analyzing chunk frames = [5714432,5714944) | 20260507 00:37:37.501 | INFO | MemInfo: { | 20260507 00:37:37.501 | INFO | "gpu" : " 17 GB", | 20260507 00:37:37.501 | INFO | "huge" : "", | 20260507 00:37:37.501 | INFO | "huge_shared" : " 27 GB", | 20260507 00:37:37.501 | INFO | "malloc" : " 14 GB", | 20260507 00:37:37.501 | INFO | "pinned" : " 29 GB" | 20260507 00:37:37.501 | INFO | } | 20260507 00:37:37.501 | INFO | Perf report for the last 5.13919 seconds: | 20260507 00:37:37.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:37:37.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:37.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:37.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:37.502 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 00:37:37.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:37.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:37.502 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:37:37.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:37.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:37.502 | INFO | Uploaded 36GiB to GPU in 2.0027s (17.9758GiB/s) | 20260507 00:37:37.502 | INFO | Downloaded 34GiB to GPU in 1.9227s (17.6835GiB/s) | 20260507 00:37:42.652 | INFO | Analyzing chunk frames = [5714944,5715456) | 20260507 00:37:42.652 | INFO | MemInfo: { | 20260507 00:37:42.652 | INFO | "gpu" : " 17 GB", | 20260507 00:37:42.652 | INFO | "huge" : "", | 20260507 00:37:42.652 | INFO | "huge_shared" : " 27 GB", | 20260507 00:37:42.652 | INFO | "malloc" : " 14 GB", | 20260507 00:37:42.652 | INFO | "pinned" : " 29 GB" | 20260507 00:37:42.652 | INFO | } | 20260507 00:37:42.652 | INFO | Perf report for the last 5.15013 seconds: | 20260507 00:37:42.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:37:42.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:42.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:42.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:42.652 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 00:37:42.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:42.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:42.652 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:37:42.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:42.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:42.652 | INFO | Uploaded 36GiB to GPU in 2.0098s (17.9122GiB/s) | 20260507 00:37:42.652 | INFO | Downloaded 34GiB to GPU in 1.9338s (17.582GiB/s) | 20260507 00:37:47.785 | INFO | Analyzing chunk frames = [5715456,5715968) | 20260507 00:37:47.785 | INFO | MemInfo: { | 20260507 00:37:47.785 | INFO | "gpu" : " 17 GB", | 20260507 00:37:47.785 | INFO | "huge" : "", | 20260507 00:37:47.785 | INFO | "huge_shared" : " 27 GB", | 20260507 00:37:47.785 | INFO | "malloc" : " 14 GB", | 20260507 00:37:47.785 | INFO | "pinned" : " 29 GB" | 20260507 00:37:47.785 | INFO | } | 20260507 00:37:47.785 | INFO | Perf report for the last 5.13377 seconds: | 20260507 00:37:47.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:37:47.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:47.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:47.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:47.785 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 00:37:47.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:47.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:47.786 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:37:47.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:47.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:47.786 | INFO | Uploaded 36GiB to GPU in 1.98187s (18.1646GiB/s) | 20260507 00:37:47.786 | INFO | Downloaded 34GiB to GPU in 1.91668s (17.739GiB/s) | 20260507 00:37:52.924 | INFO | Analyzing chunk frames = [5715968,5716480) | 20260507 00:37:52.924 | INFO | MemInfo: { | 20260507 00:37:52.924 | INFO | "gpu" : " 17 GB", | 20260507 00:37:52.924 | INFO | "huge" : "", | 20260507 00:37:52.924 | INFO | "huge_shared" : " 27 GB", | 20260507 00:37:52.924 | INFO | "malloc" : " 14 GB", | 20260507 00:37:52.924 | INFO | "pinned" : " 29 GB" | 20260507 00:37:52.924 | INFO | } | 20260507 00:37:52.925 | INFO | Perf report for the last 5.13907 seconds: | 20260507 00:37:52.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:37:52.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:52.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:52.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:52.925 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 00:37:52.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:52.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:52.925 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:37:52.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:52.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:52.925 | INFO | Uploaded 36GiB to GPU in 2.0241s (17.7857GiB/s) | 20260507 00:37:52.925 | INFO | Downloaded 34GiB to GPU in 1.94174s (17.5101GiB/s) | 20260507 00:37:58.057 | INFO | Analyzing chunk frames = [5716480,5716992) | 20260507 00:37:58.057 | INFO | MemInfo: { | 20260507 00:37:58.057 | INFO | "gpu" : " 17 GB", | 20260507 00:37:58.057 | INFO | "huge" : "", | 20260507 00:37:58.057 | INFO | "huge_shared" : " 27 GB", | 20260507 00:37:58.057 | INFO | "malloc" : " 14 GB", | 20260507 00:37:58.057 | INFO | "pinned" : " 29 GB" | 20260507 00:37:58.057 | INFO | } | 20260507 00:37:58.058 | INFO | Perf report for the last 5.13302 seconds: | 20260507 00:37:58.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:37:58.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:58.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:58.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:58.058 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 00:37:58.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:58.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:58.058 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 00:37:58.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:58.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:37:58.058 | INFO | Uploaded 36GiB to GPU in 2.0589s (17.485GiB/s) | 20260507 00:37:58.058 | INFO | Downloaded 34GiB to GPU in 1.95726s (17.3712GiB/s) | 20260507 00:38:03.192 | INFO | Analyzing chunk frames = [5716992,5717504) | 20260507 00:38:03.192 | INFO | MemInfo: { | 20260507 00:38:03.192 | INFO | "gpu" : " 17 GB", | 20260507 00:38:03.192 | INFO | "huge" : "", | 20260507 00:38:03.192 | INFO | "huge_shared" : " 27 GB", | 20260507 00:38:03.192 | INFO | "malloc" : " 14 GB", | 20260507 00:38:03.192 | INFO | "pinned" : " 29 GB" | 20260507 00:38:03.192 | INFO | } | 20260507 00:38:03.193 | INFO | Perf report for the last 5.13503 seconds: | 20260507 00:38:03.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:38:03.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:03.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:03.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:03.193 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260507 00:38:03.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:03.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:03.193 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 00:38:03.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:03.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:03.193 | INFO | Uploaded 36GiB to GPU in 1.98745s (18.1137GiB/s) | 20260507 00:38:03.193 | INFO | Downloaded 34GiB to GPU in 2.01609s (16.8644GiB/s) | 20260507 00:38:08.337 | INFO | Analyzing chunk frames = [5717504,5718016) | 20260507 00:38:08.337 | INFO | MemInfo: { | 20260507 00:38:08.337 | INFO | "gpu" : " 17 GB", | 20260507 00:38:08.337 | INFO | "huge" : "", | 20260507 00:38:08.337 | INFO | "huge_shared" : " 27 GB", | 20260507 00:38:08.337 | INFO | "malloc" : " 14 GB", | 20260507 00:38:08.337 | INFO | "pinned" : " 29 GB" | 20260507 00:38:08.337 | INFO | } | 20260507 00:38:08.338 | INFO | Perf report for the last 5.14496 seconds: | 20260507 00:38:08.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:38:08.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:08.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:08.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:08.338 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 00:38:08.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:08.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:08.338 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 00:38:08.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:08.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:08.338 | INFO | Uploaded 36GiB to GPU in 1.97253s (18.2507GiB/s) | 20260507 00:38:08.338 | INFO | Downloaded 34GiB to GPU in 1.98833s (17.0998GiB/s) | 20260507 00:38:08.338 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:38:08.338 | INFO | Total mem usage: 18 GiB | 20260507 00:38:08.338 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:38:08.338 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 00:38:08.338 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:38:08.338 | INFO | High Water Breakdown: | 20260507 00:38:08.338 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:38:08.338 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:38:08.338 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:38:08.338 | INFO | ------------------------------------------ | 20260507 00:38:08.338 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:38:08.338 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:38:08.338 | INFO | Total mem usage: 26.5 GiB | 20260507 00:38:08.338 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:38:08.338 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:38:08.338 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:38:08.338 | INFO | High Water Breakdown: | 20260507 00:38:08.338 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:38:08.338 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:38:08.338 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:38:08.338 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:38:08.338 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:38:08.338 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:38:08.338 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:38:08.338 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:38:08.338 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:38:08.338 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:38:08.338 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:38:08.338 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:38:08.338 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:38:08.338 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:38:08.338 | INFO | ------------------------------------------ | 20260507 00:38:08.338 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:38:08.338 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:38:08.338 | INFO | Total mem usage: 25.6 GiB | 20260507 00:38:08.338 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:38:08.338 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:38:08.338 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:38:08.338 | INFO | High Water Breakdown: | 20260507 00:38:08.338 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:38:08.339 | INFO | ------------------------------------------ | 20260507 00:38:08.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:38:08.339 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:38:08.339 | INFO | Total mem usage: 15.7 GiB | 20260507 00:38:08.339 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:38:08.339 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:38:08.339 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:38:08.339 | INFO | High Water Breakdown: | 20260507 00:38:08.339 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:38:08.339 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:38:08.339 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:38:08.339 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:38:08.339 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:38:08.339 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:38:08.339 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:38:08.339 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:38:08.339 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:38:08.339 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:38:08.339 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:38:08.339 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:38:08.339 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:38:08.339 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:38:08.339 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:38:08.339 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:38:08.339 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:38:08.339 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:38:08.339 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:38:08.339 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:38:08.339 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:38:08.339 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:38:08.339 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:38:08.339 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:38:08.339 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:38:08.339 | INFO | ------------------------------------------ | 20260507 00:38:08.339 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:38:08.339 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:38:08.339 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:38:08.339 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:38:08.339 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.55e+01, 1.35e+01 / 2.19e+00 | 20260507 00:38:08.339 | INFO | PerfReport: 0.30%, AggregateStats -- 9.12e-02 / 1.02e-01, 1.03e-02 / 5.99e-03 | 20260507 00:38:08.339 | INFO | PerfReport: 11.65%, Baseline -- 3.60e+00 / 4.41e+00, 5.28e-01 / 3.19e-01 | 20260507 00:38:08.339 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.49e+00, 1.75e-01 / 1.10e-01 | 20260507 00:38:08.339 | INFO | PerfReport: 16.12%, DME -- 3.98e+01 / 4.48e+01, 4.20e+00 / 2.26e+00 | 20260507 00:38:08.339 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.30e+00 / 1.50e+00 | 20260507 00:38:08.339 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.18e+00 / 2.81e+00, 3.89e-01 / 1.64e-01 | 20260507 00:38:08.339 | INFO | PerfReport: 3.15%, Metrics -- 9.73e-01 / 1.33e+00, 2.83e-01 / 1.58e-01 | 20260507 00:38:08.339 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.30e-01 / 3.84e-01, 5.20e-02 / 2.69e-02 | 20260507 00:38:08.339 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:38:08.339 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:38:08.339 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:38:08.339 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 00:38:08.339 | INFO | PerfReport: 21.58%, Upload -- 2.08e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260507 00:38:08.339 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260507 00:38:08.339 | INFO | PerfReport: 32.01%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:38:08.339 | INFO | PerfReport: 1.32%, LockUpload -- 1.27e+00 / 0.00e+00, 4.70e+00 / 0.00e+00 | 20260507 00:38:08.339 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260507 00:38:08.339 | INFO | PerfReport: 22.40%, LockCompute -- 2.16e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 00:38:08.339 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260507 00:38:08.434 | INFO | Starting a round of Baz IO | 20260507 00:38:11.569 | INFO | Populating RTMetrics report | 20260507 00:38:11.569 | INFO | RTMetrics report populated | 20260507 00:38:11.569 | INFO | Opening tmp RTMetrics json file | 20260507 00:38:11.588 | INFO | Serializing RTMetrics to json | 20260507 00:38:11.590 | INFO | Serialization done, swapping tmp file now... | 20260507 00:38:11.590 | INFO | RTMetrics done | 20260507 00:38:11.659 | INFO | Finished a round of Baz IO: Wrote 1.839 GiB in 3.224 seconds | 20260507 00:38:11.659 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.26% / 0.00% / 66.74% | 20260507 00:38:11.659 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.332065% / 7.94729e-06% / 7.54992e-05% | 20260507 00:38:11.659 | INFO | Baz IO overall: | 20260507 00:38:11.659 | INFO | BytesWritten : 2977986044567 | 20260507 00:38:11.659 | INFO | NumEvents : 1434381319682 | 20260507 00:38:13.476 | INFO | Analyzing chunk frames = [5718016,5718528) | 20260507 00:38:13.476 | INFO | MemInfo: { | 20260507 00:38:13.476 | INFO | "gpu" : " 17 GB", | 20260507 00:38:13.476 | INFO | "huge" : "", | 20260507 00:38:13.476 | INFO | "huge_shared" : " 27 GB", | 20260507 00:38:13.476 | INFO | "malloc" : " 14 GB", | 20260507 00:38:13.476 | INFO | "pinned" : " 29 GB" | 20260507 00:38:13.476 | INFO | } | 20260507 00:38:13.476 | INFO | Perf report for the last 5.13808 seconds: | 20260507 00:38:13.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:38:13.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:13.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:13.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:13.476 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 00:38:13.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:13.476 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260507 00:38:13.476 | INFO | PRE_HQ : 59.9%, 1.9, 59.6ms, 0 | 20260507 00:38:13.476 | INFO | BAZWRITER : 62.3%, 3.4, 110.3ms, 0 | 20260507 00:38:13.476 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 00:38:13.476 | INFO | Uploaded 36GiB to GPU in 2.00838s (17.9249GiB/s) | 20260507 00:38:13.476 | INFO | Downloaded 38GiB to GPU in 2.18629s (17.3811GiB/s) | 20260507 00:38:18.622 | INFO | Analyzing chunk frames = [5718528,5719040) | 20260507 00:38:18.622 | INFO | MemInfo: { | 20260507 00:38:18.622 | INFO | "gpu" : " 17 GB", | 20260507 00:38:18.622 | INFO | "huge" : "", | 20260507 00:38:18.622 | INFO | "huge_shared" : " 27 GB", | 20260507 00:38:18.622 | INFO | "malloc" : " 14 GB", | 20260507 00:38:18.622 | INFO | "pinned" : " 29 GB" | 20260507 00:38:18.622 | INFO | } | 20260507 00:38:18.622 | INFO | Perf report for the last 5.14655 seconds: | 20260507 00:38:18.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:38:18.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:18.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:18.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:18.622 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260507 00:38:18.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:18.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:18.622 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 00:38:18.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:18.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:18.622 | INFO | Uploaded 36GiB to GPU in 2.01423s (17.8729GiB/s) | 20260507 00:38:18.623 | INFO | Downloaded 34GiB to GPU in 1.9334s (17.5856GiB/s) | 20260507 00:38:23.761 | INFO | Analyzing chunk frames = [5719040,5719552) | 20260507 00:38:23.761 | INFO | MemInfo: { | 20260507 00:38:23.761 | INFO | "gpu" : " 17 GB", | 20260507 00:38:23.761 | INFO | "huge" : "", | 20260507 00:38:23.761 | INFO | "huge_shared" : " 27 GB", | 20260507 00:38:23.761 | INFO | "malloc" : " 14 GB", | 20260507 00:38:23.761 | INFO | "pinned" : " 29 GB" | 20260507 00:38:23.761 | INFO | } | 20260507 00:38:23.762 | INFO | Perf report for the last 5.13933 seconds: | 20260507 00:38:23.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:38:23.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:23.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:23.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:23.762 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:38:23.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:23.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:23.762 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:38:23.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:23.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:23.762 | INFO | Uploaded 36GiB to GPU in 1.99026s (18.0881GiB/s) | 20260507 00:38:23.762 | INFO | Downloaded 34GiB to GPU in 1.93638s (17.5585GiB/s) | 20260507 00:38:28.895 | INFO | Analyzing chunk frames = [5719552,5720064) | 20260507 00:38:28.895 | INFO | MemInfo: { | 20260507 00:38:28.895 | INFO | "gpu" : " 17 GB", | 20260507 00:38:28.895 | INFO | "huge" : "", | 20260507 00:38:28.895 | INFO | "huge_shared" : " 27 GB", | 20260507 00:38:28.895 | INFO | "malloc" : " 14 GB", | 20260507 00:38:28.895 | INFO | "pinned" : " 29 GB" | 20260507 00:38:28.895 | INFO | } | 20260507 00:38:28.896 | INFO | Perf report for the last 5.13398 seconds: | 20260507 00:38:28.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:38:28.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:28.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:28.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:28.896 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260507 00:38:28.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:28.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:28.896 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:38:28.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:28.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:28.896 | INFO | Uploaded 36GiB to GPU in 1.96553s (18.3157GiB/s) | 20260507 00:38:28.896 | INFO | Downloaded 34GiB to GPU in 1.92114s (17.6978GiB/s) | 20260507 00:38:34.033 | INFO | Analyzing chunk frames = [5720064,5720576) | 20260507 00:38:34.033 | INFO | MemInfo: { | 20260507 00:38:34.033 | INFO | "gpu" : " 17 GB", | 20260507 00:38:34.033 | INFO | "huge" : "", | 20260507 00:38:34.033 | INFO | "huge_shared" : " 27 GB", | 20260507 00:38:34.033 | INFO | "malloc" : " 14 GB", | 20260507 00:38:34.033 | INFO | "pinned" : " 29 GB" | 20260507 00:38:34.033 | INFO | } | 20260507 00:38:34.033 | INFO | Perf report for the last 5.13781 seconds: | 20260507 00:38:34.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:38:34.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:34.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:34.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:34.033 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 00:38:34.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:34.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:34.033 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:38:34.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:34.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:34.034 | INFO | Uploaded 36GiB to GPU in 2.00418s (17.9624GiB/s) | 20260507 00:38:34.034 | INFO | Downloaded 34GiB to GPU in 1.9454s (17.4771GiB/s) | 20260507 00:38:39.169 | INFO | Analyzing chunk frames = [5720576,5721088) | 20260507 00:38:39.169 | INFO | MemInfo: { | 20260507 00:38:39.169 | INFO | "gpu" : " 17 GB", | 20260507 00:38:39.169 | INFO | "huge" : "", | 20260507 00:38:39.169 | INFO | "huge_shared" : " 27 GB", | 20260507 00:38:39.169 | INFO | "malloc" : " 14 GB", | 20260507 00:38:39.169 | INFO | "pinned" : " 29 GB" | 20260507 00:38:39.169 | INFO | } | 20260507 00:38:39.169 | INFO | Perf report for the last 5.13592 seconds: | 20260507 00:38:39.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:38:39.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:39.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:39.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:39.169 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 00:38:39.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:39.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:39.169 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:38:39.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:39.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:39.170 | INFO | Uploaded 36GiB to GPU in 2.00887s (17.9205GiB/s) | 20260507 00:38:39.170 | INFO | Downloaded 34GiB to GPU in 1.93434s (17.577GiB/s) | 20260507 00:38:44.312 | INFO | Analyzing chunk frames = [5721088,5721600) | 20260507 00:38:44.312 | INFO | MemInfo: { | 20260507 00:38:44.312 | INFO | "gpu" : " 17 GB", | 20260507 00:38:44.312 | INFO | "huge" : "", | 20260507 00:38:44.312 | INFO | "huge_shared" : " 27 GB", | 20260507 00:38:44.312 | INFO | "malloc" : " 14 GB", | 20260507 00:38:44.312 | INFO | "pinned" : " 29 GB" | 20260507 00:38:44.312 | INFO | } | 20260507 00:38:44.312 | INFO | Perf report for the last 5.14254 seconds: | 20260507 00:38:44.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:38:44.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:44.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:44.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:44.312 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 00:38:44.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:44.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:44.312 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:38:44.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:44.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:44.312 | INFO | Uploaded 36GiB to GPU in 1.9601s (18.3664GiB/s) | 20260507 00:38:44.312 | INFO | Downloaded 34GiB to GPU in 1.98589s (17.1208GiB/s) | 20260507 00:38:49.452 | INFO | Analyzing chunk frames = [5721600,5722112) | 20260507 00:38:49.452 | INFO | MemInfo: { | 20260507 00:38:49.452 | INFO | "gpu" : " 17 GB", | 20260507 00:38:49.452 | INFO | "huge" : "", | 20260507 00:38:49.452 | INFO | "huge_shared" : " 27 GB", | 20260507 00:38:49.453 | INFO | "malloc" : " 14 GB", | 20260507 00:38:49.453 | INFO | "pinned" : " 29 GB" | 20260507 00:38:49.453 | INFO | } | 20260507 00:38:49.453 | INFO | Perf report for the last 5.14077 seconds: | 20260507 00:38:49.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:38:49.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:49.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:49.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:49.453 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 00:38:49.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:49.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:49.453 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 00:38:49.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:49.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:49.453 | INFO | Uploaded 36GiB to GPU in 2.00718s (17.9356GiB/s) | 20260507 00:38:49.453 | INFO | Downloaded 34GiB to GPU in 1.98769s (17.1053GiB/s) | 20260507 00:38:49.551 | INFO | Starting a round of Baz IO | 20260507 00:38:52.676 | INFO | Populating RTMetrics report | 20260507 00:38:52.676 | INFO | RTMetrics report populated | 20260507 00:38:52.676 | INFO | Opening tmp RTMetrics json file | 20260507 00:38:52.677 | INFO | Serializing RTMetrics to json | 20260507 00:38:52.678 | INFO | Serialization done, swapping tmp file now... | 20260507 00:38:52.679 | INFO | RTMetrics done | 20260507 00:38:52.766 | INFO | Finished a round of Baz IO: Wrote 1.839 GiB in 3.215 seconds | 20260507 00:38:52.766 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.26% / 0.00% / 66.74% | 20260507 00:38:52.766 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.329586% / 0% / 6.75519e-05% | 20260507 00:38:52.767 | INFO | Baz IO overall: | 20260507 00:38:52.767 | INFO | BytesWritten : 2979960845263 | 20260507 00:38:52.767 | INFO | NumEvents : 1435318790278 | 20260507 00:38:54.599 | INFO | Analyzing chunk frames = [5722112,5722624) | 20260507 00:38:54.599 | INFO | MemInfo: { | 20260507 00:38:54.599 | INFO | "gpu" : " 17 GB", | 20260507 00:38:54.599 | INFO | "huge" : "", | 20260507 00:38:54.599 | INFO | "huge_shared" : " 27 GB", | 20260507 00:38:54.599 | INFO | "malloc" : " 14 GB", | 20260507 00:38:54.599 | INFO | "pinned" : " 29 GB" | 20260507 00:38:54.599 | INFO | } | 20260507 00:38:54.599 | INFO | Perf report for the last 5.14675 seconds: | 20260507 00:38:54.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:38:54.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:54.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:54.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:54.599 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 00:38:54.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:54.599 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 00:38:54.599 | INFO | PRE_HQ : 59.5%, 1.8, 56.3ms, 0 | 20260507 00:38:54.599 | INFO | BAZWRITER : 62.1%, 3.4, 108.3ms, 0 | 20260507 00:38:54.600 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:38:54.600 | INFO | Uploaded 36GiB to GPU in 2.03044s (17.7301GiB/s) | 20260507 00:38:54.600 | INFO | Downloaded 38GiB to GPU in 2.17735s (17.4524GiB/s) | 20260507 00:38:59.734 | INFO | Analyzing chunk frames = [5722624,5723136) | 20260507 00:38:59.734 | INFO | MemInfo: { | 20260507 00:38:59.734 | INFO | "gpu" : " 17 GB", | 20260507 00:38:59.734 | INFO | "huge" : "", | 20260507 00:38:59.734 | INFO | "huge_shared" : " 27 GB", | 20260507 00:38:59.734 | INFO | "malloc" : " 14 GB", | 20260507 00:38:59.735 | INFO | "pinned" : " 29 GB" | 20260507 00:38:59.735 | INFO | } | 20260507 00:38:59.735 | INFO | Perf report for the last 5.13519 seconds: | 20260507 00:38:59.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:38:59.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:59.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:59.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:59.735 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:38:59.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:59.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:59.735 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:38:59.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:59.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:38:59.735 | INFO | Uploaded 36GiB to GPU in 2.01399s (17.875GiB/s) | 20260507 00:38:59.735 | INFO | Downloaded 34GiB to GPU in 1.93156s (17.6023GiB/s) | 20260507 00:39:04.874 | INFO | Analyzing chunk frames = [5723136,5723648) | 20260507 00:39:04.874 | INFO | MemInfo: { | 20260507 00:39:04.874 | INFO | "gpu" : " 17 GB", | 20260507 00:39:04.874 | INFO | "huge" : "", | 20260507 00:39:04.874 | INFO | "huge_shared" : " 27 GB", | 20260507 00:39:04.874 | INFO | "malloc" : " 14 GB", | 20260507 00:39:04.874 | INFO | "pinned" : " 29 GB" | 20260507 00:39:04.874 | INFO | } | 20260507 00:39:04.874 | INFO | Perf report for the last 5.1397 seconds: | 20260507 00:39:04.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:39:04.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:04.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:04.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:04.874 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:39:04.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:04.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:04.874 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:39:04.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:04.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:04.875 | INFO | Uploaded 36GiB to GPU in 1.99318s (18.0615GiB/s) | 20260507 00:39:04.875 | INFO | Downloaded 34GiB to GPU in 1.93366s (17.5833GiB/s) | 20260507 00:39:10.012 | INFO | Analyzing chunk frames = [5723648,5724160) | 20260507 00:39:10.012 | INFO | MemInfo: { | 20260507 00:39:10.012 | INFO | "gpu" : " 17 GB", | 20260507 00:39:10.012 | INFO | "huge" : "", | 20260507 00:39:10.012 | INFO | "huge_shared" : " 27 GB", | 20260507 00:39:10.012 | INFO | "malloc" : " 14 GB", | 20260507 00:39:10.012 | INFO | "pinned" : " 29 GB" | 20260507 00:39:10.012 | INFO | } | 20260507 00:39:10.012 | INFO | Perf report for the last 5.13778 seconds: | 20260507 00:39:10.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:39:10.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:10.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:10.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:10.012 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:39:10.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:10.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:10.012 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:39:10.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:10.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:10.012 | INFO | Uploaded 36GiB to GPU in 1.96329s (18.3366GiB/s) | 20260507 00:39:10.012 | INFO | Downloaded 34GiB to GPU in 1.90594s (17.839GiB/s) | 20260507 00:39:15.142 | INFO | Analyzing chunk frames = [5724160,5724672) | 20260507 00:39:15.142 | INFO | MemInfo: { | 20260507 00:39:15.142 | INFO | "gpu" : " 17 GB", | 20260507 00:39:15.142 | INFO | "huge" : "", | 20260507 00:39:15.142 | INFO | "huge_shared" : " 27 GB", | 20260507 00:39:15.142 | INFO | "malloc" : " 14 GB", | 20260507 00:39:15.142 | INFO | "pinned" : " 29 GB" | 20260507 00:39:15.142 | INFO | } | 20260507 00:39:15.142 | INFO | Perf report for the last 5.12998 seconds: | 20260507 00:39:15.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:39:15.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:15.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:15.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:15.142 | INFO | ANALYSIS : 60.2%, 3.0, 93.5ms, 0 | 20260507 00:39:15.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:15.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:15.142 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:39:15.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:15.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:15.142 | INFO | Uploaded 36GiB to GPU in 2.00398s (17.9643GiB/s) | 20260507 00:39:15.142 | INFO | Downloaded 34GiB to GPU in 1.93392s (17.5809GiB/s) | 20260507 00:39:20.288 | INFO | Analyzing chunk frames = [5724672,5725184) | 20260507 00:39:20.288 | INFO | MemInfo: { | 20260507 00:39:20.288 | INFO | "gpu" : " 17 GB", | 20260507 00:39:20.288 | INFO | "huge" : "", | 20260507 00:39:20.288 | INFO | "huge_shared" : " 27 GB", | 20260507 00:39:20.288 | INFO | "malloc" : " 14 GB", | 20260507 00:39:20.288 | INFO | "pinned" : " 29 GB" | 20260507 00:39:20.288 | INFO | } | 20260507 00:39:20.288 | INFO | Perf report for the last 5.14576 seconds: | 20260507 00:39:20.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:39:20.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:20.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:20.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:20.288 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:39:20.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:20.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:20.288 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:39:20.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:20.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:20.288 | INFO | Uploaded 36GiB to GPU in 2.02077s (17.815GiB/s) | 20260507 00:39:20.288 | INFO | Downloaded 34GiB to GPU in 1.94268s (17.5016GiB/s) | 20260507 00:39:25.431 | INFO | Analyzing chunk frames = [5725184,5725696) | 20260507 00:39:25.431 | INFO | MemInfo: { | 20260507 00:39:25.431 | INFO | "gpu" : " 17 GB", | 20260507 00:39:25.431 | INFO | "huge" : "", | 20260507 00:39:25.431 | INFO | "huge_shared" : " 27 GB", | 20260507 00:39:25.431 | INFO | "malloc" : " 14 GB", | 20260507 00:39:25.431 | INFO | "pinned" : " 29 GB" | 20260507 00:39:25.431 | INFO | } | 20260507 00:39:25.431 | INFO | Perf report for the last 5.14296 seconds: | 20260507 00:39:25.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:39:25.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:25.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:25.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:25.431 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:39:25.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:25.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:25.431 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 00:39:25.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:25.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:25.431 | INFO | Uploaded 36GiB to GPU in 1.96282s (18.341GiB/s) | 20260507 00:39:25.431 | INFO | Downloaded 34GiB to GPU in 1.99127s (17.0745GiB/s) | 20260507 00:39:30.574 | INFO | Analyzing chunk frames = [5725696,5726208) | 20260507 00:39:30.574 | INFO | MemInfo: { | 20260507 00:39:30.574 | INFO | "gpu" : " 17 GB", | 20260507 00:39:30.574 | INFO | "huge" : "", | 20260507 00:39:30.574 | INFO | "huge_shared" : " 27 GB", | 20260507 00:39:30.574 | INFO | "malloc" : " 14 GB", | 20260507 00:39:30.574 | INFO | "pinned" : " 29 GB" | 20260507 00:39:30.574 | INFO | } | 20260507 00:39:30.574 | INFO | Perf report for the last 5.14356 seconds: | 20260507 00:39:30.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:39:30.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:30.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:30.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:30.574 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260507 00:39:30.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:30.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:30.574 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 00:39:30.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:30.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:30.575 | INFO | Uploaded 36GiB to GPU in 1.98832s (18.1057GiB/s) | 20260507 00:39:30.575 | INFO | Downloaded 34GiB to GPU in 1.99611s (17.0331GiB/s) | 20260507 00:39:30.575 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:39:30.575 | INFO | Total mem usage: 18 GiB | 20260507 00:39:30.575 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:39:30.575 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 00:39:30.575 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:39:30.575 | INFO | High Water Breakdown: | 20260507 00:39:30.575 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:39:30.575 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:39:30.575 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:39:30.575 | INFO | ------------------------------------------ | 20260507 00:39:30.575 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:39:30.575 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:39:30.575 | INFO | Total mem usage: 26.5 GiB | 20260507 00:39:30.575 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:39:30.575 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:39:30.575 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:39:30.575 | INFO | High Water Breakdown: | 20260507 00:39:30.575 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:39:30.575 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:39:30.575 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:39:30.575 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:39:30.575 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:39:30.575 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:39:30.575 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:39:30.575 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:39:30.575 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:39:30.575 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:39:30.575 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:39:30.575 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:39:30.575 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:39:30.575 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:39:30.575 | INFO | ------------------------------------------ | 20260507 00:39:30.575 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:39:30.575 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:39:30.575 | INFO | Total mem usage: 25.6 GiB | 20260507 00:39:30.575 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:39:30.575 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:39:30.575 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:39:30.575 | INFO | High Water Breakdown: | 20260507 00:39:30.575 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:39:30.576 | INFO | ------------------------------------------ | 20260507 00:39:30.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:39:30.576 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:39:30.576 | INFO | Total mem usage: 15.7 GiB | 20260507 00:39:30.576 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:39:30.576 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:39:30.576 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:39:30.576 | INFO | High Water Breakdown: | 20260507 00:39:30.576 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:39:30.576 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:39:30.576 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:39:30.576 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:39:30.576 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:39:30.576 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:39:30.576 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:39:30.576 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:39:30.576 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:39:30.576 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:39:30.576 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:39:30.576 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:39:30.576 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:39:30.576 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:39:30.576 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:39:30.576 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:39:30.576 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:39:30.576 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:39:30.576 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:39:30.576 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:39:30.576 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:39:30.576 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:39:30.576 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:39:30.576 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:39:30.576 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:39:30.576 | INFO | ------------------------------------------ | 20260507 00:39:30.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:39:30.576 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:39:30.576 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:39:30.576 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:39:30.576 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.62e+01, 1.36e+01 / 2.49e+00 | 20260507 00:39:30.576 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.04e-01, 9.67e-03 / 6.39e-03 | 20260507 00:39:30.576 | INFO | PerfReport: 11.68%, Baseline -- 3.61e+00 / 4.45e+00, 5.41e-01 / 2.65e-01 | 20260507 00:39:30.576 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.45e+00, 1.74e-01 / 1.21e-01 | 20260507 00:39:30.576 | INFO | PerfReport: 16.11%, DME -- 3.98e+01 / 4.50e+01, 4.19e+00 / 2.96e+00 | 20260507 00:39:30.576 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.30e+00 / 1.68e+00 | 20260507 00:39:30.576 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.18e+00 / 2.71e+00, 3.83e-01 / 2.26e-01 | 20260507 00:39:30.576 | INFO | PerfReport: 3.14%, Metrics -- 9.72e-01 / 1.30e+00, 2.58e-01 / 1.60e-01 | 20260507 00:39:30.576 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.31e-01 / 3.83e-01, 5.22e-02 / 2.04e-02 | 20260507 00:39:30.576 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:39:30.576 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:39:30.576 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:39:30.576 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 00:39:30.576 | INFO | PerfReport: 21.54%, Upload -- 2.08e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260507 00:39:30.576 | INFO | PerfReport: 21.34%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 00:39:30.576 | INFO | PerfReport: 32.03%, Compute -- 3.09e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260507 00:39:30.576 | INFO | PerfReport: 1.32%, LockUpload -- 1.27e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 00:39:30.576 | INFO | PerfReport: 1.34%, LockDownload -- 1.29e+00 / 0.00e+00, 3.23e+00 / 0.00e+00 | 20260507 00:39:30.576 | INFO | PerfReport: 22.43%, LockCompute -- 2.17e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 00:39:30.576 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260507 00:39:30.676 | INFO | Starting a round of Baz IO | 20260507 00:39:33.791 | INFO | Populating RTMetrics report | 20260507 00:39:33.791 | INFO | RTMetrics report populated | 20260507 00:39:33.792 | INFO | Opening tmp RTMetrics json file | 20260507 00:39:33.793 | INFO | Serializing RTMetrics to json | 20260507 00:39:33.811 | INFO | Serialization done, swapping tmp file now... | 20260507 00:39:33.812 | INFO | RTMetrics done | 20260507 00:39:33.885 | INFO | Finished a round of Baz IO: Wrote 1.84 GiB in 3.209 seconds | 20260507 00:39:33.886 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.25% / 0.00% / 66.74% | 20260507 00:39:33.886 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.330067% / 0% / 6.75519e-05% | 20260507 00:39:33.886 | INFO | Baz IO overall: | 20260507 00:39:33.886 | INFO | BytesWritten : 2981936253927 | 20260507 00:39:33.886 | INFO | NumEvents : 1436257014625 | 20260507 00:39:35.710 | INFO | Analyzing chunk frames = [5726208,5726720) | 20260507 00:39:35.710 | INFO | MemInfo: { | 20260507 00:39:35.710 | INFO | "gpu" : " 17 GB", | 20260507 00:39:35.710 | INFO | "huge" : "", | 20260507 00:39:35.710 | INFO | "huge_shared" : " 27 GB", | 20260507 00:39:35.710 | INFO | "malloc" : " 14 GB", | 20260507 00:39:35.710 | INFO | "pinned" : " 29 GB" | 20260507 00:39:35.710 | INFO | } | 20260507 00:39:35.711 | INFO | Perf report for the last 5.13613 seconds: | 20260507 00:39:35.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:39:35.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:35.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:35.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:35.711 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260507 00:39:35.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:35.711 | INFO | RT_METRICS : 43.0%, 1.0, 22.5ms, 0 | 20260507 00:39:35.711 | INFO | PRE_HQ : 59.7%, 1.8, 57.1ms, 0 | 20260507 00:39:35.711 | INFO | BAZWRITER : 62.1%, 3.4, 109.5ms, 0 | 20260507 00:39:35.711 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:39:35.711 | INFO | Uploaded 36GiB to GPU in 2.03351s (17.7034GiB/s) | 20260507 00:39:35.711 | INFO | Downloaded 38GiB to GPU in 2.18244s (17.4117GiB/s) | 20260507 00:39:40.841 | INFO | Analyzing chunk frames = [5726720,5727232) | 20260507 00:39:40.841 | INFO | MemInfo: { | 20260507 00:39:40.841 | INFO | "gpu" : " 17 GB", | 20260507 00:39:40.841 | INFO | "huge" : "", | 20260507 00:39:40.841 | INFO | "huge_shared" : " 27 GB", | 20260507 00:39:40.841 | INFO | "malloc" : " 14 GB", | 20260507 00:39:40.841 | INFO | "pinned" : " 29 GB" | 20260507 00:39:40.841 | INFO | } | 20260507 00:39:40.841 | INFO | Perf report for the last 5.13028 seconds: | 20260507 00:39:40.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:39:40.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:40.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:40.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:40.841 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 00:39:40.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:40.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:40.841 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:39:40.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:40.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:40.841 | INFO | Uploaded 36GiB to GPU in 2.01256s (17.8876GiB/s) | 20260507 00:39:40.841 | INFO | Downloaded 34GiB to GPU in 1.93149s (17.603GiB/s) | 20260507 00:39:45.985 | INFO | Analyzing chunk frames = [5727232,5727744) | 20260507 00:39:45.985 | INFO | MemInfo: { | 20260507 00:39:45.985 | INFO | "gpu" : " 17 GB", | 20260507 00:39:45.985 | INFO | "huge" : "", | 20260507 00:39:45.985 | INFO | "huge_shared" : " 27 GB", | 20260507 00:39:45.986 | INFO | "malloc" : " 14 GB", | 20260507 00:39:45.986 | INFO | "pinned" : " 29 GB" | 20260507 00:39:45.986 | INFO | } | 20260507 00:39:45.986 | INFO | Perf report for the last 5.14484 seconds: | 20260507 00:39:45.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:39:45.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:45.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:45.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:45.986 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:39:45.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:45.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:45.986 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 00:39:45.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:45.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:45.986 | INFO | Uploaded 36GiB to GPU in 1.96887s (18.2846GiB/s) | 20260507 00:39:45.986 | INFO | Downloaded 34GiB to GPU in 1.93164s (17.6017GiB/s) | 20260507 00:39:51.118 | INFO | Analyzing chunk frames = [5727744,5728256) | 20260507 00:39:51.118 | INFO | MemInfo: { | 20260507 00:39:51.118 | INFO | "gpu" : " 17 GB", | 20260507 00:39:51.118 | INFO | "huge" : "", | 20260507 00:39:51.118 | INFO | "huge_shared" : " 27 GB", | 20260507 00:39:51.118 | INFO | "malloc" : " 14 GB", | 20260507 00:39:51.118 | INFO | "pinned" : " 29 GB" | 20260507 00:39:51.118 | INFO | } | 20260507 00:39:51.119 | INFO | Perf report for the last 5.13286 seconds: | 20260507 00:39:51.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:39:51.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:51.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:51.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:51.119 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 00:39:51.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:51.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:51.119 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:39:51.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:51.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:51.119 | INFO | Uploaded 36GiB to GPU in 1.97294s (18.2469GiB/s) | 20260507 00:39:51.119 | INFO | Downloaded 34GiB to GPU in 1.90525s (17.8454GiB/s) | 20260507 00:39:56.262 | INFO | Analyzing chunk frames = [5728256,5728768) | 20260507 00:39:56.262 | INFO | MemInfo: { | 20260507 00:39:56.262 | INFO | "gpu" : " 17 GB", | 20260507 00:39:56.262 | INFO | "huge" : "", | 20260507 00:39:56.262 | INFO | "huge_shared" : " 27 GB", | 20260507 00:39:56.263 | INFO | "malloc" : " 14 GB", | 20260507 00:39:56.263 | INFO | "pinned" : " 29 GB" | 20260507 00:39:56.263 | INFO | } | 20260507 00:39:56.263 | INFO | Perf report for the last 5.14414 seconds: | 20260507 00:39:56.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:39:56.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:56.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:56.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:56.263 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 00:39:56.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:56.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:56.263 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:39:56.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:56.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:39:56.263 | INFO | Uploaded 36GiB to GPU in 1.98002s (18.1817GiB/s) | 20260507 00:39:56.263 | INFO | Downloaded 34GiB to GPU in 1.92876s (17.6279GiB/s) | 20260507 00:40:01.405 | INFO | Analyzing chunk frames = [5728768,5729280) | 20260507 00:40:01.405 | INFO | MemInfo: { | 20260507 00:40:01.405 | INFO | "gpu" : " 17 GB", | 20260507 00:40:01.405 | INFO | "huge" : "", | 20260507 00:40:01.405 | INFO | "huge_shared" : " 27 GB", | 20260507 00:40:01.406 | INFO | "malloc" : " 14 GB", | 20260507 00:40:01.406 | INFO | "pinned" : " 29 GB" | 20260507 00:40:01.406 | INFO | } | 20260507 00:40:01.406 | INFO | Perf report for the last 5.14292 seconds: | 20260507 00:40:01.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:40:01.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:01.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:01.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:01.406 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 00:40:01.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:01.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:01.406 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:40:01.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:01.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:01.406 | INFO | Uploaded 36GiB to GPU in 2.0082s (17.9265GiB/s) | 20260507 00:40:01.406 | INFO | Downloaded 34GiB to GPU in 1.94455s (17.4848GiB/s) | 20260507 00:40:06.545 | INFO | Analyzing chunk frames = [5729280,5729792) | 20260507 00:40:06.545 | INFO | MemInfo: { | 20260507 00:40:06.545 | INFO | "gpu" : " 17 GB", | 20260507 00:40:06.545 | INFO | "huge" : "", | 20260507 00:40:06.545 | INFO | "huge_shared" : " 27 GB", | 20260507 00:40:06.545 | INFO | "malloc" : " 14 GB", | 20260507 00:40:06.545 | INFO | "pinned" : " 29 GB" | 20260507 00:40:06.545 | INFO | } | 20260507 00:40:06.545 | INFO | Perf report for the last 5.13921 seconds: | 20260507 00:40:06.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:40:06.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:06.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:06.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:06.545 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:40:06.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:06.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:06.545 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 00:40:06.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:06.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:06.545 | INFO | Uploaded 36GiB to GPU in 1.97027s (18.2716GiB/s) | 20260507 00:40:06.545 | INFO | Downloaded 34GiB to GPU in 2.00697s (16.941GiB/s) | 20260507 00:40:11.691 | INFO | Analyzing chunk frames = [5729792,5730304) | 20260507 00:40:11.691 | INFO | MemInfo: { | 20260507 00:40:11.691 | INFO | "gpu" : " 17 GB", | 20260507 00:40:11.691 | INFO | "huge" : "", | 20260507 00:40:11.691 | INFO | "huge_shared" : " 27 GB", | 20260507 00:40:11.691 | INFO | "malloc" : " 14 GB", | 20260507 00:40:11.692 | INFO | "pinned" : " 29 GB" | 20260507 00:40:11.692 | INFO | } | 20260507 00:40:11.692 | INFO | Perf report for the last 5.14674 seconds: | 20260507 00:40:11.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:40:11.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:11.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:11.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:11.692 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 00:40:11.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:11.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:11.692 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:40:11.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:11.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:11.692 | INFO | Uploaded 36GiB to GPU in 1.9859s (18.1278GiB/s) | 20260507 00:40:11.692 | INFO | Downloaded 34GiB to GPU in 1.98081s (17.1647GiB/s) | 20260507 00:40:11.789 | INFO | Starting a round of Baz IO | 20260507 00:40:14.907 | INFO | Populating RTMetrics report | 20260507 00:40:14.907 | INFO | RTMetrics report populated | 20260507 00:40:14.907 | INFO | Opening tmp RTMetrics json file | 20260507 00:40:14.908 | INFO | Serializing RTMetrics to json | 20260507 00:40:14.909 | INFO | Serialization done, swapping tmp file now... | 20260507 00:40:14.910 | INFO | RTMetrics done | 20260507 00:40:15.000 | INFO | Finished a round of Baz IO: Wrote 1.841 GiB in 3.211 seconds | 20260507 00:40:15.000 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.25% / 0.00% / 66.75% | 20260507 00:40:15.000 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.330726% / 7.94729e-06% / 7.15256e-05% | 20260507 00:40:15.000 | INFO | Baz IO overall: | 20260507 00:40:15.000 | INFO | BytesWritten : 2983912646732 | 20260507 00:40:15.000 | INFO | NumEvents : 1437195991390 | 20260507 00:40:16.824 | INFO | Analyzing chunk frames = [5730304,5730816) | 20260507 00:40:16.824 | INFO | MemInfo: { | 20260507 00:40:16.824 | INFO | "gpu" : " 17 GB", | 20260507 00:40:16.824 | INFO | "huge" : "", | 20260507 00:40:16.824 | INFO | "huge_shared" : " 27 GB", | 20260507 00:40:16.824 | INFO | "malloc" : " 14 GB", | 20260507 00:40:16.825 | INFO | "pinned" : " 29 GB" | 20260507 00:40:16.825 | INFO | } | 20260507 00:40:16.825 | INFO | Perf report for the last 5.13309 seconds: | 20260507 00:40:16.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:40:16.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:16.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:16.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:16.825 | INFO | ANALYSIS : 62.1%, 3.0, 96.4ms, 0 | 20260507 00:40:16.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:16.825 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 00:40:16.825 | INFO | PRE_HQ : 59.5%, 1.8, 56.3ms, 0 | 20260507 00:40:16.825 | INFO | BAZWRITER : 62.1%, 3.4, 108.5ms, 0 | 20260507 00:40:16.825 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:40:16.825 | INFO | Uploaded 36GiB to GPU in 2.0375s (17.6688GiB/s) | 20260507 00:40:16.825 | INFO | Downloaded 38GiB to GPU in 2.18459s (17.3946GiB/s) | 20260507 00:40:21.961 | INFO | Analyzing chunk frames = [5730816,5731328) | 20260507 00:40:21.961 | INFO | MemInfo: { | 20260507 00:40:21.961 | INFO | "gpu" : " 17 GB", | 20260507 00:40:21.961 | INFO | "huge" : "", | 20260507 00:40:21.961 | INFO | "huge_shared" : " 27 GB", | 20260507 00:40:21.961 | INFO | "malloc" : " 14 GB", | 20260507 00:40:21.961 | INFO | "pinned" : " 29 GB" | 20260507 00:40:21.962 | INFO | } | 20260507 00:40:21.962 | INFO | Perf report for the last 5.13683 seconds: | 20260507 00:40:21.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:40:21.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:21.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:21.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:21.962 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 00:40:21.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:21.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:21.962 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:40:21.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:21.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:21.962 | INFO | Uploaded 36GiB to GPU in 2.0134s (17.8802GiB/s) | 20260507 00:40:21.962 | INFO | Downloaded 34GiB to GPU in 1.93194s (17.5989GiB/s) | 20260507 00:40:27.095 | INFO | Analyzing chunk frames = [5731328,5731840) | 20260507 00:40:27.095 | INFO | MemInfo: { | 20260507 00:40:27.095 | INFO | "gpu" : " 17 GB", | 20260507 00:40:27.095 | INFO | "huge" : "", | 20260507 00:40:27.095 | INFO | "huge_shared" : " 27 GB", | 20260507 00:40:27.095 | INFO | "malloc" : " 14 GB", | 20260507 00:40:27.095 | INFO | "pinned" : " 29 GB" | 20260507 00:40:27.095 | INFO | } | 20260507 00:40:27.095 | INFO | Perf report for the last 5.13324 seconds: | 20260507 00:40:27.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:40:27.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:27.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:27.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:27.095 | INFO | ANALYSIS : 60.5%, 3.0, 93.5ms, 0 | 20260507 00:40:27.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:27.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:27.095 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:40:27.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:27.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:27.095 | INFO | Uploaded 36GiB to GPU in 1.99901s (18.0089GiB/s) | 20260507 00:40:27.095 | INFO | Downloaded 34GiB to GPU in 1.93467s (17.5741GiB/s) | 20260507 00:40:32.236 | INFO | Analyzing chunk frames = [5731840,5732352) | 20260507 00:40:32.236 | INFO | MemInfo: { | 20260507 00:40:32.236 | INFO | "gpu" : " 17 GB", | 20260507 00:40:32.236 | INFO | "huge" : "", | 20260507 00:40:32.236 | INFO | "huge_shared" : " 27 GB", | 20260507 00:40:32.236 | INFO | "malloc" : " 14 GB", | 20260507 00:40:32.236 | INFO | "pinned" : " 29 GB" | 20260507 00:40:32.236 | INFO | } | 20260507 00:40:32.236 | INFO | Perf report for the last 5.14141 seconds: | 20260507 00:40:32.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:40:32.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:32.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:32.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:32.236 | INFO | ANALYSIS : 59.5%, 3.0, 92.2ms, 0 | 20260507 00:40:32.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:32.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:32.236 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:40:32.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:32.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:32.237 | INFO | Uploaded 36GiB to GPU in 1.97411s (18.2361GiB/s) | 20260507 00:40:32.237 | INFO | Downloaded 34GiB to GPU in 1.87994s (18.0857GiB/s) | 20260507 00:40:37.387 | INFO | Analyzing chunk frames = [5732352,5732864) | 20260507 00:40:37.387 | INFO | MemInfo: { | 20260507 00:40:37.387 | INFO | "gpu" : " 17 GB", | 20260507 00:40:37.387 | INFO | "huge" : "", | 20260507 00:40:37.387 | INFO | "huge_shared" : " 27 GB", | 20260507 00:40:37.387 | INFO | "malloc" : " 14 GB", | 20260507 00:40:37.387 | INFO | "pinned" : " 29 GB" | 20260507 00:40:37.387 | INFO | } | 20260507 00:40:37.387 | INFO | Perf report for the last 5.15054 seconds: | 20260507 00:40:37.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:40:37.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:37.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:37.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:37.387 | INFO | ANALYSIS : 59.9%, 3.0, 93.5ms, 0 | 20260507 00:40:37.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:37.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:37.387 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:40:37.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:37.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:37.387 | INFO | Uploaded 36GiB to GPU in 1.99363s (18.0575GiB/s) | 20260507 00:40:37.387 | INFO | Downloaded 34GiB to GPU in 1.92179s (17.6918GiB/s) | 20260507 00:40:42.519 | INFO | Analyzing chunk frames = [5732864,5733376) | 20260507 00:40:42.519 | INFO | MemInfo: { | 20260507 00:40:42.519 | INFO | "gpu" : " 17 GB", | 20260507 00:40:42.519 | INFO | "huge" : "", | 20260507 00:40:42.519 | INFO | "huge_shared" : " 27 GB", | 20260507 00:40:42.519 | INFO | "malloc" : " 14 GB", | 20260507 00:40:42.519 | INFO | "pinned" : " 29 GB" | 20260507 00:40:42.519 | INFO | } | 20260507 00:40:42.519 | INFO | Perf report for the last 5.13226 seconds: | 20260507 00:40:42.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:40:42.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:42.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:42.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:42.519 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 00:40:42.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:42.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:42.519 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 00:40:42.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:42.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:42.519 | INFO | Uploaded 36GiB to GPU in 2.0594s (17.4808GiB/s) | 20260507 00:40:42.520 | INFO | Downloaded 34GiB to GPU in 1.96308s (17.3197GiB/s) | 20260507 00:40:47.660 | INFO | Analyzing chunk frames = [5733376,5733888) | 20260507 00:40:47.660 | INFO | MemInfo: { | 20260507 00:40:47.660 | INFO | "gpu" : " 17 GB", | 20260507 00:40:47.660 | INFO | "huge" : "", | 20260507 00:40:47.660 | INFO | "huge_shared" : " 27 GB", | 20260507 00:40:47.660 | INFO | "malloc" : " 14 GB", | 20260507 00:40:47.660 | INFO | "pinned" : " 29 GB" | 20260507 00:40:47.660 | INFO | } | 20260507 00:40:47.660 | INFO | Perf report for the last 5.14091 seconds: | 20260507 00:40:47.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:40:47.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:47.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:47.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:47.660 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:40:47.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:47.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:47.660 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 00:40:47.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:47.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:47.660 | INFO | Uploaded 36GiB to GPU in 1.97445s (18.2329GiB/s) | 20260507 00:40:47.660 | INFO | Downloaded 34GiB to GPU in 1.96453s (17.307GiB/s) | 20260507 00:40:52.800 | INFO | Analyzing chunk frames = [5733888,5734400) | 20260507 00:40:52.800 | INFO | MemInfo: { | 20260507 00:40:52.800 | INFO | "gpu" : " 17 GB", | 20260507 00:40:52.800 | INFO | "huge" : "", | 20260507 00:40:52.800 | INFO | "huge_shared" : " 27 GB", | 20260507 00:40:52.800 | INFO | "malloc" : " 14 GB", | 20260507 00:40:52.800 | INFO | "pinned" : " 29 GB" | 20260507 00:40:52.800 | INFO | } | 20260507 00:40:52.800 | INFO | Perf report for the last 5.13986 seconds: | 20260507 00:40:52.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:40:52.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:52.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:52.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:52.800 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:40:52.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:52.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:52.800 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:40:52.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:52.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:52.800 | INFO | Uploaded 36GiB to GPU in 1.99066s (18.0844GiB/s) | 20260507 00:40:52.800 | INFO | Downloaded 34GiB to GPU in 1.97389s (17.2248GiB/s) | 20260507 00:40:52.800 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:40:52.800 | INFO | Total mem usage: 18 GiB | 20260507 00:40:52.800 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:40:52.800 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:40:52.800 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:40:52.800 | INFO | High Water Breakdown: | 20260507 00:40:52.800 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:40:52.800 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:40:52.800 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:40:52.801 | INFO | ------------------------------------------ | 20260507 00:40:52.801 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:40:52.801 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:40:52.801 | INFO | Total mem usage: 26.5 GiB | 20260507 00:40:52.801 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:40:52.801 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:40:52.801 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:40:52.801 | INFO | High Water Breakdown: | 20260507 00:40:52.801 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:40:52.801 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:40:52.801 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:40:52.801 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:40:52.801 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:40:52.801 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:40:52.801 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:40:52.801 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:40:52.801 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:40:52.801 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:40:52.801 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:40:52.801 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:40:52.801 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:40:52.801 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:40:52.801 | INFO | ------------------------------------------ | 20260507 00:40:52.801 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:40:52.801 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:40:52.801 | INFO | Total mem usage: 25.6 GiB | 20260507 00:40:52.801 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:40:52.801 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:40:52.801 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:40:52.801 | INFO | High Water Breakdown: | 20260507 00:40:52.801 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:40:52.801 | INFO | ------------------------------------------ | 20260507 00:40:52.801 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:40:52.801 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:40:52.801 | INFO | Total mem usage: 15.7 GiB | 20260507 00:40:52.801 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:40:52.801 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:40:52.801 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 00:40:52.801 | INFO | High Water Breakdown: | 20260507 00:40:52.801 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:40:52.801 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:40:52.801 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:40:52.801 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:40:52.801 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:40:52.801 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:40:52.801 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:40:52.801 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:40:52.801 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:40:52.801 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:40:52.801 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:40:52.801 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:40:52.801 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:40:52.801 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:40:52.801 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:40:52.801 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:40:52.801 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:40:52.801 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:40:52.801 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:40:52.801 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:40:52.801 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:40:52.801 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:40:52.801 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:40:52.801 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:40:52.802 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:40:52.802 | INFO | ------------------------------------------ | 20260507 00:40:52.802 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:40:52.802 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:40:52.802 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:40:52.802 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:40:52.802 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.53e+01, 1.33e+01 / 1.98e+00 | 20260507 00:40:52.802 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.00e-01, 9.70e-03 / 4.94e-03 | 20260507 00:40:52.802 | INFO | PerfReport: 11.66%, Baseline -- 3.60e+00 / 4.40e+00, 5.36e-01 / 3.39e-01 | 20260507 00:40:52.802 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.44e+00, 1.77e-01 / 1.41e-01 | 20260507 00:40:52.802 | INFO | PerfReport: 16.06%, DME -- 3.97e+01 / 4.43e+01, 3.91e+00 / 2.21e+00 | 20260507 00:40:52.802 | INFO | PerfReport: 57.47%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.33e+00 / 1.79e+00 | 20260507 00:40:52.802 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.18e+00 / 2.68e+00, 3.83e-01 / 2.43e-01 | 20260507 00:40:52.802 | INFO | PerfReport: 3.15%, Metrics -- 9.73e-01 / 1.29e+00, 2.59e-01 / 1.25e-01 | 20260507 00:40:52.802 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.30e-01 / 3.81e-01, 5.22e-02 / 2.62e-02 | 20260507 00:40:52.802 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:40:52.802 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:40:52.802 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:40:52.802 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 00:40:52.802 | INFO | PerfReport: 21.58%, Upload -- 2.08e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260507 00:40:52.802 | INFO | PerfReport: 21.31%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260507 00:40:52.802 | INFO | PerfReport: 32.04%, Compute -- 3.09e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 00:40:52.802 | INFO | PerfReport: 1.34%, LockUpload -- 1.29e+00 / 0.00e+00, 4.87e+00 / 0.00e+00 | 20260507 00:40:52.802 | INFO | PerfReport: 1.28%, LockDownload -- 1.24e+00 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260507 00:40:52.802 | INFO | PerfReport: 22.44%, LockCompute -- 2.16e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 00:40:52.802 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260507 00:40:52.897 | INFO | Starting a round of Baz IO | 20260507 00:40:56.026 | INFO | Populating RTMetrics report | 20260507 00:40:56.026 | INFO | RTMetrics report populated | 20260507 00:40:56.026 | INFO | Opening tmp RTMetrics json file | 20260507 00:40:56.033 | INFO | Serializing RTMetrics to json | 20260507 00:40:56.046 | INFO | Serialization done, swapping tmp file now... | 20260507 00:40:56.047 | INFO | RTMetrics done | 20260507 00:40:56.117 | INFO | Finished a round of Baz IO: Wrote 1.839 GiB in 3.219 seconds | 20260507 00:40:56.117 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.25% / 0.00% / 66.75% | 20260507 00:40:56.117 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.329479% / 0% / 6.35783e-05% | 20260507 00:40:56.117 | INFO | Baz IO overall: | 20260507 00:40:56.117 | INFO | BytesWritten : 2985887372645 | 20260507 00:40:56.117 | INFO | NumEvents : 1438133299809 | 20260507 00:40:57.937 | INFO | Analyzing chunk frames = [5734400,5734912) | 20260507 00:40:57.937 | INFO | MemInfo: { | 20260507 00:40:57.937 | INFO | "gpu" : " 17 GB", | 20260507 00:40:57.937 | INFO | "huge" : "", | 20260507 00:40:57.937 | INFO | "huge_shared" : " 27 GB", | 20260507 00:40:57.937 | INFO | "malloc" : " 14 GB", | 20260507 00:40:57.937 | INFO | "pinned" : " 29 GB" | 20260507 00:40:57.937 | INFO | } | 20260507 00:40:57.937 | INFO | Perf report for the last 5.13743 seconds: | 20260507 00:40:57.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:40:57.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:57.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:57.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:57.937 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 00:40:57.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:40:57.938 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 00:40:57.938 | INFO | PRE_HQ : 59.9%, 1.8, 56.4ms, 0 | 20260507 00:40:57.938 | INFO | BAZWRITER : 62.3%, 3.4, 108.0ms, 0 | 20260507 00:40:57.938 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 00:40:57.938 | INFO | Uploaded 36GiB to GPU in 2.04s (17.647GiB/s) | 20260507 00:40:57.938 | INFO | Downloaded 38GiB to GPU in 2.18658s (17.3788GiB/s) | 20260507 00:41:03.076 | INFO | Analyzing chunk frames = [5734912,5735424) | 20260507 00:41:03.076 | INFO | MemInfo: { | 20260507 00:41:03.076 | INFO | "gpu" : " 17 GB", | 20260507 00:41:03.076 | INFO | "huge" : "", | 20260507 00:41:03.076 | INFO | "huge_shared" : " 27 GB", | 20260507 00:41:03.076 | INFO | "malloc" : " 14 GB", | 20260507 00:41:03.076 | INFO | "pinned" : " 29 GB" | 20260507 00:41:03.076 | INFO | } | 20260507 00:41:03.076 | INFO | Perf report for the last 5.13881 seconds: | 20260507 00:41:03.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:41:03.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:03.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:03.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:03.076 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:41:03.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:03.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:03.076 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:41:03.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:03.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:03.076 | INFO | Uploaded 36GiB to GPU in 2.00786s (17.9295GiB/s) | 20260507 00:41:03.076 | INFO | Downloaded 34GiB to GPU in 1.9345s (17.5756GiB/s) | 20260507 00:41:08.211 | INFO | Analyzing chunk frames = [5735424,5735936) | 20260507 00:41:08.211 | INFO | MemInfo: { | 20260507 00:41:08.211 | INFO | "gpu" : " 17 GB", | 20260507 00:41:08.211 | INFO | "huge" : "", | 20260507 00:41:08.211 | INFO | "huge_shared" : " 27 GB", | 20260507 00:41:08.212 | INFO | "malloc" : " 14 GB", | 20260507 00:41:08.212 | INFO | "pinned" : " 29 GB" | 20260507 00:41:08.212 | INFO | } | 20260507 00:41:08.212 | INFO | Perf report for the last 5.13551 seconds: | 20260507 00:41:08.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:41:08.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:08.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:08.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:08.212 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 00:41:08.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:08.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:08.212 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:41:08.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:08.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:08.212 | INFO | Uploaded 36GiB to GPU in 1.98561s (18.1304GiB/s) | 20260507 00:41:08.212 | INFO | Downloaded 34GiB to GPU in 1.93844s (17.5399GiB/s) | 20260507 00:41:13.359 | INFO | Analyzing chunk frames = [5735936,5736448) | 20260507 00:41:13.359 | INFO | MemInfo: { | 20260507 00:41:13.359 | INFO | "gpu" : " 17 GB", | 20260507 00:41:13.359 | INFO | "huge" : "", | 20260507 00:41:13.359 | INFO | "huge_shared" : " 27 GB", | 20260507 00:41:13.359 | INFO | "malloc" : " 14 GB", | 20260507 00:41:13.359 | INFO | "pinned" : " 29 GB" | 20260507 00:41:13.359 | INFO | } | 20260507 00:41:13.359 | INFO | Perf report for the last 5.14765 seconds: | 20260507 00:41:13.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:41:13.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:13.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:13.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:13.359 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260507 00:41:13.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:13.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:13.359 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:41:13.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:13.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:13.360 | INFO | Uploaded 36GiB to GPU in 2.0039s (17.965GiB/s) | 20260507 00:41:13.360 | INFO | Downloaded 34GiB to GPU in 1.91024s (17.7988GiB/s) | 20260507 00:41:18.496 | INFO | Analyzing chunk frames = [5736448,5736960) | 20260507 00:41:18.496 | INFO | MemInfo: { | 20260507 00:41:18.496 | INFO | "gpu" : " 17 GB", | 20260507 00:41:18.496 | INFO | "huge" : "", | 20260507 00:41:18.496 | INFO | "huge_shared" : " 27 GB", | 20260507 00:41:18.496 | INFO | "malloc" : " 14 GB", | 20260507 00:41:18.496 | INFO | "pinned" : " 29 GB" | 20260507 00:41:18.496 | INFO | } | 20260507 00:41:18.496 | INFO | Perf report for the last 5.13663 seconds: | 20260507 00:41:18.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:41:18.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:18.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:18.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:18.496 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 00:41:18.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:18.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:18.496 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:41:18.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:18.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:18.496 | INFO | Uploaded 36GiB to GPU in 1.99109s (18.0806GiB/s) | 20260507 00:41:18.497 | INFO | Downloaded 34GiB to GPU in 1.95338s (17.4057GiB/s) | 20260507 00:41:23.641 | INFO | Analyzing chunk frames = [5736960,5737472) | 20260507 00:41:23.641 | INFO | MemInfo: { | 20260507 00:41:23.641 | INFO | "gpu" : " 17 GB", | 20260507 00:41:23.641 | INFO | "huge" : "", | 20260507 00:41:23.641 | INFO | "huge_shared" : " 27 GB", | 20260507 00:41:23.642 | INFO | "malloc" : " 14 GB", | 20260507 00:41:23.642 | INFO | "pinned" : " 29 GB" | 20260507 00:41:23.642 | INFO | } | 20260507 00:41:23.642 | INFO | Perf report for the last 5.1457 seconds: | 20260507 00:41:23.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:41:23.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:23.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:23.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:23.642 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 00:41:23.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:23.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:23.642 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 00:41:23.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:23.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:23.642 | INFO | Uploaded 36GiB to GPU in 2.04285s (17.6225GiB/s) | 20260507 00:41:23.642 | INFO | Downloaded 34GiB to GPU in 1.96151s (17.3336GiB/s) | 20260507 00:41:28.773 | INFO | Analyzing chunk frames = [5737472,5737984) | 20260507 00:41:28.773 | INFO | MemInfo: { | 20260507 00:41:28.773 | INFO | "gpu" : " 17 GB", | 20260507 00:41:28.773 | INFO | "huge" : "", | 20260507 00:41:28.774 | INFO | "huge_shared" : " 27 GB", | 20260507 00:41:28.774 | INFO | "malloc" : " 14 GB", | 20260507 00:41:28.774 | INFO | "pinned" : " 29 GB" | 20260507 00:41:28.774 | INFO | } | 20260507 00:41:28.774 | INFO | Perf report for the last 5.13193 seconds: | 20260507 00:41:28.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:41:28.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:28.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:28.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:28.774 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 00:41:28.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:28.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:28.774 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 00:41:28.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:28.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:28.774 | INFO | Uploaded 36GiB to GPU in 1.99039s (18.0869GiB/s) | 20260507 00:41:28.774 | INFO | Downloaded 34GiB to GPU in 1.98798s (17.1028GiB/s) | 20260507 00:41:33.912 | INFO | Analyzing chunk frames = [5737984,5738496) | 20260507 00:41:33.912 | INFO | MemInfo: { | 20260507 00:41:33.912 | INFO | "gpu" : " 17 GB", | 20260507 00:41:33.912 | INFO | "huge" : "", | 20260507 00:41:33.912 | INFO | "huge_shared" : " 27 GB", | 20260507 00:41:33.912 | INFO | "malloc" : " 14 GB", | 20260507 00:41:33.912 | INFO | "pinned" : " 29 GB" | 20260507 00:41:33.912 | INFO | } | 20260507 00:41:33.912 | INFO | Perf report for the last 5.13883 seconds: | 20260507 00:41:33.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:41:33.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:33.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:33.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:33.913 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:41:33.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:33.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:33.913 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 00:41:33.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:33.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:33.913 | INFO | Uploaded 36GiB to GPU in 1.99384s (18.0556GiB/s) | 20260507 00:41:33.913 | INFO | Downloaded 34GiB to GPU in 1.98682s (17.1128GiB/s) | 20260507 00:41:34.010 | INFO | Starting a round of Baz IO | 20260507 00:41:37.137 | INFO | Populating RTMetrics report | 20260507 00:41:37.138 | INFO | RTMetrics report populated | 20260507 00:41:37.138 | INFO | Opening tmp RTMetrics json file | 20260507 00:41:37.138 | INFO | Serializing RTMetrics to json | 20260507 00:41:37.139 | INFO | Serialization done, swapping tmp file now... | 20260507 00:41:37.140 | INFO | RTMetrics done | 20260507 00:41:37.236 | INFO | Finished a round of Baz IO: Wrote 1.839 GiB in 3.225 seconds | 20260507 00:41:37.236 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.25% / 0.00% / 66.75% | 20260507 00:41:37.236 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.328648% / 0% / 7.94729e-05% | 20260507 00:41:37.236 | INFO | Baz IO overall: | 20260507 00:41:37.236 | INFO | BytesWritten : 2987861810805 | 20260507 00:41:37.236 | INFO | NumEvents : 1439070558334 | 20260507 00:41:39.050 | INFO | Analyzing chunk frames = [5738496,5739008) | 20260507 00:41:39.050 | INFO | MemInfo: { | 20260507 00:41:39.050 | INFO | "gpu" : " 17 GB", | 20260507 00:41:39.050 | INFO | "huge" : "", | 20260507 00:41:39.050 | INFO | "huge_shared" : " 27 GB", | 20260507 00:41:39.050 | INFO | "malloc" : " 14 GB", | 20260507 00:41:39.050 | INFO | "pinned" : " 29 GB" | 20260507 00:41:39.050 | INFO | } | 20260507 00:41:39.051 | INFO | Perf report for the last 5.13803 seconds: | 20260507 00:41:39.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:41:39.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:39.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:39.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:39.051 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260507 00:41:39.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:39.051 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 00:41:39.051 | INFO | PRE_HQ : 59.5%, 1.8, 56.5ms, 0 | 20260507 00:41:39.051 | INFO | BAZWRITER : 62.4%, 3.3, 106.9ms, 0 | 20260507 00:41:39.051 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:41:39.051 | INFO | Uploaded 36GiB to GPU in 2.03107s (17.7246GiB/s) | 20260507 00:41:39.051 | INFO | Downloaded 38GiB to GPU in 2.18114s (17.4221GiB/s) | 20260507 00:41:44.190 | INFO | Analyzing chunk frames = [5739008,5739520) | 20260507 00:41:44.190 | INFO | MemInfo: { | 20260507 00:41:44.190 | INFO | "gpu" : " 17 GB", | 20260507 00:41:44.190 | INFO | "huge" : "", | 20260507 00:41:44.190 | INFO | "huge_shared" : " 27 GB", | 20260507 00:41:44.190 | INFO | "malloc" : " 14 GB", | 20260507 00:41:44.190 | INFO | "pinned" : " 29 GB" | 20260507 00:41:44.190 | INFO | } | 20260507 00:41:44.190 | INFO | Perf report for the last 5.13979 seconds: | 20260507 00:41:44.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:41:44.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:44.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:44.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:44.190 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:41:44.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:44.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:44.190 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:41:44.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:44.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:44.191 | INFO | Uploaded 36GiB to GPU in 2.00834s (17.9253GiB/s) | 20260507 00:41:44.191 | INFO | Downloaded 34GiB to GPU in 1.93343s (17.5853GiB/s) | 20260507 00:41:49.332 | INFO | Analyzing chunk frames = [5739520,5740032) | 20260507 00:41:49.332 | INFO | MemInfo: { | 20260507 00:41:49.332 | INFO | "gpu" : " 17 GB", | 20260507 00:41:49.332 | INFO | "huge" : "", | 20260507 00:41:49.332 | INFO | "huge_shared" : " 27 GB", | 20260507 00:41:49.332 | INFO | "malloc" : " 14 GB", | 20260507 00:41:49.332 | INFO | "pinned" : " 29 GB" | 20260507 00:41:49.332 | INFO | } | 20260507 00:41:49.332 | INFO | Perf report for the last 5.14172 seconds: | 20260507 00:41:49.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:41:49.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:49.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:49.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:49.332 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:41:49.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:49.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:49.332 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:41:49.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:49.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:49.332 | INFO | Uploaded 36GiB to GPU in 1.98858s (18.1033GiB/s) | 20260507 00:41:49.332 | INFO | Downloaded 34GiB to GPU in 1.93s (17.6166GiB/s) | 20260507 00:41:54.472 | INFO | Analyzing chunk frames = [5740032,5740544) | 20260507 00:41:54.472 | INFO | MemInfo: { | 20260507 00:41:54.472 | INFO | "gpu" : " 17 GB", | 20260507 00:41:54.472 | INFO | "huge" : "", | 20260507 00:41:54.472 | INFO | "huge_shared" : " 27 GB", | 20260507 00:41:54.472 | INFO | "malloc" : " 14 GB", | 20260507 00:41:54.472 | INFO | "pinned" : " 29 GB" | 20260507 00:41:54.472 | INFO | } | 20260507 00:41:54.472 | INFO | Perf report for the last 5.14002 seconds: | 20260507 00:41:54.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:41:54.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:54.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:54.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:54.472 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 00:41:54.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:54.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:54.472 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:41:54.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:54.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:54.473 | INFO | Uploaded 36GiB to GPU in 1.97752s (18.2046GiB/s) | 20260507 00:41:54.473 | INFO | Downloaded 34GiB to GPU in 1.90811s (17.8187GiB/s) | 20260507 00:41:59.610 | INFO | Analyzing chunk frames = [5740544,5741056) | 20260507 00:41:59.610 | INFO | MemInfo: { | 20260507 00:41:59.610 | INFO | "gpu" : " 17 GB", | 20260507 00:41:59.610 | INFO | "huge" : "", | 20260507 00:41:59.610 | INFO | "huge_shared" : " 27 GB", | 20260507 00:41:59.610 | INFO | "malloc" : " 14 GB", | 20260507 00:41:59.610 | INFO | "pinned" : " 29 GB" | 20260507 00:41:59.610 | INFO | } | 20260507 00:41:59.611 | INFO | Perf report for the last 5.13842 seconds: | 20260507 00:41:59.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:41:59.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:59.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:59.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:59.611 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 00:41:59.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:59.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:59.611 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:41:59.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:59.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:41:59.611 | INFO | Uploaded 36GiB to GPU in 2.02222s (17.8022GiB/s) | 20260507 00:41:59.611 | INFO | Downloaded 34GiB to GPU in 1.94588s (17.4728GiB/s) | 20260507 00:42:04.748 | INFO | Analyzing chunk frames = [5741056,5741568) | 20260507 00:42:04.748 | INFO | MemInfo: { | 20260507 00:42:04.748 | INFO | "gpu" : " 17 GB", | 20260507 00:42:04.748 | INFO | "huge" : "", | 20260507 00:42:04.749 | INFO | "huge_shared" : " 27 GB", | 20260507 00:42:04.749 | INFO | "malloc" : " 14 GB", | 20260507 00:42:04.749 | INFO | "pinned" : " 29 GB" | 20260507 00:42:04.749 | INFO | } | 20260507 00:42:04.749 | INFO | Perf report for the last 5.13802 seconds: | 20260507 00:42:04.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:42:04.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:04.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:04.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:04.749 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 00:42:04.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:04.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:04.749 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 00:42:04.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:04.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:04.749 | INFO | Uploaded 36GiB to GPU in 2.05853s (17.4882GiB/s) | 20260507 00:42:04.749 | INFO | Downloaded 34GiB to GPU in 1.95166s (17.4211GiB/s) | 20260507 00:42:09.889 | INFO | Analyzing chunk frames = [5741568,5742080) | 20260507 00:42:09.889 | INFO | MemInfo: { | 20260507 00:42:09.889 | INFO | "gpu" : " 17 GB", | 20260507 00:42:09.890 | INFO | "huge" : "", | 20260507 00:42:09.890 | INFO | "huge_shared" : " 27 GB", | 20260507 00:42:09.890 | INFO | "malloc" : " 14 GB", | 20260507 00:42:09.890 | INFO | "pinned" : " 29 GB" | 20260507 00:42:09.890 | INFO | } | 20260507 00:42:09.890 | INFO | Perf report for the last 5.14104 seconds: | 20260507 00:42:09.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:42:09.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:09.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:09.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:09.890 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:42:09.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:09.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:09.890 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 00:42:09.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:09.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:09.890 | INFO | Uploaded 36GiB to GPU in 1.98613s (18.1257GiB/s) | 20260507 00:42:09.890 | INFO | Downloaded 34GiB to GPU in 2.00596s (16.9495GiB/s) | 20260507 00:42:15.025 | INFO | Analyzing chunk frames = [5742080,5742592) | 20260507 00:42:15.025 | INFO | MemInfo: { | 20260507 00:42:15.025 | INFO | "gpu" : " 17 GB", | 20260507 00:42:15.025 | INFO | "huge" : "", | 20260507 00:42:15.025 | INFO | "huge_shared" : " 27 GB", | 20260507 00:42:15.025 | INFO | "malloc" : " 14 GB", | 20260507 00:42:15.025 | INFO | "pinned" : " 29 GB" | 20260507 00:42:15.025 | INFO | } | 20260507 00:42:15.025 | INFO | Perf report for the last 5.13513 seconds: | 20260507 00:42:15.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:42:15.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:15.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:15.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:15.025 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:42:15.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:15.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:15.025 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 00:42:15.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:15.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:15.025 | INFO | Uploaded 36GiB to GPU in 2.00406s (17.9635GiB/s) | 20260507 00:42:15.025 | INFO | Downloaded 34GiB to GPU in 1.97601s (17.2064GiB/s) | 20260507 00:42:15.025 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:42:15.025 | INFO | Total mem usage: 18 GiB | 20260507 00:42:15.025 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:42:15.025 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 00:42:15.025 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:42:15.025 | INFO | High Water Breakdown: | 20260507 00:42:15.025 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:42:15.025 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:42:15.025 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:42:15.025 | INFO | ------------------------------------------ | 20260507 00:42:15.025 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:42:15.025 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:42:15.026 | INFO | Total mem usage: 26.5 GiB | 20260507 00:42:15.026 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:42:15.026 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:42:15.026 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:42:15.026 | INFO | High Water Breakdown: | 20260507 00:42:15.026 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:42:15.026 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:42:15.026 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:42:15.026 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:42:15.026 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:42:15.026 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:42:15.026 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:42:15.026 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:42:15.026 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:42:15.026 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:42:15.026 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:42:15.026 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:42:15.026 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:42:15.026 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:42:15.026 | INFO | ------------------------------------------ | 20260507 00:42:15.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:42:15.026 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:42:15.026 | INFO | Total mem usage: 25.6 GiB | 20260507 00:42:15.026 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:42:15.026 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:42:15.026 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:42:15.026 | INFO | High Water Breakdown: | 20260507 00:42:15.026 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:42:15.026 | INFO | ------------------------------------------ | 20260507 00:42:15.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:42:15.026 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:42:15.026 | INFO | Total mem usage: 15.7 GiB | 20260507 00:42:15.026 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:42:15.026 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:42:15.026 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:42:15.026 | INFO | High Water Breakdown: | 20260507 00:42:15.026 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:42:15.026 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:42:15.026 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:42:15.026 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:42:15.026 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:42:15.026 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:42:15.026 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:42:15.026 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:42:15.026 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:42:15.026 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:42:15.026 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:42:15.026 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:42:15.026 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:42:15.026 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:42:15.026 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:42:15.026 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:42:15.026 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:42:15.026 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:42:15.026 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:42:15.026 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:42:15.026 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:42:15.026 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:42:15.026 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:42:15.026 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:42:15.026 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:42:15.026 | INFO | ------------------------------------------ | 20260507 00:42:15.026 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:42:15.026 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:42:15.026 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:42:15.026 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:42:15.026 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.59e+01, 1.35e+01 / 2.39e+00 | 20260507 00:42:15.026 | INFO | PerfReport: 0.30%, AggregateStats -- 9.14e-02 / 1.03e-01, 1.04e-02 / 5.25e-03 | 20260507 00:42:15.026 | INFO | PerfReport: 11.65%, Baseline -- 3.60e+00 / 4.45e+00, 5.22e-01 / 2.38e-01 | 20260507 00:42:15.026 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.47e+00, 1.73e-01 / 1.03e-01 | 20260507 00:42:15.026 | INFO | PerfReport: 16.07%, DME -- 3.97e+01 / 4.45e+01, 4.04e+00 / 2.54e+00 | 20260507 00:42:15.026 | INFO | PerfReport: 57.47%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.34e+00 / 1.47e+00 | 20260507 00:42:15.026 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.18e+00 / 2.73e+00, 3.83e-01 / 2.14e-01 | 20260507 00:42:15.026 | INFO | PerfReport: 3.15%, Metrics -- 9.73e-01 / 1.31e+00, 2.60e-01 / 1.72e-01 | 20260507 00:42:15.026 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.31e-01 / 3.86e-01, 5.28e-02 / 2.95e-02 | 20260507 00:42:15.026 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:42:15.026 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:42:15.026 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:42:15.026 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 00:42:15.026 | INFO | PerfReport: 21.65%, Upload -- 2.09e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260507 00:42:15.026 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 00:42:15.026 | INFO | PerfReport: 32.00%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:42:15.026 | INFO | PerfReport: 1.35%, LockUpload -- 1.31e+00 / 0.00e+00, 4.86e+00 / 0.00e+00 | 20260507 00:42:15.026 | INFO | PerfReport: 1.26%, LockDownload -- 1.22e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260507 00:42:15.026 | INFO | PerfReport: 22.36%, LockCompute -- 2.16e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 00:42:15.027 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260507 00:42:15.124 | INFO | Starting a round of Baz IO | 20260507 00:42:18.250 | INFO | Populating RTMetrics report | 20260507 00:42:18.250 | INFO | RTMetrics report populated | 20260507 00:42:18.250 | INFO | Opening tmp RTMetrics json file | 20260507 00:42:18.251 | INFO | Serializing RTMetrics to json | 20260507 00:42:18.254 | INFO | Serialization done, swapping tmp file now... | 20260507 00:42:18.261 | INFO | RTMetrics done | 20260507 00:42:18.343 | INFO | Finished a round of Baz IO: Wrote 1.841 GiB in 3.219 seconds | 20260507 00:42:18.344 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.25% / 0.00% / 66.75% | 20260507 00:42:18.344 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.329606% / 3.97364e-06% / 7.54992e-05% | 20260507 00:42:18.344 | INFO | Baz IO overall: | 20260507 00:42:18.344 | INFO | BytesWritten : 2989838427967 | 20260507 00:42:18.344 | INFO | NumEvents : 1440009544687 | 20260507 00:42:20.162 | INFO | Analyzing chunk frames = [5742592,5743104) | 20260507 00:42:20.162 | INFO | MemInfo: { | 20260507 00:42:20.162 | INFO | "gpu" : " 17 GB", | 20260507 00:42:20.162 | INFO | "huge" : "", | 20260507 00:42:20.162 | INFO | "huge_shared" : " 27 GB", | 20260507 00:42:20.162 | INFO | "malloc" : " 14 GB", | 20260507 00:42:20.162 | INFO | "pinned" : " 29 GB" | 20260507 00:42:20.162 | INFO | } | 20260507 00:42:20.162 | INFO | Perf report for the last 5.13701 seconds: | 20260507 00:42:20.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:42:20.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:20.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:20.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:20.162 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 00:42:20.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:20.162 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 00:42:20.162 | INFO | PRE_HQ : 59.6%, 1.8, 56.6ms, 0 | 20260507 00:42:20.162 | INFO | BAZWRITER : 62.1%, 3.4, 108.0ms, 0 | 20260507 00:42:20.162 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:42:20.162 | INFO | Uploaded 36GiB to GPU in 2.04086s (17.6396GiB/s) | 20260507 00:42:20.162 | INFO | Downloaded 38GiB to GPU in 2.17572s (17.4655GiB/s) | 20260507 00:42:25.306 | INFO | Analyzing chunk frames = [5743104,5743616) | 20260507 00:42:25.306 | INFO | MemInfo: { | 20260507 00:42:25.306 | INFO | "gpu" : " 17 GB", | 20260507 00:42:25.306 | INFO | "huge" : "", | 20260507 00:42:25.306 | INFO | "huge_shared" : " 27 GB", | 20260507 00:42:25.306 | INFO | "malloc" : " 14 GB", | 20260507 00:42:25.306 | INFO | "pinned" : " 29 GB" | 20260507 00:42:25.306 | INFO | } | 20260507 00:42:25.306 | INFO | Perf report for the last 5.14457 seconds: | 20260507 00:42:25.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:42:25.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:25.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:25.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:25.306 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260507 00:42:25.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:25.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:25.306 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:42:25.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:25.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:25.307 | INFO | Uploaded 36GiB to GPU in 2.00479s (17.957GiB/s) | 20260507 00:42:25.307 | INFO | Downloaded 34GiB to GPU in 1.93266s (17.5923GiB/s) | 20260507 00:42:30.451 | INFO | Analyzing chunk frames = [5743616,5744128) | 20260507 00:42:30.451 | INFO | MemInfo: { | 20260507 00:42:30.451 | INFO | "gpu" : " 17 GB", | 20260507 00:42:30.451 | INFO | "huge" : "", | 20260507 00:42:30.451 | INFO | "huge_shared" : " 27 GB", | 20260507 00:42:30.451 | INFO | "malloc" : " 14 GB", | 20260507 00:42:30.451 | INFO | "pinned" : " 29 GB" | 20260507 00:42:30.451 | INFO | } | 20260507 00:42:30.451 | INFO | Perf report for the last 5.14505 seconds: | 20260507 00:42:30.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:42:30.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:30.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:30.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:30.451 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260507 00:42:30.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:30.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:30.452 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:42:30.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:30.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:30.452 | INFO | Uploaded 36GiB to GPU in 1.98607s (18.1262GiB/s) | 20260507 00:42:30.452 | INFO | Downloaded 34GiB to GPU in 1.91931s (17.7147GiB/s) | 20260507 00:42:35.591 | INFO | Analyzing chunk frames = [5744128,5744640) | 20260507 00:42:35.591 | INFO | MemInfo: { | 20260507 00:42:35.591 | INFO | "gpu" : " 17 GB", | 20260507 00:42:35.591 | INFO | "huge" : "", | 20260507 00:42:35.591 | INFO | "huge_shared" : " 27 GB", | 20260507 00:42:35.592 | INFO | "malloc" : " 14 GB", | 20260507 00:42:35.592 | INFO | "pinned" : " 29 GB" | 20260507 00:42:35.592 | INFO | } | 20260507 00:42:35.592 | INFO | Perf report for the last 5.14012 seconds: | 20260507 00:42:35.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:42:35.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:35.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:35.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:35.592 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:42:35.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:35.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:35.592 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:42:35.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:35.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:35.592 | INFO | Uploaded 36GiB to GPU in 1.98739s (18.1142GiB/s) | 20260507 00:42:35.592 | INFO | Downloaded 34GiB to GPU in 1.91625s (17.743GiB/s) | 20260507 00:42:40.733 | INFO | Analyzing chunk frames = [5744640,5745152) | 20260507 00:42:40.733 | INFO | MemInfo: { | 20260507 00:42:40.733 | INFO | "gpu" : " 17 GB", | 20260507 00:42:40.733 | INFO | "huge" : "", | 20260507 00:42:40.733 | INFO | "huge_shared" : " 27 GB", | 20260507 00:42:40.733 | INFO | "malloc" : " 14 GB", | 20260507 00:42:40.733 | INFO | "pinned" : " 29 GB" | 20260507 00:42:40.733 | INFO | } | 20260507 00:42:40.733 | INFO | Perf report for the last 5.14135 seconds: | 20260507 00:42:40.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:42:40.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:40.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:40.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:40.733 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 00:42:40.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:40.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:40.733 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 00:42:40.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:40.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:40.733 | INFO | Uploaded 36GiB to GPU in 2.00867s (17.9223GiB/s) | 20260507 00:42:40.733 | INFO | Downloaded 34GiB to GPU in 1.94071s (17.5194GiB/s) | 20260507 00:42:45.859 | INFO | Analyzing chunk frames = [5745152,5745664) | 20260507 00:42:45.859 | INFO | MemInfo: { | 20260507 00:42:45.859 | INFO | "gpu" : " 17 GB", | 20260507 00:42:45.859 | INFO | "huge" : "", | 20260507 00:42:45.859 | INFO | "huge_shared" : " 27 GB", | 20260507 00:42:45.859 | INFO | "malloc" : " 14 GB", | 20260507 00:42:45.859 | INFO | "pinned" : " 29 GB" | 20260507 00:42:45.859 | INFO | } | 20260507 00:42:45.859 | INFO | Perf report for the last 5.12627 seconds: | 20260507 00:42:45.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:42:45.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:45.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:45.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:45.859 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260507 00:42:45.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:45.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:45.859 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:42:45.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:45.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:45.860 | INFO | Uploaded 36GiB to GPU in 2.05585s (17.511GiB/s) | 20260507 00:42:45.860 | INFO | Downloaded 34GiB to GPU in 1.95501s (17.3912GiB/s) | 20260507 00:42:51.001 | INFO | Analyzing chunk frames = [5745664,5746176) | 20260507 00:42:51.001 | INFO | MemInfo: { | 20260507 00:42:51.001 | INFO | "gpu" : " 17 GB", | 20260507 00:42:51.001 | INFO | "huge" : "", | 20260507 00:42:51.001 | INFO | "huge_shared" : " 27 GB", | 20260507 00:42:51.001 | INFO | "malloc" : " 14 GB", | 20260507 00:42:51.001 | INFO | "pinned" : " 29 GB" | 20260507 00:42:51.001 | INFO | } | 20260507 00:42:51.001 | INFO | Perf report for the last 5.14162 seconds: | 20260507 00:42:51.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:42:51.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:51.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:51.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:51.001 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 00:42:51.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:51.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:51.001 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 00:42:51.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:51.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:51.001 | INFO | Uploaded 36GiB to GPU in 1.96305s (18.3388GiB/s) | 20260507 00:42:51.001 | INFO | Downloaded 34GiB to GPU in 1.99265s (17.0627GiB/s) | 20260507 00:42:56.147 | INFO | Analyzing chunk frames = [5746176,5746688) | 20260507 00:42:56.147 | INFO | MemInfo: { | 20260507 00:42:56.147 | INFO | "gpu" : " 17 GB", | 20260507 00:42:56.147 | INFO | "huge" : "", | 20260507 00:42:56.147 | INFO | "huge_shared" : " 27 GB", | 20260507 00:42:56.147 | INFO | "malloc" : " 14 GB", | 20260507 00:42:56.147 | INFO | "pinned" : " 29 GB" | 20260507 00:42:56.147 | INFO | } | 20260507 00:42:56.147 | INFO | Perf report for the last 5.14608 seconds: | 20260507 00:42:56.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:42:56.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:56.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:56.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:56.147 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 00:42:56.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:56.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:56.147 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 00:42:56.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:56.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:42:56.147 | INFO | Uploaded 36GiB to GPU in 1.9858s (18.1287GiB/s) | 20260507 00:42:56.147 | INFO | Downloaded 34GiB to GPU in 1.99166s (17.0712GiB/s) | 20260507 00:42:56.244 | INFO | Starting a round of Baz IO | 20260507 00:42:59.364 | INFO | Populating RTMetrics report | 20260507 00:42:59.364 | INFO | RTMetrics report populated | 20260507 00:42:59.364 | INFO | Opening tmp RTMetrics json file | 20260507 00:42:59.365 | INFO | Serializing RTMetrics to json | 20260507 00:42:59.366 | INFO | Serialization done, swapping tmp file now... | 20260507 00:42:59.366 | INFO | RTMetrics done | 20260507 00:42:59.454 | INFO | Finished a round of Baz IO: Wrote 1.839 GiB in 3.209 seconds | 20260507 00:42:59.454 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.25% / 0.00% / 66.75% | 20260507 00:42:59.454 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.327973% / 3.97364e-06% / 7.94729e-05% | 20260507 00:42:59.454 | INFO | Baz IO overall: | 20260507 00:42:59.454 | INFO | BytesWritten : 2991813017862 | 20260507 00:42:59.454 | INFO | NumEvents : 1440947083024 | 20260507 00:43:01.283 | INFO | Analyzing chunk frames = [5746688,5747200) | 20260507 00:43:01.283 | INFO | MemInfo: { | 20260507 00:43:01.283 | INFO | "gpu" : " 17 GB", | 20260507 00:43:01.283 | INFO | "huge" : "", | 20260507 00:43:01.283 | INFO | "huge_shared" : " 27 GB", | 20260507 00:43:01.283 | INFO | "malloc" : " 14 GB", | 20260507 00:43:01.283 | INFO | "pinned" : " 29 GB" | 20260507 00:43:01.283 | INFO | } | 20260507 00:43:01.283 | INFO | Perf report for the last 5.13652 seconds: | 20260507 00:43:01.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:43:01.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:01.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:01.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:01.284 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260507 00:43:01.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:01.284 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 00:43:01.284 | INFO | PRE_HQ : 59.7%, 1.8, 56.4ms, 0 | 20260507 00:43:01.284 | INFO | BAZWRITER : 62.1%, 3.4, 107.2ms, 0 | 20260507 00:43:01.284 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:43:01.284 | INFO | Uploaded 36GiB to GPU in 2.04432s (17.6098GiB/s) | 20260507 00:43:01.284 | INFO | Downloaded 38GiB to GPU in 2.18404s (17.399GiB/s) | 20260507 00:43:06.421 | INFO | Analyzing chunk frames = [5747200,5747712) | 20260507 00:43:06.421 | INFO | MemInfo: { | 20260507 00:43:06.421 | INFO | "gpu" : " 17 GB", | 20260507 00:43:06.421 | INFO | "huge" : "", | 20260507 00:43:06.421 | INFO | "huge_shared" : " 27 GB", | 20260507 00:43:06.421 | INFO | "malloc" : " 14 GB", | 20260507 00:43:06.421 | INFO | "pinned" : " 29 GB" | 20260507 00:43:06.421 | INFO | } | 20260507 00:43:06.421 | INFO | Perf report for the last 5.13765 seconds: | 20260507 00:43:06.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:43:06.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:06.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:06.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:06.421 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:43:06.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:06.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:06.421 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:43:06.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:06.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:06.421 | INFO | Uploaded 36GiB to GPU in 2.0131s (17.8829GiB/s) | 20260507 00:43:06.422 | INFO | Downloaded 34GiB to GPU in 1.93129s (17.6048GiB/s) | 20260507 00:43:11.559 | INFO | Analyzing chunk frames = [5747712,5748224) | 20260507 00:43:11.559 | INFO | MemInfo: { | 20260507 00:43:11.559 | INFO | "gpu" : " 17 GB", | 20260507 00:43:11.559 | INFO | "huge" : "", | 20260507 00:43:11.559 | INFO | "huge_shared" : " 27 GB", | 20260507 00:43:11.559 | INFO | "malloc" : " 14 GB", | 20260507 00:43:11.559 | INFO | "pinned" : " 29 GB" | 20260507 00:43:11.559 | INFO | } | 20260507 00:43:11.559 | INFO | Perf report for the last 5.13833 seconds: | 20260507 00:43:11.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:43:11.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:11.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:11.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:11.559 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:43:11.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:11.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:11.560 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:43:11.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:11.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:11.560 | INFO | Uploaded 36GiB to GPU in 1.99078s (18.0834GiB/s) | 20260507 00:43:11.560 | INFO | Downloaded 34GiB to GPU in 1.93004s (17.6163GiB/s) | 20260507 00:43:16.703 | INFO | Analyzing chunk frames = [5748224,5748736) | 20260507 00:43:16.703 | INFO | MemInfo: { | 20260507 00:43:16.703 | INFO | "gpu" : " 17 GB", | 20260507 00:43:16.703 | INFO | "huge" : "", | 20260507 00:43:16.703 | INFO | "huge_shared" : " 27 GB", | 20260507 00:43:16.703 | INFO | "malloc" : " 14 GB", | 20260507 00:43:16.703 | INFO | "pinned" : " 29 GB" | 20260507 00:43:16.703 | INFO | } | 20260507 00:43:16.703 | INFO | Perf report for the last 5.14387 seconds: | 20260507 00:43:16.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:43:16.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:16.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:16.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:16.703 | INFO | ANALYSIS : 60.0%, 3.0, 93.2ms, 0 | 20260507 00:43:16.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:16.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:16.703 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:43:16.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:16.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:16.704 | INFO | Uploaded 36GiB to GPU in 2.01873s (17.833GiB/s) | 20260507 00:43:16.704 | INFO | Downloaded 34GiB to GPU in 1.90965s (17.8043GiB/s) | 20260507 00:43:21.836 | INFO | Analyzing chunk frames = [5748736,5749248) | 20260507 00:43:21.836 | INFO | MemInfo: { | 20260507 00:43:21.836 | INFO | "gpu" : " 17 GB", | 20260507 00:43:21.836 | INFO | "huge" : "", | 20260507 00:43:21.836 | INFO | "huge_shared" : " 27 GB", | 20260507 00:43:21.836 | INFO | "malloc" : " 14 GB", | 20260507 00:43:21.836 | INFO | "pinned" : " 29 GB" | 20260507 00:43:21.836 | INFO | } | 20260507 00:43:21.836 | INFO | Perf report for the last 5.13295 seconds: | 20260507 00:43:21.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:43:21.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:21.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:21.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:21.836 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 00:43:21.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:21.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:21.836 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 00:43:21.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:21.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:21.837 | INFO | Uploaded 36GiB to GPU in 2.02301s (17.7953GiB/s) | 20260507 00:43:21.837 | INFO | Downloaded 34GiB to GPU in 1.9401s (17.5249GiB/s) | 20260507 00:43:26.974 | INFO | Analyzing chunk frames = [5749248,5749760) | 20260507 00:43:26.974 | INFO | MemInfo: { | 20260507 00:43:26.974 | INFO | "gpu" : " 17 GB", | 20260507 00:43:26.974 | INFO | "huge" : "", | 20260507 00:43:26.974 | INFO | "huge_shared" : " 27 GB", | 20260507 00:43:26.974 | INFO | "malloc" : " 14 GB", | 20260507 00:43:26.974 | INFO | "pinned" : " 29 GB" | 20260507 00:43:26.974 | INFO | } | 20260507 00:43:26.974 | INFO | Perf report for the last 5.13777 seconds: | 20260507 00:43:26.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:43:26.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:26.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:26.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:26.974 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 00:43:26.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:26.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:26.974 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 00:43:26.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:26.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:26.974 | INFO | Uploaded 36GiB to GPU in 2.0522s (17.5422GiB/s) | 20260507 00:43:26.974 | INFO | Downloaded 34GiB to GPU in 1.94879s (17.4467GiB/s) | 20260507 00:43:32.113 | INFO | Analyzing chunk frames = [5749760,5750272) | 20260507 00:43:32.113 | INFO | MemInfo: { | 20260507 00:43:32.113 | INFO | "gpu" : " 17 GB", | 20260507 00:43:32.113 | INFO | "huge" : "", | 20260507 00:43:32.113 | INFO | "huge_shared" : " 27 GB", | 20260507 00:43:32.113 | INFO | "malloc" : " 14 GB", | 20260507 00:43:32.113 | INFO | "pinned" : " 29 GB" | 20260507 00:43:32.113 | INFO | } | 20260507 00:43:32.113 | INFO | Perf report for the last 5.13912 seconds: | 20260507 00:43:32.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:43:32.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:32.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:32.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:32.113 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260507 00:43:32.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:32.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:32.113 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 00:43:32.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:32.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:32.114 | INFO | Uploaded 36GiB to GPU in 2.00229s (17.9794GiB/s) | 20260507 00:43:32.114 | INFO | Downloaded 34GiB to GPU in 1.99874s (17.0107GiB/s) | 20260507 00:43:37.255 | INFO | Analyzing chunk frames = [5750272,5750784) | 20260507 00:43:37.255 | INFO | MemInfo: { | 20260507 00:43:37.255 | INFO | "gpu" : " 17 GB", | 20260507 00:43:37.255 | INFO | "huge" : "", | 20260507 00:43:37.255 | INFO | "huge_shared" : " 27 GB", | 20260507 00:43:37.255 | INFO | "malloc" : " 14 GB", | 20260507 00:43:37.255 | INFO | "pinned" : " 29 GB" | 20260507 00:43:37.255 | INFO | } | 20260507 00:43:37.255 | INFO | Perf report for the last 5.14209 seconds: | 20260507 00:43:37.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:43:37.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:37.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:37.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:37.255 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 00:43:37.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:37.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:37.255 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:43:37.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:37.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:37.256 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:43:37.256 | INFO | Total mem usage: 18 GiB | 20260507 00:43:37.256 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:43:37.256 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:43:37.256 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:43:37.256 | INFO | High Water Breakdown: | 20260507 00:43:37.256 | INFO | 60.2% : 8.59 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:43:37.256 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:43:37.256 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:43:37.256 | INFO | ------------------------------------------ | 20260507 00:43:37.256 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:43:37.256 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:43:37.256 | INFO | Total mem usage: 26.5 GiB | 20260507 00:43:37.256 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:43:37.256 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:43:37.256 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:43:37.256 | INFO | Uploaded 36GiB to GPU in 1.98324s (18.1521GiB/s) | 20260507 00:43:37.256 | INFO | Downloaded 34GiB to GPU in 1.98669s (17.1139GiB/s) | 20260507 00:43:37.256 | INFO | High Water Breakdown: | 20260507 00:43:37.256 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:43:37.256 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:43:37.256 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:43:37.256 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:43:37.256 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:43:37.256 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:43:37.256 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:43:37.256 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:43:37.256 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:43:37.256 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:43:37.256 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:43:37.256 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:43:37.256 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:43:37.256 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:43:37.256 | INFO | ------------------------------------------ | 20260507 00:43:37.256 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:43:37.256 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:43:37.256 | INFO | Total mem usage: 25.6 GiB | 20260507 00:43:37.256 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:43:37.256 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:43:37.256 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:43:37.256 | INFO | High Water Breakdown: | 20260507 00:43:37.256 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:43:37.256 | INFO | ------------------------------------------ | 20260507 00:43:37.256 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:43:37.256 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:43:37.256 | INFO | Total mem usage: 15.7 GiB | 20260507 00:43:37.256 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:43:37.256 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:43:37.256 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 00:43:37.256 | INFO | High Water Breakdown: | 20260507 00:43:37.256 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:43:37.257 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:43:37.257 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:43:37.257 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:43:37.257 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:43:37.257 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:43:37.257 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:43:37.257 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:43:37.257 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:43:37.257 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:43:37.257 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:43:37.257 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:43:37.257 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:43:37.257 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:43:37.257 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:43:37.257 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:43:37.257 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:43:37.257 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:43:37.257 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:43:37.257 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:43:37.257 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:43:37.257 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:43:37.257 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:43:37.257 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:43:37.257 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:43:37.257 | INFO | ------------------------------------------ | 20260507 00:43:37.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:43:37.257 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:43:37.257 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:43:37.257 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:43:37.257 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.60e+01, 1.33e+01 / 2.36e+00 | 20260507 00:43:37.257 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 1.00e-01, 9.51e-03 / 5.66e-03 | 20260507 00:43:37.257 | INFO | PerfReport: 11.65%, Baseline -- 3.60e+00 / 4.44e+00, 5.27e-01 / 3.12e-01 | 20260507 00:43:37.257 | INFO | PerfReport: 3.23%, Histogram -- 1.00e+00 / 1.47e+00, 1.74e-01 / 1.33e-01 | 20260507 00:43:37.257 | INFO | PerfReport: 16.16%, DME -- 4.00e+01 / 4.51e+01, 4.39e+00 / 3.15e+00 | 20260507 00:43:37.257 | INFO | PerfReport: 57.40%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.30e+00 / 1.97e+00 | 20260507 00:43:37.257 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.18e+00 / 2.79e+00, 3.85e-01 / 1.88e-01 | 20260507 00:43:37.257 | INFO | PerfReport: 3.14%, Metrics -- 9.71e-01 / 1.31e+00, 2.58e-01 / 1.74e-01 | 20260507 00:43:37.257 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.31e-01 / 3.84e-01, 5.25e-02 / 2.73e-02 | 20260507 00:43:37.257 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:43:37.257 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:43:37.257 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:43:37.257 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 00:43:37.257 | INFO | PerfReport: 21.67%, Upload -- 2.09e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260507 00:43:37.257 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260507 00:43:37.257 | INFO | PerfReport: 31.99%, Compute -- 3.09e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 00:43:37.257 | INFO | PerfReport: 1.45%, LockUpload -- 1.40e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 00:43:37.257 | INFO | PerfReport: 1.21%, LockDownload -- 1.17e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260507 00:43:37.257 | INFO | PerfReport: 22.34%, LockCompute -- 2.16e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 00:43:37.257 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.68e-02 / 0.00e+00 | 20260507 00:43:37.352 | INFO | Starting a round of Baz IO | 20260507 00:43:40.476 | INFO | Populating RTMetrics report | 20260507 00:43:40.477 | INFO | RTMetrics report populated | 20260507 00:43:40.477 | INFO | Opening tmp RTMetrics json file | 20260507 00:43:40.478 | INFO | Serializing RTMetrics to json | 20260507 00:43:40.479 | INFO | Serialization done, swapping tmp file now... | 20260507 00:43:40.479 | INFO | RTMetrics done | 20260507 00:43:40.567 | INFO | Finished a round of Baz IO: Wrote 1.839 GiB in 3.214 seconds | 20260507 00:43:40.567 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.24% / 0.00% / 66.75% | 20260507 00:43:40.567 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.327849% / 0% / 6.75519e-05% | 20260507 00:43:40.567 | INFO | Baz IO overall: | 20260507 00:43:40.567 | INFO | BytesWritten : 2993787614356 | 20260507 00:43:40.567 | INFO | NumEvents : 1441884688693 | 20260507 00:43:42.403 | INFO | Analyzing chunk frames = [5750784,5751296) | 20260507 00:43:42.403 | INFO | MemInfo: { | 20260507 00:43:42.403 | INFO | "gpu" : " 17 GB", | 20260507 00:43:42.403 | INFO | "huge" : "", | 20260507 00:43:42.403 | INFO | "huge_shared" : " 27 GB", | 20260507 00:43:42.403 | INFO | "malloc" : " 14 GB", | 20260507 00:43:42.403 | INFO | "pinned" : " 29 GB" | 20260507 00:43:42.403 | INFO | } | 20260507 00:43:42.403 | INFO | Perf report for the last 5.14807 seconds: | 20260507 00:43:42.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:43:42.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:42.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:42.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:42.404 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260507 00:43:42.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:42.404 | INFO | RT_METRICS : 42.0%, 1.0, 22.1ms, 0 | 20260507 00:43:42.404 | INFO | PRE_HQ : 59.5%, 1.8, 56.6ms, 0 | 20260507 00:43:42.404 | INFO | BAZWRITER : 62.0%, 3.4, 107.4ms, 0 | 20260507 00:43:42.404 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:43:42.404 | INFO | Uploaded 36GiB to GPU in 2.00441s (17.9604GiB/s) | 20260507 00:43:42.404 | INFO | Downloaded 38GiB to GPU in 2.16728s (17.5335GiB/s) | 20260507 00:43:47.538 | INFO | Analyzing chunk frames = [5751296,5751808) | 20260507 00:43:47.538 | INFO | MemInfo: { | 20260507 00:43:47.538 | INFO | "gpu" : " 17 GB", | 20260507 00:43:47.538 | INFO | "huge" : "", | 20260507 00:43:47.538 | INFO | "huge_shared" : " 27 GB", | 20260507 00:43:47.538 | INFO | "malloc" : " 14 GB", | 20260507 00:43:47.538 | INFO | "pinned" : " 29 GB" | 20260507 00:43:47.538 | INFO | } | 20260507 00:43:47.538 | INFO | Perf report for the last 5.13475 seconds: | 20260507 00:43:47.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:43:47.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:47.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:47.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:47.538 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 00:43:47.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:47.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:47.538 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:43:47.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:47.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:47.538 | INFO | Uploaded 36GiB to GPU in 2.02769s (17.7542GiB/s) | 20260507 00:43:47.539 | INFO | Downloaded 34GiB to GPU in 1.93868s (17.5377GiB/s) | 20260507 00:43:52.681 | INFO | Analyzing chunk frames = [5751808,5752320) | 20260507 00:43:52.681 | INFO | MemInfo: { | 20260507 00:43:52.681 | INFO | "gpu" : " 17 GB", | 20260507 00:43:52.681 | INFO | "huge" : "", | 20260507 00:43:52.681 | INFO | "huge_shared" : " 27 GB", | 20260507 00:43:52.681 | INFO | "malloc" : " 14 GB", | 20260507 00:43:52.681 | INFO | "pinned" : " 29 GB" | 20260507 00:43:52.681 | INFO | } | 20260507 00:43:52.681 | INFO | Perf report for the last 5.14276 seconds: | 20260507 00:43:52.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:43:52.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:52.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:52.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:52.681 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:43:52.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:52.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:52.681 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:43:52.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:52.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:52.681 | INFO | Uploaded 36GiB to GPU in 1.99069s (18.0841GiB/s) | 20260507 00:43:52.681 | INFO | Downloaded 34GiB to GPU in 1.93771s (17.5465GiB/s) | 20260507 00:43:57.816 | INFO | Analyzing chunk frames = [5752320,5752832) | 20260507 00:43:57.816 | INFO | MemInfo: { | 20260507 00:43:57.816 | INFO | "gpu" : " 17 GB", | 20260507 00:43:57.816 | INFO | "huge" : "", | 20260507 00:43:57.816 | INFO | "huge_shared" : " 27 GB", | 20260507 00:43:57.816 | INFO | "malloc" : " 14 GB", | 20260507 00:43:57.816 | INFO | "pinned" : " 29 GB" | 20260507 00:43:57.816 | INFO | } | 20260507 00:43:57.816 | INFO | Perf report for the last 5.1351 seconds: | 20260507 00:43:57.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:43:57.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:57.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:57.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:57.816 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:43:57.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:57.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:57.816 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:43:57.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:57.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:43:57.816 | INFO | Uploaded 36GiB to GPU in 1.96476s (18.3228GiB/s) | 20260507 00:43:57.816 | INFO | Downloaded 34GiB to GPU in 1.90839s (17.8161GiB/s) | 20260507 00:44:02.950 | INFO | Analyzing chunk frames = [5752832,5753344) | 20260507 00:44:02.950 | INFO | MemInfo: { | 20260507 00:44:02.950 | INFO | "gpu" : " 17 GB", | 20260507 00:44:02.950 | INFO | "huge" : "", | 20260507 00:44:02.951 | INFO | "huge_shared" : " 27 GB", | 20260507 00:44:02.951 | INFO | "malloc" : " 14 GB", | 20260507 00:44:02.951 | INFO | "pinned" : " 29 GB" | 20260507 00:44:02.951 | INFO | } | 20260507 00:44:02.951 | INFO | Perf report for the last 5.13448 seconds: | 20260507 00:44:02.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:44:02.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:02.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:02.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:02.951 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 00:44:02.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:02.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:02.951 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:44:02.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:02.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:02.951 | INFO | Uploaded 36GiB to GPU in 1.99095s (18.0818GiB/s) | 20260507 00:44:02.951 | INFO | Downloaded 34GiB to GPU in 1.94875s (17.4471GiB/s) | 20260507 00:44:08.095 | INFO | Analyzing chunk frames = [5753344,5753856) | 20260507 00:44:08.095 | INFO | MemInfo: { | 20260507 00:44:08.095 | INFO | "gpu" : " 17 GB", | 20260507 00:44:08.095 | INFO | "huge" : "", | 20260507 00:44:08.095 | INFO | "huge_shared" : " 27 GB", | 20260507 00:44:08.095 | INFO | "malloc" : " 14 GB", | 20260507 00:44:08.095 | INFO | "pinned" : " 29 GB" | 20260507 00:44:08.095 | INFO | } | 20260507 00:44:08.095 | INFO | Perf report for the last 5.14425 seconds: | 20260507 00:44:08.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:44:08.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:08.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:08.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:08.095 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 00:44:08.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:08.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:08.095 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 00:44:08.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:08.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:08.095 | INFO | Uploaded 36GiB to GPU in 2.01654s (17.8524GiB/s) | 20260507 00:44:08.095 | INFO | Downloaded 34GiB to GPU in 1.94941s (17.4411GiB/s) | 20260507 00:44:13.236 | INFO | Analyzing chunk frames = [5753856,5754368) | 20260507 00:44:13.236 | INFO | MemInfo: { | 20260507 00:44:13.236 | INFO | "gpu" : " 17 GB", | 20260507 00:44:13.236 | INFO | "huge" : "", | 20260507 00:44:13.236 | INFO | "huge_shared" : " 27 GB", | 20260507 00:44:13.236 | INFO | "malloc" : " 14 GB", | 20260507 00:44:13.237 | INFO | "pinned" : " 29 GB" | 20260507 00:44:13.237 | INFO | } | 20260507 00:44:13.237 | INFO | Perf report for the last 5.14212 seconds: | 20260507 00:44:13.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:44:13.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:13.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:13.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:13.237 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 00:44:13.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:13.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:13.237 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 00:44:13.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:13.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:13.237 | INFO | Uploaded 36GiB to GPU in 1.97557s (18.2226GiB/s) | 20260507 00:44:13.237 | INFO | Downloaded 34GiB to GPU in 1.98464s (17.1315GiB/s) | 20260507 00:44:18.371 | INFO | Analyzing chunk frames = [5754368,5754880) | 20260507 00:44:18.371 | INFO | MemInfo: { | 20260507 00:44:18.371 | INFO | "gpu" : " 17 GB", | 20260507 00:44:18.371 | INFO | "huge" : "", | 20260507 00:44:18.371 | INFO | "huge_shared" : " 27 GB", | 20260507 00:44:18.371 | INFO | "malloc" : " 14 GB", | 20260507 00:44:18.371 | INFO | "pinned" : " 29 GB" | 20260507 00:44:18.371 | INFO | } | 20260507 00:44:18.371 | INFO | Perf report for the last 5.13375 seconds: | 20260507 00:44:18.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:44:18.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:18.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:18.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:18.371 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260507 00:44:18.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:18.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:18.371 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 00:44:18.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:18.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:18.371 | INFO | Uploaded 36GiB to GPU in 2.00924s (17.9172GiB/s) | 20260507 00:44:18.371 | INFO | Downloaded 34GiB to GPU in 1.97711s (17.1968GiB/s) | 20260507 00:44:18.469 | INFO | Starting a round of Baz IO | 20260507 00:44:21.597 | INFO | Populating RTMetrics report | 20260507 00:44:21.598 | INFO | RTMetrics report populated | 20260507 00:44:21.598 | INFO | Opening tmp RTMetrics json file | 20260507 00:44:21.598 | INFO | Serializing RTMetrics to json | 20260507 00:44:21.602 | INFO | Serialization done, swapping tmp file now... | 20260507 00:44:21.610 | INFO | RTMetrics done | 20260507 00:44:21.687 | INFO | Finished a round of Baz IO: Wrote 1.838 GiB in 3.218 seconds | 20260507 00:44:21.688 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.24% / 0.00% / 66.76% | 20260507 00:44:21.688 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.326399% / 3.97364e-06% / 6.35783e-05% | 20260507 00:44:21.688 | INFO | Baz IO overall: | 20260507 00:44:21.688 | INFO | BytesWritten : 2995761013467 | 20260507 00:44:21.688 | INFO | NumEvents : 1442821329526 | 20260507 00:44:23.516 | INFO | Analyzing chunk frames = [5754880,5755392) | 20260507 00:44:23.516 | INFO | MemInfo: { | 20260507 00:44:23.516 | INFO | "gpu" : " 17 GB", | 20260507 00:44:23.516 | INFO | "huge" : "", | 20260507 00:44:23.516 | INFO | "huge_shared" : " 27 GB", | 20260507 00:44:23.516 | INFO | "malloc" : " 14 GB", | 20260507 00:44:23.516 | INFO | "pinned" : " 29 GB" | 20260507 00:44:23.516 | INFO | } | 20260507 00:44:23.516 | INFO | Perf report for the last 5.14547 seconds: | 20260507 00:44:23.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:44:23.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:23.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:23.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:23.516 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260507 00:44:23.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:23.516 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260507 00:44:23.516 | INFO | PRE_HQ : 60.1%, 1.9, 59.8ms, 0 | 20260507 00:44:23.516 | INFO | BAZWRITER : 62.1%, 3.5, 110.3ms, 0 | 20260507 00:44:23.516 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:44:23.516 | INFO | Uploaded 36GiB to GPU in 2.03944s (17.6519GiB/s) | 20260507 00:44:23.516 | INFO | Downloaded 38GiB to GPU in 2.18149s (17.4192GiB/s) | 20260507 00:44:28.652 | INFO | Analyzing chunk frames = [5755392,5755904) | 20260507 00:44:28.652 | INFO | MemInfo: { | 20260507 00:44:28.652 | INFO | "gpu" : " 17 GB", | 20260507 00:44:28.652 | INFO | "huge" : "", | 20260507 00:44:28.652 | INFO | "huge_shared" : " 27 GB", | 20260507 00:44:28.652 | INFO | "malloc" : " 14 GB", | 20260507 00:44:28.652 | INFO | "pinned" : " 29 GB" | 20260507 00:44:28.653 | INFO | } | 20260507 00:44:28.653 | INFO | Perf report for the last 5.13635 seconds: | 20260507 00:44:28.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:44:28.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:28.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:28.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:28.653 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:44:28.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:28.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:28.653 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 00:44:28.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:28.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:28.653 | INFO | Uploaded 36GiB to GPU in 2.02922s (17.7408GiB/s) | 20260507 00:44:28.653 | INFO | Downloaded 34GiB to GPU in 1.9365s (17.5574GiB/s) | 20260507 00:44:33.789 | INFO | Analyzing chunk frames = [5755904,5756416) | 20260507 00:44:33.789 | INFO | MemInfo: { | 20260507 00:44:33.789 | INFO | "gpu" : " 17 GB", | 20260507 00:44:33.789 | INFO | "huge" : "", | 20260507 00:44:33.789 | INFO | "huge_shared" : " 27 GB", | 20260507 00:44:33.789 | INFO | "malloc" : " 14 GB", | 20260507 00:44:33.789 | INFO | "pinned" : " 29 GB" | 20260507 00:44:33.789 | INFO | } | 20260507 00:44:33.789 | INFO | Perf report for the last 5.13671 seconds: | 20260507 00:44:33.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:44:33.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:33.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:33.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:33.789 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:44:33.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:33.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:33.789 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:44:33.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:33.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:33.790 | INFO | Uploaded 36GiB to GPU in 1.99977s (18.0021GiB/s) | 20260507 00:44:33.790 | INFO | Downloaded 34GiB to GPU in 1.93444s (17.5761GiB/s) | 20260507 00:44:38.925 | INFO | Analyzing chunk frames = [5756416,5756928) | 20260507 00:44:38.925 | INFO | MemInfo: { | 20260507 00:44:38.925 | INFO | "gpu" : " 17 GB", | 20260507 00:44:38.925 | INFO | "huge" : "", | 20260507 00:44:38.925 | INFO | "huge_shared" : " 27 GB", | 20260507 00:44:38.926 | INFO | "malloc" : " 14 GB", | 20260507 00:44:38.926 | INFO | "pinned" : " 29 GB" | 20260507 00:44:38.926 | INFO | } | 20260507 00:44:38.926 | INFO | Perf report for the last 5.13642 seconds: | 20260507 00:44:38.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:44:38.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:38.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:38.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:38.926 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:44:38.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:38.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:38.926 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:44:38.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:38.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:38.926 | INFO | Uploaded 36GiB to GPU in 1.99682s (18.0287GiB/s) | 20260507 00:44:38.926 | INFO | Downloaded 34GiB to GPU in 1.90803s (17.8194GiB/s) | 20260507 00:44:44.067 | INFO | Analyzing chunk frames = [5756928,5757440) | 20260507 00:44:44.067 | INFO | MemInfo: { | 20260507 00:44:44.067 | INFO | "gpu" : " 17 GB", | 20260507 00:44:44.067 | INFO | "huge" : "", | 20260507 00:44:44.067 | INFO | "huge_shared" : " 27 GB", | 20260507 00:44:44.067 | INFO | "malloc" : " 14 GB", | 20260507 00:44:44.067 | INFO | "pinned" : " 29 GB" | 20260507 00:44:44.067 | INFO | } | 20260507 00:44:44.067 | INFO | Perf report for the last 5.14158 seconds: | 20260507 00:44:44.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:44:44.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:44.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:44.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:44.067 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 00:44:44.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:44.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:44.067 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:44:44.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:44.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:44.067 | INFO | Uploaded 36GiB to GPU in 1.9898s (18.0923GiB/s) | 20260507 00:44:44.068 | INFO | Downloaded 34GiB to GPU in 1.92712s (17.6429GiB/s) | 20260507 00:44:49.206 | INFO | Analyzing chunk frames = [5757440,5757952) | 20260507 00:44:49.206 | INFO | MemInfo: { | 20260507 00:44:49.206 | INFO | "gpu" : " 17 GB", | 20260507 00:44:49.206 | INFO | "huge" : "", | 20260507 00:44:49.206 | INFO | "huge_shared" : " 27 GB", | 20260507 00:44:49.206 | INFO | "malloc" : " 14 GB", | 20260507 00:44:49.206 | INFO | "pinned" : " 29 GB" | 20260507 00:44:49.206 | INFO | } | 20260507 00:44:49.206 | INFO | Perf report for the last 5.1387 seconds: | 20260507 00:44:49.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:44:49.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:49.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:49.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:49.206 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:44:49.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:49.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:49.206 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:44:49.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:49.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:49.206 | INFO | Uploaded 36GiB to GPU in 2.03805s (17.664GiB/s) | 20260507 00:44:49.206 | INFO | Downloaded 34GiB to GPU in 1.94034s (17.5227GiB/s) | 20260507 00:44:54.351 | INFO | Analyzing chunk frames = [5757952,5758464) | 20260507 00:44:54.351 | INFO | MemInfo: { | 20260507 00:44:54.351 | INFO | "gpu" : " 17 GB", | 20260507 00:44:54.351 | INFO | "huge" : "", | 20260507 00:44:54.351 | INFO | "huge_shared" : " 27 GB", | 20260507 00:44:54.351 | INFO | "malloc" : " 14 GB", | 20260507 00:44:54.351 | INFO | "pinned" : " 29 GB" | 20260507 00:44:54.351 | INFO | } | 20260507 00:44:54.351 | INFO | Perf report for the last 5.14487 seconds: | 20260507 00:44:54.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:44:54.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:54.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:54.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:54.351 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 00:44:54.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:54.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:54.351 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 00:44:54.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:54.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:54.351 | INFO | Uploaded 36GiB to GPU in 1.9503s (18.4587GiB/s) | 20260507 00:44:54.351 | INFO | Downloaded 34GiB to GPU in 1.98994s (17.0859GiB/s) | 20260507 00:44:59.494 | INFO | Analyzing chunk frames = [5758464,5758976) | 20260507 00:44:59.494 | INFO | MemInfo: { | 20260507 00:44:59.494 | INFO | "gpu" : " 17 GB", | 20260507 00:44:59.494 | INFO | "huge" : "", | 20260507 00:44:59.494 | INFO | "huge_shared" : " 27 GB", | 20260507 00:44:59.494 | INFO | "malloc" : " 14 GB", | 20260507 00:44:59.494 | INFO | "pinned" : " 29 GB" | 20260507 00:44:59.494 | INFO | } | 20260507 00:44:59.494 | INFO | Perf report for the last 5.14315 seconds: | 20260507 00:44:59.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:44:59.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:59.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:59.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:59.494 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260507 00:44:59.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:59.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:59.494 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 00:44:59.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:59.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:44:59.494 | INFO | Uploaded 36GiB to GPU in 2.00127s (17.9886GiB/s) | 20260507 00:44:59.494 | INFO | Downloaded 34GiB to GPU in 1.9795s (17.1761GiB/s) | 20260507 00:44:59.494 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:44:59.494 | INFO | Total mem usage: 18 GiB | 20260507 00:44:59.494 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:44:59.494 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.5%) | 20260507 00:44:59.494 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:44:59.494 | INFO | High Water Breakdown: | 20260507 00:44:59.494 | INFO | 60.2% : 8.64 GiB ( 75.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:44:59.494 | INFO | 39.7% : 5.69 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:44:59.494 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:44:59.495 | INFO | ------------------------------------------ | 20260507 00:44:59.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:44:59.495 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:44:59.495 | INFO | Total mem usage: 26.5 GiB | 20260507 00:44:59.495 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:44:59.495 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:44:59.495 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:44:59.495 | INFO | High Water Breakdown: | 20260507 00:44:59.495 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:44:59.495 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:44:59.495 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:44:59.495 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:44:59.495 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:44:59.495 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:44:59.495 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:44:59.495 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:44:59.495 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:44:59.495 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:44:59.495 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:44:59.495 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:44:59.495 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:44:59.495 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:44:59.495 | INFO | ------------------------------------------ | 20260507 00:44:59.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:44:59.495 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:44:59.495 | INFO | Total mem usage: 25.6 GiB | 20260507 00:44:59.495 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:44:59.495 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:44:59.495 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:44:59.495 | INFO | High Water Breakdown: | 20260507 00:44:59.495 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:44:59.495 | INFO | ------------------------------------------ | 20260507 00:44:59.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:44:59.495 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:44:59.495 | INFO | Total mem usage: 15.7 GiB | 20260507 00:44:59.495 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:44:59.495 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:44:59.495 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:44:59.495 | INFO | High Water Breakdown: | 20260507 00:44:59.495 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:44:59.495 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:44:59.495 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:44:59.495 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:44:59.495 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:44:59.495 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:44:59.495 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:44:59.495 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:44:59.495 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:44:59.495 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:44:59.495 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:44:59.495 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:44:59.495 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:44:59.495 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:44:59.495 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:44:59.495 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:44:59.495 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:44:59.495 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:44:59.495 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:44:59.495 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:44:59.495 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:44:59.495 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:44:59.495 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:44:59.495 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:44:59.495 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:44:59.495 | INFO | ------------------------------------------ | 20260507 00:44:59.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:44:59.495 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:44:59.495 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:44:59.495 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:44:59.495 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.60e+01, 1.35e+01 / 1.97e+00 | 20260507 00:44:59.495 | INFO | PerfReport: 0.30%, AggregateStats -- 9.14e-02 / 1.03e-01, 1.02e-02 / 3.44e-03 | 20260507 00:44:59.495 | INFO | PerfReport: 11.68%, Baseline -- 3.61e+00 / 4.48e+00, 5.37e-01 / 2.08e-01 | 20260507 00:44:59.495 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.48e+00, 1.74e-01 / 1.09e-01 | 20260507 00:44:59.495 | INFO | PerfReport: 16.11%, DME -- 3.98e+01 / 4.49e+01, 4.10e+00 / 2.12e+00 | 20260507 00:44:59.495 | INFO | PerfReport: 57.42%, FrameLabeling -- 1.77e+01 / 2.06e+01, 1.32e+00 / 1.61e+00 | 20260507 00:44:59.495 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.18e+00 / 2.74e+00, 3.83e-01 / 2.01e-01 | 20260507 00:44:59.495 | INFO | PerfReport: 3.14%, Metrics -- 9.71e-01 / 1.31e+00, 2.58e-01 / 1.35e-01 | 20260507 00:44:59.495 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.31e-01 / 3.89e-01, 5.29e-02 / 2.38e-02 | 20260507 00:44:59.495 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:44:59.495 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:44:59.495 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:44:59.496 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 00:44:59.496 | INFO | PerfReport: 21.59%, Upload -- 2.08e+01 / 0.00e+00, 7.04e+00 / 0.00e+00 | 20260507 00:44:59.496 | INFO | PerfReport: 21.31%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 00:44:59.496 | INFO | PerfReport: 32.02%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:44:59.496 | INFO | PerfReport: 1.35%, LockUpload -- 1.31e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 00:44:59.496 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260507 00:44:59.496 | INFO | PerfReport: 22.39%, LockCompute -- 2.16e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 00:44:59.496 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.29e-02 / 0.00e+00 | 20260507 00:44:59.593 | INFO | Starting a round of Baz IO | 20260507 00:45:02.734 | INFO | Populating RTMetrics report | 20260507 00:45:02.735 | INFO | RTMetrics report populated | 20260507 00:45:02.735 | INFO | Opening tmp RTMetrics json file | 20260507 00:45:02.736 | INFO | Serializing RTMetrics to json | 20260507 00:45:02.737 | INFO | Serialization done, swapping tmp file now... | 20260507 00:45:02.737 | INFO | RTMetrics done | 20260507 00:45:02.827 | INFO | Finished a round of Baz IO: Wrote 1.839 GiB in 3.234 seconds | 20260507 00:45:02.828 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.24% / 0.00% / 66.76% | 20260507 00:45:02.828 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.326606% / 7.94729e-06% / 7.94729e-05% | 20260507 00:45:02.828 | INFO | Baz IO overall: | 20260507 00:45:02.828 | INFO | BytesWritten : 2997735949347 | 20260507 00:45:02.828 | INFO | NumEvents : 1443759125323 | 20260507 00:45:04.632 | INFO | Analyzing chunk frames = [5758976,5759488) | 20260507 00:45:04.632 | INFO | MemInfo: { | 20260507 00:45:04.632 | INFO | "gpu" : " 17 GB", | 20260507 00:45:04.632 | INFO | "huge" : "", | 20260507 00:45:04.632 | INFO | "huge_shared" : " 27 GB", | 20260507 00:45:04.632 | INFO | "malloc" : " 14 GB", | 20260507 00:45:04.632 | INFO | "pinned" : " 29 GB" | 20260507 00:45:04.632 | INFO | } | 20260507 00:45:04.632 | INFO | Perf report for the last 5.13774 seconds: | 20260507 00:45:04.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:45:04.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:04.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:04.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:04.632 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 00:45:04.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:04.632 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260507 00:45:04.632 | INFO | PRE_HQ : 60.0%, 1.9, 59.3ms, 0 | 20260507 00:45:04.632 | INFO | BAZWRITER : 62.5%, 3.4, 110.1ms, 0 | 20260507 00:45:04.632 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260507 00:45:04.632 | INFO | Uploaded 36GiB to GPU in 2.01976s (17.8239GiB/s) | 20260507 00:45:04.632 | INFO | Downloaded 38GiB to GPU in 2.1737s (17.4817GiB/s) | 20260507 00:45:09.770 | INFO | Analyzing chunk frames = [5759488,5760000) | 20260507 00:45:09.770 | INFO | MemInfo: { | 20260507 00:45:09.770 | INFO | "gpu" : " 17 GB", | 20260507 00:45:09.770 | INFO | "huge" : "", | 20260507 00:45:09.770 | INFO | "huge_shared" : " 27 GB", | 20260507 00:45:09.770 | INFO | "malloc" : " 14 GB", | 20260507 00:45:09.770 | INFO | "pinned" : " 29 GB" | 20260507 00:45:09.770 | INFO | } | 20260507 00:45:09.770 | INFO | Perf report for the last 5.1379 seconds: | 20260507 00:45:09.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:45:09.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:09.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:09.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:09.770 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:45:09.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:09.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:09.770 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:45:09.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:09.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:09.770 | INFO | Uploaded 36GiB to GPU in 2.00976s (17.9126GiB/s) | 20260507 00:45:09.770 | INFO | Downloaded 34GiB to GPU in 1.93601s (17.5619GiB/s) | 20260507 00:45:14.902 | INFO | Analyzing chunk frames = [5760000,5760512) | 20260507 00:45:14.902 | INFO | MemInfo: { | 20260507 00:45:14.902 | INFO | "gpu" : " 17 GB", | 20260507 00:45:14.902 | INFO | "huge" : "", | 20260507 00:45:14.902 | INFO | "huge_shared" : " 27 GB", | 20260507 00:45:14.902 | INFO | "malloc" : " 14 GB", | 20260507 00:45:14.902 | INFO | "pinned" : " 29 GB" | 20260507 00:45:14.902 | INFO | } | 20260507 00:45:14.902 | INFO | Perf report for the last 5.13253 seconds: | 20260507 00:45:14.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:45:14.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:14.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:14.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:14.902 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 00:45:14.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:14.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:14.902 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:45:14.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:14.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:14.903 | INFO | Uploaded 36GiB to GPU in 1.9957s (18.0388GiB/s) | 20260507 00:45:14.903 | INFO | Downloaded 34GiB to GPU in 1.93772s (17.5464GiB/s) | 20260507 00:45:20.043 | INFO | Analyzing chunk frames = [5760512,5761024) | 20260507 00:45:20.043 | INFO | MemInfo: { | 20260507 00:45:20.043 | INFO | "gpu" : " 17 GB", | 20260507 00:45:20.043 | INFO | "huge" : "", | 20260507 00:45:20.043 | INFO | "huge_shared" : " 27 GB", | 20260507 00:45:20.043 | INFO | "malloc" : " 14 GB", | 20260507 00:45:20.043 | INFO | "pinned" : " 29 GB" | 20260507 00:45:20.043 | INFO | } | 20260507 00:45:20.043 | INFO | Perf report for the last 5.14119 seconds: | 20260507 00:45:20.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:45:20.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:20.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:20.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:20.044 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 00:45:20.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:20.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:20.044 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:45:20.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:20.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:20.044 | INFO | Uploaded 36GiB to GPU in 1.95075s (18.4544GiB/s) | 20260507 00:45:20.044 | INFO | Downloaded 34GiB to GPU in 1.8997s (17.8975GiB/s) | 20260507 00:45:25.193 | INFO | Analyzing chunk frames = [5761024,5761536) | 20260507 00:45:25.193 | INFO | MemInfo: { | 20260507 00:45:25.193 | INFO | "gpu" : " 17 GB", | 20260507 00:45:25.193 | INFO | "huge" : "", | 20260507 00:45:25.193 | INFO | "huge_shared" : " 27 GB", | 20260507 00:45:25.193 | INFO | "malloc" : " 14 GB", | 20260507 00:45:25.193 | INFO | "pinned" : " 29 GB" | 20260507 00:45:25.193 | INFO | } | 20260507 00:45:25.193 | INFO | Perf report for the last 5.14965 seconds: | 20260507 00:45:25.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:45:25.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:25.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:25.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:25.193 | INFO | ANALYSIS : 59.9%, 3.0, 93.4ms, 0 | 20260507 00:45:25.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:25.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:25.193 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:45:25.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:25.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:25.193 | INFO | Uploaded 36GiB to GPU in 1.98233s (18.1604GiB/s) | 20260507 00:45:25.193 | INFO | Downloaded 34GiB to GPU in 1.93747s (17.5486GiB/s) | 20260507 00:45:30.324 | INFO | Analyzing chunk frames = [5761536,5762048) | 20260507 00:45:30.324 | INFO | MemInfo: { | 20260507 00:45:30.324 | INFO | "gpu" : " 17 GB", | 20260507 00:45:30.324 | INFO | "huge" : "", | 20260507 00:45:30.324 | INFO | "huge_shared" : " 27 GB", | 20260507 00:45:30.324 | INFO | "malloc" : " 14 GB", | 20260507 00:45:30.324 | INFO | "pinned" : " 29 GB" | 20260507 00:45:30.324 | INFO | } | 20260507 00:45:30.324 | INFO | Perf report for the last 5.13066 seconds: | 20260507 00:45:30.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:45:30.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:30.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:30.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:30.324 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 00:45:30.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:30.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:30.324 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:45:30.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:30.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:30.324 | INFO | Uploaded 36GiB to GPU in 2.04269s (17.6238GiB/s) | 20260507 00:45:30.324 | INFO | Downloaded 34GiB to GPU in 1.96148s (17.3338GiB/s) | 20260507 00:45:35.465 | INFO | Analyzing chunk frames = [5762048,5762560) | 20260507 00:45:35.465 | INFO | MemInfo: { | 20260507 00:45:35.465 | INFO | "gpu" : " 17 GB", | 20260507 00:45:35.465 | INFO | "huge" : "", | 20260507 00:45:35.465 | INFO | "huge_shared" : " 27 GB", | 20260507 00:45:35.465 | INFO | "malloc" : " 14 GB", | 20260507 00:45:35.465 | INFO | "pinned" : " 29 GB" | 20260507 00:45:35.465 | INFO | } | 20260507 00:45:35.465 | INFO | Perf report for the last 5.14136 seconds: | 20260507 00:45:35.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:45:35.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:35.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:35.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:35.465 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:45:35.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:35.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:35.465 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 00:45:35.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:35.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:35.466 | INFO | Uploaded 36GiB to GPU in 1.9936s (18.0578GiB/s) | 20260507 00:45:35.466 | INFO | Downloaded 34GiB to GPU in 1.98921s (17.0923GiB/s) | 20260507 00:45:40.603 | INFO | Analyzing chunk frames = [5762560,5763072) | 20260507 00:45:40.603 | INFO | MemInfo: { | 20260507 00:45:40.603 | INFO | "gpu" : " 17 GB", | 20260507 00:45:40.603 | INFO | "huge" : "", | 20260507 00:45:40.603 | INFO | "huge_shared" : " 27 GB", | 20260507 00:45:40.603 | INFO | "malloc" : " 14 GB", | 20260507 00:45:40.603 | INFO | "pinned" : " 29 GB" | 20260507 00:45:40.603 | INFO | } | 20260507 00:45:40.603 | INFO | Perf report for the last 5.13823 seconds: | 20260507 00:45:40.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:45:40.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:40.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:40.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:40.604 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 00:45:40.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:40.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:40.604 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:45:40.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:40.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:40.604 | INFO | Uploaded 36GiB to GPU in 2.00633s (17.9432GiB/s) | 20260507 00:45:40.604 | INFO | Downloaded 34GiB to GPU in 1.97533s (17.2123GiB/s) | 20260507 00:45:40.703 | INFO | Starting a round of Baz IO | 20260507 00:45:43.819 | INFO | Populating RTMetrics report | 20260507 00:45:43.819 | INFO | RTMetrics report populated | 20260507 00:45:43.819 | INFO | Opening tmp RTMetrics json file | 20260507 00:45:43.820 | INFO | Serializing RTMetrics to json | 20260507 00:45:43.822 | INFO | Serialization done, swapping tmp file now... | 20260507 00:45:43.830 | INFO | RTMetrics done | 20260507 00:45:43.920 | INFO | Finished a round of Baz IO: Wrote 1.839 GiB in 3.217 seconds | 20260507 00:45:43.920 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.24% / 0.00% / 66.76% | 20260507 00:45:43.920 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.326145% / 3.97364e-06% / 7.54992e-05% | 20260507 00:45:43.920 | INFO | Baz IO overall: | 20260507 00:45:43.920 | INFO | BytesWritten : 2999710089890 | 20260507 00:45:43.920 | INFO | NumEvents : 1444696783931 | 20260507 00:45:45.740 | INFO | Analyzing chunk frames = [5763072,5763584) | 20260507 00:45:45.740 | INFO | MemInfo: { | 20260507 00:45:45.740 | INFO | "gpu" : " 17 GB", | 20260507 00:45:45.740 | INFO | "huge" : "", | 20260507 00:45:45.740 | INFO | "huge_shared" : " 27 GB", | 20260507 00:45:45.740 | INFO | "malloc" : " 14 GB", | 20260507 00:45:45.740 | INFO | "pinned" : " 29 GB" | 20260507 00:45:45.740 | INFO | } | 20260507 00:45:45.740 | INFO | Perf report for the last 5.13702 seconds: | 20260507 00:45:45.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:45:45.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:45.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:45.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:45.741 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 00:45:45.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:45.741 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 00:45:45.741 | INFO | PRE_HQ : 59.6%, 1.8, 56.7ms, 0 | 20260507 00:45:45.741 | INFO | BAZWRITER : 62.2%, 3.3, 106.7ms, 0 | 20260507 00:45:45.741 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 00:45:45.741 | INFO | Uploaded 36GiB to GPU in 2.0269s (17.7611GiB/s) | 20260507 00:45:45.741 | INFO | Downloaded 38GiB to GPU in 2.17689s (17.4561GiB/s) | 20260507 00:45:50.877 | INFO | Analyzing chunk frames = [5763584,5764096) | 20260507 00:45:50.877 | INFO | MemInfo: { | 20260507 00:45:50.877 | INFO | "gpu" : " 17 GB", | 20260507 00:45:50.877 | INFO | "huge" : "", | 20260507 00:45:50.877 | INFO | "huge_shared" : " 27 GB", | 20260507 00:45:50.877 | INFO | "malloc" : " 14 GB", | 20260507 00:45:50.877 | INFO | "pinned" : " 29 GB" | 20260507 00:45:50.877 | INFO | } | 20260507 00:45:50.877 | INFO | Perf report for the last 5.13688 seconds: | 20260507 00:45:50.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:45:50.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:50.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:50.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:50.877 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:45:50.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:50.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:50.877 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:45:50.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:50.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:50.878 | INFO | Uploaded 36GiB to GPU in 2.01757s (17.8432GiB/s) | 20260507 00:45:50.878 | INFO | Downloaded 34GiB to GPU in 1.93004s (17.6162GiB/s) | 20260507 00:45:56.019 | INFO | Analyzing chunk frames = [5764096,5764608) | 20260507 00:45:56.019 | INFO | MemInfo: { | 20260507 00:45:56.019 | INFO | "gpu" : " 17 GB", | 20260507 00:45:56.019 | INFO | "huge" : "", | 20260507 00:45:56.019 | INFO | "huge_shared" : " 27 GB", | 20260507 00:45:56.019 | INFO | "malloc" : " 14 GB", | 20260507 00:45:56.019 | INFO | "pinned" : " 29 GB" | 20260507 00:45:56.019 | INFO | } | 20260507 00:45:56.019 | INFO | Perf report for the last 5.1421 seconds: | 20260507 00:45:56.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:45:56.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:56.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:56.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:56.020 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:45:56.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:56.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:56.020 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:45:56.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:56.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:45:56.020 | INFO | Uploaded 36GiB to GPU in 1.99214s (18.071GiB/s) | 20260507 00:45:56.020 | INFO | Downloaded 34GiB to GPU in 1.92669s (17.6468GiB/s) | 20260507 00:46:01.165 | INFO | Analyzing chunk frames = [5764608,5765120) | 20260507 00:46:01.165 | INFO | MemInfo: { | 20260507 00:46:01.165 | INFO | "gpu" : " 17 GB", | 20260507 00:46:01.165 | INFO | "huge" : "", | 20260507 00:46:01.165 | INFO | "huge_shared" : " 27 GB", | 20260507 00:46:01.165 | INFO | "malloc" : " 14 GB", | 20260507 00:46:01.165 | INFO | "pinned" : " 29 GB" | 20260507 00:46:01.165 | INFO | } | 20260507 00:46:01.165 | INFO | Perf report for the last 5.14555 seconds: | 20260507 00:46:01.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:46:01.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:01.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:01.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:01.165 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 00:46:01.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:01.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:01.165 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:46:01.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:01.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:01.165 | INFO | Uploaded 36GiB to GPU in 2.01074s (17.9038GiB/s) | 20260507 00:46:01.165 | INFO | Downloaded 34GiB to GPU in 1.91003s (17.8008GiB/s) | 20260507 00:46:06.305 | INFO | Analyzing chunk frames = [5765120,5765632) | 20260507 00:46:06.305 | INFO | MemInfo: { | 20260507 00:46:06.305 | INFO | "gpu" : " 17 GB", | 20260507 00:46:06.305 | INFO | "huge" : "", | 20260507 00:46:06.305 | INFO | "huge_shared" : " 27 GB", | 20260507 00:46:06.305 | INFO | "malloc" : " 14 GB", | 20260507 00:46:06.305 | INFO | "pinned" : " 29 GB" | 20260507 00:46:06.305 | INFO | } | 20260507 00:46:06.305 | INFO | Perf report for the last 5.14022 seconds: | 20260507 00:46:06.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:46:06.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:06.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:06.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:06.305 | INFO | ANALYSIS : 60.2%, 3.0, 94.0ms, 0 | 20260507 00:46:06.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:06.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:06.306 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:46:06.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:06.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:06.306 | INFO | Uploaded 36GiB to GPU in 2.01464s (17.8692GiB/s) | 20260507 00:46:06.306 | INFO | Downloaded 34GiB to GPU in 1.94452s (17.485GiB/s) | 20260507 00:46:11.445 | INFO | Analyzing chunk frames = [5765632,5766144) | 20260507 00:46:11.445 | INFO | MemInfo: { | 20260507 00:46:11.445 | INFO | "gpu" : " 17 GB", | 20260507 00:46:11.445 | INFO | "huge" : "", | 20260507 00:46:11.445 | INFO | "huge_shared" : " 27 GB", | 20260507 00:46:11.445 | INFO | "malloc" : " 14 GB", | 20260507 00:46:11.445 | INFO | "pinned" : " 29 GB" | 20260507 00:46:11.445 | INFO | } | 20260507 00:46:11.445 | INFO | Perf report for the last 5.13974 seconds: | 20260507 00:46:11.445 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:46:11.445 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:11.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:11.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:11.445 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 00:46:11.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:11.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:11.445 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 00:46:11.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:11.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:11.445 | INFO | Uploaded 36GiB to GPU in 2.03097s (17.7255GiB/s) | 20260507 00:46:11.445 | INFO | Downloaded 34GiB to GPU in 1.91577s (17.7474GiB/s) | 20260507 00:46:16.579 | INFO | Analyzing chunk frames = [5766144,5766656) | 20260507 00:46:16.579 | INFO | MemInfo: { | 20260507 00:46:16.579 | INFO | "gpu" : " 17 GB", | 20260507 00:46:16.579 | INFO | "huge" : "", | 20260507 00:46:16.579 | INFO | "huge_shared" : " 27 GB", | 20260507 00:46:16.579 | INFO | "malloc" : " 14 GB", | 20260507 00:46:16.579 | INFO | "pinned" : " 29 GB" | 20260507 00:46:16.579 | INFO | } | 20260507 00:46:16.579 | INFO | Perf report for the last 5.13445 seconds: | 20260507 00:46:16.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:46:16.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:16.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:16.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:16.580 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 00:46:16.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:16.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:16.580 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:46:16.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:16.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:16.580 | INFO | Uploaded 36GiB to GPU in 1.95697s (18.3958GiB/s) | 20260507 00:46:16.580 | INFO | Downloaded 34GiB to GPU in 1.99187s (17.0693GiB/s) | 20260507 00:46:21.716 | INFO | Analyzing chunk frames = [5766656,5767168) | 20260507 00:46:21.716 | INFO | MemInfo: { | 20260507 00:46:21.716 | INFO | "gpu" : " 17 GB", | 20260507 00:46:21.716 | INFO | "huge" : "", | 20260507 00:46:21.716 | INFO | "huge_shared" : " 27 GB", | 20260507 00:46:21.716 | INFO | "malloc" : " 14 GB", | 20260507 00:46:21.716 | INFO | "pinned" : " 29 GB" | 20260507 00:46:21.716 | INFO | } | 20260507 00:46:21.716 | INFO | Perf report for the last 5.13672 seconds: | 20260507 00:46:21.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:46:21.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:21.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:21.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:21.716 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:46:21.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:21.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:21.716 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 00:46:21.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:21.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:21.716 | INFO | Uploaded 36GiB to GPU in 2.00429s (17.9615GiB/s) | 20260507 00:46:21.717 | INFO | Downloaded 34GiB to GPU in 1.99418s (17.0496GiB/s) | 20260507 00:46:21.717 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:46:21.717 | INFO | Total mem usage: 18 GiB | 20260507 00:46:21.717 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:46:21.717 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:46:21.717 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:46:21.717 | INFO | High Water Breakdown: | 20260507 00:46:21.717 | INFO | 60.2% : 8.6 GiB ( 75.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:46:21.717 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:46:21.717 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:46:21.717 | INFO | ------------------------------------------ | 20260507 00:46:21.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:46:21.717 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:46:21.717 | INFO | Total mem usage: 26.5 GiB | 20260507 00:46:21.717 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:46:21.717 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:46:21.717 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:46:21.717 | INFO | High Water Breakdown: | 20260507 00:46:21.717 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:46:21.717 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:46:21.717 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:46:21.717 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:46:21.717 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:46:21.717 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:46:21.717 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:46:21.717 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:46:21.717 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:46:21.717 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:46:21.717 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:46:21.717 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:46:21.717 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:46:21.717 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:46:21.717 | INFO | ------------------------------------------ | 20260507 00:46:21.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:46:21.717 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:46:21.717 | INFO | Total mem usage: 25.6 GiB | 20260507 00:46:21.717 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:46:21.717 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:46:21.717 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:46:21.717 | INFO | High Water Breakdown: | 20260507 00:46:21.717 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:46:21.717 | INFO | ------------------------------------------ | 20260507 00:46:21.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:46:21.717 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:46:21.717 | INFO | Total mem usage: 15.7 GiB | 20260507 00:46:21.717 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:46:21.717 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:46:21.717 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 00:46:21.717 | INFO | High Water Breakdown: | 20260507 00:46:21.717 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:46:21.717 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:46:21.717 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:46:21.717 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:46:21.717 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:46:21.717 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:46:21.718 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:46:21.718 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:46:21.718 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:46:21.718 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:46:21.718 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:46:21.718 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:46:21.718 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:46:21.718 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:46:21.718 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:46:21.718 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:46:21.718 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:46:21.718 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:46:21.718 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:46:21.718 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:46:21.718 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:46:21.718 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:46:21.718 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:46:21.718 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:46:21.718 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:46:21.718 | INFO | ------------------------------------------ | 20260507 00:46:21.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:46:21.718 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:46:21.718 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:46:21.718 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:46:21.718 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.55e+01, 1.31e+01 / 2.10e+00 | 20260507 00:46:21.718 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.01e-01, 1.02e-02 / 5.18e-03 | 20260507 00:46:21.718 | INFO | PerfReport: 11.62%, Baseline -- 3.60e+00 / 4.39e+00, 5.28e-01 / 3.56e-01 | 20260507 00:46:21.718 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.46e+00, 1.76e-01 / 1.26e-01 | 20260507 00:46:21.718 | INFO | PerfReport: 16.07%, DME -- 3.98e+01 / 4.45e+01, 3.93e+00 / 2.35e+00 | 20260507 00:46:21.718 | INFO | PerfReport: 57.54%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.40e+00 / 1.65e+00 | 20260507 00:46:21.718 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.17e+00 / 2.68e+00, 3.80e-01 / 2.38e-01 | 20260507 00:46:21.718 | INFO | PerfReport: 3.14%, Metrics -- 9.70e-01 / 1.28e+00, 2.59e-01 / 1.04e-01 | 20260507 00:46:21.718 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.30e-01 / 3.86e-01, 5.29e-02 / 2.79e-02 | 20260507 00:46:21.718 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:46:21.718 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:46:21.718 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:46:21.718 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 00:46:21.718 | INFO | PerfReport: 21.61%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260507 00:46:21.718 | INFO | PerfReport: 21.31%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260507 00:46:21.718 | INFO | PerfReport: 32.05%, Compute -- 3.09e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 00:46:21.718 | INFO | PerfReport: 1.36%, LockUpload -- 1.31e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 00:46:21.718 | INFO | PerfReport: 1.26%, LockDownload -- 1.22e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260507 00:46:21.718 | INFO | PerfReport: 22.40%, LockCompute -- 2.16e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260507 00:46:21.718 | INFO | PerfReport: 0.01%, Misc -- 1.25e-02 / 0.00e+00, 1.29e-02 / 0.00e+00 | 20260507 00:46:21.814 | INFO | Starting a round of Baz IO | 20260507 00:46:24.936 | INFO | Populating RTMetrics report | 20260507 00:46:24.937 | INFO | RTMetrics report populated | 20260507 00:46:24.937 | INFO | Opening tmp RTMetrics json file | 20260507 00:46:24.942 | INFO | Serializing RTMetrics to json | 20260507 00:46:24.957 | INFO | Serialization done, swapping tmp file now... | 20260507 00:46:24.957 | INFO | RTMetrics done | 20260507 00:46:25.029 | INFO | Finished a round of Baz IO: Wrote 1.839 GiB in 3.214 seconds | 20260507 00:46:25.029 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.24% / 0.00% / 66.76% | 20260507 00:46:25.029 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.325159% / 7.94729e-06% / 5.96046e-05% | 20260507 00:46:25.029 | INFO | Baz IO overall: | 20260507 00:46:25.029 | INFO | BytesWritten : 3001684252911 | 20260507 00:46:25.029 | INFO | NumEvents : 1445634471479 | 20260507 00:46:26.854 | INFO | Analyzing chunk frames = [5767168,5767680) | 20260507 00:46:26.854 | INFO | MemInfo: { | 20260507 00:46:26.854 | INFO | "gpu" : " 17 GB", | 20260507 00:46:26.854 | INFO | "huge" : "", | 20260507 00:46:26.854 | INFO | "huge_shared" : " 27 GB", | 20260507 00:46:26.854 | INFO | "malloc" : " 14 GB", | 20260507 00:46:26.854 | INFO | "pinned" : " 29 GB" | 20260507 00:46:26.854 | INFO | } | 20260507 00:46:26.854 | INFO | Perf report for the last 5.13745 seconds: | 20260507 00:46:26.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:46:26.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:26.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:26.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:26.854 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260507 00:46:26.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:26.854 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 00:46:26.854 | INFO | PRE_HQ : 59.7%, 1.8, 56.4ms, 0 | 20260507 00:46:26.854 | INFO | BAZWRITER : 62.1%, 3.4, 107.1ms, 0 | 20260507 00:46:26.854 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 00:46:26.854 | INFO | Uploaded 36GiB to GPU in 2.02468s (17.7806GiB/s) | 20260507 00:46:26.854 | INFO | Downloaded 38GiB to GPU in 2.17337s (17.4844GiB/s) | 20260507 00:46:31.992 | INFO | Analyzing chunk frames = [5767680,5768192) | 20260507 00:46:31.992 | INFO | MemInfo: { | 20260507 00:46:31.992 | INFO | "gpu" : " 17 GB", | 20260507 00:46:31.992 | INFO | "huge" : "", | 20260507 00:46:31.992 | INFO | "huge_shared" : " 27 GB", | 20260507 00:46:31.992 | INFO | "malloc" : " 14 GB", | 20260507 00:46:31.992 | INFO | "pinned" : " 29 GB" | 20260507 00:46:31.992 | INFO | } | 20260507 00:46:31.992 | INFO | Perf report for the last 5.13876 seconds: | 20260507 00:46:31.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:46:31.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:31.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:31.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:31.992 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:46:31.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:31.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:31.993 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:46:31.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:31.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:31.993 | INFO | Uploaded 36GiB to GPU in 2.00469s (17.9579GiB/s) | 20260507 00:46:31.993 | INFO | Downloaded 34GiB to GPU in 1.93576s (17.5641GiB/s) | 20260507 00:46:37.132 | INFO | Analyzing chunk frames = [5768192,5768704) | 20260507 00:46:37.132 | INFO | MemInfo: { | 20260507 00:46:37.132 | INFO | "gpu" : " 17 GB", | 20260507 00:46:37.132 | INFO | "huge" : "", | 20260507 00:46:37.132 | INFO | "huge_shared" : " 27 GB", | 20260507 00:46:37.132 | INFO | "malloc" : " 14 GB", | 20260507 00:46:37.132 | INFO | "pinned" : " 29 GB" | 20260507 00:46:37.132 | INFO | } | 20260507 00:46:37.133 | INFO | Perf report for the last 5.14007 seconds: | 20260507 00:46:37.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:46:37.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:37.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:37.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:37.133 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 00:46:37.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:37.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:37.133 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:46:37.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:37.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:37.133 | INFO | Uploaded 36GiB to GPU in 1.98121s (18.1707GiB/s) | 20260507 00:46:37.133 | INFO | Downloaded 34GiB to GPU in 1.93686s (17.5542GiB/s) | 20260507 00:46:42.277 | INFO | Analyzing chunk frames = [5768704,5769216) | 20260507 00:46:42.277 | INFO | MemInfo: { | 20260507 00:46:42.277 | INFO | "gpu" : " 17 GB", | 20260507 00:46:42.277 | INFO | "huge" : "", | 20260507 00:46:42.277 | INFO | "huge_shared" : " 27 GB", | 20260507 00:46:42.277 | INFO | "malloc" : " 14 GB", | 20260507 00:46:42.277 | INFO | "pinned" : " 29 GB" | 20260507 00:46:42.277 | INFO | } | 20260507 00:46:42.277 | INFO | Perf report for the last 5.1445 seconds: | 20260507 00:46:42.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:46:42.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:42.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:42.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:42.277 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 00:46:42.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:42.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:42.277 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:46:42.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:42.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:42.277 | INFO | Uploaded 36GiB to GPU in 2.00954s (17.9146GiB/s) | 20260507 00:46:42.277 | INFO | Downloaded 34GiB to GPU in 1.91165s (17.7857GiB/s) | 20260507 00:46:47.420 | INFO | Analyzing chunk frames = [5769216,5769728) | 20260507 00:46:47.420 | INFO | MemInfo: { | 20260507 00:46:47.420 | INFO | "gpu" : " 17 GB", | 20260507 00:46:47.420 | INFO | "huge" : "", | 20260507 00:46:47.420 | INFO | "huge_shared" : " 27 GB", | 20260507 00:46:47.420 | INFO | "malloc" : " 14 GB", | 20260507 00:46:47.420 | INFO | "pinned" : " 29 GB" | 20260507 00:46:47.420 | INFO | } | 20260507 00:46:47.420 | INFO | Perf report for the last 5.14314 seconds: | 20260507 00:46:47.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:46:47.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:47.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:47.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:47.420 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 00:46:47.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:47.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:47.420 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:46:47.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:47.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:47.420 | INFO | Uploaded 36GiB to GPU in 1.99735s (18.0239GiB/s) | 20260507 00:46:47.421 | INFO | Downloaded 34GiB to GPU in 1.9402s (17.5239GiB/s) | 20260507 00:46:52.552 | INFO | Analyzing chunk frames = [5769728,5770240) | 20260507 00:46:52.552 | INFO | MemInfo: { | 20260507 00:46:52.552 | INFO | "gpu" : " 17 GB", | 20260507 00:46:52.552 | INFO | "huge" : "", | 20260507 00:46:52.552 | INFO | "huge_shared" : " 27 GB", | 20260507 00:46:52.552 | INFO | "malloc" : " 14 GB", | 20260507 00:46:52.552 | INFO | "pinned" : " 29 GB" | 20260507 00:46:52.552 | INFO | } | 20260507 00:46:52.552 | INFO | Perf report for the last 5.13177 seconds: | 20260507 00:46:52.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:46:52.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:52.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:52.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:52.552 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:46:52.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:52.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:52.552 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 00:46:52.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:52.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:52.553 | INFO | Uploaded 36GiB to GPU in 2.02916s (17.7413GiB/s) | 20260507 00:46:52.553 | INFO | Downloaded 34GiB to GPU in 1.94158s (17.5115GiB/s) | 20260507 00:46:57.687 | INFO | Analyzing chunk frames = [5770240,5770752) | 20260507 00:46:57.687 | INFO | MemInfo: { | 20260507 00:46:57.687 | INFO | "gpu" : " 17 GB", | 20260507 00:46:57.687 | INFO | "huge" : "", | 20260507 00:46:57.687 | INFO | "huge_shared" : " 27 GB", | 20260507 00:46:57.687 | INFO | "malloc" : " 14 GB", | 20260507 00:46:57.687 | INFO | "pinned" : " 29 GB" | 20260507 00:46:57.687 | INFO | } | 20260507 00:46:57.687 | INFO | Perf report for the last 5.1354 seconds: | 20260507 00:46:57.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:46:57.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:57.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:57.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:57.687 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260507 00:46:57.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:57.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:57.687 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:46:57.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:57.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:46:57.688 | INFO | Uploaded 36GiB to GPU in 1.97961s (18.1854GiB/s) | 20260507 00:46:57.688 | INFO | Downloaded 34GiB to GPU in 2.00545s (16.9538GiB/s) | 20260507 00:47:02.829 | INFO | Analyzing chunk frames = [5770752,5771264) | 20260507 00:47:02.829 | INFO | MemInfo: { | 20260507 00:47:02.829 | INFO | "gpu" : " 17 GB", | 20260507 00:47:02.829 | INFO | "huge" : "", | 20260507 00:47:02.829 | INFO | "huge_shared" : " 27 GB", | 20260507 00:47:02.829 | INFO | "malloc" : " 14 GB", | 20260507 00:47:02.829 | INFO | "pinned" : " 29 GB" | 20260507 00:47:02.829 | INFO | } | 20260507 00:47:02.829 | INFO | Perf report for the last 5.14212 seconds: | 20260507 00:47:02.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:47:02.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:02.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:02.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:02.830 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:47:02.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:02.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:02.830 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 00:47:02.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:02.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:02.830 | INFO | Uploaded 36GiB to GPU in 1.99211s (18.0713GiB/s) | 20260507 00:47:02.830 | INFO | Downloaded 34GiB to GPU in 1.98506s (17.1279GiB/s) | 20260507 00:47:02.927 | INFO | Starting a round of Baz IO | 20260507 00:47:06.055 | INFO | Populating RTMetrics report | 20260507 00:47:06.055 | INFO | RTMetrics report populated | 20260507 00:47:06.055 | INFO | Opening tmp RTMetrics json file | 20260507 00:47:06.059 | INFO | Serializing RTMetrics to json | 20260507 00:47:06.074 | INFO | Serialization done, swapping tmp file now... | 20260507 00:47:06.075 | INFO | RTMetrics done | 20260507 00:47:06.148 | INFO | Finished a round of Baz IO: Wrote 1.836 GiB in 3.22 seconds | 20260507 00:47:06.148 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.23% / 0.00% / 66.76% | 20260507 00:47:06.148 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.323133% / 0% / 7.15256e-05% | 20260507 00:47:06.148 | INFO | Baz IO overall: | 20260507 00:47:06.148 | INFO | BytesWritten : 3003655442749 | 20260507 00:47:06.148 | INFO | NumEvents : 1446570095496 | 20260507 00:47:07.970 | INFO | Analyzing chunk frames = [5771264,5771776) | 20260507 00:47:07.970 | INFO | MemInfo: { | 20260507 00:47:07.970 | INFO | "gpu" : " 17 GB", | 20260507 00:47:07.970 | INFO | "huge" : "", | 20260507 00:47:07.970 | INFO | "huge_shared" : " 27 GB", | 20260507 00:47:07.970 | INFO | "malloc" : " 14 GB", | 20260507 00:47:07.971 | INFO | "pinned" : " 29 GB" | 20260507 00:47:07.971 | INFO | } | 20260507 00:47:07.971 | INFO | Perf report for the last 5.14114 seconds: | 20260507 00:47:07.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:47:07.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:07.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:07.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:07.971 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 00:47:07.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:07.971 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 00:47:07.971 | INFO | PRE_HQ : 59.7%, 1.8, 56.6ms, 0 | 20260507 00:47:07.971 | INFO | BAZWRITER : 62.2%, 3.3, 106.2ms, 0 | 20260507 00:47:07.971 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:47:07.971 | INFO | Uploaded 36GiB to GPU in 2.02332s (17.7925GiB/s) | 20260507 00:47:07.971 | INFO | Downloaded 38GiB to GPU in 2.1725s (17.4913GiB/s) | 20260507 00:47:13.113 | INFO | Analyzing chunk frames = [5771776,5772288) | 20260507 00:47:13.113 | INFO | MemInfo: { | 20260507 00:47:13.113 | INFO | "gpu" : " 17 GB", | 20260507 00:47:13.113 | INFO | "huge" : "", | 20260507 00:47:13.113 | INFO | "huge_shared" : " 27 GB", | 20260507 00:47:13.113 | INFO | "malloc" : " 14 GB", | 20260507 00:47:13.113 | INFO | "pinned" : " 29 GB" | 20260507 00:47:13.113 | INFO | } | 20260507 00:47:13.113 | INFO | Perf report for the last 5.14241 seconds: | 20260507 00:47:13.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:47:13.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:13.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:13.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:13.113 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 00:47:13.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:13.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:13.113 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:47:13.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:13.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:13.113 | INFO | Uploaded 36GiB to GPU in 1.99808s (18.0173GiB/s) | 20260507 00:47:13.113 | INFO | Downloaded 34GiB to GPU in 1.925s (17.6623GiB/s) | 20260507 00:47:18.255 | INFO | Analyzing chunk frames = [5772288,5772800) | 20260507 00:47:18.255 | INFO | MemInfo: { | 20260507 00:47:18.255 | INFO | "gpu" : " 17 GB", | 20260507 00:47:18.255 | INFO | "huge" : "", | 20260507 00:47:18.255 | INFO | "huge_shared" : " 27 GB", | 20260507 00:47:18.255 | INFO | "malloc" : " 14 GB", | 20260507 00:47:18.255 | INFO | "pinned" : " 29 GB" | 20260507 00:47:18.255 | INFO | } | 20260507 00:47:18.255 | INFO | Perf report for the last 5.14192 seconds: | 20260507 00:47:18.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:47:18.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:18.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:18.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:18.255 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 00:47:18.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:18.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:18.255 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:47:18.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:18.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:18.255 | INFO | Uploaded 36GiB to GPU in 2.0102s (17.9086GiB/s) | 20260507 00:47:18.255 | INFO | Downloaded 34GiB to GPU in 1.93407s (17.5795GiB/s) | 20260507 00:47:23.393 | INFO | Analyzing chunk frames = [5772800,5773312) | 20260507 00:47:23.393 | INFO | MemInfo: { | 20260507 00:47:23.393 | INFO | "gpu" : " 17 GB", | 20260507 00:47:23.393 | INFO | "huge" : "", | 20260507 00:47:23.393 | INFO | "huge_shared" : " 27 GB", | 20260507 00:47:23.393 | INFO | "malloc" : " 14 GB", | 20260507 00:47:23.394 | INFO | "pinned" : " 29 GB" | 20260507 00:47:23.394 | INFO | } | 20260507 00:47:23.394 | INFO | Perf report for the last 5.13856 seconds: | 20260507 00:47:23.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:47:23.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:23.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:23.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:23.394 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 00:47:23.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:23.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:23.394 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:47:23.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:23.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:23.394 | INFO | Uploaded 36GiB to GPU in 1.9839s (18.1461GiB/s) | 20260507 00:47:23.394 | INFO | Downloaded 34GiB to GPU in 1.91517s (17.753GiB/s) | 20260507 00:47:28.532 | INFO | Analyzing chunk frames = [5773312,5773824) | 20260507 00:47:28.532 | INFO | MemInfo: { | 20260507 00:47:28.532 | INFO | "gpu" : " 17 GB", | 20260507 00:47:28.532 | INFO | "huge" : "", | 20260507 00:47:28.532 | INFO | "huge_shared" : " 27 GB", | 20260507 00:47:28.532 | INFO | "malloc" : " 14 GB", | 20260507 00:47:28.532 | INFO | "pinned" : " 29 GB" | 20260507 00:47:28.532 | INFO | } | 20260507 00:47:28.532 | INFO | Perf report for the last 5.13856 seconds: | 20260507 00:47:28.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:47:28.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:28.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:28.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:28.532 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 00:47:28.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:28.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:28.532 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:47:28.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:28.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:28.532 | INFO | Uploaded 36GiB to GPU in 2.01293s (17.8844GiB/s) | 20260507 00:47:28.532 | INFO | Downloaded 34GiB to GPU in 1.94411s (17.4887GiB/s) | 20260507 00:47:33.664 | INFO | Analyzing chunk frames = [5773824,5774336) | 20260507 00:47:33.664 | INFO | MemInfo: { | 20260507 00:47:33.664 | INFO | "gpu" : " 17 GB", | 20260507 00:47:33.664 | INFO | "huge" : "", | 20260507 00:47:33.664 | INFO | "huge_shared" : " 27 GB", | 20260507 00:47:33.664 | INFO | "malloc" : " 14 GB", | 20260507 00:47:33.664 | INFO | "pinned" : " 29 GB" | 20260507 00:47:33.664 | INFO | } | 20260507 00:47:33.664 | INFO | Perf report for the last 5.13163 seconds: | 20260507 00:47:33.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:47:33.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:33.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:33.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:33.664 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 00:47:33.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:33.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:33.664 | INFO | PRE_HQ : 30.0%, 1.0, 15.5ms, 0 | 20260507 00:47:33.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:33.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:33.665 | INFO | Uploaded 36GiB to GPU in 2.04429s (17.61GiB/s) | 20260507 00:47:33.665 | INFO | Downloaded 34GiB to GPU in 1.94926s (17.4425GiB/s) | 20260507 00:47:38.803 | INFO | Analyzing chunk frames = [5774336,5774848) | 20260507 00:47:38.803 | INFO | MemInfo: { | 20260507 00:47:38.803 | INFO | "gpu" : " 17 GB", | 20260507 00:47:38.803 | INFO | "huge" : "", | 20260507 00:47:38.803 | INFO | "huge_shared" : " 27 GB", | 20260507 00:47:38.803 | INFO | "malloc" : " 14 GB", | 20260507 00:47:38.803 | INFO | "pinned" : " 29 GB" | 20260507 00:47:38.803 | INFO | } | 20260507 00:47:38.803 | INFO | Perf report for the last 5.13963 seconds: | 20260507 00:47:38.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:47:38.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:38.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:38.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:38.804 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:47:38.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:38.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:38.804 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:47:38.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:38.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:38.804 | INFO | Uploaded 36GiB to GPU in 1.96779s (18.2947GiB/s) | 20260507 00:47:38.804 | INFO | Downloaded 34GiB to GPU in 1.9842s (17.1354GiB/s) | 20260507 00:47:43.944 | INFO | Analyzing chunk frames = [5774848,5775360) | 20260507 00:47:43.944 | INFO | MemInfo: { | 20260507 00:47:43.944 | INFO | "gpu" : " 17 GB", | 20260507 00:47:43.944 | INFO | "huge" : "", | 20260507 00:47:43.944 | INFO | "huge_shared" : " 27 GB", | 20260507 00:47:43.944 | INFO | "malloc" : " 14 GB", | 20260507 00:47:43.944 | INFO | "pinned" : " 29 GB" | 20260507 00:47:43.944 | INFO | } | 20260507 00:47:43.944 | INFO | Perf report for the last 5.14049 seconds: | 20260507 00:47:43.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:47:43.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:43.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:43.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:43.944 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:47:43.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:43.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:43.944 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:47:43.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:43.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:43.944 | INFO | Uploaded 36GiB to GPU in 1.9824s (18.1598GiB/s) | 20260507 00:47:43.944 | INFO | Downloaded 34GiB to GPU in 1.99057s (17.0805GiB/s) | 20260507 00:47:43.944 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:47:43.944 | INFO | Total mem usage: 18 GiB | 20260507 00:47:43.944 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:47:43.944 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 00:47:43.944 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:47:43.944 | INFO | High Water Breakdown: | 20260507 00:47:43.944 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:47:43.944 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:47:43.944 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:47:43.944 | INFO | ------------------------------------------ | 20260507 00:47:43.944 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:47:43.944 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:47:43.944 | INFO | Total mem usage: 26.5 GiB | 20260507 00:47:43.944 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:47:43.944 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:47:43.944 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:47:43.945 | INFO | High Water Breakdown: | 20260507 00:47:43.945 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:47:43.945 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:47:43.945 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:47:43.945 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:47:43.945 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:47:43.945 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:47:43.945 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:47:43.945 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:47:43.945 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:47:43.945 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:47:43.945 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:47:43.945 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:47:43.945 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:47:43.945 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:47:43.945 | INFO | ------------------------------------------ | 20260507 00:47:43.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:47:43.945 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:47:43.945 | INFO | Total mem usage: 25.6 GiB | 20260507 00:47:43.945 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:47:43.945 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:47:43.945 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:47:43.945 | INFO | High Water Breakdown: | 20260507 00:47:43.945 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:47:43.945 | INFO | ------------------------------------------ | 20260507 00:47:43.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:47:43.945 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:47:43.945 | INFO | Total mem usage: 15.7 GiB | 20260507 00:47:43.945 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:47:43.945 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:47:43.945 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:47:43.945 | INFO | High Water Breakdown: | 20260507 00:47:43.945 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:47:43.945 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:47:43.945 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:47:43.945 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:47:43.945 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:47:43.945 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:47:43.945 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:47:43.945 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:47:43.945 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:47:43.945 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:47:43.945 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:47:43.945 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:47:43.945 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:47:43.945 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:47:43.945 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:47:43.945 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:47:43.945 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:47:43.945 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:47:43.945 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:47:43.945 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:47:43.945 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:47:43.945 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:47:43.945 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:47:43.945 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:47:43.945 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:47:43.945 | INFO | ------------------------------------------ | 20260507 00:47:43.945 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:47:43.945 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:47:43.945 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:47:43.945 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:47:43.945 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.60e+01, 1.35e+01 / 2.09e+00 | 20260507 00:47:43.945 | INFO | PerfReport: 0.30%, AggregateStats -- 9.13e-02 / 1.01e-01, 9.88e-03 / 6.58e-03 | 20260507 00:47:43.945 | INFO | PerfReport: 11.66%, Baseline -- 3.61e+00 / 4.46e+00, 5.34e-01 / 2.55e-01 | 20260507 00:47:43.945 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.47e+00, 1.75e-01 / 9.98e-02 | 20260507 00:47:43.945 | INFO | PerfReport: 16.11%, DME -- 3.99e+01 / 4.51e+01, 4.20e+00 / 2.35e+00 | 20260507 00:47:43.945 | INFO | PerfReport: 57.46%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.35e+00 / 1.58e+00 | 20260507 00:47:43.945 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.17e+00 / 2.76e+00, 3.80e-01 / 1.91e-01 | 20260507 00:47:43.945 | INFO | PerfReport: 3.13%, Metrics -- 9.70e-01 / 1.30e+00, 2.58e-01 / 1.34e-01 | 20260507 00:47:43.945 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.30e-01 / 3.81e-01, 5.23e-02 / 2.08e-02 | 20260507 00:47:43.945 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:47:43.945 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:47:43.945 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:47:43.945 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 00:47:43.945 | INFO | PerfReport: 21.60%, Upload -- 2.09e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260507 00:47:43.945 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 00:47:43.945 | INFO | PerfReport: 32.05%, Compute -- 3.10e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:47:43.945 | INFO | PerfReport: 1.34%, LockUpload -- 1.30e+00 / 0.00e+00, 4.77e+00 / 0.00e+00 | 20260507 00:47:43.945 | INFO | PerfReport: 1.27%, LockDownload -- 1.23e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260507 00:47:43.945 | INFO | PerfReport: 22.40%, LockCompute -- 2.16e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 00:47:43.945 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.46e-02 / 0.00e+00 | 20260507 00:47:44.055 | INFO | Starting a round of Baz IO | 20260507 00:47:47.168 | INFO | Populating RTMetrics report | 20260507 00:47:47.168 | INFO | RTMetrics report populated | 20260507 00:47:47.168 | INFO | Opening tmp RTMetrics json file | 20260507 00:47:47.189 | INFO | Serializing RTMetrics to json | 20260507 00:47:47.190 | INFO | Serialization done, swapping tmp file now... | 20260507 00:47:47.190 | INFO | RTMetrics done | 20260507 00:47:47.258 | INFO | Finished a round of Baz IO: Wrote 1.836 GiB in 3.203 seconds | 20260507 00:47:47.259 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.23% / 0.00% / 66.77% | 20260507 00:47:47.259 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.323395% / 0% / 7.94729e-05% | 20260507 00:47:47.259 | INFO | Baz IO overall: | 20260507 00:47:47.259 | INFO | BytesWritten : 3005627059356 | 20260507 00:47:47.259 | INFO | NumEvents : 1447505798783 | 20260507 00:47:49.087 | INFO | Analyzing chunk frames = [5775360,5775872) | 20260507 00:47:49.087 | INFO | MemInfo: { | 20260507 00:47:49.087 | INFO | "gpu" : " 17 GB", | 20260507 00:47:49.087 | INFO | "huge" : "", | 20260507 00:47:49.087 | INFO | "huge_shared" : " 27 GB", | 20260507 00:47:49.087 | INFO | "malloc" : " 14 GB", | 20260507 00:47:49.087 | INFO | "pinned" : " 29 GB" | 20260507 00:47:49.087 | INFO | } | 20260507 00:47:49.087 | INFO | Perf report for the last 5.14344 seconds: | 20260507 00:47:49.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:47:49.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:49.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:49.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:49.088 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260507 00:47:49.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:49.088 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 00:47:49.088 | INFO | PRE_HQ : 59.3%, 1.8, 56.1ms, 0 | 20260507 00:47:49.088 | INFO | BAZWRITER : 62.1%, 3.3, 106.3ms, 0 | 20260507 00:47:49.088 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:47:49.088 | INFO | Uploaded 36GiB to GPU in 2.02293s (17.796GiB/s) | 20260507 00:47:49.088 | INFO | Downloaded 38GiB to GPU in 2.18264s (17.4101GiB/s) | 20260507 00:47:54.233 | INFO | Analyzing chunk frames = [5775872,5776384) | 20260507 00:47:54.233 | INFO | MemInfo: { | 20260507 00:47:54.233 | INFO | "gpu" : " 17 GB", | 20260507 00:47:54.233 | INFO | "huge" : "", | 20260507 00:47:54.233 | INFO | "huge_shared" : " 27 GB", | 20260507 00:47:54.233 | INFO | "malloc" : " 14 GB", | 20260507 00:47:54.233 | INFO | "pinned" : " 29 GB" | 20260507 00:47:54.233 | INFO | } | 20260507 00:47:54.233 | INFO | Perf report for the last 5.14575 seconds: | 20260507 00:47:54.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:47:54.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:54.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:54.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:54.233 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 00:47:54.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:54.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:54.233 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 00:47:54.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:54.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:54.233 | INFO | Uploaded 36GiB to GPU in 2.0088s (17.9211GiB/s) | 20260507 00:47:54.234 | INFO | Downloaded 34GiB to GPU in 1.92901s (17.6256GiB/s) | 20260507 00:47:59.368 | INFO | Analyzing chunk frames = [5776384,5776896) | 20260507 00:47:59.368 | INFO | MemInfo: { | 20260507 00:47:59.368 | INFO | "gpu" : " 17 GB", | 20260507 00:47:59.368 | INFO | "huge" : "", | 20260507 00:47:59.368 | INFO | "huge_shared" : " 27 GB", | 20260507 00:47:59.368 | INFO | "malloc" : " 14 GB", | 20260507 00:47:59.368 | INFO | "pinned" : " 29 GB" | 20260507 00:47:59.368 | INFO | } | 20260507 00:47:59.368 | INFO | Perf report for the last 5.13503 seconds: | 20260507 00:47:59.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:47:59.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:59.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:59.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:59.368 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:47:59.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:59.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:59.368 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:47:59.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:59.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:47:59.369 | INFO | Uploaded 36GiB to GPU in 2.00909s (17.9186GiB/s) | 20260507 00:47:59.369 | INFO | Downloaded 34GiB to GPU in 1.93678s (17.5549GiB/s) | 20260507 00:48:04.501 | INFO | Analyzing chunk frames = [5776896,5777408) | 20260507 00:48:04.501 | INFO | MemInfo: { | 20260507 00:48:04.501 | INFO | "gpu" : " 17 GB", | 20260507 00:48:04.501 | INFO | "huge" : "", | 20260507 00:48:04.501 | INFO | "huge_shared" : " 27 GB", | 20260507 00:48:04.501 | INFO | "malloc" : " 14 GB", | 20260507 00:48:04.501 | INFO | "pinned" : " 29 GB" | 20260507 00:48:04.501 | INFO | } | 20260507 00:48:04.501 | INFO | Perf report for the last 5.13255 seconds: | 20260507 00:48:04.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:48:04.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:04.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:04.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:04.501 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 00:48:04.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:04.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:04.501 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:48:04.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:04.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:04.501 | INFO | Uploaded 36GiB to GPU in 1.97289s (18.2473GiB/s) | 20260507 00:48:04.501 | INFO | Downloaded 34GiB to GPU in 1.90949s (17.8058GiB/s) | 20260507 00:48:09.638 | INFO | Analyzing chunk frames = [5777408,5777920) | 20260507 00:48:09.638 | INFO | MemInfo: { | 20260507 00:48:09.638 | INFO | "gpu" : " 17 GB", | 20260507 00:48:09.638 | INFO | "huge" : "", | 20260507 00:48:09.638 | INFO | "huge_shared" : " 27 GB", | 20260507 00:48:09.638 | INFO | "malloc" : " 14 GB", | 20260507 00:48:09.638 | INFO | "pinned" : " 29 GB" | 20260507 00:48:09.638 | INFO | } | 20260507 00:48:09.638 | INFO | Perf report for the last 5.13702 seconds: | 20260507 00:48:09.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:48:09.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:09.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:09.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:09.638 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260507 00:48:09.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:09.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:09.638 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:48:09.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:09.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:09.638 | INFO | Uploaded 36GiB to GPU in 1.98753s (18.1129GiB/s) | 20260507 00:48:09.638 | INFO | Downloaded 34GiB to GPU in 1.92637s (17.6497GiB/s) | 20260507 00:48:14.778 | INFO | Analyzing chunk frames = [5777920,5778432) | 20260507 00:48:14.778 | INFO | MemInfo: { | 20260507 00:48:14.778 | INFO | "gpu" : " 17 GB", | 20260507 00:48:14.778 | INFO | "huge" : "", | 20260507 00:48:14.778 | INFO | "huge_shared" : " 27 GB", | 20260507 00:48:14.778 | INFO | "malloc" : " 14 GB", | 20260507 00:48:14.778 | INFO | "pinned" : " 29 GB" | 20260507 00:48:14.778 | INFO | } | 20260507 00:48:14.778 | INFO | Perf report for the last 5.1406 seconds: | 20260507 00:48:14.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:48:14.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:14.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:14.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:14.778 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:48:14.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:14.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:14.778 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260507 00:48:14.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:14.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:14.779 | INFO | Uploaded 36GiB to GPU in 2.02377s (17.7886GiB/s) | 20260507 00:48:14.779 | INFO | Downloaded 34GiB to GPU in 1.95464s (17.3945GiB/s) | 20260507 00:48:19.920 | INFO | Analyzing chunk frames = [5778432,5778944) | 20260507 00:48:19.920 | INFO | MemInfo: { | 20260507 00:48:19.920 | INFO | "gpu" : " 17 GB", | 20260507 00:48:19.920 | INFO | "huge" : "", | 20260507 00:48:19.920 | INFO | "huge_shared" : " 27 GB", | 20260507 00:48:19.920 | INFO | "malloc" : " 14 GB", | 20260507 00:48:19.920 | INFO | "pinned" : " 29 GB" | 20260507 00:48:19.920 | INFO | } | 20260507 00:48:19.920 | INFO | Perf report for the last 5.1416 seconds: | 20260507 00:48:19.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:48:19.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:19.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:19.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:19.920 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:48:19.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:19.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:19.920 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:48:19.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:19.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:19.920 | INFO | Uploaded 36GiB to GPU in 1.96125s (18.3556GiB/s) | 20260507 00:48:19.920 | INFO | Downloaded 34GiB to GPU in 1.9821s (17.1535GiB/s) | 20260507 00:48:25.063 | INFO | Analyzing chunk frames = [5778944,5779456) | 20260507 00:48:25.063 | INFO | MemInfo: { | 20260507 00:48:25.063 | INFO | "gpu" : " 17 GB", | 20260507 00:48:25.064 | INFO | "huge" : "", | 20260507 00:48:25.064 | INFO | "huge_shared" : " 27 GB", | 20260507 00:48:25.064 | INFO | "malloc" : " 14 GB", | 20260507 00:48:25.064 | INFO | "pinned" : " 29 GB" | 20260507 00:48:25.064 | INFO | } | 20260507 00:48:25.064 | INFO | Perf report for the last 5.14367 seconds: | 20260507 00:48:25.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:48:25.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:25.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:25.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:25.064 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:48:25.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:25.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:25.064 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 00:48:25.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:25.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:25.064 | INFO | Uploaded 36GiB to GPU in 1.98184s (18.165GiB/s) | 20260507 00:48:25.064 | INFO | Downloaded 34GiB to GPU in 1.98123s (17.1611GiB/s) | 20260507 00:48:25.161 | INFO | Starting a round of Baz IO | 20260507 00:48:28.281 | INFO | Populating RTMetrics report | 20260507 00:48:28.281 | INFO | RTMetrics report populated | 20260507 00:48:28.281 | INFO | Opening tmp RTMetrics json file | 20260507 00:48:28.285 | INFO | Serializing RTMetrics to json | 20260507 00:48:28.300 | INFO | Serialization done, swapping tmp file now... | 20260507 00:48:28.301 | INFO | RTMetrics done | 20260507 00:48:28.370 | INFO | Finished a round of Baz IO: Wrote 1.835 GiB in 3.209 seconds | 20260507 00:48:28.370 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.23% / 0.00% / 66.77% | 20260507 00:48:28.370 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.323379% / 0% / 6.75519e-05% | 20260507 00:48:28.370 | INFO | Baz IO overall: | 20260507 00:48:28.370 | INFO | BytesWritten : 3007597331421 | 20260507 00:48:28.370 | INFO | NumEvents : 1448440951792 | 20260507 00:48:30.206 | INFO | Analyzing chunk frames = [5779456,5779968) | 20260507 00:48:30.206 | INFO | MemInfo: { | 20260507 00:48:30.206 | INFO | "gpu" : " 17 GB", | 20260507 00:48:30.206 | INFO | "huge" : "", | 20260507 00:48:30.206 | INFO | "huge_shared" : " 27 GB", | 20260507 00:48:30.206 | INFO | "malloc" : " 14 GB", | 20260507 00:48:30.206 | INFO | "pinned" : " 29 GB" | 20260507 00:48:30.206 | INFO | } | 20260507 00:48:30.206 | INFO | Perf report for the last 5.14201 seconds: | 20260507 00:48:30.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:48:30.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:30.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:30.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:30.206 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 00:48:30.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:30.206 | INFO | RT_METRICS : 42.4%, 1.0, 22.3ms, 0 | 20260507 00:48:30.206 | INFO | PRE_HQ : 59.6%, 1.8, 56.2ms, 0 | 20260507 00:48:30.206 | INFO | BAZWRITER : 62.0%, 3.3, 106.5ms, 0 | 20260507 00:48:30.206 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 00:48:30.206 | INFO | Uploaded 36GiB to GPU in 2.02986s (17.7352GiB/s) | 20260507 00:48:30.206 | INFO | Downloaded 38GiB to GPU in 2.17877s (17.441GiB/s) | 20260507 00:48:35.344 | INFO | Analyzing chunk frames = [5779968,5780480) | 20260507 00:48:35.344 | INFO | MemInfo: { | 20260507 00:48:35.344 | INFO | "gpu" : " 17 GB", | 20260507 00:48:35.344 | INFO | "huge" : "", | 20260507 00:48:35.344 | INFO | "huge_shared" : " 27 GB", | 20260507 00:48:35.344 | INFO | "malloc" : " 14 GB", | 20260507 00:48:35.344 | INFO | "pinned" : " 29 GB" | 20260507 00:48:35.344 | INFO | } | 20260507 00:48:35.344 | INFO | Perf report for the last 5.13851 seconds: | 20260507 00:48:35.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:48:35.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:35.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:35.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:35.344 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:48:35.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:35.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:35.344 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 00:48:35.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:35.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:35.345 | INFO | Uploaded 36GiB to GPU in 2.03186s (17.7178GiB/s) | 20260507 00:48:35.345 | INFO | Downloaded 34GiB to GPU in 1.93828s (17.5413GiB/s) | 20260507 00:48:40.486 | INFO | Analyzing chunk frames = [5780480,5780992) | 20260507 00:48:40.486 | INFO | MemInfo: { | 20260507 00:48:40.486 | INFO | "gpu" : " 17 GB", | 20260507 00:48:40.486 | INFO | "huge" : "", | 20260507 00:48:40.486 | INFO | "huge_shared" : " 27 GB", | 20260507 00:48:40.486 | INFO | "malloc" : " 14 GB", | 20260507 00:48:40.486 | INFO | "pinned" : " 29 GB" | 20260507 00:48:40.486 | INFO | } | 20260507 00:48:40.486 | INFO | Perf report for the last 5.14195 seconds: | 20260507 00:48:40.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:48:40.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:40.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:40.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:40.486 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260507 00:48:40.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:40.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:40.486 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:48:40.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:40.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:40.486 | INFO | Uploaded 36GiB to GPU in 1.9798s (18.1837GiB/s) | 20260507 00:48:40.486 | INFO | Downloaded 34GiB to GPU in 1.9376s (17.5475GiB/s) | 20260507 00:48:45.618 | INFO | Analyzing chunk frames = [5780992,5781504) | 20260507 00:48:45.618 | INFO | MemInfo: { | 20260507 00:48:45.618 | INFO | "gpu" : " 17 GB", | 20260507 00:48:45.618 | INFO | "huge" : "", | 20260507 00:48:45.618 | INFO | "huge_shared" : " 27 GB", | 20260507 00:48:45.618 | INFO | "malloc" : " 14 GB", | 20260507 00:48:45.618 | INFO | "pinned" : " 29 GB" | 20260507 00:48:45.618 | INFO | } | 20260507 00:48:45.618 | INFO | Perf report for the last 5.13211 seconds: | 20260507 00:48:45.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:48:45.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:45.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:45.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:45.618 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260507 00:48:45.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:45.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:45.618 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:48:45.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:45.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:45.619 | INFO | Uploaded 36GiB to GPU in 1.97071s (18.2675GiB/s) | 20260507 00:48:45.619 | INFO | Downloaded 34GiB to GPU in 1.90936s (17.807GiB/s) | 20260507 00:48:50.754 | INFO | Analyzing chunk frames = [5781504,5782016) | 20260507 00:48:50.754 | INFO | MemInfo: { | 20260507 00:48:50.754 | INFO | "gpu" : " 17 GB", | 20260507 00:48:50.754 | INFO | "huge" : "", | 20260507 00:48:50.754 | INFO | "huge_shared" : " 27 GB", | 20260507 00:48:50.754 | INFO | "malloc" : " 14 GB", | 20260507 00:48:50.754 | INFO | "pinned" : " 29 GB" | 20260507 00:48:50.754 | INFO | } | 20260507 00:48:50.754 | INFO | Perf report for the last 5.13608 seconds: | 20260507 00:48:50.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:48:50.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:50.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:50.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:50.755 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 00:48:50.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:50.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:50.755 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:48:50.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:50.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:50.755 | INFO | Uploaded 36GiB to GPU in 1.97711s (18.2084GiB/s) | 20260507 00:48:50.755 | INFO | Downloaded 34GiB to GPU in 1.92972s (17.6191GiB/s) | 20260507 00:48:55.902 | INFO | Analyzing chunk frames = [5782016,5782528) | 20260507 00:48:55.902 | INFO | MemInfo: { | 20260507 00:48:55.902 | INFO | "gpu" : " 17 GB", | 20260507 00:48:55.902 | INFO | "huge" : "", | 20260507 00:48:55.902 | INFO | "huge_shared" : " 27 GB", | 20260507 00:48:55.902 | INFO | "malloc" : " 14 GB", | 20260507 00:48:55.902 | INFO | "pinned" : " 29 GB" | 20260507 00:48:55.902 | INFO | } | 20260507 00:48:55.902 | INFO | Perf report for the last 5.1478 seconds: | 20260507 00:48:55.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:48:55.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:55.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:55.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:55.902 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 00:48:55.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:55.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:55.902 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 00:48:55.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:55.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:48:55.903 | INFO | Uploaded 36GiB to GPU in 2.01408s (17.8742GiB/s) | 20260507 00:48:55.903 | INFO | Downloaded 34GiB to GPU in 1.94813s (17.4527GiB/s) | 20260507 00:49:01.038 | INFO | Analyzing chunk frames = [5782528,5783040) | 20260507 00:49:01.038 | INFO | MemInfo: { | 20260507 00:49:01.038 | INFO | "gpu" : " 17 GB", | 20260507 00:49:01.038 | INFO | "huge" : "", | 20260507 00:49:01.038 | INFO | "huge_shared" : " 27 GB", | 20260507 00:49:01.038 | INFO | "malloc" : " 14 GB", | 20260507 00:49:01.038 | INFO | "pinned" : " 29 GB" | 20260507 00:49:01.038 | INFO | } | 20260507 00:49:01.038 | INFO | Perf report for the last 5.13595 seconds: | 20260507 00:49:01.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:49:01.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:01.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:01.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:01.038 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:49:01.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:01.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:01.038 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:49:01.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:01.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:01.038 | INFO | Uploaded 36GiB to GPU in 1.98246s (18.1592GiB/s) | 20260507 00:49:01.038 | INFO | Downloaded 34GiB to GPU in 2.00143s (16.9879GiB/s) | 20260507 00:49:06.179 | INFO | Analyzing chunk frames = [5783040,5783552) | 20260507 00:49:06.179 | INFO | MemInfo: { | 20260507 00:49:06.179 | INFO | "gpu" : " 17 GB", | 20260507 00:49:06.179 | INFO | "huge" : "", | 20260507 00:49:06.179 | INFO | "huge_shared" : " 27 GB", | 20260507 00:49:06.179 | INFO | "malloc" : " 14 GB", | 20260507 00:49:06.179 | INFO | "pinned" : " 29 GB" | 20260507 00:49:06.179 | INFO | } | 20260507 00:49:06.179 | INFO | Perf report for the last 5.14084 seconds: | 20260507 00:49:06.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:49:06.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:06.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:06.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:06.179 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:49:06.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:06.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:06.179 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:49:06.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:06.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:06.179 | INFO | Uploaded 36GiB to GPU in 1.99237s (18.0689GiB/s) | 20260507 00:49:06.179 | INFO | Downloaded 34GiB to GPU in 1.99414s (17.0499GiB/s) | 20260507 00:49:06.179 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:49:06.179 | INFO | Total mem usage: 18 GiB | 20260507 00:49:06.179 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:49:06.179 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 00:49:06.179 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:49:06.179 | INFO | High Water Breakdown: | 20260507 00:49:06.179 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:49:06.179 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:49:06.179 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:49:06.179 | INFO | ------------------------------------------ | 20260507 00:49:06.179 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:49:06.179 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:49:06.179 | INFO | Total mem usage: 26.5 GiB | 20260507 00:49:06.179 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:49:06.179 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:49:06.180 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:49:06.180 | INFO | High Water Breakdown: | 20260507 00:49:06.180 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:49:06.180 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:49:06.180 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:49:06.180 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:49:06.180 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:49:06.180 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:49:06.180 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:49:06.180 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:49:06.180 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:49:06.180 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:49:06.180 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:49:06.180 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:49:06.180 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:49:06.180 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:49:06.180 | INFO | ------------------------------------------ | 20260507 00:49:06.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:49:06.180 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:49:06.180 | INFO | Total mem usage: 25.6 GiB | 20260507 00:49:06.180 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:49:06.180 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:49:06.180 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:49:06.180 | INFO | High Water Breakdown: | 20260507 00:49:06.180 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:49:06.180 | INFO | ------------------------------------------ | 20260507 00:49:06.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:49:06.180 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:49:06.180 | INFO | Total mem usage: 15.7 GiB | 20260507 00:49:06.180 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:49:06.180 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:49:06.180 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:49:06.180 | INFO | High Water Breakdown: | 20260507 00:49:06.180 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:49:06.180 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:49:06.180 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:49:06.180 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:49:06.180 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:49:06.180 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:49:06.180 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:49:06.180 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:49:06.180 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:49:06.180 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:49:06.180 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:49:06.180 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:49:06.180 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:49:06.180 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:49:06.180 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:49:06.180 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:49:06.180 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:49:06.180 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:49:06.180 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:49:06.180 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:49:06.180 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:49:06.180 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:49:06.180 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:49:06.180 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:49:06.180 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:49:06.180 | INFO | ------------------------------------------ | 20260507 00:49:06.180 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:49:06.180 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:49:06.180 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:49:06.180 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:49:06.180 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.65e+01, 1.35e+01 / 2.03e+00 | 20260507 00:49:06.180 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.00e-01, 9.64e-03 / 4.83e-03 | 20260507 00:49:06.180 | INFO | PerfReport: 11.67%, Baseline -- 3.61e+00 / 4.44e+00, 5.37e-01 / 2.13e-01 | 20260507 00:49:06.180 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.47e+00, 1.75e-01 / 1.20e-01 | 20260507 00:49:06.180 | INFO | PerfReport: 16.12%, DME -- 3.99e+01 / 4.52e+01, 4.43e+00 / 2.12e+00 | 20260507 00:49:06.180 | INFO | PerfReport: 57.44%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.30e+00 / 1.63e+00 | 20260507 00:49:06.180 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.17e+00 / 2.72e+00, 3.78e-01 / 2.20e-01 | 20260507 00:49:06.180 | INFO | PerfReport: 3.14%, Metrics -- 9.70e-01 / 1.29e+00, 2.55e-01 / 1.47e-01 | 20260507 00:49:06.180 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.30e-01 / 3.83e-01, 5.20e-02 / 2.21e-02 | 20260507 00:49:06.180 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:49:06.180 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:49:06.180 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:49:06.180 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 00:49:06.180 | INFO | PerfReport: 21.55%, Upload -- 2.08e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260507 00:49:06.180 | INFO | PerfReport: 21.34%, Download -- 2.06e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260507 00:49:06.180 | INFO | PerfReport: 32.03%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:49:06.180 | INFO | PerfReport: 1.29%, LockUpload -- 1.25e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260507 00:49:06.180 | INFO | PerfReport: 1.34%, LockDownload -- 1.30e+00 / 0.00e+00, 3.20e+00 / 0.00e+00 | 20260507 00:49:06.180 | INFO | PerfReport: 22.43%, LockCompute -- 2.17e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 00:49:06.180 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260507 00:49:06.279 | INFO | Starting a round of Baz IO | 20260507 00:49:09.387 | INFO | Populating RTMetrics report | 20260507 00:49:09.388 | INFO | RTMetrics report populated | 20260507 00:49:09.388 | INFO | Opening tmp RTMetrics json file | 20260507 00:49:09.389 | INFO | Serializing RTMetrics to json | 20260507 00:49:09.392 | INFO | Serialization done, swapping tmp file now... | 20260507 00:49:09.400 | INFO | RTMetrics done | 20260507 00:49:09.481 | INFO | Finished a round of Baz IO: Wrote 1.837 GiB in 3.202 seconds | 20260507 00:49:09.481 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.23% / 0.00% / 66.77% | 20260507 00:49:09.481 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.323002% / 7.94729e-06% / 8.74201e-05% | 20260507 00:49:09.481 | INFO | Baz IO overall: | 20260507 00:49:09.481 | INFO | BytesWritten : 3009570145759 | 20260507 00:49:09.481 | INFO | NumEvents : 1449378173825 | 20260507 00:49:11.319 | INFO | Analyzing chunk frames = [5783552,5784064) | 20260507 00:49:11.319 | INFO | MemInfo: { | 20260507 00:49:11.319 | INFO | "gpu" : " 17 GB", | 20260507 00:49:11.319 | INFO | "huge" : "", | 20260507 00:49:11.319 | INFO | "huge_shared" : " 27 GB", | 20260507 00:49:11.319 | INFO | "malloc" : " 14 GB", | 20260507 00:49:11.319 | INFO | "pinned" : " 29 GB" | 20260507 00:49:11.319 | INFO | } | 20260507 00:49:11.319 | INFO | Perf report for the last 5.1403 seconds: | 20260507 00:49:11.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:49:11.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:11.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:11.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:11.319 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 00:49:11.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:11.319 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 00:49:11.320 | INFO | PRE_HQ : 59.5%, 1.8, 56.7ms, 0 | 20260507 00:49:11.320 | INFO | BAZWRITER : 61.9%, 3.3, 106.5ms, 0 | 20260507 00:49:11.320 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:49:11.320 | INFO | Uploaded 36GiB to GPU in 2.01209s (17.8919GiB/s) | 20260507 00:49:11.320 | INFO | Downloaded 38GiB to GPU in 2.17818s (17.4457GiB/s) | 20260507 00:49:16.456 | INFO | Analyzing chunk frames = [5784064,5784576) | 20260507 00:49:16.456 | INFO | MemInfo: { | 20260507 00:49:16.456 | INFO | "gpu" : " 17 GB", | 20260507 00:49:16.456 | INFO | "huge" : "", | 20260507 00:49:16.456 | INFO | "huge_shared" : " 27 GB", | 20260507 00:49:16.456 | INFO | "malloc" : " 14 GB", | 20260507 00:49:16.456 | INFO | "pinned" : " 29 GB" | 20260507 00:49:16.456 | INFO | } | 20260507 00:49:16.456 | INFO | Perf report for the last 5.13694 seconds: | 20260507 00:49:16.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:49:16.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:16.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:16.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:16.456 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:49:16.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:16.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:16.456 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:49:16.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:16.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:16.456 | INFO | Uploaded 36GiB to GPU in 2.01179s (17.8945GiB/s) | 20260507 00:49:16.456 | INFO | Downloaded 34GiB to GPU in 1.93702s (17.5528GiB/s) | 20260507 00:49:21.596 | INFO | Analyzing chunk frames = [5784576,5785088) | 20260507 00:49:21.596 | INFO | MemInfo: { | 20260507 00:49:21.596 | INFO | "gpu" : " 17 GB", | 20260507 00:49:21.596 | INFO | "huge" : "", | 20260507 00:49:21.596 | INFO | "huge_shared" : " 27 GB", | 20260507 00:49:21.596 | INFO | "malloc" : " 14 GB", | 20260507 00:49:21.596 | INFO | "pinned" : " 29 GB" | 20260507 00:49:21.596 | INFO | } | 20260507 00:49:21.596 | INFO | Perf report for the last 5.1401 seconds: | 20260507 00:49:21.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:49:21.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:21.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:21.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:21.597 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:49:21.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:21.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:21.597 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:49:21.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:21.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:21.597 | INFO | Uploaded 36GiB to GPU in 1.96471s (18.3233GiB/s) | 20260507 00:49:21.597 | INFO | Downloaded 34GiB to GPU in 1.93013s (17.6154GiB/s) | 20260507 00:49:26.731 | INFO | Analyzing chunk frames = [5785088,5785600) | 20260507 00:49:26.731 | INFO | MemInfo: { | 20260507 00:49:26.731 | INFO | "gpu" : " 17 GB", | 20260507 00:49:26.731 | INFO | "huge" : "", | 20260507 00:49:26.731 | INFO | "huge_shared" : " 27 GB", | 20260507 00:49:26.731 | INFO | "malloc" : " 14 GB", | 20260507 00:49:26.731 | INFO | "pinned" : " 29 GB" | 20260507 00:49:26.731 | INFO | } | 20260507 00:49:26.731 | INFO | Perf report for the last 5.13427 seconds: | 20260507 00:49:26.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:49:26.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:26.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:26.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:26.731 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 00:49:26.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:26.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:26.731 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:49:26.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:26.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:26.731 | INFO | Uploaded 36GiB to GPU in 1.97549s (18.2234GiB/s) | 20260507 00:49:26.731 | INFO | Downloaded 34GiB to GPU in 1.90918s (17.8087GiB/s) | 20260507 00:49:31.877 | INFO | Analyzing chunk frames = [5785600,5786112) | 20260507 00:49:31.877 | INFO | MemInfo: { | 20260507 00:49:31.877 | INFO | "gpu" : " 17 GB", | 20260507 00:49:31.877 | INFO | "huge" : "", | 20260507 00:49:31.877 | INFO | "huge_shared" : " 27 GB", | 20260507 00:49:31.877 | INFO | "malloc" : " 14 GB", | 20260507 00:49:31.877 | INFO | "pinned" : " 29 GB" | 20260507 00:49:31.877 | INFO | } | 20260507 00:49:31.877 | INFO | Perf report for the last 5.14617 seconds: | 20260507 00:49:31.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:49:31.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:31.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:31.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:31.877 | INFO | ANALYSIS : 59.9%, 3.0, 93.4ms, 0 | 20260507 00:49:31.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:31.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:31.877 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:49:31.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:31.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:31.877 | INFO | Uploaded 36GiB to GPU in 2.00299s (17.9731GiB/s) | 20260507 00:49:31.877 | INFO | Downloaded 34GiB to GPU in 1.92395s (17.672GiB/s) | 20260507 00:49:37.012 | INFO | Analyzing chunk frames = [5786112,5786624) | 20260507 00:49:37.012 | INFO | MemInfo: { | 20260507 00:49:37.012 | INFO | "gpu" : " 17 GB", | 20260507 00:49:37.012 | INFO | "huge" : "", | 20260507 00:49:37.012 | INFO | "huge_shared" : " 27 GB", | 20260507 00:49:37.012 | INFO | "malloc" : " 14 GB", | 20260507 00:49:37.012 | INFO | "pinned" : " 29 GB" | 20260507 00:49:37.012 | INFO | } | 20260507 00:49:37.012 | INFO | Perf report for the last 5.13555 seconds: | 20260507 00:49:37.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:49:37.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:37.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:37.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:37.013 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 00:49:37.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:37.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:37.013 | INFO | PRE_HQ : 29.8%, 1.0, 15.4ms, 0 | 20260507 00:49:37.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:37.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:37.014 | INFO | Uploaded 36GiB to GPU in 2.02384s (17.788GiB/s) | 20260507 00:49:37.014 | INFO | Downloaded 34GiB to GPU in 1.93832s (17.541GiB/s) | 20260507 00:49:42.156 | INFO | Analyzing chunk frames = [5786624,5787136) | 20260507 00:49:42.156 | INFO | MemInfo: { | 20260507 00:49:42.156 | INFO | "gpu" : " 17 GB", | 20260507 00:49:42.156 | INFO | "huge" : "", | 20260507 00:49:42.156 | INFO | "huge_shared" : " 27 GB", | 20260507 00:49:42.156 | INFO | "malloc" : " 14 GB", | 20260507 00:49:42.156 | INFO | "pinned" : " 29 GB" | 20260507 00:49:42.156 | INFO | } | 20260507 00:49:42.157 | INFO | Perf report for the last 5.144 seconds: | 20260507 00:49:42.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:49:42.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:42.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:42.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:42.157 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:49:42.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:42.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:42.157 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 00:49:42.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:42.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:42.157 | INFO | Uploaded 36GiB to GPU in 1.98629s (18.1243GiB/s) | 20260507 00:49:42.157 | INFO | Downloaded 34GiB to GPU in 2.00419s (16.9645GiB/s) | 20260507 00:49:47.295 | INFO | Analyzing chunk frames = [5787136,5787648) | 20260507 00:49:47.295 | INFO | MemInfo: { | 20260507 00:49:47.295 | INFO | "gpu" : " 17 GB", | 20260507 00:49:47.295 | INFO | "huge" : "", | 20260507 00:49:47.295 | INFO | "huge_shared" : " 27 GB", | 20260507 00:49:47.295 | INFO | "malloc" : " 14 GB", | 20260507 00:49:47.295 | INFO | "pinned" : " 29 GB" | 20260507 00:49:47.295 | INFO | } | 20260507 00:49:47.295 | INFO | Perf report for the last 5.13893 seconds: | 20260507 00:49:47.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:49:47.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:47.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:47.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:47.296 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260507 00:49:47.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:47.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:47.296 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 00:49:47.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:47.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:47.296 | INFO | Uploaded 36GiB to GPU in 2.00154s (17.9862GiB/s) | 20260507 00:49:47.296 | INFO | Downloaded 34GiB to GPU in 1.98539s (17.1251GiB/s) | 20260507 00:49:47.394 | INFO | Starting a round of Baz IO | 20260507 00:49:50.506 | INFO | Populating RTMetrics report | 20260507 00:49:50.506 | INFO | RTMetrics report populated | 20260507 00:49:50.506 | INFO | Opening tmp RTMetrics json file | 20260507 00:49:50.507 | INFO | Serializing RTMetrics to json | 20260507 00:49:50.516 | INFO | Serialization done, swapping tmp file now... | 20260507 00:49:50.527 | INFO | RTMetrics done | 20260507 00:49:50.599 | INFO | Finished a round of Baz IO: Wrote 1.833 GiB in 3.205 seconds | 20260507 00:49:50.599 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.23% / 0.00% / 66.77% | 20260507 00:49:50.599 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.321583% / 7.94729e-06% / 6.75519e-05% | 20260507 00:49:50.599 | INFO | Baz IO overall: | 20260507 00:49:50.599 | INFO | BytesWritten : 3011538642906 | 20260507 00:49:50.599 | INFO | NumEvents : 1450312383158 | 20260507 00:49:52.432 | INFO | Analyzing chunk frames = [5787648,5788160) | 20260507 00:49:52.432 | INFO | MemInfo: { | 20260507 00:49:52.432 | INFO | "gpu" : " 17 GB", | 20260507 00:49:52.432 | INFO | "huge" : "", | 20260507 00:49:52.432 | INFO | "huge_shared" : " 27 GB", | 20260507 00:49:52.432 | INFO | "malloc" : " 14 GB", | 20260507 00:49:52.432 | INFO | "pinned" : " 29 GB" | 20260507 00:49:52.432 | INFO | } | 20260507 00:49:52.432 | INFO | Perf report for the last 5.13689 seconds: | 20260507 00:49:52.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:49:52.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:52.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:52.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:52.432 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 00:49:52.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:52.432 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 00:49:52.432 | INFO | PRE_HQ : 59.6%, 1.8, 56.2ms, 0 | 20260507 00:49:52.432 | INFO | BAZWRITER : 62.0%, 3.4, 106.8ms, 0 | 20260507 00:49:52.432 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:49:52.433 | INFO | Uploaded 36GiB to GPU in 2.03404s (17.6987GiB/s) | 20260507 00:49:52.433 | INFO | Downloaded 38GiB to GPU in 2.1888s (17.3611GiB/s) | 20260507 00:49:57.572 | INFO | Analyzing chunk frames = [5788160,5788672) | 20260507 00:49:57.572 | INFO | MemInfo: { | 20260507 00:49:57.572 | INFO | "gpu" : " 17 GB", | 20260507 00:49:57.572 | INFO | "huge" : "", | 20260507 00:49:57.572 | INFO | "huge_shared" : " 27 GB", | 20260507 00:49:57.572 | INFO | "malloc" : " 14 GB", | 20260507 00:49:57.572 | INFO | "pinned" : " 29 GB" | 20260507 00:49:57.572 | INFO | } | 20260507 00:49:57.572 | INFO | Perf report for the last 5.1395 seconds: | 20260507 00:49:57.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:49:57.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:57.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:57.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:57.572 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:49:57.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:57.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:57.572 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:49:57.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:57.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:49:57.572 | INFO | Uploaded 36GiB to GPU in 2.00669s (17.94GiB/s) | 20260507 00:49:57.572 | INFO | Downloaded 34GiB to GPU in 1.93344s (17.5852GiB/s) | 20260507 00:50:02.703 | INFO | Analyzing chunk frames = [5788672,5789184) | 20260507 00:50:02.703 | INFO | MemInfo: { | 20260507 00:50:02.703 | INFO | "gpu" : " 17 GB", | 20260507 00:50:02.703 | INFO | "huge" : "", | 20260507 00:50:02.703 | INFO | "huge_shared" : " 27 GB", | 20260507 00:50:02.703 | INFO | "malloc" : " 14 GB", | 20260507 00:50:02.703 | INFO | "pinned" : " 29 GB" | 20260507 00:50:02.703 | INFO | } | 20260507 00:50:02.703 | INFO | Perf report for the last 5.13151 seconds: | 20260507 00:50:02.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:50:02.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:02.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:02.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:02.703 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 00:50:02.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:02.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:02.703 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:50:02.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:02.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:02.704 | INFO | Uploaded 36GiB to GPU in 1.99641s (18.0324GiB/s) | 20260507 00:50:02.704 | INFO | Downloaded 34GiB to GPU in 1.93857s (17.5387GiB/s) | 20260507 00:50:07.846 | INFO | Analyzing chunk frames = [5789184,5789696) | 20260507 00:50:07.846 | INFO | MemInfo: { | 20260507 00:50:07.846 | INFO | "gpu" : " 17 GB", | 20260507 00:50:07.846 | INFO | "huge" : "", | 20260507 00:50:07.847 | INFO | "huge_shared" : " 27 GB", | 20260507 00:50:07.847 | INFO | "malloc" : " 14 GB", | 20260507 00:50:07.847 | INFO | "pinned" : " 29 GB" | 20260507 00:50:07.847 | INFO | } | 20260507 00:50:07.847 | INFO | Perf report for the last 5.14326 seconds: | 20260507 00:50:07.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:50:07.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:07.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:07.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:07.847 | INFO | ANALYSIS : 59.3%, 3.0, 92.0ms, 0 | 20260507 00:50:07.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:07.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:07.847 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 00:50:07.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:07.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:07.847 | INFO | Uploaded 36GiB to GPU in 1.94895s (18.4715GiB/s) | 20260507 00:50:07.847 | INFO | Downloaded 34GiB to GPU in 1.88293s (18.057GiB/s) | 20260507 00:50:12.992 | INFO | Analyzing chunk frames = [5789696,5790208) | 20260507 00:50:12.992 | INFO | MemInfo: { | 20260507 00:50:12.992 | INFO | "gpu" : " 17 GB", | 20260507 00:50:12.992 | INFO | "huge" : "", | 20260507 00:50:12.992 | INFO | "huge_shared" : " 27 GB", | 20260507 00:50:12.992 | INFO | "malloc" : " 14 GB", | 20260507 00:50:12.992 | INFO | "pinned" : " 29 GB" | 20260507 00:50:12.992 | INFO | } | 20260507 00:50:12.992 | INFO | Perf report for the last 5.14512 seconds: | 20260507 00:50:12.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:50:12.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:12.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:12.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:12.992 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 00:50:12.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:12.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:12.992 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 00:50:12.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:12.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:12.992 | INFO | Uploaded 36GiB to GPU in 1.98385s (18.1466GiB/s) | 20260507 00:50:12.992 | INFO | Downloaded 34GiB to GPU in 1.93107s (17.6068GiB/s) | 20260507 00:50:18.133 | INFO | Analyzing chunk frames = [5790208,5790720) | 20260507 00:50:18.133 | INFO | MemInfo: { | 20260507 00:50:18.133 | INFO | "gpu" : " 17 GB", | 20260507 00:50:18.133 | INFO | "huge" : "", | 20260507 00:50:18.133 | INFO | "huge_shared" : " 27 GB", | 20260507 00:50:18.133 | INFO | "malloc" : " 14 GB", | 20260507 00:50:18.133 | INFO | "pinned" : " 29 GB" | 20260507 00:50:18.133 | INFO | } | 20260507 00:50:18.133 | INFO | Perf report for the last 5.14127 seconds: | 20260507 00:50:18.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:50:18.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:18.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:18.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:18.133 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 00:50:18.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:18.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:18.133 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:50:18.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:18.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:18.133 | INFO | Uploaded 36GiB to GPU in 2.02466s (17.7808GiB/s) | 20260507 00:50:18.133 | INFO | Downloaded 34GiB to GPU in 1.95374s (17.4025GiB/s) | 20260507 00:50:23.270 | INFO | Analyzing chunk frames = [5790720,5791232) | 20260507 00:50:23.270 | INFO | MemInfo: { | 20260507 00:50:23.270 | INFO | "gpu" : " 17 GB", | 20260507 00:50:23.270 | INFO | "huge" : "", | 20260507 00:50:23.270 | INFO | "huge_shared" : " 27 GB", | 20260507 00:50:23.270 | INFO | "malloc" : " 14 GB", | 20260507 00:50:23.270 | INFO | "pinned" : " 29 GB" | 20260507 00:50:23.270 | INFO | } | 20260507 00:50:23.270 | INFO | Perf report for the last 5.13668 seconds: | 20260507 00:50:23.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:50:23.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:23.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:23.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:23.270 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 00:50:23.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:23.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:23.270 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 00:50:23.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:23.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:23.270 | INFO | Uploaded 36GiB to GPU in 1.98028s (18.1792GiB/s) | 20260507 00:50:23.270 | INFO | Downloaded 34GiB to GPU in 1.97219s (17.2397GiB/s) | 20260507 00:50:28.406 | INFO | Analyzing chunk frames = [5791232,5791744) | 20260507 00:50:28.406 | INFO | MemInfo: { | 20260507 00:50:28.406 | INFO | "gpu" : " 17 GB", | 20260507 00:50:28.406 | INFO | "huge" : "", | 20260507 00:50:28.406 | INFO | "huge_shared" : " 27 GB", | 20260507 00:50:28.406 | INFO | "malloc" : " 14 GB", | 20260507 00:50:28.406 | INFO | "pinned" : " 29 GB" | 20260507 00:50:28.406 | INFO | } | 20260507 00:50:28.406 | INFO | Perf report for the last 5.13596 seconds: | 20260507 00:50:28.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:50:28.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:28.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:28.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:28.406 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:50:28.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:28.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:28.406 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:50:28.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:28.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:28.406 | INFO | Uploaded 36GiB to GPU in 1.99335s (18.0601GiB/s) | 20260507 00:50:28.406 | INFO | Downloaded 34GiB to GPU in 1.97789s (17.19GiB/s) | 20260507 00:50:28.406 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:50:28.406 | INFO | Total mem usage: 18 GiB | 20260507 00:50:28.406 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:50:28.406 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 00:50:28.406 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:50:28.406 | INFO | High Water Breakdown: | 20260507 00:50:28.406 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:50:28.406 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:50:28.406 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:50:28.406 | INFO | ------------------------------------------ | 20260507 00:50:28.406 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:50:28.406 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:50:28.406 | INFO | Total mem usage: 26.5 GiB | 20260507 00:50:28.406 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:50:28.406 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:50:28.406 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:50:28.406 | INFO | High Water Breakdown: | 20260507 00:50:28.406 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:50:28.406 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:50:28.406 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:50:28.406 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:50:28.406 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:50:28.406 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:50:28.406 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:50:28.406 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:50:28.406 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:50:28.406 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:50:28.406 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:50:28.406 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:50:28.406 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:50:28.406 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:50:28.406 | INFO | ------------------------------------------ | 20260507 00:50:28.406 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:50:28.406 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:50:28.406 | INFO | Total mem usage: 25.6 GiB | 20260507 00:50:28.406 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:50:28.406 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:50:28.406 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:50:28.406 | INFO | High Water Breakdown: | 20260507 00:50:28.406 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:50:28.406 | INFO | ------------------------------------------ | 20260507 00:50:28.406 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:50:28.406 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:50:28.406 | INFO | Total mem usage: 15.7 GiB | 20260507 00:50:28.407 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:50:28.407 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:50:28.407 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 00:50:28.407 | INFO | High Water Breakdown: | 20260507 00:50:28.407 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:50:28.407 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:50:28.407 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:50:28.407 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:50:28.407 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:50:28.407 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:50:28.407 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:50:28.407 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:50:28.407 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:50:28.407 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:50:28.407 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:50:28.407 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:50:28.407 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:50:28.407 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:50:28.407 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:50:28.407 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:50:28.407 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:50:28.407 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:50:28.407 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:50:28.407 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:50:28.407 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:50:28.407 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:50:28.407 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:50:28.407 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:50:28.407 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:50:28.407 | INFO | ------------------------------------------ | 20260507 00:50:28.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:50:28.407 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:50:28.407 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:50:28.407 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:50:28.407 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.53e+01, 1.35e+01 / 2.17e+00 | 20260507 00:50:28.407 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.82e-03 / 6.22e-03 | 20260507 00:50:28.407 | INFO | PerfReport: 11.66%, Baseline -- 3.60e+00 / 4.41e+00, 5.34e-01 / 3.16e-01 | 20260507 00:50:28.407 | INFO | PerfReport: 3.25%, Histogram -- 1.01e+00 / 1.45e+00, 1.76e-01 / 1.41e-01 | 20260507 00:50:28.407 | INFO | PerfReport: 16.06%, DME -- 3.97e+01 / 4.40e+01, 3.86e+00 / 2.26e+00 | 20260507 00:50:28.407 | INFO | PerfReport: 57.51%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.35e+00 / 1.78e+00 | 20260507 00:50:28.407 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.17e+00 / 2.67e+00, 3.75e-01 / 2.47e-01 | 20260507 00:50:28.407 | INFO | PerfReport: 3.14%, Metrics -- 9.69e-01 / 1.30e+00, 2.58e-01 / 1.74e-01 | 20260507 00:50:28.407 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.30e-01 / 3.82e-01, 5.24e-02 / 2.03e-02 | 20260507 00:50:28.407 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:50:28.407 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:50:28.407 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:50:28.407 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 00:50:28.407 | INFO | PerfReport: 21.57%, Upload -- 2.08e+01 / 0.00e+00, 6.94e+00 / 0.00e+00 | 20260507 00:50:28.407 | INFO | PerfReport: 21.32%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 00:50:28.407 | INFO | PerfReport: 32.04%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:50:28.407 | INFO | PerfReport: 1.26%, LockUpload -- 1.22e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 00:50:28.407 | INFO | PerfReport: 1.35%, LockDownload -- 1.31e+00 / 0.00e+00, 3.23e+00 / 0.00e+00 | 20260507 00:50:28.407 | INFO | PerfReport: 22.44%, LockCompute -- 2.16e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 00:50:28.407 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260507 00:50:28.503 | INFO | Starting a round of Baz IO | 20260507 00:50:31.628 | INFO | Populating RTMetrics report | 20260507 00:50:31.629 | INFO | RTMetrics report populated | 20260507 00:50:31.629 | INFO | Opening tmp RTMetrics json file | 20260507 00:50:31.648 | INFO | Serializing RTMetrics to json | 20260507 00:50:31.649 | INFO | Serialization done, swapping tmp file now... | 20260507 00:50:31.650 | INFO | RTMetrics done | 20260507 00:50:31.720 | INFO | Finished a round of Baz IO: Wrote 1.834 GiB in 3.217 seconds | 20260507 00:50:31.720 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.23% / 0.00% / 66.77% | 20260507 00:50:31.720 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.319397% / 0% / 9.53674e-05% | 20260507 00:50:31.721 | INFO | Baz IO overall: | 20260507 00:50:31.721 | INFO | BytesWritten : 3013508046612 | 20260507 00:50:31.721 | INFO | NumEvents : 1451247460024 | 20260507 00:50:33.539 | INFO | Analyzing chunk frames = [5791744,5792256) | 20260507 00:50:33.539 | INFO | MemInfo: { | 20260507 00:50:33.539 | INFO | "gpu" : " 17 GB", | 20260507 00:50:33.539 | INFO | "huge" : "", | 20260507 00:50:33.539 | INFO | "huge_shared" : " 27 GB", | 20260507 00:50:33.539 | INFO | "malloc" : " 14 GB", | 20260507 00:50:33.539 | INFO | "pinned" : " 29 GB" | 20260507 00:50:33.539 | INFO | } | 20260507 00:50:33.539 | INFO | Perf report for the last 5.13356 seconds: | 20260507 00:50:33.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:50:33.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:33.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:33.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:33.539 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260507 00:50:33.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:33.539 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 00:50:33.539 | INFO | PRE_HQ : 59.7%, 1.8, 56.0ms, 0 | 20260507 00:50:33.539 | INFO | BAZWRITER : 62.2%, 3.3, 106.0ms, 0 | 20260507 00:50:33.539 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:50:33.540 | INFO | Uploaded 36GiB to GPU in 2.02659s (17.7638GiB/s) | 20260507 00:50:33.540 | INFO | Downloaded 38GiB to GPU in 2.17555s (17.4668GiB/s) | 20260507 00:50:38.680 | INFO | Analyzing chunk frames = [5792256,5792768) | 20260507 00:50:38.680 | INFO | MemInfo: { | 20260507 00:50:38.680 | INFO | "gpu" : " 17 GB", | 20260507 00:50:38.680 | INFO | "huge" : "", | 20260507 00:50:38.680 | INFO | "huge_shared" : " 27 GB", | 20260507 00:50:38.680 | INFO | "malloc" : " 14 GB", | 20260507 00:50:38.680 | INFO | "pinned" : " 29 GB" | 20260507 00:50:38.680 | INFO | } | 20260507 00:50:38.680 | INFO | Perf report for the last 5.14111 seconds: | 20260507 00:50:38.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:50:38.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:38.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:38.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:38.680 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:50:38.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:38.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:38.680 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 00:50:38.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:38.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:38.681 | INFO | Uploaded 36GiB to GPU in 2.00342s (17.9692GiB/s) | 20260507 00:50:38.681 | INFO | Downloaded 34GiB to GPU in 1.93258s (17.5931GiB/s) | 20260507 00:50:43.826 | INFO | Analyzing chunk frames = [5792768,5793280) | 20260507 00:50:43.826 | INFO | MemInfo: { | 20260507 00:50:43.826 | INFO | "gpu" : " 17 GB", | 20260507 00:50:43.826 | INFO | "huge" : "", | 20260507 00:50:43.826 | INFO | "huge_shared" : " 27 GB", | 20260507 00:50:43.826 | INFO | "malloc" : " 14 GB", | 20260507 00:50:43.826 | INFO | "pinned" : " 29 GB" | 20260507 00:50:43.826 | INFO | } | 20260507 00:50:43.826 | INFO | Perf report for the last 5.14526 seconds: | 20260507 00:50:43.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:50:43.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:43.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:43.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:43.826 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:50:43.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:43.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:43.826 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:50:43.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:43.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:43.826 | INFO | Uploaded 36GiB to GPU in 1.97515s (18.2265GiB/s) | 20260507 00:50:43.826 | INFO | Downloaded 34GiB to GPU in 1.93563s (17.5653GiB/s) | 20260507 00:50:48.959 | INFO | Analyzing chunk frames = [5793280,5793792) | 20260507 00:50:48.959 | INFO | MemInfo: { | 20260507 00:50:48.959 | INFO | "gpu" : " 17 GB", | 20260507 00:50:48.959 | INFO | "huge" : "", | 20260507 00:50:48.959 | INFO | "huge_shared" : " 27 GB", | 20260507 00:50:48.959 | INFO | "malloc" : " 14 GB", | 20260507 00:50:48.959 | INFO | "pinned" : " 29 GB" | 20260507 00:50:48.959 | INFO | } | 20260507 00:50:48.959 | INFO | Perf report for the last 5.13356 seconds: | 20260507 00:50:48.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:50:48.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:48.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:48.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:48.959 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:50:48.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:48.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:48.959 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:50:48.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:48.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:48.959 | INFO | Uploaded 36GiB to GPU in 1.96663s (18.3054GiB/s) | 20260507 00:50:48.960 | INFO | Downloaded 34GiB to GPU in 1.90954s (17.8053GiB/s) | 20260507 00:50:54.106 | INFO | Analyzing chunk frames = [5793792,5794304) | 20260507 00:50:54.106 | INFO | MemInfo: { | 20260507 00:50:54.106 | INFO | "gpu" : " 17 GB", | 20260507 00:50:54.106 | INFO | "huge" : "", | 20260507 00:50:54.106 | INFO | "huge_shared" : " 27 GB", | 20260507 00:50:54.106 | INFO | "malloc" : " 14 GB", | 20260507 00:50:54.106 | INFO | "pinned" : " 29 GB" | 20260507 00:50:54.106 | INFO | } | 20260507 00:50:54.106 | INFO | Perf report for the last 5.14647 seconds: | 20260507 00:50:54.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:50:54.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:54.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:54.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:54.106 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 00:50:54.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:54.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:54.106 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:50:54.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:54.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:54.106 | INFO | Uploaded 36GiB to GPU in 1.96389s (18.331GiB/s) | 20260507 00:50:54.106 | INFO | Downloaded 34GiB to GPU in 1.9278s (17.6367GiB/s) | 20260507 00:50:59.246 | INFO | Analyzing chunk frames = [5794304,5794816) | 20260507 00:50:59.246 | INFO | MemInfo: { | 20260507 00:50:59.246 | INFO | "gpu" : " 17 GB", | 20260507 00:50:59.246 | INFO | "huge" : "", | 20260507 00:50:59.246 | INFO | "huge_shared" : " 27 GB", | 20260507 00:50:59.246 | INFO | "malloc" : " 14 GB", | 20260507 00:50:59.246 | INFO | "pinned" : " 29 GB" | 20260507 00:50:59.246 | INFO | } | 20260507 00:50:59.246 | INFO | Perf report for the last 5.1403 seconds: | 20260507 00:50:59.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:50:59.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:59.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:59.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:59.246 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:50:59.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:59.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:59.246 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260507 00:50:59.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:59.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:50:59.246 | INFO | Uploaded 36GiB to GPU in 2.04342s (17.6175GiB/s) | 20260507 00:50:59.246 | INFO | Downloaded 34GiB to GPU in 1.95998s (17.3471GiB/s) | 20260507 00:51:04.382 | INFO | Analyzing chunk frames = [5794816,5795328) | 20260507 00:51:04.382 | INFO | MemInfo: { | 20260507 00:51:04.382 | INFO | "gpu" : " 17 GB", | 20260507 00:51:04.382 | INFO | "huge" : "", | 20260507 00:51:04.382 | INFO | "huge_shared" : " 27 GB", | 20260507 00:51:04.382 | INFO | "malloc" : " 14 GB", | 20260507 00:51:04.382 | INFO | "pinned" : " 29 GB" | 20260507 00:51:04.382 | INFO | } | 20260507 00:51:04.382 | INFO | Perf report for the last 5.1362 seconds: | 20260507 00:51:04.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:51:04.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:04.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:04.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:04.382 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 00:51:04.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:04.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:04.382 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:51:04.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:04.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:04.382 | INFO | Uploaded 36GiB to GPU in 1.96445s (18.3257GiB/s) | 20260507 00:51:04.383 | INFO | Downloaded 34GiB to GPU in 1.96055s (17.342GiB/s) | 20260507 00:51:09.518 | INFO | Analyzing chunk frames = [5795328,5795840) | 20260507 00:51:09.518 | INFO | MemInfo: { | 20260507 00:51:09.518 | INFO | "gpu" : " 17 GB", | 20260507 00:51:09.518 | INFO | "huge" : "", | 20260507 00:51:09.518 | INFO | "huge_shared" : " 27 GB", | 20260507 00:51:09.518 | INFO | "malloc" : " 14 GB", | 20260507 00:51:09.518 | INFO | "pinned" : " 29 GB" | 20260507 00:51:09.518 | INFO | } | 20260507 00:51:09.519 | INFO | Perf report for the last 5.13624 seconds: | 20260507 00:51:09.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:51:09.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:09.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:09.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:09.519 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260507 00:51:09.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:09.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:09.519 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 00:51:09.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:09.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:09.519 | INFO | Uploaded 36GiB to GPU in 2.00767s (17.9312GiB/s) | 20260507 00:51:09.519 | INFO | Downloaded 34GiB to GPU in 1.9704s (17.2554GiB/s) | 20260507 00:51:09.617 | INFO | Starting a round of Baz IO | 20260507 00:51:12.746 | INFO | Populating RTMetrics report | 20260507 00:51:12.746 | INFO | RTMetrics report populated | 20260507 00:51:12.746 | INFO | Opening tmp RTMetrics json file | 20260507 00:51:12.751 | INFO | Serializing RTMetrics to json | 20260507 00:51:12.765 | INFO | Serialization done, swapping tmp file now... | 20260507 00:51:12.766 | INFO | RTMetrics done | 20260507 00:51:12.839 | INFO | Finished a round of Baz IO: Wrote 1.833 GiB in 3.222 seconds | 20260507 00:51:12.839 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.22% / 0.00% / 66.77% | 20260507 00:51:12.839 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.319095% / 0% / 7.94729e-05% | 20260507 00:51:12.839 | INFO | Baz IO overall: | 20260507 00:51:12.839 | INFO | BytesWritten : 3015476489185 | 20260507 00:51:12.839 | INFO | NumEvents : 1452182065969 | 20260507 00:51:14.657 | INFO | Analyzing chunk frames = [5795840,5796352) | 20260507 00:51:14.657 | INFO | MemInfo: { | 20260507 00:51:14.657 | INFO | "gpu" : " 17 GB", | 20260507 00:51:14.657 | INFO | "huge" : "", | 20260507 00:51:14.657 | INFO | "huge_shared" : " 27 GB", | 20260507 00:51:14.657 | INFO | "malloc" : " 14 GB", | 20260507 00:51:14.657 | INFO | "pinned" : " 29 GB" | 20260507 00:51:14.657 | INFO | } | 20260507 00:51:14.657 | INFO | Perf report for the last 5.13845 seconds: | 20260507 00:51:14.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:51:14.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:14.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:14.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:14.657 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 00:51:14.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:14.657 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 00:51:14.657 | INFO | PRE_HQ : 60.0%, 1.8, 57.4ms, 0 | 20260507 00:51:14.657 | INFO | BAZWRITER : 62.3%, 3.3, 106.2ms, 0 | 20260507 00:51:14.657 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 00:51:14.658 | INFO | Uploaded 36GiB to GPU in 2.03522s (17.6885GiB/s) | 20260507 00:51:14.658 | INFO | Downloaded 38GiB to GPU in 2.17905s (17.4388GiB/s) | 20260507 00:51:19.804 | INFO | Analyzing chunk frames = [5796352,5796864) | 20260507 00:51:19.804 | INFO | MemInfo: { | 20260507 00:51:19.804 | INFO | "gpu" : " 17 GB", | 20260507 00:51:19.804 | INFO | "huge" : "", | 20260507 00:51:19.804 | INFO | "huge_shared" : " 27 GB", | 20260507 00:51:19.804 | INFO | "malloc" : " 14 GB", | 20260507 00:51:19.804 | INFO | "pinned" : " 29 GB" | 20260507 00:51:19.804 | INFO | } | 20260507 00:51:19.804 | INFO | Perf report for the last 5.14699 seconds: | 20260507 00:51:19.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:51:19.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:19.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:19.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:19.804 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260507 00:51:19.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:19.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:19.804 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 00:51:19.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:19.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:19.804 | INFO | Uploaded 36GiB to GPU in 2.00675s (17.9394GiB/s) | 20260507 00:51:19.804 | INFO | Downloaded 34GiB to GPU in 1.93234s (17.5953GiB/s) | 20260507 00:51:24.945 | INFO | Analyzing chunk frames = [5796864,5797376) | 20260507 00:51:24.945 | INFO | MemInfo: { | 20260507 00:51:24.945 | INFO | "gpu" : " 17 GB", | 20260507 00:51:24.945 | INFO | "huge" : "", | 20260507 00:51:24.945 | INFO | "huge_shared" : " 27 GB", | 20260507 00:51:24.945 | INFO | "malloc" : " 14 GB", | 20260507 00:51:24.945 | INFO | "pinned" : " 29 GB" | 20260507 00:51:24.945 | INFO | } | 20260507 00:51:24.945 | INFO | Perf report for the last 5.14128 seconds: | 20260507 00:51:24.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:51:24.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:24.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:24.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:24.945 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 00:51:24.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:24.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:24.945 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:51:24.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:24.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:24.945 | INFO | Uploaded 36GiB to GPU in 2.00104s (17.9906GiB/s) | 20260507 00:51:24.945 | INFO | Downloaded 34GiB to GPU in 1.93312s (17.5881GiB/s) | 20260507 00:51:30.080 | INFO | Analyzing chunk frames = [5797376,5797888) | 20260507 00:51:30.080 | INFO | MemInfo: { | 20260507 00:51:30.080 | INFO | "gpu" : " 17 GB", | 20260507 00:51:30.080 | INFO | "huge" : "", | 20260507 00:51:30.080 | INFO | "huge_shared" : " 27 GB", | 20260507 00:51:30.080 | INFO | "malloc" : " 14 GB", | 20260507 00:51:30.080 | INFO | "pinned" : " 29 GB" | 20260507 00:51:30.080 | INFO | } | 20260507 00:51:30.080 | INFO | Perf report for the last 5.13501 seconds: | 20260507 00:51:30.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:51:30.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:30.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:30.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:30.080 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 00:51:30.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:30.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:30.080 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:51:30.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:30.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:30.081 | INFO | Uploaded 36GiB to GPU in 1.98632s (18.124GiB/s) | 20260507 00:51:30.081 | INFO | Downloaded 34GiB to GPU in 1.91589s (17.7463GiB/s) | 20260507 00:51:35.220 | INFO | Analyzing chunk frames = [5797888,5798400) | 20260507 00:51:35.220 | INFO | MemInfo: { | 20260507 00:51:35.220 | INFO | "gpu" : " 17 GB", | 20260507 00:51:35.220 | INFO | "huge" : "", | 20260507 00:51:35.220 | INFO | "huge_shared" : " 27 GB", | 20260507 00:51:35.220 | INFO | "malloc" : " 14 GB", | 20260507 00:51:35.220 | INFO | "pinned" : " 29 GB" | 20260507 00:51:35.220 | INFO | } | 20260507 00:51:35.220 | INFO | Perf report for the last 5.13964 seconds: | 20260507 00:51:35.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:51:35.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:35.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:35.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:35.220 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 00:51:35.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:35.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:35.220 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 00:51:35.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:35.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:35.220 | INFO | Uploaded 36GiB to GPU in 2.014s (17.8748GiB/s) | 20260507 00:51:35.220 | INFO | Downloaded 34GiB to GPU in 1.9389s (17.5357GiB/s) | 20260507 00:51:40.356 | INFO | Analyzing chunk frames = [5798400,5798912) | 20260507 00:51:40.356 | INFO | MemInfo: { | 20260507 00:51:40.356 | INFO | "gpu" : " 17 GB", | 20260507 00:51:40.356 | INFO | "huge" : "", | 20260507 00:51:40.356 | INFO | "huge_shared" : " 27 GB", | 20260507 00:51:40.356 | INFO | "malloc" : " 14 GB", | 20260507 00:51:40.356 | INFO | "pinned" : " 29 GB" | 20260507 00:51:40.356 | INFO | } | 20260507 00:51:40.356 | INFO | Perf report for the last 5.13566 seconds: | 20260507 00:51:40.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:51:40.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:40.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:40.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:40.356 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:51:40.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:40.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:40.356 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:51:40.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:40.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:40.356 | INFO | Uploaded 36GiB to GPU in 2.05393s (17.5274GiB/s) | 20260507 00:51:40.356 | INFO | Downloaded 34GiB to GPU in 1.94823s (17.4517GiB/s) | 20260507 00:51:45.495 | INFO | Analyzing chunk frames = [5798912,5799424) | 20260507 00:51:45.495 | INFO | MemInfo: { | 20260507 00:51:45.495 | INFO | "gpu" : " 17 GB", | 20260507 00:51:45.495 | INFO | "huge" : "", | 20260507 00:51:45.495 | INFO | "huge_shared" : " 27 GB", | 20260507 00:51:45.495 | INFO | "malloc" : " 14 GB", | 20260507 00:51:45.495 | INFO | "pinned" : " 29 GB" | 20260507 00:51:45.495 | INFO | } | 20260507 00:51:45.495 | INFO | Perf report for the last 5.13924 seconds: | 20260507 00:51:45.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:51:45.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:45.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:45.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:45.495 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 00:51:45.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:45.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:45.495 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:51:45.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:45.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:45.495 | INFO | Uploaded 36GiB to GPU in 1.99747s (18.0228GiB/s) | 20260507 00:51:45.495 | INFO | Downloaded 34GiB to GPU in 2.00096s (16.9918GiB/s) | 20260507 00:51:50.635 | INFO | Analyzing chunk frames = [5799424,5799936) | 20260507 00:51:50.635 | INFO | MemInfo: { | 20260507 00:51:50.635 | INFO | "gpu" : " 17 GB", | 20260507 00:51:50.635 | INFO | "huge" : "", | 20260507 00:51:50.635 | INFO | "huge_shared" : " 27 GB", | 20260507 00:51:50.635 | INFO | "malloc" : " 14 GB", | 20260507 00:51:50.635 | INFO | "pinned" : " 29 GB" | 20260507 00:51:50.635 | INFO | } | 20260507 00:51:50.635 | INFO | Perf report for the last 5.14001 seconds: | 20260507 00:51:50.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:51:50.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:50.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:50.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:50.635 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:51:50.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:50.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:50.635 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:51:50.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:50.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:50.635 | INFO | Uploaded 36GiB to GPU in 1.98267s (18.1574GiB/s) | 20260507 00:51:50.635 | INFO | Downloaded 34GiB to GPU in 1.98095s (17.1634GiB/s) | 20260507 00:51:50.635 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:51:50.635 | INFO | Total mem usage: 18 GiB | 20260507 00:51:50.635 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:51:50.635 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 00:51:50.635 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:51:50.635 | INFO | High Water Breakdown: | 20260507 00:51:50.635 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:51:50.635 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:51:50.635 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:51:50.635 | INFO | ------------------------------------------ | 20260507 00:51:50.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:51:50.635 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:51:50.635 | INFO | Total mem usage: 26.5 GiB | 20260507 00:51:50.635 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:51:50.635 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:51:50.635 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:51:50.635 | INFO | High Water Breakdown: | 20260507 00:51:50.635 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:51:50.635 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:51:50.635 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:51:50.635 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:51:50.636 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:51:50.636 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:51:50.636 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:51:50.636 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:51:50.636 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:51:50.636 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:51:50.636 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:51:50.636 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:51:50.636 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:51:50.636 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:51:50.636 | INFO | ------------------------------------------ | 20260507 00:51:50.636 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:51:50.636 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:51:50.636 | INFO | Total mem usage: 25.6 GiB | 20260507 00:51:50.636 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:51:50.636 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:51:50.636 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:51:50.636 | INFO | High Water Breakdown: | 20260507 00:51:50.636 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:51:50.636 | INFO | ------------------------------------------ | 20260507 00:51:50.636 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:51:50.636 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:51:50.636 | INFO | Total mem usage: 15.7 GiB | 20260507 00:51:50.636 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:51:50.636 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:51:50.636 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:51:50.636 | INFO | High Water Breakdown: | 20260507 00:51:50.636 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:51:50.636 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:51:50.636 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:51:50.636 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:51:50.636 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:51:50.636 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:51:50.636 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:51:50.636 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:51:50.636 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:51:50.636 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:51:50.636 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:51:50.636 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:51:50.636 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:51:50.636 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:51:50.636 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:51:50.636 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:51:50.636 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:51:50.636 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:51:50.636 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:51:50.636 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:51:50.636 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:51:50.636 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:51:50.636 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:51:50.636 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:51:50.636 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:51:50.636 | INFO | ------------------------------------------ | 20260507 00:51:50.636 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:51:50.636 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:51:50.636 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:51:50.636 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:51:50.636 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.59e+01, 1.35e+01 / 1.95e+00 | 20260507 00:51:50.636 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.01e-01, 1.01e-02 / 4.39e-03 | 20260507 00:51:50.636 | INFO | PerfReport: 11.66%, Baseline -- 3.60e+00 / 4.46e+00, 5.25e-01 / 2.41e-01 | 20260507 00:51:50.636 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.48e+00, 1.74e-01 / 9.74e-02 | 20260507 00:51:50.636 | INFO | PerfReport: 16.12%, DME -- 3.99e+01 / 4.49e+01, 4.14e+00 / 2.42e+00 | 20260507 00:51:50.636 | INFO | PerfReport: 57.43%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.33e+00 / 1.36e+00 | 20260507 00:51:50.636 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.17e+00 / 2.73e+00, 3.82e-01 / 1.83e-01 | 20260507 00:51:50.636 | INFO | PerfReport: 3.14%, Metrics -- 9.70e-01 / 1.32e+00, 2.58e-01 / 1.35e-01 | 20260507 00:51:50.636 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.30e-01 / 3.84e-01, 5.29e-02 / 1.82e-02 | 20260507 00:51:50.636 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:51:50.636 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:51:50.636 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:51:50.636 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 00:51:50.636 | INFO | PerfReport: 21.59%, Upload -- 2.09e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260507 00:51:50.636 | INFO | PerfReport: 21.30%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260507 00:51:50.636 | INFO | PerfReport: 32.01%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:51:50.636 | INFO | PerfReport: 1.38%, LockUpload -- 1.33e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 00:51:50.636 | INFO | PerfReport: 1.29%, LockDownload -- 1.25e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260507 00:51:50.636 | INFO | PerfReport: 22.41%, LockCompute -- 2.16e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 00:51:50.636 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260507 00:51:50.732 | INFO | Starting a round of Baz IO | 20260507 00:51:53.853 | INFO | Populating RTMetrics report | 20260507 00:51:53.853 | INFO | RTMetrics report populated | 20260507 00:51:53.853 | INFO | Opening tmp RTMetrics json file | 20260507 00:51:53.872 | INFO | Serializing RTMetrics to json | 20260507 00:51:53.874 | INFO | Serialization done, swapping tmp file now... | 20260507 00:51:53.874 | INFO | RTMetrics done | 20260507 00:51:53.942 | INFO | Finished a round of Baz IO: Wrote 1.834 GiB in 3.209 seconds | 20260507 00:51:53.942 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.22% / 0.00% / 66.78% | 20260507 00:51:53.942 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.319056% / 0% / 7.94729e-05% | 20260507 00:51:53.942 | INFO | Baz IO overall: | 20260507 00:51:53.942 | INFO | BytesWritten : 3017445755578 | 20260507 00:51:53.942 | INFO | NumEvents : 1453117190811 | 20260507 00:51:55.777 | INFO | Analyzing chunk frames = [5799936,5800448) | 20260507 00:51:55.778 | INFO | MemInfo: { | 20260507 00:51:55.778 | INFO | "gpu" : " 17 GB", | 20260507 00:51:55.778 | INFO | "huge" : "", | 20260507 00:51:55.778 | INFO | "huge_shared" : " 27 GB", | 20260507 00:51:55.778 | INFO | "malloc" : " 14 GB", | 20260507 00:51:55.778 | INFO | "pinned" : " 29 GB" | 20260507 00:51:55.778 | INFO | } | 20260507 00:51:55.778 | INFO | Perf report for the last 5.1427 seconds: | 20260507 00:51:55.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:51:55.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:55.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:55.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:55.778 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 00:51:55.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:51:55.778 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 00:51:55.778 | INFO | PRE_HQ : 59.7%, 1.8, 56.3ms, 0 | 20260507 00:51:55.778 | INFO | BAZWRITER : 62.0%, 3.3, 105.0ms, 0 | 20260507 00:51:55.778 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 00:51:55.778 | INFO | Uploaded 36GiB to GPU in 2.01866s (17.8336GiB/s) | 20260507 00:51:55.778 | INFO | Downloaded 38GiB to GPU in 2.16759s (17.531GiB/s) | 20260507 00:52:00.916 | INFO | Analyzing chunk frames = [5800448,5800960) | 20260507 00:52:00.916 | INFO | MemInfo: { | 20260507 00:52:00.916 | INFO | "gpu" : " 17 GB", | 20260507 00:52:00.916 | INFO | "huge" : "", | 20260507 00:52:00.916 | INFO | "huge_shared" : " 27 GB", | 20260507 00:52:00.916 | INFO | "malloc" : " 14 GB", | 20260507 00:52:00.916 | INFO | "pinned" : " 29 GB" | 20260507 00:52:00.916 | INFO | } | 20260507 00:52:00.916 | INFO | Perf report for the last 5.13879 seconds: | 20260507 00:52:00.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:52:00.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:00.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:00.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:00.916 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:52:00.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:00.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:00.916 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 00:52:00.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:00.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:00.917 | INFO | Uploaded 36GiB to GPU in 1.98955s (18.0945GiB/s) | 20260507 00:52:00.917 | INFO | Downloaded 34GiB to GPU in 1.93453s (17.5753GiB/s) | 20260507 00:52:06.059 | INFO | Analyzing chunk frames = [5800960,5801472) | 20260507 00:52:06.059 | INFO | MemInfo: { | 20260507 00:52:06.059 | INFO | "gpu" : " 17 GB", | 20260507 00:52:06.059 | INFO | "huge" : "", | 20260507 00:52:06.059 | INFO | "huge_shared" : " 27 GB", | 20260507 00:52:06.059 | INFO | "malloc" : " 14 GB", | 20260507 00:52:06.059 | INFO | "pinned" : " 29 GB" | 20260507 00:52:06.059 | INFO | } | 20260507 00:52:06.059 | INFO | Perf report for the last 5.14276 seconds: | 20260507 00:52:06.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:52:06.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:06.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:06.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:06.059 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 00:52:06.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:06.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:06.059 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:52:06.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:06.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:06.059 | INFO | Uploaded 36GiB to GPU in 1.98977s (18.0925GiB/s) | 20260507 00:52:06.059 | INFO | Downloaded 34GiB to GPU in 1.94038s (17.5224GiB/s) | 20260507 00:52:11.200 | INFO | Analyzing chunk frames = [5801472,5801984) | 20260507 00:52:11.200 | INFO | MemInfo: { | 20260507 00:52:11.200 | INFO | "gpu" : " 17 GB", | 20260507 00:52:11.200 | INFO | "huge" : "", | 20260507 00:52:11.200 | INFO | "huge_shared" : " 27 GB", | 20260507 00:52:11.200 | INFO | "malloc" : " 14 GB", | 20260507 00:52:11.200 | INFO | "pinned" : " 29 GB" | 20260507 00:52:11.200 | INFO | } | 20260507 00:52:11.200 | INFO | Perf report for the last 5.14113 seconds: | 20260507 00:52:11.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:52:11.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:11.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:11.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:11.200 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 00:52:11.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:11.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:11.200 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:52:11.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:11.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:11.201 | INFO | Uploaded 36GiB to GPU in 1.98707s (18.1171GiB/s) | 20260507 00:52:11.201 | INFO | Downloaded 34GiB to GPU in 1.92129s (17.6964GiB/s) | 20260507 00:52:16.333 | INFO | Analyzing chunk frames = [5801984,5802496) | 20260507 00:52:16.333 | INFO | MemInfo: { | 20260507 00:52:16.333 | INFO | "gpu" : " 17 GB", | 20260507 00:52:16.333 | INFO | "huge" : "", | 20260507 00:52:16.333 | INFO | "huge_shared" : " 27 GB", | 20260507 00:52:16.333 | INFO | "malloc" : " 14 GB", | 20260507 00:52:16.333 | INFO | "pinned" : " 29 GB" | 20260507 00:52:16.333 | INFO | } | 20260507 00:52:16.333 | INFO | Perf report for the last 5.13283 seconds: | 20260507 00:52:16.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:52:16.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:16.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:16.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:16.333 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260507 00:52:16.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:16.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:16.333 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:52:16.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:16.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:16.333 | INFO | Uploaded 36GiB to GPU in 2.0068s (17.9391GiB/s) | 20260507 00:52:16.333 | INFO | Downloaded 34GiB to GPU in 1.95738s (17.3702GiB/s) | 20260507 00:52:21.468 | INFO | Analyzing chunk frames = [5802496,5803008) | 20260507 00:52:21.469 | INFO | MemInfo: { | 20260507 00:52:21.469 | INFO | "gpu" : " 17 GB", | 20260507 00:52:21.469 | INFO | "huge" : "", | 20260507 00:52:21.469 | INFO | "huge_shared" : " 27 GB", | 20260507 00:52:21.469 | INFO | "malloc" : " 14 GB", | 20260507 00:52:21.469 | INFO | "pinned" : " 29 GB" | 20260507 00:52:21.469 | INFO | } | 20260507 00:52:21.469 | INFO | Perf report for the last 5.13546 seconds: | 20260507 00:52:21.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:52:21.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:21.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:21.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:21.469 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 00:52:21.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:21.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:21.469 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 00:52:21.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:21.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:21.469 | INFO | Uploaded 36GiB to GPU in 2.02035s (17.8187GiB/s) | 20260507 00:52:21.469 | INFO | Downloaded 34GiB to GPU in 1.95791s (17.3655GiB/s) | 20260507 00:52:26.612 | INFO | Analyzing chunk frames = [5803008,5803520) | 20260507 00:52:26.612 | INFO | MemInfo: { | 20260507 00:52:26.612 | INFO | "gpu" : " 17 GB", | 20260507 00:52:26.612 | INFO | "huge" : "", | 20260507 00:52:26.612 | INFO | "huge_shared" : " 27 GB", | 20260507 00:52:26.612 | INFO | "malloc" : " 14 GB", | 20260507 00:52:26.612 | INFO | "pinned" : " 29 GB" | 20260507 00:52:26.612 | INFO | } | 20260507 00:52:26.612 | INFO | Perf report for the last 5.14354 seconds: | 20260507 00:52:26.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:52:26.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:26.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:26.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:26.612 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 00:52:26.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:26.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:26.612 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 00:52:26.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:26.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:26.612 | INFO | Uploaded 36GiB to GPU in 1.95882s (18.3784GiB/s) | 20260507 00:52:26.612 | INFO | Downloaded 34GiB to GPU in 1.97816s (17.1877GiB/s) | 20260507 00:52:31.749 | INFO | Analyzing chunk frames = [5803520,5804032) | 20260507 00:52:31.749 | INFO | MemInfo: { | 20260507 00:52:31.749 | INFO | "gpu" : " 17 GB", | 20260507 00:52:31.749 | INFO | "huge" : "", | 20260507 00:52:31.749 | INFO | "huge_shared" : " 27 GB", | 20260507 00:52:31.749 | INFO | "malloc" : " 14 GB", | 20260507 00:52:31.749 | INFO | "pinned" : " 29 GB" | 20260507 00:52:31.749 | INFO | } | 20260507 00:52:31.749 | INFO | Perf report for the last 5.13706 seconds: | 20260507 00:52:31.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:52:31.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:31.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:31.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:31.749 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:52:31.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:31.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:31.749 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:52:31.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:31.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:31.750 | INFO | Uploaded 36GiB to GPU in 1.97435s (18.2338GiB/s) | 20260507 00:52:31.750 | INFO | Downloaded 34GiB to GPU in 1.97967s (17.1746GiB/s) | 20260507 00:52:31.847 | INFO | Starting a round of Baz IO | 20260507 00:52:34.969 | INFO | Populating RTMetrics report | 20260507 00:52:34.970 | INFO | RTMetrics report populated | 20260507 00:52:34.970 | INFO | Opening tmp RTMetrics json file | 20260507 00:52:34.972 | INFO | Serializing RTMetrics to json | 20260507 00:52:34.989 | INFO | Serialization done, swapping tmp file now... | 20260507 00:52:34.989 | INFO | RTMetrics done | 20260507 00:52:35.060 | INFO | Finished a round of Baz IO: Wrote 1.831 GiB in 3.213 seconds | 20260507 00:52:35.060 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.22% / 0.00% / 66.78% | 20260507 00:52:35.060 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.317466% / 7.94729e-06% / 6.35783e-05% | 20260507 00:52:35.060 | INFO | Baz IO overall: | 20260507 00:52:35.060 | INFO | BytesWritten : 3019411883778 | 20260507 00:52:35.060 | INFO | NumEvents : 1454050128342 | 20260507 00:52:36.890 | INFO | Analyzing chunk frames = [5804032,5804544) | 20260507 00:52:36.890 | INFO | MemInfo: { | 20260507 00:52:36.890 | INFO | "gpu" : " 17 GB", | 20260507 00:52:36.890 | INFO | "huge" : "", | 20260507 00:52:36.890 | INFO | "huge_shared" : " 27 GB", | 20260507 00:52:36.890 | INFO | "malloc" : " 14 GB", | 20260507 00:52:36.891 | INFO | "pinned" : " 29 GB" | 20260507 00:52:36.891 | INFO | } | 20260507 00:52:36.891 | INFO | Perf report for the last 5.14124 seconds: | 20260507 00:52:36.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:52:36.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:36.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:36.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:36.891 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 00:52:36.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:36.891 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 00:52:36.891 | INFO | PRE_HQ : 59.7%, 1.8, 56.3ms, 0 | 20260507 00:52:36.891 | INFO | BAZWRITER : 62.1%, 3.3, 105.1ms, 0 | 20260507 00:52:36.891 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:52:36.891 | INFO | Uploaded 36GiB to GPU in 2.03035s (17.731GiB/s) | 20260507 00:52:36.891 | INFO | Downloaded 38GiB to GPU in 2.17145s (17.4999GiB/s) | 20260507 00:52:42.034 | INFO | Analyzing chunk frames = [5804544,5805056) | 20260507 00:52:42.034 | INFO | MemInfo: { | 20260507 00:52:42.034 | INFO | "gpu" : " 17 GB", | 20260507 00:52:42.034 | INFO | "huge" : "", | 20260507 00:52:42.034 | INFO | "huge_shared" : " 27 GB", | 20260507 00:52:42.034 | INFO | "malloc" : " 14 GB", | 20260507 00:52:42.034 | INFO | "pinned" : " 29 GB" | 20260507 00:52:42.034 | INFO | } | 20260507 00:52:42.034 | INFO | Perf report for the last 5.14329 seconds: | 20260507 00:52:42.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:52:42.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:42.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:42.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:42.034 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 00:52:42.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:42.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:42.034 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 00:52:42.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:42.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:42.034 | INFO | Uploaded 36GiB to GPU in 2.02025s (17.8196GiB/s) | 20260507 00:52:42.034 | INFO | Downloaded 34GiB to GPU in 1.93856s (17.5388GiB/s) | 20260507 00:52:47.173 | INFO | Analyzing chunk frames = [5805056,5805568) | 20260507 00:52:47.173 | INFO | MemInfo: { | 20260507 00:52:47.173 | INFO | "gpu" : " 17 GB", | 20260507 00:52:47.173 | INFO | "huge" : "", | 20260507 00:52:47.173 | INFO | "huge_shared" : " 27 GB", | 20260507 00:52:47.173 | INFO | "malloc" : " 14 GB", | 20260507 00:52:47.173 | INFO | "pinned" : " 29 GB" | 20260507 00:52:47.173 | INFO | } | 20260507 00:52:47.173 | INFO | Perf report for the last 5.13922 seconds: | 20260507 00:52:47.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:52:47.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:47.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:47.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:47.173 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:52:47.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:47.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:47.173 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:52:47.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:47.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:47.173 | INFO | Uploaded 36GiB to GPU in 1.98365s (18.1484GiB/s) | 20260507 00:52:47.173 | INFO | Downloaded 34GiB to GPU in 1.93951s (17.5302GiB/s) | 20260507 00:52:52.310 | INFO | Analyzing chunk frames = [5805568,5806080) | 20260507 00:52:52.310 | INFO | MemInfo: { | 20260507 00:52:52.310 | INFO | "gpu" : " 17 GB", | 20260507 00:52:52.311 | INFO | "huge" : "", | 20260507 00:52:52.311 | INFO | "huge_shared" : " 27 GB", | 20260507 00:52:52.311 | INFO | "malloc" : " 14 GB", | 20260507 00:52:52.311 | INFO | "pinned" : " 29 GB" | 20260507 00:52:52.311 | INFO | } | 20260507 00:52:52.311 | INFO | Perf report for the last 5.13754 seconds: | 20260507 00:52:52.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:52:52.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:52.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:52.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:52.311 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 00:52:52.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:52.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:52.311 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:52:52.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:52.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:52.311 | INFO | Uploaded 36GiB to GPU in 1.97151s (18.2601GiB/s) | 20260507 00:52:52.311 | INFO | Downloaded 34GiB to GPU in 1.91472s (17.7571GiB/s) | 20260507 00:52:57.444 | INFO | Analyzing chunk frames = [5806080,5806592) | 20260507 00:52:57.444 | INFO | MemInfo: { | 20260507 00:52:57.444 | INFO | "gpu" : " 17 GB", | 20260507 00:52:57.444 | INFO | "huge" : "", | 20260507 00:52:57.444 | INFO | "huge_shared" : " 27 GB", | 20260507 00:52:57.444 | INFO | "malloc" : " 14 GB", | 20260507 00:52:57.444 | INFO | "pinned" : " 29 GB" | 20260507 00:52:57.444 | INFO | } | 20260507 00:52:57.444 | INFO | Perf report for the last 5.13351 seconds: | 20260507 00:52:57.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:52:57.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:57.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:57.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:57.444 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 00:52:57.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:57.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:57.444 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:52:57.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:57.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:52:57.444 | INFO | Uploaded 36GiB to GPU in 1.99291s (18.064GiB/s) | 20260507 00:52:57.444 | INFO | Downloaded 34GiB to GPU in 1.93298s (17.5894GiB/s) | 20260507 00:53:02.582 | INFO | Analyzing chunk frames = [5806592,5807104) | 20260507 00:53:02.582 | INFO | MemInfo: { | 20260507 00:53:02.582 | INFO | "gpu" : " 17 GB", | 20260507 00:53:02.582 | INFO | "huge" : "", | 20260507 00:53:02.582 | INFO | "huge_shared" : " 27 GB", | 20260507 00:53:02.582 | INFO | "malloc" : " 14 GB", | 20260507 00:53:02.582 | INFO | "pinned" : " 29 GB" | 20260507 00:53:02.582 | INFO | } | 20260507 00:53:02.582 | INFO | Perf report for the last 5.138 seconds: | 20260507 00:53:02.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:53:02.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:02.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:02.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:02.582 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:53:02.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:02.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:02.582 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 00:53:02.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:02.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:02.582 | INFO | Uploaded 36GiB to GPU in 2.03364s (17.7023GiB/s) | 20260507 00:53:02.582 | INFO | Downloaded 34GiB to GPU in 1.95567s (17.3853GiB/s) | 20260507 00:53:07.722 | INFO | Analyzing chunk frames = [5807104,5807616) | 20260507 00:53:07.722 | INFO | MemInfo: { | 20260507 00:53:07.722 | INFO | "gpu" : " 17 GB", | 20260507 00:53:07.722 | INFO | "huge" : "", | 20260507 00:53:07.722 | INFO | "huge_shared" : " 27 GB", | 20260507 00:53:07.723 | INFO | "malloc" : " 14 GB", | 20260507 00:53:07.723 | INFO | "pinned" : " 29 GB" | 20260507 00:53:07.723 | INFO | } | 20260507 00:53:07.723 | INFO | Perf report for the last 5.14039 seconds: | 20260507 00:53:07.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:53:07.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:07.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:07.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:07.723 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 00:53:07.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:07.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:07.723 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 00:53:07.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:07.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:07.723 | INFO | Uploaded 36GiB to GPU in 1.97459s (18.2316GiB/s) | 20260507 00:53:07.723 | INFO | Downloaded 34GiB to GPU in 2.00086s (16.9927GiB/s) | 20260507 00:53:12.866 | INFO | Analyzing chunk frames = [5807616,5808128) | 20260507 00:53:12.866 | INFO | MemInfo: { | 20260507 00:53:12.866 | INFO | "gpu" : " 17 GB", | 20260507 00:53:12.866 | INFO | "huge" : "", | 20260507 00:53:12.867 | INFO | "huge_shared" : " 27 GB", | 20260507 00:53:12.867 | INFO | "malloc" : " 14 GB", | 20260507 00:53:12.867 | INFO | "pinned" : " 29 GB" | 20260507 00:53:12.867 | INFO | } | 20260507 00:53:12.867 | INFO | Perf report for the last 5.14399 seconds: | 20260507 00:53:12.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:53:12.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:12.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:12.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:12.867 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260507 00:53:12.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:12.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:12.867 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 00:53:12.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:12.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:12.867 | INFO | Uploaded 36GiB to GPU in 1.99111s (18.0803GiB/s) | 20260507 00:53:12.867 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:53:12.867 | INFO | Total mem usage: 18 GiB | 20260507 00:53:12.867 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:53:12.867 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 00:53:12.867 | INFO | Downloaded 34GiB to GPU in 1.9818s (17.1561GiB/s) | 20260507 00:53:12.867 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:53:12.867 | INFO | High Water Breakdown: | 20260507 00:53:12.867 | INFO | 60.1% : 8.57 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:53:12.867 | INFO | 39.8% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:53:12.867 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:53:12.867 | INFO | ------------------------------------------ | 20260507 00:53:12.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:53:12.867 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:53:12.867 | INFO | Total mem usage: 26.5 GiB | 20260507 00:53:12.867 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:53:12.867 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:53:12.867 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:53:12.867 | INFO | High Water Breakdown: | 20260507 00:53:12.867 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:53:12.867 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:53:12.867 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:53:12.867 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:53:12.867 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:53:12.867 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:53:12.867 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:53:12.867 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:53:12.867 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:53:12.867 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:53:12.867 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:53:12.867 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:53:12.867 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:53:12.867 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:53:12.867 | INFO | ------------------------------------------ | 20260507 00:53:12.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:53:12.867 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:53:12.867 | INFO | Total mem usage: 25.6 GiB | 20260507 00:53:12.867 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:53:12.867 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 00:53:12.867 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 00:53:12.867 | INFO | High Water Breakdown: | 20260507 00:53:12.867 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 00:53:12.867 | INFO | ------------------------------------------ | 20260507 00:53:12.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:53:12.867 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:53:12.867 | INFO | Total mem usage: 15.7 GiB | 20260507 00:53:12.867 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:53:12.867 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:53:12.867 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:53:12.867 | INFO | High Water Breakdown: | 20260507 00:53:12.867 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:53:12.867 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:53:12.867 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:53:12.867 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:53:12.867 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:53:12.867 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:53:12.867 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:53:12.867 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:53:12.867 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:53:12.867 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:53:12.867 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:53:12.867 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:53:12.867 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:53:12.867 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:53:12.868 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:53:12.868 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:53:12.868 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:53:12.868 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:53:12.868 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:53:12.868 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:53:12.868 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:53:12.868 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:53:12.868 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:53:12.868 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:53:12.868 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:53:12.868 | INFO | ------------------------------------------ | 20260507 00:53:12.868 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:53:12.868 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:53:12.868 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:53:12.868 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:53:12.868 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.59e+01, 1.35e+01 / 1.94e+00 | 20260507 00:53:12.868 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.02e-01, 9.49e-03 / 3.96e-03 | 20260507 00:53:12.868 | INFO | PerfReport: 11.68%, Baseline -- 3.61e+00 / 4.46e+00, 5.35e-01 / 2.38e-01 | 20260507 00:53:12.868 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.45e+00, 1.74e-01 / 9.77e-02 | 20260507 00:53:12.868 | INFO | PerfReport: 16.10%, DME -- 3.98e+01 / 4.47e+01, 4.01e+00 / 2.40e+00 | 20260507 00:53:12.868 | INFO | PerfReport: 57.44%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.31e+00 / 1.53e+00 | 20260507 00:53:12.868 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.17e+00 / 2.75e+00, 3.82e-01 / 2.05e-01 | 20260507 00:53:12.868 | INFO | PerfReport: 3.14%, Metrics -- 9.70e-01 / 1.30e+00, 2.58e-01 / 1.64e-01 | 20260507 00:53:12.868 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.30e-01 / 3.83e-01, 5.18e-02 / 2.11e-02 | 20260507 00:53:12.868 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:53:12.868 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:53:12.868 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:53:12.868 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 00:53:12.868 | INFO | PerfReport: 21.55%, Upload -- 2.08e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260507 00:53:12.868 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260507 00:53:12.868 | INFO | PerfReport: 32.03%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 00:53:12.868 | INFO | PerfReport: 1.26%, LockUpload -- 1.21e+00 / 0.00e+00, 4.78e+00 / 0.00e+00 | 20260507 00:53:12.868 | INFO | PerfReport: 1.36%, LockDownload -- 1.31e+00 / 0.00e+00, 3.22e+00 / 0.00e+00 | 20260507 00:53:12.868 | INFO | PerfReport: 22.41%, LockCompute -- 2.16e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 00:53:12.868 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260507 00:53:12.965 | INFO | Starting a round of Baz IO | 20260507 00:53:16.092 | INFO | Populating RTMetrics report | 20260507 00:53:16.092 | INFO | RTMetrics report populated | 20260507 00:53:16.092 | INFO | Opening tmp RTMetrics json file | 20260507 00:53:16.111 | INFO | Serializing RTMetrics to json | 20260507 00:53:16.113 | INFO | Serialization done, swapping tmp file now... | 20260507 00:53:16.113 | INFO | RTMetrics done | 20260507 00:53:16.182 | INFO | Finished a round of Baz IO: Wrote 1.832 GiB in 3.217 seconds | 20260507 00:53:16.182 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.22% / 0.00% / 66.78% | 20260507 00:53:16.182 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.317212% / 3.97364e-06% / 8.34465e-05% | 20260507 00:53:16.183 | INFO | Baz IO overall: | 20260507 00:53:16.183 | INFO | BytesWritten : 3021378529264 | 20260507 00:53:16.183 | INFO | NumEvents : 1454983439156 | 20260507 00:53:18.010 | INFO | Analyzing chunk frames = [5808128,5808640) | 20260507 00:53:18.010 | INFO | MemInfo: { | 20260507 00:53:18.010 | INFO | "gpu" : " 17 GB", | 20260507 00:53:18.010 | INFO | "huge" : "", | 20260507 00:53:18.010 | INFO | "huge_shared" : " 27 GB", | 20260507 00:53:18.010 | INFO | "malloc" : " 14 GB", | 20260507 00:53:18.010 | INFO | "pinned" : " 29 GB" | 20260507 00:53:18.010 | INFO | } | 20260507 00:53:18.010 | INFO | Perf report for the last 5.14337 seconds: | 20260507 00:53:18.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:53:18.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:18.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:18.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:18.010 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260507 00:53:18.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:18.010 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 00:53:18.010 | INFO | PRE_HQ : 59.6%, 1.8, 56.7ms, 0 | 20260507 00:53:18.010 | INFO | BAZWRITER : 62.0%, 3.3, 106.1ms, 0 | 20260507 00:53:18.010 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 00:53:18.011 | INFO | Uploaded 36GiB to GPU in 2.02186s (17.8054GiB/s) | 20260507 00:53:18.011 | INFO | Downloaded 38GiB to GPU in 2.18429s (17.3969GiB/s) | 20260507 00:53:23.146 | INFO | Analyzing chunk frames = [5808640,5809152) | 20260507 00:53:23.146 | INFO | MemInfo: { | 20260507 00:53:23.146 | INFO | "gpu" : " 17 GB", | 20260507 00:53:23.146 | INFO | "huge" : "", | 20260507 00:53:23.146 | INFO | "huge_shared" : " 27 GB", | 20260507 00:53:23.146 | INFO | "malloc" : " 14 GB", | 20260507 00:53:23.146 | INFO | "pinned" : " 29 GB" | 20260507 00:53:23.146 | INFO | } | 20260507 00:53:23.146 | INFO | Perf report for the last 5.13626 seconds: | 20260507 00:53:23.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:53:23.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:23.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:23.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:23.146 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 00:53:23.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:23.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:23.146 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:53:23.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:23.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:23.146 | INFO | Uploaded 36GiB to GPU in 2.0476s (17.5816GiB/s) | 20260507 00:53:23.146 | INFO | Downloaded 34GiB to GPU in 1.93418s (17.5785GiB/s) | 20260507 00:53:28.286 | INFO | Analyzing chunk frames = [5809152,5809664) | 20260507 00:53:28.286 | INFO | MemInfo: { | 20260507 00:53:28.286 | INFO | "gpu" : " 17 GB", | 20260507 00:53:28.286 | INFO | "huge" : "", | 20260507 00:53:28.286 | INFO | "huge_shared" : " 27 GB", | 20260507 00:53:28.286 | INFO | "malloc" : " 14 GB", | 20260507 00:53:28.287 | INFO | "pinned" : " 29 GB" | 20260507 00:53:28.287 | INFO | } | 20260507 00:53:28.287 | INFO | Perf report for the last 5.1403 seconds: | 20260507 00:53:28.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:53:28.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:28.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:28.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:28.287 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260507 00:53:28.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:28.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:28.287 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:53:28.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:28.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:28.287 | INFO | Uploaded 36GiB to GPU in 2.00752s (17.9326GiB/s) | 20260507 00:53:28.287 | INFO | Downloaded 34GiB to GPU in 1.93383s (17.5817GiB/s) | 20260507 00:53:33.424 | INFO | Analyzing chunk frames = [5809664,5810176) | 20260507 00:53:33.424 | INFO | MemInfo: { | 20260507 00:53:33.424 | INFO | "gpu" : " 17 GB", | 20260507 00:53:33.424 | INFO | "huge" : "", | 20260507 00:53:33.424 | INFO | "huge_shared" : " 27 GB", | 20260507 00:53:33.424 | INFO | "malloc" : " 14 GB", | 20260507 00:53:33.424 | INFO | "pinned" : " 29 GB" | 20260507 00:53:33.424 | INFO | } | 20260507 00:53:33.424 | INFO | Perf report for the last 5.13726 seconds: | 20260507 00:53:33.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:53:33.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:33.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:33.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:33.424 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 00:53:33.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:33.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:33.424 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:53:33.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:33.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:33.424 | INFO | Uploaded 36GiB to GPU in 2.01479s (17.8679GiB/s) | 20260507 00:53:33.424 | INFO | Downloaded 34GiB to GPU in 1.90873s (17.8129GiB/s) | 20260507 00:53:38.558 | INFO | Analyzing chunk frames = [5810176,5810688) | 20260507 00:53:38.558 | INFO | MemInfo: { | 20260507 00:53:38.558 | INFO | "gpu" : " 17 GB", | 20260507 00:53:38.558 | INFO | "huge" : "", | 20260507 00:53:38.558 | INFO | "huge_shared" : " 27 GB", | 20260507 00:53:38.558 | INFO | "malloc" : " 14 GB", | 20260507 00:53:38.558 | INFO | "pinned" : " 29 GB" | 20260507 00:53:38.558 | INFO | } | 20260507 00:53:38.558 | INFO | Perf report for the last 5.13435 seconds: | 20260507 00:53:38.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:53:38.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:38.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:38.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:38.558 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260507 00:53:38.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:38.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:38.558 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:53:38.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:38.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:38.558 | INFO | Uploaded 36GiB to GPU in 1.99226s (18.0699GiB/s) | 20260507 00:53:38.558 | INFO | Downloaded 34GiB to GPU in 1.92773s (17.6374GiB/s) | 20260507 00:53:43.704 | INFO | Analyzing chunk frames = [5810688,5811200) | 20260507 00:53:43.704 | INFO | MemInfo: { | 20260507 00:53:43.704 | INFO | "gpu" : " 17 GB", | 20260507 00:53:43.704 | INFO | "huge" : "", | 20260507 00:53:43.704 | INFO | "huge_shared" : " 27 GB", | 20260507 00:53:43.704 | INFO | "malloc" : " 14 GB", | 20260507 00:53:43.704 | INFO | "pinned" : " 29 GB" | 20260507 00:53:43.704 | INFO | } | 20260507 00:53:43.704 | INFO | Perf report for the last 5.14576 seconds: | 20260507 00:53:43.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:53:43.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:43.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:43.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:43.704 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 00:53:43.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:43.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:43.704 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:53:43.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:43.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:43.704 | INFO | Uploaded 36GiB to GPU in 2.0226s (17.7989GiB/s) | 20260507 00:53:43.704 | INFO | Downloaded 34GiB to GPU in 1.93464s (17.5743GiB/s) | 20260507 00:53:48.848 | INFO | Analyzing chunk frames = [5811200,5811712) | 20260507 00:53:48.848 | INFO | MemInfo: { | 20260507 00:53:48.848 | INFO | "gpu" : " 17 GB", | 20260507 00:53:48.848 | INFO | "huge" : "", | 20260507 00:53:48.848 | INFO | "huge_shared" : " 27 GB", | 20260507 00:53:48.848 | INFO | "malloc" : " 14 GB", | 20260507 00:53:48.848 | INFO | "pinned" : " 29 GB" | 20260507 00:53:48.848 | INFO | } | 20260507 00:53:48.848 | INFO | Perf report for the last 5.14375 seconds: | 20260507 00:53:48.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:53:48.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:48.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:48.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:48.848 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 00:53:48.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:48.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:48.848 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:53:48.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:48.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:48.848 | INFO | Uploaded 36GiB to GPU in 1.9734s (18.2427GiB/s) | 20260507 00:53:48.848 | INFO | Downloaded 34GiB to GPU in 1.95003s (17.4357GiB/s) | 20260507 00:53:53.983 | INFO | Analyzing chunk frames = [5811712,5812224) | 20260507 00:53:53.983 | INFO | MemInfo: { | 20260507 00:53:53.983 | INFO | "gpu" : " 17 GB", | 20260507 00:53:53.983 | INFO | "huge" : "", | 20260507 00:53:53.983 | INFO | "huge_shared" : " 27 GB", | 20260507 00:53:53.983 | INFO | "malloc" : " 14 GB", | 20260507 00:53:53.983 | INFO | "pinned" : " 29 GB" | 20260507 00:53:53.983 | INFO | } | 20260507 00:53:53.983 | INFO | Perf report for the last 5.13526 seconds: | 20260507 00:53:53.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:53:53.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:53.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:53.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:53.983 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 00:53:53.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:53.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:53.983 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:53:53.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:53.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:53.983 | INFO | Uploaded 36GiB to GPU in 1.98405s (18.1447GiB/s) | 20260507 00:53:53.983 | INFO | Downloaded 34GiB to GPU in 1.97082s (17.2517GiB/s) | 20260507 00:53:54.081 | INFO | Starting a round of Baz IO | 20260507 00:53:57.198 | INFO | Populating RTMetrics report | 20260507 00:53:57.199 | INFO | RTMetrics report populated | 20260507 00:53:57.199 | INFO | Opening tmp RTMetrics json file | 20260507 00:53:57.199 | INFO | Serializing RTMetrics to json | 20260507 00:53:57.201 | INFO | Serialization done, swapping tmp file now... | 20260507 00:53:57.203 | INFO | RTMetrics done | 20260507 00:53:57.292 | INFO | Finished a round of Baz IO: Wrote 1.83 GiB in 3.211 seconds | 20260507 00:53:57.292 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.22% / 0.00% / 66.78% | 20260507 00:53:57.292 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.315893% / 3.97364e-06% / 6.75519e-05% | 20260507 00:53:57.292 | INFO | Baz IO overall: | 20260507 00:53:57.292 | INFO | BytesWritten : 3023342984423 | 20260507 00:53:57.292 | INFO | NumEvents : 1455915057003 | 20260507 00:53:59.121 | INFO | Analyzing chunk frames = [5812224,5812736) | 20260507 00:53:59.121 | INFO | MemInfo: { | 20260507 00:53:59.121 | INFO | "gpu" : " 17 GB", | 20260507 00:53:59.121 | INFO | "huge" : "", | 20260507 00:53:59.121 | INFO | "huge_shared" : " 27 GB", | 20260507 00:53:59.121 | INFO | "malloc" : " 14 GB", | 20260507 00:53:59.121 | INFO | "pinned" : " 29 GB" | 20260507 00:53:59.121 | INFO | } | 20260507 00:53:59.121 | INFO | Perf report for the last 5.13831 seconds: | 20260507 00:53:59.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:53:59.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:59.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:59.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:59.121 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 00:53:59.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:53:59.121 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 00:53:59.121 | INFO | PRE_HQ : 59.5%, 1.8, 56.0ms, 0 | 20260507 00:53:59.121 | INFO | BAZWRITER : 62.1%, 3.3, 105.6ms, 0 | 20260507 00:53:59.121 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:53:59.122 | INFO | Uploaded 36GiB to GPU in 2.02829s (17.7489GiB/s) | 20260507 00:53:59.122 | INFO | Downloaded 38GiB to GPU in 2.18125s (17.4212GiB/s) | 20260507 00:54:04.258 | INFO | Analyzing chunk frames = [5812736,5813248) | 20260507 00:54:04.258 | INFO | MemInfo: { | 20260507 00:54:04.258 | INFO | "gpu" : " 17 GB", | 20260507 00:54:04.258 | INFO | "huge" : "", | 20260507 00:54:04.258 | INFO | "huge_shared" : " 27 GB", | 20260507 00:54:04.258 | INFO | "malloc" : " 14 GB", | 20260507 00:54:04.258 | INFO | "pinned" : " 29 GB" | 20260507 00:54:04.258 | INFO | } | 20260507 00:54:04.258 | INFO | Perf report for the last 5.1371 seconds: | 20260507 00:54:04.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:54:04.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:04.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:04.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:04.258 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 00:54:04.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:04.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:04.258 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:54:04.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:04.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:04.259 | INFO | Uploaded 36GiB to GPU in 2.00313s (17.9718GiB/s) | 20260507 00:54:04.259 | INFO | Downloaded 34GiB to GPU in 1.92998s (17.6168GiB/s) | 20260507 00:54:09.393 | INFO | Analyzing chunk frames = [5813248,5813760) | 20260507 00:54:09.393 | INFO | MemInfo: { | 20260507 00:54:09.393 | INFO | "gpu" : " 17 GB", | 20260507 00:54:09.393 | INFO | "huge" : "", | 20260507 00:54:09.393 | INFO | "huge_shared" : " 27 GB", | 20260507 00:54:09.393 | INFO | "malloc" : " 14 GB", | 20260507 00:54:09.393 | INFO | "pinned" : " 29 GB" | 20260507 00:54:09.393 | INFO | } | 20260507 00:54:09.393 | INFO | Perf report for the last 5.13491 seconds: | 20260507 00:54:09.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:54:09.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:09.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:09.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:09.393 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:54:09.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:09.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:09.393 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:54:09.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:09.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:09.394 | INFO | Uploaded 36GiB to GPU in 1.99051s (18.0858GiB/s) | 20260507 00:54:09.394 | INFO | Downloaded 34GiB to GPU in 1.93887s (17.536GiB/s) | 20260507 00:54:14.533 | INFO | Analyzing chunk frames = [5813760,5814272) | 20260507 00:54:14.533 | INFO | MemInfo: { | 20260507 00:54:14.533 | INFO | "gpu" : " 17 GB", | 20260507 00:54:14.533 | INFO | "huge" : "", | 20260507 00:54:14.533 | INFO | "huge_shared" : " 27 GB", | 20260507 00:54:14.533 | INFO | "malloc" : " 14 GB", | 20260507 00:54:14.533 | INFO | "pinned" : " 29 GB" | 20260507 00:54:14.533 | INFO | } | 20260507 00:54:14.533 | INFO | Perf report for the last 5.13986 seconds: | 20260507 00:54:14.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:54:14.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:14.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:14.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:14.533 | INFO | ANALYSIS : 59.4%, 3.0, 92.1ms, 0 | 20260507 00:54:14.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:14.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:14.533 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 00:54:14.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:14.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:14.533 | INFO | Uploaded 36GiB to GPU in 1.94382s (18.5202GiB/s) | 20260507 00:54:14.534 | INFO | Downloaded 34GiB to GPU in 1.88277s (18.0585GiB/s) | 20260507 00:54:19.676 | INFO | Analyzing chunk frames = [5814272,5814784) | 20260507 00:54:19.676 | INFO | MemInfo: { | 20260507 00:54:19.676 | INFO | "gpu" : " 17 GB", | 20260507 00:54:19.677 | INFO | "huge" : "", | 20260507 00:54:19.677 | INFO | "huge_shared" : " 27 GB", | 20260507 00:54:19.677 | INFO | "malloc" : " 14 GB", | 20260507 00:54:19.677 | INFO | "pinned" : " 29 GB" | 20260507 00:54:19.677 | INFO | } | 20260507 00:54:19.677 | INFO | Perf report for the last 5.14341 seconds: | 20260507 00:54:19.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:54:19.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:19.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:19.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:19.677 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 00:54:19.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:19.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:19.677 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:54:19.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:19.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:19.677 | INFO | Uploaded 36GiB to GPU in 1.98539s (18.1324GiB/s) | 20260507 00:54:19.677 | INFO | Downloaded 34GiB to GPU in 1.92393s (17.6721GiB/s) | 20260507 00:54:24.821 | INFO | Analyzing chunk frames = [5814784,5815296) | 20260507 00:54:24.821 | INFO | MemInfo: { | 20260507 00:54:24.821 | INFO | "gpu" : " 17 GB", | 20260507 00:54:24.821 | INFO | "huge" : "", | 20260507 00:54:24.821 | INFO | "huge_shared" : " 27 GB", | 20260507 00:54:24.821 | INFO | "malloc" : " 14 GB", | 20260507 00:54:24.821 | INFO | "pinned" : " 29 GB" | 20260507 00:54:24.821 | INFO | } | 20260507 00:54:24.821 | INFO | Perf report for the last 5.14474 seconds: | 20260507 00:54:24.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:54:24.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:24.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:24.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:24.821 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 00:54:24.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:24.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:24.821 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 00:54:24.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:24.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:24.822 | INFO | Uploaded 36GiB to GPU in 2.02718s (17.7586GiB/s) | 20260507 00:54:24.822 | INFO | Downloaded 34GiB to GPU in 1.95225s (17.4158GiB/s) | 20260507 00:54:29.960 | INFO | Analyzing chunk frames = [5815296,5815808) | 20260507 00:54:29.960 | INFO | MemInfo: { | 20260507 00:54:29.960 | INFO | "gpu" : " 17 GB", | 20260507 00:54:29.961 | INFO | "huge" : "", | 20260507 00:54:29.961 | INFO | "huge_shared" : " 27 GB", | 20260507 00:54:29.961 | INFO | "malloc" : " 14 GB", | 20260507 00:54:29.961 | INFO | "pinned" : " 29 GB" | 20260507 00:54:29.961 | INFO | } | 20260507 00:54:29.961 | INFO | Perf report for the last 5.13919 seconds: | 20260507 00:54:29.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:54:29.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:29.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:29.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:29.961 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 00:54:29.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:29.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:29.961 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:54:29.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:29.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:29.961 | INFO | Uploaded 36GiB to GPU in 1.98358s (18.149GiB/s) | 20260507 00:54:29.961 | INFO | Downloaded 34GiB to GPU in 1.98791s (17.1034GiB/s) | 20260507 00:54:35.095 | INFO | Analyzing chunk frames = [5815808,5816320) | 20260507 00:54:35.095 | INFO | MemInfo: { | 20260507 00:54:35.095 | INFO | "gpu" : " 17 GB", | 20260507 00:54:35.095 | INFO | "huge" : "", | 20260507 00:54:35.095 | INFO | "huge_shared" : " 27 GB", | 20260507 00:54:35.095 | INFO | "malloc" : " 14 GB", | 20260507 00:54:35.095 | INFO | "pinned" : " 29 GB" | 20260507 00:54:35.095 | INFO | } | 20260507 00:54:35.095 | INFO | Perf report for the last 5.13437 seconds: | 20260507 00:54:35.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:54:35.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:35.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:35.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:35.095 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:54:35.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:35.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:35.095 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:54:35.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:35.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:35.095 | INFO | Uploaded 36GiB to GPU in 2.00684s (17.9386GiB/s) | 20260507 00:54:35.095 | INFO | Downloaded 34GiB to GPU in 1.97014s (17.2577GiB/s) | 20260507 00:54:35.095 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:54:35.095 | INFO | Total mem usage: 18 GiB | 20260507 00:54:35.095 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:54:35.095 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.7%) | 20260507 00:54:35.095 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:54:35.095 | INFO | High Water Breakdown: | 20260507 00:54:35.095 | INFO | 60.3% : 8.63 GiB ( 75.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:54:35.095 | INFO | 39.6% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:54:35.095 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:54:35.095 | INFO | ------------------------------------------ | 20260507 00:54:35.095 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:54:35.095 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:54:35.095 | INFO | Total mem usage: 26.5 GiB | 20260507 00:54:35.095 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:54:35.095 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:54:35.095 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:54:35.095 | INFO | High Water Breakdown: | 20260507 00:54:35.095 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:54:35.095 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:54:35.095 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:54:35.095 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:54:35.095 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:54:35.095 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:54:35.095 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:54:35.096 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:54:35.096 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:54:35.096 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:54:35.096 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:54:35.096 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:54:35.096 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:54:35.096 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:54:35.096 | INFO | ------------------------------------------ | 20260507 00:54:35.096 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:54:35.096 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:54:35.096 | INFO | Total mem usage: 25.6 GiB | 20260507 00:54:35.096 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:54:35.096 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:54:35.096 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:54:35.096 | INFO | High Water Breakdown: | 20260507 00:54:35.096 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:54:35.096 | INFO | ------------------------------------------ | 20260507 00:54:35.096 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:54:35.096 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:54:35.096 | INFO | Total mem usage: 15.7 GiB | 20260507 00:54:35.096 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:54:35.096 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:54:35.096 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 00:54:35.096 | INFO | High Water Breakdown: | 20260507 00:54:35.096 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:54:35.096 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:54:35.096 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:54:35.096 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:54:35.096 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:54:35.096 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:54:35.096 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:54:35.096 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:54:35.096 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:54:35.096 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:54:35.096 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:54:35.096 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:54:35.096 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:54:35.096 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:54:35.096 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:54:35.096 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:54:35.096 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:54:35.096 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:54:35.096 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:54:35.096 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:54:35.096 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:54:35.096 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:54:35.096 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:54:35.096 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:54:35.096 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:54:35.096 | INFO | ------------------------------------------ | 20260507 00:54:35.096 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:54:35.096 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:54:35.096 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:54:35.096 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:54:35.096 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.53e+01, 1.34e+01 / 1.79e+00 | 20260507 00:54:35.096 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.02e-01, 9.92e-03 / 6.16e-03 | 20260507 00:54:35.096 | INFO | PerfReport: 11.66%, Baseline -- 3.60e+00 / 4.43e+00, 5.33e-01 / 2.74e-01 | 20260507 00:54:35.096 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.46e+00, 1.74e-01 / 1.09e-01 | 20260507 00:54:35.096 | INFO | PerfReport: 16.06%, DME -- 3.97e+01 / 4.43e+01, 3.92e+00 / 1.69e+00 | 20260507 00:54:35.096 | INFO | PerfReport: 57.52%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.36e+00 / 1.84e+00 | 20260507 00:54:35.096 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.17e+00 / 2.67e+00, 3.81e-01 / 2.49e-01 | 20260507 00:54:35.096 | INFO | PerfReport: 3.13%, Metrics -- 9.67e-01 / 1.27e+00, 2.56e-01 / 1.77e-01 | 20260507 00:54:35.096 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.29e-01 / 3.81e-01, 5.24e-02 / 2.39e-02 | 20260507 00:54:35.096 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:54:35.096 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:54:35.096 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:54:35.096 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.32e+01 / 0.00e+00 | 20260507 00:54:35.096 | INFO | PerfReport: 21.62%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260507 00:54:35.096 | INFO | PerfReport: 21.27%, Download -- 2.05e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 00:54:35.096 | INFO | PerfReport: 32.04%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 00:54:35.096 | INFO | PerfReport: 1.36%, LockUpload -- 1.31e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 00:54:35.096 | INFO | PerfReport: 1.25%, LockDownload -- 1.20e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260507 00:54:35.096 | INFO | PerfReport: 22.44%, LockCompute -- 2.16e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 00:54:35.096 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.31e-02 / 0.00e+00 | 20260507 00:54:35.194 | INFO | Starting a round of Baz IO | 20260507 00:54:38.321 | INFO | Populating RTMetrics report | 20260507 00:54:38.321 | INFO | RTMetrics report populated | 20260507 00:54:38.321 | INFO | Opening tmp RTMetrics json file | 20260507 00:54:38.341 | INFO | Serializing RTMetrics to json | 20260507 00:54:38.342 | INFO | Serialization done, swapping tmp file now... | 20260507 00:54:38.342 | INFO | RTMetrics done | 20260507 00:54:38.414 | INFO | Finished a round of Baz IO: Wrote 1.831 GiB in 3.22 seconds | 20260507 00:54:38.414 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.21% / 0.00% / 66.78% | 20260507 00:54:38.414 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.315646% / 0% / 7.94729e-05% | 20260507 00:54:38.414 | INFO | Baz IO overall: | 20260507 00:54:38.414 | INFO | BytesWritten : 3025308518775 | 20260507 00:54:38.414 | INFO | NumEvents : 1456847415932 | 20260507 00:54:40.237 | INFO | Analyzing chunk frames = [5816320,5816832) | 20260507 00:54:40.237 | INFO | MemInfo: { | 20260507 00:54:40.237 | INFO | "gpu" : " 17 GB", | 20260507 00:54:40.237 | INFO | "huge" : "", | 20260507 00:54:40.237 | INFO | "huge_shared" : " 27 GB", | 20260507 00:54:40.237 | INFO | "malloc" : " 14 GB", | 20260507 00:54:40.237 | INFO | "pinned" : " 29 GB" | 20260507 00:54:40.237 | INFO | } | 20260507 00:54:40.237 | INFO | Perf report for the last 5.14168 seconds: | 20260507 00:54:40.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:54:40.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:40.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:40.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:40.237 | INFO | ANALYSIS : 62.2%, 3.0, 96.8ms, 0 | 20260507 00:54:40.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:40.237 | INFO | RT_METRICS : 42.4%, 1.0, 22.3ms, 0 | 20260507 00:54:40.237 | INFO | PRE_HQ : 59.6%, 1.8, 56.2ms, 0 | 20260507 00:54:40.237 | INFO | BAZWRITER : 62.2%, 3.3, 106.2ms, 0 | 20260507 00:54:40.237 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:54:40.237 | INFO | Uploaded 36GiB to GPU in 2.03861s (17.6591GiB/s) | 20260507 00:54:40.237 | INFO | Downloaded 38GiB to GPU in 2.1633s (17.5658GiB/s) | 20260507 00:54:45.373 | INFO | Analyzing chunk frames = [5816832,5817344) | 20260507 00:54:45.373 | INFO | MemInfo: { | 20260507 00:54:45.373 | INFO | "gpu" : " 17 GB", | 20260507 00:54:45.373 | INFO | "huge" : "", | 20260507 00:54:45.373 | INFO | "huge_shared" : " 27 GB", | 20260507 00:54:45.373 | INFO | "malloc" : " 14 GB", | 20260507 00:54:45.373 | INFO | "pinned" : " 29 GB" | 20260507 00:54:45.373 | INFO | } | 20260507 00:54:45.373 | INFO | Perf report for the last 5.13679 seconds: | 20260507 00:54:45.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:54:45.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:45.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:45.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:45.374 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 00:54:45.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:45.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:45.374 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 00:54:45.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:45.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:45.374 | INFO | Uploaded 36GiB to GPU in 1.99626s (18.0337GiB/s) | 20260507 00:54:45.374 | INFO | Downloaded 34GiB to GPU in 1.9221s (17.6889GiB/s) | 20260507 00:54:50.513 | INFO | Analyzing chunk frames = [5817344,5817856) | 20260507 00:54:50.513 | INFO | MemInfo: { | 20260507 00:54:50.513 | INFO | "gpu" : " 17 GB", | 20260507 00:54:50.513 | INFO | "huge" : "", | 20260507 00:54:50.513 | INFO | "huge_shared" : " 27 GB", | 20260507 00:54:50.513 | INFO | "malloc" : " 14 GB", | 20260507 00:54:50.513 | INFO | "pinned" : " 29 GB" | 20260507 00:54:50.513 | INFO | } | 20260507 00:54:50.513 | INFO | Perf report for the last 5.13984 seconds: | 20260507 00:54:50.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:54:50.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:50.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:50.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:50.513 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 00:54:50.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:50.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:50.513 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:54:50.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:50.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:50.513 | INFO | Uploaded 36GiB to GPU in 1.99029s (18.0878GiB/s) | 20260507 00:54:50.514 | INFO | Downloaded 34GiB to GPU in 1.93341s (17.5855GiB/s) | 20260507 00:54:55.657 | INFO | Analyzing chunk frames = [5817856,5818368) | 20260507 00:54:55.657 | INFO | MemInfo: { | 20260507 00:54:55.657 | INFO | "gpu" : " 17 GB", | 20260507 00:54:55.657 | INFO | "huge" : "", | 20260507 00:54:55.657 | INFO | "huge_shared" : " 27 GB", | 20260507 00:54:55.657 | INFO | "malloc" : " 14 GB", | 20260507 00:54:55.657 | INFO | "pinned" : " 29 GB" | 20260507 00:54:55.657 | INFO | } | 20260507 00:54:55.657 | INFO | Perf report for the last 5.14367 seconds: | 20260507 00:54:55.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:54:55.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:55.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:55.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:55.657 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 00:54:55.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:55.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:55.657 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:54:55.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:55.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:54:55.657 | INFO | Uploaded 36GiB to GPU in 2.02005s (17.8213GiB/s) | 20260507 00:54:55.657 | INFO | Downloaded 34GiB to GPU in 1.90976s (17.8032GiB/s) | 20260507 00:55:00.791 | INFO | Analyzing chunk frames = [5818368,5818880) | 20260507 00:55:00.791 | INFO | MemInfo: { | 20260507 00:55:00.791 | INFO | "gpu" : " 17 GB", | 20260507 00:55:00.791 | INFO | "huge" : "", | 20260507 00:55:00.791 | INFO | "huge_shared" : " 27 GB", | 20260507 00:55:00.791 | INFO | "malloc" : " 14 GB", | 20260507 00:55:00.791 | INFO | "pinned" : " 29 GB" | 20260507 00:55:00.791 | INFO | } | 20260507 00:55:00.791 | INFO | Perf report for the last 5.13378 seconds: | 20260507 00:55:00.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:55:00.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:00.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:00.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:00.791 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 00:55:00.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:00.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:00.791 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:55:00.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:00.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:00.791 | INFO | Uploaded 36GiB to GPU in 2.02303s (17.795GiB/s) | 20260507 00:55:00.791 | INFO | Downloaded 34GiB to GPU in 1.9411s (17.5158GiB/s) | 20260507 00:55:05.936 | INFO | Analyzing chunk frames = [5818880,5819392) | 20260507 00:55:05.936 | INFO | MemInfo: { | 20260507 00:55:05.936 | INFO | "gpu" : " 17 GB", | 20260507 00:55:05.936 | INFO | "huge" : "", | 20260507 00:55:05.936 | INFO | "huge_shared" : " 27 GB", | 20260507 00:55:05.936 | INFO | "malloc" : " 14 GB", | 20260507 00:55:05.936 | INFO | "pinned" : " 29 GB" | 20260507 00:55:05.936 | INFO | } | 20260507 00:55:05.936 | INFO | Perf report for the last 5.14525 seconds: | 20260507 00:55:05.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:55:05.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:05.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:05.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:05.936 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:55:05.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:05.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:05.936 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 00:55:05.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:05.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:05.936 | INFO | Uploaded 36GiB to GPU in 2.04542s (17.6003GiB/s) | 20260507 00:55:05.936 | INFO | Downloaded 34GiB to GPU in 1.94321s (17.4968GiB/s) | 20260507 00:55:11.076 | INFO | Analyzing chunk frames = [5819392,5819904) | 20260507 00:55:11.076 | INFO | MemInfo: { | 20260507 00:55:11.076 | INFO | "gpu" : " 17 GB", | 20260507 00:55:11.077 | INFO | "huge" : "", | 20260507 00:55:11.077 | INFO | "huge_shared" : " 27 GB", | 20260507 00:55:11.077 | INFO | "malloc" : " 14 GB", | 20260507 00:55:11.077 | INFO | "pinned" : " 29 GB" | 20260507 00:55:11.077 | INFO | } | 20260507 00:55:11.077 | INFO | Perf report for the last 5.14053 seconds: | 20260507 00:55:11.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:55:11.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:11.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:11.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:11.077 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:55:11.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:11.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:11.077 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 00:55:11.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:11.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:11.077 | INFO | Uploaded 36GiB to GPU in 2.00822s (17.9263GiB/s) | 20260507 00:55:11.077 | INFO | Downloaded 34GiB to GPU in 1.97307s (17.2321GiB/s) | 20260507 00:55:16.212 | INFO | Analyzing chunk frames = [5819904,5820416) | 20260507 00:55:16.212 | INFO | MemInfo: { | 20260507 00:55:16.212 | INFO | "gpu" : " 17 GB", | 20260507 00:55:16.212 | INFO | "huge" : "", | 20260507 00:55:16.212 | INFO | "huge_shared" : " 27 GB", | 20260507 00:55:16.212 | INFO | "malloc" : " 14 GB", | 20260507 00:55:16.212 | INFO | "pinned" : " 29 GB" | 20260507 00:55:16.212 | INFO | } | 20260507 00:55:16.212 | INFO | Perf report for the last 5.13558 seconds: | 20260507 00:55:16.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:55:16.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:16.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:16.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:16.212 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260507 00:55:16.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:16.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:16.212 | INFO | PRE_HQ : 30.1%, 1.0, 15.7ms, 0 | 20260507 00:55:16.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:16.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:16.212 | INFO | Uploaded 36GiB to GPU in 2.00085s (17.9923GiB/s) | 20260507 00:55:16.212 | INFO | Downloaded 34GiB to GPU in 1.97489s (17.2162GiB/s) | 20260507 00:55:16.310 | INFO | Starting a round of Baz IO | 20260507 00:55:19.435 | INFO | Populating RTMetrics report | 20260507 00:55:19.436 | INFO | RTMetrics report populated | 20260507 00:55:19.436 | INFO | Opening tmp RTMetrics json file | 20260507 00:55:19.437 | INFO | Serializing RTMetrics to json | 20260507 00:55:19.455 | INFO | Serialization done, swapping tmp file now... | 20260507 00:55:19.455 | INFO | RTMetrics done | 20260507 00:55:19.527 | INFO | Finished a round of Baz IO: Wrote 1.831 GiB in 3.216 seconds | 20260507 00:55:19.527 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.21% / 0.00% / 66.79% | 20260507 00:55:19.527 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.316203% / 3.97364e-06% / 7.94729e-05% | 20260507 00:55:19.527 | INFO | Baz IO overall: | 20260507 00:55:19.527 | INFO | BytesWritten : 3027274844276 | 20260507 00:55:19.527 | INFO | NumEvents : 1457780471027 | 20260507 00:55:21.347 | INFO | Analyzing chunk frames = [5820416,5820928) | 20260507 00:55:21.347 | INFO | MemInfo: { | 20260507 00:55:21.347 | INFO | "gpu" : " 17 GB", | 20260507 00:55:21.347 | INFO | "huge" : "", | 20260507 00:55:21.347 | INFO | "huge_shared" : " 27 GB", | 20260507 00:55:21.347 | INFO | "malloc" : " 14 GB", | 20260507 00:55:21.347 | INFO | "pinned" : " 29 GB" | 20260507 00:55:21.347 | INFO | } | 20260507 00:55:21.348 | INFO | Perf report for the last 5.13528 seconds: | 20260507 00:55:21.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:55:21.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:21.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:21.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:21.348 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260507 00:55:21.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:21.348 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 00:55:21.348 | INFO | PRE_HQ : 59.7%, 1.8, 56.2ms, 0 | 20260507 00:55:21.348 | INFO | BAZWRITER : 62.1%, 3.3, 106.6ms, 0 | 20260507 00:55:21.348 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:55:21.348 | INFO | Uploaded 36GiB to GPU in 2.04167s (17.6326GiB/s) | 20260507 00:55:21.348 | INFO | Downloaded 38GiB to GPU in 2.18077s (17.425GiB/s) | 20260507 00:55:26.486 | INFO | Analyzing chunk frames = [5820928,5821440) | 20260507 00:55:26.486 | INFO | MemInfo: { | 20260507 00:55:26.486 | INFO | "gpu" : " 17 GB", | 20260507 00:55:26.486 | INFO | "huge" : "", | 20260507 00:55:26.486 | INFO | "huge_shared" : " 27 GB", | 20260507 00:55:26.486 | INFO | "malloc" : " 14 GB", | 20260507 00:55:26.486 | INFO | "pinned" : " 29 GB" | 20260507 00:55:26.486 | INFO | } | 20260507 00:55:26.486 | INFO | Perf report for the last 5.13885 seconds: | 20260507 00:55:26.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:55:26.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:26.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:26.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:26.486 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:55:26.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:26.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:26.486 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 00:55:26.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:26.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:26.487 | INFO | Uploaded 36GiB to GPU in 2.00672s (17.9398GiB/s) | 20260507 00:55:26.487 | INFO | Downloaded 34GiB to GPU in 1.92985s (17.6179GiB/s) | 20260507 00:55:31.634 | INFO | Analyzing chunk frames = [5821440,5821952) | 20260507 00:55:31.634 | INFO | MemInfo: { | 20260507 00:55:31.634 | INFO | "gpu" : " 17 GB", | 20260507 00:55:31.634 | INFO | "huge" : "", | 20260507 00:55:31.634 | INFO | "huge_shared" : " 27 GB", | 20260507 00:55:31.634 | INFO | "malloc" : " 14 GB", | 20260507 00:55:31.634 | INFO | "pinned" : " 29 GB" | 20260507 00:55:31.634 | INFO | } | 20260507 00:55:31.634 | INFO | Perf report for the last 5.14779 seconds: | 20260507 00:55:31.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:55:31.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:31.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:31.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:31.634 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 00:55:31.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:31.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:31.634 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 00:55:31.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:31.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:31.634 | INFO | Uploaded 36GiB to GPU in 1.96876s (18.2856GiB/s) | 20260507 00:55:31.634 | INFO | Downloaded 34GiB to GPU in 1.92552s (17.6576GiB/s) | 20260507 00:55:36.770 | INFO | Analyzing chunk frames = [5821952,5822464) | 20260507 00:55:36.770 | INFO | MemInfo: { | 20260507 00:55:36.770 | INFO | "gpu" : " 17 GB", | 20260507 00:55:36.770 | INFO | "huge" : "", | 20260507 00:55:36.770 | INFO | "huge_shared" : " 27 GB", | 20260507 00:55:36.770 | INFO | "malloc" : " 14 GB", | 20260507 00:55:36.770 | INFO | "pinned" : " 29 GB" | 20260507 00:55:36.770 | INFO | } | 20260507 00:55:36.770 | INFO | Perf report for the last 5.13552 seconds: | 20260507 00:55:36.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:55:36.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:36.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:36.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:36.770 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 00:55:36.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:36.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:36.770 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:55:36.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:36.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:36.770 | INFO | Uploaded 36GiB to GPU in 1.96041s (18.3635GiB/s) | 20260507 00:55:36.770 | INFO | Downloaded 34GiB to GPU in 1.90737s (17.8256GiB/s) | 20260507 00:55:41.906 | INFO | Analyzing chunk frames = [5822464,5822976) | 20260507 00:55:41.906 | INFO | MemInfo: { | 20260507 00:55:41.906 | INFO | "gpu" : " 17 GB", | 20260507 00:55:41.906 | INFO | "huge" : "", | 20260507 00:55:41.906 | INFO | "huge_shared" : " 27 GB", | 20260507 00:55:41.906 | INFO | "malloc" : " 14 GB", | 20260507 00:55:41.906 | INFO | "pinned" : " 29 GB" | 20260507 00:55:41.906 | INFO | } | 20260507 00:55:41.906 | INFO | Perf report for the last 5.13615 seconds: | 20260507 00:55:41.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:55:41.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:41.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:41.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:41.906 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260507 00:55:41.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:41.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:41.906 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:55:41.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:41.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:41.906 | INFO | Uploaded 36GiB to GPU in 2.01563s (17.8604GiB/s) | 20260507 00:55:41.906 | INFO | Downloaded 34GiB to GPU in 1.95079s (17.4288GiB/s) | 20260507 00:55:47.045 | INFO | Analyzing chunk frames = [5822976,5823488) | 20260507 00:55:47.045 | INFO | MemInfo: { | 20260507 00:55:47.045 | INFO | "gpu" : " 17 GB", | 20260507 00:55:47.045 | INFO | "huge" : "", | 20260507 00:55:47.046 | INFO | "huge_shared" : " 27 GB", | 20260507 00:55:47.046 | INFO | "malloc" : " 14 GB", | 20260507 00:55:47.046 | INFO | "pinned" : " 29 GB" | 20260507 00:55:47.046 | INFO | } | 20260507 00:55:47.046 | INFO | Perf report for the last 5.13975 seconds: | 20260507 00:55:47.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:55:47.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:47.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:47.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:47.046 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 00:55:47.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:47.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:47.046 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:55:47.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:47.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:47.046 | INFO | Uploaded 36GiB to GPU in 2.03354s (17.7031GiB/s) | 20260507 00:55:47.046 | INFO | Downloaded 34GiB to GPU in 1.95667s (17.3765GiB/s) | 20260507 00:55:52.186 | INFO | Analyzing chunk frames = [5823488,5824000) | 20260507 00:55:52.186 | INFO | MemInfo: { | 20260507 00:55:52.186 | INFO | "gpu" : " 17 GB", | 20260507 00:55:52.186 | INFO | "huge" : "", | 20260507 00:55:52.186 | INFO | "huge_shared" : " 27 GB", | 20260507 00:55:52.186 | INFO | "malloc" : " 14 GB", | 20260507 00:55:52.186 | INFO | "pinned" : " 29 GB" | 20260507 00:55:52.186 | INFO | } | 20260507 00:55:52.186 | INFO | Perf report for the last 5.14034 seconds: | 20260507 00:55:52.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:55:52.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:52.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:52.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:52.186 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 00:55:52.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:52.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:52.186 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 00:55:52.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:52.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:52.186 | INFO | Uploaded 36GiB to GPU in 1.9506s (18.4559GiB/s) | 20260507 00:55:52.186 | INFO | Downloaded 34GiB to GPU in 1.98866s (17.097GiB/s) | 20260507 00:55:57.329 | INFO | Analyzing chunk frames = [5824000,5824512) | 20260507 00:55:57.329 | INFO | MemInfo: { | 20260507 00:55:57.329 | INFO | "gpu" : " 17 GB", | 20260507 00:55:57.329 | INFO | "huge" : "", | 20260507 00:55:57.329 | INFO | "huge_shared" : " 27 GB", | 20260507 00:55:57.329 | INFO | "malloc" : " 14 GB", | 20260507 00:55:57.329 | INFO | "pinned" : " 29 GB" | 20260507 00:55:57.329 | INFO | } | 20260507 00:55:57.329 | INFO | Perf report for the last 5.14281 seconds: | 20260507 00:55:57.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:55:57.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:57.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:57.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:57.329 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 00:55:57.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:57.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:57.329 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:55:57.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:57.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:55:57.329 | INFO | Uploaded 36GiB to GPU in 1.9954s (18.0415GiB/s) | 20260507 00:55:57.329 | INFO | Downloaded 34GiB to GPU in 1.99075s (17.079GiB/s) | 20260507 00:55:57.329 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:55:57.329 | INFO | Total mem usage: 18 GiB | 20260507 00:55:57.329 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:55:57.329 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 00:55:57.329 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:55:57.329 | INFO | High Water Breakdown: | 20260507 00:55:57.329 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:55:57.329 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:55:57.329 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:55:57.329 | INFO | ------------------------------------------ | 20260507 00:55:57.329 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:55:57.329 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:55:57.329 | INFO | Total mem usage: 26.5 GiB | 20260507 00:55:57.329 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:55:57.329 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:55:57.329 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:55:57.329 | INFO | High Water Breakdown: | 20260507 00:55:57.329 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:55:57.329 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:55:57.329 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:55:57.329 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:55:57.329 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:55:57.329 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:55:57.329 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:55:57.329 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:55:57.329 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:55:57.329 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:55:57.329 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:55:57.329 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:55:57.329 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:55:57.329 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:55:57.329 | INFO | ------------------------------------------ | 20260507 00:55:57.329 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:55:57.329 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:55:57.329 | INFO | Total mem usage: 25.6 GiB | 20260507 00:55:57.329 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:55:57.329 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:55:57.329 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:55:57.329 | INFO | High Water Breakdown: | 20260507 00:55:57.329 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:55:57.329 | INFO | ------------------------------------------ | 20260507 00:55:57.329 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:55:57.330 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:55:57.330 | INFO | Total mem usage: 15.7 GiB | 20260507 00:55:57.330 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:55:57.330 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:55:57.330 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:55:57.330 | INFO | High Water Breakdown: | 20260507 00:55:57.330 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:55:57.330 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:55:57.330 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:55:57.330 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:55:57.330 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:55:57.330 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:55:57.330 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:55:57.330 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:55:57.330 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:55:57.330 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:55:57.330 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:55:57.330 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:55:57.330 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:55:57.330 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:55:57.330 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:55:57.330 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:55:57.330 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:55:57.330 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:55:57.330 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:55:57.330 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:55:57.330 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:55:57.330 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:55:57.330 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:55:57.330 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:55:57.330 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:55:57.330 | INFO | ------------------------------------------ | 20260507 00:55:57.330 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:55:57.330 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:55:57.330 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:55:57.330 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:55:57.330 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.57e+01, 1.34e+01 / 1.91e+00 | 20260507 00:55:57.330 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.76e-03 / 4.79e-03 | 20260507 00:55:57.330 | INFO | PerfReport: 11.68%, Baseline -- 3.61e+00 / 4.42e+00, 5.36e-01 / 3.46e-01 | 20260507 00:55:57.330 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.48e+00, 1.76e-01 / 1.10e-01 | 20260507 00:55:57.330 | INFO | PerfReport: 16.09%, DME -- 3.98e+01 / 4.45e+01, 4.02e+00 / 2.08e+00 | 20260507 00:55:57.330 | INFO | PerfReport: 57.49%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.36e+00 / 1.68e+00 | 20260507 00:55:57.330 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.17e+00 / 2.72e+00, 3.78e-01 / 2.33e-01 | 20260507 00:55:57.330 | INFO | PerfReport: 3.12%, Metrics -- 9.65e-01 / 1.30e+00, 2.53e-01 / 1.62e-01 | 20260507 00:55:57.330 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.30e-01 / 3.82e-01, 5.31e-02 / 2.49e-02 | 20260507 00:55:57.330 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:55:57.330 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:55:57.330 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:55:57.330 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 00:55:57.330 | INFO | PerfReport: 21.64%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260507 00:55:57.330 | INFO | PerfReport: 21.30%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260507 00:55:57.330 | INFO | PerfReport: 32.00%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 00:55:57.330 | INFO | PerfReport: 1.38%, LockUpload -- 1.33e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 00:55:57.330 | INFO | PerfReport: 1.27%, LockDownload -- 1.23e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260507 00:55:57.330 | INFO | PerfReport: 22.40%, LockCompute -- 2.16e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 00:55:57.330 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260507 00:55:57.426 | INFO | Starting a round of Baz IO | 20260507 00:56:00.552 | INFO | Populating RTMetrics report | 20260507 00:56:00.552 | INFO | RTMetrics report populated | 20260507 00:56:00.552 | INFO | Opening tmp RTMetrics json file | 20260507 00:56:00.572 | INFO | Serializing RTMetrics to json | 20260507 00:56:00.573 | INFO | Serialization done, swapping tmp file now... | 20260507 00:56:00.574 | INFO | RTMetrics done | 20260507 00:56:00.643 | INFO | Finished a round of Baz IO: Wrote 1.832 GiB in 3.217 seconds | 20260507 00:56:00.643 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.21% / 0.00% / 66.79% | 20260507 00:56:00.643 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.315356% / 3.97364e-06% / 6.75519e-05% | 20260507 00:56:00.643 | INFO | Baz IO overall: | 20260507 00:56:00.643 | INFO | BytesWritten : 3029241992416 | 20260507 00:56:00.643 | INFO | NumEvents : 1458714096314 | 20260507 00:56:02.464 | INFO | Analyzing chunk frames = [5824512,5825024) | 20260507 00:56:02.464 | INFO | MemInfo: { | 20260507 00:56:02.464 | INFO | "gpu" : " 17 GB", | 20260507 00:56:02.464 | INFO | "huge" : "", | 20260507 00:56:02.464 | INFO | "huge_shared" : " 27 GB", | 20260507 00:56:02.464 | INFO | "malloc" : " 14 GB", | 20260507 00:56:02.464 | INFO | "pinned" : " 29 GB" | 20260507 00:56:02.464 | INFO | } | 20260507 00:56:02.464 | INFO | Perf report for the last 5.13492 seconds: | 20260507 00:56:02.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:56:02.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:02.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:02.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:02.464 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260507 00:56:02.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:02.464 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 00:56:02.464 | INFO | PRE_HQ : 59.7%, 1.8, 56.1ms, 0 | 20260507 00:56:02.464 | INFO | BAZWRITER : 62.2%, 3.3, 106.1ms, 0 | 20260507 00:56:02.464 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:56:02.464 | INFO | Uploaded 36GiB to GPU in 2.03491s (17.6912GiB/s) | 20260507 00:56:02.464 | INFO | Downloaded 38GiB to GPU in 2.17678s (17.457GiB/s) | 20260507 00:56:07.607 | INFO | Analyzing chunk frames = [5825024,5825536) | 20260507 00:56:07.608 | INFO | MemInfo: { | 20260507 00:56:07.608 | INFO | "gpu" : " 17 GB", | 20260507 00:56:07.608 | INFO | "huge" : "", | 20260507 00:56:07.608 | INFO | "huge_shared" : " 27 GB", | 20260507 00:56:07.608 | INFO | "malloc" : " 14 GB", | 20260507 00:56:07.608 | INFO | "pinned" : " 29 GB" | 20260507 00:56:07.608 | INFO | } | 20260507 00:56:07.608 | INFO | Perf report for the last 5.14392 seconds: | 20260507 00:56:07.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:56:07.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:07.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:07.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:07.608 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 00:56:07.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:07.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:07.608 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 00:56:07.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:07.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:07.608 | INFO | Uploaded 36GiB to GPU in 1.99594s (18.0366GiB/s) | 20260507 00:56:07.608 | INFO | Downloaded 34GiB to GPU in 1.92372s (17.6741GiB/s) | 20260507 00:56:12.743 | INFO | Analyzing chunk frames = [5825536,5826048) | 20260507 00:56:12.743 | INFO | MemInfo: { | 20260507 00:56:12.743 | INFO | "gpu" : " 17 GB", | 20260507 00:56:12.743 | INFO | "huge" : "", | 20260507 00:56:12.743 | INFO | "huge_shared" : " 27 GB", | 20260507 00:56:12.743 | INFO | "malloc" : " 14 GB", | 20260507 00:56:12.743 | INFO | "pinned" : " 29 GB" | 20260507 00:56:12.743 | INFO | } | 20260507 00:56:12.743 | INFO | Perf report for the last 5.13569 seconds: | 20260507 00:56:12.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:56:12.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:12.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:12.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:12.743 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:56:12.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:12.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:12.743 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:56:12.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:12.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:12.744 | INFO | Uploaded 36GiB to GPU in 2.00459s (17.9588GiB/s) | 20260507 00:56:12.744 | INFO | Downloaded 34GiB to GPU in 1.93396s (17.5805GiB/s) | 20260507 00:56:17.888 | INFO | Analyzing chunk frames = [5826048,5826560) | 20260507 00:56:17.888 | INFO | MemInfo: { | 20260507 00:56:17.888 | INFO | "gpu" : " 17 GB", | 20260507 00:56:17.888 | INFO | "huge" : "", | 20260507 00:56:17.888 | INFO | "huge_shared" : " 27 GB", | 20260507 00:56:17.888 | INFO | "malloc" : " 14 GB", | 20260507 00:56:17.888 | INFO | "pinned" : " 29 GB" | 20260507 00:56:17.888 | INFO | } | 20260507 00:56:17.888 | INFO | Perf report for the last 5.14507 seconds: | 20260507 00:56:17.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:56:17.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:17.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:17.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:17.889 | INFO | ANALYSIS : 59.9%, 3.0, 93.1ms, 0 | 20260507 00:56:17.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:17.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:17.889 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:56:17.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:17.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:17.889 | INFO | Uploaded 36GiB to GPU in 2.01051s (17.9059GiB/s) | 20260507 00:56:17.889 | INFO | Downloaded 34GiB to GPU in 1.90944s (17.8063GiB/s) | 20260507 00:56:23.024 | INFO | Analyzing chunk frames = [5826560,5827072) | 20260507 00:56:23.024 | INFO | MemInfo: { | 20260507 00:56:23.024 | INFO | "gpu" : " 17 GB", | 20260507 00:56:23.024 | INFO | "huge" : "", | 20260507 00:56:23.024 | INFO | "huge_shared" : " 27 GB", | 20260507 00:56:23.024 | INFO | "malloc" : " 14 GB", | 20260507 00:56:23.024 | INFO | "pinned" : " 29 GB" | 20260507 00:56:23.024 | INFO | } | 20260507 00:56:23.025 | INFO | Perf report for the last 5.13608 seconds: | 20260507 00:56:23.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:56:23.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:23.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:23.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:23.025 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 00:56:23.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:23.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:23.025 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 00:56:23.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:23.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:23.025 | INFO | Uploaded 36GiB to GPU in 2.006s (17.9462GiB/s) | 20260507 00:56:23.025 | INFO | Downloaded 34GiB to GPU in 1.94188s (17.5088GiB/s) | 20260507 00:56:28.159 | INFO | Analyzing chunk frames = [5827072,5827584) | 20260507 00:56:28.159 | INFO | MemInfo: { | 20260507 00:56:28.159 | INFO | "gpu" : " 17 GB", | 20260507 00:56:28.159 | INFO | "huge" : "", | 20260507 00:56:28.159 | INFO | "huge_shared" : " 27 GB", | 20260507 00:56:28.159 | INFO | "malloc" : " 14 GB", | 20260507 00:56:28.159 | INFO | "pinned" : " 29 GB" | 20260507 00:56:28.159 | INFO | } | 20260507 00:56:28.159 | INFO | Perf report for the last 5.13422 seconds: | 20260507 00:56:28.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:56:28.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:28.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:28.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:28.159 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 00:56:28.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:28.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:28.159 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260507 00:56:28.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:28.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:28.159 | INFO | Uploaded 36GiB to GPU in 2.05849s (17.4885GiB/s) | 20260507 00:56:28.159 | INFO | Downloaded 34GiB to GPU in 1.9632s (17.3187GiB/s) | 20260507 00:56:33.300 | INFO | Analyzing chunk frames = [5827584,5828096) | 20260507 00:56:33.300 | INFO | MemInfo: { | 20260507 00:56:33.300 | INFO | "gpu" : " 17 GB", | 20260507 00:56:33.300 | INFO | "huge" : "", | 20260507 00:56:33.300 | INFO | "huge_shared" : " 27 GB", | 20260507 00:56:33.300 | INFO | "malloc" : " 14 GB", | 20260507 00:56:33.300 | INFO | "pinned" : " 29 GB" | 20260507 00:56:33.300 | INFO | } | 20260507 00:56:33.300 | INFO | Perf report for the last 5.14099 seconds: | 20260507 00:56:33.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:56:33.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:33.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:33.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:33.300 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:56:33.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:33.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:33.300 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:56:33.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:33.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:33.300 | INFO | Uploaded 36GiB to GPU in 1.98592s (18.1276GiB/s) | 20260507 00:56:33.300 | INFO | Downloaded 34GiB to GPU in 2.00419s (16.9645GiB/s) | 20260507 00:56:38.439 | INFO | Analyzing chunk frames = [5828096,5828608) | 20260507 00:56:38.440 | INFO | MemInfo: { | 20260507 00:56:38.440 | INFO | "gpu" : " 17 GB", | 20260507 00:56:38.440 | INFO | "huge" : "", | 20260507 00:56:38.440 | INFO | "huge_shared" : " 27 GB", | 20260507 00:56:38.440 | INFO | "malloc" : " 14 GB", | 20260507 00:56:38.440 | INFO | "pinned" : " 29 GB" | 20260507 00:56:38.440 | INFO | } | 20260507 00:56:38.440 | INFO | Perf report for the last 5.13986 seconds: | 20260507 00:56:38.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:56:38.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:38.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:38.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:38.440 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:56:38.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:38.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:38.440 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:56:38.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:38.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:38.440 | INFO | Uploaded 36GiB to GPU in 1.98554s (18.1311GiB/s) | 20260507 00:56:38.440 | INFO | Downloaded 34GiB to GPU in 1.97716s (17.1964GiB/s) | 20260507 00:56:38.537 | INFO | Starting a round of Baz IO | 20260507 00:56:41.659 | INFO | Populating RTMetrics report | 20260507 00:56:41.660 | INFO | RTMetrics report populated | 20260507 00:56:41.660 | INFO | Opening tmp RTMetrics json file | 20260507 00:56:41.664 | INFO | Serializing RTMetrics to json | 20260507 00:56:41.678 | INFO | Serialization done, swapping tmp file now... | 20260507 00:56:41.679 | INFO | RTMetrics done | 20260507 00:56:41.749 | INFO | Finished a round of Baz IO: Wrote 1.833 GiB in 3.211 seconds | 20260507 00:56:41.749 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.21% / 0.00% / 66.79% | 20260507 00:56:41.749 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.316536% / 0% / 5.16574e-05% | 20260507 00:56:41.749 | INFO | Baz IO overall: | 20260507 00:56:41.749 | INFO | BytesWritten : 3031209955627 | 20260507 00:56:41.749 | INFO | NumEvents : 1459648465838 | 20260507 00:56:43.583 | INFO | Analyzing chunk frames = [5828608,5829120) | 20260507 00:56:43.583 | INFO | MemInfo: { | 20260507 00:56:43.583 | INFO | "gpu" : " 17 GB", | 20260507 00:56:43.583 | INFO | "huge" : "", | 20260507 00:56:43.583 | INFO | "huge_shared" : " 27 GB", | 20260507 00:56:43.583 | INFO | "malloc" : " 14 GB", | 20260507 00:56:43.583 | INFO | "pinned" : " 29 GB" | 20260507 00:56:43.584 | INFO | } | 20260507 00:56:43.584 | INFO | Perf report for the last 5.14417 seconds: | 20260507 00:56:43.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:56:43.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:43.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:43.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:43.584 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 00:56:43.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:43.584 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 00:56:43.584 | INFO | PRE_HQ : 59.3%, 1.8, 56.0ms, 0 | 20260507 00:56:43.584 | INFO | BAZWRITER : 62.0%, 3.3, 105.9ms, 0 | 20260507 00:56:43.584 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:56:43.584 | INFO | Uploaded 36GiB to GPU in 2.01294s (17.8842GiB/s) | 20260507 00:56:43.584 | INFO | Downloaded 38GiB to GPU in 2.17485s (17.4724GiB/s) | 20260507 00:56:48.723 | INFO | Analyzing chunk frames = [5829120,5829632) | 20260507 00:56:48.723 | INFO | MemInfo: { | 20260507 00:56:48.723 | INFO | "gpu" : " 17 GB", | 20260507 00:56:48.723 | INFO | "huge" : "", | 20260507 00:56:48.723 | INFO | "huge_shared" : " 27 GB", | 20260507 00:56:48.723 | INFO | "malloc" : " 14 GB", | 20260507 00:56:48.723 | INFO | "pinned" : " 29 GB" | 20260507 00:56:48.723 | INFO | } | 20260507 00:56:48.723 | INFO | Perf report for the last 5.13957 seconds: | 20260507 00:56:48.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:56:48.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:48.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:48.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:48.723 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:56:48.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:48.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:48.724 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:56:48.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:48.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:48.724 | INFO | Uploaded 36GiB to GPU in 2.02337s (17.7921GiB/s) | 20260507 00:56:48.724 | INFO | Downloaded 34GiB to GPU in 1.93374s (17.5825GiB/s) | 20260507 00:56:53.860 | INFO | Analyzing chunk frames = [5829632,5830144) | 20260507 00:56:53.860 | INFO | MemInfo: { | 20260507 00:56:53.860 | INFO | "gpu" : " 17 GB", | 20260507 00:56:53.860 | INFO | "huge" : "", | 20260507 00:56:53.860 | INFO | "huge_shared" : " 27 GB", | 20260507 00:56:53.860 | INFO | "malloc" : " 14 GB", | 20260507 00:56:53.860 | INFO | "pinned" : " 29 GB" | 20260507 00:56:53.860 | INFO | } | 20260507 00:56:53.860 | INFO | Perf report for the last 5.13653 seconds: | 20260507 00:56:53.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:56:53.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:53.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:53.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:53.860 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 00:56:53.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:53.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:53.860 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:56:53.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:53.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:53.860 | INFO | Uploaded 36GiB to GPU in 1.99705s (18.0266GiB/s) | 20260507 00:56:53.860 | INFO | Downloaded 34GiB to GPU in 1.91936s (17.7142GiB/s) | 20260507 00:56:59.000 | INFO | Analyzing chunk frames = [5830144,5830656) | 20260507 00:56:59.000 | INFO | MemInfo: { | 20260507 00:56:59.000 | INFO | "gpu" : " 17 GB", | 20260507 00:56:59.000 | INFO | "huge" : "", | 20260507 00:56:59.000 | INFO | "huge_shared" : " 27 GB", | 20260507 00:56:59.000 | INFO | "malloc" : " 14 GB", | 20260507 00:56:59.000 | INFO | "pinned" : " 29 GB" | 20260507 00:56:59.000 | INFO | } | 20260507 00:56:59.000 | INFO | Perf report for the last 5.13975 seconds: | 20260507 00:56:59.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:56:59.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:59.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:59.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:59.000 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:56:59.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:59.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:59.000 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:56:59.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:59.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:56:59.000 | INFO | Uploaded 36GiB to GPU in 1.98195s (18.1639GiB/s) | 20260507 00:56:59.000 | INFO | Downloaded 34GiB to GPU in 1.9058s (17.8403GiB/s) | 20260507 00:57:04.134 | INFO | Analyzing chunk frames = [5830656,5831168) | 20260507 00:57:04.134 | INFO | MemInfo: { | 20260507 00:57:04.134 | INFO | "gpu" : " 17 GB", | 20260507 00:57:04.134 | INFO | "huge" : "", | 20260507 00:57:04.134 | INFO | "huge_shared" : " 27 GB", | 20260507 00:57:04.134 | INFO | "malloc" : " 14 GB", | 20260507 00:57:04.134 | INFO | "pinned" : " 29 GB" | 20260507 00:57:04.134 | INFO | } | 20260507 00:57:04.134 | INFO | Perf report for the last 5.13467 seconds: | 20260507 00:57:04.134 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:57:04.134 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:04.134 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:04.134 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:04.134 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 00:57:04.134 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:04.134 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:04.134 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:57:04.134 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:04.134 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:04.135 | INFO | Uploaded 36GiB to GPU in 2.00656s (17.9411GiB/s) | 20260507 00:57:04.135 | INFO | Downloaded 34GiB to GPU in 1.94018s (17.5242GiB/s) | 20260507 00:57:09.283 | INFO | Analyzing chunk frames = [5831168,5831680) | 20260507 00:57:09.283 | INFO | MemInfo: { | 20260507 00:57:09.283 | INFO | "gpu" : " 17 GB", | 20260507 00:57:09.283 | INFO | "huge" : "", | 20260507 00:57:09.283 | INFO | "huge_shared" : " 27 GB", | 20260507 00:57:09.283 | INFO | "malloc" : " 14 GB", | 20260507 00:57:09.283 | INFO | "pinned" : " 29 GB" | 20260507 00:57:09.283 | INFO | } | 20260507 00:57:09.283 | INFO | Perf report for the last 5.14893 seconds: | 20260507 00:57:09.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:57:09.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:09.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:09.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:09.283 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 00:57:09.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:09.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:09.283 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 00:57:09.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:09.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:09.284 | INFO | Uploaded 36GiB to GPU in 2.0253s (17.7751GiB/s) | 20260507 00:57:09.284 | INFO | Downloaded 34GiB to GPU in 1.92422s (17.6695GiB/s) | 20260507 00:57:14.419 | INFO | Analyzing chunk frames = [5831680,5832192) | 20260507 00:57:14.419 | INFO | MemInfo: { | 20260507 00:57:14.419 | INFO | "gpu" : " 17 GB", | 20260507 00:57:14.419 | INFO | "huge" : "", | 20260507 00:57:14.419 | INFO | "huge_shared" : " 27 GB", | 20260507 00:57:14.419 | INFO | "malloc" : " 14 GB", | 20260507 00:57:14.419 | INFO | "pinned" : " 29 GB" | 20260507 00:57:14.419 | INFO | } | 20260507 00:57:14.419 | INFO | Perf report for the last 5.13567 seconds: | 20260507 00:57:14.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:57:14.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:14.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:14.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:14.419 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 00:57:14.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:14.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:14.419 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:57:14.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:14.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:14.419 | INFO | Uploaded 36GiB to GPU in 1.99056s (18.0854GiB/s) | 20260507 00:57:14.419 | INFO | Downloaded 34GiB to GPU in 1.9714s (17.2466GiB/s) | 20260507 00:57:19.556 | INFO | Analyzing chunk frames = [5832192,5832704) | 20260507 00:57:19.556 | INFO | MemInfo: { | 20260507 00:57:19.556 | INFO | "gpu" : " 17 GB", | 20260507 00:57:19.556 | INFO | "huge" : "", | 20260507 00:57:19.556 | INFO | "huge_shared" : " 27 GB", | 20260507 00:57:19.556 | INFO | "malloc" : " 14 GB", | 20260507 00:57:19.556 | INFO | "pinned" : " 29 GB" | 20260507 00:57:19.556 | INFO | } | 20260507 00:57:19.556 | INFO | Perf report for the last 5.13713 seconds: | 20260507 00:57:19.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:57:19.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:19.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:19.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:19.556 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:57:19.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:19.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:19.556 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 00:57:19.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:19.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:19.557 | INFO | Uploaded 36GiB to GPU in 1.9933s (18.0605GiB/s) | 20260507 00:57:19.557 | INFO | Downloaded 34GiB to GPU in 1.9836s (17.1406GiB/s) | 20260507 00:57:19.557 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:57:19.557 | INFO | Total mem usage: 18 GiB | 20260507 00:57:19.557 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:57:19.557 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 00:57:19.557 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:57:19.557 | INFO | High Water Breakdown: | 20260507 00:57:19.557 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:57:19.557 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:57:19.557 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:57:19.557 | INFO | ------------------------------------------ | 20260507 00:57:19.557 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:57:19.557 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:57:19.557 | INFO | Total mem usage: 26.5 GiB | 20260507 00:57:19.557 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:57:19.557 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:57:19.557 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:57:19.557 | INFO | High Water Breakdown: | 20260507 00:57:19.557 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:57:19.557 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:57:19.557 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:57:19.557 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:57:19.557 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:57:19.557 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:57:19.557 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:57:19.557 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:57:19.557 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:57:19.557 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:57:19.557 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:57:19.557 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:57:19.557 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:57:19.557 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:57:19.557 | INFO | ------------------------------------------ | 20260507 00:57:19.557 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:57:19.557 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:57:19.557 | INFO | Total mem usage: 25.6 GiB | 20260507 00:57:19.557 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:57:19.557 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:57:19.557 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:57:19.557 | INFO | High Water Breakdown: | 20260507 00:57:19.557 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:57:19.557 | INFO | ------------------------------------------ | 20260507 00:57:19.557 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:57:19.557 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:57:19.557 | INFO | Total mem usage: 15.7 GiB | 20260507 00:57:19.557 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:57:19.557 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:57:19.557 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:57:19.557 | INFO | High Water Breakdown: | 20260507 00:57:19.557 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:57:19.557 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:57:19.557 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:57:19.557 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:57:19.557 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:57:19.557 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:57:19.557 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:57:19.557 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:57:19.557 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:57:19.557 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:57:19.557 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:57:19.557 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:57:19.557 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:57:19.557 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:57:19.557 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:57:19.557 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:57:19.557 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:57:19.557 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:57:19.557 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:57:19.558 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:57:19.558 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:57:19.558 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:57:19.558 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:57:19.558 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:57:19.558 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:57:19.558 | INFO | ------------------------------------------ | 20260507 00:57:19.558 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:57:19.558 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:57:19.558 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:57:19.558 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:57:19.558 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.62e+01, 1.35e+01 / 2.05e+00 | 20260507 00:57:19.558 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.01e-01, 1.01e-02 / 4.30e-03 | 20260507 00:57:19.558 | INFO | PerfReport: 11.64%, Baseline -- 3.60e+00 / 4.49e+00, 5.27e-01 / 2.40e-01 | 20260507 00:57:19.558 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.50e+00, 1.73e-01 / 9.70e-02 | 20260507 00:57:19.558 | INFO | PerfReport: 16.16%, DME -- 3.99e+01 / 4.55e+01, 4.32e+00 / 2.39e+00 | 20260507 00:57:19.558 | INFO | PerfReport: 57.43%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.28e+00 / 1.66e+00 | 20260507 00:57:19.558 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.17e+00 / 2.74e+00, 3.82e-01 / 2.02e-01 | 20260507 00:57:19.558 | INFO | PerfReport: 3.13%, Metrics -- 9.67e-01 / 1.30e+00, 2.58e-01 / 1.41e-01 | 20260507 00:57:19.558 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.29e-01 / 3.88e-01, 5.17e-02 / 2.60e-02 | 20260507 00:57:19.558 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:57:19.558 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:57:19.558 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:57:19.558 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 00:57:19.558 | INFO | PerfReport: 21.64%, Upload -- 2.09e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 00:57:19.558 | INFO | PerfReport: 21.29%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 00:57:19.558 | INFO | PerfReport: 31.98%, Compute -- 3.09e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260507 00:57:19.558 | INFO | PerfReport: 1.48%, LockUpload -- 1.43e+00 / 0.00e+00, 5.04e+00 / 0.00e+00 | 20260507 00:57:19.558 | INFO | PerfReport: 1.24%, LockDownload -- 1.20e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260507 00:57:19.558 | INFO | PerfReport: 22.35%, LockCompute -- 2.16e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 00:57:19.558 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.34e-02 / 0.00e+00 | 20260507 00:57:19.654 | INFO | Starting a round of Baz IO | 20260507 00:57:22.777 | INFO | Populating RTMetrics report | 20260507 00:57:22.777 | INFO | RTMetrics report populated | 20260507 00:57:22.777 | INFO | Opening tmp RTMetrics json file | 20260507 00:57:22.796 | INFO | Serializing RTMetrics to json | 20260507 00:57:22.798 | INFO | Serialization done, swapping tmp file now... | 20260507 00:57:22.798 | INFO | RTMetrics done | 20260507 00:57:22.870 | INFO | Finished a round of Baz IO: Wrote 1.831 GiB in 3.216 seconds | 20260507 00:57:22.870 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.21% / 0.00% / 66.79% | 20260507 00:57:22.870 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.313938% / 0% / 7.54992e-05% | 20260507 00:57:22.870 | INFO | Baz IO overall: | 20260507 00:57:22.870 | INFO | BytesWritten : 3033175577196 | 20260507 00:57:22.870 | INFO | NumEvents : 1460581085562 | 20260507 00:57:24.697 | INFO | Analyzing chunk frames = [5832704,5833216) | 20260507 00:57:24.697 | INFO | MemInfo: { | 20260507 00:57:24.697 | INFO | "gpu" : " 17 GB", | 20260507 00:57:24.697 | INFO | "huge" : "", | 20260507 00:57:24.697 | INFO | "huge_shared" : " 27 GB", | 20260507 00:57:24.697 | INFO | "malloc" : " 14 GB", | 20260507 00:57:24.697 | INFO | "pinned" : " 29 GB" | 20260507 00:57:24.697 | INFO | } | 20260507 00:57:24.697 | INFO | Perf report for the last 5.14127 seconds: | 20260507 00:57:24.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:57:24.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:24.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:24.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:24.698 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 00:57:24.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:24.698 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 00:57:24.698 | INFO | PRE_HQ : 59.6%, 1.8, 56.1ms, 0 | 20260507 00:57:24.698 | INFO | BAZWRITER : 62.2%, 3.3, 106.2ms, 0 | 20260507 00:57:24.698 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:57:24.698 | INFO | Uploaded 36GiB to GPU in 2.01108s (17.9008GiB/s) | 20260507 00:57:24.698 | INFO | Downloaded 38GiB to GPU in 2.16349s (17.5642GiB/s) | 20260507 00:57:29.833 | INFO | Analyzing chunk frames = [5833216,5833728) | 20260507 00:57:29.833 | INFO | MemInfo: { | 20260507 00:57:29.833 | INFO | "gpu" : " 17 GB", | 20260507 00:57:29.833 | INFO | "huge" : "", | 20260507 00:57:29.833 | INFO | "huge_shared" : " 27 GB", | 20260507 00:57:29.833 | INFO | "malloc" : " 14 GB", | 20260507 00:57:29.833 | INFO | "pinned" : " 29 GB" | 20260507 00:57:29.833 | INFO | } | 20260507 00:57:29.833 | INFO | Perf report for the last 5.13562 seconds: | 20260507 00:57:29.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:57:29.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:29.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:29.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:29.833 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:57:29.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:29.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:29.833 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:57:29.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:29.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:29.833 | INFO | Uploaded 36GiB to GPU in 2.00807s (17.9277GiB/s) | 20260507 00:57:29.833 | INFO | Downloaded 34GiB to GPU in 1.93617s (17.5604GiB/s) | 20260507 00:57:34.973 | INFO | Analyzing chunk frames = [5833728,5834240) | 20260507 00:57:34.973 | INFO | MemInfo: { | 20260507 00:57:34.973 | INFO | "gpu" : " 17 GB", | 20260507 00:57:34.973 | INFO | "huge" : "", | 20260507 00:57:34.973 | INFO | "huge_shared" : " 27 GB", | 20260507 00:57:34.973 | INFO | "malloc" : " 14 GB", | 20260507 00:57:34.973 | INFO | "pinned" : " 29 GB" | 20260507 00:57:34.973 | INFO | } | 20260507 00:57:34.973 | INFO | Perf report for the last 5.14 seconds: | 20260507 00:57:34.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:57:34.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:34.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:34.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:34.973 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:57:34.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:34.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:34.973 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:57:34.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:34.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:34.973 | INFO | Uploaded 36GiB to GPU in 1.99147s (18.0771GiB/s) | 20260507 00:57:34.973 | INFO | Downloaded 34GiB to GPU in 1.93851s (17.5392GiB/s) | 20260507 00:57:40.115 | INFO | Analyzing chunk frames = [5834240,5834752) | 20260507 00:57:40.115 | INFO | MemInfo: { | 20260507 00:57:40.115 | INFO | "gpu" : " 17 GB", | 20260507 00:57:40.115 | INFO | "huge" : "", | 20260507 00:57:40.115 | INFO | "huge_shared" : " 27 GB", | 20260507 00:57:40.115 | INFO | "malloc" : " 14 GB", | 20260507 00:57:40.115 | INFO | "pinned" : " 29 GB" | 20260507 00:57:40.115 | INFO | } | 20260507 00:57:40.115 | INFO | Perf report for the last 5.14179 seconds: | 20260507 00:57:40.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:57:40.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:40.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:40.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:40.115 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 00:57:40.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:40.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:40.115 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:57:40.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:40.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:40.115 | INFO | Uploaded 36GiB to GPU in 1.99959s (18.0037GiB/s) | 20260507 00:57:40.115 | INFO | Downloaded 34GiB to GPU in 1.9097s (17.8039GiB/s) | 20260507 00:57:45.248 | INFO | Analyzing chunk frames = [5834752,5835264) | 20260507 00:57:45.248 | INFO | MemInfo: { | 20260507 00:57:45.248 | INFO | "gpu" : " 17 GB", | 20260507 00:57:45.248 | INFO | "huge" : "", | 20260507 00:57:45.248 | INFO | "huge_shared" : " 27 GB", | 20260507 00:57:45.248 | INFO | "malloc" : " 14 GB", | 20260507 00:57:45.248 | INFO | "pinned" : " 29 GB" | 20260507 00:57:45.248 | INFO | } | 20260507 00:57:45.248 | INFO | Perf report for the last 5.13337 seconds: | 20260507 00:57:45.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:57:45.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:45.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:45.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:45.248 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 00:57:45.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:45.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:45.248 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:57:45.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:45.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:45.249 | INFO | Uploaded 36GiB to GPU in 1.97912s (18.1899GiB/s) | 20260507 00:57:45.249 | INFO | Downloaded 34GiB to GPU in 1.9383s (17.5411GiB/s) | 20260507 00:57:50.396 | INFO | Analyzing chunk frames = [5835264,5835776) | 20260507 00:57:50.396 | INFO | MemInfo: { | 20260507 00:57:50.396 | INFO | "gpu" : " 17 GB", | 20260507 00:57:50.397 | INFO | "huge" : "", | 20260507 00:57:50.397 | INFO | "huge_shared" : " 27 GB", | 20260507 00:57:50.397 | INFO | "malloc" : " 14 GB", | 20260507 00:57:50.397 | INFO | "pinned" : " 29 GB" | 20260507 00:57:50.397 | INFO | } | 20260507 00:57:50.397 | INFO | Perf report for the last 5.14841 seconds: | 20260507 00:57:50.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:57:50.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:50.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:50.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:50.397 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 00:57:50.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:50.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:50.397 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 00:57:50.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:50.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:50.397 | INFO | Uploaded 36GiB to GPU in 1.99627s (18.0336GiB/s) | 20260507 00:57:50.397 | INFO | Downloaded 34GiB to GPU in 1.94126s (17.5144GiB/s) | 20260507 00:57:55.536 | INFO | Analyzing chunk frames = [5835776,5836288) | 20260507 00:57:55.536 | INFO | MemInfo: { | 20260507 00:57:55.536 | INFO | "gpu" : " 17 GB", | 20260507 00:57:55.536 | INFO | "huge" : "", | 20260507 00:57:55.536 | INFO | "huge_shared" : " 27 GB", | 20260507 00:57:55.536 | INFO | "malloc" : " 14 GB", | 20260507 00:57:55.536 | INFO | "pinned" : " 29 GB" | 20260507 00:57:55.536 | INFO | } | 20260507 00:57:55.536 | INFO | Perf report for the last 5.13894 seconds: | 20260507 00:57:55.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:57:55.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:55.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:55.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:55.536 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:57:55.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:55.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:55.536 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:57:55.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:55.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:57:55.536 | INFO | Uploaded 36GiB to GPU in 1.98642s (18.123GiB/s) | 20260507 00:57:55.536 | INFO | Downloaded 34GiB to GPU in 1.98326s (17.1435GiB/s) | 20260507 00:58:00.668 | INFO | Analyzing chunk frames = [5836288,5836800) | 20260507 00:58:00.668 | INFO | MemInfo: { | 20260507 00:58:00.668 | INFO | "gpu" : " 17 GB", | 20260507 00:58:00.668 | INFO | "huge" : "", | 20260507 00:58:00.668 | INFO | "huge_shared" : " 27 GB", | 20260507 00:58:00.668 | INFO | "malloc" : " 14 GB", | 20260507 00:58:00.668 | INFO | "pinned" : " 29 GB" | 20260507 00:58:00.668 | INFO | } | 20260507 00:58:00.668 | INFO | Perf report for the last 5.13236 seconds: | 20260507 00:58:00.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:58:00.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:00.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:00.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:00.668 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260507 00:58:00.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:00.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:00.668 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 00:58:00.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:00.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:00.668 | INFO | Uploaded 36GiB to GPU in 1.99717s (18.0255GiB/s) | 20260507 00:58:00.668 | INFO | Downloaded 34GiB to GPU in 1.98215s (17.1531GiB/s) | 20260507 00:58:00.766 | INFO | Starting a round of Baz IO | 20260507 00:58:03.882 | INFO | Populating RTMetrics report | 20260507 00:58:03.882 | INFO | RTMetrics report populated | 20260507 00:58:03.882 | INFO | Opening tmp RTMetrics json file | 20260507 00:58:03.883 | INFO | Serializing RTMetrics to json | 20260507 00:58:03.889 | INFO | Serialization done, swapping tmp file now... | 20260507 00:58:03.902 | INFO | RTMetrics done | 20260507 00:58:03.974 | INFO | Finished a round of Baz IO: Wrote 1.83 GiB in 3.207 seconds | 20260507 00:58:03.974 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.21% / 0.00% / 66.79% | 20260507 00:58:03.974 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.313695% / 0% / 9.13938e-05% | 20260507 00:58:03.974 | INFO | Baz IO overall: | 20260507 00:58:03.974 | INFO | BytesWritten : 3035140100291 | 20260507 00:58:03.974 | INFO | NumEvents : 1461512953073 | 20260507 00:58:05.804 | INFO | Analyzing chunk frames = [5836800,5837312) | 20260507 00:58:05.804 | INFO | MemInfo: { | 20260507 00:58:05.804 | INFO | "gpu" : " 17 GB", | 20260507 00:58:05.804 | INFO | "huge" : "", | 20260507 00:58:05.804 | INFO | "huge_shared" : " 27 GB", | 20260507 00:58:05.804 | INFO | "malloc" : " 14 GB", | 20260507 00:58:05.804 | INFO | "pinned" : " 29 GB" | 20260507 00:58:05.804 | INFO | } | 20260507 00:58:05.804 | INFO | Perf report for the last 5.13596 seconds: | 20260507 00:58:05.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:58:05.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:05.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:05.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:05.804 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 00:58:05.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:05.804 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 00:58:05.804 | INFO | PRE_HQ : 59.6%, 1.8, 56.5ms, 0 | 20260507 00:58:05.804 | INFO | BAZWRITER : 62.0%, 3.3, 106.7ms, 0 | 20260507 00:58:05.804 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:58:05.804 | INFO | Uploaded 36GiB to GPU in 2.0208s (17.8147GiB/s) | 20260507 00:58:05.804 | INFO | Downloaded 38GiB to GPU in 2.18735s (17.3726GiB/s) | 20260507 00:58:10.946 | INFO | Analyzing chunk frames = [5837312,5837824) | 20260507 00:58:10.946 | INFO | MemInfo: { | 20260507 00:58:10.946 | INFO | "gpu" : " 17 GB", | 20260507 00:58:10.946 | INFO | "huge" : "", | 20260507 00:58:10.946 | INFO | "huge_shared" : " 27 GB", | 20260507 00:58:10.946 | INFO | "malloc" : " 14 GB", | 20260507 00:58:10.946 | INFO | "pinned" : " 29 GB" | 20260507 00:58:10.946 | INFO | } | 20260507 00:58:10.946 | INFO | Perf report for the last 5.14198 seconds: | 20260507 00:58:10.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:58:10.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:10.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:10.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:10.946 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:58:10.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:10.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:10.946 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:58:10.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:10.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:10.946 | INFO | Uploaded 36GiB to GPU in 2.00111s (17.99GiB/s) | 20260507 00:58:10.947 | INFO | Downloaded 34GiB to GPU in 1.93443s (17.5762GiB/s) | 20260507 00:58:16.094 | INFO | Analyzing chunk frames = [5837824,5838336) | 20260507 00:58:16.094 | INFO | MemInfo: { | 20260507 00:58:16.094 | INFO | "gpu" : " 17 GB", | 20260507 00:58:16.094 | INFO | "huge" : "", | 20260507 00:58:16.094 | INFO | "huge_shared" : " 27 GB", | 20260507 00:58:16.094 | INFO | "malloc" : " 14 GB", | 20260507 00:58:16.094 | INFO | "pinned" : " 29 GB" | 20260507 00:58:16.094 | INFO | } | 20260507 00:58:16.094 | INFO | Perf report for the last 5.14773 seconds: | 20260507 00:58:16.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:58:16.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:16.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:16.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:16.094 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 00:58:16.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:16.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:16.094 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:58:16.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:16.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:16.094 | INFO | Uploaded 36GiB to GPU in 1.98276s (18.1565GiB/s) | 20260507 00:58:16.094 | INFO | Downloaded 34GiB to GPU in 1.93579s (17.5639GiB/s) | 20260507 00:58:21.233 | INFO | Analyzing chunk frames = [5838336,5838848) | 20260507 00:58:21.233 | INFO | MemInfo: { | 20260507 00:58:21.233 | INFO | "gpu" : " 17 GB", | 20260507 00:58:21.233 | INFO | "huge" : "", | 20260507 00:58:21.233 | INFO | "huge_shared" : " 27 GB", | 20260507 00:58:21.233 | INFO | "malloc" : " 14 GB", | 20260507 00:58:21.233 | INFO | "pinned" : " 29 GB" | 20260507 00:58:21.233 | INFO | } | 20260507 00:58:21.233 | INFO | Perf report for the last 5.13908 seconds: | 20260507 00:58:21.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:58:21.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:21.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:21.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:21.233 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 00:58:21.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:21.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:21.233 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:58:21.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:21.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:21.233 | INFO | Uploaded 36GiB to GPU in 1.95703s (18.3952GiB/s) | 20260507 00:58:21.233 | INFO | Downloaded 34GiB to GPU in 1.91522s (17.7525GiB/s) | 20260507 00:58:26.372 | INFO | Analyzing chunk frames = [5838848,5839360) | 20260507 00:58:26.372 | INFO | MemInfo: { | 20260507 00:58:26.372 | INFO | "gpu" : " 17 GB", | 20260507 00:58:26.372 | INFO | "huge" : "", | 20260507 00:58:26.372 | INFO | "huge_shared" : " 27 GB", | 20260507 00:58:26.372 | INFO | "malloc" : " 14 GB", | 20260507 00:58:26.372 | INFO | "pinned" : " 29 GB" | 20260507 00:58:26.372 | INFO | } | 20260507 00:58:26.372 | INFO | Perf report for the last 5.13937 seconds: | 20260507 00:58:26.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:58:26.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:26.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:26.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:26.372 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 00:58:26.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:26.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:26.372 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:58:26.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:26.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:26.373 | INFO | Uploaded 36GiB to GPU in 2.00123s (17.989GiB/s) | 20260507 00:58:26.373 | INFO | Downloaded 34GiB to GPU in 1.94467s (17.4837GiB/s) | 20260507 00:58:31.507 | INFO | Analyzing chunk frames = [5839360,5839872) | 20260507 00:58:31.507 | INFO | MemInfo: { | 20260507 00:58:31.507 | INFO | "gpu" : " 17 GB", | 20260507 00:58:31.507 | INFO | "huge" : "", | 20260507 00:58:31.507 | INFO | "huge_shared" : " 27 GB", | 20260507 00:58:31.507 | INFO | "malloc" : " 14 GB", | 20260507 00:58:31.507 | INFO | "pinned" : " 29 GB" | 20260507 00:58:31.507 | INFO | } | 20260507 00:58:31.507 | INFO | Perf report for the last 5.13449 seconds: | 20260507 00:58:31.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:58:31.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:31.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:31.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:31.507 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 00:58:31.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:31.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:31.507 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:58:31.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:31.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:31.507 | INFO | Uploaded 36GiB to GPU in 2.01422s (17.8729GiB/s) | 20260507 00:58:31.507 | INFO | Downloaded 34GiB to GPU in 1.942s (17.5077GiB/s) | 20260507 00:58:36.646 | INFO | Analyzing chunk frames = [5839872,5840384) | 20260507 00:58:36.646 | INFO | MemInfo: { | 20260507 00:58:36.646 | INFO | "gpu" : " 17 GB", | 20260507 00:58:36.646 | INFO | "huge" : "", | 20260507 00:58:36.646 | INFO | "huge_shared" : " 27 GB", | 20260507 00:58:36.647 | INFO | "malloc" : " 14 GB", | 20260507 00:58:36.647 | INFO | "pinned" : " 29 GB" | 20260507 00:58:36.647 | INFO | } | 20260507 00:58:36.647 | INFO | Perf report for the last 5.13992 seconds: | 20260507 00:58:36.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:58:36.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:36.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:36.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:36.647 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:58:36.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:36.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:36.647 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 00:58:36.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:36.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:36.647 | INFO | Uploaded 36GiB to GPU in 1.95591s (18.4057GiB/s) | 20260507 00:58:36.647 | INFO | Downloaded 34GiB to GPU in 1.99081s (17.0785GiB/s) | 20260507 00:58:41.784 | INFO | Analyzing chunk frames = [5840384,5840896) | 20260507 00:58:41.784 | INFO | MemInfo: { | 20260507 00:58:41.784 | INFO | "gpu" : " 17 GB", | 20260507 00:58:41.784 | INFO | "huge" : "", | 20260507 00:58:41.784 | INFO | "huge_shared" : " 27 GB", | 20260507 00:58:41.784 | INFO | "malloc" : " 14 GB", | 20260507 00:58:41.784 | INFO | "pinned" : " 29 GB" | 20260507 00:58:41.784 | INFO | } | 20260507 00:58:41.784 | INFO | Perf report for the last 5.13701 seconds: | 20260507 00:58:41.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:58:41.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:41.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:41.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:41.784 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:58:41.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:41.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:41.784 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 00:58:41.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:41.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:41.784 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 00:58:41.784 | INFO | Total mem usage: 18 GiB | 20260507 00:58:41.784 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 00:58:41.784 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 00:58:41.784 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 00:58:41.784 | INFO | High Water Breakdown: | 20260507 00:58:41.784 | INFO | 60.1% : 8.58 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 00:58:41.784 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 00:58:41.784 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 00:58:41.784 | INFO | ------------------------------------------ | 20260507 00:58:41.784 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:58:41.784 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 00:58:41.784 | INFO | Total mem usage: 26.5 GiB | 20260507 00:58:41.784 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 00:58:41.784 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 00:58:41.784 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 00:58:41.784 | INFO | High Water Breakdown: | 20260507 00:58:41.784 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:58:41.784 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:58:41.784 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:58:41.784 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:58:41.784 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:58:41.784 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:58:41.784 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 00:58:41.784 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 00:58:41.784 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:58:41.784 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 00:58:41.784 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 00:58:41.784 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 00:58:41.784 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 00:58:41.784 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 00:58:41.784 | INFO | ------------------------------------------ | 20260507 00:58:41.784 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:58:41.784 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 00:58:41.784 | INFO | Uploaded 36GiB to GPU in 1.98677s (18.1199GiB/s) | 20260507 00:58:41.785 | INFO | Total mem usage: 25.6 GiB | 20260507 00:58:41.785 | INFO | Downloaded 34GiB to GPU in 1.99238s (17.065GiB/s) | 20260507 00:58:41.785 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 00:58:41.785 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 00:58:41.785 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 00:58:41.785 | INFO | High Water Breakdown: | 20260507 00:58:41.785 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 00:58:41.785 | INFO | ------------------------------------------ | 20260507 00:58:41.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:58:41.785 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 00:58:41.785 | INFO | Total mem usage: 15.7 GiB | 20260507 00:58:41.785 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 00:58:41.785 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 00:58:41.785 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 00:58:41.785 | INFO | High Water Breakdown: | 20260507 00:58:41.785 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 00:58:41.785 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 00:58:41.785 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 00:58:41.785 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 00:58:41.785 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 00:58:41.785 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 00:58:41.785 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 00:58:41.785 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 00:58:41.785 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 00:58:41.785 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 00:58:41.785 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 00:58:41.785 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 00:58:41.785 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 00:58:41.785 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 00:58:41.785 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 00:58:41.785 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 00:58:41.785 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 00:58:41.785 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 00:58:41.785 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 00:58:41.785 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 00:58:41.785 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 00:58:41.785 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 00:58:41.785 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 00:58:41.785 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 00:58:41.785 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 00:58:41.785 | INFO | ------------------------------------------ | 20260507 00:58:41.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 00:58:41.785 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 00:58:41.785 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:58:41.785 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:58:41.785 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.55e+01, 1.33e+01 / 2.21e+00 | 20260507 00:58:41.785 | INFO | PerfReport: 0.30%, AggregateStats -- 9.13e-02 / 1.02e-01, 1.02e-02 / 6.26e-03 | 20260507 00:58:41.785 | INFO | PerfReport: 11.67%, Baseline -- 3.61e+00 / 4.40e+00, 5.35e-01 / 3.31e-01 | 20260507 00:58:41.785 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.46e+00, 1.76e-01 / 1.31e-01 | 20260507 00:58:41.785 | INFO | PerfReport: 16.10%, DME -- 3.98e+01 / 4.44e+01, 4.11e+00 / 2.50e+00 | 20260507 00:58:41.785 | INFO | PerfReport: 57.44%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.31e+00 / 1.87e+00 | 20260507 00:58:41.785 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.18e+00 / 2.71e+00, 3.82e-01 / 2.41e-01 | 20260507 00:58:41.785 | INFO | PerfReport: 3.14%, Metrics -- 9.69e-01 / 1.29e+00, 2.59e-01 / 1.26e-01 | 20260507 00:58:41.785 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.30e-01 / 3.84e-01, 5.27e-02 / 2.74e-02 | 20260507 00:58:41.785 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 00:58:41.785 | INFO | PerfReport: Gathered 1536 samples. | 20260507 00:58:41.785 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 00:58:41.785 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 00:58:41.785 | INFO | PerfReport: 21.52%, Upload -- 2.08e+01 / 0.00e+00, 6.91e+00 / 0.00e+00 | 20260507 00:58:41.785 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260507 00:58:41.785 | INFO | PerfReport: 32.04%, Compute -- 3.09e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 00:58:41.785 | INFO | PerfReport: 1.27%, LockUpload -- 1.22e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260507 00:58:41.785 | INFO | PerfReport: 1.42%, LockDownload -- 1.37e+00 / 0.00e+00, 3.35e+00 / 0.00e+00 | 20260507 00:58:41.785 | INFO | PerfReport: 22.40%, LockCompute -- 2.16e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 00:58:41.785 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260507 00:58:41.882 | INFO | Starting a round of Baz IO | 20260507 00:58:45.001 | INFO | Populating RTMetrics report | 20260507 00:58:45.002 | INFO | RTMetrics report populated | 20260507 00:58:45.002 | INFO | Opening tmp RTMetrics json file | 20260507 00:58:45.004 | INFO | Serializing RTMetrics to json | 20260507 00:58:45.022 | INFO | Serialization done, swapping tmp file now... | 20260507 00:58:45.022 | INFO | RTMetrics done | 20260507 00:58:45.094 | INFO | Finished a round of Baz IO: Wrote 1.829 GiB in 3.211 seconds | 20260507 00:58:45.094 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.20% / 0.00% / 66.79% | 20260507 00:58:45.094 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.312956% / 3.97364e-06% / 7.54992e-05% | 20260507 00:58:45.094 | INFO | Baz IO overall: | 20260507 00:58:45.094 | INFO | BytesWritten : 3037104192516 | 20260507 00:58:45.094 | INFO | NumEvents : 1462444622430 | 20260507 00:58:46.922 | INFO | Analyzing chunk frames = [5840896,5841408) | 20260507 00:58:46.922 | INFO | MemInfo: { | 20260507 00:58:46.922 | INFO | "gpu" : " 17 GB", | 20260507 00:58:46.922 | INFO | "huge" : "", | 20260507 00:58:46.922 | INFO | "huge_shared" : " 27 GB", | 20260507 00:58:46.922 | INFO | "malloc" : " 14 GB", | 20260507 00:58:46.922 | INFO | "pinned" : " 29 GB" | 20260507 00:58:46.922 | INFO | } | 20260507 00:58:46.922 | INFO | Perf report for the last 5.13857 seconds: | 20260507 00:58:46.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:58:46.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:46.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:46.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:46.922 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260507 00:58:46.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:46.922 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 00:58:46.922 | INFO | PRE_HQ : 59.6%, 1.8, 56.3ms, 0 | 20260507 00:58:46.922 | INFO | BAZWRITER : 62.1%, 3.3, 106.6ms, 0 | 20260507 00:58:46.922 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:58:46.923 | INFO | Uploaded 36GiB to GPU in 2.02503s (17.7775GiB/s) | 20260507 00:58:46.923 | INFO | Downloaded 38GiB to GPU in 2.18364s (17.4021GiB/s) | 20260507 00:58:52.065 | INFO | Analyzing chunk frames = [5841408,5841920) | 20260507 00:58:52.065 | INFO | MemInfo: { | 20260507 00:58:52.065 | INFO | "gpu" : " 17 GB", | 20260507 00:58:52.065 | INFO | "huge" : "", | 20260507 00:58:52.065 | INFO | "huge_shared" : " 27 GB", | 20260507 00:58:52.065 | INFO | "malloc" : " 14 GB", | 20260507 00:58:52.065 | INFO | "pinned" : " 29 GB" | 20260507 00:58:52.065 | INFO | } | 20260507 00:58:52.065 | INFO | Perf report for the last 5.14314 seconds: | 20260507 00:58:52.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:58:52.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:52.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:52.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:52.066 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:58:52.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:52.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:52.066 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 00:58:52.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:52.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:52.066 | INFO | Uploaded 36GiB to GPU in 1.99961s (18.0035GiB/s) | 20260507 00:58:52.066 | INFO | Downloaded 34GiB to GPU in 1.93345s (17.5852GiB/s) | 20260507 00:58:57.206 | INFO | Analyzing chunk frames = [5841920,5842432) | 20260507 00:58:57.206 | INFO | MemInfo: { | 20260507 00:58:57.206 | INFO | "gpu" : " 17 GB", | 20260507 00:58:57.206 | INFO | "huge" : "", | 20260507 00:58:57.206 | INFO | "huge_shared" : " 27 GB", | 20260507 00:58:57.206 | INFO | "malloc" : " 14 GB", | 20260507 00:58:57.206 | INFO | "pinned" : " 29 GB" | 20260507 00:58:57.206 | INFO | } | 20260507 00:58:57.206 | INFO | Perf report for the last 5.14083 seconds: | 20260507 00:58:57.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:58:57.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:57.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:57.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:57.206 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:58:57.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:57.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:57.206 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 00:58:57.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:57.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:58:57.207 | INFO | Uploaded 36GiB to GPU in 1.97047s (18.2697GiB/s) | 20260507 00:58:57.207 | INFO | Downloaded 34GiB to GPU in 1.93498s (17.5713GiB/s) | 20260507 00:59:02.348 | INFO | Analyzing chunk frames = [5842432,5842944) | 20260507 00:59:02.348 | INFO | MemInfo: { | 20260507 00:59:02.348 | INFO | "gpu" : " 17 GB", | 20260507 00:59:02.348 | INFO | "huge" : "", | 20260507 00:59:02.348 | INFO | "huge_shared" : " 27 GB", | 20260507 00:59:02.348 | INFO | "malloc" : " 14 GB", | 20260507 00:59:02.348 | INFO | "pinned" : " 29 GB" | 20260507 00:59:02.348 | INFO | } | 20260507 00:59:02.348 | INFO | Perf report for the last 5.14225 seconds: | 20260507 00:59:02.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:59:02.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:02.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:02.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:02.349 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260507 00:59:02.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:02.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:02.349 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:59:02.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:02.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:02.349 | INFO | Uploaded 36GiB to GPU in 1.9679s (18.2936GiB/s) | 20260507 00:59:02.349 | INFO | Downloaded 34GiB to GPU in 1.9082s (17.8178GiB/s) | 20260507 00:59:07.480 | INFO | Analyzing chunk frames = [5842944,5843456) | 20260507 00:59:07.480 | INFO | MemInfo: { | 20260507 00:59:07.480 | INFO | "gpu" : " 17 GB", | 20260507 00:59:07.480 | INFO | "huge" : "", | 20260507 00:59:07.480 | INFO | "huge_shared" : " 27 GB", | 20260507 00:59:07.480 | INFO | "malloc" : " 14 GB", | 20260507 00:59:07.480 | INFO | "pinned" : " 29 GB" | 20260507 00:59:07.480 | INFO | } | 20260507 00:59:07.480 | INFO | Perf report for the last 5.13137 seconds: | 20260507 00:59:07.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:59:07.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:07.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:07.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:07.480 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260507 00:59:07.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:07.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:07.480 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:59:07.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:07.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:07.480 | INFO | Uploaded 36GiB to GPU in 1.98154s (18.1677GiB/s) | 20260507 00:59:07.480 | INFO | Downloaded 34GiB to GPU in 1.95213s (17.4169GiB/s) | 20260507 00:59:12.617 | INFO | Analyzing chunk frames = [5843456,5843968) | 20260507 00:59:12.617 | INFO | MemInfo: { | 20260507 00:59:12.617 | INFO | "gpu" : " 17 GB", | 20260507 00:59:12.617 | INFO | "huge" : "", | 20260507 00:59:12.617 | INFO | "huge_shared" : " 27 GB", | 20260507 00:59:12.617 | INFO | "malloc" : " 14 GB", | 20260507 00:59:12.617 | INFO | "pinned" : " 29 GB" | 20260507 00:59:12.617 | INFO | } | 20260507 00:59:12.617 | INFO | Perf report for the last 5.13716 seconds: | 20260507 00:59:12.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:59:12.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:12.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:12.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:12.617 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:59:12.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:12.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:12.617 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 00:59:12.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:12.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:12.617 | INFO | Uploaded 36GiB to GPU in 2.04448s (17.6084GiB/s) | 20260507 00:59:12.617 | INFO | Downloaded 34GiB to GPU in 1.95772s (17.3671GiB/s) | 20260507 00:59:17.756 | INFO | Analyzing chunk frames = [5843968,5844480) | 20260507 00:59:17.757 | INFO | MemInfo: { | 20260507 00:59:17.757 | INFO | "gpu" : " 17 GB", | 20260507 00:59:17.757 | INFO | "huge" : "", | 20260507 00:59:17.757 | INFO | "huge_shared" : " 27 GB", | 20260507 00:59:17.757 | INFO | "malloc" : " 14 GB", | 20260507 00:59:17.757 | INFO | "pinned" : " 29 GB" | 20260507 00:59:17.757 | INFO | } | 20260507 00:59:17.757 | INFO | Perf report for the last 5.13979 seconds: | 20260507 00:59:17.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:59:17.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:17.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:17.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:17.757 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 00:59:17.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:17.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:17.757 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 00:59:17.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:17.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:17.757 | INFO | Uploaded 36GiB to GPU in 1.95281s (18.435GiB/s) | 20260507 00:59:17.757 | INFO | Downloaded 34GiB to GPU in 1.98908s (17.0933GiB/s) | 20260507 00:59:22.897 | INFO | Analyzing chunk frames = [5844480,5844992) | 20260507 00:59:22.897 | INFO | MemInfo: { | 20260507 00:59:22.897 | INFO | "gpu" : " 17 GB", | 20260507 00:59:22.897 | INFO | "huge" : "", | 20260507 00:59:22.897 | INFO | "huge_shared" : " 27 GB", | 20260507 00:59:22.898 | INFO | "malloc" : " 14 GB", | 20260507 00:59:22.898 | INFO | "pinned" : " 29 GB" | 20260507 00:59:22.898 | INFO | } | 20260507 00:59:22.898 | INFO | Perf report for the last 5.14085 seconds: | 20260507 00:59:22.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:59:22.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:22.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:22.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:22.898 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 00:59:22.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:22.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:22.898 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 00:59:22.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:22.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:22.898 | INFO | Uploaded 36GiB to GPU in 1.97538s (18.2243GiB/s) | 20260507 00:59:22.898 | INFO | Downloaded 34GiB to GPU in 1.98129s (17.1605GiB/s) | 20260507 00:59:22.995 | INFO | Starting a round of Baz IO | 20260507 00:59:26.119 | INFO | Populating RTMetrics report | 20260507 00:59:26.119 | INFO | RTMetrics report populated | 20260507 00:59:26.119 | INFO | Opening tmp RTMetrics json file | 20260507 00:59:26.120 | INFO | Serializing RTMetrics to json | 20260507 00:59:26.123 | INFO | Serialization done, swapping tmp file now... | 20260507 00:59:26.130 | INFO | RTMetrics done | 20260507 00:59:26.213 | INFO | Finished a round of Baz IO: Wrote 1.83 GiB in 3.217 seconds | 20260507 00:59:26.213 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.20% / 0.00% / 66.80% | 20260507 00:59:26.213 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.313489% / 3.97364e-06% / 6.35783e-05% | 20260507 00:59:26.213 | INFO | Baz IO overall: | 20260507 00:59:26.213 | INFO | BytesWritten : 3039069548666 | 20260507 00:59:26.213 | INFO | NumEvents : 1463377408735 | 20260507 00:59:28.045 | INFO | Analyzing chunk frames = [5844992,5845504) | 20260507 00:59:28.045 | INFO | MemInfo: { | 20260507 00:59:28.045 | INFO | "gpu" : " 17 GB", | 20260507 00:59:28.045 | INFO | "huge" : "", | 20260507 00:59:28.045 | INFO | "huge_shared" : " 27 GB", | 20260507 00:59:28.045 | INFO | "malloc" : " 14 GB", | 20260507 00:59:28.045 | INFO | "pinned" : " 29 GB" | 20260507 00:59:28.045 | INFO | } | 20260507 00:59:28.045 | INFO | Perf report for the last 5.14772 seconds: | 20260507 00:59:28.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:59:28.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:28.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:28.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:28.045 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260507 00:59:28.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:28.046 | INFO | RT_METRICS : 42.0%, 1.0, 22.1ms, 0 | 20260507 00:59:28.046 | INFO | PRE_HQ : 59.5%, 1.8, 56.2ms, 0 | 20260507 00:59:28.046 | INFO | BAZWRITER : 62.1%, 3.4, 107.4ms, 0 | 20260507 00:59:28.046 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 00:59:28.046 | INFO | Uploaded 36GiB to GPU in 2.0303s (17.7314GiB/s) | 20260507 00:59:28.046 | INFO | Downloaded 38GiB to GPU in 2.17844s (17.4437GiB/s) | 20260507 00:59:33.188 | INFO | Analyzing chunk frames = [5845504,5846016) | 20260507 00:59:33.188 | INFO | MemInfo: { | 20260507 00:59:33.188 | INFO | "gpu" : " 17 GB", | 20260507 00:59:33.188 | INFO | "huge" : "", | 20260507 00:59:33.188 | INFO | "huge_shared" : " 27 GB", | 20260507 00:59:33.188 | INFO | "malloc" : " 14 GB", | 20260507 00:59:33.188 | INFO | "pinned" : " 29 GB" | 20260507 00:59:33.188 | INFO | } | 20260507 00:59:33.188 | INFO | Perf report for the last 5.14245 seconds: | 20260507 00:59:33.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:59:33.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:33.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:33.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:33.188 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 00:59:33.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:33.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:33.188 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 00:59:33.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:33.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:33.188 | INFO | Uploaded 36GiB to GPU in 2.03837s (17.6612GiB/s) | 20260507 00:59:33.188 | INFO | Downloaded 34GiB to GPU in 1.94267s (17.5017GiB/s) | 20260507 00:59:38.325 | INFO | Analyzing chunk frames = [5846016,5846528) | 20260507 00:59:38.325 | INFO | MemInfo: { | 20260507 00:59:38.325 | INFO | "gpu" : " 17 GB", | 20260507 00:59:38.325 | INFO | "huge" : "", | 20260507 00:59:38.325 | INFO | "huge_shared" : " 27 GB", | 20260507 00:59:38.325 | INFO | "malloc" : " 14 GB", | 20260507 00:59:38.325 | INFO | "pinned" : " 29 GB" | 20260507 00:59:38.325 | INFO | } | 20260507 00:59:38.325 | INFO | Perf report for the last 5.13722 seconds: | 20260507 00:59:38.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:59:38.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:38.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:38.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:38.325 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 00:59:38.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:38.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:38.325 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 00:59:38.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:38.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:38.325 | INFO | Uploaded 36GiB to GPU in 1.9991s (18.0081GiB/s) | 20260507 00:59:38.325 | INFO | Downloaded 34GiB to GPU in 1.94291s (17.4995GiB/s) | 20260507 00:59:43.465 | INFO | Analyzing chunk frames = [5846528,5847040) | 20260507 00:59:43.465 | INFO | MemInfo: { | 20260507 00:59:43.465 | INFO | "gpu" : " 17 GB", | 20260507 00:59:43.465 | INFO | "huge" : "", | 20260507 00:59:43.465 | INFO | "huge_shared" : " 27 GB", | 20260507 00:59:43.465 | INFO | "malloc" : " 14 GB", | 20260507 00:59:43.465 | INFO | "pinned" : " 29 GB" | 20260507 00:59:43.465 | INFO | } | 20260507 00:59:43.465 | INFO | Perf report for the last 5.14031 seconds: | 20260507 00:59:43.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:59:43.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:43.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:43.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:43.466 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 00:59:43.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:43.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:43.466 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 00:59:43.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:43.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:43.466 | INFO | Uploaded 36GiB to GPU in 1.95957s (18.3713GiB/s) | 20260507 00:59:43.466 | INFO | Downloaded 34GiB to GPU in 1.92146s (17.6949GiB/s) | 20260507 00:59:48.598 | INFO | Analyzing chunk frames = [5847040,5847552) | 20260507 00:59:48.598 | INFO | MemInfo: { | 20260507 00:59:48.598 | INFO | "gpu" : " 17 GB", | 20260507 00:59:48.598 | INFO | "huge" : "", | 20260507 00:59:48.598 | INFO | "huge_shared" : " 27 GB", | 20260507 00:59:48.598 | INFO | "malloc" : " 14 GB", | 20260507 00:59:48.598 | INFO | "pinned" : " 29 GB" | 20260507 00:59:48.598 | INFO | } | 20260507 00:59:48.598 | INFO | Perf report for the last 5.13256 seconds: | 20260507 00:59:48.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:59:48.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:48.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:48.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:48.598 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260507 00:59:48.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:48.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:48.598 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 00:59:48.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:48.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:48.598 | INFO | Uploaded 36GiB to GPU in 1.99347s (18.059GiB/s) | 20260507 00:59:48.598 | INFO | Downloaded 34GiB to GPU in 1.94511s (17.4797GiB/s) | 20260507 00:59:53.736 | INFO | Analyzing chunk frames = [5847552,5848064) | 20260507 00:59:53.736 | INFO | MemInfo: { | 20260507 00:59:53.736 | INFO | "gpu" : " 17 GB", | 20260507 00:59:53.736 | INFO | "huge" : "", | 20260507 00:59:53.736 | INFO | "huge_shared" : " 27 GB", | 20260507 00:59:53.736 | INFO | "malloc" : " 14 GB", | 20260507 00:59:53.736 | INFO | "pinned" : " 29 GB" | 20260507 00:59:53.736 | INFO | } | 20260507 00:59:53.736 | INFO | Perf report for the last 5.13803 seconds: | 20260507 00:59:53.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:59:53.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:53.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:53.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:53.736 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 00:59:53.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:53.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:53.736 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 00:59:53.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:53.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:53.736 | INFO | Uploaded 36GiB to GPU in 2.04079s (17.6402GiB/s) | 20260507 00:59:53.736 | INFO | Downloaded 34GiB to GPU in 1.96169s (17.332GiB/s) | 20260507 00:59:58.873 | INFO | Analyzing chunk frames = [5848064,5848576) | 20260507 00:59:58.873 | INFO | MemInfo: { | 20260507 00:59:58.874 | INFO | "gpu" : " 17 GB", | 20260507 00:59:58.874 | INFO | "huge" : "", | 20260507 00:59:58.874 | INFO | "huge_shared" : " 27 GB", | 20260507 00:59:58.874 | INFO | "malloc" : " 14 GB", | 20260507 00:59:58.874 | INFO | "pinned" : " 29 GB" | 20260507 00:59:58.874 | INFO | } | 20260507 00:59:58.874 | INFO | Perf report for the last 5.13765 seconds: | 20260507 00:59:58.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 00:59:58.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:58.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:58.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:58.874 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 00:59:58.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:58.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:58.874 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 00:59:58.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:58.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 00:59:58.874 | INFO | Uploaded 36GiB to GPU in 1.99935s (18.0058GiB/s) | 20260507 00:59:58.874 | INFO | Downloaded 34GiB to GPU in 1.99566s (17.0369GiB/s) | 20260507 01:00:04.013 | INFO | Analyzing chunk frames = [5848576,5849088) | 20260507 01:00:04.013 | INFO | MemInfo: { | 20260507 01:00:04.013 | INFO | "gpu" : " 17 GB", | 20260507 01:00:04.013 | INFO | "huge" : "", | 20260507 01:00:04.013 | INFO | "huge_shared" : " 27 GB", | 20260507 01:00:04.013 | INFO | "malloc" : " 14 GB", | 20260507 01:00:04.013 | INFO | "pinned" : " 29 GB" | 20260507 01:00:04.013 | INFO | } | 20260507 01:00:04.013 | INFO | Perf report for the last 5.13952 seconds: | 20260507 01:00:04.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:00:04.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:04.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:04.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:04.013 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:00:04.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:04.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:04.013 | INFO | PRE_HQ : 30.7%, 1.0, 16.0ms, 0 | 20260507 01:00:04.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:04.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:04.013 | INFO | Uploaded 36GiB to GPU in 1.98178s (18.1655GiB/s) | 20260507 01:00:04.014 | INFO | Downloaded 34GiB to GPU in 1.98721s (17.1094GiB/s) | 20260507 01:00:04.014 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:00:04.014 | INFO | Total mem usage: 18 GiB | 20260507 01:00:04.014 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:00:04.014 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:00:04.014 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:00:04.014 | INFO | High Water Breakdown: | 20260507 01:00:04.014 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:00:04.014 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:00:04.014 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:00:04.014 | INFO | ------------------------------------------ | 20260507 01:00:04.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:00:04.014 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:00:04.014 | INFO | Total mem usage: 26.5 GiB | 20260507 01:00:04.014 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:00:04.014 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:00:04.014 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:00:04.014 | INFO | High Water Breakdown: | 20260507 01:00:04.014 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:00:04.014 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:00:04.014 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:00:04.014 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:00:04.014 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:00:04.014 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:00:04.014 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:00:04.014 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:00:04.014 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:00:04.014 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:00:04.014 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:00:04.014 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:00:04.014 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:00:04.014 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:00:04.014 | INFO | ------------------------------------------ | 20260507 01:00:04.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:00:04.014 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:00:04.014 | INFO | Total mem usage: 25.6 GiB | 20260507 01:00:04.014 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:00:04.014 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 01:00:04.014 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 01:00:04.014 | INFO | High Water Breakdown: | 20260507 01:00:04.014 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 01:00:04.014 | INFO | ------------------------------------------ | 20260507 01:00:04.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:00:04.014 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:00:04.014 | INFO | Total mem usage: 15.7 GiB | 20260507 01:00:04.014 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:00:04.014 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:00:04.014 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:00:04.014 | INFO | High Water Breakdown: | 20260507 01:00:04.014 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:00:04.014 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:00:04.014 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:00:04.014 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:00:04.014 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:00:04.015 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:00:04.015 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:00:04.015 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:00:04.015 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:00:04.015 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:00:04.015 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:00:04.015 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:00:04.015 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:00:04.015 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:00:04.015 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:00:04.015 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:00:04.015 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:00:04.015 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:00:04.015 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:00:04.015 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:00:04.015 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:00:04.015 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:00:04.015 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:00:04.015 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:00:04.015 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:00:04.015 | INFO | ------------------------------------------ | 20260507 01:00:04.015 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:00:04.015 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:00:04.015 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:00:04.015 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:00:04.015 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.52e+01, 1.32e+01 / 2.07e+00 | 20260507 01:00:04.015 | INFO | PerfReport: 0.30%, AggregateStats -- 9.12e-02 / 1.01e-01, 1.01e-02 / 6.93e-03 | 20260507 01:00:04.015 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.35e+00, 5.26e-01 / 3.76e-01 | 20260507 01:00:04.015 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.45e+00, 1.71e-01 / 1.29e-01 | 20260507 01:00:04.015 | INFO | PerfReport: 16.07%, DME -- 3.97e+01 / 4.41e+01, 3.90e+00 / 2.29e+00 | 20260507 01:00:04.015 | INFO | PerfReport: 57.47%, FrameLabeling -- 1.77e+01 / 2.08e+01, 1.27e+00 / 1.63e+00 | 20260507 01:00:04.015 | INFO | PerfReport: 7.06%, PulseAccumulating -- 2.18e+00 / 2.73e+00, 3.84e-01 / 2.19e-01 | 20260507 01:00:04.015 | INFO | PerfReport: 3.14%, Metrics -- 9.68e-01 / 1.30e+00, 2.55e-01 / 1.58e-01 | 20260507 01:00:04.015 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.29e-01 / 3.83e-01, 5.16e-02 / 2.33e-02 | 20260507 01:00:04.015 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:00:04.015 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:00:04.015 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:00:04.015 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260507 01:00:04.015 | INFO | PerfReport: 21.56%, Upload -- 2.08e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260507 01:00:04.015 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260507 01:00:04.015 | INFO | PerfReport: 32.00%, Compute -- 3.09e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 01:00:04.015 | INFO | PerfReport: 1.28%, LockUpload -- 1.23e+00 / 0.00e+00, 4.78e+00 / 0.00e+00 | 20260507 01:00:04.015 | INFO | PerfReport: 1.40%, LockDownload -- 1.35e+00 / 0.00e+00, 3.29e+00 / 0.00e+00 | 20260507 01:00:04.015 | INFO | PerfReport: 22.34%, LockCompute -- 2.16e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260507 01:00:04.015 | INFO | PerfReport: 0.02%, Misc -- 1.51e-02 / 0.00e+00, 3.59e-02 / 0.00e+00 | 20260507 01:00:04.111 | INFO | Starting a round of Baz IO | 20260507 01:00:07.233 | INFO | Populating RTMetrics report | 20260507 01:00:07.233 | INFO | RTMetrics report populated | 20260507 01:00:07.233 | INFO | Opening tmp RTMetrics json file | 20260507 01:00:07.253 | INFO | Serializing RTMetrics to json | 20260507 01:00:07.254 | INFO | Serialization done, swapping tmp file now... | 20260507 01:00:07.255 | INFO | RTMetrics done | 20260507 01:00:07.327 | INFO | Finished a round of Baz IO: Wrote 1.832 GiB in 3.215 seconds | 20260507 01:00:07.327 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.20% / 0.00% / 66.80% | 20260507 01:00:07.327 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.313787% / 7.94729e-06% / 5.96046e-05% | 20260507 01:00:07.327 | INFO | Baz IO overall: | 20260507 01:00:07.327 | INFO | BytesWritten : 3041036567817 | 20260507 01:00:07.327 | INFO | NumEvents : 1464311580538 | 20260507 01:00:09.162 | INFO | Analyzing chunk frames = [5849088,5849600) | 20260507 01:00:09.162 | INFO | MemInfo: { | 20260507 01:00:09.162 | INFO | "gpu" : " 17 GB", | 20260507 01:00:09.162 | INFO | "huge" : "", | 20260507 01:00:09.162 | INFO | "huge_shared" : " 27 GB", | 20260507 01:00:09.162 | INFO | "malloc" : " 14 GB", | 20260507 01:00:09.162 | INFO | "pinned" : " 29 GB" | 20260507 01:00:09.162 | INFO | } | 20260507 01:00:09.162 | INFO | Perf report for the last 5.14873 seconds: | 20260507 01:00:09.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:00:09.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:09.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:09.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:09.162 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260507 01:00:09.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:09.162 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 01:00:09.162 | INFO | PRE_HQ : 59.4%, 1.8, 56.2ms, 0 | 20260507 01:00:09.162 | INFO | BAZWRITER : 62.0%, 3.3, 106.6ms, 0 | 20260507 01:00:09.162 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:00:09.162 | INFO | Uploaded 36GiB to GPU in 2.04686s (17.5879GiB/s) | 20260507 01:00:09.162 | INFO | Downloaded 38GiB to GPU in 2.18274s (17.4093GiB/s) | 20260507 01:00:14.297 | INFO | Analyzing chunk frames = [5849600,5850112) | 20260507 01:00:14.297 | INFO | MemInfo: { | 20260507 01:00:14.297 | INFO | "gpu" : " 17 GB", | 20260507 01:00:14.297 | INFO | "huge" : "", | 20260507 01:00:14.297 | INFO | "huge_shared" : " 27 GB", | 20260507 01:00:14.297 | INFO | "malloc" : " 14 GB", | 20260507 01:00:14.297 | INFO | "pinned" : " 29 GB" | 20260507 01:00:14.297 | INFO | } | 20260507 01:00:14.297 | INFO | Perf report for the last 5.13537 seconds: | 20260507 01:00:14.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:00:14.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:14.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:14.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:14.297 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:00:14.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:14.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:14.297 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:00:14.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:14.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:14.298 | INFO | Uploaded 36GiB to GPU in 2.04676s (17.5887GiB/s) | 20260507 01:00:14.298 | INFO | Downloaded 34GiB to GPU in 1.93497s (17.5713GiB/s) | 20260507 01:00:19.429 | INFO | Analyzing chunk frames = [5850112,5850624) | 20260507 01:00:19.429 | INFO | MemInfo: { | 20260507 01:00:19.429 | INFO | "gpu" : " 17 GB", | 20260507 01:00:19.429 | INFO | "huge" : "", | 20260507 01:00:19.429 | INFO | "huge_shared" : " 27 GB", | 20260507 01:00:19.429 | INFO | "malloc" : " 14 GB", | 20260507 01:00:19.429 | INFO | "pinned" : " 29 GB" | 20260507 01:00:19.429 | INFO | } | 20260507 01:00:19.429 | INFO | Perf report for the last 5.13207 seconds: | 20260507 01:00:19.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:00:19.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:19.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:19.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:19.430 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:00:19.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:19.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:19.430 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:00:19.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:19.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:19.430 | INFO | Uploaded 36GiB to GPU in 2.01918s (17.829GiB/s) | 20260507 01:00:19.430 | INFO | Downloaded 34GiB to GPU in 1.93645s (17.5579GiB/s) | 20260507 01:00:24.568 | INFO | Analyzing chunk frames = [5850624,5851136) | 20260507 01:00:24.568 | INFO | MemInfo: { | 20260507 01:00:24.568 | INFO | "gpu" : " 17 GB", | 20260507 01:00:24.568 | INFO | "huge" : "", | 20260507 01:00:24.568 | INFO | "huge_shared" : " 27 GB", | 20260507 01:00:24.568 | INFO | "malloc" : " 14 GB", | 20260507 01:00:24.568 | INFO | "pinned" : " 29 GB" | 20260507 01:00:24.569 | INFO | } | 20260507 01:00:24.569 | INFO | Perf report for the last 5.13914 seconds: | 20260507 01:00:24.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:00:24.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:24.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:24.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:24.569 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 01:00:24.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:24.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:24.569 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 01:00:24.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:24.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:24.569 | INFO | Uploaded 36GiB to GPU in 2.01704s (17.8479GiB/s) | 20260507 01:00:24.569 | INFO | Downloaded 34GiB to GPU in 1.90892s (17.8111GiB/s) | 20260507 01:00:29.710 | INFO | Analyzing chunk frames = [5851136,5851648) | 20260507 01:00:29.710 | INFO | MemInfo: { | 20260507 01:00:29.710 | INFO | "gpu" : " 17 GB", | 20260507 01:00:29.710 | INFO | "huge" : "", | 20260507 01:00:29.710 | INFO | "huge_shared" : " 27 GB", | 20260507 01:00:29.710 | INFO | "malloc" : " 14 GB", | 20260507 01:00:29.710 | INFO | "pinned" : " 29 GB" | 20260507 01:00:29.711 | INFO | } | 20260507 01:00:29.711 | INFO | Perf report for the last 5.14232 seconds: | 20260507 01:00:29.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:00:29.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:29.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:29.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:29.711 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 01:00:29.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:29.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:29.711 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:00:29.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:29.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:29.711 | INFO | Uploaded 36GiB to GPU in 1.99739s (18.0235GiB/s) | 20260507 01:00:29.711 | INFO | Downloaded 34GiB to GPU in 1.92771s (17.6375GiB/s) | 20260507 01:00:34.852 | INFO | Analyzing chunk frames = [5851648,5852160) | 20260507 01:00:34.852 | INFO | MemInfo: { | 20260507 01:00:34.852 | INFO | "gpu" : " 17 GB", | 20260507 01:00:34.852 | INFO | "huge" : "", | 20260507 01:00:34.852 | INFO | "huge_shared" : " 27 GB", | 20260507 01:00:34.853 | INFO | "malloc" : " 14 GB", | 20260507 01:00:34.853 | INFO | "pinned" : " 29 GB" | 20260507 01:00:34.853 | INFO | } | 20260507 01:00:34.853 | INFO | Perf report for the last 5.14172 seconds: | 20260507 01:00:34.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:00:34.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:34.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:34.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:34.853 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 01:00:34.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:34.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:34.853 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 01:00:34.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:34.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:34.853 | INFO | Uploaded 36GiB to GPU in 2.03024s (17.7319GiB/s) | 20260507 01:00:34.853 | INFO | Downloaded 34GiB to GPU in 1.93975s (17.528GiB/s) | 20260507 01:00:39.998 | INFO | Analyzing chunk frames = [5852160,5852672) | 20260507 01:00:39.998 | INFO | MemInfo: { | 20260507 01:00:39.998 | INFO | "gpu" : " 17 GB", | 20260507 01:00:39.998 | INFO | "huge" : "", | 20260507 01:00:39.998 | INFO | "huge_shared" : " 27 GB", | 20260507 01:00:39.998 | INFO | "malloc" : " 14 GB", | 20260507 01:00:39.998 | INFO | "pinned" : " 29 GB" | 20260507 01:00:39.998 | INFO | } | 20260507 01:00:39.998 | INFO | Perf report for the last 5.14552 seconds: | 20260507 01:00:39.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:00:39.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:39.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:39.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:39.998 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:00:39.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:39.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:39.998 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 01:00:39.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:39.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:39.999 | INFO | Uploaded 36GiB to GPU in 1.99611s (18.0351GiB/s) | 20260507 01:00:39.999 | INFO | Downloaded 34GiB to GPU in 1.9857s (17.1225GiB/s) | 20260507 01:00:45.136 | INFO | Analyzing chunk frames = [5852672,5853184) | 20260507 01:00:45.136 | INFO | MemInfo: { | 20260507 01:00:45.136 | INFO | "gpu" : " 17 GB", | 20260507 01:00:45.136 | INFO | "huge" : "", | 20260507 01:00:45.136 | INFO | "huge_shared" : " 27 GB", | 20260507 01:00:45.136 | INFO | "malloc" : " 14 GB", | 20260507 01:00:45.136 | INFO | "pinned" : " 29 GB" | 20260507 01:00:45.136 | INFO | } | 20260507 01:00:45.136 | INFO | Perf report for the last 5.13761 seconds: | 20260507 01:00:45.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:00:45.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:45.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:45.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:45.136 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:00:45.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:45.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:45.136 | INFO | PRE_HQ : 30.1%, 1.0, 15.6ms, 0 | 20260507 01:00:45.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:45.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:45.136 | INFO | Uploaded 36GiB to GPU in 1.99226s (18.0699GiB/s) | 20260507 01:00:45.136 | INFO | Downloaded 34GiB to GPU in 1.97926s (17.1782GiB/s) | 20260507 01:00:45.234 | INFO | Starting a round of Baz IO | 20260507 01:00:48.350 | INFO | Populating RTMetrics report | 20260507 01:00:48.351 | INFO | RTMetrics report populated | 20260507 01:00:48.351 | INFO | Opening tmp RTMetrics json file | 20260507 01:00:48.352 | INFO | Serializing RTMetrics to json | 20260507 01:00:48.371 | INFO | Serialization done, swapping tmp file now... | 20260507 01:00:48.371 | INFO | RTMetrics done | 20260507 01:00:48.441 | INFO | Finished a round of Baz IO: Wrote 1.827 GiB in 3.207 seconds | 20260507 01:00:48.441 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.20% / 0.00% / 66.80% | 20260507 01:00:48.441 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.311701% / 1.19209e-05% / 5.5631e-05% | 20260507 01:00:48.441 | INFO | Baz IO overall: | 20260507 01:00:48.441 | INFO | BytesWritten : 3042998467530 | 20260507 01:00:48.441 | INFO | NumEvents : 1465242056181 | 20260507 01:00:50.269 | INFO | Analyzing chunk frames = [5853184,5853696) | 20260507 01:00:50.269 | INFO | MemInfo: { | 20260507 01:00:50.269 | INFO | "gpu" : " 17 GB", | 20260507 01:00:50.269 | INFO | "huge" : "", | 20260507 01:00:50.269 | INFO | "huge_shared" : " 27 GB", | 20260507 01:00:50.269 | INFO | "malloc" : " 14 GB", | 20260507 01:00:50.269 | INFO | "pinned" : " 29 GB" | 20260507 01:00:50.269 | INFO | } | 20260507 01:00:50.269 | INFO | Perf report for the last 5.13317 seconds: | 20260507 01:00:50.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:00:50.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:50.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:50.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:50.269 | INFO | ANALYSIS : 62.1%, 3.0, 96.4ms, 0 | 20260507 01:00:50.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:50.269 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 01:00:50.269 | INFO | PRE_HQ : 59.7%, 1.8, 56.3ms, 0 | 20260507 01:00:50.269 | INFO | BAZWRITER : 62.1%, 3.3, 106.6ms, 0 | 20260507 01:00:50.269 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:00:50.269 | INFO | Uploaded 36GiB to GPU in 2.04455s (17.6078GiB/s) | 20260507 01:00:50.269 | INFO | Downloaded 38GiB to GPU in 2.18223s (17.4134GiB/s) | 20260507 01:00:55.410 | INFO | Analyzing chunk frames = [5853696,5854208) | 20260507 01:00:55.410 | INFO | MemInfo: { | 20260507 01:00:55.410 | INFO | "gpu" : " 17 GB", | 20260507 01:00:55.410 | INFO | "huge" : "", | 20260507 01:00:55.410 | INFO | "huge_shared" : " 27 GB", | 20260507 01:00:55.410 | INFO | "malloc" : " 14 GB", | 20260507 01:00:55.410 | INFO | "pinned" : " 29 GB" | 20260507 01:00:55.410 | INFO | } | 20260507 01:00:55.410 | INFO | Perf report for the last 5.14109 seconds: | 20260507 01:00:55.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:00:55.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:55.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:55.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:55.410 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:00:55.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:55.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:55.410 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:00:55.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:55.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:00:55.410 | INFO | Uploaded 36GiB to GPU in 2.0063s (17.9435GiB/s) | 20260507 01:00:55.410 | INFO | Downloaded 34GiB to GPU in 1.93539s (17.5675GiB/s) | 20260507 01:01:00.548 | INFO | Analyzing chunk frames = [5854208,5854720) | 20260507 01:01:00.548 | INFO | MemInfo: { | 20260507 01:01:00.548 | INFO | "gpu" : " 17 GB", | 20260507 01:01:00.548 | INFO | "huge" : "", | 20260507 01:01:00.549 | INFO | "huge_shared" : " 27 GB", | 20260507 01:01:00.549 | INFO | "malloc" : " 14 GB", | 20260507 01:01:00.549 | INFO | "pinned" : " 29 GB" | 20260507 01:01:00.549 | INFO | } | 20260507 01:01:00.549 | INFO | Perf report for the last 5.13853 seconds: | 20260507 01:01:00.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:01:00.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:00.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:00.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:00.549 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 01:01:00.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:00.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:00.549 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:01:00.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:00.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:00.549 | INFO | Uploaded 36GiB to GPU in 1.98827s (18.1062GiB/s) | 20260507 01:01:00.549 | INFO | Downloaded 34GiB to GPU in 1.93381s (17.5819GiB/s) | 20260507 01:01:05.684 | INFO | Analyzing chunk frames = [5854720,5855232) | 20260507 01:01:05.684 | INFO | MemInfo: { | 20260507 01:01:05.684 | INFO | "gpu" : " 17 GB", | 20260507 01:01:05.685 | INFO | "huge" : "", | 20260507 01:01:05.685 | INFO | "huge_shared" : " 27 GB", | 20260507 01:01:05.685 | INFO | "malloc" : " 14 GB", | 20260507 01:01:05.685 | INFO | "pinned" : " 29 GB" | 20260507 01:01:05.685 | INFO | } | 20260507 01:01:05.685 | INFO | Perf report for the last 5.13601 seconds: | 20260507 01:01:05.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:01:05.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:05.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:05.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:05.685 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 01:01:05.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:05.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:05.685 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:01:05.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:05.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:05.685 | INFO | Uploaded 36GiB to GPU in 1.99139s (18.0778GiB/s) | 20260507 01:01:05.685 | INFO | Downloaded 34GiB to GPU in 1.91122s (17.7897GiB/s) | 20260507 01:01:10.825 | INFO | Analyzing chunk frames = [5855232,5855744) | 20260507 01:01:10.825 | INFO | MemInfo: { | 20260507 01:01:10.825 | INFO | "gpu" : " 17 GB", | 20260507 01:01:10.825 | INFO | "huge" : "", | 20260507 01:01:10.825 | INFO | "huge_shared" : " 27 GB", | 20260507 01:01:10.825 | INFO | "malloc" : " 14 GB", | 20260507 01:01:10.825 | INFO | "pinned" : " 29 GB" | 20260507 01:01:10.825 | INFO | } | 20260507 01:01:10.825 | INFO | Perf report for the last 5.1406 seconds: | 20260507 01:01:10.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:01:10.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:10.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:10.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:10.825 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 01:01:10.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:10.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:10.825 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:01:10.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:10.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:10.825 | INFO | Uploaded 36GiB to GPU in 1.98299s (18.1544GiB/s) | 20260507 01:01:10.826 | INFO | Downloaded 34GiB to GPU in 1.93095s (17.6079GiB/s) | 20260507 01:01:15.967 | INFO | Analyzing chunk frames = [5855744,5856256) | 20260507 01:01:15.967 | INFO | MemInfo: { | 20260507 01:01:15.967 | INFO | "gpu" : " 17 GB", | 20260507 01:01:15.967 | INFO | "huge" : "", | 20260507 01:01:15.967 | INFO | "huge_shared" : " 27 GB", | 20260507 01:01:15.967 | INFO | "malloc" : " 14 GB", | 20260507 01:01:15.967 | INFO | "pinned" : " 29 GB" | 20260507 01:01:15.967 | INFO | } | 20260507 01:01:15.967 | INFO | Perf report for the last 5.14169 seconds: | 20260507 01:01:15.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:01:15.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:15.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:15.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:15.967 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 01:01:15.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:15.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:15.967 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:01:15.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:15.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:15.967 | INFO | Uploaded 36GiB to GPU in 2.01571s (17.8597GiB/s) | 20260507 01:01:15.967 | INFO | Downloaded 34GiB to GPU in 1.93256s (17.5933GiB/s) | 20260507 01:01:21.108 | INFO | Analyzing chunk frames = [5856256,5856768) | 20260507 01:01:21.108 | INFO | MemInfo: { | 20260507 01:01:21.108 | INFO | "gpu" : " 17 GB", | 20260507 01:01:21.108 | INFO | "huge" : "", | 20260507 01:01:21.108 | INFO | "huge_shared" : " 27 GB", | 20260507 01:01:21.108 | INFO | "malloc" : " 14 GB", | 20260507 01:01:21.108 | INFO | "pinned" : " 29 GB" | 20260507 01:01:21.108 | INFO | } | 20260507 01:01:21.108 | INFO | Perf report for the last 5.14115 seconds: | 20260507 01:01:21.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:01:21.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:21.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:21.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:21.108 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:01:21.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:21.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:21.108 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 01:01:21.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:21.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:21.108 | INFO | Uploaded 36GiB to GPU in 1.99417s (18.0526GiB/s) | 20260507 01:01:21.108 | INFO | Downloaded 34GiB to GPU in 1.99409s (17.0504GiB/s) | 20260507 01:01:26.246 | INFO | Analyzing chunk frames = [5856768,5857280) | 20260507 01:01:26.246 | INFO | MemInfo: { | 20260507 01:01:26.246 | INFO | "gpu" : " 17 GB", | 20260507 01:01:26.246 | INFO | "huge" : "", | 20260507 01:01:26.246 | INFO | "huge_shared" : " 27 GB", | 20260507 01:01:26.246 | INFO | "malloc" : " 14 GB", | 20260507 01:01:26.246 | INFO | "pinned" : " 29 GB" | 20260507 01:01:26.246 | INFO | } | 20260507 01:01:26.246 | INFO | Perf report for the last 5.13805 seconds: | 20260507 01:01:26.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:01:26.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:26.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:26.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:26.246 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:01:26.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:26.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:26.246 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 01:01:26.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:26.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:26.246 | INFO | Uploaded 36GiB to GPU in 1.99644s (18.0321GiB/s) | 20260507 01:01:26.246 | INFO | Downloaded 34GiB to GPU in 1.9763s (17.2039GiB/s) | 20260507 01:01:26.246 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:01:26.247 | INFO | Total mem usage: 18 GiB | 20260507 01:01:26.247 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:01:26.247 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:01:26.247 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:01:26.247 | INFO | High Water Breakdown: | 20260507 01:01:26.247 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:01:26.247 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:01:26.247 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:01:26.247 | INFO | ------------------------------------------ | 20260507 01:01:26.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:01:26.247 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:01:26.247 | INFO | Total mem usage: 26.5 GiB | 20260507 01:01:26.247 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:01:26.247 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:01:26.247 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:01:26.247 | INFO | High Water Breakdown: | 20260507 01:01:26.247 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:01:26.247 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:01:26.247 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:01:26.247 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:01:26.247 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:01:26.247 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:01:26.247 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:01:26.247 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:01:26.247 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:01:26.247 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:01:26.247 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:01:26.247 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:01:26.247 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:01:26.247 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:01:26.247 | INFO | ------------------------------------------ | 20260507 01:01:26.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:01:26.247 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:01:26.247 | INFO | Total mem usage: 25.6 GiB | 20260507 01:01:26.247 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:01:26.247 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:01:26.247 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:01:26.247 | INFO | High Water Breakdown: | 20260507 01:01:26.247 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:01:26.247 | INFO | ------------------------------------------ | 20260507 01:01:26.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:01:26.247 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:01:26.247 | INFO | Total mem usage: 15.7 GiB | 20260507 01:01:26.247 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:01:26.247 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:01:26.247 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:01:26.247 | INFO | High Water Breakdown: | 20260507 01:01:26.247 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:01:26.247 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:01:26.247 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:01:26.247 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:01:26.247 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:01:26.247 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:01:26.247 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:01:26.247 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:01:26.247 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:01:26.247 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:01:26.247 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:01:26.247 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:01:26.247 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:01:26.247 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:01:26.247 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:01:26.247 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:01:26.247 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:01:26.247 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:01:26.247 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:01:26.247 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:01:26.247 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:01:26.247 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:01:26.247 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:01:26.247 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:01:26.247 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:01:26.247 | INFO | ------------------------------------------ | 20260507 01:01:26.247 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:01:26.247 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:01:26.247 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:01:26.247 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:01:26.247 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.57e+01, 1.35e+01 / 2.17e+00 | 20260507 01:01:26.247 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.00e-01, 1.07e-02 / 5.98e-03 | 20260507 01:01:26.247 | INFO | PerfReport: 11.65%, Baseline -- 3.60e+00 / 4.42e+00, 5.31e-01 / 3.48e-01 | 20260507 01:01:26.247 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.44e+00, 1.72e-01 / 1.35e-01 | 20260507 01:01:26.247 | INFO | PerfReport: 16.07%, DME -- 3.97e+01 / 4.43e+01, 4.05e+00 / 2.49e+00 | 20260507 01:01:26.247 | INFO | PerfReport: 57.55%, FrameLabeling -- 1.78e+01 / 2.11e+01, 1.37e+00 / 2.00e+00 | 20260507 01:01:26.247 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.17e+00 / 2.70e+00, 3.77e-01 / 2.23e-01 | 20260507 01:01:26.247 | INFO | PerfReport: 3.13%, Metrics -- 9.66e-01 / 1.26e+00, 2.57e-01 / 1.42e-01 | 20260507 01:01:26.247 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.29e-01 / 3.82e-01, 5.26e-02 / 1.98e-02 | 20260507 01:01:26.247 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:01:26.247 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:01:26.247 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:01:26.247 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 01:01:26.247 | INFO | PerfReport: 21.69%, Upload -- 2.09e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260507 01:01:26.247 | INFO | PerfReport: 21.30%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 01:01:26.247 | INFO | PerfReport: 32.01%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 01:01:26.247 | INFO | PerfReport: 1.37%, LockUpload -- 1.32e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 01:01:26.247 | INFO | PerfReport: 1.26%, LockDownload -- 1.21e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260507 01:01:26.247 | INFO | PerfReport: 22.35%, LockCompute -- 2.16e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 01:01:26.247 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260507 01:01:26.345 | INFO | Starting a round of Baz IO | 20260507 01:01:29.460 | INFO | Populating RTMetrics report | 20260507 01:01:29.460 | INFO | RTMetrics report populated | 20260507 01:01:29.460 | INFO | Opening tmp RTMetrics json file | 20260507 01:01:29.461 | INFO | Serializing RTMetrics to json | 20260507 01:01:29.464 | INFO | Serialization done, swapping tmp file now... | 20260507 01:01:29.471 | INFO | RTMetrics done | 20260507 01:01:29.552 | INFO | Finished a round of Baz IO: Wrote 1.829 GiB in 3.207 seconds | 20260507 01:01:29.552 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.20% / 0.00% / 66.80% | 20260507 01:01:29.552 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.310357% / 3.97364e-06% / 5.96046e-05% | 20260507 01:01:29.552 | INFO | Baz IO overall: | 20260507 01:01:29.552 | INFO | BytesWritten : 3044961945738 | 20260507 01:01:29.552 | INFO | NumEvents : 1466173813878 | 20260507 01:01:31.382 | INFO | Analyzing chunk frames = [5857280,5857792) | 20260507 01:01:31.382 | INFO | MemInfo: { | 20260507 01:01:31.382 | INFO | "gpu" : " 17 GB", | 20260507 01:01:31.382 | INFO | "huge" : "", | 20260507 01:01:31.382 | INFO | "huge_shared" : " 27 GB", | 20260507 01:01:31.382 | INFO | "malloc" : " 14 GB", | 20260507 01:01:31.382 | INFO | "pinned" : " 29 GB" | 20260507 01:01:31.382 | INFO | } | 20260507 01:01:31.382 | INFO | Perf report for the last 5.13568 seconds: | 20260507 01:01:31.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:01:31.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:31.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:31.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:31.382 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260507 01:01:31.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:31.382 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 01:01:31.382 | INFO | PRE_HQ : 59.6%, 1.8, 56.7ms, 0 | 20260507 01:01:31.382 | INFO | BAZWRITER : 62.0%, 3.4, 107.7ms, 0 | 20260507 01:01:31.382 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:01:31.382 | INFO | Uploaded 36GiB to GPU in 2.02534s (17.7748GiB/s) | 20260507 01:01:31.382 | INFO | Downloaded 38GiB to GPU in 2.18369s (17.4017GiB/s) | 20260507 01:01:36.519 | INFO | Analyzing chunk frames = [5857792,5858304) | 20260507 01:01:36.519 | INFO | MemInfo: { | 20260507 01:01:36.519 | INFO | "gpu" : " 17 GB", | 20260507 01:01:36.519 | INFO | "huge" : "", | 20260507 01:01:36.519 | INFO | "huge_shared" : " 27 GB", | 20260507 01:01:36.519 | INFO | "malloc" : " 14 GB", | 20260507 01:01:36.519 | INFO | "pinned" : " 29 GB" | 20260507 01:01:36.519 | INFO | } | 20260507 01:01:36.519 | INFO | Perf report for the last 5.1368 seconds: | 20260507 01:01:36.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:01:36.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:36.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:36.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:36.519 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:01:36.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:36.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:36.519 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:01:36.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:36.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:36.519 | INFO | Uploaded 36GiB to GPU in 1.99074s (18.0838GiB/s) | 20260507 01:01:36.519 | INFO | Downloaded 34GiB to GPU in 1.93525s (17.5688GiB/s) | 20260507 01:01:41.666 | INFO | Analyzing chunk frames = [5858304,5858816) | 20260507 01:01:41.666 | INFO | MemInfo: { | 20260507 01:01:41.666 | INFO | "gpu" : " 17 GB", | 20260507 01:01:41.666 | INFO | "huge" : "", | 20260507 01:01:41.666 | INFO | "huge_shared" : " 27 GB", | 20260507 01:01:41.666 | INFO | "malloc" : " 14 GB", | 20260507 01:01:41.666 | INFO | "pinned" : " 29 GB" | 20260507 01:01:41.666 | INFO | } | 20260507 01:01:41.666 | INFO | Perf report for the last 5.1476 seconds: | 20260507 01:01:41.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:01:41.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:41.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:41.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:41.666 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 01:01:41.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:41.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:41.666 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:01:41.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:41.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:41.666 | INFO | Uploaded 36GiB to GPU in 1.96959s (18.2779GiB/s) | 20260507 01:01:41.667 | INFO | Downloaded 34GiB to GPU in 1.92907s (17.6251GiB/s) | 20260507 01:01:46.807 | INFO | Analyzing chunk frames = [5858816,5859328) | 20260507 01:01:46.807 | INFO | MemInfo: { | 20260507 01:01:46.807 | INFO | "gpu" : " 17 GB", | 20260507 01:01:46.807 | INFO | "huge" : "", | 20260507 01:01:46.807 | INFO | "huge_shared" : " 27 GB", | 20260507 01:01:46.807 | INFO | "malloc" : " 14 GB", | 20260507 01:01:46.807 | INFO | "pinned" : " 29 GB" | 20260507 01:01:46.807 | INFO | } | 20260507 01:01:46.807 | INFO | Perf report for the last 5.14094 seconds: | 20260507 01:01:46.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:01:46.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:46.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:46.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:46.807 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260507 01:01:46.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:46.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:46.807 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 01:01:46.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:46.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:46.807 | INFO | Uploaded 36GiB to GPU in 2.00447s (17.9599GiB/s) | 20260507 01:01:46.807 | INFO | Downloaded 34GiB to GPU in 1.91125s (17.7894GiB/s) | 20260507 01:01:51.944 | INFO | Analyzing chunk frames = [5859328,5859840) | 20260507 01:01:51.944 | INFO | MemInfo: { | 20260507 01:01:51.944 | INFO | "gpu" : " 17 GB", | 20260507 01:01:51.944 | INFO | "huge" : "", | 20260507 01:01:51.944 | INFO | "huge_shared" : " 27 GB", | 20260507 01:01:51.944 | INFO | "malloc" : " 14 GB", | 20260507 01:01:51.944 | INFO | "pinned" : " 29 GB" | 20260507 01:01:51.944 | INFO | } | 20260507 01:01:51.944 | INFO | Perf report for the last 5.13699 seconds: | 20260507 01:01:51.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:01:51.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:51.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:51.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:51.944 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 01:01:51.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:51.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:51.944 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:01:51.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:51.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:51.944 | INFO | Uploaded 36GiB to GPU in 2.00822s (17.9263GiB/s) | 20260507 01:01:51.945 | INFO | Downloaded 34GiB to GPU in 1.93887s (17.536GiB/s) | 20260507 01:01:57.084 | INFO | Analyzing chunk frames = [5859840,5860352) | 20260507 01:01:57.084 | INFO | MemInfo: { | 20260507 01:01:57.084 | INFO | "gpu" : " 17 GB", | 20260507 01:01:57.084 | INFO | "huge" : "", | 20260507 01:01:57.084 | INFO | "huge_shared" : " 27 GB", | 20260507 01:01:57.084 | INFO | "malloc" : " 14 GB", | 20260507 01:01:57.084 | INFO | "pinned" : " 29 GB" | 20260507 01:01:57.084 | INFO | } | 20260507 01:01:57.085 | INFO | Perf report for the last 5.14026 seconds: | 20260507 01:01:57.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:01:57.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:57.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:57.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:57.085 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 01:01:57.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:57.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:57.085 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:01:57.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:57.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:01:57.085 | INFO | Uploaded 36GiB to GPU in 2.04162s (17.633GiB/s) | 20260507 01:01:57.085 | INFO | Downloaded 34GiB to GPU in 1.9571s (17.3726GiB/s) | 20260507 01:02:02.224 | INFO | Analyzing chunk frames = [5860352,5860864) | 20260507 01:02:02.224 | INFO | MemInfo: { | 20260507 01:02:02.224 | INFO | "gpu" : " 17 GB", | 20260507 01:02:02.224 | INFO | "huge" : "", | 20260507 01:02:02.224 | INFO | "huge_shared" : " 27 GB", | 20260507 01:02:02.224 | INFO | "malloc" : " 14 GB", | 20260507 01:02:02.224 | INFO | "pinned" : " 29 GB" | 20260507 01:02:02.224 | INFO | } | 20260507 01:02:02.224 | INFO | Perf report for the last 5.13986 seconds: | 20260507 01:02:02.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:02:02.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:02.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:02.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:02.224 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:02:02.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:02.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:02.224 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 01:02:02.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:02.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:02.225 | INFO | Uploaded 36GiB to GPU in 1.97859s (18.1947GiB/s) | 20260507 01:02:02.225 | INFO | Downloaded 34GiB to GPU in 1.98514s (17.1272GiB/s) | 20260507 01:02:07.365 | INFO | Analyzing chunk frames = [5860864,5861376) | 20260507 01:02:07.365 | INFO | MemInfo: { | 20260507 01:02:07.365 | INFO | "gpu" : " 17 GB", | 20260507 01:02:07.365 | INFO | "huge" : "", | 20260507 01:02:07.365 | INFO | "huge_shared" : " 27 GB", | 20260507 01:02:07.365 | INFO | "malloc" : " 14 GB", | 20260507 01:02:07.365 | INFO | "pinned" : " 29 GB" | 20260507 01:02:07.365 | INFO | } | 20260507 01:02:07.365 | INFO | Perf report for the last 5.14025 seconds: | 20260507 01:02:07.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:02:07.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:07.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:07.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:07.365 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:02:07.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:07.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:07.365 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 01:02:07.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:07.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:07.365 | INFO | Uploaded 36GiB to GPU in 1.99148s (18.077GiB/s) | 20260507 01:02:07.365 | INFO | Downloaded 34GiB to GPU in 1.97289s (17.2336GiB/s) | 20260507 01:02:07.463 | INFO | Starting a round of Baz IO | 20260507 01:02:10.576 | INFO | Populating RTMetrics report | 20260507 01:02:10.577 | INFO | RTMetrics report populated | 20260507 01:02:10.577 | INFO | Opening tmp RTMetrics json file | 20260507 01:02:10.577 | INFO | Serializing RTMetrics to json | 20260507 01:02:10.579 | INFO | Serialization done, swapping tmp file now... | 20260507 01:02:10.579 | INFO | RTMetrics done | 20260507 01:02:10.672 | INFO | Finished a round of Baz IO: Wrote 1.828 GiB in 3.209 seconds | 20260507 01:02:10.672 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.20% / 0.00% / 66.80% | 20260507 01:02:10.672 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.308859% / 3.97364e-06% / 3.97364e-05% | 20260507 01:02:10.672 | INFO | Baz IO overall: | 20260507 01:02:10.672 | INFO | BytesWritten : 3046924903392 | 20260507 01:02:10.672 | INFO | NumEvents : 1467105284864 | 20260507 01:02:12.500 | INFO | Analyzing chunk frames = [5861376,5861888) | 20260507 01:02:12.500 | INFO | MemInfo: { | 20260507 01:02:12.500 | INFO | "gpu" : " 17 GB", | 20260507 01:02:12.500 | INFO | "huge" : "", | 20260507 01:02:12.500 | INFO | "huge_shared" : " 27 GB", | 20260507 01:02:12.500 | INFO | "malloc" : " 14 GB", | 20260507 01:02:12.500 | INFO | "pinned" : " 29 GB" | 20260507 01:02:12.500 | INFO | } | 20260507 01:02:12.500 | INFO | Perf report for the last 5.1354 seconds: | 20260507 01:02:12.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:02:12.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:12.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:12.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:12.500 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 01:02:12.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:12.500 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:02:12.500 | INFO | PRE_HQ : 59.4%, 1.8, 56.2ms, 0 | 20260507 01:02:12.500 | INFO | BAZWRITER : 62.0%, 3.4, 107.7ms, 0 | 20260507 01:02:12.500 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:02:12.500 | INFO | Uploaded 36GiB to GPU in 2.02798s (17.7517GiB/s) | 20260507 01:02:12.500 | INFO | Downloaded 38GiB to GPU in 2.18133s (17.4206GiB/s) | 20260507 01:02:17.638 | INFO | Analyzing chunk frames = [5861888,5862400) | 20260507 01:02:17.638 | INFO | MemInfo: { | 20260507 01:02:17.638 | INFO | "gpu" : " 17 GB", | 20260507 01:02:17.638 | INFO | "huge" : "", | 20260507 01:02:17.638 | INFO | "huge_shared" : " 27 GB", | 20260507 01:02:17.638 | INFO | "malloc" : " 14 GB", | 20260507 01:02:17.638 | INFO | "pinned" : " 29 GB" | 20260507 01:02:17.638 | INFO | } | 20260507 01:02:17.638 | INFO | Perf report for the last 5.13818 seconds: | 20260507 01:02:17.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:02:17.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:17.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:17.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:17.638 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:02:17.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:17.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:17.638 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:02:17.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:17.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:17.639 | INFO | Uploaded 36GiB to GPU in 1.99014s (18.0891GiB/s) | 20260507 01:02:17.639 | INFO | Downloaded 34GiB to GPU in 1.93448s (17.5757GiB/s) | 20260507 01:02:22.782 | INFO | Analyzing chunk frames = [5862400,5862912) | 20260507 01:02:22.782 | INFO | MemInfo: { | 20260507 01:02:22.782 | INFO | "gpu" : " 17 GB", | 20260507 01:02:22.782 | INFO | "huge" : "", | 20260507 01:02:22.782 | INFO | "huge_shared" : " 27 GB", | 20260507 01:02:22.782 | INFO | "malloc" : " 14 GB", | 20260507 01:02:22.782 | INFO | "pinned" : " 29 GB" | 20260507 01:02:22.782 | INFO | } | 20260507 01:02:22.782 | INFO | Perf report for the last 5.14409 seconds: | 20260507 01:02:22.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:02:22.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:22.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:22.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:22.782 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260507 01:02:22.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:22.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:22.783 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:02:22.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:22.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:22.783 | INFO | Uploaded 36GiB to GPU in 1.96762s (18.2962GiB/s) | 20260507 01:02:22.783 | INFO | Downloaded 34GiB to GPU in 1.93115s (17.6061GiB/s) | 20260507 01:02:27.918 | INFO | Analyzing chunk frames = [5862912,5863424) | 20260507 01:02:27.918 | INFO | MemInfo: { | 20260507 01:02:27.918 | INFO | "gpu" : " 17 GB", | 20260507 01:02:27.918 | INFO | "huge" : "", | 20260507 01:02:27.918 | INFO | "huge_shared" : " 27 GB", | 20260507 01:02:27.918 | INFO | "malloc" : " 14 GB", | 20260507 01:02:27.918 | INFO | "pinned" : " 29 GB" | 20260507 01:02:27.918 | INFO | } | 20260507 01:02:27.918 | INFO | Perf report for the last 5.13543 seconds: | 20260507 01:02:27.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:02:27.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:27.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:27.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:27.918 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 01:02:27.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:27.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:27.918 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:02:27.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:27.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:27.918 | INFO | Uploaded 36GiB to GPU in 1.96887s (18.2846GiB/s) | 20260507 01:02:27.918 | INFO | Downloaded 34GiB to GPU in 1.90993s (17.8017GiB/s) | 20260507 01:02:33.058 | INFO | Analyzing chunk frames = [5863424,5863936) | 20260507 01:02:33.058 | INFO | MemInfo: { | 20260507 01:02:33.058 | INFO | "gpu" : " 17 GB", | 20260507 01:02:33.058 | INFO | "huge" : "", | 20260507 01:02:33.058 | INFO | "huge_shared" : " 27 GB", | 20260507 01:02:33.058 | INFO | "malloc" : " 14 GB", | 20260507 01:02:33.058 | INFO | "pinned" : " 29 GB" | 20260507 01:02:33.058 | INFO | } | 20260507 01:02:33.058 | INFO | Perf report for the last 5.1399 seconds: | 20260507 01:02:33.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:02:33.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:33.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:33.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:33.058 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 01:02:33.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:33.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:33.058 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 01:02:33.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:33.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:33.058 | INFO | Uploaded 36GiB to GPU in 2.00093s (17.9916GiB/s) | 20260507 01:02:33.058 | INFO | Downloaded 34GiB to GPU in 1.95031s (17.4331GiB/s) | 20260507 01:02:38.200 | INFO | Analyzing chunk frames = [5863936,5864448) | 20260507 01:02:38.200 | INFO | MemInfo: { | 20260507 01:02:38.200 | INFO | "gpu" : " 17 GB", | 20260507 01:02:38.200 | INFO | "huge" : "", | 20260507 01:02:38.201 | INFO | "huge_shared" : " 27 GB", | 20260507 01:02:38.201 | INFO | "malloc" : " 14 GB", | 20260507 01:02:38.201 | INFO | "pinned" : " 29 GB" | 20260507 01:02:38.201 | INFO | } | 20260507 01:02:38.201 | INFO | Perf report for the last 5.14327 seconds: | 20260507 01:02:38.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:02:38.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:38.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:38.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:38.201 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 01:02:38.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:38.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:38.201 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:02:38.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:38.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:38.201 | INFO | Uploaded 36GiB to GPU in 2.04342s (17.6175GiB/s) | 20260507 01:02:38.201 | INFO | Downloaded 34GiB to GPU in 1.96157s (17.333GiB/s) | 20260507 01:02:43.331 | INFO | Analyzing chunk frames = [5864448,5864960) | 20260507 01:02:43.331 | INFO | MemInfo: { | 20260507 01:02:43.331 | INFO | "gpu" : " 17 GB", | 20260507 01:02:43.331 | INFO | "huge" : "", | 20260507 01:02:43.331 | INFO | "huge_shared" : " 27 GB", | 20260507 01:02:43.331 | INFO | "malloc" : " 14 GB", | 20260507 01:02:43.331 | INFO | "pinned" : " 29 GB" | 20260507 01:02:43.331 | INFO | } | 20260507 01:02:43.331 | INFO | Perf report for the last 5.13004 seconds: | 20260507 01:02:43.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:02:43.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:43.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:43.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:43.331 | INFO | ANALYSIS : 61.2%, 3.0, 95.0ms, 0 | 20260507 01:02:43.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:43.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:43.331 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 01:02:43.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:43.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:43.331 | INFO | Uploaded 36GiB to GPU in 1.98569s (18.1298GiB/s) | 20260507 01:02:43.331 | INFO | Downloaded 34GiB to GPU in 1.98255s (17.1496GiB/s) | 20260507 01:02:48.469 | INFO | Analyzing chunk frames = [5864960,5865472) | 20260507 01:02:48.469 | INFO | MemInfo: { | 20260507 01:02:48.469 | INFO | "gpu" : " 17 GB", | 20260507 01:02:48.469 | INFO | "huge" : "", | 20260507 01:02:48.469 | INFO | "huge_shared" : " 27 GB", | 20260507 01:02:48.469 | INFO | "malloc" : " 14 GB", | 20260507 01:02:48.469 | INFO | "pinned" : " 29 GB" | 20260507 01:02:48.469 | INFO | } | 20260507 01:02:48.469 | INFO | Perf report for the last 5.13808 seconds: | 20260507 01:02:48.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:02:48.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:48.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:48.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:48.469 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:02:48.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:48.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:48.469 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:02:48.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:48.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:48.469 | INFO | Uploaded 36GiB to GPU in 1.97692s (18.2101GiB/s) | 20260507 01:02:48.469 | INFO | Downloaded 34GiB to GPU in 1.97988s (17.1727GiB/s) | 20260507 01:02:48.469 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:02:48.469 | INFO | Total mem usage: 18 GiB | 20260507 01:02:48.469 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:02:48.469 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:02:48.469 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:02:48.470 | INFO | High Water Breakdown: | 20260507 01:02:48.470 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:02:48.470 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:02:48.470 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:02:48.470 | INFO | ------------------------------------------ | 20260507 01:02:48.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:02:48.470 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:02:48.470 | INFO | Total mem usage: 26.5 GiB | 20260507 01:02:48.470 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:02:48.470 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:02:48.470 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:02:48.470 | INFO | High Water Breakdown: | 20260507 01:02:48.470 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:02:48.470 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:02:48.470 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:02:48.470 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:02:48.470 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:02:48.470 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:02:48.470 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:02:48.470 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:02:48.470 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:02:48.470 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:02:48.470 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:02:48.470 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:02:48.470 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:02:48.470 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:02:48.470 | INFO | ------------------------------------------ | 20260507 01:02:48.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:02:48.470 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:02:48.470 | INFO | Total mem usage: 25.6 GiB | 20260507 01:02:48.470 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:02:48.470 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:02:48.470 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:02:48.470 | INFO | High Water Breakdown: | 20260507 01:02:48.470 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:02:48.470 | INFO | ------------------------------------------ | 20260507 01:02:48.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:02:48.470 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:02:48.470 | INFO | Total mem usage: 15.7 GiB | 20260507 01:02:48.470 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:02:48.470 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:02:48.470 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:02:48.470 | INFO | High Water Breakdown: | 20260507 01:02:48.470 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:02:48.470 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:02:48.470 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:02:48.470 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:02:48.470 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:02:48.470 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:02:48.470 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:02:48.470 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:02:48.470 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:02:48.470 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:02:48.470 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:02:48.470 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:02:48.470 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:02:48.470 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:02:48.470 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:02:48.470 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:02:48.470 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:02:48.470 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:02:48.470 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:02:48.471 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:02:48.471 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:02:48.471 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:02:48.471 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:02:48.471 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:02:48.471 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:02:48.471 | INFO | ------------------------------------------ | 20260507 01:02:48.471 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:02:48.471 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:02:48.471 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:02:48.471 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:02:48.471 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.55e+01, 1.33e+01 / 2.00e+00 | 20260507 01:02:48.471 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 9.89e-02, 9.79e-03 / 5.89e-03 | 20260507 01:02:48.471 | INFO | PerfReport: 11.66%, Baseline -- 3.60e+00 / 4.39e+00, 5.25e-01 / 3.50e-01 | 20260507 01:02:48.471 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.46e+00, 1.75e-01 / 1.28e-01 | 20260507 01:02:48.471 | INFO | PerfReport: 16.10%, DME -- 3.98e+01 / 4.45e+01, 4.15e+00 / 2.15e+00 | 20260507 01:02:48.471 | INFO | PerfReport: 57.51%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.35e+00 / 1.84e+00 | 20260507 01:02:48.471 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.17e+00 / 2.71e+00, 3.76e-01 / 2.34e-01 | 20260507 01:02:48.471 | INFO | PerfReport: 3.12%, Metrics -- 9.65e-01 / 1.28e+00, 2.57e-01 / 1.56e-01 | 20260507 01:02:48.471 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.28e-01 / 3.78e-01, 5.15e-02 / 1.99e-02 | 20260507 01:02:48.471 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:02:48.471 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:02:48.471 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:02:48.471 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 01:02:48.471 | INFO | PerfReport: 21.57%, Upload -- 2.08e+01 / 0.00e+00, 6.94e+00 / 0.00e+00 | 20260507 01:02:48.471 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.49e+00 / 0.00e+00 | 20260507 01:02:48.471 | INFO | PerfReport: 32.03%, Compute -- 3.09e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 01:02:48.471 | INFO | PerfReport: 1.29%, LockUpload -- 1.25e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 01:02:48.471 | INFO | PerfReport: 1.39%, LockDownload -- 1.34e+00 / 0.00e+00, 3.25e+00 / 0.00e+00 | 20260507 01:02:48.471 | INFO | PerfReport: 22.36%, LockCompute -- 2.16e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 01:02:48.471 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260507 01:02:48.567 | INFO | Starting a round of Baz IO | 20260507 01:02:51.692 | INFO | Populating RTMetrics report | 20260507 01:02:51.692 | INFO | RTMetrics report populated | 20260507 01:02:51.692 | INFO | Opening tmp RTMetrics json file | 20260507 01:02:51.711 | INFO | Serializing RTMetrics to json | 20260507 01:02:51.712 | INFO | Serialization done, swapping tmp file now... | 20260507 01:02:51.713 | INFO | RTMetrics done | 20260507 01:02:51.785 | INFO | Finished a round of Baz IO: Wrote 1.828 GiB in 3.218 seconds | 20260507 01:02:51.785 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.19% / 0.00% / 66.80% | 20260507 01:02:51.785 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.309487% / 0% / 5.5631e-05% | 20260507 01:02:51.785 | INFO | Baz IO overall: | 20260507 01:02:51.785 | INFO | BytesWritten : 3048887902567 | 20260507 01:02:51.785 | INFO | NumEvents : 1468036961054 | 20260507 01:02:53.615 | INFO | Analyzing chunk frames = [5865472,5865984) | 20260507 01:02:53.615 | INFO | MemInfo: { | 20260507 01:02:53.615 | INFO | "gpu" : " 17 GB", | 20260507 01:02:53.615 | INFO | "huge" : "", | 20260507 01:02:53.615 | INFO | "huge_shared" : " 27 GB", | 20260507 01:02:53.615 | INFO | "malloc" : " 14 GB", | 20260507 01:02:53.615 | INFO | "pinned" : " 29 GB" | 20260507 01:02:53.615 | INFO | } | 20260507 01:02:53.615 | INFO | Perf report for the last 5.14612 seconds: | 20260507 01:02:53.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:02:53.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:53.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:53.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:53.615 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260507 01:02:53.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:53.615 | INFO | RT_METRICS : 42.2%, 1.0, 22.2ms, 0 | 20260507 01:02:53.615 | INFO | PRE_HQ : 59.3%, 1.8, 55.9ms, 0 | 20260507 01:02:53.615 | INFO | BAZWRITER : 62.1%, 3.3, 106.0ms, 0 | 20260507 01:02:53.615 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:02:53.616 | INFO | Uploaded 36GiB to GPU in 2.01882s (17.8322GiB/s) | 20260507 01:02:53.616 | INFO | Downloaded 38GiB to GPU in 2.18835s (17.3647GiB/s) | 20260507 01:02:58.757 | INFO | Analyzing chunk frames = [5865984,5866496) | 20260507 01:02:58.757 | INFO | MemInfo: { | 20260507 01:02:58.757 | INFO | "gpu" : " 17 GB", | 20260507 01:02:58.757 | INFO | "huge" : "", | 20260507 01:02:58.757 | INFO | "huge_shared" : " 27 GB", | 20260507 01:02:58.757 | INFO | "malloc" : " 14 GB", | 20260507 01:02:58.757 | INFO | "pinned" : " 29 GB" | 20260507 01:02:58.757 | INFO | } | 20260507 01:02:58.757 | INFO | Perf report for the last 5.14203 seconds: | 20260507 01:02:58.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:02:58.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:58.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:58.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:58.757 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:02:58.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:58.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:58.757 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:02:58.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:58.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:02:58.758 | INFO | Uploaded 36GiB to GPU in 1.99352s (18.0585GiB/s) | 20260507 01:02:58.758 | INFO | Downloaded 34GiB to GPU in 1.93211s (17.5973GiB/s) | 20260507 01:03:03.895 | INFO | Analyzing chunk frames = [5866496,5867008) | 20260507 01:03:03.895 | INFO | MemInfo: { | 20260507 01:03:03.895 | INFO | "gpu" : " 17 GB", | 20260507 01:03:03.895 | INFO | "huge" : "", | 20260507 01:03:03.895 | INFO | "huge_shared" : " 27 GB", | 20260507 01:03:03.895 | INFO | "malloc" : " 14 GB", | 20260507 01:03:03.895 | INFO | "pinned" : " 29 GB" | 20260507 01:03:03.895 | INFO | } | 20260507 01:03:03.895 | INFO | Perf report for the last 5.13762 seconds: | 20260507 01:03:03.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:03:03.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:03.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:03.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:03.895 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:03:03.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:03.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:03.895 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:03:03.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:03.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:03.895 | INFO | Uploaded 36GiB to GPU in 2.00099s (17.9911GiB/s) | 20260507 01:03:03.895 | INFO | Downloaded 34GiB to GPU in 1.94046s (17.5217GiB/s) | 20260507 01:03:09.037 | INFO | Analyzing chunk frames = [5867008,5867520) | 20260507 01:03:09.037 | INFO | MemInfo: { | 20260507 01:03:09.037 | INFO | "gpu" : " 17 GB", | 20260507 01:03:09.037 | INFO | "huge" : "", | 20260507 01:03:09.038 | INFO | "huge_shared" : " 27 GB", | 20260507 01:03:09.038 | INFO | "malloc" : " 14 GB", | 20260507 01:03:09.038 | INFO | "pinned" : " 29 GB" | 20260507 01:03:09.038 | INFO | } | 20260507 01:03:09.038 | INFO | Perf report for the last 5.14261 seconds: | 20260507 01:03:09.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:03:09.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:09.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:09.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:09.038 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:03:09.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:09.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:09.038 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:03:09.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:09.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:09.038 | INFO | Uploaded 36GiB to GPU in 1.9675s (18.2974GiB/s) | 20260507 01:03:09.038 | INFO | Downloaded 34GiB to GPU in 1.90834s (17.8165GiB/s) | 20260507 01:03:14.173 | INFO | Analyzing chunk frames = [5867520,5868032) | 20260507 01:03:14.173 | INFO | MemInfo: { | 20260507 01:03:14.173 | INFO | "gpu" : " 17 GB", | 20260507 01:03:14.174 | INFO | "huge" : "", | 20260507 01:03:14.174 | INFO | "huge_shared" : " 27 GB", | 20260507 01:03:14.174 | INFO | "malloc" : " 14 GB", | 20260507 01:03:14.174 | INFO | "pinned" : " 29 GB" | 20260507 01:03:14.174 | INFO | } | 20260507 01:03:14.174 | INFO | Perf report for the last 5.13608 seconds: | 20260507 01:03:14.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:03:14.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:14.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:14.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:14.174 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 01:03:14.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:14.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:14.174 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:03:14.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:14.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:14.174 | INFO | Uploaded 36GiB to GPU in 1.99615s (18.0347GiB/s) | 20260507 01:03:14.174 | INFO | Downloaded 34GiB to GPU in 1.95152s (17.4223GiB/s) | 20260507 01:03:19.309 | INFO | Analyzing chunk frames = [5868032,5868544) | 20260507 01:03:19.309 | INFO | MemInfo: { | 20260507 01:03:19.309 | INFO | "gpu" : " 17 GB", | 20260507 01:03:19.309 | INFO | "huge" : "", | 20260507 01:03:19.309 | INFO | "huge_shared" : " 27 GB", | 20260507 01:03:19.309 | INFO | "malloc" : " 14 GB", | 20260507 01:03:19.309 | INFO | "pinned" : " 29 GB" | 20260507 01:03:19.309 | INFO | } | 20260507 01:03:19.309 | INFO | Perf report for the last 5.13496 seconds: | 20260507 01:03:19.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:03:19.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:19.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:19.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:19.309 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 01:03:19.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:19.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:19.309 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 01:03:19.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:19.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:19.309 | INFO | Uploaded 36GiB to GPU in 2.03163s (17.7197GiB/s) | 20260507 01:03:19.309 | INFO | Downloaded 34GiB to GPU in 1.9408s (17.5185GiB/s) | 20260507 01:03:24.455 | INFO | Analyzing chunk frames = [5868544,5869056) | 20260507 01:03:24.455 | INFO | MemInfo: { | 20260507 01:03:24.455 | INFO | "gpu" : " 17 GB", | 20260507 01:03:24.455 | INFO | "huge" : "", | 20260507 01:03:24.455 | INFO | "huge_shared" : " 27 GB", | 20260507 01:03:24.455 | INFO | "malloc" : " 14 GB", | 20260507 01:03:24.455 | INFO | "pinned" : " 29 GB" | 20260507 01:03:24.455 | INFO | } | 20260507 01:03:24.455 | INFO | Perf report for the last 5.14661 seconds: | 20260507 01:03:24.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:03:24.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:24.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:24.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:24.455 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260507 01:03:24.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:24.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:24.455 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 01:03:24.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:24.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:24.456 | INFO | Uploaded 36GiB to GPU in 1.97936s (18.1877GiB/s) | 20260507 01:03:24.456 | INFO | Downloaded 34GiB to GPU in 1.99949s (17.0043GiB/s) | 20260507 01:03:29.588 | INFO | Analyzing chunk frames = [5869056,5869568) | 20260507 01:03:29.588 | INFO | MemInfo: { | 20260507 01:03:29.588 | INFO | "gpu" : " 17 GB", | 20260507 01:03:29.588 | INFO | "huge" : "", | 20260507 01:03:29.588 | INFO | "huge_shared" : " 27 GB", | 20260507 01:03:29.588 | INFO | "malloc" : " 14 GB", | 20260507 01:03:29.588 | INFO | "pinned" : " 29 GB" | 20260507 01:03:29.588 | INFO | } | 20260507 01:03:29.588 | INFO | Perf report for the last 5.133 seconds: | 20260507 01:03:29.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:03:29.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:29.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:29.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:29.588 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260507 01:03:29.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:29.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:29.588 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 01:03:29.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:29.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:29.588 | INFO | Uploaded 36GiB to GPU in 1.99615s (18.0347GiB/s) | 20260507 01:03:29.588 | INFO | Downloaded 34GiB to GPU in 1.97385s (17.2252GiB/s) | 20260507 01:03:29.687 | INFO | Starting a round of Baz IO | 20260507 01:03:32.818 | INFO | Populating RTMetrics report | 20260507 01:03:32.818 | INFO | RTMetrics report populated | 20260507 01:03:32.818 | INFO | Opening tmp RTMetrics json file | 20260507 01:03:32.819 | INFO | Serializing RTMetrics to json | 20260507 01:03:32.820 | INFO | Serialization done, swapping tmp file now... | 20260507 01:03:32.821 | INFO | RTMetrics done | 20260507 01:03:32.916 | INFO | Finished a round of Baz IO: Wrote 1.828 GiB in 3.229 seconds | 20260507 01:03:32.916 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.19% / 0.00% / 66.81% | 20260507 01:03:32.916 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.308398% / 0% / 5.96046e-05% | 20260507 01:03:32.916 | INFO | Baz IO overall: | 20260507 01:03:32.916 | INFO | BytesWritten : 3050850300663 | 20260507 01:03:32.916 | INFO | NumEvents : 1468968250302 | 20260507 01:03:34.732 | INFO | Analyzing chunk frames = [5869568,5870080) | 20260507 01:03:34.732 | INFO | MemInfo: { | 20260507 01:03:34.732 | INFO | "gpu" : " 17 GB", | 20260507 01:03:34.732 | INFO | "huge" : "", | 20260507 01:03:34.732 | INFO | "huge_shared" : " 27 GB", | 20260507 01:03:34.732 | INFO | "malloc" : " 14 GB", | 20260507 01:03:34.732 | INFO | "pinned" : " 29 GB" | 20260507 01:03:34.732 | INFO | } | 20260507 01:03:34.732 | INFO | Perf report for the last 5.14339 seconds: | 20260507 01:03:34.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:03:34.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:34.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:34.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:34.732 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 01:03:34.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:34.732 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260507 01:03:34.732 | INFO | PRE_HQ : 59.9%, 1.9, 59.1ms, 0 | 20260507 01:03:34.732 | INFO | BAZWRITER : 62.4%, 3.4, 109.8ms, 0 | 20260507 01:03:34.732 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260507 01:03:34.732 | INFO | Uploaded 36GiB to GPU in 2.01205s (17.8922GiB/s) | 20260507 01:03:34.732 | INFO | Downloaded 38GiB to GPU in 2.17658s (17.4586GiB/s) | 20260507 01:03:39.873 | INFO | Analyzing chunk frames = [5870080,5870592) | 20260507 01:03:39.873 | INFO | MemInfo: { | 20260507 01:03:39.873 | INFO | "gpu" : " 17 GB", | 20260507 01:03:39.873 | INFO | "huge" : "", | 20260507 01:03:39.873 | INFO | "huge_shared" : " 27 GB", | 20260507 01:03:39.874 | INFO | "malloc" : " 14 GB", | 20260507 01:03:39.874 | INFO | "pinned" : " 29 GB" | 20260507 01:03:39.874 | INFO | } | 20260507 01:03:39.874 | INFO | Perf report for the last 5.14186 seconds: | 20260507 01:03:39.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:03:39.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:39.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:39.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:39.874 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:03:39.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:39.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:39.874 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 01:03:39.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:39.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:39.874 | INFO | Uploaded 36GiB to GPU in 2.02045s (17.8178GiB/s) | 20260507 01:03:39.874 | INFO | Downloaded 34GiB to GPU in 1.93642s (17.5581GiB/s) | 20260507 01:03:45.009 | INFO | Analyzing chunk frames = [5870592,5871104) | 20260507 01:03:45.009 | INFO | MemInfo: { | 20260507 01:03:45.009 | INFO | "gpu" : " 17 GB", | 20260507 01:03:45.009 | INFO | "huge" : "", | 20260507 01:03:45.009 | INFO | "huge_shared" : " 27 GB", | 20260507 01:03:45.009 | INFO | "malloc" : " 14 GB", | 20260507 01:03:45.009 | INFO | "pinned" : " 29 GB" | 20260507 01:03:45.009 | INFO | } | 20260507 01:03:45.009 | INFO | Perf report for the last 5.13529 seconds: | 20260507 01:03:45.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:03:45.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:45.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:45.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:45.009 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 01:03:45.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:45.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:45.009 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:03:45.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:45.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:45.009 | INFO | Uploaded 36GiB to GPU in 1.98104s (18.1723GiB/s) | 20260507 01:03:45.009 | INFO | Downloaded 34GiB to GPU in 1.94056s (17.5207GiB/s) | 20260507 01:03:50.146 | INFO | Analyzing chunk frames = [5871104,5871616) | 20260507 01:03:50.146 | INFO | MemInfo: { | 20260507 01:03:50.146 | INFO | "gpu" : " 17 GB", | 20260507 01:03:50.146 | INFO | "huge" : "", | 20260507 01:03:50.146 | INFO | "huge_shared" : " 27 GB", | 20260507 01:03:50.146 | INFO | "malloc" : " 14 GB", | 20260507 01:03:50.146 | INFO | "pinned" : " 29 GB" | 20260507 01:03:50.146 | INFO | } | 20260507 01:03:50.146 | INFO | Perf report for the last 5.13754 seconds: | 20260507 01:03:50.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:03:50.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:50.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:50.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:50.146 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:03:50.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:50.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:50.146 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 01:03:50.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:50.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:50.147 | INFO | Uploaded 36GiB to GPU in 1.97182s (18.2572GiB/s) | 20260507 01:03:50.147 | INFO | Downloaded 34GiB to GPU in 1.92057s (17.7031GiB/s) | 20260507 01:03:55.282 | INFO | Analyzing chunk frames = [5871616,5872128) | 20260507 01:03:55.282 | INFO | MemInfo: { | 20260507 01:03:55.282 | INFO | "gpu" : " 17 GB", | 20260507 01:03:55.282 | INFO | "huge" : "", | 20260507 01:03:55.282 | INFO | "huge_shared" : " 27 GB", | 20260507 01:03:55.282 | INFO | "malloc" : " 14 GB", | 20260507 01:03:55.282 | INFO | "pinned" : " 29 GB" | 20260507 01:03:55.282 | INFO | } | 20260507 01:03:55.282 | INFO | Perf report for the last 5.1355 seconds: | 20260507 01:03:55.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:03:55.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:55.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:55.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:55.282 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 01:03:55.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:55.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:55.282 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 01:03:55.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:55.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:03:55.282 | INFO | Uploaded 36GiB to GPU in 1.98948s (18.0952GiB/s) | 20260507 01:03:55.282 | INFO | Downloaded 34GiB to GPU in 1.94958s (17.4396GiB/s) | 20260507 01:04:00.426 | INFO | Analyzing chunk frames = [5872128,5872640) | 20260507 01:04:00.426 | INFO | MemInfo: { | 20260507 01:04:00.426 | INFO | "gpu" : " 17 GB", | 20260507 01:04:00.426 | INFO | "huge" : "", | 20260507 01:04:00.426 | INFO | "huge_shared" : " 27 GB", | 20260507 01:04:00.426 | INFO | "malloc" : " 14 GB", | 20260507 01:04:00.426 | INFO | "pinned" : " 29 GB" | 20260507 01:04:00.426 | INFO | } | 20260507 01:04:00.426 | INFO | Perf report for the last 5.14399 seconds: | 20260507 01:04:00.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:04:00.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:00.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:00.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:00.426 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 01:04:00.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:00.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:00.426 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:04:00.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:00.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:00.426 | INFO | Uploaded 36GiB to GPU in 2.00257s (17.9769GiB/s) | 20260507 01:04:00.426 | INFO | Downloaded 34GiB to GPU in 1.95336s (17.4059GiB/s) | 20260507 01:04:05.564 | INFO | Analyzing chunk frames = [5872640,5873152) | 20260507 01:04:05.564 | INFO | MemInfo: { | 20260507 01:04:05.564 | INFO | "gpu" : " 17 GB", | 20260507 01:04:05.564 | INFO | "huge" : "", | 20260507 01:04:05.564 | INFO | "huge_shared" : " 27 GB", | 20260507 01:04:05.564 | INFO | "malloc" : " 14 GB", | 20260507 01:04:05.564 | INFO | "pinned" : " 29 GB" | 20260507 01:04:05.564 | INFO | } | 20260507 01:04:05.565 | INFO | Perf report for the last 5.13856 seconds: | 20260507 01:04:05.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:04:05.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:05.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:05.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:05.565 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 01:04:05.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:05.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:05.565 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:04:05.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:05.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:05.565 | INFO | Uploaded 36GiB to GPU in 1.96s (18.3673GiB/s) | 20260507 01:04:05.565 | INFO | Downloaded 34GiB to GPU in 1.98681s (17.1129GiB/s) | 20260507 01:04:10.706 | INFO | Analyzing chunk frames = [5873152,5873664) | 20260507 01:04:10.706 | INFO | MemInfo: { | 20260507 01:04:10.706 | INFO | "gpu" : " 17 GB", | 20260507 01:04:10.706 | INFO | "huge" : "", | 20260507 01:04:10.706 | INFO | "huge_shared" : " 27 GB", | 20260507 01:04:10.706 | INFO | "malloc" : " 14 GB", | 20260507 01:04:10.706 | INFO | "pinned" : " 29 GB" | 20260507 01:04:10.706 | INFO | } | 20260507 01:04:10.706 | INFO | Perf report for the last 5.14153 seconds: | 20260507 01:04:10.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:04:10.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:10.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:10.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:10.706 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:04:10.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:10.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:10.706 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 01:04:10.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:10.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:10.706 | INFO | Uploaded 36GiB to GPU in 1.97985s (18.1832GiB/s) | 20260507 01:04:10.706 | INFO | Downloaded 34GiB to GPU in 1.99421s (17.0493GiB/s) | 20260507 01:04:10.706 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:04:10.706 | INFO | Total mem usage: 18 GiB | 20260507 01:04:10.706 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:04:10.706 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:04:10.706 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:04:10.706 | INFO | High Water Breakdown: | 20260507 01:04:10.706 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:04:10.706 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:04:10.706 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:04:10.706 | INFO | ------------------------------------------ | 20260507 01:04:10.706 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:04:10.706 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:04:10.706 | INFO | Total mem usage: 26.5 GiB | 20260507 01:04:10.706 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:04:10.706 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:04:10.706 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:04:10.706 | INFO | High Water Breakdown: | 20260507 01:04:10.706 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:04:10.706 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:04:10.707 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:04:10.707 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:04:10.707 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:04:10.707 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:04:10.707 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:04:10.707 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:04:10.707 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:04:10.707 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:04:10.707 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:04:10.707 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:04:10.707 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:04:10.707 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:04:10.707 | INFO | ------------------------------------------ | 20260507 01:04:10.707 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:04:10.707 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:04:10.707 | INFO | Total mem usage: 25.6 GiB | 20260507 01:04:10.707 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:04:10.707 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:04:10.707 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:04:10.707 | INFO | High Water Breakdown: | 20260507 01:04:10.707 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:04:10.707 | INFO | ------------------------------------------ | 20260507 01:04:10.707 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:04:10.707 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:04:10.707 | INFO | Total mem usage: 15.7 GiB | 20260507 01:04:10.707 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:04:10.707 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:04:10.707 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:04:10.707 | INFO | High Water Breakdown: | 20260507 01:04:10.707 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:04:10.707 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:04:10.707 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:04:10.707 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:04:10.707 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:04:10.707 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:04:10.707 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:04:10.707 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:04:10.707 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:04:10.707 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:04:10.707 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:04:10.707 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:04:10.707 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:04:10.707 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:04:10.707 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:04:10.707 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:04:10.707 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:04:10.707 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:04:10.707 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:04:10.707 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:04:10.707 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:04:10.707 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:04:10.707 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:04:10.707 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:04:10.707 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:04:10.707 | INFO | ------------------------------------------ | 20260507 01:04:10.707 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:04:10.707 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:04:10.707 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:04:10.707 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:04:10.707 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.57e+01, 1.34e+01 / 2.41e+00 | 20260507 01:04:10.707 | INFO | PerfReport: 0.30%, AggregateStats -- 9.15e-02 / 1.01e-01, 1.03e-02 / 4.67e-03 | 20260507 01:04:10.707 | INFO | PerfReport: 11.67%, Baseline -- 3.61e+00 / 4.40e+00, 5.34e-01 / 3.14e-01 | 20260507 01:04:10.707 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.43e+00, 1.73e-01 / 1.36e-01 | 20260507 01:04:10.707 | INFO | PerfReport: 16.06%, DME -- 3.97e+01 / 4.43e+01, 4.09e+00 / 2.66e+00 | 20260507 01:04:10.707 | INFO | PerfReport: 57.50%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.31e+00 / 1.53e+00 | 20260507 01:04:10.707 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.17e+00 / 2.73e+00, 3.79e-01 / 2.32e-01 | 20260507 01:04:10.707 | INFO | PerfReport: 3.13%, Metrics -- 9.67e-01 / 1.30e+00, 2.57e-01 / 1.49e-01 | 20260507 01:04:10.707 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.29e-01 / 3.77e-01, 5.22e-02 / 1.78e-02 | 20260507 01:04:10.707 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:04:10.707 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:04:10.707 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:04:10.707 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 01:04:10.707 | INFO | PerfReport: 21.53%, Upload -- 2.08e+01 / 0.00e+00, 6.85e+00 / 0.00e+00 | 20260507 01:04:10.707 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260507 01:04:10.707 | INFO | PerfReport: 32.03%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 01:04:10.707 | INFO | PerfReport: 1.26%, LockUpload -- 1.21e+00 / 0.00e+00, 4.76e+00 / 0.00e+00 | 20260507 01:04:10.707 | INFO | PerfReport: 1.41%, LockDownload -- 1.36e+00 / 0.00e+00, 3.33e+00 / 0.00e+00 | 20260507 01:04:10.707 | INFO | PerfReport: 22.38%, LockCompute -- 2.16e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 01:04:10.707 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260507 01:04:10.804 | INFO | Starting a round of Baz IO | 20260507 01:04:13.932 | INFO | Populating RTMetrics report | 20260507 01:04:13.932 | INFO | RTMetrics report populated | 20260507 01:04:13.932 | INFO | Opening tmp RTMetrics json file | 20260507 01:04:13.933 | INFO | Serializing RTMetrics to json | 20260507 01:04:13.942 | INFO | Serialization done, swapping tmp file now... | 20260507 01:04:13.953 | INFO | RTMetrics done | 20260507 01:04:14.030 | INFO | Finished a round of Baz IO: Wrote 1.827 GiB in 3.226 seconds | 20260507 01:04:14.030 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.19% / 0.00% / 66.81% | 20260507 01:04:14.030 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.307957% / 3.97364e-06% / 8.74201e-05% | 20260507 01:04:14.030 | INFO | Baz IO overall: | 20260507 01:04:14.031 | INFO | BytesWritten : 3052812060701 | 20260507 01:04:14.031 | INFO | NumEvents : 1469899140916 | 20260507 01:04:15.841 | INFO | Analyzing chunk frames = [5873664,5874176) | 20260507 01:04:15.841 | INFO | MemInfo: { | 20260507 01:04:15.841 | INFO | "gpu" : " 17 GB", | 20260507 01:04:15.841 | INFO | "huge" : "", | 20260507 01:04:15.841 | INFO | "huge_shared" : " 27 GB", | 20260507 01:04:15.841 | INFO | "malloc" : " 14 GB", | 20260507 01:04:15.841 | INFO | "pinned" : " 29 GB" | 20260507 01:04:15.841 | INFO | } | 20260507 01:04:15.841 | INFO | Perf report for the last 5.13504 seconds: | 20260507 01:04:15.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:04:15.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:15.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:15.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:15.841 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 01:04:15.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:15.841 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 01:04:15.841 | INFO | PRE_HQ : 59.7%, 1.8, 56.2ms, 0 | 20260507 01:04:15.841 | INFO | BAZWRITER : 62.4%, 3.3, 107.0ms, 0 | 20260507 01:04:15.841 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:04:15.841 | INFO | Uploaded 36GiB to GPU in 2.04121s (17.6366GiB/s) | 20260507 01:04:15.842 | INFO | Downloaded 38GiB to GPU in 2.1766s (17.4584GiB/s) | 20260507 01:04:20.979 | INFO | Analyzing chunk frames = [5874176,5874688) | 20260507 01:04:20.979 | INFO | MemInfo: { | 20260507 01:04:20.979 | INFO | "gpu" : " 17 GB", | 20260507 01:04:20.979 | INFO | "huge" : "", | 20260507 01:04:20.979 | INFO | "huge_shared" : " 27 GB", | 20260507 01:04:20.979 | INFO | "malloc" : " 14 GB", | 20260507 01:04:20.979 | INFO | "pinned" : " 29 GB" | 20260507 01:04:20.979 | INFO | } | 20260507 01:04:20.979 | INFO | Perf report for the last 5.13772 seconds: | 20260507 01:04:20.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:04:20.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:20.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:20.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:20.979 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:04:20.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:20.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:20.979 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:04:20.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:20.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:20.979 | INFO | Uploaded 36GiB to GPU in 2.00995s (17.9109GiB/s) | 20260507 01:04:20.979 | INFO | Downloaded 34GiB to GPU in 1.93446s (17.5759GiB/s) | 20260507 01:04:26.124 | INFO | Analyzing chunk frames = [5874688,5875200) | 20260507 01:04:26.124 | INFO | MemInfo: { | 20260507 01:04:26.124 | INFO | "gpu" : " 17 GB", | 20260507 01:04:26.124 | INFO | "huge" : "", | 20260507 01:04:26.124 | INFO | "huge_shared" : " 27 GB", | 20260507 01:04:26.124 | INFO | "malloc" : " 14 GB", | 20260507 01:04:26.124 | INFO | "pinned" : " 29 GB" | 20260507 01:04:26.124 | INFO | } | 20260507 01:04:26.124 | INFO | Perf report for the last 5.1451 seconds: | 20260507 01:04:26.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:04:26.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:26.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:26.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:26.124 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 01:04:26.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:26.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:26.124 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:04:26.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:26.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:26.125 | INFO | Uploaded 36GiB to GPU in 1.99042s (18.0866GiB/s) | 20260507 01:04:26.125 | INFO | Downloaded 34GiB to GPU in 1.93528s (17.5685GiB/s) | 20260507 01:04:31.259 | INFO | Analyzing chunk frames = [5875200,5875712) | 20260507 01:04:31.259 | INFO | MemInfo: { | 20260507 01:04:31.259 | INFO | "gpu" : " 17 GB", | 20260507 01:04:31.259 | INFO | "huge" : "", | 20260507 01:04:31.259 | INFO | "huge_shared" : " 27 GB", | 20260507 01:04:31.259 | INFO | "malloc" : " 14 GB", | 20260507 01:04:31.259 | INFO | "pinned" : " 29 GB" | 20260507 01:04:31.259 | INFO | } | 20260507 01:04:31.259 | INFO | Perf report for the last 5.13496 seconds: | 20260507 01:04:31.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:04:31.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:31.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:31.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:31.259 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 01:04:31.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:31.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:31.259 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:04:31.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:31.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:31.259 | INFO | Uploaded 36GiB to GPU in 1.98126s (18.1702GiB/s) | 20260507 01:04:31.259 | INFO | Downloaded 34GiB to GPU in 1.91322s (17.7711GiB/s) | 20260507 01:04:36.403 | INFO | Analyzing chunk frames = [5875712,5876224) | 20260507 01:04:36.403 | INFO | MemInfo: { | 20260507 01:04:36.403 | INFO | "gpu" : " 17 GB", | 20260507 01:04:36.403 | INFO | "huge" : "", | 20260507 01:04:36.403 | INFO | "huge_shared" : " 27 GB", | 20260507 01:04:36.403 | INFO | "malloc" : " 14 GB", | 20260507 01:04:36.403 | INFO | "pinned" : " 29 GB" | 20260507 01:04:36.403 | INFO | } | 20260507 01:04:36.403 | INFO | Perf report for the last 5.14399 seconds: | 20260507 01:04:36.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:04:36.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:36.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:36.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:36.403 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 01:04:36.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:36.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:36.403 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:04:36.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:36.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:36.403 | INFO | Uploaded 36GiB to GPU in 2.00992s (17.9112GiB/s) | 20260507 01:04:36.403 | INFO | Downloaded 34GiB to GPU in 1.94049s (17.5214GiB/s) | 20260507 01:04:41.543 | INFO | Analyzing chunk frames = [5876224,5876736) | 20260507 01:04:41.543 | INFO | MemInfo: { | 20260507 01:04:41.543 | INFO | "gpu" : " 17 GB", | 20260507 01:04:41.543 | INFO | "huge" : "", | 20260507 01:04:41.543 | INFO | "huge_shared" : " 27 GB", | 20260507 01:04:41.543 | INFO | "malloc" : " 14 GB", | 20260507 01:04:41.543 | INFO | "pinned" : " 29 GB" | 20260507 01:04:41.543 | INFO | } | 20260507 01:04:41.543 | INFO | Perf report for the last 5.14019 seconds: | 20260507 01:04:41.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:04:41.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:41.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:41.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:41.543 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 01:04:41.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:41.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:41.543 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:04:41.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:41.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:41.543 | INFO | Uploaded 36GiB to GPU in 2.02753s (17.7556GiB/s) | 20260507 01:04:41.543 | INFO | Downloaded 34GiB to GPU in 1.93603s (17.5617GiB/s) | 20260507 01:04:46.684 | INFO | Analyzing chunk frames = [5876736,5877248) | 20260507 01:04:46.684 | INFO | MemInfo: { | 20260507 01:04:46.685 | INFO | "gpu" : " 17 GB", | 20260507 01:04:46.685 | INFO | "huge" : "", | 20260507 01:04:46.685 | INFO | "huge_shared" : " 27 GB", | 20260507 01:04:46.685 | INFO | "malloc" : " 14 GB", | 20260507 01:04:46.685 | INFO | "pinned" : " 29 GB" | 20260507 01:04:46.685 | INFO | } | 20260507 01:04:46.685 | INFO | Perf report for the last 5.14152 seconds: | 20260507 01:04:46.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:04:46.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:46.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:46.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:46.685 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:04:46.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:46.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:46.685 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:04:46.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:46.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:46.685 | INFO | Uploaded 36GiB to GPU in 1.97389s (18.2381GiB/s) | 20260507 01:04:46.685 | INFO | Downloaded 34GiB to GPU in 1.99928s (17.0061GiB/s) | 20260507 01:04:51.822 | INFO | Analyzing chunk frames = [5877248,5877760) | 20260507 01:04:51.822 | INFO | MemInfo: { | 20260507 01:04:51.822 | INFO | "gpu" : " 17 GB", | 20260507 01:04:51.822 | INFO | "huge" : "", | 20260507 01:04:51.822 | INFO | "huge_shared" : " 27 GB", | 20260507 01:04:51.822 | INFO | "malloc" : " 14 GB", | 20260507 01:04:51.822 | INFO | "pinned" : " 29 GB" | 20260507 01:04:51.822 | INFO | } | 20260507 01:04:51.822 | INFO | Perf report for the last 5.13778 seconds: | 20260507 01:04:51.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:04:51.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:51.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:51.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:51.822 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:04:51.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:51.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:51.823 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 01:04:51.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:51.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:51.823 | INFO | Uploaded 36GiB to GPU in 2.00347s (17.9688GiB/s) | 20260507 01:04:51.823 | INFO | Downloaded 34GiB to GPU in 1.9759s (17.2073GiB/s) | 20260507 01:04:51.921 | INFO | Starting a round of Baz IO | 20260507 01:04:55.035 | INFO | Populating RTMetrics report | 20260507 01:04:55.036 | INFO | RTMetrics report populated | 20260507 01:04:55.036 | INFO | Opening tmp RTMetrics json file | 20260507 01:04:55.036 | INFO | Serializing RTMetrics to json | 20260507 01:04:55.038 | INFO | Serialization done, swapping tmp file now... | 20260507 01:04:55.038 | INFO | RTMetrics done | 20260507 01:04:55.127 | INFO | Finished a round of Baz IO: Wrote 1.826 GiB in 3.206 seconds | 20260507 01:04:55.127 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.19% / 0.00% / 66.81% | 20260507 01:04:55.127 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.307461% / 3.97364e-06% / 6.75519e-05% | 20260507 01:04:55.127 | INFO | Baz IO overall: | 20260507 01:04:55.127 | INFO | BytesWritten : 3054772778965 | 20260507 01:04:55.127 | INFO | NumEvents : 1470829011083 | 20260507 01:04:56.958 | INFO | Analyzing chunk frames = [5877760,5878272) | 20260507 01:04:56.958 | INFO | MemInfo: { | 20260507 01:04:56.958 | INFO | "gpu" : " 17 GB", | 20260507 01:04:56.958 | INFO | "huge" : "", | 20260507 01:04:56.958 | INFO | "huge_shared" : " 27 GB", | 20260507 01:04:56.958 | INFO | "malloc" : " 14 GB", | 20260507 01:04:56.958 | INFO | "pinned" : " 29 GB" | 20260507 01:04:56.958 | INFO | } | 20260507 01:04:56.958 | INFO | Perf report for the last 5.13539 seconds: | 20260507 01:04:56.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:04:56.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:56.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:56.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:56.958 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 01:04:56.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:04:56.958 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:04:56.958 | INFO | PRE_HQ : 59.4%, 1.8, 56.3ms, 0 | 20260507 01:04:56.958 | INFO | BAZWRITER : 62.0%, 3.3, 106.3ms, 0 | 20260507 01:04:56.958 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:04:56.958 | INFO | Uploaded 36GiB to GPU in 2.04721s (17.5849GiB/s) | 20260507 01:04:56.958 | INFO | Downloaded 38GiB to GPU in 2.17547s (17.4675GiB/s) | 20260507 01:05:02.095 | INFO | Analyzing chunk frames = [5878272,5878784) | 20260507 01:05:02.095 | INFO | MemInfo: { | 20260507 01:05:02.095 | INFO | "gpu" : " 17 GB", | 20260507 01:05:02.095 | INFO | "huge" : "", | 20260507 01:05:02.095 | INFO | "huge_shared" : " 27 GB", | 20260507 01:05:02.095 | INFO | "malloc" : " 14 GB", | 20260507 01:05:02.095 | INFO | "pinned" : " 29 GB" | 20260507 01:05:02.095 | INFO | } | 20260507 01:05:02.095 | INFO | Perf report for the last 5.13734 seconds: | 20260507 01:05:02.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:05:02.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:02.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:02.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:02.095 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:05:02.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:02.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:02.095 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:05:02.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:02.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:02.095 | INFO | Uploaded 36GiB to GPU in 2.01003s (17.9102GiB/s) | 20260507 01:05:02.095 | INFO | Downloaded 34GiB to GPU in 1.9329s (17.5902GiB/s) | 20260507 01:05:07.238 | INFO | Analyzing chunk frames = [5878784,5879296) | 20260507 01:05:07.238 | INFO | MemInfo: { | 20260507 01:05:07.238 | INFO | "gpu" : " 17 GB", | 20260507 01:05:07.238 | INFO | "huge" : "", | 20260507 01:05:07.238 | INFO | "huge_shared" : " 27 GB", | 20260507 01:05:07.238 | INFO | "malloc" : " 14 GB", | 20260507 01:05:07.238 | INFO | "pinned" : " 29 GB" | 20260507 01:05:07.238 | INFO | } | 20260507 01:05:07.238 | INFO | Perf report for the last 5.14286 seconds: | 20260507 01:05:07.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:05:07.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:07.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:07.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:07.238 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260507 01:05:07.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:07.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:07.238 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:05:07.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:07.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:07.238 | INFO | Uploaded 36GiB to GPU in 1.99416s (18.0528GiB/s) | 20260507 01:05:07.238 | INFO | Downloaded 34GiB to GPU in 1.92768s (17.6377GiB/s) | 20260507 01:05:12.379 | INFO | Analyzing chunk frames = [5879296,5879808) | 20260507 01:05:12.379 | INFO | MemInfo: { | 20260507 01:05:12.379 | INFO | "gpu" : " 17 GB", | 20260507 01:05:12.379 | INFO | "huge" : "", | 20260507 01:05:12.379 | INFO | "huge_shared" : " 27 GB", | 20260507 01:05:12.379 | INFO | "malloc" : " 14 GB", | 20260507 01:05:12.379 | INFO | "pinned" : " 29 GB" | 20260507 01:05:12.379 | INFO | } | 20260507 01:05:12.379 | INFO | Perf report for the last 5.14082 seconds: | 20260507 01:05:12.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:05:12.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:12.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:12.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:12.379 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 01:05:12.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:12.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:12.379 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:05:12.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:12.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:12.379 | INFO | Uploaded 36GiB to GPU in 1.99322s (18.0612GiB/s) | 20260507 01:05:12.379 | INFO | Downloaded 34GiB to GPU in 1.91002s (17.8009GiB/s) | 20260507 01:05:17.516 | INFO | Analyzing chunk frames = [5879808,5880320) | 20260507 01:05:17.516 | INFO | MemInfo: { | 20260507 01:05:17.516 | INFO | "gpu" : " 17 GB", | 20260507 01:05:17.516 | INFO | "huge" : "", | 20260507 01:05:17.516 | INFO | "huge_shared" : " 27 GB", | 20260507 01:05:17.516 | INFO | "malloc" : " 14 GB", | 20260507 01:05:17.516 | INFO | "pinned" : " 29 GB" | 20260507 01:05:17.516 | INFO | } | 20260507 01:05:17.516 | INFO | Perf report for the last 5.13677 seconds: | 20260507 01:05:17.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:05:17.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:17.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:17.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:17.516 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 01:05:17.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:17.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:17.516 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:05:17.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:17.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:17.516 | INFO | Uploaded 36GiB to GPU in 2.00324s (17.9709GiB/s) | 20260507 01:05:17.516 | INFO | Downloaded 34GiB to GPU in 1.94033s (17.5228GiB/s) | 20260507 01:05:22.658 | INFO | Analyzing chunk frames = [5880320,5880832) | 20260507 01:05:22.658 | INFO | MemInfo: { | 20260507 01:05:22.658 | INFO | "gpu" : " 17 GB", | 20260507 01:05:22.658 | INFO | "huge" : "", | 20260507 01:05:22.658 | INFO | "huge_shared" : " 27 GB", | 20260507 01:05:22.658 | INFO | "malloc" : " 14 GB", | 20260507 01:05:22.658 | INFO | "pinned" : " 29 GB" | 20260507 01:05:22.658 | INFO | } | 20260507 01:05:22.658 | INFO | Perf report for the last 5.1425 seconds: | 20260507 01:05:22.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:05:22.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:22.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:22.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:22.658 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 01:05:22.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:22.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:22.658 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:05:22.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:22.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:22.659 | INFO | Uploaded 36GiB to GPU in 2.01374s (17.8772GiB/s) | 20260507 01:05:22.659 | INFO | Downloaded 34GiB to GPU in 1.93605s (17.5615GiB/s) | 20260507 01:05:27.798 | INFO | Analyzing chunk frames = [5880832,5881344) | 20260507 01:05:27.798 | INFO | MemInfo: { | 20260507 01:05:27.798 | INFO | "gpu" : " 17 GB", | 20260507 01:05:27.798 | INFO | "huge" : "", | 20260507 01:05:27.798 | INFO | "huge_shared" : " 27 GB", | 20260507 01:05:27.798 | INFO | "malloc" : " 14 GB", | 20260507 01:05:27.798 | INFO | "pinned" : " 29 GB" | 20260507 01:05:27.798 | INFO | } | 20260507 01:05:27.799 | INFO | Perf report for the last 5.1404 seconds: | 20260507 01:05:27.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:05:27.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:27.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:27.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:27.799 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 01:05:27.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:27.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:27.799 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:05:27.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:27.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:27.799 | INFO | Uploaded 36GiB to GPU in 2.00727s (17.9348GiB/s) | 20260507 01:05:27.799 | INFO | Downloaded 34GiB to GPU in 1.9967s (17.0281GiB/s) | 20260507 01:05:32.932 | INFO | Analyzing chunk frames = [5881344,5881856) | 20260507 01:05:32.932 | INFO | MemInfo: { | 20260507 01:05:32.932 | INFO | "gpu" : " 17 GB", | 20260507 01:05:32.932 | INFO | "huge" : "", | 20260507 01:05:32.932 | INFO | "huge_shared" : " 27 GB", | 20260507 01:05:32.932 | INFO | "malloc" : " 14 GB", | 20260507 01:05:32.932 | INFO | "pinned" : " 29 GB" | 20260507 01:05:32.932 | INFO | } | 20260507 01:05:32.932 | INFO | Perf report for the last 5.13327 seconds: | 20260507 01:05:32.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:05:32.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:32.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:32.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:32.932 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:05:32.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:32.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:32.932 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260507 01:05:32.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:32.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:32.932 | INFO | Uploaded 36GiB to GPU in 2.00615s (17.9448GiB/s) | 20260507 01:05:32.932 | INFO | Downloaded 34GiB to GPU in 1.99286s (17.0609GiB/s) | 20260507 01:05:32.932 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:05:32.932 | INFO | Total mem usage: 18 GiB | 20260507 01:05:32.932 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:05:32.932 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:05:32.932 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:05:32.932 | INFO | High Water Breakdown: | 20260507 01:05:32.932 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:05:32.932 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:05:32.932 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:05:32.932 | INFO | ------------------------------------------ | 20260507 01:05:32.932 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:05:32.932 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:05:32.932 | INFO | Total mem usage: 26.5 GiB | 20260507 01:05:32.932 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:05:32.932 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:05:32.932 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:05:32.932 | INFO | High Water Breakdown: | 20260507 01:05:32.932 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:05:32.932 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:05:32.932 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:05:32.932 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:05:32.932 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:05:32.932 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:05:32.932 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:05:32.932 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:05:32.932 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:05:32.932 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:05:32.932 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:05:32.933 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:05:32.933 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:05:32.933 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:05:32.933 | INFO | ------------------------------------------ | 20260507 01:05:32.933 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:05:32.933 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:05:32.933 | INFO | Total mem usage: 25.6 GiB | 20260507 01:05:32.933 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:05:32.933 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:05:32.933 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:05:32.933 | INFO | High Water Breakdown: | 20260507 01:05:32.933 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:05:32.933 | INFO | ------------------------------------------ | 20260507 01:05:32.933 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:05:32.933 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:05:32.933 | INFO | Total mem usage: 15.7 GiB | 20260507 01:05:32.933 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:05:32.933 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:05:32.933 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:05:32.933 | INFO | High Water Breakdown: | 20260507 01:05:32.933 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:05:32.933 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:05:32.933 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:05:32.933 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:05:32.933 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:05:32.933 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:05:32.933 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:05:32.933 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:05:32.933 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:05:32.933 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:05:32.933 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:05:32.933 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:05:32.933 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:05:32.933 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:05:32.933 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:05:32.933 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:05:32.933 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:05:32.933 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:05:32.933 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:05:32.933 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:05:32.933 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:05:32.933 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:05:32.933 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:05:32.933 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:05:32.933 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:05:32.933 | INFO | ------------------------------------------ | 20260507 01:05:32.933 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:05:32.933 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:05:32.933 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:05:32.933 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:05:32.933 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.58e+01, 1.35e+01 / 2.04e+00 | 20260507 01:05:32.933 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 9.97e-02, 9.44e-03 / 4.64e-03 | 20260507 01:05:32.933 | INFO | PerfReport: 11.67%, Baseline -- 3.61e+00 / 4.43e+00, 5.37e-01 / 3.28e-01 | 20260507 01:05:32.933 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.46e+00, 1.74e-01 / 1.28e-01 | 20260507 01:05:32.933 | INFO | PerfReport: 16.11%, DME -- 3.98e+01 / 4.47e+01, 4.11e+00 / 2.26e+00 | 20260507 01:05:32.933 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.30e+00 / 1.68e+00 | 20260507 01:05:32.933 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.17e+00 / 2.69e+00, 3.80e-01 / 2.21e-01 | 20260507 01:05:32.933 | INFO | PerfReport: 3.13%, Metrics -- 9.67e-01 / 1.30e+00, 2.59e-01 / 1.49e-01 | 20260507 01:05:32.933 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.28e-01 / 3.85e-01, 5.14e-02 / 2.39e-02 | 20260507 01:05:32.933 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:05:32.933 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:05:32.933 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:05:32.933 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 01:05:32.933 | INFO | PerfReport: 21.66%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260507 01:05:32.933 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260507 01:05:32.933 | INFO | PerfReport: 32.01%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 01:05:32.933 | INFO | PerfReport: 1.37%, LockUpload -- 1.32e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 01:05:32.933 | INFO | PerfReport: 1.28%, LockDownload -- 1.24e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260507 01:05:32.933 | INFO | PerfReport: 22.33%, LockCompute -- 2.16e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 01:05:32.933 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.97e-02 / 0.00e+00 | 20260507 01:05:33.030 | INFO | Starting a round of Baz IO | 20260507 01:05:36.147 | INFO | Populating RTMetrics report | 20260507 01:05:36.147 | INFO | RTMetrics report populated | 20260507 01:05:36.147 | INFO | Opening tmp RTMetrics json file | 20260507 01:05:36.148 | INFO | Serializing RTMetrics to json | 20260507 01:05:36.150 | INFO | Serialization done, swapping tmp file now... | 20260507 01:05:36.153 | INFO | RTMetrics done | 20260507 01:05:36.245 | INFO | Finished a round of Baz IO: Wrote 1.828 GiB in 3.215 seconds | 20260507 01:05:36.245 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.19% / 0.00% / 66.81% | 20260507 01:05:36.245 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.307361% / 0% / 7.94729e-05% | 20260507 01:05:36.246 | INFO | Baz IO overall: | 20260507 01:05:36.246 | INFO | BytesWritten : 3056735357085 | 20260507 01:05:36.246 | INFO | NumEvents : 1471760314772 | 20260507 01:05:38.072 | INFO | Analyzing chunk frames = [5881856,5882368) | 20260507 01:05:38.072 | INFO | MemInfo: { | 20260507 01:05:38.072 | INFO | "gpu" : " 17 GB", | 20260507 01:05:38.072 | INFO | "huge" : "", | 20260507 01:05:38.072 | INFO | "huge_shared" : " 27 GB", | 20260507 01:05:38.072 | INFO | "malloc" : " 14 GB", | 20260507 01:05:38.072 | INFO | "pinned" : " 29 GB" | 20260507 01:05:38.072 | INFO | } | 20260507 01:05:38.072 | INFO | Perf report for the last 5.14015 seconds: | 20260507 01:05:38.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:05:38.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:38.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:38.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:38.072 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 01:05:38.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:38.072 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260507 01:05:38.072 | INFO | PRE_HQ : 59.9%, 1.9, 58.7ms, 0 | 20260507 01:05:38.072 | INFO | BAZWRITER : 62.0%, 3.5, 110.6ms, 0 | 20260507 01:05:38.073 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:05:38.073 | INFO | Uploaded 36GiB to GPU in 2.02766s (17.7545GiB/s) | 20260507 01:05:38.073 | INFO | Downloaded 38GiB to GPU in 2.17986s (17.4324GiB/s) | 20260507 01:05:43.212 | INFO | Analyzing chunk frames = [5882368,5882880) | 20260507 01:05:43.212 | INFO | MemInfo: { | 20260507 01:05:43.212 | INFO | "gpu" : " 17 GB", | 20260507 01:05:43.212 | INFO | "huge" : "", | 20260507 01:05:43.212 | INFO | "huge_shared" : " 27 GB", | 20260507 01:05:43.212 | INFO | "malloc" : " 14 GB", | 20260507 01:05:43.212 | INFO | "pinned" : " 29 GB" | 20260507 01:05:43.212 | INFO | } | 20260507 01:05:43.212 | INFO | Perf report for the last 5.1402 seconds: | 20260507 01:05:43.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:05:43.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:43.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:43.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:43.212 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:05:43.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:43.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:43.212 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 01:05:43.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:43.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:43.213 | INFO | Uploaded 36GiB to GPU in 2.02788s (17.7525GiB/s) | 20260507 01:05:43.213 | INFO | Downloaded 34GiB to GPU in 1.93338s (17.5858GiB/s) | 20260507 01:05:48.351 | INFO | Analyzing chunk frames = [5882880,5883392) | 20260507 01:05:48.351 | INFO | MemInfo: { | 20260507 01:05:48.351 | INFO | "gpu" : " 17 GB", | 20260507 01:05:48.351 | INFO | "huge" : "", | 20260507 01:05:48.351 | INFO | "huge_shared" : " 27 GB", | 20260507 01:05:48.351 | INFO | "malloc" : " 14 GB", | 20260507 01:05:48.351 | INFO | "pinned" : " 29 GB" | 20260507 01:05:48.351 | INFO | } | 20260507 01:05:48.351 | INFO | Perf report for the last 5.13882 seconds: | 20260507 01:05:48.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:05:48.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:48.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:48.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:48.351 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 01:05:48.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:48.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:48.351 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:05:48.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:48.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:48.351 | INFO | Uploaded 36GiB to GPU in 2.01075s (17.9038GiB/s) | 20260507 01:05:48.351 | INFO | Downloaded 34GiB to GPU in 1.93452s (17.5754GiB/s) | 20260507 01:05:53.495 | INFO | Analyzing chunk frames = [5883392,5883904) | 20260507 01:05:53.495 | INFO | MemInfo: { | 20260507 01:05:53.495 | INFO | "gpu" : " 17 GB", | 20260507 01:05:53.495 | INFO | "huge" : "", | 20260507 01:05:53.495 | INFO | "huge_shared" : " 27 GB", | 20260507 01:05:53.495 | INFO | "malloc" : " 14 GB", | 20260507 01:05:53.495 | INFO | "pinned" : " 29 GB" | 20260507 01:05:53.495 | INFO | } | 20260507 01:05:53.495 | INFO | Perf report for the last 5.14409 seconds: | 20260507 01:05:53.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:05:53.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:53.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:53.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:53.495 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 01:05:53.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:53.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:53.495 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:05:53.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:53.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:53.496 | INFO | Uploaded 36GiB to GPU in 2.01253s (17.8879GiB/s) | 20260507 01:05:53.496 | INFO | Downloaded 34GiB to GPU in 1.90951s (17.8056GiB/s) | 20260507 01:05:58.633 | INFO | Analyzing chunk frames = [5883904,5884416) | 20260507 01:05:58.633 | INFO | MemInfo: { | 20260507 01:05:58.633 | INFO | "gpu" : " 17 GB", | 20260507 01:05:58.633 | INFO | "huge" : "", | 20260507 01:05:58.633 | INFO | "huge_shared" : " 27 GB", | 20260507 01:05:58.633 | INFO | "malloc" : " 14 GB", | 20260507 01:05:58.633 | INFO | "pinned" : " 29 GB" | 20260507 01:05:58.633 | INFO | } | 20260507 01:05:58.633 | INFO | Perf report for the last 5.13781 seconds: | 20260507 01:05:58.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:05:58.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:58.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:58.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:58.633 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260507 01:05:58.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:58.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:58.633 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:05:58.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:58.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:05:58.633 | INFO | Uploaded 36GiB to GPU in 2.00348s (17.9688GiB/s) | 20260507 01:05:58.633 | INFO | Downloaded 34GiB to GPU in 1.93478s (17.573GiB/s) | 20260507 01:06:03.775 | INFO | Analyzing chunk frames = [5884416,5884928) | 20260507 01:06:03.775 | INFO | MemInfo: { | 20260507 01:06:03.775 | INFO | "gpu" : " 17 GB", | 20260507 01:06:03.775 | INFO | "huge" : "", | 20260507 01:06:03.775 | INFO | "huge_shared" : " 27 GB", | 20260507 01:06:03.775 | INFO | "malloc" : " 14 GB", | 20260507 01:06:03.775 | INFO | "pinned" : " 29 GB" | 20260507 01:06:03.775 | INFO | } | 20260507 01:06:03.775 | INFO | Perf report for the last 5.14187 seconds: | 20260507 01:06:03.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:06:03.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:03.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:03.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:03.775 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 01:06:03.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:03.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:03.775 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 01:06:03.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:03.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:03.775 | INFO | Uploaded 36GiB to GPU in 2.05992s (17.4764GiB/s) | 20260507 01:06:03.775 | INFO | Downloaded 34GiB to GPU in 1.95555s (17.3864GiB/s) | 20260507 01:06:08.912 | INFO | Analyzing chunk frames = [5884928,5885440) | 20260507 01:06:08.912 | INFO | MemInfo: { | 20260507 01:06:08.912 | INFO | "gpu" : " 17 GB", | 20260507 01:06:08.912 | INFO | "huge" : "", | 20260507 01:06:08.912 | INFO | "huge_shared" : " 27 GB", | 20260507 01:06:08.912 | INFO | "malloc" : " 14 GB", | 20260507 01:06:08.912 | INFO | "pinned" : " 29 GB" | 20260507 01:06:08.912 | INFO | } | 20260507 01:06:08.912 | INFO | Perf report for the last 5.13674 seconds: | 20260507 01:06:08.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:06:08.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:08.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:08.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:08.912 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:06:08.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:08.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:08.912 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 01:06:08.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:08.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:08.912 | INFO | Uploaded 36GiB to GPU in 1.9796s (18.1855GiB/s) | 20260507 01:06:08.912 | INFO | Downloaded 34GiB to GPU in 1.95623s (17.3803GiB/s) | 20260507 01:06:14.051 | INFO | Analyzing chunk frames = [5885440,5885952) | 20260507 01:06:14.051 | INFO | MemInfo: { | 20260507 01:06:14.051 | INFO | "gpu" : " 17 GB", | 20260507 01:06:14.051 | INFO | "huge" : "", | 20260507 01:06:14.051 | INFO | "huge_shared" : " 27 GB", | 20260507 01:06:14.051 | INFO | "malloc" : " 14 GB", | 20260507 01:06:14.051 | INFO | "pinned" : " 29 GB" | 20260507 01:06:14.051 | INFO | } | 20260507 01:06:14.051 | INFO | Perf report for the last 5.13977 seconds: | 20260507 01:06:14.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:06:14.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:14.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:14.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:14.051 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:06:14.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:14.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:14.052 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 01:06:14.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:14.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:14.052 | INFO | Uploaded 36GiB to GPU in 1.99938s (18.0056GiB/s) | 20260507 01:06:14.052 | INFO | Downloaded 34GiB to GPU in 1.95868s (17.3586GiB/s) | 20260507 01:06:14.150 | INFO | Starting a round of Baz IO | 20260507 01:06:17.270 | INFO | Populating RTMetrics report | 20260507 01:06:17.270 | INFO | RTMetrics report populated | 20260507 01:06:17.270 | INFO | Opening tmp RTMetrics json file | 20260507 01:06:17.271 | INFO | Serializing RTMetrics to json | 20260507 01:06:17.273 | INFO | Serialization done, swapping tmp file now... | 20260507 01:06:17.273 | INFO | RTMetrics done | 20260507 01:06:17.359 | INFO | Finished a round of Baz IO: Wrote 1.827 GiB in 3.209 seconds | 20260507 01:06:17.359 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.19% / 0.00% / 66.81% | 20260507 01:06:17.359 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.30698% / 0% / 0.000103315% | 20260507 01:06:17.360 | INFO | Baz IO overall: | 20260507 01:06:17.360 | INFO | BytesWritten : 3058696769889 | 20260507 01:06:17.360 | INFO | NumEvents : 1472690723004 | 20260507 01:06:19.187 | INFO | Analyzing chunk frames = [5885952,5886464) | 20260507 01:06:19.187 | INFO | MemInfo: { | 20260507 01:06:19.187 | INFO | "gpu" : " 17 GB", | 20260507 01:06:19.187 | INFO | "huge" : "", | 20260507 01:06:19.187 | INFO | "huge_shared" : " 27 GB", | 20260507 01:06:19.187 | INFO | "malloc" : " 14 GB", | 20260507 01:06:19.187 | INFO | "pinned" : " 29 GB" | 20260507 01:06:19.187 | INFO | } | 20260507 01:06:19.187 | INFO | Perf report for the last 5.13544 seconds: | 20260507 01:06:19.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:06:19.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:19.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:19.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:19.187 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 01:06:19.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:19.187 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:06:19.187 | INFO | PRE_HQ : 59.8%, 1.8, 56.3ms, 0 | 20260507 01:06:19.187 | INFO | BAZWRITER : 62.1%, 3.4, 107.7ms, 0 | 20260507 01:06:19.187 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:06:19.187 | INFO | Uploaded 36GiB to GPU in 2.02735s (17.7571GiB/s) | 20260507 01:06:19.187 | INFO | Downloaded 38GiB to GPU in 2.1738s (17.4809GiB/s) | 20260507 01:06:24.329 | INFO | Analyzing chunk frames = [5886464,5886976) | 20260507 01:06:24.329 | INFO | MemInfo: { | 20260507 01:06:24.329 | INFO | "gpu" : " 17 GB", | 20260507 01:06:24.329 | INFO | "huge" : "", | 20260507 01:06:24.329 | INFO | "huge_shared" : " 27 GB", | 20260507 01:06:24.329 | INFO | "malloc" : " 14 GB", | 20260507 01:06:24.329 | INFO | "pinned" : " 29 GB" | 20260507 01:06:24.329 | INFO | } | 20260507 01:06:24.329 | INFO | Perf report for the last 5.14198 seconds: | 20260507 01:06:24.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:06:24.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:24.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:24.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:24.329 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:06:24.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:24.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:24.329 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:06:24.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:24.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:24.329 | INFO | Uploaded 36GiB to GPU in 1.99358s (18.0579GiB/s) | 20260507 01:06:24.329 | INFO | Downloaded 34GiB to GPU in 1.92237s (17.6865GiB/s) | 20260507 01:06:29.469 | INFO | Analyzing chunk frames = [5886976,5887488) | 20260507 01:06:29.469 | INFO | MemInfo: { | 20260507 01:06:29.469 | INFO | "gpu" : " 17 GB", | 20260507 01:06:29.469 | INFO | "huge" : "", | 20260507 01:06:29.469 | INFO | "huge_shared" : " 27 GB", | 20260507 01:06:29.469 | INFO | "malloc" : " 14 GB", | 20260507 01:06:29.469 | INFO | "pinned" : " 29 GB" | 20260507 01:06:29.469 | INFO | } | 20260507 01:06:29.469 | INFO | Perf report for the last 5.14004 seconds: | 20260507 01:06:29.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:06:29.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:29.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:29.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:29.469 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 01:06:29.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:29.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:29.469 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:06:29.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:29.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:29.469 | INFO | Uploaded 36GiB to GPU in 1.99811s (18.017GiB/s) | 20260507 01:06:29.469 | INFO | Downloaded 34GiB to GPU in 1.93245s (17.5943GiB/s) | 20260507 01:06:34.606 | INFO | Analyzing chunk frames = [5887488,5888000) | 20260507 01:06:34.606 | INFO | MemInfo: { | 20260507 01:06:34.606 | INFO | "gpu" : " 17 GB", | 20260507 01:06:34.606 | INFO | "huge" : "", | 20260507 01:06:34.606 | INFO | "huge_shared" : " 27 GB", | 20260507 01:06:34.606 | INFO | "malloc" : " 14 GB", | 20260507 01:06:34.606 | INFO | "pinned" : " 29 GB" | 20260507 01:06:34.606 | INFO | } | 20260507 01:06:34.606 | INFO | Perf report for the last 5.13715 seconds: | 20260507 01:06:34.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:06:34.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:34.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:34.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:34.606 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 01:06:34.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:34.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:34.606 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:06:34.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:34.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:34.606 | INFO | Uploaded 36GiB to GPU in 1.97858s (18.1949GiB/s) | 20260507 01:06:34.606 | INFO | Downloaded 34GiB to GPU in 1.89773s (17.9161GiB/s) | 20260507 01:06:39.748 | INFO | Analyzing chunk frames = [5888000,5888512) | 20260507 01:06:39.748 | INFO | MemInfo: { | 20260507 01:06:39.748 | INFO | "gpu" : " 17 GB", | 20260507 01:06:39.748 | INFO | "huge" : "", | 20260507 01:06:39.748 | INFO | "huge_shared" : " 27 GB", | 20260507 01:06:39.748 | INFO | "malloc" : " 14 GB", | 20260507 01:06:39.748 | INFO | "pinned" : " 29 GB" | 20260507 01:06:39.748 | INFO | } | 20260507 01:06:39.748 | INFO | Perf report for the last 5.14196 seconds: | 20260507 01:06:39.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:06:39.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:39.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:39.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:39.748 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 01:06:39.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:39.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:39.748 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:06:39.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:39.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:39.748 | INFO | Uploaded 36GiB to GPU in 1.99963s (18.0033GiB/s) | 20260507 01:06:39.748 | INFO | Downloaded 34GiB to GPU in 1.92703s (17.6437GiB/s) | 20260507 01:06:44.886 | INFO | Analyzing chunk frames = [5888512,5889024) | 20260507 01:06:44.886 | INFO | MemInfo: { | 20260507 01:06:44.886 | INFO | "gpu" : " 17 GB", | 20260507 01:06:44.886 | INFO | "huge" : "", | 20260507 01:06:44.886 | INFO | "huge_shared" : " 27 GB", | 20260507 01:06:44.886 | INFO | "malloc" : " 14 GB", | 20260507 01:06:44.886 | INFO | "pinned" : " 29 GB" | 20260507 01:06:44.886 | INFO | } | 20260507 01:06:44.886 | INFO | Perf report for the last 5.13801 seconds: | 20260507 01:06:44.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:06:44.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:44.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:44.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:44.886 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:06:44.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:44.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:44.886 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:06:44.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:44.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:44.886 | INFO | Uploaded 36GiB to GPU in 2.05771s (17.4952GiB/s) | 20260507 01:06:44.886 | INFO | Downloaded 34GiB to GPU in 1.95424s (17.3981GiB/s) | 20260507 01:06:50.022 | INFO | Analyzing chunk frames = [5889024,5889536) | 20260507 01:06:50.022 | INFO | MemInfo: { | 20260507 01:06:50.022 | INFO | "gpu" : " 17 GB", | 20260507 01:06:50.022 | INFO | "huge" : "", | 20260507 01:06:50.022 | INFO | "huge_shared" : " 27 GB", | 20260507 01:06:50.022 | INFO | "malloc" : " 14 GB", | 20260507 01:06:50.022 | INFO | "pinned" : " 29 GB" | 20260507 01:06:50.022 | INFO | } | 20260507 01:06:50.022 | INFO | Perf report for the last 5.13599 seconds: | 20260507 01:06:50.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:06:50.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:50.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:50.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:50.022 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 01:06:50.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:50.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:50.022 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 01:06:50.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:50.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:50.022 | INFO | Uploaded 36GiB to GPU in 2.01846s (17.8354GiB/s) | 20260507 01:06:50.022 | INFO | Downloaded 34GiB to GPU in 1.97663s (17.201GiB/s) | 20260507 01:06:55.161 | INFO | Analyzing chunk frames = [5889536,5890048) | 20260507 01:06:55.161 | INFO | MemInfo: { | 20260507 01:06:55.161 | INFO | "gpu" : " 17 GB", | 20260507 01:06:55.161 | INFO | "huge" : "", | 20260507 01:06:55.161 | INFO | "huge_shared" : " 27 GB", | 20260507 01:06:55.161 | INFO | "malloc" : " 14 GB", | 20260507 01:06:55.161 | INFO | "pinned" : " 29 GB" | 20260507 01:06:55.161 | INFO | } | 20260507 01:06:55.161 | INFO | Perf report for the last 5.13888 seconds: | 20260507 01:06:55.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:06:55.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:55.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:55.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:55.161 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 01:06:55.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:55.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:55.161 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 01:06:55.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:55.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:06:55.161 | INFO | Uploaded 36GiB to GPU in 1.99595s (18.0365GiB/s) | 20260507 01:06:55.161 | INFO | Downloaded 34GiB to GPU in 1.97696s (17.1982GiB/s) | 20260507 01:06:55.161 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:06:55.161 | INFO | Total mem usage: 18 GiB | 20260507 01:06:55.161 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:06:55.161 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 01:06:55.161 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:06:55.161 | INFO | High Water Breakdown: | 20260507 01:06:55.161 | INFO | 60.1% : 8.58 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:06:55.161 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:06:55.161 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:06:55.161 | INFO | ------------------------------------------ | 20260507 01:06:55.161 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:06:55.161 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:06:55.161 | INFO | Total mem usage: 26.5 GiB | 20260507 01:06:55.161 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:06:55.161 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:06:55.161 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:06:55.161 | INFO | High Water Breakdown: | 20260507 01:06:55.161 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:06:55.161 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:06:55.162 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:06:55.162 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:06:55.162 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:06:55.162 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:06:55.162 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:06:55.162 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:06:55.162 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:06:55.162 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:06:55.162 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:06:55.162 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:06:55.162 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:06:55.162 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:06:55.162 | INFO | ------------------------------------------ | 20260507 01:06:55.162 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:06:55.162 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:06:55.162 | INFO | Total mem usage: 25.6 GiB | 20260507 01:06:55.162 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:06:55.162 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:06:55.162 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:06:55.162 | INFO | High Water Breakdown: | 20260507 01:06:55.162 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:06:55.162 | INFO | ------------------------------------------ | 20260507 01:06:55.162 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:06:55.162 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:06:55.162 | INFO | Total mem usage: 15.7 GiB | 20260507 01:06:55.162 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:06:55.162 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:06:55.162 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:06:55.162 | INFO | High Water Breakdown: | 20260507 01:06:55.162 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:06:55.162 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:06:55.162 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:06:55.162 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:06:55.162 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:06:55.162 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:06:55.162 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:06:55.162 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:06:55.162 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:06:55.162 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:06:55.162 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:06:55.162 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:06:55.162 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:06:55.162 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:06:55.162 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:06:55.162 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:06:55.162 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:06:55.162 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:06:55.162 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:06:55.162 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:06:55.162 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:06:55.162 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:06:55.162 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:06:55.162 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:06:55.162 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:06:55.162 | INFO | ------------------------------------------ | 20260507 01:06:55.162 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:06:55.162 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:06:55.162 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:06:55.162 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:06:55.162 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.53e+01, 1.32e+01 / 1.30e+00 | 20260507 01:06:55.162 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.00e-01, 1.06e-02 / 5.98e-03 | 20260507 01:06:55.162 | INFO | PerfReport: 11.67%, Baseline -- 3.61e+00 / 4.40e+00, 5.34e-01 / 3.77e-01 | 20260507 01:06:55.162 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.46e+00, 1.74e-01 / 1.44e-01 | 20260507 01:06:55.162 | INFO | PerfReport: 16.03%, DME -- 3.96e+01 / 4.40e+01, 3.74e+00 / 1.62e+00 | 20260507 01:06:55.162 | INFO | PerfReport: 57.57%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.41e+00 / 1.93e+00 | 20260507 01:06:55.162 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.17e+00 / 2.68e+00, 3.76e-01 / 2.53e-01 | 20260507 01:06:55.162 | INFO | PerfReport: 3.12%, Metrics -- 9.65e-01 / 1.29e+00, 2.56e-01 / 1.70e-01 | 20260507 01:06:55.162 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.28e-01 / 3.85e-01, 5.27e-02 / 3.07e-02 | 20260507 01:06:55.162 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:06:55.162 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:06:55.162 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:06:55.162 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 01:06:55.162 | INFO | PerfReport: 21.71%, Upload -- 2.10e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260507 01:06:55.162 | INFO | PerfReport: 21.26%, Download -- 2.05e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 01:06:55.162 | INFO | PerfReport: 32.00%, Compute -- 3.09e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 01:06:55.162 | INFO | PerfReport: 1.48%, LockUpload -- 1.43e+00 / 0.00e+00, 5.00e+00 / 0.00e+00 | 20260507 01:06:55.162 | INFO | PerfReport: 1.13%, LockDownload -- 1.09e+00 / 0.00e+00, 2.78e+00 / 0.00e+00 | 20260507 01:06:55.162 | INFO | PerfReport: 22.41%, LockCompute -- 2.16e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 01:06:55.162 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.17e-02 / 0.00e+00 | 20260507 01:06:55.259 | INFO | Starting a round of Baz IO | 20260507 01:06:58.382 | INFO | Populating RTMetrics report | 20260507 01:06:58.382 | INFO | RTMetrics report populated | 20260507 01:06:58.382 | INFO | Opening tmp RTMetrics json file | 20260507 01:06:58.383 | INFO | Serializing RTMetrics to json | 20260507 01:06:58.392 | INFO | Serialization done, swapping tmp file now... | 20260507 01:06:58.403 | INFO | RTMetrics done | 20260507 01:06:58.476 | INFO | Finished a round of Baz IO: Wrote 1.826 GiB in 3.216 seconds | 20260507 01:06:58.476 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.18% / 0.00% / 66.81% | 20260507 01:06:58.476 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.306459% / 0% / 5.5631e-05% | 20260507 01:06:58.476 | INFO | Baz IO overall: | 20260507 01:06:58.476 | INFO | BytesWritten : 3060657801722 | 20260507 01:06:58.476 | INFO | NumEvents : 1473620739881 | 20260507 01:07:00.304 | INFO | Analyzing chunk frames = [5890048,5890560) | 20260507 01:07:00.304 | INFO | MemInfo: { | 20260507 01:07:00.304 | INFO | "gpu" : " 17 GB", | 20260507 01:07:00.304 | INFO | "huge" : "", | 20260507 01:07:00.304 | INFO | "huge_shared" : " 27 GB", | 20260507 01:07:00.304 | INFO | "malloc" : " 14 GB", | 20260507 01:07:00.304 | INFO | "pinned" : " 29 GB" | 20260507 01:07:00.304 | INFO | } | 20260507 01:07:00.304 | INFO | Perf report for the last 5.14311 seconds: | 20260507 01:07:00.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:07:00.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:00.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:00.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:00.304 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 01:07:00.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:00.304 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260507 01:07:00.304 | INFO | PRE_HQ : 60.0%, 1.9, 59.3ms, 0 | 20260507 01:07:00.304 | INFO | BAZWRITER : 62.1%, 3.4, 108.8ms, 0 | 20260507 01:07:00.304 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 01:07:00.304 | INFO | Uploaded 36GiB to GPU in 2.01692s (17.849GiB/s) | 20260507 01:07:00.304 | INFO | Downloaded 38GiB to GPU in 2.18226s (17.4131GiB/s) | 20260507 01:07:05.450 | INFO | Analyzing chunk frames = [5890560,5891072) | 20260507 01:07:05.450 | INFO | MemInfo: { | 20260507 01:07:05.450 | INFO | "gpu" : " 17 GB", | 20260507 01:07:05.450 | INFO | "huge" : "", | 20260507 01:07:05.450 | INFO | "huge_shared" : " 27 GB", | 20260507 01:07:05.450 | INFO | "malloc" : " 14 GB", | 20260507 01:07:05.450 | INFO | "pinned" : " 29 GB" | 20260507 01:07:05.450 | INFO | } | 20260507 01:07:05.450 | INFO | Perf report for the last 5.14557 seconds: | 20260507 01:07:05.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:07:05.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:05.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:05.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:05.450 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:07:05.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:05.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:05.450 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 01:07:05.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:05.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:05.450 | INFO | Uploaded 36GiB to GPU in 1.99024s (18.0883GiB/s) | 20260507 01:07:05.450 | INFO | Downloaded 34GiB to GPU in 1.93035s (17.6134GiB/s) | 20260507 01:07:10.586 | INFO | Analyzing chunk frames = [5891072,5891584) | 20260507 01:07:10.586 | INFO | MemInfo: { | 20260507 01:07:10.586 | INFO | "gpu" : " 17 GB", | 20260507 01:07:10.586 | INFO | "huge" : "", | 20260507 01:07:10.586 | INFO | "huge_shared" : " 27 GB", | 20260507 01:07:10.586 | INFO | "malloc" : " 14 GB", | 20260507 01:07:10.586 | INFO | "pinned" : " 29 GB" | 20260507 01:07:10.586 | INFO | } | 20260507 01:07:10.586 | INFO | Perf report for the last 5.13669 seconds: | 20260507 01:07:10.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:07:10.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:10.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:10.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:10.587 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 01:07:10.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:10.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:10.587 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:07:10.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:10.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:10.587 | INFO | Uploaded 36GiB to GPU in 2.00232s (17.9792GiB/s) | 20260507 01:07:10.587 | INFO | Downloaded 34GiB to GPU in 1.93993s (17.5264GiB/s) | 20260507 01:07:15.724 | INFO | Analyzing chunk frames = [5891584,5892096) | 20260507 01:07:15.724 | INFO | MemInfo: { | 20260507 01:07:15.724 | INFO | "gpu" : " 17 GB", | 20260507 01:07:15.724 | INFO | "huge" : "", | 20260507 01:07:15.724 | INFO | "huge_shared" : " 27 GB", | 20260507 01:07:15.724 | INFO | "malloc" : " 14 GB", | 20260507 01:07:15.724 | INFO | "pinned" : " 29 GB" | 20260507 01:07:15.724 | INFO | } | 20260507 01:07:15.724 | INFO | Perf report for the last 5.13738 seconds: | 20260507 01:07:15.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:07:15.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:15.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:15.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:15.724 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 01:07:15.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:15.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:15.724 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:07:15.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:15.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:15.724 | INFO | Uploaded 36GiB to GPU in 1.97231s (18.2527GiB/s) | 20260507 01:07:15.724 | INFO | Downloaded 34GiB to GPU in 1.9148s (17.7564GiB/s) | 20260507 01:07:20.859 | INFO | Analyzing chunk frames = [5892096,5892608) | 20260507 01:07:20.859 | INFO | MemInfo: { | 20260507 01:07:20.859 | INFO | "gpu" : " 17 GB", | 20260507 01:07:20.859 | INFO | "huge" : "", | 20260507 01:07:20.859 | INFO | "huge_shared" : " 27 GB", | 20260507 01:07:20.859 | INFO | "malloc" : " 14 GB", | 20260507 01:07:20.859 | INFO | "pinned" : " 29 GB" | 20260507 01:07:20.859 | INFO | } | 20260507 01:07:20.859 | INFO | Perf report for the last 5.13538 seconds: | 20260507 01:07:20.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:07:20.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:20.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:20.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:20.859 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260507 01:07:20.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:20.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:20.859 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:07:20.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:20.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:20.860 | INFO | Uploaded 36GiB to GPU in 1.99281s (18.0649GiB/s) | 20260507 01:07:20.860 | INFO | Downloaded 34GiB to GPU in 1.95526s (17.389GiB/s) | 20260507 01:07:26.001 | INFO | Analyzing chunk frames = [5892608,5893120) | 20260507 01:07:26.001 | INFO | MemInfo: { | 20260507 01:07:26.001 | INFO | "gpu" : " 17 GB", | 20260507 01:07:26.001 | INFO | "huge" : "", | 20260507 01:07:26.001 | INFO | "huge_shared" : " 27 GB", | 20260507 01:07:26.001 | INFO | "malloc" : " 14 GB", | 20260507 01:07:26.001 | INFO | "pinned" : " 29 GB" | 20260507 01:07:26.001 | INFO | } | 20260507 01:07:26.001 | INFO | Perf report for the last 5.1422 seconds: | 20260507 01:07:26.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:07:26.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:26.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:26.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:26.002 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 01:07:26.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:26.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:26.002 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:07:26.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:26.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:26.002 | INFO | Uploaded 36GiB to GPU in 2.01684s (17.8497GiB/s) | 20260507 01:07:26.002 | INFO | Downloaded 34GiB to GPU in 1.94273s (17.5011GiB/s) | 20260507 01:07:31.141 | INFO | Analyzing chunk frames = [5893120,5893632) | 20260507 01:07:31.141 | INFO | MemInfo: { | 20260507 01:07:31.141 | INFO | "gpu" : " 17 GB", | 20260507 01:07:31.141 | INFO | "huge" : "", | 20260507 01:07:31.141 | INFO | "huge_shared" : " 27 GB", | 20260507 01:07:31.141 | INFO | "malloc" : " 14 GB", | 20260507 01:07:31.141 | INFO | "pinned" : " 29 GB" | 20260507 01:07:31.141 | INFO | } | 20260507 01:07:31.141 | INFO | Perf report for the last 5.13987 seconds: | 20260507 01:07:31.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:07:31.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:31.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:31.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:31.141 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:07:31.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:31.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:31.141 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:07:31.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:31.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:31.142 | INFO | Uploaded 36GiB to GPU in 1.99558s (18.0399GiB/s) | 20260507 01:07:31.142 | INFO | Downloaded 34GiB to GPU in 1.98802s (17.1025GiB/s) | 20260507 01:07:36.276 | INFO | Analyzing chunk frames = [5893632,5894144) | 20260507 01:07:36.276 | INFO | MemInfo: { | 20260507 01:07:36.276 | INFO | "gpu" : " 17 GB", | 20260507 01:07:36.276 | INFO | "huge" : "", | 20260507 01:07:36.276 | INFO | "huge_shared" : " 27 GB", | 20260507 01:07:36.276 | INFO | "malloc" : " 14 GB", | 20260507 01:07:36.276 | INFO | "pinned" : " 29 GB" | 20260507 01:07:36.276 | INFO | } | 20260507 01:07:36.276 | INFO | Perf report for the last 5.13513 seconds: | 20260507 01:07:36.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:07:36.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:36.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:36.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:36.277 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:07:36.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:36.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:36.277 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 01:07:36.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:36.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:36.277 | INFO | Uploaded 36GiB to GPU in 1.98227s (18.161GiB/s) | 20260507 01:07:36.277 | INFO | Downloaded 34GiB to GPU in 1.99439s (17.0479GiB/s) | 20260507 01:07:36.374 | INFO | Starting a round of Baz IO | 20260507 01:07:39.499 | INFO | Populating RTMetrics report | 20260507 01:07:39.499 | INFO | RTMetrics report populated | 20260507 01:07:39.499 | INFO | Opening tmp RTMetrics json file | 20260507 01:07:39.505 | INFO | Serializing RTMetrics to json | 20260507 01:07:39.518 | INFO | Serialization done, swapping tmp file now... | 20260507 01:07:39.519 | INFO | RTMetrics done | 20260507 01:07:39.589 | INFO | Finished a round of Baz IO: Wrote 1.825 GiB in 3.215 seconds | 20260507 01:07:39.589 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.18% / 0.00% / 66.82% | 20260507 01:07:39.590 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.304409% / 3.97364e-06% / 8.34465e-05% | 20260507 01:07:39.590 | INFO | Baz IO overall: | 20260507 01:07:39.590 | INFO | BytesWritten : 3062617518267 | 20260507 01:07:39.590 | INFO | NumEvents : 1474549851715 | 20260507 01:07:41.419 | INFO | Analyzing chunk frames = [5894144,5894656) | 20260507 01:07:41.419 | INFO | MemInfo: { | 20260507 01:07:41.419 | INFO | "gpu" : " 17 GB", | 20260507 01:07:41.419 | INFO | "huge" : "", | 20260507 01:07:41.419 | INFO | "huge_shared" : " 27 GB", | 20260507 01:07:41.419 | INFO | "malloc" : " 14 GB", | 20260507 01:07:41.419 | INFO | "pinned" : " 29 GB" | 20260507 01:07:41.419 | INFO | } | 20260507 01:07:41.419 | INFO | Perf report for the last 5.14253 seconds: | 20260507 01:07:41.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:07:41.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:41.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:41.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:41.419 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 01:07:41.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:41.419 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 01:07:41.419 | INFO | PRE_HQ : 59.3%, 1.8, 55.7ms, 0 | 20260507 01:07:41.419 | INFO | BAZWRITER : 62.1%, 3.3, 105.4ms, 0 | 20260507 01:07:41.419 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:07:41.419 | INFO | Uploaded 36GiB to GPU in 2.02793s (17.7521GiB/s) | 20260507 01:07:41.419 | INFO | Downloaded 38GiB to GPU in 2.18397s (17.3995GiB/s) | 20260507 01:07:46.558 | INFO | Analyzing chunk frames = [5894656,5895168) | 20260507 01:07:46.558 | INFO | MemInfo: { | 20260507 01:07:46.558 | INFO | "gpu" : " 17 GB", | 20260507 01:07:46.558 | INFO | "huge" : "", | 20260507 01:07:46.558 | INFO | "huge_shared" : " 27 GB", | 20260507 01:07:46.558 | INFO | "malloc" : " 14 GB", | 20260507 01:07:46.558 | INFO | "pinned" : " 29 GB" | 20260507 01:07:46.558 | INFO | } | 20260507 01:07:46.558 | INFO | Perf report for the last 5.13915 seconds: | 20260507 01:07:46.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:07:46.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:46.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:46.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:46.558 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:07:46.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:46.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:46.558 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:07:46.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:46.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:46.558 | INFO | Uploaded 36GiB to GPU in 2.00552s (17.9505GiB/s) | 20260507 01:07:46.558 | INFO | Downloaded 34GiB to GPU in 1.92984s (17.618GiB/s) | 20260507 01:07:51.700 | INFO | Analyzing chunk frames = [5895168,5895680) | 20260507 01:07:51.700 | INFO | MemInfo: { | 20260507 01:07:51.700 | INFO | "gpu" : " 17 GB", | 20260507 01:07:51.700 | INFO | "huge" : "", | 20260507 01:07:51.700 | INFO | "huge_shared" : " 27 GB", | 20260507 01:07:51.700 | INFO | "malloc" : " 14 GB", | 20260507 01:07:51.700 | INFO | "pinned" : " 29 GB" | 20260507 01:07:51.700 | INFO | } | 20260507 01:07:51.700 | INFO | Perf report for the last 5.1423 seconds: | 20260507 01:07:51.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:07:51.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:51.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:51.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:51.701 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260507 01:07:51.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:51.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:51.701 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:07:51.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:51.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:51.701 | INFO | Uploaded 36GiB to GPU in 1.99788s (18.0191GiB/s) | 20260507 01:07:51.701 | INFO | Downloaded 34GiB to GPU in 1.93352s (17.5845GiB/s) | 20260507 01:07:56.837 | INFO | Analyzing chunk frames = [5895680,5896192) | 20260507 01:07:56.837 | INFO | MemInfo: { | 20260507 01:07:56.837 | INFO | "gpu" : " 17 GB", | 20260507 01:07:56.837 | INFO | "huge" : "", | 20260507 01:07:56.837 | INFO | "huge_shared" : " 27 GB", | 20260507 01:07:56.837 | INFO | "malloc" : " 14 GB", | 20260507 01:07:56.837 | INFO | "pinned" : " 29 GB" | 20260507 01:07:56.837 | INFO | } | 20260507 01:07:56.837 | INFO | Perf report for the last 5.13693 seconds: | 20260507 01:07:56.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:07:56.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:56.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:56.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:56.837 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 01:07:56.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:56.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:56.838 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:07:56.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:56.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:07:56.838 | INFO | Uploaded 36GiB to GPU in 1.97977s (18.1839GiB/s) | 20260507 01:07:56.838 | INFO | Downloaded 34GiB to GPU in 1.91224s (17.7802GiB/s) | 20260507 01:08:01.972 | INFO | Analyzing chunk frames = [5896192,5896704) | 20260507 01:08:01.972 | INFO | MemInfo: { | 20260507 01:08:01.972 | INFO | "gpu" : " 17 GB", | 20260507 01:08:01.972 | INFO | "huge" : "", | 20260507 01:08:01.972 | INFO | "huge_shared" : " 27 GB", | 20260507 01:08:01.972 | INFO | "malloc" : " 14 GB", | 20260507 01:08:01.972 | INFO | "pinned" : " 29 GB" | 20260507 01:08:01.972 | INFO | } | 20260507 01:08:01.972 | INFO | Perf report for the last 5.13467 seconds: | 20260507 01:08:01.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:08:01.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:01.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:01.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:01.972 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 01:08:01.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:01.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:01.973 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:08:01.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:01.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:01.973 | INFO | Uploaded 36GiB to GPU in 2.01023s (17.9084GiB/s) | 20260507 01:08:01.973 | INFO | Downloaded 34GiB to GPU in 1.93594s (17.5625GiB/s) | 20260507 01:08:07.114 | INFO | Analyzing chunk frames = [5896704,5897216) | 20260507 01:08:07.114 | INFO | MemInfo: { | 20260507 01:08:07.114 | INFO | "gpu" : " 17 GB", | 20260507 01:08:07.114 | INFO | "huge" : "", | 20260507 01:08:07.114 | INFO | "huge_shared" : " 27 GB", | 20260507 01:08:07.114 | INFO | "malloc" : " 14 GB", | 20260507 01:08:07.114 | INFO | "pinned" : " 29 GB" | 20260507 01:08:07.114 | INFO | } | 20260507 01:08:07.114 | INFO | Perf report for the last 5.14179 seconds: | 20260507 01:08:07.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:08:07.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:07.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:07.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:07.114 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 01:08:07.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:07.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:07.114 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 01:08:07.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:07.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:07.114 | INFO | Uploaded 36GiB to GPU in 2.0431s (17.6203GiB/s) | 20260507 01:08:07.114 | INFO | Downloaded 34GiB to GPU in 1.94865s (17.448GiB/s) | 20260507 01:08:12.259 | INFO | Analyzing chunk frames = [5897216,5897728) | 20260507 01:08:12.259 | INFO | MemInfo: { | 20260507 01:08:12.259 | INFO | "gpu" : " 17 GB", | 20260507 01:08:12.259 | INFO | "huge" : "", | 20260507 01:08:12.259 | INFO | "huge_shared" : " 27 GB", | 20260507 01:08:12.259 | INFO | "malloc" : " 14 GB", | 20260507 01:08:12.259 | INFO | "pinned" : " 29 GB" | 20260507 01:08:12.259 | INFO | } | 20260507 01:08:12.259 | INFO | Perf report for the last 5.1448 seconds: | 20260507 01:08:12.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:08:12.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:12.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:12.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:12.259 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:08:12.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:12.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:12.259 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:08:12.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:12.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:12.259 | INFO | Uploaded 36GiB to GPU in 1.97284s (18.2478GiB/s) | 20260507 01:08:12.259 | INFO | Downloaded 34GiB to GPU in 1.99897s (17.0088GiB/s) | 20260507 01:08:17.398 | INFO | Analyzing chunk frames = [5897728,5898240) | 20260507 01:08:17.398 | INFO | MemInfo: { | 20260507 01:08:17.398 | INFO | "gpu" : " 17 GB", | 20260507 01:08:17.398 | INFO | "huge" : "", | 20260507 01:08:17.398 | INFO | "huge_shared" : " 27 GB", | 20260507 01:08:17.398 | INFO | "malloc" : " 14 GB", | 20260507 01:08:17.398 | INFO | "pinned" : " 29 GB" | 20260507 01:08:17.398 | INFO | } | 20260507 01:08:17.398 | INFO | Perf report for the last 5.13917 seconds: | 20260507 01:08:17.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:08:17.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:17.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:17.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:17.398 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:08:17.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:17.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:17.398 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:08:17.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:17.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:17.398 | INFO | Uploaded 36GiB to GPU in 1.98265s (18.1576GiB/s) | 20260507 01:08:17.398 | INFO | Downloaded 34GiB to GPU in 1.97227s (17.2391GiB/s) | 20260507 01:08:17.398 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:08:17.398 | INFO | Total mem usage: 18 GiB | 20260507 01:08:17.398 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:08:17.398 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:08:17.398 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:08:17.398 | INFO | High Water Breakdown: | 20260507 01:08:17.398 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:08:17.398 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:08:17.398 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:08:17.398 | INFO | ------------------------------------------ | 20260507 01:08:17.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:08:17.398 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:08:17.398 | INFO | Total mem usage: 26.5 GiB | 20260507 01:08:17.398 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:08:17.398 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 01:08:17.398 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260507 01:08:17.398 | INFO | High Water Breakdown: | 20260507 01:08:17.398 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:08:17.398 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:08:17.398 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:08:17.398 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:08:17.398 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:08:17.398 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:08:17.398 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:08:17.398 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:08:17.398 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:08:17.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:08:17.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:08:17.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:08:17.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:08:17.399 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:08:17.399 | INFO | ------------------------------------------ | 20260507 01:08:17.399 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:08:17.399 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:08:17.399 | INFO | Total mem usage: 25.6 GiB | 20260507 01:08:17.399 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:08:17.399 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:08:17.399 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:08:17.399 | INFO | High Water Breakdown: | 20260507 01:08:17.399 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:08:17.399 | INFO | ------------------------------------------ | 20260507 01:08:17.399 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:08:17.399 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:08:17.399 | INFO | Total mem usage: 15.7 GiB | 20260507 01:08:17.399 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:08:17.399 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:08:17.399 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:08:17.399 | INFO | High Water Breakdown: | 20260507 01:08:17.399 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:08:17.399 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:08:17.399 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:08:17.399 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:08:17.399 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:08:17.399 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:08:17.399 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:08:17.399 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:08:17.399 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:08:17.399 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:08:17.399 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:08:17.399 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:08:17.399 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:08:17.399 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:08:17.399 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:08:17.399 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:08:17.399 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:08:17.399 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:08:17.399 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:08:17.399 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:08:17.399 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:08:17.399 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:08:17.399 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:08:17.399 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:08:17.399 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:08:17.399 | INFO | ------------------------------------------ | 20260507 01:08:17.399 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:08:17.399 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:08:17.399 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:08:17.399 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:08:17.399 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.57e+01, 1.35e+01 / 2.18e+00 | 20260507 01:08:17.399 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 1.02e-02 / 5.41e-03 | 20260507 01:08:17.399 | INFO | PerfReport: 11.68%, Baseline -- 3.61e+00 / 4.44e+00, 5.37e-01 / 2.44e-01 | 20260507 01:08:17.399 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.45e+00, 1.73e-01 / 1.16e-01 | 20260507 01:08:17.399 | INFO | PerfReport: 16.12%, DME -- 3.98e+01 / 4.48e+01, 4.15e+00 / 2.39e+00 | 20260507 01:08:17.399 | INFO | PerfReport: 57.43%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.31e+00 / 1.47e+00 | 20260507 01:08:17.399 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.17e+00 / 2.74e+00, 3.83e-01 / 1.87e-01 | 20260507 01:08:17.399 | INFO | PerfReport: 3.12%, Metrics -- 9.65e-01 / 1.32e+00, 2.56e-01 / 1.15e-01 | 20260507 01:08:17.399 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.28e-01 / 3.79e-01, 5.18e-02 / 1.80e-02 | 20260507 01:08:17.399 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:08:17.399 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:08:17.399 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:08:17.399 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 01:08:17.399 | INFO | PerfReport: 21.58%, Upload -- 2.08e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260507 01:08:17.399 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260507 01:08:17.399 | INFO | PerfReport: 32.01%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 01:08:17.399 | INFO | PerfReport: 1.34%, LockUpload -- 1.29e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 01:08:17.399 | INFO | PerfReport: 1.36%, LockDownload -- 1.31e+00 / 0.00e+00, 3.21e+00 / 0.00e+00 | 20260507 01:08:17.399 | INFO | PerfReport: 22.34%, LockCompute -- 2.16e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 01:08:17.399 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.30e-02 / 0.00e+00 | 20260507 01:08:17.495 | INFO | Starting a round of Baz IO | 20260507 01:08:20.612 | INFO | Populating RTMetrics report | 20260507 01:08:20.612 | INFO | RTMetrics report populated | 20260507 01:08:20.612 | INFO | Opening tmp RTMetrics json file | 20260507 01:08:20.614 | INFO | Serializing RTMetrics to json | 20260507 01:08:20.633 | INFO | Serialization done, swapping tmp file now... | 20260507 01:08:20.633 | INFO | RTMetrics done | 20260507 01:08:20.706 | INFO | Finished a round of Baz IO: Wrote 1.826 GiB in 3.21 seconds | 20260507 01:08:20.706 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.18% / 0.00% / 66.82% | 20260507 01:08:20.706 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.304317% / 0% / 6.35783e-05% | 20260507 01:08:20.706 | INFO | Baz IO overall: | 20260507 01:08:20.706 | INFO | BytesWritten : 3064578169882 | 20260507 01:08:20.706 | INFO | NumEvents : 1475479607131 | 20260507 01:08:22.538 | INFO | Analyzing chunk frames = [5898240,5898752) | 20260507 01:08:22.538 | INFO | MemInfo: { | 20260507 01:08:22.538 | INFO | "gpu" : " 17 GB", | 20260507 01:08:22.538 | INFO | "huge" : "", | 20260507 01:08:22.538 | INFO | "huge_shared" : " 27 GB", | 20260507 01:08:22.538 | INFO | "malloc" : " 14 GB", | 20260507 01:08:22.538 | INFO | "pinned" : " 29 GB" | 20260507 01:08:22.538 | INFO | } | 20260507 01:08:22.538 | INFO | Perf report for the last 5.14034 seconds: | 20260507 01:08:22.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:08:22.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:22.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:22.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:22.538 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 01:08:22.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:22.538 | INFO | RT_METRICS : 42.2%, 1.0, 22.2ms, 0 | 20260507 01:08:22.538 | INFO | PRE_HQ : 59.6%, 1.8, 56.1ms, 0 | 20260507 01:08:22.538 | INFO | BAZWRITER : 62.1%, 3.3, 105.7ms, 0 | 20260507 01:08:22.538 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 01:08:22.539 | INFO | Uploaded 36GiB to GPU in 2.03626s (17.6794GiB/s) | 20260507 01:08:22.539 | INFO | Downloaded 38GiB to GPU in 2.19223s (17.3339GiB/s) | 20260507 01:08:27.671 | INFO | Analyzing chunk frames = [5898752,5899264) | 20260507 01:08:27.671 | INFO | MemInfo: { | 20260507 01:08:27.671 | INFO | "gpu" : " 17 GB", | 20260507 01:08:27.671 | INFO | "huge" : "", | 20260507 01:08:27.671 | INFO | "huge_shared" : " 27 GB", | 20260507 01:08:27.671 | INFO | "malloc" : " 14 GB", | 20260507 01:08:27.671 | INFO | "pinned" : " 29 GB" | 20260507 01:08:27.671 | INFO | } | 20260507 01:08:27.671 | INFO | Perf report for the last 5.13241 seconds: | 20260507 01:08:27.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:08:27.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:27.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:27.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:27.671 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:08:27.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:27.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:27.671 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:08:27.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:27.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:27.671 | INFO | Uploaded 36GiB to GPU in 2.01272s (17.8863GiB/s) | 20260507 01:08:27.671 | INFO | Downloaded 34GiB to GPU in 1.93428s (17.5776GiB/s) | 20260507 01:08:32.813 | INFO | Analyzing chunk frames = [5899264,5899776) | 20260507 01:08:32.813 | INFO | MemInfo: { | 20260507 01:08:32.813 | INFO | "gpu" : " 17 GB", | 20260507 01:08:32.813 | INFO | "huge" : "", | 20260507 01:08:32.813 | INFO | "huge_shared" : " 27 GB", | 20260507 01:08:32.813 | INFO | "malloc" : " 14 GB", | 20260507 01:08:32.813 | INFO | "pinned" : " 29 GB" | 20260507 01:08:32.813 | INFO | } | 20260507 01:08:32.813 | INFO | Perf report for the last 5.1422 seconds: | 20260507 01:08:32.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:08:32.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:32.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:32.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:32.813 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:08:32.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:32.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:32.813 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:08:32.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:32.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:32.813 | INFO | Uploaded 36GiB to GPU in 1.9854s (18.1323GiB/s) | 20260507 01:08:32.813 | INFO | Downloaded 34GiB to GPU in 1.93693s (17.5536GiB/s) | 20260507 01:08:37.955 | INFO | Analyzing chunk frames = [5899776,5900288) | 20260507 01:08:37.955 | INFO | MemInfo: { | 20260507 01:08:37.955 | INFO | "gpu" : " 17 GB", | 20260507 01:08:37.955 | INFO | "huge" : "", | 20260507 01:08:37.955 | INFO | "huge_shared" : " 27 GB", | 20260507 01:08:37.955 | INFO | "malloc" : " 14 GB", | 20260507 01:08:37.955 | INFO | "pinned" : " 29 GB" | 20260507 01:08:37.955 | INFO | } | 20260507 01:08:37.955 | INFO | Perf report for the last 5.14246 seconds: | 20260507 01:08:37.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:08:37.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:37.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:37.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:37.955 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260507 01:08:37.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:37.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:37.955 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:08:37.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:37.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:37.956 | INFO | Uploaded 36GiB to GPU in 1.99086s (18.0827GiB/s) | 20260507 01:08:37.956 | INFO | Downloaded 34GiB to GPU in 1.91484s (17.756GiB/s) | 20260507 01:08:43.090 | INFO | Analyzing chunk frames = [5900288,5900800) | 20260507 01:08:43.090 | INFO | MemInfo: { | 20260507 01:08:43.090 | INFO | "gpu" : " 17 GB", | 20260507 01:08:43.090 | INFO | "huge" : "", | 20260507 01:08:43.090 | INFO | "huge_shared" : " 27 GB", | 20260507 01:08:43.090 | INFO | "malloc" : " 14 GB", | 20260507 01:08:43.090 | INFO | "pinned" : " 29 GB" | 20260507 01:08:43.090 | INFO | } | 20260507 01:08:43.090 | INFO | Perf report for the last 5.13436 seconds: | 20260507 01:08:43.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:08:43.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:43.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:43.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:43.090 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 01:08:43.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:43.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:43.090 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:08:43.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:43.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:43.090 | INFO | Uploaded 36GiB to GPU in 2.00487s (17.9563GiB/s) | 20260507 01:08:43.090 | INFO | Downloaded 34GiB to GPU in 1.94561s (17.4753GiB/s) | 20260507 01:08:48.232 | INFO | Analyzing chunk frames = [5900800,5901312) | 20260507 01:08:48.232 | INFO | MemInfo: { | 20260507 01:08:48.232 | INFO | "gpu" : " 17 GB", | 20260507 01:08:48.232 | INFO | "huge" : "", | 20260507 01:08:48.232 | INFO | "huge_shared" : " 27 GB", | 20260507 01:08:48.232 | INFO | "malloc" : " 14 GB", | 20260507 01:08:48.232 | INFO | "pinned" : " 29 GB" | 20260507 01:08:48.232 | INFO | } | 20260507 01:08:48.232 | INFO | Perf report for the last 5.14255 seconds: | 20260507 01:08:48.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:08:48.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:48.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:48.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:48.232 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 01:08:48.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:48.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:48.232 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 01:08:48.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:48.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:48.233 | INFO | Uploaded 36GiB to GPU in 2.05275s (17.5374GiB/s) | 20260507 01:08:48.233 | INFO | Downloaded 34GiB to GPU in 1.96277s (17.3224GiB/s) | 20260507 01:08:53.367 | INFO | Analyzing chunk frames = [5901312,5901824) | 20260507 01:08:53.368 | INFO | MemInfo: { | 20260507 01:08:53.368 | INFO | "gpu" : " 17 GB", | 20260507 01:08:53.368 | INFO | "huge" : "", | 20260507 01:08:53.368 | INFO | "huge_shared" : " 27 GB", | 20260507 01:08:53.368 | INFO | "malloc" : " 14 GB", | 20260507 01:08:53.368 | INFO | "pinned" : " 29 GB" | 20260507 01:08:53.368 | INFO | } | 20260507 01:08:53.368 | INFO | Perf report for the last 5.13534 seconds: | 20260507 01:08:53.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:08:53.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:53.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:53.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:53.368 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 01:08:53.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:53.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:53.368 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 01:08:53.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:53.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:53.368 | INFO | Uploaded 36GiB to GPU in 1.96401s (18.3298GiB/s) | 20260507 01:08:53.368 | INFO | Downloaded 34GiB to GPU in 1.98981s (17.0871GiB/s) | 20260507 01:08:58.515 | INFO | Analyzing chunk frames = [5901824,5902336) | 20260507 01:08:58.515 | INFO | MemInfo: { | 20260507 01:08:58.515 | INFO | "gpu" : " 17 GB", | 20260507 01:08:58.515 | INFO | "huge" : "", | 20260507 01:08:58.515 | INFO | "huge_shared" : " 27 GB", | 20260507 01:08:58.515 | INFO | "malloc" : " 14 GB", | 20260507 01:08:58.515 | INFO | "pinned" : " 29 GB" | 20260507 01:08:58.515 | INFO | } | 20260507 01:08:58.515 | INFO | Perf report for the last 5.14707 seconds: | 20260507 01:08:58.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:08:58.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:58.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:58.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:58.515 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260507 01:08:58.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:58.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:58.515 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 01:08:58.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:58.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:08:58.515 | INFO | Uploaded 36GiB to GPU in 1.98101s (18.1725GiB/s) | 20260507 01:08:58.515 | INFO | Downloaded 34GiB to GPU in 1.98736s (17.1081GiB/s) | 20260507 01:08:58.613 | INFO | Starting a round of Baz IO | 20260507 01:09:01.729 | INFO | Populating RTMetrics report | 20260507 01:09:01.730 | INFO | RTMetrics report populated | 20260507 01:09:01.730 | INFO | Opening tmp RTMetrics json file | 20260507 01:09:01.730 | INFO | Serializing RTMetrics to json | 20260507 01:09:01.738 | INFO | Serialization done, swapping tmp file now... | 20260507 01:09:01.751 | INFO | RTMetrics done | 20260507 01:09:01.822 | INFO | Finished a round of Baz IO: Wrote 1.828 GiB in 3.208 seconds | 20260507 01:09:01.822 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.18% / 0.00% / 66.82% | 20260507 01:09:01.822 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.305287% / 3.97364e-06% / 8.74201e-05% | 20260507 01:09:01.822 | INFO | Baz IO overall: | 20260507 01:09:01.822 | INFO | BytesWritten : 3066541417444 | 20260507 01:09:01.822 | INFO | NumEvents : 1476411488370 | 20260507 01:09:03.650 | INFO | Analyzing chunk frames = [5902336,5902848) | 20260507 01:09:03.650 | INFO | MemInfo: { | 20260507 01:09:03.650 | INFO | "gpu" : " 17 GB", | 20260507 01:09:03.650 | INFO | "huge" : "", | 20260507 01:09:03.650 | INFO | "huge_shared" : " 27 GB", | 20260507 01:09:03.650 | INFO | "malloc" : " 14 GB", | 20260507 01:09:03.650 | INFO | "pinned" : " 29 GB" | 20260507 01:09:03.650 | INFO | } | 20260507 01:09:03.650 | INFO | Perf report for the last 5.13536 seconds: | 20260507 01:09:03.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:09:03.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:03.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:03.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:03.650 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 01:09:03.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:03.650 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 01:09:03.650 | INFO | PRE_HQ : 59.7%, 1.8, 56.8ms, 0 | 20260507 01:09:03.650 | INFO | BAZWRITER : 62.1%, 3.4, 106.8ms, 0 | 20260507 01:09:03.650 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:09:03.650 | INFO | Uploaded 36GiB to GPU in 2.04501s (17.6038GiB/s) | 20260507 01:09:03.650 | INFO | Downloaded 38GiB to GPU in 2.18495s (17.3917GiB/s) | 20260507 01:09:08.785 | INFO | Analyzing chunk frames = [5902848,5903360) | 20260507 01:09:08.785 | INFO | MemInfo: { | 20260507 01:09:08.785 | INFO | "gpu" : " 17 GB", | 20260507 01:09:08.785 | INFO | "huge" : "", | 20260507 01:09:08.785 | INFO | "huge_shared" : " 27 GB", | 20260507 01:09:08.785 | INFO | "malloc" : " 14 GB", | 20260507 01:09:08.785 | INFO | "pinned" : " 29 GB" | 20260507 01:09:08.785 | INFO | } | 20260507 01:09:08.785 | INFO | Perf report for the last 5.13527 seconds: | 20260507 01:09:08.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:09:08.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:08.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:08.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:08.785 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 01:09:08.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:08.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:08.785 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:09:08.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:08.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:08.786 | INFO | Uploaded 36GiB to GPU in 2.03633s (17.6789GiB/s) | 20260507 01:09:08.786 | INFO | Downloaded 34GiB to GPU in 1.93718s (17.5513GiB/s) | 20260507 01:09:13.924 | INFO | Analyzing chunk frames = [5903360,5903872) | 20260507 01:09:13.924 | INFO | MemInfo: { | 20260507 01:09:13.924 | INFO | "gpu" : " 17 GB", | 20260507 01:09:13.924 | INFO | "huge" : "", | 20260507 01:09:13.924 | INFO | "huge_shared" : " 27 GB", | 20260507 01:09:13.924 | INFO | "malloc" : " 14 GB", | 20260507 01:09:13.924 | INFO | "pinned" : " 29 GB" | 20260507 01:09:13.924 | INFO | } | 20260507 01:09:13.924 | INFO | Perf report for the last 5.13894 seconds: | 20260507 01:09:13.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:09:13.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:13.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:13.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:13.924 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 01:09:13.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:13.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:13.924 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 01:09:13.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:13.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:13.924 | INFO | Uploaded 36GiB to GPU in 2.00491s (17.9559GiB/s) | 20260507 01:09:13.924 | INFO | Downloaded 34GiB to GPU in 1.93436s (17.5769GiB/s) | 20260507 01:09:19.067 | INFO | Analyzing chunk frames = [5903872,5904384) | 20260507 01:09:19.067 | INFO | MemInfo: { | 20260507 01:09:19.067 | INFO | "gpu" : " 17 GB", | 20260507 01:09:19.067 | INFO | "huge" : "", | 20260507 01:09:19.067 | INFO | "huge_shared" : " 27 GB", | 20260507 01:09:19.067 | INFO | "malloc" : " 14 GB", | 20260507 01:09:19.067 | INFO | "pinned" : " 29 GB" | 20260507 01:09:19.067 | INFO | } | 20260507 01:09:19.067 | INFO | Perf report for the last 5.14276 seconds: | 20260507 01:09:19.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:09:19.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:19.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:19.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:19.067 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260507 01:09:19.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:19.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:19.067 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 01:09:19.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:19.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:19.067 | INFO | Uploaded 36GiB to GPU in 2.00043s (17.9961GiB/s) | 20260507 01:09:19.067 | INFO | Downloaded 34GiB to GPU in 1.90862s (17.814GiB/s) | 20260507 01:09:24.210 | INFO | Analyzing chunk frames = [5904384,5904896) | 20260507 01:09:24.210 | INFO | MemInfo: { | 20260507 01:09:24.210 | INFO | "gpu" : " 17 GB", | 20260507 01:09:24.210 | INFO | "huge" : "", | 20260507 01:09:24.210 | INFO | "huge_shared" : " 27 GB", | 20260507 01:09:24.210 | INFO | "malloc" : " 14 GB", | 20260507 01:09:24.210 | INFO | "pinned" : " 29 GB" | 20260507 01:09:24.210 | INFO | } | 20260507 01:09:24.210 | INFO | Perf report for the last 5.14285 seconds: | 20260507 01:09:24.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:09:24.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:24.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:24.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:24.210 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 01:09:24.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:24.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:24.210 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 01:09:24.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:24.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:24.210 | INFO | Uploaded 36GiB to GPU in 2.00975s (17.9127GiB/s) | 20260507 01:09:24.210 | INFO | Downloaded 34GiB to GPU in 1.94291s (17.4995GiB/s) | 20260507 01:09:29.350 | INFO | Analyzing chunk frames = [5904896,5905408) | 20260507 01:09:29.350 | INFO | MemInfo: { | 20260507 01:09:29.350 | INFO | "gpu" : " 17 GB", | 20260507 01:09:29.350 | INFO | "huge" : "", | 20260507 01:09:29.350 | INFO | "huge_shared" : " 27 GB", | 20260507 01:09:29.350 | INFO | "malloc" : " 14 GB", | 20260507 01:09:29.350 | INFO | "pinned" : " 29 GB" | 20260507 01:09:29.350 | INFO | } | 20260507 01:09:29.350 | INFO | Perf report for the last 5.14041 seconds: | 20260507 01:09:29.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:09:29.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:29.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:29.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:29.350 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:09:29.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:29.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:29.350 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 01:09:29.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:29.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:29.351 | INFO | Uploaded 36GiB to GPU in 2.05846s (17.4888GiB/s) | 20260507 01:09:29.351 | INFO | Downloaded 34GiB to GPU in 1.95274s (17.4114GiB/s) | 20260507 01:09:34.491 | INFO | Analyzing chunk frames = [5905408,5905920) | 20260507 01:09:34.491 | INFO | MemInfo: { | 20260507 01:09:34.491 | INFO | "gpu" : " 17 GB", | 20260507 01:09:34.491 | INFO | "huge" : "", | 20260507 01:09:34.491 | INFO | "huge_shared" : " 27 GB", | 20260507 01:09:34.491 | INFO | "malloc" : " 14 GB", | 20260507 01:09:34.491 | INFO | "pinned" : " 29 GB" | 20260507 01:09:34.491 | INFO | } | 20260507 01:09:34.491 | INFO | Perf report for the last 5.14072 seconds: | 20260507 01:09:34.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:09:34.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:34.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:34.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:34.491 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:09:34.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:34.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:34.491 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 01:09:34.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:34.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:34.491 | INFO | Uploaded 36GiB to GPU in 1.98195s (18.1639GiB/s) | 20260507 01:09:34.491 | INFO | Downloaded 34GiB to GPU in 1.96628s (17.2915GiB/s) | 20260507 01:09:39.624 | INFO | Analyzing chunk frames = [5905920,5906432) | 20260507 01:09:39.624 | INFO | MemInfo: { | 20260507 01:09:39.624 | INFO | "gpu" : " 17 GB", | 20260507 01:09:39.624 | INFO | "huge" : "", | 20260507 01:09:39.624 | INFO | "huge_shared" : " 27 GB", | 20260507 01:09:39.624 | INFO | "malloc" : " 14 GB", | 20260507 01:09:39.624 | INFO | "pinned" : " 29 GB" | 20260507 01:09:39.624 | INFO | } | 20260507 01:09:39.624 | INFO | Perf report for the last 5.1327 seconds: | 20260507 01:09:39.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:09:39.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:39.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:39.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:39.624 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:09:39.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:39.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:39.624 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 01:09:39.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:39.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:39.624 | INFO | Uploaded 36GiB to GPU in 1.97214s (18.2542GiB/s) | 20260507 01:09:39.624 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:09:39.624 | INFO | Total mem usage: 18 GiB | 20260507 01:09:39.624 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:09:39.624 | INFO | Downloaded 34GiB to GPU in 1.96979s (17.2607GiB/s) | 20260507 01:09:39.624 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:09:39.624 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:09:39.624 | INFO | High Water Breakdown: | 20260507 01:09:39.624 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:09:39.624 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:09:39.624 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:09:39.624 | INFO | ------------------------------------------ | 20260507 01:09:39.624 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:09:39.624 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:09:39.625 | INFO | Total mem usage: 26.5 GiB | 20260507 01:09:39.625 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:09:39.625 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:09:39.625 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:09:39.625 | INFO | High Water Breakdown: | 20260507 01:09:39.625 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:09:39.625 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:09:39.625 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:09:39.625 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:09:39.625 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:09:39.625 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:09:39.625 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:09:39.625 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:09:39.625 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:09:39.625 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:09:39.625 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:09:39.625 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:09:39.625 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:09:39.625 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:09:39.625 | INFO | ------------------------------------------ | 20260507 01:09:39.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:09:39.625 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:09:39.625 | INFO | Total mem usage: 25.6 GiB | 20260507 01:09:39.625 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:09:39.625 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 01:09:39.625 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 01:09:39.625 | INFO | High Water Breakdown: | 20260507 01:09:39.625 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 01:09:39.625 | INFO | ------------------------------------------ | 20260507 01:09:39.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:09:39.625 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:09:39.625 | INFO | Total mem usage: 15.7 GiB | 20260507 01:09:39.625 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:09:39.625 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:09:39.625 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:09:39.625 | INFO | High Water Breakdown: | 20260507 01:09:39.625 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:09:39.625 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:09:39.625 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:09:39.625 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:09:39.625 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:09:39.625 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:09:39.625 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:09:39.625 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:09:39.625 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:09:39.625 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:09:39.625 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:09:39.625 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:09:39.625 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:09:39.625 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:09:39.625 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:09:39.625 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:09:39.625 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:09:39.625 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:09:39.625 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:09:39.625 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:09:39.625 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:09:39.625 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:09:39.625 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:09:39.625 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:09:39.625 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:09:39.625 | INFO | ------------------------------------------ | 20260507 01:09:39.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:09:39.625 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:09:39.625 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:09:39.625 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:09:39.625 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.52e+01, 1.35e+01 / 1.77e+00 | 20260507 01:09:39.625 | INFO | PerfReport: 0.30%, AggregateStats -- 9.12e-02 / 1.02e-01, 9.75e-03 / 5.51e-03 | 20260507 01:09:39.625 | INFO | PerfReport: 11.65%, Baseline -- 3.60e+00 / 4.39e+00, 5.23e-01 / 3.80e-01 | 20260507 01:09:39.625 | INFO | PerfReport: 3.23%, Histogram -- 9.98e-01 / 1.46e+00, 1.71e-01 / 1.36e-01 | 20260507 01:09:39.625 | INFO | PerfReport: 16.08%, DME -- 3.97e+01 / 4.39e+01, 4.01e+00 / 1.94e+00 | 20260507 01:09:39.625 | INFO | PerfReport: 57.49%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.32e+00 / 1.59e+00 | 20260507 01:09:39.625 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.18e+00 / 2.70e+00, 3.85e-01 / 2.50e-01 | 20260507 01:09:39.625 | INFO | PerfReport: 3.13%, Metrics -- 9.68e-01 / 1.28e+00, 2.57e-01 / 1.23e-01 | 20260507 01:09:39.625 | INFO | PerfReport: 1.07%, PulseSerializing -- 3.29e-01 / 3.80e-01, 5.21e-02 / 2.29e-02 | 20260507 01:09:39.625 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:09:39.625 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:09:39.625 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:09:39.625 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 01:09:39.625 | INFO | PerfReport: 21.67%, Upload -- 2.09e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260507 01:09:39.625 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 01:09:39.625 | INFO | PerfReport: 31.99%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 01:09:39.625 | INFO | PerfReport: 1.40%, LockUpload -- 1.35e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260507 01:09:39.625 | INFO | PerfReport: 1.26%, LockDownload -- 1.21e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260507 01:09:39.625 | INFO | PerfReport: 22.31%, LockCompute -- 2.15e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 01:09:39.625 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260507 01:09:39.724 | INFO | Starting a round of Baz IO | 20260507 01:09:42.847 | INFO | Populating RTMetrics report | 20260507 01:09:42.847 | INFO | RTMetrics report populated | 20260507 01:09:42.847 | INFO | Opening tmp RTMetrics json file | 20260507 01:09:42.848 | INFO | Serializing RTMetrics to json | 20260507 01:09:42.849 | INFO | Serialization done, swapping tmp file now... | 20260507 01:09:42.851 | INFO | RTMetrics done | 20260507 01:09:42.939 | INFO | Finished a round of Baz IO: Wrote 1.827 GiB in 3.215 seconds | 20260507 01:09:42.939 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.18% / 0.00% / 66.82% | 20260507 01:09:42.939 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.305514% / 3.97364e-06% / 9.13938e-05% | 20260507 01:09:42.939 | INFO | Baz IO overall: | 20260507 01:09:42.939 | INFO | BytesWritten : 3068503635950 | 20260507 01:09:42.939 | INFO | NumEvents : 1477342796763 | 20260507 01:09:44.759 | INFO | Analyzing chunk frames = [5906432,5906944) | 20260507 01:09:44.759 | INFO | MemInfo: { | 20260507 01:09:44.759 | INFO | "gpu" : " 17 GB", | 20260507 01:09:44.759 | INFO | "huge" : "", | 20260507 01:09:44.759 | INFO | "huge_shared" : " 27 GB", | 20260507 01:09:44.759 | INFO | "malloc" : " 14 GB", | 20260507 01:09:44.759 | INFO | "pinned" : " 29 GB" | 20260507 01:09:44.759 | INFO | } | 20260507 01:09:44.759 | INFO | Perf report for the last 5.13494 seconds: | 20260507 01:09:44.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:09:44.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:44.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:44.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:44.759 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 01:09:44.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:44.759 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260507 01:09:44.759 | INFO | PRE_HQ : 60.2%, 1.9, 59.3ms, 0 | 20260507 01:09:44.759 | INFO | BAZWRITER : 62.2%, 3.4, 108.2ms, 0 | 20260507 01:09:44.759 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:09:44.759 | INFO | Uploaded 36GiB to GPU in 2.0287s (17.7453GiB/s) | 20260507 01:09:44.759 | INFO | Downloaded 38GiB to GPU in 2.16703s (17.5355GiB/s) | 20260507 01:09:49.907 | INFO | Analyzing chunk frames = [5906944,5907456) | 20260507 01:09:49.907 | INFO | MemInfo: { | 20260507 01:09:49.907 | INFO | "gpu" : " 17 GB", | 20260507 01:09:49.907 | INFO | "huge" : "", | 20260507 01:09:49.907 | INFO | "huge_shared" : " 27 GB", | 20260507 01:09:49.907 | INFO | "malloc" : " 14 GB", | 20260507 01:09:49.907 | INFO | "pinned" : " 29 GB" | 20260507 01:09:49.907 | INFO | } | 20260507 01:09:49.907 | INFO | Perf report for the last 5.14822 seconds: | 20260507 01:09:49.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:09:49.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:49.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:49.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:49.907 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260507 01:09:49.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:49.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:49.907 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 01:09:49.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:49.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:49.907 | INFO | Uploaded 36GiB to GPU in 2.00911s (17.9184GiB/s) | 20260507 01:09:49.907 | INFO | Downloaded 34GiB to GPU in 1.93014s (17.6153GiB/s) | 20260507 01:09:55.043 | INFO | Analyzing chunk frames = [5907456,5907968) | 20260507 01:09:55.043 | INFO | MemInfo: { | 20260507 01:09:55.043 | INFO | "gpu" : " 17 GB", | 20260507 01:09:55.043 | INFO | "huge" : "", | 20260507 01:09:55.043 | INFO | "huge_shared" : " 27 GB", | 20260507 01:09:55.043 | INFO | "malloc" : " 14 GB", | 20260507 01:09:55.043 | INFO | "pinned" : " 29 GB" | 20260507 01:09:55.043 | INFO | } | 20260507 01:09:55.043 | INFO | Perf report for the last 5.13588 seconds: | 20260507 01:09:55.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:09:55.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:55.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:55.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:55.043 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 01:09:55.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:55.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:55.043 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:09:55.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:55.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:09:55.043 | INFO | Uploaded 36GiB to GPU in 2.01159s (17.8963GiB/s) | 20260507 01:09:55.043 | INFO | Downloaded 34GiB to GPU in 1.93561s (17.5655GiB/s) | 20260507 01:10:00.180 | INFO | Analyzing chunk frames = [5907968,5908480) | 20260507 01:10:00.180 | INFO | MemInfo: { | 20260507 01:10:00.180 | INFO | "gpu" : " 17 GB", | 20260507 01:10:00.181 | INFO | "huge" : "", | 20260507 01:10:00.181 | INFO | "huge_shared" : " 27 GB", | 20260507 01:10:00.181 | INFO | "malloc" : " 14 GB", | 20260507 01:10:00.181 | INFO | "pinned" : " 29 GB" | 20260507 01:10:00.181 | INFO | } | 20260507 01:10:00.181 | INFO | Perf report for the last 5.13767 seconds: | 20260507 01:10:00.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:10:00.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:00.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:00.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:00.181 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 01:10:00.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:00.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:00.181 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 01:10:00.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:00.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:00.181 | INFO | Uploaded 36GiB to GPU in 1.98205s (18.163GiB/s) | 20260507 01:10:00.181 | INFO | Downloaded 34GiB to GPU in 1.90474s (17.8502GiB/s) | 20260507 01:10:05.320 | INFO | Analyzing chunk frames = [5908480,5908992) | 20260507 01:10:05.320 | INFO | MemInfo: { | 20260507 01:10:05.320 | INFO | "gpu" : " 17 GB", | 20260507 01:10:05.320 | INFO | "huge" : "", | 20260507 01:10:05.320 | INFO | "huge_shared" : " 27 GB", | 20260507 01:10:05.320 | INFO | "malloc" : " 14 GB", | 20260507 01:10:05.321 | INFO | "pinned" : " 29 GB" | 20260507 01:10:05.321 | INFO | } | 20260507 01:10:05.321 | INFO | Perf report for the last 5.13992 seconds: | 20260507 01:10:05.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:10:05.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:05.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:05.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:05.321 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 01:10:05.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:05.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:05.321 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 01:10:05.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:05.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:05.321 | INFO | Uploaded 36GiB to GPU in 1.98652s (18.1221GiB/s) | 20260507 01:10:05.321 | INFO | Downloaded 34GiB to GPU in 1.92489s (17.6633GiB/s) | 20260507 01:10:10.464 | INFO | Analyzing chunk frames = [5908992,5909504) | 20260507 01:10:10.464 | INFO | MemInfo: { | 20260507 01:10:10.464 | INFO | "gpu" : " 17 GB", | 20260507 01:10:10.464 | INFO | "huge" : "", | 20260507 01:10:10.464 | INFO | "huge_shared" : " 27 GB", | 20260507 01:10:10.464 | INFO | "malloc" : " 14 GB", | 20260507 01:10:10.464 | INFO | "pinned" : " 29 GB" | 20260507 01:10:10.464 | INFO | } | 20260507 01:10:10.464 | INFO | Perf report for the last 5.14391 seconds: | 20260507 01:10:10.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:10:10.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:10.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:10.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:10.465 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 01:10:10.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:10.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:10.465 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:10:10.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:10.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:10.465 | INFO | Uploaded 36GiB to GPU in 2.0277s (17.7541GiB/s) | 20260507 01:10:10.465 | INFO | Downloaded 34GiB to GPU in 1.93421s (17.5782GiB/s) | 20260507 01:10:15.597 | INFO | Analyzing chunk frames = [5909504,5910016) | 20260507 01:10:15.597 | INFO | MemInfo: { | 20260507 01:10:15.597 | INFO | "gpu" : " 17 GB", | 20260507 01:10:15.597 | INFO | "huge" : "", | 20260507 01:10:15.597 | INFO | "huge_shared" : " 27 GB", | 20260507 01:10:15.597 | INFO | "malloc" : " 14 GB", | 20260507 01:10:15.597 | INFO | "pinned" : " 29 GB" | 20260507 01:10:15.597 | INFO | } | 20260507 01:10:15.597 | INFO | Perf report for the last 5.1327 seconds: | 20260507 01:10:15.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:10:15.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:15.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:15.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:15.597 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:10:15.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:15.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:15.597 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 01:10:15.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:15.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:15.597 | INFO | Uploaded 36GiB to GPU in 1.98301s (18.1542GiB/s) | 20260507 01:10:15.597 | INFO | Downloaded 34GiB to GPU in 1.98673s (17.1136GiB/s) | 20260507 01:10:20.736 | INFO | Analyzing chunk frames = [5910016,5910528) | 20260507 01:10:20.736 | INFO | MemInfo: { | 20260507 01:10:20.736 | INFO | "gpu" : " 17 GB", | 20260507 01:10:20.736 | INFO | "huge" : "", | 20260507 01:10:20.736 | INFO | "huge_shared" : " 27 GB", | 20260507 01:10:20.736 | INFO | "malloc" : " 14 GB", | 20260507 01:10:20.736 | INFO | "pinned" : " 29 GB" | 20260507 01:10:20.736 | INFO | } | 20260507 01:10:20.736 | INFO | Perf report for the last 5.13891 seconds: | 20260507 01:10:20.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:10:20.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:20.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:20.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:20.736 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:10:20.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:20.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:20.736 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 01:10:20.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:20.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:20.736 | INFO | Uploaded 36GiB to GPU in 2.00025s (17.9977GiB/s) | 20260507 01:10:20.736 | INFO | Downloaded 34GiB to GPU in 1.97852s (17.1845GiB/s) | 20260507 01:10:20.834 | INFO | Starting a round of Baz IO | 20260507 01:10:23.958 | INFO | Populating RTMetrics report | 20260507 01:10:23.959 | INFO | RTMetrics report populated | 20260507 01:10:23.959 | INFO | Opening tmp RTMetrics json file | 20260507 01:10:23.960 | INFO | Serializing RTMetrics to json | 20260507 01:10:23.978 | INFO | Serialization done, swapping tmp file now... | 20260507 01:10:23.979 | INFO | RTMetrics done | 20260507 01:10:24.053 | INFO | Finished a round of Baz IO: Wrote 1.825 GiB in 3.219 seconds | 20260507 01:10:24.053 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.17% / 0.00% / 66.82% | 20260507 01:10:24.053 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.302951% / 3.97364e-06% / 0.000107288% | 20260507 01:10:24.053 | INFO | Baz IO overall: | 20260507 01:10:24.053 | INFO | BytesWritten : 3070462949976 | 20260507 01:10:24.053 | INFO | NumEvents : 1478272242866 | 20260507 01:10:25.879 | INFO | Analyzing chunk frames = [5910528,5911040) | 20260507 01:10:25.879 | INFO | MemInfo: { | 20260507 01:10:25.879 | INFO | "gpu" : " 17 GB", | 20260507 01:10:25.879 | INFO | "huge" : "", | 20260507 01:10:25.879 | INFO | "huge_shared" : " 27 GB", | 20260507 01:10:25.879 | INFO | "malloc" : " 14 GB", | 20260507 01:10:25.879 | INFO | "pinned" : " 29 GB" | 20260507 01:10:25.879 | INFO | } | 20260507 01:10:25.879 | INFO | Perf report for the last 5.1427 seconds: | 20260507 01:10:25.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:10:25.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:25.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:25.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:25.879 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 01:10:25.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:25.879 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 01:10:25.879 | INFO | PRE_HQ : 59.7%, 1.8, 56.3ms, 0 | 20260507 01:10:25.879 | INFO | BAZWRITER : 62.2%, 3.3, 106.3ms, 0 | 20260507 01:10:25.879 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:10:25.879 | INFO | Uploaded 36GiB to GPU in 2.02631s (17.7663GiB/s) | 20260507 01:10:25.879 | INFO | Downloaded 38GiB to GPU in 2.18471s (17.3936GiB/s) | 20260507 01:10:31.019 | INFO | Analyzing chunk frames = [5911040,5911552) | 20260507 01:10:31.019 | INFO | MemInfo: { | 20260507 01:10:31.019 | INFO | "gpu" : " 17 GB", | 20260507 01:10:31.019 | INFO | "huge" : "", | 20260507 01:10:31.019 | INFO | "huge_shared" : " 27 GB", | 20260507 01:10:31.019 | INFO | "malloc" : " 14 GB", | 20260507 01:10:31.019 | INFO | "pinned" : " 29 GB" | 20260507 01:10:31.019 | INFO | } | 20260507 01:10:31.019 | INFO | Perf report for the last 5.13981 seconds: | 20260507 01:10:31.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:10:31.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:31.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:31.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:31.019 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 01:10:31.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:31.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:31.019 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:10:31.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:31.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:31.019 | INFO | Uploaded 36GiB to GPU in 2.01383s (17.8764GiB/s) | 20260507 01:10:31.019 | INFO | Downloaded 34GiB to GPU in 1.93795s (17.5443GiB/s) | 20260507 01:10:36.166 | INFO | Analyzing chunk frames = [5911552,5912064) | 20260507 01:10:36.166 | INFO | MemInfo: { | 20260507 01:10:36.166 | INFO | "gpu" : " 17 GB", | 20260507 01:10:36.166 | INFO | "huge" : "", | 20260507 01:10:36.166 | INFO | "huge_shared" : " 27 GB", | 20260507 01:10:36.166 | INFO | "malloc" : " 14 GB", | 20260507 01:10:36.166 | INFO | "pinned" : " 29 GB" | 20260507 01:10:36.166 | INFO | } | 20260507 01:10:36.166 | INFO | Perf report for the last 5.14725 seconds: | 20260507 01:10:36.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:10:36.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:36.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:36.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:36.166 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260507 01:10:36.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:36.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:36.166 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:10:36.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:36.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:36.166 | INFO | Uploaded 36GiB to GPU in 2.00669s (17.94GiB/s) | 20260507 01:10:36.166 | INFO | Downloaded 34GiB to GPU in 1.93363s (17.5835GiB/s) | 20260507 01:10:41.304 | INFO | Analyzing chunk frames = [5912064,5912576) | 20260507 01:10:41.305 | INFO | MemInfo: { | 20260507 01:10:41.305 | INFO | "gpu" : " 17 GB", | 20260507 01:10:41.305 | INFO | "huge" : "", | 20260507 01:10:41.305 | INFO | "huge_shared" : " 27 GB", | 20260507 01:10:41.305 | INFO | "malloc" : " 14 GB", | 20260507 01:10:41.305 | INFO | "pinned" : " 29 GB" | 20260507 01:10:41.305 | INFO | } | 20260507 01:10:41.305 | INFO | Perf report for the last 5.13871 seconds: | 20260507 01:10:41.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:10:41.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:41.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:41.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:41.305 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:10:41.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:41.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:41.305 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 01:10:41.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:41.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:41.305 | INFO | Uploaded 36GiB to GPU in 2.00173s (17.9845GiB/s) | 20260507 01:10:41.305 | INFO | Downloaded 34GiB to GPU in 1.92677s (17.6461GiB/s) | 20260507 01:10:46.435 | INFO | Analyzing chunk frames = [5912576,5913088) | 20260507 01:10:46.435 | INFO | MemInfo: { | 20260507 01:10:46.435 | INFO | "gpu" : " 17 GB", | 20260507 01:10:46.435 | INFO | "huge" : "", | 20260507 01:10:46.435 | INFO | "huge_shared" : " 27 GB", | 20260507 01:10:46.435 | INFO | "malloc" : " 14 GB", | 20260507 01:10:46.435 | INFO | "pinned" : " 29 GB" | 20260507 01:10:46.435 | INFO | } | 20260507 01:10:46.435 | INFO | Perf report for the last 5.1308 seconds: | 20260507 01:10:46.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:10:46.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:46.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:46.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:46.436 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260507 01:10:46.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:46.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:46.436 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:10:46.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:46.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:46.436 | INFO | Uploaded 36GiB to GPU in 2.0139s (17.8757GiB/s) | 20260507 01:10:46.436 | INFO | Downloaded 34GiB to GPU in 1.93514s (17.5698GiB/s) | 20260507 01:10:51.577 | INFO | Analyzing chunk frames = [5913088,5913600) | 20260507 01:10:51.577 | INFO | MemInfo: { | 20260507 01:10:51.577 | INFO | "gpu" : " 17 GB", | 20260507 01:10:51.577 | INFO | "huge" : "", | 20260507 01:10:51.577 | INFO | "huge_shared" : " 27 GB", | 20260507 01:10:51.577 | INFO | "malloc" : " 14 GB", | 20260507 01:10:51.577 | INFO | "pinned" : " 29 GB" | 20260507 01:10:51.577 | INFO | } | 20260507 01:10:51.577 | INFO | Perf report for the last 5.14186 seconds: | 20260507 01:10:51.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:10:51.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:51.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:51.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:51.577 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 01:10:51.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:51.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:51.577 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 01:10:51.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:51.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:51.578 | INFO | Uploaded 36GiB to GPU in 2.03166s (17.7195GiB/s) | 20260507 01:10:51.578 | INFO | Downloaded 34GiB to GPU in 1.93761s (17.5474GiB/s) | 20260507 01:10:56.714 | INFO | Analyzing chunk frames = [5913600,5914112) | 20260507 01:10:56.714 | INFO | MemInfo: { | 20260507 01:10:56.714 | INFO | "gpu" : " 17 GB", | 20260507 01:10:56.714 | INFO | "huge" : "", | 20260507 01:10:56.714 | INFO | "huge_shared" : " 27 GB", | 20260507 01:10:56.714 | INFO | "malloc" : " 14 GB", | 20260507 01:10:56.714 | INFO | "pinned" : " 29 GB" | 20260507 01:10:56.714 | INFO | } | 20260507 01:10:56.714 | INFO | Perf report for the last 5.13655 seconds: | 20260507 01:10:56.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:10:56.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:56.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:56.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:56.714 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 01:10:56.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:56.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:56.714 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 01:10:56.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:56.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:10:56.714 | INFO | Uploaded 36GiB to GPU in 1.99807s (18.0174GiB/s) | 20260507 01:10:56.714 | INFO | Downloaded 34GiB to GPU in 1.97949s (17.1761GiB/s) | 20260507 01:11:01.852 | INFO | Analyzing chunk frames = [5914112,5914624) | 20260507 01:11:01.852 | INFO | MemInfo: { | 20260507 01:11:01.852 | INFO | "gpu" : " 17 GB", | 20260507 01:11:01.852 | INFO | "huge" : "", | 20260507 01:11:01.852 | INFO | "huge_shared" : " 27 GB", | 20260507 01:11:01.852 | INFO | "malloc" : " 14 GB", | 20260507 01:11:01.852 | INFO | "pinned" : " 29 GB" | 20260507 01:11:01.852 | INFO | } | 20260507 01:11:01.852 | INFO | Perf report for the last 5.13798 seconds: | 20260507 01:11:01.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:11:01.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:01.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:01.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:01.852 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:11:01.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:01.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:01.852 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260507 01:11:01.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:01.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:01.852 | INFO | Uploaded 36GiB to GPU in 1.99159s (18.076GiB/s) | 20260507 01:11:01.852 | INFO | Downloaded 34GiB to GPU in 2.00845s (16.9285GiB/s) | 20260507 01:11:01.852 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:11:01.852 | INFO | Total mem usage: 18 GiB | 20260507 01:11:01.852 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:11:01.852 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:11:01.852 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:11:01.852 | INFO | High Water Breakdown: | 20260507 01:11:01.852 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:11:01.852 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:11:01.852 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:11:01.852 | INFO | ------------------------------------------ | 20260507 01:11:01.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:11:01.852 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:11:01.852 | INFO | Total mem usage: 26.5 GiB | 20260507 01:11:01.852 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:11:01.852 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:11:01.852 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:11:01.852 | INFO | High Water Breakdown: | 20260507 01:11:01.852 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:11:01.852 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:11:01.852 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:11:01.852 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:11:01.852 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:11:01.852 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:11:01.852 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:11:01.852 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:11:01.852 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:11:01.852 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:11:01.852 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:11:01.852 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:11:01.852 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:11:01.852 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:11:01.852 | INFO | ------------------------------------------ | 20260507 01:11:01.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:11:01.852 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:11:01.852 | INFO | Total mem usage: 25.6 GiB | 20260507 01:11:01.853 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:11:01.853 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:11:01.853 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:11:01.853 | INFO | High Water Breakdown: | 20260507 01:11:01.853 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:11:01.853 | INFO | ------------------------------------------ | 20260507 01:11:01.853 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:11:01.853 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:11:01.853 | INFO | Total mem usage: 15.7 GiB | 20260507 01:11:01.853 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:11:01.853 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:11:01.853 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:11:01.853 | INFO | High Water Breakdown: | 20260507 01:11:01.853 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:11:01.853 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:11:01.853 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:11:01.853 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:11:01.853 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:11:01.853 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:11:01.853 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:11:01.853 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:11:01.853 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:11:01.853 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:11:01.853 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:11:01.853 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:11:01.853 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:11:01.853 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:11:01.853 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:11:01.853 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:11:01.853 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:11:01.853 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:11:01.853 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:11:01.853 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:11:01.853 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:11:01.853 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:11:01.853 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:11:01.853 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:11:01.853 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:11:01.853 | INFO | ------------------------------------------ | 20260507 01:11:01.853 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:11:01.853 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:11:01.853 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:11:01.853 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:11:01.853 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.58e+01, 1.32e+01 / 1.79e+00 | 20260507 01:11:01.853 | INFO | PerfReport: 0.30%, AggregateStats -- 9.16e-02 / 1.01e-01, 9.85e-03 / 4.32e-03 | 20260507 01:11:01.853 | INFO | PerfReport: 11.66%, Baseline -- 3.60e+00 / 4.47e+00, 5.31e-01 / 2.38e-01 | 20260507 01:11:01.853 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.48e+00, 1.74e-01 / 1.02e-01 | 20260507 01:11:01.853 | INFO | PerfReport: 16.08%, DME -- 3.98e+01 / 4.47e+01, 4.02e+00 / 2.38e+00 | 20260507 01:11:01.853 | INFO | PerfReport: 57.50%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.37e+00 / 1.77e+00 | 20260507 01:11:01.853 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.17e+00 / 2.70e+00, 3.79e-01 / 2.42e-01 | 20260507 01:11:01.853 | INFO | PerfReport: 3.12%, Metrics -- 9.65e-01 / 1.32e+00, 2.56e-01 / 1.39e-01 | 20260507 01:11:01.853 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.29e-01 / 3.83e-01, 5.21e-02 / 1.81e-02 | 20260507 01:11:01.853 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:11:01.853 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:11:01.853 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:11:01.853 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 01:11:01.853 | INFO | PerfReport: 21.65%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260507 01:11:01.853 | INFO | PerfReport: 21.31%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260507 01:11:01.853 | INFO | PerfReport: 32.01%, Compute -- 3.09e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 01:11:01.853 | INFO | PerfReport: 1.40%, LockUpload -- 1.35e+00 / 0.00e+00, 4.89e+00 / 0.00e+00 | 20260507 01:11:01.853 | INFO | PerfReport: 1.29%, LockDownload -- 1.25e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260507 01:11:01.853 | INFO | PerfReport: 22.32%, LockCompute -- 2.15e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 01:11:01.853 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260507 01:11:01.949 | INFO | Starting a round of Baz IO | 20260507 01:11:05.073 | INFO | Populating RTMetrics report | 20260507 01:11:05.074 | INFO | RTMetrics report populated | 20260507 01:11:05.074 | INFO | Opening tmp RTMetrics json file | 20260507 01:11:05.093 | INFO | Serializing RTMetrics to json | 20260507 01:11:05.094 | INFO | Serialization done, swapping tmp file now... | 20260507 01:11:05.095 | INFO | RTMetrics done | 20260507 01:11:05.165 | INFO | Finished a round of Baz IO: Wrote 1.825 GiB in 3.216 seconds | 20260507 01:11:05.165 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.17% / 0.00% / 66.83% | 20260507 01:11:05.166 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.302267% / 0% / 0.000103315% | 20260507 01:11:05.166 | INFO | Baz IO overall: | 20260507 01:11:05.166 | INFO | BytesWritten : 3072422146299 | 20260507 01:11:05.166 | INFO | NumEvents : 1479201667670 | 20260507 01:11:07.001 | INFO | Analyzing chunk frames = [5914624,5915136) | 20260507 01:11:07.001 | INFO | MemInfo: { | 20260507 01:11:07.001 | INFO | "gpu" : " 17 GB", | 20260507 01:11:07.001 | INFO | "huge" : "", | 20260507 01:11:07.001 | INFO | "huge_shared" : " 27 GB", | 20260507 01:11:07.001 | INFO | "malloc" : " 14 GB", | 20260507 01:11:07.001 | INFO | "pinned" : " 29 GB" | 20260507 01:11:07.001 | INFO | } | 20260507 01:11:07.001 | INFO | Perf report for the last 5.14907 seconds: | 20260507 01:11:07.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:11:07.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:07.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:07.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:07.001 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260507 01:11:07.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:07.001 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260507 01:11:07.001 | INFO | PRE_HQ : 59.6%, 1.8, 56.2ms, 0 | 20260507 01:11:07.001 | INFO | BAZWRITER : 62.0%, 3.3, 106.3ms, 0 | 20260507 01:11:07.001 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:11:07.001 | INFO | Uploaded 36GiB to GPU in 2.02265s (17.7985GiB/s) | 20260507 01:11:07.001 | INFO | Downloaded 38GiB to GPU in 2.17708s (17.4546GiB/s) | 20260507 01:11:12.135 | INFO | Analyzing chunk frames = [5915136,5915648) | 20260507 01:11:12.135 | INFO | MemInfo: { | 20260507 01:11:12.135 | INFO | "gpu" : " 17 GB", | 20260507 01:11:12.135 | INFO | "huge" : "", | 20260507 01:11:12.135 | INFO | "huge_shared" : " 27 GB", | 20260507 01:11:12.135 | INFO | "malloc" : " 14 GB", | 20260507 01:11:12.135 | INFO | "pinned" : " 29 GB" | 20260507 01:11:12.135 | INFO | } | 20260507 01:11:12.135 | INFO | Perf report for the last 5.13408 seconds: | 20260507 01:11:12.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:11:12.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:12.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:12.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:12.135 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:11:12.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:12.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:12.135 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 01:11:12.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:12.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:12.135 | INFO | Uploaded 36GiB to GPU in 2.01779s (17.8413GiB/s) | 20260507 01:11:12.135 | INFO | Downloaded 34GiB to GPU in 1.93696s (17.5533GiB/s) | 20260507 01:11:17.276 | INFO | Analyzing chunk frames = [5915648,5916160) | 20260507 01:11:17.276 | INFO | MemInfo: { | 20260507 01:11:17.276 | INFO | "gpu" : " 17 GB", | 20260507 01:11:17.276 | INFO | "huge" : "", | 20260507 01:11:17.276 | INFO | "huge_shared" : " 27 GB", | 20260507 01:11:17.276 | INFO | "malloc" : " 14 GB", | 20260507 01:11:17.276 | INFO | "pinned" : " 29 GB" | 20260507 01:11:17.276 | INFO | } | 20260507 01:11:17.276 | INFO | Perf report for the last 5.14093 seconds: | 20260507 01:11:17.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:11:17.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:17.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:17.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:17.276 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:11:17.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:17.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:17.276 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:11:17.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:17.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:17.276 | INFO | Uploaded 36GiB to GPU in 1.99709s (18.0262GiB/s) | 20260507 01:11:17.276 | INFO | Downloaded 34GiB to GPU in 1.93977s (17.5279GiB/s) | 20260507 01:11:22.416 | INFO | Analyzing chunk frames = [5916160,5916672) | 20260507 01:11:22.416 | INFO | MemInfo: { | 20260507 01:11:22.416 | INFO | "gpu" : " 17 GB", | 20260507 01:11:22.416 | INFO | "huge" : "", | 20260507 01:11:22.416 | INFO | "huge_shared" : " 27 GB", | 20260507 01:11:22.416 | INFO | "malloc" : " 14 GB", | 20260507 01:11:22.416 | INFO | "pinned" : " 29 GB" | 20260507 01:11:22.416 | INFO | } | 20260507 01:11:22.416 | INFO | Perf report for the last 5.14008 seconds: | 20260507 01:11:22.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:11:22.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:22.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:22.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:22.416 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:11:22.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:22.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:22.416 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:11:22.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:22.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:22.416 | INFO | Uploaded 36GiB to GPU in 1.9711s (18.2639GiB/s) | 20260507 01:11:22.416 | INFO | Downloaded 34GiB to GPU in 1.90795s (17.8201GiB/s) | 20260507 01:11:27.549 | INFO | Analyzing chunk frames = [5916672,5917184) | 20260507 01:11:27.549 | INFO | MemInfo: { | 20260507 01:11:27.549 | INFO | "gpu" : " 17 GB", | 20260507 01:11:27.549 | INFO | "huge" : "", | 20260507 01:11:27.549 | INFO | "huge_shared" : " 27 GB", | 20260507 01:11:27.549 | INFO | "malloc" : " 14 GB", | 20260507 01:11:27.549 | INFO | "pinned" : " 29 GB" | 20260507 01:11:27.549 | INFO | } | 20260507 01:11:27.549 | INFO | Perf report for the last 5.13323 seconds: | 20260507 01:11:27.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:11:27.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:27.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:27.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:27.549 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 01:11:27.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:27.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:27.549 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:11:27.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:27.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:27.550 | INFO | Uploaded 36GiB to GPU in 2.00719s (17.9355GiB/s) | 20260507 01:11:27.550 | INFO | Downloaded 34GiB to GPU in 1.94186s (17.509GiB/s) | 20260507 01:11:32.683 | INFO | Analyzing chunk frames = [5917184,5917696) | 20260507 01:11:32.683 | INFO | MemInfo: { | 20260507 01:11:32.683 | INFO | "gpu" : " 17 GB", | 20260507 01:11:32.683 | INFO | "huge" : "", | 20260507 01:11:32.683 | INFO | "huge_shared" : " 27 GB", | 20260507 01:11:32.683 | INFO | "malloc" : " 14 GB", | 20260507 01:11:32.683 | INFO | "pinned" : " 29 GB" | 20260507 01:11:32.683 | INFO | } | 20260507 01:11:32.683 | INFO | Perf report for the last 5.13404 seconds: | 20260507 01:11:32.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:11:32.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:32.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:32.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:32.683 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 01:11:32.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:32.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:32.683 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:11:32.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:32.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:32.684 | INFO | Uploaded 36GiB to GPU in 2.01218s (17.8911GiB/s) | 20260507 01:11:32.684 | INFO | Downloaded 34GiB to GPU in 1.9377s (17.5465GiB/s) | 20260507 01:11:37.830 | INFO | Analyzing chunk frames = [5917696,5918208) | 20260507 01:11:37.830 | INFO | MemInfo: { | 20260507 01:11:37.830 | INFO | "gpu" : " 17 GB", | 20260507 01:11:37.830 | INFO | "huge" : "", | 20260507 01:11:37.830 | INFO | "huge_shared" : " 27 GB", | 20260507 01:11:37.830 | INFO | "malloc" : " 14 GB", | 20260507 01:11:37.830 | INFO | "pinned" : " 29 GB" | 20260507 01:11:37.830 | INFO | } | 20260507 01:11:37.830 | INFO | Perf report for the last 5.14681 seconds: | 20260507 01:11:37.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:11:37.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:37.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:37.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:37.830 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 01:11:37.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:37.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:37.830 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:11:37.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:37.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:37.830 | INFO | Uploaded 36GiB to GPU in 1.94732s (18.4869GiB/s) | 20260507 01:11:37.830 | INFO | Downloaded 34GiB to GPU in 1.986s (17.1199GiB/s) | 20260507 01:11:42.973 | INFO | Analyzing chunk frames = [5918208,5918720) | 20260507 01:11:42.973 | INFO | MemInfo: { | 20260507 01:11:42.973 | INFO | "gpu" : " 17 GB", | 20260507 01:11:42.973 | INFO | "huge" : "", | 20260507 01:11:42.973 | INFO | "huge_shared" : " 27 GB", | 20260507 01:11:42.973 | INFO | "malloc" : " 14 GB", | 20260507 01:11:42.973 | INFO | "pinned" : " 29 GB" | 20260507 01:11:42.973 | INFO | } | 20260507 01:11:42.973 | INFO | Perf report for the last 5.14278 seconds: | 20260507 01:11:42.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:11:42.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:42.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:42.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:42.973 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:11:42.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:42.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:42.973 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 01:11:42.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:42.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:42.973 | INFO | Uploaded 36GiB to GPU in 1.98573s (18.1294GiB/s) | 20260507 01:11:42.973 | INFO | Downloaded 34GiB to GPU in 1.9752s (17.2134GiB/s) | 20260507 01:11:43.070 | INFO | Starting a round of Baz IO | 20260507 01:11:46.186 | INFO | Populating RTMetrics report | 20260507 01:11:46.186 | INFO | RTMetrics report populated | 20260507 01:11:46.186 | INFO | Opening tmp RTMetrics json file | 20260507 01:11:46.187 | INFO | Serializing RTMetrics to json | 20260507 01:11:46.190 | INFO | Serialization done, swapping tmp file now... | 20260507 01:11:46.195 | INFO | RTMetrics done | 20260507 01:11:46.276 | INFO | Finished a round of Baz IO: Wrote 1.822 GiB in 3.205 seconds | 20260507 01:11:46.276 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.17% / 0.00% / 66.83% | 20260507 01:11:46.276 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.301095% / 0% / 9.13938e-05% | 20260507 01:11:46.276 | INFO | Baz IO overall: | 20260507 01:11:46.276 | INFO | BytesWritten : 3074378179184 | 20260507 01:11:46.276 | INFO | NumEvents : 1480129134713 | 20260507 01:11:48.109 | INFO | Analyzing chunk frames = [5918720,5919232) | 20260507 01:11:48.109 | INFO | MemInfo: { | 20260507 01:11:48.109 | INFO | "gpu" : " 17 GB", | 20260507 01:11:48.109 | INFO | "huge" : "", | 20260507 01:11:48.109 | INFO | "huge_shared" : " 27 GB", | 20260507 01:11:48.109 | INFO | "malloc" : " 14 GB", | 20260507 01:11:48.109 | INFO | "pinned" : " 29 GB" | 20260507 01:11:48.109 | INFO | } | 20260507 01:11:48.109 | INFO | Perf report for the last 5.13627 seconds: | 20260507 01:11:48.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:11:48.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:48.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:48.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:48.109 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260507 01:11:48.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:48.109 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:11:48.109 | INFO | PRE_HQ : 59.6%, 1.8, 55.9ms, 0 | 20260507 01:11:48.109 | INFO | BAZWRITER : 62.0%, 3.3, 106.4ms, 0 | 20260507 01:11:48.109 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:11:48.110 | INFO | Uploaded 36GiB to GPU in 2.04868s (17.5723GiB/s) | 20260507 01:11:48.110 | INFO | Downloaded 38GiB to GPU in 2.18344s (17.4037GiB/s) | 20260507 01:11:53.254 | INFO | Analyzing chunk frames = [5919232,5919744) | 20260507 01:11:53.254 | INFO | MemInfo: { | 20260507 01:11:53.254 | INFO | "gpu" : " 17 GB", | 20260507 01:11:53.254 | INFO | "huge" : "", | 20260507 01:11:53.254 | INFO | "huge_shared" : " 27 GB", | 20260507 01:11:53.254 | INFO | "malloc" : " 14 GB", | 20260507 01:11:53.254 | INFO | "pinned" : " 29 GB" | 20260507 01:11:53.254 | INFO | } | 20260507 01:11:53.254 | INFO | Perf report for the last 5.14499 seconds: | 20260507 01:11:53.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:11:53.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:53.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:53.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:53.254 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:11:53.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:53.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:53.254 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:11:53.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:53.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:53.255 | INFO | Uploaded 36GiB to GPU in 2.03833s (17.6616GiB/s) | 20260507 01:11:53.255 | INFO | Downloaded 34GiB to GPU in 1.93536s (17.5678GiB/s) | 20260507 01:11:58.390 | INFO | Analyzing chunk frames = [5919744,5920256) | 20260507 01:11:58.390 | INFO | MemInfo: { | 20260507 01:11:58.390 | INFO | "gpu" : " 17 GB", | 20260507 01:11:58.390 | INFO | "huge" : "", | 20260507 01:11:58.390 | INFO | "huge_shared" : " 27 GB", | 20260507 01:11:58.390 | INFO | "malloc" : " 14 GB", | 20260507 01:11:58.390 | INFO | "pinned" : " 29 GB" | 20260507 01:11:58.390 | INFO | } | 20260507 01:11:58.390 | INFO | Perf report for the last 5.13604 seconds: | 20260507 01:11:58.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:11:58.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:58.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:58.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:58.390 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 01:11:58.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:58.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:58.390 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:11:58.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:58.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:11:58.391 | INFO | Uploaded 36GiB to GPU in 2.01112s (17.9004GiB/s) | 20260507 01:11:58.391 | INFO | Downloaded 34GiB to GPU in 1.93599s (17.5621GiB/s) | 20260507 01:12:03.525 | INFO | Analyzing chunk frames = [5920256,5920768) | 20260507 01:12:03.526 | INFO | MemInfo: { | 20260507 01:12:03.526 | INFO | "gpu" : " 17 GB", | 20260507 01:12:03.526 | INFO | "huge" : "", | 20260507 01:12:03.526 | INFO | "huge_shared" : " 27 GB", | 20260507 01:12:03.526 | INFO | "malloc" : " 14 GB", | 20260507 01:12:03.526 | INFO | "pinned" : " 29 GB" | 20260507 01:12:03.526 | INFO | } | 20260507 01:12:03.526 | INFO | Perf report for the last 5.13528 seconds: | 20260507 01:12:03.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:12:03.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:03.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:03.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:03.526 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260507 01:12:03.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:03.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:03.526 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:12:03.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:03.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:03.526 | INFO | Uploaded 36GiB to GPU in 1.99889s (18.01GiB/s) | 20260507 01:12:03.526 | INFO | Downloaded 34GiB to GPU in 1.91264s (17.7765GiB/s) | 20260507 01:12:08.664 | INFO | Analyzing chunk frames = [5920768,5921280) | 20260507 01:12:08.664 | INFO | MemInfo: { | 20260507 01:12:08.664 | INFO | "gpu" : " 17 GB", | 20260507 01:12:08.664 | INFO | "huge" : "", | 20260507 01:12:08.664 | INFO | "huge_shared" : " 27 GB", | 20260507 01:12:08.664 | INFO | "malloc" : " 14 GB", | 20260507 01:12:08.664 | INFO | "pinned" : " 29 GB" | 20260507 01:12:08.664 | INFO | } | 20260507 01:12:08.664 | INFO | Perf report for the last 5.13859 seconds: | 20260507 01:12:08.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:12:08.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:08.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:08.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:08.664 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 01:12:08.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:08.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:08.664 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:12:08.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:08.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:08.665 | INFO | Uploaded 36GiB to GPU in 1.98577s (18.129GiB/s) | 20260507 01:12:08.665 | INFO | Downloaded 34GiB to GPU in 1.92544s (17.6583GiB/s) | 20260507 01:12:13.804 | INFO | Analyzing chunk frames = [5921280,5921792) | 20260507 01:12:13.804 | INFO | MemInfo: { | 20260507 01:12:13.804 | INFO | "gpu" : " 17 GB", | 20260507 01:12:13.804 | INFO | "huge" : "", | 20260507 01:12:13.804 | INFO | "huge_shared" : " 27 GB", | 20260507 01:12:13.804 | INFO | "malloc" : " 14 GB", | 20260507 01:12:13.804 | INFO | "pinned" : " 29 GB" | 20260507 01:12:13.804 | INFO | } | 20260507 01:12:13.804 | INFO | Perf report for the last 5.14003 seconds: | 20260507 01:12:13.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:12:13.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:13.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:13.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:13.804 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:12:13.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:13.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:13.804 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 01:12:13.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:13.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:13.804 | INFO | Uploaded 36GiB to GPU in 2.06108s (17.4665GiB/s) | 20260507 01:12:13.804 | INFO | Downloaded 34GiB to GPU in 1.95479s (17.3932GiB/s) | 20260507 01:12:18.945 | INFO | Analyzing chunk frames = [5921792,5922304) | 20260507 01:12:18.945 | INFO | MemInfo: { | 20260507 01:12:18.945 | INFO | "gpu" : " 17 GB", | 20260507 01:12:18.945 | INFO | "huge" : "", | 20260507 01:12:18.946 | INFO | "huge_shared" : " 27 GB", | 20260507 01:12:18.946 | INFO | "malloc" : " 14 GB", | 20260507 01:12:18.946 | INFO | "pinned" : " 29 GB" | 20260507 01:12:18.946 | INFO | } | 20260507 01:12:18.946 | INFO | Perf report for the last 5.1413 seconds: | 20260507 01:12:18.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:12:18.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:18.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:18.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:18.946 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:12:18.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:18.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:18.946 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 01:12:18.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:18.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:18.946 | INFO | Uploaded 36GiB to GPU in 1.99061s (18.0849GiB/s) | 20260507 01:12:18.946 | INFO | Downloaded 34GiB to GPU in 1.98436s (17.134GiB/s) | 20260507 01:12:24.089 | INFO | Analyzing chunk frames = [5922304,5922816) | 20260507 01:12:24.089 | INFO | MemInfo: { | 20260507 01:12:24.089 | INFO | "gpu" : " 17 GB", | 20260507 01:12:24.089 | INFO | "huge" : "", | 20260507 01:12:24.089 | INFO | "huge_shared" : " 27 GB", | 20260507 01:12:24.089 | INFO | "malloc" : " 14 GB", | 20260507 01:12:24.089 | INFO | "pinned" : " 29 GB" | 20260507 01:12:24.089 | INFO | } | 20260507 01:12:24.089 | INFO | Perf report for the last 5.14334 seconds: | 20260507 01:12:24.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:12:24.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:24.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:24.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:24.089 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260507 01:12:24.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:24.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:24.089 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 01:12:24.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:24.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:24.089 | INFO | Uploaded 36GiB to GPU in 2.00214s (17.9808GiB/s) | 20260507 01:12:24.089 | INFO | Downloaded 34GiB to GPU in 1.97572s (17.2089GiB/s) | 20260507 01:12:24.089 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:12:24.089 | INFO | Total mem usage: 18 GiB | 20260507 01:12:24.089 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:12:24.089 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:12:24.089 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:12:24.089 | INFO | High Water Breakdown: | 20260507 01:12:24.089 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:12:24.089 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:12:24.090 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:12:24.090 | INFO | ------------------------------------------ | 20260507 01:12:24.090 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:12:24.090 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:12:24.090 | INFO | Total mem usage: 26.5 GiB | 20260507 01:12:24.090 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:12:24.090 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:12:24.090 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:12:24.090 | INFO | High Water Breakdown: | 20260507 01:12:24.090 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:12:24.090 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:12:24.090 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:12:24.090 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:12:24.090 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:12:24.090 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:12:24.090 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:12:24.090 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:12:24.090 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:12:24.090 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:12:24.090 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:12:24.090 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:12:24.090 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:12:24.090 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:12:24.090 | INFO | ------------------------------------------ | 20260507 01:12:24.090 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:12:24.090 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:12:24.090 | INFO | Total mem usage: 25.6 GiB | 20260507 01:12:24.090 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:12:24.090 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:12:24.090 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:12:24.090 | INFO | High Water Breakdown: | 20260507 01:12:24.090 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:12:24.090 | INFO | ------------------------------------------ | 20260507 01:12:24.090 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:12:24.090 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:12:24.090 | INFO | Total mem usage: 15.7 GiB | 20260507 01:12:24.090 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:12:24.090 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:12:24.090 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:12:24.090 | INFO | High Water Breakdown: | 20260507 01:12:24.090 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:12:24.090 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:12:24.090 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:12:24.090 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:12:24.090 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:12:24.090 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:12:24.090 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:12:24.090 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:12:24.090 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:12:24.090 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:12:24.090 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:12:24.090 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:12:24.090 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:12:24.090 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:12:24.090 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:12:24.090 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:12:24.090 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:12:24.090 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:12:24.090 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:12:24.090 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:12:24.090 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:12:24.090 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:12:24.090 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:12:24.090 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:12:24.090 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:12:24.090 | INFO | ------------------------------------------ | 20260507 01:12:24.090 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:12:24.090 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:12:24.090 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:12:24.090 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:12:24.090 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.58e+01, 1.34e+01 / 2.04e+00 | 20260507 01:12:24.090 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.02e-01, 9.43e-03 / 4.34e-03 | 20260507 01:12:24.090 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.46e+00, 5.29e-01 / 2.53e-01 | 20260507 01:12:24.090 | INFO | PerfReport: 3.23%, Histogram -- 9.99e-01 / 1.48e+00, 1.71e-01 / 1.01e-01 | 20260507 01:12:24.090 | INFO | PerfReport: 16.11%, DME -- 3.98e+01 / 4.48e+01, 4.08e+00 / 2.26e+00 | 20260507 01:12:24.090 | INFO | PerfReport: 57.47%, FrameLabeling -- 1.77e+01 / 2.06e+01, 1.30e+00 / 1.98e+00 | 20260507 01:12:24.090 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.17e+00 / 2.70e+00, 3.82e-01 / 2.38e-01 | 20260507 01:12:24.090 | INFO | PerfReport: 3.12%, Metrics -- 9.64e-01 / 1.28e+00, 2.55e-01 / 1.80e-01 | 20260507 01:12:24.090 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.28e-01 / 3.82e-01, 5.14e-02 / 2.40e-02 | 20260507 01:12:24.090 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:12:24.090 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:12:24.090 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:12:24.090 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 01:12:24.090 | INFO | PerfReport: 21.65%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260507 01:12:24.090 | INFO | PerfReport: 21.32%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 01:12:24.090 | INFO | PerfReport: 31.99%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 01:12:24.090 | INFO | PerfReport: 1.41%, LockUpload -- 1.36e+00 / 0.00e+00, 4.98e+00 / 0.00e+00 | 20260507 01:12:24.090 | INFO | PerfReport: 1.28%, LockDownload -- 1.24e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260507 01:12:24.090 | INFO | PerfReport: 22.33%, LockCompute -- 2.16e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 01:12:24.090 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260507 01:12:24.186 | INFO | Starting a round of Baz IO | 20260507 01:12:27.333 | INFO | Populating RTMetrics report | 20260507 01:12:27.333 | INFO | RTMetrics report populated | 20260507 01:12:27.333 | INFO | Opening tmp RTMetrics json file | 20260507 01:12:27.334 | INFO | Serializing RTMetrics to json | 20260507 01:12:27.335 | INFO | Serialization done, swapping tmp file now... | 20260507 01:12:27.336 | INFO | RTMetrics done | 20260507 01:12:27.427 | INFO | Finished a round of Baz IO: Wrote 1.824 GiB in 3.24 seconds | 20260507 01:12:27.427 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.17% / 0.00% / 66.83% | 20260507 01:12:27.427 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.297169% / 7.94729e-06% / 7.15256e-05% | 20260507 01:12:27.427 | INFO | Baz IO overall: | 20260507 01:12:27.427 | INFO | BytesWritten : 3076336225763 | 20260507 01:12:27.427 | INFO | NumEvents : 1481056166306 | 20260507 01:12:29.227 | INFO | Analyzing chunk frames = [5922816,5923328) | 20260507 01:12:29.227 | INFO | MemInfo: { | 20260507 01:12:29.227 | INFO | "gpu" : " 17 GB", | 20260507 01:12:29.227 | INFO | "huge" : "", | 20260507 01:12:29.227 | INFO | "huge_shared" : " 27 GB", | 20260507 01:12:29.227 | INFO | "malloc" : " 14 GB", | 20260507 01:12:29.227 | INFO | "pinned" : " 29 GB" | 20260507 01:12:29.227 | INFO | } | 20260507 01:12:29.227 | INFO | Perf report for the last 5.13775 seconds: | 20260507 01:12:29.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:12:29.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:29.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:29.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:29.227 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260507 01:12:29.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:29.227 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260507 01:12:29.227 | INFO | PRE_HQ : 60.2%, 1.9, 60.8ms, 0 | 20260507 01:12:29.227 | INFO | BAZWRITER : 62.6%, 3.4, 110.3ms, 0 | 20260507 01:12:29.227 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:12:29.227 | INFO | Uploaded 36GiB to GPU in 2.03025s (17.7318GiB/s) | 20260507 01:12:29.227 | INFO | Downloaded 38GiB to GPU in 2.17422s (17.4775GiB/s) | 20260507 01:12:34.361 | INFO | Analyzing chunk frames = [5923328,5923840) | 20260507 01:12:34.361 | INFO | MemInfo: { | 20260507 01:12:34.361 | INFO | "gpu" : " 17 GB", | 20260507 01:12:34.361 | INFO | "huge" : "", | 20260507 01:12:34.361 | INFO | "huge_shared" : " 27 GB", | 20260507 01:12:34.361 | INFO | "malloc" : " 14 GB", | 20260507 01:12:34.361 | INFO | "pinned" : " 29 GB" | 20260507 01:12:34.361 | INFO | } | 20260507 01:12:34.362 | INFO | Perf report for the last 5.1348 seconds: | 20260507 01:12:34.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:12:34.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:34.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:34.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:34.362 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:12:34.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:34.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:34.362 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 01:12:34.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:34.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:34.362 | INFO | Uploaded 36GiB to GPU in 2.03888s (17.6567GiB/s) | 20260507 01:12:34.362 | INFO | Downloaded 34GiB to GPU in 1.93862s (17.5382GiB/s) | 20260507 01:12:39.505 | INFO | Analyzing chunk frames = [5923840,5924352) | 20260507 01:12:39.505 | INFO | MemInfo: { | 20260507 01:12:39.505 | INFO | "gpu" : " 17 GB", | 20260507 01:12:39.505 | INFO | "huge" : "", | 20260507 01:12:39.505 | INFO | "huge_shared" : " 27 GB", | 20260507 01:12:39.505 | INFO | "malloc" : " 14 GB", | 20260507 01:12:39.505 | INFO | "pinned" : " 29 GB" | 20260507 01:12:39.505 | INFO | } | 20260507 01:12:39.505 | INFO | Perf report for the last 5.14337 seconds: | 20260507 01:12:39.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:12:39.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:39.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:39.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:39.505 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 01:12:39.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:39.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:39.505 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:12:39.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:39.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:39.505 | INFO | Uploaded 36GiB to GPU in 2.00768s (17.9312GiB/s) | 20260507 01:12:39.505 | INFO | Downloaded 34GiB to GPU in 1.93261s (17.5928GiB/s) | 20260507 01:12:44.638 | INFO | Analyzing chunk frames = [5924352,5924864) | 20260507 01:12:44.638 | INFO | MemInfo: { | 20260507 01:12:44.638 | INFO | "gpu" : " 17 GB", | 20260507 01:12:44.638 | INFO | "huge" : "", | 20260507 01:12:44.638 | INFO | "huge_shared" : " 27 GB", | 20260507 01:12:44.638 | INFO | "malloc" : " 14 GB", | 20260507 01:12:44.638 | INFO | "pinned" : " 29 GB" | 20260507 01:12:44.638 | INFO | } | 20260507 01:12:44.638 | INFO | Perf report for the last 5.1331 seconds: | 20260507 01:12:44.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:12:44.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:44.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:44.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:44.638 | INFO | ANALYSIS : 60.0%, 3.0, 92.9ms, 0 | 20260507 01:12:44.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:44.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:44.638 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:12:44.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:44.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:44.638 | INFO | Uploaded 36GiB to GPU in 1.98987s (18.0917GiB/s) | 20260507 01:12:44.638 | INFO | Downloaded 34GiB to GPU in 1.91021s (17.7991GiB/s) | 20260507 01:12:49.777 | INFO | Analyzing chunk frames = [5924864,5925376) | 20260507 01:12:49.777 | INFO | MemInfo: { | 20260507 01:12:49.777 | INFO | "gpu" : " 17 GB", | 20260507 01:12:49.777 | INFO | "huge" : "", | 20260507 01:12:49.777 | INFO | "huge_shared" : " 27 GB", | 20260507 01:12:49.777 | INFO | "malloc" : " 14 GB", | 20260507 01:12:49.777 | INFO | "pinned" : " 29 GB" | 20260507 01:12:49.777 | INFO | } | 20260507 01:12:49.777 | INFO | Perf report for the last 5.13922 seconds: | 20260507 01:12:49.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:12:49.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:49.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:49.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:49.777 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 01:12:49.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:49.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:49.777 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 01:12:49.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:49.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:49.777 | INFO | Uploaded 36GiB to GPU in 2.00232s (17.9792GiB/s) | 20260507 01:12:49.777 | INFO | Downloaded 34GiB to GPU in 1.91723s (17.7339GiB/s) | 20260507 01:12:54.923 | INFO | Analyzing chunk frames = [5925376,5925888) | 20260507 01:12:54.923 | INFO | MemInfo: { | 20260507 01:12:54.923 | INFO | "gpu" : " 17 GB", | 20260507 01:12:54.923 | INFO | "huge" : "", | 20260507 01:12:54.923 | INFO | "huge_shared" : " 27 GB", | 20260507 01:12:54.923 | INFO | "malloc" : " 14 GB", | 20260507 01:12:54.923 | INFO | "pinned" : " 29 GB" | 20260507 01:12:54.923 | INFO | } | 20260507 01:12:54.923 | INFO | Perf report for the last 5.14619 seconds: | 20260507 01:12:54.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:12:54.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:54.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:54.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:54.923 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 01:12:54.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:54.923 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:54.923 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:12:54.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:54.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:12:54.924 | INFO | Uploaded 36GiB to GPU in 2.02799s (17.7516GiB/s) | 20260507 01:12:54.924 | INFO | Downloaded 34GiB to GPU in 1.93899s (17.5349GiB/s) | 20260507 01:13:00.069 | INFO | Analyzing chunk frames = [5925888,5926400) | 20260507 01:13:00.069 | INFO | MemInfo: { | 20260507 01:13:00.069 | INFO | "gpu" : " 17 GB", | 20260507 01:13:00.069 | INFO | "huge" : "", | 20260507 01:13:00.069 | INFO | "huge_shared" : " 27 GB", | 20260507 01:13:00.069 | INFO | "malloc" : " 14 GB", | 20260507 01:13:00.069 | INFO | "pinned" : " 29 GB" | 20260507 01:13:00.069 | INFO | } | 20260507 01:13:00.069 | INFO | Perf report for the last 5.14604 seconds: | 20260507 01:13:00.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:13:00.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:00.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:00.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:00.070 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 01:13:00.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:00.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:00.070 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 01:13:00.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:00.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:00.070 | INFO | Uploaded 36GiB to GPU in 1.98041s (18.1781GiB/s) | 20260507 01:13:00.070 | INFO | Downloaded 34GiB to GPU in 1.98201s (17.1543GiB/s) | 20260507 01:13:05.202 | INFO | Analyzing chunk frames = [5926400,5926912) | 20260507 01:13:05.202 | INFO | MemInfo: { | 20260507 01:13:05.202 | INFO | "gpu" : " 17 GB", | 20260507 01:13:05.202 | INFO | "huge" : "", | 20260507 01:13:05.202 | INFO | "huge_shared" : " 27 GB", | 20260507 01:13:05.202 | INFO | "malloc" : " 14 GB", | 20260507 01:13:05.202 | INFO | "pinned" : " 29 GB" | 20260507 01:13:05.202 | INFO | } | 20260507 01:13:05.202 | INFO | Perf report for the last 5.13246 seconds: | 20260507 01:13:05.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:13:05.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:05.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:05.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:05.202 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:13:05.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:05.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:05.202 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:13:05.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:05.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:05.202 | INFO | Uploaded 36GiB to GPU in 2.00102s (17.9908GiB/s) | 20260507 01:13:05.202 | INFO | Downloaded 34GiB to GPU in 1.95896s (17.3561GiB/s) | 20260507 01:13:05.302 | INFO | Starting a round of Baz IO | 20260507 01:13:08.415 | INFO | Populating RTMetrics report | 20260507 01:13:08.415 | INFO | RTMetrics report populated | 20260507 01:13:08.415 | INFO | Opening tmp RTMetrics json file | 20260507 01:13:08.416 | INFO | Serializing RTMetrics to json | 20260507 01:13:08.417 | INFO | Serialization done, swapping tmp file now... | 20260507 01:13:08.417 | INFO | RTMetrics done | 20260507 01:13:08.512 | INFO | Finished a round of Baz IO: Wrote 1.828 GiB in 3.209 seconds | 20260507 01:13:08.512 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.17% / 0.00% / 66.83% | 20260507 01:13:08.512 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.288526% / 0% / 9.13938e-05% | 20260507 01:13:08.512 | INFO | Baz IO overall: | 20260507 01:13:08.512 | INFO | BytesWritten : 3078298514610 | 20260507 01:13:08.512 | INFO | NumEvents : 1481982343123 | 20260507 01:13:10.339 | INFO | Analyzing chunk frames = [5926912,5927424) | 20260507 01:13:10.339 | INFO | MemInfo: { | 20260507 01:13:10.339 | INFO | "gpu" : " 17 GB", | 20260507 01:13:10.339 | INFO | "huge" : "", | 20260507 01:13:10.339 | INFO | "huge_shared" : " 27 GB", | 20260507 01:13:10.339 | INFO | "malloc" : " 14 GB", | 20260507 01:13:10.340 | INFO | "pinned" : " 29 GB" | 20260507 01:13:10.340 | INFO | } | 20260507 01:13:10.340 | INFO | Perf report for the last 5.13759 seconds: | 20260507 01:13:10.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:13:10.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:10.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:10.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:10.340 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260507 01:13:10.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:10.340 | INFO | RT_METRICS : 41.7%, 1.0, 21.8ms, 0 | 20260507 01:13:10.340 | INFO | PRE_HQ : 59.5%, 1.8, 56.7ms, 0 | 20260507 01:13:10.340 | INFO | BAZWRITER : 62.1%, 3.4, 107.1ms, 0 | 20260507 01:13:10.340 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:13:10.340 | INFO | Uploaded 36GiB to GPU in 2.03581s (17.6834GiB/s) | 20260507 01:13:10.340 | INFO | Downloaded 38GiB to GPU in 2.17816s (17.4459GiB/s) | 20260507 01:13:15.478 | INFO | Analyzing chunk frames = [5927424,5927936) | 20260507 01:13:15.478 | INFO | MemInfo: { | 20260507 01:13:15.478 | INFO | "gpu" : " 17 GB", | 20260507 01:13:15.478 | INFO | "huge" : "", | 20260507 01:13:15.478 | INFO | "huge_shared" : " 27 GB", | 20260507 01:13:15.478 | INFO | "malloc" : " 14 GB", | 20260507 01:13:15.478 | INFO | "pinned" : " 29 GB" | 20260507 01:13:15.478 | INFO | } | 20260507 01:13:15.478 | INFO | Perf report for the last 5.13816 seconds: | 20260507 01:13:15.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:13:15.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:15.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:15.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:15.478 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:13:15.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:15.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:15.478 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:13:15.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:15.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:15.478 | INFO | Uploaded 36GiB to GPU in 2.00203s (17.9818GiB/s) | 20260507 01:13:15.478 | INFO | Downloaded 34GiB to GPU in 1.92621s (17.6512GiB/s) | 20260507 01:13:20.615 | INFO | Analyzing chunk frames = [5927936,5928448) | 20260507 01:13:20.615 | INFO | MemInfo: { | 20260507 01:13:20.615 | INFO | "gpu" : " 17 GB", | 20260507 01:13:20.615 | INFO | "huge" : "", | 20260507 01:13:20.615 | INFO | "huge_shared" : " 27 GB", | 20260507 01:13:20.615 | INFO | "malloc" : " 14 GB", | 20260507 01:13:20.615 | INFO | "pinned" : " 29 GB" | 20260507 01:13:20.615 | INFO | } | 20260507 01:13:20.615 | INFO | Perf report for the last 5.13691 seconds: | 20260507 01:13:20.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:13:20.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:20.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:20.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:20.615 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 01:13:20.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:20.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:20.615 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:13:20.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:20.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:20.615 | INFO | Uploaded 36GiB to GPU in 1.9998s (18.0018GiB/s) | 20260507 01:13:20.615 | INFO | Downloaded 34GiB to GPU in 1.93424s (17.578GiB/s) | 20260507 01:13:25.761 | INFO | Analyzing chunk frames = [5928448,5928960) | 20260507 01:13:25.761 | INFO | MemInfo: { | 20260507 01:13:25.761 | INFO | "gpu" : " 17 GB", | 20260507 01:13:25.761 | INFO | "huge" : "", | 20260507 01:13:25.761 | INFO | "huge_shared" : " 27 GB", | 20260507 01:13:25.761 | INFO | "malloc" : " 14 GB", | 20260507 01:13:25.761 | INFO | "pinned" : " 29 GB" | 20260507 01:13:25.761 | INFO | } | 20260507 01:13:25.761 | INFO | Perf report for the last 5.1462 seconds: | 20260507 01:13:25.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:13:25.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:25.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:25.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:25.761 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260507 01:13:25.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:25.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:25.761 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:13:25.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:25.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:25.761 | INFO | Uploaded 36GiB to GPU in 1.97536s (18.2245GiB/s) | 20260507 01:13:25.761 | INFO | Downloaded 34GiB to GPU in 1.90519s (17.846GiB/s) | 20260507 01:13:30.894 | INFO | Analyzing chunk frames = [5928960,5929472) | 20260507 01:13:30.894 | INFO | MemInfo: { | 20260507 01:13:30.894 | INFO | "gpu" : " 17 GB", | 20260507 01:13:30.894 | INFO | "huge" : "", | 20260507 01:13:30.894 | INFO | "huge_shared" : " 27 GB", | 20260507 01:13:30.894 | INFO | "malloc" : " 14 GB", | 20260507 01:13:30.894 | INFO | "pinned" : " 29 GB" | 20260507 01:13:30.894 | INFO | } | 20260507 01:13:30.894 | INFO | Perf report for the last 5.13296 seconds: | 20260507 01:13:30.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:13:30.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:30.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:30.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:30.894 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 01:13:30.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:30.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:30.894 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:13:30.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:30.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:30.894 | INFO | Uploaded 36GiB to GPU in 2.02217s (17.8027GiB/s) | 20260507 01:13:30.894 | INFO | Downloaded 34GiB to GPU in 1.94476s (17.4829GiB/s) | 20260507 01:13:36.037 | INFO | Analyzing chunk frames = [5929472,5929984) | 20260507 01:13:36.037 | INFO | MemInfo: { | 20260507 01:13:36.037 | INFO | "gpu" : " 17 GB", | 20260507 01:13:36.037 | INFO | "huge" : "", | 20260507 01:13:36.037 | INFO | "huge_shared" : " 27 GB", | 20260507 01:13:36.037 | INFO | "malloc" : " 14 GB", | 20260507 01:13:36.037 | INFO | "pinned" : " 29 GB" | 20260507 01:13:36.037 | INFO | } | 20260507 01:13:36.038 | INFO | Perf report for the last 5.14366 seconds: | 20260507 01:13:36.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:13:36.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:36.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:36.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:36.038 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 01:13:36.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:36.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:36.038 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 01:13:36.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:36.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:36.038 | INFO | Uploaded 36GiB to GPU in 2.05642s (17.5062GiB/s) | 20260507 01:13:36.038 | INFO | Downloaded 34GiB to GPU in 1.94897s (17.4451GiB/s) | 20260507 01:13:41.176 | INFO | Analyzing chunk frames = [5929984,5930496) | 20260507 01:13:41.176 | INFO | MemInfo: { | 20260507 01:13:41.176 | INFO | "gpu" : " 17 GB", | 20260507 01:13:41.176 | INFO | "huge" : "", | 20260507 01:13:41.176 | INFO | "huge_shared" : " 27 GB", | 20260507 01:13:41.176 | INFO | "malloc" : " 14 GB", | 20260507 01:13:41.176 | INFO | "pinned" : " 29 GB" | 20260507 01:13:41.176 | INFO | } | 20260507 01:13:41.176 | INFO | Perf report for the last 5.13816 seconds: | 20260507 01:13:41.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:13:41.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:41.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:41.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:41.176 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 01:13:41.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:41.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:41.176 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 01:13:41.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:41.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:41.176 | INFO | Uploaded 36GiB to GPU in 1.98392s (18.1459GiB/s) | 20260507 01:13:41.176 | INFO | Downloaded 34GiB to GPU in 1.95309s (17.4083GiB/s) | 20260507 01:13:46.312 | INFO | Analyzing chunk frames = [5930496,5931008) | 20260507 01:13:46.312 | INFO | MemInfo: { | 20260507 01:13:46.312 | INFO | "gpu" : " 17 GB", | 20260507 01:13:46.312 | INFO | "huge" : "", | 20260507 01:13:46.312 | INFO | "huge_shared" : " 27 GB", | 20260507 01:13:46.312 | INFO | "malloc" : " 14 GB", | 20260507 01:13:46.312 | INFO | "pinned" : " 29 GB" | 20260507 01:13:46.312 | INFO | } | 20260507 01:13:46.312 | INFO | Perf report for the last 5.13667 seconds: | 20260507 01:13:46.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:13:46.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:46.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:46.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:46.312 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:13:46.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:46.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:46.312 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 01:13:46.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:46.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:46.313 | INFO | Uploaded 36GiB to GPU in 1.99886s (18.0102GiB/s) | 20260507 01:13:46.313 | INFO | Downloaded 34GiB to GPU in 1.9668s (17.287GiB/s) | 20260507 01:13:46.313 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:13:46.313 | INFO | Total mem usage: 18 GiB | 20260507 01:13:46.313 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:13:46.313 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:13:46.313 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:13:46.313 | INFO | High Water Breakdown: | 20260507 01:13:46.313 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:13:46.313 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:13:46.313 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:13:46.313 | INFO | ------------------------------------------ | 20260507 01:13:46.313 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:13:46.313 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:13:46.313 | INFO | Total mem usage: 26.5 GiB | 20260507 01:13:46.313 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:13:46.313 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:13:46.313 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:13:46.313 | INFO | High Water Breakdown: | 20260507 01:13:46.313 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:13:46.313 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:13:46.313 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:13:46.313 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:13:46.313 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:13:46.313 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:13:46.313 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:13:46.313 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:13:46.313 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:13:46.313 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:13:46.313 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:13:46.313 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:13:46.313 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:13:46.313 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:13:46.313 | INFO | ------------------------------------------ | 20260507 01:13:46.313 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:13:46.313 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:13:46.313 | INFO | Total mem usage: 25.6 GiB | 20260507 01:13:46.313 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:13:46.313 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:13:46.313 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:13:46.313 | INFO | High Water Breakdown: | 20260507 01:13:46.313 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:13:46.313 | INFO | ------------------------------------------ | 20260507 01:13:46.313 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:13:46.313 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:13:46.313 | INFO | Total mem usage: 15.7 GiB | 20260507 01:13:46.313 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:13:46.313 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:13:46.313 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:13:46.313 | INFO | High Water Breakdown: | 20260507 01:13:46.313 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:13:46.313 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:13:46.313 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:13:46.313 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:13:46.313 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:13:46.313 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:13:46.313 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:13:46.313 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:13:46.313 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:13:46.313 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:13:46.313 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:13:46.313 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:13:46.313 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:13:46.313 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:13:46.313 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:13:46.313 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:13:46.313 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:13:46.313 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:13:46.313 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:13:46.313 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:13:46.314 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:13:46.314 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:13:46.314 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:13:46.314 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:13:46.314 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:13:46.314 | INFO | ------------------------------------------ | 20260507 01:13:46.314 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:13:46.314 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:13:46.314 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:13:46.314 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:13:46.314 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.54e+01, 1.35e+01 / 2.00e+00 | 20260507 01:13:46.314 | INFO | PerfReport: 0.30%, AggregateStats -- 9.16e-02 / 1.03e-01, 9.63e-03 / 5.92e-03 | 20260507 01:13:46.314 | INFO | PerfReport: 11.68%, Baseline -- 3.61e+00 / 4.43e+00, 5.30e-01 / 3.12e-01 | 20260507 01:13:46.314 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.47e+00, 1.71e-01 / 1.16e-01 | 20260507 01:13:46.314 | INFO | PerfReport: 16.14%, DME -- 3.99e+01 / 4.42e+01, 4.01e+00 / 2.78e+00 | 20260507 01:13:46.314 | INFO | PerfReport: 57.47%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.34e+00 / 2.06e+00 | 20260507 01:13:46.314 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.16e+00 / 2.68e+00, 3.80e-01 / 2.48e-01 | 20260507 01:13:46.314 | INFO | PerfReport: 3.11%, Metrics -- 9.60e-01 / 1.27e+00, 2.55e-01 / 1.39e-01 | 20260507 01:13:46.314 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.25e-01 / 3.74e-01, 5.19e-02 / 1.92e-02 | 20260507 01:13:46.314 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:13:46.314 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:13:46.314 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:13:46.314 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 01:13:46.314 | INFO | PerfReport: 21.69%, Upload -- 2.09e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260507 01:13:46.314 | INFO | PerfReport: 21.25%, Download -- 2.05e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 01:13:46.314 | INFO | PerfReport: 32.01%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 01:13:46.314 | INFO | PerfReport: 1.46%, LockUpload -- 1.41e+00 / 0.00e+00, 4.94e+00 / 0.00e+00 | 20260507 01:13:46.314 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260507 01:13:46.314 | INFO | PerfReport: 22.36%, LockCompute -- 2.16e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 01:13:46.314 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260507 01:13:46.410 | INFO | Starting a round of Baz IO | 20260507 01:13:49.532 | INFO | Populating RTMetrics report | 20260507 01:13:49.532 | INFO | RTMetrics report populated | 20260507 01:13:49.532 | INFO | Opening tmp RTMetrics json file | 20260507 01:13:49.533 | INFO | Serializing RTMetrics to json | 20260507 01:13:49.534 | INFO | Serialization done, swapping tmp file now... | 20260507 01:13:49.535 | INFO | RTMetrics done | 20260507 01:13:49.628 | INFO | Finished a round of Baz IO: Wrote 1.825 GiB in 3.217 seconds | 20260507 01:13:49.628 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.17% / 0.00% / 66.83% | 20260507 01:13:49.628 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.289087% / 0% / 8.74201e-05% | 20260507 01:13:49.628 | INFO | Baz IO overall: | 20260507 01:13:49.628 | INFO | BytesWritten : 3080257585392 | 20260507 01:13:49.628 | INFO | NumEvents : 1482907398366 | 20260507 01:13:51.446 | INFO | Analyzing chunk frames = [5931008,5931520) | 20260507 01:13:51.446 | INFO | MemInfo: { | 20260507 01:13:51.446 | INFO | "gpu" : " 17 GB", | 20260507 01:13:51.446 | INFO | "huge" : "", | 20260507 01:13:51.446 | INFO | "huge_shared" : " 27 GB", | 20260507 01:13:51.446 | INFO | "malloc" : " 14 GB", | 20260507 01:13:51.446 | INFO | "pinned" : " 29 GB" | 20260507 01:13:51.446 | INFO | } | 20260507 01:13:51.446 | INFO | Perf report for the last 5.13407 seconds: | 20260507 01:13:51.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:13:51.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:51.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:51.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:51.447 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260507 01:13:51.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:51.447 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 01:13:51.447 | INFO | PRE_HQ : 59.7%, 1.8, 56.2ms, 0 | 20260507 01:13:51.447 | INFO | BAZWRITER : 62.2%, 3.3, 106.5ms, 0 | 20260507 01:13:51.447 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:13:51.447 | INFO | Uploaded 36GiB to GPU in 2.03605s (17.6813GiB/s) | 20260507 01:13:51.447 | INFO | Downloaded 38GiB to GPU in 2.17594s (17.4637GiB/s) | 20260507 01:13:56.589 | INFO | Analyzing chunk frames = [5931520,5932032) | 20260507 01:13:56.589 | INFO | MemInfo: { | 20260507 01:13:56.589 | INFO | "gpu" : " 17 GB", | 20260507 01:13:56.589 | INFO | "huge" : "", | 20260507 01:13:56.589 | INFO | "huge_shared" : " 27 GB", | 20260507 01:13:56.589 | INFO | "malloc" : " 14 GB", | 20260507 01:13:56.589 | INFO | "pinned" : " 29 GB" | 20260507 01:13:56.589 | INFO | } | 20260507 01:13:56.589 | INFO | Perf report for the last 5.14301 seconds: | 20260507 01:13:56.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:13:56.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:56.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:56.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:56.590 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 01:13:56.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:56.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:56.590 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 01:13:56.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:56.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:13:56.590 | INFO | Uploaded 36GiB to GPU in 2.00934s (17.9163GiB/s) | 20260507 01:13:56.590 | INFO | Downloaded 34GiB to GPU in 1.93196s (17.5987GiB/s) | 20260507 01:14:01.734 | INFO | Analyzing chunk frames = [5932032,5932544) | 20260507 01:14:01.734 | INFO | MemInfo: { | 20260507 01:14:01.734 | INFO | "gpu" : " 17 GB", | 20260507 01:14:01.734 | INFO | "huge" : "", | 20260507 01:14:01.734 | INFO | "huge_shared" : " 27 GB", | 20260507 01:14:01.734 | INFO | "malloc" : " 14 GB", | 20260507 01:14:01.734 | INFO | "pinned" : " 29 GB" | 20260507 01:14:01.734 | INFO | } | 20260507 01:14:01.734 | INFO | Perf report for the last 5.14472 seconds: | 20260507 01:14:01.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:14:01.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:01.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:01.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:01.734 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 01:14:01.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:01.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:01.734 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:14:01.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:01.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:01.735 | INFO | Uploaded 36GiB to GPU in 1.99864s (18.0123GiB/s) | 20260507 01:14:01.735 | INFO | Downloaded 34GiB to GPU in 1.93225s (17.5961GiB/s) | 20260507 01:14:06.876 | INFO | Analyzing chunk frames = [5932544,5933056) | 20260507 01:14:06.876 | INFO | MemInfo: { | 20260507 01:14:06.876 | INFO | "gpu" : " 17 GB", | 20260507 01:14:06.876 | INFO | "huge" : "", | 20260507 01:14:06.876 | INFO | "huge_shared" : " 27 GB", | 20260507 01:14:06.876 | INFO | "malloc" : " 14 GB", | 20260507 01:14:06.876 | INFO | "pinned" : " 29 GB" | 20260507 01:14:06.876 | INFO | } | 20260507 01:14:06.876 | INFO | Perf report for the last 5.14182 seconds: | 20260507 01:14:06.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:14:06.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:06.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:06.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:06.876 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 01:14:06.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:06.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:06.876 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:14:06.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:06.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:06.876 | INFO | Uploaded 36GiB to GPU in 1.97307s (18.2456GiB/s) | 20260507 01:14:06.876 | INFO | Downloaded 34GiB to GPU in 1.90651s (17.8337GiB/s) | 20260507 01:14:12.014 | INFO | Analyzing chunk frames = [5933056,5933568) | 20260507 01:14:12.014 | INFO | MemInfo: { | 20260507 01:14:12.014 | INFO | "gpu" : " 17 GB", | 20260507 01:14:12.014 | INFO | "huge" : "", | 20260507 01:14:12.014 | INFO | "huge_shared" : " 27 GB", | 20260507 01:14:12.014 | INFO | "malloc" : " 14 GB", | 20260507 01:14:12.014 | INFO | "pinned" : " 29 GB" | 20260507 01:14:12.014 | INFO | } | 20260507 01:14:12.014 | INFO | Perf report for the last 5.1382 seconds: | 20260507 01:14:12.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:14:12.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:12.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:12.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:12.014 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 01:14:12.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:12.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:12.014 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:14:12.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:12.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:12.015 | INFO | Uploaded 36GiB to GPU in 2.03213s (17.7154GiB/s) | 20260507 01:14:12.015 | INFO | Downloaded 34GiB to GPU in 1.93351s (17.5846GiB/s) | 20260507 01:14:17.153 | INFO | Analyzing chunk frames = [5933568,5934080) | 20260507 01:14:17.153 | INFO | MemInfo: { | 20260507 01:14:17.153 | INFO | "gpu" : " 17 GB", | 20260507 01:14:17.153 | INFO | "huge" : "", | 20260507 01:14:17.153 | INFO | "huge_shared" : " 27 GB", | 20260507 01:14:17.153 | INFO | "malloc" : " 14 GB", | 20260507 01:14:17.153 | INFO | "pinned" : " 29 GB" | 20260507 01:14:17.153 | INFO | } | 20260507 01:14:17.153 | INFO | Perf report for the last 5.13919 seconds: | 20260507 01:14:17.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:14:17.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:17.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:17.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:17.154 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 01:14:17.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:17.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:17.154 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:14:17.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:17.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:17.154 | INFO | Uploaded 36GiB to GPU in 2.05748s (17.4971GiB/s) | 20260507 01:14:17.154 | INFO | Downloaded 34GiB to GPU in 1.96355s (17.3156GiB/s) | 20260507 01:14:22.294 | INFO | Analyzing chunk frames = [5934080,5934592) | 20260507 01:14:22.294 | INFO | MemInfo: { | 20260507 01:14:22.294 | INFO | "gpu" : " 17 GB", | 20260507 01:14:22.294 | INFO | "huge" : "", | 20260507 01:14:22.294 | INFO | "huge_shared" : " 27 GB", | 20260507 01:14:22.294 | INFO | "malloc" : " 14 GB", | 20260507 01:14:22.294 | INFO | "pinned" : " 29 GB" | 20260507 01:14:22.294 | INFO | } | 20260507 01:14:22.294 | INFO | Perf report for the last 5.14057 seconds: | 20260507 01:14:22.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:14:22.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:22.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:22.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:22.294 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 01:14:22.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:22.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:22.294 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 01:14:22.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:22.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:22.294 | INFO | Uploaded 36GiB to GPU in 1.98946s (18.0954GiB/s) | 20260507 01:14:22.294 | INFO | Downloaded 34GiB to GPU in 1.9818s (17.1562GiB/s) | 20260507 01:14:27.425 | INFO | Analyzing chunk frames = [5934592,5935104) | 20260507 01:14:27.425 | INFO | MemInfo: { | 20260507 01:14:27.426 | INFO | "gpu" : " 17 GB", | 20260507 01:14:27.426 | INFO | "huge" : "", | 20260507 01:14:27.426 | INFO | "huge_shared" : " 27 GB", | 20260507 01:14:27.426 | INFO | "malloc" : " 14 GB", | 20260507 01:14:27.426 | INFO | "pinned" : " 29 GB" | 20260507 01:14:27.426 | INFO | } | 20260507 01:14:27.426 | INFO | Perf report for the last 5.1316 seconds: | 20260507 01:14:27.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:14:27.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:27.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:27.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:27.426 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260507 01:14:27.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:27.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:27.426 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 01:14:27.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:27.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:27.426 | INFO | Uploaded 36GiB to GPU in 1.9954s (18.0415GiB/s) | 20260507 01:14:27.426 | INFO | Downloaded 34GiB to GPU in 1.97893s (17.181GiB/s) | 20260507 01:14:27.524 | INFO | Starting a round of Baz IO | 20260507 01:14:30.648 | INFO | Populating RTMetrics report | 20260507 01:14:30.648 | INFO | RTMetrics report populated | 20260507 01:14:30.648 | INFO | Opening tmp RTMetrics json file | 20260507 01:14:30.649 | INFO | Serializing RTMetrics to json | 20260507 01:14:30.655 | INFO | Serialization done, swapping tmp file now... | 20260507 01:14:30.669 | INFO | RTMetrics done | 20260507 01:14:30.741 | INFO | Finished a round of Baz IO: Wrote 1.818 GiB in 3.216 seconds | 20260507 01:14:30.741 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.17% / 0.00% / 66.83% | 20260507 01:14:30.741 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.289285% / 0% / 6.35783e-05% | 20260507 01:14:30.741 | INFO | Baz IO overall: | 20260507 01:14:30.741 | INFO | BytesWritten : 3082209821999 | 20260507 01:14:30.741 | INFO | NumEvents : 1483830456582 | 20260507 01:14:32.564 | INFO | Analyzing chunk frames = [5935104,5935616) | 20260507 01:14:32.564 | INFO | MemInfo: { | 20260507 01:14:32.564 | INFO | "gpu" : " 17 GB", | 20260507 01:14:32.564 | INFO | "huge" : "", | 20260507 01:14:32.564 | INFO | "huge_shared" : " 27 GB", | 20260507 01:14:32.564 | INFO | "malloc" : " 14 GB", | 20260507 01:14:32.565 | INFO | "pinned" : " 29 GB" | 20260507 01:14:32.565 | INFO | } | 20260507 01:14:32.565 | INFO | Perf report for the last 5.1389 seconds: | 20260507 01:14:32.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:14:32.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:32.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:32.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:32.565 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260507 01:14:32.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:32.565 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 01:14:32.565 | INFO | PRE_HQ : 59.6%, 1.8, 56.2ms, 0 | 20260507 01:14:32.565 | INFO | BAZWRITER : 62.2%, 3.3, 106.5ms, 0 | 20260507 01:14:32.565 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 01:14:32.565 | INFO | Uploaded 36GiB to GPU in 2.02313s (17.7942GiB/s) | 20260507 01:14:32.565 | INFO | Downloaded 38GiB to GPU in 2.17595s (17.4636GiB/s) | 20260507 01:14:37.707 | INFO | Analyzing chunk frames = [5935616,5936128) | 20260507 01:14:37.707 | INFO | MemInfo: { | 20260507 01:14:37.707 | INFO | "gpu" : " 17 GB", | 20260507 01:14:37.707 | INFO | "huge" : "", | 20260507 01:14:37.707 | INFO | "huge_shared" : " 27 GB", | 20260507 01:14:37.707 | INFO | "malloc" : " 14 GB", | 20260507 01:14:37.707 | INFO | "pinned" : " 29 GB" | 20260507 01:14:37.707 | INFO | } | 20260507 01:14:37.707 | INFO | Perf report for the last 5.14242 seconds: | 20260507 01:14:37.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:14:37.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:37.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:37.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:37.707 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:14:37.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:37.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:37.707 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:14:37.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:37.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:37.707 | INFO | Uploaded 36GiB to GPU in 1.98586s (18.1281GiB/s) | 20260507 01:14:37.707 | INFO | Downloaded 34GiB to GPU in 1.93372s (17.5827GiB/s) | 20260507 01:14:42.845 | INFO | Analyzing chunk frames = [5936128,5936640) | 20260507 01:14:42.845 | INFO | MemInfo: { | 20260507 01:14:42.845 | INFO | "gpu" : " 17 GB", | 20260507 01:14:42.845 | INFO | "huge" : "", | 20260507 01:14:42.845 | INFO | "huge_shared" : " 27 GB", | 20260507 01:14:42.845 | INFO | "malloc" : " 14 GB", | 20260507 01:14:42.845 | INFO | "pinned" : " 29 GB" | 20260507 01:14:42.845 | INFO | } | 20260507 01:14:42.845 | INFO | Perf report for the last 5.13844 seconds: | 20260507 01:14:42.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:14:42.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:42.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:42.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:42.846 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 01:14:42.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:42.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:42.846 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:14:42.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:42.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:42.846 | INFO | Uploaded 36GiB to GPU in 1.99387s (18.0553GiB/s) | 20260507 01:14:42.846 | INFO | Downloaded 34GiB to GPU in 1.94169s (17.5105GiB/s) | 20260507 01:14:47.993 | INFO | Analyzing chunk frames = [5936640,5937152) | 20260507 01:14:47.993 | INFO | MemInfo: { | 20260507 01:14:47.993 | INFO | "gpu" : " 17 GB", | 20260507 01:14:47.993 | INFO | "huge" : "", | 20260507 01:14:47.993 | INFO | "huge_shared" : " 27 GB", | 20260507 01:14:47.993 | INFO | "malloc" : " 14 GB", | 20260507 01:14:47.993 | INFO | "pinned" : " 29 GB" | 20260507 01:14:47.993 | INFO | } | 20260507 01:14:47.993 | INFO | Perf report for the last 5.14788 seconds: | 20260507 01:14:47.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:14:47.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:47.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:47.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:47.993 | INFO | ANALYSIS : 59.8%, 3.0, 93.0ms, 0 | 20260507 01:14:47.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:47.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:47.993 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:14:47.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:47.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:47.994 | INFO | Uploaded 36GiB to GPU in 1.99816s (18.0166GiB/s) | 20260507 01:14:47.994 | INFO | Downloaded 34GiB to GPU in 1.91112s (17.7906GiB/s) | 20260507 01:14:53.132 | INFO | Analyzing chunk frames = [5937152,5937664) | 20260507 01:14:53.132 | INFO | MemInfo: { | 20260507 01:14:53.132 | INFO | "gpu" : " 17 GB", | 20260507 01:14:53.132 | INFO | "huge" : "", | 20260507 01:14:53.132 | INFO | "huge_shared" : " 27 GB", | 20260507 01:14:53.132 | INFO | "malloc" : " 14 GB", | 20260507 01:14:53.132 | INFO | "pinned" : " 29 GB" | 20260507 01:14:53.132 | INFO | } | 20260507 01:14:53.132 | INFO | Perf report for the last 5.13886 seconds: | 20260507 01:14:53.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:14:53.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:53.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:53.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:53.132 | INFO | ANALYSIS : 60.2%, 3.0, 94.0ms, 0 | 20260507 01:14:53.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:53.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:53.132 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 01:14:53.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:53.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:53.132 | INFO | Uploaded 36GiB to GPU in 2.03635s (17.6787GiB/s) | 20260507 01:14:53.132 | INFO | Downloaded 34GiB to GPU in 1.9384s (17.5402GiB/s) | 20260507 01:14:58.265 | INFO | Analyzing chunk frames = [5937664,5938176) | 20260507 01:14:58.265 | INFO | MemInfo: { | 20260507 01:14:58.265 | INFO | "gpu" : " 17 GB", | 20260507 01:14:58.265 | INFO | "huge" : "", | 20260507 01:14:58.265 | INFO | "huge_shared" : " 27 GB", | 20260507 01:14:58.265 | INFO | "malloc" : " 14 GB", | 20260507 01:14:58.265 | INFO | "pinned" : " 29 GB" | 20260507 01:14:58.265 | INFO | } | 20260507 01:14:58.265 | INFO | Perf report for the last 5.13311 seconds: | 20260507 01:14:58.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:14:58.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:58.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:58.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:58.265 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 01:14:58.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:58.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:58.265 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:14:58.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:58.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:14:58.266 | INFO | Uploaded 36GiB to GPU in 2.04799s (17.5782GiB/s) | 20260507 01:14:58.266 | INFO | Downloaded 34GiB to GPU in 1.9647s (17.3054GiB/s) | 20260507 01:15:03.401 | INFO | Analyzing chunk frames = [5938176,5938688) | 20260507 01:15:03.401 | INFO | MemInfo: { | 20260507 01:15:03.401 | INFO | "gpu" : " 17 GB", | 20260507 01:15:03.401 | INFO | "huge" : "", | 20260507 01:15:03.402 | INFO | "huge_shared" : " 27 GB", | 20260507 01:15:03.402 | INFO | "malloc" : " 14 GB", | 20260507 01:15:03.402 | INFO | "pinned" : " 29 GB" | 20260507 01:15:03.402 | INFO | } | 20260507 01:15:03.402 | INFO | Perf report for the last 5.13621 seconds: | 20260507 01:15:03.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:15:03.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:03.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:03.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:03.402 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:15:03.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:03.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:03.402 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:15:03.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:03.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:03.402 | INFO | Uploaded 36GiB to GPU in 1.98062s (18.1761GiB/s) | 20260507 01:15:03.402 | INFO | Downloaded 34GiB to GPU in 2.00198s (16.9831GiB/s) | 20260507 01:15:08.543 | INFO | Analyzing chunk frames = [5938688,5939200) | 20260507 01:15:08.543 | INFO | MemInfo: { | 20260507 01:15:08.543 | INFO | "gpu" : " 17 GB", | 20260507 01:15:08.543 | INFO | "huge" : "", | 20260507 01:15:08.543 | INFO | "huge_shared" : " 27 GB", | 20260507 01:15:08.543 | INFO | "malloc" : " 14 GB", | 20260507 01:15:08.543 | INFO | "pinned" : " 29 GB" | 20260507 01:15:08.543 | INFO | } | 20260507 01:15:08.543 | INFO | Perf report for the last 5.1411 seconds: | 20260507 01:15:08.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:15:08.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:08.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:08.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:08.543 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:15:08.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:08.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:08.543 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:15:08.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:08.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:08.543 | INFO | Uploaded 36GiB to GPU in 1.98889s (18.1005GiB/s) | 20260507 01:15:08.543 | INFO | Downloaded 34GiB to GPU in 1.96918s (17.2661GiB/s) | 20260507 01:15:08.543 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:15:08.543 | INFO | Total mem usage: 18 GiB | 20260507 01:15:08.543 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:15:08.543 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:15:08.543 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:15:08.543 | INFO | High Water Breakdown: | 20260507 01:15:08.543 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:15:08.543 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:15:08.543 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:15:08.543 | INFO | ------------------------------------------ | 20260507 01:15:08.543 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:15:08.543 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:15:08.543 | INFO | Total mem usage: 26.5 GiB | 20260507 01:15:08.543 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:15:08.543 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:15:08.543 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:15:08.543 | INFO | High Water Breakdown: | 20260507 01:15:08.543 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:15:08.543 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:15:08.543 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:15:08.543 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:15:08.543 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:15:08.543 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:15:08.543 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:15:08.543 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:15:08.543 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:15:08.543 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:15:08.543 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:15:08.543 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:15:08.543 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:15:08.544 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:15:08.544 | INFO | ------------------------------------------ | 20260507 01:15:08.544 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:15:08.544 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:15:08.544 | INFO | Total mem usage: 25.6 GiB | 20260507 01:15:08.544 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:15:08.544 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:15:08.544 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:15:08.544 | INFO | High Water Breakdown: | 20260507 01:15:08.544 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:15:08.544 | INFO | ------------------------------------------ | 20260507 01:15:08.544 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:15:08.544 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:15:08.544 | INFO | Total mem usage: 15.7 GiB | 20260507 01:15:08.544 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:15:08.544 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:15:08.544 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:15:08.544 | INFO | High Water Breakdown: | 20260507 01:15:08.544 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:15:08.544 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:15:08.544 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:15:08.544 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:15:08.544 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:15:08.544 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:15:08.544 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:15:08.544 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:15:08.544 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:15:08.544 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:15:08.544 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:15:08.544 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:15:08.544 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:15:08.544 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:15:08.544 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:15:08.544 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:15:08.544 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:15:08.544 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:15:08.544 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:15:08.544 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:15:08.544 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:15:08.544 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:15:08.544 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:15:08.544 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:15:08.544 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:15:08.544 | INFO | ------------------------------------------ | 20260507 01:15:08.544 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:15:08.544 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:15:08.544 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:15:08.544 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:15:08.544 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.57e+01, 1.34e+01 / 2.12e+00 | 20260507 01:15:08.544 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.02e-01, 1.00e-02 / 7.38e-03 | 20260507 01:15:08.544 | INFO | PerfReport: 11.63%, Baseline -- 3.59e+00 / 4.52e+00, 5.12e-01 / 2.00e-01 | 20260507 01:15:08.544 | INFO | PerfReport: 3.23%, Histogram -- 9.95e-01 / 1.50e+00, 1.72e-01 / 1.02e-01 | 20260507 01:15:08.544 | INFO | PerfReport: 16.19%, DME -- 3.99e+01 / 4.50e+01, 4.20e+00 / 2.17e+00 | 20260507 01:15:08.544 | INFO | PerfReport: 57.47%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.30e+00 / 1.66e+00 | 20260507 01:15:08.544 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.17e+00 / 2.75e+00, 3.82e-01 / 2.01e-01 | 20260507 01:15:08.544 | INFO | PerfReport: 3.11%, Metrics -- 9.60e-01 / 1.28e+00, 2.55e-01 / 1.29e-01 | 20260507 01:15:08.544 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.25e-01 / 3.82e-01, 5.16e-02 / 3.06e-02 | 20260507 01:15:08.544 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:15:08.544 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:15:08.544 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:15:08.544 | INFO | PerfReport: Full Execution -- 9.66e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 01:15:08.544 | INFO | PerfReport: 21.66%, Upload -- 2.09e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 01:15:08.544 | INFO | PerfReport: 21.32%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 01:15:08.544 | INFO | PerfReport: 31.93%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 01:15:08.544 | INFO | PerfReport: 1.44%, LockUpload -- 1.39e+00 / 0.00e+00, 5.03e+00 / 0.00e+00 | 20260507 01:15:08.544 | INFO | PerfReport: 1.31%, LockDownload -- 1.27e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260507 01:15:08.544 | INFO | PerfReport: 22.32%, LockCompute -- 2.16e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 01:15:08.544 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260507 01:15:08.639 | INFO | Starting a round of Baz IO | 20260507 01:15:11.760 | INFO | Populating RTMetrics report | 20260507 01:15:11.761 | INFO | RTMetrics report populated | 20260507 01:15:11.761 | INFO | Opening tmp RTMetrics json file | 20260507 01:15:11.762 | INFO | Serializing RTMetrics to json | 20260507 01:15:11.772 | INFO | Serialization done, swapping tmp file now... | 20260507 01:15:11.782 | INFO | RTMetrics done | 20260507 01:15:11.851 | INFO | Finished a round of Baz IO: Wrote 1.819 GiB in 3.211 seconds | 20260507 01:15:11.851 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.16% / 0.00% / 66.83% | 20260507 01:15:11.851 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.289679% / 3.97364e-06% / 9.53674e-05% | 20260507 01:15:11.852 | INFO | Baz IO overall: | 20260507 01:15:11.852 | INFO | BytesWritten : 3084163161415 | 20260507 01:15:11.852 | INFO | NumEvents : 1484755578361 | 20260507 01:15:13.683 | INFO | Analyzing chunk frames = [5939200,5939712) | 20260507 01:15:13.683 | INFO | MemInfo: { | 20260507 01:15:13.683 | INFO | "gpu" : " 17 GB", | 20260507 01:15:13.683 | INFO | "huge" : "", | 20260507 01:15:13.683 | INFO | "huge_shared" : " 27 GB", | 20260507 01:15:13.683 | INFO | "malloc" : " 14 GB", | 20260507 01:15:13.683 | INFO | "pinned" : " 29 GB" | 20260507 01:15:13.683 | INFO | } | 20260507 01:15:13.683 | INFO | Perf report for the last 5.14064 seconds: | 20260507 01:15:13.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:15:13.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:13.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:13.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:13.684 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 01:15:13.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:13.684 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:15:13.684 | INFO | PRE_HQ : 59.3%, 1.8, 55.7ms, 0 | 20260507 01:15:13.684 | INFO | BAZWRITER : 62.1%, 3.3, 106.0ms, 0 | 20260507 01:15:13.684 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:15:13.684 | INFO | Uploaded 36GiB to GPU in 2.01517s (17.8645GiB/s) | 20260507 01:15:13.684 | INFO | Downloaded 38GiB to GPU in 2.17689s (17.4561GiB/s) | 20260507 01:15:18.820 | INFO | Analyzing chunk frames = [5939712,5940224) | 20260507 01:15:18.820 | INFO | MemInfo: { | 20260507 01:15:18.820 | INFO | "gpu" : " 17 GB", | 20260507 01:15:18.820 | INFO | "huge" : "", | 20260507 01:15:18.820 | INFO | "huge_shared" : " 27 GB", | 20260507 01:15:18.820 | INFO | "malloc" : " 14 GB", | 20260507 01:15:18.820 | INFO | "pinned" : " 29 GB" | 20260507 01:15:18.820 | INFO | } | 20260507 01:15:18.820 | INFO | Perf report for the last 5.13666 seconds: | 20260507 01:15:18.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:15:18.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:18.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:18.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:18.820 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:15:18.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:18.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:18.820 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:15:18.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:18.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:18.820 | INFO | Uploaded 36GiB to GPU in 2.02656s (17.7641GiB/s) | 20260507 01:15:18.820 | INFO | Downloaded 34GiB to GPU in 1.93194s (17.5989GiB/s) | 20260507 01:15:23.965 | INFO | Analyzing chunk frames = [5940224,5940736) | 20260507 01:15:23.965 | INFO | MemInfo: { | 20260507 01:15:23.965 | INFO | "gpu" : " 17 GB", | 20260507 01:15:23.965 | INFO | "huge" : "", | 20260507 01:15:23.965 | INFO | "huge_shared" : " 27 GB", | 20260507 01:15:23.965 | INFO | "malloc" : " 14 GB", | 20260507 01:15:23.965 | INFO | "pinned" : " 29 GB" | 20260507 01:15:23.965 | INFO | } | 20260507 01:15:23.965 | INFO | Perf report for the last 5.14549 seconds: | 20260507 01:15:23.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:15:23.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:23.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:23.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:23.966 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 01:15:23.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:23.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:23.966 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:15:23.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:23.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:23.966 | INFO | Uploaded 36GiB to GPU in 2.00719s (17.9355GiB/s) | 20260507 01:15:23.966 | INFO | Downloaded 34GiB to GPU in 1.93363s (17.5835GiB/s) | 20260507 01:15:29.107 | INFO | Analyzing chunk frames = [5940736,5941248) | 20260507 01:15:29.107 | INFO | MemInfo: { | 20260507 01:15:29.107 | INFO | "gpu" : " 17 GB", | 20260507 01:15:29.107 | INFO | "huge" : "", | 20260507 01:15:29.107 | INFO | "huge_shared" : " 27 GB", | 20260507 01:15:29.107 | INFO | "malloc" : " 14 GB", | 20260507 01:15:29.107 | INFO | "pinned" : " 29 GB" | 20260507 01:15:29.107 | INFO | } | 20260507 01:15:29.107 | INFO | Perf report for the last 5.1419 seconds: | 20260507 01:15:29.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:15:29.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:29.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:29.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:29.107 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:15:29.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:29.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:29.107 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:15:29.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:29.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:29.108 | INFO | Uploaded 36GiB to GPU in 1.97871s (18.1937GiB/s) | 20260507 01:15:29.108 | INFO | Downloaded 34GiB to GPU in 1.90814s (17.8184GiB/s) | 20260507 01:15:34.245 | INFO | Analyzing chunk frames = [5941248,5941760) | 20260507 01:15:34.245 | INFO | MemInfo: { | 20260507 01:15:34.245 | INFO | "gpu" : " 17 GB", | 20260507 01:15:34.245 | INFO | "huge" : "", | 20260507 01:15:34.245 | INFO | "huge_shared" : " 27 GB", | 20260507 01:15:34.245 | INFO | "malloc" : " 14 GB", | 20260507 01:15:34.245 | INFO | "pinned" : " 29 GB" | 20260507 01:15:34.245 | INFO | } | 20260507 01:15:34.246 | INFO | Perf report for the last 5.13812 seconds: | 20260507 01:15:34.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:15:34.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:34.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:34.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:34.246 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 01:15:34.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:34.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:34.246 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:15:34.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:34.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:34.246 | INFO | Uploaded 36GiB to GPU in 2.03329s (17.7053GiB/s) | 20260507 01:15:34.246 | INFO | Downloaded 34GiB to GPU in 1.93852s (17.5392GiB/s) | 20260507 01:15:39.380 | INFO | Analyzing chunk frames = [5941760,5942272) | 20260507 01:15:39.380 | INFO | MemInfo: { | 20260507 01:15:39.380 | INFO | "gpu" : " 17 GB", | 20260507 01:15:39.380 | INFO | "huge" : "", | 20260507 01:15:39.380 | INFO | "huge_shared" : " 27 GB", | 20260507 01:15:39.380 | INFO | "malloc" : " 14 GB", | 20260507 01:15:39.380 | INFO | "pinned" : " 29 GB" | 20260507 01:15:39.380 | INFO | } | 20260507 01:15:39.380 | INFO | Perf report for the last 5.13418 seconds: | 20260507 01:15:39.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:15:39.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:39.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:39.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:39.380 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 01:15:39.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:39.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:39.380 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 01:15:39.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:39.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:39.380 | INFO | Uploaded 36GiB to GPU in 2.05696s (17.5016GiB/s) | 20260507 01:15:39.380 | INFO | Downloaded 34GiB to GPU in 1.96038s (17.3436GiB/s) | 20260507 01:15:44.513 | INFO | Analyzing chunk frames = [5942272,5942784) | 20260507 01:15:44.513 | INFO | MemInfo: { | 20260507 01:15:44.513 | INFO | "gpu" : " 17 GB", | 20260507 01:15:44.513 | INFO | "huge" : "", | 20260507 01:15:44.513 | INFO | "huge_shared" : " 27 GB", | 20260507 01:15:44.513 | INFO | "malloc" : " 14 GB", | 20260507 01:15:44.513 | INFO | "pinned" : " 29 GB" | 20260507 01:15:44.513 | INFO | } | 20260507 01:15:44.513 | INFO | Perf report for the last 5.13296 seconds: | 20260507 01:15:44.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:15:44.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:44.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:44.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:44.513 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260507 01:15:44.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:44.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:44.513 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 01:15:44.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:44.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:44.513 | INFO | Uploaded 36GiB to GPU in 2.00065s (17.9942GiB/s) | 20260507 01:15:44.513 | INFO | Downloaded 34GiB to GPU in 2.00364s (16.9691GiB/s) | 20260507 01:15:49.656 | INFO | Analyzing chunk frames = [5942784,5943296) | 20260507 01:15:49.656 | INFO | MemInfo: { | 20260507 01:15:49.656 | INFO | "gpu" : " 17 GB", | 20260507 01:15:49.656 | INFO | "huge" : "", | 20260507 01:15:49.656 | INFO | "huge_shared" : " 27 GB", | 20260507 01:15:49.656 | INFO | "malloc" : " 14 GB", | 20260507 01:15:49.656 | INFO | "pinned" : " 29 GB" | 20260507 01:15:49.656 | INFO | } | 20260507 01:15:49.656 | INFO | Perf report for the last 5.1436 seconds: | 20260507 01:15:49.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:15:49.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:49.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:49.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:49.656 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:15:49.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:49.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:49.656 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 01:15:49.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:49.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:49.657 | INFO | Uploaded 36GiB to GPU in 1.9789s (18.1919GiB/s) | 20260507 01:15:49.657 | INFO | Downloaded 34GiB to GPU in 1.97487s (17.2163GiB/s) | 20260507 01:15:49.753 | INFO | Starting a round of Baz IO | 20260507 01:15:52.876 | INFO | Populating RTMetrics report | 20260507 01:15:52.876 | INFO | RTMetrics report populated | 20260507 01:15:52.876 | INFO | Opening tmp RTMetrics json file | 20260507 01:15:52.877 | INFO | Serializing RTMetrics to json | 20260507 01:15:52.883 | INFO | Serialization done, swapping tmp file now... | 20260507 01:15:52.897 | INFO | RTMetrics done | 20260507 01:15:52.968 | INFO | Finished a round of Baz IO: Wrote 1.817 GiB in 3.215 seconds | 20260507 01:15:52.968 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.16% / 0.00% / 66.84% | 20260507 01:15:52.968 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.290211% / 3.97364e-06% / 7.94729e-05% | 20260507 01:15:52.968 | INFO | Baz IO overall: | 20260507 01:15:52.968 | INFO | BytesWritten : 3086114304681 | 20260507 01:15:52.968 | INFO | NumEvents : 1485679542839 | 20260507 01:15:54.806 | INFO | Analyzing chunk frames = [5943296,5943808) | 20260507 01:15:54.806 | INFO | MemInfo: { | 20260507 01:15:54.806 | INFO | "gpu" : " 17 GB", | 20260507 01:15:54.806 | INFO | "huge" : "", | 20260507 01:15:54.806 | INFO | "huge_shared" : " 27 GB", | 20260507 01:15:54.806 | INFO | "malloc" : " 14 GB", | 20260507 01:15:54.806 | INFO | "pinned" : " 29 GB" | 20260507 01:15:54.806 | INFO | } | 20260507 01:15:54.806 | INFO | Perf report for the last 5.14951 seconds: | 20260507 01:15:54.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:15:54.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:54.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:54.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:54.806 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 01:15:54.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:54.806 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:15:54.806 | INFO | PRE_HQ : 59.4%, 1.8, 55.9ms, 0 | 20260507 01:15:54.806 | INFO | BAZWRITER : 62.0%, 3.3, 106.4ms, 0 | 20260507 01:15:54.806 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:15:54.806 | INFO | Uploaded 36GiB to GPU in 2.03042s (17.7304GiB/s) | 20260507 01:15:54.806 | INFO | Downloaded 38GiB to GPU in 2.17072s (17.5057GiB/s) | 20260507 01:15:59.937 | INFO | Analyzing chunk frames = [5943808,5944320) | 20260507 01:15:59.937 | INFO | MemInfo: { | 20260507 01:15:59.937 | INFO | "gpu" : " 17 GB", | 20260507 01:15:59.937 | INFO | "huge" : "", | 20260507 01:15:59.938 | INFO | "huge_shared" : " 27 GB", | 20260507 01:15:59.938 | INFO | "malloc" : " 14 GB", | 20260507 01:15:59.938 | INFO | "pinned" : " 29 GB" | 20260507 01:15:59.938 | INFO | } | 20260507 01:15:59.938 | INFO | Perf report for the last 5.13173 seconds: | 20260507 01:15:59.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:15:59.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:59.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:59.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:59.938 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:15:59.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:59.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:59.938 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:15:59.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:59.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:15:59.938 | INFO | Uploaded 36GiB to GPU in 2.02612s (17.7679GiB/s) | 20260507 01:15:59.938 | INFO | Downloaded 34GiB to GPU in 1.93743s (17.549GiB/s) | 20260507 01:16:05.076 | INFO | Analyzing chunk frames = [5944320,5944832) | 20260507 01:16:05.076 | INFO | MemInfo: { | 20260507 01:16:05.076 | INFO | "gpu" : " 17 GB", | 20260507 01:16:05.076 | INFO | "huge" : "", | 20260507 01:16:05.076 | INFO | "huge_shared" : " 27 GB", | 20260507 01:16:05.076 | INFO | "malloc" : " 14 GB", | 20260507 01:16:05.076 | INFO | "pinned" : " 29 GB" | 20260507 01:16:05.076 | INFO | } | 20260507 01:16:05.076 | INFO | Perf report for the last 5.13861 seconds: | 20260507 01:16:05.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:16:05.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:05.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:05.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:05.076 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:16:05.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:05.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:05.076 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:16:05.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:05.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:05.076 | INFO | Uploaded 36GiB to GPU in 1.99147s (18.0771GiB/s) | 20260507 01:16:05.076 | INFO | Downloaded 34GiB to GPU in 1.94179s (17.5096GiB/s) | 20260507 01:16:10.210 | INFO | Analyzing chunk frames = [5944832,5945344) | 20260507 01:16:10.210 | INFO | MemInfo: { | 20260507 01:16:10.210 | INFO | "gpu" : " 17 GB", | 20260507 01:16:10.210 | INFO | "huge" : "", | 20260507 01:16:10.210 | INFO | "huge_shared" : " 27 GB", | 20260507 01:16:10.210 | INFO | "malloc" : " 14 GB", | 20260507 01:16:10.210 | INFO | "pinned" : " 29 GB" | 20260507 01:16:10.210 | INFO | } | 20260507 01:16:10.210 | INFO | Perf report for the last 5.13383 seconds: | 20260507 01:16:10.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:16:10.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:10.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:10.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:10.210 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 01:16:10.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:10.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:10.210 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:16:10.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:10.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:10.210 | INFO | Uploaded 36GiB to GPU in 1.95433s (18.4207GiB/s) | 20260507 01:16:10.210 | INFO | Downloaded 34GiB to GPU in 1.91094s (17.7923GiB/s) | 20260507 01:16:15.353 | INFO | Analyzing chunk frames = [5945344,5945856) | 20260507 01:16:15.353 | INFO | MemInfo: { | 20260507 01:16:15.353 | INFO | "gpu" : " 17 GB", | 20260507 01:16:15.353 | INFO | "huge" : "", | 20260507 01:16:15.353 | INFO | "huge_shared" : " 27 GB", | 20260507 01:16:15.353 | INFO | "malloc" : " 14 GB", | 20260507 01:16:15.353 | INFO | "pinned" : " 29 GB" | 20260507 01:16:15.353 | INFO | } | 20260507 01:16:15.354 | INFO | Perf report for the last 5.14348 seconds: | 20260507 01:16:15.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:16:15.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:15.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:15.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:15.354 | INFO | ANALYSIS : 59.9%, 3.0, 93.4ms, 0 | 20260507 01:16:15.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:15.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:15.354 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:16:15.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:15.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:15.354 | INFO | Uploaded 36GiB to GPU in 1.98382s (18.1468GiB/s) | 20260507 01:16:15.354 | INFO | Downloaded 34GiB to GPU in 1.9255s (17.6577GiB/s) | 20260507 01:16:20.491 | INFO | Analyzing chunk frames = [5945856,5946368) | 20260507 01:16:20.491 | INFO | MemInfo: { | 20260507 01:16:20.491 | INFO | "gpu" : " 17 GB", | 20260507 01:16:20.491 | INFO | "huge" : "", | 20260507 01:16:20.491 | INFO | "huge_shared" : " 27 GB", | 20260507 01:16:20.491 | INFO | "malloc" : " 14 GB", | 20260507 01:16:20.491 | INFO | "pinned" : " 29 GB" | 20260507 01:16:20.491 | INFO | } | 20260507 01:16:20.492 | INFO | Perf report for the last 5.13797 seconds: | 20260507 01:16:20.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:16:20.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:20.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:20.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:20.492 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 01:16:20.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:20.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:20.492 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 01:16:20.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:20.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:20.492 | INFO | Uploaded 36GiB to GPU in 2.04087s (17.6396GiB/s) | 20260507 01:16:20.492 | INFO | Downloaded 34GiB to GPU in 1.96002s (17.3467GiB/s) | 20260507 01:16:25.631 | INFO | Analyzing chunk frames = [5946368,5946880) | 20260507 01:16:25.631 | INFO | MemInfo: { | 20260507 01:16:25.631 | INFO | "gpu" : " 17 GB", | 20260507 01:16:25.631 | INFO | "huge" : "", | 20260507 01:16:25.631 | INFO | "huge_shared" : " 27 GB", | 20260507 01:16:25.631 | INFO | "malloc" : " 14 GB", | 20260507 01:16:25.631 | INFO | "pinned" : " 29 GB" | 20260507 01:16:25.631 | INFO | } | 20260507 01:16:25.631 | INFO | Perf report for the last 5.1395 seconds: | 20260507 01:16:25.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:16:25.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:25.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:25.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:25.631 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:16:25.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:25.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:25.631 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:16:25.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:25.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:25.632 | INFO | Uploaded 36GiB to GPU in 1.97423s (18.235GiB/s) | 20260507 01:16:25.632 | INFO | Downloaded 34GiB to GPU in 1.99723s (17.0236GiB/s) | 20260507 01:16:30.775 | INFO | Analyzing chunk frames = [5946880,5947392) | 20260507 01:16:30.775 | INFO | MemInfo: { | 20260507 01:16:30.775 | INFO | "gpu" : " 17 GB", | 20260507 01:16:30.775 | INFO | "huge" : "", | 20260507 01:16:30.775 | INFO | "huge_shared" : " 27 GB", | 20260507 01:16:30.775 | INFO | "malloc" : " 14 GB", | 20260507 01:16:30.775 | INFO | "pinned" : " 29 GB" | 20260507 01:16:30.775 | INFO | } | 20260507 01:16:30.775 | INFO | Perf report for the last 5.14393 seconds: | 20260507 01:16:30.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:16:30.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:30.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:30.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:30.775 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:16:30.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:30.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:30.775 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:16:30.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:30.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:30.775 | INFO | Uploaded 36GiB to GPU in 1.98661s (18.1214GiB/s) | 20260507 01:16:30.775 | INFO | Downloaded 34GiB to GPU in 1.97969s (17.1744GiB/s) | 20260507 01:16:30.775 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:16:30.775 | INFO | Total mem usage: 18 GiB | 20260507 01:16:30.775 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:16:30.775 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 01:16:30.775 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:16:30.775 | INFO | High Water Breakdown: | 20260507 01:16:30.775 | INFO | 60.1% : 8.58 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:16:30.775 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:16:30.775 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:16:30.775 | INFO | ------------------------------------------ | 20260507 01:16:30.775 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:16:30.775 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:16:30.775 | INFO | Total mem usage: 26.5 GiB | 20260507 01:16:30.775 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:16:30.775 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:16:30.775 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:16:30.775 | INFO | High Water Breakdown: | 20260507 01:16:30.775 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:16:30.776 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:16:30.776 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:16:30.776 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:16:30.776 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:16:30.776 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:16:30.776 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:16:30.776 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:16:30.776 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:16:30.776 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:16:30.776 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:16:30.776 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:16:30.776 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:16:30.776 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:16:30.776 | INFO | ------------------------------------------ | 20260507 01:16:30.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:16:30.776 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:16:30.776 | INFO | Total mem usage: 25.6 GiB | 20260507 01:16:30.776 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:16:30.776 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:16:30.776 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:16:30.776 | INFO | High Water Breakdown: | 20260507 01:16:30.776 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:16:30.776 | INFO | ------------------------------------------ | 20260507 01:16:30.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:16:30.776 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:16:30.776 | INFO | Total mem usage: 15.7 GiB | 20260507 01:16:30.776 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:16:30.776 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:16:30.776 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:16:30.776 | INFO | High Water Breakdown: | 20260507 01:16:30.776 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:16:30.776 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:16:30.776 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:16:30.776 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:16:30.776 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:16:30.776 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:16:30.776 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:16:30.776 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:16:30.776 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:16:30.776 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:16:30.776 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:16:30.776 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:16:30.776 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:16:30.776 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:16:30.776 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:16:30.776 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:16:30.776 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:16:30.776 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:16:30.776 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:16:30.776 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:16:30.776 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:16:30.776 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:16:30.776 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:16:30.776 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:16:30.776 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:16:30.776 | INFO | ------------------------------------------ | 20260507 01:16:30.776 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:16:30.776 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:16:30.776 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:16:30.776 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:16:30.776 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.60e+01, 1.34e+01 / 2.05e+00 | 20260507 01:16:30.776 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.02e-01, 9.30e-03 / 3.37e-03 | 20260507 01:16:30.776 | INFO | PerfReport: 11.66%, Baseline -- 3.60e+00 / 4.49e+00, 5.21e-01 / 1.76e-01 | 20260507 01:16:30.776 | INFO | PerfReport: 3.22%, Histogram -- 9.94e-01 / 1.48e+00, 1.68e-01 / 1.06e-01 | 20260507 01:16:30.776 | INFO | PerfReport: 16.16%, DME -- 3.99e+01 / 4.51e+01, 4.14e+00 / 2.01e+00 | 20260507 01:16:30.776 | INFO | PerfReport: 57.46%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.28e+00 / 1.85e+00 | 20260507 01:16:30.776 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.17e+00 / 2.72e+00, 3.85e-01 / 2.33e-01 | 20260507 01:16:30.776 | INFO | PerfReport: 3.11%, Metrics -- 9.61e-01 / 1.30e+00, 2.56e-01 / 1.73e-01 | 20260507 01:16:30.776 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.26e-01 / 3.84e-01, 5.14e-02 / 2.55e-02 | 20260507 01:16:30.776 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:16:30.776 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:16:30.776 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:16:30.776 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 01:16:30.776 | INFO | PerfReport: 21.64%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260507 01:16:30.776 | INFO | PerfReport: 21.34%, Download -- 2.06e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260507 01:16:30.776 | INFO | PerfReport: 31.97%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 01:16:30.776 | INFO | PerfReport: 1.42%, LockUpload -- 1.37e+00 / 0.00e+00, 4.96e+00 / 0.00e+00 | 20260507 01:16:30.776 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260507 01:16:30.776 | INFO | PerfReport: 22.33%, LockCompute -- 2.16e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 01:16:30.777 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260507 01:16:30.883 | INFO | Starting a round of Baz IO | 20260507 01:16:33.993 | INFO | Populating RTMetrics report | 20260507 01:16:33.994 | INFO | RTMetrics report populated | 20260507 01:16:33.994 | INFO | Opening tmp RTMetrics json file | 20260507 01:16:34.013 | INFO | Serializing RTMetrics to json | 20260507 01:16:34.014 | INFO | Serialization done, swapping tmp file now... | 20260507 01:16:34.015 | INFO | RTMetrics done | 20260507 01:16:34.084 | INFO | Finished a round of Baz IO: Wrote 1.816 GiB in 3.2 seconds | 20260507 01:16:34.084 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.16% / 0.00% / 66.84% | 20260507 01:16:34.084 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.28981% / 0% / 7.94729e-05% | 20260507 01:16:34.084 | INFO | Baz IO overall: | 20260507 01:16:34.084 | INFO | BytesWritten : 3088064691144 | 20260507 01:16:34.084 | INFO | NumEvents : 1486603211858 | 20260507 01:16:35.916 | INFO | Analyzing chunk frames = [5947392,5947904) | 20260507 01:16:35.916 | INFO | MemInfo: { | 20260507 01:16:35.916 | INFO | "gpu" : " 17 GB", | 20260507 01:16:35.916 | INFO | "huge" : "", | 20260507 01:16:35.916 | INFO | "huge_shared" : " 27 GB", | 20260507 01:16:35.916 | INFO | "malloc" : " 14 GB", | 20260507 01:16:35.916 | INFO | "pinned" : " 29 GB" | 20260507 01:16:35.916 | INFO | } | 20260507 01:16:35.916 | INFO | Perf report for the last 5.14103 seconds: | 20260507 01:16:35.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:16:35.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:35.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:35.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:35.916 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260507 01:16:35.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:35.916 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 01:16:35.916 | INFO | PRE_HQ : 59.6%, 1.8, 56.2ms, 0 | 20260507 01:16:35.916 | INFO | BAZWRITER : 62.1%, 3.3, 105.8ms, 0 | 20260507 01:16:35.916 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:16:35.916 | INFO | Uploaded 36GiB to GPU in 2.03253s (17.7119GiB/s) | 20260507 01:16:35.916 | INFO | Downloaded 38GiB to GPU in 2.17611s (17.4624GiB/s) | 20260507 01:16:41.060 | INFO | Analyzing chunk frames = [5947904,5948416) | 20260507 01:16:41.060 | INFO | MemInfo: { | 20260507 01:16:41.060 | INFO | "gpu" : " 17 GB", | 20260507 01:16:41.060 | INFO | "huge" : "", | 20260507 01:16:41.060 | INFO | "huge_shared" : " 27 GB", | 20260507 01:16:41.060 | INFO | "malloc" : " 14 GB", | 20260507 01:16:41.060 | INFO | "pinned" : " 29 GB" | 20260507 01:16:41.060 | INFO | } | 20260507 01:16:41.060 | INFO | Perf report for the last 5.14442 seconds: | 20260507 01:16:41.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:16:41.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:41.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:41.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:41.061 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:16:41.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:41.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:41.061 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:16:41.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:41.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:41.061 | INFO | Uploaded 36GiB to GPU in 2.0395s (17.6514GiB/s) | 20260507 01:16:41.061 | INFO | Downloaded 34GiB to GPU in 1.94027s (17.5234GiB/s) | 20260507 01:16:46.189 | INFO | Analyzing chunk frames = [5948416,5948928) | 20260507 01:16:46.189 | INFO | MemInfo: { | 20260507 01:16:46.189 | INFO | "gpu" : " 17 GB", | 20260507 01:16:46.189 | INFO | "huge" : "", | 20260507 01:16:46.190 | INFO | "huge_shared" : " 27 GB", | 20260507 01:16:46.190 | INFO | "malloc" : " 14 GB", | 20260507 01:16:46.190 | INFO | "pinned" : " 29 GB" | 20260507 01:16:46.190 | INFO | } | 20260507 01:16:46.190 | INFO | Perf report for the last 5.12918 seconds: | 20260507 01:16:46.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:16:46.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:46.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:46.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:46.190 | INFO | ANALYSIS : 60.5%, 3.0, 93.4ms, 0 | 20260507 01:16:46.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:46.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:46.190 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:16:46.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:46.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:46.190 | INFO | Uploaded 36GiB to GPU in 2.01629s (17.8546GiB/s) | 20260507 01:16:46.190 | INFO | Downloaded 34GiB to GPU in 1.93821s (17.5419GiB/s) | 20260507 01:16:51.329 | INFO | Analyzing chunk frames = [5948928,5949440) | 20260507 01:16:51.329 | INFO | MemInfo: { | 20260507 01:16:51.329 | INFO | "gpu" : " 17 GB", | 20260507 01:16:51.329 | INFO | "huge" : "", | 20260507 01:16:51.329 | INFO | "huge_shared" : " 27 GB", | 20260507 01:16:51.329 | INFO | "malloc" : " 14 GB", | 20260507 01:16:51.329 | INFO | "pinned" : " 29 GB" | 20260507 01:16:51.329 | INFO | } | 20260507 01:16:51.329 | INFO | Perf report for the last 5.13928 seconds: | 20260507 01:16:51.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:16:51.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:51.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:51.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:51.329 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:16:51.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:51.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:51.329 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:16:51.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:51.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:51.329 | INFO | Uploaded 36GiB to GPU in 1.97882s (18.1927GiB/s) | 20260507 01:16:51.329 | INFO | Downloaded 34GiB to GPU in 1.89989s (17.8958GiB/s) | 20260507 01:16:56.472 | INFO | Analyzing chunk frames = [5949440,5949952) | 20260507 01:16:56.472 | INFO | MemInfo: { | 20260507 01:16:56.472 | INFO | "gpu" : " 17 GB", | 20260507 01:16:56.472 | INFO | "huge" : "", | 20260507 01:16:56.472 | INFO | "huge_shared" : " 27 GB", | 20260507 01:16:56.472 | INFO | "malloc" : " 14 GB", | 20260507 01:16:56.472 | INFO | "pinned" : " 29 GB" | 20260507 01:16:56.472 | INFO | } | 20260507 01:16:56.472 | INFO | Perf report for the last 5.14315 seconds: | 20260507 01:16:56.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:16:56.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:56.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:56.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:56.472 | INFO | ANALYSIS : 59.9%, 3.0, 93.4ms, 0 | 20260507 01:16:56.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:56.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:56.472 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:16:56.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:56.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:16:56.472 | INFO | Uploaded 36GiB to GPU in 1.99742s (18.0233GiB/s) | 20260507 01:16:56.472 | INFO | Downloaded 34GiB to GPU in 1.92447s (17.6672GiB/s) | 20260507 01:17:01.609 | INFO | Analyzing chunk frames = [5949952,5950464) | 20260507 01:17:01.609 | INFO | MemInfo: { | 20260507 01:17:01.609 | INFO | "gpu" : " 17 GB", | 20260507 01:17:01.609 | INFO | "huge" : "", | 20260507 01:17:01.609 | INFO | "huge_shared" : " 27 GB", | 20260507 01:17:01.609 | INFO | "malloc" : " 14 GB", | 20260507 01:17:01.609 | INFO | "pinned" : " 29 GB" | 20260507 01:17:01.609 | INFO | } | 20260507 01:17:01.609 | INFO | Perf report for the last 5.13659 seconds: | 20260507 01:17:01.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:17:01.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:01.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:01.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:01.609 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:17:01.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:01.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:01.609 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 01:17:01.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:01.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:01.609 | INFO | Uploaded 36GiB to GPU in 2.05874s (17.4864GiB/s) | 20260507 01:17:01.609 | INFO | Downloaded 34GiB to GPU in 1.95304s (17.4088GiB/s) | 20260507 01:17:06.748 | INFO | Analyzing chunk frames = [5950464,5950976) | 20260507 01:17:06.748 | INFO | MemInfo: { | 20260507 01:17:06.748 | INFO | "gpu" : " 17 GB", | 20260507 01:17:06.748 | INFO | "huge" : "", | 20260507 01:17:06.748 | INFO | "huge_shared" : " 27 GB", | 20260507 01:17:06.748 | INFO | "malloc" : " 14 GB", | 20260507 01:17:06.748 | INFO | "pinned" : " 29 GB" | 20260507 01:17:06.748 | INFO | } | 20260507 01:17:06.748 | INFO | Perf report for the last 5.13962 seconds: | 20260507 01:17:06.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:17:06.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:06.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:06.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:06.748 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:17:06.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:06.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:06.748 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 01:17:06.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:06.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:06.749 | INFO | Uploaded 36GiB to GPU in 1.99454s (18.0493GiB/s) | 20260507 01:17:06.749 | INFO | Downloaded 34GiB to GPU in 1.99877s (17.0105GiB/s) | 20260507 01:17:11.890 | INFO | Analyzing chunk frames = [5950976,5951488) | 20260507 01:17:11.890 | INFO | MemInfo: { | 20260507 01:17:11.890 | INFO | "gpu" : " 17 GB", | 20260507 01:17:11.890 | INFO | "huge" : "", | 20260507 01:17:11.890 | INFO | "huge_shared" : " 27 GB", | 20260507 01:17:11.890 | INFO | "malloc" : " 14 GB", | 20260507 01:17:11.890 | INFO | "pinned" : " 29 GB" | 20260507 01:17:11.890 | INFO | } | 20260507 01:17:11.890 | INFO | Perf report for the last 5.14162 seconds: | 20260507 01:17:11.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:17:11.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:11.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:11.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:11.890 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:17:11.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:11.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:11.890 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 01:17:11.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:11.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:11.890 | INFO | Uploaded 36GiB to GPU in 2.00973s (17.9129GiB/s) | 20260507 01:17:11.890 | INFO | Downloaded 34GiB to GPU in 1.97617s (17.205GiB/s) | 20260507 01:17:11.988 | INFO | Starting a round of Baz IO | 20260507 01:17:15.104 | INFO | Populating RTMetrics report | 20260507 01:17:15.104 | INFO | RTMetrics report populated | 20260507 01:17:15.104 | INFO | Opening tmp RTMetrics json file | 20260507 01:17:15.105 | INFO | Serializing RTMetrics to json | 20260507 01:17:15.106 | INFO | Serialization done, swapping tmp file now... | 20260507 01:17:15.106 | INFO | RTMetrics done | 20260507 01:17:15.197 | INFO | Finished a round of Baz IO: Wrote 1.815 GiB in 3.208 seconds | 20260507 01:17:15.197 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.16% / 0.00% / 66.84% | 20260507 01:17:15.197 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.289873% / 3.97364e-06% / 5.5631e-05% | 20260507 01:17:15.197 | INFO | Baz IO overall: | 20260507 01:17:15.197 | INFO | BytesWritten : 3090013430364 | 20260507 01:17:15.197 | INFO | NumEvents : 1487525999976 | 20260507 01:17:17.026 | INFO | Analyzing chunk frames = [5951488,5952000) | 20260507 01:17:17.026 | INFO | MemInfo: { | 20260507 01:17:17.026 | INFO | "gpu" : " 17 GB", | 20260507 01:17:17.026 | INFO | "huge" : "", | 20260507 01:17:17.026 | INFO | "huge_shared" : " 27 GB", | 20260507 01:17:17.026 | INFO | "malloc" : " 14 GB", | 20260507 01:17:17.026 | INFO | "pinned" : " 29 GB" | 20260507 01:17:17.026 | INFO | } | 20260507 01:17:17.027 | INFO | Perf report for the last 5.13653 seconds: | 20260507 01:17:17.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:17:17.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:17.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:17.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:17.027 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 01:17:17.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:17.027 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:17:17.027 | INFO | PRE_HQ : 59.6%, 1.8, 56.4ms, 0 | 20260507 01:17:17.027 | INFO | BAZWRITER : 62.1%, 3.3, 106.2ms, 0 | 20260507 01:17:17.027 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:17:17.027 | INFO | Uploaded 36GiB to GPU in 2.05039s (17.5576GiB/s) | 20260507 01:17:17.027 | INFO | Downloaded 38GiB to GPU in 2.18982s (17.3531GiB/s) | 20260507 01:17:22.170 | INFO | Analyzing chunk frames = [5952000,5952512) | 20260507 01:17:22.170 | INFO | MemInfo: { | 20260507 01:17:22.170 | INFO | "gpu" : " 17 GB", | 20260507 01:17:22.170 | INFO | "huge" : "", | 20260507 01:17:22.170 | INFO | "huge_shared" : " 27 GB", | 20260507 01:17:22.170 | INFO | "malloc" : " 14 GB", | 20260507 01:17:22.170 | INFO | "pinned" : " 29 GB" | 20260507 01:17:22.170 | INFO | } | 20260507 01:17:22.170 | INFO | Perf report for the last 5.14333 seconds: | 20260507 01:17:22.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:17:22.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:22.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:22.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:22.170 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:17:22.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:22.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:22.170 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:17:22.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:22.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:22.170 | INFO | Uploaded 36GiB to GPU in 2.0201s (17.8209GiB/s) | 20260507 01:17:22.170 | INFO | Downloaded 34GiB to GPU in 1.93323s (17.5871GiB/s) | 20260507 01:17:27.304 | INFO | Analyzing chunk frames = [5952512,5953024) | 20260507 01:17:27.304 | INFO | MemInfo: { | 20260507 01:17:27.304 | INFO | "gpu" : " 17 GB", | 20260507 01:17:27.304 | INFO | "huge" : "", | 20260507 01:17:27.304 | INFO | "huge_shared" : " 27 GB", | 20260507 01:17:27.304 | INFO | "malloc" : " 14 GB", | 20260507 01:17:27.304 | INFO | "pinned" : " 29 GB" | 20260507 01:17:27.304 | INFO | } | 20260507 01:17:27.304 | INFO | Perf report for the last 5.13408 seconds: | 20260507 01:17:27.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:17:27.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:27.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:27.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:27.304 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 01:17:27.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:27.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:27.304 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:17:27.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:27.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:27.304 | INFO | Uploaded 36GiB to GPU in 1.99929s (18.0064GiB/s) | 20260507 01:17:27.304 | INFO | Downloaded 34GiB to GPU in 1.93442s (17.5763GiB/s) | 20260507 01:17:32.443 | INFO | Analyzing chunk frames = [5953024,5953536) | 20260507 01:17:32.443 | INFO | MemInfo: { | 20260507 01:17:32.443 | INFO | "gpu" : " 17 GB", | 20260507 01:17:32.443 | INFO | "huge" : "", | 20260507 01:17:32.443 | INFO | "huge_shared" : " 27 GB", | 20260507 01:17:32.443 | INFO | "malloc" : " 14 GB", | 20260507 01:17:32.443 | INFO | "pinned" : " 29 GB" | 20260507 01:17:32.443 | INFO | } | 20260507 01:17:32.443 | INFO | Perf report for the last 5.13874 seconds: | 20260507 01:17:32.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:17:32.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:32.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:32.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:32.443 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260507 01:17:32.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:32.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:32.443 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:17:32.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:32.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:32.443 | INFO | Uploaded 36GiB to GPU in 2.00011s (17.999GiB/s) | 20260507 01:17:32.443 | INFO | Downloaded 34GiB to GPU in 1.90967s (17.8042GiB/s) | 20260507 01:17:37.586 | INFO | Analyzing chunk frames = [5953536,5954048) | 20260507 01:17:37.586 | INFO | MemInfo: { | 20260507 01:17:37.586 | INFO | "gpu" : " 17 GB", | 20260507 01:17:37.586 | INFO | "huge" : "", | 20260507 01:17:37.586 | INFO | "huge_shared" : " 27 GB", | 20260507 01:17:37.586 | INFO | "malloc" : " 14 GB", | 20260507 01:17:37.586 | INFO | "pinned" : " 29 GB" | 20260507 01:17:37.586 | INFO | } | 20260507 01:17:37.586 | INFO | Perf report for the last 5.14332 seconds: | 20260507 01:17:37.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:17:37.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:37.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:37.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:37.586 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 01:17:37.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:37.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:37.586 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:17:37.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:37.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:37.586 | INFO | Uploaded 36GiB to GPU in 1.98607s (18.1263GiB/s) | 20260507 01:17:37.586 | INFO | Downloaded 34GiB to GPU in 1.92598s (17.6533GiB/s) | 20260507 01:17:42.729 | INFO | Analyzing chunk frames = [5954048,5954560) | 20260507 01:17:42.729 | INFO | MemInfo: { | 20260507 01:17:42.729 | INFO | "gpu" : " 17 GB", | 20260507 01:17:42.729 | INFO | "huge" : "", | 20260507 01:17:42.729 | INFO | "huge_shared" : " 27 GB", | 20260507 01:17:42.729 | INFO | "malloc" : " 14 GB", | 20260507 01:17:42.729 | INFO | "pinned" : " 29 GB" | 20260507 01:17:42.729 | INFO | } | 20260507 01:17:42.729 | INFO | Perf report for the last 5.1427 seconds: | 20260507 01:17:42.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:17:42.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:42.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:42.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:42.729 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 01:17:42.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:42.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:42.729 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:17:42.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:42.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:42.729 | INFO | Uploaded 36GiB to GPU in 2.05298s (17.5355GiB/s) | 20260507 01:17:42.729 | INFO | Downloaded 34GiB to GPU in 1.95249s (17.4136GiB/s) | 20260507 01:17:47.870 | INFO | Analyzing chunk frames = [5954560,5955072) | 20260507 01:17:47.870 | INFO | MemInfo: { | 20260507 01:17:47.870 | INFO | "gpu" : " 17 GB", | 20260507 01:17:47.870 | INFO | "huge" : "", | 20260507 01:17:47.870 | INFO | "huge_shared" : " 27 GB", | 20260507 01:17:47.870 | INFO | "malloc" : " 14 GB", | 20260507 01:17:47.870 | INFO | "pinned" : " 29 GB" | 20260507 01:17:47.870 | INFO | } | 20260507 01:17:47.870 | INFO | Perf report for the last 5.14124 seconds: | 20260507 01:17:47.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:17:47.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:47.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:47.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:47.870 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:17:47.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:47.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:47.870 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 01:17:47.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:47.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:47.870 | INFO | Uploaded 36GiB to GPU in 1.99191s (18.0731GiB/s) | 20260507 01:17:47.870 | INFO | Downloaded 34GiB to GPU in 1.97912s (17.1793GiB/s) | 20260507 01:17:53.007 | INFO | Analyzing chunk frames = [5955072,5955584) | 20260507 01:17:53.007 | INFO | MemInfo: { | 20260507 01:17:53.007 | INFO | "gpu" : " 17 GB", | 20260507 01:17:53.007 | INFO | "huge" : "", | 20260507 01:17:53.007 | INFO | "huge_shared" : " 27 GB", | 20260507 01:17:53.007 | INFO | "malloc" : " 14 GB", | 20260507 01:17:53.007 | INFO | "pinned" : " 29 GB" | 20260507 01:17:53.007 | INFO | } | 20260507 01:17:53.007 | INFO | Perf report for the last 5.13713 seconds: | 20260507 01:17:53.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:17:53.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:53.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:53.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:53.007 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:17:53.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:53.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:53.007 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 01:17:53.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:53.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:53.007 | INFO | Uploaded 36GiB to GPU in 2.0078s (17.9301GiB/s) | 20260507 01:17:53.007 | INFO | Downloaded 34GiB to GPU in 1.96979s (17.2607GiB/s) | 20260507 01:17:53.008 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:17:53.008 | INFO | Total mem usage: 18 GiB | 20260507 01:17:53.008 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:17:53.008 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 01:17:53.008 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:17:53.008 | INFO | High Water Breakdown: | 20260507 01:17:53.008 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:17:53.008 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:17:53.008 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:17:53.008 | INFO | ------------------------------------------ | 20260507 01:17:53.008 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:17:53.008 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:17:53.008 | INFO | Total mem usage: 26.5 GiB | 20260507 01:17:53.008 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:17:53.008 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:17:53.008 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:17:53.008 | INFO | High Water Breakdown: | 20260507 01:17:53.008 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:17:53.008 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:17:53.008 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:17:53.008 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:17:53.008 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:17:53.008 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:17:53.008 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:17:53.008 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:17:53.008 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:17:53.008 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:17:53.008 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:17:53.008 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:17:53.008 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:17:53.008 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:17:53.008 | INFO | ------------------------------------------ | 20260507 01:17:53.008 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:17:53.008 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:17:53.008 | INFO | Total mem usage: 25.6 GiB | 20260507 01:17:53.008 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:17:53.008 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:17:53.008 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:17:53.008 | INFO | High Water Breakdown: | 20260507 01:17:53.008 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:17:53.008 | INFO | ------------------------------------------ | 20260507 01:17:53.008 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:17:53.008 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:17:53.008 | INFO | Total mem usage: 15.7 GiB | 20260507 01:17:53.008 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:17:53.008 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:17:53.008 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:17:53.008 | INFO | High Water Breakdown: | 20260507 01:17:53.008 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:17:53.008 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:17:53.008 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:17:53.008 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:17:53.008 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:17:53.008 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:17:53.008 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:17:53.008 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:17:53.008 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:17:53.008 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:17:53.008 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:17:53.008 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:17:53.008 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:17:53.008 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:17:53.008 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:17:53.008 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:17:53.008 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:17:53.008 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:17:53.008 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:17:53.008 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:17:53.008 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:17:53.008 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:17:53.008 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:17:53.008 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:17:53.008 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:17:53.008 | INFO | ------------------------------------------ | 20260507 01:17:53.008 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:17:53.008 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:17:53.008 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:17:53.008 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:17:53.008 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.56e+01, 1.33e+01 / 1.85e+00 | 20260507 01:17:53.008 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.02e-01, 9.62e-03 / 7.97e-03 | 20260507 01:17:53.008 | INFO | PerfReport: 11.65%, Baseline -- 3.59e+00 / 4.45e+00, 5.23e-01 / 2.87e-01 | 20260507 01:17:53.008 | INFO | PerfReport: 3.24%, Histogram -- 9.99e-01 / 1.47e+00, 1.72e-01 / 1.27e-01 | 20260507 01:17:53.008 | INFO | PerfReport: 16.10%, DME -- 3.97e+01 / 4.44e+01, 3.98e+00 / 2.47e+00 | 20260507 01:17:53.008 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.33e+00 / 2.17e+00 | 20260507 01:17:53.008 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.17e+00 / 2.69e+00, 3.81e-01 / 2.59e-01 | 20260507 01:17:53.008 | INFO | PerfReport: 3.11%, Metrics -- 9.60e-01 / 1.29e+00, 2.56e-01 / 1.57e-01 | 20260507 01:17:53.008 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.26e-01 / 3.76e-01, 5.19e-02 / 2.46e-02 | 20260507 01:17:53.008 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:17:53.008 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:17:53.009 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:17:53.009 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 01:17:53.009 | INFO | PerfReport: 21.75%, Upload -- 2.10e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260507 01:17:53.009 | INFO | PerfReport: 21.32%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 01:17:53.009 | INFO | PerfReport: 31.98%, Compute -- 3.09e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 01:17:53.009 | INFO | PerfReport: 1.45%, LockUpload -- 1.40e+00 / 0.00e+00, 5.01e+00 / 0.00e+00 | 20260507 01:17:53.009 | INFO | PerfReport: 1.19%, LockDownload -- 1.15e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260507 01:17:53.009 | INFO | PerfReport: 22.30%, LockCompute -- 2.15e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 01:17:53.009 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260507 01:17:53.108 | INFO | Starting a round of Baz IO | 20260507 01:17:56.217 | INFO | Populating RTMetrics report | 20260507 01:17:56.217 | INFO | RTMetrics report populated | 20260507 01:17:56.217 | INFO | Opening tmp RTMetrics json file | 20260507 01:17:56.218 | INFO | Serializing RTMetrics to json | 20260507 01:17:56.221 | INFO | Serialization done, swapping tmp file now... | 20260507 01:17:56.228 | INFO | RTMetrics done | 20260507 01:17:56.308 | INFO | Finished a round of Baz IO: Wrote 1.817 GiB in 3.199 seconds | 20260507 01:17:56.308 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.16% / 0.00% / 66.84% | 20260507 01:17:56.308 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.290406% / 0% / 7.15256e-05% | 20260507 01:17:56.308 | INFO | Baz IO overall: | 20260507 01:17:56.308 | INFO | BytesWritten : 3091964019217 | 20260507 01:17:56.308 | INFO | NumEvents : 1488450362315 | 20260507 01:17:58.143 | INFO | Analyzing chunk frames = [5955584,5956096) | 20260507 01:17:58.143 | INFO | MemInfo: { | 20260507 01:17:58.143 | INFO | "gpu" : " 17 GB", | 20260507 01:17:58.143 | INFO | "huge" : "", | 20260507 01:17:58.143 | INFO | "huge_shared" : " 27 GB", | 20260507 01:17:58.143 | INFO | "malloc" : " 14 GB", | 20260507 01:17:58.143 | INFO | "pinned" : " 29 GB" | 20260507 01:17:58.143 | INFO | } | 20260507 01:17:58.143 | INFO | Perf report for the last 5.13581 seconds: | 20260507 01:17:58.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:17:58.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:58.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:58.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:58.143 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260507 01:17:58.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:17:58.143 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:17:58.143 | INFO | PRE_HQ : 59.4%, 1.8, 56.3ms, 0 | 20260507 01:17:58.143 | INFO | BAZWRITER : 61.9%, 3.3, 105.7ms, 0 | 20260507 01:17:58.143 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:17:58.143 | INFO | Uploaded 36GiB to GPU in 2.03344s (17.704GiB/s) | 20260507 01:17:58.143 | INFO | Downloaded 38GiB to GPU in 2.174s (17.4793GiB/s) | 20260507 01:18:03.278 | INFO | Analyzing chunk frames = [5956096,5956608) | 20260507 01:18:03.278 | INFO | MemInfo: { | 20260507 01:18:03.278 | INFO | "gpu" : " 17 GB", | 20260507 01:18:03.278 | INFO | "huge" : "", | 20260507 01:18:03.278 | INFO | "huge_shared" : " 27 GB", | 20260507 01:18:03.278 | INFO | "malloc" : " 14 GB", | 20260507 01:18:03.278 | INFO | "pinned" : " 29 GB" | 20260507 01:18:03.278 | INFO | } | 20260507 01:18:03.278 | INFO | Perf report for the last 5.13535 seconds: | 20260507 01:18:03.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:18:03.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:03.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:03.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:03.278 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:18:03.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:03.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:03.278 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:18:03.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:03.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:03.279 | INFO | Uploaded 36GiB to GPU in 1.99451s (18.0495GiB/s) | 20260507 01:18:03.279 | INFO | Downloaded 34GiB to GPU in 1.92973s (17.619GiB/s) | 20260507 01:18:08.421 | INFO | Analyzing chunk frames = [5956608,5957120) | 20260507 01:18:08.421 | INFO | MemInfo: { | 20260507 01:18:08.421 | INFO | "gpu" : " 17 GB", | 20260507 01:18:08.421 | INFO | "huge" : "", | 20260507 01:18:08.421 | INFO | "huge_shared" : " 27 GB", | 20260507 01:18:08.421 | INFO | "malloc" : " 14 GB", | 20260507 01:18:08.421 | INFO | "pinned" : " 29 GB" | 20260507 01:18:08.421 | INFO | } | 20260507 01:18:08.421 | INFO | Perf report for the last 5.14259 seconds: | 20260507 01:18:08.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:18:08.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:08.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:08.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:08.421 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:18:08.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:08.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:08.421 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 01:18:08.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:08.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:08.421 | INFO | Uploaded 36GiB to GPU in 1.97307s (18.2457GiB/s) | 20260507 01:18:08.421 | INFO | Downloaded 34GiB to GPU in 1.92799s (17.635GiB/s) | 20260507 01:18:13.562 | INFO | Analyzing chunk frames = [5957120,5957632) | 20260507 01:18:13.562 | INFO | MemInfo: { | 20260507 01:18:13.562 | INFO | "gpu" : " 17 GB", | 20260507 01:18:13.562 | INFO | "huge" : "", | 20260507 01:18:13.562 | INFO | "huge_shared" : " 27 GB", | 20260507 01:18:13.562 | INFO | "malloc" : " 14 GB", | 20260507 01:18:13.562 | INFO | "pinned" : " 29 GB" | 20260507 01:18:13.562 | INFO | } | 20260507 01:18:13.562 | INFO | Perf report for the last 5.14077 seconds: | 20260507 01:18:13.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:18:13.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:13.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:13.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:13.562 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 01:18:13.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:13.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:13.562 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:18:13.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:13.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:13.562 | INFO | Uploaded 36GiB to GPU in 2.00168s (17.9849GiB/s) | 20260507 01:18:13.562 | INFO | Downloaded 34GiB to GPU in 1.91056s (17.7958GiB/s) | 20260507 01:18:18.704 | INFO | Analyzing chunk frames = [5957632,5958144) | 20260507 01:18:18.704 | INFO | MemInfo: { | 20260507 01:18:18.704 | INFO | "gpu" : " 17 GB", | 20260507 01:18:18.704 | INFO | "huge" : "", | 20260507 01:18:18.704 | INFO | "huge_shared" : " 27 GB", | 20260507 01:18:18.704 | INFO | "malloc" : " 14 GB", | 20260507 01:18:18.704 | INFO | "pinned" : " 29 GB" | 20260507 01:18:18.704 | INFO | } | 20260507 01:18:18.704 | INFO | Perf report for the last 5.14234 seconds: | 20260507 01:18:18.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:18:18.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:18.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:18.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:18.704 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 01:18:18.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:18.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:18.704 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:18:18.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:18.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:18.704 | INFO | Uploaded 36GiB to GPU in 2.01688s (17.8494GiB/s) | 20260507 01:18:18.704 | INFO | Downloaded 34GiB to GPU in 1.94963s (17.4392GiB/s) | 20260507 01:18:23.842 | INFO | Analyzing chunk frames = [5958144,5958656) | 20260507 01:18:23.842 | INFO | MemInfo: { | 20260507 01:18:23.842 | INFO | "gpu" : " 17 GB", | 20260507 01:18:23.842 | INFO | "huge" : "", | 20260507 01:18:23.842 | INFO | "huge_shared" : " 27 GB", | 20260507 01:18:23.842 | INFO | "malloc" : " 14 GB", | 20260507 01:18:23.842 | INFO | "pinned" : " 29 GB" | 20260507 01:18:23.842 | INFO | } | 20260507 01:18:23.842 | INFO | Perf report for the last 5.13812 seconds: | 20260507 01:18:23.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:18:23.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:23.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:23.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:23.842 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260507 01:18:23.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:23.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:23.842 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 01:18:23.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:23.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:23.842 | INFO | Uploaded 36GiB to GPU in 2.03516s (17.6891GiB/s) | 20260507 01:18:23.842 | INFO | Downloaded 34GiB to GPU in 1.93279s (17.5911GiB/s) | 20260507 01:18:28.978 | INFO | Analyzing chunk frames = [5958656,5959168) | 20260507 01:18:28.978 | INFO | MemInfo: { | 20260507 01:18:28.978 | INFO | "gpu" : " 17 GB", | 20260507 01:18:28.978 | INFO | "huge" : "", | 20260507 01:18:28.978 | INFO | "huge_shared" : " 27 GB", | 20260507 01:18:28.978 | INFO | "malloc" : " 14 GB", | 20260507 01:18:28.978 | INFO | "pinned" : " 29 GB" | 20260507 01:18:28.978 | INFO | } | 20260507 01:18:28.978 | INFO | Perf report for the last 5.13601 seconds: | 20260507 01:18:28.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:18:28.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:28.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:28.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:28.978 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:18:28.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:28.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:28.978 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:18:28.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:28.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:28.978 | INFO | Uploaded 36GiB to GPU in 1.98187s (18.1647GiB/s) | 20260507 01:18:28.978 | INFO | Downloaded 34GiB to GPU in 1.95367s (17.4032GiB/s) | 20260507 01:18:34.122 | INFO | Analyzing chunk frames = [5959168,5959680) | 20260507 01:18:34.122 | INFO | MemInfo: { | 20260507 01:18:34.122 | INFO | "gpu" : " 17 GB", | 20260507 01:18:34.122 | INFO | "huge" : "", | 20260507 01:18:34.122 | INFO | "huge_shared" : " 27 GB", | 20260507 01:18:34.122 | INFO | "malloc" : " 14 GB", | 20260507 01:18:34.122 | INFO | "pinned" : " 29 GB" | 20260507 01:18:34.122 | INFO | } | 20260507 01:18:34.122 | INFO | Perf report for the last 5.14411 seconds: | 20260507 01:18:34.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:18:34.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:34.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:34.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:34.122 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:18:34.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:34.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:34.122 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 01:18:34.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:34.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:34.123 | INFO | Uploaded 36GiB to GPU in 1.98833s (18.1057GiB/s) | 20260507 01:18:34.123 | INFO | Downloaded 34GiB to GPU in 1.97643s (17.2027GiB/s) | 20260507 01:18:34.220 | INFO | Starting a round of Baz IO | 20260507 01:18:37.344 | INFO | Populating RTMetrics report | 20260507 01:18:37.344 | INFO | RTMetrics report populated | 20260507 01:18:37.344 | INFO | Opening tmp RTMetrics json file | 20260507 01:18:37.348 | INFO | Serializing RTMetrics to json | 20260507 01:18:37.363 | INFO | Serialization done, swapping tmp file now... | 20260507 01:18:37.364 | INFO | RTMetrics done | 20260507 01:18:37.435 | INFO | Finished a round of Baz IO: Wrote 1.815 GiB in 3.215 seconds | 20260507 01:18:37.435 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.15% / 0.00% / 66.84% | 20260507 01:18:37.435 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.289472% / 0% / 8.34465e-05% | 20260507 01:18:37.435 | INFO | Baz IO overall: | 20260507 01:18:37.435 | INFO | BytesWritten : 3093912843869 | 20260507 01:18:37.435 | INFO | NumEvents : 1489373311979 | 20260507 01:18:39.255 | INFO | Analyzing chunk frames = [5959680,5960192) | 20260507 01:18:39.255 | INFO | MemInfo: { | 20260507 01:18:39.255 | INFO | "gpu" : " 17 GB", | 20260507 01:18:39.255 | INFO | "huge" : "", | 20260507 01:18:39.255 | INFO | "huge_shared" : " 27 GB", | 20260507 01:18:39.255 | INFO | "malloc" : " 14 GB", | 20260507 01:18:39.255 | INFO | "pinned" : " 29 GB" | 20260507 01:18:39.255 | INFO | } | 20260507 01:18:39.255 | INFO | Perf report for the last 5.13236 seconds: | 20260507 01:18:39.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:18:39.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:39.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:39.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:39.255 | INFO | ANALYSIS : 62.2%, 3.0, 96.7ms, 0 | 20260507 01:18:39.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:39.255 | INFO | RT_METRICS : 42.3%, 1.0, 22.1ms, 0 | 20260507 01:18:39.255 | INFO | PRE_HQ : 59.7%, 1.8, 55.8ms, 0 | 20260507 01:18:39.255 | INFO | BAZWRITER : 62.2%, 3.3, 106.0ms, 0 | 20260507 01:18:39.255 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:18:39.255 | INFO | Uploaded 36GiB to GPU in 2.03908s (17.6551GiB/s) | 20260507 01:18:39.256 | INFO | Downloaded 38GiB to GPU in 2.17165s (17.4982GiB/s) | 20260507 01:18:44.395 | INFO | Analyzing chunk frames = [5960192,5960704) | 20260507 01:18:44.395 | INFO | MemInfo: { | 20260507 01:18:44.395 | INFO | "gpu" : " 17 GB", | 20260507 01:18:44.395 | INFO | "huge" : "", | 20260507 01:18:44.395 | INFO | "huge_shared" : " 27 GB", | 20260507 01:18:44.395 | INFO | "malloc" : " 14 GB", | 20260507 01:18:44.395 | INFO | "pinned" : " 29 GB" | 20260507 01:18:44.395 | INFO | } | 20260507 01:18:44.395 | INFO | Perf report for the last 5.14017 seconds: | 20260507 01:18:44.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:18:44.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:44.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:44.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:44.395 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:18:44.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:44.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:44.395 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 01:18:44.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:44.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:44.395 | INFO | Uploaded 36GiB to GPU in 1.99951s (18.0044GiB/s) | 20260507 01:18:44.395 | INFO | Downloaded 34GiB to GPU in 1.92656s (17.648GiB/s) | 20260507 01:18:49.536 | INFO | Analyzing chunk frames = [5960704,5961216) | 20260507 01:18:49.536 | INFO | MemInfo: { | 20260507 01:18:49.536 | INFO | "gpu" : " 17 GB", | 20260507 01:18:49.536 | INFO | "huge" : "", | 20260507 01:18:49.536 | INFO | "huge_shared" : " 27 GB", | 20260507 01:18:49.536 | INFO | "malloc" : " 14 GB", | 20260507 01:18:49.536 | INFO | "pinned" : " 29 GB" | 20260507 01:18:49.536 | INFO | } | 20260507 01:18:49.536 | INFO | Perf report for the last 5.14105 seconds: | 20260507 01:18:49.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:18:49.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:49.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:49.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:49.536 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 01:18:49.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:49.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:49.536 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:18:49.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:49.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:49.536 | INFO | Uploaded 36GiB to GPU in 1.99392s (18.0549GiB/s) | 20260507 01:18:49.536 | INFO | Downloaded 34GiB to GPU in 1.93131s (17.6046GiB/s) | 20260507 01:18:54.676 | INFO | Analyzing chunk frames = [5961216,5961728) | 20260507 01:18:54.676 | INFO | MemInfo: { | 20260507 01:18:54.676 | INFO | "gpu" : " 17 GB", | 20260507 01:18:54.676 | INFO | "huge" : "", | 20260507 01:18:54.676 | INFO | "huge_shared" : " 27 GB", | 20260507 01:18:54.676 | INFO | "malloc" : " 14 GB", | 20260507 01:18:54.676 | INFO | "pinned" : " 29 GB" | 20260507 01:18:54.676 | INFO | } | 20260507 01:18:54.676 | INFO | Perf report for the last 5.13997 seconds: | 20260507 01:18:54.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:18:54.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:54.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:54.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:54.676 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260507 01:18:54.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:54.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:54.676 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:18:54.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:54.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:54.676 | INFO | Uploaded 36GiB to GPU in 2.00146s (17.9868GiB/s) | 20260507 01:18:54.676 | INFO | Downloaded 34GiB to GPU in 1.90998s (17.8012GiB/s) | 20260507 01:18:59.817 | INFO | Analyzing chunk frames = [5961728,5962240) | 20260507 01:18:59.817 | INFO | MemInfo: { | 20260507 01:18:59.817 | INFO | "gpu" : " 17 GB", | 20260507 01:18:59.817 | INFO | "huge" : "", | 20260507 01:18:59.817 | INFO | "huge_shared" : " 27 GB", | 20260507 01:18:59.817 | INFO | "malloc" : " 14 GB", | 20260507 01:18:59.817 | INFO | "pinned" : " 29 GB" | 20260507 01:18:59.817 | INFO | } | 20260507 01:18:59.817 | INFO | Perf report for the last 5.14157 seconds: | 20260507 01:18:59.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:18:59.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:59.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:59.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:59.818 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 01:18:59.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:59.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:59.818 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:18:59.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:59.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:18:59.818 | INFO | Uploaded 36GiB to GPU in 2.00712s (17.9361GiB/s) | 20260507 01:18:59.818 | INFO | Downloaded 34GiB to GPU in 1.9451s (17.4798GiB/s) | 20260507 01:19:04.955 | INFO | Analyzing chunk frames = [5962240,5962752) | 20260507 01:19:04.955 | INFO | MemInfo: { | 20260507 01:19:04.955 | INFO | "gpu" : " 17 GB", | 20260507 01:19:04.955 | INFO | "huge" : "", | 20260507 01:19:04.955 | INFO | "huge_shared" : " 27 GB", | 20260507 01:19:04.955 | INFO | "malloc" : " 14 GB", | 20260507 01:19:04.955 | INFO | "pinned" : " 29 GB" | 20260507 01:19:04.955 | INFO | } | 20260507 01:19:04.955 | INFO | Perf report for the last 5.1373 seconds: | 20260507 01:19:04.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:19:04.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:04.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:04.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:04.955 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 01:19:04.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:04.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:04.955 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:19:04.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:04.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:04.955 | INFO | Uploaded 36GiB to GPU in 2.0574s (17.4978GiB/s) | 20260507 01:19:04.955 | INFO | Downloaded 34GiB to GPU in 1.95677s (17.3756GiB/s) | 20260507 01:19:10.095 | INFO | Analyzing chunk frames = [5962752,5963264) | 20260507 01:19:10.095 | INFO | MemInfo: { | 20260507 01:19:10.095 | INFO | "gpu" : " 17 GB", | 20260507 01:19:10.095 | INFO | "huge" : "", | 20260507 01:19:10.095 | INFO | "huge_shared" : " 27 GB", | 20260507 01:19:10.095 | INFO | "malloc" : " 14 GB", | 20260507 01:19:10.095 | INFO | "pinned" : " 29 GB" | 20260507 01:19:10.095 | INFO | } | 20260507 01:19:10.095 | INFO | Perf report for the last 5.14045 seconds: | 20260507 01:19:10.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:19:10.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:10.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:10.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:10.095 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 01:19:10.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:10.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:10.095 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:19:10.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:10.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:10.096 | INFO | Uploaded 36GiB to GPU in 1.9813s (18.1699GiB/s) | 20260507 01:19:10.096 | INFO | Downloaded 34GiB to GPU in 1.95993s (17.3475GiB/s) | 20260507 01:19:15.235 | INFO | Analyzing chunk frames = [5963264,5963776) | 20260507 01:19:15.235 | INFO | MemInfo: { | 20260507 01:19:15.235 | INFO | "gpu" : " 17 GB", | 20260507 01:19:15.235 | INFO | "huge" : "", | 20260507 01:19:15.235 | INFO | "huge_shared" : " 27 GB", | 20260507 01:19:15.235 | INFO | "malloc" : " 14 GB", | 20260507 01:19:15.235 | INFO | "pinned" : " 29 GB" | 20260507 01:19:15.235 | INFO | } | 20260507 01:19:15.235 | INFO | Perf report for the last 5.13946 seconds: | 20260507 01:19:15.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:19:15.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:15.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:15.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:15.235 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:19:15.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:15.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:15.235 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:19:15.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:15.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:15.235 | INFO | Uploaded 36GiB to GPU in 1.97007s (18.2734GiB/s) | 20260507 01:19:15.235 | INFO | Downloaded 34GiB to GPU in 1.96557s (17.2978GiB/s) | 20260507 01:19:15.235 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:19:15.235 | INFO | Total mem usage: 18 GiB | 20260507 01:19:15.235 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:19:15.235 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 01:19:15.235 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:19:15.235 | INFO | High Water Breakdown: | 20260507 01:19:15.235 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:19:15.235 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:19:15.235 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:19:15.235 | INFO | ------------------------------------------ | 20260507 01:19:15.235 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:19:15.235 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:19:15.235 | INFO | Total mem usage: 26.5 GiB | 20260507 01:19:15.235 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:19:15.235 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:19:15.235 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:19:15.235 | INFO | High Water Breakdown: | 20260507 01:19:15.235 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:19:15.235 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:19:15.235 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:19:15.235 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:19:15.235 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:19:15.235 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:19:15.235 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:19:15.235 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:19:15.235 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:19:15.235 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:19:15.235 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:19:15.235 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:19:15.235 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:19:15.235 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:19:15.235 | INFO | ------------------------------------------ | 20260507 01:19:15.235 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:19:15.235 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:19:15.235 | INFO | Total mem usage: 25.6 GiB | 20260507 01:19:15.235 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:19:15.235 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:19:15.235 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:19:15.235 | INFO | High Water Breakdown: | 20260507 01:19:15.235 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:19:15.235 | INFO | ------------------------------------------ | 20260507 01:19:15.235 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:19:15.235 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:19:15.235 | INFO | Total mem usage: 15.7 GiB | 20260507 01:19:15.235 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:19:15.235 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:19:15.236 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:19:15.236 | INFO | High Water Breakdown: | 20260507 01:19:15.236 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:19:15.236 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:19:15.236 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:19:15.236 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:19:15.236 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:19:15.236 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:19:15.236 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:19:15.236 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:19:15.236 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:19:15.236 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:19:15.236 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:19:15.236 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:19:15.236 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:19:15.236 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:19:15.236 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:19:15.236 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:19:15.236 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:19:15.236 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:19:15.236 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:19:15.236 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:19:15.236 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:19:15.236 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:19:15.236 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:19:15.236 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:19:15.236 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:19:15.236 | INFO | ------------------------------------------ | 20260507 01:19:15.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:19:15.236 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:19:15.236 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:19:15.236 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:19:15.236 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.49e+01, 1.34e+01 / 1.53e+00 | 20260507 01:19:15.236 | INFO | PerfReport: 0.29%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.34e-03 / 1.09e-02 | 20260507 01:19:15.236 | INFO | PerfReport: 11.66%, Baseline -- 3.60e+00 / 4.30e+00, 5.34e-01 / 3.87e-01 | 20260507 01:19:15.236 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.44e+00, 1.77e-01 / 1.36e-01 | 20260507 01:19:15.236 | INFO | PerfReport: 16.01%, DME -- 3.96e+01 / 4.36e+01, 3.63e+00 / 1.54e+00 | 20260507 01:19:15.236 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.43e+00 / 1.91e+00 | 20260507 01:19:15.236 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.16e+00 / 2.71e+00, 3.76e-01 / 2.29e-01 | 20260507 01:19:15.236 | INFO | PerfReport: 3.10%, Metrics -- 9.59e-01 / 1.31e+00, 2.60e-01 / 1.94e-01 | 20260507 01:19:15.236 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.27e-01 / 3.78e-01, 5.20e-02 / 2.40e-02 | 20260507 01:19:15.236 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:19:15.236 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:19:15.236 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:19:15.236 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 01:19:15.236 | INFO | PerfReport: 21.65%, Upload -- 2.09e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260507 01:19:15.236 | INFO | PerfReport: 21.27%, Download -- 2.05e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260507 01:19:15.236 | INFO | PerfReport: 32.03%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 01:19:15.236 | INFO | PerfReport: 1.42%, LockUpload -- 1.37e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 01:19:15.236 | INFO | PerfReport: 1.19%, LockDownload -- 1.15e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260507 01:19:15.236 | INFO | PerfReport: 22.43%, LockCompute -- 2.16e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 01:19:15.236 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260507 01:19:15.331 | INFO | Starting a round of Baz IO | 20260507 01:19:18.453 | INFO | Populating RTMetrics report | 20260507 01:19:18.453 | INFO | RTMetrics report populated | 20260507 01:19:18.453 | INFO | Opening tmp RTMetrics json file | 20260507 01:19:18.472 | INFO | Serializing RTMetrics to json | 20260507 01:19:18.473 | INFO | Serialization done, swapping tmp file now... | 20260507 01:19:18.473 | INFO | RTMetrics done | 20260507 01:19:18.544 | INFO | Finished a round of Baz IO: Wrote 1.817 GiB in 3.212 seconds | 20260507 01:19:18.544 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.15% / 0.00% / 66.85% | 20260507 01:19:18.544 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.290116% / 0% / 9.13938e-05% | 20260507 01:19:18.544 | INFO | Baz IO overall: | 20260507 01:19:18.544 | INFO | BytesWritten : 3095863975738 | 20260507 01:19:18.544 | INFO | NumEvents : 1490298221395 | 20260507 01:19:20.370 | INFO | Analyzing chunk frames = [5963776,5964288) | 20260507 01:19:20.370 | INFO | MemInfo: { | 20260507 01:19:20.370 | INFO | "gpu" : " 17 GB", | 20260507 01:19:20.370 | INFO | "huge" : "", | 20260507 01:19:20.370 | INFO | "huge_shared" : " 27 GB", | 20260507 01:19:20.370 | INFO | "malloc" : " 14 GB", | 20260507 01:19:20.370 | INFO | "pinned" : " 29 GB" | 20260507 01:19:20.370 | INFO | } | 20260507 01:19:20.370 | INFO | Perf report for the last 5.13569 seconds: | 20260507 01:19:20.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:19:20.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:20.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:20.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:20.371 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 01:19:20.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:20.371 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 01:19:20.371 | INFO | PRE_HQ : 59.5%, 1.8, 55.7ms, 0 | 20260507 01:19:20.371 | INFO | BAZWRITER : 62.1%, 3.3, 106.4ms, 0 | 20260507 01:19:20.371 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 01:19:20.371 | INFO | Uploaded 36GiB to GPU in 2.018s (17.8395GiB/s) | 20260507 01:19:20.371 | INFO | Downloaded 38GiB to GPU in 2.17485s (17.4725GiB/s) | 20260507 01:19:25.522 | INFO | Analyzing chunk frames = [5964288,5964800) | 20260507 01:19:25.522 | INFO | MemInfo: { | 20260507 01:19:25.522 | INFO | "gpu" : " 17 GB", | 20260507 01:19:25.522 | INFO | "huge" : "", | 20260507 01:19:25.522 | INFO | "huge_shared" : " 27 GB", | 20260507 01:19:25.522 | INFO | "malloc" : " 14 GB", | 20260507 01:19:25.522 | INFO | "pinned" : " 29 GB" | 20260507 01:19:25.522 | INFO | } | 20260507 01:19:25.522 | INFO | Perf report for the last 5.15144 seconds: | 20260507 01:19:25.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:19:25.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:25.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:25.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:25.522 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 01:19:25.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:25.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:25.522 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 01:19:25.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:25.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:25.522 | INFO | Uploaded 36GiB to GPU in 1.99454s (18.0493GiB/s) | 20260507 01:19:25.522 | INFO | Downloaded 34GiB to GPU in 1.93798s (17.544GiB/s) | 20260507 01:19:30.654 | INFO | Analyzing chunk frames = [5964800,5965312) | 20260507 01:19:30.654 | INFO | MemInfo: { | 20260507 01:19:30.654 | INFO | "gpu" : " 17 GB", | 20260507 01:19:30.654 | INFO | "huge" : "", | 20260507 01:19:30.654 | INFO | "huge_shared" : " 27 GB", | 20260507 01:19:30.654 | INFO | "malloc" : " 14 GB", | 20260507 01:19:30.654 | INFO | "pinned" : " 29 GB" | 20260507 01:19:30.655 | INFO | } | 20260507 01:19:30.655 | INFO | Perf report for the last 5.13275 seconds: | 20260507 01:19:30.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:19:30.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:30.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:30.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:30.655 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260507 01:19:30.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:30.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:30.655 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:19:30.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:30.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:30.655 | INFO | Uploaded 36GiB to GPU in 2.03195s (17.7169GiB/s) | 20260507 01:19:30.655 | INFO | Downloaded 34GiB to GPU in 1.95761s (17.3681GiB/s) | 20260507 01:19:35.789 | INFO | Analyzing chunk frames = [5965312,5965824) | 20260507 01:19:35.789 | INFO | MemInfo: { | 20260507 01:19:35.789 | INFO | "gpu" : " 17 GB", | 20260507 01:19:35.789 | INFO | "huge" : "", | 20260507 01:19:35.789 | INFO | "huge_shared" : " 27 GB", | 20260507 01:19:35.789 | INFO | "malloc" : " 14 GB", | 20260507 01:19:35.789 | INFO | "pinned" : " 29 GB" | 20260507 01:19:35.789 | INFO | } | 20260507 01:19:35.789 | INFO | Perf report for the last 5.13426 seconds: | 20260507 01:19:35.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:19:35.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:35.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:35.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:35.789 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 01:19:35.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:35.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:35.789 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:19:35.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:35.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:35.789 | INFO | Uploaded 36GiB to GPU in 1.97377s (18.2392GiB/s) | 20260507 01:19:35.789 | INFO | Downloaded 34GiB to GPU in 1.91929s (17.7149GiB/s) | 20260507 01:19:40.934 | INFO | Analyzing chunk frames = [5965824,5966336) | 20260507 01:19:40.934 | INFO | MemInfo: { | 20260507 01:19:40.934 | INFO | "gpu" : " 17 GB", | 20260507 01:19:40.935 | INFO | "huge" : "", | 20260507 01:19:40.935 | INFO | "huge_shared" : " 27 GB", | 20260507 01:19:40.935 | INFO | "malloc" : " 14 GB", | 20260507 01:19:40.935 | INFO | "pinned" : " 29 GB" | 20260507 01:19:40.935 | INFO | } | 20260507 01:19:40.935 | INFO | Perf report for the last 5.14578 seconds: | 20260507 01:19:40.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:19:40.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:40.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:40.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:40.935 | INFO | ANALYSIS : 59.9%, 3.0, 93.4ms, 0 | 20260507 01:19:40.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:40.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:40.935 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:19:40.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:40.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:40.935 | INFO | Uploaded 36GiB to GPU in 1.9858s (18.1287GiB/s) | 20260507 01:19:40.935 | INFO | Downloaded 34GiB to GPU in 1.9462s (17.47GiB/s) | 20260507 01:19:46.071 | INFO | Analyzing chunk frames = [5966336,5966848) | 20260507 01:19:46.071 | INFO | MemInfo: { | 20260507 01:19:46.071 | INFO | "gpu" : " 17 GB", | 20260507 01:19:46.071 | INFO | "huge" : "", | 20260507 01:19:46.071 | INFO | "huge_shared" : " 27 GB", | 20260507 01:19:46.071 | INFO | "malloc" : " 14 GB", | 20260507 01:19:46.071 | INFO | "pinned" : " 29 GB" | 20260507 01:19:46.071 | INFO | } | 20260507 01:19:46.071 | INFO | Perf report for the last 5.13646 seconds: | 20260507 01:19:46.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:19:46.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:46.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:46.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:46.071 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 01:19:46.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:46.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:46.071 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 01:19:46.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:46.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:46.071 | INFO | Uploaded 36GiB to GPU in 2.04335s (17.6181GiB/s) | 20260507 01:19:46.071 | INFO | Downloaded 34GiB to GPU in 1.96608s (17.2933GiB/s) | 20260507 01:19:51.205 | INFO | Analyzing chunk frames = [5966848,5967360) | 20260507 01:19:51.205 | INFO | MemInfo: { | 20260507 01:19:51.205 | INFO | "gpu" : " 17 GB", | 20260507 01:19:51.205 | INFO | "huge" : "", | 20260507 01:19:51.205 | INFO | "huge_shared" : " 27 GB", | 20260507 01:19:51.205 | INFO | "malloc" : " 14 GB", | 20260507 01:19:51.205 | INFO | "pinned" : " 29 GB" | 20260507 01:19:51.205 | INFO | } | 20260507 01:19:51.205 | INFO | Perf report for the last 5.13399 seconds: | 20260507 01:19:51.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:19:51.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:51.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:51.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:51.205 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 01:19:51.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:51.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:51.206 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 01:19:51.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:51.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:51.206 | INFO | Uploaded 36GiB to GPU in 1.98283s (18.1559GiB/s) | 20260507 01:19:51.206 | INFO | Downloaded 34GiB to GPU in 2.0074s (16.9373GiB/s) | 20260507 01:19:56.346 | INFO | Analyzing chunk frames = [5967360,5967872) | 20260507 01:19:56.346 | INFO | MemInfo: { | 20260507 01:19:56.346 | INFO | "gpu" : " 17 GB", | 20260507 01:19:56.346 | INFO | "huge" : "", | 20260507 01:19:56.346 | INFO | "huge_shared" : " 27 GB", | 20260507 01:19:56.346 | INFO | "malloc" : " 14 GB", | 20260507 01:19:56.346 | INFO | "pinned" : " 29 GB" | 20260507 01:19:56.346 | INFO | } | 20260507 01:19:56.346 | INFO | Perf report for the last 5.14066 seconds: | 20260507 01:19:56.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:19:56.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:56.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:56.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:56.346 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:19:56.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:56.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:56.346 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:19:56.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:56.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:19:56.346 | INFO | Uploaded 36GiB to GPU in 1.986s (18.1269GiB/s) | 20260507 01:19:56.346 | INFO | Downloaded 34GiB to GPU in 1.98672s (17.1136GiB/s) | 20260507 01:19:56.443 | INFO | Starting a round of Baz IO | 20260507 01:19:59.570 | INFO | Populating RTMetrics report | 20260507 01:19:59.570 | INFO | RTMetrics report populated | 20260507 01:19:59.570 | INFO | Opening tmp RTMetrics json file | 20260507 01:19:59.584 | INFO | Serializing RTMetrics to json | 20260507 01:19:59.586 | INFO | Serialization done, swapping tmp file now... | 20260507 01:19:59.586 | INFO | RTMetrics done | 20260507 01:19:59.664 | INFO | Finished a round of Baz IO: Wrote 1.817 GiB in 3.22 seconds | 20260507 01:19:59.664 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.15% / 0.00% / 66.85% | 20260507 01:19:59.664 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.289826% / 3.97364e-06% / 9.13938e-05% | 20260507 01:19:59.664 | INFO | Baz IO overall: | 20260507 01:19:59.664 | INFO | BytesWritten : 3097814623747 | 20260507 01:19:59.664 | INFO | NumEvents : 1491222864874 | 20260507 01:20:01.489 | INFO | Analyzing chunk frames = [5967872,5968384) | 20260507 01:20:01.489 | INFO | MemInfo: { | 20260507 01:20:01.489 | INFO | "gpu" : " 17 GB", | 20260507 01:20:01.489 | INFO | "huge" : "", | 20260507 01:20:01.489 | INFO | "huge_shared" : " 27 GB", | 20260507 01:20:01.489 | INFO | "malloc" : " 14 GB", | 20260507 01:20:01.489 | INFO | "pinned" : " 29 GB" | 20260507 01:20:01.489 | INFO | } | 20260507 01:20:01.489 | INFO | Perf report for the last 5.14291 seconds: | 20260507 01:20:01.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:20:01.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:01.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:01.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:01.489 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 01:20:01.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:01.489 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 01:20:01.489 | INFO | PRE_HQ : 59.1%, 1.8, 56.3ms, 0 | 20260507 01:20:01.489 | INFO | BAZWRITER : 62.2%, 3.4, 107.2ms, 0 | 20260507 01:20:01.489 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:20:01.489 | INFO | Uploaded 36GiB to GPU in 2.01315s (17.8825GiB/s) | 20260507 01:20:01.489 | INFO | Downloaded 38GiB to GPU in 2.1774s (17.452GiB/s) | 20260507 01:20:06.627 | INFO | Analyzing chunk frames = [5968384,5968896) | 20260507 01:20:06.627 | INFO | MemInfo: { | 20260507 01:20:06.627 | INFO | "gpu" : " 17 GB", | 20260507 01:20:06.627 | INFO | "huge" : "", | 20260507 01:20:06.627 | INFO | "huge_shared" : " 27 GB", | 20260507 01:20:06.627 | INFO | "malloc" : " 14 GB", | 20260507 01:20:06.627 | INFO | "pinned" : " 29 GB" | 20260507 01:20:06.627 | INFO | } | 20260507 01:20:06.627 | INFO | Perf report for the last 5.13849 seconds: | 20260507 01:20:06.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:20:06.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:06.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:06.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:06.627 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:20:06.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:06.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:06.627 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:20:06.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:06.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:06.628 | INFO | Uploaded 36GiB to GPU in 2.00174s (17.9844GiB/s) | 20260507 01:20:06.628 | INFO | Downloaded 34GiB to GPU in 1.93108s (17.6067GiB/s) | 20260507 01:20:11.771 | INFO | Analyzing chunk frames = [5968896,5969408) | 20260507 01:20:11.771 | INFO | MemInfo: { | 20260507 01:20:11.771 | INFO | "gpu" : " 17 GB", | 20260507 01:20:11.771 | INFO | "huge" : "", | 20260507 01:20:11.771 | INFO | "huge_shared" : " 27 GB", | 20260507 01:20:11.771 | INFO | "malloc" : " 14 GB", | 20260507 01:20:11.771 | INFO | "pinned" : " 29 GB" | 20260507 01:20:11.771 | INFO | } | 20260507 01:20:11.771 | INFO | Perf report for the last 5.14331 seconds: | 20260507 01:20:11.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:20:11.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:11.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:11.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:11.771 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:20:11.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:11.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:11.771 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:20:11.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:11.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:11.771 | INFO | Uploaded 36GiB to GPU in 2.01131s (17.8988GiB/s) | 20260507 01:20:11.771 | INFO | Downloaded 34GiB to GPU in 1.9349s (17.5719GiB/s) | 20260507 01:20:16.905 | INFO | Analyzing chunk frames = [5969408,5969920) | 20260507 01:20:16.905 | INFO | MemInfo: { | 20260507 01:20:16.905 | INFO | "gpu" : " 17 GB", | 20260507 01:20:16.905 | INFO | "huge" : "", | 20260507 01:20:16.905 | INFO | "huge_shared" : " 27 GB", | 20260507 01:20:16.905 | INFO | "malloc" : " 14 GB", | 20260507 01:20:16.905 | INFO | "pinned" : " 29 GB" | 20260507 01:20:16.905 | INFO | } | 20260507 01:20:16.905 | INFO | Perf report for the last 5.13439 seconds: | 20260507 01:20:16.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:20:16.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:16.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:16.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:16.905 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 01:20:16.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:16.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:16.905 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:20:16.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:16.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:16.905 | INFO | Uploaded 36GiB to GPU in 1.9864s (18.1232GiB/s) | 20260507 01:20:16.905 | INFO | Downloaded 34GiB to GPU in 1.91899s (17.7177GiB/s) | 20260507 01:20:22.042 | INFO | Analyzing chunk frames = [5969920,5970432) | 20260507 01:20:22.042 | INFO | MemInfo: { | 20260507 01:20:22.042 | INFO | "gpu" : " 17 GB", | 20260507 01:20:22.042 | INFO | "huge" : "", | 20260507 01:20:22.042 | INFO | "huge_shared" : " 27 GB", | 20260507 01:20:22.042 | INFO | "malloc" : " 14 GB", | 20260507 01:20:22.042 | INFO | "pinned" : " 29 GB" | 20260507 01:20:22.042 | INFO | } | 20260507 01:20:22.042 | INFO | Perf report for the last 5.13705 seconds: | 20260507 01:20:22.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:20:22.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:22.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:22.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:22.042 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 01:20:22.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:22.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:22.042 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:20:22.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:22.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:22.042 | INFO | Uploaded 36GiB to GPU in 2.00837s (17.925GiB/s) | 20260507 01:20:22.042 | INFO | Downloaded 34GiB to GPU in 1.94044s (17.5218GiB/s) | 20260507 01:20:27.184 | INFO | Analyzing chunk frames = [5970432,5970944) | 20260507 01:20:27.184 | INFO | MemInfo: { | 20260507 01:20:27.184 | INFO | "gpu" : " 17 GB", | 20260507 01:20:27.184 | INFO | "huge" : "", | 20260507 01:20:27.184 | INFO | "huge_shared" : " 27 GB", | 20260507 01:20:27.184 | INFO | "malloc" : " 14 GB", | 20260507 01:20:27.184 | INFO | "pinned" : " 29 GB" | 20260507 01:20:27.185 | INFO | } | 20260507 01:20:27.185 | INFO | Perf report for the last 5.14252 seconds: | 20260507 01:20:27.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:20:27.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:27.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:27.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:27.185 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 01:20:27.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:27.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:27.185 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 01:20:27.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:27.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:27.185 | INFO | Uploaded 36GiB to GPU in 2.04337s (17.618GiB/s) | 20260507 01:20:27.185 | INFO | Downloaded 34GiB to GPU in 1.94783s (17.4554GiB/s) | 20260507 01:20:32.324 | INFO | Analyzing chunk frames = [5970944,5971456) | 20260507 01:20:32.324 | INFO | MemInfo: { | 20260507 01:20:32.324 | INFO | "gpu" : " 17 GB", | 20260507 01:20:32.324 | INFO | "huge" : "", | 20260507 01:20:32.324 | INFO | "huge_shared" : " 27 GB", | 20260507 01:20:32.324 | INFO | "malloc" : " 14 GB", | 20260507 01:20:32.324 | INFO | "pinned" : " 29 GB" | 20260507 01:20:32.324 | INFO | } | 20260507 01:20:32.324 | INFO | Perf report for the last 5.13959 seconds: | 20260507 01:20:32.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:20:32.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:32.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:32.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:32.324 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:20:32.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:32.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:32.324 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:20:32.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:32.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:32.325 | INFO | Uploaded 36GiB to GPU in 1.96776s (18.2949GiB/s) | 20260507 01:20:32.325 | INFO | Downloaded 34GiB to GPU in 2.00025s (16.9978GiB/s) | 20260507 01:20:37.465 | INFO | Analyzing chunk frames = [5971456,5971968) | 20260507 01:20:37.465 | INFO | MemInfo: { | 20260507 01:20:37.465 | INFO | "gpu" : " 17 GB", | 20260507 01:20:37.465 | INFO | "huge" : "", | 20260507 01:20:37.465 | INFO | "huge_shared" : " 27 GB", | 20260507 01:20:37.465 | INFO | "malloc" : " 14 GB", | 20260507 01:20:37.465 | INFO | "pinned" : " 29 GB" | 20260507 01:20:37.465 | INFO | } | 20260507 01:20:37.465 | INFO | Perf report for the last 5.14079 seconds: | 20260507 01:20:37.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:20:37.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:37.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:37.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:37.465 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:20:37.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:37.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:37.465 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 01:20:37.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:37.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:37.465 | INFO | Uploaded 36GiB to GPU in 1.99156s (18.0763GiB/s) | 20260507 01:20:37.466 | INFO | Downloaded 34GiB to GPU in 1.96977s (17.2609GiB/s) | 20260507 01:20:37.466 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:20:37.466 | INFO | Total mem usage: 18 GiB | 20260507 01:20:37.466 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:20:37.466 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:20:37.466 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:20:37.466 | INFO | High Water Breakdown: | 20260507 01:20:37.466 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:20:37.466 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:20:37.466 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:20:37.466 | INFO | ------------------------------------------ | 20260507 01:20:37.466 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:20:37.466 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:20:37.466 | INFO | Total mem usage: 26.5 GiB | 20260507 01:20:37.466 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:20:37.466 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:20:37.466 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:20:37.466 | INFO | High Water Breakdown: | 20260507 01:20:37.466 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:20:37.466 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:20:37.466 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:20:37.466 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:20:37.466 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:20:37.466 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:20:37.466 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:20:37.466 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:20:37.466 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:20:37.466 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:20:37.466 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:20:37.466 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:20:37.466 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:20:37.466 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:20:37.466 | INFO | ------------------------------------------ | 20260507 01:20:37.466 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:20:37.466 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:20:37.466 | INFO | Total mem usage: 25.6 GiB | 20260507 01:20:37.466 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:20:37.466 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:20:37.466 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:20:37.466 | INFO | High Water Breakdown: | 20260507 01:20:37.466 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:20:37.466 | INFO | ------------------------------------------ | 20260507 01:20:37.466 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:20:37.466 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:20:37.466 | INFO | Total mem usage: 15.7 GiB | 20260507 01:20:37.466 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:20:37.466 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:20:37.466 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:20:37.466 | INFO | High Water Breakdown: | 20260507 01:20:37.466 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:20:37.466 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:20:37.466 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:20:37.466 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:20:37.466 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:20:37.466 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:20:37.466 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:20:37.466 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:20:37.466 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:20:37.466 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:20:37.466 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:20:37.466 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:20:37.466 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:20:37.466 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:20:37.466 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:20:37.466 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:20:37.466 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:20:37.466 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:20:37.466 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:20:37.466 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:20:37.466 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:20:37.466 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:20:37.466 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:20:37.466 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:20:37.466 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:20:37.466 | INFO | ------------------------------------------ | 20260507 01:20:37.466 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:20:37.466 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:20:37.466 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:20:37.466 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:20:37.466 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.59e+01, 1.34e+01 / 2.23e+00 | 20260507 01:20:37.466 | INFO | PerfReport: 0.30%, AggregateStats -- 9.12e-02 / 1.01e-01, 1.00e-02 / 4.54e-03 | 20260507 01:20:37.467 | INFO | PerfReport: 11.66%, Baseline -- 3.60e+00 / 4.38e+00, 5.29e-01 / 3.42e-01 | 20260507 01:20:37.467 | INFO | PerfReport: 3.24%, Histogram -- 9.99e-01 / 1.47e+00, 1.73e-01 / 1.28e-01 | 20260507 01:20:37.467 | INFO | PerfReport: 16.19%, DME -- 4.00e+01 / 4.53e+01, 4.32e+00 / 2.62e+00 | 20260507 01:20:37.467 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.25e+00 / 1.54e+00 | 20260507 01:20:37.467 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.17e+00 / 2.74e+00, 3.87e-01 / 2.17e-01 | 20260507 01:20:37.467 | INFO | PerfReport: 3.11%, Metrics -- 9.61e-01 / 1.30e+00, 2.55e-01 / 1.53e-01 | 20260507 01:20:37.467 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.26e-01 / 3.78e-01, 5.16e-02 / 1.73e-02 | 20260507 01:20:37.467 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:20:37.467 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:20:37.467 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:20:37.467 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 01:20:37.467 | INFO | PerfReport: 21.60%, Upload -- 2.09e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260507 01:20:37.467 | INFO | PerfReport: 21.39%, Download -- 2.07e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260507 01:20:37.467 | INFO | PerfReport: 31.98%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 01:20:37.467 | INFO | PerfReport: 1.36%, LockUpload -- 1.32e+00 / 0.00e+00, 4.89e+00 / 0.00e+00 | 20260507 01:20:37.467 | INFO | PerfReport: 1.37%, LockDownload -- 1.32e+00 / 0.00e+00, 3.22e+00 / 0.00e+00 | 20260507 01:20:37.467 | INFO | PerfReport: 22.28%, LockCompute -- 2.15e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 01:20:37.467 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260507 01:20:37.562 | INFO | Starting a round of Baz IO | 20260507 01:20:40.677 | INFO | Populating RTMetrics report | 20260507 01:20:40.678 | INFO | RTMetrics report populated | 20260507 01:20:40.678 | INFO | Opening tmp RTMetrics json file | 20260507 01:20:40.678 | INFO | Serializing RTMetrics to json | 20260507 01:20:40.679 | INFO | Serialization done, swapping tmp file now... | 20260507 01:20:40.681 | INFO | RTMetrics done | 20260507 01:20:40.766 | INFO | Finished a round of Baz IO: Wrote 1.816 GiB in 3.204 seconds | 20260507 01:20:40.766 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.15% / 0.00% / 66.85% | 20260507 01:20:40.766 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.28799% / 3.97364e-06% / 9.13938e-05% | 20260507 01:20:40.766 | INFO | Baz IO overall: | 20260507 01:20:40.766 | INFO | BytesWritten : 3099764118002 | 20260507 01:20:40.766 | INFO | NumEvents : 1492146825332 | 20260507 01:20:42.606 | INFO | Analyzing chunk frames = [5971968,5972480) | 20260507 01:20:42.606 | INFO | MemInfo: { | 20260507 01:20:42.606 | INFO | "gpu" : " 17 GB", | 20260507 01:20:42.606 | INFO | "huge" : "", | 20260507 01:20:42.606 | INFO | "huge_shared" : " 27 GB", | 20260507 01:20:42.606 | INFO | "malloc" : " 14 GB", | 20260507 01:20:42.606 | INFO | "pinned" : " 29 GB" | 20260507 01:20:42.606 | INFO | } | 20260507 01:20:42.607 | INFO | Perf report for the last 5.14149 seconds: | 20260507 01:20:42.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:20:42.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:42.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:42.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:42.607 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260507 01:20:42.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:42.607 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:20:42.607 | INFO | PRE_HQ : 59.4%, 1.8, 56.0ms, 0 | 20260507 01:20:42.607 | INFO | BAZWRITER : 61.9%, 3.3, 106.1ms, 0 | 20260507 01:20:42.607 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:20:42.607 | INFO | Uploaded 36GiB to GPU in 2.02871s (17.7453GiB/s) | 20260507 01:20:42.607 | INFO | Downloaded 38GiB to GPU in 2.18581s (17.3849GiB/s) | 20260507 01:20:47.741 | INFO | Analyzing chunk frames = [5972480,5972992) | 20260507 01:20:47.741 | INFO | MemInfo: { | 20260507 01:20:47.741 | INFO | "gpu" : " 17 GB", | 20260507 01:20:47.741 | INFO | "huge" : "", | 20260507 01:20:47.741 | INFO | "huge_shared" : " 27 GB", | 20260507 01:20:47.741 | INFO | "malloc" : " 14 GB", | 20260507 01:20:47.741 | INFO | "pinned" : " 29 GB" | 20260507 01:20:47.741 | INFO | } | 20260507 01:20:47.741 | INFO | Perf report for the last 5.1348 seconds: | 20260507 01:20:47.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:20:47.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:47.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:47.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:47.742 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:20:47.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:47.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:47.742 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:20:47.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:47.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:47.742 | INFO | Uploaded 36GiB to GPU in 2.01378s (17.8768GiB/s) | 20260507 01:20:47.742 | INFO | Downloaded 34GiB to GPU in 1.93413s (17.579GiB/s) | 20260507 01:20:52.880 | INFO | Analyzing chunk frames = [5972992,5973504) | 20260507 01:20:52.880 | INFO | MemInfo: { | 20260507 01:20:52.880 | INFO | "gpu" : " 17 GB", | 20260507 01:20:52.880 | INFO | "huge" : "", | 20260507 01:20:52.880 | INFO | "huge_shared" : " 27 GB", | 20260507 01:20:52.880 | INFO | "malloc" : " 14 GB", | 20260507 01:20:52.880 | INFO | "pinned" : " 29 GB" | 20260507 01:20:52.880 | INFO | } | 20260507 01:20:52.880 | INFO | Perf report for the last 5.13879 seconds: | 20260507 01:20:52.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:20:52.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:52.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:52.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:52.880 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 01:20:52.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:52.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:52.880 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:20:52.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:52.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:52.881 | INFO | Uploaded 36GiB to GPU in 1.98095s (18.1731GiB/s) | 20260507 01:20:52.881 | INFO | Downloaded 34GiB to GPU in 1.93726s (17.5506GiB/s) | 20260507 01:20:58.021 | INFO | Analyzing chunk frames = [5973504,5974016) | 20260507 01:20:58.021 | INFO | MemInfo: { | 20260507 01:20:58.021 | INFO | "gpu" : " 17 GB", | 20260507 01:20:58.021 | INFO | "huge" : "", | 20260507 01:20:58.021 | INFO | "huge_shared" : " 27 GB", | 20260507 01:20:58.021 | INFO | "malloc" : " 14 GB", | 20260507 01:20:58.021 | INFO | "pinned" : " 29 GB" | 20260507 01:20:58.021 | INFO | } | 20260507 01:20:58.021 | INFO | Perf report for the last 5.14096 seconds: | 20260507 01:20:58.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:20:58.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:58.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:58.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:58.022 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 01:20:58.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:58.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:58.022 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:20:58.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:58.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:20:58.022 | INFO | Uploaded 36GiB to GPU in 1.95218s (18.4409GiB/s) | 20260507 01:20:58.022 | INFO | Downloaded 34GiB to GPU in 1.91101s (17.7917GiB/s) | 20260507 01:21:03.155 | INFO | Analyzing chunk frames = [5974016,5974528) | 20260507 01:21:03.155 | INFO | MemInfo: { | 20260507 01:21:03.155 | INFO | "gpu" : " 17 GB", | 20260507 01:21:03.155 | INFO | "huge" : "", | 20260507 01:21:03.155 | INFO | "huge_shared" : " 27 GB", | 20260507 01:21:03.155 | INFO | "malloc" : " 14 GB", | 20260507 01:21:03.155 | INFO | "pinned" : " 29 GB" | 20260507 01:21:03.155 | INFO | } | 20260507 01:21:03.155 | INFO | Perf report for the last 5.13382 seconds: | 20260507 01:21:03.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:21:03.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:03.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:03.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:03.155 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 01:21:03.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:03.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:03.155 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 01:21:03.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:03.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:03.155 | INFO | Uploaded 36GiB to GPU in 1.99616s (18.0346GiB/s) | 20260507 01:21:03.155 | INFO | Downloaded 34GiB to GPU in 1.95162s (17.4214GiB/s) | 20260507 01:21:08.297 | INFO | Analyzing chunk frames = [5974528,5975040) | 20260507 01:21:08.297 | INFO | MemInfo: { | 20260507 01:21:08.297 | INFO | "gpu" : " 17 GB", | 20260507 01:21:08.297 | INFO | "huge" : "", | 20260507 01:21:08.297 | INFO | "huge_shared" : " 27 GB", | 20260507 01:21:08.297 | INFO | "malloc" : " 14 GB", | 20260507 01:21:08.297 | INFO | "pinned" : " 29 GB" | 20260507 01:21:08.297 | INFO | } | 20260507 01:21:08.297 | INFO | Perf report for the last 5.14171 seconds: | 20260507 01:21:08.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:21:08.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:08.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:08.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:08.297 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 01:21:08.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:08.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:08.297 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 01:21:08.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:08.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:08.297 | INFO | Uploaded 36GiB to GPU in 2.00549s (17.9507GiB/s) | 20260507 01:21:08.297 | INFO | Downloaded 34GiB to GPU in 1.9325s (17.5938GiB/s) | 20260507 01:21:13.444 | INFO | Analyzing chunk frames = [5975040,5975552) | 20260507 01:21:13.444 | INFO | MemInfo: { | 20260507 01:21:13.444 | INFO | "gpu" : " 17 GB", | 20260507 01:21:13.444 | INFO | "huge" : "", | 20260507 01:21:13.444 | INFO | "huge_shared" : " 27 GB", | 20260507 01:21:13.444 | INFO | "malloc" : " 14 GB", | 20260507 01:21:13.444 | INFO | "pinned" : " 29 GB" | 20260507 01:21:13.444 | INFO | } | 20260507 01:21:13.444 | INFO | Perf report for the last 5.14758 seconds: | 20260507 01:21:13.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:21:13.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:13.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:13.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:13.444 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:21:13.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:13.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:13.445 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 01:21:13.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:13.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:13.445 | INFO | Uploaded 36GiB to GPU in 1.97419s (18.2353GiB/s) | 20260507 01:21:13.445 | INFO | Downloaded 34GiB to GPU in 1.99469s (17.0453GiB/s) | 20260507 01:21:18.582 | INFO | Analyzing chunk frames = [5975552,5976064) | 20260507 01:21:18.582 | INFO | MemInfo: { | 20260507 01:21:18.582 | INFO | "gpu" : " 17 GB", | 20260507 01:21:18.582 | INFO | "huge" : "", | 20260507 01:21:18.582 | INFO | "huge_shared" : " 27 GB", | 20260507 01:21:18.582 | INFO | "malloc" : " 14 GB", | 20260507 01:21:18.582 | INFO | "pinned" : " 29 GB" | 20260507 01:21:18.582 | INFO | } | 20260507 01:21:18.582 | INFO | Perf report for the last 5.13815 seconds: | 20260507 01:21:18.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:21:18.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:18.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:18.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:18.582 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:21:18.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:18.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:18.582 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:21:18.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:18.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:18.583 | INFO | Uploaded 36GiB to GPU in 2.00282s (17.9746GiB/s) | 20260507 01:21:18.583 | INFO | Downloaded 34GiB to GPU in 1.97418s (17.2223GiB/s) | 20260507 01:21:18.680 | INFO | Starting a round of Baz IO | 20260507 01:21:21.793 | INFO | Populating RTMetrics report | 20260507 01:21:21.793 | INFO | RTMetrics report populated | 20260507 01:21:21.793 | INFO | Opening tmp RTMetrics json file | 20260507 01:21:21.794 | INFO | Serializing RTMetrics to json | 20260507 01:21:21.801 | INFO | Serialization done, swapping tmp file now... | 20260507 01:21:21.814 | INFO | RTMetrics done | 20260507 01:21:21.884 | INFO | Finished a round of Baz IO: Wrote 1.814 GiB in 3.203 seconds | 20260507 01:21:21.884 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.15% / 0.00% / 66.85% | 20260507 01:21:21.884 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.287644% / 0% / 8.74201e-05% | 20260507 01:21:21.884 | INFO | Baz IO overall: | 20260507 01:21:21.884 | INFO | BytesWritten : 3101711638147 | 20260507 01:21:21.884 | INFO | NumEvents : 1493069545030 | 20260507 01:21:23.716 | INFO | Analyzing chunk frames = [5976064,5976576) | 20260507 01:21:23.716 | INFO | MemInfo: { | 20260507 01:21:23.716 | INFO | "gpu" : " 17 GB", | 20260507 01:21:23.716 | INFO | "huge" : "", | 20260507 01:21:23.716 | INFO | "huge_shared" : " 27 GB", | 20260507 01:21:23.716 | INFO | "malloc" : " 14 GB", | 20260507 01:21:23.716 | INFO | "pinned" : " 29 GB" | 20260507 01:21:23.716 | INFO | } | 20260507 01:21:23.716 | INFO | Perf report for the last 5.13386 seconds: | 20260507 01:21:23.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:21:23.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:23.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:23.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:23.716 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260507 01:21:23.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:23.716 | INFO | RT_METRICS : 42.3%, 1.0, 22.1ms, 0 | 20260507 01:21:23.716 | INFO | PRE_HQ : 59.4%, 1.8, 55.9ms, 0 | 20260507 01:21:23.716 | INFO | BAZWRITER : 62.0%, 3.3, 106.2ms, 0 | 20260507 01:21:23.716 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:21:23.717 | INFO | Uploaded 36GiB to GPU in 2.03343s (17.7041GiB/s) | 20260507 01:21:23.717 | INFO | Downloaded 38GiB to GPU in 2.18443s (17.3958GiB/s) | 20260507 01:21:28.855 | INFO | Analyzing chunk frames = [5976576,5977088) | 20260507 01:21:28.855 | INFO | MemInfo: { | 20260507 01:21:28.855 | INFO | "gpu" : " 17 GB", | 20260507 01:21:28.855 | INFO | "huge" : "", | 20260507 01:21:28.855 | INFO | "huge_shared" : " 27 GB", | 20260507 01:21:28.855 | INFO | "malloc" : " 14 GB", | 20260507 01:21:28.855 | INFO | "pinned" : " 29 GB" | 20260507 01:21:28.855 | INFO | } | 20260507 01:21:28.855 | INFO | Perf report for the last 5.13864 seconds: | 20260507 01:21:28.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:21:28.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:28.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:28.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:28.855 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:21:28.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:28.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:28.855 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:21:28.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:28.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:28.855 | INFO | Uploaded 36GiB to GPU in 2.02404s (17.7862GiB/s) | 20260507 01:21:28.855 | INFO | Downloaded 34GiB to GPU in 1.93208s (17.5976GiB/s) | 20260507 01:21:33.992 | INFO | Analyzing chunk frames = [5977088,5977600) | 20260507 01:21:33.992 | INFO | MemInfo: { | 20260507 01:21:33.992 | INFO | "gpu" : " 17 GB", | 20260507 01:21:33.992 | INFO | "huge" : "", | 20260507 01:21:33.992 | INFO | "huge_shared" : " 27 GB", | 20260507 01:21:33.992 | INFO | "malloc" : " 14 GB", | 20260507 01:21:33.992 | INFO | "pinned" : " 29 GB" | 20260507 01:21:33.992 | INFO | } | 20260507 01:21:33.992 | INFO | Perf report for the last 5.13721 seconds: | 20260507 01:21:33.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:21:33.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:33.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:33.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:33.992 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 01:21:33.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:33.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:33.992 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:21:33.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:33.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:33.992 | INFO | Uploaded 36GiB to GPU in 2.00706s (17.9367GiB/s) | 20260507 01:21:33.992 | INFO | Downloaded 34GiB to GPU in 1.93502s (17.5709GiB/s) | 20260507 01:21:39.137 | INFO | Analyzing chunk frames = [5977600,5978112) | 20260507 01:21:39.137 | INFO | MemInfo: { | 20260507 01:21:39.137 | INFO | "gpu" : " 17 GB", | 20260507 01:21:39.137 | INFO | "huge" : "", | 20260507 01:21:39.137 | INFO | "huge_shared" : " 27 GB", | 20260507 01:21:39.137 | INFO | "malloc" : " 14 GB", | 20260507 01:21:39.137 | INFO | "pinned" : " 29 GB" | 20260507 01:21:39.137 | INFO | } | 20260507 01:21:39.137 | INFO | Perf report for the last 5.14514 seconds: | 20260507 01:21:39.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:21:39.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:39.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:39.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:39.137 | INFO | ANALYSIS : 60.0%, 3.0, 93.2ms, 0 | 20260507 01:21:39.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:39.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:39.137 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:21:39.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:39.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:39.138 | INFO | Uploaded 36GiB to GPU in 2.01917s (17.8291GiB/s) | 20260507 01:21:39.138 | INFO | Downloaded 34GiB to GPU in 1.90846s (17.8154GiB/s) | 20260507 01:21:44.276 | INFO | Analyzing chunk frames = [5978112,5978624) | 20260507 01:21:44.276 | INFO | MemInfo: { | 20260507 01:21:44.276 | INFO | "gpu" : " 17 GB", | 20260507 01:21:44.276 | INFO | "huge" : "", | 20260507 01:21:44.276 | INFO | "huge_shared" : " 27 GB", | 20260507 01:21:44.276 | INFO | "malloc" : " 14 GB", | 20260507 01:21:44.276 | INFO | "pinned" : " 29 GB" | 20260507 01:21:44.276 | INFO | } | 20260507 01:21:44.276 | INFO | Perf report for the last 5.13857 seconds: | 20260507 01:21:44.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:21:44.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:44.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:44.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:44.276 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 01:21:44.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:44.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:44.276 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 01:21:44.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:44.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:44.276 | INFO | Uploaded 36GiB to GPU in 2.00848s (17.924GiB/s) | 20260507 01:21:44.276 | INFO | Downloaded 34GiB to GPU in 1.94451s (17.4852GiB/s) | 20260507 01:21:49.415 | INFO | Analyzing chunk frames = [5978624,5979136) | 20260507 01:21:49.415 | INFO | MemInfo: { | 20260507 01:21:49.415 | INFO | "gpu" : " 17 GB", | 20260507 01:21:49.415 | INFO | "huge" : "", | 20260507 01:21:49.415 | INFO | "huge_shared" : " 27 GB", | 20260507 01:21:49.415 | INFO | "malloc" : " 14 GB", | 20260507 01:21:49.415 | INFO | "pinned" : " 29 GB" | 20260507 01:21:49.415 | INFO | } | 20260507 01:21:49.416 | INFO | Perf report for the last 5.13963 seconds: | 20260507 01:21:49.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:21:49.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:49.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:49.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:49.416 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 01:21:49.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:49.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:49.416 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 01:21:49.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:49.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:49.416 | INFO | Uploaded 36GiB to GPU in 2.05265s (17.5383GiB/s) | 20260507 01:21:49.416 | INFO | Downloaded 34GiB to GPU in 1.94907s (17.4442GiB/s) | 20260507 01:21:54.552 | INFO | Analyzing chunk frames = [5979136,5979648) | 20260507 01:21:54.552 | INFO | MemInfo: { | 20260507 01:21:54.552 | INFO | "gpu" : " 17 GB", | 20260507 01:21:54.552 | INFO | "huge" : "", | 20260507 01:21:54.552 | INFO | "huge_shared" : " 27 GB", | 20260507 01:21:54.552 | INFO | "malloc" : " 14 GB", | 20260507 01:21:54.552 | INFO | "pinned" : " 29 GB" | 20260507 01:21:54.553 | INFO | } | 20260507 01:21:54.553 | INFO | Perf report for the last 5.13701 seconds: | 20260507 01:21:54.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:21:54.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:54.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:54.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:54.553 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:21:54.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:54.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:54.553 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 01:21:54.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:54.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:54.553 | INFO | Uploaded 36GiB to GPU in 1.99636s (18.0328GiB/s) | 20260507 01:21:54.553 | INFO | Downloaded 34GiB to GPU in 1.98543s (17.1248GiB/s) | 20260507 01:21:59.693 | INFO | Analyzing chunk frames = [5979648,5980160) | 20260507 01:21:59.693 | INFO | MemInfo: { | 20260507 01:21:59.693 | INFO | "gpu" : " 17 GB", | 20260507 01:21:59.693 | INFO | "huge" : "", | 20260507 01:21:59.693 | INFO | "huge_shared" : " 27 GB", | 20260507 01:21:59.693 | INFO | "malloc" : " 14 GB", | 20260507 01:21:59.693 | INFO | "pinned" : " 29 GB" | 20260507 01:21:59.693 | INFO | } | 20260507 01:21:59.693 | INFO | Perf report for the last 5.14022 seconds: | 20260507 01:21:59.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:21:59.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:59.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:59.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:59.693 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:21:59.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:59.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:59.693 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 01:21:59.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:59.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:21:59.693 | INFO | Uploaded 36GiB to GPU in 1.99294s (18.0638GiB/s) | 20260507 01:21:59.693 | INFO | Downloaded 34GiB to GPU in 1.97853s (17.1845GiB/s) | 20260507 01:21:59.693 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:21:59.693 | INFO | Total mem usage: 18 GiB | 20260507 01:21:59.693 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:21:59.693 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 01:21:59.693 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:21:59.693 | INFO | High Water Breakdown: | 20260507 01:21:59.693 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:21:59.693 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:21:59.693 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:21:59.693 | INFO | ------------------------------------------ | 20260507 01:21:59.693 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:21:59.693 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:21:59.693 | INFO | Total mem usage: 26.5 GiB | 20260507 01:21:59.693 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:21:59.693 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:21:59.693 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:21:59.693 | INFO | High Water Breakdown: | 20260507 01:21:59.693 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:21:59.693 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:21:59.693 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:21:59.694 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:21:59.694 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:21:59.694 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:21:59.694 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:21:59.694 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:21:59.694 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:21:59.694 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:21:59.694 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:21:59.694 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:21:59.694 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:21:59.694 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:21:59.694 | INFO | ------------------------------------------ | 20260507 01:21:59.694 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:21:59.694 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:21:59.694 | INFO | Total mem usage: 25.6 GiB | 20260507 01:21:59.694 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:21:59.694 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:21:59.694 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:21:59.694 | INFO | High Water Breakdown: | 20260507 01:21:59.694 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:21:59.694 | INFO | ------------------------------------------ | 20260507 01:21:59.694 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:21:59.694 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:21:59.694 | INFO | Total mem usage: 15.7 GiB | 20260507 01:21:59.694 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:21:59.694 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:21:59.694 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:21:59.694 | INFO | High Water Breakdown: | 20260507 01:21:59.694 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:21:59.694 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:21:59.694 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:21:59.694 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:21:59.694 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:21:59.694 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:21:59.694 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:21:59.694 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:21:59.694 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:21:59.694 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:21:59.694 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:21:59.694 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:21:59.694 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:21:59.694 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:21:59.694 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:21:59.694 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:21:59.694 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:21:59.694 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:21:59.694 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:21:59.694 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:21:59.694 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:21:59.694 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:21:59.694 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:21:59.694 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:21:59.694 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:21:59.694 | INFO | ------------------------------------------ | 20260507 01:21:59.694 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:21:59.694 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:21:59.694 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:21:59.694 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:21:59.694 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.61e+01, 1.36e+01 / 2.13e+00 | 20260507 01:21:59.694 | INFO | PerfReport: 0.30%, AggregateStats -- 9.12e-02 / 1.01e-01, 9.68e-03 / 5.32e-03 | 20260507 01:21:59.694 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.36e+00, 5.31e-01 / 3.07e-01 | 20260507 01:21:59.694 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.46e+00, 1.74e-01 / 1.23e-01 | 20260507 01:21:59.694 | INFO | PerfReport: 16.15%, DME -- 3.99e+01 / 4.51e+01, 4.25e+00 / 2.22e+00 | 20260507 01:21:59.694 | INFO | PerfReport: 57.48%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.31e+00 / 1.64e+00 | 20260507 01:21:59.694 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.16e+00 / 2.72e+00, 3.77e-01 / 1.83e-01 | 20260507 01:21:59.694 | INFO | PerfReport: 3.12%, Metrics -- 9.62e-01 / 1.28e+00, 2.57e-01 / 1.29e-01 | 20260507 01:21:59.694 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.26e-01 / 3.75e-01, 5.13e-02 / 1.70e-02 | 20260507 01:21:59.694 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:21:59.694 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:21:59.694 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:21:59.694 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 01:21:59.694 | INFO | PerfReport: 21.65%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260507 01:21:59.694 | INFO | PerfReport: 21.34%, Download -- 2.06e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260507 01:21:59.694 | INFO | PerfReport: 32.01%, Compute -- 3.09e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260507 01:21:59.694 | INFO | PerfReport: 1.37%, LockUpload -- 1.32e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260507 01:21:59.694 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.16e+00 / 0.00e+00 | 20260507 01:21:59.695 | INFO | PerfReport: 22.31%, LockCompute -- 2.15e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 01:21:59.695 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260507 01:21:59.791 | INFO | Starting a round of Baz IO | 20260507 01:22:02.915 | INFO | Populating RTMetrics report | 20260507 01:22:02.915 | INFO | RTMetrics report populated | 20260507 01:22:02.915 | INFO | Opening tmp RTMetrics json file | 20260507 01:22:02.934 | INFO | Serializing RTMetrics to json | 20260507 01:22:02.935 | INFO | Serialization done, swapping tmp file now... | 20260507 01:22:02.936 | INFO | RTMetrics done | 20260507 01:22:03.007 | INFO | Finished a round of Baz IO: Wrote 1.815 GiB in 3.216 seconds | 20260507 01:22:03.007 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.15% / 0.00% / 66.85% | 20260507 01:22:03.007 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.287533% / 3.97364e-06% / 5.96046e-05% | 20260507 01:22:03.007 | INFO | Baz IO overall: | 20260507 01:22:03.007 | INFO | BytesWritten : 3103660146064 | 20260507 01:22:03.007 | INFO | NumEvents : 1493992979232 | 20260507 01:22:04.831 | INFO | Analyzing chunk frames = [5980160,5980672) | 20260507 01:22:04.831 | INFO | MemInfo: { | 20260507 01:22:04.831 | INFO | "gpu" : " 17 GB", | 20260507 01:22:04.832 | INFO | "huge" : "", | 20260507 01:22:04.832 | INFO | "huge_shared" : " 27 GB", | 20260507 01:22:04.832 | INFO | "malloc" : " 14 GB", | 20260507 01:22:04.832 | INFO | "pinned" : " 29 GB" | 20260507 01:22:04.832 | INFO | } | 20260507 01:22:04.832 | INFO | Perf report for the last 5.13872 seconds: | 20260507 01:22:04.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:22:04.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:04.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:04.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:04.832 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260507 01:22:04.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:04.832 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 01:22:04.832 | INFO | PRE_HQ : 59.5%, 1.8, 56.4ms, 0 | 20260507 01:22:04.832 | INFO | BAZWRITER : 62.1%, 3.3, 106.0ms, 0 | 20260507 01:22:04.832 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:22:04.832 | INFO | Uploaded 36GiB to GPU in 2.02345s (17.7914GiB/s) | 20260507 01:22:04.832 | INFO | Downloaded 38GiB to GPU in 2.18142s (17.4198GiB/s) | 20260507 01:22:09.967 | INFO | Analyzing chunk frames = [5980672,5981184) | 20260507 01:22:09.967 | INFO | MemInfo: { | 20260507 01:22:09.967 | INFO | "gpu" : " 17 GB", | 20260507 01:22:09.967 | INFO | "huge" : "", | 20260507 01:22:09.967 | INFO | "huge_shared" : " 27 GB", | 20260507 01:22:09.967 | INFO | "malloc" : " 14 GB", | 20260507 01:22:09.967 | INFO | "pinned" : " 29 GB" | 20260507 01:22:09.967 | INFO | } | 20260507 01:22:09.967 | INFO | Perf report for the last 5.13569 seconds: | 20260507 01:22:09.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:22:09.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:09.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:09.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:09.967 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:22:09.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:09.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:09.967 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 01:22:09.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:09.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:09.968 | INFO | Uploaded 36GiB to GPU in 2.00142s (17.9872GiB/s) | 20260507 01:22:09.968 | INFO | Downloaded 34GiB to GPU in 1.93216s (17.5969GiB/s) | 20260507 01:22:15.109 | INFO | Analyzing chunk frames = [5981184,5981696) | 20260507 01:22:15.109 | INFO | MemInfo: { | 20260507 01:22:15.109 | INFO | "gpu" : " 17 GB", | 20260507 01:22:15.109 | INFO | "huge" : "", | 20260507 01:22:15.109 | INFO | "huge_shared" : " 27 GB", | 20260507 01:22:15.109 | INFO | "malloc" : " 14 GB", | 20260507 01:22:15.109 | INFO | "pinned" : " 29 GB" | 20260507 01:22:15.109 | INFO | } | 20260507 01:22:15.109 | INFO | Perf report for the last 5.14161 seconds: | 20260507 01:22:15.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:22:15.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:15.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:15.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:15.109 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:22:15.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:15.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:15.109 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:22:15.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:15.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:15.109 | INFO | Uploaded 36GiB to GPU in 1.98667s (18.1207GiB/s) | 20260507 01:22:15.109 | INFO | Downloaded 34GiB to GPU in 1.93431s (17.5773GiB/s) | 20260507 01:22:20.254 | INFO | Analyzing chunk frames = [5981696,5982208) | 20260507 01:22:20.254 | INFO | MemInfo: { | 20260507 01:22:20.254 | INFO | "gpu" : " 17 GB", | 20260507 01:22:20.254 | INFO | "huge" : "", | 20260507 01:22:20.254 | INFO | "huge_shared" : " 27 GB", | 20260507 01:22:20.254 | INFO | "malloc" : " 14 GB", | 20260507 01:22:20.254 | INFO | "pinned" : " 29 GB" | 20260507 01:22:20.254 | INFO | } | 20260507 01:22:20.254 | INFO | Perf report for the last 5.14473 seconds: | 20260507 01:22:20.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:22:20.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:20.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:20.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:20.254 | INFO | ANALYSIS : 59.2%, 3.0, 91.9ms, 0 | 20260507 01:22:20.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:20.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:20.254 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:22:20.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:20.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:20.254 | INFO | Uploaded 36GiB to GPU in 1.94735s (18.4867GiB/s) | 20260507 01:22:20.255 | INFO | Downloaded 34GiB to GPU in 1.87752s (18.109GiB/s) | 20260507 01:22:25.392 | INFO | Analyzing chunk frames = [5982208,5982720) | 20260507 01:22:25.392 | INFO | MemInfo: { | 20260507 01:22:25.392 | INFO | "gpu" : " 17 GB", | 20260507 01:22:25.392 | INFO | "huge" : "", | 20260507 01:22:25.392 | INFO | "huge_shared" : " 27 GB", | 20260507 01:22:25.392 | INFO | "malloc" : " 14 GB", | 20260507 01:22:25.392 | INFO | "pinned" : " 29 GB" | 20260507 01:22:25.392 | INFO | } | 20260507 01:22:25.392 | INFO | Perf report for the last 5.1386 seconds: | 20260507 01:22:25.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:22:25.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:25.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:25.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:25.392 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 01:22:25.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:25.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:25.392 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:22:25.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:25.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:25.393 | INFO | Uploaded 36GiB to GPU in 1.98184s (18.1649GiB/s) | 20260507 01:22:25.393 | INFO | Downloaded 34GiB to GPU in 1.93593s (17.5626GiB/s) | 20260507 01:22:30.528 | INFO | Analyzing chunk frames = [5982720,5983232) | 20260507 01:22:30.528 | INFO | MemInfo: { | 20260507 01:22:30.528 | INFO | "gpu" : " 17 GB", | 20260507 01:22:30.528 | INFO | "huge" : "", | 20260507 01:22:30.528 | INFO | "huge_shared" : " 27 GB", | 20260507 01:22:30.528 | INFO | "malloc" : " 14 GB", | 20260507 01:22:30.528 | INFO | "pinned" : " 29 GB" | 20260507 01:22:30.528 | INFO | } | 20260507 01:22:30.529 | INFO | Perf report for the last 5.13629 seconds: | 20260507 01:22:30.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:22:30.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:30.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:30.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:30.529 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 01:22:30.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:30.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:30.529 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 01:22:30.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:30.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:30.529 | INFO | Uploaded 36GiB to GPU in 2.0489s (17.5704GiB/s) | 20260507 01:22:30.529 | INFO | Downloaded 34GiB to GPU in 1.95419s (17.3985GiB/s) | 20260507 01:22:35.666 | INFO | Analyzing chunk frames = [5983232,5983744) | 20260507 01:22:35.666 | INFO | MemInfo: { | 20260507 01:22:35.666 | INFO | "gpu" : " 17 GB", | 20260507 01:22:35.666 | INFO | "huge" : "", | 20260507 01:22:35.666 | INFO | "huge_shared" : " 27 GB", | 20260507 01:22:35.666 | INFO | "malloc" : " 14 GB", | 20260507 01:22:35.666 | INFO | "pinned" : " 29 GB" | 20260507 01:22:35.666 | INFO | } | 20260507 01:22:35.666 | INFO | Perf report for the last 5.13771 seconds: | 20260507 01:22:35.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:22:35.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:35.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:35.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:35.666 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:22:35.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:35.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:35.667 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 01:22:35.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:35.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:35.667 | INFO | Uploaded 36GiB to GPU in 1.98347s (18.15GiB/s) | 20260507 01:22:35.667 | INFO | Downloaded 34GiB to GPU in 1.98519s (17.1268GiB/s) | 20260507 01:22:40.804 | INFO | Analyzing chunk frames = [5983744,5984256) | 20260507 01:22:40.804 | INFO | MemInfo: { | 20260507 01:22:40.804 | INFO | "gpu" : " 17 GB", | 20260507 01:22:40.804 | INFO | "huge" : "", | 20260507 01:22:40.804 | INFO | "huge_shared" : " 27 GB", | 20260507 01:22:40.804 | INFO | "malloc" : " 14 GB", | 20260507 01:22:40.804 | INFO | "pinned" : " 29 GB" | 20260507 01:22:40.804 | INFO | } | 20260507 01:22:40.804 | INFO | Perf report for the last 5.13768 seconds: | 20260507 01:22:40.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:22:40.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:40.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:40.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:40.804 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:22:40.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:40.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:40.804 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:22:40.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:40.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:40.804 | INFO | Uploaded 36GiB to GPU in 1.97513s (18.2266GiB/s) | 20260507 01:22:40.804 | INFO | Downloaded 34GiB to GPU in 1.9746s (17.2187GiB/s) | 20260507 01:22:40.901 | INFO | Starting a round of Baz IO | 20260507 01:22:44.023 | INFO | Populating RTMetrics report | 20260507 01:22:44.023 | INFO | RTMetrics report populated | 20260507 01:22:44.023 | INFO | Opening tmp RTMetrics json file | 20260507 01:22:44.031 | INFO | Serializing RTMetrics to json | 20260507 01:22:44.041 | INFO | Serialization done, swapping tmp file now... | 20260507 01:22:44.041 | INFO | RTMetrics done | 20260507 01:22:44.112 | INFO | Finished a round of Baz IO: Wrote 1.814 GiB in 3.211 seconds | 20260507 01:22:44.112 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.15% / 0.00% / 66.85% | 20260507 01:22:44.112 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.287163% / 0% / 7.94729e-05% | 20260507 01:22:44.112 | INFO | Baz IO overall: | 20260507 01:22:44.112 | INFO | BytesWritten : 3105607430084 | 20260507 01:22:44.112 | INFO | NumEvents : 1494915439015 | 20260507 01:22:45.946 | INFO | Analyzing chunk frames = [5984256,5984768) | 20260507 01:22:45.946 | INFO | MemInfo: { | 20260507 01:22:45.946 | INFO | "gpu" : " 17 GB", | 20260507 01:22:45.946 | INFO | "huge" : "", | 20260507 01:22:45.946 | INFO | "huge_shared" : " 27 GB", | 20260507 01:22:45.946 | INFO | "malloc" : " 14 GB", | 20260507 01:22:45.946 | INFO | "pinned" : " 29 GB" | 20260507 01:22:45.946 | INFO | } | 20260507 01:22:45.946 | INFO | Perf report for the last 5.14187 seconds: | 20260507 01:22:45.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:22:45.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:45.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:45.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:45.946 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 01:22:45.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:45.946 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 01:22:45.946 | INFO | PRE_HQ : 59.7%, 1.8, 56.1ms, 0 | 20260507 01:22:45.946 | INFO | BAZWRITER : 62.0%, 3.3, 106.0ms, 0 | 20260507 01:22:45.946 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 01:22:45.946 | INFO | Uploaded 36GiB to GPU in 2.02926s (17.7404GiB/s) | 20260507 01:22:45.946 | INFO | Downloaded 38GiB to GPU in 2.18067s (17.4258GiB/s) | 20260507 01:22:51.088 | INFO | Analyzing chunk frames = [5984768,5985280) | 20260507 01:22:51.088 | INFO | MemInfo: { | 20260507 01:22:51.088 | INFO | "gpu" : " 17 GB", | 20260507 01:22:51.088 | INFO | "huge" : "", | 20260507 01:22:51.088 | INFO | "huge_shared" : " 27 GB", | 20260507 01:22:51.088 | INFO | "malloc" : " 14 GB", | 20260507 01:22:51.088 | INFO | "pinned" : " 29 GB" | 20260507 01:22:51.088 | INFO | } | 20260507 01:22:51.088 | INFO | Perf report for the last 5.14227 seconds: | 20260507 01:22:51.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:22:51.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:51.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:51.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:51.088 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 01:22:51.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:51.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:51.088 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:22:51.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:51.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:51.089 | INFO | Uploaded 36GiB to GPU in 1.98934s (18.0965GiB/s) | 20260507 01:22:51.089 | INFO | Downloaded 34GiB to GPU in 1.93144s (17.6035GiB/s) | 20260507 01:22:56.235 | INFO | Analyzing chunk frames = [5985280,5985792) | 20260507 01:22:56.235 | INFO | MemInfo: { | 20260507 01:22:56.235 | INFO | "gpu" : " 17 GB", | 20260507 01:22:56.235 | INFO | "huge" : "", | 20260507 01:22:56.235 | INFO | "huge_shared" : " 27 GB", | 20260507 01:22:56.235 | INFO | "malloc" : " 14 GB", | 20260507 01:22:56.235 | INFO | "pinned" : " 29 GB" | 20260507 01:22:56.235 | INFO | } | 20260507 01:22:56.235 | INFO | Perf report for the last 5.14671 seconds: | 20260507 01:22:56.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:22:56.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:56.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:56.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:56.235 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 01:22:56.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:56.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:56.235 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 01:22:56.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:56.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:22:56.235 | INFO | Uploaded 36GiB to GPU in 1.9885s (18.1041GiB/s) | 20260507 01:22:56.235 | INFO | Downloaded 34GiB to GPU in 1.9362s (17.5601GiB/s) | 20260507 01:23:01.365 | INFO | Analyzing chunk frames = [5985792,5986304) | 20260507 01:23:01.365 | INFO | MemInfo: { | 20260507 01:23:01.365 | INFO | "gpu" : " 17 GB", | 20260507 01:23:01.365 | INFO | "huge" : "", | 20260507 01:23:01.365 | INFO | "huge_shared" : " 27 GB", | 20260507 01:23:01.366 | INFO | "malloc" : " 14 GB", | 20260507 01:23:01.366 | INFO | "pinned" : " 29 GB" | 20260507 01:23:01.366 | INFO | } | 20260507 01:23:01.366 | INFO | Perf report for the last 5.13085 seconds: | 20260507 01:23:01.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:23:01.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:01.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:01.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:01.366 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260507 01:23:01.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:01.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:01.366 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:23:01.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:01.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:01.366 | INFO | Uploaded 36GiB to GPU in 1.96364s (18.3333GiB/s) | 20260507 01:23:01.366 | INFO | Downloaded 34GiB to GPU in 1.92294s (17.6813GiB/s) | 20260507 01:23:06.506 | INFO | Analyzing chunk frames = [5986304,5986816) | 20260507 01:23:06.506 | INFO | MemInfo: { | 20260507 01:23:06.506 | INFO | "gpu" : " 17 GB", | 20260507 01:23:06.506 | INFO | "huge" : "", | 20260507 01:23:06.506 | INFO | "huge_shared" : " 27 GB", | 20260507 01:23:06.506 | INFO | "malloc" : " 14 GB", | 20260507 01:23:06.506 | INFO | "pinned" : " 29 GB" | 20260507 01:23:06.506 | INFO | } | 20260507 01:23:06.507 | INFO | Perf report for the last 5.1409 seconds: | 20260507 01:23:06.507 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:23:06.507 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:06.507 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:06.507 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:06.507 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 01:23:06.507 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:06.507 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:06.507 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:23:06.507 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:06.507 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:06.507 | INFO | Uploaded 36GiB to GPU in 2.00413s (17.9629GiB/s) | 20260507 01:23:06.507 | INFO | Downloaded 34GiB to GPU in 1.94879s (17.4467GiB/s) | 20260507 01:23:11.644 | INFO | Analyzing chunk frames = [5986816,5987328) | 20260507 01:23:11.644 | INFO | MemInfo: { | 20260507 01:23:11.644 | INFO | "gpu" : " 17 GB", | 20260507 01:23:11.644 | INFO | "huge" : "", | 20260507 01:23:11.644 | INFO | "huge_shared" : " 27 GB", | 20260507 01:23:11.644 | INFO | "malloc" : " 14 GB", | 20260507 01:23:11.644 | INFO | "pinned" : " 29 GB" | 20260507 01:23:11.644 | INFO | } | 20260507 01:23:11.644 | INFO | Perf report for the last 5.13736 seconds: | 20260507 01:23:11.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:23:11.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:11.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:11.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:11.644 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 01:23:11.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:11.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:11.644 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 01:23:11.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:11.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:11.644 | INFO | Uploaded 36GiB to GPU in 2.04914s (17.5684GiB/s) | 20260507 01:23:11.644 | INFO | Downloaded 34GiB to GPU in 1.96399s (17.3117GiB/s) | 20260507 01:23:16.781 | INFO | Analyzing chunk frames = [5987328,5987840) | 20260507 01:23:16.781 | INFO | MemInfo: { | 20260507 01:23:16.781 | INFO | "gpu" : " 17 GB", | 20260507 01:23:16.781 | INFO | "huge" : "", | 20260507 01:23:16.781 | INFO | "huge_shared" : " 27 GB", | 20260507 01:23:16.781 | INFO | "malloc" : " 14 GB", | 20260507 01:23:16.781 | INFO | "pinned" : " 29 GB" | 20260507 01:23:16.781 | INFO | } | 20260507 01:23:16.781 | INFO | Perf report for the last 5.13744 seconds: | 20260507 01:23:16.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:23:16.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:16.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:16.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:16.781 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:23:16.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:16.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:16.782 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:23:16.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:16.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:16.782 | INFO | Uploaded 36GiB to GPU in 1.99285s (18.0645GiB/s) | 20260507 01:23:16.782 | INFO | Downloaded 34GiB to GPU in 1.98814s (17.1014GiB/s) | 20260507 01:23:21.922 | INFO | Analyzing chunk frames = [5987840,5988352) | 20260507 01:23:21.922 | INFO | MemInfo: { | 20260507 01:23:21.922 | INFO | "gpu" : " 17 GB", | 20260507 01:23:21.922 | INFO | "huge" : "", | 20260507 01:23:21.922 | INFO | "huge_shared" : " 27 GB", | 20260507 01:23:21.922 | INFO | "malloc" : " 14 GB", | 20260507 01:23:21.922 | INFO | "pinned" : " 29 GB" | 20260507 01:23:21.922 | INFO | } | 20260507 01:23:21.922 | INFO | Perf report for the last 5.14043 seconds: | 20260507 01:23:21.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:23:21.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:21.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:21.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:21.922 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 01:23:21.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:21.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:21.922 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:23:21.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:21.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:21.922 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:23:21.922 | INFO | Uploaded 36GiB to GPU in 2.00315s (17.9717GiB/s) | 20260507 01:23:21.922 | INFO | Total mem usage: 18 GiB | 20260507 01:23:21.922 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:23:21.922 | INFO | Downloaded 34GiB to GPU in 1.97594s (17.207GiB/s) | 20260507 01:23:21.922 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 01:23:21.922 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:23:21.922 | INFO | High Water Breakdown: | 20260507 01:23:21.922 | INFO | 60.1% : 8.58 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:23:21.922 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:23:21.922 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:23:21.922 | INFO | ------------------------------------------ | 20260507 01:23:21.922 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:23:21.922 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:23:21.922 | INFO | Total mem usage: 26.5 GiB | 20260507 01:23:21.922 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:23:21.922 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:23:21.922 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:23:21.922 | INFO | High Water Breakdown: | 20260507 01:23:21.922 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:23:21.922 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:23:21.922 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:23:21.922 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:23:21.923 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:23:21.923 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:23:21.923 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:23:21.923 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:23:21.923 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:23:21.923 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:23:21.923 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:23:21.923 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:23:21.923 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:23:21.923 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:23:21.923 | INFO | ------------------------------------------ | 20260507 01:23:21.923 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:23:21.923 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:23:21.923 | INFO | Total mem usage: 25.6 GiB | 20260507 01:23:21.923 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:23:21.923 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:23:21.923 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:23:21.923 | INFO | High Water Breakdown: | 20260507 01:23:21.923 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:23:21.923 | INFO | ------------------------------------------ | 20260507 01:23:21.923 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:23:21.923 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:23:21.923 | INFO | Total mem usage: 15.7 GiB | 20260507 01:23:21.923 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:23:21.923 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:23:21.923 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:23:21.923 | INFO | High Water Breakdown: | 20260507 01:23:21.923 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:23:21.923 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:23:21.923 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:23:21.923 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:23:21.923 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:23:21.923 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:23:21.923 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:23:21.923 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:23:21.923 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:23:21.923 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:23:21.923 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:23:21.923 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:23:21.923 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:23:21.923 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:23:21.923 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:23:21.923 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:23:21.923 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:23:21.923 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:23:21.923 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:23:21.923 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:23:21.923 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:23:21.923 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:23:21.923 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:23:21.923 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:23:21.923 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:23:21.923 | INFO | ------------------------------------------ | 20260507 01:23:21.923 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:23:21.923 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:23:21.923 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:23:21.923 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:23:21.923 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.56e+01, 1.35e+01 / 2.02e+00 | 20260507 01:23:21.923 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 1.01e-01, 9.78e-03 / 6.94e-03 | 20260507 01:23:21.923 | INFO | PerfReport: 11.66%, Baseline -- 3.60e+00 / 4.47e+00, 5.21e-01 / 2.63e-01 | 20260507 01:23:21.923 | INFO | PerfReport: 3.23%, Histogram -- 9.97e-01 / 1.48e+00, 1.70e-01 / 9.44e-02 | 20260507 01:23:21.923 | INFO | PerfReport: 16.11%, DME -- 3.98e+01 / 4.48e+01, 3.91e+00 / 2.47e+00 | 20260507 01:23:21.923 | INFO | PerfReport: 57.52%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.35e+00 / 1.59e+00 | 20260507 01:23:21.923 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.17e+00 / 2.72e+00, 3.84e-01 / 2.06e-01 | 20260507 01:23:21.923 | INFO | PerfReport: 3.10%, Metrics -- 9.58e-01 / 1.27e+00, 2.54e-01 / 1.22e-01 | 20260507 01:23:21.923 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.26e-01 / 3.79e-01, 5.17e-02 / 2.13e-02 | 20260507 01:23:21.923 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:23:21.923 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:23:21.923 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:23:21.923 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 01:23:21.923 | INFO | PerfReport: 21.59%, Upload -- 2.08e+01 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260507 01:23:21.923 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 01:23:21.923 | INFO | PerfReport: 32.02%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 01:23:21.923 | INFO | PerfReport: 1.30%, LockUpload -- 1.25e+00 / 0.00e+00, 4.87e+00 / 0.00e+00 | 20260507 01:23:21.923 | INFO | PerfReport: 1.36%, LockDownload -- 1.31e+00 / 0.00e+00, 3.18e+00 / 0.00e+00 | 20260507 01:23:21.923 | INFO | PerfReport: 22.37%, LockCompute -- 2.16e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 01:23:21.923 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260507 01:23:22.019 | INFO | Starting a round of Baz IO | 20260507 01:23:25.137 | INFO | Populating RTMetrics report | 20260507 01:23:25.138 | INFO | RTMetrics report populated | 20260507 01:23:25.138 | INFO | Opening tmp RTMetrics json file | 20260507 01:23:25.140 | INFO | Serializing RTMetrics to json | 20260507 01:23:25.158 | INFO | Serialization done, swapping tmp file now... | 20260507 01:23:25.158 | INFO | RTMetrics done | 20260507 01:23:25.227 | INFO | Finished a round of Baz IO: Wrote 1.813 GiB in 3.207 seconds | 20260507 01:23:25.227 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.14% / 0.00% / 66.85% | 20260507 01:23:25.227 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.286988% / 0% / 7.94729e-05% | 20260507 01:23:25.227 | INFO | Baz IO overall: | 20260507 01:23:25.227 | INFO | BytesWritten : 3107554612747 | 20260507 01:23:25.227 | INFO | NumEvents : 1495838030425 | 20260507 01:23:27.062 | INFO | Analyzing chunk frames = [5988352,5988864) | 20260507 01:23:27.062 | INFO | MemInfo: { | 20260507 01:23:27.062 | INFO | "gpu" : " 17 GB", | 20260507 01:23:27.062 | INFO | "huge" : "", | 20260507 01:23:27.062 | INFO | "huge_shared" : " 27 GB", | 20260507 01:23:27.062 | INFO | "malloc" : " 14 GB", | 20260507 01:23:27.062 | INFO | "pinned" : " 29 GB" | 20260507 01:23:27.062 | INFO | } | 20260507 01:23:27.062 | INFO | Perf report for the last 5.14022 seconds: | 20260507 01:23:27.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:23:27.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:27.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:27.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:27.062 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 01:23:27.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:27.062 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 01:23:27.062 | INFO | PRE_HQ : 59.5%, 1.8, 56.0ms, 0 | 20260507 01:23:27.062 | INFO | BAZWRITER : 61.9%, 3.3, 105.6ms, 0 | 20260507 01:23:27.062 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:23:27.062 | INFO | Uploaded 36GiB to GPU in 2.02706s (17.7597GiB/s) | 20260507 01:23:27.063 | INFO | Downloaded 38GiB to GPU in 2.1757s (17.4657GiB/s) | 20260507 01:23:32.201 | INFO | Analyzing chunk frames = [5988864,5989376) | 20260507 01:23:32.201 | INFO | MemInfo: { | 20260507 01:23:32.201 | INFO | "gpu" : " 17 GB", | 20260507 01:23:32.201 | INFO | "huge" : "", | 20260507 01:23:32.202 | INFO | "huge_shared" : " 27 GB", | 20260507 01:23:32.202 | INFO | "malloc" : " 14 GB", | 20260507 01:23:32.202 | INFO | "pinned" : " 29 GB" | 20260507 01:23:32.202 | INFO | } | 20260507 01:23:32.202 | INFO | Perf report for the last 5.13951 seconds: | 20260507 01:23:32.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:23:32.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:32.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:32.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:32.202 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:23:32.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:32.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:32.202 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:23:32.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:32.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:32.202 | INFO | Uploaded 36GiB to GPU in 2.00363s (17.9674GiB/s) | 20260507 01:23:32.202 | INFO | Downloaded 34GiB to GPU in 1.93127s (17.605GiB/s) | 20260507 01:23:37.345 | INFO | Analyzing chunk frames = [5989376,5989888) | 20260507 01:23:37.345 | INFO | MemInfo: { | 20260507 01:23:37.345 | INFO | "gpu" : " 17 GB", | 20260507 01:23:37.345 | INFO | "huge" : "", | 20260507 01:23:37.345 | INFO | "huge_shared" : " 27 GB", | 20260507 01:23:37.345 | INFO | "malloc" : " 14 GB", | 20260507 01:23:37.345 | INFO | "pinned" : " 29 GB" | 20260507 01:23:37.345 | INFO | } | 20260507 01:23:37.345 | INFO | Perf report for the last 5.14311 seconds: | 20260507 01:23:37.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:23:37.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:37.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:37.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:37.345 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260507 01:23:37.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:37.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:37.345 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:23:37.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:37.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:37.345 | INFO | Uploaded 36GiB to GPU in 1.97859s (18.1947GiB/s) | 20260507 01:23:37.345 | INFO | Downloaded 34GiB to GPU in 1.92875s (17.628GiB/s) | 20260507 01:23:42.481 | INFO | Analyzing chunk frames = [5989888,5990400) | 20260507 01:23:42.481 | INFO | MemInfo: { | 20260507 01:23:42.481 | INFO | "gpu" : " 17 GB", | 20260507 01:23:42.481 | INFO | "huge" : "", | 20260507 01:23:42.481 | INFO | "huge_shared" : " 27 GB", | 20260507 01:23:42.481 | INFO | "malloc" : " 14 GB", | 20260507 01:23:42.481 | INFO | "pinned" : " 29 GB" | 20260507 01:23:42.481 | INFO | } | 20260507 01:23:42.481 | INFO | Perf report for the last 5.1367 seconds: | 20260507 01:23:42.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:23:42.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:42.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:42.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:42.481 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 01:23:42.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:42.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:42.482 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:23:42.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:42.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:42.482 | INFO | Uploaded 36GiB to GPU in 1.97688s (18.2105GiB/s) | 20260507 01:23:42.482 | INFO | Downloaded 34GiB to GPU in 1.90749s (17.8245GiB/s) | 20260507 01:23:47.618 | INFO | Analyzing chunk frames = [5990400,5990912) | 20260507 01:23:47.618 | INFO | MemInfo: { | 20260507 01:23:47.618 | INFO | "gpu" : " 17 GB", | 20260507 01:23:47.618 | INFO | "huge" : "", | 20260507 01:23:47.619 | INFO | "huge_shared" : " 27 GB", | 20260507 01:23:47.619 | INFO | "malloc" : " 14 GB", | 20260507 01:23:47.619 | INFO | "pinned" : " 29 GB" | 20260507 01:23:47.619 | INFO | } | 20260507 01:23:47.619 | INFO | Perf report for the last 5.13722 seconds: | 20260507 01:23:47.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:23:47.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:47.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:47.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:47.619 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 01:23:47.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:47.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:47.619 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:23:47.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:47.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:47.619 | INFO | Uploaded 36GiB to GPU in 2.02131s (17.8103GiB/s) | 20260507 01:23:47.619 | INFO | Downloaded 34GiB to GPU in 1.94402s (17.4895GiB/s) | 20260507 01:23:52.755 | INFO | Analyzing chunk frames = [5990912,5991424) | 20260507 01:23:52.755 | INFO | MemInfo: { | 20260507 01:23:52.755 | INFO | "gpu" : " 17 GB", | 20260507 01:23:52.755 | INFO | "huge" : "", | 20260507 01:23:52.755 | INFO | "huge_shared" : " 27 GB", | 20260507 01:23:52.755 | INFO | "malloc" : " 14 GB", | 20260507 01:23:52.755 | INFO | "pinned" : " 29 GB" | 20260507 01:23:52.755 | INFO | } | 20260507 01:23:52.755 | INFO | Perf report for the last 5.13645 seconds: | 20260507 01:23:52.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:23:52.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:52.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:52.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:52.755 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:23:52.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:52.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:52.755 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 01:23:52.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:52.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:52.755 | INFO | Uploaded 36GiB to GPU in 2.05892s (17.4849GiB/s) | 20260507 01:23:52.755 | INFO | Downloaded 34GiB to GPU in 1.96355s (17.3156GiB/s) | 20260507 01:23:57.899 | INFO | Analyzing chunk frames = [5991424,5991936) | 20260507 01:23:57.899 | INFO | MemInfo: { | 20260507 01:23:57.899 | INFO | "gpu" : " 17 GB", | 20260507 01:23:57.900 | INFO | "huge" : "", | 20260507 01:23:57.900 | INFO | "huge_shared" : " 27 GB", | 20260507 01:23:57.900 | INFO | "malloc" : " 14 GB", | 20260507 01:23:57.900 | INFO | "pinned" : " 29 GB" | 20260507 01:23:57.900 | INFO | } | 20260507 01:23:57.900 | INFO | Perf report for the last 5.14442 seconds: | 20260507 01:23:57.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:23:57.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:57.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:57.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:57.900 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:23:57.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:57.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:57.900 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 01:23:57.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:57.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:23:57.900 | INFO | Uploaded 36GiB to GPU in 1.98085s (18.174GiB/s) | 20260507 01:23:57.900 | INFO | Downloaded 34GiB to GPU in 2.00091s (16.9922GiB/s) | 20260507 01:24:03.047 | INFO | Analyzing chunk frames = [5991936,5992448) | 20260507 01:24:03.047 | INFO | MemInfo: { | 20260507 01:24:03.047 | INFO | "gpu" : " 17 GB", | 20260507 01:24:03.047 | INFO | "huge" : "", | 20260507 01:24:03.047 | INFO | "huge_shared" : " 27 GB", | 20260507 01:24:03.047 | INFO | "malloc" : " 14 GB", | 20260507 01:24:03.047 | INFO | "pinned" : " 29 GB" | 20260507 01:24:03.047 | INFO | } | 20260507 01:24:03.047 | INFO | Perf report for the last 5.14791 seconds: | 20260507 01:24:03.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:24:03.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:03.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:03.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:03.048 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260507 01:24:03.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:03.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:03.048 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 01:24:03.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:03.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:03.048 | INFO | Uploaded 36GiB to GPU in 1.99005s (18.09GiB/s) | 20260507 01:24:03.048 | INFO | Downloaded 34GiB to GPU in 1.98099s (17.1631GiB/s) | 20260507 01:24:03.144 | INFO | Starting a round of Baz IO | 20260507 01:24:06.286 | INFO | Populating RTMetrics report | 20260507 01:24:06.287 | INFO | RTMetrics report populated | 20260507 01:24:06.287 | INFO | Opening tmp RTMetrics json file | 20260507 01:24:06.287 | INFO | Serializing RTMetrics to json | 20260507 01:24:06.289 | INFO | Serialization done, swapping tmp file now... | 20260507 01:24:06.289 | INFO | RTMetrics done | 20260507 01:24:06.379 | INFO | Finished a round of Baz IO: Wrote 1.813 GiB in 3.234 seconds | 20260507 01:24:06.379 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.14% / 0.00% / 66.86% | 20260507 01:24:06.379 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.286293% / 3.97364e-06% / 7.94729e-05% | 20260507 01:24:06.379 | INFO | Baz IO overall: | 20260507 01:24:06.379 | INFO | BytesWritten : 3109501203054 | 20260507 01:24:06.379 | INFO | NumEvents : 1496760441295 | 20260507 01:24:08.181 | INFO | Analyzing chunk frames = [5992448,5992960) | 20260507 01:24:08.181 | INFO | MemInfo: { | 20260507 01:24:08.181 | INFO | "gpu" : " 17 GB", | 20260507 01:24:08.181 | INFO | "huge" : "", | 20260507 01:24:08.181 | INFO | "huge_shared" : " 27 GB", | 20260507 01:24:08.181 | INFO | "malloc" : " 14 GB", | 20260507 01:24:08.181 | INFO | "pinned" : " 29 GB" | 20260507 01:24:08.181 | INFO | } | 20260507 01:24:08.181 | INFO | Perf report for the last 5.13395 seconds: | 20260507 01:24:08.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:24:08.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:08.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:08.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:08.181 | INFO | ANALYSIS : 62.0%, 3.0, 96.2ms, 0 | 20260507 01:24:08.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:08.182 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260507 01:24:08.182 | INFO | PRE_HQ : 60.2%, 1.9, 60.0ms, 0 | 20260507 01:24:08.182 | INFO | BAZWRITER : 62.6%, 3.4, 110.4ms, 0 | 20260507 01:24:08.182 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 01:24:08.182 | INFO | Uploaded 36GiB to GPU in 2.03655s (17.677GiB/s) | 20260507 01:24:08.182 | INFO | Downloaded 38GiB to GPU in 2.18428s (17.397GiB/s) | 20260507 01:24:13.321 | INFO | Analyzing chunk frames = [5992960,5993472) | 20260507 01:24:13.321 | INFO | MemInfo: { | 20260507 01:24:13.321 | INFO | "gpu" : " 17 GB", | 20260507 01:24:13.321 | INFO | "huge" : "", | 20260507 01:24:13.321 | INFO | "huge_shared" : " 27 GB", | 20260507 01:24:13.321 | INFO | "malloc" : " 14 GB", | 20260507 01:24:13.321 | INFO | "pinned" : " 29 GB" | 20260507 01:24:13.321 | INFO | } | 20260507 01:24:13.321 | INFO | Perf report for the last 5.13932 seconds: | 20260507 01:24:13.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:24:13.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:13.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:13.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:13.321 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:24:13.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:13.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:13.321 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 01:24:13.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:13.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:13.321 | INFO | Uploaded 36GiB to GPU in 2.02424s (17.7844GiB/s) | 20260507 01:24:13.321 | INFO | Downloaded 34GiB to GPU in 1.94308s (17.498GiB/s) | 20260507 01:24:18.457 | INFO | Analyzing chunk frames = [5993472,5993984) | 20260507 01:24:18.457 | INFO | MemInfo: { | 20260507 01:24:18.457 | INFO | "gpu" : " 17 GB", | 20260507 01:24:18.457 | INFO | "huge" : "", | 20260507 01:24:18.457 | INFO | "huge_shared" : " 27 GB", | 20260507 01:24:18.457 | INFO | "malloc" : " 14 GB", | 20260507 01:24:18.457 | INFO | "pinned" : " 29 GB" | 20260507 01:24:18.457 | INFO | } | 20260507 01:24:18.457 | INFO | Perf report for the last 5.13614 seconds: | 20260507 01:24:18.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:24:18.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:18.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:18.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:18.457 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 01:24:18.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:18.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:18.457 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:24:18.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:18.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:18.457 | INFO | Uploaded 36GiB to GPU in 1.99112s (18.0803GiB/s) | 20260507 01:24:18.457 | INFO | Downloaded 34GiB to GPU in 1.93898s (17.535GiB/s) | 20260507 01:24:23.595 | INFO | Analyzing chunk frames = [5993984,5994496) | 20260507 01:24:23.595 | INFO | MemInfo: { | 20260507 01:24:23.595 | INFO | "gpu" : " 17 GB", | 20260507 01:24:23.595 | INFO | "huge" : "", | 20260507 01:24:23.595 | INFO | "huge_shared" : " 27 GB", | 20260507 01:24:23.595 | INFO | "malloc" : " 14 GB", | 20260507 01:24:23.595 | INFO | "pinned" : " 29 GB" | 20260507 01:24:23.595 | INFO | } | 20260507 01:24:23.595 | INFO | Perf report for the last 5.1381 seconds: | 20260507 01:24:23.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:24:23.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:23.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:23.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:23.595 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 01:24:23.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:23.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:23.595 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:24:23.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:23.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:23.595 | INFO | Uploaded 36GiB to GPU in 1.95807s (18.3855GiB/s) | 20260507 01:24:23.596 | INFO | Downloaded 34GiB to GPU in 1.90171s (17.8786GiB/s) | 20260507 01:24:28.739 | INFO | Analyzing chunk frames = [5994496,5995008) | 20260507 01:24:28.739 | INFO | MemInfo: { | 20260507 01:24:28.739 | INFO | "gpu" : " 17 GB", | 20260507 01:24:28.739 | INFO | "huge" : "", | 20260507 01:24:28.739 | INFO | "huge_shared" : " 27 GB", | 20260507 01:24:28.739 | INFO | "malloc" : " 14 GB", | 20260507 01:24:28.739 | INFO | "pinned" : " 29 GB" | 20260507 01:24:28.739 | INFO | } | 20260507 01:24:28.739 | INFO | Perf report for the last 5.14442 seconds: | 20260507 01:24:28.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:24:28.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:28.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:28.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:28.740 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 01:24:28.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:28.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:28.740 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:24:28.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:28.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:28.740 | INFO | Uploaded 36GiB to GPU in 1.98772s (18.1112GiB/s) | 20260507 01:24:28.740 | INFO | Downloaded 34GiB to GPU in 1.94865s (17.4479GiB/s) | 20260507 01:24:33.871 | INFO | Analyzing chunk frames = [5995008,5995520) | 20260507 01:24:33.871 | INFO | MemInfo: { | 20260507 01:24:33.871 | INFO | "gpu" : " 17 GB", | 20260507 01:24:33.871 | INFO | "huge" : "", | 20260507 01:24:33.871 | INFO | "huge_shared" : " 27 GB", | 20260507 01:24:33.871 | INFO | "malloc" : " 14 GB", | 20260507 01:24:33.871 | INFO | "pinned" : " 29 GB" | 20260507 01:24:33.871 | INFO | } | 20260507 01:24:33.871 | INFO | Perf report for the last 5.13165 seconds: | 20260507 01:24:33.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:24:33.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:33.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:33.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:33.871 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 01:24:33.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:33.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:33.871 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:24:33.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:33.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:33.871 | INFO | Uploaded 36GiB to GPU in 2.04984s (17.5623GiB/s) | 20260507 01:24:33.871 | INFO | Downloaded 34GiB to GPU in 1.96507s (17.3022GiB/s) | 20260507 01:24:39.012 | INFO | Analyzing chunk frames = [5995520,5996032) | 20260507 01:24:39.012 | INFO | MemInfo: { | 20260507 01:24:39.012 | INFO | "gpu" : " 17 GB", | 20260507 01:24:39.012 | INFO | "huge" : "", | 20260507 01:24:39.012 | INFO | "huge_shared" : " 27 GB", | 20260507 01:24:39.012 | INFO | "malloc" : " 14 GB", | 20260507 01:24:39.012 | INFO | "pinned" : " 29 GB" | 20260507 01:24:39.012 | INFO | } | 20260507 01:24:39.012 | INFO | Perf report for the last 5.14052 seconds: | 20260507 01:24:39.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:24:39.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:39.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:39.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:39.012 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:24:39.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:39.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:39.012 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 01:24:39.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:39.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:39.012 | INFO | Uploaded 36GiB to GPU in 1.96493s (18.3213GiB/s) | 20260507 01:24:39.012 | INFO | Downloaded 34GiB to GPU in 1.99759s (17.0205GiB/s) | 20260507 01:24:44.154 | INFO | Analyzing chunk frames = [5996032,5996544) | 20260507 01:24:44.154 | INFO | MemInfo: { | 20260507 01:24:44.154 | INFO | "gpu" : " 17 GB", | 20260507 01:24:44.154 | INFO | "huge" : "", | 20260507 01:24:44.155 | INFO | "huge_shared" : " 27 GB", | 20260507 01:24:44.155 | INFO | "malloc" : " 14 GB", | 20260507 01:24:44.155 | INFO | "pinned" : " 29 GB" | 20260507 01:24:44.155 | INFO | } | 20260507 01:24:44.155 | INFO | Perf report for the last 5.14363 seconds: | 20260507 01:24:44.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:24:44.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:44.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:44.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:44.155 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:24:44.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:44.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:44.155 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 01:24:44.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:44.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:44.156 | INFO | Uploaded 36GiB to GPU in 1.96185s (18.35GiB/s) | 20260507 01:24:44.156 | INFO | Downloaded 34GiB to GPU in 1.97688s (17.1988GiB/s) | 20260507 01:24:44.156 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:24:44.156 | INFO | Total mem usage: 18 GiB | 20260507 01:24:44.156 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:24:44.156 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:24:44.156 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:24:44.156 | INFO | High Water Breakdown: | 20260507 01:24:44.156 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:24:44.156 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:24:44.156 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:24:44.156 | INFO | ------------------------------------------ | 20260507 01:24:44.156 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:24:44.156 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:24:44.156 | INFO | Total mem usage: 26.5 GiB | 20260507 01:24:44.156 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:24:44.156 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:24:44.156 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:24:44.156 | INFO | High Water Breakdown: | 20260507 01:24:44.156 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:24:44.156 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:24:44.156 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:24:44.156 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:24:44.156 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:24:44.156 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:24:44.156 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:24:44.156 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:24:44.156 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:24:44.156 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:24:44.156 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:24:44.156 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:24:44.156 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:24:44.156 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:24:44.156 | INFO | ------------------------------------------ | 20260507 01:24:44.156 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:24:44.156 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:24:44.156 | INFO | Total mem usage: 25.6 GiB | 20260507 01:24:44.156 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:24:44.156 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:24:44.156 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:24:44.156 | INFO | High Water Breakdown: | 20260507 01:24:44.156 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:24:44.156 | INFO | ------------------------------------------ | 20260507 01:24:44.156 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:24:44.156 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:24:44.156 | INFO | Total mem usage: 15.7 GiB | 20260507 01:24:44.156 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:24:44.156 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:24:44.156 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:24:44.156 | INFO | High Water Breakdown: | 20260507 01:24:44.156 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:24:44.156 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:24:44.156 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:24:44.156 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:24:44.156 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:24:44.156 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:24:44.156 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:24:44.156 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:24:44.156 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:24:44.156 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:24:44.156 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:24:44.156 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:24:44.156 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:24:44.156 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:24:44.156 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:24:44.156 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:24:44.156 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:24:44.156 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:24:44.156 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:24:44.156 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:24:44.156 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:24:44.156 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:24:44.156 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:24:44.156 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:24:44.157 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:24:44.157 | INFO | ------------------------------------------ | 20260507 01:24:44.157 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:24:44.157 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:24:44.157 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:24:44.157 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:24:44.157 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.59e+01, 1.33e+01 / 2.40e+00 | 20260507 01:24:44.157 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.02e-01, 1.00e-02 / 4.47e-03 | 20260507 01:24:44.157 | INFO | PerfReport: 11.64%, Baseline -- 3.59e+00 / 4.45e+00, 5.17e-01 / 2.42e-01 | 20260507 01:24:44.157 | INFO | PerfReport: 3.24%, Histogram -- 9.98e-01 / 1.49e+00, 1.73e-01 / 9.89e-02 | 20260507 01:24:44.157 | INFO | PerfReport: 16.16%, DME -- 3.99e+01 / 4.51e+01, 4.25e+00 / 2.71e+00 | 20260507 01:24:44.157 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.25e+00 / 1.74e+00 | 20260507 01:24:44.157 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.17e+00 / 2.73e+00, 3.86e-01 / 2.02e-01 | 20260507 01:24:44.157 | INFO | PerfReport: 3.11%, Metrics -- 9.61e-01 / 1.30e+00, 2.56e-01 / 1.30e-01 | 20260507 01:24:44.157 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.27e-01 / 3.82e-01, 5.17e-02 / 2.26e-02 | 20260507 01:24:44.157 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:24:44.157 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:24:44.157 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:24:44.157 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 01:24:44.157 | INFO | PerfReport: 21.60%, Upload -- 2.08e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260507 01:24:44.157 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 01:24:44.157 | INFO | PerfReport: 31.97%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 01:24:44.157 | INFO | PerfReport: 1.38%, LockUpload -- 1.33e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 01:24:44.157 | INFO | PerfReport: 1.33%, LockDownload -- 1.29e+00 / 0.00e+00, 3.16e+00 / 0.00e+00 | 20260507 01:24:44.157 | INFO | PerfReport: 22.33%, LockCompute -- 2.15e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 01:24:44.157 | INFO | PerfReport: 0.02%, Misc -- 1.53e-02 / 0.00e+00, 2.99e-02 / 0.00e+00 | 20260507 01:24:44.254 | INFO | Starting a round of Baz IO | 20260507 01:24:47.376 | INFO | Populating RTMetrics report | 20260507 01:24:47.377 | INFO | RTMetrics report populated | 20260507 01:24:47.377 | INFO | Opening tmp RTMetrics json file | 20260507 01:24:47.379 | INFO | Serializing RTMetrics to json | 20260507 01:24:47.397 | INFO | Serialization done, swapping tmp file now... | 20260507 01:24:47.397 | INFO | RTMetrics done | 20260507 01:24:47.467 | INFO | Finished a round of Baz IO: Wrote 1.813 GiB in 3.213 seconds | 20260507 01:24:47.467 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.14% / 0.00% / 66.86% | 20260507 01:24:47.467 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.286341% / 0% / 5.5631e-05% | 20260507 01:24:47.467 | INFO | Baz IO overall: | 20260507 01:24:47.467 | INFO | BytesWritten : 3111448362289 | 20260507 01:24:47.467 | INFO | NumEvents : 1497683268979 | 20260507 01:24:49.295 | INFO | Analyzing chunk frames = [5996544,5997056) | 20260507 01:24:49.295 | INFO | MemInfo: { | 20260507 01:24:49.295 | INFO | "gpu" : " 17 GB", | 20260507 01:24:49.295 | INFO | "huge" : "", | 20260507 01:24:49.295 | INFO | "huge_shared" : " 27 GB", | 20260507 01:24:49.295 | INFO | "malloc" : " 14 GB", | 20260507 01:24:49.295 | INFO | "pinned" : " 29 GB" | 20260507 01:24:49.295 | INFO | } | 20260507 01:24:49.295 | INFO | Perf report for the last 5.13953 seconds: | 20260507 01:24:49.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:24:49.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:49.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:49.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:49.295 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260507 01:24:49.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:49.295 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 01:24:49.295 | INFO | PRE_HQ : 60.2%, 1.9, 59.2ms, 0 | 20260507 01:24:49.295 | INFO | BAZWRITER : 62.0%, 3.4, 109.0ms, 0 | 20260507 01:24:49.295 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 01:24:49.295 | INFO | Uploaded 36GiB to GPU in 2.03888s (17.6568GiB/s) | 20260507 01:24:49.295 | INFO | Downloaded 38GiB to GPU in 2.17854s (17.4428GiB/s) | 20260507 01:24:54.428 | INFO | Analyzing chunk frames = [5997056,5997568) | 20260507 01:24:54.428 | INFO | MemInfo: { | 20260507 01:24:54.428 | INFO | "gpu" : " 17 GB", | 20260507 01:24:54.428 | INFO | "huge" : "", | 20260507 01:24:54.428 | INFO | "huge_shared" : " 27 GB", | 20260507 01:24:54.428 | INFO | "malloc" : " 14 GB", | 20260507 01:24:54.428 | INFO | "pinned" : " 29 GB" | 20260507 01:24:54.428 | INFO | } | 20260507 01:24:54.428 | INFO | Perf report for the last 5.13339 seconds: | 20260507 01:24:54.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:24:54.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:54.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:54.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:54.428 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:24:54.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:54.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:54.428 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 01:24:54.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:54.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:54.429 | INFO | Uploaded 36GiB to GPU in 2.04249s (17.6255GiB/s) | 20260507 01:24:54.429 | INFO | Downloaded 34GiB to GPU in 1.94291s (17.4995GiB/s) | 20260507 01:24:59.567 | INFO | Analyzing chunk frames = [5997568,5998080) | 20260507 01:24:59.567 | INFO | MemInfo: { | 20260507 01:24:59.567 | INFO | "gpu" : " 17 GB", | 20260507 01:24:59.567 | INFO | "huge" : "", | 20260507 01:24:59.567 | INFO | "huge_shared" : " 27 GB", | 20260507 01:24:59.567 | INFO | "malloc" : " 14 GB", | 20260507 01:24:59.567 | INFO | "pinned" : " 29 GB" | 20260507 01:24:59.567 | INFO | } | 20260507 01:24:59.567 | INFO | Perf report for the last 5.13901 seconds: | 20260507 01:24:59.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:24:59.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:59.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:59.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:59.567 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:24:59.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:59.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:59.568 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:24:59.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:59.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:24:59.568 | INFO | Uploaded 36GiB to GPU in 1.97373s (18.2396GiB/s) | 20260507 01:24:59.568 | INFO | Downloaded 34GiB to GPU in 1.93627s (17.5595GiB/s) | 20260507 01:25:04.708 | INFO | Analyzing chunk frames = [5998080,5998592) | 20260507 01:25:04.708 | INFO | MemInfo: { | 20260507 01:25:04.708 | INFO | "gpu" : " 17 GB", | 20260507 01:25:04.708 | INFO | "huge" : "", | 20260507 01:25:04.708 | INFO | "huge_shared" : " 27 GB", | 20260507 01:25:04.708 | INFO | "malloc" : " 14 GB", | 20260507 01:25:04.708 | INFO | "pinned" : " 29 GB" | 20260507 01:25:04.708 | INFO | } | 20260507 01:25:04.708 | INFO | Perf report for the last 5.14092 seconds: | 20260507 01:25:04.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:25:04.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:04.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:04.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:04.708 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260507 01:25:04.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:04.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:04.708 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:25:04.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:04.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:04.708 | INFO | Uploaded 36GiB to GPU in 2.01403s (17.8746GiB/s) | 20260507 01:25:04.709 | INFO | Downloaded 34GiB to GPU in 1.91135s (17.7885GiB/s) | 20260507 01:25:09.852 | INFO | Analyzing chunk frames = [5998592,5999104) | 20260507 01:25:09.852 | INFO | MemInfo: { | 20260507 01:25:09.852 | INFO | "gpu" : " 17 GB", | 20260507 01:25:09.852 | INFO | "huge" : "", | 20260507 01:25:09.852 | INFO | "huge_shared" : " 27 GB", | 20260507 01:25:09.852 | INFO | "malloc" : " 14 GB", | 20260507 01:25:09.852 | INFO | "pinned" : " 29 GB" | 20260507 01:25:09.852 | INFO | } | 20260507 01:25:09.852 | INFO | Perf report for the last 5.14412 seconds: | 20260507 01:25:09.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:25:09.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:09.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:09.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:09.853 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 01:25:09.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:09.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:09.853 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:25:09.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:09.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:09.853 | INFO | Uploaded 36GiB to GPU in 1.99535s (18.042GiB/s) | 20260507 01:25:09.853 | INFO | Downloaded 34GiB to GPU in 1.93773s (17.5463GiB/s) | 20260507 01:25:14.993 | INFO | Analyzing chunk frames = [5999104,5999616) | 20260507 01:25:14.993 | INFO | MemInfo: { | 20260507 01:25:14.993 | INFO | "gpu" : " 17 GB", | 20260507 01:25:14.993 | INFO | "huge" : "", | 20260507 01:25:14.993 | INFO | "huge_shared" : " 27 GB", | 20260507 01:25:14.993 | INFO | "malloc" : " 14 GB", | 20260507 01:25:14.993 | INFO | "pinned" : " 29 GB" | 20260507 01:25:14.994 | INFO | } | 20260507 01:25:14.994 | INFO | Perf report for the last 5.14123 seconds: | 20260507 01:25:14.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:25:14.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:14.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:14.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:14.994 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 01:25:14.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:14.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:14.994 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:25:14.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:14.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:14.994 | INFO | Uploaded 36GiB to GPU in 2.05121s (17.5506GiB/s) | 20260507 01:25:14.994 | INFO | Downloaded 34GiB to GPU in 1.9617s (17.3319GiB/s) | 20260507 01:25:20.129 | INFO | Analyzing chunk frames = [5999616,6000128) | 20260507 01:25:20.129 | INFO | MemInfo: { | 20260507 01:25:20.129 | INFO | "gpu" : " 17 GB", | 20260507 01:25:20.129 | INFO | "huge" : "", | 20260507 01:25:20.129 | INFO | "huge_shared" : " 27 GB", | 20260507 01:25:20.129 | INFO | "malloc" : " 14 GB", | 20260507 01:25:20.129 | INFO | "pinned" : " 29 GB" | 20260507 01:25:20.129 | INFO | } | 20260507 01:25:20.129 | INFO | Perf report for the last 5.13554 seconds: | 20260507 01:25:20.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:25:20.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:20.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:20.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:20.129 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260507 01:25:20.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:20.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:20.129 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 01:25:20.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:20.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:20.130 | INFO | Uploaded 36GiB to GPU in 1.98411s (18.1441GiB/s) | 20260507 01:25:20.130 | INFO | Downloaded 34GiB to GPU in 1.98586s (17.121GiB/s) | 20260507 01:25:25.270 | INFO | Analyzing chunk frames = [6000128,6000640) | 20260507 01:25:25.270 | INFO | MemInfo: { | 20260507 01:25:25.270 | INFO | "gpu" : " 17 GB", | 20260507 01:25:25.270 | INFO | "huge" : "", | 20260507 01:25:25.270 | INFO | "huge_shared" : " 27 GB", | 20260507 01:25:25.270 | INFO | "malloc" : " 14 GB", | 20260507 01:25:25.270 | INFO | "pinned" : " 29 GB" | 20260507 01:25:25.270 | INFO | } | 20260507 01:25:25.270 | INFO | Perf report for the last 5.14067 seconds: | 20260507 01:25:25.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:25:25.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:25.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:25.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:25.270 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:25:25.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:25.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:25.270 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 01:25:25.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:25.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:25.270 | INFO | Uploaded 36GiB to GPU in 1.99608s (18.0354GiB/s) | 20260507 01:25:25.270 | INFO | Downloaded 34GiB to GPU in 1.97217s (17.2399GiB/s) | 20260507 01:25:25.367 | INFO | Starting a round of Baz IO | 20260507 01:25:28.488 | INFO | Populating RTMetrics report | 20260507 01:25:28.488 | INFO | RTMetrics report populated | 20260507 01:25:28.488 | INFO | Opening tmp RTMetrics json file | 20260507 01:25:28.493 | INFO | Serializing RTMetrics to json | 20260507 01:25:28.507 | INFO | Serialization done, swapping tmp file now... | 20260507 01:25:28.507 | INFO | RTMetrics done | 20260507 01:25:28.582 | INFO | Finished a round of Baz IO: Wrote 1.812 GiB in 3.215 seconds | 20260507 01:25:28.582 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.14% / 0.00% / 66.86% | 20260507 01:25:28.582 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.285482% / 3.97364e-06% / 7.54992e-05% | 20260507 01:25:28.582 | INFO | Baz IO overall: | 20260507 01:25:28.582 | INFO | BytesWritten : 3113394509766 | 20260507 01:25:28.582 | INFO | NumEvents : 1498605482252 | 20260507 01:25:30.413 | INFO | Analyzing chunk frames = [6000640,6001152) | 20260507 01:25:30.413 | INFO | MemInfo: { | 20260507 01:25:30.413 | INFO | "gpu" : " 17 GB", | 20260507 01:25:30.413 | INFO | "huge" : "", | 20260507 01:25:30.413 | INFO | "huge_shared" : " 27 GB", | 20260507 01:25:30.413 | INFO | "malloc" : " 14 GB", | 20260507 01:25:30.413 | INFO | "pinned" : " 29 GB" | 20260507 01:25:30.413 | INFO | } | 20260507 01:25:30.414 | INFO | Perf report for the last 5.14367 seconds: | 20260507 01:25:30.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:25:30.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:30.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:30.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:30.414 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 01:25:30.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:30.414 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:25:30.414 | INFO | PRE_HQ : 59.4%, 1.8, 55.7ms, 0 | 20260507 01:25:30.414 | INFO | BAZWRITER : 62.1%, 3.3, 105.8ms, 0 | 20260507 01:25:30.414 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:25:30.414 | INFO | Uploaded 36GiB to GPU in 2.02981s (17.7357GiB/s) | 20260507 01:25:30.414 | INFO | Downloaded 38GiB to GPU in 2.17461s (17.4744GiB/s) | 20260507 01:25:35.548 | INFO | Analyzing chunk frames = [6001152,6001664) | 20260507 01:25:35.548 | INFO | MemInfo: { | 20260507 01:25:35.548 | INFO | "gpu" : " 17 GB", | 20260507 01:25:35.548 | INFO | "huge" : "", | 20260507 01:25:35.548 | INFO | "huge_shared" : " 27 GB", | 20260507 01:25:35.548 | INFO | "malloc" : " 14 GB", | 20260507 01:25:35.548 | INFO | "pinned" : " 29 GB" | 20260507 01:25:35.548 | INFO | } | 20260507 01:25:35.548 | INFO | Perf report for the last 5.13452 seconds: | 20260507 01:25:35.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:25:35.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:35.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:35.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:35.548 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:25:35.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:35.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:35.548 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:25:35.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:35.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:35.548 | INFO | Uploaded 36GiB to GPU in 2.03903s (17.6554GiB/s) | 20260507 01:25:35.548 | INFO | Downloaded 34GiB to GPU in 1.93449s (17.5757GiB/s) | 20260507 01:25:40.682 | INFO | Analyzing chunk frames = [6001664,6002176) | 20260507 01:25:40.682 | INFO | MemInfo: { | 20260507 01:25:40.682 | INFO | "gpu" : " 17 GB", | 20260507 01:25:40.682 | INFO | "huge" : "", | 20260507 01:25:40.682 | INFO | "huge_shared" : " 27 GB", | 20260507 01:25:40.682 | INFO | "malloc" : " 14 GB", | 20260507 01:25:40.683 | INFO | "pinned" : " 29 GB" | 20260507 01:25:40.683 | INFO | } | 20260507 01:25:40.683 | INFO | Perf report for the last 5.13449 seconds: | 20260507 01:25:40.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:25:40.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:40.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:40.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:40.683 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 01:25:40.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:40.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:40.683 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:25:40.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:40.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:40.683 | INFO | Uploaded 36GiB to GPU in 2.01138s (17.8982GiB/s) | 20260507 01:25:40.683 | INFO | Downloaded 34GiB to GPU in 1.93445s (17.5761GiB/s) | 20260507 01:25:45.819 | INFO | Analyzing chunk frames = [6002176,6002688) | 20260507 01:25:45.819 | INFO | MemInfo: { | 20260507 01:25:45.819 | INFO | "gpu" : " 17 GB", | 20260507 01:25:45.819 | INFO | "huge" : "", | 20260507 01:25:45.819 | INFO | "huge_shared" : " 27 GB", | 20260507 01:25:45.819 | INFO | "malloc" : " 14 GB", | 20260507 01:25:45.820 | INFO | "pinned" : " 29 GB" | 20260507 01:25:45.820 | INFO | } | 20260507 01:25:45.820 | INFO | Perf report for the last 5.13711 seconds: | 20260507 01:25:45.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:25:45.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:45.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:45.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:45.820 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260507 01:25:45.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:45.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:45.820 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:25:45.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:45.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:45.820 | INFO | Uploaded 36GiB to GPU in 2.00987s (17.9116GiB/s) | 20260507 01:25:45.820 | INFO | Downloaded 34GiB to GPU in 1.9094s (17.8066GiB/s) | 20260507 01:25:50.962 | INFO | Analyzing chunk frames = [6002688,6003200) | 20260507 01:25:50.962 | INFO | MemInfo: { | 20260507 01:25:50.962 | INFO | "gpu" : " 17 GB", | 20260507 01:25:50.962 | INFO | "huge" : "", | 20260507 01:25:50.962 | INFO | "huge_shared" : " 27 GB", | 20260507 01:25:50.962 | INFO | "malloc" : " 14 GB", | 20260507 01:25:50.962 | INFO | "pinned" : " 29 GB" | 20260507 01:25:50.962 | INFO | } | 20260507 01:25:50.962 | INFO | Perf report for the last 5.14187 seconds: | 20260507 01:25:50.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:25:50.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:50.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:50.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:50.962 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 01:25:50.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:50.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:50.962 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:25:50.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:50.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:50.962 | INFO | Uploaded 36GiB to GPU in 1.99046s (18.0862GiB/s) | 20260507 01:25:50.962 | INFO | Downloaded 34GiB to GPU in 1.93483s (17.5726GiB/s) | 20260507 01:25:56.104 | INFO | Analyzing chunk frames = [6003200,6003712) | 20260507 01:25:56.104 | INFO | MemInfo: { | 20260507 01:25:56.104 | INFO | "gpu" : " 17 GB", | 20260507 01:25:56.104 | INFO | "huge" : "", | 20260507 01:25:56.104 | INFO | "huge_shared" : " 27 GB", | 20260507 01:25:56.104 | INFO | "malloc" : " 14 GB", | 20260507 01:25:56.104 | INFO | "pinned" : " 29 GB" | 20260507 01:25:56.104 | INFO | } | 20260507 01:25:56.104 | INFO | Perf report for the last 5.14272 seconds: | 20260507 01:25:56.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:25:56.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:56.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:56.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:56.104 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 01:25:56.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:56.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:56.104 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 01:25:56.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:56.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:25:56.105 | INFO | Uploaded 36GiB to GPU in 2.04095s (17.6388GiB/s) | 20260507 01:25:56.105 | INFO | Downloaded 34GiB to GPU in 1.94383s (17.4913GiB/s) | 20260507 01:26:01.243 | INFO | Analyzing chunk frames = [6003712,6004224) | 20260507 01:26:01.243 | INFO | MemInfo: { | 20260507 01:26:01.243 | INFO | "gpu" : " 17 GB", | 20260507 01:26:01.243 | INFO | "huge" : "", | 20260507 01:26:01.243 | INFO | "huge_shared" : " 27 GB", | 20260507 01:26:01.243 | INFO | "malloc" : " 14 GB", | 20260507 01:26:01.243 | INFO | "pinned" : " 29 GB" | 20260507 01:26:01.243 | INFO | } | 20260507 01:26:01.243 | INFO | Perf report for the last 5.1384 seconds: | 20260507 01:26:01.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:26:01.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:01.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:01.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:01.243 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:26:01.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:01.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:01.243 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:26:01.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:01.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:01.243 | INFO | Uploaded 36GiB to GPU in 1.9899s (18.0914GiB/s) | 20260507 01:26:01.243 | INFO | Downloaded 34GiB to GPU in 1.99499s (17.0427GiB/s) | 20260507 01:26:06.380 | INFO | Analyzing chunk frames = [6004224,6004736) | 20260507 01:26:06.380 | INFO | MemInfo: { | 20260507 01:26:06.380 | INFO | "gpu" : " 17 GB", | 20260507 01:26:06.380 | INFO | "huge" : "", | 20260507 01:26:06.380 | INFO | "huge_shared" : " 27 GB", | 20260507 01:26:06.380 | INFO | "malloc" : " 14 GB", | 20260507 01:26:06.380 | INFO | "pinned" : " 29 GB" | 20260507 01:26:06.380 | INFO | } | 20260507 01:26:06.380 | INFO | Perf report for the last 5.13739 seconds: | 20260507 01:26:06.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:26:06.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:06.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:06.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:06.380 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 01:26:06.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:06.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:06.380 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 01:26:06.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:06.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:06.380 | INFO | Uploaded 36GiB to GPU in 2.01528s (17.8635GiB/s) | 20260507 01:26:06.380 | INFO | Downloaded 34GiB to GPU in 1.97807s (17.1884GiB/s) | 20260507 01:26:06.381 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:26:06.381 | INFO | Total mem usage: 18 GiB | 20260507 01:26:06.381 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:26:06.381 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:26:06.381 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:26:06.381 | INFO | High Water Breakdown: | 20260507 01:26:06.381 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:26:06.381 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:26:06.381 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:26:06.381 | INFO | ------------------------------------------ | 20260507 01:26:06.381 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:26:06.381 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:26:06.381 | INFO | Total mem usage: 26.5 GiB | 20260507 01:26:06.381 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:26:06.381 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:26:06.381 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:26:06.381 | INFO | High Water Breakdown: | 20260507 01:26:06.381 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:26:06.381 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:26:06.381 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:26:06.381 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:26:06.381 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:26:06.381 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:26:06.381 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:26:06.381 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:26:06.381 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:26:06.381 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:26:06.381 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:26:06.381 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:26:06.381 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:26:06.381 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:26:06.381 | INFO | ------------------------------------------ | 20260507 01:26:06.381 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:26:06.381 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:26:06.381 | INFO | Total mem usage: 25.6 GiB | 20260507 01:26:06.381 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:26:06.381 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:26:06.381 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:26:06.381 | INFO | High Water Breakdown: | 20260507 01:26:06.381 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:26:06.381 | INFO | ------------------------------------------ | 20260507 01:26:06.381 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:26:06.381 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:26:06.381 | INFO | Total mem usage: 15.7 GiB | 20260507 01:26:06.381 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:26:06.381 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:26:06.381 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:26:06.381 | INFO | High Water Breakdown: | 20260507 01:26:06.381 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:26:06.381 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:26:06.381 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:26:06.381 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:26:06.381 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:26:06.381 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:26:06.381 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:26:06.381 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:26:06.381 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:26:06.381 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:26:06.381 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:26:06.381 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:26:06.381 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:26:06.381 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:26:06.381 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:26:06.381 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:26:06.381 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:26:06.381 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:26:06.381 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:26:06.381 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:26:06.381 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:26:06.381 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:26:06.381 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:26:06.381 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:26:06.381 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:26:06.381 | INFO | ------------------------------------------ | 20260507 01:26:06.381 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:26:06.382 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:26:06.382 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:26:06.382 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:26:06.382 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.58e+01, 1.35e+01 / 1.69e+00 | 20260507 01:26:06.382 | INFO | PerfReport: 0.29%, AggregateStats -- 9.10e-02 / 1.02e-01, 9.70e-03 / 6.59e-03 | 20260507 01:26:06.382 | INFO | PerfReport: 11.65%, Baseline -- 3.60e+00 / 4.46e+00, 5.24e-01 / 3.10e-01 | 20260507 01:26:06.382 | INFO | PerfReport: 3.24%, Histogram -- 9.99e-01 / 1.48e+00, 1.73e-01 / 1.33e-01 | 20260507 01:26:06.382 | INFO | PerfReport: 16.08%, DME -- 3.97e+01 / 4.46e+01, 3.95e+00 / 1.69e+00 | 20260507 01:26:06.382 | INFO | PerfReport: 57.55%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.33e+00 / 1.77e+00 | 20260507 01:26:06.382 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.16e+00 / 2.70e+00, 3.79e-01 / 2.27e-01 | 20260507 01:26:06.382 | INFO | PerfReport: 3.11%, Metrics -- 9.60e-01 / 1.26e+00, 2.56e-01 / 1.41e-01 | 20260507 01:26:06.382 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.26e-01 / 3.77e-01, 5.12e-02 / 2.04e-02 | 20260507 01:26:06.382 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:26:06.382 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:26:06.382 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:26:06.382 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 01:26:06.382 | INFO | PerfReport: 21.73%, Upload -- 2.10e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260507 01:26:06.382 | INFO | PerfReport: 21.34%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 01:26:06.382 | INFO | PerfReport: 31.97%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 01:26:06.382 | INFO | PerfReport: 1.41%, LockUpload -- 1.36e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 01:26:06.382 | INFO | PerfReport: 1.22%, LockDownload -- 1.17e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260507 01:26:06.382 | INFO | PerfReport: 22.31%, LockCompute -- 2.15e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 01:26:06.382 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260507 01:26:06.479 | INFO | Starting a round of Baz IO | 20260507 01:26:09.612 | INFO | Populating RTMetrics report | 20260507 01:26:09.612 | INFO | RTMetrics report populated | 20260507 01:26:09.612 | INFO | Opening tmp RTMetrics json file | 20260507 01:26:09.613 | INFO | Serializing RTMetrics to json | 20260507 01:26:09.614 | INFO | Serialization done, swapping tmp file now... | 20260507 01:26:09.614 | INFO | RTMetrics done | 20260507 01:26:09.704 | INFO | Finished a round of Baz IO: Wrote 1.812 GiB in 3.225 seconds | 20260507 01:26:09.704 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.14% / 0.00% / 66.86% | 20260507 01:26:09.704 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.284696% / 7.94729e-06% / 8.74201e-05% | 20260507 01:26:09.704 | INFO | Baz IO overall: | 20260507 01:26:09.704 | INFO | BytesWritten : 3115339806182 | 20260507 01:26:09.704 | INFO | NumEvents : 1499526883054 | 20260507 01:26:11.519 | INFO | Analyzing chunk frames = [6004736,6005248) | 20260507 01:26:11.519 | INFO | MemInfo: { | 20260507 01:26:11.519 | INFO | "gpu" : " 17 GB", | 20260507 01:26:11.519 | INFO | "huge" : "", | 20260507 01:26:11.519 | INFO | "huge_shared" : " 27 GB", | 20260507 01:26:11.519 | INFO | "malloc" : " 14 GB", | 20260507 01:26:11.519 | INFO | "pinned" : " 29 GB" | 20260507 01:26:11.519 | INFO | } | 20260507 01:26:11.519 | INFO | Perf report for the last 5.13866 seconds: | 20260507 01:26:11.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:26:11.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:11.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:11.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:11.519 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 01:26:11.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:11.519 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260507 01:26:11.519 | INFO | PRE_HQ : 60.6%, 2.0, 61.6ms, 0 | 20260507 01:26:11.519 | INFO | BAZWRITER : 62.3%, 3.5, 111.3ms, 0 | 20260507 01:26:11.519 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 01:26:11.519 | INFO | Uploaded 36GiB to GPU in 2.02744s (17.7564GiB/s) | 20260507 01:26:11.519 | INFO | Downloaded 38GiB to GPU in 2.18189s (17.4161GiB/s) | 20260507 01:26:16.665 | INFO | Analyzing chunk frames = [6005248,6005760) | 20260507 01:26:16.665 | INFO | MemInfo: { | 20260507 01:26:16.665 | INFO | "gpu" : " 17 GB", | 20260507 01:26:16.665 | INFO | "huge" : "", | 20260507 01:26:16.665 | INFO | "huge_shared" : " 27 GB", | 20260507 01:26:16.665 | INFO | "malloc" : " 14 GB", | 20260507 01:26:16.665 | INFO | "pinned" : " 29 GB" | 20260507 01:26:16.665 | INFO | } | 20260507 01:26:16.665 | INFO | Perf report for the last 5.14654 seconds: | 20260507 01:26:16.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:26:16.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:16.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:16.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:16.666 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 01:26:16.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:16.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:16.666 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 01:26:16.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:16.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:16.666 | INFO | Uploaded 36GiB to GPU in 2.00294s (17.9736GiB/s) | 20260507 01:26:16.666 | INFO | Downloaded 34GiB to GPU in 1.92881s (17.6275GiB/s) | 20260507 01:26:21.797 | INFO | Analyzing chunk frames = [6005760,6006272) | 20260507 01:26:21.797 | INFO | MemInfo: { | 20260507 01:26:21.797 | INFO | "gpu" : " 17 GB", | 20260507 01:26:21.797 | INFO | "huge" : "", | 20260507 01:26:21.797 | INFO | "huge_shared" : " 27 GB", | 20260507 01:26:21.797 | INFO | "malloc" : " 14 GB", | 20260507 01:26:21.797 | INFO | "pinned" : " 29 GB" | 20260507 01:26:21.797 | INFO | } | 20260507 01:26:21.797 | INFO | Perf report for the last 5.13175 seconds: | 20260507 01:26:21.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:26:21.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:21.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:21.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:21.797 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 01:26:21.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:21.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:21.797 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:26:21.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:21.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:21.797 | INFO | Uploaded 36GiB to GPU in 2.01456s (17.8699GiB/s) | 20260507 01:26:21.797 | INFO | Downloaded 34GiB to GPU in 1.93734s (17.5498GiB/s) | 20260507 01:26:26.941 | INFO | Analyzing chunk frames = [6006272,6006784) | 20260507 01:26:26.941 | INFO | MemInfo: { | 20260507 01:26:26.941 | INFO | "gpu" : " 17 GB", | 20260507 01:26:26.941 | INFO | "huge" : "", | 20260507 01:26:26.941 | INFO | "huge_shared" : " 27 GB", | 20260507 01:26:26.941 | INFO | "malloc" : " 14 GB", | 20260507 01:26:26.941 | INFO | "pinned" : " 29 GB" | 20260507 01:26:26.941 | INFO | } | 20260507 01:26:26.941 | INFO | Perf report for the last 5.14377 seconds: | 20260507 01:26:26.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:26:26.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:26.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:26.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:26.941 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:26:26.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:26.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:26.941 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:26:26.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:26.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:26.941 | INFO | Uploaded 36GiB to GPU in 1.99419s (18.0524GiB/s) | 20260507 01:26:26.941 | INFO | Downloaded 34GiB to GPU in 1.91143s (17.7877GiB/s) | 20260507 01:26:32.082 | INFO | Analyzing chunk frames = [6006784,6007296) | 20260507 01:26:32.082 | INFO | MemInfo: { | 20260507 01:26:32.082 | INFO | "gpu" : " 17 GB", | 20260507 01:26:32.082 | INFO | "huge" : "", | 20260507 01:26:32.082 | INFO | "huge_shared" : " 27 GB", | 20260507 01:26:32.082 | INFO | "malloc" : " 14 GB", | 20260507 01:26:32.082 | INFO | "pinned" : " 29 GB" | 20260507 01:26:32.082 | INFO | } | 20260507 01:26:32.082 | INFO | Perf report for the last 5.14122 seconds: | 20260507 01:26:32.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:26:32.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:32.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:32.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:32.082 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 01:26:32.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:32.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:32.082 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:26:32.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:32.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:32.082 | INFO | Uploaded 36GiB to GPU in 2.01267s (17.8867GiB/s) | 20260507 01:26:32.082 | INFO | Downloaded 34GiB to GPU in 1.95044s (17.432GiB/s) | 20260507 01:26:37.221 | INFO | Analyzing chunk frames = [6007296,6007808) | 20260507 01:26:37.221 | INFO | MemInfo: { | 20260507 01:26:37.221 | INFO | "gpu" : " 17 GB", | 20260507 01:26:37.221 | INFO | "huge" : "", | 20260507 01:26:37.221 | INFO | "huge_shared" : " 27 GB", | 20260507 01:26:37.221 | INFO | "malloc" : " 14 GB", | 20260507 01:26:37.221 | INFO | "pinned" : " 29 GB" | 20260507 01:26:37.221 | INFO | } | 20260507 01:26:37.221 | INFO | Perf report for the last 5.13926 seconds: | 20260507 01:26:37.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:26:37.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:37.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:37.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:37.222 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 01:26:37.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:37.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:37.222 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:26:37.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:37.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:37.222 | INFO | Uploaded 36GiB to GPU in 2.02927s (17.7404GiB/s) | 20260507 01:26:37.222 | INFO | Downloaded 34GiB to GPU in 1.93589s (17.563GiB/s) | 20260507 01:26:42.360 | INFO | Analyzing chunk frames = [6007808,6008320) | 20260507 01:26:42.360 | INFO | MemInfo: { | 20260507 01:26:42.360 | INFO | "gpu" : " 17 GB", | 20260507 01:26:42.360 | INFO | "huge" : "", | 20260507 01:26:42.360 | INFO | "huge_shared" : " 27 GB", | 20260507 01:26:42.360 | INFO | "malloc" : " 14 GB", | 20260507 01:26:42.360 | INFO | "pinned" : " 29 GB" | 20260507 01:26:42.360 | INFO | } | 20260507 01:26:42.360 | INFO | Perf report for the last 5.13866 seconds: | 20260507 01:26:42.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:26:42.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:42.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:42.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:42.360 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:26:42.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:42.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:42.360 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 01:26:42.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:42.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:42.361 | INFO | Uploaded 36GiB to GPU in 1.98943s (18.0956GiB/s) | 20260507 01:26:42.361 | INFO | Downloaded 34GiB to GPU in 1.98867s (17.0968GiB/s) | 20260507 01:26:47.495 | INFO | Analyzing chunk frames = [6008320,6008832) | 20260507 01:26:47.495 | INFO | MemInfo: { | 20260507 01:26:47.495 | INFO | "gpu" : " 17 GB", | 20260507 01:26:47.495 | INFO | "huge" : "", | 20260507 01:26:47.495 | INFO | "huge_shared" : " 27 GB", | 20260507 01:26:47.495 | INFO | "malloc" : " 14 GB", | 20260507 01:26:47.495 | INFO | "pinned" : " 29 GB" | 20260507 01:26:47.495 | INFO | } | 20260507 01:26:47.495 | INFO | Perf report for the last 5.13451 seconds: | 20260507 01:26:47.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:26:47.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:47.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:47.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:47.495 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 01:26:47.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:47.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:47.495 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 01:26:47.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:47.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:47.495 | INFO | Uploaded 36GiB to GPU in 2.00086s (17.9923GiB/s) | 20260507 01:26:47.495 | INFO | Downloaded 34GiB to GPU in 1.96946s (17.2636GiB/s) | 20260507 01:26:47.593 | INFO | Starting a round of Baz IO | 20260507 01:26:50.706 | INFO | Populating RTMetrics report | 20260507 01:26:50.706 | INFO | RTMetrics report populated | 20260507 01:26:50.706 | INFO | Opening tmp RTMetrics json file | 20260507 01:26:50.707 | INFO | Serializing RTMetrics to json | 20260507 01:26:50.713 | INFO | Serialization done, swapping tmp file now... | 20260507 01:26:50.727 | INFO | RTMetrics done | 20260507 01:26:50.798 | INFO | Finished a round of Baz IO: Wrote 1.811 GiB in 3.204 seconds | 20260507 01:26:50.798 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.14% / 0.00% / 66.86% | 20260507 01:26:50.798 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.284886% / 7.94729e-06% / 8.74201e-05% | 20260507 01:26:50.798 | INFO | Baz IO overall: | 20260507 01:26:50.798 | INFO | BytesWritten : 3117284731524 | 20260507 01:26:50.798 | INFO | NumEvents : 1500448024396 | 20260507 01:26:52.632 | INFO | Analyzing chunk frames = [6008832,6009344) | 20260507 01:26:52.632 | INFO | MemInfo: { | 20260507 01:26:52.632 | INFO | "gpu" : " 17 GB", | 20260507 01:26:52.632 | INFO | "huge" : "", | 20260507 01:26:52.632 | INFO | "huge_shared" : " 27 GB", | 20260507 01:26:52.632 | INFO | "malloc" : " 14 GB", | 20260507 01:26:52.632 | INFO | "pinned" : " 29 GB" | 20260507 01:26:52.632 | INFO | } | 20260507 01:26:52.633 | INFO | Perf report for the last 5.13807 seconds: | 20260507 01:26:52.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:26:52.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:52.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:52.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:52.633 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260507 01:26:52.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:52.633 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 01:26:52.633 | INFO | PRE_HQ : 59.4%, 1.8, 56.0ms, 0 | 20260507 01:26:52.633 | INFO | BAZWRITER : 61.9%, 3.3, 106.4ms, 0 | 20260507 01:26:52.633 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:26:52.633 | INFO | Uploaded 36GiB to GPU in 2.0288s (17.7445GiB/s) | 20260507 01:26:52.633 | INFO | Downloaded 38GiB to GPU in 2.18967s (17.3542GiB/s) | 20260507 01:26:57.775 | INFO | Analyzing chunk frames = [6009344,6009856) | 20260507 01:26:57.775 | INFO | MemInfo: { | 20260507 01:26:57.775 | INFO | "gpu" : " 17 GB", | 20260507 01:26:57.775 | INFO | "huge" : "", | 20260507 01:26:57.775 | INFO | "huge_shared" : " 27 GB", | 20260507 01:26:57.775 | INFO | "malloc" : " 14 GB", | 20260507 01:26:57.775 | INFO | "pinned" : " 29 GB" | 20260507 01:26:57.775 | INFO | } | 20260507 01:26:57.775 | INFO | Perf report for the last 5.14204 seconds: | 20260507 01:26:57.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:26:57.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:57.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:57.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:57.775 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:26:57.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:57.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:57.775 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:26:57.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:57.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:26:57.775 | INFO | Uploaded 36GiB to GPU in 1.99364s (18.0575GiB/s) | 20260507 01:26:57.775 | INFO | Downloaded 34GiB to GPU in 1.93327s (17.5868GiB/s) | 20260507 01:27:02.914 | INFO | Analyzing chunk frames = [6009856,6010368) | 20260507 01:27:02.914 | INFO | MemInfo: { | 20260507 01:27:02.914 | INFO | "gpu" : " 17 GB", | 20260507 01:27:02.914 | INFO | "huge" : "", | 20260507 01:27:02.914 | INFO | "huge_shared" : " 27 GB", | 20260507 01:27:02.914 | INFO | "malloc" : " 14 GB", | 20260507 01:27:02.914 | INFO | "pinned" : " 29 GB" | 20260507 01:27:02.914 | INFO | } | 20260507 01:27:02.914 | INFO | Perf report for the last 5.13905 seconds: | 20260507 01:27:02.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:27:02.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:02.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:02.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:02.914 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 01:27:02.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:02.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:02.914 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 01:27:02.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:02.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:02.914 | INFO | Uploaded 36GiB to GPU in 1.98243s (18.1596GiB/s) | 20260507 01:27:02.914 | INFO | Downloaded 34GiB to GPU in 1.93639s (17.5584GiB/s) | 20260507 01:27:08.054 | INFO | Analyzing chunk frames = [6010368,6010880) | 20260507 01:27:08.054 | INFO | MemInfo: { | 20260507 01:27:08.054 | INFO | "gpu" : " 17 GB", | 20260507 01:27:08.054 | INFO | "huge" : "", | 20260507 01:27:08.054 | INFO | "huge_shared" : " 27 GB", | 20260507 01:27:08.054 | INFO | "malloc" : " 14 GB", | 20260507 01:27:08.054 | INFO | "pinned" : " 29 GB" | 20260507 01:27:08.054 | INFO | } | 20260507 01:27:08.055 | INFO | Perf report for the last 5.14074 seconds: | 20260507 01:27:08.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:27:08.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:08.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:08.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:08.055 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 01:27:08.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:08.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:08.055 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 01:27:08.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:08.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:08.055 | INFO | Uploaded 36GiB to GPU in 1.99467s (18.0481GiB/s) | 20260507 01:27:08.055 | INFO | Downloaded 34GiB to GPU in 1.91089s (17.7928GiB/s) | 20260507 01:27:13.193 | INFO | Analyzing chunk frames = [6010880,6011392) | 20260507 01:27:13.193 | INFO | MemInfo: { | 20260507 01:27:13.193 | INFO | "gpu" : " 17 GB", | 20260507 01:27:13.193 | INFO | "huge" : "", | 20260507 01:27:13.193 | INFO | "huge_shared" : " 27 GB", | 20260507 01:27:13.193 | INFO | "malloc" : " 14 GB", | 20260507 01:27:13.193 | INFO | "pinned" : " 29 GB" | 20260507 01:27:13.193 | INFO | } | 20260507 01:27:13.193 | INFO | Perf report for the last 5.13883 seconds: | 20260507 01:27:13.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:27:13.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:13.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:13.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:13.193 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 01:27:13.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:13.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:13.193 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 01:27:13.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:13.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:13.194 | INFO | Uploaded 36GiB to GPU in 1.9864s (18.1232GiB/s) | 20260507 01:27:13.194 | INFO | Downloaded 34GiB to GPU in 1.94086s (17.518GiB/s) | 20260507 01:27:18.331 | INFO | Analyzing chunk frames = [6011392,6011904) | 20260507 01:27:18.331 | INFO | MemInfo: { | 20260507 01:27:18.331 | INFO | "gpu" : " 17 GB", | 20260507 01:27:18.331 | INFO | "huge" : "", | 20260507 01:27:18.331 | INFO | "huge_shared" : " 27 GB", | 20260507 01:27:18.331 | INFO | "malloc" : " 14 GB", | 20260507 01:27:18.331 | INFO | "pinned" : " 29 GB" | 20260507 01:27:18.331 | INFO | } | 20260507 01:27:18.331 | INFO | Perf report for the last 5.13757 seconds: | 20260507 01:27:18.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:27:18.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:18.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:18.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:18.331 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 01:27:18.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:18.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:18.331 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:27:18.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:18.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:18.331 | INFO | Uploaded 36GiB to GPU in 2.04144s (17.6346GiB/s) | 20260507 01:27:18.331 | INFO | Downloaded 34GiB to GPU in 1.95868s (17.3586GiB/s) | 20260507 01:27:23.472 | INFO | Analyzing chunk frames = [6011904,6012416) | 20260507 01:27:23.472 | INFO | MemInfo: { | 20260507 01:27:23.472 | INFO | "gpu" : " 17 GB", | 20260507 01:27:23.472 | INFO | "huge" : "", | 20260507 01:27:23.472 | INFO | "huge_shared" : " 27 GB", | 20260507 01:27:23.472 | INFO | "malloc" : " 14 GB", | 20260507 01:27:23.472 | INFO | "pinned" : " 29 GB" | 20260507 01:27:23.472 | INFO | } | 20260507 01:27:23.472 | INFO | Perf report for the last 5.14102 seconds: | 20260507 01:27:23.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:27:23.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:23.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:23.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:23.472 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:27:23.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:23.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:23.472 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 01:27:23.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:23.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:23.473 | INFO | Uploaded 36GiB to GPU in 1.98077s (18.1748GiB/s) | 20260507 01:27:23.473 | INFO | Downloaded 34GiB to GPU in 2.00042s (16.9964GiB/s) | 20260507 01:27:28.611 | INFO | Analyzing chunk frames = [6012416,6012928) | 20260507 01:27:28.611 | INFO | MemInfo: { | 20260507 01:27:28.611 | INFO | "gpu" : " 17 GB", | 20260507 01:27:28.611 | INFO | "huge" : "", | 20260507 01:27:28.611 | INFO | "huge_shared" : " 27 GB", | 20260507 01:27:28.611 | INFO | "malloc" : " 14 GB", | 20260507 01:27:28.611 | INFO | "pinned" : " 29 GB" | 20260507 01:27:28.611 | INFO | } | 20260507 01:27:28.611 | INFO | Perf report for the last 5.1389 seconds: | 20260507 01:27:28.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:27:28.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:28.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:28.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:28.611 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:27:28.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:28.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:28.611 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 01:27:28.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:28.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:28.611 | INFO | Uploaded 36GiB to GPU in 1.99619s (18.0343GiB/s) | 20260507 01:27:28.611 | INFO | Downloaded 34GiB to GPU in 1.98283s (17.1472GiB/s) | 20260507 01:27:28.611 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:27:28.611 | INFO | Total mem usage: 18 GiB | 20260507 01:27:28.611 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:27:28.612 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:27:28.612 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:27:28.612 | INFO | High Water Breakdown: | 20260507 01:27:28.612 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:27:28.612 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:27:28.612 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:27:28.612 | INFO | ------------------------------------------ | 20260507 01:27:28.612 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:27:28.612 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:27:28.612 | INFO | Total mem usage: 26.5 GiB | 20260507 01:27:28.612 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:27:28.612 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:27:28.612 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:27:28.612 | INFO | High Water Breakdown: | 20260507 01:27:28.612 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:27:28.612 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:27:28.612 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:27:28.612 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:27:28.612 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:27:28.612 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:27:28.612 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:27:28.612 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:27:28.612 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:27:28.612 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:27:28.612 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:27:28.612 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:27:28.612 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:27:28.612 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:27:28.612 | INFO | ------------------------------------------ | 20260507 01:27:28.612 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:27:28.612 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:27:28.612 | INFO | Total mem usage: 25.6 GiB | 20260507 01:27:28.612 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:27:28.612 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:27:28.612 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:27:28.612 | INFO | High Water Breakdown: | 20260507 01:27:28.612 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:27:28.612 | INFO | ------------------------------------------ | 20260507 01:27:28.612 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:27:28.612 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:27:28.612 | INFO | Total mem usage: 15.7 GiB | 20260507 01:27:28.612 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:27:28.612 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:27:28.612 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:27:28.612 | INFO | High Water Breakdown: | 20260507 01:27:28.612 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:27:28.612 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:27:28.612 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:27:28.612 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:27:28.612 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:27:28.612 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:27:28.612 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:27:28.612 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:27:28.612 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:27:28.612 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:27:28.612 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:27:28.612 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:27:28.612 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:27:28.612 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:27:28.612 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:27:28.612 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:27:28.612 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:27:28.612 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:27:28.612 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:27:28.612 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:27:28.612 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:27:28.612 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:27:28.612 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:27:28.613 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:27:28.613 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:27:28.613 | INFO | ------------------------------------------ | 20260507 01:27:28.613 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:27:28.613 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:27:28.613 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:27:28.613 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:27:28.613 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.55e+01, 1.34e+01 / 1.91e+00 | 20260507 01:27:28.613 | INFO | PerfReport: 0.30%, AggregateStats -- 9.14e-02 / 1.01e-01, 9.93e-03 / 6.09e-03 | 20260507 01:27:28.613 | INFO | PerfReport: 11.69%, Baseline -- 3.61e+00 / 4.33e+00, 5.37e-01 / 3.50e-01 | 20260507 01:27:28.613 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.46e+00, 1.76e-01 / 1.26e-01 | 20260507 01:27:28.613 | INFO | PerfReport: 16.08%, DME -- 3.97e+01 / 4.44e+01, 4.02e+00 / 2.24e+00 | 20260507 01:27:28.613 | INFO | PerfReport: 57.50%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.33e+00 / 1.85e+00 | 20260507 01:27:28.613 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.17e+00 / 2.69e+00, 3.81e-01 / 2.29e-01 | 20260507 01:27:28.613 | INFO | PerfReport: 3.11%, Metrics -- 9.61e-01 / 1.29e+00, 2.56e-01 / 1.49e-01 | 20260507 01:27:28.613 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.26e-01 / 3.73e-01, 5.11e-02 / 2.04e-02 | 20260507 01:27:28.613 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:27:28.613 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:27:28.613 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:27:28.613 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 01:27:28.613 | INFO | PerfReport: 21.65%, Upload -- 2.09e+01 / 0.00e+00, 7.04e+00 / 0.00e+00 | 20260507 01:27:28.613 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260507 01:27:28.613 | INFO | PerfReport: 32.02%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 01:27:28.613 | INFO | PerfReport: 1.33%, LockUpload -- 1.28e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 01:27:28.613 | INFO | PerfReport: 1.33%, LockDownload -- 1.28e+00 / 0.00e+00, 3.17e+00 / 0.00e+00 | 20260507 01:27:28.613 | INFO | PerfReport: 22.28%, LockCompute -- 2.15e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 01:27:28.613 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 01:27:28.709 | INFO | Starting a round of Baz IO | 20260507 01:27:31.828 | INFO | Populating RTMetrics report | 20260507 01:27:31.828 | INFO | RTMetrics report populated | 20260507 01:27:31.828 | INFO | Opening tmp RTMetrics json file | 20260507 01:27:31.846 | INFO | Serializing RTMetrics to json | 20260507 01:27:31.847 | INFO | Serialization done, swapping tmp file now... | 20260507 01:27:31.848 | INFO | RTMetrics done | 20260507 01:27:31.921 | INFO | Finished a round of Baz IO: Wrote 1.813 GiB in 3.212 seconds | 20260507 01:27:31.921 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.13% / 0.00% / 66.86% | 20260507 01:27:31.921 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.283917% / 7.94729e-06% / 6.35783e-05% | 20260507 01:27:31.921 | INFO | Baz IO overall: | 20260507 01:27:31.921 | INFO | BytesWritten : 3119230965624 | 20260507 01:27:31.921 | INFO | NumEvents : 1501370153082 | 20260507 01:27:33.750 | INFO | Analyzing chunk frames = [6012928,6013440) | 20260507 01:27:33.750 | INFO | MemInfo: { | 20260507 01:27:33.750 | INFO | "gpu" : " 17 GB", | 20260507 01:27:33.750 | INFO | "huge" : "", | 20260507 01:27:33.750 | INFO | "huge_shared" : " 27 GB", | 20260507 01:27:33.750 | INFO | "malloc" : " 14 GB", | 20260507 01:27:33.750 | INFO | "pinned" : " 29 GB" | 20260507 01:27:33.750 | INFO | } | 20260507 01:27:33.750 | INFO | Perf report for the last 5.13889 seconds: | 20260507 01:27:33.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:27:33.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:33.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:33.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:33.750 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 01:27:33.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:33.750 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 01:27:33.750 | INFO | PRE_HQ : 59.6%, 1.8, 56.2ms, 0 | 20260507 01:27:33.750 | INFO | BAZWRITER : 62.0%, 3.4, 106.9ms, 0 | 20260507 01:27:33.750 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:27:33.750 | INFO | Uploaded 36GiB to GPU in 2.03028s (17.7316GiB/s) | 20260507 01:27:33.750 | INFO | Downloaded 38GiB to GPU in 2.17911s (17.4383GiB/s) | 20260507 01:27:38.889 | INFO | Analyzing chunk frames = [6013440,6013952) | 20260507 01:27:38.889 | INFO | MemInfo: { | 20260507 01:27:38.889 | INFO | "gpu" : " 17 GB", | 20260507 01:27:38.889 | INFO | "huge" : "", | 20260507 01:27:38.889 | INFO | "huge_shared" : " 27 GB", | 20260507 01:27:38.889 | INFO | "malloc" : " 14 GB", | 20260507 01:27:38.889 | INFO | "pinned" : " 29 GB" | 20260507 01:27:38.890 | INFO | } | 20260507 01:27:38.890 | INFO | Perf report for the last 5.13985 seconds: | 20260507 01:27:38.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:27:38.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:38.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:38.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:38.890 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:27:38.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:38.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:38.890 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:27:38.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:38.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:38.890 | INFO | Uploaded 36GiB to GPU in 2.01584s (17.8586GiB/s) | 20260507 01:27:38.890 | INFO | Downloaded 34GiB to GPU in 1.93877s (17.5369GiB/s) | 20260507 01:27:44.034 | INFO | Analyzing chunk frames = [6013952,6014464) | 20260507 01:27:44.034 | INFO | MemInfo: { | 20260507 01:27:44.034 | INFO | "gpu" : " 17 GB", | 20260507 01:27:44.034 | INFO | "huge" : "", | 20260507 01:27:44.034 | INFO | "huge_shared" : " 27 GB", | 20260507 01:27:44.034 | INFO | "malloc" : " 14 GB", | 20260507 01:27:44.034 | INFO | "pinned" : " 29 GB" | 20260507 01:27:44.034 | INFO | } | 20260507 01:27:44.034 | INFO | Perf report for the last 5.14463 seconds: | 20260507 01:27:44.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:27:44.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:44.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:44.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:44.034 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260507 01:27:44.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:44.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:44.034 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:27:44.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:44.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:44.035 | INFO | Uploaded 36GiB to GPU in 1.99809s (18.0172GiB/s) | 20260507 01:27:44.035 | INFO | Downloaded 34GiB to GPU in 1.93419s (17.5784GiB/s) | 20260507 01:27:49.173 | INFO | Analyzing chunk frames = [6014464,6014976) | 20260507 01:27:49.173 | INFO | MemInfo: { | 20260507 01:27:49.173 | INFO | "gpu" : " 17 GB", | 20260507 01:27:49.173 | INFO | "huge" : "", | 20260507 01:27:49.173 | INFO | "huge_shared" : " 27 GB", | 20260507 01:27:49.173 | INFO | "malloc" : " 14 GB", | 20260507 01:27:49.173 | INFO | "pinned" : " 29 GB" | 20260507 01:27:49.173 | INFO | } | 20260507 01:27:49.174 | INFO | Perf report for the last 5.13917 seconds: | 20260507 01:27:49.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:27:49.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:49.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:49.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:49.174 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:27:49.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:49.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:49.174 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:27:49.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:49.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:49.174 | INFO | Uploaded 36GiB to GPU in 1.96494s (18.3212GiB/s) | 20260507 01:27:49.174 | INFO | Downloaded 34GiB to GPU in 1.91026s (17.7986GiB/s) | 20260507 01:27:54.308 | INFO | Analyzing chunk frames = [6014976,6015488) | 20260507 01:27:54.308 | INFO | MemInfo: { | 20260507 01:27:54.308 | INFO | "gpu" : " 17 GB", | 20260507 01:27:54.308 | INFO | "huge" : "", | 20260507 01:27:54.308 | INFO | "huge_shared" : " 27 GB", | 20260507 01:27:54.308 | INFO | "malloc" : " 14 GB", | 20260507 01:27:54.308 | INFO | "pinned" : " 29 GB" | 20260507 01:27:54.308 | INFO | } | 20260507 01:27:54.308 | INFO | Perf report for the last 5.1349 seconds: | 20260507 01:27:54.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:27:54.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:54.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:54.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:54.308 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 01:27:54.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:54.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:54.309 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:27:54.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:54.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:54.309 | INFO | Uploaded 36GiB to GPU in 1.99459s (18.0488GiB/s) | 20260507 01:27:54.309 | INFO | Downloaded 34GiB to GPU in 1.94989s (17.4369GiB/s) | 20260507 01:27:59.447 | INFO | Analyzing chunk frames = [6015488,6016000) | 20260507 01:27:59.447 | INFO | MemInfo: { | 20260507 01:27:59.447 | INFO | "gpu" : " 17 GB", | 20260507 01:27:59.447 | INFO | "huge" : "", | 20260507 01:27:59.447 | INFO | "huge_shared" : " 27 GB", | 20260507 01:27:59.447 | INFO | "malloc" : " 14 GB", | 20260507 01:27:59.447 | INFO | "pinned" : " 29 GB" | 20260507 01:27:59.447 | INFO | } | 20260507 01:27:59.447 | INFO | Perf report for the last 5.13875 seconds: | 20260507 01:27:59.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:27:59.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:59.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:59.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:59.447 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 01:27:59.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:59.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:59.447 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:27:59.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:59.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:27:59.447 | INFO | Uploaded 36GiB to GPU in 2.04831s (17.5755GiB/s) | 20260507 01:27:59.447 | INFO | Downloaded 34GiB to GPU in 1.95637s (17.3791GiB/s) | 20260507 01:28:04.581 | INFO | Analyzing chunk frames = [6016000,6016512) | 20260507 01:28:04.581 | INFO | MemInfo: { | 20260507 01:28:04.581 | INFO | "gpu" : " 17 GB", | 20260507 01:28:04.581 | INFO | "huge" : "", | 20260507 01:28:04.581 | INFO | "huge_shared" : " 27 GB", | 20260507 01:28:04.581 | INFO | "malloc" : " 14 GB", | 20260507 01:28:04.581 | INFO | "pinned" : " 29 GB" | 20260507 01:28:04.581 | INFO | } | 20260507 01:28:04.581 | INFO | Perf report for the last 5.13418 seconds: | 20260507 01:28:04.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:28:04.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:04.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:04.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:04.581 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260507 01:28:04.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:04.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:04.581 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 01:28:04.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:04.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:04.582 | INFO | Uploaded 36GiB to GPU in 1.98922s (18.0975GiB/s) | 20260507 01:28:04.582 | INFO | Downloaded 34GiB to GPU in 1.98757s (17.1063GiB/s) | 20260507 01:28:09.725 | INFO | Analyzing chunk frames = [6016512,6017024) | 20260507 01:28:09.726 | INFO | MemInfo: { | 20260507 01:28:09.726 | INFO | "gpu" : " 17 GB", | 20260507 01:28:09.726 | INFO | "huge" : "", | 20260507 01:28:09.726 | INFO | "huge_shared" : " 27 GB", | 20260507 01:28:09.726 | INFO | "malloc" : " 14 GB", | 20260507 01:28:09.726 | INFO | "pinned" : " 29 GB" | 20260507 01:28:09.726 | INFO | } | 20260507 01:28:09.726 | INFO | Perf report for the last 5.14434 seconds: | 20260507 01:28:09.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:28:09.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:09.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:09.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:09.726 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:28:09.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:09.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:09.726 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 01:28:09.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:09.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:09.726 | INFO | Uploaded 36GiB to GPU in 1.98572s (18.1294GiB/s) | 20260507 01:28:09.726 | INFO | Downloaded 34GiB to GPU in 1.9761s (17.2056GiB/s) | 20260507 01:28:09.822 | INFO | Starting a round of Baz IO | 20260507 01:28:12.947 | INFO | Populating RTMetrics report | 20260507 01:28:12.947 | INFO | RTMetrics report populated | 20260507 01:28:12.947 | INFO | Opening tmp RTMetrics json file | 20260507 01:28:12.951 | INFO | Serializing RTMetrics to json | 20260507 01:28:12.966 | INFO | Serialization done, swapping tmp file now... | 20260507 01:28:12.966 | INFO | RTMetrics done | 20260507 01:28:13.036 | INFO | Finished a round of Baz IO: Wrote 1.814 GiB in 3.213 seconds | 20260507 01:28:13.036 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.13% / 0.00% / 66.87% | 20260507 01:28:13.036 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.283523% / 3.97364e-06% / 5.96046e-05% | 20260507 01:28:13.036 | INFO | Baz IO overall: | 20260507 01:28:13.036 | INFO | BytesWritten : 3121178673862 | 20260507 01:28:13.036 | INFO | NumEvents : 1502293360721 | 20260507 01:28:14.871 | INFO | Analyzing chunk frames = [6017024,6017536) | 20260507 01:28:14.871 | INFO | MemInfo: { | 20260507 01:28:14.871 | INFO | "gpu" : " 17 GB", | 20260507 01:28:14.871 | INFO | "huge" : "", | 20260507 01:28:14.871 | INFO | "huge_shared" : " 27 GB", | 20260507 01:28:14.871 | INFO | "malloc" : " 14 GB", | 20260507 01:28:14.871 | INFO | "pinned" : " 29 GB" | 20260507 01:28:14.871 | INFO | } | 20260507 01:28:14.871 | INFO | Perf report for the last 5.14549 seconds: | 20260507 01:28:14.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:28:14.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:14.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:14.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:14.871 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260507 01:28:14.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:14.871 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 01:28:14.871 | INFO | PRE_HQ : 59.4%, 1.8, 56.0ms, 0 | 20260507 01:28:14.871 | INFO | BAZWRITER : 61.9%, 3.3, 106.2ms, 0 | 20260507 01:28:14.871 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:28:14.871 | INFO | Uploaded 36GiB to GPU in 2.03222s (17.7146GiB/s) | 20260507 01:28:14.871 | INFO | Downloaded 38GiB to GPU in 2.18022s (17.4294GiB/s) | 20260507 01:28:20.004 | INFO | Analyzing chunk frames = [6017536,6018048) | 20260507 01:28:20.004 | INFO | MemInfo: { | 20260507 01:28:20.004 | INFO | "gpu" : " 17 GB", | 20260507 01:28:20.004 | INFO | "huge" : "", | 20260507 01:28:20.004 | INFO | "huge_shared" : " 27 GB", | 20260507 01:28:20.004 | INFO | "malloc" : " 14 GB", | 20260507 01:28:20.004 | INFO | "pinned" : " 29 GB" | 20260507 01:28:20.004 | INFO | } | 20260507 01:28:20.004 | INFO | Perf report for the last 5.1329 seconds: | 20260507 01:28:20.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:28:20.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:20.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:20.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:20.004 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:28:20.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:20.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:20.004 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:28:20.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:20.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:20.004 | INFO | Uploaded 36GiB to GPU in 2.02351s (17.7909GiB/s) | 20260507 01:28:20.004 | INFO | Downloaded 34GiB to GPU in 1.93081s (17.6092GiB/s) | 20260507 01:28:25.146 | INFO | Analyzing chunk frames = [6018048,6018560) | 20260507 01:28:25.146 | INFO | MemInfo: { | 20260507 01:28:25.146 | INFO | "gpu" : " 17 GB", | 20260507 01:28:25.146 | INFO | "huge" : "", | 20260507 01:28:25.146 | INFO | "huge_shared" : " 27 GB", | 20260507 01:28:25.146 | INFO | "malloc" : " 14 GB", | 20260507 01:28:25.146 | INFO | "pinned" : " 29 GB" | 20260507 01:28:25.146 | INFO | } | 20260507 01:28:25.146 | INFO | Perf report for the last 5.14189 seconds: | 20260507 01:28:25.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:28:25.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:25.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:25.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:25.146 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260507 01:28:25.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:25.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:25.146 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:28:25.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:25.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:25.146 | INFO | Uploaded 36GiB to GPU in 1.97791s (18.2011GiB/s) | 20260507 01:28:25.146 | INFO | Downloaded 34GiB to GPU in 1.91974s (17.7108GiB/s) | 20260507 01:28:30.288 | INFO | Analyzing chunk frames = [6018560,6019072) | 20260507 01:28:30.288 | INFO | MemInfo: { | 20260507 01:28:30.288 | INFO | "gpu" : " 17 GB", | 20260507 01:28:30.288 | INFO | "huge" : "", | 20260507 01:28:30.288 | INFO | "huge_shared" : " 27 GB", | 20260507 01:28:30.288 | INFO | "malloc" : " 14 GB", | 20260507 01:28:30.288 | INFO | "pinned" : " 29 GB" | 20260507 01:28:30.288 | INFO | } | 20260507 01:28:30.288 | INFO | Perf report for the last 5.14241 seconds: | 20260507 01:28:30.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:28:30.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:30.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:30.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:30.289 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:28:30.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:30.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:30.289 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:28:30.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:30.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:30.289 | INFO | Uploaded 36GiB to GPU in 1.99574s (18.0384GiB/s) | 20260507 01:28:30.289 | INFO | Downloaded 34GiB to GPU in 1.91048s (17.7966GiB/s) | 20260507 01:28:35.424 | INFO | Analyzing chunk frames = [6019072,6019584) | 20260507 01:28:35.424 | INFO | MemInfo: { | 20260507 01:28:35.424 | INFO | "gpu" : " 17 GB", | 20260507 01:28:35.424 | INFO | "huge" : "", | 20260507 01:28:35.424 | INFO | "huge_shared" : " 27 GB", | 20260507 01:28:35.424 | INFO | "malloc" : " 14 GB", | 20260507 01:28:35.424 | INFO | "pinned" : " 29 GB" | 20260507 01:28:35.424 | INFO | } | 20260507 01:28:35.424 | INFO | Perf report for the last 5.13573 seconds: | 20260507 01:28:35.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:28:35.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:35.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:35.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:35.424 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 01:28:35.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:35.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:35.424 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:28:35.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:35.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:35.425 | INFO | Uploaded 36GiB to GPU in 2.02444s (17.7827GiB/s) | 20260507 01:28:35.425 | INFO | Downloaded 34GiB to GPU in 1.94507s (17.4801GiB/s) | 20260507 01:28:40.561 | INFO | Analyzing chunk frames = [6019584,6020096) | 20260507 01:28:40.561 | INFO | MemInfo: { | 20260507 01:28:40.561 | INFO | "gpu" : " 17 GB", | 20260507 01:28:40.561 | INFO | "huge" : "", | 20260507 01:28:40.561 | INFO | "huge_shared" : " 27 GB", | 20260507 01:28:40.561 | INFO | "malloc" : " 14 GB", | 20260507 01:28:40.561 | INFO | "pinned" : " 29 GB" | 20260507 01:28:40.561 | INFO | } | 20260507 01:28:40.561 | INFO | Perf report for the last 5.13693 seconds: | 20260507 01:28:40.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:28:40.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:40.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:40.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:40.561 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:28:40.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:40.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:40.561 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 01:28:40.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:40.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:40.561 | INFO | Uploaded 36GiB to GPU in 2.06037s (17.4726GiB/s) | 20260507 01:28:40.561 | INFO | Downloaded 34GiB to GPU in 1.96108s (17.3374GiB/s) | 20260507 01:28:45.702 | INFO | Analyzing chunk frames = [6020096,6020608) | 20260507 01:28:45.702 | INFO | MemInfo: { | 20260507 01:28:45.702 | INFO | "gpu" : " 17 GB", | 20260507 01:28:45.702 | INFO | "huge" : "", | 20260507 01:28:45.702 | INFO | "huge_shared" : " 27 GB", | 20260507 01:28:45.702 | INFO | "malloc" : " 14 GB", | 20260507 01:28:45.702 | INFO | "pinned" : " 29 GB" | 20260507 01:28:45.702 | INFO | } | 20260507 01:28:45.703 | INFO | Perf report for the last 5.14134 seconds: | 20260507 01:28:45.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:28:45.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:45.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:45.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:45.703 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:28:45.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:45.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:45.703 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 01:28:45.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:45.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:45.703 | INFO | Uploaded 36GiB to GPU in 2.00086s (17.9923GiB/s) | 20260507 01:28:45.703 | INFO | Downloaded 34GiB to GPU in 1.97546s (17.2112GiB/s) | 20260507 01:28:50.840 | INFO | Analyzing chunk frames = [6020608,6021120) | 20260507 01:28:50.840 | INFO | MemInfo: { | 20260507 01:28:50.840 | INFO | "gpu" : " 17 GB", | 20260507 01:28:50.840 | INFO | "huge" : "", | 20260507 01:28:50.840 | INFO | "huge_shared" : " 27 GB", | 20260507 01:28:50.840 | INFO | "malloc" : " 14 GB", | 20260507 01:28:50.840 | INFO | "pinned" : " 29 GB" | 20260507 01:28:50.840 | INFO | } | 20260507 01:28:50.840 | INFO | Perf report for the last 5.13764 seconds: | 20260507 01:28:50.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:28:50.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:50.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:50.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:50.840 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:28:50.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:50.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:50.840 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 01:28:50.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:50.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:50.840 | INFO | Uploaded 36GiB to GPU in 1.99174s (18.0746GiB/s) | 20260507 01:28:50.840 | INFO | Downloaded 34GiB to GPU in 1.97407s (17.2233GiB/s) | 20260507 01:28:50.840 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:28:50.840 | INFO | Total mem usage: 18 GiB | 20260507 01:28:50.840 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:28:50.840 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:28:50.841 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:28:50.841 | INFO | High Water Breakdown: | 20260507 01:28:50.841 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:28:50.841 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:28:50.841 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:28:50.841 | INFO | ------------------------------------------ | 20260507 01:28:50.841 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:28:50.841 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:28:50.841 | INFO | Total mem usage: 26.5 GiB | 20260507 01:28:50.841 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:28:50.841 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:28:50.841 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:28:50.841 | INFO | High Water Breakdown: | 20260507 01:28:50.841 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:28:50.841 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:28:50.841 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:28:50.841 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:28:50.841 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:28:50.841 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:28:50.841 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:28:50.841 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:28:50.841 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:28:50.841 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:28:50.841 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:28:50.841 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:28:50.841 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:28:50.841 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:28:50.841 | INFO | ------------------------------------------ | 20260507 01:28:50.841 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:28:50.841 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:28:50.841 | INFO | Total mem usage: 25.6 GiB | 20260507 01:28:50.841 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:28:50.841 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:28:50.841 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:28:50.841 | INFO | High Water Breakdown: | 20260507 01:28:50.841 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:28:50.841 | INFO | ------------------------------------------ | 20260507 01:28:50.841 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:28:50.841 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:28:50.841 | INFO | Total mem usage: 15.7 GiB | 20260507 01:28:50.841 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:28:50.841 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:28:50.841 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:28:50.841 | INFO | High Water Breakdown: | 20260507 01:28:50.841 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:28:50.841 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:28:50.841 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:28:50.841 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:28:50.841 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:28:50.841 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:28:50.841 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:28:50.841 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:28:50.841 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:28:50.841 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:28:50.841 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:28:50.841 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:28:50.841 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:28:50.841 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:28:50.841 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:28:50.841 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:28:50.841 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:28:50.841 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:28:50.841 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:28:50.841 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:28:50.841 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:28:50.841 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:28:50.841 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:28:50.841 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:28:50.841 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:28:50.841 | INFO | ------------------------------------------ | 20260507 01:28:50.841 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:28:50.841 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:28:50.841 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:28:50.841 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:28:50.842 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.59e+01, 1.36e+01 / 2.04e+00 | 20260507 01:28:50.842 | INFO | PerfReport: 0.30%, AggregateStats -- 9.12e-02 / 1.03e-01, 1.00e-02 / 6.77e-03 | 20260507 01:28:50.842 | INFO | PerfReport: 11.65%, Baseline -- 3.60e+00 / 4.51e+00, 5.16e-01 / 1.87e-01 | 20260507 01:28:50.842 | INFO | PerfReport: 3.22%, Histogram -- 9.94e-01 / 1.48e+00, 1.68e-01 / 9.23e-02 | 20260507 01:28:50.842 | INFO | PerfReport: 16.14%, DME -- 3.98e+01 / 4.51e+01, 4.16e+00 / 2.65e+00 | 20260507 01:28:50.842 | INFO | PerfReport: 57.50%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.31e+00 / 1.76e+00 | 20260507 01:28:50.842 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.17e+00 / 2.74e+00, 3.81e-01 / 2.07e-01 | 20260507 01:28:50.842 | INFO | PerfReport: 3.11%, Metrics -- 9.60e-01 / 1.30e+00, 2.57e-01 / 1.93e-01 | 20260507 01:28:50.842 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.26e-01 / 3.86e-01, 5.24e-02 / 2.81e-02 | 20260507 01:28:50.842 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:28:50.842 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:28:50.842 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:28:50.842 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 01:28:50.842 | INFO | PerfReport: 21.67%, Upload -- 2.09e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 01:28:50.842 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260507 01:28:50.842 | INFO | PerfReport: 31.96%, Compute -- 3.09e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260507 01:28:50.842 | INFO | PerfReport: 1.41%, LockUpload -- 1.36e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 01:28:50.842 | INFO | PerfReport: 1.28%, LockDownload -- 1.24e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260507 01:28:50.842 | INFO | PerfReport: 22.33%, LockCompute -- 2.16e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 01:28:50.842 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.64e-02 / 0.00e+00 | 20260507 01:28:50.938 | INFO | Starting a round of Baz IO | 20260507 01:28:54.059 | INFO | Populating RTMetrics report | 20260507 01:28:54.059 | INFO | RTMetrics report populated | 20260507 01:28:54.059 | INFO | Opening tmp RTMetrics json file | 20260507 01:28:54.060 | INFO | Serializing RTMetrics to json | 20260507 01:28:54.070 | INFO | Serialization done, swapping tmp file now... | 20260507 01:28:54.080 | INFO | RTMetrics done | 20260507 01:28:54.151 | INFO | Finished a round of Baz IO: Wrote 1.815 GiB in 3.213 seconds | 20260507 01:28:54.151 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.13% / 0.00% / 66.87% | 20260507 01:28:54.151 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.284878% / 3.97364e-06% / 6.75519e-05% | 20260507 01:28:54.151 | INFO | Baz IO overall: | 20260507 01:28:54.151 | INFO | BytesWritten : 3123127750114 | 20260507 01:28:54.151 | INFO | NumEvents : 1503217648051 | 20260507 01:28:55.981 | INFO | Analyzing chunk frames = [6021120,6021632) | 20260507 01:28:55.981 | INFO | MemInfo: { | 20260507 01:28:55.981 | INFO | "gpu" : " 17 GB", | 20260507 01:28:55.981 | INFO | "huge" : "", | 20260507 01:28:55.981 | INFO | "huge_shared" : " 27 GB", | 20260507 01:28:55.981 | INFO | "malloc" : " 14 GB", | 20260507 01:28:55.981 | INFO | "pinned" : " 29 GB" | 20260507 01:28:55.981 | INFO | } | 20260507 01:28:55.981 | INFO | Perf report for the last 5.14078 seconds: | 20260507 01:28:55.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:28:55.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:55.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:55.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:55.981 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260507 01:28:55.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:28:55.981 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 01:28:55.981 | INFO | PRE_HQ : 59.7%, 1.8, 56.3ms, 0 | 20260507 01:28:55.981 | INFO | BAZWRITER : 62.0%, 3.4, 107.0ms, 0 | 20260507 01:28:55.981 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:28:55.981 | INFO | Uploaded 36GiB to GPU in 2.02577s (17.7711GiB/s) | 20260507 01:28:55.981 | INFO | Downloaded 38GiB to GPU in 2.17911s (17.4383GiB/s) | 20260507 01:29:01.128 | INFO | Analyzing chunk frames = [6021632,6022144) | 20260507 01:29:01.128 | INFO | MemInfo: { | 20260507 01:29:01.128 | INFO | "gpu" : " 17 GB", | 20260507 01:29:01.128 | INFO | "huge" : "", | 20260507 01:29:01.128 | INFO | "huge_shared" : " 27 GB", | 20260507 01:29:01.128 | INFO | "malloc" : " 14 GB", | 20260507 01:29:01.128 | INFO | "pinned" : " 29 GB" | 20260507 01:29:01.128 | INFO | } | 20260507 01:29:01.128 | INFO | Perf report for the last 5.1473 seconds: | 20260507 01:29:01.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:29:01.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:01.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:01.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:01.128 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 01:29:01.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:01.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:01.128 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 01:29:01.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:01.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:01.129 | INFO | Uploaded 36GiB to GPU in 1.99468s (18.048GiB/s) | 20260507 01:29:01.129 | INFO | Downloaded 34GiB to GPU in 1.92976s (17.6188GiB/s) | 20260507 01:29:06.261 | INFO | Analyzing chunk frames = [6022144,6022656) | 20260507 01:29:06.261 | INFO | MemInfo: { | 20260507 01:29:06.261 | INFO | "gpu" : " 17 GB", | 20260507 01:29:06.261 | INFO | "huge" : "", | 20260507 01:29:06.261 | INFO | "huge_shared" : " 27 GB", | 20260507 01:29:06.261 | INFO | "malloc" : " 14 GB", | 20260507 01:29:06.261 | INFO | "pinned" : " 29 GB" | 20260507 01:29:06.261 | INFO | } | 20260507 01:29:06.261 | INFO | Perf report for the last 5.13288 seconds: | 20260507 01:29:06.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:29:06.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:06.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:06.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:06.261 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 01:29:06.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:06.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:06.261 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:29:06.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:06.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:06.262 | INFO | Uploaded 36GiB to GPU in 1.98526s (18.1337GiB/s) | 20260507 01:29:06.262 | INFO | Downloaded 34GiB to GPU in 1.94404s (17.4894GiB/s) | 20260507 01:29:11.399 | INFO | Analyzing chunk frames = [6022656,6023168) | 20260507 01:29:11.399 | INFO | MemInfo: { | 20260507 01:29:11.399 | INFO | "gpu" : " 17 GB", | 20260507 01:29:11.399 | INFO | "huge" : "", | 20260507 01:29:11.399 | INFO | "huge_shared" : " 27 GB", | 20260507 01:29:11.399 | INFO | "malloc" : " 14 GB", | 20260507 01:29:11.399 | INFO | "pinned" : " 29 GB" | 20260507 01:29:11.400 | INFO | } | 20260507 01:29:11.400 | INFO | Perf report for the last 5.13845 seconds: | 20260507 01:29:11.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:29:11.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:11.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:11.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:11.400 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 01:29:11.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:11.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:11.400 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:29:11.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:11.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:11.400 | INFO | Uploaded 36GiB to GPU in 1.97327s (18.2438GiB/s) | 20260507 01:29:11.400 | INFO | Downloaded 34GiB to GPU in 1.92174s (17.6923GiB/s) | 20260507 01:29:16.541 | INFO | Analyzing chunk frames = [6023168,6023680) | 20260507 01:29:16.541 | INFO | MemInfo: { | 20260507 01:29:16.541 | INFO | "gpu" : " 17 GB", | 20260507 01:29:16.541 | INFO | "huge" : "", | 20260507 01:29:16.541 | INFO | "huge_shared" : " 27 GB", | 20260507 01:29:16.541 | INFO | "malloc" : " 14 GB", | 20260507 01:29:16.541 | INFO | "pinned" : " 29 GB" | 20260507 01:29:16.541 | INFO | } | 20260507 01:29:16.541 | INFO | Perf report for the last 5.14158 seconds: | 20260507 01:29:16.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:29:16.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:16.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:16.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:16.541 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 01:29:16.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:16.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:16.541 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 01:29:16.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:16.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:16.541 | INFO | Uploaded 36GiB to GPU in 1.98409s (18.1443GiB/s) | 20260507 01:29:16.542 | INFO | Downloaded 34GiB to GPU in 1.94819s (17.4521GiB/s) | 20260507 01:29:21.676 | INFO | Analyzing chunk frames = [6023680,6024192) | 20260507 01:29:21.676 | INFO | MemInfo: { | 20260507 01:29:21.676 | INFO | "gpu" : " 17 GB", | 20260507 01:29:21.676 | INFO | "huge" : "", | 20260507 01:29:21.676 | INFO | "huge_shared" : " 27 GB", | 20260507 01:29:21.676 | INFO | "malloc" : " 14 GB", | 20260507 01:29:21.676 | INFO | "pinned" : " 29 GB" | 20260507 01:29:21.676 | INFO | } | 20260507 01:29:21.676 | INFO | Perf report for the last 5.1349 seconds: | 20260507 01:29:21.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:29:21.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:21.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:21.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:21.676 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 01:29:21.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:21.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:21.676 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:29:21.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:21.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:21.677 | INFO | Uploaded 36GiB to GPU in 2.01845s (17.8355GiB/s) | 20260507 01:29:21.677 | INFO | Downloaded 34GiB to GPU in 1.95323s (17.4071GiB/s) | 20260507 01:29:26.820 | INFO | Analyzing chunk frames = [6024192,6024704) | 20260507 01:29:26.820 | INFO | MemInfo: { | 20260507 01:29:26.821 | INFO | "gpu" : " 17 GB", | 20260507 01:29:26.821 | INFO | "huge" : "", | 20260507 01:29:26.821 | INFO | "huge_shared" : " 27 GB", | 20260507 01:29:26.821 | INFO | "malloc" : " 14 GB", | 20260507 01:29:26.821 | INFO | "pinned" : " 29 GB" | 20260507 01:29:26.821 | INFO | } | 20260507 01:29:26.821 | INFO | Perf report for the last 5.1445 seconds: | 20260507 01:29:26.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:29:26.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:26.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:26.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:26.821 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:29:26.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:26.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:26.821 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 01:29:26.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:26.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:26.821 | INFO | Uploaded 36GiB to GPU in 1.99204s (18.0719GiB/s) | 20260507 01:29:26.821 | INFO | Downloaded 34GiB to GPU in 1.98949s (17.0898GiB/s) | 20260507 01:29:31.957 | INFO | Analyzing chunk frames = [6024704,6025216) | 20260507 01:29:31.957 | INFO | MemInfo: { | 20260507 01:29:31.957 | INFO | "gpu" : " 17 GB", | 20260507 01:29:31.957 | INFO | "huge" : "", | 20260507 01:29:31.957 | INFO | "huge_shared" : " 27 GB", | 20260507 01:29:31.957 | INFO | "malloc" : " 14 GB", | 20260507 01:29:31.957 | INFO | "pinned" : " 29 GB" | 20260507 01:29:31.957 | INFO | } | 20260507 01:29:31.958 | INFO | Perf report for the last 5.1369 seconds: | 20260507 01:29:31.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:29:31.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:31.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:31.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:31.958 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:29:31.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:31.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:31.958 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 01:29:31.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:31.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:31.958 | INFO | Uploaded 36GiB to GPU in 1.99664s (18.0303GiB/s) | 20260507 01:29:31.958 | INFO | Downloaded 34GiB to GPU in 1.98226s (17.1521GiB/s) | 20260507 01:29:32.057 | INFO | Starting a round of Baz IO | 20260507 01:29:35.176 | INFO | Populating RTMetrics report | 20260507 01:29:35.177 | INFO | RTMetrics report populated | 20260507 01:29:35.177 | INFO | Opening tmp RTMetrics json file | 20260507 01:29:35.178 | INFO | Serializing RTMetrics to json | 20260507 01:29:35.196 | INFO | Serialization done, swapping tmp file now... | 20260507 01:29:35.197 | INFO | RTMetrics done | 20260507 01:29:35.265 | INFO | Finished a round of Baz IO: Wrote 1.814 GiB in 3.207 seconds | 20260507 01:29:35.265 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.13% / 0.00% / 66.87% | 20260507 01:29:35.265 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.283949% / 0% / 7.94729e-05% | 20260507 01:29:35.265 | INFO | Baz IO overall: | 20260507 01:29:35.265 | INFO | BytesWritten : 3125076041033 | 20260507 01:29:35.265 | INFO | NumEvents : 1504141169464 | 20260507 01:29:37.097 | INFO | Analyzing chunk frames = [6025216,6025728) | 20260507 01:29:37.097 | INFO | MemInfo: { | 20260507 01:29:37.097 | INFO | "gpu" : " 17 GB", | 20260507 01:29:37.097 | INFO | "huge" : "", | 20260507 01:29:37.097 | INFO | "huge_shared" : " 27 GB", | 20260507 01:29:37.097 | INFO | "malloc" : " 14 GB", | 20260507 01:29:37.097 | INFO | "pinned" : " 29 GB" | 20260507 01:29:37.097 | INFO | } | 20260507 01:29:37.097 | INFO | Perf report for the last 5.13956 seconds: | 20260507 01:29:37.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:29:37.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:37.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:37.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:37.097 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 01:29:37.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:37.097 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 01:29:37.097 | INFO | PRE_HQ : 59.6%, 1.8, 55.8ms, 0 | 20260507 01:29:37.097 | INFO | BAZWRITER : 62.0%, 3.3, 105.8ms, 0 | 20260507 01:29:37.097 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:29:37.097 | INFO | Uploaded 36GiB to GPU in 2.02877s (17.7447GiB/s) | 20260507 01:29:37.098 | INFO | Downloaded 38GiB to GPU in 2.17207s (17.4949GiB/s) | 20260507 01:29:42.234 | INFO | Analyzing chunk frames = [6025728,6026240) | 20260507 01:29:42.234 | INFO | MemInfo: { | 20260507 01:29:42.234 | INFO | "gpu" : " 17 GB", | 20260507 01:29:42.234 | INFO | "huge" : "", | 20260507 01:29:42.234 | INFO | "huge_shared" : " 27 GB", | 20260507 01:29:42.234 | INFO | "malloc" : " 14 GB", | 20260507 01:29:42.234 | INFO | "pinned" : " 29 GB" | 20260507 01:29:42.234 | INFO | } | 20260507 01:29:42.234 | INFO | Perf report for the last 5.13668 seconds: | 20260507 01:29:42.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:29:42.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:42.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:42.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:42.234 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 01:29:42.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:42.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:42.234 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:29:42.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:42.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:42.234 | INFO | Uploaded 36GiB to GPU in 1.98426s (18.1428GiB/s) | 20260507 01:29:42.234 | INFO | Downloaded 34GiB to GPU in 1.9306s (17.6111GiB/s) | 20260507 01:29:47.374 | INFO | Analyzing chunk frames = [6026240,6026752) | 20260507 01:29:47.374 | INFO | MemInfo: { | 20260507 01:29:47.374 | INFO | "gpu" : " 17 GB", | 20260507 01:29:47.374 | INFO | "huge" : "", | 20260507 01:29:47.374 | INFO | "huge_shared" : " 27 GB", | 20260507 01:29:47.374 | INFO | "malloc" : " 14 GB", | 20260507 01:29:47.374 | INFO | "pinned" : " 29 GB" | 20260507 01:29:47.374 | INFO | } | 20260507 01:29:47.374 | INFO | Perf report for the last 5.14041 seconds: | 20260507 01:29:47.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:29:47.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:47.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:47.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:47.374 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:29:47.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:47.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:47.375 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:29:47.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:47.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:47.375 | INFO | Uploaded 36GiB to GPU in 1.99198s (18.0724GiB/s) | 20260507 01:29:47.375 | INFO | Downloaded 34GiB to GPU in 1.93933s (17.5318GiB/s) | 20260507 01:29:52.510 | INFO | Analyzing chunk frames = [6026752,6027264) | 20260507 01:29:52.510 | INFO | MemInfo: { | 20260507 01:29:52.510 | INFO | "gpu" : " 17 GB", | 20260507 01:29:52.510 | INFO | "huge" : "", | 20260507 01:29:52.510 | INFO | "huge_shared" : " 27 GB", | 20260507 01:29:52.510 | INFO | "malloc" : " 14 GB", | 20260507 01:29:52.510 | INFO | "pinned" : " 29 GB" | 20260507 01:29:52.510 | INFO | } | 20260507 01:29:52.511 | INFO | Perf report for the last 5.13629 seconds: | 20260507 01:29:52.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:29:52.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:52.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:52.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:52.511 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:29:52.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:52.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:52.511 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:29:52.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:52.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:52.511 | INFO | Uploaded 36GiB to GPU in 1.96622s (18.3092GiB/s) | 20260507 01:29:52.511 | INFO | Downloaded 34GiB to GPU in 1.91101s (17.7916GiB/s) | 20260507 01:29:57.647 | INFO | Analyzing chunk frames = [6027264,6027776) | 20260507 01:29:57.647 | INFO | MemInfo: { | 20260507 01:29:57.647 | INFO | "gpu" : " 17 GB", | 20260507 01:29:57.647 | INFO | "huge" : "", | 20260507 01:29:57.647 | INFO | "huge_shared" : " 27 GB", | 20260507 01:29:57.647 | INFO | "malloc" : " 14 GB", | 20260507 01:29:57.647 | INFO | "pinned" : " 29 GB" | 20260507 01:29:57.647 | INFO | } | 20260507 01:29:57.647 | INFO | Perf report for the last 5.13674 seconds: | 20260507 01:29:57.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:29:57.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:57.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:57.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:57.647 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 01:29:57.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:57.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:57.647 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:29:57.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:57.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:29:57.648 | INFO | Uploaded 36GiB to GPU in 1.97829s (18.1975GiB/s) | 20260507 01:29:57.648 | INFO | Downloaded 34GiB to GPU in 1.95296s (17.4095GiB/s) | 20260507 01:30:02.791 | INFO | Analyzing chunk frames = [6027776,6028288) | 20260507 01:30:02.791 | INFO | MemInfo: { | 20260507 01:30:02.791 | INFO | "gpu" : " 17 GB", | 20260507 01:30:02.791 | INFO | "huge" : "", | 20260507 01:30:02.791 | INFO | "huge_shared" : " 27 GB", | 20260507 01:30:02.791 | INFO | "malloc" : " 14 GB", | 20260507 01:30:02.791 | INFO | "pinned" : " 29 GB" | 20260507 01:30:02.791 | INFO | } | 20260507 01:30:02.791 | INFO | Perf report for the last 5.14398 seconds: | 20260507 01:30:02.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:30:02.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:02.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:02.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:02.791 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 01:30:02.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:02.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:02.792 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:30:02.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:02.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:02.792 | INFO | Uploaded 36GiB to GPU in 1.98653s (18.1221GiB/s) | 20260507 01:30:02.792 | INFO | Downloaded 34GiB to GPU in 1.93858s (17.5386GiB/s) | 20260507 01:30:07.939 | INFO | Analyzing chunk frames = [6028288,6028800) | 20260507 01:30:07.939 | INFO | MemInfo: { | 20260507 01:30:07.939 | INFO | "gpu" : " 17 GB", | 20260507 01:30:07.939 | INFO | "huge" : "", | 20260507 01:30:07.939 | INFO | "huge_shared" : " 27 GB", | 20260507 01:30:07.939 | INFO | "malloc" : " 14 GB", | 20260507 01:30:07.939 | INFO | "pinned" : " 29 GB" | 20260507 01:30:07.939 | INFO | } | 20260507 01:30:07.939 | INFO | Perf report for the last 5.14733 seconds: | 20260507 01:30:07.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:30:07.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:07.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:07.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:07.939 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:30:07.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:07.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:07.939 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:30:07.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:07.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:07.939 | INFO | Uploaded 36GiB to GPU in 1.96892s (18.2841GiB/s) | 20260507 01:30:07.939 | INFO | Downloaded 34GiB to GPU in 2.00102s (16.9914GiB/s) | 20260507 01:30:13.078 | INFO | Analyzing chunk frames = [6028800,6029312) | 20260507 01:30:13.078 | INFO | MemInfo: { | 20260507 01:30:13.078 | INFO | "gpu" : " 17 GB", | 20260507 01:30:13.078 | INFO | "huge" : "", | 20260507 01:30:13.078 | INFO | "huge_shared" : " 27 GB", | 20260507 01:30:13.078 | INFO | "malloc" : " 14 GB", | 20260507 01:30:13.078 | INFO | "pinned" : " 29 GB" | 20260507 01:30:13.078 | INFO | } | 20260507 01:30:13.078 | INFO | Perf report for the last 5.13936 seconds: | 20260507 01:30:13.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:30:13.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:13.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:13.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:13.078 | INFO | ANALYSIS : 61.3%, 3.0, 95.4ms, 0 | 20260507 01:30:13.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:13.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:13.078 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 01:30:13.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:13.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:13.078 | INFO | Uploaded 36GiB to GPU in 2.01124s (17.8994GiB/s) | 20260507 01:30:13.078 | INFO | Downloaded 34GiB to GPU in 1.97827s (17.1867GiB/s) | 20260507 01:30:13.079 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:30:13.079 | INFO | Total mem usage: 18 GiB | 20260507 01:30:13.079 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:30:13.079 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:30:13.079 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:30:13.079 | INFO | High Water Breakdown: | 20260507 01:30:13.079 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:30:13.079 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:30:13.079 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:30:13.079 | INFO | ------------------------------------------ | 20260507 01:30:13.079 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:30:13.079 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:30:13.079 | INFO | Total mem usage: 26.5 GiB | 20260507 01:30:13.079 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:30:13.079 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:30:13.079 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:30:13.079 | INFO | High Water Breakdown: | 20260507 01:30:13.079 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:30:13.079 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:30:13.079 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:30:13.079 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:30:13.079 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:30:13.079 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:30:13.079 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:30:13.079 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:30:13.079 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:30:13.079 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:30:13.079 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:30:13.079 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:30:13.079 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:30:13.079 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:30:13.079 | INFO | ------------------------------------------ | 20260507 01:30:13.079 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:30:13.079 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:30:13.079 | INFO | Total mem usage: 25.6 GiB | 20260507 01:30:13.079 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:30:13.079 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:30:13.079 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:30:13.079 | INFO | High Water Breakdown: | 20260507 01:30:13.079 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:30:13.079 | INFO | ------------------------------------------ | 20260507 01:30:13.079 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:30:13.079 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:30:13.079 | INFO | Total mem usage: 15.7 GiB | 20260507 01:30:13.079 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:30:13.079 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:30:13.079 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:30:13.079 | INFO | High Water Breakdown: | 20260507 01:30:13.079 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:30:13.079 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:30:13.079 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:30:13.079 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:30:13.079 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:30:13.079 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:30:13.079 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:30:13.079 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:30:13.079 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:30:13.079 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:30:13.079 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:30:13.079 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:30:13.079 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:30:13.079 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:30:13.079 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:30:13.079 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:30:13.079 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:30:13.079 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:30:13.079 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:30:13.079 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:30:13.079 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:30:13.079 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:30:13.079 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:30:13.079 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:30:13.079 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:30:13.079 | INFO | ------------------------------------------ | 20260507 01:30:13.080 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:30:13.080 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:30:13.080 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:30:13.080 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:30:13.080 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.58e+01, 1.35e+01 / 2.20e+00 | 20260507 01:30:13.080 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.03e-01, 9.77e-03 / 9.39e-03 | 20260507 01:30:13.080 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.43e+00, 5.31e-01 / 2.99e-01 | 20260507 01:30:13.080 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.46e+00, 1.74e-01 / 1.30e-01 | 20260507 01:30:13.080 | INFO | PerfReport: 16.10%, DME -- 3.98e+01 / 4.46e+01, 4.14e+00 / 3.07e+00 | 20260507 01:30:13.080 | INFO | PerfReport: 57.52%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.33e+00 / 2.15e+00 | 20260507 01:30:13.080 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.16e+00 / 2.72e+00, 3.78e-01 / 2.46e-01 | 20260507 01:30:13.080 | INFO | PerfReport: 3.11%, Metrics -- 9.61e-01 / 1.30e+00, 2.57e-01 / 1.42e-01 | 20260507 01:30:13.080 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.26e-01 / 3.76e-01, 5.18e-02 / 1.86e-02 | 20260507 01:30:13.080 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:30:13.080 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:30:13.080 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:30:13.080 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260507 01:30:13.080 | INFO | PerfReport: 21.53%, Upload -- 2.08e+01 / 0.00e+00, 6.85e+00 / 0.00e+00 | 20260507 01:30:13.080 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260507 01:30:13.080 | INFO | PerfReport: 32.02%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 01:30:13.080 | INFO | PerfReport: 1.26%, LockUpload -- 1.21e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 01:30:13.080 | INFO | PerfReport: 1.43%, LockDownload -- 1.38e+00 / 0.00e+00, 3.33e+00 / 0.00e+00 | 20260507 01:30:13.080 | INFO | PerfReport: 22.37%, LockCompute -- 2.16e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 01:30:13.080 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.48e-02 / 0.00e+00 | 20260507 01:30:13.176 | INFO | Starting a round of Baz IO | 20260507 01:30:16.290 | INFO | Populating RTMetrics report | 20260507 01:30:16.291 | INFO | RTMetrics report populated | 20260507 01:30:16.291 | INFO | Opening tmp RTMetrics json file | 20260507 01:30:16.291 | INFO | Serializing RTMetrics to json | 20260507 01:30:16.293 | INFO | Serialization done, swapping tmp file now... | 20260507 01:30:16.294 | INFO | RTMetrics done | 20260507 01:30:16.380 | INFO | Finished a round of Baz IO: Wrote 1.812 GiB in 3.203 seconds | 20260507 01:30:16.380 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.13% / 0.00% / 66.87% | 20260507 01:30:16.380 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.282339% / 0% / 7.54992e-05% | 20260507 01:30:16.380 | INFO | Baz IO overall: | 20260507 01:30:16.380 | INFO | BytesWritten : 3127021528998 | 20260507 01:30:16.380 | INFO | NumEvents : 1505063021608 | 20260507 01:30:18.207 | INFO | Analyzing chunk frames = [6029312,6029824) | 20260507 01:30:18.207 | INFO | MemInfo: { | 20260507 01:30:18.207 | INFO | "gpu" : " 17 GB", | 20260507 01:30:18.207 | INFO | "huge" : "", | 20260507 01:30:18.207 | INFO | "huge_shared" : " 27 GB", | 20260507 01:30:18.207 | INFO | "malloc" : " 14 GB", | 20260507 01:30:18.207 | INFO | "pinned" : " 29 GB" | 20260507 01:30:18.207 | INFO | } | 20260507 01:30:18.207 | INFO | Perf report for the last 5.12951 seconds: | 20260507 01:30:18.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:30:18.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:18.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:18.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:18.208 | INFO | ANALYSIS : 62.1%, 3.0, 96.3ms, 0 | 20260507 01:30:18.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:18.208 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 01:30:18.208 | INFO | PRE_HQ : 59.5%, 1.8, 56.4ms, 0 | 20260507 01:30:18.208 | INFO | BAZWRITER : 62.1%, 3.3, 105.8ms, 0 | 20260507 01:30:18.208 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:30:18.208 | INFO | Uploaded 36GiB to GPU in 2.04617s (17.5939GiB/s) | 20260507 01:30:18.208 | INFO | Downloaded 38GiB to GPU in 2.18285s (17.4084GiB/s) | 20260507 01:30:23.346 | INFO | Analyzing chunk frames = [6029824,6030336) | 20260507 01:30:23.346 | INFO | MemInfo: { | 20260507 01:30:23.346 | INFO | "gpu" : " 17 GB", | 20260507 01:30:23.346 | INFO | "huge" : "", | 20260507 01:30:23.346 | INFO | "huge_shared" : " 27 GB", | 20260507 01:30:23.346 | INFO | "malloc" : " 14 GB", | 20260507 01:30:23.346 | INFO | "pinned" : " 29 GB" | 20260507 01:30:23.346 | INFO | } | 20260507 01:30:23.346 | INFO | Perf report for the last 5.13849 seconds: | 20260507 01:30:23.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:30:23.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:23.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:23.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:23.346 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:30:23.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:23.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:23.346 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 01:30:23.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:23.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:23.346 | INFO | Uploaded 36GiB to GPU in 2.0083s (17.9256GiB/s) | 20260507 01:30:23.347 | INFO | Downloaded 34GiB to GPU in 1.92994s (17.6171GiB/s) | 20260507 01:30:28.491 | INFO | Analyzing chunk frames = [6030336,6030848) | 20260507 01:30:28.491 | INFO | MemInfo: { | 20260507 01:30:28.491 | INFO | "gpu" : " 17 GB", | 20260507 01:30:28.491 | INFO | "huge" : "", | 20260507 01:30:28.491 | INFO | "huge_shared" : " 27 GB", | 20260507 01:30:28.491 | INFO | "malloc" : " 14 GB", | 20260507 01:30:28.491 | INFO | "pinned" : " 29 GB" | 20260507 01:30:28.491 | INFO | } | 20260507 01:30:28.491 | INFO | Perf report for the last 5.14523 seconds: | 20260507 01:30:28.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:30:28.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:28.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:28.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:28.491 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260507 01:30:28.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:28.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:28.492 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:30:28.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:28.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:28.492 | INFO | Uploaded 36GiB to GPU in 1.99696s (18.0274GiB/s) | 20260507 01:30:28.492 | INFO | Downloaded 34GiB to GPU in 1.92973s (17.6191GiB/s) | 20260507 01:30:33.626 | INFO | Analyzing chunk frames = [6030848,6031360) | 20260507 01:30:33.626 | INFO | MemInfo: { | 20260507 01:30:33.626 | INFO | "gpu" : " 17 GB", | 20260507 01:30:33.626 | INFO | "huge" : "", | 20260507 01:30:33.626 | INFO | "huge_shared" : " 27 GB", | 20260507 01:30:33.626 | INFO | "malloc" : " 14 GB", | 20260507 01:30:33.626 | INFO | "pinned" : " 29 GB" | 20260507 01:30:33.626 | INFO | } | 20260507 01:30:33.626 | INFO | Perf report for the last 5.13504 seconds: | 20260507 01:30:33.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:30:33.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:33.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:33.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:33.627 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 01:30:33.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:33.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:33.627 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:30:33.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:33.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:33.627 | INFO | Uploaded 36GiB to GPU in 1.97585s (18.22GiB/s) | 20260507 01:30:33.627 | INFO | Downloaded 34GiB to GPU in 1.90447s (17.8527GiB/s) | 20260507 01:30:38.768 | INFO | Analyzing chunk frames = [6031360,6031872) | 20260507 01:30:38.768 | INFO | MemInfo: { | 20260507 01:30:38.768 | INFO | "gpu" : " 17 GB", | 20260507 01:30:38.768 | INFO | "huge" : "", | 20260507 01:30:38.768 | INFO | "huge_shared" : " 27 GB", | 20260507 01:30:38.768 | INFO | "malloc" : " 14 GB", | 20260507 01:30:38.768 | INFO | "pinned" : " 29 GB" | 20260507 01:30:38.768 | INFO | } | 20260507 01:30:38.768 | INFO | Perf report for the last 5.14173 seconds: | 20260507 01:30:38.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:30:38.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:38.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:38.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:38.768 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 01:30:38.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:38.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:38.768 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:30:38.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:38.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:38.768 | INFO | Uploaded 36GiB to GPU in 1.98781s (18.1103GiB/s) | 20260507 01:30:38.768 | INFO | Downloaded 34GiB to GPU in 1.92644s (17.6492GiB/s) | 20260507 01:30:43.906 | INFO | Analyzing chunk frames = [6031872,6032384) | 20260507 01:30:43.906 | INFO | MemInfo: { | 20260507 01:30:43.906 | INFO | "gpu" : " 17 GB", | 20260507 01:30:43.906 | INFO | "huge" : "", | 20260507 01:30:43.906 | INFO | "huge_shared" : " 27 GB", | 20260507 01:30:43.906 | INFO | "malloc" : " 14 GB", | 20260507 01:30:43.906 | INFO | "pinned" : " 29 GB" | 20260507 01:30:43.906 | INFO | } | 20260507 01:30:43.906 | INFO | Perf report for the last 5.13842 seconds: | 20260507 01:30:43.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:30:43.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:43.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:43.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:43.906 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 01:30:43.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:43.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:43.907 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:30:43.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:43.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:43.907 | INFO | Uploaded 36GiB to GPU in 2.03945s (17.6518GiB/s) | 20260507 01:30:43.907 | INFO | Downloaded 34GiB to GPU in 1.94526s (17.4783GiB/s) | 20260507 01:30:49.052 | INFO | Analyzing chunk frames = [6032384,6032896) | 20260507 01:30:49.052 | INFO | MemInfo: { | 20260507 01:30:49.052 | INFO | "gpu" : " 17 GB", | 20260507 01:30:49.052 | INFO | "huge" : "", | 20260507 01:30:49.052 | INFO | "huge_shared" : " 27 GB", | 20260507 01:30:49.052 | INFO | "malloc" : " 14 GB", | 20260507 01:30:49.052 | INFO | "pinned" : " 29 GB" | 20260507 01:30:49.052 | INFO | } | 20260507 01:30:49.052 | INFO | Perf report for the last 5.14567 seconds: | 20260507 01:30:49.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:30:49.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:49.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:49.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:49.052 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 01:30:49.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:49.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:49.052 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:30:49.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:49.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:49.053 | INFO | Uploaded 36GiB to GPU in 1.95585s (18.4063GiB/s) | 20260507 01:30:49.053 | INFO | Downloaded 34GiB to GPU in 1.98768s (17.1054GiB/s) | 20260507 01:30:54.185 | INFO | Analyzing chunk frames = [6032896,6033408) | 20260507 01:30:54.185 | INFO | MemInfo: { | 20260507 01:30:54.185 | INFO | "gpu" : " 17 GB", | 20260507 01:30:54.185 | INFO | "huge" : "", | 20260507 01:30:54.185 | INFO | "huge_shared" : " 27 GB", | 20260507 01:30:54.185 | INFO | "malloc" : " 14 GB", | 20260507 01:30:54.185 | INFO | "pinned" : " 29 GB" | 20260507 01:30:54.185 | INFO | } | 20260507 01:30:54.185 | INFO | Perf report for the last 5.13266 seconds: | 20260507 01:30:54.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:30:54.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:54.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:54.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:54.185 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 01:30:54.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:54.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:54.185 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 01:30:54.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:54.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:54.185 | INFO | Uploaded 36GiB to GPU in 2.00289s (17.9741GiB/s) | 20260507 01:30:54.185 | INFO | Downloaded 34GiB to GPU in 1.98325s (17.1436GiB/s) | 20260507 01:30:54.284 | INFO | Starting a round of Baz IO | 20260507 01:30:57.391 | INFO | Populating RTMetrics report | 20260507 01:30:57.391 | INFO | RTMetrics report populated | 20260507 01:30:57.391 | INFO | Opening tmp RTMetrics json file | 20260507 01:30:57.392 | INFO | Serializing RTMetrics to json | 20260507 01:30:57.393 | INFO | Serialization done, swapping tmp file now... | 20260507 01:30:57.393 | INFO | RTMetrics done | 20260507 01:30:57.482 | INFO | Finished a round of Baz IO: Wrote 1.813 GiB in 3.197 seconds | 20260507 01:30:57.482 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.13% / 0.00% / 66.87% | 20260507 01:30:57.482 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.282339% / 0% / 4.76837e-05% | 20260507 01:30:57.482 | INFO | Baz IO overall: | 20260507 01:30:57.482 | INFO | BytesWritten : 3128968325563 | 20260507 01:30:57.482 | INFO | NumEvents : 1505985716933 | 20260507 01:30:59.324 | INFO | Analyzing chunk frames = [6033408,6033920) | 20260507 01:30:59.324 | INFO | MemInfo: { | 20260507 01:30:59.324 | INFO | "gpu" : " 17 GB", | 20260507 01:30:59.324 | INFO | "huge" : "", | 20260507 01:30:59.324 | INFO | "huge_shared" : " 27 GB", | 20260507 01:30:59.324 | INFO | "malloc" : " 14 GB", | 20260507 01:30:59.324 | INFO | "pinned" : " 29 GB" | 20260507 01:30:59.324 | INFO | } | 20260507 01:30:59.324 | INFO | Perf report for the last 5.1392 seconds: | 20260507 01:30:59.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:30:59.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:59.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:59.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:59.324 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 01:30:59.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:30:59.324 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260507 01:30:59.324 | INFO | PRE_HQ : 59.9%, 1.9, 59.8ms, 0 | 20260507 01:30:59.324 | INFO | BAZWRITER : 61.8%, 3.5, 110.3ms, 0 | 20260507 01:30:59.324 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:30:59.324 | INFO | Uploaded 36GiB to GPU in 2.02331s (17.7926GiB/s) | 20260507 01:30:59.324 | INFO | Downloaded 38GiB to GPU in 2.17887s (17.4403GiB/s) | 20260507 01:31:04.462 | INFO | Analyzing chunk frames = [6033920,6034432) | 20260507 01:31:04.462 | INFO | MemInfo: { | 20260507 01:31:04.462 | INFO | "gpu" : " 17 GB", | 20260507 01:31:04.462 | INFO | "huge" : "", | 20260507 01:31:04.462 | INFO | "huge_shared" : " 27 GB", | 20260507 01:31:04.462 | INFO | "malloc" : " 14 GB", | 20260507 01:31:04.462 | INFO | "pinned" : " 29 GB" | 20260507 01:31:04.462 | INFO | } | 20260507 01:31:04.462 | INFO | Perf report for the last 5.13834 seconds: | 20260507 01:31:04.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:31:04.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:04.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:04.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:04.462 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:31:04.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:04.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:04.462 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 01:31:04.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:04.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:04.463 | INFO | Uploaded 36GiB to GPU in 2.03359s (17.7026GiB/s) | 20260507 01:31:04.463 | INFO | Downloaded 34GiB to GPU in 1.9416s (17.5113GiB/s) | 20260507 01:31:09.601 | INFO | Analyzing chunk frames = [6034432,6034944) | 20260507 01:31:09.601 | INFO | MemInfo: { | 20260507 01:31:09.601 | INFO | "gpu" : " 17 GB", | 20260507 01:31:09.601 | INFO | "huge" : "", | 20260507 01:31:09.601 | INFO | "huge_shared" : " 27 GB", | 20260507 01:31:09.601 | INFO | "malloc" : " 14 GB", | 20260507 01:31:09.601 | INFO | "pinned" : " 29 GB" | 20260507 01:31:09.601 | INFO | } | 20260507 01:31:09.601 | INFO | Perf report for the last 5.13845 seconds: | 20260507 01:31:09.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:31:09.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:09.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:09.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:09.601 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 01:31:09.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:09.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:09.601 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:31:09.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:09.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:09.601 | INFO | Uploaded 36GiB to GPU in 1.98124s (18.1704GiB/s) | 20260507 01:31:09.601 | INFO | Downloaded 34GiB to GPU in 1.93736s (17.5496GiB/s) | 20260507 01:31:14.756 | INFO | Analyzing chunk frames = [6034944,6035456) | 20260507 01:31:14.756 | INFO | MemInfo: { | 20260507 01:31:14.756 | INFO | "gpu" : " 17 GB", | 20260507 01:31:14.756 | INFO | "huge" : "", | 20260507 01:31:14.756 | INFO | "huge_shared" : " 27 GB", | 20260507 01:31:14.756 | INFO | "malloc" : " 14 GB", | 20260507 01:31:14.756 | INFO | "pinned" : " 29 GB" | 20260507 01:31:14.756 | INFO | } | 20260507 01:31:14.756 | INFO | Perf report for the last 5.15514 seconds: | 20260507 01:31:14.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:31:14.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:14.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:14.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:14.756 | INFO | ANALYSIS : 59.8%, 3.0, 93.0ms, 0 | 20260507 01:31:14.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:14.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:14.756 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:31:14.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:14.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:14.756 | INFO | Uploaded 36GiB to GPU in 2.00649s (17.9418GiB/s) | 20260507 01:31:14.756 | INFO | Downloaded 34GiB to GPU in 1.90974s (17.8035GiB/s) | 20260507 01:31:19.889 | INFO | Analyzing chunk frames = [6035456,6035968) | 20260507 01:31:19.889 | INFO | MemInfo: { | 20260507 01:31:19.889 | INFO | "gpu" : " 17 GB", | 20260507 01:31:19.889 | INFO | "huge" : "", | 20260507 01:31:19.889 | INFO | "huge_shared" : " 27 GB", | 20260507 01:31:19.889 | INFO | "malloc" : " 14 GB", | 20260507 01:31:19.889 | INFO | "pinned" : " 29 GB" | 20260507 01:31:19.889 | INFO | } | 20260507 01:31:19.889 | INFO | Perf report for the last 5.13263 seconds: | 20260507 01:31:19.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:31:19.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:19.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:19.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:19.889 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 01:31:19.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:19.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:19.889 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:31:19.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:19.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:19.889 | INFO | Uploaded 36GiB to GPU in 2.02067s (17.8158GiB/s) | 20260507 01:31:19.889 | INFO | Downloaded 34GiB to GPU in 1.9421s (17.5068GiB/s) | 20260507 01:31:25.025 | INFO | Analyzing chunk frames = [6035968,6036480) | 20260507 01:31:25.025 | INFO | MemInfo: { | 20260507 01:31:25.025 | INFO | "gpu" : " 17 GB", | 20260507 01:31:25.025 | INFO | "huge" : "", | 20260507 01:31:25.025 | INFO | "huge_shared" : " 27 GB", | 20260507 01:31:25.025 | INFO | "malloc" : " 14 GB", | 20260507 01:31:25.025 | INFO | "pinned" : " 29 GB" | 20260507 01:31:25.025 | INFO | } | 20260507 01:31:25.025 | INFO | Perf report for the last 5.13664 seconds: | 20260507 01:31:25.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:31:25.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:25.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:25.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:25.025 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 01:31:25.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:25.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:25.025 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:31:25.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:25.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:25.026 | INFO | Uploaded 36GiB to GPU in 2.04543s (17.6002GiB/s) | 20260507 01:31:25.026 | INFO | Downloaded 34GiB to GPU in 1.96011s (17.346GiB/s) | 20260507 01:31:30.163 | INFO | Analyzing chunk frames = [6036480,6036992) | 20260507 01:31:30.163 | INFO | MemInfo: { | 20260507 01:31:30.163 | INFO | "gpu" : " 17 GB", | 20260507 01:31:30.163 | INFO | "huge" : "", | 20260507 01:31:30.163 | INFO | "huge_shared" : " 27 GB", | 20260507 01:31:30.163 | INFO | "malloc" : " 14 GB", | 20260507 01:31:30.163 | INFO | "pinned" : " 29 GB" | 20260507 01:31:30.163 | INFO | } | 20260507 01:31:30.163 | INFO | Perf report for the last 5.1382 seconds: | 20260507 01:31:30.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:31:30.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:30.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:30.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:30.164 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:31:30.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:30.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:30.164 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 01:31:30.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:30.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:30.164 | INFO | Uploaded 36GiB to GPU in 1.97675s (18.2117GiB/s) | 20260507 01:31:30.164 | INFO | Downloaded 34GiB to GPU in 1.98778s (17.1045GiB/s) | 20260507 01:31:35.301 | INFO | Analyzing chunk frames = [6036992,6037504) | 20260507 01:31:35.301 | INFO | MemInfo: { | 20260507 01:31:35.301 | INFO | "gpu" : " 17 GB", | 20260507 01:31:35.301 | INFO | "huge" : "", | 20260507 01:31:35.301 | INFO | "huge_shared" : " 27 GB", | 20260507 01:31:35.301 | INFO | "malloc" : " 14 GB", | 20260507 01:31:35.301 | INFO | "pinned" : " 29 GB" | 20260507 01:31:35.301 | INFO | } | 20260507 01:31:35.301 | INFO | Perf report for the last 5.13775 seconds: | 20260507 01:31:35.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:31:35.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:35.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:35.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:35.301 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 01:31:35.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:35.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:35.302 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 01:31:35.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:35.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:35.302 | INFO | Uploaded 36GiB to GPU in 1.99894s (18.0096GiB/s) | 20260507 01:31:35.302 | INFO | Downloaded 34GiB to GPU in 1.9763s (17.2038GiB/s) | 20260507 01:31:35.302 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:31:35.302 | INFO | Total mem usage: 18 GiB | 20260507 01:31:35.302 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:31:35.302 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.1%) | 20260507 01:31:35.302 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:31:35.302 | INFO | High Water Breakdown: | 20260507 01:31:35.302 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:31:35.302 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:31:35.302 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:31:35.302 | INFO | ------------------------------------------ | 20260507 01:31:35.302 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:31:35.302 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:31:35.302 | INFO | Total mem usage: 26.5 GiB | 20260507 01:31:35.302 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:31:35.302 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:31:35.302 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:31:35.302 | INFO | High Water Breakdown: | 20260507 01:31:35.302 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:31:35.302 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:31:35.302 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:31:35.302 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:31:35.302 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:31:35.302 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:31:35.302 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:31:35.303 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:31:35.303 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:31:35.303 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:31:35.303 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:31:35.303 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:31:35.303 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:31:35.303 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:31:35.303 | INFO | ------------------------------------------ | 20260507 01:31:35.303 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:31:35.303 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:31:35.303 | INFO | Total mem usage: 25.6 GiB | 20260507 01:31:35.303 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:31:35.303 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:31:35.303 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:31:35.303 | INFO | High Water Breakdown: | 20260507 01:31:35.303 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:31:35.303 | INFO | ------------------------------------------ | 20260507 01:31:35.303 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:31:35.303 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:31:35.303 | INFO | Total mem usage: 15.7 GiB | 20260507 01:31:35.303 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:31:35.303 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:31:35.303 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:31:35.303 | INFO | High Water Breakdown: | 20260507 01:31:35.303 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:31:35.303 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:31:35.303 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:31:35.303 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:31:35.303 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:31:35.303 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:31:35.303 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:31:35.303 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:31:35.303 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:31:35.303 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:31:35.303 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:31:35.303 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:31:35.303 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:31:35.303 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:31:35.303 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:31:35.303 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:31:35.303 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:31:35.303 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:31:35.303 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:31:35.303 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:31:35.303 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:31:35.303 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:31:35.303 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:31:35.303 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:31:35.303 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:31:35.303 | INFO | ------------------------------------------ | 20260507 01:31:35.303 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:31:35.303 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:31:35.303 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:31:35.303 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:31:35.303 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.57e+01, 1.35e+01 / 1.84e+00 | 20260507 01:31:35.303 | INFO | PerfReport: 0.30%, AggregateStats -- 9.13e-02 / 1.02e-01, 1.00e-02 / 5.92e-03 | 20260507 01:31:35.303 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.45e+00, 5.32e-01 / 2.79e-01 | 20260507 01:31:35.303 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.49e+00, 1.73e-01 / 9.92e-02 | 20260507 01:31:35.303 | INFO | PerfReport: 16.09%, DME -- 3.97e+01 / 4.45e+01, 4.01e+00 / 2.59e+00 | 20260507 01:31:35.303 | INFO | PerfReport: 57.49%, FrameLabeling -- 1.77e+01 / 2.09e+01, 1.31e+00 / 1.75e+00 | 20260507 01:31:35.303 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.17e+00 / 2.69e+00, 3.83e-01 / 2.40e-01 | 20260507 01:31:35.303 | INFO | PerfReport: 3.11%, Metrics -- 9.61e-01 / 1.27e+00, 2.58e-01 / 1.29e-01 | 20260507 01:31:35.303 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.26e-01 / 3.74e-01, 5.15e-02 / 1.93e-02 | 20260507 01:31:35.303 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:31:35.303 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:31:35.303 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:31:35.303 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 01:31:35.303 | INFO | PerfReport: 21.67%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260507 01:31:35.303 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 01:31:35.303 | INFO | PerfReport: 32.01%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 01:31:35.303 | INFO | PerfReport: 1.36%, LockUpload -- 1.31e+00 / 0.00e+00, 4.94e+00 / 0.00e+00 | 20260507 01:31:35.303 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.10e+00 / 0.00e+00 | 20260507 01:31:35.303 | INFO | PerfReport: 22.29%, LockCompute -- 2.15e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 01:31:35.303 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260507 01:31:35.400 | INFO | Starting a round of Baz IO | 20260507 01:31:38.518 | INFO | Populating RTMetrics report | 20260507 01:31:38.519 | INFO | RTMetrics report populated | 20260507 01:31:38.519 | INFO | Opening tmp RTMetrics json file | 20260507 01:31:38.538 | INFO | Serializing RTMetrics to json | 20260507 01:31:38.539 | INFO | Serialization done, swapping tmp file now... | 20260507 01:31:38.539 | INFO | RTMetrics done | 20260507 01:31:38.610 | INFO | Finished a round of Baz IO: Wrote 1.81 GiB in 3.209 seconds | 20260507 01:31:38.610 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.12% / 0.00% / 66.87% | 20260507 01:31:38.610 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.281858% / 7.94729e-06% / 5.96046e-05% | 20260507 01:31:38.610 | INFO | Baz IO overall: | 20260507 01:31:38.610 | INFO | BytesWritten : 3130912246120 | 20260507 01:31:38.610 | INFO | NumEvents : 1506906348833 | 20260507 01:31:40.440 | INFO | Analyzing chunk frames = [6037504,6038016) | 20260507 01:31:40.440 | INFO | MemInfo: { | 20260507 01:31:40.440 | INFO | "gpu" : " 17 GB", | 20260507 01:31:40.440 | INFO | "huge" : "", | 20260507 01:31:40.440 | INFO | "huge_shared" : " 27 GB", | 20260507 01:31:40.440 | INFO | "malloc" : " 14 GB", | 20260507 01:31:40.440 | INFO | "pinned" : " 29 GB" | 20260507 01:31:40.440 | INFO | } | 20260507 01:31:40.440 | INFO | Perf report for the last 5.13875 seconds: | 20260507 01:31:40.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:31:40.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:40.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:40.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:40.440 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260507 01:31:40.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:40.440 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 01:31:40.440 | INFO | PRE_HQ : 59.5%, 1.8, 56.0ms, 0 | 20260507 01:31:40.440 | INFO | BAZWRITER : 61.9%, 3.3, 106.1ms, 0 | 20260507 01:31:40.440 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 01:31:40.440 | INFO | Uploaded 36GiB to GPU in 2.02889s (17.7437GiB/s) | 20260507 01:31:40.440 | INFO | Downloaded 38GiB to GPU in 2.17797s (17.4475GiB/s) | 20260507 01:31:45.588 | INFO | Analyzing chunk frames = [6038016,6038528) | 20260507 01:31:45.588 | INFO | MemInfo: { | 20260507 01:31:45.588 | INFO | "gpu" : " 17 GB", | 20260507 01:31:45.588 | INFO | "huge" : "", | 20260507 01:31:45.588 | INFO | "huge_shared" : " 27 GB", | 20260507 01:31:45.588 | INFO | "malloc" : " 14 GB", | 20260507 01:31:45.588 | INFO | "pinned" : " 29 GB" | 20260507 01:31:45.588 | INFO | } | 20260507 01:31:45.588 | INFO | Perf report for the last 5.14771 seconds: | 20260507 01:31:45.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:31:45.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:45.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:45.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:45.588 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 01:31:45.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:45.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:45.588 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:31:45.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:45.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:45.588 | INFO | Uploaded 36GiB to GPU in 2.00859s (17.923GiB/s) | 20260507 01:31:45.588 | INFO | Downloaded 34GiB to GPU in 1.93482s (17.5727GiB/s) | 20260507 01:31:50.724 | INFO | Analyzing chunk frames = [6038528,6039040) | 20260507 01:31:50.724 | INFO | MemInfo: { | 20260507 01:31:50.724 | INFO | "gpu" : " 17 GB", | 20260507 01:31:50.724 | INFO | "huge" : "", | 20260507 01:31:50.724 | INFO | "huge_shared" : " 27 GB", | 20260507 01:31:50.724 | INFO | "malloc" : " 14 GB", | 20260507 01:31:50.724 | INFO | "pinned" : " 29 GB" | 20260507 01:31:50.724 | INFO | } | 20260507 01:31:50.724 | INFO | Perf report for the last 5.1363 seconds: | 20260507 01:31:50.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:31:50.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:50.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:50.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:50.724 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:31:50.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:50.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:50.724 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:31:50.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:50.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:50.724 | INFO | Uploaded 36GiB to GPU in 2.00434s (17.9611GiB/s) | 20260507 01:31:50.725 | INFO | Downloaded 34GiB to GPU in 1.94243s (17.5039GiB/s) | 20260507 01:31:55.859 | INFO | Analyzing chunk frames = [6039040,6039552) | 20260507 01:31:55.859 | INFO | MemInfo: { | 20260507 01:31:55.859 | INFO | "gpu" : " 17 GB", | 20260507 01:31:55.859 | INFO | "huge" : "", | 20260507 01:31:55.859 | INFO | "huge_shared" : " 27 GB", | 20260507 01:31:55.859 | INFO | "malloc" : " 14 GB", | 20260507 01:31:55.859 | INFO | "pinned" : " 29 GB" | 20260507 01:31:55.859 | INFO | } | 20260507 01:31:55.859 | INFO | Perf report for the last 5.13474 seconds: | 20260507 01:31:55.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:31:55.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:55.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:55.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:55.859 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 01:31:55.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:55.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:55.859 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:31:55.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:55.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:31:55.859 | INFO | Uploaded 36GiB to GPU in 1.95606s (18.4043GiB/s) | 20260507 01:31:55.859 | INFO | Downloaded 34GiB to GPU in 1.91119s (17.79GiB/s) | 20260507 01:32:00.997 | INFO | Analyzing chunk frames = [6039552,6040064) | 20260507 01:32:00.997 | INFO | MemInfo: { | 20260507 01:32:00.997 | INFO | "gpu" : " 17 GB", | 20260507 01:32:00.997 | INFO | "huge" : "", | 20260507 01:32:00.997 | INFO | "huge_shared" : " 27 GB", | 20260507 01:32:00.997 | INFO | "malloc" : " 14 GB", | 20260507 01:32:00.997 | INFO | "pinned" : " 29 GB" | 20260507 01:32:00.997 | INFO | } | 20260507 01:32:00.998 | INFO | Perf report for the last 5.13875 seconds: | 20260507 01:32:00.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:32:00.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:00.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:00.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:00.998 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 01:32:00.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:00.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:00.998 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 01:32:00.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:00.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:00.998 | INFO | Uploaded 36GiB to GPU in 1.98769s (18.1115GiB/s) | 20260507 01:32:00.998 | INFO | Downloaded 34GiB to GPU in 1.93969s (17.5286GiB/s) | 20260507 01:32:06.137 | INFO | Analyzing chunk frames = [6040064,6040576) | 20260507 01:32:06.137 | INFO | MemInfo: { | 20260507 01:32:06.137 | INFO | "gpu" : " 17 GB", | 20260507 01:32:06.137 | INFO | "huge" : "", | 20260507 01:32:06.137 | INFO | "huge_shared" : " 27 GB", | 20260507 01:32:06.137 | INFO | "malloc" : " 14 GB", | 20260507 01:32:06.137 | INFO | "pinned" : " 29 GB" | 20260507 01:32:06.137 | INFO | } | 20260507 01:32:06.137 | INFO | Perf report for the last 5.13968 seconds: | 20260507 01:32:06.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:32:06.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:06.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:06.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:06.137 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 01:32:06.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:06.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:06.137 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:32:06.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:06.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:06.138 | INFO | Uploaded 36GiB to GPU in 2.01323s (17.8817GiB/s) | 20260507 01:32:06.138 | INFO | Downloaded 34GiB to GPU in 1.94215s (17.5064GiB/s) | 20260507 01:32:11.269 | INFO | Analyzing chunk frames = [6040576,6041088) | 20260507 01:32:11.269 | INFO | MemInfo: { | 20260507 01:32:11.269 | INFO | "gpu" : " 17 GB", | 20260507 01:32:11.269 | INFO | "huge" : "", | 20260507 01:32:11.269 | INFO | "huge_shared" : " 27 GB", | 20260507 01:32:11.269 | INFO | "malloc" : " 14 GB", | 20260507 01:32:11.269 | INFO | "pinned" : " 29 GB" | 20260507 01:32:11.269 | INFO | } | 20260507 01:32:11.269 | INFO | Perf report for the last 5.13215 seconds: | 20260507 01:32:11.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:32:11.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:11.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:11.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:11.270 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:32:11.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:11.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:11.270 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:32:11.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:11.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:11.270 | INFO | Uploaded 36GiB to GPU in 1.98933s (18.0966GiB/s) | 20260507 01:32:11.270 | INFO | Downloaded 34GiB to GPU in 1.9823s (17.1518GiB/s) | 20260507 01:32:16.414 | INFO | Analyzing chunk frames = [6041088,6041600) | 20260507 01:32:16.414 | INFO | MemInfo: { | 20260507 01:32:16.414 | INFO | "gpu" : " 17 GB", | 20260507 01:32:16.414 | INFO | "huge" : "", | 20260507 01:32:16.414 | INFO | "huge_shared" : " 27 GB", | 20260507 01:32:16.414 | INFO | "malloc" : " 14 GB", | 20260507 01:32:16.414 | INFO | "pinned" : " 29 GB" | 20260507 01:32:16.414 | INFO | } | 20260507 01:32:16.414 | INFO | Perf report for the last 5.14501 seconds: | 20260507 01:32:16.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:32:16.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:16.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:16.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:16.415 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:32:16.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:16.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:16.415 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 01:32:16.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:16.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:16.415 | INFO | Uploaded 36GiB to GPU in 1.9771s (18.2085GiB/s) | 20260507 01:32:16.415 | INFO | Downloaded 34GiB to GPU in 1.98878s (17.0959GiB/s) | 20260507 01:32:16.511 | INFO | Starting a round of Baz IO | 20260507 01:32:19.632 | INFO | Populating RTMetrics report | 20260507 01:32:19.632 | INFO | RTMetrics report populated | 20260507 01:32:19.632 | INFO | Opening tmp RTMetrics json file | 20260507 01:32:19.635 | INFO | Serializing RTMetrics to json | 20260507 01:32:19.651 | INFO | Serialization done, swapping tmp file now... | 20260507 01:32:19.651 | INFO | RTMetrics done | 20260507 01:32:19.721 | INFO | Finished a round of Baz IO: Wrote 1.811 GiB in 3.209 seconds | 20260507 01:32:19.721 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.12% / 0.00% / 66.88% | 20260507 01:32:19.721 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.280746% / 3.97364e-06% / 7.54992e-05% | 20260507 01:32:19.721 | INFO | Baz IO overall: | 20260507 01:32:19.721 | INFO | BytesWritten : 3132856716163 | 20260507 01:32:19.721 | INFO | NumEvents : 1507827763173 | 20260507 01:32:21.555 | INFO | Analyzing chunk frames = [6041600,6042112) | 20260507 01:32:21.555 | INFO | MemInfo: { | 20260507 01:32:21.555 | INFO | "gpu" : " 17 GB", | 20260507 01:32:21.556 | INFO | "huge" : "", | 20260507 01:32:21.556 | INFO | "huge_shared" : " 27 GB", | 20260507 01:32:21.556 | INFO | "malloc" : " 14 GB", | 20260507 01:32:21.556 | INFO | "pinned" : " 29 GB" | 20260507 01:32:21.556 | INFO | } | 20260507 01:32:21.556 | INFO | Perf report for the last 5.14123 seconds: | 20260507 01:32:21.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:32:21.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:21.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:21.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:21.556 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 01:32:21.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:21.556 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 01:32:21.556 | INFO | PRE_HQ : 59.5%, 1.8, 56.1ms, 0 | 20260507 01:32:21.556 | INFO | BAZWRITER : 61.9%, 3.3, 105.0ms, 0 | 20260507 01:32:21.556 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:32:21.556 | INFO | Uploaded 36GiB to GPU in 2.0071s (17.9363GiB/s) | 20260507 01:32:21.556 | INFO | Downloaded 38GiB to GPU in 2.18001s (17.4311GiB/s) | 20260507 01:32:26.703 | INFO | Analyzing chunk frames = [6042112,6042624) | 20260507 01:32:26.703 | INFO | MemInfo: { | 20260507 01:32:26.703 | INFO | "gpu" : " 17 GB", | 20260507 01:32:26.703 | INFO | "huge" : "", | 20260507 01:32:26.703 | INFO | "huge_shared" : " 27 GB", | 20260507 01:32:26.703 | INFO | "malloc" : " 14 GB", | 20260507 01:32:26.703 | INFO | "pinned" : " 29 GB" | 20260507 01:32:26.703 | INFO | } | 20260507 01:32:26.703 | INFO | Perf report for the last 5.14738 seconds: | 20260507 01:32:26.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:32:26.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:26.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:26.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:26.703 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 01:32:26.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:26.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:26.703 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:32:26.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:26.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:26.703 | INFO | Uploaded 36GiB to GPU in 1.99189s (18.0733GiB/s) | 20260507 01:32:26.703 | INFO | Downloaded 34GiB to GPU in 1.93476s (17.5732GiB/s) | 20260507 01:32:31.843 | INFO | Analyzing chunk frames = [6042624,6043136) | 20260507 01:32:31.843 | INFO | MemInfo: { | 20260507 01:32:31.843 | INFO | "gpu" : " 17 GB", | 20260507 01:32:31.843 | INFO | "huge" : "", | 20260507 01:32:31.843 | INFO | "huge_shared" : " 27 GB", | 20260507 01:32:31.843 | INFO | "malloc" : " 14 GB", | 20260507 01:32:31.843 | INFO | "pinned" : " 29 GB" | 20260507 01:32:31.843 | INFO | } | 20260507 01:32:31.843 | INFO | Perf report for the last 5.13975 seconds: | 20260507 01:32:31.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:32:31.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:31.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:31.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:31.843 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:32:31.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:31.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:31.843 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:32:31.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:31.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:31.843 | INFO | Uploaded 36GiB to GPU in 1.98665s (18.121GiB/s) | 20260507 01:32:31.843 | INFO | Downloaded 34GiB to GPU in 1.9422s (17.5059GiB/s) | 20260507 01:32:36.976 | INFO | Analyzing chunk frames = [6043136,6043648) | 20260507 01:32:36.976 | INFO | MemInfo: { | 20260507 01:32:36.976 | INFO | "gpu" : " 17 GB", | 20260507 01:32:36.976 | INFO | "huge" : "", | 20260507 01:32:36.976 | INFO | "huge_shared" : " 27 GB", | 20260507 01:32:36.976 | INFO | "malloc" : " 14 GB", | 20260507 01:32:36.976 | INFO | "pinned" : " 29 GB" | 20260507 01:32:36.976 | INFO | } | 20260507 01:32:36.976 | INFO | Perf report for the last 5.13322 seconds: | 20260507 01:32:36.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:32:36.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:36.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:36.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:36.976 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 01:32:36.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:36.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:36.976 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:32:36.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:36.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:36.976 | INFO | Uploaded 36GiB to GPU in 1.96251s (18.3439GiB/s) | 20260507 01:32:36.976 | INFO | Downloaded 34GiB to GPU in 1.92093s (17.6998GiB/s) | 20260507 01:32:42.113 | INFO | Analyzing chunk frames = [6043648,6044160) | 20260507 01:32:42.113 | INFO | MemInfo: { | 20260507 01:32:42.113 | INFO | "gpu" : " 17 GB", | 20260507 01:32:42.113 | INFO | "huge" : "", | 20260507 01:32:42.113 | INFO | "huge_shared" : " 27 GB", | 20260507 01:32:42.113 | INFO | "malloc" : " 14 GB", | 20260507 01:32:42.113 | INFO | "pinned" : " 29 GB" | 20260507 01:32:42.113 | INFO | } | 20260507 01:32:42.114 | INFO | Perf report for the last 5.13744 seconds: | 20260507 01:32:42.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:32:42.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:42.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:42.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:42.114 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 01:32:42.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:42.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:42.114 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:32:42.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:42.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:42.114 | INFO | Uploaded 36GiB to GPU in 1.98651s (18.1223GiB/s) | 20260507 01:32:42.114 | INFO | Downloaded 34GiB to GPU in 1.95327s (17.4067GiB/s) | 20260507 01:32:47.254 | INFO | Analyzing chunk frames = [6044160,6044672) | 20260507 01:32:47.254 | INFO | MemInfo: { | 20260507 01:32:47.254 | INFO | "gpu" : " 17 GB", | 20260507 01:32:47.254 | INFO | "huge" : "", | 20260507 01:32:47.254 | INFO | "huge_shared" : " 27 GB", | 20260507 01:32:47.254 | INFO | "malloc" : " 14 GB", | 20260507 01:32:47.254 | INFO | "pinned" : " 29 GB" | 20260507 01:32:47.254 | INFO | } | 20260507 01:32:47.254 | INFO | Perf report for the last 5.14046 seconds: | 20260507 01:32:47.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:32:47.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:47.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:47.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:47.254 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 01:32:47.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:47.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:47.254 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:32:47.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:47.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:47.254 | INFO | Uploaded 36GiB to GPU in 2.00145s (17.9869GiB/s) | 20260507 01:32:47.254 | INFO | Downloaded 34GiB to GPU in 1.94074s (17.5191GiB/s) | 20260507 01:32:52.393 | INFO | Analyzing chunk frames = [6044672,6045184) | 20260507 01:32:52.393 | INFO | MemInfo: { | 20260507 01:32:52.393 | INFO | "gpu" : " 17 GB", | 20260507 01:32:52.393 | INFO | "huge" : "", | 20260507 01:32:52.394 | INFO | "huge_shared" : " 27 GB", | 20260507 01:32:52.394 | INFO | "malloc" : " 14 GB", | 20260507 01:32:52.394 | INFO | "pinned" : " 29 GB" | 20260507 01:32:52.394 | INFO | } | 20260507 01:32:52.394 | INFO | Perf report for the last 5.13956 seconds: | 20260507 01:32:52.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:32:52.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:52.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:52.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:52.394 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:32:52.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:52.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:52.394 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 01:32:52.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:52.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:52.394 | INFO | Uploaded 36GiB to GPU in 1.97579s (18.2206GiB/s) | 20260507 01:32:52.394 | INFO | Downloaded 34GiB to GPU in 1.98049s (17.1674GiB/s) | 20260507 01:32:57.536 | INFO | Analyzing chunk frames = [6045184,6045696) | 20260507 01:32:57.536 | INFO | MemInfo: { | 20260507 01:32:57.536 | INFO | "gpu" : " 17 GB", | 20260507 01:32:57.536 | INFO | "huge" : "", | 20260507 01:32:57.536 | INFO | "huge_shared" : " 27 GB", | 20260507 01:32:57.536 | INFO | "malloc" : " 14 GB", | 20260507 01:32:57.536 | INFO | "pinned" : " 29 GB" | 20260507 01:32:57.536 | INFO | } | 20260507 01:32:57.536 | INFO | Perf report for the last 5.1422 seconds: | 20260507 01:32:57.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:32:57.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:57.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:57.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:57.536 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:32:57.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:57.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:57.536 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 01:32:57.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:57.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:32:57.536 | INFO | Uploaded 36GiB to GPU in 2.00884s (17.9208GiB/s) | 20260507 01:32:57.536 | INFO | Downloaded 34GiB to GPU in 1.96262s (17.3237GiB/s) | 20260507 01:32:57.536 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:32:57.536 | INFO | Total mem usage: 18 GiB | 20260507 01:32:57.536 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:32:57.536 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:32:57.536 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:32:57.536 | INFO | High Water Breakdown: | 20260507 01:32:57.536 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:32:57.536 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:32:57.536 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:32:57.536 | INFO | ------------------------------------------ | 20260507 01:32:57.536 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:32:57.536 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:32:57.536 | INFO | Total mem usage: 26.5 GiB | 20260507 01:32:57.536 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:32:57.536 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:32:57.536 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:32:57.536 | INFO | High Water Breakdown: | 20260507 01:32:57.536 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:32:57.536 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:32:57.537 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:32:57.537 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:32:57.537 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:32:57.537 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:32:57.537 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:32:57.537 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:32:57.537 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:32:57.537 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:32:57.537 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:32:57.537 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:32:57.537 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:32:57.537 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:32:57.537 | INFO | ------------------------------------------ | 20260507 01:32:57.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:32:57.537 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:32:57.537 | INFO | Total mem usage: 25.6 GiB | 20260507 01:32:57.537 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:32:57.537 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:32:57.537 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:32:57.537 | INFO | High Water Breakdown: | 20260507 01:32:57.537 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:32:57.537 | INFO | ------------------------------------------ | 20260507 01:32:57.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:32:57.537 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:32:57.537 | INFO | Total mem usage: 15.7 GiB | 20260507 01:32:57.537 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:32:57.537 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:32:57.537 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:32:57.537 | INFO | High Water Breakdown: | 20260507 01:32:57.537 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:32:57.537 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:32:57.537 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:32:57.537 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:32:57.537 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:32:57.537 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:32:57.537 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:32:57.537 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:32:57.537 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:32:57.537 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:32:57.537 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:32:57.537 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:32:57.537 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:32:57.537 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:32:57.537 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:32:57.537 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:32:57.537 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:32:57.537 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:32:57.537 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:32:57.537 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:32:57.537 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:32:57.537 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:32:57.537 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:32:57.537 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:32:57.537 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:32:57.537 | INFO | ------------------------------------------ | 20260507 01:32:57.537 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:32:57.537 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:32:57.537 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:32:57.537 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:32:57.537 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.55e+01, 1.29e+01 / 1.92e+00 | 20260507 01:32:57.537 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.67e-03 / 5.64e-03 | 20260507 01:32:57.537 | INFO | PerfReport: 11.68%, Baseline -- 3.61e+00 / 4.46e+00, 5.35e-01 / 2.91e-01 | 20260507 01:32:57.537 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.46e+00, 1.75e-01 / 1.29e-01 | 20260507 01:32:57.537 | INFO | PerfReport: 16.06%, DME -- 3.97e+01 / 4.43e+01, 3.93e+00 / 2.43e+00 | 20260507 01:32:57.537 | INFO | PerfReport: 57.54%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.34e+00 / 1.95e+00 | 20260507 01:32:57.537 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.16e+00 / 2.72e+00, 3.82e-01 / 2.14e-01 | 20260507 01:32:57.537 | INFO | PerfReport: 3.11%, Metrics -- 9.59e-01 / 1.29e+00, 2.55e-01 / 1.57e-01 | 20260507 01:32:57.537 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.26e-01 / 3.77e-01, 5.13e-02 / 1.96e-02 | 20260507 01:32:57.537 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:32:57.537 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:32:57.537 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:32:57.537 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 01:32:57.537 | INFO | PerfReport: 21.54%, Upload -- 2.08e+01 / 0.00e+00, 6.88e+00 / 0.00e+00 | 20260507 01:32:57.538 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 01:32:57.538 | INFO | PerfReport: 32.04%, Compute -- 3.09e+01 / 0.00e+00, 1.29e+01 / 0.00e+00 | 20260507 01:32:57.538 | INFO | PerfReport: 1.28%, LockUpload -- 1.23e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 01:32:57.538 | INFO | PerfReport: 1.43%, LockDownload -- 1.38e+00 / 0.00e+00, 3.36e+00 / 0.00e+00 | 20260507 01:32:57.538 | INFO | PerfReport: 22.34%, LockCompute -- 2.15e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260507 01:32:57.538 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.40e-02 / 0.00e+00 | 20260507 01:32:57.633 | INFO | Starting a round of Baz IO | 20260507 01:33:00.775 | INFO | Populating RTMetrics report | 20260507 01:33:00.775 | INFO | RTMetrics report populated | 20260507 01:33:00.775 | INFO | Opening tmp RTMetrics json file | 20260507 01:33:00.776 | INFO | Serializing RTMetrics to json | 20260507 01:33:00.777 | INFO | Serialization done, swapping tmp file now... | 20260507 01:33:00.778 | INFO | RTMetrics done | 20260507 01:33:00.870 | INFO | Finished a round of Baz IO: Wrote 1.81 GiB in 3.236 seconds | 20260507 01:33:00.870 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.12% / 0.00% / 66.88% | 20260507 01:33:00.870 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.280301% / 3.97364e-06% / 7.94729e-05% | 20260507 01:33:00.870 | INFO | Baz IO overall: | 20260507 01:33:00.870 | INFO | BytesWritten : 3134799700638 | 20260507 01:33:00.870 | INFO | NumEvents : 1508748191068 | 20260507 01:33:02.672 | INFO | Analyzing chunk frames = [6045696,6046208) | 20260507 01:33:02.672 | INFO | MemInfo: { | 20260507 01:33:02.672 | INFO | "gpu" : " 17 GB", | 20260507 01:33:02.672 | INFO | "huge" : "", | 20260507 01:33:02.672 | INFO | "huge_shared" : " 27 GB", | 20260507 01:33:02.672 | INFO | "malloc" : " 14 GB", | 20260507 01:33:02.672 | INFO | "pinned" : " 29 GB" | 20260507 01:33:02.672 | INFO | } | 20260507 01:33:02.672 | INFO | Perf report for the last 5.13602 seconds: | 20260507 01:33:02.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:33:02.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:02.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:02.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:02.672 | INFO | ANALYSIS : 62.3%, 2.9, 94.9ms, 0 | 20260507 01:33:02.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:02.672 | INFO | RT_METRICS : 44.7%, 1.0, 23.4ms, 0 | 20260507 01:33:02.672 | INFO | PRE_HQ : 60.7%, 1.9, 60.0ms, 0 | 20260507 01:33:02.672 | INFO | BAZWRITER : 62.6%, 3.8, 122.7ms, 0 | 20260507 01:33:02.673 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260507 01:33:02.673 | INFO | Uploaded 36GiB to GPU in 2.07531s (17.3468GiB/s) | 20260507 01:33:02.673 | INFO | Downloaded 38GiB to GPU in 2.17454s (17.4749GiB/s) | 20260507 01:33:07.813 | INFO | Analyzing chunk frames = [6046208,6046720) | 20260507 01:33:07.813 | INFO | MemInfo: { | 20260507 01:33:07.813 | INFO | "gpu" : " 17 GB", | 20260507 01:33:07.813 | INFO | "huge" : "", | 20260507 01:33:07.813 | INFO | "huge_shared" : " 27 GB", | 20260507 01:33:07.813 | INFO | "malloc" : " 14 GB", | 20260507 01:33:07.813 | INFO | "pinned" : " 29 GB" | 20260507 01:33:07.813 | INFO | } | 20260507 01:33:07.813 | INFO | Perf report for the last 5.14077 seconds: | 20260507 01:33:07.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:33:07.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:07.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:07.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:07.813 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 01:33:07.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:07.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:07.813 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 01:33:07.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:07.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:07.813 | INFO | Uploaded 36GiB to GPU in 1.99512s (18.044GiB/s) | 20260507 01:33:07.813 | INFO | Downloaded 34GiB to GPU in 1.92311s (17.6797GiB/s) | 20260507 01:33:12.951 | INFO | Analyzing chunk frames = [6046720,6047232) | 20260507 01:33:12.951 | INFO | MemInfo: { | 20260507 01:33:12.951 | INFO | "gpu" : " 17 GB", | 20260507 01:33:12.951 | INFO | "huge" : "", | 20260507 01:33:12.951 | INFO | "huge_shared" : " 27 GB", | 20260507 01:33:12.951 | INFO | "malloc" : " 14 GB", | 20260507 01:33:12.951 | INFO | "pinned" : " 29 GB" | 20260507 01:33:12.951 | INFO | } | 20260507 01:33:12.951 | INFO | Perf report for the last 5.13886 seconds: | 20260507 01:33:12.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:33:12.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:12.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:12.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:12.952 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 01:33:12.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:12.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:12.952 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:33:12.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:12.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:12.952 | INFO | Uploaded 36GiB to GPU in 1.9907s (18.0841GiB/s) | 20260507 01:33:12.952 | INFO | Downloaded 34GiB to GPU in 1.91848s (17.7223GiB/s) | 20260507 01:33:18.087 | INFO | Analyzing chunk frames = [6047232,6047744) | 20260507 01:33:18.087 | INFO | MemInfo: { | 20260507 01:33:18.087 | INFO | "gpu" : " 17 GB", | 20260507 01:33:18.087 | INFO | "huge" : "", | 20260507 01:33:18.087 | INFO | "huge_shared" : " 27 GB", | 20260507 01:33:18.087 | INFO | "malloc" : " 14 GB", | 20260507 01:33:18.087 | INFO | "pinned" : " 29 GB" | 20260507 01:33:18.087 | INFO | } | 20260507 01:33:18.087 | INFO | Perf report for the last 5.1357 seconds: | 20260507 01:33:18.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:33:18.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:18.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:18.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:18.087 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 01:33:18.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:18.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:18.087 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:33:18.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:18.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:18.088 | INFO | Uploaded 36GiB to GPU in 1.97481s (18.2296GiB/s) | 20260507 01:33:18.088 | INFO | Downloaded 34GiB to GPU in 1.90886s (17.8117GiB/s) | 20260507 01:33:23.222 | INFO | Analyzing chunk frames = [6047744,6048256) | 20260507 01:33:23.222 | INFO | MemInfo: { | 20260507 01:33:23.222 | INFO | "gpu" : " 17 GB", | 20260507 01:33:23.222 | INFO | "huge" : "", | 20260507 01:33:23.222 | INFO | "huge_shared" : " 27 GB", | 20260507 01:33:23.222 | INFO | "malloc" : " 14 GB", | 20260507 01:33:23.222 | INFO | "pinned" : " 29 GB" | 20260507 01:33:23.222 | INFO | } | 20260507 01:33:23.222 | INFO | Perf report for the last 5.13475 seconds: | 20260507 01:33:23.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:33:23.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:23.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:23.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:23.222 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 01:33:23.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:23.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:23.222 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:33:23.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:23.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:23.222 | INFO | Uploaded 36GiB to GPU in 2.00945s (17.9153GiB/s) | 20260507 01:33:23.222 | INFO | Downloaded 34GiB to GPU in 1.94482s (17.4823GiB/s) | 20260507 01:33:28.364 | INFO | Analyzing chunk frames = [6048256,6048768) | 20260507 01:33:28.364 | INFO | MemInfo: { | 20260507 01:33:28.364 | INFO | "gpu" : " 17 GB", | 20260507 01:33:28.364 | INFO | "huge" : "", | 20260507 01:33:28.364 | INFO | "huge_shared" : " 27 GB", | 20260507 01:33:28.364 | INFO | "malloc" : " 14 GB", | 20260507 01:33:28.364 | INFO | "pinned" : " 29 GB" | 20260507 01:33:28.364 | INFO | } | 20260507 01:33:28.364 | INFO | Perf report for the last 5.14243 seconds: | 20260507 01:33:28.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:33:28.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:28.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:28.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:28.365 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 01:33:28.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:28.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:28.365 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:33:28.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:28.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:28.365 | INFO | Uploaded 36GiB to GPU in 2.01401s (17.8748GiB/s) | 20260507 01:33:28.365 | INFO | Downloaded 34GiB to GPU in 1.94337s (17.4954GiB/s) | 20260507 01:33:33.515 | INFO | Analyzing chunk frames = [6048768,6049280) | 20260507 01:33:33.515 | INFO | MemInfo: { | 20260507 01:33:33.515 | INFO | "gpu" : " 17 GB", | 20260507 01:33:33.515 | INFO | "huge" : "", | 20260507 01:33:33.515 | INFO | "huge_shared" : " 27 GB", | 20260507 01:33:33.515 | INFO | "malloc" : " 14 GB", | 20260507 01:33:33.515 | INFO | "pinned" : " 29 GB" | 20260507 01:33:33.515 | INFO | } | 20260507 01:33:33.515 | INFO | Perf report for the last 5.15047 seconds: | 20260507 01:33:33.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:33:33.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:33.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:33.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:33.515 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 01:33:33.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:33.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:33.515 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 01:33:33.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:33.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:33.515 | INFO | Uploaded 36GiB to GPU in 1.97029s (18.2714GiB/s) | 20260507 01:33:33.515 | INFO | Downloaded 34GiB to GPU in 1.98471s (17.1309GiB/s) | 20260507 01:33:38.653 | INFO | Analyzing chunk frames = [6049280,6049792) | 20260507 01:33:38.653 | INFO | MemInfo: { | 20260507 01:33:38.653 | INFO | "gpu" : " 17 GB", | 20260507 01:33:38.653 | INFO | "huge" : "", | 20260507 01:33:38.653 | INFO | "huge_shared" : " 27 GB", | 20260507 01:33:38.653 | INFO | "malloc" : " 14 GB", | 20260507 01:33:38.653 | INFO | "pinned" : " 29 GB" | 20260507 01:33:38.653 | INFO | } | 20260507 01:33:38.653 | INFO | Perf report for the last 5.13833 seconds: | 20260507 01:33:38.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:33:38.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:38.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:38.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:38.653 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:33:38.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:38.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:38.653 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:33:38.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:38.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:38.653 | INFO | Uploaded 36GiB to GPU in 2.00952s (17.9147GiB/s) | 20260507 01:33:38.654 | INFO | Downloaded 34GiB to GPU in 1.97015s (17.2576GiB/s) | 20260507 01:33:38.750 | INFO | Starting a round of Baz IO | 20260507 01:33:41.856 | INFO | Populating RTMetrics report | 20260507 01:33:41.856 | INFO | RTMetrics report populated | 20260507 01:33:41.856 | INFO | Opening tmp RTMetrics json file | 20260507 01:33:41.857 | INFO | Serializing RTMetrics to json | 20260507 01:33:41.858 | INFO | Serialization done, swapping tmp file now... | 20260507 01:33:41.858 | INFO | RTMetrics done | 20260507 01:33:41.948 | INFO | Finished a round of Baz IO: Wrote 1.81 GiB in 3.197 seconds | 20260507 01:33:41.948 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.12% / 0.00% / 66.88% | 20260507 01:33:41.948 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.279415% / 3.97364e-06% / 8.74201e-05% | 20260507 01:33:41.948 | INFO | Baz IO overall: | 20260507 01:33:41.948 | INFO | BytesWritten : 3136743097520 | 20260507 01:33:41.948 | INFO | NumEvents : 1509669158532 | 20260507 01:33:43.790 | INFO | Analyzing chunk frames = [6049792,6050304) | 20260507 01:33:43.791 | INFO | MemInfo: { | 20260507 01:33:43.791 | INFO | "gpu" : " 17 GB", | 20260507 01:33:43.791 | INFO | "huge" : "", | 20260507 01:33:43.791 | INFO | "huge_shared" : " 27 GB", | 20260507 01:33:43.791 | INFO | "malloc" : " 14 GB", | 20260507 01:33:43.791 | INFO | "pinned" : " 29 GB" | 20260507 01:33:43.791 | INFO | } | 20260507 01:33:43.791 | INFO | Perf report for the last 5.13746 seconds: | 20260507 01:33:43.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:33:43.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:43.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:43.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:43.791 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260507 01:33:43.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:43.791 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 01:33:43.791 | INFO | PRE_HQ : 59.3%, 1.8, 55.9ms, 0 | 20260507 01:33:43.791 | INFO | BAZWRITER : 61.8%, 3.4, 107.7ms, 0 | 20260507 01:33:43.791 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:33:43.791 | INFO | Uploaded 36GiB to GPU in 2.04745s (17.5828GiB/s) | 20260507 01:33:43.791 | INFO | Downloaded 38GiB to GPU in 2.1786s (17.4424GiB/s) | 20260507 01:33:48.925 | INFO | Analyzing chunk frames = [6050304,6050816) | 20260507 01:33:48.925 | INFO | MemInfo: { | 20260507 01:33:48.925 | INFO | "gpu" : " 17 GB", | 20260507 01:33:48.925 | INFO | "huge" : "", | 20260507 01:33:48.925 | INFO | "huge_shared" : " 27 GB", | 20260507 01:33:48.925 | INFO | "malloc" : " 14 GB", | 20260507 01:33:48.925 | INFO | "pinned" : " 29 GB" | 20260507 01:33:48.925 | INFO | } | 20260507 01:33:48.925 | INFO | Perf report for the last 5.13461 seconds: | 20260507 01:33:48.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:33:48.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:48.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:48.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:48.925 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260507 01:33:48.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:48.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:48.925 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:33:48.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:48.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:48.926 | INFO | Uploaded 36GiB to GPU in 2.05285s (17.5366GiB/s) | 20260507 01:33:48.926 | INFO | Downloaded 34GiB to GPU in 1.9411s (17.5159GiB/s) | 20260507 01:33:54.065 | INFO | Analyzing chunk frames = [6050816,6051328) | 20260507 01:33:54.065 | INFO | MemInfo: { | 20260507 01:33:54.065 | INFO | "gpu" : " 17 GB", | 20260507 01:33:54.065 | INFO | "huge" : "", | 20260507 01:33:54.065 | INFO | "huge_shared" : " 27 GB", | 20260507 01:33:54.065 | INFO | "malloc" : " 14 GB", | 20260507 01:33:54.065 | INFO | "pinned" : " 29 GB" | 20260507 01:33:54.065 | INFO | } | 20260507 01:33:54.065 | INFO | Perf report for the last 5.13945 seconds: | 20260507 01:33:54.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:33:54.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:54.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:54.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:54.065 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:33:54.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:54.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:54.065 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:33:54.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:54.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:54.065 | INFO | Uploaded 36GiB to GPU in 1.98521s (18.1341GiB/s) | 20260507 01:33:54.065 | INFO | Downloaded 34GiB to GPU in 1.93036s (17.6133GiB/s) | 20260507 01:33:59.207 | INFO | Analyzing chunk frames = [6051328,6051840) | 20260507 01:33:59.207 | INFO | MemInfo: { | 20260507 01:33:59.207 | INFO | "gpu" : " 17 GB", | 20260507 01:33:59.207 | INFO | "huge" : "", | 20260507 01:33:59.207 | INFO | "huge_shared" : " 27 GB", | 20260507 01:33:59.207 | INFO | "malloc" : " 14 GB", | 20260507 01:33:59.207 | INFO | "pinned" : " 29 GB" | 20260507 01:33:59.207 | INFO | } | 20260507 01:33:59.207 | INFO | Perf report for the last 5.14191 seconds: | 20260507 01:33:59.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:33:59.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:59.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:59.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:59.207 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260507 01:33:59.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:59.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:59.207 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:33:59.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:59.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:33:59.207 | INFO | Uploaded 36GiB to GPU in 2.00949s (17.915GiB/s) | 20260507 01:33:59.207 | INFO | Downloaded 34GiB to GPU in 1.90845s (17.8155GiB/s) | 20260507 01:34:04.344 | INFO | Analyzing chunk frames = [6051840,6052352) | 20260507 01:34:04.344 | INFO | MemInfo: { | 20260507 01:34:04.344 | INFO | "gpu" : " 17 GB", | 20260507 01:34:04.344 | INFO | "huge" : "", | 20260507 01:34:04.344 | INFO | "huge_shared" : " 27 GB", | 20260507 01:34:04.344 | INFO | "malloc" : " 14 GB", | 20260507 01:34:04.344 | INFO | "pinned" : " 29 GB" | 20260507 01:34:04.344 | INFO | } | 20260507 01:34:04.344 | INFO | Perf report for the last 5.13705 seconds: | 20260507 01:34:04.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:34:04.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:04.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:04.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:04.344 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 01:34:04.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:04.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:04.344 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:34:04.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:04.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:04.344 | INFO | Uploaded 36GiB to GPU in 2.01863s (17.8339GiB/s) | 20260507 01:34:04.344 | INFO | Downloaded 34GiB to GPU in 1.93796s (17.5443GiB/s) | 20260507 01:34:09.480 | INFO | Analyzing chunk frames = [6052352,6052864) | 20260507 01:34:09.480 | INFO | MemInfo: { | 20260507 01:34:09.480 | INFO | "gpu" : " 17 GB", | 20260507 01:34:09.480 | INFO | "huge" : "", | 20260507 01:34:09.480 | INFO | "huge_shared" : " 27 GB", | 20260507 01:34:09.480 | INFO | "malloc" : " 14 GB", | 20260507 01:34:09.480 | INFO | "pinned" : " 29 GB" | 20260507 01:34:09.480 | INFO | } | 20260507 01:34:09.480 | INFO | Perf report for the last 5.13646 seconds: | 20260507 01:34:09.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:34:09.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:09.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:09.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:09.480 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:34:09.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:09.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:09.481 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 01:34:09.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:09.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:09.481 | INFO | Uploaded 36GiB to GPU in 2.05757s (17.4964GiB/s) | 20260507 01:34:09.481 | INFO | Downloaded 34GiB to GPU in 1.95985s (17.3483GiB/s) | 20260507 01:34:14.626 | INFO | Analyzing chunk frames = [6052864,6053376) | 20260507 01:34:14.626 | INFO | MemInfo: { | 20260507 01:34:14.626 | INFO | "gpu" : " 17 GB", | 20260507 01:34:14.626 | INFO | "huge" : "", | 20260507 01:34:14.626 | INFO | "huge_shared" : " 27 GB", | 20260507 01:34:14.626 | INFO | "malloc" : " 14 GB", | 20260507 01:34:14.626 | INFO | "pinned" : " 29 GB" | 20260507 01:34:14.626 | INFO | } | 20260507 01:34:14.626 | INFO | Perf report for the last 5.14596 seconds: | 20260507 01:34:14.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:34:14.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:14.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:14.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:14.626 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:34:14.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:14.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:14.626 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 01:34:14.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:14.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:14.626 | INFO | Uploaded 36GiB to GPU in 1.98767s (18.1117GiB/s) | 20260507 01:34:14.627 | INFO | Downloaded 34GiB to GPU in 2.00417s (16.9646GiB/s) | 20260507 01:34:19.762 | INFO | Analyzing chunk frames = [6053376,6053888) | 20260507 01:34:19.762 | INFO | MemInfo: { | 20260507 01:34:19.762 | INFO | "gpu" : " 17 GB", | 20260507 01:34:19.762 | INFO | "huge" : "", | 20260507 01:34:19.762 | INFO | "huge_shared" : " 27 GB", | 20260507 01:34:19.762 | INFO | "malloc" : " 14 GB", | 20260507 01:34:19.762 | INFO | "pinned" : " 29 GB" | 20260507 01:34:19.762 | INFO | } | 20260507 01:34:19.762 | INFO | Perf report for the last 5.13602 seconds: | 20260507 01:34:19.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:34:19.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:19.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:19.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:19.762 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:34:19.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:19.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:19.762 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 01:34:19.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:19.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:19.763 | INFO | Uploaded 36GiB to GPU in 2.00841s (17.9246GiB/s) | 20260507 01:34:19.763 | INFO | Downloaded 34GiB to GPU in 1.96936s (17.2645GiB/s) | 20260507 01:34:19.763 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:34:19.763 | INFO | Total mem usage: 18 GiB | 20260507 01:34:19.763 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:34:19.763 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -20.3%) | 20260507 01:34:19.763 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:34:19.763 | INFO | High Water Breakdown: | 20260507 01:34:19.763 | INFO | 61% : 9.13 GiB ( 80% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:34:19.763 | INFO | 38.9% : 5.82 GiB ( 86.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:34:19.763 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:34:19.763 | INFO | ------------------------------------------ | 20260507 01:34:19.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:34:19.763 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:34:19.763 | INFO | Total mem usage: 26.5 GiB | 20260507 01:34:19.763 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:34:19.763 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:34:19.763 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:34:19.763 | INFO | High Water Breakdown: | 20260507 01:34:19.763 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:34:19.763 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:34:19.763 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:34:19.763 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:34:19.763 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:34:19.763 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:34:19.763 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:34:19.763 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:34:19.763 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:34:19.763 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:34:19.763 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:34:19.763 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:34:19.763 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:34:19.763 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:34:19.763 | INFO | ------------------------------------------ | 20260507 01:34:19.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:34:19.763 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:34:19.763 | INFO | Total mem usage: 25.6 GiB | 20260507 01:34:19.763 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:34:19.763 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:34:19.763 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:34:19.763 | INFO | High Water Breakdown: | 20260507 01:34:19.763 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:34:19.763 | INFO | ------------------------------------------ | 20260507 01:34:19.763 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:34:19.763 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:34:19.763 | INFO | Total mem usage: 15.7 GiB | 20260507 01:34:19.763 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:34:19.763 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:34:19.763 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:34:19.763 | INFO | High Water Breakdown: | 20260507 01:34:19.763 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:34:19.763 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:34:19.763 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:34:19.763 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:34:19.763 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:34:19.763 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:34:19.763 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:34:19.763 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:34:19.763 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:34:19.763 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:34:19.763 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:34:19.764 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:34:19.764 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:34:19.764 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:34:19.764 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:34:19.764 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:34:19.764 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:34:19.764 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:34:19.764 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:34:19.764 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:34:19.764 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:34:19.764 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:34:19.764 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:34:19.764 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:34:19.764 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:34:19.764 | INFO | ------------------------------------------ | 20260507 01:34:19.764 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:34:19.764 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:34:19.764 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:34:19.764 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:34:19.764 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.58e+01, 1.36e+01 / 2.26e+00 | 20260507 01:34:19.764 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.00e-01, 9.94e-03 / 6.32e-03 | 20260507 01:34:19.764 | INFO | PerfReport: 11.65%, Baseline -- 3.59e+00 / 4.41e+00, 5.17e-01 / 3.06e-01 | 20260507 01:34:19.764 | INFO | PerfReport: 3.24%, Histogram -- 9.99e-01 / 1.48e+00, 1.73e-01 / 1.13e-01 | 20260507 01:34:19.764 | INFO | PerfReport: 16.14%, DME -- 3.98e+01 / 4.47e+01, 4.08e+00 / 2.78e+00 | 20260507 01:34:19.764 | INFO | PerfReport: 57.48%, FrameLabeling -- 1.77e+01 / 2.09e+01, 1.25e+00 / 1.60e+00 | 20260507 01:34:19.764 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.16e+00 / 2.67e+00, 3.80e-01 / 2.39e-01 | 20260507 01:34:19.764 | INFO | PerfReport: 3.11%, Metrics -- 9.59e-01 / 1.23e+00, 2.59e-01 / 1.25e-01 | 20260507 01:34:19.764 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.25e-01 / 3.69e-01, 5.17e-02 / 1.98e-02 | 20260507 01:34:19.764 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:34:19.764 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:34:19.764 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:34:19.764 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 01:34:19.764 | INFO | PerfReport: 21.76%, Upload -- 2.10e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260507 01:34:19.764 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 01:34:19.764 | INFO | PerfReport: 31.99%, Compute -- 3.08e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260507 01:34:19.764 | INFO | PerfReport: 1.49%, LockUpload -- 1.44e+00 / 0.00e+00, 5.07e+00 / 0.00e+00 | 20260507 01:34:19.764 | INFO | PerfReport: 1.21%, LockDownload -- 1.16e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260507 01:34:19.764 | INFO | PerfReport: 22.18%, LockCompute -- 2.14e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 01:34:19.764 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260507 01:34:19.860 | INFO | Starting a round of Baz IO | 20260507 01:34:22.972 | INFO | Populating RTMetrics report | 20260507 01:34:22.973 | INFO | RTMetrics report populated | 20260507 01:34:22.973 | INFO | Opening tmp RTMetrics json file | 20260507 01:34:22.973 | INFO | Serializing RTMetrics to json | 20260507 01:34:22.975 | INFO | Serialization done, swapping tmp file now... | 20260507 01:34:22.975 | INFO | RTMetrics done | 20260507 01:34:23.068 | INFO | Finished a round of Baz IO: Wrote 1.809 GiB in 3.208 seconds | 20260507 01:34:23.068 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.12% / 0.00% / 66.88% | 20260507 01:34:23.068 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.279287% / 0% / 5.16574e-05% | 20260507 01:34:23.068 | INFO | Baz IO overall: | 20260507 01:34:23.068 | INFO | BytesWritten : 3138685373946 | 20260507 01:34:23.068 | INFO | NumEvents : 1510589354470 | 20260507 01:34:24.903 | INFO | Analyzing chunk frames = [6053888,6054400) | 20260507 01:34:24.903 | INFO | MemInfo: { | 20260507 01:34:24.903 | INFO | "gpu" : " 17 GB", | 20260507 01:34:24.903 | INFO | "huge" : "", | 20260507 01:34:24.903 | INFO | "huge_shared" : " 27 GB", | 20260507 01:34:24.903 | INFO | "malloc" : " 14 GB", | 20260507 01:34:24.903 | INFO | "pinned" : " 29 GB" | 20260507 01:34:24.903 | INFO | } | 20260507 01:34:24.903 | INFO | Perf report for the last 5.14085 seconds: | 20260507 01:34:24.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:34:24.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:24.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:24.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:24.903 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260507 01:34:24.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:24.903 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 01:34:24.903 | INFO | PRE_HQ : 59.3%, 1.8, 56.2ms, 0 | 20260507 01:34:24.903 | INFO | BAZWRITER : 62.0%, 3.3, 105.4ms, 0 | 20260507 01:34:24.903 | INFO | BAZ_SAMPLING : 1.3%, 1.0, 0.7ms, 0 | 20260507 01:34:24.903 | INFO | Uploaded 36GiB to GPU in 2.04181s (17.6314GiB/s) | 20260507 01:34:24.903 | INFO | Downloaded 38GiB to GPU in 2.18652s (17.3792GiB/s) | 20260507 01:34:30.035 | INFO | Analyzing chunk frames = [6054400,6054912) | 20260507 01:34:30.035 | INFO | MemInfo: { | 20260507 01:34:30.035 | INFO | "gpu" : " 17 GB", | 20260507 01:34:30.035 | INFO | "huge" : "", | 20260507 01:34:30.035 | INFO | "huge_shared" : " 27 GB", | 20260507 01:34:30.035 | INFO | "malloc" : " 14 GB", | 20260507 01:34:30.035 | INFO | "pinned" : " 29 GB" | 20260507 01:34:30.035 | INFO | } | 20260507 01:34:30.035 | INFO | Perf report for the last 5.13157 seconds: | 20260507 01:34:30.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:34:30.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:30.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:30.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:30.035 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 01:34:30.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:30.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:30.035 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:34:30.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:30.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:30.035 | INFO | Uploaded 36GiB to GPU in 2.03784s (17.6658GiB/s) | 20260507 01:34:30.035 | INFO | Downloaded 34GiB to GPU in 1.93865s (17.538GiB/s) | 20260507 01:34:35.175 | INFO | Analyzing chunk frames = [6054912,6055424) | 20260507 01:34:35.175 | INFO | MemInfo: { | 20260507 01:34:35.175 | INFO | "gpu" : " 17 GB", | 20260507 01:34:35.175 | INFO | "huge" : "", | 20260507 01:34:35.175 | INFO | "huge_shared" : " 27 GB", | 20260507 01:34:35.175 | INFO | "malloc" : " 14 GB", | 20260507 01:34:35.175 | INFO | "pinned" : " 29 GB" | 20260507 01:34:35.175 | INFO | } | 20260507 01:34:35.175 | INFO | Perf report for the last 5.14008 seconds: | 20260507 01:34:35.175 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:34:35.175 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:35.175 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:35.175 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:35.175 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 01:34:35.175 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:35.175 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:35.175 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:34:35.175 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:35.175 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:35.175 | INFO | Uploaded 36GiB to GPU in 1.9802s (18.18GiB/s) | 20260507 01:34:35.175 | INFO | Downloaded 34GiB to GPU in 1.92875s (17.628GiB/s) | 20260507 01:34:40.317 | INFO | Analyzing chunk frames = [6055424,6055936) | 20260507 01:34:40.317 | INFO | MemInfo: { | 20260507 01:34:40.317 | INFO | "gpu" : " 17 GB", | 20260507 01:34:40.317 | INFO | "huge" : "", | 20260507 01:34:40.317 | INFO | "huge_shared" : " 27 GB", | 20260507 01:34:40.317 | INFO | "malloc" : " 14 GB", | 20260507 01:34:40.317 | INFO | "pinned" : " 29 GB" | 20260507 01:34:40.317 | INFO | } | 20260507 01:34:40.317 | INFO | Perf report for the last 5.14256 seconds: | 20260507 01:34:40.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:34:40.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:40.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:40.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:40.317 | INFO | ANALYSIS : 59.4%, 3.0, 92.2ms, 0 | 20260507 01:34:40.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:40.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:40.317 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:34:40.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:40.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:40.317 | INFO | Uploaded 36GiB to GPU in 1.97301s (18.2462GiB/s) | 20260507 01:34:40.318 | INFO | Downloaded 34GiB to GPU in 1.87941s (18.0908GiB/s) | 20260507 01:34:45.464 | INFO | Analyzing chunk frames = [6055936,6056448) | 20260507 01:34:45.464 | INFO | MemInfo: { | 20260507 01:34:45.464 | INFO | "gpu" : " 17 GB", | 20260507 01:34:45.464 | INFO | "huge" : "", | 20260507 01:34:45.464 | INFO | "huge_shared" : " 27 GB", | 20260507 01:34:45.464 | INFO | "malloc" : " 14 GB", | 20260507 01:34:45.464 | INFO | "pinned" : " 29 GB" | 20260507 01:34:45.464 | INFO | } | 20260507 01:34:45.464 | INFO | Perf report for the last 5.14652 seconds: | 20260507 01:34:45.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:34:45.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:45.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:45.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:45.464 | INFO | ANALYSIS : 59.9%, 3.0, 93.4ms, 0 | 20260507 01:34:45.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:45.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:45.464 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:34:45.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:45.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:45.464 | INFO | Uploaded 36GiB to GPU in 1.99926s (18.0067GiB/s) | 20260507 01:34:45.464 | INFO | Downloaded 34GiB to GPU in 1.93547s (17.5668GiB/s) | 20260507 01:34:50.599 | INFO | Analyzing chunk frames = [6056448,6056960) | 20260507 01:34:50.599 | INFO | MemInfo: { | 20260507 01:34:50.599 | INFO | "gpu" : " 17 GB", | 20260507 01:34:50.599 | INFO | "huge" : "", | 20260507 01:34:50.599 | INFO | "huge_shared" : " 27 GB", | 20260507 01:34:50.599 | INFO | "malloc" : " 14 GB", | 20260507 01:34:50.599 | INFO | "pinned" : " 29 GB" | 20260507 01:34:50.599 | INFO | } | 20260507 01:34:50.599 | INFO | Perf report for the last 5.13526 seconds: | 20260507 01:34:50.599 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:34:50.599 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:50.599 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:50.599 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:50.599 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 01:34:50.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:50.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:50.599 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:34:50.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:50.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:50.599 | INFO | Uploaded 36GiB to GPU in 2.03814s (17.6632GiB/s) | 20260507 01:34:50.599 | INFO | Downloaded 34GiB to GPU in 1.93192s (17.599GiB/s) | 20260507 01:34:55.734 | INFO | Analyzing chunk frames = [6056960,6057472) | 20260507 01:34:55.734 | INFO | MemInfo: { | 20260507 01:34:55.734 | INFO | "gpu" : " 17 GB", | 20260507 01:34:55.734 | INFO | "huge" : "", | 20260507 01:34:55.734 | INFO | "huge_shared" : " 27 GB", | 20260507 01:34:55.734 | INFO | "malloc" : " 14 GB", | 20260507 01:34:55.734 | INFO | "pinned" : " 29 GB" | 20260507 01:34:55.734 | INFO | } | 20260507 01:34:55.735 | INFO | Perf report for the last 5.13539 seconds: | 20260507 01:34:55.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:34:55.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:55.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:55.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:55.735 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:34:55.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:55.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:55.735 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 01:34:55.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:55.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:34:55.735 | INFO | Uploaded 36GiB to GPU in 1.97851s (18.1955GiB/s) | 20260507 01:34:55.735 | INFO | Downloaded 34GiB to GPU in 1.95577s (17.3845GiB/s) | 20260507 01:35:00.879 | INFO | Analyzing chunk frames = [6057472,6057984) | 20260507 01:35:00.879 | INFO | MemInfo: { | 20260507 01:35:00.879 | INFO | "gpu" : " 17 GB", | 20260507 01:35:00.879 | INFO | "huge" : "", | 20260507 01:35:00.879 | INFO | "huge_shared" : " 27 GB", | 20260507 01:35:00.879 | INFO | "malloc" : " 14 GB", | 20260507 01:35:00.879 | INFO | "pinned" : " 29 GB" | 20260507 01:35:00.879 | INFO | } | 20260507 01:35:00.879 | INFO | Perf report for the last 5.14432 seconds: | 20260507 01:35:00.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:35:00.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:00.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:00.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:00.879 | INFO | ANALYSIS : 61.1%, 3.0, 95.3ms, 0 | 20260507 01:35:00.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:00.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:00.879 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 01:35:00.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:00.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:00.879 | INFO | Uploaded 36GiB to GPU in 2.00775s (17.9305GiB/s) | 20260507 01:35:00.879 | INFO | Downloaded 34GiB to GPU in 1.96072s (17.3405GiB/s) | 20260507 01:35:00.977 | INFO | Starting a round of Baz IO | 20260507 01:35:04.096 | INFO | Populating RTMetrics report | 20260507 01:35:04.096 | INFO | RTMetrics report populated | 20260507 01:35:04.096 | INFO | Opening tmp RTMetrics json file | 20260507 01:35:04.097 | INFO | Serializing RTMetrics to json | 20260507 01:35:04.113 | INFO | Serialization done, swapping tmp file now... | 20260507 01:35:04.116 | INFO | RTMetrics done | 20260507 01:35:04.185 | INFO | Finished a round of Baz IO: Wrote 1.808 GiB in 3.208 seconds | 20260507 01:35:04.185 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.12% / 0.00% / 66.88% | 20260507 01:35:04.185 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.277817% / 0% / 5.5631e-05% | 20260507 01:35:04.185 | INFO | Baz IO overall: | 20260507 01:35:04.185 | INFO | BytesWritten : 3140626536773 | 20260507 01:35:04.185 | INFO | NumEvents : 1511508914506 | 20260507 01:35:06.016 | INFO | Analyzing chunk frames = [6057984,6058496) | 20260507 01:35:06.016 | INFO | MemInfo: { | 20260507 01:35:06.016 | INFO | "gpu" : " 17 GB", | 20260507 01:35:06.016 | INFO | "huge" : "", | 20260507 01:35:06.016 | INFO | "huge_shared" : " 27 GB", | 20260507 01:35:06.016 | INFO | "malloc" : " 14 GB", | 20260507 01:35:06.016 | INFO | "pinned" : " 29 GB" | 20260507 01:35:06.016 | INFO | } | 20260507 01:35:06.016 | INFO | Perf report for the last 5.13723 seconds: | 20260507 01:35:06.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:35:06.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:06.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:06.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:06.016 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260507 01:35:06.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:06.016 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 01:35:06.016 | INFO | PRE_HQ : 59.6%, 1.8, 56.3ms, 0 | 20260507 01:35:06.016 | INFO | BAZWRITER : 62.0%, 3.4, 108.8ms, 0 | 20260507 01:35:06.016 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:35:06.016 | INFO | Uploaded 36GiB to GPU in 2.03197s (17.7168GiB/s) | 20260507 01:35:06.016 | INFO | Downloaded 38GiB to GPU in 2.17961s (17.4343GiB/s) | 20260507 01:35:11.155 | INFO | Analyzing chunk frames = [6058496,6059008) | 20260507 01:35:11.155 | INFO | MemInfo: { | 20260507 01:35:11.155 | INFO | "gpu" : " 17 GB", | 20260507 01:35:11.155 | INFO | "huge" : "", | 20260507 01:35:11.155 | INFO | "huge_shared" : " 27 GB", | 20260507 01:35:11.155 | INFO | "malloc" : " 14 GB", | 20260507 01:35:11.155 | INFO | "pinned" : " 29 GB" | 20260507 01:35:11.155 | INFO | } | 20260507 01:35:11.155 | INFO | Perf report for the last 5.13906 seconds: | 20260507 01:35:11.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:35:11.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:11.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:11.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:11.155 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:35:11.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:11.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:11.155 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:35:11.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:11.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:11.155 | INFO | Uploaded 36GiB to GPU in 2.0031s (17.9722GiB/s) | 20260507 01:35:11.155 | INFO | Downloaded 34GiB to GPU in 1.932s (17.5984GiB/s) | 20260507 01:35:16.301 | INFO | Analyzing chunk frames = [6059008,6059520) | 20260507 01:35:16.301 | INFO | MemInfo: { | 20260507 01:35:16.301 | INFO | "gpu" : " 17 GB", | 20260507 01:35:16.301 | INFO | "huge" : "", | 20260507 01:35:16.301 | INFO | "huge_shared" : " 27 GB", | 20260507 01:35:16.301 | INFO | "malloc" : " 14 GB", | 20260507 01:35:16.301 | INFO | "pinned" : " 29 GB" | 20260507 01:35:16.301 | INFO | } | 20260507 01:35:16.301 | INFO | Perf report for the last 5.14598 seconds: | 20260507 01:35:16.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:35:16.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:16.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:16.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:16.301 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 01:35:16.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:16.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:16.301 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:35:16.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:16.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:16.301 | INFO | Uploaded 36GiB to GPU in 2.00256s (17.977GiB/s) | 20260507 01:35:16.302 | INFO | Downloaded 34GiB to GPU in 1.93446s (17.576GiB/s) | 20260507 01:35:21.435 | INFO | Analyzing chunk frames = [6059520,6060032) | 20260507 01:35:21.435 | INFO | MemInfo: { | 20260507 01:35:21.435 | INFO | "gpu" : " 17 GB", | 20260507 01:35:21.435 | INFO | "huge" : "", | 20260507 01:35:21.435 | INFO | "huge_shared" : " 27 GB", | 20260507 01:35:21.435 | INFO | "malloc" : " 14 GB", | 20260507 01:35:21.435 | INFO | "pinned" : " 29 GB" | 20260507 01:35:21.435 | INFO | } | 20260507 01:35:21.435 | INFO | Perf report for the last 5.13389 seconds: | 20260507 01:35:21.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:35:21.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:21.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:21.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:21.435 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260507 01:35:21.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:21.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:21.435 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 01:35:21.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:21.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:21.435 | INFO | Uploaded 36GiB to GPU in 1.99815s (18.0167GiB/s) | 20260507 01:35:21.435 | INFO | Downloaded 34GiB to GPU in 1.92126s (17.6967GiB/s) | 20260507 01:35:26.578 | INFO | Analyzing chunk frames = [6060032,6060544) | 20260507 01:35:26.578 | INFO | MemInfo: { | 20260507 01:35:26.578 | INFO | "gpu" : " 17 GB", | 20260507 01:35:26.578 | INFO | "huge" : "", | 20260507 01:35:26.578 | INFO | "huge_shared" : " 27 GB", | 20260507 01:35:26.578 | INFO | "malloc" : " 14 GB", | 20260507 01:35:26.578 | INFO | "pinned" : " 29 GB" | 20260507 01:35:26.578 | INFO | } | 20260507 01:35:26.578 | INFO | Perf report for the last 5.14311 seconds: | 20260507 01:35:26.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:35:26.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:26.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:26.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:26.578 | INFO | ANALYSIS : 60.2%, 3.0, 94.0ms, 0 | 20260507 01:35:26.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:26.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:26.578 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 01:35:26.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:26.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:26.578 | INFO | Uploaded 36GiB to GPU in 2.02182s (17.8058GiB/s) | 20260507 01:35:26.579 | INFO | Downloaded 34GiB to GPU in 1.94308s (17.498GiB/s) | 20260507 01:35:31.718 | INFO | Analyzing chunk frames = [6060544,6061056) | 20260507 01:35:31.718 | INFO | MemInfo: { | 20260507 01:35:31.718 | INFO | "gpu" : " 17 GB", | 20260507 01:35:31.718 | INFO | "huge" : "", | 20260507 01:35:31.718 | INFO | "huge_shared" : " 27 GB", | 20260507 01:35:31.718 | INFO | "malloc" : " 14 GB", | 20260507 01:35:31.718 | INFO | "pinned" : " 29 GB" | 20260507 01:35:31.718 | INFO | } | 20260507 01:35:31.718 | INFO | Perf report for the last 5.13951 seconds: | 20260507 01:35:31.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:35:31.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:31.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:31.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:31.718 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 01:35:31.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:31.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:31.718 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:35:31.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:31.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:31.718 | INFO | Uploaded 36GiB to GPU in 2.05788s (17.4938GiB/s) | 20260507 01:35:31.718 | INFO | Downloaded 34GiB to GPU in 1.95782s (17.3663GiB/s) | 20260507 01:35:36.849 | INFO | Analyzing chunk frames = [6061056,6061568) | 20260507 01:35:36.849 | INFO | MemInfo: { | 20260507 01:35:36.849 | INFO | "gpu" : " 17 GB", | 20260507 01:35:36.850 | INFO | "huge" : "", | 20260507 01:35:36.850 | INFO | "huge_shared" : " 27 GB", | 20260507 01:35:36.850 | INFO | "malloc" : " 14 GB", | 20260507 01:35:36.850 | INFO | "pinned" : " 29 GB" | 20260507 01:35:36.850 | INFO | } | 20260507 01:35:36.850 | INFO | Perf report for the last 5.13195 seconds: | 20260507 01:35:36.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:35:36.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:36.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:36.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:36.850 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 01:35:36.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:36.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:36.850 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:35:36.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:36.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:36.850 | INFO | Uploaded 36GiB to GPU in 1.97867s (18.194GiB/s) | 20260507 01:35:36.850 | INFO | Downloaded 34GiB to GPU in 1.96812s (17.2753GiB/s) | 20260507 01:35:41.989 | INFO | Analyzing chunk frames = [6061568,6062080) | 20260507 01:35:41.989 | INFO | MemInfo: { | 20260507 01:35:41.989 | INFO | "gpu" : " 17 GB", | 20260507 01:35:41.989 | INFO | "huge" : "", | 20260507 01:35:41.989 | INFO | "huge_shared" : " 27 GB", | 20260507 01:35:41.989 | INFO | "malloc" : " 14 GB", | 20260507 01:35:41.989 | INFO | "pinned" : " 29 GB" | 20260507 01:35:41.989 | INFO | } | 20260507 01:35:41.989 | INFO | Perf report for the last 5.13968 seconds: | 20260507 01:35:41.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:35:41.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:41.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:41.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:41.990 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:35:41.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:41.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:41.990 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 01:35:41.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:41.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:41.990 | INFO | Uploaded 36GiB to GPU in 1.99221s (18.0704GiB/s) | 20260507 01:35:41.990 | INFO | Downloaded 34GiB to GPU in 1.97102s (17.25GiB/s) | 20260507 01:35:41.990 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:35:41.990 | INFO | Total mem usage: 18 GiB | 20260507 01:35:41.990 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:35:41.990 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:35:41.990 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:35:41.990 | INFO | High Water Breakdown: | 20260507 01:35:41.990 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:35:41.990 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:35:41.990 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:35:41.990 | INFO | ------------------------------------------ | 20260507 01:35:41.990 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:35:41.990 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:35:41.990 | INFO | Total mem usage: 26.5 GiB | 20260507 01:35:41.990 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:35:41.990 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:35:41.990 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:35:41.990 | INFO | High Water Breakdown: | 20260507 01:35:41.990 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:35:41.990 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:35:41.990 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:35:41.990 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:35:41.990 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:35:41.990 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:35:41.990 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:35:41.990 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:35:41.990 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:35:41.990 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:35:41.990 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:35:41.990 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:35:41.990 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:35:41.990 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:35:41.990 | INFO | ------------------------------------------ | 20260507 01:35:41.990 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:35:41.990 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:35:41.990 | INFO | Total mem usage: 25.6 GiB | 20260507 01:35:41.990 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:35:41.990 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:35:41.990 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:35:41.990 | INFO | High Water Breakdown: | 20260507 01:35:41.990 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:35:41.990 | INFO | ------------------------------------------ | 20260507 01:35:41.990 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:35:41.990 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:35:41.990 | INFO | Total mem usage: 15.7 GiB | 20260507 01:35:41.991 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:35:41.991 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:35:41.991 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:35:41.991 | INFO | High Water Breakdown: | 20260507 01:35:41.991 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:35:41.991 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:35:41.991 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:35:41.991 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:35:41.991 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:35:41.991 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:35:41.991 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:35:41.991 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:35:41.991 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:35:41.991 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:35:41.991 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:35:41.991 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:35:41.991 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:35:41.991 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:35:41.991 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:35:41.991 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:35:41.991 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:35:41.991 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:35:41.991 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:35:41.991 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:35:41.991 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:35:41.991 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:35:41.991 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:35:41.991 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:35:41.991 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:35:41.991 | INFO | ------------------------------------------ | 20260507 01:35:41.991 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:35:41.991 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:35:41.991 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:35:41.991 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:35:41.991 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.50e+01, 1.33e+01 / 1.65e+00 | 20260507 01:35:41.991 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.03e-01, 9.79e-03 / 1.56e-02 | 20260507 01:35:41.991 | INFO | PerfReport: 11.65%, Baseline -- 3.59e+00 / 4.41e+00, 5.18e-01 / 3.18e-01 | 20260507 01:35:41.991 | INFO | PerfReport: 3.24%, Histogram -- 9.99e-01 / 1.46e+00, 1.72e-01 / 1.32e-01 | 20260507 01:35:41.991 | INFO | PerfReport: 16.07%, DME -- 3.97e+01 / 4.42e+01, 3.84e+00 / 1.94e+00 | 20260507 01:35:41.991 | INFO | PerfReport: 57.60%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.37e+00 / 1.47e+00 | 20260507 01:35:41.991 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.16e+00 / 2.73e+00, 3.75e-01 / 1.97e-01 | 20260507 01:35:41.991 | INFO | PerfReport: 3.10%, Metrics -- 9.55e-01 / 1.31e+00, 2.53e-01 / 1.23e-01 | 20260507 01:35:41.991 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.25e-01 / 3.81e-01, 5.23e-02 / 2.29e-02 | 20260507 01:35:41.991 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:35:41.991 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:35:41.991 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:35:41.991 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 01:35:41.991 | INFO | PerfReport: 21.70%, Upload -- 2.09e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260507 01:35:41.991 | INFO | PerfReport: 21.28%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 01:35:41.991 | INFO | PerfReport: 31.98%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 01:35:41.991 | INFO | PerfReport: 1.47%, LockUpload -- 1.42e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 01:35:41.991 | INFO | PerfReport: 1.19%, LockDownload -- 1.15e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260507 01:35:41.991 | INFO | PerfReport: 22.37%, LockCompute -- 2.16e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 01:35:41.991 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260507 01:35:42.086 | INFO | Starting a round of Baz IO | 20260507 01:35:45.207 | INFO | Populating RTMetrics report | 20260507 01:35:45.207 | INFO | RTMetrics report populated | 20260507 01:35:45.207 | INFO | Opening tmp RTMetrics json file | 20260507 01:35:45.208 | INFO | Serializing RTMetrics to json | 20260507 01:35:45.210 | INFO | Serialization done, swapping tmp file now... | 20260507 01:35:45.215 | INFO | RTMetrics done | 20260507 01:35:45.300 | INFO | Finished a round of Baz IO: Wrote 1.807 GiB in 3.213 seconds | 20260507 01:35:45.300 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.11% / 0.00% / 66.88% | 20260507 01:35:45.300 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.277233% / 0% / 7.15256e-05% | 20260507 01:35:45.300 | INFO | Baz IO overall: | 20260507 01:35:45.300 | INFO | BytesWritten : 3142566949237 | 20260507 01:35:45.300 | INFO | NumEvents : 1512427944729 | 20260507 01:35:47.129 | INFO | Analyzing chunk frames = [6062080,6062592) | 20260507 01:35:47.129 | INFO | MemInfo: { | 20260507 01:35:47.129 | INFO | "gpu" : " 17 GB", | 20260507 01:35:47.129 | INFO | "huge" : "", | 20260507 01:35:47.129 | INFO | "huge_shared" : " 27 GB", | 20260507 01:35:47.129 | INFO | "malloc" : " 14 GB", | 20260507 01:35:47.129 | INFO | "pinned" : " 29 GB" | 20260507 01:35:47.129 | INFO | } | 20260507 01:35:47.129 | INFO | Perf report for the last 5.13968 seconds: | 20260507 01:35:47.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:35:47.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:47.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:47.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:47.129 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 01:35:47.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:47.129 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 01:35:47.129 | INFO | PRE_HQ : 59.6%, 1.8, 56.1ms, 0 | 20260507 01:35:47.129 | INFO | BAZWRITER : 62.1%, 3.3, 105.5ms, 0 | 20260507 01:35:47.129 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 01:35:47.129 | INFO | Uploaded 36GiB to GPU in 2.02673s (17.7626GiB/s) | 20260507 01:35:47.129 | INFO | Downloaded 38GiB to GPU in 2.17539s (17.4682GiB/s) | 20260507 01:35:52.271 | INFO | Analyzing chunk frames = [6062592,6063104) | 20260507 01:35:52.271 | INFO | MemInfo: { | 20260507 01:35:52.271 | INFO | "gpu" : " 17 GB", | 20260507 01:35:52.271 | INFO | "huge" : "", | 20260507 01:35:52.271 | INFO | "huge_shared" : " 27 GB", | 20260507 01:35:52.271 | INFO | "malloc" : " 14 GB", | 20260507 01:35:52.271 | INFO | "pinned" : " 29 GB" | 20260507 01:35:52.271 | INFO | } | 20260507 01:35:52.272 | INFO | Perf report for the last 5.14244 seconds: | 20260507 01:35:52.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:35:52.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:52.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:52.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:52.272 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:35:52.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:52.272 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:52.272 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:35:52.272 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:52.272 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:52.272 | INFO | Uploaded 36GiB to GPU in 2.01023s (17.9084GiB/s) | 20260507 01:35:52.272 | INFO | Downloaded 34GiB to GPU in 1.93771s (17.5465GiB/s) | 20260507 01:35:57.411 | INFO | Analyzing chunk frames = [6063104,6063616) | 20260507 01:35:57.411 | INFO | MemInfo: { | 20260507 01:35:57.411 | INFO | "gpu" : " 17 GB", | 20260507 01:35:57.411 | INFO | "huge" : "", | 20260507 01:35:57.411 | INFO | "huge_shared" : " 27 GB", | 20260507 01:35:57.411 | INFO | "malloc" : " 14 GB", | 20260507 01:35:57.411 | INFO | "pinned" : " 29 GB" | 20260507 01:35:57.411 | INFO | } | 20260507 01:35:57.411 | INFO | Perf report for the last 5.13923 seconds: | 20260507 01:35:57.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:35:57.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:57.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:57.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:57.411 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260507 01:35:57.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:57.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:57.411 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:35:57.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:57.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:35:57.411 | INFO | Uploaded 36GiB to GPU in 2.0086s (17.923GiB/s) | 20260507 01:35:57.411 | INFO | Downloaded 34GiB to GPU in 1.93195s (17.5988GiB/s) | 20260507 01:36:02.551 | INFO | Analyzing chunk frames = [6063616,6064128) | 20260507 01:36:02.551 | INFO | MemInfo: { | 20260507 01:36:02.551 | INFO | "gpu" : " 17 GB", | 20260507 01:36:02.551 | INFO | "huge" : "", | 20260507 01:36:02.551 | INFO | "huge_shared" : " 27 GB", | 20260507 01:36:02.551 | INFO | "malloc" : " 14 GB", | 20260507 01:36:02.552 | INFO | "pinned" : " 29 GB" | 20260507 01:36:02.552 | INFO | } | 20260507 01:36:02.552 | INFO | Perf report for the last 5.14091 seconds: | 20260507 01:36:02.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:36:02.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:02.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:02.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:02.552 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 01:36:02.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:02.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:02.552 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:36:02.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:02.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:02.552 | INFO | Uploaded 36GiB to GPU in 1.98396s (18.1455GiB/s) | 20260507 01:36:02.552 | INFO | Downloaded 34GiB to GPU in 1.91936s (17.7143GiB/s) | 20260507 01:36:07.686 | INFO | Analyzing chunk frames = [6064128,6064640) | 20260507 01:36:07.686 | INFO | MemInfo: { | 20260507 01:36:07.686 | INFO | "gpu" : " 17 GB", | 20260507 01:36:07.686 | INFO | "huge" : "", | 20260507 01:36:07.686 | INFO | "huge_shared" : " 27 GB", | 20260507 01:36:07.686 | INFO | "malloc" : " 14 GB", | 20260507 01:36:07.686 | INFO | "pinned" : " 29 GB" | 20260507 01:36:07.686 | INFO | } | 20260507 01:36:07.686 | INFO | Perf report for the last 5.13466 seconds: | 20260507 01:36:07.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:36:07.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:07.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:07.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:07.686 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 01:36:07.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:07.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:07.686 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:36:07.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:07.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:07.687 | INFO | Uploaded 36GiB to GPU in 2.0157s (17.8598GiB/s) | 20260507 01:36:07.687 | INFO | Downloaded 34GiB to GPU in 1.93956s (17.5297GiB/s) | 20260507 01:36:12.828 | INFO | Analyzing chunk frames = [6064640,6065152) | 20260507 01:36:12.828 | INFO | MemInfo: { | 20260507 01:36:12.828 | INFO | "gpu" : " 17 GB", | 20260507 01:36:12.828 | INFO | "huge" : "", | 20260507 01:36:12.828 | INFO | "huge_shared" : " 27 GB", | 20260507 01:36:12.828 | INFO | "malloc" : " 14 GB", | 20260507 01:36:12.828 | INFO | "pinned" : " 29 GB" | 20260507 01:36:12.828 | INFO | } | 20260507 01:36:12.828 | INFO | Perf report for the last 5.14206 seconds: | 20260507 01:36:12.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:36:12.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:12.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:12.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:12.828 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 01:36:12.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:12.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:12.829 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 01:36:12.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:12.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:12.829 | INFO | Uploaded 36GiB to GPU in 2.05511s (17.5173GiB/s) | 20260507 01:36:12.829 | INFO | Downloaded 34GiB to GPU in 1.94858s (17.4486GiB/s) | 20260507 01:36:17.969 | INFO | Analyzing chunk frames = [6065152,6065664) | 20260507 01:36:17.969 | INFO | MemInfo: { | 20260507 01:36:17.969 | INFO | "gpu" : " 17 GB", | 20260507 01:36:17.969 | INFO | "huge" : "", | 20260507 01:36:17.969 | INFO | "huge_shared" : " 27 GB", | 20260507 01:36:17.969 | INFO | "malloc" : " 14 GB", | 20260507 01:36:17.969 | INFO | "pinned" : " 29 GB" | 20260507 01:36:17.969 | INFO | } | 20260507 01:36:17.969 | INFO | Perf report for the last 5.14097 seconds: | 20260507 01:36:17.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:36:17.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:17.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:17.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:17.969 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:36:17.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:17.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:17.969 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 01:36:17.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:17.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:17.970 | INFO | Uploaded 36GiB to GPU in 1.98726s (18.1154GiB/s) | 20260507 01:36:17.970 | INFO | Downloaded 34GiB to GPU in 1.98785s (17.1039GiB/s) | 20260507 01:36:23.109 | INFO | Analyzing chunk frames = [6065664,6066176) | 20260507 01:36:23.109 | INFO | MemInfo: { | 20260507 01:36:23.109 | INFO | "gpu" : " 17 GB", | 20260507 01:36:23.109 | INFO | "huge" : "", | 20260507 01:36:23.109 | INFO | "huge_shared" : " 27 GB", | 20260507 01:36:23.109 | INFO | "malloc" : " 14 GB", | 20260507 01:36:23.109 | INFO | "pinned" : " 29 GB" | 20260507 01:36:23.109 | INFO | } | 20260507 01:36:23.109 | INFO | Perf report for the last 5.13946 seconds: | 20260507 01:36:23.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:36:23.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:23.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:23.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:23.109 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:36:23.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:23.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:23.109 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:36:23.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:23.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:23.109 | INFO | Uploaded 36GiB to GPU in 1.97898s (18.1912GiB/s) | 20260507 01:36:23.109 | INFO | Downloaded 34GiB to GPU in 1.98516s (17.1271GiB/s) | 20260507 01:36:23.206 | INFO | Starting a round of Baz IO | 20260507 01:36:26.315 | INFO | Populating RTMetrics report | 20260507 01:36:26.315 | INFO | RTMetrics report populated | 20260507 01:36:26.315 | INFO | Opening tmp RTMetrics json file | 20260507 01:36:26.317 | INFO | Serializing RTMetrics to json | 20260507 01:36:26.335 | INFO | Serialization done, swapping tmp file now... | 20260507 01:36:26.336 | INFO | RTMetrics done | 20260507 01:36:26.408 | INFO | Finished a round of Baz IO: Wrote 1.807 GiB in 3.201 seconds | 20260507 01:36:26.408 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.11% / 0.00% / 66.89% | 20260507 01:36:26.408 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.277098% / 7.94729e-06% / 4.76837e-05% | 20260507 01:36:26.408 | INFO | Baz IO overall: | 20260507 01:36:26.408 | INFO | BytesWritten : 3144507417858 | 20260507 01:36:26.408 | INFO | NumEvents : 1513346960052 | 20260507 01:36:28.249 | INFO | Analyzing chunk frames = [6066176,6066688) | 20260507 01:36:28.249 | INFO | MemInfo: { | 20260507 01:36:28.249 | INFO | "gpu" : " 17 GB", | 20260507 01:36:28.249 | INFO | "huge" : "", | 20260507 01:36:28.249 | INFO | "huge_shared" : " 27 GB", | 20260507 01:36:28.249 | INFO | "malloc" : " 14 GB", | 20260507 01:36:28.249 | INFO | "pinned" : " 29 GB" | 20260507 01:36:28.249 | INFO | } | 20260507 01:36:28.249 | INFO | Perf report for the last 5.14046 seconds: | 20260507 01:36:28.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:36:28.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:28.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:28.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:28.249 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 01:36:28.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:28.249 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 01:36:28.249 | INFO | PRE_HQ : 59.4%, 1.8, 56.2ms, 0 | 20260507 01:36:28.249 | INFO | BAZWRITER : 61.9%, 3.3, 106.5ms, 0 | 20260507 01:36:28.249 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:36:28.250 | INFO | Uploaded 36GiB to GPU in 2.01172s (17.8952GiB/s) | 20260507 01:36:28.250 | INFO | Downloaded 38GiB to GPU in 2.17863s (17.4421GiB/s) | 20260507 01:36:33.382 | INFO | Analyzing chunk frames = [6066688,6067200) | 20260507 01:36:33.382 | INFO | MemInfo: { | 20260507 01:36:33.382 | INFO | "gpu" : " 17 GB", | 20260507 01:36:33.382 | INFO | "huge" : "", | 20260507 01:36:33.382 | INFO | "huge_shared" : " 27 GB", | 20260507 01:36:33.382 | INFO | "malloc" : " 14 GB", | 20260507 01:36:33.382 | INFO | "pinned" : " 29 GB" | 20260507 01:36:33.382 | INFO | } | 20260507 01:36:33.382 | INFO | Perf report for the last 5.13251 seconds: | 20260507 01:36:33.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:36:33.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:33.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:33.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:33.382 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:36:33.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:33.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:33.382 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:36:33.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:33.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:33.382 | INFO | Uploaded 36GiB to GPU in 2.02962s (17.7374GiB/s) | 20260507 01:36:33.382 | INFO | Downloaded 34GiB to GPU in 1.94002s (17.5256GiB/s) | 20260507 01:36:38.521 | INFO | Analyzing chunk frames = [6067200,6067712) | 20260507 01:36:38.521 | INFO | MemInfo: { | 20260507 01:36:38.521 | INFO | "gpu" : " 17 GB", | 20260507 01:36:38.521 | INFO | "huge" : "", | 20260507 01:36:38.521 | INFO | "huge_shared" : " 27 GB", | 20260507 01:36:38.522 | INFO | "malloc" : " 14 GB", | 20260507 01:36:38.522 | INFO | "pinned" : " 29 GB" | 20260507 01:36:38.522 | INFO | } | 20260507 01:36:38.522 | INFO | Perf report for the last 5.13978 seconds: | 20260507 01:36:38.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:36:38.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:38.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:38.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:38.522 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 01:36:38.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:38.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:38.522 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:36:38.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:38.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:38.522 | INFO | Uploaded 36GiB to GPU in 1.98592s (18.1276GiB/s) | 20260507 01:36:38.522 | INFO | Downloaded 34GiB to GPU in 1.93519s (17.5694GiB/s) | 20260507 01:36:43.667 | INFO | Analyzing chunk frames = [6067712,6068224) | 20260507 01:36:43.667 | INFO | MemInfo: { | 20260507 01:36:43.667 | INFO | "gpu" : " 17 GB", | 20260507 01:36:43.667 | INFO | "huge" : "", | 20260507 01:36:43.667 | INFO | "huge_shared" : " 27 GB", | 20260507 01:36:43.667 | INFO | "malloc" : " 14 GB", | 20260507 01:36:43.667 | INFO | "pinned" : " 29 GB" | 20260507 01:36:43.667 | INFO | } | 20260507 01:36:43.668 | INFO | Perf report for the last 5.14589 seconds: | 20260507 01:36:43.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:36:43.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:43.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:43.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:43.668 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260507 01:36:43.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:43.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:43.668 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:36:43.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:43.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:43.668 | INFO | Uploaded 36GiB to GPU in 2.00112s (17.99GiB/s) | 20260507 01:36:43.668 | INFO | Downloaded 34GiB to GPU in 1.91109s (17.7909GiB/s) | 20260507 01:36:48.798 | INFO | Analyzing chunk frames = [6068224,6068736) | 20260507 01:36:48.798 | INFO | MemInfo: { | 20260507 01:36:48.798 | INFO | "gpu" : " 17 GB", | 20260507 01:36:48.798 | INFO | "huge" : "", | 20260507 01:36:48.798 | INFO | "huge_shared" : " 27 GB", | 20260507 01:36:48.798 | INFO | "malloc" : " 14 GB", | 20260507 01:36:48.798 | INFO | "pinned" : " 29 GB" | 20260507 01:36:48.799 | INFO | } | 20260507 01:36:48.799 | INFO | Perf report for the last 5.13102 seconds: | 20260507 01:36:48.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:36:48.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:48.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:48.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:48.799 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 01:36:48.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:48.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:48.799 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 01:36:48.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:48.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:48.799 | INFO | Uploaded 36GiB to GPU in 2.00948s (17.9151GiB/s) | 20260507 01:36:48.799 | INFO | Downloaded 34GiB to GPU in 1.94362s (17.4932GiB/s) | 20260507 01:36:53.939 | INFO | Analyzing chunk frames = [6068736,6069248) | 20260507 01:36:53.939 | INFO | MemInfo: { | 20260507 01:36:53.939 | INFO | "gpu" : " 17 GB", | 20260507 01:36:53.939 | INFO | "huge" : "", | 20260507 01:36:53.939 | INFO | "huge_shared" : " 27 GB", | 20260507 01:36:53.939 | INFO | "malloc" : " 14 GB", | 20260507 01:36:53.939 | INFO | "pinned" : " 29 GB" | 20260507 01:36:53.939 | INFO | } | 20260507 01:36:53.939 | INFO | Perf report for the last 5.1403 seconds: | 20260507 01:36:53.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:36:53.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:53.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:53.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:53.939 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 01:36:53.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:53.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:53.939 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:36:53.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:53.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:53.940 | INFO | Uploaded 36GiB to GPU in 2.00747s (17.933GiB/s) | 20260507 01:36:53.940 | INFO | Downloaded 34GiB to GPU in 1.93977s (17.5278GiB/s) | 20260507 01:36:59.083 | INFO | Analyzing chunk frames = [6069248,6069760) | 20260507 01:36:59.083 | INFO | MemInfo: { | 20260507 01:36:59.083 | INFO | "gpu" : " 17 GB", | 20260507 01:36:59.083 | INFO | "huge" : "", | 20260507 01:36:59.083 | INFO | "huge_shared" : " 27 GB", | 20260507 01:36:59.083 | INFO | "malloc" : " 14 GB", | 20260507 01:36:59.083 | INFO | "pinned" : " 29 GB" | 20260507 01:36:59.083 | INFO | } | 20260507 01:36:59.083 | INFO | Perf report for the last 5.14447 seconds: | 20260507 01:36:59.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:36:59.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:59.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:59.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:59.083 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:36:59.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:59.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:59.083 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:36:59.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:59.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:36:59.084 | INFO | Uploaded 36GiB to GPU in 1.96725s (18.2997GiB/s) | 20260507 01:36:59.084 | INFO | Downloaded 34GiB to GPU in 1.99793s (17.0176GiB/s) | 20260507 01:37:04.226 | INFO | Analyzing chunk frames = [6069760,6070272) | 20260507 01:37:04.226 | INFO | MemInfo: { | 20260507 01:37:04.226 | INFO | "gpu" : " 17 GB", | 20260507 01:37:04.226 | INFO | "huge" : "", | 20260507 01:37:04.226 | INFO | "huge_shared" : " 27 GB", | 20260507 01:37:04.226 | INFO | "malloc" : " 14 GB", | 20260507 01:37:04.226 | INFO | "pinned" : " 29 GB" | 20260507 01:37:04.226 | INFO | } | 20260507 01:37:04.226 | INFO | Perf report for the last 5.14244 seconds: | 20260507 01:37:04.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:37:04.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:04.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:04.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:04.226 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 01:37:04.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:04.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:04.226 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260507 01:37:04.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:04.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:04.226 | INFO | Uploaded 36GiB to GPU in 1.99256s (18.0672GiB/s) | 20260507 01:37:04.226 | INFO | Downloaded 34GiB to GPU in 1.98057s (17.1668GiB/s) | 20260507 01:37:04.226 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:37:04.226 | INFO | Total mem usage: 18 GiB | 20260507 01:37:04.226 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:37:04.226 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:37:04.226 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:37:04.226 | INFO | High Water Breakdown: | 20260507 01:37:04.226 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:37:04.226 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:37:04.226 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:37:04.226 | INFO | ------------------------------------------ | 20260507 01:37:04.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:37:04.226 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:37:04.226 | INFO | Total mem usage: 26.5 GiB | 20260507 01:37:04.226 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:37:04.226 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:37:04.226 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:37:04.226 | INFO | High Water Breakdown: | 20260507 01:37:04.226 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:37:04.226 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:37:04.226 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:37:04.226 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:37:04.226 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:37:04.226 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:37:04.226 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:37:04.226 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:37:04.226 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:37:04.226 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:37:04.226 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:37:04.226 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:37:04.226 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:37:04.226 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:37:04.226 | INFO | ------------------------------------------ | 20260507 01:37:04.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:37:04.226 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:37:04.226 | INFO | Total mem usage: 25.6 GiB | 20260507 01:37:04.226 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:37:04.226 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:37:04.226 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:37:04.226 | INFO | High Water Breakdown: | 20260507 01:37:04.226 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:37:04.226 | INFO | ------------------------------------------ | 20260507 01:37:04.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:37:04.226 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:37:04.226 | INFO | Total mem usage: 15.7 GiB | 20260507 01:37:04.226 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:37:04.227 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:37:04.227 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:37:04.227 | INFO | High Water Breakdown: | 20260507 01:37:04.227 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:37:04.227 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:37:04.227 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:37:04.227 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:37:04.227 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:37:04.227 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:37:04.227 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:37:04.227 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:37:04.227 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:37:04.227 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:37:04.227 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:37:04.227 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:37:04.227 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:37:04.227 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:37:04.227 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:37:04.227 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:37:04.227 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:37:04.227 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:37:04.227 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:37:04.227 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:37:04.227 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:37:04.227 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:37:04.227 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:37:04.227 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:37:04.227 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:37:04.227 | INFO | ------------------------------------------ | 20260507 01:37:04.227 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:37:04.227 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:37:04.227 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:37:04.227 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:37:04.227 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.59e+01, 1.34e+01 / 2.31e+00 | 20260507 01:37:04.227 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 1.01e-01, 1.01e-02 / 6.95e-03 | 20260507 01:37:04.227 | INFO | PerfReport: 11.68%, Baseline -- 3.60e+00 / 4.44e+00, 5.24e-01 / 2.94e-01 | 20260507 01:37:04.227 | INFO | PerfReport: 3.23%, Histogram -- 9.96e-01 / 1.46e+00, 1.69e-01 / 1.15e-01 | 20260507 01:37:04.227 | INFO | PerfReport: 16.17%, DME -- 3.99e+01 / 4.53e+01, 4.29e+00 / 2.60e+00 | 20260507 01:37:04.227 | INFO | PerfReport: 57.45%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.26e+00 / 1.51e+00 | 20260507 01:37:04.227 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.17e+00 / 2.72e+00, 3.83e-01 / 2.10e-01 | 20260507 01:37:04.227 | INFO | PerfReport: 3.10%, Metrics -- 9.58e-01 / 1.29e+00, 2.55e-01 / 1.54e-01 | 20260507 01:37:04.227 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.25e-01 / 3.77e-01, 5.13e-02 / 2.70e-02 | 20260507 01:37:04.227 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:37:04.227 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:37:04.227 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:37:04.227 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 01:37:04.227 | INFO | PerfReport: 21.63%, Upload -- 2.09e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260507 01:37:04.227 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 01:37:04.227 | INFO | PerfReport: 31.97%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 01:37:04.227 | INFO | PerfReport: 1.40%, LockUpload -- 1.35e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 01:37:04.227 | INFO | PerfReport: 1.33%, LockDownload -- 1.28e+00 / 0.00e+00, 3.16e+00 / 0.00e+00 | 20260507 01:37:04.227 | INFO | PerfReport: 22.29%, LockCompute -- 2.15e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 01:37:04.227 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260507 01:37:04.323 | INFO | Starting a round of Baz IO | 20260507 01:37:07.436 | INFO | Populating RTMetrics report | 20260507 01:37:07.437 | INFO | RTMetrics report populated | 20260507 01:37:07.437 | INFO | Opening tmp RTMetrics json file | 20260507 01:37:07.438 | INFO | Serializing RTMetrics to json | 20260507 01:37:07.452 | INFO | Serialization done, swapping tmp file now... | 20260507 01:37:07.457 | INFO | RTMetrics done | 20260507 01:37:07.529 | INFO | Finished a round of Baz IO: Wrote 1.808 GiB in 3.206 seconds | 20260507 01:37:07.530 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.11% / 0.00% / 66.89% | 20260507 01:37:07.530 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.276343% / 0% / 7.54992e-05% | 20260507 01:37:07.530 | INFO | Baz IO overall: | 20260507 01:37:07.530 | INFO | BytesWritten : 3146448584354 | 20260507 01:37:07.530 | INFO | NumEvents : 1514266717777 | 20260507 01:37:09.361 | INFO | Analyzing chunk frames = [6070272,6070784) | 20260507 01:37:09.361 | INFO | MemInfo: { | 20260507 01:37:09.361 | INFO | "gpu" : " 17 GB", | 20260507 01:37:09.361 | INFO | "huge" : "", | 20260507 01:37:09.361 | INFO | "huge_shared" : " 27 GB", | 20260507 01:37:09.361 | INFO | "malloc" : " 14 GB", | 20260507 01:37:09.361 | INFO | "pinned" : " 29 GB" | 20260507 01:37:09.361 | INFO | } | 20260507 01:37:09.361 | INFO | Perf report for the last 5.13541 seconds: | 20260507 01:37:09.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:37:09.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:09.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:09.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:09.361 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260507 01:37:09.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:09.361 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 01:37:09.361 | INFO | PRE_HQ : 59.5%, 1.8, 55.9ms, 0 | 20260507 01:37:09.361 | INFO | BAZWRITER : 62.0%, 3.3, 106.0ms, 0 | 20260507 01:37:09.361 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 01:37:09.361 | INFO | Uploaded 36GiB to GPU in 2.04087s (17.6395GiB/s) | 20260507 01:37:09.362 | INFO | Downloaded 38GiB to GPU in 2.18285s (17.4084GiB/s) | 20260507 01:37:14.497 | INFO | Analyzing chunk frames = [6070784,6071296) | 20260507 01:37:14.497 | INFO | MemInfo: { | 20260507 01:37:14.497 | INFO | "gpu" : " 17 GB", | 20260507 01:37:14.497 | INFO | "huge" : "", | 20260507 01:37:14.497 | INFO | "huge_shared" : " 27 GB", | 20260507 01:37:14.498 | INFO | "malloc" : " 14 GB", | 20260507 01:37:14.498 | INFO | "pinned" : " 29 GB" | 20260507 01:37:14.498 | INFO | } | 20260507 01:37:14.498 | INFO | Perf report for the last 5.13634 seconds: | 20260507 01:37:14.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:37:14.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:14.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:14.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:14.498 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:37:14.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:14.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:14.498 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:37:14.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:14.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:14.498 | INFO | Uploaded 36GiB to GPU in 2.03914s (17.6545GiB/s) | 20260507 01:37:14.498 | INFO | Downloaded 34GiB to GPU in 1.93517s (17.5695GiB/s) | 20260507 01:37:19.639 | INFO | Analyzing chunk frames = [6071296,6071808) | 20260507 01:37:19.639 | INFO | MemInfo: { | 20260507 01:37:19.639 | INFO | "gpu" : " 17 GB", | 20260507 01:37:19.639 | INFO | "huge" : "", | 20260507 01:37:19.639 | INFO | "huge_shared" : " 27 GB", | 20260507 01:37:19.639 | INFO | "malloc" : " 14 GB", | 20260507 01:37:19.639 | INFO | "pinned" : " 29 GB" | 20260507 01:37:19.639 | INFO | } | 20260507 01:37:19.639 | INFO | Perf report for the last 5.1415 seconds: | 20260507 01:37:19.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:37:19.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:19.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:19.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:19.639 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:37:19.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:19.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:19.639 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:37:19.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:19.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:19.639 | INFO | Uploaded 36GiB to GPU in 2.00101s (17.9909GiB/s) | 20260507 01:37:19.639 | INFO | Downloaded 34GiB to GPU in 1.93545s (17.567GiB/s) | 20260507 01:37:24.776 | INFO | Analyzing chunk frames = [6071808,6072320) | 20260507 01:37:24.776 | INFO | MemInfo: { | 20260507 01:37:24.776 | INFO | "gpu" : " 17 GB", | 20260507 01:37:24.776 | INFO | "huge" : "", | 20260507 01:37:24.776 | INFO | "huge_shared" : " 27 GB", | 20260507 01:37:24.777 | INFO | "malloc" : " 14 GB", | 20260507 01:37:24.777 | INFO | "pinned" : " 29 GB" | 20260507 01:37:24.777 | INFO | } | 20260507 01:37:24.777 | INFO | Perf report for the last 5.13747 seconds: | 20260507 01:37:24.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:37:24.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:24.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:24.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:24.777 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 01:37:24.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:24.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:24.777 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:37:24.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:24.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:24.777 | INFO | Uploaded 36GiB to GPU in 1.98929s (18.0969GiB/s) | 20260507 01:37:24.777 | INFO | Downloaded 34GiB to GPU in 1.91336s (17.7698GiB/s) | 20260507 01:37:29.929 | INFO | Analyzing chunk frames = [6072320,6072832) | 20260507 01:37:29.929 | INFO | MemInfo: { | 20260507 01:37:29.929 | INFO | "gpu" : " 17 GB", | 20260507 01:37:29.929 | INFO | "huge" : "", | 20260507 01:37:29.929 | INFO | "huge_shared" : " 27 GB", | 20260507 01:37:29.929 | INFO | "malloc" : " 14 GB", | 20260507 01:37:29.929 | INFO | "pinned" : " 29 GB" | 20260507 01:37:29.929 | INFO | } | 20260507 01:37:29.929 | INFO | Perf report for the last 5.15237 seconds: | 20260507 01:37:29.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:37:29.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:29.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:29.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:29.929 | INFO | ANALYSIS : 59.9%, 3.0, 93.5ms, 0 | 20260507 01:37:29.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:29.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:29.929 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:37:29.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:29.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:29.929 | INFO | Uploaded 36GiB to GPU in 2.00291s (17.9739GiB/s) | 20260507 01:37:29.929 | INFO | Downloaded 34GiB to GPU in 1.93457s (17.5749GiB/s) | 20260507 01:37:35.058 | INFO | Analyzing chunk frames = [6072832,6073344) | 20260507 01:37:35.058 | INFO | MemInfo: { | 20260507 01:37:35.058 | INFO | "gpu" : " 17 GB", | 20260507 01:37:35.058 | INFO | "huge" : "", | 20260507 01:37:35.058 | INFO | "huge_shared" : " 27 GB", | 20260507 01:37:35.058 | INFO | "malloc" : " 14 GB", | 20260507 01:37:35.058 | INFO | "pinned" : " 29 GB" | 20260507 01:37:35.058 | INFO | } | 20260507 01:37:35.058 | INFO | Perf report for the last 5.1291 seconds: | 20260507 01:37:35.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:37:35.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:35.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:35.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:35.058 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 01:37:35.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:35.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:35.058 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:37:35.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:35.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:35.058 | INFO | Uploaded 36GiB to GPU in 2.036s (17.6817GiB/s) | 20260507 01:37:35.058 | INFO | Downloaded 34GiB to GPU in 1.92107s (17.6985GiB/s) | 20260507 01:37:40.198 | INFO | Analyzing chunk frames = [6073344,6073856) | 20260507 01:37:40.198 | INFO | MemInfo: { | 20260507 01:37:40.198 | INFO | "gpu" : " 17 GB", | 20260507 01:37:40.198 | INFO | "huge" : "", | 20260507 01:37:40.198 | INFO | "huge_shared" : " 27 GB", | 20260507 01:37:40.198 | INFO | "malloc" : " 14 GB", | 20260507 01:37:40.198 | INFO | "pinned" : " 29 GB" | 20260507 01:37:40.198 | INFO | } | 20260507 01:37:40.198 | INFO | Perf report for the last 5.14028 seconds: | 20260507 01:37:40.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:37:40.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:40.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:40.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:40.198 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:37:40.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:40.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:40.198 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:37:40.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:40.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:40.199 | INFO | Uploaded 36GiB to GPU in 1.97807s (18.1995GiB/s) | 20260507 01:37:40.199 | INFO | Downloaded 34GiB to GPU in 1.98525s (17.1263GiB/s) | 20260507 01:37:45.340 | INFO | Analyzing chunk frames = [6073856,6074368) | 20260507 01:37:45.340 | INFO | MemInfo: { | 20260507 01:37:45.340 | INFO | "gpu" : " 17 GB", | 20260507 01:37:45.340 | INFO | "huge" : "", | 20260507 01:37:45.340 | INFO | "huge_shared" : " 27 GB", | 20260507 01:37:45.340 | INFO | "malloc" : " 14 GB", | 20260507 01:37:45.340 | INFO | "pinned" : " 29 GB" | 20260507 01:37:45.340 | INFO | } | 20260507 01:37:45.340 | INFO | Perf report for the last 5.14196 seconds: | 20260507 01:37:45.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:37:45.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:45.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:45.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:45.340 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 01:37:45.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:45.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:45.340 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 01:37:45.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:45.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:45.341 | INFO | Uploaded 36GiB to GPU in 1.99167s (18.0753GiB/s) | 20260507 01:37:45.341 | INFO | Downloaded 34GiB to GPU in 1.98216s (17.153GiB/s) | 20260507 01:37:45.438 | INFO | Starting a round of Baz IO | 20260507 01:37:48.548 | INFO | Populating RTMetrics report | 20260507 01:37:48.548 | INFO | RTMetrics report populated | 20260507 01:37:48.548 | INFO | Opening tmp RTMetrics json file | 20260507 01:37:48.549 | INFO | Serializing RTMetrics to json | 20260507 01:37:48.550 | INFO | Serialization done, swapping tmp file now... | 20260507 01:37:48.550 | INFO | RTMetrics done | 20260507 01:37:48.638 | INFO | Finished a round of Baz IO: Wrote 1.807 GiB in 3.199 seconds | 20260507 01:37:48.638 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.11% / 0.00% / 66.89% | 20260507 01:37:48.638 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.27554% / 3.97364e-06% / 8.34465e-05% | 20260507 01:37:48.638 | INFO | Baz IO overall: | 20260507 01:37:48.638 | INFO | BytesWritten : 3148389257900 | 20260507 01:37:48.638 | INFO | NumEvents : 1515186177482 | 20260507 01:37:50.472 | INFO | Analyzing chunk frames = [6074368,6074880) | 20260507 01:37:50.472 | INFO | MemInfo: { | 20260507 01:37:50.472 | INFO | "gpu" : " 17 GB", | 20260507 01:37:50.472 | INFO | "huge" : "", | 20260507 01:37:50.472 | INFO | "huge_shared" : " 27 GB", | 20260507 01:37:50.472 | INFO | "malloc" : " 14 GB", | 20260507 01:37:50.472 | INFO | "pinned" : " 29 GB" | 20260507 01:37:50.472 | INFO | } | 20260507 01:37:50.472 | INFO | Perf report for the last 5.13186 seconds: | 20260507 01:37:50.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:37:50.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:50.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:50.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:50.472 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 01:37:50.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:50.472 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 01:37:50.472 | INFO | PRE_HQ : 59.6%, 1.8, 56.2ms, 0 | 20260507 01:37:50.472 | INFO | BAZWRITER : 61.8%, 3.3, 106.1ms, 0 | 20260507 01:37:50.472 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:37:50.472 | INFO | Uploaded 36GiB to GPU in 2.02267s (17.7983GiB/s) | 20260507 01:37:50.472 | INFO | Downloaded 38GiB to GPU in 2.18465s (17.3941GiB/s) | 20260507 01:37:55.611 | INFO | Analyzing chunk frames = [6074880,6075392) | 20260507 01:37:55.611 | INFO | MemInfo: { | 20260507 01:37:55.611 | INFO | "gpu" : " 17 GB", | 20260507 01:37:55.611 | INFO | "huge" : "", | 20260507 01:37:55.611 | INFO | "huge_shared" : " 27 GB", | 20260507 01:37:55.611 | INFO | "malloc" : " 14 GB", | 20260507 01:37:55.611 | INFO | "pinned" : " 29 GB" | 20260507 01:37:55.611 | INFO | } | 20260507 01:37:55.611 | INFO | Perf report for the last 5.13864 seconds: | 20260507 01:37:55.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:37:55.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:55.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:55.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:55.611 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:37:55.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:55.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:55.611 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:37:55.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:55.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:37:55.611 | INFO | Uploaded 36GiB to GPU in 2.02653s (17.7644GiB/s) | 20260507 01:37:55.611 | INFO | Downloaded 34GiB to GPU in 1.94018s (17.5241GiB/s) | 20260507 01:38:00.754 | INFO | Analyzing chunk frames = [6075392,6075904) | 20260507 01:38:00.754 | INFO | MemInfo: { | 20260507 01:38:00.754 | INFO | "gpu" : " 17 GB", | 20260507 01:38:00.754 | INFO | "huge" : "", | 20260507 01:38:00.754 | INFO | "huge_shared" : " 27 GB", | 20260507 01:38:00.754 | INFO | "malloc" : " 14 GB", | 20260507 01:38:00.754 | INFO | "pinned" : " 29 GB" | 20260507 01:38:00.754 | INFO | } | 20260507 01:38:00.754 | INFO | Perf report for the last 5.14354 seconds: | 20260507 01:38:00.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:38:00.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:00.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:00.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:00.754 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:38:00.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:00.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:00.755 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:38:00.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:00.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:00.755 | INFO | Uploaded 36GiB to GPU in 1.99962s (18.0034GiB/s) | 20260507 01:38:00.755 | INFO | Downloaded 34GiB to GPU in 1.93906s (17.5342GiB/s) | 20260507 01:38:05.892 | INFO | Analyzing chunk frames = [6075904,6076416) | 20260507 01:38:05.892 | INFO | MemInfo: { | 20260507 01:38:05.892 | INFO | "gpu" : " 17 GB", | 20260507 01:38:05.892 | INFO | "huge" : "", | 20260507 01:38:05.892 | INFO | "huge_shared" : " 27 GB", | 20260507 01:38:05.892 | INFO | "malloc" : " 14 GB", | 20260507 01:38:05.892 | INFO | "pinned" : " 29 GB" | 20260507 01:38:05.892 | INFO | } | 20260507 01:38:05.892 | INFO | Perf report for the last 5.13784 seconds: | 20260507 01:38:05.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:38:05.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:05.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:05.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:05.892 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:38:05.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:05.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:05.892 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:38:05.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:05.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:05.893 | INFO | Uploaded 36GiB to GPU in 1.97564s (18.2219GiB/s) | 20260507 01:38:05.893 | INFO | Downloaded 34GiB to GPU in 1.91259s (17.7769GiB/s) | 20260507 01:38:11.036 | INFO | Analyzing chunk frames = [6076416,6076928) | 20260507 01:38:11.036 | INFO | MemInfo: { | 20260507 01:38:11.036 | INFO | "gpu" : " 17 GB", | 20260507 01:38:11.036 | INFO | "huge" : "", | 20260507 01:38:11.036 | INFO | "huge_shared" : " 27 GB", | 20260507 01:38:11.036 | INFO | "malloc" : " 14 GB", | 20260507 01:38:11.036 | INFO | "pinned" : " 29 GB" | 20260507 01:38:11.036 | INFO | } | 20260507 01:38:11.036 | INFO | Perf report for the last 5.14409 seconds: | 20260507 01:38:11.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:38:11.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:11.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:11.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:11.036 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 01:38:11.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:11.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:11.036 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:38:11.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:11.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:11.037 | INFO | Uploaded 36GiB to GPU in 1.97846s (18.196GiB/s) | 20260507 01:38:11.037 | INFO | Downloaded 34GiB to GPU in 1.93131s (17.6046GiB/s) | 20260507 01:38:16.173 | INFO | Analyzing chunk frames = [6076928,6077440) | 20260507 01:38:16.173 | INFO | MemInfo: { | 20260507 01:38:16.173 | INFO | "gpu" : " 17 GB", | 20260507 01:38:16.173 | INFO | "huge" : "", | 20260507 01:38:16.173 | INFO | "huge_shared" : " 27 GB", | 20260507 01:38:16.173 | INFO | "malloc" : " 14 GB", | 20260507 01:38:16.173 | INFO | "pinned" : " 29 GB" | 20260507 01:38:16.174 | INFO | } | 20260507 01:38:16.174 | INFO | Perf report for the last 5.13722 seconds: | 20260507 01:38:16.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:38:16.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:16.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:16.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:16.174 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 01:38:16.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:16.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:16.174 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:38:16.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:16.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:16.174 | INFO | Uploaded 36GiB to GPU in 2.02996s (17.7344GiB/s) | 20260507 01:38:16.174 | INFO | Downloaded 34GiB to GPU in 1.94992s (17.4366GiB/s) | 20260507 01:38:21.312 | INFO | Analyzing chunk frames = [6077440,6077952) | 20260507 01:38:21.312 | INFO | MemInfo: { | 20260507 01:38:21.312 | INFO | "gpu" : " 17 GB", | 20260507 01:38:21.312 | INFO | "huge" : "", | 20260507 01:38:21.312 | INFO | "huge_shared" : " 27 GB", | 20260507 01:38:21.312 | INFO | "malloc" : " 14 GB", | 20260507 01:38:21.312 | INFO | "pinned" : " 29 GB" | 20260507 01:38:21.312 | INFO | } | 20260507 01:38:21.312 | INFO | Perf report for the last 5.13853 seconds: | 20260507 01:38:21.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:38:21.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:21.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:21.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:21.312 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:38:21.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:21.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:21.312 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 01:38:21.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:21.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:21.312 | INFO | Uploaded 36GiB to GPU in 1.97776s (18.2024GiB/s) | 20260507 01:38:21.312 | INFO | Downloaded 34GiB to GPU in 1.98353s (17.1411GiB/s) | 20260507 01:38:26.445 | INFO | Analyzing chunk frames = [6077952,6078464) | 20260507 01:38:26.445 | INFO | MemInfo: { | 20260507 01:38:26.445 | INFO | "gpu" : " 17 GB", | 20260507 01:38:26.445 | INFO | "huge" : "", | 20260507 01:38:26.445 | INFO | "huge_shared" : " 27 GB", | 20260507 01:38:26.445 | INFO | "malloc" : " 14 GB", | 20260507 01:38:26.445 | INFO | "pinned" : " 29 GB" | 20260507 01:38:26.445 | INFO | } | 20260507 01:38:26.446 | INFO | Perf report for the last 5.13338 seconds: | 20260507 01:38:26.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:38:26.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:26.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:26.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:26.446 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:38:26.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:26.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:26.446 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:38:26.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:26.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:26.446 | INFO | Uploaded 36GiB to GPU in 2.01464s (17.8692GiB/s) | 20260507 01:38:26.446 | INFO | Downloaded 34GiB to GPU in 1.97493s (17.2158GiB/s) | 20260507 01:38:26.446 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:38:26.446 | INFO | Total mem usage: 18 GiB | 20260507 01:38:26.446 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:38:26.446 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:38:26.446 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:38:26.446 | INFO | High Water Breakdown: | 20260507 01:38:26.446 | INFO | 60.2% : 8.59 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:38:26.446 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:38:26.446 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:38:26.446 | INFO | ------------------------------------------ | 20260507 01:38:26.446 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:38:26.446 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:38:26.446 | INFO | Total mem usage: 26.5 GiB | 20260507 01:38:26.446 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:38:26.446 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:38:26.446 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:38:26.446 | INFO | High Water Breakdown: | 20260507 01:38:26.446 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:38:26.446 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:38:26.446 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:38:26.446 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:38:26.446 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:38:26.446 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:38:26.446 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:38:26.446 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:38:26.446 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:38:26.446 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:38:26.447 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:38:26.447 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:38:26.447 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:38:26.447 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:38:26.447 | INFO | ------------------------------------------ | 20260507 01:38:26.447 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:38:26.447 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:38:26.447 | INFO | Total mem usage: 25.6 GiB | 20260507 01:38:26.447 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:38:26.447 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:38:26.447 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:38:26.447 | INFO | High Water Breakdown: | 20260507 01:38:26.447 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:38:26.447 | INFO | ------------------------------------------ | 20260507 01:38:26.447 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:38:26.447 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:38:26.447 | INFO | Total mem usage: 15.7 GiB | 20260507 01:38:26.447 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:38:26.447 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:38:26.447 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:38:26.447 | INFO | High Water Breakdown: | 20260507 01:38:26.447 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:38:26.447 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:38:26.447 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:38:26.447 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:38:26.447 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:38:26.447 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:38:26.447 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:38:26.447 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:38:26.447 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:38:26.447 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:38:26.447 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:38:26.447 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:38:26.447 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:38:26.447 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:38:26.447 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:38:26.447 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:38:26.447 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:38:26.447 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:38:26.447 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:38:26.447 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:38:26.447 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:38:26.447 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:38:26.447 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:38:26.447 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:38:26.447 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:38:26.447 | INFO | ------------------------------------------ | 20260507 01:38:26.447 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:38:26.447 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:38:26.447 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:38:26.447 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:38:26.447 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.53e+01, 1.35e+01 / 1.49e+00 | 20260507 01:38:26.447 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 1.02e-01, 1.00e-02 / 5.27e-03 | 20260507 01:38:26.447 | INFO | PerfReport: 11.65%, Baseline -- 3.60e+00 / 4.41e+00, 5.21e-01 / 2.85e-01 | 20260507 01:38:26.447 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.46e+00, 1.72e-01 / 9.75e-02 | 20260507 01:38:26.447 | INFO | PerfReport: 16.04%, DME -- 3.96e+01 / 4.40e+01, 3.85e+00 / 2.08e+00 | 20260507 01:38:26.447 | INFO | PerfReport: 57.65%, FrameLabeling -- 1.78e+01 / 2.11e+01, 1.39e+00 / 1.70e+00 | 20260507 01:38:26.447 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.15e+00 / 2.63e+00, 3.71e-01 / 2.42e-01 | 20260507 01:38:26.447 | INFO | PerfReport: 3.10%, Metrics -- 9.56e-01 / 1.28e+00, 2.55e-01 / 9.81e-02 | 20260507 01:38:26.447 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.25e-01 / 3.77e-01, 5.17e-02 / 1.89e-02 | 20260507 01:38:26.447 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:38:26.447 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:38:26.447 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:38:26.447 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260507 01:38:26.447 | INFO | PerfReport: 21.67%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260507 01:38:26.447 | INFO | PerfReport: 21.34%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 01:38:26.447 | INFO | PerfReport: 32.02%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 01:38:26.447 | INFO | PerfReport: 1.37%, LockUpload -- 1.32e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 01:38:26.447 | INFO | PerfReport: 1.28%, LockDownload -- 1.24e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260507 01:38:26.447 | INFO | PerfReport: 22.29%, LockCompute -- 2.15e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 01:38:26.447 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260507 01:38:26.544 | INFO | Starting a round of Baz IO | 20260507 01:38:29.667 | INFO | Populating RTMetrics report | 20260507 01:38:29.668 | INFO | RTMetrics report populated | 20260507 01:38:29.668 | INFO | Opening tmp RTMetrics json file | 20260507 01:38:29.685 | INFO | Serializing RTMetrics to json | 20260507 01:38:29.686 | INFO | Serialization done, swapping tmp file now... | 20260507 01:38:29.686 | INFO | RTMetrics done | 20260507 01:38:29.756 | INFO | Finished a round of Baz IO: Wrote 1.807 GiB in 3.212 seconds | 20260507 01:38:29.756 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.11% / 0.00% / 66.89% | 20260507 01:38:29.756 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.275449% / 7.94729e-06% / 5.96046e-05% | 20260507 01:38:29.756 | INFO | Baz IO overall: | 20260507 01:38:29.756 | INFO | BytesWritten : 3150329280685 | 20260507 01:38:29.756 | INFO | NumEvents : 1516105228607 | 20260507 01:38:31.590 | INFO | Analyzing chunk frames = [6078464,6078976) | 20260507 01:38:31.590 | INFO | MemInfo: { | 20260507 01:38:31.590 | INFO | "gpu" : " 17 GB", | 20260507 01:38:31.590 | INFO | "huge" : "", | 20260507 01:38:31.590 | INFO | "huge_shared" : " 27 GB", | 20260507 01:38:31.590 | INFO | "malloc" : " 14 GB", | 20260507 01:38:31.590 | INFO | "pinned" : " 29 GB" | 20260507 01:38:31.590 | INFO | } | 20260507 01:38:31.591 | INFO | Perf report for the last 5.14495 seconds: | 20260507 01:38:31.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:38:31.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:31.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:31.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:31.591 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 01:38:31.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:31.591 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:38:31.591 | INFO | PRE_HQ : 59.3%, 1.8, 56.0ms, 0 | 20260507 01:38:31.591 | INFO | BAZWRITER : 62.0%, 3.3, 104.5ms, 0 | 20260507 01:38:31.591 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:38:31.591 | INFO | Uploaded 36GiB to GPU in 2.03668s (17.6758GiB/s) | 20260507 01:38:31.591 | INFO | Downloaded 38GiB to GPU in 2.18177s (17.417GiB/s) | 20260507 01:38:36.732 | INFO | Analyzing chunk frames = [6078976,6079488) | 20260507 01:38:36.732 | INFO | MemInfo: { | 20260507 01:38:36.732 | INFO | "gpu" : " 17 GB", | 20260507 01:38:36.732 | INFO | "huge" : "", | 20260507 01:38:36.732 | INFO | "huge_shared" : " 27 GB", | 20260507 01:38:36.732 | INFO | "malloc" : " 14 GB", | 20260507 01:38:36.732 | INFO | "pinned" : " 29 GB" | 20260507 01:38:36.732 | INFO | } | 20260507 01:38:36.732 | INFO | Perf report for the last 5.14159 seconds: | 20260507 01:38:36.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:38:36.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:36.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:36.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:36.732 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:38:36.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:36.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:36.732 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:38:36.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:36.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:36.732 | INFO | Uploaded 36GiB to GPU in 2.00554s (17.9503GiB/s) | 20260507 01:38:36.732 | INFO | Downloaded 34GiB to GPU in 1.92812s (17.6337GiB/s) | 20260507 01:38:41.878 | INFO | Analyzing chunk frames = [6079488,6080000) | 20260507 01:38:41.878 | INFO | MemInfo: { | 20260507 01:38:41.878 | INFO | "gpu" : " 17 GB", | 20260507 01:38:41.878 | INFO | "huge" : "", | 20260507 01:38:41.878 | INFO | "huge_shared" : " 27 GB", | 20260507 01:38:41.878 | INFO | "malloc" : " 14 GB", | 20260507 01:38:41.878 | INFO | "pinned" : " 29 GB" | 20260507 01:38:41.878 | INFO | } | 20260507 01:38:41.878 | INFO | Perf report for the last 5.1458 seconds: | 20260507 01:38:41.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:38:41.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:41.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:41.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:41.878 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260507 01:38:41.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:41.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:41.878 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:38:41.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:41.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:41.878 | INFO | Uploaded 36GiB to GPU in 2.01233s (17.8897GiB/s) | 20260507 01:38:41.878 | INFO | Downloaded 34GiB to GPU in 1.93454s (17.5753GiB/s) | 20260507 01:38:47.016 | INFO | Analyzing chunk frames = [6080000,6080512) | 20260507 01:38:47.016 | INFO | MemInfo: { | 20260507 01:38:47.016 | INFO | "gpu" : " 17 GB", | 20260507 01:38:47.016 | INFO | "huge" : "", | 20260507 01:38:47.016 | INFO | "huge_shared" : " 27 GB", | 20260507 01:38:47.016 | INFO | "malloc" : " 14 GB", | 20260507 01:38:47.016 | INFO | "pinned" : " 29 GB" | 20260507 01:38:47.016 | INFO | } | 20260507 01:38:47.017 | INFO | Perf report for the last 5.1386 seconds: | 20260507 01:38:47.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:38:47.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:47.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:47.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:47.017 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:38:47.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:47.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:47.017 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:38:47.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:47.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:47.017 | INFO | Uploaded 36GiB to GPU in 2.00512s (17.9541GiB/s) | 20260507 01:38:47.017 | INFO | Downloaded 34GiB to GPU in 1.93397s (17.5804GiB/s) | 20260507 01:38:52.154 | INFO | Analyzing chunk frames = [6080512,6081024) | 20260507 01:38:52.154 | INFO | MemInfo: { | 20260507 01:38:52.154 | INFO | "gpu" : " 17 GB", | 20260507 01:38:52.154 | INFO | "huge" : "", | 20260507 01:38:52.154 | INFO | "huge_shared" : " 27 GB", | 20260507 01:38:52.154 | INFO | "malloc" : " 14 GB", | 20260507 01:38:52.154 | INFO | "pinned" : " 29 GB" | 20260507 01:38:52.154 | INFO | } | 20260507 01:38:52.154 | INFO | Perf report for the last 5.1373 seconds: | 20260507 01:38:52.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:38:52.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:52.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:52.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:52.154 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 01:38:52.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:52.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:52.154 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:38:52.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:52.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:52.154 | INFO | Uploaded 36GiB to GPU in 2.01484s (17.8674GiB/s) | 20260507 01:38:52.154 | INFO | Downloaded 34GiB to GPU in 1.94944s (17.4409GiB/s) | 20260507 01:38:57.289 | INFO | Analyzing chunk frames = [6081024,6081536) | 20260507 01:38:57.289 | INFO | MemInfo: { | 20260507 01:38:57.289 | INFO | "gpu" : " 17 GB", | 20260507 01:38:57.289 | INFO | "huge" : "", | 20260507 01:38:57.289 | INFO | "huge_shared" : " 27 GB", | 20260507 01:38:57.289 | INFO | "malloc" : " 14 GB", | 20260507 01:38:57.289 | INFO | "pinned" : " 29 GB" | 20260507 01:38:57.289 | INFO | } | 20260507 01:38:57.289 | INFO | Perf report for the last 5.13563 seconds: | 20260507 01:38:57.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:38:57.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:57.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:57.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:57.290 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 01:38:57.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:57.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:57.290 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:38:57.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:57.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:38:57.290 | INFO | Uploaded 36GiB to GPU in 2.05755s (17.4965GiB/s) | 20260507 01:38:57.290 | INFO | Downloaded 34GiB to GPU in 1.95771s (17.3672GiB/s) | 20260507 01:39:02.423 | INFO | Analyzing chunk frames = [6081536,6082048) | 20260507 01:39:02.423 | INFO | MemInfo: { | 20260507 01:39:02.423 | INFO | "gpu" : " 17 GB", | 20260507 01:39:02.423 | INFO | "huge" : "", | 20260507 01:39:02.423 | INFO | "huge_shared" : " 27 GB", | 20260507 01:39:02.423 | INFO | "malloc" : " 14 GB", | 20260507 01:39:02.423 | INFO | "pinned" : " 29 GB" | 20260507 01:39:02.423 | INFO | } | 20260507 01:39:02.423 | INFO | Perf report for the last 5.13365 seconds: | 20260507 01:39:02.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:39:02.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:02.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:02.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:02.423 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260507 01:39:02.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:02.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:02.423 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 01:39:02.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:02.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:02.423 | INFO | Uploaded 36GiB to GPU in 1.99955s (18.0041GiB/s) | 20260507 01:39:02.423 | INFO | Downloaded 34GiB to GPU in 1.97001s (17.2588GiB/s) | 20260507 01:39:07.565 | INFO | Analyzing chunk frames = [6082048,6082560) | 20260507 01:39:07.565 | INFO | MemInfo: { | 20260507 01:39:07.565 | INFO | "gpu" : " 17 GB", | 20260507 01:39:07.565 | INFO | "huge" : "", | 20260507 01:39:07.565 | INFO | "huge_shared" : " 27 GB", | 20260507 01:39:07.565 | INFO | "malloc" : " 14 GB", | 20260507 01:39:07.565 | INFO | "pinned" : " 29 GB" | 20260507 01:39:07.565 | INFO | } | 20260507 01:39:07.565 | INFO | Perf report for the last 5.14197 seconds: | 20260507 01:39:07.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:39:07.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:07.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:07.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:07.565 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:39:07.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:07.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:07.565 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 01:39:07.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:07.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:07.565 | INFO | Uploaded 36GiB to GPU in 1.99396s (18.0546GiB/s) | 20260507 01:39:07.565 | INFO | Downloaded 34GiB to GPU in 1.97959s (17.1753GiB/s) | 20260507 01:39:07.662 | INFO | Starting a round of Baz IO | 20260507 01:39:10.783 | INFO | Populating RTMetrics report | 20260507 01:39:10.783 | INFO | RTMetrics report populated | 20260507 01:39:10.783 | INFO | Opening tmp RTMetrics json file | 20260507 01:39:10.784 | INFO | Serializing RTMetrics to json | 20260507 01:39:10.797 | INFO | Serialization done, swapping tmp file now... | 20260507 01:39:10.803 | INFO | RTMetrics done | 20260507 01:39:10.875 | INFO | Finished a round of Baz IO: Wrote 1.806 GiB in 3.213 seconds | 20260507 01:39:10.875 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.11% / 0.00% / 66.89% | 20260507 01:39:10.875 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.273518% / 3.97364e-06% / 7.15256e-05% | 20260507 01:39:10.875 | INFO | Baz IO overall: | 20260507 01:39:10.875 | INFO | BytesWritten : 3152268120246 | 20260507 01:39:10.875 | INFO | NumEvents : 1517023171476 | 20260507 01:39:12.706 | INFO | Analyzing chunk frames = [6082560,6083072) | 20260507 01:39:12.706 | INFO | MemInfo: { | 20260507 01:39:12.706 | INFO | "gpu" : " 17 GB", | 20260507 01:39:12.706 | INFO | "huge" : "", | 20260507 01:39:12.706 | INFO | "huge_shared" : " 27 GB", | 20260507 01:39:12.706 | INFO | "malloc" : " 14 GB", | 20260507 01:39:12.706 | INFO | "pinned" : " 29 GB" | 20260507 01:39:12.706 | INFO | } | 20260507 01:39:12.706 | INFO | Perf report for the last 5.14136 seconds: | 20260507 01:39:12.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:39:12.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:12.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:12.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:12.707 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 01:39:12.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:12.707 | INFO | RT_METRICS : 42.0%, 1.0, 22.1ms, 0 | 20260507 01:39:12.707 | INFO | PRE_HQ : 59.3%, 1.8, 55.8ms, 0 | 20260507 01:39:12.707 | INFO | BAZWRITER : 62.1%, 3.3, 104.6ms, 0 | 20260507 01:39:12.707 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:39:12.707 | INFO | Uploaded 36GiB to GPU in 2.02921s (17.7409GiB/s) | 20260507 01:39:12.707 | INFO | Downloaded 38GiB to GPU in 2.1758s (17.4648GiB/s) | 20260507 01:39:17.843 | INFO | Analyzing chunk frames = [6083072,6083584) | 20260507 01:39:17.843 | INFO | MemInfo: { | 20260507 01:39:17.843 | INFO | "gpu" : " 17 GB", | 20260507 01:39:17.843 | INFO | "huge" : "", | 20260507 01:39:17.843 | INFO | "huge_shared" : " 27 GB", | 20260507 01:39:17.843 | INFO | "malloc" : " 14 GB", | 20260507 01:39:17.843 | INFO | "pinned" : " 29 GB" | 20260507 01:39:17.843 | INFO | } | 20260507 01:39:17.843 | INFO | Perf report for the last 5.137 seconds: | 20260507 01:39:17.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:39:17.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:17.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:17.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:17.843 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:39:17.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:17.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:17.844 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:39:17.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:17.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:17.844 | INFO | Uploaded 36GiB to GPU in 1.99145s (18.0773GiB/s) | 20260507 01:39:17.844 | INFO | Downloaded 34GiB to GPU in 1.93036s (17.6133GiB/s) | 20260507 01:39:22.986 | INFO | Analyzing chunk frames = [6083584,6084096) | 20260507 01:39:22.986 | INFO | MemInfo: { | 20260507 01:39:22.986 | INFO | "gpu" : " 17 GB", | 20260507 01:39:22.986 | INFO | "huge" : "", | 20260507 01:39:22.986 | INFO | "huge_shared" : " 27 GB", | 20260507 01:39:22.986 | INFO | "malloc" : " 14 GB", | 20260507 01:39:22.986 | INFO | "pinned" : " 29 GB" | 20260507 01:39:22.986 | INFO | } | 20260507 01:39:22.986 | INFO | Perf report for the last 5.14227 seconds: | 20260507 01:39:22.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:39:22.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:22.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:22.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:22.986 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:39:22.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:22.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:22.986 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:39:22.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:22.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:22.986 | INFO | Uploaded 36GiB to GPU in 1.9942s (18.0524GiB/s) | 20260507 01:39:22.986 | INFO | Downloaded 34GiB to GPU in 1.9408s (17.5186GiB/s) | 20260507 01:39:28.126 | INFO | Analyzing chunk frames = [6084096,6084608) | 20260507 01:39:28.126 | INFO | MemInfo: { | 20260507 01:39:28.126 | INFO | "gpu" : " 17 GB", | 20260507 01:39:28.126 | INFO | "huge" : "", | 20260507 01:39:28.126 | INFO | "huge_shared" : " 27 GB", | 20260507 01:39:28.126 | INFO | "malloc" : " 14 GB", | 20260507 01:39:28.126 | INFO | "pinned" : " 29 GB" | 20260507 01:39:28.126 | INFO | } | 20260507 01:39:28.127 | INFO | Perf report for the last 5.14082 seconds: | 20260507 01:39:28.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:39:28.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:28.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:28.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:28.127 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 01:39:28.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:28.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:28.127 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:39:28.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:28.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:28.127 | INFO | Uploaded 36GiB to GPU in 1.98269s (18.1571GiB/s) | 20260507 01:39:28.127 | INFO | Downloaded 34GiB to GPU in 1.91717s (17.7345GiB/s) | 20260507 01:39:33.261 | INFO | Analyzing chunk frames = [6084608,6085120) | 20260507 01:39:33.261 | INFO | MemInfo: { | 20260507 01:39:33.261 | INFO | "gpu" : " 17 GB", | 20260507 01:39:33.261 | INFO | "huge" : "", | 20260507 01:39:33.261 | INFO | "huge_shared" : " 27 GB", | 20260507 01:39:33.261 | INFO | "malloc" : " 14 GB", | 20260507 01:39:33.261 | INFO | "pinned" : " 29 GB" | 20260507 01:39:33.261 | INFO | } | 20260507 01:39:33.261 | INFO | Perf report for the last 5.13474 seconds: | 20260507 01:39:33.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:39:33.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:33.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:33.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:33.262 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 01:39:33.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:33.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:33.262 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:39:33.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:33.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:33.262 | INFO | Uploaded 36GiB to GPU in 2.02822s (17.7495GiB/s) | 20260507 01:39:33.262 | INFO | Downloaded 34GiB to GPU in 1.94941s (17.4412GiB/s) | 20260507 01:39:38.402 | INFO | Analyzing chunk frames = [6085120,6085632) | 20260507 01:39:38.402 | INFO | MemInfo: { | 20260507 01:39:38.402 | INFO | "gpu" : " 17 GB", | 20260507 01:39:38.402 | INFO | "huge" : "", | 20260507 01:39:38.402 | INFO | "huge_shared" : " 27 GB", | 20260507 01:39:38.402 | INFO | "malloc" : " 14 GB", | 20260507 01:39:38.402 | INFO | "pinned" : " 29 GB" | 20260507 01:39:38.402 | INFO | } | 20260507 01:39:38.402 | INFO | Perf report for the last 5.14072 seconds: | 20260507 01:39:38.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:39:38.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:38.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:38.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:38.402 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 01:39:38.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:38.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:38.402 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 01:39:38.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:38.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:38.403 | INFO | Uploaded 36GiB to GPU in 2.03s (17.734GiB/s) | 20260507 01:39:38.403 | INFO | Downloaded 34GiB to GPU in 1.95413s (17.3991GiB/s) | 20260507 01:39:43.539 | INFO | Analyzing chunk frames = [6085632,6086144) | 20260507 01:39:43.539 | INFO | MemInfo: { | 20260507 01:39:43.539 | INFO | "gpu" : " 17 GB", | 20260507 01:39:43.539 | INFO | "huge" : "", | 20260507 01:39:43.539 | INFO | "huge_shared" : " 27 GB", | 20260507 01:39:43.539 | INFO | "malloc" : " 14 GB", | 20260507 01:39:43.539 | INFO | "pinned" : " 29 GB" | 20260507 01:39:43.539 | INFO | } | 20260507 01:39:43.539 | INFO | Perf report for the last 5.13727 seconds: | 20260507 01:39:43.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:39:43.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:43.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:43.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:43.539 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:39:43.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:43.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:43.539 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:39:43.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:43.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:43.540 | INFO | Uploaded 36GiB to GPU in 1.99112s (18.0803GiB/s) | 20260507 01:39:43.540 | INFO | Downloaded 34GiB to GPU in 2.00566s (16.952GiB/s) | 20260507 01:39:48.689 | INFO | Analyzing chunk frames = [6086144,6086656) | 20260507 01:39:48.689 | INFO | MemInfo: { | 20260507 01:39:48.689 | INFO | "gpu" : " 17 GB", | 20260507 01:39:48.689 | INFO | "huge" : "", | 20260507 01:39:48.689 | INFO | "huge_shared" : " 27 GB", | 20260507 01:39:48.689 | INFO | "malloc" : " 14 GB", | 20260507 01:39:48.689 | INFO | "pinned" : " 29 GB" | 20260507 01:39:48.689 | INFO | } | 20260507 01:39:48.689 | INFO | Perf report for the last 5.14951 seconds: | 20260507 01:39:48.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:39:48.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:48.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:48.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:48.689 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:39:48.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:48.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:48.689 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:39:48.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:48.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:48.689 | INFO | Uploaded 36GiB to GPU in 1.99901s (18.0089GiB/s) | 20260507 01:39:48.689 | INFO | Downloaded 34GiB to GPU in 1.97382s (17.2254GiB/s) | 20260507 01:39:48.689 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:39:48.689 | INFO | Total mem usage: 18 GiB | 20260507 01:39:48.689 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:39:48.689 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 01:39:48.689 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:39:48.689 | INFO | High Water Breakdown: | 20260507 01:39:48.689 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:39:48.689 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:39:48.689 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:39:48.690 | INFO | ------------------------------------------ | 20260507 01:39:48.690 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:39:48.690 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:39:48.690 | INFO | Total mem usage: 26.5 GiB | 20260507 01:39:48.690 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:39:48.690 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:39:48.690 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:39:48.690 | INFO | High Water Breakdown: | 20260507 01:39:48.690 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:39:48.690 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:39:48.690 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:39:48.690 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:39:48.690 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:39:48.690 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:39:48.690 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:39:48.690 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:39:48.690 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:39:48.690 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:39:48.690 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:39:48.690 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:39:48.690 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:39:48.690 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:39:48.690 | INFO | ------------------------------------------ | 20260507 01:39:48.690 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:39:48.690 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:39:48.690 | INFO | Total mem usage: 25.6 GiB | 20260507 01:39:48.690 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:39:48.690 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:39:48.690 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:39:48.690 | INFO | High Water Breakdown: | 20260507 01:39:48.690 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:39:48.690 | INFO | ------------------------------------------ | 20260507 01:39:48.690 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:39:48.690 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:39:48.690 | INFO | Total mem usage: 15.7 GiB | 20260507 01:39:48.690 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:39:48.690 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:39:48.690 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:39:48.690 | INFO | High Water Breakdown: | 20260507 01:39:48.690 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:39:48.690 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:39:48.690 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:39:48.690 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:39:48.690 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:39:48.690 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:39:48.690 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:39:48.690 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:39:48.690 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:39:48.690 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:39:48.690 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:39:48.690 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:39:48.690 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:39:48.690 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:39:48.690 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:39:48.690 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:39:48.690 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:39:48.690 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:39:48.690 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:39:48.690 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:39:48.690 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:39:48.690 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:39:48.690 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:39:48.690 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:39:48.690 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:39:48.690 | INFO | ------------------------------------------ | 20260507 01:39:48.690 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:39:48.690 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:39:48.690 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:39:48.690 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:39:48.690 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.57e+01, 1.34e+01 / 2.26e+00 | 20260507 01:39:48.690 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 1.01e-01, 9.66e-03 / 3.98e-03 | 20260507 01:39:48.690 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.47e+00, 5.24e-01 / 1.90e-01 | 20260507 01:39:48.690 | INFO | PerfReport: 3.23%, Histogram -- 9.97e-01 / 1.47e+00, 1.70e-01 / 9.77e-02 | 20260507 01:39:48.690 | INFO | PerfReport: 16.12%, DME -- 3.98e+01 / 4.48e+01, 4.10e+00 / 2.51e+00 | 20260507 01:39:48.690 | INFO | PerfReport: 57.49%, FrameLabeling -- 1.77e+01 / 2.05e+01, 1.26e+00 / 1.52e+00 | 20260507 01:39:48.690 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.17e+00 / 2.73e+00, 3.86e-01 / 2.10e-01 | 20260507 01:39:48.690 | INFO | PerfReport: 3.10%, Metrics -- 9.56e-01 / 1.31e+00, 2.56e-01 / 1.76e-01 | 20260507 01:39:48.691 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.25e-01 / 3.79e-01, 5.08e-02 / 2.38e-02 | 20260507 01:39:48.691 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:39:48.691 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:39:48.691 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:39:48.691 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 01:39:48.691 | INFO | PerfReport: 21.71%, Upload -- 2.09e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260507 01:39:48.691 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 01:39:48.691 | INFO | PerfReport: 31.95%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 01:39:48.691 | INFO | PerfReport: 1.40%, LockUpload -- 1.35e+00 / 0.00e+00, 4.94e+00 / 0.00e+00 | 20260507 01:39:48.691 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260507 01:39:48.691 | INFO | PerfReport: 22.24%, LockCompute -- 2.15e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 01:39:48.691 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.33e-02 / 0.00e+00 | 20260507 01:39:48.785 | INFO | Starting a round of Baz IO | 20260507 01:39:51.900 | INFO | Populating RTMetrics report | 20260507 01:39:51.900 | INFO | RTMetrics report populated | 20260507 01:39:51.900 | INFO | Opening tmp RTMetrics json file | 20260507 01:39:51.919 | INFO | Serializing RTMetrics to json | 20260507 01:39:51.921 | INFO | Serialization done, swapping tmp file now... | 20260507 01:39:51.921 | INFO | RTMetrics done | 20260507 01:39:51.995 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.209 seconds | 20260507 01:39:51.995 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.10% / 0.00% / 66.89% | 20260507 01:39:51.995 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.273399% / 0% / 7.15256e-05% | 20260507 01:39:51.995 | INFO | Baz IO overall: | 20260507 01:39:51.995 | INFO | BytesWritten : 3154205633473 | 20260507 01:39:51.995 | INFO | NumEvents : 1517940119023 | 20260507 01:39:53.824 | INFO | Analyzing chunk frames = [6086656,6087168) | 20260507 01:39:53.824 | INFO | MemInfo: { | 20260507 01:39:53.824 | INFO | "gpu" : " 17 GB", | 20260507 01:39:53.824 | INFO | "huge" : "", | 20260507 01:39:53.824 | INFO | "huge_shared" : " 27 GB", | 20260507 01:39:53.824 | INFO | "malloc" : " 14 GB", | 20260507 01:39:53.824 | INFO | "pinned" : " 29 GB" | 20260507 01:39:53.824 | INFO | } | 20260507 01:39:53.824 | INFO | Perf report for the last 5.13541 seconds: | 20260507 01:39:53.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:39:53.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:53.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:53.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:53.824 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 01:39:53.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:53.824 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 01:39:53.824 | INFO | PRE_HQ : 59.3%, 1.8, 55.8ms, 0 | 20260507 01:39:53.824 | INFO | BAZWRITER : 62.1%, 3.3, 105.3ms, 0 | 20260507 01:39:53.824 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:39:53.824 | INFO | Uploaded 36GiB to GPU in 2.04118s (17.6369GiB/s) | 20260507 01:39:53.824 | INFO | Downloaded 38GiB to GPU in 2.16686s (17.5369GiB/s) | 20260507 01:39:58.963 | INFO | Analyzing chunk frames = [6087168,6087680) | 20260507 01:39:58.963 | INFO | MemInfo: { | 20260507 01:39:58.963 | INFO | "gpu" : " 17 GB", | 20260507 01:39:58.963 | INFO | "huge" : "", | 20260507 01:39:58.963 | INFO | "huge_shared" : " 27 GB", | 20260507 01:39:58.963 | INFO | "malloc" : " 14 GB", | 20260507 01:39:58.963 | INFO | "pinned" : " 29 GB" | 20260507 01:39:58.963 | INFO | } | 20260507 01:39:58.963 | INFO | Perf report for the last 5.13912 seconds: | 20260507 01:39:58.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:39:58.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:58.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:58.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:58.964 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:39:58.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:58.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:58.964 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:39:58.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:58.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:39:58.964 | INFO | Uploaded 36GiB to GPU in 2.03963s (17.6503GiB/s) | 20260507 01:39:58.964 | INFO | Downloaded 34GiB to GPU in 1.93586s (17.5632GiB/s) | 20260507 01:40:04.101 | INFO | Analyzing chunk frames = [6087680,6088192) | 20260507 01:40:04.101 | INFO | MemInfo: { | 20260507 01:40:04.101 | INFO | "gpu" : " 17 GB", | 20260507 01:40:04.102 | INFO | "huge" : "", | 20260507 01:40:04.102 | INFO | "huge_shared" : " 27 GB", | 20260507 01:40:04.102 | INFO | "malloc" : " 14 GB", | 20260507 01:40:04.102 | INFO | "pinned" : " 29 GB" | 20260507 01:40:04.102 | INFO | } | 20260507 01:40:04.102 | INFO | Perf report for the last 5.13832 seconds: | 20260507 01:40:04.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:40:04.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:04.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:04.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:04.102 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:40:04.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:04.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:04.102 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 01:40:04.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:04.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:04.102 | INFO | Uploaded 36GiB to GPU in 2.00482s (17.9567GiB/s) | 20260507 01:40:04.102 | INFO | Downloaded 34GiB to GPU in 1.93478s (17.573GiB/s) | 20260507 01:40:09.239 | INFO | Analyzing chunk frames = [6088192,6088704) | 20260507 01:40:09.239 | INFO | MemInfo: { | 20260507 01:40:09.239 | INFO | "gpu" : " 17 GB", | 20260507 01:40:09.239 | INFO | "huge" : "", | 20260507 01:40:09.239 | INFO | "huge_shared" : " 27 GB", | 20260507 01:40:09.239 | INFO | "malloc" : " 14 GB", | 20260507 01:40:09.239 | INFO | "pinned" : " 29 GB" | 20260507 01:40:09.239 | INFO | } | 20260507 01:40:09.239 | INFO | Perf report for the last 5.13715 seconds: | 20260507 01:40:09.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:40:09.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:09.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:09.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:09.239 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 01:40:09.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:09.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:09.239 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:40:09.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:09.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:09.239 | INFO | Uploaded 36GiB to GPU in 1.97308s (18.2456GiB/s) | 20260507 01:40:09.239 | INFO | Downloaded 34GiB to GPU in 1.90947s (17.806GiB/s) | 20260507 01:40:14.375 | INFO | Analyzing chunk frames = [6088704,6089216) | 20260507 01:40:14.375 | INFO | MemInfo: { | 20260507 01:40:14.375 | INFO | "gpu" : " 17 GB", | 20260507 01:40:14.375 | INFO | "huge" : "", | 20260507 01:40:14.375 | INFO | "huge_shared" : " 27 GB", | 20260507 01:40:14.375 | INFO | "malloc" : " 14 GB", | 20260507 01:40:14.375 | INFO | "pinned" : " 29 GB" | 20260507 01:40:14.375 | INFO | } | 20260507 01:40:14.375 | INFO | Perf report for the last 5.13626 seconds: | 20260507 01:40:14.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:40:14.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:14.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:14.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:14.375 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 01:40:14.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:14.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:14.375 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:40:14.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:14.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:14.375 | INFO | Uploaded 36GiB to GPU in 2.00716s (17.9358GiB/s) | 20260507 01:40:14.375 | INFO | Downloaded 34GiB to GPU in 1.93835s (17.5407GiB/s) | 20260507 01:40:19.517 | INFO | Analyzing chunk frames = [6089216,6089728) | 20260507 01:40:19.517 | INFO | MemInfo: { | 20260507 01:40:19.517 | INFO | "gpu" : " 17 GB", | 20260507 01:40:19.517 | INFO | "huge" : "", | 20260507 01:40:19.517 | INFO | "huge_shared" : " 27 GB", | 20260507 01:40:19.517 | INFO | "malloc" : " 14 GB", | 20260507 01:40:19.517 | INFO | "pinned" : " 29 GB" | 20260507 01:40:19.517 | INFO | } | 20260507 01:40:19.517 | INFO | Perf report for the last 5.142 seconds: | 20260507 01:40:19.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:40:19.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:19.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:19.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:19.517 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 01:40:19.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:19.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:19.517 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 01:40:19.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:19.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:19.517 | INFO | Uploaded 36GiB to GPU in 2.05724s (17.4992GiB/s) | 20260507 01:40:19.517 | INFO | Downloaded 34GiB to GPU in 1.96007s (17.3463GiB/s) | 20260507 01:40:24.653 | INFO | Analyzing chunk frames = [6089728,6090240) | 20260507 01:40:24.653 | INFO | MemInfo: { | 20260507 01:40:24.653 | INFO | "gpu" : " 17 GB", | 20260507 01:40:24.653 | INFO | "huge" : "", | 20260507 01:40:24.653 | INFO | "huge_shared" : " 27 GB", | 20260507 01:40:24.653 | INFO | "malloc" : " 14 GB", | 20260507 01:40:24.654 | INFO | "pinned" : " 29 GB" | 20260507 01:40:24.654 | INFO | } | 20260507 01:40:24.654 | INFO | Perf report for the last 5.13637 seconds: | 20260507 01:40:24.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:40:24.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:24.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:24.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:24.654 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:40:24.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:24.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:24.654 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:40:24.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:24.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:24.654 | INFO | Uploaded 36GiB to GPU in 1.98484s (18.1375GiB/s) | 20260507 01:40:24.654 | INFO | Downloaded 34GiB to GPU in 2.00205s (16.9826GiB/s) | 20260507 01:40:29.797 | INFO | Analyzing chunk frames = [6090240,6090752) | 20260507 01:40:29.797 | INFO | MemInfo: { | 20260507 01:40:29.797 | INFO | "gpu" : " 17 GB", | 20260507 01:40:29.797 | INFO | "huge" : "", | 20260507 01:40:29.797 | INFO | "huge_shared" : " 27 GB", | 20260507 01:40:29.797 | INFO | "malloc" : " 14 GB", | 20260507 01:40:29.797 | INFO | "pinned" : " 29 GB" | 20260507 01:40:29.798 | INFO | } | 20260507 01:40:29.798 | INFO | Perf report for the last 5.14404 seconds: | 20260507 01:40:29.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:40:29.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:29.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:29.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:29.798 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:40:29.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:29.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:29.798 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 01:40:29.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:29.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:29.798 | INFO | Uploaded 36GiB to GPU in 1.9825s (18.1589GiB/s) | 20260507 01:40:29.798 | INFO | Downloaded 34GiB to GPU in 1.97105s (17.2497GiB/s) | 20260507 01:40:29.894 | INFO | Starting a round of Baz IO | 20260507 01:40:33.008 | INFO | Populating RTMetrics report | 20260507 01:40:33.008 | INFO | RTMetrics report populated | 20260507 01:40:33.008 | INFO | Opening tmp RTMetrics json file | 20260507 01:40:33.009 | INFO | Serializing RTMetrics to json | 20260507 01:40:33.010 | INFO | Serialization done, swapping tmp file now... | 20260507 01:40:33.010 | INFO | RTMetrics done | 20260507 01:40:33.099 | INFO | Finished a round of Baz IO: Wrote 1.806 GiB in 3.204 seconds | 20260507 01:40:33.099 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.10% / 0.00% / 66.90% | 20260507 01:40:33.099 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.27287% / 0% / 8.34465e-05% | 20260507 01:40:33.099 | INFO | Baz IO overall: | 20260507 01:40:33.099 | INFO | BytesWritten : 3156144713305 | 20260507 01:40:33.099 | INFO | NumEvents : 1518858391974 | 20260507 01:40:34.940 | INFO | Analyzing chunk frames = [6090752,6091264) | 20260507 01:40:34.940 | INFO | MemInfo: { | 20260507 01:40:34.940 | INFO | "gpu" : " 17 GB", | 20260507 01:40:34.940 | INFO | "huge" : "", | 20260507 01:40:34.940 | INFO | "huge_shared" : " 27 GB", | 20260507 01:40:34.940 | INFO | "malloc" : " 14 GB", | 20260507 01:40:34.940 | INFO | "pinned" : " 29 GB" | 20260507 01:40:34.940 | INFO | } | 20260507 01:40:34.940 | INFO | Perf report for the last 5.1423 seconds: | 20260507 01:40:34.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:40:34.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:34.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:34.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:34.940 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260507 01:40:34.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:34.940 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 01:40:34.940 | INFO | PRE_HQ : 59.4%, 1.8, 55.9ms, 0 | 20260507 01:40:34.940 | INFO | BAZWRITER : 61.9%, 3.3, 105.3ms, 0 | 20260507 01:40:34.940 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:40:34.940 | INFO | Uploaded 36GiB to GPU in 2.0402s (17.6454GiB/s) | 20260507 01:40:34.940 | INFO | Downloaded 38GiB to GPU in 2.18266s (17.41GiB/s) | 20260507 01:40:40.075 | INFO | Analyzing chunk frames = [6091264,6091776) | 20260507 01:40:40.075 | INFO | MemInfo: { | 20260507 01:40:40.075 | INFO | "gpu" : " 17 GB", | 20260507 01:40:40.075 | INFO | "huge" : "", | 20260507 01:40:40.075 | INFO | "huge_shared" : " 27 GB", | 20260507 01:40:40.075 | INFO | "malloc" : " 14 GB", | 20260507 01:40:40.076 | INFO | "pinned" : " 29 GB" | 20260507 01:40:40.076 | INFO | } | 20260507 01:40:40.076 | INFO | Perf report for the last 5.13576 seconds: | 20260507 01:40:40.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:40:40.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:40.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:40.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:40.076 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:40:40.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:40.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:40.076 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:40:40.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:40.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:40.076 | INFO | Uploaded 36GiB to GPU in 2.03751s (17.6686GiB/s) | 20260507 01:40:40.076 | INFO | Downloaded 34GiB to GPU in 1.93546s (17.5669GiB/s) | 20260507 01:40:45.211 | INFO | Analyzing chunk frames = [6091776,6092288) | 20260507 01:40:45.211 | INFO | MemInfo: { | 20260507 01:40:45.211 | INFO | "gpu" : " 17 GB", | 20260507 01:40:45.211 | INFO | "huge" : "", | 20260507 01:40:45.211 | INFO | "huge_shared" : " 27 GB", | 20260507 01:40:45.211 | INFO | "malloc" : " 14 GB", | 20260507 01:40:45.211 | INFO | "pinned" : " 29 GB" | 20260507 01:40:45.211 | INFO | } | 20260507 01:40:45.211 | INFO | Perf report for the last 5.13564 seconds: | 20260507 01:40:45.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:40:45.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:45.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:45.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:45.211 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 01:40:45.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:45.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:45.211 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:40:45.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:45.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:45.212 | INFO | Uploaded 36GiB to GPU in 1.99713s (18.0259GiB/s) | 20260507 01:40:45.212 | INFO | Downloaded 34GiB to GPU in 1.9337s (17.5828GiB/s) | 20260507 01:40:50.353 | INFO | Analyzing chunk frames = [6092288,6092800) | 20260507 01:40:50.353 | INFO | MemInfo: { | 20260507 01:40:50.353 | INFO | "gpu" : " 17 GB", | 20260507 01:40:50.353 | INFO | "huge" : "", | 20260507 01:40:50.353 | INFO | "huge_shared" : " 27 GB", | 20260507 01:40:50.353 | INFO | "malloc" : " 14 GB", | 20260507 01:40:50.353 | INFO | "pinned" : " 29 GB" | 20260507 01:40:50.353 | INFO | } | 20260507 01:40:50.353 | INFO | Perf report for the last 5.14207 seconds: | 20260507 01:40:50.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:40:50.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:50.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:50.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:50.354 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 01:40:50.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:50.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:50.354 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:40:50.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:50.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:50.354 | INFO | Uploaded 36GiB to GPU in 1.97681s (18.2112GiB/s) | 20260507 01:40:50.354 | INFO | Downloaded 34GiB to GPU in 1.90729s (17.8263GiB/s) | 20260507 01:40:55.493 | INFO | Analyzing chunk frames = [6092800,6093312) | 20260507 01:40:55.493 | INFO | MemInfo: { | 20260507 01:40:55.493 | INFO | "gpu" : " 17 GB", | 20260507 01:40:55.493 | INFO | "huge" : "", | 20260507 01:40:55.493 | INFO | "huge_shared" : " 27 GB", | 20260507 01:40:55.493 | INFO | "malloc" : " 14 GB", | 20260507 01:40:55.493 | INFO | "pinned" : " 29 GB" | 20260507 01:40:55.493 | INFO | } | 20260507 01:40:55.493 | INFO | Perf report for the last 5.13941 seconds: | 20260507 01:40:55.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:40:55.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:55.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:55.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:55.493 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 01:40:55.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:55.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:55.493 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:40:55.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:55.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:40:55.493 | INFO | Uploaded 36GiB to GPU in 2.01593s (17.8578GiB/s) | 20260507 01:40:55.493 | INFO | Downloaded 34GiB to GPU in 1.944s (17.4897GiB/s) | 20260507 01:41:00.633 | INFO | Analyzing chunk frames = [6093312,6093824) | 20260507 01:41:00.633 | INFO | MemInfo: { | 20260507 01:41:00.633 | INFO | "gpu" : " 17 GB", | 20260507 01:41:00.633 | INFO | "huge" : "", | 20260507 01:41:00.633 | INFO | "huge_shared" : " 27 GB", | 20260507 01:41:00.633 | INFO | "malloc" : " 14 GB", | 20260507 01:41:00.633 | INFO | "pinned" : " 29 GB" | 20260507 01:41:00.633 | INFO | } | 20260507 01:41:00.633 | INFO | Perf report for the last 5.14013 seconds: | 20260507 01:41:00.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:41:00.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:00.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:00.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:00.633 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:41:00.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:00.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:00.633 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 01:41:00.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:00.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:00.633 | INFO | Uploaded 36GiB to GPU in 2.05904s (17.4839GiB/s) | 20260507 01:41:00.634 | INFO | Downloaded 34GiB to GPU in 1.96418s (17.31GiB/s) | 20260507 01:41:05.773 | INFO | Analyzing chunk frames = [6093824,6094336) | 20260507 01:41:05.773 | INFO | MemInfo: { | 20260507 01:41:05.773 | INFO | "gpu" : " 17 GB", | 20260507 01:41:05.773 | INFO | "huge" : "", | 20260507 01:41:05.773 | INFO | "huge_shared" : " 27 GB", | 20260507 01:41:05.773 | INFO | "malloc" : " 14 GB", | 20260507 01:41:05.773 | INFO | "pinned" : " 29 GB" | 20260507 01:41:05.773 | INFO | } | 20260507 01:41:05.773 | INFO | Perf report for the last 5.14032 seconds: | 20260507 01:41:05.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:41:05.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:05.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:05.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:05.774 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:41:05.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:05.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:05.774 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 01:41:05.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:05.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:05.774 | INFO | Uploaded 36GiB to GPU in 1.99668s (18.0299GiB/s) | 20260507 01:41:05.774 | INFO | Downloaded 34GiB to GPU in 1.98074s (17.1653GiB/s) | 20260507 01:41:10.914 | INFO | Analyzing chunk frames = [6094336,6094848) | 20260507 01:41:10.914 | INFO | MemInfo: { | 20260507 01:41:10.914 | INFO | "gpu" : " 17 GB", | 20260507 01:41:10.914 | INFO | "huge" : "", | 20260507 01:41:10.914 | INFO | "huge_shared" : " 27 GB", | 20260507 01:41:10.914 | INFO | "malloc" : " 14 GB", | 20260507 01:41:10.914 | INFO | "pinned" : " 29 GB" | 20260507 01:41:10.914 | INFO | } | 20260507 01:41:10.914 | INFO | Perf report for the last 5.14024 seconds: | 20260507 01:41:10.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:41:10.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:10.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:10.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:10.914 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:41:10.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:10.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:10.914 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 01:41:10.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:10.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:10.914 | INFO | Uploaded 36GiB to GPU in 2.00078s (17.993GiB/s) | 20260507 01:41:10.914 | INFO | Downloaded 34GiB to GPU in 1.97869s (17.1831GiB/s) | 20260507 01:41:10.914 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:41:10.914 | INFO | Total mem usage: 18 GiB | 20260507 01:41:10.914 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:41:10.914 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 01:41:10.914 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:41:10.914 | INFO | High Water Breakdown: | 20260507 01:41:10.914 | INFO | 60.2% : 8.58 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:41:10.914 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:41:10.914 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:41:10.914 | INFO | ------------------------------------------ | 20260507 01:41:10.914 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:41:10.914 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:41:10.914 | INFO | Total mem usage: 26.5 GiB | 20260507 01:41:10.914 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:41:10.914 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:41:10.914 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:41:10.914 | INFO | High Water Breakdown: | 20260507 01:41:10.914 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:41:10.914 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:41:10.914 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:41:10.914 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:41:10.914 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:41:10.914 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:41:10.914 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:41:10.915 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:41:10.915 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:41:10.915 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:41:10.915 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:41:10.915 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:41:10.915 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:41:10.915 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:41:10.915 | INFO | ------------------------------------------ | 20260507 01:41:10.915 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:41:10.915 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:41:10.915 | INFO | Total mem usage: 25.6 GiB | 20260507 01:41:10.915 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:41:10.915 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:41:10.915 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:41:10.915 | INFO | High Water Breakdown: | 20260507 01:41:10.915 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:41:10.915 | INFO | ------------------------------------------ | 20260507 01:41:10.915 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:41:10.915 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:41:10.915 | INFO | Total mem usage: 15.7 GiB | 20260507 01:41:10.915 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:41:10.915 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:41:10.915 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:41:10.915 | INFO | High Water Breakdown: | 20260507 01:41:10.915 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:41:10.915 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:41:10.915 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:41:10.915 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:41:10.915 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:41:10.915 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:41:10.915 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:41:10.915 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:41:10.915 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:41:10.915 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:41:10.915 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:41:10.915 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:41:10.915 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:41:10.915 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:41:10.915 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:41:10.915 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:41:10.915 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:41:10.915 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:41:10.915 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:41:10.915 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:41:10.915 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:41:10.915 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:41:10.915 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:41:10.915 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:41:10.915 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:41:10.915 | INFO | ------------------------------------------ | 20260507 01:41:10.915 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:41:10.915 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:41:10.915 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:41:10.915 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:41:10.915 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.55e+01, 1.34e+01 / 2.27e+00 | 20260507 01:41:10.915 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 9.96e-02, 9.68e-03 / 5.39e-03 | 20260507 01:41:10.915 | INFO | PerfReport: 11.64%, Baseline -- 3.59e+00 / 4.39e+00, 5.18e-01 / 3.43e-01 | 20260507 01:41:10.915 | INFO | PerfReport: 3.23%, Histogram -- 9.94e-01 / 1.45e+00, 1.70e-01 / 1.40e-01 | 20260507 01:41:10.915 | INFO | PerfReport: 16.11%, DME -- 3.97e+01 / 4.45e+01, 4.06e+00 / 2.40e+00 | 20260507 01:41:10.915 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.77e+01 / 2.07e+01, 1.28e+00 / 1.66e+00 | 20260507 01:41:10.915 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.16e+00 / 2.70e+00, 3.84e-01 / 2.35e-01 | 20260507 01:41:10.915 | INFO | PerfReport: 3.11%, Metrics -- 9.60e-01 / 1.27e+00, 2.56e-01 / 1.39e-01 | 20260507 01:41:10.915 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.25e-01 / 3.77e-01, 5.19e-02 / 2.87e-02 | 20260507 01:41:10.915 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:41:10.915 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:41:10.915 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:41:10.915 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 01:41:10.915 | INFO | PerfReport: 21.74%, Upload -- 2.10e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260507 01:41:10.915 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260507 01:41:10.915 | INFO | PerfReport: 31.94%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 01:41:10.915 | INFO | PerfReport: 1.49%, LockUpload -- 1.43e+00 / 0.00e+00, 5.03e+00 / 0.00e+00 | 20260507 01:41:10.915 | INFO | PerfReport: 1.20%, LockDownload -- 1.16e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260507 01:41:10.915 | INFO | PerfReport: 22.26%, LockCompute -- 2.15e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 01:41:10.915 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260507 01:41:11.011 | INFO | Starting a round of Baz IO | 20260507 01:41:14.135 | INFO | Populating RTMetrics report | 20260507 01:41:14.135 | INFO | RTMetrics report populated | 20260507 01:41:14.135 | INFO | Opening tmp RTMetrics json file | 20260507 01:41:14.136 | INFO | Serializing RTMetrics to json | 20260507 01:41:14.147 | INFO | Serialization done, swapping tmp file now... | 20260507 01:41:14.156 | INFO | RTMetrics done | 20260507 01:41:14.228 | INFO | Finished a round of Baz IO: Wrote 1.805 GiB in 3.216 seconds | 20260507 01:41:14.228 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.10% / 0.00% / 66.90% | 20260507 01:41:14.228 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.27281% / 3.97364e-06% / 7.94729e-05% | 20260507 01:41:14.228 | INFO | Baz IO overall: | 20260507 01:41:14.228 | INFO | BytesWritten : 3158083158080 | 20260507 01:41:14.228 | INFO | NumEvents : 1519776305412 | 20260507 01:41:16.054 | INFO | Analyzing chunk frames = [6094848,6095360) | 20260507 01:41:16.054 | INFO | MemInfo: { | 20260507 01:41:16.054 | INFO | "gpu" : " 17 GB", | 20260507 01:41:16.054 | INFO | "huge" : "", | 20260507 01:41:16.054 | INFO | "huge_shared" : " 27 GB", | 20260507 01:41:16.054 | INFO | "malloc" : " 14 GB", | 20260507 01:41:16.054 | INFO | "pinned" : " 29 GB" | 20260507 01:41:16.054 | INFO | } | 20260507 01:41:16.054 | INFO | Perf report for the last 5.14086 seconds: | 20260507 01:41:16.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:41:16.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:16.055 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:16.055 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:16.055 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 01:41:16.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:16.055 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 01:41:16.055 | INFO | PRE_HQ : 59.7%, 1.8, 56.1ms, 0 | 20260507 01:41:16.055 | INFO | BAZWRITER : 62.1%, 3.3, 105.8ms, 0 | 20260507 01:41:16.055 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:41:16.055 | INFO | Uploaded 36GiB to GPU in 2.03745s (17.6691GiB/s) | 20260507 01:41:16.055 | INFO | Downloaded 38GiB to GPU in 2.186s (17.3834GiB/s) | 20260507 01:41:21.188 | INFO | Analyzing chunk frames = [6095360,6095872) | 20260507 01:41:21.188 | INFO | MemInfo: { | 20260507 01:41:21.188 | INFO | "gpu" : " 17 GB", | 20260507 01:41:21.188 | INFO | "huge" : "", | 20260507 01:41:21.188 | INFO | "huge_shared" : " 27 GB", | 20260507 01:41:21.188 | INFO | "malloc" : " 14 GB", | 20260507 01:41:21.188 | INFO | "pinned" : " 29 GB" | 20260507 01:41:21.188 | INFO | } | 20260507 01:41:21.188 | INFO | Perf report for the last 5.13339 seconds: | 20260507 01:41:21.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:41:21.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:21.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:21.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:21.188 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:41:21.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:21.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:21.188 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 01:41:21.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:21.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:21.188 | INFO | Uploaded 36GiB to GPU in 2.02345s (17.7914GiB/s) | 20260507 01:41:21.188 | INFO | Downloaded 34GiB to GPU in 1.93862s (17.5383GiB/s) | 20260507 01:41:26.324 | INFO | Analyzing chunk frames = [6095872,6096384) | 20260507 01:41:26.324 | INFO | MemInfo: { | 20260507 01:41:26.324 | INFO | "gpu" : " 17 GB", | 20260507 01:41:26.324 | INFO | "huge" : "", | 20260507 01:41:26.324 | INFO | "huge_shared" : " 27 GB", | 20260507 01:41:26.324 | INFO | "malloc" : " 14 GB", | 20260507 01:41:26.324 | INFO | "pinned" : " 29 GB" | 20260507 01:41:26.324 | INFO | } | 20260507 01:41:26.324 | INFO | Perf report for the last 5.13654 seconds: | 20260507 01:41:26.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:41:26.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:26.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:26.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:26.325 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 01:41:26.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:26.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:26.325 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:41:26.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:26.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:26.325 | INFO | Uploaded 36GiB to GPU in 1.9721s (18.2546GiB/s) | 20260507 01:41:26.325 | INFO | Downloaded 34GiB to GPU in 1.94093s (17.5174GiB/s) | 20260507 01:41:31.467 | INFO | Analyzing chunk frames = [6096384,6096896) | 20260507 01:41:31.467 | INFO | MemInfo: { | 20260507 01:41:31.467 | INFO | "gpu" : " 17 GB", | 20260507 01:41:31.467 | INFO | "huge" : "", | 20260507 01:41:31.467 | INFO | "huge_shared" : " 27 GB", | 20260507 01:41:31.467 | INFO | "malloc" : " 14 GB", | 20260507 01:41:31.467 | INFO | "pinned" : " 29 GB" | 20260507 01:41:31.467 | INFO | } | 20260507 01:41:31.467 | INFO | Perf report for the last 5.14265 seconds: | 20260507 01:41:31.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:41:31.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:31.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:31.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:31.467 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260507 01:41:31.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:31.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:31.467 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:41:31.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:31.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:31.467 | INFO | Uploaded 36GiB to GPU in 2.00378s (17.966GiB/s) | 20260507 01:41:31.467 | INFO | Downloaded 34GiB to GPU in 1.91254s (17.7774GiB/s) | 20260507 01:41:36.609 | INFO | Analyzing chunk frames = [6096896,6097408) | 20260507 01:41:36.609 | INFO | MemInfo: { | 20260507 01:41:36.609 | INFO | "gpu" : " 17 GB", | 20260507 01:41:36.609 | INFO | "huge" : "", | 20260507 01:41:36.609 | INFO | "huge_shared" : " 27 GB", | 20260507 01:41:36.610 | INFO | "malloc" : " 14 GB", | 20260507 01:41:36.610 | INFO | "pinned" : " 29 GB" | 20260507 01:41:36.610 | INFO | } | 20260507 01:41:36.610 | INFO | Perf report for the last 5.14251 seconds: | 20260507 01:41:36.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:41:36.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:36.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:36.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:36.610 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 01:41:36.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:36.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:36.610 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 01:41:36.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:36.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:36.610 | INFO | Uploaded 36GiB to GPU in 1.98388s (18.1462GiB/s) | 20260507 01:41:36.610 | INFO | Downloaded 34GiB to GPU in 1.93211s (17.5973GiB/s) | 20260507 01:41:41.752 | INFO | Analyzing chunk frames = [6097408,6097920) | 20260507 01:41:41.752 | INFO | MemInfo: { | 20260507 01:41:41.752 | INFO | "gpu" : " 17 GB", | 20260507 01:41:41.752 | INFO | "huge" : "", | 20260507 01:41:41.752 | INFO | "huge_shared" : " 27 GB", | 20260507 01:41:41.752 | INFO | "malloc" : " 14 GB", | 20260507 01:41:41.752 | INFO | "pinned" : " 29 GB" | 20260507 01:41:41.752 | INFO | } | 20260507 01:41:41.753 | INFO | Perf report for the last 5.14295 seconds: | 20260507 01:41:41.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:41:41.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:41.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:41.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:41.753 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 01:41:41.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:41.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:41.753 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 01:41:41.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:41.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:41.753 | INFO | Uploaded 36GiB to GPU in 2.04816s (17.5768GiB/s) | 20260507 01:41:41.753 | INFO | Downloaded 34GiB to GPU in 1.96022s (17.345GiB/s) | 20260507 01:41:46.893 | INFO | Analyzing chunk frames = [6097920,6098432) | 20260507 01:41:46.893 | INFO | MemInfo: { | 20260507 01:41:46.893 | INFO | "gpu" : " 17 GB", | 20260507 01:41:46.893 | INFO | "huge" : "", | 20260507 01:41:46.893 | INFO | "huge_shared" : " 27 GB", | 20260507 01:41:46.893 | INFO | "malloc" : " 14 GB", | 20260507 01:41:46.893 | INFO | "pinned" : " 29 GB" | 20260507 01:41:46.893 | INFO | } | 20260507 01:41:46.893 | INFO | Perf report for the last 5.1405 seconds: | 20260507 01:41:46.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:41:46.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:46.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:46.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:46.893 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 01:41:46.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:46.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:46.893 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 01:41:46.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:46.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:46.894 | INFO | Uploaded 36GiB to GPU in 1.97068s (18.2678GiB/s) | 20260507 01:41:46.894 | INFO | Downloaded 34GiB to GPU in 1.96724s (17.2831GiB/s) | 20260507 01:41:52.023 | INFO | Analyzing chunk frames = [6098432,6098944) | 20260507 01:41:52.023 | INFO | MemInfo: { | 20260507 01:41:52.023 | INFO | "gpu" : " 17 GB", | 20260507 01:41:52.023 | INFO | "huge" : "", | 20260507 01:41:52.023 | INFO | "huge_shared" : " 27 GB", | 20260507 01:41:52.023 | INFO | "malloc" : " 14 GB", | 20260507 01:41:52.023 | INFO | "pinned" : " 29 GB" | 20260507 01:41:52.023 | INFO | } | 20260507 01:41:52.023 | INFO | Perf report for the last 5.12989 seconds: | 20260507 01:41:52.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:41:52.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:52.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:52.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:52.023 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:41:52.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:52.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:52.023 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 01:41:52.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:52.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:52.023 | INFO | Uploaded 36GiB to GPU in 1.99046s (18.0862GiB/s) | 20260507 01:41:52.023 | INFO | Downloaded 34GiB to GPU in 1.97649s (17.2022GiB/s) | 20260507 01:41:52.120 | INFO | Starting a round of Baz IO | 20260507 01:41:55.236 | INFO | Populating RTMetrics report | 20260507 01:41:55.237 | INFO | RTMetrics report populated | 20260507 01:41:55.237 | INFO | Opening tmp RTMetrics json file | 20260507 01:41:55.242 | INFO | Serializing RTMetrics to json | 20260507 01:41:55.256 | INFO | Serialization done, swapping tmp file now... | 20260507 01:41:55.256 | INFO | RTMetrics done | 20260507 01:41:55.326 | INFO | Finished a round of Baz IO: Wrote 1.808 GiB in 3.205 seconds | 20260507 01:41:55.326 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.10% / 0.00% / 66.90% | 20260507 01:41:55.326 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.273971% / 3.97364e-06% / 7.94729e-05% | 20260507 01:41:55.326 | INFO | Baz IO overall: | 20260507 01:41:55.326 | INFO | BytesWritten : 3160024158519 | 20260507 01:41:55.326 | INFO | NumEvents : 1520696256505 | 20260507 01:41:57.164 | INFO | Analyzing chunk frames = [6098944,6099456) | 20260507 01:41:57.164 | INFO | MemInfo: { | 20260507 01:41:57.164 | INFO | "gpu" : " 17 GB", | 20260507 01:41:57.164 | INFO | "huge" : "", | 20260507 01:41:57.164 | INFO | "huge_shared" : " 27 GB", | 20260507 01:41:57.164 | INFO | "malloc" : " 14 GB", | 20260507 01:41:57.164 | INFO | "pinned" : " 29 GB" | 20260507 01:41:57.164 | INFO | } | 20260507 01:41:57.164 | INFO | Perf report for the last 5.14088 seconds: | 20260507 01:41:57.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:41:57.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:57.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:57.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:57.164 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 01:41:57.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:41:57.164 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:41:57.164 | INFO | PRE_HQ : 59.4%, 1.8, 55.9ms, 0 | 20260507 01:41:57.164 | INFO | BAZWRITER : 61.9%, 3.3, 106.5ms, 0 | 20260507 01:41:57.164 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:41:57.165 | INFO | Uploaded 36GiB to GPU in 2.02728s (17.7577GiB/s) | 20260507 01:41:57.165 | INFO | Downloaded 38GiB to GPU in 2.1796s (17.4344GiB/s) | 20260507 01:42:02.303 | INFO | Analyzing chunk frames = [6099456,6099968) | 20260507 01:42:02.303 | INFO | MemInfo: { | 20260507 01:42:02.303 | INFO | "gpu" : " 17 GB", | 20260507 01:42:02.303 | INFO | "huge" : "", | 20260507 01:42:02.303 | INFO | "huge_shared" : " 27 GB", | 20260507 01:42:02.303 | INFO | "malloc" : " 14 GB", | 20260507 01:42:02.303 | INFO | "pinned" : " 29 GB" | 20260507 01:42:02.303 | INFO | } | 20260507 01:42:02.303 | INFO | Perf report for the last 5.13891 seconds: | 20260507 01:42:02.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:42:02.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:02.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:02.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:02.303 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:42:02.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:02.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:02.303 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:42:02.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:02.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:02.303 | INFO | Uploaded 36GiB to GPU in 1.99152s (18.0766GiB/s) | 20260507 01:42:02.303 | INFO | Downloaded 34GiB to GPU in 1.93084s (17.6089GiB/s) | 20260507 01:42:07.442 | INFO | Analyzing chunk frames = [6099968,6100480) | 20260507 01:42:07.442 | INFO | MemInfo: { | 20260507 01:42:07.442 | INFO | "gpu" : " 17 GB", | 20260507 01:42:07.442 | INFO | "huge" : "", | 20260507 01:42:07.442 | INFO | "huge_shared" : " 27 GB", | 20260507 01:42:07.442 | INFO | "malloc" : " 14 GB", | 20260507 01:42:07.442 | INFO | "pinned" : " 29 GB" | 20260507 01:42:07.442 | INFO | } | 20260507 01:42:07.442 | INFO | Perf report for the last 5.13915 seconds: | 20260507 01:42:07.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:42:07.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:07.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:07.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:07.442 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 01:42:07.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:07.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:07.443 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:42:07.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:07.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:07.443 | INFO | Uploaded 36GiB to GPU in 1.98025s (18.1795GiB/s) | 20260507 01:42:07.443 | INFO | Downloaded 34GiB to GPU in 1.93589s (17.563GiB/s) | 20260507 01:42:12.583 | INFO | Analyzing chunk frames = [6100480,6100992) | 20260507 01:42:12.583 | INFO | MemInfo: { | 20260507 01:42:12.583 | INFO | "gpu" : " 17 GB", | 20260507 01:42:12.583 | INFO | "huge" : "", | 20260507 01:42:12.583 | INFO | "huge_shared" : " 27 GB", | 20260507 01:42:12.583 | INFO | "malloc" : " 14 GB", | 20260507 01:42:12.583 | INFO | "pinned" : " 29 GB" | 20260507 01:42:12.583 | INFO | } | 20260507 01:42:12.583 | INFO | Perf report for the last 5.14088 seconds: | 20260507 01:42:12.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:42:12.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:12.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:12.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:12.583 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:42:12.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:12.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:12.583 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:42:12.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:12.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:12.583 | INFO | Uploaded 36GiB to GPU in 1.9526s (18.4369GiB/s) | 20260507 01:42:12.583 | INFO | Downloaded 34GiB to GPU in 1.926s (17.6532GiB/s) | 20260507 01:42:17.727 | INFO | Analyzing chunk frames = [6100992,6101504) | 20260507 01:42:17.727 | INFO | MemInfo: { | 20260507 01:42:17.727 | INFO | "gpu" : " 17 GB", | 20260507 01:42:17.727 | INFO | "huge" : "", | 20260507 01:42:17.727 | INFO | "huge_shared" : " 27 GB", | 20260507 01:42:17.727 | INFO | "malloc" : " 14 GB", | 20260507 01:42:17.727 | INFO | "pinned" : " 29 GB" | 20260507 01:42:17.727 | INFO | } | 20260507 01:42:17.727 | INFO | Perf report for the last 5.14387 seconds: | 20260507 01:42:17.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:42:17.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:17.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:17.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:17.727 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 01:42:17.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:17.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:17.727 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:42:17.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:17.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:17.728 | INFO | Uploaded 36GiB to GPU in 1.99284s (18.0646GiB/s) | 20260507 01:42:17.728 | INFO | Downloaded 34GiB to GPU in 1.94827s (17.4514GiB/s) | 20260507 01:42:22.869 | INFO | Analyzing chunk frames = [6101504,6102016) | 20260507 01:42:22.869 | INFO | MemInfo: { | 20260507 01:42:22.869 | INFO | "gpu" : " 17 GB", | 20260507 01:42:22.869 | INFO | "huge" : "", | 20260507 01:42:22.869 | INFO | "huge_shared" : " 27 GB", | 20260507 01:42:22.869 | INFO | "malloc" : " 14 GB", | 20260507 01:42:22.869 | INFO | "pinned" : " 29 GB" | 20260507 01:42:22.869 | INFO | } | 20260507 01:42:22.869 | INFO | Perf report for the last 5.14272 seconds: | 20260507 01:42:22.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:42:22.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:22.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:22.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:22.870 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 01:42:22.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:22.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:22.870 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:42:22.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:22.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:22.870 | INFO | Uploaded 36GiB to GPU in 2.03752s (17.6686GiB/s) | 20260507 01:42:22.870 | INFO | Downloaded 34GiB to GPU in 1.95892s (17.3565GiB/s) | 20260507 01:42:28.006 | INFO | Analyzing chunk frames = [6102016,6102528) | 20260507 01:42:28.006 | INFO | MemInfo: { | 20260507 01:42:28.006 | INFO | "gpu" : " 17 GB", | 20260507 01:42:28.006 | INFO | "huge" : "", | 20260507 01:42:28.006 | INFO | "huge_shared" : " 27 GB", | 20260507 01:42:28.006 | INFO | "malloc" : " 14 GB", | 20260507 01:42:28.006 | INFO | "pinned" : " 29 GB" | 20260507 01:42:28.006 | INFO | } | 20260507 01:42:28.006 | INFO | Perf report for the last 5.13677 seconds: | 20260507 01:42:28.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:42:28.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:28.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:28.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:28.006 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 01:42:28.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:28.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:28.006 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 01:42:28.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:28.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:28.007 | INFO | Uploaded 36GiB to GPU in 1.96757s (18.2967GiB/s) | 20260507 01:42:28.007 | INFO | Downloaded 34GiB to GPU in 1.97524s (17.2131GiB/s) | 20260507 01:42:33.141 | INFO | Analyzing chunk frames = [6102528,6103040) | 20260507 01:42:33.141 | INFO | MemInfo: { | 20260507 01:42:33.141 | INFO | "gpu" : " 17 GB", | 20260507 01:42:33.141 | INFO | "huge" : "", | 20260507 01:42:33.141 | INFO | "huge_shared" : " 27 GB", | 20260507 01:42:33.141 | INFO | "malloc" : " 14 GB", | 20260507 01:42:33.141 | INFO | "pinned" : " 29 GB" | 20260507 01:42:33.141 | INFO | } | 20260507 01:42:33.141 | INFO | Perf report for the last 5.13442 seconds: | 20260507 01:42:33.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:42:33.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:33.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:33.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:33.141 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 01:42:33.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:33.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:33.141 | INFO | PRE_HQ : 30.2%, 1.0, 15.7ms, 0 | 20260507 01:42:33.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:33.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:33.141 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:42:33.141 | INFO | Total mem usage: 18 GiB | 20260507 01:42:33.141 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:42:33.141 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:42:33.141 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:42:33.141 | INFO | High Water Breakdown: | 20260507 01:42:33.141 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:42:33.141 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:42:33.141 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:42:33.141 | INFO | ------------------------------------------ | 20260507 01:42:33.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:42:33.141 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:42:33.141 | INFO | Total mem usage: 26.5 GiB | 20260507 01:42:33.141 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:42:33.141 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:42:33.141 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:42:33.141 | INFO | High Water Breakdown: | 20260507 01:42:33.141 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:42:33.141 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:42:33.141 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:42:33.141 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:42:33.141 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:42:33.141 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:42:33.141 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:42:33.141 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:42:33.141 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:42:33.141 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:42:33.141 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:42:33.141 | INFO | Uploaded 36GiB to GPU in 1.99173s (18.0747GiB/s) | 20260507 01:42:33.141 | INFO | Downloaded 34GiB to GPU in 1.97744s (17.194GiB/s) | 20260507 01:42:33.141 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:42:33.141 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:42:33.141 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:42:33.141 | INFO | ------------------------------------------ | 20260507 01:42:33.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:42:33.141 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:42:33.141 | INFO | Total mem usage: 25.6 GiB | 20260507 01:42:33.141 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:42:33.141 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:42:33.141 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:42:33.141 | INFO | High Water Breakdown: | 20260507 01:42:33.142 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:42:33.142 | INFO | ------------------------------------------ | 20260507 01:42:33.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:42:33.142 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:42:33.142 | INFO | Total mem usage: 15.7 GiB | 20260507 01:42:33.142 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:42:33.142 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:42:33.142 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:42:33.142 | INFO | High Water Breakdown: | 20260507 01:42:33.142 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:42:33.142 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:42:33.142 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:42:33.142 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:42:33.142 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:42:33.142 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:42:33.142 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:42:33.142 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:42:33.142 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:42:33.142 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:42:33.142 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:42:33.142 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:42:33.142 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:42:33.142 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:42:33.142 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:42:33.142 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:42:33.142 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:42:33.142 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:42:33.142 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:42:33.142 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:42:33.142 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:42:33.142 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:42:33.142 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:42:33.142 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:42:33.142 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:42:33.142 | INFO | ------------------------------------------ | 20260507 01:42:33.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:42:33.142 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:42:33.142 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:42:33.142 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:42:33.142 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.49e+01, 1.34e+01 / 1.24e+00 | 20260507 01:42:33.142 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 1.01e-01, 1.06e-02 / 5.27e-03 | 20260507 01:42:33.142 | INFO | PerfReport: 11.69%, Baseline -- 3.60e+00 / 4.46e+00, 5.32e-01 / 2.87e-01 | 20260507 01:42:33.142 | INFO | PerfReport: 3.24%, Histogram -- 9.98e-01 / 1.46e+00, 1.71e-01 / 1.31e-01 | 20260507 01:42:33.142 | INFO | PerfReport: 16.04%, DME -- 3.96e+01 / 4.39e+01, 3.72e+00 / 1.66e+00 | 20260507 01:42:33.142 | INFO | PerfReport: 57.55%, FrameLabeling -- 1.77e+01 / 2.07e+01, 1.32e+00 / 1.86e+00 | 20260507 01:42:33.142 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.17e+00 / 2.71e+00, 3.82e-01 / 2.15e-01 | 20260507 01:42:33.142 | INFO | PerfReport: 3.11%, Metrics -- 9.59e-01 / 1.28e+00, 2.56e-01 / 1.47e-01 | 20260507 01:42:33.142 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.25e-01 / 3.74e-01, 5.15e-02 / 1.37e-02 | 20260507 01:42:33.142 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:42:33.142 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:42:33.142 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:42:33.142 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 01:42:33.142 | INFO | PerfReport: 21.60%, Upload -- 2.08e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260507 01:42:33.142 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260507 01:42:33.142 | INFO | PerfReport: 32.00%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 01:42:33.142 | INFO | PerfReport: 1.27%, LockUpload -- 1.23e+00 / 0.00e+00, 4.68e+00 / 0.00e+00 | 20260507 01:42:33.143 | INFO | PerfReport: 1.38%, LockDownload -- 1.33e+00 / 0.00e+00, 3.21e+00 / 0.00e+00 | 20260507 01:42:33.143 | INFO | PerfReport: 22.35%, LockCompute -- 2.15e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 01:42:33.143 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260507 01:42:33.238 | INFO | Starting a round of Baz IO | 20260507 01:42:36.360 | INFO | Populating RTMetrics report | 20260507 01:42:36.360 | INFO | RTMetrics report populated | 20260507 01:42:36.360 | INFO | Opening tmp RTMetrics json file | 20260507 01:42:36.379 | INFO | Serializing RTMetrics to json | 20260507 01:42:36.380 | INFO | Serialization done, swapping tmp file now... | 20260507 01:42:36.380 | INFO | RTMetrics done | 20260507 01:42:36.452 | INFO | Finished a round of Baz IO: Wrote 1.804 GiB in 3.213 seconds | 20260507 01:42:36.452 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.10% / 0.00% / 66.90% | 20260507 01:42:36.452 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.271114% / 0% / 7.94729e-05% | 20260507 01:42:36.452 | INFO | Baz IO overall: | 20260507 01:42:36.452 | INFO | BytesWritten : 3161961668472 | 20260507 01:42:36.452 | INFO | NumEvents : 1521613513739 | 20260507 01:42:38.274 | INFO | Analyzing chunk frames = [6103040,6103552) | 20260507 01:42:38.274 | INFO | MemInfo: { | 20260507 01:42:38.274 | INFO | "gpu" : " 17 GB", | 20260507 01:42:38.274 | INFO | "huge" : "", | 20260507 01:42:38.274 | INFO | "huge_shared" : " 27 GB", | 20260507 01:42:38.274 | INFO | "malloc" : " 14 GB", | 20260507 01:42:38.274 | INFO | "pinned" : " 29 GB" | 20260507 01:42:38.274 | INFO | } | 20260507 01:42:38.274 | INFO | Perf report for the last 5.13296 seconds: | 20260507 01:42:38.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:42:38.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:38.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:38.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:38.274 | INFO | ANALYSIS : 62.2%, 3.0, 96.6ms, 0 | 20260507 01:42:38.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:38.274 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 01:42:38.274 | INFO | PRE_HQ : 60.2%, 1.9, 58.4ms, 0 | 20260507 01:42:38.274 | INFO | BAZWRITER : 62.2%, 3.4, 108.8ms, 0 | 20260507 01:42:38.274 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:42:38.274 | INFO | Uploaded 36GiB to GPU in 2.04642s (17.5917GiB/s) | 20260507 01:42:38.274 | INFO | Downloaded 38GiB to GPU in 2.18171s (17.4175GiB/s) | 20260507 01:42:43.420 | INFO | Analyzing chunk frames = [6103552,6104064) | 20260507 01:42:43.420 | INFO | MemInfo: { | 20260507 01:42:43.420 | INFO | "gpu" : " 17 GB", | 20260507 01:42:43.420 | INFO | "huge" : "", | 20260507 01:42:43.420 | INFO | "huge_shared" : " 27 GB", | 20260507 01:42:43.420 | INFO | "malloc" : " 14 GB", | 20260507 01:42:43.420 | INFO | "pinned" : " 29 GB" | 20260507 01:42:43.420 | INFO | } | 20260507 01:42:43.420 | INFO | Perf report for the last 5.146 seconds: | 20260507 01:42:43.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:42:43.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:43.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:43.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:43.420 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:42:43.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:43.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:43.420 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 01:42:43.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:43.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:43.420 | INFO | Uploaded 36GiB to GPU in 1.99273s (18.0657GiB/s) | 20260507 01:42:43.420 | INFO | Downloaded 34GiB to GPU in 1.93248s (17.594GiB/s) | 20260507 01:42:48.563 | INFO | Analyzing chunk frames = [6104064,6104576) | 20260507 01:42:48.563 | INFO | MemInfo: { | 20260507 01:42:48.563 | INFO | "gpu" : " 17 GB", | 20260507 01:42:48.563 | INFO | "huge" : "", | 20260507 01:42:48.563 | INFO | "huge_shared" : " 27 GB", | 20260507 01:42:48.563 | INFO | "malloc" : " 14 GB", | 20260507 01:42:48.563 | INFO | "pinned" : " 29 GB" | 20260507 01:42:48.563 | INFO | } | 20260507 01:42:48.563 | INFO | Perf report for the last 5.14359 seconds: | 20260507 01:42:48.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:42:48.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:48.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:48.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:48.564 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 01:42:48.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:48.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:48.564 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:42:48.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:48.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:48.564 | INFO | Uploaded 36GiB to GPU in 1.9886s (18.1032GiB/s) | 20260507 01:42:48.564 | INFO | Downloaded 34GiB to GPU in 1.93755s (17.5479GiB/s) | 20260507 01:42:53.703 | INFO | Analyzing chunk frames = [6104576,6105088) | 20260507 01:42:53.703 | INFO | MemInfo: { | 20260507 01:42:53.703 | INFO | "gpu" : " 17 GB", | 20260507 01:42:53.703 | INFO | "huge" : "", | 20260507 01:42:53.703 | INFO | "huge_shared" : " 27 GB", | 20260507 01:42:53.703 | INFO | "malloc" : " 14 GB", | 20260507 01:42:53.703 | INFO | "pinned" : " 29 GB" | 20260507 01:42:53.703 | INFO | } | 20260507 01:42:53.703 | INFO | Perf report for the last 5.14005 seconds: | 20260507 01:42:53.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:42:53.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:53.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:53.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:53.704 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 01:42:53.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:53.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:53.704 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:42:53.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:53.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:53.704 | INFO | Uploaded 36GiB to GPU in 1.96849s (18.2881GiB/s) | 20260507 01:42:53.704 | INFO | Downloaded 34GiB to GPU in 1.90821s (17.8178GiB/s) | 20260507 01:42:58.840 | INFO | Analyzing chunk frames = [6105088,6105600) | 20260507 01:42:58.840 | INFO | MemInfo: { | 20260507 01:42:58.840 | INFO | "gpu" : " 17 GB", | 20260507 01:42:58.840 | INFO | "huge" : "", | 20260507 01:42:58.840 | INFO | "huge_shared" : " 27 GB", | 20260507 01:42:58.840 | INFO | "malloc" : " 14 GB", | 20260507 01:42:58.840 | INFO | "pinned" : " 29 GB" | 20260507 01:42:58.840 | INFO | } | 20260507 01:42:58.840 | INFO | Perf report for the last 5.1368 seconds: | 20260507 01:42:58.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:42:58.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:58.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:58.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:58.840 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 01:42:58.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:58.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:58.840 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:42:58.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:58.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:42:58.840 | INFO | Uploaded 36GiB to GPU in 2.02908s (17.7421GiB/s) | 20260507 01:42:58.840 | INFO | Downloaded 34GiB to GPU in 1.9464s (17.4682GiB/s) | 20260507 01:43:03.978 | INFO | Analyzing chunk frames = [6105600,6106112) | 20260507 01:43:03.978 | INFO | MemInfo: { | 20260507 01:43:03.978 | INFO | "gpu" : " 17 GB", | 20260507 01:43:03.978 | INFO | "huge" : "", | 20260507 01:43:03.978 | INFO | "huge_shared" : " 27 GB", | 20260507 01:43:03.978 | INFO | "malloc" : " 14 GB", | 20260507 01:43:03.978 | INFO | "pinned" : " 29 GB" | 20260507 01:43:03.978 | INFO | } | 20260507 01:43:03.978 | INFO | Perf report for the last 5.13825 seconds: | 20260507 01:43:03.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:43:03.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:03.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:03.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:03.979 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 01:43:03.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:03.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:03.979 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:43:03.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:03.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:03.979 | INFO | Uploaded 36GiB to GPU in 2.0439s (17.6134GiB/s) | 20260507 01:43:03.979 | INFO | Downloaded 34GiB to GPU in 1.95595s (17.3829GiB/s) | 20260507 01:43:09.119 | INFO | Analyzing chunk frames = [6106112,6106624) | 20260507 01:43:09.119 | INFO | MemInfo: { | 20260507 01:43:09.119 | INFO | "gpu" : " 17 GB", | 20260507 01:43:09.119 | INFO | "huge" : "", | 20260507 01:43:09.119 | INFO | "huge_shared" : " 27 GB", | 20260507 01:43:09.119 | INFO | "malloc" : " 14 GB", | 20260507 01:43:09.120 | INFO | "pinned" : " 29 GB" | 20260507 01:43:09.120 | INFO | } | 20260507 01:43:09.120 | INFO | Perf report for the last 5.14122 seconds: | 20260507 01:43:09.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:43:09.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:09.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:09.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:09.120 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 01:43:09.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:09.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:09.120 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 01:43:09.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:09.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:09.120 | INFO | Uploaded 36GiB to GPU in 1.95154s (18.447GiB/s) | 20260507 01:43:09.120 | INFO | Downloaded 34GiB to GPU in 1.99256s (17.0635GiB/s) | 20260507 01:43:14.250 | INFO | Analyzing chunk frames = [6106624,6107136) | 20260507 01:43:14.250 | INFO | MemInfo: { | 20260507 01:43:14.250 | INFO | "gpu" : " 17 GB", | 20260507 01:43:14.250 | INFO | "huge" : "", | 20260507 01:43:14.250 | INFO | "huge_shared" : " 27 GB", | 20260507 01:43:14.250 | INFO | "malloc" : " 14 GB", | 20260507 01:43:14.250 | INFO | "pinned" : " 29 GB" | 20260507 01:43:14.250 | INFO | } | 20260507 01:43:14.250 | INFO | Perf report for the last 5.13076 seconds: | 20260507 01:43:14.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:43:14.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:14.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:14.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:14.250 | INFO | ANALYSIS : 61.5%, 3.0, 95.5ms, 0 | 20260507 01:43:14.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:14.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:14.251 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 01:43:14.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:14.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:14.251 | INFO | Uploaded 36GiB to GPU in 2.00412s (17.963GiB/s) | 20260507 01:43:14.251 | INFO | Downloaded 34GiB to GPU in 2.00284s (16.9759GiB/s) | 20260507 01:43:14.349 | INFO | Starting a round of Baz IO | 20260507 01:43:17.470 | INFO | Populating RTMetrics report | 20260507 01:43:17.470 | INFO | RTMetrics report populated | 20260507 01:43:17.470 | INFO | Opening tmp RTMetrics json file | 20260507 01:43:17.474 | INFO | Serializing RTMetrics to json | 20260507 01:43:17.489 | INFO | Serialization done, swapping tmp file now... | 20260507 01:43:17.489 | INFO | RTMetrics done | 20260507 01:43:17.561 | INFO | Finished a round of Baz IO: Wrote 1.805 GiB in 3.212 seconds | 20260507 01:43:17.561 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.10% / 0.00% / 66.90% | 20260507 01:43:17.561 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.271785% / 3.97364e-06% / 8.34465e-05% | 20260507 01:43:17.561 | INFO | Baz IO overall: | 20260507 01:43:17.561 | INFO | BytesWritten : 3163899434636 | 20260507 01:43:17.561 | INFO | NumEvents : 1522531082622 | 20260507 01:43:19.395 | INFO | Analyzing chunk frames = [6107136,6107648) | 20260507 01:43:19.395 | INFO | MemInfo: { | 20260507 01:43:19.395 | INFO | "gpu" : " 17 GB", | 20260507 01:43:19.395 | INFO | "huge" : "", | 20260507 01:43:19.395 | INFO | "huge_shared" : " 27 GB", | 20260507 01:43:19.395 | INFO | "malloc" : " 14 GB", | 20260507 01:43:19.395 | INFO | "pinned" : " 29 GB" | 20260507 01:43:19.395 | INFO | } | 20260507 01:43:19.395 | INFO | Perf report for the last 5.14475 seconds: | 20260507 01:43:19.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:43:19.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:19.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:19.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:19.395 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260507 01:43:19.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:19.395 | INFO | RT_METRICS : 42.0%, 1.0, 22.1ms, 0 | 20260507 01:43:19.395 | INFO | PRE_HQ : 59.4%, 1.8, 55.9ms, 0 | 20260507 01:43:19.395 | INFO | BAZWRITER : 62.0%, 3.3, 105.7ms, 0 | 20260507 01:43:19.395 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:43:19.396 | INFO | Uploaded 36GiB to GPU in 2.03729s (17.6705GiB/s) | 20260507 01:43:19.396 | INFO | Downloaded 38GiB to GPU in 2.18482s (17.3927GiB/s) | 20260507 01:43:24.535 | INFO | Analyzing chunk frames = [6107648,6108160) | 20260507 01:43:24.535 | INFO | MemInfo: { | 20260507 01:43:24.535 | INFO | "gpu" : " 17 GB", | 20260507 01:43:24.535 | INFO | "huge" : "", | 20260507 01:43:24.535 | INFO | "huge_shared" : " 27 GB", | 20260507 01:43:24.535 | INFO | "malloc" : " 14 GB", | 20260507 01:43:24.535 | INFO | "pinned" : " 29 GB" | 20260507 01:43:24.535 | INFO | } | 20260507 01:43:24.535 | INFO | Perf report for the last 5.14017 seconds: | 20260507 01:43:24.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:43:24.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:24.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:24.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:24.535 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 01:43:24.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:24.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:24.535 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:43:24.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:24.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:24.536 | INFO | Uploaded 36GiB to GPU in 2.00683s (17.9387GiB/s) | 20260507 01:43:24.536 | INFO | Downloaded 34GiB to GPU in 1.93034s (17.6135GiB/s) | 20260507 01:43:29.677 | INFO | Analyzing chunk frames = [6108160,6108672) | 20260507 01:43:29.677 | INFO | MemInfo: { | 20260507 01:43:29.677 | INFO | "gpu" : " 17 GB", | 20260507 01:43:29.677 | INFO | "huge" : "", | 20260507 01:43:29.677 | INFO | "huge_shared" : " 27 GB", | 20260507 01:43:29.677 | INFO | "malloc" : " 14 GB", | 20260507 01:43:29.677 | INFO | "pinned" : " 29 GB" | 20260507 01:43:29.677 | INFO | } | 20260507 01:43:29.677 | INFO | Perf report for the last 5.14181 seconds: | 20260507 01:43:29.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:43:29.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:29.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:29.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:29.677 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 01:43:29.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:29.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:29.677 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:43:29.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:29.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:29.677 | INFO | Uploaded 36GiB to GPU in 1.99011s (18.0894GiB/s) | 20260507 01:43:29.677 | INFO | Downloaded 34GiB to GPU in 1.93967s (17.5287GiB/s) | 20260507 01:43:34.815 | INFO | Analyzing chunk frames = [6108672,6109184) | 20260507 01:43:34.815 | INFO | MemInfo: { | 20260507 01:43:34.815 | INFO | "gpu" : " 17 GB", | 20260507 01:43:34.815 | INFO | "huge" : "", | 20260507 01:43:34.815 | INFO | "huge_shared" : " 27 GB", | 20260507 01:43:34.815 | INFO | "malloc" : " 14 GB", | 20260507 01:43:34.815 | INFO | "pinned" : " 29 GB" | 20260507 01:43:34.815 | INFO | } | 20260507 01:43:34.815 | INFO | Perf report for the last 5.13826 seconds: | 20260507 01:43:34.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:43:34.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:34.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:34.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:34.816 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 01:43:34.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:34.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:34.816 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:43:34.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:34.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:34.816 | INFO | Uploaded 36GiB to GPU in 1.98333s (18.1513GiB/s) | 20260507 01:43:34.816 | INFO | Downloaded 34GiB to GPU in 1.92182s (17.6916GiB/s) | 20260507 01:43:39.953 | INFO | Analyzing chunk frames = [6109184,6109696) | 20260507 01:43:39.953 | INFO | MemInfo: { | 20260507 01:43:39.953 | INFO | "gpu" : " 17 GB", | 20260507 01:43:39.953 | INFO | "huge" : "", | 20260507 01:43:39.953 | INFO | "huge_shared" : " 27 GB", | 20260507 01:43:39.953 | INFO | "malloc" : " 14 GB", | 20260507 01:43:39.953 | INFO | "pinned" : " 29 GB" | 20260507 01:43:39.953 | INFO | } | 20260507 01:43:39.953 | INFO | Perf report for the last 5.13756 seconds: | 20260507 01:43:39.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:43:39.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:39.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:39.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:39.953 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 01:43:39.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:39.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:39.953 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:43:39.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:39.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:39.953 | INFO | Uploaded 36GiB to GPU in 2.00915s (17.918GiB/s) | 20260507 01:43:39.953 | INFO | Downloaded 34GiB to GPU in 1.93713s (17.5518GiB/s) | 20260507 01:43:45.088 | INFO | Analyzing chunk frames = [6109696,6110208) | 20260507 01:43:45.088 | INFO | MemInfo: { | 20260507 01:43:45.088 | INFO | "gpu" : " 17 GB", | 20260507 01:43:45.088 | INFO | "huge" : "", | 20260507 01:43:45.088 | INFO | "huge_shared" : " 27 GB", | 20260507 01:43:45.088 | INFO | "malloc" : " 14 GB", | 20260507 01:43:45.089 | INFO | "pinned" : " 29 GB" | 20260507 01:43:45.089 | INFO | } | 20260507 01:43:45.089 | INFO | Perf report for the last 5.13548 seconds: | 20260507 01:43:45.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:43:45.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:45.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:45.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:45.089 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 01:43:45.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:45.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:45.089 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:43:45.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:45.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:45.089 | INFO | Uploaded 36GiB to GPU in 2.03354s (17.7031GiB/s) | 20260507 01:43:45.089 | INFO | Downloaded 34GiB to GPU in 1.95239s (17.4146GiB/s) | 20260507 01:43:50.225 | INFO | Analyzing chunk frames = [6110208,6110720) | 20260507 01:43:50.225 | INFO | MemInfo: { | 20260507 01:43:50.225 | INFO | "gpu" : " 17 GB", | 20260507 01:43:50.225 | INFO | "huge" : "", | 20260507 01:43:50.225 | INFO | "huge_shared" : " 27 GB", | 20260507 01:43:50.225 | INFO | "malloc" : " 14 GB", | 20260507 01:43:50.225 | INFO | "pinned" : " 29 GB" | 20260507 01:43:50.225 | INFO | } | 20260507 01:43:50.225 | INFO | Perf report for the last 5.13624 seconds: | 20260507 01:43:50.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:43:50.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:50.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:50.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:50.225 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260507 01:43:50.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:50.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:50.226 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 01:43:50.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:50.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:50.226 | INFO | Uploaded 36GiB to GPU in 1.99525s (18.0429GiB/s) | 20260507 01:43:50.226 | INFO | Downloaded 34GiB to GPU in 2.00481s (16.9592GiB/s) | 20260507 01:43:55.368 | INFO | Analyzing chunk frames = [6110720,6111232) | 20260507 01:43:55.368 | INFO | MemInfo: { | 20260507 01:43:55.368 | INFO | "gpu" : " 17 GB", | 20260507 01:43:55.368 | INFO | "huge" : "", | 20260507 01:43:55.368 | INFO | "huge_shared" : " 27 GB", | 20260507 01:43:55.368 | INFO | "malloc" : " 14 GB", | 20260507 01:43:55.368 | INFO | "pinned" : " 29 GB" | 20260507 01:43:55.368 | INFO | } | 20260507 01:43:55.368 | INFO | Perf report for the last 5.14357 seconds: | 20260507 01:43:55.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:43:55.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:55.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:55.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:55.369 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:43:55.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:55.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:55.369 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 01:43:55.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:55.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:43:55.369 | INFO | Uploaded 36GiB to GPU in 1.98933s (18.0966GiB/s) | 20260507 01:43:55.369 | INFO | Downloaded 34GiB to GPU in 1.99995s (17.0004GiB/s) | 20260507 01:43:55.369 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:43:55.369 | INFO | Total mem usage: 18 GiB | 20260507 01:43:55.369 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:43:55.369 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 01:43:55.369 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:43:55.369 | INFO | High Water Breakdown: | 20260507 01:43:55.369 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:43:55.369 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:43:55.369 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:43:55.369 | INFO | ------------------------------------------ | 20260507 01:43:55.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:43:55.369 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:43:55.369 | INFO | Total mem usage: 26.5 GiB | 20260507 01:43:55.369 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:43:55.369 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:43:55.369 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:43:55.369 | INFO | High Water Breakdown: | 20260507 01:43:55.369 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:43:55.369 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:43:55.369 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:43:55.369 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:43:55.369 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:43:55.369 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:43:55.369 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:43:55.369 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:43:55.369 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:43:55.369 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:43:55.369 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:43:55.369 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:43:55.369 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:43:55.369 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:43:55.369 | INFO | ------------------------------------------ | 20260507 01:43:55.369 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:43:55.369 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:43:55.369 | INFO | Total mem usage: 25.6 GiB | 20260507 01:43:55.369 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:43:55.369 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:43:55.370 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:43:55.370 | INFO | High Water Breakdown: | 20260507 01:43:55.370 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:43:55.370 | INFO | ------------------------------------------ | 20260507 01:43:55.370 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:43:55.370 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:43:55.370 | INFO | Total mem usage: 15.7 GiB | 20260507 01:43:55.370 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:43:55.370 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:43:55.370 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:43:55.370 | INFO | High Water Breakdown: | 20260507 01:43:55.370 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:43:55.370 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:43:55.370 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:43:55.370 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:43:55.370 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:43:55.370 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:43:55.370 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:43:55.370 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:43:55.370 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:43:55.370 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:43:55.370 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:43:55.370 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:43:55.370 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:43:55.370 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:43:55.370 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:43:55.370 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:43:55.370 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:43:55.370 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:43:55.370 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:43:55.370 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:43:55.370 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:43:55.370 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:43:55.370 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:43:55.370 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:43:55.370 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:43:55.370 | INFO | ------------------------------------------ | 20260507 01:43:55.370 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:43:55.370 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:43:55.370 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:43:55.370 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:43:55.370 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.56e+01, 1.33e+01 / 2.29e+00 | 20260507 01:43:55.370 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.02e-01, 1.00e-02 / 4.52e-03 | 20260507 01:43:55.370 | INFO | PerfReport: 11.68%, Baseline -- 3.60e+00 / 4.50e+00, 5.24e-01 / 1.63e-01 | 20260507 01:43:55.370 | INFO | PerfReport: 3.24%, Histogram -- 9.98e-01 / 1.50e+00, 1.71e-01 / 9.64e-02 | 20260507 01:43:55.370 | INFO | PerfReport: 16.16%, DME -- 3.99e+01 / 4.50e+01, 4.20e+00 / 3.15e+00 | 20260507 01:43:55.370 | INFO | PerfReport: 57.41%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.23e+00 / 2.02e+00 | 20260507 01:43:55.370 | INFO | PerfReport: 7.04%, PulseAccumulating -- 2.17e+00 / 2.76e+00, 3.88e-01 / 2.08e-01 | 20260507 01:43:55.370 | INFO | PerfReport: 3.11%, Metrics -- 9.58e-01 / 1.27e+00, 2.54e-01 / 1.73e-01 | 20260507 01:43:55.370 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.26e-01 / 3.79e-01, 5.14e-02 / 2.00e-02 | 20260507 01:43:55.370 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:43:55.370 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:43:55.370 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:43:55.370 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 01:43:55.370 | INFO | PerfReport: 21.64%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260507 01:43:55.370 | INFO | PerfReport: 21.41%, Download -- 2.07e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260507 01:43:55.370 | INFO | PerfReport: 31.96%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 01:43:55.370 | INFO | PerfReport: 1.33%, LockUpload -- 1.28e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 01:43:55.370 | INFO | PerfReport: 1.42%, LockDownload -- 1.37e+00 / 0.00e+00, 3.23e+00 / 0.00e+00 | 20260507 01:43:55.370 | INFO | PerfReport: 22.22%, LockCompute -- 2.14e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 01:43:55.370 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260507 01:43:55.465 | INFO | Starting a round of Baz IO | 20260507 01:43:58.586 | INFO | Populating RTMetrics report | 20260507 01:43:58.586 | INFO | RTMetrics report populated | 20260507 01:43:58.586 | INFO | Opening tmp RTMetrics json file | 20260507 01:43:58.588 | INFO | Serializing RTMetrics to json | 20260507 01:43:58.606 | INFO | Serialization done, swapping tmp file now... | 20260507 01:43:58.606 | INFO | RTMetrics done | 20260507 01:43:58.676 | INFO | Finished a round of Baz IO: Wrote 1.807 GiB in 3.211 seconds | 20260507 01:43:58.676 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.10% / 0.00% / 66.90% | 20260507 01:43:58.676 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.272151% / 0% / 7.54992e-05% | 20260507 01:43:58.676 | INFO | Baz IO overall: | 20260507 01:43:58.676 | INFO | BytesWritten : 3165839588697 | 20260507 01:43:58.676 | INFO | NumEvents : 1523450508504 | 20260507 01:44:00.515 | INFO | Analyzing chunk frames = [6111232,6111744) | 20260507 01:44:00.515 | INFO | MemInfo: { | 20260507 01:44:00.515 | INFO | "gpu" : " 17 GB", | 20260507 01:44:00.515 | INFO | "huge" : "", | 20260507 01:44:00.515 | INFO | "huge_shared" : " 27 GB", | 20260507 01:44:00.515 | INFO | "malloc" : " 14 GB", | 20260507 01:44:00.515 | INFO | "pinned" : " 29 GB" | 20260507 01:44:00.515 | INFO | } | 20260507 01:44:00.515 | INFO | Perf report for the last 5.14668 seconds: | 20260507 01:44:00.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:44:00.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:00.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:00.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:00.515 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 01:44:00.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:00.515 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:44:00.515 | INFO | PRE_HQ : 59.5%, 1.8, 56.1ms, 0 | 20260507 01:44:00.515 | INFO | BAZWRITER : 62.0%, 3.3, 106.2ms, 0 | 20260507 01:44:00.515 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:44:00.515 | INFO | Uploaded 36GiB to GPU in 2.02773s (17.7539GiB/s) | 20260507 01:44:00.516 | INFO | Downloaded 38GiB to GPU in 2.1712s (17.5018GiB/s) | 20260507 01:44:05.649 | INFO | Analyzing chunk frames = [6111744,6112256) | 20260507 01:44:05.649 | INFO | MemInfo: { | 20260507 01:44:05.649 | INFO | "gpu" : " 17 GB", | 20260507 01:44:05.649 | INFO | "huge" : "", | 20260507 01:44:05.649 | INFO | "huge_shared" : " 27 GB", | 20260507 01:44:05.649 | INFO | "malloc" : " 14 GB", | 20260507 01:44:05.649 | INFO | "pinned" : " 29 GB" | 20260507 01:44:05.649 | INFO | } | 20260507 01:44:05.649 | INFO | Perf report for the last 5.13415 seconds: | 20260507 01:44:05.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:44:05.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:05.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:05.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:05.649 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 01:44:05.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:05.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:05.649 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:44:05.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:05.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:05.650 | INFO | Uploaded 36GiB to GPU in 2.04177s (17.6317GiB/s) | 20260507 01:44:05.650 | INFO | Downloaded 34GiB to GPU in 1.9405s (17.5213GiB/s) | 20260507 01:44:10.786 | INFO | Analyzing chunk frames = [6112256,6112768) | 20260507 01:44:10.786 | INFO | MemInfo: { | 20260507 01:44:10.786 | INFO | "gpu" : " 17 GB", | 20260507 01:44:10.786 | INFO | "huge" : "", | 20260507 01:44:10.786 | INFO | "huge_shared" : " 27 GB", | 20260507 01:44:10.786 | INFO | "malloc" : " 14 GB", | 20260507 01:44:10.786 | INFO | "pinned" : " 29 GB" | 20260507 01:44:10.786 | INFO | } | 20260507 01:44:10.786 | INFO | Perf report for the last 5.13672 seconds: | 20260507 01:44:10.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:44:10.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:10.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:10.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:10.786 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 01:44:10.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:10.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:10.786 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:44:10.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:10.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:10.787 | INFO | Uploaded 36GiB to GPU in 1.97015s (18.2728GiB/s) | 20260507 01:44:10.787 | INFO | Downloaded 34GiB to GPU in 1.94125s (17.5144GiB/s) | 20260507 01:44:15.928 | INFO | Analyzing chunk frames = [6112768,6113280) | 20260507 01:44:15.928 | INFO | MemInfo: { | 20260507 01:44:15.928 | INFO | "gpu" : " 17 GB", | 20260507 01:44:15.928 | INFO | "huge" : "", | 20260507 01:44:15.928 | INFO | "huge_shared" : " 27 GB", | 20260507 01:44:15.928 | INFO | "malloc" : " 14 GB", | 20260507 01:44:15.928 | INFO | "pinned" : " 29 GB" | 20260507 01:44:15.928 | INFO | } | 20260507 01:44:15.928 | INFO | Perf report for the last 5.14249 seconds: | 20260507 01:44:15.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:44:15.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:15.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:15.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:15.929 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 01:44:15.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:15.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:15.929 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:44:15.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:15.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:15.929 | INFO | Uploaded 36GiB to GPU in 1.95521s (18.4124GiB/s) | 20260507 01:44:15.929 | INFO | Downloaded 34GiB to GPU in 1.90518s (17.8461GiB/s) | 20260507 01:44:21.064 | INFO | Analyzing chunk frames = [6113280,6113792) | 20260507 01:44:21.064 | INFO | MemInfo: { | 20260507 01:44:21.064 | INFO | "gpu" : " 17 GB", | 20260507 01:44:21.064 | INFO | "huge" : "", | 20260507 01:44:21.064 | INFO | "huge_shared" : " 27 GB", | 20260507 01:44:21.064 | INFO | "malloc" : " 14 GB", | 20260507 01:44:21.064 | INFO | "pinned" : " 29 GB" | 20260507 01:44:21.064 | INFO | } | 20260507 01:44:21.064 | INFO | Perf report for the last 5.13556 seconds: | 20260507 01:44:21.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:44:21.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:21.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:21.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:21.064 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 01:44:21.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:21.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:21.064 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:44:21.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:21.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:21.065 | INFO | Uploaded 36GiB to GPU in 1.98909s (18.0987GiB/s) | 20260507 01:44:21.065 | INFO | Downloaded 34GiB to GPU in 1.94953s (17.4401GiB/s) | 20260507 01:44:26.206 | INFO | Analyzing chunk frames = [6113792,6114304) | 20260507 01:44:26.206 | INFO | MemInfo: { | 20260507 01:44:26.206 | INFO | "gpu" : " 17 GB", | 20260507 01:44:26.206 | INFO | "huge" : "", | 20260507 01:44:26.206 | INFO | "huge_shared" : " 27 GB", | 20260507 01:44:26.206 | INFO | "malloc" : " 14 GB", | 20260507 01:44:26.206 | INFO | "pinned" : " 29 GB" | 20260507 01:44:26.206 | INFO | } | 20260507 01:44:26.206 | INFO | Perf report for the last 5.14239 seconds: | 20260507 01:44:26.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:44:26.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:26.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:26.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:26.206 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 01:44:26.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:26.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:26.207 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 01:44:26.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:26.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:26.207 | INFO | Uploaded 36GiB to GPU in 2.03749s (17.6688GiB/s) | 20260507 01:44:26.207 | INFO | Downloaded 34GiB to GPU in 1.95992s (17.3476GiB/s) | 20260507 01:44:31.346 | INFO | Analyzing chunk frames = [6114304,6114816) | 20260507 01:44:31.346 | INFO | MemInfo: { | 20260507 01:44:31.346 | INFO | "gpu" : " 17 GB", | 20260507 01:44:31.346 | INFO | "huge" : "", | 20260507 01:44:31.346 | INFO | "huge_shared" : " 27 GB", | 20260507 01:44:31.346 | INFO | "malloc" : " 14 GB", | 20260507 01:44:31.346 | INFO | "pinned" : " 29 GB" | 20260507 01:44:31.346 | INFO | } | 20260507 01:44:31.346 | INFO | Perf report for the last 5.13961 seconds: | 20260507 01:44:31.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:44:31.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:31.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:31.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:31.346 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 01:44:31.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:31.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:31.346 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 01:44:31.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:31.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:31.346 | INFO | Uploaded 36GiB to GPU in 1.96113s (18.3567GiB/s) | 20260507 01:44:31.346 | INFO | Downloaded 34GiB to GPU in 1.9915s (17.0726GiB/s) | 20260507 01:44:36.481 | INFO | Analyzing chunk frames = [6114816,6115328) | 20260507 01:44:36.481 | INFO | MemInfo: { | 20260507 01:44:36.481 | INFO | "gpu" : " 17 GB", | 20260507 01:44:36.481 | INFO | "huge" : "", | 20260507 01:44:36.481 | INFO | "huge_shared" : " 27 GB", | 20260507 01:44:36.481 | INFO | "malloc" : " 14 GB", | 20260507 01:44:36.481 | INFO | "pinned" : " 29 GB" | 20260507 01:44:36.481 | INFO | } | 20260507 01:44:36.481 | INFO | Perf report for the last 5.13522 seconds: | 20260507 01:44:36.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:44:36.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:36.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:36.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:36.481 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:44:36.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:36.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:36.481 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 01:44:36.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:36.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:36.482 | INFO | Uploaded 36GiB to GPU in 1.98849s (18.1042GiB/s) | 20260507 01:44:36.482 | INFO | Downloaded 34GiB to GPU in 2.00051s (16.9956GiB/s) | 20260507 01:44:36.579 | INFO | Starting a round of Baz IO | 20260507 01:44:39.699 | INFO | Populating RTMetrics report | 20260507 01:44:39.700 | INFO | RTMetrics report populated | 20260507 01:44:39.700 | INFO | Opening tmp RTMetrics json file | 20260507 01:44:39.708 | INFO | Serializing RTMetrics to json | 20260507 01:44:39.719 | INFO | Serialization done, swapping tmp file now... | 20260507 01:44:39.719 | INFO | RTMetrics done | 20260507 01:44:39.790 | INFO | Finished a round of Baz IO: Wrote 1.806 GiB in 3.21 seconds | 20260507 01:44:39.790 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.09% / 0.00% / 66.90% | 20260507 01:44:39.790 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.270883% / 3.97364e-06% / 6.75519e-05% | 20260507 01:44:39.790 | INFO | Baz IO overall: | 20260507 01:44:39.790 | INFO | BytesWritten : 3167778546206 | 20260507 01:44:39.790 | INFO | NumEvents : 1524369600554 | 20260507 01:44:41.625 | INFO | Analyzing chunk frames = [6115328,6115840) | 20260507 01:44:41.625 | INFO | MemInfo: { | 20260507 01:44:41.625 | INFO | "gpu" : " 17 GB", | 20260507 01:44:41.625 | INFO | "huge" : "", | 20260507 01:44:41.625 | INFO | "huge_shared" : " 27 GB", | 20260507 01:44:41.625 | INFO | "malloc" : " 14 GB", | 20260507 01:44:41.625 | INFO | "pinned" : " 29 GB" | 20260507 01:44:41.625 | INFO | } | 20260507 01:44:41.625 | INFO | Perf report for the last 5.14399 seconds: | 20260507 01:44:41.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:44:41.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:41.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:41.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:41.625 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260507 01:44:41.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:41.625 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 01:44:41.625 | INFO | PRE_HQ : 59.6%, 1.8, 56.0ms, 0 | 20260507 01:44:41.625 | INFO | BAZWRITER : 62.0%, 3.3, 105.5ms, 0 | 20260507 01:44:41.625 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:44:41.626 | INFO | Uploaded 36GiB to GPU in 2.02366s (17.7896GiB/s) | 20260507 01:44:41.626 | INFO | Downloaded 38GiB to GPU in 2.18143s (17.4197GiB/s) | 20260507 01:44:46.771 | INFO | Analyzing chunk frames = [6115840,6116352) | 20260507 01:44:46.771 | INFO | MemInfo: { | 20260507 01:44:46.771 | INFO | "gpu" : " 17 GB", | 20260507 01:44:46.771 | INFO | "huge" : "", | 20260507 01:44:46.771 | INFO | "huge_shared" : " 27 GB", | 20260507 01:44:46.772 | INFO | "malloc" : " 14 GB", | 20260507 01:44:46.772 | INFO | "pinned" : " 29 GB" | 20260507 01:44:46.772 | INFO | } | 20260507 01:44:46.772 | INFO | Perf report for the last 5.14628 seconds: | 20260507 01:44:46.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:44:46.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:46.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:46.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:46.772 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 01:44:46.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:46.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:46.772 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 01:44:46.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:46.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:46.772 | INFO | Uploaded 36GiB to GPU in 2.02284s (17.7967GiB/s) | 20260507 01:44:46.772 | INFO | Downloaded 34GiB to GPU in 1.93652s (17.5572GiB/s) | 20260507 01:44:51.904 | INFO | Analyzing chunk frames = [6116352,6116864) | 20260507 01:44:51.904 | INFO | MemInfo: { | 20260507 01:44:51.904 | INFO | "gpu" : " 17 GB", | 20260507 01:44:51.904 | INFO | "huge" : "", | 20260507 01:44:51.904 | INFO | "huge_shared" : " 27 GB", | 20260507 01:44:51.904 | INFO | "malloc" : " 14 GB", | 20260507 01:44:51.904 | INFO | "pinned" : " 29 GB" | 20260507 01:44:51.904 | INFO | } | 20260507 01:44:51.904 | INFO | Perf report for the last 5.13212 seconds: | 20260507 01:44:51.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:44:51.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:51.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:51.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:51.904 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 01:44:51.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:51.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:51.904 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:44:51.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:51.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:51.904 | INFO | Uploaded 36GiB to GPU in 2.0076s (17.9318GiB/s) | 20260507 01:44:51.904 | INFO | Downloaded 34GiB to GPU in 1.94293s (17.4993GiB/s) | 20260507 01:44:57.038 | INFO | Analyzing chunk frames = [6116864,6117376) | 20260507 01:44:57.038 | INFO | MemInfo: { | 20260507 01:44:57.038 | INFO | "gpu" : " 17 GB", | 20260507 01:44:57.038 | INFO | "huge" : "", | 20260507 01:44:57.038 | INFO | "huge_shared" : " 27 GB", | 20260507 01:44:57.038 | INFO | "malloc" : " 14 GB", | 20260507 01:44:57.038 | INFO | "pinned" : " 29 GB" | 20260507 01:44:57.038 | INFO | } | 20260507 01:44:57.038 | INFO | Perf report for the last 5.13472 seconds: | 20260507 01:44:57.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:44:57.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:57.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:57.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:57.039 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 01:44:57.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:57.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:57.039 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:44:57.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:57.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:44:57.039 | INFO | Uploaded 36GiB to GPU in 1.95296s (18.4335GiB/s) | 20260507 01:44:57.039 | INFO | Downloaded 34GiB to GPU in 1.90025s (17.8924GiB/s) | 20260507 01:45:02.182 | INFO | Analyzing chunk frames = [6117376,6117888) | 20260507 01:45:02.182 | INFO | MemInfo: { | 20260507 01:45:02.182 | INFO | "gpu" : " 17 GB", | 20260507 01:45:02.182 | INFO | "huge" : "", | 20260507 01:45:02.182 | INFO | "huge_shared" : " 27 GB", | 20260507 01:45:02.182 | INFO | "malloc" : " 14 GB", | 20260507 01:45:02.183 | INFO | "pinned" : " 29 GB" | 20260507 01:45:02.183 | INFO | } | 20260507 01:45:02.183 | INFO | Perf report for the last 5.14425 seconds: | 20260507 01:45:02.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:45:02.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:02.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:02.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:02.183 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 01:45:02.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:02.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:02.183 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 01:45:02.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:02.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:02.183 | INFO | Uploaded 36GiB to GPU in 1.99067s (18.0843GiB/s) | 20260507 01:45:02.183 | INFO | Downloaded 34GiB to GPU in 1.92654s (17.6482GiB/s) | 20260507 01:45:07.325 | INFO | Analyzing chunk frames = [6117888,6118400) | 20260507 01:45:07.325 | INFO | MemInfo: { | 20260507 01:45:07.325 | INFO | "gpu" : " 17 GB", | 20260507 01:45:07.325 | INFO | "huge" : "", | 20260507 01:45:07.325 | INFO | "huge_shared" : " 27 GB", | 20260507 01:45:07.325 | INFO | "malloc" : " 14 GB", | 20260507 01:45:07.325 | INFO | "pinned" : " 29 GB" | 20260507 01:45:07.325 | INFO | } | 20260507 01:45:07.325 | INFO | Perf report for the last 5.14236 seconds: | 20260507 01:45:07.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:45:07.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:07.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:07.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:07.325 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 01:45:07.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:07.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:07.325 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:45:07.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:07.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:07.325 | INFO | Uploaded 36GiB to GPU in 2.02676s (17.7623GiB/s) | 20260507 01:45:07.325 | INFO | Downloaded 34GiB to GPU in 1.95006s (17.4354GiB/s) | 20260507 01:45:12.464 | INFO | Analyzing chunk frames = [6118400,6118912) | 20260507 01:45:12.464 | INFO | MemInfo: { | 20260507 01:45:12.464 | INFO | "gpu" : " 17 GB", | 20260507 01:45:12.464 | INFO | "huge" : "", | 20260507 01:45:12.464 | INFO | "huge_shared" : " 27 GB", | 20260507 01:45:12.464 | INFO | "malloc" : " 14 GB", | 20260507 01:45:12.464 | INFO | "pinned" : " 29 GB" | 20260507 01:45:12.464 | INFO | } | 20260507 01:45:12.464 | INFO | Perf report for the last 5.13899 seconds: | 20260507 01:45:12.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:45:12.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:12.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:12.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:12.464 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:45:12.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:12.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:12.464 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 01:45:12.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:12.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:12.464 | INFO | Uploaded 36GiB to GPU in 1.99083s (18.0829GiB/s) | 20260507 01:45:12.464 | INFO | Downloaded 34GiB to GPU in 1.98976s (17.0875GiB/s) | 20260507 01:45:17.603 | INFO | Analyzing chunk frames = [6118912,6119424) | 20260507 01:45:17.603 | INFO | MemInfo: { | 20260507 01:45:17.603 | INFO | "gpu" : " 17 GB", | 20260507 01:45:17.603 | INFO | "huge" : "", | 20260507 01:45:17.603 | INFO | "huge_shared" : " 27 GB", | 20260507 01:45:17.603 | INFO | "malloc" : " 14 GB", | 20260507 01:45:17.603 | INFO | "pinned" : " 29 GB" | 20260507 01:45:17.603 | INFO | } | 20260507 01:45:17.604 | INFO | Perf report for the last 5.13951 seconds: | 20260507 01:45:17.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:45:17.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:17.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:17.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:17.604 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:45:17.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:17.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:17.604 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260507 01:45:17.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:17.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:17.604 | INFO | Uploaded 36GiB to GPU in 2.01191s (17.8934GiB/s) | 20260507 01:45:17.604 | INFO | Downloaded 34GiB to GPU in 2.00615s (16.9479GiB/s) | 20260507 01:45:17.604 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:45:17.604 | INFO | Total mem usage: 18 GiB | 20260507 01:45:17.604 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:45:17.604 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 01:45:17.604 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:45:17.604 | INFO | High Water Breakdown: | 20260507 01:45:17.604 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:45:17.604 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:45:17.604 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:45:17.604 | INFO | ------------------------------------------ | 20260507 01:45:17.604 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:45:17.604 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:45:17.604 | INFO | Total mem usage: 26.5 GiB | 20260507 01:45:17.604 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:45:17.604 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:45:17.604 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:45:17.604 | INFO | High Water Breakdown: | 20260507 01:45:17.604 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:45:17.604 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:45:17.604 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:45:17.604 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:45:17.604 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:45:17.604 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:45:17.604 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:45:17.604 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:45:17.605 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:45:17.605 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:45:17.605 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:45:17.605 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:45:17.605 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:45:17.605 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:45:17.605 | INFO | ------------------------------------------ | 20260507 01:45:17.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:45:17.605 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:45:17.605 | INFO | Total mem usage: 25.6 GiB | 20260507 01:45:17.605 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:45:17.605 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 01:45:17.605 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 01:45:17.605 | INFO | High Water Breakdown: | 20260507 01:45:17.605 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 01:45:17.605 | INFO | ------------------------------------------ | 20260507 01:45:17.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:45:17.605 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:45:17.605 | INFO | Total mem usage: 15.7 GiB | 20260507 01:45:17.605 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:45:17.605 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:45:17.605 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:45:17.605 | INFO | High Water Breakdown: | 20260507 01:45:17.605 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:45:17.605 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:45:17.605 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:45:17.605 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:45:17.605 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:45:17.605 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:45:17.605 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:45:17.605 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:45:17.605 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:45:17.605 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:45:17.605 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:45:17.605 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:45:17.605 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:45:17.605 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:45:17.605 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:45:17.605 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:45:17.605 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:45:17.605 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:45:17.605 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:45:17.605 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:45:17.605 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:45:17.605 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:45:17.605 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:45:17.605 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:45:17.605 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:45:17.605 | INFO | ------------------------------------------ | 20260507 01:45:17.605 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:45:17.605 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:45:17.605 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:45:17.605 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:45:17.605 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.53e+01, 1.35e+01 / 1.67e+00 | 20260507 01:45:17.605 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.02e-01, 9.88e-03 / 5.32e-03 | 20260507 01:45:17.605 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.48e+00, 5.27e-01 / 1.99e-01 | 20260507 01:45:17.605 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.48e+00, 1.73e-01 / 7.20e-02 | 20260507 01:45:17.605 | INFO | PerfReport: 16.07%, DME -- 3.97e+01 / 4.44e+01, 3.95e+00 / 2.09e+00 | 20260507 01:45:17.605 | INFO | PerfReport: 57.55%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.33e+00 / 1.55e+00 | 20260507 01:45:17.605 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.16e+00 / 2.69e+00, 3.83e-01 / 2.27e-01 | 20260507 01:45:17.605 | INFO | PerfReport: 3.11%, Metrics -- 9.58e-01 / 1.27e+00, 2.56e-01 / 1.23e-01 | 20260507 01:45:17.605 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.25e-01 / 3.80e-01, 5.18e-02 / 1.94e-02 | 20260507 01:45:17.606 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:45:17.606 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:45:17.606 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:45:17.606 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 01:45:17.606 | INFO | PerfReport: 21.62%, Upload -- 2.08e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260507 01:45:17.606 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260507 01:45:17.606 | INFO | PerfReport: 32.02%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 01:45:17.606 | INFO | PerfReport: 1.27%, LockUpload -- 1.23e+00 / 0.00e+00, 4.68e+00 / 0.00e+00 | 20260507 01:45:17.606 | INFO | PerfReport: 1.36%, LockDownload -- 1.31e+00 / 0.00e+00, 3.19e+00 / 0.00e+00 | 20260507 01:45:17.606 | INFO | PerfReport: 22.31%, LockCompute -- 2.15e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 01:45:17.606 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260507 01:45:17.704 | INFO | Starting a round of Baz IO | 20260507 01:45:20.813 | INFO | Populating RTMetrics report | 20260507 01:45:20.813 | INFO | RTMetrics report populated | 20260507 01:45:20.813 | INFO | Opening tmp RTMetrics json file | 20260507 01:45:20.814 | INFO | Serializing RTMetrics to json | 20260507 01:45:20.815 | INFO | Serialization done, swapping tmp file now... | 20260507 01:45:20.815 | INFO | RTMetrics done | 20260507 01:45:20.907 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.203 seconds | 20260507 01:45:20.907 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.09% / 0.00% / 66.91% | 20260507 01:45:20.907 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.269922% / 3.97364e-06% / 5.5631e-05% | 20260507 01:45:20.907 | INFO | Baz IO overall: | 20260507 01:45:20.907 | INFO | BytesWritten : 3169714718453 | 20260507 01:45:20.907 | INFO | NumEvents : 1525286458845 | 20260507 01:45:22.740 | INFO | Analyzing chunk frames = [6119424,6119936) | 20260507 01:45:22.740 | INFO | MemInfo: { | 20260507 01:45:22.740 | INFO | "gpu" : " 17 GB", | 20260507 01:45:22.740 | INFO | "huge" : "", | 20260507 01:45:22.740 | INFO | "huge_shared" : " 27 GB", | 20260507 01:45:22.740 | INFO | "malloc" : " 14 GB", | 20260507 01:45:22.740 | INFO | "pinned" : " 29 GB" | 20260507 01:45:22.740 | INFO | } | 20260507 01:45:22.740 | INFO | Perf report for the last 5.13674 seconds: | 20260507 01:45:22.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:45:22.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:22.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:22.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:22.741 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 01:45:22.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:22.741 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 01:45:22.741 | INFO | PRE_HQ : 59.5%, 1.8, 55.9ms, 0 | 20260507 01:45:22.741 | INFO | BAZWRITER : 62.0%, 3.3, 106.0ms, 0 | 20260507 01:45:22.741 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:45:22.741 | INFO | Uploaded 36GiB to GPU in 2.0459s (17.5962GiB/s) | 20260507 01:45:22.741 | INFO | Downloaded 38GiB to GPU in 2.19167s (17.3384GiB/s) | 20260507 01:45:27.875 | INFO | Analyzing chunk frames = [6119936,6120448) | 20260507 01:45:27.875 | INFO | MemInfo: { | 20260507 01:45:27.875 | INFO | "gpu" : " 17 GB", | 20260507 01:45:27.875 | INFO | "huge" : "", | 20260507 01:45:27.876 | INFO | "huge_shared" : " 27 GB", | 20260507 01:45:27.876 | INFO | "malloc" : " 14 GB", | 20260507 01:45:27.876 | INFO | "pinned" : " 29 GB" | 20260507 01:45:27.876 | INFO | } | 20260507 01:45:27.876 | INFO | Perf report for the last 5.13525 seconds: | 20260507 01:45:27.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:45:27.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:27.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:27.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:27.876 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:45:27.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:27.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:27.876 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 01:45:27.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:27.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:27.876 | INFO | Uploaded 36GiB to GPU in 2.03521s (17.6886GiB/s) | 20260507 01:45:27.876 | INFO | Downloaded 34GiB to GPU in 1.93819s (17.5422GiB/s) | 20260507 01:45:33.015 | INFO | Analyzing chunk frames = [6120448,6120960) | 20260507 01:45:33.015 | INFO | MemInfo: { | 20260507 01:45:33.015 | INFO | "gpu" : " 17 GB", | 20260507 01:45:33.015 | INFO | "huge" : "", | 20260507 01:45:33.015 | INFO | "huge_shared" : " 27 GB", | 20260507 01:45:33.015 | INFO | "malloc" : " 14 GB", | 20260507 01:45:33.015 | INFO | "pinned" : " 29 GB" | 20260507 01:45:33.015 | INFO | } | 20260507 01:45:33.015 | INFO | Perf report for the last 5.13942 seconds: | 20260507 01:45:33.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:45:33.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:33.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:33.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:33.015 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 01:45:33.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:33.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:33.015 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:45:33.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:33.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:33.015 | INFO | Uploaded 36GiB to GPU in 1.98706s (18.1172GiB/s) | 20260507 01:45:33.015 | INFO | Downloaded 34GiB to GPU in 1.93035s (17.6133GiB/s) | 20260507 01:45:38.159 | INFO | Analyzing chunk frames = [6120960,6121472) | 20260507 01:45:38.159 | INFO | MemInfo: { | 20260507 01:45:38.159 | INFO | "gpu" : " 17 GB", | 20260507 01:45:38.159 | INFO | "huge" : "", | 20260507 01:45:38.159 | INFO | "huge_shared" : " 27 GB", | 20260507 01:45:38.159 | INFO | "malloc" : " 14 GB", | 20260507 01:45:38.159 | INFO | "pinned" : " 29 GB" | 20260507 01:45:38.159 | INFO | } | 20260507 01:45:38.159 | INFO | Perf report for the last 5.14437 seconds: | 20260507 01:45:38.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:45:38.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:38.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:38.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:38.160 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:45:38.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:38.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:38.160 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:45:38.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:38.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:38.160 | INFO | Uploaded 36GiB to GPU in 1.97543s (18.2238GiB/s) | 20260507 01:45:38.160 | INFO | Downloaded 34GiB to GPU in 1.91166s (17.7856GiB/s) | 20260507 01:45:43.294 | INFO | Analyzing chunk frames = [6121472,6121984) | 20260507 01:45:43.294 | INFO | MemInfo: { | 20260507 01:45:43.294 | INFO | "gpu" : " 17 GB", | 20260507 01:45:43.294 | INFO | "huge" : "", | 20260507 01:45:43.294 | INFO | "huge_shared" : " 27 GB", | 20260507 01:45:43.294 | INFO | "malloc" : " 14 GB", | 20260507 01:45:43.294 | INFO | "pinned" : " 29 GB" | 20260507 01:45:43.294 | INFO | } | 20260507 01:45:43.294 | INFO | Perf report for the last 5.13422 seconds: | 20260507 01:45:43.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:45:43.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:43.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:43.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:43.294 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 01:45:43.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:43.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:43.294 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:45:43.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:43.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:43.294 | INFO | Uploaded 36GiB to GPU in 2.01295s (17.8842GiB/s) | 20260507 01:45:43.294 | INFO | Downloaded 34GiB to GPU in 1.94976s (17.438GiB/s) | 20260507 01:45:48.442 | INFO | Analyzing chunk frames = [6121984,6122496) | 20260507 01:45:48.442 | INFO | MemInfo: { | 20260507 01:45:48.442 | INFO | "gpu" : " 17 GB", | 20260507 01:45:48.442 | INFO | "huge" : "", | 20260507 01:45:48.442 | INFO | "huge_shared" : " 27 GB", | 20260507 01:45:48.442 | INFO | "malloc" : " 14 GB", | 20260507 01:45:48.442 | INFO | "pinned" : " 29 GB" | 20260507 01:45:48.442 | INFO | } | 20260507 01:45:48.443 | INFO | Perf report for the last 5.14878 seconds: | 20260507 01:45:48.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:45:48.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:48.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:48.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:48.443 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 01:45:48.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:48.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:48.443 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:45:48.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:48.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:48.443 | INFO | Uploaded 36GiB to GPU in 2.04064s (17.6416GiB/s) | 20260507 01:45:48.443 | INFO | Downloaded 34GiB to GPU in 1.9427s (17.5014GiB/s) | 20260507 01:45:53.583 | INFO | Analyzing chunk frames = [6122496,6123008) | 20260507 01:45:53.583 | INFO | MemInfo: { | 20260507 01:45:53.583 | INFO | "gpu" : " 17 GB", | 20260507 01:45:53.583 | INFO | "huge" : "", | 20260507 01:45:53.583 | INFO | "huge_shared" : " 27 GB", | 20260507 01:45:53.583 | INFO | "malloc" : " 14 GB", | 20260507 01:45:53.583 | INFO | "pinned" : " 29 GB" | 20260507 01:45:53.583 | INFO | } | 20260507 01:45:53.583 | INFO | Perf report for the last 5.1405 seconds: | 20260507 01:45:53.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:45:53.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:53.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:53.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:53.583 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 01:45:53.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:53.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:53.583 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 01:45:53.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:53.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:53.584 | INFO | Uploaded 36GiB to GPU in 1.96953s (18.2784GiB/s) | 20260507 01:45:53.584 | INFO | Downloaded 34GiB to GPU in 1.95855s (17.3598GiB/s) | 20260507 01:45:58.718 | INFO | Analyzing chunk frames = [6123008,6123520) | 20260507 01:45:58.718 | INFO | MemInfo: { | 20260507 01:45:58.718 | INFO | "gpu" : " 17 GB", | 20260507 01:45:58.718 | INFO | "huge" : "", | 20260507 01:45:58.718 | INFO | "huge_shared" : " 27 GB", | 20260507 01:45:58.719 | INFO | "malloc" : " 14 GB", | 20260507 01:45:58.719 | INFO | "pinned" : " 29 GB" | 20260507 01:45:58.719 | INFO | } | 20260507 01:45:58.719 | INFO | Perf report for the last 5.13557 seconds: | 20260507 01:45:58.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:45:58.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:58.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:58.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:58.719 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:45:58.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:58.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:58.719 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 01:45:58.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:58.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:45:58.719 | INFO | Uploaded 36GiB to GPU in 2.01984s (17.8232GiB/s) | 20260507 01:45:58.719 | INFO | Downloaded 34GiB to GPU in 1.99815s (17.0158GiB/s) | 20260507 01:45:58.817 | INFO | Starting a round of Baz IO | 20260507 01:46:01.935 | INFO | Populating RTMetrics report | 20260507 01:46:01.935 | INFO | RTMetrics report populated | 20260507 01:46:01.935 | INFO | Opening tmp RTMetrics json file | 20260507 01:46:01.936 | INFO | Serializing RTMetrics to json | 20260507 01:46:01.952 | INFO | Serialization done, swapping tmp file now... | 20260507 01:46:01.955 | INFO | RTMetrics done | 20260507 01:46:02.029 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.212 seconds | 20260507 01:46:02.029 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.09% / 0.00% / 66.91% | 20260507 01:46:02.029 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.268575% / 3.97364e-06% / 5.5631e-05% | 20260507 01:46:02.029 | INFO | Baz IO overall: | 20260507 01:46:02.029 | INFO | BytesWritten : 3171650902438 | 20260507 01:46:02.029 | INFO | NumEvents : 1526203274086 | 20260507 01:46:03.854 | INFO | Analyzing chunk frames = [6123520,6124032) | 20260507 01:46:03.854 | INFO | MemInfo: { | 20260507 01:46:03.854 | INFO | "gpu" : " 17 GB", | 20260507 01:46:03.854 | INFO | "huge" : "", | 20260507 01:46:03.854 | INFO | "huge_shared" : " 27 GB", | 20260507 01:46:03.854 | INFO | "malloc" : " 14 GB", | 20260507 01:46:03.854 | INFO | "pinned" : " 29 GB" | 20260507 01:46:03.854 | INFO | } | 20260507 01:46:03.854 | INFO | Perf report for the last 5.13581 seconds: | 20260507 01:46:03.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:46:03.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:03.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:03.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:03.854 | INFO | ANALYSIS : 62.1%, 2.9, 95.4ms, 0 | 20260507 01:46:03.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:03.855 | INFO | RT_METRICS : 46.6%, 1.0, 24.4ms, 0 | 20260507 01:46:03.855 | INFO | PRE_HQ : 60.6%, 2.0, 62.1ms, 0 | 20260507 01:46:03.855 | INFO | BAZWRITER : 62.1%, 3.5, 113.0ms, 0 | 20260507 01:46:03.855 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:46:03.855 | INFO | Uploaded 36GiB to GPU in 2.05953s (17.4797GiB/s) | 20260507 01:46:03.855 | INFO | Downloaded 38GiB to GPU in 2.17598s (17.4634GiB/s) | 20260507 01:46:08.994 | INFO | Analyzing chunk frames = [6124032,6124544) | 20260507 01:46:08.994 | INFO | MemInfo: { | 20260507 01:46:08.994 | INFO | "gpu" : " 17 GB", | 20260507 01:46:08.994 | INFO | "huge" : "", | 20260507 01:46:08.994 | INFO | "huge_shared" : " 27 GB", | 20260507 01:46:08.994 | INFO | "malloc" : " 14 GB", | 20260507 01:46:08.994 | INFO | "pinned" : " 29 GB" | 20260507 01:46:08.994 | INFO | } | 20260507 01:46:08.994 | INFO | Perf report for the last 5.13932 seconds: | 20260507 01:46:08.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:46:08.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:08.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:08.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:08.994 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:46:08.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:08.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:08.994 | INFO | PRE_HQ : 27.3%, 1.0, 14.1ms, 0 | 20260507 01:46:08.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:08.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:08.994 | INFO | Uploaded 36GiB to GPU in 2.00009s (17.9992GiB/s) | 20260507 01:46:08.994 | INFO | Downloaded 34GiB to GPU in 1.92368s (17.6744GiB/s) | 20260507 01:46:14.135 | INFO | Analyzing chunk frames = [6124544,6125056) | 20260507 01:46:14.135 | INFO | MemInfo: { | 20260507 01:46:14.135 | INFO | "gpu" : " 17 GB", | 20260507 01:46:14.135 | INFO | "huge" : "", | 20260507 01:46:14.135 | INFO | "huge_shared" : " 27 GB", | 20260507 01:46:14.135 | INFO | "malloc" : " 14 GB", | 20260507 01:46:14.135 | INFO | "pinned" : " 29 GB" | 20260507 01:46:14.135 | INFO | } | 20260507 01:46:14.136 | INFO | Perf report for the last 5.14172 seconds: | 20260507 01:46:14.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:46:14.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:14.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:14.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:14.136 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:46:14.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:14.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:14.136 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:46:14.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:14.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:14.136 | INFO | Uploaded 36GiB to GPU in 1.98086s (18.1739GiB/s) | 20260507 01:46:14.136 | INFO | Downloaded 34GiB to GPU in 1.92679s (17.646GiB/s) | 20260507 01:46:19.277 | INFO | Analyzing chunk frames = [6125056,6125568) | 20260507 01:46:19.277 | INFO | MemInfo: { | 20260507 01:46:19.277 | INFO | "gpu" : " 17 GB", | 20260507 01:46:19.277 | INFO | "huge" : "", | 20260507 01:46:19.277 | INFO | "huge_shared" : " 27 GB", | 20260507 01:46:19.277 | INFO | "malloc" : " 14 GB", | 20260507 01:46:19.277 | INFO | "pinned" : " 29 GB" | 20260507 01:46:19.277 | INFO | } | 20260507 01:46:19.277 | INFO | Perf report for the last 5.14167 seconds: | 20260507 01:46:19.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:46:19.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:19.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:19.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:19.277 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 01:46:19.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:19.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:19.277 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:46:19.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:19.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:19.277 | INFO | Uploaded 36GiB to GPU in 1.96864s (18.2867GiB/s) | 20260507 01:46:19.277 | INFO | Downloaded 34GiB to GPU in 1.89975s (17.8971GiB/s) | 20260507 01:46:24.413 | INFO | Analyzing chunk frames = [6125568,6126080) | 20260507 01:46:24.413 | INFO | MemInfo: { | 20260507 01:46:24.413 | INFO | "gpu" : " 17 GB", | 20260507 01:46:24.413 | INFO | "huge" : "", | 20260507 01:46:24.413 | INFO | "huge_shared" : " 27 GB", | 20260507 01:46:24.413 | INFO | "malloc" : " 14 GB", | 20260507 01:46:24.413 | INFO | "pinned" : " 29 GB" | 20260507 01:46:24.413 | INFO | } | 20260507 01:46:24.413 | INFO | Perf report for the last 5.13586 seconds: | 20260507 01:46:24.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:46:24.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:24.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:24.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:24.413 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 01:46:24.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:24.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:24.413 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:46:24.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:24.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:24.414 | INFO | Uploaded 36GiB to GPU in 2.03794s (17.6649GiB/s) | 20260507 01:46:24.414 | INFO | Downloaded 34GiB to GPU in 1.93331s (17.5865GiB/s) | 20260507 01:46:29.549 | INFO | Analyzing chunk frames = [6126080,6126592) | 20260507 01:46:29.549 | INFO | MemInfo: { | 20260507 01:46:29.549 | INFO | "gpu" : " 17 GB", | 20260507 01:46:29.549 | INFO | "huge" : "", | 20260507 01:46:29.549 | INFO | "huge_shared" : " 27 GB", | 20260507 01:46:29.550 | INFO | "malloc" : " 14 GB", | 20260507 01:46:29.550 | INFO | "pinned" : " 29 GB" | 20260507 01:46:29.550 | INFO | } | 20260507 01:46:29.550 | INFO | Perf report for the last 5.13655 seconds: | 20260507 01:46:29.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:46:29.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:29.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:29.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:29.550 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 01:46:29.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:29.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:29.550 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:46:29.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:29.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:29.550 | INFO | Uploaded 36GiB to GPU in 2.05851s (17.4884GiB/s) | 20260507 01:46:29.550 | INFO | Downloaded 34GiB to GPU in 1.9631s (17.3196GiB/s) | 20260507 01:46:34.691 | INFO | Analyzing chunk frames = [6126592,6127104) | 20260507 01:46:34.691 | INFO | MemInfo: { | 20260507 01:46:34.691 | INFO | "gpu" : " 17 GB", | 20260507 01:46:34.691 | INFO | "huge" : "", | 20260507 01:46:34.691 | INFO | "huge_shared" : " 27 GB", | 20260507 01:46:34.691 | INFO | "malloc" : " 14 GB", | 20260507 01:46:34.691 | INFO | "pinned" : " 29 GB" | 20260507 01:46:34.691 | INFO | } | 20260507 01:46:34.691 | INFO | Perf report for the last 5.14177 seconds: | 20260507 01:46:34.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:46:34.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:34.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:34.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:34.691 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:46:34.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:34.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:34.691 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 01:46:34.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:34.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:34.692 | INFO | Uploaded 36GiB to GPU in 1.99367s (18.0572GiB/s) | 20260507 01:46:34.692 | INFO | Downloaded 34GiB to GPU in 1.98079s (17.1649GiB/s) | 20260507 01:46:39.832 | INFO | Analyzing chunk frames = [6127104,6127616) | 20260507 01:46:39.832 | INFO | MemInfo: { | 20260507 01:46:39.832 | INFO | "gpu" : " 17 GB", | 20260507 01:46:39.832 | INFO | "huge" : "", | 20260507 01:46:39.832 | INFO | "huge_shared" : " 27 GB", | 20260507 01:46:39.832 | INFO | "malloc" : " 14 GB", | 20260507 01:46:39.832 | INFO | "pinned" : " 29 GB" | 20260507 01:46:39.832 | INFO | } | 20260507 01:46:39.832 | INFO | Perf report for the last 5.1403 seconds: | 20260507 01:46:39.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:46:39.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:39.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:39.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:39.832 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:46:39.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:39.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:39.832 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 01:46:39.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:39.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:39.832 | INFO | Uploaded 36GiB to GPU in 1.98937s (18.0962GiB/s) | 20260507 01:46:39.832 | INFO | Downloaded 34GiB to GPU in 1.97978s (17.1736GiB/s) | 20260507 01:46:39.832 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:46:39.832 | INFO | Total mem usage: 18 GiB | 20260507 01:46:39.832 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:46:39.832 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -25.4%) | 20260507 01:46:39.832 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:46:39.832 | INFO | High Water Breakdown: | 20260507 01:46:39.832 | INFO | 60.2% : 8.65 GiB ( 75.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:46:39.832 | INFO | 39.6% : 5.69 GiB ( 85% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:46:39.832 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:46:39.832 | INFO | ------------------------------------------ | 20260507 01:46:39.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:46:39.832 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:46:39.832 | INFO | Total mem usage: 26.5 GiB | 20260507 01:46:39.832 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:46:39.832 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:46:39.832 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:46:39.832 | INFO | High Water Breakdown: | 20260507 01:46:39.832 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:46:39.832 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:46:39.832 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:46:39.832 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:46:39.832 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:46:39.832 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:46:39.832 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:46:39.832 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:46:39.832 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:46:39.832 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:46:39.832 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:46:39.832 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:46:39.832 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:46:39.832 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:46:39.832 | INFO | ------------------------------------------ | 20260507 01:46:39.832 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:46:39.832 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:46:39.832 | INFO | Total mem usage: 25.6 GiB | 20260507 01:46:39.832 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:46:39.832 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:46:39.833 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:46:39.833 | INFO | High Water Breakdown: | 20260507 01:46:39.833 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:46:39.833 | INFO | ------------------------------------------ | 20260507 01:46:39.833 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:46:39.833 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:46:39.833 | INFO | Total mem usage: 15.7 GiB | 20260507 01:46:39.833 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:46:39.833 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:46:39.833 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:46:39.833 | INFO | High Water Breakdown: | 20260507 01:46:39.833 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:46:39.833 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:46:39.833 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:46:39.833 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:46:39.833 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:46:39.833 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:46:39.833 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:46:39.833 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:46:39.833 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:46:39.833 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:46:39.833 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:46:39.833 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:46:39.833 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:46:39.833 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:46:39.833 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:46:39.833 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:46:39.833 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:46:39.833 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:46:39.833 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:46:39.833 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:46:39.833 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:46:39.833 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:46:39.833 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:46:39.833 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:46:39.833 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:46:39.833 | INFO | ------------------------------------------ | 20260507 01:46:39.833 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:46:39.833 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:46:39.833 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:46:39.833 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:46:39.833 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.51e+01, 1.33e+01 / 1.66e+00 | 20260507 01:46:39.833 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.75e-03 / 5.20e-03 | 20260507 01:46:39.833 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.38e+00, 5.20e-01 / 3.11e-01 | 20260507 01:46:39.833 | INFO | PerfReport: 3.24%, Histogram -- 9.98e-01 / 1.49e+00, 1.73e-01 / 1.03e-01 | 20260507 01:46:39.833 | INFO | PerfReport: 16.09%, DME -- 3.97e+01 / 4.43e+01, 3.93e+00 / 2.15e+00 | 20260507 01:46:39.833 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.77e+01 / 2.05e+01, 1.31e+00 / 1.86e+00 | 20260507 01:46:39.833 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.16e+00 / 2.69e+00, 3.83e-01 / 2.29e-01 | 20260507 01:46:39.833 | INFO | PerfReport: 3.11%, Metrics -- 9.59e-01 / 1.28e+00, 2.58e-01 / 1.46e-01 | 20260507 01:46:39.833 | INFO | PerfReport: 1.06%, PulseSerializing -- 3.25e-01 / 3.78e-01, 5.23e-02 / 2.23e-02 | 20260507 01:46:39.833 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:46:39.833 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:46:39.833 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:46:39.833 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 01:46:39.833 | INFO | PerfReport: 21.74%, Upload -- 2.09e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260507 01:46:39.833 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260507 01:46:39.833 | INFO | PerfReport: 31.99%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 01:46:39.833 | INFO | PerfReport: 1.47%, LockUpload -- 1.41e+00 / 0.00e+00, 5.03e+00 / 0.00e+00 | 20260507 01:46:39.833 | INFO | PerfReport: 1.24%, LockDownload -- 1.20e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260507 01:46:39.833 | INFO | PerfReport: 22.20%, LockCompute -- 2.14e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 01:46:39.833 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260507 01:46:39.929 | INFO | Starting a round of Baz IO | 20260507 01:46:43.042 | INFO | Populating RTMetrics report | 20260507 01:46:43.042 | INFO | RTMetrics report populated | 20260507 01:46:43.042 | INFO | Opening tmp RTMetrics json file | 20260507 01:46:43.044 | INFO | Serializing RTMetrics to json | 20260507 01:46:43.062 | INFO | Serialization done, swapping tmp file now... | 20260507 01:46:43.063 | INFO | RTMetrics done | 20260507 01:46:43.134 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.205 seconds | 20260507 01:46:43.134 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.09% / 0.00% / 66.91% | 20260507 01:46:43.134 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.268058% / 3.97364e-06% / 7.15256e-05% | 20260507 01:46:43.134 | INFO | Baz IO overall: | 20260507 01:46:43.134 | INFO | BytesWritten : 3173586533941 | 20260507 01:46:43.134 | INFO | NumEvents : 1527119873834 | 20260507 01:46:44.968 | INFO | Analyzing chunk frames = [6127616,6128128) | 20260507 01:46:44.968 | INFO | MemInfo: { | 20260507 01:46:44.968 | INFO | "gpu" : " 17 GB", | 20260507 01:46:44.968 | INFO | "huge" : "", | 20260507 01:46:44.968 | INFO | "huge_shared" : " 27 GB", | 20260507 01:46:44.968 | INFO | "malloc" : " 14 GB", | 20260507 01:46:44.968 | INFO | "pinned" : " 29 GB" | 20260507 01:46:44.968 | INFO | } | 20260507 01:46:44.968 | INFO | Perf report for the last 5.13674 seconds: | 20260507 01:46:44.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:46:44.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:44.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:44.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:44.969 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260507 01:46:44.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:44.969 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 01:46:44.969 | INFO | PRE_HQ : 59.3%, 1.8, 55.7ms, 0 | 20260507 01:46:44.969 | INFO | BAZWRITER : 62.0%, 3.3, 105.5ms, 0 | 20260507 01:46:44.969 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:46:44.969 | INFO | Uploaded 36GiB to GPU in 2.02522s (17.7758GiB/s) | 20260507 01:46:44.969 | INFO | Downloaded 38GiB to GPU in 2.17851s (17.4431GiB/s) | 20260507 01:46:50.108 | INFO | Analyzing chunk frames = [6128128,6128640) | 20260507 01:46:50.108 | INFO | MemInfo: { | 20260507 01:46:50.108 | INFO | "gpu" : " 17 GB", | 20260507 01:46:50.108 | INFO | "huge" : "", | 20260507 01:46:50.108 | INFO | "huge_shared" : " 27 GB", | 20260507 01:46:50.108 | INFO | "malloc" : " 14 GB", | 20260507 01:46:50.108 | INFO | "pinned" : " 29 GB" | 20260507 01:46:50.108 | INFO | } | 20260507 01:46:50.108 | INFO | Perf report for the last 5.13932 seconds: | 20260507 01:46:50.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:46:50.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:50.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:50.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:50.108 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:46:50.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:50.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:50.108 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 01:46:50.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:50.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:50.108 | INFO | Uploaded 36GiB to GPU in 2.01158s (17.8964GiB/s) | 20260507 01:46:50.108 | INFO | Downloaded 34GiB to GPU in 1.93579s (17.5639GiB/s) | 20260507 01:46:55.247 | INFO | Analyzing chunk frames = [6128640,6129152) | 20260507 01:46:55.247 | INFO | MemInfo: { | 20260507 01:46:55.247 | INFO | "gpu" : " 17 GB", | 20260507 01:46:55.247 | INFO | "huge" : "", | 20260507 01:46:55.247 | INFO | "huge_shared" : " 27 GB", | 20260507 01:46:55.247 | INFO | "malloc" : " 14 GB", | 20260507 01:46:55.247 | INFO | "pinned" : " 29 GB" | 20260507 01:46:55.247 | INFO | } | 20260507 01:46:55.247 | INFO | Perf report for the last 5.13928 seconds: | 20260507 01:46:55.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:46:55.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:55.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:55.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:55.247 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 01:46:55.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:55.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:55.247 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:46:55.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:55.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:46:55.247 | INFO | Uploaded 36GiB to GPU in 1.96765s (18.296GiB/s) | 20260507 01:46:55.247 | INFO | Downloaded 34GiB to GPU in 1.93413s (17.579GiB/s) | 20260507 01:47:00.393 | INFO | Analyzing chunk frames = [6129152,6129664) | 20260507 01:47:00.393 | INFO | MemInfo: { | 20260507 01:47:00.393 | INFO | "gpu" : " 17 GB", | 20260507 01:47:00.393 | INFO | "huge" : "", | 20260507 01:47:00.393 | INFO | "huge_shared" : " 27 GB", | 20260507 01:47:00.393 | INFO | "malloc" : " 14 GB", | 20260507 01:47:00.393 | INFO | "pinned" : " 29 GB" | 20260507 01:47:00.393 | INFO | } | 20260507 01:47:00.393 | INFO | Perf report for the last 5.14559 seconds: | 20260507 01:47:00.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:47:00.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:00.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:00.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:00.393 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 01:47:00.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:00.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:00.393 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:47:00.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:00.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:00.393 | INFO | Uploaded 36GiB to GPU in 1.98092s (18.1734GiB/s) | 20260507 01:47:00.393 | INFO | Downloaded 34GiB to GPU in 1.91289s (17.7741GiB/s) | 20260507 01:47:05.529 | INFO | Analyzing chunk frames = [6129664,6130176) | 20260507 01:47:05.529 | INFO | MemInfo: { | 20260507 01:47:05.529 | INFO | "gpu" : " 17 GB", | 20260507 01:47:05.529 | INFO | "huge" : "", | 20260507 01:47:05.529 | INFO | "huge_shared" : " 27 GB", | 20260507 01:47:05.529 | INFO | "malloc" : " 14 GB", | 20260507 01:47:05.530 | INFO | "pinned" : " 29 GB" | 20260507 01:47:05.530 | INFO | } | 20260507 01:47:05.530 | INFO | Perf report for the last 5.13692 seconds: | 20260507 01:47:05.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:47:05.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:05.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:05.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:05.530 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 01:47:05.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:05.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:05.530 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:47:05.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:05.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:05.530 | INFO | Uploaded 36GiB to GPU in 1.99819s (18.0163GiB/s) | 20260507 01:47:05.530 | INFO | Downloaded 34GiB to GPU in 1.94808s (17.453GiB/s) | 20260507 01:47:10.663 | INFO | Analyzing chunk frames = [6130176,6130688) | 20260507 01:47:10.663 | INFO | MemInfo: { | 20260507 01:47:10.663 | INFO | "gpu" : " 17 GB", | 20260507 01:47:10.663 | INFO | "huge" : "", | 20260507 01:47:10.663 | INFO | "huge_shared" : " 27 GB", | 20260507 01:47:10.663 | INFO | "malloc" : " 14 GB", | 20260507 01:47:10.663 | INFO | "pinned" : " 29 GB" | 20260507 01:47:10.663 | INFO | } | 20260507 01:47:10.663 | INFO | Perf report for the last 5.13347 seconds: | 20260507 01:47:10.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:47:10.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:10.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:10.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:10.663 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 01:47:10.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:10.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:10.663 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:47:10.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:10.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:10.664 | INFO | Uploaded 36GiB to GPU in 2.04662s (17.59GiB/s) | 20260507 01:47:10.664 | INFO | Downloaded 34GiB to GPU in 1.96497s (17.303GiB/s) | 20260507 01:47:15.804 | INFO | Analyzing chunk frames = [6130688,6131200) | 20260507 01:47:15.804 | INFO | MemInfo: { | 20260507 01:47:15.804 | INFO | "gpu" : " 17 GB", | 20260507 01:47:15.804 | INFO | "huge" : "", | 20260507 01:47:15.804 | INFO | "huge_shared" : " 27 GB", | 20260507 01:47:15.804 | INFO | "malloc" : " 14 GB", | 20260507 01:47:15.804 | INFO | "pinned" : " 29 GB" | 20260507 01:47:15.804 | INFO | } | 20260507 01:47:15.804 | INFO | Perf report for the last 5.14134 seconds: | 20260507 01:47:15.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:47:15.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:15.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:15.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:15.805 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:47:15.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:15.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:15.805 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 01:47:15.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:15.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:15.805 | INFO | Uploaded 36GiB to GPU in 1.99614s (18.0348GiB/s) | 20260507 01:47:15.805 | INFO | Downloaded 34GiB to GPU in 1.98728s (17.1088GiB/s) | 20260507 01:47:20.946 | INFO | Analyzing chunk frames = [6131200,6131712) | 20260507 01:47:20.946 | INFO | MemInfo: { | 20260507 01:47:20.946 | INFO | "gpu" : " 17 GB", | 20260507 01:47:20.946 | INFO | "huge" : "", | 20260507 01:47:20.946 | INFO | "huge_shared" : " 27 GB", | 20260507 01:47:20.946 | INFO | "malloc" : " 14 GB", | 20260507 01:47:20.946 | INFO | "pinned" : " 29 GB" | 20260507 01:47:20.946 | INFO | } | 20260507 01:47:20.946 | INFO | Perf report for the last 5.14135 seconds: | 20260507 01:47:20.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:47:20.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:20.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:20.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:20.946 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:47:20.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:20.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:20.946 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 01:47:20.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:20.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:20.947 | INFO | Uploaded 36GiB to GPU in 1.98143s (18.1687GiB/s) | 20260507 01:47:20.947 | INFO | Downloaded 34GiB to GPU in 1.98379s (17.1389GiB/s) | 20260507 01:47:21.043 | INFO | Starting a round of Baz IO | 20260507 01:47:24.160 | INFO | Populating RTMetrics report | 20260507 01:47:24.161 | INFO | RTMetrics report populated | 20260507 01:47:24.161 | INFO | Opening tmp RTMetrics json file | 20260507 01:47:24.162 | INFO | Serializing RTMetrics to json | 20260507 01:47:24.180 | INFO | Serialization done, swapping tmp file now... | 20260507 01:47:24.181 | INFO | RTMetrics done | 20260507 01:47:24.253 | INFO | Finished a round of Baz IO: Wrote 1.805 GiB in 3.209 seconds | 20260507 01:47:24.253 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.09% / 0.00% / 66.91% | 20260507 01:47:24.253 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.269544% / 3.97364e-06% / 4.76837e-05% | 20260507 01:47:24.253 | INFO | Baz IO overall: | 20260507 01:47:24.253 | INFO | BytesWritten : 3175524494388 | 20260507 01:47:24.253 | INFO | NumEvents : 1528038242909 | 20260507 01:47:26.086 | INFO | Analyzing chunk frames = [6131712,6132224) | 20260507 01:47:26.086 | INFO | MemInfo: { | 20260507 01:47:26.086 | INFO | "gpu" : " 17 GB", | 20260507 01:47:26.086 | INFO | "huge" : "", | 20260507 01:47:26.086 | INFO | "huge_shared" : " 27 GB", | 20260507 01:47:26.086 | INFO | "malloc" : " 14 GB", | 20260507 01:47:26.086 | INFO | "pinned" : " 29 GB" | 20260507 01:47:26.086 | INFO | } | 20260507 01:47:26.087 | INFO | Perf report for the last 5.14065 seconds: | 20260507 01:47:26.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:47:26.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:26.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:26.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:26.087 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 01:47:26.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:26.087 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 01:47:26.087 | INFO | PRE_HQ : 59.5%, 1.8, 56.0ms, 0 | 20260507 01:47:26.087 | INFO | BAZWRITER : 62.0%, 3.4, 107.1ms, 0 | 20260507 01:47:26.087 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:47:26.087 | INFO | Uploaded 36GiB to GPU in 2.02458s (17.7814GiB/s) | 20260507 01:47:26.087 | INFO | Downloaded 38GiB to GPU in 2.17318s (17.4859GiB/s) | 20260507 01:47:31.228 | INFO | Analyzing chunk frames = [6132224,6132736) | 20260507 01:47:31.228 | INFO | MemInfo: { | 20260507 01:47:31.228 | INFO | "gpu" : " 17 GB", | 20260507 01:47:31.228 | INFO | "huge" : "", | 20260507 01:47:31.228 | INFO | "huge_shared" : " 27 GB", | 20260507 01:47:31.228 | INFO | "malloc" : " 14 GB", | 20260507 01:47:31.228 | INFO | "pinned" : " 29 GB" | 20260507 01:47:31.228 | INFO | } | 20260507 01:47:31.228 | INFO | Perf report for the last 5.14126 seconds: | 20260507 01:47:31.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:47:31.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:31.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:31.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:31.228 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:47:31.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:31.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:31.228 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:47:31.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:31.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:31.228 | INFO | Uploaded 36GiB to GPU in 2.00776s (17.9304GiB/s) | 20260507 01:47:31.228 | INFO | Downloaded 34GiB to GPU in 1.93238s (17.5949GiB/s) | 20260507 01:47:36.363 | INFO | Analyzing chunk frames = [6132736,6133248) | 20260507 01:47:36.363 | INFO | MemInfo: { | 20260507 01:47:36.363 | INFO | "gpu" : " 17 GB", | 20260507 01:47:36.363 | INFO | "huge" : "", | 20260507 01:47:36.363 | INFO | "huge_shared" : " 27 GB", | 20260507 01:47:36.363 | INFO | "malloc" : " 14 GB", | 20260507 01:47:36.363 | INFO | "pinned" : " 29 GB" | 20260507 01:47:36.363 | INFO | } | 20260507 01:47:36.363 | INFO | Perf report for the last 5.13521 seconds: | 20260507 01:47:36.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:47:36.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:36.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:36.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:36.363 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:47:36.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:36.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:36.363 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:47:36.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:36.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:36.363 | INFO | Uploaded 36GiB to GPU in 2.01056s (17.9055GiB/s) | 20260507 01:47:36.363 | INFO | Downloaded 34GiB to GPU in 1.93852s (17.5391GiB/s) | 20260507 01:47:41.504 | INFO | Analyzing chunk frames = [6133248,6133760) | 20260507 01:47:41.504 | INFO | MemInfo: { | 20260507 01:47:41.504 | INFO | "gpu" : " 17 GB", | 20260507 01:47:41.504 | INFO | "huge" : "", | 20260507 01:47:41.504 | INFO | "huge_shared" : " 27 GB", | 20260507 01:47:41.504 | INFO | "malloc" : " 14 GB", | 20260507 01:47:41.504 | INFO | "pinned" : " 29 GB" | 20260507 01:47:41.504 | INFO | } | 20260507 01:47:41.504 | INFO | Perf report for the last 5.14144 seconds: | 20260507 01:47:41.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:47:41.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:41.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:41.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:41.505 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:47:41.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:41.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:41.505 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:47:41.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:41.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:41.505 | INFO | Uploaded 36GiB to GPU in 1.97925s (18.1887GiB/s) | 20260507 01:47:41.505 | INFO | Downloaded 34GiB to GPU in 1.90595s (17.8389GiB/s) | 20260507 01:47:46.643 | INFO | Analyzing chunk frames = [6133760,6134272) | 20260507 01:47:46.643 | INFO | MemInfo: { | 20260507 01:47:46.643 | INFO | "gpu" : " 17 GB", | 20260507 01:47:46.643 | INFO | "huge" : "", | 20260507 01:47:46.643 | INFO | "huge_shared" : " 27 GB", | 20260507 01:47:46.643 | INFO | "malloc" : " 14 GB", | 20260507 01:47:46.643 | INFO | "pinned" : " 29 GB" | 20260507 01:47:46.643 | INFO | } | 20260507 01:47:46.643 | INFO | Perf report for the last 5.13874 seconds: | 20260507 01:47:46.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:47:46.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:46.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:46.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:46.643 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 01:47:46.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:46.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:46.643 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:47:46.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:46.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:46.643 | INFO | Uploaded 36GiB to GPU in 2.01881s (17.8323GiB/s) | 20260507 01:47:46.644 | INFO | Downloaded 34GiB to GPU in 1.94478s (17.4827GiB/s) | 20260507 01:47:51.784 | INFO | Analyzing chunk frames = [6134272,6134784) | 20260507 01:47:51.784 | INFO | MemInfo: { | 20260507 01:47:51.784 | INFO | "gpu" : " 17 GB", | 20260507 01:47:51.784 | INFO | "huge" : "", | 20260507 01:47:51.784 | INFO | "huge_shared" : " 27 GB", | 20260507 01:47:51.784 | INFO | "malloc" : " 14 GB", | 20260507 01:47:51.784 | INFO | "pinned" : " 29 GB" | 20260507 01:47:51.784 | INFO | } | 20260507 01:47:51.784 | INFO | Perf report for the last 5.14069 seconds: | 20260507 01:47:51.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:47:51.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:51.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:51.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:51.784 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 01:47:51.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:51.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:51.784 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:47:51.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:51.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:51.784 | INFO | Uploaded 36GiB to GPU in 2.05263s (17.5384GiB/s) | 20260507 01:47:51.784 | INFO | Downloaded 34GiB to GPU in 1.94965s (17.439GiB/s) | 20260507 01:47:56.919 | INFO | Analyzing chunk frames = [6134784,6135296) | 20260507 01:47:56.919 | INFO | MemInfo: { | 20260507 01:47:56.919 | INFO | "gpu" : " 17 GB", | 20260507 01:47:56.919 | INFO | "huge" : "", | 20260507 01:47:56.919 | INFO | "huge_shared" : " 27 GB", | 20260507 01:47:56.919 | INFO | "malloc" : " 14 GB", | 20260507 01:47:56.919 | INFO | "pinned" : " 29 GB" | 20260507 01:47:56.919 | INFO | } | 20260507 01:47:56.919 | INFO | Perf report for the last 5.13504 seconds: | 20260507 01:47:56.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:47:56.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:56.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:56.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:56.919 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:47:56.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:56.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:56.919 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 01:47:56.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:56.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:47:56.919 | INFO | Uploaded 36GiB to GPU in 1.98026s (18.1794GiB/s) | 20260507 01:47:56.919 | INFO | Downloaded 34GiB to GPU in 1.98607s (17.1193GiB/s) | 20260507 01:48:02.058 | INFO | Analyzing chunk frames = [6135296,6135808) | 20260507 01:48:02.058 | INFO | MemInfo: { | 20260507 01:48:02.058 | INFO | "gpu" : " 17 GB", | 20260507 01:48:02.058 | INFO | "huge" : "", | 20260507 01:48:02.058 | INFO | "huge_shared" : " 27 GB", | 20260507 01:48:02.058 | INFO | "malloc" : " 14 GB", | 20260507 01:48:02.058 | INFO | "pinned" : " 29 GB" | 20260507 01:48:02.058 | INFO | } | 20260507 01:48:02.058 | INFO | Perf report for the last 5.13879 seconds: | 20260507 01:48:02.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:48:02.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:02.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:02.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:02.058 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:48:02.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:02.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:02.058 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 01:48:02.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:02.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:02.058 | INFO | Uploaded 36GiB to GPU in 1.99335s (18.06GiB/s) | 20260507 01:48:02.058 | INFO | Downloaded 34GiB to GPU in 1.9808s (17.1647GiB/s) | 20260507 01:48:02.058 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:48:02.058 | INFO | Total mem usage: 18 GiB | 20260507 01:48:02.058 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:48:02.058 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:48:02.058 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:48:02.058 | INFO | High Water Breakdown: | 20260507 01:48:02.058 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:48:02.058 | INFO | 39.7% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:48:02.058 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:48:02.058 | INFO | ------------------------------------------ | 20260507 01:48:02.058 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:48:02.058 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:48:02.058 | INFO | Total mem usage: 26.5 GiB | 20260507 01:48:02.058 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:48:02.058 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:48:02.058 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:48:02.058 | INFO | High Water Breakdown: | 20260507 01:48:02.058 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:48:02.058 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:48:02.058 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:48:02.058 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:48:02.058 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:48:02.058 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:48:02.058 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:48:02.058 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:48:02.058 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:48:02.058 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:48:02.058 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:48:02.058 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:48:02.058 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:48:02.058 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:48:02.058 | INFO | ------------------------------------------ | 20260507 01:48:02.058 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:48:02.058 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:48:02.058 | INFO | Total mem usage: 25.6 GiB | 20260507 01:48:02.058 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:48:02.058 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:48:02.058 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:48:02.058 | INFO | High Water Breakdown: | 20260507 01:48:02.058 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:48:02.058 | INFO | ------------------------------------------ | 20260507 01:48:02.059 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:48:02.059 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:48:02.059 | INFO | Total mem usage: 15.7 GiB | 20260507 01:48:02.059 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:48:02.059 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:48:02.059 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:48:02.059 | INFO | High Water Breakdown: | 20260507 01:48:02.059 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:48:02.059 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:48:02.059 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:48:02.059 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:48:02.059 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:48:02.059 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:48:02.059 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:48:02.059 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:48:02.059 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:48:02.059 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:48:02.059 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:48:02.059 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:48:02.059 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:48:02.059 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:48:02.059 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:48:02.059 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:48:02.059 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:48:02.059 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:48:02.059 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:48:02.059 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:48:02.059 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:48:02.059 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:48:02.059 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:48:02.059 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:48:02.059 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:48:02.059 | INFO | ------------------------------------------ | 20260507 01:48:02.059 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:48:02.059 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:48:02.059 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:48:02.059 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:48:02.059 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.57e+01, 1.34e+01 / 1.87e+00 | 20260507 01:48:02.059 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.01e-01, 9.50e-03 / 5.35e-03 | 20260507 01:48:02.059 | INFO | PerfReport: 11.64%, Baseline -- 3.59e+00 / 4.47e+00, 5.13e-01 / 2.66e-01 | 20260507 01:48:02.059 | INFO | PerfReport: 3.23%, Histogram -- 9.96e-01 / 1.48e+00, 1.71e-01 / 9.03e-02 | 20260507 01:48:02.059 | INFO | PerfReport: 16.15%, DME -- 3.99e+01 / 4.50e+01, 4.18e+00 / 2.22e+00 | 20260507 01:48:02.059 | INFO | PerfReport: 57.52%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.33e+00 / 1.63e+00 | 20260507 01:48:02.059 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.16e+00 / 2.71e+00, 3.80e-01 / 1.89e-01 | 20260507 01:48:02.059 | INFO | PerfReport: 3.10%, Metrics -- 9.56e-01 / 1.29e+00, 2.55e-01 / 1.55e-01 | 20260507 01:48:02.059 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.25e-01 / 3.81e-01, 5.27e-02 / 2.71e-02 | 20260507 01:48:02.059 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:48:02.059 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:48:02.059 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:48:02.059 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 01:48:02.059 | INFO | PerfReport: 21.64%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260507 01:48:02.059 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 01:48:02.059 | INFO | PerfReport: 31.98%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 01:48:02.059 | INFO | PerfReport: 1.39%, LockUpload -- 1.34e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 01:48:02.059 | INFO | PerfReport: 1.36%, LockDownload -- 1.31e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260507 01:48:02.059 | INFO | PerfReport: 22.26%, LockCompute -- 2.15e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 01:48:02.059 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.43e-02 / 0.00e+00 | 20260507 01:48:02.155 | INFO | Starting a round of Baz IO | 20260507 01:48:05.278 | INFO | Populating RTMetrics report | 20260507 01:48:05.279 | INFO | RTMetrics report populated | 20260507 01:48:05.279 | INFO | Opening tmp RTMetrics json file | 20260507 01:48:05.297 | INFO | Serializing RTMetrics to json | 20260507 01:48:05.298 | INFO | Serialization done, swapping tmp file now... | 20260507 01:48:05.299 | INFO | RTMetrics done | 20260507 01:48:05.367 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.212 seconds | 20260507 01:48:05.367 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.09% / 0.00% / 66.91% | 20260507 01:48:05.367 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.268006% / 0% / 7.54992e-05% | 20260507 01:48:05.367 | INFO | Baz IO overall: | 20260507 01:48:05.367 | INFO | BytesWritten : 3177460830255 | 20260507 01:48:05.367 | INFO | NumEvents : 1528955294334 | 20260507 01:48:07.203 | INFO | Analyzing chunk frames = [6135808,6136320) | 20260507 01:48:07.203 | INFO | MemInfo: { | 20260507 01:48:07.203 | INFO | "gpu" : " 17 GB", | 20260507 01:48:07.203 | INFO | "huge" : "", | 20260507 01:48:07.203 | INFO | "huge_shared" : " 27 GB", | 20260507 01:48:07.203 | INFO | "malloc" : " 14 GB", | 20260507 01:48:07.203 | INFO | "pinned" : " 29 GB" | 20260507 01:48:07.203 | INFO | } | 20260507 01:48:07.203 | INFO | Perf report for the last 5.14509 seconds: | 20260507 01:48:07.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:48:07.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:07.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:07.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:07.203 | INFO | ANALYSIS : 62.1%, 3.0, 96.7ms, 0 | 20260507 01:48:07.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:07.203 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:48:07.203 | INFO | PRE_HQ : 59.4%, 1.8, 55.7ms, 0 | 20260507 01:48:07.203 | INFO | BAZWRITER : 62.0%, 3.3, 106.2ms, 0 | 20260507 01:48:07.203 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:48:07.203 | INFO | Uploaded 36GiB to GPU in 2.04583s (17.5967GiB/s) | 20260507 01:48:07.203 | INFO | Downloaded 38GiB to GPU in 2.18474s (17.3934GiB/s) | 20260507 01:48:12.342 | INFO | Analyzing chunk frames = [6136320,6136832) | 20260507 01:48:12.342 | INFO | MemInfo: { | 20260507 01:48:12.342 | INFO | "gpu" : " 17 GB", | 20260507 01:48:12.342 | INFO | "huge" : "", | 20260507 01:48:12.342 | INFO | "huge_shared" : " 27 GB", | 20260507 01:48:12.342 | INFO | "malloc" : " 14 GB", | 20260507 01:48:12.342 | INFO | "pinned" : " 29 GB" | 20260507 01:48:12.342 | INFO | } | 20260507 01:48:12.342 | INFO | Perf report for the last 5.13904 seconds: | 20260507 01:48:12.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:48:12.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:12.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:12.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:12.342 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:48:12.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:12.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:12.342 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 01:48:12.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:12.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:12.342 | INFO | Uploaded 36GiB to GPU in 2.01552s (17.8614GiB/s) | 20260507 01:48:12.342 | INFO | Downloaded 34GiB to GPU in 1.93437s (17.5768GiB/s) | 20260507 01:48:17.479 | INFO | Analyzing chunk frames = [6136832,6137344) | 20260507 01:48:17.479 | INFO | MemInfo: { | 20260507 01:48:17.479 | INFO | "gpu" : " 17 GB", | 20260507 01:48:17.479 | INFO | "huge" : "", | 20260507 01:48:17.479 | INFO | "huge_shared" : " 27 GB", | 20260507 01:48:17.479 | INFO | "malloc" : " 14 GB", | 20260507 01:48:17.479 | INFO | "pinned" : " 29 GB" | 20260507 01:48:17.479 | INFO | } | 20260507 01:48:17.479 | INFO | Perf report for the last 5.13698 seconds: | 20260507 01:48:17.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:48:17.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:17.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:17.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:17.479 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:48:17.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:17.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:17.479 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:48:17.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:17.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:17.479 | INFO | Uploaded 36GiB to GPU in 2.01748s (17.844GiB/s) | 20260507 01:48:17.479 | INFO | Downloaded 34GiB to GPU in 1.93923s (17.5327GiB/s) | 20260507 01:48:22.614 | INFO | Analyzing chunk frames = [6137344,6137856) | 20260507 01:48:22.614 | INFO | MemInfo: { | 20260507 01:48:22.614 | INFO | "gpu" : " 17 GB", | 20260507 01:48:22.614 | INFO | "huge" : "", | 20260507 01:48:22.614 | INFO | "huge_shared" : " 27 GB", | 20260507 01:48:22.614 | INFO | "malloc" : " 14 GB", | 20260507 01:48:22.614 | INFO | "pinned" : " 29 GB" | 20260507 01:48:22.614 | INFO | } | 20260507 01:48:22.614 | INFO | Perf report for the last 5.13482 seconds: | 20260507 01:48:22.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:48:22.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:22.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:22.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:22.614 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 01:48:22.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:22.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:22.614 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:48:22.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:22.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:22.614 | INFO | Uploaded 36GiB to GPU in 1.98516s (18.1345GiB/s) | 20260507 01:48:22.614 | INFO | Downloaded 34GiB to GPU in 1.9173s (17.7332GiB/s) | 20260507 01:48:27.751 | INFO | Analyzing chunk frames = [6137856,6138368) | 20260507 01:48:27.752 | INFO | MemInfo: { | 20260507 01:48:27.752 | INFO | "gpu" : " 17 GB", | 20260507 01:48:27.752 | INFO | "huge" : "", | 20260507 01:48:27.752 | INFO | "huge_shared" : " 27 GB", | 20260507 01:48:27.752 | INFO | "malloc" : " 14 GB", | 20260507 01:48:27.752 | INFO | "pinned" : " 29 GB" | 20260507 01:48:27.752 | INFO | } | 20260507 01:48:27.752 | INFO | Perf report for the last 5.13791 seconds: | 20260507 01:48:27.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:48:27.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:27.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:27.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:27.752 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 01:48:27.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:27.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:27.752 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:48:27.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:27.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:27.752 | INFO | Uploaded 36GiB to GPU in 2.00498s (17.9553GiB/s) | 20260507 01:48:27.752 | INFO | Downloaded 34GiB to GPU in 1.94802s (17.4537GiB/s) | 20260507 01:48:32.903 | INFO | Analyzing chunk frames = [6138368,6138880) | 20260507 01:48:32.903 | INFO | MemInfo: { | 20260507 01:48:32.903 | INFO | "gpu" : " 17 GB", | 20260507 01:48:32.903 | INFO | "huge" : "", | 20260507 01:48:32.903 | INFO | "huge_shared" : " 27 GB", | 20260507 01:48:32.903 | INFO | "malloc" : " 14 GB", | 20260507 01:48:32.903 | INFO | "pinned" : " 29 GB" | 20260507 01:48:32.903 | INFO | } | 20260507 01:48:32.903 | INFO | Perf report for the last 5.15105 seconds: | 20260507 01:48:32.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:48:32.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:32.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:32.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:32.903 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 01:48:32.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:32.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:32.903 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 01:48:32.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:32.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:32.903 | INFO | Uploaded 36GiB to GPU in 2.02907s (17.7422GiB/s) | 20260507 01:48:32.903 | INFO | Downloaded 34GiB to GPU in 1.93912s (17.5337GiB/s) | 20260507 01:48:38.041 | INFO | Analyzing chunk frames = [6138880,6139392) | 20260507 01:48:38.041 | INFO | MemInfo: { | 20260507 01:48:38.041 | INFO | "gpu" : " 17 GB", | 20260507 01:48:38.041 | INFO | "huge" : "", | 20260507 01:48:38.041 | INFO | "huge_shared" : " 27 GB", | 20260507 01:48:38.041 | INFO | "malloc" : " 14 GB", | 20260507 01:48:38.041 | INFO | "pinned" : " 29 GB" | 20260507 01:48:38.041 | INFO | } | 20260507 01:48:38.041 | INFO | Perf report for the last 5.13852 seconds: | 20260507 01:48:38.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:48:38.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:38.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:38.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:38.041 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:48:38.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:38.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:38.041 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:48:38.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:38.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:38.041 | INFO | Uploaded 36GiB to GPU in 2.00673s (17.9397GiB/s) | 20260507 01:48:38.042 | INFO | Downloaded 34GiB to GPU in 1.97638s (17.2032GiB/s) | 20260507 01:48:43.182 | INFO | Analyzing chunk frames = [6139392,6139904) | 20260507 01:48:43.182 | INFO | MemInfo: { | 20260507 01:48:43.182 | INFO | "gpu" : " 17 GB", | 20260507 01:48:43.182 | INFO | "huge" : "", | 20260507 01:48:43.182 | INFO | "huge_shared" : " 27 GB", | 20260507 01:48:43.182 | INFO | "malloc" : " 14 GB", | 20260507 01:48:43.182 | INFO | "pinned" : " 29 GB" | 20260507 01:48:43.182 | INFO | } | 20260507 01:48:43.182 | INFO | Perf report for the last 5.1408 seconds: | 20260507 01:48:43.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:48:43.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:43.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:43.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:43.182 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:48:43.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:43.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:43.182 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:48:43.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:43.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:43.182 | INFO | Uploaded 36GiB to GPU in 2.02248s (17.8GiB/s) | 20260507 01:48:43.182 | INFO | Downloaded 34GiB to GPU in 1.97961s (17.1751GiB/s) | 20260507 01:48:43.279 | INFO | Starting a round of Baz IO | 20260507 01:48:46.390 | INFO | Populating RTMetrics report | 20260507 01:48:46.391 | INFO | RTMetrics report populated | 20260507 01:48:46.391 | INFO | Opening tmp RTMetrics json file | 20260507 01:48:46.392 | INFO | Serializing RTMetrics to json | 20260507 01:48:46.410 | INFO | Serialization done, swapping tmp file now... | 20260507 01:48:46.411 | INFO | RTMetrics done | 20260507 01:48:46.481 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.201 seconds | 20260507 01:48:46.481 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.08% / 0.00% / 66.91% | 20260507 01:48:46.481 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.267919% / 3.97364e-06% / 9.93411e-05% | 20260507 01:48:46.481 | INFO | Baz IO overall: | 20260507 01:48:46.481 | INFO | BytesWritten : 3179396054776 | 20260507 01:48:46.481 | INFO | NumEvents : 1529871655426 | 20260507 01:48:48.316 | INFO | Analyzing chunk frames = [6139904,6140416) | 20260507 01:48:48.316 | INFO | MemInfo: { | 20260507 01:48:48.316 | INFO | "gpu" : " 17 GB", | 20260507 01:48:48.316 | INFO | "huge" : "", | 20260507 01:48:48.316 | INFO | "huge_shared" : " 27 GB", | 20260507 01:48:48.316 | INFO | "malloc" : " 14 GB", | 20260507 01:48:48.316 | INFO | "pinned" : " 29 GB" | 20260507 01:48:48.316 | INFO | } | 20260507 01:48:48.316 | INFO | Perf report for the last 5.13386 seconds: | 20260507 01:48:48.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:48:48.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:48.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:48.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:48.316 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 01:48:48.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:48.316 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 01:48:48.316 | INFO | PRE_HQ : 59.3%, 1.8, 55.8ms, 0 | 20260507 01:48:48.316 | INFO | BAZWRITER : 61.9%, 3.4, 107.4ms, 0 | 20260507 01:48:48.316 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:48:48.316 | INFO | Uploaded 36GiB to GPU in 2.03827s (17.662GiB/s) | 20260507 01:48:48.316 | INFO | Downloaded 38GiB to GPU in 2.1776s (17.4504GiB/s) | 20260507 01:48:53.454 | INFO | Analyzing chunk frames = [6140416,6140928) | 20260507 01:48:53.454 | INFO | MemInfo: { | 20260507 01:48:53.454 | INFO | "gpu" : " 17 GB", | 20260507 01:48:53.454 | INFO | "huge" : "", | 20260507 01:48:53.454 | INFO | "huge_shared" : " 27 GB", | 20260507 01:48:53.454 | INFO | "malloc" : " 14 GB", | 20260507 01:48:53.454 | INFO | "pinned" : " 29 GB" | 20260507 01:48:53.454 | INFO | } | 20260507 01:48:53.454 | INFO | Perf report for the last 5.1381 seconds: | 20260507 01:48:53.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:48:53.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:53.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:53.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:53.454 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:48:53.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:53.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:53.454 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 01:48:53.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:53.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:53.455 | INFO | Uploaded 36GiB to GPU in 1.99443s (18.0503GiB/s) | 20260507 01:48:53.455 | INFO | Downloaded 34GiB to GPU in 1.92519s (17.6606GiB/s) | 20260507 01:48:58.595 | INFO | Analyzing chunk frames = [6140928,6141440) | 20260507 01:48:58.595 | INFO | MemInfo: { | 20260507 01:48:58.595 | INFO | "gpu" : " 17 GB", | 20260507 01:48:58.595 | INFO | "huge" : "", | 20260507 01:48:58.595 | INFO | "huge_shared" : " 27 GB", | 20260507 01:48:58.595 | INFO | "malloc" : " 14 GB", | 20260507 01:48:58.595 | INFO | "pinned" : " 29 GB" | 20260507 01:48:58.595 | INFO | } | 20260507 01:48:58.595 | INFO | Perf report for the last 5.14064 seconds: | 20260507 01:48:58.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:48:58.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:58.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:58.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:58.595 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 01:48:58.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:58.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:58.595 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:48:58.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:58.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:48:58.595 | INFO | Uploaded 36GiB to GPU in 2.00976s (17.9126GiB/s) | 20260507 01:48:58.595 | INFO | Downloaded 34GiB to GPU in 1.93513s (17.5699GiB/s) | 20260507 01:49:03.732 | INFO | Analyzing chunk frames = [6141440,6141952) | 20260507 01:49:03.732 | INFO | MemInfo: { | 20260507 01:49:03.732 | INFO | "gpu" : " 17 GB", | 20260507 01:49:03.732 | INFO | "huge" : "", | 20260507 01:49:03.733 | INFO | "huge_shared" : " 27 GB", | 20260507 01:49:03.733 | INFO | "malloc" : " 14 GB", | 20260507 01:49:03.733 | INFO | "pinned" : " 29 GB" | 20260507 01:49:03.733 | INFO | } | 20260507 01:49:03.733 | INFO | Perf report for the last 5.13792 seconds: | 20260507 01:49:03.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:49:03.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:03.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:03.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:03.733 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:49:03.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:03.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:03.733 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:49:03.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:03.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:03.733 | INFO | Uploaded 36GiB to GPU in 1.9873s (18.115GiB/s) | 20260507 01:49:03.733 | INFO | Downloaded 34GiB to GPU in 1.92038s (17.7048GiB/s) | 20260507 01:49:08.869 | INFO | Analyzing chunk frames = [6141952,6142464) | 20260507 01:49:08.869 | INFO | MemInfo: { | 20260507 01:49:08.869 | INFO | "gpu" : " 17 GB", | 20260507 01:49:08.869 | INFO | "huge" : "", | 20260507 01:49:08.869 | INFO | "huge_shared" : " 27 GB", | 20260507 01:49:08.869 | INFO | "malloc" : " 14 GB", | 20260507 01:49:08.869 | INFO | "pinned" : " 29 GB" | 20260507 01:49:08.869 | INFO | } | 20260507 01:49:08.869 | INFO | Perf report for the last 5.13673 seconds: | 20260507 01:49:08.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:49:08.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:08.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:08.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:08.869 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 01:49:08.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:08.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:08.869 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:49:08.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:08.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:08.870 | INFO | Uploaded 36GiB to GPU in 2.01313s (17.8826GiB/s) | 20260507 01:49:08.870 | INFO | Downloaded 34GiB to GPU in 1.9404s (17.5222GiB/s) | 20260507 01:49:14.012 | INFO | Analyzing chunk frames = [6142464,6142976) | 20260507 01:49:14.012 | INFO | MemInfo: { | 20260507 01:49:14.012 | INFO | "gpu" : " 17 GB", | 20260507 01:49:14.012 | INFO | "huge" : "", | 20260507 01:49:14.012 | INFO | "huge_shared" : " 27 GB", | 20260507 01:49:14.012 | INFO | "malloc" : " 14 GB", | 20260507 01:49:14.012 | INFO | "pinned" : " 29 GB" | 20260507 01:49:14.012 | INFO | } | 20260507 01:49:14.012 | INFO | Perf report for the last 5.14267 seconds: | 20260507 01:49:14.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:49:14.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:14.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:14.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:14.012 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 01:49:14.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:14.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:14.012 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 01:49:14.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:14.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:14.012 | INFO | Uploaded 36GiB to GPU in 2.01744s (17.8444GiB/s) | 20260507 01:49:14.012 | INFO | Downloaded 34GiB to GPU in 1.9432s (17.4969GiB/s) | 20260507 01:49:19.152 | INFO | Analyzing chunk frames = [6142976,6143488) | 20260507 01:49:19.152 | INFO | MemInfo: { | 20260507 01:49:19.152 | INFO | "gpu" : " 17 GB", | 20260507 01:49:19.152 | INFO | "huge" : "", | 20260507 01:49:19.152 | INFO | "huge_shared" : " 27 GB", | 20260507 01:49:19.152 | INFO | "malloc" : " 14 GB", | 20260507 01:49:19.152 | INFO | "pinned" : " 29 GB" | 20260507 01:49:19.152 | INFO | } | 20260507 01:49:19.153 | INFO | Perf report for the last 5.14046 seconds: | 20260507 01:49:19.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:49:19.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:19.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:19.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:19.153 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:49:19.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:19.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:19.153 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:49:19.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:19.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:19.153 | INFO | Uploaded 36GiB to GPU in 1.98131s (18.1698GiB/s) | 20260507 01:49:19.153 | INFO | Downloaded 34GiB to GPU in 1.99915s (17.0072GiB/s) | 20260507 01:49:24.292 | INFO | Analyzing chunk frames = [6143488,6144000) | 20260507 01:49:24.292 | INFO | MemInfo: { | 20260507 01:49:24.292 | INFO | "gpu" : " 17 GB", | 20260507 01:49:24.292 | INFO | "huge" : "", | 20260507 01:49:24.292 | INFO | "huge_shared" : " 27 GB", | 20260507 01:49:24.292 | INFO | "malloc" : " 14 GB", | 20260507 01:49:24.292 | INFO | "pinned" : " 29 GB" | 20260507 01:49:24.292 | INFO | } | 20260507 01:49:24.292 | INFO | Perf report for the last 5.13936 seconds: | 20260507 01:49:24.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:49:24.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:24.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:24.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:24.292 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:49:24.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:24.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:24.292 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:49:24.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:24.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:24.292 | INFO | Uploaded 36GiB to GPU in 2.00012s (17.999GiB/s) | 20260507 01:49:24.292 | INFO | Downloaded 34GiB to GPU in 1.97722s (17.1958GiB/s) | 20260507 01:49:24.292 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:49:24.292 | INFO | Total mem usage: 18 GiB | 20260507 01:49:24.292 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:49:24.292 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 01:49:24.292 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:49:24.292 | INFO | High Water Breakdown: | 20260507 01:49:24.292 | INFO | 60.2% : 8.58 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:49:24.292 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:49:24.292 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:49:24.292 | INFO | ------------------------------------------ | 20260507 01:49:24.292 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:49:24.292 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:49:24.292 | INFO | Total mem usage: 26.5 GiB | 20260507 01:49:24.292 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:49:24.292 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:49:24.292 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:49:24.292 | INFO | High Water Breakdown: | 20260507 01:49:24.292 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:49:24.292 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:49:24.293 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:49:24.293 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:49:24.293 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:49:24.293 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:49:24.293 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:49:24.293 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:49:24.293 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:49:24.293 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:49:24.293 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:49:24.293 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:49:24.293 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:49:24.293 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:49:24.293 | INFO | ------------------------------------------ | 20260507 01:49:24.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:49:24.293 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:49:24.293 | INFO | Total mem usage: 25.6 GiB | 20260507 01:49:24.293 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:49:24.293 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:49:24.293 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:49:24.293 | INFO | High Water Breakdown: | 20260507 01:49:24.293 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:49:24.293 | INFO | ------------------------------------------ | 20260507 01:49:24.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:49:24.293 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:49:24.293 | INFO | Total mem usage: 15.7 GiB | 20260507 01:49:24.293 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:49:24.293 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:49:24.293 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:49:24.293 | INFO | High Water Breakdown: | 20260507 01:49:24.293 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:49:24.293 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:49:24.293 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:49:24.293 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:49:24.293 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:49:24.293 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:49:24.293 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:49:24.293 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:49:24.293 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:49:24.293 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:49:24.293 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:49:24.293 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:49:24.293 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:49:24.293 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:49:24.293 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:49:24.293 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:49:24.293 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:49:24.293 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:49:24.293 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:49:24.293 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:49:24.293 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:49:24.293 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:49:24.293 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:49:24.293 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:49:24.293 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:49:24.293 | INFO | ------------------------------------------ | 20260507 01:49:24.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:49:24.293 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:49:24.293 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:49:24.293 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:49:24.293 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.55e+01, 1.33e+01 / 1.67e+00 | 20260507 01:49:24.293 | INFO | PerfReport: 0.30%, AggregateStats -- 9.12e-02 / 1.02e-01, 9.92e-03 / 5.11e-03 | 20260507 01:49:24.293 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.43e+00, 5.30e-01 / 2.92e-01 | 20260507 01:49:24.293 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.47e+00, 1.73e-01 / 1.06e-01 | 20260507 01:49:24.293 | INFO | PerfReport: 16.09%, DME -- 3.97e+01 / 4.45e+01, 3.90e+00 / 2.35e+00 | 20260507 01:49:24.293 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.77e+01 / 2.07e+01, 1.31e+00 / 1.79e+00 | 20260507 01:49:24.293 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.16e+00 / 2.68e+00, 3.82e-01 / 2.46e-01 | 20260507 01:49:24.293 | INFO | PerfReport: 3.10%, Metrics -- 9.56e-01 / 1.27e+00, 2.56e-01 / 1.14e-01 | 20260507 01:49:24.293 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.25e-01 / 3.82e-01, 5.26e-02 / 2.08e-02 | 20260507 01:49:24.293 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:49:24.293 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:49:24.293 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:49:24.293 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 01:49:24.293 | INFO | PerfReport: 21.71%, Upload -- 2.09e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260507 01:49:24.293 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260507 01:49:24.293 | INFO | PerfReport: 31.97%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 01:49:24.293 | INFO | PerfReport: 1.44%, LockUpload -- 1.39e+00 / 0.00e+00, 5.01e+00 / 0.00e+00 | 20260507 01:49:24.293 | INFO | PerfReport: 1.27%, LockDownload -- 1.22e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260507 01:49:24.293 | INFO | PerfReport: 22.25%, LockCompute -- 2.15e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 01:49:24.293 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260507 01:49:24.388 | INFO | Starting a round of Baz IO | 20260507 01:49:27.499 | INFO | Populating RTMetrics report | 20260507 01:49:27.500 | INFO | RTMetrics report populated | 20260507 01:49:27.500 | INFO | Opening tmp RTMetrics json file | 20260507 01:49:27.501 | INFO | Serializing RTMetrics to json | 20260507 01:49:27.502 | INFO | Serialization done, swapping tmp file now... | 20260507 01:49:27.504 | INFO | RTMetrics done | 20260507 01:49:27.593 | INFO | Finished a round of Baz IO: Wrote 1.805 GiB in 3.204 seconds | 20260507 01:49:27.593 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.08% / 0.00% / 66.92% | 20260507 01:49:27.593 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.267772% / 0% / 9.93411e-05% | 20260507 01:49:27.593 | INFO | Baz IO overall: | 20260507 01:49:27.593 | INFO | BytesWritten : 3181334672621 | 20260507 01:49:27.593 | INFO | NumEvents : 1530790793967 | 20260507 01:49:29.429 | INFO | Analyzing chunk frames = [6144000,6144512) | 20260507 01:49:29.429 | INFO | MemInfo: { | 20260507 01:49:29.429 | INFO | "gpu" : " 17 GB", | 20260507 01:49:29.429 | INFO | "huge" : "", | 20260507 01:49:29.429 | INFO | "huge_shared" : " 27 GB", | 20260507 01:49:29.429 | INFO | "malloc" : " 14 GB", | 20260507 01:49:29.429 | INFO | "pinned" : " 29 GB" | 20260507 01:49:29.429 | INFO | } | 20260507 01:49:29.429 | INFO | Perf report for the last 5.13717 seconds: | 20260507 01:49:29.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:49:29.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:29.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:29.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:29.429 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 01:49:29.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:29.429 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 01:49:29.429 | INFO | PRE_HQ : 59.3%, 1.8, 55.7ms, 0 | 20260507 01:49:29.429 | INFO | BAZWRITER : 62.0%, 3.4, 107.2ms, 0 | 20260507 01:49:29.429 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:49:29.429 | INFO | Uploaded 36GiB to GPU in 2.03865s (17.6587GiB/s) | 20260507 01:49:29.429 | INFO | Downloaded 38GiB to GPU in 2.18014s (17.4301GiB/s) | 20260507 01:49:34.564 | INFO | Analyzing chunk frames = [6144512,6145024) | 20260507 01:49:34.564 | INFO | MemInfo: { | 20260507 01:49:34.564 | INFO | "gpu" : " 17 GB", | 20260507 01:49:34.564 | INFO | "huge" : "", | 20260507 01:49:34.564 | INFO | "huge_shared" : " 27 GB", | 20260507 01:49:34.564 | INFO | "malloc" : " 14 GB", | 20260507 01:49:34.564 | INFO | "pinned" : " 29 GB" | 20260507 01:49:34.564 | INFO | } | 20260507 01:49:34.564 | INFO | Perf report for the last 5.13528 seconds: | 20260507 01:49:34.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:49:34.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:34.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:34.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:34.564 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:49:34.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:34.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:34.564 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:49:34.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:34.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:34.565 | INFO | Uploaded 36GiB to GPU in 2.03056s (17.7291GiB/s) | 20260507 01:49:34.565 | INFO | Downloaded 34GiB to GPU in 1.93549s (17.5666GiB/s) | 20260507 01:49:39.709 | INFO | Analyzing chunk frames = [6145024,6145536) | 20260507 01:49:39.709 | INFO | MemInfo: { | 20260507 01:49:39.709 | INFO | "gpu" : " 17 GB", | 20260507 01:49:39.709 | INFO | "huge" : "", | 20260507 01:49:39.710 | INFO | "huge_shared" : " 27 GB", | 20260507 01:49:39.710 | INFO | "malloc" : " 14 GB", | 20260507 01:49:39.710 | INFO | "pinned" : " 29 GB" | 20260507 01:49:39.710 | INFO | } | 20260507 01:49:39.710 | INFO | Perf report for the last 5.14523 seconds: | 20260507 01:49:39.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:49:39.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:39.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:39.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:39.710 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:49:39.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:39.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:39.710 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 01:49:39.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:39.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:39.710 | INFO | Uploaded 36GiB to GPU in 1.98464s (18.1393GiB/s) | 20260507 01:49:39.710 | INFO | Downloaded 34GiB to GPU in 1.92898s (17.6259GiB/s) | 20260507 01:49:44.857 | INFO | Analyzing chunk frames = [6145536,6146048) | 20260507 01:49:44.857 | INFO | MemInfo: { | 20260507 01:49:44.857 | INFO | "gpu" : " 17 GB", | 20260507 01:49:44.857 | INFO | "huge" : "", | 20260507 01:49:44.857 | INFO | "huge_shared" : " 27 GB", | 20260507 01:49:44.857 | INFO | "malloc" : " 14 GB", | 20260507 01:49:44.857 | INFO | "pinned" : " 29 GB" | 20260507 01:49:44.857 | INFO | } | 20260507 01:49:44.857 | INFO | Perf report for the last 5.14778 seconds: | 20260507 01:49:44.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:49:44.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:44.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:44.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:44.857 | INFO | ANALYSIS : 59.7%, 3.0, 92.8ms, 0 | 20260507 01:49:44.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:44.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:44.858 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:49:44.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:44.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:44.858 | INFO | Uploaded 36GiB to GPU in 1.97613s (18.2174GiB/s) | 20260507 01:49:44.858 | INFO | Downloaded 34GiB to GPU in 1.9087s (17.8132GiB/s) | 20260507 01:49:49.997 | INFO | Analyzing chunk frames = [6146048,6146560) | 20260507 01:49:49.997 | INFO | MemInfo: { | 20260507 01:49:49.997 | INFO | "gpu" : " 17 GB", | 20260507 01:49:49.997 | INFO | "huge" : "", | 20260507 01:49:49.997 | INFO | "huge_shared" : " 27 GB", | 20260507 01:49:49.997 | INFO | "malloc" : " 14 GB", | 20260507 01:49:49.997 | INFO | "pinned" : " 29 GB" | 20260507 01:49:49.997 | INFO | } | 20260507 01:49:49.998 | INFO | Perf report for the last 5.14011 seconds: | 20260507 01:49:49.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:49:49.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:49.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:49.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:49.998 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 01:49:49.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:49.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:49.998 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:49:49.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:49.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:49.998 | INFO | Uploaded 36GiB to GPU in 2.04174s (17.632GiB/s) | 20260507 01:49:49.998 | INFO | Downloaded 34GiB to GPU in 1.93935s (17.5316GiB/s) | 20260507 01:49:55.125 | INFO | Analyzing chunk frames = [6146560,6147072) | 20260507 01:49:55.125 | INFO | MemInfo: { | 20260507 01:49:55.125 | INFO | "gpu" : " 17 GB", | 20260507 01:49:55.125 | INFO | "huge" : "", | 20260507 01:49:55.125 | INFO | "huge_shared" : " 27 GB", | 20260507 01:49:55.125 | INFO | "malloc" : " 14 GB", | 20260507 01:49:55.125 | INFO | "pinned" : " 29 GB" | 20260507 01:49:55.125 | INFO | } | 20260507 01:49:55.125 | INFO | Perf report for the last 5.12732 seconds: | 20260507 01:49:55.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:49:55.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:55.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:55.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:55.125 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 01:49:55.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:55.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:55.125 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 01:49:55.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:55.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:49:55.125 | INFO | Uploaded 36GiB to GPU in 2.02008s (17.8211GiB/s) | 20260507 01:49:55.125 | INFO | Downloaded 34GiB to GPU in 1.9292s (17.6239GiB/s) | 20260507 01:50:00.267 | INFO | Analyzing chunk frames = [6147072,6147584) | 20260507 01:50:00.267 | INFO | MemInfo: { | 20260507 01:50:00.267 | INFO | "gpu" : " 17 GB", | 20260507 01:50:00.267 | INFO | "huge" : "", | 20260507 01:50:00.267 | INFO | "huge_shared" : " 27 GB", | 20260507 01:50:00.267 | INFO | "malloc" : " 14 GB", | 20260507 01:50:00.267 | INFO | "pinned" : " 29 GB" | 20260507 01:50:00.267 | INFO | } | 20260507 01:50:00.267 | INFO | Perf report for the last 5.14232 seconds: | 20260507 01:50:00.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:50:00.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:00.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:00.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:00.267 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:50:00.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:00.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:00.267 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:50:00.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:00.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:00.267 | INFO | Uploaded 36GiB to GPU in 2.00069s (17.9938GiB/s) | 20260507 01:50:00.267 | INFO | Downloaded 34GiB to GPU in 1.98292s (17.1464GiB/s) | 20260507 01:50:05.405 | INFO | Analyzing chunk frames = [6147584,6148096) | 20260507 01:50:05.405 | INFO | MemInfo: { | 20260507 01:50:05.405 | INFO | "gpu" : " 17 GB", | 20260507 01:50:05.405 | INFO | "huge" : "", | 20260507 01:50:05.405 | INFO | "huge_shared" : " 27 GB", | 20260507 01:50:05.405 | INFO | "malloc" : " 14 GB", | 20260507 01:50:05.405 | INFO | "pinned" : " 29 GB" | 20260507 01:50:05.405 | INFO | } | 20260507 01:50:05.405 | INFO | Perf report for the last 5.13794 seconds: | 20260507 01:50:05.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:50:05.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:05.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:05.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:05.405 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:50:05.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:05.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:05.405 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 01:50:05.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:05.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:05.405 | INFO | Uploaded 36GiB to GPU in 2.00847s (17.9241GiB/s) | 20260507 01:50:05.405 | INFO | Downloaded 34GiB to GPU in 1.9933s (17.0571GiB/s) | 20260507 01:50:05.502 | INFO | Starting a round of Baz IO | 20260507 01:50:08.616 | INFO | Populating RTMetrics report | 20260507 01:50:08.616 | INFO | RTMetrics report populated | 20260507 01:50:08.616 | INFO | Opening tmp RTMetrics json file | 20260507 01:50:08.618 | INFO | Serializing RTMetrics to json | 20260507 01:50:08.636 | INFO | Serialization done, swapping tmp file now... | 20260507 01:50:08.637 | INFO | RTMetrics done | 20260507 01:50:08.708 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.205 seconds | 20260507 01:50:08.708 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.08% / 0.00% / 66.92% | 20260507 01:50:08.708 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.266528% / 3.97364e-06% / 7.15256e-05% | 20260507 01:50:08.708 | INFO | Baz IO overall: | 20260507 01:50:08.708 | INFO | BytesWritten : 3183270177914 | 20260507 01:50:08.708 | INFO | NumEvents : 1531707163799 | 20260507 01:50:10.542 | INFO | Analyzing chunk frames = [6148096,6148608) | 20260507 01:50:10.542 | INFO | MemInfo: { | 20260507 01:50:10.542 | INFO | "gpu" : " 17 GB", | 20260507 01:50:10.542 | INFO | "huge" : "", | 20260507 01:50:10.542 | INFO | "huge_shared" : " 27 GB", | 20260507 01:50:10.542 | INFO | "malloc" : " 14 GB", | 20260507 01:50:10.542 | INFO | "pinned" : " 29 GB" | 20260507 01:50:10.542 | INFO | } | 20260507 01:50:10.542 | INFO | Perf report for the last 5.13707 seconds: | 20260507 01:50:10.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:50:10.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:10.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:10.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:10.542 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260507 01:50:10.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:10.542 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 01:50:10.542 | INFO | PRE_HQ : 59.8%, 1.8, 57.2ms, 0 | 20260507 01:50:10.542 | INFO | BAZWRITER : 62.0%, 3.4, 108.7ms, 0 | 20260507 01:50:10.542 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 01:50:10.543 | INFO | Uploaded 36GiB to GPU in 2.04778s (17.58GiB/s) | 20260507 01:50:10.543 | INFO | Downloaded 38GiB to GPU in 2.18172s (17.4174GiB/s) | 20260507 01:50:15.680 | INFO | Analyzing chunk frames = [6148608,6149120) | 20260507 01:50:15.680 | INFO | MemInfo: { | 20260507 01:50:15.680 | INFO | "gpu" : " 17 GB", | 20260507 01:50:15.680 | INFO | "huge" : "", | 20260507 01:50:15.680 | INFO | "huge_shared" : " 27 GB", | 20260507 01:50:15.680 | INFO | "malloc" : " 14 GB", | 20260507 01:50:15.680 | INFO | "pinned" : " 29 GB" | 20260507 01:50:15.680 | INFO | } | 20260507 01:50:15.680 | INFO | Perf report for the last 5.1382 seconds: | 20260507 01:50:15.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:50:15.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:15.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:15.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:15.680 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:50:15.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:15.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:15.680 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 01:50:15.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:15.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:15.681 | INFO | Uploaded 36GiB to GPU in 2.01533s (17.8631GiB/s) | 20260507 01:50:15.681 | INFO | Downloaded 34GiB to GPU in 1.93176s (17.6006GiB/s) | 20260507 01:50:20.819 | INFO | Analyzing chunk frames = [6149120,6149632) | 20260507 01:50:20.819 | INFO | MemInfo: { | 20260507 01:50:20.819 | INFO | "gpu" : " 17 GB", | 20260507 01:50:20.819 | INFO | "huge" : "", | 20260507 01:50:20.819 | INFO | "huge_shared" : " 27 GB", | 20260507 01:50:20.819 | INFO | "malloc" : " 14 GB", | 20260507 01:50:20.819 | INFO | "pinned" : " 29 GB" | 20260507 01:50:20.819 | INFO | } | 20260507 01:50:20.819 | INFO | Perf report for the last 5.13882 seconds: | 20260507 01:50:20.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:50:20.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:20.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:20.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:20.819 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 01:50:20.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:20.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:20.819 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:50:20.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:20.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:20.819 | INFO | Uploaded 36GiB to GPU in 2.00766s (17.9314GiB/s) | 20260507 01:50:20.819 | INFO | Downloaded 34GiB to GPU in 1.93488s (17.5721GiB/s) | 20260507 01:50:25.964 | INFO | Analyzing chunk frames = [6149632,6150144) | 20260507 01:50:25.964 | INFO | MemInfo: { | 20260507 01:50:25.964 | INFO | "gpu" : " 17 GB", | 20260507 01:50:25.964 | INFO | "huge" : "", | 20260507 01:50:25.964 | INFO | "huge_shared" : " 27 GB", | 20260507 01:50:25.964 | INFO | "malloc" : " 14 GB", | 20260507 01:50:25.964 | INFO | "pinned" : " 29 GB" | 20260507 01:50:25.964 | INFO | } | 20260507 01:50:25.964 | INFO | Perf report for the last 5.14447 seconds: | 20260507 01:50:25.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:50:25.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:25.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:25.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:25.964 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 01:50:25.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:25.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:25.964 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:50:25.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:25.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:25.964 | INFO | Uploaded 36GiB to GPU in 2.01998s (17.822GiB/s) | 20260507 01:50:25.964 | INFO | Downloaded 34GiB to GPU in 1.91003s (17.8007GiB/s) | 20260507 01:50:31.103 | INFO | Analyzing chunk frames = [6150144,6150656) | 20260507 01:50:31.103 | INFO | MemInfo: { | 20260507 01:50:31.103 | INFO | "gpu" : " 17 GB", | 20260507 01:50:31.103 | INFO | "huge" : "", | 20260507 01:50:31.103 | INFO | "huge_shared" : " 27 GB", | 20260507 01:50:31.103 | INFO | "malloc" : " 14 GB", | 20260507 01:50:31.103 | INFO | "pinned" : " 29 GB" | 20260507 01:50:31.103 | INFO | } | 20260507 01:50:31.103 | INFO | Perf report for the last 5.13934 seconds: | 20260507 01:50:31.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:50:31.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:31.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:31.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:31.103 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 01:50:31.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:31.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:31.103 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:50:31.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:31.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:31.103 | INFO | Uploaded 36GiB to GPU in 2.00733s (17.9342GiB/s) | 20260507 01:50:31.103 | INFO | Downloaded 34GiB to GPU in 1.9503s (17.4332GiB/s) | 20260507 01:50:36.237 | INFO | Analyzing chunk frames = [6150656,6151168) | 20260507 01:50:36.237 | INFO | MemInfo: { | 20260507 01:50:36.237 | INFO | "gpu" : " 17 GB", | 20260507 01:50:36.237 | INFO | "huge" : "", | 20260507 01:50:36.237 | INFO | "huge_shared" : " 27 GB", | 20260507 01:50:36.237 | INFO | "malloc" : " 14 GB", | 20260507 01:50:36.237 | INFO | "pinned" : " 29 GB" | 20260507 01:50:36.237 | INFO | } | 20260507 01:50:36.238 | INFO | Perf report for the last 5.13441 seconds: | 20260507 01:50:36.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:50:36.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:36.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:36.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:36.238 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 01:50:36.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:36.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:36.238 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:50:36.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:36.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:36.238 | INFO | Uploaded 36GiB to GPU in 2.05821s (17.491GiB/s) | 20260507 01:50:36.238 | INFO | Downloaded 34GiB to GPU in 1.95418s (17.3986GiB/s) | 20260507 01:50:41.376 | INFO | Analyzing chunk frames = [6151168,6151680) | 20260507 01:50:41.376 | INFO | MemInfo: { | 20260507 01:50:41.376 | INFO | "gpu" : " 17 GB", | 20260507 01:50:41.376 | INFO | "huge" : "", | 20260507 01:50:41.376 | INFO | "huge_shared" : " 27 GB", | 20260507 01:50:41.376 | INFO | "malloc" : " 14 GB", | 20260507 01:50:41.376 | INFO | "pinned" : " 29 GB" | 20260507 01:50:41.376 | INFO | } | 20260507 01:50:41.376 | INFO | Perf report for the last 5.1382 seconds: | 20260507 01:50:41.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:50:41.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:41.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:41.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:41.376 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:50:41.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:41.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:41.376 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 01:50:41.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:41.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:41.376 | INFO | Uploaded 36GiB to GPU in 1.99452s (18.0494GiB/s) | 20260507 01:50:41.376 | INFO | Downloaded 34GiB to GPU in 1.98234s (17.1515GiB/s) | 20260507 01:50:46.515 | INFO | Analyzing chunk frames = [6151680,6152192) | 20260507 01:50:46.515 | INFO | MemInfo: { | 20260507 01:50:46.515 | INFO | "gpu" : " 17 GB", | 20260507 01:50:46.515 | INFO | "huge" : "", | 20260507 01:50:46.515 | INFO | "huge_shared" : " 27 GB", | 20260507 01:50:46.515 | INFO | "malloc" : " 14 GB", | 20260507 01:50:46.515 | INFO | "pinned" : " 29 GB" | 20260507 01:50:46.515 | INFO | } | 20260507 01:50:46.515 | INFO | Perf report for the last 5.13947 seconds: | 20260507 01:50:46.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:50:46.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:46.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:46.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:46.515 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:50:46.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:46.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:46.515 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:50:46.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:46.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:46.516 | INFO | Uploaded 36GiB to GPU in 1.98952s (18.0948GiB/s) | 20260507 01:50:46.516 | INFO | Downloaded 34GiB to GPU in 1.98091s (17.1638GiB/s) | 20260507 01:50:46.516 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:50:46.516 | INFO | Total mem usage: 18 GiB | 20260507 01:50:46.516 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:50:46.516 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 01:50:46.516 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:50:46.516 | INFO | High Water Breakdown: | 20260507 01:50:46.516 | INFO | 60.2% : 8.58 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:50:46.516 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:50:46.516 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:50:46.516 | INFO | ------------------------------------------ | 20260507 01:50:46.516 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:50:46.516 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:50:46.516 | INFO | Total mem usage: 26.5 GiB | 20260507 01:50:46.516 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:50:46.516 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:50:46.516 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:50:46.516 | INFO | High Water Breakdown: | 20260507 01:50:46.516 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:50:46.516 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:50:46.516 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:50:46.516 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:50:46.516 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:50:46.516 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:50:46.516 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:50:46.516 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:50:46.516 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:50:46.516 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:50:46.516 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:50:46.516 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:50:46.516 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:50:46.516 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:50:46.516 | INFO | ------------------------------------------ | 20260507 01:50:46.516 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:50:46.516 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:50:46.516 | INFO | Total mem usage: 25.6 GiB | 20260507 01:50:46.516 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:50:46.516 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:50:46.516 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:50:46.516 | INFO | High Water Breakdown: | 20260507 01:50:46.516 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:50:46.516 | INFO | ------------------------------------------ | 20260507 01:50:46.516 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:50:46.516 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:50:46.516 | INFO | Total mem usage: 15.7 GiB | 20260507 01:50:46.516 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:50:46.516 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:50:46.516 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:50:46.516 | INFO | High Water Breakdown: | 20260507 01:50:46.516 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:50:46.516 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:50:46.516 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:50:46.516 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:50:46.516 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:50:46.516 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:50:46.516 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:50:46.516 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:50:46.516 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:50:46.516 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:50:46.516 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:50:46.516 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:50:46.516 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:50:46.516 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:50:46.516 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:50:46.516 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:50:46.516 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:50:46.516 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:50:46.517 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:50:46.517 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:50:46.517 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:50:46.517 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:50:46.517 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:50:46.517 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:50:46.517 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:50:46.517 | INFO | ------------------------------------------ | 20260507 01:50:46.517 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:50:46.517 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:50:46.517 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:50:46.517 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:50:46.517 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.53e+01, 1.35e+01 / 2.19e+00 | 20260507 01:50:46.517 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.00e-01, 9.77e-03 / 5.28e-03 | 20260507 01:50:46.517 | INFO | PerfReport: 11.64%, Baseline -- 3.59e+00 / 4.39e+00, 5.14e-01 / 3.19e-01 | 20260507 01:50:46.517 | INFO | PerfReport: 3.23%, Histogram -- 9.96e-01 / 1.49e+00, 1.71e-01 / 1.03e-01 | 20260507 01:50:46.517 | INFO | PerfReport: 16.07%, DME -- 3.97e+01 / 4.40e+01, 3.86e+00 / 2.57e+00 | 20260507 01:50:46.517 | INFO | PerfReport: 57.62%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.38e+00 / 1.82e+00 | 20260507 01:50:46.517 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.16e+00 / 2.70e+00, 3.78e-01 / 2.20e-01 | 20260507 01:50:46.517 | INFO | PerfReport: 3.10%, Metrics -- 9.56e-01 / 1.26e+00, 2.57e-01 / 1.57e-01 | 20260507 01:50:46.517 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.25e-01 / 3.75e-01, 5.27e-02 / 2.77e-02 | 20260507 01:50:46.517 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:50:46.517 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:50:46.517 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:50:46.517 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 01:50:46.517 | INFO | PerfReport: 21.76%, Upload -- 2.10e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260507 01:50:46.517 | INFO | PerfReport: 21.34%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 01:50:46.517 | INFO | PerfReport: 31.98%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 01:50:46.517 | INFO | PerfReport: 1.48%, LockUpload -- 1.43e+00 / 0.00e+00, 4.98e+00 / 0.00e+00 | 20260507 01:50:46.517 | INFO | PerfReport: 1.17%, LockDownload -- 1.13e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260507 01:50:46.517 | INFO | PerfReport: 22.25%, LockCompute -- 2.15e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 01:50:46.517 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.61e-02 / 0.00e+00 | 20260507 01:50:46.612 | INFO | Starting a round of Baz IO | 20260507 01:50:49.736 | INFO | Populating RTMetrics report | 20260507 01:50:49.736 | INFO | RTMetrics report populated | 20260507 01:50:49.736 | INFO | Opening tmp RTMetrics json file | 20260507 01:50:49.754 | INFO | Serializing RTMetrics to json | 20260507 01:50:49.755 | INFO | Serialization done, swapping tmp file now... | 20260507 01:50:49.755 | INFO | RTMetrics done | 20260507 01:50:49.831 | INFO | Finished a round of Baz IO: Wrote 1.803 GiB in 3.218 seconds | 20260507 01:50:49.831 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.08% / 0.00% / 66.92% | 20260507 01:50:49.831 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.266159% / 3.97364e-06% / 5.16574e-05% | 20260507 01:50:49.831 | INFO | Baz IO overall: | 20260507 01:50:49.831 | INFO | BytesWritten : 3185206273879 | 20260507 01:50:49.831 | INFO | NumEvents : 1532624194939 | 20260507 01:50:51.668 | INFO | Analyzing chunk frames = [6152192,6152704) | 20260507 01:50:51.668 | INFO | MemInfo: { | 20260507 01:50:51.668 | INFO | "gpu" : " 17 GB", | 20260507 01:50:51.668 | INFO | "huge" : "", | 20260507 01:50:51.668 | INFO | "huge_shared" : " 27 GB", | 20260507 01:50:51.668 | INFO | "malloc" : " 14 GB", | 20260507 01:50:51.668 | INFO | "pinned" : " 29 GB" | 20260507 01:50:51.668 | INFO | } | 20260507 01:50:51.668 | INFO | Perf report for the last 5.15287 seconds: | 20260507 01:50:51.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:50:51.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:51.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:51.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:51.668 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260507 01:50:51.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:51.668 | INFO | RT_METRICS : 42.0%, 1.0, 22.1ms, 0 | 20260507 01:50:51.668 | INFO | PRE_HQ : 59.4%, 1.8, 55.9ms, 0 | 20260507 01:50:51.668 | INFO | BAZWRITER : 61.9%, 3.3, 106.7ms, 0 | 20260507 01:50:51.668 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:50:51.668 | INFO | Uploaded 36GiB to GPU in 2.02007s (17.8212GiB/s) | 20260507 01:50:51.668 | INFO | Downloaded 38GiB to GPU in 2.17665s (17.458GiB/s) | 20260507 01:50:56.804 | INFO | Analyzing chunk frames = [6152704,6153216) | 20260507 01:50:56.804 | INFO | MemInfo: { | 20260507 01:50:56.804 | INFO | "gpu" : " 17 GB", | 20260507 01:50:56.804 | INFO | "huge" : "", | 20260507 01:50:56.804 | INFO | "huge_shared" : " 27 GB", | 20260507 01:50:56.804 | INFO | "malloc" : " 14 GB", | 20260507 01:50:56.804 | INFO | "pinned" : " 29 GB" | 20260507 01:50:56.804 | INFO | } | 20260507 01:50:56.804 | INFO | Perf report for the last 5.1356 seconds: | 20260507 01:50:56.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:50:56.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:56.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:56.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:56.804 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:50:56.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:56.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:56.804 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:50:56.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:56.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:50:56.804 | INFO | Uploaded 36GiB to GPU in 2.04899s (17.5696GiB/s) | 20260507 01:50:56.804 | INFO | Downloaded 34GiB to GPU in 1.94025s (17.5235GiB/s) | 20260507 01:51:01.939 | INFO | Analyzing chunk frames = [6153216,6153728) | 20260507 01:51:01.939 | INFO | MemInfo: { | 20260507 01:51:01.939 | INFO | "gpu" : " 17 GB", | 20260507 01:51:01.939 | INFO | "huge" : "", | 20260507 01:51:01.939 | INFO | "huge_shared" : " 27 GB", | 20260507 01:51:01.939 | INFO | "malloc" : " 14 GB", | 20260507 01:51:01.939 | INFO | "pinned" : " 29 GB" | 20260507 01:51:01.939 | INFO | } | 20260507 01:51:01.939 | INFO | Perf report for the last 5.13496 seconds: | 20260507 01:51:01.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:51:01.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:01.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:01.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:01.939 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 01:51:01.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:01.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:01.939 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:51:01.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:01.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:01.939 | INFO | Uploaded 36GiB to GPU in 2.0143s (17.8722GiB/s) | 20260507 01:51:01.939 | INFO | Downloaded 34GiB to GPU in 1.93694s (17.5534GiB/s) | 20260507 01:51:07.080 | INFO | Analyzing chunk frames = [6153728,6154240) | 20260507 01:51:07.080 | INFO | MemInfo: { | 20260507 01:51:07.080 | INFO | "gpu" : " 17 GB", | 20260507 01:51:07.080 | INFO | "huge" : "", | 20260507 01:51:07.080 | INFO | "huge_shared" : " 27 GB", | 20260507 01:51:07.080 | INFO | "malloc" : " 14 GB", | 20260507 01:51:07.080 | INFO | "pinned" : " 29 GB" | 20260507 01:51:07.080 | INFO | } | 20260507 01:51:07.080 | INFO | Perf report for the last 5.14165 seconds: | 20260507 01:51:07.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:51:07.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:07.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:07.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:07.080 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:51:07.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:07.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:07.080 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:51:07.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:07.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:07.081 | INFO | Uploaded 36GiB to GPU in 1.97911s (18.19GiB/s) | 20260507 01:51:07.081 | INFO | Downloaded 34GiB to GPU in 1.90603s (17.8381GiB/s) | 20260507 01:51:12.218 | INFO | Analyzing chunk frames = [6154240,6154752) | 20260507 01:51:12.218 | INFO | MemInfo: { | 20260507 01:51:12.218 | INFO | "gpu" : " 17 GB", | 20260507 01:51:12.218 | INFO | "huge" : "", | 20260507 01:51:12.218 | INFO | "huge_shared" : " 27 GB", | 20260507 01:51:12.218 | INFO | "malloc" : " 14 GB", | 20260507 01:51:12.218 | INFO | "pinned" : " 29 GB" | 20260507 01:51:12.218 | INFO | } | 20260507 01:51:12.218 | INFO | Perf report for the last 5.13778 seconds: | 20260507 01:51:12.218 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:51:12.218 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:12.218 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:12.218 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:12.218 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 01:51:12.218 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:12.218 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:12.218 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:51:12.218 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:12.218 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:12.218 | INFO | Uploaded 36GiB to GPU in 1.98551s (18.1313GiB/s) | 20260507 01:51:12.218 | INFO | Downloaded 34GiB to GPU in 1.94266s (17.5017GiB/s) | 20260507 01:51:17.353 | INFO | Analyzing chunk frames = [6154752,6155264) | 20260507 01:51:17.353 | INFO | MemInfo: { | 20260507 01:51:17.353 | INFO | "gpu" : " 17 GB", | 20260507 01:51:17.353 | INFO | "huge" : "", | 20260507 01:51:17.353 | INFO | "huge_shared" : " 27 GB", | 20260507 01:51:17.353 | INFO | "malloc" : " 14 GB", | 20260507 01:51:17.353 | INFO | "pinned" : " 29 GB" | 20260507 01:51:17.353 | INFO | } | 20260507 01:51:17.353 | INFO | Perf report for the last 5.13506 seconds: | 20260507 01:51:17.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:51:17.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:17.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:17.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:17.353 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:51:17.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:17.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:17.353 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 01:51:17.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:17.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:17.353 | INFO | Uploaded 36GiB to GPU in 2.05934s (17.4813GiB/s) | 20260507 01:51:17.353 | INFO | Downloaded 34GiB to GPU in 1.96s (17.3469GiB/s) | 20260507 01:51:22.492 | INFO | Analyzing chunk frames = [6155264,6155776) | 20260507 01:51:22.492 | INFO | MemInfo: { | 20260507 01:51:22.492 | INFO | "gpu" : " 17 GB", | 20260507 01:51:22.492 | INFO | "huge" : "", | 20260507 01:51:22.492 | INFO | "huge_shared" : " 27 GB", | 20260507 01:51:22.492 | INFO | "malloc" : " 14 GB", | 20260507 01:51:22.492 | INFO | "pinned" : " 29 GB" | 20260507 01:51:22.492 | INFO | } | 20260507 01:51:22.492 | INFO | Perf report for the last 5.13868 seconds: | 20260507 01:51:22.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:51:22.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:22.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:22.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:22.492 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:51:22.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:22.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:22.492 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 01:51:22.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:22.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:22.492 | INFO | Uploaded 36GiB to GPU in 2.00214s (17.9807GiB/s) | 20260507 01:51:22.492 | INFO | Downloaded 34GiB to GPU in 1.97841s (17.1856GiB/s) | 20260507 01:51:27.633 | INFO | Analyzing chunk frames = [6155776,6156288) | 20260507 01:51:27.633 | INFO | MemInfo: { | 20260507 01:51:27.633 | INFO | "gpu" : " 17 GB", | 20260507 01:51:27.633 | INFO | "huge" : "", | 20260507 01:51:27.633 | INFO | "huge_shared" : " 27 GB", | 20260507 01:51:27.633 | INFO | "malloc" : " 14 GB", | 20260507 01:51:27.633 | INFO | "pinned" : " 29 GB" | 20260507 01:51:27.634 | INFO | } | 20260507 01:51:27.634 | INFO | Perf report for the last 5.14163 seconds: | 20260507 01:51:27.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:51:27.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:27.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:27.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:27.634 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:51:27.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:27.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:27.634 | INFO | PRE_HQ : 29.4%, 1.0, 15.2ms, 0 | 20260507 01:51:27.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:27.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:27.634 | INFO | Uploaded 36GiB to GPU in 1.99513s (18.044GiB/s) | 20260507 01:51:27.634 | INFO | Downloaded 34GiB to GPU in 1.98181s (17.1561GiB/s) | 20260507 01:51:27.730 | INFO | Starting a round of Baz IO | 20260507 01:51:30.850 | INFO | Populating RTMetrics report | 20260507 01:51:30.851 | INFO | RTMetrics report populated | 20260507 01:51:30.851 | INFO | Opening tmp RTMetrics json file | 20260507 01:51:30.851 | INFO | Serializing RTMetrics to json | 20260507 01:51:30.854 | INFO | Serialization done, swapping tmp file now... | 20260507 01:51:30.861 | INFO | RTMetrics done | 20260507 01:51:30.944 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.213 seconds | 20260507 01:51:30.944 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.08% / 0.00% / 66.92% | 20260507 01:51:30.944 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.265388% / 3.97364e-06% / 8.34465e-05% | 20260507 01:51:30.944 | INFO | Baz IO overall: | 20260507 01:51:30.944 | INFO | BytesWritten : 3187140383678 | 20260507 01:51:30.944 | INFO | NumEvents : 1533539681631 | 20260507 01:51:32.774 | INFO | Analyzing chunk frames = [6156288,6156800) | 20260507 01:51:32.774 | INFO | MemInfo: { | 20260507 01:51:32.774 | INFO | "gpu" : " 17 GB", | 20260507 01:51:32.774 | INFO | "huge" : "", | 20260507 01:51:32.774 | INFO | "huge_shared" : " 27 GB", | 20260507 01:51:32.774 | INFO | "malloc" : " 14 GB", | 20260507 01:51:32.774 | INFO | "pinned" : " 29 GB" | 20260507 01:51:32.774 | INFO | } | 20260507 01:51:32.774 | INFO | Perf report for the last 5.14057 seconds: | 20260507 01:51:32.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:51:32.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:32.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:32.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:32.774 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 01:51:32.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:32.774 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260507 01:51:32.774 | INFO | PRE_HQ : 59.9%, 1.9, 59.7ms, 0 | 20260507 01:51:32.774 | INFO | BAZWRITER : 62.1%, 3.4, 109.9ms, 0 | 20260507 01:51:32.774 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:51:32.775 | INFO | Uploaded 36GiB to GPU in 2.0213s (17.8103GiB/s) | 20260507 01:51:32.775 | INFO | Downloaded 38GiB to GPU in 2.17548s (17.4674GiB/s) | 20260507 01:51:37.915 | INFO | Analyzing chunk frames = [6156800,6157312) | 20260507 01:51:37.915 | INFO | MemInfo: { | 20260507 01:51:37.915 | INFO | "gpu" : " 17 GB", | 20260507 01:51:37.916 | INFO | "huge" : "", | 20260507 01:51:37.916 | INFO | "huge_shared" : " 27 GB", | 20260507 01:51:37.916 | INFO | "malloc" : " 14 GB", | 20260507 01:51:37.916 | INFO | "pinned" : " 29 GB" | 20260507 01:51:37.916 | INFO | } | 20260507 01:51:37.916 | INFO | Perf report for the last 5.14158 seconds: | 20260507 01:51:37.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:51:37.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:37.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:37.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:37.916 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:51:37.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:37.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:37.916 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 01:51:37.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:37.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:37.916 | INFO | Uploaded 36GiB to GPU in 2.01282s (17.8853GiB/s) | 20260507 01:51:37.916 | INFO | Downloaded 34GiB to GPU in 1.9371s (17.552GiB/s) | 20260507 01:51:43.056 | INFO | Analyzing chunk frames = [6157312,6157824) | 20260507 01:51:43.056 | INFO | MemInfo: { | 20260507 01:51:43.056 | INFO | "gpu" : " 17 GB", | 20260507 01:51:43.056 | INFO | "huge" : "", | 20260507 01:51:43.056 | INFO | "huge_shared" : " 27 GB", | 20260507 01:51:43.056 | INFO | "malloc" : " 14 GB", | 20260507 01:51:43.056 | INFO | "pinned" : " 29 GB" | 20260507 01:51:43.056 | INFO | } | 20260507 01:51:43.056 | INFO | Perf report for the last 5.14035 seconds: | 20260507 01:51:43.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:51:43.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:43.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:43.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:43.056 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 01:51:43.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:43.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:43.056 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:51:43.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:43.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:43.056 | INFO | Uploaded 36GiB to GPU in 1.99164s (18.0755GiB/s) | 20260507 01:51:43.056 | INFO | Downloaded 34GiB to GPU in 1.93968s (17.5286GiB/s) | 20260507 01:51:48.190 | INFO | Analyzing chunk frames = [6157824,6158336) | 20260507 01:51:48.190 | INFO | MemInfo: { | 20260507 01:51:48.190 | INFO | "gpu" : " 17 GB", | 20260507 01:51:48.190 | INFO | "huge" : "", | 20260507 01:51:48.190 | INFO | "huge_shared" : " 27 GB", | 20260507 01:51:48.190 | INFO | "malloc" : " 14 GB", | 20260507 01:51:48.190 | INFO | "pinned" : " 29 GB" | 20260507 01:51:48.190 | INFO | } | 20260507 01:51:48.190 | INFO | Perf report for the last 5.13384 seconds: | 20260507 01:51:48.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:51:48.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:48.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:48.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:48.190 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 01:51:48.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:48.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:48.190 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:51:48.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:48.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:48.190 | INFO | Uploaded 36GiB to GPU in 1.9534s (18.4294GiB/s) | 20260507 01:51:48.190 | INFO | Downloaded 34GiB to GPU in 1.92153s (17.6942GiB/s) | 20260507 01:51:53.329 | INFO | Analyzing chunk frames = [6158336,6158848) | 20260507 01:51:53.329 | INFO | MemInfo: { | 20260507 01:51:53.329 | INFO | "gpu" : " 17 GB", | 20260507 01:51:53.329 | INFO | "huge" : "", | 20260507 01:51:53.329 | INFO | "huge_shared" : " 27 GB", | 20260507 01:51:53.329 | INFO | "malloc" : " 14 GB", | 20260507 01:51:53.329 | INFO | "pinned" : " 29 GB" | 20260507 01:51:53.329 | INFO | } | 20260507 01:51:53.329 | INFO | Perf report for the last 5.13937 seconds: | 20260507 01:51:53.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:51:53.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:53.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:53.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:53.329 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 01:51:53.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:53.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:53.329 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:51:53.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:53.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:53.330 | INFO | Uploaded 36GiB to GPU in 1.9953s (18.0424GiB/s) | 20260507 01:51:53.330 | INFO | Downloaded 34GiB to GPU in 1.95309s (17.4083GiB/s) | 20260507 01:51:58.469 | INFO | Analyzing chunk frames = [6158848,6159360) | 20260507 01:51:58.469 | INFO | MemInfo: { | 20260507 01:51:58.469 | INFO | "gpu" : " 17 GB", | 20260507 01:51:58.469 | INFO | "huge" : "", | 20260507 01:51:58.469 | INFO | "huge_shared" : " 27 GB", | 20260507 01:51:58.469 | INFO | "malloc" : " 14 GB", | 20260507 01:51:58.469 | INFO | "pinned" : " 29 GB" | 20260507 01:51:58.469 | INFO | } | 20260507 01:51:58.469 | INFO | Perf report for the last 5.13968 seconds: | 20260507 01:51:58.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:51:58.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:58.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:58.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:58.469 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 01:51:58.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:58.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:58.469 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:51:58.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:58.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:51:58.469 | INFO | Uploaded 36GiB to GPU in 2.01608s (17.8564GiB/s) | 20260507 01:51:58.469 | INFO | Downloaded 34GiB to GPU in 1.94088s (17.5178GiB/s) | 20260507 01:52:03.608 | INFO | Analyzing chunk frames = [6159360,6159872) | 20260507 01:52:03.608 | INFO | MemInfo: { | 20260507 01:52:03.608 | INFO | "gpu" : " 17 GB", | 20260507 01:52:03.608 | INFO | "huge" : "", | 20260507 01:52:03.608 | INFO | "huge_shared" : " 27 GB", | 20260507 01:52:03.608 | INFO | "malloc" : " 14 GB", | 20260507 01:52:03.608 | INFO | "pinned" : " 29 GB" | 20260507 01:52:03.608 | INFO | } | 20260507 01:52:03.608 | INFO | Perf report for the last 5.1392 seconds: | 20260507 01:52:03.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:52:03.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:03.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:03.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:03.608 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:52:03.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:03.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:03.608 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:52:03.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:03.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:03.608 | INFO | Uploaded 36GiB to GPU in 1.96648s (18.3069GiB/s) | 20260507 01:52:03.608 | INFO | Downloaded 34GiB to GPU in 2.00193s (16.9836GiB/s) | 20260507 01:52:08.753 | INFO | Analyzing chunk frames = [6159872,6160384) | 20260507 01:52:08.753 | INFO | MemInfo: { | 20260507 01:52:08.753 | INFO | "gpu" : " 17 GB", | 20260507 01:52:08.753 | INFO | "huge" : "", | 20260507 01:52:08.753 | INFO | "huge_shared" : " 27 GB", | 20260507 01:52:08.753 | INFO | "malloc" : " 14 GB", | 20260507 01:52:08.753 | INFO | "pinned" : " 29 GB" | 20260507 01:52:08.753 | INFO | } | 20260507 01:52:08.753 | INFO | Perf report for the last 5.1447 seconds: | 20260507 01:52:08.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:52:08.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:08.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:08.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:08.753 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:52:08.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:08.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:08.753 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:52:08.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:08.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:08.753 | INFO | Uploaded 36GiB to GPU in 1.99193s (18.0729GiB/s) | 20260507 01:52:08.753 | INFO | Downloaded 34GiB to GPU in 1.97013s (17.2578GiB/s) | 20260507 01:52:08.753 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:52:08.753 | INFO | Total mem usage: 18 GiB | 20260507 01:52:08.753 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:52:08.753 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.9%) | 20260507 01:52:08.753 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:52:08.753 | INFO | High Water Breakdown: | 20260507 01:52:08.753 | INFO | 60.2% : 8.62 GiB ( 75.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:52:08.754 | INFO | 39.6% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:52:08.754 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:52:08.754 | INFO | ------------------------------------------ | 20260507 01:52:08.754 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:52:08.754 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:52:08.754 | INFO | Total mem usage: 26.5 GiB | 20260507 01:52:08.754 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:52:08.754 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:52:08.754 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:52:08.754 | INFO | High Water Breakdown: | 20260507 01:52:08.754 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:52:08.754 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:52:08.754 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:52:08.754 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:52:08.754 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:52:08.754 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:52:08.754 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:52:08.754 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:52:08.754 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:52:08.754 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:52:08.754 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:52:08.754 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:52:08.754 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:52:08.754 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:52:08.754 | INFO | ------------------------------------------ | 20260507 01:52:08.754 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:52:08.754 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:52:08.754 | INFO | Total mem usage: 25.6 GiB | 20260507 01:52:08.754 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:52:08.754 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:52:08.754 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:52:08.754 | INFO | High Water Breakdown: | 20260507 01:52:08.754 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:52:08.754 | INFO | ------------------------------------------ | 20260507 01:52:08.754 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:52:08.754 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:52:08.754 | INFO | Total mem usage: 15.7 GiB | 20260507 01:52:08.754 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:52:08.754 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:52:08.754 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:52:08.754 | INFO | High Water Breakdown: | 20260507 01:52:08.754 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:52:08.754 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:52:08.754 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:52:08.754 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:52:08.754 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:52:08.754 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:52:08.754 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:52:08.754 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:52:08.754 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:52:08.754 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:52:08.754 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:52:08.754 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:52:08.754 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:52:08.754 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:52:08.754 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:52:08.754 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:52:08.754 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:52:08.754 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:52:08.754 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:52:08.754 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:52:08.754 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:52:08.754 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:52:08.754 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:52:08.754 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:52:08.754 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:52:08.754 | INFO | ------------------------------------------ | 20260507 01:52:08.754 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:52:08.754 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:52:08.754 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:52:08.754 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:52:08.754 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.55e+01, 1.33e+01 / 1.98e+00 | 20260507 01:52:08.754 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 1.02e-01, 1.06e-02 / 4.63e-03 | 20260507 01:52:08.754 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.49e+00, 5.23e-01 / 2.43e-01 | 20260507 01:52:08.754 | INFO | PerfReport: 3.24%, Histogram -- 9.99e-01 / 1.49e+00, 1.75e-01 / 1.05e-01 | 20260507 01:52:08.754 | INFO | PerfReport: 16.07%, DME -- 3.96e+01 / 4.42e+01, 3.90e+00 / 2.24e+00 | 20260507 01:52:08.754 | INFO | PerfReport: 57.56%, FrameLabeling -- 1.77e+01 / 2.09e+01, 1.33e+00 / 1.69e+00 | 20260507 01:52:08.754 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.16e+00 / 2.72e+00, 3.82e-01 / 2.41e-01 | 20260507 01:52:08.754 | INFO | PerfReport: 3.10%, Metrics -- 9.55e-01 / 1.29e+00, 2.53e-01 / 1.67e-01 | 20260507 01:52:08.754 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.25e-01 / 3.82e-01, 5.19e-02 / 2.85e-02 | 20260507 01:52:08.754 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:52:08.754 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:52:08.754 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:52:08.754 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 01:52:08.754 | INFO | PerfReport: 21.64%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260507 01:52:08.755 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 01:52:08.755 | INFO | PerfReport: 31.98%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 01:52:08.755 | INFO | PerfReport: 1.38%, LockUpload -- 1.33e+00 / 0.00e+00, 4.94e+00 / 0.00e+00 | 20260507 01:52:08.755 | INFO | PerfReport: 1.33%, LockDownload -- 1.28e+00 / 0.00e+00, 3.15e+00 / 0.00e+00 | 20260507 01:52:08.755 | INFO | PerfReport: 22.29%, LockCompute -- 2.15e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 01:52:08.755 | INFO | PerfReport: 0.02%, Misc -- 1.45e-02 / 0.00e+00, 2.15e-02 / 0.00e+00 | 20260507 01:52:08.850 | INFO | Starting a round of Baz IO | 20260507 01:52:11.959 | INFO | Populating RTMetrics report | 20260507 01:52:11.959 | INFO | RTMetrics report populated | 20260507 01:52:11.959 | INFO | Opening tmp RTMetrics json file | 20260507 01:52:11.966 | INFO | Serializing RTMetrics to json | 20260507 01:52:11.980 | INFO | Serialization done, swapping tmp file now... | 20260507 01:52:11.980 | INFO | RTMetrics done | 20260507 01:52:12.054 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.204 seconds | 20260507 01:52:12.054 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.08% / 0.00% / 66.92% | 20260507 01:52:12.054 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.265372% / 0% / 8.34465e-05% | 20260507 01:52:12.054 | INFO | Baz IO overall: | 20260507 01:52:12.054 | INFO | BytesWritten : 3189074988796 | 20260507 01:52:12.054 | INFO | NumEvents : 1534455602345 | 20260507 01:52:13.894 | INFO | Analyzing chunk frames = [6160384,6160896) | 20260507 01:52:13.894 | INFO | MemInfo: { | 20260507 01:52:13.894 | INFO | "gpu" : " 17 GB", | 20260507 01:52:13.894 | INFO | "huge" : "", | 20260507 01:52:13.894 | INFO | "huge_shared" : " 27 GB", | 20260507 01:52:13.894 | INFO | "malloc" : " 14 GB", | 20260507 01:52:13.894 | INFO | "pinned" : " 29 GB" | 20260507 01:52:13.894 | INFO | } | 20260507 01:52:13.894 | INFO | Perf report for the last 5.14076 seconds: | 20260507 01:52:13.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:52:13.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:13.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:13.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:13.894 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 01:52:13.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:13.894 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:52:13.894 | INFO | PRE_HQ : 59.3%, 1.8, 55.8ms, 0 | 20260507 01:52:13.894 | INFO | BAZWRITER : 61.9%, 3.4, 106.7ms, 0 | 20260507 01:52:13.894 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 01:52:13.894 | INFO | Uploaded 36GiB to GPU in 2.0229s (17.7963GiB/s) | 20260507 01:52:13.894 | INFO | Downloaded 38GiB to GPU in 2.18807s (17.3669GiB/s) | 20260507 01:52:19.033 | INFO | Analyzing chunk frames = [6160896,6161408) | 20260507 01:52:19.033 | INFO | MemInfo: { | 20260507 01:52:19.033 | INFO | "gpu" : " 17 GB", | 20260507 01:52:19.033 | INFO | "huge" : "", | 20260507 01:52:19.033 | INFO | "huge_shared" : " 27 GB", | 20260507 01:52:19.033 | INFO | "malloc" : " 14 GB", | 20260507 01:52:19.033 | INFO | "pinned" : " 29 GB" | 20260507 01:52:19.033 | INFO | } | 20260507 01:52:19.033 | INFO | Perf report for the last 5.13934 seconds: | 20260507 01:52:19.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:52:19.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:19.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:19.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:19.033 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 01:52:19.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:19.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:19.033 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:52:19.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:19.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:19.033 | INFO | Uploaded 36GiB to GPU in 2.00842s (17.9245GiB/s) | 20260507 01:52:19.033 | INFO | Downloaded 34GiB to GPU in 1.931s (17.6075GiB/s) | 20260507 01:52:24.169 | INFO | Analyzing chunk frames = [6161408,6161920) | 20260507 01:52:24.169 | INFO | MemInfo: { | 20260507 01:52:24.169 | INFO | "gpu" : " 17 GB", | 20260507 01:52:24.169 | INFO | "huge" : "", | 20260507 01:52:24.169 | INFO | "huge_shared" : " 27 GB", | 20260507 01:52:24.169 | INFO | "malloc" : " 14 GB", | 20260507 01:52:24.169 | INFO | "pinned" : " 29 GB" | 20260507 01:52:24.169 | INFO | } | 20260507 01:52:24.169 | INFO | Perf report for the last 5.13587 seconds: | 20260507 01:52:24.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:52:24.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:24.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:24.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:24.169 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 01:52:24.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:24.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:24.169 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:52:24.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:24.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:24.169 | INFO | Uploaded 36GiB to GPU in 1.99094s (18.0819GiB/s) | 20260507 01:52:24.169 | INFO | Downloaded 34GiB to GPU in 1.92246s (17.6857GiB/s) | 20260507 01:52:29.302 | INFO | Analyzing chunk frames = [6161920,6162432) | 20260507 01:52:29.302 | INFO | MemInfo: { | 20260507 01:52:29.302 | INFO | "gpu" : " 17 GB", | 20260507 01:52:29.302 | INFO | "huge" : "", | 20260507 01:52:29.302 | INFO | "huge_shared" : " 27 GB", | 20260507 01:52:29.302 | INFO | "malloc" : " 14 GB", | 20260507 01:52:29.302 | INFO | "pinned" : " 29 GB" | 20260507 01:52:29.302 | INFO | } | 20260507 01:52:29.302 | INFO | Perf report for the last 5.13286 seconds: | 20260507 01:52:29.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:52:29.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:29.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:29.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:29.302 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 01:52:29.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:29.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:29.302 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:52:29.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:29.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:29.302 | INFO | Uploaded 36GiB to GPU in 1.97412s (18.236GiB/s) | 20260507 01:52:29.302 | INFO | Downloaded 34GiB to GPU in 1.91691s (17.7369GiB/s) | 20260507 01:52:34.447 | INFO | Analyzing chunk frames = [6162432,6162944) | 20260507 01:52:34.447 | INFO | MemInfo: { | 20260507 01:52:34.447 | INFO | "gpu" : " 17 GB", | 20260507 01:52:34.447 | INFO | "huge" : "", | 20260507 01:52:34.447 | INFO | "huge_shared" : " 27 GB", | 20260507 01:52:34.447 | INFO | "malloc" : " 14 GB", | 20260507 01:52:34.447 | INFO | "pinned" : " 29 GB" | 20260507 01:52:34.447 | INFO | } | 20260507 01:52:34.447 | INFO | Perf report for the last 5.14557 seconds: | 20260507 01:52:34.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:52:34.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:34.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:34.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:34.447 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 01:52:34.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:34.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:34.447 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:52:34.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:34.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:34.448 | INFO | Uploaded 36GiB to GPU in 1.99574s (18.0385GiB/s) | 20260507 01:52:34.448 | INFO | Downloaded 34GiB to GPU in 1.94688s (17.4638GiB/s) | 20260507 01:52:39.580 | INFO | Analyzing chunk frames = [6162944,6163456) | 20260507 01:52:39.580 | INFO | MemInfo: { | 20260507 01:52:39.580 | INFO | "gpu" : " 17 GB", | 20260507 01:52:39.580 | INFO | "huge" : "", | 20260507 01:52:39.580 | INFO | "huge_shared" : " 27 GB", | 20260507 01:52:39.580 | INFO | "malloc" : " 14 GB", | 20260507 01:52:39.580 | INFO | "pinned" : " 29 GB" | 20260507 01:52:39.580 | INFO | } | 20260507 01:52:39.581 | INFO | Perf report for the last 5.13313 seconds: | 20260507 01:52:39.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:52:39.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:39.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:39.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:39.581 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 01:52:39.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:39.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:39.581 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 01:52:39.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:39.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:39.581 | INFO | Uploaded 36GiB to GPU in 2.0499s (17.5619GiB/s) | 20260507 01:52:39.581 | INFO | Downloaded 34GiB to GPU in 1.96857s (17.2714GiB/s) | 20260507 01:52:44.722 | INFO | Analyzing chunk frames = [6163456,6163968) | 20260507 01:52:44.722 | INFO | MemInfo: { | 20260507 01:52:44.722 | INFO | "gpu" : " 17 GB", | 20260507 01:52:44.722 | INFO | "huge" : "", | 20260507 01:52:44.722 | INFO | "huge_shared" : " 27 GB", | 20260507 01:52:44.722 | INFO | "malloc" : " 14 GB", | 20260507 01:52:44.722 | INFO | "pinned" : " 29 GB" | 20260507 01:52:44.722 | INFO | } | 20260507 01:52:44.722 | INFO | Perf report for the last 5.1417 seconds: | 20260507 01:52:44.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:52:44.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:44.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:44.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:44.722 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:52:44.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:44.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:44.722 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:52:44.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:44.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:44.722 | INFO | Uploaded 36GiB to GPU in 1.95954s (18.3716GiB/s) | 20260507 01:52:44.723 | INFO | Downloaded 34GiB to GPU in 1.99461s (17.0459GiB/s) | 20260507 01:52:49.867 | INFO | Analyzing chunk frames = [6163968,6164480) | 20260507 01:52:49.867 | INFO | MemInfo: { | 20260507 01:52:49.867 | INFO | "gpu" : " 17 GB", | 20260507 01:52:49.867 | INFO | "huge" : "", | 20260507 01:52:49.867 | INFO | "huge_shared" : " 27 GB", | 20260507 01:52:49.867 | INFO | "malloc" : " 14 GB", | 20260507 01:52:49.867 | INFO | "pinned" : " 29 GB" | 20260507 01:52:49.867 | INFO | } | 20260507 01:52:49.868 | INFO | Perf report for the last 5.14528 seconds: | 20260507 01:52:49.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:52:49.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:49.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:49.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:49.868 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:52:49.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:49.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:49.868 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:52:49.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:49.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:49.868 | INFO | Uploaded 36GiB to GPU in 1.98305s (18.1539GiB/s) | 20260507 01:52:49.868 | INFO | Downloaded 34GiB to GPU in 1.97496s (17.2156GiB/s) | 20260507 01:52:49.964 | INFO | Starting a round of Baz IO | 20260507 01:52:53.079 | INFO | Populating RTMetrics report | 20260507 01:52:53.079 | INFO | RTMetrics report populated | 20260507 01:52:53.079 | INFO | Opening tmp RTMetrics json file | 20260507 01:52:53.081 | INFO | Serializing RTMetrics to json | 20260507 01:52:53.099 | INFO | Serialization done, swapping tmp file now... | 20260507 01:52:53.099 | INFO | RTMetrics done | 20260507 01:52:53.176 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.211 seconds | 20260507 01:52:53.176 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.08% / 0.00% / 66.92% | 20260507 01:52:53.176 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.264156% / 3.97364e-06% / 7.94729e-05% | 20260507 01:52:53.176 | INFO | Baz IO overall: | 20260507 01:52:53.176 | INFO | BytesWritten : 3191009132921 | 20260507 01:52:53.176 | INFO | NumEvents : 1535371305412 | 20260507 01:52:55.002 | INFO | Analyzing chunk frames = [6164480,6164992) | 20260507 01:52:55.002 | INFO | MemInfo: { | 20260507 01:52:55.002 | INFO | "gpu" : " 17 GB", | 20260507 01:52:55.002 | INFO | "huge" : "", | 20260507 01:52:55.002 | INFO | "huge_shared" : " 27 GB", | 20260507 01:52:55.002 | INFO | "malloc" : " 14 GB", | 20260507 01:52:55.002 | INFO | "pinned" : " 29 GB" | 20260507 01:52:55.002 | INFO | } | 20260507 01:52:55.003 | INFO | Perf report for the last 5.13499 seconds: | 20260507 01:52:55.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:52:55.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:55.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:55.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:55.003 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 01:52:55.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:52:55.003 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:52:55.003 | INFO | PRE_HQ : 59.5%, 1.8, 55.7ms, 0 | 20260507 01:52:55.003 | INFO | BAZWRITER : 62.1%, 3.3, 106.0ms, 0 | 20260507 01:52:55.003 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:52:55.003 | INFO | Uploaded 36GiB to GPU in 2.03784s (17.6657GiB/s) | 20260507 01:52:55.003 | INFO | Downloaded 38GiB to GPU in 2.18196s (17.4155GiB/s) | 20260507 01:53:00.140 | INFO | Analyzing chunk frames = [6164992,6165504) | 20260507 01:53:00.140 | INFO | MemInfo: { | 20260507 01:53:00.140 | INFO | "gpu" : " 17 GB", | 20260507 01:53:00.140 | INFO | "huge" : "", | 20260507 01:53:00.140 | INFO | "huge_shared" : " 27 GB", | 20260507 01:53:00.140 | INFO | "malloc" : " 14 GB", | 20260507 01:53:00.140 | INFO | "pinned" : " 29 GB" | 20260507 01:53:00.140 | INFO | } | 20260507 01:53:00.140 | INFO | Perf report for the last 5.1375 seconds: | 20260507 01:53:00.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:53:00.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:00.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:00.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:00.140 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 01:53:00.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:00.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:00.140 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 01:53:00.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:00.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:00.140 | INFO | Uploaded 36GiB to GPU in 1.98423s (18.1431GiB/s) | 20260507 01:53:00.140 | INFO | Downloaded 34GiB to GPU in 1.93005s (17.6162GiB/s) | 20260507 01:53:05.285 | INFO | Analyzing chunk frames = [6165504,6166016) | 20260507 01:53:05.285 | INFO | MemInfo: { | 20260507 01:53:05.285 | INFO | "gpu" : " 17 GB", | 20260507 01:53:05.285 | INFO | "huge" : "", | 20260507 01:53:05.285 | INFO | "huge_shared" : " 27 GB", | 20260507 01:53:05.285 | INFO | "malloc" : " 14 GB", | 20260507 01:53:05.285 | INFO | "pinned" : " 29 GB" | 20260507 01:53:05.285 | INFO | } | 20260507 01:53:05.285 | INFO | Perf report for the last 5.14532 seconds: | 20260507 01:53:05.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:53:05.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:05.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:05.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:05.285 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 01:53:05.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:05.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:05.285 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:53:05.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:05.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:05.286 | INFO | Uploaded 36GiB to GPU in 2.00588s (17.9472GiB/s) | 20260507 01:53:05.286 | INFO | Downloaded 34GiB to GPU in 1.93351s (17.5846GiB/s) | 20260507 01:53:10.420 | INFO | Analyzing chunk frames = [6166016,6166528) | 20260507 01:53:10.420 | INFO | MemInfo: { | 20260507 01:53:10.420 | INFO | "gpu" : " 17 GB", | 20260507 01:53:10.420 | INFO | "huge" : "", | 20260507 01:53:10.420 | INFO | "huge_shared" : " 27 GB", | 20260507 01:53:10.420 | INFO | "malloc" : " 14 GB", | 20260507 01:53:10.420 | INFO | "pinned" : " 29 GB" | 20260507 01:53:10.420 | INFO | } | 20260507 01:53:10.420 | INFO | Perf report for the last 5.13487 seconds: | 20260507 01:53:10.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:53:10.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:10.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:10.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:10.420 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 01:53:10.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:10.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:10.420 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:53:10.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:10.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:10.420 | INFO | Uploaded 36GiB to GPU in 1.97373s (18.2396GiB/s) | 20260507 01:53:10.421 | INFO | Downloaded 34GiB to GPU in 1.89938s (17.9006GiB/s) | 20260507 01:53:15.557 | INFO | Analyzing chunk frames = [6166528,6167040) | 20260507 01:53:15.557 | INFO | MemInfo: { | 20260507 01:53:15.557 | INFO | "gpu" : " 17 GB", | 20260507 01:53:15.557 | INFO | "huge" : "", | 20260507 01:53:15.557 | INFO | "huge_shared" : " 27 GB", | 20260507 01:53:15.557 | INFO | "malloc" : " 14 GB", | 20260507 01:53:15.557 | INFO | "pinned" : " 29 GB" | 20260507 01:53:15.557 | INFO | } | 20260507 01:53:15.557 | INFO | Perf report for the last 5.13708 seconds: | 20260507 01:53:15.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:53:15.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:15.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:15.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:15.557 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 01:53:15.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:15.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:15.557 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:53:15.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:15.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:15.558 | INFO | Uploaded 36GiB to GPU in 2.00656s (17.9411GiB/s) | 20260507 01:53:15.558 | INFO | Downloaded 34GiB to GPU in 1.94929s (17.4422GiB/s) | 20260507 01:53:20.706 | INFO | Analyzing chunk frames = [6167040,6167552) | 20260507 01:53:20.706 | INFO | MemInfo: { | 20260507 01:53:20.706 | INFO | "gpu" : " 17 GB", | 20260507 01:53:20.706 | INFO | "huge" : "", | 20260507 01:53:20.706 | INFO | "huge_shared" : " 27 GB", | 20260507 01:53:20.706 | INFO | "malloc" : " 14 GB", | 20260507 01:53:20.706 | INFO | "pinned" : " 29 GB" | 20260507 01:53:20.706 | INFO | } | 20260507 01:53:20.706 | INFO | Perf report for the last 5.14888 seconds: | 20260507 01:53:20.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:53:20.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:20.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:20.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:20.706 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 01:53:20.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:20.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:20.706 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 01:53:20.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:20.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:20.706 | INFO | Uploaded 36GiB to GPU in 2.06057s (17.4709GiB/s) | 20260507 01:53:20.706 | INFO | Downloaded 34GiB to GPU in 1.95805s (17.3642GiB/s) | 20260507 01:53:25.838 | INFO | Analyzing chunk frames = [6167552,6168064) | 20260507 01:53:25.838 | INFO | MemInfo: { | 20260507 01:53:25.838 | INFO | "gpu" : " 17 GB", | 20260507 01:53:25.838 | INFO | "huge" : "", | 20260507 01:53:25.838 | INFO | "huge_shared" : " 27 GB", | 20260507 01:53:25.838 | INFO | "malloc" : " 14 GB", | 20260507 01:53:25.838 | INFO | "pinned" : " 29 GB" | 20260507 01:53:25.838 | INFO | } | 20260507 01:53:25.838 | INFO | Perf report for the last 5.13206 seconds: | 20260507 01:53:25.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:53:25.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:25.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:25.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:25.838 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 01:53:25.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:25.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:25.838 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 01:53:25.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:25.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:25.839 | INFO | Uploaded 36GiB to GPU in 1.97229s (18.2529GiB/s) | 20260507 01:53:25.839 | INFO | Downloaded 34GiB to GPU in 1.95678s (17.3755GiB/s) | 20260507 01:53:30.980 | INFO | Analyzing chunk frames = [6168064,6168576) | 20260507 01:53:30.980 | INFO | MemInfo: { | 20260507 01:53:30.980 | INFO | "gpu" : " 17 GB", | 20260507 01:53:30.980 | INFO | "huge" : "", | 20260507 01:53:30.980 | INFO | "huge_shared" : " 27 GB", | 20260507 01:53:30.981 | INFO | "malloc" : " 14 GB", | 20260507 01:53:30.981 | INFO | "pinned" : " 29 GB" | 20260507 01:53:30.981 | INFO | } | 20260507 01:53:30.981 | INFO | Perf report for the last 5.14227 seconds: | 20260507 01:53:30.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:53:30.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:30.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:30.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:30.981 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:53:30.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:30.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:30.981 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 01:53:30.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:30.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:30.981 | INFO | Uploaded 36GiB to GPU in 1.9814s (18.169GiB/s) | 20260507 01:53:30.981 | INFO | Downloaded 34GiB to GPU in 1.9723s (17.2387GiB/s) | 20260507 01:53:30.981 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:53:30.981 | INFO | Total mem usage: 18 GiB | 20260507 01:53:30.981 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:53:30.981 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 01:53:30.981 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:53:30.981 | INFO | High Water Breakdown: | 20260507 01:53:30.981 | INFO | 60.1% : 8.58 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:53:30.981 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:53:30.981 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:53:30.981 | INFO | ------------------------------------------ | 20260507 01:53:30.981 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:53:30.981 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:53:30.981 | INFO | Total mem usage: 26.5 GiB | 20260507 01:53:30.981 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:53:30.981 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:53:30.981 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:53:30.981 | INFO | High Water Breakdown: | 20260507 01:53:30.981 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:53:30.981 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:53:30.981 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:53:30.981 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:53:30.981 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:53:30.981 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:53:30.981 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:53:30.981 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:53:30.981 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:53:30.981 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:53:30.981 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:53:30.981 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:53:30.981 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:53:30.981 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:53:30.981 | INFO | ------------------------------------------ | 20260507 01:53:30.981 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:53:30.981 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:53:30.981 | INFO | Total mem usage: 25.6 GiB | 20260507 01:53:30.981 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:53:30.981 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:53:30.981 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:53:30.981 | INFO | High Water Breakdown: | 20260507 01:53:30.981 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:53:30.981 | INFO | ------------------------------------------ | 20260507 01:53:30.981 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:53:30.981 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:53:30.981 | INFO | Total mem usage: 15.7 GiB | 20260507 01:53:30.981 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:53:30.981 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:53:30.981 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:53:30.981 | INFO | High Water Breakdown: | 20260507 01:53:30.981 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:53:30.981 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:53:30.982 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:53:30.982 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:53:30.982 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:53:30.982 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:53:30.982 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:53:30.982 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:53:30.982 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:53:30.982 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:53:30.982 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:53:30.982 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:53:30.982 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:53:30.982 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:53:30.982 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:53:30.982 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:53:30.982 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:53:30.982 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:53:30.982 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:53:30.982 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:53:30.982 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:53:30.982 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:53:30.982 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:53:30.982 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:53:30.982 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:53:30.982 | INFO | ------------------------------------------ | 20260507 01:53:30.982 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:53:30.982 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:53:30.982 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:53:30.982 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:53:30.982 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.51e+01, 1.34e+01 / 1.81e+00 | 20260507 01:53:30.982 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 1.01e-01, 9.43e-03 / 5.03e-03 | 20260507 01:53:30.982 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.42e+00, 5.30e-01 / 3.23e-01 | 20260507 01:53:30.982 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.46e+00, 1.74e-01 / 1.22e-01 | 20260507 01:53:30.982 | INFO | PerfReport: 16.07%, DME -- 3.96e+01 / 4.38e+01, 3.88e+00 / 2.06e+00 | 20260507 01:53:30.982 | INFO | PerfReport: 57.55%, FrameLabeling -- 1.77e+01 / 2.10e+01, 1.29e+00 / 1.67e+00 | 20260507 01:53:30.982 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.16e+00 / 2.70e+00, 3.83e-01 / 2.55e-01 | 20260507 01:53:30.982 | INFO | PerfReport: 3.11%, Metrics -- 9.58e-01 / 1.25e+00, 2.57e-01 / 1.50e-01 | 20260507 01:53:30.982 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.25e-01 / 3.77e-01, 5.24e-02 / 2.59e-02 | 20260507 01:53:30.982 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:53:30.982 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:53:30.982 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:53:30.982 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 01:53:30.982 | INFO | PerfReport: 21.62%, Upload -- 2.08e+01 / 0.00e+00, 7.04e+00 / 0.00e+00 | 20260507 01:53:30.982 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260507 01:53:30.982 | INFO | PerfReport: 31.99%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 01:53:30.982 | INFO | PerfReport: 1.37%, LockUpload -- 1.32e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260507 01:53:30.982 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.16e+00 / 0.00e+00 | 20260507 01:53:30.982 | INFO | PerfReport: 22.34%, LockCompute -- 2.15e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 01:53:30.982 | INFO | PerfReport: 0.02%, Misc -- 1.53e-02 / 0.00e+00, 2.93e-02 / 0.00e+00 | 20260507 01:53:31.090 | INFO | Starting a round of Baz IO | 20260507 01:53:34.189 | INFO | Populating RTMetrics report | 20260507 01:53:34.190 | INFO | RTMetrics report populated | 20260507 01:53:34.190 | INFO | Opening tmp RTMetrics json file | 20260507 01:53:34.209 | INFO | Serializing RTMetrics to json | 20260507 01:53:34.210 | INFO | Serialization done, swapping tmp file now... | 20260507 01:53:34.211 | INFO | RTMetrics done | 20260507 01:53:34.280 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.19 seconds | 20260507 01:53:34.280 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.07% / 0.00% / 66.92% | 20260507 01:53:34.280 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.263023% / 7.94729e-06% / 8.74201e-05% | 20260507 01:53:34.280 | INFO | Baz IO overall: | 20260507 01:53:34.280 | INFO | BytesWritten : 3192940678949 | 20260507 01:53:34.280 | INFO | NumEvents : 1536285264328 | 20260507 01:53:36.121 | INFO | Analyzing chunk frames = [6168576,6169088) | 20260507 01:53:36.121 | INFO | MemInfo: { | 20260507 01:53:36.121 | INFO | "gpu" : " 17 GB", | 20260507 01:53:36.121 | INFO | "huge" : "", | 20260507 01:53:36.121 | INFO | "huge_shared" : " 27 GB", | 20260507 01:53:36.121 | INFO | "malloc" : " 14 GB", | 20260507 01:53:36.121 | INFO | "pinned" : " 29 GB" | 20260507 01:53:36.121 | INFO | } | 20260507 01:53:36.121 | INFO | Perf report for the last 5.14005 seconds: | 20260507 01:53:36.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:53:36.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:36.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:36.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:36.121 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260507 01:53:36.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:36.121 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:53:36.121 | INFO | PRE_HQ : 59.3%, 1.8, 55.9ms, 0 | 20260507 01:53:36.121 | INFO | BAZWRITER : 61.9%, 3.3, 106.1ms, 0 | 20260507 01:53:36.121 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:53:36.121 | INFO | Uploaded 36GiB to GPU in 2.01877s (17.8326GiB/s) | 20260507 01:53:36.121 | INFO | Downloaded 38GiB to GPU in 2.17865s (17.442GiB/s) | 20260507 01:53:41.253 | INFO | Analyzing chunk frames = [6169088,6169600) | 20260507 01:53:41.253 | INFO | MemInfo: { | 20260507 01:53:41.253 | INFO | "gpu" : " 17 GB", | 20260507 01:53:41.253 | INFO | "huge" : "", | 20260507 01:53:41.253 | INFO | "huge_shared" : " 27 GB", | 20260507 01:53:41.253 | INFO | "malloc" : " 14 GB", | 20260507 01:53:41.253 | INFO | "pinned" : " 29 GB" | 20260507 01:53:41.253 | INFO | } | 20260507 01:53:41.253 | INFO | Perf report for the last 5.13248 seconds: | 20260507 01:53:41.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:53:41.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:41.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:41.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:41.253 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 01:53:41.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:41.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:41.253 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:53:41.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:41.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:41.253 | INFO | Uploaded 36GiB to GPU in 2.03278s (17.7097GiB/s) | 20260507 01:53:41.254 | INFO | Downloaded 34GiB to GPU in 1.94438s (17.4863GiB/s) | 20260507 01:53:46.397 | INFO | Analyzing chunk frames = [6169600,6170112) | 20260507 01:53:46.397 | INFO | MemInfo: { | 20260507 01:53:46.397 | INFO | "gpu" : " 17 GB", | 20260507 01:53:46.397 | INFO | "huge" : "", | 20260507 01:53:46.397 | INFO | "huge_shared" : " 27 GB", | 20260507 01:53:46.397 | INFO | "malloc" : " 14 GB", | 20260507 01:53:46.397 | INFO | "pinned" : " 29 GB" | 20260507 01:53:46.397 | INFO | } | 20260507 01:53:46.397 | INFO | Perf report for the last 5.14388 seconds: | 20260507 01:53:46.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:53:46.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:46.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:46.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:46.397 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:53:46.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:46.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:46.397 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:53:46.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:46.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:46.397 | INFO | Uploaded 36GiB to GPU in 1.99537s (18.0418GiB/s) | 20260507 01:53:46.397 | INFO | Downloaded 34GiB to GPU in 1.9395s (17.5303GiB/s) | 20260507 01:53:51.540 | INFO | Analyzing chunk frames = [6170112,6170624) | 20260507 01:53:51.540 | INFO | MemInfo: { | 20260507 01:53:51.540 | INFO | "gpu" : " 17 GB", | 20260507 01:53:51.540 | INFO | "huge" : "", | 20260507 01:53:51.540 | INFO | "huge_shared" : " 27 GB", | 20260507 01:53:51.540 | INFO | "malloc" : " 14 GB", | 20260507 01:53:51.540 | INFO | "pinned" : " 29 GB" | 20260507 01:53:51.540 | INFO | } | 20260507 01:53:51.540 | INFO | Perf report for the last 5.14329 seconds: | 20260507 01:53:51.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:53:51.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:51.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:51.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:51.540 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 01:53:51.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:51.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:51.540 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:53:51.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:51.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:51.541 | INFO | Uploaded 36GiB to GPU in 1.98125s (18.1704GiB/s) | 20260507 01:53:51.541 | INFO | Downloaded 34GiB to GPU in 1.91559s (17.7491GiB/s) | 20260507 01:53:56.674 | INFO | Analyzing chunk frames = [6170624,6171136) | 20260507 01:53:56.674 | INFO | MemInfo: { | 20260507 01:53:56.674 | INFO | "gpu" : " 17 GB", | 20260507 01:53:56.674 | INFO | "huge" : "", | 20260507 01:53:56.674 | INFO | "huge_shared" : " 27 GB", | 20260507 01:53:56.674 | INFO | "malloc" : " 14 GB", | 20260507 01:53:56.674 | INFO | "pinned" : " 29 GB" | 20260507 01:53:56.674 | INFO | } | 20260507 01:53:56.674 | INFO | Perf report for the last 5.13372 seconds: | 20260507 01:53:56.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:53:56.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:56.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:56.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:56.674 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 01:53:56.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:56.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:56.674 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:53:56.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:56.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:53:56.674 | INFO | Uploaded 36GiB to GPU in 1.9888s (18.1014GiB/s) | 20260507 01:53:56.674 | INFO | Downloaded 34GiB to GPU in 1.95276s (17.4112GiB/s) | 20260507 01:54:01.816 | INFO | Analyzing chunk frames = [6171136,6171648) | 20260507 01:54:01.816 | INFO | MemInfo: { | 20260507 01:54:01.816 | INFO | "gpu" : " 17 GB", | 20260507 01:54:01.816 | INFO | "huge" : "", | 20260507 01:54:01.816 | INFO | "huge_shared" : " 27 GB", | 20260507 01:54:01.816 | INFO | "malloc" : " 14 GB", | 20260507 01:54:01.816 | INFO | "pinned" : " 29 GB" | 20260507 01:54:01.816 | INFO | } | 20260507 01:54:01.816 | INFO | Perf report for the last 5.14197 seconds: | 20260507 01:54:01.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:54:01.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:01.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:01.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:01.816 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 01:54:01.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:01.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:01.816 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 01:54:01.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:01.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:01.816 | INFO | Uploaded 36GiB to GPU in 2.01906s (17.8301GiB/s) | 20260507 01:54:01.816 | INFO | Downloaded 34GiB to GPU in 1.94723s (17.4607GiB/s) | 20260507 01:54:06.961 | INFO | Analyzing chunk frames = [6171648,6172160) | 20260507 01:54:06.961 | INFO | MemInfo: { | 20260507 01:54:06.961 | INFO | "gpu" : " 17 GB", | 20260507 01:54:06.961 | INFO | "huge" : "", | 20260507 01:54:06.961 | INFO | "huge_shared" : " 27 GB", | 20260507 01:54:06.961 | INFO | "malloc" : " 14 GB", | 20260507 01:54:06.961 | INFO | "pinned" : " 29 GB" | 20260507 01:54:06.961 | INFO | } | 20260507 01:54:06.961 | INFO | Perf report for the last 5.14478 seconds: | 20260507 01:54:06.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:54:06.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:06.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:06.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:06.961 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:54:06.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:06.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:06.961 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:54:06.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:06.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:06.961 | INFO | Uploaded 36GiB to GPU in 1.97619s (18.2168GiB/s) | 20260507 01:54:06.961 | INFO | Downloaded 34GiB to GPU in 1.9866s (17.1147GiB/s) | 20260507 01:54:12.094 | INFO | Analyzing chunk frames = [6172160,6172672) | 20260507 01:54:12.094 | INFO | MemInfo: { | 20260507 01:54:12.094 | INFO | "gpu" : " 17 GB", | 20260507 01:54:12.094 | INFO | "huge" : "", | 20260507 01:54:12.094 | INFO | "huge_shared" : " 27 GB", | 20260507 01:54:12.094 | INFO | "malloc" : " 14 GB", | 20260507 01:54:12.094 | INFO | "pinned" : " 29 GB" | 20260507 01:54:12.094 | INFO | } | 20260507 01:54:12.094 | INFO | Perf report for the last 5.13295 seconds: | 20260507 01:54:12.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:54:12.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:12.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:12.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:12.094 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260507 01:54:12.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:12.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:12.094 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:54:12.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:12.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:12.094 | INFO | Uploaded 36GiB to GPU in 1.98177s (18.1656GiB/s) | 20260507 01:54:12.094 | INFO | Downloaded 34GiB to GPU in 1.96615s (17.2927GiB/s) | 20260507 01:54:12.191 | INFO | Starting a round of Baz IO | 20260507 01:54:15.307 | INFO | Populating RTMetrics report | 20260507 01:54:15.308 | INFO | RTMetrics report populated | 20260507 01:54:15.308 | INFO | Opening tmp RTMetrics json file | 20260507 01:54:15.309 | INFO | Serializing RTMetrics to json | 20260507 01:54:15.310 | INFO | Serialization done, swapping tmp file now... | 20260507 01:54:15.310 | INFO | RTMetrics done | 20260507 01:54:15.397 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.206 seconds | 20260507 01:54:15.397 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.07% / 0.00% / 66.93% | 20260507 01:54:15.397 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.263457% / 0% / 7.54992e-05% | 20260507 01:54:15.398 | INFO | Baz IO overall: | 20260507 01:54:15.398 | INFO | BytesWritten : 3194873120030 | 20260507 01:54:15.398 | INFO | NumEvents : 1537199971387 | 20260507 01:54:17.232 | INFO | Analyzing chunk frames = [6172672,6173184) | 20260507 01:54:17.232 | INFO | MemInfo: { | 20260507 01:54:17.232 | INFO | "gpu" : " 17 GB", | 20260507 01:54:17.232 | INFO | "huge" : "", | 20260507 01:54:17.232 | INFO | "huge_shared" : " 27 GB", | 20260507 01:54:17.232 | INFO | "malloc" : " 14 GB", | 20260507 01:54:17.232 | INFO | "pinned" : " 29 GB" | 20260507 01:54:17.232 | INFO | } | 20260507 01:54:17.232 | INFO | Perf report for the last 5.13844 seconds: | 20260507 01:54:17.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:54:17.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:17.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:17.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:17.232 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 01:54:17.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:17.232 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 01:54:17.232 | INFO | PRE_HQ : 59.4%, 1.8, 55.9ms, 0 | 20260507 01:54:17.232 | INFO | BAZWRITER : 62.0%, 3.3, 105.4ms, 0 | 20260507 01:54:17.232 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:54:17.232 | INFO | Uploaded 36GiB to GPU in 2.0355s (17.6861GiB/s) | 20260507 01:54:17.233 | INFO | Downloaded 38GiB to GPU in 2.18135s (17.4204GiB/s) | 20260507 01:54:22.371 | INFO | Analyzing chunk frames = [6173184,6173696) | 20260507 01:54:22.371 | INFO | MemInfo: { | 20260507 01:54:22.371 | INFO | "gpu" : " 17 GB", | 20260507 01:54:22.371 | INFO | "huge" : "", | 20260507 01:54:22.371 | INFO | "huge_shared" : " 27 GB", | 20260507 01:54:22.371 | INFO | "malloc" : " 14 GB", | 20260507 01:54:22.371 | INFO | "pinned" : " 29 GB" | 20260507 01:54:22.371 | INFO | } | 20260507 01:54:22.371 | INFO | Perf report for the last 5.1391 seconds: | 20260507 01:54:22.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:54:22.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:22.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:22.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:22.371 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:54:22.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:22.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:22.371 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 01:54:22.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:22.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:22.371 | INFO | Uploaded 36GiB to GPU in 1.9901s (18.0895GiB/s) | 20260507 01:54:22.372 | INFO | Downloaded 34GiB to GPU in 1.93545s (17.5669GiB/s) | 20260507 01:54:27.514 | INFO | Analyzing chunk frames = [6173696,6174208) | 20260507 01:54:27.514 | INFO | MemInfo: { | 20260507 01:54:27.514 | INFO | "gpu" : " 17 GB", | 20260507 01:54:27.514 | INFO | "huge" : "", | 20260507 01:54:27.514 | INFO | "huge_shared" : " 27 GB", | 20260507 01:54:27.514 | INFO | "malloc" : " 14 GB", | 20260507 01:54:27.514 | INFO | "pinned" : " 29 GB" | 20260507 01:54:27.514 | INFO | } | 20260507 01:54:27.514 | INFO | Perf report for the last 5.14297 seconds: | 20260507 01:54:27.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:54:27.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:27.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:27.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:27.514 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 01:54:27.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:27.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:27.514 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:54:27.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:27.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:27.515 | INFO | Uploaded 36GiB to GPU in 1.99103s (18.0811GiB/s) | 20260507 01:54:27.515 | INFO | Downloaded 34GiB to GPU in 1.93876s (17.5369GiB/s) | 20260507 01:54:32.651 | INFO | Analyzing chunk frames = [6174208,6174720) | 20260507 01:54:32.651 | INFO | MemInfo: { | 20260507 01:54:32.651 | INFO | "gpu" : " 17 GB", | 20260507 01:54:32.651 | INFO | "huge" : "", | 20260507 01:54:32.651 | INFO | "huge_shared" : " 27 GB", | 20260507 01:54:32.651 | INFO | "malloc" : " 14 GB", | 20260507 01:54:32.651 | INFO | "pinned" : " 29 GB" | 20260507 01:54:32.651 | INFO | } | 20260507 01:54:32.651 | INFO | Perf report for the last 5.13694 seconds: | 20260507 01:54:32.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:54:32.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:32.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:32.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:32.651 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 01:54:32.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:32.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:32.651 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:54:32.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:32.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:32.652 | INFO | Uploaded 36GiB to GPU in 1.96357s (18.3339GiB/s) | 20260507 01:54:32.652 | INFO | Downloaded 34GiB to GPU in 1.92371s (17.6742GiB/s) | 20260507 01:54:37.797 | INFO | Analyzing chunk frames = [6174720,6175232) | 20260507 01:54:37.797 | INFO | MemInfo: { | 20260507 01:54:37.797 | INFO | "gpu" : " 17 GB", | 20260507 01:54:37.797 | INFO | "huge" : "", | 20260507 01:54:37.797 | INFO | "huge_shared" : " 27 GB", | 20260507 01:54:37.797 | INFO | "malloc" : " 14 GB", | 20260507 01:54:37.797 | INFO | "pinned" : " 29 GB" | 20260507 01:54:37.797 | INFO | } | 20260507 01:54:37.797 | INFO | Perf report for the last 5.14599 seconds: | 20260507 01:54:37.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:54:37.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:37.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:37.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:37.797 | INFO | ANALYSIS : 59.8%, 3.0, 93.3ms, 0 | 20260507 01:54:37.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:37.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:37.797 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 01:54:37.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:37.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:37.798 | INFO | Uploaded 36GiB to GPU in 1.96382s (18.3316GiB/s) | 20260507 01:54:37.798 | INFO | Downloaded 34GiB to GPU in 1.93975s (17.5281GiB/s) | 20260507 01:54:42.936 | INFO | Analyzing chunk frames = [6175232,6175744) | 20260507 01:54:42.936 | INFO | MemInfo: { | 20260507 01:54:42.936 | INFO | "gpu" : " 17 GB", | 20260507 01:54:42.936 | INFO | "huge" : "", | 20260507 01:54:42.936 | INFO | "huge_shared" : " 27 GB", | 20260507 01:54:42.936 | INFO | "malloc" : " 14 GB", | 20260507 01:54:42.936 | INFO | "pinned" : " 29 GB" | 20260507 01:54:42.936 | INFO | } | 20260507 01:54:42.936 | INFO | Perf report for the last 5.13907 seconds: | 20260507 01:54:42.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:54:42.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:42.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:42.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:42.936 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 01:54:42.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:42.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:42.937 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 01:54:42.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:42.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:42.937 | INFO | Uploaded 36GiB to GPU in 2.0263s (17.7664GiB/s) | 20260507 01:54:42.937 | INFO | Downloaded 34GiB to GPU in 1.9527s (17.4118GiB/s) | 20260507 01:54:48.067 | INFO | Analyzing chunk frames = [6175744,6176256) | 20260507 01:54:48.067 | INFO | MemInfo: { | 20260507 01:54:48.067 | INFO | "gpu" : " 17 GB", | 20260507 01:54:48.067 | INFO | "huge" : "", | 20260507 01:54:48.067 | INFO | "huge_shared" : " 27 GB", | 20260507 01:54:48.067 | INFO | "malloc" : " 14 GB", | 20260507 01:54:48.068 | INFO | "pinned" : " 29 GB" | 20260507 01:54:48.068 | INFO | } | 20260507 01:54:48.068 | INFO | Perf report for the last 5.13115 seconds: | 20260507 01:54:48.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:54:48.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:48.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:48.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:48.068 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260507 01:54:48.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:48.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:48.068 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:54:48.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:48.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:48.068 | INFO | Uploaded 36GiB to GPU in 1.97857s (18.195GiB/s) | 20260507 01:54:48.068 | INFO | Downloaded 34GiB to GPU in 1.98487s (17.1296GiB/s) | 20260507 01:54:53.207 | INFO | Analyzing chunk frames = [6176256,6176768) | 20260507 01:54:53.207 | INFO | MemInfo: { | 20260507 01:54:53.207 | INFO | "gpu" : " 17 GB", | 20260507 01:54:53.207 | INFO | "huge" : "", | 20260507 01:54:53.207 | INFO | "huge_shared" : " 27 GB", | 20260507 01:54:53.207 | INFO | "malloc" : " 14 GB", | 20260507 01:54:53.207 | INFO | "pinned" : " 29 GB" | 20260507 01:54:53.207 | INFO | } | 20260507 01:54:53.207 | INFO | Perf report for the last 5.13946 seconds: | 20260507 01:54:53.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:54:53.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:53.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:53.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:53.207 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 01:54:53.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:53.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:53.207 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 01:54:53.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:53.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:53.207 | INFO | Uploaded 36GiB to GPU in 2.00713s (17.9361GiB/s) | 20260507 01:54:53.207 | INFO | Downloaded 34GiB to GPU in 1.98308s (17.145GiB/s) | 20260507 01:54:53.208 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:54:53.208 | INFO | Total mem usage: 18 GiB | 20260507 01:54:53.208 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:54:53.208 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:54:53.208 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:54:53.208 | INFO | High Water Breakdown: | 20260507 01:54:53.208 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:54:53.208 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:54:53.208 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:54:53.208 | INFO | ------------------------------------------ | 20260507 01:54:53.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:54:53.208 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:54:53.208 | INFO | Total mem usage: 26.5 GiB | 20260507 01:54:53.208 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:54:53.208 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:54:53.208 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:54:53.208 | INFO | High Water Breakdown: | 20260507 01:54:53.208 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:54:53.208 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:54:53.208 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:54:53.208 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:54:53.208 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:54:53.208 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:54:53.208 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:54:53.208 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:54:53.208 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:54:53.208 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:54:53.208 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:54:53.208 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:54:53.208 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:54:53.208 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:54:53.208 | INFO | ------------------------------------------ | 20260507 01:54:53.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:54:53.208 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:54:53.208 | INFO | Total mem usage: 25.6 GiB | 20260507 01:54:53.208 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:54:53.208 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:54:53.208 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:54:53.208 | INFO | High Water Breakdown: | 20260507 01:54:53.208 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:54:53.208 | INFO | ------------------------------------------ | 20260507 01:54:53.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:54:53.208 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:54:53.208 | INFO | Total mem usage: 15.7 GiB | 20260507 01:54:53.208 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:54:53.208 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:54:53.208 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:54:53.208 | INFO | High Water Breakdown: | 20260507 01:54:53.208 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:54:53.208 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:54:53.208 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:54:53.208 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:54:53.208 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:54:53.208 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:54:53.208 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:54:53.208 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:54:53.208 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:54:53.208 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:54:53.208 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:54:53.208 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:54:53.208 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:54:53.208 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:54:53.208 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:54:53.208 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:54:53.208 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:54:53.208 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:54:53.208 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:54:53.208 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:54:53.208 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:54:53.208 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:54:53.208 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:54:53.208 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:54:53.208 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:54:53.208 | INFO | ------------------------------------------ | 20260507 01:54:53.208 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:54:53.208 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:54:53.208 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:54:53.208 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:54:53.208 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.52e+01, 1.34e+01 / 1.81e+00 | 20260507 01:54:53.208 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 1.00e-01, 1.01e-02 / 5.22e-03 | 20260507 01:54:53.208 | INFO | PerfReport: 11.68%, Baseline -- 3.60e+00 / 4.42e+00, 5.30e-01 / 3.05e-01 | 20260507 01:54:53.208 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.45e+00, 1.75e-01 / 1.28e-01 | 20260507 01:54:53.208 | INFO | PerfReport: 16.04%, DME -- 3.96e+01 / 4.40e+01, 3.94e+00 / 2.18e+00 | 20260507 01:54:53.208 | INFO | PerfReport: 57.58%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.36e+00 / 2.10e+00 | 20260507 01:54:53.208 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.16e+00 / 2.67e+00, 3.80e-01 / 2.61e-01 | 20260507 01:54:53.208 | INFO | PerfReport: 3.09%, Metrics -- 9.54e-01 / 1.28e+00, 2.54e-01 / 1.02e-01 | 20260507 01:54:53.208 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.25e-01 / 3.78e-01, 5.19e-02 / 2.28e-02 | 20260507 01:54:53.208 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:54:53.208 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:54:53.208 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:54:53.208 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 01:54:53.208 | INFO | PerfReport: 21.59%, Upload -- 2.08e+01 / 0.00e+00, 6.90e+00 / 0.00e+00 | 20260507 01:54:53.209 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260507 01:54:53.209 | INFO | PerfReport: 32.03%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 01:54:53.209 | INFO | PerfReport: 1.24%, LockUpload -- 1.20e+00 / 0.00e+00, 4.76e+00 / 0.00e+00 | 20260507 01:54:53.209 | INFO | PerfReport: 1.42%, LockDownload -- 1.37e+00 / 0.00e+00, 3.29e+00 / 0.00e+00 | 20260507 01:54:53.209 | INFO | PerfReport: 22.30%, LockCompute -- 2.15e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 01:54:53.209 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260507 01:54:53.304 | INFO | Starting a round of Baz IO | 20260507 01:54:56.425 | INFO | Populating RTMetrics report | 20260507 01:54:56.425 | INFO | RTMetrics report populated | 20260507 01:54:56.425 | INFO | Opening tmp RTMetrics json file | 20260507 01:54:56.426 | INFO | Serializing RTMetrics to json | 20260507 01:54:56.427 | INFO | Serialization done, swapping tmp file now... | 20260507 01:54:56.430 | INFO | RTMetrics done | 20260507 01:54:56.513 | INFO | Finished a round of Baz IO: Wrote 1.802 GiB in 3.208 seconds | 20260507 01:54:56.513 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.07% / 0.00% / 66.93% | 20260507 01:54:56.513 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.264068% / 0% / 7.94729e-05% | 20260507 01:54:56.513 | INFO | Baz IO overall: | 20260507 01:54:56.513 | INFO | BytesWritten : 3196808191797 | 20260507 01:54:56.513 | INFO | NumEvents : 1538117161865 | 20260507 01:54:58.345 | INFO | Analyzing chunk frames = [6176768,6177280) | 20260507 01:54:58.345 | INFO | MemInfo: { | 20260507 01:54:58.345 | INFO | "gpu" : " 17 GB", | 20260507 01:54:58.345 | INFO | "huge" : "", | 20260507 01:54:58.345 | INFO | "huge_shared" : " 27 GB", | 20260507 01:54:58.345 | INFO | "malloc" : " 14 GB", | 20260507 01:54:58.345 | INFO | "pinned" : " 29 GB" | 20260507 01:54:58.345 | INFO | } | 20260507 01:54:58.345 | INFO | Perf report for the last 5.13776 seconds: | 20260507 01:54:58.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:54:58.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:58.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:58.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:58.345 | INFO | ANALYSIS : 62.1%, 3.0, 96.5ms, 0 | 20260507 01:54:58.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:54:58.345 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 01:54:58.345 | INFO | PRE_HQ : 59.3%, 1.8, 55.8ms, 0 | 20260507 01:54:58.345 | INFO | BAZWRITER : 62.0%, 3.3, 105.5ms, 0 | 20260507 01:54:58.345 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:54:58.345 | INFO | Uploaded 36GiB to GPU in 2.0235s (17.7909GiB/s) | 20260507 01:54:58.345 | INFO | Downloaded 38GiB to GPU in 2.18405s (17.3989GiB/s) | 20260507 01:55:03.486 | INFO | Analyzing chunk frames = [6177280,6177792) | 20260507 01:55:03.486 | INFO | MemInfo: { | 20260507 01:55:03.486 | INFO | "gpu" : " 17 GB", | 20260507 01:55:03.486 | INFO | "huge" : "", | 20260507 01:55:03.486 | INFO | "huge_shared" : " 27 GB", | 20260507 01:55:03.486 | INFO | "malloc" : " 14 GB", | 20260507 01:55:03.486 | INFO | "pinned" : " 29 GB" | 20260507 01:55:03.486 | INFO | } | 20260507 01:55:03.486 | INFO | Perf report for the last 5.14141 seconds: | 20260507 01:55:03.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:55:03.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:03.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:03.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:03.486 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:55:03.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:03.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:03.486 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:55:03.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:03.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:03.487 | INFO | Uploaded 36GiB to GPU in 2.00355s (17.9681GiB/s) | 20260507 01:55:03.487 | INFO | Downloaded 34GiB to GPU in 1.92927s (17.6233GiB/s) | 20260507 01:55:08.633 | INFO | Analyzing chunk frames = [6177792,6178304) | 20260507 01:55:08.633 | INFO | MemInfo: { | 20260507 01:55:08.633 | INFO | "gpu" : " 17 GB", | 20260507 01:55:08.633 | INFO | "huge" : "", | 20260507 01:55:08.633 | INFO | "huge_shared" : " 27 GB", | 20260507 01:55:08.633 | INFO | "malloc" : " 14 GB", | 20260507 01:55:08.633 | INFO | "pinned" : " 29 GB" | 20260507 01:55:08.633 | INFO | } | 20260507 01:55:08.633 | INFO | Perf report for the last 5.14715 seconds: | 20260507 01:55:08.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:55:08.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:08.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:08.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:08.633 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260507 01:55:08.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:08.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:08.633 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:55:08.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:08.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:08.634 | INFO | Uploaded 36GiB to GPU in 2.00077s (17.9931GiB/s) | 20260507 01:55:08.634 | INFO | Downloaded 34GiB to GPU in 1.93114s (17.6062GiB/s) | 20260507 01:55:13.773 | INFO | Analyzing chunk frames = [6178304,6178816) | 20260507 01:55:13.773 | INFO | MemInfo: { | 20260507 01:55:13.773 | INFO | "gpu" : " 17 GB", | 20260507 01:55:13.773 | INFO | "huge" : "", | 20260507 01:55:13.773 | INFO | "huge_shared" : " 27 GB", | 20260507 01:55:13.773 | INFO | "malloc" : " 14 GB", | 20260507 01:55:13.773 | INFO | "pinned" : " 29 GB" | 20260507 01:55:13.773 | INFO | } | 20260507 01:55:13.773 | INFO | Perf report for the last 5.13956 seconds: | 20260507 01:55:13.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:55:13.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:13.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:13.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:13.773 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 01:55:13.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:13.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:13.773 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:55:13.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:13.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:13.773 | INFO | Uploaded 36GiB to GPU in 1.99437s (18.0508GiB/s) | 20260507 01:55:13.773 | INFO | Downloaded 34GiB to GPU in 1.91685s (17.7375GiB/s) | 20260507 01:55:18.905 | INFO | Analyzing chunk frames = [6178816,6179328) | 20260507 01:55:18.905 | INFO | MemInfo: { | 20260507 01:55:18.905 | INFO | "gpu" : " 17 GB", | 20260507 01:55:18.905 | INFO | "huge" : "", | 20260507 01:55:18.905 | INFO | "huge_shared" : " 27 GB", | 20260507 01:55:18.905 | INFO | "malloc" : " 14 GB", | 20260507 01:55:18.905 | INFO | "pinned" : " 29 GB" | 20260507 01:55:18.905 | INFO | } | 20260507 01:55:18.905 | INFO | Perf report for the last 5.13208 seconds: | 20260507 01:55:18.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:55:18.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:18.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:18.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:18.905 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 01:55:18.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:18.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:18.905 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:55:18.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:18.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:18.905 | INFO | Uploaded 36GiB to GPU in 2.01554s (17.8612GiB/s) | 20260507 01:55:18.905 | INFO | Downloaded 34GiB to GPU in 1.94299s (17.4988GiB/s) | 20260507 01:55:24.043 | INFO | Analyzing chunk frames = [6179328,6179840) | 20260507 01:55:24.043 | INFO | MemInfo: { | 20260507 01:55:24.043 | INFO | "gpu" : " 17 GB", | 20260507 01:55:24.043 | INFO | "huge" : "", | 20260507 01:55:24.044 | INFO | "huge_shared" : " 27 GB", | 20260507 01:55:24.044 | INFO | "malloc" : " 14 GB", | 20260507 01:55:24.044 | INFO | "pinned" : " 29 GB" | 20260507 01:55:24.044 | INFO | } | 20260507 01:55:24.044 | INFO | Perf report for the last 5.13849 seconds: | 20260507 01:55:24.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:55:24.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:24.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:24.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:24.044 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 01:55:24.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:24.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:24.044 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 01:55:24.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:24.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:24.044 | INFO | Uploaded 36GiB to GPU in 2.04121s (17.6366GiB/s) | 20260507 01:55:24.044 | INFO | Downloaded 34GiB to GPU in 1.94172s (17.5103GiB/s) | 20260507 01:55:29.181 | INFO | Analyzing chunk frames = [6179840,6180352) | 20260507 01:55:29.181 | INFO | MemInfo: { | 20260507 01:55:29.181 | INFO | "gpu" : " 17 GB", | 20260507 01:55:29.181 | INFO | "huge" : "", | 20260507 01:55:29.181 | INFO | "huge_shared" : " 27 GB", | 20260507 01:55:29.181 | INFO | "malloc" : " 14 GB", | 20260507 01:55:29.181 | INFO | "pinned" : " 29 GB" | 20260507 01:55:29.181 | INFO | } | 20260507 01:55:29.181 | INFO | Perf report for the last 5.13723 seconds: | 20260507 01:55:29.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:55:29.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:29.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:29.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:29.181 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 01:55:29.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:29.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:29.181 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 01:55:29.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:29.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:29.181 | INFO | Uploaded 36GiB to GPU in 1.94516s (18.5075GiB/s) | 20260507 01:55:29.181 | INFO | Downloaded 34GiB to GPU in 1.98829s (17.1001GiB/s) | 20260507 01:55:34.317 | INFO | Analyzing chunk frames = [6180352,6180864) | 20260507 01:55:34.317 | INFO | MemInfo: { | 20260507 01:55:34.317 | INFO | "gpu" : " 17 GB", | 20260507 01:55:34.317 | INFO | "huge" : "", | 20260507 01:55:34.317 | INFO | "huge_shared" : " 27 GB", | 20260507 01:55:34.317 | INFO | "malloc" : " 14 GB", | 20260507 01:55:34.317 | INFO | "pinned" : " 29 GB" | 20260507 01:55:34.317 | INFO | } | 20260507 01:55:34.317 | INFO | Perf report for the last 5.13612 seconds: | 20260507 01:55:34.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:55:34.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:34.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:34.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:34.317 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 01:55:34.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:34.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:34.317 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 01:55:34.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:34.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:34.317 | INFO | Uploaded 36GiB to GPU in 2.00191s (17.9828GiB/s) | 20260507 01:55:34.317 | INFO | Downloaded 34GiB to GPU in 1.98191s (17.1552GiB/s) | 20260507 01:55:34.414 | INFO | Starting a round of Baz IO | 20260507 01:55:37.532 | INFO | Populating RTMetrics report | 20260507 01:55:37.533 | INFO | RTMetrics report populated | 20260507 01:55:37.533 | INFO | Opening tmp RTMetrics json file | 20260507 01:55:37.538 | INFO | Serializing RTMetrics to json | 20260507 01:55:37.551 | INFO | Serialization done, swapping tmp file now... | 20260507 01:55:37.552 | INFO | RTMetrics done | 20260507 01:55:37.622 | INFO | Finished a round of Baz IO: Wrote 1.801 GiB in 3.208 seconds | 20260507 01:55:37.622 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.07% / 0.00% / 66.93% | 20260507 01:55:37.622 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.262209% / 3.97364e-06% / 6.35783e-05% | 20260507 01:55:37.622 | INFO | Baz IO overall: | 20260507 01:55:37.622 | INFO | BytesWritten : 3198741551432 | 20260507 01:55:37.622 | INFO | NumEvents : 1539033178503 | 20260507 01:55:39.468 | INFO | Analyzing chunk frames = [6180864,6181376) | 20260507 01:55:39.468 | INFO | MemInfo: { | 20260507 01:55:39.468 | INFO | "gpu" : " 17 GB", | 20260507 01:55:39.468 | INFO | "huge" : "", | 20260507 01:55:39.468 | INFO | "huge_shared" : " 27 GB", | 20260507 01:55:39.468 | INFO | "malloc" : " 14 GB", | 20260507 01:55:39.468 | INFO | "pinned" : " 29 GB" | 20260507 01:55:39.468 | INFO | } | 20260507 01:55:39.468 | INFO | Perf report for the last 5.15136 seconds: | 20260507 01:55:39.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:55:39.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:39.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:39.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:39.468 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260507 01:55:39.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:39.468 | INFO | RT_METRICS : 42.2%, 1.0, 22.2ms, 0 | 20260507 01:55:39.468 | INFO | PRE_HQ : 59.4%, 1.8, 56.4ms, 0 | 20260507 01:55:39.468 | INFO | BAZWRITER : 61.9%, 3.3, 105.2ms, 0 | 20260507 01:55:39.468 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:55:39.469 | INFO | Uploaded 36GiB to GPU in 2.01596s (17.8575GiB/s) | 20260507 01:55:39.469 | INFO | Downloaded 38GiB to GPU in 2.17693s (17.4558GiB/s) | 20260507 01:55:44.606 | INFO | Analyzing chunk frames = [6181376,6181888) | 20260507 01:55:44.606 | INFO | MemInfo: { | 20260507 01:55:44.606 | INFO | "gpu" : " 17 GB", | 20260507 01:55:44.606 | INFO | "huge" : "", | 20260507 01:55:44.606 | INFO | "huge_shared" : " 27 GB", | 20260507 01:55:44.606 | INFO | "malloc" : " 14 GB", | 20260507 01:55:44.606 | INFO | "pinned" : " 29 GB" | 20260507 01:55:44.606 | INFO | } | 20260507 01:55:44.606 | INFO | Perf report for the last 5.13804 seconds: | 20260507 01:55:44.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:55:44.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:44.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:44.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:44.606 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:55:44.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:44.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:44.606 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:55:44.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:44.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:44.607 | INFO | Uploaded 36GiB to GPU in 2.03278s (17.7097GiB/s) | 20260507 01:55:44.607 | INFO | Downloaded 34GiB to GPU in 1.93804s (17.5435GiB/s) | 20260507 01:55:49.749 | INFO | Analyzing chunk frames = [6181888,6182400) | 20260507 01:55:49.749 | INFO | MemInfo: { | 20260507 01:55:49.749 | INFO | "gpu" : " 17 GB", | 20260507 01:55:49.749 | INFO | "huge" : "", | 20260507 01:55:49.749 | INFO | "huge_shared" : " 27 GB", | 20260507 01:55:49.749 | INFO | "malloc" : " 14 GB", | 20260507 01:55:49.750 | INFO | "pinned" : " 29 GB" | 20260507 01:55:49.750 | INFO | } | 20260507 01:55:49.750 | INFO | Perf report for the last 5.14319 seconds: | 20260507 01:55:49.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:55:49.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:49.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:49.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:49.750 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:55:49.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:49.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:49.750 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 01:55:49.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:49.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:49.750 | INFO | Uploaded 36GiB to GPU in 2.0054s (17.9515GiB/s) | 20260507 01:55:49.750 | INFO | Downloaded 34GiB to GPU in 1.93801s (17.5438GiB/s) | 20260507 01:55:54.887 | INFO | Analyzing chunk frames = [6182400,6182912) | 20260507 01:55:54.887 | INFO | MemInfo: { | 20260507 01:55:54.887 | INFO | "gpu" : " 17 GB", | 20260507 01:55:54.887 | INFO | "huge" : "", | 20260507 01:55:54.888 | INFO | "huge_shared" : " 27 GB", | 20260507 01:55:54.888 | INFO | "malloc" : " 14 GB", | 20260507 01:55:54.888 | INFO | "pinned" : " 29 GB" | 20260507 01:55:54.888 | INFO | } | 20260507 01:55:54.888 | INFO | Perf report for the last 5.13803 seconds: | 20260507 01:55:54.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:55:54.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:54.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:54.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:54.888 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 01:55:54.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:54.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:54.888 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:55:54.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:54.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:55:54.888 | INFO | Uploaded 36GiB to GPU in 1.98333s (18.1513GiB/s) | 20260507 01:55:54.888 | INFO | Downloaded 34GiB to GPU in 1.91918s (17.7159GiB/s) | 20260507 01:56:00.021 | INFO | Analyzing chunk frames = [6182912,6183424) | 20260507 01:56:00.021 | INFO | MemInfo: { | 20260507 01:56:00.021 | INFO | "gpu" : " 17 GB", | 20260507 01:56:00.021 | INFO | "huge" : "", | 20260507 01:56:00.021 | INFO | "huge_shared" : " 27 GB", | 20260507 01:56:00.021 | INFO | "malloc" : " 14 GB", | 20260507 01:56:00.021 | INFO | "pinned" : " 29 GB" | 20260507 01:56:00.021 | INFO | } | 20260507 01:56:00.022 | INFO | Perf report for the last 5.13393 seconds: | 20260507 01:56:00.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:56:00.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:00.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:00.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:00.022 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 01:56:00.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:00.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:00.022 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:56:00.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:00.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:00.022 | INFO | Uploaded 36GiB to GPU in 2.00864s (17.9225GiB/s) | 20260507 01:56:00.022 | INFO | Downloaded 34GiB to GPU in 1.93715s (17.5516GiB/s) | 20260507 01:56:05.156 | INFO | Analyzing chunk frames = [6183424,6183936) | 20260507 01:56:05.156 | INFO | MemInfo: { | 20260507 01:56:05.156 | INFO | "gpu" : " 17 GB", | 20260507 01:56:05.157 | INFO | "huge" : "", | 20260507 01:56:05.157 | INFO | "huge_shared" : " 27 GB", | 20260507 01:56:05.157 | INFO | "malloc" : " 14 GB", | 20260507 01:56:05.157 | INFO | "pinned" : " 29 GB" | 20260507 01:56:05.157 | INFO | } | 20260507 01:56:05.157 | INFO | Perf report for the last 5.13509 seconds: | 20260507 01:56:05.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:56:05.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:05.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:05.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:05.157 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:56:05.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:05.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:05.157 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 01:56:05.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:05.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:05.157 | INFO | Uploaded 36GiB to GPU in 2.05846s (17.4888GiB/s) | 20260507 01:56:05.157 | INFO | Downloaded 34GiB to GPU in 1.96133s (17.3352GiB/s) | 20260507 01:56:10.299 | INFO | Analyzing chunk frames = [6183936,6184448) | 20260507 01:56:10.299 | INFO | MemInfo: { | 20260507 01:56:10.299 | INFO | "gpu" : " 17 GB", | 20260507 01:56:10.299 | INFO | "huge" : "", | 20260507 01:56:10.299 | INFO | "huge_shared" : " 27 GB", | 20260507 01:56:10.299 | INFO | "malloc" : " 14 GB", | 20260507 01:56:10.299 | INFO | "pinned" : " 29 GB" | 20260507 01:56:10.299 | INFO | } | 20260507 01:56:10.299 | INFO | Perf report for the last 5.14268 seconds: | 20260507 01:56:10.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:56:10.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:10.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:10.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:10.299 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 01:56:10.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:10.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:10.300 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:56:10.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:10.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:10.300 | INFO | Uploaded 36GiB to GPU in 1.95633s (18.4018GiB/s) | 20260507 01:56:10.300 | INFO | Downloaded 34GiB to GPU in 1.98795s (17.1031GiB/s) | 20260507 01:56:15.443 | INFO | Analyzing chunk frames = [6184448,6184960) | 20260507 01:56:15.443 | INFO | MemInfo: { | 20260507 01:56:15.443 | INFO | "gpu" : " 17 GB", | 20260507 01:56:15.443 | INFO | "huge" : "", | 20260507 01:56:15.443 | INFO | "huge_shared" : " 27 GB", | 20260507 01:56:15.443 | INFO | "malloc" : " 14 GB", | 20260507 01:56:15.443 | INFO | "pinned" : " 29 GB" | 20260507 01:56:15.443 | INFO | } | 20260507 01:56:15.443 | INFO | Perf report for the last 5.14396 seconds: | 20260507 01:56:15.443 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:56:15.443 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:15.443 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:15.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:15.443 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:56:15.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:15.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:15.443 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 01:56:15.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:15.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:15.444 | INFO | Uploaded 36GiB to GPU in 1.9887s (18.1022GiB/s) | 20260507 01:56:15.444 | INFO | Downloaded 34GiB to GPU in 1.98155s (17.1583GiB/s) | 20260507 01:56:15.444 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:56:15.444 | INFO | Total mem usage: 18 GiB | 20260507 01:56:15.444 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:56:15.444 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 01:56:15.444 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:56:15.444 | INFO | High Water Breakdown: | 20260507 01:56:15.444 | INFO | 60.2% : 8.58 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:56:15.444 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:56:15.444 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:56:15.444 | INFO | ------------------------------------------ | 20260507 01:56:15.444 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:56:15.444 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:56:15.444 | INFO | Total mem usage: 26.5 GiB | 20260507 01:56:15.444 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:56:15.444 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:56:15.444 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:56:15.444 | INFO | High Water Breakdown: | 20260507 01:56:15.444 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:56:15.444 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:56:15.444 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:56:15.444 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:56:15.444 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:56:15.444 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:56:15.444 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:56:15.444 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:56:15.444 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:56:15.444 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:56:15.444 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:56:15.444 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:56:15.444 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:56:15.444 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:56:15.444 | INFO | ------------------------------------------ | 20260507 01:56:15.444 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:56:15.444 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:56:15.444 | INFO | Total mem usage: 25.6 GiB | 20260507 01:56:15.444 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:56:15.444 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:56:15.444 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:56:15.444 | INFO | High Water Breakdown: | 20260507 01:56:15.444 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:56:15.444 | INFO | ------------------------------------------ | 20260507 01:56:15.444 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:56:15.444 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:56:15.444 | INFO | Total mem usage: 15.7 GiB | 20260507 01:56:15.444 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:56:15.444 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:56:15.444 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 01:56:15.444 | INFO | High Water Breakdown: | 20260507 01:56:15.444 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:56:15.444 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:56:15.444 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:56:15.444 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:56:15.444 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:56:15.444 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:56:15.444 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:56:15.444 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:56:15.444 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:56:15.444 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:56:15.444 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:56:15.444 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:56:15.444 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:56:15.444 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:56:15.444 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:56:15.444 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:56:15.444 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:56:15.444 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:56:15.444 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:56:15.444 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:56:15.444 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:56:15.444 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:56:15.444 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:56:15.444 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:56:15.444 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:56:15.444 | INFO | ------------------------------------------ | 20260507 01:56:15.444 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:56:15.444 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:56:15.444 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:56:15.444 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:56:15.445 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.56e+01, 1.32e+01 / 2.17e+00 | 20260507 01:56:15.445 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.00e-01, 9.11e-03 / 5.92e-03 | 20260507 01:56:15.445 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.42e+00, 5.21e-01 / 2.89e-01 | 20260507 01:56:15.445 | INFO | PerfReport: 3.23%, Histogram -- 9.94e-01 / 1.48e+00, 1.71e-01 / 1.15e-01 | 20260507 01:56:15.445 | INFO | PerfReport: 16.16%, DME -- 3.98e+01 / 4.48e+01, 4.12e+00 / 2.85e+00 | 20260507 01:56:15.445 | INFO | PerfReport: 57.48%, FrameLabeling -- 1.77e+01 / 2.05e+01, 1.24e+00 / 1.79e+00 | 20260507 01:56:15.445 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.17e+00 / 2.73e+00, 3.83e-01 / 2.34e-01 | 20260507 01:56:15.445 | INFO | PerfReport: 3.10%, Metrics -- 9.55e-01 / 1.28e+00, 2.53e-01 / 1.57e-01 | 20260507 01:56:15.445 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.24e-01 / 3.78e-01, 5.22e-02 / 2.33e-02 | 20260507 01:56:15.445 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:56:15.445 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:56:15.445 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:56:15.445 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 01:56:15.445 | INFO | PerfReport: 21.65%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260507 01:56:15.445 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260507 01:56:15.445 | INFO | PerfReport: 31.94%, Compute -- 3.08e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 01:56:15.445 | INFO | PerfReport: 1.46%, LockUpload -- 1.41e+00 / 0.00e+00, 4.99e+00 / 0.00e+00 | 20260507 01:56:15.445 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.17e+00 / 0.00e+00 | 20260507 01:56:15.445 | INFO | PerfReport: 22.26%, LockCompute -- 2.15e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 01:56:15.445 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260507 01:56:15.543 | INFO | Starting a round of Baz IO | 20260507 01:56:18.655 | INFO | Populating RTMetrics report | 20260507 01:56:18.655 | INFO | RTMetrics report populated | 20260507 01:56:18.655 | INFO | Opening tmp RTMetrics json file | 20260507 01:56:18.656 | INFO | Serializing RTMetrics to json | 20260507 01:56:18.657 | INFO | Serialization done, swapping tmp file now... | 20260507 01:56:18.658 | INFO | RTMetrics done | 20260507 01:56:18.743 | INFO | Finished a round of Baz IO: Wrote 1.8 GiB in 3.2 seconds | 20260507 01:56:18.743 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.07% / 0.00% / 66.93% | 20260507 01:56:18.747 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.261732% / 3.97364e-06% / 3.97364e-05% | 20260507 01:56:18.747 | INFO | Baz IO overall: | 20260507 01:56:18.747 | INFO | BytesWritten : 3200674140373 | 20260507 01:56:18.747 | INFO | NumEvents : 1539948472750 | 20260507 01:56:20.587 | INFO | Analyzing chunk frames = [6184960,6185472) | 20260507 01:56:20.587 | INFO | MemInfo: { | 20260507 01:56:20.587 | INFO | "gpu" : " 17 GB", | 20260507 01:56:20.587 | INFO | "huge" : "", | 20260507 01:56:20.587 | INFO | "huge_shared" : " 27 GB", | 20260507 01:56:20.587 | INFO | "malloc" : " 14 GB", | 20260507 01:56:20.587 | INFO | "pinned" : " 29 GB" | 20260507 01:56:20.587 | INFO | } | 20260507 01:56:20.587 | INFO | Perf report for the last 5.14368 seconds: | 20260507 01:56:20.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:56:20.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:20.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:20.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:20.587 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260507 01:56:20.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:20.587 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 01:56:20.587 | INFO | PRE_HQ : 59.5%, 1.8, 55.8ms, 0 | 20260507 01:56:20.587 | INFO | BAZWRITER : 61.9%, 3.3, 105.3ms, 0 | 20260507 01:56:20.587 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:56:20.587 | INFO | Uploaded 36GiB to GPU in 2.03722s (17.6712GiB/s) | 20260507 01:56:20.587 | INFO | Downloaded 38GiB to GPU in 2.17591s (17.464GiB/s) | 20260507 01:56:25.719 | INFO | Analyzing chunk frames = [6185472,6185984) | 20260507 01:56:25.719 | INFO | MemInfo: { | 20260507 01:56:25.719 | INFO | "gpu" : " 17 GB", | 20260507 01:56:25.719 | INFO | "huge" : "", | 20260507 01:56:25.719 | INFO | "huge_shared" : " 27 GB", | 20260507 01:56:25.719 | INFO | "malloc" : " 14 GB", | 20260507 01:56:25.719 | INFO | "pinned" : " 29 GB" | 20260507 01:56:25.719 | INFO | } | 20260507 01:56:25.719 | INFO | Perf report for the last 5.1317 seconds: | 20260507 01:56:25.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:56:25.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:25.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:25.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:25.719 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 01:56:25.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:25.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:25.719 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 01:56:25.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:25.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:25.719 | INFO | Uploaded 36GiB to GPU in 2.0004s (17.9964GiB/s) | 20260507 01:56:25.719 | INFO | Downloaded 34GiB to GPU in 1.9346s (17.5747GiB/s) | 20260507 01:56:30.856 | INFO | Analyzing chunk frames = [6185984,6186496) | 20260507 01:56:30.856 | INFO | MemInfo: { | 20260507 01:56:30.856 | INFO | "gpu" : " 17 GB", | 20260507 01:56:30.856 | INFO | "huge" : "", | 20260507 01:56:30.856 | INFO | "huge_shared" : " 27 GB", | 20260507 01:56:30.857 | INFO | "malloc" : " 14 GB", | 20260507 01:56:30.857 | INFO | "pinned" : " 29 GB" | 20260507 01:56:30.857 | INFO | } | 20260507 01:56:30.857 | INFO | Perf report for the last 5.13782 seconds: | 20260507 01:56:30.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:56:30.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:30.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:30.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:30.857 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:56:30.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:30.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:30.857 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:56:30.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:30.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:30.857 | INFO | Uploaded 36GiB to GPU in 1.99427s (18.0517GiB/s) | 20260507 01:56:30.857 | INFO | Downloaded 34GiB to GPU in 1.94266s (17.5018GiB/s) | 20260507 01:56:35.997 | INFO | Analyzing chunk frames = [6186496,6187008) | 20260507 01:56:35.997 | INFO | MemInfo: { | 20260507 01:56:35.997 | INFO | "gpu" : " 17 GB", | 20260507 01:56:35.997 | INFO | "huge" : "", | 20260507 01:56:35.997 | INFO | "huge_shared" : " 27 GB", | 20260507 01:56:35.997 | INFO | "malloc" : " 14 GB", | 20260507 01:56:35.997 | INFO | "pinned" : " 29 GB" | 20260507 01:56:35.997 | INFO | } | 20260507 01:56:35.997 | INFO | Perf report for the last 5.1401 seconds: | 20260507 01:56:35.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:56:35.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:35.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:35.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:35.997 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 01:56:35.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:35.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:35.997 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:56:35.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:35.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:35.997 | INFO | Uploaded 36GiB to GPU in 1.95984s (18.3689GiB/s) | 20260507 01:56:35.997 | INFO | Downloaded 34GiB to GPU in 1.91206s (17.7819GiB/s) | 20260507 01:56:41.134 | INFO | Analyzing chunk frames = [6187008,6187520) | 20260507 01:56:41.134 | INFO | MemInfo: { | 20260507 01:56:41.135 | INFO | "gpu" : " 17 GB", | 20260507 01:56:41.135 | INFO | "huge" : "", | 20260507 01:56:41.135 | INFO | "huge_shared" : " 27 GB", | 20260507 01:56:41.135 | INFO | "malloc" : " 14 GB", | 20260507 01:56:41.135 | INFO | "pinned" : " 29 GB" | 20260507 01:56:41.135 | INFO | } | 20260507 01:56:41.135 | INFO | Perf report for the last 5.13803 seconds: | 20260507 01:56:41.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:56:41.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:41.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:41.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:41.135 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 01:56:41.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:41.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:41.135 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 01:56:41.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:41.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:41.135 | INFO | Uploaded 36GiB to GPU in 1.96082s (18.3596GiB/s) | 20260507 01:56:41.135 | INFO | Downloaded 34GiB to GPU in 1.94614s (17.4705GiB/s) | 20260507 01:56:46.276 | INFO | Analyzing chunk frames = [6187520,6188032) | 20260507 01:56:46.276 | INFO | MemInfo: { | 20260507 01:56:46.276 | INFO | "gpu" : " 17 GB", | 20260507 01:56:46.276 | INFO | "huge" : "", | 20260507 01:56:46.276 | INFO | "huge_shared" : " 27 GB", | 20260507 01:56:46.276 | INFO | "malloc" : " 14 GB", | 20260507 01:56:46.276 | INFO | "pinned" : " 29 GB" | 20260507 01:56:46.276 | INFO | } | 20260507 01:56:46.276 | INFO | Perf report for the last 5.14108 seconds: | 20260507 01:56:46.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:56:46.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:46.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:46.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:46.276 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 01:56:46.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:46.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:46.276 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 01:56:46.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:46.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:46.276 | INFO | Uploaded 36GiB to GPU in 2.04077s (17.6404GiB/s) | 20260507 01:56:46.276 | INFO | Downloaded 34GiB to GPU in 1.95921s (17.3539GiB/s) | 20260507 01:56:51.419 | INFO | Analyzing chunk frames = [6188032,6188544) | 20260507 01:56:51.419 | INFO | MemInfo: { | 20260507 01:56:51.419 | INFO | "gpu" : " 17 GB", | 20260507 01:56:51.419 | INFO | "huge" : "", | 20260507 01:56:51.419 | INFO | "huge_shared" : " 27 GB", | 20260507 01:56:51.419 | INFO | "malloc" : " 14 GB", | 20260507 01:56:51.419 | INFO | "pinned" : " 29 GB" | 20260507 01:56:51.419 | INFO | } | 20260507 01:56:51.419 | INFO | Perf report for the last 5.14319 seconds: | 20260507 01:56:51.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:56:51.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:51.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:51.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:51.419 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 01:56:51.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:51.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:51.419 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 01:56:51.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:51.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:51.419 | INFO | Uploaded 36GiB to GPU in 1.96231s (18.3457GiB/s) | 20260507 01:56:51.419 | INFO | Downloaded 34GiB to GPU in 1.98216s (17.153GiB/s) | 20260507 01:56:56.561 | INFO | Analyzing chunk frames = [6188544,6189056) | 20260507 01:56:56.561 | INFO | MemInfo: { | 20260507 01:56:56.561 | INFO | "gpu" : " 17 GB", | 20260507 01:56:56.561 | INFO | "huge" : "", | 20260507 01:56:56.561 | INFO | "huge_shared" : " 27 GB", | 20260507 01:56:56.561 | INFO | "malloc" : " 14 GB", | 20260507 01:56:56.561 | INFO | "pinned" : " 29 GB" | 20260507 01:56:56.561 | INFO | } | 20260507 01:56:56.561 | INFO | Perf report for the last 5.14178 seconds: | 20260507 01:56:56.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:56:56.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:56.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:56.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:56.561 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:56:56.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:56.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:56.561 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 01:56:56.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:56.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:56:56.561 | INFO | Uploaded 36GiB to GPU in 1.99747s (18.0228GiB/s) | 20260507 01:56:56.561 | INFO | Downloaded 34GiB to GPU in 1.96812s (17.2754GiB/s) | 20260507 01:56:56.670 | INFO | Starting a round of Baz IO | 20260507 01:56:59.770 | INFO | Populating RTMetrics report | 20260507 01:56:59.770 | INFO | RTMetrics report populated | 20260507 01:56:59.770 | INFO | Opening tmp RTMetrics json file | 20260507 01:56:59.771 | INFO | Serializing RTMetrics to json | 20260507 01:56:59.772 | INFO | Serialization done, swapping tmp file now... | 20260507 01:56:59.772 | INFO | RTMetrics done | 20260507 01:56:59.860 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.189 seconds | 20260507 01:56:59.860 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.07% / 0.00% / 66.93% | 20260507 01:56:59.860 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.261927% / 3.97364e-06% / 9.53674e-05% | 20260507 01:56:59.860 | INFO | Baz IO overall: | 20260507 01:56:59.860 | INFO | BytesWritten : 3202605412096 | 20260507 01:56:59.860 | INFO | NumEvents : 1540862811305 | 20260507 01:57:01.700 | INFO | Analyzing chunk frames = [6189056,6189568) | 20260507 01:57:01.700 | INFO | MemInfo: { | 20260507 01:57:01.700 | INFO | "gpu" : " 17 GB", | 20260507 01:57:01.700 | INFO | "huge" : "", | 20260507 01:57:01.700 | INFO | "huge_shared" : " 27 GB", | 20260507 01:57:01.700 | INFO | "malloc" : " 14 GB", | 20260507 01:57:01.700 | INFO | "pinned" : " 29 GB" | 20260507 01:57:01.700 | INFO | } | 20260507 01:57:01.700 | INFO | Perf report for the last 5.13917 seconds: | 20260507 01:57:01.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:57:01.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:01.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:01.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:01.700 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 01:57:01.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:01.700 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 01:57:01.700 | INFO | PRE_HQ : 59.6%, 1.8, 56.5ms, 0 | 20260507 01:57:01.700 | INFO | BAZWRITER : 62.1%, 3.3, 105.5ms, 0 | 20260507 01:57:01.700 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:57:01.700 | INFO | Uploaded 36GiB to GPU in 2.04869s (17.5722GiB/s) | 20260507 01:57:01.700 | INFO | Downloaded 38GiB to GPU in 2.17941s (17.4359GiB/s) | 20260507 01:57:06.840 | INFO | Analyzing chunk frames = [6189568,6190080) | 20260507 01:57:06.840 | INFO | MemInfo: { | 20260507 01:57:06.840 | INFO | "gpu" : " 17 GB", | 20260507 01:57:06.840 | INFO | "huge" : "", | 20260507 01:57:06.840 | INFO | "huge_shared" : " 27 GB", | 20260507 01:57:06.840 | INFO | "malloc" : " 14 GB", | 20260507 01:57:06.840 | INFO | "pinned" : " 29 GB" | 20260507 01:57:06.840 | INFO | } | 20260507 01:57:06.840 | INFO | Perf report for the last 5.14037 seconds: | 20260507 01:57:06.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:57:06.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:06.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:06.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:06.840 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:57:06.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:06.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:06.840 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:57:06.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:06.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:06.841 | INFO | Uploaded 36GiB to GPU in 2.04865s (17.5726GiB/s) | 20260507 01:57:06.841 | INFO | Downloaded 34GiB to GPU in 1.93878s (17.5368GiB/s) | 20260507 01:57:11.973 | INFO | Analyzing chunk frames = [6190080,6190592) | 20260507 01:57:11.974 | INFO | MemInfo: { | 20260507 01:57:11.974 | INFO | "gpu" : " 17 GB", | 20260507 01:57:11.974 | INFO | "huge" : "", | 20260507 01:57:11.974 | INFO | "huge_shared" : " 27 GB", | 20260507 01:57:11.974 | INFO | "malloc" : " 14 GB", | 20260507 01:57:11.974 | INFO | "pinned" : " 29 GB" | 20260507 01:57:11.974 | INFO | } | 20260507 01:57:11.974 | INFO | Perf report for the last 5.13328 seconds: | 20260507 01:57:11.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:57:11.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:11.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:11.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:11.974 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 01:57:11.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:11.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:11.974 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:57:11.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:11.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:11.974 | INFO | Uploaded 36GiB to GPU in 2.01647s (17.853GiB/s) | 20260507 01:57:11.974 | INFO | Downloaded 34GiB to GPU in 1.93689s (17.5539GiB/s) | 20260507 01:57:17.107 | INFO | Analyzing chunk frames = [6190592,6191104) | 20260507 01:57:17.107 | INFO | MemInfo: { | 20260507 01:57:17.107 | INFO | "gpu" : " 17 GB", | 20260507 01:57:17.107 | INFO | "huge" : "", | 20260507 01:57:17.107 | INFO | "huge_shared" : " 27 GB", | 20260507 01:57:17.107 | INFO | "malloc" : " 14 GB", | 20260507 01:57:17.107 | INFO | "pinned" : " 29 GB" | 20260507 01:57:17.107 | INFO | } | 20260507 01:57:17.107 | INFO | Perf report for the last 5.13369 seconds: | 20260507 01:57:17.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:57:17.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:17.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:17.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:17.107 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 01:57:17.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:17.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:17.107 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:57:17.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:17.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:17.108 | INFO | Uploaded 36GiB to GPU in 1.97799s (18.2003GiB/s) | 20260507 01:57:17.108 | INFO | Downloaded 34GiB to GPU in 1.90842s (17.8157GiB/s) | 20260507 01:57:22.253 | INFO | Analyzing chunk frames = [6191104,6191616) | 20260507 01:57:22.253 | INFO | MemInfo: { | 20260507 01:57:22.253 | INFO | "gpu" : " 17 GB", | 20260507 01:57:22.253 | INFO | "huge" : "", | 20260507 01:57:22.253 | INFO | "huge_shared" : " 27 GB", | 20260507 01:57:22.253 | INFO | "malloc" : " 14 GB", | 20260507 01:57:22.253 | INFO | "pinned" : " 29 GB" | 20260507 01:57:22.253 | INFO | } | 20260507 01:57:22.253 | INFO | Perf report for the last 5.14593 seconds: | 20260507 01:57:22.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:57:22.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:22.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:22.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:22.253 | INFO | ANALYSIS : 59.9%, 3.0, 93.4ms, 0 | 20260507 01:57:22.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:22.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:22.253 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 01:57:22.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:22.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:22.254 | INFO | Uploaded 36GiB to GPU in 1.99331s (18.0604GiB/s) | 20260507 01:57:22.254 | INFO | Downloaded 34GiB to GPU in 1.92491s (17.6632GiB/s) | 20260507 01:57:27.392 | INFO | Analyzing chunk frames = [6191616,6192128) | 20260507 01:57:27.392 | INFO | MemInfo: { | 20260507 01:57:27.392 | INFO | "gpu" : " 17 GB", | 20260507 01:57:27.392 | INFO | "huge" : "", | 20260507 01:57:27.392 | INFO | "huge_shared" : " 27 GB", | 20260507 01:57:27.392 | INFO | "malloc" : " 14 GB", | 20260507 01:57:27.392 | INFO | "pinned" : " 29 GB" | 20260507 01:57:27.392 | INFO | } | 20260507 01:57:27.392 | INFO | Perf report for the last 5.1387 seconds: | 20260507 01:57:27.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:57:27.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:27.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:27.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:27.392 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 01:57:27.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:27.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:27.392 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 01:57:27.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:27.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:27.392 | INFO | Uploaded 36GiB to GPU in 2.00048s (17.9957GiB/s) | 20260507 01:57:27.392 | INFO | Downloaded 34GiB to GPU in 1.96613s (17.2928GiB/s) | 20260507 01:57:32.532 | INFO | Analyzing chunk frames = [6192128,6192640) | 20260507 01:57:32.532 | INFO | MemInfo: { | 20260507 01:57:32.532 | INFO | "gpu" : " 17 GB", | 20260507 01:57:32.532 | INFO | "huge" : "", | 20260507 01:57:32.532 | INFO | "huge_shared" : " 27 GB", | 20260507 01:57:32.532 | INFO | "malloc" : " 14 GB", | 20260507 01:57:32.532 | INFO | "pinned" : " 29 GB" | 20260507 01:57:32.532 | INFO | } | 20260507 01:57:32.532 | INFO | Perf report for the last 5.14032 seconds: | 20260507 01:57:32.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:57:32.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:32.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:32.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:32.532 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 01:57:32.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:32.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:32.532 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 01:57:32.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:32.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:32.533 | INFO | Uploaded 36GiB to GPU in 1.97197s (18.2559GiB/s) | 20260507 01:57:32.533 | INFO | Downloaded 34GiB to GPU in 1.96541s (17.2992GiB/s) | 20260507 01:57:37.673 | INFO | Analyzing chunk frames = [6192640,6193152) | 20260507 01:57:37.673 | INFO | MemInfo: { | 20260507 01:57:37.673 | INFO | "gpu" : " 17 GB", | 20260507 01:57:37.673 | INFO | "huge" : "", | 20260507 01:57:37.673 | INFO | "huge_shared" : " 27 GB", | 20260507 01:57:37.673 | INFO | "malloc" : " 14 GB", | 20260507 01:57:37.673 | INFO | "pinned" : " 29 GB" | 20260507 01:57:37.673 | INFO | } | 20260507 01:57:37.673 | INFO | Perf report for the last 5.14098 seconds: | 20260507 01:57:37.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:57:37.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:37.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:37.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:37.673 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 01:57:37.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:37.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:37.673 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 01:57:37.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:37.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:37.674 | INFO | Uploaded 36GiB to GPU in 2.01049s (17.9061GiB/s) | 20260507 01:57:37.674 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:57:37.674 | INFO | Downloaded 34GiB to GPU in 1.97101s (17.25GiB/s) | 20260507 01:57:37.674 | INFO | Total mem usage: 18 GiB | 20260507 01:57:37.674 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:57:37.674 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 01:57:37.674 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:57:37.674 | INFO | High Water Breakdown: | 20260507 01:57:37.674 | INFO | 60.2% : 8.58 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:57:37.674 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:57:37.674 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:57:37.674 | INFO | ------------------------------------------ | 20260507 01:57:37.674 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:57:37.674 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:57:37.674 | INFO | Total mem usage: 26.5 GiB | 20260507 01:57:37.674 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:57:37.674 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:57:37.674 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:57:37.674 | INFO | High Water Breakdown: | 20260507 01:57:37.674 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:57:37.674 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:57:37.674 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:57:37.674 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:57:37.674 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:57:37.674 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:57:37.674 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:57:37.674 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:57:37.674 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:57:37.674 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:57:37.674 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:57:37.674 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:57:37.674 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:57:37.674 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:57:37.674 | INFO | ------------------------------------------ | 20260507 01:57:37.674 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:57:37.674 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:57:37.674 | INFO | Total mem usage: 25.6 GiB | 20260507 01:57:37.674 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:57:37.674 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:57:37.674 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:57:37.674 | INFO | High Water Breakdown: | 20260507 01:57:37.674 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:57:37.674 | INFO | ------------------------------------------ | 20260507 01:57:37.674 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:57:37.674 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:57:37.674 | INFO | Total mem usage: 15.7 GiB | 20260507 01:57:37.674 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:57:37.674 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:57:37.674 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:57:37.674 | INFO | High Water Breakdown: | 20260507 01:57:37.674 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:57:37.674 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:57:37.674 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:57:37.674 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:57:37.674 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:57:37.674 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:57:37.674 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:57:37.674 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:57:37.674 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:57:37.674 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:57:37.674 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:57:37.674 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:57:37.674 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:57:37.674 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:57:37.674 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:57:37.674 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:57:37.674 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:57:37.674 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:57:37.674 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:57:37.674 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:57:37.674 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:57:37.674 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:57:37.674 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:57:37.674 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:57:37.674 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:57:37.674 | INFO | ------------------------------------------ | 20260507 01:57:37.674 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:57:37.674 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:57:37.674 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:57:37.674 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:57:37.674 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.53e+01, 1.35e+01 / 1.62e+00 | 20260507 01:57:37.674 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.00e-01, 1.02e-02 / 6.38e-03 | 20260507 01:57:37.674 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.36e+00, 5.34e-01 / 3.75e-01 | 20260507 01:57:37.674 | INFO | PerfReport: 3.24%, Histogram -- 9.99e-01 / 1.42e+00, 1.72e-01 / 1.42e-01 | 20260507 01:57:37.674 | INFO | PerfReport: 16.05%, DME -- 3.96e+01 / 4.40e+01, 3.92e+00 / 1.91e+00 | 20260507 01:57:37.674 | INFO | PerfReport: 57.61%, FrameLabeling -- 1.78e+01 / 2.11e+01, 1.36e+00 / 1.55e+00 | 20260507 01:57:37.674 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.16e+00 / 2.63e+00, 3.77e-01 / 2.64e-01 | 20260507 01:57:37.674 | INFO | PerfReport: 3.09%, Metrics -- 9.53e-01 / 1.25e+00, 2.52e-01 / 1.15e-01 | 20260507 01:57:37.675 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.25e-01 / 3.76e-01, 5.31e-02 / 2.26e-02 | 20260507 01:57:37.675 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:57:37.675 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:57:37.675 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:57:37.675 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260507 01:57:37.675 | INFO | PerfReport: 21.64%, Upload -- 2.08e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260507 01:57:37.675 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260507 01:57:37.675 | INFO | PerfReport: 32.02%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 01:57:37.675 | INFO | PerfReport: 1.36%, LockUpload -- 1.31e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 01:57:37.675 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.18e+00 / 0.00e+00 | 20260507 01:57:37.675 | INFO | PerfReport: 22.28%, LockCompute -- 2.15e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 01:57:37.675 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260507 01:57:37.770 | INFO | Starting a round of Baz IO | 20260507 01:57:40.882 | INFO | Populating RTMetrics report | 20260507 01:57:40.882 | INFO | RTMetrics report populated | 20260507 01:57:40.882 | INFO | Opening tmp RTMetrics json file | 20260507 01:57:40.883 | INFO | Serializing RTMetrics to json | 20260507 01:57:40.884 | INFO | Serialization done, swapping tmp file now... | 20260507 01:57:40.885 | INFO | RTMetrics done | 20260507 01:57:40.970 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.199 seconds | 20260507 01:57:40.970 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.07% / 0.00% / 66.93% | 20260507 01:57:40.970 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.260202% / 0% / 6.75519e-05% | 20260507 01:57:40.970 | INFO | Baz IO overall: | 20260507 01:57:40.970 | INFO | BytesWritten : 3204536026807 | 20260507 01:57:40.970 | INFO | NumEvents : 1541777013172 | 20260507 01:57:42.815 | INFO | Analyzing chunk frames = [6193152,6193664) | 20260507 01:57:42.815 | INFO | MemInfo: { | 20260507 01:57:42.815 | INFO | "gpu" : " 17 GB", | 20260507 01:57:42.815 | INFO | "huge" : "", | 20260507 01:57:42.815 | INFO | "huge_shared" : " 27 GB", | 20260507 01:57:42.815 | INFO | "malloc" : " 14 GB", | 20260507 01:57:42.815 | INFO | "pinned" : " 29 GB" | 20260507 01:57:42.815 | INFO | } | 20260507 01:57:42.815 | INFO | Perf report for the last 5.14136 seconds: | 20260507 01:57:42.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:57:42.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:42.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:42.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:42.815 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 01:57:42.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:42.815 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260507 01:57:42.815 | INFO | PRE_HQ : 60.1%, 1.9, 59.8ms, 0 | 20260507 01:57:42.815 | INFO | BAZWRITER : 61.8%, 3.4, 109.4ms, 0 | 20260507 01:57:42.815 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:57:42.815 | INFO | Uploaded 36GiB to GPU in 2.03084s (17.7267GiB/s) | 20260507 01:57:42.815 | INFO | Downloaded 38GiB to GPU in 2.1691s (17.5188GiB/s) | 20260507 01:57:47.945 | INFO | Analyzing chunk frames = [6193664,6194176) | 20260507 01:57:47.945 | INFO | MemInfo: { | 20260507 01:57:47.945 | INFO | "gpu" : " 17 GB", | 20260507 01:57:47.945 | INFO | "huge" : "", | 20260507 01:57:47.945 | INFO | "huge_shared" : " 27 GB", | 20260507 01:57:47.945 | INFO | "malloc" : " 14 GB", | 20260507 01:57:47.945 | INFO | "pinned" : " 29 GB" | 20260507 01:57:47.945 | INFO | } | 20260507 01:57:47.945 | INFO | Perf report for the last 5.13045 seconds: | 20260507 01:57:47.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:57:47.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:47.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:47.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:47.945 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 01:57:47.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:47.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:47.945 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:57:47.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:47.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:47.945 | INFO | Uploaded 36GiB to GPU in 2.04022s (17.6452GiB/s) | 20260507 01:57:47.946 | INFO | Downloaded 34GiB to GPU in 1.93896s (17.5352GiB/s) | 20260507 01:57:53.082 | INFO | Analyzing chunk frames = [6194176,6194688) | 20260507 01:57:53.082 | INFO | MemInfo: { | 20260507 01:57:53.082 | INFO | "gpu" : " 17 GB", | 20260507 01:57:53.082 | INFO | "huge" : "", | 20260507 01:57:53.082 | INFO | "huge_shared" : " 27 GB", | 20260507 01:57:53.082 | INFO | "malloc" : " 14 GB", | 20260507 01:57:53.082 | INFO | "pinned" : " 29 GB" | 20260507 01:57:53.082 | INFO | } | 20260507 01:57:53.083 | INFO | Perf report for the last 5.13732 seconds: | 20260507 01:57:53.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:57:53.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:53.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:53.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:53.083 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 01:57:53.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:53.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:53.083 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:57:53.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:53.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:53.083 | INFO | Uploaded 36GiB to GPU in 2.00526s (17.9528GiB/s) | 20260507 01:57:53.083 | INFO | Downloaded 34GiB to GPU in 1.93404s (17.5798GiB/s) | 20260507 01:57:58.229 | INFO | Analyzing chunk frames = [6194688,6195200) | 20260507 01:57:58.229 | INFO | MemInfo: { | 20260507 01:57:58.229 | INFO | "gpu" : " 17 GB", | 20260507 01:57:58.229 | INFO | "huge" : "", | 20260507 01:57:58.229 | INFO | "huge_shared" : " 27 GB", | 20260507 01:57:58.229 | INFO | "malloc" : " 14 GB", | 20260507 01:57:58.229 | INFO | "pinned" : " 29 GB" | 20260507 01:57:58.229 | INFO | } | 20260507 01:57:58.229 | INFO | Perf report for the last 5.14642 seconds: | 20260507 01:57:58.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:57:58.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:58.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:58.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:58.229 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260507 01:57:58.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:58.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:58.229 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 01:57:58.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:58.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:57:58.230 | INFO | Uploaded 36GiB to GPU in 2.00004s (17.9997GiB/s) | 20260507 01:57:58.230 | INFO | Downloaded 34GiB to GPU in 1.91233s (17.7794GiB/s) | 20260507 01:58:03.363 | INFO | Analyzing chunk frames = [6195200,6195712) | 20260507 01:58:03.363 | INFO | MemInfo: { | 20260507 01:58:03.363 | INFO | "gpu" : " 17 GB", | 20260507 01:58:03.363 | INFO | "huge" : "", | 20260507 01:58:03.363 | INFO | "huge_shared" : " 27 GB", | 20260507 01:58:03.363 | INFO | "malloc" : " 14 GB", | 20260507 01:58:03.363 | INFO | "pinned" : " 29 GB" | 20260507 01:58:03.363 | INFO | } | 20260507 01:58:03.363 | INFO | Perf report for the last 5.13402 seconds: | 20260507 01:58:03.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:58:03.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:03.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:03.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:03.363 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 01:58:03.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:03.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:03.363 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:58:03.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:03.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:03.363 | INFO | Uploaded 36GiB to GPU in 2.01764s (17.8426GiB/s) | 20260507 01:58:03.363 | INFO | Downloaded 34GiB to GPU in 1.95205s (17.4176GiB/s) | 20260507 01:58:08.513 | INFO | Analyzing chunk frames = [6195712,6196224) | 20260507 01:58:08.513 | INFO | MemInfo: { | 20260507 01:58:08.513 | INFO | "gpu" : " 17 GB", | 20260507 01:58:08.513 | INFO | "huge" : "", | 20260507 01:58:08.513 | INFO | "huge_shared" : " 27 GB", | 20260507 01:58:08.513 | INFO | "malloc" : " 14 GB", | 20260507 01:58:08.513 | INFO | "pinned" : " 29 GB" | 20260507 01:58:08.513 | INFO | } | 20260507 01:58:08.513 | INFO | Perf report for the last 5.14994 seconds: | 20260507 01:58:08.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:58:08.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:08.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:08.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:08.513 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260507 01:58:08.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:08.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:08.513 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:58:08.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:08.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:08.513 | INFO | Uploaded 36GiB to GPU in 2.05757s (17.4964GiB/s) | 20260507 01:58:08.513 | INFO | Downloaded 34GiB to GPU in 1.95252s (17.4134GiB/s) | 20260507 01:58:13.647 | INFO | Analyzing chunk frames = [6196224,6196736) | 20260507 01:58:13.648 | INFO | MemInfo: { | 20260507 01:58:13.648 | INFO | "gpu" : " 17 GB", | 20260507 01:58:13.648 | INFO | "huge" : "", | 20260507 01:58:13.648 | INFO | "huge_shared" : " 27 GB", | 20260507 01:58:13.648 | INFO | "malloc" : " 14 GB", | 20260507 01:58:13.648 | INFO | "pinned" : " 29 GB" | 20260507 01:58:13.648 | INFO | } | 20260507 01:58:13.648 | INFO | Perf report for the last 5.13476 seconds: | 20260507 01:58:13.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:58:13.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:13.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:13.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:13.648 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 01:58:13.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:13.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:13.648 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 01:58:13.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:13.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:13.648 | INFO | Uploaded 36GiB to GPU in 1.97294s (18.2468GiB/s) | 20260507 01:58:13.648 | INFO | Downloaded 34GiB to GPU in 1.97262s (17.236GiB/s) | 20260507 01:58:18.784 | INFO | Analyzing chunk frames = [6196736,6197248) | 20260507 01:58:18.784 | INFO | MemInfo: { | 20260507 01:58:18.784 | INFO | "gpu" : " 17 GB", | 20260507 01:58:18.784 | INFO | "huge" : "", | 20260507 01:58:18.784 | INFO | "huge_shared" : " 27 GB", | 20260507 01:58:18.784 | INFO | "malloc" : " 14 GB", | 20260507 01:58:18.784 | INFO | "pinned" : " 29 GB" | 20260507 01:58:18.784 | INFO | } | 20260507 01:58:18.784 | INFO | Perf report for the last 5.13614 seconds: | 20260507 01:58:18.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:58:18.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:18.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:18.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:18.784 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:58:18.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:18.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:18.784 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 01:58:18.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:18.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:18.784 | INFO | Uploaded 36GiB to GPU in 1.98868s (18.1025GiB/s) | 20260507 01:58:18.784 | INFO | Downloaded 34GiB to GPU in 1.97212s (17.2403GiB/s) | 20260507 01:58:18.881 | INFO | Starting a round of Baz IO | 20260507 01:58:21.990 | INFO | Populating RTMetrics report | 20260507 01:58:21.991 | INFO | RTMetrics report populated | 20260507 01:58:21.991 | INFO | Opening tmp RTMetrics json file | 20260507 01:58:21.991 | INFO | Serializing RTMetrics to json | 20260507 01:58:21.992 | INFO | Serialization done, swapping tmp file now... | 20260507 01:58:21.993 | INFO | RTMetrics done | 20260507 01:58:22.079 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.198 seconds | 20260507 01:58:22.079 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.06% / 0.00% / 66.93% | 20260507 01:58:22.079 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.259952% / 7.94729e-06% / 9.93411e-05% | 20260507 01:58:22.079 | INFO | Baz IO overall: | 20260507 01:58:22.079 | INFO | BytesWritten : 3206466949766 | 20260507 01:58:22.080 | INFO | NumEvents : 1542691639002 | 20260507 01:58:23.920 | INFO | Analyzing chunk frames = [6197248,6197760) | 20260507 01:58:23.920 | INFO | MemInfo: { | 20260507 01:58:23.920 | INFO | "gpu" : " 17 GB", | 20260507 01:58:23.920 | INFO | "huge" : "", | 20260507 01:58:23.920 | INFO | "huge_shared" : " 27 GB", | 20260507 01:58:23.920 | INFO | "malloc" : " 14 GB", | 20260507 01:58:23.920 | INFO | "pinned" : " 29 GB" | 20260507 01:58:23.920 | INFO | } | 20260507 01:58:23.920 | INFO | Perf report for the last 5.13639 seconds: | 20260507 01:58:23.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:58:23.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:23.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:23.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:23.920 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 01:58:23.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:23.920 | INFO | RT_METRICS : 41.9%, 1.0, 21.9ms, 0 | 20260507 01:58:23.920 | INFO | PRE_HQ : 59.3%, 1.8, 55.8ms, 0 | 20260507 01:58:23.920 | INFO | BAZWRITER : 61.9%, 3.3, 105.1ms, 0 | 20260507 01:58:23.920 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:58:23.921 | INFO | Uploaded 36GiB to GPU in 2.02014s (17.8206GiB/s) | 20260507 01:58:23.921 | INFO | Downloaded 38GiB to GPU in 2.18246s (17.4115GiB/s) | 20260507 01:58:29.059 | INFO | Analyzing chunk frames = [6197760,6198272) | 20260507 01:58:29.059 | INFO | MemInfo: { | 20260507 01:58:29.059 | INFO | "gpu" : " 17 GB", | 20260507 01:58:29.059 | INFO | "huge" : "", | 20260507 01:58:29.059 | INFO | "huge_shared" : " 27 GB", | 20260507 01:58:29.059 | INFO | "malloc" : " 14 GB", | 20260507 01:58:29.059 | INFO | "pinned" : " 29 GB" | 20260507 01:58:29.059 | INFO | } | 20260507 01:58:29.059 | INFO | Perf report for the last 5.13876 seconds: | 20260507 01:58:29.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:58:29.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:29.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:29.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:29.059 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:58:29.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:29.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:29.059 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 01:58:29.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:29.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:29.059 | INFO | Uploaded 36GiB to GPU in 2.00213s (17.9809GiB/s) | 20260507 01:58:29.059 | INFO | Downloaded 34GiB to GPU in 1.93447s (17.5758GiB/s) | 20260507 01:58:34.202 | INFO | Analyzing chunk frames = [6198272,6198784) | 20260507 01:58:34.202 | INFO | MemInfo: { | 20260507 01:58:34.202 | INFO | "gpu" : " 17 GB", | 20260507 01:58:34.202 | INFO | "huge" : "", | 20260507 01:58:34.202 | INFO | "huge_shared" : " 27 GB", | 20260507 01:58:34.202 | INFO | "malloc" : " 14 GB", | 20260507 01:58:34.202 | INFO | "pinned" : " 29 GB" | 20260507 01:58:34.202 | INFO | } | 20260507 01:58:34.202 | INFO | Perf report for the last 5.14279 seconds: | 20260507 01:58:34.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:58:34.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:34.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:34.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:34.202 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 01:58:34.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:34.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:34.202 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:58:34.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:34.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:34.203 | INFO | Uploaded 36GiB to GPU in 1.99073s (18.0839GiB/s) | 20260507 01:58:34.203 | INFO | Downloaded 34GiB to GPU in 1.93413s (17.579GiB/s) | 20260507 01:58:39.343 | INFO | Analyzing chunk frames = [6198784,6199296) | 20260507 01:58:39.343 | INFO | MemInfo: { | 20260507 01:58:39.343 | INFO | "gpu" : " 17 GB", | 20260507 01:58:39.343 | INFO | "huge" : "", | 20260507 01:58:39.343 | INFO | "huge_shared" : " 27 GB", | 20260507 01:58:39.343 | INFO | "malloc" : " 14 GB", | 20260507 01:58:39.343 | INFO | "pinned" : " 29 GB" | 20260507 01:58:39.343 | INFO | } | 20260507 01:58:39.343 | INFO | Perf report for the last 5.14115 seconds: | 20260507 01:58:39.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:58:39.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:39.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:39.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:39.343 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 01:58:39.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:39.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:39.343 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:58:39.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:39.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:39.344 | INFO | Uploaded 36GiB to GPU in 1.96278s (18.3413GiB/s) | 20260507 01:58:39.344 | INFO | Downloaded 34GiB to GPU in 1.90957s (17.8051GiB/s) | 20260507 01:58:44.483 | INFO | Analyzing chunk frames = [6199296,6199808) | 20260507 01:58:44.483 | INFO | MemInfo: { | 20260507 01:58:44.483 | INFO | "gpu" : " 17 GB", | 20260507 01:58:44.483 | INFO | "huge" : "", | 20260507 01:58:44.483 | INFO | "huge_shared" : " 27 GB", | 20260507 01:58:44.483 | INFO | "malloc" : " 14 GB", | 20260507 01:58:44.483 | INFO | "pinned" : " 29 GB" | 20260507 01:58:44.483 | INFO | } | 20260507 01:58:44.483 | INFO | Perf report for the last 5.1403 seconds: | 20260507 01:58:44.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:58:44.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:44.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:44.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:44.483 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 01:58:44.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:44.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:44.483 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 01:58:44.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:44.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:44.484 | INFO | Uploaded 36GiB to GPU in 1.99145s (18.0773GiB/s) | 20260507 01:58:44.484 | INFO | Downloaded 34GiB to GPU in 1.94141s (17.5131GiB/s) | 20260507 01:58:49.627 | INFO | Analyzing chunk frames = [6199808,6200320) | 20260507 01:58:49.627 | INFO | MemInfo: { | 20260507 01:58:49.627 | INFO | "gpu" : " 17 GB", | 20260507 01:58:49.627 | INFO | "huge" : "", | 20260507 01:58:49.627 | INFO | "huge_shared" : " 27 GB", | 20260507 01:58:49.627 | INFO | "malloc" : " 14 GB", | 20260507 01:58:49.627 | INFO | "pinned" : " 29 GB" | 20260507 01:58:49.627 | INFO | } | 20260507 01:58:49.627 | INFO | Perf report for the last 5.14371 seconds: | 20260507 01:58:49.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:58:49.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:49.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:49.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:49.627 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 01:58:49.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:49.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:49.627 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:58:49.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:49.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:49.628 | INFO | Uploaded 36GiB to GPU in 2.04797s (17.5784GiB/s) | 20260507 01:58:49.628 | INFO | Downloaded 34GiB to GPU in 1.96785s (17.2777GiB/s) | 20260507 01:58:54.758 | INFO | Analyzing chunk frames = [6200320,6200832) | 20260507 01:58:54.758 | INFO | MemInfo: { | 20260507 01:58:54.758 | INFO | "gpu" : " 17 GB", | 20260507 01:58:54.758 | INFO | "huge" : "", | 20260507 01:58:54.758 | INFO | "huge_shared" : " 27 GB", | 20260507 01:58:54.758 | INFO | "malloc" : " 14 GB", | 20260507 01:58:54.758 | INFO | "pinned" : " 29 GB" | 20260507 01:58:54.758 | INFO | } | 20260507 01:58:54.758 | INFO | Perf report for the last 5.13097 seconds: | 20260507 01:58:54.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:58:54.758 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:54.758 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:54.758 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:54.758 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 01:58:54.758 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:54.758 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:54.758 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 01:58:54.758 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:54.758 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:54.758 | INFO | Uploaded 36GiB to GPU in 1.96627s (18.3088GiB/s) | 20260507 01:58:54.758 | INFO | Downloaded 34GiB to GPU in 1.95804s (17.3643GiB/s) | 20260507 01:58:59.895 | INFO | Analyzing chunk frames = [6200832,6201344) | 20260507 01:58:59.895 | INFO | MemInfo: { | 20260507 01:58:59.895 | INFO | "gpu" : " 17 GB", | 20260507 01:58:59.895 | INFO | "huge" : "", | 20260507 01:58:59.895 | INFO | "huge_shared" : " 27 GB", | 20260507 01:58:59.895 | INFO | "malloc" : " 14 GB", | 20260507 01:58:59.895 | INFO | "pinned" : " 29 GB" | 20260507 01:58:59.895 | INFO | } | 20260507 01:58:59.895 | INFO | Perf report for the last 5.13744 seconds: | 20260507 01:58:59.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:58:59.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:59.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:59.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:59.896 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 01:58:59.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:59.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:59.896 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:58:59.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:59.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:58:59.896 | INFO | Uploaded 36GiB to GPU in 1.98985s (18.0918GiB/s) | 20260507 01:58:59.896 | INFO | Downloaded 34GiB to GPU in 1.98655s (17.1151GiB/s) | 20260507 01:58:59.896 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 01:58:59.896 | INFO | Total mem usage: 18 GiB | 20260507 01:58:59.896 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 01:58:59.896 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 01:58:59.896 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 01:58:59.896 | INFO | High Water Breakdown: | 20260507 01:58:59.896 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 01:58:59.896 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 01:58:59.896 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 01:58:59.896 | INFO | ------------------------------------------ | 20260507 01:58:59.896 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:58:59.896 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 01:58:59.896 | INFO | Total mem usage: 26.5 GiB | 20260507 01:58:59.896 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 01:58:59.896 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 01:58:59.896 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 01:58:59.896 | INFO | High Water Breakdown: | 20260507 01:58:59.896 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:58:59.896 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:58:59.896 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:58:59.896 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:58:59.896 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:58:59.896 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:58:59.896 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 01:58:59.896 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 01:58:59.896 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:58:59.896 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 01:58:59.896 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 01:58:59.896 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 01:58:59.896 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 01:58:59.896 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 01:58:59.896 | INFO | ------------------------------------------ | 20260507 01:58:59.896 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:58:59.896 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 01:58:59.896 | INFO | Total mem usage: 25.6 GiB | 20260507 01:58:59.896 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 01:58:59.896 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 01:58:59.896 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 01:58:59.896 | INFO | High Water Breakdown: | 20260507 01:58:59.896 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 01:58:59.896 | INFO | ------------------------------------------ | 20260507 01:58:59.896 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:58:59.896 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 01:58:59.896 | INFO | Total mem usage: 15.7 GiB | 20260507 01:58:59.896 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 01:58:59.896 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 01:58:59.896 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 01:58:59.896 | INFO | High Water Breakdown: | 20260507 01:58:59.896 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 01:58:59.896 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 01:58:59.896 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 01:58:59.896 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 01:58:59.896 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 01:58:59.896 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 01:58:59.896 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 01:58:59.896 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 01:58:59.896 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 01:58:59.896 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 01:58:59.896 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 01:58:59.896 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 01:58:59.896 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 01:58:59.896 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 01:58:59.896 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 01:58:59.896 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 01:58:59.896 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 01:58:59.896 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 01:58:59.896 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 01:58:59.896 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 01:58:59.896 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 01:58:59.896 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 01:58:59.896 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 01:58:59.896 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 01:58:59.896 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 01:58:59.896 | INFO | ------------------------------------------ | 20260507 01:58:59.896 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 01:58:59.897 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 01:58:59.897 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:58:59.897 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:58:59.897 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.49e+01, 1.31e+01 / 1.64e+00 | 20260507 01:58:59.897 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.02e-01, 9.62e-03 / 7.29e-03 | 20260507 01:58:59.897 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.36e+00, 5.20e-01 / 3.56e-01 | 20260507 01:58:59.897 | INFO | PerfReport: 3.24%, Histogram -- 9.98e-01 / 1.45e+00, 1.70e-01 / 1.26e-01 | 20260507 01:58:59.897 | INFO | PerfReport: 16.08%, DME -- 3.96e+01 / 4.39e+01, 3.85e+00 / 1.77e+00 | 20260507 01:58:59.897 | INFO | PerfReport: 57.55%, FrameLabeling -- 1.77e+01 / 2.07e+01, 1.31e+00 / 2.00e+00 | 20260507 01:58:59.897 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.16e+00 / 2.69e+00, 3.85e-01 / 2.46e-01 | 20260507 01:58:59.897 | INFO | PerfReport: 3.10%, Metrics -- 9.57e-01 / 1.28e+00, 2.58e-01 / 1.36e-01 | 20260507 01:58:59.897 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.24e-01 / 3.77e-01, 5.28e-02 / 2.01e-02 | 20260507 01:58:59.897 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 01:58:59.897 | INFO | PerfReport: Gathered 1536 samples. | 20260507 01:58:59.897 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 01:58:59.897 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 01:58:59.897 | INFO | PerfReport: 21.66%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260507 01:58:59.897 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 01:58:59.897 | INFO | PerfReport: 31.96%, Compute -- 3.08e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 01:58:59.897 | INFO | PerfReport: 1.41%, LockUpload -- 1.36e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 01:58:59.897 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260507 01:58:59.897 | INFO | PerfReport: 22.28%, LockCompute -- 2.15e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 01:58:59.897 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260507 01:58:59.993 | INFO | Starting a round of Baz IO | 20260507 01:59:03.110 | INFO | Populating RTMetrics report | 20260507 01:59:03.110 | INFO | RTMetrics report populated | 20260507 01:59:03.110 | INFO | Opening tmp RTMetrics json file | 20260507 01:59:03.125 | INFO | Serializing RTMetrics to json | 20260507 01:59:03.127 | INFO | Serialization done, swapping tmp file now... | 20260507 01:59:03.127 | INFO | RTMetrics done | 20260507 01:59:03.196 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.203 seconds | 20260507 01:59:03.196 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.06% / 0.00% / 66.94% | 20260507 01:59:03.196 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.260254% / 7.94729e-06% / 6.75519e-05% | 20260507 01:59:03.196 | INFO | Baz IO overall: | 20260507 01:59:03.196 | INFO | BytesWritten : 3208397938284 | 20260507 01:59:03.196 | INFO | NumEvents : 1543605988825 | 20260507 01:59:05.036 | INFO | Analyzing chunk frames = [6201344,6201856) | 20260507 01:59:05.036 | INFO | MemInfo: { | 20260507 01:59:05.036 | INFO | "gpu" : " 17 GB", | 20260507 01:59:05.036 | INFO | "huge" : "", | 20260507 01:59:05.036 | INFO | "huge_shared" : " 27 GB", | 20260507 01:59:05.036 | INFO | "malloc" : " 14 GB", | 20260507 01:59:05.036 | INFO | "pinned" : " 29 GB" | 20260507 01:59:05.036 | INFO | } | 20260507 01:59:05.036 | INFO | Perf report for the last 5.14075 seconds: | 20260507 01:59:05.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:59:05.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:05.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:05.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:05.036 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260507 01:59:05.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:05.036 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 01:59:05.036 | INFO | PRE_HQ : 59.4%, 1.8, 55.7ms, 0 | 20260507 01:59:05.036 | INFO | BAZWRITER : 61.9%, 3.3, 106.5ms, 0 | 20260507 01:59:05.036 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 01:59:05.036 | INFO | Uploaded 36GiB to GPU in 2.00587s (17.9474GiB/s) | 20260507 01:59:05.036 | INFO | Downloaded 38GiB to GPU in 2.17315s (17.4861GiB/s) | 20260507 01:59:10.179 | INFO | Analyzing chunk frames = [6201856,6202368) | 20260507 01:59:10.179 | INFO | MemInfo: { | 20260507 01:59:10.179 | INFO | "gpu" : " 17 GB", | 20260507 01:59:10.179 | INFO | "huge" : "", | 20260507 01:59:10.179 | INFO | "huge_shared" : " 27 GB", | 20260507 01:59:10.179 | INFO | "malloc" : " 14 GB", | 20260507 01:59:10.179 | INFO | "pinned" : " 29 GB" | 20260507 01:59:10.179 | INFO | } | 20260507 01:59:10.179 | INFO | Perf report for the last 5.14274 seconds: | 20260507 01:59:10.179 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:59:10.179 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:10.179 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:10.179 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:10.179 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 01:59:10.179 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:10.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:10.179 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 01:59:10.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:10.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:10.179 | INFO | Uploaded 36GiB to GPU in 2.0055s (17.9506GiB/s) | 20260507 01:59:10.179 | INFO | Downloaded 34GiB to GPU in 1.93291s (17.59GiB/s) | 20260507 01:59:15.316 | INFO | Analyzing chunk frames = [6202368,6202880) | 20260507 01:59:15.316 | INFO | MemInfo: { | 20260507 01:59:15.316 | INFO | "gpu" : " 17 GB", | 20260507 01:59:15.316 | INFO | "huge" : "", | 20260507 01:59:15.316 | INFO | "huge_shared" : " 27 GB", | 20260507 01:59:15.316 | INFO | "malloc" : " 14 GB", | 20260507 01:59:15.317 | INFO | "pinned" : " 29 GB" | 20260507 01:59:15.317 | INFO | } | 20260507 01:59:15.317 | INFO | Perf report for the last 5.13773 seconds: | 20260507 01:59:15.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:59:15.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:15.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:15.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:15.317 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 01:59:15.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:15.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:15.317 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:59:15.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:15.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:15.317 | INFO | Uploaded 36GiB to GPU in 1.98443s (18.1412GiB/s) | 20260507 01:59:15.317 | INFO | Downloaded 34GiB to GPU in 1.93818s (17.5423GiB/s) | 20260507 01:59:20.459 | INFO | Analyzing chunk frames = [6202880,6203392) | 20260507 01:59:20.459 | INFO | MemInfo: { | 20260507 01:59:20.459 | INFO | "gpu" : " 17 GB", | 20260507 01:59:20.459 | INFO | "huge" : "", | 20260507 01:59:20.459 | INFO | "huge_shared" : " 27 GB", | 20260507 01:59:20.459 | INFO | "malloc" : " 14 GB", | 20260507 01:59:20.459 | INFO | "pinned" : " 29 GB" | 20260507 01:59:20.459 | INFO | } | 20260507 01:59:20.459 | INFO | Perf report for the last 5.14222 seconds: | 20260507 01:59:20.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:59:20.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:20.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:20.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:20.459 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 01:59:20.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:20.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:20.459 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 01:59:20.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:20.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:20.459 | INFO | Uploaded 36GiB to GPU in 1.97148s (18.2604GiB/s) | 20260507 01:59:20.459 | INFO | Downloaded 34GiB to GPU in 1.91056s (17.7958GiB/s) | 20260507 01:59:25.599 | INFO | Analyzing chunk frames = [6203392,6203904) | 20260507 01:59:25.599 | INFO | MemInfo: { | 20260507 01:59:25.599 | INFO | "gpu" : " 17 GB", | 20260507 01:59:25.600 | INFO | "huge" : "", | 20260507 01:59:25.600 | INFO | "huge_shared" : " 27 GB", | 20260507 01:59:25.600 | INFO | "malloc" : " 14 GB", | 20260507 01:59:25.600 | INFO | "pinned" : " 29 GB" | 20260507 01:59:25.600 | INFO | } | 20260507 01:59:25.600 | INFO | Perf report for the last 5.14071 seconds: | 20260507 01:59:25.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:59:25.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:25.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:25.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:25.600 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 01:59:25.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:25.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:25.600 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 01:59:25.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:25.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:25.600 | INFO | Uploaded 36GiB to GPU in 2.01977s (17.8238GiB/s) | 20260507 01:59:25.600 | INFO | Downloaded 34GiB to GPU in 1.94256s (17.5027GiB/s) | 20260507 01:59:30.734 | INFO | Analyzing chunk frames = [6203904,6204416) | 20260507 01:59:30.734 | INFO | MemInfo: { | 20260507 01:59:30.734 | INFO | "gpu" : " 17 GB", | 20260507 01:59:30.734 | INFO | "huge" : "", | 20260507 01:59:30.734 | INFO | "huge_shared" : " 27 GB", | 20260507 01:59:30.734 | INFO | "malloc" : " 14 GB", | 20260507 01:59:30.734 | INFO | "pinned" : " 29 GB" | 20260507 01:59:30.734 | INFO | } | 20260507 01:59:30.734 | INFO | Perf report for the last 5.13478 seconds: | 20260507 01:59:30.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:59:30.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:30.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:30.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:30.735 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 01:59:30.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:30.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:30.735 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 01:59:30.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:30.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:30.735 | INFO | Uploaded 36GiB to GPU in 2.02895s (17.7431GiB/s) | 20260507 01:59:30.735 | INFO | Downloaded 34GiB to GPU in 1.92057s (17.7031GiB/s) | 20260507 01:59:35.875 | INFO | Analyzing chunk frames = [6204416,6204928) | 20260507 01:59:35.875 | INFO | MemInfo: { | 20260507 01:59:35.875 | INFO | "gpu" : " 17 GB", | 20260507 01:59:35.875 | INFO | "huge" : "", | 20260507 01:59:35.875 | INFO | "huge_shared" : " 27 GB", | 20260507 01:59:35.875 | INFO | "malloc" : " 14 GB", | 20260507 01:59:35.875 | INFO | "pinned" : " 29 GB" | 20260507 01:59:35.875 | INFO | } | 20260507 01:59:35.875 | INFO | Perf report for the last 5.14075 seconds: | 20260507 01:59:35.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:59:35.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:35.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:35.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:35.875 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:59:35.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:35.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:35.875 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 01:59:35.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:35.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:35.875 | INFO | Uploaded 36GiB to GPU in 1.9866s (18.1214GiB/s) | 20260507 01:59:35.875 | INFO | Downloaded 34GiB to GPU in 2.00225s (16.9809GiB/s) | 20260507 01:59:41.011 | INFO | Analyzing chunk frames = [6204928,6205440) | 20260507 01:59:41.011 | INFO | MemInfo: { | 20260507 01:59:41.011 | INFO | "gpu" : " 17 GB", | 20260507 01:59:41.011 | INFO | "huge" : "", | 20260507 01:59:41.011 | INFO | "huge_shared" : " 27 GB", | 20260507 01:59:41.011 | INFO | "malloc" : " 14 GB", | 20260507 01:59:41.011 | INFO | "pinned" : " 29 GB" | 20260507 01:59:41.011 | INFO | } | 20260507 01:59:41.011 | INFO | Perf report for the last 5.13598 seconds: | 20260507 01:59:41.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:59:41.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:41.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:41.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:41.011 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 01:59:41.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:41.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:41.011 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 01:59:41.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:41.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:41.011 | INFO | Uploaded 36GiB to GPU in 1.97339s (18.2427GiB/s) | 20260507 01:59:41.012 | INFO | Downloaded 34GiB to GPU in 1.9784s (17.1856GiB/s) | 20260507 01:59:41.109 | INFO | Starting a round of Baz IO | 20260507 01:59:44.231 | INFO | Populating RTMetrics report | 20260507 01:59:44.232 | INFO | RTMetrics report populated | 20260507 01:59:44.232 | INFO | Opening tmp RTMetrics json file | 20260507 01:59:44.235 | INFO | Serializing RTMetrics to json | 20260507 01:59:44.250 | INFO | Serialization done, swapping tmp file now... | 20260507 01:59:44.251 | INFO | RTMetrics done | 20260507 01:59:44.319 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.21 seconds | 20260507 01:59:44.319 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.06% / 0.00% / 66.94% | 20260507 01:59:44.319 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.260266% / 0% / 7.94729e-05% | 20260507 01:59:44.319 | INFO | Baz IO overall: | 20260507 01:59:44.319 | INFO | BytesWritten : 3210329356125 | 20260507 01:59:44.319 | INFO | NumEvents : 1544520708124 | 20260507 01:59:46.157 | INFO | Analyzing chunk frames = [6205440,6205952) | 20260507 01:59:46.157 | INFO | MemInfo: { | 20260507 01:59:46.157 | INFO | "gpu" : " 17 GB", | 20260507 01:59:46.157 | INFO | "huge" : "", | 20260507 01:59:46.157 | INFO | "huge_shared" : " 27 GB", | 20260507 01:59:46.157 | INFO | "malloc" : " 14 GB", | 20260507 01:59:46.157 | INFO | "pinned" : " 29 GB" | 20260507 01:59:46.157 | INFO | } | 20260507 01:59:46.157 | INFO | Perf report for the last 5.14584 seconds: | 20260507 01:59:46.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:59:46.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:46.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:46.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:46.157 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260507 01:59:46.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:46.157 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 01:59:46.157 | INFO | PRE_HQ : 59.5%, 1.8, 55.8ms, 0 | 20260507 01:59:46.157 | INFO | BAZWRITER : 62.0%, 3.3, 105.7ms, 0 | 20260507 01:59:46.157 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 01:59:46.157 | INFO | Uploaded 36GiB to GPU in 2.03147s (17.7211GiB/s) | 20260507 01:59:46.157 | INFO | Downloaded 38GiB to GPU in 2.18653s (17.3791GiB/s) | 20260507 01:59:51.297 | INFO | Analyzing chunk frames = [6205952,6206464) | 20260507 01:59:51.297 | INFO | MemInfo: { | 20260507 01:59:51.297 | INFO | "gpu" : " 17 GB", | 20260507 01:59:51.297 | INFO | "huge" : "", | 20260507 01:59:51.297 | INFO | "huge_shared" : " 27 GB", | 20260507 01:59:51.297 | INFO | "malloc" : " 14 GB", | 20260507 01:59:51.297 | INFO | "pinned" : " 29 GB" | 20260507 01:59:51.297 | INFO | } | 20260507 01:59:51.297 | INFO | Perf report for the last 5.14012 seconds: | 20260507 01:59:51.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:59:51.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:51.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:51.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:51.297 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 01:59:51.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:51.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:51.297 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 01:59:51.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:51.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:51.297 | INFO | Uploaded 36GiB to GPU in 2.04098s (17.6386GiB/s) | 20260507 01:59:51.297 | INFO | Downloaded 34GiB to GPU in 1.94342s (17.4949GiB/s) | 20260507 01:59:56.430 | INFO | Analyzing chunk frames = [6206464,6206976) | 20260507 01:59:56.430 | INFO | MemInfo: { | 20260507 01:59:56.430 | INFO | "gpu" : " 17 GB", | 20260507 01:59:56.430 | INFO | "huge" : "", | 20260507 01:59:56.430 | INFO | "huge_shared" : " 27 GB", | 20260507 01:59:56.430 | INFO | "malloc" : " 14 GB", | 20260507 01:59:56.430 | INFO | "pinned" : " 29 GB" | 20260507 01:59:56.430 | INFO | } | 20260507 01:59:56.430 | INFO | Perf report for the last 5.13292 seconds: | 20260507 01:59:56.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 01:59:56.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:56.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:56.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:56.430 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 01:59:56.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:56.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:56.430 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 01:59:56.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:56.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 01:59:56.430 | INFO | Uploaded 36GiB to GPU in 1.98752s (18.113GiB/s) | 20260507 01:59:56.430 | INFO | Downloaded 34GiB to GPU in 1.94173s (17.5102GiB/s) | 20260507 02:00:01.573 | INFO | Analyzing chunk frames = [6206976,6207488) | 20260507 02:00:01.573 | INFO | MemInfo: { | 20260507 02:00:01.573 | INFO | "gpu" : " 17 GB", | 20260507 02:00:01.573 | INFO | "huge" : "", | 20260507 02:00:01.573 | INFO | "huge_shared" : " 27 GB", | 20260507 02:00:01.573 | INFO | "malloc" : " 14 GB", | 20260507 02:00:01.573 | INFO | "pinned" : " 29 GB" | 20260507 02:00:01.573 | INFO | } | 20260507 02:00:01.573 | INFO | Perf report for the last 5.14254 seconds: | 20260507 02:00:01.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:00:01.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:01.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:01.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:01.573 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:00:01.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:01.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:01.573 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 02:00:01.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:01.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:01.573 | INFO | Uploaded 36GiB to GPU in 1.98517s (18.1345GiB/s) | 20260507 02:00:01.573 | INFO | Downloaded 34GiB to GPU in 1.90647s (17.834GiB/s) | 20260507 02:00:06.711 | INFO | Analyzing chunk frames = [6207488,6208000) | 20260507 02:00:06.711 | INFO | MemInfo: { | 20260507 02:00:06.711 | INFO | "gpu" : " 17 GB", | 20260507 02:00:06.711 | INFO | "huge" : "", | 20260507 02:00:06.711 | INFO | "huge_shared" : " 27 GB", | 20260507 02:00:06.711 | INFO | "malloc" : " 14 GB", | 20260507 02:00:06.711 | INFO | "pinned" : " 29 GB" | 20260507 02:00:06.711 | INFO | } | 20260507 02:00:06.711 | INFO | Perf report for the last 5.13823 seconds: | 20260507 02:00:06.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:00:06.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:06.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:06.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:06.711 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 02:00:06.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:06.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:06.711 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:00:06.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:06.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:06.711 | INFO | Uploaded 36GiB to GPU in 2.00272s (17.9756GiB/s) | 20260507 02:00:06.711 | INFO | Downloaded 34GiB to GPU in 1.94822s (17.4518GiB/s) | 20260507 02:00:11.848 | INFO | Analyzing chunk frames = [6208000,6208512) | 20260507 02:00:11.848 | INFO | MemInfo: { | 20260507 02:00:11.848 | INFO | "gpu" : " 17 GB", | 20260507 02:00:11.848 | INFO | "huge" : "", | 20260507 02:00:11.848 | INFO | "huge_shared" : " 27 GB", | 20260507 02:00:11.848 | INFO | "malloc" : " 14 GB", | 20260507 02:00:11.848 | INFO | "pinned" : " 29 GB" | 20260507 02:00:11.848 | INFO | } | 20260507 02:00:11.848 | INFO | Perf report for the last 5.13709 seconds: | 20260507 02:00:11.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:00:11.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:11.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:11.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:11.848 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 02:00:11.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:11.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:11.848 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:00:11.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:11.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:11.848 | INFO | Uploaded 36GiB to GPU in 2.03864s (17.6588GiB/s) | 20260507 02:00:11.848 | INFO | Downloaded 34GiB to GPU in 1.95731s (17.3708GiB/s) | 20260507 02:00:16.983 | INFO | Analyzing chunk frames = [6208512,6209024) | 20260507 02:00:16.983 | INFO | MemInfo: { | 20260507 02:00:16.983 | INFO | "gpu" : " 17 GB", | 20260507 02:00:16.983 | INFO | "huge" : "", | 20260507 02:00:16.983 | INFO | "huge_shared" : " 27 GB", | 20260507 02:00:16.983 | INFO | "malloc" : " 14 GB", | 20260507 02:00:16.983 | INFO | "pinned" : " 29 GB" | 20260507 02:00:16.983 | INFO | } | 20260507 02:00:16.983 | INFO | Perf report for the last 5.13532 seconds: | 20260507 02:00:16.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:00:16.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:16.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:16.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:16.983 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 02:00:16.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:16.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:16.983 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:00:16.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:16.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:16.984 | INFO | Uploaded 36GiB to GPU in 1.98086s (18.1739GiB/s) | 20260507 02:00:16.984 | INFO | Downloaded 34GiB to GPU in 1.9849s (17.1293GiB/s) | 20260507 02:00:22.124 | INFO | Analyzing chunk frames = [6209024,6209536) | 20260507 02:00:22.124 | INFO | MemInfo: { | 20260507 02:00:22.124 | INFO | "gpu" : " 17 GB", | 20260507 02:00:22.124 | INFO | "huge" : "", | 20260507 02:00:22.124 | INFO | "huge_shared" : " 27 GB", | 20260507 02:00:22.124 | INFO | "malloc" : " 14 GB", | 20260507 02:00:22.124 | INFO | "pinned" : " 29 GB" | 20260507 02:00:22.124 | INFO | } | 20260507 02:00:22.124 | INFO | Perf report for the last 5.14073 seconds: | 20260507 02:00:22.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:00:22.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:22.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:22.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:22.124 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:00:22.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:22.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:22.124 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:00:22.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:22.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:22.124 | INFO | Uploaded 36GiB to GPU in 1.99253s (18.0675GiB/s) | 20260507 02:00:22.124 | INFO | Downloaded 34GiB to GPU in 1.96233s (17.3264GiB/s) | 20260507 02:00:22.124 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:00:22.124 | INFO | Total mem usage: 18 GiB | 20260507 02:00:22.124 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:00:22.125 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:00:22.125 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:00:22.125 | INFO | High Water Breakdown: | 20260507 02:00:22.125 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:00:22.125 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:00:22.125 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:00:22.125 | INFO | ------------------------------------------ | 20260507 02:00:22.125 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:00:22.125 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:00:22.125 | INFO | Total mem usage: 26.5 GiB | 20260507 02:00:22.125 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:00:22.125 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:00:22.125 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:00:22.125 | INFO | High Water Breakdown: | 20260507 02:00:22.125 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:00:22.125 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:00:22.125 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:00:22.125 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:00:22.125 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:00:22.125 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:00:22.125 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:00:22.125 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:00:22.125 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:00:22.125 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:00:22.125 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:00:22.125 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:00:22.125 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:00:22.125 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:00:22.125 | INFO | ------------------------------------------ | 20260507 02:00:22.125 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:00:22.125 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:00:22.125 | INFO | Total mem usage: 25.6 GiB | 20260507 02:00:22.125 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:00:22.125 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:00:22.125 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:00:22.125 | INFO | High Water Breakdown: | 20260507 02:00:22.125 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:00:22.125 | INFO | ------------------------------------------ | 20260507 02:00:22.125 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:00:22.125 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:00:22.125 | INFO | Total mem usage: 15.7 GiB | 20260507 02:00:22.125 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:00:22.125 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:00:22.125 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 02:00:22.125 | INFO | High Water Breakdown: | 20260507 02:00:22.125 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:00:22.125 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:00:22.125 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:00:22.125 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:00:22.125 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:00:22.125 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:00:22.125 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:00:22.125 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:00:22.125 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:00:22.125 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:00:22.125 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:00:22.125 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:00:22.125 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:00:22.125 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:00:22.125 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:00:22.125 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:00:22.125 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:00:22.125 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:00:22.125 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:00:22.125 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:00:22.125 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:00:22.125 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:00:22.125 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:00:22.125 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:00:22.125 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:00:22.125 | INFO | ------------------------------------------ | 20260507 02:00:22.125 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:00:22.125 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:00:22.125 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:00:22.125 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:00:22.125 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.59e+01, 1.34e+01 / 2.04e+00 | 20260507 02:00:22.125 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 1.00e-01, 9.67e-03 / 3.80e-03 | 20260507 02:00:22.125 | INFO | PerfReport: 11.64%, Baseline -- 3.59e+00 / 4.47e+00, 5.18e-01 / 2.43e-01 | 20260507 02:00:22.125 | INFO | PerfReport: 3.24%, Histogram -- 9.99e-01 / 1.49e+00, 1.72e-01 / 8.21e-02 | 20260507 02:00:22.125 | INFO | PerfReport: 16.15%, DME -- 3.99e+01 / 4.53e+01, 4.25e+00 / 2.11e+00 | 20260507 02:00:22.125 | INFO | PerfReport: 57.54%, FrameLabeling -- 1.78e+01 / 2.01e+01, 1.34e+00 / 1.07e+00 | 20260507 02:00:22.125 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.16e+00 / 2.71e+00, 3.82e-01 / 1.54e-01 | 20260507 02:00:22.125 | INFO | PerfReport: 3.09%, Metrics -- 9.54e-01 / 1.29e+00, 2.56e-01 / 1.31e-01 | 20260507 02:00:22.125 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.24e-01 / 3.82e-01, 5.35e-02 / 2.66e-02 | 20260507 02:00:22.126 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:00:22.126 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:00:22.126 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:00:22.126 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 02:00:22.126 | INFO | PerfReport: 21.62%, Upload -- 2.09e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260507 02:00:22.126 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 02:00:22.126 | INFO | PerfReport: 32.00%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 02:00:22.126 | INFO | PerfReport: 1.34%, LockUpload -- 1.30e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 02:00:22.126 | INFO | PerfReport: 1.38%, LockDownload -- 1.33e+00 / 0.00e+00, 3.20e+00 / 0.00e+00 | 20260507 02:00:22.126 | INFO | PerfReport: 22.29%, LockCompute -- 2.15e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 02:00:22.126 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260507 02:00:22.221 | INFO | Starting a round of Baz IO | 20260507 02:00:25.341 | INFO | Populating RTMetrics report | 20260507 02:00:25.342 | INFO | RTMetrics report populated | 20260507 02:00:25.342 | INFO | Opening tmp RTMetrics json file | 20260507 02:00:25.360 | INFO | Serializing RTMetrics to json | 20260507 02:00:25.362 | INFO | Serialization done, swapping tmp file now... | 20260507 02:00:25.362 | INFO | RTMetrics done | 20260507 02:00:25.430 | INFO | Finished a round of Baz IO: Wrote 1.799 GiB in 3.209 seconds | 20260507 02:00:25.431 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.06% / 0.00% / 66.94% | 20260507 02:00:25.431 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.25967% / 0% / 8.74201e-05% | 20260507 02:00:25.431 | INFO | Baz IO overall: | 20260507 02:00:25.431 | INFO | BytesWritten : 3212260886203 | 20260507 02:00:25.431 | INFO | NumEvents : 1545435543714 | 20260507 02:00:27.272 | INFO | Analyzing chunk frames = [6209536,6210048) | 20260507 02:00:27.272 | INFO | MemInfo: { | 20260507 02:00:27.272 | INFO | "gpu" : " 17 GB", | 20260507 02:00:27.272 | INFO | "huge" : "", | 20260507 02:00:27.272 | INFO | "huge_shared" : " 27 GB", | 20260507 02:00:27.272 | INFO | "malloc" : " 14 GB", | 20260507 02:00:27.272 | INFO | "pinned" : " 29 GB" | 20260507 02:00:27.272 | INFO | } | 20260507 02:00:27.272 | INFO | Perf report for the last 5.14804 seconds: | 20260507 02:00:27.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:00:27.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:27.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:27.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:27.272 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260507 02:00:27.272 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:27.272 | INFO | RT_METRICS : 42.0%, 1.0, 22.1ms, 0 | 20260507 02:00:27.272 | INFO | PRE_HQ : 59.3%, 1.8, 55.6ms, 0 | 20260507 02:00:27.272 | INFO | BAZWRITER : 61.9%, 3.3, 105.0ms, 0 | 20260507 02:00:27.272 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:00:27.272 | INFO | Uploaded 36GiB to GPU in 2.02082s (17.8146GiB/s) | 20260507 02:00:27.273 | INFO | Downloaded 38GiB to GPU in 2.18207s (17.4147GiB/s) | 20260507 02:00:32.413 | INFO | Analyzing chunk frames = [6210048,6210560) | 20260507 02:00:32.413 | INFO | MemInfo: { | 20260507 02:00:32.413 | INFO | "gpu" : " 17 GB", | 20260507 02:00:32.413 | INFO | "huge" : "", | 20260507 02:00:32.413 | INFO | "huge_shared" : " 27 GB", | 20260507 02:00:32.413 | INFO | "malloc" : " 14 GB", | 20260507 02:00:32.413 | INFO | "pinned" : " 29 GB" | 20260507 02:00:32.413 | INFO | } | 20260507 02:00:32.413 | INFO | Perf report for the last 5.14066 seconds: | 20260507 02:00:32.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:00:32.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:32.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:32.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:32.413 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:00:32.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:32.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:32.413 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:00:32.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:32.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:32.413 | INFO | Uploaded 36GiB to GPU in 2.03339s (17.7044GiB/s) | 20260507 02:00:32.413 | INFO | Downloaded 34GiB to GPU in 1.93596s (17.5623GiB/s) | 20260507 02:00:37.551 | INFO | Analyzing chunk frames = [6210560,6211072) | 20260507 02:00:37.551 | INFO | MemInfo: { | 20260507 02:00:37.551 | INFO | "gpu" : " 17 GB", | 20260507 02:00:37.551 | INFO | "huge" : "", | 20260507 02:00:37.551 | INFO | "huge_shared" : " 27 GB", | 20260507 02:00:37.551 | INFO | "malloc" : " 14 GB", | 20260507 02:00:37.551 | INFO | "pinned" : " 29 GB" | 20260507 02:00:37.551 | INFO | } | 20260507 02:00:37.551 | INFO | Perf report for the last 5.13805 seconds: | 20260507 02:00:37.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:00:37.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:37.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:37.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:37.551 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260507 02:00:37.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:37.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:37.551 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 02:00:37.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:37.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:37.551 | INFO | Uploaded 36GiB to GPU in 2.04847s (17.5741GiB/s) | 20260507 02:00:37.551 | INFO | Downloaded 34GiB to GPU in 1.94969s (17.4386GiB/s) | 20260507 02:00:42.684 | INFO | Analyzing chunk frames = [6211072,6211584) | 20260507 02:00:42.684 | INFO | MemInfo: { | 20260507 02:00:42.684 | INFO | "gpu" : " 17 GB", | 20260507 02:00:42.684 | INFO | "huge" : "", | 20260507 02:00:42.684 | INFO | "huge_shared" : " 27 GB", | 20260507 02:00:42.684 | INFO | "malloc" : " 14 GB", | 20260507 02:00:42.684 | INFO | "pinned" : " 29 GB" | 20260507 02:00:42.684 | INFO | } | 20260507 02:00:42.684 | INFO | Perf report for the last 5.13271 seconds: | 20260507 02:00:42.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:00:42.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:42.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:42.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:42.684 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 02:00:42.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:42.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:42.684 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:00:42.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:42.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:42.684 | INFO | Uploaded 36GiB to GPU in 1.99963s (18.0034GiB/s) | 20260507 02:00:42.684 | INFO | Downloaded 34GiB to GPU in 1.91919s (17.7158GiB/s) | 20260507 02:00:47.827 | INFO | Analyzing chunk frames = [6211584,6212096) | 20260507 02:00:47.827 | INFO | MemInfo: { | 20260507 02:00:47.827 | INFO | "gpu" : " 17 GB", | 20260507 02:00:47.827 | INFO | "huge" : "", | 20260507 02:00:47.827 | INFO | "huge_shared" : " 27 GB", | 20260507 02:00:47.827 | INFO | "malloc" : " 14 GB", | 20260507 02:00:47.827 | INFO | "pinned" : " 29 GB" | 20260507 02:00:47.827 | INFO | } | 20260507 02:00:47.827 | INFO | Perf report for the last 5.14312 seconds: | 20260507 02:00:47.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:00:47.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:47.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:47.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:47.827 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 02:00:47.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:47.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:47.827 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:00:47.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:47.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:47.827 | INFO | Uploaded 36GiB to GPU in 1.99069s (18.0842GiB/s) | 20260507 02:00:47.827 | INFO | Downloaded 34GiB to GPU in 1.93426s (17.5778GiB/s) | 20260507 02:00:52.970 | INFO | Analyzing chunk frames = [6212096,6212608) | 20260507 02:00:52.970 | INFO | MemInfo: { | 20260507 02:00:52.970 | INFO | "gpu" : " 17 GB", | 20260507 02:00:52.970 | INFO | "huge" : "", | 20260507 02:00:52.970 | INFO | "huge_shared" : " 27 GB", | 20260507 02:00:52.970 | INFO | "malloc" : " 14 GB", | 20260507 02:00:52.970 | INFO | "pinned" : " 29 GB" | 20260507 02:00:52.970 | INFO | } | 20260507 02:00:52.970 | INFO | Perf report for the last 5.14296 seconds: | 20260507 02:00:52.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:00:52.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:52.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:52.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:52.970 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 02:00:52.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:52.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:52.970 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 02:00:52.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:52.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:52.970 | INFO | Uploaded 36GiB to GPU in 2.04794s (17.5786GiB/s) | 20260507 02:00:52.970 | INFO | Downloaded 34GiB to GPU in 1.94844s (17.4499GiB/s) | 20260507 02:00:58.105 | INFO | Analyzing chunk frames = [6212608,6213120) | 20260507 02:00:58.105 | INFO | MemInfo: { | 20260507 02:00:58.105 | INFO | "gpu" : " 17 GB", | 20260507 02:00:58.105 | INFO | "huge" : "", | 20260507 02:00:58.105 | INFO | "huge_shared" : " 27 GB", | 20260507 02:00:58.105 | INFO | "malloc" : " 14 GB", | 20260507 02:00:58.105 | INFO | "pinned" : " 29 GB" | 20260507 02:00:58.105 | INFO | } | 20260507 02:00:58.105 | INFO | Perf report for the last 5.13557 seconds: | 20260507 02:00:58.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:00:58.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:58.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:58.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:58.105 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 02:00:58.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:58.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:58.105 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 02:00:58.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:58.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:00:58.105 | INFO | Uploaded 36GiB to GPU in 1.97887s (18.1922GiB/s) | 20260507 02:00:58.105 | INFO | Downloaded 34GiB to GPU in 1.96228s (17.3268GiB/s) | 20260507 02:01:03.241 | INFO | Analyzing chunk frames = [6213120,6213632) | 20260507 02:01:03.241 | INFO | MemInfo: { | 20260507 02:01:03.241 | INFO | "gpu" : " 17 GB", | 20260507 02:01:03.241 | INFO | "huge" : "", | 20260507 02:01:03.241 | INFO | "huge_shared" : " 27 GB", | 20260507 02:01:03.241 | INFO | "malloc" : " 14 GB", | 20260507 02:01:03.241 | INFO | "pinned" : " 29 GB" | 20260507 02:01:03.241 | INFO | } | 20260507 02:01:03.241 | INFO | Perf report for the last 5.13604 seconds: | 20260507 02:01:03.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:01:03.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:03.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:03.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:03.241 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 02:01:03.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:03.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:03.241 | INFO | PRE_HQ : 30.3%, 1.0, 15.7ms, 0 | 20260507 02:01:03.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:03.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:03.242 | INFO | Uploaded 36GiB to GPU in 1.98829s (18.106GiB/s) | 20260507 02:01:03.242 | INFO | Downloaded 34GiB to GPU in 1.97978s (17.1736GiB/s) | 20260507 02:01:03.338 | INFO | Starting a round of Baz IO | 20260507 02:01:06.455 | INFO | Populating RTMetrics report | 20260507 02:01:06.456 | INFO | RTMetrics report populated | 20260507 02:01:06.456 | INFO | Opening tmp RTMetrics json file | 20260507 02:01:06.458 | INFO | Serializing RTMetrics to json | 20260507 02:01:06.475 | INFO | Serialization done, swapping tmp file now... | 20260507 02:01:06.475 | INFO | RTMetrics done | 20260507 02:01:06.545 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.207 seconds | 20260507 02:01:06.545 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.06% / 0.00% / 66.94% | 20260507 02:01:06.545 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.259129% / 0% / 8.74201e-05% | 20260507 02:01:06.545 | INFO | Baz IO overall: | 20260507 02:01:06.545 | INFO | BytesWritten : 3214190664788 | 20260507 02:01:06.545 | INFO | NumEvents : 1546349440253 | 20260507 02:01:08.386 | INFO | Analyzing chunk frames = [6213632,6214144) | 20260507 02:01:08.386 | INFO | MemInfo: { | 20260507 02:01:08.386 | INFO | "gpu" : " 17 GB", | 20260507 02:01:08.386 | INFO | "huge" : "", | 20260507 02:01:08.386 | INFO | "huge_shared" : " 27 GB", | 20260507 02:01:08.386 | INFO | "malloc" : " 14 GB", | 20260507 02:01:08.386 | INFO | "pinned" : " 29 GB" | 20260507 02:01:08.386 | INFO | } | 20260507 02:01:08.386 | INFO | Perf report for the last 5.14457 seconds: | 20260507 02:01:08.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:01:08.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:08.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:08.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:08.386 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260507 02:01:08.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:08.386 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 02:01:08.386 | INFO | PRE_HQ : 59.6%, 1.8, 56.0ms, 0 | 20260507 02:01:08.386 | INFO | BAZWRITER : 61.8%, 3.3, 105.5ms, 0 | 20260507 02:01:08.386 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:01:08.386 | INFO | Uploaded 36GiB to GPU in 2.02437s (17.7833GiB/s) | 20260507 02:01:08.386 | INFO | Downloaded 38GiB to GPU in 2.18214s (17.4141GiB/s) | 20260507 02:01:13.526 | INFO | Analyzing chunk frames = [6214144,6214656) | 20260507 02:01:13.526 | INFO | MemInfo: { | 20260507 02:01:13.526 | INFO | "gpu" : " 17 GB", | 20260507 02:01:13.526 | INFO | "huge" : "", | 20260507 02:01:13.526 | INFO | "huge_shared" : " 27 GB", | 20260507 02:01:13.526 | INFO | "malloc" : " 14 GB", | 20260507 02:01:13.526 | INFO | "pinned" : " 29 GB" | 20260507 02:01:13.526 | INFO | } | 20260507 02:01:13.526 | INFO | Perf report for the last 5.14048 seconds: | 20260507 02:01:13.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:01:13.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:13.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:13.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:13.527 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:01:13.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:13.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:13.527 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:01:13.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:13.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:13.527 | INFO | Uploaded 36GiB to GPU in 2.03099s (17.7253GiB/s) | 20260507 02:01:13.527 | INFO | Downloaded 34GiB to GPU in 1.93808s (17.5431GiB/s) | 20260507 02:01:18.663 | INFO | Analyzing chunk frames = [6214656,6215168) | 20260507 02:01:18.663 | INFO | MemInfo: { | 20260507 02:01:18.663 | INFO | "gpu" : " 17 GB", | 20260507 02:01:18.663 | INFO | "huge" : "", | 20260507 02:01:18.663 | INFO | "huge_shared" : " 27 GB", | 20260507 02:01:18.663 | INFO | "malloc" : " 14 GB", | 20260507 02:01:18.663 | INFO | "pinned" : " 29 GB" | 20260507 02:01:18.663 | INFO | } | 20260507 02:01:18.663 | INFO | Perf report for the last 5.13684 seconds: | 20260507 02:01:18.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:01:18.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:18.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:18.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:18.663 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 02:01:18.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:18.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:18.663 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:01:18.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:18.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:18.664 | INFO | Uploaded 36GiB to GPU in 1.98119s (18.1709GiB/s) | 20260507 02:01:18.664 | INFO | Downloaded 34GiB to GPU in 1.94485s (17.482GiB/s) | 20260507 02:01:23.797 | INFO | Analyzing chunk frames = [6215168,6215680) | 20260507 02:01:23.797 | INFO | MemInfo: { | 20260507 02:01:23.797 | INFO | "gpu" : " 17 GB", | 20260507 02:01:23.797 | INFO | "huge" : "", | 20260507 02:01:23.797 | INFO | "huge_shared" : " 27 GB", | 20260507 02:01:23.797 | INFO | "malloc" : " 14 GB", | 20260507 02:01:23.797 | INFO | "pinned" : " 29 GB" | 20260507 02:01:23.797 | INFO | } | 20260507 02:01:23.797 | INFO | Perf report for the last 5.13365 seconds: | 20260507 02:01:23.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:01:23.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:23.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:23.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:23.797 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 02:01:23.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:23.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:23.797 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:01:23.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:23.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:23.797 | INFO | Uploaded 36GiB to GPU in 1.9554s (18.4105GiB/s) | 20260507 02:01:23.797 | INFO | Downloaded 34GiB to GPU in 1.92021s (17.7064GiB/s) | 20260507 02:01:28.937 | INFO | Analyzing chunk frames = [6215680,6216192) | 20260507 02:01:28.937 | INFO | MemInfo: { | 20260507 02:01:28.937 | INFO | "gpu" : " 17 GB", | 20260507 02:01:28.937 | INFO | "huge" : "", | 20260507 02:01:28.937 | INFO | "huge_shared" : " 27 GB", | 20260507 02:01:28.937 | INFO | "malloc" : " 14 GB", | 20260507 02:01:28.937 | INFO | "pinned" : " 29 GB" | 20260507 02:01:28.937 | INFO | } | 20260507 02:01:28.937 | INFO | Perf report for the last 5.13967 seconds: | 20260507 02:01:28.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:01:28.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:28.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:28.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:28.937 | INFO | ANALYSIS : 60.2%, 3.0, 94.0ms, 0 | 20260507 02:01:28.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:28.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:28.937 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 02:01:28.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:28.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:28.937 | INFO | Uploaded 36GiB to GPU in 1.99377s (18.0563GiB/s) | 20260507 02:01:28.937 | INFO | Downloaded 34GiB to GPU in 1.94511s (17.4797GiB/s) | 20260507 02:01:34.080 | INFO | Analyzing chunk frames = [6216192,6216704) | 20260507 02:01:34.080 | INFO | MemInfo: { | 20260507 02:01:34.080 | INFO | "gpu" : " 17 GB", | 20260507 02:01:34.080 | INFO | "huge" : "", | 20260507 02:01:34.081 | INFO | "huge_shared" : " 27 GB", | 20260507 02:01:34.081 | INFO | "malloc" : " 14 GB", | 20260507 02:01:34.081 | INFO | "pinned" : " 29 GB" | 20260507 02:01:34.081 | INFO | } | 20260507 02:01:34.081 | INFO | Perf report for the last 5.14438 seconds: | 20260507 02:01:34.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:01:34.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:34.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:34.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:34.081 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 02:01:34.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:34.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:34.081 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:01:34.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:34.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:34.081 | INFO | Uploaded 36GiB to GPU in 2.00995s (17.9109GiB/s) | 20260507 02:01:34.081 | INFO | Downloaded 34GiB to GPU in 1.93851s (17.5393GiB/s) | 20260507 02:01:39.227 | INFO | Analyzing chunk frames = [6216704,6217216) | 20260507 02:01:39.227 | INFO | MemInfo: { | 20260507 02:01:39.227 | INFO | "gpu" : " 17 GB", | 20260507 02:01:39.227 | INFO | "huge" : "", | 20260507 02:01:39.227 | INFO | "huge_shared" : " 27 GB", | 20260507 02:01:39.227 | INFO | "malloc" : " 14 GB", | 20260507 02:01:39.227 | INFO | "pinned" : " 29 GB" | 20260507 02:01:39.227 | INFO | } | 20260507 02:01:39.227 | INFO | Perf report for the last 5.14563 seconds: | 20260507 02:01:39.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:01:39.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:39.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:39.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:39.227 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:01:39.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:39.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:39.227 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 02:01:39.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:39.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:39.227 | INFO | Uploaded 36GiB to GPU in 1.98424s (18.143GiB/s) | 20260507 02:01:39.227 | INFO | Downloaded 34GiB to GPU in 1.98596s (17.1202GiB/s) | 20260507 02:01:44.362 | INFO | Analyzing chunk frames = [6217216,6217728) | 20260507 02:01:44.362 | INFO | MemInfo: { | 20260507 02:01:44.362 | INFO | "gpu" : " 17 GB", | 20260507 02:01:44.362 | INFO | "huge" : "", | 20260507 02:01:44.362 | INFO | "huge_shared" : " 27 GB", | 20260507 02:01:44.362 | INFO | "malloc" : " 14 GB", | 20260507 02:01:44.362 | INFO | "pinned" : " 29 GB" | 20260507 02:01:44.362 | INFO | } | 20260507 02:01:44.363 | INFO | Perf report for the last 5.13582 seconds: | 20260507 02:01:44.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:01:44.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:44.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:44.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:44.363 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:01:44.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:44.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:44.363 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:01:44.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:44.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:44.363 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:01:44.363 | INFO | Total mem usage: 18 GiB | 20260507 02:01:44.363 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:01:44.363 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:01:44.363 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:01:44.363 | INFO | High Water Breakdown: | 20260507 02:01:44.363 | INFO | 60.2% : 8.58 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:01:44.363 | INFO | Uploaded 36GiB to GPU in 1.99531s (18.0423GiB/s) | 20260507 02:01:44.363 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:01:44.363 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:01:44.363 | INFO | ------------------------------------------ | 20260507 02:01:44.363 | INFO | Downloaded 34GiB to GPU in 1.96403s (17.3113GiB/s) | 20260507 02:01:44.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:01:44.363 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:01:44.363 | INFO | Total mem usage: 26.5 GiB | 20260507 02:01:44.363 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:01:44.363 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:01:44.363 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:01:44.363 | INFO | High Water Breakdown: | 20260507 02:01:44.363 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:01:44.363 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:01:44.363 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:01:44.363 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:01:44.363 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:01:44.363 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:01:44.363 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:01:44.363 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:01:44.363 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:01:44.363 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:01:44.363 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:01:44.363 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:01:44.363 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:01:44.363 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:01:44.363 | INFO | ------------------------------------------ | 20260507 02:01:44.363 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:01:44.364 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:01:44.364 | INFO | Total mem usage: 25.6 GiB | 20260507 02:01:44.364 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:01:44.364 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 02:01:44.364 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 02:01:44.364 | INFO | High Water Breakdown: | 20260507 02:01:44.364 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 02:01:44.364 | INFO | ------------------------------------------ | 20260507 02:01:44.364 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:01:44.364 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:01:44.364 | INFO | Total mem usage: 15.7 GiB | 20260507 02:01:44.364 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:01:44.364 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:01:44.364 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 02:01:44.364 | INFO | High Water Breakdown: | 20260507 02:01:44.364 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:01:44.364 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:01:44.364 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:01:44.364 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:01:44.364 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:01:44.364 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:01:44.364 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:01:44.364 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:01:44.364 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:01:44.364 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:01:44.364 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:01:44.364 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:01:44.364 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:01:44.364 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:01:44.364 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:01:44.364 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:01:44.364 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:01:44.364 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:01:44.364 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:01:44.364 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:01:44.364 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:01:44.364 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:01:44.364 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:01:44.364 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:01:44.364 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:01:44.364 | INFO | ------------------------------------------ | 20260507 02:01:44.364 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:01:44.364 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:01:44.364 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:01:44.364 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:01:44.364 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.55e+01, 1.34e+01 / 1.54e+00 | 20260507 02:01:44.364 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.00e-01, 9.52e-03 / 3.69e-03 | 20260507 02:01:44.364 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.52e+00, 5.22e-01 / 1.90e-01 | 20260507 02:01:44.364 | INFO | PerfReport: 3.23%, Histogram -- 9.96e-01 / 1.48e+00, 1.72e-01 / 8.41e-02 | 20260507 02:01:44.364 | INFO | PerfReport: 16.14%, DME -- 3.98e+01 / 4.48e+01, 4.05e+00 / 2.25e+00 | 20260507 02:01:44.364 | INFO | PerfReport: 57.49%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.27e+00 / 1.67e+00 | 20260507 02:01:44.364 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.16e+00 / 2.72e+00, 3.84e-01 / 2.06e-01 | 20260507 02:01:44.364 | INFO | PerfReport: 3.10%, Metrics -- 9.55e-01 / 1.27e+00, 2.55e-01 / 1.32e-01 | 20260507 02:01:44.364 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.24e-01 / 3.75e-01, 5.23e-02 / 1.10e-02 | 20260507 02:01:44.364 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:01:44.364 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:01:44.364 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:01:44.364 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 02:01:44.364 | INFO | PerfReport: 21.65%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260507 02:01:44.364 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260507 02:01:44.364 | INFO | PerfReport: 31.95%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 02:01:44.364 | INFO | PerfReport: 1.43%, LockUpload -- 1.38e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 02:01:44.364 | INFO | PerfReport: 1.34%, LockDownload -- 1.30e+00 / 0.00e+00, 3.18e+00 / 0.00e+00 | 20260507 02:01:44.364 | INFO | PerfReport: 22.26%, LockCompute -- 2.15e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 02:01:44.364 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260507 02:01:44.460 | INFO | Starting a round of Baz IO | 20260507 02:01:47.567 | INFO | Populating RTMetrics report | 20260507 02:01:47.567 | INFO | RTMetrics report populated | 20260507 02:01:47.567 | INFO | Opening tmp RTMetrics json file | 20260507 02:01:47.568 | INFO | Serializing RTMetrics to json | 20260507 02:01:47.585 | INFO | Serialization done, swapping tmp file now... | 20260507 02:01:47.588 | INFO | RTMetrics done | 20260507 02:01:47.661 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.201 seconds | 20260507 02:01:47.661 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.06% / 0.00% / 66.94% | 20260507 02:01:47.661 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.257246% / 0% / 9.13938e-05% | 20260507 02:01:47.661 | INFO | Baz IO overall: | 20260507 02:01:47.661 | INFO | BytesWritten : 3216119875901 | 20260507 02:01:47.661 | INFO | NumEvents : 1547262799688 | 20260507 02:01:49.501 | INFO | Analyzing chunk frames = [6217728,6218240) | 20260507 02:01:49.501 | INFO | MemInfo: { | 20260507 02:01:49.501 | INFO | "gpu" : " 17 GB", | 20260507 02:01:49.501 | INFO | "huge" : "", | 20260507 02:01:49.501 | INFO | "huge_shared" : " 27 GB", | 20260507 02:01:49.501 | INFO | "malloc" : " 14 GB", | 20260507 02:01:49.501 | INFO | "pinned" : " 29 GB" | 20260507 02:01:49.501 | INFO | } | 20260507 02:01:49.501 | INFO | Perf report for the last 5.13833 seconds: | 20260507 02:01:49.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:01:49.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:49.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:49.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:49.501 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 02:01:49.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:49.501 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 02:01:49.501 | INFO | PRE_HQ : 59.3%, 1.8, 55.8ms, 0 | 20260507 02:01:49.501 | INFO | BAZWRITER : 61.9%, 3.3, 105.6ms, 0 | 20260507 02:01:49.501 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:01:49.501 | INFO | Uploaded 36GiB to GPU in 2.04389s (17.6135GiB/s) | 20260507 02:01:49.501 | INFO | Downloaded 38GiB to GPU in 2.17708s (17.4546GiB/s) | 20260507 02:01:54.639 | INFO | Analyzing chunk frames = [6218240,6218752) | 20260507 02:01:54.639 | INFO | MemInfo: { | 20260507 02:01:54.639 | INFO | "gpu" : " 17 GB", | 20260507 02:01:54.639 | INFO | "huge" : "", | 20260507 02:01:54.639 | INFO | "huge_shared" : " 27 GB", | 20260507 02:01:54.639 | INFO | "malloc" : " 14 GB", | 20260507 02:01:54.639 | INFO | "pinned" : " 29 GB" | 20260507 02:01:54.639 | INFO | } | 20260507 02:01:54.639 | INFO | Perf report for the last 5.1385 seconds: | 20260507 02:01:54.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:01:54.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:54.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:54.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:54.639 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:01:54.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:54.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:54.639 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:01:54.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:54.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:54.640 | INFO | Uploaded 36GiB to GPU in 2.0435s (17.6168GiB/s) | 20260507 02:01:54.640 | INFO | Downloaded 34GiB to GPU in 1.93822s (17.5419GiB/s) | 20260507 02:01:59.778 | INFO | Analyzing chunk frames = [6218752,6219264) | 20260507 02:01:59.778 | INFO | MemInfo: { | 20260507 02:01:59.778 | INFO | "gpu" : " 17 GB", | 20260507 02:01:59.778 | INFO | "huge" : "", | 20260507 02:01:59.778 | INFO | "huge_shared" : " 27 GB", | 20260507 02:01:59.778 | INFO | "malloc" : " 14 GB", | 20260507 02:01:59.778 | INFO | "pinned" : " 29 GB" | 20260507 02:01:59.778 | INFO | } | 20260507 02:01:59.778 | INFO | Perf report for the last 5.13876 seconds: | 20260507 02:01:59.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:01:59.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:59.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:59.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:59.778 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 02:01:59.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:59.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:59.778 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:01:59.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:59.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:01:59.778 | INFO | Uploaded 36GiB to GPU in 1.99251s (18.0676GiB/s) | 20260507 02:01:59.778 | INFO | Downloaded 34GiB to GPU in 1.93721s (17.5511GiB/s) | 20260507 02:02:04.915 | INFO | Analyzing chunk frames = [6219264,6219776) | 20260507 02:02:04.915 | INFO | MemInfo: { | 20260507 02:02:04.915 | INFO | "gpu" : " 17 GB", | 20260507 02:02:04.915 | INFO | "huge" : "", | 20260507 02:02:04.915 | INFO | "huge_shared" : " 27 GB", | 20260507 02:02:04.915 | INFO | "malloc" : " 14 GB", | 20260507 02:02:04.916 | INFO | "pinned" : " 29 GB" | 20260507 02:02:04.916 | INFO | } | 20260507 02:02:04.916 | INFO | Perf report for the last 5.13741 seconds: | 20260507 02:02:04.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:02:04.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:04.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:04.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:04.916 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 02:02:04.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:04.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:04.916 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 02:02:04.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:04.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:04.916 | INFO | Uploaded 36GiB to GPU in 1.99197s (18.0726GiB/s) | 20260507 02:02:04.916 | INFO | Downloaded 34GiB to GPU in 1.91046s (17.7967GiB/s) | 20260507 02:02:10.057 | INFO | Analyzing chunk frames = [6219776,6220288) | 20260507 02:02:10.057 | INFO | MemInfo: { | 20260507 02:02:10.057 | INFO | "gpu" : " 17 GB", | 20260507 02:02:10.057 | INFO | "huge" : "", | 20260507 02:02:10.057 | INFO | "huge_shared" : " 27 GB", | 20260507 02:02:10.057 | INFO | "malloc" : " 14 GB", | 20260507 02:02:10.057 | INFO | "pinned" : " 29 GB" | 20260507 02:02:10.057 | INFO | } | 20260507 02:02:10.057 | INFO | Perf report for the last 5.14172 seconds: | 20260507 02:02:10.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:02:10.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:10.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:10.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:10.057 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 02:02:10.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:10.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:10.057 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:02:10.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:10.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:10.058 | INFO | Uploaded 36GiB to GPU in 2.00659s (17.9409GiB/s) | 20260507 02:02:10.058 | INFO | Downloaded 34GiB to GPU in 1.94714s (17.4615GiB/s) | 20260507 02:02:15.191 | INFO | Analyzing chunk frames = [6220288,6220800) | 20260507 02:02:15.191 | INFO | MemInfo: { | 20260507 02:02:15.191 | INFO | "gpu" : " 17 GB", | 20260507 02:02:15.191 | INFO | "huge" : "", | 20260507 02:02:15.191 | INFO | "huge_shared" : " 27 GB", | 20260507 02:02:15.191 | INFO | "malloc" : " 14 GB", | 20260507 02:02:15.191 | INFO | "pinned" : " 29 GB" | 20260507 02:02:15.191 | INFO | } | 20260507 02:02:15.191 | INFO | Perf report for the last 5.13408 seconds: | 20260507 02:02:15.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:02:15.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:15.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:15.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:15.191 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 02:02:15.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:15.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:15.191 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 02:02:15.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:15.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:15.192 | INFO | Uploaded 36GiB to GPU in 2.03186s (17.7177GiB/s) | 20260507 02:02:15.192 | INFO | Downloaded 34GiB to GPU in 1.91716s (17.7345GiB/s) | 20260507 02:02:20.340 | INFO | Analyzing chunk frames = [6220800,6221312) | 20260507 02:02:20.340 | INFO | MemInfo: { | 20260507 02:02:20.340 | INFO | "gpu" : " 17 GB", | 20260507 02:02:20.340 | INFO | "huge" : "", | 20260507 02:02:20.340 | INFO | "huge_shared" : " 27 GB", | 20260507 02:02:20.340 | INFO | "malloc" : " 14 GB", | 20260507 02:02:20.340 | INFO | "pinned" : " 29 GB" | 20260507 02:02:20.340 | INFO | } | 20260507 02:02:20.340 | INFO | Perf report for the last 5.1488 seconds: | 20260507 02:02:20.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:02:20.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:20.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:20.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:20.340 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 02:02:20.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:20.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:20.340 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:02:20.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:20.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:20.340 | INFO | Uploaded 36GiB to GPU in 1.97233s (18.2525GiB/s) | 20260507 02:02:20.340 | INFO | Downloaded 34GiB to GPU in 1.98885s (17.0953GiB/s) | 20260507 02:02:25.481 | INFO | Analyzing chunk frames = [6221312,6221824) | 20260507 02:02:25.481 | INFO | MemInfo: { | 20260507 02:02:25.481 | INFO | "gpu" : " 17 GB", | 20260507 02:02:25.481 | INFO | "huge" : "", | 20260507 02:02:25.481 | INFO | "huge_shared" : " 27 GB", | 20260507 02:02:25.481 | INFO | "malloc" : " 14 GB", | 20260507 02:02:25.481 | INFO | "pinned" : " 29 GB" | 20260507 02:02:25.481 | INFO | } | 20260507 02:02:25.481 | INFO | Perf report for the last 5.1408 seconds: | 20260507 02:02:25.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:02:25.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:25.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:25.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:25.481 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:02:25.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:25.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:25.481 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:02:25.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:25.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:25.481 | INFO | Uploaded 36GiB to GPU in 2.0067s (17.9399GiB/s) | 20260507 02:02:25.481 | INFO | Downloaded 34GiB to GPU in 1.99674s (17.0278GiB/s) | 20260507 02:02:25.578 | INFO | Starting a round of Baz IO | 20260507 02:02:28.686 | INFO | Populating RTMetrics report | 20260507 02:02:28.686 | INFO | RTMetrics report populated | 20260507 02:02:28.686 | INFO | Opening tmp RTMetrics json file | 20260507 02:02:28.707 | INFO | Serializing RTMetrics to json | 20260507 02:02:28.710 | INFO | Serialization done, swapping tmp file now... | 20260507 02:02:28.712 | INFO | RTMetrics done | 20260507 02:02:28.850 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.271 seconds | 20260507 02:02:28.850 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.06% / 0.00% / 66.94% | 20260507 02:02:28.850 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.257564% / 3.97364e-06% / 0.000107288% | 20260507 02:02:28.850 | INFO | Baz IO overall: | 20260507 02:02:28.850 | INFO | BytesWritten : 3218047983890 | 20260507 02:02:28.850 | INFO | NumEvents : 1548175318042 | 20260507 02:02:30.614 | INFO | Analyzing chunk frames = [6221824,6222336) | 20260507 02:02:30.614 | INFO | MemInfo: { | 20260507 02:02:30.614 | INFO | "gpu" : " 17 GB", | 20260507 02:02:30.614 | INFO | "huge" : "", | 20260507 02:02:30.614 | INFO | "huge_shared" : " 27 GB", | 20260507 02:02:30.614 | INFO | "malloc" : " 14 GB", | 20260507 02:02:30.614 | INFO | "pinned" : " 29 GB" | 20260507 02:02:30.614 | INFO | } | 20260507 02:02:30.614 | INFO | Perf report for the last 5.13292 seconds: | 20260507 02:02:30.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:02:30.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:30.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:30.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:30.614 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260507 02:02:30.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:30.614 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 02:02:30.614 | INFO | PRE_HQ : 59.3%, 1.8, 55.7ms, 0 | 20260507 02:02:30.614 | INFO | BAZWRITER : 63.2%, 3.5, 114.6ms, 0 | 20260507 02:02:30.614 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:02:30.614 | INFO | Uploaded 36GiB to GPU in 2.03893s (17.6564GiB/s) | 20260507 02:02:30.614 | INFO | Downloaded 38GiB to GPU in 2.16899s (17.5196GiB/s) | 20260507 02:02:35.747 | INFO | Analyzing chunk frames = [6222336,6222848) | 20260507 02:02:35.747 | INFO | MemInfo: { | 20260507 02:02:35.747 | INFO | "gpu" : " 17 GB", | 20260507 02:02:35.747 | INFO | "huge" : "", | 20260507 02:02:35.747 | INFO | "huge_shared" : " 27 GB", | 20260507 02:02:35.747 | INFO | "malloc" : " 14 GB", | 20260507 02:02:35.747 | INFO | "pinned" : " 29 GB" | 20260507 02:02:35.748 | INFO | } | 20260507 02:02:35.748 | INFO | Perf report for the last 5.1336 seconds: | 20260507 02:02:35.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:02:35.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:35.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:35.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:35.748 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:02:35.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:35.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:35.748 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:02:35.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:35.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:35.748 | INFO | Uploaded 36GiB to GPU in 2.04167s (17.6326GiB/s) | 20260507 02:02:35.748 | INFO | Downloaded 34GiB to GPU in 1.93678s (17.5549GiB/s) | 20260507 02:02:40.889 | INFO | Analyzing chunk frames = [6222848,6223360) | 20260507 02:02:40.889 | INFO | MemInfo: { | 20260507 02:02:40.889 | INFO | "gpu" : " 17 GB", | 20260507 02:02:40.889 | INFO | "huge" : "", | 20260507 02:02:40.889 | INFO | "huge_shared" : " 27 GB", | 20260507 02:02:40.889 | INFO | "malloc" : " 14 GB", | 20260507 02:02:40.889 | INFO | "pinned" : " 29 GB" | 20260507 02:02:40.889 | INFO | } | 20260507 02:02:40.889 | INFO | Perf report for the last 5.14128 seconds: | 20260507 02:02:40.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:02:40.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:40.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:40.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:40.889 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 02:02:40.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:40.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:40.889 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:02:40.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:40.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:40.889 | INFO | Uploaded 36GiB to GPU in 1.98789s (18.1097GiB/s) | 20260507 02:02:40.889 | INFO | Downloaded 34GiB to GPU in 1.93266s (17.5924GiB/s) | 20260507 02:02:46.033 | INFO | Analyzing chunk frames = [6223360,6223872) | 20260507 02:02:46.033 | INFO | MemInfo: { | 20260507 02:02:46.033 | INFO | "gpu" : " 17 GB", | 20260507 02:02:46.033 | INFO | "huge" : "", | 20260507 02:02:46.033 | INFO | "huge_shared" : " 27 GB", | 20260507 02:02:46.033 | INFO | "malloc" : " 14 GB", | 20260507 02:02:46.033 | INFO | "pinned" : " 29 GB" | 20260507 02:02:46.033 | INFO | } | 20260507 02:02:46.033 | INFO | Perf report for the last 5.14393 seconds: | 20260507 02:02:46.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:02:46.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:46.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:46.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:46.033 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260507 02:02:46.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:46.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:46.033 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:02:46.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:46.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:46.033 | INFO | Uploaded 36GiB to GPU in 1.99429s (18.0515GiB/s) | 20260507 02:02:46.033 | INFO | Downloaded 34GiB to GPU in 1.91164s (17.7857GiB/s) | 20260507 02:02:51.171 | INFO | Analyzing chunk frames = [6223872,6224384) | 20260507 02:02:51.171 | INFO | MemInfo: { | 20260507 02:02:51.171 | INFO | "gpu" : " 17 GB", | 20260507 02:02:51.171 | INFO | "huge" : "", | 20260507 02:02:51.172 | INFO | "huge_shared" : " 27 GB", | 20260507 02:02:51.172 | INFO | "malloc" : " 14 GB", | 20260507 02:02:51.172 | INFO | "pinned" : " 29 GB" | 20260507 02:02:51.172 | INFO | } | 20260507 02:02:51.172 | INFO | Perf report for the last 5.13887 seconds: | 20260507 02:02:51.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:02:51.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:51.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:51.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:51.172 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 02:02:51.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:51.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:51.172 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:02:51.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:51.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:51.172 | INFO | Uploaded 36GiB to GPU in 2.01455s (17.87GiB/s) | 20260507 02:02:51.172 | INFO | Downloaded 34GiB to GPU in 1.93928s (17.5323GiB/s) | 20260507 02:02:56.313 | INFO | Analyzing chunk frames = [6224384,6224896) | 20260507 02:02:56.313 | INFO | MemInfo: { | 20260507 02:02:56.313 | INFO | "gpu" : " 17 GB", | 20260507 02:02:56.313 | INFO | "huge" : "", | 20260507 02:02:56.314 | INFO | "huge_shared" : " 27 GB", | 20260507 02:02:56.314 | INFO | "malloc" : " 14 GB", | 20260507 02:02:56.314 | INFO | "pinned" : " 29 GB" | 20260507 02:02:56.314 | INFO | } | 20260507 02:02:56.314 | INFO | Perf report for the last 5.142 seconds: | 20260507 02:02:56.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:02:56.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:56.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:56.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:56.314 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260507 02:02:56.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:56.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:56.314 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 02:02:56.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:56.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:02:56.314 | INFO | Uploaded 36GiB to GPU in 2.03372s (17.7016GiB/s) | 20260507 02:02:56.314 | INFO | Downloaded 34GiB to GPU in 1.93064s (17.6108GiB/s) | 20260507 02:03:01.454 | INFO | Analyzing chunk frames = [6224896,6225408) | 20260507 02:03:01.454 | INFO | MemInfo: { | 20260507 02:03:01.454 | INFO | "gpu" : " 17 GB", | 20260507 02:03:01.454 | INFO | "huge" : "", | 20260507 02:03:01.454 | INFO | "huge_shared" : " 27 GB", | 20260507 02:03:01.454 | INFO | "malloc" : " 14 GB", | 20260507 02:03:01.454 | INFO | "pinned" : " 29 GB" | 20260507 02:03:01.454 | INFO | } | 20260507 02:03:01.454 | INFO | Perf report for the last 5.14001 seconds: | 20260507 02:03:01.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:03:01.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:01.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:01.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:01.454 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:03:01.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:01.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:01.454 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 02:03:01.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:01.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:01.454 | INFO | Uploaded 36GiB to GPU in 1.94638s (18.4958GiB/s) | 20260507 02:03:01.454 | INFO | Downloaded 34GiB to GPU in 1.97234s (17.2384GiB/s) | 20260507 02:03:06.594 | INFO | Analyzing chunk frames = [6225408,6225920) | 20260507 02:03:06.594 | INFO | MemInfo: { | 20260507 02:03:06.594 | INFO | "gpu" : " 17 GB", | 20260507 02:03:06.594 | INFO | "huge" : "", | 20260507 02:03:06.594 | INFO | "huge_shared" : " 27 GB", | 20260507 02:03:06.594 | INFO | "malloc" : " 14 GB", | 20260507 02:03:06.594 | INFO | "pinned" : " 29 GB" | 20260507 02:03:06.594 | INFO | } | 20260507 02:03:06.594 | INFO | Perf report for the last 5.14004 seconds: | 20260507 02:03:06.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:03:06.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:06.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:06.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:06.594 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 02:03:06.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:06.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:06.594 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:03:06.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:06.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:06.594 | INFO | Uploaded 36GiB to GPU in 1.96732s (18.299GiB/s) | 20260507 02:03:06.594 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:03:06.594 | INFO | Total mem usage: 18 GiB | 20260507 02:03:06.594 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:03:06.594 | INFO | Recent High Water Marks Sum: 14.5 GiB (Savings: -24%) | 20260507 02:03:06.594 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:03:06.594 | INFO | High Water Breakdown: | 20260507 02:03:06.594 | INFO | 60.4% : 8.77 GiB ( 76.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:03:06.594 | INFO | 39.5% : 5.74 GiB ( 85.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:03:06.594 | INFO | 0.137% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:03:06.594 | INFO | ------------------------------------------ | 20260507 02:03:06.594 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:03:06.594 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:03:06.594 | INFO | Total mem usage: 26.5 GiB | 20260507 02:03:06.594 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:03:06.594 | INFO | Downloaded 34GiB to GPU in 1.94591s (17.4726GiB/s) | 20260507 02:03:06.594 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:03:06.594 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:03:06.594 | INFO | High Water Breakdown: | 20260507 02:03:06.594 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:03:06.595 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:03:06.595 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:03:06.595 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:03:06.595 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:03:06.595 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:03:06.595 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:03:06.595 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:03:06.595 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:03:06.595 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:03:06.595 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:03:06.595 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:03:06.595 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:03:06.595 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:03:06.595 | INFO | ------------------------------------------ | 20260507 02:03:06.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:03:06.595 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:03:06.595 | INFO | Total mem usage: 25.6 GiB | 20260507 02:03:06.595 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:03:06.595 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 02:03:06.595 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 02:03:06.595 | INFO | High Water Breakdown: | 20260507 02:03:06.595 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 02:03:06.595 | INFO | ------------------------------------------ | 20260507 02:03:06.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:03:06.595 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:03:06.595 | INFO | Total mem usage: 15.7 GiB | 20260507 02:03:06.595 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:03:06.595 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:03:06.595 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:03:06.595 | INFO | High Water Breakdown: | 20260507 02:03:06.595 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:03:06.595 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:03:06.595 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:03:06.595 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:03:06.595 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:03:06.595 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:03:06.595 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:03:06.595 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:03:06.595 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:03:06.595 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:03:06.595 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:03:06.595 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:03:06.595 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:03:06.595 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:03:06.595 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:03:06.595 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:03:06.595 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:03:06.595 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:03:06.595 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:03:06.595 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:03:06.595 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:03:06.595 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:03:06.595 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:03:06.595 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:03:06.595 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:03:06.595 | INFO | ------------------------------------------ | 20260507 02:03:06.595 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:03:06.595 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:03:06.595 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:03:06.595 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:03:06.595 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.52e+01, 1.35e+01 / 1.92e+00 | 20260507 02:03:06.595 | INFO | PerfReport: 0.30%, AggregateStats -- 9.12e-02 / 1.03e-01, 1.08e-02 / 6.80e-03 | 20260507 02:03:06.595 | INFO | PerfReport: 11.66%, Baseline -- 3.60e+00 / 4.41e+00, 5.25e-01 / 2.88e-01 | 20260507 02:03:06.595 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.47e+00, 1.76e-01 / 1.04e-01 | 20260507 02:03:06.595 | INFO | PerfReport: 16.04%, DME -- 3.96e+01 / 4.41e+01, 3.75e+00 / 2.07e+00 | 20260507 02:03:06.595 | INFO | PerfReport: 57.62%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.36e+00 / 1.77e+00 | 20260507 02:03:06.596 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.16e+00 / 2.65e+00, 3.78e-01 / 2.41e-01 | 20260507 02:03:06.596 | INFO | PerfReport: 3.10%, Metrics -- 9.55e-01 / 1.28e+00, 2.57e-01 / 1.45e-01 | 20260507 02:03:06.596 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.24e-01 / 3.77e-01, 5.15e-02 / 2.42e-02 | 20260507 02:03:06.596 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:03:06.596 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:03:06.596 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:03:06.596 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 02:03:06.596 | INFO | PerfReport: 21.70%, Upload -- 2.09e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 02:03:06.596 | INFO | PerfReport: 21.32%, Download -- 2.05e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260507 02:03:06.596 | INFO | PerfReport: 32.03%, Compute -- 3.09e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 02:03:06.596 | INFO | PerfReport: 1.42%, LockUpload -- 1.37e+00 / 0.00e+00, 4.72e+00 / 0.00e+00 | 20260507 02:03:06.596 | INFO | PerfReport: 1.23%, LockDownload -- 1.19e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260507 02:03:06.596 | INFO | PerfReport: 22.28%, LockCompute -- 2.15e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 02:03:06.596 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260507 02:03:06.691 | INFO | Starting a round of Baz IO | 20260507 02:03:09.802 | INFO | Populating RTMetrics report | 20260507 02:03:09.802 | INFO | RTMetrics report populated | 20260507 02:03:09.802 | INFO | Opening tmp RTMetrics json file | 20260507 02:03:09.821 | INFO | Serializing RTMetrics to json | 20260507 02:03:09.822 | INFO | Serialization done, swapping tmp file now... | 20260507 02:03:09.823 | INFO | RTMetrics done | 20260507 02:03:09.892 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.2 seconds | 20260507 02:03:09.892 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.05% / 0.00% / 66.94% | 20260507 02:03:09.892 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.257115% / 0% / 9.13938e-05% | 20260507 02:03:09.892 | INFO | Baz IO overall: | 20260507 02:03:09.892 | INFO | BytesWritten : 3219977174242 | 20260507 02:03:09.892 | INFO | NumEvents : 1549088671751 | 20260507 02:03:11.736 | INFO | Analyzing chunk frames = [6225920,6226432) | 20260507 02:03:11.736 | INFO | MemInfo: { | 20260507 02:03:11.736 | INFO | "gpu" : " 17 GB", | 20260507 02:03:11.736 | INFO | "huge" : "", | 20260507 02:03:11.736 | INFO | "huge_shared" : " 27 GB", | 20260507 02:03:11.736 | INFO | "malloc" : " 14 GB", | 20260507 02:03:11.736 | INFO | "pinned" : " 29 GB" | 20260507 02:03:11.736 | INFO | } | 20260507 02:03:11.736 | INFO | Perf report for the last 5.14236 seconds: | 20260507 02:03:11.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:03:11.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:11.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:11.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:11.736 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260507 02:03:11.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:11.736 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 02:03:11.736 | INFO | PRE_HQ : 59.3%, 1.8, 55.6ms, 0 | 20260507 02:03:11.736 | INFO | BAZWRITER : 61.8%, 3.3, 103.9ms, 0 | 20260507 02:03:11.736 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:03:11.737 | INFO | Uploaded 36GiB to GPU in 2.0399s (17.648GiB/s) | 20260507 02:03:11.737 | INFO | Downloaded 38GiB to GPU in 2.18801s (17.3673GiB/s) | 20260507 02:03:16.862 | INFO | Analyzing chunk frames = [6226432,6226944) | 20260507 02:03:16.862 | INFO | MemInfo: { | 20260507 02:03:16.862 | INFO | "gpu" : " 17 GB", | 20260507 02:03:16.862 | INFO | "huge" : "", | 20260507 02:03:16.862 | INFO | "huge_shared" : " 27 GB", | 20260507 02:03:16.862 | INFO | "malloc" : " 14 GB", | 20260507 02:03:16.862 | INFO | "pinned" : " 29 GB" | 20260507 02:03:16.863 | INFO | } | 20260507 02:03:16.863 | INFO | Perf report for the last 5.1264 seconds: | 20260507 02:03:16.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:03:16.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:16.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:16.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:16.863 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 02:03:16.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:16.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:16.863 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:03:16.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:16.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:16.863 | INFO | Uploaded 36GiB to GPU in 2.04086s (17.6396GiB/s) | 20260507 02:03:16.863 | INFO | Downloaded 34GiB to GPU in 1.93705s (17.5525GiB/s) | 20260507 02:03:22.008 | INFO | Analyzing chunk frames = [6226944,6227456) | 20260507 02:03:22.008 | INFO | MemInfo: { | 20260507 02:03:22.008 | INFO | "gpu" : " 17 GB", | 20260507 02:03:22.008 | INFO | "huge" : "", | 20260507 02:03:22.008 | INFO | "huge_shared" : " 27 GB", | 20260507 02:03:22.008 | INFO | "malloc" : " 14 GB", | 20260507 02:03:22.008 | INFO | "pinned" : " 29 GB" | 20260507 02:03:22.008 | INFO | } | 20260507 02:03:22.008 | INFO | Perf report for the last 5.14558 seconds: | 20260507 02:03:22.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:03:22.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:22.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:22.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:22.008 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260507 02:03:22.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:22.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:22.008 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 02:03:22.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:22.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:22.008 | INFO | Uploaded 36GiB to GPU in 2.0038s (17.9658GiB/s) | 20260507 02:03:22.008 | INFO | Downloaded 34GiB to GPU in 1.93454s (17.5752GiB/s) | 20260507 02:03:27.152 | INFO | Analyzing chunk frames = [6227456,6227968) | 20260507 02:03:27.152 | INFO | MemInfo: { | 20260507 02:03:27.152 | INFO | "gpu" : " 17 GB", | 20260507 02:03:27.152 | INFO | "huge" : "", | 20260507 02:03:27.152 | INFO | "huge_shared" : " 27 GB", | 20260507 02:03:27.152 | INFO | "malloc" : " 14 GB", | 20260507 02:03:27.152 | INFO | "pinned" : " 29 GB" | 20260507 02:03:27.152 | INFO | } | 20260507 02:03:27.152 | INFO | Perf report for the last 5.14433 seconds: | 20260507 02:03:27.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:03:27.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:27.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:27.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:27.152 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 02:03:27.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:27.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:27.153 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:03:27.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:27.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:27.153 | INFO | Uploaded 36GiB to GPU in 1.98145s (18.1685GiB/s) | 20260507 02:03:27.153 | INFO | Downloaded 34GiB to GPU in 1.91206s (17.7818GiB/s) | 20260507 02:03:32.292 | INFO | Analyzing chunk frames = [6227968,6228480) | 20260507 02:03:32.292 | INFO | MemInfo: { | 20260507 02:03:32.292 | INFO | "gpu" : " 17 GB", | 20260507 02:03:32.292 | INFO | "huge" : "", | 20260507 02:03:32.292 | INFO | "huge_shared" : " 27 GB", | 20260507 02:03:32.292 | INFO | "malloc" : " 14 GB", | 20260507 02:03:32.292 | INFO | "pinned" : " 29 GB" | 20260507 02:03:32.292 | INFO | } | 20260507 02:03:32.292 | INFO | Perf report for the last 5.13965 seconds: | 20260507 02:03:32.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:03:32.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:32.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:32.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:32.292 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 02:03:32.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:32.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:32.292 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:03:32.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:32.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:32.292 | INFO | Uploaded 36GiB to GPU in 2.04573s (17.5976GiB/s) | 20260507 02:03:32.292 | INFO | Downloaded 34GiB to GPU in 1.9333s (17.5865GiB/s) | 20260507 02:03:37.425 | INFO | Analyzing chunk frames = [6228480,6228992) | 20260507 02:03:37.425 | INFO | MemInfo: { | 20260507 02:03:37.425 | INFO | "gpu" : " 17 GB", | 20260507 02:03:37.425 | INFO | "huge" : "", | 20260507 02:03:37.425 | INFO | "huge_shared" : " 27 GB", | 20260507 02:03:37.425 | INFO | "malloc" : " 14 GB", | 20260507 02:03:37.425 | INFO | "pinned" : " 29 GB" | 20260507 02:03:37.425 | INFO | } | 20260507 02:03:37.425 | INFO | Perf report for the last 5.13328 seconds: | 20260507 02:03:37.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:03:37.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:37.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:37.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:37.426 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 02:03:37.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:37.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:37.426 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 02:03:37.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:37.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:37.426 | INFO | Uploaded 36GiB to GPU in 2.04185s (17.6311GiB/s) | 20260507 02:03:37.426 | INFO | Downloaded 34GiB to GPU in 1.91044s (17.7969GiB/s) | 20260507 02:03:42.563 | INFO | Analyzing chunk frames = [6228992,6229504) | 20260507 02:03:42.563 | INFO | MemInfo: { | 20260507 02:03:42.563 | INFO | "gpu" : " 17 GB", | 20260507 02:03:42.563 | INFO | "huge" : "", | 20260507 02:03:42.563 | INFO | "huge_shared" : " 27 GB", | 20260507 02:03:42.563 | INFO | "malloc" : " 14 GB", | 20260507 02:03:42.563 | INFO | "pinned" : " 29 GB" | 20260507 02:03:42.563 | INFO | } | 20260507 02:03:42.564 | INFO | Perf report for the last 5.13807 seconds: | 20260507 02:03:42.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:03:42.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:42.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:42.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:42.564 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:03:42.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:42.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:42.564 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:03:42.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:42.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:42.564 | INFO | Uploaded 36GiB to GPU in 1.98541s (18.1323GiB/s) | 20260507 02:03:42.564 | INFO | Downloaded 34GiB to GPU in 1.97332s (17.2299GiB/s) | 20260507 02:03:47.701 | INFO | Analyzing chunk frames = [6229504,6230016) | 20260507 02:03:47.701 | INFO | MemInfo: { | 20260507 02:03:47.701 | INFO | "gpu" : " 17 GB", | 20260507 02:03:47.701 | INFO | "huge" : "", | 20260507 02:03:47.701 | INFO | "huge_shared" : " 27 GB", | 20260507 02:03:47.701 | INFO | "malloc" : " 14 GB", | 20260507 02:03:47.701 | INFO | "pinned" : " 29 GB" | 20260507 02:03:47.701 | INFO | } | 20260507 02:03:47.701 | INFO | Perf report for the last 5.13793 seconds: | 20260507 02:03:47.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:03:47.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:47.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:47.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:47.701 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 02:03:47.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:47.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:47.702 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:03:47.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:47.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:47.702 | INFO | Uploaded 36GiB to GPU in 1.9587s (18.3796GiB/s) | 20260507 02:03:47.702 | INFO | Downloaded 34GiB to GPU in 1.9595s (17.3514GiB/s) | 20260507 02:03:47.798 | INFO | Starting a round of Baz IO | 20260507 02:03:50.917 | INFO | Populating RTMetrics report | 20260507 02:03:50.917 | INFO | RTMetrics report populated | 20260507 02:03:50.917 | INFO | Opening tmp RTMetrics json file | 20260507 02:03:50.923 | INFO | Serializing RTMetrics to json | 20260507 02:03:50.935 | INFO | Serialization done, swapping tmp file now... | 20260507 02:03:50.936 | INFO | RTMetrics done | 20260507 02:03:51.005 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.207 seconds | 20260507 02:03:51.005 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.05% / 0.00% / 66.95% | 20260507 02:03:51.005 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.258056% / 7.94729e-06% / 9.53674e-05% | 20260507 02:03:51.005 | INFO | Baz IO overall: | 20260507 02:03:51.005 | INFO | BytesWritten : 3221907122552 | 20260507 02:03:51.005 | INFO | NumEvents : 1550002513746 | 20260507 02:03:52.840 | INFO | Analyzing chunk frames = [6230016,6230528) | 20260507 02:03:52.840 | INFO | MemInfo: { | 20260507 02:03:52.840 | INFO | "gpu" : " 17 GB", | 20260507 02:03:52.840 | INFO | "huge" : "", | 20260507 02:03:52.841 | INFO | "huge_shared" : " 27 GB", | 20260507 02:03:52.841 | INFO | "malloc" : " 14 GB", | 20260507 02:03:52.841 | INFO | "pinned" : " 29 GB" | 20260507 02:03:52.841 | INFO | } | 20260507 02:03:52.841 | INFO | Perf report for the last 5.13913 seconds: | 20260507 02:03:52.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:03:52.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:52.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:52.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:52.841 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 02:03:52.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:52.841 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 02:03:52.841 | INFO | PRE_HQ : 60.2%, 1.8, 56.5ms, 0 | 20260507 02:03:52.841 | INFO | BAZWRITER : 62.0%, 3.3, 103.9ms, 0 | 20260507 02:03:52.841 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:03:52.841 | INFO | Uploaded 36GiB to GPU in 2.01546s (17.8619GiB/s) | 20260507 02:03:52.841 | INFO | Downloaded 38GiB to GPU in 2.17382s (17.4807GiB/s) | 20260507 02:03:57.985 | INFO | Analyzing chunk frames = [6230528,6231040) | 20260507 02:03:57.985 | INFO | MemInfo: { | 20260507 02:03:57.985 | INFO | "gpu" : " 17 GB", | 20260507 02:03:57.985 | INFO | "huge" : "", | 20260507 02:03:57.985 | INFO | "huge_shared" : " 27 GB", | 20260507 02:03:57.985 | INFO | "malloc" : " 14 GB", | 20260507 02:03:57.985 | INFO | "pinned" : " 29 GB" | 20260507 02:03:57.985 | INFO | } | 20260507 02:03:57.985 | INFO | Perf report for the last 5.14466 seconds: | 20260507 02:03:57.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:03:57.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:57.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:57.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:57.985 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:03:57.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:57.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:57.985 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:03:57.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:57.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:03:57.986 | INFO | Uploaded 36GiB to GPU in 1.99175s (18.0746GiB/s) | 20260507 02:03:57.986 | INFO | Downloaded 34GiB to GPU in 1.93004s (17.6162GiB/s) | 20260507 02:04:03.124 | INFO | Analyzing chunk frames = [6231040,6231552) | 20260507 02:04:03.124 | INFO | MemInfo: { | 20260507 02:04:03.124 | INFO | "gpu" : " 17 GB", | 20260507 02:04:03.124 | INFO | "huge" : "", | 20260507 02:04:03.124 | INFO | "huge_shared" : " 27 GB", | 20260507 02:04:03.124 | INFO | "malloc" : " 14 GB", | 20260507 02:04:03.124 | INFO | "pinned" : " 29 GB" | 20260507 02:04:03.124 | INFO | } | 20260507 02:04:03.124 | INFO | Perf report for the last 5.13846 seconds: | 20260507 02:04:03.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:04:03.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:03.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:03.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:03.124 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 02:04:03.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:03.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:03.124 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:04:03.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:03.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:03.124 | INFO | Uploaded 36GiB to GPU in 2.00318s (17.9715GiB/s) | 20260507 02:04:03.124 | INFO | Downloaded 34GiB to GPU in 1.94052s (17.521GiB/s) | 20260507 02:04:08.263 | INFO | Analyzing chunk frames = [6231552,6232064) | 20260507 02:04:08.263 | INFO | MemInfo: { | 20260507 02:04:08.263 | INFO | "gpu" : " 17 GB", | 20260507 02:04:08.263 | INFO | "huge" : "", | 20260507 02:04:08.263 | INFO | "huge_shared" : " 27 GB", | 20260507 02:04:08.263 | INFO | "malloc" : " 14 GB", | 20260507 02:04:08.263 | INFO | "pinned" : " 29 GB" | 20260507 02:04:08.263 | INFO | } | 20260507 02:04:08.263 | INFO | Perf report for the last 5.13913 seconds: | 20260507 02:04:08.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:04:08.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:08.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:08.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:08.263 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:04:08.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:08.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:08.263 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 02:04:08.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:08.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:08.263 | INFO | Uploaded 36GiB to GPU in 1.96569s (18.3142GiB/s) | 20260507 02:04:08.263 | INFO | Downloaded 34GiB to GPU in 1.92187s (17.6911GiB/s) | 20260507 02:04:13.410 | INFO | Analyzing chunk frames = [6232064,6232576) | 20260507 02:04:13.410 | INFO | MemInfo: { | 20260507 02:04:13.410 | INFO | "gpu" : " 17 GB", | 20260507 02:04:13.410 | INFO | "huge" : "", | 20260507 02:04:13.410 | INFO | "huge_shared" : " 27 GB", | 20260507 02:04:13.410 | INFO | "malloc" : " 14 GB", | 20260507 02:04:13.410 | INFO | "pinned" : " 29 GB" | 20260507 02:04:13.410 | INFO | } | 20260507 02:04:13.410 | INFO | Perf report for the last 5.14739 seconds: | 20260507 02:04:13.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:04:13.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:13.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:13.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:13.410 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 02:04:13.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:13.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:13.411 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 02:04:13.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:13.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:13.411 | INFO | Uploaded 36GiB to GPU in 2.01582s (17.8588GiB/s) | 20260507 02:04:13.411 | INFO | Downloaded 34GiB to GPU in 1.94183s (17.5092GiB/s) | 20260507 02:04:18.541 | INFO | Analyzing chunk frames = [6232576,6233088) | 20260507 02:04:18.541 | INFO | MemInfo: { | 20260507 02:04:18.541 | INFO | "gpu" : " 17 GB", | 20260507 02:04:18.541 | INFO | "huge" : "", | 20260507 02:04:18.541 | INFO | "huge_shared" : " 27 GB", | 20260507 02:04:18.541 | INFO | "malloc" : " 14 GB", | 20260507 02:04:18.541 | INFO | "pinned" : " 29 GB" | 20260507 02:04:18.541 | INFO | } | 20260507 02:04:18.541 | INFO | Perf report for the last 5.13053 seconds: | 20260507 02:04:18.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:04:18.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:18.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:18.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:18.541 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 02:04:18.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:18.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:18.541 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:04:18.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:18.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:18.541 | INFO | Uploaded 36GiB to GPU in 2.04733s (17.5839GiB/s) | 20260507 02:04:18.541 | INFO | Downloaded 34GiB to GPU in 1.95551s (17.3868GiB/s) | 20260507 02:04:23.676 | INFO | Analyzing chunk frames = [6233088,6233600) | 20260507 02:04:23.676 | INFO | MemInfo: { | 20260507 02:04:23.676 | INFO | "gpu" : " 17 GB", | 20260507 02:04:23.676 | INFO | "huge" : "", | 20260507 02:04:23.676 | INFO | "huge_shared" : " 27 GB", | 20260507 02:04:23.676 | INFO | "malloc" : " 14 GB", | 20260507 02:04:23.676 | INFO | "pinned" : " 29 GB" | 20260507 02:04:23.676 | INFO | } | 20260507 02:04:23.676 | INFO | Perf report for the last 5.13555 seconds: | 20260507 02:04:23.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:04:23.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:23.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:23.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:23.676 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 02:04:23.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:23.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:23.677 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 02:04:23.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:23.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:23.677 | INFO | Uploaded 36GiB to GPU in 1.98734s (18.1147GiB/s) | 20260507 02:04:23.677 | INFO | Downloaded 34GiB to GPU in 1.9882s (17.1009GiB/s) | 20260507 02:04:28.818 | INFO | Analyzing chunk frames = [6233600,6234112) | 20260507 02:04:28.818 | INFO | MemInfo: { | 20260507 02:04:28.818 | INFO | "gpu" : " 17 GB", | 20260507 02:04:28.818 | INFO | "huge" : "", | 20260507 02:04:28.818 | INFO | "huge_shared" : " 27 GB", | 20260507 02:04:28.818 | INFO | "malloc" : " 14 GB", | 20260507 02:04:28.818 | INFO | "pinned" : " 29 GB" | 20260507 02:04:28.818 | INFO | } | 20260507 02:04:28.818 | INFO | Perf report for the last 5.14159 seconds: | 20260507 02:04:28.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:04:28.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:28.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:28.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:28.818 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:04:28.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:28.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:28.818 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:04:28.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:28.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:28.818 | INFO | Uploaded 36GiB to GPU in 1.98529s (18.1334GiB/s) | 20260507 02:04:28.818 | INFO | Downloaded 34GiB to GPU in 1.97645s (17.2026GiB/s) | 20260507 02:04:28.818 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:04:28.818 | INFO | Total mem usage: 18 GiB | 20260507 02:04:28.818 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:04:28.818 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:04:28.818 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:04:28.818 | INFO | High Water Breakdown: | 20260507 02:04:28.818 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:04:28.819 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:04:28.819 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:04:28.819 | INFO | ------------------------------------------ | 20260507 02:04:28.819 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:04:28.819 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:04:28.819 | INFO | Total mem usage: 26.5 GiB | 20260507 02:04:28.819 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:04:28.819 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:04:28.819 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:04:28.819 | INFO | High Water Breakdown: | 20260507 02:04:28.819 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:04:28.819 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:04:28.819 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:04:28.819 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:04:28.819 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:04:28.819 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:04:28.819 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:04:28.819 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:04:28.819 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:04:28.819 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:04:28.819 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:04:28.819 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:04:28.819 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:04:28.819 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:04:28.819 | INFO | ------------------------------------------ | 20260507 02:04:28.819 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:04:28.819 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:04:28.819 | INFO | Total mem usage: 25.6 GiB | 20260507 02:04:28.819 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:04:28.819 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:04:28.819 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:04:28.819 | INFO | High Water Breakdown: | 20260507 02:04:28.819 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:04:28.819 | INFO | ------------------------------------------ | 20260507 02:04:28.819 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:04:28.819 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:04:28.819 | INFO | Total mem usage: 15.7 GiB | 20260507 02:04:28.819 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:04:28.819 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:04:28.819 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 02:04:28.819 | INFO | High Water Breakdown: | 20260507 02:04:28.819 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:04:28.819 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:04:28.819 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:04:28.819 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:04:28.819 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:04:28.819 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:04:28.819 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:04:28.819 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:04:28.819 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:04:28.819 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:04:28.819 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:04:28.819 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:04:28.819 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:04:28.819 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:04:28.819 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:04:28.819 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:04:28.819 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:04:28.819 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:04:28.819 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:04:28.819 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:04:28.819 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:04:28.819 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:04:28.819 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:04:28.819 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:04:28.819 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:04:28.819 | INFO | ------------------------------------------ | 20260507 02:04:28.819 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:04:28.819 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:04:28.819 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:04:28.819 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:04:28.819 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.52e+01, 1.33e+01 / 1.65e+00 | 20260507 02:04:28.819 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 1.01e-01, 1.02e-02 / 4.69e-03 | 20260507 02:04:28.819 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.52e+00, 5.26e-01 / 1.52e-01 | 20260507 02:04:28.819 | INFO | PerfReport: 3.24%, Histogram -- 9.99e-01 / 1.49e+00, 1.72e-01 / 8.81e-02 | 20260507 02:04:28.819 | INFO | PerfReport: 16.09%, DME -- 3.97e+01 / 4.43e+01, 3.96e+00 / 2.28e+00 | 20260507 02:04:28.819 | INFO | PerfReport: 57.56%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.35e+00 / 1.68e+00 | 20260507 02:04:28.819 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.16e+00 / 2.74e+00, 3.82e-01 / 1.89e-01 | 20260507 02:04:28.819 | INFO | PerfReport: 3.10%, Metrics -- 9.55e-01 / 1.29e+00, 2.57e-01 / 1.07e-01 | 20260507 02:04:28.819 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.24e-01 / 3.76e-01, 5.19e-02 / 1.97e-02 | 20260507 02:04:28.819 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:04:28.819 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:04:28.819 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:04:28.819 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 02:04:28.819 | INFO | PerfReport: 21.68%, Upload -- 2.09e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 02:04:28.819 | INFO | PerfReport: 21.32%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 02:04:28.819 | INFO | PerfReport: 32.00%, Compute -- 3.09e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 02:04:28.819 | INFO | PerfReport: 1.41%, LockUpload -- 1.36e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 02:04:28.819 | INFO | PerfReport: 1.29%, LockDownload -- 1.25e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260507 02:04:28.819 | INFO | PerfReport: 22.29%, LockCompute -- 2.15e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 02:04:28.819 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260507 02:04:28.916 | INFO | Starting a round of Baz IO | 20260507 02:04:32.036 | INFO | Populating RTMetrics report | 20260507 02:04:32.037 | INFO | RTMetrics report populated | 20260507 02:04:32.037 | INFO | Opening tmp RTMetrics json file | 20260507 02:04:32.044 | INFO | Serializing RTMetrics to json | 20260507 02:04:32.057 | INFO | Serialization done, swapping tmp file now... | 20260507 02:04:32.057 | INFO | RTMetrics done | 20260507 02:04:32.127 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.211 seconds | 20260507 02:04:32.128 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.05% / 0.00% / 66.95% | 20260507 02:04:32.128 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.255835% / 0% / 5.96046e-05% | 20260507 02:04:32.128 | INFO | Baz IO overall: | 20260507 02:04:32.128 | INFO | BytesWritten : 3223835966158 | 20260507 02:04:32.128 | INFO | NumEvents : 1550915595839 | 20260507 02:04:33.953 | INFO | Analyzing chunk frames = [6234112,6234624) | 20260507 02:04:33.953 | INFO | MemInfo: { | 20260507 02:04:33.953 | INFO | "gpu" : " 17 GB", | 20260507 02:04:33.953 | INFO | "huge" : "", | 20260507 02:04:33.953 | INFO | "huge_shared" : " 27 GB", | 20260507 02:04:33.953 | INFO | "malloc" : " 14 GB", | 20260507 02:04:33.953 | INFO | "pinned" : " 29 GB" | 20260507 02:04:33.953 | INFO | } | 20260507 02:04:33.953 | INFO | Perf report for the last 5.1354 seconds: | 20260507 02:04:33.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:04:33.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:33.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:33.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:33.953 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 02:04:33.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:33.954 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 02:04:33.954 | INFO | PRE_HQ : 59.7%, 1.8, 55.9ms, 0 | 20260507 02:04:33.954 | INFO | BAZWRITER : 62.1%, 3.3, 103.9ms, 0 | 20260507 02:04:33.954 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:04:33.954 | INFO | Uploaded 36GiB to GPU in 2.04568s (17.5981GiB/s) | 20260507 02:04:33.954 | INFO | Downloaded 38GiB to GPU in 2.19443s (17.3166GiB/s) | 20260507 02:04:39.096 | INFO | Analyzing chunk frames = [6234624,6235136) | 20260507 02:04:39.096 | INFO | MemInfo: { | 20260507 02:04:39.096 | INFO | "gpu" : " 17 GB", | 20260507 02:04:39.096 | INFO | "huge" : "", | 20260507 02:04:39.096 | INFO | "huge_shared" : " 27 GB", | 20260507 02:04:39.096 | INFO | "malloc" : " 14 GB", | 20260507 02:04:39.096 | INFO | "pinned" : " 29 GB" | 20260507 02:04:39.096 | INFO | } | 20260507 02:04:39.096 | INFO | Perf report for the last 5.14235 seconds: | 20260507 02:04:39.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:04:39.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:39.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:39.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:39.096 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:04:39.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:39.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:39.096 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:04:39.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:39.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:39.096 | INFO | Uploaded 36GiB to GPU in 1.99233s (18.0693GiB/s) | 20260507 02:04:39.096 | INFO | Downloaded 34GiB to GPU in 1.9338s (17.5819GiB/s) | 20260507 02:04:44.242 | INFO | Analyzing chunk frames = [6235136,6235648) | 20260507 02:04:44.242 | INFO | MemInfo: { | 20260507 02:04:44.242 | INFO | "gpu" : " 17 GB", | 20260507 02:04:44.242 | INFO | "huge" : "", | 20260507 02:04:44.242 | INFO | "huge_shared" : " 27 GB", | 20260507 02:04:44.242 | INFO | "malloc" : " 14 GB", | 20260507 02:04:44.242 | INFO | "pinned" : " 29 GB" | 20260507 02:04:44.242 | INFO | } | 20260507 02:04:44.242 | INFO | Perf report for the last 5.14634 seconds: | 20260507 02:04:44.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:04:44.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:44.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:44.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:44.242 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 02:04:44.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:44.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:44.242 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:04:44.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:44.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:44.242 | INFO | Uploaded 36GiB to GPU in 2.00185s (17.9833GiB/s) | 20260507 02:04:44.242 | INFO | Downloaded 34GiB to GPU in 1.9399s (17.5267GiB/s) | 20260507 02:04:49.380 | INFO | Analyzing chunk frames = [6235648,6236160) | 20260507 02:04:49.380 | INFO | MemInfo: { | 20260507 02:04:49.380 | INFO | "gpu" : " 17 GB", | 20260507 02:04:49.380 | INFO | "huge" : "", | 20260507 02:04:49.380 | INFO | "huge_shared" : " 27 GB", | 20260507 02:04:49.380 | INFO | "malloc" : " 14 GB", | 20260507 02:04:49.380 | INFO | "pinned" : " 29 GB" | 20260507 02:04:49.380 | INFO | } | 20260507 02:04:49.380 | INFO | Perf report for the last 5.13769 seconds: | 20260507 02:04:49.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:04:49.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:49.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:49.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:49.380 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:04:49.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:49.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:49.380 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:04:49.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:49.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:49.380 | INFO | Uploaded 36GiB to GPU in 1.98412s (18.1441GiB/s) | 20260507 02:04:49.380 | INFO | Downloaded 34GiB to GPU in 1.9316s (17.602GiB/s) | 20260507 02:04:54.515 | INFO | Analyzing chunk frames = [6236160,6236672) | 20260507 02:04:54.515 | INFO | MemInfo: { | 20260507 02:04:54.515 | INFO | "gpu" : " 17 GB", | 20260507 02:04:54.515 | INFO | "huge" : "", | 20260507 02:04:54.515 | INFO | "huge_shared" : " 27 GB", | 20260507 02:04:54.515 | INFO | "malloc" : " 14 GB", | 20260507 02:04:54.515 | INFO | "pinned" : " 29 GB" | 20260507 02:04:54.515 | INFO | } | 20260507 02:04:54.515 | INFO | Perf report for the last 5.13555 seconds: | 20260507 02:04:54.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:04:54.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:54.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:54.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:54.515 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 02:04:54.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:54.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:54.516 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:04:54.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:54.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:54.516 | INFO | Uploaded 36GiB to GPU in 2.0403s (17.6444GiB/s) | 20260507 02:04:54.516 | INFO | Downloaded 34GiB to GPU in 1.93965s (17.5289GiB/s) | 20260507 02:04:59.652 | INFO | Analyzing chunk frames = [6236672,6237184) | 20260507 02:04:59.652 | INFO | MemInfo: { | 20260507 02:04:59.652 | INFO | "gpu" : " 17 GB", | 20260507 02:04:59.652 | INFO | "huge" : "", | 20260507 02:04:59.652 | INFO | "huge_shared" : " 27 GB", | 20260507 02:04:59.652 | INFO | "malloc" : " 14 GB", | 20260507 02:04:59.652 | INFO | "pinned" : " 29 GB" | 20260507 02:04:59.652 | INFO | } | 20260507 02:04:59.652 | INFO | Perf report for the last 5.13661 seconds: | 20260507 02:04:59.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:04:59.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:59.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:59.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:59.652 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 02:04:59.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:59.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:59.652 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 02:04:59.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:59.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:04:59.652 | INFO | Uploaded 36GiB to GPU in 2.05082s (17.5539GiB/s) | 20260507 02:04:59.652 | INFO | Downloaded 34GiB to GPU in 1.96341s (17.3168GiB/s) | 20260507 02:05:04.791 | INFO | Analyzing chunk frames = [6237184,6237696) | 20260507 02:05:04.791 | INFO | MemInfo: { | 20260507 02:05:04.791 | INFO | "gpu" : " 17 GB", | 20260507 02:05:04.791 | INFO | "huge" : "", | 20260507 02:05:04.791 | INFO | "huge_shared" : " 27 GB", | 20260507 02:05:04.791 | INFO | "malloc" : " 14 GB", | 20260507 02:05:04.791 | INFO | "pinned" : " 29 GB" | 20260507 02:05:04.791 | INFO | } | 20260507 02:05:04.792 | INFO | Perf report for the last 5.13948 seconds: | 20260507 02:05:04.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:05:04.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:04.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:04.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:04.792 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:05:04.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:04.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:04.792 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:05:04.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:04.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:04.792 | INFO | Uploaded 36GiB to GPU in 1.97252s (18.2508GiB/s) | 20260507 02:05:04.792 | INFO | Downloaded 34GiB to GPU in 1.99697s (17.0258GiB/s) | 20260507 02:05:09.931 | INFO | Analyzing chunk frames = [6237696,6238208) | 20260507 02:05:09.931 | INFO | MemInfo: { | 20260507 02:05:09.931 | INFO | "gpu" : " 17 GB", | 20260507 02:05:09.931 | INFO | "huge" : "", | 20260507 02:05:09.931 | INFO | "huge_shared" : " 27 GB", | 20260507 02:05:09.931 | INFO | "malloc" : " 14 GB", | 20260507 02:05:09.931 | INFO | "pinned" : " 29 GB" | 20260507 02:05:09.931 | INFO | } | 20260507 02:05:09.932 | INFO | Perf report for the last 5.13999 seconds: | 20260507 02:05:09.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:05:09.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:09.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:09.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:09.932 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:05:09.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:09.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:09.932 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:05:09.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:09.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:09.932 | INFO | Uploaded 36GiB to GPU in 1.993s (18.0632GiB/s) | 20260507 02:05:09.932 | INFO | Downloaded 34GiB to GPU in 1.97046s (17.2549GiB/s) | 20260507 02:05:10.028 | INFO | Starting a round of Baz IO | 20260507 02:05:13.143 | INFO | Populating RTMetrics report | 20260507 02:05:13.143 | INFO | RTMetrics report populated | 20260507 02:05:13.143 | INFO | Opening tmp RTMetrics json file | 20260507 02:05:13.144 | INFO | Serializing RTMetrics to json | 20260507 02:05:13.163 | INFO | Serialization done, swapping tmp file now... | 20260507 02:05:13.163 | INFO | RTMetrics done | 20260507 02:05:13.230 | INFO | Finished a round of Baz IO: Wrote 1.798 GiB in 3.201 seconds | 20260507 02:05:13.230 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.05% / 0.00% / 66.95% | 20260507 02:05:13.230 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.256944% / 1.19209e-05% / 0.000111262% | 20260507 02:05:13.230 | INFO | Baz IO overall: | 20260507 02:05:13.230 | INFO | BytesWritten : 3225766259921 | 20260507 02:05:13.230 | INFO | NumEvents : 1551829936008 | 20260507 02:05:15.079 | INFO | Analyzing chunk frames = [6238208,6238720) | 20260507 02:05:15.079 | INFO | MemInfo: { | 20260507 02:05:15.079 | INFO | "gpu" : " 17 GB", | 20260507 02:05:15.079 | INFO | "huge" : "", | 20260507 02:05:15.079 | INFO | "huge_shared" : " 27 GB", | 20260507 02:05:15.079 | INFO | "malloc" : " 14 GB", | 20260507 02:05:15.079 | INFO | "pinned" : " 29 GB" | 20260507 02:05:15.079 | INFO | } | 20260507 02:05:15.079 | INFO | Perf report for the last 5.14751 seconds: | 20260507 02:05:15.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:05:15.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:15.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:15.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:15.079 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260507 02:05:15.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:15.079 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 02:05:15.079 | INFO | PRE_HQ : 59.4%, 1.8, 56.0ms, 0 | 20260507 02:05:15.079 | INFO | BAZWRITER : 61.8%, 3.3, 104.4ms, 0 | 20260507 02:05:15.079 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:05:15.079 | INFO | Uploaded 36GiB to GPU in 2.03144s (17.7214GiB/s) | 20260507 02:05:15.079 | INFO | Downloaded 38GiB to GPU in 2.1803s (17.4288GiB/s) | 20260507 02:05:20.214 | INFO | Analyzing chunk frames = [6238720,6239232) | 20260507 02:05:20.214 | INFO | MemInfo: { | 20260507 02:05:20.214 | INFO | "gpu" : " 17 GB", | 20260507 02:05:20.214 | INFO | "huge" : "", | 20260507 02:05:20.214 | INFO | "huge_shared" : " 27 GB", | 20260507 02:05:20.214 | INFO | "malloc" : " 14 GB", | 20260507 02:05:20.214 | INFO | "pinned" : " 29 GB" | 20260507 02:05:20.214 | INFO | } | 20260507 02:05:20.214 | INFO | Perf report for the last 5.13475 seconds: | 20260507 02:05:20.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:05:20.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:20.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:20.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:20.214 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:05:20.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:20.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:20.214 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:05:20.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:20.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:20.214 | INFO | Uploaded 36GiB to GPU in 2.04781s (17.5798GiB/s) | 20260507 02:05:20.214 | INFO | Downloaded 34GiB to GPU in 1.93523s (17.569GiB/s) | 20260507 02:05:25.356 | INFO | Analyzing chunk frames = [6239232,6239744) | 20260507 02:05:25.356 | INFO | MemInfo: { | 20260507 02:05:25.356 | INFO | "gpu" : " 17 GB", | 20260507 02:05:25.356 | INFO | "huge" : "", | 20260507 02:05:25.356 | INFO | "huge_shared" : " 27 GB", | 20260507 02:05:25.356 | INFO | "malloc" : " 14 GB", | 20260507 02:05:25.356 | INFO | "pinned" : " 29 GB" | 20260507 02:05:25.356 | INFO | } | 20260507 02:05:25.356 | INFO | Perf report for the last 5.14252 seconds: | 20260507 02:05:25.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:05:25.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:25.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:25.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:25.357 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 02:05:25.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:25.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:25.357 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:05:25.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:25.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:25.357 | INFO | Uploaded 36GiB to GPU in 2.00841s (17.9246GiB/s) | 20260507 02:05:25.357 | INFO | Downloaded 34GiB to GPU in 1.93501s (17.5709GiB/s) | 20260507 02:05:30.493 | INFO | Analyzing chunk frames = [6239744,6240256) | 20260507 02:05:30.493 | INFO | MemInfo: { | 20260507 02:05:30.493 | INFO | "gpu" : " 17 GB", | 20260507 02:05:30.493 | INFO | "huge" : "", | 20260507 02:05:30.493 | INFO | "huge_shared" : " 27 GB", | 20260507 02:05:30.493 | INFO | "malloc" : " 14 GB", | 20260507 02:05:30.493 | INFO | "pinned" : " 29 GB" | 20260507 02:05:30.494 | INFO | } | 20260507 02:05:30.494 | INFO | Perf report for the last 5.13723 seconds: | 20260507 02:05:30.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:05:30.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:30.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:30.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:30.494 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 02:05:30.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:30.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:30.494 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:05:30.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:30.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:30.494 | INFO | Uploaded 36GiB to GPU in 1.98633s (18.1239GiB/s) | 20260507 02:05:30.494 | INFO | Downloaded 34GiB to GPU in 1.91681s (17.7378GiB/s) | 20260507 02:05:35.627 | INFO | Analyzing chunk frames = [6240256,6240768) | 20260507 02:05:35.627 | INFO | MemInfo: { | 20260507 02:05:35.627 | INFO | "gpu" : " 17 GB", | 20260507 02:05:35.627 | INFO | "huge" : "", | 20260507 02:05:35.627 | INFO | "huge_shared" : " 27 GB", | 20260507 02:05:35.627 | INFO | "malloc" : " 14 GB", | 20260507 02:05:35.627 | INFO | "pinned" : " 29 GB" | 20260507 02:05:35.627 | INFO | } | 20260507 02:05:35.628 | INFO | Perf report for the last 5.13422 seconds: | 20260507 02:05:35.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:05:35.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:35.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:35.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:35.628 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 02:05:35.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:35.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:35.628 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:05:35.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:35.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:35.628 | INFO | Uploaded 36GiB to GPU in 2.02033s (17.8189GiB/s) | 20260507 02:05:35.628 | INFO | Downloaded 34GiB to GPU in 1.94485s (17.482GiB/s) | 20260507 02:05:40.767 | INFO | Analyzing chunk frames = [6240768,6241280) | 20260507 02:05:40.767 | INFO | MemInfo: { | 20260507 02:05:40.767 | INFO | "gpu" : " 17 GB", | 20260507 02:05:40.767 | INFO | "huge" : "", | 20260507 02:05:40.767 | INFO | "huge_shared" : " 27 GB", | 20260507 02:05:40.767 | INFO | "malloc" : " 14 GB", | 20260507 02:05:40.767 | INFO | "pinned" : " 29 GB" | 20260507 02:05:40.767 | INFO | } | 20260507 02:05:40.767 | INFO | Perf report for the last 5.13896 seconds: | 20260507 02:05:40.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:05:40.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:40.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:40.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:40.767 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:05:40.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:40.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:40.767 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 02:05:40.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:40.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:40.767 | INFO | Uploaded 36GiB to GPU in 2.05808s (17.4921GiB/s) | 20260507 02:05:40.767 | INFO | Downloaded 34GiB to GPU in 1.95906s (17.3553GiB/s) | 20260507 02:05:45.906 | INFO | Analyzing chunk frames = [6241280,6241792) | 20260507 02:05:45.906 | INFO | MemInfo: { | 20260507 02:05:45.906 | INFO | "gpu" : " 17 GB", | 20260507 02:05:45.906 | INFO | "huge" : "", | 20260507 02:05:45.906 | INFO | "huge_shared" : " 27 GB", | 20260507 02:05:45.906 | INFO | "malloc" : " 14 GB", | 20260507 02:05:45.906 | INFO | "pinned" : " 29 GB" | 20260507 02:05:45.906 | INFO | } | 20260507 02:05:45.906 | INFO | Perf report for the last 5.13934 seconds: | 20260507 02:05:45.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:05:45.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:45.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:45.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:45.906 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:05:45.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:45.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:45.906 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 02:05:45.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:45.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:45.906 | INFO | Uploaded 36GiB to GPU in 1.97981s (18.1835GiB/s) | 20260507 02:05:45.906 | INFO | Downloaded 34GiB to GPU in 1.99777s (17.019GiB/s) | 20260507 02:05:51.052 | INFO | Analyzing chunk frames = [6241792,6242304) | 20260507 02:05:51.052 | INFO | MemInfo: { | 20260507 02:05:51.052 | INFO | "gpu" : " 17 GB", | 20260507 02:05:51.052 | INFO | "huge" : "", | 20260507 02:05:51.052 | INFO | "huge_shared" : " 27 GB", | 20260507 02:05:51.052 | INFO | "malloc" : " 14 GB", | 20260507 02:05:51.052 | INFO | "pinned" : " 29 GB" | 20260507 02:05:51.053 | INFO | } | 20260507 02:05:51.053 | INFO | Perf report for the last 5.14656 seconds: | 20260507 02:05:51.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:05:51.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:51.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:51.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:51.053 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 02:05:51.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:51.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:51.053 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 02:05:51.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:51.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:51.053 | INFO | Uploaded 36GiB to GPU in 1.98094s (18.1732GiB/s) | 20260507 02:05:51.053 | INFO | Downloaded 34GiB to GPU in 1.98755s (17.1065GiB/s) | 20260507 02:05:51.053 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:05:51.053 | INFO | Total mem usage: 18 GiB | 20260507 02:05:51.053 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:05:51.053 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:05:51.053 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:05:51.053 | INFO | High Water Breakdown: | 20260507 02:05:51.053 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:05:51.053 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:05:51.053 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:05:51.053 | INFO | ------------------------------------------ | 20260507 02:05:51.053 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:05:51.053 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:05:51.053 | INFO | Total mem usage: 26.5 GiB | 20260507 02:05:51.053 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:05:51.053 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:05:51.053 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:05:51.053 | INFO | High Water Breakdown: | 20260507 02:05:51.054 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:05:51.054 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:05:51.054 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:05:51.054 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:05:51.054 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:05:51.054 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:05:51.054 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:05:51.054 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:05:51.054 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:05:51.054 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:05:51.054 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:05:51.054 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:05:51.054 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:05:51.054 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:05:51.054 | INFO | ------------------------------------------ | 20260507 02:05:51.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:05:51.054 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:05:51.054 | INFO | Total mem usage: 25.6 GiB | 20260507 02:05:51.054 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:05:51.054 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:05:51.054 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:05:51.054 | INFO | High Water Breakdown: | 20260507 02:05:51.054 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:05:51.054 | INFO | ------------------------------------------ | 20260507 02:05:51.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:05:51.054 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:05:51.054 | INFO | Total mem usage: 15.7 GiB | 20260507 02:05:51.054 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:05:51.054 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:05:51.054 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 02:05:51.054 | INFO | High Water Breakdown: | 20260507 02:05:51.054 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:05:51.054 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:05:51.054 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:05:51.054 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:05:51.054 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:05:51.054 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:05:51.054 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:05:51.054 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:05:51.054 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:05:51.054 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:05:51.054 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:05:51.054 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:05:51.054 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:05:51.054 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:05:51.054 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:05:51.054 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:05:51.054 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:05:51.054 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:05:51.054 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:05:51.054 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:05:51.054 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:05:51.054 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:05:51.054 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:05:51.054 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:05:51.054 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:05:51.054 | INFO | ------------------------------------------ | 20260507 02:05:51.054 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:05:51.054 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:05:51.054 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:05:51.054 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:05:51.054 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.55e+01, 1.33e+01 / 1.96e+00 | 20260507 02:05:51.054 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 1.01e-01, 9.47e-03 / 6.19e-03 | 20260507 02:05:51.054 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.46e+00, 5.21e-01 / 2.21e-01 | 20260507 02:05:51.054 | INFO | PerfReport: 3.21%, Histogram -- 9.90e-01 / 1.45e+00, 1.65e-01 / 9.94e-02 | 20260507 02:05:51.054 | INFO | PerfReport: 16.16%, DME -- 3.98e+01 / 4.50e+01, 4.15e+00 / 2.48e+00 | 20260507 02:05:51.054 | INFO | PerfReport: 57.47%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.24e+00 / 1.48e+00 | 20260507 02:05:51.054 | INFO | PerfReport: 7.05%, PulseAccumulating -- 2.17e+00 / 2.75e+00, 3.91e-01 / 2.05e-01 | 20260507 02:05:51.054 | INFO | PerfReport: 3.10%, Metrics -- 9.55e-01 / 1.28e+00, 2.53e-01 / 1.10e-01 | 20260507 02:05:51.054 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.24e-01 / 3.81e-01, 5.13e-02 / 2.23e-02 | 20260507 02:05:51.054 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:05:51.054 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:05:51.054 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:05:51.054 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 02:05:51.054 | INFO | PerfReport: 21.72%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260507 02:05:51.054 | INFO | PerfReport: 21.41%, Download -- 2.07e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260507 02:05:51.054 | INFO | PerfReport: 31.94%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 02:05:51.054 | INFO | PerfReport: 1.43%, LockUpload -- 1.38e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 02:05:51.054 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260507 02:05:51.054 | INFO | PerfReport: 22.19%, LockCompute -- 2.14e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 02:05:51.054 | INFO | PerfReport: 0.02%, Misc -- 1.45e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260507 02:05:51.150 | INFO | Starting a round of Baz IO | 20260507 02:05:54.262 | INFO | Populating RTMetrics report | 20260507 02:05:54.262 | INFO | RTMetrics report populated | 20260507 02:05:54.262 | INFO | Opening tmp RTMetrics json file | 20260507 02:05:54.263 | INFO | Serializing RTMetrics to json | 20260507 02:05:54.264 | INFO | Serialization done, swapping tmp file now... | 20260507 02:05:54.265 | INFO | RTMetrics done | 20260507 02:05:54.353 | INFO | Finished a round of Baz IO: Wrote 1.795 GiB in 3.203 seconds | 20260507 02:05:54.353 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.05% / 0.00% / 66.95% | 20260507 02:05:54.353 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.255124% / 1.19209e-05% / 0.000107288% | 20260507 02:05:54.353 | INFO | Baz IO overall: | 20260507 02:05:54.353 | INFO | BytesWritten : 3227693652797 | 20260507 02:05:54.353 | INFO | NumEvents : 1552742135884 | 20260507 02:05:56.197 | INFO | Analyzing chunk frames = [6242304,6242816) | 20260507 02:05:56.197 | INFO | MemInfo: { | 20260507 02:05:56.197 | INFO | "gpu" : " 17 GB", | 20260507 02:05:56.197 | INFO | "huge" : "", | 20260507 02:05:56.197 | INFO | "huge_shared" : " 27 GB", | 20260507 02:05:56.197 | INFO | "malloc" : " 14 GB", | 20260507 02:05:56.197 | INFO | "pinned" : " 29 GB" | 20260507 02:05:56.197 | INFO | } | 20260507 02:05:56.197 | INFO | Perf report for the last 5.14433 seconds: | 20260507 02:05:56.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:05:56.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:56.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:56.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:56.197 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260507 02:05:56.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:05:56.197 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 02:05:56.197 | INFO | PRE_HQ : 59.4%, 1.8, 55.7ms, 0 | 20260507 02:05:56.197 | INFO | BAZWRITER : 61.9%, 3.3, 104.9ms, 0 | 20260507 02:05:56.197 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:05:56.197 | INFO | Uploaded 36GiB to GPU in 2.04019s (17.6455GiB/s) | 20260507 02:05:56.198 | INFO | Downloaded 38GiB to GPU in 2.1933s (17.3255GiB/s) | 20260507 02:06:01.330 | INFO | Analyzing chunk frames = [6242816,6243328) | 20260507 02:06:01.330 | INFO | MemInfo: { | 20260507 02:06:01.330 | INFO | "gpu" : " 17 GB", | 20260507 02:06:01.330 | INFO | "huge" : "", | 20260507 02:06:01.330 | INFO | "huge_shared" : " 27 GB", | 20260507 02:06:01.330 | INFO | "malloc" : " 14 GB", | 20260507 02:06:01.330 | INFO | "pinned" : " 29 GB" | 20260507 02:06:01.330 | INFO | } | 20260507 02:06:01.330 | INFO | Perf report for the last 5.13286 seconds: | 20260507 02:06:01.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:06:01.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:01.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:01.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:01.330 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:06:01.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:01.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:01.330 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 02:06:01.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:01.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:01.330 | INFO | Uploaded 36GiB to GPU in 1.98428s (18.1426GiB/s) | 20260507 02:06:01.330 | INFO | Downloaded 34GiB to GPU in 1.9208s (17.701GiB/s) | 20260507 02:06:06.467 | INFO | Analyzing chunk frames = [6243328,6243840) | 20260507 02:06:06.467 | INFO | MemInfo: { | 20260507 02:06:06.467 | INFO | "gpu" : " 17 GB", | 20260507 02:06:06.467 | INFO | "huge" : "", | 20260507 02:06:06.467 | INFO | "huge_shared" : " 27 GB", | 20260507 02:06:06.467 | INFO | "malloc" : " 14 GB", | 20260507 02:06:06.467 | INFO | "pinned" : " 29 GB" | 20260507 02:06:06.467 | INFO | } | 20260507 02:06:06.467 | INFO | Perf report for the last 5.13703 seconds: | 20260507 02:06:06.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:06:06.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:06.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:06.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:06.467 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 02:06:06.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:06.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:06.467 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:06:06.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:06.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:06.467 | INFO | Uploaded 36GiB to GPU in 1.99328s (18.0607GiB/s) | 20260507 02:06:06.467 | INFO | Downloaded 34GiB to GPU in 1.93855s (17.5389GiB/s) | 20260507 02:06:11.604 | INFO | Analyzing chunk frames = [6243840,6244352) | 20260507 02:06:11.604 | INFO | MemInfo: { | 20260507 02:06:11.604 | INFO | "gpu" : " 17 GB", | 20260507 02:06:11.604 | INFO | "huge" : "", | 20260507 02:06:11.604 | INFO | "huge_shared" : " 27 GB", | 20260507 02:06:11.604 | INFO | "malloc" : " 14 GB", | 20260507 02:06:11.604 | INFO | "pinned" : " 29 GB" | 20260507 02:06:11.604 | INFO | } | 20260507 02:06:11.605 | INFO | Perf report for the last 5.13761 seconds: | 20260507 02:06:11.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:06:11.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:11.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:11.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:11.605 | INFO | ANALYSIS : 59.5%, 3.0, 92.2ms, 0 | 20260507 02:06:11.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:11.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:11.605 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:06:11.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:11.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:11.605 | INFO | Uploaded 36GiB to GPU in 1.96268s (18.3423GiB/s) | 20260507 02:06:11.605 | INFO | Downloaded 34GiB to GPU in 1.9038s (17.859GiB/s) | 20260507 02:06:16.744 | INFO | Analyzing chunk frames = [6244352,6244864) | 20260507 02:06:16.744 | INFO | MemInfo: { | 20260507 02:06:16.744 | INFO | "gpu" : " 17 GB", | 20260507 02:06:16.744 | INFO | "huge" : "", | 20260507 02:06:16.744 | INFO | "huge_shared" : " 27 GB", | 20260507 02:06:16.744 | INFO | "malloc" : " 14 GB", | 20260507 02:06:16.744 | INFO | "pinned" : " 29 GB" | 20260507 02:06:16.744 | INFO | } | 20260507 02:06:16.744 | INFO | Perf report for the last 5.13969 seconds: | 20260507 02:06:16.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:06:16.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:16.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:16.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:16.744 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 02:06:16.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:16.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:16.744 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:06:16.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:16.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:16.745 | INFO | Uploaded 36GiB to GPU in 1.99012s (18.0893GiB/s) | 20260507 02:06:16.745 | INFO | Downloaded 34GiB to GPU in 1.94555s (17.4758GiB/s) | 20260507 02:06:21.884 | INFO | Analyzing chunk frames = [6244864,6245376) | 20260507 02:06:21.884 | INFO | MemInfo: { | 20260507 02:06:21.884 | INFO | "gpu" : " 17 GB", | 20260507 02:06:21.884 | INFO | "huge" : "", | 20260507 02:06:21.885 | INFO | "huge_shared" : " 27 GB", | 20260507 02:06:21.885 | INFO | "malloc" : " 14 GB", | 20260507 02:06:21.885 | INFO | "pinned" : " 29 GB" | 20260507 02:06:21.885 | INFO | } | 20260507 02:06:21.885 | INFO | Perf report for the last 5.14041 seconds: | 20260507 02:06:21.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:06:21.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:21.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:21.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:21.885 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:06:21.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:21.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:21.885 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 02:06:21.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:21.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:21.885 | INFO | Uploaded 36GiB to GPU in 2.05102s (17.5522GiB/s) | 20260507 02:06:21.885 | INFO | Downloaded 34GiB to GPU in 1.96693s (17.2858GiB/s) | 20260507 02:06:27.021 | INFO | Analyzing chunk frames = [6245376,6245888) | 20260507 02:06:27.021 | INFO | MemInfo: { | 20260507 02:06:27.021 | INFO | "gpu" : " 17 GB", | 20260507 02:06:27.021 | INFO | "huge" : "", | 20260507 02:06:27.021 | INFO | "huge_shared" : " 27 GB", | 20260507 02:06:27.021 | INFO | "malloc" : " 14 GB", | 20260507 02:06:27.021 | INFO | "pinned" : " 29 GB" | 20260507 02:06:27.021 | INFO | } | 20260507 02:06:27.021 | INFO | Perf report for the last 5.13664 seconds: | 20260507 02:06:27.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:06:27.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:27.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:27.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:27.022 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:06:27.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:27.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:27.022 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:06:27.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:27.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:27.022 | INFO | Uploaded 36GiB to GPU in 1.97879s (18.193GiB/s) | 20260507 02:06:27.022 | INFO | Downloaded 34GiB to GPU in 1.96407s (17.311GiB/s) | 20260507 02:06:32.165 | INFO | Analyzing chunk frames = [6245888,6246400) | 20260507 02:06:32.165 | INFO | MemInfo: { | 20260507 02:06:32.165 | INFO | "gpu" : " 17 GB", | 20260507 02:06:32.165 | INFO | "huge" : "", | 20260507 02:06:32.165 | INFO | "huge_shared" : " 27 GB", | 20260507 02:06:32.165 | INFO | "malloc" : " 14 GB", | 20260507 02:06:32.165 | INFO | "pinned" : " 29 GB" | 20260507 02:06:32.165 | INFO | } | 20260507 02:06:32.165 | INFO | Perf report for the last 5.14343 seconds: | 20260507 02:06:32.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:06:32.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:32.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:32.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:32.165 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 02:06:32.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:32.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:32.165 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 02:06:32.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:32.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:32.165 | INFO | Uploaded 36GiB to GPU in 2.00431s (17.9613GiB/s) | 20260507 02:06:32.165 | INFO | Downloaded 34GiB to GPU in 1.96186s (17.3305GiB/s) | 20260507 02:06:32.262 | INFO | Starting a round of Baz IO | 20260507 02:06:35.372 | INFO | Populating RTMetrics report | 20260507 02:06:35.372 | INFO | RTMetrics report populated | 20260507 02:06:35.372 | INFO | Opening tmp RTMetrics json file | 20260507 02:06:35.374 | INFO | Serializing RTMetrics to json | 20260507 02:06:35.392 | INFO | Serialization done, swapping tmp file now... | 20260507 02:06:35.392 | INFO | RTMetrics done | 20260507 02:06:35.464 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.202 seconds | 20260507 02:06:35.464 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.05% / 0.00% / 66.95% | 20260507 02:06:35.464 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.255088% / 0% / 9.93411e-05% | 20260507 02:06:35.464 | INFO | Baz IO overall: | 20260507 02:06:35.464 | INFO | BytesWritten : 3229622423023 | 20260507 02:06:35.464 | INFO | NumEvents : 1553655367432 | 20260507 02:06:37.306 | INFO | Analyzing chunk frames = [6246400,6246912) | 20260507 02:06:37.306 | INFO | MemInfo: { | 20260507 02:06:37.306 | INFO | "gpu" : " 17 GB", | 20260507 02:06:37.306 | INFO | "huge" : "", | 20260507 02:06:37.306 | INFO | "huge_shared" : " 27 GB", | 20260507 02:06:37.306 | INFO | "malloc" : " 14 GB", | 20260507 02:06:37.306 | INFO | "pinned" : " 29 GB" | 20260507 02:06:37.306 | INFO | } | 20260507 02:06:37.306 | INFO | Perf report for the last 5.14166 seconds: | 20260507 02:06:37.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:06:37.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:37.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:37.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:37.306 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 02:06:37.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:37.307 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 02:06:37.307 | INFO | PRE_HQ : 59.2%, 1.8, 55.9ms, 0 | 20260507 02:06:37.307 | INFO | BAZWRITER : 61.8%, 3.3, 106.3ms, 0 | 20260507 02:06:37.307 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:06:37.307 | INFO | Uploaded 36GiB to GPU in 2.03618s (17.6802GiB/s) | 20260507 02:06:37.307 | INFO | Downloaded 38GiB to GPU in 2.17962s (17.4342GiB/s) | 20260507 02:06:42.448 | INFO | Analyzing chunk frames = [6246912,6247424) | 20260507 02:06:42.448 | INFO | MemInfo: { | 20260507 02:06:42.449 | INFO | "gpu" : " 17 GB", | 20260507 02:06:42.449 | INFO | "huge" : "", | 20260507 02:06:42.449 | INFO | "huge_shared" : " 27 GB", | 20260507 02:06:42.449 | INFO | "malloc" : " 14 GB", | 20260507 02:06:42.449 | INFO | "pinned" : " 29 GB" | 20260507 02:06:42.449 | INFO | } | 20260507 02:06:42.449 | INFO | Perf report for the last 5.1421 seconds: | 20260507 02:06:42.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:06:42.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:42.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:42.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:42.449 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 02:06:42.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:42.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:42.449 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:06:42.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:42.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:42.449 | INFO | Uploaded 36GiB to GPU in 2.04062s (17.6417GiB/s) | 20260507 02:06:42.449 | INFO | Downloaded 34GiB to GPU in 1.93579s (17.5639GiB/s) | 20260507 02:06:47.576 | INFO | Analyzing chunk frames = [6247424,6247936) | 20260507 02:06:47.576 | INFO | MemInfo: { | 20260507 02:06:47.576 | INFO | "gpu" : " 17 GB", | 20260507 02:06:47.576 | INFO | "huge" : "", | 20260507 02:06:47.576 | INFO | "huge_shared" : " 27 GB", | 20260507 02:06:47.576 | INFO | "malloc" : " 14 GB", | 20260507 02:06:47.576 | INFO | "pinned" : " 29 GB" | 20260507 02:06:47.576 | INFO | } | 20260507 02:06:47.577 | INFO | Perf report for the last 5.12803 seconds: | 20260507 02:06:47.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:06:47.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:47.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:47.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:47.577 | INFO | ANALYSIS : 60.5%, 3.0, 93.5ms, 0 | 20260507 02:06:47.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:47.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:47.577 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:06:47.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:47.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:47.577 | INFO | Uploaded 36GiB to GPU in 2.01878s (17.8326GiB/s) | 20260507 02:06:47.577 | INFO | Downloaded 34GiB to GPU in 1.94093s (17.5174GiB/s) | 20260507 02:06:52.718 | INFO | Analyzing chunk frames = [6247936,6248448) | 20260507 02:06:52.718 | INFO | MemInfo: { | 20260507 02:06:52.718 | INFO | "gpu" : " 17 GB", | 20260507 02:06:52.718 | INFO | "huge" : "", | 20260507 02:06:52.718 | INFO | "huge_shared" : " 27 GB", | 20260507 02:06:52.718 | INFO | "malloc" : " 14 GB", | 20260507 02:06:52.718 | INFO | "pinned" : " 29 GB" | 20260507 02:06:52.718 | INFO | } | 20260507 02:06:52.718 | INFO | Perf report for the last 5.14173 seconds: | 20260507 02:06:52.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:06:52.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:52.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:52.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:52.719 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260507 02:06:52.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:52.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:52.719 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 02:06:52.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:52.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:52.719 | INFO | Uploaded 36GiB to GPU in 2.01838s (17.8361GiB/s) | 20260507 02:06:52.719 | INFO | Downloaded 34GiB to GPU in 1.90964s (17.8044GiB/s) | 20260507 02:06:57.864 | INFO | Analyzing chunk frames = [6248448,6248960) | 20260507 02:06:57.864 | INFO | MemInfo: { | 20260507 02:06:57.864 | INFO | "gpu" : " 17 GB", | 20260507 02:06:57.864 | INFO | "huge" : "", | 20260507 02:06:57.864 | INFO | "huge_shared" : " 27 GB", | 20260507 02:06:57.864 | INFO | "malloc" : " 14 GB", | 20260507 02:06:57.864 | INFO | "pinned" : " 29 GB" | 20260507 02:06:57.864 | INFO | } | 20260507 02:06:57.864 | INFO | Perf report for the last 5.14582 seconds: | 20260507 02:06:57.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:06:57.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:57.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:57.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:57.864 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 02:06:57.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:57.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:57.864 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:06:57.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:57.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:06:57.865 | INFO | Uploaded 36GiB to GPU in 1.99322s (18.0613GiB/s) | 20260507 02:06:57.865 | INFO | Downloaded 34GiB to GPU in 1.93619s (17.5602GiB/s) | 20260507 02:07:03.002 | INFO | Analyzing chunk frames = [6248960,6249472) | 20260507 02:07:03.002 | INFO | MemInfo: { | 20260507 02:07:03.002 | INFO | "gpu" : " 17 GB", | 20260507 02:07:03.002 | INFO | "huge" : "", | 20260507 02:07:03.002 | INFO | "huge_shared" : " 27 GB", | 20260507 02:07:03.002 | INFO | "malloc" : " 14 GB", | 20260507 02:07:03.002 | INFO | "pinned" : " 29 GB" | 20260507 02:07:03.002 | INFO | } | 20260507 02:07:03.002 | INFO | Perf report for the last 5.13777 seconds: | 20260507 02:07:03.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:07:03.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:03.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:03.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:03.002 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:07:03.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:03.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:03.002 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 02:07:03.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:03.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:03.002 | INFO | Uploaded 36GiB to GPU in 2.05768s (17.4954GiB/s) | 20260507 02:07:03.002 | INFO | Downloaded 34GiB to GPU in 1.95879s (17.3577GiB/s) | 20260507 02:07:08.139 | INFO | Analyzing chunk frames = [6249472,6249984) | 20260507 02:07:08.139 | INFO | MemInfo: { | 20260507 02:07:08.139 | INFO | "gpu" : " 17 GB", | 20260507 02:07:08.139 | INFO | "huge" : "", | 20260507 02:07:08.139 | INFO | "huge_shared" : " 27 GB", | 20260507 02:07:08.139 | INFO | "malloc" : " 14 GB", | 20260507 02:07:08.139 | INFO | "pinned" : " 29 GB" | 20260507 02:07:08.139 | INFO | } | 20260507 02:07:08.140 | INFO | Perf report for the last 5.13754 seconds: | 20260507 02:07:08.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:07:08.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:08.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:08.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:08.140 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:07:08.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:08.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:08.140 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 02:07:08.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:08.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:08.140 | INFO | Uploaded 36GiB to GPU in 1.97236s (18.2522GiB/s) | 20260507 02:07:08.140 | INFO | Downloaded 34GiB to GPU in 1.97302s (17.2324GiB/s) | 20260507 02:07:13.278 | INFO | Analyzing chunk frames = [6249984,6250496) | 20260507 02:07:13.278 | INFO | MemInfo: { | 20260507 02:07:13.278 | INFO | "gpu" : " 17 GB", | 20260507 02:07:13.278 | INFO | "huge" : "", | 20260507 02:07:13.278 | INFO | "huge_shared" : " 27 GB", | 20260507 02:07:13.278 | INFO | "malloc" : " 14 GB", | 20260507 02:07:13.278 | INFO | "pinned" : " 29 GB" | 20260507 02:07:13.278 | INFO | } | 20260507 02:07:13.278 | INFO | Perf report for the last 5.13847 seconds: | 20260507 02:07:13.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:07:13.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:13.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:13.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:13.278 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 02:07:13.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:13.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:13.278 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:07:13.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:13.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:13.278 | INFO | Uploaded 36GiB to GPU in 1.95873s (18.3793GiB/s) | 20260507 02:07:13.279 | INFO | Downloaded 34GiB to GPU in 1.96947s (17.2636GiB/s) | 20260507 02:07:13.279 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:07:13.279 | INFO | Total mem usage: 18 GiB | 20260507 02:07:13.279 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:07:13.279 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:07:13.279 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:07:13.279 | INFO | High Water Breakdown: | 20260507 02:07:13.279 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:07:13.279 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:07:13.279 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:07:13.279 | INFO | ------------------------------------------ | 20260507 02:07:13.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:07:13.279 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:07:13.279 | INFO | Total mem usage: 26.5 GiB | 20260507 02:07:13.279 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:07:13.279 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:07:13.279 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:07:13.279 | INFO | High Water Breakdown: | 20260507 02:07:13.279 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:07:13.279 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:07:13.279 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:07:13.279 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:07:13.279 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:07:13.279 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:07:13.279 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:07:13.279 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:07:13.279 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:07:13.279 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:07:13.279 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:07:13.279 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:07:13.279 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:07:13.279 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:07:13.279 | INFO | ------------------------------------------ | 20260507 02:07:13.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:07:13.279 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:07:13.279 | INFO | Total mem usage: 25.6 GiB | 20260507 02:07:13.279 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:07:13.279 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:07:13.279 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:07:13.279 | INFO | High Water Breakdown: | 20260507 02:07:13.279 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:07:13.279 | INFO | ------------------------------------------ | 20260507 02:07:13.279 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:07:13.279 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:07:13.279 | INFO | Total mem usage: 15.7 GiB | 20260507 02:07:13.279 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:07:13.279 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:07:13.279 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:07:13.279 | INFO | High Water Breakdown: | 20260507 02:07:13.279 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:07:13.279 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:07:13.279 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:07:13.279 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:07:13.279 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:07:13.279 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:07:13.279 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:07:13.279 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:07:13.279 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:07:13.279 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:07:13.279 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:07:13.280 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:07:13.280 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:07:13.280 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:07:13.280 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:07:13.280 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:07:13.280 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:07:13.280 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:07:13.280 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:07:13.280 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:07:13.280 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:07:13.280 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:07:13.280 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:07:13.280 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:07:13.280 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:07:13.280 | INFO | ------------------------------------------ | 20260507 02:07:13.280 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:07:13.280 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:07:13.280 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:07:13.280 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:07:13.280 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.48e+01, 1.33e+01 / 1.37e+00 | 20260507 02:07:13.280 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 9.95e-02, 9.73e-03 / 5.50e-03 | 20260507 02:07:13.280 | INFO | PerfReport: 11.64%, Baseline -- 3.59e+00 / 4.36e+00, 5.22e-01 / 3.73e-01 | 20260507 02:07:13.280 | INFO | PerfReport: 3.23%, Histogram -- 9.96e-01 / 1.41e+00, 1.69e-01 / 1.44e-01 | 20260507 02:07:13.280 | INFO | PerfReport: 16.04%, DME -- 3.96e+01 / 4.38e+01, 3.67e+00 / 1.21e+00 | 20260507 02:07:13.280 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.34e+00 / 1.46e+00 | 20260507 02:07:13.280 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.16e+00 / 2.66e+00, 3.77e-01 / 2.23e-01 | 20260507 02:07:13.280 | INFO | PerfReport: 3.10%, Metrics -- 9.56e-01 / 1.29e+00, 2.58e-01 / 1.62e-01 | 20260507 02:07:13.280 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.24e-01 / 3.77e-01, 5.25e-02 / 2.35e-02 | 20260507 02:07:13.280 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:07:13.280 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:07:13.280 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:07:13.280 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 02:07:13.280 | INFO | PerfReport: 21.69%, Upload -- 2.09e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260507 02:07:13.280 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 02:07:13.280 | INFO | PerfReport: 31.99%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 02:07:13.280 | INFO | PerfReport: 1.43%, LockUpload -- 1.38e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 02:07:13.280 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260507 02:07:13.280 | INFO | PerfReport: 22.29%, LockCompute -- 2.15e+01 / 0.00e+00, 1.79e+01 / 0.00e+00 | 20260507 02:07:13.280 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 02:07:13.375 | INFO | Starting a round of Baz IO | 20260507 02:07:16.483 | INFO | Populating RTMetrics report | 20260507 02:07:16.483 | INFO | RTMetrics report populated | 20260507 02:07:16.483 | INFO | Opening tmp RTMetrics json file | 20260507 02:07:16.502 | INFO | Serializing RTMetrics to json | 20260507 02:07:16.503 | INFO | Serialization done, swapping tmp file now... | 20260507 02:07:16.504 | INFO | RTMetrics done | 20260507 02:07:16.572 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.196 seconds | 20260507 02:07:16.572 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.05% / 0.00% / 66.95% | 20260507 02:07:16.572 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.255573% / 0% / 7.94729e-05% | 20260507 02:07:16.572 | INFO | Baz IO overall: | 20260507 02:07:16.572 | INFO | BytesWritten : 3231551343763 | 20260507 02:07:16.572 | INFO | NumEvents : 1554568980338 | 20260507 02:07:18.418 | INFO | Analyzing chunk frames = [6250496,6251008) | 20260507 02:07:18.418 | INFO | MemInfo: { | 20260507 02:07:18.418 | INFO | "gpu" : " 17 GB", | 20260507 02:07:18.418 | INFO | "huge" : "", | 20260507 02:07:18.418 | INFO | "huge_shared" : " 27 GB", | 20260507 02:07:18.418 | INFO | "malloc" : " 14 GB", | 20260507 02:07:18.418 | INFO | "pinned" : " 29 GB" | 20260507 02:07:18.418 | INFO | } | 20260507 02:07:18.418 | INFO | Perf report for the last 5.13996 seconds: | 20260507 02:07:18.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:07:18.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:18.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:18.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:18.418 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260507 02:07:18.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:18.418 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 02:07:18.418 | INFO | PRE_HQ : 59.5%, 1.8, 56.5ms, 0 | 20260507 02:07:18.418 | INFO | BAZWRITER : 61.8%, 3.4, 106.6ms, 0 | 20260507 02:07:18.418 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:07:18.418 | INFO | Uploaded 36GiB to GPU in 2.00766s (17.9314GiB/s) | 20260507 02:07:18.418 | INFO | Downloaded 38GiB to GPU in 2.18143s (17.4197GiB/s) | 20260507 02:07:23.555 | INFO | Analyzing chunk frames = [6251008,6251520) | 20260507 02:07:23.555 | INFO | MemInfo: { | 20260507 02:07:23.555 | INFO | "gpu" : " 17 GB", | 20260507 02:07:23.555 | INFO | "huge" : "", | 20260507 02:07:23.555 | INFO | "huge_shared" : " 27 GB", | 20260507 02:07:23.555 | INFO | "malloc" : " 14 GB", | 20260507 02:07:23.555 | INFO | "pinned" : " 29 GB" | 20260507 02:07:23.555 | INFO | } | 20260507 02:07:23.555 | INFO | Perf report for the last 5.13675 seconds: | 20260507 02:07:23.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:07:23.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:23.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:23.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:23.555 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:07:23.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:23.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:23.555 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:07:23.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:23.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:23.555 | INFO | Uploaded 36GiB to GPU in 1.99881s (18.0107GiB/s) | 20260507 02:07:23.556 | INFO | Downloaded 34GiB to GPU in 1.92967s (17.6196GiB/s) | 20260507 02:07:28.691 | INFO | Analyzing chunk frames = [6251520,6252032) | 20260507 02:07:28.691 | INFO | MemInfo: { | 20260507 02:07:28.691 | INFO | "gpu" : " 17 GB", | 20260507 02:07:28.691 | INFO | "huge" : "", | 20260507 02:07:28.691 | INFO | "huge_shared" : " 27 GB", | 20260507 02:07:28.691 | INFO | "malloc" : " 14 GB", | 20260507 02:07:28.691 | INFO | "pinned" : " 29 GB" | 20260507 02:07:28.691 | INFO | } | 20260507 02:07:28.691 | INFO | Perf report for the last 5.13644 seconds: | 20260507 02:07:28.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:07:28.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:28.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:28.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:28.691 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 02:07:28.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:28.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:28.691 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:07:28.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:28.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:28.691 | INFO | Uploaded 36GiB to GPU in 1.99905s (18.0086GiB/s) | 20260507 02:07:28.692 | INFO | Downloaded 34GiB to GPU in 1.93684s (17.5544GiB/s) | 20260507 02:07:33.833 | INFO | Analyzing chunk frames = [6252032,6252544) | 20260507 02:07:33.833 | INFO | MemInfo: { | 20260507 02:07:33.833 | INFO | "gpu" : " 17 GB", | 20260507 02:07:33.833 | INFO | "huge" : "", | 20260507 02:07:33.833 | INFO | "huge_shared" : " 27 GB", | 20260507 02:07:33.833 | INFO | "malloc" : " 14 GB", | 20260507 02:07:33.833 | INFO | "pinned" : " 29 GB" | 20260507 02:07:33.833 | INFO | } | 20260507 02:07:33.833 | INFO | Perf report for the last 5.14137 seconds: | 20260507 02:07:33.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:07:33.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:33.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:33.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:33.833 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 02:07:33.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:33.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:33.833 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 02:07:33.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:33.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:33.834 | INFO | Uploaded 36GiB to GPU in 1.99394s (18.0547GiB/s) | 20260507 02:07:33.834 | INFO | Downloaded 34GiB to GPU in 1.91452s (17.759GiB/s) | 20260507 02:07:38.971 | INFO | Analyzing chunk frames = [6252544,6253056) | 20260507 02:07:38.971 | INFO | MemInfo: { | 20260507 02:07:38.971 | INFO | "gpu" : " 17 GB", | 20260507 02:07:38.971 | INFO | "huge" : "", | 20260507 02:07:38.971 | INFO | "huge_shared" : " 27 GB", | 20260507 02:07:38.971 | INFO | "malloc" : " 14 GB", | 20260507 02:07:38.971 | INFO | "pinned" : " 29 GB" | 20260507 02:07:38.971 | INFO | } | 20260507 02:07:38.971 | INFO | Perf report for the last 5.13863 seconds: | 20260507 02:07:38.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:07:38.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:38.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:38.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:38.971 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 02:07:38.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:38.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:38.971 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:07:38.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:38.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:38.971 | INFO | Uploaded 36GiB to GPU in 1.98803s (18.1084GiB/s) | 20260507 02:07:38.972 | INFO | Downloaded 34GiB to GPU in 1.95068s (17.4298GiB/s) | 20260507 02:07:44.123 | INFO | Analyzing chunk frames = [6253056,6253568) | 20260507 02:07:44.123 | INFO | MemInfo: { | 20260507 02:07:44.123 | INFO | "gpu" : " 17 GB", | 20260507 02:07:44.123 | INFO | "huge" : "", | 20260507 02:07:44.123 | INFO | "huge_shared" : " 27 GB", | 20260507 02:07:44.123 | INFO | "malloc" : " 14 GB", | 20260507 02:07:44.123 | INFO | "pinned" : " 29 GB" | 20260507 02:07:44.123 | INFO | } | 20260507 02:07:44.123 | INFO | Perf report for the last 5.15162 seconds: | 20260507 02:07:44.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:07:44.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:44.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:44.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:44.123 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 02:07:44.123 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:44.123 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:44.123 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 02:07:44.123 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:44.123 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:44.123 | INFO | Uploaded 36GiB to GPU in 1.99657s (18.0309GiB/s) | 20260507 02:07:44.123 | INFO | Downloaded 34GiB to GPU in 1.9446s (17.4843GiB/s) | 20260507 02:07:49.260 | INFO | Analyzing chunk frames = [6253568,6254080) | 20260507 02:07:49.260 | INFO | MemInfo: { | 20260507 02:07:49.260 | INFO | "gpu" : " 17 GB", | 20260507 02:07:49.260 | INFO | "huge" : "", | 20260507 02:07:49.260 | INFO | "huge_shared" : " 27 GB", | 20260507 02:07:49.260 | INFO | "malloc" : " 14 GB", | 20260507 02:07:49.260 | INFO | "pinned" : " 29 GB" | 20260507 02:07:49.260 | INFO | } | 20260507 02:07:49.260 | INFO | Perf report for the last 5.13695 seconds: | 20260507 02:07:49.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:07:49.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:49.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:49.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:49.260 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:07:49.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:49.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:49.260 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 02:07:49.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:49.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:49.260 | INFO | Uploaded 36GiB to GPU in 1.96411s (18.3289GiB/s) | 20260507 02:07:49.260 | INFO | Downloaded 34GiB to GPU in 1.97606s (17.2059GiB/s) | 20260507 02:07:54.396 | INFO | Analyzing chunk frames = [6254080,6254592) | 20260507 02:07:54.396 | INFO | MemInfo: { | 20260507 02:07:54.396 | INFO | "gpu" : " 17 GB", | 20260507 02:07:54.396 | INFO | "huge" : "", | 20260507 02:07:54.396 | INFO | "huge_shared" : " 27 GB", | 20260507 02:07:54.396 | INFO | "malloc" : " 14 GB", | 20260507 02:07:54.396 | INFO | "pinned" : " 29 GB" | 20260507 02:07:54.396 | INFO | } | 20260507 02:07:54.396 | INFO | Perf report for the last 5.13631 seconds: | 20260507 02:07:54.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:07:54.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:54.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:54.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:54.396 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 02:07:54.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:54.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:54.396 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:07:54.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:54.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:54.397 | INFO | Uploaded 36GiB to GPU in 1.96873s (18.2859GiB/s) | 20260507 02:07:54.397 | INFO | Downloaded 34GiB to GPU in 1.95375s (17.4024GiB/s) | 20260507 02:07:54.494 | INFO | Starting a round of Baz IO | 20260507 02:07:57.604 | INFO | Populating RTMetrics report | 20260507 02:07:57.605 | INFO | RTMetrics report populated | 20260507 02:07:57.605 | INFO | Opening tmp RTMetrics json file | 20260507 02:07:57.607 | INFO | Serializing RTMetrics to json | 20260507 02:07:57.623 | INFO | Serialization done, swapping tmp file now... | 20260507 02:07:57.624 | INFO | RTMetrics done | 20260507 02:07:57.695 | INFO | Finished a round of Baz IO: Wrote 1.794 GiB in 3.201 seconds | 20260507 02:07:57.695 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.04% / 0.00% / 66.95% | 20260507 02:07:57.695 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.253483% / 3.97364e-06% / 9.53674e-05% | 20260507 02:07:57.695 | INFO | Baz IO overall: | 20260507 02:07:57.695 | INFO | BytesWritten : 3233477796759 | 20260507 02:07:57.695 | INFO | NumEvents : 1555480761686 | 20260507 02:07:59.529 | INFO | Analyzing chunk frames = [6254592,6255104) | 20260507 02:07:59.529 | INFO | MemInfo: { | 20260507 02:07:59.529 | INFO | "gpu" : " 17 GB", | 20260507 02:07:59.529 | INFO | "huge" : "", | 20260507 02:07:59.529 | INFO | "huge_shared" : " 27 GB", | 20260507 02:07:59.529 | INFO | "malloc" : " 14 GB", | 20260507 02:07:59.529 | INFO | "pinned" : " 29 GB" | 20260507 02:07:59.529 | INFO | } | 20260507 02:07:59.529 | INFO | Perf report for the last 5.13326 seconds: | 20260507 02:07:59.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:07:59.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:59.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:59.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:59.529 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260507 02:07:59.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:07:59.529 | INFO | RT_METRICS : 42.1%, 1.0, 22.0ms, 0 | 20260507 02:07:59.530 | INFO | PRE_HQ : 59.3%, 1.8, 55.7ms, 0 | 20260507 02:07:59.530 | INFO | BAZWRITER : 61.9%, 3.3, 104.5ms, 0 | 20260507 02:07:59.530 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:07:59.530 | INFO | Uploaded 36GiB to GPU in 2.02166s (17.8071GiB/s) | 20260507 02:07:59.530 | INFO | Downloaded 38GiB to GPU in 2.17753s (17.4509GiB/s) | 20260507 02:08:04.670 | INFO | Analyzing chunk frames = [6255104,6255616) | 20260507 02:08:04.670 | INFO | MemInfo: { | 20260507 02:08:04.670 | INFO | "gpu" : " 17 GB", | 20260507 02:08:04.670 | INFO | "huge" : "", | 20260507 02:08:04.670 | INFO | "huge_shared" : " 27 GB", | 20260507 02:08:04.670 | INFO | "malloc" : " 14 GB", | 20260507 02:08:04.670 | INFO | "pinned" : " 29 GB" | 20260507 02:08:04.670 | INFO | } | 20260507 02:08:04.670 | INFO | Perf report for the last 5.14105 seconds: | 20260507 02:08:04.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:08:04.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:04.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:04.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:04.670 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:08:04.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:04.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:04.670 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:08:04.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:04.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:04.671 | INFO | Uploaded 36GiB to GPU in 2.00644s (17.9422GiB/s) | 20260507 02:08:04.671 | INFO | Downloaded 34GiB to GPU in 1.93321s (17.5873GiB/s) | 20260507 02:08:09.813 | INFO | Analyzing chunk frames = [6255616,6256128) | 20260507 02:08:09.813 | INFO | MemInfo: { | 20260507 02:08:09.813 | INFO | "gpu" : " 17 GB", | 20260507 02:08:09.813 | INFO | "huge" : "", | 20260507 02:08:09.813 | INFO | "huge_shared" : " 27 GB", | 20260507 02:08:09.813 | INFO | "malloc" : " 14 GB", | 20260507 02:08:09.813 | INFO | "pinned" : " 29 GB" | 20260507 02:08:09.813 | INFO | } | 20260507 02:08:09.813 | INFO | Perf report for the last 5.1423 seconds: | 20260507 02:08:09.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:08:09.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:09.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:09.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:09.813 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 02:08:09.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:09.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:09.813 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:08:09.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:09.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:09.813 | INFO | Uploaded 36GiB to GPU in 2.00384s (17.9655GiB/s) | 20260507 02:08:09.813 | INFO | Downloaded 34GiB to GPU in 1.93673s (17.5553GiB/s) | 20260507 02:08:14.954 | INFO | Analyzing chunk frames = [6256128,6256640) | 20260507 02:08:14.954 | INFO | MemInfo: { | 20260507 02:08:14.954 | INFO | "gpu" : " 17 GB", | 20260507 02:08:14.954 | INFO | "huge" : "", | 20260507 02:08:14.954 | INFO | "huge_shared" : " 27 GB", | 20260507 02:08:14.954 | INFO | "malloc" : " 14 GB", | 20260507 02:08:14.954 | INFO | "pinned" : " 29 GB" | 20260507 02:08:14.954 | INFO | } | 20260507 02:08:14.954 | INFO | Perf report for the last 5.14118 seconds: | 20260507 02:08:14.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:08:14.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:14.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:14.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:14.954 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 02:08:14.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:14.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:14.954 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 02:08:14.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:14.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:14.954 | INFO | Uploaded 36GiB to GPU in 1.98058s (18.1765GiB/s) | 20260507 02:08:14.954 | INFO | Downloaded 34GiB to GPU in 1.91207s (17.7818GiB/s) | 20260507 02:08:20.092 | INFO | Analyzing chunk frames = [6256640,6257152) | 20260507 02:08:20.092 | INFO | MemInfo: { | 20260507 02:08:20.092 | INFO | "gpu" : " 17 GB", | 20260507 02:08:20.092 | INFO | "huge" : "", | 20260507 02:08:20.092 | INFO | "huge_shared" : " 27 GB", | 20260507 02:08:20.092 | INFO | "malloc" : " 14 GB", | 20260507 02:08:20.092 | INFO | "pinned" : " 29 GB" | 20260507 02:08:20.092 | INFO | } | 20260507 02:08:20.092 | INFO | Perf report for the last 5.13812 seconds: | 20260507 02:08:20.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:08:20.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:20.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:20.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:20.092 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 02:08:20.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:20.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:20.092 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:08:20.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:20.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:20.092 | INFO | Uploaded 36GiB to GPU in 2.03158s (17.7202GiB/s) | 20260507 02:08:20.092 | INFO | Downloaded 34GiB to GPU in 1.93312s (17.5881GiB/s) | 20260507 02:08:25.232 | INFO | Analyzing chunk frames = [6257152,6257664) | 20260507 02:08:25.232 | INFO | MemInfo: { | 20260507 02:08:25.232 | INFO | "gpu" : " 17 GB", | 20260507 02:08:25.232 | INFO | "huge" : "", | 20260507 02:08:25.232 | INFO | "huge_shared" : " 27 GB", | 20260507 02:08:25.232 | INFO | "malloc" : " 14 GB", | 20260507 02:08:25.232 | INFO | "pinned" : " 29 GB" | 20260507 02:08:25.232 | INFO | } | 20260507 02:08:25.232 | INFO | Perf report for the last 5.13968 seconds: | 20260507 02:08:25.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:08:25.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:25.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:25.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:25.232 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 02:08:25.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:25.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:25.232 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 02:08:25.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:25.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:25.232 | INFO | Uploaded 36GiB to GPU in 2.03574s (17.684GiB/s) | 20260507 02:08:25.232 | INFO | Downloaded 34GiB to GPU in 1.92262s (17.6842GiB/s) | 20260507 02:08:30.370 | INFO | Analyzing chunk frames = [6257664,6258176) | 20260507 02:08:30.370 | INFO | MemInfo: { | 20260507 02:08:30.370 | INFO | "gpu" : " 17 GB", | 20260507 02:08:30.370 | INFO | "huge" : "", | 20260507 02:08:30.370 | INFO | "huge_shared" : " 27 GB", | 20260507 02:08:30.370 | INFO | "malloc" : " 14 GB", | 20260507 02:08:30.370 | INFO | "pinned" : " 29 GB" | 20260507 02:08:30.370 | INFO | } | 20260507 02:08:30.370 | INFO | Perf report for the last 5.13791 seconds: | 20260507 02:08:30.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:08:30.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:30.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:30.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:30.370 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:08:30.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:30.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:30.370 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 02:08:30.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:30.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:30.370 | INFO | Uploaded 36GiB to GPU in 2.00116s (17.9896GiB/s) | 20260507 02:08:30.370 | INFO | Downloaded 34GiB to GPU in 1.99282s (17.0613GiB/s) | 20260507 02:08:35.506 | INFO | Analyzing chunk frames = [6258176,6258688) | 20260507 02:08:35.506 | INFO | MemInfo: { | 20260507 02:08:35.506 | INFO | "gpu" : " 17 GB", | 20260507 02:08:35.506 | INFO | "huge" : "", | 20260507 02:08:35.506 | INFO | "huge_shared" : " 27 GB", | 20260507 02:08:35.506 | INFO | "malloc" : " 14 GB", | 20260507 02:08:35.506 | INFO | "pinned" : " 29 GB" | 20260507 02:08:35.506 | INFO | } | 20260507 02:08:35.506 | INFO | Perf report for the last 5.13606 seconds: | 20260507 02:08:35.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:08:35.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:35.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:35.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:35.506 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 02:08:35.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:35.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:35.506 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:08:35.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:35.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:35.506 | INFO | Uploaded 36GiB to GPU in 2.00028s (17.9975GiB/s) | 20260507 02:08:35.506 | INFO | Downloaded 34GiB to GPU in 1.97545s (17.2113GiB/s) | 20260507 02:08:35.506 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:08:35.506 | INFO | Total mem usage: 18 GiB | 20260507 02:08:35.506 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:08:35.506 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 02:08:35.506 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:08:35.506 | INFO | High Water Breakdown: | 20260507 02:08:35.506 | INFO | 60.1% : 8.57 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:08:35.506 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:08:35.506 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:08:35.506 | INFO | ------------------------------------------ | 20260507 02:08:35.506 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:08:35.507 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:08:35.507 | INFO | Total mem usage: 26.5 GiB | 20260507 02:08:35.507 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:08:35.507 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:08:35.507 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:08:35.507 | INFO | High Water Breakdown: | 20260507 02:08:35.507 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:08:35.507 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:08:35.507 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:08:35.507 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:08:35.507 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:08:35.507 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:08:35.507 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:08:35.507 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:08:35.507 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:08:35.507 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:08:35.507 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:08:35.507 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:08:35.507 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:08:35.507 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:08:35.507 | INFO | ------------------------------------------ | 20260507 02:08:35.507 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:08:35.507 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:08:35.507 | INFO | Total mem usage: 25.6 GiB | 20260507 02:08:35.507 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:08:35.507 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:08:35.507 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:08:35.507 | INFO | High Water Breakdown: | 20260507 02:08:35.507 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:08:35.507 | INFO | ------------------------------------------ | 20260507 02:08:35.507 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:08:35.507 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:08:35.507 | INFO | Total mem usage: 15.7 GiB | 20260507 02:08:35.507 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:08:35.507 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:08:35.507 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260507 02:08:35.507 | INFO | High Water Breakdown: | 20260507 02:08:35.507 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:08:35.507 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:08:35.507 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:08:35.507 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:08:35.507 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:08:35.507 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:08:35.507 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:08:35.507 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:08:35.507 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:08:35.507 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:08:35.507 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:08:35.507 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:08:35.507 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:08:35.507 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:08:35.507 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:08:35.507 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:08:35.507 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:08:35.507 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:08:35.507 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:08:35.507 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:08:35.507 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:08:35.507 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:08:35.507 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:08:35.507 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:08:35.507 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:08:35.507 | INFO | ------------------------------------------ | 20260507 02:08:35.507 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:08:35.507 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:08:35.507 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:08:35.507 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:08:35.507 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.52e+01, 1.33e+01 / 1.78e+00 | 20260507 02:08:35.507 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 9.97e-02, 9.80e-03 / 6.70e-03 | 20260507 02:08:35.507 | INFO | PerfReport: 11.68%, Baseline -- 3.60e+00 / 4.39e+00, 5.30e-01 / 3.48e-01 | 20260507 02:08:35.507 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.45e+00, 1.74e-01 / 1.39e-01 | 20260507 02:08:35.507 | INFO | PerfReport: 16.07%, DME -- 3.97e+01 / 4.42e+01, 3.83e+00 / 2.17e+00 | 20260507 02:08:35.507 | INFO | PerfReport: 57.54%, FrameLabeling -- 1.77e+01 / 2.07e+01, 1.31e+00 / 1.37e+00 | 20260507 02:08:35.507 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.16e+00 / 2.71e+00, 3.83e-01 / 2.13e-01 | 20260507 02:08:35.507 | INFO | PerfReport: 3.09%, Metrics -- 9.54e-01 / 1.27e+00, 2.56e-01 / 1.34e-01 | 20260507 02:08:35.507 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.24e-01 / 3.73e-01, 5.22e-02 / 1.79e-02 | 20260507 02:08:35.508 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:08:35.508 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:08:35.508 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:08:35.508 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 02:08:35.508 | INFO | PerfReport: 21.62%, Upload -- 2.08e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260507 02:08:35.508 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 02:08:35.508 | INFO | PerfReport: 32.02%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 02:08:35.508 | INFO | PerfReport: 1.35%, LockUpload -- 1.30e+00 / 0.00e+00, 4.79e+00 / 0.00e+00 | 20260507 02:08:35.508 | INFO | PerfReport: 1.33%, LockDownload -- 1.29e+00 / 0.00e+00, 3.20e+00 / 0.00e+00 | 20260507 02:08:35.508 | INFO | PerfReport: 22.32%, LockCompute -- 2.15e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 02:08:35.508 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 3.14e-02 / 0.00e+00 | 20260507 02:08:35.603 | INFO | Starting a round of Baz IO | 20260507 02:08:38.714 | INFO | Populating RTMetrics report | 20260507 02:08:38.715 | INFO | RTMetrics report populated | 20260507 02:08:38.715 | INFO | Opening tmp RTMetrics json file | 20260507 02:08:38.717 | INFO | Serializing RTMetrics to json | 20260507 02:08:38.735 | INFO | Serialization done, swapping tmp file now... | 20260507 02:08:38.735 | INFO | RTMetrics done | 20260507 02:08:38.810 | INFO | Finished a round of Baz IO: Wrote 1.792 GiB in 3.206 seconds | 20260507 02:08:38.810 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.04% / 0.00% / 66.96% | 20260507 02:08:38.810 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.252509% / 0% / 9.53674e-05% | 20260507 02:08:38.810 | INFO | Baz IO overall: | 20260507 02:08:38.810 | INFO | BytesWritten : 3235402388505 | 20260507 02:08:38.810 | INFO | NumEvents : 1556391097470 | 20260507 02:08:40.640 | INFO | Analyzing chunk frames = [6258688,6259200) | 20260507 02:08:40.640 | INFO | MemInfo: { | 20260507 02:08:40.640 | INFO | "gpu" : " 17 GB", | 20260507 02:08:40.640 | INFO | "huge" : "", | 20260507 02:08:40.640 | INFO | "huge_shared" : " 27 GB", | 20260507 02:08:40.640 | INFO | "malloc" : " 14 GB", | 20260507 02:08:40.640 | INFO | "pinned" : " 29 GB" | 20260507 02:08:40.640 | INFO | } | 20260507 02:08:40.640 | INFO | Perf report for the last 5.1346 seconds: | 20260507 02:08:40.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:08:40.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:40.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:40.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:40.641 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 02:08:40.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:40.641 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 02:08:40.641 | INFO | PRE_HQ : 59.7%, 1.8, 56.2ms, 0 | 20260507 02:08:40.641 | INFO | BAZWRITER : 62.0%, 3.3, 105.7ms, 0 | 20260507 02:08:40.641 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:08:40.641 | INFO | Uploaded 36GiB to GPU in 2.02984s (17.7353GiB/s) | 20260507 02:08:40.641 | INFO | Downloaded 38GiB to GPU in 2.18751s (17.3713GiB/s) | 20260507 02:08:45.784 | INFO | Analyzing chunk frames = [6259200,6259712) | 20260507 02:08:45.784 | INFO | MemInfo: { | 20260507 02:08:45.784 | INFO | "gpu" : " 17 GB", | 20260507 02:08:45.784 | INFO | "huge" : "", | 20260507 02:08:45.784 | INFO | "huge_shared" : " 27 GB", | 20260507 02:08:45.784 | INFO | "malloc" : " 14 GB", | 20260507 02:08:45.784 | INFO | "pinned" : " 29 GB" | 20260507 02:08:45.784 | INFO | } | 20260507 02:08:45.784 | INFO | Perf report for the last 5.1437 seconds: | 20260507 02:08:45.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:08:45.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:45.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:45.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:45.784 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 02:08:45.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:45.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:45.784 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:08:45.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:45.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:45.784 | INFO | Uploaded 36GiB to GPU in 1.99155s (18.0764GiB/s) | 20260507 02:08:45.784 | INFO | Downloaded 34GiB to GPU in 1.93112s (17.6063GiB/s) | 20260507 02:08:50.928 | INFO | Analyzing chunk frames = [6259712,6260224) | 20260507 02:08:50.928 | INFO | MemInfo: { | 20260507 02:08:50.928 | INFO | "gpu" : " 17 GB", | 20260507 02:08:50.928 | INFO | "huge" : "", | 20260507 02:08:50.928 | INFO | "huge_shared" : " 27 GB", | 20260507 02:08:50.928 | INFO | "malloc" : " 14 GB", | 20260507 02:08:50.928 | INFO | "pinned" : " 29 GB" | 20260507 02:08:50.928 | INFO | } | 20260507 02:08:50.928 | INFO | Perf report for the last 5.14424 seconds: | 20260507 02:08:50.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:08:50.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:50.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:50.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:50.929 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 02:08:50.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:50.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:50.929 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:08:50.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:50.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:50.929 | INFO | Uploaded 36GiB to GPU in 1.99063s (18.0847GiB/s) | 20260507 02:08:50.929 | INFO | Downloaded 34GiB to GPU in 1.93703s (17.5527GiB/s) | 20260507 02:08:56.068 | INFO | Analyzing chunk frames = [6260224,6260736) | 20260507 02:08:56.068 | INFO | MemInfo: { | 20260507 02:08:56.068 | INFO | "gpu" : " 17 GB", | 20260507 02:08:56.068 | INFO | "huge" : "", | 20260507 02:08:56.068 | INFO | "huge_shared" : " 27 GB", | 20260507 02:08:56.068 | INFO | "malloc" : " 14 GB", | 20260507 02:08:56.068 | INFO | "pinned" : " 29 GB" | 20260507 02:08:56.068 | INFO | } | 20260507 02:08:56.068 | INFO | Perf report for the last 5.13971 seconds: | 20260507 02:08:56.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:08:56.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:56.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:56.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:56.068 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 02:08:56.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:56.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:56.068 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:08:56.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:56.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:08:56.069 | INFO | Uploaded 36GiB to GPU in 1.95938s (18.3731GiB/s) | 20260507 02:08:56.069 | INFO | Downloaded 34GiB to GPU in 1.90647s (17.834GiB/s) | 20260507 02:09:01.208 | INFO | Analyzing chunk frames = [6260736,6261248) | 20260507 02:09:01.208 | INFO | MemInfo: { | 20260507 02:09:01.208 | INFO | "gpu" : " 17 GB", | 20260507 02:09:01.208 | INFO | "huge" : "", | 20260507 02:09:01.208 | INFO | "huge_shared" : " 27 GB", | 20260507 02:09:01.208 | INFO | "malloc" : " 14 GB", | 20260507 02:09:01.208 | INFO | "pinned" : " 29 GB" | 20260507 02:09:01.208 | INFO | } | 20260507 02:09:01.208 | INFO | Perf report for the last 5.1402 seconds: | 20260507 02:09:01.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:09:01.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:01.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:01.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:01.209 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 02:09:01.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:01.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:01.209 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 02:09:01.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:01.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:01.209 | INFO | Uploaded 36GiB to GPU in 2.03085s (17.7266GiB/s) | 20260507 02:09:01.209 | INFO | Downloaded 34GiB to GPU in 1.93615s (17.5606GiB/s) | 20260507 02:09:06.343 | INFO | Analyzing chunk frames = [6261248,6261760) | 20260507 02:09:06.343 | INFO | MemInfo: { | 20260507 02:09:06.343 | INFO | "gpu" : " 17 GB", | 20260507 02:09:06.343 | INFO | "huge" : "", | 20260507 02:09:06.343 | INFO | "huge_shared" : " 27 GB", | 20260507 02:09:06.343 | INFO | "malloc" : " 14 GB", | 20260507 02:09:06.343 | INFO | "pinned" : " 29 GB" | 20260507 02:09:06.343 | INFO | } | 20260507 02:09:06.343 | INFO | Perf report for the last 5.13516 seconds: | 20260507 02:09:06.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:09:06.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:06.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:06.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:06.344 | INFO | ANALYSIS : 60.6%, 3.0, 93.5ms, 0 | 20260507 02:09:06.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:06.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:06.344 | INFO | PRE_HQ : 25.8%, 1.0, 13.3ms, 0 | 20260507 02:09:06.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:06.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:06.344 | INFO | Uploaded 36GiB to GPU in 2.03749s (17.6688GiB/s) | 20260507 02:09:06.344 | INFO | Downloaded 34GiB to GPU in 1.92527s (17.6598GiB/s) | 20260507 02:09:11.483 | INFO | Analyzing chunk frames = [6261760,6262272) | 20260507 02:09:11.483 | INFO | MemInfo: { | 20260507 02:09:11.483 | INFO | "gpu" : " 17 GB", | 20260507 02:09:11.483 | INFO | "huge" : "", | 20260507 02:09:11.484 | INFO | "huge_shared" : " 27 GB", | 20260507 02:09:11.484 | INFO | "malloc" : " 14 GB", | 20260507 02:09:11.484 | INFO | "pinned" : " 29 GB" | 20260507 02:09:11.484 | INFO | } | 20260507 02:09:11.484 | INFO | Perf report for the last 5.14006 seconds: | 20260507 02:09:11.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:09:11.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:11.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:11.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:11.484 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:09:11.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:11.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:11.484 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 02:09:11.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:11.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:11.484 | INFO | Uploaded 36GiB to GPU in 1.99483s (18.0467GiB/s) | 20260507 02:09:11.484 | INFO | Downloaded 34GiB to GPU in 1.98592s (17.1206GiB/s) | 20260507 02:09:16.621 | INFO | Analyzing chunk frames = [6262272,6262784) | 20260507 02:09:16.621 | INFO | MemInfo: { | 20260507 02:09:16.621 | INFO | "gpu" : " 17 GB", | 20260507 02:09:16.621 | INFO | "huge" : "", | 20260507 02:09:16.621 | INFO | "huge_shared" : " 27 GB", | 20260507 02:09:16.621 | INFO | "malloc" : " 14 GB", | 20260507 02:09:16.621 | INFO | "pinned" : " 29 GB" | 20260507 02:09:16.621 | INFO | } | 20260507 02:09:16.621 | INFO | Perf report for the last 5.13735 seconds: | 20260507 02:09:16.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:09:16.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:16.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:16.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:16.621 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 02:09:16.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:16.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:16.621 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 02:09:16.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:16.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:16.621 | INFO | Uploaded 36GiB to GPU in 2.00253s (17.9772GiB/s) | 20260507 02:09:16.621 | INFO | Downloaded 34GiB to GPU in 1.97952s (17.1759GiB/s) | 20260507 02:09:16.718 | INFO | Starting a round of Baz IO | 20260507 02:09:19.834 | INFO | Populating RTMetrics report | 20260507 02:09:19.834 | INFO | RTMetrics report populated | 20260507 02:09:19.834 | INFO | Opening tmp RTMetrics json file | 20260507 02:09:19.835 | INFO | Serializing RTMetrics to json | 20260507 02:09:19.836 | INFO | Serialization done, swapping tmp file now... | 20260507 02:09:19.836 | INFO | RTMetrics done | 20260507 02:09:19.925 | INFO | Finished a round of Baz IO: Wrote 1.794 GiB in 3.206 seconds | 20260507 02:09:19.925 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.04% / 0.00% / 66.96% | 20260507 02:09:19.925 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.25239% / 3.97364e-06% / 0.000107288% | 20260507 02:09:19.925 | INFO | Baz IO overall: | 20260507 02:09:19.925 | INFO | BytesWritten : 3237328277260 | 20260507 02:09:19.925 | INFO | NumEvents : 1557302306811 | 20260507 02:09:21.762 | INFO | Analyzing chunk frames = [6262784,6263296) | 20260507 02:09:21.762 | INFO | MemInfo: { | 20260507 02:09:21.762 | INFO | "gpu" : " 17 GB", | 20260507 02:09:21.762 | INFO | "huge" : "", | 20260507 02:09:21.762 | INFO | "huge_shared" : " 27 GB", | 20260507 02:09:21.762 | INFO | "malloc" : " 14 GB", | 20260507 02:09:21.762 | INFO | "pinned" : " 29 GB" | 20260507 02:09:21.762 | INFO | } | 20260507 02:09:21.763 | INFO | Perf report for the last 5.14161 seconds: | 20260507 02:09:21.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:09:21.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:21.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:21.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:21.763 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 02:09:21.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:21.763 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 02:09:21.763 | INFO | PRE_HQ : 59.2%, 1.8, 56.1ms, 0 | 20260507 02:09:21.763 | INFO | BAZWRITER : 61.9%, 3.3, 105.6ms, 0 | 20260507 02:09:21.763 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:09:21.763 | INFO | Uploaded 36GiB to GPU in 2.00667s (17.9402GiB/s) | 20260507 02:09:21.763 | INFO | Downloaded 38GiB to GPU in 2.18276s (17.4091GiB/s) | 20260507 02:09:26.901 | INFO | Analyzing chunk frames = [6263296,6263808) | 20260507 02:09:26.901 | INFO | MemInfo: { | 20260507 02:09:26.901 | INFO | "gpu" : " 17 GB", | 20260507 02:09:26.901 | INFO | "huge" : "", | 20260507 02:09:26.901 | INFO | "huge_shared" : " 27 GB", | 20260507 02:09:26.901 | INFO | "malloc" : " 14 GB", | 20260507 02:09:26.901 | INFO | "pinned" : " 29 GB" | 20260507 02:09:26.901 | INFO | } | 20260507 02:09:26.901 | INFO | Perf report for the last 5.13877 seconds: | 20260507 02:09:26.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:09:26.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:26.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:26.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:26.901 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:09:26.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:26.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:26.901 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:09:26.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:26.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:26.902 | INFO | Uploaded 36GiB to GPU in 2.02534s (17.7748GiB/s) | 20260507 02:09:26.902 | INFO | Downloaded 34GiB to GPU in 1.93723s (17.5508GiB/s) | 20260507 02:09:32.042 | INFO | Analyzing chunk frames = [6263808,6264320) | 20260507 02:09:32.042 | INFO | MemInfo: { | 20260507 02:09:32.042 | INFO | "gpu" : " 17 GB", | 20260507 02:09:32.042 | INFO | "huge" : "", | 20260507 02:09:32.042 | INFO | "huge_shared" : " 27 GB", | 20260507 02:09:32.042 | INFO | "malloc" : " 14 GB", | 20260507 02:09:32.042 | INFO | "pinned" : " 29 GB" | 20260507 02:09:32.042 | INFO | } | 20260507 02:09:32.042 | INFO | Perf report for the last 5.1407 seconds: | 20260507 02:09:32.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:09:32.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:32.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:32.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:32.042 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 02:09:32.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:32.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:32.042 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 02:09:32.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:32.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:32.042 | INFO | Uploaded 36GiB to GPU in 1.97623s (18.2165GiB/s) | 20260507 02:09:32.042 | INFO | Downloaded 34GiB to GPU in 1.94119s (17.515GiB/s) | 20260507 02:09:37.188 | INFO | Analyzing chunk frames = [6264320,6264832) | 20260507 02:09:37.188 | INFO | MemInfo: { | 20260507 02:09:37.188 | INFO | "gpu" : " 17 GB", | 20260507 02:09:37.188 | INFO | "huge" : "", | 20260507 02:09:37.188 | INFO | "huge_shared" : " 27 GB", | 20260507 02:09:37.188 | INFO | "malloc" : " 14 GB", | 20260507 02:09:37.188 | INFO | "pinned" : " 29 GB" | 20260507 02:09:37.188 | INFO | } | 20260507 02:09:37.188 | INFO | Perf report for the last 5.14567 seconds: | 20260507 02:09:37.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:09:37.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:37.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:37.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:37.188 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 02:09:37.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:37.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:37.188 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 02:09:37.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:37.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:37.188 | INFO | Uploaded 36GiB to GPU in 1.9617s (18.3514GiB/s) | 20260507 02:09:37.188 | INFO | Downloaded 34GiB to GPU in 1.92131s (17.6962GiB/s) | 20260507 02:09:42.319 | INFO | Analyzing chunk frames = [6264832,6265344) | 20260507 02:09:42.319 | INFO | MemInfo: { | 20260507 02:09:42.319 | INFO | "gpu" : " 17 GB", | 20260507 02:09:42.319 | INFO | "huge" : "", | 20260507 02:09:42.319 | INFO | "huge_shared" : " 27 GB", | 20260507 02:09:42.319 | INFO | "malloc" : " 14 GB", | 20260507 02:09:42.319 | INFO | "pinned" : " 29 GB" | 20260507 02:09:42.319 | INFO | } | 20260507 02:09:42.319 | INFO | Perf report for the last 5.13105 seconds: | 20260507 02:09:42.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:09:42.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:42.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:42.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:42.319 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 02:09:42.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:42.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:42.319 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:09:42.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:42.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:42.319 | INFO | Uploaded 36GiB to GPU in 2.01559s (17.8608GiB/s) | 20260507 02:09:42.319 | INFO | Downloaded 34GiB to GPU in 1.93521s (17.5691GiB/s) | 20260507 02:09:47.452 | INFO | Analyzing chunk frames = [6265344,6265856) | 20260507 02:09:47.452 | INFO | MemInfo: { | 20260507 02:09:47.452 | INFO | "gpu" : " 17 GB", | 20260507 02:09:47.452 | INFO | "huge" : "", | 20260507 02:09:47.452 | INFO | "huge_shared" : " 27 GB", | 20260507 02:09:47.452 | INFO | "malloc" : " 14 GB", | 20260507 02:09:47.452 | INFO | "pinned" : " 29 GB" | 20260507 02:09:47.452 | INFO | } | 20260507 02:09:47.452 | INFO | Perf report for the last 5.13331 seconds: | 20260507 02:09:47.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:09:47.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:47.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:47.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:47.452 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:09:47.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:47.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:47.452 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 02:09:47.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:47.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:47.452 | INFO | Uploaded 36GiB to GPU in 2.05191s (17.5446GiB/s) | 20260507 02:09:47.452 | INFO | Downloaded 34GiB to GPU in 1.97213s (17.2402GiB/s) | 20260507 02:09:52.597 | INFO | Analyzing chunk frames = [6265856,6266368) | 20260507 02:09:52.597 | INFO | MemInfo: { | 20260507 02:09:52.597 | INFO | "gpu" : " 17 GB", | 20260507 02:09:52.597 | INFO | "huge" : "", | 20260507 02:09:52.597 | INFO | "huge_shared" : " 27 GB", | 20260507 02:09:52.597 | INFO | "malloc" : " 14 GB", | 20260507 02:09:52.597 | INFO | "pinned" : " 29 GB" | 20260507 02:09:52.597 | INFO | } | 20260507 02:09:52.597 | INFO | Perf report for the last 5.14506 seconds: | 20260507 02:09:52.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:09:52.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:52.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:52.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:52.597 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:09:52.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:52.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:52.598 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 02:09:52.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:52.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:52.598 | INFO | Uploaded 36GiB to GPU in 1.97417s (18.2355GiB/s) | 20260507 02:09:52.598 | INFO | Downloaded 34GiB to GPU in 1.99908s (17.0079GiB/s) | 20260507 02:09:57.734 | INFO | Analyzing chunk frames = [6266368,6266880) | 20260507 02:09:57.734 | INFO | MemInfo: { | 20260507 02:09:57.734 | INFO | "gpu" : " 17 GB", | 20260507 02:09:57.734 | INFO | "huge" : "", | 20260507 02:09:57.734 | INFO | "huge_shared" : " 27 GB", | 20260507 02:09:57.734 | INFO | "malloc" : " 14 GB", | 20260507 02:09:57.734 | INFO | "pinned" : " 29 GB" | 20260507 02:09:57.734 | INFO | } | 20260507 02:09:57.734 | INFO | Perf report for the last 5.13696 seconds: | 20260507 02:09:57.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:09:57.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:57.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:57.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:57.734 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 02:09:57.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:57.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:57.734 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 02:09:57.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:57.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:09:57.735 | INFO | Uploaded 36GiB to GPU in 2.00294s (17.9736GiB/s) | 20260507 02:09:57.735 | INFO | Downloaded 34GiB to GPU in 1.97626s (17.2042GiB/s) | 20260507 02:09:57.735 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:09:57.735 | INFO | Total mem usage: 18 GiB | 20260507 02:09:57.735 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:09:57.735 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:09:57.735 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:09:57.735 | INFO | High Water Breakdown: | 20260507 02:09:57.735 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:09:57.735 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:09:57.735 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:09:57.735 | INFO | ------------------------------------------ | 20260507 02:09:57.735 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:09:57.735 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:09:57.735 | INFO | Total mem usage: 26.5 GiB | 20260507 02:09:57.735 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:09:57.735 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:09:57.735 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:09:57.735 | INFO | High Water Breakdown: | 20260507 02:09:57.735 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:09:57.735 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:09:57.735 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:09:57.735 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:09:57.735 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:09:57.735 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:09:57.735 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:09:57.735 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:09:57.735 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:09:57.735 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:09:57.735 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:09:57.735 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:09:57.735 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:09:57.735 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:09:57.735 | INFO | ------------------------------------------ | 20260507 02:09:57.735 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:09:57.735 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:09:57.735 | INFO | Total mem usage: 25.6 GiB | 20260507 02:09:57.735 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:09:57.735 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:09:57.735 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:09:57.735 | INFO | High Water Breakdown: | 20260507 02:09:57.735 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:09:57.735 | INFO | ------------------------------------------ | 20260507 02:09:57.736 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:09:57.736 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:09:57.736 | INFO | Total mem usage: 15.7 GiB | 20260507 02:09:57.736 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:09:57.736 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:09:57.736 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:09:57.736 | INFO | High Water Breakdown: | 20260507 02:09:57.736 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:09:57.736 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:09:57.736 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:09:57.736 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:09:57.736 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:09:57.736 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:09:57.736 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:09:57.736 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:09:57.736 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:09:57.736 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:09:57.736 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:09:57.736 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:09:57.736 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:09:57.736 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:09:57.736 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:09:57.736 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:09:57.736 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:09:57.736 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:09:57.736 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:09:57.736 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:09:57.736 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:09:57.736 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:09:57.736 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:09:57.736 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:09:57.736 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:09:57.736 | INFO | ------------------------------------------ | 20260507 02:09:57.736 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:09:57.736 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:09:57.736 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:09:57.736 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:09:57.736 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.56e+01, 1.34e+01 / 2.14e+00 | 20260507 02:09:57.736 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.51e-03 / 3.83e-03 | 20260507 02:09:57.736 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.49e+00, 5.21e-01 / 1.64e-01 | 20260507 02:09:57.736 | INFO | PerfReport: 3.23%, Histogram -- 9.95e-01 / 1.47e+00, 1.69e-01 / 8.69e-02 | 20260507 02:09:57.736 | INFO | PerfReport: 16.13%, DME -- 3.98e+01 / 4.49e+01, 4.20e+00 / 2.36e+00 | 20260507 02:09:57.736 | INFO | PerfReport: 57.54%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.31e+00 / 1.71e+00 | 20260507 02:09:57.736 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.16e+00 / 2.69e+00, 3.80e-01 / 2.35e-01 | 20260507 02:09:57.736 | INFO | PerfReport: 3.09%, Metrics -- 9.53e-01 / 1.27e+00, 2.55e-01 / 1.31e-01 | 20260507 02:09:57.736 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.24e-01 / 3.77e-01, 5.24e-02 / 2.46e-02 | 20260507 02:09:57.736 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:09:57.736 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:09:57.736 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:09:57.736 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 02:09:57.736 | INFO | PerfReport: 21.64%, Upload -- 2.09e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260507 02:09:57.736 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 02:09:57.736 | INFO | PerfReport: 31.98%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 02:09:57.736 | INFO | PerfReport: 1.34%, LockUpload -- 1.29e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260507 02:09:57.736 | INFO | PerfReport: 1.40%, LockDownload -- 1.35e+00 / 0.00e+00, 3.28e+00 / 0.00e+00 | 20260507 02:09:57.736 | INFO | PerfReport: 22.26%, LockCompute -- 2.15e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 02:09:57.736 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260507 02:09:57.833 | INFO | Starting a round of Baz IO | 20260507 02:10:00.958 | INFO | Populating RTMetrics report | 20260507 02:10:00.958 | INFO | RTMetrics report populated | 20260507 02:10:00.958 | INFO | Opening tmp RTMetrics json file | 20260507 02:10:00.977 | INFO | Serializing RTMetrics to json | 20260507 02:10:00.978 | INFO | Serialization done, swapping tmp file now... | 20260507 02:10:00.979 | INFO | RTMetrics done | 20260507 02:10:01.049 | INFO | Finished a round of Baz IO: Wrote 1.793 GiB in 3.216 seconds | 20260507 02:10:01.049 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.04% / 0.00% / 66.96% | 20260507 02:10:01.049 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.252652% / 3.97364e-06% / 0.000111262% | 20260507 02:10:01.049 | INFO | Baz IO overall: | 20260507 02:10:01.049 | INFO | BytesWritten : 3239253414535 | 20260507 02:10:01.049 | INFO | NumEvents : 1558212964615 | 20260507 02:10:02.878 | INFO | Analyzing chunk frames = [6266880,6267392) | 20260507 02:10:02.878 | INFO | MemInfo: { | 20260507 02:10:02.878 | INFO | "gpu" : " 17 GB", | 20260507 02:10:02.878 | INFO | "huge" : "", | 20260507 02:10:02.878 | INFO | "huge_shared" : " 27 GB", | 20260507 02:10:02.878 | INFO | "malloc" : " 14 GB", | 20260507 02:10:02.878 | INFO | "pinned" : " 29 GB" | 20260507 02:10:02.878 | INFO | } | 20260507 02:10:02.878 | INFO | Perf report for the last 5.14357 seconds: | 20260507 02:10:02.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:10:02.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:02.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:02.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:02.878 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 02:10:02.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:02.878 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 02:10:02.878 | INFO | PRE_HQ : 60.2%, 1.9, 60.0ms, 0 | 20260507 02:10:02.878 | INFO | BAZWRITER : 62.1%, 3.4, 109.5ms, 0 | 20260507 02:10:02.878 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:10:02.878 | INFO | Uploaded 36GiB to GPU in 2.04799s (17.5782GiB/s) | 20260507 02:10:02.878 | INFO | Downloaded 38GiB to GPU in 2.17959s (17.4345GiB/s) | 20260507 02:10:08.018 | INFO | Analyzing chunk frames = [6267392,6267904) | 20260507 02:10:08.018 | INFO | MemInfo: { | 20260507 02:10:08.018 | INFO | "gpu" : " 17 GB", | 20260507 02:10:08.018 | INFO | "huge" : "", | 20260507 02:10:08.018 | INFO | "huge_shared" : " 27 GB", | 20260507 02:10:08.018 | INFO | "malloc" : " 14 GB", | 20260507 02:10:08.018 | INFO | "pinned" : " 29 GB" | 20260507 02:10:08.018 | INFO | } | 20260507 02:10:08.018 | INFO | Perf report for the last 5.14068 seconds: | 20260507 02:10:08.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:10:08.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:08.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:08.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:08.019 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:10:08.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:08.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:08.019 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 02:10:08.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:08.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:08.019 | INFO | Uploaded 36GiB to GPU in 2.03647s (17.6777GiB/s) | 20260507 02:10:08.019 | INFO | Downloaded 34GiB to GPU in 1.93799s (17.544GiB/s) | 20260507 02:10:13.159 | INFO | Analyzing chunk frames = [6267904,6268416) | 20260507 02:10:13.159 | INFO | MemInfo: { | 20260507 02:10:13.159 | INFO | "gpu" : " 17 GB", | 20260507 02:10:13.159 | INFO | "huge" : "", | 20260507 02:10:13.159 | INFO | "huge_shared" : " 27 GB", | 20260507 02:10:13.159 | INFO | "malloc" : " 14 GB", | 20260507 02:10:13.159 | INFO | "pinned" : " 29 GB" | 20260507 02:10:13.159 | INFO | } | 20260507 02:10:13.159 | INFO | Perf report for the last 5.14053 seconds: | 20260507 02:10:13.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:10:13.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:13.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:13.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:13.159 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 02:10:13.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:13.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:13.159 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:10:13.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:13.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:13.159 | INFO | Uploaded 36GiB to GPU in 2.0222s (17.8024GiB/s) | 20260507 02:10:13.159 | INFO | Downloaded 34GiB to GPU in 1.93972s (17.5283GiB/s) | 20260507 02:10:18.295 | INFO | Analyzing chunk frames = [6268416,6268928) | 20260507 02:10:18.295 | INFO | MemInfo: { | 20260507 02:10:18.295 | INFO | "gpu" : " 17 GB", | 20260507 02:10:18.295 | INFO | "huge" : "", | 20260507 02:10:18.295 | INFO | "huge_shared" : " 27 GB", | 20260507 02:10:18.295 | INFO | "malloc" : " 14 GB", | 20260507 02:10:18.295 | INFO | "pinned" : " 29 GB" | 20260507 02:10:18.295 | INFO | } | 20260507 02:10:18.295 | INFO | Perf report for the last 5.13586 seconds: | 20260507 02:10:18.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:10:18.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:18.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:18.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:18.295 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 02:10:18.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:18.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:18.295 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:10:18.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:18.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:18.295 | INFO | Uploaded 36GiB to GPU in 1.99153s (18.0766GiB/s) | 20260507 02:10:18.295 | INFO | Downloaded 34GiB to GPU in 1.91841s (17.723GiB/s) | 20260507 02:10:23.437 | INFO | Analyzing chunk frames = [6268928,6269440) | 20260507 02:10:23.437 | INFO | MemInfo: { | 20260507 02:10:23.437 | INFO | "gpu" : " 17 GB", | 20260507 02:10:23.437 | INFO | "huge" : "", | 20260507 02:10:23.437 | INFO | "huge_shared" : " 27 GB", | 20260507 02:10:23.437 | INFO | "malloc" : " 14 GB", | 20260507 02:10:23.437 | INFO | "pinned" : " 29 GB" | 20260507 02:10:23.437 | INFO | } | 20260507 02:10:23.437 | INFO | Perf report for the last 5.14213 seconds: | 20260507 02:10:23.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:10:23.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:23.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:23.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:23.437 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 02:10:23.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:23.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:23.437 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:10:23.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:23.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:23.437 | INFO | Uploaded 36GiB to GPU in 1.9874s (18.1142GiB/s) | 20260507 02:10:23.437 | INFO | Downloaded 34GiB to GPU in 1.92472s (17.6649GiB/s) | 20260507 02:10:28.573 | INFO | Analyzing chunk frames = [6269440,6269952) | 20260507 02:10:28.573 | INFO | MemInfo: { | 20260507 02:10:28.573 | INFO | "gpu" : " 17 GB", | 20260507 02:10:28.573 | INFO | "huge" : "", | 20260507 02:10:28.573 | INFO | "huge_shared" : " 27 GB", | 20260507 02:10:28.573 | INFO | "malloc" : " 14 GB", | 20260507 02:10:28.573 | INFO | "pinned" : " 29 GB" | 20260507 02:10:28.573 | INFO | } | 20260507 02:10:28.573 | INFO | Perf report for the last 5.13632 seconds: | 20260507 02:10:28.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:10:28.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:28.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:28.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:28.573 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 02:10:28.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:28.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:28.573 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:10:28.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:28.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:28.574 | INFO | Uploaded 36GiB to GPU in 2.0287s (17.7454GiB/s) | 20260507 02:10:28.574 | INFO | Downloaded 34GiB to GPU in 1.93772s (17.5464GiB/s) | 20260507 02:10:33.715 | INFO | Analyzing chunk frames = [6269952,6270464) | 20260507 02:10:33.715 | INFO | MemInfo: { | 20260507 02:10:33.715 | INFO | "gpu" : " 17 GB", | 20260507 02:10:33.715 | INFO | "huge" : "", | 20260507 02:10:33.715 | INFO | "huge_shared" : " 27 GB", | 20260507 02:10:33.715 | INFO | "malloc" : " 14 GB", | 20260507 02:10:33.715 | INFO | "pinned" : " 29 GB" | 20260507 02:10:33.715 | INFO | } | 20260507 02:10:33.715 | INFO | Perf report for the last 5.14178 seconds: | 20260507 02:10:33.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:10:33.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:33.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:33.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:33.715 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:10:33.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:33.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:33.715 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 02:10:33.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:33.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:33.715 | INFO | Uploaded 36GiB to GPU in 1.97033s (18.271GiB/s) | 20260507 02:10:33.715 | INFO | Downloaded 34GiB to GPU in 1.94682s (17.4644GiB/s) | 20260507 02:10:38.854 | INFO | Analyzing chunk frames = [6270464,6270976) | 20260507 02:10:38.854 | INFO | MemInfo: { | 20260507 02:10:38.854 | INFO | "gpu" : " 17 GB", | 20260507 02:10:38.854 | INFO | "huge" : "", | 20260507 02:10:38.854 | INFO | "huge_shared" : " 27 GB", | 20260507 02:10:38.854 | INFO | "malloc" : " 14 GB", | 20260507 02:10:38.854 | INFO | "pinned" : " 29 GB" | 20260507 02:10:38.854 | INFO | } | 20260507 02:10:38.854 | INFO | Perf report for the last 5.13908 seconds: | 20260507 02:10:38.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:10:38.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:38.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:38.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:38.854 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:10:38.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:38.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:38.854 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:10:38.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:38.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:38.854 | INFO | Uploaded 36GiB to GPU in 1.9923s (18.0696GiB/s) | 20260507 02:10:38.854 | INFO | Downloaded 34GiB to GPU in 1.96104s (17.3377GiB/s) | 20260507 02:10:38.954 | INFO | Starting a round of Baz IO | 20260507 02:10:42.064 | INFO | Populating RTMetrics report | 20260507 02:10:42.065 | INFO | RTMetrics report populated | 20260507 02:10:42.065 | INFO | Opening tmp RTMetrics json file | 20260507 02:10:42.066 | INFO | Serializing RTMetrics to json | 20260507 02:10:42.084 | INFO | Serialization done, swapping tmp file now... | 20260507 02:10:42.085 | INFO | RTMetrics done | 20260507 02:10:42.156 | INFO | Finished a round of Baz IO: Wrote 1.793 GiB in 3.202 seconds | 20260507 02:10:42.156 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.04% / 0.00% / 66.96% | 20260507 02:10:42.156 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.251706% / 7.94729e-06% / 8.74201e-05% | 20260507 02:10:42.156 | INFO | Baz IO overall: | 20260507 02:10:42.156 | INFO | BytesWritten : 3241178551802 | 20260507 02:10:42.156 | INFO | NumEvents : 1559123837079 | 20260507 02:10:43.996 | INFO | Analyzing chunk frames = [6270976,6271488) | 20260507 02:10:43.996 | INFO | MemInfo: { | 20260507 02:10:43.996 | INFO | "gpu" : " 17 GB", | 20260507 02:10:43.996 | INFO | "huge" : "", | 20260507 02:10:43.996 | INFO | "huge_shared" : " 27 GB", | 20260507 02:10:43.996 | INFO | "malloc" : " 14 GB", | 20260507 02:10:43.996 | INFO | "pinned" : " 29 GB" | 20260507 02:10:43.996 | INFO | } | 20260507 02:10:43.996 | INFO | Perf report for the last 5.14208 seconds: | 20260507 02:10:43.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:10:43.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:43.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:43.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:43.996 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260507 02:10:43.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:43.996 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 02:10:43.996 | INFO | PRE_HQ : 59.5%, 1.8, 56.1ms, 0 | 20260507 02:10:43.996 | INFO | BAZWRITER : 61.9%, 3.3, 106.4ms, 0 | 20260507 02:10:43.996 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:10:43.997 | INFO | Uploaded 36GiB to GPU in 2.01896s (17.831GiB/s) | 20260507 02:10:43.997 | INFO | Downloaded 38GiB to GPU in 2.17556s (17.4667GiB/s) | 20260507 02:10:49.132 | INFO | Analyzing chunk frames = [6271488,6272000) | 20260507 02:10:49.132 | INFO | MemInfo: { | 20260507 02:10:49.132 | INFO | "gpu" : " 17 GB", | 20260507 02:10:49.132 | INFO | "huge" : "", | 20260507 02:10:49.132 | INFO | "huge_shared" : " 27 GB", | 20260507 02:10:49.132 | INFO | "malloc" : " 14 GB", | 20260507 02:10:49.132 | INFO | "pinned" : " 29 GB" | 20260507 02:10:49.132 | INFO | } | 20260507 02:10:49.132 | INFO | Perf report for the last 5.13553 seconds: | 20260507 02:10:49.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:10:49.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:49.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:49.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:49.132 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:10:49.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:49.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:49.132 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:10:49.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:49.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:49.132 | INFO | Uploaded 36GiB to GPU in 2.04271s (17.6236GiB/s) | 20260507 02:10:49.132 | INFO | Downloaded 34GiB to GPU in 1.93481s (17.5728GiB/s) | 20260507 02:10:54.271 | INFO | Analyzing chunk frames = [6272000,6272512) | 20260507 02:10:54.271 | INFO | MemInfo: { | 20260507 02:10:54.271 | INFO | "gpu" : " 17 GB", | 20260507 02:10:54.271 | INFO | "huge" : "", | 20260507 02:10:54.271 | INFO | "huge_shared" : " 27 GB", | 20260507 02:10:54.271 | INFO | "malloc" : " 14 GB", | 20260507 02:10:54.271 | INFO | "pinned" : " 29 GB" | 20260507 02:10:54.271 | INFO | } | 20260507 02:10:54.271 | INFO | Perf report for the last 5.13889 seconds: | 20260507 02:10:54.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:10:54.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:54.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:54.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:54.271 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 02:10:54.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:54.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:54.271 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 02:10:54.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:54.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:54.271 | INFO | Uploaded 36GiB to GPU in 2.00745s (17.9332GiB/s) | 20260507 02:10:54.271 | INFO | Downloaded 34GiB to GPU in 1.93767s (17.5469GiB/s) | 20260507 02:10:59.406 | INFO | Analyzing chunk frames = [6272512,6273024) | 20260507 02:10:59.406 | INFO | MemInfo: { | 20260507 02:10:59.406 | INFO | "gpu" : " 17 GB", | 20260507 02:10:59.406 | INFO | "huge" : "", | 20260507 02:10:59.406 | INFO | "huge_shared" : " 27 GB", | 20260507 02:10:59.406 | INFO | "malloc" : " 14 GB", | 20260507 02:10:59.406 | INFO | "pinned" : " 29 GB" | 20260507 02:10:59.406 | INFO | } | 20260507 02:10:59.406 | INFO | Perf report for the last 5.13517 seconds: | 20260507 02:10:59.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:10:59.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:59.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:59.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:59.406 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 02:10:59.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:59.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:59.406 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:10:59.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:59.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:10:59.406 | INFO | Uploaded 36GiB to GPU in 1.98362s (18.1486GiB/s) | 20260507 02:10:59.406 | INFO | Downloaded 34GiB to GPU in 1.90609s (17.8376GiB/s) | 20260507 02:11:04.542 | INFO | Analyzing chunk frames = [6273024,6273536) | 20260507 02:11:04.542 | INFO | MemInfo: { | 20260507 02:11:04.542 | INFO | "gpu" : " 17 GB", | 20260507 02:11:04.542 | INFO | "huge" : "", | 20260507 02:11:04.542 | INFO | "huge_shared" : " 27 GB", | 20260507 02:11:04.542 | INFO | "malloc" : " 14 GB", | 20260507 02:11:04.542 | INFO | "pinned" : " 29 GB" | 20260507 02:11:04.542 | INFO | } | 20260507 02:11:04.543 | INFO | Perf report for the last 5.13658 seconds: | 20260507 02:11:04.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:11:04.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:04.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:04.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:04.543 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 02:11:04.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:04.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:04.543 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:11:04.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:04.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:04.543 | INFO | Uploaded 36GiB to GPU in 1.99807s (18.0174GiB/s) | 20260507 02:11:04.543 | INFO | Downloaded 34GiB to GPU in 1.93359s (17.5839GiB/s) | 20260507 02:11:09.689 | INFO | Analyzing chunk frames = [6273536,6274048) | 20260507 02:11:09.689 | INFO | MemInfo: { | 20260507 02:11:09.689 | INFO | "gpu" : " 17 GB", | 20260507 02:11:09.689 | INFO | "huge" : "", | 20260507 02:11:09.689 | INFO | "huge_shared" : " 27 GB", | 20260507 02:11:09.689 | INFO | "malloc" : " 14 GB", | 20260507 02:11:09.689 | INFO | "pinned" : " 29 GB" | 20260507 02:11:09.689 | INFO | } | 20260507 02:11:09.689 | INFO | Perf report for the last 5.14636 seconds: | 20260507 02:11:09.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:11:09.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:09.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:09.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:09.689 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 02:11:09.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:09.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:09.689 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 02:11:09.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:09.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:09.689 | INFO | Uploaded 36GiB to GPU in 2.02561s (17.7724GiB/s) | 20260507 02:11:09.689 | INFO | Downloaded 34GiB to GPU in 1.93113s (17.6062GiB/s) | 20260507 02:11:14.835 | INFO | Analyzing chunk frames = [6274048,6274560) | 20260507 02:11:14.835 | INFO | MemInfo: { | 20260507 02:11:14.835 | INFO | "gpu" : " 17 GB", | 20260507 02:11:14.835 | INFO | "huge" : "", | 20260507 02:11:14.835 | INFO | "huge_shared" : " 27 GB", | 20260507 02:11:14.835 | INFO | "malloc" : " 14 GB", | 20260507 02:11:14.835 | INFO | "pinned" : " 29 GB" | 20260507 02:11:14.835 | INFO | } | 20260507 02:11:14.836 | INFO | Perf report for the last 5.14661 seconds: | 20260507 02:11:14.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:11:14.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:14.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:14.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:14.836 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260507 02:11:14.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:14.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:14.836 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 02:11:14.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:14.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:14.836 | INFO | Uploaded 36GiB to GPU in 2.0041s (17.9632GiB/s) | 20260507 02:11:14.836 | INFO | Downloaded 34GiB to GPU in 1.99103s (17.0765GiB/s) | 20260507 02:11:19.972 | INFO | Analyzing chunk frames = [6274560,6275072) | 20260507 02:11:19.972 | INFO | MemInfo: { | 20260507 02:11:19.972 | INFO | "gpu" : " 17 GB", | 20260507 02:11:19.972 | INFO | "huge" : "", | 20260507 02:11:19.972 | INFO | "huge_shared" : " 27 GB", | 20260507 02:11:19.972 | INFO | "malloc" : " 14 GB", | 20260507 02:11:19.972 | INFO | "pinned" : " 29 GB" | 20260507 02:11:19.972 | INFO | } | 20260507 02:11:19.972 | INFO | Perf report for the last 5.1368 seconds: | 20260507 02:11:19.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:11:19.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:19.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:19.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:19.972 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:11:19.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:19.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:19.972 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:11:19.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:19.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:19.973 | INFO | Uploaded 36GiB to GPU in 2.00073s (17.9934GiB/s) | 20260507 02:11:19.973 | INFO | Downloaded 34GiB to GPU in 1.97073s (17.2524GiB/s) | 20260507 02:11:19.973 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:11:19.973 | INFO | Total mem usage: 18 GiB | 20260507 02:11:19.973 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:11:19.973 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:11:19.973 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:11:19.973 | INFO | High Water Breakdown: | 20260507 02:11:19.973 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:11:19.973 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:11:19.973 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:11:19.973 | INFO | ------------------------------------------ | 20260507 02:11:19.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:11:19.973 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:11:19.973 | INFO | Total mem usage: 26.5 GiB | 20260507 02:11:19.973 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:11:19.973 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:11:19.973 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:11:19.973 | INFO | High Water Breakdown: | 20260507 02:11:19.973 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:11:19.973 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:11:19.973 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:11:19.973 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:11:19.973 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:11:19.973 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:11:19.973 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:11:19.973 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:11:19.973 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:11:19.973 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:11:19.973 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:11:19.973 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:11:19.973 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:11:19.973 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:11:19.973 | INFO | ------------------------------------------ | 20260507 02:11:19.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:11:19.973 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:11:19.973 | INFO | Total mem usage: 25.6 GiB | 20260507 02:11:19.973 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:11:19.973 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:11:19.973 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:11:19.973 | INFO | High Water Breakdown: | 20260507 02:11:19.973 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:11:19.973 | INFO | ------------------------------------------ | 20260507 02:11:19.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:11:19.973 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:11:19.973 | INFO | Total mem usage: 15.7 GiB | 20260507 02:11:19.973 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:11:19.973 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:11:19.973 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:11:19.973 | INFO | High Water Breakdown: | 20260507 02:11:19.973 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:11:19.973 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:11:19.973 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:11:19.973 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:11:19.973 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:11:19.973 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:11:19.973 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:11:19.973 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:11:19.973 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:11:19.973 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:11:19.974 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:11:19.974 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:11:19.974 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:11:19.974 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:11:19.974 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:11:19.974 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:11:19.974 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:11:19.974 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:11:19.974 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:11:19.974 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:11:19.974 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:11:19.974 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:11:19.974 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:11:19.974 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:11:19.974 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:11:19.974 | INFO | ------------------------------------------ | 20260507 02:11:19.974 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:11:19.974 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:11:19.974 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:11:19.974 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:11:19.974 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.54e+01, 1.34e+01 / 1.87e+00 | 20260507 02:11:19.974 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 9.90e-02, 9.30e-03 / 6.27e-03 | 20260507 02:11:19.974 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.47e+00, 5.30e-01 / 3.07e-01 | 20260507 02:11:19.974 | INFO | PerfReport: 3.24%, Histogram -- 9.99e-01 / 1.45e+00, 1.73e-01 / 1.24e-01 | 20260507 02:11:19.974 | INFO | PerfReport: 16.06%, DME -- 3.96e+01 / 4.42e+01, 3.74e+00 / 2.11e+00 | 20260507 02:11:19.974 | INFO | PerfReport: 57.58%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.32e+00 / 1.71e+00 | 20260507 02:11:19.974 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.16e+00 / 2.64e+00, 3.77e-01 / 2.84e-01 | 20260507 02:11:19.974 | INFO | PerfReport: 3.10%, Metrics -- 9.55e-01 / 1.29e+00, 2.54e-01 / 1.81e-01 | 20260507 02:11:19.974 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.24e-01 / 3.76e-01, 5.20e-02 / 1.61e-02 | 20260507 02:11:19.974 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:11:19.974 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:11:19.974 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:11:19.974 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 02:11:19.974 | INFO | PerfReport: 21.72%, Upload -- 2.09e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260507 02:11:19.974 | INFO | PerfReport: 21.29%, Download -- 2.05e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 02:11:19.974 | INFO | PerfReport: 31.99%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 02:11:19.974 | INFO | PerfReport: 1.47%, LockUpload -- 1.42e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 02:11:19.974 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260507 02:11:19.974 | INFO | PerfReport: 22.30%, LockCompute -- 2.15e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 02:11:19.974 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260507 02:11:20.069 | INFO | Starting a round of Baz IO | 20260507 02:11:23.173 | INFO | Populating RTMetrics report | 20260507 02:11:23.174 | INFO | RTMetrics report populated | 20260507 02:11:23.174 | INFO | Opening tmp RTMetrics json file | 20260507 02:11:23.176 | INFO | Serializing RTMetrics to json | 20260507 02:11:23.194 | INFO | Serialization done, swapping tmp file now... | 20260507 02:11:23.194 | INFO | RTMetrics done | 20260507 02:11:23.267 | INFO | Finished a round of Baz IO: Wrote 1.795 GiB in 3.197 seconds | 20260507 02:11:23.267 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.04% / 0.00% / 66.96% | 20260507 02:11:23.267 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.252676% / 3.97364e-06% / 9.93411e-05% | 20260507 02:11:23.267 | INFO | Baz IO overall: | 20260507 02:11:23.267 | INFO | BytesWritten : 3243105845991 | 20260507 02:11:23.267 | INFO | NumEvents : 1560036386411 | 20260507 02:11:25.105 | INFO | Analyzing chunk frames = [6275072,6275584) | 20260507 02:11:25.105 | INFO | MemInfo: { | 20260507 02:11:25.105 | INFO | "gpu" : " 17 GB", | 20260507 02:11:25.105 | INFO | "huge" : "", | 20260507 02:11:25.105 | INFO | "huge_shared" : " 27 GB", | 20260507 02:11:25.105 | INFO | "malloc" : " 14 GB", | 20260507 02:11:25.105 | INFO | "pinned" : " 29 GB" | 20260507 02:11:25.105 | INFO | } | 20260507 02:11:25.105 | INFO | Perf report for the last 5.13229 seconds: | 20260507 02:11:25.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:11:25.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:25.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:25.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:25.105 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260507 02:11:25.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:25.105 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 02:11:25.105 | INFO | PRE_HQ : 59.4%, 1.8, 55.8ms, 0 | 20260507 02:11:25.105 | INFO | BAZWRITER : 61.9%, 3.4, 106.7ms, 0 | 20260507 02:11:25.105 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:11:25.105 | INFO | Uploaded 36GiB to GPU in 2.03206s (17.7161GiB/s) | 20260507 02:11:25.105 | INFO | Downloaded 38GiB to GPU in 2.18044s (17.4277GiB/s) | 20260507 02:11:30.247 | INFO | Analyzing chunk frames = [6275584,6276096) | 20260507 02:11:30.247 | INFO | MemInfo: { | 20260507 02:11:30.247 | INFO | "gpu" : " 17 GB", | 20260507 02:11:30.247 | INFO | "huge" : "", | 20260507 02:11:30.247 | INFO | "huge_shared" : " 27 GB", | 20260507 02:11:30.247 | INFO | "malloc" : " 14 GB", | 20260507 02:11:30.247 | INFO | "pinned" : " 29 GB" | 20260507 02:11:30.247 | INFO | } | 20260507 02:11:30.247 | INFO | Perf report for the last 5.1424 seconds: | 20260507 02:11:30.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:11:30.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:30.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:30.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:30.247 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:11:30.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:30.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:30.247 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:11:30.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:30.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:30.247 | INFO | Uploaded 36GiB to GPU in 2.01438s (17.8715GiB/s) | 20260507 02:11:30.247 | INFO | Downloaded 34GiB to GPU in 1.93232s (17.5955GiB/s) | 20260507 02:11:35.383 | INFO | Analyzing chunk frames = [6276096,6276608) | 20260507 02:11:35.383 | INFO | MemInfo: { | 20260507 02:11:35.383 | INFO | "gpu" : " 17 GB", | 20260507 02:11:35.383 | INFO | "huge" : "", | 20260507 02:11:35.383 | INFO | "huge_shared" : " 27 GB", | 20260507 02:11:35.383 | INFO | "malloc" : " 14 GB", | 20260507 02:11:35.383 | INFO | "pinned" : " 29 GB" | 20260507 02:11:35.383 | INFO | } | 20260507 02:11:35.383 | INFO | Perf report for the last 5.13608 seconds: | 20260507 02:11:35.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:11:35.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:35.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:35.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:35.383 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 02:11:35.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:35.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:35.383 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 02:11:35.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:35.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:35.383 | INFO | Uploaded 36GiB to GPU in 1.98815s (18.1073GiB/s) | 20260507 02:11:35.383 | INFO | Downloaded 34GiB to GPU in 1.93687s (17.5541GiB/s) | 20260507 02:11:40.526 | INFO | Analyzing chunk frames = [6276608,6277120) | 20260507 02:11:40.526 | INFO | MemInfo: { | 20260507 02:11:40.526 | INFO | "gpu" : " 17 GB", | 20260507 02:11:40.526 | INFO | "huge" : "", | 20260507 02:11:40.526 | INFO | "huge_shared" : " 27 GB", | 20260507 02:11:40.526 | INFO | "malloc" : " 14 GB", | 20260507 02:11:40.526 | INFO | "pinned" : " 29 GB" | 20260507 02:11:40.526 | INFO | } | 20260507 02:11:40.527 | INFO | Perf report for the last 5.1434 seconds: | 20260507 02:11:40.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:11:40.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:40.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:40.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:40.527 | INFO | ANALYSIS : 59.7%, 3.0, 92.8ms, 0 | 20260507 02:11:40.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:40.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:40.527 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:11:40.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:40.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:40.527 | INFO | Uploaded 36GiB to GPU in 1.9679s (18.2936GiB/s) | 20260507 02:11:40.527 | INFO | Downloaded 34GiB to GPU in 1.90904s (17.81GiB/s) | 20260507 02:11:45.670 | INFO | Analyzing chunk frames = [6277120,6277632) | 20260507 02:11:45.670 | INFO | MemInfo: { | 20260507 02:11:45.670 | INFO | "gpu" : " 17 GB", | 20260507 02:11:45.670 | INFO | "huge" : "", | 20260507 02:11:45.670 | INFO | "huge_shared" : " 27 GB", | 20260507 02:11:45.670 | INFO | "malloc" : " 14 GB", | 20260507 02:11:45.670 | INFO | "pinned" : " 29 GB" | 20260507 02:11:45.670 | INFO | } | 20260507 02:11:45.670 | INFO | Perf report for the last 5.14342 seconds: | 20260507 02:11:45.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:11:45.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:45.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:45.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:45.670 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 02:11:45.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:45.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:45.670 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:11:45.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:45.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:45.670 | INFO | Uploaded 36GiB to GPU in 2.01212s (17.8915GiB/s) | 20260507 02:11:45.670 | INFO | Downloaded 34GiB to GPU in 1.94703s (17.4625GiB/s) | 20260507 02:11:50.798 | INFO | Analyzing chunk frames = [6277632,6278144) | 20260507 02:11:50.798 | INFO | MemInfo: { | 20260507 02:11:50.798 | INFO | "gpu" : " 17 GB", | 20260507 02:11:50.798 | INFO | "huge" : "", | 20260507 02:11:50.798 | INFO | "huge_shared" : " 27 GB", | 20260507 02:11:50.798 | INFO | "malloc" : " 14 GB", | 20260507 02:11:50.798 | INFO | "pinned" : " 29 GB" | 20260507 02:11:50.798 | INFO | } | 20260507 02:11:50.798 | INFO | Perf report for the last 5.12816 seconds: | 20260507 02:11:50.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:11:50.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:50.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:50.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:50.798 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260507 02:11:50.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:50.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:50.798 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 02:11:50.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:50.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:50.798 | INFO | Uploaded 36GiB to GPU in 2.05136s (17.5494GiB/s) | 20260507 02:11:50.798 | INFO | Downloaded 34GiB to GPU in 1.96842s (17.2727GiB/s) | 20260507 02:11:55.945 | INFO | Analyzing chunk frames = [6278144,6278656) | 20260507 02:11:55.945 | INFO | MemInfo: { | 20260507 02:11:55.945 | INFO | "gpu" : " 17 GB", | 20260507 02:11:55.945 | INFO | "huge" : "", | 20260507 02:11:55.945 | INFO | "huge_shared" : " 27 GB", | 20260507 02:11:55.945 | INFO | "malloc" : " 14 GB", | 20260507 02:11:55.945 | INFO | "pinned" : " 29 GB" | 20260507 02:11:55.945 | INFO | } | 20260507 02:11:55.945 | INFO | Perf report for the last 5.1472 seconds: | 20260507 02:11:55.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:11:55.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:55.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:55.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:55.945 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 02:11:55.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:55.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:55.945 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:11:55.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:55.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:11:55.946 | INFO | Uploaded 36GiB to GPU in 1.98656s (18.1217GiB/s) | 20260507 02:11:55.946 | INFO | Downloaded 34GiB to GPU in 1.99456s (17.0464GiB/s) | 20260507 02:12:01.091 | INFO | Analyzing chunk frames = [6278656,6279168) | 20260507 02:12:01.091 | INFO | MemInfo: { | 20260507 02:12:01.091 | INFO | "gpu" : " 17 GB", | 20260507 02:12:01.091 | INFO | "huge" : "", | 20260507 02:12:01.091 | INFO | "huge_shared" : " 27 GB", | 20260507 02:12:01.091 | INFO | "malloc" : " 14 GB", | 20260507 02:12:01.091 | INFO | "pinned" : " 29 GB" | 20260507 02:12:01.091 | INFO | } | 20260507 02:12:01.091 | INFO | Perf report for the last 5.1459 seconds: | 20260507 02:12:01.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:12:01.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:01.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:01.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:01.091 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 02:12:01.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:01.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:01.092 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 02:12:01.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:01.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:01.092 | INFO | Uploaded 36GiB to GPU in 2.00954s (17.9145GiB/s) | 20260507 02:12:01.092 | INFO | Downloaded 34GiB to GPU in 1.9691s (17.2667GiB/s) | 20260507 02:12:01.188 | INFO | Starting a round of Baz IO | 20260507 02:12:04.292 | INFO | Populating RTMetrics report | 20260507 02:12:04.292 | INFO | RTMetrics report populated | 20260507 02:12:04.292 | INFO | Opening tmp RTMetrics json file | 20260507 02:12:04.298 | INFO | Serializing RTMetrics to json | 20260507 02:12:04.310 | INFO | Serialization done, swapping tmp file now... | 20260507 02:12:04.311 | INFO | RTMetrics done | 20260507 02:12:04.386 | INFO | Finished a round of Baz IO: Wrote 1.794 GiB in 3.198 seconds | 20260507 02:12:04.386 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.04% / 0.00% / 66.96% | 20260507 02:12:04.386 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.252128% / 3.97364e-06% / 7.94729e-05% | 20260507 02:12:04.386 | INFO | Baz IO overall: | 20260507 02:12:04.386 | INFO | BytesWritten : 3245031797918 | 20260507 02:12:04.386 | INFO | NumEvents : 1560948063113 | 20260507 02:12:06.220 | INFO | Analyzing chunk frames = [6279168,6279680) | 20260507 02:12:06.220 | INFO | MemInfo: { | 20260507 02:12:06.220 | INFO | "gpu" : " 17 GB", | 20260507 02:12:06.220 | INFO | "huge" : "", | 20260507 02:12:06.220 | INFO | "huge_shared" : " 27 GB", | 20260507 02:12:06.220 | INFO | "malloc" : " 14 GB", | 20260507 02:12:06.220 | INFO | "pinned" : " 29 GB" | 20260507 02:12:06.220 | INFO | } | 20260507 02:12:06.220 | INFO | Perf report for the last 5.12883 seconds: | 20260507 02:12:06.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:12:06.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:06.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:06.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:06.220 | INFO | ANALYSIS : 61.9%, 3.0, 96.0ms, 0 | 20260507 02:12:06.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:06.220 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260507 02:12:06.220 | INFO | PRE_HQ : 59.4%, 1.8, 56.0ms, 0 | 20260507 02:12:06.220 | INFO | BAZWRITER : 62.0%, 3.4, 106.7ms, 0 | 20260507 02:12:06.220 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 02:12:06.220 | INFO | Uploaded 36GiB to GPU in 2.02615s (17.7677GiB/s) | 20260507 02:12:06.220 | INFO | Downloaded 38GiB to GPU in 2.17831s (17.4448GiB/s) | 20260507 02:12:11.352 | INFO | Analyzing chunk frames = [6279680,6280192) | 20260507 02:12:11.352 | INFO | MemInfo: { | 20260507 02:12:11.352 | INFO | "gpu" : " 17 GB", | 20260507 02:12:11.352 | INFO | "huge" : "", | 20260507 02:12:11.352 | INFO | "huge_shared" : " 27 GB", | 20260507 02:12:11.352 | INFO | "malloc" : " 14 GB", | 20260507 02:12:11.352 | INFO | "pinned" : " 29 GB" | 20260507 02:12:11.352 | INFO | } | 20260507 02:12:11.352 | INFO | Perf report for the last 5.13217 seconds: | 20260507 02:12:11.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:12:11.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:11.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:11.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:11.353 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:12:11.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:11.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:11.353 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 02:12:11.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:11.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:11.353 | INFO | Uploaded 36GiB to GPU in 2.02277s (17.7974GiB/s) | 20260507 02:12:11.353 | INFO | Downloaded 34GiB to GPU in 1.9302s (17.6147GiB/s) | 20260507 02:12:16.501 | INFO | Analyzing chunk frames = [6280192,6280704) | 20260507 02:12:16.501 | INFO | MemInfo: { | 20260507 02:12:16.501 | INFO | "gpu" : " 17 GB", | 20260507 02:12:16.501 | INFO | "huge" : "", | 20260507 02:12:16.501 | INFO | "huge_shared" : " 27 GB", | 20260507 02:12:16.501 | INFO | "malloc" : " 14 GB", | 20260507 02:12:16.501 | INFO | "pinned" : " 29 GB" | 20260507 02:12:16.501 | INFO | } | 20260507 02:12:16.501 | INFO | Perf report for the last 5.14851 seconds: | 20260507 02:12:16.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:12:16.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:16.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:16.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:16.501 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 02:12:16.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:16.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:16.501 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:12:16.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:16.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:16.501 | INFO | Uploaded 36GiB to GPU in 1.97911s (18.19GiB/s) | 20260507 02:12:16.501 | INFO | Downloaded 34GiB to GPU in 1.92225s (17.6876GiB/s) | 20260507 02:12:21.643 | INFO | Analyzing chunk frames = [6280704,6281216) | 20260507 02:12:21.643 | INFO | MemInfo: { | 20260507 02:12:21.643 | INFO | "gpu" : " 17 GB", | 20260507 02:12:21.643 | INFO | "huge" : "", | 20260507 02:12:21.643 | INFO | "huge_shared" : " 27 GB", | 20260507 02:12:21.643 | INFO | "malloc" : " 14 GB", | 20260507 02:12:21.643 | INFO | "pinned" : " 29 GB" | 20260507 02:12:21.643 | INFO | } | 20260507 02:12:21.643 | INFO | Perf report for the last 5.14201 seconds: | 20260507 02:12:21.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:12:21.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:21.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:21.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:21.643 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 02:12:21.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:21.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:21.643 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:12:21.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:21.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:21.643 | INFO | Uploaded 36GiB to GPU in 2.01765s (17.8426GiB/s) | 20260507 02:12:21.643 | INFO | Downloaded 34GiB to GPU in 1.90902s (17.8102GiB/s) | 20260507 02:12:26.781 | INFO | Analyzing chunk frames = [6281216,6281728) | 20260507 02:12:26.781 | INFO | MemInfo: { | 20260507 02:12:26.781 | INFO | "gpu" : " 17 GB", | 20260507 02:12:26.781 | INFO | "huge" : "", | 20260507 02:12:26.781 | INFO | "huge_shared" : " 27 GB", | 20260507 02:12:26.781 | INFO | "malloc" : " 14 GB", | 20260507 02:12:26.781 | INFO | "pinned" : " 29 GB" | 20260507 02:12:26.781 | INFO | } | 20260507 02:12:26.781 | INFO | Perf report for the last 5.13817 seconds: | 20260507 02:12:26.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:12:26.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:26.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:26.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:26.781 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 02:12:26.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:26.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:26.781 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 02:12:26.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:26.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:26.782 | INFO | Uploaded 36GiB to GPU in 2.01641s (17.8535GiB/s) | 20260507 02:12:26.782 | INFO | Downloaded 34GiB to GPU in 1.9395s (17.5303GiB/s) | 20260507 02:12:31.920 | INFO | Analyzing chunk frames = [6281728,6282240) | 20260507 02:12:31.920 | INFO | MemInfo: { | 20260507 02:12:31.920 | INFO | "gpu" : " 17 GB", | 20260507 02:12:31.920 | INFO | "huge" : "", | 20260507 02:12:31.920 | INFO | "huge_shared" : " 27 GB", | 20260507 02:12:31.920 | INFO | "malloc" : " 14 GB", | 20260507 02:12:31.920 | INFO | "pinned" : " 29 GB" | 20260507 02:12:31.920 | INFO | } | 20260507 02:12:31.920 | INFO | Perf report for the last 5.13871 seconds: | 20260507 02:12:31.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:12:31.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:31.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:31.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:31.920 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 02:12:31.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:31.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:31.920 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 02:12:31.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:31.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:31.920 | INFO | Uploaded 36GiB to GPU in 2.03712s (17.672GiB/s) | 20260507 02:12:31.920 | INFO | Downloaded 34GiB to GPU in 1.92155s (17.6941GiB/s) | 20260507 02:12:37.058 | INFO | Analyzing chunk frames = [6282240,6282752) | 20260507 02:12:37.058 | INFO | MemInfo: { | 20260507 02:12:37.058 | INFO | "gpu" : " 17 GB", | 20260507 02:12:37.058 | INFO | "huge" : "", | 20260507 02:12:37.058 | INFO | "huge_shared" : " 27 GB", | 20260507 02:12:37.058 | INFO | "malloc" : " 14 GB", | 20260507 02:12:37.058 | INFO | "pinned" : " 29 GB" | 20260507 02:12:37.058 | INFO | } | 20260507 02:12:37.058 | INFO | Perf report for the last 5.13813 seconds: | 20260507 02:12:37.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:12:37.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:37.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:37.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:37.058 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:12:37.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:37.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:37.058 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 02:12:37.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:37.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:37.058 | INFO | Uploaded 36GiB to GPU in 1.96211s (18.3476GiB/s) | 20260507 02:12:37.058 | INFO | Downloaded 34GiB to GPU in 1.95706s (17.373GiB/s) | 20260507 02:12:42.197 | INFO | Analyzing chunk frames = [6282752,6283264) | 20260507 02:12:42.197 | INFO | MemInfo: { | 20260507 02:12:42.197 | INFO | "gpu" : " 17 GB", | 20260507 02:12:42.197 | INFO | "huge" : "", | 20260507 02:12:42.197 | INFO | "huge_shared" : " 27 GB", | 20260507 02:12:42.197 | INFO | "malloc" : " 14 GB", | 20260507 02:12:42.197 | INFO | "pinned" : " 29 GB" | 20260507 02:12:42.197 | INFO | } | 20260507 02:12:42.197 | INFO | Perf report for the last 5.13891 seconds: | 20260507 02:12:42.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:12:42.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:42.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:42.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:42.197 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 02:12:42.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:42.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:42.197 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 02:12:42.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:42.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:42.197 | INFO | Uploaded 36GiB to GPU in 2.01253s (17.888GiB/s) | 20260507 02:12:42.197 | INFO | Downloaded 34GiB to GPU in 1.97744s (17.194GiB/s) | 20260507 02:12:42.197 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:12:42.197 | INFO | Total mem usage: 18 GiB | 20260507 02:12:42.197 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:12:42.197 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 02:12:42.197 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:12:42.197 | INFO | High Water Breakdown: | 20260507 02:12:42.197 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:12:42.197 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:12:42.197 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:12:42.197 | INFO | ------------------------------------------ | 20260507 02:12:42.197 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:12:42.197 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:12:42.197 | INFO | Total mem usage: 26.5 GiB | 20260507 02:12:42.197 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:12:42.197 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:12:42.197 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:12:42.197 | INFO | High Water Breakdown: | 20260507 02:12:42.197 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:12:42.198 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:12:42.198 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:12:42.198 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:12:42.198 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:12:42.198 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:12:42.198 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:12:42.198 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:12:42.198 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:12:42.198 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:12:42.198 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:12:42.198 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:12:42.198 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:12:42.198 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:12:42.198 | INFO | ------------------------------------------ | 20260507 02:12:42.198 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:12:42.198 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:12:42.198 | INFO | Total mem usage: 25.6 GiB | 20260507 02:12:42.198 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:12:42.198 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:12:42.198 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:12:42.198 | INFO | High Water Breakdown: | 20260507 02:12:42.198 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:12:42.198 | INFO | ------------------------------------------ | 20260507 02:12:42.198 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:12:42.198 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:12:42.198 | INFO | Total mem usage: 15.7 GiB | 20260507 02:12:42.198 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:12:42.198 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:12:42.198 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:12:42.198 | INFO | High Water Breakdown: | 20260507 02:12:42.198 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:12:42.198 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:12:42.198 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:12:42.198 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:12:42.198 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:12:42.198 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:12:42.198 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:12:42.198 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:12:42.198 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:12:42.198 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:12:42.198 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:12:42.198 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:12:42.198 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:12:42.198 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:12:42.198 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:12:42.198 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:12:42.198 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:12:42.198 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:12:42.198 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:12:42.198 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:12:42.198 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:12:42.198 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:12:42.198 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:12:42.198 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:12:42.198 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:12:42.198 | INFO | ------------------------------------------ | 20260507 02:12:42.198 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:12:42.198 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:12:42.198 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:12:42.198 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:12:42.198 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.59e+01, 1.34e+01 / 1.91e+00 | 20260507 02:12:42.198 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 9.97e-02, 9.95e-03 / 4.41e-03 | 20260507 02:12:42.198 | INFO | PerfReport: 11.65%, Baseline -- 3.60e+00 / 4.42e+00, 5.18e-01 / 2.43e-01 | 20260507 02:12:42.198 | INFO | PerfReport: 3.24%, Histogram -- 1.00e+00 / 1.46e+00, 1.72e-01 / 1.27e-01 | 20260507 02:12:42.198 | INFO | PerfReport: 16.14%, DME -- 3.98e+01 / 4.50e+01, 4.07e+00 / 2.43e+00 | 20260507 02:12:42.198 | INFO | PerfReport: 57.59%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.36e+00 / 1.94e+00 | 20260507 02:12:42.198 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.15e+00 / 2.72e+00, 3.77e-01 / 2.13e-01 | 20260507 02:12:42.198 | INFO | PerfReport: 3.08%, Metrics -- 9.50e-01 / 1.25e+00, 2.58e-01 / 1.35e-01 | 20260507 02:12:42.198 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.23e-01 / 3.71e-01, 5.32e-02 / 1.66e-02 | 20260507 02:12:42.198 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:12:42.198 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:12:42.198 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:12:42.198 | INFO | PerfReport: Full Execution -- 9.65e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 02:12:42.198 | INFO | PerfReport: 21.69%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260507 02:12:42.198 | INFO | PerfReport: 21.31%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260507 02:12:42.198 | INFO | PerfReport: 32.00%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 02:12:42.198 | INFO | PerfReport: 1.43%, LockUpload -- 1.38e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260507 02:12:42.198 | INFO | PerfReport: 1.27%, LockDownload -- 1.23e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260507 02:12:42.198 | INFO | PerfReport: 22.28%, LockCompute -- 2.15e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 02:12:42.198 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260507 02:12:42.296 | INFO | Starting a round of Baz IO | 20260507 02:12:45.418 | INFO | Populating RTMetrics report | 20260507 02:12:45.418 | INFO | RTMetrics report populated | 20260507 02:12:45.418 | INFO | Opening tmp RTMetrics json file | 20260507 02:12:45.419 | INFO | Serializing RTMetrics to json | 20260507 02:12:45.420 | INFO | Serialization done, swapping tmp file now... | 20260507 02:12:45.420 | INFO | RTMetrics done | 20260507 02:12:45.515 | INFO | Finished a round of Baz IO: Wrote 1.796 GiB in 3.219 seconds | 20260507 02:12:45.515 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.03% / 0.00% / 66.96% | 20260507 02:12:45.515 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.243855% / 0% / 7.54992e-05% | 20260507 02:12:45.515 | INFO | Baz IO overall: | 20260507 02:12:45.515 | INFO | BytesWritten : 3246960368378 | 20260507 02:12:45.515 | INFO | NumEvents : 1561856735535 | 20260507 02:12:47.342 | INFO | Analyzing chunk frames = [6283264,6283776) | 20260507 02:12:47.342 | INFO | MemInfo: { | 20260507 02:12:47.342 | INFO | "gpu" : " 17 GB", | 20260507 02:12:47.342 | INFO | "huge" : "", | 20260507 02:12:47.342 | INFO | "huge_shared" : " 27 GB", | 20260507 02:12:47.342 | INFO | "malloc" : " 14 GB", | 20260507 02:12:47.342 | INFO | "pinned" : " 29 GB" | 20260507 02:12:47.342 | INFO | } | 20260507 02:12:47.342 | INFO | Perf report for the last 5.14487 seconds: | 20260507 02:12:47.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:12:47.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:47.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:47.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:47.342 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 02:12:47.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:47.342 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260507 02:12:47.342 | INFO | PRE_HQ : 60.2%, 1.9, 58.9ms, 0 | 20260507 02:12:47.342 | INFO | BAZWRITER : 62.2%, 3.4, 110.3ms, 0 | 20260507 02:12:47.342 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:12:47.342 | INFO | Uploaded 36GiB to GPU in 2.04659s (17.5902GiB/s) | 20260507 02:12:47.342 | INFO | Downloaded 38GiB to GPU in 2.18129s (17.4209GiB/s) | 20260507 02:12:52.473 | INFO | Analyzing chunk frames = [6283776,6284288) | 20260507 02:12:52.473 | INFO | MemInfo: { | 20260507 02:12:52.473 | INFO | "gpu" : " 17 GB", | 20260507 02:12:52.473 | INFO | "huge" : "", | 20260507 02:12:52.473 | INFO | "huge_shared" : " 27 GB", | 20260507 02:12:52.473 | INFO | "malloc" : " 14 GB", | 20260507 02:12:52.473 | INFO | "pinned" : " 29 GB" | 20260507 02:12:52.473 | INFO | } | 20260507 02:12:52.473 | INFO | Perf report for the last 5.13111 seconds: | 20260507 02:12:52.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:12:52.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:52.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:52.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:52.473 | INFO | ANALYSIS : 61.3%, 3.0, 95.2ms, 0 | 20260507 02:12:52.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:52.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:52.473 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 02:12:52.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:52.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:52.473 | INFO | Uploaded 36GiB to GPU in 2.04272s (17.6236GiB/s) | 20260507 02:12:52.473 | INFO | Downloaded 34GiB to GPU in 1.94s (17.5258GiB/s) | 20260507 02:12:57.613 | INFO | Analyzing chunk frames = [6284288,6284800) | 20260507 02:12:57.613 | INFO | MemInfo: { | 20260507 02:12:57.613 | INFO | "gpu" : " 17 GB", | 20260507 02:12:57.613 | INFO | "huge" : "", | 20260507 02:12:57.613 | INFO | "huge_shared" : " 27 GB", | 20260507 02:12:57.613 | INFO | "malloc" : " 14 GB", | 20260507 02:12:57.613 | INFO | "pinned" : " 29 GB" | 20260507 02:12:57.613 | INFO | } | 20260507 02:12:57.613 | INFO | Perf report for the last 5.14057 seconds: | 20260507 02:12:57.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:12:57.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:57.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:57.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:57.613 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 02:12:57.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:57.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:57.614 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:12:57.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:57.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:12:57.614 | INFO | Uploaded 36GiB to GPU in 2.00245s (17.978GiB/s) | 20260507 02:12:57.614 | INFO | Downloaded 34GiB to GPU in 1.93652s (17.5573GiB/s) | 20260507 02:13:02.755 | INFO | Analyzing chunk frames = [6284800,6285312) | 20260507 02:13:02.755 | INFO | MemInfo: { | 20260507 02:13:02.755 | INFO | "gpu" : " 17 GB", | 20260507 02:13:02.755 | INFO | "huge" : "", | 20260507 02:13:02.755 | INFO | "huge_shared" : " 27 GB", | 20260507 02:13:02.755 | INFO | "malloc" : " 14 GB", | 20260507 02:13:02.755 | INFO | "pinned" : " 29 GB" | 20260507 02:13:02.755 | INFO | } | 20260507 02:13:02.755 | INFO | Perf report for the last 5.14178 seconds: | 20260507 02:13:02.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:13:02.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:02.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:02.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:02.755 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 02:13:02.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:02.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:02.755 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:13:02.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:02.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:02.756 | INFO | Uploaded 36GiB to GPU in 1.96659s (18.3058GiB/s) | 20260507 02:13:02.756 | INFO | Downloaded 34GiB to GPU in 1.89581s (17.9343GiB/s) | 20260507 02:13:07.898 | INFO | Analyzing chunk frames = [6285312,6285824) | 20260507 02:13:07.898 | INFO | MemInfo: { | 20260507 02:13:07.898 | INFO | "gpu" : " 17 GB", | 20260507 02:13:07.898 | INFO | "huge" : "", | 20260507 02:13:07.898 | INFO | "huge_shared" : " 27 GB", | 20260507 02:13:07.898 | INFO | "malloc" : " 14 GB", | 20260507 02:13:07.898 | INFO | "pinned" : " 29 GB" | 20260507 02:13:07.898 | INFO | } | 20260507 02:13:07.898 | INFO | Perf report for the last 5.14316 seconds: | 20260507 02:13:07.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:13:07.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:07.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:07.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:07.898 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 02:13:07.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:07.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:07.899 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 02:13:07.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:07.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:07.899 | INFO | Uploaded 36GiB to GPU in 2.00838s (17.9249GiB/s) | 20260507 02:13:07.899 | INFO | Downloaded 34GiB to GPU in 1.93691s (17.5537GiB/s) | 20260507 02:13:13.033 | INFO | Analyzing chunk frames = [6285824,6286336) | 20260507 02:13:13.033 | INFO | MemInfo: { | 20260507 02:13:13.033 | INFO | "gpu" : " 17 GB", | 20260507 02:13:13.033 | INFO | "huge" : "", | 20260507 02:13:13.033 | INFO | "huge_shared" : " 27 GB", | 20260507 02:13:13.033 | INFO | "malloc" : " 14 GB", | 20260507 02:13:13.033 | INFO | "pinned" : " 29 GB" | 20260507 02:13:13.033 | INFO | } | 20260507 02:13:13.033 | INFO | Perf report for the last 5.13503 seconds: | 20260507 02:13:13.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:13:13.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:13.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:13.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:13.033 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 02:13:13.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:13.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:13.034 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:13:13.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:13.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:13.034 | INFO | Uploaded 36GiB to GPU in 2.05239s (17.5405GiB/s) | 20260507 02:13:13.034 | INFO | Downloaded 34GiB to GPU in 1.90928s (17.8077GiB/s) | 20260507 02:13:18.170 | INFO | Analyzing chunk frames = [6286336,6286848) | 20260507 02:13:18.170 | INFO | MemInfo: { | 20260507 02:13:18.170 | INFO | "gpu" : " 17 GB", | 20260507 02:13:18.170 | INFO | "huge" : "", | 20260507 02:13:18.170 | INFO | "huge_shared" : " 27 GB", | 20260507 02:13:18.170 | INFO | "malloc" : " 14 GB", | 20260507 02:13:18.170 | INFO | "pinned" : " 29 GB" | 20260507 02:13:18.170 | INFO | } | 20260507 02:13:18.170 | INFO | Perf report for the last 5.13699 seconds: | 20260507 02:13:18.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:13:18.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:18.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:18.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:18.170 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:13:18.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:18.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:18.171 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:13:18.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:18.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:18.171 | INFO | Uploaded 36GiB to GPU in 1.98377s (18.1473GiB/s) | 20260507 02:13:18.171 | INFO | Downloaded 34GiB to GPU in 1.9847s (17.131GiB/s) | 20260507 02:13:23.308 | INFO | Analyzing chunk frames = [6286848,6287360) | 20260507 02:13:23.308 | INFO | MemInfo: { | 20260507 02:13:23.308 | INFO | "gpu" : " 17 GB", | 20260507 02:13:23.308 | INFO | "huge" : "", | 20260507 02:13:23.308 | INFO | "huge_shared" : " 27 GB", | 20260507 02:13:23.308 | INFO | "malloc" : " 14 GB", | 20260507 02:13:23.308 | INFO | "pinned" : " 29 GB" | 20260507 02:13:23.308 | INFO | } | 20260507 02:13:23.308 | INFO | Perf report for the last 5.13732 seconds: | 20260507 02:13:23.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:13:23.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:23.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:23.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:23.308 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 02:13:23.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:23.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:23.308 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 02:13:23.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:23.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:23.308 | INFO | Uploaded 36GiB to GPU in 1.9979s (18.019GiB/s) | 20260507 02:13:23.308 | INFO | Downloaded 34GiB to GPU in 1.9687s (17.2703GiB/s) | 20260507 02:13:23.406 | INFO | Starting a round of Baz IO | 20260507 02:13:26.552 | INFO | Populating RTMetrics report | 20260507 02:13:26.552 | INFO | RTMetrics report populated | 20260507 02:13:26.552 | INFO | Opening tmp RTMetrics json file | 20260507 02:13:26.553 | INFO | Serializing RTMetrics to json | 20260507 02:13:26.554 | INFO | Serialization done, swapping tmp file now... | 20260507 02:13:26.555 | INFO | RTMetrics done | 20260507 02:13:26.646 | INFO | Finished a round of Baz IO: Wrote 1.797 GiB in 3.24 seconds | 20260507 02:13:26.646 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.03% / 0.00% / 66.97% | 20260507 02:13:26.646 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.24211% / 0% / 7.15256e-05% | 20260507 02:13:26.646 | INFO | Baz IO overall: | 20260507 02:13:26.646 | INFO | BytesWritten : 3248889904919 | 20260507 02:13:26.646 | INFO | NumEvents : 1562764956690 | 20260507 02:13:28.448 | INFO | Analyzing chunk frames = [6287360,6287872) | 20260507 02:13:28.448 | INFO | MemInfo: { | 20260507 02:13:28.448 | INFO | "gpu" : " 17 GB", | 20260507 02:13:28.448 | INFO | "huge" : "", | 20260507 02:13:28.448 | INFO | "huge_shared" : " 27 GB", | 20260507 02:13:28.448 | INFO | "malloc" : " 14 GB", | 20260507 02:13:28.448 | INFO | "pinned" : " 29 GB" | 20260507 02:13:28.448 | INFO | } | 20260507 02:13:28.448 | INFO | Perf report for the last 5.14053 seconds: | 20260507 02:13:28.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:13:28.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:28.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:28.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:28.448 | INFO | ANALYSIS : 62.1%, 3.0, 96.3ms, 0 | 20260507 02:13:28.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:28.448 | INFO | RT_METRICS : 45.3%, 1.0, 23.8ms, 0 | 20260507 02:13:28.448 | INFO | PRE_HQ : 60.0%, 2.0, 61.4ms, 0 | 20260507 02:13:28.448 | INFO | BAZWRITER : 62.6%, 3.5, 113.0ms, 0 | 20260507 02:13:28.448 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 02:13:28.449 | INFO | Uploaded 36GiB to GPU in 2.02346s (17.7914GiB/s) | 20260507 02:13:28.449 | INFO | Downloaded 38GiB to GPU in 2.18023s (17.4293GiB/s) | 20260507 02:13:33.595 | INFO | Analyzing chunk frames = [6287872,6288384) | 20260507 02:13:33.595 | INFO | MemInfo: { | 20260507 02:13:33.595 | INFO | "gpu" : " 17 GB", | 20260507 02:13:33.595 | INFO | "huge" : "", | 20260507 02:13:33.595 | INFO | "huge_shared" : " 27 GB", | 20260507 02:13:33.595 | INFO | "malloc" : " 14 GB", | 20260507 02:13:33.595 | INFO | "pinned" : " 29 GB" | 20260507 02:13:33.595 | INFO | } | 20260507 02:13:33.595 | INFO | Perf report for the last 5.14657 seconds: | 20260507 02:13:33.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:13:33.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:33.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:33.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:33.595 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:13:33.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:33.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:33.595 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 02:13:33.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:33.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:33.595 | INFO | Uploaded 36GiB to GPU in 1.99809s (18.0172GiB/s) | 20260507 02:13:33.595 | INFO | Downloaded 34GiB to GPU in 1.92694s (17.6445GiB/s) | 20260507 02:13:38.728 | INFO | Analyzing chunk frames = [6288384,6288896) | 20260507 02:13:38.728 | INFO | MemInfo: { | 20260507 02:13:38.728 | INFO | "gpu" : " 17 GB", | 20260507 02:13:38.728 | INFO | "huge" : "", | 20260507 02:13:38.728 | INFO | "huge_shared" : " 27 GB", | 20260507 02:13:38.728 | INFO | "malloc" : " 14 GB", | 20260507 02:13:38.728 | INFO | "pinned" : " 29 GB" | 20260507 02:13:38.728 | INFO | } | 20260507 02:13:38.728 | INFO | Perf report for the last 5.13301 seconds: | 20260507 02:13:38.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:13:38.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:38.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:38.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:38.728 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 02:13:38.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:38.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:38.728 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:13:38.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:38.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:38.728 | INFO | Uploaded 36GiB to GPU in 2.00998s (17.9107GiB/s) | 20260507 02:13:38.728 | INFO | Downloaded 34GiB to GPU in 1.94038s (17.5224GiB/s) | 20260507 02:13:43.863 | INFO | Analyzing chunk frames = [6288896,6289408) | 20260507 02:13:43.863 | INFO | MemInfo: { | 20260507 02:13:43.863 | INFO | "gpu" : " 17 GB", | 20260507 02:13:43.863 | INFO | "huge" : "", | 20260507 02:13:43.863 | INFO | "huge_shared" : " 27 GB", | 20260507 02:13:43.863 | INFO | "malloc" : " 14 GB", | 20260507 02:13:43.863 | INFO | "pinned" : " 29 GB" | 20260507 02:13:43.863 | INFO | } | 20260507 02:13:43.863 | INFO | Perf report for the last 5.13529 seconds: | 20260507 02:13:43.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:13:43.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:43.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:43.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:43.863 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 02:13:43.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:43.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:43.863 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:13:43.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:43.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:43.863 | INFO | Uploaded 36GiB to GPU in 1.99002s (18.0903GiB/s) | 20260507 02:13:43.863 | INFO | Downloaded 34GiB to GPU in 1.91665s (17.7393GiB/s) | 20260507 02:13:49.011 | INFO | Analyzing chunk frames = [6289408,6289920) | 20260507 02:13:49.011 | INFO | MemInfo: { | 20260507 02:13:49.011 | INFO | "gpu" : " 17 GB", | 20260507 02:13:49.011 | INFO | "huge" : "", | 20260507 02:13:49.011 | INFO | "huge_shared" : " 27 GB", | 20260507 02:13:49.011 | INFO | "malloc" : " 14 GB", | 20260507 02:13:49.011 | INFO | "pinned" : " 29 GB" | 20260507 02:13:49.011 | INFO | } | 20260507 02:13:49.011 | INFO | Perf report for the last 5.14825 seconds: | 20260507 02:13:49.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:13:49.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:49.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:49.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:49.011 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 02:13:49.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:49.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:49.012 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 02:13:49.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:49.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:49.012 | INFO | Uploaded 36GiB to GPU in 2.00847s (17.9241GiB/s) | 20260507 02:13:49.012 | INFO | Downloaded 34GiB to GPU in 1.93679s (17.5548GiB/s) | 20260507 02:13:54.151 | INFO | Analyzing chunk frames = [6289920,6290432) | 20260507 02:13:54.151 | INFO | MemInfo: { | 20260507 02:13:54.151 | INFO | "gpu" : " 17 GB", | 20260507 02:13:54.151 | INFO | "huge" : "", | 20260507 02:13:54.151 | INFO | "huge_shared" : " 27 GB", | 20260507 02:13:54.151 | INFO | "malloc" : " 14 GB", | 20260507 02:13:54.151 | INFO | "pinned" : " 29 GB" | 20260507 02:13:54.151 | INFO | } | 20260507 02:13:54.152 | INFO | Perf report for the last 5.14006 seconds: | 20260507 02:13:54.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:13:54.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:54.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:54.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:54.152 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:13:54.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:54.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:54.152 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:13:54.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:54.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:54.152 | INFO | Uploaded 36GiB to GPU in 2.0568s (17.5029GiB/s) | 20260507 02:13:54.152 | INFO | Downloaded 34GiB to GPU in 1.95854s (17.3599GiB/s) | 20260507 02:13:59.287 | INFO | Analyzing chunk frames = [6290432,6290944) | 20260507 02:13:59.287 | INFO | MemInfo: { | 20260507 02:13:59.287 | INFO | "gpu" : " 17 GB", | 20260507 02:13:59.287 | INFO | "huge" : "", | 20260507 02:13:59.287 | INFO | "huge_shared" : " 27 GB", | 20260507 02:13:59.287 | INFO | "malloc" : " 14 GB", | 20260507 02:13:59.287 | INFO | "pinned" : " 29 GB" | 20260507 02:13:59.287 | INFO | } | 20260507 02:13:59.287 | INFO | Perf report for the last 5.1353 seconds: | 20260507 02:13:59.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:13:59.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:59.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:59.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:59.287 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:13:59.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:59.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:59.287 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:13:59.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:59.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:13:59.287 | INFO | Uploaded 36GiB to GPU in 1.96743s (18.298GiB/s) | 20260507 02:13:59.287 | INFO | Downloaded 34GiB to GPU in 1.95268s (17.412GiB/s) | 20260507 02:14:04.426 | INFO | Analyzing chunk frames = [6290944,6291456) | 20260507 02:14:04.426 | INFO | MemInfo: { | 20260507 02:14:04.426 | INFO | "gpu" : " 17 GB", | 20260507 02:14:04.427 | INFO | "huge" : "", | 20260507 02:14:04.427 | INFO | "huge_shared" : " 27 GB", | 20260507 02:14:04.427 | INFO | "malloc" : " 14 GB", | 20260507 02:14:04.427 | INFO | "pinned" : " 29 GB" | 20260507 02:14:04.427 | INFO | } | 20260507 02:14:04.427 | INFO | Perf report for the last 5.13975 seconds: | 20260507 02:14:04.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:14:04.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:04.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:04.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:04.427 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:14:04.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:04.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:04.427 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:14:04.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:04.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:04.427 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:14:04.427 | INFO | Total mem usage: 18 GiB | 20260507 02:14:04.427 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:14:04.427 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:14:04.427 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:14:04.427 | INFO | High Water Breakdown: | 20260507 02:14:04.427 | INFO | Uploaded 36GiB to GPU in 1.97443s (18.2331GiB/s) | 20260507 02:14:04.427 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:14:04.427 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:14:04.427 | INFO | Downloaded 34GiB to GPU in 1.97014s (17.2576GiB/s) | 20260507 02:14:04.427 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:14:04.427 | INFO | ------------------------------------------ | 20260507 02:14:04.427 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:14:04.427 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:14:04.427 | INFO | Total mem usage: 26.5 GiB | 20260507 02:14:04.427 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:14:04.427 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 02:14:04.427 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260507 02:14:04.427 | INFO | High Water Breakdown: | 20260507 02:14:04.427 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:14:04.427 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:14:04.427 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:14:04.427 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:14:04.427 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:14:04.427 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:14:04.427 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:14:04.427 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:14:04.427 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:14:04.427 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:14:04.427 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:14:04.427 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:14:04.427 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:14:04.427 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:14:04.427 | INFO | ------------------------------------------ | 20260507 02:14:04.427 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:14:04.427 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:14:04.427 | INFO | Total mem usage: 25.6 GiB | 20260507 02:14:04.427 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:14:04.427 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:14:04.427 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:14:04.427 | INFO | High Water Breakdown: | 20260507 02:14:04.427 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:14:04.427 | INFO | ------------------------------------------ | 20260507 02:14:04.427 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:14:04.427 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:14:04.427 | INFO | Total mem usage: 15.7 GiB | 20260507 02:14:04.427 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:14:04.427 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:14:04.427 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 02:14:04.427 | INFO | High Water Breakdown: | 20260507 02:14:04.427 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:14:04.427 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:14:04.427 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:14:04.427 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:14:04.427 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:14:04.428 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:14:04.428 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:14:04.428 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:14:04.428 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:14:04.428 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:14:04.428 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:14:04.428 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:14:04.428 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:14:04.428 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:14:04.428 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:14:04.428 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:14:04.428 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:14:04.428 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:14:04.428 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:14:04.428 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:14:04.428 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:14:04.428 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:14:04.428 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:14:04.428 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:14:04.428 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:14:04.428 | INFO | ------------------------------------------ | 20260507 02:14:04.428 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:14:04.428 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:14:04.428 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:14:04.428 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:14:04.428 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.51e+01, 1.34e+01 / 1.47e+00 | 20260507 02:14:04.428 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.01e-01, 9.14e-03 / 5.11e-03 | 20260507 02:14:04.428 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.46e+00, 5.24e-01 / 2.73e-01 | 20260507 02:14:04.428 | INFO | PerfReport: 3.23%, Histogram -- 9.96e-01 / 1.46e+00, 1.71e-01 / 1.05e-01 | 20260507 02:14:04.428 | INFO | PerfReport: 16.07%, DME -- 3.96e+01 / 4.40e+01, 3.64e+00 / 1.63e+00 | 20260507 02:14:04.428 | INFO | PerfReport: 57.64%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.36e+00 / 1.79e+00 | 20260507 02:14:04.428 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.15e+00 / 2.68e+00, 3.78e-01 / 2.48e-01 | 20260507 02:14:04.428 | INFO | PerfReport: 3.08%, Metrics -- 9.51e-01 / 1.26e+00, 2.56e-01 / 1.52e-01 | 20260507 02:14:04.428 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.22e-01 / 3.74e-01, 5.27e-02 / 2.87e-02 | 20260507 02:14:04.428 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:14:04.428 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:14:04.428 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:14:04.428 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 02:14:04.428 | INFO | PerfReport: 21.69%, Upload -- 2.09e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260507 02:14:04.428 | INFO | PerfReport: 21.29%, Download -- 2.05e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260507 02:14:04.428 | INFO | PerfReport: 31.98%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 02:14:04.428 | INFO | PerfReport: 1.48%, LockUpload -- 1.43e+00 / 0.00e+00, 4.94e+00 / 0.00e+00 | 20260507 02:14:04.428 | INFO | PerfReport: 1.21%, LockDownload -- 1.17e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260507 02:14:04.428 | INFO | PerfReport: 22.32%, LockCompute -- 2.15e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 02:14:04.428 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 02:14:04.524 | INFO | Starting a round of Baz IO | 20260507 02:14:07.646 | INFO | Populating RTMetrics report | 20260507 02:14:07.646 | INFO | RTMetrics report populated | 20260507 02:14:07.646 | INFO | Opening tmp RTMetrics json file | 20260507 02:14:07.647 | INFO | Serializing RTMetrics to json | 20260507 02:14:07.656 | INFO | Serialization done, swapping tmp file now... | 20260507 02:14:07.667 | INFO | RTMetrics done | 20260507 02:14:07.742 | INFO | Finished a round of Baz IO: Wrote 1.791 GiB in 3.217 seconds | 20260507 02:14:07.742 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.03% / 0.00% / 66.97% | 20260507 02:14:07.742 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.241677% / 0% / 0.000103315% | 20260507 02:14:07.742 | INFO | Baz IO overall: | 20260507 02:14:07.742 | INFO | BytesWritten : 3250813376427 | 20260507 02:14:07.742 | INFO | NumEvents : 1563671787553 | 20260507 02:14:09.568 | INFO | Analyzing chunk frames = [6291456,6291968) | 20260507 02:14:09.568 | INFO | MemInfo: { | 20260507 02:14:09.568 | INFO | "gpu" : " 17 GB", | 20260507 02:14:09.568 | INFO | "huge" : "", | 20260507 02:14:09.568 | INFO | "huge_shared" : " 27 GB", | 20260507 02:14:09.568 | INFO | "malloc" : " 14 GB", | 20260507 02:14:09.568 | INFO | "pinned" : " 29 GB" | 20260507 02:14:09.568 | INFO | } | 20260507 02:14:09.568 | INFO | Perf report for the last 5.14117 seconds: | 20260507 02:14:09.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:14:09.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:09.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:09.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:09.568 | INFO | ANALYSIS : 62.1%, 3.0, 96.6ms, 0 | 20260507 02:14:09.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:09.568 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 02:14:09.568 | INFO | PRE_HQ : 59.5%, 1.8, 55.5ms, 0 | 20260507 02:14:09.568 | INFO | BAZWRITER : 62.2%, 3.3, 106.8ms, 0 | 20260507 02:14:09.568 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:14:09.568 | INFO | Uploaded 36GiB to GPU in 2.02785s (17.7528GiB/s) | 20260507 02:14:09.568 | INFO | Downloaded 38GiB to GPU in 2.17906s (17.4387GiB/s) | 20260507 02:14:14.709 | INFO | Analyzing chunk frames = [6291968,6292480) | 20260507 02:14:14.709 | INFO | MemInfo: { | 20260507 02:14:14.709 | INFO | "gpu" : " 17 GB", | 20260507 02:14:14.709 | INFO | "huge" : "", | 20260507 02:14:14.709 | INFO | "huge_shared" : " 27 GB", | 20260507 02:14:14.710 | INFO | "malloc" : " 14 GB", | 20260507 02:14:14.710 | INFO | "pinned" : " 29 GB" | 20260507 02:14:14.710 | INFO | } | 20260507 02:14:14.710 | INFO | Perf report for the last 5.1418 seconds: | 20260507 02:14:14.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:14:14.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:14.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:14.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:14.710 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:14:14.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:14.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:14.710 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:14:14.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:14.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:14.710 | INFO | Uploaded 36GiB to GPU in 1.99857s (18.0129GiB/s) | 20260507 02:14:14.710 | INFO | Downloaded 34GiB to GPU in 1.93523s (17.569GiB/s) | 20260507 02:14:19.844 | INFO | Analyzing chunk frames = [6292480,6292992) | 20260507 02:14:19.844 | INFO | MemInfo: { | 20260507 02:14:19.844 | INFO | "gpu" : " 17 GB", | 20260507 02:14:19.844 | INFO | "huge" : "", | 20260507 02:14:19.844 | INFO | "huge_shared" : " 27 GB", | 20260507 02:14:19.844 | INFO | "malloc" : " 14 GB", | 20260507 02:14:19.844 | INFO | "pinned" : " 29 GB" | 20260507 02:14:19.844 | INFO | } | 20260507 02:14:19.844 | INFO | Perf report for the last 5.13487 seconds: | 20260507 02:14:19.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:14:19.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:19.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:19.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:19.844 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 02:14:19.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:19.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:19.845 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:14:19.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:19.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:19.845 | INFO | Uploaded 36GiB to GPU in 1.98805s (18.1082GiB/s) | 20260507 02:14:19.845 | INFO | Downloaded 34GiB to GPU in 1.94414s (17.4885GiB/s) | 20260507 02:14:24.981 | INFO | Analyzing chunk frames = [6292992,6293504) | 20260507 02:14:24.981 | INFO | MemInfo: { | 20260507 02:14:24.981 | INFO | "gpu" : " 17 GB", | 20260507 02:14:24.981 | INFO | "huge" : "", | 20260507 02:14:24.981 | INFO | "huge_shared" : " 27 GB", | 20260507 02:14:24.981 | INFO | "malloc" : " 14 GB", | 20260507 02:14:24.981 | INFO | "pinned" : " 29 GB" | 20260507 02:14:24.981 | INFO | } | 20260507 02:14:24.981 | INFO | Perf report for the last 5.13644 seconds: | 20260507 02:14:24.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:14:24.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:24.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:24.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:24.982 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 02:14:24.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:24.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:24.982 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:14:24.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:24.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:24.982 | INFO | Uploaded 36GiB to GPU in 1.95592s (18.4056GiB/s) | 20260507 02:14:24.982 | INFO | Downloaded 34GiB to GPU in 1.90258s (17.8705GiB/s) | 20260507 02:14:30.121 | INFO | Analyzing chunk frames = [6293504,6294016) | 20260507 02:14:30.121 | INFO | MemInfo: { | 20260507 02:14:30.121 | INFO | "gpu" : " 17 GB", | 20260507 02:14:30.121 | INFO | "huge" : "", | 20260507 02:14:30.121 | INFO | "huge_shared" : " 27 GB", | 20260507 02:14:30.121 | INFO | "malloc" : " 14 GB", | 20260507 02:14:30.121 | INFO | "pinned" : " 29 GB" | 20260507 02:14:30.121 | INFO | } | 20260507 02:14:30.121 | INFO | Perf report for the last 5.14045 seconds: | 20260507 02:14:30.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:14:30.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:30.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:30.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:30.121 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 02:14:30.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:30.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:30.121 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:14:30.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:30.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:30.122 | INFO | Uploaded 36GiB to GPU in 2.00558s (17.9499GiB/s) | 20260507 02:14:30.122 | INFO | Downloaded 34GiB to GPU in 1.94842s (17.45GiB/s) | 20260507 02:14:35.261 | INFO | Analyzing chunk frames = [6294016,6294528) | 20260507 02:14:35.261 | INFO | MemInfo: { | 20260507 02:14:35.261 | INFO | "gpu" : " 17 GB", | 20260507 02:14:35.261 | INFO | "huge" : "", | 20260507 02:14:35.261 | INFO | "huge_shared" : " 27 GB", | 20260507 02:14:35.261 | INFO | "malloc" : " 14 GB", | 20260507 02:14:35.261 | INFO | "pinned" : " 29 GB" | 20260507 02:14:35.261 | INFO | } | 20260507 02:14:35.262 | INFO | Perf report for the last 5.14017 seconds: | 20260507 02:14:35.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:14:35.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:35.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:35.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:35.262 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 02:14:35.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:35.262 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:35.262 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 02:14:35.262 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:35.262 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:35.262 | INFO | Uploaded 36GiB to GPU in 2.01983s (17.8232GiB/s) | 20260507 02:14:35.262 | INFO | Downloaded 34GiB to GPU in 1.9476s (17.4574GiB/s) | 20260507 02:14:40.400 | INFO | Analyzing chunk frames = [6294528,6295040) | 20260507 02:14:40.400 | INFO | MemInfo: { | 20260507 02:14:40.400 | INFO | "gpu" : " 17 GB", | 20260507 02:14:40.400 | INFO | "huge" : "", | 20260507 02:14:40.400 | INFO | "huge_shared" : " 27 GB", | 20260507 02:14:40.400 | INFO | "malloc" : " 14 GB", | 20260507 02:14:40.400 | INFO | "pinned" : " 29 GB" | 20260507 02:14:40.400 | INFO | } | 20260507 02:14:40.400 | INFO | Perf report for the last 5.13813 seconds: | 20260507 02:14:40.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:14:40.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:40.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:40.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:40.400 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 02:14:40.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:40.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:40.400 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:14:40.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:40.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:40.400 | INFO | Uploaded 36GiB to GPU in 1.9461s (18.4986GiB/s) | 20260507 02:14:40.400 | INFO | Downloaded 34GiB to GPU in 1.98948s (17.0899GiB/s) | 20260507 02:14:45.543 | INFO | Analyzing chunk frames = [6295040,6295552) | 20260507 02:14:45.543 | INFO | MemInfo: { | 20260507 02:14:45.543 | INFO | "gpu" : " 17 GB", | 20260507 02:14:45.543 | INFO | "huge" : "", | 20260507 02:14:45.543 | INFO | "huge_shared" : " 27 GB", | 20260507 02:14:45.543 | INFO | "malloc" : " 14 GB", | 20260507 02:14:45.543 | INFO | "pinned" : " 29 GB" | 20260507 02:14:45.544 | INFO | } | 20260507 02:14:45.544 | INFO | Perf report for the last 5.14388 seconds: | 20260507 02:14:45.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:14:45.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:45.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:45.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:45.544 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 02:14:45.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:45.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:45.544 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:14:45.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:45.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:45.544 | INFO | Uploaded 36GiB to GPU in 1.98945s (18.0954GiB/s) | 20260507 02:14:45.544 | INFO | Downloaded 34GiB to GPU in 1.98331s (17.1431GiB/s) | 20260507 02:14:45.640 | INFO | Starting a round of Baz IO | 20260507 02:14:48.755 | INFO | Populating RTMetrics report | 20260507 02:14:48.755 | INFO | RTMetrics report populated | 20260507 02:14:48.755 | INFO | Opening tmp RTMetrics json file | 20260507 02:14:48.756 | INFO | Serializing RTMetrics to json | 20260507 02:14:48.757 | INFO | Serialization done, swapping tmp file now... | 20260507 02:14:48.757 | INFO | RTMetrics done | 20260507 02:14:48.849 | INFO | Finished a round of Baz IO: Wrote 1.788 GiB in 3.208 seconds | 20260507 02:14:48.849 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.03% / 0.00% / 66.97% | 20260507 02:14:48.849 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.242333% / 0% / 9.93411e-05% | 20260507 02:14:48.849 | INFO | Baz IO overall: | 20260507 02:14:48.849 | INFO | BytesWritten : 3252733751729 | 20260507 02:14:48.849 | INFO | NumEvents : 1564578476184 | 20260507 02:14:50.682 | INFO | Analyzing chunk frames = [6295552,6296064) | 20260507 02:14:50.682 | INFO | MemInfo: { | 20260507 02:14:50.682 | INFO | "gpu" : " 17 GB", | 20260507 02:14:50.682 | INFO | "huge" : "", | 20260507 02:14:50.682 | INFO | "huge_shared" : " 27 GB", | 20260507 02:14:50.682 | INFO | "malloc" : " 14 GB", | 20260507 02:14:50.682 | INFO | "pinned" : " 29 GB" | 20260507 02:14:50.682 | INFO | } | 20260507 02:14:50.682 | INFO | Perf report for the last 5.13882 seconds: | 20260507 02:14:50.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:14:50.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:50.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:50.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:50.682 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 02:14:50.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:50.682 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 02:14:50.682 | INFO | PRE_HQ : 59.3%, 1.8, 55.6ms, 0 | 20260507 02:14:50.683 | INFO | BAZWRITER : 62.0%, 3.3, 104.5ms, 0 | 20260507 02:14:50.683 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:14:50.683 | INFO | Uploaded 36GiB to GPU in 2.04243s (17.6261GiB/s) | 20260507 02:14:50.683 | INFO | Downloaded 38GiB to GPU in 2.17257s (17.4908GiB/s) | 20260507 02:14:55.818 | INFO | Analyzing chunk frames = [6296064,6296576) | 20260507 02:14:55.818 | INFO | MemInfo: { | 20260507 02:14:55.818 | INFO | "gpu" : " 17 GB", | 20260507 02:14:55.818 | INFO | "huge" : "", | 20260507 02:14:55.818 | INFO | "huge_shared" : " 27 GB", | 20260507 02:14:55.818 | INFO | "malloc" : " 14 GB", | 20260507 02:14:55.818 | INFO | "pinned" : " 29 GB" | 20260507 02:14:55.818 | INFO | } | 20260507 02:14:55.818 | INFO | Perf report for the last 5.13599 seconds: | 20260507 02:14:55.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:14:55.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:55.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:55.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:55.818 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:14:55.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:55.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:55.819 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 02:14:55.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:55.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:14:55.819 | INFO | Uploaded 36GiB to GPU in 2.03826s (17.6621GiB/s) | 20260507 02:14:55.819 | INFO | Downloaded 34GiB to GPU in 1.93918s (17.5332GiB/s) | 20260507 02:15:00.961 | INFO | Analyzing chunk frames = [6296576,6297088) | 20260507 02:15:00.961 | INFO | MemInfo: { | 20260507 02:15:00.961 | INFO | "gpu" : " 17 GB", | 20260507 02:15:00.961 | INFO | "huge" : "", | 20260507 02:15:00.961 | INFO | "huge_shared" : " 27 GB", | 20260507 02:15:00.961 | INFO | "malloc" : " 14 GB", | 20260507 02:15:00.961 | INFO | "pinned" : " 29 GB" | 20260507 02:15:00.961 | INFO | } | 20260507 02:15:00.961 | INFO | Perf report for the last 5.1425 seconds: | 20260507 02:15:00.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:15:00.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:00.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:00.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:00.961 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 02:15:00.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:00.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:00.961 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:15:00.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:00.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:00.961 | INFO | Uploaded 36GiB to GPU in 2.00746s (17.9331GiB/s) | 20260507 02:15:00.961 | INFO | Downloaded 34GiB to GPU in 1.93514s (17.5698GiB/s) | 20260507 02:15:06.098 | INFO | Analyzing chunk frames = [6297088,6297600) | 20260507 02:15:06.098 | INFO | MemInfo: { | 20260507 02:15:06.098 | INFO | "gpu" : " 17 GB", | 20260507 02:15:06.098 | INFO | "huge" : "", | 20260507 02:15:06.098 | INFO | "huge_shared" : " 27 GB", | 20260507 02:15:06.098 | INFO | "malloc" : " 14 GB", | 20260507 02:15:06.098 | INFO | "pinned" : " 29 GB" | 20260507 02:15:06.098 | INFO | } | 20260507 02:15:06.098 | INFO | Perf report for the last 5.13747 seconds: | 20260507 02:15:06.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:15:06.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:06.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:06.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:06.099 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 02:15:06.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:06.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:06.099 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 02:15:06.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:06.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:06.099 | INFO | Uploaded 36GiB to GPU in 1.97504s (18.2274GiB/s) | 20260507 02:15:06.099 | INFO | Downloaded 34GiB to GPU in 1.90818s (17.8181GiB/s) | 20260507 02:15:11.241 | INFO | Analyzing chunk frames = [6297600,6298112) | 20260507 02:15:11.241 | INFO | MemInfo: { | 20260507 02:15:11.241 | INFO | "gpu" : " 17 GB", | 20260507 02:15:11.241 | INFO | "huge" : "", | 20260507 02:15:11.241 | INFO | "huge_shared" : " 27 GB", | 20260507 02:15:11.241 | INFO | "malloc" : " 14 GB", | 20260507 02:15:11.242 | INFO | "pinned" : " 29 GB" | 20260507 02:15:11.242 | INFO | } | 20260507 02:15:11.242 | INFO | Perf report for the last 5.14325 seconds: | 20260507 02:15:11.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:15:11.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:11.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:11.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:11.242 | INFO | ANALYSIS : 60.2%, 3.0, 94.0ms, 0 | 20260507 02:15:11.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:11.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:11.242 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:15:11.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:11.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:11.242 | INFO | Uploaded 36GiB to GPU in 2.01999s (17.8219GiB/s) | 20260507 02:15:11.242 | INFO | Downloaded 34GiB to GPU in 1.94589s (17.4727GiB/s) | 20260507 02:15:16.385 | INFO | Analyzing chunk frames = [6298112,6298624) | 20260507 02:15:16.385 | INFO | MemInfo: { | 20260507 02:15:16.385 | INFO | "gpu" : " 17 GB", | 20260507 02:15:16.385 | INFO | "huge" : "", | 20260507 02:15:16.385 | INFO | "huge_shared" : " 27 GB", | 20260507 02:15:16.385 | INFO | "malloc" : " 14 GB", | 20260507 02:15:16.385 | INFO | "pinned" : " 29 GB" | 20260507 02:15:16.386 | INFO | } | 20260507 02:15:16.386 | INFO | Perf report for the last 5.14397 seconds: | 20260507 02:15:16.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:15:16.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:16.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:16.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:16.386 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 02:15:16.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:16.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:16.386 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 02:15:16.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:16.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:16.386 | INFO | Uploaded 36GiB to GPU in 2.03645s (17.6778GiB/s) | 20260507 02:15:16.386 | INFO | Downloaded 34GiB to GPU in 1.93441s (17.5765GiB/s) | 20260507 02:15:21.518 | INFO | Analyzing chunk frames = [6298624,6299136) | 20260507 02:15:21.518 | INFO | MemInfo: { | 20260507 02:15:21.518 | INFO | "gpu" : " 17 GB", | 20260507 02:15:21.518 | INFO | "huge" : "", | 20260507 02:15:21.518 | INFO | "huge_shared" : " 27 GB", | 20260507 02:15:21.518 | INFO | "malloc" : " 14 GB", | 20260507 02:15:21.518 | INFO | "pinned" : " 29 GB" | 20260507 02:15:21.518 | INFO | } | 20260507 02:15:21.518 | INFO | Perf report for the last 5.13269 seconds: | 20260507 02:15:21.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:15:21.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:21.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:21.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:21.518 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 02:15:21.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:21.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:21.518 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:15:21.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:21.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:21.519 | INFO | Uploaded 36GiB to GPU in 1.9687s (18.2862GiB/s) | 20260507 02:15:21.519 | INFO | Downloaded 34GiB to GPU in 1.95356s (17.4041GiB/s) | 20260507 02:15:26.656 | INFO | Analyzing chunk frames = [6299136,6299648) | 20260507 02:15:26.656 | INFO | MemInfo: { | 20260507 02:15:26.656 | INFO | "gpu" : " 17 GB", | 20260507 02:15:26.656 | INFO | "huge" : "", | 20260507 02:15:26.656 | INFO | "huge_shared" : " 27 GB", | 20260507 02:15:26.657 | INFO | "malloc" : " 14 GB", | 20260507 02:15:26.657 | INFO | "pinned" : " 29 GB" | 20260507 02:15:26.657 | INFO | } | 20260507 02:15:26.657 | INFO | Perf report for the last 5.13816 seconds: | 20260507 02:15:26.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:15:26.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:26.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:26.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:26.657 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:15:26.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:26.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:26.657 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 02:15:26.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:26.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:26.657 | INFO | Uploaded 36GiB to GPU in 1.99385s (18.0556GiB/s) | 20260507 02:15:26.657 | INFO | Downloaded 34GiB to GPU in 1.97588s (17.2076GiB/s) | 20260507 02:15:26.657 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:15:26.657 | INFO | Total mem usage: 18 GiB | 20260507 02:15:26.657 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:15:26.657 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 02:15:26.657 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:15:26.657 | INFO | High Water Breakdown: | 20260507 02:15:26.657 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:15:26.657 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:15:26.657 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:15:26.657 | INFO | ------------------------------------------ | 20260507 02:15:26.657 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:15:26.657 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:15:26.657 | INFO | Total mem usage: 26.5 GiB | 20260507 02:15:26.657 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:15:26.657 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:15:26.657 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:15:26.657 | INFO | High Water Breakdown: | 20260507 02:15:26.657 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:15:26.657 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:15:26.657 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:15:26.657 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:15:26.657 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:15:26.657 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:15:26.657 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:15:26.657 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:15:26.657 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:15:26.657 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:15:26.657 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:15:26.657 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:15:26.657 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:15:26.657 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:15:26.657 | INFO | ------------------------------------------ | 20260507 02:15:26.657 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:15:26.657 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:15:26.657 | INFO | Total mem usage: 25.6 GiB | 20260507 02:15:26.657 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:15:26.657 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:15:26.658 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:15:26.658 | INFO | High Water Breakdown: | 20260507 02:15:26.658 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:15:26.658 | INFO | ------------------------------------------ | 20260507 02:15:26.658 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:15:26.658 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:15:26.658 | INFO | Total mem usage: 15.7 GiB | 20260507 02:15:26.658 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:15:26.658 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:15:26.658 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:15:26.658 | INFO | High Water Breakdown: | 20260507 02:15:26.658 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:15:26.658 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:15:26.658 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:15:26.658 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:15:26.658 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:15:26.658 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:15:26.658 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:15:26.658 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:15:26.658 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:15:26.658 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:15:26.658 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:15:26.658 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:15:26.658 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:15:26.658 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:15:26.658 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:15:26.658 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:15:26.658 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:15:26.658 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:15:26.658 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:15:26.658 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:15:26.658 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:15:26.658 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:15:26.658 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:15:26.658 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:15:26.658 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:15:26.658 | INFO | ------------------------------------------ | 20260507 02:15:26.658 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:15:26.658 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:15:26.658 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:15:26.658 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:15:26.658 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.56e+01, 1.34e+01 / 2.35e+00 | 20260507 02:15:26.658 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.16e-03 / 3.93e-03 | 20260507 02:15:26.658 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.45e+00, 5.20e-01 / 2.24e-01 | 20260507 02:15:26.658 | INFO | PerfReport: 3.23%, Histogram -- 9.95e-01 / 1.44e+00, 1.68e-01 / 1.25e-01 | 20260507 02:15:26.658 | INFO | PerfReport: 16.16%, DME -- 3.99e+01 / 4.50e+01, 4.02e+00 / 2.50e+00 | 20260507 02:15:26.658 | INFO | PerfReport: 57.51%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.28e+00 / 1.58e+00 | 20260507 02:15:26.658 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.16e+00 / 2.72e+00, 3.84e-01 / 1.96e-01 | 20260507 02:15:26.658 | INFO | PerfReport: 3.08%, Metrics -- 9.50e-01 / 1.29e+00, 2.54e-01 / 1.74e-01 | 20260507 02:15:26.658 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.25e-01 / 3.80e-01, 1.13e-01 / 2.40e-02 | 20260507 02:15:26.658 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:15:26.658 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:15:26.658 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:15:26.658 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 02:15:26.658 | INFO | PerfReport: 21.62%, Upload -- 2.08e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260507 02:15:26.658 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260507 02:15:26.658 | INFO | PerfReport: 31.98%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 02:15:26.658 | INFO | PerfReport: 1.42%, LockUpload -- 1.37e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 02:15:26.658 | INFO | PerfReport: 1.34%, LockDownload -- 1.29e+00 / 0.00e+00, 3.17e+00 / 0.00e+00 | 20260507 02:15:26.658 | INFO | PerfReport: 22.30%, LockCompute -- 2.15e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 02:15:26.658 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.12e-02 / 0.00e+00 | 20260507 02:15:26.753 | INFO | Starting a round of Baz IO | 20260507 02:15:29.870 | INFO | Populating RTMetrics report | 20260507 02:15:29.871 | INFO | RTMetrics report populated | 20260507 02:15:29.871 | INFO | Opening tmp RTMetrics json file | 20260507 02:15:29.889 | INFO | Serializing RTMetrics to json | 20260507 02:15:29.890 | INFO | Serialization done, swapping tmp file now... | 20260507 02:15:29.891 | INFO | RTMetrics done | 20260507 02:15:29.961 | INFO | Finished a round of Baz IO: Wrote 1.788 GiB in 3.207 seconds | 20260507 02:15:29.961 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.03% / 0.00% / 66.97% | 20260507 02:15:29.961 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.242325% / 0% / 8.34465e-05% | 20260507 02:15:29.961 | INFO | Baz IO overall: | 20260507 02:15:29.961 | INFO | BytesWritten : 3254653765608 | 20260507 02:15:29.961 | INFO | NumEvents : 1565485940688 | 20260507 02:15:31.795 | INFO | Analyzing chunk frames = [6299648,6300160) | 20260507 02:15:31.795 | INFO | MemInfo: { | 20260507 02:15:31.795 | INFO | "gpu" : " 17 GB", | 20260507 02:15:31.795 | INFO | "huge" : "", | 20260507 02:15:31.795 | INFO | "huge_shared" : " 27 GB", | 20260507 02:15:31.795 | INFO | "malloc" : " 14 GB", | 20260507 02:15:31.795 | INFO | "pinned" : " 29 GB" | 20260507 02:15:31.795 | INFO | } | 20260507 02:15:31.795 | INFO | Perf report for the last 5.13809 seconds: | 20260507 02:15:31.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:15:31.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:31.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:31.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:31.795 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 02:15:31.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:31.795 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 02:15:31.795 | INFO | PRE_HQ : 59.5%, 1.8, 55.9ms, 0 | 20260507 02:15:31.795 | INFO | BAZWRITER : 62.0%, 3.3, 105.2ms, 0 | 20260507 02:15:31.795 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:15:31.795 | INFO | Uploaded 36GiB to GPU in 2.03741s (17.6695GiB/s) | 20260507 02:15:31.795 | INFO | Downloaded 38GiB to GPU in 2.17774s (17.4493GiB/s) | 20260507 02:15:36.930 | INFO | Analyzing chunk frames = [6300160,6300672) | 20260507 02:15:36.930 | INFO | MemInfo: { | 20260507 02:15:36.930 | INFO | "gpu" : " 17 GB", | 20260507 02:15:36.930 | INFO | "huge" : "", | 20260507 02:15:36.930 | INFO | "huge_shared" : " 27 GB", | 20260507 02:15:36.930 | INFO | "malloc" : " 14 GB", | 20260507 02:15:36.930 | INFO | "pinned" : " 29 GB" | 20260507 02:15:36.930 | INFO | } | 20260507 02:15:36.930 | INFO | Perf report for the last 5.13533 seconds: | 20260507 02:15:36.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:15:36.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:36.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:36.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:36.930 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:15:36.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:36.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:36.930 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:15:36.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:36.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:36.930 | INFO | Uploaded 36GiB to GPU in 2.03348s (17.7036GiB/s) | 20260507 02:15:36.930 | INFO | Downloaded 34GiB to GPU in 1.9386s (17.5384GiB/s) | 20260507 02:15:42.071 | INFO | Analyzing chunk frames = [6300672,6301184) | 20260507 02:15:42.071 | INFO | MemInfo: { | 20260507 02:15:42.071 | INFO | "gpu" : " 17 GB", | 20260507 02:15:42.071 | INFO | "huge" : "", | 20260507 02:15:42.071 | INFO | "huge_shared" : " 27 GB", | 20260507 02:15:42.071 | INFO | "malloc" : " 14 GB", | 20260507 02:15:42.071 | INFO | "pinned" : " 29 GB" | 20260507 02:15:42.071 | INFO | } | 20260507 02:15:42.071 | INFO | Perf report for the last 5.14106 seconds: | 20260507 02:15:42.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:15:42.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:42.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:42.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:42.071 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:15:42.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:42.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:42.071 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:15:42.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:42.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:42.072 | INFO | Uploaded 36GiB to GPU in 1.97964s (18.1851GiB/s) | 20260507 02:15:42.072 | INFO | Downloaded 34GiB to GPU in 1.93692s (17.5536GiB/s) | 20260507 02:15:47.216 | INFO | Analyzing chunk frames = [6301184,6301696) | 20260507 02:15:47.216 | INFO | MemInfo: { | 20260507 02:15:47.216 | INFO | "gpu" : " 17 GB", | 20260507 02:15:47.216 | INFO | "huge" : "", | 20260507 02:15:47.216 | INFO | "huge_shared" : " 27 GB", | 20260507 02:15:47.216 | INFO | "malloc" : " 14 GB", | 20260507 02:15:47.216 | INFO | "pinned" : " 29 GB" | 20260507 02:15:47.216 | INFO | } | 20260507 02:15:47.217 | INFO | Perf report for the last 5.14548 seconds: | 20260507 02:15:47.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:15:47.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:47.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:47.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:47.217 | INFO | ANALYSIS : 59.7%, 3.0, 92.8ms, 0 | 20260507 02:15:47.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:47.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:47.217 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:15:47.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:47.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:47.217 | INFO | Uploaded 36GiB to GPU in 1.98298s (18.1545GiB/s) | 20260507 02:15:47.217 | INFO | Downloaded 34GiB to GPU in 1.91012s (17.8GiB/s) | 20260507 02:15:52.356 | INFO | Analyzing chunk frames = [6301696,6302208) | 20260507 02:15:52.356 | INFO | MemInfo: { | 20260507 02:15:52.356 | INFO | "gpu" : " 17 GB", | 20260507 02:15:52.356 | INFO | "huge" : "", | 20260507 02:15:52.356 | INFO | "huge_shared" : " 27 GB", | 20260507 02:15:52.356 | INFO | "malloc" : " 14 GB", | 20260507 02:15:52.356 | INFO | "pinned" : " 29 GB" | 20260507 02:15:52.356 | INFO | } | 20260507 02:15:52.356 | INFO | Perf report for the last 5.13945 seconds: | 20260507 02:15:52.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:15:52.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:52.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:52.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:52.356 | INFO | ANALYSIS : 59.9%, 3.0, 93.2ms, 0 | 20260507 02:15:52.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:52.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:52.356 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:15:52.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:52.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:52.356 | INFO | Uploaded 36GiB to GPU in 1.99566s (18.0392GiB/s) | 20260507 02:15:52.356 | INFO | Downloaded 34GiB to GPU in 1.94468s (17.4836GiB/s) | 20260507 02:15:57.499 | INFO | Analyzing chunk frames = [6302208,6302720) | 20260507 02:15:57.499 | INFO | MemInfo: { | 20260507 02:15:57.499 | INFO | "gpu" : " 17 GB", | 20260507 02:15:57.499 | INFO | "huge" : "", | 20260507 02:15:57.499 | INFO | "huge_shared" : " 27 GB", | 20260507 02:15:57.499 | INFO | "malloc" : " 14 GB", | 20260507 02:15:57.499 | INFO | "pinned" : " 29 GB" | 20260507 02:15:57.499 | INFO | } | 20260507 02:15:57.499 | INFO | Perf report for the last 5.14255 seconds: | 20260507 02:15:57.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:15:57.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:57.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:57.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:57.499 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 02:15:57.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:57.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:57.499 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:15:57.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:57.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:15:57.499 | INFO | Uploaded 36GiB to GPU in 2.04533s (17.6011GiB/s) | 20260507 02:15:57.499 | INFO | Downloaded 34GiB to GPU in 1.95961s (17.3504GiB/s) | 20260507 02:16:02.634 | INFO | Analyzing chunk frames = [6302720,6303232) | 20260507 02:16:02.634 | INFO | MemInfo: { | 20260507 02:16:02.634 | INFO | "gpu" : " 17 GB", | 20260507 02:16:02.634 | INFO | "huge" : "", | 20260507 02:16:02.634 | INFO | "huge_shared" : " 27 GB", | 20260507 02:16:02.634 | INFO | "malloc" : " 14 GB", | 20260507 02:16:02.634 | INFO | "pinned" : " 29 GB" | 20260507 02:16:02.634 | INFO | } | 20260507 02:16:02.634 | INFO | Perf report for the last 5.13549 seconds: | 20260507 02:16:02.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:16:02.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:02.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:02.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:02.634 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:16:02.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:02.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:02.634 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 02:16:02.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:02.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:02.634 | INFO | Uploaded 36GiB to GPU in 1.96736s (18.2986GiB/s) | 20260507 02:16:02.634 | INFO | Downloaded 34GiB to GPU in 1.98498s (17.1286GiB/s) | 20260507 02:16:07.774 | INFO | Analyzing chunk frames = [6303232,6303744) | 20260507 02:16:07.774 | INFO | MemInfo: { | 20260507 02:16:07.774 | INFO | "gpu" : " 17 GB", | 20260507 02:16:07.774 | INFO | "huge" : "", | 20260507 02:16:07.774 | INFO | "huge_shared" : " 27 GB", | 20260507 02:16:07.774 | INFO | "malloc" : " 14 GB", | 20260507 02:16:07.774 | INFO | "pinned" : " 29 GB" | 20260507 02:16:07.774 | INFO | } | 20260507 02:16:07.774 | INFO | Perf report for the last 5.13993 seconds: | 20260507 02:16:07.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:16:07.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:07.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:07.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:07.774 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 02:16:07.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:07.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:07.774 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260507 02:16:07.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:07.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:07.774 | INFO | Uploaded 36GiB to GPU in 2.00165s (17.9851GiB/s) | 20260507 02:16:07.774 | INFO | Downloaded 34GiB to GPU in 1.98164s (17.1575GiB/s) | 20260507 02:16:07.872 | INFO | Starting a round of Baz IO | 20260507 02:16:10.980 | INFO | Populating RTMetrics report | 20260507 02:16:10.980 | INFO | RTMetrics report populated | 20260507 02:16:10.980 | INFO | Opening tmp RTMetrics json file | 20260507 02:16:10.981 | INFO | Serializing RTMetrics to json | 20260507 02:16:10.982 | INFO | Serialization done, swapping tmp file now... | 20260507 02:16:10.982 | INFO | RTMetrics done | 20260507 02:16:11.071 | INFO | Finished a round of Baz IO: Wrote 1.788 GiB in 3.199 seconds | 20260507 02:16:11.071 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.03% / 0.00% / 66.97% | 20260507 02:16:11.071 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.241963% / 3.97364e-06% / 7.94729e-05% | 20260507 02:16:11.071 | INFO | Baz IO overall: | 20260507 02:16:11.071 | INFO | BytesWritten : 3256573277909 | 20260507 02:16:11.071 | INFO | NumEvents : 1566393809847 | 20260507 02:16:12.908 | INFO | Analyzing chunk frames = [6303744,6304256) | 20260507 02:16:12.908 | INFO | MemInfo: { | 20260507 02:16:12.908 | INFO | "gpu" : " 17 GB", | 20260507 02:16:12.908 | INFO | "huge" : "", | 20260507 02:16:12.908 | INFO | "huge_shared" : " 27 GB", | 20260507 02:16:12.908 | INFO | "malloc" : " 14 GB", | 20260507 02:16:12.908 | INFO | "pinned" : " 29 GB" | 20260507 02:16:12.908 | INFO | } | 20260507 02:16:12.908 | INFO | Perf report for the last 5.13406 seconds: | 20260507 02:16:12.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:16:12.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:12.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:12.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:12.908 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 02:16:12.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:12.908 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260507 02:16:12.908 | INFO | PRE_HQ : 60.2%, 1.9, 60.2ms, 0 | 20260507 02:16:12.908 | INFO | BAZWRITER : 61.9%, 3.4, 109.2ms, 0 | 20260507 02:16:12.908 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 02:16:12.908 | INFO | Uploaded 36GiB to GPU in 2.01175s (17.8949GiB/s) | 20260507 02:16:12.908 | INFO | Downloaded 38GiB to GPU in 2.1796s (17.4344GiB/s) | 20260507 02:16:18.053 | INFO | Analyzing chunk frames = [6304256,6304768) | 20260507 02:16:18.053 | INFO | MemInfo: { | 20260507 02:16:18.053 | INFO | "gpu" : " 17 GB", | 20260507 02:16:18.053 | INFO | "huge" : "", | 20260507 02:16:18.053 | INFO | "huge_shared" : " 27 GB", | 20260507 02:16:18.053 | INFO | "malloc" : " 14 GB", | 20260507 02:16:18.053 | INFO | "pinned" : " 29 GB" | 20260507 02:16:18.053 | INFO | } | 20260507 02:16:18.053 | INFO | Perf report for the last 5.14526 seconds: | 20260507 02:16:18.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:16:18.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:18.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:18.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:18.053 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 02:16:18.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:18.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:18.054 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:16:18.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:18.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:18.054 | INFO | Uploaded 36GiB to GPU in 2.02169s (17.8069GiB/s) | 20260507 02:16:18.054 | INFO | Downloaded 34GiB to GPU in 1.94096s (17.5171GiB/s) | 20260507 02:16:23.192 | INFO | Analyzing chunk frames = [6304768,6305280) | 20260507 02:16:23.192 | INFO | MemInfo: { | 20260507 02:16:23.192 | INFO | "gpu" : " 17 GB", | 20260507 02:16:23.192 | INFO | "huge" : "", | 20260507 02:16:23.192 | INFO | "huge_shared" : " 27 GB", | 20260507 02:16:23.192 | INFO | "malloc" : " 14 GB", | 20260507 02:16:23.192 | INFO | "pinned" : " 29 GB" | 20260507 02:16:23.192 | INFO | } | 20260507 02:16:23.192 | INFO | Perf report for the last 5.13882 seconds: | 20260507 02:16:23.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:16:23.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:23.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:23.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:23.192 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 02:16:23.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:23.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:23.192 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 02:16:23.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:23.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:23.193 | INFO | Uploaded 36GiB to GPU in 2.00468s (17.958GiB/s) | 20260507 02:16:23.193 | INFO | Downloaded 34GiB to GPU in 1.94491s (17.4815GiB/s) | 20260507 02:16:28.326 | INFO | Analyzing chunk frames = [6305280,6305792) | 20260507 02:16:28.326 | INFO | MemInfo: { | 20260507 02:16:28.327 | INFO | "gpu" : " 17 GB", | 20260507 02:16:28.327 | INFO | "huge" : "", | 20260507 02:16:28.327 | INFO | "huge_shared" : " 27 GB", | 20260507 02:16:28.327 | INFO | "malloc" : " 14 GB", | 20260507 02:16:28.327 | INFO | "pinned" : " 29 GB" | 20260507 02:16:28.327 | INFO | } | 20260507 02:16:28.327 | INFO | Perf report for the last 5.13452 seconds: | 20260507 02:16:28.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:16:28.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:28.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:28.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:28.327 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260507 02:16:28.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:28.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:28.327 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:16:28.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:28.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:28.327 | INFO | Uploaded 36GiB to GPU in 1.98553s (18.1312GiB/s) | 20260507 02:16:28.327 | INFO | Downloaded 34GiB to GPU in 1.91714s (17.7348GiB/s) | 20260507 02:16:33.471 | INFO | Analyzing chunk frames = [6305792,6306304) | 20260507 02:16:33.471 | INFO | MemInfo: { | 20260507 02:16:33.471 | INFO | "gpu" : " 17 GB", | 20260507 02:16:33.471 | INFO | "huge" : "", | 20260507 02:16:33.471 | INFO | "huge_shared" : " 27 GB", | 20260507 02:16:33.471 | INFO | "malloc" : " 14 GB", | 20260507 02:16:33.471 | INFO | "pinned" : " 29 GB" | 20260507 02:16:33.471 | INFO | } | 20260507 02:16:33.471 | INFO | Perf report for the last 5.1443 seconds: | 20260507 02:16:33.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:16:33.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:33.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:33.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:33.471 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 02:16:33.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:33.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:33.471 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:16:33.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:33.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:33.471 | INFO | Uploaded 36GiB to GPU in 1.99845s (18.014GiB/s) | 20260507 02:16:33.471 | INFO | Downloaded 34GiB to GPU in 1.93125s (17.6052GiB/s) | 20260507 02:16:38.612 | INFO | Analyzing chunk frames = [6306304,6306816) | 20260507 02:16:38.612 | INFO | MemInfo: { | 20260507 02:16:38.612 | INFO | "gpu" : " 17 GB", | 20260507 02:16:38.612 | INFO | "huge" : "", | 20260507 02:16:38.612 | INFO | "huge_shared" : " 27 GB", | 20260507 02:16:38.612 | INFO | "malloc" : " 14 GB", | 20260507 02:16:38.612 | INFO | "pinned" : " 29 GB" | 20260507 02:16:38.612 | INFO | } | 20260507 02:16:38.612 | INFO | Perf report for the last 5.14061 seconds: | 20260507 02:16:38.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:16:38.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:38.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:38.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:38.612 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:16:38.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:38.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:38.612 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:16:38.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:38.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:38.612 | INFO | Uploaded 36GiB to GPU in 2.0573s (17.4987GiB/s) | 20260507 02:16:38.612 | INFO | Downloaded 34GiB to GPU in 1.95578s (17.3843GiB/s) | 20260507 02:16:43.745 | INFO | Analyzing chunk frames = [6306816,6307328) | 20260507 02:16:43.745 | INFO | MemInfo: { | 20260507 02:16:43.745 | INFO | "gpu" : " 17 GB", | 20260507 02:16:43.745 | INFO | "huge" : "", | 20260507 02:16:43.745 | INFO | "huge_shared" : " 27 GB", | 20260507 02:16:43.745 | INFO | "malloc" : " 14 GB", | 20260507 02:16:43.745 | INFO | "pinned" : " 29 GB" | 20260507 02:16:43.745 | INFO | } | 20260507 02:16:43.745 | INFO | Perf report for the last 5.13364 seconds: | 20260507 02:16:43.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:16:43.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:43.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:43.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:43.745 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:16:43.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:43.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:43.745 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:16:43.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:43.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:43.746 | INFO | Uploaded 36GiB to GPU in 1.98139s (18.1691GiB/s) | 20260507 02:16:43.746 | INFO | Downloaded 34GiB to GPU in 1.95979s (17.3488GiB/s) | 20260507 02:16:48.886 | INFO | Analyzing chunk frames = [6307328,6307840) | 20260507 02:16:48.886 | INFO | MemInfo: { | 20260507 02:16:48.886 | INFO | "gpu" : " 17 GB", | 20260507 02:16:48.886 | INFO | "huge" : "", | 20260507 02:16:48.886 | INFO | "huge_shared" : " 27 GB", | 20260507 02:16:48.886 | INFO | "malloc" : " 14 GB", | 20260507 02:16:48.886 | INFO | "pinned" : " 29 GB" | 20260507 02:16:48.886 | INFO | } | 20260507 02:16:48.886 | INFO | Perf report for the last 5.14109 seconds: | 20260507 02:16:48.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:16:48.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:48.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:48.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:48.887 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:16:48.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:48.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:48.887 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:16:48.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:48.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:48.887 | INFO | Uploaded 36GiB to GPU in 1.9837s (18.1479GiB/s) | 20260507 02:16:48.887 | INFO | Downloaded 34GiB to GPU in 1.9713s (17.2475GiB/s) | 20260507 02:16:48.887 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:16:48.887 | INFO | Total mem usage: 18 GiB | 20260507 02:16:48.887 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:16:48.887 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:16:48.887 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:16:48.887 | INFO | High Water Breakdown: | 20260507 02:16:48.887 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:16:48.887 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:16:48.887 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:16:48.887 | INFO | ------------------------------------------ | 20260507 02:16:48.887 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:16:48.887 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:16:48.887 | INFO | Total mem usage: 26.5 GiB | 20260507 02:16:48.887 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:16:48.887 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:16:48.887 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:16:48.887 | INFO | High Water Breakdown: | 20260507 02:16:48.887 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:16:48.887 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:16:48.887 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:16:48.887 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:16:48.887 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:16:48.887 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:16:48.887 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:16:48.887 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:16:48.887 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:16:48.887 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:16:48.887 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:16:48.887 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:16:48.887 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:16:48.887 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:16:48.887 | INFO | ------------------------------------------ | 20260507 02:16:48.887 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:16:48.887 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:16:48.887 | INFO | Total mem usage: 25.6 GiB | 20260507 02:16:48.887 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:16:48.887 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:16:48.887 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:16:48.887 | INFO | High Water Breakdown: | 20260507 02:16:48.887 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:16:48.887 | INFO | ------------------------------------------ | 20260507 02:16:48.887 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:16:48.888 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:16:48.888 | INFO | Total mem usage: 15.7 GiB | 20260507 02:16:48.888 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:16:48.888 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:16:48.888 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:16:48.888 | INFO | High Water Breakdown: | 20260507 02:16:48.888 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:16:48.888 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:16:48.888 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:16:48.888 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:16:48.888 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:16:48.888 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:16:48.888 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:16:48.888 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:16:48.888 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:16:48.888 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:16:48.888 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:16:48.888 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:16:48.888 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:16:48.888 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:16:48.888 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:16:48.888 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:16:48.888 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:16:48.888 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:16:48.888 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:16:48.888 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:16:48.888 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:16:48.888 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:16:48.888 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:16:48.888 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:16:48.888 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:16:48.888 | INFO | ------------------------------------------ | 20260507 02:16:48.888 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:16:48.888 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:16:48.888 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:16:48.888 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:16:48.888 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.50e+01, 1.35e+01 / 1.47e+00 | 20260507 02:16:48.888 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.01e-01, 9.40e-03 / 5.84e-03 | 20260507 02:16:48.888 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.43e+00, 5.26e-01 / 3.06e-01 | 20260507 02:16:48.888 | INFO | PerfReport: 3.23%, Histogram -- 9.94e-01 / 1.46e+00, 1.70e-01 / 1.14e-01 | 20260507 02:16:48.888 | INFO | PerfReport: 16.06%, DME -- 3.96e+01 / 4.38e+01, 3.71e+00 / 1.69e+00 | 20260507 02:16:48.888 | INFO | PerfReport: 57.62%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.33e+00 / 1.54e+00 | 20260507 02:16:48.888 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.15e+00 / 2.66e+00, 3.81e-01 / 2.52e-01 | 20260507 02:16:48.888 | INFO | PerfReport: 3.09%, Metrics -- 9.51e-01 / 1.27e+00, 2.54e-01 / 1.44e-01 | 20260507 02:16:48.888 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.22e-01 / 3.77e-01, 5.23e-02 / 2.60e-02 | 20260507 02:16:48.888 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:16:48.888 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:16:48.888 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:16:48.888 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260507 02:16:48.888 | INFO | PerfReport: 21.68%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260507 02:16:48.888 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260507 02:16:48.888 | INFO | PerfReport: 31.98%, Compute -- 3.08e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 02:16:48.888 | INFO | PerfReport: 1.38%, LockUpload -- 1.33e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 02:16:48.888 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.10e+00 / 0.00e+00 | 20260507 02:16:48.888 | INFO | PerfReport: 22.26%, LockCompute -- 2.14e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 02:16:48.888 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260507 02:16:48.983 | INFO | Starting a round of Baz IO | 20260507 02:16:52.101 | INFO | Populating RTMetrics report | 20260507 02:16:52.101 | INFO | RTMetrics report populated | 20260507 02:16:52.101 | INFO | Opening tmp RTMetrics json file | 20260507 02:16:52.121 | INFO | Serializing RTMetrics to json | 20260507 02:16:52.122 | INFO | Serialization done, swapping tmp file now... | 20260507 02:16:52.123 | INFO | RTMetrics done | 20260507 02:16:52.191 | INFO | Finished a round of Baz IO: Wrote 1.786 GiB in 3.208 seconds | 20260507 02:16:52.191 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.03% / 0.00% / 66.97% | 20260507 02:16:52.191 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.242472% / 0% / 7.15256e-05% | 20260507 02:16:52.191 | INFO | Baz IO overall: | 20260507 02:16:52.191 | INFO | BytesWritten : 3258490991927 | 20260507 02:16:52.191 | INFO | NumEvents : 1567300993991 | 20260507 02:16:54.023 | INFO | Analyzing chunk frames = [6307840,6308352) | 20260507 02:16:54.023 | INFO | MemInfo: { | 20260507 02:16:54.023 | INFO | "gpu" : " 17 GB", | 20260507 02:16:54.023 | INFO | "huge" : "", | 20260507 02:16:54.023 | INFO | "huge_shared" : " 27 GB", | 20260507 02:16:54.023 | INFO | "malloc" : " 14 GB", | 20260507 02:16:54.023 | INFO | "pinned" : " 29 GB" | 20260507 02:16:54.023 | INFO | } | 20260507 02:16:54.023 | INFO | Perf report for the last 5.13671 seconds: | 20260507 02:16:54.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:16:54.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:54.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:54.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:54.023 | INFO | ANALYSIS : 62.1%, 2.9, 95.6ms, 0 | 20260507 02:16:54.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:54.023 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 02:16:54.023 | INFO | PRE_HQ : 59.7%, 1.8, 55.8ms, 0 | 20260507 02:16:54.023 | INFO | BAZWRITER : 62.0%, 3.7, 116.8ms, 0 | 20260507 02:16:54.023 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 02:16:54.023 | INFO | Uploaded 36GiB to GPU in 2.04761s (17.5815GiB/s) | 20260507 02:16:54.023 | INFO | Downloaded 38GiB to GPU in 2.19594s (17.3047GiB/s) | 20260507 02:16:59.170 | INFO | Analyzing chunk frames = [6308352,6308864) | 20260507 02:16:59.170 | INFO | MemInfo: { | 20260507 02:16:59.170 | INFO | "gpu" : " 17 GB", | 20260507 02:16:59.170 | INFO | "huge" : "", | 20260507 02:16:59.170 | INFO | "huge_shared" : " 27 GB", | 20260507 02:16:59.170 | INFO | "malloc" : " 14 GB", | 20260507 02:16:59.170 | INFO | "pinned" : " 29 GB" | 20260507 02:16:59.170 | INFO | } | 20260507 02:16:59.170 | INFO | Perf report for the last 5.1466 seconds: | 20260507 02:16:59.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:16:59.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:59.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:59.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:59.170 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 02:16:59.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:59.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:59.170 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:16:59.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:59.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:16:59.172 | INFO | Uploaded 36GiB to GPU in 2.00867s (17.9223GiB/s) | 20260507 02:16:59.172 | INFO | Downloaded 34GiB to GPU in 1.93338s (17.5858GiB/s) | 20260507 02:17:04.308 | INFO | Analyzing chunk frames = [6308864,6309376) | 20260507 02:17:04.308 | INFO | MemInfo: { | 20260507 02:17:04.308 | INFO | "gpu" : " 17 GB", | 20260507 02:17:04.308 | INFO | "huge" : "", | 20260507 02:17:04.308 | INFO | "huge_shared" : " 27 GB", | 20260507 02:17:04.308 | INFO | "malloc" : " 14 GB", | 20260507 02:17:04.308 | INFO | "pinned" : " 29 GB" | 20260507 02:17:04.308 | INFO | } | 20260507 02:17:04.308 | INFO | Perf report for the last 5.1387 seconds: | 20260507 02:17:04.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:17:04.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:04.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:04.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:04.309 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 02:17:04.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:04.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:04.309 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:17:04.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:04.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:04.309 | INFO | Uploaded 36GiB to GPU in 1.97817s (18.1986GiB/s) | 20260507 02:17:04.309 | INFO | Downloaded 34GiB to GPU in 1.94129s (17.5141GiB/s) | 20260507 02:17:09.446 | INFO | Analyzing chunk frames = [6309376,6309888) | 20260507 02:17:09.446 | INFO | MemInfo: { | 20260507 02:17:09.446 | INFO | "gpu" : " 17 GB", | 20260507 02:17:09.446 | INFO | "huge" : "", | 20260507 02:17:09.446 | INFO | "huge_shared" : " 27 GB", | 20260507 02:17:09.446 | INFO | "malloc" : " 14 GB", | 20260507 02:17:09.446 | INFO | "pinned" : " 29 GB" | 20260507 02:17:09.446 | INFO | } | 20260507 02:17:09.447 | INFO | Perf report for the last 5.13798 seconds: | 20260507 02:17:09.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:17:09.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:09.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:09.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:09.447 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 02:17:09.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:09.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:09.447 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:17:09.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:09.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:09.447 | INFO | Uploaded 36GiB to GPU in 1.96475s (18.3229GiB/s) | 20260507 02:17:09.447 | INFO | Downloaded 34GiB to GPU in 1.91212s (17.7813GiB/s) | 20260507 02:17:14.582 | INFO | Analyzing chunk frames = [6309888,6310400) | 20260507 02:17:14.582 | INFO | MemInfo: { | 20260507 02:17:14.582 | INFO | "gpu" : " 17 GB", | 20260507 02:17:14.582 | INFO | "huge" : "", | 20260507 02:17:14.582 | INFO | "huge_shared" : " 27 GB", | 20260507 02:17:14.582 | INFO | "malloc" : " 14 GB", | 20260507 02:17:14.582 | INFO | "pinned" : " 29 GB" | 20260507 02:17:14.582 | INFO | } | 20260507 02:17:14.582 | INFO | Perf report for the last 5.13509 seconds: | 20260507 02:17:14.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:17:14.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:14.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:14.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:14.582 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 02:17:14.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:14.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:14.582 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 02:17:14.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:14.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:14.582 | INFO | Uploaded 36GiB to GPU in 1.99357s (18.0581GiB/s) | 20260507 02:17:14.582 | INFO | Downloaded 34GiB to GPU in 1.9557s (17.3851GiB/s) | 20260507 02:17:19.727 | INFO | Analyzing chunk frames = [6310400,6310912) | 20260507 02:17:19.727 | INFO | MemInfo: { | 20260507 02:17:19.727 | INFO | "gpu" : " 17 GB", | 20260507 02:17:19.727 | INFO | "huge" : "", | 20260507 02:17:19.727 | INFO | "huge_shared" : " 27 GB", | 20260507 02:17:19.727 | INFO | "malloc" : " 14 GB", | 20260507 02:17:19.727 | INFO | "pinned" : " 29 GB" | 20260507 02:17:19.727 | INFO | } | 20260507 02:17:19.727 | INFO | Perf report for the last 5.14519 seconds: | 20260507 02:17:19.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:17:19.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:19.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:19.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:19.727 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 02:17:19.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:19.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:19.727 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 02:17:19.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:19.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:19.727 | INFO | Uploaded 36GiB to GPU in 2.04305s (17.6207GiB/s) | 20260507 02:17:19.727 | INFO | Downloaded 34GiB to GPU in 1.95397s (17.4004GiB/s) | 20260507 02:17:24.865 | INFO | Analyzing chunk frames = [6310912,6311424) | 20260507 02:17:24.865 | INFO | MemInfo: { | 20260507 02:17:24.865 | INFO | "gpu" : " 17 GB", | 20260507 02:17:24.865 | INFO | "huge" : "", | 20260507 02:17:24.865 | INFO | "huge_shared" : " 27 GB", | 20260507 02:17:24.865 | INFO | "malloc" : " 14 GB", | 20260507 02:17:24.865 | INFO | "pinned" : " 29 GB" | 20260507 02:17:24.865 | INFO | } | 20260507 02:17:24.866 | INFO | Perf report for the last 5.13875 seconds: | 20260507 02:17:24.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:17:24.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:24.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:24.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:24.866 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:17:24.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:24.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:24.866 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:17:24.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:24.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:24.866 | INFO | Uploaded 36GiB to GPU in 1.96423s (18.3278GiB/s) | 20260507 02:17:24.866 | INFO | Downloaded 34GiB to GPU in 1.9551s (17.3904GiB/s) | 20260507 02:17:29.997 | INFO | Analyzing chunk frames = [6311424,6311936) | 20260507 02:17:29.997 | INFO | MemInfo: { | 20260507 02:17:29.997 | INFO | "gpu" : " 17 GB", | 20260507 02:17:29.997 | INFO | "huge" : "", | 20260507 02:17:29.997 | INFO | "huge_shared" : " 27 GB", | 20260507 02:17:29.997 | INFO | "malloc" : " 14 GB", | 20260507 02:17:29.997 | INFO | "pinned" : " 29 GB" | 20260507 02:17:29.997 | INFO | } | 20260507 02:17:29.997 | INFO | Perf report for the last 5.13156 seconds: | 20260507 02:17:29.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:17:29.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:29.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:29.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:29.997 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 02:17:29.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:29.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:29.997 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:17:29.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:29.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:29.997 | INFO | Uploaded 36GiB to GPU in 1.97293s (18.247GiB/s) | 20260507 02:17:29.997 | INFO | Downloaded 34GiB to GPU in 1.97003s (17.2587GiB/s) | 20260507 02:17:30.094 | INFO | Starting a round of Baz IO | 20260507 02:17:33.209 | INFO | Populating RTMetrics report | 20260507 02:17:33.209 | INFO | RTMetrics report populated | 20260507 02:17:33.209 | INFO | Opening tmp RTMetrics json file | 20260507 02:17:33.211 | INFO | Serializing RTMetrics to json | 20260507 02:17:33.229 | INFO | Serialization done, swapping tmp file now... | 20260507 02:17:33.229 | INFO | RTMetrics done | 20260507 02:17:33.300 | INFO | Finished a round of Baz IO: Wrote 1.785 GiB in 3.206 seconds | 20260507 02:17:33.300 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.02% / 0.00% / 66.97% | 20260507 02:17:33.300 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.242825% / 0% / 9.53674e-05% | 20260507 02:17:33.300 | INFO | Baz IO overall: | 20260507 02:17:33.300 | INFO | BytesWritten : 3260407529485 | 20260507 02:17:33.300 | INFO | NumEvents : 1568207195847 | 20260507 02:17:35.141 | INFO | Analyzing chunk frames = [6311936,6312448) | 20260507 02:17:35.141 | INFO | MemInfo: { | 20260507 02:17:35.141 | INFO | "gpu" : " 17 GB", | 20260507 02:17:35.141 | INFO | "huge" : "", | 20260507 02:17:35.141 | INFO | "huge_shared" : " 27 GB", | 20260507 02:17:35.141 | INFO | "malloc" : " 14 GB", | 20260507 02:17:35.141 | INFO | "pinned" : " 29 GB" | 20260507 02:17:35.141 | INFO | } | 20260507 02:17:35.141 | INFO | Perf report for the last 5.14402 seconds: | 20260507 02:17:35.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:17:35.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:35.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:35.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:35.141 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260507 02:17:35.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:35.141 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 02:17:35.141 | INFO | PRE_HQ : 59.3%, 1.8, 55.8ms, 0 | 20260507 02:17:35.141 | INFO | BAZWRITER : 61.9%, 3.3, 104.3ms, 0 | 20260507 02:17:35.141 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:17:35.141 | INFO | Uploaded 36GiB to GPU in 2.03923s (17.6537GiB/s) | 20260507 02:17:35.141 | INFO | Downloaded 38GiB to GPU in 2.18426s (17.3972GiB/s) | 20260507 02:17:40.286 | INFO | Analyzing chunk frames = [6312448,6312960) | 20260507 02:17:40.286 | INFO | MemInfo: { | 20260507 02:17:40.286 | INFO | "gpu" : " 17 GB", | 20260507 02:17:40.286 | INFO | "huge" : "", | 20260507 02:17:40.286 | INFO | "huge_shared" : " 27 GB", | 20260507 02:17:40.286 | INFO | "malloc" : " 14 GB", | 20260507 02:17:40.286 | INFO | "pinned" : " 29 GB" | 20260507 02:17:40.286 | INFO | } | 20260507 02:17:40.286 | INFO | Perf report for the last 5.14507 seconds: | 20260507 02:17:40.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:17:40.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:40.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:40.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:40.286 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:17:40.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:40.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:40.286 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:17:40.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:40.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:40.287 | INFO | Uploaded 36GiB to GPU in 1.99266s (18.0663GiB/s) | 20260507 02:17:40.287 | INFO | Downloaded 34GiB to GPU in 1.93307s (17.5886GiB/s) | 20260507 02:17:45.430 | INFO | Analyzing chunk frames = [6312960,6313472) | 20260507 02:17:45.430 | INFO | MemInfo: { | 20260507 02:17:45.430 | INFO | "gpu" : " 17 GB", | 20260507 02:17:45.430 | INFO | "huge" : "", | 20260507 02:17:45.430 | INFO | "huge_shared" : " 27 GB", | 20260507 02:17:45.430 | INFO | "malloc" : " 14 GB", | 20260507 02:17:45.430 | INFO | "pinned" : " 29 GB" | 20260507 02:17:45.430 | INFO | } | 20260507 02:17:45.430 | INFO | Perf report for the last 5.14347 seconds: | 20260507 02:17:45.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:17:45.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:45.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:45.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:45.430 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:17:45.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:45.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:45.430 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:17:45.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:45.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:45.430 | INFO | Uploaded 36GiB to GPU in 1.99752s (18.0224GiB/s) | 20260507 02:17:45.430 | INFO | Downloaded 34GiB to GPU in 1.93966s (17.5288GiB/s) | 20260507 02:17:50.560 | INFO | Analyzing chunk frames = [6313472,6313984) | 20260507 02:17:50.560 | INFO | MemInfo: { | 20260507 02:17:50.560 | INFO | "gpu" : " 17 GB", | 20260507 02:17:50.561 | INFO | "huge" : "", | 20260507 02:17:50.561 | INFO | "huge_shared" : " 27 GB", | 20260507 02:17:50.561 | INFO | "malloc" : " 14 GB", | 20260507 02:17:50.561 | INFO | "pinned" : " 29 GB" | 20260507 02:17:50.561 | INFO | } | 20260507 02:17:50.561 | INFO | Perf report for the last 5.13095 seconds: | 20260507 02:17:50.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:17:50.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:50.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:50.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:50.561 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 02:17:50.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:50.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:50.561 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:17:50.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:50.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:50.561 | INFO | Uploaded 36GiB to GPU in 1.98086s (18.174GiB/s) | 20260507 02:17:50.561 | INFO | Downloaded 34GiB to GPU in 1.93674s (17.5553GiB/s) | 20260507 02:17:55.696 | INFO | Analyzing chunk frames = [6313984,6314496) | 20260507 02:17:55.696 | INFO | MemInfo: { | 20260507 02:17:55.696 | INFO | "gpu" : " 17 GB", | 20260507 02:17:55.696 | INFO | "huge" : "", | 20260507 02:17:55.697 | INFO | "huge_shared" : " 27 GB", | 20260507 02:17:55.697 | INFO | "malloc" : " 14 GB", | 20260507 02:17:55.697 | INFO | "pinned" : " 29 GB" | 20260507 02:17:55.697 | INFO | } | 20260507 02:17:55.697 | INFO | Perf report for the last 5.13589 seconds: | 20260507 02:17:55.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:17:55.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:55.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:55.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:55.697 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 02:17:55.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:55.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:55.697 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:17:55.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:55.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:17:55.697 | INFO | Uploaded 36GiB to GPU in 2.00727s (17.9348GiB/s) | 20260507 02:17:55.697 | INFO | Downloaded 34GiB to GPU in 1.93954s (17.53GiB/s) | 20260507 02:18:00.836 | INFO | Analyzing chunk frames = [6314496,6315008) | 20260507 02:18:00.836 | INFO | MemInfo: { | 20260507 02:18:00.836 | INFO | "gpu" : " 17 GB", | 20260507 02:18:00.836 | INFO | "huge" : "", | 20260507 02:18:00.836 | INFO | "huge_shared" : " 27 GB", | 20260507 02:18:00.836 | INFO | "malloc" : " 14 GB", | 20260507 02:18:00.836 | INFO | "pinned" : " 29 GB" | 20260507 02:18:00.837 | INFO | } | 20260507 02:18:00.837 | INFO | Perf report for the last 5.13992 seconds: | 20260507 02:18:00.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:18:00.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:00.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:00.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:00.837 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:18:00.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:00.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:00.837 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 02:18:00.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:00.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:00.837 | INFO | Uploaded 36GiB to GPU in 2.05028s (17.5586GiB/s) | 20260507 02:18:00.837 | INFO | Downloaded 34GiB to GPU in 1.96688s (17.2862GiB/s) | 20260507 02:18:05.977 | INFO | Analyzing chunk frames = [6315008,6315520) | 20260507 02:18:05.977 | INFO | MemInfo: { | 20260507 02:18:05.977 | INFO | "gpu" : " 17 GB", | 20260507 02:18:05.977 | INFO | "huge" : "", | 20260507 02:18:05.977 | INFO | "huge_shared" : " 27 GB", | 20260507 02:18:05.977 | INFO | "malloc" : " 14 GB", | 20260507 02:18:05.977 | INFO | "pinned" : " 29 GB" | 20260507 02:18:05.977 | INFO | } | 20260507 02:18:05.977 | INFO | Perf report for the last 5.14062 seconds: | 20260507 02:18:05.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:18:05.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:05.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:05.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:05.977 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 02:18:05.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:05.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:05.977 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 02:18:05.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:05.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:05.977 | INFO | Uploaded 36GiB to GPU in 1.94919s (18.4692GiB/s) | 20260507 02:18:05.977 | INFO | Downloaded 34GiB to GPU in 1.99164s (17.0713GiB/s) | 20260507 02:18:11.121 | INFO | Analyzing chunk frames = [6315520,6316032) | 20260507 02:18:11.121 | INFO | MemInfo: { | 20260507 02:18:11.121 | INFO | "gpu" : " 17 GB", | 20260507 02:18:11.121 | INFO | "huge" : "", | 20260507 02:18:11.121 | INFO | "huge_shared" : " 27 GB", | 20260507 02:18:11.122 | INFO | "malloc" : " 14 GB", | 20260507 02:18:11.122 | INFO | "pinned" : " 29 GB" | 20260507 02:18:11.122 | INFO | } | 20260507 02:18:11.122 | INFO | Perf report for the last 5.14442 seconds: | 20260507 02:18:11.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:18:11.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:11.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:11.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:11.122 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:18:11.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:11.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:11.122 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 02:18:11.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:11.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:11.122 | INFO | Uploaded 36GiB to GPU in 1.98639s (18.1234GiB/s) | 20260507 02:18:11.122 | INFO | Downloaded 34GiB to GPU in 1.98273s (17.1481GiB/s) | 20260507 02:18:11.122 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:18:11.122 | INFO | Total mem usage: 18 GiB | 20260507 02:18:11.122 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:18:11.122 | INFO | Recent High Water Marks Sum: 15.1 GiB (Savings: -19%) | 20260507 02:18:11.122 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:18:11.122 | INFO | High Water Breakdown: | 20260507 02:18:11.122 | INFO | 61.1% : 9.24 GiB ( 81% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:18:11.122 | INFO | 38.8% : 5.86 GiB ( 87.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:18:11.122 | INFO | 0.132% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:18:11.122 | INFO | ------------------------------------------ | 20260507 02:18:11.122 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:18:11.122 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:18:11.122 | INFO | Total mem usage: 26.5 GiB | 20260507 02:18:11.122 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:18:11.122 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:18:11.122 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:18:11.122 | INFO | High Water Breakdown: | 20260507 02:18:11.122 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:18:11.122 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:18:11.122 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:18:11.122 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:18:11.122 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:18:11.122 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:18:11.122 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:18:11.122 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:18:11.122 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:18:11.122 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:18:11.122 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:18:11.122 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:18:11.122 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:18:11.122 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:18:11.122 | INFO | ------------------------------------------ | 20260507 02:18:11.122 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:18:11.122 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:18:11.122 | INFO | Total mem usage: 25.6 GiB | 20260507 02:18:11.122 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:18:11.122 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:18:11.122 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:18:11.122 | INFO | High Water Breakdown: | 20260507 02:18:11.122 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:18:11.122 | INFO | ------------------------------------------ | 20260507 02:18:11.122 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:18:11.122 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:18:11.122 | INFO | Total mem usage: 15.7 GiB | 20260507 02:18:11.122 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:18:11.122 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:18:11.122 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 02:18:11.122 | INFO | High Water Breakdown: | 20260507 02:18:11.122 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:18:11.122 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:18:11.122 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:18:11.122 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:18:11.122 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:18:11.122 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:18:11.122 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:18:11.122 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:18:11.122 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:18:11.122 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:18:11.122 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:18:11.122 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:18:11.122 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:18:11.122 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:18:11.122 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:18:11.122 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:18:11.122 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:18:11.123 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:18:11.123 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:18:11.123 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:18:11.123 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:18:11.123 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:18:11.123 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:18:11.123 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:18:11.123 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:18:11.123 | INFO | ------------------------------------------ | 20260507 02:18:11.123 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:18:11.123 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:18:11.123 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:18:11.123 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:18:11.123 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.52e+01, 1.32e+01 / 1.86e+00 | 20260507 02:18:11.123 | INFO | PerfReport: 0.30%, AggregateStats -- 9.14e-02 / 1.02e-01, 9.67e-03 / 4.88e-03 | 20260507 02:18:11.123 | INFO | PerfReport: 11.68%, Baseline -- 3.60e+00 / 4.47e+00, 5.26e-01 / 2.35e-01 | 20260507 02:18:11.123 | INFO | PerfReport: 3.23%, Histogram -- 9.96e-01 / 1.49e+00, 1.71e-01 / 1.15e-01 | 20260507 02:18:11.123 | INFO | PerfReport: 16.10%, DME -- 3.97e+01 / 4.45e+01, 3.97e+00 / 2.17e+00 | 20260507 02:18:11.123 | INFO | PerfReport: 57.52%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.25e+00 / 1.65e+00 | 20260507 02:18:11.123 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.17e+00 / 2.73e+00, 3.91e-01 / 2.17e-01 | 20260507 02:18:11.123 | INFO | PerfReport: 3.09%, Metrics -- 9.50e-01 / 1.28e+00, 2.52e-01 / 1.36e-01 | 20260507 02:18:11.123 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.22e-01 / 3.81e-01, 5.20e-02 / 2.64e-02 | 20260507 02:18:11.123 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:18:11.123 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:18:11.123 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:18:11.123 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 02:18:11.123 | INFO | PerfReport: 21.62%, Upload -- 2.08e+01 / 0.00e+00, 6.94e+00 / 0.00e+00 | 20260507 02:18:11.123 | INFO | PerfReport: 21.43%, Download -- 2.06e+01 / 0.00e+00, 7.48e+00 / 0.00e+00 | 20260507 02:18:11.123 | INFO | PerfReport: 31.99%, Compute -- 3.08e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 02:18:11.123 | INFO | PerfReport: 1.32%, LockUpload -- 1.27e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 02:18:11.123 | INFO | PerfReport: 1.43%, LockDownload -- 1.37e+00 / 0.00e+00, 3.27e+00 / 0.00e+00 | 20260507 02:18:11.123 | INFO | PerfReport: 22.21%, LockCompute -- 2.14e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 02:18:11.123 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260507 02:18:11.218 | INFO | Starting a round of Baz IO | 20260507 02:18:14.330 | INFO | Populating RTMetrics report | 20260507 02:18:14.330 | INFO | RTMetrics report populated | 20260507 02:18:14.330 | INFO | Opening tmp RTMetrics json file | 20260507 02:18:14.332 | INFO | Serializing RTMetrics to json | 20260507 02:18:14.337 | INFO | Serialization done, swapping tmp file now... | 20260507 02:18:14.350 | INFO | RTMetrics done | 20260507 02:18:14.419 | INFO | Finished a round of Baz IO: Wrote 1.784 GiB in 3.2 seconds | 20260507 02:18:14.419 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.02% / 0.00% / 66.98% | 20260507 02:18:14.419 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.242066% / 0% / 7.15256e-05% | 20260507 02:18:14.419 | INFO | Baz IO overall: | 20260507 02:18:14.419 | INFO | BytesWritten : 3262323314240 | 20260507 02:18:14.419 | INFO | NumEvents : 1569113089580 | 20260507 02:18:16.254 | INFO | Analyzing chunk frames = [6316032,6316544) | 20260507 02:18:16.254 | INFO | MemInfo: { | 20260507 02:18:16.254 | INFO | "gpu" : " 17 GB", | 20260507 02:18:16.254 | INFO | "huge" : "", | 20260507 02:18:16.254 | INFO | "huge_shared" : " 27 GB", | 20260507 02:18:16.254 | INFO | "malloc" : " 14 GB", | 20260507 02:18:16.254 | INFO | "pinned" : " 29 GB" | 20260507 02:18:16.254 | INFO | } | 20260507 02:18:16.254 | INFO | Perf report for the last 5.1322 seconds: | 20260507 02:18:16.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:18:16.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:16.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:16.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:16.254 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260507 02:18:16.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:16.254 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 02:18:16.254 | INFO | PRE_HQ : 59.5%, 1.8, 55.9ms, 0 | 20260507 02:18:16.254 | INFO | BAZWRITER : 61.9%, 3.3, 104.2ms, 0 | 20260507 02:18:16.254 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:18:16.254 | INFO | Uploaded 36GiB to GPU in 2.03309s (17.707GiB/s) | 20260507 02:18:16.254 | INFO | Downloaded 38GiB to GPU in 2.18318s (17.4058GiB/s) | 20260507 02:18:21.397 | INFO | Analyzing chunk frames = [6316544,6317056) | 20260507 02:18:21.397 | INFO | MemInfo: { | 20260507 02:18:21.397 | INFO | "gpu" : " 17 GB", | 20260507 02:18:21.397 | INFO | "huge" : "", | 20260507 02:18:21.398 | INFO | "huge_shared" : " 27 GB", | 20260507 02:18:21.398 | INFO | "malloc" : " 14 GB", | 20260507 02:18:21.398 | INFO | "pinned" : " 29 GB" | 20260507 02:18:21.398 | INFO | } | 20260507 02:18:21.398 | INFO | Perf report for the last 5.14375 seconds: | 20260507 02:18:21.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:18:21.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:21.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:21.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:21.398 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:18:21.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:21.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:21.398 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:18:21.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:21.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:21.398 | INFO | Uploaded 36GiB to GPU in 1.99104s (18.081GiB/s) | 20260507 02:18:21.398 | INFO | Downloaded 34GiB to GPU in 1.93018s (17.6149GiB/s) | 20260507 02:18:26.538 | INFO | Analyzing chunk frames = [6317056,6317568) | 20260507 02:18:26.538 | INFO | MemInfo: { | 20260507 02:18:26.538 | INFO | "gpu" : " 17 GB", | 20260507 02:18:26.538 | INFO | "huge" : "", | 20260507 02:18:26.538 | INFO | "huge_shared" : " 27 GB", | 20260507 02:18:26.538 | INFO | "malloc" : " 14 GB", | 20260507 02:18:26.538 | INFO | "pinned" : " 29 GB" | 20260507 02:18:26.538 | INFO | } | 20260507 02:18:26.538 | INFO | Perf report for the last 5.14036 seconds: | 20260507 02:18:26.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:18:26.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:26.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:26.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:26.538 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 02:18:26.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:26.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:26.538 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:18:26.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:26.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:26.538 | INFO | Uploaded 36GiB to GPU in 1.99645s (18.032GiB/s) | 20260507 02:18:26.538 | INFO | Downloaded 34GiB to GPU in 1.93908s (17.5341GiB/s) | 20260507 02:18:31.674 | INFO | Analyzing chunk frames = [6317568,6318080) | 20260507 02:18:31.674 | INFO | MemInfo: { | 20260507 02:18:31.674 | INFO | "gpu" : " 17 GB", | 20260507 02:18:31.674 | INFO | "huge" : "", | 20260507 02:18:31.674 | INFO | "huge_shared" : " 27 GB", | 20260507 02:18:31.674 | INFO | "malloc" : " 14 GB", | 20260507 02:18:31.674 | INFO | "pinned" : " 29 GB" | 20260507 02:18:31.674 | INFO | } | 20260507 02:18:31.674 | INFO | Perf report for the last 5.13607 seconds: | 20260507 02:18:31.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:18:31.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:31.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:31.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:31.674 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:18:31.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:31.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:31.674 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:18:31.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:31.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:31.674 | INFO | Uploaded 36GiB to GPU in 1.95621s (18.4029GiB/s) | 20260507 02:18:31.674 | INFO | Downloaded 34GiB to GPU in 1.91705s (17.7356GiB/s) | 20260507 02:18:36.813 | INFO | Analyzing chunk frames = [6318080,6318592) | 20260507 02:18:36.813 | INFO | MemInfo: { | 20260507 02:18:36.813 | INFO | "gpu" : " 17 GB", | 20260507 02:18:36.813 | INFO | "huge" : "", | 20260507 02:18:36.813 | INFO | "huge_shared" : " 27 GB", | 20260507 02:18:36.813 | INFO | "malloc" : " 14 GB", | 20260507 02:18:36.813 | INFO | "pinned" : " 29 GB" | 20260507 02:18:36.813 | INFO | } | 20260507 02:18:36.813 | INFO | Perf report for the last 5.13871 seconds: | 20260507 02:18:36.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:18:36.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:36.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:36.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:36.813 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 02:18:36.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:36.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:36.813 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 02:18:36.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:36.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:36.813 | INFO | Uploaded 36GiB to GPU in 2.02952s (17.7382GiB/s) | 20260507 02:18:36.813 | INFO | Downloaded 34GiB to GPU in 1.94139s (17.5132GiB/s) | 20260507 02:18:41.952 | INFO | Analyzing chunk frames = [6318592,6319104) | 20260507 02:18:41.952 | INFO | MemInfo: { | 20260507 02:18:41.952 | INFO | "gpu" : " 17 GB", | 20260507 02:18:41.952 | INFO | "huge" : "", | 20260507 02:18:41.952 | INFO | "huge_shared" : " 27 GB", | 20260507 02:18:41.952 | INFO | "malloc" : " 14 GB", | 20260507 02:18:41.952 | INFO | "pinned" : " 29 GB" | 20260507 02:18:41.952 | INFO | } | 20260507 02:18:41.952 | INFO | Perf report for the last 5.13903 seconds: | 20260507 02:18:41.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:18:41.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:41.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:41.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:41.952 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 02:18:41.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:41.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:41.952 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:18:41.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:41.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:41.952 | INFO | Uploaded 36GiB to GPU in 2.05026s (17.5588GiB/s) | 20260507 02:18:41.952 | INFO | Downloaded 34GiB to GPU in 1.96228s (17.3268GiB/s) | 20260507 02:18:47.094 | INFO | Analyzing chunk frames = [6319104,6319616) | 20260507 02:18:47.094 | INFO | MemInfo: { | 20260507 02:18:47.094 | INFO | "gpu" : " 17 GB", | 20260507 02:18:47.094 | INFO | "huge" : "", | 20260507 02:18:47.094 | INFO | "huge_shared" : " 27 GB", | 20260507 02:18:47.094 | INFO | "malloc" : " 14 GB", | 20260507 02:18:47.094 | INFO | "pinned" : " 29 GB" | 20260507 02:18:47.094 | INFO | } | 20260507 02:18:47.094 | INFO | Perf report for the last 5.14254 seconds: | 20260507 02:18:47.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:18:47.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:47.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:47.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:47.094 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:18:47.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:47.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:47.094 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 02:18:47.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:47.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:47.095 | INFO | Uploaded 36GiB to GPU in 1.97018s (18.2725GiB/s) | 20260507 02:18:47.095 | INFO | Downloaded 34GiB to GPU in 1.98409s (17.1363GiB/s) | 20260507 02:18:52.234 | INFO | Analyzing chunk frames = [6319616,6320128) | 20260507 02:18:52.234 | INFO | MemInfo: { | 20260507 02:18:52.234 | INFO | "gpu" : " 17 GB", | 20260507 02:18:52.234 | INFO | "huge" : "", | 20260507 02:18:52.234 | INFO | "huge_shared" : " 27 GB", | 20260507 02:18:52.234 | INFO | "malloc" : " 14 GB", | 20260507 02:18:52.234 | INFO | "pinned" : " 29 GB" | 20260507 02:18:52.234 | INFO | } | 20260507 02:18:52.234 | INFO | Perf report for the last 5.13987 seconds: | 20260507 02:18:52.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:18:52.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:52.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:52.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:52.234 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:18:52.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:52.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:52.234 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:18:52.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:52.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:52.234 | INFO | Uploaded 36GiB to GPU in 1.98244s (18.1595GiB/s) | 20260507 02:18:52.234 | INFO | Downloaded 34GiB to GPU in 1.97353s (17.2281GiB/s) | 20260507 02:18:52.331 | INFO | Starting a round of Baz IO | 20260507 02:18:55.442 | INFO | Populating RTMetrics report | 20260507 02:18:55.442 | INFO | RTMetrics report populated | 20260507 02:18:55.442 | INFO | Opening tmp RTMetrics json file | 20260507 02:18:55.443 | INFO | Serializing RTMetrics to json | 20260507 02:18:55.446 | INFO | Serialization done, swapping tmp file now... | 20260507 02:18:55.453 | INFO | RTMetrics done | 20260507 02:18:55.530 | INFO | Finished a round of Baz IO: Wrote 1.785 GiB in 3.198 seconds | 20260507 02:18:55.530 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.02% / 0.00% / 66.98% | 20260507 02:18:55.530 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.242949% / 0% / 5.5631e-05% | 20260507 02:18:55.530 | INFO | Baz IO overall: | 20260507 02:18:55.530 | INFO | BytesWritten : 3264239992840 | 20260507 02:18:55.530 | INFO | NumEvents : 1570019959557 | 20260507 02:18:57.377 | INFO | Analyzing chunk frames = [6320128,6320640) | 20260507 02:18:57.377 | INFO | MemInfo: { | 20260507 02:18:57.377 | INFO | "gpu" : " 17 GB", | 20260507 02:18:57.377 | INFO | "huge" : "", | 20260507 02:18:57.377 | INFO | "huge_shared" : " 27 GB", | 20260507 02:18:57.377 | INFO | "malloc" : " 14 GB", | 20260507 02:18:57.377 | INFO | "pinned" : " 29 GB" | 20260507 02:18:57.377 | INFO | } | 20260507 02:18:57.377 | INFO | Perf report for the last 5.14323 seconds: | 20260507 02:18:57.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:18:57.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:57.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:57.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:57.378 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260507 02:18:57.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:18:57.378 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 02:18:57.378 | INFO | PRE_HQ : 59.3%, 1.8, 55.9ms, 0 | 20260507 02:18:57.378 | INFO | BAZWRITER : 61.8%, 3.3, 105.1ms, 0 | 20260507 02:18:57.378 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:18:57.378 | INFO | Uploaded 36GiB to GPU in 2.03137s (17.722GiB/s) | 20260507 02:18:57.378 | INFO | Downloaded 38GiB to GPU in 2.18557s (17.3867GiB/s) | 20260507 02:19:02.514 | INFO | Analyzing chunk frames = [6320640,6321152) | 20260507 02:19:02.514 | INFO | MemInfo: { | 20260507 02:19:02.514 | INFO | "gpu" : " 17 GB", | 20260507 02:19:02.514 | INFO | "huge" : "", | 20260507 02:19:02.514 | INFO | "huge_shared" : " 27 GB", | 20260507 02:19:02.514 | INFO | "malloc" : " 14 GB", | 20260507 02:19:02.514 | INFO | "pinned" : " 29 GB" | 20260507 02:19:02.514 | INFO | } | 20260507 02:19:02.514 | INFO | Perf report for the last 5.13661 seconds: | 20260507 02:19:02.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:19:02.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:02.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:02.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:02.514 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:19:02.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:02.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:02.514 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:19:02.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:02.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:02.514 | INFO | Uploaded 36GiB to GPU in 1.98115s (18.1713GiB/s) | 20260507 02:19:02.514 | INFO | Downloaded 34GiB to GPU in 1.9293s (17.6229GiB/s) | 20260507 02:19:07.646 | INFO | Analyzing chunk frames = [6321152,6321664) | 20260507 02:19:07.646 | INFO | MemInfo: { | 20260507 02:19:07.646 | INFO | "gpu" : " 17 GB", | 20260507 02:19:07.646 | INFO | "huge" : "", | 20260507 02:19:07.646 | INFO | "huge_shared" : " 27 GB", | 20260507 02:19:07.646 | INFO | "malloc" : " 14 GB", | 20260507 02:19:07.646 | INFO | "pinned" : " 29 GB" | 20260507 02:19:07.646 | INFO | } | 20260507 02:19:07.646 | INFO | Perf report for the last 5.13192 seconds: | 20260507 02:19:07.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:19:07.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:07.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:07.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:07.646 | INFO | ANALYSIS : 60.3%, 3.0, 93.2ms, 0 | 20260507 02:19:07.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:07.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:07.646 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:19:07.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:07.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:07.646 | INFO | Uploaded 36GiB to GPU in 1.98627s (18.1244GiB/s) | 20260507 02:19:07.646 | INFO | Downloaded 34GiB to GPU in 1.94279s (17.5006GiB/s) | 20260507 02:19:12.790 | INFO | Analyzing chunk frames = [6321664,6322176) | 20260507 02:19:12.790 | INFO | MemInfo: { | 20260507 02:19:12.790 | INFO | "gpu" : " 17 GB", | 20260507 02:19:12.790 | INFO | "huge" : "", | 20260507 02:19:12.790 | INFO | "huge_shared" : " 27 GB", | 20260507 02:19:12.791 | INFO | "malloc" : " 14 GB", | 20260507 02:19:12.791 | INFO | "pinned" : " 29 GB" | 20260507 02:19:12.791 | INFO | } | 20260507 02:19:12.791 | INFO | Perf report for the last 5.14452 seconds: | 20260507 02:19:12.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:19:12.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:12.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:12.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:12.791 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260507 02:19:12.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:12.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:12.791 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:19:12.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:12.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:12.791 | INFO | Uploaded 36GiB to GPU in 2.00285s (17.9744GiB/s) | 20260507 02:19:12.791 | INFO | Downloaded 34GiB to GPU in 1.91069s (17.7946GiB/s) | 20260507 02:19:17.931 | INFO | Analyzing chunk frames = [6322176,6322688) | 20260507 02:19:17.931 | INFO | MemInfo: { | 20260507 02:19:17.931 | INFO | "gpu" : " 17 GB", | 20260507 02:19:17.931 | INFO | "huge" : "", | 20260507 02:19:17.931 | INFO | "huge_shared" : " 27 GB", | 20260507 02:19:17.932 | INFO | "malloc" : " 14 GB", | 20260507 02:19:17.932 | INFO | "pinned" : " 29 GB" | 20260507 02:19:17.932 | INFO | } | 20260507 02:19:17.932 | INFO | Perf report for the last 5.14106 seconds: | 20260507 02:19:17.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:19:17.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:17.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:17.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:17.932 | INFO | ANALYSIS : 59.8%, 3.0, 93.2ms, 0 | 20260507 02:19:17.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:17.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:17.932 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:19:17.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:17.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:17.932 | INFO | Uploaded 36GiB to GPU in 1.97825s (18.1979GiB/s) | 20260507 02:19:17.932 | INFO | Downloaded 34GiB to GPU in 1.94615s (17.4704GiB/s) | 20260507 02:19:23.067 | INFO | Analyzing chunk frames = [6322688,6323200) | 20260507 02:19:23.067 | INFO | MemInfo: { | 20260507 02:19:23.067 | INFO | "gpu" : " 17 GB", | 20260507 02:19:23.067 | INFO | "huge" : "", | 20260507 02:19:23.067 | INFO | "huge_shared" : " 27 GB", | 20260507 02:19:23.067 | INFO | "malloc" : " 14 GB", | 20260507 02:19:23.067 | INFO | "pinned" : " 29 GB" | 20260507 02:19:23.067 | INFO | } | 20260507 02:19:23.067 | INFO | Perf report for the last 5.13568 seconds: | 20260507 02:19:23.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:19:23.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:23.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:23.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:23.067 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:19:23.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:23.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:23.067 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 02:19:23.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:23.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:23.068 | INFO | Uploaded 36GiB to GPU in 2.04934s (17.5666GiB/s) | 20260507 02:19:23.068 | INFO | Downloaded 34GiB to GPU in 1.97174s (17.2436GiB/s) | 20260507 02:19:28.205 | INFO | Analyzing chunk frames = [6323200,6323712) | 20260507 02:19:28.205 | INFO | MemInfo: { | 20260507 02:19:28.205 | INFO | "gpu" : " 17 GB", | 20260507 02:19:28.205 | INFO | "huge" : "", | 20260507 02:19:28.205 | INFO | "huge_shared" : " 27 GB", | 20260507 02:19:28.205 | INFO | "malloc" : " 14 GB", | 20260507 02:19:28.205 | INFO | "pinned" : " 29 GB" | 20260507 02:19:28.205 | INFO | } | 20260507 02:19:28.205 | INFO | Perf report for the last 5.1375 seconds: | 20260507 02:19:28.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:19:28.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:28.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:28.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:28.205 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:19:28.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:28.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:28.205 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 02:19:28.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:28.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:28.205 | INFO | Uploaded 36GiB to GPU in 1.99663s (18.0304GiB/s) | 20260507 02:19:28.205 | INFO | Downloaded 34GiB to GPU in 1.98689s (17.1122GiB/s) | 20260507 02:19:33.346 | INFO | Analyzing chunk frames = [6323712,6324224) | 20260507 02:19:33.346 | INFO | MemInfo: { | 20260507 02:19:33.346 | INFO | "gpu" : " 17 GB", | 20260507 02:19:33.346 | INFO | "huge" : "", | 20260507 02:19:33.346 | INFO | "huge_shared" : " 27 GB", | 20260507 02:19:33.346 | INFO | "malloc" : " 14 GB", | 20260507 02:19:33.346 | INFO | "pinned" : " 29 GB" | 20260507 02:19:33.346 | INFO | } | 20260507 02:19:33.346 | INFO | Perf report for the last 5.14154 seconds: | 20260507 02:19:33.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:19:33.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:33.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:33.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:33.346 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:19:33.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:33.346 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:33.346 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:19:33.346 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:33.346 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:33.347 | INFO | Uploaded 36GiB to GPU in 1.98018s (18.1801GiB/s) | 20260507 02:19:33.347 | INFO | Downloaded 34GiB to GPU in 1.9796s (17.1752GiB/s) | 20260507 02:19:33.347 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:19:33.347 | INFO | Total mem usage: 18 GiB | 20260507 02:19:33.347 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:19:33.347 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:19:33.347 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:19:33.347 | INFO | High Water Breakdown: | 20260507 02:19:33.347 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:19:33.347 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:19:33.347 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:19:33.347 | INFO | ------------------------------------------ | 20260507 02:19:33.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:19:33.347 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:19:33.347 | INFO | Total mem usage: 26.5 GiB | 20260507 02:19:33.347 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:19:33.347 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:19:33.347 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:19:33.347 | INFO | High Water Breakdown: | 20260507 02:19:33.347 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:19:33.347 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:19:33.347 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:19:33.347 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:19:33.347 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:19:33.347 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:19:33.347 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:19:33.347 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:19:33.347 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:19:33.347 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:19:33.347 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:19:33.347 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:19:33.347 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:19:33.347 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:19:33.347 | INFO | ------------------------------------------ | 20260507 02:19:33.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:19:33.347 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:19:33.347 | INFO | Total mem usage: 25.6 GiB | 20260507 02:19:33.347 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:19:33.347 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:19:33.347 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:19:33.347 | INFO | High Water Breakdown: | 20260507 02:19:33.347 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:19:33.347 | INFO | ------------------------------------------ | 20260507 02:19:33.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:19:33.347 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:19:33.347 | INFO | Total mem usage: 15.7 GiB | 20260507 02:19:33.347 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:19:33.347 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:19:33.347 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:19:33.347 | INFO | High Water Breakdown: | 20260507 02:19:33.347 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:19:33.347 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:19:33.347 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:19:33.347 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:19:33.347 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:19:33.347 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:19:33.347 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:19:33.347 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:19:33.347 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:19:33.347 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:19:33.347 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:19:33.347 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:19:33.347 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:19:33.347 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:19:33.347 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:19:33.347 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:19:33.347 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:19:33.347 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:19:33.347 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:19:33.347 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:19:33.347 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:19:33.347 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:19:33.347 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:19:33.347 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:19:33.347 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:19:33.347 | INFO | ------------------------------------------ | 20260507 02:19:33.347 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:19:33.347 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:19:33.347 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:19:33.347 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:19:33.347 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.52e+01, 1.36e+01 / 2.00e+00 | 20260507 02:19:33.347 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 9.95e-02, 9.41e-03 / 5.55e-03 | 20260507 02:19:33.348 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.40e+00, 5.23e-01 / 3.42e-01 | 20260507 02:19:33.348 | INFO | PerfReport: 3.23%, Histogram -- 9.96e-01 / 1.44e+00, 1.70e-01 / 1.39e-01 | 20260507 02:19:33.348 | INFO | PerfReport: 16.08%, DME -- 3.97e+01 / 4.41e+01, 3.88e+00 / 2.25e+00 | 20260507 02:19:33.348 | INFO | PerfReport: 57.64%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.36e+00 / 1.52e+00 | 20260507 02:19:33.348 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.15e+00 / 2.68e+00, 3.80e-01 / 2.17e-01 | 20260507 02:19:33.348 | INFO | PerfReport: 3.07%, Metrics -- 9.48e-01 / 1.27e+00, 2.49e-01 / 1.29e-01 | 20260507 02:19:33.348 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.22e-01 / 3.73e-01, 5.09e-02 / 2.51e-02 | 20260507 02:19:33.348 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:19:33.348 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:19:33.348 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:19:33.348 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 02:19:33.348 | INFO | PerfReport: 21.63%, Upload -- 2.08e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260507 02:19:33.348 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.51e+00 / 0.00e+00 | 20260507 02:19:33.348 | INFO | PerfReport: 32.00%, Compute -- 3.08e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260507 02:19:33.348 | INFO | PerfReport: 1.31%, LockUpload -- 1.27e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 02:19:33.348 | INFO | PerfReport: 1.38%, LockDownload -- 1.33e+00 / 0.00e+00, 3.24e+00 / 0.00e+00 | 20260507 02:19:33.348 | INFO | PerfReport: 22.26%, LockCompute -- 2.14e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 02:19:33.348 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260507 02:19:33.443 | INFO | Starting a round of Baz IO | 20260507 02:19:36.560 | INFO | Populating RTMetrics report | 20260507 02:19:36.560 | INFO | RTMetrics report populated | 20260507 02:19:36.560 | INFO | Opening tmp RTMetrics json file | 20260507 02:19:36.579 | INFO | Serializing RTMetrics to json | 20260507 02:19:36.580 | INFO | Serialization done, swapping tmp file now... | 20260507 02:19:36.580 | INFO | RTMetrics done | 20260507 02:19:36.650 | INFO | Finished a round of Baz IO: Wrote 1.784 GiB in 3.206 seconds | 20260507 02:19:36.650 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.02% / 0.00% / 66.98% | 20260507 02:19:36.650 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.241176% / 3.97364e-06% / 8.74201e-05% | 20260507 02:19:36.650 | INFO | Baz IO overall: | 20260507 02:19:36.650 | INFO | BytesWritten : 3266155870256 | 20260507 02:19:36.650 | INFO | NumEvents : 1570925884470 | 20260507 02:19:38.487 | INFO | Analyzing chunk frames = [6324224,6324736) | 20260507 02:19:38.487 | INFO | MemInfo: { | 20260507 02:19:38.487 | INFO | "gpu" : " 17 GB", | 20260507 02:19:38.487 | INFO | "huge" : "", | 20260507 02:19:38.487 | INFO | "huge_shared" : " 27 GB", | 20260507 02:19:38.487 | INFO | "malloc" : " 14 GB", | 20260507 02:19:38.487 | INFO | "pinned" : " 29 GB" | 20260507 02:19:38.487 | INFO | } | 20260507 02:19:38.487 | INFO | Perf report for the last 5.14064 seconds: | 20260507 02:19:38.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:19:38.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:38.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:38.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:38.487 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 02:19:38.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:38.487 | INFO | RT_METRICS : 42.0%, 1.0, 22.1ms, 0 | 20260507 02:19:38.487 | INFO | PRE_HQ : 59.5%, 1.8, 55.7ms, 0 | 20260507 02:19:38.487 | INFO | BAZWRITER : 62.0%, 3.3, 103.8ms, 0 | 20260507 02:19:38.487 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:19:38.487 | INFO | Uploaded 36GiB to GPU in 2.02878s (17.7447GiB/s) | 20260507 02:19:38.487 | INFO | Downloaded 38GiB to GPU in 2.1773s (17.4528GiB/s) | 20260507 02:19:43.624 | INFO | Analyzing chunk frames = [6324736,6325248) | 20260507 02:19:43.624 | INFO | MemInfo: { | 20260507 02:19:43.624 | INFO | "gpu" : " 17 GB", | 20260507 02:19:43.624 | INFO | "huge" : "", | 20260507 02:19:43.624 | INFO | "huge_shared" : " 27 GB", | 20260507 02:19:43.624 | INFO | "malloc" : " 14 GB", | 20260507 02:19:43.624 | INFO | "pinned" : " 29 GB" | 20260507 02:19:43.624 | INFO | } | 20260507 02:19:43.624 | INFO | Perf report for the last 5.13664 seconds: | 20260507 02:19:43.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:19:43.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:43.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:43.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:43.624 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:19:43.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:43.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:43.624 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:19:43.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:43.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:43.624 | INFO | Uploaded 36GiB to GPU in 2.02157s (17.8079GiB/s) | 20260507 02:19:43.624 | INFO | Downloaded 34GiB to GPU in 1.93836s (17.5406GiB/s) | 20260507 02:19:48.762 | INFO | Analyzing chunk frames = [6325248,6325760) | 20260507 02:19:48.762 | INFO | MemInfo: { | 20260507 02:19:48.762 | INFO | "gpu" : " 17 GB", | 20260507 02:19:48.762 | INFO | "huge" : "", | 20260507 02:19:48.762 | INFO | "huge_shared" : " 27 GB", | 20260507 02:19:48.762 | INFO | "malloc" : " 14 GB", | 20260507 02:19:48.762 | INFO | "pinned" : " 29 GB" | 20260507 02:19:48.762 | INFO | } | 20260507 02:19:48.762 | INFO | Perf report for the last 5.13828 seconds: | 20260507 02:19:48.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:19:48.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:48.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:48.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:48.762 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 02:19:48.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:48.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:48.762 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:19:48.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:48.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:48.762 | INFO | Uploaded 36GiB to GPU in 1.99065s (18.0846GiB/s) | 20260507 02:19:48.762 | INFO | Downloaded 34GiB to GPU in 1.94384s (17.4911GiB/s) | 20260507 02:19:53.902 | INFO | Analyzing chunk frames = [6325760,6326272) | 20260507 02:19:53.902 | INFO | MemInfo: { | 20260507 02:19:53.902 | INFO | "gpu" : " 17 GB", | 20260507 02:19:53.902 | INFO | "huge" : "", | 20260507 02:19:53.902 | INFO | "huge_shared" : " 27 GB", | 20260507 02:19:53.902 | INFO | "malloc" : " 14 GB", | 20260507 02:19:53.902 | INFO | "pinned" : " 29 GB" | 20260507 02:19:53.902 | INFO | } | 20260507 02:19:53.902 | INFO | Perf report for the last 5.14042 seconds: | 20260507 02:19:53.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:19:53.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:53.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:53.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:53.902 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 02:19:53.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:53.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:53.902 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:19:53.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:53.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:53.903 | INFO | Uploaded 36GiB to GPU in 1.94167s (18.5407GiB/s) | 20260507 02:19:53.903 | INFO | Downloaded 34GiB to GPU in 1.90346s (17.8622GiB/s) | 20260507 02:19:59.042 | INFO | Analyzing chunk frames = [6326272,6326784) | 20260507 02:19:59.042 | INFO | MemInfo: { | 20260507 02:19:59.042 | INFO | "gpu" : " 17 GB", | 20260507 02:19:59.042 | INFO | "huge" : "", | 20260507 02:19:59.042 | INFO | "huge_shared" : " 27 GB", | 20260507 02:19:59.042 | INFO | "malloc" : " 14 GB", | 20260507 02:19:59.042 | INFO | "pinned" : " 29 GB" | 20260507 02:19:59.042 | INFO | } | 20260507 02:19:59.042 | INFO | Perf report for the last 5.13943 seconds: | 20260507 02:19:59.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:19:59.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:59.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:59.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:59.042 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 02:19:59.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:59.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:59.042 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:19:59.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:59.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:19:59.042 | INFO | Uploaded 36GiB to GPU in 1.99876s (18.0112GiB/s) | 20260507 02:19:59.042 | INFO | Downloaded 34GiB to GPU in 1.94879s (17.4467GiB/s) | 20260507 02:20:04.192 | INFO | Analyzing chunk frames = [6326784,6327296) | 20260507 02:20:04.192 | INFO | MemInfo: { | 20260507 02:20:04.192 | INFO | "gpu" : " 17 GB", | 20260507 02:20:04.192 | INFO | "huge" : "", | 20260507 02:20:04.192 | INFO | "huge_shared" : " 27 GB", | 20260507 02:20:04.192 | INFO | "malloc" : " 14 GB", | 20260507 02:20:04.192 | INFO | "pinned" : " 29 GB" | 20260507 02:20:04.192 | INFO | } | 20260507 02:20:04.192 | INFO | Perf report for the last 5.15021 seconds: | 20260507 02:20:04.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:20:04.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:04.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:04.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:04.192 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 02:20:04.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:04.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:04.192 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 02:20:04.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:04.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:04.192 | INFO | Uploaded 36GiB to GPU in 2.03857s (17.6594GiB/s) | 20260507 02:20:04.192 | INFO | Downloaded 34GiB to GPU in 1.96011s (17.346GiB/s) | 20260507 02:20:09.326 | INFO | Analyzing chunk frames = [6327296,6327808) | 20260507 02:20:09.326 | INFO | MemInfo: { | 20260507 02:20:09.326 | INFO | "gpu" : " 17 GB", | 20260507 02:20:09.326 | INFO | "huge" : "", | 20260507 02:20:09.326 | INFO | "huge_shared" : " 27 GB", | 20260507 02:20:09.326 | INFO | "malloc" : " 14 GB", | 20260507 02:20:09.326 | INFO | "pinned" : " 29 GB" | 20260507 02:20:09.326 | INFO | } | 20260507 02:20:09.326 | INFO | Perf report for the last 5.13359 seconds: | 20260507 02:20:09.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:20:09.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:09.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:09.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:09.326 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:20:09.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:09.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:09.326 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:20:09.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:09.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:09.326 | INFO | Uploaded 36GiB to GPU in 1.96835s (18.2895GiB/s) | 20260507 02:20:09.326 | INFO | Downloaded 34GiB to GPU in 1.97104s (17.2497GiB/s) | 20260507 02:20:14.464 | INFO | Analyzing chunk frames = [6327808,6328320) | 20260507 02:20:14.464 | INFO | MemInfo: { | 20260507 02:20:14.464 | INFO | "gpu" : " 17 GB", | 20260507 02:20:14.464 | INFO | "huge" : "", | 20260507 02:20:14.464 | INFO | "huge_shared" : " 27 GB", | 20260507 02:20:14.464 | INFO | "malloc" : " 14 GB", | 20260507 02:20:14.464 | INFO | "pinned" : " 29 GB" | 20260507 02:20:14.464 | INFO | } | 20260507 02:20:14.464 | INFO | Perf report for the last 5.13835 seconds: | 20260507 02:20:14.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:20:14.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:14.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:14.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:14.464 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:20:14.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:14.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:14.464 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:20:14.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:14.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:14.464 | INFO | Uploaded 36GiB to GPU in 1.99507s (18.0445GiB/s) | 20260507 02:20:14.464 | INFO | Downloaded 34GiB to GPU in 1.96935s (17.2646GiB/s) | 20260507 02:20:14.561 | INFO | Starting a round of Baz IO | 20260507 02:20:17.674 | INFO | Populating RTMetrics report | 20260507 02:20:17.674 | INFO | RTMetrics report populated | 20260507 02:20:17.674 | INFO | Opening tmp RTMetrics json file | 20260507 02:20:17.676 | INFO | Serializing RTMetrics to json | 20260507 02:20:17.693 | INFO | Serialization done, swapping tmp file now... | 20260507 02:20:17.694 | INFO | RTMetrics done | 20260507 02:20:17.764 | INFO | Finished a round of Baz IO: Wrote 1.783 GiB in 3.203 seconds | 20260507 02:20:17.764 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.02% / 0.00% / 66.98% | 20260507 02:20:17.764 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.241689% / 0% / 7.94729e-05% | 20260507 02:20:17.765 | INFO | Baz IO overall: | 20260507 02:20:17.765 | INFO | BytesWritten : 3268070570242 | 20260507 02:20:17.765 | INFO | NumEvents : 1571831234782 | 20260507 02:20:19.604 | INFO | Analyzing chunk frames = [6328320,6328832) | 20260507 02:20:19.604 | INFO | MemInfo: { | 20260507 02:20:19.604 | INFO | "gpu" : " 17 GB", | 20260507 02:20:19.604 | INFO | "huge" : "", | 20260507 02:20:19.604 | INFO | "huge_shared" : " 27 GB", | 20260507 02:20:19.604 | INFO | "malloc" : " 14 GB", | 20260507 02:20:19.604 | INFO | "pinned" : " 29 GB" | 20260507 02:20:19.604 | INFO | } | 20260507 02:20:19.604 | INFO | Perf report for the last 5.13962 seconds: | 20260507 02:20:19.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:20:19.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:19.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:19.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:19.604 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260507 02:20:19.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:19.604 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 02:20:19.604 | INFO | PRE_HQ : 59.2%, 1.8, 55.6ms, 0 | 20260507 02:20:19.604 | INFO | BAZWRITER : 61.9%, 3.3, 104.3ms, 0 | 20260507 02:20:19.604 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:20:19.604 | INFO | Uploaded 36GiB to GPU in 2.05351s (17.5309GiB/s) | 20260507 02:20:19.604 | INFO | Downloaded 38GiB to GPU in 2.18347s (17.4035GiB/s) | 20260507 02:20:24.740 | INFO | Analyzing chunk frames = [6328832,6329344) | 20260507 02:20:24.740 | INFO | MemInfo: { | 20260507 02:20:24.740 | INFO | "gpu" : " 17 GB", | 20260507 02:20:24.740 | INFO | "huge" : "", | 20260507 02:20:24.740 | INFO | "huge_shared" : " 27 GB", | 20260507 02:20:24.740 | INFO | "malloc" : " 14 GB", | 20260507 02:20:24.740 | INFO | "pinned" : " 29 GB" | 20260507 02:20:24.740 | INFO | } | 20260507 02:20:24.740 | INFO | Perf report for the last 5.13672 seconds: | 20260507 02:20:24.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:20:24.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:24.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:24.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:24.740 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 02:20:24.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:24.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:24.740 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:20:24.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:24.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:24.741 | INFO | Uploaded 36GiB to GPU in 2.03102s (17.7251GiB/s) | 20260507 02:20:24.741 | INFO | Downloaded 34GiB to GPU in 1.93736s (17.5497GiB/s) | 20260507 02:20:29.881 | INFO | Analyzing chunk frames = [6329344,6329856) | 20260507 02:20:29.881 | INFO | MemInfo: { | 20260507 02:20:29.881 | INFO | "gpu" : " 17 GB", | 20260507 02:20:29.881 | INFO | "huge" : "", | 20260507 02:20:29.881 | INFO | "huge_shared" : " 27 GB", | 20260507 02:20:29.881 | INFO | "malloc" : " 14 GB", | 20260507 02:20:29.881 | INFO | "pinned" : " 29 GB" | 20260507 02:20:29.881 | INFO | } | 20260507 02:20:29.881 | INFO | Perf report for the last 5.14089 seconds: | 20260507 02:20:29.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:20:29.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:29.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:29.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:29.881 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:20:29.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:29.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:29.881 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 02:20:29.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:29.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:29.881 | INFO | Uploaded 36GiB to GPU in 1.99965s (18.0032GiB/s) | 20260507 02:20:29.882 | INFO | Downloaded 34GiB to GPU in 1.93498s (17.5712GiB/s) | 20260507 02:20:35.017 | INFO | Analyzing chunk frames = [6329856,6330368) | 20260507 02:20:35.017 | INFO | MemInfo: { | 20260507 02:20:35.017 | INFO | "gpu" : " 17 GB", | 20260507 02:20:35.017 | INFO | "huge" : "", | 20260507 02:20:35.017 | INFO | "huge_shared" : " 27 GB", | 20260507 02:20:35.017 | INFO | "malloc" : " 14 GB", | 20260507 02:20:35.017 | INFO | "pinned" : " 29 GB" | 20260507 02:20:35.017 | INFO | } | 20260507 02:20:35.017 | INFO | Perf report for the last 5.13579 seconds: | 20260507 02:20:35.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:20:35.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:35.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:35.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:35.017 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:20:35.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:35.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:35.017 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:20:35.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:35.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:35.017 | INFO | Uploaded 36GiB to GPU in 1.96338s (18.3357GiB/s) | 20260507 02:20:35.017 | INFO | Downloaded 34GiB to GPU in 1.89877s (17.9063GiB/s) | 20260507 02:20:40.158 | INFO | Analyzing chunk frames = [6330368,6330880) | 20260507 02:20:40.158 | INFO | MemInfo: { | 20260507 02:20:40.158 | INFO | "gpu" : " 17 GB", | 20260507 02:20:40.158 | INFO | "huge" : "", | 20260507 02:20:40.158 | INFO | "huge_shared" : " 27 GB", | 20260507 02:20:40.158 | INFO | "malloc" : " 14 GB", | 20260507 02:20:40.158 | INFO | "pinned" : " 29 GB" | 20260507 02:20:40.158 | INFO | } | 20260507 02:20:40.158 | INFO | Perf report for the last 5.14088 seconds: | 20260507 02:20:40.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:20:40.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:40.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:40.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:40.158 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 02:20:40.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:40.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:40.158 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:20:40.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:40.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:40.159 | INFO | Uploaded 36GiB to GPU in 2.01651s (17.8526GiB/s) | 20260507 02:20:40.159 | INFO | Downloaded 34GiB to GPU in 1.93828s (17.5413GiB/s) | 20260507 02:20:45.299 | INFO | Analyzing chunk frames = [6330880,6331392) | 20260507 02:20:45.299 | INFO | MemInfo: { | 20260507 02:20:45.299 | INFO | "gpu" : " 17 GB", | 20260507 02:20:45.299 | INFO | "huge" : "", | 20260507 02:20:45.300 | INFO | "huge_shared" : " 27 GB", | 20260507 02:20:45.300 | INFO | "malloc" : " 14 GB", | 20260507 02:20:45.300 | INFO | "pinned" : " 29 GB" | 20260507 02:20:45.300 | INFO | } | 20260507 02:20:45.300 | INFO | Perf report for the last 5.14158 seconds: | 20260507 02:20:45.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:20:45.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:45.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:45.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:45.300 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 02:20:45.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:45.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:45.300 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:20:45.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:45.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:45.300 | INFO | Uploaded 36GiB to GPU in 2.02277s (17.7973GiB/s) | 20260507 02:20:45.300 | INFO | Downloaded 34GiB to GPU in 1.93441s (17.5764GiB/s) | 20260507 02:20:50.440 | INFO | Analyzing chunk frames = [6331392,6331904) | 20260507 02:20:50.440 | INFO | MemInfo: { | 20260507 02:20:50.440 | INFO | "gpu" : " 17 GB", | 20260507 02:20:50.440 | INFO | "huge" : "", | 20260507 02:20:50.440 | INFO | "huge_shared" : " 27 GB", | 20260507 02:20:50.440 | INFO | "malloc" : " 14 GB", | 20260507 02:20:50.440 | INFO | "pinned" : " 29 GB" | 20260507 02:20:50.440 | INFO | } | 20260507 02:20:50.440 | INFO | Perf report for the last 5.14012 seconds: | 20260507 02:20:50.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:20:50.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:50.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:50.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:50.440 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:20:50.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:50.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:50.440 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 02:20:50.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:50.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:50.440 | INFO | Uploaded 36GiB to GPU in 1.98804s (18.1083GiB/s) | 20260507 02:20:50.440 | INFO | Downloaded 34GiB to GPU in 1.98218s (17.1529GiB/s) | 20260507 02:20:55.577 | INFO | Analyzing chunk frames = [6331904,6332416) | 20260507 02:20:55.577 | INFO | MemInfo: { | 20260507 02:20:55.577 | INFO | "gpu" : " 17 GB", | 20260507 02:20:55.577 | INFO | "huge" : "", | 20260507 02:20:55.577 | INFO | "huge_shared" : " 27 GB", | 20260507 02:20:55.577 | INFO | "malloc" : " 14 GB", | 20260507 02:20:55.577 | INFO | "pinned" : " 29 GB" | 20260507 02:20:55.577 | INFO | } | 20260507 02:20:55.577 | INFO | Perf report for the last 5.13752 seconds: | 20260507 02:20:55.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:20:55.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:55.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:55.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:55.577 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 02:20:55.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:55.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:55.577 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:20:55.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:55.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:20:55.578 | INFO | Uploaded 36GiB to GPU in 1.99791s (18.0188GiB/s) | 20260507 02:20:55.578 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:20:55.578 | INFO | Downloaded 34GiB to GPU in 1.97383s (17.2254GiB/s) | 20260507 02:20:55.578 | INFO | Total mem usage: 18 GiB | 20260507 02:20:55.578 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:20:55.578 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 02:20:55.578 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:20:55.578 | INFO | High Water Breakdown: | 20260507 02:20:55.578 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:20:55.578 | INFO | 39.8% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:20:55.578 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:20:55.578 | INFO | ------------------------------------------ | 20260507 02:20:55.578 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:20:55.578 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:20:55.578 | INFO | Total mem usage: 26.5 GiB | 20260507 02:20:55.578 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:20:55.578 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:20:55.578 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:20:55.578 | INFO | High Water Breakdown: | 20260507 02:20:55.578 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:20:55.578 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:20:55.578 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:20:55.578 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:20:55.578 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:20:55.578 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:20:55.578 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:20:55.578 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:20:55.578 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:20:55.578 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:20:55.578 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:20:55.578 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:20:55.578 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:20:55.578 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:20:55.578 | INFO | ------------------------------------------ | 20260507 02:20:55.578 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:20:55.578 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:20:55.578 | INFO | Total mem usage: 25.6 GiB | 20260507 02:20:55.578 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:20:55.578 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:20:55.578 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:20:55.578 | INFO | High Water Breakdown: | 20260507 02:20:55.578 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:20:55.578 | INFO | ------------------------------------------ | 20260507 02:20:55.578 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:20:55.578 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:20:55.578 | INFO | Total mem usage: 15.7 GiB | 20260507 02:20:55.578 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:20:55.578 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:20:55.578 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:20:55.578 | INFO | High Water Breakdown: | 20260507 02:20:55.578 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:20:55.578 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:20:55.578 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:20:55.578 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:20:55.578 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:20:55.578 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:20:55.578 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:20:55.578 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:20:55.578 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:20:55.578 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:20:55.578 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:20:55.578 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:20:55.578 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:20:55.578 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:20:55.578 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:20:55.578 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:20:55.578 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:20:55.578 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:20:55.578 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:20:55.578 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:20:55.578 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:20:55.578 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:20:55.578 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:20:55.578 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:20:55.578 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:20:55.578 | INFO | ------------------------------------------ | 20260507 02:20:55.578 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:20:55.578 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:20:55.578 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:20:55.578 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:20:55.578 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.53e+01, 1.32e+01 / 1.96e+00 | 20260507 02:20:55.578 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 9.99e-02, 9.39e-03 / 3.91e-03 | 20260507 02:20:55.578 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.48e+00, 5.23e-01 / 2.24e-01 | 20260507 02:20:55.579 | INFO | PerfReport: 3.23%, Histogram -- 9.94e-01 / 1.48e+00, 1.70e-01 / 1.13e-01 | 20260507 02:20:55.579 | INFO | PerfReport: 16.12%, DME -- 3.97e+01 / 4.44e+01, 3.92e+00 / 2.32e+00 | 20260507 02:20:55.579 | INFO | PerfReport: 57.56%, FrameLabeling -- 1.77e+01 / 2.05e+01, 1.31e+00 / 1.85e+00 | 20260507 02:20:55.579 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.15e+00 / 2.68e+00, 3.83e-01 / 2.18e-01 | 20260507 02:20:55.579 | INFO | PerfReport: 3.08%, Metrics -- 9.50e-01 / 1.28e+00, 2.53e-01 / 2.01e-01 | 20260507 02:20:55.579 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.21e-01 / 3.71e-01, 5.05e-02 / 2.14e-02 | 20260507 02:20:55.579 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:20:55.579 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:20:55.579 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:20:55.579 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 02:20:55.579 | INFO | PerfReport: 21.66%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260507 02:20:55.579 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260507 02:20:55.579 | INFO | PerfReport: 31.97%, Compute -- 3.08e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 02:20:55.579 | INFO | PerfReport: 1.37%, LockUpload -- 1.32e+00 / 0.00e+00, 4.86e+00 / 0.00e+00 | 20260507 02:20:55.579 | INFO | PerfReport: 1.33%, LockDownload -- 1.28e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260507 02:20:55.579 | INFO | PerfReport: 22.30%, LockCompute -- 2.15e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 02:20:55.579 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260507 02:20:55.674 | INFO | Starting a round of Baz IO | 20260507 02:20:58.786 | INFO | Populating RTMetrics report | 20260507 02:20:58.786 | INFO | RTMetrics report populated | 20260507 02:20:58.786 | INFO | Opening tmp RTMetrics json file | 20260507 02:20:58.804 | INFO | Serializing RTMetrics to json | 20260507 02:20:58.805 | INFO | Serialization done, swapping tmp file now... | 20260507 02:20:58.806 | INFO | RTMetrics done | 20260507 02:20:58.881 | INFO | Finished a round of Baz IO: Wrote 1.786 GiB in 3.207 seconds | 20260507 02:20:58.881 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.02% / 0.00% / 66.98% | 20260507 02:20:58.881 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.242496% / 0% / 9.53674e-05% | 20260507 02:20:58.882 | INFO | Baz IO overall: | 20260507 02:20:58.882 | INFO | BytesWritten : 3269988213335 | 20260507 02:20:58.882 | INFO | NumEvents : 1572738867265 | 20260507 02:21:00.712 | INFO | Analyzing chunk frames = [6332416,6332928) | 20260507 02:21:00.712 | INFO | MemInfo: { | 20260507 02:21:00.712 | INFO | "gpu" : " 17 GB", | 20260507 02:21:00.712 | INFO | "huge" : "", | 20260507 02:21:00.712 | INFO | "huge_shared" : " 27 GB", | 20260507 02:21:00.712 | INFO | "malloc" : " 14 GB", | 20260507 02:21:00.712 | INFO | "pinned" : " 29 GB" | 20260507 02:21:00.712 | INFO | } | 20260507 02:21:00.712 | INFO | Perf report for the last 5.13517 seconds: | 20260507 02:21:00.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:21:00.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:00.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:00.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:00.712 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260507 02:21:00.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:00.713 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 02:21:00.713 | INFO | PRE_HQ : 59.6%, 1.8, 55.8ms, 0 | 20260507 02:21:00.713 | INFO | BAZWRITER : 62.1%, 3.3, 104.0ms, 0 | 20260507 02:21:00.713 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:21:00.713 | INFO | Uploaded 36GiB to GPU in 2.03145s (17.7213GiB/s) | 20260507 02:21:00.713 | INFO | Downloaded 38GiB to GPU in 2.18452s (17.3952GiB/s) | 20260507 02:21:05.854 | INFO | Analyzing chunk frames = [6332928,6333440) | 20260507 02:21:05.854 | INFO | MemInfo: { | 20260507 02:21:05.854 | INFO | "gpu" : " 17 GB", | 20260507 02:21:05.854 | INFO | "huge" : "", | 20260507 02:21:05.854 | INFO | "huge_shared" : " 27 GB", | 20260507 02:21:05.854 | INFO | "malloc" : " 14 GB", | 20260507 02:21:05.854 | INFO | "pinned" : " 29 GB" | 20260507 02:21:05.854 | INFO | } | 20260507 02:21:05.854 | INFO | Perf report for the last 5.14161 seconds: | 20260507 02:21:05.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:21:05.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:05.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:05.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:05.854 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:21:05.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:05.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:05.854 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:21:05.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:05.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:05.854 | INFO | Uploaded 36GiB to GPU in 1.99435s (18.051GiB/s) | 20260507 02:21:05.854 | INFO | Downloaded 34GiB to GPU in 1.93161s (17.6019GiB/s) | 20260507 02:21:10.997 | INFO | Analyzing chunk frames = [6333440,6333952) | 20260507 02:21:10.997 | INFO | MemInfo: { | 20260507 02:21:10.997 | INFO | "gpu" : " 17 GB", | 20260507 02:21:10.997 | INFO | "huge" : "", | 20260507 02:21:10.997 | INFO | "huge_shared" : " 27 GB", | 20260507 02:21:10.997 | INFO | "malloc" : " 14 GB", | 20260507 02:21:10.997 | INFO | "pinned" : " 29 GB" | 20260507 02:21:10.997 | INFO | } | 20260507 02:21:10.997 | INFO | Perf report for the last 5.14336 seconds: | 20260507 02:21:10.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:21:10.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:10.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:10.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:10.997 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:21:10.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:10.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:10.997 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:21:10.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:10.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:10.998 | INFO | Uploaded 36GiB to GPU in 1.98415s (18.1437GiB/s) | 20260507 02:21:10.998 | INFO | Downloaded 34GiB to GPU in 1.93668s (17.5558GiB/s) | 20260507 02:21:16.143 | INFO | Analyzing chunk frames = [6333952,6334464) | 20260507 02:21:16.143 | INFO | MemInfo: { | 20260507 02:21:16.143 | INFO | "gpu" : " 17 GB", | 20260507 02:21:16.143 | INFO | "huge" : "", | 20260507 02:21:16.143 | INFO | "huge_shared" : " 27 GB", | 20260507 02:21:16.143 | INFO | "malloc" : " 14 GB", | 20260507 02:21:16.143 | INFO | "pinned" : " 29 GB" | 20260507 02:21:16.143 | INFO | } | 20260507 02:21:16.143 | INFO | Perf report for the last 5.14531 seconds: | 20260507 02:21:16.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:21:16.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:16.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:16.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:16.143 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 02:21:16.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:16.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:16.143 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:21:16.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:16.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:16.143 | INFO | Uploaded 36GiB to GPU in 1.97456s (18.2319GiB/s) | 20260507 02:21:16.143 | INFO | Downloaded 34GiB to GPU in 1.90905s (17.8099GiB/s) | 20260507 02:21:21.280 | INFO | Analyzing chunk frames = [6334464,6334976) | 20260507 02:21:21.280 | INFO | MemInfo: { | 20260507 02:21:21.280 | INFO | "gpu" : " 17 GB", | 20260507 02:21:21.280 | INFO | "huge" : "", | 20260507 02:21:21.280 | INFO | "huge_shared" : " 27 GB", | 20260507 02:21:21.280 | INFO | "malloc" : " 14 GB", | 20260507 02:21:21.280 | INFO | "pinned" : " 29 GB" | 20260507 02:21:21.280 | INFO | } | 20260507 02:21:21.280 | INFO | Perf report for the last 5.13748 seconds: | 20260507 02:21:21.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:21:21.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:21.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:21.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:21.280 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 02:21:21.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:21.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:21.280 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 02:21:21.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:21.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:21.280 | INFO | Uploaded 36GiB to GPU in 2.04903s (17.5693GiB/s) | 20260507 02:21:21.281 | INFO | Downloaded 34GiB to GPU in 1.95334s (17.4061GiB/s) | 20260507 02:21:26.407 | INFO | Analyzing chunk frames = [6334976,6335488) | 20260507 02:21:26.407 | INFO | MemInfo: { | 20260507 02:21:26.407 | INFO | "gpu" : " 17 GB", | 20260507 02:21:26.407 | INFO | "huge" : "", | 20260507 02:21:26.408 | INFO | "huge_shared" : " 27 GB", | 20260507 02:21:26.408 | INFO | "malloc" : " 14 GB", | 20260507 02:21:26.408 | INFO | "pinned" : " 29 GB" | 20260507 02:21:26.408 | INFO | } | 20260507 02:21:26.408 | INFO | Perf report for the last 5.12736 seconds: | 20260507 02:21:26.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:21:26.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:26.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:26.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:26.408 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 02:21:26.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:26.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:26.408 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 02:21:26.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:26.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:26.408 | INFO | Uploaded 36GiB to GPU in 2.01567s (17.8601GiB/s) | 20260507 02:21:26.408 | INFO | Downloaded 34GiB to GPU in 1.93585s (17.5633GiB/s) | 20260507 02:21:31.553 | INFO | Analyzing chunk frames = [6335488,6336000) | 20260507 02:21:31.553 | INFO | MemInfo: { | 20260507 02:21:31.553 | INFO | "gpu" : " 17 GB", | 20260507 02:21:31.553 | INFO | "huge" : "", | 20260507 02:21:31.553 | INFO | "huge_shared" : " 27 GB", | 20260507 02:21:31.553 | INFO | "malloc" : " 14 GB", | 20260507 02:21:31.553 | INFO | "pinned" : " 29 GB" | 20260507 02:21:31.553 | INFO | } | 20260507 02:21:31.553 | INFO | Perf report for the last 5.14508 seconds: | 20260507 02:21:31.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:21:31.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:31.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:31.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:31.553 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:21:31.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:31.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:31.553 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 02:21:31.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:31.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:31.553 | INFO | Uploaded 36GiB to GPU in 1.96957s (18.2781GiB/s) | 20260507 02:21:31.553 | INFO | Downloaded 34GiB to GPU in 2.00104s (16.9911GiB/s) | 20260507 02:21:36.694 | INFO | Analyzing chunk frames = [6336000,6336512) | 20260507 02:21:36.694 | INFO | MemInfo: { | 20260507 02:21:36.694 | INFO | "gpu" : " 17 GB", | 20260507 02:21:36.694 | INFO | "huge" : "", | 20260507 02:21:36.694 | INFO | "huge_shared" : " 27 GB", | 20260507 02:21:36.694 | INFO | "malloc" : " 14 GB", | 20260507 02:21:36.694 | INFO | "pinned" : " 29 GB" | 20260507 02:21:36.694 | INFO | } | 20260507 02:21:36.694 | INFO | Perf report for the last 5.14162 seconds: | 20260507 02:21:36.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:21:36.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:36.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:36.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:36.694 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:21:36.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:36.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:36.694 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:21:36.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:36.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:36.695 | INFO | Uploaded 36GiB to GPU in 2.00654s (17.9414GiB/s) | 20260507 02:21:36.695 | INFO | Downloaded 34GiB to GPU in 1.96728s (17.2828GiB/s) | 20260507 02:21:36.791 | INFO | Starting a round of Baz IO | 20260507 02:21:39.902 | INFO | Populating RTMetrics report | 20260507 02:21:39.902 | INFO | RTMetrics report populated | 20260507 02:21:39.902 | INFO | Opening tmp RTMetrics json file | 20260507 02:21:39.904 | INFO | Serializing RTMetrics to json | 20260507 02:21:39.921 | INFO | Serialization done, swapping tmp file now... | 20260507 02:21:39.922 | INFO | RTMetrics done | 20260507 02:21:39.993 | INFO | Finished a round of Baz IO: Wrote 1.785 GiB in 3.201 seconds | 20260507 02:21:39.993 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.02% / 0.00% / 66.98% | 20260507 02:21:39.993 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.241236% / 3.97364e-06% / 5.96046e-05% | 20260507 02:21:39.993 | INFO | Baz IO overall: | 20260507 02:21:39.993 | INFO | BytesWritten : 3271904523190 | 20260507 02:21:39.993 | INFO | NumEvents : 1573645576730 | 20260507 02:21:41.831 | INFO | Analyzing chunk frames = [6336512,6337024) | 20260507 02:21:41.832 | INFO | MemInfo: { | 20260507 02:21:41.832 | INFO | "gpu" : " 17 GB", | 20260507 02:21:41.832 | INFO | "huge" : "", | 20260507 02:21:41.832 | INFO | "huge_shared" : " 27 GB", | 20260507 02:21:41.832 | INFO | "malloc" : " 14 GB", | 20260507 02:21:41.832 | INFO | "pinned" : " 29 GB" | 20260507 02:21:41.832 | INFO | } | 20260507 02:21:41.832 | INFO | Perf report for the last 5.13725 seconds: | 20260507 02:21:41.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:21:41.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:41.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:41.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:41.832 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 02:21:41.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:41.832 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 02:21:41.832 | INFO | PRE_HQ : 59.5%, 1.8, 56.2ms, 0 | 20260507 02:21:41.832 | INFO | BAZWRITER : 61.9%, 3.3, 105.3ms, 0 | 20260507 02:21:41.832 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:21:41.832 | INFO | Uploaded 36GiB to GPU in 2.04154s (17.6338GiB/s) | 20260507 02:21:41.832 | INFO | Downloaded 38GiB to GPU in 2.18186s (17.4163GiB/s) | 20260507 02:21:46.970 | INFO | Analyzing chunk frames = [6337024,6337536) | 20260507 02:21:46.970 | INFO | MemInfo: { | 20260507 02:21:46.970 | INFO | "gpu" : " 17 GB", | 20260507 02:21:46.970 | INFO | "huge" : "", | 20260507 02:21:46.970 | INFO | "huge_shared" : " 27 GB", | 20260507 02:21:46.970 | INFO | "malloc" : " 14 GB", | 20260507 02:21:46.970 | INFO | "pinned" : " 29 GB" | 20260507 02:21:46.970 | INFO | } | 20260507 02:21:46.970 | INFO | Perf report for the last 5.13852 seconds: | 20260507 02:21:46.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:21:46.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:46.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:46.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:46.970 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:21:46.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:46.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:46.970 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:21:46.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:46.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:46.970 | INFO | Uploaded 36GiB to GPU in 2.04041s (17.6435GiB/s) | 20260507 02:21:46.970 | INFO | Downloaded 34GiB to GPU in 1.93665s (17.5561GiB/s) | 20260507 02:21:52.108 | INFO | Analyzing chunk frames = [6337536,6338048) | 20260507 02:21:52.108 | INFO | MemInfo: { | 20260507 02:21:52.108 | INFO | "gpu" : " 17 GB", | 20260507 02:21:52.108 | INFO | "huge" : "", | 20260507 02:21:52.108 | INFO | "huge_shared" : " 27 GB", | 20260507 02:21:52.108 | INFO | "malloc" : " 14 GB", | 20260507 02:21:52.108 | INFO | "pinned" : " 29 GB" | 20260507 02:21:52.108 | INFO | } | 20260507 02:21:52.108 | INFO | Perf report for the last 5.13759 seconds: | 20260507 02:21:52.108 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:21:52.108 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:52.108 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:52.108 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:52.108 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 02:21:52.108 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:52.108 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:52.108 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:21:52.108 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:52.108 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:52.108 | INFO | Uploaded 36GiB to GPU in 1.99912s (18.0079GiB/s) | 20260507 02:21:52.108 | INFO | Downloaded 34GiB to GPU in 1.93604s (17.5616GiB/s) | 20260507 02:21:57.250 | INFO | Analyzing chunk frames = [6338048,6338560) | 20260507 02:21:57.250 | INFO | MemInfo: { | 20260507 02:21:57.250 | INFO | "gpu" : " 17 GB", | 20260507 02:21:57.250 | INFO | "huge" : "", | 20260507 02:21:57.250 | INFO | "huge_shared" : " 27 GB", | 20260507 02:21:57.250 | INFO | "malloc" : " 14 GB", | 20260507 02:21:57.250 | INFO | "pinned" : " 29 GB" | 20260507 02:21:57.250 | INFO | } | 20260507 02:21:57.250 | INFO | Perf report for the last 5.14197 seconds: | 20260507 02:21:57.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:21:57.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:57.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:57.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:57.250 | INFO | ANALYSIS : 59.8%, 3.0, 92.1ms, 0 | 20260507 02:21:57.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:57.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:57.250 | INFO | PRE_HQ : 25.5%, 1.0, 13.1ms, 0 | 20260507 02:21:57.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:57.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:21:57.250 | INFO | Uploaded 36GiB to GPU in 1.98146s (18.1684GiB/s) | 20260507 02:21:57.250 | INFO | Downloaded 34GiB to GPU in 1.90045s (17.8905GiB/s) | 20260507 02:22:02.391 | INFO | Analyzing chunk frames = [6338560,6339072) | 20260507 02:22:02.391 | INFO | MemInfo: { | 20260507 02:22:02.391 | INFO | "gpu" : " 17 GB", | 20260507 02:22:02.391 | INFO | "huge" : "", | 20260507 02:22:02.391 | INFO | "huge_shared" : " 27 GB", | 20260507 02:22:02.391 | INFO | "malloc" : " 14 GB", | 20260507 02:22:02.391 | INFO | "pinned" : " 29 GB" | 20260507 02:22:02.391 | INFO | } | 20260507 02:22:02.391 | INFO | Perf report for the last 5.14169 seconds: | 20260507 02:22:02.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:22:02.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:02.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:02.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:02.391 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 02:22:02.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:02.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:02.392 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:22:02.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:02.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:02.392 | INFO | Uploaded 36GiB to GPU in 2.0287s (17.7453GiB/s) | 20260507 02:22:02.392 | INFO | Downloaded 34GiB to GPU in 1.92923s (17.6236GiB/s) | 20260507 02:22:07.524 | INFO | Analyzing chunk frames = [6339072,6339584) | 20260507 02:22:07.524 | INFO | MemInfo: { | 20260507 02:22:07.524 | INFO | "gpu" : " 17 GB", | 20260507 02:22:07.524 | INFO | "huge" : "", | 20260507 02:22:07.524 | INFO | "huge_shared" : " 27 GB", | 20260507 02:22:07.524 | INFO | "malloc" : " 14 GB", | 20260507 02:22:07.524 | INFO | "pinned" : " 29 GB" | 20260507 02:22:07.524 | INFO | } | 20260507 02:22:07.524 | INFO | Perf report for the last 5.13238 seconds: | 20260507 02:22:07.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:22:07.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:07.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:07.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:07.524 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 02:22:07.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:07.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:07.524 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 02:22:07.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:07.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:07.525 | INFO | Uploaded 36GiB to GPU in 2.02099s (17.8131GiB/s) | 20260507 02:22:07.525 | INFO | Downloaded 34GiB to GPU in 1.91852s (17.722GiB/s) | 20260507 02:22:12.662 | INFO | Analyzing chunk frames = [6339584,6340096) | 20260507 02:22:12.662 | INFO | MemInfo: { | 20260507 02:22:12.662 | INFO | "gpu" : " 17 GB", | 20260507 02:22:12.662 | INFO | "huge" : "", | 20260507 02:22:12.662 | INFO | "huge_shared" : " 27 GB", | 20260507 02:22:12.662 | INFO | "malloc" : " 14 GB", | 20260507 02:22:12.662 | INFO | "pinned" : " 29 GB" | 20260507 02:22:12.662 | INFO | } | 20260507 02:22:12.662 | INFO | Perf report for the last 5.13817 seconds: | 20260507 02:22:12.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:22:12.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:12.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:12.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:12.662 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 02:22:12.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:12.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:12.662 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:22:12.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:12.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:12.662 | INFO | Uploaded 36GiB to GPU in 1.97578s (18.2206GiB/s) | 20260507 02:22:12.662 | INFO | Downloaded 34GiB to GPU in 1.98823s (17.1006GiB/s) | 20260507 02:22:17.804 | INFO | Analyzing chunk frames = [6340096,6340608) | 20260507 02:22:17.804 | INFO | MemInfo: { | 20260507 02:22:17.804 | INFO | "gpu" : " 17 GB", | 20260507 02:22:17.804 | INFO | "huge" : "", | 20260507 02:22:17.804 | INFO | "huge_shared" : " 27 GB", | 20260507 02:22:17.804 | INFO | "malloc" : " 14 GB", | 20260507 02:22:17.804 | INFO | "pinned" : " 29 GB" | 20260507 02:22:17.804 | INFO | } | 20260507 02:22:17.804 | INFO | Perf report for the last 5.14188 seconds: | 20260507 02:22:17.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:22:17.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:17.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:17.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:17.804 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:22:17.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:17.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:17.804 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:22:17.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:17.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:17.804 | INFO | Uploaded 36GiB to GPU in 1.97206s (18.255GiB/s) | 20260507 02:22:17.804 | INFO | Downloaded 34GiB to GPU in 1.97796s (17.1895GiB/s) | 20260507 02:22:17.804 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:22:17.804 | INFO | Total mem usage: 18 GiB | 20260507 02:22:17.804 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:22:17.804 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:22:17.804 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:22:17.804 | INFO | High Water Breakdown: | 20260507 02:22:17.804 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:22:17.804 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:22:17.804 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:22:17.804 | INFO | ------------------------------------------ | 20260507 02:22:17.804 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:22:17.804 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:22:17.804 | INFO | Total mem usage: 26.5 GiB | 20260507 02:22:17.804 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:22:17.804 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:22:17.804 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:22:17.804 | INFO | High Water Breakdown: | 20260507 02:22:17.804 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:22:17.804 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:22:17.804 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:22:17.804 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:22:17.804 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:22:17.804 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:22:17.804 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:22:17.804 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:22:17.804 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:22:17.804 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:22:17.804 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:22:17.805 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:22:17.805 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:22:17.805 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:22:17.805 | INFO | ------------------------------------------ | 20260507 02:22:17.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:22:17.805 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:22:17.805 | INFO | Total mem usage: 25.6 GiB | 20260507 02:22:17.805 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:22:17.805 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:22:17.805 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:22:17.805 | INFO | High Water Breakdown: | 20260507 02:22:17.805 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:22:17.805 | INFO | ------------------------------------------ | 20260507 02:22:17.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:22:17.805 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:22:17.805 | INFO | Total mem usage: 15.7 GiB | 20260507 02:22:17.805 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:22:17.805 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:22:17.805 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:22:17.805 | INFO | High Water Breakdown: | 20260507 02:22:17.805 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:22:17.805 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:22:17.805 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:22:17.805 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:22:17.805 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:22:17.805 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:22:17.805 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:22:17.805 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:22:17.805 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:22:17.805 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:22:17.805 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:22:17.805 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:22:17.805 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:22:17.805 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:22:17.805 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:22:17.805 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:22:17.805 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:22:17.805 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:22:17.805 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:22:17.805 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:22:17.805 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:22:17.805 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:22:17.805 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:22:17.805 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:22:17.805 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:22:17.805 | INFO | ------------------------------------------ | 20260507 02:22:17.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:22:17.805 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:22:17.805 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:22:17.805 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:22:17.805 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.56e+01, 1.35e+01 / 2.26e+00 | 20260507 02:22:17.805 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 1.05e-01, 9.93e-03 / 1.40e-02 | 20260507 02:22:17.805 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.44e+00, 5.25e-01 / 2.37e-01 | 20260507 02:22:17.805 | INFO | PerfReport: 3.24%, Histogram -- 9.98e-01 / 1.47e+00, 1.73e-01 / 1.23e-01 | 20260507 02:22:17.805 | INFO | PerfReport: 16.12%, DME -- 3.98e+01 / 4.48e+01, 4.06e+00 / 2.43e+00 | 20260507 02:22:17.805 | INFO | PerfReport: 57.55%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.29e+00 / 1.55e+00 | 20260507 02:22:17.805 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.16e+00 / 2.73e+00, 3.84e-01 / 2.04e-01 | 20260507 02:22:17.805 | INFO | PerfReport: 3.08%, Metrics -- 9.49e-01 / 1.27e+00, 2.52e-01 / 1.13e-01 | 20260507 02:22:17.805 | INFO | PerfReport: 1.05%, PulseSerializing -- 3.23e-01 / 3.77e-01, 5.21e-02 / 2.39e-02 | 20260507 02:22:17.805 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:22:17.805 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:22:17.805 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:22:17.805 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 02:22:17.805 | INFO | PerfReport: 21.66%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260507 02:22:17.805 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260507 02:22:17.805 | INFO | PerfReport: 31.97%, Compute -- 3.08e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 02:22:17.805 | INFO | PerfReport: 1.43%, LockUpload -- 1.38e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 02:22:17.805 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.14e+00 / 0.00e+00 | 20260507 02:22:17.805 | INFO | PerfReport: 22.28%, LockCompute -- 2.15e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 02:22:17.805 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260507 02:22:17.901 | INFO | Starting a round of Baz IO | 20260507 02:22:21.017 | INFO | Populating RTMetrics report | 20260507 02:22:21.017 | INFO | RTMetrics report populated | 20260507 02:22:21.017 | INFO | Opening tmp RTMetrics json file | 20260507 02:22:21.037 | INFO | Serializing RTMetrics to json | 20260507 02:22:21.038 | INFO | Serialization done, swapping tmp file now... | 20260507 02:22:21.039 | INFO | RTMetrics done | 20260507 02:22:21.111 | INFO | Finished a round of Baz IO: Wrote 1.784 GiB in 3.209 seconds | 20260507 02:22:21.111 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.01% / 0.00% / 66.98% | 20260507 02:22:21.111 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.239964% / 3.97364e-06% / 7.94729e-05% | 20260507 02:22:21.111 | INFO | Baz IO overall: | 20260507 02:22:21.111 | INFO | BytesWritten : 3273820086498 | 20260507 02:22:21.111 | INFO | NumEvents : 1574551718261 | 20260507 02:22:22.958 | INFO | Analyzing chunk frames = [6340608,6341120) | 20260507 02:22:22.958 | INFO | MemInfo: { | 20260507 02:22:22.958 | INFO | "gpu" : " 17 GB", | 20260507 02:22:22.958 | INFO | "huge" : "", | 20260507 02:22:22.958 | INFO | "huge_shared" : " 27 GB", | 20260507 02:22:22.958 | INFO | "malloc" : " 14 GB", | 20260507 02:22:22.958 | INFO | "pinned" : " 29 GB" | 20260507 02:22:22.958 | INFO | } | 20260507 02:22:22.958 | INFO | Perf report for the last 5.15423 seconds: | 20260507 02:22:22.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:22:22.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:22.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:22.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:22.958 | INFO | ANALYSIS : 61.8%, 3.0, 96.4ms, 0 | 20260507 02:22:22.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:22.958 | INFO | RT_METRICS : 42.0%, 1.0, 22.1ms, 0 | 20260507 02:22:22.958 | INFO | PRE_HQ : 59.1%, 1.8, 55.6ms, 0 | 20260507 02:22:22.958 | INFO | BAZWRITER : 61.9%, 3.2, 103.6ms, 0 | 20260507 02:22:22.958 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:22:22.958 | INFO | Uploaded 36GiB to GPU in 2.02885s (17.7441GiB/s) | 20260507 02:22:22.958 | INFO | Downloaded 38GiB to GPU in 2.17811s (17.4463GiB/s) | 20260507 02:22:28.089 | INFO | Analyzing chunk frames = [6341120,6341632) | 20260507 02:22:28.089 | INFO | MemInfo: { | 20260507 02:22:28.089 | INFO | "gpu" : " 17 GB", | 20260507 02:22:28.089 | INFO | "huge" : "", | 20260507 02:22:28.089 | INFO | "huge_shared" : " 27 GB", | 20260507 02:22:28.089 | INFO | "malloc" : " 14 GB", | 20260507 02:22:28.089 | INFO | "pinned" : " 29 GB" | 20260507 02:22:28.089 | INFO | } | 20260507 02:22:28.089 | INFO | Perf report for the last 5.13114 seconds: | 20260507 02:22:28.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:22:28.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:28.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:28.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:28.089 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 02:22:28.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:28.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:28.089 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 02:22:28.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:28.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:28.090 | INFO | Uploaded 36GiB to GPU in 1.99921s (18.0071GiB/s) | 20260507 02:22:28.090 | INFO | Downloaded 34GiB to GPU in 1.93362s (17.5836GiB/s) | 20260507 02:22:33.225 | INFO | Analyzing chunk frames = [6341632,6342144) | 20260507 02:22:33.225 | INFO | MemInfo: { | 20260507 02:22:33.225 | INFO | "gpu" : " 17 GB", | 20260507 02:22:33.225 | INFO | "huge" : "", | 20260507 02:22:33.225 | INFO | "huge_shared" : " 27 GB", | 20260507 02:22:33.225 | INFO | "malloc" : " 14 GB", | 20260507 02:22:33.225 | INFO | "pinned" : " 29 GB" | 20260507 02:22:33.225 | INFO | } | 20260507 02:22:33.225 | INFO | Perf report for the last 5.13576 seconds: | 20260507 02:22:33.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:22:33.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:33.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:33.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:33.225 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 02:22:33.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:33.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:33.225 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:22:33.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:33.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:33.225 | INFO | Uploaded 36GiB to GPU in 1.97967s (18.1848GiB/s) | 20260507 02:22:33.225 | INFO | Downloaded 34GiB to GPU in 1.94261s (17.5022GiB/s) | 20260507 02:22:38.364 | INFO | Analyzing chunk frames = [6342144,6342656) | 20260507 02:22:38.364 | INFO | MemInfo: { | 20260507 02:22:38.364 | INFO | "gpu" : " 17 GB", | 20260507 02:22:38.364 | INFO | "huge" : "", | 20260507 02:22:38.364 | INFO | "huge_shared" : " 27 GB", | 20260507 02:22:38.364 | INFO | "malloc" : " 14 GB", | 20260507 02:22:38.364 | INFO | "pinned" : " 29 GB" | 20260507 02:22:38.364 | INFO | } | 20260507 02:22:38.364 | INFO | Perf report for the last 5.1387 seconds: | 20260507 02:22:38.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:22:38.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:38.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:38.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:38.364 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 02:22:38.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:38.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:38.364 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:22:38.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:38.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:38.364 | INFO | Uploaded 36GiB to GPU in 1.95398s (18.4239GiB/s) | 20260507 02:22:38.364 | INFO | Downloaded 34GiB to GPU in 1.91086s (17.793GiB/s) | 20260507 02:22:43.502 | INFO | Analyzing chunk frames = [6342656,6343168) | 20260507 02:22:43.502 | INFO | MemInfo: { | 20260507 02:22:43.502 | INFO | "gpu" : " 17 GB", | 20260507 02:22:43.502 | INFO | "huge" : "", | 20260507 02:22:43.502 | INFO | "huge_shared" : " 27 GB", | 20260507 02:22:43.502 | INFO | "malloc" : " 14 GB", | 20260507 02:22:43.502 | INFO | "pinned" : " 29 GB" | 20260507 02:22:43.502 | INFO | } | 20260507 02:22:43.502 | INFO | Perf report for the last 5.13819 seconds: | 20260507 02:22:43.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:22:43.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:43.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:43.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:43.502 | INFO | ANALYSIS : 59.9%, 3.0, 93.2ms, 0 | 20260507 02:22:43.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:43.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:43.502 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:22:43.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:43.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:43.502 | INFO | Uploaded 36GiB to GPU in 1.98832s (18.1057GiB/s) | 20260507 02:22:43.502 | INFO | Downloaded 34GiB to GPU in 1.94816s (17.4523GiB/s) | 20260507 02:22:48.641 | INFO | Analyzing chunk frames = [6343168,6343680) | 20260507 02:22:48.641 | INFO | MemInfo: { | 20260507 02:22:48.641 | INFO | "gpu" : " 17 GB", | 20260507 02:22:48.641 | INFO | "huge" : "", | 20260507 02:22:48.641 | INFO | "huge_shared" : " 27 GB", | 20260507 02:22:48.641 | INFO | "malloc" : " 14 GB", | 20260507 02:22:48.641 | INFO | "pinned" : " 29 GB" | 20260507 02:22:48.641 | INFO | } | 20260507 02:22:48.641 | INFO | Perf report for the last 5.13898 seconds: | 20260507 02:22:48.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:22:48.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:48.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:48.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:48.641 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:22:48.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:48.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:48.641 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 02:22:48.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:48.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:48.641 | INFO | Uploaded 36GiB to GPU in 2.0518s (17.5455GiB/s) | 20260507 02:22:48.641 | INFO | Downloaded 34GiB to GPU in 1.96931s (17.2649GiB/s) | 20260507 02:22:53.782 | INFO | Analyzing chunk frames = [6343680,6344192) | 20260507 02:22:53.782 | INFO | MemInfo: { | 20260507 02:22:53.782 | INFO | "gpu" : " 17 GB", | 20260507 02:22:53.782 | INFO | "huge" : "", | 20260507 02:22:53.782 | INFO | "huge_shared" : " 27 GB", | 20260507 02:22:53.782 | INFO | "malloc" : " 14 GB", | 20260507 02:22:53.782 | INFO | "pinned" : " 29 GB" | 20260507 02:22:53.782 | INFO | } | 20260507 02:22:53.782 | INFO | Perf report for the last 5.14132 seconds: | 20260507 02:22:53.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:22:53.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:53.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:53.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:53.782 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:22:53.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:53.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:53.782 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 02:22:53.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:53.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:53.783 | INFO | Uploaded 36GiB to GPU in 1.97935s (18.1878GiB/s) | 20260507 02:22:53.783 | INFO | Downloaded 34GiB to GPU in 1.98819s (17.101GiB/s) | 20260507 02:22:58.919 | INFO | Analyzing chunk frames = [6344192,6344704) | 20260507 02:22:58.919 | INFO | MemInfo: { | 20260507 02:22:58.919 | INFO | "gpu" : " 17 GB", | 20260507 02:22:58.919 | INFO | "huge" : "", | 20260507 02:22:58.919 | INFO | "huge_shared" : " 27 GB", | 20260507 02:22:58.919 | INFO | "malloc" : " 14 GB", | 20260507 02:22:58.919 | INFO | "pinned" : " 29 GB" | 20260507 02:22:58.919 | INFO | } | 20260507 02:22:58.919 | INFO | Perf report for the last 5.13677 seconds: | 20260507 02:22:58.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:22:58.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:58.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:58.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:58.919 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:22:58.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:58.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:58.919 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 02:22:58.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:58.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:22:58.919 | INFO | Uploaded 36GiB to GPU in 2.002s (17.982GiB/s) | 20260507 02:22:58.919 | INFO | Downloaded 34GiB to GPU in 1.96827s (17.2741GiB/s) | 20260507 02:22:59.016 | INFO | Starting a round of Baz IO | 20260507 02:23:02.132 | INFO | Populating RTMetrics report | 20260507 02:23:02.133 | INFO | RTMetrics report populated | 20260507 02:23:02.133 | INFO | Opening tmp RTMetrics json file | 20260507 02:23:02.136 | INFO | Serializing RTMetrics to json | 20260507 02:23:02.151 | INFO | Serialization done, swapping tmp file now... | 20260507 02:23:02.152 | INFO | RTMetrics done | 20260507 02:23:02.223 | INFO | Finished a round of Baz IO: Wrote 1.783 GiB in 3.206 seconds | 20260507 02:23:02.223 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.01% / 0.00% / 66.99% | 20260507 02:23:02.223 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.239805% / 0% / 4.76837e-05% | 20260507 02:23:02.223 | INFO | Baz IO overall: | 20260507 02:23:02.223 | INFO | BytesWritten : 3275734972334 | 20260507 02:23:02.223 | INFO | NumEvents : 1575457565171 | 20260507 02:23:04.064 | INFO | Analyzing chunk frames = [6344704,6345216) | 20260507 02:23:04.064 | INFO | MemInfo: { | 20260507 02:23:04.064 | INFO | "gpu" : " 17 GB", | 20260507 02:23:04.064 | INFO | "huge" : "", | 20260507 02:23:04.064 | INFO | "huge_shared" : " 27 GB", | 20260507 02:23:04.064 | INFO | "malloc" : " 14 GB", | 20260507 02:23:04.064 | INFO | "pinned" : " 29 GB" | 20260507 02:23:04.064 | INFO | } | 20260507 02:23:04.064 | INFO | Perf report for the last 5.14454 seconds: | 20260507 02:23:04.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:23:04.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:04.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:04.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:04.064 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260507 02:23:04.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:04.064 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 02:23:04.064 | INFO | PRE_HQ : 59.5%, 1.8, 55.8ms, 0 | 20260507 02:23:04.064 | INFO | BAZWRITER : 61.9%, 3.3, 104.7ms, 0 | 20260507 02:23:04.064 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 02:23:04.064 | INFO | Uploaded 36GiB to GPU in 2.03398s (17.6992GiB/s) | 20260507 02:23:04.064 | INFO | Downloaded 38GiB to GPU in 2.18013s (17.4302GiB/s) | 20260507 02:23:09.203 | INFO | Analyzing chunk frames = [6345216,6345728) | 20260507 02:23:09.203 | INFO | MemInfo: { | 20260507 02:23:09.203 | INFO | "gpu" : " 17 GB", | 20260507 02:23:09.203 | INFO | "huge" : "", | 20260507 02:23:09.203 | INFO | "huge_shared" : " 27 GB", | 20260507 02:23:09.203 | INFO | "malloc" : " 14 GB", | 20260507 02:23:09.203 | INFO | "pinned" : " 29 GB" | 20260507 02:23:09.203 | INFO | } | 20260507 02:23:09.203 | INFO | Perf report for the last 5.13945 seconds: | 20260507 02:23:09.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:23:09.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:09.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:09.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:09.203 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:23:09.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:09.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:09.203 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:23:09.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:09.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:09.203 | INFO | Uploaded 36GiB to GPU in 2.02535s (17.7747GiB/s) | 20260507 02:23:09.203 | INFO | Downloaded 34GiB to GPU in 1.93236s (17.595GiB/s) | 20260507 02:23:14.340 | INFO | Analyzing chunk frames = [6345728,6346240) | 20260507 02:23:14.340 | INFO | MemInfo: { | 20260507 02:23:14.340 | INFO | "gpu" : " 17 GB", | 20260507 02:23:14.340 | INFO | "huge" : "", | 20260507 02:23:14.340 | INFO | "huge_shared" : " 27 GB", | 20260507 02:23:14.340 | INFO | "malloc" : " 14 GB", | 20260507 02:23:14.340 | INFO | "pinned" : " 29 GB" | 20260507 02:23:14.340 | INFO | } | 20260507 02:23:14.340 | INFO | Perf report for the last 5.13687 seconds: | 20260507 02:23:14.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:23:14.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:14.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:14.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:14.340 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 02:23:14.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:14.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:14.340 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:23:14.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:14.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:14.340 | INFO | Uploaded 36GiB to GPU in 2.00546s (17.951GiB/s) | 20260507 02:23:14.340 | INFO | Downloaded 34GiB to GPU in 1.93485s (17.5724GiB/s) | 20260507 02:23:19.472 | INFO | Analyzing chunk frames = [6346240,6346752) | 20260507 02:23:19.472 | INFO | MemInfo: { | 20260507 02:23:19.472 | INFO | "gpu" : " 17 GB", | 20260507 02:23:19.472 | INFO | "huge" : "", | 20260507 02:23:19.472 | INFO | "huge_shared" : " 27 GB", | 20260507 02:23:19.472 | INFO | "malloc" : " 14 GB", | 20260507 02:23:19.472 | INFO | "pinned" : " 29 GB" | 20260507 02:23:19.472 | INFO | } | 20260507 02:23:19.472 | INFO | Perf report for the last 5.13168 seconds: | 20260507 02:23:19.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:23:19.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:19.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:19.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:19.472 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260507 02:23:19.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:19.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:19.472 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:23:19.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:19.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:19.472 | INFO | Uploaded 36GiB to GPU in 1.98239s (18.1599GiB/s) | 20260507 02:23:19.472 | INFO | Downloaded 34GiB to GPU in 1.91974s (17.7108GiB/s) | 20260507 02:23:24.618 | INFO | Analyzing chunk frames = [6346752,6347264) | 20260507 02:23:24.618 | INFO | MemInfo: { | 20260507 02:23:24.618 | INFO | "gpu" : " 17 GB", | 20260507 02:23:24.618 | INFO | "huge" : "", | 20260507 02:23:24.618 | INFO | "huge_shared" : " 27 GB", | 20260507 02:23:24.618 | INFO | "malloc" : " 14 GB", | 20260507 02:23:24.618 | INFO | "pinned" : " 29 GB" | 20260507 02:23:24.618 | INFO | } | 20260507 02:23:24.618 | INFO | Perf report for the last 5.14607 seconds: | 20260507 02:23:24.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:23:24.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:24.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:24.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:24.618 | INFO | ANALYSIS : 59.8%, 3.0, 93.2ms, 0 | 20260507 02:23:24.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:24.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:24.618 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:23:24.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:24.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:24.618 | INFO | Uploaded 36GiB to GPU in 1.98618s (18.1253GiB/s) | 20260507 02:23:24.618 | INFO | Downloaded 34GiB to GPU in 1.93276s (17.5914GiB/s) | 20260507 02:23:29.759 | INFO | Analyzing chunk frames = [6347264,6347776) | 20260507 02:23:29.759 | INFO | MemInfo: { | 20260507 02:23:29.759 | INFO | "gpu" : " 17 GB", | 20260507 02:23:29.759 | INFO | "huge" : "", | 20260507 02:23:29.759 | INFO | "huge_shared" : " 27 GB", | 20260507 02:23:29.759 | INFO | "malloc" : " 14 GB", | 20260507 02:23:29.759 | INFO | "pinned" : " 29 GB" | 20260507 02:23:29.759 | INFO | } | 20260507 02:23:29.759 | INFO | Perf report for the last 5.14109 seconds: | 20260507 02:23:29.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:23:29.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:29.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:29.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:29.759 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 02:23:29.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:29.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:29.759 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:23:29.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:29.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:29.759 | INFO | Uploaded 36GiB to GPU in 2.04797s (17.5783GiB/s) | 20260507 02:23:29.759 | INFO | Downloaded 34GiB to GPU in 1.94649s (17.4674GiB/s) | 20260507 02:23:34.903 | INFO | Analyzing chunk frames = [6347776,6348288) | 20260507 02:23:34.903 | INFO | MemInfo: { | 20260507 02:23:34.903 | INFO | "gpu" : " 17 GB", | 20260507 02:23:34.903 | INFO | "huge" : "", | 20260507 02:23:34.903 | INFO | "huge_shared" : " 27 GB", | 20260507 02:23:34.903 | INFO | "malloc" : " 14 GB", | 20260507 02:23:34.903 | INFO | "pinned" : " 29 GB" | 20260507 02:23:34.903 | INFO | } | 20260507 02:23:34.904 | INFO | Perf report for the last 5.1446 seconds: | 20260507 02:23:34.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:23:34.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:34.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:34.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:34.904 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:23:34.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:34.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:34.904 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 02:23:34.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:34.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:34.904 | INFO | Uploaded 36GiB to GPU in 1.97577s (18.2208GiB/s) | 20260507 02:23:34.904 | INFO | Downloaded 34GiB to GPU in 1.97405s (17.2234GiB/s) | 20260507 02:23:40.042 | INFO | Analyzing chunk frames = [6348288,6348800) | 20260507 02:23:40.042 | INFO | MemInfo: { | 20260507 02:23:40.042 | INFO | "gpu" : " 17 GB", | 20260507 02:23:40.042 | INFO | "huge" : "", | 20260507 02:23:40.042 | INFO | "huge_shared" : " 27 GB", | 20260507 02:23:40.042 | INFO | "malloc" : " 14 GB", | 20260507 02:23:40.042 | INFO | "pinned" : " 29 GB" | 20260507 02:23:40.042 | INFO | } | 20260507 02:23:40.042 | INFO | Perf report for the last 5.13839 seconds: | 20260507 02:23:40.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:23:40.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:40.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:40.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:40.042 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:23:40.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:40.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:40.042 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 02:23:40.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:40.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:40.042 | INFO | Uploaded 36GiB to GPU in 1.98923s (18.0974GiB/s) | 20260507 02:23:40.042 | INFO | Downloaded 34GiB to GPU in 1.97325s (17.2305GiB/s) | 20260507 02:23:40.042 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:23:40.042 | INFO | Total mem usage: 18 GiB | 20260507 02:23:40.042 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:23:40.042 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:23:40.042 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:23:40.042 | INFO | High Water Breakdown: | 20260507 02:23:40.042 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:23:40.042 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:23:40.043 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:23:40.043 | INFO | ------------------------------------------ | 20260507 02:23:40.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:23:40.043 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:23:40.043 | INFO | Total mem usage: 26.5 GiB | 20260507 02:23:40.043 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:23:40.043 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:23:40.043 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:23:40.043 | INFO | High Water Breakdown: | 20260507 02:23:40.043 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:23:40.043 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:23:40.043 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:23:40.043 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:23:40.043 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:23:40.043 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:23:40.043 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:23:40.043 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:23:40.043 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:23:40.043 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:23:40.043 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:23:40.043 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:23:40.043 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:23:40.043 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:23:40.043 | INFO | ------------------------------------------ | 20260507 02:23:40.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:23:40.043 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:23:40.043 | INFO | Total mem usage: 25.6 GiB | 20260507 02:23:40.043 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:23:40.043 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:23:40.043 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:23:40.043 | INFO | High Water Breakdown: | 20260507 02:23:40.043 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:23:40.043 | INFO | ------------------------------------------ | 20260507 02:23:40.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:23:40.043 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:23:40.043 | INFO | Total mem usage: 15.7 GiB | 20260507 02:23:40.043 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:23:40.043 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:23:40.043 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 02:23:40.043 | INFO | High Water Breakdown: | 20260507 02:23:40.043 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:23:40.043 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:23:40.043 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:23:40.043 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:23:40.043 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:23:40.043 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:23:40.043 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:23:40.043 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:23:40.043 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:23:40.043 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:23:40.043 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:23:40.043 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:23:40.043 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:23:40.043 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:23:40.043 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:23:40.043 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:23:40.043 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:23:40.043 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:23:40.043 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:23:40.043 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:23:40.043 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:23:40.043 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:23:40.043 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:23:40.043 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:23:40.043 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:23:40.043 | INFO | ------------------------------------------ | 20260507 02:23:40.043 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:23:40.043 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:23:40.043 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:23:40.043 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:23:40.043 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.54e+01, 1.35e+01 / 1.83e+00 | 20260507 02:23:40.043 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 1.00e-01, 9.68e-03 / 5.73e-03 | 20260507 02:23:40.043 | INFO | PerfReport: 11.66%, Baseline -- 3.60e+00 / 4.46e+00, 5.30e-01 / 3.14e-01 | 20260507 02:23:40.043 | INFO | PerfReport: 3.23%, Histogram -- 9.97e-01 / 1.46e+00, 1.71e-01 / 1.18e-01 | 20260507 02:23:40.043 | INFO | PerfReport: 16.06%, DME -- 3.96e+01 / 4.41e+01, 3.95e+00 / 2.10e+00 | 20260507 02:23:40.043 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.78e+01 / 2.09e+01, 1.37e+00 / 1.94e+00 | 20260507 02:23:40.043 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.15e+00 / 2.70e+00, 3.79e-01 / 2.46e-01 | 20260507 02:23:40.043 | INFO | PerfReport: 3.08%, Metrics -- 9.49e-01 / 1.28e+00, 2.55e-01 / 1.64e-01 | 20260507 02:23:40.043 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.21e-01 / 3.73e-01, 5.12e-02 / 2.42e-02 | 20260507 02:23:40.043 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:23:40.043 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:23:40.043 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:23:40.043 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 02:23:40.043 | INFO | PerfReport: 21.65%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260507 02:23:40.043 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260507 02:23:40.043 | INFO | PerfReport: 32.02%, Compute -- 3.08e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 02:23:40.043 | INFO | PerfReport: 1.36%, LockUpload -- 1.31e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 02:23:40.043 | INFO | PerfReport: 1.29%, LockDownload -- 1.25e+00 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260507 02:23:40.043 | INFO | PerfReport: 22.29%, LockCompute -- 2.15e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 02:23:40.043 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260507 02:23:40.139 | INFO | Starting a round of Baz IO | 20260507 02:23:43.242 | INFO | Populating RTMetrics report | 20260507 02:23:43.242 | INFO | RTMetrics report populated | 20260507 02:23:43.242 | INFO | Opening tmp RTMetrics json file | 20260507 02:23:43.262 | INFO | Serializing RTMetrics to json | 20260507 02:23:43.263 | INFO | Serialization done, swapping tmp file now... | 20260507 02:23:43.263 | INFO | RTMetrics done | 20260507 02:23:43.335 | INFO | Finished a round of Baz IO: Wrote 1.784 GiB in 3.195 seconds | 20260507 02:23:43.335 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.01% / 0.00% / 66.99% | 20260507 02:23:43.335 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.239913% / 0% / 3.97364e-05% | 20260507 02:23:43.335 | INFO | Baz IO overall: | 20260507 02:23:43.335 | INFO | BytesWritten : 3277650449967 | 20260507 02:23:43.335 | INFO | NumEvents : 1576363613070 | 20260507 02:23:45.175 | INFO | Analyzing chunk frames = [6348800,6349312) | 20260507 02:23:45.175 | INFO | MemInfo: { | 20260507 02:23:45.175 | INFO | "gpu" : " 17 GB", | 20260507 02:23:45.175 | INFO | "huge" : "", | 20260507 02:23:45.175 | INFO | "huge_shared" : " 27 GB", | 20260507 02:23:45.175 | INFO | "malloc" : " 14 GB", | 20260507 02:23:45.175 | INFO | "pinned" : " 29 GB" | 20260507 02:23:45.175 | INFO | } | 20260507 02:23:45.176 | INFO | Perf report for the last 5.13361 seconds: | 20260507 02:23:45.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:23:45.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:45.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:45.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:45.176 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260507 02:23:45.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:45.176 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 02:23:45.176 | INFO | PRE_HQ : 59.3%, 1.8, 55.9ms, 0 | 20260507 02:23:45.176 | INFO | BAZWRITER : 61.8%, 3.3, 104.6ms, 0 | 20260507 02:23:45.176 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:23:45.176 | INFO | Uploaded 36GiB to GPU in 2.02243s (17.8004GiB/s) | 20260507 02:23:45.176 | INFO | Downloaded 38GiB to GPU in 2.18088s (17.4242GiB/s) | 20260507 02:23:50.321 | INFO | Analyzing chunk frames = [6349312,6349824) | 20260507 02:23:50.321 | INFO | MemInfo: { | 20260507 02:23:50.321 | INFO | "gpu" : " 17 GB", | 20260507 02:23:50.321 | INFO | "huge" : "", | 20260507 02:23:50.321 | INFO | "huge_shared" : " 27 GB", | 20260507 02:23:50.321 | INFO | "malloc" : " 14 GB", | 20260507 02:23:50.321 | INFO | "pinned" : " 29 GB" | 20260507 02:23:50.321 | INFO | } | 20260507 02:23:50.321 | INFO | Perf report for the last 5.14585 seconds: | 20260507 02:23:50.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:23:50.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:50.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:50.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:50.321 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 02:23:50.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:50.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:50.321 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:23:50.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:50.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:50.322 | INFO | Uploaded 36GiB to GPU in 1.99082s (18.083GiB/s) | 20260507 02:23:50.322 | INFO | Downloaded 34GiB to GPU in 1.93316s (17.5877GiB/s) | 20260507 02:23:55.452 | INFO | Analyzing chunk frames = [6349824,6350336) | 20260507 02:23:55.452 | INFO | MemInfo: { | 20260507 02:23:55.452 | INFO | "gpu" : " 17 GB", | 20260507 02:23:55.452 | INFO | "huge" : "", | 20260507 02:23:55.452 | INFO | "huge_shared" : " 27 GB", | 20260507 02:23:55.452 | INFO | "malloc" : " 14 GB", | 20260507 02:23:55.453 | INFO | "pinned" : " 29 GB" | 20260507 02:23:55.453 | INFO | } | 20260507 02:23:55.453 | INFO | Perf report for the last 5.13118 seconds: | 20260507 02:23:55.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:23:55.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:55.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:55.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:55.453 | INFO | ANALYSIS : 60.3%, 3.0, 93.2ms, 0 | 20260507 02:23:55.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:55.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:55.453 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:23:55.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:55.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:23:55.453 | INFO | Uploaded 36GiB to GPU in 2.00464s (17.9583GiB/s) | 20260507 02:23:55.453 | INFO | Downloaded 34GiB to GPU in 1.94027s (17.5233GiB/s) | 20260507 02:24:00.591 | INFO | Analyzing chunk frames = [6350336,6350848) | 20260507 02:24:00.591 | INFO | MemInfo: { | 20260507 02:24:00.591 | INFO | "gpu" : " 17 GB", | 20260507 02:24:00.591 | INFO | "huge" : "", | 20260507 02:24:00.592 | INFO | "huge_shared" : " 27 GB", | 20260507 02:24:00.592 | INFO | "malloc" : " 14 GB", | 20260507 02:24:00.592 | INFO | "pinned" : " 29 GB" | 20260507 02:24:00.592 | INFO | } | 20260507 02:24:00.592 | INFO | Perf report for the last 5.139 seconds: | 20260507 02:24:00.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:24:00.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:00.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:00.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:00.592 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 02:24:00.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:00.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:00.592 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:24:00.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:00.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:00.592 | INFO | Uploaded 36GiB to GPU in 1.95974s (18.3698GiB/s) | 20260507 02:24:00.592 | INFO | Downloaded 34GiB to GPU in 1.90191s (17.8768GiB/s) | 20260507 02:24:05.743 | INFO | Analyzing chunk frames = [6350848,6351360) | 20260507 02:24:05.743 | INFO | MemInfo: { | 20260507 02:24:05.743 | INFO | "gpu" : " 17 GB", | 20260507 02:24:05.743 | INFO | "huge" : "", | 20260507 02:24:05.743 | INFO | "huge_shared" : " 27 GB", | 20260507 02:24:05.743 | INFO | "malloc" : " 14 GB", | 20260507 02:24:05.743 | INFO | "pinned" : " 29 GB" | 20260507 02:24:05.743 | INFO | } | 20260507 02:24:05.743 | INFO | Perf report for the last 5.15178 seconds: | 20260507 02:24:05.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:24:05.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:05.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:05.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:05.743 | INFO | ANALYSIS : 60.0%, 3.0, 93.9ms, 0 | 20260507 02:24:05.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:05.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:05.743 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 02:24:05.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:05.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:05.744 | INFO | Uploaded 36GiB to GPU in 2.01266s (17.8868GiB/s) | 20260507 02:24:05.744 | INFO | Downloaded 34GiB to GPU in 1.94471s (17.4833GiB/s) | 20260507 02:24:10.873 | INFO | Analyzing chunk frames = [6351360,6351872) | 20260507 02:24:10.873 | INFO | MemInfo: { | 20260507 02:24:10.873 | INFO | "gpu" : " 17 GB", | 20260507 02:24:10.873 | INFO | "huge" : "", | 20260507 02:24:10.873 | INFO | "huge_shared" : " 27 GB", | 20260507 02:24:10.873 | INFO | "malloc" : " 14 GB", | 20260507 02:24:10.873 | INFO | "pinned" : " 29 GB" | 20260507 02:24:10.873 | INFO | } | 20260507 02:24:10.873 | INFO | Perf report for the last 5.12929 seconds: | 20260507 02:24:10.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:24:10.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:10.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:10.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:10.873 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 02:24:10.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:10.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:10.873 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 02:24:10.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:10.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:10.873 | INFO | Uploaded 36GiB to GPU in 2.02432s (17.7838GiB/s) | 20260507 02:24:10.873 | INFO | Downloaded 34GiB to GPU in 1.93412s (17.579GiB/s) | 20260507 02:24:16.010 | INFO | Analyzing chunk frames = [6351872,6352384) | 20260507 02:24:16.010 | INFO | MemInfo: { | 20260507 02:24:16.010 | INFO | "gpu" : " 17 GB", | 20260507 02:24:16.010 | INFO | "huge" : "", | 20260507 02:24:16.010 | INFO | "huge_shared" : " 27 GB", | 20260507 02:24:16.010 | INFO | "malloc" : " 14 GB", | 20260507 02:24:16.010 | INFO | "pinned" : " 29 GB" | 20260507 02:24:16.010 | INFO | } | 20260507 02:24:16.010 | INFO | Perf report for the last 5.13752 seconds: | 20260507 02:24:16.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:24:16.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:16.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:16.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:16.010 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 02:24:16.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:16.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:16.010 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 02:24:16.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:16.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:16.011 | INFO | Uploaded 36GiB to GPU in 1.99673s (18.0295GiB/s) | 20260507 02:24:16.011 | INFO | Downloaded 34GiB to GPU in 1.99295s (17.0602GiB/s) | 20260507 02:24:21.154 | INFO | Analyzing chunk frames = [6352384,6352896) | 20260507 02:24:21.154 | INFO | MemInfo: { | 20260507 02:24:21.154 | INFO | "gpu" : " 17 GB", | 20260507 02:24:21.154 | INFO | "huge" : "", | 20260507 02:24:21.154 | INFO | "huge_shared" : " 27 GB", | 20260507 02:24:21.154 | INFO | "malloc" : " 14 GB", | 20260507 02:24:21.154 | INFO | "pinned" : " 29 GB" | 20260507 02:24:21.154 | INFO | } | 20260507 02:24:21.154 | INFO | Perf report for the last 5.14401 seconds: | 20260507 02:24:21.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:24:21.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:21.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:21.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:21.154 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:24:21.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:21.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:21.154 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 02:24:21.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:21.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:21.155 | INFO | Uploaded 36GiB to GPU in 2.01074s (17.9039GiB/s) | 20260507 02:24:21.155 | INFO | Downloaded 34GiB to GPU in 1.97446s (17.2199GiB/s) | 20260507 02:24:21.251 | INFO | Starting a round of Baz IO | 20260507 02:24:24.363 | INFO | Populating RTMetrics report | 20260507 02:24:24.363 | INFO | RTMetrics report populated | 20260507 02:24:24.363 | INFO | Opening tmp RTMetrics json file | 20260507 02:24:24.364 | INFO | Serializing RTMetrics to json | 20260507 02:24:24.380 | INFO | Serialization done, swapping tmp file now... | 20260507 02:24:24.383 | INFO | RTMetrics done | 20260507 02:24:24.451 | INFO | Finished a round of Baz IO: Wrote 1.784 GiB in 3.199 seconds | 20260507 02:24:24.451 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.01% / 0.00% / 66.99% | 20260507 02:24:24.451 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.239746% / 0% / 5.5631e-05% | 20260507 02:24:24.451 | INFO | Baz IO overall: | 20260507 02:24:24.451 | INFO | BytesWritten : 3279566101333 | 20260507 02:24:24.451 | INFO | NumEvents : 1577269808494 | 20260507 02:24:26.288 | INFO | Analyzing chunk frames = [6352896,6353408) | 20260507 02:24:26.288 | INFO | MemInfo: { | 20260507 02:24:26.288 | INFO | "gpu" : " 17 GB", | 20260507 02:24:26.288 | INFO | "huge" : "", | 20260507 02:24:26.288 | INFO | "huge_shared" : " 27 GB", | 20260507 02:24:26.288 | INFO | "malloc" : " 14 GB", | 20260507 02:24:26.288 | INFO | "pinned" : " 29 GB" | 20260507 02:24:26.288 | INFO | } | 20260507 02:24:26.288 | INFO | Perf report for the last 5.13393 seconds: | 20260507 02:24:26.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:24:26.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:26.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:26.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:26.288 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260507 02:24:26.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:26.288 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 02:24:26.288 | INFO | PRE_HQ : 59.5%, 1.8, 56.9ms, 0 | 20260507 02:24:26.288 | INFO | BAZWRITER : 61.9%, 3.3, 105.6ms, 0 | 20260507 02:24:26.288 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:24:26.288 | INFO | Uploaded 36GiB to GPU in 2.03716s (17.6717GiB/s) | 20260507 02:24:26.289 | INFO | Downloaded 38GiB to GPU in 2.17722s (17.4535GiB/s) | 20260507 02:24:31.422 | INFO | Analyzing chunk frames = [6353408,6353920) | 20260507 02:24:31.422 | INFO | MemInfo: { | 20260507 02:24:31.422 | INFO | "gpu" : " 17 GB", | 20260507 02:24:31.422 | INFO | "huge" : "", | 20260507 02:24:31.422 | INFO | "huge_shared" : " 27 GB", | 20260507 02:24:31.422 | INFO | "malloc" : " 14 GB", | 20260507 02:24:31.422 | INFO | "pinned" : " 29 GB" | 20260507 02:24:31.422 | INFO | } | 20260507 02:24:31.422 | INFO | Perf report for the last 5.13401 seconds: | 20260507 02:24:31.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:24:31.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:31.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:31.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:31.422 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 02:24:31.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:31.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:31.422 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 02:24:31.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:31.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:31.423 | INFO | Uploaded 36GiB to GPU in 2.02894s (17.7432GiB/s) | 20260507 02:24:31.423 | INFO | Downloaded 34GiB to GPU in 1.94372s (17.4923GiB/s) | 20260507 02:24:36.569 | INFO | Analyzing chunk frames = [6353920,6354432) | 20260507 02:24:36.569 | INFO | MemInfo: { | 20260507 02:24:36.569 | INFO | "gpu" : " 17 GB", | 20260507 02:24:36.569 | INFO | "huge" : "", | 20260507 02:24:36.569 | INFO | "huge_shared" : " 27 GB", | 20260507 02:24:36.569 | INFO | "malloc" : " 14 GB", | 20260507 02:24:36.569 | INFO | "pinned" : " 29 GB" | 20260507 02:24:36.569 | INFO | } | 20260507 02:24:36.569 | INFO | Perf report for the last 5.14671 seconds: | 20260507 02:24:36.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:24:36.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:36.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:36.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:36.569 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260507 02:24:36.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:36.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:36.569 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:24:36.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:36.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:36.569 | INFO | Uploaded 36GiB to GPU in 1.98872s (18.1021GiB/s) | 20260507 02:24:36.569 | INFO | Downloaded 34GiB to GPU in 1.92925s (17.6235GiB/s) | 20260507 02:24:41.710 | INFO | Analyzing chunk frames = [6354432,6354944) | 20260507 02:24:41.710 | INFO | MemInfo: { | 20260507 02:24:41.710 | INFO | "gpu" : " 17 GB", | 20260507 02:24:41.710 | INFO | "huge" : "", | 20260507 02:24:41.710 | INFO | "huge_shared" : " 27 GB", | 20260507 02:24:41.710 | INFO | "malloc" : " 14 GB", | 20260507 02:24:41.710 | INFO | "pinned" : " 29 GB" | 20260507 02:24:41.710 | INFO | } | 20260507 02:24:41.710 | INFO | Perf report for the last 5.14139 seconds: | 20260507 02:24:41.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:24:41.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:41.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:41.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:41.710 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:24:41.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:41.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:41.711 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:24:41.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:41.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:41.711 | INFO | Uploaded 36GiB to GPU in 1.97095s (18.2653GiB/s) | 20260507 02:24:41.711 | INFO | Downloaded 34GiB to GPU in 1.90038s (17.8911GiB/s) | 20260507 02:24:46.851 | INFO | Analyzing chunk frames = [6354944,6355456) | 20260507 02:24:46.851 | INFO | MemInfo: { | 20260507 02:24:46.851 | INFO | "gpu" : " 17 GB", | 20260507 02:24:46.851 | INFO | "huge" : "", | 20260507 02:24:46.851 | INFO | "huge_shared" : " 27 GB", | 20260507 02:24:46.851 | INFO | "malloc" : " 14 GB", | 20260507 02:24:46.851 | INFO | "pinned" : " 29 GB" | 20260507 02:24:46.851 | INFO | } | 20260507 02:24:46.851 | INFO | Perf report for the last 5.14045 seconds: | 20260507 02:24:46.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:24:46.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:46.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:46.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:46.851 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 02:24:46.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:46.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:46.851 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 02:24:46.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:46.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:46.851 | INFO | Uploaded 36GiB to GPU in 2.02613s (17.7678GiB/s) | 20260507 02:24:46.851 | INFO | Downloaded 34GiB to GPU in 1.93268s (17.5921GiB/s) | 20260507 02:24:51.996 | INFO | Analyzing chunk frames = [6355456,6355968) | 20260507 02:24:51.996 | INFO | MemInfo: { | 20260507 02:24:51.996 | INFO | "gpu" : " 17 GB", | 20260507 02:24:51.996 | INFO | "huge" : "", | 20260507 02:24:51.996 | INFO | "huge_shared" : " 27 GB", | 20260507 02:24:51.996 | INFO | "malloc" : " 14 GB", | 20260507 02:24:51.996 | INFO | "pinned" : " 29 GB" | 20260507 02:24:51.996 | INFO | } | 20260507 02:24:51.996 | INFO | Perf report for the last 5.14496 seconds: | 20260507 02:24:51.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:24:51.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:51.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:51.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:51.996 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 02:24:51.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:51.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:51.996 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:24:51.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:51.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:51.996 | INFO | Uploaded 36GiB to GPU in 2.05657s (17.5049GiB/s) | 20260507 02:24:51.996 | INFO | Downloaded 34GiB to GPU in 1.96232s (17.3264GiB/s) | 20260507 02:24:57.129 | INFO | Analyzing chunk frames = [6355968,6356480) | 20260507 02:24:57.130 | INFO | MemInfo: { | 20260507 02:24:57.130 | INFO | "gpu" : " 17 GB", | 20260507 02:24:57.130 | INFO | "huge" : "", | 20260507 02:24:57.130 | INFO | "huge_shared" : " 27 GB", | 20260507 02:24:57.130 | INFO | "malloc" : " 14 GB", | 20260507 02:24:57.130 | INFO | "pinned" : " 29 GB" | 20260507 02:24:57.130 | INFO | } | 20260507 02:24:57.130 | INFO | Perf report for the last 5.13385 seconds: | 20260507 02:24:57.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:24:57.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:57.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:57.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:57.130 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 02:24:57.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:57.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:57.130 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:24:57.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:57.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:24:57.130 | INFO | Uploaded 36GiB to GPU in 1.97235s (18.2524GiB/s) | 20260507 02:24:57.130 | INFO | Downloaded 34GiB to GPU in 1.97358s (17.2276GiB/s) | 20260507 02:25:02.264 | INFO | Analyzing chunk frames = [6356480,6356992) | 20260507 02:25:02.264 | INFO | MemInfo: { | 20260507 02:25:02.264 | INFO | "gpu" : " 17 GB", | 20260507 02:25:02.264 | INFO | "huge" : "", | 20260507 02:25:02.264 | INFO | "huge_shared" : " 27 GB", | 20260507 02:25:02.264 | INFO | "malloc" : " 14 GB", | 20260507 02:25:02.264 | INFO | "pinned" : " 29 GB" | 20260507 02:25:02.264 | INFO | } | 20260507 02:25:02.264 | INFO | Perf report for the last 5.13449 seconds: | 20260507 02:25:02.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:25:02.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:02.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:02.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:02.264 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:25:02.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:02.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:02.264 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 02:25:02.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:02.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:02.264 | INFO | Uploaded 36GiB to GPU in 1.97084s (18.2663GiB/s) | 20260507 02:25:02.264 | INFO | Downloaded 34GiB to GPU in 1.97069s (17.2528GiB/s) | 20260507 02:25:02.264 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:25:02.264 | INFO | Total mem usage: 18 GiB | 20260507 02:25:02.264 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:25:02.264 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:25:02.265 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:25:02.265 | INFO | High Water Breakdown: | 20260507 02:25:02.265 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:25:02.265 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:25:02.265 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:25:02.265 | INFO | ------------------------------------------ | 20260507 02:25:02.265 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:25:02.265 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:25:02.265 | INFO | Total mem usage: 26.5 GiB | 20260507 02:25:02.265 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:25:02.265 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:25:02.265 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:25:02.265 | INFO | High Water Breakdown: | 20260507 02:25:02.265 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:25:02.265 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:25:02.265 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:25:02.265 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:25:02.265 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:25:02.265 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:25:02.265 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:25:02.265 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:25:02.265 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:25:02.265 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:25:02.265 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:25:02.265 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:25:02.265 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:25:02.265 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:25:02.265 | INFO | ------------------------------------------ | 20260507 02:25:02.265 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:25:02.265 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:25:02.265 | INFO | Total mem usage: 25.6 GiB | 20260507 02:25:02.265 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:25:02.265 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:25:02.265 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:25:02.265 | INFO | High Water Breakdown: | 20260507 02:25:02.265 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:25:02.265 | INFO | ------------------------------------------ | 20260507 02:25:02.265 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:25:02.265 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:25:02.265 | INFO | Total mem usage: 15.7 GiB | 20260507 02:25:02.265 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:25:02.265 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:25:02.265 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:25:02.265 | INFO | High Water Breakdown: | 20260507 02:25:02.265 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:25:02.265 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:25:02.265 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:25:02.265 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:25:02.265 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:25:02.265 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:25:02.265 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:25:02.265 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:25:02.265 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:25:02.265 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:25:02.265 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:25:02.266 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:25:02.266 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:25:02.266 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:25:02.266 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:25:02.266 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:25:02.266 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:25:02.266 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:25:02.266 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:25:02.266 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:25:02.266 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:25:02.266 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:25:02.266 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:25:02.266 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:25:02.266 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:25:02.266 | INFO | ------------------------------------------ | 20260507 02:25:02.266 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:25:02.266 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:25:02.266 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:25:02.266 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:25:02.266 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.49e+01, 1.33e+01 / 2.02e+00 | 20260507 02:25:02.266 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.00e-01, 1.00e-02 / 6.25e-03 | 20260507 02:25:02.266 | INFO | PerfReport: 11.65%, Baseline -- 3.59e+00 / 4.38e+00, 5.20e-01 / 3.51e-01 | 20260507 02:25:02.266 | INFO | PerfReport: 3.23%, Histogram -- 9.95e-01 / 1.44e+00, 1.70e-01 / 1.26e-01 | 20260507 02:25:02.266 | INFO | PerfReport: 16.12%, DME -- 3.97e+01 / 4.42e+01, 3.97e+00 / 2.37e+00 | 20260507 02:25:02.266 | INFO | PerfReport: 57.56%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.29e+00 / 1.63e+00 | 20260507 02:25:02.266 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.16e+00 / 2.70e+00, 3.89e-01 / 2.18e-01 | 20260507 02:25:02.266 | INFO | PerfReport: 3.08%, Metrics -- 9.49e-01 / 1.28e+00, 2.53e-01 / 1.36e-01 | 20260507 02:25:02.266 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.21e-01 / 3.73e-01, 5.16e-02 / 2.18e-02 | 20260507 02:25:02.266 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:25:02.266 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:25:02.266 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:25:02.266 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 02:25:02.266 | INFO | PerfReport: 21.67%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260507 02:25:02.266 | INFO | PerfReport: 21.34%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260507 02:25:02.266 | INFO | PerfReport: 31.98%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 02:25:02.266 | INFO | PerfReport: 1.42%, LockUpload -- 1.37e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 02:25:02.266 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.15e+00 / 0.00e+00 | 20260507 02:25:02.266 | INFO | PerfReport: 22.26%, LockCompute -- 2.14e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 02:25:02.266 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260507 02:25:02.361 | INFO | Starting a round of Baz IO | 20260507 02:25:05.480 | INFO | Populating RTMetrics report | 20260507 02:25:05.481 | INFO | RTMetrics report populated | 20260507 02:25:05.481 | INFO | Opening tmp RTMetrics json file | 20260507 02:25:05.497 | INFO | Serializing RTMetrics to json | 20260507 02:25:05.501 | INFO | Serialization done, swapping tmp file now... | 20260507 02:25:05.501 | INFO | RTMetrics done | 20260507 02:25:05.574 | INFO | Finished a round of Baz IO: Wrote 1.783 GiB in 3.212 seconds | 20260507 02:25:05.574 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.01% / 0.00% / 66.99% | 20260507 02:25:05.574 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.238689% / 3.97364e-06% / 6.35783e-05% | 20260507 02:25:05.574 | INFO | Baz IO overall: | 20260507 02:25:05.574 | INFO | BytesWritten : 3281480637920 | 20260507 02:25:05.574 | INFO | NumEvents : 1578175429654 | 20260507 02:25:07.405 | INFO | Analyzing chunk frames = [6356992,6357504) | 20260507 02:25:07.405 | INFO | MemInfo: { | 20260507 02:25:07.405 | INFO | "gpu" : " 17 GB", | 20260507 02:25:07.405 | INFO | "huge" : "", | 20260507 02:25:07.405 | INFO | "huge_shared" : " 27 GB", | 20260507 02:25:07.405 | INFO | "malloc" : " 14 GB", | 20260507 02:25:07.405 | INFO | "pinned" : " 29 GB" | 20260507 02:25:07.405 | INFO | } | 20260507 02:25:07.405 | INFO | Perf report for the last 5.14134 seconds: | 20260507 02:25:07.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:25:07.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:07.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:07.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:07.406 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 02:25:07.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:07.406 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260507 02:25:07.406 | INFO | PRE_HQ : 59.9%, 1.9, 58.7ms, 0 | 20260507 02:25:07.406 | INFO | BAZWRITER : 62.1%, 3.4, 108.6ms, 0 | 20260507 02:25:07.406 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 02:25:07.406 | INFO | Uploaded 36GiB to GPU in 2.04124s (17.6364GiB/s) | 20260507 02:25:07.406 | INFO | Downloaded 38GiB to GPU in 2.18027s (17.4291GiB/s) | 20260507 02:25:12.547 | INFO | Analyzing chunk frames = [6357504,6358016) | 20260507 02:25:12.547 | INFO | MemInfo: { | 20260507 02:25:12.547 | INFO | "gpu" : " 17 GB", | 20260507 02:25:12.547 | INFO | "huge" : "", | 20260507 02:25:12.547 | INFO | "huge_shared" : " 27 GB", | 20260507 02:25:12.547 | INFO | "malloc" : " 14 GB", | 20260507 02:25:12.547 | INFO | "pinned" : " 29 GB" | 20260507 02:25:12.547 | INFO | } | 20260507 02:25:12.547 | INFO | Perf report for the last 5.14135 seconds: | 20260507 02:25:12.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:25:12.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:12.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:12.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:12.547 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:25:12.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:12.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:12.547 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 02:25:12.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:12.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:12.547 | INFO | Uploaded 36GiB to GPU in 2.03383s (17.7006GiB/s) | 20260507 02:25:12.547 | INFO | Downloaded 34GiB to GPU in 1.94211s (17.5067GiB/s) | 20260507 02:25:17.679 | INFO | Analyzing chunk frames = [6358016,6358528) | 20260507 02:25:17.679 | INFO | MemInfo: { | 20260507 02:25:17.679 | INFO | "gpu" : " 17 GB", | 20260507 02:25:17.679 | INFO | "huge" : "", | 20260507 02:25:17.679 | INFO | "huge_shared" : " 27 GB", | 20260507 02:25:17.679 | INFO | "malloc" : " 14 GB", | 20260507 02:25:17.679 | INFO | "pinned" : " 29 GB" | 20260507 02:25:17.679 | INFO | } | 20260507 02:25:17.679 | INFO | Perf report for the last 5.13239 seconds: | 20260507 02:25:17.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:25:17.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:17.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:17.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:17.679 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 02:25:17.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:17.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:17.679 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:25:17.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:17.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:17.679 | INFO | Uploaded 36GiB to GPU in 1.99076s (18.0835GiB/s) | 20260507 02:25:17.680 | INFO | Downloaded 34GiB to GPU in 1.94518s (17.4791GiB/s) | 20260507 02:25:22.824 | INFO | Analyzing chunk frames = [6358528,6359040) | 20260507 02:25:22.824 | INFO | MemInfo: { | 20260507 02:25:22.824 | INFO | "gpu" : " 17 GB", | 20260507 02:25:22.824 | INFO | "huge" : "", | 20260507 02:25:22.824 | INFO | "huge_shared" : " 27 GB", | 20260507 02:25:22.824 | INFO | "malloc" : " 14 GB", | 20260507 02:25:22.824 | INFO | "pinned" : " 29 GB" | 20260507 02:25:22.824 | INFO | } | 20260507 02:25:22.824 | INFO | Perf report for the last 5.14504 seconds: | 20260507 02:25:22.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:25:22.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:22.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:22.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:22.824 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 02:25:22.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:22.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:22.824 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:25:22.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:22.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:22.825 | INFO | Uploaded 36GiB to GPU in 2.01393s (17.8755GiB/s) | 20260507 02:25:22.825 | INFO | Downloaded 34GiB to GPU in 1.91048s (17.7965GiB/s) | 20260507 02:25:27.968 | INFO | Analyzing chunk frames = [6359040,6359552) | 20260507 02:25:27.968 | INFO | MemInfo: { | 20260507 02:25:27.968 | INFO | "gpu" : " 17 GB", | 20260507 02:25:27.968 | INFO | "huge" : "", | 20260507 02:25:27.968 | INFO | "huge_shared" : " 27 GB", | 20260507 02:25:27.968 | INFO | "malloc" : " 14 GB", | 20260507 02:25:27.968 | INFO | "pinned" : " 29 GB" | 20260507 02:25:27.968 | INFO | } | 20260507 02:25:27.968 | INFO | Perf report for the last 5.14338 seconds: | 20260507 02:25:27.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:25:27.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:27.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:27.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:27.968 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 02:25:27.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:27.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:27.968 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:25:27.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:27.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:27.968 | INFO | Uploaded 36GiB to GPU in 2.00645s (17.9422GiB/s) | 20260507 02:25:27.968 | INFO | Downloaded 34GiB to GPU in 1.95158s (17.4218GiB/s) | 20260507 02:25:33.102 | INFO | Analyzing chunk frames = [6359552,6360064) | 20260507 02:25:33.102 | INFO | MemInfo: { | 20260507 02:25:33.102 | INFO | "gpu" : " 17 GB", | 20260507 02:25:33.102 | INFO | "huge" : "", | 20260507 02:25:33.102 | INFO | "huge_shared" : " 27 GB", | 20260507 02:25:33.102 | INFO | "malloc" : " 14 GB", | 20260507 02:25:33.102 | INFO | "pinned" : " 29 GB" | 20260507 02:25:33.102 | INFO | } | 20260507 02:25:33.103 | INFO | Perf report for the last 5.1348 seconds: | 20260507 02:25:33.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:25:33.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:33.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:33.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:33.103 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 02:25:33.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:33.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:33.103 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:25:33.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:33.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:33.103 | INFO | Uploaded 36GiB to GPU in 2.04392s (17.6132GiB/s) | 20260507 02:25:33.103 | INFO | Downloaded 34GiB to GPU in 1.96403s (17.3114GiB/s) | 20260507 02:25:38.235 | INFO | Analyzing chunk frames = [6360064,6360576) | 20260507 02:25:38.235 | INFO | MemInfo: { | 20260507 02:25:38.235 | INFO | "gpu" : " 17 GB", | 20260507 02:25:38.235 | INFO | "huge" : "", | 20260507 02:25:38.235 | INFO | "huge_shared" : " 27 GB", | 20260507 02:25:38.235 | INFO | "malloc" : " 14 GB", | 20260507 02:25:38.235 | INFO | "pinned" : " 29 GB" | 20260507 02:25:38.235 | INFO | } | 20260507 02:25:38.235 | INFO | Perf report for the last 5.13243 seconds: | 20260507 02:25:38.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:25:38.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:38.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:38.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:38.235 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:25:38.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:38.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:38.235 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:25:38.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:38.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:38.235 | INFO | Uploaded 36GiB to GPU in 1.98626s (18.1245GiB/s) | 20260507 02:25:38.235 | INFO | Downloaded 34GiB to GPU in 1.96507s (17.3022GiB/s) | 20260507 02:25:43.377 | INFO | Analyzing chunk frames = [6360576,6361088) | 20260507 02:25:43.377 | INFO | MemInfo: { | 20260507 02:25:43.377 | INFO | "gpu" : " 17 GB", | 20260507 02:25:43.378 | INFO | "huge" : "", | 20260507 02:25:43.378 | INFO | "huge_shared" : " 27 GB", | 20260507 02:25:43.378 | INFO | "malloc" : " 14 GB", | 20260507 02:25:43.378 | INFO | "pinned" : " 29 GB" | 20260507 02:25:43.378 | INFO | } | 20260507 02:25:43.378 | INFO | Perf report for the last 5.14262 seconds: | 20260507 02:25:43.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:25:43.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:43.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:43.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:43.378 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:25:43.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:43.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:43.378 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 02:25:43.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:43.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:43.378 | INFO | Uploaded 36GiB to GPU in 1.96928s (18.2808GiB/s) | 20260507 02:25:43.378 | INFO | Downloaded 34GiB to GPU in 1.96015s (17.3456GiB/s) | 20260507 02:25:43.474 | INFO | Starting a round of Baz IO | 20260507 02:25:46.595 | INFO | Populating RTMetrics report | 20260507 02:25:46.596 | INFO | RTMetrics report populated | 20260507 02:25:46.596 | INFO | Opening tmp RTMetrics json file | 20260507 02:25:46.601 | INFO | Serializing RTMetrics to json | 20260507 02:25:46.614 | INFO | Serialization done, swapping tmp file now... | 20260507 02:25:46.615 | INFO | RTMetrics done | 20260507 02:25:46.688 | INFO | Finished a round of Baz IO: Wrote 1.783 GiB in 3.213 seconds | 20260507 02:25:46.688 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.01% / 0.00% / 66.99% | 20260507 02:25:46.688 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.23886% / 7.94729e-06% / 8.34465e-05% | 20260507 02:25:46.688 | INFO | Baz IO overall: | 20260507 02:25:46.688 | INFO | BytesWritten : 3283395643620 | 20260507 02:25:46.688 | INFO | NumEvents : 1579081489310 | 20260507 02:25:48.520 | INFO | Analyzing chunk frames = [6361088,6361600) | 20260507 02:25:48.520 | INFO | MemInfo: { | 20260507 02:25:48.520 | INFO | "gpu" : " 17 GB", | 20260507 02:25:48.520 | INFO | "huge" : "", | 20260507 02:25:48.520 | INFO | "huge_shared" : " 27 GB", | 20260507 02:25:48.520 | INFO | "malloc" : " 14 GB", | 20260507 02:25:48.520 | INFO | "pinned" : " 29 GB" | 20260507 02:25:48.520 | INFO | } | 20260507 02:25:48.520 | INFO | Perf report for the last 5.14212 seconds: | 20260507 02:25:48.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:25:48.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:48.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:48.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:48.520 | INFO | ANALYSIS : 62.0%, 3.0, 96.6ms, 0 | 20260507 02:25:48.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:48.520 | INFO | RT_METRICS : 42.2%, 1.0, 22.2ms, 0 | 20260507 02:25:48.520 | INFO | PRE_HQ : 59.4%, 1.8, 55.9ms, 0 | 20260507 02:25:48.520 | INFO | BAZWRITER : 61.9%, 3.3, 104.8ms, 0 | 20260507 02:25:48.520 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:25:48.520 | INFO | Uploaded 36GiB to GPU in 2.02966s (17.7369GiB/s) | 20260507 02:25:48.520 | INFO | Downloaded 38GiB to GPU in 2.1841s (17.3984GiB/s) | 20260507 02:25:53.665 | INFO | Analyzing chunk frames = [6361600,6362112) | 20260507 02:25:53.665 | INFO | MemInfo: { | 20260507 02:25:53.665 | INFO | "gpu" : " 17 GB", | 20260507 02:25:53.665 | INFO | "huge" : "", | 20260507 02:25:53.665 | INFO | "huge_shared" : " 27 GB", | 20260507 02:25:53.665 | INFO | "malloc" : " 14 GB", | 20260507 02:25:53.665 | INFO | "pinned" : " 29 GB" | 20260507 02:25:53.665 | INFO | } | 20260507 02:25:53.665 | INFO | Perf report for the last 5.14532 seconds: | 20260507 02:25:53.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:25:53.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:53.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:53.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:53.665 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 02:25:53.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:53.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:53.665 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:25:53.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:53.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:53.665 | INFO | Uploaded 36GiB to GPU in 2.01034s (17.9074GiB/s) | 20260507 02:25:53.665 | INFO | Downloaded 34GiB to GPU in 1.93398s (17.5803GiB/s) | 20260507 02:25:58.805 | INFO | Analyzing chunk frames = [6362112,6362624) | 20260507 02:25:58.805 | INFO | MemInfo: { | 20260507 02:25:58.805 | INFO | "gpu" : " 17 GB", | 20260507 02:25:58.805 | INFO | "huge" : "", | 20260507 02:25:58.805 | INFO | "huge_shared" : " 27 GB", | 20260507 02:25:58.805 | INFO | "malloc" : " 14 GB", | 20260507 02:25:58.805 | INFO | "pinned" : " 29 GB" | 20260507 02:25:58.805 | INFO | } | 20260507 02:25:58.805 | INFO | Perf report for the last 5.1399 seconds: | 20260507 02:25:58.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:25:58.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:58.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:58.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:58.805 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 02:25:58.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:58.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:58.805 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:25:58.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:58.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:25:58.805 | INFO | Uploaded 36GiB to GPU in 1.98466s (18.1391GiB/s) | 20260507 02:25:58.805 | INFO | Downloaded 34GiB to GPU in 1.94469s (17.4835GiB/s) | 20260507 02:26:03.941 | INFO | Analyzing chunk frames = [6362624,6363136) | 20260507 02:26:03.941 | INFO | MemInfo: { | 20260507 02:26:03.941 | INFO | "gpu" : " 17 GB", | 20260507 02:26:03.941 | INFO | "huge" : "", | 20260507 02:26:03.941 | INFO | "huge_shared" : " 27 GB", | 20260507 02:26:03.941 | INFO | "malloc" : " 14 GB", | 20260507 02:26:03.941 | INFO | "pinned" : " 29 GB" | 20260507 02:26:03.941 | INFO | } | 20260507 02:26:03.941 | INFO | Perf report for the last 5.13597 seconds: | 20260507 02:26:03.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:26:03.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:03.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:03.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:03.941 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 02:26:03.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:03.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:03.941 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:26:03.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:03.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:03.941 | INFO | Uploaded 36GiB to GPU in 2.01036s (17.9072GiB/s) | 20260507 02:26:03.941 | INFO | Downloaded 34GiB to GPU in 1.93928s (17.5323GiB/s) | 20260507 02:26:09.080 | INFO | Analyzing chunk frames = [6363136,6363648) | 20260507 02:26:09.080 | INFO | MemInfo: { | 20260507 02:26:09.080 | INFO | "gpu" : " 17 GB", | 20260507 02:26:09.080 | INFO | "huge" : "", | 20260507 02:26:09.080 | INFO | "huge_shared" : " 27 GB", | 20260507 02:26:09.080 | INFO | "malloc" : " 14 GB", | 20260507 02:26:09.080 | INFO | "pinned" : " 29 GB" | 20260507 02:26:09.080 | INFO | } | 20260507 02:26:09.080 | INFO | Perf report for the last 5.1386 seconds: | 20260507 02:26:09.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:26:09.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:09.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:09.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:09.080 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 02:26:09.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:09.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:09.080 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:26:09.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:09.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:09.080 | INFO | Uploaded 36GiB to GPU in 2.01888s (17.8317GiB/s) | 20260507 02:26:09.080 | INFO | Downloaded 34GiB to GPU in 1.94008s (17.5251GiB/s) | 20260507 02:26:14.215 | INFO | Analyzing chunk frames = [6363648,6364160) | 20260507 02:26:14.215 | INFO | MemInfo: { | 20260507 02:26:14.215 | INFO | "gpu" : " 17 GB", | 20260507 02:26:14.215 | INFO | "huge" : "", | 20260507 02:26:14.215 | INFO | "huge_shared" : " 27 GB", | 20260507 02:26:14.215 | INFO | "malloc" : " 14 GB", | 20260507 02:26:14.215 | INFO | "pinned" : " 29 GB" | 20260507 02:26:14.215 | INFO | } | 20260507 02:26:14.215 | INFO | Perf report for the last 5.13566 seconds: | 20260507 02:26:14.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:26:14.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:14.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:14.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:14.215 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 02:26:14.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:14.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:14.215 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:26:14.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:14.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:14.215 | INFO | Uploaded 36GiB to GPU in 2.03358s (17.7028GiB/s) | 20260507 02:26:14.216 | INFO | Downloaded 34GiB to GPU in 1.95232s (17.4152GiB/s) | 20260507 02:26:19.358 | INFO | Analyzing chunk frames = [6364160,6364672) | 20260507 02:26:19.358 | INFO | MemInfo: { | 20260507 02:26:19.358 | INFO | "gpu" : " 17 GB", | 20260507 02:26:19.358 | INFO | "huge" : "", | 20260507 02:26:19.358 | INFO | "huge_shared" : " 27 GB", | 20260507 02:26:19.358 | INFO | "malloc" : " 14 GB", | 20260507 02:26:19.358 | INFO | "pinned" : " 29 GB" | 20260507 02:26:19.358 | INFO | } | 20260507 02:26:19.358 | INFO | Perf report for the last 5.1426 seconds: | 20260507 02:26:19.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:26:19.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:19.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:19.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:19.358 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:26:19.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:19.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:19.358 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 02:26:19.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:19.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:19.358 | INFO | Uploaded 36GiB to GPU in 1.97592s (18.2194GiB/s) | 20260507 02:26:19.358 | INFO | Downloaded 34GiB to GPU in 2.00196s (16.9834GiB/s) | 20260507 02:26:24.494 | INFO | Analyzing chunk frames = [6364672,6365184) | 20260507 02:26:24.494 | INFO | MemInfo: { | 20260507 02:26:24.494 | INFO | "gpu" : " 17 GB", | 20260507 02:26:24.494 | INFO | "huge" : "", | 20260507 02:26:24.494 | INFO | "huge_shared" : " 27 GB", | 20260507 02:26:24.494 | INFO | "malloc" : " 14 GB", | 20260507 02:26:24.494 | INFO | "pinned" : " 29 GB" | 20260507 02:26:24.494 | INFO | } | 20260507 02:26:24.494 | INFO | Perf report for the last 5.13644 seconds: | 20260507 02:26:24.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:26:24.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:24.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:24.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:24.494 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:26:24.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:24.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:24.494 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:26:24.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:24.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:24.495 | INFO | Uploaded 36GiB to GPU in 1.99701s (18.027GiB/s) | 20260507 02:26:24.495 | INFO | Downloaded 34GiB to GPU in 1.97935s (17.1773GiB/s) | 20260507 02:26:24.495 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:26:24.495 | INFO | Total mem usage: 18 GiB | 20260507 02:26:24.495 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:26:24.495 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:26:24.495 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:26:24.495 | INFO | High Water Breakdown: | 20260507 02:26:24.495 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:26:24.495 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:26:24.495 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:26:24.495 | INFO | ------------------------------------------ | 20260507 02:26:24.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:26:24.495 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:26:24.495 | INFO | Total mem usage: 26.5 GiB | 20260507 02:26:24.495 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:26:24.495 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:26:24.495 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:26:24.495 | INFO | High Water Breakdown: | 20260507 02:26:24.495 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:26:24.495 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:26:24.495 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:26:24.495 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:26:24.495 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:26:24.495 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:26:24.495 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:26:24.495 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:26:24.495 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:26:24.495 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:26:24.495 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:26:24.495 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:26:24.495 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:26:24.495 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:26:24.495 | INFO | ------------------------------------------ | 20260507 02:26:24.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:26:24.495 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:26:24.495 | INFO | Total mem usage: 25.6 GiB | 20260507 02:26:24.495 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:26:24.495 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:26:24.495 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:26:24.495 | INFO | High Water Breakdown: | 20260507 02:26:24.495 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:26:24.495 | INFO | ------------------------------------------ | 20260507 02:26:24.495 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:26:24.495 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:26:24.495 | INFO | Total mem usage: 15.7 GiB | 20260507 02:26:24.495 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:26:24.495 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:26:24.495 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 02:26:24.495 | INFO | High Water Breakdown: | 20260507 02:26:24.495 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:26:24.495 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:26:24.495 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:26:24.495 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:26:24.495 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:26:24.495 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:26:24.495 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:26:24.495 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:26:24.495 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:26:24.495 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:26:24.495 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:26:24.495 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:26:24.495 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:26:24.495 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:26:24.495 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:26:24.495 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:26:24.495 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:26:24.495 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:26:24.495 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:26:24.495 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:26:24.495 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:26:24.496 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:26:24.496 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:26:24.496 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:26:24.496 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:26:24.496 | INFO | ------------------------------------------ | 20260507 02:26:24.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:26:24.496 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:26:24.496 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:26:24.496 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:26:24.496 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.54e+01, 1.33e+01 / 2.10e+00 | 20260507 02:26:24.496 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 9.97e-02, 1.00e-02 / 5.00e-03 | 20260507 02:26:24.496 | INFO | PerfReport: 11.64%, Baseline -- 3.58e+00 / 4.43e+00, 5.16e-01 / 3.18e-01 | 20260507 02:26:24.496 | INFO | PerfReport: 3.22%, Histogram -- 9.93e-01 / 1.47e+00, 1.68e-01 / 1.19e-01 | 20260507 02:26:24.496 | INFO | PerfReport: 16.11%, DME -- 3.97e+01 / 4.47e+01, 4.00e+00 / 2.44e+00 | 20260507 02:26:24.496 | INFO | PerfReport: 57.61%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.30e+00 / 1.72e+00 | 20260507 02:26:24.496 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.15e+00 / 2.72e+00, 3.83e-01 / 1.94e-01 | 20260507 02:26:24.496 | INFO | PerfReport: 3.08%, Metrics -- 9.50e-01 / 1.29e+00, 2.55e-01 / 1.13e-01 | 20260507 02:26:24.496 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.21e-01 / 3.78e-01, 5.06e-02 / 2.13e-02 | 20260507 02:26:24.496 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:26:24.496 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:26:24.496 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:26:24.496 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 02:26:24.496 | INFO | PerfReport: 21.71%, Upload -- 2.09e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260507 02:26:24.496 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260507 02:26:24.496 | INFO | PerfReport: 31.95%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 02:26:24.496 | INFO | PerfReport: 1.39%, LockUpload -- 1.34e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 02:26:24.496 | INFO | PerfReport: 1.34%, LockDownload -- 1.29e+00 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260507 02:26:24.496 | INFO | PerfReport: 22.19%, LockCompute -- 2.14e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 02:26:24.496 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260507 02:26:24.591 | INFO | Starting a round of Baz IO | 20260507 02:26:27.706 | INFO | Populating RTMetrics report | 20260507 02:26:27.707 | INFO | RTMetrics report populated | 20260507 02:26:27.707 | INFO | Opening tmp RTMetrics json file | 20260507 02:26:27.725 | INFO | Serializing RTMetrics to json | 20260507 02:26:27.726 | INFO | Serialization done, swapping tmp file now... | 20260507 02:26:27.726 | INFO | RTMetrics done | 20260507 02:26:27.796 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.205 seconds | 20260507 02:26:27.796 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.01% / 0.00% / 66.99% | 20260507 02:26:27.796 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.237254% / 3.97364e-06% / 6.75519e-05% | 20260507 02:26:27.796 | INFO | Baz IO overall: | 20260507 02:26:27.796 | INFO | BytesWritten : 3285307429343 | 20260507 02:26:27.796 | INFO | NumEvents : 1579985362674 | 20260507 02:26:29.638 | INFO | Analyzing chunk frames = [6365184,6365696) | 20260507 02:26:29.638 | INFO | MemInfo: { | 20260507 02:26:29.638 | INFO | "gpu" : " 17 GB", | 20260507 02:26:29.638 | INFO | "huge" : "", | 20260507 02:26:29.638 | INFO | "huge_shared" : " 27 GB", | 20260507 02:26:29.638 | INFO | "malloc" : " 14 GB", | 20260507 02:26:29.638 | INFO | "pinned" : " 29 GB" | 20260507 02:26:29.638 | INFO | } | 20260507 02:26:29.638 | INFO | Perf report for the last 5.14391 seconds: | 20260507 02:26:29.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:26:29.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:29.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:29.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:29.638 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260507 02:26:29.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:29.638 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 02:26:29.638 | INFO | PRE_HQ : 59.4%, 1.8, 55.5ms, 0 | 20260507 02:26:29.638 | INFO | BAZWRITER : 61.9%, 3.3, 104.3ms, 0 | 20260507 02:26:29.638 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:26:29.639 | INFO | Uploaded 36GiB to GPU in 2.01324s (17.8816GiB/s) | 20260507 02:26:29.639 | INFO | Downloaded 38GiB to GPU in 2.17445s (17.4757GiB/s) | 20260507 02:26:34.781 | INFO | Analyzing chunk frames = [6365696,6366208) | 20260507 02:26:34.781 | INFO | MemInfo: { | 20260507 02:26:34.781 | INFO | "gpu" : " 17 GB", | 20260507 02:26:34.781 | INFO | "huge" : "", | 20260507 02:26:34.781 | INFO | "huge_shared" : " 27 GB", | 20260507 02:26:34.781 | INFO | "malloc" : " 14 GB", | 20260507 02:26:34.781 | INFO | "pinned" : " 29 GB" | 20260507 02:26:34.781 | INFO | } | 20260507 02:26:34.781 | INFO | Perf report for the last 5.14306 seconds: | 20260507 02:26:34.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:26:34.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:34.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:34.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:34.781 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:26:34.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:34.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:34.781 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:26:34.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:34.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:34.782 | INFO | Uploaded 36GiB to GPU in 2.01971s (17.8244GiB/s) | 20260507 02:26:34.782 | INFO | Downloaded 34GiB to GPU in 1.93634s (17.5589GiB/s) | 20260507 02:26:39.919 | INFO | Analyzing chunk frames = [6366208,6366720) | 20260507 02:26:39.919 | INFO | MemInfo: { | 20260507 02:26:39.919 | INFO | "gpu" : " 17 GB", | 20260507 02:26:39.919 | INFO | "huge" : "", | 20260507 02:26:39.919 | INFO | "huge_shared" : " 27 GB", | 20260507 02:26:39.920 | INFO | "malloc" : " 14 GB", | 20260507 02:26:39.920 | INFO | "pinned" : " 29 GB" | 20260507 02:26:39.920 | INFO | } | 20260507 02:26:39.920 | INFO | Perf report for the last 5.13829 seconds: | 20260507 02:26:39.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:26:39.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:39.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:39.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:39.920 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 02:26:39.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:39.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:39.920 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:26:39.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:39.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:39.920 | INFO | Uploaded 36GiB to GPU in 2.01548s (17.8618GiB/s) | 20260507 02:26:39.920 | INFO | Downloaded 34GiB to GPU in 1.94566s (17.4748GiB/s) | 20260507 02:26:45.050 | INFO | Analyzing chunk frames = [6366720,6367232) | 20260507 02:26:45.050 | INFO | MemInfo: { | 20260507 02:26:45.050 | INFO | "gpu" : " 17 GB", | 20260507 02:26:45.050 | INFO | "huge" : "", | 20260507 02:26:45.050 | INFO | "huge_shared" : " 27 GB", | 20260507 02:26:45.050 | INFO | "malloc" : " 14 GB", | 20260507 02:26:45.050 | INFO | "pinned" : " 29 GB" | 20260507 02:26:45.050 | INFO | } | 20260507 02:26:45.050 | INFO | Perf report for the last 5.1306 seconds: | 20260507 02:26:45.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:26:45.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:45.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:45.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:45.050 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 02:26:45.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:45.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:45.050 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 02:26:45.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:45.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:45.050 | INFO | Uploaded 36GiB to GPU in 1.98175s (18.1658GiB/s) | 20260507 02:26:45.050 | INFO | Downloaded 34GiB to GPU in 1.93332s (17.5863GiB/s) | 20260507 02:26:50.186 | INFO | Analyzing chunk frames = [6367232,6367744) | 20260507 02:26:50.186 | INFO | MemInfo: { | 20260507 02:26:50.186 | INFO | "gpu" : " 17 GB", | 20260507 02:26:50.187 | INFO | "huge" : "", | 20260507 02:26:50.187 | INFO | "huge_shared" : " 27 GB", | 20260507 02:26:50.187 | INFO | "malloc" : " 14 GB", | 20260507 02:26:50.187 | INFO | "pinned" : " 29 GB" | 20260507 02:26:50.187 | INFO | } | 20260507 02:26:50.187 | INFO | Perf report for the last 5.13641 seconds: | 20260507 02:26:50.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:26:50.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:50.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:50.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:50.187 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 02:26:50.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:50.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:50.187 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:26:50.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:50.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:50.187 | INFO | Uploaded 36GiB to GPU in 2.00735s (17.9341GiB/s) | 20260507 02:26:50.187 | INFO | Downloaded 34GiB to GPU in 1.94997s (17.4361GiB/s) | 20260507 02:26:55.332 | INFO | Analyzing chunk frames = [6367744,6368256) | 20260507 02:26:55.332 | INFO | MemInfo: { | 20260507 02:26:55.332 | INFO | "gpu" : " 17 GB", | 20260507 02:26:55.332 | INFO | "huge" : "", | 20260507 02:26:55.332 | INFO | "huge_shared" : " 27 GB", | 20260507 02:26:55.332 | INFO | "malloc" : " 14 GB", | 20260507 02:26:55.332 | INFO | "pinned" : " 29 GB" | 20260507 02:26:55.332 | INFO | } | 20260507 02:26:55.333 | INFO | Perf report for the last 5.1459 seconds: | 20260507 02:26:55.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:26:55.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:55.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:55.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:55.333 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 02:26:55.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:55.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:55.333 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 02:26:55.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:55.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:26:55.333 | INFO | Uploaded 36GiB to GPU in 2.00696s (17.9376GiB/s) | 20260507 02:26:55.333 | INFO | Downloaded 34GiB to GPU in 1.94875s (17.4471GiB/s) | 20260507 02:27:00.471 | INFO | Analyzing chunk frames = [6368256,6368768) | 20260507 02:27:00.471 | INFO | MemInfo: { | 20260507 02:27:00.471 | INFO | "gpu" : " 17 GB", | 20260507 02:27:00.471 | INFO | "huge" : "", | 20260507 02:27:00.471 | INFO | "huge_shared" : " 27 GB", | 20260507 02:27:00.471 | INFO | "malloc" : " 14 GB", | 20260507 02:27:00.471 | INFO | "pinned" : " 29 GB" | 20260507 02:27:00.471 | INFO | } | 20260507 02:27:00.471 | INFO | Perf report for the last 5.13846 seconds: | 20260507 02:27:00.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:27:00.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:00.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:00.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:00.471 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:27:00.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:00.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:00.471 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:27:00.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:00.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:00.471 | INFO | Uploaded 36GiB to GPU in 1.99429s (18.0515GiB/s) | 20260507 02:27:00.471 | INFO | Downloaded 34GiB to GPU in 1.99472s (17.045GiB/s) | 20260507 02:27:05.611 | INFO | Analyzing chunk frames = [6368768,6369280) | 20260507 02:27:05.611 | INFO | MemInfo: { | 20260507 02:27:05.611 | INFO | "gpu" : " 17 GB", | 20260507 02:27:05.611 | INFO | "huge" : "", | 20260507 02:27:05.611 | INFO | "huge_shared" : " 27 GB", | 20260507 02:27:05.611 | INFO | "malloc" : " 14 GB", | 20260507 02:27:05.611 | INFO | "pinned" : " 29 GB" | 20260507 02:27:05.611 | INFO | } | 20260507 02:27:05.611 | INFO | Perf report for the last 5.14016 seconds: | 20260507 02:27:05.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:27:05.611 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:05.611 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:05.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:05.611 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:27:05.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:05.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:05.611 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:27:05.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:05.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:05.612 | INFO | Uploaded 36GiB to GPU in 2.00067s (17.9939GiB/s) | 20260507 02:27:05.612 | INFO | Downloaded 34GiB to GPU in 1.98327s (17.1434GiB/s) | 20260507 02:27:05.708 | INFO | Starting a round of Baz IO | 20260507 02:27:08.816 | INFO | Populating RTMetrics report | 20260507 02:27:08.817 | INFO | RTMetrics report populated | 20260507 02:27:08.817 | INFO | Opening tmp RTMetrics json file | 20260507 02:27:08.817 | INFO | Serializing RTMetrics to json | 20260507 02:27:08.818 | INFO | Serialization done, swapping tmp file now... | 20260507 02:27:08.819 | INFO | RTMetrics done | 20260507 02:27:08.903 | INFO | Finished a round of Baz IO: Wrote 1.781 GiB in 3.195 seconds | 20260507 02:27:08.903 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.00% / 0.00% / 66.99% | 20260507 02:27:08.903 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.236412% / 7.94729e-06% / 6.75519e-05% | 20260507 02:27:08.903 | INFO | Baz IO overall: | 20260507 02:27:08.903 | INFO | BytesWritten : 3287219426056 | 20260507 02:27:08.903 | INFO | NumEvents : 1580889307011 | 20260507 02:27:10.752 | INFO | Analyzing chunk frames = [6369280,6369792) | 20260507 02:27:10.752 | INFO | MemInfo: { | 20260507 02:27:10.752 | INFO | "gpu" : " 17 GB", | 20260507 02:27:10.752 | INFO | "huge" : "", | 20260507 02:27:10.752 | INFO | "huge_shared" : " 27 GB", | 20260507 02:27:10.752 | INFO | "malloc" : " 14 GB", | 20260507 02:27:10.752 | INFO | "pinned" : " 29 GB" | 20260507 02:27:10.752 | INFO | } | 20260507 02:27:10.752 | INFO | Perf report for the last 5.14057 seconds: | 20260507 02:27:10.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:27:10.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:10.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:10.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:10.752 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 02:27:10.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:10.752 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 02:27:10.752 | INFO | PRE_HQ : 59.3%, 1.8, 55.7ms, 0 | 20260507 02:27:10.752 | INFO | BAZWRITER : 61.7%, 3.3, 103.8ms, 0 | 20260507 02:27:10.752 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:27:10.752 | INFO | Uploaded 36GiB to GPU in 2.03227s (17.7142GiB/s) | 20260507 02:27:10.752 | INFO | Downloaded 38GiB to GPU in 2.1896s (17.3548GiB/s) | 20260507 02:27:15.892 | INFO | Analyzing chunk frames = [6369792,6370304) | 20260507 02:27:15.892 | INFO | MemInfo: { | 20260507 02:27:15.892 | INFO | "gpu" : " 17 GB", | 20260507 02:27:15.892 | INFO | "huge" : "", | 20260507 02:27:15.892 | INFO | "huge_shared" : " 27 GB", | 20260507 02:27:15.892 | INFO | "malloc" : " 14 GB", | 20260507 02:27:15.892 | INFO | "pinned" : " 29 GB" | 20260507 02:27:15.892 | INFO | } | 20260507 02:27:15.893 | INFO | Perf report for the last 5.14075 seconds: | 20260507 02:27:15.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:27:15.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:15.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:15.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:15.893 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:27:15.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:15.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:15.893 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:27:15.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:15.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:15.893 | INFO | Uploaded 36GiB to GPU in 2.03218s (17.715GiB/s) | 20260507 02:27:15.893 | INFO | Downloaded 34GiB to GPU in 1.93995s (17.5262GiB/s) | 20260507 02:27:21.031 | INFO | Analyzing chunk frames = [6370304,6370816) | 20260507 02:27:21.031 | INFO | MemInfo: { | 20260507 02:27:21.031 | INFO | "gpu" : " 17 GB", | 20260507 02:27:21.031 | INFO | "huge" : "", | 20260507 02:27:21.031 | INFO | "huge_shared" : " 27 GB", | 20260507 02:27:21.031 | INFO | "malloc" : " 14 GB", | 20260507 02:27:21.031 | INFO | "pinned" : " 29 GB" | 20260507 02:27:21.031 | INFO | } | 20260507 02:27:21.031 | INFO | Perf report for the last 5.13837 seconds: | 20260507 02:27:21.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:27:21.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:21.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:21.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:21.031 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 02:27:21.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:21.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:21.031 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:27:21.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:21.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:21.031 | INFO | Uploaded 36GiB to GPU in 1.9967s (18.0297GiB/s) | 20260507 02:27:21.031 | INFO | Downloaded 34GiB to GPU in 1.93805s (17.5434GiB/s) | 20260507 02:27:26.170 | INFO | Analyzing chunk frames = [6370816,6371328) | 20260507 02:27:26.170 | INFO | MemInfo: { | 20260507 02:27:26.170 | INFO | "gpu" : " 17 GB", | 20260507 02:27:26.170 | INFO | "huge" : "", | 20260507 02:27:26.170 | INFO | "huge_shared" : " 27 GB", | 20260507 02:27:26.170 | INFO | "malloc" : " 14 GB", | 20260507 02:27:26.170 | INFO | "pinned" : " 29 GB" | 20260507 02:27:26.170 | INFO | } | 20260507 02:27:26.170 | INFO | Perf report for the last 5.1393 seconds: | 20260507 02:27:26.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:27:26.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:26.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:26.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:26.170 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 02:27:26.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:26.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:26.170 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:27:26.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:26.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:26.170 | INFO | Uploaded 36GiB to GPU in 1.97607s (18.218GiB/s) | 20260507 02:27:26.171 | INFO | Downloaded 34GiB to GPU in 1.90325s (17.8641GiB/s) | 20260507 02:27:31.309 | INFO | Analyzing chunk frames = [6371328,6371840) | 20260507 02:27:31.309 | INFO | MemInfo: { | 20260507 02:27:31.309 | INFO | "gpu" : " 17 GB", | 20260507 02:27:31.309 | INFO | "huge" : "", | 20260507 02:27:31.309 | INFO | "huge_shared" : " 27 GB", | 20260507 02:27:31.309 | INFO | "malloc" : " 14 GB", | 20260507 02:27:31.309 | INFO | "pinned" : " 29 GB" | 20260507 02:27:31.309 | INFO | } | 20260507 02:27:31.309 | INFO | Perf report for the last 5.13895 seconds: | 20260507 02:27:31.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:27:31.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:31.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:31.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:31.309 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 02:27:31.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:31.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:31.309 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:27:31.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:31.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:31.310 | INFO | Uploaded 36GiB to GPU in 2.00177s (17.9841GiB/s) | 20260507 02:27:31.310 | INFO | Downloaded 34GiB to GPU in 1.95385s (17.4015GiB/s) | 20260507 02:27:36.448 | INFO | Analyzing chunk frames = [6371840,6372352) | 20260507 02:27:36.448 | INFO | MemInfo: { | 20260507 02:27:36.448 | INFO | "gpu" : " 17 GB", | 20260507 02:27:36.448 | INFO | "huge" : "", | 20260507 02:27:36.448 | INFO | "huge_shared" : " 27 GB", | 20260507 02:27:36.448 | INFO | "malloc" : " 14 GB", | 20260507 02:27:36.448 | INFO | "pinned" : " 29 GB" | 20260507 02:27:36.448 | INFO | } | 20260507 02:27:36.448 | INFO | Perf report for the last 5.13898 seconds: | 20260507 02:27:36.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:27:36.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:36.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:36.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:36.448 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 02:27:36.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:36.448 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:36.448 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:27:36.448 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:36.448 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:36.448 | INFO | Uploaded 36GiB to GPU in 2.0494s (17.5662GiB/s) | 20260507 02:27:36.449 | INFO | Downloaded 34GiB to GPU in 1.96597s (17.2943GiB/s) | 20260507 02:27:41.586 | INFO | Analyzing chunk frames = [6372352,6372864) | 20260507 02:27:41.586 | INFO | MemInfo: { | 20260507 02:27:41.586 | INFO | "gpu" : " 17 GB", | 20260507 02:27:41.586 | INFO | "huge" : "", | 20260507 02:27:41.586 | INFO | "huge_shared" : " 27 GB", | 20260507 02:27:41.586 | INFO | "malloc" : " 14 GB", | 20260507 02:27:41.586 | INFO | "pinned" : " 29 GB" | 20260507 02:27:41.586 | INFO | } | 20260507 02:27:41.586 | INFO | Perf report for the last 5.13783 seconds: | 20260507 02:27:41.586 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:27:41.586 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:41.586 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:41.586 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:41.586 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:27:41.586 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:41.586 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:41.586 | INFO | PRE_HQ : 29.8%, 1.0, 15.5ms, 0 | 20260507 02:27:41.586 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:41.586 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:41.586 | INFO | Uploaded 36GiB to GPU in 1.98109s (18.1719GiB/s) | 20260507 02:27:41.586 | INFO | Downloaded 34GiB to GPU in 2.00196s (16.9834GiB/s) | 20260507 02:27:46.725 | INFO | Analyzing chunk frames = [6372864,6373376) | 20260507 02:27:46.725 | INFO | MemInfo: { | 20260507 02:27:46.725 | INFO | "gpu" : " 17 GB", | 20260507 02:27:46.725 | INFO | "huge" : "", | 20260507 02:27:46.725 | INFO | "huge_shared" : " 27 GB", | 20260507 02:27:46.725 | INFO | "malloc" : " 14 GB", | 20260507 02:27:46.725 | INFO | "pinned" : " 29 GB" | 20260507 02:27:46.725 | INFO | } | 20260507 02:27:46.725 | INFO | Perf report for the last 5.1391 seconds: | 20260507 02:27:46.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:27:46.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:46.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:46.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:46.725 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:27:46.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:46.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:46.725 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:27:46.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:46.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:46.725 | INFO | Uploaded 36GiB to GPU in 2.00696s (17.9376GiB/s) | 20260507 02:27:46.726 | INFO | Downloaded 34GiB to GPU in 1.97476s (17.2173GiB/s) | 20260507 02:27:46.726 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:27:46.726 | INFO | Total mem usage: 18 GiB | 20260507 02:27:46.726 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:27:46.726 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:27:46.726 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:27:46.726 | INFO | High Water Breakdown: | 20260507 02:27:46.726 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:27:46.726 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:27:46.726 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:27:46.726 | INFO | ------------------------------------------ | 20260507 02:27:46.726 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:27:46.726 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:27:46.726 | INFO | Total mem usage: 26.5 GiB | 20260507 02:27:46.726 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:27:46.726 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:27:46.726 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:27:46.726 | INFO | High Water Breakdown: | 20260507 02:27:46.726 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:27:46.726 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:27:46.726 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:27:46.726 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:27:46.726 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:27:46.726 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:27:46.726 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:27:46.726 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:27:46.726 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:27:46.726 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:27:46.726 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:27:46.726 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:27:46.726 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:27:46.726 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:27:46.726 | INFO | ------------------------------------------ | 20260507 02:27:46.726 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:27:46.726 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:27:46.726 | INFO | Total mem usage: 25.6 GiB | 20260507 02:27:46.726 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:27:46.726 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:27:46.726 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:27:46.726 | INFO | High Water Breakdown: | 20260507 02:27:46.726 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:27:46.726 | INFO | ------------------------------------------ | 20260507 02:27:46.726 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:27:46.726 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:27:46.726 | INFO | Total mem usage: 15.7 GiB | 20260507 02:27:46.726 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:27:46.726 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:27:46.726 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:27:46.726 | INFO | High Water Breakdown: | 20260507 02:27:46.726 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:27:46.726 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:27:46.726 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:27:46.726 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:27:46.726 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:27:46.726 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:27:46.726 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:27:46.726 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:27:46.726 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:27:46.726 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:27:46.726 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:27:46.726 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:27:46.726 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:27:46.726 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:27:46.726 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:27:46.726 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:27:46.726 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:27:46.726 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:27:46.726 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:27:46.726 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:27:46.726 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:27:46.726 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:27:46.726 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:27:46.726 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:27:46.726 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:27:46.726 | INFO | ------------------------------------------ | 20260507 02:27:46.726 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:27:46.726 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:27:46.726 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:27:46.726 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:27:46.726 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.53e+01, 1.33e+01 / 2.26e+00 | 20260507 02:27:46.726 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.81e-03 / 5.24e-03 | 20260507 02:27:46.726 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.45e+00, 5.22e-01 / 2.61e-01 | 20260507 02:27:46.726 | INFO | PerfReport: 3.24%, Histogram -- 9.97e-01 / 1.48e+00, 1.72e-01 / 1.20e-01 | 20260507 02:27:46.726 | INFO | PerfReport: 16.10%, DME -- 3.97e+01 / 4.42e+01, 3.95e+00 / 2.80e+00 | 20260507 02:27:46.726 | INFO | PerfReport: 57.58%, FrameLabeling -- 1.77e+01 / 2.08e+01, 1.28e+00 / 2.10e+00 | 20260507 02:27:46.726 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.16e+00 / 2.67e+00, 3.83e-01 / 2.55e-01 | 20260507 02:27:46.726 | INFO | PerfReport: 3.09%, Metrics -- 9.50e-01 / 1.28e+00, 2.53e-01 / 1.58e-01 | 20260507 02:27:46.726 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.21e-01 / 3.71e-01, 5.00e-02 / 2.13e-02 | 20260507 02:27:46.726 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:27:46.726 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:27:46.726 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:27:46.726 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260507 02:27:46.726 | INFO | PerfReport: 21.70%, Upload -- 2.09e+01 / 0.00e+00, 7.04e+00 / 0.00e+00 | 20260507 02:27:46.726 | INFO | PerfReport: 21.45%, Download -- 2.07e+01 / 0.00e+00, 7.50e+00 / 0.00e+00 | 20260507 02:27:46.726 | INFO | PerfReport: 31.97%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 02:27:46.726 | INFO | PerfReport: 1.32%, LockUpload -- 1.27e+00 / 0.00e+00, 4.86e+00 / 0.00e+00 | 20260507 02:27:46.726 | INFO | PerfReport: 1.36%, LockDownload -- 1.31e+00 / 0.00e+00, 3.18e+00 / 0.00e+00 | 20260507 02:27:46.726 | INFO | PerfReport: 22.19%, LockCompute -- 2.14e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 02:27:46.726 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260507 02:27:46.822 | INFO | Starting a round of Baz IO | 20260507 02:27:49.931 | INFO | Populating RTMetrics report | 20260507 02:27:49.931 | INFO | RTMetrics report populated | 20260507 02:27:49.931 | INFO | Opening tmp RTMetrics json file | 20260507 02:27:49.933 | INFO | Serializing RTMetrics to json | 20260507 02:27:49.949 | INFO | Serialization done, swapping tmp file now... | 20260507 02:27:49.951 | INFO | RTMetrics done | 20260507 02:27:50.017 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.194 seconds | 20260507 02:27:50.017 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.00% / 0.00% / 67.00% | 20260507 02:27:50.017 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.236483% / 0% / 7.15256e-05% | 20260507 02:27:50.017 | INFO | Baz IO overall: | 20260507 02:27:50.017 | INFO | BytesWritten : 3289130731548 | 20260507 02:27:50.017 | INFO | NumEvents : 1581792923853 | 20260507 02:27:51.864 | INFO | Analyzing chunk frames = [6373376,6373888) | 20260507 02:27:51.864 | INFO | MemInfo: { | 20260507 02:27:51.864 | INFO | "gpu" : " 17 GB", | 20260507 02:27:51.864 | INFO | "huge" : "", | 20260507 02:27:51.864 | INFO | "huge_shared" : " 27 GB", | 20260507 02:27:51.864 | INFO | "malloc" : " 14 GB", | 20260507 02:27:51.864 | INFO | "pinned" : " 29 GB" | 20260507 02:27:51.864 | INFO | } | 20260507 02:27:51.864 | INFO | Perf report for the last 5.13902 seconds: | 20260507 02:27:51.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:27:51.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:51.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:51.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:51.864 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 02:27:51.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:51.864 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 02:27:51.864 | INFO | PRE_HQ : 59.2%, 1.8, 55.6ms, 0 | 20260507 02:27:51.864 | INFO | BAZWRITER : 61.8%, 3.3, 104.1ms, 0 | 20260507 02:27:51.864 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:27:51.864 | INFO | Uploaded 36GiB to GPU in 2.04385s (17.6138GiB/s) | 20260507 02:27:51.864 | INFO | Downloaded 38GiB to GPU in 2.18418s (17.3978GiB/s) | 20260507 02:27:56.999 | INFO | Analyzing chunk frames = [6373888,6374400) | 20260507 02:27:56.999 | INFO | MemInfo: { | 20260507 02:27:56.999 | INFO | "gpu" : " 17 GB", | 20260507 02:27:56.999 | INFO | "huge" : "", | 20260507 02:27:57.000 | INFO | "huge_shared" : " 27 GB", | 20260507 02:27:57.000 | INFO | "malloc" : " 14 GB", | 20260507 02:27:57.000 | INFO | "pinned" : " 29 GB" | 20260507 02:27:57.000 | INFO | } | 20260507 02:27:57.000 | INFO | Perf report for the last 5.13539 seconds: | 20260507 02:27:57.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:27:57.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:57.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:57.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:57.000 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:27:57.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:57.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:57.000 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:27:57.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:57.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:27:57.000 | INFO | Uploaded 36GiB to GPU in 2.03845s (17.6605GiB/s) | 20260507 02:27:57.000 | INFO | Downloaded 34GiB to GPU in 1.93751s (17.5483GiB/s) | 20260507 02:28:02.142 | INFO | Analyzing chunk frames = [6374400,6374912) | 20260507 02:28:02.142 | INFO | MemInfo: { | 20260507 02:28:02.142 | INFO | "gpu" : " 17 GB", | 20260507 02:28:02.142 | INFO | "huge" : "", | 20260507 02:28:02.142 | INFO | "huge_shared" : " 27 GB", | 20260507 02:28:02.143 | INFO | "malloc" : " 14 GB", | 20260507 02:28:02.143 | INFO | "pinned" : " 29 GB" | 20260507 02:28:02.143 | INFO | } | 20260507 02:28:02.143 | INFO | Perf report for the last 5.14298 seconds: | 20260507 02:28:02.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:28:02.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:02.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:02.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:02.143 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:28:02.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:02.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:02.143 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:28:02.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:02.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:02.143 | INFO | Uploaded 36GiB to GPU in 1.99131s (18.0785GiB/s) | 20260507 02:28:02.143 | INFO | Downloaded 34GiB to GPU in 1.93308s (17.5886GiB/s) | 20260507 02:28:07.285 | INFO | Analyzing chunk frames = [6374912,6375424) | 20260507 02:28:07.285 | INFO | MemInfo: { | 20260507 02:28:07.285 | INFO | "gpu" : " 17 GB", | 20260507 02:28:07.285 | INFO | "huge" : "", | 20260507 02:28:07.285 | INFO | "huge_shared" : " 27 GB", | 20260507 02:28:07.285 | INFO | "malloc" : " 14 GB", | 20260507 02:28:07.285 | INFO | "pinned" : " 29 GB" | 20260507 02:28:07.285 | INFO | } | 20260507 02:28:07.285 | INFO | Perf report for the last 5.14219 seconds: | 20260507 02:28:07.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:28:07.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:07.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:07.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:07.285 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260507 02:28:07.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:07.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:07.285 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:28:07.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:07.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:07.285 | INFO | Uploaded 36GiB to GPU in 2.0127s (17.8864GiB/s) | 20260507 02:28:07.285 | INFO | Downloaded 34GiB to GPU in 1.90934s (17.8072GiB/s) | 20260507 02:28:12.424 | INFO | Analyzing chunk frames = [6375424,6375936) | 20260507 02:28:12.424 | INFO | MemInfo: { | 20260507 02:28:12.424 | INFO | "gpu" : " 17 GB", | 20260507 02:28:12.424 | INFO | "huge" : "", | 20260507 02:28:12.424 | INFO | "huge_shared" : " 27 GB", | 20260507 02:28:12.424 | INFO | "malloc" : " 14 GB", | 20260507 02:28:12.424 | INFO | "pinned" : " 29 GB" | 20260507 02:28:12.424 | INFO | } | 20260507 02:28:12.424 | INFO | Perf report for the last 5.1397 seconds: | 20260507 02:28:12.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:28:12.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:12.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:12.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:12.425 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 02:28:12.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:12.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:12.425 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:28:12.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:12.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:12.425 | INFO | Uploaded 36GiB to GPU in 2.0104s (17.9069GiB/s) | 20260507 02:28:12.425 | INFO | Downloaded 34GiB to GPU in 1.95009s (17.4351GiB/s) | 20260507 02:28:17.562 | INFO | Analyzing chunk frames = [6375936,6376448) | 20260507 02:28:17.562 | INFO | MemInfo: { | 20260507 02:28:17.562 | INFO | "gpu" : " 17 GB", | 20260507 02:28:17.562 | INFO | "huge" : "", | 20260507 02:28:17.562 | INFO | "huge_shared" : " 27 GB", | 20260507 02:28:17.562 | INFO | "malloc" : " 14 GB", | 20260507 02:28:17.562 | INFO | "pinned" : " 29 GB" | 20260507 02:28:17.562 | INFO | } | 20260507 02:28:17.562 | INFO | Perf report for the last 5.13791 seconds: | 20260507 02:28:17.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:28:17.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:17.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:17.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:17.562 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:28:17.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:17.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:17.563 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:28:17.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:17.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:17.563 | INFO | Uploaded 36GiB to GPU in 2.05908s (17.4836GiB/s) | 20260507 02:28:17.563 | INFO | Downloaded 34GiB to GPU in 1.96247s (17.3251GiB/s) | 20260507 02:28:22.701 | INFO | Analyzing chunk frames = [6376448,6376960) | 20260507 02:28:22.701 | INFO | MemInfo: { | 20260507 02:28:22.701 | INFO | "gpu" : " 17 GB", | 20260507 02:28:22.701 | INFO | "huge" : "", | 20260507 02:28:22.701 | INFO | "huge_shared" : " 27 GB", | 20260507 02:28:22.701 | INFO | "malloc" : " 14 GB", | 20260507 02:28:22.701 | INFO | "pinned" : " 29 GB" | 20260507 02:28:22.701 | INFO | } | 20260507 02:28:22.701 | INFO | Perf report for the last 5.13891 seconds: | 20260507 02:28:22.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:28:22.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:22.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:22.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:22.701 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:28:22.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:22.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:22.701 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 02:28:22.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:22.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:22.702 | INFO | Uploaded 36GiB to GPU in 1.9746s (18.2316GiB/s) | 20260507 02:28:22.702 | INFO | Downloaded 34GiB to GPU in 1.95129s (17.4244GiB/s) | 20260507 02:28:27.838 | INFO | Analyzing chunk frames = [6376960,6377472) | 20260507 02:28:27.838 | INFO | MemInfo: { | 20260507 02:28:27.838 | INFO | "gpu" : " 17 GB", | 20260507 02:28:27.838 | INFO | "huge" : "", | 20260507 02:28:27.838 | INFO | "huge_shared" : " 27 GB", | 20260507 02:28:27.838 | INFO | "malloc" : " 14 GB", | 20260507 02:28:27.838 | INFO | "pinned" : " 29 GB" | 20260507 02:28:27.838 | INFO | } | 20260507 02:28:27.838 | INFO | Perf report for the last 5.13681 seconds: | 20260507 02:28:27.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:28:27.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:27.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:27.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:27.838 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:28:27.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:27.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:27.838 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:28:27.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:27.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:27.839 | INFO | Uploaded 36GiB to GPU in 1.98025s (18.1795GiB/s) | 20260507 02:28:27.839 | INFO | Downloaded 34GiB to GPU in 1.9538s (17.402GiB/s) | 20260507 02:28:27.935 | INFO | Starting a round of Baz IO | 20260507 02:28:31.052 | INFO | Populating RTMetrics report | 20260507 02:28:31.053 | INFO | RTMetrics report populated | 20260507 02:28:31.053 | INFO | Opening tmp RTMetrics json file | 20260507 02:28:31.055 | INFO | Serializing RTMetrics to json | 20260507 02:28:31.072 | INFO | Serialization done, swapping tmp file now... | 20260507 02:28:31.072 | INFO | RTMetrics done | 20260507 02:28:31.142 | INFO | Finished a round of Baz IO: Wrote 1.779 GiB in 3.207 seconds | 20260507 02:28:31.142 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.00% / 0.00% / 67.00% | 20260507 02:28:31.142 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.235828% / 0% / 8.34465e-05% | 20260507 02:28:31.142 | INFO | Baz IO overall: | 20260507 02:28:31.142 | INFO | BytesWritten : 3291040948624 | 20260507 02:28:31.142 | INFO | NumEvents : 1582695813410 | 20260507 02:28:32.983 | INFO | Analyzing chunk frames = [6377472,6377984) | 20260507 02:28:32.983 | INFO | MemInfo: { | 20260507 02:28:32.983 | INFO | "gpu" : " 17 GB", | 20260507 02:28:32.983 | INFO | "huge" : "", | 20260507 02:28:32.983 | INFO | "huge_shared" : " 27 GB", | 20260507 02:28:32.983 | INFO | "malloc" : " 14 GB", | 20260507 02:28:32.983 | INFO | "pinned" : " 29 GB" | 20260507 02:28:32.983 | INFO | } | 20260507 02:28:32.983 | INFO | Perf report for the last 5.14453 seconds: | 20260507 02:28:32.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:28:32.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:32.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:32.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:32.983 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260507 02:28:32.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:32.983 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 02:28:32.983 | INFO | PRE_HQ : 59.4%, 1.8, 55.7ms, 0 | 20260507 02:28:32.983 | INFO | BAZWRITER : 61.9%, 3.3, 104.4ms, 0 | 20260507 02:28:32.983 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:28:32.983 | INFO | Uploaded 36GiB to GPU in 2.04452s (17.608GiB/s) | 20260507 02:28:32.983 | INFO | Downloaded 38GiB to GPU in 2.18544s (17.3878GiB/s) | 20260507 02:28:38.125 | INFO | Analyzing chunk frames = [6377984,6378496) | 20260507 02:28:38.125 | INFO | MemInfo: { | 20260507 02:28:38.125 | INFO | "gpu" : " 17 GB", | 20260507 02:28:38.125 | INFO | "huge" : "", | 20260507 02:28:38.125 | INFO | "huge_shared" : " 27 GB", | 20260507 02:28:38.125 | INFO | "malloc" : " 14 GB", | 20260507 02:28:38.125 | INFO | "pinned" : " 29 GB" | 20260507 02:28:38.125 | INFO | } | 20260507 02:28:38.125 | INFO | Perf report for the last 5.14199 seconds: | 20260507 02:28:38.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:28:38.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:38.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:38.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:38.125 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:28:38.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:38.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:38.125 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:28:38.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:38.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:38.125 | INFO | Uploaded 36GiB to GPU in 2.04215s (17.6285GiB/s) | 20260507 02:28:38.125 | INFO | Downloaded 34GiB to GPU in 1.93418s (17.5785GiB/s) | 20260507 02:28:43.257 | INFO | Analyzing chunk frames = [6378496,6379008) | 20260507 02:28:43.257 | INFO | MemInfo: { | 20260507 02:28:43.257 | INFO | "gpu" : " 17 GB", | 20260507 02:28:43.257 | INFO | "huge" : "", | 20260507 02:28:43.257 | INFO | "huge_shared" : " 27 GB", | 20260507 02:28:43.257 | INFO | "malloc" : " 14 GB", | 20260507 02:28:43.257 | INFO | "pinned" : " 29 GB" | 20260507 02:28:43.257 | INFO | } | 20260507 02:28:43.257 | INFO | Perf report for the last 5.13272 seconds: | 20260507 02:28:43.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:28:43.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:43.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:43.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:43.257 | INFO | ANALYSIS : 60.4%, 3.0, 93.3ms, 0 | 20260507 02:28:43.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:43.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:43.257 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 02:28:43.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:43.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:43.258 | INFO | Uploaded 36GiB to GPU in 2.01166s (17.8956GiB/s) | 20260507 02:28:43.258 | INFO | Downloaded 34GiB to GPU in 1.93864s (17.5381GiB/s) | 20260507 02:28:48.404 | INFO | Analyzing chunk frames = [6379008,6379520) | 20260507 02:28:48.404 | INFO | MemInfo: { | 20260507 02:28:48.404 | INFO | "gpu" : " 17 GB", | 20260507 02:28:48.404 | INFO | "huge" : "", | 20260507 02:28:48.404 | INFO | "huge_shared" : " 27 GB", | 20260507 02:28:48.404 | INFO | "malloc" : " 14 GB", | 20260507 02:28:48.404 | INFO | "pinned" : " 29 GB" | 20260507 02:28:48.404 | INFO | } | 20260507 02:28:48.404 | INFO | Perf report for the last 5.14641 seconds: | 20260507 02:28:48.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:28:48.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:48.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:48.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:48.404 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260507 02:28:48.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:48.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:48.404 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:28:48.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:48.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:48.404 | INFO | Uploaded 36GiB to GPU in 2.01151s (17.897GiB/s) | 20260507 02:28:48.404 | INFO | Downloaded 34GiB to GPU in 1.90905s (17.8099GiB/s) | 20260507 02:28:53.542 | INFO | Analyzing chunk frames = [6379520,6380032) | 20260507 02:28:53.542 | INFO | MemInfo: { | 20260507 02:28:53.542 | INFO | "gpu" : " 17 GB", | 20260507 02:28:53.542 | INFO | "huge" : "", | 20260507 02:28:53.542 | INFO | "huge_shared" : " 27 GB", | 20260507 02:28:53.542 | INFO | "malloc" : " 14 GB", | 20260507 02:28:53.542 | INFO | "pinned" : " 29 GB" | 20260507 02:28:53.542 | INFO | } | 20260507 02:28:53.542 | INFO | Perf report for the last 5.13801 seconds: | 20260507 02:28:53.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:28:53.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:53.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:53.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:53.542 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 02:28:53.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:53.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:53.542 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:28:53.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:53.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:53.542 | INFO | Uploaded 36GiB to GPU in 2.02576s (17.7711GiB/s) | 20260507 02:28:53.542 | INFO | Downloaded 34GiB to GPU in 1.93231s (17.5955GiB/s) | 20260507 02:28:58.674 | INFO | Analyzing chunk frames = [6380032,6380544) | 20260507 02:28:58.674 | INFO | MemInfo: { | 20260507 02:28:58.674 | INFO | "gpu" : " 17 GB", | 20260507 02:28:58.674 | INFO | "huge" : "", | 20260507 02:28:58.674 | INFO | "huge_shared" : " 27 GB", | 20260507 02:28:58.674 | INFO | "malloc" : " 14 GB", | 20260507 02:28:58.674 | INFO | "pinned" : " 29 GB" | 20260507 02:28:58.674 | INFO | } | 20260507 02:28:58.674 | INFO | Perf report for the last 5.13232 seconds: | 20260507 02:28:58.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:28:58.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:58.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:58.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:58.674 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 02:28:58.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:58.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:58.674 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 02:28:58.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:58.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:28:58.675 | INFO | Uploaded 36GiB to GPU in 2.03877s (17.6577GiB/s) | 20260507 02:28:58.675 | INFO | Downloaded 34GiB to GPU in 1.92132s (17.6962GiB/s) | 20260507 02:29:03.819 | INFO | Analyzing chunk frames = [6380544,6381056) | 20260507 02:29:03.819 | INFO | MemInfo: { | 20260507 02:29:03.819 | INFO | "gpu" : " 17 GB", | 20260507 02:29:03.819 | INFO | "huge" : "", | 20260507 02:29:03.819 | INFO | "huge_shared" : " 27 GB", | 20260507 02:29:03.819 | INFO | "malloc" : " 14 GB", | 20260507 02:29:03.819 | INFO | "pinned" : " 29 GB" | 20260507 02:29:03.819 | INFO | } | 20260507 02:29:03.819 | INFO | Perf report for the last 5.14483 seconds: | 20260507 02:29:03.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:29:03.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:03.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:03.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:03.819 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 02:29:03.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:03.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:03.819 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 02:29:03.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:03.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:03.819 | INFO | Uploaded 36GiB to GPU in 1.96494s (18.3212GiB/s) | 20260507 02:29:03.819 | INFO | Downloaded 34GiB to GPU in 1.95584s (17.3838GiB/s) | 20260507 02:29:08.952 | INFO | Analyzing chunk frames = [6381056,6381568) | 20260507 02:29:08.952 | INFO | MemInfo: { | 20260507 02:29:08.952 | INFO | "gpu" : " 17 GB", | 20260507 02:29:08.952 | INFO | "huge" : "", | 20260507 02:29:08.952 | INFO | "huge_shared" : " 27 GB", | 20260507 02:29:08.952 | INFO | "malloc" : " 14 GB", | 20260507 02:29:08.952 | INFO | "pinned" : " 29 GB" | 20260507 02:29:08.952 | INFO | } | 20260507 02:29:08.952 | INFO | Perf report for the last 5.13336 seconds: | 20260507 02:29:08.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:29:08.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:08.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:08.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:08.952 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 02:29:08.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:08.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:08.952 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 02:29:08.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:08.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:08.953 | INFO | Uploaded 36GiB to GPU in 1.96972s (18.2767GiB/s) | 20260507 02:29:08.953 | INFO | Downloaded 34GiB to GPU in 1.95502s (17.3912GiB/s) | 20260507 02:29:08.953 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:29:08.953 | INFO | Total mem usage: 18 GiB | 20260507 02:29:08.953 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:29:08.953 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:29:08.953 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:29:08.953 | INFO | High Water Breakdown: | 20260507 02:29:08.953 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:29:08.953 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:29:08.953 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:29:08.953 | INFO | ------------------------------------------ | 20260507 02:29:08.953 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:29:08.953 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:29:08.953 | INFO | Total mem usage: 26.5 GiB | 20260507 02:29:08.953 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:29:08.953 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:29:08.953 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:29:08.953 | INFO | High Water Breakdown: | 20260507 02:29:08.953 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:29:08.953 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:29:08.953 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:29:08.953 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:29:08.953 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:29:08.953 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:29:08.953 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:29:08.953 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:29:08.953 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:29:08.953 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:29:08.953 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:29:08.953 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:29:08.953 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:29:08.953 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:29:08.953 | INFO | ------------------------------------------ | 20260507 02:29:08.953 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:29:08.953 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:29:08.953 | INFO | Total mem usage: 25.6 GiB | 20260507 02:29:08.953 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:29:08.953 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 02:29:08.953 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 02:29:08.953 | INFO | High Water Breakdown: | 20260507 02:29:08.953 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 02:29:08.953 | INFO | ------------------------------------------ | 20260507 02:29:08.953 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:29:08.953 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:29:08.953 | INFO | Total mem usage: 15.7 GiB | 20260507 02:29:08.953 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:29:08.953 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:29:08.953 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 02:29:08.953 | INFO | High Water Breakdown: | 20260507 02:29:08.953 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:29:08.953 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:29:08.953 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:29:08.953 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:29:08.953 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:29:08.953 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:29:08.953 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:29:08.953 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:29:08.953 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:29:08.953 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:29:08.953 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:29:08.953 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:29:08.953 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:29:08.953 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:29:08.953 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:29:08.953 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:29:08.953 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:29:08.953 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:29:08.953 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:29:08.953 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:29:08.953 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:29:08.953 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:29:08.953 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:29:08.953 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:29:08.953 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:29:08.953 | INFO | ------------------------------------------ | 20260507 02:29:08.953 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:29:08.954 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:29:08.954 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:29:08.954 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:29:08.954 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.48e+01, 1.33e+01 / 1.26e+00 | 20260507 02:29:08.954 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.00e-01, 9.22e-03 / 8.14e-03 | 20260507 02:29:08.954 | INFO | PerfReport: 11.65%, Baseline -- 3.59e+00 / 4.33e+00, 5.18e-01 / 3.55e-01 | 20260507 02:29:08.954 | INFO | PerfReport: 3.23%, Histogram -- 9.96e-01 / 1.45e+00, 1.72e-01 / 1.27e-01 | 20260507 02:29:08.954 | INFO | PerfReport: 16.05%, DME -- 3.96e+01 / 4.40e+01, 3.72e+00 / 1.70e+00 | 20260507 02:29:08.954 | INFO | PerfReport: 57.69%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.38e+00 / 1.87e+00 | 20260507 02:29:08.954 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.15e+00 / 2.70e+00, 3.76e-01 / 2.20e-01 | 20260507 02:29:08.954 | INFO | PerfReport: 3.07%, Metrics -- 9.46e-01 / 1.25e+00, 2.54e-01 / 1.31e-01 | 20260507 02:29:08.954 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.20e-01 / 3.68e-01, 5.08e-02 / 2.07e-02 | 20260507 02:29:08.954 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:29:08.954 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:29:08.954 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:29:08.954 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 02:29:08.954 | INFO | PerfReport: 21.77%, Upload -- 2.10e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260507 02:29:08.954 | INFO | PerfReport: 21.30%, Download -- 2.05e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 02:29:08.954 | INFO | PerfReport: 31.98%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 02:29:08.954 | INFO | PerfReport: 1.51%, LockUpload -- 1.46e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 02:29:08.954 | INFO | PerfReport: 1.19%, LockDownload -- 1.15e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260507 02:29:08.954 | INFO | PerfReport: 22.23%, LockCompute -- 2.14e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 02:29:08.954 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.34e-02 / 0.00e+00 | 20260507 02:29:09.049 | INFO | Starting a round of Baz IO | 20260507 02:29:12.163 | INFO | Populating RTMetrics report | 20260507 02:29:12.163 | INFO | RTMetrics report populated | 20260507 02:29:12.163 | INFO | Opening tmp RTMetrics json file | 20260507 02:29:12.181 | INFO | Serializing RTMetrics to json | 20260507 02:29:12.182 | INFO | Serialization done, swapping tmp file now... | 20260507 02:29:12.183 | INFO | RTMetrics done | 20260507 02:29:12.251 | INFO | Finished a round of Baz IO: Wrote 1.779 GiB in 3.201 seconds | 20260507 02:29:12.251 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.00% / 0.00% / 67.00% | 20260507 02:29:12.251 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.23526% / 0% / 9.53674e-05% | 20260507 02:29:12.251 | INFO | Baz IO overall: | 20260507 02:29:12.251 | INFO | BytesWritten : 3292950648859 | 20260507 02:29:12.251 | INFO | NumEvents : 1583598332233 | 20260507 02:29:14.094 | INFO | Analyzing chunk frames = [6381568,6382080) | 20260507 02:29:14.094 | INFO | MemInfo: { | 20260507 02:29:14.094 | INFO | "gpu" : " 17 GB", | 20260507 02:29:14.094 | INFO | "huge" : "", | 20260507 02:29:14.095 | INFO | "huge_shared" : " 27 GB", | 20260507 02:29:14.095 | INFO | "malloc" : " 14 GB", | 20260507 02:29:14.095 | INFO | "pinned" : " 29 GB" | 20260507 02:29:14.095 | INFO | } | 20260507 02:29:14.095 | INFO | Perf report for the last 5.14217 seconds: | 20260507 02:29:14.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:29:14.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:14.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:14.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:14.095 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260507 02:29:14.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:14.095 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 02:29:14.095 | INFO | PRE_HQ : 59.3%, 1.8, 55.4ms, 0 | 20260507 02:29:14.095 | INFO | BAZWRITER : 61.8%, 3.3, 103.7ms, 0 | 20260507 02:29:14.095 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:29:14.095 | INFO | Uploaded 36GiB to GPU in 2.02155s (17.8081GiB/s) | 20260507 02:29:14.095 | INFO | Downloaded 38GiB to GPU in 2.17199s (17.4954GiB/s) | 20260507 02:29:19.234 | INFO | Analyzing chunk frames = [6382080,6382592) | 20260507 02:29:19.234 | INFO | MemInfo: { | 20260507 02:29:19.234 | INFO | "gpu" : " 17 GB", | 20260507 02:29:19.234 | INFO | "huge" : "", | 20260507 02:29:19.234 | INFO | "huge_shared" : " 27 GB", | 20260507 02:29:19.234 | INFO | "malloc" : " 14 GB", | 20260507 02:29:19.234 | INFO | "pinned" : " 29 GB" | 20260507 02:29:19.234 | INFO | } | 20260507 02:29:19.234 | INFO | Perf report for the last 5.13961 seconds: | 20260507 02:29:19.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:29:19.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:19.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:19.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:19.234 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:29:19.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:19.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:19.234 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:29:19.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:19.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:19.234 | INFO | Uploaded 36GiB to GPU in 2.01329s (17.8812GiB/s) | 20260507 02:29:19.234 | INFO | Downloaded 34GiB to GPU in 1.93137s (17.6041GiB/s) | 20260507 02:29:24.376 | INFO | Analyzing chunk frames = [6382592,6383104) | 20260507 02:29:24.376 | INFO | MemInfo: { | 20260507 02:29:24.376 | INFO | "gpu" : " 17 GB", | 20260507 02:29:24.376 | INFO | "huge" : "", | 20260507 02:29:24.376 | INFO | "huge_shared" : " 27 GB", | 20260507 02:29:24.376 | INFO | "malloc" : " 14 GB", | 20260507 02:29:24.376 | INFO | "pinned" : " 29 GB" | 20260507 02:29:24.376 | INFO | } | 20260507 02:29:24.376 | INFO | Perf report for the last 5.14217 seconds: | 20260507 02:29:24.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:29:24.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:24.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:24.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:24.376 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:29:24.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:24.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:24.376 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:29:24.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:24.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:24.377 | INFO | Uploaded 36GiB to GPU in 1.99632s (18.0331GiB/s) | 20260507 02:29:24.377 | INFO | Downloaded 34GiB to GPU in 1.93591s (17.5628GiB/s) | 20260507 02:29:29.508 | INFO | Analyzing chunk frames = [6383104,6383616) | 20260507 02:29:29.508 | INFO | MemInfo: { | 20260507 02:29:29.508 | INFO | "gpu" : " 17 GB", | 20260507 02:29:29.508 | INFO | "huge" : "", | 20260507 02:29:29.508 | INFO | "huge_shared" : " 27 GB", | 20260507 02:29:29.508 | INFO | "malloc" : " 14 GB", | 20260507 02:29:29.508 | INFO | "pinned" : " 29 GB" | 20260507 02:29:29.508 | INFO | } | 20260507 02:29:29.508 | INFO | Perf report for the last 5.13197 seconds: | 20260507 02:29:29.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:29:29.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:29.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:29.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:29.508 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 02:29:29.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:29.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:29.509 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:29:29.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:29.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:29.509 | INFO | Uploaded 36GiB to GPU in 1.96384s (18.3315GiB/s) | 20260507 02:29:29.509 | INFO | Downloaded 34GiB to GPU in 1.90334s (17.8633GiB/s) | 20260507 02:29:34.655 | INFO | Analyzing chunk frames = [6383616,6384128) | 20260507 02:29:34.655 | INFO | MemInfo: { | 20260507 02:29:34.655 | INFO | "gpu" : " 17 GB", | 20260507 02:29:34.655 | INFO | "huge" : "", | 20260507 02:29:34.655 | INFO | "huge_shared" : " 27 GB", | 20260507 02:29:34.655 | INFO | "malloc" : " 14 GB", | 20260507 02:29:34.655 | INFO | "pinned" : " 29 GB" | 20260507 02:29:34.655 | INFO | } | 20260507 02:29:34.655 | INFO | Perf report for the last 5.14636 seconds: | 20260507 02:29:34.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:29:34.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:34.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:34.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:34.655 | INFO | ANALYSIS : 59.8%, 3.0, 93.2ms, 0 | 20260507 02:29:34.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:34.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:34.655 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:29:34.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:34.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:34.655 | INFO | Uploaded 36GiB to GPU in 1.9866s (18.1214GiB/s) | 20260507 02:29:34.655 | INFO | Downloaded 34GiB to GPU in 1.93209s (17.5975GiB/s) | 20260507 02:29:39.788 | INFO | Analyzing chunk frames = [6384128,6384640) | 20260507 02:29:39.788 | INFO | MemInfo: { | 20260507 02:29:39.788 | INFO | "gpu" : " 17 GB", | 20260507 02:29:39.788 | INFO | "huge" : "", | 20260507 02:29:39.788 | INFO | "huge_shared" : " 27 GB", | 20260507 02:29:39.788 | INFO | "malloc" : " 14 GB", | 20260507 02:29:39.788 | INFO | "pinned" : " 29 GB" | 20260507 02:29:39.788 | INFO | } | 20260507 02:29:39.789 | INFO | Perf report for the last 5.13376 seconds: | 20260507 02:29:39.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:29:39.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:39.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:39.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:39.789 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 02:29:39.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:39.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:39.789 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 02:29:39.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:39.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:39.789 | INFO | Uploaded 36GiB to GPU in 2.05897s (17.4845GiB/s) | 20260507 02:29:39.789 | INFO | Downloaded 34GiB to GPU in 1.95897s (17.3561GiB/s) | 20260507 02:29:44.928 | INFO | Analyzing chunk frames = [6384640,6385152) | 20260507 02:29:44.928 | INFO | MemInfo: { | 20260507 02:29:44.928 | INFO | "gpu" : " 17 GB", | 20260507 02:29:44.928 | INFO | "huge" : "", | 20260507 02:29:44.928 | INFO | "huge_shared" : " 27 GB", | 20260507 02:29:44.928 | INFO | "malloc" : " 14 GB", | 20260507 02:29:44.928 | INFO | "pinned" : " 29 GB" | 20260507 02:29:44.928 | INFO | } | 20260507 02:29:44.928 | INFO | Perf report for the last 5.13967 seconds: | 20260507 02:29:44.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:29:44.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:44.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:44.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:44.928 | INFO | ANALYSIS : 61.1%, 3.0, 94.1ms, 0 | 20260507 02:29:44.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:44.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:44.928 | INFO | PRE_HQ : 26.0%, 1.0, 13.3ms, 0 | 20260507 02:29:44.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:44.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:44.928 | INFO | Uploaded 36GiB to GPU in 2.00946s (17.9153GiB/s) | 20260507 02:29:44.929 | INFO | Downloaded 34GiB to GPU in 1.9946s (17.046GiB/s) | 20260507 02:29:50.076 | INFO | Analyzing chunk frames = [6385152,6385664) | 20260507 02:29:50.076 | INFO | MemInfo: { | 20260507 02:29:50.076 | INFO | "gpu" : " 17 GB", | 20260507 02:29:50.076 | INFO | "huge" : "", | 20260507 02:29:50.076 | INFO | "huge_shared" : " 27 GB", | 20260507 02:29:50.076 | INFO | "malloc" : " 14 GB", | 20260507 02:29:50.076 | INFO | "pinned" : " 29 GB" | 20260507 02:29:50.076 | INFO | } | 20260507 02:29:50.076 | INFO | Perf report for the last 5.14798 seconds: | 20260507 02:29:50.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:29:50.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:50.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:50.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:50.076 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 02:29:50.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:50.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:50.076 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 02:29:50.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:50.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:50.076 | INFO | Uploaded 36GiB to GPU in 2.00459s (17.9588GiB/s) | 20260507 02:29:50.076 | INFO | Downloaded 34GiB to GPU in 1.97678s (17.1997GiB/s) | 20260507 02:29:50.173 | INFO | Starting a round of Baz IO | 20260507 02:29:53.278 | INFO | Populating RTMetrics report | 20260507 02:29:53.278 | INFO | RTMetrics report populated | 20260507 02:29:53.278 | INFO | Opening tmp RTMetrics json file | 20260507 02:29:53.281 | INFO | Serializing RTMetrics to json | 20260507 02:29:53.297 | INFO | Serialization done, swapping tmp file now... | 20260507 02:29:53.298 | INFO | RTMetrics done | 20260507 02:29:53.370 | INFO | Finished a round of Baz IO: Wrote 1.779 GiB in 3.197 seconds | 20260507 02:29:53.370 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.00% / 0.00% / 67.00% | 20260507 02:29:53.370 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.234636% / 3.97364e-06% / 9.53674e-05% | 20260507 02:29:53.370 | INFO | Baz IO overall: | 20260507 02:29:53.370 | INFO | BytesWritten : 3294860840089 | 20260507 02:29:53.370 | INFO | NumEvents : 1584501294102 | 20260507 02:29:55.214 | INFO | Analyzing chunk frames = [6385664,6386176) | 20260507 02:29:55.214 | INFO | MemInfo: { | 20260507 02:29:55.214 | INFO | "gpu" : " 17 GB", | 20260507 02:29:55.214 | INFO | "huge" : "", | 20260507 02:29:55.214 | INFO | "huge_shared" : " 27 GB", | 20260507 02:29:55.214 | INFO | "malloc" : " 14 GB", | 20260507 02:29:55.214 | INFO | "pinned" : " 29 GB" | 20260507 02:29:55.214 | INFO | } | 20260507 02:29:55.214 | INFO | Perf report for the last 5.13752 seconds: | 20260507 02:29:55.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:29:55.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:55.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:55.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:55.214 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260507 02:29:55.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:29:55.214 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 02:29:55.214 | INFO | PRE_HQ : 59.1%, 1.8, 55.5ms, 0 | 20260507 02:29:55.214 | INFO | BAZWRITER : 61.7%, 3.3, 105.3ms, 0 | 20260507 02:29:55.214 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:29:55.214 | INFO | Uploaded 36GiB to GPU in 2.0098s (17.9123GiB/s) | 20260507 02:29:55.214 | INFO | Downloaded 38GiB to GPU in 2.1785s (17.4432GiB/s) | 20260507 02:30:00.353 | INFO | Analyzing chunk frames = [6386176,6386688) | 20260507 02:30:00.353 | INFO | MemInfo: { | 20260507 02:30:00.353 | INFO | "gpu" : " 17 GB", | 20260507 02:30:00.353 | INFO | "huge" : "", | 20260507 02:30:00.353 | INFO | "huge_shared" : " 27 GB", | 20260507 02:30:00.353 | INFO | "malloc" : " 14 GB", | 20260507 02:30:00.353 | INFO | "pinned" : " 29 GB" | 20260507 02:30:00.353 | INFO | } | 20260507 02:30:00.353 | INFO | Perf report for the last 5.13964 seconds: | 20260507 02:30:00.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:30:00.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:00.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:00.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:00.353 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:30:00.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:00.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:00.353 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:30:00.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:00.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:00.354 | INFO | Uploaded 36GiB to GPU in 2.02401s (17.7865GiB/s) | 20260507 02:30:00.354 | INFO | Downloaded 34GiB to GPU in 1.93526s (17.5687GiB/s) | 20260507 02:30:05.488 | INFO | Analyzing chunk frames = [6386688,6387200) | 20260507 02:30:05.488 | INFO | MemInfo: { | 20260507 02:30:05.488 | INFO | "gpu" : " 17 GB", | 20260507 02:30:05.488 | INFO | "huge" : "", | 20260507 02:30:05.488 | INFO | "huge_shared" : " 27 GB", | 20260507 02:30:05.488 | INFO | "malloc" : " 14 GB", | 20260507 02:30:05.488 | INFO | "pinned" : " 29 GB" | 20260507 02:30:05.488 | INFO | } | 20260507 02:30:05.488 | INFO | Perf report for the last 5.13457 seconds: | 20260507 02:30:05.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:30:05.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:05.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:05.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:05.488 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 02:30:05.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:05.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:05.488 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:30:05.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:05.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:05.488 | INFO | Uploaded 36GiB to GPU in 2.01171s (17.8952GiB/s) | 20260507 02:30:05.488 | INFO | Downloaded 34GiB to GPU in 1.94114s (17.5155GiB/s) | 20260507 02:30:10.622 | INFO | Analyzing chunk frames = [6387200,6387712) | 20260507 02:30:10.622 | INFO | MemInfo: { | 20260507 02:30:10.622 | INFO | "gpu" : " 17 GB", | 20260507 02:30:10.622 | INFO | "huge" : "", | 20260507 02:30:10.622 | INFO | "huge_shared" : " 27 GB", | 20260507 02:30:10.622 | INFO | "malloc" : " 14 GB", | 20260507 02:30:10.622 | INFO | "pinned" : " 29 GB" | 20260507 02:30:10.622 | INFO | } | 20260507 02:30:10.622 | INFO | Perf report for the last 5.13421 seconds: | 20260507 02:30:10.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:30:10.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:10.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:10.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:10.622 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 02:30:10.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:10.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:10.622 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:30:10.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:10.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:10.622 | INFO | Uploaded 36GiB to GPU in 1.94976s (18.4638GiB/s) | 20260507 02:30:10.622 | INFO | Downloaded 34GiB to GPU in 1.89739s (17.9194GiB/s) | 20260507 02:30:15.763 | INFO | Analyzing chunk frames = [6387712,6388224) | 20260507 02:30:15.763 | INFO | MemInfo: { | 20260507 02:30:15.763 | INFO | "gpu" : " 17 GB", | 20260507 02:30:15.763 | INFO | "huge" : "", | 20260507 02:30:15.764 | INFO | "huge_shared" : " 27 GB", | 20260507 02:30:15.764 | INFO | "malloc" : " 14 GB", | 20260507 02:30:15.764 | INFO | "pinned" : " 29 GB" | 20260507 02:30:15.764 | INFO | } | 20260507 02:30:15.764 | INFO | Perf report for the last 5.14148 seconds: | 20260507 02:30:15.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:30:15.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:15.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:15.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:15.764 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 02:30:15.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:15.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:15.764 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:30:15.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:15.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:15.764 | INFO | Uploaded 36GiB to GPU in 2.00502s (17.9549GiB/s) | 20260507 02:30:15.764 | INFO | Downloaded 34GiB to GPU in 1.94637s (17.4685GiB/s) | 20260507 02:30:20.910 | INFO | Analyzing chunk frames = [6388224,6388736) | 20260507 02:30:20.910 | INFO | MemInfo: { | 20260507 02:30:20.910 | INFO | "gpu" : " 17 GB", | 20260507 02:30:20.910 | INFO | "huge" : "", | 20260507 02:30:20.910 | INFO | "huge_shared" : " 27 GB", | 20260507 02:30:20.910 | INFO | "malloc" : " 14 GB", | 20260507 02:30:20.910 | INFO | "pinned" : " 29 GB" | 20260507 02:30:20.910 | INFO | } | 20260507 02:30:20.910 | INFO | Perf report for the last 5.14653 seconds: | 20260507 02:30:20.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:30:20.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:20.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:20.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:20.910 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 02:30:20.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:20.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:20.910 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:30:20.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:20.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:20.910 | INFO | Uploaded 36GiB to GPU in 2.02759s (17.7551GiB/s) | 20260507 02:30:20.910 | INFO | Downloaded 34GiB to GPU in 1.94594s (17.4722GiB/s) | 20260507 02:30:26.048 | INFO | Analyzing chunk frames = [6388736,6389248) | 20260507 02:30:26.048 | INFO | MemInfo: { | 20260507 02:30:26.048 | INFO | "gpu" : " 17 GB", | 20260507 02:30:26.048 | INFO | "huge" : "", | 20260507 02:30:26.048 | INFO | "huge_shared" : " 27 GB", | 20260507 02:30:26.048 | INFO | "malloc" : " 14 GB", | 20260507 02:30:26.048 | INFO | "pinned" : " 29 GB" | 20260507 02:30:26.048 | INFO | } | 20260507 02:30:26.048 | INFO | Perf report for the last 5.1376 seconds: | 20260507 02:30:26.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:30:26.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:26.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:26.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:26.048 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 02:30:26.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:26.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:26.048 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:30:26.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:26.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:26.048 | INFO | Uploaded 36GiB to GPU in 1.98458s (18.1399GiB/s) | 20260507 02:30:26.048 | INFO | Downloaded 34GiB to GPU in 2.00177s (16.9849GiB/s) | 20260507 02:30:31.190 | INFO | Analyzing chunk frames = [6389248,6389760) | 20260507 02:30:31.190 | INFO | MemInfo: { | 20260507 02:30:31.190 | INFO | "gpu" : " 17 GB", | 20260507 02:30:31.190 | INFO | "huge" : "", | 20260507 02:30:31.190 | INFO | "huge_shared" : " 27 GB", | 20260507 02:30:31.190 | INFO | "malloc" : " 14 GB", | 20260507 02:30:31.190 | INFO | "pinned" : " 29 GB" | 20260507 02:30:31.190 | INFO | } | 20260507 02:30:31.190 | INFO | Perf report for the last 5.14258 seconds: | 20260507 02:30:31.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:30:31.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:31.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:31.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:31.190 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 02:30:31.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:31.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:31.190 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 02:30:31.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:31.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:31.191 | INFO | Uploaded 36GiB to GPU in 2.00616s (17.9447GiB/s) | 20260507 02:30:31.191 | INFO | Downloaded 34GiB to GPU in 1.98404s (17.1368GiB/s) | 20260507 02:30:31.191 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:30:31.191 | INFO | Total mem usage: 18 GiB | 20260507 02:30:31.191 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:30:31.191 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:30:31.191 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:30:31.191 | INFO | High Water Breakdown: | 20260507 02:30:31.191 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:30:31.191 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:30:31.191 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:30:31.191 | INFO | ------------------------------------------ | 20260507 02:30:31.191 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:30:31.191 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:30:31.191 | INFO | Total mem usage: 26.5 GiB | 20260507 02:30:31.191 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:30:31.191 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:30:31.191 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:30:31.191 | INFO | High Water Breakdown: | 20260507 02:30:31.191 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:30:31.191 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:30:31.191 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:30:31.191 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:30:31.191 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:30:31.191 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:30:31.191 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:30:31.191 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:30:31.191 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:30:31.191 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:30:31.191 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:30:31.191 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:30:31.191 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:30:31.191 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:30:31.191 | INFO | ------------------------------------------ | 20260507 02:30:31.191 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:30:31.191 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:30:31.191 | INFO | Total mem usage: 25.6 GiB | 20260507 02:30:31.191 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:30:31.191 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 02:30:31.191 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 02:30:31.191 | INFO | High Water Breakdown: | 20260507 02:30:31.191 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 02:30:31.191 | INFO | ------------------------------------------ | 20260507 02:30:31.191 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:30:31.191 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:30:31.191 | INFO | Total mem usage: 15.7 GiB | 20260507 02:30:31.191 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:30:31.191 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:30:31.191 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:30:31.191 | INFO | High Water Breakdown: | 20260507 02:30:31.191 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:30:31.191 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:30:31.191 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:30:31.191 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:30:31.191 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:30:31.191 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:30:31.191 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:30:31.191 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:30:31.191 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:30:31.191 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:30:31.191 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:30:31.191 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:30:31.191 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:30:31.191 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:30:31.191 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:30:31.191 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:30:31.191 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:30:31.191 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:30:31.191 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:30:31.191 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:30:31.191 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:30:31.191 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:30:31.191 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:30:31.191 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:30:31.191 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:30:31.191 | INFO | ------------------------------------------ | 20260507 02:30:31.191 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:30:31.191 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:30:31.191 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:30:31.191 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:30:31.191 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.57e+01, 1.35e+01 / 1.95e+00 | 20260507 02:30:31.192 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.42e-03 / 4.99e-03 | 20260507 02:30:31.192 | INFO | PerfReport: 11.68%, Baseline -- 3.60e+00 / 4.47e+00, 5.26e-01 / 2.26e-01 | 20260507 02:30:31.192 | INFO | PerfReport: 3.23%, Histogram -- 9.96e-01 / 1.48e+00, 1.73e-01 / 8.18e-02 | 20260507 02:30:31.192 | INFO | PerfReport: 16.13%, DME -- 3.98e+01 / 4.46e+01, 4.07e+00 / 2.44e+00 | 20260507 02:30:31.192 | INFO | PerfReport: 57.55%, FrameLabeling -- 1.77e+01 / 2.07e+01, 1.28e+00 / 1.66e+00 | 20260507 02:30:31.192 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.15e+00 / 2.67e+00, 3.81e-01 / 2.28e-01 | 20260507 02:30:31.192 | INFO | PerfReport: 3.08%, Metrics -- 9.49e-01 / 1.27e+00, 2.56e-01 / 1.65e-01 | 20260507 02:30:31.192 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.22e-01 / 3.80e-01, 5.17e-02 / 2.45e-02 | 20260507 02:30:31.192 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:30:31.192 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:30:31.192 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:30:31.192 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 02:30:31.192 | INFO | PerfReport: 21.66%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260507 02:30:31.192 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 02:30:31.192 | INFO | PerfReport: 31.97%, Compute -- 3.08e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 02:30:31.192 | INFO | PerfReport: 1.40%, LockUpload -- 1.35e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 02:30:31.192 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260507 02:30:31.192 | INFO | PerfReport: 22.30%, LockCompute -- 2.15e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 02:30:31.192 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.24e-02 / 0.00e+00 | 20260507 02:30:31.287 | INFO | Starting a round of Baz IO | 20260507 02:30:34.394 | INFO | Populating RTMetrics report | 20260507 02:30:34.395 | INFO | RTMetrics report populated | 20260507 02:30:34.395 | INFO | Opening tmp RTMetrics json file | 20260507 02:30:34.413 | INFO | Serializing RTMetrics to json | 20260507 02:30:34.414 | INFO | Serialization done, swapping tmp file now... | 20260507 02:30:34.415 | INFO | RTMetrics done | 20260507 02:30:34.485 | INFO | Finished a round of Baz IO: Wrote 1.779 GiB in 3.197 seconds | 20260507 02:30:34.485 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.00% / 0.00% / 67.00% | 20260507 02:30:34.485 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.235037% / 3.97364e-06% / 7.15256e-05% | 20260507 02:30:34.485 | INFO | Baz IO overall: | 20260507 02:30:34.485 | INFO | BytesWritten : 3296770889601 | 20260507 02:30:34.485 | INFO | NumEvents : 1585404322923 | 20260507 02:30:36.331 | INFO | Analyzing chunk frames = [6389760,6390272) | 20260507 02:30:36.331 | INFO | MemInfo: { | 20260507 02:30:36.331 | INFO | "gpu" : " 17 GB", | 20260507 02:30:36.331 | INFO | "huge" : "", | 20260507 02:30:36.331 | INFO | "huge_shared" : " 27 GB", | 20260507 02:30:36.331 | INFO | "malloc" : " 14 GB", | 20260507 02:30:36.331 | INFO | "pinned" : " 29 GB" | 20260507 02:30:36.331 | INFO | } | 20260507 02:30:36.331 | INFO | Perf report for the last 5.14059 seconds: | 20260507 02:30:36.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:30:36.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:36.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:36.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:36.331 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 02:30:36.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:36.331 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 02:30:36.331 | INFO | PRE_HQ : 59.3%, 1.8, 56.1ms, 0 | 20260507 02:30:36.331 | INFO | BAZWRITER : 61.8%, 3.3, 105.6ms, 0 | 20260507 02:30:36.331 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:30:36.331 | INFO | Uploaded 36GiB to GPU in 2.02887s (17.7439GiB/s) | 20260507 02:30:36.331 | INFO | Downloaded 38GiB to GPU in 2.18473s (17.3934GiB/s) | 20260507 02:30:41.466 | INFO | Analyzing chunk frames = [6390272,6390784) | 20260507 02:30:41.466 | INFO | MemInfo: { | 20260507 02:30:41.466 | INFO | "gpu" : " 17 GB", | 20260507 02:30:41.466 | INFO | "huge" : "", | 20260507 02:30:41.466 | INFO | "huge_shared" : " 27 GB", | 20260507 02:30:41.466 | INFO | "malloc" : " 14 GB", | 20260507 02:30:41.466 | INFO | "pinned" : " 29 GB" | 20260507 02:30:41.466 | INFO | } | 20260507 02:30:41.466 | INFO | Perf report for the last 5.13528 seconds: | 20260507 02:30:41.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:30:41.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:41.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:41.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:41.466 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 02:30:41.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:41.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:41.466 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:30:41.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:41.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:41.467 | INFO | Uploaded 36GiB to GPU in 2.03625s (17.6796GiB/s) | 20260507 02:30:41.467 | INFO | Downloaded 34GiB to GPU in 1.94131s (17.514GiB/s) | 20260507 02:30:46.605 | INFO | Analyzing chunk frames = [6390784,6391296) | 20260507 02:30:46.605 | INFO | MemInfo: { | 20260507 02:30:46.605 | INFO | "gpu" : " 17 GB", | 20260507 02:30:46.605 | INFO | "huge" : "", | 20260507 02:30:46.605 | INFO | "huge_shared" : " 27 GB", | 20260507 02:30:46.605 | INFO | "malloc" : " 14 GB", | 20260507 02:30:46.605 | INFO | "pinned" : " 29 GB" | 20260507 02:30:46.605 | INFO | } | 20260507 02:30:46.605 | INFO | Perf report for the last 5.13858 seconds: | 20260507 02:30:46.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:30:46.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:46.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:46.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:46.605 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 02:30:46.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:46.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:46.605 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:30:46.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:46.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:46.605 | INFO | Uploaded 36GiB to GPU in 2.00202s (17.9818GiB/s) | 20260507 02:30:46.605 | INFO | Downloaded 34GiB to GPU in 1.9401s (17.5249GiB/s) | 20260507 02:30:51.744 | INFO | Analyzing chunk frames = [6391296,6391808) | 20260507 02:30:51.744 | INFO | MemInfo: { | 20260507 02:30:51.744 | INFO | "gpu" : " 17 GB", | 20260507 02:30:51.744 | INFO | "huge" : "", | 20260507 02:30:51.744 | INFO | "huge_shared" : " 27 GB", | 20260507 02:30:51.744 | INFO | "malloc" : " 14 GB", | 20260507 02:30:51.744 | INFO | "pinned" : " 29 GB" | 20260507 02:30:51.744 | INFO | } | 20260507 02:30:51.744 | INFO | Perf report for the last 5.13895 seconds: | 20260507 02:30:51.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:30:51.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:51.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:51.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:51.744 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 02:30:51.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:51.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:51.744 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:30:51.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:51.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:51.744 | INFO | Uploaded 36GiB to GPU in 1.98607s (18.1263GiB/s) | 20260507 02:30:51.744 | INFO | Downloaded 34GiB to GPU in 1.92241s (17.6861GiB/s) | 20260507 02:30:56.882 | INFO | Analyzing chunk frames = [6391808,6392320) | 20260507 02:30:56.882 | INFO | MemInfo: { | 20260507 02:30:56.882 | INFO | "gpu" : " 17 GB", | 20260507 02:30:56.882 | INFO | "huge" : "", | 20260507 02:30:56.882 | INFO | "huge_shared" : " 27 GB", | 20260507 02:30:56.882 | INFO | "malloc" : " 14 GB", | 20260507 02:30:56.882 | INFO | "pinned" : " 29 GB" | 20260507 02:30:56.882 | INFO | } | 20260507 02:30:56.882 | INFO | Perf report for the last 5.13793 seconds: | 20260507 02:30:56.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:30:56.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:56.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:56.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:56.882 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 02:30:56.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:56.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:56.882 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:30:56.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:56.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:30:56.882 | INFO | Uploaded 36GiB to GPU in 2.01448s (17.8706GiB/s) | 20260507 02:30:56.882 | INFO | Downloaded 34GiB to GPU in 1.93499s (17.5711GiB/s) | 20260507 02:31:02.022 | INFO | Analyzing chunk frames = [6392320,6392832) | 20260507 02:31:02.022 | INFO | MemInfo: { | 20260507 02:31:02.022 | INFO | "gpu" : " 17 GB", | 20260507 02:31:02.022 | INFO | "huge" : "", | 20260507 02:31:02.022 | INFO | "huge_shared" : " 27 GB", | 20260507 02:31:02.022 | INFO | "malloc" : " 14 GB", | 20260507 02:31:02.022 | INFO | "pinned" : " 29 GB" | 20260507 02:31:02.022 | INFO | } | 20260507 02:31:02.022 | INFO | Perf report for the last 5.14011 seconds: | 20260507 02:31:02.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:31:02.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:02.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:02.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:02.022 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 02:31:02.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:02.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:02.022 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:31:02.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:02.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:02.022 | INFO | Uploaded 36GiB to GPU in 2.04378s (17.6144GiB/s) | 20260507 02:31:02.022 | INFO | Downloaded 34GiB to GPU in 1.9567s (17.3762GiB/s) | 20260507 02:31:07.164 | INFO | Analyzing chunk frames = [6392832,6393344) | 20260507 02:31:07.164 | INFO | MemInfo: { | 20260507 02:31:07.164 | INFO | "gpu" : " 17 GB", | 20260507 02:31:07.164 | INFO | "huge" : "", | 20260507 02:31:07.164 | INFO | "huge_shared" : " 27 GB", | 20260507 02:31:07.164 | INFO | "malloc" : " 14 GB", | 20260507 02:31:07.164 | INFO | "pinned" : " 29 GB" | 20260507 02:31:07.164 | INFO | } | 20260507 02:31:07.164 | INFO | Perf report for the last 5.14202 seconds: | 20260507 02:31:07.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:31:07.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:07.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:07.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:07.164 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 02:31:07.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:07.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:07.164 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:31:07.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:07.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:07.164 | INFO | Uploaded 36GiB to GPU in 1.96399s (18.33GiB/s) | 20260507 02:31:07.164 | INFO | Downloaded 34GiB to GPU in 1.95834s (17.3616GiB/s) | 20260507 02:31:12.306 | INFO | Analyzing chunk frames = [6393344,6393856) | 20260507 02:31:12.306 | INFO | MemInfo: { | 20260507 02:31:12.306 | INFO | "gpu" : " 17 GB", | 20260507 02:31:12.306 | INFO | "huge" : "", | 20260507 02:31:12.306 | INFO | "huge_shared" : " 27 GB", | 20260507 02:31:12.306 | INFO | "malloc" : " 14 GB", | 20260507 02:31:12.306 | INFO | "pinned" : " 29 GB" | 20260507 02:31:12.306 | INFO | } | 20260507 02:31:12.306 | INFO | Perf report for the last 5.14183 seconds: | 20260507 02:31:12.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:31:12.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:12.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:12.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:12.306 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 02:31:12.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:12.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:12.306 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:31:12.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:12.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:12.306 | INFO | Uploaded 36GiB to GPU in 1.96519s (18.3188GiB/s) | 20260507 02:31:12.306 | INFO | Downloaded 34GiB to GPU in 1.96161s (17.3327GiB/s) | 20260507 02:31:12.403 | INFO | Starting a round of Baz IO | 20260507 02:31:15.508 | INFO | Populating RTMetrics report | 20260507 02:31:15.508 | INFO | RTMetrics report populated | 20260507 02:31:15.509 | INFO | Opening tmp RTMetrics json file | 20260507 02:31:15.509 | INFO | Serializing RTMetrics to json | 20260507 02:31:15.520 | INFO | Serialization done, swapping tmp file now... | 20260507 02:31:15.529 | INFO | RTMetrics done | 20260507 02:31:15.602 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.199 seconds | 20260507 02:31:15.602 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 33.00% / 0.00% / 67.00% | 20260507 02:31:15.602 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.234489% / 7.94729e-06% / 7.15256e-05% | 20260507 02:31:15.602 | INFO | Baz IO overall: | 20260507 02:31:15.602 | INFO | BytesWritten : 3298680120482 | 20260507 02:31:15.602 | INFO | NumEvents : 1586306520355 | 20260507 02:31:17.441 | INFO | Analyzing chunk frames = [6393856,6394368) | 20260507 02:31:17.441 | INFO | MemInfo: { | 20260507 02:31:17.441 | INFO | "gpu" : " 17 GB", | 20260507 02:31:17.441 | INFO | "huge" : "", | 20260507 02:31:17.441 | INFO | "huge_shared" : " 27 GB", | 20260507 02:31:17.441 | INFO | "malloc" : " 14 GB", | 20260507 02:31:17.441 | INFO | "pinned" : " 29 GB" | 20260507 02:31:17.441 | INFO | } | 20260507 02:31:17.441 | INFO | Perf report for the last 5.13496 seconds: | 20260507 02:31:17.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:31:17.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:17.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:17.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:17.441 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 02:31:17.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:17.441 | INFO | RT_METRICS : 42.1%, 1.0, 22.0ms, 0 | 20260507 02:31:17.441 | INFO | PRE_HQ : 59.3%, 1.8, 55.6ms, 0 | 20260507 02:31:17.441 | INFO | BAZWRITER : 61.9%, 3.3, 105.1ms, 0 | 20260507 02:31:17.441 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:31:17.441 | INFO | Uploaded 36GiB to GPU in 2.01901s (17.8305GiB/s) | 20260507 02:31:17.441 | INFO | Downloaded 38GiB to GPU in 2.18323s (17.4054GiB/s) | 20260507 02:31:22.573 | INFO | Analyzing chunk frames = [6394368,6394880) | 20260507 02:31:22.573 | INFO | MemInfo: { | 20260507 02:31:22.573 | INFO | "gpu" : " 17 GB", | 20260507 02:31:22.573 | INFO | "huge" : "", | 20260507 02:31:22.573 | INFO | "huge_shared" : " 27 GB", | 20260507 02:31:22.574 | INFO | "malloc" : " 14 GB", | 20260507 02:31:22.574 | INFO | "pinned" : " 29 GB" | 20260507 02:31:22.574 | INFO | } | 20260507 02:31:22.574 | INFO | Perf report for the last 5.13276 seconds: | 20260507 02:31:22.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:31:22.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:22.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:22.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:22.574 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 02:31:22.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:22.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:22.574 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:31:22.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:22.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:22.574 | INFO | Uploaded 36GiB to GPU in 2.02299s (17.7954GiB/s) | 20260507 02:31:22.574 | INFO | Downloaded 34GiB to GPU in 1.93267s (17.5922GiB/s) | 20260507 02:31:27.717 | INFO | Analyzing chunk frames = [6394880,6395392) | 20260507 02:31:27.717 | INFO | MemInfo: { | 20260507 02:31:27.717 | INFO | "gpu" : " 17 GB", | 20260507 02:31:27.717 | INFO | "huge" : "", | 20260507 02:31:27.717 | INFO | "huge_shared" : " 27 GB", | 20260507 02:31:27.717 | INFO | "malloc" : " 14 GB", | 20260507 02:31:27.717 | INFO | "pinned" : " 29 GB" | 20260507 02:31:27.717 | INFO | } | 20260507 02:31:27.718 | INFO | Perf report for the last 5.14398 seconds: | 20260507 02:31:27.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:31:27.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:27.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:27.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:27.718 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 02:31:27.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:27.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:27.718 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:31:27.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:27.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:27.718 | INFO | Uploaded 36GiB to GPU in 2.00803s (17.928GiB/s) | 20260507 02:31:27.718 | INFO | Downloaded 34GiB to GPU in 1.93808s (17.5431GiB/s) | 20260507 02:31:32.868 | INFO | Analyzing chunk frames = [6395392,6395904) | 20260507 02:31:32.868 | INFO | MemInfo: { | 20260507 02:31:32.868 | INFO | "gpu" : " 17 GB", | 20260507 02:31:32.868 | INFO | "huge" : "", | 20260507 02:31:32.868 | INFO | "huge_shared" : " 27 GB", | 20260507 02:31:32.868 | INFO | "malloc" : " 14 GB", | 20260507 02:31:32.868 | INFO | "pinned" : " 29 GB" | 20260507 02:31:32.868 | INFO | } | 20260507 02:31:32.868 | INFO | Perf report for the last 5.15045 seconds: | 20260507 02:31:32.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:31:32.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:32.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:32.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:32.868 | INFO | ANALYSIS : 59.6%, 3.0, 92.7ms, 0 | 20260507 02:31:32.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:32.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:32.868 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:31:32.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:32.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:32.868 | INFO | Uploaded 36GiB to GPU in 1.98295s (18.1547GiB/s) | 20260507 02:31:32.868 | INFO | Downloaded 34GiB to GPU in 1.91171s (17.7851GiB/s) | 20260507 02:31:38.003 | INFO | Analyzing chunk frames = [6395904,6396416) | 20260507 02:31:38.003 | INFO | MemInfo: { | 20260507 02:31:38.003 | INFO | "gpu" : " 17 GB", | 20260507 02:31:38.003 | INFO | "huge" : "", | 20260507 02:31:38.003 | INFO | "huge_shared" : " 27 GB", | 20260507 02:31:38.003 | INFO | "malloc" : " 14 GB", | 20260507 02:31:38.003 | INFO | "pinned" : " 29 GB" | 20260507 02:31:38.003 | INFO | } | 20260507 02:31:38.003 | INFO | Perf report for the last 5.13538 seconds: | 20260507 02:31:38.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:31:38.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:38.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:38.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:38.003 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 02:31:38.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:38.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:38.004 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:31:38.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:38.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:38.004 | INFO | Uploaded 36GiB to GPU in 2.02356s (17.7904GiB/s) | 20260507 02:31:38.004 | INFO | Downloaded 34GiB to GPU in 1.94105s (17.5163GiB/s) | 20260507 02:31:43.140 | INFO | Analyzing chunk frames = [6396416,6396928) | 20260507 02:31:43.140 | INFO | MemInfo: { | 20260507 02:31:43.140 | INFO | "gpu" : " 17 GB", | 20260507 02:31:43.140 | INFO | "huge" : "", | 20260507 02:31:43.140 | INFO | "huge_shared" : " 27 GB", | 20260507 02:31:43.140 | INFO | "malloc" : " 14 GB", | 20260507 02:31:43.140 | INFO | "pinned" : " 29 GB" | 20260507 02:31:43.140 | INFO | } | 20260507 02:31:43.140 | INFO | Perf report for the last 5.13679 seconds: | 20260507 02:31:43.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:31:43.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:43.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:43.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:43.140 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 02:31:43.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:43.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:43.140 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 02:31:43.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:43.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:43.140 | INFO | Uploaded 36GiB to GPU in 2.02736s (17.7571GiB/s) | 20260507 02:31:43.141 | INFO | Downloaded 34GiB to GPU in 1.92992s (17.6173GiB/s) | 20260507 02:31:48.272 | INFO | Analyzing chunk frames = [6396928,6397440) | 20260507 02:31:48.272 | INFO | MemInfo: { | 20260507 02:31:48.272 | INFO | "gpu" : " 17 GB", | 20260507 02:31:48.272 | INFO | "huge" : "", | 20260507 02:31:48.273 | INFO | "huge_shared" : " 27 GB", | 20260507 02:31:48.273 | INFO | "malloc" : " 14 GB", | 20260507 02:31:48.273 | INFO | "pinned" : " 29 GB" | 20260507 02:31:48.273 | INFO | } | 20260507 02:31:48.273 | INFO | Perf report for the last 5.13244 seconds: | 20260507 02:31:48.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:31:48.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:48.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:48.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:48.273 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260507 02:31:48.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:48.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:48.273 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 02:31:48.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:48.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:48.273 | INFO | Uploaded 36GiB to GPU in 1.9564s (18.4011GiB/s) | 20260507 02:31:48.273 | INFO | Downloaded 34GiB to GPU in 1.96397s (17.3119GiB/s) | 20260507 02:31:53.415 | INFO | Analyzing chunk frames = [6397440,6397952) | 20260507 02:31:53.415 | INFO | MemInfo: { | 20260507 02:31:53.415 | INFO | "gpu" : " 17 GB", | 20260507 02:31:53.415 | INFO | "huge" : "", | 20260507 02:31:53.415 | INFO | "huge_shared" : " 27 GB", | 20260507 02:31:53.415 | INFO | "malloc" : " 14 GB", | 20260507 02:31:53.415 | INFO | "pinned" : " 29 GB" | 20260507 02:31:53.415 | INFO | } | 20260507 02:31:53.415 | INFO | Perf report for the last 5.14257 seconds: | 20260507 02:31:53.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:31:53.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:53.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:53.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:53.415 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:31:53.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:53.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:53.415 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 02:31:53.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:53.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:53.415 | INFO | Uploaded 36GiB to GPU in 2.0306s (17.7287GiB/s) | 20260507 02:31:53.416 | INFO | Downloaded 34GiB to GPU in 1.97201s (17.2413GiB/s) | 20260507 02:31:53.416 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:31:53.416 | INFO | Total mem usage: 18 GiB | 20260507 02:31:53.416 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:31:53.416 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:31:53.416 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:31:53.416 | INFO | High Water Breakdown: | 20260507 02:31:53.416 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:31:53.416 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:31:53.416 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:31:53.416 | INFO | ------------------------------------------ | 20260507 02:31:53.416 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:31:53.416 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:31:53.416 | INFO | Total mem usage: 26.5 GiB | 20260507 02:31:53.416 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:31:53.416 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:31:53.416 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:31:53.416 | INFO | High Water Breakdown: | 20260507 02:31:53.416 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:31:53.416 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:31:53.416 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:31:53.416 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:31:53.416 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:31:53.416 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:31:53.416 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:31:53.416 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:31:53.416 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:31:53.416 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:31:53.416 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:31:53.416 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:31:53.416 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:31:53.416 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:31:53.416 | INFO | ------------------------------------------ | 20260507 02:31:53.416 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:31:53.416 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:31:53.416 | INFO | Total mem usage: 25.6 GiB | 20260507 02:31:53.416 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:31:53.416 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 02:31:53.416 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 02:31:53.416 | INFO | High Water Breakdown: | 20260507 02:31:53.416 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 02:31:53.416 | INFO | ------------------------------------------ | 20260507 02:31:53.416 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:31:53.416 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:31:53.416 | INFO | Total mem usage: 15.7 GiB | 20260507 02:31:53.416 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:31:53.416 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:31:53.416 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:31:53.416 | INFO | High Water Breakdown: | 20260507 02:31:53.416 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:31:53.416 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:31:53.416 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:31:53.416 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:31:53.416 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:31:53.416 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:31:53.416 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:31:53.416 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:31:53.416 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:31:53.416 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:31:53.416 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:31:53.416 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:31:53.416 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:31:53.416 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:31:53.416 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:31:53.416 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:31:53.416 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:31:53.416 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:31:53.416 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:31:53.416 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:31:53.416 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:31:53.416 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:31:53.416 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:31:53.416 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:31:53.416 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:31:53.416 | INFO | ------------------------------------------ | 20260507 02:31:53.416 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:31:53.416 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:31:53.416 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:31:53.416 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:31:53.416 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.50e+01, 1.33e+01 / 1.17e+00 | 20260507 02:31:53.416 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 1.02e-01, 9.80e-03 / 8.08e-03 | 20260507 02:31:53.416 | INFO | PerfReport: 11.68%, Baseline -- 3.60e+00 / 4.45e+00, 5.21e-01 / 2.13e-01 | 20260507 02:31:53.416 | INFO | PerfReport: 3.23%, Histogram -- 9.95e-01 / 1.45e+00, 1.68e-01 / 9.50e-02 | 20260507 02:31:53.416 | INFO | PerfReport: 16.08%, DME -- 3.96e+01 / 4.41e+01, 3.82e+00 / 1.82e+00 | 20260507 02:31:53.416 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.34e+00 / 1.65e+00 | 20260507 02:31:53.416 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.15e+00 / 2.65e+00, 3.78e-01 / 2.18e-01 | 20260507 02:31:53.416 | INFO | PerfReport: 3.08%, Metrics -- 9.47e-01 / 1.24e+00, 2.51e-01 / 1.08e-01 | 20260507 02:31:53.416 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.21e-01 / 3.73e-01, 5.11e-02 / 1.60e-02 | 20260507 02:31:53.416 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:31:53.416 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:31:53.416 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:31:53.416 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 02:31:53.416 | INFO | PerfReport: 21.70%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260507 02:31:53.416 | INFO | PerfReport: 21.34%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260507 02:31:53.416 | INFO | PerfReport: 31.98%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 02:31:53.416 | INFO | PerfReport: 1.39%, LockUpload -- 1.34e+00 / 0.00e+00, 4.74e+00 / 0.00e+00 | 20260507 02:31:53.416 | INFO | PerfReport: 1.36%, LockDownload -- 1.31e+00 / 0.00e+00, 3.19e+00 / 0.00e+00 | 20260507 02:31:53.416 | INFO | PerfReport: 22.21%, LockCompute -- 2.14e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 02:31:53.416 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260507 02:31:53.514 | INFO | Starting a round of Baz IO | 20260507 02:31:56.625 | INFO | Populating RTMetrics report | 20260507 02:31:56.625 | INFO | RTMetrics report populated | 20260507 02:31:56.625 | INFO | Opening tmp RTMetrics json file | 20260507 02:31:56.644 | INFO | Serializing RTMetrics to json | 20260507 02:31:56.646 | INFO | Serialization done, swapping tmp file now... | 20260507 02:31:56.646 | INFO | RTMetrics done | 20260507 02:31:56.724 | INFO | Finished a round of Baz IO: Wrote 1.779 GiB in 3.21 seconds | 20260507 02:31:56.724 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.99% / 0.00% / 67.00% | 20260507 02:31:56.724 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.234103% / 3.97364e-06% / 7.54992e-05% | 20260507 02:31:56.724 | INFO | Baz IO overall: | 20260507 02:31:56.724 | INFO | BytesWritten : 3300590195952 | 20260507 02:31:56.724 | INFO | NumEvents : 1587209628668 | 20260507 02:31:58.552 | INFO | Analyzing chunk frames = [6397952,6398464) | 20260507 02:31:58.552 | INFO | MemInfo: { | 20260507 02:31:58.552 | INFO | "gpu" : " 17 GB", | 20260507 02:31:58.552 | INFO | "huge" : "", | 20260507 02:31:58.552 | INFO | "huge_shared" : " 27 GB", | 20260507 02:31:58.552 | INFO | "malloc" : " 14 GB", | 20260507 02:31:58.552 | INFO | "pinned" : " 29 GB" | 20260507 02:31:58.552 | INFO | } | 20260507 02:31:58.552 | INFO | Perf report for the last 5.13726 seconds: | 20260507 02:31:58.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:31:58.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:58.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:58.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:58.553 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260507 02:31:58.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:31:58.553 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 02:31:58.553 | INFO | PRE_HQ : 59.4%, 1.8, 55.7ms, 0 | 20260507 02:31:58.553 | INFO | BAZWRITER : 62.1%, 3.3, 105.7ms, 0 | 20260507 02:31:58.553 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:31:58.553 | INFO | Uploaded 36GiB to GPU in 2.03486s (17.6916GiB/s) | 20260507 02:31:58.553 | INFO | Downloaded 38GiB to GPU in 2.18732s (17.3728GiB/s) | 20260507 02:32:03.692 | INFO | Analyzing chunk frames = [6398464,6398976) | 20260507 02:32:03.692 | INFO | MemInfo: { | 20260507 02:32:03.692 | INFO | "gpu" : " 17 GB", | 20260507 02:32:03.692 | INFO | "huge" : "", | 20260507 02:32:03.692 | INFO | "huge_shared" : " 27 GB", | 20260507 02:32:03.692 | INFO | "malloc" : " 14 GB", | 20260507 02:32:03.692 | INFO | "pinned" : " 29 GB" | 20260507 02:32:03.692 | INFO | } | 20260507 02:32:03.692 | INFO | Perf report for the last 5.13946 seconds: | 20260507 02:32:03.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:32:03.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:03.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:03.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:03.692 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:32:03.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:03.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:03.692 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:32:03.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:03.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:03.692 | INFO | Uploaded 36GiB to GPU in 1.9884s (18.105GiB/s) | 20260507 02:32:03.692 | INFO | Downloaded 34GiB to GPU in 1.93302s (17.5891GiB/s) | 20260507 02:32:08.832 | INFO | Analyzing chunk frames = [6398976,6399488) | 20260507 02:32:08.832 | INFO | MemInfo: { | 20260507 02:32:08.832 | INFO | "gpu" : " 17 GB", | 20260507 02:32:08.832 | INFO | "huge" : "", | 20260507 02:32:08.832 | INFO | "huge_shared" : " 27 GB", | 20260507 02:32:08.832 | INFO | "malloc" : " 14 GB", | 20260507 02:32:08.832 | INFO | "pinned" : " 29 GB" | 20260507 02:32:08.832 | INFO | } | 20260507 02:32:08.832 | INFO | Perf report for the last 5.14039 seconds: | 20260507 02:32:08.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:32:08.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:08.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:08.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:08.832 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 02:32:08.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:08.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:08.832 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:32:08.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:08.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:08.833 | INFO | Uploaded 36GiB to GPU in 2.00394s (17.9646GiB/s) | 20260507 02:32:08.833 | INFO | Downloaded 34GiB to GPU in 1.9406s (17.5203GiB/s) | 20260507 02:32:13.976 | INFO | Analyzing chunk frames = [6399488,6400000) | 20260507 02:32:13.976 | INFO | MemInfo: { | 20260507 02:32:13.976 | INFO | "gpu" : " 17 GB", | 20260507 02:32:13.976 | INFO | "huge" : "", | 20260507 02:32:13.976 | INFO | "huge_shared" : " 27 GB", | 20260507 02:32:13.976 | INFO | "malloc" : " 14 GB", | 20260507 02:32:13.976 | INFO | "pinned" : " 29 GB" | 20260507 02:32:13.976 | INFO | } | 20260507 02:32:13.976 | INFO | Perf report for the last 5.14338 seconds: | 20260507 02:32:13.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:32:13.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:13.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:13.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:13.976 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 02:32:13.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:13.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:13.976 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:32:13.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:13.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:13.976 | INFO | Uploaded 36GiB to GPU in 1.95055s (18.4564GiB/s) | 20260507 02:32:13.976 | INFO | Downloaded 34GiB to GPU in 1.90371s (17.8599GiB/s) | 20260507 02:32:19.122 | INFO | Analyzing chunk frames = [6400000,6400512) | 20260507 02:32:19.122 | INFO | MemInfo: { | 20260507 02:32:19.122 | INFO | "gpu" : " 17 GB", | 20260507 02:32:19.122 | INFO | "huge" : "", | 20260507 02:32:19.122 | INFO | "huge_shared" : " 27 GB", | 20260507 02:32:19.122 | INFO | "malloc" : " 14 GB", | 20260507 02:32:19.122 | INFO | "pinned" : " 29 GB" | 20260507 02:32:19.122 | INFO | } | 20260507 02:32:19.122 | INFO | Perf report for the last 5.1461 seconds: | 20260507 02:32:19.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:32:19.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:19.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:19.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:19.122 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 02:32:19.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:19.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:19.122 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:32:19.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:19.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:19.122 | INFO | Uploaded 36GiB to GPU in 2.02752s (17.7557GiB/s) | 20260507 02:32:19.122 | INFO | Downloaded 34GiB to GPU in 1.94311s (17.4977GiB/s) | 20260507 02:32:24.253 | INFO | Analyzing chunk frames = [6400512,6401024) | 20260507 02:32:24.253 | INFO | MemInfo: { | 20260507 02:32:24.253 | INFO | "gpu" : " 17 GB", | 20260507 02:32:24.253 | INFO | "huge" : "", | 20260507 02:32:24.253 | INFO | "huge_shared" : " 27 GB", | 20260507 02:32:24.253 | INFO | "malloc" : " 14 GB", | 20260507 02:32:24.253 | INFO | "pinned" : " 29 GB" | 20260507 02:32:24.253 | INFO | } | 20260507 02:32:24.253 | INFO | Perf report for the last 5.13115 seconds: | 20260507 02:32:24.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:32:24.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:24.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:24.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:24.253 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 02:32:24.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:24.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:24.253 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 02:32:24.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:24.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:24.253 | INFO | Uploaded 36GiB to GPU in 2.0283s (17.7489GiB/s) | 20260507 02:32:24.253 | INFO | Downloaded 34GiB to GPU in 1.90563s (17.8419GiB/s) | 20260507 02:32:29.389 | INFO | Analyzing chunk frames = [6401024,6401536) | 20260507 02:32:29.389 | INFO | MemInfo: { | 20260507 02:32:29.389 | INFO | "gpu" : " 17 GB", | 20260507 02:32:29.389 | INFO | "huge" : "", | 20260507 02:32:29.389 | INFO | "huge_shared" : " 27 GB", | 20260507 02:32:29.389 | INFO | "malloc" : " 14 GB", | 20260507 02:32:29.389 | INFO | "pinned" : " 29 GB" | 20260507 02:32:29.389 | INFO | } | 20260507 02:32:29.389 | INFO | Perf report for the last 5.13615 seconds: | 20260507 02:32:29.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:32:29.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:29.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:29.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:29.389 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 02:32:29.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:29.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:29.389 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:32:29.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:29.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:29.389 | INFO | Uploaded 36GiB to GPU in 1.96502s (18.3204GiB/s) | 20260507 02:32:29.389 | INFO | Downloaded 34GiB to GPU in 1.96905s (17.2672GiB/s) | 20260507 02:32:34.530 | INFO | Analyzing chunk frames = [6401536,6402048) | 20260507 02:32:34.530 | INFO | MemInfo: { | 20260507 02:32:34.530 | INFO | "gpu" : " 17 GB", | 20260507 02:32:34.530 | INFO | "huge" : "", | 20260507 02:32:34.530 | INFO | "huge_shared" : " 27 GB", | 20260507 02:32:34.530 | INFO | "malloc" : " 14 GB", | 20260507 02:32:34.530 | INFO | "pinned" : " 29 GB" | 20260507 02:32:34.530 | INFO | } | 20260507 02:32:34.530 | INFO | Perf report for the last 5.14067 seconds: | 20260507 02:32:34.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:32:34.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:34.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:34.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:34.530 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:32:34.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:34.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:34.530 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 02:32:34.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:34.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:34.530 | INFO | Uploaded 36GiB to GPU in 1.96872s (18.286GiB/s) | 20260507 02:32:34.530 | INFO | Downloaded 34GiB to GPU in 1.95847s (17.3605GiB/s) | 20260507 02:32:34.636 | INFO | Starting a round of Baz IO | 20260507 02:32:37.744 | INFO | Populating RTMetrics report | 20260507 02:32:37.745 | INFO | RTMetrics report populated | 20260507 02:32:37.745 | INFO | Opening tmp RTMetrics json file | 20260507 02:32:37.750 | INFO | Serializing RTMetrics to json | 20260507 02:32:37.764 | INFO | Serialization done, swapping tmp file now... | 20260507 02:32:37.765 | INFO | RTMetrics done | 20260507 02:32:37.832 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.196 seconds | 20260507 02:32:37.832 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.99% / 0.00% / 67.01% | 20260507 02:32:37.832 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.233161% / 0% / 7.15256e-05% | 20260507 02:32:37.832 | INFO | Baz IO overall: | 20260507 02:32:37.832 | INFO | BytesWritten : 3302499467876 | 20260507 02:32:37.833 | INFO | NumEvents : 1588112135669 | 20260507 02:32:39.672 | INFO | Analyzing chunk frames = [6402048,6402560) | 20260507 02:32:39.672 | INFO | MemInfo: { | 20260507 02:32:39.672 | INFO | "gpu" : " 17 GB", | 20260507 02:32:39.672 | INFO | "huge" : "", | 20260507 02:32:39.672 | INFO | "huge_shared" : " 27 GB", | 20260507 02:32:39.672 | INFO | "malloc" : " 14 GB", | 20260507 02:32:39.672 | INFO | "pinned" : " 29 GB" | 20260507 02:32:39.672 | INFO | } | 20260507 02:32:39.672 | INFO | Perf report for the last 5.14258 seconds: | 20260507 02:32:39.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:32:39.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:39.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:39.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:39.673 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260507 02:32:39.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:39.673 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260507 02:32:39.673 | INFO | PRE_HQ : 59.4%, 1.9, 59.0ms, 0 | 20260507 02:32:39.673 | INFO | BAZWRITER : 62.2%, 3.4, 110.0ms, 0 | 20260507 02:32:39.673 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 02:32:39.673 | INFO | Uploaded 36GiB to GPU in 2.05247s (17.5399GiB/s) | 20260507 02:32:39.673 | INFO | Downloaded 38GiB to GPU in 2.17125s (17.5014GiB/s) | 20260507 02:32:44.812 | INFO | Analyzing chunk frames = [6402560,6403072) | 20260507 02:32:44.812 | INFO | MemInfo: { | 20260507 02:32:44.812 | INFO | "gpu" : " 17 GB", | 20260507 02:32:44.812 | INFO | "huge" : "", | 20260507 02:32:44.812 | INFO | "huge_shared" : " 27 GB", | 20260507 02:32:44.812 | INFO | "malloc" : " 14 GB", | 20260507 02:32:44.812 | INFO | "pinned" : " 29 GB" | 20260507 02:32:44.812 | INFO | } | 20260507 02:32:44.812 | INFO | Perf report for the last 5.13997 seconds: | 20260507 02:32:44.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:32:44.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:44.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:44.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:44.813 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:32:44.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:44.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:44.813 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:32:44.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:44.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:44.813 | INFO | Uploaded 36GiB to GPU in 2.03369s (17.7018GiB/s) | 20260507 02:32:44.813 | INFO | Downloaded 34GiB to GPU in 1.93988s (17.5269GiB/s) | 20260507 02:32:49.950 | INFO | Analyzing chunk frames = [6403072,6403584) | 20260507 02:32:49.950 | INFO | MemInfo: { | 20260507 02:32:49.950 | INFO | "gpu" : " 17 GB", | 20260507 02:32:49.950 | INFO | "huge" : "", | 20260507 02:32:49.950 | INFO | "huge_shared" : " 27 GB", | 20260507 02:32:49.950 | INFO | "malloc" : " 14 GB", | 20260507 02:32:49.950 | INFO | "pinned" : " 29 GB" | 20260507 02:32:49.950 | INFO | } | 20260507 02:32:49.950 | INFO | Perf report for the last 5.13755 seconds: | 20260507 02:32:49.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:32:49.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:49.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:49.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:49.950 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 02:32:49.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:49.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:49.950 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:32:49.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:49.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:49.950 | INFO | Uploaded 36GiB to GPU in 2.01433s (17.872GiB/s) | 20260507 02:32:49.950 | INFO | Downloaded 34GiB to GPU in 1.94045s (17.5217GiB/s) | 20260507 02:32:55.089 | INFO | Analyzing chunk frames = [6403584,6404096) | 20260507 02:32:55.089 | INFO | MemInfo: { | 20260507 02:32:55.089 | INFO | "gpu" : " 17 GB", | 20260507 02:32:55.089 | INFO | "huge" : "", | 20260507 02:32:55.089 | INFO | "huge_shared" : " 27 GB", | 20260507 02:32:55.089 | INFO | "malloc" : " 14 GB", | 20260507 02:32:55.089 | INFO | "pinned" : " 29 GB" | 20260507 02:32:55.089 | INFO | } | 20260507 02:32:55.089 | INFO | Perf report for the last 5.13882 seconds: | 20260507 02:32:55.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:32:55.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:55.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:55.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:55.089 | INFO | ANALYSIS : 60.6%, 2.9, 93.6ms, 0 | 20260507 02:32:55.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:55.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:55.089 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 02:32:55.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:55.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:32:55.089 | INFO | Uploaded 36GiB to GPU in 2.0226s (17.7989GiB/s) | 20260507 02:32:55.089 | INFO | Downloaded 34GiB to GPU in 1.93935s (17.5316GiB/s) | 20260507 02:33:00.224 | INFO | Analyzing chunk frames = [6404096,6404608) | 20260507 02:33:00.224 | INFO | MemInfo: { | 20260507 02:33:00.224 | INFO | "gpu" : " 17 GB", | 20260507 02:33:00.224 | INFO | "huge" : "", | 20260507 02:33:00.224 | INFO | "huge_shared" : " 27 GB", | 20260507 02:33:00.224 | INFO | "malloc" : " 14 GB", | 20260507 02:33:00.224 | INFO | "pinned" : " 29 GB" | 20260507 02:33:00.224 | INFO | } | 20260507 02:33:00.224 | INFO | Perf report for the last 5.13529 seconds: | 20260507 02:33:00.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:33:00.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:00.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:00.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:00.224 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 02:33:00.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:00.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:00.224 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:33:00.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:00.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:00.224 | INFO | Uploaded 36GiB to GPU in 2.0204s (17.8182GiB/s) | 20260507 02:33:00.224 | INFO | Downloaded 34GiB to GPU in 1.93812s (17.5427GiB/s) | 20260507 02:33:05.363 | INFO | Analyzing chunk frames = [6404608,6405120) | 20260507 02:33:05.363 | INFO | MemInfo: { | 20260507 02:33:05.363 | INFO | "gpu" : " 17 GB", | 20260507 02:33:05.364 | INFO | "huge" : "", | 20260507 02:33:05.364 | INFO | "huge_shared" : " 27 GB", | 20260507 02:33:05.364 | INFO | "malloc" : " 14 GB", | 20260507 02:33:05.364 | INFO | "pinned" : " 29 GB" | 20260507 02:33:05.364 | INFO | } | 20260507 02:33:05.364 | INFO | Perf report for the last 5.13943 seconds: | 20260507 02:33:05.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:33:05.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:05.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:05.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:05.364 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 02:33:05.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:05.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:05.364 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:33:05.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:05.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:05.364 | INFO | Uploaded 36GiB to GPU in 2.04792s (17.5788GiB/s) | 20260507 02:33:05.364 | INFO | Downloaded 34GiB to GPU in 1.94551s (17.4762GiB/s) | 20260507 02:33:10.505 | INFO | Analyzing chunk frames = [6405120,6405632) | 20260507 02:33:10.505 | INFO | MemInfo: { | 20260507 02:33:10.505 | INFO | "gpu" : " 17 GB", | 20260507 02:33:10.505 | INFO | "huge" : "", | 20260507 02:33:10.505 | INFO | "huge_shared" : " 27 GB", | 20260507 02:33:10.505 | INFO | "malloc" : " 14 GB", | 20260507 02:33:10.505 | INFO | "pinned" : " 29 GB" | 20260507 02:33:10.505 | INFO | } | 20260507 02:33:10.505 | INFO | Perf report for the last 5.14158 seconds: | 20260507 02:33:10.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:33:10.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:10.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:10.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:10.505 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:33:10.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:10.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:10.505 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:33:10.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:10.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:10.505 | INFO | Uploaded 36GiB to GPU in 1.96753s (18.2971GiB/s) | 20260507 02:33:10.505 | INFO | Downloaded 34GiB to GPU in 1.99744s (17.0218GiB/s) | 20260507 02:33:15.644 | INFO | Analyzing chunk frames = [6405632,6406144) | 20260507 02:33:15.644 | INFO | MemInfo: { | 20260507 02:33:15.644 | INFO | "gpu" : " 17 GB", | 20260507 02:33:15.644 | INFO | "huge" : "", | 20260507 02:33:15.644 | INFO | "huge_shared" : " 27 GB", | 20260507 02:33:15.644 | INFO | "malloc" : " 14 GB", | 20260507 02:33:15.644 | INFO | "pinned" : " 29 GB" | 20260507 02:33:15.644 | INFO | } | 20260507 02:33:15.644 | INFO | Perf report for the last 5.13876 seconds: | 20260507 02:33:15.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:33:15.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:15.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:15.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:15.644 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:33:15.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:15.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:15.644 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 02:33:15.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:15.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:15.644 | INFO | Uploaded 36GiB to GPU in 1.99818s (18.0164GiB/s) | 20260507 02:33:15.644 | INFO | Downloaded 34GiB to GPU in 1.96751s (17.2807GiB/s) | 20260507 02:33:15.644 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:33:15.644 | INFO | Total mem usage: 18 GiB | 20260507 02:33:15.644 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:33:15.644 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 02:33:15.645 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:33:15.645 | INFO | High Water Breakdown: | 20260507 02:33:15.645 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:33:15.645 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:33:15.645 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:33:15.645 | INFO | ------------------------------------------ | 20260507 02:33:15.645 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:33:15.645 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:33:15.645 | INFO | Total mem usage: 26.5 GiB | 20260507 02:33:15.645 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:33:15.645 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 02:33:15.645 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 02:33:15.645 | INFO | High Water Breakdown: | 20260507 02:33:15.645 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:33:15.645 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:33:15.645 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:33:15.645 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:33:15.645 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:33:15.645 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:33:15.645 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:33:15.645 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:33:15.645 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:33:15.645 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:33:15.645 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:33:15.645 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:33:15.645 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:33:15.645 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:33:15.645 | INFO | ------------------------------------------ | 20260507 02:33:15.645 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:33:15.645 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:33:15.645 | INFO | Total mem usage: 25.6 GiB | 20260507 02:33:15.645 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:33:15.645 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:33:15.645 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:33:15.645 | INFO | High Water Breakdown: | 20260507 02:33:15.645 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:33:15.645 | INFO | ------------------------------------------ | 20260507 02:33:15.645 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:33:15.645 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:33:15.645 | INFO | Total mem usage: 15.7 GiB | 20260507 02:33:15.645 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:33:15.645 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:33:15.645 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:33:15.645 | INFO | High Water Breakdown: | 20260507 02:33:15.645 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:33:15.645 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:33:15.645 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:33:15.645 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:33:15.645 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:33:15.645 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:33:15.645 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:33:15.645 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:33:15.645 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:33:15.645 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:33:15.645 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:33:15.645 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:33:15.645 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:33:15.645 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:33:15.645 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:33:15.645 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:33:15.645 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:33:15.645 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:33:15.645 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:33:15.645 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:33:15.645 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:33:15.645 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:33:15.645 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:33:15.645 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:33:15.646 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:33:15.646 | INFO | ------------------------------------------ | 20260507 02:33:15.646 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:33:15.646 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:33:15.646 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:33:15.646 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:33:15.646 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.54e+01, 1.34e+01 / 1.98e+00 | 20260507 02:33:15.646 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.01e-01, 1.04e-02 / 5.02e-03 | 20260507 02:33:15.646 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.43e+00, 5.17e-01 / 3.05e-01 | 20260507 02:33:15.646 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.45e+00, 1.69e-01 / 1.32e-01 | 20260507 02:33:15.646 | INFO | PerfReport: 16.20%, DME -- 3.99e+01 / 4.52e+01, 4.18e+00 / 2.08e+00 | 20260507 02:33:15.646 | INFO | PerfReport: 57.48%, FrameLabeling -- 1.77e+01 / 1.98e+01, 1.23e+00 / 1.31e+00 | 20260507 02:33:15.646 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.16e+00 / 2.72e+00, 3.87e-01 / 2.15e-01 | 20260507 02:33:15.646 | INFO | PerfReport: 3.08%, Metrics -- 9.48e-01 / 1.29e+00, 2.52e-01 / 1.53e-01 | 20260507 02:33:15.646 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.21e-01 / 3.77e-01, 5.04e-02 / 2.31e-02 | 20260507 02:33:15.646 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:33:15.646 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:33:15.646 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:33:15.646 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 02:33:15.646 | INFO | PerfReport: 21.69%, Upload -- 2.09e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 02:33:15.646 | INFO | PerfReport: 21.33%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260507 02:33:15.646 | INFO | PerfReport: 31.93%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 02:33:15.646 | INFO | PerfReport: 1.50%, LockUpload -- 1.44e+00 / 0.00e+00, 5.01e+00 / 0.00e+00 | 20260507 02:33:15.646 | INFO | PerfReport: 1.35%, LockDownload -- 1.30e+00 / 0.00e+00, 3.20e+00 / 0.00e+00 | 20260507 02:33:15.646 | INFO | PerfReport: 22.19%, LockCompute -- 2.14e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 02:33:15.646 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260507 02:33:15.741 | INFO | Starting a round of Baz IO | 20260507 02:33:18.858 | INFO | Populating RTMetrics report | 20260507 02:33:18.858 | INFO | RTMetrics report populated | 20260507 02:33:18.859 | INFO | Opening tmp RTMetrics json file | 20260507 02:33:18.860 | INFO | Serializing RTMetrics to json | 20260507 02:33:18.864 | INFO | Serialization done, swapping tmp file now... | 20260507 02:33:18.873 | INFO | RTMetrics done | 20260507 02:33:18.952 | INFO | Finished a round of Baz IO: Wrote 1.78 GiB in 3.211 seconds | 20260507 02:33:18.952 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.99% / 0.00% / 67.01% | 20260507 02:33:18.952 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.233579% / 3.97364e-06% / 5.16574e-05% | 20260507 02:33:18.952 | INFO | Baz IO overall: | 20260507 02:33:18.952 | INFO | BytesWritten : 3304410383857 | 20260507 02:33:18.952 | INFO | NumEvents : 1589015759388 | 20260507 02:33:20.784 | INFO | Analyzing chunk frames = [6406144,6406656) | 20260507 02:33:20.784 | INFO | MemInfo: { | 20260507 02:33:20.784 | INFO | "gpu" : " 17 GB", | 20260507 02:33:20.784 | INFO | "huge" : "", | 20260507 02:33:20.784 | INFO | "huge_shared" : " 27 GB", | 20260507 02:33:20.784 | INFO | "malloc" : " 14 GB", | 20260507 02:33:20.784 | INFO | "pinned" : " 29 GB" | 20260507 02:33:20.784 | INFO | } | 20260507 02:33:20.784 | INFO | Perf report for the last 5.14039 seconds: | 20260507 02:33:20.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:33:20.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:20.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:20.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:20.784 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 02:33:20.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:20.784 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260507 02:33:20.784 | INFO | PRE_HQ : 59.7%, 1.9, 59.4ms, 0 | 20260507 02:33:20.784 | INFO | BAZWRITER : 62.1%, 3.4, 110.0ms, 0 | 20260507 02:33:20.784 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:33:20.785 | INFO | Uploaded 36GiB to GPU in 2.02697s (17.7605GiB/s) | 20260507 02:33:20.785 | INFO | Downloaded 38GiB to GPU in 2.18022s (17.4294GiB/s) | 20260507 02:33:25.926 | INFO | Analyzing chunk frames = [6406656,6407168) | 20260507 02:33:25.926 | INFO | MemInfo: { | 20260507 02:33:25.926 | INFO | "gpu" : " 17 GB", | 20260507 02:33:25.926 | INFO | "huge" : "", | 20260507 02:33:25.926 | INFO | "huge_shared" : " 27 GB", | 20260507 02:33:25.926 | INFO | "malloc" : " 14 GB", | 20260507 02:33:25.926 | INFO | "pinned" : " 29 GB" | 20260507 02:33:25.926 | INFO | } | 20260507 02:33:25.926 | INFO | Perf report for the last 5.14163 seconds: | 20260507 02:33:25.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:33:25.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:25.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:25.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:25.926 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:33:25.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:25.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:25.926 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 02:33:25.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:25.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:25.927 | INFO | Uploaded 36GiB to GPU in 2.02908s (17.742GiB/s) | 20260507 02:33:25.927 | INFO | Downloaded 34GiB to GPU in 1.93654s (17.5571GiB/s) | 20260507 02:33:31.066 | INFO | Analyzing chunk frames = [6407168,6407680) | 20260507 02:33:31.066 | INFO | MemInfo: { | 20260507 02:33:31.066 | INFO | "gpu" : " 17 GB", | 20260507 02:33:31.066 | INFO | "huge" : "", | 20260507 02:33:31.066 | INFO | "huge_shared" : " 27 GB", | 20260507 02:33:31.066 | INFO | "malloc" : " 14 GB", | 20260507 02:33:31.066 | INFO | "pinned" : " 29 GB" | 20260507 02:33:31.066 | INFO | } | 20260507 02:33:31.066 | INFO | Perf report for the last 5.14029 seconds: | 20260507 02:33:31.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:33:31.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:31.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:31.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:31.066 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:33:31.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:31.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:31.066 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:33:31.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:31.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:31.067 | INFO | Uploaded 36GiB to GPU in 2.0105s (17.906GiB/s) | 20260507 02:33:31.067 | INFO | Downloaded 34GiB to GPU in 1.93866s (17.5379GiB/s) | 20260507 02:33:36.201 | INFO | Analyzing chunk frames = [6407680,6408192) | 20260507 02:33:36.201 | INFO | MemInfo: { | 20260507 02:33:36.201 | INFO | "gpu" : " 17 GB", | 20260507 02:33:36.201 | INFO | "huge" : "", | 20260507 02:33:36.201 | INFO | "huge_shared" : " 27 GB", | 20260507 02:33:36.201 | INFO | "malloc" : " 14 GB", | 20260507 02:33:36.201 | INFO | "pinned" : " 29 GB" | 20260507 02:33:36.202 | INFO | } | 20260507 02:33:36.202 | INFO | Perf report for the last 5.13522 seconds: | 20260507 02:33:36.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:33:36.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:36.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:36.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:36.202 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 02:33:36.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:36.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:36.202 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:33:36.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:36.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:36.202 | INFO | Uploaded 36GiB to GPU in 2.00771s (17.9308GiB/s) | 20260507 02:33:36.202 | INFO | Downloaded 34GiB to GPU in 1.92686s (17.6453GiB/s) | 20260507 02:33:41.341 | INFO | Analyzing chunk frames = [6408192,6408704) | 20260507 02:33:41.341 | INFO | MemInfo: { | 20260507 02:33:41.341 | INFO | "gpu" : " 17 GB", | 20260507 02:33:41.341 | INFO | "huge" : "", | 20260507 02:33:41.341 | INFO | "huge_shared" : " 27 GB", | 20260507 02:33:41.341 | INFO | "malloc" : " 14 GB", | 20260507 02:33:41.341 | INFO | "pinned" : " 29 GB" | 20260507 02:33:41.341 | INFO | } | 20260507 02:33:41.341 | INFO | Perf report for the last 5.13983 seconds: | 20260507 02:33:41.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:33:41.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:41.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:41.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:41.342 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 02:33:41.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:41.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:41.342 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:33:41.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:41.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:41.342 | INFO | Uploaded 36GiB to GPU in 2.02004s (17.8215GiB/s) | 20260507 02:33:41.342 | INFO | Downloaded 34GiB to GPU in 1.94126s (17.5144GiB/s) | 20260507 02:33:46.488 | INFO | Analyzing chunk frames = [6408704,6409216) | 20260507 02:33:46.488 | INFO | MemInfo: { | 20260507 02:33:46.488 | INFO | "gpu" : " 17 GB", | 20260507 02:33:46.488 | INFO | "huge" : "", | 20260507 02:33:46.488 | INFO | "huge_shared" : " 27 GB", | 20260507 02:33:46.488 | INFO | "malloc" : " 14 GB", | 20260507 02:33:46.488 | INFO | "pinned" : " 29 GB" | 20260507 02:33:46.488 | INFO | } | 20260507 02:33:46.488 | INFO | Perf report for the last 5.14649 seconds: | 20260507 02:33:46.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:33:46.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:46.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:46.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:46.488 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 02:33:46.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:46.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:46.488 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:33:46.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:46.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:46.488 | INFO | Uploaded 36GiB to GPU in 2.05813s (17.4916GiB/s) | 20260507 02:33:46.488 | INFO | Downloaded 34GiB to GPU in 1.955s (17.3913GiB/s) | 20260507 02:33:51.627 | INFO | Analyzing chunk frames = [6409216,6409728) | 20260507 02:33:51.627 | INFO | MemInfo: { | 20260507 02:33:51.627 | INFO | "gpu" : " 17 GB", | 20260507 02:33:51.627 | INFO | "huge" : "", | 20260507 02:33:51.627 | INFO | "huge_shared" : " 27 GB", | 20260507 02:33:51.627 | INFO | "malloc" : " 14 GB", | 20260507 02:33:51.627 | INFO | "pinned" : " 29 GB" | 20260507 02:33:51.627 | INFO | } | 20260507 02:33:51.627 | INFO | Perf report for the last 5.13932 seconds: | 20260507 02:33:51.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:33:51.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:51.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:51.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:51.627 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:33:51.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:51.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:51.627 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:33:51.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:51.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:51.627 | INFO | Uploaded 36GiB to GPU in 2.01462s (17.8693GiB/s) | 20260507 02:33:51.627 | INFO | Downloaded 34GiB to GPU in 1.97346s (17.2286GiB/s) | 20260507 02:33:56.766 | INFO | Analyzing chunk frames = [6409728,6410240) | 20260507 02:33:56.766 | INFO | MemInfo: { | 20260507 02:33:56.766 | INFO | "gpu" : " 17 GB", | 20260507 02:33:56.766 | INFO | "huge" : "", | 20260507 02:33:56.766 | INFO | "huge_shared" : " 27 GB", | 20260507 02:33:56.766 | INFO | "malloc" : " 14 GB", | 20260507 02:33:56.766 | INFO | "pinned" : " 29 GB" | 20260507 02:33:56.766 | INFO | } | 20260507 02:33:56.766 | INFO | Perf report for the last 5.13863 seconds: | 20260507 02:33:56.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:33:56.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:56.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:56.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:56.766 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 02:33:56.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:56.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:56.766 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 02:33:56.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:56.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:33:56.766 | INFO | Uploaded 36GiB to GPU in 1.9923s (18.0695GiB/s) | 20260507 02:33:56.766 | INFO | Downloaded 34GiB to GPU in 1.97373s (17.2262GiB/s) | 20260507 02:33:56.863 | INFO | Starting a round of Baz IO | 20260507 02:33:59.978 | INFO | Populating RTMetrics report | 20260507 02:33:59.978 | INFO | RTMetrics report populated | 20260507 02:33:59.978 | INFO | Opening tmp RTMetrics json file | 20260507 02:33:59.979 | INFO | Serializing RTMetrics to json | 20260507 02:33:59.980 | INFO | Serialization done, swapping tmp file now... | 20260507 02:33:59.980 | INFO | RTMetrics done | 20260507 02:34:00.080 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.217 seconds | 20260507 02:34:00.080 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.99% / 0.00% / 67.01% | 20260507 02:34:00.080 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.23342% / 7.94729e-06% / 9.53674e-05% | 20260507 02:34:00.081 | INFO | Baz IO overall: | 20260507 02:34:00.081 | INFO | BytesWritten : 3306319679650 | 20260507 02:34:00.081 | INFO | NumEvents : 1589918195216 | 20260507 02:34:01.900 | INFO | Analyzing chunk frames = [6410240,6410752) | 20260507 02:34:01.900 | INFO | MemInfo: { | 20260507 02:34:01.900 | INFO | "gpu" : " 17 GB", | 20260507 02:34:01.900 | INFO | "huge" : "", | 20260507 02:34:01.900 | INFO | "huge_shared" : " 27 GB", | 20260507 02:34:01.900 | INFO | "malloc" : " 14 GB", | 20260507 02:34:01.900 | INFO | "pinned" : " 29 GB" | 20260507 02:34:01.900 | INFO | } | 20260507 02:34:01.900 | INFO | Perf report for the last 5.13429 seconds: | 20260507 02:34:01.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:34:01.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:01.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:01.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:01.900 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260507 02:34:01.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:01.900 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260507 02:34:01.900 | INFO | PRE_HQ : 60.2%, 1.9, 60.1ms, 0 | 20260507 02:34:01.900 | INFO | BAZWRITER : 62.3%, 3.5, 111.2ms, 0 | 20260507 02:34:01.900 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:34:01.900 | INFO | Uploaded 36GiB to GPU in 2.03814s (17.6632GiB/s) | 20260507 02:34:01.901 | INFO | Downloaded 38GiB to GPU in 2.18437s (17.3963GiB/s) | 20260507 02:34:07.038 | INFO | Analyzing chunk frames = [6410752,6411264) | 20260507 02:34:07.038 | INFO | MemInfo: { | 20260507 02:34:07.038 | INFO | "gpu" : " 17 GB", | 20260507 02:34:07.038 | INFO | "huge" : "", | 20260507 02:34:07.038 | INFO | "huge_shared" : " 27 GB", | 20260507 02:34:07.038 | INFO | "malloc" : " 14 GB", | 20260507 02:34:07.038 | INFO | "pinned" : " 29 GB" | 20260507 02:34:07.038 | INFO | } | 20260507 02:34:07.038 | INFO | Perf report for the last 5.13756 seconds: | 20260507 02:34:07.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:34:07.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:07.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:07.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:07.038 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:34:07.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:07.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:07.038 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 02:34:07.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:07.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:07.038 | INFO | Uploaded 36GiB to GPU in 2.03577s (17.6837GiB/s) | 20260507 02:34:07.038 | INFO | Downloaded 34GiB to GPU in 1.94092s (17.5175GiB/s) | 20260507 02:34:12.174 | INFO | Analyzing chunk frames = [6411264,6411776) | 20260507 02:34:12.174 | INFO | MemInfo: { | 20260507 02:34:12.174 | INFO | "gpu" : " 17 GB", | 20260507 02:34:12.174 | INFO | "huge" : "", | 20260507 02:34:12.174 | INFO | "huge_shared" : " 27 GB", | 20260507 02:34:12.174 | INFO | "malloc" : " 14 GB", | 20260507 02:34:12.174 | INFO | "pinned" : " 29 GB" | 20260507 02:34:12.174 | INFO | } | 20260507 02:34:12.174 | INFO | Perf report for the last 5.13642 seconds: | 20260507 02:34:12.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:34:12.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:12.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:12.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:12.174 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 02:34:12.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:12.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:12.174 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:34:12.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:12.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:12.174 | INFO | Uploaded 36GiB to GPU in 1.99462s (18.0485GiB/s) | 20260507 02:34:12.174 | INFO | Downloaded 34GiB to GPU in 1.94057s (17.5207GiB/s) | 20260507 02:34:17.322 | INFO | Analyzing chunk frames = [6411776,6412288) | 20260507 02:34:17.322 | INFO | MemInfo: { | 20260507 02:34:17.322 | INFO | "gpu" : " 17 GB", | 20260507 02:34:17.322 | INFO | "huge" : "", | 20260507 02:34:17.322 | INFO | "huge_shared" : " 27 GB", | 20260507 02:34:17.322 | INFO | "malloc" : " 14 GB", | 20260507 02:34:17.322 | INFO | "pinned" : " 29 GB" | 20260507 02:34:17.322 | INFO | } | 20260507 02:34:17.322 | INFO | Perf report for the last 5.14757 seconds: | 20260507 02:34:17.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:34:17.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:17.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:17.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:17.322 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 02:34:17.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:17.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:17.322 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 02:34:17.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:17.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:17.322 | INFO | Uploaded 36GiB to GPU in 1.96346s (18.335GiB/s) | 20260507 02:34:17.322 | INFO | Downloaded 34GiB to GPU in 1.90742s (17.8251GiB/s) | 20260507 02:34:22.461 | INFO | Analyzing chunk frames = [6412288,6412800) | 20260507 02:34:22.461 | INFO | MemInfo: { | 20260507 02:34:22.461 | INFO | "gpu" : " 17 GB", | 20260507 02:34:22.461 | INFO | "huge" : "", | 20260507 02:34:22.461 | INFO | "huge_shared" : " 27 GB", | 20260507 02:34:22.461 | INFO | "malloc" : " 14 GB", | 20260507 02:34:22.461 | INFO | "pinned" : " 29 GB" | 20260507 02:34:22.461 | INFO | } | 20260507 02:34:22.461 | INFO | Perf report for the last 5.1394 seconds: | 20260507 02:34:22.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:34:22.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:22.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:22.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:22.461 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 02:34:22.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:22.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:22.461 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:34:22.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:22.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:22.461 | INFO | Uploaded 36GiB to GPU in 2.0172s (17.8465GiB/s) | 20260507 02:34:22.461 | INFO | Downloaded 34GiB to GPU in 1.93775s (17.5461GiB/s) | 20260507 02:34:27.596 | INFO | Analyzing chunk frames = [6412800,6413312) | 20260507 02:34:27.596 | INFO | MemInfo: { | 20260507 02:34:27.596 | INFO | "gpu" : " 17 GB", | 20260507 02:34:27.596 | INFO | "huge" : "", | 20260507 02:34:27.596 | INFO | "huge_shared" : " 27 GB", | 20260507 02:34:27.596 | INFO | "malloc" : " 14 GB", | 20260507 02:34:27.596 | INFO | "pinned" : " 29 GB" | 20260507 02:34:27.596 | INFO | } | 20260507 02:34:27.596 | INFO | Perf report for the last 5.13492 seconds: | 20260507 02:34:27.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:34:27.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:27.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:27.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:27.596 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 02:34:27.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:27.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:27.596 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:34:27.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:27.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:27.596 | INFO | Uploaded 36GiB to GPU in 2.0481s (17.5773GiB/s) | 20260507 02:34:27.596 | INFO | Downloaded 34GiB to GPU in 1.96997s (17.2591GiB/s) | 20260507 02:34:32.735 | INFO | Analyzing chunk frames = [6413312,6413824) | 20260507 02:34:32.735 | INFO | MemInfo: { | 20260507 02:34:32.735 | INFO | "gpu" : " 17 GB", | 20260507 02:34:32.735 | INFO | "huge" : "", | 20260507 02:34:32.735 | INFO | "huge_shared" : " 27 GB", | 20260507 02:34:32.735 | INFO | "malloc" : " 14 GB", | 20260507 02:34:32.735 | INFO | "pinned" : " 29 GB" | 20260507 02:34:32.735 | INFO | } | 20260507 02:34:32.735 | INFO | Perf report for the last 5.13861 seconds: | 20260507 02:34:32.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:34:32.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:32.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:32.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:32.735 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:34:32.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:32.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:32.735 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 02:34:32.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:32.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:32.735 | INFO | Uploaded 36GiB to GPU in 1.98649s (18.1224GiB/s) | 20260507 02:34:32.735 | INFO | Downloaded 34GiB to GPU in 1.98584s (17.1212GiB/s) | 20260507 02:34:37.876 | INFO | Analyzing chunk frames = [6413824,6414336) | 20260507 02:34:37.876 | INFO | MemInfo: { | 20260507 02:34:37.876 | INFO | "gpu" : " 17 GB", | 20260507 02:34:37.876 | INFO | "huge" : "", | 20260507 02:34:37.876 | INFO | "huge_shared" : " 27 GB", | 20260507 02:34:37.876 | INFO | "malloc" : " 14 GB", | 20260507 02:34:37.876 | INFO | "pinned" : " 29 GB" | 20260507 02:34:37.876 | INFO | } | 20260507 02:34:37.876 | INFO | Perf report for the last 5.14148 seconds: | 20260507 02:34:37.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:34:37.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:37.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:37.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:37.876 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:34:37.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:37.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:37.876 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:34:37.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:37.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:37.877 | INFO | Uploaded 36GiB to GPU in 1.98786s (18.1099GiB/s) | 20260507 02:34:37.877 | INFO | Downloaded 34GiB to GPU in 1.97128s (17.2477GiB/s) | 20260507 02:34:37.877 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:34:37.877 | INFO | Total mem usage: 18 GiB | 20260507 02:34:37.877 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:34:37.877 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:34:37.877 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:34:37.877 | INFO | High Water Breakdown: | 20260507 02:34:37.877 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:34:37.877 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:34:37.877 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:34:37.877 | INFO | ------------------------------------------ | 20260507 02:34:37.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:34:37.877 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:34:37.877 | INFO | Total mem usage: 26.5 GiB | 20260507 02:34:37.877 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:34:37.877 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:34:37.877 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:34:37.877 | INFO | High Water Breakdown: | 20260507 02:34:37.877 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:34:37.877 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:34:37.877 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:34:37.877 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:34:37.877 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:34:37.877 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:34:37.877 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:34:37.877 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:34:37.877 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:34:37.877 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:34:37.877 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:34:37.877 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:34:37.877 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:34:37.877 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:34:37.877 | INFO | ------------------------------------------ | 20260507 02:34:37.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:34:37.877 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:34:37.877 | INFO | Total mem usage: 25.6 GiB | 20260507 02:34:37.877 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:34:37.877 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:34:37.877 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:34:37.877 | INFO | High Water Breakdown: | 20260507 02:34:37.877 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:34:37.877 | INFO | ------------------------------------------ | 20260507 02:34:37.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:34:37.877 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:34:37.877 | INFO | Total mem usage: 15.7 GiB | 20260507 02:34:37.877 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:34:37.877 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:34:37.877 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 02:34:37.877 | INFO | High Water Breakdown: | 20260507 02:34:37.877 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:34:37.877 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:34:37.877 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:34:37.877 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:34:37.877 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:34:37.877 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:34:37.877 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:34:37.877 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:34:37.877 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:34:37.877 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:34:37.877 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:34:37.877 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:34:37.877 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:34:37.877 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:34:37.877 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:34:37.877 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:34:37.877 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:34:37.877 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:34:37.877 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:34:37.877 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:34:37.877 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:34:37.877 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:34:37.877 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:34:37.877 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:34:37.877 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:34:37.877 | INFO | ------------------------------------------ | 20260507 02:34:37.877 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:34:37.877 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:34:37.877 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:34:37.877 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:34:37.877 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.54e+01, 1.35e+01 / 1.82e+00 | 20260507 02:34:37.877 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.01e-01, 1.07e-02 / 5.11e-03 | 20260507 02:34:37.877 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.48e+00, 5.10e-01 / 2.59e-01 | 20260507 02:34:37.877 | INFO | PerfReport: 3.22%, Histogram -- 9.91e-01 / 1.49e+00, 1.68e-01 / 1.05e-01 | 20260507 02:34:37.877 | INFO | PerfReport: 16.12%, DME -- 3.97e+01 / 4.47e+01, 4.05e+00 / 2.06e+00 | 20260507 02:34:37.877 | INFO | PerfReport: 57.61%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.30e+00 / 1.55e+00 | 20260507 02:34:37.877 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.15e+00 / 2.70e+00, 3.85e-01 / 2.17e-01 | 20260507 02:34:37.877 | INFO | PerfReport: 3.07%, Metrics -- 9.44e-01 / 1.28e+00, 2.52e-01 / 1.09e-01 | 20260507 02:34:37.877 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.21e-01 / 3.74e-01, 5.12e-02 / 2.01e-02 | 20260507 02:34:37.877 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:34:37.877 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:34:37.877 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:34:37.877 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 02:34:37.877 | INFO | PerfReport: 21.77%, Upload -- 2.10e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260507 02:34:37.878 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260507 02:34:37.878 | INFO | PerfReport: 31.92%, Compute -- 3.08e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 02:34:37.878 | INFO | PerfReport: 1.46%, LockUpload -- 1.41e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 02:34:37.878 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260507 02:34:37.878 | INFO | PerfReport: 22.16%, LockCompute -- 2.14e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 02:34:37.878 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260507 02:34:37.973 | INFO | Starting a round of Baz IO | 20260507 02:34:41.082 | INFO | Populating RTMetrics report | 20260507 02:34:41.082 | INFO | RTMetrics report populated | 20260507 02:34:41.082 | INFO | Opening tmp RTMetrics json file | 20260507 02:34:41.100 | INFO | Serializing RTMetrics to json | 20260507 02:34:41.101 | INFO | Serialization done, swapping tmp file now... | 20260507 02:34:41.102 | INFO | RTMetrics done | 20260507 02:34:41.173 | INFO | Finished a round of Baz IO: Wrote 1.777 GiB in 3.199 seconds | 20260507 02:34:41.173 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.99% / 0.00% / 67.01% | 20260507 02:34:41.173 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.231675% / 3.97364e-06% / 5.96046e-05% | 20260507 02:34:41.173 | INFO | Baz IO overall: | 20260507 02:34:41.173 | INFO | BytesWritten : 3308227318954 | 20260507 02:34:41.173 | INFO | NumEvents : 1590819306388 | 20260507 02:34:43.012 | INFO | Analyzing chunk frames = [6414336,6414848) | 20260507 02:34:43.012 | INFO | MemInfo: { | 20260507 02:34:43.012 | INFO | "gpu" : " 17 GB", | 20260507 02:34:43.012 | INFO | "huge" : "", | 20260507 02:34:43.012 | INFO | "huge_shared" : " 27 GB", | 20260507 02:34:43.012 | INFO | "malloc" : " 14 GB", | 20260507 02:34:43.012 | INFO | "pinned" : " 29 GB" | 20260507 02:34:43.012 | INFO | } | 20260507 02:34:43.013 | INFO | Perf report for the last 5.13626 seconds: | 20260507 02:34:43.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:34:43.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:43.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:43.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:43.013 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 02:34:43.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:43.013 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 02:34:43.013 | INFO | PRE_HQ : 59.4%, 1.8, 55.7ms, 0 | 20260507 02:34:43.013 | INFO | BAZWRITER : 61.9%, 3.3, 105.9ms, 0 | 20260507 02:34:43.013 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:34:43.013 | INFO | Uploaded 36GiB to GPU in 2.04289s (17.6221GiB/s) | 20260507 02:34:43.013 | INFO | Downloaded 38GiB to GPU in 2.18419s (17.3977GiB/s) | 20260507 02:34:48.154 | INFO | Analyzing chunk frames = [6414848,6415360) | 20260507 02:34:48.154 | INFO | MemInfo: { | 20260507 02:34:48.154 | INFO | "gpu" : " 17 GB", | 20260507 02:34:48.154 | INFO | "huge" : "", | 20260507 02:34:48.154 | INFO | "huge_shared" : " 27 GB", | 20260507 02:34:48.154 | INFO | "malloc" : " 14 GB", | 20260507 02:34:48.154 | INFO | "pinned" : " 29 GB" | 20260507 02:34:48.154 | INFO | } | 20260507 02:34:48.154 | INFO | Perf report for the last 5.1417 seconds: | 20260507 02:34:48.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:34:48.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:48.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:48.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:48.154 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:34:48.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:48.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:48.154 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:34:48.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:48.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:48.155 | INFO | Uploaded 36GiB to GPU in 2.02352s (17.7907GiB/s) | 20260507 02:34:48.155 | INFO | Downloaded 34GiB to GPU in 1.93632s (17.559GiB/s) | 20260507 02:34:53.295 | INFO | Analyzing chunk frames = [6415360,6415872) | 20260507 02:34:53.295 | INFO | MemInfo: { | 20260507 02:34:53.295 | INFO | "gpu" : " 17 GB", | 20260507 02:34:53.295 | INFO | "huge" : "", | 20260507 02:34:53.295 | INFO | "huge_shared" : " 27 GB", | 20260507 02:34:53.295 | INFO | "malloc" : " 14 GB", | 20260507 02:34:53.295 | INFO | "pinned" : " 29 GB" | 20260507 02:34:53.295 | INFO | } | 20260507 02:34:53.295 | INFO | Perf report for the last 5.14118 seconds: | 20260507 02:34:53.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:34:53.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:53.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:53.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:53.296 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 02:34:53.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:53.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:53.296 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:34:53.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:53.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:53.296 | INFO | Uploaded 36GiB to GPU in 2.01381s (17.8766GiB/s) | 20260507 02:34:53.296 | INFO | Downloaded 34GiB to GPU in 1.9347s (17.5738GiB/s) | 20260507 02:34:58.442 | INFO | Analyzing chunk frames = [6415872,6416384) | 20260507 02:34:58.442 | INFO | MemInfo: { | 20260507 02:34:58.442 | INFO | "gpu" : " 17 GB", | 20260507 02:34:58.442 | INFO | "huge" : "", | 20260507 02:34:58.442 | INFO | "huge_shared" : " 27 GB", | 20260507 02:34:58.442 | INFO | "malloc" : " 14 GB", | 20260507 02:34:58.442 | INFO | "pinned" : " 29 GB" | 20260507 02:34:58.442 | INFO | } | 20260507 02:34:58.442 | INFO | Perf report for the last 5.1464 seconds: | 20260507 02:34:58.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:34:58.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:58.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:58.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:58.442 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 02:34:58.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:58.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:58.442 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:34:58.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:58.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:34:58.442 | INFO | Uploaded 36GiB to GPU in 1.99166s (18.0753GiB/s) | 20260507 02:34:58.442 | INFO | Downloaded 34GiB to GPU in 1.91s (17.8011GiB/s) | 20260507 02:35:03.581 | INFO | Analyzing chunk frames = [6416384,6416896) | 20260507 02:35:03.581 | INFO | MemInfo: { | 20260507 02:35:03.581 | INFO | "gpu" : " 17 GB", | 20260507 02:35:03.581 | INFO | "huge" : "", | 20260507 02:35:03.581 | INFO | "huge_shared" : " 27 GB", | 20260507 02:35:03.581 | INFO | "malloc" : " 14 GB", | 20260507 02:35:03.581 | INFO | "pinned" : " 29 GB" | 20260507 02:35:03.581 | INFO | } | 20260507 02:35:03.581 | INFO | Perf report for the last 5.13955 seconds: | 20260507 02:35:03.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:35:03.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:03.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:03.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:03.581 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 02:35:03.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:03.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:03.581 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:35:03.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:03.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:03.582 | INFO | Uploaded 36GiB to GPU in 2.04025s (17.6449GiB/s) | 20260507 02:35:03.582 | INFO | Downloaded 34GiB to GPU in 1.93825s (17.5416GiB/s) | 20260507 02:35:08.710 | INFO | Analyzing chunk frames = [6416896,6417408) | 20260507 02:35:08.710 | INFO | MemInfo: { | 20260507 02:35:08.710 | INFO | "gpu" : " 17 GB", | 20260507 02:35:08.710 | INFO | "huge" : "", | 20260507 02:35:08.710 | INFO | "huge_shared" : " 27 GB", | 20260507 02:35:08.710 | INFO | "malloc" : " 14 GB", | 20260507 02:35:08.710 | INFO | "pinned" : " 29 GB" | 20260507 02:35:08.710 | INFO | } | 20260507 02:35:08.711 | INFO | Perf report for the last 5.12907 seconds: | 20260507 02:35:08.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:35:08.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:08.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:08.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:08.711 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 02:35:08.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:08.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:08.711 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 02:35:08.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:08.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:08.711 | INFO | Uploaded 36GiB to GPU in 2.0373s (17.6704GiB/s) | 20260507 02:35:08.711 | INFO | Downloaded 34GiB to GPU in 1.93447s (17.5759GiB/s) | 20260507 02:35:13.845 | INFO | Analyzing chunk frames = [6417408,6417920) | 20260507 02:35:13.845 | INFO | MemInfo: { | 20260507 02:35:13.845 | INFO | "gpu" : " 17 GB", | 20260507 02:35:13.845 | INFO | "huge" : "", | 20260507 02:35:13.845 | INFO | "huge_shared" : " 27 GB", | 20260507 02:35:13.845 | INFO | "malloc" : " 14 GB", | 20260507 02:35:13.845 | INFO | "pinned" : " 29 GB" | 20260507 02:35:13.845 | INFO | } | 20260507 02:35:13.845 | INFO | Perf report for the last 5.13487 seconds: | 20260507 02:35:13.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:35:13.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:13.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:13.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:13.845 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 02:35:13.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:13.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:13.845 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:35:13.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:13.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:13.846 | INFO | Uploaded 36GiB to GPU in 1.98106s (18.1721GiB/s) | 20260507 02:35:13.846 | INFO | Downloaded 34GiB to GPU in 1.95262s (17.4125GiB/s) | 20260507 02:35:18.993 | INFO | Analyzing chunk frames = [6417920,6418432) | 20260507 02:35:18.993 | INFO | MemInfo: { | 20260507 02:35:18.993 | INFO | "gpu" : " 17 GB", | 20260507 02:35:18.993 | INFO | "huge" : "", | 20260507 02:35:18.993 | INFO | "huge_shared" : " 27 GB", | 20260507 02:35:18.993 | INFO | "malloc" : " 14 GB", | 20260507 02:35:18.993 | INFO | "pinned" : " 29 GB" | 20260507 02:35:18.993 | INFO | } | 20260507 02:35:18.993 | INFO | Perf report for the last 5.14749 seconds: | 20260507 02:35:18.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:35:18.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:18.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:18.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:18.993 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 02:35:18.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:18.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:18.993 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 02:35:18.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:18.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:18.993 | INFO | Uploaded 36GiB to GPU in 1.96625s (18.309GiB/s) | 20260507 02:35:18.993 | INFO | Downloaded 34GiB to GPU in 1.97079s (17.252GiB/s) | 20260507 02:35:19.089 | INFO | Starting a round of Baz IO | 20260507 02:35:22.200 | INFO | Populating RTMetrics report | 20260507 02:35:22.201 | INFO | RTMetrics report populated | 20260507 02:35:22.201 | INFO | Opening tmp RTMetrics json file | 20260507 02:35:22.205 | INFO | Serializing RTMetrics to json | 20260507 02:35:22.219 | INFO | Serialization done, swapping tmp file now... | 20260507 02:35:22.219 | INFO | RTMetrics done | 20260507 02:35:22.293 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.203 seconds | 20260507 02:35:22.293 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.99% / 0.00% / 67.01% | 20260507 02:35:22.293 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.232025% / 3.97364e-06% / 9.13938e-05% | 20260507 02:35:22.293 | INFO | Baz IO overall: | 20260507 02:35:22.293 | INFO | BytesWritten : 3310136354178 | 20260507 02:35:22.293 | INFO | NumEvents : 1591721657557 | 20260507 02:35:24.136 | INFO | Analyzing chunk frames = [6418432,6418944) | 20260507 02:35:24.136 | INFO | MemInfo: { | 20260507 02:35:24.136 | INFO | "gpu" : " 17 GB", | 20260507 02:35:24.136 | INFO | "huge" : "", | 20260507 02:35:24.136 | INFO | "huge_shared" : " 27 GB", | 20260507 02:35:24.136 | INFO | "malloc" : " 14 GB", | 20260507 02:35:24.136 | INFO | "pinned" : " 29 GB" | 20260507 02:35:24.136 | INFO | } | 20260507 02:35:24.136 | INFO | Perf report for the last 5.14312 seconds: | 20260507 02:35:24.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:35:24.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:24.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:24.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:24.136 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260507 02:35:24.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:24.136 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 02:35:24.136 | INFO | PRE_HQ : 59.4%, 1.8, 55.6ms, 0 | 20260507 02:35:24.136 | INFO | BAZWRITER : 61.9%, 3.4, 106.9ms, 0 | 20260507 02:35:24.136 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:35:24.136 | INFO | Uploaded 36GiB to GPU in 2.0211s (17.8121GiB/s) | 20260507 02:35:24.136 | INFO | Downloaded 38GiB to GPU in 2.18181s (17.4167GiB/s) | 20260507 02:35:29.277 | INFO | Analyzing chunk frames = [6418944,6419456) | 20260507 02:35:29.277 | INFO | MemInfo: { | 20260507 02:35:29.277 | INFO | "gpu" : " 17 GB", | 20260507 02:35:29.277 | INFO | "huge" : "", | 20260507 02:35:29.277 | INFO | "huge_shared" : " 27 GB", | 20260507 02:35:29.277 | INFO | "malloc" : " 14 GB", | 20260507 02:35:29.277 | INFO | "pinned" : " 29 GB" | 20260507 02:35:29.277 | INFO | } | 20260507 02:35:29.277 | INFO | Perf report for the last 5.14119 seconds: | 20260507 02:35:29.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:35:29.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:29.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:29.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:29.277 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:35:29.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:29.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:29.277 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:35:29.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:29.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:29.278 | INFO | Uploaded 36GiB to GPU in 2.03751s (17.6686GiB/s) | 20260507 02:35:29.278 | INFO | Downloaded 34GiB to GPU in 1.94145s (17.5126GiB/s) | 20260507 02:35:34.412 | INFO | Analyzing chunk frames = [6419456,6419968) | 20260507 02:35:34.412 | INFO | MemInfo: { | 20260507 02:35:34.412 | INFO | "gpu" : " 17 GB", | 20260507 02:35:34.412 | INFO | "huge" : "", | 20260507 02:35:34.412 | INFO | "huge_shared" : " 27 GB", | 20260507 02:35:34.412 | INFO | "malloc" : " 14 GB", | 20260507 02:35:34.412 | INFO | "pinned" : " 29 GB" | 20260507 02:35:34.412 | INFO | } | 20260507 02:35:34.412 | INFO | Perf report for the last 5.13494 seconds: | 20260507 02:35:34.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:35:34.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:34.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:34.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:34.412 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 02:35:34.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:34.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:34.412 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:35:34.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:34.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:34.412 | INFO | Uploaded 36GiB to GPU in 2.01718s (17.8467GiB/s) | 20260507 02:35:34.412 | INFO | Downloaded 34GiB to GPU in 1.95604s (17.3821GiB/s) | 20260507 02:35:39.547 | INFO | Analyzing chunk frames = [6419968,6420480) | 20260507 02:35:39.547 | INFO | MemInfo: { | 20260507 02:35:39.547 | INFO | "gpu" : " 17 GB", | 20260507 02:35:39.547 | INFO | "huge" : "", | 20260507 02:35:39.547 | INFO | "huge_shared" : " 27 GB", | 20260507 02:35:39.547 | INFO | "malloc" : " 14 GB", | 20260507 02:35:39.547 | INFO | "pinned" : " 29 GB" | 20260507 02:35:39.547 | INFO | } | 20260507 02:35:39.547 | INFO | Perf report for the last 5.13446 seconds: | 20260507 02:35:39.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:35:39.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:39.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:39.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:39.547 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 02:35:39.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:39.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:39.547 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:35:39.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:39.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:39.547 | INFO | Uploaded 36GiB to GPU in 1.9746s (18.2316GiB/s) | 20260507 02:35:39.547 | INFO | Downloaded 34GiB to GPU in 1.92573s (17.6556GiB/s) | 20260507 02:35:44.688 | INFO | Analyzing chunk frames = [6420480,6420992) | 20260507 02:35:44.688 | INFO | MemInfo: { | 20260507 02:35:44.688 | INFO | "gpu" : " 17 GB", | 20260507 02:35:44.688 | INFO | "huge" : "", | 20260507 02:35:44.688 | INFO | "huge_shared" : " 27 GB", | 20260507 02:35:44.688 | INFO | "malloc" : " 14 GB", | 20260507 02:35:44.688 | INFO | "pinned" : " 29 GB" | 20260507 02:35:44.688 | INFO | } | 20260507 02:35:44.688 | INFO | Perf report for the last 5.14115 seconds: | 20260507 02:35:44.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:35:44.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:44.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:44.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:44.688 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 02:35:44.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:44.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:44.688 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 02:35:44.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:44.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:44.688 | INFO | Uploaded 36GiB to GPU in 2.00084s (17.9924GiB/s) | 20260507 02:35:44.688 | INFO | Downloaded 34GiB to GPU in 1.94871s (17.4474GiB/s) | 20260507 02:35:49.825 | INFO | Analyzing chunk frames = [6420992,6421504) | 20260507 02:35:49.825 | INFO | MemInfo: { | 20260507 02:35:49.825 | INFO | "gpu" : " 17 GB", | 20260507 02:35:49.825 | INFO | "huge" : "", | 20260507 02:35:49.825 | INFO | "huge_shared" : " 27 GB", | 20260507 02:35:49.825 | INFO | "malloc" : " 14 GB", | 20260507 02:35:49.825 | INFO | "pinned" : " 29 GB" | 20260507 02:35:49.825 | INFO | } | 20260507 02:35:49.825 | INFO | Perf report for the last 5.13703 seconds: | 20260507 02:35:49.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:35:49.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:49.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:49.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:49.825 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 02:35:49.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:49.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:49.825 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 02:35:49.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:49.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:49.825 | INFO | Uploaded 36GiB to GPU in 2.04218s (17.6282GiB/s) | 20260507 02:35:49.825 | INFO | Downloaded 34GiB to GPU in 1.96116s (17.3366GiB/s) | 20260507 02:35:54.968 | INFO | Analyzing chunk frames = [6421504,6422016) | 20260507 02:35:54.968 | INFO | MemInfo: { | 20260507 02:35:54.968 | INFO | "gpu" : " 17 GB", | 20260507 02:35:54.968 | INFO | "huge" : "", | 20260507 02:35:54.968 | INFO | "huge_shared" : " 27 GB", | 20260507 02:35:54.968 | INFO | "malloc" : " 14 GB", | 20260507 02:35:54.968 | INFO | "pinned" : " 29 GB" | 20260507 02:35:54.968 | INFO | } | 20260507 02:35:54.968 | INFO | Perf report for the last 5.14355 seconds: | 20260507 02:35:54.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:35:54.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:54.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:54.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:54.968 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:35:54.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:54.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:54.968 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:35:54.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:54.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:35:54.969 | INFO | Uploaded 36GiB to GPU in 1.98243s (18.1595GiB/s) | 20260507 02:35:54.969 | INFO | Downloaded 34GiB to GPU in 1.9875s (17.1069GiB/s) | 20260507 02:36:00.107 | INFO | Analyzing chunk frames = [6422016,6422528) | 20260507 02:36:00.107 | INFO | MemInfo: { | 20260507 02:36:00.107 | INFO | "gpu" : " 17 GB", | 20260507 02:36:00.107 | INFO | "huge" : "", | 20260507 02:36:00.107 | INFO | "huge_shared" : " 27 GB", | 20260507 02:36:00.107 | INFO | "malloc" : " 14 GB", | 20260507 02:36:00.107 | INFO | "pinned" : " 29 GB" | 20260507 02:36:00.107 | INFO | } | 20260507 02:36:00.107 | INFO | Perf report for the last 5.1388 seconds: | 20260507 02:36:00.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:36:00.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:00.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:00.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:00.107 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:36:00.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:00.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:00.107 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:36:00.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:00.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:00.107 | INFO | Uploaded 36GiB to GPU in 1.98589s (18.1279GiB/s) | 20260507 02:36:00.108 | INFO | Downloaded 34GiB to GPU in 1.97738s (17.1945GiB/s) | 20260507 02:36:00.108 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:36:00.108 | INFO | Total mem usage: 18 GiB | 20260507 02:36:00.108 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:36:00.108 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 02:36:00.108 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:36:00.108 | INFO | High Water Breakdown: | 20260507 02:36:00.108 | INFO | 60.2% : 8.58 GiB ( 75.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:36:00.108 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:36:00.108 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:36:00.108 | INFO | ------------------------------------------ | 20260507 02:36:00.108 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:36:00.108 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:36:00.108 | INFO | Total mem usage: 26.5 GiB | 20260507 02:36:00.108 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:36:00.108 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:36:00.108 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:36:00.108 | INFO | High Water Breakdown: | 20260507 02:36:00.108 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:36:00.108 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:36:00.108 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:36:00.108 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:36:00.108 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:36:00.108 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:36:00.108 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:36:00.108 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:36:00.108 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:36:00.108 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:36:00.108 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:36:00.108 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:36:00.108 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:36:00.108 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:36:00.108 | INFO | ------------------------------------------ | 20260507 02:36:00.108 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:36:00.108 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:36:00.108 | INFO | Total mem usage: 25.6 GiB | 20260507 02:36:00.108 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:36:00.108 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:36:00.108 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:36:00.108 | INFO | High Water Breakdown: | 20260507 02:36:00.108 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:36:00.108 | INFO | ------------------------------------------ | 20260507 02:36:00.108 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:36:00.108 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:36:00.108 | INFO | Total mem usage: 15.7 GiB | 20260507 02:36:00.108 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:36:00.108 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:36:00.108 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:36:00.108 | INFO | High Water Breakdown: | 20260507 02:36:00.108 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:36:00.108 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:36:00.108 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:36:00.108 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:36:00.108 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:36:00.108 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:36:00.108 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:36:00.108 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:36:00.108 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:36:00.108 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:36:00.108 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:36:00.108 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:36:00.108 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:36:00.108 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:36:00.108 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:36:00.108 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:36:00.108 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:36:00.108 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:36:00.108 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:36:00.108 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:36:00.108 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:36:00.108 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:36:00.108 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:36:00.108 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:36:00.108 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:36:00.108 | INFO | ------------------------------------------ | 20260507 02:36:00.108 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:36:00.108 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:36:00.108 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:36:00.108 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:36:00.108 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.52e+01, 1.34e+01 / 1.59e+00 | 20260507 02:36:00.108 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.41e-03 / 4.56e-03 | 20260507 02:36:00.108 | INFO | PerfReport: 11.68%, Baseline -- 3.60e+00 / 4.40e+00, 5.20e-01 / 2.64e-01 | 20260507 02:36:00.108 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.47e+00, 1.74e-01 / 1.05e-01 | 20260507 02:36:00.108 | INFO | PerfReport: 16.11%, DME -- 3.97e+01 / 4.44e+01, 3.88e+00 / 1.88e+00 | 20260507 02:36:00.108 | INFO | PerfReport: 57.55%, FrameLabeling -- 1.77e+01 / 2.05e+01, 1.28e+00 / 1.67e+00 | 20260507 02:36:00.108 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.15e+00 / 2.70e+00, 3.83e-01 / 2.13e-01 | 20260507 02:36:00.108 | INFO | PerfReport: 3.07%, Metrics -- 9.46e-01 / 1.27e+00, 2.52e-01 / 9.47e-02 | 20260507 02:36:00.108 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.21e-01 / 3.67e-01, 5.11e-02 / 1.52e-02 | 20260507 02:36:00.108 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:36:00.108 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:36:00.108 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:36:00.108 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 02:36:00.109 | INFO | PerfReport: 21.73%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260507 02:36:00.109 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 02:36:00.109 | INFO | PerfReport: 31.96%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 02:36:00.109 | INFO | PerfReport: 1.40%, LockUpload -- 1.35e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 02:36:00.109 | INFO | PerfReport: 1.35%, LockDownload -- 1.30e+00 / 0.00e+00, 3.16e+00 / 0.00e+00 | 20260507 02:36:00.109 | INFO | PerfReport: 22.17%, LockCompute -- 2.14e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 02:36:00.109 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.50e-02 / 0.00e+00 | 20260507 02:36:00.203 | INFO | Starting a round of Baz IO | 20260507 02:36:03.313 | INFO | Populating RTMetrics report | 20260507 02:36:03.313 | INFO | RTMetrics report populated | 20260507 02:36:03.313 | INFO | Opening tmp RTMetrics json file | 20260507 02:36:03.315 | INFO | Serializing RTMetrics to json | 20260507 02:36:03.333 | INFO | Serialization done, swapping tmp file now... | 20260507 02:36:03.333 | INFO | RTMetrics done | 20260507 02:36:03.405 | INFO | Finished a round of Baz IO: Wrote 1.778 GiB in 3.201 seconds | 20260507 02:36:03.405 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.99% / 0.00% / 67.01% | 20260507 02:36:03.405 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.232279% / 0% / 5.16574e-05% | 20260507 02:36:03.406 | INFO | Baz IO overall: | 20260507 02:36:03.406 | INFO | BytesWritten : 3312045550945 | 20260507 02:36:03.406 | INFO | NumEvents : 1592624028339 | 20260507 02:36:05.251 | INFO | Analyzing chunk frames = [6422528,6423040) | 20260507 02:36:05.251 | INFO | MemInfo: { | 20260507 02:36:05.251 | INFO | "gpu" : " 17 GB", | 20260507 02:36:05.251 | INFO | "huge" : "", | 20260507 02:36:05.251 | INFO | "huge_shared" : " 27 GB", | 20260507 02:36:05.251 | INFO | "malloc" : " 14 GB", | 20260507 02:36:05.251 | INFO | "pinned" : " 29 GB" | 20260507 02:36:05.251 | INFO | } | 20260507 02:36:05.251 | INFO | Perf report for the last 5.14413 seconds: | 20260507 02:36:05.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:36:05.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:05.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:05.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:05.251 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 02:36:05.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:05.251 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 02:36:05.251 | INFO | PRE_HQ : 59.3%, 1.8, 55.7ms, 0 | 20260507 02:36:05.251 | INFO | BAZWRITER : 61.8%, 3.3, 104.1ms, 0 | 20260507 02:36:05.251 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:36:05.252 | INFO | Uploaded 36GiB to GPU in 2.02608s (17.7683GiB/s) | 20260507 02:36:05.252 | INFO | Downloaded 38GiB to GPU in 2.18633s (17.3807GiB/s) | 20260507 02:36:10.389 | INFO | Analyzing chunk frames = [6423040,6423552) | 20260507 02:36:10.389 | INFO | MemInfo: { | 20260507 02:36:10.389 | INFO | "gpu" : " 17 GB", | 20260507 02:36:10.389 | INFO | "huge" : "", | 20260507 02:36:10.389 | INFO | "huge_shared" : " 27 GB", | 20260507 02:36:10.389 | INFO | "malloc" : " 14 GB", | 20260507 02:36:10.389 | INFO | "pinned" : " 29 GB" | 20260507 02:36:10.389 | INFO | } | 20260507 02:36:10.389 | INFO | Perf report for the last 5.13731 seconds: | 20260507 02:36:10.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:36:10.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:10.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:10.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:10.389 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:36:10.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:10.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:10.389 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:36:10.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:10.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:10.389 | INFO | Uploaded 36GiB to GPU in 2.03696s (17.6734GiB/s) | 20260507 02:36:10.389 | INFO | Downloaded 34GiB to GPU in 1.94015s (17.5245GiB/s) | 20260507 02:36:15.528 | INFO | Analyzing chunk frames = [6423552,6424064) | 20260507 02:36:15.528 | INFO | MemInfo: { | 20260507 02:36:15.528 | INFO | "gpu" : " 17 GB", | 20260507 02:36:15.529 | INFO | "huge" : "", | 20260507 02:36:15.529 | INFO | "huge_shared" : " 27 GB", | 20260507 02:36:15.529 | INFO | "malloc" : " 14 GB", | 20260507 02:36:15.529 | INFO | "pinned" : " 29 GB" | 20260507 02:36:15.529 | INFO | } | 20260507 02:36:15.529 | INFO | Perf report for the last 5.14032 seconds: | 20260507 02:36:15.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:36:15.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:15.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:15.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:15.529 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 02:36:15.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:15.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:15.529 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 02:36:15.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:15.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:15.529 | INFO | Uploaded 36GiB to GPU in 1.98405s (18.1447GiB/s) | 20260507 02:36:15.529 | INFO | Downloaded 34GiB to GPU in 1.94339s (17.4952GiB/s) | 20260507 02:36:20.661 | INFO | Analyzing chunk frames = [6424064,6424576) | 20260507 02:36:20.661 | INFO | MemInfo: { | 20260507 02:36:20.661 | INFO | "gpu" : " 17 GB", | 20260507 02:36:20.661 | INFO | "huge" : "", | 20260507 02:36:20.662 | INFO | "huge_shared" : " 27 GB", | 20260507 02:36:20.662 | INFO | "malloc" : " 14 GB", | 20260507 02:36:20.662 | INFO | "pinned" : " 29 GB" | 20260507 02:36:20.662 | INFO | } | 20260507 02:36:20.662 | INFO | Perf report for the last 5.13258 seconds: | 20260507 02:36:20.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:36:20.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:20.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:20.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:20.662 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 02:36:20.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:20.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:20.662 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:36:20.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:20.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:20.662 | INFO | Uploaded 36GiB to GPU in 1.97683s (18.211GiB/s) | 20260507 02:36:20.662 | INFO | Downloaded 34GiB to GPU in 1.93555s (17.5661GiB/s) | 20260507 02:36:25.798 | INFO | Analyzing chunk frames = [6424576,6425088) | 20260507 02:36:25.798 | INFO | MemInfo: { | 20260507 02:36:25.798 | INFO | "gpu" : " 17 GB", | 20260507 02:36:25.798 | INFO | "huge" : "", | 20260507 02:36:25.798 | INFO | "huge_shared" : " 27 GB", | 20260507 02:36:25.798 | INFO | "malloc" : " 14 GB", | 20260507 02:36:25.798 | INFO | "pinned" : " 29 GB" | 20260507 02:36:25.798 | INFO | } | 20260507 02:36:25.798 | INFO | Perf report for the last 5.1365 seconds: | 20260507 02:36:25.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:36:25.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:25.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:25.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:25.798 | INFO | ANALYSIS : 59.9%, 3.0, 93.2ms, 0 | 20260507 02:36:25.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:25.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:25.798 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:36:25.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:25.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:25.798 | INFO | Uploaded 36GiB to GPU in 2.00118s (17.9894GiB/s) | 20260507 02:36:25.798 | INFO | Downloaded 34GiB to GPU in 1.95012s (17.4349GiB/s) | 20260507 02:36:30.945 | INFO | Analyzing chunk frames = [6425088,6425600) | 20260507 02:36:30.945 | INFO | MemInfo: { | 20260507 02:36:30.945 | INFO | "gpu" : " 17 GB", | 20260507 02:36:30.945 | INFO | "huge" : "", | 20260507 02:36:30.945 | INFO | "huge_shared" : " 27 GB", | 20260507 02:36:30.945 | INFO | "malloc" : " 14 GB", | 20260507 02:36:30.946 | INFO | "pinned" : " 29 GB" | 20260507 02:36:30.946 | INFO | } | 20260507 02:36:30.946 | INFO | Perf report for the last 5.14742 seconds: | 20260507 02:36:30.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:36:30.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:30.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:30.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:30.946 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 02:36:30.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:30.946 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:30.946 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 02:36:30.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:30.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:30.946 | INFO | Uploaded 36GiB to GPU in 2.01402s (17.8747GiB/s) | 20260507 02:36:30.946 | INFO | Downloaded 34GiB to GPU in 1.93822s (17.5419GiB/s) | 20260507 02:36:36.084 | INFO | Analyzing chunk frames = [6425600,6426112) | 20260507 02:36:36.084 | INFO | MemInfo: { | 20260507 02:36:36.084 | INFO | "gpu" : " 17 GB", | 20260507 02:36:36.084 | INFO | "huge" : "", | 20260507 02:36:36.084 | INFO | "huge_shared" : " 27 GB", | 20260507 02:36:36.084 | INFO | "malloc" : " 14 GB", | 20260507 02:36:36.084 | INFO | "pinned" : " 29 GB" | 20260507 02:36:36.084 | INFO | } | 20260507 02:36:36.084 | INFO | Perf report for the last 5.13875 seconds: | 20260507 02:36:36.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:36:36.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:36.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:36.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:36.084 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 02:36:36.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:36.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:36.084 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:36:36.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:36.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:36.085 | INFO | Uploaded 36GiB to GPU in 1.96489s (18.3216GiB/s) | 20260507 02:36:36.085 | INFO | Downloaded 34GiB to GPU in 1.96164s (17.3325GiB/s) | 20260507 02:36:41.224 | INFO | Analyzing chunk frames = [6426112,6426624) | 20260507 02:36:41.224 | INFO | MemInfo: { | 20260507 02:36:41.224 | INFO | "gpu" : " 17 GB", | 20260507 02:36:41.224 | INFO | "huge" : "", | 20260507 02:36:41.224 | INFO | "huge_shared" : " 27 GB", | 20260507 02:36:41.224 | INFO | "malloc" : " 14 GB", | 20260507 02:36:41.224 | INFO | "pinned" : " 29 GB" | 20260507 02:36:41.224 | INFO | } | 20260507 02:36:41.224 | INFO | Perf report for the last 5.13969 seconds: | 20260507 02:36:41.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:36:41.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:41.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:41.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:41.224 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 02:36:41.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:41.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:41.224 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:36:41.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:41.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:41.225 | INFO | Uploaded 36GiB to GPU in 1.97889s (18.192GiB/s) | 20260507 02:36:41.225 | INFO | Downloaded 34GiB to GPU in 1.9661s (17.2931GiB/s) | 20260507 02:36:41.321 | INFO | Starting a round of Baz IO | 20260507 02:36:44.428 | INFO | Populating RTMetrics report | 20260507 02:36:44.429 | INFO | RTMetrics report populated | 20260507 02:36:44.429 | INFO | Opening tmp RTMetrics json file | 20260507 02:36:44.430 | INFO | Serializing RTMetrics to json | 20260507 02:36:44.446 | INFO | Serialization done, swapping tmp file now... | 20260507 02:36:44.449 | INFO | RTMetrics done | 20260507 02:36:44.523 | INFO | Finished a round of Baz IO: Wrote 1.777 GiB in 3.202 seconds | 20260507 02:36:44.523 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.99% / 0.00% / 67.01% | 20260507 02:36:44.523 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.2321% / 3.97364e-06% / 7.94729e-05% | 20260507 02:36:44.523 | INFO | Baz IO overall: | 20260507 02:36:44.523 | INFO | BytesWritten : 3313953345809 | 20260507 02:36:44.523 | INFO | NumEvents : 1593525677213 | 20260507 02:36:46.361 | INFO | Analyzing chunk frames = [6426624,6427136) | 20260507 02:36:46.361 | INFO | MemInfo: { | 20260507 02:36:46.361 | INFO | "gpu" : " 17 GB", | 20260507 02:36:46.361 | INFO | "huge" : "", | 20260507 02:36:46.361 | INFO | "huge_shared" : " 27 GB", | 20260507 02:36:46.361 | INFO | "malloc" : " 14 GB", | 20260507 02:36:46.361 | INFO | "pinned" : " 29 GB" | 20260507 02:36:46.361 | INFO | } | 20260507 02:36:46.361 | INFO | Perf report for the last 5.13707 seconds: | 20260507 02:36:46.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:36:46.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:46.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:46.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:46.361 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 02:36:46.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:46.361 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 02:36:46.361 | INFO | PRE_HQ : 59.3%, 1.8, 55.5ms, 0 | 20260507 02:36:46.361 | INFO | BAZWRITER : 61.9%, 3.4, 106.8ms, 0 | 20260507 02:36:46.361 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:36:46.361 | INFO | Uploaded 36GiB to GPU in 2.03684s (17.6744GiB/s) | 20260507 02:36:46.361 | INFO | Downloaded 38GiB to GPU in 2.17508s (17.4706GiB/s) | 20260507 02:36:51.499 | INFO | Analyzing chunk frames = [6427136,6427648) | 20260507 02:36:51.499 | INFO | MemInfo: { | 20260507 02:36:51.499 | INFO | "gpu" : " 17 GB", | 20260507 02:36:51.499 | INFO | "huge" : "", | 20260507 02:36:51.499 | INFO | "huge_shared" : " 27 GB", | 20260507 02:36:51.499 | INFO | "malloc" : " 14 GB", | 20260507 02:36:51.499 | INFO | "pinned" : " 29 GB" | 20260507 02:36:51.499 | INFO | } | 20260507 02:36:51.499 | INFO | Perf report for the last 5.13782 seconds: | 20260507 02:36:51.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:36:51.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:51.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:51.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:51.499 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:36:51.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:51.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:51.499 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 02:36:51.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:51.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:51.499 | INFO | Uploaded 36GiB to GPU in 2.02917s (17.7412GiB/s) | 20260507 02:36:51.499 | INFO | Downloaded 34GiB to GPU in 1.93291s (17.5901GiB/s) | 20260507 02:36:56.640 | INFO | Analyzing chunk frames = [6427648,6428160) | 20260507 02:36:56.640 | INFO | MemInfo: { | 20260507 02:36:56.640 | INFO | "gpu" : " 17 GB", | 20260507 02:36:56.640 | INFO | "huge" : "", | 20260507 02:36:56.640 | INFO | "huge_shared" : " 27 GB", | 20260507 02:36:56.640 | INFO | "malloc" : " 14 GB", | 20260507 02:36:56.640 | INFO | "pinned" : " 29 GB" | 20260507 02:36:56.640 | INFO | } | 20260507 02:36:56.640 | INFO | Perf report for the last 5.14092 seconds: | 20260507 02:36:56.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:36:56.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:56.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:56.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:56.640 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:36:56.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:56.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:56.640 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:36:56.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:56.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:36:56.641 | INFO | Uploaded 36GiB to GPU in 1.98872s (18.1021GiB/s) | 20260507 02:36:56.641 | INFO | Downloaded 34GiB to GPU in 1.93248s (17.5939GiB/s) | 20260507 02:37:01.784 | INFO | Analyzing chunk frames = [6428160,6428672) | 20260507 02:37:01.784 | INFO | MemInfo: { | 20260507 02:37:01.784 | INFO | "gpu" : " 17 GB", | 20260507 02:37:01.784 | INFO | "huge" : "", | 20260507 02:37:01.784 | INFO | "huge_shared" : " 27 GB", | 20260507 02:37:01.784 | INFO | "malloc" : " 14 GB", | 20260507 02:37:01.784 | INFO | "pinned" : " 29 GB" | 20260507 02:37:01.784 | INFO | } | 20260507 02:37:01.784 | INFO | Perf report for the last 5.1441 seconds: | 20260507 02:37:01.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:37:01.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:01.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:01.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:01.784 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 02:37:01.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:01.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:01.784 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 02:37:01.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:01.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:01.784 | INFO | Uploaded 36GiB to GPU in 1.97962s (18.1854GiB/s) | 20260507 02:37:01.784 | INFO | Downloaded 34GiB to GPU in 1.9071s (17.8281GiB/s) | 20260507 02:37:06.919 | INFO | Analyzing chunk frames = [6428672,6429184) | 20260507 02:37:06.919 | INFO | MemInfo: { | 20260507 02:37:06.919 | INFO | "gpu" : " 17 GB", | 20260507 02:37:06.919 | INFO | "huge" : "", | 20260507 02:37:06.919 | INFO | "huge_shared" : " 27 GB", | 20260507 02:37:06.919 | INFO | "malloc" : " 14 GB", | 20260507 02:37:06.919 | INFO | "pinned" : " 29 GB" | 20260507 02:37:06.919 | INFO | } | 20260507 02:37:06.919 | INFO | Perf report for the last 5.13483 seconds: | 20260507 02:37:06.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:37:06.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:06.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:06.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:06.919 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 02:37:06.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:06.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:06.919 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:37:06.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:06.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:06.919 | INFO | Uploaded 36GiB to GPU in 2.0116s (17.8962GiB/s) | 20260507 02:37:06.919 | INFO | Downloaded 34GiB to GPU in 1.94106s (17.5162GiB/s) | 20260507 02:37:12.052 | INFO | Analyzing chunk frames = [6429184,6429696) | 20260507 02:37:12.052 | INFO | MemInfo: { | 20260507 02:37:12.052 | INFO | "gpu" : " 17 GB", | 20260507 02:37:12.052 | INFO | "huge" : "", | 20260507 02:37:12.052 | INFO | "huge_shared" : " 27 GB", | 20260507 02:37:12.052 | INFO | "malloc" : " 14 GB", | 20260507 02:37:12.052 | INFO | "pinned" : " 29 GB" | 20260507 02:37:12.052 | INFO | } | 20260507 02:37:12.052 | INFO | Perf report for the last 5.13352 seconds: | 20260507 02:37:12.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:37:12.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:12.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:12.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:12.052 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 02:37:12.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:12.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:12.052 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:37:12.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:12.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:12.053 | INFO | Uploaded 36GiB to GPU in 2.05685s (17.5025GiB/s) | 20260507 02:37:12.053 | INFO | Downloaded 34GiB to GPU in 1.96355s (17.3156GiB/s) | 20260507 02:37:17.199 | INFO | Analyzing chunk frames = [6429696,6430208) | 20260507 02:37:17.199 | INFO | MemInfo: { | 20260507 02:37:17.199 | INFO | "gpu" : " 17 GB", | 20260507 02:37:17.199 | INFO | "huge" : "", | 20260507 02:37:17.199 | INFO | "huge_shared" : " 27 GB", | 20260507 02:37:17.199 | INFO | "malloc" : " 14 GB", | 20260507 02:37:17.199 | INFO | "pinned" : " 29 GB" | 20260507 02:37:17.199 | INFO | } | 20260507 02:37:17.199 | INFO | Perf report for the last 5.14659 seconds: | 20260507 02:37:17.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:37:17.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:17.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:17.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:17.199 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 02:37:17.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:17.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:17.199 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:37:17.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:17.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:17.199 | INFO | Uploaded 36GiB to GPU in 1.98982s (18.0921GiB/s) | 20260507 02:37:17.199 | INFO | Downloaded 34GiB to GPU in 2.00158s (16.9866GiB/s) | 20260507 02:37:22.336 | INFO | Analyzing chunk frames = [6430208,6430720) | 20260507 02:37:22.336 | INFO | MemInfo: { | 20260507 02:37:22.336 | INFO | "gpu" : " 17 GB", | 20260507 02:37:22.336 | INFO | "huge" : "", | 20260507 02:37:22.336 | INFO | "huge_shared" : " 27 GB", | 20260507 02:37:22.336 | INFO | "malloc" : " 14 GB", | 20260507 02:37:22.336 | INFO | "pinned" : " 29 GB" | 20260507 02:37:22.336 | INFO | } | 20260507 02:37:22.336 | INFO | Perf report for the last 5.13689 seconds: | 20260507 02:37:22.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:37:22.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:22.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:22.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:22.336 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 02:37:22.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:22.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:22.336 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 02:37:22.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:22.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:22.336 | INFO | Uploaded 36GiB to GPU in 1.99324s (18.0611GiB/s) | 20260507 02:37:22.336 | INFO | Downloaded 34GiB to GPU in 1.98671s (17.1137GiB/s) | 20260507 02:37:22.336 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:37:22.336 | INFO | Total mem usage: 18 GiB | 20260507 02:37:22.336 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:37:22.336 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:37:22.336 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:37:22.336 | INFO | High Water Breakdown: | 20260507 02:37:22.336 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:37:22.336 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:37:22.336 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:37:22.336 | INFO | ------------------------------------------ | 20260507 02:37:22.336 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:37:22.336 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:37:22.336 | INFO | Total mem usage: 26.5 GiB | 20260507 02:37:22.336 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:37:22.336 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:37:22.337 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:37:22.337 | INFO | High Water Breakdown: | 20260507 02:37:22.337 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:37:22.337 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:37:22.337 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:37:22.337 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:37:22.337 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:37:22.337 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:37:22.337 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:37:22.337 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:37:22.337 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:37:22.337 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:37:22.337 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:37:22.337 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:37:22.337 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:37:22.337 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:37:22.337 | INFO | ------------------------------------------ | 20260507 02:37:22.337 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:37:22.337 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:37:22.337 | INFO | Total mem usage: 25.6 GiB | 20260507 02:37:22.337 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:37:22.337 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 02:37:22.337 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 02:37:22.337 | INFO | High Water Breakdown: | 20260507 02:37:22.337 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 02:37:22.337 | INFO | ------------------------------------------ | 20260507 02:37:22.337 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:37:22.337 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:37:22.337 | INFO | Total mem usage: 15.7 GiB | 20260507 02:37:22.337 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:37:22.337 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:37:22.337 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:37:22.337 | INFO | High Water Breakdown: | 20260507 02:37:22.337 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:37:22.337 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:37:22.337 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:37:22.337 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:37:22.337 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:37:22.337 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:37:22.337 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:37:22.337 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:37:22.337 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:37:22.337 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:37:22.337 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:37:22.337 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:37:22.337 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:37:22.337 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:37:22.337 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:37:22.337 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:37:22.337 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:37:22.337 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:37:22.337 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:37:22.337 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:37:22.337 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:37:22.337 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:37:22.337 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:37:22.337 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:37:22.337 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:37:22.337 | INFO | ------------------------------------------ | 20260507 02:37:22.337 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:37:22.337 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:37:22.337 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:37:22.337 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:37:22.337 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.53e+01, 1.31e+01 / 2.07e+00 | 20260507 02:37:22.337 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 9.92e-02, 9.47e-03 / 5.87e-03 | 20260507 02:37:22.337 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.35e+00, 5.16e-01 / 3.32e-01 | 20260507 02:37:22.337 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.43e+00, 1.67e-01 / 1.17e-01 | 20260507 02:37:22.337 | INFO | PerfReport: 16.13%, DME -- 3.97e+01 / 4.46e+01, 4.06e+00 / 2.48e+00 | 20260507 02:37:22.337 | INFO | PerfReport: 57.58%, FrameLabeling -- 1.77e+01 / 2.05e+01, 1.26e+00 / 1.87e+00 | 20260507 02:37:22.337 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.15e+00 / 2.69e+00, 3.82e-01 / 2.44e-01 | 20260507 02:37:22.337 | INFO | PerfReport: 3.07%, Metrics -- 9.46e-01 / 1.26e+00, 2.53e-01 / 1.25e-01 | 20260507 02:37:22.337 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.20e-01 / 3.70e-01, 5.08e-02 / 2.51e-02 | 20260507 02:37:22.337 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:37:22.337 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:37:22.337 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:37:22.337 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 02:37:22.337 | INFO | PerfReport: 21.67%, Upload -- 2.09e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260507 02:37:22.337 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260507 02:37:22.337 | INFO | PerfReport: 31.95%, Compute -- 3.08e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 02:37:22.337 | INFO | PerfReport: 1.41%, LockUpload -- 1.35e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 02:37:22.337 | INFO | PerfReport: 1.35%, LockDownload -- 1.30e+00 / 0.00e+00, 3.20e+00 / 0.00e+00 | 20260507 02:37:22.337 | INFO | PerfReport: 22.20%, LockCompute -- 2.14e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 02:37:22.337 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260507 02:37:22.433 | INFO | Starting a round of Baz IO | 20260507 02:37:25.548 | INFO | Populating RTMetrics report | 20260507 02:37:25.549 | INFO | RTMetrics report populated | 20260507 02:37:25.549 | INFO | Opening tmp RTMetrics json file | 20260507 02:37:25.550 | INFO | Serializing RTMetrics to json | 20260507 02:37:25.551 | INFO | Serialization done, swapping tmp file now... | 20260507 02:37:25.552 | INFO | RTMetrics done | 20260507 02:37:25.645 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.212 seconds | 20260507 02:37:25.645 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.98% / 0.00% / 67.01% | 20260507 02:37:25.645 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.23065% / 3.97364e-06% / 9.53674e-05% | 20260507 02:37:25.645 | INFO | Baz IO overall: | 20260507 02:37:25.645 | INFO | BytesWritten : 3315859537793 | 20260507 02:37:25.645 | INFO | NumEvents : 1594425927019 | 20260507 02:37:27.472 | INFO | Analyzing chunk frames = [6430720,6431232) | 20260507 02:37:27.472 | INFO | MemInfo: { | 20260507 02:37:27.472 | INFO | "gpu" : " 17 GB", | 20260507 02:37:27.472 | INFO | "huge" : "", | 20260507 02:37:27.472 | INFO | "huge_shared" : " 27 GB", | 20260507 02:37:27.472 | INFO | "malloc" : " 14 GB", | 20260507 02:37:27.472 | INFO | "pinned" : " 29 GB" | 20260507 02:37:27.472 | INFO | } | 20260507 02:37:27.472 | INFO | Perf report for the last 5.13654 seconds: | 20260507 02:37:27.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:37:27.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:27.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:27.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:27.472 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260507 02:37:27.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:27.472 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260507 02:37:27.472 | INFO | PRE_HQ : 60.1%, 1.9, 59.3ms, 0 | 20260507 02:37:27.472 | INFO | BAZWRITER : 62.1%, 3.5, 110.5ms, 0 | 20260507 02:37:27.472 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 02:37:27.473 | INFO | Uploaded 36GiB to GPU in 2.03754s (17.6683GiB/s) | 20260507 02:37:27.473 | INFO | Downloaded 38GiB to GPU in 2.18429s (17.3969GiB/s) | 20260507 02:37:32.617 | INFO | Analyzing chunk frames = [6431232,6431744) | 20260507 02:37:32.617 | INFO | MemInfo: { | 20260507 02:37:32.617 | INFO | "gpu" : " 17 GB", | 20260507 02:37:32.617 | INFO | "huge" : "", | 20260507 02:37:32.617 | INFO | "huge_shared" : " 27 GB", | 20260507 02:37:32.617 | INFO | "malloc" : " 14 GB", | 20260507 02:37:32.617 | INFO | "pinned" : " 29 GB" | 20260507 02:37:32.617 | INFO | } | 20260507 02:37:32.617 | INFO | Perf report for the last 5.14429 seconds: | 20260507 02:37:32.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:37:32.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:32.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:32.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:32.617 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:37:32.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:32.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:32.617 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:37:32.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:32.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:32.617 | INFO | Uploaded 36GiB to GPU in 2.03081s (17.7269GiB/s) | 20260507 02:37:32.617 | INFO | Downloaded 34GiB to GPU in 1.94034s (17.5227GiB/s) | 20260507 02:37:37.761 | INFO | Analyzing chunk frames = [6431744,6432256) | 20260507 02:37:37.761 | INFO | MemInfo: { | 20260507 02:37:37.761 | INFO | "gpu" : " 17 GB", | 20260507 02:37:37.761 | INFO | "huge" : "", | 20260507 02:37:37.761 | INFO | "huge_shared" : " 27 GB", | 20260507 02:37:37.761 | INFO | "malloc" : " 14 GB", | 20260507 02:37:37.761 | INFO | "pinned" : " 29 GB" | 20260507 02:37:37.761 | INFO | } | 20260507 02:37:37.761 | INFO | Perf report for the last 5.14458 seconds: | 20260507 02:37:37.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:37:37.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:37.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:37.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:37.761 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:37:37.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:37.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:37.761 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 02:37:37.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:37.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:37.762 | INFO | Uploaded 36GiB to GPU in 1.99233s (18.0693GiB/s) | 20260507 02:37:37.762 | INFO | Downloaded 34GiB to GPU in 1.93793s (17.5445GiB/s) | 20260507 02:37:42.900 | INFO | Analyzing chunk frames = [6432256,6432768) | 20260507 02:37:42.900 | INFO | MemInfo: { | 20260507 02:37:42.900 | INFO | "gpu" : " 17 GB", | 20260507 02:37:42.900 | INFO | "huge" : "", | 20260507 02:37:42.901 | INFO | "huge_shared" : " 27 GB", | 20260507 02:37:42.901 | INFO | "malloc" : " 14 GB", | 20260507 02:37:42.901 | INFO | "pinned" : " 29 GB" | 20260507 02:37:42.901 | INFO | } | 20260507 02:37:42.901 | INFO | Perf report for the last 5.13935 seconds: | 20260507 02:37:42.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:37:42.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:42.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:42.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:42.901 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:37:42.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:42.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:42.901 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:37:42.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:42.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:42.901 | INFO | Uploaded 36GiB to GPU in 1.97786s (18.2015GiB/s) | 20260507 02:37:42.901 | INFO | Downloaded 34GiB to GPU in 1.91966s (17.7115GiB/s) | 20260507 02:37:48.031 | INFO | Analyzing chunk frames = [6432768,6433280) | 20260507 02:37:48.031 | INFO | MemInfo: { | 20260507 02:37:48.031 | INFO | "gpu" : " 17 GB", | 20260507 02:37:48.031 | INFO | "huge" : "", | 20260507 02:37:48.031 | INFO | "huge_shared" : " 27 GB", | 20260507 02:37:48.032 | INFO | "malloc" : " 14 GB", | 20260507 02:37:48.032 | INFO | "pinned" : " 29 GB" | 20260507 02:37:48.032 | INFO | } | 20260507 02:37:48.032 | INFO | Perf report for the last 5.13089 seconds: | 20260507 02:37:48.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:37:48.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:48.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:48.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:48.032 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 02:37:48.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:48.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:48.032 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:37:48.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:48.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:48.032 | INFO | Uploaded 36GiB to GPU in 2.02821s (17.7497GiB/s) | 20260507 02:37:48.032 | INFO | Downloaded 34GiB to GPU in 1.94808s (17.4531GiB/s) | 20260507 02:37:53.167 | INFO | Analyzing chunk frames = [6433280,6433792) | 20260507 02:37:53.167 | INFO | MemInfo: { | 20260507 02:37:53.167 | INFO | "gpu" : " 17 GB", | 20260507 02:37:53.167 | INFO | "huge" : "", | 20260507 02:37:53.167 | INFO | "huge_shared" : " 27 GB", | 20260507 02:37:53.167 | INFO | "malloc" : " 14 GB", | 20260507 02:37:53.167 | INFO | "pinned" : " 29 GB" | 20260507 02:37:53.167 | INFO | } | 20260507 02:37:53.167 | INFO | Perf report for the last 5.13538 seconds: | 20260507 02:37:53.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:37:53.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:53.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:53.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:53.167 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 02:37:53.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:53.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:53.167 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:37:53.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:53.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:53.167 | INFO | Uploaded 36GiB to GPU in 2.0463s (17.5927GiB/s) | 20260507 02:37:53.167 | INFO | Downloaded 34GiB to GPU in 1.96264s (17.3236GiB/s) | 20260507 02:37:58.310 | INFO | Analyzing chunk frames = [6433792,6434304) | 20260507 02:37:58.310 | INFO | MemInfo: { | 20260507 02:37:58.310 | INFO | "gpu" : " 17 GB", | 20260507 02:37:58.310 | INFO | "huge" : "", | 20260507 02:37:58.310 | INFO | "huge_shared" : " 27 GB", | 20260507 02:37:58.310 | INFO | "malloc" : " 14 GB", | 20260507 02:37:58.310 | INFO | "pinned" : " 29 GB" | 20260507 02:37:58.310 | INFO | } | 20260507 02:37:58.310 | INFO | Perf report for the last 5.1428 seconds: | 20260507 02:37:58.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:37:58.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:58.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:58.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:58.310 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:37:58.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:58.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:58.310 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:37:58.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:58.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:37:58.310 | INFO | Uploaded 36GiB to GPU in 1.97529s (18.2252GiB/s) | 20260507 02:37:58.310 | INFO | Downloaded 34GiB to GPU in 2.00229s (16.9806GiB/s) | 20260507 02:38:03.447 | INFO | Analyzing chunk frames = [6434304,6434816) | 20260507 02:38:03.447 | INFO | MemInfo: { | 20260507 02:38:03.447 | INFO | "gpu" : " 17 GB", | 20260507 02:38:03.447 | INFO | "huge" : "", | 20260507 02:38:03.447 | INFO | "huge_shared" : " 27 GB", | 20260507 02:38:03.447 | INFO | "malloc" : " 14 GB", | 20260507 02:38:03.447 | INFO | "pinned" : " 29 GB" | 20260507 02:38:03.447 | INFO | } | 20260507 02:38:03.447 | INFO | Perf report for the last 5.13713 seconds: | 20260507 02:38:03.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:38:03.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:03.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:03.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:03.447 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:38:03.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:03.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:03.447 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 02:38:03.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:03.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:03.447 | INFO | Uploaded 36GiB to GPU in 1.98718s (18.1162GiB/s) | 20260507 02:38:03.447 | INFO | Downloaded 34GiB to GPU in 1.97344s (17.2288GiB/s) | 20260507 02:38:03.545 | INFO | Starting a round of Baz IO | 20260507 02:38:06.664 | INFO | Populating RTMetrics report | 20260507 02:38:06.664 | INFO | RTMetrics report populated | 20260507 02:38:06.664 | INFO | Opening tmp RTMetrics json file | 20260507 02:38:06.666 | INFO | Serializing RTMetrics to json | 20260507 02:38:06.679 | INFO | Serialization done, swapping tmp file now... | 20260507 02:38:06.685 | INFO | RTMetrics done | 20260507 02:38:06.761 | INFO | Finished a round of Baz IO: Wrote 1.776 GiB in 3.215 seconds | 20260507 02:38:06.761 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.98% / 0.00% / 67.02% | 20260507 02:38:06.761 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.230245% / 3.97364e-06% / 7.54992e-05% | 20260507 02:38:06.761 | INFO | Baz IO overall: | 20260507 02:38:06.761 | INFO | BytesWritten : 3317766146467 | 20260507 02:38:06.761 | INFO | NumEvents : 1595327161510 | 20260507 02:38:08.594 | INFO | Analyzing chunk frames = [6434816,6435328) | 20260507 02:38:08.594 | INFO | MemInfo: { | 20260507 02:38:08.594 | INFO | "gpu" : " 17 GB", | 20260507 02:38:08.594 | INFO | "huge" : "", | 20260507 02:38:08.594 | INFO | "huge_shared" : " 27 GB", | 20260507 02:38:08.594 | INFO | "malloc" : " 14 GB", | 20260507 02:38:08.594 | INFO | "pinned" : " 29 GB" | 20260507 02:38:08.594 | INFO | } | 20260507 02:38:08.594 | INFO | Perf report for the last 5.14696 seconds: | 20260507 02:38:08.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:38:08.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:08.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:08.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:08.594 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260507 02:38:08.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:08.594 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260507 02:38:08.594 | INFO | PRE_HQ : 59.7%, 1.9, 59.2ms, 0 | 20260507 02:38:08.594 | INFO | BAZWRITER : 62.1%, 3.4, 109.7ms, 0 | 20260507 02:38:08.594 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:38:08.594 | INFO | Uploaded 36GiB to GPU in 2.01968s (17.8246GiB/s) | 20260507 02:38:08.594 | INFO | Downloaded 38GiB to GPU in 2.17965s (17.434GiB/s) | 20260507 02:38:13.735 | INFO | Analyzing chunk frames = [6435328,6435840) | 20260507 02:38:13.735 | INFO | MemInfo: { | 20260507 02:38:13.735 | INFO | "gpu" : " 17 GB", | 20260507 02:38:13.735 | INFO | "huge" : "", | 20260507 02:38:13.735 | INFO | "huge_shared" : " 27 GB", | 20260507 02:38:13.735 | INFO | "malloc" : " 14 GB", | 20260507 02:38:13.735 | INFO | "pinned" : " 29 GB" | 20260507 02:38:13.735 | INFO | } | 20260507 02:38:13.735 | INFO | Perf report for the last 5.14108 seconds: | 20260507 02:38:13.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:38:13.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:13.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:13.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:13.735 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:38:13.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:13.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:13.735 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:38:13.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:13.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:13.735 | INFO | Uploaded 36GiB to GPU in 2.0341s (17.6983GiB/s) | 20260507 02:38:13.735 | INFO | Downloaded 34GiB to GPU in 1.94282s (17.5003GiB/s) | 20260507 02:38:18.870 | INFO | Analyzing chunk frames = [6435840,6436352) | 20260507 02:38:18.870 | INFO | MemInfo: { | 20260507 02:38:18.870 | INFO | "gpu" : " 17 GB", | 20260507 02:38:18.870 | INFO | "huge" : "", | 20260507 02:38:18.870 | INFO | "huge_shared" : " 27 GB", | 20260507 02:38:18.870 | INFO | "malloc" : " 14 GB", | 20260507 02:38:18.870 | INFO | "pinned" : " 29 GB" | 20260507 02:38:18.870 | INFO | } | 20260507 02:38:18.870 | INFO | Perf report for the last 5.13494 seconds: | 20260507 02:38:18.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:38:18.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:18.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:18.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:18.870 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 02:38:18.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:18.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:18.870 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:38:18.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:18.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:18.870 | INFO | Uploaded 36GiB to GPU in 2.00124s (17.9888GiB/s) | 20260507 02:38:18.870 | INFO | Downloaded 34GiB to GPU in 1.94305s (17.4983GiB/s) | 20260507 02:38:24.008 | INFO | Analyzing chunk frames = [6436352,6436864) | 20260507 02:38:24.008 | INFO | MemInfo: { | 20260507 02:38:24.008 | INFO | "gpu" : " 17 GB", | 20260507 02:38:24.008 | INFO | "huge" : "", | 20260507 02:38:24.008 | INFO | "huge_shared" : " 27 GB", | 20260507 02:38:24.008 | INFO | "malloc" : " 14 GB", | 20260507 02:38:24.008 | INFO | "pinned" : " 29 GB" | 20260507 02:38:24.008 | INFO | } | 20260507 02:38:24.008 | INFO | Perf report for the last 5.1384 seconds: | 20260507 02:38:24.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:38:24.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:24.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:24.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:24.008 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 02:38:24.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:24.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:24.008 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:38:24.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:24.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:24.009 | INFO | Uploaded 36GiB to GPU in 1.96023s (18.3652GiB/s) | 20260507 02:38:24.009 | INFO | Downloaded 34GiB to GPU in 1.91638s (17.7418GiB/s) | 20260507 02:38:29.144 | INFO | Analyzing chunk frames = [6436864,6437376) | 20260507 02:38:29.144 | INFO | MemInfo: { | 20260507 02:38:29.144 | INFO | "gpu" : " 17 GB", | 20260507 02:38:29.144 | INFO | "huge" : "", | 20260507 02:38:29.144 | INFO | "huge_shared" : " 27 GB", | 20260507 02:38:29.144 | INFO | "malloc" : " 14 GB", | 20260507 02:38:29.144 | INFO | "pinned" : " 29 GB" | 20260507 02:38:29.144 | INFO | } | 20260507 02:38:29.144 | INFO | Perf report for the last 5.13579 seconds: | 20260507 02:38:29.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:38:29.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:29.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:29.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:29.144 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 02:38:29.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:29.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:29.144 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:38:29.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:29.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:29.144 | INFO | Uploaded 36GiB to GPU in 1.9864s (18.1233GiB/s) | 20260507 02:38:29.144 | INFO | Downloaded 34GiB to GPU in 1.94911s (17.4438GiB/s) | 20260507 02:38:34.289 | INFO | Analyzing chunk frames = [6437376,6437888) | 20260507 02:38:34.289 | INFO | MemInfo: { | 20260507 02:38:34.289 | INFO | "gpu" : " 17 GB", | 20260507 02:38:34.289 | INFO | "huge" : "", | 20260507 02:38:34.289 | INFO | "huge_shared" : " 27 GB", | 20260507 02:38:34.289 | INFO | "malloc" : " 14 GB", | 20260507 02:38:34.289 | INFO | "pinned" : " 29 GB" | 20260507 02:38:34.289 | INFO | } | 20260507 02:38:34.289 | INFO | Perf report for the last 5.14482 seconds: | 20260507 02:38:34.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:38:34.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:34.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:34.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:34.289 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 02:38:34.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:34.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:34.289 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:38:34.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:34.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:34.289 | INFO | Uploaded 36GiB to GPU in 2.01778s (17.8414GiB/s) | 20260507 02:38:34.289 | INFO | Downloaded 34GiB to GPU in 1.94717s (17.4612GiB/s) | 20260507 02:38:39.427 | INFO | Analyzing chunk frames = [6437888,6438400) | 20260507 02:38:39.427 | INFO | MemInfo: { | 20260507 02:38:39.427 | INFO | "gpu" : " 17 GB", | 20260507 02:38:39.427 | INFO | "huge" : "", | 20260507 02:38:39.427 | INFO | "huge_shared" : " 27 GB", | 20260507 02:38:39.427 | INFO | "malloc" : " 14 GB", | 20260507 02:38:39.427 | INFO | "pinned" : " 29 GB" | 20260507 02:38:39.427 | INFO | } | 20260507 02:38:39.427 | INFO | Perf report for the last 5.13809 seconds: | 20260507 02:38:39.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:38:39.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:39.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:39.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:39.427 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:38:39.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:39.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:39.427 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:38:39.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:39.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:39.427 | INFO | Uploaded 36GiB to GPU in 1.96963s (18.2776GiB/s) | 20260507 02:38:39.427 | INFO | Downloaded 34GiB to GPU in 1.97563s (17.2097GiB/s) | 20260507 02:38:44.572 | INFO | Analyzing chunk frames = [6438400,6438912) | 20260507 02:38:44.572 | INFO | MemInfo: { | 20260507 02:38:44.572 | INFO | "gpu" : " 17 GB", | 20260507 02:38:44.572 | INFO | "huge" : "", | 20260507 02:38:44.572 | INFO | "huge_shared" : " 27 GB", | 20260507 02:38:44.572 | INFO | "malloc" : " 14 GB", | 20260507 02:38:44.572 | INFO | "pinned" : " 29 GB" | 20260507 02:38:44.572 | INFO | } | 20260507 02:38:44.572 | INFO | Perf report for the last 5.14503 seconds: | 20260507 02:38:44.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:38:44.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:44.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:44.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:44.572 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 02:38:44.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:44.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:44.572 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:38:44.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:44.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:44.573 | INFO | Uploaded 36GiB to GPU in 1.96129s (18.3553GiB/s) | 20260507 02:38:44.573 | INFO | Downloaded 34GiB to GPU in 1.95299s (17.4092GiB/s) | 20260507 02:38:44.573 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:38:44.573 | INFO | Total mem usage: 18 GiB | 20260507 02:38:44.573 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:38:44.573 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:38:44.573 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:38:44.573 | INFO | High Water Breakdown: | 20260507 02:38:44.573 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:38:44.573 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:38:44.573 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:38:44.573 | INFO | ------------------------------------------ | 20260507 02:38:44.573 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:38:44.573 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:38:44.573 | INFO | Total mem usage: 26.5 GiB | 20260507 02:38:44.573 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:38:44.573 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:38:44.573 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:38:44.573 | INFO | High Water Breakdown: | 20260507 02:38:44.573 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:38:44.573 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:38:44.573 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:38:44.573 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:38:44.573 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:38:44.573 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:38:44.573 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:38:44.573 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:38:44.573 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:38:44.573 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:38:44.573 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:38:44.573 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:38:44.573 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:38:44.573 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:38:44.573 | INFO | ------------------------------------------ | 20260507 02:38:44.573 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:38:44.573 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:38:44.573 | INFO | Total mem usage: 25.6 GiB | 20260507 02:38:44.573 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:38:44.573 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:38:44.573 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:38:44.573 | INFO | High Water Breakdown: | 20260507 02:38:44.573 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:38:44.573 | INFO | ------------------------------------------ | 20260507 02:38:44.573 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:38:44.573 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:38:44.573 | INFO | Total mem usage: 15.7 GiB | 20260507 02:38:44.573 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:38:44.573 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:38:44.573 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:38:44.573 | INFO | High Water Breakdown: | 20260507 02:38:44.573 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:38:44.573 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:38:44.573 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:38:44.573 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:38:44.573 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:38:44.573 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:38:44.573 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:38:44.573 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:38:44.573 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:38:44.573 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:38:44.573 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:38:44.574 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:38:44.574 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:38:44.574 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:38:44.574 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:38:44.574 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:38:44.574 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:38:44.574 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:38:44.574 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:38:44.574 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:38:44.574 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:38:44.574 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:38:44.574 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:38:44.574 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:38:44.574 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:38:44.574 | INFO | ------------------------------------------ | 20260507 02:38:44.574 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:38:44.574 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:38:44.574 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:38:44.574 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:38:44.574 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.54e+01, 1.33e+01 / 1.81e+00 | 20260507 02:38:44.574 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.02e-01, 1.03e-02 / 9.16e-03 | 20260507 02:38:44.574 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.45e+00, 5.19e-01 / 2.26e-01 | 20260507 02:38:44.574 | INFO | PerfReport: 3.24%, Histogram -- 9.96e-01 / 1.47e+00, 1.69e-01 / 1.01e-01 | 20260507 02:38:44.574 | INFO | PerfReport: 16.11%, DME -- 3.97e+01 / 4.44e+01, 3.89e+00 / 2.30e+00 | 20260507 02:38:44.574 | INFO | PerfReport: 57.57%, FrameLabeling -- 1.77e+01 / 2.06e+01, 1.28e+00 / 1.80e+00 | 20260507 02:38:44.574 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.16e+00 / 2.71e+00, 3.82e-01 / 2.32e-01 | 20260507 02:38:44.574 | INFO | PerfReport: 3.07%, Metrics -- 9.46e-01 / 1.27e+00, 2.52e-01 / 1.59e-01 | 20260507 02:38:44.574 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.20e-01 / 3.72e-01, 5.03e-02 / 2.37e-02 | 20260507 02:38:44.574 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:38:44.574 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:38:44.574 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:38:44.574 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260507 02:38:44.574 | INFO | PerfReport: 21.65%, Upload -- 2.09e+01 / 0.00e+00, 6.94e+00 / 0.00e+00 | 20260507 02:38:44.574 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260507 02:38:44.574 | INFO | PerfReport: 31.97%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 02:38:44.574 | INFO | PerfReport: 1.33%, LockUpload -- 1.28e+00 / 0.00e+00, 4.76e+00 / 0.00e+00 | 20260507 02:38:44.574 | INFO | PerfReport: 1.40%, LockDownload -- 1.35e+00 / 0.00e+00, 3.25e+00 / 0.00e+00 | 20260507 02:38:44.574 | INFO | PerfReport: 22.21%, LockCompute -- 2.14e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 02:38:44.574 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260507 02:38:44.669 | INFO | Starting a round of Baz IO | 20260507 02:38:47.778 | INFO | Populating RTMetrics report | 20260507 02:38:47.779 | INFO | RTMetrics report populated | 20260507 02:38:47.779 | INFO | Opening tmp RTMetrics json file | 20260507 02:38:47.798 | INFO | Serializing RTMetrics to json | 20260507 02:38:47.799 | INFO | Serialization done, swapping tmp file now... | 20260507 02:38:47.800 | INFO | RTMetrics done | 20260507 02:38:47.874 | INFO | Finished a round of Baz IO: Wrote 1.776 GiB in 3.204 seconds | 20260507 02:38:47.874 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.98% / 0.00% / 67.02% | 20260507 02:38:47.874 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.230114% / 3.97364e-06% / 8.74201e-05% | 20260507 02:38:47.874 | INFO | Baz IO overall: | 20260507 02:38:47.874 | INFO | BytesWritten : 3319672705343 | 20260507 02:38:47.874 | INFO | NumEvents : 1596228535559 | 20260507 02:38:49.710 | INFO | Analyzing chunk frames = [6438912,6439424) | 20260507 02:38:49.710 | INFO | MemInfo: { | 20260507 02:38:49.710 | INFO | "gpu" : " 17 GB", | 20260507 02:38:49.710 | INFO | "huge" : "", | 20260507 02:38:49.710 | INFO | "huge_shared" : " 27 GB", | 20260507 02:38:49.710 | INFO | "malloc" : " 14 GB", | 20260507 02:38:49.710 | INFO | "pinned" : " 29 GB" | 20260507 02:38:49.710 | INFO | } | 20260507 02:38:49.710 | INFO | Perf report for the last 5.13769 seconds: | 20260507 02:38:49.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:38:49.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:49.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:49.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:49.710 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 02:38:49.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:49.710 | INFO | RT_METRICS : 42.1%, 1.0, 22.0ms, 0 | 20260507 02:38:49.710 | INFO | PRE_HQ : 59.3%, 1.8, 55.6ms, 0 | 20260507 02:38:49.710 | INFO | BAZWRITER : 61.8%, 3.3, 105.5ms, 0 | 20260507 02:38:49.710 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:38:49.710 | INFO | Uploaded 36GiB to GPU in 2.03969s (17.6497GiB/s) | 20260507 02:38:49.710 | INFO | Downloaded 38GiB to GPU in 2.18512s (17.3904GiB/s) | 20260507 02:38:54.848 | INFO | Analyzing chunk frames = [6439424,6439936) | 20260507 02:38:54.848 | INFO | MemInfo: { | 20260507 02:38:54.848 | INFO | "gpu" : " 17 GB", | 20260507 02:38:54.848 | INFO | "huge" : "", | 20260507 02:38:54.848 | INFO | "huge_shared" : " 27 GB", | 20260507 02:38:54.848 | INFO | "malloc" : " 14 GB", | 20260507 02:38:54.848 | INFO | "pinned" : " 29 GB" | 20260507 02:38:54.848 | INFO | } | 20260507 02:38:54.848 | INFO | Perf report for the last 5.13844 seconds: | 20260507 02:38:54.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:38:54.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:54.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:54.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:54.848 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:38:54.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:54.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:54.848 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 02:38:54.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:54.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:54.849 | INFO | Uploaded 36GiB to GPU in 1.99315s (18.0619GiB/s) | 20260507 02:38:54.849 | INFO | Downloaded 34GiB to GPU in 1.93483s (17.5726GiB/s) | 20260507 02:38:59.985 | INFO | Analyzing chunk frames = [6439936,6440448) | 20260507 02:38:59.985 | INFO | MemInfo: { | 20260507 02:38:59.985 | INFO | "gpu" : " 17 GB", | 20260507 02:38:59.985 | INFO | "huge" : "", | 20260507 02:38:59.985 | INFO | "huge_shared" : " 27 GB", | 20260507 02:38:59.985 | INFO | "malloc" : " 14 GB", | 20260507 02:38:59.985 | INFO | "pinned" : " 29 GB" | 20260507 02:38:59.985 | INFO | } | 20260507 02:38:59.985 | INFO | Perf report for the last 5.13688 seconds: | 20260507 02:38:59.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:38:59.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:59.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:59.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:59.985 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:38:59.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:59.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:59.985 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:38:59.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:59.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:38:59.985 | INFO | Uploaded 36GiB to GPU in 1.98876s (18.1018GiB/s) | 20260507 02:38:59.986 | INFO | Downloaded 34GiB to GPU in 1.94256s (17.5026GiB/s) | 20260507 02:39:05.121 | INFO | Analyzing chunk frames = [6440448,6440960) | 20260507 02:39:05.121 | INFO | MemInfo: { | 20260507 02:39:05.121 | INFO | "gpu" : " 17 GB", | 20260507 02:39:05.121 | INFO | "huge" : "", | 20260507 02:39:05.121 | INFO | "huge_shared" : " 27 GB", | 20260507 02:39:05.121 | INFO | "malloc" : " 14 GB", | 20260507 02:39:05.121 | INFO | "pinned" : " 29 GB" | 20260507 02:39:05.121 | INFO | } | 20260507 02:39:05.121 | INFO | Perf report for the last 5.13579 seconds: | 20260507 02:39:05.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:39:05.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:05.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:05.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:05.121 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:39:05.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:05.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:05.121 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:39:05.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:05.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:05.121 | INFO | Uploaded 36GiB to GPU in 1.95212s (18.4415GiB/s) | 20260507 02:39:05.121 | INFO | Downloaded 34GiB to GPU in 1.91051s (17.7963GiB/s) | 20260507 02:39:10.265 | INFO | Analyzing chunk frames = [6440960,6441472) | 20260507 02:39:10.265 | INFO | MemInfo: { | 20260507 02:39:10.265 | INFO | "gpu" : " 17 GB", | 20260507 02:39:10.265 | INFO | "huge" : "", | 20260507 02:39:10.265 | INFO | "huge_shared" : " 27 GB", | 20260507 02:39:10.265 | INFO | "malloc" : " 14 GB", | 20260507 02:39:10.265 | INFO | "pinned" : " 29 GB" | 20260507 02:39:10.265 | INFO | } | 20260507 02:39:10.265 | INFO | Perf report for the last 5.14429 seconds: | 20260507 02:39:10.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:39:10.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:10.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:10.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:10.265 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 02:39:10.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:10.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:10.265 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:39:10.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:10.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:10.266 | INFO | Uploaded 36GiB to GPU in 1.98634s (18.1238GiB/s) | 20260507 02:39:10.266 | INFO | Downloaded 34GiB to GPU in 1.93626s (17.5596GiB/s) | 20260507 02:39:15.407 | INFO | Analyzing chunk frames = [6441472,6441984) | 20260507 02:39:15.407 | INFO | MemInfo: { | 20260507 02:39:15.407 | INFO | "gpu" : " 17 GB", | 20260507 02:39:15.407 | INFO | "huge" : "", | 20260507 02:39:15.407 | INFO | "huge_shared" : " 27 GB", | 20260507 02:39:15.407 | INFO | "malloc" : " 14 GB", | 20260507 02:39:15.407 | INFO | "pinned" : " 29 GB" | 20260507 02:39:15.407 | INFO | } | 20260507 02:39:15.407 | INFO | Perf report for the last 5.14214 seconds: | 20260507 02:39:15.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:39:15.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:15.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:15.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:15.408 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 02:39:15.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:15.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:15.408 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 02:39:15.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:15.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:15.408 | INFO | Uploaded 36GiB to GPU in 2.05053s (17.5564GiB/s) | 20260507 02:39:15.408 | INFO | Downloaded 34GiB to GPU in 1.96731s (17.2825GiB/s) | 20260507 02:39:20.545 | INFO | Analyzing chunk frames = [6441984,6442496) | 20260507 02:39:20.545 | INFO | MemInfo: { | 20260507 02:39:20.545 | INFO | "gpu" : " 17 GB", | 20260507 02:39:20.545 | INFO | "huge" : "", | 20260507 02:39:20.545 | INFO | "huge_shared" : " 27 GB", | 20260507 02:39:20.545 | INFO | "malloc" : " 14 GB", | 20260507 02:39:20.545 | INFO | "pinned" : " 29 GB" | 20260507 02:39:20.545 | INFO | } | 20260507 02:39:20.545 | INFO | Perf report for the last 5.13771 seconds: | 20260507 02:39:20.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:39:20.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:20.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:20.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:20.545 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:39:20.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:20.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:20.545 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:39:20.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:20.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:20.546 | INFO | Uploaded 36GiB to GPU in 1.96819s (18.2909GiB/s) | 20260507 02:39:20.546 | INFO | Downloaded 34GiB to GPU in 1.97769s (17.1917GiB/s) | 20260507 02:39:25.682 | INFO | Analyzing chunk frames = [6442496,6443008) | 20260507 02:39:25.682 | INFO | MemInfo: { | 20260507 02:39:25.682 | INFO | "gpu" : " 17 GB", | 20260507 02:39:25.682 | INFO | "huge" : "", | 20260507 02:39:25.682 | INFO | "huge_shared" : " 27 GB", | 20260507 02:39:25.682 | INFO | "malloc" : " 14 GB", | 20260507 02:39:25.682 | INFO | "pinned" : " 29 GB" | 20260507 02:39:25.682 | INFO | } | 20260507 02:39:25.682 | INFO | Perf report for the last 5.13662 seconds: | 20260507 02:39:25.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:39:25.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:25.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:25.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:25.682 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 02:39:25.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:25.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:25.682 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 02:39:25.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:25.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:25.682 | INFO | Uploaded 36GiB to GPU in 1.96353s (18.3344GiB/s) | 20260507 02:39:25.682 | INFO | Downloaded 34GiB to GPU in 1.96789s (17.2774GiB/s) | 20260507 02:39:25.779 | INFO | Starting a round of Baz IO | 20260507 02:39:28.888 | INFO | Populating RTMetrics report | 20260507 02:39:28.889 | INFO | RTMetrics report populated | 20260507 02:39:28.889 | INFO | Opening tmp RTMetrics json file | 20260507 02:39:28.889 | INFO | Serializing RTMetrics to json | 20260507 02:39:28.890 | INFO | Serialization done, swapping tmp file now... | 20260507 02:39:28.891 | INFO | RTMetrics done | 20260507 02:39:28.989 | INFO | Finished a round of Baz IO: Wrote 1.774 GiB in 3.21 seconds | 20260507 02:39:28.989 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.98% / 0.00% / 67.02% | 20260507 02:39:28.989 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.22794% / 3.97364e-06% / 8.34465e-05% | 20260507 02:39:28.989 | INFO | Baz IO overall: | 20260507 02:39:28.989 | INFO | BytesWritten : 3321577332374 | 20260507 02:39:28.989 | INFO | NumEvents : 1597128427799 | 20260507 02:39:30.817 | INFO | Analyzing chunk frames = [6443008,6443520) | 20260507 02:39:30.817 | INFO | MemInfo: { | 20260507 02:39:30.817 | INFO | "gpu" : " 17 GB", | 20260507 02:39:30.817 | INFO | "huge" : "", | 20260507 02:39:30.817 | INFO | "huge_shared" : " 27 GB", | 20260507 02:39:30.817 | INFO | "malloc" : " 14 GB", | 20260507 02:39:30.817 | INFO | "pinned" : " 29 GB" | 20260507 02:39:30.817 | INFO | } | 20260507 02:39:30.817 | INFO | Perf report for the last 5.13501 seconds: | 20260507 02:39:30.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:39:30.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:30.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:30.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:30.817 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 02:39:30.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:30.817 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 02:39:30.817 | INFO | PRE_HQ : 59.6%, 1.8, 56.2ms, 0 | 20260507 02:39:30.817 | INFO | BAZWRITER : 62.1%, 3.3, 105.9ms, 0 | 20260507 02:39:30.817 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:39:30.817 | INFO | Uploaded 36GiB to GPU in 2.01641s (17.8535GiB/s) | 20260507 02:39:30.817 | INFO | Downloaded 38GiB to GPU in 2.18256s (17.4107GiB/s) | 20260507 02:39:35.959 | INFO | Analyzing chunk frames = [6443520,6444032) | 20260507 02:39:35.959 | INFO | MemInfo: { | 20260507 02:39:35.959 | INFO | "gpu" : " 17 GB", | 20260507 02:39:35.959 | INFO | "huge" : "", | 20260507 02:39:35.959 | INFO | "huge_shared" : " 27 GB", | 20260507 02:39:35.959 | INFO | "malloc" : " 14 GB", | 20260507 02:39:35.959 | INFO | "pinned" : " 29 GB" | 20260507 02:39:35.959 | INFO | } | 20260507 02:39:35.959 | INFO | Perf report for the last 5.14198 seconds: | 20260507 02:39:35.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:39:35.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:35.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:35.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:35.959 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 02:39:35.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:35.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:35.959 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:39:35.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:35.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:35.959 | INFO | Uploaded 36GiB to GPU in 1.98991s (18.0913GiB/s) | 20260507 02:39:35.959 | INFO | Downloaded 34GiB to GPU in 1.93087s (17.6086GiB/s) | 20260507 02:39:41.099 | INFO | Analyzing chunk frames = [6444032,6444544) | 20260507 02:39:41.099 | INFO | MemInfo: { | 20260507 02:39:41.099 | INFO | "gpu" : " 17 GB", | 20260507 02:39:41.099 | INFO | "huge" : "", | 20260507 02:39:41.099 | INFO | "huge_shared" : " 27 GB", | 20260507 02:39:41.099 | INFO | "malloc" : " 14 GB", | 20260507 02:39:41.099 | INFO | "pinned" : " 29 GB" | 20260507 02:39:41.099 | INFO | } | 20260507 02:39:41.099 | INFO | Perf report for the last 5.14031 seconds: | 20260507 02:39:41.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:39:41.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:41.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:41.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:41.099 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:39:41.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:41.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:41.099 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:39:41.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:41.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:41.099 | INFO | Uploaded 36GiB to GPU in 1.98939s (18.096GiB/s) | 20260507 02:39:41.099 | INFO | Downloaded 34GiB to GPU in 1.93777s (17.5459GiB/s) | 20260507 02:39:46.239 | INFO | Analyzing chunk frames = [6444544,6445056) | 20260507 02:39:46.239 | INFO | MemInfo: { | 20260507 02:39:46.239 | INFO | "gpu" : " 17 GB", | 20260507 02:39:46.239 | INFO | "huge" : "", | 20260507 02:39:46.239 | INFO | "huge_shared" : " 27 GB", | 20260507 02:39:46.239 | INFO | "malloc" : " 14 GB", | 20260507 02:39:46.239 | INFO | "pinned" : " 29 GB" | 20260507 02:39:46.239 | INFO | } | 20260507 02:39:46.239 | INFO | Perf report for the last 5.13944 seconds: | 20260507 02:39:46.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:39:46.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:46.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:46.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:46.239 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 02:39:46.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:46.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:46.239 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:39:46.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:46.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:46.239 | INFO | Uploaded 36GiB to GPU in 1.97627s (18.2161GiB/s) | 20260507 02:39:46.239 | INFO | Downloaded 34GiB to GPU in 1.915s (17.7546GiB/s) | 20260507 02:39:51.377 | INFO | Analyzing chunk frames = [6445056,6445568) | 20260507 02:39:51.377 | INFO | MemInfo: { | 20260507 02:39:51.377 | INFO | "gpu" : " 17 GB", | 20260507 02:39:51.377 | INFO | "huge" : "", | 20260507 02:39:51.377 | INFO | "huge_shared" : " 27 GB", | 20260507 02:39:51.377 | INFO | "malloc" : " 14 GB", | 20260507 02:39:51.377 | INFO | "pinned" : " 29 GB" | 20260507 02:39:51.377 | INFO | } | 20260507 02:39:51.377 | INFO | Perf report for the last 5.13866 seconds: | 20260507 02:39:51.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:39:51.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:51.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:51.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:51.377 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 02:39:51.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:51.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:51.377 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:39:51.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:51.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:51.378 | INFO | Uploaded 36GiB to GPU in 2.005s (17.9551GiB/s) | 20260507 02:39:51.378 | INFO | Downloaded 34GiB to GPU in 1.94548s (17.4764GiB/s) | 20260507 02:39:56.526 | INFO | Analyzing chunk frames = [6445568,6446080) | 20260507 02:39:56.526 | INFO | MemInfo: { | 20260507 02:39:56.526 | INFO | "gpu" : " 17 GB", | 20260507 02:39:56.526 | INFO | "huge" : "", | 20260507 02:39:56.526 | INFO | "huge_shared" : " 27 GB", | 20260507 02:39:56.526 | INFO | "malloc" : " 14 GB", | 20260507 02:39:56.526 | INFO | "pinned" : " 29 GB" | 20260507 02:39:56.526 | INFO | } | 20260507 02:39:56.526 | INFO | Perf report for the last 5.14878 seconds: | 20260507 02:39:56.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:39:56.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:56.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:56.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:56.526 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 02:39:56.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:56.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:56.526 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:39:56.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:56.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:39:56.526 | INFO | Uploaded 36GiB to GPU in 2.04985s (17.5622GiB/s) | 20260507 02:39:56.527 | INFO | Downloaded 34GiB to GPU in 1.95675s (17.3757GiB/s) | 20260507 02:40:01.657 | INFO | Analyzing chunk frames = [6446080,6446592) | 20260507 02:40:01.657 | INFO | MemInfo: { | 20260507 02:40:01.657 | INFO | "gpu" : " 17 GB", | 20260507 02:40:01.657 | INFO | "huge" : "", | 20260507 02:40:01.657 | INFO | "huge_shared" : " 27 GB", | 20260507 02:40:01.657 | INFO | "malloc" : " 14 GB", | 20260507 02:40:01.657 | INFO | "pinned" : " 29 GB" | 20260507 02:40:01.657 | INFO | } | 20260507 02:40:01.657 | INFO | Perf report for the last 5.13126 seconds: | 20260507 02:40:01.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:40:01.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:01.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:01.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:01.658 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 02:40:01.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:01.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:01.658 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 02:40:01.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:01.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:01.658 | INFO | Uploaded 36GiB to GPU in 1.96814s (18.2914GiB/s) | 20260507 02:40:01.658 | INFO | Downloaded 34GiB to GPU in 1.97728s (17.1953GiB/s) | 20260507 02:40:06.794 | INFO | Analyzing chunk frames = [6446592,6447104) | 20260507 02:40:06.794 | INFO | MemInfo: { | 20260507 02:40:06.794 | INFO | "gpu" : " 17 GB", | 20260507 02:40:06.794 | INFO | "huge" : "", | 20260507 02:40:06.794 | INFO | "huge_shared" : " 27 GB", | 20260507 02:40:06.794 | INFO | "malloc" : " 14 GB", | 20260507 02:40:06.794 | INFO | "pinned" : " 29 GB" | 20260507 02:40:06.794 | INFO | } | 20260507 02:40:06.794 | INFO | Perf report for the last 5.13645 seconds: | 20260507 02:40:06.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:40:06.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:06.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:06.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:06.794 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:40:06.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:06.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:06.794 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 02:40:06.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:06.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:06.794 | INFO | Uploaded 36GiB to GPU in 1.9879s (18.1095GiB/s) | 20260507 02:40:06.794 | INFO | Downloaded 34GiB to GPU in 1.9781s (17.1882GiB/s) | 20260507 02:40:06.794 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:40:06.794 | INFO | Total mem usage: 18 GiB | 20260507 02:40:06.794 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:40:06.794 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:40:06.794 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:40:06.794 | INFO | High Water Breakdown: | 20260507 02:40:06.794 | INFO | 60.2% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:40:06.794 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:40:06.794 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:40:06.794 | INFO | ------------------------------------------ | 20260507 02:40:06.794 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:40:06.794 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:40:06.794 | INFO | Total mem usage: 26.5 GiB | 20260507 02:40:06.794 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:40:06.794 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:40:06.794 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:40:06.795 | INFO | High Water Breakdown: | 20260507 02:40:06.795 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:40:06.795 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:40:06.795 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:40:06.795 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:40:06.795 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:40:06.795 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:40:06.795 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:40:06.795 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:40:06.795 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:40:06.795 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:40:06.795 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:40:06.795 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:40:06.795 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:40:06.795 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:40:06.795 | INFO | ------------------------------------------ | 20260507 02:40:06.795 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:40:06.795 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:40:06.795 | INFO | Total mem usage: 25.6 GiB | 20260507 02:40:06.795 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:40:06.795 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 02:40:06.795 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 02:40:06.795 | INFO | High Water Breakdown: | 20260507 02:40:06.795 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 02:40:06.795 | INFO | ------------------------------------------ | 20260507 02:40:06.795 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:40:06.795 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:40:06.795 | INFO | Total mem usage: 15.7 GiB | 20260507 02:40:06.795 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:40:06.795 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:40:06.795 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:40:06.795 | INFO | High Water Breakdown: | 20260507 02:40:06.795 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:40:06.795 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:40:06.795 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:40:06.795 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:40:06.795 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:40:06.795 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:40:06.795 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:40:06.795 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:40:06.795 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:40:06.795 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:40:06.795 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:40:06.795 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:40:06.795 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:40:06.795 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:40:06.795 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:40:06.795 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:40:06.795 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:40:06.795 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:40:06.795 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:40:06.795 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:40:06.795 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:40:06.795 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:40:06.795 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:40:06.795 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:40:06.795 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:40:06.795 | INFO | ------------------------------------------ | 20260507 02:40:06.795 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:40:06.795 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:40:06.795 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:40:06.795 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:40:06.795 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.50e+01, 1.35e+01 / 1.59e+00 | 20260507 02:40:06.795 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 1.00e-01, 9.39e-03 / 6.14e-03 | 20260507 02:40:06.795 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.39e+00, 5.22e-01 / 3.54e-01 | 20260507 02:40:06.795 | INFO | PerfReport: 3.23%, Histogram -- 9.95e-01 / 1.44e+00, 1.70e-01 / 1.34e-01 | 20260507 02:40:06.795 | INFO | PerfReport: 16.07%, DME -- 3.96e+01 / 4.41e+01, 3.81e+00 / 1.42e+00 | 20260507 02:40:06.795 | INFO | PerfReport: 57.62%, FrameLabeling -- 1.77e+01 / 2.07e+01, 1.33e+00 / 1.45e+00 | 20260507 02:40:06.795 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.15e+00 / 2.66e+00, 3.81e-01 / 2.00e-01 | 20260507 02:40:06.795 | INFO | PerfReport: 3.08%, Metrics -- 9.47e-01 / 1.26e+00, 2.53e-01 / 1.11e-01 | 20260507 02:40:06.795 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.21e-01 / 3.73e-01, 5.08e-02 / 1.85e-02 | 20260507 02:40:06.795 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:40:06.795 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:40:06.795 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:40:06.795 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 02:40:06.795 | INFO | PerfReport: 21.60%, Upload -- 2.08e+01 / 0.00e+00, 6.95e+00 / 0.00e+00 | 20260507 02:40:06.795 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260507 02:40:06.795 | INFO | PerfReport: 32.00%, Compute -- 3.08e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 02:40:06.795 | INFO | PerfReport: 1.31%, LockUpload -- 1.26e+00 / 0.00e+00, 4.68e+00 / 0.00e+00 | 20260507 02:40:06.795 | INFO | PerfReport: 1.39%, LockDownload -- 1.34e+00 / 0.00e+00, 3.27e+00 / 0.00e+00 | 20260507 02:40:06.795 | INFO | PerfReport: 22.28%, LockCompute -- 2.14e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 02:40:06.795 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.51e-02 / 0.00e+00 | 20260507 02:40:06.891 | INFO | Starting a round of Baz IO | 20260507 02:40:09.998 | INFO | Populating RTMetrics report | 20260507 02:40:09.998 | INFO | RTMetrics report populated | 20260507 02:40:09.998 | INFO | Opening tmp RTMetrics json file | 20260507 02:40:10.017 | INFO | Serializing RTMetrics to json | 20260507 02:40:10.018 | INFO | Serialization done, swapping tmp file now... | 20260507 02:40:10.019 | INFO | RTMetrics done | 20260507 02:40:10.092 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.2 seconds | 20260507 02:40:10.092 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.98% / 0.00% / 67.02% | 20260507 02:40:10.092 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.228437% / 0% / 8.74201e-05% | 20260507 02:40:10.092 | INFO | Baz IO overall: | 20260507 02:40:10.092 | INFO | BytesWritten : 3323482921308 | 20260507 02:40:10.092 | INFO | NumEvents : 1598029165570 | 20260507 02:40:11.941 | INFO | Analyzing chunk frames = [6447104,6447616) | 20260507 02:40:11.941 | INFO | MemInfo: { | 20260507 02:40:11.941 | INFO | "gpu" : " 17 GB", | 20260507 02:40:11.941 | INFO | "huge" : "", | 20260507 02:40:11.941 | INFO | "huge_shared" : " 27 GB", | 20260507 02:40:11.941 | INFO | "malloc" : " 14 GB", | 20260507 02:40:11.941 | INFO | "pinned" : " 29 GB" | 20260507 02:40:11.941 | INFO | } | 20260507 02:40:11.941 | INFO | Perf report for the last 5.14759 seconds: | 20260507 02:40:11.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:40:11.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:11.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:11.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:11.942 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260507 02:40:11.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:11.942 | INFO | RT_METRICS : 41.8%, 1.0, 22.0ms, 0 | 20260507 02:40:11.942 | INFO | PRE_HQ : 59.2%, 1.8, 55.5ms, 0 | 20260507 02:40:11.942 | INFO | BAZWRITER : 61.8%, 3.3, 106.1ms, 0 | 20260507 02:40:11.942 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:40:11.942 | INFO | Uploaded 36GiB to GPU in 2.0364s (17.6782GiB/s) | 20260507 02:40:11.942 | INFO | Downloaded 38GiB to GPU in 2.18421s (17.3976GiB/s) | 20260507 02:40:17.072 | INFO | Analyzing chunk frames = [6447616,6448128) | 20260507 02:40:17.072 | INFO | MemInfo: { | 20260507 02:40:17.072 | INFO | "gpu" : " 17 GB", | 20260507 02:40:17.072 | INFO | "huge" : "", | 20260507 02:40:17.072 | INFO | "huge_shared" : " 27 GB", | 20260507 02:40:17.072 | INFO | "malloc" : " 14 GB", | 20260507 02:40:17.072 | INFO | "pinned" : " 29 GB" | 20260507 02:40:17.073 | INFO | } | 20260507 02:40:17.073 | INFO | Perf report for the last 5.1313 seconds: | 20260507 02:40:17.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:40:17.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:17.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:17.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:17.073 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 02:40:17.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:17.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:17.073 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:40:17.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:17.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:17.073 | INFO | Uploaded 36GiB to GPU in 2.03808s (17.6637GiB/s) | 20260507 02:40:17.073 | INFO | Downloaded 34GiB to GPU in 1.94302s (17.4985GiB/s) | 20260507 02:40:22.216 | INFO | Analyzing chunk frames = [6448128,6448640) | 20260507 02:40:22.216 | INFO | MemInfo: { | 20260507 02:40:22.216 | INFO | "gpu" : " 17 GB", | 20260507 02:40:22.216 | INFO | "huge" : "", | 20260507 02:40:22.216 | INFO | "huge_shared" : " 27 GB", | 20260507 02:40:22.216 | INFO | "malloc" : " 14 GB", | 20260507 02:40:22.216 | INFO | "pinned" : " 29 GB" | 20260507 02:40:22.216 | INFO | } | 20260507 02:40:22.216 | INFO | Perf report for the last 5.14327 seconds: | 20260507 02:40:22.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:40:22.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:22.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:22.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:22.216 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 02:40:22.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:22.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:22.216 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:40:22.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:22.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:22.216 | INFO | Uploaded 36GiB to GPU in 1.97581s (18.2204GiB/s) | 20260507 02:40:22.216 | INFO | Downloaded 34GiB to GPU in 1.94098s (17.5169GiB/s) | 20260507 02:40:27.353 | INFO | Analyzing chunk frames = [6448640,6449152) | 20260507 02:40:27.353 | INFO | MemInfo: { | 20260507 02:40:27.353 | INFO | "gpu" : " 17 GB", | 20260507 02:40:27.353 | INFO | "huge" : "", | 20260507 02:40:27.353 | INFO | "huge_shared" : " 27 GB", | 20260507 02:40:27.353 | INFO | "malloc" : " 14 GB", | 20260507 02:40:27.353 | INFO | "pinned" : " 29 GB" | 20260507 02:40:27.353 | INFO | } | 20260507 02:40:27.353 | INFO | Perf report for the last 5.13684 seconds: | 20260507 02:40:27.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:40:27.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:27.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:27.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:27.353 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:40:27.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:27.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:27.353 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:40:27.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:27.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:27.353 | INFO | Uploaded 36GiB to GPU in 1.95746s (18.3912GiB/s) | 20260507 02:40:27.353 | INFO | Downloaded 34GiB to GPU in 1.92288s (17.6818GiB/s) | 20260507 02:40:32.489 | INFO | Analyzing chunk frames = [6449152,6449664) | 20260507 02:40:32.489 | INFO | MemInfo: { | 20260507 02:40:32.489 | INFO | "gpu" : " 17 GB", | 20260507 02:40:32.489 | INFO | "huge" : "", | 20260507 02:40:32.489 | INFO | "huge_shared" : " 27 GB", | 20260507 02:40:32.489 | INFO | "malloc" : " 14 GB", | 20260507 02:40:32.489 | INFO | "pinned" : " 29 GB" | 20260507 02:40:32.489 | INFO | } | 20260507 02:40:32.489 | INFO | Perf report for the last 5.13597 seconds: | 20260507 02:40:32.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:40:32.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:32.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:32.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:32.489 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 02:40:32.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:32.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:32.489 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:40:32.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:32.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:32.489 | INFO | Uploaded 36GiB to GPU in 2.01549s (17.8616GiB/s) | 20260507 02:40:32.489 | INFO | Downloaded 34GiB to GPU in 1.93551s (17.5664GiB/s) | 20260507 02:40:37.626 | INFO | Analyzing chunk frames = [6449664,6450176) | 20260507 02:40:37.626 | INFO | MemInfo: { | 20260507 02:40:37.626 | INFO | "gpu" : " 17 GB", | 20260507 02:40:37.626 | INFO | "huge" : "", | 20260507 02:40:37.626 | INFO | "huge_shared" : " 27 GB", | 20260507 02:40:37.626 | INFO | "malloc" : " 14 GB", | 20260507 02:40:37.626 | INFO | "pinned" : " 29 GB" | 20260507 02:40:37.626 | INFO | } | 20260507 02:40:37.626 | INFO | Perf report for the last 5.13731 seconds: | 20260507 02:40:37.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:40:37.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:37.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:37.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:37.626 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 02:40:37.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:37.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:37.626 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:40:37.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:37.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:37.626 | INFO | Uploaded 36GiB to GPU in 2.04887s (17.5706GiB/s) | 20260507 02:40:37.627 | INFO | Downloaded 34GiB to GPU in 1.96018s (17.3454GiB/s) | 20260507 02:40:42.771 | INFO | Analyzing chunk frames = [6450176,6450688) | 20260507 02:40:42.771 | INFO | MemInfo: { | 20260507 02:40:42.771 | INFO | "gpu" : " 17 GB", | 20260507 02:40:42.771 | INFO | "huge" : "", | 20260507 02:40:42.771 | INFO | "huge_shared" : " 27 GB", | 20260507 02:40:42.771 | INFO | "malloc" : " 14 GB", | 20260507 02:40:42.771 | INFO | "pinned" : " 29 GB" | 20260507 02:40:42.771 | INFO | } | 20260507 02:40:42.771 | INFO | Perf report for the last 5.14521 seconds: | 20260507 02:40:42.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:40:42.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:42.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:42.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:42.771 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:40:42.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:42.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:42.772 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 02:40:42.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:42.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:42.772 | INFO | Uploaded 36GiB to GPU in 1.96831s (18.2898GiB/s) | 20260507 02:40:42.772 | INFO | Downloaded 34GiB to GPU in 1.9978s (17.0187GiB/s) | 20260507 02:40:47.913 | INFO | Analyzing chunk frames = [6450688,6451200) | 20260507 02:40:47.913 | INFO | MemInfo: { | 20260507 02:40:47.913 | INFO | "gpu" : " 17 GB", | 20260507 02:40:47.913 | INFO | "huge" : "", | 20260507 02:40:47.913 | INFO | "huge_shared" : " 27 GB", | 20260507 02:40:47.913 | INFO | "malloc" : " 14 GB", | 20260507 02:40:47.913 | INFO | "pinned" : " 29 GB" | 20260507 02:40:47.913 | INFO | } | 20260507 02:40:47.913 | INFO | Perf report for the last 5.14155 seconds: | 20260507 02:40:47.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:40:47.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:47.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:47.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:47.913 | INFO | ANALYSIS : 61.1%, 3.0, 95.2ms, 0 | 20260507 02:40:47.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:47.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:47.913 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 02:40:47.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:47.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:47.913 | INFO | Uploaded 36GiB to GPU in 2.01411s (17.8739GiB/s) | 20260507 02:40:47.913 | INFO | Downloaded 34GiB to GPU in 1.97416s (17.2225GiB/s) | 20260507 02:40:48.010 | INFO | Starting a round of Baz IO | 20260507 02:40:51.118 | INFO | Populating RTMetrics report | 20260507 02:40:51.119 | INFO | RTMetrics report populated | 20260507 02:40:51.119 | INFO | Opening tmp RTMetrics json file | 20260507 02:40:51.121 | INFO | Serializing RTMetrics to json | 20260507 02:40:51.138 | INFO | Serialization done, swapping tmp file now... | 20260507 02:40:51.139 | INFO | RTMetrics done | 20260507 02:40:51.210 | INFO | Finished a round of Baz IO: Wrote 1.774 GiB in 3.2 seconds | 20260507 02:40:51.210 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.98% / 0.00% / 67.02% | 20260507 02:40:51.210 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.227408% / 0% / 8.74201e-05% | 20260507 02:40:51.210 | INFO | Baz IO overall: | 20260507 02:40:51.210 | INFO | BytesWritten : 3325387411532 | 20260507 02:40:51.210 | INFO | NumEvents : 1598929149644 | 20260507 02:40:53.052 | INFO | Analyzing chunk frames = [6451200,6451712) | 20260507 02:40:53.052 | INFO | MemInfo: { | 20260507 02:40:53.052 | INFO | "gpu" : " 17 GB", | 20260507 02:40:53.052 | INFO | "huge" : "", | 20260507 02:40:53.052 | INFO | "huge_shared" : " 27 GB", | 20260507 02:40:53.052 | INFO | "malloc" : " 14 GB", | 20260507 02:40:53.052 | INFO | "pinned" : " 29 GB" | 20260507 02:40:53.052 | INFO | } | 20260507 02:40:53.052 | INFO | Perf report for the last 5.13935 seconds: | 20260507 02:40:53.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:40:53.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:53.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:53.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:53.052 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 02:40:53.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:53.052 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 02:40:53.052 | INFO | PRE_HQ : 59.2%, 1.8, 55.4ms, 0 | 20260507 02:40:53.052 | INFO | BAZWRITER : 61.9%, 3.3, 105.4ms, 0 | 20260507 02:40:53.052 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:40:53.053 | INFO | Uploaded 36GiB to GPU in 2.04625s (17.5932GiB/s) | 20260507 02:40:53.053 | INFO | Downloaded 38GiB to GPU in 2.18229s (17.4129GiB/s) | 20260507 02:40:58.190 | INFO | Analyzing chunk frames = [6451712,6452224) | 20260507 02:40:58.190 | INFO | MemInfo: { | 20260507 02:40:58.190 | INFO | "gpu" : " 17 GB", | 20260507 02:40:58.190 | INFO | "huge" : "", | 20260507 02:40:58.190 | INFO | "huge_shared" : " 27 GB", | 20260507 02:40:58.190 | INFO | "malloc" : " 14 GB", | 20260507 02:40:58.190 | INFO | "pinned" : " 29 GB" | 20260507 02:40:58.190 | INFO | } | 20260507 02:40:58.190 | INFO | Perf report for the last 5.13788 seconds: | 20260507 02:40:58.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:40:58.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:58.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:58.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:58.190 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 02:40:58.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:58.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:58.190 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:40:58.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:58.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:40:58.190 | INFO | Uploaded 36GiB to GPU in 2.01507s (17.8654GiB/s) | 20260507 02:40:58.190 | INFO | Downloaded 34GiB to GPU in 1.93301s (17.5891GiB/s) | 20260507 02:41:03.327 | INFO | Analyzing chunk frames = [6452224,6452736) | 20260507 02:41:03.327 | INFO | MemInfo: { | 20260507 02:41:03.327 | INFO | "gpu" : " 17 GB", | 20260507 02:41:03.327 | INFO | "huge" : "", | 20260507 02:41:03.327 | INFO | "huge_shared" : " 27 GB", | 20260507 02:41:03.327 | INFO | "malloc" : " 14 GB", | 20260507 02:41:03.327 | INFO | "pinned" : " 29 GB" | 20260507 02:41:03.327 | INFO | } | 20260507 02:41:03.327 | INFO | Perf report for the last 5.13697 seconds: | 20260507 02:41:03.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:41:03.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:03.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:03.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:03.327 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 02:41:03.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:03.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:03.327 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:41:03.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:03.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:03.327 | INFO | Uploaded 36GiB to GPU in 2.0085s (17.9238GiB/s) | 20260507 02:41:03.327 | INFO | Downloaded 34GiB to GPU in 1.93291s (17.5901GiB/s) | 20260507 02:41:08.467 | INFO | Analyzing chunk frames = [6452736,6453248) | 20260507 02:41:08.467 | INFO | MemInfo: { | 20260507 02:41:08.467 | INFO | "gpu" : " 17 GB", | 20260507 02:41:08.467 | INFO | "huge" : "", | 20260507 02:41:08.467 | INFO | "huge_shared" : " 27 GB", | 20260507 02:41:08.467 | INFO | "malloc" : " 14 GB", | 20260507 02:41:08.468 | INFO | "pinned" : " 29 GB" | 20260507 02:41:08.468 | INFO | } | 20260507 02:41:08.468 | INFO | Perf report for the last 5.14038 seconds: | 20260507 02:41:08.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:41:08.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:08.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:08.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:08.468 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 02:41:08.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:08.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:08.468 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:41:08.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:08.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:08.468 | INFO | Uploaded 36GiB to GPU in 1.96461s (18.3242GiB/s) | 20260507 02:41:08.468 | INFO | Downloaded 34GiB to GPU in 1.90826s (17.8173GiB/s) | 20260507 02:41:13.613 | INFO | Analyzing chunk frames = [6453248,6453760) | 20260507 02:41:13.613 | INFO | MemInfo: { | 20260507 02:41:13.613 | INFO | "gpu" : " 17 GB", | 20260507 02:41:13.613 | INFO | "huge" : "", | 20260507 02:41:13.613 | INFO | "huge_shared" : " 27 GB", | 20260507 02:41:13.613 | INFO | "malloc" : " 14 GB", | 20260507 02:41:13.613 | INFO | "pinned" : " 29 GB" | 20260507 02:41:13.613 | INFO | } | 20260507 02:41:13.613 | INFO | Perf report for the last 5.1451 seconds: | 20260507 02:41:13.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:41:13.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:13.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:13.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:13.613 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 02:41:13.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:13.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:13.613 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 02:41:13.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:13.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:13.613 | INFO | Uploaded 36GiB to GPU in 2.01083s (17.9031GiB/s) | 20260507 02:41:13.613 | INFO | Downloaded 34GiB to GPU in 1.94142s (17.5129GiB/s) | 20260507 02:41:18.750 | INFO | Analyzing chunk frames = [6453760,6454272) | 20260507 02:41:18.750 | INFO | MemInfo: { | 20260507 02:41:18.750 | INFO | "gpu" : " 17 GB", | 20260507 02:41:18.750 | INFO | "huge" : "", | 20260507 02:41:18.750 | INFO | "huge_shared" : " 27 GB", | 20260507 02:41:18.750 | INFO | "malloc" : " 14 GB", | 20260507 02:41:18.750 | INFO | "pinned" : " 29 GB" | 20260507 02:41:18.750 | INFO | } | 20260507 02:41:18.750 | INFO | Perf report for the last 5.13777 seconds: | 20260507 02:41:18.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:41:18.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:18.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:18.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:18.751 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 02:41:18.751 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:18.751 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:18.751 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:41:18.751 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:18.751 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:18.751 | INFO | Uploaded 36GiB to GPU in 2.04049s (17.6428GiB/s) | 20260507 02:41:18.751 | INFO | Downloaded 34GiB to GPU in 1.91603s (17.745GiB/s) | 20260507 02:41:23.890 | INFO | Analyzing chunk frames = [6454272,6454784) | 20260507 02:41:23.890 | INFO | MemInfo: { | 20260507 02:41:23.890 | INFO | "gpu" : " 17 GB", | 20260507 02:41:23.890 | INFO | "huge" : "", | 20260507 02:41:23.890 | INFO | "huge_shared" : " 27 GB", | 20260507 02:41:23.890 | INFO | "malloc" : " 14 GB", | 20260507 02:41:23.890 | INFO | "pinned" : " 29 GB" | 20260507 02:41:23.890 | INFO | } | 20260507 02:41:23.890 | INFO | Perf report for the last 5.14002 seconds: | 20260507 02:41:23.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:41:23.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:23.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:23.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:23.891 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:41:23.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:23.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:23.891 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 02:41:23.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:23.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:23.891 | INFO | Uploaded 36GiB to GPU in 1.97828s (18.1976GiB/s) | 20260507 02:41:23.891 | INFO | Downloaded 34GiB to GPU in 1.97786s (17.1903GiB/s) | 20260507 02:41:29.033 | INFO | Analyzing chunk frames = [6454784,6455296) | 20260507 02:41:29.033 | INFO | MemInfo: { | 20260507 02:41:29.033 | INFO | "gpu" : " 17 GB", | 20260507 02:41:29.033 | INFO | "huge" : "", | 20260507 02:41:29.033 | INFO | "huge_shared" : " 27 GB", | 20260507 02:41:29.033 | INFO | "malloc" : " 14 GB", | 20260507 02:41:29.034 | INFO | "pinned" : " 29 GB" | 20260507 02:41:29.034 | INFO | } | 20260507 02:41:29.034 | INFO | Perf report for the last 5.14314 seconds: | 20260507 02:41:29.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:41:29.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:29.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:29.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:29.034 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 02:41:29.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:29.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:29.034 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:41:29.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:29.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:29.034 | INFO | Uploaded 36GiB to GPU in 1.96387s (18.3312GiB/s) | 20260507 02:41:29.034 | INFO | Downloaded 34GiB to GPU in 1.95895s (17.3563GiB/s) | 20260507 02:41:29.034 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:41:29.034 | INFO | Total mem usage: 18 GiB | 20260507 02:41:29.034 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:41:29.034 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:41:29.034 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:41:29.034 | INFO | High Water Breakdown: | 20260507 02:41:29.034 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:41:29.034 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:41:29.034 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:41:29.034 | INFO | ------------------------------------------ | 20260507 02:41:29.034 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:41:29.034 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:41:29.034 | INFO | Total mem usage: 26.5 GiB | 20260507 02:41:29.034 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:41:29.034 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:41:29.034 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 02:41:29.034 | INFO | High Water Breakdown: | 20260507 02:41:29.034 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:41:29.034 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:41:29.034 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:41:29.034 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:41:29.034 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:41:29.034 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:41:29.034 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:41:29.034 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:41:29.034 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:41:29.034 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:41:29.034 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:41:29.035 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:41:29.035 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:41:29.035 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:41:29.035 | INFO | ------------------------------------------ | 20260507 02:41:29.035 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:41:29.035 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:41:29.035 | INFO | Total mem usage: 25.6 GiB | 20260507 02:41:29.035 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:41:29.035 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:41:29.035 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:41:29.035 | INFO | High Water Breakdown: | 20260507 02:41:29.035 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:41:29.035 | INFO | ------------------------------------------ | 20260507 02:41:29.035 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:41:29.035 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:41:29.035 | INFO | Total mem usage: 15.7 GiB | 20260507 02:41:29.035 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:41:29.035 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:41:29.035 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:41:29.035 | INFO | High Water Breakdown: | 20260507 02:41:29.035 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:41:29.035 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:41:29.035 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:41:29.035 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:41:29.035 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:41:29.035 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:41:29.035 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:41:29.035 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:41:29.035 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:41:29.035 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:41:29.035 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:41:29.035 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:41:29.035 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:41:29.035 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:41:29.035 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:41:29.035 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:41:29.035 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:41:29.035 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:41:29.035 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:41:29.035 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:41:29.035 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:41:29.035 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:41:29.035 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:41:29.035 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:41:29.035 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:41:29.035 | INFO | ------------------------------------------ | 20260507 02:41:29.035 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:41:29.035 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:41:29.035 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:41:29.035 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:41:29.035 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.55e+01, 1.32e+01 / 2.04e+00 | 20260507 02:41:29.035 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.39e-03 / 4.50e-03 | 20260507 02:41:29.035 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.49e+00, 5.26e-01 / 2.95e-01 | 20260507 02:41:29.035 | INFO | PerfReport: 3.24%, Histogram -- 9.99e-01 / 1.45e+00, 1.71e-01 / 1.24e-01 | 20260507 02:41:29.035 | INFO | PerfReport: 16.13%, DME -- 3.97e+01 / 4.46e+01, 4.12e+00 / 2.17e+00 | 20260507 02:41:29.035 | INFO | PerfReport: 57.60%, FrameLabeling -- 1.77e+01 / 2.06e+01, 1.32e+00 / 1.42e+00 | 20260507 02:41:29.035 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.14e+00 / 2.63e+00, 3.77e-01 / 2.51e-01 | 20260507 02:41:29.035 | INFO | PerfReport: 3.07%, Metrics -- 9.46e-01 / 1.25e+00, 2.54e-01 / 1.08e-01 | 20260507 02:41:29.035 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.20e-01 / 3.67e-01, 5.08e-02 / 1.54e-02 | 20260507 02:41:29.035 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:41:29.035 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:41:29.035 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:41:29.035 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 02:41:29.035 | INFO | PerfReport: 21.67%, Upload -- 2.09e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260507 02:41:29.035 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 02:41:29.035 | INFO | PerfReport: 31.97%, Compute -- 3.08e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 02:41:29.035 | INFO | PerfReport: 1.44%, LockUpload -- 1.38e+00 / 0.00e+00, 4.89e+00 / 0.00e+00 | 20260507 02:41:29.035 | INFO | PerfReport: 1.31%, LockDownload -- 1.27e+00 / 0.00e+00, 3.16e+00 / 0.00e+00 | 20260507 02:41:29.035 | INFO | PerfReport: 22.24%, LockCompute -- 2.14e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 02:41:29.035 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260507 02:41:29.131 | INFO | Starting a round of Baz IO | 20260507 02:41:32.237 | INFO | Populating RTMetrics report | 20260507 02:41:32.237 | INFO | RTMetrics report populated | 20260507 02:41:32.237 | INFO | Opening tmp RTMetrics json file | 20260507 02:41:32.239 | INFO | Serializing RTMetrics to json | 20260507 02:41:32.243 | INFO | Serialization done, swapping tmp file now... | 20260507 02:41:32.255 | INFO | RTMetrics done | 20260507 02:41:32.330 | INFO | Finished a round of Baz IO: Wrote 1.776 GiB in 3.199 seconds | 20260507 02:41:32.331 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.98% / 0.00% / 67.02% | 20260507 02:41:32.331 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.228659% / 0% / 6.75519e-05% | 20260507 02:41:32.331 | INFO | Baz IO overall: | 20260507 02:41:32.331 | INFO | BytesWritten : 3327294862577 | 20260507 02:41:32.331 | INFO | NumEvents : 1599831486009 | 20260507 02:41:34.167 | INFO | Analyzing chunk frames = [6455296,6455808) | 20260507 02:41:34.167 | INFO | MemInfo: { | 20260507 02:41:34.167 | INFO | "gpu" : " 17 GB", | 20260507 02:41:34.167 | INFO | "huge" : "", | 20260507 02:41:34.167 | INFO | "huge_shared" : " 27 GB", | 20260507 02:41:34.167 | INFO | "malloc" : " 14 GB", | 20260507 02:41:34.167 | INFO | "pinned" : " 29 GB" | 20260507 02:41:34.167 | INFO | } | 20260507 02:41:34.167 | INFO | Perf report for the last 5.13293 seconds: | 20260507 02:41:34.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:41:34.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:34.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:34.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:34.167 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260507 02:41:34.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:34.167 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 02:41:34.167 | INFO | PRE_HQ : 59.4%, 1.8, 55.4ms, 0 | 20260507 02:41:34.167 | INFO | BAZWRITER : 61.9%, 3.3, 104.4ms, 0 | 20260507 02:41:34.167 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:41:34.167 | INFO | Uploaded 36GiB to GPU in 2.04668s (17.5895GiB/s) | 20260507 02:41:34.167 | INFO | Downloaded 38GiB to GPU in 2.18239s (17.4121GiB/s) | 20260507 02:41:39.305 | INFO | Analyzing chunk frames = [6455808,6456320) | 20260507 02:41:39.305 | INFO | MemInfo: { | 20260507 02:41:39.305 | INFO | "gpu" : " 17 GB", | 20260507 02:41:39.305 | INFO | "huge" : "", | 20260507 02:41:39.305 | INFO | "huge_shared" : " 27 GB", | 20260507 02:41:39.305 | INFO | "malloc" : " 14 GB", | 20260507 02:41:39.305 | INFO | "pinned" : " 29 GB" | 20260507 02:41:39.305 | INFO | } | 20260507 02:41:39.305 | INFO | Perf report for the last 5.13848 seconds: | 20260507 02:41:39.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:41:39.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:39.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:39.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:39.305 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:41:39.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:39.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:39.305 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 02:41:39.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:39.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:39.305 | INFO | Uploaded 36GiB to GPU in 2.03867s (17.6586GiB/s) | 20260507 02:41:39.306 | INFO | Downloaded 34GiB to GPU in 1.93722s (17.5509GiB/s) | 20260507 02:41:44.437 | INFO | Analyzing chunk frames = [6456320,6456832) | 20260507 02:41:44.437 | INFO | MemInfo: { | 20260507 02:41:44.437 | INFO | "gpu" : " 17 GB", | 20260507 02:41:44.437 | INFO | "huge" : "", | 20260507 02:41:44.438 | INFO | "huge_shared" : " 27 GB", | 20260507 02:41:44.438 | INFO | "malloc" : " 14 GB", | 20260507 02:41:44.438 | INFO | "pinned" : " 29 GB" | 20260507 02:41:44.438 | INFO | } | 20260507 02:41:44.438 | INFO | Perf report for the last 5.13256 seconds: | 20260507 02:41:44.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:41:44.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:44.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:44.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:44.438 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 02:41:44.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:44.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:44.438 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:41:44.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:44.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:44.438 | INFO | Uploaded 36GiB to GPU in 2.00903s (17.9191GiB/s) | 20260507 02:41:44.438 | INFO | Downloaded 34GiB to GPU in 1.93956s (17.5297GiB/s) | 20260507 02:41:49.581 | INFO | Analyzing chunk frames = [6456832,6457344) | 20260507 02:41:49.581 | INFO | MemInfo: { | 20260507 02:41:49.581 | INFO | "gpu" : " 17 GB", | 20260507 02:41:49.581 | INFO | "huge" : "", | 20260507 02:41:49.581 | INFO | "huge_shared" : " 27 GB", | 20260507 02:41:49.581 | INFO | "malloc" : " 14 GB", | 20260507 02:41:49.581 | INFO | "pinned" : " 29 GB" | 20260507 02:41:49.581 | INFO | } | 20260507 02:41:49.581 | INFO | Perf report for the last 5.14352 seconds: | 20260507 02:41:49.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:41:49.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:49.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:49.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:49.581 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260507 02:41:49.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:49.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:49.581 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 02:41:49.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:49.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:49.582 | INFO | Uploaded 36GiB to GPU in 2.01021s (17.9086GiB/s) | 20260507 02:41:49.582 | INFO | Downloaded 34GiB to GPU in 1.91222s (17.7803GiB/s) | 20260507 02:41:54.728 | INFO | Analyzing chunk frames = [6457344,6457856) | 20260507 02:41:54.728 | INFO | MemInfo: { | 20260507 02:41:54.728 | INFO | "gpu" : " 17 GB", | 20260507 02:41:54.728 | INFO | "huge" : "", | 20260507 02:41:54.728 | INFO | "huge_shared" : " 27 GB", | 20260507 02:41:54.728 | INFO | "malloc" : " 14 GB", | 20260507 02:41:54.728 | INFO | "pinned" : " 29 GB" | 20260507 02:41:54.728 | INFO | } | 20260507 02:41:54.728 | INFO | Perf report for the last 5.14648 seconds: | 20260507 02:41:54.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:41:54.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:54.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:54.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:54.728 | INFO | ANALYSIS : 59.9%, 3.0, 93.4ms, 0 | 20260507 02:41:54.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:54.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:54.728 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:41:54.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:54.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:54.728 | INFO | Uploaded 36GiB to GPU in 2.02109s (17.8122GiB/s) | 20260507 02:41:54.728 | INFO | Downloaded 34GiB to GPU in 1.94517s (17.4792GiB/s) | 20260507 02:41:59.864 | INFO | Analyzing chunk frames = [6457856,6458368) | 20260507 02:41:59.864 | INFO | MemInfo: { | 20260507 02:41:59.864 | INFO | "gpu" : " 17 GB", | 20260507 02:41:59.864 | INFO | "huge" : "", | 20260507 02:41:59.864 | INFO | "huge_shared" : " 27 GB", | 20260507 02:41:59.864 | INFO | "malloc" : " 14 GB", | 20260507 02:41:59.864 | INFO | "pinned" : " 29 GB" | 20260507 02:41:59.864 | INFO | } | 20260507 02:41:59.864 | INFO | Perf report for the last 5.13663 seconds: | 20260507 02:41:59.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:41:59.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:59.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:59.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:59.864 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 02:41:59.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:59.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:59.864 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 02:41:59.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:59.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:41:59.865 | INFO | Uploaded 36GiB to GPU in 2.02319s (17.7937GiB/s) | 20260507 02:41:59.865 | INFO | Downloaded 34GiB to GPU in 1.91754s (17.7311GiB/s) | 20260507 02:42:05.000 | INFO | Analyzing chunk frames = [6458368,6458880) | 20260507 02:42:05.000 | INFO | MemInfo: { | 20260507 02:42:05.000 | INFO | "gpu" : " 17 GB", | 20260507 02:42:05.001 | INFO | "huge" : "", | 20260507 02:42:05.001 | INFO | "huge_shared" : " 27 GB", | 20260507 02:42:05.001 | INFO | "malloc" : " 14 GB", | 20260507 02:42:05.001 | INFO | "pinned" : " 29 GB" | 20260507 02:42:05.001 | INFO | } | 20260507 02:42:05.001 | INFO | Perf report for the last 5.13621 seconds: | 20260507 02:42:05.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:42:05.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:05.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:05.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:05.001 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 02:42:05.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:05.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:05.001 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:42:05.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:05.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:05.001 | INFO | Uploaded 36GiB to GPU in 1.935s (18.6047GiB/s) | 20260507 02:42:05.001 | INFO | Downloaded 34GiB to GPU in 1.97337s (17.2294GiB/s) | 20260507 02:42:10.137 | INFO | Analyzing chunk frames = [6458880,6459392) | 20260507 02:42:10.137 | INFO | MemInfo: { | 20260507 02:42:10.137 | INFO | "gpu" : " 17 GB", | 20260507 02:42:10.137 | INFO | "huge" : "", | 20260507 02:42:10.137 | INFO | "huge_shared" : " 27 GB", | 20260507 02:42:10.137 | INFO | "malloc" : " 14 GB", | 20260507 02:42:10.137 | INFO | "pinned" : " 29 GB" | 20260507 02:42:10.137 | INFO | } | 20260507 02:42:10.137 | INFO | Perf report for the last 5.13608 seconds: | 20260507 02:42:10.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:42:10.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:10.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:10.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:10.137 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 02:42:10.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:10.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:10.137 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:42:10.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:10.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:10.137 | INFO | Uploaded 36GiB to GPU in 1.97255s (18.2504GiB/s) | 20260507 02:42:10.137 | INFO | Downloaded 34GiB to GPU in 1.95335s (17.406GiB/s) | 20260507 02:42:10.234 | INFO | Starting a round of Baz IO | 20260507 02:42:13.346 | INFO | Populating RTMetrics report | 20260507 02:42:13.347 | INFO | RTMetrics report populated | 20260507 02:42:13.347 | INFO | Opening tmp RTMetrics json file | 20260507 02:42:13.350 | INFO | Serializing RTMetrics to json | 20260507 02:42:13.365 | INFO | Serialization done, swapping tmp file now... | 20260507 02:42:13.366 | INFO | RTMetrics done | 20260507 02:42:13.440 | INFO | Finished a round of Baz IO: Wrote 1.774 GiB in 3.205 seconds | 20260507 02:42:13.440 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.97% / 0.00% / 67.02% | 20260507 02:42:13.440 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.226653% / 0% / 6.35783e-05% | 20260507 02:42:13.440 | INFO | Baz IO overall: | 20260507 02:42:13.440 | INFO | BytesWritten : 3329199290147 | 20260507 02:42:13.440 | INFO | NumEvents : 1600731522711 | 20260507 02:42:15.279 | INFO | Analyzing chunk frames = [6459392,6459904) | 20260507 02:42:15.279 | INFO | MemInfo: { | 20260507 02:42:15.279 | INFO | "gpu" : " 17 GB", | 20260507 02:42:15.279 | INFO | "huge" : "", | 20260507 02:42:15.279 | INFO | "huge_shared" : " 27 GB", | 20260507 02:42:15.279 | INFO | "malloc" : " 14 GB", | 20260507 02:42:15.279 | INFO | "pinned" : " 29 GB" | 20260507 02:42:15.279 | INFO | } | 20260507 02:42:15.279 | INFO | Perf report for the last 5.14238 seconds: | 20260507 02:42:15.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:42:15.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:15.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:15.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:15.279 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260507 02:42:15.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:15.279 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 02:42:15.279 | INFO | PRE_HQ : 59.3%, 1.8, 55.3ms, 0 | 20260507 02:42:15.279 | INFO | BAZWRITER : 61.8%, 3.3, 105.8ms, 0 | 20260507 02:42:15.279 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:42:15.280 | INFO | Uploaded 36GiB to GPU in 2.02091s (17.8137GiB/s) | 20260507 02:42:15.280 | INFO | Downloaded 38GiB to GPU in 2.17672s (17.4574GiB/s) | 20260507 02:42:20.417 | INFO | Analyzing chunk frames = [6459904,6460416) | 20260507 02:42:20.417 | INFO | MemInfo: { | 20260507 02:42:20.417 | INFO | "gpu" : " 17 GB", | 20260507 02:42:20.417 | INFO | "huge" : "", | 20260507 02:42:20.417 | INFO | "huge_shared" : " 27 GB", | 20260507 02:42:20.417 | INFO | "malloc" : " 14 GB", | 20260507 02:42:20.417 | INFO | "pinned" : " 29 GB" | 20260507 02:42:20.417 | INFO | } | 20260507 02:42:20.417 | INFO | Perf report for the last 5.13823 seconds: | 20260507 02:42:20.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:42:20.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:20.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:20.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:20.417 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:42:20.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:20.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:20.417 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 02:42:20.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:20.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:20.418 | INFO | Uploaded 36GiB to GPU in 1.99307s (18.0626GiB/s) | 20260507 02:42:20.418 | INFO | Downloaded 34GiB to GPU in 1.93115s (17.6061GiB/s) | 20260507 02:42:25.560 | INFO | Analyzing chunk frames = [6460416,6460928) | 20260507 02:42:25.560 | INFO | MemInfo: { | 20260507 02:42:25.560 | INFO | "gpu" : " 17 GB", | 20260507 02:42:25.560 | INFO | "huge" : "", | 20260507 02:42:25.560 | INFO | "huge_shared" : " 27 GB", | 20260507 02:42:25.560 | INFO | "malloc" : " 14 GB", | 20260507 02:42:25.560 | INFO | "pinned" : " 29 GB" | 20260507 02:42:25.560 | INFO | } | 20260507 02:42:25.560 | INFO | Perf report for the last 5.14256 seconds: | 20260507 02:42:25.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:42:25.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:25.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:25.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:25.560 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:42:25.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:25.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:25.560 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:42:25.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:25.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:25.560 | INFO | Uploaded 36GiB to GPU in 1.99374s (18.0565GiB/s) | 20260507 02:42:25.560 | INFO | Downloaded 34GiB to GPU in 1.93932s (17.5319GiB/s) | 20260507 02:42:30.701 | INFO | Analyzing chunk frames = [6460928,6461440) | 20260507 02:42:30.701 | INFO | MemInfo: { | 20260507 02:42:30.701 | INFO | "gpu" : " 17 GB", | 20260507 02:42:30.701 | INFO | "huge" : "", | 20260507 02:42:30.701 | INFO | "huge_shared" : " 27 GB", | 20260507 02:42:30.701 | INFO | "malloc" : " 14 GB", | 20260507 02:42:30.701 | INFO | "pinned" : " 29 GB" | 20260507 02:42:30.701 | INFO | } | 20260507 02:42:30.701 | INFO | Perf report for the last 5.14092 seconds: | 20260507 02:42:30.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:42:30.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:30.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:30.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:30.701 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:42:30.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:30.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:30.701 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:42:30.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:30.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:30.701 | INFO | Uploaded 36GiB to GPU in 1.9778s (18.2021GiB/s) | 20260507 02:42:30.701 | INFO | Downloaded 34GiB to GPU in 1.91905s (17.7171GiB/s) | 20260507 02:42:35.842 | INFO | Analyzing chunk frames = [6461440,6461952) | 20260507 02:42:35.842 | INFO | MemInfo: { | 20260507 02:42:35.842 | INFO | "gpu" : " 17 GB", | 20260507 02:42:35.842 | INFO | "huge" : "", | 20260507 02:42:35.842 | INFO | "huge_shared" : " 27 GB", | 20260507 02:42:35.842 | INFO | "malloc" : " 14 GB", | 20260507 02:42:35.842 | INFO | "pinned" : " 29 GB" | 20260507 02:42:35.842 | INFO | } | 20260507 02:42:35.842 | INFO | Perf report for the last 5.14143 seconds: | 20260507 02:42:35.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:42:35.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:35.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:35.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:35.842 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 02:42:35.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:35.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:35.842 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:42:35.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:35.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:35.842 | INFO | Uploaded 36GiB to GPU in 2.03401s (17.699GiB/s) | 20260507 02:42:35.842 | INFO | Downloaded 34GiB to GPU in 1.93021s (17.6146GiB/s) | 20260507 02:42:40.981 | INFO | Analyzing chunk frames = [6461952,6462464) | 20260507 02:42:40.981 | INFO | MemInfo: { | 20260507 02:42:40.981 | INFO | "gpu" : " 17 GB", | 20260507 02:42:40.981 | INFO | "huge" : "", | 20260507 02:42:40.981 | INFO | "huge_shared" : " 27 GB", | 20260507 02:42:40.981 | INFO | "malloc" : " 14 GB", | 20260507 02:42:40.981 | INFO | "pinned" : " 29 GB" | 20260507 02:42:40.981 | INFO | } | 20260507 02:42:40.982 | INFO | Perf report for the last 5.13945 seconds: | 20260507 02:42:40.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:42:40.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:40.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:40.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:40.982 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 02:42:40.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:40.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:40.982 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 02:42:40.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:40.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:40.982 | INFO | Uploaded 36GiB to GPU in 2.05432s (17.5241GiB/s) | 20260507 02:42:40.982 | INFO | Downloaded 34GiB to GPU in 1.97256s (17.2365GiB/s) | 20260507 02:42:46.117 | INFO | Analyzing chunk frames = [6462464,6462976) | 20260507 02:42:46.117 | INFO | MemInfo: { | 20260507 02:42:46.117 | INFO | "gpu" : " 17 GB", | 20260507 02:42:46.117 | INFO | "huge" : "", | 20260507 02:42:46.117 | INFO | "huge_shared" : " 27 GB", | 20260507 02:42:46.117 | INFO | "malloc" : " 14 GB", | 20260507 02:42:46.117 | INFO | "pinned" : " 29 GB" | 20260507 02:42:46.117 | INFO | } | 20260507 02:42:46.117 | INFO | Perf report for the last 5.13537 seconds: | 20260507 02:42:46.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:42:46.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:46.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:46.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:46.117 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:42:46.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:46.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:46.117 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:42:46.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:46.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:46.117 | INFO | Uploaded 36GiB to GPU in 1.9727s (18.2491GiB/s) | 20260507 02:42:46.117 | INFO | Downloaded 34GiB to GPU in 1.96189s (17.3302GiB/s) | 20260507 02:42:51.256 | INFO | Analyzing chunk frames = [6462976,6463488) | 20260507 02:42:51.256 | INFO | MemInfo: { | 20260507 02:42:51.256 | INFO | "gpu" : " 17 GB", | 20260507 02:42:51.256 | INFO | "huge" : "", | 20260507 02:42:51.256 | INFO | "huge_shared" : " 27 GB", | 20260507 02:42:51.256 | INFO | "malloc" : " 14 GB", | 20260507 02:42:51.256 | INFO | "pinned" : " 29 GB" | 20260507 02:42:51.256 | INFO | } | 20260507 02:42:51.256 | INFO | Perf report for the last 5.13936 seconds: | 20260507 02:42:51.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:42:51.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:51.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:51.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:51.257 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 02:42:51.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:51.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:51.257 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:42:51.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:51.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:51.257 | INFO | Uploaded 36GiB to GPU in 1.98329s (18.1516GiB/s) | 20260507 02:42:51.257 | INFO | Downloaded 34GiB to GPU in 1.96183s (17.3307GiB/s) | 20260507 02:42:51.257 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:42:51.257 | INFO | Total mem usage: 18 GiB | 20260507 02:42:51.257 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:42:51.257 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:42:51.257 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:42:51.257 | INFO | High Water Breakdown: | 20260507 02:42:51.257 | INFO | 60.2% : 8.58 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:42:51.257 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:42:51.257 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:42:51.257 | INFO | ------------------------------------------ | 20260507 02:42:51.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:42:51.257 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:42:51.257 | INFO | Total mem usage: 26.5 GiB | 20260507 02:42:51.257 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:42:51.257 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:42:51.257 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:42:51.257 | INFO | High Water Breakdown: | 20260507 02:42:51.257 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:42:51.257 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:42:51.257 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:42:51.257 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:42:51.257 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:42:51.257 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:42:51.257 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:42:51.257 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:42:51.257 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:42:51.257 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:42:51.257 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:42:51.257 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:42:51.257 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:42:51.257 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:42:51.257 | INFO | ------------------------------------------ | 20260507 02:42:51.257 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:42:51.257 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:42:51.257 | INFO | Total mem usage: 25.6 GiB | 20260507 02:42:51.257 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:42:51.257 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 02:42:51.257 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 02:42:51.257 | INFO | High Water Breakdown: | 20260507 02:42:51.257 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 02:42:51.257 | INFO | ------------------------------------------ | 20260507 02:42:51.258 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:42:51.258 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:42:51.258 | INFO | Total mem usage: 15.7 GiB | 20260507 02:42:51.258 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:42:51.258 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:42:51.258 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 02:42:51.258 | INFO | High Water Breakdown: | 20260507 02:42:51.258 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:42:51.258 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:42:51.258 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:42:51.258 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:42:51.258 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:42:51.258 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:42:51.258 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:42:51.258 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:42:51.258 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:42:51.258 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:42:51.258 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:42:51.258 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:42:51.258 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:42:51.258 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:42:51.258 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:42:51.258 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:42:51.258 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:42:51.258 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:42:51.258 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:42:51.258 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:42:51.258 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:42:51.258 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:42:51.258 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:42:51.258 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:42:51.258 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:42:51.258 | INFO | ------------------------------------------ | 20260507 02:42:51.258 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:42:51.258 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:42:51.258 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:42:51.258 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:42:51.258 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.49e+01, 1.33e+01 / 1.08e+00 | 20260507 02:42:51.258 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.00e-01, 9.65e-03 / 5.14e-03 | 20260507 02:42:51.258 | INFO | PerfReport: 11.68%, Baseline -- 3.60e+00 / 4.49e+00, 5.31e-01 / 2.75e-01 | 20260507 02:42:51.258 | INFO | PerfReport: 3.23%, Histogram -- 9.96e-01 / 1.46e+00, 1.71e-01 / 1.09e-01 | 20260507 02:42:51.258 | INFO | PerfReport: 16.03%, DME -- 3.95e+01 / 4.40e+01, 3.72e+00 / 1.16e+00 | 20260507 02:42:51.258 | INFO | PerfReport: 57.68%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.39e+00 / 1.37e+00 | 20260507 02:42:51.258 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.15e+00 / 2.71e+00, 3.80e-01 / 2.15e-01 | 20260507 02:42:51.258 | INFO | PerfReport: 3.07%, Metrics -- 9.45e-01 / 1.25e+00, 2.51e-01 / 1.23e-01 | 20260507 02:42:51.258 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.21e-01 / 3.70e-01, 5.13e-02 / 1.79e-02 | 20260507 02:42:51.258 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:42:51.258 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:42:51.258 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:42:51.258 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 02:42:51.258 | INFO | PerfReport: 21.70%, Upload -- 2.09e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 02:42:51.258 | INFO | PerfReport: 21.34%, Download -- 2.05e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 02:42:51.258 | INFO | PerfReport: 32.01%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 02:42:51.258 | INFO | PerfReport: 1.40%, LockUpload -- 1.35e+00 / 0.00e+00, 4.67e+00 / 0.00e+00 | 20260507 02:42:51.258 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260507 02:42:51.258 | INFO | PerfReport: 22.26%, LockCompute -- 2.14e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 02:42:51.258 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.62e-02 / 0.00e+00 | 20260507 02:42:51.353 | INFO | Starting a round of Baz IO | 20260507 02:42:54.464 | INFO | Populating RTMetrics report | 20260507 02:42:54.464 | INFO | RTMetrics report populated | 20260507 02:42:54.464 | INFO | Opening tmp RTMetrics json file | 20260507 02:42:54.481 | INFO | Serializing RTMetrics to json | 20260507 02:42:54.482 | INFO | Serialization done, swapping tmp file now... | 20260507 02:42:54.483 | INFO | RTMetrics done | 20260507 02:42:54.556 | INFO | Finished a round of Baz IO: Wrote 1.773 GiB in 3.202 seconds | 20260507 02:42:54.556 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.97% / 0.00% / 67.03% | 20260507 02:42:54.556 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.226712% / 0% / 8.74201e-05% | 20260507 02:42:54.556 | INFO | Baz IO overall: | 20260507 02:42:54.556 | INFO | BytesWritten : 3331102566293 | 20260507 02:42:54.556 | INFO | NumEvents : 1601630625575 | 20260507 02:42:56.401 | INFO | Analyzing chunk frames = [6463488,6464000) | 20260507 02:42:56.401 | INFO | MemInfo: { | 20260507 02:42:56.401 | INFO | "gpu" : " 17 GB", | 20260507 02:42:56.401 | INFO | "huge" : "", | 20260507 02:42:56.401 | INFO | "huge_shared" : " 27 GB", | 20260507 02:42:56.401 | INFO | "malloc" : " 14 GB", | 20260507 02:42:56.401 | INFO | "pinned" : " 29 GB" | 20260507 02:42:56.401 | INFO | } | 20260507 02:42:56.401 | INFO | Perf report for the last 5.14424 seconds: | 20260507 02:42:56.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:42:56.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:56.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:56.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:56.401 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260507 02:42:56.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:42:56.401 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 02:42:56.401 | INFO | PRE_HQ : 59.2%, 1.8, 55.5ms, 0 | 20260507 02:42:56.401 | INFO | BAZWRITER : 61.8%, 3.3, 105.7ms, 0 | 20260507 02:42:56.401 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:42:56.401 | INFO | Uploaded 36GiB to GPU in 2.03818s (17.6628GiB/s) | 20260507 02:42:56.401 | INFO | Downloaded 38GiB to GPU in 2.18086s (17.4243GiB/s) | 20260507 02:43:01.536 | INFO | Analyzing chunk frames = [6464000,6464512) | 20260507 02:43:01.536 | INFO | MemInfo: { | 20260507 02:43:01.536 | INFO | "gpu" : " 17 GB", | 20260507 02:43:01.536 | INFO | "huge" : "", | 20260507 02:43:01.536 | INFO | "huge_shared" : " 27 GB", | 20260507 02:43:01.536 | INFO | "malloc" : " 14 GB", | 20260507 02:43:01.536 | INFO | "pinned" : " 29 GB" | 20260507 02:43:01.536 | INFO | } | 20260507 02:43:01.536 | INFO | Perf report for the last 5.13532 seconds: | 20260507 02:43:01.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:43:01.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:01.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:01.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:01.536 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:43:01.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:01.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:01.536 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:43:01.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:01.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:01.536 | INFO | Uploaded 36GiB to GPU in 2.03496s (17.6907GiB/s) | 20260507 02:43:01.536 | INFO | Downloaded 34GiB to GPU in 1.93796s (17.5442GiB/s) | 20260507 02:43:06.681 | INFO | Analyzing chunk frames = [6464512,6465024) | 20260507 02:43:06.681 | INFO | MemInfo: { | 20260507 02:43:06.681 | INFO | "gpu" : " 17 GB", | 20260507 02:43:06.681 | INFO | "huge" : "", | 20260507 02:43:06.681 | INFO | "huge_shared" : " 27 GB", | 20260507 02:43:06.681 | INFO | "malloc" : " 14 GB", | 20260507 02:43:06.681 | INFO | "pinned" : " 29 GB" | 20260507 02:43:06.681 | INFO | } | 20260507 02:43:06.681 | INFO | Perf report for the last 5.14511 seconds: | 20260507 02:43:06.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:43:06.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:06.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:06.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:06.681 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 02:43:06.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:06.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:06.681 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:43:06.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:06.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:06.681 | INFO | Uploaded 36GiB to GPU in 2.01599s (17.8573GiB/s) | 20260507 02:43:06.681 | INFO | Downloaded 34GiB to GPU in 1.94185s (17.5091GiB/s) | 20260507 02:43:11.815 | INFO | Analyzing chunk frames = [6465024,6465536) | 20260507 02:43:11.815 | INFO | MemInfo: { | 20260507 02:43:11.815 | INFO | "gpu" : " 17 GB", | 20260507 02:43:11.815 | INFO | "huge" : "", | 20260507 02:43:11.815 | INFO | "huge_shared" : " 27 GB", | 20260507 02:43:11.815 | INFO | "malloc" : " 14 GB", | 20260507 02:43:11.815 | INFO | "pinned" : " 29 GB" | 20260507 02:43:11.815 | INFO | } | 20260507 02:43:11.815 | INFO | Perf report for the last 5.13381 seconds: | 20260507 02:43:11.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:43:11.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:11.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:11.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:11.815 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 02:43:11.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:11.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:11.815 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:43:11.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:11.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:11.815 | INFO | Uploaded 36GiB to GPU in 2.0065s (17.9417GiB/s) | 20260507 02:43:11.815 | INFO | Downloaded 34GiB to GPU in 1.93024s (17.6144GiB/s) | 20260507 02:43:16.952 | INFO | Analyzing chunk frames = [6465536,6466048) | 20260507 02:43:16.952 | INFO | MemInfo: { | 20260507 02:43:16.952 | INFO | "gpu" : " 17 GB", | 20260507 02:43:16.952 | INFO | "huge" : "", | 20260507 02:43:16.952 | INFO | "huge_shared" : " 27 GB", | 20260507 02:43:16.952 | INFO | "malloc" : " 14 GB", | 20260507 02:43:16.952 | INFO | "pinned" : " 29 GB" | 20260507 02:43:16.952 | INFO | } | 20260507 02:43:16.952 | INFO | Perf report for the last 5.13726 seconds: | 20260507 02:43:16.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:43:16.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:16.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:16.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:16.952 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 02:43:16.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:16.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:16.952 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:43:16.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:16.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:16.953 | INFO | Uploaded 36GiB to GPU in 2.02889s (17.7437GiB/s) | 20260507 02:43:16.953 | INFO | Downloaded 34GiB to GPU in 1.9396s (17.5294GiB/s) | 20260507 02:43:22.093 | INFO | Analyzing chunk frames = [6466048,6466560) | 20260507 02:43:22.093 | INFO | MemInfo: { | 20260507 02:43:22.093 | INFO | "gpu" : " 17 GB", | 20260507 02:43:22.093 | INFO | "huge" : "", | 20260507 02:43:22.093 | INFO | "huge_shared" : " 27 GB", | 20260507 02:43:22.093 | INFO | "malloc" : " 14 GB", | 20260507 02:43:22.093 | INFO | "pinned" : " 29 GB" | 20260507 02:43:22.093 | INFO | } | 20260507 02:43:22.093 | INFO | Perf report for the last 5.14047 seconds: | 20260507 02:43:22.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:43:22.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:22.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:22.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:22.093 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 02:43:22.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:22.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:22.093 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 02:43:22.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:22.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:22.093 | INFO | Uploaded 36GiB to GPU in 2.05713s (17.5001GiB/s) | 20260507 02:43:22.093 | INFO | Downloaded 34GiB to GPU in 1.95569s (17.3852GiB/s) | 20260507 02:43:27.232 | INFO | Analyzing chunk frames = [6466560,6467072) | 20260507 02:43:27.232 | INFO | MemInfo: { | 20260507 02:43:27.232 | INFO | "gpu" : " 17 GB", | 20260507 02:43:27.232 | INFO | "huge" : "", | 20260507 02:43:27.232 | INFO | "huge_shared" : " 27 GB", | 20260507 02:43:27.232 | INFO | "malloc" : " 14 GB", | 20260507 02:43:27.232 | INFO | "pinned" : " 29 GB" | 20260507 02:43:27.232 | INFO | } | 20260507 02:43:27.232 | INFO | Perf report for the last 5.13948 seconds: | 20260507 02:43:27.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:43:27.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:27.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:27.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:27.232 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:43:27.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:27.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:27.232 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 02:43:27.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:27.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:27.233 | INFO | Uploaded 36GiB to GPU in 1.98588s (18.1279GiB/s) | 20260507 02:43:27.233 | INFO | Downloaded 34GiB to GPU in 1.99756s (17.0208GiB/s) | 20260507 02:43:32.369 | INFO | Analyzing chunk frames = [6467072,6467584) | 20260507 02:43:32.369 | INFO | MemInfo: { | 20260507 02:43:32.369 | INFO | "gpu" : " 17 GB", | 20260507 02:43:32.369 | INFO | "huge" : "", | 20260507 02:43:32.369 | INFO | "huge_shared" : " 27 GB", | 20260507 02:43:32.369 | INFO | "malloc" : " 14 GB", | 20260507 02:43:32.369 | INFO | "pinned" : " 29 GB" | 20260507 02:43:32.369 | INFO | } | 20260507 02:43:32.369 | INFO | Perf report for the last 5.13671 seconds: | 20260507 02:43:32.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:43:32.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:32.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:32.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:32.369 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:43:32.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:32.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:32.369 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 02:43:32.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:32.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:32.369 | INFO | Uploaded 36GiB to GPU in 2.00479s (17.957GiB/s) | 20260507 02:43:32.369 | INFO | Downloaded 34GiB to GPU in 1.96526s (17.3005GiB/s) | 20260507 02:43:32.468 | INFO | Starting a round of Baz IO | 20260507 02:43:35.590 | INFO | Populating RTMetrics report | 20260507 02:43:35.590 | INFO | RTMetrics report populated | 20260507 02:43:35.590 | INFO | Opening tmp RTMetrics json file | 20260507 02:43:35.592 | INFO | Serializing RTMetrics to json | 20260507 02:43:35.593 | INFO | Serialization done, swapping tmp file now... | 20260507 02:43:35.593 | INFO | RTMetrics done | 20260507 02:43:35.684 | INFO | Finished a round of Baz IO: Wrote 1.774 GiB in 3.216 seconds | 20260507 02:43:35.684 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.97% / 0.00% / 67.03% | 20260507 02:43:35.684 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.227559% / 0% / 5.96046e-05% | 20260507 02:43:35.684 | INFO | Baz IO overall: | 20260507 02:43:35.684 | INFO | BytesWritten : 3333007635610 | 20260507 02:43:35.684 | INFO | NumEvents : 1602531096813 | 20260507 02:43:37.512 | INFO | Analyzing chunk frames = [6467584,6468096) | 20260507 02:43:37.512 | INFO | MemInfo: { | 20260507 02:43:37.512 | INFO | "gpu" : " 17 GB", | 20260507 02:43:37.512 | INFO | "huge" : "", | 20260507 02:43:37.512 | INFO | "huge_shared" : " 27 GB", | 20260507 02:43:37.512 | INFO | "malloc" : " 14 GB", | 20260507 02:43:37.512 | INFO | "pinned" : " 29 GB" | 20260507 02:43:37.512 | INFO | } | 20260507 02:43:37.512 | INFO | Perf report for the last 5.14306 seconds: | 20260507 02:43:37.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:43:37.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:37.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:37.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:37.512 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260507 02:43:37.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:37.512 | INFO | RT_METRICS : 44.9%, 1.0, 23.6ms, 0 | 20260507 02:43:37.512 | INFO | PRE_HQ : 60.0%, 1.9, 60.6ms, 0 | 20260507 02:43:37.512 | INFO | BAZWRITER : 62.2%, 3.5, 111.1ms, 0 | 20260507 02:43:37.512 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 02:43:37.512 | INFO | Uploaded 36GiB to GPU in 2.0339s (17.7GiB/s) | 20260507 02:43:37.512 | INFO | Downloaded 38GiB to GPU in 2.1887s (17.3619GiB/s) | 20260507 02:43:42.647 | INFO | Analyzing chunk frames = [6468096,6468608) | 20260507 02:43:42.647 | INFO | MemInfo: { | 20260507 02:43:42.647 | INFO | "gpu" : " 17 GB", | 20260507 02:43:42.647 | INFO | "huge" : "", | 20260507 02:43:42.647 | INFO | "huge_shared" : " 27 GB", | 20260507 02:43:42.647 | INFO | "malloc" : " 14 GB", | 20260507 02:43:42.647 | INFO | "pinned" : " 29 GB" | 20260507 02:43:42.647 | INFO | } | 20260507 02:43:42.647 | INFO | Perf report for the last 5.13478 seconds: | 20260507 02:43:42.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:43:42.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:42.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:42.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:42.647 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:43:42.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:42.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:42.647 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:43:42.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:42.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:42.647 | INFO | Uploaded 36GiB to GPU in 2.04684s (17.5881GiB/s) | 20260507 02:43:42.647 | INFO | Downloaded 34GiB to GPU in 1.93684s (17.5544GiB/s) | 20260507 02:43:47.791 | INFO | Analyzing chunk frames = [6468608,6469120) | 20260507 02:43:47.791 | INFO | MemInfo: { | 20260507 02:43:47.791 | INFO | "gpu" : " 17 GB", | 20260507 02:43:47.791 | INFO | "huge" : "", | 20260507 02:43:47.791 | INFO | "huge_shared" : " 27 GB", | 20260507 02:43:47.792 | INFO | "malloc" : " 14 GB", | 20260507 02:43:47.792 | INFO | "pinned" : " 29 GB" | 20260507 02:43:47.792 | INFO | } | 20260507 02:43:47.792 | INFO | Perf report for the last 5.14482 seconds: | 20260507 02:43:47.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:43:47.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:47.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:47.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:47.792 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:43:47.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:47.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:47.792 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:43:47.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:47.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:47.792 | INFO | Uploaded 36GiB to GPU in 2.00469s (17.9579GiB/s) | 20260507 02:43:47.792 | INFO | Downloaded 34GiB to GPU in 1.93842s (17.54GiB/s) | 20260507 02:43:52.927 | INFO | Analyzing chunk frames = [6469120,6469632) | 20260507 02:43:52.927 | INFO | MemInfo: { | 20260507 02:43:52.927 | INFO | "gpu" : " 17 GB", | 20260507 02:43:52.927 | INFO | "huge" : "", | 20260507 02:43:52.927 | INFO | "huge_shared" : " 27 GB", | 20260507 02:43:52.927 | INFO | "malloc" : " 14 GB", | 20260507 02:43:52.927 | INFO | "pinned" : " 29 GB" | 20260507 02:43:52.927 | INFO | } | 20260507 02:43:52.927 | INFO | Perf report for the last 5.13549 seconds: | 20260507 02:43:52.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:43:52.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:52.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:52.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:52.927 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 02:43:52.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:52.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:52.927 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:43:52.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:52.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:52.927 | INFO | Uploaded 36GiB to GPU in 1.99645s (18.032GiB/s) | 20260507 02:43:52.927 | INFO | Downloaded 34GiB to GPU in 1.91844s (17.7227GiB/s) | 20260507 02:43:58.065 | INFO | Analyzing chunk frames = [6469632,6470144) | 20260507 02:43:58.065 | INFO | MemInfo: { | 20260507 02:43:58.065 | INFO | "gpu" : " 17 GB", | 20260507 02:43:58.065 | INFO | "huge" : "", | 20260507 02:43:58.066 | INFO | "huge_shared" : " 27 GB", | 20260507 02:43:58.066 | INFO | "malloc" : " 14 GB", | 20260507 02:43:58.066 | INFO | "pinned" : " 29 GB" | 20260507 02:43:58.066 | INFO | } | 20260507 02:43:58.066 | INFO | Perf report for the last 5.13852 seconds: | 20260507 02:43:58.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:43:58.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:58.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:58.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:58.066 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 02:43:58.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:58.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:58.066 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 02:43:58.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:58.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:43:58.066 | INFO | Uploaded 36GiB to GPU in 2.0137s (17.8776GiB/s) | 20260507 02:43:58.066 | INFO | Downloaded 34GiB to GPU in 1.94961s (17.4393GiB/s) | 20260507 02:44:03.205 | INFO | Analyzing chunk frames = [6470144,6470656) | 20260507 02:44:03.205 | INFO | MemInfo: { | 20260507 02:44:03.205 | INFO | "gpu" : " 17 GB", | 20260507 02:44:03.205 | INFO | "huge" : "", | 20260507 02:44:03.205 | INFO | "huge_shared" : " 27 GB", | 20260507 02:44:03.205 | INFO | "malloc" : " 14 GB", | 20260507 02:44:03.205 | INFO | "pinned" : " 29 GB" | 20260507 02:44:03.205 | INFO | } | 20260507 02:44:03.205 | INFO | Perf report for the last 5.13967 seconds: | 20260507 02:44:03.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:44:03.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:03.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:03.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:03.205 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:44:03.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:03.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:03.205 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:44:03.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:03.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:03.206 | INFO | Uploaded 36GiB to GPU in 2.05748s (17.4971GiB/s) | 20260507 02:44:03.206 | INFO | Downloaded 34GiB to GPU in 1.95877s (17.3578GiB/s) | 20260507 02:44:08.353 | INFO | Analyzing chunk frames = [6470656,6471168) | 20260507 02:44:08.353 | INFO | MemInfo: { | 20260507 02:44:08.353 | INFO | "gpu" : " 17 GB", | 20260507 02:44:08.353 | INFO | "huge" : "", | 20260507 02:44:08.353 | INFO | "huge_shared" : " 27 GB", | 20260507 02:44:08.353 | INFO | "malloc" : " 14 GB", | 20260507 02:44:08.353 | INFO | "pinned" : " 29 GB" | 20260507 02:44:08.353 | INFO | } | 20260507 02:44:08.353 | INFO | Perf report for the last 5.14796 seconds: | 20260507 02:44:08.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:44:08.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:08.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:08.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:08.353 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:44:08.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:08.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:08.353 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 02:44:08.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:08.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:08.354 | INFO | Uploaded 36GiB to GPU in 1.96426s (18.3275GiB/s) | 20260507 02:44:08.354 | INFO | Downloaded 34GiB to GPU in 1.99734s (17.0226GiB/s) | 20260507 02:44:13.490 | INFO | Analyzing chunk frames = [6471168,6471680) | 20260507 02:44:13.490 | INFO | MemInfo: { | 20260507 02:44:13.490 | INFO | "gpu" : " 17 GB", | 20260507 02:44:13.490 | INFO | "huge" : "", | 20260507 02:44:13.490 | INFO | "huge_shared" : " 27 GB", | 20260507 02:44:13.490 | INFO | "malloc" : " 14 GB", | 20260507 02:44:13.490 | INFO | "pinned" : " 29 GB" | 20260507 02:44:13.490 | INFO | } | 20260507 02:44:13.490 | INFO | Perf report for the last 5.13648 seconds: | 20260507 02:44:13.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:44:13.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:13.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:13.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:13.490 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 02:44:13.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:13.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:13.490 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 02:44:13.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:13.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:13.490 | INFO | Uploaded 36GiB to GPU in 1.99573s (18.0385GiB/s) | 20260507 02:44:13.490 | INFO | Downloaded 34GiB to GPU in 1.96763s (17.2797GiB/s) | 20260507 02:44:13.490 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:44:13.490 | INFO | Total mem usage: 18 GiB | 20260507 02:44:13.490 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:44:13.490 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:44:13.490 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:44:13.490 | INFO | High Water Breakdown: | 20260507 02:44:13.490 | INFO | 60.2% : 8.58 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:44:13.490 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:44:13.490 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:44:13.490 | INFO | ------------------------------------------ | 20260507 02:44:13.490 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:44:13.490 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:44:13.491 | INFO | Total mem usage: 26.5 GiB | 20260507 02:44:13.491 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:44:13.491 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 02:44:13.491 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 02:44:13.491 | INFO | High Water Breakdown: | 20260507 02:44:13.491 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:44:13.491 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:44:13.491 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:44:13.491 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:44:13.491 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:44:13.491 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:44:13.491 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:44:13.491 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:44:13.491 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:44:13.491 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:44:13.491 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:44:13.491 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:44:13.491 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:44:13.491 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:44:13.491 | INFO | ------------------------------------------ | 20260507 02:44:13.491 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:44:13.491 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:44:13.491 | INFO | Total mem usage: 25.6 GiB | 20260507 02:44:13.491 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:44:13.491 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:44:13.491 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:44:13.491 | INFO | High Water Breakdown: | 20260507 02:44:13.491 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:44:13.491 | INFO | ------------------------------------------ | 20260507 02:44:13.491 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:44:13.491 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:44:13.491 | INFO | Total mem usage: 15.7 GiB | 20260507 02:44:13.491 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:44:13.491 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:44:13.491 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 02:44:13.491 | INFO | High Water Breakdown: | 20260507 02:44:13.491 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:44:13.491 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:44:13.491 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:44:13.491 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:44:13.491 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:44:13.491 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:44:13.491 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:44:13.491 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:44:13.491 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:44:13.491 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:44:13.491 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:44:13.491 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:44:13.491 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:44:13.491 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:44:13.491 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:44:13.491 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:44:13.491 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:44:13.491 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:44:13.491 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:44:13.491 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:44:13.491 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:44:13.491 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:44:13.491 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:44:13.491 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:44:13.491 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:44:13.491 | INFO | ------------------------------------------ | 20260507 02:44:13.491 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:44:13.491 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:44:13.491 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:44:13.491 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:44:13.491 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.51e+01, 1.34e+01 / 2.06e+00 | 20260507 02:44:13.491 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.02e-01, 9.22e-03 / 4.09e-03 | 20260507 02:44:13.491 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.41e+00, 5.09e-01 / 2.73e-01 | 20260507 02:44:13.491 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.48e+00, 1.66e-01 / 1.03e-01 | 20260507 02:44:13.491 | INFO | PerfReport: 16.12%, DME -- 3.97e+01 / 4.42e+01, 3.93e+00 / 2.11e+00 | 20260507 02:44:13.491 | INFO | PerfReport: 57.55%, FrameLabeling -- 1.77e+01 / 2.05e+01, 1.21e+00 / 1.20e+00 | 20260507 02:44:13.491 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.16e+00 / 2.73e+00, 3.90e-01 / 1.92e-01 | 20260507 02:44:13.491 | INFO | PerfReport: 3.08%, Metrics -- 9.46e-01 / 1.27e+00, 2.50e-01 / 1.38e-01 | 20260507 02:44:13.491 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.20e-01 / 3.76e-01, 5.10e-02 / 2.62e-02 | 20260507 02:44:13.491 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:44:13.491 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:44:13.491 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:44:13.491 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 02:44:13.491 | INFO | PerfReport: 21.80%, Upload -- 2.10e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260507 02:44:13.491 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260507 02:44:13.491 | INFO | PerfReport: 31.89%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 02:44:13.491 | INFO | PerfReport: 1.53%, LockUpload -- 1.48e+00 / 0.00e+00, 5.01e+00 / 0.00e+00 | 20260507 02:44:13.491 | INFO | PerfReport: 1.25%, LockDownload -- 1.21e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260507 02:44:13.491 | INFO | PerfReport: 22.09%, LockCompute -- 2.13e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 02:44:13.491 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260507 02:44:13.587 | INFO | Starting a round of Baz IO | 20260507 02:44:16.694 | INFO | Populating RTMetrics report | 20260507 02:44:16.695 | INFO | RTMetrics report populated | 20260507 02:44:16.695 | INFO | Opening tmp RTMetrics json file | 20260507 02:44:16.696 | INFO | Serializing RTMetrics to json | 20260507 02:44:16.713 | INFO | Serialization done, swapping tmp file now... | 20260507 02:44:16.715 | INFO | RTMetrics done | 20260507 02:44:16.792 | INFO | Finished a round of Baz IO: Wrote 1.775 GiB in 3.205 seconds | 20260507 02:44:16.792 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.97% / 0.00% / 67.03% | 20260507 02:44:16.792 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.227388% / 0% / 9.13938e-05% | 20260507 02:44:16.792 | INFO | Baz IO overall: | 20260507 02:44:16.792 | INFO | BytesWritten : 3334914028608 | 20260507 02:44:16.792 | INFO | NumEvents : 1603432828760 | 20260507 02:44:18.628 | INFO | Analyzing chunk frames = [6471680,6472192) | 20260507 02:44:18.628 | INFO | MemInfo: { | 20260507 02:44:18.628 | INFO | "gpu" : " 17 GB", | 20260507 02:44:18.628 | INFO | "huge" : "", | 20260507 02:44:18.628 | INFO | "huge_shared" : " 27 GB", | 20260507 02:44:18.628 | INFO | "malloc" : " 14 GB", | 20260507 02:44:18.628 | INFO | "pinned" : " 29 GB" | 20260507 02:44:18.628 | INFO | } | 20260507 02:44:18.628 | INFO | Perf report for the last 5.13793 seconds: | 20260507 02:44:18.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:44:18.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:18.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:18.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:18.628 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 02:44:18.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:18.628 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 02:44:18.628 | INFO | PRE_HQ : 59.2%, 1.8, 55.8ms, 0 | 20260507 02:44:18.628 | INFO | BAZWRITER : 61.9%, 3.4, 106.6ms, 0 | 20260507 02:44:18.628 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:44:18.628 | INFO | Uploaded 36GiB to GPU in 2.02841s (17.7479GiB/s) | 20260507 02:44:18.628 | INFO | Downloaded 38GiB to GPU in 2.17687s (17.4562GiB/s) | 20260507 02:44:23.764 | INFO | Analyzing chunk frames = [6472192,6472704) | 20260507 02:44:23.764 | INFO | MemInfo: { | 20260507 02:44:23.764 | INFO | "gpu" : " 17 GB", | 20260507 02:44:23.764 | INFO | "huge" : "", | 20260507 02:44:23.764 | INFO | "huge_shared" : " 27 GB", | 20260507 02:44:23.764 | INFO | "malloc" : " 14 GB", | 20260507 02:44:23.764 | INFO | "pinned" : " 29 GB" | 20260507 02:44:23.764 | INFO | } | 20260507 02:44:23.764 | INFO | Perf report for the last 5.13609 seconds: | 20260507 02:44:23.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:44:23.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:23.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:23.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:23.764 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:44:23.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:23.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:23.764 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:44:23.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:23.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:23.764 | INFO | Uploaded 36GiB to GPU in 2.00391s (17.9649GiB/s) | 20260507 02:44:23.764 | INFO | Downloaded 34GiB to GPU in 1.93089s (17.6085GiB/s) | 20260507 02:44:28.902 | INFO | Analyzing chunk frames = [6472704,6473216) | 20260507 02:44:28.902 | INFO | MemInfo: { | 20260507 02:44:28.902 | INFO | "gpu" : " 17 GB", | 20260507 02:44:28.902 | INFO | "huge" : "", | 20260507 02:44:28.902 | INFO | "huge_shared" : " 27 GB", | 20260507 02:44:28.902 | INFO | "malloc" : " 14 GB", | 20260507 02:44:28.902 | INFO | "pinned" : " 29 GB" | 20260507 02:44:28.902 | INFO | } | 20260507 02:44:28.902 | INFO | Perf report for the last 5.13855 seconds: | 20260507 02:44:28.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:44:28.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:28.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:28.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:28.902 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:44:28.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:28.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:28.902 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:44:28.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:28.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:28.903 | INFO | Uploaded 36GiB to GPU in 2.00695s (17.9377GiB/s) | 20260507 02:44:28.903 | INFO | Downloaded 34GiB to GPU in 1.93632s (17.5591GiB/s) | 20260507 02:44:34.044 | INFO | Analyzing chunk frames = [6473216,6473728) | 20260507 02:44:34.044 | INFO | MemInfo: { | 20260507 02:44:34.044 | INFO | "gpu" : " 17 GB", | 20260507 02:44:34.044 | INFO | "huge" : "", | 20260507 02:44:34.044 | INFO | "huge_shared" : " 27 GB", | 20260507 02:44:34.045 | INFO | "malloc" : " 14 GB", | 20260507 02:44:34.045 | INFO | "pinned" : " 29 GB" | 20260507 02:44:34.045 | INFO | } | 20260507 02:44:34.045 | INFO | Perf report for the last 5.14215 seconds: | 20260507 02:44:34.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:44:34.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:34.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:34.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:34.045 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:44:34.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:34.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:34.045 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:44:34.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:34.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:34.045 | INFO | Uploaded 36GiB to GPU in 1.98281s (18.156GiB/s) | 20260507 02:44:34.045 | INFO | Downloaded 34GiB to GPU in 1.90872s (17.813GiB/s) | 20260507 02:44:39.183 | INFO | Analyzing chunk frames = [6473728,6474240) | 20260507 02:44:39.183 | INFO | MemInfo: { | 20260507 02:44:39.183 | INFO | "gpu" : " 17 GB", | 20260507 02:44:39.183 | INFO | "huge" : "", | 20260507 02:44:39.183 | INFO | "huge_shared" : " 27 GB", | 20260507 02:44:39.183 | INFO | "malloc" : " 14 GB", | 20260507 02:44:39.183 | INFO | "pinned" : " 29 GB" | 20260507 02:44:39.183 | INFO | } | 20260507 02:44:39.183 | INFO | Perf report for the last 5.13826 seconds: | 20260507 02:44:39.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:44:39.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:39.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:39.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:39.183 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 02:44:39.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:39.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:39.183 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:44:39.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:39.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:39.183 | INFO | Uploaded 36GiB to GPU in 2.02775s (17.7537GiB/s) | 20260507 02:44:39.183 | INFO | Downloaded 34GiB to GPU in 1.9447s (17.4834GiB/s) | 20260507 02:44:44.323 | INFO | Analyzing chunk frames = [6474240,6474752) | 20260507 02:44:44.323 | INFO | MemInfo: { | 20260507 02:44:44.323 | INFO | "gpu" : " 17 GB", | 20260507 02:44:44.323 | INFO | "huge" : "", | 20260507 02:44:44.323 | INFO | "huge_shared" : " 27 GB", | 20260507 02:44:44.323 | INFO | "malloc" : " 14 GB", | 20260507 02:44:44.323 | INFO | "pinned" : " 29 GB" | 20260507 02:44:44.323 | INFO | } | 20260507 02:44:44.323 | INFO | Perf report for the last 5.14054 seconds: | 20260507 02:44:44.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:44:44.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:44.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:44.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:44.323 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 02:44:44.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:44.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:44.324 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:44:44.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:44.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:44.324 | INFO | Uploaded 36GiB to GPU in 2.0565s (17.5054GiB/s) | 20260507 02:44:44.324 | INFO | Downloaded 34GiB to GPU in 1.96279s (17.3223GiB/s) | 20260507 02:44:49.463 | INFO | Analyzing chunk frames = [6474752,6475264) | 20260507 02:44:49.463 | INFO | MemInfo: { | 20260507 02:44:49.463 | INFO | "gpu" : " 17 GB", | 20260507 02:44:49.463 | INFO | "huge" : "", | 20260507 02:44:49.463 | INFO | "huge_shared" : " 27 GB", | 20260507 02:44:49.463 | INFO | "malloc" : " 14 GB", | 20260507 02:44:49.463 | INFO | "pinned" : " 29 GB" | 20260507 02:44:49.463 | INFO | } | 20260507 02:44:49.463 | INFO | Perf report for the last 5.13938 seconds: | 20260507 02:44:49.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:44:49.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:49.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:49.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:49.463 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 02:44:49.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:49.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:49.463 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 02:44:49.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:49.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:49.463 | INFO | Uploaded 36GiB to GPU in 1.97713s (18.2083GiB/s) | 20260507 02:44:49.463 | INFO | Downloaded 34GiB to GPU in 1.95517s (17.3898GiB/s) | 20260507 02:44:54.601 | INFO | Analyzing chunk frames = [6475264,6475776) | 20260507 02:44:54.601 | INFO | MemInfo: { | 20260507 02:44:54.601 | INFO | "gpu" : " 17 GB", | 20260507 02:44:54.601 | INFO | "huge" : "", | 20260507 02:44:54.602 | INFO | "huge_shared" : " 27 GB", | 20260507 02:44:54.602 | INFO | "malloc" : " 14 GB", | 20260507 02:44:54.602 | INFO | "pinned" : " 29 GB" | 20260507 02:44:54.602 | INFO | } | 20260507 02:44:54.602 | INFO | Perf report for the last 5.13891 seconds: | 20260507 02:44:54.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:44:54.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:54.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:54.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:54.602 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 02:44:54.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:54.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:54.602 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:44:54.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:54.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:54.602 | INFO | Uploaded 36GiB to GPU in 1.97388s (18.2382GiB/s) | 20260507 02:44:54.602 | INFO | Downloaded 34GiB to GPU in 1.95942s (17.352GiB/s) | 20260507 02:44:54.713 | INFO | Starting a round of Baz IO | 20260507 02:44:57.808 | INFO | Populating RTMetrics report | 20260507 02:44:57.809 | INFO | RTMetrics report populated | 20260507 02:44:57.809 | INFO | Opening tmp RTMetrics json file | 20260507 02:44:57.810 | INFO | Serializing RTMetrics to json | 20260507 02:44:57.828 | INFO | Serialization done, swapping tmp file now... | 20260507 02:44:57.828 | INFO | RTMetrics done | 20260507 02:44:57.902 | INFO | Finished a round of Baz IO: Wrote 1.774 GiB in 3.188 seconds | 20260507 02:44:57.902 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.97% / 0.00% / 67.03% | 20260507 02:44:57.902 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.226883% / 3.97364e-06% / 8.74201e-05% | 20260507 02:44:57.902 | INFO | Baz IO overall: | 20260507 02:44:57.902 | INFO | BytesWritten : 3336818906852 | 20260507 02:44:57.902 | INFO | NumEvents : 1604333361680 | 20260507 02:44:59.738 | INFO | Analyzing chunk frames = [6475776,6476288) | 20260507 02:44:59.738 | INFO | MemInfo: { | 20260507 02:44:59.738 | INFO | "gpu" : " 17 GB", | 20260507 02:44:59.739 | INFO | "huge" : "", | 20260507 02:44:59.739 | INFO | "huge_shared" : " 27 GB", | 20260507 02:44:59.739 | INFO | "malloc" : " 14 GB", | 20260507 02:44:59.739 | INFO | "pinned" : " 29 GB" | 20260507 02:44:59.739 | INFO | } | 20260507 02:44:59.739 | INFO | Perf report for the last 5.13692 seconds: | 20260507 02:44:59.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:44:59.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:59.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:59.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:59.739 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 02:44:59.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:44:59.739 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 02:44:59.739 | INFO | PRE_HQ : 59.3%, 1.8, 55.5ms, 0 | 20260507 02:44:59.739 | INFO | BAZWRITER : 62.1%, 3.4, 107.1ms, 0 | 20260507 02:44:59.739 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:44:59.739 | INFO | Uploaded 36GiB to GPU in 2.03558s (17.6853GiB/s) | 20260507 02:44:59.739 | INFO | Downloaded 38GiB to GPU in 2.18314s (17.4062GiB/s) | 20260507 02:45:04.878 | INFO | Analyzing chunk frames = [6476288,6476800) | 20260507 02:45:04.878 | INFO | MemInfo: { | 20260507 02:45:04.878 | INFO | "gpu" : " 17 GB", | 20260507 02:45:04.878 | INFO | "huge" : "", | 20260507 02:45:04.878 | INFO | "huge_shared" : " 27 GB", | 20260507 02:45:04.878 | INFO | "malloc" : " 14 GB", | 20260507 02:45:04.878 | INFO | "pinned" : " 29 GB" | 20260507 02:45:04.878 | INFO | } | 20260507 02:45:04.878 | INFO | Perf report for the last 5.13935 seconds: | 20260507 02:45:04.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:45:04.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:04.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:04.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:04.878 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:45:04.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:04.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:04.878 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:45:04.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:04.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:04.878 | INFO | Uploaded 36GiB to GPU in 2.02571s (17.7716GiB/s) | 20260507 02:45:04.879 | INFO | Downloaded 34GiB to GPU in 1.94221s (17.5058GiB/s) | 20260507 02:45:10.024 | INFO | Analyzing chunk frames = [6476800,6477312) | 20260507 02:45:10.025 | INFO | MemInfo: { | 20260507 02:45:10.025 | INFO | "gpu" : " 17 GB", | 20260507 02:45:10.025 | INFO | "huge" : "", | 20260507 02:45:10.025 | INFO | "huge_shared" : " 27 GB", | 20260507 02:45:10.025 | INFO | "malloc" : " 14 GB", | 20260507 02:45:10.025 | INFO | "pinned" : " 29 GB" | 20260507 02:45:10.025 | INFO | } | 20260507 02:45:10.025 | INFO | Perf report for the last 5.14677 seconds: | 20260507 02:45:10.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:45:10.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:10.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:10.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:10.025 | INFO | ANALYSIS : 60.0%, 3.0, 93.2ms, 0 | 20260507 02:45:10.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:10.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:10.025 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 02:45:10.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:10.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:10.025 | INFO | Uploaded 36GiB to GPU in 1.98854s (18.1037GiB/s) | 20260507 02:45:10.025 | INFO | Downloaded 34GiB to GPU in 1.93998s (17.5259GiB/s) | 20260507 02:45:15.159 | INFO | Analyzing chunk frames = [6477312,6477824) | 20260507 02:45:15.159 | INFO | MemInfo: { | 20260507 02:45:15.159 | INFO | "gpu" : " 17 GB", | 20260507 02:45:15.159 | INFO | "huge" : "", | 20260507 02:45:15.159 | INFO | "huge_shared" : " 27 GB", | 20260507 02:45:15.159 | INFO | "malloc" : " 14 GB", | 20260507 02:45:15.159 | INFO | "pinned" : " 29 GB" | 20260507 02:45:15.159 | INFO | } | 20260507 02:45:15.159 | INFO | Perf report for the last 5.13423 seconds: | 20260507 02:45:15.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:45:15.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:15.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:15.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:15.159 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:45:15.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:15.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:15.159 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:45:15.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:15.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:15.159 | INFO | Uploaded 36GiB to GPU in 1.96883s (18.285GiB/s) | 20260507 02:45:15.159 | INFO | Downloaded 34GiB to GPU in 1.92759s (17.6386GiB/s) | 20260507 02:45:20.303 | INFO | Analyzing chunk frames = [6477824,6478336) | 20260507 02:45:20.303 | INFO | MemInfo: { | 20260507 02:45:20.303 | INFO | "gpu" : " 17 GB", | 20260507 02:45:20.303 | INFO | "huge" : "", | 20260507 02:45:20.303 | INFO | "huge_shared" : " 27 GB", | 20260507 02:45:20.303 | INFO | "malloc" : " 14 GB", | 20260507 02:45:20.303 | INFO | "pinned" : " 29 GB" | 20260507 02:45:20.303 | INFO | } | 20260507 02:45:20.303 | INFO | Perf report for the last 5.14411 seconds: | 20260507 02:45:20.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:45:20.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:20.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:20.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:20.303 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 02:45:20.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:20.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:20.304 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:45:20.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:20.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:20.304 | INFO | Uploaded 36GiB to GPU in 1.99704s (18.0267GiB/s) | 20260507 02:45:20.304 | INFO | Downloaded 34GiB to GPU in 1.949s (17.4448GiB/s) | 20260507 02:45:25.441 | INFO | Analyzing chunk frames = [6478336,6478848) | 20260507 02:45:25.441 | INFO | MemInfo: { | 20260507 02:45:25.441 | INFO | "gpu" : " 17 GB", | 20260507 02:45:25.441 | INFO | "huge" : "", | 20260507 02:45:25.441 | INFO | "huge_shared" : " 27 GB", | 20260507 02:45:25.441 | INFO | "malloc" : " 14 GB", | 20260507 02:45:25.441 | INFO | "pinned" : " 29 GB" | 20260507 02:45:25.441 | INFO | } | 20260507 02:45:25.441 | INFO | Perf report for the last 5.13774 seconds: | 20260507 02:45:25.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:45:25.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:25.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:25.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:25.441 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 02:45:25.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:25.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:25.441 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 02:45:25.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:25.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:25.441 | INFO | Uploaded 36GiB to GPU in 2.02314s (17.7942GiB/s) | 20260507 02:45:25.442 | INFO | Downloaded 34GiB to GPU in 1.93993s (17.5264GiB/s) | 20260507 02:45:30.572 | INFO | Analyzing chunk frames = [6478848,6479360) | 20260507 02:45:30.572 | INFO | MemInfo: { | 20260507 02:45:30.572 | INFO | "gpu" : " 17 GB", | 20260507 02:45:30.573 | INFO | "huge" : "", | 20260507 02:45:30.573 | INFO | "huge_shared" : " 27 GB", | 20260507 02:45:30.573 | INFO | "malloc" : " 14 GB", | 20260507 02:45:30.573 | INFO | "pinned" : " 29 GB" | 20260507 02:45:30.573 | INFO | } | 20260507 02:45:30.573 | INFO | Perf report for the last 5.13171 seconds: | 20260507 02:45:30.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:45:30.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:30.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:30.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:30.573 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 02:45:30.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:30.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:30.573 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:45:30.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:30.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:30.573 | INFO | Uploaded 36GiB to GPU in 1.95938s (18.3732GiB/s) | 20260507 02:45:30.573 | INFO | Downloaded 34GiB to GPU in 1.95609s (17.3816GiB/s) | 20260507 02:45:35.709 | INFO | Analyzing chunk frames = [6479360,6479872) | 20260507 02:45:35.709 | INFO | MemInfo: { | 20260507 02:45:35.709 | INFO | "gpu" : " 17 GB", | 20260507 02:45:35.709 | INFO | "huge" : "", | 20260507 02:45:35.709 | INFO | "huge_shared" : " 27 GB", | 20260507 02:45:35.709 | INFO | "malloc" : " 14 GB", | 20260507 02:45:35.709 | INFO | "pinned" : " 29 GB" | 20260507 02:45:35.709 | INFO | } | 20260507 02:45:35.709 | INFO | Perf report for the last 5.13677 seconds: | 20260507 02:45:35.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:45:35.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:35.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:35.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:35.710 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:45:35.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:35.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:35.710 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 02:45:35.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:35.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:35.710 | INFO | Uploaded 36GiB to GPU in 1.961s (18.3579GiB/s) | 20260507 02:45:35.710 | INFO | Downloaded 34GiB to GPU in 1.96417s (17.3101GiB/s) | 20260507 02:45:35.710 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:45:35.710 | INFO | Total mem usage: 18 GiB | 20260507 02:45:35.710 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:45:35.710 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:45:35.710 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:45:35.710 | INFO | High Water Breakdown: | 20260507 02:45:35.710 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:45:35.710 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:45:35.710 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:45:35.710 | INFO | ------------------------------------------ | 20260507 02:45:35.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:45:35.710 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:45:35.710 | INFO | Total mem usage: 26.5 GiB | 20260507 02:45:35.710 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:45:35.710 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:45:35.710 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:45:35.710 | INFO | High Water Breakdown: | 20260507 02:45:35.710 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:45:35.710 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:45:35.710 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:45:35.710 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:45:35.710 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:45:35.710 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:45:35.710 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:45:35.710 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:45:35.710 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:45:35.710 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:45:35.710 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:45:35.710 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:45:35.710 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:45:35.710 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:45:35.710 | INFO | ------------------------------------------ | 20260507 02:45:35.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:45:35.710 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:45:35.710 | INFO | Total mem usage: 25.6 GiB | 20260507 02:45:35.710 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:45:35.710 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 02:45:35.710 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 02:45:35.710 | INFO | High Water Breakdown: | 20260507 02:45:35.710 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 02:45:35.710 | INFO | ------------------------------------------ | 20260507 02:45:35.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:45:35.710 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:45:35.710 | INFO | Total mem usage: 15.7 GiB | 20260507 02:45:35.711 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:45:35.711 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:45:35.711 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:45:35.711 | INFO | High Water Breakdown: | 20260507 02:45:35.711 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:45:35.711 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:45:35.711 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:45:35.711 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:45:35.711 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:45:35.711 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:45:35.711 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:45:35.711 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:45:35.711 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:45:35.711 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:45:35.711 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:45:35.711 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:45:35.711 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:45:35.711 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:45:35.711 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:45:35.711 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:45:35.711 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:45:35.711 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:45:35.711 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:45:35.711 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:45:35.711 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:45:35.711 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:45:35.711 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:45:35.711 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:45:35.711 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:45:35.711 | INFO | ------------------------------------------ | 20260507 02:45:35.711 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:45:35.711 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:45:35.711 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:45:35.711 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:45:35.711 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.49e+01, 1.33e+01 / 1.72e+00 | 20260507 02:45:35.711 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.01e-01, 1.00e-02 / 7.00e-03 | 20260507 02:45:35.711 | INFO | PerfReport: 11.68%, Baseline -- 3.60e+00 / 4.40e+00, 5.21e-01 / 3.21e-01 | 20260507 02:45:35.711 | INFO | PerfReport: 3.24%, Histogram -- 9.98e-01 / 1.43e+00, 1.71e-01 / 1.28e-01 | 20260507 02:45:35.711 | INFO | PerfReport: 16.06%, DME -- 3.96e+01 / 4.41e+01, 3.93e+00 / 1.93e+00 | 20260507 02:45:35.711 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.35e+00 / 1.54e+00 | 20260507 02:45:35.711 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.14e+00 / 2.61e+00, 3.74e-01 / 2.67e-01 | 20260507 02:45:35.711 | INFO | PerfReport: 3.06%, Metrics -- 9.44e-01 / 1.27e+00, 2.50e-01 / 1.27e-01 | 20260507 02:45:35.711 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.20e-01 / 3.69e-01, 5.09e-02 / 2.06e-02 | 20260507 02:45:35.711 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:45:35.711 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:45:35.711 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:45:35.711 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 02:45:35.711 | INFO | PerfReport: 21.64%, Upload -- 2.08e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260507 02:45:35.711 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 02:45:35.711 | INFO | PerfReport: 32.00%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 02:45:35.711 | INFO | PerfReport: 1.42%, LockUpload -- 1.37e+00 / 0.00e+00, 4.75e+00 / 0.00e+00 | 20260507 02:45:35.711 | INFO | PerfReport: 1.33%, LockDownload -- 1.28e+00 / 0.00e+00, 3.18e+00 / 0.00e+00 | 20260507 02:45:35.711 | INFO | PerfReport: 22.23%, LockCompute -- 2.14e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 02:45:35.711 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.63e-02 / 0.00e+00 | 20260507 02:45:35.806 | INFO | Starting a round of Baz IO | 20260507 02:45:38.924 | INFO | Populating RTMetrics report | 20260507 02:45:38.924 | INFO | RTMetrics report populated | 20260507 02:45:38.924 | INFO | Opening tmp RTMetrics json file | 20260507 02:45:38.942 | INFO | Serializing RTMetrics to json | 20260507 02:45:38.943 | INFO | Serialization done, swapping tmp file now... | 20260507 02:45:38.944 | INFO | RTMetrics done | 20260507 02:45:39.016 | INFO | Finished a round of Baz IO: Wrote 1.774 GiB in 3.209 seconds | 20260507 02:45:39.016 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.97% / 0.00% / 67.03% | 20260507 02:45:39.016 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.226219% / 3.97364e-06% / 6.75519e-05% | 20260507 02:45:39.016 | INFO | Baz IO overall: | 20260507 02:45:39.016 | INFO | BytesWritten : 3338723823161 | 20260507 02:45:39.016 | INFO | NumEvents : 1605233904275 | 20260507 02:45:40.855 | INFO | Analyzing chunk frames = [6479872,6480384) | 20260507 02:45:40.855 | INFO | MemInfo: { | 20260507 02:45:40.855 | INFO | "gpu" : " 17 GB", | 20260507 02:45:40.855 | INFO | "huge" : "", | 20260507 02:45:40.855 | INFO | "huge_shared" : " 27 GB", | 20260507 02:45:40.855 | INFO | "malloc" : " 14 GB", | 20260507 02:45:40.855 | INFO | "pinned" : " 29 GB" | 20260507 02:45:40.855 | INFO | } | 20260507 02:45:40.855 | INFO | Perf report for the last 5.14532 seconds: | 20260507 02:45:40.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:45:40.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:40.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:40.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:40.855 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260507 02:45:40.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:40.855 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 02:45:40.855 | INFO | PRE_HQ : 59.4%, 1.8, 55.7ms, 0 | 20260507 02:45:40.855 | INFO | BAZWRITER : 62.0%, 3.3, 105.9ms, 0 | 20260507 02:45:40.855 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:45:40.855 | INFO | Uploaded 36GiB to GPU in 2.03352s (17.7033GiB/s) | 20260507 02:45:40.855 | INFO | Downloaded 38GiB to GPU in 2.17949s (17.4353GiB/s) | 20260507 02:45:46.000 | INFO | Analyzing chunk frames = [6480384,6480896) | 20260507 02:45:46.000 | INFO | MemInfo: { | 20260507 02:45:46.000 | INFO | "gpu" : " 17 GB", | 20260507 02:45:46.000 | INFO | "huge" : "", | 20260507 02:45:46.000 | INFO | "huge_shared" : " 27 GB", | 20260507 02:45:46.000 | INFO | "malloc" : " 14 GB", | 20260507 02:45:46.000 | INFO | "pinned" : " 29 GB" | 20260507 02:45:46.000 | INFO | } | 20260507 02:45:46.000 | INFO | Perf report for the last 5.14529 seconds: | 20260507 02:45:46.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:45:46.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:46.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:46.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:46.000 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 02:45:46.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:46.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:46.000 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:45:46.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:46.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:46.000 | INFO | Uploaded 36GiB to GPU in 1.99128s (18.0788GiB/s) | 20260507 02:45:46.001 | INFO | Downloaded 34GiB to GPU in 1.9313s (17.6048GiB/s) | 20260507 02:45:51.142 | INFO | Analyzing chunk frames = [6480896,6481408) | 20260507 02:45:51.142 | INFO | MemInfo: { | 20260507 02:45:51.142 | INFO | "gpu" : " 17 GB", | 20260507 02:45:51.142 | INFO | "huge" : "", | 20260507 02:45:51.142 | INFO | "huge_shared" : " 27 GB", | 20260507 02:45:51.142 | INFO | "malloc" : " 14 GB", | 20260507 02:45:51.142 | INFO | "pinned" : " 29 GB" | 20260507 02:45:51.142 | INFO | } | 20260507 02:45:51.142 | INFO | Perf report for the last 5.14241 seconds: | 20260507 02:45:51.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:45:51.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:51.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:51.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:51.142 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 02:45:51.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:51.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:51.143 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 02:45:51.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:51.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:51.143 | INFO | Uploaded 36GiB to GPU in 2.02848s (17.7473GiB/s) | 20260507 02:45:51.143 | INFO | Downloaded 34GiB to GPU in 1.9562s (17.3806GiB/s) | 20260507 02:45:56.277 | INFO | Analyzing chunk frames = [6481408,6481920) | 20260507 02:45:56.277 | INFO | MemInfo: { | 20260507 02:45:56.277 | INFO | "gpu" : " 17 GB", | 20260507 02:45:56.277 | INFO | "huge" : "", | 20260507 02:45:56.277 | INFO | "huge_shared" : " 27 GB", | 20260507 02:45:56.277 | INFO | "malloc" : " 14 GB", | 20260507 02:45:56.277 | INFO | "pinned" : " 29 GB" | 20260507 02:45:56.277 | INFO | } | 20260507 02:45:56.277 | INFO | Perf report for the last 5.1345 seconds: | 20260507 02:45:56.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:45:56.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:56.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:56.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:56.277 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:45:56.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:56.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:56.277 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 02:45:56.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:56.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:45:56.277 | INFO | Uploaded 36GiB to GPU in 2.00358s (17.9679GiB/s) | 20260507 02:45:56.277 | INFO | Downloaded 34GiB to GPU in 1.94284s (17.5001GiB/s) | 20260507 02:46:01.409 | INFO | Analyzing chunk frames = [6481920,6482432) | 20260507 02:46:01.409 | INFO | MemInfo: { | 20260507 02:46:01.409 | INFO | "gpu" : " 17 GB", | 20260507 02:46:01.409 | INFO | "huge" : "", | 20260507 02:46:01.409 | INFO | "huge_shared" : " 27 GB", | 20260507 02:46:01.409 | INFO | "malloc" : " 14 GB", | 20260507 02:46:01.409 | INFO | "pinned" : " 29 GB" | 20260507 02:46:01.409 | INFO | } | 20260507 02:46:01.409 | INFO | Perf report for the last 5.13211 seconds: | 20260507 02:46:01.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:46:01.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:01.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:01.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:01.409 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 02:46:01.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:01.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:01.409 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 02:46:01.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:01.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:01.409 | INFO | Uploaded 36GiB to GPU in 2.01978s (17.8237GiB/s) | 20260507 02:46:01.409 | INFO | Downloaded 34GiB to GPU in 1.95061s (17.4305GiB/s) | 20260507 02:46:06.549 | INFO | Analyzing chunk frames = [6482432,6482944) | 20260507 02:46:06.549 | INFO | MemInfo: { | 20260507 02:46:06.549 | INFO | "gpu" : " 17 GB", | 20260507 02:46:06.549 | INFO | "huge" : "", | 20260507 02:46:06.549 | INFO | "huge_shared" : " 27 GB", | 20260507 02:46:06.549 | INFO | "malloc" : " 14 GB", | 20260507 02:46:06.549 | INFO | "pinned" : " 29 GB" | 20260507 02:46:06.549 | INFO | } | 20260507 02:46:06.549 | INFO | Perf report for the last 5.13971 seconds: | 20260507 02:46:06.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:46:06.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:06.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:06.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:06.549 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 02:46:06.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:06.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:06.549 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:46:06.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:06.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:06.549 | INFO | Uploaded 36GiB to GPU in 2.01223s (17.8906GiB/s) | 20260507 02:46:06.549 | INFO | Downloaded 34GiB to GPU in 1.94592s (17.4724GiB/s) | 20260507 02:46:11.689 | INFO | Analyzing chunk frames = [6482944,6483456) | 20260507 02:46:11.689 | INFO | MemInfo: { | 20260507 02:46:11.689 | INFO | "gpu" : " 17 GB", | 20260507 02:46:11.689 | INFO | "huge" : "", | 20260507 02:46:11.689 | INFO | "huge_shared" : " 27 GB", | 20260507 02:46:11.689 | INFO | "malloc" : " 14 GB", | 20260507 02:46:11.689 | INFO | "pinned" : " 29 GB" | 20260507 02:46:11.689 | INFO | } | 20260507 02:46:11.689 | INFO | Perf report for the last 5.13993 seconds: | 20260507 02:46:11.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:46:11.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:11.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:11.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:11.689 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 02:46:11.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:11.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:11.689 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 02:46:11.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:11.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:11.689 | INFO | Uploaded 36GiB to GPU in 1.94748s (18.4854GiB/s) | 20260507 02:46:11.689 | INFO | Downloaded 34GiB to GPU in 1.99349s (17.0555GiB/s) | 20260507 02:46:16.825 | INFO | Analyzing chunk frames = [6483456,6483968) | 20260507 02:46:16.825 | INFO | MemInfo: { | 20260507 02:46:16.825 | INFO | "gpu" : " 17 GB", | 20260507 02:46:16.825 | INFO | "huge" : "", | 20260507 02:46:16.825 | INFO | "huge_shared" : " 27 GB", | 20260507 02:46:16.825 | INFO | "malloc" : " 14 GB", | 20260507 02:46:16.825 | INFO | "pinned" : " 29 GB" | 20260507 02:46:16.825 | INFO | } | 20260507 02:46:16.825 | INFO | Perf report for the last 5.13648 seconds: | 20260507 02:46:16.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:46:16.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:16.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:16.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:16.825 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:46:16.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:16.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:16.825 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:46:16.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:16.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:16.825 | INFO | Uploaded 36GiB to GPU in 1.99216s (18.0709GiB/s) | 20260507 02:46:16.826 | INFO | Downloaded 34GiB to GPU in 1.97238s (17.2381GiB/s) | 20260507 02:46:16.922 | INFO | Starting a round of Baz IO | 20260507 02:46:20.031 | INFO | Populating RTMetrics report | 20260507 02:46:20.031 | INFO | RTMetrics report populated | 20260507 02:46:20.031 | INFO | Opening tmp RTMetrics json file | 20260507 02:46:20.032 | INFO | Serializing RTMetrics to json | 20260507 02:46:20.033 | INFO | Serialization done, swapping tmp file now... | 20260507 02:46:20.033 | INFO | RTMetrics done | 20260507 02:46:20.124 | INFO | Finished a round of Baz IO: Wrote 1.773 GiB in 3.202 seconds | 20260507 02:46:20.124 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.97% / 0.00% / 67.03% | 20260507 02:46:20.124 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.22529% / 0% / 5.5631e-05% | 20260507 02:46:20.124 | INFO | Baz IO overall: | 20260507 02:46:20.125 | INFO | BytesWritten : 3340627719610 | 20260507 02:46:20.125 | INFO | NumEvents : 1606133548376 | 20260507 02:46:21.968 | INFO | Analyzing chunk frames = [6483968,6484480) | 20260507 02:46:21.968 | INFO | MemInfo: { | 20260507 02:46:21.968 | INFO | "gpu" : " 17 GB", | 20260507 02:46:21.968 | INFO | "huge" : "", | 20260507 02:46:21.968 | INFO | "huge_shared" : " 27 GB", | 20260507 02:46:21.968 | INFO | "malloc" : " 14 GB", | 20260507 02:46:21.968 | INFO | "pinned" : " 29 GB" | 20260507 02:46:21.968 | INFO | } | 20260507 02:46:21.968 | INFO | Perf report for the last 5.14255 seconds: | 20260507 02:46:21.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:46:21.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:21.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:21.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:21.968 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 02:46:21.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:21.968 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 02:46:21.968 | INFO | PRE_HQ : 59.3%, 1.8, 55.7ms, 0 | 20260507 02:46:21.968 | INFO | BAZWRITER : 61.9%, 3.3, 105.6ms, 0 | 20260507 02:46:21.968 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:46:21.968 | INFO | Uploaded 36GiB to GPU in 2.02854s (17.7467GiB/s) | 20260507 02:46:21.968 | INFO | Downloaded 38GiB to GPU in 2.18515s (17.3901GiB/s) | 20260507 02:46:27.109 | INFO | Analyzing chunk frames = [6484480,6484992) | 20260507 02:46:27.109 | INFO | MemInfo: { | 20260507 02:46:27.109 | INFO | "gpu" : " 17 GB", | 20260507 02:46:27.109 | INFO | "huge" : "", | 20260507 02:46:27.109 | INFO | "huge_shared" : " 27 GB", | 20260507 02:46:27.109 | INFO | "malloc" : " 14 GB", | 20260507 02:46:27.109 | INFO | "pinned" : " 29 GB" | 20260507 02:46:27.109 | INFO | } | 20260507 02:46:27.109 | INFO | Perf report for the last 5.14116 seconds: | 20260507 02:46:27.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:46:27.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:27.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:27.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:27.109 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:46:27.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:27.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:27.109 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:46:27.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:27.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:27.109 | INFO | Uploaded 36GiB to GPU in 2.0091s (17.9184GiB/s) | 20260507 02:46:27.109 | INFO | Downloaded 34GiB to GPU in 1.93294s (17.5898GiB/s) | 20260507 02:46:32.252 | INFO | Analyzing chunk frames = [6484992,6485504) | 20260507 02:46:32.252 | INFO | MemInfo: { | 20260507 02:46:32.252 | INFO | "gpu" : " 17 GB", | 20260507 02:46:32.252 | INFO | "huge" : "", | 20260507 02:46:32.252 | INFO | "huge_shared" : " 27 GB", | 20260507 02:46:32.252 | INFO | "malloc" : " 14 GB", | 20260507 02:46:32.252 | INFO | "pinned" : " 29 GB" | 20260507 02:46:32.252 | INFO | } | 20260507 02:46:32.252 | INFO | Perf report for the last 5.1428 seconds: | 20260507 02:46:32.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:46:32.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:32.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:32.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:32.252 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:46:32.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:32.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:32.252 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:46:32.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:32.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:32.252 | INFO | Uploaded 36GiB to GPU in 2.00296s (17.9734GiB/s) | 20260507 02:46:32.252 | INFO | Downloaded 34GiB to GPU in 1.94165s (17.5109GiB/s) | 20260507 02:46:37.393 | INFO | Analyzing chunk frames = [6485504,6486016) | 20260507 02:46:37.393 | INFO | MemInfo: { | 20260507 02:46:37.393 | INFO | "gpu" : " 17 GB", | 20260507 02:46:37.393 | INFO | "huge" : "", | 20260507 02:46:37.393 | INFO | "huge_shared" : " 27 GB", | 20260507 02:46:37.393 | INFO | "malloc" : " 14 GB", | 20260507 02:46:37.393 | INFO | "pinned" : " 29 GB" | 20260507 02:46:37.393 | INFO | } | 20260507 02:46:37.393 | INFO | Perf report for the last 5.14119 seconds: | 20260507 02:46:37.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:46:37.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:37.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:37.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:37.393 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 02:46:37.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:37.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:37.393 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:46:37.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:37.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:37.393 | INFO | Uploaded 36GiB to GPU in 1.99684s (18.0285GiB/s) | 20260507 02:46:37.393 | INFO | Downloaded 34GiB to GPU in 1.93217s (17.5968GiB/s) | 20260507 02:46:42.521 | INFO | Analyzing chunk frames = [6486016,6486528) | 20260507 02:46:42.521 | INFO | MemInfo: { | 20260507 02:46:42.521 | INFO | "gpu" : " 17 GB", | 20260507 02:46:42.521 | INFO | "huge" : "", | 20260507 02:46:42.521 | INFO | "huge_shared" : " 27 GB", | 20260507 02:46:42.521 | INFO | "malloc" : " 14 GB", | 20260507 02:46:42.521 | INFO | "pinned" : " 29 GB" | 20260507 02:46:42.522 | INFO | } | 20260507 02:46:42.522 | INFO | Perf report for the last 5.12858 seconds: | 20260507 02:46:42.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:46:42.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:42.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:42.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:42.522 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 02:46:42.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:42.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:42.522 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:46:42.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:42.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:42.522 | INFO | Uploaded 36GiB to GPU in 2.02255s (17.7993GiB/s) | 20260507 02:46:42.522 | INFO | Downloaded 34GiB to GPU in 1.94268s (17.5016GiB/s) | 20260507 02:46:47.660 | INFO | Analyzing chunk frames = [6486528,6487040) | 20260507 02:46:47.660 | INFO | MemInfo: { | 20260507 02:46:47.660 | INFO | "gpu" : " 17 GB", | 20260507 02:46:47.660 | INFO | "huge" : "", | 20260507 02:46:47.660 | INFO | "huge_shared" : " 27 GB", | 20260507 02:46:47.660 | INFO | "malloc" : " 14 GB", | 20260507 02:46:47.660 | INFO | "pinned" : " 29 GB" | 20260507 02:46:47.660 | INFO | } | 20260507 02:46:47.660 | INFO | Perf report for the last 5.13839 seconds: | 20260507 02:46:47.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:46:47.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:47.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:47.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:47.660 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:46:47.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:47.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:47.660 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 02:46:47.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:47.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:47.660 | INFO | Uploaded 36GiB to GPU in 2.05056s (17.5562GiB/s) | 20260507 02:46:47.660 | INFO | Downloaded 34GiB to GPU in 1.96181s (17.331GiB/s) | 20260507 02:46:52.812 | INFO | Analyzing chunk frames = [6487040,6487552) | 20260507 02:46:52.812 | INFO | MemInfo: { | 20260507 02:46:52.812 | INFO | "gpu" : " 17 GB", | 20260507 02:46:52.812 | INFO | "huge" : "", | 20260507 02:46:52.812 | INFO | "huge_shared" : " 27 GB", | 20260507 02:46:52.812 | INFO | "malloc" : " 14 GB", | 20260507 02:46:52.812 | INFO | "pinned" : " 29 GB" | 20260507 02:46:52.812 | INFO | } | 20260507 02:46:52.812 | INFO | Perf report for the last 5.15224 seconds: | 20260507 02:46:52.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:46:52.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:52.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:52.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:52.812 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 02:46:52.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:52.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:52.812 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 02:46:52.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:52.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:52.812 | INFO | Uploaded 36GiB to GPU in 1.95449s (18.4191GiB/s) | 20260507 02:46:52.813 | INFO | Downloaded 34GiB to GPU in 1.98193s (17.155GiB/s) | 20260507 02:46:57.948 | INFO | Analyzing chunk frames = [6487552,6488064) | 20260507 02:46:57.948 | INFO | MemInfo: { | 20260507 02:46:57.948 | INFO | "gpu" : " 17 GB", | 20260507 02:46:57.948 | INFO | "huge" : "", | 20260507 02:46:57.948 | INFO | "huge_shared" : " 27 GB", | 20260507 02:46:57.948 | INFO | "malloc" : " 14 GB", | 20260507 02:46:57.948 | INFO | "pinned" : " 29 GB" | 20260507 02:46:57.948 | INFO | } | 20260507 02:46:57.948 | INFO | Perf report for the last 5.13592 seconds: | 20260507 02:46:57.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:46:57.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:57.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:57.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:57.948 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 02:46:57.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:57.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:57.948 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 02:46:57.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:57.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:46:57.949 | INFO | Uploaded 36GiB to GPU in 2.00431s (17.9613GiB/s) | 20260507 02:46:57.949 | INFO | Downloaded 34GiB to GPU in 1.9687s (17.2703GiB/s) | 20260507 02:46:57.949 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:46:57.949 | INFO | Total mem usage: 18 GiB | 20260507 02:46:57.949 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:46:57.949 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 02:46:57.949 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:46:57.949 | INFO | High Water Breakdown: | 20260507 02:46:57.949 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:46:57.949 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:46:57.949 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:46:57.949 | INFO | ------------------------------------------ | 20260507 02:46:57.949 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:46:57.949 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:46:57.949 | INFO | Total mem usage: 26.5 GiB | 20260507 02:46:57.949 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:46:57.949 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:46:57.949 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:46:57.949 | INFO | High Water Breakdown: | 20260507 02:46:57.949 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:46:57.949 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:46:57.949 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:46:57.949 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:46:57.949 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:46:57.949 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:46:57.949 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:46:57.949 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:46:57.949 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:46:57.949 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:46:57.949 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:46:57.949 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:46:57.949 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:46:57.949 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:46:57.949 | INFO | ------------------------------------------ | 20260507 02:46:57.949 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:46:57.949 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:46:57.949 | INFO | Total mem usage: 25.6 GiB | 20260507 02:46:57.949 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:46:57.949 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 02:46:57.949 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 02:46:57.949 | INFO | High Water Breakdown: | 20260507 02:46:57.949 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 02:46:57.950 | INFO | ------------------------------------------ | 20260507 02:46:57.950 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:46:57.950 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:46:57.950 | INFO | Total mem usage: 15.7 GiB | 20260507 02:46:57.950 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:46:57.950 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:46:57.950 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 02:46:57.950 | INFO | High Water Breakdown: | 20260507 02:46:57.950 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:46:57.950 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:46:57.950 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:46:57.950 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:46:57.950 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:46:57.950 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:46:57.950 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:46:57.950 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:46:57.950 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:46:57.950 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:46:57.950 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:46:57.950 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:46:57.950 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:46:57.950 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:46:57.950 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:46:57.950 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:46:57.950 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:46:57.950 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:46:57.950 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:46:57.950 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:46:57.950 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:46:57.950 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:46:57.950 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:46:57.950 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:46:57.950 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:46:57.950 | INFO | ------------------------------------------ | 20260507 02:46:57.950 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:46:57.950 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:46:57.950 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:46:57.950 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:46:57.950 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.53e+01, 1.33e+01 / 2.18e+00 | 20260507 02:46:57.950 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 9.99e-02, 1.03e-02 / 5.82e-03 | 20260507 02:46:57.950 | INFO | PerfReport: 11.71%, Baseline -- 3.60e+00 / 4.44e+00, 5.31e-01 / 2.59e-01 | 20260507 02:46:57.950 | INFO | PerfReport: 3.23%, Histogram -- 9.94e-01 / 1.46e+00, 1.67e-01 / 1.18e-01 | 20260507 02:46:57.950 | INFO | PerfReport: 16.18%, DME -- 3.98e+01 / 4.50e+01, 4.22e+00 / 2.63e+00 | 20260507 02:46:57.950 | INFO | PerfReport: 57.44%, FrameLabeling -- 1.77e+01 / 1.98e+01, 1.15e+00 / 1.59e+00 | 20260507 02:46:57.950 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.16e+00 / 2.77e+00, 3.90e-01 / 1.93e-01 | 20260507 02:46:57.950 | INFO | PerfReport: 3.08%, Metrics -- 9.46e-01 / 1.29e+00, 2.50e-01 / 1.13e-01 | 20260507 02:46:57.950 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.20e-01 / 3.78e-01, 5.02e-02 / 2.52e-02 | 20260507 02:46:57.950 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:46:57.950 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:46:57.950 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:46:57.950 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 02:46:57.950 | INFO | PerfReport: 21.69%, Upload -- 2.09e+01 / 0.00e+00, 7.00e+00 / 0.00e+00 | 20260507 02:46:57.950 | INFO | PerfReport: 21.44%, Download -- 2.07e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 02:46:57.950 | INFO | PerfReport: 31.94%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 02:46:57.950 | INFO | PerfReport: 1.37%, LockUpload -- 1.32e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 02:46:57.950 | INFO | PerfReport: 1.43%, LockDownload -- 1.38e+00 / 0.00e+00, 3.30e+00 / 0.00e+00 | 20260507 02:46:57.950 | INFO | PerfReport: 22.12%, LockCompute -- 2.13e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 02:46:57.950 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.22e-02 / 0.00e+00 | 20260507 02:46:58.045 | INFO | Starting a round of Baz IO | 20260507 02:47:01.152 | INFO | Populating RTMetrics report | 20260507 02:47:01.152 | INFO | RTMetrics report populated | 20260507 02:47:01.152 | INFO | Opening tmp RTMetrics json file | 20260507 02:47:01.153 | INFO | Serializing RTMetrics to json | 20260507 02:47:01.162 | INFO | Serialization done, swapping tmp file now... | 20260507 02:47:01.173 | INFO | RTMetrics done | 20260507 02:47:01.248 | INFO | Finished a round of Baz IO: Wrote 1.774 GiB in 3.202 seconds | 20260507 02:47:01.248 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.96% / 0.00% / 67.03% | 20260507 02:47:01.248 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.225115% / 0% / 5.96046e-05% | 20260507 02:47:01.248 | INFO | Baz IO overall: | 20260507 02:47:01.248 | INFO | BytesWritten : 3342532041912 | 20260507 02:47:01.248 | INFO | NumEvents : 1607033774013 | 20260507 02:47:03.082 | INFO | Analyzing chunk frames = [6488064,6488576) | 20260507 02:47:03.082 | INFO | MemInfo: { | 20260507 02:47:03.082 | INFO | "gpu" : " 17 GB", | 20260507 02:47:03.082 | INFO | "huge" : "", | 20260507 02:47:03.082 | INFO | "huge_shared" : " 27 GB", | 20260507 02:47:03.082 | INFO | "malloc" : " 14 GB", | 20260507 02:47:03.082 | INFO | "pinned" : " 29 GB" | 20260507 02:47:03.082 | INFO | } | 20260507 02:47:03.082 | INFO | Perf report for the last 5.13408 seconds: | 20260507 02:47:03.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:47:03.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:03.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:03.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:03.082 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 02:47:03.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:03.082 | INFO | RT_METRICS : 42.1%, 1.0, 22.0ms, 0 | 20260507 02:47:03.083 | INFO | PRE_HQ : 59.3%, 1.8, 55.7ms, 0 | 20260507 02:47:03.083 | INFO | BAZWRITER : 61.9%, 3.3, 105.6ms, 0 | 20260507 02:47:03.083 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:47:03.083 | INFO | Uploaded 36GiB to GPU in 2.05166s (17.5467GiB/s) | 20260507 02:47:03.083 | INFO | Downloaded 38GiB to GPU in 2.18283s (17.4086GiB/s) | 20260507 02:47:08.227 | INFO | Analyzing chunk frames = [6488576,6489088) | 20260507 02:47:08.227 | INFO | MemInfo: { | 20260507 02:47:08.227 | INFO | "gpu" : " 17 GB", | 20260507 02:47:08.227 | INFO | "huge" : "", | 20260507 02:47:08.227 | INFO | "huge_shared" : " 27 GB", | 20260507 02:47:08.227 | INFO | "malloc" : " 14 GB", | 20260507 02:47:08.227 | INFO | "pinned" : " 29 GB" | 20260507 02:47:08.227 | INFO | } | 20260507 02:47:08.227 | INFO | Perf report for the last 5.14434 seconds: | 20260507 02:47:08.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:47:08.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:08.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:08.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:08.227 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:47:08.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:08.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:08.227 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 02:47:08.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:08.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:08.227 | INFO | Uploaded 36GiB to GPU in 2.04589s (17.5963GiB/s) | 20260507 02:47:08.227 | INFO | Downloaded 34GiB to GPU in 1.93407s (17.5795GiB/s) | 20260507 02:47:13.365 | INFO | Analyzing chunk frames = [6489088,6489600) | 20260507 02:47:13.365 | INFO | MemInfo: { | 20260507 02:47:13.365 | INFO | "gpu" : " 17 GB", | 20260507 02:47:13.365 | INFO | "huge" : "", | 20260507 02:47:13.365 | INFO | "huge_shared" : " 27 GB", | 20260507 02:47:13.365 | INFO | "malloc" : " 14 GB", | 20260507 02:47:13.365 | INFO | "pinned" : " 29 GB" | 20260507 02:47:13.365 | INFO | } | 20260507 02:47:13.366 | INFO | Perf report for the last 5.13895 seconds: | 20260507 02:47:13.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:47:13.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:13.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:13.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:13.366 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:47:13.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:13.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:13.366 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:47:13.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:13.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:13.366 | INFO | Uploaded 36GiB to GPU in 2.01018s (17.9088GiB/s) | 20260507 02:47:13.366 | INFO | Downloaded 34GiB to GPU in 1.93912s (17.5338GiB/s) | 20260507 02:47:18.501 | INFO | Analyzing chunk frames = [6489600,6490112) | 20260507 02:47:18.501 | INFO | MemInfo: { | 20260507 02:47:18.501 | INFO | "gpu" : " 17 GB", | 20260507 02:47:18.501 | INFO | "huge" : "", | 20260507 02:47:18.501 | INFO | "huge_shared" : " 27 GB", | 20260507 02:47:18.501 | INFO | "malloc" : " 14 GB", | 20260507 02:47:18.501 | INFO | "pinned" : " 29 GB" | 20260507 02:47:18.501 | INFO | } | 20260507 02:47:18.502 | INFO | Perf report for the last 5.1359 seconds: | 20260507 02:47:18.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:47:18.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:18.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:18.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:18.502 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 02:47:18.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:18.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:18.502 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:47:18.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:18.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:18.502 | INFO | Uploaded 36GiB to GPU in 1.97373s (18.2396GiB/s) | 20260507 02:47:18.502 | INFO | Downloaded 34GiB to GPU in 1.90806s (17.8191GiB/s) | 20260507 02:47:23.642 | INFO | Analyzing chunk frames = [6490112,6490624) | 20260507 02:47:23.642 | INFO | MemInfo: { | 20260507 02:47:23.642 | INFO | "gpu" : " 17 GB", | 20260507 02:47:23.642 | INFO | "huge" : "", | 20260507 02:47:23.642 | INFO | "huge_shared" : " 27 GB", | 20260507 02:47:23.642 | INFO | "malloc" : " 14 GB", | 20260507 02:47:23.642 | INFO | "pinned" : " 29 GB" | 20260507 02:47:23.642 | INFO | } | 20260507 02:47:23.643 | INFO | Perf report for the last 5.14113 seconds: | 20260507 02:47:23.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:47:23.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:23.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:23.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:23.643 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 02:47:23.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:23.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:23.643 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:47:23.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:23.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:23.643 | INFO | Uploaded 36GiB to GPU in 2.00638s (17.9428GiB/s) | 20260507 02:47:23.643 | INFO | Downloaded 34GiB to GPU in 1.94287s (17.4999GiB/s) | 20260507 02:47:28.783 | INFO | Analyzing chunk frames = [6490624,6491136) | 20260507 02:47:28.783 | INFO | MemInfo: { | 20260507 02:47:28.783 | INFO | "gpu" : " 17 GB", | 20260507 02:47:28.783 | INFO | "huge" : "", | 20260507 02:47:28.783 | INFO | "huge_shared" : " 27 GB", | 20260507 02:47:28.783 | INFO | "malloc" : " 14 GB", | 20260507 02:47:28.783 | INFO | "pinned" : " 29 GB" | 20260507 02:47:28.783 | INFO | } | 20260507 02:47:28.783 | INFO | Perf report for the last 5.14041 seconds: | 20260507 02:47:28.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:47:28.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:28.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:28.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:28.783 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:47:28.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:28.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:28.783 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 02:47:28.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:28.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:28.784 | INFO | Uploaded 36GiB to GPU in 2.05894s (17.4848GiB/s) | 20260507 02:47:28.784 | INFO | Downloaded 34GiB to GPU in 1.96517s (17.3013GiB/s) | 20260507 02:47:33.919 | INFO | Analyzing chunk frames = [6491136,6491648) | 20260507 02:47:33.919 | INFO | MemInfo: { | 20260507 02:47:33.919 | INFO | "gpu" : " 17 GB", | 20260507 02:47:33.919 | INFO | "huge" : "", | 20260507 02:47:33.919 | INFO | "huge_shared" : " 27 GB", | 20260507 02:47:33.919 | INFO | "malloc" : " 14 GB", | 20260507 02:47:33.919 | INFO | "pinned" : " 29 GB" | 20260507 02:47:33.919 | INFO | } | 20260507 02:47:33.919 | INFO | Perf report for the last 5.13615 seconds: | 20260507 02:47:33.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:47:33.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:33.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:33.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:33.919 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 02:47:33.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:33.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:33.919 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 02:47:33.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:33.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:33.919 | INFO | Uploaded 36GiB to GPU in 1.98099s (18.1727GiB/s) | 20260507 02:47:33.919 | INFO | Downloaded 34GiB to GPU in 1.96216s (17.3278GiB/s) | 20260507 02:47:39.066 | INFO | Analyzing chunk frames = [6491648,6492160) | 20260507 02:47:39.066 | INFO | MemInfo: { | 20260507 02:47:39.066 | INFO | "gpu" : " 17 GB", | 20260507 02:47:39.066 | INFO | "huge" : "", | 20260507 02:47:39.066 | INFO | "huge_shared" : " 27 GB", | 20260507 02:47:39.066 | INFO | "malloc" : " 14 GB", | 20260507 02:47:39.066 | INFO | "pinned" : " 29 GB" | 20260507 02:47:39.066 | INFO | } | 20260507 02:47:39.066 | INFO | Perf report for the last 5.14718 seconds: | 20260507 02:47:39.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:47:39.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:39.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:39.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:39.067 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 02:47:39.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:39.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:39.067 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 02:47:39.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:39.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:39.067 | INFO | Uploaded 36GiB to GPU in 1.95115s (18.4506GiB/s) | 20260507 02:47:39.067 | INFO | Downloaded 34GiB to GPU in 1.96379s (17.3135GiB/s) | 20260507 02:47:39.164 | INFO | Starting a round of Baz IO | 20260507 02:47:42.263 | INFO | Populating RTMetrics report | 20260507 02:47:42.263 | INFO | RTMetrics report populated | 20260507 02:47:42.263 | INFO | Opening tmp RTMetrics json file | 20260507 02:47:42.264 | INFO | Serializing RTMetrics to json | 20260507 02:47:42.265 | INFO | Serialization done, swapping tmp file now... | 20260507 02:47:42.265 | INFO | RTMetrics done | 20260507 02:47:42.361 | INFO | Finished a round of Baz IO: Wrote 1.773 GiB in 3.197 seconds | 20260507 02:47:42.361 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.96% / 0.00% / 67.04% | 20260507 02:47:42.361 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.224868% / 0% / 6.35783e-05% | 20260507 02:47:42.361 | INFO | Baz IO overall: | 20260507 02:47:42.361 | INFO | BytesWritten : 3344435401133 | 20260507 02:47:42.361 | INFO | NumEvents : 1607933372453 | 20260507 02:47:44.201 | INFO | Analyzing chunk frames = [6492160,6492672) | 20260507 02:47:44.201 | INFO | MemInfo: { | 20260507 02:47:44.201 | INFO | "gpu" : " 17 GB", | 20260507 02:47:44.201 | INFO | "huge" : "", | 20260507 02:47:44.201 | INFO | "huge_shared" : " 27 GB", | 20260507 02:47:44.201 | INFO | "malloc" : " 14 GB", | 20260507 02:47:44.201 | INFO | "pinned" : " 29 GB" | 20260507 02:47:44.201 | INFO | } | 20260507 02:47:44.201 | INFO | Perf report for the last 5.13463 seconds: | 20260507 02:47:44.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:47:44.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:44.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:44.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:44.201 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 02:47:44.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:44.201 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 02:47:44.201 | INFO | PRE_HQ : 59.4%, 1.8, 55.9ms, 0 | 20260507 02:47:44.201 | INFO | BAZWRITER : 61.9%, 3.3, 105.4ms, 0 | 20260507 02:47:44.201 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:47:44.201 | INFO | Uploaded 36GiB to GPU in 2.01564s (17.8603GiB/s) | 20260507 02:47:44.201 | INFO | Downloaded 38GiB to GPU in 2.17487s (17.4723GiB/s) | 20260507 02:47:49.335 | INFO | Analyzing chunk frames = [6492672,6493184) | 20260507 02:47:49.335 | INFO | MemInfo: { | 20260507 02:47:49.335 | INFO | "gpu" : " 17 GB", | 20260507 02:47:49.335 | INFO | "huge" : "", | 20260507 02:47:49.335 | INFO | "huge_shared" : " 27 GB", | 20260507 02:47:49.335 | INFO | "malloc" : " 14 GB", | 20260507 02:47:49.335 | INFO | "pinned" : " 29 GB" | 20260507 02:47:49.335 | INFO | } | 20260507 02:47:49.335 | INFO | Perf report for the last 5.1344 seconds: | 20260507 02:47:49.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:47:49.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:49.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:49.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:49.336 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:47:49.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:49.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:49.336 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:47:49.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:49.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:49.336 | INFO | Uploaded 36GiB to GPU in 1.98696s (18.1181GiB/s) | 20260507 02:47:49.336 | INFO | Downloaded 34GiB to GPU in 1.93398s (17.5803GiB/s) | 20260507 02:47:54.475 | INFO | Analyzing chunk frames = [6493184,6493696) | 20260507 02:47:54.475 | INFO | MemInfo: { | 20260507 02:47:54.475 | INFO | "gpu" : " 17 GB", | 20260507 02:47:54.475 | INFO | "huge" : "", | 20260507 02:47:54.475 | INFO | "huge_shared" : " 27 GB", | 20260507 02:47:54.475 | INFO | "malloc" : " 14 GB", | 20260507 02:47:54.475 | INFO | "pinned" : " 29 GB" | 20260507 02:47:54.475 | INFO | } | 20260507 02:47:54.475 | INFO | Perf report for the last 5.13989 seconds: | 20260507 02:47:54.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:47:54.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:54.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:54.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:54.475 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:47:54.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:54.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:54.476 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:47:54.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:54.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:54.476 | INFO | Uploaded 36GiB to GPU in 2.00631s (17.9434GiB/s) | 20260507 02:47:54.476 | INFO | Downloaded 34GiB to GPU in 1.94169s (17.5105GiB/s) | 20260507 02:47:59.615 | INFO | Analyzing chunk frames = [6493696,6494208) | 20260507 02:47:59.615 | INFO | MemInfo: { | 20260507 02:47:59.615 | INFO | "gpu" : " 17 GB", | 20260507 02:47:59.615 | INFO | "huge" : "", | 20260507 02:47:59.615 | INFO | "huge_shared" : " 27 GB", | 20260507 02:47:59.615 | INFO | "malloc" : " 14 GB", | 20260507 02:47:59.615 | INFO | "pinned" : " 29 GB" | 20260507 02:47:59.615 | INFO | } | 20260507 02:47:59.615 | INFO | Perf report for the last 5.13944 seconds: | 20260507 02:47:59.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:47:59.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:59.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:59.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:59.615 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:47:59.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:59.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:59.615 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:47:59.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:59.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:47:59.615 | INFO | Uploaded 36GiB to GPU in 1.96105s (18.3575GiB/s) | 20260507 02:47:59.615 | INFO | Downloaded 34GiB to GPU in 1.90336s (17.8632GiB/s) | 20260507 02:48:04.762 | INFO | Analyzing chunk frames = [6494208,6494720) | 20260507 02:48:04.762 | INFO | MemInfo: { | 20260507 02:48:04.762 | INFO | "gpu" : " 17 GB", | 20260507 02:48:04.762 | INFO | "huge" : "", | 20260507 02:48:04.762 | INFO | "huge_shared" : " 27 GB", | 20260507 02:48:04.762 | INFO | "malloc" : " 14 GB", | 20260507 02:48:04.762 | INFO | "pinned" : " 29 GB" | 20260507 02:48:04.762 | INFO | } | 20260507 02:48:04.762 | INFO | Perf report for the last 5.14696 seconds: | 20260507 02:48:04.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:48:04.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:04.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:04.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:04.762 | INFO | ANALYSIS : 59.8%, 3.0, 93.2ms, 0 | 20260507 02:48:04.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:04.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:04.762 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:48:04.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:04.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:04.762 | INFO | Uploaded 36GiB to GPU in 1.99001s (18.0903GiB/s) | 20260507 02:48:04.762 | INFO | Downloaded 34GiB to GPU in 1.93656s (17.5569GiB/s) | 20260507 02:48:09.898 | INFO | Analyzing chunk frames = [6494720,6495232) | 20260507 02:48:09.898 | INFO | MemInfo: { | 20260507 02:48:09.898 | INFO | "gpu" : " 17 GB", | 20260507 02:48:09.898 | INFO | "huge" : "", | 20260507 02:48:09.898 | INFO | "huge_shared" : " 27 GB", | 20260507 02:48:09.898 | INFO | "malloc" : " 14 GB", | 20260507 02:48:09.898 | INFO | "pinned" : " 29 GB" | 20260507 02:48:09.898 | INFO | } | 20260507 02:48:09.898 | INFO | Perf report for the last 5.13595 seconds: | 20260507 02:48:09.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:48:09.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:09.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:09.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:09.898 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 02:48:09.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:09.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:09.898 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:48:09.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:09.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:09.898 | INFO | Uploaded 36GiB to GPU in 2.02875s (17.7449GiB/s) | 20260507 02:48:09.898 | INFO | Downloaded 34GiB to GPU in 1.92205s (17.6895GiB/s) | 20260507 02:48:15.032 | INFO | Analyzing chunk frames = [6495232,6495744) | 20260507 02:48:15.032 | INFO | MemInfo: { | 20260507 02:48:15.032 | INFO | "gpu" : " 17 GB", | 20260507 02:48:15.032 | INFO | "huge" : "", | 20260507 02:48:15.032 | INFO | "huge_shared" : " 27 GB", | 20260507 02:48:15.032 | INFO | "malloc" : " 14 GB", | 20260507 02:48:15.032 | INFO | "pinned" : " 29 GB" | 20260507 02:48:15.032 | INFO | } | 20260507 02:48:15.032 | INFO | Perf report for the last 5.13459 seconds: | 20260507 02:48:15.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:48:15.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:15.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:15.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:15.032 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 02:48:15.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:15.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:15.032 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 02:48:15.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:15.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:15.033 | INFO | Uploaded 36GiB to GPU in 1.96396s (18.3303GiB/s) | 20260507 02:48:15.033 | INFO | Downloaded 34GiB to GPU in 1.97689s (17.1987GiB/s) | 20260507 02:48:20.176 | INFO | Analyzing chunk frames = [6495744,6496256) | 20260507 02:48:20.176 | INFO | MemInfo: { | 20260507 02:48:20.176 | INFO | "gpu" : " 17 GB", | 20260507 02:48:20.176 | INFO | "huge" : "", | 20260507 02:48:20.177 | INFO | "huge_shared" : " 27 GB", | 20260507 02:48:20.177 | INFO | "malloc" : " 14 GB", | 20260507 02:48:20.177 | INFO | "pinned" : " 29 GB" | 20260507 02:48:20.177 | INFO | } | 20260507 02:48:20.177 | INFO | Perf report for the last 5.1443 seconds: | 20260507 02:48:20.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:48:20.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:20.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:20.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:20.177 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 02:48:20.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:20.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:20.177 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 02:48:20.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:20.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:20.177 | INFO | Uploaded 36GiB to GPU in 1.95196s (18.443GiB/s) | 20260507 02:48:20.177 | INFO | Downloaded 34GiB to GPU in 1.97467s (17.2181GiB/s) | 20260507 02:48:20.177 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:48:20.177 | INFO | Total mem usage: 18 GiB | 20260507 02:48:20.177 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:48:20.177 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 02:48:20.177 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:48:20.177 | INFO | High Water Breakdown: | 20260507 02:48:20.177 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:48:20.177 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:48:20.177 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:48:20.177 | INFO | ------------------------------------------ | 20260507 02:48:20.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:48:20.177 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:48:20.177 | INFO | Total mem usage: 26.5 GiB | 20260507 02:48:20.177 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:48:20.177 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:48:20.177 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:48:20.177 | INFO | High Water Breakdown: | 20260507 02:48:20.177 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:48:20.177 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:48:20.177 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:48:20.177 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:48:20.177 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:48:20.177 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:48:20.177 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:48:20.177 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:48:20.177 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:48:20.177 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:48:20.177 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:48:20.177 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:48:20.177 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:48:20.177 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:48:20.177 | INFO | ------------------------------------------ | 20260507 02:48:20.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:48:20.177 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:48:20.177 | INFO | Total mem usage: 25.6 GiB | 20260507 02:48:20.177 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:48:20.177 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 02:48:20.177 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 02:48:20.177 | INFO | High Water Breakdown: | 20260507 02:48:20.177 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 02:48:20.177 | INFO | ------------------------------------------ | 20260507 02:48:20.177 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:48:20.177 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:48:20.177 | INFO | Total mem usage: 15.7 GiB | 20260507 02:48:20.177 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:48:20.177 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:48:20.177 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:48:20.177 | INFO | High Water Breakdown: | 20260507 02:48:20.177 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:48:20.177 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:48:20.178 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:48:20.178 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:48:20.178 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:48:20.178 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:48:20.178 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:48:20.178 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:48:20.178 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:48:20.178 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:48:20.178 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:48:20.178 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:48:20.178 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:48:20.178 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:48:20.178 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:48:20.178 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:48:20.178 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:48:20.178 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:48:20.178 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:48:20.178 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:48:20.178 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:48:20.178 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:48:20.178 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:48:20.178 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:48:20.178 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:48:20.178 | INFO | ------------------------------------------ | 20260507 02:48:20.178 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:48:20.178 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:48:20.178 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:48:20.178 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:48:20.178 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.49e+01, 1.34e+01 / 1.30e+00 | 20260507 02:48:20.178 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 9.99e-02, 9.33e-03 / 5.18e-03 | 20260507 02:48:20.178 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.44e+00, 5.27e-01 / 3.36e-01 | 20260507 02:48:20.178 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.44e+00, 1.75e-01 / 1.28e-01 | 20260507 02:48:20.178 | INFO | PerfReport: 16.03%, DME -- 3.95e+01 / 4.38e+01, 3.77e+00 / 1.70e+00 | 20260507 02:48:20.178 | INFO | PerfReport: 57.71%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.40e+00 / 1.58e+00 | 20260507 02:48:20.178 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.63e+00, 3.75e-01 / 2.43e-01 | 20260507 02:48:20.178 | INFO | PerfReport: 3.07%, Metrics -- 9.46e-01 / 1.24e+00, 2.55e-01 / 1.38e-01 | 20260507 02:48:20.178 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.20e-01 / 3.73e-01, 5.09e-02 / 1.96e-02 | 20260507 02:48:20.178 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:48:20.178 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:48:20.178 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:48:20.178 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 02:48:20.178 | INFO | PerfReport: 21.63%, Upload -- 2.08e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260507 02:48:20.178 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 02:48:20.178 | INFO | PerfReport: 32.04%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 02:48:20.178 | INFO | PerfReport: 1.40%, LockUpload -- 1.35e+00 / 0.00e+00, 4.69e+00 / 0.00e+00 | 20260507 02:48:20.178 | INFO | PerfReport: 1.27%, LockDownload -- 1.22e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260507 02:48:20.178 | INFO | PerfReport: 22.29%, LockCompute -- 2.15e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 02:48:20.178 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260507 02:48:20.274 | INFO | Starting a round of Baz IO | 20260507 02:48:23.383 | INFO | Populating RTMetrics report | 20260507 02:48:23.383 | INFO | RTMetrics report populated | 20260507 02:48:23.383 | INFO | Opening tmp RTMetrics json file | 20260507 02:48:23.402 | INFO | Serializing RTMetrics to json | 20260507 02:48:23.403 | INFO | Serialization done, swapping tmp file now... | 20260507 02:48:23.404 | INFO | RTMetrics done | 20260507 02:48:23.477 | INFO | Finished a round of Baz IO: Wrote 1.771 GiB in 3.203 seconds | 20260507 02:48:23.477 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.96% / 0.00% / 67.04% | 20260507 02:48:23.478 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.223482% / 1.19209e-05% / 6.35783e-05% | 20260507 02:48:23.478 | INFO | Baz IO overall: | 20260507 02:48:23.478 | INFO | BytesWritten : 3346336933515 | 20260507 02:48:23.478 | INFO | NumEvents : 1608831895509 | 20260507 02:48:25.312 | INFO | Analyzing chunk frames = [6496256,6496768) | 20260507 02:48:25.312 | INFO | MemInfo: { | 20260507 02:48:25.312 | INFO | "gpu" : " 17 GB", | 20260507 02:48:25.312 | INFO | "huge" : "", | 20260507 02:48:25.312 | INFO | "huge_shared" : " 27 GB", | 20260507 02:48:25.312 | INFO | "malloc" : " 14 GB", | 20260507 02:48:25.312 | INFO | "pinned" : " 29 GB" | 20260507 02:48:25.312 | INFO | } | 20260507 02:48:25.313 | INFO | Perf report for the last 5.13585 seconds: | 20260507 02:48:25.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:48:25.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:25.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:25.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:25.313 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 02:48:25.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:25.313 | INFO | RT_METRICS : 42.1%, 1.0, 22.0ms, 0 | 20260507 02:48:25.313 | INFO | PRE_HQ : 59.4%, 1.8, 55.9ms, 0 | 20260507 02:48:25.313 | INFO | BAZWRITER : 62.0%, 3.3, 106.2ms, 0 | 20260507 02:48:25.313 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:48:25.313 | INFO | Uploaded 36GiB to GPU in 2.0375s (17.6688GiB/s) | 20260507 02:48:25.313 | INFO | Downloaded 38GiB to GPU in 2.18302s (17.407GiB/s) | 20260507 02:48:30.454 | INFO | Analyzing chunk frames = [6496768,6497280) | 20260507 02:48:30.454 | INFO | MemInfo: { | 20260507 02:48:30.454 | INFO | "gpu" : " 17 GB", | 20260507 02:48:30.454 | INFO | "huge" : "", | 20260507 02:48:30.454 | INFO | "huge_shared" : " 27 GB", | 20260507 02:48:30.454 | INFO | "malloc" : " 14 GB", | 20260507 02:48:30.454 | INFO | "pinned" : " 29 GB" | 20260507 02:48:30.454 | INFO | } | 20260507 02:48:30.454 | INFO | Perf report for the last 5.14172 seconds: | 20260507 02:48:30.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:48:30.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:30.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:30.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:30.454 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:48:30.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:30.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:30.454 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:48:30.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:30.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:30.455 | INFO | Uploaded 36GiB to GPU in 2.01153s (17.8969GiB/s) | 20260507 02:48:30.455 | INFO | Downloaded 34GiB to GPU in 1.93918s (17.5331GiB/s) | 20260507 02:48:35.594 | INFO | Analyzing chunk frames = [6497280,6497792) | 20260507 02:48:35.594 | INFO | MemInfo: { | 20260507 02:48:35.594 | INFO | "gpu" : " 17 GB", | 20260507 02:48:35.594 | INFO | "huge" : "", | 20260507 02:48:35.594 | INFO | "huge_shared" : " 27 GB", | 20260507 02:48:35.594 | INFO | "malloc" : " 14 GB", | 20260507 02:48:35.594 | INFO | "pinned" : " 29 GB" | 20260507 02:48:35.594 | INFO | } | 20260507 02:48:35.594 | INFO | Perf report for the last 5.13962 seconds: | 20260507 02:48:35.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:48:35.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:35.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:35.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:35.594 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:48:35.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:35.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:35.594 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:48:35.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:35.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:35.594 | INFO | Uploaded 36GiB to GPU in 1.98382s (18.1468GiB/s) | 20260507 02:48:35.594 | INFO | Downloaded 34GiB to GPU in 1.9428s (17.5005GiB/s) | 20260507 02:48:40.731 | INFO | Analyzing chunk frames = [6497792,6498304) | 20260507 02:48:40.731 | INFO | MemInfo: { | 20260507 02:48:40.731 | INFO | "gpu" : " 17 GB", | 20260507 02:48:40.731 | INFO | "huge" : "", | 20260507 02:48:40.731 | INFO | "huge_shared" : " 27 GB", | 20260507 02:48:40.731 | INFO | "malloc" : " 14 GB", | 20260507 02:48:40.731 | INFO | "pinned" : " 29 GB" | 20260507 02:48:40.731 | INFO | } | 20260507 02:48:40.731 | INFO | Perf report for the last 5.13695 seconds: | 20260507 02:48:40.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:48:40.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:40.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:40.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:40.731 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:48:40.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:40.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:40.731 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:48:40.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:40.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:40.731 | INFO | Uploaded 36GiB to GPU in 1.95437s (18.4203GiB/s) | 20260507 02:48:40.731 | INFO | Downloaded 34GiB to GPU in 1.91473s (17.757GiB/s) | 20260507 02:48:45.871 | INFO | Analyzing chunk frames = [6498304,6498816) | 20260507 02:48:45.871 | INFO | MemInfo: { | 20260507 02:48:45.871 | INFO | "gpu" : " 17 GB", | 20260507 02:48:45.871 | INFO | "huge" : "", | 20260507 02:48:45.871 | INFO | "huge_shared" : " 27 GB", | 20260507 02:48:45.871 | INFO | "malloc" : " 14 GB", | 20260507 02:48:45.871 | INFO | "pinned" : " 29 GB" | 20260507 02:48:45.871 | INFO | } | 20260507 02:48:45.871 | INFO | Perf report for the last 5.14015 seconds: | 20260507 02:48:45.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:48:45.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:45.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:45.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:45.871 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 02:48:45.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:45.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:45.871 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:48:45.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:45.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:45.871 | INFO | Uploaded 36GiB to GPU in 1.99592s (18.0368GiB/s) | 20260507 02:48:45.871 | INFO | Downloaded 34GiB to GPU in 1.93867s (17.5378GiB/s) | 20260507 02:48:51.011 | INFO | Analyzing chunk frames = [6498816,6499328) | 20260507 02:48:51.011 | INFO | MemInfo: { | 20260507 02:48:51.011 | INFO | "gpu" : " 17 GB", | 20260507 02:48:51.011 | INFO | "huge" : "", | 20260507 02:48:51.011 | INFO | "huge_shared" : " 27 GB", | 20260507 02:48:51.011 | INFO | "malloc" : " 14 GB", | 20260507 02:48:51.011 | INFO | "pinned" : " 29 GB" | 20260507 02:48:51.011 | INFO | } | 20260507 02:48:51.012 | INFO | Perf report for the last 5.14067 seconds: | 20260507 02:48:51.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:48:51.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:51.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:51.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:51.012 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 02:48:51.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:51.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:51.012 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 02:48:51.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:51.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:51.012 | INFO | Uploaded 36GiB to GPU in 2.04246s (17.6258GiB/s) | 20260507 02:48:51.012 | INFO | Downloaded 34GiB to GPU in 1.96228s (17.3267GiB/s) | 20260507 02:48:56.146 | INFO | Analyzing chunk frames = [6499328,6499840) | 20260507 02:48:56.146 | INFO | MemInfo: { | 20260507 02:48:56.146 | INFO | "gpu" : " 17 GB", | 20260507 02:48:56.146 | INFO | "huge" : "", | 20260507 02:48:56.146 | INFO | "huge_shared" : " 27 GB", | 20260507 02:48:56.146 | INFO | "malloc" : " 14 GB", | 20260507 02:48:56.146 | INFO | "pinned" : " 29 GB" | 20260507 02:48:56.146 | INFO | } | 20260507 02:48:56.146 | INFO | Perf report for the last 5.13404 seconds: | 20260507 02:48:56.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:48:56.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:56.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:56.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:56.146 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:48:56.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:56.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:56.146 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:48:56.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:56.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:48:56.146 | INFO | Uploaded 36GiB to GPU in 1.96926s (18.2809GiB/s) | 20260507 02:48:56.146 | INFO | Downloaded 34GiB to GPU in 1.97498s (17.2153GiB/s) | 20260507 02:49:01.281 | INFO | Analyzing chunk frames = [6499840,6500352) | 20260507 02:49:01.281 | INFO | MemInfo: { | 20260507 02:49:01.281 | INFO | "gpu" : " 17 GB", | 20260507 02:49:01.281 | INFO | "huge" : "", | 20260507 02:49:01.281 | INFO | "huge_shared" : " 27 GB", | 20260507 02:49:01.282 | INFO | "malloc" : " 14 GB", | 20260507 02:49:01.282 | INFO | "pinned" : " 29 GB" | 20260507 02:49:01.282 | INFO | } | 20260507 02:49:01.282 | INFO | Perf report for the last 5.1359 seconds: | 20260507 02:49:01.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:49:01.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:01.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:01.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:01.282 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 02:49:01.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:01.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:01.282 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 02:49:01.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:01.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:01.282 | INFO | Uploaded 36GiB to GPU in 1.98355s (18.1493GiB/s) | 20260507 02:49:01.282 | INFO | Downloaded 34GiB to GPU in 1.97748s (17.1936GiB/s) | 20260507 02:49:01.379 | INFO | Starting a round of Baz IO | 20260507 02:49:04.497 | INFO | Populating RTMetrics report | 20260507 02:49:04.497 | INFO | RTMetrics report populated | 20260507 02:49:04.497 | INFO | Opening tmp RTMetrics json file | 20260507 02:49:04.504 | INFO | Serializing RTMetrics to json | 20260507 02:49:04.515 | INFO | Serialization done, swapping tmp file now... | 20260507 02:49:04.515 | INFO | RTMetrics done | 20260507 02:49:04.585 | INFO | Finished a round of Baz IO: Wrote 1.773 GiB in 3.206 seconds | 20260507 02:49:04.585 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.96% / 0.00% / 67.04% | 20260507 02:49:04.585 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.223871% / 0% / 7.15256e-05% | 20260507 02:49:04.585 | INFO | Baz IO overall: | 20260507 02:49:04.585 | INFO | BytesWritten : 3348240421337 | 20260507 02:49:04.585 | INFO | NumEvents : 1609732055441 | 20260507 02:49:06.430 | INFO | Analyzing chunk frames = [6500352,6500864) | 20260507 02:49:06.430 | INFO | MemInfo: { | 20260507 02:49:06.430 | INFO | "gpu" : " 17 GB", | 20260507 02:49:06.430 | INFO | "huge" : "", | 20260507 02:49:06.430 | INFO | "huge_shared" : " 27 GB", | 20260507 02:49:06.430 | INFO | "malloc" : " 14 GB", | 20260507 02:49:06.430 | INFO | "pinned" : " 29 GB" | 20260507 02:49:06.430 | INFO | } | 20260507 02:49:06.430 | INFO | Perf report for the last 5.14853 seconds: | 20260507 02:49:06.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:49:06.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:06.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:06.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:06.430 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260507 02:49:06.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:06.430 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 02:49:06.430 | INFO | PRE_HQ : 59.4%, 1.8, 55.8ms, 0 | 20260507 02:49:06.430 | INFO | BAZWRITER : 61.9%, 3.3, 104.5ms, 0 | 20260507 02:49:06.430 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:49:06.431 | INFO | Uploaded 36GiB to GPU in 2.03016s (17.7326GiB/s) | 20260507 02:49:06.431 | INFO | Downloaded 38GiB to GPU in 2.17914s (17.4381GiB/s) | 20260507 02:49:11.568 | INFO | Analyzing chunk frames = [6500864,6501376) | 20260507 02:49:11.568 | INFO | MemInfo: { | 20260507 02:49:11.568 | INFO | "gpu" : " 17 GB", | 20260507 02:49:11.568 | INFO | "huge" : "", | 20260507 02:49:11.568 | INFO | "huge_shared" : " 27 GB", | 20260507 02:49:11.568 | INFO | "malloc" : " 14 GB", | 20260507 02:49:11.568 | INFO | "pinned" : " 29 GB" | 20260507 02:49:11.568 | INFO | } | 20260507 02:49:11.568 | INFO | Perf report for the last 5.13809 seconds: | 20260507 02:49:11.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:49:11.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:11.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:11.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:11.568 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:49:11.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:11.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:11.568 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 02:49:11.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:11.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:11.569 | INFO | Uploaded 36GiB to GPU in 2.00904s (17.919GiB/s) | 20260507 02:49:11.569 | INFO | Downloaded 34GiB to GPU in 1.93672s (17.5555GiB/s) | 20260507 02:49:16.714 | INFO | Analyzing chunk frames = [6501376,6501888) | 20260507 02:49:16.715 | INFO | MemInfo: { | 20260507 02:49:16.715 | INFO | "gpu" : " 17 GB", | 20260507 02:49:16.715 | INFO | "huge" : "", | 20260507 02:49:16.715 | INFO | "huge_shared" : " 27 GB", | 20260507 02:49:16.715 | INFO | "malloc" : " 14 GB", | 20260507 02:49:16.715 | INFO | "pinned" : " 29 GB" | 20260507 02:49:16.715 | INFO | } | 20260507 02:49:16.715 | INFO | Perf report for the last 5.14627 seconds: | 20260507 02:49:16.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:49:16.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:16.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:16.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:16.715 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 02:49:16.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:16.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:16.715 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:49:16.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:16.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:16.715 | INFO | Uploaded 36GiB to GPU in 1.99231s (18.0694GiB/s) | 20260507 02:49:16.715 | INFO | Downloaded 34GiB to GPU in 1.94195s (17.5081GiB/s) | 20260507 02:49:21.846 | INFO | Analyzing chunk frames = [6501888,6502400) | 20260507 02:49:21.846 | INFO | MemInfo: { | 20260507 02:49:21.846 | INFO | "gpu" : " 17 GB", | 20260507 02:49:21.846 | INFO | "huge" : "", | 20260507 02:49:21.846 | INFO | "huge_shared" : " 27 GB", | 20260507 02:49:21.847 | INFO | "malloc" : " 14 GB", | 20260507 02:49:21.847 | INFO | "pinned" : " 29 GB" | 20260507 02:49:21.847 | INFO | } | 20260507 02:49:21.847 | INFO | Perf report for the last 5.13213 seconds: | 20260507 02:49:21.847 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:49:21.847 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:21.847 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:21.847 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:21.847 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 02:49:21.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:21.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:21.847 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:49:21.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:21.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:21.847 | INFO | Uploaded 36GiB to GPU in 1.99873s (18.0114GiB/s) | 20260507 02:49:21.847 | INFO | Downloaded 34GiB to GPU in 1.93676s (17.5551GiB/s) | 20260507 02:49:26.980 | INFO | Analyzing chunk frames = [6502400,6502912) | 20260507 02:49:26.980 | INFO | MemInfo: { | 20260507 02:49:26.980 | INFO | "gpu" : " 17 GB", | 20260507 02:49:26.980 | INFO | "huge" : "", | 20260507 02:49:26.980 | INFO | "huge_shared" : " 27 GB", | 20260507 02:49:26.980 | INFO | "malloc" : " 14 GB", | 20260507 02:49:26.980 | INFO | "pinned" : " 29 GB" | 20260507 02:49:26.980 | INFO | } | 20260507 02:49:26.980 | INFO | Perf report for the last 5.13327 seconds: | 20260507 02:49:26.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:49:26.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:26.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:26.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:26.980 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 02:49:26.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:26.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:26.980 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:49:26.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:26.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:26.980 | INFO | Uploaded 36GiB to GPU in 2.02249s (17.7998GiB/s) | 20260507 02:49:26.980 | INFO | Downloaded 34GiB to GPU in 1.94252s (17.503GiB/s) | 20260507 02:49:32.121 | INFO | Analyzing chunk frames = [6502912,6503424) | 20260507 02:49:32.121 | INFO | MemInfo: { | 20260507 02:49:32.121 | INFO | "gpu" : " 17 GB", | 20260507 02:49:32.121 | INFO | "huge" : "", | 20260507 02:49:32.121 | INFO | "huge_shared" : " 27 GB", | 20260507 02:49:32.121 | INFO | "malloc" : " 14 GB", | 20260507 02:49:32.121 | INFO | "pinned" : " 29 GB" | 20260507 02:49:32.122 | INFO | } | 20260507 02:49:32.122 | INFO | Perf report for the last 5.14164 seconds: | 20260507 02:49:32.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:49:32.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:32.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:32.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:32.122 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 02:49:32.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:32.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:32.122 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:49:32.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:32.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:32.122 | INFO | Uploaded 36GiB to GPU in 2.01536s (17.8628GiB/s) | 20260507 02:49:32.122 | INFO | Downloaded 34GiB to GPU in 1.9458s (17.4735GiB/s) | 20260507 02:49:37.269 | INFO | Analyzing chunk frames = [6503424,6503936) | 20260507 02:49:37.269 | INFO | MemInfo: { | 20260507 02:49:37.269 | INFO | "gpu" : " 17 GB", | 20260507 02:49:37.269 | INFO | "huge" : "", | 20260507 02:49:37.269 | INFO | "huge_shared" : " 27 GB", | 20260507 02:49:37.269 | INFO | "malloc" : " 14 GB", | 20260507 02:49:37.269 | INFO | "pinned" : " 29 GB" | 20260507 02:49:37.269 | INFO | } | 20260507 02:49:37.270 | INFO | Perf report for the last 5.14782 seconds: | 20260507 02:49:37.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:49:37.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:37.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:37.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:37.270 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 02:49:37.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:37.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:37.270 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:49:37.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:37.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:37.270 | INFO | Uploaded 36GiB to GPU in 1.98221s (18.1616GiB/s) | 20260507 02:49:37.270 | INFO | Downloaded 34GiB to GPU in 1.99878s (17.0104GiB/s) | 20260507 02:49:42.406 | INFO | Analyzing chunk frames = [6503936,6504448) | 20260507 02:49:42.406 | INFO | MemInfo: { | 20260507 02:49:42.406 | INFO | "gpu" : " 17 GB", | 20260507 02:49:42.406 | INFO | "huge" : "", | 20260507 02:49:42.406 | INFO | "huge_shared" : " 27 GB", | 20260507 02:49:42.406 | INFO | "malloc" : " 14 GB", | 20260507 02:49:42.406 | INFO | "pinned" : " 29 GB" | 20260507 02:49:42.406 | INFO | } | 20260507 02:49:42.406 | INFO | Perf report for the last 5.13666 seconds: | 20260507 02:49:42.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:49:42.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:42.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:42.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:42.406 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:49:42.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:42.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:42.406 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:49:42.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:42.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:42.406 | INFO | Uploaded 36GiB to GPU in 1.99578s (18.0381GiB/s) | 20260507 02:49:42.407 | INFO | Downloaded 34GiB to GPU in 1.97s (17.2589GiB/s) | 20260507 02:49:42.407 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:49:42.407 | INFO | Total mem usage: 18 GiB | 20260507 02:49:42.407 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:49:42.407 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:49:42.407 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:49:42.407 | INFO | High Water Breakdown: | 20260507 02:49:42.407 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:49:42.407 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:49:42.407 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:49:42.407 | INFO | ------------------------------------------ | 20260507 02:49:42.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:49:42.407 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:49:42.407 | INFO | Total mem usage: 26.5 GiB | 20260507 02:49:42.407 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:49:42.407 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:49:42.407 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:49:42.407 | INFO | High Water Breakdown: | 20260507 02:49:42.407 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:49:42.407 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:49:42.407 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:49:42.407 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:49:42.407 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:49:42.407 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:49:42.407 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:49:42.407 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:49:42.407 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:49:42.407 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:49:42.407 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:49:42.407 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:49:42.407 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:49:42.407 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:49:42.407 | INFO | ------------------------------------------ | 20260507 02:49:42.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:49:42.407 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:49:42.407 | INFO | Total mem usage: 25.6 GiB | 20260507 02:49:42.407 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:49:42.407 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 02:49:42.407 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 02:49:42.407 | INFO | High Water Breakdown: | 20260507 02:49:42.407 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 02:49:42.407 | INFO | ------------------------------------------ | 20260507 02:49:42.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:49:42.407 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:49:42.407 | INFO | Total mem usage: 15.7 GiB | 20260507 02:49:42.407 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:49:42.407 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:49:42.407 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:49:42.407 | INFO | High Water Breakdown: | 20260507 02:49:42.407 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:49:42.407 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:49:42.407 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:49:42.407 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:49:42.407 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:49:42.407 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:49:42.407 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:49:42.407 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:49:42.407 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:49:42.407 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:49:42.407 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:49:42.407 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:49:42.407 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:49:42.407 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:49:42.407 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:49:42.407 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:49:42.407 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:49:42.407 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:49:42.407 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:49:42.407 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:49:42.407 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:49:42.407 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:49:42.407 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:49:42.407 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:49:42.407 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:49:42.407 | INFO | ------------------------------------------ | 20260507 02:49:42.407 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:49:42.408 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:49:42.408 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:49:42.408 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:49:42.408 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.55e+01, 1.35e+01 / 1.82e+00 | 20260507 02:49:42.408 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.00e-01, 9.89e-03 / 5.22e-03 | 20260507 02:49:42.408 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.51e+00, 5.18e-01 / 2.28e-01 | 20260507 02:49:42.408 | INFO | PerfReport: 3.22%, Histogram -- 9.92e-01 / 1.45e+00, 1.67e-01 / 1.20e-01 | 20260507 02:49:42.408 | INFO | PerfReport: 16.16%, DME -- 3.98e+01 / 4.49e+01, 4.11e+00 / 2.15e+00 | 20260507 02:49:42.408 | INFO | PerfReport: 57.52%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.22e+00 / 1.24e+00 | 20260507 02:49:42.408 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.16e+00 / 2.71e+00, 3.85e-01 / 1.95e-01 | 20260507 02:49:42.408 | INFO | PerfReport: 3.07%, Metrics -- 9.45e-01 / 1.27e+00, 2.49e-01 / 1.27e-01 | 20260507 02:49:42.408 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.19e-01 / 3.72e-01, 5.03e-02 / 2.01e-02 | 20260507 02:49:42.408 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:49:42.408 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:49:42.408 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:49:42.408 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 02:49:42.408 | INFO | PerfReport: 21.64%, Upload -- 2.08e+01 / 0.00e+00, 6.91e+00 / 0.00e+00 | 20260507 02:49:42.408 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260507 02:49:42.408 | INFO | PerfReport: 31.93%, Compute -- 3.08e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 02:49:42.408 | INFO | PerfReport: 1.37%, LockUpload -- 1.32e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 02:49:42.408 | INFO | PerfReport: 1.45%, LockDownload -- 1.39e+00 / 0.00e+00, 3.31e+00 / 0.00e+00 | 20260507 02:49:42.408 | INFO | PerfReport: 22.19%, LockCompute -- 2.14e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 02:49:42.408 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260507 02:49:42.503 | INFO | Starting a round of Baz IO | 20260507 02:49:45.612 | INFO | Populating RTMetrics report | 20260507 02:49:45.613 | INFO | RTMetrics report populated | 20260507 02:49:45.613 | INFO | Opening tmp RTMetrics json file | 20260507 02:49:45.627 | INFO | Serializing RTMetrics to json | 20260507 02:49:45.632 | INFO | Serialization done, swapping tmp file now... | 20260507 02:49:45.633 | INFO | RTMetrics done | 20260507 02:49:45.702 | INFO | Finished a round of Baz IO: Wrote 1.772 GiB in 3.198 seconds | 20260507 02:49:45.702 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.96% / 0.00% / 67.04% | 20260507 02:49:45.702 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.22289% / 3.97364e-06% / 7.15256e-05% | 20260507 02:49:45.702 | INFO | Baz IO overall: | 20260507 02:49:45.702 | INFO | BytesWritten : 3350143199322 | 20260507 02:49:45.702 | INFO | NumEvents : 1610631581497 | 20260507 02:49:47.542 | INFO | Analyzing chunk frames = [6504448,6504960) | 20260507 02:49:47.542 | INFO | MemInfo: { | 20260507 02:49:47.542 | INFO | "gpu" : " 17 GB", | 20260507 02:49:47.542 | INFO | "huge" : "", | 20260507 02:49:47.542 | INFO | "huge_shared" : " 27 GB", | 20260507 02:49:47.542 | INFO | "malloc" : " 14 GB", | 20260507 02:49:47.542 | INFO | "pinned" : " 29 GB" | 20260507 02:49:47.542 | INFO | } | 20260507 02:49:47.542 | INFO | Perf report for the last 5.13566 seconds: | 20260507 02:49:47.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:49:47.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:47.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:47.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:47.542 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 02:49:47.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:47.542 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 02:49:47.542 | INFO | PRE_HQ : 59.2%, 1.8, 55.6ms, 0 | 20260507 02:49:47.542 | INFO | BAZWRITER : 61.9%, 3.3, 106.4ms, 0 | 20260507 02:49:47.542 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:49:47.542 | INFO | Uploaded 36GiB to GPU in 2.04928s (17.5672GiB/s) | 20260507 02:49:47.542 | INFO | Downloaded 38GiB to GPU in 2.18204s (17.4149GiB/s) | 20260507 02:49:52.683 | INFO | Analyzing chunk frames = [6504960,6505472) | 20260507 02:49:52.683 | INFO | MemInfo: { | 20260507 02:49:52.683 | INFO | "gpu" : " 17 GB", | 20260507 02:49:52.683 | INFO | "huge" : "", | 20260507 02:49:52.683 | INFO | "huge_shared" : " 27 GB", | 20260507 02:49:52.683 | INFO | "malloc" : " 14 GB", | 20260507 02:49:52.683 | INFO | "pinned" : " 29 GB" | 20260507 02:49:52.683 | INFO | } | 20260507 02:49:52.684 | INFO | Perf report for the last 5.14166 seconds: | 20260507 02:49:52.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:49:52.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:52.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:52.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:52.684 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:49:52.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:52.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:52.684 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:49:52.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:52.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:52.684 | INFO | Uploaded 36GiB to GPU in 2.02246s (17.8001GiB/s) | 20260507 02:49:52.684 | INFO | Downloaded 34GiB to GPU in 1.93078s (17.6095GiB/s) | 20260507 02:49:57.824 | INFO | Analyzing chunk frames = [6505472,6505984) | 20260507 02:49:57.824 | INFO | MemInfo: { | 20260507 02:49:57.824 | INFO | "gpu" : " 17 GB", | 20260507 02:49:57.824 | INFO | "huge" : "", | 20260507 02:49:57.825 | INFO | "huge_shared" : " 27 GB", | 20260507 02:49:57.825 | INFO | "malloc" : " 14 GB", | 20260507 02:49:57.825 | INFO | "pinned" : " 29 GB" | 20260507 02:49:57.825 | INFO | } | 20260507 02:49:57.825 | INFO | Perf report for the last 5.14114 seconds: | 20260507 02:49:57.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:49:57.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:57.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:57.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:57.825 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 02:49:57.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:57.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:57.825 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:49:57.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:57.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:49:57.825 | INFO | Uploaded 36GiB to GPU in 2.00564s (17.9494GiB/s) | 20260507 02:49:57.825 | INFO | Downloaded 34GiB to GPU in 1.93773s (17.5463GiB/s) | 20260507 02:50:02.957 | INFO | Analyzing chunk frames = [6505984,6506496) | 20260507 02:50:02.957 | INFO | MemInfo: { | 20260507 02:50:02.957 | INFO | "gpu" : " 17 GB", | 20260507 02:50:02.957 | INFO | "huge" : "", | 20260507 02:50:02.958 | INFO | "huge_shared" : " 27 GB", | 20260507 02:50:02.958 | INFO | "malloc" : " 14 GB", | 20260507 02:50:02.958 | INFO | "pinned" : " 29 GB" | 20260507 02:50:02.958 | INFO | } | 20260507 02:50:02.958 | INFO | Perf report for the last 5.13305 seconds: | 20260507 02:50:02.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:50:02.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:02.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:02.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:02.958 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 02:50:02.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:02.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:02.958 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:50:02.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:02.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:02.958 | INFO | Uploaded 36GiB to GPU in 1.98738s (18.1143GiB/s) | 20260507 02:50:02.958 | INFO | Downloaded 34GiB to GPU in 1.91738s (17.7325GiB/s) | 20260507 02:50:08.102 | INFO | Analyzing chunk frames = [6506496,6507008) | 20260507 02:50:08.102 | INFO | MemInfo: { | 20260507 02:50:08.102 | INFO | "gpu" : " 17 GB", | 20260507 02:50:08.102 | INFO | "huge" : "", | 20260507 02:50:08.102 | INFO | "huge_shared" : " 27 GB", | 20260507 02:50:08.102 | INFO | "malloc" : " 14 GB", | 20260507 02:50:08.102 | INFO | "pinned" : " 29 GB" | 20260507 02:50:08.102 | INFO | } | 20260507 02:50:08.102 | INFO | Perf report for the last 5.14403 seconds: | 20260507 02:50:08.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:50:08.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:08.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:08.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:08.102 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 02:50:08.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:08.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:08.102 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:50:08.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:08.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:08.102 | INFO | Uploaded 36GiB to GPU in 2.01719s (17.8466GiB/s) | 20260507 02:50:08.102 | INFO | Downloaded 34GiB to GPU in 1.93971s (17.5284GiB/s) | 20260507 02:50:13.239 | INFO | Analyzing chunk frames = [6507008,6507520) | 20260507 02:50:13.239 | INFO | MemInfo: { | 20260507 02:50:13.239 | INFO | "gpu" : " 17 GB", | 20260507 02:50:13.239 | INFO | "huge" : "", | 20260507 02:50:13.239 | INFO | "huge_shared" : " 27 GB", | 20260507 02:50:13.239 | INFO | "malloc" : " 14 GB", | 20260507 02:50:13.239 | INFO | "pinned" : " 29 GB" | 20260507 02:50:13.239 | INFO | } | 20260507 02:50:13.239 | INFO | Perf report for the last 5.13746 seconds: | 20260507 02:50:13.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:50:13.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:13.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:13.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:13.239 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 02:50:13.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:13.239 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:13.239 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 02:50:13.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:13.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:13.239 | INFO | Uploaded 36GiB to GPU in 2.03489s (17.6914GiB/s) | 20260507 02:50:13.239 | INFO | Downloaded 34GiB to GPU in 1.92281s (17.6825GiB/s) | 20260507 02:50:18.382 | INFO | Analyzing chunk frames = [6507520,6508032) | 20260507 02:50:18.382 | INFO | MemInfo: { | 20260507 02:50:18.382 | INFO | "gpu" : " 17 GB", | 20260507 02:50:18.382 | INFO | "huge" : "", | 20260507 02:50:18.382 | INFO | "huge_shared" : " 27 GB", | 20260507 02:50:18.382 | INFO | "malloc" : " 14 GB", | 20260507 02:50:18.382 | INFO | "pinned" : " 29 GB" | 20260507 02:50:18.382 | INFO | } | 20260507 02:50:18.382 | INFO | Perf report for the last 5.14253 seconds: | 20260507 02:50:18.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:50:18.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:18.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:18.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:18.382 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 02:50:18.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:18.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:18.382 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:50:18.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:18.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:18.382 | INFO | Uploaded 36GiB to GPU in 1.98298s (18.1545GiB/s) | 20260507 02:50:18.382 | INFO | Downloaded 34GiB to GPU in 1.98567s (17.1227GiB/s) | 20260507 02:50:23.527 | INFO | Analyzing chunk frames = [6508032,6508544) | 20260507 02:50:23.527 | INFO | MemInfo: { | 20260507 02:50:23.527 | INFO | "gpu" : " 17 GB", | 20260507 02:50:23.527 | INFO | "huge" : "", | 20260507 02:50:23.527 | INFO | "huge_shared" : " 27 GB", | 20260507 02:50:23.527 | INFO | "malloc" : " 14 GB", | 20260507 02:50:23.527 | INFO | "pinned" : " 29 GB" | 20260507 02:50:23.527 | INFO | } | 20260507 02:50:23.527 | INFO | Perf report for the last 5.14525 seconds: | 20260507 02:50:23.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:50:23.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:23.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:23.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:23.527 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 02:50:23.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:23.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:23.527 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 02:50:23.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:23.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:23.527 | INFO | Uploaded 36GiB to GPU in 1.9799s (18.1828GiB/s) | 20260507 02:50:23.527 | INFO | Downloaded 34GiB to GPU in 1.97666s (17.2008GiB/s) | 20260507 02:50:23.624 | INFO | Starting a round of Baz IO | 20260507 02:50:26.724 | INFO | Populating RTMetrics report | 20260507 02:50:26.724 | INFO | RTMetrics report populated | 20260507 02:50:26.724 | INFO | Opening tmp RTMetrics json file | 20260507 02:50:26.729 | INFO | Serializing RTMetrics to json | 20260507 02:50:26.743 | INFO | Serialization done, swapping tmp file now... | 20260507 02:50:26.744 | INFO | RTMetrics done | 20260507 02:50:26.815 | INFO | Finished a round of Baz IO: Wrote 1.769 GiB in 3.191 seconds | 20260507 02:50:26.816 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.96% / 0.00% / 67.04% | 20260507 02:50:26.816 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.22155% / 3.97364e-06% / 7.15256e-05% | 20260507 02:50:26.816 | INFO | Baz IO overall: | 20260507 02:50:26.816 | INFO | BytesWritten : 3352043001437 | 20260507 02:50:26.816 | INFO | NumEvents : 1611528996831 | 20260507 02:50:28.659 | INFO | Analyzing chunk frames = [6508544,6509056) | 20260507 02:50:28.659 | INFO | MemInfo: { | 20260507 02:50:28.659 | INFO | "gpu" : " 17 GB", | 20260507 02:50:28.659 | INFO | "huge" : "", | 20260507 02:50:28.659 | INFO | "huge_shared" : " 27 GB", | 20260507 02:50:28.659 | INFO | "malloc" : " 14 GB", | 20260507 02:50:28.659 | INFO | "pinned" : " 29 GB" | 20260507 02:50:28.659 | INFO | } | 20260507 02:50:28.659 | INFO | Perf report for the last 5.13229 seconds: | 20260507 02:50:28.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:50:28.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:28.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:28.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:28.659 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 02:50:28.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:28.659 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 02:50:28.659 | INFO | PRE_HQ : 59.4%, 1.8, 55.6ms, 0 | 20260507 02:50:28.659 | INFO | BAZWRITER : 61.8%, 3.3, 103.9ms, 0 | 20260507 02:50:28.660 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:50:28.660 | INFO | Uploaded 36GiB to GPU in 2.0363s (17.6792GiB/s) | 20260507 02:50:28.660 | INFO | Downloaded 38GiB to GPU in 2.17571s (17.4656GiB/s) | 20260507 02:50:33.799 | INFO | Analyzing chunk frames = [6509056,6509568) | 20260507 02:50:33.799 | INFO | MemInfo: { | 20260507 02:50:33.799 | INFO | "gpu" : " 17 GB", | 20260507 02:50:33.799 | INFO | "huge" : "", | 20260507 02:50:33.799 | INFO | "huge_shared" : " 27 GB", | 20260507 02:50:33.799 | INFO | "malloc" : " 14 GB", | 20260507 02:50:33.799 | INFO | "pinned" : " 29 GB" | 20260507 02:50:33.799 | INFO | } | 20260507 02:50:33.799 | INFO | Perf report for the last 5.13981 seconds: | 20260507 02:50:33.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:50:33.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:33.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:33.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:33.799 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:50:33.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:33.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:33.799 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:50:33.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:33.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:33.800 | INFO | Uploaded 36GiB to GPU in 2.03645s (17.6778GiB/s) | 20260507 02:50:33.800 | INFO | Downloaded 34GiB to GPU in 1.94187s (17.5089GiB/s) | 20260507 02:50:38.932 | INFO | Analyzing chunk frames = [6509568,6510080) | 20260507 02:50:38.932 | INFO | MemInfo: { | 20260507 02:50:38.932 | INFO | "gpu" : " 17 GB", | 20260507 02:50:38.932 | INFO | "huge" : "", | 20260507 02:50:38.932 | INFO | "huge_shared" : " 27 GB", | 20260507 02:50:38.932 | INFO | "malloc" : " 14 GB", | 20260507 02:50:38.932 | INFO | "pinned" : " 29 GB" | 20260507 02:50:38.932 | INFO | } | 20260507 02:50:38.932 | INFO | Perf report for the last 5.13261 seconds: | 20260507 02:50:38.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:50:38.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:38.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:38.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:38.932 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 02:50:38.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:38.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:38.932 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:50:38.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:38.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:38.932 | INFO | Uploaded 36GiB to GPU in 1.99532s (18.0422GiB/s) | 20260507 02:50:38.932 | INFO | Downloaded 34GiB to GPU in 1.94067s (17.5197GiB/s) | 20260507 02:50:44.076 | INFO | Analyzing chunk frames = [6510080,6510592) | 20260507 02:50:44.076 | INFO | MemInfo: { | 20260507 02:50:44.076 | INFO | "gpu" : " 17 GB", | 20260507 02:50:44.076 | INFO | "huge" : "", | 20260507 02:50:44.076 | INFO | "huge_shared" : " 27 GB", | 20260507 02:50:44.076 | INFO | "malloc" : " 14 GB", | 20260507 02:50:44.077 | INFO | "pinned" : " 29 GB" | 20260507 02:50:44.077 | INFO | } | 20260507 02:50:44.077 | INFO | Perf report for the last 5.14476 seconds: | 20260507 02:50:44.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:50:44.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:44.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:44.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:44.077 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 02:50:44.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:44.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:44.077 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 02:50:44.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:44.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:44.077 | INFO | Uploaded 36GiB to GPU in 2.0066s (17.9408GiB/s) | 20260507 02:50:44.077 | INFO | Downloaded 34GiB to GPU in 1.91039s (17.7974GiB/s) | 20260507 02:50:49.226 | INFO | Analyzing chunk frames = [6510592,6511104) | 20260507 02:50:49.226 | INFO | MemInfo: { | 20260507 02:50:49.226 | INFO | "gpu" : " 17 GB", | 20260507 02:50:49.226 | INFO | "huge" : "", | 20260507 02:50:49.226 | INFO | "huge_shared" : " 27 GB", | 20260507 02:50:49.226 | INFO | "malloc" : " 14 GB", | 20260507 02:50:49.226 | INFO | "pinned" : " 29 GB" | 20260507 02:50:49.226 | INFO | } | 20260507 02:50:49.226 | INFO | Perf report for the last 5.1492 seconds: | 20260507 02:50:49.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:50:49.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:49.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:49.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:49.226 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 02:50:49.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:49.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:49.226 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:50:49.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:49.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:49.226 | INFO | Uploaded 36GiB to GPU in 2.00496s (17.9555GiB/s) | 20260507 02:50:49.226 | INFO | Downloaded 34GiB to GPU in 1.9453s (17.478GiB/s) | 20260507 02:50:54.361 | INFO | Analyzing chunk frames = [6511104,6511616) | 20260507 02:50:54.361 | INFO | MemInfo: { | 20260507 02:50:54.361 | INFO | "gpu" : " 17 GB", | 20260507 02:50:54.361 | INFO | "huge" : "", | 20260507 02:50:54.361 | INFO | "huge_shared" : " 27 GB", | 20260507 02:50:54.361 | INFO | "malloc" : " 14 GB", | 20260507 02:50:54.361 | INFO | "pinned" : " 29 GB" | 20260507 02:50:54.361 | INFO | } | 20260507 02:50:54.361 | INFO | Perf report for the last 5.13563 seconds: | 20260507 02:50:54.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:50:54.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:54.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:54.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:54.362 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 02:50:54.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:54.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:54.362 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:50:54.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:54.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:54.362 | INFO | Uploaded 36GiB to GPU in 2.02494s (17.7783GiB/s) | 20260507 02:50:54.362 | INFO | Downloaded 34GiB to GPU in 1.92025s (17.7061GiB/s) | 20260507 02:50:59.488 | INFO | Analyzing chunk frames = [6511616,6512128) | 20260507 02:50:59.488 | INFO | MemInfo: { | 20260507 02:50:59.488 | INFO | "gpu" : " 17 GB", | 20260507 02:50:59.488 | INFO | "huge" : "", | 20260507 02:50:59.488 | INFO | "huge_shared" : " 27 GB", | 20260507 02:50:59.488 | INFO | "malloc" : " 14 GB", | 20260507 02:50:59.488 | INFO | "pinned" : " 29 GB" | 20260507 02:50:59.488 | INFO | } | 20260507 02:50:59.488 | INFO | Perf report for the last 5.127 seconds: | 20260507 02:50:59.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:50:59.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:59.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:59.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:59.489 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 02:50:59.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:59.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:59.489 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 02:50:59.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:59.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:50:59.489 | INFO | Uploaded 36GiB to GPU in 1.98307s (18.1537GiB/s) | 20260507 02:50:59.489 | INFO | Downloaded 34GiB to GPU in 1.9684s (17.2729GiB/s) | 20260507 02:51:04.633 | INFO | Analyzing chunk frames = [6512128,6512640) | 20260507 02:51:04.633 | INFO | MemInfo: { | 20260507 02:51:04.633 | INFO | "gpu" : " 17 GB", | 20260507 02:51:04.633 | INFO | "huge" : "", | 20260507 02:51:04.633 | INFO | "huge_shared" : " 27 GB", | 20260507 02:51:04.633 | INFO | "malloc" : " 14 GB", | 20260507 02:51:04.633 | INFO | "pinned" : " 29 GB" | 20260507 02:51:04.633 | INFO | } | 20260507 02:51:04.634 | INFO | Perf report for the last 5.14505 seconds: | 20260507 02:51:04.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:51:04.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:04.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:04.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:04.634 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:51:04.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:04.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:04.634 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:51:04.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:04.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:04.634 | INFO | Uploaded 36GiB to GPU in 2.02415s (17.7852GiB/s) | 20260507 02:51:04.634 | INFO | Downloaded 34GiB to GPU in 1.9619s (17.3301GiB/s) | 20260507 02:51:04.634 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:51:04.634 | INFO | Total mem usage: 18 GiB | 20260507 02:51:04.634 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:51:04.634 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:51:04.634 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:51:04.634 | INFO | High Water Breakdown: | 20260507 02:51:04.634 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:51:04.634 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:51:04.634 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:51:04.634 | INFO | ------------------------------------------ | 20260507 02:51:04.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:51:04.634 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:51:04.634 | INFO | Total mem usage: 26.5 GiB | 20260507 02:51:04.634 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:51:04.634 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:51:04.634 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:51:04.634 | INFO | High Water Breakdown: | 20260507 02:51:04.634 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:51:04.634 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:51:04.634 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:51:04.634 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:51:04.634 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:51:04.634 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:51:04.634 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:51:04.634 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:51:04.634 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:51:04.634 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:51:04.634 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:51:04.634 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:51:04.634 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:51:04.634 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:51:04.634 | INFO | ------------------------------------------ | 20260507 02:51:04.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:51:04.634 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:51:04.634 | INFO | Total mem usage: 25.6 GiB | 20260507 02:51:04.634 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:51:04.634 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 02:51:04.634 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 02:51:04.634 | INFO | High Water Breakdown: | 20260507 02:51:04.634 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 02:51:04.634 | INFO | ------------------------------------------ | 20260507 02:51:04.634 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:51:04.634 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:51:04.634 | INFO | Total mem usage: 15.7 GiB | 20260507 02:51:04.634 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:51:04.634 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:51:04.634 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:51:04.634 | INFO | High Water Breakdown: | 20260507 02:51:04.634 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:51:04.634 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:51:04.634 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:51:04.634 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:51:04.634 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:51:04.634 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:51:04.634 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:51:04.634 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:51:04.634 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:51:04.634 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:51:04.635 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:51:04.635 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:51:04.635 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:51:04.635 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:51:04.635 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:51:04.635 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:51:04.635 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:51:04.635 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:51:04.635 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:51:04.635 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:51:04.635 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:51:04.635 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:51:04.635 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:51:04.635 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:51:04.635 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:51:04.635 | INFO | ------------------------------------------ | 20260507 02:51:04.635 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:51:04.635 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:51:04.635 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:51:04.635 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:51:04.635 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.53e+01, 1.34e+01 / 1.81e+00 | 20260507 02:51:04.635 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.02e-01, 9.81e-03 / 7.34e-03 | 20260507 02:51:04.635 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.46e+00, 5.20e-01 / 2.58e-01 | 20260507 02:51:04.635 | INFO | PerfReport: 3.23%, Histogram -- 9.94e-01 / 1.46e+00, 1.69e-01 / 1.20e-01 | 20260507 02:51:04.635 | INFO | PerfReport: 16.08%, DME -- 3.96e+01 / 4.45e+01, 3.84e+00 / 1.88e+00 | 20260507 02:51:04.635 | INFO | PerfReport: 57.67%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.38e+00 / 1.65e+00 | 20260507 02:51:04.635 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.15e+00 / 2.68e+00, 3.79e-01 / 2.26e-01 | 20260507 02:51:04.635 | INFO | PerfReport: 3.06%, Metrics -- 9.44e-01 / 1.26e+00, 2.54e-01 / 1.61e-01 | 20260507 02:51:04.635 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.19e-01 / 3.74e-01, 5.16e-02 / 2.53e-02 | 20260507 02:51:04.635 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:51:04.635 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:51:04.635 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:51:04.635 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 02:51:04.635 | INFO | PerfReport: 21.77%, Upload -- 2.10e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 02:51:04.635 | INFO | PerfReport: 21.34%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260507 02:51:04.635 | INFO | PerfReport: 32.00%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 02:51:04.635 | INFO | PerfReport: 1.35%, LockUpload -- 1.30e+00 / 0.00e+00, 4.62e+00 / 0.00e+00 | 20260507 02:51:04.635 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260507 02:51:04.635 | INFO | PerfReport: 22.21%, LockCompute -- 2.14e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 02:51:04.635 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260507 02:51:04.732 | INFO | Starting a round of Baz IO | 20260507 02:51:07.842 | INFO | Populating RTMetrics report | 20260507 02:51:07.842 | INFO | RTMetrics report populated | 20260507 02:51:07.842 | INFO | Opening tmp RTMetrics json file | 20260507 02:51:07.861 | INFO | Serializing RTMetrics to json | 20260507 02:51:07.862 | INFO | Serialization done, swapping tmp file now... | 20260507 02:51:07.862 | INFO | RTMetrics done | 20260507 02:51:07.933 | INFO | Finished a round of Baz IO: Wrote 1.77 GiB in 3.2 seconds | 20260507 02:51:07.933 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.96% / 0.00% / 67.04% | 20260507 02:51:07.933 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.222003% / 3.97364e-06% / 8.74201e-05% | 20260507 02:51:07.933 | INFO | Baz IO overall: | 20260507 02:51:07.933 | INFO | BytesWritten : 3353943155366 | 20260507 02:51:07.933 | INFO | NumEvents : 1612426884098 | 20260507 02:51:09.773 | INFO | Analyzing chunk frames = [6512640,6513152) | 20260507 02:51:09.773 | INFO | MemInfo: { | 20260507 02:51:09.773 | INFO | "gpu" : " 17 GB", | 20260507 02:51:09.773 | INFO | "huge" : "", | 20260507 02:51:09.773 | INFO | "huge_shared" : " 27 GB", | 20260507 02:51:09.773 | INFO | "malloc" : " 14 GB", | 20260507 02:51:09.773 | INFO | "pinned" : " 29 GB" | 20260507 02:51:09.773 | INFO | } | 20260507 02:51:09.773 | INFO | Perf report for the last 5.13963 seconds: | 20260507 02:51:09.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:51:09.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:09.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:09.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:09.773 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260507 02:51:09.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:09.773 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 02:51:09.773 | INFO | PRE_HQ : 59.3%, 1.8, 55.5ms, 0 | 20260507 02:51:09.773 | INFO | BAZWRITER : 61.9%, 3.3, 105.5ms, 0 | 20260507 02:51:09.773 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:51:09.773 | INFO | Uploaded 36GiB to GPU in 2.04134s (17.6355GiB/s) | 20260507 02:51:09.773 | INFO | Downloaded 38GiB to GPU in 2.19354s (17.3236GiB/s) | 20260507 02:51:14.910 | INFO | Analyzing chunk frames = [6513152,6513664) | 20260507 02:51:14.910 | INFO | MemInfo: { | 20260507 02:51:14.910 | INFO | "gpu" : " 17 GB", | 20260507 02:51:14.910 | INFO | "huge" : "", | 20260507 02:51:14.910 | INFO | "huge_shared" : " 27 GB", | 20260507 02:51:14.910 | INFO | "malloc" : " 14 GB", | 20260507 02:51:14.910 | INFO | "pinned" : " 29 GB" | 20260507 02:51:14.910 | INFO | } | 20260507 02:51:14.910 | INFO | Perf report for the last 5.13687 seconds: | 20260507 02:51:14.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:51:14.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:14.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:14.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:14.910 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:51:14.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:14.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:14.910 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 02:51:14.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:14.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:14.911 | INFO | Uploaded 36GiB to GPU in 2.0281s (17.7506GiB/s) | 20260507 02:51:14.911 | INFO | Downloaded 34GiB to GPU in 1.94098s (17.5169GiB/s) | 20260507 02:51:20.051 | INFO | Analyzing chunk frames = [6513664,6514176) | 20260507 02:51:20.051 | INFO | MemInfo: { | 20260507 02:51:20.051 | INFO | "gpu" : " 17 GB", | 20260507 02:51:20.051 | INFO | "huge" : "", | 20260507 02:51:20.051 | INFO | "huge_shared" : " 27 GB", | 20260507 02:51:20.051 | INFO | "malloc" : " 14 GB", | 20260507 02:51:20.051 | INFO | "pinned" : " 29 GB" | 20260507 02:51:20.051 | INFO | } | 20260507 02:51:20.051 | INFO | Perf report for the last 5.14101 seconds: | 20260507 02:51:20.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:51:20.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:20.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:20.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:20.051 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 02:51:20.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:20.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:20.051 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:51:20.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:20.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:20.051 | INFO | Uploaded 36GiB to GPU in 1.99562s (18.0395GiB/s) | 20260507 02:51:20.051 | INFO | Downloaded 34GiB to GPU in 1.93818s (17.5423GiB/s) | 20260507 02:51:25.191 | INFO | Analyzing chunk frames = [6514176,6514688) | 20260507 02:51:25.191 | INFO | MemInfo: { | 20260507 02:51:25.191 | INFO | "gpu" : " 17 GB", | 20260507 02:51:25.191 | INFO | "huge" : "", | 20260507 02:51:25.191 | INFO | "huge_shared" : " 27 GB", | 20260507 02:51:25.191 | INFO | "malloc" : " 14 GB", | 20260507 02:51:25.191 | INFO | "pinned" : " 29 GB" | 20260507 02:51:25.191 | INFO | } | 20260507 02:51:25.191 | INFO | Perf report for the last 5.13999 seconds: | 20260507 02:51:25.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:51:25.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:25.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:25.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:25.191 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 02:51:25.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:25.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:25.191 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:51:25.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:25.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:25.191 | INFO | Uploaded 36GiB to GPU in 1.96875s (18.2857GiB/s) | 20260507 02:51:25.191 | INFO | Downloaded 34GiB to GPU in 1.9094s (17.8067GiB/s) | 20260507 02:51:30.335 | INFO | Analyzing chunk frames = [6514688,6515200) | 20260507 02:51:30.335 | INFO | MemInfo: { | 20260507 02:51:30.335 | INFO | "gpu" : " 17 GB", | 20260507 02:51:30.335 | INFO | "huge" : "", | 20260507 02:51:30.335 | INFO | "huge_shared" : " 27 GB", | 20260507 02:51:30.335 | INFO | "malloc" : " 14 GB", | 20260507 02:51:30.335 | INFO | "pinned" : " 29 GB" | 20260507 02:51:30.335 | INFO | } | 20260507 02:51:30.335 | INFO | Perf report for the last 5.14369 seconds: | 20260507 02:51:30.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:51:30.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:30.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:30.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:30.335 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 02:51:30.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:30.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:30.335 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:51:30.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:30.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:30.335 | INFO | Uploaded 36GiB to GPU in 1.99424s (18.052GiB/s) | 20260507 02:51:30.335 | INFO | Downloaded 34GiB to GPU in 1.94492s (17.4815GiB/s) | 20260507 02:51:35.474 | INFO | Analyzing chunk frames = [6515200,6515712) | 20260507 02:51:35.474 | INFO | MemInfo: { | 20260507 02:51:35.474 | INFO | "gpu" : " 17 GB", | 20260507 02:51:35.474 | INFO | "huge" : "", | 20260507 02:51:35.474 | INFO | "huge_shared" : " 27 GB", | 20260507 02:51:35.474 | INFO | "malloc" : " 14 GB", | 20260507 02:51:35.474 | INFO | "pinned" : " 29 GB" | 20260507 02:51:35.474 | INFO | } | 20260507 02:51:35.474 | INFO | Perf report for the last 5.13961 seconds: | 20260507 02:51:35.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:51:35.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:35.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:35.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:35.474 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 02:51:35.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:35.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:35.475 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 02:51:35.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:35.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:35.475 | INFO | Uploaded 36GiB to GPU in 2.03356s (17.703GiB/s) | 20260507 02:51:35.475 | INFO | Downloaded 34GiB to GPU in 1.91926s (17.7152GiB/s) | 20260507 02:51:40.607 | INFO | Analyzing chunk frames = [6515712,6516224) | 20260507 02:51:40.607 | INFO | MemInfo: { | 20260507 02:51:40.607 | INFO | "gpu" : " 17 GB", | 20260507 02:51:40.607 | INFO | "huge" : "", | 20260507 02:51:40.607 | INFO | "huge_shared" : " 27 GB", | 20260507 02:51:40.607 | INFO | "malloc" : " 14 GB", | 20260507 02:51:40.607 | INFO | "pinned" : " 29 GB" | 20260507 02:51:40.607 | INFO | } | 20260507 02:51:40.607 | INFO | Perf report for the last 5.13259 seconds: | 20260507 02:51:40.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:51:40.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:40.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:40.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:40.607 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 02:51:40.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:40.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:40.607 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:51:40.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:40.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:40.607 | INFO | Uploaded 36GiB to GPU in 1.96806s (18.2921GiB/s) | 20260507 02:51:40.607 | INFO | Downloaded 34GiB to GPU in 1.97132s (17.2473GiB/s) | 20260507 02:51:45.747 | INFO | Analyzing chunk frames = [6516224,6516736) | 20260507 02:51:45.747 | INFO | MemInfo: { | 20260507 02:51:45.747 | INFO | "gpu" : " 17 GB", | 20260507 02:51:45.747 | INFO | "huge" : "", | 20260507 02:51:45.747 | INFO | "huge_shared" : " 27 GB", | 20260507 02:51:45.747 | INFO | "malloc" : " 14 GB", | 20260507 02:51:45.747 | INFO | "pinned" : " 29 GB" | 20260507 02:51:45.747 | INFO | } | 20260507 02:51:45.747 | INFO | Perf report for the last 5.14044 seconds: | 20260507 02:51:45.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:51:45.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:45.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:45.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:45.748 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:51:45.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:45.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:45.748 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 02:51:45.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:45.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:45.748 | INFO | Uploaded 36GiB to GPU in 1.9795s (18.1864GiB/s) | 20260507 02:51:45.748 | INFO | Downloaded 34GiB to GPU in 1.97952s (17.1759GiB/s) | 20260507 02:51:45.844 | INFO | Starting a round of Baz IO | 20260507 02:51:48.964 | INFO | Populating RTMetrics report | 20260507 02:51:48.964 | INFO | RTMetrics report populated | 20260507 02:51:48.964 | INFO | Opening tmp RTMetrics json file | 20260507 02:51:48.970 | INFO | Serializing RTMetrics to json | 20260507 02:51:48.983 | INFO | Serialization done, swapping tmp file now... | 20260507 02:51:48.983 | INFO | RTMetrics done | 20260507 02:51:49.055 | INFO | Finished a round of Baz IO: Wrote 1.769 GiB in 3.211 seconds | 20260507 02:51:49.055 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.96% / 0.00% / 67.04% | 20260507 02:51:49.055 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.221654% / 0% / 0.000107288% | 20260507 02:51:49.055 | INFO | Baz IO overall: | 20260507 02:51:49.055 | INFO | BytesWritten : 3355842811928 | 20260507 02:51:49.056 | INFO | NumEvents : 1613324364774 | 20260507 02:51:50.892 | INFO | Analyzing chunk frames = [6516736,6517248) | 20260507 02:51:50.893 | INFO | MemInfo: { | 20260507 02:51:50.893 | INFO | "gpu" : " 17 GB", | 20260507 02:51:50.893 | INFO | "huge" : "", | 20260507 02:51:50.893 | INFO | "huge_shared" : " 27 GB", | 20260507 02:51:50.893 | INFO | "malloc" : " 14 GB", | 20260507 02:51:50.893 | INFO | "pinned" : " 29 GB" | 20260507 02:51:50.893 | INFO | } | 20260507 02:51:50.893 | INFO | Perf report for the last 5.14522 seconds: | 20260507 02:51:50.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:51:50.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:50.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:50.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:50.893 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 02:51:50.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:50.893 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 02:51:50.893 | INFO | PRE_HQ : 59.3%, 1.8, 55.6ms, 0 | 20260507 02:51:50.893 | INFO | BAZWRITER : 62.0%, 3.3, 104.9ms, 0 | 20260507 02:51:50.893 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:51:50.893 | INFO | Uploaded 36GiB to GPU in 2.03148s (17.721GiB/s) | 20260507 02:51:50.893 | INFO | Downloaded 38GiB to GPU in 2.17587s (17.4643GiB/s) | 20260507 02:51:56.030 | INFO | Analyzing chunk frames = [6517248,6517760) | 20260507 02:51:56.030 | INFO | MemInfo: { | 20260507 02:51:56.030 | INFO | "gpu" : " 17 GB", | 20260507 02:51:56.030 | INFO | "huge" : "", | 20260507 02:51:56.030 | INFO | "huge_shared" : " 27 GB", | 20260507 02:51:56.030 | INFO | "malloc" : " 14 GB", | 20260507 02:51:56.030 | INFO | "pinned" : " 29 GB" | 20260507 02:51:56.031 | INFO | } | 20260507 02:51:56.031 | INFO | Perf report for the last 5.13802 seconds: | 20260507 02:51:56.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:51:56.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:56.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:56.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:56.031 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:51:56.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:56.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:56.031 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:51:56.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:56.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:51:56.031 | INFO | Uploaded 36GiB to GPU in 2.02874s (17.745GiB/s) | 20260507 02:51:56.031 | INFO | Downloaded 34GiB to GPU in 1.93887s (17.536GiB/s) | 20260507 02:52:01.169 | INFO | Analyzing chunk frames = [6517760,6518272) | 20260507 02:52:01.169 | INFO | MemInfo: { | 20260507 02:52:01.169 | INFO | "gpu" : " 17 GB", | 20260507 02:52:01.169 | INFO | "huge" : "", | 20260507 02:52:01.169 | INFO | "huge_shared" : " 27 GB", | 20260507 02:52:01.169 | INFO | "malloc" : " 14 GB", | 20260507 02:52:01.169 | INFO | "pinned" : " 29 GB" | 20260507 02:52:01.169 | INFO | } | 20260507 02:52:01.169 | INFO | Perf report for the last 5.13826 seconds: | 20260507 02:52:01.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:52:01.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:01.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:01.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:01.169 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 02:52:01.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:01.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:01.169 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:52:01.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:01.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:01.169 | INFO | Uploaded 36GiB to GPU in 1.98198s (18.1636GiB/s) | 20260507 02:52:01.169 | INFO | Downloaded 34GiB to GPU in 1.94191s (17.5085GiB/s) | 20260507 02:52:06.302 | INFO | Analyzing chunk frames = [6518272,6518784) | 20260507 02:52:06.302 | INFO | MemInfo: { | 20260507 02:52:06.302 | INFO | "gpu" : " 17 GB", | 20260507 02:52:06.302 | INFO | "huge" : "", | 20260507 02:52:06.302 | INFO | "huge_shared" : " 27 GB", | 20260507 02:52:06.302 | INFO | "malloc" : " 14 GB", | 20260507 02:52:06.302 | INFO | "pinned" : " 29 GB" | 20260507 02:52:06.302 | INFO | } | 20260507 02:52:06.302 | INFO | Perf report for the last 5.13331 seconds: | 20260507 02:52:06.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:52:06.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:06.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:06.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:06.302 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 02:52:06.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:06.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:06.302 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:52:06.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:06.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:06.303 | INFO | Uploaded 36GiB to GPU in 1.9864s (18.1232GiB/s) | 20260507 02:52:06.303 | INFO | Downloaded 34GiB to GPU in 1.93073s (17.6099GiB/s) | 20260507 02:52:11.446 | INFO | Analyzing chunk frames = [6518784,6519296) | 20260507 02:52:11.446 | INFO | MemInfo: { | 20260507 02:52:11.446 | INFO | "gpu" : " 17 GB", | 20260507 02:52:11.446 | INFO | "huge" : "", | 20260507 02:52:11.446 | INFO | "huge_shared" : " 27 GB", | 20260507 02:52:11.446 | INFO | "malloc" : " 14 GB", | 20260507 02:52:11.446 | INFO | "pinned" : " 29 GB" | 20260507 02:52:11.446 | INFO | } | 20260507 02:52:11.446 | INFO | Perf report for the last 5.14334 seconds: | 20260507 02:52:11.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:52:11.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:11.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:11.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:11.446 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 02:52:11.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:11.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:11.446 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:52:11.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:11.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:11.446 | INFO | Uploaded 36GiB to GPU in 2.01255s (17.8877GiB/s) | 20260507 02:52:11.446 | INFO | Downloaded 34GiB to GPU in 1.94887s (17.446GiB/s) | 20260507 02:52:16.588 | INFO | Analyzing chunk frames = [6519296,6519808) | 20260507 02:52:16.588 | INFO | MemInfo: { | 20260507 02:52:16.588 | INFO | "gpu" : " 17 GB", | 20260507 02:52:16.588 | INFO | "huge" : "", | 20260507 02:52:16.589 | INFO | "huge_shared" : " 27 GB", | 20260507 02:52:16.589 | INFO | "malloc" : " 14 GB", | 20260507 02:52:16.589 | INFO | "pinned" : " 29 GB" | 20260507 02:52:16.589 | INFO | } | 20260507 02:52:16.589 | INFO | Perf report for the last 5.14306 seconds: | 20260507 02:52:16.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:52:16.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:16.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:16.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:16.589 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 02:52:16.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:16.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:16.589 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:52:16.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:16.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:16.589 | INFO | Uploaded 36GiB to GPU in 2.03933s (17.6528GiB/s) | 20260507 02:52:16.589 | INFO | Downloaded 34GiB to GPU in 1.95061s (17.4304GiB/s) | 20260507 02:52:21.728 | INFO | Analyzing chunk frames = [6519808,6520320) | 20260507 02:52:21.728 | INFO | MemInfo: { | 20260507 02:52:21.728 | INFO | "gpu" : " 17 GB", | 20260507 02:52:21.728 | INFO | "huge" : "", | 20260507 02:52:21.728 | INFO | "huge_shared" : " 27 GB", | 20260507 02:52:21.728 | INFO | "malloc" : " 14 GB", | 20260507 02:52:21.728 | INFO | "pinned" : " 29 GB" | 20260507 02:52:21.728 | INFO | } | 20260507 02:52:21.728 | INFO | Perf report for the last 5.13929 seconds: | 20260507 02:52:21.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:52:21.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:21.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:21.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:21.728 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:52:21.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:21.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:21.728 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:52:21.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:21.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:21.729 | INFO | Uploaded 36GiB to GPU in 1.96724s (18.2997GiB/s) | 20260507 02:52:21.729 | INFO | Downloaded 34GiB to GPU in 1.96231s (17.3265GiB/s) | 20260507 02:52:26.865 | INFO | Analyzing chunk frames = [6520320,6520832) | 20260507 02:52:26.865 | INFO | MemInfo: { | 20260507 02:52:26.865 | INFO | "gpu" : " 17 GB", | 20260507 02:52:26.865 | INFO | "huge" : "", | 20260507 02:52:26.865 | INFO | "huge_shared" : " 27 GB", | 20260507 02:52:26.865 | INFO | "malloc" : " 14 GB", | 20260507 02:52:26.865 | INFO | "pinned" : " 29 GB" | 20260507 02:52:26.865 | INFO | } | 20260507 02:52:26.865 | INFO | Perf report for the last 5.1373 seconds: | 20260507 02:52:26.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:52:26.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:26.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:26.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:26.865 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:52:26.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:26.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:26.865 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 02:52:26.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:26.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:26.866 | INFO | Uploaded 36GiB to GPU in 1.99945s (18.0049GiB/s) | 20260507 02:52:26.866 | INFO | Downloaded 34GiB to GPU in 1.96968s (17.2617GiB/s) | 20260507 02:52:26.866 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:52:26.866 | INFO | Total mem usage: 18 GiB | 20260507 02:52:26.866 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:52:26.866 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 02:52:26.866 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:52:26.866 | INFO | High Water Breakdown: | 20260507 02:52:26.866 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:52:26.866 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:52:26.866 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:52:26.866 | INFO | ------------------------------------------ | 20260507 02:52:26.866 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:52:26.866 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:52:26.866 | INFO | Total mem usage: 26.5 GiB | 20260507 02:52:26.866 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:52:26.866 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:52:26.866 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:52:26.866 | INFO | High Water Breakdown: | 20260507 02:52:26.866 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:52:26.866 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:52:26.866 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:52:26.866 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:52:26.866 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:52:26.866 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:52:26.866 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:52:26.866 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:52:26.866 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:52:26.866 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:52:26.866 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:52:26.866 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:52:26.866 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:52:26.866 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:52:26.866 | INFO | ------------------------------------------ | 20260507 02:52:26.866 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:52:26.866 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:52:26.866 | INFO | Total mem usage: 25.6 GiB | 20260507 02:52:26.866 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:52:26.866 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 02:52:26.866 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 02:52:26.866 | INFO | High Water Breakdown: | 20260507 02:52:26.866 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 02:52:26.866 | INFO | ------------------------------------------ | 20260507 02:52:26.866 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:52:26.866 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:52:26.866 | INFO | Total mem usage: 15.7 GiB | 20260507 02:52:26.866 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:52:26.866 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:52:26.866 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 02:52:26.866 | INFO | High Water Breakdown: | 20260507 02:52:26.866 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:52:26.866 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:52:26.866 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:52:26.866 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:52:26.866 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:52:26.866 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:52:26.866 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:52:26.866 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:52:26.866 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:52:26.866 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:52:26.866 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:52:26.866 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:52:26.866 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:52:26.866 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:52:26.866 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:52:26.866 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:52:26.866 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:52:26.866 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:52:26.866 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:52:26.866 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:52:26.866 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:52:26.866 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:52:26.866 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:52:26.866 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:52:26.866 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:52:26.867 | INFO | ------------------------------------------ | 20260507 02:52:26.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:52:26.867 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:52:26.867 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:52:26.867 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:52:26.867 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.52e+01, 1.35e+01 / 1.92e+00 | 20260507 02:52:26.867 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.84e-03 / 8.50e-03 | 20260507 02:52:26.867 | INFO | PerfReport: 11.68%, Baseline -- 3.60e+00 / 4.42e+00, 5.23e-01 / 3.06e-01 | 20260507 02:52:26.867 | INFO | PerfReport: 3.24%, Histogram -- 9.98e-01 / 1.47e+00, 1.71e-01 / 1.22e-01 | 20260507 02:52:26.867 | INFO | PerfReport: 16.11%, DME -- 3.97e+01 / 4.47e+01, 4.03e+00 / 2.20e+00 | 20260507 02:52:26.867 | INFO | PerfReport: 57.59%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.33e+00 / 1.58e+00 | 20260507 02:52:26.867 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.15e+00 / 2.71e+00, 3.81e-01 / 2.23e-01 | 20260507 02:52:26.867 | INFO | PerfReport: 3.06%, Metrics -- 9.42e-01 / 1.27e+00, 2.47e-01 / 1.20e-01 | 20260507 02:52:26.867 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.20e-01 / 3.79e-01, 5.10e-02 / 2.71e-02 | 20260507 02:52:26.867 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:52:26.867 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:52:26.867 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:52:26.867 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 02:52:26.867 | INFO | PerfReport: 21.67%, Upload -- 2.09e+01 / 0.00e+00, 6.98e+00 / 0.00e+00 | 20260507 02:52:26.867 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260507 02:52:26.867 | INFO | PerfReport: 31.98%, Compute -- 3.08e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 02:52:26.867 | INFO | PerfReport: 1.33%, LockUpload -- 1.28e+00 / 0.00e+00, 4.72e+00 / 0.00e+00 | 20260507 02:52:26.867 | INFO | PerfReport: 1.41%, LockDownload -- 1.35e+00 / 0.00e+00, 3.22e+00 / 0.00e+00 | 20260507 02:52:26.867 | INFO | PerfReport: 22.22%, LockCompute -- 2.14e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 02:52:26.867 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260507 02:52:26.962 | INFO | Starting a round of Baz IO | 20260507 02:52:30.072 | INFO | Populating RTMetrics report | 20260507 02:52:30.073 | INFO | RTMetrics report populated | 20260507 02:52:30.073 | INFO | Opening tmp RTMetrics json file | 20260507 02:52:30.092 | INFO | Serializing RTMetrics to json | 20260507 02:52:30.093 | INFO | Serialization done, swapping tmp file now... | 20260507 02:52:30.094 | INFO | RTMetrics done | 20260507 02:52:30.164 | INFO | Finished a round of Baz IO: Wrote 1.771 GiB in 3.201 seconds | 20260507 02:52:30.164 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.95% / 0.00% / 67.04% | 20260507 02:52:30.164 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.222588% / 0% / 9.93411e-05% | 20260507 02:52:30.164 | INFO | Baz IO overall: | 20260507 02:52:30.164 | INFO | BytesWritten : 3357744105399 | 20260507 02:52:30.164 | INFO | NumEvents : 1614223168080 | 20260507 02:52:32.005 | INFO | Analyzing chunk frames = [6520832,6521344) | 20260507 02:52:32.005 | INFO | MemInfo: { | 20260507 02:52:32.005 | INFO | "gpu" : " 17 GB", | 20260507 02:52:32.005 | INFO | "huge" : "", | 20260507 02:52:32.005 | INFO | "huge_shared" : " 27 GB", | 20260507 02:52:32.005 | INFO | "malloc" : " 14 GB", | 20260507 02:52:32.005 | INFO | "pinned" : " 29 GB" | 20260507 02:52:32.005 | INFO | } | 20260507 02:52:32.005 | INFO | Perf report for the last 5.13964 seconds: | 20260507 02:52:32.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:52:32.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:32.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:32.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:32.005 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 02:52:32.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:32.005 | INFO | RT_METRICS : 44.2%, 1.0, 23.2ms, 0 | 20260507 02:52:32.005 | INFO | PRE_HQ : 59.8%, 1.9, 58.8ms, 0 | 20260507 02:52:32.005 | INFO | BAZWRITER : 61.9%, 3.4, 108.6ms, 0 | 20260507 02:52:32.005 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:52:32.005 | INFO | Uploaded 36GiB to GPU in 2.03408s (17.6984GiB/s) | 20260507 02:52:32.005 | INFO | Downloaded 38GiB to GPU in 2.17866s (17.4419GiB/s) | 20260507 02:52:37.147 | INFO | Analyzing chunk frames = [6521344,6521856) | 20260507 02:52:37.147 | INFO | MemInfo: { | 20260507 02:52:37.147 | INFO | "gpu" : " 17 GB", | 20260507 02:52:37.147 | INFO | "huge" : "", | 20260507 02:52:37.147 | INFO | "huge_shared" : " 27 GB", | 20260507 02:52:37.147 | INFO | "malloc" : " 14 GB", | 20260507 02:52:37.147 | INFO | "pinned" : " 29 GB" | 20260507 02:52:37.147 | INFO | } | 20260507 02:52:37.147 | INFO | Perf report for the last 5.14209 seconds: | 20260507 02:52:37.147 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:52:37.147 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:37.147 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:37.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:37.147 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 02:52:37.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:37.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:37.147 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 02:52:37.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:37.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:37.147 | INFO | Uploaded 36GiB to GPU in 2.03607s (17.6811GiB/s) | 20260507 02:52:37.147 | INFO | Downloaded 34GiB to GPU in 1.93625s (17.5597GiB/s) | 20260507 02:52:42.280 | INFO | Analyzing chunk frames = [6521856,6522368) | 20260507 02:52:42.280 | INFO | MemInfo: { | 20260507 02:52:42.280 | INFO | "gpu" : " 17 GB", | 20260507 02:52:42.280 | INFO | "huge" : "", | 20260507 02:52:42.280 | INFO | "huge_shared" : " 27 GB", | 20260507 02:52:42.280 | INFO | "malloc" : " 14 GB", | 20260507 02:52:42.280 | INFO | "pinned" : " 29 GB" | 20260507 02:52:42.280 | INFO | } | 20260507 02:52:42.280 | INFO | Perf report for the last 5.13289 seconds: | 20260507 02:52:42.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:52:42.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:42.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:42.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:42.280 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 02:52:42.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:42.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:42.280 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:52:42.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:42.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:42.280 | INFO | Uploaded 36GiB to GPU in 2.00568s (17.949GiB/s) | 20260507 02:52:42.280 | INFO | Downloaded 34GiB to GPU in 1.93735s (17.5498GiB/s) | 20260507 02:52:47.419 | INFO | Analyzing chunk frames = [6522368,6522880) | 20260507 02:52:47.419 | INFO | MemInfo: { | 20260507 02:52:47.419 | INFO | "gpu" : " 17 GB", | 20260507 02:52:47.419 | INFO | "huge" : "", | 20260507 02:52:47.419 | INFO | "huge_shared" : " 27 GB", | 20260507 02:52:47.419 | INFO | "malloc" : " 14 GB", | 20260507 02:52:47.419 | INFO | "pinned" : " 29 GB" | 20260507 02:52:47.419 | INFO | } | 20260507 02:52:47.419 | INFO | Perf report for the last 5.13917 seconds: | 20260507 02:52:47.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:52:47.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:47.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:47.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:47.419 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 02:52:47.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:47.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:47.419 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:52:47.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:47.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:47.419 | INFO | Uploaded 36GiB to GPU in 1.98294s (18.1548GiB/s) | 20260507 02:52:47.419 | INFO | Downloaded 34GiB to GPU in 1.90752s (17.8242GiB/s) | 20260507 02:52:52.561 | INFO | Analyzing chunk frames = [6522880,6523392) | 20260507 02:52:52.561 | INFO | MemInfo: { | 20260507 02:52:52.561 | INFO | "gpu" : " 17 GB", | 20260507 02:52:52.561 | INFO | "huge" : "", | 20260507 02:52:52.561 | INFO | "huge_shared" : " 27 GB", | 20260507 02:52:52.561 | INFO | "malloc" : " 14 GB", | 20260507 02:52:52.561 | INFO | "pinned" : " 29 GB" | 20260507 02:52:52.561 | INFO | } | 20260507 02:52:52.561 | INFO | Perf report for the last 5.14175 seconds: | 20260507 02:52:52.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:52:52.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:52.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:52.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:52.561 | INFO | ANALYSIS : 60.2%, 3.0, 94.0ms, 0 | 20260507 02:52:52.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:52.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:52.561 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:52:52.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:52.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:52.561 | INFO | Uploaded 36GiB to GPU in 2.04018s (17.6455GiB/s) | 20260507 02:52:52.561 | INFO | Downloaded 34GiB to GPU in 1.93021s (17.6146GiB/s) | 20260507 02:52:57.698 | INFO | Analyzing chunk frames = [6523392,6523904) | 20260507 02:52:57.698 | INFO | MemInfo: { | 20260507 02:52:57.698 | INFO | "gpu" : " 17 GB", | 20260507 02:52:57.698 | INFO | "huge" : "", | 20260507 02:52:57.698 | INFO | "huge_shared" : " 27 GB", | 20260507 02:52:57.698 | INFO | "malloc" : " 14 GB", | 20260507 02:52:57.698 | INFO | "pinned" : " 29 GB" | 20260507 02:52:57.698 | INFO | } | 20260507 02:52:57.698 | INFO | Perf report for the last 5.13731 seconds: | 20260507 02:52:57.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:52:57.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:57.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:57.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:57.698 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 02:52:57.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:57.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:57.698 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:52:57.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:57.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:52:57.699 | INFO | Uploaded 36GiB to GPU in 2.05617s (17.5082GiB/s) | 20260507 02:52:57.699 | INFO | Downloaded 34GiB to GPU in 1.95435s (17.3971GiB/s) | 20260507 02:53:02.841 | INFO | Analyzing chunk frames = [6523904,6524416) | 20260507 02:53:02.841 | INFO | MemInfo: { | 20260507 02:53:02.841 | INFO | "gpu" : " 17 GB", | 20260507 02:53:02.841 | INFO | "huge" : "", | 20260507 02:53:02.841 | INFO | "huge_shared" : " 27 GB", | 20260507 02:53:02.841 | INFO | "malloc" : " 14 GB", | 20260507 02:53:02.841 | INFO | "pinned" : " 29 GB" | 20260507 02:53:02.841 | INFO | } | 20260507 02:53:02.841 | INFO | Perf report for the last 5.14305 seconds: | 20260507 02:53:02.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:53:02.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:02.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:02.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:02.841 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:53:02.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:02.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:02.842 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:53:02.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:02.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:02.842 | INFO | Uploaded 36GiB to GPU in 1.99913s (18.0079GiB/s) | 20260507 02:53:02.842 | INFO | Downloaded 34GiB to GPU in 1.98133s (17.1602GiB/s) | 20260507 02:53:07.982 | INFO | Analyzing chunk frames = [6524416,6524928) | 20260507 02:53:07.982 | INFO | MemInfo: { | 20260507 02:53:07.982 | INFO | "gpu" : " 17 GB", | 20260507 02:53:07.982 | INFO | "huge" : "", | 20260507 02:53:07.982 | INFO | "huge_shared" : " 27 GB", | 20260507 02:53:07.982 | INFO | "malloc" : " 14 GB", | 20260507 02:53:07.982 | INFO | "pinned" : " 29 GB" | 20260507 02:53:07.982 | INFO | } | 20260507 02:53:07.982 | INFO | Perf report for the last 5.14078 seconds: | 20260507 02:53:07.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:53:07.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:07.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:07.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:07.982 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:53:07.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:07.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:07.982 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:53:07.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:07.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:07.982 | INFO | Uploaded 36GiB to GPU in 1.98911s (18.0986GiB/s) | 20260507 02:53:07.982 | INFO | Downloaded 34GiB to GPU in 1.97199s (17.2415GiB/s) | 20260507 02:53:08.079 | INFO | Starting a round of Baz IO | 20260507 02:53:11.188 | INFO | Populating RTMetrics report | 20260507 02:53:11.189 | INFO | RTMetrics report populated | 20260507 02:53:11.189 | INFO | Opening tmp RTMetrics json file | 20260507 02:53:11.190 | INFO | Serializing RTMetrics to json | 20260507 02:53:11.208 | INFO | Serialization done, swapping tmp file now... | 20260507 02:53:11.209 | INFO | RTMetrics done | 20260507 02:53:11.281 | INFO | Finished a round of Baz IO: Wrote 1.771 GiB in 3.202 seconds | 20260507 02:53:11.281 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.95% / 0.00% / 67.05% | 20260507 02:53:11.281 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.22105% / 0% / 6.75519e-05% | 20260507 02:53:11.281 | INFO | Baz IO overall: | 20260507 02:53:11.281 | INFO | BytesWritten : 3359645194358 | 20260507 02:53:11.281 | INFO | NumEvents : 1615121743993 | 20260507 02:53:13.117 | INFO | Analyzing chunk frames = [6524928,6525440) | 20260507 02:53:13.117 | INFO | MemInfo: { | 20260507 02:53:13.117 | INFO | "gpu" : " 17 GB", | 20260507 02:53:13.117 | INFO | "huge" : "", | 20260507 02:53:13.117 | INFO | "huge_shared" : " 27 GB", | 20260507 02:53:13.117 | INFO | "malloc" : " 14 GB", | 20260507 02:53:13.117 | INFO | "pinned" : " 29 GB" | 20260507 02:53:13.117 | INFO | } | 20260507 02:53:13.117 | INFO | Perf report for the last 5.1346 seconds: | 20260507 02:53:13.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:53:13.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:13.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:13.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:13.117 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 02:53:13.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:13.117 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 02:53:13.117 | INFO | PRE_HQ : 59.3%, 1.8, 55.6ms, 0 | 20260507 02:53:13.117 | INFO | BAZWRITER : 62.0%, 3.3, 106.2ms, 0 | 20260507 02:53:13.117 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:53:13.118 | INFO | Uploaded 36GiB to GPU in 2.0496s (17.5644GiB/s) | 20260507 02:53:13.118 | INFO | Downloaded 38GiB to GPU in 2.1749s (17.4721GiB/s) | 20260507 02:53:18.253 | INFO | Analyzing chunk frames = [6525440,6525952) | 20260507 02:53:18.253 | INFO | MemInfo: { | 20260507 02:53:18.253 | INFO | "gpu" : " 17 GB", | 20260507 02:53:18.253 | INFO | "huge" : "", | 20260507 02:53:18.253 | INFO | "huge_shared" : " 27 GB", | 20260507 02:53:18.253 | INFO | "malloc" : " 14 GB", | 20260507 02:53:18.253 | INFO | "pinned" : " 29 GB" | 20260507 02:53:18.253 | INFO | } | 20260507 02:53:18.253 | INFO | Perf report for the last 5.13637 seconds: | 20260507 02:53:18.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:53:18.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:18.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:18.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:18.253 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:53:18.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:18.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:18.253 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:53:18.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:18.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:18.253 | INFO | Uploaded 36GiB to GPU in 2.03418s (17.6976GiB/s) | 20260507 02:53:18.254 | INFO | Downloaded 34GiB to GPU in 1.93717s (17.5514GiB/s) | 20260507 02:53:23.395 | INFO | Analyzing chunk frames = [6525952,6526464) | 20260507 02:53:23.395 | INFO | MemInfo: { | 20260507 02:53:23.395 | INFO | "gpu" : " 17 GB", | 20260507 02:53:23.395 | INFO | "huge" : "", | 20260507 02:53:23.395 | INFO | "huge_shared" : " 27 GB", | 20260507 02:53:23.395 | INFO | "malloc" : " 14 GB", | 20260507 02:53:23.395 | INFO | "pinned" : " 29 GB" | 20260507 02:53:23.395 | INFO | } | 20260507 02:53:23.395 | INFO | Perf report for the last 5.14212 seconds: | 20260507 02:53:23.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:53:23.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:23.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:23.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:23.395 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 02:53:23.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:23.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:23.395 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:53:23.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:23.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:23.396 | INFO | Uploaded 36GiB to GPU in 2.00295s (17.9735GiB/s) | 20260507 02:53:23.396 | INFO | Downloaded 34GiB to GPU in 1.93388s (17.5813GiB/s) | 20260507 02:53:28.540 | INFO | Analyzing chunk frames = [6526464,6526976) | 20260507 02:53:28.540 | INFO | MemInfo: { | 20260507 02:53:28.540 | INFO | "gpu" : " 17 GB", | 20260507 02:53:28.540 | INFO | "huge" : "", | 20260507 02:53:28.540 | INFO | "huge_shared" : " 27 GB", | 20260507 02:53:28.540 | INFO | "malloc" : " 14 GB", | 20260507 02:53:28.540 | INFO | "pinned" : " 29 GB" | 20260507 02:53:28.540 | INFO | } | 20260507 02:53:28.540 | INFO | Perf report for the last 5.14461 seconds: | 20260507 02:53:28.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:53:28.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:28.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:28.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:28.540 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 02:53:28.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:28.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:28.540 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:53:28.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:28.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:28.540 | INFO | Uploaded 36GiB to GPU in 1.98269s (18.1571GiB/s) | 20260507 02:53:28.540 | INFO | Downloaded 34GiB to GPU in 1.90838s (17.8162GiB/s) | 20260507 02:53:33.677 | INFO | Analyzing chunk frames = [6526976,6527488) | 20260507 02:53:33.677 | INFO | MemInfo: { | 20260507 02:53:33.677 | INFO | "gpu" : " 17 GB", | 20260507 02:53:33.677 | INFO | "huge" : "", | 20260507 02:53:33.677 | INFO | "huge_shared" : " 27 GB", | 20260507 02:53:33.677 | INFO | "malloc" : " 14 GB", | 20260507 02:53:33.677 | INFO | "pinned" : " 29 GB" | 20260507 02:53:33.677 | INFO | } | 20260507 02:53:33.677 | INFO | Perf report for the last 5.1372 seconds: | 20260507 02:53:33.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:53:33.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:33.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:33.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:33.677 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 02:53:33.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:33.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:33.677 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:53:33.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:33.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:33.677 | INFO | Uploaded 36GiB to GPU in 2.04033s (17.6442GiB/s) | 20260507 02:53:33.678 | INFO | Downloaded 34GiB to GPU in 1.93901s (17.5347GiB/s) | 20260507 02:53:38.817 | INFO | Analyzing chunk frames = [6527488,6528000) | 20260507 02:53:38.817 | INFO | MemInfo: { | 20260507 02:53:38.817 | INFO | "gpu" : " 17 GB", | 20260507 02:53:38.817 | INFO | "huge" : "", | 20260507 02:53:38.817 | INFO | "huge_shared" : " 27 GB", | 20260507 02:53:38.817 | INFO | "malloc" : " 14 GB", | 20260507 02:53:38.817 | INFO | "pinned" : " 29 GB" | 20260507 02:53:38.817 | INFO | } | 20260507 02:53:38.817 | INFO | Perf report for the last 5.14011 seconds: | 20260507 02:53:38.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:53:38.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:38.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:38.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:38.817 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 02:53:38.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:38.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:38.817 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 02:53:38.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:38.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:38.818 | INFO | Uploaded 36GiB to GPU in 2.02803s (17.7512GiB/s) | 20260507 02:53:38.818 | INFO | Downloaded 34GiB to GPU in 1.91974s (17.7107GiB/s) | 20260507 02:53:43.954 | INFO | Analyzing chunk frames = [6528000,6528512) | 20260507 02:53:43.954 | INFO | MemInfo: { | 20260507 02:53:43.954 | INFO | "gpu" : " 17 GB", | 20260507 02:53:43.954 | INFO | "huge" : "", | 20260507 02:53:43.954 | INFO | "huge_shared" : " 27 GB", | 20260507 02:53:43.954 | INFO | "malloc" : " 14 GB", | 20260507 02:53:43.954 | INFO | "pinned" : " 29 GB" | 20260507 02:53:43.954 | INFO | } | 20260507 02:53:43.954 | INFO | Perf report for the last 5.13695 seconds: | 20260507 02:53:43.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:53:43.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:43.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:43.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:43.954 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 02:53:43.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:43.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:43.954 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:53:43.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:43.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:43.954 | INFO | Uploaded 36GiB to GPU in 1.97457s (18.2318GiB/s) | 20260507 02:53:43.954 | INFO | Downloaded 34GiB to GPU in 1.97023s (17.2568GiB/s) | 20260507 02:53:49.096 | INFO | Analyzing chunk frames = [6528512,6529024) | 20260507 02:53:49.096 | INFO | MemInfo: { | 20260507 02:53:49.096 | INFO | "gpu" : " 17 GB", | 20260507 02:53:49.096 | INFO | "huge" : "", | 20260507 02:53:49.096 | INFO | "huge_shared" : " 27 GB", | 20260507 02:53:49.096 | INFO | "malloc" : " 14 GB", | 20260507 02:53:49.096 | INFO | "pinned" : " 29 GB" | 20260507 02:53:49.096 | INFO | } | 20260507 02:53:49.096 | INFO | Perf report for the last 5.14203 seconds: | 20260507 02:53:49.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:53:49.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:49.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:49.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:49.096 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 02:53:49.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:49.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:49.096 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:53:49.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:49.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:49.096 | INFO | Uploaded 36GiB to GPU in 1.98408s (18.1444GiB/s) | 20260507 02:53:49.096 | INFO | Downloaded 34GiB to GPU in 1.96234s (17.3262GiB/s) | 20260507 02:53:49.097 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:53:49.097 | INFO | Total mem usage: 18 GiB | 20260507 02:53:49.097 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:53:49.097 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:53:49.097 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:53:49.097 | INFO | High Water Breakdown: | 20260507 02:53:49.097 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:53:49.097 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:53:49.097 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:53:49.097 | INFO | ------------------------------------------ | 20260507 02:53:49.097 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:53:49.097 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:53:49.097 | INFO | Total mem usage: 26.5 GiB | 20260507 02:53:49.097 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:53:49.097 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:53:49.097 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:53:49.097 | INFO | High Water Breakdown: | 20260507 02:53:49.097 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:53:49.097 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:53:49.097 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:53:49.097 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:53:49.097 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:53:49.097 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:53:49.097 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:53:49.097 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:53:49.097 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:53:49.097 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:53:49.097 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:53:49.097 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:53:49.097 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:53:49.097 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:53:49.097 | INFO | ------------------------------------------ | 20260507 02:53:49.097 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:53:49.097 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:53:49.097 | INFO | Total mem usage: 25.6 GiB | 20260507 02:53:49.097 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:53:49.097 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 02:53:49.097 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 02:53:49.097 | INFO | High Water Breakdown: | 20260507 02:53:49.097 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 02:53:49.097 | INFO | ------------------------------------------ | 20260507 02:53:49.097 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:53:49.097 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:53:49.097 | INFO | Total mem usage: 15.7 GiB | 20260507 02:53:49.097 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:53:49.097 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:53:49.097 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 02:53:49.097 | INFO | High Water Breakdown: | 20260507 02:53:49.097 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:53:49.097 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:53:49.097 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:53:49.097 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:53:49.097 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:53:49.097 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:53:49.097 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:53:49.097 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:53:49.097 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:53:49.097 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:53:49.097 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:53:49.097 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:53:49.097 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:53:49.097 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:53:49.098 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:53:49.098 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:53:49.098 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:53:49.098 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:53:49.098 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:53:49.098 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:53:49.098 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:53:49.098 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:53:49.098 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:53:49.098 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:53:49.098 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:53:49.098 | INFO | ------------------------------------------ | 20260507 02:53:49.098 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:53:49.098 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:53:49.098 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:53:49.098 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:53:49.098 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.52e+01, 1.34e+01 / 1.45e+00 | 20260507 02:53:49.098 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.00e-01, 9.59e-03 / 5.77e-03 | 20260507 02:53:49.098 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.47e+00, 5.16e-01 / 2.14e-01 | 20260507 02:53:49.098 | INFO | PerfReport: 3.22%, Histogram -- 9.91e-01 / 1.48e+00, 1.69e-01 / 8.74e-02 | 20260507 02:53:49.098 | INFO | PerfReport: 16.11%, DME -- 3.97e+01 / 4.44e+01, 3.97e+00 / 1.89e+00 | 20260507 02:53:49.098 | INFO | PerfReport: 57.61%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.30e+00 / 1.70e+00 | 20260507 02:53:49.098 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.15e+00 / 2.71e+00, 3.79e-01 / 2.08e-01 | 20260507 02:53:49.098 | INFO | PerfReport: 3.07%, Metrics -- 9.44e-01 / 1.28e+00, 2.50e-01 / 1.13e-01 | 20260507 02:53:49.098 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.20e-01 / 3.76e-01, 5.13e-02 / 1.98e-02 | 20260507 02:53:49.098 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:53:49.098 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:53:49.098 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:53:49.098 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 02:53:49.098 | INFO | PerfReport: 21.78%, Upload -- 2.10e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260507 02:53:49.098 | INFO | PerfReport: 21.31%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260507 02:53:49.098 | INFO | PerfReport: 31.95%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 02:53:49.098 | INFO | PerfReport: 1.53%, LockUpload -- 1.48e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 02:53:49.098 | INFO | PerfReport: 1.21%, LockDownload -- 1.16e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260507 02:53:49.098 | INFO | PerfReport: 22.20%, LockCompute -- 2.14e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 02:53:49.098 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260507 02:53:49.193 | INFO | Starting a round of Baz IO | 20260507 02:53:52.299 | INFO | Populating RTMetrics report | 20260507 02:53:52.300 | INFO | RTMetrics report populated | 20260507 02:53:52.300 | INFO | Opening tmp RTMetrics json file | 20260507 02:53:52.318 | INFO | Serializing RTMetrics to json | 20260507 02:53:52.319 | INFO | Serialization done, swapping tmp file now... | 20260507 02:53:52.319 | INFO | RTMetrics done | 20260507 02:53:52.393 | INFO | Finished a round of Baz IO: Wrote 1.771 GiB in 3.2 seconds | 20260507 02:53:52.393 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.95% / 0.00% / 67.05% | 20260507 02:53:52.393 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.221165% / 3.97364e-06% / 7.54992e-05% | 20260507 02:53:52.393 | INFO | Baz IO overall: | 20260507 02:53:52.393 | INFO | BytesWritten : 3361546353521 | 20260507 02:53:52.393 | INFO | NumEvents : 1616020606375 | 20260507 02:53:54.230 | INFO | Analyzing chunk frames = [6529024,6529536) | 20260507 02:53:54.230 | INFO | MemInfo: { | 20260507 02:53:54.230 | INFO | "gpu" : " 17 GB", | 20260507 02:53:54.230 | INFO | "huge" : "", | 20260507 02:53:54.230 | INFO | "huge_shared" : " 27 GB", | 20260507 02:53:54.230 | INFO | "malloc" : " 14 GB", | 20260507 02:53:54.230 | INFO | "pinned" : " 29 GB" | 20260507 02:53:54.230 | INFO | } | 20260507 02:53:54.230 | INFO | Perf report for the last 5.1339 seconds: | 20260507 02:53:54.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:53:54.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:54.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:54.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:54.230 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260507 02:53:54.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:54.230 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 02:53:54.230 | INFO | PRE_HQ : 59.2%, 1.8, 55.4ms, 0 | 20260507 02:53:54.230 | INFO | BAZWRITER : 61.9%, 3.3, 103.8ms, 0 | 20260507 02:53:54.230 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:53:54.230 | INFO | Uploaded 36GiB to GPU in 2.03515s (17.6891GiB/s) | 20260507 02:53:54.230 | INFO | Downloaded 38GiB to GPU in 2.18259s (17.4105GiB/s) | 20260507 02:53:59.371 | INFO | Analyzing chunk frames = [6529536,6530048) | 20260507 02:53:59.371 | INFO | MemInfo: { | 20260507 02:53:59.371 | INFO | "gpu" : " 17 GB", | 20260507 02:53:59.371 | INFO | "huge" : "", | 20260507 02:53:59.371 | INFO | "huge_shared" : " 27 GB", | 20260507 02:53:59.371 | INFO | "malloc" : " 14 GB", | 20260507 02:53:59.371 | INFO | "pinned" : " 29 GB" | 20260507 02:53:59.371 | INFO | } | 20260507 02:53:59.371 | INFO | Perf report for the last 5.14064 seconds: | 20260507 02:53:59.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:53:59.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:59.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:59.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:59.371 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:53:59.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:59.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:59.371 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 02:53:59.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:59.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:53:59.371 | INFO | Uploaded 36GiB to GPU in 2.02373s (17.7889GiB/s) | 20260507 02:53:59.371 | INFO | Downloaded 34GiB to GPU in 1.93546s (17.5669GiB/s) | 20260507 02:54:04.512 | INFO | Analyzing chunk frames = [6530048,6530560) | 20260507 02:54:04.512 | INFO | MemInfo: { | 20260507 02:54:04.512 | INFO | "gpu" : " 17 GB", | 20260507 02:54:04.512 | INFO | "huge" : "", | 20260507 02:54:04.512 | INFO | "huge_shared" : " 27 GB", | 20260507 02:54:04.512 | INFO | "malloc" : " 14 GB", | 20260507 02:54:04.512 | INFO | "pinned" : " 29 GB" | 20260507 02:54:04.512 | INFO | } | 20260507 02:54:04.512 | INFO | Perf report for the last 5.14148 seconds: | 20260507 02:54:04.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:54:04.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:04.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:04.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:04.512 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:54:04.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:04.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:04.512 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:54:04.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:04.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:04.513 | INFO | Uploaded 36GiB to GPU in 2.00351s (17.9684GiB/s) | 20260507 02:54:04.513 | INFO | Downloaded 34GiB to GPU in 1.93639s (17.5584GiB/s) | 20260507 02:54:09.652 | INFO | Analyzing chunk frames = [6530560,6531072) | 20260507 02:54:09.652 | INFO | MemInfo: { | 20260507 02:54:09.652 | INFO | "gpu" : " 17 GB", | 20260507 02:54:09.652 | INFO | "huge" : "", | 20260507 02:54:09.652 | INFO | "huge_shared" : " 27 GB", | 20260507 02:54:09.652 | INFO | "malloc" : " 14 GB", | 20260507 02:54:09.652 | INFO | "pinned" : " 29 GB" | 20260507 02:54:09.652 | INFO | } | 20260507 02:54:09.653 | INFO | Perf report for the last 5.14018 seconds: | 20260507 02:54:09.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:54:09.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:09.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:09.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:09.653 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 02:54:09.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:09.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:09.653 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:54:09.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:09.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:09.653 | INFO | Uploaded 36GiB to GPU in 1.99684s (18.0285GiB/s) | 20260507 02:54:09.653 | INFO | Downloaded 34GiB to GPU in 1.91856s (17.7217GiB/s) | 20260507 02:54:14.791 | INFO | Analyzing chunk frames = [6531072,6531584) | 20260507 02:54:14.791 | INFO | MemInfo: { | 20260507 02:54:14.791 | INFO | "gpu" : " 17 GB", | 20260507 02:54:14.791 | INFO | "huge" : "", | 20260507 02:54:14.791 | INFO | "huge_shared" : " 27 GB", | 20260507 02:54:14.791 | INFO | "malloc" : " 14 GB", | 20260507 02:54:14.791 | INFO | "pinned" : " 29 GB" | 20260507 02:54:14.791 | INFO | } | 20260507 02:54:14.791 | INFO | Perf report for the last 5.13869 seconds: | 20260507 02:54:14.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:54:14.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:14.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:14.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:14.791 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 02:54:14.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:14.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:14.791 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:54:14.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:14.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:14.791 | INFO | Uploaded 36GiB to GPU in 2.03805s (17.6639GiB/s) | 20260507 02:54:14.791 | INFO | Downloaded 34GiB to GPU in 1.93313s (17.588GiB/s) | 20260507 02:54:19.928 | INFO | Analyzing chunk frames = [6531584,6532096) | 20260507 02:54:19.928 | INFO | MemInfo: { | 20260507 02:54:19.928 | INFO | "gpu" : " 17 GB", | 20260507 02:54:19.928 | INFO | "huge" : "", | 20260507 02:54:19.928 | INFO | "huge_shared" : " 27 GB", | 20260507 02:54:19.928 | INFO | "malloc" : " 14 GB", | 20260507 02:54:19.928 | INFO | "pinned" : " 29 GB" | 20260507 02:54:19.928 | INFO | } | 20260507 02:54:19.928 | INFO | Perf report for the last 5.13661 seconds: | 20260507 02:54:19.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:54:19.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:19.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:19.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:19.928 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 02:54:19.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:19.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:19.928 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 02:54:19.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:19.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:19.928 | INFO | Uploaded 36GiB to GPU in 2.0342s (17.6974GiB/s) | 20260507 02:54:19.928 | INFO | Downloaded 34GiB to GPU in 1.90064s (17.8887GiB/s) | 20260507 02:54:25.066 | INFO | Analyzing chunk frames = [6532096,6532608) | 20260507 02:54:25.066 | INFO | MemInfo: { | 20260507 02:54:25.066 | INFO | "gpu" : " 17 GB", | 20260507 02:54:25.066 | INFO | "huge" : "", | 20260507 02:54:25.066 | INFO | "huge_shared" : " 27 GB", | 20260507 02:54:25.066 | INFO | "malloc" : " 14 GB", | 20260507 02:54:25.066 | INFO | "pinned" : " 29 GB" | 20260507 02:54:25.066 | INFO | } | 20260507 02:54:25.066 | INFO | Perf report for the last 5.1386 seconds: | 20260507 02:54:25.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:54:25.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:25.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:25.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:25.066 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:54:25.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:25.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:25.066 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 02:54:25.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:25.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:25.067 | INFO | Uploaded 36GiB to GPU in 1.99216s (18.0708GiB/s) | 20260507 02:54:25.067 | INFO | Downloaded 34GiB to GPU in 1.98049s (17.1675GiB/s) | 20260507 02:54:30.206 | INFO | Analyzing chunk frames = [6532608,6533120) | 20260507 02:54:30.206 | INFO | MemInfo: { | 20260507 02:54:30.206 | INFO | "gpu" : " 17 GB", | 20260507 02:54:30.207 | INFO | "huge" : "", | 20260507 02:54:30.207 | INFO | "huge_shared" : " 27 GB", | 20260507 02:54:30.207 | INFO | "malloc" : " 14 GB", | 20260507 02:54:30.207 | INFO | "pinned" : " 29 GB" | 20260507 02:54:30.207 | INFO | } | 20260507 02:54:30.207 | INFO | Perf report for the last 5.14029 seconds: | 20260507 02:54:30.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:54:30.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:30.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:30.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:30.207 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:54:30.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:30.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:30.207 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:54:30.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:30.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:30.207 | INFO | Uploaded 36GiB to GPU in 2.00369s (17.9669GiB/s) | 20260507 02:54:30.207 | INFO | Downloaded 34GiB to GPU in 1.97638s (17.2031GiB/s) | 20260507 02:54:30.304 | INFO | Starting a round of Baz IO | 20260507 02:54:33.419 | INFO | Populating RTMetrics report | 20260507 02:54:33.419 | INFO | RTMetrics report populated | 20260507 02:54:33.419 | INFO | Opening tmp RTMetrics json file | 20260507 02:54:33.427 | INFO | Serializing RTMetrics to json | 20260507 02:54:33.437 | INFO | Serialization done, swapping tmp file now... | 20260507 02:54:33.437 | INFO | RTMetrics done | 20260507 02:54:33.508 | INFO | Finished a round of Baz IO: Wrote 1.769 GiB in 3.204 seconds | 20260507 02:54:33.508 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.95% / 0.00% / 67.05% | 20260507 02:54:33.508 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.220478% / 0% / 5.5631e-05% | 20260507 02:54:33.508 | INFO | Baz IO overall: | 20260507 02:54:33.508 | INFO | BytesWritten : 3363446248563 | 20260507 02:54:33.508 | INFO | NumEvents : 1616918161033 | 20260507 02:54:35.346 | INFO | Analyzing chunk frames = [6533120,6533632) | 20260507 02:54:35.346 | INFO | MemInfo: { | 20260507 02:54:35.346 | INFO | "gpu" : " 17 GB", | 20260507 02:54:35.346 | INFO | "huge" : "", | 20260507 02:54:35.346 | INFO | "huge_shared" : " 27 GB", | 20260507 02:54:35.346 | INFO | "malloc" : " 14 GB", | 20260507 02:54:35.346 | INFO | "pinned" : " 29 GB" | 20260507 02:54:35.346 | INFO | } | 20260507 02:54:35.346 | INFO | Perf report for the last 5.1393 seconds: | 20260507 02:54:35.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:54:35.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:35.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:35.346 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:35.346 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260507 02:54:35.346 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:35.346 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 02:54:35.346 | INFO | PRE_HQ : 59.4%, 1.8, 55.4ms, 0 | 20260507 02:54:35.346 | INFO | BAZWRITER : 61.9%, 3.3, 104.6ms, 0 | 20260507 02:54:35.346 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:54:35.346 | INFO | Uploaded 36GiB to GPU in 2.0355s (17.6861GiB/s) | 20260507 02:54:35.346 | INFO | Downloaded 38GiB to GPU in 2.18057s (17.4266GiB/s) | 20260507 02:54:40.492 | INFO | Analyzing chunk frames = [6533632,6534144) | 20260507 02:54:40.492 | INFO | MemInfo: { | 20260507 02:54:40.492 | INFO | "gpu" : " 17 GB", | 20260507 02:54:40.492 | INFO | "huge" : "", | 20260507 02:54:40.492 | INFO | "huge_shared" : " 27 GB", | 20260507 02:54:40.492 | INFO | "malloc" : " 14 GB", | 20260507 02:54:40.492 | INFO | "pinned" : " 29 GB" | 20260507 02:54:40.492 | INFO | } | 20260507 02:54:40.492 | INFO | Perf report for the last 5.14586 seconds: | 20260507 02:54:40.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:54:40.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:40.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:40.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:40.492 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 02:54:40.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:40.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:40.492 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:54:40.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:40.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:40.492 | INFO | Uploaded 36GiB to GPU in 2.03803s (17.6641GiB/s) | 20260507 02:54:40.492 | INFO | Downloaded 34GiB to GPU in 1.93854s (17.539GiB/s) | 20260507 02:54:45.630 | INFO | Analyzing chunk frames = [6534144,6534656) | 20260507 02:54:45.630 | INFO | MemInfo: { | 20260507 02:54:45.630 | INFO | "gpu" : " 17 GB", | 20260507 02:54:45.630 | INFO | "huge" : "", | 20260507 02:54:45.630 | INFO | "huge_shared" : " 27 GB", | 20260507 02:54:45.630 | INFO | "malloc" : " 14 GB", | 20260507 02:54:45.630 | INFO | "pinned" : " 29 GB" | 20260507 02:54:45.630 | INFO | } | 20260507 02:54:45.630 | INFO | Perf report for the last 5.13794 seconds: | 20260507 02:54:45.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:54:45.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:45.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:45.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:45.630 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 02:54:45.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:45.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:45.630 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:54:45.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:45.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:45.630 | INFO | Uploaded 36GiB to GPU in 2.0211s (17.812GiB/s) | 20260507 02:54:45.630 | INFO | Downloaded 34GiB to GPU in 1.93757s (17.5478GiB/s) | 20260507 02:54:50.763 | INFO | Analyzing chunk frames = [6534656,6535168) | 20260507 02:54:50.763 | INFO | MemInfo: { | 20260507 02:54:50.763 | INFO | "gpu" : " 17 GB", | 20260507 02:54:50.763 | INFO | "huge" : "", | 20260507 02:54:50.763 | INFO | "huge_shared" : " 27 GB", | 20260507 02:54:50.763 | INFO | "malloc" : " 14 GB", | 20260507 02:54:50.763 | INFO | "pinned" : " 29 GB" | 20260507 02:54:50.763 | INFO | } | 20260507 02:54:50.763 | INFO | Perf report for the last 5.13281 seconds: | 20260507 02:54:50.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:54:50.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:50.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:50.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:50.763 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 02:54:50.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:50.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:50.763 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:54:50.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:50.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:50.763 | INFO | Uploaded 36GiB to GPU in 2.0065s (17.9417GiB/s) | 20260507 02:54:50.763 | INFO | Downloaded 34GiB to GPU in 1.93003s (17.6163GiB/s) | 20260507 02:54:55.904 | INFO | Analyzing chunk frames = [6535168,6535680) | 20260507 02:54:55.904 | INFO | MemInfo: { | 20260507 02:54:55.904 | INFO | "gpu" : " 17 GB", | 20260507 02:54:55.904 | INFO | "huge" : "", | 20260507 02:54:55.904 | INFO | "huge_shared" : " 27 GB", | 20260507 02:54:55.904 | INFO | "malloc" : " 14 GB", | 20260507 02:54:55.904 | INFO | "pinned" : " 29 GB" | 20260507 02:54:55.904 | INFO | } | 20260507 02:54:55.904 | INFO | Perf report for the last 5.14155 seconds: | 20260507 02:54:55.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:54:55.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:55.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:55.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:55.904 | INFO | ANALYSIS : 59.8%, 3.0, 93.2ms, 0 | 20260507 02:54:55.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:55.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:55.904 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:54:55.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:55.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:54:55.905 | INFO | Uploaded 36GiB to GPU in 2.00062s (17.9944GiB/s) | 20260507 02:54:55.905 | INFO | Downloaded 34GiB to GPU in 1.94257s (17.5026GiB/s) | 20260507 02:55:01.047 | INFO | Analyzing chunk frames = [6535680,6536192) | 20260507 02:55:01.047 | INFO | MemInfo: { | 20260507 02:55:01.047 | INFO | "gpu" : " 17 GB", | 20260507 02:55:01.047 | INFO | "huge" : "", | 20260507 02:55:01.047 | INFO | "huge_shared" : " 27 GB", | 20260507 02:55:01.047 | INFO | "malloc" : " 14 GB", | 20260507 02:55:01.047 | INFO | "pinned" : " 29 GB" | 20260507 02:55:01.047 | INFO | } | 20260507 02:55:01.047 | INFO | Perf report for the last 5.143 seconds: | 20260507 02:55:01.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:55:01.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:01.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:01.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:01.047 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:55:01.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:01.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:01.047 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 02:55:01.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:01.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:01.048 | INFO | Uploaded 36GiB to GPU in 2.05931s (17.4816GiB/s) | 20260507 02:55:01.048 | INFO | Downloaded 34GiB to GPU in 1.9569s (17.3745GiB/s) | 20260507 02:55:06.182 | INFO | Analyzing chunk frames = [6536192,6536704) | 20260507 02:55:06.182 | INFO | MemInfo: { | 20260507 02:55:06.182 | INFO | "gpu" : " 17 GB", | 20260507 02:55:06.182 | INFO | "huge" : "", | 20260507 02:55:06.182 | INFO | "huge_shared" : " 27 GB", | 20260507 02:55:06.183 | INFO | "malloc" : " 14 GB", | 20260507 02:55:06.183 | INFO | "pinned" : " 29 GB" | 20260507 02:55:06.183 | INFO | } | 20260507 02:55:06.183 | INFO | Perf report for the last 5.13531 seconds: | 20260507 02:55:06.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:55:06.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:06.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:06.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:06.183 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:55:06.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:06.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:06.183 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 02:55:06.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:06.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:06.183 | INFO | Uploaded 36GiB to GPU in 1.99978s (18.002GiB/s) | 20260507 02:55:06.183 | INFO | Downloaded 34GiB to GPU in 1.97774s (17.1913GiB/s) | 20260507 02:55:11.321 | INFO | Analyzing chunk frames = [6536704,6537216) | 20260507 02:55:11.321 | INFO | MemInfo: { | 20260507 02:55:11.321 | INFO | "gpu" : " 17 GB", | 20260507 02:55:11.321 | INFO | "huge" : "", | 20260507 02:55:11.321 | INFO | "huge_shared" : " 27 GB", | 20260507 02:55:11.321 | INFO | "malloc" : " 14 GB", | 20260507 02:55:11.321 | INFO | "pinned" : " 29 GB" | 20260507 02:55:11.321 | INFO | } | 20260507 02:55:11.321 | INFO | Perf report for the last 5.13813 seconds: | 20260507 02:55:11.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:55:11.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:11.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:11.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:11.321 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:55:11.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:11.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:11.321 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:55:11.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:11.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:11.321 | INFO | Uploaded 36GiB to GPU in 2.00838s (17.9249GiB/s) | 20260507 02:55:11.321 | INFO | Downloaded 34GiB to GPU in 1.9673s (17.2825GiB/s) | 20260507 02:55:11.321 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:55:11.321 | INFO | Total mem usage: 18 GiB | 20260507 02:55:11.321 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:55:11.321 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 02:55:11.321 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:55:11.321 | INFO | High Water Breakdown: | 20260507 02:55:11.321 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:55:11.321 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:55:11.321 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:55:11.321 | INFO | ------------------------------------------ | 20260507 02:55:11.321 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:55:11.321 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:55:11.321 | INFO | Total mem usage: 26.5 GiB | 20260507 02:55:11.321 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:55:11.321 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:55:11.321 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:55:11.321 | INFO | High Water Breakdown: | 20260507 02:55:11.321 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:55:11.321 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:55:11.321 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:55:11.321 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:55:11.321 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:55:11.322 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:55:11.322 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:55:11.322 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:55:11.322 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:55:11.322 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:55:11.322 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:55:11.322 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:55:11.322 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:55:11.322 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:55:11.322 | INFO | ------------------------------------------ | 20260507 02:55:11.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:55:11.322 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:55:11.322 | INFO | Total mem usage: 25.6 GiB | 20260507 02:55:11.322 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:55:11.322 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:55:11.322 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:55:11.322 | INFO | High Water Breakdown: | 20260507 02:55:11.322 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:55:11.322 | INFO | ------------------------------------------ | 20260507 02:55:11.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:55:11.322 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:55:11.322 | INFO | Total mem usage: 15.7 GiB | 20260507 02:55:11.322 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:55:11.322 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:55:11.322 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 02:55:11.322 | INFO | High Water Breakdown: | 20260507 02:55:11.322 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:55:11.322 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:55:11.322 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:55:11.322 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:55:11.322 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:55:11.322 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:55:11.322 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:55:11.322 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:55:11.322 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:55:11.322 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:55:11.322 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:55:11.322 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:55:11.322 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:55:11.322 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:55:11.322 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:55:11.322 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:55:11.322 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:55:11.322 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:55:11.322 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:55:11.322 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:55:11.322 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:55:11.322 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:55:11.322 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:55:11.322 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:55:11.322 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:55:11.322 | INFO | ------------------------------------------ | 20260507 02:55:11.322 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:55:11.322 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:55:11.322 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:55:11.322 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:55:11.322 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.51e+01, 1.33e+01 / 2.00e+00 | 20260507 02:55:11.322 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.01e-01, 9.65e-03 / 5.40e-03 | 20260507 02:55:11.322 | INFO | PerfReport: 11.65%, Baseline -- 3.59e+00 / 4.45e+00, 5.11e-01 / 2.82e-01 | 20260507 02:55:11.322 | INFO | PerfReport: 3.23%, Histogram -- 9.94e-01 / 1.48e+00, 1.72e-01 / 1.22e-01 | 20260507 02:55:11.322 | INFO | PerfReport: 16.09%, DME -- 3.96e+01 / 4.42e+01, 3.86e+00 / 2.18e+00 | 20260507 02:55:11.322 | INFO | PerfReport: 57.65%, FrameLabeling -- 1.77e+01 / 2.05e+01, 1.32e+00 / 1.85e+00 | 20260507 02:55:11.322 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.15e+00 / 2.73e+00, 3.84e-01 / 2.26e-01 | 20260507 02:55:11.322 | INFO | PerfReport: 3.07%, Metrics -- 9.44e-01 / 1.28e+00, 2.55e-01 / 1.54e-01 | 20260507 02:55:11.322 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.19e-01 / 3.70e-01, 5.11e-02 / 2.19e-02 | 20260507 02:55:11.322 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:55:11.322 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:55:11.322 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:55:11.322 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 02:55:11.322 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260507 02:55:11.322 | INFO | PerfReport: 21.34%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260507 02:55:11.322 | INFO | PerfReport: 31.93%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 02:55:11.322 | INFO | PerfReport: 1.53%, LockUpload -- 1.47e+00 / 0.00e+00, 5.04e+00 / 0.00e+00 | 20260507 02:55:11.322 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260507 02:55:11.322 | INFO | PerfReport: 22.14%, LockCompute -- 2.13e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 02:55:11.322 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260507 02:55:11.418 | INFO | Starting a round of Baz IO | 20260507 02:55:14.537 | INFO | Populating RTMetrics report | 20260507 02:55:14.537 | INFO | RTMetrics report populated | 20260507 02:55:14.537 | INFO | Opening tmp RTMetrics json file | 20260507 02:55:14.805 | INFO | Serializing RTMetrics to json | 20260507 02:55:14.806 | INFO | Serialization done, swapping tmp file now... | 20260507 02:55:14.807 | INFO | RTMetrics done | 20260507 02:55:14.821 | INFO | Finished a round of Baz IO: Wrote 1.767 GiB in 3.402 seconds | 20260507 02:55:14.821 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.95% / 0.00% / 67.05% | 20260507 02:55:14.821 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.219731% / 3.97364e-06% / 6.35783e-05% | 20260507 02:55:14.821 | INFO | Baz IO overall: | 20260507 02:55:14.821 | INFO | BytesWritten : 3365344003759 | 20260507 02:55:14.821 | INFO | NumEvents : 1617814257869 | 20260507 02:55:16.462 | INFO | Analyzing chunk frames = [6537216,6537728) | 20260507 02:55:16.462 | INFO | MemInfo: { | 20260507 02:55:16.462 | INFO | "gpu" : " 17 GB", | 20260507 02:55:16.462 | INFO | "huge" : "", | 20260507 02:55:16.462 | INFO | "huge_shared" : " 27 GB", | 20260507 02:55:16.462 | INFO | "malloc" : " 14 GB", | 20260507 02:55:16.462 | INFO | "pinned" : " 29 GB" | 20260507 02:55:16.462 | INFO | } | 20260507 02:55:16.463 | INFO | Perf report for the last 5.14176 seconds: | 20260507 02:55:16.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:55:16.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:16.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:16.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:16.463 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 02:55:16.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:16.463 | INFO | RT_METRICS : 46.8%, 1.0, 24.6ms, 0 | 20260507 02:55:16.463 | INFO | PRE_HQ : 59.4%, 1.8, 55.5ms, 0 | 20260507 02:55:16.463 | INFO | BAZWRITER : 65.8%, 3.3, 111.6ms, 0 | 20260507 02:55:16.463 | INFO | BAZ_SAMPLING : 5.0%, 1.0, 2.6ms, 0 | 20260507 02:55:16.463 | INFO | Uploaded 36GiB to GPU in 2.03869s (17.6584GiB/s) | 20260507 02:55:16.463 | INFO | Downloaded 38GiB to GPU in 2.17327s (17.4851GiB/s) | 20260507 02:55:21.605 | INFO | Analyzing chunk frames = [6537728,6538240) | 20260507 02:55:21.605 | INFO | MemInfo: { | 20260507 02:55:21.605 | INFO | "gpu" : " 17 GB", | 20260507 02:55:21.605 | INFO | "huge" : "", | 20260507 02:55:21.605 | INFO | "huge_shared" : " 27 GB", | 20260507 02:55:21.605 | INFO | "malloc" : " 14 GB", | 20260507 02:55:21.605 | INFO | "pinned" : " 29 GB" | 20260507 02:55:21.605 | INFO | } | 20260507 02:55:21.605 | INFO | Perf report for the last 5.1423 seconds: | 20260507 02:55:21.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:55:21.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:21.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:21.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:21.605 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:55:21.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:21.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:21.605 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 02:55:21.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:21.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:21.605 | INFO | Uploaded 36GiB to GPU in 2.03256s (17.7116GiB/s) | 20260507 02:55:21.605 | INFO | Downloaded 34GiB to GPU in 1.93599s (17.562GiB/s) | 20260507 02:55:26.737 | INFO | Analyzing chunk frames = [6538240,6538752) | 20260507 02:55:26.737 | INFO | MemInfo: { | 20260507 02:55:26.737 | INFO | "gpu" : " 17 GB", | 20260507 02:55:26.737 | INFO | "huge" : "", | 20260507 02:55:26.737 | INFO | "huge_shared" : " 27 GB", | 20260507 02:55:26.737 | INFO | "malloc" : " 14 GB", | 20260507 02:55:26.737 | INFO | "pinned" : " 29 GB" | 20260507 02:55:26.737 | INFO | } | 20260507 02:55:26.737 | INFO | Perf report for the last 5.13245 seconds: | 20260507 02:55:26.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:55:26.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:26.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:26.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:26.737 | INFO | ANALYSIS : 60.8%, 3.0, 94.0ms, 0 | 20260507 02:55:26.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:26.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:26.737 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:55:26.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:26.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:26.737 | INFO | Uploaded 36GiB to GPU in 2.04005s (17.6466GiB/s) | 20260507 02:55:26.738 | INFO | Downloaded 34GiB to GPU in 1.95261s (17.4126GiB/s) | 20260507 02:55:31.876 | INFO | Analyzing chunk frames = [6538752,6539264) | 20260507 02:55:31.876 | INFO | MemInfo: { | 20260507 02:55:31.876 | INFO | "gpu" : " 17 GB", | 20260507 02:55:31.876 | INFO | "huge" : "", | 20260507 02:55:31.876 | INFO | "huge_shared" : " 27 GB", | 20260507 02:55:31.876 | INFO | "malloc" : " 14 GB", | 20260507 02:55:31.876 | INFO | "pinned" : " 29 GB" | 20260507 02:55:31.876 | INFO | } | 20260507 02:55:31.876 | INFO | Perf report for the last 5.1391 seconds: | 20260507 02:55:31.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:55:31.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:31.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:31.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:31.877 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:55:31.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:31.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:31.877 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:55:31.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:31.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:31.877 | INFO | Uploaded 36GiB to GPU in 1.99108s (18.0806GiB/s) | 20260507 02:55:31.877 | INFO | Downloaded 34GiB to GPU in 1.91762s (17.7303GiB/s) | 20260507 02:55:37.020 | INFO | Analyzing chunk frames = [6539264,6539776) | 20260507 02:55:37.020 | INFO | MemInfo: { | 20260507 02:55:37.020 | INFO | "gpu" : " 17 GB", | 20260507 02:55:37.020 | INFO | "huge" : "", | 20260507 02:55:37.020 | INFO | "huge_shared" : " 27 GB", | 20260507 02:55:37.020 | INFO | "malloc" : " 14 GB", | 20260507 02:55:37.020 | INFO | "pinned" : " 29 GB" | 20260507 02:55:37.020 | INFO | } | 20260507 02:55:37.020 | INFO | Perf report for the last 5.14379 seconds: | 20260507 02:55:37.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:55:37.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:37.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:37.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:37.020 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 02:55:37.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:37.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:37.020 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 02:55:37.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:37.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:37.021 | INFO | Uploaded 36GiB to GPU in 2.01323s (17.8817GiB/s) | 20260507 02:55:37.021 | INFO | Downloaded 34GiB to GPU in 1.94894s (17.4454GiB/s) | 20260507 02:55:42.165 | INFO | Analyzing chunk frames = [6539776,6540288) | 20260507 02:55:42.165 | INFO | MemInfo: { | 20260507 02:55:42.165 | INFO | "gpu" : " 17 GB", | 20260507 02:55:42.165 | INFO | "huge" : "", | 20260507 02:55:42.165 | INFO | "huge_shared" : " 27 GB", | 20260507 02:55:42.165 | INFO | "malloc" : " 14 GB", | 20260507 02:55:42.165 | INFO | "pinned" : " 29 GB" | 20260507 02:55:42.165 | INFO | } | 20260507 02:55:42.165 | INFO | Perf report for the last 5.14514 seconds: | 20260507 02:55:42.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:55:42.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:42.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:42.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:42.165 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 02:55:42.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:42.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:42.165 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 02:55:42.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:42.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:42.166 | INFO | Uploaded 36GiB to GPU in 2.05856s (17.4879GiB/s) | 20260507 02:55:42.166 | INFO | Downloaded 34GiB to GPU in 1.96058s (17.3418GiB/s) | 20260507 02:55:47.303 | INFO | Analyzing chunk frames = [6540288,6540800) | 20260507 02:55:47.303 | INFO | MemInfo: { | 20260507 02:55:47.303 | INFO | "gpu" : " 17 GB", | 20260507 02:55:47.303 | INFO | "huge" : "", | 20260507 02:55:47.303 | INFO | "huge_shared" : " 27 GB", | 20260507 02:55:47.303 | INFO | "malloc" : " 14 GB", | 20260507 02:55:47.303 | INFO | "pinned" : " 29 GB" | 20260507 02:55:47.303 | INFO | } | 20260507 02:55:47.303 | INFO | Perf report for the last 5.13743 seconds: | 20260507 02:55:47.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:55:47.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:47.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:47.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:47.303 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:55:47.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:47.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:47.303 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:55:47.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:47.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:47.303 | INFO | Uploaded 36GiB to GPU in 1.98109s (18.1718GiB/s) | 20260507 02:55:47.303 | INFO | Downloaded 34GiB to GPU in 1.97031s (17.2561GiB/s) | 20260507 02:55:52.436 | INFO | Analyzing chunk frames = [6540800,6541312) | 20260507 02:55:52.436 | INFO | MemInfo: { | 20260507 02:55:52.436 | INFO | "gpu" : " 17 GB", | 20260507 02:55:52.436 | INFO | "huge" : "", | 20260507 02:55:52.436 | INFO | "huge_shared" : " 27 GB", | 20260507 02:55:52.436 | INFO | "malloc" : " 14 GB", | 20260507 02:55:52.436 | INFO | "pinned" : " 29 GB" | 20260507 02:55:52.436 | INFO | } | 20260507 02:55:52.436 | INFO | Perf report for the last 5.13356 seconds: | 20260507 02:55:52.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:55:52.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:52.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:52.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:52.437 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 02:55:52.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:52.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:52.437 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 02:55:52.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:52.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:52.437 | INFO | Uploaded 36GiB to GPU in 2.00033s (17.997GiB/s) | 20260507 02:55:52.437 | INFO | Downloaded 34GiB to GPU in 1.97089s (17.2511GiB/s) | 20260507 02:55:52.533 | INFO | Starting a round of Baz IO | 20260507 02:55:55.640 | INFO | Populating RTMetrics report | 20260507 02:55:55.641 | INFO | RTMetrics report populated | 20260507 02:55:55.641 | INFO | Opening tmp RTMetrics json file | 20260507 02:55:55.646 | INFO | Serializing RTMetrics to json | 20260507 02:55:55.659 | INFO | Serialization done, swapping tmp file now... | 20260507 02:55:55.659 | INFO | RTMetrics done | 20260507 02:55:55.729 | INFO | Finished a round of Baz IO: Wrote 1.768 GiB in 3.196 seconds | 20260507 02:55:55.729 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.95% / 0.00% / 67.05% | 20260507 02:55:55.729 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.218892% / 3.97364e-06% / 7.94729e-05% | 20260507 02:55:55.729 | INFO | Baz IO overall: | 20260507 02:55:55.729 | INFO | BytesWritten : 3367242181575 | 20260507 02:55:55.729 | INFO | NumEvents : 1618710599547 | 20260507 02:55:57.572 | INFO | Analyzing chunk frames = [6541312,6541824) | 20260507 02:55:57.572 | INFO | MemInfo: { | 20260507 02:55:57.572 | INFO | "gpu" : " 17 GB", | 20260507 02:55:57.572 | INFO | "huge" : "", | 20260507 02:55:57.572 | INFO | "huge_shared" : " 27 GB", | 20260507 02:55:57.572 | INFO | "malloc" : " 14 GB", | 20260507 02:55:57.572 | INFO | "pinned" : " 29 GB" | 20260507 02:55:57.572 | INFO | } | 20260507 02:55:57.572 | INFO | Perf report for the last 5.13562 seconds: | 20260507 02:55:57.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:55:57.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:57.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:57.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:57.572 | INFO | ANALYSIS : 61.8%, 3.0, 95.6ms, 0 | 20260507 02:55:57.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:55:57.572 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 02:55:57.572 | INFO | PRE_HQ : 59.2%, 1.8, 55.0ms, 0 | 20260507 02:55:57.572 | INFO | BAZWRITER : 61.7%, 3.7, 116.4ms, 0 | 20260507 02:55:57.572 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:55:57.572 | INFO | Uploaded 36GiB to GPU in 2.05508s (17.5176GiB/s) | 20260507 02:55:57.572 | INFO | Downloaded 38GiB to GPU in 2.19354s (17.3236GiB/s) | 20260507 02:56:02.710 | INFO | Analyzing chunk frames = [6541824,6542336) | 20260507 02:56:02.710 | INFO | MemInfo: { | 20260507 02:56:02.710 | INFO | "gpu" : " 17 GB", | 20260507 02:56:02.710 | INFO | "huge" : "", | 20260507 02:56:02.710 | INFO | "huge_shared" : " 27 GB", | 20260507 02:56:02.710 | INFO | "malloc" : " 14 GB", | 20260507 02:56:02.710 | INFO | "pinned" : " 29 GB" | 20260507 02:56:02.710 | INFO | } | 20260507 02:56:02.711 | INFO | Perf report for the last 5.13862 seconds: | 20260507 02:56:02.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:56:02.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:02.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:02.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:02.711 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:56:02.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:02.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:02.711 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:56:02.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:02.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:02.711 | INFO | Uploaded 36GiB to GPU in 2.02342s (17.7917GiB/s) | 20260507 02:56:02.711 | INFO | Downloaded 34GiB to GPU in 1.93633s (17.559GiB/s) | 20260507 02:56:07.858 | INFO | Analyzing chunk frames = [6542336,6542848) | 20260507 02:56:07.858 | INFO | MemInfo: { | 20260507 02:56:07.858 | INFO | "gpu" : " 17 GB", | 20260507 02:56:07.858 | INFO | "huge" : "", | 20260507 02:56:07.858 | INFO | "huge_shared" : " 27 GB", | 20260507 02:56:07.858 | INFO | "malloc" : " 14 GB", | 20260507 02:56:07.858 | INFO | "pinned" : " 29 GB" | 20260507 02:56:07.858 | INFO | } | 20260507 02:56:07.858 | INFO | Perf report for the last 5.14696 seconds: | 20260507 02:56:07.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:56:07.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:07.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:07.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:07.858 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260507 02:56:07.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:07.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:07.858 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 02:56:07.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:07.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:07.858 | INFO | Uploaded 36GiB to GPU in 2.00021s (17.9981GiB/s) | 20260507 02:56:07.858 | INFO | Downloaded 34GiB to GPU in 1.9332s (17.5874GiB/s) | 20260507 02:56:13.000 | INFO | Analyzing chunk frames = [6542848,6543360) | 20260507 02:56:13.000 | INFO | MemInfo: { | 20260507 02:56:13.000 | INFO | "gpu" : " 17 GB", | 20260507 02:56:13.000 | INFO | "huge" : "", | 20260507 02:56:13.000 | INFO | "huge_shared" : " 27 GB", | 20260507 02:56:13.000 | INFO | "malloc" : " 14 GB", | 20260507 02:56:13.000 | INFO | "pinned" : " 29 GB" | 20260507 02:56:13.000 | INFO | } | 20260507 02:56:13.000 | INFO | Perf report for the last 5.14285 seconds: | 20260507 02:56:13.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:56:13.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:13.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:13.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:13.001 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:56:13.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:13.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:13.001 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:56:13.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:13.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:13.001 | INFO | Uploaded 36GiB to GPU in 1.97429s (18.2344GiB/s) | 20260507 02:56:13.001 | INFO | Downloaded 34GiB to GPU in 1.90705s (17.8285GiB/s) | 20260507 02:56:18.136 | INFO | Analyzing chunk frames = [6543360,6543872) | 20260507 02:56:18.136 | INFO | MemInfo: { | 20260507 02:56:18.136 | INFO | "gpu" : " 17 GB", | 20260507 02:56:18.136 | INFO | "huge" : "", | 20260507 02:56:18.136 | INFO | "huge_shared" : " 27 GB", | 20260507 02:56:18.136 | INFO | "malloc" : " 14 GB", | 20260507 02:56:18.136 | INFO | "pinned" : " 29 GB" | 20260507 02:56:18.136 | INFO | } | 20260507 02:56:18.136 | INFO | Perf report for the last 5.13559 seconds: | 20260507 02:56:18.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:56:18.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:18.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:18.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:18.136 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 02:56:18.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:18.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:18.136 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:56:18.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:18.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:18.136 | INFO | Uploaded 36GiB to GPU in 2.05506s (17.5177GiB/s) | 20260507 02:56:18.136 | INFO | Downloaded 34GiB to GPU in 1.94892s (17.4455GiB/s) | 20260507 02:56:23.270 | INFO | Analyzing chunk frames = [6543872,6544384) | 20260507 02:56:23.270 | INFO | MemInfo: { | 20260507 02:56:23.270 | INFO | "gpu" : " 17 GB", | 20260507 02:56:23.270 | INFO | "huge" : "", | 20260507 02:56:23.270 | INFO | "huge_shared" : " 27 GB", | 20260507 02:56:23.270 | INFO | "malloc" : " 14 GB", | 20260507 02:56:23.270 | INFO | "pinned" : " 29 GB" | 20260507 02:56:23.270 | INFO | } | 20260507 02:56:23.270 | INFO | Perf report for the last 5.13386 seconds: | 20260507 02:56:23.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:56:23.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:23.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:23.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:23.270 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:56:23.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:23.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:23.270 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:56:23.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:23.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:23.270 | INFO | Uploaded 36GiB to GPU in 2.05715s (17.4999GiB/s) | 20260507 02:56:23.270 | INFO | Downloaded 34GiB to GPU in 1.96399s (17.3117GiB/s) | 20260507 02:56:28.416 | INFO | Analyzing chunk frames = [6544384,6544896) | 20260507 02:56:28.416 | INFO | MemInfo: { | 20260507 02:56:28.416 | INFO | "gpu" : " 17 GB", | 20260507 02:56:28.416 | INFO | "huge" : "", | 20260507 02:56:28.416 | INFO | "huge_shared" : " 27 GB", | 20260507 02:56:28.416 | INFO | "malloc" : " 14 GB", | 20260507 02:56:28.416 | INFO | "pinned" : " 29 GB" | 20260507 02:56:28.416 | INFO | } | 20260507 02:56:28.416 | INFO | Perf report for the last 5.14579 seconds: | 20260507 02:56:28.416 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:56:28.416 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:28.416 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:28.416 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:28.416 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:56:28.416 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:28.416 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:28.416 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 02:56:28.416 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:28.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:28.416 | INFO | Uploaded 36GiB to GPU in 1.98026s (18.1794GiB/s) | 20260507 02:56:28.416 | INFO | Downloaded 34GiB to GPU in 1.99714s (17.0244GiB/s) | 20260507 02:56:33.549 | INFO | Analyzing chunk frames = [6544896,6545408) | 20260507 02:56:33.549 | INFO | MemInfo: { | 20260507 02:56:33.549 | INFO | "gpu" : " 17 GB", | 20260507 02:56:33.549 | INFO | "huge" : "", | 20260507 02:56:33.549 | INFO | "huge_shared" : " 27 GB", | 20260507 02:56:33.549 | INFO | "malloc" : " 14 GB", | 20260507 02:56:33.549 | INFO | "pinned" : " 29 GB" | 20260507 02:56:33.549 | INFO | } | 20260507 02:56:33.549 | INFO | Perf report for the last 5.13366 seconds: | 20260507 02:56:33.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:56:33.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:33.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:33.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:33.550 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:56:33.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:33.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:33.550 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:56:33.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:33.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:33.550 | INFO | Uploaded 36GiB to GPU in 1.99357s (18.0581GiB/s) | 20260507 02:56:33.550 | INFO | Downloaded 34GiB to GPU in 1.96931s (17.2649GiB/s) | 20260507 02:56:33.550 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:56:33.550 | INFO | Total mem usage: 18 GiB | 20260507 02:56:33.550 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:56:33.550 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -20%) | 20260507 02:56:33.550 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:56:33.550 | INFO | High Water Breakdown: | 20260507 02:56:33.550 | INFO | 61% : 9.15 GiB ( 80.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:56:33.550 | INFO | 38.9% : 5.84 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:56:33.550 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:56:33.550 | INFO | ------------------------------------------ | 20260507 02:56:33.550 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:56:33.550 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:56:33.550 | INFO | Total mem usage: 26.5 GiB | 20260507 02:56:33.550 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:56:33.550 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:56:33.550 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:56:33.550 | INFO | High Water Breakdown: | 20260507 02:56:33.550 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:56:33.550 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:56:33.550 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:56:33.550 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:56:33.550 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:56:33.550 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:56:33.550 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:56:33.550 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:56:33.550 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:56:33.550 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:56:33.550 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:56:33.550 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:56:33.550 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:56:33.550 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:56:33.550 | INFO | ------------------------------------------ | 20260507 02:56:33.550 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:56:33.550 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:56:33.550 | INFO | Total mem usage: 25.6 GiB | 20260507 02:56:33.550 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:56:33.550 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:56:33.550 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:56:33.550 | INFO | High Water Breakdown: | 20260507 02:56:33.550 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:56:33.550 | INFO | ------------------------------------------ | 20260507 02:56:33.550 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:56:33.550 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:56:33.550 | INFO | Total mem usage: 15.7 GiB | 20260507 02:56:33.550 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:56:33.550 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:56:33.550 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:56:33.550 | INFO | High Water Breakdown: | 20260507 02:56:33.550 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:56:33.550 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:56:33.550 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:56:33.550 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:56:33.550 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:56:33.551 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:56:33.551 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:56:33.551 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:56:33.551 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:56:33.551 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:56:33.551 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:56:33.551 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:56:33.551 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:56:33.551 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:56:33.551 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:56:33.551 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:56:33.551 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:56:33.551 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:56:33.551 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:56:33.551 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:56:33.551 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:56:33.551 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:56:33.551 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:56:33.551 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:56:33.551 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:56:33.551 | INFO | ------------------------------------------ | 20260507 02:56:33.551 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:56:33.551 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:56:33.551 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:56:33.551 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:56:33.551 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.52e+01, 1.34e+01 / 1.88e+00 | 20260507 02:56:33.551 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 9.39e-03 / 5.16e-03 | 20260507 02:56:33.551 | INFO | PerfReport: 11.63%, Baseline -- 3.57e+00 / 4.44e+00, 5.07e-01 / 2.50e-01 | 20260507 02:56:33.551 | INFO | PerfReport: 3.22%, Histogram -- 9.91e-01 / 1.48e+00, 1.70e-01 / 9.34e-02 | 20260507 02:56:33.551 | INFO | PerfReport: 16.16%, DME -- 3.97e+01 / 4.44e+01, 3.96e+00 / 2.37e+00 | 20260507 02:56:33.551 | INFO | PerfReport: 57.57%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.22e+00 / 1.45e+00 | 20260507 02:56:33.551 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.16e+00 / 2.74e+00, 4.14e-01 / 1.73e-01 | 20260507 02:56:33.551 | INFO | PerfReport: 3.07%, Metrics -- 9.43e-01 / 1.24e+00, 2.50e-01 / 1.18e-01 | 20260507 02:56:33.551 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.19e-01 / 3.71e-01, 5.07e-02 / 1.78e-02 | 20260507 02:56:33.551 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:56:33.551 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:56:33.551 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:56:33.551 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 02:56:33.551 | INFO | PerfReport: 21.81%, Upload -- 2.10e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260507 02:56:33.551 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260507 02:56:33.551 | INFO | PerfReport: 31.89%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 02:56:33.551 | INFO | PerfReport: 1.57%, LockUpload -- 1.52e+00 / 0.00e+00, 5.06e+00 / 0.00e+00 | 20260507 02:56:33.551 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260507 02:56:33.551 | INFO | PerfReport: 22.10%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 02:56:33.551 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260507 02:56:33.649 | INFO | Starting a round of Baz IO | 20260507 02:56:36.757 | INFO | Populating RTMetrics report | 20260507 02:56:36.757 | INFO | RTMetrics report populated | 20260507 02:56:36.757 | INFO | Opening tmp RTMetrics json file | 20260507 02:56:36.758 | INFO | Serializing RTMetrics to json | 20260507 02:56:36.759 | INFO | Serialization done, swapping tmp file now... | 20260507 02:56:36.761 | INFO | RTMetrics done | 20260507 02:56:36.846 | INFO | Finished a round of Baz IO: Wrote 1.767 GiB in 3.196 seconds | 20260507 02:56:36.846 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.95% / 0.00% / 67.05% | 20260507 02:56:36.846 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.218574% / 3.97364e-06% / 8.34465e-05% | 20260507 02:56:36.846 | INFO | Baz IO overall: | 20260507 02:56:36.846 | INFO | BytesWritten : 3369139781468 | 20260507 02:56:36.846 | INFO | NumEvents : 1619606649567 | 20260507 02:56:38.689 | INFO | Analyzing chunk frames = [6545408,6545920) | 20260507 02:56:38.689 | INFO | MemInfo: { | 20260507 02:56:38.689 | INFO | "gpu" : " 17 GB", | 20260507 02:56:38.689 | INFO | "huge" : "", | 20260507 02:56:38.689 | INFO | "huge_shared" : " 27 GB", | 20260507 02:56:38.689 | INFO | "malloc" : " 14 GB", | 20260507 02:56:38.689 | INFO | "pinned" : " 29 GB" | 20260507 02:56:38.689 | INFO | } | 20260507 02:56:38.689 | INFO | Perf report for the last 5.13952 seconds: | 20260507 02:56:38.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:56:38.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:38.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:38.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:38.689 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260507 02:56:38.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:38.689 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 02:56:38.689 | INFO | PRE_HQ : 59.3%, 1.8, 55.5ms, 0 | 20260507 02:56:38.689 | INFO | BAZWRITER : 61.8%, 3.3, 103.9ms, 0 | 20260507 02:56:38.689 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:56:38.689 | INFO | Uploaded 36GiB to GPU in 2.04473s (17.6062GiB/s) | 20260507 02:56:38.689 | INFO | Downloaded 38GiB to GPU in 2.17961s (17.4344GiB/s) | 20260507 02:56:43.834 | INFO | Analyzing chunk frames = [6545920,6546432) | 20260507 02:56:43.834 | INFO | MemInfo: { | 20260507 02:56:43.834 | INFO | "gpu" : " 17 GB", | 20260507 02:56:43.834 | INFO | "huge" : "", | 20260507 02:56:43.834 | INFO | "huge_shared" : " 27 GB", | 20260507 02:56:43.835 | INFO | "malloc" : " 14 GB", | 20260507 02:56:43.835 | INFO | "pinned" : " 29 GB" | 20260507 02:56:43.835 | INFO | } | 20260507 02:56:43.835 | INFO | Perf report for the last 5.1456 seconds: | 20260507 02:56:43.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:56:43.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:43.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:43.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:43.835 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:56:43.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:43.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:43.835 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:56:43.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:43.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:43.835 | INFO | Uploaded 36GiB to GPU in 2.03839s (17.661GiB/s) | 20260507 02:56:43.835 | INFO | Downloaded 34GiB to GPU in 1.93763s (17.5472GiB/s) | 20260507 02:56:48.973 | INFO | Analyzing chunk frames = [6546432,6546944) | 20260507 02:56:48.973 | INFO | MemInfo: { | 20260507 02:56:48.973 | INFO | "gpu" : " 17 GB", | 20260507 02:56:48.973 | INFO | "huge" : "", | 20260507 02:56:48.973 | INFO | "huge_shared" : " 27 GB", | 20260507 02:56:48.973 | INFO | "malloc" : " 14 GB", | 20260507 02:56:48.973 | INFO | "pinned" : " 29 GB" | 20260507 02:56:48.973 | INFO | } | 20260507 02:56:48.973 | INFO | Perf report for the last 5.13876 seconds: | 20260507 02:56:48.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:56:48.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:48.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:48.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:48.973 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 02:56:48.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:48.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:48.974 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:56:48.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:48.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:48.974 | INFO | Uploaded 36GiB to GPU in 2.01229s (17.8901GiB/s) | 20260507 02:56:48.974 | INFO | Downloaded 34GiB to GPU in 1.94006s (17.5252GiB/s) | 20260507 02:56:54.115 | INFO | Analyzing chunk frames = [6546944,6547456) | 20260507 02:56:54.115 | INFO | MemInfo: { | 20260507 02:56:54.115 | INFO | "gpu" : " 17 GB", | 20260507 02:56:54.115 | INFO | "huge" : "", | 20260507 02:56:54.115 | INFO | "huge_shared" : " 27 GB", | 20260507 02:56:54.115 | INFO | "malloc" : " 14 GB", | 20260507 02:56:54.115 | INFO | "pinned" : " 29 GB" | 20260507 02:56:54.115 | INFO | } | 20260507 02:56:54.115 | INFO | Perf report for the last 5.14173 seconds: | 20260507 02:56:54.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:56:54.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:54.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:54.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:54.115 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 02:56:54.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:54.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:54.115 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:56:54.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:54.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:54.116 | INFO | Uploaded 36GiB to GPU in 1.98469s (18.1388GiB/s) | 20260507 02:56:54.116 | INFO | Downloaded 34GiB to GPU in 1.92648s (17.6488GiB/s) | 20260507 02:56:59.251 | INFO | Analyzing chunk frames = [6547456,6547968) | 20260507 02:56:59.251 | INFO | MemInfo: { | 20260507 02:56:59.251 | INFO | "gpu" : " 17 GB", | 20260507 02:56:59.251 | INFO | "huge" : "", | 20260507 02:56:59.251 | INFO | "huge_shared" : " 27 GB", | 20260507 02:56:59.251 | INFO | "malloc" : " 14 GB", | 20260507 02:56:59.251 | INFO | "pinned" : " 29 GB" | 20260507 02:56:59.251 | INFO | } | 20260507 02:56:59.251 | INFO | Perf report for the last 5.13582 seconds: | 20260507 02:56:59.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:56:59.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:59.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:59.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:59.251 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 02:56:59.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:59.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:59.251 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:56:59.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:59.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:56:59.251 | INFO | Uploaded 36GiB to GPU in 2.05772s (17.4951GiB/s) | 20260507 02:56:59.251 | INFO | Downloaded 34GiB to GPU in 1.94892s (17.4456GiB/s) | 20260507 02:57:04.385 | INFO | Analyzing chunk frames = [6547968,6548480) | 20260507 02:57:04.385 | INFO | MemInfo: { | 20260507 02:57:04.385 | INFO | "gpu" : " 17 GB", | 20260507 02:57:04.386 | INFO | "huge" : "", | 20260507 02:57:04.386 | INFO | "huge_shared" : " 27 GB", | 20260507 02:57:04.386 | INFO | "malloc" : " 14 GB", | 20260507 02:57:04.386 | INFO | "pinned" : " 29 GB" | 20260507 02:57:04.386 | INFO | } | 20260507 02:57:04.386 | INFO | Perf report for the last 5.13481 seconds: | 20260507 02:57:04.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:57:04.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:04.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:04.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:04.386 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:57:04.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:04.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:04.386 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:57:04.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:04.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:04.386 | INFO | Uploaded 36GiB to GPU in 2.05871s (17.4867GiB/s) | 20260507 02:57:04.386 | INFO | Downloaded 34GiB to GPU in 1.96014s (17.3457GiB/s) | 20260507 02:57:09.521 | INFO | Analyzing chunk frames = [6548480,6548992) | 20260507 02:57:09.521 | INFO | MemInfo: { | 20260507 02:57:09.521 | INFO | "gpu" : " 17 GB", | 20260507 02:57:09.521 | INFO | "huge" : "", | 20260507 02:57:09.521 | INFO | "huge_shared" : " 27 GB", | 20260507 02:57:09.521 | INFO | "malloc" : " 14 GB", | 20260507 02:57:09.521 | INFO | "pinned" : " 29 GB" | 20260507 02:57:09.521 | INFO | } | 20260507 02:57:09.521 | INFO | Perf report for the last 5.13526 seconds: | 20260507 02:57:09.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:57:09.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:09.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:09.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:09.521 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 02:57:09.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:09.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:09.521 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 02:57:09.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:09.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:09.521 | INFO | Uploaded 36GiB to GPU in 1.99846s (18.0138GiB/s) | 20260507 02:57:09.521 | INFO | Downloaded 34GiB to GPU in 1.97862s (17.1837GiB/s) | 20260507 02:57:14.668 | INFO | Analyzing chunk frames = [6548992,6549504) | 20260507 02:57:14.668 | INFO | MemInfo: { | 20260507 02:57:14.668 | INFO | "gpu" : " 17 GB", | 20260507 02:57:14.668 | INFO | "huge" : "", | 20260507 02:57:14.668 | INFO | "huge_shared" : " 27 GB", | 20260507 02:57:14.668 | INFO | "malloc" : " 14 GB", | 20260507 02:57:14.668 | INFO | "pinned" : " 29 GB" | 20260507 02:57:14.668 | INFO | } | 20260507 02:57:14.668 | INFO | Perf report for the last 5.14727 seconds: | 20260507 02:57:14.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:57:14.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:14.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:14.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:14.668 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 02:57:14.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:14.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:14.668 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 02:57:14.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:14.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:14.669 | INFO | Uploaded 36GiB to GPU in 1.98656s (18.1218GiB/s) | 20260507 02:57:14.669 | INFO | Downloaded 34GiB to GPU in 1.97317s (17.2311GiB/s) | 20260507 02:57:14.766 | INFO | Starting a round of Baz IO | 20260507 02:57:17.881 | INFO | Populating RTMetrics report | 20260507 02:57:17.881 | INFO | RTMetrics report populated | 20260507 02:57:17.881 | INFO | Opening tmp RTMetrics json file | 20260507 02:57:17.886 | INFO | Serializing RTMetrics to json | 20260507 02:57:17.899 | INFO | Serialization done, swapping tmp file now... | 20260507 02:57:17.900 | INFO | RTMetrics done | 20260507 02:57:17.973 | INFO | Finished a round of Baz IO: Wrote 1.768 GiB in 3.206 seconds | 20260507 02:57:17.973 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.95% / 0.00% / 67.05% | 20260507 02:57:17.973 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.218248% / 3.97364e-06% / 5.5631e-05% | 20260507 02:57:17.973 | INFO | Baz IO overall: | 20260507 02:57:17.973 | INFO | BytesWritten : 3371038109011 | 20260507 02:57:17.973 | INFO | NumEvents : 1620503326463 | 20260507 02:57:19.811 | INFO | Analyzing chunk frames = [6549504,6550016) | 20260507 02:57:19.811 | INFO | MemInfo: { | 20260507 02:57:19.811 | INFO | "gpu" : " 17 GB", | 20260507 02:57:19.811 | INFO | "huge" : "", | 20260507 02:57:19.812 | INFO | "huge_shared" : " 27 GB", | 20260507 02:57:19.812 | INFO | "malloc" : " 14 GB", | 20260507 02:57:19.812 | INFO | "pinned" : " 29 GB" | 20260507 02:57:19.812 | INFO | } | 20260507 02:57:19.812 | INFO | Perf report for the last 5.14327 seconds: | 20260507 02:57:19.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:57:19.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:19.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:19.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:19.812 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260507 02:57:19.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:19.812 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 02:57:19.812 | INFO | PRE_HQ : 59.3%, 1.8, 55.5ms, 0 | 20260507 02:57:19.812 | INFO | BAZWRITER : 62.0%, 3.3, 104.8ms, 0 | 20260507 02:57:19.812 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:57:19.812 | INFO | Uploaded 36GiB to GPU in 2.04822s (17.5762GiB/s) | 20260507 02:57:19.812 | INFO | Downloaded 38GiB to GPU in 2.18257s (17.4107GiB/s) | 20260507 02:57:24.952 | INFO | Analyzing chunk frames = [6550016,6550528) | 20260507 02:57:24.952 | INFO | MemInfo: { | 20260507 02:57:24.952 | INFO | "gpu" : " 17 GB", | 20260507 02:57:24.952 | INFO | "huge" : "", | 20260507 02:57:24.952 | INFO | "huge_shared" : " 27 GB", | 20260507 02:57:24.952 | INFO | "malloc" : " 14 GB", | 20260507 02:57:24.952 | INFO | "pinned" : " 29 GB" | 20260507 02:57:24.952 | INFO | } | 20260507 02:57:24.952 | INFO | Perf report for the last 5.14015 seconds: | 20260507 02:57:24.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:57:24.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:24.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:24.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:24.952 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:57:24.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:24.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:24.952 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:57:24.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:24.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:24.952 | INFO | Uploaded 36GiB to GPU in 1.99963s (18.0034GiB/s) | 20260507 02:57:24.952 | INFO | Downloaded 34GiB to GPU in 1.93284s (17.5907GiB/s) | 20260507 02:57:30.082 | INFO | Analyzing chunk frames = [6550528,6551040) | 20260507 02:57:30.082 | INFO | MemInfo: { | 20260507 02:57:30.082 | INFO | "gpu" : " 17 GB", | 20260507 02:57:30.082 | INFO | "huge" : "", | 20260507 02:57:30.082 | INFO | "huge_shared" : " 27 GB", | 20260507 02:57:30.082 | INFO | "malloc" : " 14 GB", | 20260507 02:57:30.082 | INFO | "pinned" : " 29 GB" | 20260507 02:57:30.082 | INFO | } | 20260507 02:57:30.082 | INFO | Perf report for the last 5.1301 seconds: | 20260507 02:57:30.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:57:30.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:30.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:30.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:30.082 | INFO | ANALYSIS : 60.8%, 3.0, 93.9ms, 0 | 20260507 02:57:30.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:30.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:30.082 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:57:30.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:30.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:30.082 | INFO | Uploaded 36GiB to GPU in 2.05361s (17.5301GiB/s) | 20260507 02:57:30.082 | INFO | Downloaded 34GiB to GPU in 1.94834s (17.4507GiB/s) | 20260507 02:57:35.224 | INFO | Analyzing chunk frames = [6551040,6551552) | 20260507 02:57:35.224 | INFO | MemInfo: { | 20260507 02:57:35.224 | INFO | "gpu" : " 17 GB", | 20260507 02:57:35.224 | INFO | "huge" : "", | 20260507 02:57:35.224 | INFO | "huge_shared" : " 27 GB", | 20260507 02:57:35.224 | INFO | "malloc" : " 14 GB", | 20260507 02:57:35.224 | INFO | "pinned" : " 29 GB" | 20260507 02:57:35.224 | INFO | } | 20260507 02:57:35.224 | INFO | Perf report for the last 5.14173 seconds: | 20260507 02:57:35.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:57:35.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:35.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:35.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:35.224 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 02:57:35.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:35.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:35.224 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:57:35.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:35.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:35.224 | INFO | Uploaded 36GiB to GPU in 1.97757s (18.2042GiB/s) | 20260507 02:57:35.224 | INFO | Downloaded 34GiB to GPU in 1.90667s (17.8321GiB/s) | 20260507 02:57:40.368 | INFO | Analyzing chunk frames = [6551552,6552064) | 20260507 02:57:40.368 | INFO | MemInfo: { | 20260507 02:57:40.368 | INFO | "gpu" : " 17 GB", | 20260507 02:57:40.368 | INFO | "huge" : "", | 20260507 02:57:40.368 | INFO | "huge_shared" : " 27 GB", | 20260507 02:57:40.368 | INFO | "malloc" : " 14 GB", | 20260507 02:57:40.368 | INFO | "pinned" : " 29 GB" | 20260507 02:57:40.368 | INFO | } | 20260507 02:57:40.368 | INFO | Perf report for the last 5.14459 seconds: | 20260507 02:57:40.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:57:40.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:40.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:40.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:40.368 | INFO | ANALYSIS : 59.8%, 3.0, 93.2ms, 0 | 20260507 02:57:40.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:40.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:40.368 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:57:40.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:40.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:40.368 | INFO | Uploaded 36GiB to GPU in 1.99715s (18.0257GiB/s) | 20260507 02:57:40.368 | INFO | Downloaded 34GiB to GPU in 1.93771s (17.5465GiB/s) | 20260507 02:57:45.504 | INFO | Analyzing chunk frames = [6552064,6552576) | 20260507 02:57:45.504 | INFO | MemInfo: { | 20260507 02:57:45.504 | INFO | "gpu" : " 17 GB", | 20260507 02:57:45.504 | INFO | "huge" : "", | 20260507 02:57:45.504 | INFO | "huge_shared" : " 27 GB", | 20260507 02:57:45.504 | INFO | "malloc" : " 14 GB", | 20260507 02:57:45.504 | INFO | "pinned" : " 29 GB" | 20260507 02:57:45.504 | INFO | } | 20260507 02:57:45.504 | INFO | Perf report for the last 5.1361 seconds: | 20260507 02:57:45.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:57:45.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:45.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:45.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:45.504 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 02:57:45.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:45.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:45.504 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 02:57:45.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:45.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:45.505 | INFO | Uploaded 36GiB to GPU in 2.0376s (17.6678GiB/s) | 20260507 02:57:45.505 | INFO | Downloaded 34GiB to GPU in 1.93159s (17.6021GiB/s) | 20260507 02:57:50.645 | INFO | Analyzing chunk frames = [6552576,6553088) | 20260507 02:57:50.645 | INFO | MemInfo: { | 20260507 02:57:50.645 | INFO | "gpu" : " 17 GB", | 20260507 02:57:50.645 | INFO | "huge" : "", | 20260507 02:57:50.645 | INFO | "huge_shared" : " 27 GB", | 20260507 02:57:50.645 | INFO | "malloc" : " 14 GB", | 20260507 02:57:50.645 | INFO | "pinned" : " 29 GB" | 20260507 02:57:50.645 | INFO | } | 20260507 02:57:50.645 | INFO | Perf report for the last 5.14047 seconds: | 20260507 02:57:50.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:57:50.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:50.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:50.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:50.645 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 02:57:50.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:50.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:50.645 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 02:57:50.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:50.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:50.645 | INFO | Uploaded 36GiB to GPU in 1.97086s (18.2662GiB/s) | 20260507 02:57:50.645 | INFO | Downloaded 34GiB to GPU in 1.95132s (17.4241GiB/s) | 20260507 02:57:55.784 | INFO | Analyzing chunk frames = [6553088,6553600) | 20260507 02:57:55.784 | INFO | MemInfo: { | 20260507 02:57:55.784 | INFO | "gpu" : " 17 GB", | 20260507 02:57:55.784 | INFO | "huge" : "", | 20260507 02:57:55.784 | INFO | "huge_shared" : " 27 GB", | 20260507 02:57:55.785 | INFO | "malloc" : " 14 GB", | 20260507 02:57:55.785 | INFO | "pinned" : " 29 GB" | 20260507 02:57:55.785 | INFO | } | 20260507 02:57:55.785 | INFO | Perf report for the last 5.13976 seconds: | 20260507 02:57:55.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:57:55.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:55.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:55.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:55.785 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 02:57:55.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:55.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:55.785 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 02:57:55.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:55.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:57:55.785 | INFO | Uploaded 36GiB to GPU in 1.98302s (18.1541GiB/s) | 20260507 02:57:55.785 | INFO | Downloaded 34GiB to GPU in 1.9772s (17.1961GiB/s) | 20260507 02:57:55.785 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:57:55.785 | INFO | Total mem usage: 18 GiB | 20260507 02:57:55.785 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:57:55.785 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 02:57:55.785 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:57:55.785 | INFO | High Water Breakdown: | 20260507 02:57:55.785 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:57:55.785 | INFO | 39.8% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:57:55.785 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:57:55.785 | INFO | ------------------------------------------ | 20260507 02:57:55.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:57:55.785 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:57:55.785 | INFO | Total mem usage: 26.5 GiB | 20260507 02:57:55.785 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:57:55.785 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:57:55.785 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 02:57:55.785 | INFO | High Water Breakdown: | 20260507 02:57:55.785 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:57:55.785 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:57:55.785 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:57:55.785 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:57:55.785 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:57:55.785 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:57:55.785 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:57:55.785 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:57:55.785 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:57:55.785 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:57:55.785 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:57:55.785 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:57:55.785 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:57:55.785 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:57:55.785 | INFO | ------------------------------------------ | 20260507 02:57:55.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:57:55.785 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:57:55.785 | INFO | Total mem usage: 25.6 GiB | 20260507 02:57:55.785 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:57:55.785 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:57:55.785 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:57:55.785 | INFO | High Water Breakdown: | 20260507 02:57:55.785 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:57:55.785 | INFO | ------------------------------------------ | 20260507 02:57:55.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:57:55.785 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:57:55.785 | INFO | Total mem usage: 15.7 GiB | 20260507 02:57:55.785 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:57:55.785 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:57:55.785 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 02:57:55.785 | INFO | High Water Breakdown: | 20260507 02:57:55.785 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:57:55.785 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:57:55.785 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:57:55.785 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:57:55.785 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:57:55.785 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:57:55.785 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:57:55.785 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:57:55.785 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:57:55.785 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:57:55.785 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:57:55.785 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:57:55.785 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:57:55.785 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:57:55.785 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:57:55.785 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:57:55.785 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:57:55.785 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:57:55.785 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:57:55.785 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:57:55.785 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:57:55.785 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:57:55.785 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:57:55.786 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:57:55.786 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:57:55.786 | INFO | ------------------------------------------ | 20260507 02:57:55.786 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:57:55.786 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:57:55.786 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:57:55.786 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:57:55.786 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.51e+01, 1.33e+01 / 1.73e+00 | 20260507 02:57:55.786 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 9.85e-03 / 8.72e-03 | 20260507 02:57:55.786 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.46e+00, 5.14e-01 / 3.28e-01 | 20260507 02:57:55.786 | INFO | PerfReport: 3.23%, Histogram -- 9.94e-01 / 1.47e+00, 1.71e-01 / 1.11e-01 | 20260507 02:57:55.786 | INFO | PerfReport: 16.10%, DME -- 3.96e+01 / 4.42e+01, 3.90e+00 / 2.11e+00 | 20260507 02:57:55.786 | INFO | PerfReport: 57.65%, FrameLabeling -- 1.77e+01 / 2.05e+01, 1.33e+00 / 1.82e+00 | 20260507 02:57:55.786 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.15e+00 / 2.69e+00, 3.81e-01 / 2.32e-01 | 20260507 02:57:55.786 | INFO | PerfReport: 3.06%, Metrics -- 9.42e-01 / 1.28e+00, 2.51e-01 / 1.40e-01 | 20260507 02:57:55.786 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.19e-01 / 3.80e-01, 5.13e-02 / 2.59e-02 | 20260507 02:57:55.786 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:57:55.786 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:57:55.786 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:57:55.786 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 02:57:55.786 | INFO | PerfReport: 21.78%, Upload -- 2.10e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260507 02:57:55.786 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 02:57:55.786 | INFO | PerfReport: 31.93%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 02:57:55.786 | INFO | PerfReport: 1.58%, LockUpload -- 1.53e+00 / 0.00e+00, 5.03e+00 / 0.00e+00 | 20260507 02:57:55.786 | INFO | PerfReport: 1.18%, LockDownload -- 1.14e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260507 02:57:55.786 | INFO | PerfReport: 22.16%, LockCompute -- 2.14e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 02:57:55.786 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260507 02:57:55.881 | INFO | Starting a round of Baz IO | 20260507 02:57:58.992 | INFO | Populating RTMetrics report | 20260507 02:57:58.992 | INFO | RTMetrics report populated | 20260507 02:57:58.992 | INFO | Opening tmp RTMetrics json file | 20260507 02:57:58.993 | INFO | Serializing RTMetrics to json | 20260507 02:57:59.005 | INFO | Serialization done, swapping tmp file now... | 20260507 02:57:59.013 | INFO | RTMetrics done | 20260507 02:57:59.090 | INFO | Finished a round of Baz IO: Wrote 1.766 GiB in 3.209 seconds | 20260507 02:57:59.090 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.94% / 0.00% / 67.05% | 20260507 02:57:59.090 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.216726% / 0% / 4.76837e-05% | 20260507 02:57:59.090 | INFO | Baz IO overall: | 20260507 02:57:59.090 | INFO | BytesWritten : 3372933916047 | 20260507 02:57:59.091 | INFO | NumEvents : 1621397950987 | 20260507 02:58:00.925 | INFO | Analyzing chunk frames = [6553600,6554112) | 20260507 02:58:00.925 | INFO | MemInfo: { | 20260507 02:58:00.925 | INFO | "gpu" : " 17 GB", | 20260507 02:58:00.925 | INFO | "huge" : "", | 20260507 02:58:00.925 | INFO | "huge_shared" : " 27 GB", | 20260507 02:58:00.925 | INFO | "malloc" : " 14 GB", | 20260507 02:58:00.925 | INFO | "pinned" : " 29 GB" | 20260507 02:58:00.925 | INFO | } | 20260507 02:58:00.925 | INFO | Perf report for the last 5.14067 seconds: | 20260507 02:58:00.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:58:00.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:00.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:00.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:00.925 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 02:58:00.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:00.925 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 02:58:00.925 | INFO | PRE_HQ : 59.3%, 1.8, 55.8ms, 0 | 20260507 02:58:00.925 | INFO | BAZWRITER : 61.8%, 3.3, 104.6ms, 0 | 20260507 02:58:00.925 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:58:00.925 | INFO | Uploaded 36GiB to GPU in 2.03849s (17.6601GiB/s) | 20260507 02:58:00.926 | INFO | Downloaded 38GiB to GPU in 2.18519s (17.3898GiB/s) | 20260507 02:58:06.059 | INFO | Analyzing chunk frames = [6554112,6554624) | 20260507 02:58:06.059 | INFO | MemInfo: { | 20260507 02:58:06.059 | INFO | "gpu" : " 17 GB", | 20260507 02:58:06.059 | INFO | "huge" : "", | 20260507 02:58:06.059 | INFO | "huge_shared" : " 27 GB", | 20260507 02:58:06.059 | INFO | "malloc" : " 14 GB", | 20260507 02:58:06.059 | INFO | "pinned" : " 29 GB" | 20260507 02:58:06.059 | INFO | } | 20260507 02:58:06.059 | INFO | Perf report for the last 5.1338 seconds: | 20260507 02:58:06.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:58:06.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:06.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:06.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:06.059 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:58:06.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:06.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:06.059 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:58:06.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:06.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:06.059 | INFO | Uploaded 36GiB to GPU in 2.00779s (17.9302GiB/s) | 20260507 02:58:06.059 | INFO | Downloaded 34GiB to GPU in 1.93194s (17.5989GiB/s) | 20260507 02:58:11.200 | INFO | Analyzing chunk frames = [6554624,6555136) | 20260507 02:58:11.200 | INFO | MemInfo: { | 20260507 02:58:11.200 | INFO | "gpu" : " 17 GB", | 20260507 02:58:11.200 | INFO | "huge" : "", | 20260507 02:58:11.200 | INFO | "huge_shared" : " 27 GB", | 20260507 02:58:11.200 | INFO | "malloc" : " 14 GB", | 20260507 02:58:11.200 | INFO | "pinned" : " 29 GB" | 20260507 02:58:11.200 | INFO | } | 20260507 02:58:11.200 | INFO | Perf report for the last 5.14085 seconds: | 20260507 02:58:11.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:58:11.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:11.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:11.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:11.200 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:58:11.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:11.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:11.200 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 02:58:11.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:11.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:11.200 | INFO | Uploaded 36GiB to GPU in 2.01396s (17.8752GiB/s) | 20260507 02:58:11.200 | INFO | Downloaded 34GiB to GPU in 1.93858s (17.5386GiB/s) | 20260507 02:58:16.339 | INFO | Analyzing chunk frames = [6555136,6555648) | 20260507 02:58:16.339 | INFO | MemInfo: { | 20260507 02:58:16.339 | INFO | "gpu" : " 17 GB", | 20260507 02:58:16.339 | INFO | "huge" : "", | 20260507 02:58:16.339 | INFO | "huge_shared" : " 27 GB", | 20260507 02:58:16.339 | INFO | "malloc" : " 14 GB", | 20260507 02:58:16.339 | INFO | "pinned" : " 29 GB" | 20260507 02:58:16.339 | INFO | } | 20260507 02:58:16.339 | INFO | Perf report for the last 5.13891 seconds: | 20260507 02:58:16.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:58:16.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:16.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:16.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:16.339 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 02:58:16.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:16.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:16.339 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:58:16.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:16.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:16.339 | INFO | Uploaded 36GiB to GPU in 1.98821s (18.1067GiB/s) | 20260507 02:58:16.339 | INFO | Downloaded 34GiB to GPU in 1.92022s (17.7063GiB/s) | 20260507 02:58:21.477 | INFO | Analyzing chunk frames = [6555648,6556160) | 20260507 02:58:21.477 | INFO | MemInfo: { | 20260507 02:58:21.477 | INFO | "gpu" : " 17 GB", | 20260507 02:58:21.477 | INFO | "huge" : "", | 20260507 02:58:21.477 | INFO | "huge_shared" : " 27 GB", | 20260507 02:58:21.477 | INFO | "malloc" : " 14 GB", | 20260507 02:58:21.477 | INFO | "pinned" : " 29 GB" | 20260507 02:58:21.477 | INFO | } | 20260507 02:58:21.477 | INFO | Perf report for the last 5.13789 seconds: | 20260507 02:58:21.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:58:21.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:21.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:21.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:21.477 | INFO | ANALYSIS : 59.9%, 3.0, 93.2ms, 0 | 20260507 02:58:21.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:21.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:21.477 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 02:58:21.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:21.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:21.477 | INFO | Uploaded 36GiB to GPU in 2.0186s (17.8341GiB/s) | 20260507 02:58:21.477 | INFO | Downloaded 34GiB to GPU in 1.94932s (17.4419GiB/s) | 20260507 02:58:26.622 | INFO | Analyzing chunk frames = [6556160,6556672) | 20260507 02:58:26.622 | INFO | MemInfo: { | 20260507 02:58:26.622 | INFO | "gpu" : " 17 GB", | 20260507 02:58:26.622 | INFO | "huge" : "", | 20260507 02:58:26.622 | INFO | "huge_shared" : " 27 GB", | 20260507 02:58:26.622 | INFO | "malloc" : " 14 GB", | 20260507 02:58:26.622 | INFO | "pinned" : " 29 GB" | 20260507 02:58:26.622 | INFO | } | 20260507 02:58:26.622 | INFO | Perf report for the last 5.14536 seconds: | 20260507 02:58:26.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:58:26.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:26.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:26.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:26.622 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:58:26.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:26.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:26.622 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260507 02:58:26.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:26.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:26.622 | INFO | Uploaded 36GiB to GPU in 2.06011s (17.4748GiB/s) | 20260507 02:58:26.622 | INFO | Downloaded 34GiB to GPU in 1.95672s (17.376GiB/s) | 20260507 02:58:31.759 | INFO | Analyzing chunk frames = [6556672,6557184) | 20260507 02:58:31.759 | INFO | MemInfo: { | 20260507 02:58:31.759 | INFO | "gpu" : " 17 GB", | 20260507 02:58:31.759 | INFO | "huge" : "", | 20260507 02:58:31.759 | INFO | "huge_shared" : " 27 GB", | 20260507 02:58:31.759 | INFO | "malloc" : " 14 GB", | 20260507 02:58:31.759 | INFO | "pinned" : " 29 GB" | 20260507 02:58:31.759 | INFO | } | 20260507 02:58:31.759 | INFO | Perf report for the last 5.13677 seconds: | 20260507 02:58:31.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:58:31.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:31.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:31.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:31.759 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:58:31.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:31.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:31.759 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 02:58:31.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:31.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:31.759 | INFO | Uploaded 36GiB to GPU in 1.99074s (18.0837GiB/s) | 20260507 02:58:31.759 | INFO | Downloaded 34GiB to GPU in 1.98461s (17.1319GiB/s) | 20260507 02:58:36.894 | INFO | Analyzing chunk frames = [6557184,6557696) | 20260507 02:58:36.894 | INFO | MemInfo: { | 20260507 02:58:36.894 | INFO | "gpu" : " 17 GB", | 20260507 02:58:36.894 | INFO | "huge" : "", | 20260507 02:58:36.894 | INFO | "huge_shared" : " 27 GB", | 20260507 02:58:36.894 | INFO | "malloc" : " 14 GB", | 20260507 02:58:36.894 | INFO | "pinned" : " 29 GB" | 20260507 02:58:36.894 | INFO | } | 20260507 02:58:36.894 | INFO | Perf report for the last 5.13524 seconds: | 20260507 02:58:36.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:58:36.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:36.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:36.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:36.894 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 02:58:36.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:36.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:36.894 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:58:36.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:36.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:36.894 | INFO | Uploaded 36GiB to GPU in 2.00904s (17.919GiB/s) | 20260507 02:58:36.894 | INFO | Downloaded 34GiB to GPU in 1.96696s (17.2855GiB/s) | 20260507 02:58:36.991 | INFO | Starting a round of Baz IO | 20260507 02:58:40.100 | INFO | Populating RTMetrics report | 20260507 02:58:40.100 | INFO | RTMetrics report populated | 20260507 02:58:40.100 | INFO | Opening tmp RTMetrics json file | 20260507 02:58:40.105 | INFO | Serializing RTMetrics to json | 20260507 02:58:40.120 | INFO | Serialization done, swapping tmp file now... | 20260507 02:58:40.120 | INFO | RTMetrics done | 20260507 02:58:40.191 | INFO | Finished a round of Baz IO: Wrote 1.766 GiB in 3.2 seconds | 20260507 02:58:40.191 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.94% / 0.00% / 67.06% | 20260507 02:58:40.191 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.217621% / 0% / 8.34465e-05% | 20260507 02:58:40.191 | INFO | Baz IO overall: | 20260507 02:58:40.191 | INFO | BytesWritten : 3374830375732 | 20260507 02:58:40.191 | INFO | NumEvents : 1622293260910 | 20260507 02:58:42.034 | INFO | Analyzing chunk frames = [6557696,6558208) | 20260507 02:58:42.034 | INFO | MemInfo: { | 20260507 02:58:42.034 | INFO | "gpu" : " 17 GB", | 20260507 02:58:42.034 | INFO | "huge" : "", | 20260507 02:58:42.034 | INFO | "huge_shared" : " 27 GB", | 20260507 02:58:42.034 | INFO | "malloc" : " 14 GB", | 20260507 02:58:42.034 | INFO | "pinned" : " 29 GB" | 20260507 02:58:42.034 | INFO | } | 20260507 02:58:42.034 | INFO | Perf report for the last 5.13963 seconds: | 20260507 02:58:42.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:58:42.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:42.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:42.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:42.034 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 02:58:42.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:42.034 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 02:58:42.034 | INFO | PRE_HQ : 59.3%, 1.8, 55.5ms, 0 | 20260507 02:58:42.034 | INFO | BAZWRITER : 61.9%, 3.3, 104.0ms, 0 | 20260507 02:58:42.034 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:58:42.034 | INFO | Uploaded 36GiB to GPU in 2.05051s (17.5567GiB/s) | 20260507 02:58:42.034 | INFO | Downloaded 38GiB to GPU in 2.18484s (17.3926GiB/s) | 20260507 02:58:47.173 | INFO | Analyzing chunk frames = [6558208,6558720) | 20260507 02:58:47.173 | INFO | MemInfo: { | 20260507 02:58:47.173 | INFO | "gpu" : " 17 GB", | 20260507 02:58:47.173 | INFO | "huge" : "", | 20260507 02:58:47.173 | INFO | "huge_shared" : " 27 GB", | 20260507 02:58:47.173 | INFO | "malloc" : " 14 GB", | 20260507 02:58:47.173 | INFO | "pinned" : " 29 GB" | 20260507 02:58:47.173 | INFO | } | 20260507 02:58:47.173 | INFO | Perf report for the last 5.13947 seconds: | 20260507 02:58:47.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:58:47.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:47.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:47.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:47.173 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 02:58:47.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:47.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:47.173 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 02:58:47.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:47.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:47.173 | INFO | Uploaded 36GiB to GPU in 2.00539s (17.9516GiB/s) | 20260507 02:58:47.173 | INFO | Downloaded 34GiB to GPU in 1.93146s (17.6032GiB/s) | 20260507 02:58:52.318 | INFO | Analyzing chunk frames = [6558720,6559232) | 20260507 02:58:52.318 | INFO | MemInfo: { | 20260507 02:58:52.318 | INFO | "gpu" : " 17 GB", | 20260507 02:58:52.318 | INFO | "huge" : "", | 20260507 02:58:52.318 | INFO | "huge_shared" : " 27 GB", | 20260507 02:58:52.318 | INFO | "malloc" : " 14 GB", | 20260507 02:58:52.318 | INFO | "pinned" : " 29 GB" | 20260507 02:58:52.318 | INFO | } | 20260507 02:58:52.318 | INFO | Perf report for the last 5.14483 seconds: | 20260507 02:58:52.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:58:52.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:52.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:52.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:52.318 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 02:58:52.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:52.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:52.318 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:58:52.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:52.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:52.318 | INFO | Uploaded 36GiB to GPU in 1.99917s (18.0074GiB/s) | 20260507 02:58:52.318 | INFO | Downloaded 34GiB to GPU in 1.93679s (17.5548GiB/s) | 20260507 02:58:57.458 | INFO | Analyzing chunk frames = [6559232,6559744) | 20260507 02:58:57.458 | INFO | MemInfo: { | 20260507 02:58:57.458 | INFO | "gpu" : " 17 GB", | 20260507 02:58:57.458 | INFO | "huge" : "", | 20260507 02:58:57.458 | INFO | "huge_shared" : " 27 GB", | 20260507 02:58:57.458 | INFO | "malloc" : " 14 GB", | 20260507 02:58:57.458 | INFO | "pinned" : " 29 GB" | 20260507 02:58:57.458 | INFO | } | 20260507 02:58:57.458 | INFO | Perf report for the last 5.14001 seconds: | 20260507 02:58:57.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:58:57.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:57.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:57.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:57.458 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 02:58:57.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:57.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:57.458 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:58:57.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:57.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:58:57.458 | INFO | Uploaded 36GiB to GPU in 1.99834s (18.015GiB/s) | 20260507 02:58:57.458 | INFO | Downloaded 34GiB to GPU in 1.92315s (17.6794GiB/s) | 20260507 02:59:02.596 | INFO | Analyzing chunk frames = [6559744,6560256) | 20260507 02:59:02.596 | INFO | MemInfo: { | 20260507 02:59:02.596 | INFO | "gpu" : " 17 GB", | 20260507 02:59:02.596 | INFO | "huge" : "", | 20260507 02:59:02.596 | INFO | "huge_shared" : " 27 GB", | 20260507 02:59:02.596 | INFO | "malloc" : " 14 GB", | 20260507 02:59:02.596 | INFO | "pinned" : " 29 GB" | 20260507 02:59:02.596 | INFO | } | 20260507 02:59:02.596 | INFO | Perf report for the last 5.13778 seconds: | 20260507 02:59:02.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:59:02.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:02.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:02.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:02.596 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 02:59:02.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:02.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:02.596 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 02:59:02.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:02.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:02.596 | INFO | Uploaded 36GiB to GPU in 2.02007s (17.8212GiB/s) | 20260507 02:59:02.596 | INFO | Downloaded 34GiB to GPU in 1.9391s (17.5339GiB/s) | 20260507 02:59:07.736 | INFO | Analyzing chunk frames = [6560256,6560768) | 20260507 02:59:07.736 | INFO | MemInfo: { | 20260507 02:59:07.736 | INFO | "gpu" : " 17 GB", | 20260507 02:59:07.736 | INFO | "huge" : "", | 20260507 02:59:07.736 | INFO | "huge_shared" : " 27 GB", | 20260507 02:59:07.736 | INFO | "malloc" : " 14 GB", | 20260507 02:59:07.736 | INFO | "pinned" : " 29 GB" | 20260507 02:59:07.736 | INFO | } | 20260507 02:59:07.736 | INFO | Perf report for the last 5.14056 seconds: | 20260507 02:59:07.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:59:07.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:07.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:07.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:07.737 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 02:59:07.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:07.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:07.737 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 02:59:07.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:07.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:07.737 | INFO | Uploaded 36GiB to GPU in 2.03277s (17.7098GiB/s) | 20260507 02:59:07.737 | INFO | Downloaded 34GiB to GPU in 1.93228s (17.5958GiB/s) | 20260507 02:59:12.873 | INFO | Analyzing chunk frames = [6560768,6561280) | 20260507 02:59:12.873 | INFO | MemInfo: { | 20260507 02:59:12.873 | INFO | "gpu" : " 17 GB", | 20260507 02:59:12.873 | INFO | "huge" : "", | 20260507 02:59:12.873 | INFO | "huge_shared" : " 27 GB", | 20260507 02:59:12.873 | INFO | "malloc" : " 14 GB", | 20260507 02:59:12.873 | INFO | "pinned" : " 29 GB" | 20260507 02:59:12.873 | INFO | } | 20260507 02:59:12.873 | INFO | Perf report for the last 5.13684 seconds: | 20260507 02:59:12.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:59:12.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:12.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:12.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:12.873 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 02:59:12.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:12.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:12.873 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 02:59:12.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:12.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:12.874 | INFO | Uploaded 36GiB to GPU in 1.98266s (18.1574GiB/s) | 20260507 02:59:12.874 | INFO | Downloaded 34GiB to GPU in 1.95596s (17.3827GiB/s) | 20260507 02:59:18.012 | INFO | Analyzing chunk frames = [6561280,6561792) | 20260507 02:59:18.012 | INFO | MemInfo: { | 20260507 02:59:18.012 | INFO | "gpu" : " 17 GB", | 20260507 02:59:18.012 | INFO | "huge" : "", | 20260507 02:59:18.012 | INFO | "huge_shared" : " 27 GB", | 20260507 02:59:18.012 | INFO | "malloc" : " 14 GB", | 20260507 02:59:18.012 | INFO | "pinned" : " 29 GB" | 20260507 02:59:18.012 | INFO | } | 20260507 02:59:18.012 | INFO | Perf report for the last 5.13865 seconds: | 20260507 02:59:18.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:59:18.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:18.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:18.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:18.012 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 02:59:18.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:18.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:18.012 | INFO | PRE_HQ : 30.0%, 1.0, 15.6ms, 0 | 20260507 02:59:18.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:18.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:18.012 | INFO | Uploaded 36GiB to GPU in 1.99926s (18.0066GiB/s) | 20260507 02:59:18.012 | INFO | Downloaded 34GiB to GPU in 1.96784s (17.2778GiB/s) | 20260507 02:59:18.012 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 02:59:18.012 | INFO | Total mem usage: 18 GiB | 20260507 02:59:18.012 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 02:59:18.012 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 02:59:18.012 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 02:59:18.012 | INFO | High Water Breakdown: | 20260507 02:59:18.012 | INFO | 60.2% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 02:59:18.012 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 02:59:18.012 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 02:59:18.012 | INFO | ------------------------------------------ | 20260507 02:59:18.012 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:59:18.012 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 02:59:18.012 | INFO | Total mem usage: 26.5 GiB | 20260507 02:59:18.012 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 02:59:18.012 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 02:59:18.012 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 02:59:18.012 | INFO | High Water Breakdown: | 20260507 02:59:18.012 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:59:18.013 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:59:18.013 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:59:18.013 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:59:18.013 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:59:18.013 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:59:18.013 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 02:59:18.013 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 02:59:18.013 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:59:18.013 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 02:59:18.013 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 02:59:18.013 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 02:59:18.013 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 02:59:18.013 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 02:59:18.013 | INFO | ------------------------------------------ | 20260507 02:59:18.013 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:59:18.013 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 02:59:18.013 | INFO | Total mem usage: 25.6 GiB | 20260507 02:59:18.013 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 02:59:18.013 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 02:59:18.013 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 02:59:18.013 | INFO | High Water Breakdown: | 20260507 02:59:18.013 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 02:59:18.013 | INFO | ------------------------------------------ | 20260507 02:59:18.013 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:59:18.013 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 02:59:18.013 | INFO | Total mem usage: 15.7 GiB | 20260507 02:59:18.013 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 02:59:18.013 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 02:59:18.013 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 02:59:18.013 | INFO | High Water Breakdown: | 20260507 02:59:18.013 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 02:59:18.013 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 02:59:18.013 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 02:59:18.013 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 02:59:18.013 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 02:59:18.013 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 02:59:18.013 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 02:59:18.013 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 02:59:18.013 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 02:59:18.013 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 02:59:18.013 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 02:59:18.013 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 02:59:18.013 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 02:59:18.013 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 02:59:18.013 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 02:59:18.013 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 02:59:18.013 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 02:59:18.013 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 02:59:18.013 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 02:59:18.013 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 02:59:18.013 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 02:59:18.013 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 02:59:18.013 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 02:59:18.013 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 02:59:18.013 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 02:59:18.013 | INFO | ------------------------------------------ | 20260507 02:59:18.013 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 02:59:18.013 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 02:59:18.013 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:59:18.013 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:59:18.013 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.50e+01, 1.34e+01 / 1.97e+00 | 20260507 02:59:18.013 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 1.02e-01, 9.78e-03 / 5.08e-03 | 20260507 02:59:18.013 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.42e+00, 5.17e-01 / 2.91e-01 | 20260507 02:59:18.013 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.46e+00, 1.74e-01 / 1.22e-01 | 20260507 02:59:18.013 | INFO | PerfReport: 16.07%, DME -- 3.96e+01 / 4.41e+01, 3.86e+00 / 2.19e+00 | 20260507 02:59:18.013 | INFO | PerfReport: 57.67%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.35e+00 / 1.58e+00 | 20260507 02:59:18.013 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.14e+00 / 2.69e+00, 3.79e-01 / 2.23e-01 | 20260507 02:59:18.013 | INFO | PerfReport: 3.06%, Metrics -- 9.41e-01 / 1.27e+00, 2.49e-01 / 1.55e-01 | 20260507 02:59:18.013 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.20e-01 / 3.70e-01, 5.05e-02 / 1.66e-02 | 20260507 02:59:18.013 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 02:59:18.013 | INFO | PerfReport: Gathered 1536 samples. | 20260507 02:59:18.013 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 02:59:18.013 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 02:59:18.013 | INFO | PerfReport: 21.77%, Upload -- 2.10e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260507 02:59:18.013 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260507 02:59:18.013 | INFO | PerfReport: 31.97%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 02:59:18.013 | INFO | PerfReport: 1.48%, LockUpload -- 1.43e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 02:59:18.013 | INFO | PerfReport: 1.23%, LockDownload -- 1.19e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260507 02:59:18.013 | INFO | PerfReport: 22.18%, LockCompute -- 2.14e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 02:59:18.013 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260507 02:59:18.108 | INFO | Starting a round of Baz IO | 20260507 02:59:21.217 | INFO | Populating RTMetrics report | 20260507 02:59:21.217 | INFO | RTMetrics report populated | 20260507 02:59:21.217 | INFO | Opening tmp RTMetrics json file | 20260507 02:59:21.236 | INFO | Serializing RTMetrics to json | 20260507 02:59:21.237 | INFO | Serialization done, swapping tmp file now... | 20260507 02:59:21.238 | INFO | RTMetrics done | 20260507 02:59:21.308 | INFO | Finished a round of Baz IO: Wrote 1.765 GiB in 3.199 seconds | 20260507 02:59:21.308 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.94% / 0.00% / 67.06% | 20260507 02:59:21.308 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.216929% / 3.97364e-06% / 5.5631e-05% | 20260507 02:59:21.308 | INFO | Baz IO overall: | 20260507 02:59:21.308 | INFO | BytesWritten : 3376725063141 | 20260507 02:59:21.308 | INFO | NumEvents : 1623187225608 | 20260507 02:59:23.150 | INFO | Analyzing chunk frames = [6561792,6562304) | 20260507 02:59:23.150 | INFO | MemInfo: { | 20260507 02:59:23.150 | INFO | "gpu" : " 17 GB", | 20260507 02:59:23.150 | INFO | "huge" : "", | 20260507 02:59:23.150 | INFO | "huge_shared" : " 27 GB", | 20260507 02:59:23.150 | INFO | "malloc" : " 14 GB", | 20260507 02:59:23.150 | INFO | "pinned" : " 29 GB" | 20260507 02:59:23.150 | INFO | } | 20260507 02:59:23.150 | INFO | Perf report for the last 5.13792 seconds: | 20260507 02:59:23.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:59:23.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:23.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:23.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:23.150 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 02:59:23.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:23.150 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 02:59:23.150 | INFO | PRE_HQ : 59.4%, 1.8, 56.1ms, 0 | 20260507 02:59:23.150 | INFO | BAZWRITER : 61.8%, 3.3, 105.3ms, 0 | 20260507 02:59:23.150 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 02:59:23.150 | INFO | Uploaded 36GiB to GPU in 2.05473s (17.5206GiB/s) | 20260507 02:59:23.150 | INFO | Downloaded 38GiB to GPU in 2.18249s (17.4113GiB/s) | 20260507 02:59:28.287 | INFO | Analyzing chunk frames = [6562304,6562816) | 20260507 02:59:28.287 | INFO | MemInfo: { | 20260507 02:59:28.287 | INFO | "gpu" : " 17 GB", | 20260507 02:59:28.287 | INFO | "huge" : "", | 20260507 02:59:28.287 | INFO | "huge_shared" : " 27 GB", | 20260507 02:59:28.287 | INFO | "malloc" : " 14 GB", | 20260507 02:59:28.287 | INFO | "pinned" : " 29 GB" | 20260507 02:59:28.287 | INFO | } | 20260507 02:59:28.287 | INFO | Perf report for the last 5.13696 seconds: | 20260507 02:59:28.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:59:28.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:28.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:28.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:28.287 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 02:59:28.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:28.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:28.287 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 02:59:28.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:28.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:28.287 | INFO | Uploaded 36GiB to GPU in 2.03476s (17.6925GiB/s) | 20260507 02:59:28.287 | INFO | Downloaded 34GiB to GPU in 1.93373s (17.5826GiB/s) | 20260507 02:59:33.430 | INFO | Analyzing chunk frames = [6562816,6563328) | 20260507 02:59:33.430 | INFO | MemInfo: { | 20260507 02:59:33.431 | INFO | "gpu" : " 17 GB", | 20260507 02:59:33.431 | INFO | "huge" : "", | 20260507 02:59:33.431 | INFO | "huge_shared" : " 27 GB", | 20260507 02:59:33.431 | INFO | "malloc" : " 14 GB", | 20260507 02:59:33.431 | INFO | "pinned" : " 29 GB" | 20260507 02:59:33.431 | INFO | } | 20260507 02:59:33.431 | INFO | Perf report for the last 5.14371 seconds: | 20260507 02:59:33.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:59:33.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:33.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:33.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:33.431 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 02:59:33.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:33.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:33.431 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 02:59:33.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:33.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:33.431 | INFO | Uploaded 36GiB to GPU in 1.98835s (18.1054GiB/s) | 20260507 02:59:33.431 | INFO | Downloaded 34GiB to GPU in 1.93439s (17.5766GiB/s) | 20260507 02:59:38.578 | INFO | Analyzing chunk frames = [6563328,6563840) | 20260507 02:59:38.578 | INFO | MemInfo: { | 20260507 02:59:38.578 | INFO | "gpu" : " 17 GB", | 20260507 02:59:38.578 | INFO | "huge" : "", | 20260507 02:59:38.578 | INFO | "huge_shared" : " 27 GB", | 20260507 02:59:38.578 | INFO | "malloc" : " 14 GB", | 20260507 02:59:38.578 | INFO | "pinned" : " 29 GB" | 20260507 02:59:38.578 | INFO | } | 20260507 02:59:38.578 | INFO | Perf report for the last 5.14719 seconds: | 20260507 02:59:38.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:59:38.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:38.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:38.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:38.578 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 02:59:38.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:38.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:38.578 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 02:59:38.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:38.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:38.578 | INFO | Uploaded 36GiB to GPU in 1.97965s (18.185GiB/s) | 20260507 02:59:38.578 | INFO | Downloaded 34GiB to GPU in 1.9098s (17.8029GiB/s) | 20260507 02:59:43.707 | INFO | Analyzing chunk frames = [6563840,6564352) | 20260507 02:59:43.707 | INFO | MemInfo: { | 20260507 02:59:43.707 | INFO | "gpu" : " 17 GB", | 20260507 02:59:43.707 | INFO | "huge" : "", | 20260507 02:59:43.707 | INFO | "huge_shared" : " 27 GB", | 20260507 02:59:43.707 | INFO | "malloc" : " 14 GB", | 20260507 02:59:43.707 | INFO | "pinned" : " 29 GB" | 20260507 02:59:43.707 | INFO | } | 20260507 02:59:43.707 | INFO | Perf report for the last 5.12903 seconds: | 20260507 02:59:43.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:59:43.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:43.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:43.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:43.707 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 02:59:43.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:43.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:43.707 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 02:59:43.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:43.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:43.707 | INFO | Uploaded 36GiB to GPU in 2.05818s (17.4912GiB/s) | 20260507 02:59:43.707 | INFO | Downloaded 34GiB to GPU in 1.94684s (17.4642GiB/s) | 20260507 02:59:48.843 | INFO | Analyzing chunk frames = [6564352,6564864) | 20260507 02:59:48.843 | INFO | MemInfo: { | 20260507 02:59:48.843 | INFO | "gpu" : " 17 GB", | 20260507 02:59:48.843 | INFO | "huge" : "", | 20260507 02:59:48.843 | INFO | "huge_shared" : " 27 GB", | 20260507 02:59:48.843 | INFO | "malloc" : " 14 GB", | 20260507 02:59:48.843 | INFO | "pinned" : " 29 GB" | 20260507 02:59:48.843 | INFO | } | 20260507 02:59:48.843 | INFO | Perf report for the last 5.13645 seconds: | 20260507 02:59:48.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:59:48.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:48.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:48.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:48.843 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 02:59:48.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:48.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:48.843 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 02:59:48.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:48.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:48.844 | INFO | Uploaded 36GiB to GPU in 2.0589s (17.4851GiB/s) | 20260507 02:59:48.844 | INFO | Downloaded 34GiB to GPU in 1.96412s (17.3105GiB/s) | 20260507 02:59:53.988 | INFO | Analyzing chunk frames = [6564864,6565376) | 20260507 02:59:53.988 | INFO | MemInfo: { | 20260507 02:59:53.988 | INFO | "gpu" : " 17 GB", | 20260507 02:59:53.988 | INFO | "huge" : "", | 20260507 02:59:53.988 | INFO | "huge_shared" : " 27 GB", | 20260507 02:59:53.988 | INFO | "malloc" : " 14 GB", | 20260507 02:59:53.988 | INFO | "pinned" : " 29 GB" | 20260507 02:59:53.988 | INFO | } | 20260507 02:59:53.988 | INFO | Perf report for the last 5.14473 seconds: | 20260507 02:59:53.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:59:53.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:53.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:53.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:53.988 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 02:59:53.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:53.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:53.988 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 02:59:53.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:53.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:53.988 | INFO | Uploaded 36GiB to GPU in 1.99779s (18.0199GiB/s) | 20260507 02:59:53.988 | INFO | Downloaded 34GiB to GPU in 1.99503s (17.0423GiB/s) | 20260507 02:59:59.130 | INFO | Analyzing chunk frames = [6565376,6565888) | 20260507 02:59:59.130 | INFO | MemInfo: { | 20260507 02:59:59.130 | INFO | "gpu" : " 17 GB", | 20260507 02:59:59.130 | INFO | "huge" : "", | 20260507 02:59:59.130 | INFO | "huge_shared" : " 27 GB", | 20260507 02:59:59.130 | INFO | "malloc" : " 14 GB", | 20260507 02:59:59.130 | INFO | "pinned" : " 29 GB" | 20260507 02:59:59.130 | INFO | } | 20260507 02:59:59.130 | INFO | Perf report for the last 5.14171 seconds: | 20260507 02:59:59.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 02:59:59.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:59.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:59.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:59.130 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 02:59:59.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:59.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:59.130 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 02:59:59.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:59.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 02:59:59.130 | INFO | Uploaded 36GiB to GPU in 2.00389s (17.965GiB/s) | 20260507 02:59:59.130 | INFO | Downloaded 34GiB to GPU in 1.96716s (17.2838GiB/s) | 20260507 02:59:59.227 | INFO | Starting a round of Baz IO | 20260507 03:00:02.344 | INFO | Populating RTMetrics report | 20260507 03:00:02.345 | INFO | RTMetrics report populated | 20260507 03:00:02.345 | INFO | Opening tmp RTMetrics json file | 20260507 03:00:02.354 | INFO | Serializing RTMetrics to json | 20260507 03:00:02.362 | INFO | Serialization done, swapping tmp file now... | 20260507 03:00:02.363 | INFO | RTMetrics done | 20260507 03:00:02.432 | INFO | Finished a round of Baz IO: Wrote 1.765 GiB in 3.204 seconds | 20260507 03:00:02.432 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.94% / 0.00% / 67.06% | 20260507 03:00:02.432 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.216027% / 7.94729e-06% / 7.54992e-05% | 20260507 03:00:02.432 | INFO | Baz IO overall: | 20260507 03:00:02.432 | INFO | BytesWritten : 3378620264752 | 20260507 03:00:02.432 | INFO | NumEvents : 1624082264037 | 20260507 03:00:04.268 | INFO | Analyzing chunk frames = [6565888,6566400) | 20260507 03:00:04.268 | INFO | MemInfo: { | 20260507 03:00:04.268 | INFO | "gpu" : " 17 GB", | 20260507 03:00:04.268 | INFO | "huge" : "", | 20260507 03:00:04.268 | INFO | "huge_shared" : " 27 GB", | 20260507 03:00:04.268 | INFO | "malloc" : " 14 GB", | 20260507 03:00:04.268 | INFO | "pinned" : " 29 GB" | 20260507 03:00:04.268 | INFO | } | 20260507 03:00:04.269 | INFO | Perf report for the last 5.13875 seconds: | 20260507 03:00:04.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:00:04.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:04.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:04.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:04.269 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 03:00:04.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:04.269 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260507 03:00:04.269 | INFO | PRE_HQ : 60.2%, 1.9, 59.5ms, 0 | 20260507 03:00:04.269 | INFO | BAZWRITER : 62.0%, 3.4, 108.4ms, 0 | 20260507 03:00:04.269 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:00:04.269 | INFO | Uploaded 36GiB to GPU in 2.0576s (17.4961GiB/s) | 20260507 03:00:04.269 | INFO | Downloaded 38GiB to GPU in 2.18577s (17.3852GiB/s) | 20260507 03:00:09.409 | INFO | Analyzing chunk frames = [6566400,6566912) | 20260507 03:00:09.409 | INFO | MemInfo: { | 20260507 03:00:09.409 | INFO | "gpu" : " 17 GB", | 20260507 03:00:09.409 | INFO | "huge" : "", | 20260507 03:00:09.409 | INFO | "huge_shared" : " 27 GB", | 20260507 03:00:09.409 | INFO | "malloc" : " 14 GB", | 20260507 03:00:09.409 | INFO | "pinned" : " 29 GB" | 20260507 03:00:09.409 | INFO | } | 20260507 03:00:09.409 | INFO | Perf report for the last 5.14023 seconds: | 20260507 03:00:09.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:00:09.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:09.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:09.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:09.409 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 03:00:09.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:09.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:09.409 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 03:00:09.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:09.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:09.409 | INFO | Uploaded 36GiB to GPU in 2.05201s (17.5438GiB/s) | 20260507 03:00:09.409 | INFO | Downloaded 34GiB to GPU in 1.94187s (17.5089GiB/s) | 20260507 03:00:14.545 | INFO | Analyzing chunk frames = [6566912,6567424) | 20260507 03:00:14.545 | INFO | MemInfo: { | 20260507 03:00:14.545 | INFO | "gpu" : " 17 GB", | 20260507 03:00:14.545 | INFO | "huge" : "", | 20260507 03:00:14.545 | INFO | "huge_shared" : " 27 GB", | 20260507 03:00:14.545 | INFO | "malloc" : " 14 GB", | 20260507 03:00:14.545 | INFO | "pinned" : " 29 GB" | 20260507 03:00:14.545 | INFO | } | 20260507 03:00:14.545 | INFO | Perf report for the last 5.1362 seconds: | 20260507 03:00:14.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:00:14.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:14.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:14.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:14.545 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 03:00:14.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:14.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:14.545 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:00:14.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:14.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:14.545 | INFO | Uploaded 36GiB to GPU in 2.0103s (17.9078GiB/s) | 20260507 03:00:14.545 | INFO | Downloaded 34GiB to GPU in 1.9403s (17.5231GiB/s) | 20260507 03:00:19.684 | INFO | Analyzing chunk frames = [6567424,6567936) | 20260507 03:00:19.684 | INFO | MemInfo: { | 20260507 03:00:19.684 | INFO | "gpu" : " 17 GB", | 20260507 03:00:19.684 | INFO | "huge" : "", | 20260507 03:00:19.684 | INFO | "huge_shared" : " 27 GB", | 20260507 03:00:19.684 | INFO | "malloc" : " 14 GB", | 20260507 03:00:19.684 | INFO | "pinned" : " 29 GB" | 20260507 03:00:19.685 | INFO | } | 20260507 03:00:19.685 | INFO | Perf report for the last 5.13952 seconds: | 20260507 03:00:19.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:00:19.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:19.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:19.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:19.685 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:00:19.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:19.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:19.685 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:00:19.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:19.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:19.685 | INFO | Uploaded 36GiB to GPU in 1.97198s (18.2557GiB/s) | 20260507 03:00:19.685 | INFO | Downloaded 34GiB to GPU in 1.90949s (17.8058GiB/s) | 20260507 03:00:24.821 | INFO | Analyzing chunk frames = [6567936,6568448) | 20260507 03:00:24.821 | INFO | MemInfo: { | 20260507 03:00:24.821 | INFO | "gpu" : " 17 GB", | 20260507 03:00:24.821 | INFO | "huge" : "", | 20260507 03:00:24.821 | INFO | "huge_shared" : " 27 GB", | 20260507 03:00:24.821 | INFO | "malloc" : " 14 GB", | 20260507 03:00:24.821 | INFO | "pinned" : " 29 GB" | 20260507 03:00:24.821 | INFO | } | 20260507 03:00:24.821 | INFO | Perf report for the last 5.13675 seconds: | 20260507 03:00:24.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:00:24.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:24.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:24.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:24.821 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 03:00:24.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:24.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:24.821 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 03:00:24.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:24.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:24.821 | INFO | Uploaded 36GiB to GPU in 2.01539s (17.8626GiB/s) | 20260507 03:00:24.822 | INFO | Downloaded 34GiB to GPU in 1.94863s (17.4482GiB/s) | 20260507 03:00:29.962 | INFO | Analyzing chunk frames = [6568448,6568960) | 20260507 03:00:29.962 | INFO | MemInfo: { | 20260507 03:00:29.962 | INFO | "gpu" : " 17 GB", | 20260507 03:00:29.962 | INFO | "huge" : "", | 20260507 03:00:29.962 | INFO | "huge_shared" : " 27 GB", | 20260507 03:00:29.962 | INFO | "malloc" : " 14 GB", | 20260507 03:00:29.962 | INFO | "pinned" : " 29 GB" | 20260507 03:00:29.962 | INFO | } | 20260507 03:00:29.962 | INFO | Perf report for the last 5.14106 seconds: | 20260507 03:00:29.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:00:29.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:29.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:29.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:29.962 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:00:29.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:29.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:29.962 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 03:00:29.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:29.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:29.963 | INFO | Uploaded 36GiB to GPU in 2.05851s (17.4884GiB/s) | 20260507 03:00:29.963 | INFO | Downloaded 34GiB to GPU in 1.95385s (17.4015GiB/s) | 20260507 03:00:35.101 | INFO | Analyzing chunk frames = [6568960,6569472) | 20260507 03:00:35.101 | INFO | MemInfo: { | 20260507 03:00:35.101 | INFO | "gpu" : " 17 GB", | 20260507 03:00:35.101 | INFO | "huge" : "", | 20260507 03:00:35.101 | INFO | "huge_shared" : " 27 GB", | 20260507 03:00:35.101 | INFO | "malloc" : " 14 GB", | 20260507 03:00:35.101 | INFO | "pinned" : " 29 GB" | 20260507 03:00:35.101 | INFO | } | 20260507 03:00:35.101 | INFO | Perf report for the last 5.13857 seconds: | 20260507 03:00:35.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:00:35.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:35.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:35.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:35.101 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 03:00:35.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:35.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:35.101 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 03:00:35.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:35.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:35.101 | INFO | Uploaded 36GiB to GPU in 1.98819s (18.1069GiB/s) | 20260507 03:00:35.101 | INFO | Downloaded 34GiB to GPU in 1.96954s (17.263GiB/s) | 20260507 03:00:40.250 | INFO | Analyzing chunk frames = [6569472,6569984) | 20260507 03:00:40.250 | INFO | MemInfo: { | 20260507 03:00:40.250 | INFO | "gpu" : " 17 GB", | 20260507 03:00:40.250 | INFO | "huge" : "", | 20260507 03:00:40.250 | INFO | "huge_shared" : " 27 GB", | 20260507 03:00:40.250 | INFO | "malloc" : " 14 GB", | 20260507 03:00:40.250 | INFO | "pinned" : " 29 GB" | 20260507 03:00:40.250 | INFO | } | 20260507 03:00:40.250 | INFO | Perf report for the last 5.14883 seconds: | 20260507 03:00:40.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:00:40.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:40.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:40.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:40.250 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 03:00:40.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:40.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:40.250 | INFO | PRE_HQ : 29.9%, 1.0, 15.6ms, 0 | 20260507 03:00:40.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:40.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:40.250 | INFO | Uploaded 36GiB to GPU in 1.97684s (18.2108GiB/s) | 20260507 03:00:40.250 | INFO | Downloaded 34GiB to GPU in 1.9553s (17.3887GiB/s) | 20260507 03:00:40.250 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:00:40.250 | INFO | Total mem usage: 18 GiB | 20260507 03:00:40.250 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:00:40.250 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:00:40.250 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:00:40.250 | INFO | High Water Breakdown: | 20260507 03:00:40.250 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:00:40.250 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:00:40.250 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:00:40.250 | INFO | ------------------------------------------ | 20260507 03:00:40.250 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:00:40.250 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:00:40.250 | INFO | Total mem usage: 26.5 GiB | 20260507 03:00:40.250 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:00:40.250 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 03:00:40.250 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 03:00:40.250 | INFO | High Water Breakdown: | 20260507 03:00:40.250 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:00:40.250 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:00:40.250 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:00:40.250 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:00:40.251 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:00:40.251 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:00:40.251 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:00:40.251 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:00:40.251 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:00:40.251 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:00:40.251 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:00:40.251 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:00:40.251 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:00:40.251 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:00:40.251 | INFO | ------------------------------------------ | 20260507 03:00:40.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:00:40.251 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:00:40.251 | INFO | Total mem usage: 25.6 GiB | 20260507 03:00:40.251 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:00:40.251 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 03:00:40.251 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 03:00:40.251 | INFO | High Water Breakdown: | 20260507 03:00:40.251 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 03:00:40.251 | INFO | ------------------------------------------ | 20260507 03:00:40.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:00:40.251 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:00:40.251 | INFO | Total mem usage: 15.7 GiB | 20260507 03:00:40.251 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:00:40.251 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:00:40.251 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 03:00:40.251 | INFO | High Water Breakdown: | 20260507 03:00:40.251 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:00:40.251 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:00:40.251 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:00:40.251 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:00:40.251 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:00:40.251 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:00:40.251 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:00:40.251 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:00:40.251 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:00:40.251 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:00:40.251 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:00:40.251 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:00:40.251 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:00:40.251 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:00:40.251 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:00:40.251 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:00:40.251 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:00:40.251 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:00:40.251 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:00:40.251 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:00:40.251 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:00:40.251 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:00:40.251 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:00:40.251 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:00:40.251 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:00:40.251 | INFO | ------------------------------------------ | 20260507 03:00:40.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:00:40.251 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:00:40.251 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:00:40.251 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:00:40.251 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.31e+01 / 1.80e+00 | 20260507 03:00:40.251 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.02e-01, 9.11e-03 / 5.29e-03 | 20260507 03:00:40.252 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.44e+00, 5.08e-01 / 2.68e-01 | 20260507 03:00:40.252 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.48e+00, 1.66e-01 / 1.03e-01 | 20260507 03:00:40.252 | INFO | PerfReport: 16.08%, DME -- 3.95e+01 / 4.39e+01, 3.85e+00 / 1.87e+00 | 20260507 03:00:40.252 | INFO | PerfReport: 57.62%, FrameLabeling -- 1.77e+01 / 2.07e+01, 1.25e+00 / 1.76e+00 | 20260507 03:00:40.252 | INFO | PerfReport: 7.01%, PulseAccumulating -- 2.15e+00 / 2.70e+00, 3.83e-01 / 2.29e-01 | 20260507 03:00:40.252 | INFO | PerfReport: 3.07%, Metrics -- 9.43e-01 / 1.27e+00, 2.49e-01 / 1.24e-01 | 20260507 03:00:40.252 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.19e-01 / 3.74e-01, 5.05e-02 / 2.63e-02 | 20260507 03:00:40.252 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:00:40.252 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:00:40.252 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:00:40.252 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 03:00:40.252 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260507 03:00:40.252 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 03:00:40.252 | INFO | PerfReport: 31.89%, Compute -- 3.07e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 03:00:40.252 | INFO | PerfReport: 1.56%, LockUpload -- 1.50e+00 / 0.00e+00, 5.02e+00 / 0.00e+00 | 20260507 03:00:40.252 | INFO | PerfReport: 1.21%, LockDownload -- 1.16e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260507 03:00:40.252 | INFO | PerfReport: 22.14%, LockCompute -- 2.13e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 03:00:40.252 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260507 03:00:40.347 | INFO | Starting a round of Baz IO | 20260507 03:00:43.457 | INFO | Populating RTMetrics report | 20260507 03:00:43.457 | INFO | RTMetrics report populated | 20260507 03:00:43.457 | INFO | Opening tmp RTMetrics json file | 20260507 03:00:43.475 | INFO | Serializing RTMetrics to json | 20260507 03:00:43.477 | INFO | Serialization done, swapping tmp file now... | 20260507 03:00:43.477 | INFO | RTMetrics done | 20260507 03:00:43.549 | INFO | Finished a round of Baz IO: Wrote 1.765 GiB in 3.202 seconds | 20260507 03:00:43.549 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.94% / 0.00% / 67.06% | 20260507 03:00:43.549 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.216134% / 0% / 7.94729e-05% | 20260507 03:00:43.549 | INFO | Baz IO overall: | 20260507 03:00:43.549 | INFO | BytesWritten : 3380515756983 | 20260507 03:00:43.549 | INFO | NumEvents : 1624977555818 | 20260507 03:00:45.385 | INFO | Analyzing chunk frames = [6569984,6570496) | 20260507 03:00:45.385 | INFO | MemInfo: { | 20260507 03:00:45.385 | INFO | "gpu" : " 17 GB", | 20260507 03:00:45.385 | INFO | "huge" : "", | 20260507 03:00:45.385 | INFO | "huge_shared" : " 27 GB", | 20260507 03:00:45.385 | INFO | "malloc" : " 14 GB", | 20260507 03:00:45.385 | INFO | "pinned" : " 29 GB" | 20260507 03:00:45.385 | INFO | } | 20260507 03:00:45.385 | INFO | Perf report for the last 5.1353 seconds: | 20260507 03:00:45.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:00:45.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:45.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:45.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:45.385 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 03:00:45.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:45.385 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260507 03:00:45.385 | INFO | PRE_HQ : 60.2%, 1.9, 59.3ms, 0 | 20260507 03:00:45.385 | INFO | BAZWRITER : 61.9%, 3.4, 109.4ms, 0 | 20260507 03:00:45.385 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:00:45.385 | INFO | Uploaded 36GiB to GPU in 2.0411s (17.6375GiB/s) | 20260507 03:00:45.386 | INFO | Downloaded 38GiB to GPU in 2.17811s (17.4464GiB/s) | 20260507 03:00:50.523 | INFO | Analyzing chunk frames = [6570496,6571008) | 20260507 03:00:50.523 | INFO | MemInfo: { | 20260507 03:00:50.523 | INFO | "gpu" : " 17 GB", | 20260507 03:00:50.523 | INFO | "huge" : "", | 20260507 03:00:50.523 | INFO | "huge_shared" : " 27 GB", | 20260507 03:00:50.523 | INFO | "malloc" : " 14 GB", | 20260507 03:00:50.523 | INFO | "pinned" : " 29 GB" | 20260507 03:00:50.523 | INFO | } | 20260507 03:00:50.524 | INFO | Perf report for the last 5.1384 seconds: | 20260507 03:00:50.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:00:50.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:50.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:50.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:50.524 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:00:50.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:50.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:50.524 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:00:50.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:50.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:50.524 | INFO | Uploaded 36GiB to GPU in 1.99613s (18.0349GiB/s) | 20260507 03:00:50.524 | INFO | Downloaded 34GiB to GPU in 1.92876s (17.6279GiB/s) | 20260507 03:00:55.659 | INFO | Analyzing chunk frames = [6571008,6571520) | 20260507 03:00:55.659 | INFO | MemInfo: { | 20260507 03:00:55.659 | INFO | "gpu" : " 17 GB", | 20260507 03:00:55.659 | INFO | "huge" : "", | 20260507 03:00:55.659 | INFO | "huge_shared" : " 27 GB", | 20260507 03:00:55.659 | INFO | "malloc" : " 14 GB", | 20260507 03:00:55.659 | INFO | "pinned" : " 29 GB" | 20260507 03:00:55.659 | INFO | } | 20260507 03:00:55.659 | INFO | Perf report for the last 5.13569 seconds: | 20260507 03:00:55.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:00:55.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:55.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:55.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:55.659 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 03:00:55.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:55.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:55.659 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:00:55.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:55.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:00:55.659 | INFO | Uploaded 36GiB to GPU in 2.00963s (17.9137GiB/s) | 20260507 03:00:55.659 | INFO | Downloaded 34GiB to GPU in 1.93996s (17.5262GiB/s) | 20260507 03:01:00.792 | INFO | Analyzing chunk frames = [6571520,6572032) | 20260507 03:01:00.792 | INFO | MemInfo: { | 20260507 03:01:00.792 | INFO | "gpu" : " 17 GB", | 20260507 03:01:00.792 | INFO | "huge" : "", | 20260507 03:01:00.792 | INFO | "huge_shared" : " 27 GB", | 20260507 03:01:00.792 | INFO | "malloc" : " 14 GB", | 20260507 03:01:00.793 | INFO | "pinned" : " 29 GB" | 20260507 03:01:00.793 | INFO | } | 20260507 03:01:00.793 | INFO | Perf report for the last 5.13343 seconds: | 20260507 03:01:00.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:01:00.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:00.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:00.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:00.793 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260507 03:01:00.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:00.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:00.793 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:01:00.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:00.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:00.793 | INFO | Uploaded 36GiB to GPU in 1.991s (18.0814GiB/s) | 20260507 03:01:00.793 | INFO | Downloaded 34GiB to GPU in 1.9172s (17.7342GiB/s) | 20260507 03:01:05.942 | INFO | Analyzing chunk frames = [6572032,6572544) | 20260507 03:01:05.942 | INFO | MemInfo: { | 20260507 03:01:05.942 | INFO | "gpu" : " 17 GB", | 20260507 03:01:05.942 | INFO | "huge" : "", | 20260507 03:01:05.942 | INFO | "huge_shared" : " 27 GB", | 20260507 03:01:05.942 | INFO | "malloc" : " 14 GB", | 20260507 03:01:05.942 | INFO | "pinned" : " 29 GB" | 20260507 03:01:05.942 | INFO | } | 20260507 03:01:05.942 | INFO | Perf report for the last 5.14911 seconds: | 20260507 03:01:05.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:01:05.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:05.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:05.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:05.942 | INFO | ANALYSIS : 59.8%, 3.0, 93.3ms, 0 | 20260507 03:01:05.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:05.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:05.942 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 03:01:05.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:05.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:05.942 | INFO | Uploaded 36GiB to GPU in 1.9935s (18.0587GiB/s) | 20260507 03:01:05.942 | INFO | Downloaded 34GiB to GPU in 1.93197s (17.5986GiB/s) | 20260507 03:01:11.079 | INFO | Analyzing chunk frames = [6572544,6573056) | 20260507 03:01:11.079 | INFO | MemInfo: { | 20260507 03:01:11.079 | INFO | "gpu" : " 17 GB", | 20260507 03:01:11.079 | INFO | "huge" : "", | 20260507 03:01:11.079 | INFO | "huge_shared" : " 27 GB", | 20260507 03:01:11.079 | INFO | "malloc" : " 14 GB", | 20260507 03:01:11.079 | INFO | "pinned" : " 29 GB" | 20260507 03:01:11.079 | INFO | } | 20260507 03:01:11.079 | INFO | Perf report for the last 5.13738 seconds: | 20260507 03:01:11.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:01:11.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:11.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:11.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:11.079 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:01:11.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:11.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:11.079 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:01:11.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:11.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:11.079 | INFO | Uploaded 36GiB to GPU in 2.0571s (17.5003GiB/s) | 20260507 03:01:11.080 | INFO | Downloaded 34GiB to GPU in 1.95827s (17.3623GiB/s) | 20260507 03:01:16.220 | INFO | Analyzing chunk frames = [6573056,6573568) | 20260507 03:01:16.220 | INFO | MemInfo: { | 20260507 03:01:16.220 | INFO | "gpu" : " 17 GB", | 20260507 03:01:16.220 | INFO | "huge" : "", | 20260507 03:01:16.220 | INFO | "huge_shared" : " 27 GB", | 20260507 03:01:16.220 | INFO | "malloc" : " 14 GB", | 20260507 03:01:16.220 | INFO | "pinned" : " 29 GB" | 20260507 03:01:16.221 | INFO | } | 20260507 03:01:16.221 | INFO | Perf report for the last 5.14152 seconds: | 20260507 03:01:16.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:01:16.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:16.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:16.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:16.221 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:01:16.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:16.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:16.221 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 03:01:16.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:16.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:16.221 | INFO | Uploaded 36GiB to GPU in 1.97896s (18.1913GiB/s) | 20260507 03:01:16.221 | INFO | Downloaded 34GiB to GPU in 1.97246s (17.2373GiB/s) | 20260507 03:01:21.355 | INFO | Analyzing chunk frames = [6573568,6574080) | 20260507 03:01:21.355 | INFO | MemInfo: { | 20260507 03:01:21.355 | INFO | "gpu" : " 17 GB", | 20260507 03:01:21.355 | INFO | "huge" : "", | 20260507 03:01:21.355 | INFO | "huge_shared" : " 27 GB", | 20260507 03:01:21.355 | INFO | "malloc" : " 14 GB", | 20260507 03:01:21.355 | INFO | "pinned" : " 29 GB" | 20260507 03:01:21.355 | INFO | } | 20260507 03:01:21.355 | INFO | Perf report for the last 5.13454 seconds: | 20260507 03:01:21.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:01:21.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:21.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:21.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:21.355 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 03:01:21.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:21.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:21.356 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 03:01:21.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:21.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:21.356 | INFO | Uploaded 36GiB to GPU in 1.99707s (18.0264GiB/s) | 20260507 03:01:21.356 | INFO | Downloaded 34GiB to GPU in 1.96975s (17.2611GiB/s) | 20260507 03:01:21.452 | INFO | Starting a round of Baz IO | 20260507 03:01:24.565 | INFO | Populating RTMetrics report | 20260507 03:01:24.566 | INFO | RTMetrics report populated | 20260507 03:01:24.566 | INFO | Opening tmp RTMetrics json file | 20260507 03:01:24.567 | INFO | Serializing RTMetrics to json | 20260507 03:01:24.585 | INFO | Serialization done, swapping tmp file now... | 20260507 03:01:24.585 | INFO | RTMetrics done | 20260507 03:01:24.658 | INFO | Finished a round of Baz IO: Wrote 1.764 GiB in 3.206 seconds | 20260507 03:01:24.658 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.94% / 0.00% / 67.06% | 20260507 03:01:24.658 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.21505% / 3.97364e-06% / 8.34465e-05% | 20260507 03:01:24.658 | INFO | Baz IO overall: | 20260507 03:01:24.658 | INFO | BytesWritten : 3382409519078 | 20260507 03:01:24.658 | INFO | NumEvents : 1625871814106 | 20260507 03:01:26.492 | INFO | Analyzing chunk frames = [6574080,6574592) | 20260507 03:01:26.492 | INFO | MemInfo: { | 20260507 03:01:26.492 | INFO | "gpu" : " 17 GB", | 20260507 03:01:26.492 | INFO | "huge" : "", | 20260507 03:01:26.492 | INFO | "huge_shared" : " 27 GB", | 20260507 03:01:26.492 | INFO | "malloc" : " 14 GB", | 20260507 03:01:26.492 | INFO | "pinned" : " 29 GB" | 20260507 03:01:26.492 | INFO | } | 20260507 03:01:26.492 | INFO | Perf report for the last 5.13691 seconds: | 20260507 03:01:26.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:01:26.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:26.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:26.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:26.492 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 03:01:26.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:26.492 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260507 03:01:26.492 | INFO | PRE_HQ : 60.3%, 1.9, 59.8ms, 0 | 20260507 03:01:26.492 | INFO | BAZWRITER : 62.0%, 3.4, 109.0ms, 0 | 20260507 03:01:26.492 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 03:01:26.492 | INFO | Uploaded 36GiB to GPU in 2.03659s (17.6766GiB/s) | 20260507 03:01:26.492 | INFO | Downloaded 38GiB to GPU in 2.18471s (17.3936GiB/s) | 20260507 03:01:31.633 | INFO | Analyzing chunk frames = [6574592,6575104) | 20260507 03:01:31.633 | INFO | MemInfo: { | 20260507 03:01:31.633 | INFO | "gpu" : " 17 GB", | 20260507 03:01:31.633 | INFO | "huge" : "", | 20260507 03:01:31.633 | INFO | "huge_shared" : " 27 GB", | 20260507 03:01:31.633 | INFO | "malloc" : " 14 GB", | 20260507 03:01:31.633 | INFO | "pinned" : " 29 GB" | 20260507 03:01:31.633 | INFO | } | 20260507 03:01:31.633 | INFO | Perf report for the last 5.14087 seconds: | 20260507 03:01:31.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:01:31.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:31.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:31.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:31.633 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 03:01:31.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:31.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:31.633 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 03:01:31.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:31.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:31.634 | INFO | Uploaded 36GiB to GPU in 2.02199s (17.8043GiB/s) | 20260507 03:01:31.634 | INFO | Downloaded 34GiB to GPU in 1.9343s (17.5774GiB/s) | 20260507 03:01:36.773 | INFO | Analyzing chunk frames = [6575104,6575616) | 20260507 03:01:36.773 | INFO | MemInfo: { | 20260507 03:01:36.773 | INFO | "gpu" : " 17 GB", | 20260507 03:01:36.773 | INFO | "huge" : "", | 20260507 03:01:36.773 | INFO | "huge_shared" : " 27 GB", | 20260507 03:01:36.773 | INFO | "malloc" : " 14 GB", | 20260507 03:01:36.774 | INFO | "pinned" : " 29 GB" | 20260507 03:01:36.774 | INFO | } | 20260507 03:01:36.774 | INFO | Perf report for the last 5.14049 seconds: | 20260507 03:01:36.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:01:36.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:36.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:36.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:36.774 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 03:01:36.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:36.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:36.774 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:01:36.774 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:36.774 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:36.774 | INFO | Uploaded 36GiB to GPU in 2.00956s (17.9143GiB/s) | 20260507 03:01:36.774 | INFO | Downloaded 34GiB to GPU in 1.93721s (17.551GiB/s) | 20260507 03:01:41.917 | INFO | Analyzing chunk frames = [6575616,6576128) | 20260507 03:01:41.917 | INFO | MemInfo: { | 20260507 03:01:41.917 | INFO | "gpu" : " 17 GB", | 20260507 03:01:41.917 | INFO | "huge" : "", | 20260507 03:01:41.917 | INFO | "huge_shared" : " 27 GB", | 20260507 03:01:41.917 | INFO | "malloc" : " 14 GB", | 20260507 03:01:41.917 | INFO | "pinned" : " 29 GB" | 20260507 03:01:41.917 | INFO | } | 20260507 03:01:41.917 | INFO | Perf report for the last 5.14392 seconds: | 20260507 03:01:41.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:01:41.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:41.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:41.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:41.918 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 03:01:41.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:41.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:41.918 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:01:41.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:41.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:41.918 | INFO | Uploaded 36GiB to GPU in 1.96358s (18.3338GiB/s) | 20260507 03:01:41.918 | INFO | Downloaded 34GiB to GPU in 1.90322s (17.8645GiB/s) | 20260507 03:01:47.053 | INFO | Analyzing chunk frames = [6576128,6576640) | 20260507 03:01:47.053 | INFO | MemInfo: { | 20260507 03:01:47.053 | INFO | "gpu" : " 17 GB", | 20260507 03:01:47.053 | INFO | "huge" : "", | 20260507 03:01:47.053 | INFO | "huge_shared" : " 27 GB", | 20260507 03:01:47.053 | INFO | "malloc" : " 14 GB", | 20260507 03:01:47.053 | INFO | "pinned" : " 29 GB" | 20260507 03:01:47.053 | INFO | } | 20260507 03:01:47.054 | INFO | Perf report for the last 5.13599 seconds: | 20260507 03:01:47.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:01:47.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:47.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:47.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:47.054 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 03:01:47.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:47.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:47.054 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:01:47.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:47.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:47.054 | INFO | Uploaded 36GiB to GPU in 2.02619s (17.7673GiB/s) | 20260507 03:01:47.054 | INFO | Downloaded 34GiB to GPU in 1.93408s (17.5794GiB/s) | 20260507 03:01:52.191 | INFO | Analyzing chunk frames = [6576640,6577152) | 20260507 03:01:52.191 | INFO | MemInfo: { | 20260507 03:01:52.191 | INFO | "gpu" : " 17 GB", | 20260507 03:01:52.191 | INFO | "huge" : "", | 20260507 03:01:52.191 | INFO | "huge_shared" : " 27 GB", | 20260507 03:01:52.191 | INFO | "malloc" : " 14 GB", | 20260507 03:01:52.191 | INFO | "pinned" : " 29 GB" | 20260507 03:01:52.191 | INFO | } | 20260507 03:01:52.191 | INFO | Perf report for the last 5.13794 seconds: | 20260507 03:01:52.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:01:52.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:52.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:52.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:52.192 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 03:01:52.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:52.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:52.192 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:01:52.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:52.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:52.192 | INFO | Uploaded 36GiB to GPU in 2.05639s (17.5064GiB/s) | 20260507 03:01:52.192 | INFO | Downloaded 34GiB to GPU in 1.95265s (17.4122GiB/s) | 20260507 03:01:57.335 | INFO | Analyzing chunk frames = [6577152,6577664) | 20260507 03:01:57.335 | INFO | MemInfo: { | 20260507 03:01:57.335 | INFO | "gpu" : " 17 GB", | 20260507 03:01:57.335 | INFO | "huge" : "", | 20260507 03:01:57.335 | INFO | "huge_shared" : " 27 GB", | 20260507 03:01:57.335 | INFO | "malloc" : " 14 GB", | 20260507 03:01:57.335 | INFO | "pinned" : " 29 GB" | 20260507 03:01:57.335 | INFO | } | 20260507 03:01:57.335 | INFO | Perf report for the last 5.14374 seconds: | 20260507 03:01:57.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:01:57.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:57.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:57.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:57.335 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:01:57.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:57.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:57.335 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 03:01:57.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:57.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:01:57.336 | INFO | Uploaded 36GiB to GPU in 1.99074s (18.0837GiB/s) | 20260507 03:01:57.336 | INFO | Downloaded 34GiB to GPU in 1.98211s (17.1534GiB/s) | 20260507 03:02:02.468 | INFO | Analyzing chunk frames = [6577664,6578176) | 20260507 03:02:02.468 | INFO | MemInfo: { | 20260507 03:02:02.468 | INFO | "gpu" : " 17 GB", | 20260507 03:02:02.468 | INFO | "huge" : "", | 20260507 03:02:02.468 | INFO | "huge_shared" : " 27 GB", | 20260507 03:02:02.468 | INFO | "malloc" : " 14 GB", | 20260507 03:02:02.468 | INFO | "pinned" : " 29 GB" | 20260507 03:02:02.468 | INFO | } | 20260507 03:02:02.468 | INFO | Perf report for the last 5.13297 seconds: | 20260507 03:02:02.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:02:02.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:02.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:02.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:02.468 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:02:02.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:02.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:02.468 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 03:02:02.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:02.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:02.468 | INFO | Uploaded 36GiB to GPU in 2.00518s (17.9535GiB/s) | 20260507 03:02:02.468 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:02:02.469 | INFO | Downloaded 34GiB to GPU in 1.97288s (17.2337GiB/s) | 20260507 03:02:02.469 | INFO | Total mem usage: 18 GiB | 20260507 03:02:02.469 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:02:02.469 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 03:02:02.469 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:02:02.469 | INFO | High Water Breakdown: | 20260507 03:02:02.469 | INFO | 60.2% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:02:02.469 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:02:02.469 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:02:02.469 | INFO | ------------------------------------------ | 20260507 03:02:02.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:02:02.469 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:02:02.469 | INFO | Total mem usage: 26.5 GiB | 20260507 03:02:02.469 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:02:02.469 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:02:02.469 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:02:02.469 | INFO | High Water Breakdown: | 20260507 03:02:02.469 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:02:02.469 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:02:02.469 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:02:02.469 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:02:02.469 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:02:02.469 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:02:02.469 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:02:02.469 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:02:02.469 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:02:02.469 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:02:02.469 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:02:02.469 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:02:02.469 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:02:02.469 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:02:02.469 | INFO | ------------------------------------------ | 20260507 03:02:02.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:02:02.469 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:02:02.469 | INFO | Total mem usage: 25.6 GiB | 20260507 03:02:02.469 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:02:02.469 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:02:02.469 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:02:02.469 | INFO | High Water Breakdown: | 20260507 03:02:02.469 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:02:02.469 | INFO | ------------------------------------------ | 20260507 03:02:02.469 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:02:02.469 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:02:02.469 | INFO | Total mem usage: 15.7 GiB | 20260507 03:02:02.469 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:02:02.469 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:02:02.469 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 03:02:02.469 | INFO | High Water Breakdown: | 20260507 03:02:02.469 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:02:02.469 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:02:02.469 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:02:02.469 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:02:02.469 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:02:02.469 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:02:02.470 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:02:02.470 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:02:02.470 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:02:02.470 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:02:02.470 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:02:02.470 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:02:02.470 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:02:02.470 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:02:02.470 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:02:02.470 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:02:02.470 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:02:02.470 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:02:02.470 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:02:02.470 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:02:02.470 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:02:02.470 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:02:02.470 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:02:02.470 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:02:02.470 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:02:02.470 | INFO | ------------------------------------------ | 20260507 03:02:02.470 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:02:02.470 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:02:02.470 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:02:02.470 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:02:02.470 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.50e+01, 1.33e+01 / 1.81e+00 | 20260507 03:02:02.470 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.00e-01, 9.35e-03 / 5.37e-03 | 20260507 03:02:02.470 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.42e+00, 5.15e-01 / 3.33e-01 | 20260507 03:02:02.470 | INFO | PerfReport: 3.23%, Histogram -- 9.94e-01 / 1.47e+00, 1.71e-01 / 1.23e-01 | 20260507 03:02:02.470 | INFO | PerfReport: 16.11%, DME -- 3.97e+01 / 4.41e+01, 3.86e+00 / 2.37e+00 | 20260507 03:02:02.470 | INFO | PerfReport: 57.62%, FrameLabeling -- 1.77e+01 / 2.06e+01, 1.30e+00 / 1.74e+00 | 20260507 03:02:02.470 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.15e+00 / 2.72e+00, 3.81e-01 / 1.96e-01 | 20260507 03:02:02.470 | INFO | PerfReport: 3.06%, Metrics -- 9.43e-01 / 1.28e+00, 2.50e-01 / 1.35e-01 | 20260507 03:02:02.470 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.20e-01 / 3.75e-01, 5.08e-02 / 2.50e-02 | 20260507 03:02:02.470 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:02:02.470 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:02:02.470 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:02:02.470 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 03:02:02.470 | INFO | PerfReport: 21.75%, Upload -- 2.09e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260507 03:02:02.470 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 03:02:02.470 | INFO | PerfReport: 31.96%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 03:02:02.470 | INFO | PerfReport: 1.47%, LockUpload -- 1.42e+00 / 0.00e+00, 4.86e+00 / 0.00e+00 | 20260507 03:02:02.470 | INFO | PerfReport: 1.25%, LockDownload -- 1.20e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260507 03:02:02.470 | INFO | PerfReport: 22.20%, LockCompute -- 2.14e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 03:02:02.470 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260507 03:02:02.565 | INFO | Starting a round of Baz IO | 20260507 03:02:05.673 | INFO | Populating RTMetrics report | 20260507 03:02:05.673 | INFO | RTMetrics report populated | 20260507 03:02:05.673 | INFO | Opening tmp RTMetrics json file | 20260507 03:02:05.691 | INFO | Serializing RTMetrics to json | 20260507 03:02:05.692 | INFO | Serialization done, swapping tmp file now... | 20260507 03:02:05.693 | INFO | RTMetrics done | 20260507 03:02:05.768 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.202 seconds | 20260507 03:02:05.768 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.94% / 0.00% / 67.06% | 20260507 03:02:05.768 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.214589% / 0% / 8.74201e-05% | 20260507 03:02:05.768 | INFO | Baz IO overall: | 20260507 03:02:05.768 | INFO | BytesWritten : 3384302340956 | 20260507 03:02:05.768 | INFO | NumEvents : 1626765323560 | 20260507 03:02:07.608 | INFO | Analyzing chunk frames = [6578176,6578688) | 20260507 03:02:07.608 | INFO | MemInfo: { | 20260507 03:02:07.608 | INFO | "gpu" : " 17 GB", | 20260507 03:02:07.608 | INFO | "huge" : "", | 20260507 03:02:07.608 | INFO | "huge_shared" : " 27 GB", | 20260507 03:02:07.608 | INFO | "malloc" : " 14 GB", | 20260507 03:02:07.608 | INFO | "pinned" : " 29 GB" | 20260507 03:02:07.608 | INFO | } | 20260507 03:02:07.608 | INFO | Perf report for the last 5.13985 seconds: | 20260507 03:02:07.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:02:07.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:07.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:07.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:07.608 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 03:02:07.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:07.608 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 03:02:07.608 | INFO | PRE_HQ : 59.3%, 1.8, 55.5ms, 0 | 20260507 03:02:07.608 | INFO | BAZWRITER : 61.9%, 3.3, 106.6ms, 0 | 20260507 03:02:07.608 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:02:07.608 | INFO | Uploaded 36GiB to GPU in 2.04378s (17.6144GiB/s) | 20260507 03:02:07.608 | INFO | Downloaded 38GiB to GPU in 2.187s (17.3754GiB/s) | 20260507 03:02:12.749 | INFO | Analyzing chunk frames = [6578688,6579200) | 20260507 03:02:12.749 | INFO | MemInfo: { | 20260507 03:02:12.749 | INFO | "gpu" : " 17 GB", | 20260507 03:02:12.749 | INFO | "huge" : "", | 20260507 03:02:12.749 | INFO | "huge_shared" : " 27 GB", | 20260507 03:02:12.749 | INFO | "malloc" : " 14 GB", | 20260507 03:02:12.749 | INFO | "pinned" : " 29 GB" | 20260507 03:02:12.749 | INFO | } | 20260507 03:02:12.750 | INFO | Perf report for the last 5.14161 seconds: | 20260507 03:02:12.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:02:12.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:12.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:12.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:12.750 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:02:12.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:12.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:12.750 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 03:02:12.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:12.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:12.750 | INFO | Uploaded 36GiB to GPU in 2.02301s (17.7953GiB/s) | 20260507 03:02:12.750 | INFO | Downloaded 34GiB to GPU in 1.93452s (17.5754GiB/s) | 20260507 03:02:17.899 | INFO | Analyzing chunk frames = [6579200,6579712) | 20260507 03:02:17.899 | INFO | MemInfo: { | 20260507 03:02:17.899 | INFO | "gpu" : " 17 GB", | 20260507 03:02:17.899 | INFO | "huge" : "", | 20260507 03:02:17.899 | INFO | "huge_shared" : " 27 GB", | 20260507 03:02:17.899 | INFO | "malloc" : " 14 GB", | 20260507 03:02:17.899 | INFO | "pinned" : " 29 GB" | 20260507 03:02:17.899 | INFO | } | 20260507 03:02:17.899 | INFO | Perf report for the last 5.14916 seconds: | 20260507 03:02:17.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:02:17.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:17.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:17.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:17.899 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260507 03:02:17.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:17.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:17.899 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:02:17.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:17.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:17.899 | INFO | Uploaded 36GiB to GPU in 2.0075s (17.9328GiB/s) | 20260507 03:02:17.899 | INFO | Downloaded 34GiB to GPU in 1.93553s (17.5662GiB/s) | 20260507 03:02:23.031 | INFO | Analyzing chunk frames = [6579712,6580224) | 20260507 03:02:23.031 | INFO | MemInfo: { | 20260507 03:02:23.031 | INFO | "gpu" : " 17 GB", | 20260507 03:02:23.031 | INFO | "huge" : "", | 20260507 03:02:23.031 | INFO | "huge_shared" : " 27 GB", | 20260507 03:02:23.031 | INFO | "malloc" : " 14 GB", | 20260507 03:02:23.032 | INFO | "pinned" : " 29 GB" | 20260507 03:02:23.032 | INFO | } | 20260507 03:02:23.032 | INFO | Perf report for the last 5.1329 seconds: | 20260507 03:02:23.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:02:23.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:23.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:23.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:23.032 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 03:02:23.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:23.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:23.032 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:02:23.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:23.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:23.032 | INFO | Uploaded 36GiB to GPU in 2.00979s (17.9123GiB/s) | 20260507 03:02:23.032 | INFO | Downloaded 34GiB to GPU in 1.93909s (17.534GiB/s) | 20260507 03:02:28.163 | INFO | Analyzing chunk frames = [6580224,6580736) | 20260507 03:02:28.163 | INFO | MemInfo: { | 20260507 03:02:28.163 | INFO | "gpu" : " 17 GB", | 20260507 03:02:28.163 | INFO | "huge" : "", | 20260507 03:02:28.163 | INFO | "huge_shared" : " 27 GB", | 20260507 03:02:28.163 | INFO | "malloc" : " 14 GB", | 20260507 03:02:28.164 | INFO | "pinned" : " 29 GB" | 20260507 03:02:28.164 | INFO | } | 20260507 03:02:28.164 | INFO | Perf report for the last 5.1322 seconds: | 20260507 03:02:28.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:02:28.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:28.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:28.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:28.164 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 03:02:28.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:28.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:28.164 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:02:28.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:28.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:28.164 | INFO | Uploaded 36GiB to GPU in 2.04905s (17.5691GiB/s) | 20260507 03:02:28.164 | INFO | Downloaded 34GiB to GPU in 1.9454s (17.4771GiB/s) | 20260507 03:02:33.304 | INFO | Analyzing chunk frames = [6580736,6581248) | 20260507 03:02:33.304 | INFO | MemInfo: { | 20260507 03:02:33.304 | INFO | "gpu" : " 17 GB", | 20260507 03:02:33.304 | INFO | "huge" : "", | 20260507 03:02:33.304 | INFO | "huge_shared" : " 27 GB", | 20260507 03:02:33.304 | INFO | "malloc" : " 14 GB", | 20260507 03:02:33.304 | INFO | "pinned" : " 29 GB" | 20260507 03:02:33.304 | INFO | } | 20260507 03:02:33.305 | INFO | Perf report for the last 5.14056 seconds: | 20260507 03:02:33.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:02:33.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:33.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:33.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:33.305 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:02:33.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:33.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:33.305 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 03:02:33.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:33.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:33.305 | INFO | Uploaded 36GiB to GPU in 2.05899s (17.4843GiB/s) | 20260507 03:02:33.305 | INFO | Downloaded 34GiB to GPU in 1.96431s (17.3089GiB/s) | 20260507 03:02:38.451 | INFO | Analyzing chunk frames = [6581248,6581760) | 20260507 03:02:38.451 | INFO | MemInfo: { | 20260507 03:02:38.451 | INFO | "gpu" : " 17 GB", | 20260507 03:02:38.451 | INFO | "huge" : "", | 20260507 03:02:38.451 | INFO | "huge_shared" : " 27 GB", | 20260507 03:02:38.451 | INFO | "malloc" : " 14 GB", | 20260507 03:02:38.451 | INFO | "pinned" : " 29 GB" | 20260507 03:02:38.451 | INFO | } | 20260507 03:02:38.451 | INFO | Perf report for the last 5.14688 seconds: | 20260507 03:02:38.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:02:38.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:38.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:38.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:38.452 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 03:02:38.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:38.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:38.452 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 03:02:38.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:38.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:38.452 | INFO | Uploaded 36GiB to GPU in 1.99571s (18.0387GiB/s) | 20260507 03:02:38.452 | INFO | Downloaded 34GiB to GPU in 1.99902s (17.0084GiB/s) | 20260507 03:02:43.590 | INFO | Analyzing chunk frames = [6581760,6582272) | 20260507 03:02:43.590 | INFO | MemInfo: { | 20260507 03:02:43.590 | INFO | "gpu" : " 17 GB", | 20260507 03:02:43.590 | INFO | "huge" : "", | 20260507 03:02:43.590 | INFO | "huge_shared" : " 27 GB", | 20260507 03:02:43.590 | INFO | "malloc" : " 14 GB", | 20260507 03:02:43.590 | INFO | "pinned" : " 29 GB" | 20260507 03:02:43.590 | INFO | } | 20260507 03:02:43.590 | INFO | Perf report for the last 5.13833 seconds: | 20260507 03:02:43.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:02:43.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:43.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:43.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:43.590 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:02:43.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:43.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:43.590 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 03:02:43.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:43.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:43.590 | INFO | Uploaded 36GiB to GPU in 2.00774s (17.9306GiB/s) | 20260507 03:02:43.590 | INFO | Downloaded 34GiB to GPU in 1.96689s (17.2862GiB/s) | 20260507 03:02:43.686 | INFO | Starting a round of Baz IO | 20260507 03:02:46.791 | INFO | Populating RTMetrics report | 20260507 03:02:46.791 | INFO | RTMetrics report populated | 20260507 03:02:46.791 | INFO | Opening tmp RTMetrics json file | 20260507 03:02:46.792 | INFO | Serializing RTMetrics to json | 20260507 03:02:46.793 | INFO | Serialization done, swapping tmp file now... | 20260507 03:02:46.794 | INFO | RTMetrics done | 20260507 03:02:46.881 | INFO | Finished a round of Baz IO: Wrote 1.764 GiB in 3.195 seconds | 20260507 03:02:46.882 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.94% / 0.00% / 67.06% | 20260507 03:02:46.882 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.214644% / 7.94729e-06% / 0.000123183% | 20260507 03:02:46.882 | INFO | Baz IO overall: | 20260507 03:02:46.882 | INFO | BytesWritten : 3386195890302 | 20260507 03:02:46.882 | INFO | NumEvents : 1627659340136 | 20260507 03:02:48.727 | INFO | Analyzing chunk frames = [6582272,6582784) | 20260507 03:02:48.727 | INFO | MemInfo: { | 20260507 03:02:48.727 | INFO | "gpu" : " 17 GB", | 20260507 03:02:48.727 | INFO | "huge" : "", | 20260507 03:02:48.727 | INFO | "huge_shared" : " 27 GB", | 20260507 03:02:48.727 | INFO | "malloc" : " 14 GB", | 20260507 03:02:48.727 | INFO | "pinned" : " 29 GB" | 20260507 03:02:48.727 | INFO | } | 20260507 03:02:48.727 | INFO | Perf report for the last 5.13723 seconds: | 20260507 03:02:48.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:02:48.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:48.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:48.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:48.727 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 03:02:48.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:48.727 | INFO | RT_METRICS : 41.7%, 1.0, 21.8ms, 0 | 20260507 03:02:48.727 | INFO | PRE_HQ : 59.4%, 1.8, 55.8ms, 0 | 20260507 03:02:48.727 | INFO | BAZWRITER : 61.6%, 3.4, 106.3ms, 0 | 20260507 03:02:48.727 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:02:48.727 | INFO | Uploaded 36GiB to GPU in 2.03222s (17.7146GiB/s) | 20260507 03:02:48.727 | INFO | Downloaded 38GiB to GPU in 2.1797s (17.4336GiB/s) | 20260507 03:02:53.864 | INFO | Analyzing chunk frames = [6582784,6583296) | 20260507 03:02:53.864 | INFO | MemInfo: { | 20260507 03:02:53.864 | INFO | "gpu" : " 17 GB", | 20260507 03:02:53.864 | INFO | "huge" : "", | 20260507 03:02:53.864 | INFO | "huge_shared" : " 27 GB", | 20260507 03:02:53.864 | INFO | "malloc" : " 14 GB", | 20260507 03:02:53.864 | INFO | "pinned" : " 29 GB" | 20260507 03:02:53.864 | INFO | } | 20260507 03:02:53.864 | INFO | Perf report for the last 5.13693 seconds: | 20260507 03:02:53.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:02:53.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:53.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:53.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:53.864 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:02:53.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:53.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:53.864 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:02:53.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:53.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:53.864 | INFO | Uploaded 36GiB to GPU in 2.04251s (17.6254GiB/s) | 20260507 03:02:53.864 | INFO | Downloaded 34GiB to GPU in 1.94222s (17.5058GiB/s) | 20260507 03:02:59.005 | INFO | Analyzing chunk frames = [6583296,6583808) | 20260507 03:02:59.005 | INFO | MemInfo: { | 20260507 03:02:59.005 | INFO | "gpu" : " 17 GB", | 20260507 03:02:59.005 | INFO | "huge" : "", | 20260507 03:02:59.005 | INFO | "huge_shared" : " 27 GB", | 20260507 03:02:59.005 | INFO | "malloc" : " 14 GB", | 20260507 03:02:59.005 | INFO | "pinned" : " 29 GB" | 20260507 03:02:59.006 | INFO | } | 20260507 03:02:59.006 | INFO | Perf report for the last 5.14161 seconds: | 20260507 03:02:59.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:02:59.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:59.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:59.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:59.006 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 03:02:59.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:59.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:59.006 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:02:59.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:59.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:02:59.006 | INFO | Uploaded 36GiB to GPU in 2.00416s (17.9626GiB/s) | 20260507 03:02:59.006 | INFO | Downloaded 34GiB to GPU in 1.93491s (17.5719GiB/s) | 20260507 03:03:04.145 | INFO | Analyzing chunk frames = [6583808,6584320) | 20260507 03:03:04.145 | INFO | MemInfo: { | 20260507 03:03:04.145 | INFO | "gpu" : " 17 GB", | 20260507 03:03:04.145 | INFO | "huge" : "", | 20260507 03:03:04.145 | INFO | "huge_shared" : " 27 GB", | 20260507 03:03:04.145 | INFO | "malloc" : " 14 GB", | 20260507 03:03:04.145 | INFO | "pinned" : " 29 GB" | 20260507 03:03:04.145 | INFO | } | 20260507 03:03:04.145 | INFO | Perf report for the last 5.13949 seconds: | 20260507 03:03:04.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:03:04.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:04.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:04.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:04.145 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 03:03:04.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:04.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:04.145 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 03:03:04.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:04.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:04.146 | INFO | Uploaded 36GiB to GPU in 1.97025s (18.2718GiB/s) | 20260507 03:03:04.146 | INFO | Downloaded 34GiB to GPU in 1.89744s (17.9189GiB/s) | 20260507 03:03:09.282 | INFO | Analyzing chunk frames = [6584320,6584832) | 20260507 03:03:09.282 | INFO | MemInfo: { | 20260507 03:03:09.282 | INFO | "gpu" : " 17 GB", | 20260507 03:03:09.282 | INFO | "huge" : "", | 20260507 03:03:09.282 | INFO | "huge_shared" : " 27 GB", | 20260507 03:03:09.282 | INFO | "malloc" : " 14 GB", | 20260507 03:03:09.282 | INFO | "pinned" : " 29 GB" | 20260507 03:03:09.282 | INFO | } | 20260507 03:03:09.282 | INFO | Perf report for the last 5.13683 seconds: | 20260507 03:03:09.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:03:09.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:09.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:09.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:09.282 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 03:03:09.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:09.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:09.282 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:03:09.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:09.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:09.282 | INFO | Uploaded 36GiB to GPU in 2.01797s (17.8397GiB/s) | 20260507 03:03:09.282 | INFO | Downloaded 34GiB to GPU in 1.93866s (17.5379GiB/s) | 20260507 03:03:14.424 | INFO | Analyzing chunk frames = [6584832,6585344) | 20260507 03:03:14.424 | INFO | MemInfo: { | 20260507 03:03:14.424 | INFO | "gpu" : " 17 GB", | 20260507 03:03:14.424 | INFO | "huge" : "", | 20260507 03:03:14.424 | INFO | "huge_shared" : " 27 GB", | 20260507 03:03:14.424 | INFO | "malloc" : " 14 GB", | 20260507 03:03:14.424 | INFO | "pinned" : " 29 GB" | 20260507 03:03:14.424 | INFO | } | 20260507 03:03:14.424 | INFO | Perf report for the last 5.14207 seconds: | 20260507 03:03:14.424 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:03:14.424 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:14.424 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:14.424 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:14.424 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:03:14.424 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:14.424 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:14.424 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 03:03:14.424 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:14.424 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:14.424 | INFO | Uploaded 36GiB to GPU in 2.05955s (17.4796GiB/s) | 20260507 03:03:14.424 | INFO | Downloaded 34GiB to GPU in 1.96622s (17.2921GiB/s) | 20260507 03:03:19.564 | INFO | Analyzing chunk frames = [6585344,6585856) | 20260507 03:03:19.564 | INFO | MemInfo: { | 20260507 03:03:19.564 | INFO | "gpu" : " 17 GB", | 20260507 03:03:19.565 | INFO | "huge" : "", | 20260507 03:03:19.565 | INFO | "huge_shared" : " 27 GB", | 20260507 03:03:19.565 | INFO | "malloc" : " 14 GB", | 20260507 03:03:19.565 | INFO | "pinned" : " 29 GB" | 20260507 03:03:19.565 | INFO | } | 20260507 03:03:19.565 | INFO | Perf report for the last 5.14069 seconds: | 20260507 03:03:19.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:03:19.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:19.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:19.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:19.565 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 03:03:19.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:19.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:19.565 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 03:03:19.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:19.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:19.565 | INFO | Uploaded 36GiB to GPU in 1.97343s (18.2424GiB/s) | 20260507 03:03:19.565 | INFO | Downloaded 34GiB to GPU in 1.96262s (17.3238GiB/s) | 20260507 03:03:24.708 | INFO | Analyzing chunk frames = [6585856,6586368) | 20260507 03:03:24.708 | INFO | MemInfo: { | 20260507 03:03:24.708 | INFO | "gpu" : " 17 GB", | 20260507 03:03:24.708 | INFO | "huge" : "", | 20260507 03:03:24.708 | INFO | "huge_shared" : " 27 GB", | 20260507 03:03:24.708 | INFO | "malloc" : " 14 GB", | 20260507 03:03:24.708 | INFO | "pinned" : " 29 GB" | 20260507 03:03:24.708 | INFO | } | 20260507 03:03:24.708 | INFO | Perf report for the last 5.14305 seconds: | 20260507 03:03:24.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:03:24.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:24.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:24.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:24.708 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:03:24.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:24.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:24.708 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:03:24.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:24.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:24.708 | INFO | Uploaded 36GiB to GPU in 1.97444s (18.233GiB/s) | 20260507 03:03:24.708 | INFO | Downloaded 34GiB to GPU in 1.96425s (17.3094GiB/s) | 20260507 03:03:24.708 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:03:24.708 | INFO | Total mem usage: 18 GiB | 20260507 03:03:24.708 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:03:24.708 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 03:03:24.708 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:03:24.708 | INFO | High Water Breakdown: | 20260507 03:03:24.708 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:03:24.708 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:03:24.708 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:03:24.708 | INFO | ------------------------------------------ | 20260507 03:03:24.708 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:03:24.708 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:03:24.708 | INFO | Total mem usage: 26.5 GiB | 20260507 03:03:24.708 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:03:24.708 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:03:24.708 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:03:24.708 | INFO | High Water Breakdown: | 20260507 03:03:24.708 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:03:24.708 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:03:24.708 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:03:24.708 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:03:24.708 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:03:24.708 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:03:24.708 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:03:24.708 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:03:24.708 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:03:24.708 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:03:24.708 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:03:24.708 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:03:24.708 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:03:24.709 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:03:24.709 | INFO | ------------------------------------------ | 20260507 03:03:24.709 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:03:24.709 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:03:24.709 | INFO | Total mem usage: 25.6 GiB | 20260507 03:03:24.709 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:03:24.709 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 03:03:24.709 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 03:03:24.709 | INFO | High Water Breakdown: | 20260507 03:03:24.709 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 03:03:24.709 | INFO | ------------------------------------------ | 20260507 03:03:24.709 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:03:24.709 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:03:24.709 | INFO | Total mem usage: 15.7 GiB | 20260507 03:03:24.709 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:03:24.709 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:03:24.709 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:03:24.709 | INFO | High Water Breakdown: | 20260507 03:03:24.709 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:03:24.709 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:03:24.709 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:03:24.709 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:03:24.709 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:03:24.709 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:03:24.709 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:03:24.709 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:03:24.709 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:03:24.709 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:03:24.709 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:03:24.709 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:03:24.709 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:03:24.709 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:03:24.709 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:03:24.709 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:03:24.709 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:03:24.709 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:03:24.709 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:03:24.709 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:03:24.709 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:03:24.709 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:03:24.709 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:03:24.709 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:03:24.709 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:03:24.709 | INFO | ------------------------------------------ | 20260507 03:03:24.709 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:03:24.709 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:03:24.709 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:03:24.709 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:03:24.709 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.51e+01, 1.34e+01 / 1.56e+00 | 20260507 03:03:24.709 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.02e-01, 9.32e-03 / 6.56e-03 | 20260507 03:03:24.709 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.50e+00, 5.08e-01 / 1.76e-01 | 20260507 03:03:24.709 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.46e+00, 1.65e-01 / 1.06e-01 | 20260507 03:03:24.709 | INFO | PerfReport: 16.16%, DME -- 3.97e+01 / 4.48e+01, 3.94e+00 / 1.97e+00 | 20260507 03:03:24.709 | INFO | PerfReport: 57.54%, FrameLabeling -- 1.77e+01 / 2.00e+01, 1.19e+00 / 1.57e+00 | 20260507 03:03:24.709 | INFO | PerfReport: 7.03%, PulseAccumulating -- 2.16e+00 / 2.70e+00, 3.90e-01 / 2.30e-01 | 20260507 03:03:24.709 | INFO | PerfReport: 3.07%, Metrics -- 9.43e-01 / 1.22e+00, 2.51e-01 / 1.24e-01 | 20260507 03:03:24.709 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.19e-01 / 3.74e-01, 5.08e-02 / 2.38e-02 | 20260507 03:03:24.709 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:03:24.709 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:03:24.709 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:03:24.709 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 03:03:24.709 | INFO | PerfReport: 21.80%, Upload -- 2.10e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260507 03:03:24.709 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 03:03:24.709 | INFO | PerfReport: 31.88%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 03:03:24.709 | INFO | PerfReport: 1.56%, LockUpload -- 1.50e+00 / 0.00e+00, 5.00e+00 / 0.00e+00 | 20260507 03:03:24.709 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260507 03:03:24.709 | INFO | PerfReport: 22.14%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 03:03:24.709 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260507 03:03:24.814 | INFO | Starting a round of Baz IO | 20260507 03:03:27.911 | INFO | Populating RTMetrics report | 20260507 03:03:27.912 | INFO | RTMetrics report populated | 20260507 03:03:27.912 | INFO | Opening tmp RTMetrics json file | 20260507 03:03:27.931 | INFO | Serializing RTMetrics to json | 20260507 03:03:27.932 | INFO | Serialization done, swapping tmp file now... | 20260507 03:03:27.932 | INFO | RTMetrics done | 20260507 03:03:28.003 | INFO | Finished a round of Baz IO: Wrote 1.762 GiB in 3.189 seconds | 20260507 03:03:28.003 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.93% / 0.00% / 67.06% | 20260507 03:03:28.003 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.214187% / 3.97364e-06% / 8.34465e-05% | 20260507 03:03:28.003 | INFO | Baz IO overall: | 20260507 03:03:28.003 | INFO | BytesWritten : 3388087674651 | 20260507 03:03:28.003 | INFO | NumEvents : 1628552079111 | 20260507 03:03:29.842 | INFO | Analyzing chunk frames = [6586368,6586880) | 20260507 03:03:29.842 | INFO | MemInfo: { | 20260507 03:03:29.842 | INFO | "gpu" : " 17 GB", | 20260507 03:03:29.842 | INFO | "huge" : "", | 20260507 03:03:29.842 | INFO | "huge_shared" : " 27 GB", | 20260507 03:03:29.842 | INFO | "malloc" : " 14 GB", | 20260507 03:03:29.842 | INFO | "pinned" : " 29 GB" | 20260507 03:03:29.842 | INFO | } | 20260507 03:03:29.842 | INFO | Perf report for the last 5.13391 seconds: | 20260507 03:03:29.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:03:29.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:29.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:29.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:29.842 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 03:03:29.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:29.842 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 03:03:29.842 | INFO | PRE_HQ : 59.3%, 1.8, 55.5ms, 0 | 20260507 03:03:29.842 | INFO | BAZWRITER : 62.1%, 3.3, 105.3ms, 0 | 20260507 03:03:29.842 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:03:29.842 | INFO | Uploaded 36GiB to GPU in 2.04827s (17.5758GiB/s) | 20260507 03:03:29.842 | INFO | Downloaded 38GiB to GPU in 2.18206s (17.4147GiB/s) | 20260507 03:03:34.982 | INFO | Analyzing chunk frames = [6586880,6587392) | 20260507 03:03:34.982 | INFO | MemInfo: { | 20260507 03:03:34.982 | INFO | "gpu" : " 17 GB", | 20260507 03:03:34.982 | INFO | "huge" : "", | 20260507 03:03:34.982 | INFO | "huge_shared" : " 27 GB", | 20260507 03:03:34.982 | INFO | "malloc" : " 14 GB", | 20260507 03:03:34.982 | INFO | "pinned" : " 29 GB" | 20260507 03:03:34.982 | INFO | } | 20260507 03:03:34.982 | INFO | Perf report for the last 5.14017 seconds: | 20260507 03:03:34.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:03:34.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:34.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:34.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:34.982 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:03:34.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:34.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:34.982 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:03:34.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:34.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:34.982 | INFO | Uploaded 36GiB to GPU in 2.03221s (17.7147GiB/s) | 20260507 03:03:34.982 | INFO | Downloaded 34GiB to GPU in 1.93846s (17.5397GiB/s) | 20260507 03:03:40.114 | INFO | Analyzing chunk frames = [6587392,6587904) | 20260507 03:03:40.114 | INFO | MemInfo: { | 20260507 03:03:40.114 | INFO | "gpu" : " 17 GB", | 20260507 03:03:40.114 | INFO | "huge" : "", | 20260507 03:03:40.114 | INFO | "huge_shared" : " 27 GB", | 20260507 03:03:40.114 | INFO | "malloc" : " 14 GB", | 20260507 03:03:40.114 | INFO | "pinned" : " 29 GB" | 20260507 03:03:40.114 | INFO | } | 20260507 03:03:40.114 | INFO | Perf report for the last 5.13186 seconds: | 20260507 03:03:40.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:03:40.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:40.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:40.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:40.114 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 03:03:40.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:40.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:40.114 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:03:40.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:40.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:40.114 | INFO | Uploaded 36GiB to GPU in 2.02222s (17.8022GiB/s) | 20260507 03:03:40.114 | INFO | Downloaded 34GiB to GPU in 1.93879s (17.5367GiB/s) | 20260507 03:03:45.258 | INFO | Analyzing chunk frames = [6587904,6588416) | 20260507 03:03:45.258 | INFO | MemInfo: { | 20260507 03:03:45.258 | INFO | "gpu" : " 17 GB", | 20260507 03:03:45.258 | INFO | "huge" : "", | 20260507 03:03:45.258 | INFO | "huge_shared" : " 27 GB", | 20260507 03:03:45.258 | INFO | "malloc" : " 14 GB", | 20260507 03:03:45.258 | INFO | "pinned" : " 29 GB" | 20260507 03:03:45.258 | INFO | } | 20260507 03:03:45.258 | INFO | Perf report for the last 5.14436 seconds: | 20260507 03:03:45.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:03:45.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:45.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:45.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:45.258 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 03:03:45.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:45.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:45.258 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:03:45.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:45.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:45.258 | INFO | Uploaded 36GiB to GPU in 1.98482s (18.1377GiB/s) | 20260507 03:03:45.258 | INFO | Downloaded 34GiB to GPU in 1.90606s (17.8379GiB/s) | 20260507 03:03:50.407 | INFO | Analyzing chunk frames = [6588416,6588928) | 20260507 03:03:50.407 | INFO | MemInfo: { | 20260507 03:03:50.407 | INFO | "gpu" : " 17 GB", | 20260507 03:03:50.407 | INFO | "huge" : "", | 20260507 03:03:50.407 | INFO | "huge_shared" : " 27 GB", | 20260507 03:03:50.407 | INFO | "malloc" : " 14 GB", | 20260507 03:03:50.407 | INFO | "pinned" : " 29 GB" | 20260507 03:03:50.407 | INFO | } | 20260507 03:03:50.407 | INFO | Perf report for the last 5.14906 seconds: | 20260507 03:03:50.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:03:50.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:50.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:50.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:50.407 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 03:03:50.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:50.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:50.407 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:03:50.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:50.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:50.407 | INFO | Uploaded 36GiB to GPU in 2.01937s (17.8273GiB/s) | 20260507 03:03:50.408 | INFO | Downloaded 34GiB to GPU in 1.94384s (17.4912GiB/s) | 20260507 03:03:55.541 | INFO | Analyzing chunk frames = [6588928,6589440) | 20260507 03:03:55.541 | INFO | MemInfo: { | 20260507 03:03:55.541 | INFO | "gpu" : " 17 GB", | 20260507 03:03:55.541 | INFO | "huge" : "", | 20260507 03:03:55.541 | INFO | "huge_shared" : " 27 GB", | 20260507 03:03:55.541 | INFO | "malloc" : " 14 GB", | 20260507 03:03:55.541 | INFO | "pinned" : " 29 GB" | 20260507 03:03:55.541 | INFO | } | 20260507 03:03:55.541 | INFO | Perf report for the last 5.134 seconds: | 20260507 03:03:55.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:03:55.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:55.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:55.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:55.541 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 03:03:55.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:55.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:55.541 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 03:03:55.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:55.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:03:55.541 | INFO | Uploaded 36GiB to GPU in 2.03461s (17.6938GiB/s) | 20260507 03:03:55.542 | INFO | Downloaded 34GiB to GPU in 1.91198s (17.7826GiB/s) | 20260507 03:04:00.678 | INFO | Analyzing chunk frames = [6589440,6589952) | 20260507 03:04:00.678 | INFO | MemInfo: { | 20260507 03:04:00.678 | INFO | "gpu" : " 17 GB", | 20260507 03:04:00.678 | INFO | "huge" : "", | 20260507 03:04:00.678 | INFO | "huge_shared" : " 27 GB", | 20260507 03:04:00.678 | INFO | "malloc" : " 14 GB", | 20260507 03:04:00.678 | INFO | "pinned" : " 29 GB" | 20260507 03:04:00.678 | INFO | } | 20260507 03:04:00.678 | INFO | Perf report for the last 5.137 seconds: | 20260507 03:04:00.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:04:00.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:00.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:00.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:00.678 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 03:04:00.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:00.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:00.679 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 03:04:00.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:00.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:00.679 | INFO | Uploaded 36GiB to GPU in 2.0332s (17.7061GiB/s) | 20260507 03:04:00.679 | INFO | Downloaded 34GiB to GPU in 1.97402s (17.2238GiB/s) | 20260507 03:04:05.814 | INFO | Analyzing chunk frames = [6589952,6590464) | 20260507 03:04:05.814 | INFO | MemInfo: { | 20260507 03:04:05.814 | INFO | "gpu" : " 17 GB", | 20260507 03:04:05.814 | INFO | "huge" : "", | 20260507 03:04:05.814 | INFO | "huge_shared" : " 27 GB", | 20260507 03:04:05.814 | INFO | "malloc" : " 14 GB", | 20260507 03:04:05.815 | INFO | "pinned" : " 29 GB" | 20260507 03:04:05.815 | INFO | } | 20260507 03:04:05.815 | INFO | Perf report for the last 5.13641 seconds: | 20260507 03:04:05.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:04:05.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:05.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:05.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:05.815 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 03:04:05.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:05.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:05.815 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 03:04:05.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:05.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:05.815 | INFO | Uploaded 36GiB to GPU in 1.95984s (18.3688GiB/s) | 20260507 03:04:05.815 | INFO | Downloaded 34GiB to GPU in 1.95838s (17.3613GiB/s) | 20260507 03:04:05.912 | INFO | Starting a round of Baz IO | 20260507 03:04:09.027 | INFO | Populating RTMetrics report | 20260507 03:04:09.027 | INFO | RTMetrics report populated | 20260507 03:04:09.027 | INFO | Opening tmp RTMetrics json file | 20260507 03:04:09.031 | INFO | Serializing RTMetrics to json | 20260507 03:04:09.046 | INFO | Serialization done, swapping tmp file now... | 20260507 03:04:09.046 | INFO | RTMetrics done | 20260507 03:04:09.121 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.209 seconds | 20260507 03:04:09.121 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.93% / 0.00% / 67.07% | 20260507 03:04:09.121 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.214032% / 7.94729e-06% / 9.13938e-05% | 20260507 03:04:09.121 | INFO | Baz IO overall: | 20260507 03:04:09.121 | INFO | BytesWritten : 3389980913483 | 20260507 03:04:09.121 | INFO | NumEvents : 1629446075418 | 20260507 03:04:10.955 | INFO | Analyzing chunk frames = [6590464,6590976) | 20260507 03:04:10.955 | INFO | MemInfo: { | 20260507 03:04:10.955 | INFO | "gpu" : " 17 GB", | 20260507 03:04:10.955 | INFO | "huge" : "", | 20260507 03:04:10.955 | INFO | "huge_shared" : " 27 GB", | 20260507 03:04:10.955 | INFO | "malloc" : " 14 GB", | 20260507 03:04:10.955 | INFO | "pinned" : " 29 GB" | 20260507 03:04:10.955 | INFO | } | 20260507 03:04:10.955 | INFO | Perf report for the last 5.14055 seconds: | 20260507 03:04:10.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:04:10.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:10.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:10.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:10.955 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 03:04:10.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:10.955 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260507 03:04:10.955 | INFO | PRE_HQ : 59.5%, 1.9, 57.9ms, 0 | 20260507 03:04:10.955 | INFO | BAZWRITER : 62.0%, 3.4, 109.6ms, 0 | 20260507 03:04:10.955 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:04:10.955 | INFO | Uploaded 36GiB to GPU in 2.03618s (17.6801GiB/s) | 20260507 03:04:10.956 | INFO | Downloaded 38GiB to GPU in 2.18427s (17.3971GiB/s) | 20260507 03:04:16.091 | INFO | Analyzing chunk frames = [6590976,6591488) | 20260507 03:04:16.091 | INFO | MemInfo: { | 20260507 03:04:16.091 | INFO | "gpu" : " 17 GB", | 20260507 03:04:16.091 | INFO | "huge" : "", | 20260507 03:04:16.091 | INFO | "huge_shared" : " 27 GB", | 20260507 03:04:16.091 | INFO | "malloc" : " 14 GB", | 20260507 03:04:16.091 | INFO | "pinned" : " 29 GB" | 20260507 03:04:16.091 | INFO | } | 20260507 03:04:16.091 | INFO | Perf report for the last 5.13555 seconds: | 20260507 03:04:16.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:04:16.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:16.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:16.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:16.091 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:04:16.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:16.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:16.091 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 03:04:16.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:16.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:16.091 | INFO | Uploaded 36GiB to GPU in 2.01268s (17.8866GiB/s) | 20260507 03:04:16.091 | INFO | Downloaded 34GiB to GPU in 1.93686s (17.5542GiB/s) | 20260507 03:04:21.233 | INFO | Analyzing chunk frames = [6591488,6592000) | 20260507 03:04:21.233 | INFO | MemInfo: { | 20260507 03:04:21.233 | INFO | "gpu" : " 17 GB", | 20260507 03:04:21.233 | INFO | "huge" : "", | 20260507 03:04:21.233 | INFO | "huge_shared" : " 27 GB", | 20260507 03:04:21.233 | INFO | "malloc" : " 14 GB", | 20260507 03:04:21.233 | INFO | "pinned" : " 29 GB" | 20260507 03:04:21.233 | INFO | } | 20260507 03:04:21.234 | INFO | Perf report for the last 5.14271 seconds: | 20260507 03:04:21.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:04:21.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:21.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:21.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:21.234 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 03:04:21.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:21.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:21.234 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:04:21.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:21.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:21.234 | INFO | Uploaded 36GiB to GPU in 1.98683s (18.1193GiB/s) | 20260507 03:04:21.234 | INFO | Downloaded 34GiB to GPU in 1.94153s (17.5119GiB/s) | 20260507 03:04:26.371 | INFO | Analyzing chunk frames = [6592000,6592512) | 20260507 03:04:26.371 | INFO | MemInfo: { | 20260507 03:04:26.371 | INFO | "gpu" : " 17 GB", | 20260507 03:04:26.371 | INFO | "huge" : "", | 20260507 03:04:26.371 | INFO | "huge_shared" : " 27 GB", | 20260507 03:04:26.371 | INFO | "malloc" : " 14 GB", | 20260507 03:04:26.371 | INFO | "pinned" : " 29 GB" | 20260507 03:04:26.371 | INFO | } | 20260507 03:04:26.371 | INFO | Perf report for the last 5.13719 seconds: | 20260507 03:04:26.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:04:26.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:26.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:26.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:26.371 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 03:04:26.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:26.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:26.371 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:04:26.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:26.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:26.371 | INFO | Uploaded 36GiB to GPU in 1.96224s (18.3464GiB/s) | 20260507 03:04:26.371 | INFO | Downloaded 34GiB to GPU in 1.92961s (17.6202GiB/s) | 20260507 03:04:31.514 | INFO | Analyzing chunk frames = [6592512,6593024) | 20260507 03:04:31.514 | INFO | MemInfo: { | 20260507 03:04:31.514 | INFO | "gpu" : " 17 GB", | 20260507 03:04:31.514 | INFO | "huge" : "", | 20260507 03:04:31.514 | INFO | "huge_shared" : " 27 GB", | 20260507 03:04:31.514 | INFO | "malloc" : " 14 GB", | 20260507 03:04:31.514 | INFO | "pinned" : " 29 GB" | 20260507 03:04:31.514 | INFO | } | 20260507 03:04:31.514 | INFO | Perf report for the last 5.14332 seconds: | 20260507 03:04:31.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:04:31.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:31.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:31.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:31.514 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 03:04:31.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:31.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:31.514 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:04:31.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:31.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:31.514 | INFO | Uploaded 36GiB to GPU in 2.01014s (17.9092GiB/s) | 20260507 03:04:31.514 | INFO | Downloaded 34GiB to GPU in 1.95147s (17.4228GiB/s) | 20260507 03:04:36.659 | INFO | Analyzing chunk frames = [6593024,6593536) | 20260507 03:04:36.659 | INFO | MemInfo: { | 20260507 03:04:36.659 | INFO | "gpu" : " 17 GB", | 20260507 03:04:36.659 | INFO | "huge" : "", | 20260507 03:04:36.659 | INFO | "huge_shared" : " 27 GB", | 20260507 03:04:36.659 | INFO | "malloc" : " 14 GB", | 20260507 03:04:36.659 | INFO | "pinned" : " 29 GB" | 20260507 03:04:36.659 | INFO | } | 20260507 03:04:36.659 | INFO | Perf report for the last 5.14531 seconds: | 20260507 03:04:36.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:04:36.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:36.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:36.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:36.659 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 03:04:36.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:36.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:36.660 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:04:36.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:36.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:36.660 | INFO | Uploaded 36GiB to GPU in 2.02324s (17.7933GiB/s) | 20260507 03:04:36.660 | INFO | Downloaded 34GiB to GPU in 1.93364s (17.5834GiB/s) | 20260507 03:04:41.791 | INFO | Analyzing chunk frames = [6593536,6594048) | 20260507 03:04:41.791 | INFO | MemInfo: { | 20260507 03:04:41.791 | INFO | "gpu" : " 17 GB", | 20260507 03:04:41.791 | INFO | "huge" : "", | 20260507 03:04:41.791 | INFO | "huge_shared" : " 27 GB", | 20260507 03:04:41.791 | INFO | "malloc" : " 14 GB", | 20260507 03:04:41.791 | INFO | "pinned" : " 29 GB" | 20260507 03:04:41.791 | INFO | } | 20260507 03:04:41.791 | INFO | Perf report for the last 5.13151 seconds: | 20260507 03:04:41.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:04:41.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:41.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:41.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:41.791 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 03:04:41.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:41.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:41.791 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 03:04:41.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:41.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:41.791 | INFO | Uploaded 36GiB to GPU in 1.96167s (18.3517GiB/s) | 20260507 03:04:41.791 | INFO | Downloaded 34GiB to GPU in 1.96282s (17.322GiB/s) | 20260507 03:04:46.926 | INFO | Analyzing chunk frames = [6594048,6594560) | 20260507 03:04:46.926 | INFO | MemInfo: { | 20260507 03:04:46.926 | INFO | "gpu" : " 17 GB", | 20260507 03:04:46.926 | INFO | "huge" : "", | 20260507 03:04:46.926 | INFO | "huge_shared" : " 27 GB", | 20260507 03:04:46.926 | INFO | "malloc" : " 14 GB", | 20260507 03:04:46.926 | INFO | "pinned" : " 29 GB" | 20260507 03:04:46.926 | INFO | } | 20260507 03:04:46.926 | INFO | Perf report for the last 5.13548 seconds: | 20260507 03:04:46.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:04:46.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:46.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:46.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:46.926 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:04:46.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:46.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:46.927 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 03:04:46.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:46.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:46.927 | INFO | Uploaded 36GiB to GPU in 1.99467s (18.0481GiB/s) | 20260507 03:04:46.927 | INFO | Downloaded 34GiB to GPU in 1.96874s (17.2699GiB/s) | 20260507 03:04:46.927 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:04:46.927 | INFO | Total mem usage: 18 GiB | 20260507 03:04:46.927 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:04:46.927 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:04:46.927 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:04:46.927 | INFO | High Water Breakdown: | 20260507 03:04:46.927 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:04:46.927 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:04:46.927 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:04:46.927 | INFO | ------------------------------------------ | 20260507 03:04:46.927 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:04:46.927 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:04:46.927 | INFO | Total mem usage: 26.5 GiB | 20260507 03:04:46.927 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:04:46.927 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:04:46.927 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:04:46.927 | INFO | High Water Breakdown: | 20260507 03:04:46.927 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:04:46.927 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:04:46.927 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:04:46.927 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:04:46.927 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:04:46.927 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:04:46.927 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:04:46.927 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:04:46.927 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:04:46.927 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:04:46.927 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:04:46.927 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:04:46.927 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:04:46.927 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:04:46.927 | INFO | ------------------------------------------ | 20260507 03:04:46.927 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:04:46.927 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:04:46.927 | INFO | Total mem usage: 25.6 GiB | 20260507 03:04:46.927 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:04:46.927 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:04:46.927 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:04:46.927 | INFO | High Water Breakdown: | 20260507 03:04:46.927 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:04:46.927 | INFO | ------------------------------------------ | 20260507 03:04:46.927 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:04:46.927 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:04:46.927 | INFO | Total mem usage: 15.7 GiB | 20260507 03:04:46.928 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:04:46.928 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:04:46.928 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:04:46.928 | INFO | High Water Breakdown: | 20260507 03:04:46.928 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:04:46.928 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:04:46.928 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:04:46.928 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:04:46.928 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:04:46.928 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:04:46.928 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:04:46.928 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:04:46.928 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:04:46.928 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:04:46.928 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:04:46.928 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:04:46.928 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:04:46.928 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:04:46.928 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:04:46.928 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:04:46.928 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:04:46.928 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:04:46.928 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:04:46.928 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:04:46.928 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:04:46.928 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:04:46.928 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:04:46.928 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:04:46.928 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:04:46.928 | INFO | ------------------------------------------ | 20260507 03:04:46.928 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:04:46.928 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:04:46.928 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:04:46.928 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:04:46.928 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.48e+01, 1.33e+01 / 1.68e+00 | 20260507 03:04:46.928 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.02e-01, 9.73e-03 / 6.70e-03 | 20260507 03:04:46.928 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.34e+00, 5.23e-01 / 3.14e-01 | 20260507 03:04:46.928 | INFO | PerfReport: 3.24%, Histogram -- 9.99e-01 / 1.44e+00, 1.72e-01 / 1.40e-01 | 20260507 03:04:46.928 | INFO | PerfReport: 16.04%, DME -- 3.95e+01 / 4.39e+01, 3.86e+00 / 1.94e+00 | 20260507 03:04:46.928 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.39e+00 / 1.90e+00 | 20260507 03:04:46.928 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.67e+00, 3.78e-01 / 2.37e-01 | 20260507 03:04:46.928 | INFO | PerfReport: 3.06%, Metrics -- 9.42e-01 / 1.27e+00, 2.53e-01 / 1.55e-01 | 20260507 03:04:46.928 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.19e-01 / 3.71e-01, 5.07e-02 / 2.33e-02 | 20260507 03:04:46.928 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:04:46.928 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:04:46.928 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:04:46.928 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 03:04:46.928 | INFO | PerfReport: 21.73%, Upload -- 2.09e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260507 03:04:46.928 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260507 03:04:46.928 | INFO | PerfReport: 32.01%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 03:04:46.928 | INFO | PerfReport: 1.37%, LockUpload -- 1.32e+00 / 0.00e+00, 4.63e+00 / 0.00e+00 | 20260507 03:04:46.928 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260507 03:04:46.928 | INFO | PerfReport: 22.25%, LockCompute -- 2.14e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 03:04:46.928 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.48e-02 / 0.00e+00 | 20260507 03:04:47.026 | INFO | Starting a round of Baz IO | 20260507 03:04:50.138 | INFO | Populating RTMetrics report | 20260507 03:04:50.138 | INFO | RTMetrics report populated | 20260507 03:04:50.138 | INFO | Opening tmp RTMetrics json file | 20260507 03:04:50.157 | INFO | Serializing RTMetrics to json | 20260507 03:04:50.158 | INFO | Serialization done, swapping tmp file now... | 20260507 03:04:50.158 | INFO | RTMetrics done | 20260507 03:04:50.232 | INFO | Finished a round of Baz IO: Wrote 1.764 GiB in 3.205 seconds | 20260507 03:04:50.232 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.93% / 0.00% / 67.07% | 20260507 03:04:50.232 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.213957% / 0% / 8.34465e-05% | 20260507 03:04:50.232 | INFO | Baz IO overall: | 20260507 03:04:50.232 | INFO | BytesWritten : 3391875000495 | 20260507 03:04:50.232 | INFO | NumEvents : 1630340407624 | 20260507 03:04:52.069 | INFO | Analyzing chunk frames = [6594560,6595072) | 20260507 03:04:52.070 | INFO | MemInfo: { | 20260507 03:04:52.070 | INFO | "gpu" : " 17 GB", | 20260507 03:04:52.070 | INFO | "huge" : "", | 20260507 03:04:52.070 | INFO | "huge_shared" : " 27 GB", | 20260507 03:04:52.070 | INFO | "malloc" : " 14 GB", | 20260507 03:04:52.070 | INFO | "pinned" : " 29 GB" | 20260507 03:04:52.070 | INFO | } | 20260507 03:04:52.070 | INFO | Perf report for the last 5.14319 seconds: | 20260507 03:04:52.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:04:52.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:52.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:52.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:52.070 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260507 03:04:52.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:52.070 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 03:04:52.070 | INFO | PRE_HQ : 59.6%, 1.8, 55.9ms, 0 | 20260507 03:04:52.070 | INFO | BAZWRITER : 62.0%, 3.3, 106.5ms, 0 | 20260507 03:04:52.070 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:04:52.070 | INFO | Uploaded 36GiB to GPU in 2.03417s (17.6977GiB/s) | 20260507 03:04:52.070 | INFO | Downloaded 38GiB to GPU in 2.17444s (17.4758GiB/s) | 20260507 03:04:57.212 | INFO | Analyzing chunk frames = [6595072,6595584) | 20260507 03:04:57.212 | INFO | MemInfo: { | 20260507 03:04:57.212 | INFO | "gpu" : " 17 GB", | 20260507 03:04:57.212 | INFO | "huge" : "", | 20260507 03:04:57.212 | INFO | "huge_shared" : " 27 GB", | 20260507 03:04:57.212 | INFO | "malloc" : " 14 GB", | 20260507 03:04:57.212 | INFO | "pinned" : " 29 GB" | 20260507 03:04:57.212 | INFO | } | 20260507 03:04:57.212 | INFO | Perf report for the last 5.14265 seconds: | 20260507 03:04:57.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:04:57.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:57.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:57.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:57.212 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:04:57.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:57.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:57.212 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:04:57.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:57.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:04:57.213 | INFO | Uploaded 36GiB to GPU in 2.0097s (17.9131GiB/s) | 20260507 03:04:57.213 | INFO | Downloaded 34GiB to GPU in 1.93334s (17.5862GiB/s) | 20260507 03:05:02.350 | INFO | Analyzing chunk frames = [6595584,6596096) | 20260507 03:05:02.350 | INFO | MemInfo: { | 20260507 03:05:02.350 | INFO | "gpu" : " 17 GB", | 20260507 03:05:02.350 | INFO | "huge" : "", | 20260507 03:05:02.350 | INFO | "huge_shared" : " 27 GB", | 20260507 03:05:02.350 | INFO | "malloc" : " 14 GB", | 20260507 03:05:02.350 | INFO | "pinned" : " 29 GB" | 20260507 03:05:02.351 | INFO | } | 20260507 03:05:02.351 | INFO | Perf report for the last 5.13828 seconds: | 20260507 03:05:02.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:05:02.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:02.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:02.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:02.351 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 03:05:02.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:02.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:02.351 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:05:02.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:02.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:02.351 | INFO | Uploaded 36GiB to GPU in 2.01207s (17.8921GiB/s) | 20260507 03:05:02.351 | INFO | Downloaded 34GiB to GPU in 1.93787s (17.5451GiB/s) | 20260507 03:05:07.490 | INFO | Analyzing chunk frames = [6596096,6596608) | 20260507 03:05:07.490 | INFO | MemInfo: { | 20260507 03:05:07.490 | INFO | "gpu" : " 17 GB", | 20260507 03:05:07.490 | INFO | "huge" : "", | 20260507 03:05:07.490 | INFO | "huge_shared" : " 27 GB", | 20260507 03:05:07.490 | INFO | "malloc" : " 14 GB", | 20260507 03:05:07.490 | INFO | "pinned" : " 29 GB" | 20260507 03:05:07.490 | INFO | } | 20260507 03:05:07.490 | INFO | Perf report for the last 5.13974 seconds: | 20260507 03:05:07.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:05:07.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:07.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:07.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:07.490 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 03:05:07.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:07.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:07.490 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 03:05:07.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:07.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:07.491 | INFO | Uploaded 36GiB to GPU in 2.01187s (17.8938GiB/s) | 20260507 03:05:07.491 | INFO | Downloaded 34GiB to GPU in 1.93136s (17.6042GiB/s) | 20260507 03:05:12.626 | INFO | Analyzing chunk frames = [6596608,6597120) | 20260507 03:05:12.626 | INFO | MemInfo: { | 20260507 03:05:12.626 | INFO | "gpu" : " 17 GB", | 20260507 03:05:12.626 | INFO | "huge" : "", | 20260507 03:05:12.626 | INFO | "huge_shared" : " 27 GB", | 20260507 03:05:12.626 | INFO | "malloc" : " 14 GB", | 20260507 03:05:12.626 | INFO | "pinned" : " 29 GB" | 20260507 03:05:12.626 | INFO | } | 20260507 03:05:12.626 | INFO | Perf report for the last 5.13546 seconds: | 20260507 03:05:12.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:05:12.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:12.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:12.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:12.626 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 03:05:12.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:12.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:12.626 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:05:12.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:12.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:12.626 | INFO | Uploaded 36GiB to GPU in 2.03978s (17.649GiB/s) | 20260507 03:05:12.626 | INFO | Downloaded 34GiB to GPU in 1.93931s (17.532GiB/s) | 20260507 03:05:17.768 | INFO | Analyzing chunk frames = [6597120,6597632) | 20260507 03:05:17.768 | INFO | MemInfo: { | 20260507 03:05:17.768 | INFO | "gpu" : " 17 GB", | 20260507 03:05:17.768 | INFO | "huge" : "", | 20260507 03:05:17.768 | INFO | "huge_shared" : " 27 GB", | 20260507 03:05:17.768 | INFO | "malloc" : " 14 GB", | 20260507 03:05:17.768 | INFO | "pinned" : " 29 GB" | 20260507 03:05:17.768 | INFO | } | 20260507 03:05:17.768 | INFO | Perf report for the last 5.14251 seconds: | 20260507 03:05:17.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:05:17.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:17.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:17.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:17.768 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:05:17.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:17.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:17.769 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 03:05:17.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:17.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:17.769 | INFO | Uploaded 36GiB to GPU in 2.05603s (17.5095GiB/s) | 20260507 03:05:17.769 | INFO | Downloaded 34GiB to GPU in 1.9587s (17.3585GiB/s) | 20260507 03:05:22.906 | INFO | Analyzing chunk frames = [6597632,6598144) | 20260507 03:05:22.906 | INFO | MemInfo: { | 20260507 03:05:22.906 | INFO | "gpu" : " 17 GB", | 20260507 03:05:22.906 | INFO | "huge" : "", | 20260507 03:05:22.906 | INFO | "huge_shared" : " 27 GB", | 20260507 03:05:22.906 | INFO | "malloc" : " 14 GB", | 20260507 03:05:22.906 | INFO | "pinned" : " 29 GB" | 20260507 03:05:22.906 | INFO | } | 20260507 03:05:22.906 | INFO | Perf report for the last 5.13777 seconds: | 20260507 03:05:22.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:05:22.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:22.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:22.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:22.906 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:05:22.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:22.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:22.906 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 03:05:22.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:22.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:22.907 | INFO | Uploaded 36GiB to GPU in 1.99134s (18.0783GiB/s) | 20260507 03:05:22.907 | INFO | Downloaded 34GiB to GPU in 1.97425s (17.2217GiB/s) | 20260507 03:05:28.050 | INFO | Analyzing chunk frames = [6598144,6598656) | 20260507 03:05:28.050 | INFO | MemInfo: { | 20260507 03:05:28.050 | INFO | "gpu" : " 17 GB", | 20260507 03:05:28.051 | INFO | "huge" : "", | 20260507 03:05:28.051 | INFO | "huge_shared" : " 27 GB", | 20260507 03:05:28.051 | INFO | "malloc" : " 14 GB", | 20260507 03:05:28.051 | INFO | "pinned" : " 29 GB" | 20260507 03:05:28.051 | INFO | } | 20260507 03:05:28.051 | INFO | Perf report for the last 5.14459 seconds: | 20260507 03:05:28.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:05:28.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:28.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:28.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:28.051 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:05:28.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:28.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:28.051 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 03:05:28.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:28.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:28.051 | INFO | Uploaded 36GiB to GPU in 1.98854s (18.1037GiB/s) | 20260507 03:05:28.051 | INFO | Downloaded 34GiB to GPU in 1.97601s (17.2064GiB/s) | 20260507 03:05:28.148 | INFO | Starting a round of Baz IO | 20260507 03:05:31.258 | INFO | Populating RTMetrics report | 20260507 03:05:31.259 | INFO | RTMetrics report populated | 20260507 03:05:31.259 | INFO | Opening tmp RTMetrics json file | 20260507 03:05:31.263 | INFO | Serializing RTMetrics to json | 20260507 03:05:31.277 | INFO | Serialization done, swapping tmp file now... | 20260507 03:05:31.278 | INFO | RTMetrics done | 20260507 03:05:31.350 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.202 seconds | 20260507 03:05:31.350 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.93% / 0.00% / 67.07% | 20260507 03:05:31.350 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.212999% / 3.97364e-06% / 7.94729e-05% | 20260507 03:05:31.350 | INFO | Baz IO overall: | 20260507 03:05:31.350 | INFO | BytesWritten : 3393768008200 | 20260507 03:05:31.350 | INFO | NumEvents : 1631234284712 | 20260507 03:05:33.186 | INFO | Analyzing chunk frames = [6598656,6599168) | 20260507 03:05:33.186 | INFO | MemInfo: { | 20260507 03:05:33.186 | INFO | "gpu" : " 17 GB", | 20260507 03:05:33.186 | INFO | "huge" : "", | 20260507 03:05:33.186 | INFO | "huge_shared" : " 27 GB", | 20260507 03:05:33.186 | INFO | "malloc" : " 14 GB", | 20260507 03:05:33.186 | INFO | "pinned" : " 29 GB" | 20260507 03:05:33.186 | INFO | } | 20260507 03:05:33.187 | INFO | Perf report for the last 5.1358 seconds: | 20260507 03:05:33.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:05:33.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:33.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:33.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:33.187 | INFO | ANALYSIS : 61.9%, 3.0, 96.3ms, 0 | 20260507 03:05:33.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:33.187 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 03:05:33.187 | INFO | PRE_HQ : 59.4%, 1.8, 55.9ms, 0 | 20260507 03:05:33.187 | INFO | BAZWRITER : 61.9%, 3.3, 105.7ms, 0 | 20260507 03:05:33.187 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:05:33.187 | INFO | Uploaded 36GiB to GPU in 2.0483s (17.5756GiB/s) | 20260507 03:05:33.187 | INFO | Downloaded 38GiB to GPU in 2.19606s (17.3038GiB/s) | 20260507 03:05:38.324 | INFO | Analyzing chunk frames = [6599168,6599680) | 20260507 03:05:38.324 | INFO | MemInfo: { | 20260507 03:05:38.324 | INFO | "gpu" : " 17 GB", | 20260507 03:05:38.324 | INFO | "huge" : "", | 20260507 03:05:38.324 | INFO | "huge_shared" : " 27 GB", | 20260507 03:05:38.324 | INFO | "malloc" : " 14 GB", | 20260507 03:05:38.324 | INFO | "pinned" : " 29 GB" | 20260507 03:05:38.324 | INFO | } | 20260507 03:05:38.324 | INFO | Perf report for the last 5.13778 seconds: | 20260507 03:05:38.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:05:38.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:38.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:38.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:38.324 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:05:38.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:38.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:38.324 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:05:38.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:38.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:38.324 | INFO | Uploaded 36GiB to GPU in 2.03599s (17.6819GiB/s) | 20260507 03:05:38.325 | INFO | Downloaded 34GiB to GPU in 1.94211s (17.5067GiB/s) | 20260507 03:05:43.463 | INFO | Analyzing chunk frames = [6599680,6600192) | 20260507 03:05:43.463 | INFO | MemInfo: { | 20260507 03:05:43.463 | INFO | "gpu" : " 17 GB", | 20260507 03:05:43.463 | INFO | "huge" : "", | 20260507 03:05:43.464 | INFO | "huge_shared" : " 27 GB", | 20260507 03:05:43.464 | INFO | "malloc" : " 14 GB", | 20260507 03:05:43.464 | INFO | "pinned" : " 29 GB" | 20260507 03:05:43.464 | INFO | } | 20260507 03:05:43.464 | INFO | Perf report for the last 5.13927 seconds: | 20260507 03:05:43.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:05:43.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:43.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:43.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:43.464 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 03:05:43.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:43.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:43.464 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:05:43.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:43.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:43.464 | INFO | Uploaded 36GiB to GPU in 1.9856s (18.1305GiB/s) | 20260507 03:05:43.464 | INFO | Downloaded 34GiB to GPU in 1.92358s (17.6753GiB/s) | 20260507 03:05:48.603 | INFO | Analyzing chunk frames = [6600192,6600704) | 20260507 03:05:48.603 | INFO | MemInfo: { | 20260507 03:05:48.603 | INFO | "gpu" : " 17 GB", | 20260507 03:05:48.603 | INFO | "huge" : "", | 20260507 03:05:48.603 | INFO | "huge_shared" : " 27 GB", | 20260507 03:05:48.603 | INFO | "malloc" : " 14 GB", | 20260507 03:05:48.603 | INFO | "pinned" : " 29 GB" | 20260507 03:05:48.604 | INFO | } | 20260507 03:05:48.604 | INFO | Perf report for the last 5.13997 seconds: | 20260507 03:05:48.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:05:48.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:48.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:48.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:48.604 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:05:48.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:48.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:48.604 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 03:05:48.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:48.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:48.604 | INFO | Uploaded 36GiB to GPU in 1.98666s (18.1209GiB/s) | 20260507 03:05:48.604 | INFO | Downloaded 34GiB to GPU in 1.92105s (17.6987GiB/s) | 20260507 03:05:53.743 | INFO | Analyzing chunk frames = [6600704,6601216) | 20260507 03:05:53.743 | INFO | MemInfo: { | 20260507 03:05:53.743 | INFO | "gpu" : " 17 GB", | 20260507 03:05:53.743 | INFO | "huge" : "", | 20260507 03:05:53.743 | INFO | "huge_shared" : " 27 GB", | 20260507 03:05:53.743 | INFO | "malloc" : " 14 GB", | 20260507 03:05:53.743 | INFO | "pinned" : " 29 GB" | 20260507 03:05:53.743 | INFO | } | 20260507 03:05:53.743 | INFO | Perf report for the last 5.13985 seconds: | 20260507 03:05:53.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:05:53.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:53.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:53.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:53.743 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 03:05:53.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:53.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:53.744 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 03:05:53.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:53.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:53.744 | INFO | Uploaded 36GiB to GPU in 2.04746s (17.5828GiB/s) | 20260507 03:05:53.744 | INFO | Downloaded 34GiB to GPU in 1.94097s (17.517GiB/s) | 20260507 03:05:58.877 | INFO | Analyzing chunk frames = [6601216,6601728) | 20260507 03:05:58.877 | INFO | MemInfo: { | 20260507 03:05:58.877 | INFO | "gpu" : " 17 GB", | 20260507 03:05:58.877 | INFO | "huge" : "", | 20260507 03:05:58.877 | INFO | "huge_shared" : " 27 GB", | 20260507 03:05:58.877 | INFO | "malloc" : " 14 GB", | 20260507 03:05:58.877 | INFO | "pinned" : " 29 GB" | 20260507 03:05:58.877 | INFO | } | 20260507 03:05:58.877 | INFO | Perf report for the last 5.13366 seconds: | 20260507 03:05:58.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:05:58.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:58.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:58.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:58.877 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 03:05:58.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:58.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:58.877 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:05:58.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:58.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:05:58.877 | INFO | Uploaded 36GiB to GPU in 2.02377s (17.7886GiB/s) | 20260507 03:05:58.877 | INFO | Downloaded 34GiB to GPU in 1.93762s (17.5473GiB/s) | 20260507 03:06:04.026 | INFO | Analyzing chunk frames = [6601728,6602240) | 20260507 03:06:04.026 | INFO | MemInfo: { | 20260507 03:06:04.026 | INFO | "gpu" : " 17 GB", | 20260507 03:06:04.026 | INFO | "huge" : "", | 20260507 03:06:04.026 | INFO | "huge_shared" : " 27 GB", | 20260507 03:06:04.026 | INFO | "malloc" : " 14 GB", | 20260507 03:06:04.026 | INFO | "pinned" : " 29 GB" | 20260507 03:06:04.026 | INFO | } | 20260507 03:06:04.026 | INFO | Perf report for the last 5.14868 seconds: | 20260507 03:06:04.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:06:04.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:04.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:04.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:04.026 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 03:06:04.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:04.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:04.026 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 03:06:04.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:04.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:04.026 | INFO | Uploaded 36GiB to GPU in 1.96028s (18.3647GiB/s) | 20260507 03:06:04.026 | INFO | Downloaded 34GiB to GPU in 1.99539s (17.0393GiB/s) | 20260507 03:06:09.165 | INFO | Analyzing chunk frames = [6602240,6602752) | 20260507 03:06:09.165 | INFO | MemInfo: { | 20260507 03:06:09.165 | INFO | "gpu" : " 17 GB", | 20260507 03:06:09.165 | INFO | "huge" : "", | 20260507 03:06:09.165 | INFO | "huge_shared" : " 27 GB", | 20260507 03:06:09.165 | INFO | "malloc" : " 14 GB", | 20260507 03:06:09.165 | INFO | "pinned" : " 29 GB" | 20260507 03:06:09.165 | INFO | } | 20260507 03:06:09.165 | INFO | Perf report for the last 5.13924 seconds: | 20260507 03:06:09.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:06:09.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:09.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:09.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:09.165 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:06:09.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:09.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:09.165 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 03:06:09.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:09.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:09.165 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:06:09.165 | INFO | Total mem usage: 18 GiB | 20260507 03:06:09.165 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:06:09.165 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 03:06:09.165 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:06:09.165 | INFO | High Water Breakdown: | 20260507 03:06:09.165 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:06:09.165 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:06:09.165 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:06:09.165 | INFO | ------------------------------------------ | 20260507 03:06:09.165 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:06:09.165 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:06:09.165 | INFO | Uploaded 36GiB to GPU in 1.98605s (18.1264GiB/s) | 20260507 03:06:09.165 | INFO | Total mem usage: 26.5 GiB | 20260507 03:06:09.165 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:06:09.166 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:06:09.165 | INFO | Downloaded 34GiB to GPU in 1.97705s (17.1973GiB/s) | 20260507 03:06:09.166 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:06:09.166 | INFO | High Water Breakdown: | 20260507 03:06:09.166 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:06:09.166 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:06:09.166 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:06:09.166 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:06:09.166 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:06:09.166 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:06:09.166 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:06:09.166 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:06:09.166 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:06:09.166 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:06:09.166 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:06:09.166 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:06:09.166 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:06:09.166 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:06:09.166 | INFO | ------------------------------------------ | 20260507 03:06:09.166 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:06:09.166 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:06:09.166 | INFO | Total mem usage: 25.6 GiB | 20260507 03:06:09.166 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:06:09.166 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 03:06:09.166 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 03:06:09.166 | INFO | High Water Breakdown: | 20260507 03:06:09.166 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 03:06:09.166 | INFO | ------------------------------------------ | 20260507 03:06:09.166 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:06:09.166 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:06:09.166 | INFO | Total mem usage: 15.7 GiB | 20260507 03:06:09.166 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:06:09.166 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:06:09.166 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:06:09.166 | INFO | High Water Breakdown: | 20260507 03:06:09.166 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:06:09.166 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:06:09.166 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:06:09.166 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:06:09.166 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:06:09.166 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:06:09.166 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:06:09.166 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:06:09.166 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:06:09.166 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:06:09.166 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:06:09.166 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:06:09.166 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:06:09.166 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:06:09.166 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:06:09.166 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:06:09.166 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:06:09.166 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:06:09.166 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:06:09.166 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:06:09.166 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:06:09.166 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:06:09.166 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:06:09.166 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:06:09.166 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:06:09.166 | INFO | ------------------------------------------ | 20260507 03:06:09.166 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:06:09.166 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:06:09.166 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:06:09.166 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:06:09.166 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.52e+01, 1.31e+01 / 2.15e+00 | 20260507 03:06:09.166 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.01e-01, 9.84e-03 / 5.58e-03 | 20260507 03:06:09.166 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.44e+00, 5.10e-01 / 2.23e-01 | 20260507 03:06:09.166 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.47e+00, 1.67e-01 / 1.06e-01 | 20260507 03:06:09.166 | INFO | PerfReport: 16.16%, DME -- 3.98e+01 / 4.45e+01, 4.14e+00 / 2.20e+00 | 20260507 03:06:09.166 | INFO | PerfReport: 57.59%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.25e+00 / 1.25e+00 | 20260507 03:06:09.167 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.15e+00 / 2.75e+00, 3.85e-01 / 1.71e-01 | 20260507 03:06:09.167 | INFO | PerfReport: 3.06%, Metrics -- 9.40e-01 / 1.27e+00, 2.49e-01 / 9.46e-02 | 20260507 03:06:09.167 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.18e-01 / 3.73e-01, 5.03e-02 / 2.49e-02 | 20260507 03:06:09.167 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:06:09.167 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:06:09.167 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:06:09.167 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 03:06:09.167 | INFO | PerfReport: 21.76%, Upload -- 2.10e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260507 03:06:09.167 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 03:06:09.167 | INFO | PerfReport: 31.92%, Compute -- 3.08e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 03:06:09.167 | INFO | PerfReport: 1.45%, LockUpload -- 1.40e+00 / 0.00e+00, 4.89e+00 / 0.00e+00 | 20260507 03:06:09.167 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.10e+00 / 0.00e+00 | 20260507 03:06:09.167 | INFO | PerfReport: 22.15%, LockCompute -- 2.13e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 03:06:09.167 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260507 03:06:09.262 | INFO | Starting a round of Baz IO | 20260507 03:06:12.368 | INFO | Populating RTMetrics report | 20260507 03:06:12.369 | INFO | RTMetrics report populated | 20260507 03:06:12.369 | INFO | Opening tmp RTMetrics json file | 20260507 03:06:12.388 | INFO | Serializing RTMetrics to json | 20260507 03:06:12.389 | INFO | Serialization done, swapping tmp file now... | 20260507 03:06:12.389 | INFO | RTMetrics done | 20260507 03:06:12.462 | INFO | Finished a round of Baz IO: Wrote 1.761 GiB in 3.199 seconds | 20260507 03:06:12.462 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.93% / 0.00% / 67.07% | 20260507 03:06:12.462 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.212232% / 0% / 6.35783e-05% | 20260507 03:06:12.462 | INFO | Baz IO overall: | 20260507 03:06:12.462 | INFO | BytesWritten : 3395658854067 | 20260507 03:06:12.462 | INFO | NumEvents : 1632126434123 | 20260507 03:06:14.302 | INFO | Analyzing chunk frames = [6602752,6603264) | 20260507 03:06:14.302 | INFO | MemInfo: { | 20260507 03:06:14.302 | INFO | "gpu" : " 17 GB", | 20260507 03:06:14.302 | INFO | "huge" : "", | 20260507 03:06:14.302 | INFO | "huge_shared" : " 27 GB", | 20260507 03:06:14.302 | INFO | "malloc" : " 14 GB", | 20260507 03:06:14.302 | INFO | "pinned" : " 29 GB" | 20260507 03:06:14.302 | INFO | } | 20260507 03:06:14.302 | INFO | Perf report for the last 5.13671 seconds: | 20260507 03:06:14.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:06:14.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:14.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:14.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:14.302 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 03:06:14.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:14.302 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 03:06:14.302 | INFO | PRE_HQ : 59.3%, 1.8, 55.4ms, 0 | 20260507 03:06:14.302 | INFO | BAZWRITER : 61.9%, 3.4, 106.5ms, 0 | 20260507 03:06:14.302 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:06:14.302 | INFO | Uploaded 36GiB to GPU in 2.05224s (17.5418GiB/s) | 20260507 03:06:14.302 | INFO | Downloaded 38GiB to GPU in 2.18428s (17.397GiB/s) | 20260507 03:06:19.435 | INFO | Analyzing chunk frames = [6603264,6603776) | 20260507 03:06:19.435 | INFO | MemInfo: { | 20260507 03:06:19.435 | INFO | "gpu" : " 17 GB", | 20260507 03:06:19.435 | INFO | "huge" : "", | 20260507 03:06:19.435 | INFO | "huge_shared" : " 27 GB", | 20260507 03:06:19.435 | INFO | "malloc" : " 14 GB", | 20260507 03:06:19.435 | INFO | "pinned" : " 29 GB" | 20260507 03:06:19.435 | INFO | } | 20260507 03:06:19.435 | INFO | Perf report for the last 5.13361 seconds: | 20260507 03:06:19.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:06:19.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:19.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:19.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:19.435 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 03:06:19.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:19.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:19.435 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 03:06:19.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:19.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:19.436 | INFO | Uploaded 36GiB to GPU in 2.04249s (17.6256GiB/s) | 20260507 03:06:19.436 | INFO | Downloaded 34GiB to GPU in 1.94271s (17.5013GiB/s) | 20260507 03:06:24.577 | INFO | Analyzing chunk frames = [6603776,6604288) | 20260507 03:06:24.577 | INFO | MemInfo: { | 20260507 03:06:24.577 | INFO | "gpu" : " 17 GB", | 20260507 03:06:24.577 | INFO | "huge" : "", | 20260507 03:06:24.577 | INFO | "huge_shared" : " 27 GB", | 20260507 03:06:24.577 | INFO | "malloc" : " 14 GB", | 20260507 03:06:24.577 | INFO | "pinned" : " 29 GB" | 20260507 03:06:24.577 | INFO | } | 20260507 03:06:24.577 | INFO | Perf report for the last 5.14146 seconds: | 20260507 03:06:24.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:06:24.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:24.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:24.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:24.577 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 03:06:24.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:24.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:24.577 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:06:24.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:24.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:24.577 | INFO | Uploaded 36GiB to GPU in 1.99913s (18.0078GiB/s) | 20260507 03:06:24.577 | INFO | Downloaded 34GiB to GPU in 1.93982s (17.5274GiB/s) | 20260507 03:06:29.716 | INFO | Analyzing chunk frames = [6604288,6604800) | 20260507 03:06:29.716 | INFO | MemInfo: { | 20260507 03:06:29.716 | INFO | "gpu" : " 17 GB", | 20260507 03:06:29.716 | INFO | "huge" : "", | 20260507 03:06:29.716 | INFO | "huge_shared" : " 27 GB", | 20260507 03:06:29.716 | INFO | "malloc" : " 14 GB", | 20260507 03:06:29.716 | INFO | "pinned" : " 29 GB" | 20260507 03:06:29.716 | INFO | } | 20260507 03:06:29.716 | INFO | Perf report for the last 5.13887 seconds: | 20260507 03:06:29.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:06:29.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:29.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:29.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:29.716 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 03:06:29.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:29.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:29.716 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:06:29.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:29.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:29.716 | INFO | Uploaded 36GiB to GPU in 1.97457s (18.2318GiB/s) | 20260507 03:06:29.716 | INFO | Downloaded 34GiB to GPU in 1.90825s (17.8173GiB/s) | 20260507 03:06:34.858 | INFO | Analyzing chunk frames = [6604800,6605312) | 20260507 03:06:34.858 | INFO | MemInfo: { | 20260507 03:06:34.858 | INFO | "gpu" : " 17 GB", | 20260507 03:06:34.858 | INFO | "huge" : "", | 20260507 03:06:34.859 | INFO | "huge_shared" : " 27 GB", | 20260507 03:06:34.859 | INFO | "malloc" : " 14 GB", | 20260507 03:06:34.859 | INFO | "pinned" : " 29 GB" | 20260507 03:06:34.859 | INFO | } | 20260507 03:06:34.859 | INFO | Perf report for the last 5.14284 seconds: | 20260507 03:06:34.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:06:34.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:34.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:34.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:34.859 | INFO | ANALYSIS : 59.8%, 3.0, 93.1ms, 0 | 20260507 03:06:34.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:34.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:34.859 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:06:34.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:34.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:34.859 | INFO | Uploaded 36GiB to GPU in 1.98935s (18.0964GiB/s) | 20260507 03:06:34.859 | INFO | Downloaded 34GiB to GPU in 1.94994s (17.4364GiB/s) | 20260507 03:06:39.998 | INFO | Analyzing chunk frames = [6605312,6605824) | 20260507 03:06:39.998 | INFO | MemInfo: { | 20260507 03:06:39.998 | INFO | "gpu" : " 17 GB", | 20260507 03:06:39.998 | INFO | "huge" : "", | 20260507 03:06:39.998 | INFO | "huge_shared" : " 27 GB", | 20260507 03:06:39.998 | INFO | "malloc" : " 14 GB", | 20260507 03:06:39.998 | INFO | "pinned" : " 29 GB" | 20260507 03:06:39.998 | INFO | } | 20260507 03:06:39.998 | INFO | Perf report for the last 5.13935 seconds: | 20260507 03:06:39.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:06:39.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:39.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:39.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:39.998 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:06:39.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:39.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:39.998 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 03:06:39.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:39.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:39.998 | INFO | Uploaded 36GiB to GPU in 2.05192s (17.5445GiB/s) | 20260507 03:06:39.998 | INFO | Downloaded 34GiB to GPU in 1.96371s (17.3141GiB/s) | 20260507 03:06:45.135 | INFO | Analyzing chunk frames = [6605824,6606336) | 20260507 03:06:45.135 | INFO | MemInfo: { | 20260507 03:06:45.135 | INFO | "gpu" : " 17 GB", | 20260507 03:06:45.135 | INFO | "huge" : "", | 20260507 03:06:45.135 | INFO | "huge_shared" : " 27 GB", | 20260507 03:06:45.135 | INFO | "malloc" : " 14 GB", | 20260507 03:06:45.135 | INFO | "pinned" : " 29 GB" | 20260507 03:06:45.135 | INFO | } | 20260507 03:06:45.135 | INFO | Perf report for the last 5.13713 seconds: | 20260507 03:06:45.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:06:45.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:45.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:45.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:45.135 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:06:45.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:45.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:45.135 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 03:06:45.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:45.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:45.135 | INFO | Uploaded 36GiB to GPU in 1.97421s (18.2351GiB/s) | 20260507 03:06:45.135 | INFO | Downloaded 34GiB to GPU in 1.9769s (17.1986GiB/s) | 20260507 03:06:50.281 | INFO | Analyzing chunk frames = [6606336,6606848) | 20260507 03:06:50.281 | INFO | MemInfo: { | 20260507 03:06:50.281 | INFO | "gpu" : " 17 GB", | 20260507 03:06:50.281 | INFO | "huge" : "", | 20260507 03:06:50.281 | INFO | "huge_shared" : " 27 GB", | 20260507 03:06:50.281 | INFO | "malloc" : " 14 GB", | 20260507 03:06:50.281 | INFO | "pinned" : " 29 GB" | 20260507 03:06:50.281 | INFO | } | 20260507 03:06:50.281 | INFO | Perf report for the last 5.14602 seconds: | 20260507 03:06:50.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:06:50.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:50.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:50.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:50.281 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 03:06:50.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:50.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:50.281 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:06:50.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:50.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:50.281 | INFO | Uploaded 36GiB to GPU in 1.99324s (18.061GiB/s) | 20260507 03:06:50.281 | INFO | Downloaded 34GiB to GPU in 1.97543s (17.2115GiB/s) | 20260507 03:06:50.378 | INFO | Starting a round of Baz IO | 20260507 03:06:53.486 | INFO | Populating RTMetrics report | 20260507 03:06:53.486 | INFO | RTMetrics report populated | 20260507 03:06:53.486 | INFO | Opening tmp RTMetrics json file | 20260507 03:06:53.487 | INFO | Serializing RTMetrics to json | 20260507 03:06:53.488 | INFO | Serialization done, swapping tmp file now... | 20260507 03:06:53.489 | INFO | RTMetrics done | 20260507 03:06:53.577 | INFO | Finished a round of Baz IO: Wrote 1.76 GiB in 3.199 seconds | 20260507 03:06:53.578 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.93% / 0.00% / 67.07% | 20260507 03:06:53.578 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.211787% / 7.94729e-06% / 7.94729e-05% | 20260507 03:06:53.578 | INFO | Baz IO overall: | 20260507 03:06:53.578 | INFO | BytesWritten : 3397549001919 | 20260507 03:06:53.578 | INFO | NumEvents : 1633018143566 | 20260507 03:06:55.414 | INFO | Analyzing chunk frames = [6606848,6607360) | 20260507 03:06:55.414 | INFO | MemInfo: { | 20260507 03:06:55.414 | INFO | "gpu" : " 17 GB", | 20260507 03:06:55.414 | INFO | "huge" : "", | 20260507 03:06:55.414 | INFO | "huge_shared" : " 27 GB", | 20260507 03:06:55.414 | INFO | "malloc" : " 14 GB", | 20260507 03:06:55.415 | INFO | "pinned" : " 29 GB" | 20260507 03:06:55.415 | INFO | } | 20260507 03:06:55.415 | INFO | Perf report for the last 5.13343 seconds: | 20260507 03:06:55.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:06:55.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:55.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:55.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:55.415 | INFO | ANALYSIS : 61.9%, 3.0, 96.1ms, 0 | 20260507 03:06:55.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:06:55.415 | INFO | RT_METRICS : 41.5%, 1.0, 21.7ms, 0 | 20260507 03:06:55.415 | INFO | PRE_HQ : 59.3%, 1.8, 55.3ms, 0 | 20260507 03:06:55.415 | INFO | BAZWRITER : 61.9%, 3.3, 105.8ms, 0 | 20260507 03:06:55.415 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:06:55.415 | INFO | Uploaded 36GiB to GPU in 2.046s (17.5953GiB/s) | 20260507 03:06:55.415 | INFO | Downloaded 38GiB to GPU in 2.18635s (17.3806GiB/s) | 20260507 03:07:00.550 | INFO | Analyzing chunk frames = [6607360,6607872) | 20260507 03:07:00.550 | INFO | MemInfo: { | 20260507 03:07:00.550 | INFO | "gpu" : " 17 GB", | 20260507 03:07:00.550 | INFO | "huge" : "", | 20260507 03:07:00.550 | INFO | "huge_shared" : " 27 GB", | 20260507 03:07:00.550 | INFO | "malloc" : " 14 GB", | 20260507 03:07:00.550 | INFO | "pinned" : " 29 GB" | 20260507 03:07:00.550 | INFO | } | 20260507 03:07:00.550 | INFO | Perf report for the last 5.13579 seconds: | 20260507 03:07:00.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:07:00.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:00.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:00.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:00.550 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:07:00.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:00.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:00.550 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:07:00.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:00.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:00.551 | INFO | Uploaded 36GiB to GPU in 2.03432s (17.6963GiB/s) | 20260507 03:07:00.551 | INFO | Downloaded 34GiB to GPU in 1.95009s (17.4351GiB/s) | 20260507 03:07:05.693 | INFO | Analyzing chunk frames = [6607872,6608384) | 20260507 03:07:05.693 | INFO | MemInfo: { | 20260507 03:07:05.693 | INFO | "gpu" : " 17 GB", | 20260507 03:07:05.693 | INFO | "huge" : "", | 20260507 03:07:05.693 | INFO | "huge_shared" : " 27 GB", | 20260507 03:07:05.693 | INFO | "malloc" : " 14 GB", | 20260507 03:07:05.693 | INFO | "pinned" : " 29 GB" | 20260507 03:07:05.693 | INFO | } | 20260507 03:07:05.693 | INFO | Perf report for the last 5.14262 seconds: | 20260507 03:07:05.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:07:05.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:05.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:05.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:05.693 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 03:07:05.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:05.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:05.693 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:07:05.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:05.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:05.693 | INFO | Uploaded 36GiB to GPU in 1.987s (18.1177GiB/s) | 20260507 03:07:05.693 | INFO | Downloaded 34GiB to GPU in 1.94001s (17.5257GiB/s) | 20260507 03:07:10.834 | INFO | Analyzing chunk frames = [6608384,6608896) | 20260507 03:07:10.834 | INFO | MemInfo: { | 20260507 03:07:10.834 | INFO | "gpu" : " 17 GB", | 20260507 03:07:10.834 | INFO | "huge" : "", | 20260507 03:07:10.834 | INFO | "huge_shared" : " 27 GB", | 20260507 03:07:10.834 | INFO | "malloc" : " 14 GB", | 20260507 03:07:10.834 | INFO | "pinned" : " 29 GB" | 20260507 03:07:10.834 | INFO | } | 20260507 03:07:10.834 | INFO | Perf report for the last 5.14095 seconds: | 20260507 03:07:10.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:07:10.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:10.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:10.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:10.834 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:07:10.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:10.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:10.834 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:07:10.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:10.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:10.834 | INFO | Uploaded 36GiB to GPU in 1.96052s (18.3624GiB/s) | 20260507 03:07:10.834 | INFO | Downloaded 34GiB to GPU in 1.91807s (17.7262GiB/s) | 20260507 03:07:15.979 | INFO | Analyzing chunk frames = [6608896,6609408) | 20260507 03:07:15.979 | INFO | MemInfo: { | 20260507 03:07:15.979 | INFO | "gpu" : " 17 GB", | 20260507 03:07:15.979 | INFO | "huge" : "", | 20260507 03:07:15.979 | INFO | "huge_shared" : " 27 GB", | 20260507 03:07:15.979 | INFO | "malloc" : " 14 GB", | 20260507 03:07:15.979 | INFO | "pinned" : " 29 GB" | 20260507 03:07:15.979 | INFO | } | 20260507 03:07:15.979 | INFO | Perf report for the last 5.14484 seconds: | 20260507 03:07:15.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:07:15.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:15.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:15.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:15.979 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 03:07:15.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:15.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:15.979 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:07:15.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:15.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:15.979 | INFO | Uploaded 36GiB to GPU in 2.00627s (17.9438GiB/s) | 20260507 03:07:15.979 | INFO | Downloaded 34GiB to GPU in 1.94279s (17.5006GiB/s) | 20260507 03:07:21.115 | INFO | Analyzing chunk frames = [6609408,6609920) | 20260507 03:07:21.115 | INFO | MemInfo: { | 20260507 03:07:21.115 | INFO | "gpu" : " 17 GB", | 20260507 03:07:21.115 | INFO | "huge" : "", | 20260507 03:07:21.115 | INFO | "huge_shared" : " 27 GB", | 20260507 03:07:21.115 | INFO | "malloc" : " 14 GB", | 20260507 03:07:21.115 | INFO | "pinned" : " 29 GB" | 20260507 03:07:21.115 | INFO | } | 20260507 03:07:21.115 | INFO | Perf report for the last 5.13612 seconds: | 20260507 03:07:21.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:07:21.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:21.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:21.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:21.115 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 03:07:21.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:21.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:21.115 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 03:07:21.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:21.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:21.115 | INFO | Uploaded 36GiB to GPU in 2.02448s (17.7824GiB/s) | 20260507 03:07:21.115 | INFO | Downloaded 34GiB to GPU in 1.92753s (17.6391GiB/s) | 20260507 03:07:26.251 | INFO | Analyzing chunk frames = [6609920,6610432) | 20260507 03:07:26.251 | INFO | MemInfo: { | 20260507 03:07:26.251 | INFO | "gpu" : " 17 GB", | 20260507 03:07:26.251 | INFO | "huge" : "", | 20260507 03:07:26.251 | INFO | "huge_shared" : " 27 GB", | 20260507 03:07:26.251 | INFO | "malloc" : " 14 GB", | 20260507 03:07:26.251 | INFO | "pinned" : " 29 GB" | 20260507 03:07:26.251 | INFO | } | 20260507 03:07:26.251 | INFO | Perf report for the last 5.13616 seconds: | 20260507 03:07:26.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:07:26.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:26.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:26.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:26.251 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:07:26.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:26.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:26.251 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 03:07:26.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:26.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:26.251 | INFO | Uploaded 36GiB to GPU in 1.96822s (18.2907GiB/s) | 20260507 03:07:26.251 | INFO | Downloaded 34GiB to GPU in 1.98457s (17.1322GiB/s) | 20260507 03:07:31.391 | INFO | Analyzing chunk frames = [6610432,6610944) | 20260507 03:07:31.391 | INFO | MemInfo: { | 20260507 03:07:31.391 | INFO | "gpu" : " 17 GB", | 20260507 03:07:31.391 | INFO | "huge" : "", | 20260507 03:07:31.391 | INFO | "huge_shared" : " 27 GB", | 20260507 03:07:31.391 | INFO | "malloc" : " 14 GB", | 20260507 03:07:31.391 | INFO | "pinned" : " 29 GB" | 20260507 03:07:31.391 | INFO | } | 20260507 03:07:31.391 | INFO | Perf report for the last 5.14012 seconds: | 20260507 03:07:31.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:07:31.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:31.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:31.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:31.391 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 03:07:31.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:31.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:31.391 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:07:31.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:31.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:31.392 | INFO | Uploaded 36GiB to GPU in 1.97s (18.2741GiB/s) | 20260507 03:07:31.392 | INFO | Downloaded 34GiB to GPU in 1.95362s (17.4036GiB/s) | 20260507 03:07:31.392 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:07:31.392 | INFO | Total mem usage: 18 GiB | 20260507 03:07:31.392 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:07:31.392 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 03:07:31.392 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:07:31.392 | INFO | High Water Breakdown: | 20260507 03:07:31.392 | INFO | 60.2% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:07:31.392 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:07:31.392 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:07:31.392 | INFO | ------------------------------------------ | 20260507 03:07:31.392 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:07:31.392 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:07:31.392 | INFO | Total mem usage: 26.5 GiB | 20260507 03:07:31.392 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:07:31.392 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:07:31.392 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 03:07:31.392 | INFO | High Water Breakdown: | 20260507 03:07:31.392 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:07:31.392 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:07:31.392 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:07:31.392 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:07:31.392 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:07:31.392 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:07:31.392 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:07:31.392 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:07:31.392 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:07:31.392 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:07:31.392 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:07:31.392 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:07:31.392 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:07:31.392 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:07:31.392 | INFO | ------------------------------------------ | 20260507 03:07:31.392 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:07:31.392 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:07:31.392 | INFO | Total mem usage: 25.6 GiB | 20260507 03:07:31.392 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:07:31.392 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 03:07:31.392 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 03:07:31.392 | INFO | High Water Breakdown: | 20260507 03:07:31.392 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 03:07:31.392 | INFO | ------------------------------------------ | 20260507 03:07:31.392 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:07:31.392 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:07:31.392 | INFO | Total mem usage: 15.7 GiB | 20260507 03:07:31.392 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:07:31.392 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:07:31.392 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:07:31.392 | INFO | High Water Breakdown: | 20260507 03:07:31.392 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:07:31.392 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:07:31.392 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:07:31.392 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:07:31.392 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:07:31.392 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:07:31.392 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:07:31.392 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:07:31.392 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:07:31.392 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:07:31.392 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:07:31.392 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:07:31.392 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:07:31.392 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:07:31.392 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:07:31.392 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:07:31.392 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:07:31.392 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:07:31.392 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:07:31.392 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:07:31.392 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:07:31.392 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:07:31.392 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:07:31.392 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:07:31.392 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:07:31.392 | INFO | ------------------------------------------ | 20260507 03:07:31.392 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:07:31.392 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:07:31.392 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:07:31.392 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:07:31.392 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.51e+01, 1.31e+01 / 1.80e+00 | 20260507 03:07:31.392 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.01e-01, 1.00e-02 / 4.67e-03 | 20260507 03:07:31.392 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.44e+00, 5.18e-01 / 2.65e-01 | 20260507 03:07:31.392 | INFO | PerfReport: 3.23%, Histogram -- 9.94e-01 / 1.47e+00, 1.68e-01 / 9.24e-02 | 20260507 03:07:31.392 | INFO | PerfReport: 16.09%, DME -- 3.96e+01 / 4.42e+01, 3.94e+00 / 2.20e+00 | 20260507 03:07:31.392 | INFO | PerfReport: 57.65%, FrameLabeling -- 1.77e+01 / 2.06e+01, 1.30e+00 / 1.91e+00 | 20260507 03:07:31.392 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.15e+00 / 2.71e+00, 3.81e-01 / 2.38e-01 | 20260507 03:07:31.392 | INFO | PerfReport: 3.06%, Metrics -- 9.40e-01 / 1.24e+00, 2.48e-01 / 1.26e-01 | 20260507 03:07:31.392 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.18e-01 / 3.66e-01, 4.99e-02 / 1.97e-02 | 20260507 03:07:31.392 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:07:31.392 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:07:31.392 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:07:31.392 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 03:07:31.392 | INFO | PerfReport: 21.69%, Upload -- 2.09e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260507 03:07:31.392 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 03:07:31.392 | INFO | PerfReport: 31.97%, Compute -- 3.08e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 03:07:31.392 | INFO | PerfReport: 1.34%, LockUpload -- 1.29e+00 / 0.00e+00, 4.72e+00 / 0.00e+00 | 20260507 03:07:31.392 | INFO | PerfReport: 1.41%, LockDownload -- 1.35e+00 / 0.00e+00, 3.26e+00 / 0.00e+00 | 20260507 03:07:31.393 | INFO | PerfReport: 22.18%, LockCompute -- 2.14e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 03:07:31.393 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.59e-02 / 0.00e+00 | 20260507 03:07:31.488 | INFO | Starting a round of Baz IO | 20260507 03:07:34.602 | INFO | Populating RTMetrics report | 20260507 03:07:34.603 | INFO | RTMetrics report populated | 20260507 03:07:34.603 | INFO | Opening tmp RTMetrics json file | 20260507 03:07:34.615 | INFO | Serializing RTMetrics to json | 20260507 03:07:34.622 | INFO | Serialization done, swapping tmp file now... | 20260507 03:07:34.623 | INFO | RTMetrics done | 20260507 03:07:34.699 | INFO | Finished a round of Baz IO: Wrote 1.761 GiB in 3.211 seconds | 20260507 03:07:34.699 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.93% / 0.00% / 67.07% | 20260507 03:07:34.699 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.211267% / 0% / 5.5631e-05% | 20260507 03:07:34.699 | INFO | Baz IO overall: | 20260507 03:07:34.699 | INFO | BytesWritten : 3399440361576 | 20260507 03:07:34.699 | INFO | NumEvents : 1633910884211 | 20260507 03:07:36.528 | INFO | Analyzing chunk frames = [6610944,6611456) | 20260507 03:07:36.528 | INFO | MemInfo: { | 20260507 03:07:36.528 | INFO | "gpu" : " 17 GB", | 20260507 03:07:36.528 | INFO | "huge" : "", | 20260507 03:07:36.529 | INFO | "huge_shared" : " 27 GB", | 20260507 03:07:36.529 | INFO | "malloc" : " 14 GB", | 20260507 03:07:36.529 | INFO | "pinned" : " 29 GB" | 20260507 03:07:36.529 | INFO | } | 20260507 03:07:36.529 | INFO | Perf report for the last 5.13739 seconds: | 20260507 03:07:36.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:07:36.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:36.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:36.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:36.529 | INFO | ANALYSIS : 62.0%, 3.0, 96.3ms, 0 | 20260507 03:07:36.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:36.529 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 03:07:36.529 | INFO | PRE_HQ : 59.6%, 1.8, 55.9ms, 0 | 20260507 03:07:36.529 | INFO | BAZWRITER : 62.0%, 3.3, 106.5ms, 0 | 20260507 03:07:36.529 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:07:36.529 | INFO | Uploaded 36GiB to GPU in 2.02323s (17.7934GiB/s) | 20260507 03:07:36.529 | INFO | Downloaded 38GiB to GPU in 2.16647s (17.5401GiB/s) | 20260507 03:07:41.675 | INFO | Analyzing chunk frames = [6611456,6611968) | 20260507 03:07:41.675 | INFO | MemInfo: { | 20260507 03:07:41.675 | INFO | "gpu" : " 17 GB", | 20260507 03:07:41.675 | INFO | "huge" : "", | 20260507 03:07:41.675 | INFO | "huge_shared" : " 27 GB", | 20260507 03:07:41.675 | INFO | "malloc" : " 14 GB", | 20260507 03:07:41.675 | INFO | "pinned" : " 29 GB" | 20260507 03:07:41.675 | INFO | } | 20260507 03:07:41.675 | INFO | Perf report for the last 5.1462 seconds: | 20260507 03:07:41.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:07:41.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:41.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:41.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:41.675 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 03:07:41.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:41.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:41.675 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:07:41.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:41.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:41.675 | INFO | Uploaded 36GiB to GPU in 2.02138s (17.8096GiB/s) | 20260507 03:07:41.675 | INFO | Downloaded 34GiB to GPU in 1.93325s (17.587GiB/s) | 20260507 03:07:46.816 | INFO | Analyzing chunk frames = [6611968,6612480) | 20260507 03:07:46.816 | INFO | MemInfo: { | 20260507 03:07:46.816 | INFO | "gpu" : " 17 GB", | 20260507 03:07:46.816 | INFO | "huge" : "", | 20260507 03:07:46.816 | INFO | "huge_shared" : " 27 GB", | 20260507 03:07:46.816 | INFO | "malloc" : " 14 GB", | 20260507 03:07:46.816 | INFO | "pinned" : " 29 GB" | 20260507 03:07:46.816 | INFO | } | 20260507 03:07:46.816 | INFO | Perf report for the last 5.14079 seconds: | 20260507 03:07:46.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:07:46.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:46.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:46.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:46.816 | INFO | ANALYSIS : 60.0%, 3.0, 93.3ms, 0 | 20260507 03:07:46.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:46.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:46.816 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:07:46.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:46.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:46.816 | INFO | Uploaded 36GiB to GPU in 2.01457s (17.8698GiB/s) | 20260507 03:07:46.816 | INFO | Downloaded 34GiB to GPU in 1.92514s (17.6611GiB/s) | 20260507 03:07:51.947 | INFO | Analyzing chunk frames = [6612480,6612992) | 20260507 03:07:51.947 | INFO | MemInfo: { | 20260507 03:07:51.947 | INFO | "gpu" : " 17 GB", | 20260507 03:07:51.947 | INFO | "huge" : "", | 20260507 03:07:51.947 | INFO | "huge_shared" : " 27 GB", | 20260507 03:07:51.947 | INFO | "malloc" : " 14 GB", | 20260507 03:07:51.947 | INFO | "pinned" : " 29 GB" | 20260507 03:07:51.947 | INFO | } | 20260507 03:07:51.947 | INFO | Perf report for the last 5.13101 seconds: | 20260507 03:07:51.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:07:51.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:51.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:51.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:51.947 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 03:07:51.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:51.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:51.947 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:07:51.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:51.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:51.947 | INFO | Uploaded 36GiB to GPU in 1.98495s (18.1365GiB/s) | 20260507 03:07:51.947 | INFO | Downloaded 34GiB to GPU in 1.91422s (17.7618GiB/s) | 20260507 03:07:57.086 | INFO | Analyzing chunk frames = [6612992,6613504) | 20260507 03:07:57.086 | INFO | MemInfo: { | 20260507 03:07:57.086 | INFO | "gpu" : " 17 GB", | 20260507 03:07:57.086 | INFO | "huge" : "", | 20260507 03:07:57.086 | INFO | "huge_shared" : " 27 GB", | 20260507 03:07:57.086 | INFO | "malloc" : " 14 GB", | 20260507 03:07:57.086 | INFO | "pinned" : " 29 GB" | 20260507 03:07:57.086 | INFO | } | 20260507 03:07:57.086 | INFO | Perf report for the last 5.13933 seconds: | 20260507 03:07:57.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:07:57.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:57.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:57.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:57.086 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 03:07:57.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:57.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:57.086 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 03:07:57.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:57.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:07:57.086 | INFO | Uploaded 36GiB to GPU in 2.00873s (17.9218GiB/s) | 20260507 03:07:57.086 | INFO | Downloaded 34GiB to GPU in 1.94064s (17.52GiB/s) | 20260507 03:08:02.226 | INFO | Analyzing chunk frames = [6613504,6614016) | 20260507 03:08:02.226 | INFO | MemInfo: { | 20260507 03:08:02.226 | INFO | "gpu" : " 17 GB", | 20260507 03:08:02.226 | INFO | "huge" : "", | 20260507 03:08:02.226 | INFO | "huge_shared" : " 27 GB", | 20260507 03:08:02.226 | INFO | "malloc" : " 14 GB", | 20260507 03:08:02.226 | INFO | "pinned" : " 29 GB" | 20260507 03:08:02.226 | INFO | } | 20260507 03:08:02.226 | INFO | Perf report for the last 5.13996 seconds: | 20260507 03:08:02.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:08:02.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:02.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:02.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:02.226 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:08:02.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:02.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:02.226 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 03:08:02.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:02.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:02.226 | INFO | Uploaded 36GiB to GPU in 2.05763s (17.4958GiB/s) | 20260507 03:08:02.226 | INFO | Downloaded 34GiB to GPU in 1.95493s (17.392GiB/s) | 20260507 03:08:07.363 | INFO | Analyzing chunk frames = [6614016,6614528) | 20260507 03:08:07.363 | INFO | MemInfo: { | 20260507 03:08:07.363 | INFO | "gpu" : " 17 GB", | 20260507 03:08:07.363 | INFO | "huge" : "", | 20260507 03:08:07.363 | INFO | "huge_shared" : " 27 GB", | 20260507 03:08:07.363 | INFO | "malloc" : " 14 GB", | 20260507 03:08:07.363 | INFO | "pinned" : " 29 GB" | 20260507 03:08:07.363 | INFO | } | 20260507 03:08:07.363 | INFO | Perf report for the last 5.13677 seconds: | 20260507 03:08:07.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:08:07.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:07.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:07.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:07.363 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:08:07.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:07.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:07.363 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:08:07.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:07.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:07.363 | INFO | Uploaded 36GiB to GPU in 1.99135s (18.0782GiB/s) | 20260507 03:08:07.363 | INFO | Downloaded 34GiB to GPU in 1.97783s (17.1906GiB/s) | 20260507 03:08:12.500 | INFO | Analyzing chunk frames = [6614528,6615040) | 20260507 03:08:12.500 | INFO | MemInfo: { | 20260507 03:08:12.500 | INFO | "gpu" : " 17 GB", | 20260507 03:08:12.500 | INFO | "huge" : "", | 20260507 03:08:12.500 | INFO | "huge_shared" : " 27 GB", | 20260507 03:08:12.500 | INFO | "malloc" : " 14 GB", | 20260507 03:08:12.500 | INFO | "pinned" : " 29 GB" | 20260507 03:08:12.500 | INFO | } | 20260507 03:08:12.500 | INFO | Perf report for the last 5.137 seconds: | 20260507 03:08:12.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:08:12.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:12.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:12.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:12.500 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:08:12.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:12.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:12.500 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:08:12.500 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:12.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:12.500 | INFO | Uploaded 36GiB to GPU in 1.99712s (18.026GiB/s) | 20260507 03:08:12.500 | INFO | Downloaded 34GiB to GPU in 1.97913s (17.1793GiB/s) | 20260507 03:08:12.599 | INFO | Starting a round of Baz IO | 20260507 03:08:15.709 | INFO | Populating RTMetrics report | 20260507 03:08:15.709 | INFO | RTMetrics report populated | 20260507 03:08:15.709 | INFO | Opening tmp RTMetrics json file | 20260507 03:08:15.710 | INFO | Serializing RTMetrics to json | 20260507 03:08:15.723 | INFO | Serialization done, swapping tmp file now... | 20260507 03:08:15.729 | INFO | RTMetrics done | 20260507 03:08:15.801 | INFO | Finished a round of Baz IO: Wrote 1.762 GiB in 3.201 seconds | 20260507 03:08:15.801 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.92% / 0.00% / 67.07% | 20260507 03:08:15.801 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.210273% / 0% / 8.34465e-05% | 20260507 03:08:15.801 | INFO | Baz IO overall: | 20260507 03:08:15.801 | INFO | BytesWritten : 3401332544283 | 20260507 03:08:15.801 | INFO | NumEvents : 1634804164552 | 20260507 03:08:17.644 | INFO | Analyzing chunk frames = [6615040,6615552) | 20260507 03:08:17.644 | INFO | MemInfo: { | 20260507 03:08:17.644 | INFO | "gpu" : " 17 GB", | 20260507 03:08:17.644 | INFO | "huge" : "", | 20260507 03:08:17.644 | INFO | "huge_shared" : " 27 GB", | 20260507 03:08:17.644 | INFO | "malloc" : " 14 GB", | 20260507 03:08:17.644 | INFO | "pinned" : " 29 GB" | 20260507 03:08:17.644 | INFO | } | 20260507 03:08:17.644 | INFO | Perf report for the last 5.1441 seconds: | 20260507 03:08:17.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:08:17.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:17.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:17.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:17.644 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 03:08:17.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:17.644 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:08:17.644 | INFO | PRE_HQ : 59.2%, 1.8, 55.6ms, 0 | 20260507 03:08:17.644 | INFO | BAZWRITER : 61.9%, 3.3, 105.8ms, 0 | 20260507 03:08:17.644 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:08:17.644 | INFO | Uploaded 36GiB to GPU in 2.02616s (17.7676GiB/s) | 20260507 03:08:17.644 | INFO | Downloaded 38GiB to GPU in 2.17384s (17.4806GiB/s) | 20260507 03:08:22.792 | INFO | Analyzing chunk frames = [6615552,6616064) | 20260507 03:08:22.792 | INFO | MemInfo: { | 20260507 03:08:22.792 | INFO | "gpu" : " 17 GB", | 20260507 03:08:22.792 | INFO | "huge" : "", | 20260507 03:08:22.792 | INFO | "huge_shared" : " 27 GB", | 20260507 03:08:22.792 | INFO | "malloc" : " 14 GB", | 20260507 03:08:22.792 | INFO | "pinned" : " 29 GB" | 20260507 03:08:22.792 | INFO | } | 20260507 03:08:22.792 | INFO | Perf report for the last 5.14789 seconds: | 20260507 03:08:22.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:08:22.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:22.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:22.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:22.792 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 03:08:22.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:22.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:22.792 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:08:22.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:22.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:22.792 | INFO | Uploaded 36GiB to GPU in 2.02027s (17.8194GiB/s) | 20260507 03:08:22.792 | INFO | Downloaded 34GiB to GPU in 1.93897s (17.5351GiB/s) | 20260507 03:08:27.932 | INFO | Analyzing chunk frames = [6616064,6616576) | 20260507 03:08:27.932 | INFO | MemInfo: { | 20260507 03:08:27.932 | INFO | "gpu" : " 17 GB", | 20260507 03:08:27.932 | INFO | "huge" : "", | 20260507 03:08:27.932 | INFO | "huge_shared" : " 27 GB", | 20260507 03:08:27.932 | INFO | "malloc" : " 14 GB", | 20260507 03:08:27.932 | INFO | "pinned" : " 29 GB" | 20260507 03:08:27.932 | INFO | } | 20260507 03:08:27.932 | INFO | Perf report for the last 5.13995 seconds: | 20260507 03:08:27.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:08:27.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:27.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:27.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:27.932 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 03:08:27.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:27.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:27.932 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:08:27.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:27.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:27.932 | INFO | Uploaded 36GiB to GPU in 2.03383s (17.7006GiB/s) | 20260507 03:08:27.932 | INFO | Downloaded 34GiB to GPU in 1.95855s (17.3598GiB/s) | 20260507 03:08:33.063 | INFO | Analyzing chunk frames = [6616576,6617088) | 20260507 03:08:33.063 | INFO | MemInfo: { | 20260507 03:08:33.063 | INFO | "gpu" : " 17 GB", | 20260507 03:08:33.063 | INFO | "huge" : "", | 20260507 03:08:33.063 | INFO | "huge_shared" : " 27 GB", | 20260507 03:08:33.063 | INFO | "malloc" : " 14 GB", | 20260507 03:08:33.063 | INFO | "pinned" : " 29 GB" | 20260507 03:08:33.063 | INFO | } | 20260507 03:08:33.063 | INFO | Perf report for the last 5.13087 seconds: | 20260507 03:08:33.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:08:33.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:33.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:33.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:33.063 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 03:08:33.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:33.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:33.063 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:08:33.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:33.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:33.063 | INFO | Uploaded 36GiB to GPU in 2.01146s (17.8975GiB/s) | 20260507 03:08:33.063 | INFO | Downloaded 34GiB to GPU in 1.94568s (17.4746GiB/s) | 20260507 03:08:38.201 | INFO | Analyzing chunk frames = [6617088,6617600) | 20260507 03:08:38.201 | INFO | MemInfo: { | 20260507 03:08:38.201 | INFO | "gpu" : " 17 GB", | 20260507 03:08:38.201 | INFO | "huge" : "", | 20260507 03:08:38.201 | INFO | "huge_shared" : " 27 GB", | 20260507 03:08:38.201 | INFO | "malloc" : " 14 GB", | 20260507 03:08:38.201 | INFO | "pinned" : " 29 GB" | 20260507 03:08:38.201 | INFO | } | 20260507 03:08:38.201 | INFO | Perf report for the last 5.13851 seconds: | 20260507 03:08:38.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:08:38.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:38.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:38.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:38.201 | INFO | ANALYSIS : 59.9%, 3.0, 93.2ms, 0 | 20260507 03:08:38.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:38.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:38.201 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:08:38.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:38.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:38.201 | INFO | Uploaded 36GiB to GPU in 2.00246s (17.9779GiB/s) | 20260507 03:08:38.201 | INFO | Downloaded 34GiB to GPU in 1.95521s (17.3894GiB/s) | 20260507 03:08:43.339 | INFO | Analyzing chunk frames = [6617600,6618112) | 20260507 03:08:43.339 | INFO | MemInfo: { | 20260507 03:08:43.339 | INFO | "gpu" : " 17 GB", | 20260507 03:08:43.339 | INFO | "huge" : "", | 20260507 03:08:43.339 | INFO | "huge_shared" : " 27 GB", | 20260507 03:08:43.339 | INFO | "malloc" : " 14 GB", | 20260507 03:08:43.339 | INFO | "pinned" : " 29 GB" | 20260507 03:08:43.339 | INFO | } | 20260507 03:08:43.339 | INFO | Perf report for the last 5.13816 seconds: | 20260507 03:08:43.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:08:43.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:43.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:43.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:43.339 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:08:43.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:43.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:43.339 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 03:08:43.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:43.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:43.340 | INFO | Uploaded 36GiB to GPU in 2.04795s (17.5785GiB/s) | 20260507 03:08:43.340 | INFO | Downloaded 34GiB to GPU in 1.96318s (17.3188GiB/s) | 20260507 03:08:48.479 | INFO | Analyzing chunk frames = [6618112,6618624) | 20260507 03:08:48.479 | INFO | MemInfo: { | 20260507 03:08:48.479 | INFO | "gpu" : " 17 GB", | 20260507 03:08:48.479 | INFO | "huge" : "", | 20260507 03:08:48.479 | INFO | "huge_shared" : " 27 GB", | 20260507 03:08:48.479 | INFO | "malloc" : " 14 GB", | 20260507 03:08:48.479 | INFO | "pinned" : " 29 GB" | 20260507 03:08:48.479 | INFO | } | 20260507 03:08:48.479 | INFO | Perf report for the last 5.13941 seconds: | 20260507 03:08:48.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:08:48.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:48.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:48.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:48.479 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:08:48.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:48.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:48.479 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 03:08:48.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:48.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:48.479 | INFO | Uploaded 36GiB to GPU in 1.98527s (18.1336GiB/s) | 20260507 03:08:48.479 | INFO | Downloaded 34GiB to GPU in 1.98469s (17.1312GiB/s) | 20260507 03:08:53.624 | INFO | Analyzing chunk frames = [6618624,6619136) | 20260507 03:08:53.624 | INFO | MemInfo: { | 20260507 03:08:53.624 | INFO | "gpu" : " 17 GB", | 20260507 03:08:53.624 | INFO | "huge" : "", | 20260507 03:08:53.624 | INFO | "huge_shared" : " 27 GB", | 20260507 03:08:53.624 | INFO | "malloc" : " 14 GB", | 20260507 03:08:53.624 | INFO | "pinned" : " 29 GB" | 20260507 03:08:53.624 | INFO | } | 20260507 03:08:53.624 | INFO | Perf report for the last 5.1449 seconds: | 20260507 03:08:53.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:08:53.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:53.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:53.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:53.624 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 03:08:53.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:53.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:53.624 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 03:08:53.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:53.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:53.624 | INFO | Uploaded 36GiB to GPU in 2.00889s (17.9204GiB/s) | 20260507 03:08:53.624 | INFO | Downloaded 34GiB to GPU in 1.96784s (17.2779GiB/s) | 20260507 03:08:53.624 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:08:53.624 | INFO | Total mem usage: 18 GiB | 20260507 03:08:53.624 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:08:53.624 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:08:53.624 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:08:53.624 | INFO | High Water Breakdown: | 20260507 03:08:53.624 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:08:53.624 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:08:53.624 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:08:53.624 | INFO | ------------------------------------------ | 20260507 03:08:53.624 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:08:53.624 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:08:53.624 | INFO | Total mem usage: 26.5 GiB | 20260507 03:08:53.624 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:08:53.624 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:08:53.624 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:08:53.624 | INFO | High Water Breakdown: | 20260507 03:08:53.624 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:08:53.624 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:08:53.624 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:08:53.625 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:08:53.625 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:08:53.625 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:08:53.625 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:08:53.625 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:08:53.625 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:08:53.625 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:08:53.625 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:08:53.625 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:08:53.625 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:08:53.625 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:08:53.625 | INFO | ------------------------------------------ | 20260507 03:08:53.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:08:53.625 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:08:53.625 | INFO | Total mem usage: 25.6 GiB | 20260507 03:08:53.625 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:08:53.625 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 03:08:53.625 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 03:08:53.625 | INFO | High Water Breakdown: | 20260507 03:08:53.625 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 03:08:53.625 | INFO | ------------------------------------------ | 20260507 03:08:53.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:08:53.625 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:08:53.625 | INFO | Total mem usage: 15.7 GiB | 20260507 03:08:53.625 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:08:53.625 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:08:53.625 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 03:08:53.625 | INFO | High Water Breakdown: | 20260507 03:08:53.625 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:08:53.625 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:08:53.625 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:08:53.625 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:08:53.625 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:08:53.625 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:08:53.625 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:08:53.625 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:08:53.625 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:08:53.625 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:08:53.625 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:08:53.625 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:08:53.625 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:08:53.625 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:08:53.625 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:08:53.625 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:08:53.625 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:08:53.625 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:08:53.625 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:08:53.625 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:08:53.625 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:08:53.625 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:08:53.625 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:08:53.625 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:08:53.625 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:08:53.625 | INFO | ------------------------------------------ | 20260507 03:08:53.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:08:53.625 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:08:53.625 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:08:53.625 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:08:53.625 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.55e+01, 1.34e+01 / 1.80e+00 | 20260507 03:08:53.625 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.00e-01, 9.99e-03 / 3.95e-03 | 20260507 03:08:53.625 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.48e+00, 5.13e-01 / 2.15e-01 | 20260507 03:08:53.625 | INFO | PerfReport: 3.22%, Histogram -- 9.92e-01 / 1.48e+00, 1.68e-01 / 8.78e-02 | 20260507 03:08:53.625 | INFO | PerfReport: 16.15%, DME -- 3.98e+01 / 4.49e+01, 4.04e+00 / 2.13e+00 | 20260507 03:08:53.625 | INFO | PerfReport: 57.58%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.27e+00 / 1.32e+00 | 20260507 03:08:53.625 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.15e+00 / 2.73e+00, 3.85e-01 / 1.69e-01 | 20260507 03:08:53.625 | INFO | PerfReport: 3.05%, Metrics -- 9.40e-01 / 1.29e+00, 2.48e-01 / 1.36e-01 | 20260507 03:08:53.625 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.18e-01 / 3.82e-01, 5.03e-02 / 2.86e-02 | 20260507 03:08:53.625 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:08:53.625 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:08:53.625 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:08:53.625 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 03:08:53.625 | INFO | PerfReport: 21.77%, Upload -- 2.10e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260507 03:08:53.625 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 03:08:53.625 | INFO | PerfReport: 31.92%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 03:08:53.625 | INFO | PerfReport: 1.45%, LockUpload -- 1.40e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 03:08:53.625 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260507 03:08:53.625 | INFO | PerfReport: 22.14%, LockCompute -- 2.13e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 03:08:53.625 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260507 03:08:53.721 | INFO | Starting a round of Baz IO | 20260507 03:08:56.827 | INFO | Populating RTMetrics report | 20260507 03:08:56.827 | INFO | RTMetrics report populated | 20260507 03:08:56.827 | INFO | Opening tmp RTMetrics json file | 20260507 03:08:56.828 | INFO | Serializing RTMetrics to json | 20260507 03:08:56.837 | INFO | Serialization done, swapping tmp file now... | 20260507 03:08:56.848 | INFO | RTMetrics done | 20260507 03:08:56.920 | INFO | Finished a round of Baz IO: Wrote 1.761 GiB in 3.199 seconds | 20260507 03:08:56.920 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.92% / 0.00% / 67.08% | 20260507 03:08:56.920 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.21071% / 3.97364e-06% / 7.54992e-05% | 20260507 03:08:56.920 | INFO | Baz IO overall: | 20260507 03:08:56.920 | INFO | BytesWritten : 3403223806020 | 20260507 03:08:56.920 | INFO | NumEvents : 1635696790103 | 20260507 03:08:58.764 | INFO | Analyzing chunk frames = [6619136,6619648) | 20260507 03:08:58.764 | INFO | MemInfo: { | 20260507 03:08:58.764 | INFO | "gpu" : " 17 GB", | 20260507 03:08:58.764 | INFO | "huge" : "", | 20260507 03:08:58.764 | INFO | "huge_shared" : " 27 GB", | 20260507 03:08:58.764 | INFO | "malloc" : " 14 GB", | 20260507 03:08:58.764 | INFO | "pinned" : " 29 GB" | 20260507 03:08:58.764 | INFO | } | 20260507 03:08:58.764 | INFO | Perf report for the last 5.14032 seconds: | 20260507 03:08:58.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:08:58.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:58.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:58.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:58.764 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 03:08:58.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:08:58.764 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 03:08:58.764 | INFO | PRE_HQ : 59.3%, 1.8, 55.7ms, 0 | 20260507 03:08:58.764 | INFO | BAZWRITER : 61.9%, 3.3, 106.3ms, 0 | 20260507 03:08:58.764 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:08:58.764 | INFO | Uploaded 36GiB to GPU in 2.04446s (17.6085GiB/s) | 20260507 03:08:58.764 | INFO | Downloaded 38GiB to GPU in 2.1771s (17.4544GiB/s) | 20260507 03:09:03.902 | INFO | Analyzing chunk frames = [6619648,6620160) | 20260507 03:09:03.902 | INFO | MemInfo: { | 20260507 03:09:03.902 | INFO | "gpu" : " 17 GB", | 20260507 03:09:03.902 | INFO | "huge" : "", | 20260507 03:09:03.902 | INFO | "huge_shared" : " 27 GB", | 20260507 03:09:03.902 | INFO | "malloc" : " 14 GB", | 20260507 03:09:03.902 | INFO | "pinned" : " 29 GB" | 20260507 03:09:03.902 | INFO | } | 20260507 03:09:03.902 | INFO | Perf report for the last 5.13822 seconds: | 20260507 03:09:03.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:09:03.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:03.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:03.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:03.902 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:09:03.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:03.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:03.902 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:09:03.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:03.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:03.902 | INFO | Uploaded 36GiB to GPU in 1.99828s (18.0155GiB/s) | 20260507 03:09:03.903 | INFO | Downloaded 34GiB to GPU in 1.92037s (17.7049GiB/s) | 20260507 03:09:09.040 | INFO | Analyzing chunk frames = [6620160,6620672) | 20260507 03:09:09.040 | INFO | MemInfo: { | 20260507 03:09:09.040 | INFO | "gpu" : " 17 GB", | 20260507 03:09:09.040 | INFO | "huge" : "", | 20260507 03:09:09.040 | INFO | "huge_shared" : " 27 GB", | 20260507 03:09:09.040 | INFO | "malloc" : " 14 GB", | 20260507 03:09:09.040 | INFO | "pinned" : " 29 GB" | 20260507 03:09:09.040 | INFO | } | 20260507 03:09:09.040 | INFO | Perf report for the last 5.13806 seconds: | 20260507 03:09:09.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:09:09.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:09.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:09.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:09.040 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 03:09:09.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:09.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:09.040 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:09:09.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:09.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:09.041 | INFO | Uploaded 36GiB to GPU in 2.0168s (17.85GiB/s) | 20260507 03:09:09.041 | INFO | Downloaded 34GiB to GPU in 1.93965s (17.529GiB/s) | 20260507 03:09:14.176 | INFO | Analyzing chunk frames = [6620672,6621184) | 20260507 03:09:14.176 | INFO | MemInfo: { | 20260507 03:09:14.176 | INFO | "gpu" : " 17 GB", | 20260507 03:09:14.176 | INFO | "huge" : "", | 20260507 03:09:14.176 | INFO | "huge_shared" : " 27 GB", | 20260507 03:09:14.176 | INFO | "malloc" : " 14 GB", | 20260507 03:09:14.176 | INFO | "pinned" : " 29 GB" | 20260507 03:09:14.176 | INFO | } | 20260507 03:09:14.176 | INFO | Perf report for the last 5.13572 seconds: | 20260507 03:09:14.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:09:14.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:14.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:14.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:14.176 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 03:09:14.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:14.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:14.176 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:09:14.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:14.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:14.176 | INFO | Uploaded 36GiB to GPU in 1.99791s (18.0188GiB/s) | 20260507 03:09:14.176 | INFO | Downloaded 34GiB to GPU in 1.9181s (17.7259GiB/s) | 20260507 03:09:19.315 | INFO | Analyzing chunk frames = [6621184,6621696) | 20260507 03:09:19.315 | INFO | MemInfo: { | 20260507 03:09:19.315 | INFO | "gpu" : " 17 GB", | 20260507 03:09:19.315 | INFO | "huge" : "", | 20260507 03:09:19.315 | INFO | "huge_shared" : " 27 GB", | 20260507 03:09:19.315 | INFO | "malloc" : " 14 GB", | 20260507 03:09:19.315 | INFO | "pinned" : " 29 GB" | 20260507 03:09:19.315 | INFO | } | 20260507 03:09:19.315 | INFO | Perf report for the last 5.13883 seconds: | 20260507 03:09:19.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:09:19.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:19.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:19.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:19.315 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 03:09:19.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:19.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:19.315 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:09:19.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:19.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:19.315 | INFO | Uploaded 36GiB to GPU in 2.02932s (17.74GiB/s) | 20260507 03:09:19.315 | INFO | Downloaded 34GiB to GPU in 1.93557s (17.5658GiB/s) | 20260507 03:09:24.455 | INFO | Analyzing chunk frames = [6621696,6622208) | 20260507 03:09:24.455 | INFO | MemInfo: { | 20260507 03:09:24.455 | INFO | "gpu" : " 17 GB", | 20260507 03:09:24.455 | INFO | "huge" : "", | 20260507 03:09:24.455 | INFO | "huge_shared" : " 27 GB", | 20260507 03:09:24.455 | INFO | "malloc" : " 14 GB", | 20260507 03:09:24.455 | INFO | "pinned" : " 29 GB" | 20260507 03:09:24.455 | INFO | } | 20260507 03:09:24.455 | INFO | Perf report for the last 5.14041 seconds: | 20260507 03:09:24.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:09:24.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:24.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:24.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:24.455 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 03:09:24.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:24.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:24.455 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 03:09:24.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:24.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:24.456 | INFO | Uploaded 36GiB to GPU in 2.04434s (17.6096GiB/s) | 20260507 03:09:24.456 | INFO | Downloaded 34GiB to GPU in 1.9341s (17.5793GiB/s) | 20260507 03:09:29.597 | INFO | Analyzing chunk frames = [6622208,6622720) | 20260507 03:09:29.597 | INFO | MemInfo: { | 20260507 03:09:29.597 | INFO | "gpu" : " 17 GB", | 20260507 03:09:29.597 | INFO | "huge" : "", | 20260507 03:09:29.597 | INFO | "huge_shared" : " 27 GB", | 20260507 03:09:29.597 | INFO | "malloc" : " 14 GB", | 20260507 03:09:29.597 | INFO | "pinned" : " 29 GB" | 20260507 03:09:29.597 | INFO | } | 20260507 03:09:29.597 | INFO | Perf report for the last 5.14136 seconds: | 20260507 03:09:29.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:09:29.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:29.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:29.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:29.597 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:09:29.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:29.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:29.597 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:09:29.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:29.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:29.597 | INFO | Uploaded 36GiB to GPU in 2.00412s (17.963GiB/s) | 20260507 03:09:29.597 | INFO | Downloaded 34GiB to GPU in 1.98478s (17.1303GiB/s) | 20260507 03:09:34.738 | INFO | Analyzing chunk frames = [6622720,6623232) | 20260507 03:09:34.738 | INFO | MemInfo: { | 20260507 03:09:34.738 | INFO | "gpu" : " 17 GB", | 20260507 03:09:34.738 | INFO | "huge" : "", | 20260507 03:09:34.738 | INFO | "huge_shared" : " 27 GB", | 20260507 03:09:34.738 | INFO | "malloc" : " 14 GB", | 20260507 03:09:34.738 | INFO | "pinned" : " 29 GB" | 20260507 03:09:34.738 | INFO | } | 20260507 03:09:34.738 | INFO | Perf report for the last 5.14174 seconds: | 20260507 03:09:34.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:09:34.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:34.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:34.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:34.739 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 03:09:34.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:34.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:34.739 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 03:09:34.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:34.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:34.739 | INFO | Uploaded 36GiB to GPU in 2.00649s (17.9418GiB/s) | 20260507 03:09:34.739 | INFO | Downloaded 34GiB to GPU in 1.97075s (17.2523GiB/s) | 20260507 03:09:34.835 | INFO | Starting a round of Baz IO | 20260507 03:09:37.939 | INFO | Populating RTMetrics report | 20260507 03:09:37.939 | INFO | RTMetrics report populated | 20260507 03:09:37.939 | INFO | Opening tmp RTMetrics json file | 20260507 03:09:37.940 | INFO | Serializing RTMetrics to json | 20260507 03:09:37.951 | INFO | Serialization done, swapping tmp file now... | 20260507 03:09:37.960 | INFO | RTMetrics done | 20260507 03:09:38.030 | INFO | Finished a round of Baz IO: Wrote 1.76 GiB in 3.195 seconds | 20260507 03:09:38.031 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.92% / 0.00% / 67.08% | 20260507 03:09:38.031 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.21011% / 3.97364e-06% / 9.13938e-05% | 20260507 03:09:38.031 | INFO | Baz IO overall: | 20260507 03:09:38.031 | INFO | BytesWritten : 3405113310435 | 20260507 03:09:38.031 | INFO | NumEvents : 1636588217715 | 20260507 03:09:39.876 | INFO | Analyzing chunk frames = [6623232,6623744) | 20260507 03:09:39.876 | INFO | MemInfo: { | 20260507 03:09:39.876 | INFO | "gpu" : " 17 GB", | 20260507 03:09:39.876 | INFO | "huge" : "", | 20260507 03:09:39.876 | INFO | "huge_shared" : " 27 GB", | 20260507 03:09:39.876 | INFO | "malloc" : " 14 GB", | 20260507 03:09:39.876 | INFO | "pinned" : " 29 GB" | 20260507 03:09:39.876 | INFO | } | 20260507 03:09:39.876 | INFO | Perf report for the last 5.13782 seconds: | 20260507 03:09:39.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:09:39.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:39.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:39.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:39.876 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 03:09:39.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:39.876 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 03:09:39.876 | INFO | PRE_HQ : 59.3%, 1.8, 55.4ms, 0 | 20260507 03:09:39.876 | INFO | BAZWRITER : 61.8%, 3.3, 105.2ms, 0 | 20260507 03:09:39.876 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:09:39.876 | INFO | Uploaded 36GiB to GPU in 2.04465s (17.6069GiB/s) | 20260507 03:09:39.877 | INFO | Downloaded 38GiB to GPU in 2.17826s (17.4452GiB/s) | 20260507 03:09:45.016 | INFO | Analyzing chunk frames = [6623744,6624256) | 20260507 03:09:45.016 | INFO | MemInfo: { | 20260507 03:09:45.016 | INFO | "gpu" : " 17 GB", | 20260507 03:09:45.016 | INFO | "huge" : "", | 20260507 03:09:45.016 | INFO | "huge_shared" : " 27 GB", | 20260507 03:09:45.016 | INFO | "malloc" : " 14 GB", | 20260507 03:09:45.016 | INFO | "pinned" : " 29 GB" | 20260507 03:09:45.016 | INFO | } | 20260507 03:09:45.016 | INFO | Perf report for the last 5.13985 seconds: | 20260507 03:09:45.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:09:45.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:45.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:45.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:45.016 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:09:45.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:45.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:45.016 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:09:45.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:45.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:45.017 | INFO | Uploaded 36GiB to GPU in 2.00704s (17.9369GiB/s) | 20260507 03:09:45.017 | INFO | Downloaded 34GiB to GPU in 1.93125s (17.6052GiB/s) | 20260507 03:09:50.147 | INFO | Analyzing chunk frames = [6624256,6624768) | 20260507 03:09:50.147 | INFO | MemInfo: { | 20260507 03:09:50.147 | INFO | "gpu" : " 17 GB", | 20260507 03:09:50.147 | INFO | "huge" : "", | 20260507 03:09:50.147 | INFO | "huge_shared" : " 27 GB", | 20260507 03:09:50.147 | INFO | "malloc" : " 14 GB", | 20260507 03:09:50.147 | INFO | "pinned" : " 29 GB" | 20260507 03:09:50.148 | INFO | } | 20260507 03:09:50.148 | INFO | Perf report for the last 5.13138 seconds: | 20260507 03:09:50.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:09:50.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:50.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:50.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:50.148 | INFO | ANALYSIS : 60.4%, 3.0, 93.3ms, 0 | 20260507 03:09:50.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:50.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:50.148 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 03:09:50.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:50.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:50.148 | INFO | Uploaded 36GiB to GPU in 2.02085s (17.8143GiB/s) | 20260507 03:09:50.148 | INFO | Downloaded 34GiB to GPU in 1.93809s (17.5431GiB/s) | 20260507 03:09:55.292 | INFO | Analyzing chunk frames = [6624768,6625280) | 20260507 03:09:55.292 | INFO | MemInfo: { | 20260507 03:09:55.292 | INFO | "gpu" : " 17 GB", | 20260507 03:09:55.292 | INFO | "huge" : "", | 20260507 03:09:55.292 | INFO | "huge_shared" : " 27 GB", | 20260507 03:09:55.292 | INFO | "malloc" : " 14 GB", | 20260507 03:09:55.292 | INFO | "pinned" : " 29 GB" | 20260507 03:09:55.292 | INFO | } | 20260507 03:09:55.292 | INFO | Perf report for the last 5.14466 seconds: | 20260507 03:09:55.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:09:55.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:55.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:55.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:55.292 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260507 03:09:55.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:55.292 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:55.292 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:09:55.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:55.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:09:55.292 | INFO | Uploaded 36GiB to GPU in 2.01853s (17.8348GiB/s) | 20260507 03:09:55.292 | INFO | Downloaded 34GiB to GPU in 1.90994s (17.8016GiB/s) | 20260507 03:10:00.437 | INFO | Analyzing chunk frames = [6625280,6625792) | 20260507 03:10:00.437 | INFO | MemInfo: { | 20260507 03:10:00.437 | INFO | "gpu" : " 17 GB", | 20260507 03:10:00.437 | INFO | "huge" : "", | 20260507 03:10:00.437 | INFO | "huge_shared" : " 27 GB", | 20260507 03:10:00.437 | INFO | "malloc" : " 14 GB", | 20260507 03:10:00.437 | INFO | "pinned" : " 29 GB" | 20260507 03:10:00.437 | INFO | } | 20260507 03:10:00.437 | INFO | Perf report for the last 5.14469 seconds: | 20260507 03:10:00.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:10:00.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:00.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:00.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:00.437 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 03:10:00.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:00.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:00.437 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:10:00.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:00.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:00.437 | INFO | Uploaded 36GiB to GPU in 2.00625s (17.9439GiB/s) | 20260507 03:10:00.437 | INFO | Downloaded 34GiB to GPU in 1.94978s (17.4379GiB/s) | 20260507 03:10:05.573 | INFO | Analyzing chunk frames = [6625792,6626304) | 20260507 03:10:05.573 | INFO | MemInfo: { | 20260507 03:10:05.573 | INFO | "gpu" : " 17 GB", | 20260507 03:10:05.573 | INFO | "huge" : "", | 20260507 03:10:05.573 | INFO | "huge_shared" : " 27 GB", | 20260507 03:10:05.573 | INFO | "malloc" : " 14 GB", | 20260507 03:10:05.573 | INFO | "pinned" : " 29 GB" | 20260507 03:10:05.573 | INFO | } | 20260507 03:10:05.573 | INFO | Perf report for the last 5.13637 seconds: | 20260507 03:10:05.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:10:05.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:05.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:05.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:05.573 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 03:10:05.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:05.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:05.573 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 03:10:05.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:05.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:05.574 | INFO | Uploaded 36GiB to GPU in 2.04955s (17.5648GiB/s) | 20260507 03:10:05.574 | INFO | Downloaded 34GiB to GPU in 1.92255s (17.6848GiB/s) | 20260507 03:10:10.710 | INFO | Analyzing chunk frames = [6626304,6626816) | 20260507 03:10:10.710 | INFO | MemInfo: { | 20260507 03:10:10.710 | INFO | "gpu" : " 17 GB", | 20260507 03:10:10.710 | INFO | "huge" : "", | 20260507 03:10:10.710 | INFO | "huge_shared" : " 27 GB", | 20260507 03:10:10.710 | INFO | "malloc" : " 14 GB", | 20260507 03:10:10.710 | INFO | "pinned" : " 29 GB" | 20260507 03:10:10.710 | INFO | } | 20260507 03:10:10.710 | INFO | Perf report for the last 5.13701 seconds: | 20260507 03:10:10.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:10:10.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:10.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:10.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:10.710 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:10:10.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:10.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:10.710 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 03:10:10.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:10.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:10.711 | INFO | Uploaded 36GiB to GPU in 1.96126s (18.3556GiB/s) | 20260507 03:10:10.711 | INFO | Downloaded 34GiB to GPU in 1.97769s (17.1918GiB/s) | 20260507 03:10:15.851 | INFO | Analyzing chunk frames = [6626816,6627328) | 20260507 03:10:15.851 | INFO | MemInfo: { | 20260507 03:10:15.851 | INFO | "gpu" : " 17 GB", | 20260507 03:10:15.851 | INFO | "huge" : "", | 20260507 03:10:15.851 | INFO | "huge_shared" : " 27 GB", | 20260507 03:10:15.851 | INFO | "malloc" : " 14 GB", | 20260507 03:10:15.851 | INFO | "pinned" : " 29 GB" | 20260507 03:10:15.851 | INFO | } | 20260507 03:10:15.851 | INFO | Perf report for the last 5.14083 seconds: | 20260507 03:10:15.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:10:15.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:15.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:15.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:15.851 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:10:15.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:15.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:15.851 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 03:10:15.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:15.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:15.851 | INFO | Uploaded 36GiB to GPU in 1.97637s (18.2152GiB/s) | 20260507 03:10:15.851 | INFO | Downloaded 34GiB to GPU in 1.95359s (17.4039GiB/s) | 20260507 03:10:15.852 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:10:15.852 | INFO | Total mem usage: 18 GiB | 20260507 03:10:15.852 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:10:15.852 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:10:15.852 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:10:15.852 | INFO | High Water Breakdown: | 20260507 03:10:15.852 | INFO | 60.1% : 8.57 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:10:15.852 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:10:15.852 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:10:15.852 | INFO | ------------------------------------------ | 20260507 03:10:15.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:10:15.852 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:10:15.852 | INFO | Total mem usage: 26.5 GiB | 20260507 03:10:15.852 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:10:15.852 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:10:15.852 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:10:15.852 | INFO | High Water Breakdown: | 20260507 03:10:15.852 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:10:15.852 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:10:15.852 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:10:15.852 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:10:15.852 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:10:15.852 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:10:15.852 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:10:15.852 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:10:15.852 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:10:15.852 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:10:15.852 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:10:15.852 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:10:15.852 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:10:15.852 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:10:15.852 | INFO | ------------------------------------------ | 20260507 03:10:15.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:10:15.852 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:10:15.852 | INFO | Total mem usage: 25.6 GiB | 20260507 03:10:15.852 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:10:15.852 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 03:10:15.852 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 03:10:15.852 | INFO | High Water Breakdown: | 20260507 03:10:15.852 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 03:10:15.852 | INFO | ------------------------------------------ | 20260507 03:10:15.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:10:15.852 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:10:15.852 | INFO | Total mem usage: 15.7 GiB | 20260507 03:10:15.852 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:10:15.852 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:10:15.852 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:10:15.852 | INFO | High Water Breakdown: | 20260507 03:10:15.852 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:10:15.852 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:10:15.852 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:10:15.852 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:10:15.852 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:10:15.852 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:10:15.852 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:10:15.852 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:10:15.852 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:10:15.852 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:10:15.852 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:10:15.852 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:10:15.852 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:10:15.852 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:10:15.852 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:10:15.852 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:10:15.852 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:10:15.852 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:10:15.852 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:10:15.852 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:10:15.852 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:10:15.852 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:10:15.852 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:10:15.852 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:10:15.852 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:10:15.852 | INFO | ------------------------------------------ | 20260507 03:10:15.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:10:15.852 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:10:15.852 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:10:15.852 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:10:15.852 | INFO | PerfReport: Full Execution -- 3.09e+01 / 7.51e+01, 1.34e+01 / 1.62e+00 | 20260507 03:10:15.852 | INFO | PerfReport: 0.29%, AggregateStats -- 9.09e-02 / 1.00e-01, 9.79e-03 / 5.80e-03 | 20260507 03:10:15.852 | INFO | PerfReport: 11.65%, Baseline -- 3.60e+00 / 4.35e+00, 5.21e-01 / 3.88e-01 | 20260507 03:10:15.852 | INFO | PerfReport: 3.26%, Histogram -- 1.01e+00 / 1.45e+00, 1.77e-01 / 1.35e-01 | 20260507 03:10:15.852 | INFO | PerfReport: 16.04%, DME -- 3.96e+01 / 4.39e+01, 3.45e+00 / 1.42e+00 | 20260507 03:10:15.852 | INFO | PerfReport: 57.78%, FrameLabeling -- 1.78e+01 / 2.10e+01, 1.47e+00 / 1.51e+00 | 20260507 03:10:15.852 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.13e+00 / 2.64e+00, 3.63e-01 / 2.41e-01 | 20260507 03:10:15.852 | INFO | PerfReport: 3.04%, Metrics -- 9.38e-01 / 1.28e+00, 2.49e-01 / 1.39e-01 | 20260507 03:10:15.852 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.19e-01 / 3.72e-01, 5.12e-02 / 2.67e-02 | 20260507 03:10:15.852 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:10:15.852 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:10:15.852 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:10:15.852 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 03:10:15.852 | INFO | PerfReport: 21.79%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260507 03:10:15.852 | INFO | PerfReport: 21.32%, Download -- 2.05e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 03:10:15.852 | INFO | PerfReport: 32.05%, Compute -- 3.09e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 03:10:15.853 | INFO | PerfReport: 1.46%, LockUpload -- 1.41e+00 / 0.00e+00, 4.77e+00 / 0.00e+00 | 20260507 03:10:15.853 | INFO | PerfReport: 1.17%, LockDownload -- 1.13e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260507 03:10:15.853 | INFO | PerfReport: 22.19%, LockCompute -- 2.14e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 03:10:15.853 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260507 03:10:15.963 | INFO | Starting a round of Baz IO | 20260507 03:10:19.057 | INFO | Populating RTMetrics report | 20260507 03:10:19.058 | INFO | RTMetrics report populated | 20260507 03:10:19.058 | INFO | Opening tmp RTMetrics json file | 20260507 03:10:19.059 | INFO | Serializing RTMetrics to json | 20260507 03:10:19.071 | INFO | Serialization done, swapping tmp file now... | 20260507 03:10:19.078 | INFO | RTMetrics done | 20260507 03:10:19.148 | INFO | Finished a round of Baz IO: Wrote 1.76 GiB in 3.185 seconds | 20260507 03:10:19.148 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.92% / 0.00% / 67.08% | 20260507 03:10:19.148 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.210353% / 3.97364e-06% / 7.94729e-05% | 20260507 03:10:19.148 | INFO | Baz IO overall: | 20260507 03:10:19.148 | INFO | BytesWritten : 3407003465068 | 20260507 03:10:19.148 | INFO | NumEvents : 1637480245389 | 20260507 03:10:20.988 | INFO | Analyzing chunk frames = [6627328,6627840) | 20260507 03:10:20.988 | INFO | MemInfo: { | 20260507 03:10:20.988 | INFO | "gpu" : " 17 GB", | 20260507 03:10:20.988 | INFO | "huge" : "", | 20260507 03:10:20.988 | INFO | "huge_shared" : " 27 GB", | 20260507 03:10:20.988 | INFO | "malloc" : " 14 GB", | 20260507 03:10:20.988 | INFO | "pinned" : " 29 GB" | 20260507 03:10:20.988 | INFO | } | 20260507 03:10:20.989 | INFO | Perf report for the last 5.13733 seconds: | 20260507 03:10:20.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:10:20.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:20.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:20.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:20.989 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 03:10:20.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:20.989 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 03:10:20.989 | INFO | PRE_HQ : 59.2%, 1.8, 55.6ms, 0 | 20260507 03:10:20.989 | INFO | BAZWRITER : 61.9%, 3.3, 106.0ms, 0 | 20260507 03:10:20.989 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:10:20.989 | INFO | Uploaded 36GiB to GPU in 2.04697s (17.587GiB/s) | 20260507 03:10:20.989 | INFO | Downloaded 38GiB to GPU in 2.18143s (17.4198GiB/s) | 20260507 03:10:26.126 | INFO | Analyzing chunk frames = [6627840,6628352) | 20260507 03:10:26.126 | INFO | MemInfo: { | 20260507 03:10:26.126 | INFO | "gpu" : " 17 GB", | 20260507 03:10:26.126 | INFO | "huge" : "", | 20260507 03:10:26.126 | INFO | "huge_shared" : " 27 GB", | 20260507 03:10:26.126 | INFO | "malloc" : " 14 GB", | 20260507 03:10:26.126 | INFO | "pinned" : " 29 GB" | 20260507 03:10:26.126 | INFO | } | 20260507 03:10:26.126 | INFO | Perf report for the last 5.13754 seconds: | 20260507 03:10:26.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:10:26.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:26.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:26.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:26.126 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:10:26.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:26.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:26.126 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:10:26.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:26.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:26.126 | INFO | Uploaded 36GiB to GPU in 2.0338s (17.7008GiB/s) | 20260507 03:10:26.126 | INFO | Downloaded 34GiB to GPU in 1.93946s (17.5307GiB/s) | 20260507 03:10:31.268 | INFO | Analyzing chunk frames = [6628352,6628864) | 20260507 03:10:31.268 | INFO | MemInfo: { | 20260507 03:10:31.268 | INFO | "gpu" : " 17 GB", | 20260507 03:10:31.268 | INFO | "huge" : "", | 20260507 03:10:31.268 | INFO | "huge_shared" : " 27 GB", | 20260507 03:10:31.269 | INFO | "malloc" : " 14 GB", | 20260507 03:10:31.269 | INFO | "pinned" : " 29 GB" | 20260507 03:10:31.269 | INFO | } | 20260507 03:10:31.269 | INFO | Perf report for the last 5.14257 seconds: | 20260507 03:10:31.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:10:31.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:31.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:31.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:31.269 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 03:10:31.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:31.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:31.269 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:10:31.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:31.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:31.269 | INFO | Uploaded 36GiB to GPU in 2.00794s (17.9288GiB/s) | 20260507 03:10:31.269 | INFO | Downloaded 34GiB to GPU in 1.93899s (17.5349GiB/s) | 20260507 03:10:36.408 | INFO | Analyzing chunk frames = [6628864,6629376) | 20260507 03:10:36.408 | INFO | MemInfo: { | 20260507 03:10:36.408 | INFO | "gpu" : " 17 GB", | 20260507 03:10:36.408 | INFO | "huge" : "", | 20260507 03:10:36.408 | INFO | "huge_shared" : " 27 GB", | 20260507 03:10:36.408 | INFO | "malloc" : " 14 GB", | 20260507 03:10:36.409 | INFO | "pinned" : " 29 GB" | 20260507 03:10:36.409 | INFO | } | 20260507 03:10:36.409 | INFO | Perf report for the last 5.13989 seconds: | 20260507 03:10:36.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:10:36.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:36.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:36.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:36.409 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 03:10:36.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:36.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:36.409 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:10:36.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:36.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:36.409 | INFO | Uploaded 36GiB to GPU in 1.98707s (18.1171GiB/s) | 20260507 03:10:36.409 | INFO | Downloaded 34GiB to GPU in 1.90822s (17.8176GiB/s) | 20260507 03:10:41.553 | INFO | Analyzing chunk frames = [6629376,6629888) | 20260507 03:10:41.553 | INFO | MemInfo: { | 20260507 03:10:41.553 | INFO | "gpu" : " 17 GB", | 20260507 03:10:41.553 | INFO | "huge" : "", | 20260507 03:10:41.553 | INFO | "huge_shared" : " 27 GB", | 20260507 03:10:41.553 | INFO | "malloc" : " 14 GB", | 20260507 03:10:41.553 | INFO | "pinned" : " 29 GB" | 20260507 03:10:41.553 | INFO | } | 20260507 03:10:41.553 | INFO | Perf report for the last 5.14487 seconds: | 20260507 03:10:41.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:10:41.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:41.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:41.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:41.553 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 03:10:41.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:41.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:41.554 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:10:41.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:41.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:41.554 | INFO | Uploaded 36GiB to GPU in 2.02231s (17.8014GiB/s) | 20260507 03:10:41.554 | INFO | Downloaded 34GiB to GPU in 1.94458s (17.4845GiB/s) | 20260507 03:10:46.687 | INFO | Analyzing chunk frames = [6629888,6630400) | 20260507 03:10:46.687 | INFO | MemInfo: { | 20260507 03:10:46.687 | INFO | "gpu" : " 17 GB", | 20260507 03:10:46.687 | INFO | "huge" : "", | 20260507 03:10:46.687 | INFO | "huge_shared" : " 27 GB", | 20260507 03:10:46.687 | INFO | "malloc" : " 14 GB", | 20260507 03:10:46.687 | INFO | "pinned" : " 29 GB" | 20260507 03:10:46.687 | INFO | } | 20260507 03:10:46.687 | INFO | Perf report for the last 5.13329 seconds: | 20260507 03:10:46.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:10:46.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:46.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:46.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:46.687 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 03:10:46.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:46.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:46.687 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:10:46.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:46.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:46.687 | INFO | Uploaded 36GiB to GPU in 2.03805s (17.664GiB/s) | 20260507 03:10:46.687 | INFO | Downloaded 34GiB to GPU in 1.9367s (17.5556GiB/s) | 20260507 03:10:51.822 | INFO | Analyzing chunk frames = [6630400,6630912) | 20260507 03:10:51.822 | INFO | MemInfo: { | 20260507 03:10:51.822 | INFO | "gpu" : " 17 GB", | 20260507 03:10:51.822 | INFO | "huge" : "", | 20260507 03:10:51.822 | INFO | "huge_shared" : " 27 GB", | 20260507 03:10:51.822 | INFO | "malloc" : " 14 GB", | 20260507 03:10:51.822 | INFO | "pinned" : " 29 GB" | 20260507 03:10:51.822 | INFO | } | 20260507 03:10:51.822 | INFO | Perf report for the last 5.1351 seconds: | 20260507 03:10:51.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:10:51.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:51.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:51.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:51.822 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:10:51.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:51.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:51.822 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:10:51.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:51.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:51.822 | INFO | Uploaded 36GiB to GPU in 1.97392s (18.2378GiB/s) | 20260507 03:10:51.822 | INFO | Downloaded 34GiB to GPU in 1.98509s (17.1277GiB/s) | 20260507 03:10:56.963 | INFO | Analyzing chunk frames = [6630912,6631424) | 20260507 03:10:56.963 | INFO | MemInfo: { | 20260507 03:10:56.963 | INFO | "gpu" : " 17 GB", | 20260507 03:10:56.963 | INFO | "huge" : "", | 20260507 03:10:56.963 | INFO | "huge_shared" : " 27 GB", | 20260507 03:10:56.963 | INFO | "malloc" : " 14 GB", | 20260507 03:10:56.963 | INFO | "pinned" : " 29 GB" | 20260507 03:10:56.963 | INFO | } | 20260507 03:10:56.963 | INFO | Perf report for the last 5.14121 seconds: | 20260507 03:10:56.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:10:56.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:56.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:56.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:56.963 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:10:56.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:56.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:56.963 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:10:56.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:56.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:10:56.963 | INFO | Uploaded 36GiB to GPU in 1.95168s (18.4456GiB/s) | 20260507 03:10:56.963 | INFO | Downloaded 34GiB to GPU in 1.96422s (17.3097GiB/s) | 20260507 03:10:57.060 | INFO | Starting a round of Baz IO | 20260507 03:11:00.171 | INFO | Populating RTMetrics report | 20260507 03:11:00.172 | INFO | RTMetrics report populated | 20260507 03:11:00.172 | INFO | Opening tmp RTMetrics json file | 20260507 03:11:00.175 | INFO | Serializing RTMetrics to json | 20260507 03:11:00.190 | INFO | Serialization done, swapping tmp file now... | 20260507 03:11:00.191 | INFO | RTMetrics done | 20260507 03:11:00.264 | INFO | Finished a round of Baz IO: Wrote 1.762 GiB in 3.204 seconds | 20260507 03:11:00.264 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.92% / 0.00% / 67.08% | 20260507 03:11:00.264 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.210599% / 3.97364e-06% / 8.74201e-05% | 20260507 03:11:00.264 | INFO | Baz IO overall: | 20260507 03:11:00.264 | INFO | BytesWritten : 3408894946601 | 20260507 03:11:00.264 | INFO | NumEvents : 1638373568178 | 20260507 03:11:02.105 | INFO | Analyzing chunk frames = [6631424,6631936) | 20260507 03:11:02.105 | INFO | MemInfo: { | 20260507 03:11:02.105 | INFO | "gpu" : " 17 GB", | 20260507 03:11:02.105 | INFO | "huge" : "", | 20260507 03:11:02.105 | INFO | "huge_shared" : " 27 GB", | 20260507 03:11:02.105 | INFO | "malloc" : " 14 GB", | 20260507 03:11:02.105 | INFO | "pinned" : " 29 GB" | 20260507 03:11:02.105 | INFO | } | 20260507 03:11:02.105 | INFO | Perf report for the last 5.14173 seconds: | 20260507 03:11:02.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:11:02.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:02.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:02.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:02.105 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 03:11:02.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:02.105 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 03:11:02.105 | INFO | PRE_HQ : 59.2%, 1.8, 55.4ms, 0 | 20260507 03:11:02.105 | INFO | BAZWRITER : 61.9%, 3.3, 106.3ms, 0 | 20260507 03:11:02.105 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:11:02.105 | INFO | Uploaded 36GiB to GPU in 2.0436s (17.616GiB/s) | 20260507 03:11:02.105 | INFO | Downloaded 38GiB to GPU in 2.18439s (17.3962GiB/s) | 20260507 03:11:07.245 | INFO | Analyzing chunk frames = [6631936,6632448) | 20260507 03:11:07.245 | INFO | MemInfo: { | 20260507 03:11:07.245 | INFO | "gpu" : " 17 GB", | 20260507 03:11:07.245 | INFO | "huge" : "", | 20260507 03:11:07.245 | INFO | "huge_shared" : " 27 GB", | 20260507 03:11:07.246 | INFO | "malloc" : " 14 GB", | 20260507 03:11:07.246 | INFO | "pinned" : " 29 GB" | 20260507 03:11:07.246 | INFO | } | 20260507 03:11:07.246 | INFO | Perf report for the last 5.14073 seconds: | 20260507 03:11:07.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:11:07.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:07.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:07.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:07.246 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:11:07.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:07.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:07.246 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:11:07.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:07.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:07.246 | INFO | Uploaded 36GiB to GPU in 2.03002s (17.7338GiB/s) | 20260507 03:11:07.246 | INFO | Downloaded 34GiB to GPU in 1.93856s (17.5388GiB/s) | 20260507 03:11:12.387 | INFO | Analyzing chunk frames = [6632448,6632960) | 20260507 03:11:12.387 | INFO | MemInfo: { | 20260507 03:11:12.387 | INFO | "gpu" : " 17 GB", | 20260507 03:11:12.387 | INFO | "huge" : "", | 20260507 03:11:12.387 | INFO | "huge_shared" : " 27 GB", | 20260507 03:11:12.387 | INFO | "malloc" : " 14 GB", | 20260507 03:11:12.387 | INFO | "pinned" : " 29 GB" | 20260507 03:11:12.387 | INFO | } | 20260507 03:11:12.387 | INFO | Perf report for the last 5.14179 seconds: | 20260507 03:11:12.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:11:12.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:12.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:12.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:12.387 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 03:11:12.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:12.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:12.387 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:11:12.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:12.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:12.388 | INFO | Uploaded 36GiB to GPU in 2.01193s (17.8933GiB/s) | 20260507 03:11:12.388 | INFO | Downloaded 34GiB to GPU in 1.93939s (17.5313GiB/s) | 20260507 03:11:17.520 | INFO | Analyzing chunk frames = [6632960,6633472) | 20260507 03:11:17.520 | INFO | MemInfo: { | 20260507 03:11:17.520 | INFO | "gpu" : " 17 GB", | 20260507 03:11:17.520 | INFO | "huge" : "", | 20260507 03:11:17.520 | INFO | "huge_shared" : " 27 GB", | 20260507 03:11:17.520 | INFO | "malloc" : " 14 GB", | 20260507 03:11:17.520 | INFO | "pinned" : " 29 GB" | 20260507 03:11:17.520 | INFO | } | 20260507 03:11:17.520 | INFO | Perf report for the last 5.13305 seconds: | 20260507 03:11:17.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:11:17.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:17.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:17.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:17.520 | INFO | ANALYSIS : 59.7%, 3.0, 92.4ms, 0 | 20260507 03:11:17.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:17.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:17.521 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:11:17.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:17.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:17.521 | INFO | Uploaded 36GiB to GPU in 1.96132s (18.355GiB/s) | 20260507 03:11:17.521 | INFO | Downloaded 34GiB to GPU in 1.91713s (17.7349GiB/s) | 20260507 03:11:22.664 | INFO | Analyzing chunk frames = [6633472,6633984) | 20260507 03:11:22.664 | INFO | MemInfo: { | 20260507 03:11:22.664 | INFO | "gpu" : " 17 GB", | 20260507 03:11:22.664 | INFO | "huge" : "", | 20260507 03:11:22.664 | INFO | "huge_shared" : " 27 GB", | 20260507 03:11:22.664 | INFO | "malloc" : " 14 GB", | 20260507 03:11:22.664 | INFO | "pinned" : " 29 GB" | 20260507 03:11:22.664 | INFO | } | 20260507 03:11:22.664 | INFO | Perf report for the last 5.14326 seconds: | 20260507 03:11:22.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:11:22.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:22.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:22.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:22.664 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 03:11:22.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:22.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:22.664 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 03:11:22.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:22.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:22.664 | INFO | Uploaded 36GiB to GPU in 2.00899s (17.9195GiB/s) | 20260507 03:11:22.664 | INFO | Downloaded 34GiB to GPU in 1.94073s (17.5191GiB/s) | 20260507 03:11:27.802 | INFO | Analyzing chunk frames = [6633984,6634496) | 20260507 03:11:27.802 | INFO | MemInfo: { | 20260507 03:11:27.802 | INFO | "gpu" : " 17 GB", | 20260507 03:11:27.802 | INFO | "huge" : "", | 20260507 03:11:27.802 | INFO | "huge_shared" : " 27 GB", | 20260507 03:11:27.802 | INFO | "malloc" : " 14 GB", | 20260507 03:11:27.802 | INFO | "pinned" : " 29 GB" | 20260507 03:11:27.802 | INFO | } | 20260507 03:11:27.802 | INFO | Perf report for the last 5.1387 seconds: | 20260507 03:11:27.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:11:27.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:27.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:27.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:27.802 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:11:27.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:27.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:27.802 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 03:11:27.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:27.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:27.803 | INFO | Uploaded 36GiB to GPU in 2.05165s (17.5469GiB/s) | 20260507 03:11:27.803 | INFO | Downloaded 34GiB to GPU in 1.95968s (17.3498GiB/s) | 20260507 03:11:32.941 | INFO | Analyzing chunk frames = [6634496,6635008) | 20260507 03:11:32.941 | INFO | MemInfo: { | 20260507 03:11:32.941 | INFO | "gpu" : " 17 GB", | 20260507 03:11:32.941 | INFO | "huge" : "", | 20260507 03:11:32.941 | INFO | "huge_shared" : " 27 GB", | 20260507 03:11:32.941 | INFO | "malloc" : " 14 GB", | 20260507 03:11:32.941 | INFO | "pinned" : " 29 GB" | 20260507 03:11:32.941 | INFO | } | 20260507 03:11:32.941 | INFO | Perf report for the last 5.13902 seconds: | 20260507 03:11:32.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:11:32.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:32.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:32.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:32.942 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:11:32.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:32.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:32.942 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 03:11:32.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:32.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:32.942 | INFO | Uploaded 36GiB to GPU in 1.9954s (18.0415GiB/s) | 20260507 03:11:32.942 | INFO | Downloaded 34GiB to GPU in 1.99052s (17.0809GiB/s) | 20260507 03:11:38.083 | INFO | Analyzing chunk frames = [6635008,6635520) | 20260507 03:11:38.083 | INFO | MemInfo: { | 20260507 03:11:38.083 | INFO | "gpu" : " 17 GB", | 20260507 03:11:38.083 | INFO | "huge" : "", | 20260507 03:11:38.083 | INFO | "huge_shared" : " 27 GB", | 20260507 03:11:38.083 | INFO | "malloc" : " 14 GB", | 20260507 03:11:38.083 | INFO | "pinned" : " 29 GB" | 20260507 03:11:38.083 | INFO | } | 20260507 03:11:38.083 | INFO | Perf report for the last 5.14124 seconds: | 20260507 03:11:38.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:11:38.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:38.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:38.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:38.083 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 03:11:38.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:38.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:38.083 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:11:38.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:38.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:38.083 | INFO | Uploaded 36GiB to GPU in 2.00131s (17.9883GiB/s) | 20260507 03:11:38.083 | INFO | Downloaded 34GiB to GPU in 1.97771s (17.1916GiB/s) | 20260507 03:11:38.083 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:11:38.083 | INFO | Total mem usage: 18 GiB | 20260507 03:11:38.083 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:11:38.083 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:11:38.083 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:11:38.083 | INFO | High Water Breakdown: | 20260507 03:11:38.083 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:11:38.083 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:11:38.083 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:11:38.083 | INFO | ------------------------------------------ | 20260507 03:11:38.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:11:38.083 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:11:38.083 | INFO | Total mem usage: 26.5 GiB | 20260507 03:11:38.083 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:11:38.083 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:11:38.083 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:11:38.083 | INFO | High Water Breakdown: | 20260507 03:11:38.083 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:11:38.083 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:11:38.083 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:11:38.083 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:11:38.083 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:11:38.083 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:11:38.083 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:11:38.083 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:11:38.083 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:11:38.083 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:11:38.083 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:11:38.083 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:11:38.083 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:11:38.083 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:11:38.083 | INFO | ------------------------------------------ | 20260507 03:11:38.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:11:38.083 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:11:38.083 | INFO | Total mem usage: 25.6 GiB | 20260507 03:11:38.083 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:11:38.083 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 03:11:38.083 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 03:11:38.083 | INFO | High Water Breakdown: | 20260507 03:11:38.083 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 03:11:38.083 | INFO | ------------------------------------------ | 20260507 03:11:38.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:11:38.083 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:11:38.083 | INFO | Total mem usage: 15.7 GiB | 20260507 03:11:38.083 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:11:38.083 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:11:38.083 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:11:38.084 | INFO | High Water Breakdown: | 20260507 03:11:38.084 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:11:38.084 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:11:38.084 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:11:38.084 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:11:38.084 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:11:38.084 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:11:38.084 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:11:38.084 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:11:38.084 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:11:38.084 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:11:38.084 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:11:38.084 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:11:38.084 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:11:38.084 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:11:38.084 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:11:38.084 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:11:38.084 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:11:38.084 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:11:38.084 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:11:38.084 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:11:38.084 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:11:38.084 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:11:38.084 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:11:38.084 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:11:38.084 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:11:38.084 | INFO | ------------------------------------------ | 20260507 03:11:38.084 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:11:38.084 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:11:38.084 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:11:38.084 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:11:38.084 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.53e+01, 1.33e+01 / 2.15e+00 | 20260507 03:11:38.084 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.00e-01, 9.49e-03 / 4.32e-03 | 20260507 03:11:38.084 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.43e+00, 5.13e-01 / 2.47e-01 | 20260507 03:11:38.084 | INFO | PerfReport: 3.22%, Histogram -- 9.90e-01 / 1.46e+00, 1.65e-01 / 1.07e-01 | 20260507 03:11:38.084 | INFO | PerfReport: 16.12%, DME -- 3.97e+01 / 4.45e+01, 4.02e+00 / 2.21e+00 | 20260507 03:11:38.084 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.77e+01 / 2.05e+01, 1.33e+00 / 1.61e+00 | 20260507 03:11:38.084 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.14e+00 / 2.66e+00, 3.79e-01 / 2.41e-01 | 20260507 03:11:38.084 | INFO | PerfReport: 3.05%, Metrics -- 9.39e-01 / 1.28e+00, 2.51e-01 / 1.62e-01 | 20260507 03:11:38.084 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.18e-01 / 3.76e-01, 5.06e-02 / 2.10e-02 | 20260507 03:11:38.084 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:11:38.084 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:11:38.084 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:11:38.084 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 03:11:38.084 | INFO | PerfReport: 21.74%, Upload -- 2.09e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260507 03:11:38.084 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 03:11:38.084 | INFO | PerfReport: 31.96%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 03:11:38.084 | INFO | PerfReport: 1.41%, LockUpload -- 1.36e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 03:11:38.084 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260507 03:11:38.084 | INFO | PerfReport: 22.18%, LockCompute -- 2.14e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 03:11:38.084 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.82e-02 / 0.00e+00 | 20260507 03:11:38.180 | INFO | Starting a round of Baz IO | 20260507 03:11:41.287 | INFO | Populating RTMetrics report | 20260507 03:11:41.287 | INFO | RTMetrics report populated | 20260507 03:11:41.287 | INFO | Opening tmp RTMetrics json file | 20260507 03:11:41.305 | INFO | Serializing RTMetrics to json | 20260507 03:11:41.306 | INFO | Serialization done, swapping tmp file now... | 20260507 03:11:41.307 | INFO | RTMetrics done | 20260507 03:11:41.377 | INFO | Finished a round of Baz IO: Wrote 1.76 GiB in 3.197 seconds | 20260507 03:11:41.377 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.92% / 0.00% / 67.08% | 20260507 03:11:41.377 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.209208% / 3.97364e-06% / 6.75519e-05% | 20260507 03:11:41.377 | INFO | Baz IO overall: | 20260507 03:11:41.377 | INFO | BytesWritten : 3410784967881 | 20260507 03:11:41.377 | INFO | NumEvents : 1639265822588 | 20260507 03:11:43.221 | INFO | Analyzing chunk frames = [6635520,6636032) | 20260507 03:11:43.221 | INFO | MemInfo: { | 20260507 03:11:43.221 | INFO | "gpu" : " 17 GB", | 20260507 03:11:43.221 | INFO | "huge" : "", | 20260507 03:11:43.221 | INFO | "huge_shared" : " 27 GB", | 20260507 03:11:43.221 | INFO | "malloc" : " 14 GB", | 20260507 03:11:43.221 | INFO | "pinned" : " 29 GB" | 20260507 03:11:43.221 | INFO | } | 20260507 03:11:43.222 | INFO | Perf report for the last 5.13882 seconds: | 20260507 03:11:43.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:11:43.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:43.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:43.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:43.222 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 03:11:43.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:43.222 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:11:43.222 | INFO | PRE_HQ : 59.2%, 1.8, 55.5ms, 0 | 20260507 03:11:43.222 | INFO | BAZWRITER : 61.8%, 3.3, 104.7ms, 0 | 20260507 03:11:43.222 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:11:43.222 | INFO | Uploaded 36GiB to GPU in 2.03877s (17.6577GiB/s) | 20260507 03:11:43.222 | INFO | Downloaded 38GiB to GPU in 2.18733s (17.3728GiB/s) | 20260507 03:11:48.360 | INFO | Analyzing chunk frames = [6636032,6636544) | 20260507 03:11:48.360 | INFO | MemInfo: { | 20260507 03:11:48.360 | INFO | "gpu" : " 17 GB", | 20260507 03:11:48.360 | INFO | "huge" : "", | 20260507 03:11:48.360 | INFO | "huge_shared" : " 27 GB", | 20260507 03:11:48.360 | INFO | "malloc" : " 14 GB", | 20260507 03:11:48.360 | INFO | "pinned" : " 29 GB" | 20260507 03:11:48.360 | INFO | } | 20260507 03:11:48.360 | INFO | Perf report for the last 5.13819 seconds: | 20260507 03:11:48.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:11:48.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:48.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:48.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:48.360 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:11:48.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:48.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:48.360 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:11:48.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:48.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:48.360 | INFO | Uploaded 36GiB to GPU in 2.02408s (17.7859GiB/s) | 20260507 03:11:48.360 | INFO | Downloaded 34GiB to GPU in 1.93853s (17.539GiB/s) | 20260507 03:11:53.499 | INFO | Analyzing chunk frames = [6636544,6637056) | 20260507 03:11:53.499 | INFO | MemInfo: { | 20260507 03:11:53.499 | INFO | "gpu" : " 17 GB", | 20260507 03:11:53.499 | INFO | "huge" : "", | 20260507 03:11:53.499 | INFO | "huge_shared" : " 27 GB", | 20260507 03:11:53.499 | INFO | "malloc" : " 14 GB", | 20260507 03:11:53.499 | INFO | "pinned" : " 29 GB" | 20260507 03:11:53.499 | INFO | } | 20260507 03:11:53.499 | INFO | Perf report for the last 5.13969 seconds: | 20260507 03:11:53.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:11:53.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:53.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:53.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:53.499 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 03:11:53.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:53.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:53.499 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:11:53.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:53.500 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:53.500 | INFO | Uploaded 36GiB to GPU in 1.99665s (18.0302GiB/s) | 20260507 03:11:53.500 | INFO | Downloaded 34GiB to GPU in 1.9429s (17.4996GiB/s) | 20260507 03:11:58.637 | INFO | Analyzing chunk frames = [6637056,6637568) | 20260507 03:11:58.638 | INFO | MemInfo: { | 20260507 03:11:58.638 | INFO | "gpu" : " 17 GB", | 20260507 03:11:58.638 | INFO | "huge" : "", | 20260507 03:11:58.638 | INFO | "huge_shared" : " 27 GB", | 20260507 03:11:58.638 | INFO | "malloc" : " 14 GB", | 20260507 03:11:58.638 | INFO | "pinned" : " 29 GB" | 20260507 03:11:58.638 | INFO | } | 20260507 03:11:58.638 | INFO | Perf report for the last 5.13816 seconds: | 20260507 03:11:58.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:11:58.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:58.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:58.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:58.638 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:11:58.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:58.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:58.638 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:11:58.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:58.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:11:58.638 | INFO | Uploaded 36GiB to GPU in 1.97144s (18.2608GiB/s) | 20260507 03:11:58.638 | INFO | Downloaded 34GiB to GPU in 1.93297s (17.5896GiB/s) | 20260507 03:12:03.773 | INFO | Analyzing chunk frames = [6637568,6638080) | 20260507 03:12:03.773 | INFO | MemInfo: { | 20260507 03:12:03.773 | INFO | "gpu" : " 17 GB", | 20260507 03:12:03.773 | INFO | "huge" : "", | 20260507 03:12:03.773 | INFO | "huge_shared" : " 27 GB", | 20260507 03:12:03.773 | INFO | "malloc" : " 14 GB", | 20260507 03:12:03.773 | INFO | "pinned" : " 29 GB" | 20260507 03:12:03.773 | INFO | } | 20260507 03:12:03.773 | INFO | Perf report for the last 5.13513 seconds: | 20260507 03:12:03.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:12:03.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:03.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:03.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:03.773 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 03:12:03.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:03.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:03.773 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:12:03.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:03.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:03.773 | INFO | Uploaded 36GiB to GPU in 2.02829s (17.749GiB/s) | 20260507 03:12:03.773 | INFO | Downloaded 34GiB to GPU in 1.94446s (17.4856GiB/s) | 20260507 03:12:08.918 | INFO | Analyzing chunk frames = [6638080,6638592) | 20260507 03:12:08.918 | INFO | MemInfo: { | 20260507 03:12:08.918 | INFO | "gpu" : " 17 GB", | 20260507 03:12:08.918 | INFO | "huge" : "", | 20260507 03:12:08.918 | INFO | "huge_shared" : " 27 GB", | 20260507 03:12:08.918 | INFO | "malloc" : " 14 GB", | 20260507 03:12:08.918 | INFO | "pinned" : " 29 GB" | 20260507 03:12:08.918 | INFO | } | 20260507 03:12:08.918 | INFO | Perf report for the last 5.14499 seconds: | 20260507 03:12:08.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:12:08.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:08.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:08.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:08.918 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 03:12:08.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:08.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:08.918 | INFO | PRE_HQ : 27.4%, 1.0, 14.3ms, 0 | 20260507 03:12:08.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:08.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:08.918 | INFO | Uploaded 36GiB to GPU in 2.04565s (17.5983GiB/s) | 20260507 03:12:08.918 | INFO | Downloaded 34GiB to GPU in 1.96356s (17.3155GiB/s) | 20260507 03:12:14.063 | INFO | Analyzing chunk frames = [6638592,6639104) | 20260507 03:12:14.063 | INFO | MemInfo: { | 20260507 03:12:14.063 | INFO | "gpu" : " 17 GB", | 20260507 03:12:14.063 | INFO | "huge" : "", | 20260507 03:12:14.063 | INFO | "huge_shared" : " 27 GB", | 20260507 03:12:14.063 | INFO | "malloc" : " 14 GB", | 20260507 03:12:14.063 | INFO | "pinned" : " 29 GB" | 20260507 03:12:14.063 | INFO | } | 20260507 03:12:14.063 | INFO | Perf report for the last 5.14518 seconds: | 20260507 03:12:14.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:12:14.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:14.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:14.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:14.063 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:12:14.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:14.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:14.063 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 03:12:14.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:14.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:14.063 | INFO | Uploaded 36GiB to GPU in 1.98131s (18.1698GiB/s) | 20260507 03:12:14.063 | INFO | Downloaded 34GiB to GPU in 1.97951s (17.1759GiB/s) | 20260507 03:12:19.201 | INFO | Analyzing chunk frames = [6639104,6639616) | 20260507 03:12:19.201 | INFO | MemInfo: { | 20260507 03:12:19.201 | INFO | "gpu" : " 17 GB", | 20260507 03:12:19.201 | INFO | "huge" : "", | 20260507 03:12:19.202 | INFO | "huge_shared" : " 27 GB", | 20260507 03:12:19.202 | INFO | "malloc" : " 14 GB", | 20260507 03:12:19.202 | INFO | "pinned" : " 29 GB" | 20260507 03:12:19.202 | INFO | } | 20260507 03:12:19.202 | INFO | Perf report for the last 5.13871 seconds: | 20260507 03:12:19.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:12:19.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:19.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:19.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:19.202 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:12:19.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:19.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:19.202 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 03:12:19.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:19.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:19.202 | INFO | Uploaded 36GiB to GPU in 1.98698s (18.118GiB/s) | 20260507 03:12:19.202 | INFO | Downloaded 34GiB to GPU in 1.9673s (17.2825GiB/s) | 20260507 03:12:19.299 | INFO | Starting a round of Baz IO | 20260507 03:12:22.410 | INFO | Populating RTMetrics report | 20260507 03:12:22.410 | INFO | RTMetrics report populated | 20260507 03:12:22.410 | INFO | Opening tmp RTMetrics json file | 20260507 03:12:22.414 | INFO | Serializing RTMetrics to json | 20260507 03:12:22.428 | INFO | Serialization done, swapping tmp file now... | 20260507 03:12:22.429 | INFO | RTMetrics done | 20260507 03:12:22.505 | INFO | Finished a round of Baz IO: Wrote 1.762 GiB in 3.206 seconds | 20260507 03:12:22.505 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.92% / 0.00% / 67.08% | 20260507 03:12:22.505 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.209169% / 3.97364e-06% / 6.75519e-05% | 20260507 03:12:22.505 | INFO | Baz IO overall: | 20260507 03:12:22.505 | INFO | BytesWritten : 3412677204141 | 20260507 03:12:22.505 | INFO | NumEvents : 1640158944341 | 20260507 03:12:24.336 | INFO | Analyzing chunk frames = [6639616,6640128) | 20260507 03:12:24.336 | INFO | MemInfo: { | 20260507 03:12:24.336 | INFO | "gpu" : " 17 GB", | 20260507 03:12:24.336 | INFO | "huge" : "", | 20260507 03:12:24.336 | INFO | "huge_shared" : " 27 GB", | 20260507 03:12:24.336 | INFO | "malloc" : " 14 GB", | 20260507 03:12:24.336 | INFO | "pinned" : " 29 GB" | 20260507 03:12:24.336 | INFO | } | 20260507 03:12:24.336 | INFO | Perf report for the last 5.13448 seconds: | 20260507 03:12:24.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:12:24.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:24.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:24.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:24.336 | INFO | ANALYSIS : 61.9%, 2.9, 95.4ms, 0 | 20260507 03:12:24.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:24.336 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260507 03:12:24.336 | INFO | PRE_HQ : 60.3%, 1.9, 60.9ms, 0 | 20260507 03:12:24.336 | INFO | BAZWRITER : 62.1%, 3.5, 112.2ms, 0 | 20260507 03:12:24.336 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:12:24.336 | INFO | Uploaded 36GiB to GPU in 2.05046s (17.557GiB/s) | 20260507 03:12:24.336 | INFO | Downloaded 38GiB to GPU in 2.17143s (17.5GiB/s) | 20260507 03:12:29.471 | INFO | Analyzing chunk frames = [6640128,6640640) | 20260507 03:12:29.471 | INFO | MemInfo: { | 20260507 03:12:29.471 | INFO | "gpu" : " 17 GB", | 20260507 03:12:29.471 | INFO | "huge" : "", | 20260507 03:12:29.471 | INFO | "huge_shared" : " 27 GB", | 20260507 03:12:29.471 | INFO | "malloc" : " 14 GB", | 20260507 03:12:29.471 | INFO | "pinned" : " 29 GB" | 20260507 03:12:29.471 | INFO | } | 20260507 03:12:29.471 | INFO | Perf report for the last 5.13505 seconds: | 20260507 03:12:29.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:12:29.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:29.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:29.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:29.471 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:12:29.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:29.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:29.471 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 03:12:29.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:29.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:29.472 | INFO | Uploaded 36GiB to GPU in 1.97712s (18.2083GiB/s) | 20260507 03:12:29.472 | INFO | Downloaded 34GiB to GPU in 1.93382s (17.5818GiB/s) | 20260507 03:12:34.614 | INFO | Analyzing chunk frames = [6640640,6641152) | 20260507 03:12:34.614 | INFO | MemInfo: { | 20260507 03:12:34.614 | INFO | "gpu" : " 17 GB", | 20260507 03:12:34.614 | INFO | "huge" : "", | 20260507 03:12:34.614 | INFO | "huge_shared" : " 27 GB", | 20260507 03:12:34.614 | INFO | "malloc" : " 14 GB", | 20260507 03:12:34.614 | INFO | "pinned" : " 29 GB" | 20260507 03:12:34.614 | INFO | } | 20260507 03:12:34.614 | INFO | Perf report for the last 5.14267 seconds: | 20260507 03:12:34.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:12:34.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:34.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:34.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:34.614 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 03:12:34.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:34.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:34.614 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:12:34.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:34.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:34.614 | INFO | Uploaded 36GiB to GPU in 1.99085s (18.0827GiB/s) | 20260507 03:12:34.614 | INFO | Downloaded 34GiB to GPU in 1.93862s (17.5383GiB/s) | 20260507 03:12:39.757 | INFO | Analyzing chunk frames = [6641152,6641664) | 20260507 03:12:39.757 | INFO | MemInfo: { | 20260507 03:12:39.757 | INFO | "gpu" : " 17 GB", | 20260507 03:12:39.757 | INFO | "huge" : "", | 20260507 03:12:39.757 | INFO | "huge_shared" : " 27 GB", | 20260507 03:12:39.757 | INFO | "malloc" : " 14 GB", | 20260507 03:12:39.757 | INFO | "pinned" : " 29 GB" | 20260507 03:12:39.757 | INFO | } | 20260507 03:12:39.757 | INFO | Perf report for the last 5.14331 seconds: | 20260507 03:12:39.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:12:39.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:39.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:39.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:39.757 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 03:12:39.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:39.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:39.757 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:12:39.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:39.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:39.757 | INFO | Uploaded 36GiB to GPU in 1.95888s (18.3778GiB/s) | 20260507 03:12:39.758 | INFO | Downloaded 34GiB to GPU in 1.91585s (17.7467GiB/s) | 20260507 03:12:44.893 | INFO | Analyzing chunk frames = [6641664,6642176) | 20260507 03:12:44.893 | INFO | MemInfo: { | 20260507 03:12:44.893 | INFO | "gpu" : " 17 GB", | 20260507 03:12:44.893 | INFO | "huge" : "", | 20260507 03:12:44.893 | INFO | "huge_shared" : " 27 GB", | 20260507 03:12:44.893 | INFO | "malloc" : " 14 GB", | 20260507 03:12:44.893 | INFO | "pinned" : " 29 GB" | 20260507 03:12:44.893 | INFO | } | 20260507 03:12:44.893 | INFO | Perf report for the last 5.13603 seconds: | 20260507 03:12:44.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:12:44.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:44.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:44.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:44.893 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 03:12:44.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:44.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:44.893 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:12:44.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:44.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:44.893 | INFO | Uploaded 36GiB to GPU in 2.0176s (17.843GiB/s) | 20260507 03:12:44.894 | INFO | Downloaded 34GiB to GPU in 1.94861s (17.4483GiB/s) | 20260507 03:12:50.035 | INFO | Analyzing chunk frames = [6642176,6642688) | 20260507 03:12:50.036 | INFO | MemInfo: { | 20260507 03:12:50.036 | INFO | "gpu" : " 17 GB", | 20260507 03:12:50.036 | INFO | "huge" : "", | 20260507 03:12:50.036 | INFO | "huge_shared" : " 27 GB", | 20260507 03:12:50.036 | INFO | "malloc" : " 14 GB", | 20260507 03:12:50.036 | INFO | "pinned" : " 29 GB" | 20260507 03:12:50.036 | INFO | } | 20260507 03:12:50.036 | INFO | Perf report for the last 5.14231 seconds: | 20260507 03:12:50.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:12:50.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:50.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:50.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:50.036 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:12:50.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:50.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:50.036 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:12:50.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:50.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:50.036 | INFO | Uploaded 36GiB to GPU in 2.04875s (17.5717GiB/s) | 20260507 03:12:50.036 | INFO | Downloaded 34GiB to GPU in 1.96524s (17.3007GiB/s) | 20260507 03:12:55.173 | INFO | Analyzing chunk frames = [6642688,6643200) | 20260507 03:12:55.173 | INFO | MemInfo: { | 20260507 03:12:55.173 | INFO | "gpu" : " 17 GB", | 20260507 03:12:55.173 | INFO | "huge" : "", | 20260507 03:12:55.173 | INFO | "huge_shared" : " 27 GB", | 20260507 03:12:55.173 | INFO | "malloc" : " 14 GB", | 20260507 03:12:55.173 | INFO | "pinned" : " 29 GB" | 20260507 03:12:55.173 | INFO | } | 20260507 03:12:55.173 | INFO | Perf report for the last 5.13753 seconds: | 20260507 03:12:55.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:12:55.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:55.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:55.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:55.173 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:12:55.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:55.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:55.173 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 03:12:55.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:55.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:12:55.173 | INFO | Uploaded 36GiB to GPU in 1.97282s (18.248GiB/s) | 20260507 03:12:55.173 | INFO | Downloaded 34GiB to GPU in 1.97907s (17.1798GiB/s) | 20260507 03:13:00.313 | INFO | Analyzing chunk frames = [6643200,6643712) | 20260507 03:13:00.313 | INFO | MemInfo: { | 20260507 03:13:00.313 | INFO | "gpu" : " 17 GB", | 20260507 03:13:00.313 | INFO | "huge" : "", | 20260507 03:13:00.313 | INFO | "huge_shared" : " 27 GB", | 20260507 03:13:00.313 | INFO | "malloc" : " 14 GB", | 20260507 03:13:00.313 | INFO | "pinned" : " 29 GB" | 20260507 03:13:00.313 | INFO | } | 20260507 03:13:00.313 | INFO | Perf report for the last 5.14005 seconds: | 20260507 03:13:00.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:13:00.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:00.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:00.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:00.313 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:13:00.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:00.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:00.313 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:13:00.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:00.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:00.314 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:13:00.314 | INFO | Total mem usage: 18 GiB | 20260507 03:13:00.314 | INFO | Uploaded 36GiB to GPU in 1.96409s (18.3291GiB/s) | 20260507 03:13:00.314 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:13:00.314 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.2%) | 20260507 03:13:00.314 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:13:00.314 | INFO | High Water Breakdown: | 20260507 03:13:00.314 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:13:00.314 | INFO | Downloaded 34GiB to GPU in 1.97157s (17.2452GiB/s) | 20260507 03:13:00.314 | INFO | 39.8% : 5.67 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:13:00.314 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:13:00.314 | INFO | ------------------------------------------ | 20260507 03:13:00.314 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:13:00.314 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:13:00.314 | INFO | Total mem usage: 26.5 GiB | 20260507 03:13:00.314 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:13:00.314 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:13:00.314 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:13:00.314 | INFO | High Water Breakdown: | 20260507 03:13:00.314 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:13:00.314 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:13:00.314 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:13:00.314 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:13:00.314 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:13:00.314 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:13:00.314 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:13:00.314 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:13:00.314 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:13:00.314 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:13:00.314 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:13:00.314 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:13:00.314 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:13:00.314 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:13:00.314 | INFO | ------------------------------------------ | 20260507 03:13:00.314 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:13:00.314 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:13:00.314 | INFO | Total mem usage: 25.6 GiB | 20260507 03:13:00.314 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:13:00.314 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:13:00.314 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:13:00.314 | INFO | High Water Breakdown: | 20260507 03:13:00.314 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:13:00.314 | INFO | ------------------------------------------ | 20260507 03:13:00.314 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:13:00.314 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:13:00.314 | INFO | Total mem usage: 15.7 GiB | 20260507 03:13:00.314 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:13:00.314 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:13:00.314 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:13:00.314 | INFO | High Water Breakdown: | 20260507 03:13:00.314 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:13:00.314 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:13:00.314 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:13:00.314 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:13:00.314 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:13:00.314 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:13:00.314 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:13:00.314 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:13:00.314 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:13:00.314 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:13:00.314 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:13:00.314 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:13:00.314 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:13:00.314 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:13:00.314 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:13:00.314 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:13:00.314 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:13:00.314 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:13:00.314 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:13:00.314 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:13:00.314 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:13:00.314 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:13:00.314 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:13:00.314 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:13:00.314 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:13:00.314 | INFO | ------------------------------------------ | 20260507 03:13:00.314 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:13:00.314 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:13:00.314 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:13:00.314 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:13:00.314 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.51e+01, 1.33e+01 / 1.35e+00 | 20260507 03:13:00.314 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.67e-03 / 4.49e-03 | 20260507 03:13:00.314 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.45e+00, 5.19e-01 / 2.51e-01 | 20260507 03:13:00.314 | INFO | PerfReport: 3.24%, Histogram -- 9.95e-01 / 1.45e+00, 1.70e-01 / 1.07e-01 | 20260507 03:13:00.314 | INFO | PerfReport: 16.13%, DME -- 3.97e+01 / 4.45e+01, 4.06e+00 / 1.54e+00 | 20260507 03:13:00.314 | INFO | PerfReport: 57.55%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.26e+00 / 1.27e+00 | 20260507 03:13:00.314 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.15e+00 / 2.68e+00, 3.88e-01 / 1.87e-01 | 20260507 03:13:00.314 | INFO | PerfReport: 3.06%, Metrics -- 9.42e-01 / 1.27e+00, 2.50e-01 / 1.12e-01 | 20260507 03:13:00.314 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.19e-01 / 3.72e-01, 4.99e-02 / 2.07e-02 | 20260507 03:13:00.314 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:13:00.314 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:13:00.314 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:13:00.314 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 03:13:00.314 | INFO | PerfReport: 21.69%, Upload -- 2.09e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260507 03:13:00.314 | INFO | PerfReport: 21.43%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 03:13:00.314 | INFO | PerfReport: 31.97%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 03:13:00.314 | INFO | PerfReport: 1.35%, LockUpload -- 1.30e+00 / 0.00e+00, 4.77e+00 / 0.00e+00 | 20260507 03:13:00.314 | INFO | PerfReport: 1.43%, LockDownload -- 1.38e+00 / 0.00e+00, 3.28e+00 / 0.00e+00 | 20260507 03:13:00.315 | INFO | PerfReport: 22.11%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 03:13:00.315 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260507 03:13:00.410 | INFO | Starting a round of Baz IO | 20260507 03:13:03.517 | INFO | Populating RTMetrics report | 20260507 03:13:03.517 | INFO | RTMetrics report populated | 20260507 03:13:03.517 | INFO | Opening tmp RTMetrics json file | 20260507 03:13:03.518 | INFO | Serializing RTMetrics to json | 20260507 03:13:03.520 | INFO | Serialization done, swapping tmp file now... | 20260507 03:13:03.521 | INFO | RTMetrics done | 20260507 03:13:03.613 | INFO | Finished a round of Baz IO: Wrote 1.766 GiB in 3.203 seconds | 20260507 03:13:03.613 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.92% / 0.00% / 67.08% | 20260507 03:13:03.613 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.206049% / 0% / 6.35783e-05% | 20260507 03:13:03.613 | INFO | Baz IO overall: | 20260507 03:13:03.613 | INFO | BytesWritten : 3414573421695 | 20260507 03:13:03.613 | INFO | NumEvents : 1641051946529 | 20260507 03:13:05.448 | INFO | Analyzing chunk frames = [6643712,6644224) | 20260507 03:13:05.448 | INFO | MemInfo: { | 20260507 03:13:05.448 | INFO | "gpu" : " 17 GB", | 20260507 03:13:05.448 | INFO | "huge" : "", | 20260507 03:13:05.448 | INFO | "huge_shared" : " 27 GB", | 20260507 03:13:05.448 | INFO | "malloc" : " 14 GB", | 20260507 03:13:05.448 | INFO | "pinned" : " 29 GB" | 20260507 03:13:05.448 | INFO | } | 20260507 03:13:05.448 | INFO | Perf report for the last 5.13507 seconds: | 20260507 03:13:05.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:13:05.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:05.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:05.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:05.449 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 03:13:05.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:05.449 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 03:13:05.449 | INFO | PRE_HQ : 59.3%, 1.8, 55.3ms, 0 | 20260507 03:13:05.449 | INFO | BAZWRITER : 62.0%, 3.3, 105.4ms, 0 | 20260507 03:13:05.449 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:13:05.449 | INFO | Uploaded 36GiB to GPU in 2.03669s (17.6757GiB/s) | 20260507 03:13:05.449 | INFO | Downloaded 38GiB to GPU in 2.18406s (17.3987GiB/s) | 20260507 03:13:10.587 | INFO | Analyzing chunk frames = [6644224,6644736) | 20260507 03:13:10.587 | INFO | MemInfo: { | 20260507 03:13:10.587 | INFO | "gpu" : " 17 GB", | 20260507 03:13:10.587 | INFO | "huge" : "", | 20260507 03:13:10.587 | INFO | "huge_shared" : " 27 GB", | 20260507 03:13:10.587 | INFO | "malloc" : " 14 GB", | 20260507 03:13:10.587 | INFO | "pinned" : " 29 GB" | 20260507 03:13:10.587 | INFO | } | 20260507 03:13:10.587 | INFO | Perf report for the last 5.13894 seconds: | 20260507 03:13:10.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:13:10.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:10.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:10.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:10.587 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:13:10.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:10.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:10.587 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:13:10.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:10.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:10.588 | INFO | Uploaded 36GiB to GPU in 2.03485s (17.6917GiB/s) | 20260507 03:13:10.588 | INFO | Downloaded 34GiB to GPU in 1.94063s (17.5201GiB/s) | 20260507 03:13:15.727 | INFO | Analyzing chunk frames = [6644736,6645248) | 20260507 03:13:15.727 | INFO | MemInfo: { | 20260507 03:13:15.727 | INFO | "gpu" : " 17 GB", | 20260507 03:13:15.727 | INFO | "huge" : "", | 20260507 03:13:15.727 | INFO | "huge_shared" : " 27 GB", | 20260507 03:13:15.727 | INFO | "malloc" : " 14 GB", | 20260507 03:13:15.727 | INFO | "pinned" : " 29 GB" | 20260507 03:13:15.727 | INFO | } | 20260507 03:13:15.727 | INFO | Perf report for the last 5.13988 seconds: | 20260507 03:13:15.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:13:15.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:15.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:15.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:15.727 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 03:13:15.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:15.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:15.727 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:13:15.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:15.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:15.728 | INFO | Uploaded 36GiB to GPU in 1.98742s (18.114GiB/s) | 20260507 03:13:15.728 | INFO | Downloaded 34GiB to GPU in 1.94134s (17.5137GiB/s) | 20260507 03:13:20.868 | INFO | Analyzing chunk frames = [6645248,6645760) | 20260507 03:13:20.868 | INFO | MemInfo: { | 20260507 03:13:20.868 | INFO | "gpu" : " 17 GB", | 20260507 03:13:20.868 | INFO | "huge" : "", | 20260507 03:13:20.868 | INFO | "huge_shared" : " 27 GB", | 20260507 03:13:20.868 | INFO | "malloc" : " 14 GB", | 20260507 03:13:20.868 | INFO | "pinned" : " 29 GB" | 20260507 03:13:20.868 | INFO | } | 20260507 03:13:20.868 | INFO | Perf report for the last 5.14053 seconds: | 20260507 03:13:20.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:13:20.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:20.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:20.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:20.868 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:13:20.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:20.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:20.868 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:13:20.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:20.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:20.868 | INFO | Uploaded 36GiB to GPU in 1.95716s (18.394GiB/s) | 20260507 03:13:20.868 | INFO | Downloaded 34GiB to GPU in 1.9171s (17.7351GiB/s) | 20260507 03:13:26.017 | INFO | Analyzing chunk frames = [6645760,6646272) | 20260507 03:13:26.017 | INFO | MemInfo: { | 20260507 03:13:26.017 | INFO | "gpu" : " 17 GB", | 20260507 03:13:26.017 | INFO | "huge" : "", | 20260507 03:13:26.017 | INFO | "huge_shared" : " 27 GB", | 20260507 03:13:26.017 | INFO | "malloc" : " 14 GB", | 20260507 03:13:26.017 | INFO | "pinned" : " 29 GB" | 20260507 03:13:26.017 | INFO | } | 20260507 03:13:26.017 | INFO | Perf report for the last 5.14955 seconds: | 20260507 03:13:26.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:13:26.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:26.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:26.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:26.017 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 03:13:26.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:26.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:26.017 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 03:13:26.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:26.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:26.017 | INFO | Uploaded 36GiB to GPU in 2.00977s (17.9125GiB/s) | 20260507 03:13:26.017 | INFO | Downloaded 34GiB to GPU in 1.9446s (17.4843GiB/s) | 20260507 03:13:31.152 | INFO | Analyzing chunk frames = [6646272,6646784) | 20260507 03:13:31.152 | INFO | MemInfo: { | 20260507 03:13:31.152 | INFO | "gpu" : " 17 GB", | 20260507 03:13:31.152 | INFO | "huge" : "", | 20260507 03:13:31.152 | INFO | "huge_shared" : " 27 GB", | 20260507 03:13:31.152 | INFO | "malloc" : " 14 GB", | 20260507 03:13:31.152 | INFO | "pinned" : " 29 GB" | 20260507 03:13:31.152 | INFO | } | 20260507 03:13:31.152 | INFO | Perf report for the last 5.13499 seconds: | 20260507 03:13:31.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:13:31.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:31.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:31.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:31.152 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 03:13:31.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:31.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:31.152 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 03:13:31.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:31.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:31.152 | INFO | Uploaded 36GiB to GPU in 2.03071s (17.7278GiB/s) | 20260507 03:13:31.153 | INFO | Downloaded 34GiB to GPU in 1.91516s (17.7531GiB/s) | 20260507 03:13:36.288 | INFO | Analyzing chunk frames = [6646784,6647296) | 20260507 03:13:36.288 | INFO | MemInfo: { | 20260507 03:13:36.288 | INFO | "gpu" : " 17 GB", | 20260507 03:13:36.288 | INFO | "huge" : "", | 20260507 03:13:36.288 | INFO | "huge_shared" : " 27 GB", | 20260507 03:13:36.288 | INFO | "malloc" : " 14 GB", | 20260507 03:13:36.288 | INFO | "pinned" : " 29 GB" | 20260507 03:13:36.288 | INFO | } | 20260507 03:13:36.288 | INFO | Perf report for the last 5.13554 seconds: | 20260507 03:13:36.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:13:36.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:36.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:36.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:36.288 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:13:36.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:36.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:36.288 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 03:13:36.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:36.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:36.288 | INFO | Uploaded 36GiB to GPU in 1.98993s (18.0911GiB/s) | 20260507 03:13:36.288 | INFO | Downloaded 34GiB to GPU in 1.97759s (17.1927GiB/s) | 20260507 03:13:41.425 | INFO | Analyzing chunk frames = [6647296,6647808) | 20260507 03:13:41.425 | INFO | MemInfo: { | 20260507 03:13:41.425 | INFO | "gpu" : " 17 GB", | 20260507 03:13:41.425 | INFO | "huge" : "", | 20260507 03:13:41.425 | INFO | "huge_shared" : " 27 GB", | 20260507 03:13:41.426 | INFO | "malloc" : " 14 GB", | 20260507 03:13:41.426 | INFO | "pinned" : " 29 GB" | 20260507 03:13:41.426 | INFO | } | 20260507 03:13:41.426 | INFO | Perf report for the last 5.1379 seconds: | 20260507 03:13:41.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:13:41.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:41.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:41.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:41.426 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:13:41.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:41.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:41.426 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 03:13:41.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:41.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:41.426 | INFO | Uploaded 36GiB to GPU in 1.97362s (18.2406GiB/s) | 20260507 03:13:41.426 | INFO | Downloaded 34GiB to GPU in 1.9454s (17.4771GiB/s) | 20260507 03:13:41.534 | INFO | Starting a round of Baz IO | 20260507 03:13:44.639 | INFO | Populating RTMetrics report | 20260507 03:13:44.639 | INFO | RTMetrics report populated | 20260507 03:13:44.639 | INFO | Opening tmp RTMetrics json file | 20260507 03:13:44.640 | INFO | Serializing RTMetrics to json | 20260507 03:13:44.641 | INFO | Serialization done, swapping tmp file now... | 20260507 03:13:44.642 | INFO | RTMetrics done | 20260507 03:13:44.736 | INFO | Finished a round of Baz IO: Wrote 1.765 GiB in 3.201 seconds | 20260507 03:13:44.736 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.92% / 0.00% / 67.08% | 20260507 03:13:44.736 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.207567% / 3.97364e-06% / 5.5631e-05% | 20260507 03:13:44.736 | INFO | Baz IO overall: | 20260507 03:13:44.736 | INFO | BytesWritten : 3416468114917 | 20260507 03:13:44.736 | INFO | NumEvents : 1641944495421 | 20260507 03:13:46.565 | INFO | Analyzing chunk frames = [6647808,6648320) | 20260507 03:13:46.565 | INFO | MemInfo: { | 20260507 03:13:46.565 | INFO | "gpu" : " 17 GB", | 20260507 03:13:46.565 | INFO | "huge" : "", | 20260507 03:13:46.565 | INFO | "huge_shared" : " 27 GB", | 20260507 03:13:46.565 | INFO | "malloc" : " 14 GB", | 20260507 03:13:46.565 | INFO | "pinned" : " 29 GB" | 20260507 03:13:46.566 | INFO | } | 20260507 03:13:46.566 | INFO | Perf report for the last 5.13985 seconds: | 20260507 03:13:46.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:13:46.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:46.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:46.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:46.566 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 03:13:46.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:46.566 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 03:13:46.566 | INFO | PRE_HQ : 59.4%, 1.8, 55.6ms, 0 | 20260507 03:13:46.566 | INFO | BAZWRITER : 62.3%, 3.3, 105.4ms, 0 | 20260507 03:13:46.566 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:13:46.566 | INFO | Uploaded 36GiB to GPU in 2.03192s (17.7172GiB/s) | 20260507 03:13:46.566 | INFO | Downloaded 38GiB to GPU in 2.18211s (17.4143GiB/s) | 20260507 03:13:51.709 | INFO | Analyzing chunk frames = [6648320,6648832) | 20260507 03:13:51.709 | INFO | MemInfo: { | 20260507 03:13:51.709 | INFO | "gpu" : " 17 GB", | 20260507 03:13:51.709 | INFO | "huge" : "", | 20260507 03:13:51.709 | INFO | "huge_shared" : " 27 GB", | 20260507 03:13:51.709 | INFO | "malloc" : " 14 GB", | 20260507 03:13:51.709 | INFO | "pinned" : " 29 GB" | 20260507 03:13:51.709 | INFO | } | 20260507 03:13:51.709 | INFO | Perf report for the last 5.14336 seconds: | 20260507 03:13:51.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:13:51.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:51.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:51.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:51.709 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:13:51.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:51.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:51.709 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:13:51.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:51.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:51.709 | INFO | Uploaded 36GiB to GPU in 2.02539s (17.7744GiB/s) | 20260507 03:13:51.709 | INFO | Downloaded 34GiB to GPU in 1.94165s (17.5109GiB/s) | 20260507 03:13:56.844 | INFO | Analyzing chunk frames = [6648832,6649344) | 20260507 03:13:56.844 | INFO | MemInfo: { | 20260507 03:13:56.844 | INFO | "gpu" : " 17 GB", | 20260507 03:13:56.844 | INFO | "huge" : "", | 20260507 03:13:56.844 | INFO | "huge_shared" : " 27 GB", | 20260507 03:13:56.844 | INFO | "malloc" : " 14 GB", | 20260507 03:13:56.844 | INFO | "pinned" : " 29 GB" | 20260507 03:13:56.844 | INFO | } | 20260507 03:13:56.844 | INFO | Perf report for the last 5.13491 seconds: | 20260507 03:13:56.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:13:56.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:56.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:56.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:56.844 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 03:13:56.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:56.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:56.844 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 03:13:56.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:56.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:13:56.844 | INFO | Uploaded 36GiB to GPU in 2.02121s (17.8111GiB/s) | 20260507 03:13:56.844 | INFO | Downloaded 34GiB to GPU in 1.95372s (17.4027GiB/s) | 20260507 03:14:01.986 | INFO | Analyzing chunk frames = [6649344,6649856) | 20260507 03:14:01.986 | INFO | MemInfo: { | 20260507 03:14:01.986 | INFO | "gpu" : " 17 GB", | 20260507 03:14:01.986 | INFO | "huge" : "", | 20260507 03:14:01.986 | INFO | "huge_shared" : " 27 GB", | 20260507 03:14:01.986 | INFO | "malloc" : " 14 GB", | 20260507 03:14:01.986 | INFO | "pinned" : " 29 GB" | 20260507 03:14:01.986 | INFO | } | 20260507 03:14:01.986 | INFO | Perf report for the last 5.14252 seconds: | 20260507 03:14:01.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:14:01.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:01.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:01.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:01.986 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 03:14:01.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:01.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:01.986 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:14:01.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:01.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:01.987 | INFO | Uploaded 36GiB to GPU in 1.97903s (18.1907GiB/s) | 20260507 03:14:01.987 | INFO | Downloaded 34GiB to GPU in 1.92433s (17.6684GiB/s) | 20260507 03:14:07.128 | INFO | Analyzing chunk frames = [6649856,6650368) | 20260507 03:14:07.128 | INFO | MemInfo: { | 20260507 03:14:07.128 | INFO | "gpu" : " 17 GB", | 20260507 03:14:07.128 | INFO | "huge" : "", | 20260507 03:14:07.128 | INFO | "huge_shared" : " 27 GB", | 20260507 03:14:07.128 | INFO | "malloc" : " 14 GB", | 20260507 03:14:07.128 | INFO | "pinned" : " 29 GB" | 20260507 03:14:07.128 | INFO | } | 20260507 03:14:07.129 | INFO | Perf report for the last 5.14217 seconds: | 20260507 03:14:07.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:14:07.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:07.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:07.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:07.129 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 03:14:07.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:07.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:07.129 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:14:07.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:07.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:07.129 | INFO | Uploaded 36GiB to GPU in 2.02729s (17.7577GiB/s) | 20260507 03:14:07.129 | INFO | Downloaded 34GiB to GPU in 1.94064s (17.52GiB/s) | 20260507 03:14:12.263 | INFO | Analyzing chunk frames = [6650368,6650880) | 20260507 03:14:12.263 | INFO | MemInfo: { | 20260507 03:14:12.263 | INFO | "gpu" : " 17 GB", | 20260507 03:14:12.263 | INFO | "huge" : "", | 20260507 03:14:12.263 | INFO | "huge_shared" : " 27 GB", | 20260507 03:14:12.263 | INFO | "malloc" : " 14 GB", | 20260507 03:14:12.263 | INFO | "pinned" : " 29 GB" | 20260507 03:14:12.263 | INFO | } | 20260507 03:14:12.263 | INFO | Perf report for the last 5.13424 seconds: | 20260507 03:14:12.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:14:12.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:12.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:12.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:12.263 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 03:14:12.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:12.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:12.263 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 03:14:12.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:12.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:12.263 | INFO | Uploaded 36GiB to GPU in 2.03829s (17.6618GiB/s) | 20260507 03:14:12.263 | INFO | Downloaded 34GiB to GPU in 1.9253s (17.6596GiB/s) | 20260507 03:14:17.399 | INFO | Analyzing chunk frames = [6650880,6651392) | 20260507 03:14:17.399 | INFO | MemInfo: { | 20260507 03:14:17.399 | INFO | "gpu" : " 17 GB", | 20260507 03:14:17.399 | INFO | "huge" : "", | 20260507 03:14:17.399 | INFO | "huge_shared" : " 27 GB", | 20260507 03:14:17.399 | INFO | "malloc" : " 14 GB", | 20260507 03:14:17.399 | INFO | "pinned" : " 29 GB" | 20260507 03:14:17.399 | INFO | } | 20260507 03:14:17.400 | INFO | Perf report for the last 5.13667 seconds: | 20260507 03:14:17.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:14:17.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:17.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:17.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:17.400 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:14:17.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:17.400 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:17.400 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 03:14:17.400 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:17.400 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:17.400 | INFO | Uploaded 36GiB to GPU in 1.96844s (18.2886GiB/s) | 20260507 03:14:17.400 | INFO | Downloaded 34GiB to GPU in 1.96974s (17.2612GiB/s) | 20260507 03:14:22.538 | INFO | Analyzing chunk frames = [6651392,6651904) | 20260507 03:14:22.538 | INFO | MemInfo: { | 20260507 03:14:22.538 | INFO | "gpu" : " 17 GB", | 20260507 03:14:22.538 | INFO | "huge" : "", | 20260507 03:14:22.538 | INFO | "huge_shared" : " 27 GB", | 20260507 03:14:22.538 | INFO | "malloc" : " 14 GB", | 20260507 03:14:22.538 | INFO | "pinned" : " 29 GB" | 20260507 03:14:22.538 | INFO | } | 20260507 03:14:22.538 | INFO | Perf report for the last 5.13873 seconds: | 20260507 03:14:22.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:14:22.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:22.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:22.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:22.538 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:14:22.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:22.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:22.538 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 03:14:22.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:22.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:22.538 | INFO | Uploaded 36GiB to GPU in 1.95877s (18.3789GiB/s) | 20260507 03:14:22.539 | INFO | Downloaded 34GiB to GPU in 1.97221s (17.2395GiB/s) | 20260507 03:14:22.539 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:14:22.539 | INFO | Total mem usage: 18 GiB | 20260507 03:14:22.539 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:14:22.539 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:14:22.539 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:14:22.539 | INFO | High Water Breakdown: | 20260507 03:14:22.539 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:14:22.539 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:14:22.539 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:14:22.539 | INFO | ------------------------------------------ | 20260507 03:14:22.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:14:22.539 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:14:22.539 | INFO | Total mem usage: 26.5 GiB | 20260507 03:14:22.539 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:14:22.539 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:14:22.539 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:14:22.539 | INFO | High Water Breakdown: | 20260507 03:14:22.539 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:14:22.539 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:14:22.539 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:14:22.539 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:14:22.539 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:14:22.539 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:14:22.539 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:14:22.539 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:14:22.539 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:14:22.539 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:14:22.539 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:14:22.539 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:14:22.539 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:14:22.539 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:14:22.539 | INFO | ------------------------------------------ | 20260507 03:14:22.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:14:22.539 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:14:22.539 | INFO | Total mem usage: 25.6 GiB | 20260507 03:14:22.539 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:14:22.539 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:14:22.539 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:14:22.539 | INFO | High Water Breakdown: | 20260507 03:14:22.539 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:14:22.539 | INFO | ------------------------------------------ | 20260507 03:14:22.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:14:22.539 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:14:22.539 | INFO | Total mem usage: 15.7 GiB | 20260507 03:14:22.539 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:14:22.539 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:14:22.539 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 03:14:22.539 | INFO | High Water Breakdown: | 20260507 03:14:22.539 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:14:22.539 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:14:22.539 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:14:22.539 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:14:22.539 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:14:22.539 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:14:22.539 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:14:22.539 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:14:22.539 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:14:22.539 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:14:22.539 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:14:22.539 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:14:22.539 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:14:22.539 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:14:22.539 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:14:22.539 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:14:22.539 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:14:22.539 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:14:22.539 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:14:22.539 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:14:22.539 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:14:22.539 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:14:22.539 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:14:22.539 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:14:22.539 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:14:22.539 | INFO | ------------------------------------------ | 20260507 03:14:22.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:14:22.539 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:14:22.539 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:14:22.539 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:14:22.539 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.48e+01, 1.33e+01 / 1.51e+00 | 20260507 03:14:22.539 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.02e-01, 9.46e-03 / 6.17e-03 | 20260507 03:14:22.539 | INFO | PerfReport: 11.68%, Baseline -- 3.60e+00 / 4.47e+00, 5.21e-01 / 2.32e-01 | 20260507 03:14:22.539 | INFO | PerfReport: 3.23%, Histogram -- 9.96e-01 / 1.46e+00, 1.71e-01 / 9.02e-02 | 20260507 03:14:22.540 | INFO | PerfReport: 16.12%, DME -- 3.97e+01 / 4.44e+01, 3.97e+00 / 1.67e+00 | 20260507 03:14:22.540 | INFO | PerfReport: 57.59%, FrameLabeling -- 1.77e+01 / 2.00e+01, 1.31e+00 / 1.54e+00 | 20260507 03:14:22.540 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.15e+00 / 2.74e+00, 3.85e-01 / 2.01e-01 | 20260507 03:14:22.540 | INFO | PerfReport: 3.06%, Metrics -- 9.43e-01 / 1.26e+00, 2.50e-01 / 1.43e-01 | 20260507 03:14:22.540 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.19e-01 / 3.72e-01, 4.99e-02 / 2.20e-02 | 20260507 03:14:22.540 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:14:22.540 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:14:22.540 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:14:22.540 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 03:14:22.540 | INFO | PerfReport: 21.69%, Upload -- 2.09e+01 / 0.00e+00, 7.01e+00 / 0.00e+00 | 20260507 03:14:22.540 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260507 03:14:22.540 | INFO | PerfReport: 32.00%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 03:14:22.540 | INFO | PerfReport: 1.33%, LockUpload -- 1.28e+00 / 0.00e+00, 4.64e+00 / 0.00e+00 | 20260507 03:14:22.540 | INFO | PerfReport: 1.42%, LockDownload -- 1.37e+00 / 0.00e+00, 3.26e+00 / 0.00e+00 | 20260507 03:14:22.540 | INFO | PerfReport: 22.20%, LockCompute -- 2.14e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 03:14:22.540 | INFO | PerfReport: 0.02%, Misc -- 1.45e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260507 03:14:22.635 | INFO | Starting a round of Baz IO | 20260507 03:14:25.751 | INFO | Populating RTMetrics report | 20260507 03:14:25.751 | INFO | RTMetrics report populated | 20260507 03:14:25.751 | INFO | Opening tmp RTMetrics json file | 20260507 03:14:25.770 | INFO | Serializing RTMetrics to json | 20260507 03:14:25.771 | INFO | Serialization done, swapping tmp file now... | 20260507 03:14:25.772 | INFO | RTMetrics done | 20260507 03:14:25.847 | INFO | Finished a round of Baz IO: Wrote 1.763 GiB in 3.211 seconds | 20260507 03:14:25.847 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.91% / 0.00% / 67.08% | 20260507 03:14:25.847 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.20779% / 3.97364e-06% / 7.54992e-05% | 20260507 03:14:25.847 | INFO | Baz IO overall: | 20260507 03:14:25.847 | INFO | BytesWritten : 3418360609916 | 20260507 03:14:25.847 | INFO | NumEvents : 1642836696585 | 20260507 03:14:27.677 | INFO | Analyzing chunk frames = [6651904,6652416) | 20260507 03:14:27.677 | INFO | MemInfo: { | 20260507 03:14:27.677 | INFO | "gpu" : " 17 GB", | 20260507 03:14:27.677 | INFO | "huge" : "", | 20260507 03:14:27.677 | INFO | "huge_shared" : " 27 GB", | 20260507 03:14:27.677 | INFO | "malloc" : " 14 GB", | 20260507 03:14:27.677 | INFO | "pinned" : " 29 GB" | 20260507 03:14:27.677 | INFO | } | 20260507 03:14:27.677 | INFO | Perf report for the last 5.13925 seconds: | 20260507 03:14:27.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:14:27.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:27.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:27.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:27.678 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 03:14:27.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:27.678 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 03:14:27.678 | INFO | PRE_HQ : 59.7%, 1.9, 58.3ms, 0 | 20260507 03:14:27.678 | INFO | BAZWRITER : 62.1%, 3.4, 107.9ms, 0 | 20260507 03:14:27.678 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:14:27.678 | INFO | Uploaded 36GiB to GPU in 2.0414s (17.635GiB/s) | 20260507 03:14:27.678 | INFO | Downloaded 38GiB to GPU in 2.18018s (17.4297GiB/s) | 20260507 03:14:32.821 | INFO | Analyzing chunk frames = [6652416,6652928) | 20260507 03:14:32.821 | INFO | MemInfo: { | 20260507 03:14:32.821 | INFO | "gpu" : " 17 GB", | 20260507 03:14:32.821 | INFO | "huge" : "", | 20260507 03:14:32.821 | INFO | "huge_shared" : " 27 GB", | 20260507 03:14:32.821 | INFO | "malloc" : " 14 GB", | 20260507 03:14:32.821 | INFO | "pinned" : " 29 GB" | 20260507 03:14:32.821 | INFO | } | 20260507 03:14:32.821 | INFO | Perf report for the last 5.14341 seconds: | 20260507 03:14:32.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:14:32.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:32.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:32.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:32.821 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:14:32.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:32.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:32.821 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 03:14:32.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:32.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:32.821 | INFO | Uploaded 36GiB to GPU in 2.01706s (17.8477GiB/s) | 20260507 03:14:32.821 | INFO | Downloaded 34GiB to GPU in 1.94028s (17.5232GiB/s) | 20260507 03:14:37.960 | INFO | Analyzing chunk frames = [6652928,6653440) | 20260507 03:14:37.960 | INFO | MemInfo: { | 20260507 03:14:37.960 | INFO | "gpu" : " 17 GB", | 20260507 03:14:37.960 | INFO | "huge" : "", | 20260507 03:14:37.960 | INFO | "huge_shared" : " 27 GB", | 20260507 03:14:37.960 | INFO | "malloc" : " 14 GB", | 20260507 03:14:37.960 | INFO | "pinned" : " 29 GB" | 20260507 03:14:37.960 | INFO | } | 20260507 03:14:37.960 | INFO | Perf report for the last 5.13946 seconds: | 20260507 03:14:37.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:14:37.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:37.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:37.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:37.960 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 03:14:37.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:37.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:37.960 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:14:37.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:37.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:37.961 | INFO | Uploaded 36GiB to GPU in 1.98756s (18.1126GiB/s) | 20260507 03:14:37.961 | INFO | Downloaded 34GiB to GPU in 1.94352s (17.494GiB/s) | 20260507 03:14:43.099 | INFO | Analyzing chunk frames = [6653440,6653952) | 20260507 03:14:43.099 | INFO | MemInfo: { | 20260507 03:14:43.099 | INFO | "gpu" : " 17 GB", | 20260507 03:14:43.099 | INFO | "huge" : "", | 20260507 03:14:43.099 | INFO | "huge_shared" : " 27 GB", | 20260507 03:14:43.099 | INFO | "malloc" : " 14 GB", | 20260507 03:14:43.099 | INFO | "pinned" : " 29 GB" | 20260507 03:14:43.099 | INFO | } | 20260507 03:14:43.099 | INFO | Perf report for the last 5.13855 seconds: | 20260507 03:14:43.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:14:43.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:43.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:43.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:43.099 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:14:43.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:43.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:43.099 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:14:43.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:43.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:43.099 | INFO | Uploaded 36GiB to GPU in 1.99397s (18.0545GiB/s) | 20260507 03:14:43.099 | INFO | Downloaded 34GiB to GPU in 1.93389s (17.5812GiB/s) | 20260507 03:14:48.234 | INFO | Analyzing chunk frames = [6653952,6654464) | 20260507 03:14:48.234 | INFO | MemInfo: { | 20260507 03:14:48.234 | INFO | "gpu" : " 17 GB", | 20260507 03:14:48.234 | INFO | "huge" : "", | 20260507 03:14:48.234 | INFO | "huge_shared" : " 27 GB", | 20260507 03:14:48.234 | INFO | "malloc" : " 14 GB", | 20260507 03:14:48.234 | INFO | "pinned" : " 29 GB" | 20260507 03:14:48.234 | INFO | } | 20260507 03:14:48.234 | INFO | Perf report for the last 5.13514 seconds: | 20260507 03:14:48.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:14:48.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:48.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:48.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:48.234 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 03:14:48.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:48.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:48.234 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 03:14:48.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:48.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:48.234 | INFO | Uploaded 36GiB to GPU in 2.03953s (17.6512GiB/s) | 20260507 03:14:48.234 | INFO | Downloaded 34GiB to GPU in 1.9451s (17.4798GiB/s) | 20260507 03:14:53.377 | INFO | Analyzing chunk frames = [6654464,6654976) | 20260507 03:14:53.377 | INFO | MemInfo: { | 20260507 03:14:53.377 | INFO | "gpu" : " 17 GB", | 20260507 03:14:53.377 | INFO | "huge" : "", | 20260507 03:14:53.377 | INFO | "huge_shared" : " 27 GB", | 20260507 03:14:53.377 | INFO | "malloc" : " 14 GB", | 20260507 03:14:53.377 | INFO | "pinned" : " 29 GB" | 20260507 03:14:53.377 | INFO | } | 20260507 03:14:53.377 | INFO | Perf report for the last 5.14292 seconds: | 20260507 03:14:53.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:14:53.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:53.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:53.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:53.377 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 03:14:53.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:53.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:53.377 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 03:14:53.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:53.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:53.377 | INFO | Uploaded 36GiB to GPU in 2.03014s (17.7328GiB/s) | 20260507 03:14:53.377 | INFO | Downloaded 34GiB to GPU in 1.94911s (17.4439GiB/s) | 20260507 03:14:58.514 | INFO | Analyzing chunk frames = [6654976,6655488) | 20260507 03:14:58.514 | INFO | MemInfo: { | 20260507 03:14:58.514 | INFO | "gpu" : " 17 GB", | 20260507 03:14:58.514 | INFO | "huge" : "", | 20260507 03:14:58.514 | INFO | "huge_shared" : " 27 GB", | 20260507 03:14:58.514 | INFO | "malloc" : " 14 GB", | 20260507 03:14:58.514 | INFO | "pinned" : " 29 GB" | 20260507 03:14:58.514 | INFO | } | 20260507 03:14:58.515 | INFO | Perf report for the last 5.13744 seconds: | 20260507 03:14:58.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:14:58.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:58.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:58.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:58.515 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:14:58.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:58.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:58.515 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:14:58.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:58.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:14:58.515 | INFO | Uploaded 36GiB to GPU in 1.96213s (18.3474GiB/s) | 20260507 03:14:58.515 | INFO | Downloaded 34GiB to GPU in 2.0023s (16.9805GiB/s) | 20260507 03:15:03.660 | INFO | Analyzing chunk frames = [6655488,6656000) | 20260507 03:15:03.660 | INFO | MemInfo: { | 20260507 03:15:03.660 | INFO | "gpu" : " 17 GB", | 20260507 03:15:03.660 | INFO | "huge" : "", | 20260507 03:15:03.660 | INFO | "huge_shared" : " 27 GB", | 20260507 03:15:03.660 | INFO | "malloc" : " 14 GB", | 20260507 03:15:03.660 | INFO | "pinned" : " 29 GB" | 20260507 03:15:03.660 | INFO | } | 20260507 03:15:03.660 | INFO | Perf report for the last 5.14592 seconds: | 20260507 03:15:03.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:15:03.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:03.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:03.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:03.661 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 03:15:03.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:03.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:03.661 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:15:03.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:03.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:03.661 | INFO | Uploaded 36GiB to GPU in 2.00239s (17.9785GiB/s) | 20260507 03:15:03.661 | INFO | Downloaded 34GiB to GPU in 1.96982s (17.2605GiB/s) | 20260507 03:15:03.757 | INFO | Starting a round of Baz IO | 20260507 03:15:06.859 | INFO | Populating RTMetrics report | 20260507 03:15:06.859 | INFO | RTMetrics report populated | 20260507 03:15:06.859 | INFO | Opening tmp RTMetrics json file | 20260507 03:15:06.862 | INFO | Serializing RTMetrics to json | 20260507 03:15:06.878 | INFO | Serialization done, swapping tmp file now... | 20260507 03:15:06.879 | INFO | RTMetrics done | 20260507 03:15:06.952 | INFO | Finished a round of Baz IO: Wrote 1.761 GiB in 3.194 seconds | 20260507 03:15:06.952 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.91% / 0.00% / 67.09% | 20260507 03:15:06.952 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.207857% / 7.94729e-06% / 6.75519e-05% | 20260507 03:15:06.952 | INFO | Baz IO overall: | 20260507 03:15:06.952 | INFO | BytesWritten : 3420251902102 | 20260507 03:15:06.952 | INFO | NumEvents : 1643728760161 | 20260507 03:15:08.796 | INFO | Analyzing chunk frames = [6656000,6656512) | 20260507 03:15:08.796 | INFO | MemInfo: { | 20260507 03:15:08.796 | INFO | "gpu" : " 17 GB", | 20260507 03:15:08.796 | INFO | "huge" : "", | 20260507 03:15:08.796 | INFO | "huge_shared" : " 27 GB", | 20260507 03:15:08.796 | INFO | "malloc" : " 14 GB", | 20260507 03:15:08.796 | INFO | "pinned" : " 29 GB" | 20260507 03:15:08.796 | INFO | } | 20260507 03:15:08.796 | INFO | Perf report for the last 5.13595 seconds: | 20260507 03:15:08.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:15:08.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:08.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:08.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:08.796 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 03:15:08.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:08.796 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:15:08.796 | INFO | PRE_HQ : 59.2%, 1.8, 55.4ms, 0 | 20260507 03:15:08.796 | INFO | BAZWRITER : 61.8%, 3.3, 105.9ms, 0 | 20260507 03:15:08.796 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:15:08.797 | INFO | Uploaded 36GiB to GPU in 2.06104s (17.4669GiB/s) | 20260507 03:15:08.797 | INFO | Downloaded 38GiB to GPU in 2.17942s (17.4358GiB/s) | 20260507 03:15:13.933 | INFO | Analyzing chunk frames = [6656512,6657024) | 20260507 03:15:13.933 | INFO | MemInfo: { | 20260507 03:15:13.933 | INFO | "gpu" : " 17 GB", | 20260507 03:15:13.933 | INFO | "huge" : "", | 20260507 03:15:13.933 | INFO | "huge_shared" : " 27 GB", | 20260507 03:15:13.933 | INFO | "malloc" : " 14 GB", | 20260507 03:15:13.933 | INFO | "pinned" : " 29 GB" | 20260507 03:15:13.933 | INFO | } | 20260507 03:15:13.933 | INFO | Perf report for the last 5.13702 seconds: | 20260507 03:15:13.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:15:13.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:13.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:13.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:13.933 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:15:13.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:13.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:13.933 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:15:13.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:13.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:13.934 | INFO | Uploaded 36GiB to GPU in 2.03786s (17.6656GiB/s) | 20260507 03:15:13.934 | INFO | Downloaded 34GiB to GPU in 1.93592s (17.5627GiB/s) | 20260507 03:15:19.076 | INFO | Analyzing chunk frames = [6657024,6657536) | 20260507 03:15:19.076 | INFO | MemInfo: { | 20260507 03:15:19.076 | INFO | "gpu" : " 17 GB", | 20260507 03:15:19.076 | INFO | "huge" : "", | 20260507 03:15:19.076 | INFO | "huge_shared" : " 27 GB", | 20260507 03:15:19.076 | INFO | "malloc" : " 14 GB", | 20260507 03:15:19.076 | INFO | "pinned" : " 29 GB" | 20260507 03:15:19.076 | INFO | } | 20260507 03:15:19.076 | INFO | Perf report for the last 5.1424 seconds: | 20260507 03:15:19.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:15:19.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:19.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:19.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:19.076 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 03:15:19.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:19.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:19.076 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:15:19.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:19.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:19.076 | INFO | Uploaded 36GiB to GPU in 2.01668s (17.8511GiB/s) | 20260507 03:15:19.076 | INFO | Downloaded 34GiB to GPU in 1.93674s (17.5552GiB/s) | 20260507 03:15:24.210 | INFO | Analyzing chunk frames = [6657536,6658048) | 20260507 03:15:24.210 | INFO | MemInfo: { | 20260507 03:15:24.210 | INFO | "gpu" : " 17 GB", | 20260507 03:15:24.210 | INFO | "huge" : "", | 20260507 03:15:24.210 | INFO | "huge_shared" : " 27 GB", | 20260507 03:15:24.210 | INFO | "malloc" : " 14 GB", | 20260507 03:15:24.210 | INFO | "pinned" : " 29 GB" | 20260507 03:15:24.210 | INFO | } | 20260507 03:15:24.210 | INFO | Perf report for the last 5.13425 seconds: | 20260507 03:15:24.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:15:24.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:24.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:24.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:24.210 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 03:15:24.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:24.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:24.210 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:15:24.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:24.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:24.210 | INFO | Uploaded 36GiB to GPU in 1.97893s (18.1916GiB/s) | 20260507 03:15:24.210 | INFO | Downloaded 34GiB to GPU in 1.90985s (17.8024GiB/s) | 20260507 03:15:29.346 | INFO | Analyzing chunk frames = [6658048,6658560) | 20260507 03:15:29.346 | INFO | MemInfo: { | 20260507 03:15:29.346 | INFO | "gpu" : " 17 GB", | 20260507 03:15:29.346 | INFO | "huge" : "", | 20260507 03:15:29.346 | INFO | "huge_shared" : " 27 GB", | 20260507 03:15:29.346 | INFO | "malloc" : " 14 GB", | 20260507 03:15:29.346 | INFO | "pinned" : " 29 GB" | 20260507 03:15:29.346 | INFO | } | 20260507 03:15:29.346 | INFO | Perf report for the last 5.13632 seconds: | 20260507 03:15:29.346 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:15:29.346 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:29.346 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:29.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:29.347 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 03:15:29.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:29.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:29.347 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:15:29.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:29.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:29.347 | INFO | Uploaded 36GiB to GPU in 2.02924s (17.7406GiB/s) | 20260507 03:15:29.347 | INFO | Downloaded 34GiB to GPU in 1.93652s (17.5573GiB/s) | 20260507 03:15:34.485 | INFO | Analyzing chunk frames = [6658560,6659072) | 20260507 03:15:34.485 | INFO | MemInfo: { | 20260507 03:15:34.485 | INFO | "gpu" : " 17 GB", | 20260507 03:15:34.485 | INFO | "huge" : "", | 20260507 03:15:34.485 | INFO | "huge_shared" : " 27 GB", | 20260507 03:15:34.485 | INFO | "malloc" : " 14 GB", | 20260507 03:15:34.485 | INFO | "pinned" : " 29 GB" | 20260507 03:15:34.485 | INFO | } | 20260507 03:15:34.485 | INFO | Perf report for the last 5.13897 seconds: | 20260507 03:15:34.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:15:34.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:34.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:34.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:34.485 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:15:34.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:34.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:34.486 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 03:15:34.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:34.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:34.486 | INFO | Uploaded 36GiB to GPU in 2.05271s (17.5378GiB/s) | 20260507 03:15:34.486 | INFO | Downloaded 34GiB to GPU in 1.95016s (17.4345GiB/s) | 20260507 03:15:39.635 | INFO | Analyzing chunk frames = [6659072,6659584) | 20260507 03:15:39.635 | INFO | MemInfo: { | 20260507 03:15:39.635 | INFO | "gpu" : " 17 GB", | 20260507 03:15:39.635 | INFO | "huge" : "", | 20260507 03:15:39.635 | INFO | "huge_shared" : " 27 GB", | 20260507 03:15:39.635 | INFO | "malloc" : " 14 GB", | 20260507 03:15:39.635 | INFO | "pinned" : " 29 GB" | 20260507 03:15:39.635 | INFO | } | 20260507 03:15:39.635 | INFO | Perf report for the last 5.14927 seconds: | 20260507 03:15:39.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:15:39.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:39.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:39.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:39.635 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 03:15:39.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:39.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:39.635 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 03:15:39.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:39.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:39.635 | INFO | Uploaded 36GiB to GPU in 1.97196s (18.2559GiB/s) | 20260507 03:15:39.635 | INFO | Downloaded 34GiB to GPU in 1.99648s (17.03GiB/s) | 20260507 03:15:44.772 | INFO | Analyzing chunk frames = [6659584,6660096) | 20260507 03:15:44.772 | INFO | MemInfo: { | 20260507 03:15:44.772 | INFO | "gpu" : " 17 GB", | 20260507 03:15:44.772 | INFO | "huge" : "", | 20260507 03:15:44.772 | INFO | "huge_shared" : " 27 GB", | 20260507 03:15:44.772 | INFO | "malloc" : " 14 GB", | 20260507 03:15:44.772 | INFO | "pinned" : " 29 GB" | 20260507 03:15:44.772 | INFO | } | 20260507 03:15:44.772 | INFO | Perf report for the last 5.1378 seconds: | 20260507 03:15:44.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:15:44.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:44.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:44.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:44.773 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:15:44.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:44.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:44.773 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:15:44.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:44.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:44.773 | INFO | Uploaded 36GiB to GPU in 1.99055s (18.0855GiB/s) | 20260507 03:15:44.773 | INFO | Downloaded 34GiB to GPU in 1.9675s (17.2808GiB/s) | 20260507 03:15:44.773 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:15:44.773 | INFO | Total mem usage: 18 GiB | 20260507 03:15:44.773 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:15:44.773 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 03:15:44.773 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:15:44.773 | INFO | High Water Breakdown: | 20260507 03:15:44.773 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:15:44.773 | INFO | 39.8% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:15:44.773 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:15:44.773 | INFO | ------------------------------------------ | 20260507 03:15:44.773 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:15:44.773 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:15:44.773 | INFO | Total mem usage: 26.5 GiB | 20260507 03:15:44.773 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:15:44.773 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:15:44.773 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:15:44.773 | INFO | High Water Breakdown: | 20260507 03:15:44.773 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:15:44.773 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:15:44.773 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:15:44.773 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:15:44.773 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:15:44.773 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:15:44.773 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:15:44.773 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:15:44.773 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:15:44.773 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:15:44.773 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:15:44.773 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:15:44.773 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:15:44.773 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:15:44.773 | INFO | ------------------------------------------ | 20260507 03:15:44.773 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:15:44.773 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:15:44.773 | INFO | Total mem usage: 25.6 GiB | 20260507 03:15:44.773 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:15:44.773 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 03:15:44.773 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 03:15:44.773 | INFO | High Water Breakdown: | 20260507 03:15:44.773 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 03:15:44.773 | INFO | ------------------------------------------ | 20260507 03:15:44.773 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:15:44.773 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:15:44.773 | INFO | Total mem usage: 15.7 GiB | 20260507 03:15:44.773 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:15:44.773 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:15:44.773 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 03:15:44.773 | INFO | High Water Breakdown: | 20260507 03:15:44.773 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:15:44.773 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:15:44.773 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:15:44.773 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:15:44.773 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:15:44.773 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:15:44.773 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:15:44.773 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:15:44.773 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:15:44.773 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:15:44.773 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:15:44.773 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:15:44.773 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:15:44.773 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:15:44.773 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:15:44.773 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:15:44.773 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:15:44.773 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:15:44.774 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:15:44.774 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:15:44.774 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:15:44.774 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:15:44.774 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:15:44.774 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:15:44.774 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:15:44.774 | INFO | ------------------------------------------ | 20260507 03:15:44.774 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:15:44.774 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:15:44.774 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:15:44.774 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:15:44.774 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.60e+01, 1.33e+01 / 1.80e+00 | 20260507 03:15:44.774 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.00e-01, 9.76e-03 / 4.93e-03 | 20260507 03:15:44.774 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.49e+00, 5.16e-01 / 1.97e-01 | 20260507 03:15:44.774 | INFO | PerfReport: 3.22%, Histogram -- 9.91e-01 / 1.47e+00, 1.65e-01 / 1.10e-01 | 20260507 03:15:44.774 | INFO | PerfReport: 16.21%, DME -- 3.99e+01 / 4.55e+01, 4.40e+00 / 2.27e+00 | 20260507 03:15:44.774 | INFO | PerfReport: 57.48%, FrameLabeling -- 1.77e+01 / 2.00e+01, 1.17e+00 / 1.38e+00 | 20260507 03:15:44.774 | INFO | PerfReport: 7.02%, PulseAccumulating -- 2.16e+00 / 2.75e+00, 3.90e-01 / 1.80e-01 | 20260507 03:15:44.774 | INFO | PerfReport: 3.06%, Metrics -- 9.41e-01 / 1.25e+00, 2.48e-01 / 1.40e-01 | 20260507 03:15:44.774 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.18e-01 / 3.71e-01, 5.02e-02 / 1.44e-02 | 20260507 03:15:44.774 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:15:44.774 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:15:44.774 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:15:44.774 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 03:15:44.774 | INFO | PerfReport: 21.76%, Upload -- 2.10e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 03:15:44.774 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 03:15:44.774 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 03:15:44.774 | INFO | PerfReport: 1.45%, LockUpload -- 1.40e+00 / 0.00e+00, 4.96e+00 / 0.00e+00 | 20260507 03:15:44.774 | INFO | PerfReport: 1.34%, LockDownload -- 1.30e+00 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260507 03:15:44.774 | INFO | PerfReport: 22.11%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 03:15:44.774 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260507 03:15:44.870 | INFO | Starting a round of Baz IO | 20260507 03:15:47.983 | INFO | Populating RTMetrics report | 20260507 03:15:47.983 | INFO | RTMetrics report populated | 20260507 03:15:47.983 | INFO | Opening tmp RTMetrics json file | 20260507 03:15:48.001 | INFO | Serializing RTMetrics to json | 20260507 03:15:48.002 | INFO | Serialization done, swapping tmp file now... | 20260507 03:15:48.003 | INFO | RTMetrics done | 20260507 03:15:48.069 | INFO | Finished a round of Baz IO: Wrote 1.761 GiB in 3.199 seconds | 20260507 03:15:48.069 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.91% / 0.00% / 67.09% | 20260507 03:15:48.069 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.207174% / 0% / 0.000103315% | 20260507 03:15:48.069 | INFO | Baz IO overall: | 20260507 03:15:48.069 | INFO | BytesWritten : 3422143197002 | 20260507 03:15:48.069 | INFO | NumEvents : 1644621457113 | 20260507 03:15:49.910 | INFO | Analyzing chunk frames = [6660096,6660608) | 20260507 03:15:49.910 | INFO | MemInfo: { | 20260507 03:15:49.910 | INFO | "gpu" : " 17 GB", | 20260507 03:15:49.910 | INFO | "huge" : "", | 20260507 03:15:49.910 | INFO | "huge_shared" : " 27 GB", | 20260507 03:15:49.910 | INFO | "malloc" : " 14 GB", | 20260507 03:15:49.910 | INFO | "pinned" : " 29 GB" | 20260507 03:15:49.910 | INFO | } | 20260507 03:15:49.911 | INFO | Perf report for the last 5.138 seconds: | 20260507 03:15:49.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:15:49.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:49.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:49.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:49.911 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 03:15:49.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:49.911 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 03:15:49.911 | INFO | PRE_HQ : 59.5%, 1.8, 55.9ms, 0 | 20260507 03:15:49.911 | INFO | BAZWRITER : 61.8%, 3.3, 105.9ms, 0 | 20260507 03:15:49.911 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:15:49.911 | INFO | Uploaded 36GiB to GPU in 2.05787s (17.4939GiB/s) | 20260507 03:15:49.911 | INFO | Downloaded 38GiB to GPU in 2.18383s (17.4006GiB/s) | 20260507 03:15:55.045 | INFO | Analyzing chunk frames = [6660608,6661120) | 20260507 03:15:55.045 | INFO | MemInfo: { | 20260507 03:15:55.045 | INFO | "gpu" : " 17 GB", | 20260507 03:15:55.045 | INFO | "huge" : "", | 20260507 03:15:55.045 | INFO | "huge_shared" : " 27 GB", | 20260507 03:15:55.045 | INFO | "malloc" : " 14 GB", | 20260507 03:15:55.045 | INFO | "pinned" : " 29 GB" | 20260507 03:15:55.045 | INFO | } | 20260507 03:15:55.045 | INFO | Perf report for the last 5.13475 seconds: | 20260507 03:15:55.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:15:55.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:55.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:55.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:55.045 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 03:15:55.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:55.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:55.045 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:15:55.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:55.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:15:55.045 | INFO | Uploaded 36GiB to GPU in 2.05112s (17.5514GiB/s) | 20260507 03:15:55.046 | INFO | Downloaded 34GiB to GPU in 1.9403s (17.5231GiB/s) | 20260507 03:16:00.185 | INFO | Analyzing chunk frames = [6661120,6661632) | 20260507 03:16:00.185 | INFO | MemInfo: { | 20260507 03:16:00.185 | INFO | "gpu" : " 17 GB", | 20260507 03:16:00.185 | INFO | "huge" : "", | 20260507 03:16:00.186 | INFO | "huge_shared" : " 27 GB", | 20260507 03:16:00.186 | INFO | "malloc" : " 14 GB", | 20260507 03:16:00.186 | INFO | "pinned" : " 29 GB" | 20260507 03:16:00.186 | INFO | } | 20260507 03:16:00.186 | INFO | Perf report for the last 5.14031 seconds: | 20260507 03:16:00.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:16:00.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:00.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:00.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:00.186 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 03:16:00.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:00.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:00.186 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:16:00.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:00.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:00.186 | INFO | Uploaded 36GiB to GPU in 2.00944s (17.9154GiB/s) | 20260507 03:16:00.186 | INFO | Downloaded 34GiB to GPU in 1.93671s (17.5555GiB/s) | 20260507 03:16:05.336 | INFO | Analyzing chunk frames = [6661632,6662144) | 20260507 03:16:05.336 | INFO | MemInfo: { | 20260507 03:16:05.336 | INFO | "gpu" : " 17 GB", | 20260507 03:16:05.336 | INFO | "huge" : "", | 20260507 03:16:05.336 | INFO | "huge_shared" : " 27 GB", | 20260507 03:16:05.336 | INFO | "malloc" : " 14 GB", | 20260507 03:16:05.336 | INFO | "pinned" : " 29 GB" | 20260507 03:16:05.336 | INFO | } | 20260507 03:16:05.336 | INFO | Perf report for the last 5.1507 seconds: | 20260507 03:16:05.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:16:05.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:05.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:05.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:05.336 | INFO | ANALYSIS : 59.7%, 3.0, 92.8ms, 0 | 20260507 03:16:05.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:05.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:05.336 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 03:16:05.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:05.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:05.336 | INFO | Uploaded 36GiB to GPU in 1.98069s (18.1755GiB/s) | 20260507 03:16:05.337 | INFO | Downloaded 34GiB to GPU in 1.90579s (17.8404GiB/s) | 20260507 03:16:10.469 | INFO | Analyzing chunk frames = [6662144,6662656) | 20260507 03:16:10.469 | INFO | MemInfo: { | 20260507 03:16:10.469 | INFO | "gpu" : " 17 GB", | 20260507 03:16:10.469 | INFO | "huge" : "", | 20260507 03:16:10.469 | INFO | "huge_shared" : " 27 GB", | 20260507 03:16:10.469 | INFO | "malloc" : " 14 GB", | 20260507 03:16:10.469 | INFO | "pinned" : " 29 GB" | 20260507 03:16:10.469 | INFO | } | 20260507 03:16:10.469 | INFO | Perf report for the last 5.13252 seconds: | 20260507 03:16:10.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:16:10.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:10.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:10.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:10.469 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 03:16:10.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:10.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:10.469 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 03:16:10.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:10.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:10.469 | INFO | Uploaded 36GiB to GPU in 2.03925s (17.6535GiB/s) | 20260507 03:16:10.469 | INFO | Downloaded 34GiB to GPU in 1.94115s (17.5154GiB/s) | 20260507 03:16:15.609 | INFO | Analyzing chunk frames = [6662656,6663168) | 20260507 03:16:15.609 | INFO | MemInfo: { | 20260507 03:16:15.609 | INFO | "gpu" : " 17 GB", | 20260507 03:16:15.609 | INFO | "huge" : "", | 20260507 03:16:15.609 | INFO | "huge_shared" : " 27 GB", | 20260507 03:16:15.609 | INFO | "malloc" : " 14 GB", | 20260507 03:16:15.609 | INFO | "pinned" : " 29 GB" | 20260507 03:16:15.609 | INFO | } | 20260507 03:16:15.609 | INFO | Perf report for the last 5.14033 seconds: | 20260507 03:16:15.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:16:15.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:15.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:15.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:15.610 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 03:16:15.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:15.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:15.610 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:16:15.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:15.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:15.611 | INFO | Uploaded 36GiB to GPU in 2.03492s (17.6911GiB/s) | 20260507 03:16:15.611 | INFO | Downloaded 34GiB to GPU in 1.91888s (17.7187GiB/s) | 20260507 03:16:20.742 | INFO | Analyzing chunk frames = [6663168,6663680) | 20260507 03:16:20.742 | INFO | MemInfo: { | 20260507 03:16:20.742 | INFO | "gpu" : " 17 GB", | 20260507 03:16:20.742 | INFO | "huge" : "", | 20260507 03:16:20.742 | INFO | "huge_shared" : " 27 GB", | 20260507 03:16:20.742 | INFO | "malloc" : " 14 GB", | 20260507 03:16:20.742 | INFO | "pinned" : " 29 GB" | 20260507 03:16:20.742 | INFO | } | 20260507 03:16:20.742 | INFO | Perf report for the last 5.13289 seconds: | 20260507 03:16:20.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:16:20.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:20.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:20.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:20.742 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 03:16:20.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:20.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:20.742 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 03:16:20.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:20.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:20.742 | INFO | Uploaded 36GiB to GPU in 1.97854s (18.1952GiB/s) | 20260507 03:16:20.742 | INFO | Downloaded 34GiB to GPU in 1.95336s (17.4059GiB/s) | 20260507 03:16:25.888 | INFO | Analyzing chunk frames = [6663680,6664192) | 20260507 03:16:25.888 | INFO | MemInfo: { | 20260507 03:16:25.888 | INFO | "gpu" : " 17 GB", | 20260507 03:16:25.888 | INFO | "huge" : "", | 20260507 03:16:25.888 | INFO | "huge_shared" : " 27 GB", | 20260507 03:16:25.888 | INFO | "malloc" : " 14 GB", | 20260507 03:16:25.888 | INFO | "pinned" : " 29 GB" | 20260507 03:16:25.888 | INFO | } | 20260507 03:16:25.888 | INFO | Perf report for the last 5.14613 seconds: | 20260507 03:16:25.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:16:25.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:25.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:25.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:25.888 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 03:16:25.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:25.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:25.888 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:16:25.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:25.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:25.889 | INFO | Uploaded 36GiB to GPU in 1.96176s (18.3509GiB/s) | 20260507 03:16:25.889 | INFO | Downloaded 34GiB to GPU in 1.95838s (17.3613GiB/s) | 20260507 03:16:25.985 | INFO | Starting a round of Baz IO | 20260507 03:16:29.094 | INFO | Populating RTMetrics report | 20260507 03:16:29.094 | INFO | RTMetrics report populated | 20260507 03:16:29.094 | INFO | Opening tmp RTMetrics json file | 20260507 03:16:29.095 | INFO | Serializing RTMetrics to json | 20260507 03:16:29.104 | INFO | Serialization done, swapping tmp file now... | 20260507 03:16:29.114 | INFO | RTMetrics done | 20260507 03:16:29.185 | INFO | Finished a round of Baz IO: Wrote 1.76 GiB in 3.199 seconds | 20260507 03:16:29.185 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.91% / 0.00% / 67.09% | 20260507 03:16:29.185 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.205882% / 0% / 0.000103315% | 20260507 03:16:29.185 | INFO | Baz IO overall: | 20260507 03:16:29.185 | INFO | BytesWritten : 3424033305927 | 20260507 03:16:29.185 | INFO | NumEvents : 1645513260098 | 20260507 03:16:31.028 | INFO | Analyzing chunk frames = [6664192,6664704) | 20260507 03:16:31.028 | INFO | MemInfo: { | 20260507 03:16:31.028 | INFO | "gpu" : " 17 GB", | 20260507 03:16:31.028 | INFO | "huge" : "", | 20260507 03:16:31.028 | INFO | "huge_shared" : " 27 GB", | 20260507 03:16:31.028 | INFO | "malloc" : " 14 GB", | 20260507 03:16:31.028 | INFO | "pinned" : " 29 GB" | 20260507 03:16:31.028 | INFO | } | 20260507 03:16:31.028 | INFO | Perf report for the last 5.13957 seconds: | 20260507 03:16:31.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:16:31.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:31.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:31.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:31.028 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 03:16:31.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:31.028 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 03:16:31.028 | INFO | PRE_HQ : 59.2%, 1.8, 55.5ms, 0 | 20260507 03:16:31.028 | INFO | BAZWRITER : 61.8%, 3.3, 104.9ms, 0 | 20260507 03:16:31.028 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:16:31.028 | INFO | Uploaded 36GiB to GPU in 2.03562s (17.6851GiB/s) | 20260507 03:16:31.028 | INFO | Downloaded 38GiB to GPU in 2.18953s (17.3553GiB/s) | 20260507 03:16:36.167 | INFO | Analyzing chunk frames = [6664704,6665216) | 20260507 03:16:36.167 | INFO | MemInfo: { | 20260507 03:16:36.167 | INFO | "gpu" : " 17 GB", | 20260507 03:16:36.167 | INFO | "huge" : "", | 20260507 03:16:36.167 | INFO | "huge_shared" : " 27 GB", | 20260507 03:16:36.167 | INFO | "malloc" : " 14 GB", | 20260507 03:16:36.167 | INFO | "pinned" : " 29 GB" | 20260507 03:16:36.168 | INFO | } | 20260507 03:16:36.168 | INFO | Perf report for the last 5.13974 seconds: | 20260507 03:16:36.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:16:36.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:36.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:36.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:36.168 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:16:36.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:36.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:36.168 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:16:36.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:36.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:36.168 | INFO | Uploaded 36GiB to GPU in 2.04225s (17.6277GiB/s) | 20260507 03:16:36.168 | INFO | Downloaded 34GiB to GPU in 1.94163s (17.5111GiB/s) | 20260507 03:16:41.303 | INFO | Analyzing chunk frames = [6665216,6665728) | 20260507 03:16:41.303 | INFO | MemInfo: { | 20260507 03:16:41.303 | INFO | "gpu" : " 17 GB", | 20260507 03:16:41.303 | INFO | "huge" : "", | 20260507 03:16:41.303 | INFO | "huge_shared" : " 27 GB", | 20260507 03:16:41.303 | INFO | "malloc" : " 14 GB", | 20260507 03:16:41.303 | INFO | "pinned" : " 29 GB" | 20260507 03:16:41.303 | INFO | } | 20260507 03:16:41.303 | INFO | Perf report for the last 5.13576 seconds: | 20260507 03:16:41.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:16:41.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:41.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:41.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:41.303 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 03:16:41.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:41.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:41.303 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:16:41.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:41.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:41.303 | INFO | Uploaded 36GiB to GPU in 2.02709s (17.7595GiB/s) | 20260507 03:16:41.304 | INFO | Downloaded 34GiB to GPU in 1.95393s (17.4008GiB/s) | 20260507 03:16:46.441 | INFO | Analyzing chunk frames = [6665728,6666240) | 20260507 03:16:46.441 | INFO | MemInfo: { | 20260507 03:16:46.441 | INFO | "gpu" : " 17 GB", | 20260507 03:16:46.441 | INFO | "huge" : "", | 20260507 03:16:46.441 | INFO | "huge_shared" : " 27 GB", | 20260507 03:16:46.441 | INFO | "malloc" : " 14 GB", | 20260507 03:16:46.441 | INFO | "pinned" : " 29 GB" | 20260507 03:16:46.441 | INFO | } | 20260507 03:16:46.441 | INFO | Perf report for the last 5.13765 seconds: | 20260507 03:16:46.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:16:46.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:46.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:46.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:46.441 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 03:16:46.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:46.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:46.441 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:16:46.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:46.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:46.441 | INFO | Uploaded 36GiB to GPU in 1.96185s (18.3501GiB/s) | 20260507 03:16:46.441 | INFO | Downloaded 34GiB to GPU in 1.92328s (17.6781GiB/s) | 20260507 03:16:51.583 | INFO | Analyzing chunk frames = [6666240,6666752) | 20260507 03:16:51.583 | INFO | MemInfo: { | 20260507 03:16:51.583 | INFO | "gpu" : " 17 GB", | 20260507 03:16:51.583 | INFO | "huge" : "", | 20260507 03:16:51.583 | INFO | "huge_shared" : " 27 GB", | 20260507 03:16:51.583 | INFO | "malloc" : " 14 GB", | 20260507 03:16:51.583 | INFO | "pinned" : " 29 GB" | 20260507 03:16:51.583 | INFO | } | 20260507 03:16:51.583 | INFO | Perf report for the last 5.1418 seconds: | 20260507 03:16:51.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:16:51.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:51.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:51.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:51.583 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 03:16:51.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:51.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:51.583 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:16:51.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:51.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:51.583 | INFO | Uploaded 36GiB to GPU in 1.99805s (18.0176GiB/s) | 20260507 03:16:51.583 | INFO | Downloaded 34GiB to GPU in 1.94975s (17.4381GiB/s) | 20260507 03:16:56.722 | INFO | Analyzing chunk frames = [6666752,6667264) | 20260507 03:16:56.722 | INFO | MemInfo: { | 20260507 03:16:56.722 | INFO | "gpu" : " 17 GB", | 20260507 03:16:56.722 | INFO | "huge" : "", | 20260507 03:16:56.722 | INFO | "huge_shared" : " 27 GB", | 20260507 03:16:56.722 | INFO | "malloc" : " 14 GB", | 20260507 03:16:56.722 | INFO | "pinned" : " 29 GB" | 20260507 03:16:56.722 | INFO | } | 20260507 03:16:56.722 | INFO | Perf report for the last 5.13913 seconds: | 20260507 03:16:56.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:16:56.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:56.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:56.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:56.722 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:16:56.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:56.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:56.722 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 03:16:56.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:56.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:16:56.722 | INFO | Uploaded 36GiB to GPU in 2.04985s (17.5623GiB/s) | 20260507 03:16:56.722 | INFO | Downloaded 34GiB to GPU in 1.97217s (17.2399GiB/s) | 20260507 03:17:01.858 | INFO | Analyzing chunk frames = [6667264,6667776) | 20260507 03:17:01.858 | INFO | MemInfo: { | 20260507 03:17:01.858 | INFO | "gpu" : " 17 GB", | 20260507 03:17:01.858 | INFO | "huge" : "", | 20260507 03:17:01.858 | INFO | "huge_shared" : " 27 GB", | 20260507 03:17:01.858 | INFO | "malloc" : " 14 GB", | 20260507 03:17:01.858 | INFO | "pinned" : " 29 GB" | 20260507 03:17:01.858 | INFO | } | 20260507 03:17:01.858 | INFO | Perf report for the last 5.13637 seconds: | 20260507 03:17:01.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:17:01.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:01.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:01.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:01.858 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 03:17:01.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:01.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:01.858 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 03:17:01.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:01.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:01.858 | INFO | Uploaded 36GiB to GPU in 1.97606s (18.2181GiB/s) | 20260507 03:17:01.858 | INFO | Downloaded 34GiB to GPU in 1.97427s (17.2216GiB/s) | 20260507 03:17:06.993 | INFO | Analyzing chunk frames = [6667776,6668288) | 20260507 03:17:06.993 | INFO | MemInfo: { | 20260507 03:17:06.993 | INFO | "gpu" : " 17 GB", | 20260507 03:17:06.993 | INFO | "huge" : "", | 20260507 03:17:06.993 | INFO | "huge_shared" : " 27 GB", | 20260507 03:17:06.993 | INFO | "malloc" : " 14 GB", | 20260507 03:17:06.993 | INFO | "pinned" : " 29 GB" | 20260507 03:17:06.993 | INFO | } | 20260507 03:17:06.993 | INFO | Perf report for the last 5.13502 seconds: | 20260507 03:17:06.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:17:06.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:06.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:06.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:06.993 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:17:06.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:06.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:06.993 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:17:06.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:06.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:06.994 | INFO | Uploaded 36GiB to GPU in 1.95809s (18.3853GiB/s) | 20260507 03:17:06.994 | INFO | Downloaded 34GiB to GPU in 1.96921s (17.2658GiB/s) | 20260507 03:17:06.994 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:17:06.994 | INFO | Total mem usage: 18 GiB | 20260507 03:17:06.994 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:17:06.994 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 03:17:06.994 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:17:06.994 | INFO | High Water Breakdown: | 20260507 03:17:06.994 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:17:06.994 | INFO | 39.8% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:17:06.994 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:17:06.994 | INFO | ------------------------------------------ | 20260507 03:17:06.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:17:06.994 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:17:06.994 | INFO | Total mem usage: 26.5 GiB | 20260507 03:17:06.994 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:17:06.994 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:17:06.994 | INFO | Unused Cache: 1.78 GiB (Waste: 6.7%) | 20260507 03:17:06.994 | INFO | High Water Breakdown: | 20260507 03:17:06.994 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:17:06.994 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:17:06.994 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:17:06.994 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:17:06.994 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:17:06.994 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:17:06.994 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:17:06.994 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:17:06.994 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:17:06.994 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:17:06.994 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:17:06.994 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:17:06.994 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:17:06.994 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:17:06.994 | INFO | ------------------------------------------ | 20260507 03:17:06.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:17:06.994 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:17:06.994 | INFO | Total mem usage: 25.6 GiB | 20260507 03:17:06.994 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:17:06.994 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:17:06.994 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:17:06.994 | INFO | High Water Breakdown: | 20260507 03:17:06.994 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:17:06.994 | INFO | ------------------------------------------ | 20260507 03:17:06.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:17:06.994 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:17:06.994 | INFO | Total mem usage: 15.7 GiB | 20260507 03:17:06.994 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:17:06.994 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:17:06.994 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:17:06.994 | INFO | High Water Breakdown: | 20260507 03:17:06.994 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:17:06.994 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:17:06.994 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:17:06.994 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:17:06.994 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:17:06.994 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:17:06.994 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:17:06.994 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:17:06.994 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:17:06.994 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:17:06.994 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:17:06.994 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:17:06.994 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:17:06.994 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:17:06.994 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:17:06.994 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:17:06.994 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:17:06.994 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:17:06.994 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:17:06.994 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:17:06.994 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:17:06.994 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:17:06.994 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:17:06.994 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:17:06.994 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:17:06.994 | INFO | ------------------------------------------ | 20260507 03:17:06.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:17:06.994 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:17:06.994 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:17:06.994 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:17:06.994 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.52e+01, 1.34e+01 / 1.77e+00 | 20260507 03:17:06.994 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.03e-01, 1.02e-02 / 5.81e-03 | 20260507 03:17:06.994 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.46e+00, 5.20e-01 / 2.52e-01 | 20260507 03:17:06.994 | INFO | PerfReport: 3.23%, Histogram -- 9.94e-01 / 1.47e+00, 1.69e-01 / 9.80e-02 | 20260507 03:17:06.994 | INFO | PerfReport: 16.15%, DME -- 3.98e+01 / 4.47e+01, 4.08e+00 / 2.10e+00 | 20260507 03:17:06.994 | INFO | PerfReport: 57.58%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.29e+00 / 1.26e+00 | 20260507 03:17:06.994 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.15e+00 / 2.69e+00, 3.83e-01 / 2.14e-01 | 20260507 03:17:06.994 | INFO | PerfReport: 3.06%, Metrics -- 9.40e-01 / 1.23e+00, 2.48e-01 / 1.28e-01 | 20260507 03:17:06.994 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.18e-01 / 3.67e-01, 5.01e-02 / 2.02e-02 | 20260507 03:17:06.995 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:17:06.995 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:17:06.995 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:17:06.995 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 03:17:06.995 | INFO | PerfReport: 21.74%, Upload -- 2.09e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260507 03:17:06.995 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 03:17:06.995 | INFO | PerfReport: 31.94%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 03:17:06.995 | INFO | PerfReport: 1.45%, LockUpload -- 1.40e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 03:17:06.995 | INFO | PerfReport: 1.36%, LockDownload -- 1.31e+00 / 0.00e+00, 3.15e+00 / 0.00e+00 | 20260507 03:17:06.995 | INFO | PerfReport: 22.13%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 03:17:06.995 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260507 03:17:07.090 | INFO | Starting a round of Baz IO | 20260507 03:17:10.205 | INFO | Populating RTMetrics report | 20260507 03:17:10.205 | INFO | RTMetrics report populated | 20260507 03:17:10.205 | INFO | Opening tmp RTMetrics json file | 20260507 03:17:10.224 | INFO | Serializing RTMetrics to json | 20260507 03:17:10.225 | INFO | Serialization done, swapping tmp file now... | 20260507 03:17:10.226 | INFO | RTMetrics done | 20260507 03:17:10.298 | INFO | Finished a round of Baz IO: Wrote 1.759 GiB in 3.208 seconds | 20260507 03:17:10.298 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.91% / 0.00% / 67.09% | 20260507 03:17:10.299 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.205239% / 3.97364e-06% / 6.35783e-05% | 20260507 03:17:10.299 | INFO | Baz IO overall: | 20260507 03:17:10.299 | INFO | BytesWritten : 3425922274067 | 20260507 03:17:10.299 | INFO | NumEvents : 1646404574383 | 20260507 03:17:12.139 | INFO | Analyzing chunk frames = [6668288,6668800) | 20260507 03:17:12.139 | INFO | MemInfo: { | 20260507 03:17:12.139 | INFO | "gpu" : " 17 GB", | 20260507 03:17:12.139 | INFO | "huge" : "", | 20260507 03:17:12.139 | INFO | "huge_shared" : " 27 GB", | 20260507 03:17:12.139 | INFO | "malloc" : " 14 GB", | 20260507 03:17:12.139 | INFO | "pinned" : " 29 GB" | 20260507 03:17:12.139 | INFO | } | 20260507 03:17:12.139 | INFO | Perf report for the last 5.14568 seconds: | 20260507 03:17:12.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:17:12.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:12.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:12.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:12.139 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260507 03:17:12.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:12.139 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 03:17:12.139 | INFO | PRE_HQ : 59.2%, 1.8, 55.2ms, 0 | 20260507 03:17:12.139 | INFO | BAZWRITER : 62.0%, 3.3, 105.4ms, 0 | 20260507 03:17:12.139 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:17:12.139 | INFO | Uploaded 36GiB to GPU in 2.02501s (17.7777GiB/s) | 20260507 03:17:12.139 | INFO | Downloaded 38GiB to GPU in 2.17942s (17.4358GiB/s) | 20260507 03:17:17.289 | INFO | Analyzing chunk frames = [6668800,6669312) | 20260507 03:17:17.289 | INFO | MemInfo: { | 20260507 03:17:17.289 | INFO | "gpu" : " 17 GB", | 20260507 03:17:17.289 | INFO | "huge" : "", | 20260507 03:17:17.289 | INFO | "huge_shared" : " 27 GB", | 20260507 03:17:17.289 | INFO | "malloc" : " 14 GB", | 20260507 03:17:17.289 | INFO | "pinned" : " 29 GB" | 20260507 03:17:17.289 | INFO | } | 20260507 03:17:17.289 | INFO | Perf report for the last 5.15015 seconds: | 20260507 03:17:17.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:17:17.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:17.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:17.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:17.289 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260507 03:17:17.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:17.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:17.289 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:17:17.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:17.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:17.289 | INFO | Uploaded 36GiB to GPU in 2.03375s (17.7013GiB/s) | 20260507 03:17:17.289 | INFO | Downloaded 34GiB to GPU in 1.94353s (17.4939GiB/s) | 20260507 03:17:22.422 | INFO | Analyzing chunk frames = [6669312,6669824) | 20260507 03:17:22.422 | INFO | MemInfo: { | 20260507 03:17:22.422 | INFO | "gpu" : " 17 GB", | 20260507 03:17:22.422 | INFO | "huge" : "", | 20260507 03:17:22.422 | INFO | "huge_shared" : " 27 GB", | 20260507 03:17:22.422 | INFO | "malloc" : " 14 GB", | 20260507 03:17:22.422 | INFO | "pinned" : " 29 GB" | 20260507 03:17:22.423 | INFO | } | 20260507 03:17:22.423 | INFO | Perf report for the last 5.13336 seconds: | 20260507 03:17:22.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:17:22.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:22.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:22.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:22.423 | INFO | ANALYSIS : 60.7%, 3.0, 93.8ms, 0 | 20260507 03:17:22.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:22.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:22.423 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:17:22.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:22.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:22.423 | INFO | Uploaded 36GiB to GPU in 2.04207s (17.6291GiB/s) | 20260507 03:17:22.423 | INFO | Downloaded 34GiB to GPU in 1.95307s (17.4085GiB/s) | 20260507 03:17:27.556 | INFO | Analyzing chunk frames = [6669824,6670336) | 20260507 03:17:27.556 | INFO | MemInfo: { | 20260507 03:17:27.556 | INFO | "gpu" : " 17 GB", | 20260507 03:17:27.556 | INFO | "huge" : "", | 20260507 03:17:27.556 | INFO | "huge_shared" : " 27 GB", | 20260507 03:17:27.556 | INFO | "malloc" : " 14 GB", | 20260507 03:17:27.556 | INFO | "pinned" : " 29 GB" | 20260507 03:17:27.556 | INFO | } | 20260507 03:17:27.556 | INFO | Perf report for the last 5.13337 seconds: | 20260507 03:17:27.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:17:27.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:27.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:27.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:27.556 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 03:17:27.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:27.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:27.556 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:17:27.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:27.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:27.556 | INFO | Uploaded 36GiB to GPU in 2.01554s (17.8613GiB/s) | 20260507 03:17:27.556 | INFO | Downloaded 34GiB to GPU in 1.94079s (17.5187GiB/s) | 20260507 03:17:32.695 | INFO | Analyzing chunk frames = [6670336,6670848) | 20260507 03:17:32.695 | INFO | MemInfo: { | 20260507 03:17:32.695 | INFO | "gpu" : " 17 GB", | 20260507 03:17:32.695 | INFO | "huge" : "", | 20260507 03:17:32.695 | INFO | "huge_shared" : " 27 GB", | 20260507 03:17:32.695 | INFO | "malloc" : " 14 GB", | 20260507 03:17:32.695 | INFO | "pinned" : " 29 GB" | 20260507 03:17:32.695 | INFO | } | 20260507 03:17:32.695 | INFO | Perf report for the last 5.13924 seconds: | 20260507 03:17:32.695 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:17:32.695 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:32.695 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:32.695 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:32.695 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 03:17:32.695 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:32.695 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:32.695 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 03:17:32.695 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:32.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:32.696 | INFO | Uploaded 36GiB to GPU in 2.01149s (17.8972GiB/s) | 20260507 03:17:32.696 | INFO | Downloaded 34GiB to GPU in 1.95124s (17.4248GiB/s) | 20260507 03:17:37.834 | INFO | Analyzing chunk frames = [6670848,6671360) | 20260507 03:17:37.834 | INFO | MemInfo: { | 20260507 03:17:37.834 | INFO | "gpu" : " 17 GB", | 20260507 03:17:37.834 | INFO | "huge" : "", | 20260507 03:17:37.834 | INFO | "huge_shared" : " 27 GB", | 20260507 03:17:37.834 | INFO | "malloc" : " 14 GB", | 20260507 03:17:37.834 | INFO | "pinned" : " 29 GB" | 20260507 03:17:37.834 | INFO | } | 20260507 03:17:37.835 | INFO | Perf report for the last 5.13932 seconds: | 20260507 03:17:37.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:17:37.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:37.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:37.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:37.835 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:17:37.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:37.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:37.835 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 03:17:37.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:37.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:37.835 | INFO | Uploaded 36GiB to GPU in 2.03485s (17.6918GiB/s) | 20260507 03:17:37.835 | INFO | Downloaded 34GiB to GPU in 1.94876s (17.447GiB/s) | 20260507 03:17:42.977 | INFO | Analyzing chunk frames = [6671360,6671872) | 20260507 03:17:42.977 | INFO | MemInfo: { | 20260507 03:17:42.977 | INFO | "gpu" : " 17 GB", | 20260507 03:17:42.977 | INFO | "huge" : "", | 20260507 03:17:42.977 | INFO | "huge_shared" : " 27 GB", | 20260507 03:17:42.977 | INFO | "malloc" : " 14 GB", | 20260507 03:17:42.977 | INFO | "pinned" : " 29 GB" | 20260507 03:17:42.977 | INFO | } | 20260507 03:17:42.977 | INFO | Perf report for the last 5.1428 seconds: | 20260507 03:17:42.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:17:42.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:42.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:42.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:42.977 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 03:17:42.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:42.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:42.977 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:17:42.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:42.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:42.978 | INFO | Uploaded 36GiB to GPU in 1.96975s (18.2764GiB/s) | 20260507 03:17:42.978 | INFO | Downloaded 34GiB to GPU in 1.9788s (17.1822GiB/s) | 20260507 03:17:48.116 | INFO | Analyzing chunk frames = [6671872,6672384) | 20260507 03:17:48.116 | INFO | MemInfo: { | 20260507 03:17:48.116 | INFO | "gpu" : " 17 GB", | 20260507 03:17:48.116 | INFO | "huge" : "", | 20260507 03:17:48.116 | INFO | "huge_shared" : " 27 GB", | 20260507 03:17:48.116 | INFO | "malloc" : " 14 GB", | 20260507 03:17:48.116 | INFO | "pinned" : " 29 GB" | 20260507 03:17:48.116 | INFO | } | 20260507 03:17:48.116 | INFO | Perf report for the last 5.1391 seconds: | 20260507 03:17:48.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:17:48.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:48.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:48.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:48.116 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:17:48.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:48.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:48.117 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260507 03:17:48.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:48.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:48.117 | INFO | Uploaded 36GiB to GPU in 1.99029s (18.0878GiB/s) | 20260507 03:17:48.117 | INFO | Downloaded 34GiB to GPU in 1.9686s (17.2711GiB/s) | 20260507 03:17:48.226 | INFO | Starting a round of Baz IO | 20260507 03:17:51.321 | INFO | Populating RTMetrics report | 20260507 03:17:51.321 | INFO | RTMetrics report populated | 20260507 03:17:51.321 | INFO | Opening tmp RTMetrics json file | 20260507 03:17:51.322 | INFO | Serializing RTMetrics to json | 20260507 03:17:51.334 | INFO | Serialization done, swapping tmp file now... | 20260507 03:17:51.342 | INFO | RTMetrics done | 20260507 03:17:51.417 | INFO | Finished a round of Baz IO: Wrote 1.759 GiB in 3.19 seconds | 20260507 03:17:51.417 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.91% / 0.00% / 67.09% | 20260507 03:17:51.417 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.205616% / 0% / 7.15256e-05% | 20260507 03:17:51.417 | INFO | Baz IO overall: | 20260507 03:17:51.417 | INFO | BytesWritten : 3427811452921 | 20260507 03:17:51.417 | INFO | NumEvents : 1647296143756 | 20260507 03:17:53.253 | INFO | Analyzing chunk frames = [6672384,6672896) | 20260507 03:17:53.253 | INFO | MemInfo: { | 20260507 03:17:53.253 | INFO | "gpu" : " 17 GB", | 20260507 03:17:53.253 | INFO | "huge" : "", | 20260507 03:17:53.253 | INFO | "huge_shared" : " 27 GB", | 20260507 03:17:53.253 | INFO | "malloc" : " 14 GB", | 20260507 03:17:53.253 | INFO | "pinned" : " 29 GB" | 20260507 03:17:53.253 | INFO | } | 20260507 03:17:53.253 | INFO | Perf report for the last 5.13662 seconds: | 20260507 03:17:53.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:17:53.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:53.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:53.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:53.253 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 03:17:53.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:53.253 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 03:17:53.253 | INFO | PRE_HQ : 59.3%, 1.8, 55.9ms, 0 | 20260507 03:17:53.253 | INFO | BAZWRITER : 62.0%, 3.3, 106.0ms, 0 | 20260507 03:17:53.253 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:17:53.253 | INFO | Uploaded 36GiB to GPU in 2.05055s (17.5563GiB/s) | 20260507 03:17:53.253 | INFO | Downloaded 38GiB to GPU in 2.17836s (17.4443GiB/s) | 20260507 03:17:58.398 | INFO | Analyzing chunk frames = [6672896,6673408) | 20260507 03:17:58.398 | INFO | MemInfo: { | 20260507 03:17:58.398 | INFO | "gpu" : " 17 GB", | 20260507 03:17:58.398 | INFO | "huge" : "", | 20260507 03:17:58.398 | INFO | "huge_shared" : " 27 GB", | 20260507 03:17:58.398 | INFO | "malloc" : " 14 GB", | 20260507 03:17:58.398 | INFO | "pinned" : " 29 GB" | 20260507 03:17:58.398 | INFO | } | 20260507 03:17:58.398 | INFO | Perf report for the last 5.14516 seconds: | 20260507 03:17:58.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:17:58.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:58.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:58.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:58.398 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 03:17:58.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:58.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:58.398 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:17:58.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:58.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:17:58.398 | INFO | Uploaded 36GiB to GPU in 2.04124s (17.6363GiB/s) | 20260507 03:17:58.399 | INFO | Downloaded 34GiB to GPU in 1.93954s (17.5299GiB/s) | 20260507 03:18:03.531 | INFO | Analyzing chunk frames = [6673408,6673920) | 20260507 03:18:03.531 | INFO | MemInfo: { | 20260507 03:18:03.531 | INFO | "gpu" : " 17 GB", | 20260507 03:18:03.531 | INFO | "huge" : "", | 20260507 03:18:03.532 | INFO | "huge_shared" : " 27 GB", | 20260507 03:18:03.532 | INFO | "malloc" : " 14 GB", | 20260507 03:18:03.532 | INFO | "pinned" : " 29 GB" | 20260507 03:18:03.532 | INFO | } | 20260507 03:18:03.532 | INFO | Perf report for the last 5.13335 seconds: | 20260507 03:18:03.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:18:03.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:03.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:03.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:03.532 | INFO | ANALYSIS : 60.7%, 3.0, 93.8ms, 0 | 20260507 03:18:03.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:03.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:03.532 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:18:03.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:03.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:03.532 | INFO | Uploaded 36GiB to GPU in 2.03856s (17.6595GiB/s) | 20260507 03:18:03.532 | INFO | Downloaded 34GiB to GPU in 1.95219s (17.4163GiB/s) | 20260507 03:18:08.673 | INFO | Analyzing chunk frames = [6673920,6674432) | 20260507 03:18:08.673 | INFO | MemInfo: { | 20260507 03:18:08.673 | INFO | "gpu" : " 17 GB", | 20260507 03:18:08.673 | INFO | "huge" : "", | 20260507 03:18:08.673 | INFO | "huge_shared" : " 27 GB", | 20260507 03:18:08.673 | INFO | "malloc" : " 14 GB", | 20260507 03:18:08.673 | INFO | "pinned" : " 29 GB" | 20260507 03:18:08.673 | INFO | } | 20260507 03:18:08.673 | INFO | Perf report for the last 5.14103 seconds: | 20260507 03:18:08.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:18:08.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:08.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:08.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:08.673 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 03:18:08.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:08.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:08.673 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:18:08.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:08.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:08.673 | INFO | Uploaded 36GiB to GPU in 1.96282s (18.3409GiB/s) | 20260507 03:18:08.673 | INFO | Downloaded 34GiB to GPU in 1.89221s (17.9684GiB/s) | 20260507 03:18:13.810 | INFO | Analyzing chunk frames = [6674432,6674944) | 20260507 03:18:13.810 | INFO | MemInfo: { | 20260507 03:18:13.810 | INFO | "gpu" : " 17 GB", | 20260507 03:18:13.810 | INFO | "huge" : "", | 20260507 03:18:13.810 | INFO | "huge_shared" : " 27 GB", | 20260507 03:18:13.810 | INFO | "malloc" : " 14 GB", | 20260507 03:18:13.810 | INFO | "pinned" : " 29 GB" | 20260507 03:18:13.810 | INFO | } | 20260507 03:18:13.810 | INFO | Perf report for the last 5.13764 seconds: | 20260507 03:18:13.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:18:13.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:13.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:13.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:13.810 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 03:18:13.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:13.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:13.810 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:18:13.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:13.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:13.810 | INFO | Uploaded 36GiB to GPU in 2.01904s (17.8303GiB/s) | 20260507 03:18:13.811 | INFO | Downloaded 34GiB to GPU in 1.94727s (17.4604GiB/s) | 20260507 03:18:18.952 | INFO | Analyzing chunk frames = [6674944,6675456) | 20260507 03:18:18.952 | INFO | MemInfo: { | 20260507 03:18:18.952 | INFO | "gpu" : " 17 GB", | 20260507 03:18:18.952 | INFO | "huge" : "", | 20260507 03:18:18.952 | INFO | "huge_shared" : " 27 GB", | 20260507 03:18:18.952 | INFO | "malloc" : " 14 GB", | 20260507 03:18:18.952 | INFO | "pinned" : " 29 GB" | 20260507 03:18:18.952 | INFO | } | 20260507 03:18:18.952 | INFO | Perf report for the last 5.14171 seconds: | 20260507 03:18:18.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:18:18.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:18.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:18.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:18.952 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:18:18.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:18.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:18.952 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:18:18.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:18.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:18.952 | INFO | Uploaded 36GiB to GPU in 2.04755s (17.582GiB/s) | 20260507 03:18:18.952 | INFO | Downloaded 34GiB to GPU in 1.9478s (17.4556GiB/s) | 20260507 03:18:24.093 | INFO | Analyzing chunk frames = [6675456,6675968) | 20260507 03:18:24.093 | INFO | MemInfo: { | 20260507 03:18:24.093 | INFO | "gpu" : " 17 GB", | 20260507 03:18:24.093 | INFO | "huge" : "", | 20260507 03:18:24.093 | INFO | "huge_shared" : " 27 GB", | 20260507 03:18:24.093 | INFO | "malloc" : " 14 GB", | 20260507 03:18:24.093 | INFO | "pinned" : " 29 GB" | 20260507 03:18:24.093 | INFO | } | 20260507 03:18:24.093 | INFO | Perf report for the last 5.14093 seconds: | 20260507 03:18:24.093 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:18:24.093 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:24.093 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:24.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:24.093 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:18:24.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:24.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:24.093 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 03:18:24.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:24.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:24.093 | INFO | Uploaded 36GiB to GPU in 1.97035s (18.2709GiB/s) | 20260507 03:18:24.093 | INFO | Downloaded 34GiB to GPU in 1.95995s (17.3474GiB/s) | 20260507 03:18:29.236 | INFO | Analyzing chunk frames = [6675968,6676480) | 20260507 03:18:29.236 | INFO | MemInfo: { | 20260507 03:18:29.236 | INFO | "gpu" : " 17 GB", | 20260507 03:18:29.236 | INFO | "huge" : "", | 20260507 03:18:29.236 | INFO | "huge_shared" : " 27 GB", | 20260507 03:18:29.236 | INFO | "malloc" : " 14 GB", | 20260507 03:18:29.236 | INFO | "pinned" : " 29 GB" | 20260507 03:18:29.236 | INFO | } | 20260507 03:18:29.236 | INFO | Perf report for the last 5.14279 seconds: | 20260507 03:18:29.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:18:29.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:29.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:29.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:29.236 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 03:18:29.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:29.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:29.236 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:18:29.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:29.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:29.236 | INFO | Uploaded 36GiB to GPU in 1.96359s (18.3338GiB/s) | 20260507 03:18:29.236 | INFO | Downloaded 34GiB to GPU in 1.94985s (17.4372GiB/s) | 20260507 03:18:29.236 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:18:29.236 | INFO | Total mem usage: 18 GiB | 20260507 03:18:29.236 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:18:29.236 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 03:18:29.236 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:18:29.236 | INFO | High Water Breakdown: | 20260507 03:18:29.236 | INFO | 60.2% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:18:29.236 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:18:29.237 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:18:29.237 | INFO | ------------------------------------------ | 20260507 03:18:29.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:18:29.237 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:18:29.237 | INFO | Total mem usage: 26.5 GiB | 20260507 03:18:29.237 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:18:29.237 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:18:29.237 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:18:29.237 | INFO | High Water Breakdown: | 20260507 03:18:29.237 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:18:29.237 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:18:29.237 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:18:29.237 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:18:29.237 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:18:29.237 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:18:29.237 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:18:29.237 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:18:29.237 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:18:29.237 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:18:29.237 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:18:29.237 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:18:29.237 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:18:29.237 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:18:29.237 | INFO | ------------------------------------------ | 20260507 03:18:29.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:18:29.237 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:18:29.237 | INFO | Total mem usage: 25.6 GiB | 20260507 03:18:29.237 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:18:29.237 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:18:29.237 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:18:29.237 | INFO | High Water Breakdown: | 20260507 03:18:29.237 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:18:29.237 | INFO | ------------------------------------------ | 20260507 03:18:29.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:18:29.237 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:18:29.237 | INFO | Total mem usage: 15.7 GiB | 20260507 03:18:29.237 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:18:29.237 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:18:29.237 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:18:29.237 | INFO | High Water Breakdown: | 20260507 03:18:29.237 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:18:29.237 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:18:29.237 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:18:29.237 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:18:29.237 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:18:29.237 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:18:29.237 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:18:29.237 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:18:29.237 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:18:29.237 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:18:29.237 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:18:29.237 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:18:29.237 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:18:29.237 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:18:29.237 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:18:29.237 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:18:29.237 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:18:29.237 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:18:29.237 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:18:29.237 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:18:29.237 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:18:29.237 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:18:29.237 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:18:29.237 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:18:29.237 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:18:29.237 | INFO | ------------------------------------------ | 20260507 03:18:29.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:18:29.238 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:18:29.238 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:18:29.238 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:18:29.238 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.55e+01, 1.35e+01 / 2.10e+00 | 20260507 03:18:29.238 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.01e-01, 9.59e-03 / 5.62e-03 | 20260507 03:18:29.238 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.48e+00, 5.13e-01 / 2.04e-01 | 20260507 03:18:29.238 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.49e+00, 1.65e-01 / 9.72e-02 | 20260507 03:18:29.238 | INFO | PerfReport: 16.23%, DME -- 3.99e+01 / 4.54e+01, 4.13e+00 / 2.40e+00 | 20260507 03:18:29.238 | INFO | PerfReport: 57.50%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.21e+00 / 1.58e+00 | 20260507 03:18:29.238 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.15e+00 / 2.71e+00, 3.86e-01 / 2.33e-01 | 20260507 03:18:29.238 | INFO | PerfReport: 3.05%, Metrics -- 9.39e-01 / 1.25e+00, 2.49e-01 / 1.27e-01 | 20260507 03:18:29.238 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.18e-01 / 3.70e-01, 4.97e-02 / 2.31e-02 | 20260507 03:18:29.238 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:18:29.238 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:18:29.238 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:18:29.238 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 03:18:29.238 | INFO | PerfReport: 21.77%, Upload -- 2.10e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260507 03:18:29.238 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260507 03:18:29.238 | INFO | PerfReport: 31.92%, Compute -- 3.08e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 03:18:29.238 | INFO | PerfReport: 1.48%, LockUpload -- 1.43e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 03:18:29.238 | INFO | PerfReport: 1.31%, LockDownload -- 1.27e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260507 03:18:29.238 | INFO | PerfReport: 22.13%, LockCompute -- 2.13e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 03:18:29.238 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260507 03:18:29.333 | INFO | Starting a round of Baz IO | 20260507 03:18:32.435 | INFO | Populating RTMetrics report | 20260507 03:18:32.435 | INFO | RTMetrics report populated | 20260507 03:18:32.435 | INFO | Opening tmp RTMetrics json file | 20260507 03:18:32.436 | INFO | Serializing RTMetrics to json | 20260507 03:18:32.437 | INFO | Serialization done, swapping tmp file now... | 20260507 03:18:32.438 | INFO | RTMetrics done | 20260507 03:18:32.531 | INFO | Finished a round of Baz IO: Wrote 1.758 GiB in 3.197 seconds | 20260507 03:18:32.531 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.91% / 0.00% / 67.09% | 20260507 03:18:32.531 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.205553% / 0% / 8.74201e-05% | 20260507 03:18:32.531 | INFO | Baz IO overall: | 20260507 03:18:32.531 | INFO | BytesWritten : 3429699428431 | 20260507 03:18:32.531 | INFO | NumEvents : 1648186435370 | 20260507 03:18:34.367 | INFO | Analyzing chunk frames = [6676480,6676992) | 20260507 03:18:34.367 | INFO | MemInfo: { | 20260507 03:18:34.367 | INFO | "gpu" : " 17 GB", | 20260507 03:18:34.367 | INFO | "huge" : "", | 20260507 03:18:34.367 | INFO | "huge_shared" : " 27 GB", | 20260507 03:18:34.367 | INFO | "malloc" : " 14 GB", | 20260507 03:18:34.367 | INFO | "pinned" : " 29 GB" | 20260507 03:18:34.367 | INFO | } | 20260507 03:18:34.367 | INFO | Perf report for the last 5.13115 seconds: | 20260507 03:18:34.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:18:34.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:34.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:34.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:34.367 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260507 03:18:34.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:34.367 | INFO | RT_METRICS : 41.5%, 1.0, 21.7ms, 0 | 20260507 03:18:34.367 | INFO | PRE_HQ : 59.2%, 1.8, 55.1ms, 0 | 20260507 03:18:34.367 | INFO | BAZWRITER : 61.9%, 3.3, 105.0ms, 0 | 20260507 03:18:34.367 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:18:34.367 | INFO | Uploaded 36GiB to GPU in 2.06219s (17.4572GiB/s) | 20260507 03:18:34.367 | INFO | Downloaded 38GiB to GPU in 2.18101s (17.4231GiB/s) | 20260507 03:18:39.501 | INFO | Analyzing chunk frames = [6676992,6677504) | 20260507 03:18:39.501 | INFO | MemInfo: { | 20260507 03:18:39.501 | INFO | "gpu" : " 17 GB", | 20260507 03:18:39.501 | INFO | "huge" : "", | 20260507 03:18:39.501 | INFO | "huge_shared" : " 27 GB", | 20260507 03:18:39.501 | INFO | "malloc" : " 14 GB", | 20260507 03:18:39.501 | INFO | "pinned" : " 29 GB" | 20260507 03:18:39.501 | INFO | } | 20260507 03:18:39.501 | INFO | Perf report for the last 5.1341 seconds: | 20260507 03:18:39.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:18:39.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:39.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:39.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:39.501 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 03:18:39.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:39.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:39.501 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:18:39.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:39.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:39.501 | INFO | Uploaded 36GiB to GPU in 2.04966s (17.5639GiB/s) | 20260507 03:18:39.501 | INFO | Downloaded 34GiB to GPU in 1.94316s (17.4972GiB/s) | 20260507 03:18:44.653 | INFO | Analyzing chunk frames = [6677504,6678016) | 20260507 03:18:44.653 | INFO | MemInfo: { | 20260507 03:18:44.653 | INFO | "gpu" : " 17 GB", | 20260507 03:18:44.653 | INFO | "huge" : "", | 20260507 03:18:44.653 | INFO | "huge_shared" : " 27 GB", | 20260507 03:18:44.653 | INFO | "malloc" : " 14 GB", | 20260507 03:18:44.653 | INFO | "pinned" : " 29 GB" | 20260507 03:18:44.653 | INFO | } | 20260507 03:18:44.653 | INFO | Perf report for the last 5.15205 seconds: | 20260507 03:18:44.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:18:44.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:44.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:44.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:44.653 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 03:18:44.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:44.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:44.653 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 03:18:44.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:44.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:44.653 | INFO | Uploaded 36GiB to GPU in 1.98308s (18.1536GiB/s) | 20260507 03:18:44.653 | INFO | Downloaded 34GiB to GPU in 1.93105s (17.607GiB/s) | 20260507 03:18:49.788 | INFO | Analyzing chunk frames = [6678016,6678528) | 20260507 03:18:49.788 | INFO | MemInfo: { | 20260507 03:18:49.788 | INFO | "gpu" : " 17 GB", | 20260507 03:18:49.788 | INFO | "huge" : "", | 20260507 03:18:49.788 | INFO | "huge_shared" : " 27 GB", | 20260507 03:18:49.788 | INFO | "malloc" : " 14 GB", | 20260507 03:18:49.788 | INFO | "pinned" : " 29 GB" | 20260507 03:18:49.788 | INFO | } | 20260507 03:18:49.789 | INFO | Perf report for the last 5.13539 seconds: | 20260507 03:18:49.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:18:49.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:49.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:49.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:49.789 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 03:18:49.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:49.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:49.789 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:18:49.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:49.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:49.789 | INFO | Uploaded 36GiB to GPU in 1.99235s (18.0691GiB/s) | 20260507 03:18:49.789 | INFO | Downloaded 34GiB to GPU in 1.91983s (17.7099GiB/s) | 20260507 03:18:54.931 | INFO | Analyzing chunk frames = [6678528,6679040) | 20260507 03:18:54.931 | INFO | MemInfo: { | 20260507 03:18:54.931 | INFO | "gpu" : " 17 GB", | 20260507 03:18:54.931 | INFO | "huge" : "", | 20260507 03:18:54.931 | INFO | "huge_shared" : " 27 GB", | 20260507 03:18:54.931 | INFO | "malloc" : " 14 GB", | 20260507 03:18:54.931 | INFO | "pinned" : " 29 GB" | 20260507 03:18:54.931 | INFO | } | 20260507 03:18:54.931 | INFO | Perf report for the last 5.14294 seconds: | 20260507 03:18:54.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:18:54.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:54.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:54.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:54.932 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 03:18:54.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:54.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:54.932 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 03:18:54.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:54.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:18:54.932 | INFO | Uploaded 36GiB to GPU in 2.03258s (17.7114GiB/s) | 20260507 03:18:54.932 | INFO | Downloaded 34GiB to GPU in 1.93907s (17.5342GiB/s) | 20260507 03:19:00.065 | INFO | Analyzing chunk frames = [6679040,6679552) | 20260507 03:19:00.065 | INFO | MemInfo: { | 20260507 03:19:00.065 | INFO | "gpu" : " 17 GB", | 20260507 03:19:00.065 | INFO | "huge" : "", | 20260507 03:19:00.065 | INFO | "huge_shared" : " 27 GB", | 20260507 03:19:00.066 | INFO | "malloc" : " 14 GB", | 20260507 03:19:00.066 | INFO | "pinned" : " 29 GB" | 20260507 03:19:00.066 | INFO | } | 20260507 03:19:00.066 | INFO | Perf report for the last 5.13414 seconds: | 20260507 03:19:00.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:19:00.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:00.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:00.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:00.066 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 03:19:00.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:00.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:00.066 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 03:19:00.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:00.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:00.066 | INFO | Uploaded 36GiB to GPU in 2.03957s (17.6508GiB/s) | 20260507 03:19:00.066 | INFO | Downloaded 34GiB to GPU in 1.91114s (17.7904GiB/s) | 20260507 03:19:05.208 | INFO | Analyzing chunk frames = [6679552,6680064) | 20260507 03:19:05.208 | INFO | MemInfo: { | 20260507 03:19:05.208 | INFO | "gpu" : " 17 GB", | 20260507 03:19:05.208 | INFO | "huge" : "", | 20260507 03:19:05.208 | INFO | "huge_shared" : " 27 GB", | 20260507 03:19:05.208 | INFO | "malloc" : " 14 GB", | 20260507 03:19:05.208 | INFO | "pinned" : " 29 GB" | 20260507 03:19:05.208 | INFO | } | 20260507 03:19:05.208 | INFO | Perf report for the last 5.14247 seconds: | 20260507 03:19:05.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:19:05.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:05.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:05.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:05.208 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:19:05.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:05.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:05.208 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:19:05.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:05.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:05.209 | INFO | Uploaded 36GiB to GPU in 1.96635s (18.3081GiB/s) | 20260507 03:19:05.209 | INFO | Downloaded 34GiB to GPU in 1.95568s (17.3853GiB/s) | 20260507 03:19:10.342 | INFO | Analyzing chunk frames = [6680064,6680576) | 20260507 03:19:10.342 | INFO | MemInfo: { | 20260507 03:19:10.342 | INFO | "gpu" : " 17 GB", | 20260507 03:19:10.342 | INFO | "huge" : "", | 20260507 03:19:10.342 | INFO | "huge_shared" : " 27 GB", | 20260507 03:19:10.342 | INFO | "malloc" : " 14 GB", | 20260507 03:19:10.342 | INFO | "pinned" : " 29 GB" | 20260507 03:19:10.343 | INFO | } | 20260507 03:19:10.343 | INFO | Perf report for the last 5.13456 seconds: | 20260507 03:19:10.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:19:10.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:10.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:10.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:10.343 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:19:10.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:10.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:10.343 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:19:10.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:10.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:10.343 | INFO | Uploaded 36GiB to GPU in 1.97327s (18.2439GiB/s) | 20260507 03:19:10.343 | INFO | Downloaded 34GiB to GPU in 1.9775s (17.1935GiB/s) | 20260507 03:19:10.439 | INFO | Starting a round of Baz IO | 20260507 03:19:13.549 | INFO | Populating RTMetrics report | 20260507 03:19:13.550 | INFO | RTMetrics report populated | 20260507 03:19:13.550 | INFO | Opening tmp RTMetrics json file | 20260507 03:19:13.559 | INFO | Serializing RTMetrics to json | 20260507 03:19:13.566 | INFO | Serialization done, swapping tmp file now... | 20260507 03:19:13.566 | INFO | RTMetrics done | 20260507 03:19:13.641 | INFO | Finished a round of Baz IO: Wrote 1.76 GiB in 3.202 seconds | 20260507 03:19:13.641 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.91% / 0.00% / 67.09% | 20260507 03:19:13.641 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.20562% / 0% / 7.94729e-05% | 20260507 03:19:13.641 | INFO | Baz IO overall: | 20260507 03:19:13.641 | INFO | BytesWritten : 3431588681835 | 20260507 03:19:13.641 | INFO | NumEvents : 1649077683604 | 20260507 03:19:15.483 | INFO | Analyzing chunk frames = [6680576,6681088) | 20260507 03:19:15.483 | INFO | MemInfo: { | 20260507 03:19:15.483 | INFO | "gpu" : " 17 GB", | 20260507 03:19:15.483 | INFO | "huge" : "", | 20260507 03:19:15.483 | INFO | "huge_shared" : " 27 GB", | 20260507 03:19:15.483 | INFO | "malloc" : " 14 GB", | 20260507 03:19:15.483 | INFO | "pinned" : " 29 GB" | 20260507 03:19:15.483 | INFO | } | 20260507 03:19:15.484 | INFO | Perf report for the last 5.14084 seconds: | 20260507 03:19:15.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:19:15.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:15.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:15.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:15.484 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 03:19:15.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:15.484 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:19:15.484 | INFO | PRE_HQ : 59.6%, 1.8, 55.8ms, 0 | 20260507 03:19:15.484 | INFO | BAZWRITER : 61.8%, 3.3, 105.0ms, 0 | 20260507 03:19:15.484 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:19:15.484 | INFO | Uploaded 36GiB to GPU in 2.04277s (17.6231GiB/s) | 20260507 03:19:15.484 | INFO | Downloaded 38GiB to GPU in 2.18446s (17.3956GiB/s) | 20260507 03:19:20.624 | INFO | Analyzing chunk frames = [6681088,6681600) | 20260507 03:19:20.624 | INFO | MemInfo: { | 20260507 03:19:20.624 | INFO | "gpu" : " 17 GB", | 20260507 03:19:20.624 | INFO | "huge" : "", | 20260507 03:19:20.624 | INFO | "huge_shared" : " 27 GB", | 20260507 03:19:20.624 | INFO | "malloc" : " 14 GB", | 20260507 03:19:20.624 | INFO | "pinned" : " 29 GB" | 20260507 03:19:20.624 | INFO | } | 20260507 03:19:20.624 | INFO | Perf report for the last 5.14082 seconds: | 20260507 03:19:20.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:19:20.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:20.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:20.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:20.625 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:19:20.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:20.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:20.625 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:19:20.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:20.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:20.625 | INFO | Uploaded 36GiB to GPU in 2.03558s (17.6853GiB/s) | 20260507 03:19:20.625 | INFO | Downloaded 34GiB to GPU in 1.93809s (17.5431GiB/s) | 20260507 03:19:25.761 | INFO | Analyzing chunk frames = [6681600,6682112) | 20260507 03:19:25.761 | INFO | MemInfo: { | 20260507 03:19:25.761 | INFO | "gpu" : " 17 GB", | 20260507 03:19:25.761 | INFO | "huge" : "", | 20260507 03:19:25.761 | INFO | "huge_shared" : " 27 GB", | 20260507 03:19:25.761 | INFO | "malloc" : " 14 GB", | 20260507 03:19:25.761 | INFO | "pinned" : " 29 GB" | 20260507 03:19:25.761 | INFO | } | 20260507 03:19:25.761 | INFO | Perf report for the last 5.13683 seconds: | 20260507 03:19:25.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:19:25.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:25.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:25.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:25.761 | INFO | ANALYSIS : 60.3%, 3.0, 93.2ms, 0 | 20260507 03:19:25.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:25.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:25.761 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:19:25.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:25.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:25.761 | INFO | Uploaded 36GiB to GPU in 2.01738s (17.8449GiB/s) | 20260507 03:19:25.762 | INFO | Downloaded 34GiB to GPU in 1.93736s (17.5497GiB/s) | 20260507 03:19:30.906 | INFO | Analyzing chunk frames = [6682112,6682624) | 20260507 03:19:30.906 | INFO | MemInfo: { | 20260507 03:19:30.906 | INFO | "gpu" : " 17 GB", | 20260507 03:19:30.906 | INFO | "huge" : "", | 20260507 03:19:30.906 | INFO | "huge_shared" : " 27 GB", | 20260507 03:19:30.906 | INFO | "malloc" : " 14 GB", | 20260507 03:19:30.906 | INFO | "pinned" : " 29 GB" | 20260507 03:19:30.906 | INFO | } | 20260507 03:19:30.906 | INFO | Perf report for the last 5.14508 seconds: | 20260507 03:19:30.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:19:30.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:30.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:30.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:30.906 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 03:19:30.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:30.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:30.906 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:19:30.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:30.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:30.907 | INFO | Uploaded 36GiB to GPU in 1.99629s (18.0335GiB/s) | 20260507 03:19:30.907 | INFO | Downloaded 34GiB to GPU in 1.91638s (17.7418GiB/s) | 20260507 03:19:36.041 | INFO | Analyzing chunk frames = [6682624,6683136) | 20260507 03:19:36.041 | INFO | MemInfo: { | 20260507 03:19:36.041 | INFO | "gpu" : " 17 GB", | 20260507 03:19:36.041 | INFO | "huge" : "", | 20260507 03:19:36.041 | INFO | "huge_shared" : " 27 GB", | 20260507 03:19:36.041 | INFO | "malloc" : " 14 GB", | 20260507 03:19:36.041 | INFO | "pinned" : " 29 GB" | 20260507 03:19:36.041 | INFO | } | 20260507 03:19:36.041 | INFO | Perf report for the last 5.13484 seconds: | 20260507 03:19:36.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:19:36.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:36.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:36.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:36.041 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 03:19:36.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:36.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:36.041 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:19:36.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:36.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:36.041 | INFO | Uploaded 36GiB to GPU in 2.02817s (17.75GiB/s) | 20260507 03:19:36.041 | INFO | Downloaded 34GiB to GPU in 1.93804s (17.5435GiB/s) | 20260507 03:19:41.177 | INFO | Analyzing chunk frames = [6683136,6683648) | 20260507 03:19:41.177 | INFO | MemInfo: { | 20260507 03:19:41.177 | INFO | "gpu" : " 17 GB", | 20260507 03:19:41.177 | INFO | "huge" : "", | 20260507 03:19:41.177 | INFO | "huge_shared" : " 27 GB", | 20260507 03:19:41.178 | INFO | "malloc" : " 14 GB", | 20260507 03:19:41.178 | INFO | "pinned" : " 29 GB" | 20260507 03:19:41.178 | INFO | } | 20260507 03:19:41.178 | INFO | Perf report for the last 5.13641 seconds: | 20260507 03:19:41.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:19:41.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:41.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:41.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:41.178 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 03:19:41.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:41.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:41.178 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 03:19:41.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:41.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:41.178 | INFO | Uploaded 36GiB to GPU in 2.03404s (17.6988GiB/s) | 20260507 03:19:41.178 | INFO | Downloaded 34GiB to GPU in 1.9332s (17.5874GiB/s) | 20260507 03:19:46.313 | INFO | Analyzing chunk frames = [6683648,6684160) | 20260507 03:19:46.313 | INFO | MemInfo: { | 20260507 03:19:46.313 | INFO | "gpu" : " 17 GB", | 20260507 03:19:46.313 | INFO | "huge" : "", | 20260507 03:19:46.313 | INFO | "huge_shared" : " 27 GB", | 20260507 03:19:46.313 | INFO | "malloc" : " 14 GB", | 20260507 03:19:46.313 | INFO | "pinned" : " 29 GB" | 20260507 03:19:46.314 | INFO | } | 20260507 03:19:46.314 | INFO | Perf report for the last 5.13614 seconds: | 20260507 03:19:46.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:19:46.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:46.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:46.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:46.314 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:19:46.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:46.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:46.314 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 03:19:46.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:46.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:46.314 | INFO | Uploaded 36GiB to GPU in 1.9777s (18.2029GiB/s) | 20260507 03:19:46.314 | INFO | Downloaded 34GiB to GPU in 1.9637s (17.3142GiB/s) | 20260507 03:19:51.458 | INFO | Analyzing chunk frames = [6684160,6684672) | 20260507 03:19:51.458 | INFO | MemInfo: { | 20260507 03:19:51.458 | INFO | "gpu" : " 17 GB", | 20260507 03:19:51.458 | INFO | "huge" : "", | 20260507 03:19:51.458 | INFO | "huge_shared" : " 27 GB", | 20260507 03:19:51.458 | INFO | "malloc" : " 14 GB", | 20260507 03:19:51.458 | INFO | "pinned" : " 29 GB" | 20260507 03:19:51.458 | INFO | } | 20260507 03:19:51.458 | INFO | Perf report for the last 5.14433 seconds: | 20260507 03:19:51.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:19:51.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:51.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:51.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:51.458 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:19:51.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:51.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:51.458 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 03:19:51.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:51.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:51.459 | INFO | Uploaded 36GiB to GPU in 1.97023s (18.272GiB/s) | 20260507 03:19:51.459 | INFO | Downloaded 34GiB to GPU in 1.96421s (17.3098GiB/s) | 20260507 03:19:51.459 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:19:51.459 | INFO | Total mem usage: 18 GiB | 20260507 03:19:51.459 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:19:51.459 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:19:51.459 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:19:51.459 | INFO | High Water Breakdown: | 20260507 03:19:51.459 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:19:51.459 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:19:51.459 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:19:51.459 | INFO | ------------------------------------------ | 20260507 03:19:51.459 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:19:51.459 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:19:51.459 | INFO | Total mem usage: 26.5 GiB | 20260507 03:19:51.459 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:19:51.459 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:19:51.459 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:19:51.459 | INFO | High Water Breakdown: | 20260507 03:19:51.459 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:19:51.459 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:19:51.459 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:19:51.459 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:19:51.459 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:19:51.459 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:19:51.459 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:19:51.459 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:19:51.459 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:19:51.459 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:19:51.459 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:19:51.459 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:19:51.459 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:19:51.459 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:19:51.459 | INFO | ------------------------------------------ | 20260507 03:19:51.459 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:19:51.459 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:19:51.459 | INFO | Total mem usage: 25.6 GiB | 20260507 03:19:51.459 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:19:51.459 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:19:51.459 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:19:51.459 | INFO | High Water Breakdown: | 20260507 03:19:51.459 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:19:51.459 | INFO | ------------------------------------------ | 20260507 03:19:51.459 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:19:51.459 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:19:51.459 | INFO | Total mem usage: 15.7 GiB | 20260507 03:19:51.459 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:19:51.459 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:19:51.459 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:19:51.459 | INFO | High Water Breakdown: | 20260507 03:19:51.459 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:19:51.459 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:19:51.459 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:19:51.459 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:19:51.459 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:19:51.459 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:19:51.459 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:19:51.459 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:19:51.459 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:19:51.459 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:19:51.459 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:19:51.459 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:19:51.459 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:19:51.459 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:19:51.459 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:19:51.459 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:19:51.459 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:19:51.459 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:19:51.459 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:19:51.459 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:19:51.460 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:19:51.460 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:19:51.460 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:19:51.460 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:19:51.460 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:19:51.460 | INFO | ------------------------------------------ | 20260507 03:19:51.460 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:19:51.460 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:19:51.460 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:19:51.460 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:19:51.460 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.47e+01, 1.33e+01 / 1.36e+00 | 20260507 03:19:51.460 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 9.63e-03 / 4.70e-03 | 20260507 03:19:51.460 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.49e+00, 5.21e-01 / 2.26e-01 | 20260507 03:19:51.460 | INFO | PerfReport: 3.23%, Histogram -- 9.95e-01 / 1.47e+00, 1.71e-01 / 1.02e-01 | 20260507 03:19:51.460 | INFO | PerfReport: 16.08%, DME -- 3.96e+01 / 4.39e+01, 3.85e+00 / 1.47e+00 | 20260507 03:19:51.460 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.77e+01 / 2.05e+01, 1.30e+00 / 1.42e+00 | 20260507 03:19:51.460 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.15e+00 / 2.68e+00, 3.80e-01 / 2.07e-01 | 20260507 03:19:51.460 | INFO | PerfReport: 3.07%, Metrics -- 9.43e-01 / 1.24e+00, 2.52e-01 / 1.53e-01 | 20260507 03:19:51.460 | INFO | PerfReport: 1.04%, PulseSerializing -- 3.19e-01 / 3.70e-01, 5.01e-02 / 2.12e-02 | 20260507 03:19:51.460 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:19:51.460 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:19:51.460 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:19:51.460 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 03:19:51.460 | INFO | PerfReport: 21.78%, Upload -- 2.10e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260507 03:19:51.460 | INFO | PerfReport: 21.33%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260507 03:19:51.460 | INFO | PerfReport: 31.96%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 03:19:51.460 | INFO | PerfReport: 1.48%, LockUpload -- 1.42e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 03:19:51.460 | INFO | PerfReport: 1.27%, LockDownload -- 1.22e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260507 03:19:51.460 | INFO | PerfReport: 22.17%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 03:19:51.460 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.36e-02 / 0.00e+00 | 20260507 03:19:51.556 | INFO | Starting a round of Baz IO | 20260507 03:19:54.669 | INFO | Populating RTMetrics report | 20260507 03:19:54.669 | INFO | RTMetrics report populated | 20260507 03:19:54.669 | INFO | Opening tmp RTMetrics json file | 20260507 03:19:54.688 | INFO | Serializing RTMetrics to json | 20260507 03:19:54.689 | INFO | Serialization done, swapping tmp file now... | 20260507 03:19:54.690 | INFO | RTMetrics done | 20260507 03:19:54.764 | INFO | Finished a round of Baz IO: Wrote 1.76 GiB in 3.207 seconds | 20260507 03:19:54.764 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.90% / 0.00% / 67.09% | 20260507 03:19:54.764 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.205708% / 3.97364e-06% / 8.74201e-05% | 20260507 03:19:54.764 | INFO | Baz IO overall: | 20260507 03:19:54.764 | INFO | BytesWritten : 3433477988823 | 20260507 03:19:54.764 | INFO | NumEvents : 1649969002078 | 20260507 03:19:56.611 | INFO | Analyzing chunk frames = [6684672,6685184) | 20260507 03:19:56.611 | INFO | MemInfo: { | 20260507 03:19:56.611 | INFO | "gpu" : " 17 GB", | 20260507 03:19:56.611 | INFO | "huge" : "", | 20260507 03:19:56.611 | INFO | "huge_shared" : " 27 GB", | 20260507 03:19:56.611 | INFO | "malloc" : " 14 GB", | 20260507 03:19:56.611 | INFO | "pinned" : " 29 GB" | 20260507 03:19:56.611 | INFO | } | 20260507 03:19:56.611 | INFO | Perf report for the last 5.15322 seconds: | 20260507 03:19:56.611 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:19:56.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:56.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:56.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:56.612 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260507 03:19:56.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:19:56.612 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 03:19:56.612 | INFO | PRE_HQ : 59.2%, 1.8, 55.4ms, 0 | 20260507 03:19:56.612 | INFO | BAZWRITER : 61.9%, 3.3, 104.4ms, 0 | 20260507 03:19:56.612 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:19:56.612 | INFO | Uploaded 36GiB to GPU in 2.02593s (17.7696GiB/s) | 20260507 03:19:56.612 | INFO | Downloaded 38GiB to GPU in 2.1764s (17.46GiB/s) | 20260507 03:20:01.747 | INFO | Analyzing chunk frames = [6685184,6685696) | 20260507 03:20:01.747 | INFO | MemInfo: { | 20260507 03:20:01.747 | INFO | "gpu" : " 17 GB", | 20260507 03:20:01.747 | INFO | "huge" : "", | 20260507 03:20:01.747 | INFO | "huge_shared" : " 27 GB", | 20260507 03:20:01.747 | INFO | "malloc" : " 14 GB", | 20260507 03:20:01.747 | INFO | "pinned" : " 29 GB" | 20260507 03:20:01.748 | INFO | } | 20260507 03:20:01.748 | INFO | Perf report for the last 5.13632 seconds: | 20260507 03:20:01.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:20:01.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:01.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:01.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:01.748 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:20:01.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:01.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:01.748 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 03:20:01.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:01.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:01.748 | INFO | Uploaded 36GiB to GPU in 1.99964s (18.0032GiB/s) | 20260507 03:20:01.748 | INFO | Downloaded 34GiB to GPU in 1.93425s (17.5778GiB/s) | 20260507 03:20:06.884 | INFO | Analyzing chunk frames = [6685696,6686208) | 20260507 03:20:06.884 | INFO | MemInfo: { | 20260507 03:20:06.884 | INFO | "gpu" : " 17 GB", | 20260507 03:20:06.884 | INFO | "huge" : "", | 20260507 03:20:06.884 | INFO | "huge_shared" : " 27 GB", | 20260507 03:20:06.884 | INFO | "malloc" : " 14 GB", | 20260507 03:20:06.884 | INFO | "pinned" : " 29 GB" | 20260507 03:20:06.884 | INFO | } | 20260507 03:20:06.884 | INFO | Perf report for the last 5.1363 seconds: | 20260507 03:20:06.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:20:06.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:06.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:06.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:06.884 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 03:20:06.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:06.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:06.884 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:20:06.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:06.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:06.884 | INFO | Uploaded 36GiB to GPU in 2.04489s (17.6048GiB/s) | 20260507 03:20:06.884 | INFO | Downloaded 34GiB to GPU in 1.95169s (17.4208GiB/s) | 20260507 03:20:12.017 | INFO | Analyzing chunk frames = [6686208,6686720) | 20260507 03:20:12.017 | INFO | MemInfo: { | 20260507 03:20:12.017 | INFO | "gpu" : " 17 GB", | 20260507 03:20:12.017 | INFO | "huge" : "", | 20260507 03:20:12.017 | INFO | "huge_shared" : " 27 GB", | 20260507 03:20:12.017 | INFO | "malloc" : " 14 GB", | 20260507 03:20:12.017 | INFO | "pinned" : " 29 GB" | 20260507 03:20:12.017 | INFO | } | 20260507 03:20:12.017 | INFO | Perf report for the last 5.13285 seconds: | 20260507 03:20:12.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:20:12.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:12.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:12.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:12.017 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 03:20:12.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:12.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:12.017 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 03:20:12.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:12.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:12.017 | INFO | Uploaded 36GiB to GPU in 1.97278s (18.2483GiB/s) | 20260507 03:20:12.017 | INFO | Downloaded 34GiB to GPU in 1.93086s (17.6088GiB/s) | 20260507 03:20:17.155 | INFO | Analyzing chunk frames = [6686720,6687232) | 20260507 03:20:17.155 | INFO | MemInfo: { | 20260507 03:20:17.155 | INFO | "gpu" : " 17 GB", | 20260507 03:20:17.155 | INFO | "huge" : "", | 20260507 03:20:17.155 | INFO | "huge_shared" : " 27 GB", | 20260507 03:20:17.155 | INFO | "malloc" : " 14 GB", | 20260507 03:20:17.155 | INFO | "pinned" : " 29 GB" | 20260507 03:20:17.155 | INFO | } | 20260507 03:20:17.156 | INFO | Perf report for the last 5.1387 seconds: | 20260507 03:20:17.156 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:20:17.156 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:17.156 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:17.156 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:17.156 | INFO | ANALYSIS : 59.9%, 3.0, 93.2ms, 0 | 20260507 03:20:17.156 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:17.156 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:17.156 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:20:17.156 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:17.156 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:17.156 | INFO | Uploaded 36GiB to GPU in 1.99933s (18.006GiB/s) | 20260507 03:20:17.156 | INFO | Downloaded 34GiB to GPU in 1.94656s (17.4668GiB/s) | 20260507 03:20:22.296 | INFO | Analyzing chunk frames = [6687232,6687744) | 20260507 03:20:22.297 | INFO | MemInfo: { | 20260507 03:20:22.297 | INFO | "gpu" : " 17 GB", | 20260507 03:20:22.297 | INFO | "huge" : "", | 20260507 03:20:22.297 | INFO | "huge_shared" : " 27 GB", | 20260507 03:20:22.297 | INFO | "malloc" : " 14 GB", | 20260507 03:20:22.297 | INFO | "pinned" : " 29 GB" | 20260507 03:20:22.297 | INFO | } | 20260507 03:20:22.297 | INFO | Perf report for the last 5.14123 seconds: | 20260507 03:20:22.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:20:22.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:22.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:22.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:22.297 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 03:20:22.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:22.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:22.297 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 03:20:22.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:22.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:22.297 | INFO | Uploaded 36GiB to GPU in 2.01987s (17.823GiB/s) | 20260507 03:20:22.297 | INFO | Downloaded 34GiB to GPU in 1.93782s (17.5455GiB/s) | 20260507 03:20:27.439 | INFO | Analyzing chunk frames = [6687744,6688256) | 20260507 03:20:27.439 | INFO | MemInfo: { | 20260507 03:20:27.439 | INFO | "gpu" : " 17 GB", | 20260507 03:20:27.439 | INFO | "huge" : "", | 20260507 03:20:27.439 | INFO | "huge_shared" : " 27 GB", | 20260507 03:20:27.439 | INFO | "malloc" : " 14 GB", | 20260507 03:20:27.439 | INFO | "pinned" : " 29 GB" | 20260507 03:20:27.439 | INFO | } | 20260507 03:20:27.440 | INFO | Perf report for the last 5.14274 seconds: | 20260507 03:20:27.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:20:27.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:27.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:27.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:27.440 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:20:27.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:27.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:27.440 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 03:20:27.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:27.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:27.440 | INFO | Uploaded 36GiB to GPU in 1.96981s (18.2759GiB/s) | 20260507 03:20:27.440 | INFO | Downloaded 34GiB to GPU in 1.97142s (17.2465GiB/s) | 20260507 03:20:32.574 | INFO | Analyzing chunk frames = [6688256,6688768) | 20260507 03:20:32.574 | INFO | MemInfo: { | 20260507 03:20:32.574 | INFO | "gpu" : " 17 GB", | 20260507 03:20:32.574 | INFO | "huge" : "", | 20260507 03:20:32.574 | INFO | "huge_shared" : " 27 GB", | 20260507 03:20:32.574 | INFO | "malloc" : " 14 GB", | 20260507 03:20:32.574 | INFO | "pinned" : " 29 GB" | 20260507 03:20:32.574 | INFO | } | 20260507 03:20:32.575 | INFO | Perf report for the last 5.13533 seconds: | 20260507 03:20:32.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:20:32.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:32.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:32.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:32.575 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 03:20:32.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:32.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:32.575 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:20:32.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:32.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:32.575 | INFO | Uploaded 36GiB to GPU in 1.96838s (18.2891GiB/s) | 20260507 03:20:32.575 | INFO | Downloaded 34GiB to GPU in 1.9664s (17.2905GiB/s) | 20260507 03:20:32.672 | INFO | Starting a round of Baz IO | 20260507 03:20:35.778 | INFO | Populating RTMetrics report | 20260507 03:20:35.779 | INFO | RTMetrics report populated | 20260507 03:20:35.779 | INFO | Opening tmp RTMetrics json file | 20260507 03:20:35.779 | INFO | Serializing RTMetrics to json | 20260507 03:20:35.792 | INFO | Serialization done, swapping tmp file now... | 20260507 03:20:35.799 | INFO | RTMetrics done | 20260507 03:20:35.871 | INFO | Finished a round of Baz IO: Wrote 1.758 GiB in 3.199 seconds | 20260507 03:20:35.871 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.90% / 0.00% / 67.10% | 20260507 03:20:35.871 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.205123% / 0% / 7.94729e-05% | 20260507 03:20:35.871 | INFO | Baz IO overall: | 20260507 03:20:35.871 | INFO | BytesWritten : 3435366126883 | 20260507 03:20:35.871 | INFO | NumEvents : 1650860032514 | 20260507 03:20:37.716 | INFO | Analyzing chunk frames = [6688768,6689280) | 20260507 03:20:37.716 | INFO | MemInfo: { | 20260507 03:20:37.716 | INFO | "gpu" : " 17 GB", | 20260507 03:20:37.716 | INFO | "huge" : "", | 20260507 03:20:37.716 | INFO | "huge_shared" : " 27 GB", | 20260507 03:20:37.716 | INFO | "malloc" : " 14 GB", | 20260507 03:20:37.716 | INFO | "pinned" : " 29 GB" | 20260507 03:20:37.716 | INFO | } | 20260507 03:20:37.716 | INFO | Perf report for the last 5.14104 seconds: | 20260507 03:20:37.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:20:37.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:37.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:37.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:37.716 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 03:20:37.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:37.716 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:20:37.716 | INFO | PRE_HQ : 59.0%, 1.8, 55.4ms, 0 | 20260507 03:20:37.716 | INFO | BAZWRITER : 61.8%, 3.3, 104.9ms, 0 | 20260507 03:20:37.716 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:20:37.716 | INFO | Uploaded 36GiB to GPU in 2.03171s (17.7191GiB/s) | 20260507 03:20:37.716 | INFO | Downloaded 38GiB to GPU in 2.18061s (17.4263GiB/s) | 20260507 03:20:42.854 | INFO | Analyzing chunk frames = [6689280,6689792) | 20260507 03:20:42.854 | INFO | MemInfo: { | 20260507 03:20:42.854 | INFO | "gpu" : " 17 GB", | 20260507 03:20:42.854 | INFO | "huge" : "", | 20260507 03:20:42.854 | INFO | "huge_shared" : " 27 GB", | 20260507 03:20:42.854 | INFO | "malloc" : " 14 GB", | 20260507 03:20:42.854 | INFO | "pinned" : " 29 GB" | 20260507 03:20:42.854 | INFO | } | 20260507 03:20:42.855 | INFO | Perf report for the last 5.13858 seconds: | 20260507 03:20:42.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:20:42.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:42.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:42.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:42.855 | INFO | ANALYSIS : 60.9%, 3.0, 93.9ms, 0 | 20260507 03:20:42.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:42.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:42.855 | INFO | PRE_HQ : 24.7%, 1.0, 12.7ms, 0 | 20260507 03:20:42.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:42.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:42.855 | INFO | Uploaded 36GiB to GPU in 2.0283s (17.7489GiB/s) | 20260507 03:20:42.855 | INFO | Downloaded 34GiB to GPU in 1.93401s (17.58GiB/s) | 20260507 03:20:47.990 | INFO | Analyzing chunk frames = [6689792,6690304) | 20260507 03:20:47.990 | INFO | MemInfo: { | 20260507 03:20:47.990 | INFO | "gpu" : " 17 GB", | 20260507 03:20:47.990 | INFO | "huge" : "", | 20260507 03:20:47.991 | INFO | "huge_shared" : " 27 GB", | 20260507 03:20:47.991 | INFO | "malloc" : " 14 GB", | 20260507 03:20:47.991 | INFO | "pinned" : " 29 GB" | 20260507 03:20:47.991 | INFO | } | 20260507 03:20:47.991 | INFO | Perf report for the last 5.13614 seconds: | 20260507 03:20:47.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:20:47.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:47.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:47.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:47.991 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 03:20:47.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:47.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:47.991 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:20:47.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:47.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:47.991 | INFO | Uploaded 36GiB to GPU in 2.00887s (17.9205GiB/s) | 20260507 03:20:47.991 | INFO | Downloaded 34GiB to GPU in 1.93788s (17.5449GiB/s) | 20260507 03:20:53.129 | INFO | Analyzing chunk frames = [6690304,6690816) | 20260507 03:20:53.129 | INFO | MemInfo: { | 20260507 03:20:53.129 | INFO | "gpu" : " 17 GB", | 20260507 03:20:53.129 | INFO | "huge" : "", | 20260507 03:20:53.129 | INFO | "huge_shared" : " 27 GB", | 20260507 03:20:53.129 | INFO | "malloc" : " 14 GB", | 20260507 03:20:53.129 | INFO | "pinned" : " 29 GB" | 20260507 03:20:53.129 | INFO | } | 20260507 03:20:53.129 | INFO | Perf report for the last 5.13831 seconds: | 20260507 03:20:53.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:20:53.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:53.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:53.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:53.129 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:20:53.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:53.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:53.129 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:20:53.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:53.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:53.129 | INFO | Uploaded 36GiB to GPU in 1.98129s (18.17GiB/s) | 20260507 03:20:53.129 | INFO | Downloaded 34GiB to GPU in 1.91976s (17.7105GiB/s) | 20260507 03:20:58.268 | INFO | Analyzing chunk frames = [6690816,6691328) | 20260507 03:20:58.268 | INFO | MemInfo: { | 20260507 03:20:58.268 | INFO | "gpu" : " 17 GB", | 20260507 03:20:58.268 | INFO | "huge" : "", | 20260507 03:20:58.268 | INFO | "huge_shared" : " 27 GB", | 20260507 03:20:58.268 | INFO | "malloc" : " 14 GB", | 20260507 03:20:58.268 | INFO | "pinned" : " 29 GB" | 20260507 03:20:58.268 | INFO | } | 20260507 03:20:58.269 | INFO | Perf report for the last 5.13953 seconds: | 20260507 03:20:58.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:20:58.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:58.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:58.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:58.269 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 03:20:58.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:58.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:58.269 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:20:58.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:58.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:20:58.269 | INFO | Uploaded 36GiB to GPU in 2.02462s (17.7811GiB/s) | 20260507 03:20:58.269 | INFO | Downloaded 34GiB to GPU in 1.94556s (17.4757GiB/s) | 20260507 03:21:03.414 | INFO | Analyzing chunk frames = [6691328,6691840) | 20260507 03:21:03.414 | INFO | MemInfo: { | 20260507 03:21:03.414 | INFO | "gpu" : " 17 GB", | 20260507 03:21:03.414 | INFO | "huge" : "", | 20260507 03:21:03.414 | INFO | "huge_shared" : " 27 GB", | 20260507 03:21:03.414 | INFO | "malloc" : " 14 GB", | 20260507 03:21:03.414 | INFO | "pinned" : " 29 GB" | 20260507 03:21:03.414 | INFO | } | 20260507 03:21:03.414 | INFO | Perf report for the last 5.14586 seconds: | 20260507 03:21:03.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:21:03.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:03.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:03.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:03.414 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 03:21:03.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:03.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:03.415 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 03:21:03.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:03.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:03.415 | INFO | Uploaded 36GiB to GPU in 2.05137s (17.5492GiB/s) | 20260507 03:21:03.415 | INFO | Downloaded 34GiB to GPU in 1.95114s (17.4257GiB/s) | 20260507 03:21:08.556 | INFO | Analyzing chunk frames = [6691840,6692352) | 20260507 03:21:08.556 | INFO | MemInfo: { | 20260507 03:21:08.556 | INFO | "gpu" : " 17 GB", | 20260507 03:21:08.556 | INFO | "huge" : "", | 20260507 03:21:08.556 | INFO | "huge_shared" : " 27 GB", | 20260507 03:21:08.556 | INFO | "malloc" : " 14 GB", | 20260507 03:21:08.556 | INFO | "pinned" : " 29 GB" | 20260507 03:21:08.556 | INFO | } | 20260507 03:21:08.556 | INFO | Perf report for the last 5.14167 seconds: | 20260507 03:21:08.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:21:08.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:08.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:08.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:08.556 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:21:08.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:08.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:08.556 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 03:21:08.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:08.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:08.557 | INFO | Uploaded 36GiB to GPU in 1.94736s (18.4866GiB/s) | 20260507 03:21:08.557 | INFO | Downloaded 34GiB to GPU in 1.96972s (17.2613GiB/s) | 20260507 03:21:13.693 | INFO | Analyzing chunk frames = [6692352,6692864) | 20260507 03:21:13.693 | INFO | MemInfo: { | 20260507 03:21:13.693 | INFO | "gpu" : " 17 GB", | 20260507 03:21:13.693 | INFO | "huge" : "", | 20260507 03:21:13.693 | INFO | "huge_shared" : " 27 GB", | 20260507 03:21:13.693 | INFO | "malloc" : " 14 GB", | 20260507 03:21:13.693 | INFO | "pinned" : " 29 GB" | 20260507 03:21:13.693 | INFO | } | 20260507 03:21:13.693 | INFO | Perf report for the last 5.13704 seconds: | 20260507 03:21:13.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:21:13.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:13.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:13.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:13.693 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:21:13.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:13.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:13.693 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 03:21:13.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:13.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:13.693 | INFO | Uploaded 36GiB to GPU in 1.97942s (18.1871GiB/s) | 20260507 03:21:13.693 | INFO | Downloaded 34GiB to GPU in 1.96404s (17.3113GiB/s) | 20260507 03:21:13.694 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:21:13.694 | INFO | Total mem usage: 18 GiB | 20260507 03:21:13.694 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:21:13.694 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:21:13.694 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:21:13.694 | INFO | High Water Breakdown: | 20260507 03:21:13.694 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:21:13.694 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:21:13.694 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:21:13.694 | INFO | ------------------------------------------ | 20260507 03:21:13.694 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:21:13.694 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:21:13.694 | INFO | Total mem usage: 26.5 GiB | 20260507 03:21:13.694 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:21:13.694 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:21:13.694 | INFO | Unused Cache: 1.78 GiB (Waste: 6.7%) | 20260507 03:21:13.694 | INFO | High Water Breakdown: | 20260507 03:21:13.694 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:21:13.694 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:21:13.694 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:21:13.694 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:21:13.694 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:21:13.694 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:21:13.694 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:21:13.694 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:21:13.694 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:21:13.694 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:21:13.694 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:21:13.694 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:21:13.694 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:21:13.694 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:21:13.694 | INFO | ------------------------------------------ | 20260507 03:21:13.694 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:21:13.694 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:21:13.694 | INFO | Total mem usage: 25.6 GiB | 20260507 03:21:13.694 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:21:13.694 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:21:13.694 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:21:13.694 | INFO | High Water Breakdown: | 20260507 03:21:13.694 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:21:13.694 | INFO | ------------------------------------------ | 20260507 03:21:13.694 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:21:13.694 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:21:13.694 | INFO | Total mem usage: 15.7 GiB | 20260507 03:21:13.694 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:21:13.694 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:21:13.694 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:21:13.694 | INFO | High Water Breakdown: | 20260507 03:21:13.694 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:21:13.694 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:21:13.694 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:21:13.694 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:21:13.694 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:21:13.694 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:21:13.694 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:21:13.694 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:21:13.694 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:21:13.694 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:21:13.694 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:21:13.694 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:21:13.694 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:21:13.694 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:21:13.694 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:21:13.694 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:21:13.694 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:21:13.694 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:21:13.695 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:21:13.695 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:21:13.695 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:21:13.695 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:21:13.695 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:21:13.695 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:21:13.695 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:21:13.695 | INFO | ------------------------------------------ | 20260507 03:21:13.695 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:21:13.695 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:21:13.695 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:21:13.695 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:21:13.695 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.52e+01, 1.34e+01 / 1.61e+00 | 20260507 03:21:13.695 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 1.00e-01, 1.02e-02 / 6.07e-03 | 20260507 03:21:13.695 | INFO | PerfReport: 11.69%, Baseline -- 3.60e+00 / 4.47e+00, 5.25e-01 / 2.78e-01 | 20260507 03:21:13.695 | INFO | PerfReport: 3.23%, Histogram -- 9.96e-01 / 1.44e+00, 1.70e-01 / 1.30e-01 | 20260507 03:21:13.695 | INFO | PerfReport: 16.11%, DME -- 3.97e+01 / 4.44e+01, 3.96e+00 / 1.73e+00 | 20260507 03:21:13.695 | INFO | PerfReport: 57.60%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.30e+00 / 1.43e+00 | 20260507 03:21:13.695 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.15e+00 / 2.69e+00, 3.85e-01 / 2.24e-01 | 20260507 03:21:13.695 | INFO | PerfReport: 3.05%, Metrics -- 9.38e-01 / 1.27e+00, 2.46e-01 / 1.20e-01 | 20260507 03:21:13.695 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.18e-01 / 3.69e-01, 5.01e-02 / 2.44e-02 | 20260507 03:21:13.695 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:21:13.695 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:21:13.695 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:21:13.695 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 03:21:13.695 | INFO | PerfReport: 21.68%, Upload -- 2.09e+01 / 0.00e+00, 7.04e+00 / 0.00e+00 | 20260507 03:21:13.695 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260507 03:21:13.695 | INFO | PerfReport: 31.99%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 03:21:13.695 | INFO | PerfReport: 1.40%, LockUpload -- 1.35e+00 / 0.00e+00, 4.70e+00 / 0.00e+00 | 20260507 03:21:13.695 | INFO | PerfReport: 1.34%, LockDownload -- 1.29e+00 / 0.00e+00, 3.21e+00 / 0.00e+00 | 20260507 03:21:13.695 | INFO | PerfReport: 22.19%, LockCompute -- 2.14e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 03:21:13.695 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.22e-02 / 0.00e+00 | 20260507 03:21:13.790 | INFO | Starting a round of Baz IO | 20260507 03:21:16.895 | INFO | Populating RTMetrics report | 20260507 03:21:16.895 | INFO | RTMetrics report populated | 20260507 03:21:16.895 | INFO | Opening tmp RTMetrics json file | 20260507 03:21:16.914 | INFO | Serializing RTMetrics to json | 20260507 03:21:16.915 | INFO | Serialization done, swapping tmp file now... | 20260507 03:21:16.915 | INFO | RTMetrics done | 20260507 03:21:16.985 | INFO | Finished a round of Baz IO: Wrote 1.756 GiB in 3.195 seconds | 20260507 03:21:16.985 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.90% / 0.00% / 67.10% | 20260507 03:21:16.985 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.203808% / 0% / 0.000103315% | 20260507 03:21:16.985 | INFO | Baz IO overall: | 20260507 03:21:16.985 | INFO | BytesWritten : 3437251771175 | 20260507 03:21:16.985 | INFO | NumEvents : 1651749518515 | 20260507 03:21:18.825 | INFO | Analyzing chunk frames = [6692864,6693376) | 20260507 03:21:18.825 | INFO | MemInfo: { | 20260507 03:21:18.825 | INFO | "gpu" : " 17 GB", | 20260507 03:21:18.825 | INFO | "huge" : "", | 20260507 03:21:18.825 | INFO | "huge_shared" : " 27 GB", | 20260507 03:21:18.825 | INFO | "malloc" : " 14 GB", | 20260507 03:21:18.825 | INFO | "pinned" : " 29 GB" | 20260507 03:21:18.825 | INFO | } | 20260507 03:21:18.826 | INFO | Perf report for the last 5.13234 seconds: | 20260507 03:21:18.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:21:18.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:18.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:18.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:18.826 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 03:21:18.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:18.826 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 03:21:18.826 | INFO | PRE_HQ : 60.3%, 1.9, 59.6ms, 0 | 20260507 03:21:18.826 | INFO | BAZWRITER : 61.7%, 3.4, 108.3ms, 0 | 20260507 03:21:18.826 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:21:18.826 | INFO | Uploaded 36GiB to GPU in 2.0371s (17.6722GiB/s) | 20260507 03:21:18.826 | INFO | Downloaded 38GiB to GPU in 2.17345s (17.4837GiB/s) | 20260507 03:21:23.973 | INFO | Analyzing chunk frames = [6693376,6693888) | 20260507 03:21:23.973 | INFO | MemInfo: { | 20260507 03:21:23.973 | INFO | "gpu" : " 17 GB", | 20260507 03:21:23.973 | INFO | "huge" : "", | 20260507 03:21:23.973 | INFO | "huge_shared" : " 27 GB", | 20260507 03:21:23.973 | INFO | "malloc" : " 14 GB", | 20260507 03:21:23.973 | INFO | "pinned" : " 29 GB" | 20260507 03:21:23.973 | INFO | } | 20260507 03:21:23.973 | INFO | Perf report for the last 5.14773 seconds: | 20260507 03:21:23.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:21:23.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:23.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:23.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:23.973 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:21:23.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:23.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:23.973 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 03:21:23.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:23.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:23.974 | INFO | Uploaded 36GiB to GPU in 2.01997s (17.8221GiB/s) | 20260507 03:21:23.974 | INFO | Downloaded 34GiB to GPU in 1.93839s (17.5403GiB/s) | 20260507 03:21:29.107 | INFO | Analyzing chunk frames = [6693888,6694400) | 20260507 03:21:29.107 | INFO | MemInfo: { | 20260507 03:21:29.107 | INFO | "gpu" : " 17 GB", | 20260507 03:21:29.107 | INFO | "huge" : "", | 20260507 03:21:29.107 | INFO | "huge_shared" : " 27 GB", | 20260507 03:21:29.107 | INFO | "malloc" : " 14 GB", | 20260507 03:21:29.107 | INFO | "pinned" : " 29 GB" | 20260507 03:21:29.107 | INFO | } | 20260507 03:21:29.107 | INFO | Perf report for the last 5.13396 seconds: | 20260507 03:21:29.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:21:29.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:29.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:29.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:29.107 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 03:21:29.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:29.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:29.107 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:21:29.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:29.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:29.108 | INFO | Uploaded 36GiB to GPU in 1.97705s (18.209GiB/s) | 20260507 03:21:29.108 | INFO | Downloaded 34GiB to GPU in 1.94185s (17.5091GiB/s) | 20260507 03:21:34.252 | INFO | Analyzing chunk frames = [6694400,6694912) | 20260507 03:21:34.252 | INFO | MemInfo: { | 20260507 03:21:34.252 | INFO | "gpu" : " 17 GB", | 20260507 03:21:34.252 | INFO | "huge" : "", | 20260507 03:21:34.252 | INFO | "huge_shared" : " 27 GB", | 20260507 03:21:34.252 | INFO | "malloc" : " 14 GB", | 20260507 03:21:34.252 | INFO | "pinned" : " 29 GB" | 20260507 03:21:34.252 | INFO | } | 20260507 03:21:34.252 | INFO | Perf report for the last 5.14503 seconds: | 20260507 03:21:34.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:21:34.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:34.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:34.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:34.252 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 03:21:34.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:34.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:34.252 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:21:34.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:34.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:34.253 | INFO | Uploaded 36GiB to GPU in 1.96396s (18.3303GiB/s) | 20260507 03:21:34.253 | INFO | Downloaded 34GiB to GPU in 1.91024s (17.7988GiB/s) | 20260507 03:21:39.392 | INFO | Analyzing chunk frames = [6694912,6695424) | 20260507 03:21:39.392 | INFO | MemInfo: { | 20260507 03:21:39.392 | INFO | "gpu" : " 17 GB", | 20260507 03:21:39.392 | INFO | "huge" : "", | 20260507 03:21:39.392 | INFO | "huge_shared" : " 27 GB", | 20260507 03:21:39.392 | INFO | "malloc" : " 14 GB", | 20260507 03:21:39.392 | INFO | "pinned" : " 29 GB" | 20260507 03:21:39.392 | INFO | } | 20260507 03:21:39.392 | INFO | Perf report for the last 5.13979 seconds: | 20260507 03:21:39.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:21:39.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:39.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:39.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:39.392 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 03:21:39.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:39.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:39.392 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:21:39.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:39.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:39.392 | INFO | Uploaded 36GiB to GPU in 2.04377s (17.6145GiB/s) | 20260507 03:21:39.392 | INFO | Downloaded 34GiB to GPU in 1.9465s (17.4673GiB/s) | 20260507 03:21:44.525 | INFO | Analyzing chunk frames = [6695424,6695936) | 20260507 03:21:44.525 | INFO | MemInfo: { | 20260507 03:21:44.525 | INFO | "gpu" : " 17 GB", | 20260507 03:21:44.525 | INFO | "huge" : "", | 20260507 03:21:44.526 | INFO | "huge_shared" : " 27 GB", | 20260507 03:21:44.526 | INFO | "malloc" : " 14 GB", | 20260507 03:21:44.526 | INFO | "pinned" : " 29 GB" | 20260507 03:21:44.526 | INFO | } | 20260507 03:21:44.526 | INFO | Perf report for the last 5.13368 seconds: | 20260507 03:21:44.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:21:44.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:44.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:44.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:44.526 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 03:21:44.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:44.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:44.526 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:21:44.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:44.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:44.526 | INFO | Uploaded 36GiB to GPU in 2.02996s (17.7343GiB/s) | 20260507 03:21:44.526 | INFO | Downloaded 34GiB to GPU in 1.92048s (17.7039GiB/s) | 20260507 03:21:49.667 | INFO | Analyzing chunk frames = [6695936,6696448) | 20260507 03:21:49.667 | INFO | MemInfo: { | 20260507 03:21:49.667 | INFO | "gpu" : " 17 GB", | 20260507 03:21:49.667 | INFO | "huge" : "", | 20260507 03:21:49.667 | INFO | "huge_shared" : " 27 GB", | 20260507 03:21:49.667 | INFO | "malloc" : " 14 GB", | 20260507 03:21:49.667 | INFO | "pinned" : " 29 GB" | 20260507 03:21:49.667 | INFO | } | 20260507 03:21:49.667 | INFO | Perf report for the last 5.14141 seconds: | 20260507 03:21:49.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:21:49.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:49.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:49.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:49.667 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:21:49.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:49.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:49.667 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:21:49.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:49.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:49.667 | INFO | Uploaded 36GiB to GPU in 1.98953s (18.0947GiB/s) | 20260507 03:21:49.667 | INFO | Downloaded 34GiB to GPU in 1.98489s (17.1294GiB/s) | 20260507 03:21:54.801 | INFO | Analyzing chunk frames = [6696448,6696960) | 20260507 03:21:54.801 | INFO | MemInfo: { | 20260507 03:21:54.801 | INFO | "gpu" : " 17 GB", | 20260507 03:21:54.801 | INFO | "huge" : "", | 20260507 03:21:54.801 | INFO | "huge_shared" : " 27 GB", | 20260507 03:21:54.801 | INFO | "malloc" : " 14 GB", | 20260507 03:21:54.801 | INFO | "pinned" : " 29 GB" | 20260507 03:21:54.801 | INFO | } | 20260507 03:21:54.801 | INFO | Perf report for the last 5.13367 seconds: | 20260507 03:21:54.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:21:54.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:54.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:54.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:54.801 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 03:21:54.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:54.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:54.801 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 03:21:54.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:54.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:54.801 | INFO | Uploaded 36GiB to GPU in 1.99536s (18.0418GiB/s) | 20260507 03:21:54.801 | INFO | Downloaded 34GiB to GPU in 1.97156s (17.2453GiB/s) | 20260507 03:21:54.897 | INFO | Starting a round of Baz IO | 20260507 03:21:58.004 | INFO | Populating RTMetrics report | 20260507 03:21:58.005 | INFO | RTMetrics report populated | 20260507 03:21:58.005 | INFO | Opening tmp RTMetrics json file | 20260507 03:21:58.007 | INFO | Serializing RTMetrics to json | 20260507 03:21:58.023 | INFO | Serialization done, swapping tmp file now... | 20260507 03:21:58.023 | INFO | RTMetrics done | 20260507 03:21:58.096 | INFO | Finished a round of Baz IO: Wrote 1.753 GiB in 3.198 seconds | 20260507 03:21:58.096 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.90% / 0.00% / 67.10% | 20260507 03:21:58.096 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.202986% / 7.94729e-06% / 9.13938e-05% | 20260507 03:21:58.096 | INFO | Baz IO overall: | 20260507 03:21:58.096 | INFO | BytesWritten : 3439134565232 | 20260507 03:21:58.096 | INFO | NumEvents : 1652637336506 | 20260507 03:21:59.944 | INFO | Analyzing chunk frames = [6696960,6697472) | 20260507 03:21:59.944 | INFO | MemInfo: { | 20260507 03:21:59.944 | INFO | "gpu" : " 17 GB", | 20260507 03:21:59.944 | INFO | "huge" : "", | 20260507 03:21:59.944 | INFO | "huge_shared" : " 27 GB", | 20260507 03:21:59.944 | INFO | "malloc" : " 14 GB", | 20260507 03:21:59.944 | INFO | "pinned" : " 29 GB" | 20260507 03:21:59.944 | INFO | } | 20260507 03:21:59.944 | INFO | Perf report for the last 5.14317 seconds: | 20260507 03:21:59.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:21:59.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:59.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:59.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:59.944 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 03:21:59.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:21:59.944 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 03:21:59.944 | INFO | PRE_HQ : 59.1%, 1.8, 55.3ms, 0 | 20260507 03:21:59.944 | INFO | BAZWRITER : 61.8%, 3.3, 104.8ms, 0 | 20260507 03:21:59.944 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:21:59.944 | INFO | Uploaded 36GiB to GPU in 2.04031s (17.6444GiB/s) | 20260507 03:21:59.944 | INFO | Downloaded 38GiB to GPU in 2.18463s (17.3942GiB/s) | 20260507 03:22:05.080 | INFO | Analyzing chunk frames = [6697472,6697984) | 20260507 03:22:05.080 | INFO | MemInfo: { | 20260507 03:22:05.080 | INFO | "gpu" : " 17 GB", | 20260507 03:22:05.080 | INFO | "huge" : "", | 20260507 03:22:05.080 | INFO | "huge_shared" : " 27 GB", | 20260507 03:22:05.080 | INFO | "malloc" : " 14 GB", | 20260507 03:22:05.080 | INFO | "pinned" : " 29 GB" | 20260507 03:22:05.080 | INFO | } | 20260507 03:22:05.080 | INFO | Perf report for the last 5.13587 seconds: | 20260507 03:22:05.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:22:05.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:05.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:05.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:05.080 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:22:05.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:05.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:05.080 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:22:05.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:05.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:05.080 | INFO | Uploaded 36GiB to GPU in 2.04517s (17.6024GiB/s) | 20260507 03:22:05.080 | INFO | Downloaded 34GiB to GPU in 1.93813s (17.5427GiB/s) | 20260507 03:22:10.222 | INFO | Analyzing chunk frames = [6697984,6698496) | 20260507 03:22:10.222 | INFO | MemInfo: { | 20260507 03:22:10.222 | INFO | "gpu" : " 17 GB", | 20260507 03:22:10.222 | INFO | "huge" : "", | 20260507 03:22:10.222 | INFO | "huge_shared" : " 27 GB", | 20260507 03:22:10.222 | INFO | "malloc" : " 14 GB", | 20260507 03:22:10.222 | INFO | "pinned" : " 29 GB" | 20260507 03:22:10.222 | INFO | } | 20260507 03:22:10.222 | INFO | Perf report for the last 5.142 seconds: | 20260507 03:22:10.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:22:10.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:10.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:10.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:10.222 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 03:22:10.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:10.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:10.222 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:22:10.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:10.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:10.222 | INFO | Uploaded 36GiB to GPU in 2.00232s (17.9792GiB/s) | 20260507 03:22:10.222 | INFO | Downloaded 34GiB to GPU in 1.93307s (17.5886GiB/s) | 20260507 03:22:15.363 | INFO | Analyzing chunk frames = [6698496,6699008) | 20260507 03:22:15.363 | INFO | MemInfo: { | 20260507 03:22:15.363 | INFO | "gpu" : " 17 GB", | 20260507 03:22:15.363 | INFO | "huge" : "", | 20260507 03:22:15.363 | INFO | "huge_shared" : " 27 GB", | 20260507 03:22:15.363 | INFO | "malloc" : " 14 GB", | 20260507 03:22:15.364 | INFO | "pinned" : " 29 GB" | 20260507 03:22:15.364 | INFO | } | 20260507 03:22:15.364 | INFO | Perf report for the last 5.14164 seconds: | 20260507 03:22:15.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:22:15.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:15.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:15.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:15.364 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 03:22:15.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:15.364 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:15.364 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:22:15.364 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:15.364 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:15.364 | INFO | Uploaded 36GiB to GPU in 1.98167s (18.1665GiB/s) | 20260507 03:22:15.364 | INFO | Downloaded 34GiB to GPU in 1.90723s (17.8269GiB/s) | 20260507 03:22:20.506 | INFO | Analyzing chunk frames = [6699008,6699520) | 20260507 03:22:20.506 | INFO | MemInfo: { | 20260507 03:22:20.506 | INFO | "gpu" : " 17 GB", | 20260507 03:22:20.506 | INFO | "huge" : "", | 20260507 03:22:20.506 | INFO | "huge_shared" : " 27 GB", | 20260507 03:22:20.506 | INFO | "malloc" : " 14 GB", | 20260507 03:22:20.506 | INFO | "pinned" : " 29 GB" | 20260507 03:22:20.506 | INFO | } | 20260507 03:22:20.506 | INFO | Perf report for the last 5.14246 seconds: | 20260507 03:22:20.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:22:20.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:20.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:20.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:20.506 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 03:22:20.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:20.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:20.506 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:22:20.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:20.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:20.506 | INFO | Uploaded 36GiB to GPU in 2.03577s (17.6837GiB/s) | 20260507 03:22:20.506 | INFO | Downloaded 34GiB to GPU in 1.94167s (17.5107GiB/s) | 20260507 03:22:25.639 | INFO | Analyzing chunk frames = [6699520,6700032) | 20260507 03:22:25.639 | INFO | MemInfo: { | 20260507 03:22:25.639 | INFO | "gpu" : " 17 GB", | 20260507 03:22:25.639 | INFO | "huge" : "", | 20260507 03:22:25.639 | INFO | "huge_shared" : " 27 GB", | 20260507 03:22:25.639 | INFO | "malloc" : " 14 GB", | 20260507 03:22:25.639 | INFO | "pinned" : " 29 GB" | 20260507 03:22:25.639 | INFO | } | 20260507 03:22:25.639 | INFO | Perf report for the last 5.13318 seconds: | 20260507 03:22:25.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:22:25.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:25.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:25.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:25.639 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 03:22:25.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:25.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:25.639 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 03:22:25.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:25.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:25.640 | INFO | Uploaded 36GiB to GPU in 2.04528s (17.6015GiB/s) | 20260507 03:22:25.640 | INFO | Downloaded 34GiB to GPU in 1.91015s (17.7997GiB/s) | 20260507 03:22:30.774 | INFO | Analyzing chunk frames = [6700032,6700544) | 20260507 03:22:30.774 | INFO | MemInfo: { | 20260507 03:22:30.774 | INFO | "gpu" : " 17 GB", | 20260507 03:22:30.774 | INFO | "huge" : "", | 20260507 03:22:30.775 | INFO | "huge_shared" : " 27 GB", | 20260507 03:22:30.775 | INFO | "malloc" : " 14 GB", | 20260507 03:22:30.775 | INFO | "pinned" : " 29 GB" | 20260507 03:22:30.775 | INFO | } | 20260507 03:22:30.775 | INFO | Perf report for the last 5.13534 seconds: | 20260507 03:22:30.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:22:30.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:30.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:30.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:30.775 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:22:30.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:30.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:30.775 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 03:22:30.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:30.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:30.775 | INFO | Uploaded 36GiB to GPU in 2.00983s (17.912GiB/s) | 20260507 03:22:30.775 | INFO | Downloaded 34GiB to GPU in 1.98039s (17.1684GiB/s) | 20260507 03:22:35.923 | INFO | Analyzing chunk frames = [6700544,6701056) | 20260507 03:22:35.923 | INFO | MemInfo: { | 20260507 03:22:35.923 | INFO | "gpu" : " 17 GB", | 20260507 03:22:35.923 | INFO | "huge" : "", | 20260507 03:22:35.923 | INFO | "huge_shared" : " 27 GB", | 20260507 03:22:35.923 | INFO | "malloc" : " 14 GB", | 20260507 03:22:35.923 | INFO | "pinned" : " 29 GB" | 20260507 03:22:35.923 | INFO | } | 20260507 03:22:35.923 | INFO | Perf report for the last 5.14881 seconds: | 20260507 03:22:35.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:22:35.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:35.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:35.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:35.924 | INFO | ANALYSIS : 61.0%, 3.0, 95.2ms, 0 | 20260507 03:22:35.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:35.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:35.924 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 03:22:35.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:35.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:35.924 | INFO | Uploaded 36GiB to GPU in 1.99669s (18.0299GiB/s) | 20260507 03:22:35.924 | INFO | Downloaded 34GiB to GPU in 1.98053s (17.1671GiB/s) | 20260507 03:22:35.924 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:22:35.924 | INFO | Total mem usage: 18 GiB | 20260507 03:22:35.924 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:22:35.924 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:22:35.924 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:22:35.924 | INFO | High Water Breakdown: | 20260507 03:22:35.924 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:22:35.924 | INFO | 39.7% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:22:35.924 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:22:35.924 | INFO | ------------------------------------------ | 20260507 03:22:35.924 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:22:35.924 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:22:35.924 | INFO | Total mem usage: 26.5 GiB | 20260507 03:22:35.924 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:22:35.924 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:22:35.924 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:22:35.924 | INFO | High Water Breakdown: | 20260507 03:22:35.924 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:22:35.924 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:22:35.924 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:22:35.924 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:22:35.924 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:22:35.924 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:22:35.924 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:22:35.924 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:22:35.924 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:22:35.924 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:22:35.924 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:22:35.924 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:22:35.924 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:22:35.924 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:22:35.924 | INFO | ------------------------------------------ | 20260507 03:22:35.924 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:22:35.924 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:22:35.924 | INFO | Total mem usage: 25.6 GiB | 20260507 03:22:35.924 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:22:35.924 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 03:22:35.924 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 03:22:35.924 | INFO | High Water Breakdown: | 20260507 03:22:35.924 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 03:22:35.924 | INFO | ------------------------------------------ | 20260507 03:22:35.924 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:22:35.924 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:22:35.924 | INFO | Total mem usage: 15.7 GiB | 20260507 03:22:35.924 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:22:35.924 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:22:35.924 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:22:35.924 | INFO | High Water Breakdown: | 20260507 03:22:35.924 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:22:35.924 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:22:35.924 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:22:35.924 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:22:35.924 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:22:35.924 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:22:35.924 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:22:35.924 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:22:35.924 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:22:35.924 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:22:35.924 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:22:35.924 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:22:35.924 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:22:35.924 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:22:35.925 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:22:35.925 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:22:35.925 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:22:35.925 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:22:35.925 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:22:35.925 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:22:35.925 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:22:35.925 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:22:35.925 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:22:35.925 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:22:35.925 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:22:35.925 | INFO | ------------------------------------------ | 20260507 03:22:35.925 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:22:35.925 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:22:35.925 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:22:35.925 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:22:35.925 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.52e+01, 1.35e+01 / 1.48e+00 | 20260507 03:22:35.925 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 1.02e-01, 9.33e-03 / 4.74e-03 | 20260507 03:22:35.925 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.46e+00, 5.19e-01 / 2.30e-01 | 20260507 03:22:35.925 | INFO | PerfReport: 3.23%, Histogram -- 9.94e-01 / 1.48e+00, 1.71e-01 / 8.71e-02 | 20260507 03:22:35.925 | INFO | PerfReport: 16.08%, DME -- 3.96e+01 / 4.42e+01, 3.94e+00 / 1.61e+00 | 20260507 03:22:35.925 | INFO | PerfReport: 57.68%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.37e+00 / 1.50e+00 | 20260507 03:22:35.925 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.14e+00 / 2.67e+00, 3.78e-01 / 2.31e-01 | 20260507 03:22:35.925 | INFO | PerfReport: 3.05%, Metrics -- 9.38e-01 / 1.25e+00, 2.52e-01 / 1.14e-01 | 20260507 03:22:35.925 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.18e-01 / 3.79e-01, 5.07e-02 / 3.05e-02 | 20260507 03:22:35.925 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:22:35.925 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:22:35.925 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:22:35.925 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 03:22:35.925 | INFO | PerfReport: 21.78%, Upload -- 2.10e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260507 03:22:35.925 | INFO | PerfReport: 21.34%, Download -- 2.05e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260507 03:22:35.925 | INFO | PerfReport: 31.97%, Compute -- 3.08e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 03:22:35.925 | INFO | PerfReport: 1.41%, LockUpload -- 1.36e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 03:22:35.925 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260507 03:22:35.925 | INFO | PerfReport: 22.20%, LockCompute -- 2.14e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 03:22:35.925 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.32e-02 / 0.00e+00 | 20260507 03:22:36.021 | INFO | Starting a round of Baz IO | 20260507 03:22:39.124 | INFO | Populating RTMetrics report | 20260507 03:22:39.124 | INFO | RTMetrics report populated | 20260507 03:22:39.124 | INFO | Opening tmp RTMetrics json file | 20260507 03:22:39.143 | INFO | Serializing RTMetrics to json | 20260507 03:22:39.144 | INFO | Serialization done, swapping tmp file now... | 20260507 03:22:39.144 | INFO | RTMetrics done | 20260507 03:22:39.214 | INFO | Finished a round of Baz IO: Wrote 1.753 GiB in 3.194 seconds | 20260507 03:22:39.215 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.90% / 0.00% / 67.10% | 20260507 03:22:39.215 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.202088% / 7.94729e-06% / 0.000115236% | 20260507 03:22:39.215 | INFO | Baz IO overall: | 20260507 03:22:39.215 | INFO | BytesWritten : 3441016483448 | 20260507 03:22:39.215 | INFO | NumEvents : 1653524618621 | 20260507 03:22:41.063 | INFO | Analyzing chunk frames = [6701056,6701568) | 20260507 03:22:41.063 | INFO | MemInfo: { | 20260507 03:22:41.063 | INFO | "gpu" : " 17 GB", | 20260507 03:22:41.063 | INFO | "huge" : "", | 20260507 03:22:41.063 | INFO | "huge_shared" : " 27 GB", | 20260507 03:22:41.063 | INFO | "malloc" : " 14 GB", | 20260507 03:22:41.063 | INFO | "pinned" : " 29 GB" | 20260507 03:22:41.063 | INFO | } | 20260507 03:22:41.063 | INFO | Perf report for the last 5.14002 seconds: | 20260507 03:22:41.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:22:41.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:41.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:41.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:41.064 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 03:22:41.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:41.064 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 03:22:41.064 | INFO | PRE_HQ : 59.3%, 1.8, 56.1ms, 0 | 20260507 03:22:41.064 | INFO | BAZWRITER : 61.7%, 3.3, 104.8ms, 0 | 20260507 03:22:41.064 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 03:22:41.064 | INFO | Uploaded 36GiB to GPU in 2.02505s (17.7774GiB/s) | 20260507 03:22:41.064 | INFO | Downloaded 38GiB to GPU in 2.18884s (17.3608GiB/s) | 20260507 03:22:46.199 | INFO | Analyzing chunk frames = [6701568,6702080) | 20260507 03:22:46.199 | INFO | MemInfo: { | 20260507 03:22:46.199 | INFO | "gpu" : " 17 GB", | 20260507 03:22:46.199 | INFO | "huge" : "", | 20260507 03:22:46.199 | INFO | "huge_shared" : " 27 GB", | 20260507 03:22:46.199 | INFO | "malloc" : " 14 GB", | 20260507 03:22:46.199 | INFO | "pinned" : " 29 GB" | 20260507 03:22:46.199 | INFO | } | 20260507 03:22:46.199 | INFO | Perf report for the last 5.13565 seconds: | 20260507 03:22:46.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:22:46.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:46.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:46.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:46.199 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:22:46.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:46.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:46.199 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:22:46.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:46.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:46.200 | INFO | Uploaded 36GiB to GPU in 1.99314s (18.062GiB/s) | 20260507 03:22:46.200 | INFO | Downloaded 34GiB to GPU in 1.93685s (17.5543GiB/s) | 20260507 03:22:51.337 | INFO | Analyzing chunk frames = [6702080,6702592) | 20260507 03:22:51.337 | INFO | MemInfo: { | 20260507 03:22:51.337 | INFO | "gpu" : " 17 GB", | 20260507 03:22:51.337 | INFO | "huge" : "", | 20260507 03:22:51.337 | INFO | "huge_shared" : " 27 GB", | 20260507 03:22:51.337 | INFO | "malloc" : " 14 GB", | 20260507 03:22:51.337 | INFO | "pinned" : " 29 GB" | 20260507 03:22:51.337 | INFO | } | 20260507 03:22:51.337 | INFO | Perf report for the last 5.13801 seconds: | 20260507 03:22:51.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:22:51.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:51.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:51.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:51.337 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 03:22:51.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:51.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:51.337 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 03:22:51.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:51.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:51.337 | INFO | Uploaded 36GiB to GPU in 2.01555s (17.8611GiB/s) | 20260507 03:22:51.337 | INFO | Downloaded 34GiB to GPU in 1.94167s (17.5107GiB/s) | 20260507 03:22:56.480 | INFO | Analyzing chunk frames = [6702592,6703104) | 20260507 03:22:56.480 | INFO | MemInfo: { | 20260507 03:22:56.480 | INFO | "gpu" : " 17 GB", | 20260507 03:22:56.480 | INFO | "huge" : "", | 20260507 03:22:56.480 | INFO | "huge_shared" : " 27 GB", | 20260507 03:22:56.480 | INFO | "malloc" : " 14 GB", | 20260507 03:22:56.480 | INFO | "pinned" : " 29 GB" | 20260507 03:22:56.480 | INFO | } | 20260507 03:22:56.480 | INFO | Perf report for the last 5.14313 seconds: | 20260507 03:22:56.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:22:56.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:56.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:56.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:56.480 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:22:56.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:56.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:56.480 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:22:56.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:56.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:22:56.481 | INFO | Uploaded 36GiB to GPU in 1.99045s (18.0864GiB/s) | 20260507 03:22:56.481 | INFO | Downloaded 34GiB to GPU in 1.92138s (17.6956GiB/s) | 20260507 03:23:01.613 | INFO | Analyzing chunk frames = [6703104,6703616) | 20260507 03:23:01.613 | INFO | MemInfo: { | 20260507 03:23:01.613 | INFO | "gpu" : " 17 GB", | 20260507 03:23:01.613 | INFO | "huge" : "", | 20260507 03:23:01.613 | INFO | "huge_shared" : " 27 GB", | 20260507 03:23:01.613 | INFO | "malloc" : " 14 GB", | 20260507 03:23:01.613 | INFO | "pinned" : " 29 GB" | 20260507 03:23:01.613 | INFO | } | 20260507 03:23:01.613 | INFO | Perf report for the last 5.13298 seconds: | 20260507 03:23:01.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:23:01.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:01.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:01.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:01.613 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 03:23:01.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:01.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:01.613 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:23:01.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:01.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:01.614 | INFO | Uploaded 36GiB to GPU in 2.01766s (17.8424GiB/s) | 20260507 03:23:01.614 | INFO | Downloaded 34GiB to GPU in 1.94117s (17.5152GiB/s) | 20260507 03:23:06.759 | INFO | Analyzing chunk frames = [6703616,6704128) | 20260507 03:23:06.759 | INFO | MemInfo: { | 20260507 03:23:06.759 | INFO | "gpu" : " 17 GB", | 20260507 03:23:06.759 | INFO | "huge" : "", | 20260507 03:23:06.759 | INFO | "huge_shared" : " 27 GB", | 20260507 03:23:06.759 | INFO | "malloc" : " 14 GB", | 20260507 03:23:06.759 | INFO | "pinned" : " 29 GB" | 20260507 03:23:06.759 | INFO | } | 20260507 03:23:06.759 | INFO | Perf report for the last 5.14555 seconds: | 20260507 03:23:06.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:23:06.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:06.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:06.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:06.759 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 03:23:06.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:06.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:06.759 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 03:23:06.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:06.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:06.759 | INFO | Uploaded 36GiB to GPU in 2.01176s (17.8948GiB/s) | 20260507 03:23:06.759 | INFO | Downloaded 34GiB to GPU in 1.95163s (17.4214GiB/s) | 20260507 03:23:11.892 | INFO | Analyzing chunk frames = [6704128,6704640) | 20260507 03:23:11.892 | INFO | MemInfo: { | 20260507 03:23:11.892 | INFO | "gpu" : " 17 GB", | 20260507 03:23:11.892 | INFO | "huge" : "", | 20260507 03:23:11.892 | INFO | "huge_shared" : " 27 GB", | 20260507 03:23:11.893 | INFO | "malloc" : " 14 GB", | 20260507 03:23:11.893 | INFO | "pinned" : " 29 GB" | 20260507 03:23:11.893 | INFO | } | 20260507 03:23:11.893 | INFO | Perf report for the last 5.13388 seconds: | 20260507 03:23:11.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:23:11.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:11.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:11.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:11.893 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:23:11.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:11.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:11.893 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 03:23:11.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:11.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:11.893 | INFO | Uploaded 36GiB to GPU in 1.97524s (18.2256GiB/s) | 20260507 03:23:11.893 | INFO | Downloaded 34GiB to GPU in 1.96581s (17.2957GiB/s) | 20260507 03:23:17.036 | INFO | Analyzing chunk frames = [6704640,6705152) | 20260507 03:23:17.036 | INFO | MemInfo: { | 20260507 03:23:17.036 | INFO | "gpu" : " 17 GB", | 20260507 03:23:17.036 | INFO | "huge" : "", | 20260507 03:23:17.036 | INFO | "huge_shared" : " 27 GB", | 20260507 03:23:17.036 | INFO | "malloc" : " 14 GB", | 20260507 03:23:17.036 | INFO | "pinned" : " 29 GB" | 20260507 03:23:17.036 | INFO | } | 20260507 03:23:17.037 | INFO | Perf report for the last 5.1438 seconds: | 20260507 03:23:17.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:23:17.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:17.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:17.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:17.037 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:23:17.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:17.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:17.037 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 03:23:17.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:17.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:17.037 | INFO | Uploaded 36GiB to GPU in 1.9712s (18.263GiB/s) | 20260507 03:23:17.037 | INFO | Downloaded 34GiB to GPU in 1.97091s (17.2509GiB/s) | 20260507 03:23:17.133 | INFO | Starting a round of Baz IO | 20260507 03:23:20.243 | INFO | Populating RTMetrics report | 20260507 03:23:20.243 | INFO | RTMetrics report populated | 20260507 03:23:20.243 | INFO | Opening tmp RTMetrics json file | 20260507 03:23:20.248 | INFO | Serializing RTMetrics to json | 20260507 03:23:20.261 | INFO | Serialization done, swapping tmp file now... | 20260507 03:23:20.262 | INFO | RTMetrics done | 20260507 03:23:20.331 | INFO | Finished a round of Baz IO: Wrote 1.753 GiB in 3.197 seconds | 20260507 03:23:20.331 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.90% / 0.00% / 67.10% | 20260507 03:23:20.331 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.202294% / 0% / 7.94729e-05% | 20260507 03:23:20.331 | INFO | Baz IO overall: | 20260507 03:23:20.331 | INFO | BytesWritten : 3442899050911 | 20260507 03:23:20.331 | INFO | NumEvents : 1654412631092 | 20260507 03:23:22.177 | INFO | Analyzing chunk frames = [6705152,6705664) | 20260507 03:23:22.177 | INFO | MemInfo: { | 20260507 03:23:22.177 | INFO | "gpu" : " 17 GB", | 20260507 03:23:22.177 | INFO | "huge" : "", | 20260507 03:23:22.177 | INFO | "huge_shared" : " 27 GB", | 20260507 03:23:22.177 | INFO | "malloc" : " 14 GB", | 20260507 03:23:22.177 | INFO | "pinned" : " 29 GB" | 20260507 03:23:22.177 | INFO | } | 20260507 03:23:22.177 | INFO | Perf report for the last 5.14024 seconds: | 20260507 03:23:22.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:23:22.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:22.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:22.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:22.177 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 03:23:22.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:22.177 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:23:22.177 | INFO | PRE_HQ : 59.2%, 1.8, 55.4ms, 0 | 20260507 03:23:22.177 | INFO | BAZWRITER : 61.7%, 3.3, 104.2ms, 0 | 20260507 03:23:22.177 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:23:22.177 | INFO | Uploaded 36GiB to GPU in 2.03726s (17.6708GiB/s) | 20260507 03:23:22.177 | INFO | Downloaded 38GiB to GPU in 2.18107s (17.4227GiB/s) | 20260507 03:23:27.315 | INFO | Analyzing chunk frames = [6705664,6706176) | 20260507 03:23:27.316 | INFO | MemInfo: { | 20260507 03:23:27.316 | INFO | "gpu" : " 17 GB", | 20260507 03:23:27.316 | INFO | "huge" : "", | 20260507 03:23:27.316 | INFO | "huge_shared" : " 27 GB", | 20260507 03:23:27.316 | INFO | "malloc" : " 14 GB", | 20260507 03:23:27.316 | INFO | "pinned" : " 29 GB" | 20260507 03:23:27.316 | INFO | } | 20260507 03:23:27.316 | INFO | Perf report for the last 5.13899 seconds: | 20260507 03:23:27.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:23:27.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:27.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:27.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:27.316 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:23:27.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:27.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:27.316 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:23:27.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:27.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:27.316 | INFO | Uploaded 36GiB to GPU in 2.02472s (17.7803GiB/s) | 20260507 03:23:27.316 | INFO | Downloaded 34GiB to GPU in 1.93567s (17.565GiB/s) | 20260507 03:23:32.449 | INFO | Analyzing chunk frames = [6706176,6706688) | 20260507 03:23:32.449 | INFO | MemInfo: { | 20260507 03:23:32.449 | INFO | "gpu" : " 17 GB", | 20260507 03:23:32.449 | INFO | "huge" : "", | 20260507 03:23:32.450 | INFO | "huge_shared" : " 27 GB", | 20260507 03:23:32.450 | INFO | "malloc" : " 14 GB", | 20260507 03:23:32.450 | INFO | "pinned" : " 29 GB" | 20260507 03:23:32.450 | INFO | } | 20260507 03:23:32.450 | INFO | Perf report for the last 5.13398 seconds: | 20260507 03:23:32.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:23:32.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:32.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:32.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:32.450 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 03:23:32.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:32.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:32.450 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:23:32.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:32.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:32.450 | INFO | Uploaded 36GiB to GPU in 2.04147s (17.6344GiB/s) | 20260507 03:23:32.450 | INFO | Downloaded 34GiB to GPU in 1.95162s (17.4214GiB/s) | 20260507 03:23:37.588 | INFO | Analyzing chunk frames = [6706688,6707200) | 20260507 03:23:37.588 | INFO | MemInfo: { | 20260507 03:23:37.588 | INFO | "gpu" : " 17 GB", | 20260507 03:23:37.588 | INFO | "huge" : "", | 20260507 03:23:37.588 | INFO | "huge_shared" : " 27 GB", | 20260507 03:23:37.588 | INFO | "malloc" : " 14 GB", | 20260507 03:23:37.588 | INFO | "pinned" : " 29 GB" | 20260507 03:23:37.588 | INFO | } | 20260507 03:23:37.588 | INFO | Perf report for the last 5.13814 seconds: | 20260507 03:23:37.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:23:37.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:37.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:37.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:37.588 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:23:37.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:37.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:37.588 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 03:23:37.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:37.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:37.588 | INFO | Uploaded 36GiB to GPU in 1.98523s (18.1339GiB/s) | 20260507 03:23:37.588 | INFO | Downloaded 34GiB to GPU in 1.92137s (17.6957GiB/s) | 20260507 03:23:42.732 | INFO | Analyzing chunk frames = [6707200,6707712) | 20260507 03:23:42.732 | INFO | MemInfo: { | 20260507 03:23:42.732 | INFO | "gpu" : " 17 GB", | 20260507 03:23:42.732 | INFO | "huge" : "", | 20260507 03:23:42.732 | INFO | "huge_shared" : " 27 GB", | 20260507 03:23:42.732 | INFO | "malloc" : " 14 GB", | 20260507 03:23:42.732 | INFO | "pinned" : " 29 GB" | 20260507 03:23:42.732 | INFO | } | 20260507 03:23:42.732 | INFO | Perf report for the last 5.14411 seconds: | 20260507 03:23:42.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:23:42.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:42.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:42.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:42.732 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 03:23:42.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:42.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:42.732 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 03:23:42.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:42.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:42.732 | INFO | Uploaded 36GiB to GPU in 2.01356s (17.8788GiB/s) | 20260507 03:23:42.732 | INFO | Downloaded 34GiB to GPU in 1.95119s (17.4253GiB/s) | 20260507 03:23:47.877 | INFO | Analyzing chunk frames = [6707712,6708224) | 20260507 03:23:47.877 | INFO | MemInfo: { | 20260507 03:23:47.877 | INFO | "gpu" : " 17 GB", | 20260507 03:23:47.877 | INFO | "huge" : "", | 20260507 03:23:47.877 | INFO | "huge_shared" : " 27 GB", | 20260507 03:23:47.877 | INFO | "malloc" : " 14 GB", | 20260507 03:23:47.877 | INFO | "pinned" : " 29 GB" | 20260507 03:23:47.877 | INFO | } | 20260507 03:23:47.877 | INFO | Perf report for the last 5.14535 seconds: | 20260507 03:23:47.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:23:47.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:47.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:47.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:47.877 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 03:23:47.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:47.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:47.878 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 03:23:47.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:47.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:47.878 | INFO | Uploaded 36GiB to GPU in 2.05968s (17.4784GiB/s) | 20260507 03:23:47.878 | INFO | Downloaded 34GiB to GPU in 1.96175s (17.3314GiB/s) | 20260507 03:23:53.020 | INFO | Analyzing chunk frames = [6708224,6708736) | 20260507 03:23:53.020 | INFO | MemInfo: { | 20260507 03:23:53.020 | INFO | "gpu" : " 17 GB", | 20260507 03:23:53.020 | INFO | "huge" : "", | 20260507 03:23:53.020 | INFO | "huge_shared" : " 27 GB", | 20260507 03:23:53.020 | INFO | "malloc" : " 14 GB", | 20260507 03:23:53.020 | INFO | "pinned" : " 29 GB" | 20260507 03:23:53.020 | INFO | } | 20260507 03:23:53.020 | INFO | Perf report for the last 5.14277 seconds: | 20260507 03:23:53.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:23:53.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:53.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:53.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:53.020 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 03:23:53.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:53.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:53.020 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 03:23:53.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:53.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:53.021 | INFO | Uploaded 36GiB to GPU in 2.04144s (17.6346GiB/s) | 20260507 03:23:53.021 | INFO | Downloaded 34GiB to GPU in 1.99465s (17.0456GiB/s) | 20260507 03:23:58.152 | INFO | Analyzing chunk frames = [6708736,6709248) | 20260507 03:23:58.152 | INFO | MemInfo: { | 20260507 03:23:58.152 | INFO | "gpu" : " 17 GB", | 20260507 03:23:58.152 | INFO | "huge" : "", | 20260507 03:23:58.152 | INFO | "huge_shared" : " 27 GB", | 20260507 03:23:58.152 | INFO | "malloc" : " 14 GB", | 20260507 03:23:58.152 | INFO | "pinned" : " 29 GB" | 20260507 03:23:58.152 | INFO | } | 20260507 03:23:58.152 | INFO | Perf report for the last 5.13211 seconds: | 20260507 03:23:58.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:23:58.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:58.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:58.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:58.152 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 03:23:58.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:58.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:58.152 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:23:58.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:58.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:23:58.153 | INFO | Uploaded 36GiB to GPU in 1.97069s (18.2677GiB/s) | 20260507 03:23:58.153 | INFO | Downloaded 34GiB to GPU in 1.95827s (17.3623GiB/s) | 20260507 03:23:58.153 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:23:58.153 | INFO | Total mem usage: 18 GiB | 20260507 03:23:58.153 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:23:58.153 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:23:58.153 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:23:58.153 | INFO | High Water Breakdown: | 20260507 03:23:58.153 | INFO | 60.1% : 8.57 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:23:58.153 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:23:58.153 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:23:58.153 | INFO | ------------------------------------------ | 20260507 03:23:58.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:23:58.153 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:23:58.153 | INFO | Total mem usage: 26.5 GiB | 20260507 03:23:58.153 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:23:58.153 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:23:58.153 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:23:58.153 | INFO | High Water Breakdown: | 20260507 03:23:58.153 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:23:58.153 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:23:58.153 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:23:58.153 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:23:58.153 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:23:58.153 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:23:58.153 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:23:58.153 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:23:58.153 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:23:58.153 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:23:58.153 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:23:58.153 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:23:58.153 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:23:58.153 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:23:58.153 | INFO | ------------------------------------------ | 20260507 03:23:58.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:23:58.153 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:23:58.153 | INFO | Total mem usage: 25.6 GiB | 20260507 03:23:58.153 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:23:58.153 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:23:58.153 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:23:58.153 | INFO | High Water Breakdown: | 20260507 03:23:58.153 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:23:58.153 | INFO | ------------------------------------------ | 20260507 03:23:58.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:23:58.153 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:23:58.153 | INFO | Total mem usage: 15.7 GiB | 20260507 03:23:58.153 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:23:58.153 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:23:58.153 | INFO | Unused Cache: 862 MiB (Waste: 5.37%) | 20260507 03:23:58.153 | INFO | High Water Breakdown: | 20260507 03:23:58.153 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:23:58.153 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:23:58.153 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:23:58.153 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:23:58.153 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:23:58.153 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:23:58.153 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:23:58.153 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:23:58.153 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:23:58.153 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:23:58.153 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:23:58.153 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:23:58.153 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:23:58.153 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:23:58.153 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:23:58.153 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:23:58.153 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:23:58.153 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:23:58.153 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:23:58.153 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:23:58.153 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:23:58.153 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:23:58.153 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:23:58.153 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:23:58.153 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:23:58.153 | INFO | ------------------------------------------ | 20260507 03:23:58.153 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:23:58.154 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:23:58.154 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:23:58.154 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:23:58.154 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.48e+01, 1.33e+01 / 1.43e+00 | 20260507 03:23:58.154 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 9.97e-02, 9.81e-03 / 5.48e-03 | 20260507 03:23:58.154 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.39e+00, 5.24e-01 / 3.39e-01 | 20260507 03:23:58.154 | INFO | PerfReport: 3.25%, Histogram -- 9.99e-01 / 1.46e+00, 1.73e-01 / 1.32e-01 | 20260507 03:23:58.154 | INFO | PerfReport: 16.12%, DME -- 3.97e+01 / 4.41e+01, 3.92e+00 / 1.85e+00 | 20260507 03:23:58.154 | INFO | PerfReport: 57.59%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.27e+00 / 1.55e+00 | 20260507 03:23:58.154 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.15e+00 / 2.71e+00, 3.83e-01 / 1.84e-01 | 20260507 03:23:58.154 | INFO | PerfReport: 3.05%, Metrics -- 9.38e-01 / 1.26e+00, 2.50e-01 / 1.19e-01 | 20260507 03:23:58.154 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.17e-01 / 3.69e-01, 5.02e-02 / 1.76e-02 | 20260507 03:23:58.154 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:23:58.154 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:23:58.154 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:23:58.154 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 03:23:58.154 | INFO | PerfReport: 21.76%, Upload -- 2.09e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260507 03:23:58.154 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.47e+00 / 0.00e+00 | 20260507 03:23:58.154 | INFO | PerfReport: 31.96%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 03:23:58.154 | INFO | PerfReport: 1.40%, LockUpload -- 1.35e+00 / 0.00e+00, 4.74e+00 / 0.00e+00 | 20260507 03:23:58.154 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260507 03:23:58.154 | INFO | PerfReport: 22.14%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 03:23:58.154 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260507 03:23:58.249 | INFO | Starting a round of Baz IO | 20260507 03:24:01.358 | INFO | Populating RTMetrics report | 20260507 03:24:01.358 | INFO | RTMetrics report populated | 20260507 03:24:01.358 | INFO | Opening tmp RTMetrics json file | 20260507 03:24:01.377 | INFO | Serializing RTMetrics to json | 20260507 03:24:01.379 | INFO | Serialization done, swapping tmp file now... | 20260507 03:24:01.379 | INFO | RTMetrics done | 20260507 03:24:01.450 | INFO | Finished a round of Baz IO: Wrote 1.753 GiB in 3.2 seconds | 20260507 03:24:01.450 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.90% / 0.00% / 67.10% | 20260507 03:24:01.450 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.201507% / 0% / 7.54992e-05% | 20260507 03:24:01.450 | INFO | Baz IO overall: | 20260507 03:24:01.450 | INFO | BytesWritten : 3444781379720 | 20260507 03:24:01.450 | INFO | NumEvents : 1655300632550 | 20260507 03:24:03.288 | INFO | Analyzing chunk frames = [6709248,6709760) | 20260507 03:24:03.288 | INFO | MemInfo: { | 20260507 03:24:03.288 | INFO | "gpu" : " 17 GB", | 20260507 03:24:03.288 | INFO | "huge" : "", | 20260507 03:24:03.288 | INFO | "huge_shared" : " 27 GB", | 20260507 03:24:03.288 | INFO | "malloc" : " 14 GB", | 20260507 03:24:03.288 | INFO | "pinned" : " 29 GB" | 20260507 03:24:03.288 | INFO | } | 20260507 03:24:03.288 | INFO | Perf report for the last 5.13536 seconds: | 20260507 03:24:03.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:24:03.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:03.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:03.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:03.288 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 03:24:03.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:03.288 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 03:24:03.288 | INFO | PRE_HQ : 59.3%, 1.8, 55.6ms, 0 | 20260507 03:24:03.288 | INFO | BAZWRITER : 61.9%, 3.3, 103.8ms, 0 | 20260507 03:24:03.288 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:24:03.288 | INFO | Uploaded 36GiB to GPU in 2.03314s (17.7066GiB/s) | 20260507 03:24:03.288 | INFO | Downloaded 38GiB to GPU in 2.18485s (17.3925GiB/s) | 20260507 03:24:08.423 | INFO | Analyzing chunk frames = [6709760,6710272) | 20260507 03:24:08.423 | INFO | MemInfo: { | 20260507 03:24:08.423 | INFO | "gpu" : " 17 GB", | 20260507 03:24:08.423 | INFO | "huge" : "", | 20260507 03:24:08.423 | INFO | "huge_shared" : " 27 GB", | 20260507 03:24:08.423 | INFO | "malloc" : " 14 GB", | 20260507 03:24:08.427 | INFO | "pinned" : " 29 GB" | 20260507 03:24:08.427 | INFO | } | 20260507 03:24:08.427 | INFO | Perf report for the last 5.13925 seconds: | 20260507 03:24:08.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:24:08.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:08.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:08.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:08.427 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:24:08.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:08.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:08.427 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:24:08.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:08.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:08.427 | INFO | Uploaded 36GiB to GPU in 1.99389s (18.0551GiB/s) | 20260507 03:24:08.427 | INFO | Downloaded 34GiB to GPU in 1.93233s (17.5954GiB/s) | 20260507 03:24:13.569 | INFO | Analyzing chunk frames = [6710272,6710784) | 20260507 03:24:13.569 | INFO | MemInfo: { | 20260507 03:24:13.569 | INFO | "gpu" : " 17 GB", | 20260507 03:24:13.569 | INFO | "huge" : "", | 20260507 03:24:13.569 | INFO | "huge_shared" : " 27 GB", | 20260507 03:24:13.569 | INFO | "malloc" : " 14 GB", | 20260507 03:24:13.569 | INFO | "pinned" : " 29 GB" | 20260507 03:24:13.569 | INFO | } | 20260507 03:24:13.569 | INFO | Perf report for the last 5.14208 seconds: | 20260507 03:24:13.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:24:13.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:13.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:13.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:13.569 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 03:24:13.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:13.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:13.569 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:24:13.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:13.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:13.569 | INFO | Uploaded 36GiB to GPU in 1.99283s (18.0648GiB/s) | 20260507 03:24:13.569 | INFO | Downloaded 34GiB to GPU in 1.92779s (17.6368GiB/s) | 20260507 03:24:18.712 | INFO | Analyzing chunk frames = [6710784,6711296) | 20260507 03:24:18.712 | INFO | MemInfo: { | 20260507 03:24:18.712 | INFO | "gpu" : " 17 GB", | 20260507 03:24:18.712 | INFO | "huge" : "", | 20260507 03:24:18.712 | INFO | "huge_shared" : " 27 GB", | 20260507 03:24:18.712 | INFO | "malloc" : " 14 GB", | 20260507 03:24:18.712 | INFO | "pinned" : " 29 GB" | 20260507 03:24:18.712 | INFO | } | 20260507 03:24:18.712 | INFO | Perf report for the last 5.14309 seconds: | 20260507 03:24:18.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:24:18.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:18.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:18.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:18.712 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 03:24:18.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:18.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:18.712 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:24:18.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:18.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:18.712 | INFO | Uploaded 36GiB to GPU in 1.97152s (18.26GiB/s) | 20260507 03:24:18.712 | INFO | Downloaded 34GiB to GPU in 1.92243s (17.6859GiB/s) | 20260507 03:24:23.844 | INFO | Analyzing chunk frames = [6711296,6711808) | 20260507 03:24:23.844 | INFO | MemInfo: { | 20260507 03:24:23.844 | INFO | "gpu" : " 17 GB", | 20260507 03:24:23.844 | INFO | "huge" : "", | 20260507 03:24:23.844 | INFO | "huge_shared" : " 27 GB", | 20260507 03:24:23.844 | INFO | "malloc" : " 14 GB", | 20260507 03:24:23.844 | INFO | "pinned" : " 29 GB" | 20260507 03:24:23.844 | INFO | } | 20260507 03:24:23.844 | INFO | Perf report for the last 5.13176 seconds: | 20260507 03:24:23.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:24:23.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:23.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:23.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:23.844 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 03:24:23.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:23.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:23.844 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 03:24:23.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:23.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:23.844 | INFO | Uploaded 36GiB to GPU in 2.05081s (17.554GiB/s) | 20260507 03:24:23.844 | INFO | Downloaded 34GiB to GPU in 1.95292s (17.4098GiB/s) | 20260507 03:24:28.989 | INFO | Analyzing chunk frames = [6711808,6712320) | 20260507 03:24:28.989 | INFO | MemInfo: { | 20260507 03:24:28.989 | INFO | "gpu" : " 17 GB", | 20260507 03:24:28.989 | INFO | "huge" : "", | 20260507 03:24:28.989 | INFO | "huge_shared" : " 27 GB", | 20260507 03:24:28.989 | INFO | "malloc" : " 14 GB", | 20260507 03:24:28.989 | INFO | "pinned" : " 29 GB" | 20260507 03:24:28.989 | INFO | } | 20260507 03:24:28.989 | INFO | Perf report for the last 5.14536 seconds: | 20260507 03:24:28.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:24:28.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:28.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:28.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:28.989 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 03:24:28.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:28.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:28.989 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 03:24:28.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:28.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:28.990 | INFO | Uploaded 36GiB to GPU in 2.04666s (17.5896GiB/s) | 20260507 03:24:28.990 | INFO | Downloaded 34GiB to GPU in 1.95913s (17.3547GiB/s) | 20260507 03:24:34.131 | INFO | Analyzing chunk frames = [6712320,6712832) | 20260507 03:24:34.131 | INFO | MemInfo: { | 20260507 03:24:34.131 | INFO | "gpu" : " 17 GB", | 20260507 03:24:34.131 | INFO | "huge" : "", | 20260507 03:24:34.131 | INFO | "huge_shared" : " 27 GB", | 20260507 03:24:34.131 | INFO | "malloc" : " 14 GB", | 20260507 03:24:34.131 | INFO | "pinned" : " 29 GB" | 20260507 03:24:34.131 | INFO | } | 20260507 03:24:34.131 | INFO | Perf report for the last 5.14152 seconds: | 20260507 03:24:34.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:24:34.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:34.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:34.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:34.131 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:24:34.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:34.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:34.131 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 03:24:34.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:34.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:34.131 | INFO | Uploaded 36GiB to GPU in 1.97193s (18.2562GiB/s) | 20260507 03:24:34.131 | INFO | Downloaded 34GiB to GPU in 1.97334s (17.2297GiB/s) | 20260507 03:24:39.264 | INFO | Analyzing chunk frames = [6712832,6713344) | 20260507 03:24:39.264 | INFO | MemInfo: { | 20260507 03:24:39.264 | INFO | "gpu" : " 17 GB", | 20260507 03:24:39.264 | INFO | "huge" : "", | 20260507 03:24:39.264 | INFO | "huge_shared" : " 27 GB", | 20260507 03:24:39.264 | INFO | "malloc" : " 14 GB", | 20260507 03:24:39.264 | INFO | "pinned" : " 29 GB" | 20260507 03:24:39.264 | INFO | } | 20260507 03:24:39.264 | INFO | Perf report for the last 5.13331 seconds: | 20260507 03:24:39.264 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:24:39.264 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:39.264 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:39.264 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:39.264 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 03:24:39.264 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:39.264 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:39.264 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 03:24:39.264 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:39.264 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:39.264 | INFO | Uploaded 36GiB to GPU in 1.97545s (18.2237GiB/s) | 20260507 03:24:39.264 | INFO | Downloaded 34GiB to GPU in 1.96685s (17.2866GiB/s) | 20260507 03:24:39.361 | INFO | Starting a round of Baz IO | 20260507 03:24:42.463 | INFO | Populating RTMetrics report | 20260507 03:24:42.464 | INFO | RTMetrics report populated | 20260507 03:24:42.464 | INFO | Opening tmp RTMetrics json file | 20260507 03:24:42.466 | INFO | Serializing RTMetrics to json | 20260507 03:24:42.482 | INFO | Serialization done, swapping tmp file now... | 20260507 03:24:42.483 | INFO | RTMetrics done | 20260507 03:24:42.557 | INFO | Finished a round of Baz IO: Wrote 1.753 GiB in 3.196 seconds | 20260507 03:24:42.557 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.90% / 0.00% / 67.10% | 20260507 03:24:42.557 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.201348% / 0% / 0.000107288% | 20260507 03:24:42.557 | INFO | Baz IO overall: | 20260507 03:24:42.557 | INFO | BytesWritten : 3446663674405 | 20260507 03:24:42.557 | INFO | NumEvents : 1656188282349 | 20260507 03:24:44.398 | INFO | Analyzing chunk frames = [6713344,6713856) | 20260507 03:24:44.398 | INFO | MemInfo: { | 20260507 03:24:44.398 | INFO | "gpu" : " 17 GB", | 20260507 03:24:44.398 | INFO | "huge" : "", | 20260507 03:24:44.398 | INFO | "huge_shared" : " 27 GB", | 20260507 03:24:44.399 | INFO | "malloc" : " 14 GB", | 20260507 03:24:44.399 | INFO | "pinned" : " 29 GB" | 20260507 03:24:44.399 | INFO | } | 20260507 03:24:44.399 | INFO | Perf report for the last 5.13445 seconds: | 20260507 03:24:44.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:24:44.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:44.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:44.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:44.399 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260507 03:24:44.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:44.399 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:24:44.399 | INFO | PRE_HQ : 59.2%, 1.8, 55.3ms, 0 | 20260507 03:24:44.399 | INFO | BAZWRITER : 61.9%, 3.3, 104.1ms, 0 | 20260507 03:24:44.399 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:24:44.399 | INFO | Uploaded 36GiB to GPU in 2.0375s (17.6687GiB/s) | 20260507 03:24:44.399 | INFO | Downloaded 38GiB to GPU in 2.1701s (17.5107GiB/s) | 20260507 03:24:49.544 | INFO | Analyzing chunk frames = [6713856,6714368) | 20260507 03:24:49.544 | INFO | MemInfo: { | 20260507 03:24:49.544 | INFO | "gpu" : " 17 GB", | 20260507 03:24:49.544 | INFO | "huge" : "", | 20260507 03:24:49.544 | INFO | "huge_shared" : " 27 GB", | 20260507 03:24:49.544 | INFO | "malloc" : " 14 GB", | 20260507 03:24:49.544 | INFO | "pinned" : " 29 GB" | 20260507 03:24:49.544 | INFO | } | 20260507 03:24:49.544 | INFO | Perf report for the last 5.14521 seconds: | 20260507 03:24:49.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:24:49.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:49.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:49.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:49.544 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 03:24:49.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:49.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:49.544 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:24:49.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:49.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:49.544 | INFO | Uploaded 36GiB to GPU in 2.00899s (17.9194GiB/s) | 20260507 03:24:49.544 | INFO | Downloaded 34GiB to GPU in 1.92818s (17.6332GiB/s) | 20260507 03:24:54.684 | INFO | Analyzing chunk frames = [6714368,6714880) | 20260507 03:24:54.684 | INFO | MemInfo: { | 20260507 03:24:54.684 | INFO | "gpu" : " 17 GB", | 20260507 03:24:54.684 | INFO | "huge" : "", | 20260507 03:24:54.684 | INFO | "huge_shared" : " 27 GB", | 20260507 03:24:54.684 | INFO | "malloc" : " 14 GB", | 20260507 03:24:54.684 | INFO | "pinned" : " 29 GB" | 20260507 03:24:54.684 | INFO | } | 20260507 03:24:54.684 | INFO | Perf report for the last 5.14001 seconds: | 20260507 03:24:54.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:24:54.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:54.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:54.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:54.684 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 03:24:54.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:54.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:54.684 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 03:24:54.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:54.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:54.684 | INFO | Uploaded 36GiB to GPU in 2.0069s (17.9381GiB/s) | 20260507 03:24:54.684 | INFO | Downloaded 34GiB to GPU in 1.937s (17.5529GiB/s) | 20260507 03:24:59.823 | INFO | Analyzing chunk frames = [6714880,6715392) | 20260507 03:24:59.823 | INFO | MemInfo: { | 20260507 03:24:59.823 | INFO | "gpu" : " 17 GB", | 20260507 03:24:59.823 | INFO | "huge" : "", | 20260507 03:24:59.823 | INFO | "huge_shared" : " 27 GB", | 20260507 03:24:59.823 | INFO | "malloc" : " 14 GB", | 20260507 03:24:59.823 | INFO | "pinned" : " 29 GB" | 20260507 03:24:59.823 | INFO | } | 20260507 03:24:59.823 | INFO | Perf report for the last 5.13952 seconds: | 20260507 03:24:59.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:24:59.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:59.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:59.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:59.823 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:24:59.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:59.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:59.823 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:24:59.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:59.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:24:59.824 | INFO | Uploaded 36GiB to GPU in 1.97957s (18.1858GiB/s) | 20260507 03:24:59.824 | INFO | Downloaded 34GiB to GPU in 1.92054s (17.7033GiB/s) | 20260507 03:25:04.964 | INFO | Analyzing chunk frames = [6715392,6715904) | 20260507 03:25:04.964 | INFO | MemInfo: { | 20260507 03:25:04.964 | INFO | "gpu" : " 17 GB", | 20260507 03:25:04.964 | INFO | "huge" : "", | 20260507 03:25:04.964 | INFO | "huge_shared" : " 27 GB", | 20260507 03:25:04.964 | INFO | "malloc" : " 14 GB", | 20260507 03:25:04.964 | INFO | "pinned" : " 29 GB" | 20260507 03:25:04.964 | INFO | } | 20260507 03:25:04.964 | INFO | Perf report for the last 5.14069 seconds: | 20260507 03:25:04.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:25:04.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:04.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:04.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:04.964 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 03:25:04.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:04.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:04.964 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:25:04.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:04.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:04.964 | INFO | Uploaded 36GiB to GPU in 2.05849s (17.4885GiB/s) | 20260507 03:25:04.964 | INFO | Downloaded 34GiB to GPU in 1.94874s (17.4472GiB/s) | 20260507 03:25:10.099 | INFO | Analyzing chunk frames = [6715904,6716416) | 20260507 03:25:10.099 | INFO | MemInfo: { | 20260507 03:25:10.099 | INFO | "gpu" : " 17 GB", | 20260507 03:25:10.099 | INFO | "huge" : "", | 20260507 03:25:10.099 | INFO | "huge_shared" : " 27 GB", | 20260507 03:25:10.099 | INFO | "malloc" : " 14 GB", | 20260507 03:25:10.099 | INFO | "pinned" : " 29 GB" | 20260507 03:25:10.099 | INFO | } | 20260507 03:25:10.099 | INFO | Perf report for the last 5.13497 seconds: | 20260507 03:25:10.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:25:10.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:10.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:10.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:10.099 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 03:25:10.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:10.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:10.099 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 03:25:10.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:10.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:10.099 | INFO | Uploaded 36GiB to GPU in 2.04277s (17.6231GiB/s) | 20260507 03:25:10.099 | INFO | Downloaded 34GiB to GPU in 1.90909s (17.8095GiB/s) | 20260507 03:25:15.238 | INFO | Analyzing chunk frames = [6716416,6716928) | 20260507 03:25:15.238 | INFO | MemInfo: { | 20260507 03:25:15.238 | INFO | "gpu" : " 17 GB", | 20260507 03:25:15.238 | INFO | "huge" : "", | 20260507 03:25:15.238 | INFO | "huge_shared" : " 27 GB", | 20260507 03:25:15.238 | INFO | "malloc" : " 14 GB", | 20260507 03:25:15.238 | INFO | "pinned" : " 29 GB" | 20260507 03:25:15.238 | INFO | } | 20260507 03:25:15.238 | INFO | Perf report for the last 5.13928 seconds: | 20260507 03:25:15.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:25:15.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:15.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:15.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:15.238 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:25:15.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:15.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:15.238 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 03:25:15.239 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:15.239 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:15.239 | INFO | Uploaded 36GiB to GPU in 1.99702s (18.0269GiB/s) | 20260507 03:25:15.239 | INFO | Downloaded 34GiB to GPU in 1.98199s (17.1545GiB/s) | 20260507 03:25:20.380 | INFO | Analyzing chunk frames = [6716928,6717440) | 20260507 03:25:20.380 | INFO | MemInfo: { | 20260507 03:25:20.380 | INFO | "gpu" : " 17 GB", | 20260507 03:25:20.380 | INFO | "huge" : "", | 20260507 03:25:20.380 | INFO | "huge_shared" : " 27 GB", | 20260507 03:25:20.380 | INFO | "malloc" : " 14 GB", | 20260507 03:25:20.380 | INFO | "pinned" : " 29 GB" | 20260507 03:25:20.380 | INFO | } | 20260507 03:25:20.380 | INFO | Perf report for the last 5.14163 seconds: | 20260507 03:25:20.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:25:20.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:20.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:20.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:20.380 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:25:20.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:20.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:20.380 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:25:20.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:20.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:20.380 | INFO | Uploaded 36GiB to GPU in 1.97527s (18.2253GiB/s) | 20260507 03:25:20.380 | INFO | Downloaded 34GiB to GPU in 1.96806s (17.2759GiB/s) | 20260507 03:25:20.381 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:25:20.381 | INFO | Total mem usage: 18 GiB | 20260507 03:25:20.381 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:25:20.381 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:25:20.381 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:25:20.381 | INFO | High Water Breakdown: | 20260507 03:25:20.381 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:25:20.381 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:25:20.381 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:25:20.381 | INFO | ------------------------------------------ | 20260507 03:25:20.381 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:25:20.381 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:25:20.381 | INFO | Total mem usage: 26.5 GiB | 20260507 03:25:20.381 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:25:20.381 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:25:20.381 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:25:20.381 | INFO | High Water Breakdown: | 20260507 03:25:20.381 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:25:20.381 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:25:20.381 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:25:20.381 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:25:20.381 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:25:20.381 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:25:20.381 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:25:20.381 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:25:20.381 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:25:20.381 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:25:20.381 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:25:20.381 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:25:20.381 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:25:20.381 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:25:20.381 | INFO | ------------------------------------------ | 20260507 03:25:20.381 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:25:20.381 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:25:20.381 | INFO | Total mem usage: 25.6 GiB | 20260507 03:25:20.381 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:25:20.381 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 03:25:20.381 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 03:25:20.381 | INFO | High Water Breakdown: | 20260507 03:25:20.381 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 03:25:20.381 | INFO | ------------------------------------------ | 20260507 03:25:20.381 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:25:20.381 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:25:20.381 | INFO | Total mem usage: 15.7 GiB | 20260507 03:25:20.381 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:25:20.381 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:25:20.381 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:25:20.381 | INFO | High Water Breakdown: | 20260507 03:25:20.381 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:25:20.381 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:25:20.381 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:25:20.381 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:25:20.381 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:25:20.381 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:25:20.381 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:25:20.381 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:25:20.381 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:25:20.381 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:25:20.381 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:25:20.381 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:25:20.381 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:25:20.381 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:25:20.381 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:25:20.381 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:25:20.381 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:25:20.381 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:25:20.381 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:25:20.381 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:25:20.381 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:25:20.381 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:25:20.381 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:25:20.381 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:25:20.381 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:25:20.381 | INFO | ------------------------------------------ | 20260507 03:25:20.381 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:25:20.381 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:25:20.381 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:25:20.381 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:25:20.381 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.32e+01 / 1.63e+00 | 20260507 03:25:20.381 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.00e-01, 9.15e-03 / 5.30e-03 | 20260507 03:25:20.381 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.41e+00, 5.08e-01 / 2.35e-01 | 20260507 03:25:20.381 | INFO | PerfReport: 3.22%, Histogram -- 9.90e-01 / 1.47e+00, 1.67e-01 / 1.18e-01 | 20260507 03:25:20.381 | INFO | PerfReport: 16.10%, DME -- 3.96e+01 / 4.41e+01, 3.78e+00 / 1.83e+00 | 20260507 03:25:20.381 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.27e+00 / 1.60e+00 | 20260507 03:25:20.381 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.15e+00 / 2.67e+00, 3.89e-01 / 2.59e-01 | 20260507 03:25:20.381 | INFO | PerfReport: 3.05%, Metrics -- 9.39e-01 / 1.26e+00, 2.50e-01 / 1.58e-01 | 20260507 03:25:20.381 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.17e-01 / 3.72e-01, 4.95e-02 / 2.43e-02 | 20260507 03:25:20.381 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:25:20.381 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:25:20.381 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:25:20.381 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 03:25:20.381 | INFO | PerfReport: 21.74%, Upload -- 2.09e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260507 03:25:20.381 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260507 03:25:20.381 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 03:25:20.381 | INFO | PerfReport: 1.47%, LockUpload -- 1.41e+00 / 0.00e+00, 4.87e+00 / 0.00e+00 | 20260507 03:25:20.381 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260507 03:25:20.381 | INFO | PerfReport: 22.18%, LockCompute -- 2.14e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 03:25:20.381 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.30e-02 / 0.00e+00 | 20260507 03:25:20.477 | INFO | Starting a round of Baz IO | 20260507 03:25:23.581 | INFO | Populating RTMetrics report | 20260507 03:25:23.582 | INFO | RTMetrics report populated | 20260507 03:25:23.582 | INFO | Opening tmp RTMetrics json file | 20260507 03:25:23.585 | INFO | Serializing RTMetrics to json | 20260507 03:25:23.601 | INFO | Serialization done, swapping tmp file now... | 20260507 03:25:23.601 | INFO | RTMetrics done | 20260507 03:25:23.676 | INFO | Finished a round of Baz IO: Wrote 1.752 GiB in 3.199 seconds | 20260507 03:25:23.676 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.89% / 0.00% / 67.10% | 20260507 03:25:23.676 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.200172% / 3.97364e-06% / 9.93411e-05% | 20260507 03:25:23.676 | INFO | Baz IO overall: | 20260507 03:25:23.676 | INFO | BytesWritten : 3448544333500 | 20260507 03:25:23.676 | INFO | NumEvents : 1657074844183 | 20260507 03:25:25.519 | INFO | Analyzing chunk frames = [6717440,6717952) | 20260507 03:25:25.519 | INFO | MemInfo: { | 20260507 03:25:25.519 | INFO | "gpu" : " 17 GB", | 20260507 03:25:25.519 | INFO | "huge" : "", | 20260507 03:25:25.519 | INFO | "huge_shared" : " 27 GB", | 20260507 03:25:25.519 | INFO | "malloc" : " 14 GB", | 20260507 03:25:25.519 | INFO | "pinned" : " 29 GB" | 20260507 03:25:25.519 | INFO | } | 20260507 03:25:25.519 | INFO | Perf report for the last 5.13857 seconds: | 20260507 03:25:25.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:25:25.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:25.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:25.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:25.519 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 03:25:25.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:25.519 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 03:25:25.519 | INFO | PRE_HQ : 59.1%, 1.8, 55.2ms, 0 | 20260507 03:25:25.519 | INFO | BAZWRITER : 61.8%, 3.2, 102.6ms, 0 | 20260507 03:25:25.519 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:25:25.519 | INFO | Uploaded 36GiB to GPU in 2.02984s (17.7354GiB/s) | 20260507 03:25:25.519 | INFO | Downloaded 38GiB to GPU in 2.18932s (17.357GiB/s) | 20260507 03:25:30.661 | INFO | Analyzing chunk frames = [6717952,6718464) | 20260507 03:25:30.661 | INFO | MemInfo: { | 20260507 03:25:30.661 | INFO | "gpu" : " 17 GB", | 20260507 03:25:30.661 | INFO | "huge" : "", | 20260507 03:25:30.661 | INFO | "huge_shared" : " 27 GB", | 20260507 03:25:30.661 | INFO | "malloc" : " 14 GB", | 20260507 03:25:30.661 | INFO | "pinned" : " 29 GB" | 20260507 03:25:30.661 | INFO | } | 20260507 03:25:30.661 | INFO | Perf report for the last 5.1425 seconds: | 20260507 03:25:30.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:25:30.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:30.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:30.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:30.661 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:25:30.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:30.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:30.661 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:25:30.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:30.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:30.662 | INFO | Uploaded 36GiB to GPU in 1.99096s (18.0817GiB/s) | 20260507 03:25:30.662 | INFO | Downloaded 34GiB to GPU in 1.93604s (17.5616GiB/s) | 20260507 03:25:35.798 | INFO | Analyzing chunk frames = [6718464,6718976) | 20260507 03:25:35.798 | INFO | MemInfo: { | 20260507 03:25:35.798 | INFO | "gpu" : " 17 GB", | 20260507 03:25:35.798 | INFO | "huge" : "", | 20260507 03:25:35.798 | INFO | "huge_shared" : " 27 GB", | 20260507 03:25:35.798 | INFO | "malloc" : " 14 GB", | 20260507 03:25:35.799 | INFO | "pinned" : " 29 GB" | 20260507 03:25:35.799 | INFO | } | 20260507 03:25:35.799 | INFO | Perf report for the last 5.13746 seconds: | 20260507 03:25:35.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:25:35.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:35.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:35.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:35.799 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 03:25:35.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:35.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:35.799 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:25:35.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:35.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:35.799 | INFO | Uploaded 36GiB to GPU in 2.03216s (17.7151GiB/s) | 20260507 03:25:35.799 | INFO | Downloaded 34GiB to GPU in 1.95579s (17.3842GiB/s) | 20260507 03:25:40.938 | INFO | Analyzing chunk frames = [6718976,6719488) | 20260507 03:25:40.938 | INFO | MemInfo: { | 20260507 03:25:40.938 | INFO | "gpu" : " 17 GB", | 20260507 03:25:40.938 | INFO | "huge" : "", | 20260507 03:25:40.938 | INFO | "huge_shared" : " 27 GB", | 20260507 03:25:40.938 | INFO | "malloc" : " 14 GB", | 20260507 03:25:40.938 | INFO | "pinned" : " 29 GB" | 20260507 03:25:40.938 | INFO | } | 20260507 03:25:40.938 | INFO | Perf report for the last 5.13933 seconds: | 20260507 03:25:40.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:25:40.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:40.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:40.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:40.938 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:25:40.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:40.938 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:40.938 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:25:40.938 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:40.938 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:40.938 | INFO | Uploaded 36GiB to GPU in 1.98927s (18.0971GiB/s) | 20260507 03:25:40.939 | INFO | Downloaded 34GiB to GPU in 1.92181s (17.6917GiB/s) | 20260507 03:25:46.076 | INFO | Analyzing chunk frames = [6719488,6720000) | 20260507 03:25:46.076 | INFO | MemInfo: { | 20260507 03:25:46.076 | INFO | "gpu" : " 17 GB", | 20260507 03:25:46.076 | INFO | "huge" : "", | 20260507 03:25:46.076 | INFO | "huge_shared" : " 27 GB", | 20260507 03:25:46.076 | INFO | "malloc" : " 14 GB", | 20260507 03:25:46.076 | INFO | "pinned" : " 29 GB" | 20260507 03:25:46.076 | INFO | } | 20260507 03:25:46.076 | INFO | Perf report for the last 5.13799 seconds: | 20260507 03:25:46.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:25:46.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:46.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:46.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:46.076 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 03:25:46.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:46.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:46.076 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 03:25:46.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:46.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:46.076 | INFO | Uploaded 36GiB to GPU in 2.02304s (17.795GiB/s) | 20260507 03:25:46.076 | INFO | Downloaded 34GiB to GPU in 1.9433s (17.496GiB/s) | 20260507 03:25:51.214 | INFO | Analyzing chunk frames = [6720000,6720512) | 20260507 03:25:51.214 | INFO | MemInfo: { | 20260507 03:25:51.214 | INFO | "gpu" : " 17 GB", | 20260507 03:25:51.214 | INFO | "huge" : "", | 20260507 03:25:51.214 | INFO | "huge_shared" : " 27 GB", | 20260507 03:25:51.214 | INFO | "malloc" : " 14 GB", | 20260507 03:25:51.214 | INFO | "pinned" : " 29 GB" | 20260507 03:25:51.214 | INFO | } | 20260507 03:25:51.214 | INFO | Perf report for the last 5.1378 seconds: | 20260507 03:25:51.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:25:51.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:51.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:51.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:51.214 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 03:25:51.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:51.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:51.214 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:25:51.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:51.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:51.214 | INFO | Uploaded 36GiB to GPU in 2.00972s (17.913GiB/s) | 20260507 03:25:51.214 | INFO | Downloaded 34GiB to GPU in 1.93108s (17.6067GiB/s) | 20260507 03:25:56.351 | INFO | Analyzing chunk frames = [6720512,6721024) | 20260507 03:25:56.351 | INFO | MemInfo: { | 20260507 03:25:56.351 | INFO | "gpu" : " 17 GB", | 20260507 03:25:56.351 | INFO | "huge" : "", | 20260507 03:25:56.351 | INFO | "huge_shared" : " 27 GB", | 20260507 03:25:56.351 | INFO | "malloc" : " 14 GB", | 20260507 03:25:56.351 | INFO | "pinned" : " 29 GB" | 20260507 03:25:56.351 | INFO | } | 20260507 03:25:56.351 | INFO | Perf report for the last 5.13756 seconds: | 20260507 03:25:56.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:25:56.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:56.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:56.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:56.351 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 03:25:56.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:56.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:56.351 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:25:56.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:56.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:25:56.352 | INFO | Uploaded 36GiB to GPU in 1.9785s (18.1956GiB/s) | 20260507 03:25:56.352 | INFO | Downloaded 34GiB to GPU in 1.98458s (17.1321GiB/s) | 20260507 03:26:01.494 | INFO | Analyzing chunk frames = [6721024,6721536) | 20260507 03:26:01.494 | INFO | MemInfo: { | 20260507 03:26:01.494 | INFO | "gpu" : " 17 GB", | 20260507 03:26:01.494 | INFO | "huge" : "", | 20260507 03:26:01.494 | INFO | "huge_shared" : " 27 GB", | 20260507 03:26:01.494 | INFO | "malloc" : " 14 GB", | 20260507 03:26:01.494 | INFO | "pinned" : " 29 GB" | 20260507 03:26:01.494 | INFO | } | 20260507 03:26:01.494 | INFO | Perf report for the last 5.14243 seconds: | 20260507 03:26:01.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:26:01.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:01.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:01.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:01.494 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 03:26:01.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:01.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:01.494 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:26:01.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:01.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:01.494 | INFO | Uploaded 36GiB to GPU in 1.98848s (18.1042GiB/s) | 20260507 03:26:01.494 | INFO | Downloaded 34GiB to GPU in 1.97573s (17.2089GiB/s) | 20260507 03:26:01.591 | INFO | Starting a round of Baz IO | 20260507 03:26:04.695 | INFO | Populating RTMetrics report | 20260507 03:26:04.696 | INFO | RTMetrics report populated | 20260507 03:26:04.696 | INFO | Opening tmp RTMetrics json file | 20260507 03:26:04.700 | INFO | Serializing RTMetrics to json | 20260507 03:26:04.714 | INFO | Serialization done, swapping tmp file now... | 20260507 03:26:04.714 | INFO | RTMetrics done | 20260507 03:26:04.784 | INFO | Finished a round of Baz IO: Wrote 1.753 GiB in 3.193 seconds | 20260507 03:26:04.784 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.89% / 0.00% / 67.11% | 20260507 03:26:04.784 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.20086% / 0% / 7.94729e-05% | 20260507 03:26:04.784 | INFO | Baz IO overall: | 20260507 03:26:04.784 | INFO | BytesWritten : 3450426803163 | 20260507 03:26:04.784 | INFO | NumEvents : 1657962607164 | 20260507 03:26:06.639 | INFO | Analyzing chunk frames = [6721536,6722048) | 20260507 03:26:06.639 | INFO | MemInfo: { | 20260507 03:26:06.639 | INFO | "gpu" : " 17 GB", | 20260507 03:26:06.639 | INFO | "huge" : "", | 20260507 03:26:06.639 | INFO | "huge_shared" : " 27 GB", | 20260507 03:26:06.639 | INFO | "malloc" : " 14 GB", | 20260507 03:26:06.639 | INFO | "pinned" : " 29 GB" | 20260507 03:26:06.639 | INFO | } | 20260507 03:26:06.639 | INFO | Perf report for the last 5.14546 seconds: | 20260507 03:26:06.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:26:06.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:06.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:06.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:06.639 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 03:26:06.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:06.639 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:26:06.639 | INFO | PRE_HQ : 59.1%, 1.8, 55.5ms, 0 | 20260507 03:26:06.639 | INFO | BAZWRITER : 61.7%, 3.2, 102.0ms, 0 | 20260507 03:26:06.639 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 03:26:06.640 | INFO | Uploaded 36GiB to GPU in 2.03847s (17.6603GiB/s) | 20260507 03:26:06.640 | INFO | Downloaded 38GiB to GPU in 2.18315s (17.4061GiB/s) | 20260507 03:26:11.772 | INFO | Analyzing chunk frames = [6722048,6722560) | 20260507 03:26:11.772 | INFO | MemInfo: { | 20260507 03:26:11.772 | INFO | "gpu" : " 17 GB", | 20260507 03:26:11.772 | INFO | "huge" : "", | 20260507 03:26:11.772 | INFO | "huge_shared" : " 27 GB", | 20260507 03:26:11.772 | INFO | "malloc" : " 14 GB", | 20260507 03:26:11.772 | INFO | "pinned" : " 29 GB" | 20260507 03:26:11.772 | INFO | } | 20260507 03:26:11.772 | INFO | Perf report for the last 5.13258 seconds: | 20260507 03:26:11.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:26:11.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:11.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:11.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:11.772 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 03:26:11.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:11.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:11.772 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:26:11.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:11.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:11.772 | INFO | Uploaded 36GiB to GPU in 1.98861s (18.1031GiB/s) | 20260507 03:26:11.772 | INFO | Downloaded 34GiB to GPU in 1.9287s (17.6285GiB/s) | 20260507 03:26:16.910 | INFO | Analyzing chunk frames = [6722560,6723072) | 20260507 03:26:16.910 | INFO | MemInfo: { | 20260507 03:26:16.910 | INFO | "gpu" : " 17 GB", | 20260507 03:26:16.910 | INFO | "huge" : "", | 20260507 03:26:16.911 | INFO | "huge_shared" : " 27 GB", | 20260507 03:26:16.911 | INFO | "malloc" : " 14 GB", | 20260507 03:26:16.911 | INFO | "pinned" : " 29 GB" | 20260507 03:26:16.911 | INFO | } | 20260507 03:26:16.911 | INFO | Perf report for the last 5.13879 seconds: | 20260507 03:26:16.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:26:16.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:16.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:16.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:16.911 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 03:26:16.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:16.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:16.911 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:26:16.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:16.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:16.911 | INFO | Uploaded 36GiB to GPU in 2.01099s (17.9017GiB/s) | 20260507 03:26:16.911 | INFO | Downloaded 34GiB to GPU in 1.9402s (17.524GiB/s) | 20260507 03:26:22.049 | INFO | Analyzing chunk frames = [6723072,6723584) | 20260507 03:26:22.049 | INFO | MemInfo: { | 20260507 03:26:22.049 | INFO | "gpu" : " 17 GB", | 20260507 03:26:22.049 | INFO | "huge" : "", | 20260507 03:26:22.049 | INFO | "huge_shared" : " 27 GB", | 20260507 03:26:22.049 | INFO | "malloc" : " 14 GB", | 20260507 03:26:22.049 | INFO | "pinned" : " 29 GB" | 20260507 03:26:22.049 | INFO | } | 20260507 03:26:22.049 | INFO | Perf report for the last 5.13835 seconds: | 20260507 03:26:22.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:26:22.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:22.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:22.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:22.049 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 03:26:22.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:22.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:22.049 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:26:22.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:22.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:22.049 | INFO | Uploaded 36GiB to GPU in 1.99606s (18.0355GiB/s) | 20260507 03:26:22.049 | INFO | Downloaded 34GiB to GPU in 1.91786s (17.7281GiB/s) | 20260507 03:26:27.193 | INFO | Analyzing chunk frames = [6723584,6724096) | 20260507 03:26:27.193 | INFO | MemInfo: { | 20260507 03:26:27.193 | INFO | "gpu" : " 17 GB", | 20260507 03:26:27.193 | INFO | "huge" : "", | 20260507 03:26:27.193 | INFO | "huge_shared" : " 27 GB", | 20260507 03:26:27.193 | INFO | "malloc" : " 14 GB", | 20260507 03:26:27.193 | INFO | "pinned" : " 29 GB" | 20260507 03:26:27.193 | INFO | } | 20260507 03:26:27.193 | INFO | Perf report for the last 5.14447 seconds: | 20260507 03:26:27.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:26:27.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:27.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:27.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:27.194 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 03:26:27.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:27.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:27.194 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 03:26:27.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:27.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:27.194 | INFO | Uploaded 36GiB to GPU in 2.00789s (17.9293GiB/s) | 20260507 03:26:27.194 | INFO | Downloaded 34GiB to GPU in 1.94013s (17.5246GiB/s) | 20260507 03:26:32.343 | INFO | Analyzing chunk frames = [6724096,6724608) | 20260507 03:26:32.343 | INFO | MemInfo: { | 20260507 03:26:32.343 | INFO | "gpu" : " 17 GB", | 20260507 03:26:32.343 | INFO | "huge" : "", | 20260507 03:26:32.343 | INFO | "huge_shared" : " 27 GB", | 20260507 03:26:32.343 | INFO | "malloc" : " 14 GB", | 20260507 03:26:32.343 | INFO | "pinned" : " 29 GB" | 20260507 03:26:32.343 | INFO | } | 20260507 03:26:32.343 | INFO | Perf report for the last 5.14995 seconds: | 20260507 03:26:32.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:26:32.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:32.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:32.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:32.343 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 03:26:32.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:32.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:32.344 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 03:26:32.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:32.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:32.344 | INFO | Uploaded 36GiB to GPU in 2.05876s (17.4863GiB/s) | 20260507 03:26:32.344 | INFO | Downloaded 34GiB to GPU in 1.966s (17.294GiB/s) | 20260507 03:26:37.469 | INFO | Analyzing chunk frames = [6724608,6725120) | 20260507 03:26:37.469 | INFO | MemInfo: { | 20260507 03:26:37.469 | INFO | "gpu" : " 17 GB", | 20260507 03:26:37.469 | INFO | "huge" : "", | 20260507 03:26:37.469 | INFO | "huge_shared" : " 27 GB", | 20260507 03:26:37.469 | INFO | "malloc" : " 14 GB", | 20260507 03:26:37.469 | INFO | "pinned" : " 29 GB" | 20260507 03:26:37.469 | INFO | } | 20260507 03:26:37.469 | INFO | Perf report for the last 5.12588 seconds: | 20260507 03:26:37.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:26:37.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:37.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:37.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:37.469 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 03:26:37.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:37.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:37.469 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 03:26:37.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:37.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:37.470 | INFO | Uploaded 36GiB to GPU in 1.93762s (18.5795GiB/s) | 20260507 03:26:37.470 | INFO | Downloaded 34GiB to GPU in 1.9716s (17.2449GiB/s) | 20260507 03:26:42.608 | INFO | Analyzing chunk frames = [6725120,6725632) | 20260507 03:26:42.608 | INFO | MemInfo: { | 20260507 03:26:42.608 | INFO | "gpu" : " 17 GB", | 20260507 03:26:42.608 | INFO | "huge" : "", | 20260507 03:26:42.608 | INFO | "huge_shared" : " 27 GB", | 20260507 03:26:42.608 | INFO | "malloc" : " 14 GB", | 20260507 03:26:42.608 | INFO | "pinned" : " 29 GB" | 20260507 03:26:42.608 | INFO | } | 20260507 03:26:42.609 | INFO | Perf report for the last 5.13919 seconds: | 20260507 03:26:42.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:26:42.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:42.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:42.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:42.609 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 03:26:42.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:42.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:42.609 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:26:42.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:42.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:42.609 | INFO | Uploaded 36GiB to GPU in 1.96248s (18.3441GiB/s) | 20260507 03:26:42.609 | INFO | Downloaded 34GiB to GPU in 1.94963s (17.4392GiB/s) | 20260507 03:26:42.609 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:26:42.609 | INFO | Total mem usage: 18 GiB | 20260507 03:26:42.609 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:26:42.609 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 03:26:42.609 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:26:42.609 | INFO | High Water Breakdown: | 20260507 03:26:42.609 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:26:42.609 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:26:42.609 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:26:42.609 | INFO | ------------------------------------------ | 20260507 03:26:42.609 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:26:42.609 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:26:42.609 | INFO | Total mem usage: 26.5 GiB | 20260507 03:26:42.609 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:26:42.609 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:26:42.609 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:26:42.609 | INFO | High Water Breakdown: | 20260507 03:26:42.609 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:26:42.609 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:26:42.609 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:26:42.609 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:26:42.609 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:26:42.609 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:26:42.609 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:26:42.609 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:26:42.610 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:26:42.610 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:26:42.610 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:26:42.610 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:26:42.610 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:26:42.610 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:26:42.610 | INFO | ------------------------------------------ | 20260507 03:26:42.610 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:26:42.610 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:26:42.610 | INFO | Total mem usage: 25.6 GiB | 20260507 03:26:42.610 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:26:42.610 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:26:42.610 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:26:42.610 | INFO | High Water Breakdown: | 20260507 03:26:42.610 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:26:42.610 | INFO | ------------------------------------------ | 20260507 03:26:42.610 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:26:42.610 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:26:42.610 | INFO | Total mem usage: 15.7 GiB | 20260507 03:26:42.610 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:26:42.610 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:26:42.610 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:26:42.610 | INFO | High Water Breakdown: | 20260507 03:26:42.610 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:26:42.610 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:26:42.610 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:26:42.610 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:26:42.610 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:26:42.610 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:26:42.610 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:26:42.610 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:26:42.610 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:26:42.610 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:26:42.610 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:26:42.610 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:26:42.610 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:26:42.610 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:26:42.610 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:26:42.610 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:26:42.610 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:26:42.610 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:26:42.610 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:26:42.610 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:26:42.610 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:26:42.610 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:26:42.610 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:26:42.610 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:26:42.610 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:26:42.610 | INFO | ------------------------------------------ | 20260507 03:26:42.610 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:26:42.610 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:26:42.610 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:26:42.610 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:26:42.610 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.55e+01, 1.33e+01 / 1.98e+00 | 20260507 03:26:42.610 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 9.90e-02, 9.40e-03 / 4.71e-03 | 20260507 03:26:42.610 | INFO | PerfReport: 11.68%, Baseline -- 3.60e+00 / 4.40e+00, 5.32e-01 / 3.44e-01 | 20260507 03:26:42.610 | INFO | PerfReport: 3.24%, Histogram -- 9.99e-01 / 1.43e+00, 1.74e-01 / 1.29e-01 | 20260507 03:26:42.610 | INFO | PerfReport: 16.16%, DME -- 3.98e+01 / 4.49e+01, 4.21e+00 / 2.40e+00 | 20260507 03:26:42.610 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.34e+00 / 1.66e+00 | 20260507 03:26:42.610 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.13e+00 / 2.67e+00, 3.72e-01 / 2.36e-01 | 20260507 03:26:42.610 | INFO | PerfReport: 3.04%, Metrics -- 9.37e-01 / 1.26e+00, 2.50e-01 / 1.42e-01 | 20260507 03:26:42.610 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.18e-01 / 3.70e-01, 5.05e-02 / 2.20e-02 | 20260507 03:26:42.610 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:26:42.610 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:26:42.610 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:26:42.610 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 03:26:42.610 | INFO | PerfReport: 21.66%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260507 03:26:42.610 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260507 03:26:42.610 | INFO | PerfReport: 32.01%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 03:26:42.610 | INFO | PerfReport: 1.41%, LockUpload -- 1.36e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260507 03:26:42.610 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.14e+00 / 0.00e+00 | 20260507 03:26:42.610 | INFO | PerfReport: 22.20%, LockCompute -- 2.14e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 03:26:42.610 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260507 03:26:42.706 | INFO | Starting a round of Baz IO | 20260507 03:26:45.812 | INFO | Populating RTMetrics report | 20260507 03:26:45.812 | INFO | RTMetrics report populated | 20260507 03:26:45.812 | INFO | Opening tmp RTMetrics json file | 20260507 03:26:45.832 | INFO | Serializing RTMetrics to json | 20260507 03:26:45.833 | INFO | Serialization done, swapping tmp file now... | 20260507 03:26:45.833 | INFO | RTMetrics done | 20260507 03:26:45.907 | INFO | Finished a round of Baz IO: Wrote 1.753 GiB in 3.2 seconds | 20260507 03:26:45.907 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.89% / 0.00% / 67.11% | 20260507 03:26:45.907 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.20049% / 3.97364e-06% / 6.75519e-05% | 20260507 03:26:45.907 | INFO | Baz IO overall: | 20260507 03:26:45.907 | INFO | BytesWritten : 3452308610151 | 20260507 03:26:45.907 | INFO | NumEvents : 1658850050061 | 20260507 03:26:47.752 | INFO | Analyzing chunk frames = [6725632,6726144) | 20260507 03:26:47.752 | INFO | MemInfo: { | 20260507 03:26:47.752 | INFO | "gpu" : " 17 GB", | 20260507 03:26:47.752 | INFO | "huge" : "", | 20260507 03:26:47.752 | INFO | "huge_shared" : " 27 GB", | 20260507 03:26:47.752 | INFO | "malloc" : " 14 GB", | 20260507 03:26:47.752 | INFO | "pinned" : " 29 GB" | 20260507 03:26:47.752 | INFO | } | 20260507 03:26:47.752 | INFO | Perf report for the last 5.14316 seconds: | 20260507 03:26:47.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:26:47.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:47.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:47.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:47.752 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 03:26:47.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:47.752 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 03:26:47.752 | INFO | PRE_HQ : 58.9%, 1.8, 55.4ms, 0 | 20260507 03:26:47.752 | INFO | BAZWRITER : 61.8%, 3.3, 105.6ms, 0 | 20260507 03:26:47.752 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:26:47.752 | INFO | Uploaded 36GiB to GPU in 2.03207s (17.7159GiB/s) | 20260507 03:26:47.752 | INFO | Downloaded 38GiB to GPU in 2.1774s (17.452GiB/s) | 20260507 03:26:52.885 | INFO | Analyzing chunk frames = [6726144,6726656) | 20260507 03:26:52.885 | INFO | MemInfo: { | 20260507 03:26:52.885 | INFO | "gpu" : " 17 GB", | 20260507 03:26:52.885 | INFO | "huge" : "", | 20260507 03:26:52.885 | INFO | "huge_shared" : " 27 GB", | 20260507 03:26:52.885 | INFO | "malloc" : " 14 GB", | 20260507 03:26:52.885 | INFO | "pinned" : " 29 GB" | 20260507 03:26:52.885 | INFO | } | 20260507 03:26:52.885 | INFO | Perf report for the last 5.13349 seconds: | 20260507 03:26:52.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:26:52.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:52.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:52.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:52.885 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:26:52.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:52.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:52.885 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:26:52.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:52.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:52.886 | INFO | Uploaded 36GiB to GPU in 2.03924s (17.6536GiB/s) | 20260507 03:26:52.886 | INFO | Downloaded 34GiB to GPU in 1.9445s (17.4852GiB/s) | 20260507 03:26:58.024 | INFO | Analyzing chunk frames = [6726656,6727168) | 20260507 03:26:58.024 | INFO | MemInfo: { | 20260507 03:26:58.024 | INFO | "gpu" : " 17 GB", | 20260507 03:26:58.024 | INFO | "huge" : "", | 20260507 03:26:58.025 | INFO | "huge_shared" : " 27 GB", | 20260507 03:26:58.025 | INFO | "malloc" : " 14 GB", | 20260507 03:26:58.025 | INFO | "pinned" : " 29 GB" | 20260507 03:26:58.025 | INFO | } | 20260507 03:26:58.025 | INFO | Perf report for the last 5.13948 seconds: | 20260507 03:26:58.025 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:26:58.025 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:58.025 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:58.025 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:58.025 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 03:26:58.025 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:58.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:58.025 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:26:58.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:58.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:26:58.025 | INFO | Uploaded 36GiB to GPU in 1.97951s (18.1864GiB/s) | 20260507 03:26:58.025 | INFO | Downloaded 34GiB to GPU in 1.94297s (17.499GiB/s) | 20260507 03:27:03.169 | INFO | Analyzing chunk frames = [6727168,6727680) | 20260507 03:27:03.169 | INFO | MemInfo: { | 20260507 03:27:03.169 | INFO | "gpu" : " 17 GB", | 20260507 03:27:03.169 | INFO | "huge" : "", | 20260507 03:27:03.169 | INFO | "huge_shared" : " 27 GB", | 20260507 03:27:03.169 | INFO | "malloc" : " 14 GB", | 20260507 03:27:03.169 | INFO | "pinned" : " 29 GB" | 20260507 03:27:03.169 | INFO | } | 20260507 03:27:03.169 | INFO | Perf report for the last 5.14457 seconds: | 20260507 03:27:03.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:27:03.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:03.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:03.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:03.169 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 03:27:03.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:03.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:03.169 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:27:03.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:03.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:03.170 | INFO | Uploaded 36GiB to GPU in 1.96679s (18.3039GiB/s) | 20260507 03:27:03.170 | INFO | Downloaded 34GiB to GPU in 1.91672s (17.7387GiB/s) | 20260507 03:27:08.306 | INFO | Analyzing chunk frames = [6727680,6728192) | 20260507 03:27:08.306 | INFO | MemInfo: { | 20260507 03:27:08.306 | INFO | "gpu" : " 17 GB", | 20260507 03:27:08.306 | INFO | "huge" : "", | 20260507 03:27:08.307 | INFO | "huge_shared" : " 27 GB", | 20260507 03:27:08.307 | INFO | "malloc" : " 14 GB", | 20260507 03:27:08.307 | INFO | "pinned" : " 29 GB" | 20260507 03:27:08.307 | INFO | } | 20260507 03:27:08.307 | INFO | Perf report for the last 5.13735 seconds: | 20260507 03:27:08.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:27:08.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:08.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:08.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:08.307 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 03:27:08.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:08.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:08.307 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:27:08.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:08.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:08.307 | INFO | Uploaded 36GiB to GPU in 2.0233s (17.7927GiB/s) | 20260507 03:27:08.307 | INFO | Downloaded 34GiB to GPU in 1.93324s (17.5871GiB/s) | 20260507 03:27:13.447 | INFO | Analyzing chunk frames = [6728192,6728704) | 20260507 03:27:13.447 | INFO | MemInfo: { | 20260507 03:27:13.447 | INFO | "gpu" : " 17 GB", | 20260507 03:27:13.447 | INFO | "huge" : "", | 20260507 03:27:13.447 | INFO | "huge_shared" : " 27 GB", | 20260507 03:27:13.447 | INFO | "malloc" : " 14 GB", | 20260507 03:27:13.447 | INFO | "pinned" : " 29 GB" | 20260507 03:27:13.447 | INFO | } | 20260507 03:27:13.447 | INFO | Perf report for the last 5.14066 seconds: | 20260507 03:27:13.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:27:13.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:13.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:13.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:13.447 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 03:27:13.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:13.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:13.447 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 03:27:13.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:13.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:13.448 | INFO | Uploaded 36GiB to GPU in 2.03479s (17.6922GiB/s) | 20260507 03:27:13.448 | INFO | Downloaded 34GiB to GPU in 1.95306s (17.4086GiB/s) | 20260507 03:27:18.584 | INFO | Analyzing chunk frames = [6728704,6729216) | 20260507 03:27:18.584 | INFO | MemInfo: { | 20260507 03:27:18.584 | INFO | "gpu" : " 17 GB", | 20260507 03:27:18.584 | INFO | "huge" : "", | 20260507 03:27:18.584 | INFO | "huge_shared" : " 27 GB", | 20260507 03:27:18.584 | INFO | "malloc" : " 14 GB", | 20260507 03:27:18.584 | INFO | "pinned" : " 29 GB" | 20260507 03:27:18.584 | INFO | } | 20260507 03:27:18.584 | INFO | Perf report for the last 5.13682 seconds: | 20260507 03:27:18.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:27:18.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:18.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:18.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:18.584 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:27:18.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:18.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:18.584 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 03:27:18.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:18.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:18.584 | INFO | Uploaded 36GiB to GPU in 1.96744s (18.2979GiB/s) | 20260507 03:27:18.584 | INFO | Downloaded 34GiB to GPU in 1.96948s (17.2634GiB/s) | 20260507 03:27:23.725 | INFO | Analyzing chunk frames = [6729216,6729728) | 20260507 03:27:23.726 | INFO | MemInfo: { | 20260507 03:27:23.726 | INFO | "gpu" : " 17 GB", | 20260507 03:27:23.726 | INFO | "huge" : "", | 20260507 03:27:23.726 | INFO | "huge_shared" : " 27 GB", | 20260507 03:27:23.726 | INFO | "malloc" : " 14 GB", | 20260507 03:27:23.726 | INFO | "pinned" : " 29 GB" | 20260507 03:27:23.726 | INFO | } | 20260507 03:27:23.726 | INFO | Perf report for the last 5.14166 seconds: | 20260507 03:27:23.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:27:23.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:23.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:23.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:23.726 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:27:23.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:23.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:23.726 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 03:27:23.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:23.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:23.726 | INFO | Uploaded 36GiB to GPU in 1.98936s (18.0962GiB/s) | 20260507 03:27:23.726 | INFO | Downloaded 34GiB to GPU in 1.97327s (17.2303GiB/s) | 20260507 03:27:23.823 | INFO | Starting a round of Baz IO | 20260507 03:27:26.925 | INFO | Populating RTMetrics report | 20260507 03:27:26.925 | INFO | RTMetrics report populated | 20260507 03:27:26.925 | INFO | Opening tmp RTMetrics json file | 20260507 03:27:26.926 | INFO | Serializing RTMetrics to json | 20260507 03:27:26.943 | INFO | Serialization done, swapping tmp file now... | 20260507 03:27:26.945 | INFO | RTMetrics done | 20260507 03:27:27.018 | INFO | Finished a round of Baz IO: Wrote 1.752 GiB in 3.195 seconds | 20260507 03:27:27.018 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.89% / 0.00% / 67.11% | 20260507 03:27:27.018 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.200113% / 0% / 6.35783e-05% | 20260507 03:27:27.019 | INFO | Baz IO overall: | 20260507 03:27:27.019 | INFO | BytesWritten : 3454189703460 | 20260507 03:27:27.019 | INFO | NumEvents : 1659737127013 | 20260507 03:27:28.854 | INFO | Analyzing chunk frames = [6729728,6730240) | 20260507 03:27:28.854 | INFO | MemInfo: { | 20260507 03:27:28.854 | INFO | "gpu" : " 17 GB", | 20260507 03:27:28.854 | INFO | "huge" : "", | 20260507 03:27:28.854 | INFO | "huge_shared" : " 27 GB", | 20260507 03:27:28.854 | INFO | "malloc" : " 14 GB", | 20260507 03:27:28.854 | INFO | "pinned" : " 29 GB" | 20260507 03:27:28.854 | INFO | } | 20260507 03:27:28.854 | INFO | Perf report for the last 5.12825 seconds: | 20260507 03:27:28.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:27:28.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:28.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:28.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:28.854 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260507 03:27:28.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:28.854 | INFO | RT_METRICS : 41.9%, 1.0, 21.9ms, 0 | 20260507 03:27:28.854 | INFO | PRE_HQ : 59.4%, 1.8, 55.3ms, 0 | 20260507 03:27:28.854 | INFO | BAZWRITER : 61.9%, 3.3, 103.6ms, 0 | 20260507 03:27:28.854 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:27:28.854 | INFO | Uploaded 36GiB to GPU in 2.03421s (17.6973GiB/s) | 20260507 03:27:28.854 | INFO | Downloaded 38GiB to GPU in 2.1628s (17.5698GiB/s) | 20260507 03:27:34.003 | INFO | Analyzing chunk frames = [6730240,6730752) | 20260507 03:27:34.003 | INFO | MemInfo: { | 20260507 03:27:34.003 | INFO | "gpu" : " 17 GB", | 20260507 03:27:34.003 | INFO | "huge" : "", | 20260507 03:27:34.003 | INFO | "huge_shared" : " 27 GB", | 20260507 03:27:34.003 | INFO | "malloc" : " 14 GB", | 20260507 03:27:34.003 | INFO | "pinned" : " 29 GB" | 20260507 03:27:34.003 | INFO | } | 20260507 03:27:34.003 | INFO | Perf report for the last 5.14916 seconds: | 20260507 03:27:34.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:27:34.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:34.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:34.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:34.003 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260507 03:27:34.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:34.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:34.003 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 03:27:34.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:34.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:34.004 | INFO | Uploaded 36GiB to GPU in 2.00481s (17.9568GiB/s) | 20260507 03:27:34.004 | INFO | Downloaded 34GiB to GPU in 1.93321s (17.5873GiB/s) | 20260507 03:27:39.148 | INFO | Analyzing chunk frames = [6730752,6731264) | 20260507 03:27:39.148 | INFO | MemInfo: { | 20260507 03:27:39.148 | INFO | "gpu" : " 17 GB", | 20260507 03:27:39.148 | INFO | "huge" : "", | 20260507 03:27:39.148 | INFO | "huge_shared" : " 27 GB", | 20260507 03:27:39.148 | INFO | "malloc" : " 14 GB", | 20260507 03:27:39.148 | INFO | "pinned" : " 29 GB" | 20260507 03:27:39.148 | INFO | } | 20260507 03:27:39.148 | INFO | Perf report for the last 5.14517 seconds: | 20260507 03:27:39.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:27:39.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:39.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:39.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:39.148 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260507 03:27:39.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:39.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:39.149 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:27:39.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:39.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:39.149 | INFO | Uploaded 36GiB to GPU in 2.01722s (17.8463GiB/s) | 20260507 03:27:39.149 | INFO | Downloaded 34GiB to GPU in 1.93583s (17.5635GiB/s) | 20260507 03:27:44.290 | INFO | Analyzing chunk frames = [6731264,6731776) | 20260507 03:27:44.290 | INFO | MemInfo: { | 20260507 03:27:44.290 | INFO | "gpu" : " 17 GB", | 20260507 03:27:44.290 | INFO | "huge" : "", | 20260507 03:27:44.290 | INFO | "huge_shared" : " 27 GB", | 20260507 03:27:44.290 | INFO | "malloc" : " 14 GB", | 20260507 03:27:44.290 | INFO | "pinned" : " 29 GB" | 20260507 03:27:44.290 | INFO | } | 20260507 03:27:44.290 | INFO | Perf report for the last 5.14196 seconds: | 20260507 03:27:44.290 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:27:44.290 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:44.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:44.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:44.290 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 03:27:44.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:44.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:44.290 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:27:44.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:44.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:44.291 | INFO | Uploaded 36GiB to GPU in 2.02218s (17.8025GiB/s) | 20260507 03:27:44.291 | INFO | Downloaded 34GiB to GPU in 1.93389s (17.5812GiB/s) | 20260507 03:27:49.423 | INFO | Analyzing chunk frames = [6731776,6732288) | 20260507 03:27:49.423 | INFO | MemInfo: { | 20260507 03:27:49.423 | INFO | "gpu" : " 17 GB", | 20260507 03:27:49.423 | INFO | "huge" : "", | 20260507 03:27:49.423 | INFO | "huge_shared" : " 27 GB", | 20260507 03:27:49.423 | INFO | "malloc" : " 14 GB", | 20260507 03:27:49.423 | INFO | "pinned" : " 29 GB" | 20260507 03:27:49.423 | INFO | } | 20260507 03:27:49.423 | INFO | Perf report for the last 5.13275 seconds: | 20260507 03:27:49.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:27:49.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:49.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:49.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:49.423 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260507 03:27:49.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:49.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:49.423 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:27:49.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:49.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:49.423 | INFO | Uploaded 36GiB to GPU in 2.03811s (17.6634GiB/s) | 20260507 03:27:49.424 | INFO | Downloaded 34GiB to GPU in 1.91402s (17.7637GiB/s) | 20260507 03:27:54.557 | INFO | Analyzing chunk frames = [6732288,6732800) | 20260507 03:27:54.557 | INFO | MemInfo: { | 20260507 03:27:54.557 | INFO | "gpu" : " 17 GB", | 20260507 03:27:54.558 | INFO | "huge" : "", | 20260507 03:27:54.558 | INFO | "huge_shared" : " 27 GB", | 20260507 03:27:54.558 | INFO | "malloc" : " 14 GB", | 20260507 03:27:54.558 | INFO | "pinned" : " 29 GB" | 20260507 03:27:54.558 | INFO | } | 20260507 03:27:54.558 | INFO | Perf report for the last 5.13455 seconds: | 20260507 03:27:54.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:27:54.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:54.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:54.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:54.558 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 03:27:54.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:54.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:54.558 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:27:54.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:54.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:54.558 | INFO | Uploaded 36GiB to GPU in 2.05732s (17.4985GiB/s) | 20260507 03:27:54.558 | INFO | Downloaded 34GiB to GPU in 1.95899s (17.3559GiB/s) | 20260507 03:27:59.699 | INFO | Analyzing chunk frames = [6732800,6733312) | 20260507 03:27:59.699 | INFO | MemInfo: { | 20260507 03:27:59.699 | INFO | "gpu" : " 17 GB", | 20260507 03:27:59.699 | INFO | "huge" : "", | 20260507 03:27:59.699 | INFO | "huge_shared" : " 27 GB", | 20260507 03:27:59.699 | INFO | "malloc" : " 14 GB", | 20260507 03:27:59.699 | INFO | "pinned" : " 29 GB" | 20260507 03:27:59.699 | INFO | } | 20260507 03:27:59.699 | INFO | Perf report for the last 5.14144 seconds: | 20260507 03:27:59.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:27:59.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:59.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:59.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:59.699 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:27:59.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:59.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:59.699 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:27:59.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:59.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:27:59.700 | INFO | Uploaded 36GiB to GPU in 1.95872s (18.3794GiB/s) | 20260507 03:27:59.700 | INFO | Downloaded 34GiB to GPU in 1.99426s (17.0489GiB/s) | 20260507 03:28:04.837 | INFO | Analyzing chunk frames = [6733312,6733824) | 20260507 03:28:04.837 | INFO | MemInfo: { | 20260507 03:28:04.837 | INFO | "gpu" : " 17 GB", | 20260507 03:28:04.837 | INFO | "huge" : "", | 20260507 03:28:04.837 | INFO | "huge_shared" : " 27 GB", | 20260507 03:28:04.837 | INFO | "malloc" : " 14 GB", | 20260507 03:28:04.837 | INFO | "pinned" : " 29 GB" | 20260507 03:28:04.837 | INFO | } | 20260507 03:28:04.837 | INFO | Perf report for the last 5.13802 seconds: | 20260507 03:28:04.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:28:04.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:04.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:04.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:04.837 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:28:04.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:04.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:04.837 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 03:28:04.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:04.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:04.837 | INFO | Uploaded 36GiB to GPU in 2.00142s (17.9872GiB/s) | 20260507 03:28:04.838 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:28:04.838 | INFO | Total mem usage: 18 GiB | 20260507 03:28:04.838 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:28:04.838 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 03:28:04.838 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:28:04.838 | INFO | High Water Breakdown: | 20260507 03:28:04.838 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:28:04.838 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:28:04.838 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:28:04.838 | INFO | ------------------------------------------ | 20260507 03:28:04.838 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:28:04.838 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:28:04.838 | INFO | Total mem usage: 26.5 GiB | 20260507 03:28:04.838 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:28:04.838 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:28:04.838 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:28:04.838 | INFO | Downloaded 34GiB to GPU in 1.98429s (17.1346GiB/s) | 20260507 03:28:04.838 | INFO | High Water Breakdown: | 20260507 03:28:04.838 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:28:04.838 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:28:04.838 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:28:04.838 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:28:04.838 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:28:04.838 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:28:04.838 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:28:04.838 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:28:04.838 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:28:04.838 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:28:04.838 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:28:04.838 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:28:04.838 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:28:04.838 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:28:04.838 | INFO | ------------------------------------------ | 20260507 03:28:04.838 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:28:04.838 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:28:04.838 | INFO | Total mem usage: 25.6 GiB | 20260507 03:28:04.838 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:28:04.838 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 03:28:04.838 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 03:28:04.838 | INFO | High Water Breakdown: | 20260507 03:28:04.838 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 03:28:04.838 | INFO | ------------------------------------------ | 20260507 03:28:04.838 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:28:04.838 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:28:04.838 | INFO | Total mem usage: 15.7 GiB | 20260507 03:28:04.838 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:28:04.838 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:28:04.838 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:28:04.838 | INFO | High Water Breakdown: | 20260507 03:28:04.838 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:28:04.838 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:28:04.838 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:28:04.838 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:28:04.838 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:28:04.838 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:28:04.838 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:28:04.838 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:28:04.838 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:28:04.838 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:28:04.838 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:28:04.838 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:28:04.838 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:28:04.838 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:28:04.838 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:28:04.839 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:28:04.839 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:28:04.839 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:28:04.839 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:28:04.839 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:28:04.839 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:28:04.839 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:28:04.839 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:28:04.839 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:28:04.839 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:28:04.839 | INFO | ------------------------------------------ | 20260507 03:28:04.839 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:28:04.839 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:28:04.839 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:28:04.839 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:28:04.839 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.51e+01, 1.33e+01 / 1.67e+00 | 20260507 03:28:04.839 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.02e-01, 9.46e-03 / 6.05e-03 | 20260507 03:28:04.839 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.41e+00, 5.09e-01 / 2.60e-01 | 20260507 03:28:04.839 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.45e+00, 1.65e-01 / 1.20e-01 | 20260507 03:28:04.839 | INFO | PerfReport: 16.13%, DME -- 3.97e+01 / 4.45e+01, 3.89e+00 / 1.93e+00 | 20260507 03:28:04.839 | INFO | PerfReport: 57.62%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.25e+00 / 1.46e+00 | 20260507 03:28:04.839 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.15e+00 / 2.70e+00, 3.84e-01 / 2.06e-01 | 20260507 03:28:04.839 | INFO | PerfReport: 3.05%, Metrics -- 9.38e-01 / 1.25e+00, 2.47e-01 / 1.18e-01 | 20260507 03:28:04.839 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.18e-01 / 3.69e-01, 4.97e-02 / 1.90e-02 | 20260507 03:28:04.839 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:28:04.839 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:28:04.839 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:28:04.839 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 03:28:04.839 | INFO | PerfReport: 21.75%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260507 03:28:04.839 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 03:28:04.839 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 03:28:04.839 | INFO | PerfReport: 1.50%, LockUpload -- 1.45e+00 / 0.00e+00, 4.96e+00 / 0.00e+00 | 20260507 03:28:04.839 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260507 03:28:04.839 | INFO | PerfReport: 22.11%, LockCompute -- 2.13e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 03:28:04.839 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.29e-02 / 0.00e+00 | 20260507 03:28:04.934 | INFO | Starting a round of Baz IO | 20260507 03:28:08.040 | INFO | Populating RTMetrics report | 20260507 03:28:08.040 | INFO | RTMetrics report populated | 20260507 03:28:08.040 | INFO | Opening tmp RTMetrics json file | 20260507 03:28:08.041 | INFO | Serializing RTMetrics to json | 20260507 03:28:08.042 | INFO | Serialization done, swapping tmp file now... | 20260507 03:28:08.042 | INFO | RTMetrics done | 20260507 03:28:08.132 | INFO | Finished a round of Baz IO: Wrote 1.753 GiB in 3.198 seconds | 20260507 03:28:08.133 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.89% / 0.00% / 67.11% | 20260507 03:28:08.133 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.199942% / 3.97364e-06% / 5.96046e-05% | 20260507 03:28:08.133 | INFO | Baz IO overall: | 20260507 03:28:08.133 | INFO | BytesWritten : 3456071871601 | 20260507 03:28:08.133 | INFO | NumEvents : 1660625047068 | 20260507 03:28:09.981 | INFO | Analyzing chunk frames = [6733824,6734336) | 20260507 03:28:09.981 | INFO | MemInfo: { | 20260507 03:28:09.981 | INFO | "gpu" : " 17 GB", | 20260507 03:28:09.981 | INFO | "huge" : "", | 20260507 03:28:09.981 | INFO | "huge_shared" : " 27 GB", | 20260507 03:28:09.981 | INFO | "malloc" : " 14 GB", | 20260507 03:28:09.981 | INFO | "pinned" : " 29 GB" | 20260507 03:28:09.981 | INFO | } | 20260507 03:28:09.981 | INFO | Perf report for the last 5.1438 seconds: | 20260507 03:28:09.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:28:09.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:09.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:09.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:09.981 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 03:28:09.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:09.981 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 03:28:09.981 | INFO | PRE_HQ : 59.1%, 1.8, 55.3ms, 0 | 20260507 03:28:09.981 | INFO | BAZWRITER : 61.8%, 3.3, 106.3ms, 0 | 20260507 03:28:09.981 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:28:09.981 | INFO | Uploaded 36GiB to GPU in 2.03121s (17.7234GiB/s) | 20260507 03:28:09.981 | INFO | Downloaded 38GiB to GPU in 2.18052s (17.427GiB/s) | 20260507 03:28:15.119 | INFO | Analyzing chunk frames = [6734336,6734848) | 20260507 03:28:15.119 | INFO | MemInfo: { | 20260507 03:28:15.119 | INFO | "gpu" : " 17 GB", | 20260507 03:28:15.119 | INFO | "huge" : "", | 20260507 03:28:15.119 | INFO | "huge_shared" : " 27 GB", | 20260507 03:28:15.119 | INFO | "malloc" : " 14 GB", | 20260507 03:28:15.119 | INFO | "pinned" : " 29 GB" | 20260507 03:28:15.119 | INFO | } | 20260507 03:28:15.119 | INFO | Perf report for the last 5.13838 seconds: | 20260507 03:28:15.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:28:15.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:15.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:15.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:15.119 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:28:15.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:15.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:15.120 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:28:15.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:15.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:15.120 | INFO | Uploaded 36GiB to GPU in 2.03929s (17.6532GiB/s) | 20260507 03:28:15.120 | INFO | Downloaded 34GiB to GPU in 1.94147s (17.5125GiB/s) | 20260507 03:28:20.258 | INFO | Analyzing chunk frames = [6734848,6735360) | 20260507 03:28:20.258 | INFO | MemInfo: { | 20260507 03:28:20.258 | INFO | "gpu" : " 17 GB", | 20260507 03:28:20.258 | INFO | "huge" : "", | 20260507 03:28:20.258 | INFO | "huge_shared" : " 27 GB", | 20260507 03:28:20.258 | INFO | "malloc" : " 14 GB", | 20260507 03:28:20.258 | INFO | "pinned" : " 29 GB" | 20260507 03:28:20.258 | INFO | } | 20260507 03:28:20.258 | INFO | Perf report for the last 5.13883 seconds: | 20260507 03:28:20.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:28:20.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:20.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:20.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:20.258 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 03:28:20.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:20.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:20.258 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:28:20.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:20.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:20.258 | INFO | Uploaded 36GiB to GPU in 2.01898s (17.8308GiB/s) | 20260507 03:28:20.259 | INFO | Downloaded 34GiB to GPU in 1.94395s (17.4902GiB/s) | 20260507 03:28:25.395 | INFO | Analyzing chunk frames = [6735360,6735872) | 20260507 03:28:25.395 | INFO | MemInfo: { | 20260507 03:28:25.395 | INFO | "gpu" : " 17 GB", | 20260507 03:28:25.395 | INFO | "huge" : "", | 20260507 03:28:25.395 | INFO | "huge_shared" : " 27 GB", | 20260507 03:28:25.395 | INFO | "malloc" : " 14 GB", | 20260507 03:28:25.395 | INFO | "pinned" : " 29 GB" | 20260507 03:28:25.395 | INFO | } | 20260507 03:28:25.395 | INFO | Perf report for the last 5.13684 seconds: | 20260507 03:28:25.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:28:25.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:25.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:25.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:25.395 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 03:28:25.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:25.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:25.395 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:28:25.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:25.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:25.395 | INFO | Uploaded 36GiB to GPU in 1.97816s (18.1987GiB/s) | 20260507 03:28:25.395 | INFO | Downloaded 34GiB to GPU in 1.92261s (17.6843GiB/s) | 20260507 03:28:30.533 | INFO | Analyzing chunk frames = [6735872,6736384) | 20260507 03:28:30.533 | INFO | MemInfo: { | 20260507 03:28:30.533 | INFO | "gpu" : " 17 GB", | 20260507 03:28:30.533 | INFO | "huge" : "", | 20260507 03:28:30.533 | INFO | "huge_shared" : " 27 GB", | 20260507 03:28:30.533 | INFO | "malloc" : " 14 GB", | 20260507 03:28:30.533 | INFO | "pinned" : " 29 GB" | 20260507 03:28:30.533 | INFO | } | 20260507 03:28:30.533 | INFO | Perf report for the last 5.13783 seconds: | 20260507 03:28:30.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:28:30.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:30.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:30.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:30.533 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 03:28:30.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:30.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:30.533 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:28:30.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:30.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:30.533 | INFO | Uploaded 36GiB to GPU in 2.01002s (17.9103GiB/s) | 20260507 03:28:30.533 | INFO | Downloaded 34GiB to GPU in 1.9371s (17.552GiB/s) | 20260507 03:28:35.670 | INFO | Analyzing chunk frames = [6736384,6736896) | 20260507 03:28:35.670 | INFO | MemInfo: { | 20260507 03:28:35.670 | INFO | "gpu" : " 17 GB", | 20260507 03:28:35.670 | INFO | "huge" : "", | 20260507 03:28:35.670 | INFO | "huge_shared" : " 27 GB", | 20260507 03:28:35.670 | INFO | "malloc" : " 14 GB", | 20260507 03:28:35.670 | INFO | "pinned" : " 29 GB" | 20260507 03:28:35.670 | INFO | } | 20260507 03:28:35.670 | INFO | Perf report for the last 5.13689 seconds: | 20260507 03:28:35.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:28:35.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:35.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:35.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:35.670 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 03:28:35.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:35.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:35.670 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 03:28:35.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:35.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:35.670 | INFO | Uploaded 36GiB to GPU in 2.02489s (17.7788GiB/s) | 20260507 03:28:35.670 | INFO | Downloaded 34GiB to GPU in 1.93937s (17.5314GiB/s) | 20260507 03:28:40.811 | INFO | Analyzing chunk frames = [6736896,6737408) | 20260507 03:28:40.811 | INFO | MemInfo: { | 20260507 03:28:40.811 | INFO | "gpu" : " 17 GB", | 20260507 03:28:40.811 | INFO | "huge" : "", | 20260507 03:28:40.811 | INFO | "huge_shared" : " 27 GB", | 20260507 03:28:40.811 | INFO | "malloc" : " 14 GB", | 20260507 03:28:40.811 | INFO | "pinned" : " 29 GB" | 20260507 03:28:40.811 | INFO | } | 20260507 03:28:40.811 | INFO | Perf report for the last 5.14145 seconds: | 20260507 03:28:40.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:28:40.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:40.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:40.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:40.811 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:28:40.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:40.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:40.811 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 03:28:40.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:40.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:40.812 | INFO | Uploaded 36GiB to GPU in 1.9844s (18.1415GiB/s) | 20260507 03:28:40.812 | INFO | Downloaded 34GiB to GPU in 1.98633s (17.117GiB/s) | 20260507 03:28:45.958 | INFO | Analyzing chunk frames = [6737408,6737920) | 20260507 03:28:45.958 | INFO | MemInfo: { | 20260507 03:28:45.958 | INFO | "gpu" : " 17 GB", | 20260507 03:28:45.958 | INFO | "huge" : "", | 20260507 03:28:45.958 | INFO | "huge_shared" : " 27 GB", | 20260507 03:28:45.959 | INFO | "malloc" : " 14 GB", | 20260507 03:28:45.959 | INFO | "pinned" : " 29 GB" | 20260507 03:28:45.959 | INFO | } | 20260507 03:28:45.959 | INFO | Perf report for the last 5.14739 seconds: | 20260507 03:28:45.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:28:45.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:45.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:45.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:45.959 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 03:28:45.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:45.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:45.959 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 03:28:45.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:45.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:45.959 | INFO | Uploaded 36GiB to GPU in 1.96643s (18.3073GiB/s) | 20260507 03:28:45.959 | INFO | Downloaded 34GiB to GPU in 1.96967s (17.2618GiB/s) | 20260507 03:28:46.055 | INFO | Starting a round of Baz IO | 20260507 03:28:49.159 | INFO | Populating RTMetrics report | 20260507 03:28:49.159 | INFO | RTMetrics report populated | 20260507 03:28:49.159 | INFO | Opening tmp RTMetrics json file | 20260507 03:28:49.163 | INFO | Serializing RTMetrics to json | 20260507 03:28:49.177 | INFO | Serialization done, swapping tmp file now... | 20260507 03:28:49.178 | INFO | RTMetrics done | 20260507 03:28:49.247 | INFO | Finished a round of Baz IO: Wrote 1.752 GiB in 3.191 seconds | 20260507 03:28:49.247 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.89% / 0.00% / 67.11% | 20260507 03:28:49.247 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.199823% / 0% / 7.94729e-05% | 20260507 03:28:49.247 | INFO | Baz IO overall: | 20260507 03:28:49.247 | INFO | BytesWritten : 3457953487850 | 20260507 03:28:49.247 | INFO | NumEvents : 1661512245350 | 20260507 03:28:51.099 | INFO | Analyzing chunk frames = [6737920,6738432) | 20260507 03:28:51.099 | INFO | MemInfo: { | 20260507 03:28:51.099 | INFO | "gpu" : " 17 GB", | 20260507 03:28:51.099 | INFO | "huge" : "", | 20260507 03:28:51.099 | INFO | "huge_shared" : " 27 GB", | 20260507 03:28:51.099 | INFO | "malloc" : " 14 GB", | 20260507 03:28:51.099 | INFO | "pinned" : " 29 GB" | 20260507 03:28:51.099 | INFO | } | 20260507 03:28:51.099 | INFO | Perf report for the last 5.14066 seconds: | 20260507 03:28:51.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:28:51.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:51.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:51.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:51.099 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 03:28:51.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:51.099 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:28:51.099 | INFO | PRE_HQ : 59.3%, 1.8, 55.4ms, 0 | 20260507 03:28:51.099 | INFO | BAZWRITER : 61.7%, 3.3, 105.9ms, 0 | 20260507 03:28:51.100 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:28:51.100 | INFO | Uploaded 36GiB to GPU in 2.03906s (17.6552GiB/s) | 20260507 03:28:51.100 | INFO | Downloaded 38GiB to GPU in 2.17515s (17.4701GiB/s) | 20260507 03:28:56.234 | INFO | Analyzing chunk frames = [6738432,6738944) | 20260507 03:28:56.234 | INFO | MemInfo: { | 20260507 03:28:56.234 | INFO | "gpu" : " 17 GB", | 20260507 03:28:56.234 | INFO | "huge" : "", | 20260507 03:28:56.234 | INFO | "huge_shared" : " 27 GB", | 20260507 03:28:56.234 | INFO | "malloc" : " 14 GB", | 20260507 03:28:56.234 | INFO | "pinned" : " 29 GB" | 20260507 03:28:56.234 | INFO | } | 20260507 03:28:56.234 | INFO | Perf report for the last 5.13511 seconds: | 20260507 03:28:56.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:28:56.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:56.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:56.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:56.235 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:28:56.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:56.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:56.235 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:28:56.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:56.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:28:56.235 | INFO | Uploaded 36GiB to GPU in 1.98708s (18.1171GiB/s) | 20260507 03:28:56.235 | INFO | Downloaded 34GiB to GPU in 1.92418s (17.6699GiB/s) | 20260507 03:29:01.369 | INFO | Analyzing chunk frames = [6738944,6739456) | 20260507 03:29:01.369 | INFO | MemInfo: { | 20260507 03:29:01.369 | INFO | "gpu" : " 17 GB", | 20260507 03:29:01.369 | INFO | "huge" : "", | 20260507 03:29:01.369 | INFO | "huge_shared" : " 27 GB", | 20260507 03:29:01.369 | INFO | "malloc" : " 14 GB", | 20260507 03:29:01.370 | INFO | "pinned" : " 29 GB" | 20260507 03:29:01.370 | INFO | } | 20260507 03:29:01.370 | INFO | Perf report for the last 5.1351 seconds: | 20260507 03:29:01.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:29:01.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:01.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:01.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:01.370 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 03:29:01.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:01.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:01.370 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:29:01.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:01.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:01.370 | INFO | Uploaded 36GiB to GPU in 2.0237s (17.7892GiB/s) | 20260507 03:29:01.370 | INFO | Downloaded 34GiB to GPU in 1.95087s (17.4282GiB/s) | 20260507 03:29:06.512 | INFO | Analyzing chunk frames = [6739456,6739968) | 20260507 03:29:06.512 | INFO | MemInfo: { | 20260507 03:29:06.512 | INFO | "gpu" : " 17 GB", | 20260507 03:29:06.512 | INFO | "huge" : "", | 20260507 03:29:06.512 | INFO | "huge_shared" : " 27 GB", | 20260507 03:29:06.512 | INFO | "malloc" : " 14 GB", | 20260507 03:29:06.512 | INFO | "pinned" : " 29 GB" | 20260507 03:29:06.512 | INFO | } | 20260507 03:29:06.513 | INFO | Perf report for the last 5.14291 seconds: | 20260507 03:29:06.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:29:06.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:06.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:06.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:06.513 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 03:29:06.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:06.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:06.513 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:29:06.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:06.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:06.513 | INFO | Uploaded 36GiB to GPU in 1.98239s (18.1599GiB/s) | 20260507 03:29:06.513 | INFO | Downloaded 34GiB to GPU in 1.9062s (17.8366GiB/s) | 20260507 03:29:11.649 | INFO | Analyzing chunk frames = [6739968,6740480) | 20260507 03:29:11.649 | INFO | MemInfo: { | 20260507 03:29:11.649 | INFO | "gpu" : " 17 GB", | 20260507 03:29:11.649 | INFO | "huge" : "", | 20260507 03:29:11.649 | INFO | "huge_shared" : " 27 GB", | 20260507 03:29:11.649 | INFO | "malloc" : " 14 GB", | 20260507 03:29:11.650 | INFO | "pinned" : " 29 GB" | 20260507 03:29:11.650 | INFO | } | 20260507 03:29:11.650 | INFO | Perf report for the last 5.13715 seconds: | 20260507 03:29:11.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:29:11.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:11.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:11.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:11.650 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 03:29:11.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:11.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:11.650 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:29:11.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:11.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:11.650 | INFO | Uploaded 36GiB to GPU in 2.02719s (17.7586GiB/s) | 20260507 03:29:11.650 | INFO | Downloaded 34GiB to GPU in 1.94293s (17.4994GiB/s) | 20260507 03:29:16.792 | INFO | Analyzing chunk frames = [6740480,6740992) | 20260507 03:29:16.792 | INFO | MemInfo: { | 20260507 03:29:16.792 | INFO | "gpu" : " 17 GB", | 20260507 03:29:16.792 | INFO | "huge" : "", | 20260507 03:29:16.792 | INFO | "huge_shared" : " 27 GB", | 20260507 03:29:16.792 | INFO | "malloc" : " 14 GB", | 20260507 03:29:16.792 | INFO | "pinned" : " 29 GB" | 20260507 03:29:16.792 | INFO | } | 20260507 03:29:16.792 | INFO | Perf report for the last 5.14253 seconds: | 20260507 03:29:16.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:29:16.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:16.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:16.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:16.792 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 03:29:16.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:16.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:16.792 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 03:29:16.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:16.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:16.793 | INFO | Uploaded 36GiB to GPU in 2.03579s (17.6835GiB/s) | 20260507 03:29:16.793 | INFO | Downloaded 34GiB to GPU in 1.9359s (17.5629GiB/s) | 20260507 03:29:21.930 | INFO | Analyzing chunk frames = [6740992,6741504) | 20260507 03:29:21.930 | INFO | MemInfo: { | 20260507 03:29:21.930 | INFO | "gpu" : " 17 GB", | 20260507 03:29:21.930 | INFO | "huge" : "", | 20260507 03:29:21.930 | INFO | "huge_shared" : " 27 GB", | 20260507 03:29:21.930 | INFO | "malloc" : " 14 GB", | 20260507 03:29:21.930 | INFO | "pinned" : " 29 GB" | 20260507 03:29:21.930 | INFO | } | 20260507 03:29:21.930 | INFO | Perf report for the last 5.13806 seconds: | 20260507 03:29:21.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:29:21.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:21.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:21.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:21.930 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:29:21.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:21.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:21.930 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 03:29:21.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:21.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:21.931 | INFO | Uploaded 36GiB to GPU in 1.98114s (18.1714GiB/s) | 20260507 03:29:21.931 | INFO | Downloaded 34GiB to GPU in 1.97297s (17.2329GiB/s) | 20260507 03:29:27.077 | INFO | Analyzing chunk frames = [6741504,6742016) | 20260507 03:29:27.077 | INFO | MemInfo: { | 20260507 03:29:27.077 | INFO | "gpu" : " 17 GB", | 20260507 03:29:27.077 | INFO | "huge" : "", | 20260507 03:29:27.077 | INFO | "huge_shared" : " 27 GB", | 20260507 03:29:27.077 | INFO | "malloc" : " 14 GB", | 20260507 03:29:27.077 | INFO | "pinned" : " 29 GB" | 20260507 03:29:27.077 | INFO | } | 20260507 03:29:27.077 | INFO | Perf report for the last 5.14631 seconds: | 20260507 03:29:27.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:29:27.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:27.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:27.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:27.077 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 03:29:27.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:27.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:27.077 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 03:29:27.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:27.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:27.077 | INFO | Uploaded 36GiB to GPU in 1.95782s (18.3878GiB/s) | 20260507 03:29:27.077 | INFO | Downloaded 34GiB to GPU in 1.96522s (17.3009GiB/s) | 20260507 03:29:27.077 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:29:27.077 | INFO | Total mem usage: 18 GiB | 20260507 03:29:27.077 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:29:27.077 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -26.3%) | 20260507 03:29:27.077 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:29:27.077 | INFO | High Water Breakdown: | 20260507 03:29:27.077 | INFO | 60.2% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:29:27.077 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:29:27.077 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:29:27.077 | INFO | ------------------------------------------ | 20260507 03:29:27.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:29:27.077 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:29:27.077 | INFO | Total mem usage: 26.5 GiB | 20260507 03:29:27.077 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:29:27.077 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:29:27.077 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:29:27.077 | INFO | High Water Breakdown: | 20260507 03:29:27.077 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:29:27.077 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:29:27.077 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:29:27.077 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:29:27.077 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:29:27.077 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:29:27.077 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:29:27.077 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:29:27.077 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:29:27.077 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:29:27.077 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:29:27.077 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:29:27.077 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:29:27.077 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:29:27.077 | INFO | ------------------------------------------ | 20260507 03:29:27.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:29:27.077 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:29:27.078 | INFO | Total mem usage: 25.6 GiB | 20260507 03:29:27.078 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:29:27.078 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:29:27.078 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:29:27.078 | INFO | High Water Breakdown: | 20260507 03:29:27.078 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:29:27.078 | INFO | ------------------------------------------ | 20260507 03:29:27.078 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:29:27.078 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:29:27.078 | INFO | Total mem usage: 15.7 GiB | 20260507 03:29:27.078 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:29:27.078 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:29:27.078 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:29:27.078 | INFO | High Water Breakdown: | 20260507 03:29:27.078 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:29:27.078 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:29:27.078 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:29:27.078 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:29:27.078 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:29:27.078 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:29:27.078 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:29:27.078 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:29:27.078 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:29:27.078 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:29:27.078 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:29:27.078 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:29:27.078 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:29:27.078 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:29:27.078 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:29:27.078 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:29:27.078 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:29:27.078 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:29:27.078 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:29:27.078 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:29:27.078 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:29:27.078 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:29:27.078 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:29:27.078 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:29:27.078 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:29:27.078 | INFO | ------------------------------------------ | 20260507 03:29:27.078 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:29:27.078 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:29:27.078 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:29:27.078 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:29:27.078 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.56e+01, 1.34e+01 / 2.13e+00 | 20260507 03:29:27.078 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.00e-01, 9.72e-03 / 5.37e-03 | 20260507 03:29:27.078 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.45e+00, 5.21e-01 / 2.86e-01 | 20260507 03:29:27.078 | INFO | PerfReport: 3.24%, Histogram -- 9.98e-01 / 1.47e+00, 1.73e-01 / 1.23e-01 | 20260507 03:29:27.078 | INFO | PerfReport: 16.15%, DME -- 3.98e+01 / 4.48e+01, 4.18e+00 / 2.34e+00 | 20260507 03:29:27.078 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.33e+00 / 1.64e+00 | 20260507 03:29:27.078 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.72e+00, 3.81e-01 / 2.05e-01 | 20260507 03:29:27.078 | INFO | PerfReport: 3.04%, Metrics -- 9.37e-01 / 1.26e+00, 2.51e-01 / 1.18e-01 | 20260507 03:29:27.078 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.17e-01 / 3.72e-01, 5.08e-02 / 2.22e-02 | 20260507 03:29:27.078 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:29:27.078 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:29:27.078 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:29:27.078 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 03:29:27.078 | INFO | PerfReport: 21.69%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260507 03:29:27.078 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260507 03:29:27.078 | INFO | PerfReport: 32.00%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 03:29:27.078 | INFO | PerfReport: 1.42%, LockUpload -- 1.37e+00 / 0.00e+00, 4.77e+00 / 0.00e+00 | 20260507 03:29:27.078 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.15e+00 / 0.00e+00 | 20260507 03:29:27.078 | INFO | PerfReport: 22.19%, LockCompute -- 2.14e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 03:29:27.078 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.90e-02 / 0.00e+00 | 20260507 03:29:27.174 | INFO | Starting a round of Baz IO | 20260507 03:29:30.272 | INFO | Populating RTMetrics report | 20260507 03:29:30.273 | INFO | RTMetrics report populated | 20260507 03:29:30.273 | INFO | Opening tmp RTMetrics json file | 20260507 03:29:30.291 | INFO | Serializing RTMetrics to json | 20260507 03:29:30.292 | INFO | Serialization done, swapping tmp file now... | 20260507 03:29:30.293 | INFO | RTMetrics done | 20260507 03:29:30.360 | INFO | Finished a round of Baz IO: Wrote 1.753 GiB in 3.186 seconds | 20260507 03:29:30.360 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.89% / 0.00% / 67.11% | 20260507 03:29:30.360 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.199803% / 0% / 5.96046e-05% | 20260507 03:29:30.360 | INFO | Baz IO overall: | 20260507 03:29:30.360 | INFO | BytesWritten : 3459835923811 | 20260507 03:29:30.360 | INFO | NumEvents : 1662400101893 | 20260507 03:29:32.208 | INFO | Analyzing chunk frames = [6742016,6742528) | 20260507 03:29:32.208 | INFO | MemInfo: { | 20260507 03:29:32.208 | INFO | "gpu" : " 17 GB", | 20260507 03:29:32.208 | INFO | "huge" : "", | 20260507 03:29:32.208 | INFO | "huge_shared" : " 27 GB", | 20260507 03:29:32.208 | INFO | "malloc" : " 14 GB", | 20260507 03:29:32.208 | INFO | "pinned" : " 29 GB" | 20260507 03:29:32.208 | INFO | } | 20260507 03:29:32.208 | INFO | Perf report for the last 5.13143 seconds: | 20260507 03:29:32.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:29:32.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:32.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:32.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:32.208 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 03:29:32.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:32.208 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 03:29:32.208 | INFO | PRE_HQ : 59.2%, 1.8, 55.5ms, 0 | 20260507 03:29:32.208 | INFO | BAZWRITER : 61.7%, 3.3, 104.3ms, 0 | 20260507 03:29:32.208 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:29:32.208 | INFO | Uploaded 36GiB to GPU in 2.03653s (17.6772GiB/s) | 20260507 03:29:32.208 | INFO | Downloaded 38GiB to GPU in 2.18136s (17.4204GiB/s) | 20260507 03:29:37.347 | INFO | Analyzing chunk frames = [6742528,6743040) | 20260507 03:29:37.347 | INFO | MemInfo: { | 20260507 03:29:37.347 | INFO | "gpu" : " 17 GB", | 20260507 03:29:37.347 | INFO | "huge" : "", | 20260507 03:29:37.347 | INFO | "huge_shared" : " 27 GB", | 20260507 03:29:37.347 | INFO | "malloc" : " 14 GB", | 20260507 03:29:37.347 | INFO | "pinned" : " 29 GB" | 20260507 03:29:37.347 | INFO | } | 20260507 03:29:37.347 | INFO | Perf report for the last 5.13877 seconds: | 20260507 03:29:37.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:29:37.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:37.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:37.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:37.347 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:29:37.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:37.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:37.347 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:29:37.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:37.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:37.347 | INFO | Uploaded 36GiB to GPU in 1.98059s (18.1764GiB/s) | 20260507 03:29:37.347 | INFO | Downloaded 34GiB to GPU in 1.93417s (17.5786GiB/s) | 20260507 03:29:42.487 | INFO | Analyzing chunk frames = [6743040,6743552) | 20260507 03:29:42.487 | INFO | MemInfo: { | 20260507 03:29:42.487 | INFO | "gpu" : " 17 GB", | 20260507 03:29:42.487 | INFO | "huge" : "", | 20260507 03:29:42.487 | INFO | "huge_shared" : " 27 GB", | 20260507 03:29:42.488 | INFO | "malloc" : " 14 GB", | 20260507 03:29:42.488 | INFO | "pinned" : " 29 GB" | 20260507 03:29:42.488 | INFO | } | 20260507 03:29:42.488 | INFO | Perf report for the last 5.14084 seconds: | 20260507 03:29:42.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:29:42.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:42.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:42.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:42.488 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 03:29:42.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:42.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:42.488 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:29:42.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:42.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:42.488 | INFO | Uploaded 36GiB to GPU in 1.99693s (18.0277GiB/s) | 20260507 03:29:42.488 | INFO | Downloaded 34GiB to GPU in 1.94418s (17.4881GiB/s) | 20260507 03:29:47.620 | INFO | Analyzing chunk frames = [6743552,6744064) | 20260507 03:29:47.620 | INFO | MemInfo: { | 20260507 03:29:47.620 | INFO | "gpu" : " 17 GB", | 20260507 03:29:47.620 | INFO | "huge" : "", | 20260507 03:29:47.620 | INFO | "huge_shared" : " 27 GB", | 20260507 03:29:47.620 | INFO | "malloc" : " 14 GB", | 20260507 03:29:47.620 | INFO | "pinned" : " 29 GB" | 20260507 03:29:47.620 | INFO | } | 20260507 03:29:47.621 | INFO | Perf report for the last 5.13282 seconds: | 20260507 03:29:47.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:29:47.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:47.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:47.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:47.621 | INFO | ANALYSIS : 59.7%, 3.0, 92.4ms, 0 | 20260507 03:29:47.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:47.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:47.621 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 03:29:47.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:47.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:47.621 | INFO | Uploaded 36GiB to GPU in 1.9713s (18.2621GiB/s) | 20260507 03:29:47.621 | INFO | Downloaded 34GiB to GPU in 1.9073s (17.8262GiB/s) | 20260507 03:29:52.765 | INFO | Analyzing chunk frames = [6744064,6744576) | 20260507 03:29:52.765 | INFO | MemInfo: { | 20260507 03:29:52.765 | INFO | "gpu" : " 17 GB", | 20260507 03:29:52.765 | INFO | "huge" : "", | 20260507 03:29:52.765 | INFO | "huge_shared" : " 27 GB", | 20260507 03:29:52.765 | INFO | "malloc" : " 14 GB", | 20260507 03:29:52.765 | INFO | "pinned" : " 29 GB" | 20260507 03:29:52.765 | INFO | } | 20260507 03:29:52.765 | INFO | Perf report for the last 5.14481 seconds: | 20260507 03:29:52.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:29:52.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:52.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:52.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:52.765 | INFO | ANALYSIS : 59.8%, 3.0, 93.2ms, 0 | 20260507 03:29:52.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:52.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:52.765 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:29:52.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:52.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:52.766 | INFO | Uploaded 36GiB to GPU in 1.98222s (18.1614GiB/s) | 20260507 03:29:52.766 | INFO | Downloaded 34GiB to GPU in 1.92764s (17.6381GiB/s) | 20260507 03:29:57.916 | INFO | Analyzing chunk frames = [6744576,6745088) | 20260507 03:29:57.916 | INFO | MemInfo: { | 20260507 03:29:57.916 | INFO | "gpu" : " 17 GB", | 20260507 03:29:57.916 | INFO | "huge" : "", | 20260507 03:29:57.916 | INFO | "huge_shared" : " 27 GB", | 20260507 03:29:57.916 | INFO | "malloc" : " 14 GB", | 20260507 03:29:57.916 | INFO | "pinned" : " 29 GB" | 20260507 03:29:57.916 | INFO | } | 20260507 03:29:57.916 | INFO | Perf report for the last 5.15054 seconds: | 20260507 03:29:57.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:29:57.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:57.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:57.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:57.916 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 03:29:57.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:57.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:57.916 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 03:29:57.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:57.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:29:57.916 | INFO | Uploaded 36GiB to GPU in 2.04885s (17.5708GiB/s) | 20260507 03:29:57.916 | INFO | Downloaded 34GiB to GPU in 1.96739s (17.2818GiB/s) | 20260507 03:30:03.046 | INFO | Analyzing chunk frames = [6745088,6745600) | 20260507 03:30:03.046 | INFO | MemInfo: { | 20260507 03:30:03.046 | INFO | "gpu" : " 17 GB", | 20260507 03:30:03.046 | INFO | "huge" : "", | 20260507 03:30:03.046 | INFO | "huge_shared" : " 27 GB", | 20260507 03:30:03.046 | INFO | "malloc" : " 14 GB", | 20260507 03:30:03.046 | INFO | "pinned" : " 29 GB" | 20260507 03:30:03.046 | INFO | } | 20260507 03:30:03.046 | INFO | Perf report for the last 5.13058 seconds: | 20260507 03:30:03.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:30:03.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:03.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:03.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:03.047 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 03:30:03.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:03.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:03.047 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 03:30:03.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:03.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:03.047 | INFO | Uploaded 36GiB to GPU in 2.04545s (17.6GiB/s) | 20260507 03:30:03.047 | INFO | Downloaded 34GiB to GPU in 1.98613s (17.1187GiB/s) | 20260507 03:30:08.178 | INFO | Analyzing chunk frames = [6745600,6746112) | 20260507 03:30:08.178 | INFO | MemInfo: { | 20260507 03:30:08.178 | INFO | "gpu" : " 17 GB", | 20260507 03:30:08.178 | INFO | "huge" : "", | 20260507 03:30:08.178 | INFO | "huge_shared" : " 27 GB", | 20260507 03:30:08.178 | INFO | "malloc" : " 14 GB", | 20260507 03:30:08.178 | INFO | "pinned" : " 29 GB" | 20260507 03:30:08.178 | INFO | } | 20260507 03:30:08.178 | INFO | Perf report for the last 5.13188 seconds: | 20260507 03:30:08.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:30:08.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:08.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:08.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:08.178 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:30:08.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:08.179 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:08.179 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 03:30:08.179 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:08.179 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:08.179 | INFO | Uploaded 36GiB to GPU in 1.97778s (18.2022GiB/s) | 20260507 03:30:08.179 | INFO | Downloaded 34GiB to GPU in 1.96599s (17.2941GiB/s) | 20260507 03:30:08.275 | INFO | Starting a round of Baz IO | 20260507 03:30:11.379 | INFO | Populating RTMetrics report | 20260507 03:30:11.379 | INFO | RTMetrics report populated | 20260507 03:30:11.379 | INFO | Opening tmp RTMetrics json file | 20260507 03:30:11.387 | INFO | Serializing RTMetrics to json | 20260507 03:30:11.396 | INFO | Serialization done, swapping tmp file now... | 20260507 03:30:11.397 | INFO | RTMetrics done | 20260507 03:30:11.470 | INFO | Finished a round of Baz IO: Wrote 1.752 GiB in 3.195 seconds | 20260507 03:30:11.470 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.89% / 0.00% / 67.11% | 20260507 03:30:11.470 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.199505% / 0% / 5.96046e-05% | 20260507 03:30:11.470 | INFO | Baz IO overall: | 20260507 03:30:11.470 | INFO | BytesWritten : 3461716623767 | 20260507 03:30:11.470 | INFO | NumEvents : 1663286902443 | 20260507 03:30:13.327 | INFO | Analyzing chunk frames = [6746112,6746624) | 20260507 03:30:13.327 | INFO | MemInfo: { | 20260507 03:30:13.327 | INFO | "gpu" : " 17 GB", | 20260507 03:30:13.327 | INFO | "huge" : "", | 20260507 03:30:13.327 | INFO | "huge_shared" : " 27 GB", | 20260507 03:30:13.327 | INFO | "malloc" : " 14 GB", | 20260507 03:30:13.327 | INFO | "pinned" : " 29 GB" | 20260507 03:30:13.327 | INFO | } | 20260507 03:30:13.327 | INFO | Perf report for the last 5.14854 seconds: | 20260507 03:30:13.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:30:13.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:13.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:13.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:13.327 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260507 03:30:13.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:13.327 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 03:30:13.327 | INFO | PRE_HQ : 59.1%, 1.8, 55.3ms, 0 | 20260507 03:30:13.327 | INFO | BAZWRITER : 61.7%, 3.2, 102.6ms, 0 | 20260507 03:30:13.327 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:30:13.327 | INFO | Uploaded 36GiB to GPU in 2.02625s (17.7669GiB/s) | 20260507 03:30:13.327 | INFO | Downloaded 38GiB to GPU in 2.18741s (17.3722GiB/s) | 20260507 03:30:18.462 | INFO | Analyzing chunk frames = [6746624,6747136) | 20260507 03:30:18.462 | INFO | MemInfo: { | 20260507 03:30:18.462 | INFO | "gpu" : " 17 GB", | 20260507 03:30:18.462 | INFO | "huge" : "", | 20260507 03:30:18.462 | INFO | "huge_shared" : " 27 GB", | 20260507 03:30:18.462 | INFO | "malloc" : " 14 GB", | 20260507 03:30:18.462 | INFO | "pinned" : " 29 GB" | 20260507 03:30:18.462 | INFO | } | 20260507 03:30:18.463 | INFO | Perf report for the last 5.13556 seconds: | 20260507 03:30:18.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:30:18.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:18.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:18.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:18.463 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:30:18.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:18.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:18.463 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:30:18.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:18.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:18.463 | INFO | Uploaded 36GiB to GPU in 2.03655s (17.677GiB/s) | 20260507 03:30:18.463 | INFO | Downloaded 34GiB to GPU in 1.94237s (17.5044GiB/s) | 20260507 03:30:23.604 | INFO | Analyzing chunk frames = [6747136,6747648) | 20260507 03:30:23.604 | INFO | MemInfo: { | 20260507 03:30:23.604 | INFO | "gpu" : " 17 GB", | 20260507 03:30:23.604 | INFO | "huge" : "", | 20260507 03:30:23.604 | INFO | "huge_shared" : " 27 GB", | 20260507 03:30:23.604 | INFO | "malloc" : " 14 GB", | 20260507 03:30:23.604 | INFO | "pinned" : " 29 GB" | 20260507 03:30:23.604 | INFO | } | 20260507 03:30:23.604 | INFO | Perf report for the last 5.14123 seconds: | 20260507 03:30:23.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:30:23.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:23.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:23.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:23.604 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 03:30:23.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:23.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:23.604 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:30:23.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:23.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:23.604 | INFO | Uploaded 36GiB to GPU in 1.98661s (18.1213GiB/s) | 20260507 03:30:23.604 | INFO | Downloaded 34GiB to GPU in 1.94296s (17.4991GiB/s) | 20260507 03:30:28.742 | INFO | Analyzing chunk frames = [6747648,6748160) | 20260507 03:30:28.742 | INFO | MemInfo: { | 20260507 03:30:28.742 | INFO | "gpu" : " 17 GB", | 20260507 03:30:28.742 | INFO | "huge" : "", | 20260507 03:30:28.742 | INFO | "huge_shared" : " 27 GB", | 20260507 03:30:28.742 | INFO | "malloc" : " 14 GB", | 20260507 03:30:28.742 | INFO | "pinned" : " 29 GB" | 20260507 03:30:28.742 | INFO | } | 20260507 03:30:28.742 | INFO | Perf report for the last 5.13841 seconds: | 20260507 03:30:28.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:30:28.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:28.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:28.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:28.742 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 03:30:28.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:28.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:28.742 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:30:28.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:28.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:28.742 | INFO | Uploaded 36GiB to GPU in 1.98774s (18.111GiB/s) | 20260507 03:30:28.742 | INFO | Downloaded 34GiB to GPU in 1.93413s (17.5789GiB/s) | 20260507 03:30:33.879 | INFO | Analyzing chunk frames = [6748160,6748672) | 20260507 03:30:33.879 | INFO | MemInfo: { | 20260507 03:30:33.879 | INFO | "gpu" : " 17 GB", | 20260507 03:30:33.879 | INFO | "huge" : "", | 20260507 03:30:33.879 | INFO | "huge_shared" : " 27 GB", | 20260507 03:30:33.879 | INFO | "malloc" : " 14 GB", | 20260507 03:30:33.879 | INFO | "pinned" : " 29 GB" | 20260507 03:30:33.880 | INFO | } | 20260507 03:30:33.880 | INFO | Perf report for the last 5.13754 seconds: | 20260507 03:30:33.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:30:33.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:33.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:33.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:33.880 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 03:30:33.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:33.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:33.880 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:30:33.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:33.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:33.880 | INFO | Uploaded 36GiB to GPU in 1.99112s (18.0803GiB/s) | 20260507 03:30:33.880 | INFO | Downloaded 34GiB to GPU in 1.94765s (17.4569GiB/s) | 20260507 03:30:39.022 | INFO | Analyzing chunk frames = [6748672,6749184) | 20260507 03:30:39.022 | INFO | MemInfo: { | 20260507 03:30:39.022 | INFO | "gpu" : " 17 GB", | 20260507 03:30:39.022 | INFO | "huge" : "", | 20260507 03:30:39.022 | INFO | "huge_shared" : " 27 GB", | 20260507 03:30:39.022 | INFO | "malloc" : " 14 GB", | 20260507 03:30:39.022 | INFO | "pinned" : " 29 GB" | 20260507 03:30:39.022 | INFO | } | 20260507 03:30:39.022 | INFO | Perf report for the last 5.14275 seconds: | 20260507 03:30:39.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:30:39.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:39.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:39.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:39.023 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 03:30:39.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:39.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:39.023 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:30:39.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:39.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:39.023 | INFO | Uploaded 36GiB to GPU in 2.05029s (17.5585GiB/s) | 20260507 03:30:39.023 | INFO | Downloaded 34GiB to GPU in 1.96887s (17.2688GiB/s) | 20260507 03:30:44.157 | INFO | Analyzing chunk frames = [6749184,6749696) | 20260507 03:30:44.157 | INFO | MemInfo: { | 20260507 03:30:44.157 | INFO | "gpu" : " 17 GB", | 20260507 03:30:44.157 | INFO | "huge" : "", | 20260507 03:30:44.157 | INFO | "huge_shared" : " 27 GB", | 20260507 03:30:44.157 | INFO | "malloc" : " 14 GB", | 20260507 03:30:44.157 | INFO | "pinned" : " 29 GB" | 20260507 03:30:44.157 | INFO | } | 20260507 03:30:44.158 | INFO | Perf report for the last 5.13504 seconds: | 20260507 03:30:44.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:30:44.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:44.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:44.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:44.158 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 03:30:44.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:44.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:44.158 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 03:30:44.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:44.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:44.158 | INFO | Uploaded 36GiB to GPU in 1.98942s (18.0957GiB/s) | 20260507 03:30:44.158 | INFO | Downloaded 34GiB to GPU in 1.96793s (17.277GiB/s) | 20260507 03:30:49.295 | INFO | Analyzing chunk frames = [6749696,6750208) | 20260507 03:30:49.295 | INFO | MemInfo: { | 20260507 03:30:49.295 | INFO | "gpu" : " 17 GB", | 20260507 03:30:49.295 | INFO | "huge" : "", | 20260507 03:30:49.295 | INFO | "huge_shared" : " 27 GB", | 20260507 03:30:49.295 | INFO | "malloc" : " 14 GB", | 20260507 03:30:49.295 | INFO | "pinned" : " 29 GB" | 20260507 03:30:49.295 | INFO | } | 20260507 03:30:49.295 | INFO | Perf report for the last 5.13748 seconds: | 20260507 03:30:49.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:30:49.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:49.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:49.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:49.295 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 03:30:49.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:49.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:49.295 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:30:49.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:49.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:49.295 | INFO | Uploaded 36GiB to GPU in 1.99215s (18.0709GiB/s) | 20260507 03:30:49.295 | INFO | Downloaded 34GiB to GPU in 1.96785s (17.2777GiB/s) | 20260507 03:30:49.295 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:30:49.295 | INFO | Total mem usage: 18 GiB | 20260507 03:30:49.295 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:30:49.295 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 03:30:49.295 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:30:49.295 | INFO | High Water Breakdown: | 20260507 03:30:49.295 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:30:49.295 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:30:49.295 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:30:49.295 | INFO | ------------------------------------------ | 20260507 03:30:49.295 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:30:49.295 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:30:49.295 | INFO | Total mem usage: 26.5 GiB | 20260507 03:30:49.295 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:30:49.296 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:30:49.296 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:30:49.296 | INFO | High Water Breakdown: | 20260507 03:30:49.296 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:30:49.296 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:30:49.296 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:30:49.296 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:30:49.296 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:30:49.296 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:30:49.296 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:30:49.296 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:30:49.296 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:30:49.296 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:30:49.296 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:30:49.296 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:30:49.296 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:30:49.296 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:30:49.296 | INFO | ------------------------------------------ | 20260507 03:30:49.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:30:49.296 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:30:49.296 | INFO | Total mem usage: 25.6 GiB | 20260507 03:30:49.296 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:30:49.296 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:30:49.296 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:30:49.296 | INFO | High Water Breakdown: | 20260507 03:30:49.296 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:30:49.296 | INFO | ------------------------------------------ | 20260507 03:30:49.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:30:49.296 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:30:49.296 | INFO | Total mem usage: 15.7 GiB | 20260507 03:30:49.296 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:30:49.296 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:30:49.296 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 03:30:49.296 | INFO | High Water Breakdown: | 20260507 03:30:49.296 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:30:49.296 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:30:49.296 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:30:49.296 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:30:49.296 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:30:49.296 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:30:49.296 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:30:49.296 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:30:49.296 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:30:49.296 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:30:49.296 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:30:49.296 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:30:49.296 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:30:49.296 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:30:49.296 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:30:49.296 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:30:49.296 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:30:49.296 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:30:49.296 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:30:49.296 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:30:49.296 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:30:49.296 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:30:49.296 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:30:49.296 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:30:49.296 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:30:49.296 | INFO | ------------------------------------------ | 20260507 03:30:49.296 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:30:49.296 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:30:49.296 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:30:49.296 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:30:49.296 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.47e+01, 1.34e+01 / 1.50e+00 | 20260507 03:30:49.296 | INFO | PerfReport: 0.29%, AggregateStats -- 9.02e-02 / 1.01e-01, 9.15e-03 / 4.85e-03 | 20260507 03:30:49.296 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.43e+00, 5.16e-01 / 2.95e-01 | 20260507 03:30:49.296 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.45e+00, 1.69e-01 / 1.19e-01 | 20260507 03:30:49.296 | INFO | PerfReport: 16.07%, DME -- 3.95e+01 / 4.39e+01, 3.84e+00 / 2.01e+00 | 20260507 03:30:49.296 | INFO | PerfReport: 57.69%, FrameLabeling -- 1.77e+01 / 2.06e+01, 1.33e+00 / 1.94e+00 | 20260507 03:30:49.296 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.15e+00 / 2.69e+00, 3.97e-01 / 2.19e-01 | 20260507 03:30:49.296 | INFO | PerfReport: 3.05%, Metrics -- 9.38e-01 / 1.24e+00, 2.49e-01 / 1.39e-01 | 20260507 03:30:49.296 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.17e-01 / 3.70e-01, 4.93e-02 / 2.47e-02 | 20260507 03:30:49.296 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:30:49.296 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:30:49.296 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:30:49.296 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 03:30:49.296 | INFO | PerfReport: 21.72%, Upload -- 2.09e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260507 03:30:49.296 | INFO | PerfReport: 21.43%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 03:30:49.296 | INFO | PerfReport: 31.98%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 03:30:49.296 | INFO | PerfReport: 1.33%, LockUpload -- 1.28e+00 / 0.00e+00, 4.68e+00 / 0.00e+00 | 20260507 03:30:49.296 | INFO | PerfReport: 1.36%, LockDownload -- 1.31e+00 / 0.00e+00, 3.21e+00 / 0.00e+00 | 20260507 03:30:49.296 | INFO | PerfReport: 22.17%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 03:30:49.296 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260507 03:30:49.392 | INFO | Starting a round of Baz IO | 20260507 03:30:52.496 | INFO | Populating RTMetrics report | 20260507 03:30:52.497 | INFO | RTMetrics report populated | 20260507 03:30:52.497 | INFO | Opening tmp RTMetrics json file | 20260507 03:30:52.516 | INFO | Serializing RTMetrics to json | 20260507 03:30:52.517 | INFO | Serialization done, swapping tmp file now... | 20260507 03:30:52.517 | INFO | RTMetrics done | 20260507 03:30:52.585 | INFO | Finished a round of Baz IO: Wrote 1.751 GiB in 3.193 seconds | 20260507 03:30:52.585 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.88% / 0.00% / 67.11% | 20260507 03:30:52.585 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.19844% / 3.97364e-06% / 5.96046e-05% | 20260507 03:30:52.585 | INFO | Baz IO overall: | 20260507 03:30:52.585 | INFO | BytesWritten : 3463597216498 | 20260507 03:30:52.585 | INFO | NumEvents : 1664173760951 | 20260507 03:30:54.435 | INFO | Analyzing chunk frames = [6750208,6750720) | 20260507 03:30:54.435 | INFO | MemInfo: { | 20260507 03:30:54.435 | INFO | "gpu" : " 17 GB", | 20260507 03:30:54.435 | INFO | "huge" : "", | 20260507 03:30:54.435 | INFO | "huge_shared" : " 27 GB", | 20260507 03:30:54.435 | INFO | "malloc" : " 14 GB", | 20260507 03:30:54.435 | INFO | "pinned" : " 29 GB" | 20260507 03:30:54.435 | INFO | } | 20260507 03:30:54.435 | INFO | Perf report for the last 5.14004 seconds: | 20260507 03:30:54.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:30:54.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:54.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:54.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:54.435 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 03:30:54.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:54.435 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:30:54.435 | INFO | PRE_HQ : 59.1%, 1.8, 55.2ms, 0 | 20260507 03:30:54.435 | INFO | BAZWRITER : 61.7%, 3.3, 104.0ms, 0 | 20260507 03:30:54.435 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:30:54.435 | INFO | Uploaded 36GiB to GPU in 2.04278s (17.6231GiB/s) | 20260507 03:30:54.436 | INFO | Downloaded 38GiB to GPU in 2.18186s (17.4163GiB/s) | 20260507 03:30:59.577 | INFO | Analyzing chunk frames = [6750720,6751232) | 20260507 03:30:59.577 | INFO | MemInfo: { | 20260507 03:30:59.577 | INFO | "gpu" : " 17 GB", | 20260507 03:30:59.577 | INFO | "huge" : "", | 20260507 03:30:59.578 | INFO | "huge_shared" : " 27 GB", | 20260507 03:30:59.578 | INFO | "malloc" : " 14 GB", | 20260507 03:30:59.578 | INFO | "pinned" : " 29 GB" | 20260507 03:30:59.578 | INFO | } | 20260507 03:30:59.578 | INFO | Perf report for the last 5.14256 seconds: | 20260507 03:30:59.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:30:59.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:59.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:59.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:59.578 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:30:59.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:59.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:59.578 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:30:59.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:59.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:30:59.578 | INFO | Uploaded 36GiB to GPU in 2.02254s (17.7994GiB/s) | 20260507 03:30:59.578 | INFO | Downloaded 34GiB to GPU in 1.93943s (17.5309GiB/s) | 20260507 03:31:04.719 | INFO | Analyzing chunk frames = [6751232,6751744) | 20260507 03:31:04.720 | INFO | MemInfo: { | 20260507 03:31:04.720 | INFO | "gpu" : " 17 GB", | 20260507 03:31:04.720 | INFO | "huge" : "", | 20260507 03:31:04.720 | INFO | "huge_shared" : " 27 GB", | 20260507 03:31:04.720 | INFO | "malloc" : " 14 GB", | 20260507 03:31:04.720 | INFO | "pinned" : " 29 GB" | 20260507 03:31:04.720 | INFO | } | 20260507 03:31:04.720 | INFO | Perf report for the last 5.14208 seconds: | 20260507 03:31:04.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:31:04.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:04.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:04.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:04.720 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260507 03:31:04.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:04.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:04.720 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:31:04.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:04.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:04.720 | INFO | Uploaded 36GiB to GPU in 2.01447s (17.8707GiB/s) | 20260507 03:31:04.720 | INFO | Downloaded 34GiB to GPU in 1.93305s (17.5888GiB/s) | 20260507 03:31:09.859 | INFO | Analyzing chunk frames = [6751744,6752256) | 20260507 03:31:09.859 | INFO | MemInfo: { | 20260507 03:31:09.859 | INFO | "gpu" : " 17 GB", | 20260507 03:31:09.859 | INFO | "huge" : "", | 20260507 03:31:09.859 | INFO | "huge_shared" : " 27 GB", | 20260507 03:31:09.859 | INFO | "malloc" : " 14 GB", | 20260507 03:31:09.859 | INFO | "pinned" : " 29 GB" | 20260507 03:31:09.859 | INFO | } | 20260507 03:31:09.859 | INFO | Perf report for the last 5.1393 seconds: | 20260507 03:31:09.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:31:09.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:09.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:09.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:09.859 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:31:09.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:09.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:09.859 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:31:09.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:09.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:09.859 | INFO | Uploaded 36GiB to GPU in 1.98584s (18.1283GiB/s) | 20260507 03:31:09.859 | INFO | Downloaded 34GiB to GPU in 1.92231s (17.687GiB/s) | 20260507 03:31:14.995 | INFO | Analyzing chunk frames = [6752256,6752768) | 20260507 03:31:14.995 | INFO | MemInfo: { | 20260507 03:31:14.995 | INFO | "gpu" : " 17 GB", | 20260507 03:31:14.995 | INFO | "huge" : "", | 20260507 03:31:14.995 | INFO | "huge_shared" : " 27 GB", | 20260507 03:31:14.995 | INFO | "malloc" : " 14 GB", | 20260507 03:31:14.995 | INFO | "pinned" : " 29 GB" | 20260507 03:31:14.995 | INFO | } | 20260507 03:31:14.995 | INFO | Perf report for the last 5.13639 seconds: | 20260507 03:31:14.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:31:14.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:14.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:14.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:14.996 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 03:31:14.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:14.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:14.996 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:31:14.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:14.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:14.996 | INFO | Uploaded 36GiB to GPU in 2.05358s (17.5304GiB/s) | 20260507 03:31:14.996 | INFO | Downloaded 34GiB to GPU in 1.94589s (17.4727GiB/s) | 20260507 03:31:20.129 | INFO | Analyzing chunk frames = [6752768,6753280) | 20260507 03:31:20.129 | INFO | MemInfo: { | 20260507 03:31:20.129 | INFO | "gpu" : " 17 GB", | 20260507 03:31:20.129 | INFO | "huge" : "", | 20260507 03:31:20.129 | INFO | "huge_shared" : " 27 GB", | 20260507 03:31:20.129 | INFO | "malloc" : " 14 GB", | 20260507 03:31:20.129 | INFO | "pinned" : " 29 GB" | 20260507 03:31:20.129 | INFO | } | 20260507 03:31:20.129 | INFO | Perf report for the last 5.13356 seconds: | 20260507 03:31:20.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:31:20.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:20.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:20.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:20.129 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 03:31:20.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:20.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:20.129 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 03:31:20.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:20.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:20.129 | INFO | Uploaded 36GiB to GPU in 2.0351s (17.6896GiB/s) | 20260507 03:31:20.129 | INFO | Downloaded 34GiB to GPU in 1.92265s (17.6839GiB/s) | 20260507 03:31:25.274 | INFO | Analyzing chunk frames = [6753280,6753792) | 20260507 03:31:25.274 | INFO | MemInfo: { | 20260507 03:31:25.274 | INFO | "gpu" : " 17 GB", | 20260507 03:31:25.274 | INFO | "huge" : "", | 20260507 03:31:25.274 | INFO | "huge_shared" : " 27 GB", | 20260507 03:31:25.274 | INFO | "malloc" : " 14 GB", | 20260507 03:31:25.274 | INFO | "pinned" : " 29 GB" | 20260507 03:31:25.274 | INFO | } | 20260507 03:31:25.274 | INFO | Perf report for the last 5.14517 seconds: | 20260507 03:31:25.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:31:25.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:25.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:25.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:25.274 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 03:31:25.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:25.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:25.274 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 03:31:25.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:25.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:25.275 | INFO | Uploaded 36GiB to GPU in 1.98196s (18.1638GiB/s) | 20260507 03:31:25.275 | INFO | Downloaded 34GiB to GPU in 1.98426s (17.1349GiB/s) | 20260507 03:31:30.420 | INFO | Analyzing chunk frames = [6753792,6754304) | 20260507 03:31:30.420 | INFO | MemInfo: { | 20260507 03:31:30.420 | INFO | "gpu" : " 17 GB", | 20260507 03:31:30.420 | INFO | "huge" : "", | 20260507 03:31:30.420 | INFO | "huge_shared" : " 27 GB", | 20260507 03:31:30.420 | INFO | "malloc" : " 14 GB", | 20260507 03:31:30.420 | INFO | "pinned" : " 29 GB" | 20260507 03:31:30.420 | INFO | } | 20260507 03:31:30.420 | INFO | Perf report for the last 5.14561 seconds: | 20260507 03:31:30.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:31:30.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:30.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:30.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:30.420 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 03:31:30.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:30.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:30.420 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 03:31:30.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:30.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:30.420 | INFO | Uploaded 36GiB to GPU in 1.97203s (18.2553GiB/s) | 20260507 03:31:30.420 | INFO | Downloaded 34GiB to GPU in 1.96982s (17.2604GiB/s) | 20260507 03:31:30.517 | INFO | Starting a round of Baz IO | 20260507 03:31:33.618 | INFO | Populating RTMetrics report | 20260507 03:31:33.619 | INFO | RTMetrics report populated | 20260507 03:31:33.619 | INFO | Opening tmp RTMetrics json file | 20260507 03:31:33.621 | INFO | Serializing RTMetrics to json | 20260507 03:31:33.639 | INFO | Serialization done, swapping tmp file now... | 20260507 03:31:33.640 | INFO | RTMetrics done | 20260507 03:31:33.722 | INFO | Finished a round of Baz IO: Wrote 1.752 GiB in 3.204 seconds | 20260507 03:31:33.722 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.88% / 0.00% / 67.12% | 20260507 03:31:33.722 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.198571% / 3.97364e-06% / 8.74201e-05% | 20260507 03:31:33.722 | INFO | Baz IO overall: | 20260507 03:31:33.722 | INFO | BytesWritten : 3465478362285 | 20260507 03:31:33.722 | INFO | NumEvents : 1665060938174 | 20260507 03:31:35.555 | INFO | Analyzing chunk frames = [6754304,6754816) | 20260507 03:31:35.555 | INFO | MemInfo: { | 20260507 03:31:35.555 | INFO | "gpu" : " 17 GB", | 20260507 03:31:35.555 | INFO | "huge" : "", | 20260507 03:31:35.555 | INFO | "huge_shared" : " 27 GB", | 20260507 03:31:35.555 | INFO | "malloc" : " 14 GB", | 20260507 03:31:35.555 | INFO | "pinned" : " 29 GB" | 20260507 03:31:35.555 | INFO | } | 20260507 03:31:35.555 | INFO | Perf report for the last 5.1354 seconds: | 20260507 03:31:35.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:31:35.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:35.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:35.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:35.555 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 03:31:35.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:35.555 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260507 03:31:35.555 | INFO | PRE_HQ : 60.0%, 1.9, 59.6ms, 0 | 20260507 03:31:35.555 | INFO | BAZWRITER : 62.0%, 3.4, 109.0ms, 0 | 20260507 03:31:35.555 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 03:31:35.556 | INFO | Uploaded 36GiB to GPU in 2.03649s (17.6775GiB/s) | 20260507 03:31:35.556 | INFO | Downloaded 38GiB to GPU in 2.1709s (17.5042GiB/s) | 20260507 03:31:40.698 | INFO | Analyzing chunk frames = [6754816,6755328) | 20260507 03:31:40.698 | INFO | MemInfo: { | 20260507 03:31:40.698 | INFO | "gpu" : " 17 GB", | 20260507 03:31:40.698 | INFO | "huge" : "", | 20260507 03:31:40.698 | INFO | "huge_shared" : " 27 GB", | 20260507 03:31:40.698 | INFO | "malloc" : " 14 GB", | 20260507 03:31:40.698 | INFO | "pinned" : " 29 GB" | 20260507 03:31:40.698 | INFO | } | 20260507 03:31:40.698 | INFO | Perf report for the last 5.14281 seconds: | 20260507 03:31:40.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:31:40.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:40.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:40.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:40.698 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:31:40.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:40.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:40.698 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:31:40.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:40.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:40.698 | INFO | Uploaded 36GiB to GPU in 2.03677s (17.6751GiB/s) | 20260507 03:31:40.698 | INFO | Downloaded 34GiB to GPU in 1.94014s (17.5245GiB/s) | 20260507 03:31:45.833 | INFO | Analyzing chunk frames = [6755328,6755840) | 20260507 03:31:45.833 | INFO | MemInfo: { | 20260507 03:31:45.833 | INFO | "gpu" : " 17 GB", | 20260507 03:31:45.833 | INFO | "huge" : "", | 20260507 03:31:45.833 | INFO | "huge_shared" : " 27 GB", | 20260507 03:31:45.833 | INFO | "malloc" : " 14 GB", | 20260507 03:31:45.833 | INFO | "pinned" : " 29 GB" | 20260507 03:31:45.833 | INFO | } | 20260507 03:31:45.833 | INFO | Perf report for the last 5.13527 seconds: | 20260507 03:31:45.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:31:45.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:45.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:45.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:45.833 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 03:31:45.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:45.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:45.833 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 03:31:45.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:45.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:45.834 | INFO | Uploaded 36GiB to GPU in 2.00509s (17.9543GiB/s) | 20260507 03:31:45.834 | INFO | Downloaded 34GiB to GPU in 1.93995s (17.5262GiB/s) | 20260507 03:31:50.970 | INFO | Analyzing chunk frames = [6755840,6756352) | 20260507 03:31:50.970 | INFO | MemInfo: { | 20260507 03:31:50.970 | INFO | "gpu" : " 17 GB", | 20260507 03:31:50.970 | INFO | "huge" : "", | 20260507 03:31:50.970 | INFO | "huge_shared" : " 27 GB", | 20260507 03:31:50.970 | INFO | "malloc" : " 14 GB", | 20260507 03:31:50.970 | INFO | "pinned" : " 29 GB" | 20260507 03:31:50.970 | INFO | } | 20260507 03:31:50.970 | INFO | Perf report for the last 5.13678 seconds: | 20260507 03:31:50.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:31:50.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:50.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:50.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:50.970 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 03:31:50.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:50.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:50.970 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:31:50.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:50.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:50.970 | INFO | Uploaded 36GiB to GPU in 1.95972s (18.37GiB/s) | 20260507 03:31:50.970 | INFO | Downloaded 34GiB to GPU in 1.91392s (17.7646GiB/s) | 20260507 03:31:56.119 | INFO | Analyzing chunk frames = [6756352,6756864) | 20260507 03:31:56.119 | INFO | MemInfo: { | 20260507 03:31:56.119 | INFO | "gpu" : " 17 GB", | 20260507 03:31:56.119 | INFO | "huge" : "", | 20260507 03:31:56.119 | INFO | "huge_shared" : " 27 GB", | 20260507 03:31:56.119 | INFO | "malloc" : " 14 GB", | 20260507 03:31:56.119 | INFO | "pinned" : " 29 GB" | 20260507 03:31:56.119 | INFO | } | 20260507 03:31:56.119 | INFO | Perf report for the last 5.14889 seconds: | 20260507 03:31:56.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:31:56.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:56.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:56.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:56.119 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 03:31:56.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:56.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:56.119 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:31:56.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:56.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:31:56.119 | INFO | Uploaded 36GiB to GPU in 2.01828s (17.837GiB/s) | 20260507 03:31:56.119 | INFO | Downloaded 34GiB to GPU in 1.94058s (17.5205GiB/s) | 20260507 03:32:01.252 | INFO | Analyzing chunk frames = [6756864,6757376) | 20260507 03:32:01.252 | INFO | MemInfo: { | 20260507 03:32:01.252 | INFO | "gpu" : " 17 GB", | 20260507 03:32:01.252 | INFO | "huge" : "", | 20260507 03:32:01.252 | INFO | "huge_shared" : " 27 GB", | 20260507 03:32:01.252 | INFO | "malloc" : " 14 GB", | 20260507 03:32:01.252 | INFO | "pinned" : " 29 GB" | 20260507 03:32:01.252 | INFO | } | 20260507 03:32:01.252 | INFO | Perf report for the last 5.13277 seconds: | 20260507 03:32:01.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:32:01.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:01.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:01.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:01.252 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 03:32:01.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:01.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:01.252 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:32:01.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:01.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:01.252 | INFO | Uploaded 36GiB to GPU in 2.04845s (17.5742GiB/s) | 20260507 03:32:01.252 | INFO | Downloaded 34GiB to GPU in 1.91176s (17.7847GiB/s) | 20260507 03:32:06.387 | INFO | Analyzing chunk frames = [6757376,6757888) | 20260507 03:32:06.387 | INFO | MemInfo: { | 20260507 03:32:06.387 | INFO | "gpu" : " 17 GB", | 20260507 03:32:06.387 | INFO | "huge" : "", | 20260507 03:32:06.387 | INFO | "huge_shared" : " 27 GB", | 20260507 03:32:06.387 | INFO | "malloc" : " 14 GB", | 20260507 03:32:06.387 | INFO | "pinned" : " 29 GB" | 20260507 03:32:06.387 | INFO | } | 20260507 03:32:06.387 | INFO | Perf report for the last 5.13534 seconds: | 20260507 03:32:06.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:32:06.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:06.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:06.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:06.387 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:32:06.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:06.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:06.387 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 03:32:06.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:06.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:06.388 | INFO | Uploaded 36GiB to GPU in 1.97371s (18.2398GiB/s) | 20260507 03:32:06.388 | INFO | Downloaded 34GiB to GPU in 1.9641s (17.3107GiB/s) | 20260507 03:32:11.531 | INFO | Analyzing chunk frames = [6757888,6758400) | 20260507 03:32:11.531 | INFO | MemInfo: { | 20260507 03:32:11.531 | INFO | "gpu" : " 17 GB", | 20260507 03:32:11.531 | INFO | "huge" : "", | 20260507 03:32:11.531 | INFO | "huge_shared" : " 27 GB", | 20260507 03:32:11.531 | INFO | "malloc" : " 14 GB", | 20260507 03:32:11.531 | INFO | "pinned" : " 29 GB" | 20260507 03:32:11.531 | INFO | } | 20260507 03:32:11.531 | INFO | Perf report for the last 5.14427 seconds: | 20260507 03:32:11.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:32:11.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:11.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:11.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:11.532 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 03:32:11.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:11.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:11.532 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 03:32:11.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:11.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:11.532 | INFO | Uploaded 36GiB to GPU in 1.96966s (18.2773GiB/s) | 20260507 03:32:11.532 | INFO | Downloaded 34GiB to GPU in 1.97142s (17.2464GiB/s) | 20260507 03:32:11.532 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:32:11.532 | INFO | Total mem usage: 18 GiB | 20260507 03:32:11.532 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:32:11.532 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 03:32:11.532 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:32:11.532 | INFO | High Water Breakdown: | 20260507 03:32:11.532 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:32:11.532 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:32:11.532 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:32:11.532 | INFO | ------------------------------------------ | 20260507 03:32:11.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:32:11.532 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:32:11.532 | INFO | Total mem usage: 26.5 GiB | 20260507 03:32:11.532 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:32:11.532 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:32:11.532 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:32:11.532 | INFO | High Water Breakdown: | 20260507 03:32:11.532 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:32:11.532 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:32:11.532 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:32:11.532 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:32:11.532 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:32:11.532 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:32:11.532 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:32:11.532 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:32:11.532 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:32:11.532 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:32:11.532 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:32:11.532 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:32:11.532 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:32:11.532 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:32:11.532 | INFO | ------------------------------------------ | 20260507 03:32:11.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:32:11.532 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:32:11.532 | INFO | Total mem usage: 25.6 GiB | 20260507 03:32:11.532 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:32:11.532 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:32:11.532 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:32:11.532 | INFO | High Water Breakdown: | 20260507 03:32:11.532 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:32:11.532 | INFO | ------------------------------------------ | 20260507 03:32:11.532 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:32:11.532 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:32:11.532 | INFO | Total mem usage: 15.7 GiB | 20260507 03:32:11.533 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:32:11.533 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:32:11.533 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:32:11.533 | INFO | High Water Breakdown: | 20260507 03:32:11.533 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:32:11.533 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:32:11.533 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:32:11.533 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:32:11.533 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:32:11.533 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:32:11.533 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:32:11.533 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:32:11.533 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:32:11.533 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:32:11.533 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:32:11.533 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:32:11.533 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:32:11.533 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:32:11.533 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:32:11.533 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:32:11.533 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:32:11.533 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:32:11.533 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:32:11.533 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:32:11.533 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:32:11.533 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:32:11.533 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:32:11.533 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:32:11.533 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:32:11.533 | INFO | ------------------------------------------ | 20260507 03:32:11.533 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:32:11.533 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:32:11.533 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:32:11.533 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:32:11.533 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.48e+01, 1.33e+01 / 1.54e+00 | 20260507 03:32:11.533 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.01e-01, 1.01e-02 / 4.77e-03 | 20260507 03:32:11.533 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.47e+00, 5.22e-01 / 2.61e-01 | 20260507 03:32:11.533 | INFO | PerfReport: 3.23%, Histogram -- 9.95e-01 / 1.50e+00, 1.72e-01 / 1.08e-01 | 20260507 03:32:11.533 | INFO | PerfReport: 16.07%, DME -- 3.96e+01 / 4.41e+01, 3.88e+00 / 1.93e+00 | 20260507 03:32:11.533 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.33e+00 / 1.61e+00 | 20260507 03:32:11.533 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.14e+00 / 2.70e+00, 3.82e-01 / 2.09e-01 | 20260507 03:32:11.533 | INFO | PerfReport: 3.05%, Metrics -- 9.38e-01 / 1.25e+00, 2.49e-01 / 1.43e-01 | 20260507 03:32:11.533 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.18e-01 / 3.71e-01, 5.02e-02 / 1.88e-02 | 20260507 03:32:11.533 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:32:11.533 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:32:11.533 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:32:11.533 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 03:32:11.533 | INFO | PerfReport: 21.76%, Upload -- 2.09e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260507 03:32:11.533 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260507 03:32:11.533 | INFO | PerfReport: 31.98%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 03:32:11.533 | INFO | PerfReport: 1.43%, LockUpload -- 1.38e+00 / 0.00e+00, 4.79e+00 / 0.00e+00 | 20260507 03:32:11.533 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260507 03:32:11.534 | INFO | PerfReport: 22.16%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 03:32:11.534 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.25e-02 / 0.00e+00 | 20260507 03:32:11.629 | INFO | Starting a round of Baz IO | 20260507 03:32:14.736 | INFO | Populating RTMetrics report | 20260507 03:32:14.736 | INFO | RTMetrics report populated | 20260507 03:32:14.736 | INFO | Opening tmp RTMetrics json file | 20260507 03:32:14.738 | INFO | Serializing RTMetrics to json | 20260507 03:32:14.756 | INFO | Serialization done, swapping tmp file now... | 20260507 03:32:14.756 | INFO | RTMetrics done | 20260507 03:32:14.827 | INFO | Finished a round of Baz IO: Wrote 1.751 GiB in 3.198 seconds | 20260507 03:32:14.828 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.88% / 0.00% / 67.12% | 20260507 03:32:14.828 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.197752% / 0% / 7.94729e-05% | 20260507 03:32:14.828 | INFO | Baz IO overall: | 20260507 03:32:14.828 | INFO | BytesWritten : 3467358619460 | 20260507 03:32:14.828 | INFO | NumEvents : 1665948126431 | 20260507 03:32:16.676 | INFO | Analyzing chunk frames = [6758400,6758912) | 20260507 03:32:16.676 | INFO | MemInfo: { | 20260507 03:32:16.676 | INFO | "gpu" : " 17 GB", | 20260507 03:32:16.676 | INFO | "huge" : "", | 20260507 03:32:16.676 | INFO | "huge_shared" : " 27 GB", | 20260507 03:32:16.676 | INFO | "malloc" : " 14 GB", | 20260507 03:32:16.676 | INFO | "pinned" : " 29 GB" | 20260507 03:32:16.676 | INFO | } | 20260507 03:32:16.676 | INFO | Perf report for the last 5.14453 seconds: | 20260507 03:32:16.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:32:16.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:16.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:16.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:16.676 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 03:32:16.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:16.676 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 03:32:16.676 | INFO | PRE_HQ : 59.1%, 1.8, 55.3ms, 0 | 20260507 03:32:16.676 | INFO | BAZWRITER : 61.8%, 3.3, 105.0ms, 0 | 20260507 03:32:16.676 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:32:16.676 | INFO | Uploaded 36GiB to GPU in 2.04526s (17.6017GiB/s) | 20260507 03:32:16.676 | INFO | Downloaded 38GiB to GPU in 2.19254s (17.3315GiB/s) | 20260507 03:32:21.803 | INFO | Analyzing chunk frames = [6758912,6759424) | 20260507 03:32:21.803 | INFO | MemInfo: { | 20260507 03:32:21.803 | INFO | "gpu" : " 17 GB", | 20260507 03:32:21.803 | INFO | "huge" : "", | 20260507 03:32:21.803 | INFO | "huge_shared" : " 27 GB", | 20260507 03:32:21.803 | INFO | "malloc" : " 14 GB", | 20260507 03:32:21.803 | INFO | "pinned" : " 29 GB" | 20260507 03:32:21.803 | INFO | } | 20260507 03:32:21.803 | INFO | Perf report for the last 5.12659 seconds: | 20260507 03:32:21.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:32:21.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:21.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:21.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:21.803 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 03:32:21.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:21.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:21.803 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:32:21.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:21.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:21.803 | INFO | Uploaded 36GiB to GPU in 1.99472s (18.0477GiB/s) | 20260507 03:32:21.803 | INFO | Downloaded 34GiB to GPU in 1.93092s (17.6082GiB/s) | 20260507 03:32:26.938 | INFO | Analyzing chunk frames = [6759424,6759936) | 20260507 03:32:26.938 | INFO | MemInfo: { | 20260507 03:32:26.938 | INFO | "gpu" : " 17 GB", | 20260507 03:32:26.938 | INFO | "huge" : "", | 20260507 03:32:26.938 | INFO | "huge_shared" : " 27 GB", | 20260507 03:32:26.938 | INFO | "malloc" : " 14 GB", | 20260507 03:32:26.938 | INFO | "pinned" : " 29 GB" | 20260507 03:32:26.938 | INFO | } | 20260507 03:32:26.939 | INFO | Perf report for the last 5.1359 seconds: | 20260507 03:32:26.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:32:26.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:26.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:26.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:26.939 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 03:32:26.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:26.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:26.939 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:32:26.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:26.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:26.939 | INFO | Uploaded 36GiB to GPU in 2.00508s (17.9544GiB/s) | 20260507 03:32:26.939 | INFO | Downloaded 34GiB to GPU in 1.94248s (17.5034GiB/s) | 20260507 03:32:32.086 | INFO | Analyzing chunk frames = [6759936,6760448) | 20260507 03:32:32.086 | INFO | MemInfo: { | 20260507 03:32:32.086 | INFO | "gpu" : " 17 GB", | 20260507 03:32:32.086 | INFO | "huge" : "", | 20260507 03:32:32.086 | INFO | "huge_shared" : " 27 GB", | 20260507 03:32:32.086 | INFO | "malloc" : " 14 GB", | 20260507 03:32:32.087 | INFO | "pinned" : " 29 GB" | 20260507 03:32:32.087 | INFO | } | 20260507 03:32:32.087 | INFO | Perf report for the last 5.14814 seconds: | 20260507 03:32:32.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:32:32.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:32.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:32.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:32.087 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 03:32:32.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:32.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:32.087 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:32:32.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:32.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:32.087 | INFO | Uploaded 36GiB to GPU in 1.97548s (18.2235GiB/s) | 20260507 03:32:32.087 | INFO | Downloaded 34GiB to GPU in 1.91546s (17.7503GiB/s) | 20260507 03:32:37.227 | INFO | Analyzing chunk frames = [6760448,6760960) | 20260507 03:32:37.227 | INFO | MemInfo: { | 20260507 03:32:37.227 | INFO | "gpu" : " 17 GB", | 20260507 03:32:37.227 | INFO | "huge" : "", | 20260507 03:32:37.227 | INFO | "huge_shared" : " 27 GB", | 20260507 03:32:37.227 | INFO | "malloc" : " 14 GB", | 20260507 03:32:37.227 | INFO | "pinned" : " 29 GB" | 20260507 03:32:37.227 | INFO | } | 20260507 03:32:37.227 | INFO | Perf report for the last 5.14011 seconds: | 20260507 03:32:37.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:32:37.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:37.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:37.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:37.227 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 03:32:37.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:37.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:37.227 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:32:37.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:37.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:37.227 | INFO | Uploaded 36GiB to GPU in 2.00357s (17.9679GiB/s) | 20260507 03:32:37.227 | INFO | Downloaded 34GiB to GPU in 1.94907s (17.4442GiB/s) | 20260507 03:32:42.368 | INFO | Analyzing chunk frames = [6760960,6761472) | 20260507 03:32:42.368 | INFO | MemInfo: { | 20260507 03:32:42.368 | INFO | "gpu" : " 17 GB", | 20260507 03:32:42.368 | INFO | "huge" : "", | 20260507 03:32:42.368 | INFO | "huge_shared" : " 27 GB", | 20260507 03:32:42.368 | INFO | "malloc" : " 14 GB", | 20260507 03:32:42.368 | INFO | "pinned" : " 29 GB" | 20260507 03:32:42.368 | INFO | } | 20260507 03:32:42.368 | INFO | Perf report for the last 5.14158 seconds: | 20260507 03:32:42.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:32:42.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:42.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:42.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:42.368 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 03:32:42.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:42.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:42.368 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 03:32:42.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:42.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:42.369 | INFO | Uploaded 36GiB to GPU in 2.0269s (17.7611GiB/s) | 20260507 03:32:42.369 | INFO | Downloaded 34GiB to GPU in 1.90698s (17.8292GiB/s) | 20260507 03:32:47.507 | INFO | Analyzing chunk frames = [6761472,6761984) | 20260507 03:32:47.507 | INFO | MemInfo: { | 20260507 03:32:47.507 | INFO | "gpu" : " 17 GB", | 20260507 03:32:47.507 | INFO | "huge" : "", | 20260507 03:32:47.507 | INFO | "huge_shared" : " 27 GB", | 20260507 03:32:47.507 | INFO | "malloc" : " 14 GB", | 20260507 03:32:47.507 | INFO | "pinned" : " 29 GB" | 20260507 03:32:47.507 | INFO | } | 20260507 03:32:47.507 | INFO | Perf report for the last 5.13898 seconds: | 20260507 03:32:47.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:32:47.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:47.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:47.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:47.508 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:32:47.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:47.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:47.508 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 03:32:47.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:47.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:47.508 | INFO | Uploaded 36GiB to GPU in 1.9902s (18.0886GiB/s) | 20260507 03:32:47.508 | INFO | Downloaded 34GiB to GPU in 1.97032s (17.2561GiB/s) | 20260507 03:32:52.646 | INFO | Analyzing chunk frames = [6761984,6762496) | 20260507 03:32:52.646 | INFO | MemInfo: { | 20260507 03:32:52.646 | INFO | "gpu" : " 17 GB", | 20260507 03:32:52.646 | INFO | "huge" : "", | 20260507 03:32:52.646 | INFO | "huge_shared" : " 27 GB", | 20260507 03:32:52.646 | INFO | "malloc" : " 14 GB", | 20260507 03:32:52.646 | INFO | "pinned" : " 29 GB" | 20260507 03:32:52.646 | INFO | } | 20260507 03:32:52.646 | INFO | Perf report for the last 5.13853 seconds: | 20260507 03:32:52.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:32:52.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:52.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:52.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:52.646 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 03:32:52.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:52.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:52.646 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:32:52.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:52.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:52.646 | INFO | Uploaded 36GiB to GPU in 1.95911s (18.3757GiB/s) | 20260507 03:32:52.646 | INFO | Downloaded 34GiB to GPU in 1.95328s (17.4066GiB/s) | 20260507 03:32:52.743 | INFO | Starting a round of Baz IO | 20260507 03:32:55.845 | INFO | Populating RTMetrics report | 20260507 03:32:55.845 | INFO | RTMetrics report populated | 20260507 03:32:55.845 | INFO | Opening tmp RTMetrics json file | 20260507 03:32:55.848 | INFO | Serializing RTMetrics to json | 20260507 03:32:55.864 | INFO | Serialization done, swapping tmp file now... | 20260507 03:32:55.864 | INFO | RTMetrics done | 20260507 03:32:55.937 | INFO | Finished a round of Baz IO: Wrote 1.749 GiB in 3.194 seconds | 20260507 03:32:55.937 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.88% / 0.00% / 67.12% | 20260507 03:32:55.937 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.196981% / 0% / 9.53674e-05% | 20260507 03:32:55.937 | INFO | Baz IO overall: | 20260507 03:32:55.937 | INFO | BytesWritten : 3469236872260 | 20260507 03:32:55.937 | INFO | NumEvents : 1666833879750 | 20260507 03:32:57.783 | INFO | Analyzing chunk frames = [6762496,6763008) | 20260507 03:32:57.783 | INFO | MemInfo: { | 20260507 03:32:57.783 | INFO | "gpu" : " 17 GB", | 20260507 03:32:57.783 | INFO | "huge" : "", | 20260507 03:32:57.783 | INFO | "huge_shared" : " 27 GB", | 20260507 03:32:57.783 | INFO | "malloc" : " 14 GB", | 20260507 03:32:57.783 | INFO | "pinned" : " 29 GB" | 20260507 03:32:57.783 | INFO | } | 20260507 03:32:57.783 | INFO | Perf report for the last 5.13684 seconds: | 20260507 03:32:57.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:32:57.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:57.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:57.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:57.783 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 03:32:57.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:32:57.783 | INFO | RT_METRICS : 41.9%, 1.0, 21.9ms, 0 | 20260507 03:32:57.783 | INFO | PRE_HQ : 59.1%, 1.8, 55.5ms, 0 | 20260507 03:32:57.783 | INFO | BAZWRITER : 61.8%, 3.3, 104.2ms, 0 | 20260507 03:32:57.783 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:32:57.783 | INFO | Uploaded 36GiB to GPU in 2.02525s (17.7756GiB/s) | 20260507 03:32:57.783 | INFO | Downloaded 38GiB to GPU in 2.18479s (17.3929GiB/s) | 20260507 03:33:02.920 | INFO | Analyzing chunk frames = [6763008,6763520) | 20260507 03:33:02.920 | INFO | MemInfo: { | 20260507 03:33:02.920 | INFO | "gpu" : " 17 GB", | 20260507 03:33:02.921 | INFO | "huge" : "", | 20260507 03:33:02.921 | INFO | "huge_shared" : " 27 GB", | 20260507 03:33:02.921 | INFO | "malloc" : " 14 GB", | 20260507 03:33:02.921 | INFO | "pinned" : " 29 GB" | 20260507 03:33:02.921 | INFO | } | 20260507 03:33:02.921 | INFO | Perf report for the last 5.13794 seconds: | 20260507 03:33:02.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:33:02.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:02.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:02.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:02.921 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:33:02.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:02.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:02.921 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:33:02.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:02.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:02.921 | INFO | Uploaded 36GiB to GPU in 2.01473s (17.8684GiB/s) | 20260507 03:33:02.921 | INFO | Downloaded 34GiB to GPU in 1.93582s (17.5636GiB/s) | 20260507 03:33:08.066 | INFO | Analyzing chunk frames = [6763520,6764032) | 20260507 03:33:08.066 | INFO | MemInfo: { | 20260507 03:33:08.066 | INFO | "gpu" : " 17 GB", | 20260507 03:33:08.066 | INFO | "huge" : "", | 20260507 03:33:08.066 | INFO | "huge_shared" : " 27 GB", | 20260507 03:33:08.066 | INFO | "malloc" : " 14 GB", | 20260507 03:33:08.066 | INFO | "pinned" : " 29 GB" | 20260507 03:33:08.066 | INFO | } | 20260507 03:33:08.066 | INFO | Perf report for the last 5.14561 seconds: | 20260507 03:33:08.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:33:08.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:08.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:08.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:08.066 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 03:33:08.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:08.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:08.066 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:33:08.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:08.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:08.067 | INFO | Uploaded 36GiB to GPU in 1.97829s (18.1976GiB/s) | 20260507 03:33:08.067 | INFO | Downloaded 34GiB to GPU in 1.94229s (17.5051GiB/s) | 20260507 03:33:13.201 | INFO | Analyzing chunk frames = [6764032,6764544) | 20260507 03:33:13.201 | INFO | MemInfo: { | 20260507 03:33:13.201 | INFO | "gpu" : " 17 GB", | 20260507 03:33:13.201 | INFO | "huge" : "", | 20260507 03:33:13.201 | INFO | "huge_shared" : " 27 GB", | 20260507 03:33:13.201 | INFO | "malloc" : " 14 GB", | 20260507 03:33:13.201 | INFO | "pinned" : " 29 GB" | 20260507 03:33:13.201 | INFO | } | 20260507 03:33:13.201 | INFO | Perf report for the last 5.13441 seconds: | 20260507 03:33:13.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:33:13.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:13.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:13.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:13.201 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 03:33:13.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:13.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:13.201 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:33:13.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:13.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:13.201 | INFO | Uploaded 36GiB to GPU in 2.0203s (17.8191GiB/s) | 20260507 03:33:13.201 | INFO | Downloaded 34GiB to GPU in 1.94021s (17.5238GiB/s) | 20260507 03:33:18.342 | INFO | Analyzing chunk frames = [6764544,6765056) | 20260507 03:33:18.342 | INFO | MemInfo: { | 20260507 03:33:18.342 | INFO | "gpu" : " 17 GB", | 20260507 03:33:18.342 | INFO | "huge" : "", | 20260507 03:33:18.342 | INFO | "huge_shared" : " 27 GB", | 20260507 03:33:18.342 | INFO | "malloc" : " 14 GB", | 20260507 03:33:18.342 | INFO | "pinned" : " 29 GB" | 20260507 03:33:18.342 | INFO | } | 20260507 03:33:18.342 | INFO | Perf report for the last 5.14169 seconds: | 20260507 03:33:18.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:33:18.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:18.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:18.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:18.343 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 03:33:18.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:18.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:18.343 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 03:33:18.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:18.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:18.343 | INFO | Uploaded 36GiB to GPU in 2.02153s (17.8083GiB/s) | 20260507 03:33:18.343 | INFO | Downloaded 34GiB to GPU in 1.94337s (17.4953GiB/s) | 20260507 03:33:23.483 | INFO | Analyzing chunk frames = [6765056,6765568) | 20260507 03:33:23.483 | INFO | MemInfo: { | 20260507 03:33:23.483 | INFO | "gpu" : " 17 GB", | 20260507 03:33:23.483 | INFO | "huge" : "", | 20260507 03:33:23.483 | INFO | "huge_shared" : " 27 GB", | 20260507 03:33:23.483 | INFO | "malloc" : " 14 GB", | 20260507 03:33:23.483 | INFO | "pinned" : " 29 GB" | 20260507 03:33:23.483 | INFO | } | 20260507 03:33:23.483 | INFO | Perf report for the last 5.14079 seconds: | 20260507 03:33:23.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:33:23.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:23.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:23.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:23.483 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 03:33:23.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:23.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:23.483 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 03:33:23.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:23.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:23.484 | INFO | Uploaded 36GiB to GPU in 2.03738s (17.6697GiB/s) | 20260507 03:33:23.484 | INFO | Downloaded 34GiB to GPU in 1.92631s (17.6504GiB/s) | 20260507 03:33:28.621 | INFO | Analyzing chunk frames = [6765568,6766080) | 20260507 03:33:28.621 | INFO | MemInfo: { | 20260507 03:33:28.621 | INFO | "gpu" : " 17 GB", | 20260507 03:33:28.621 | INFO | "huge" : "", | 20260507 03:33:28.621 | INFO | "huge_shared" : " 27 GB", | 20260507 03:33:28.621 | INFO | "malloc" : " 14 GB", | 20260507 03:33:28.621 | INFO | "pinned" : " 29 GB" | 20260507 03:33:28.621 | INFO | } | 20260507 03:33:28.621 | INFO | Perf report for the last 5.13752 seconds: | 20260507 03:33:28.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:33:28.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:28.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:28.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:28.621 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:33:28.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:28.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:28.621 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:33:28.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:28.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:28.621 | INFO | Uploaded 36GiB to GPU in 1.97408s (18.2364GiB/s) | 20260507 03:33:28.621 | INFO | Downloaded 34GiB to GPU in 1.98415s (17.1358GiB/s) | 20260507 03:33:33.755 | INFO | Analyzing chunk frames = [6766080,6766592) | 20260507 03:33:33.755 | INFO | MemInfo: { | 20260507 03:33:33.755 | INFO | "gpu" : " 17 GB", | 20260507 03:33:33.755 | INFO | "huge" : "", | 20260507 03:33:33.755 | INFO | "huge_shared" : " 27 GB", | 20260507 03:33:33.755 | INFO | "malloc" : " 14 GB", | 20260507 03:33:33.755 | INFO | "pinned" : " 29 GB" | 20260507 03:33:33.755 | INFO | } | 20260507 03:33:33.755 | INFO | Perf report for the last 5.13417 seconds: | 20260507 03:33:33.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:33:33.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:33.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:33.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:33.755 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 03:33:33.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:33.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:33.755 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:33:33.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:33.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:33.755 | INFO | Uploaded 36GiB to GPU in 1.96771s (18.2953GiB/s) | 20260507 03:33:33.755 | INFO | Downloaded 34GiB to GPU in 1.95793s (17.3652GiB/s) | 20260507 03:33:33.755 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:33:33.755 | INFO | Total mem usage: 18 GiB | 20260507 03:33:33.755 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:33:33.755 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:33:33.755 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:33:33.755 | INFO | High Water Breakdown: | 20260507 03:33:33.756 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:33:33.756 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:33:33.756 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:33:33.756 | INFO | ------------------------------------------ | 20260507 03:33:33.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:33:33.756 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:33:33.756 | INFO | Total mem usage: 26.5 GiB | 20260507 03:33:33.756 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:33:33.756 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:33:33.756 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:33:33.756 | INFO | High Water Breakdown: | 20260507 03:33:33.756 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:33:33.756 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:33:33.756 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:33:33.756 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:33:33.756 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:33:33.756 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:33:33.756 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:33:33.756 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:33:33.756 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:33:33.756 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:33:33.756 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:33:33.756 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:33:33.756 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:33:33.756 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:33:33.756 | INFO | ------------------------------------------ | 20260507 03:33:33.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:33:33.756 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:33:33.756 | INFO | Total mem usage: 25.6 GiB | 20260507 03:33:33.756 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:33:33.756 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:33:33.756 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:33:33.756 | INFO | High Water Breakdown: | 20260507 03:33:33.756 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:33:33.756 | INFO | ------------------------------------------ | 20260507 03:33:33.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:33:33.756 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:33:33.756 | INFO | Total mem usage: 15.7 GiB | 20260507 03:33:33.756 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:33:33.756 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:33:33.756 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 03:33:33.756 | INFO | High Water Breakdown: | 20260507 03:33:33.756 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:33:33.756 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:33:33.756 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:33:33.756 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:33:33.756 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:33:33.756 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:33:33.756 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:33:33.756 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:33:33.756 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:33:33.756 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:33:33.756 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:33:33.756 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:33:33.756 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:33:33.756 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:33:33.756 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:33:33.756 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:33:33.756 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:33:33.756 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:33:33.756 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:33:33.756 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:33:33.756 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:33:33.756 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:33:33.756 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:33:33.756 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:33:33.756 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:33:33.756 | INFO | ------------------------------------------ | 20260507 03:33:33.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:33:33.756 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:33:33.756 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:33:33.756 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:33:33.756 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.48e+01, 1.35e+01 / 1.46e+00 | 20260507 03:33:33.756 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.00e-01, 9.78e-03 / 6.45e-03 | 20260507 03:33:33.756 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.37e+00, 5.13e-01 / 3.19e-01 | 20260507 03:33:33.756 | INFO | PerfReport: 3.23%, Histogram -- 9.95e-01 / 1.42e+00, 1.68e-01 / 1.44e-01 | 20260507 03:33:33.756 | INFO | PerfReport: 16.08%, DME -- 3.96e+01 / 4.42e+01, 3.90e+00 / 1.76e+00 | 20260507 03:33:33.756 | INFO | PerfReport: 57.68%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.35e+00 / 1.68e+00 | 20260507 03:33:33.756 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.14e+00 / 2.68e+00, 3.79e-01 / 2.38e-01 | 20260507 03:33:33.756 | INFO | PerfReport: 3.05%, Metrics -- 9.39e-01 / 1.27e+00, 2.49e-01 / 1.38e-01 | 20260507 03:33:33.756 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.18e-01 / 3.71e-01, 4.95e-02 / 1.86e-02 | 20260507 03:33:33.757 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:33:33.757 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:33:33.757 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:33:33.757 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 03:33:33.757 | INFO | PerfReport: 21.69%, Upload -- 2.09e+01 / 0.00e+00, 6.96e+00 / 0.00e+00 | 20260507 03:33:33.757 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 03:33:33.757 | INFO | PerfReport: 32.02%, Compute -- 3.08e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 03:33:33.757 | INFO | PerfReport: 1.31%, LockUpload -- 1.26e+00 / 0.00e+00, 4.61e+00 / 0.00e+00 | 20260507 03:33:33.757 | INFO | PerfReport: 1.39%, LockDownload -- 1.34e+00 / 0.00e+00, 3.25e+00 / 0.00e+00 | 20260507 03:33:33.757 | INFO | PerfReport: 22.19%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 03:33:33.757 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260507 03:33:33.852 | INFO | Starting a round of Baz IO | 20260507 03:33:36.970 | INFO | Populating RTMetrics report | 20260507 03:33:36.970 | INFO | RTMetrics report populated | 20260507 03:33:36.970 | INFO | Opening tmp RTMetrics json file | 20260507 03:33:36.987 | INFO | Serializing RTMetrics to json | 20260507 03:33:36.988 | INFO | Serialization done, swapping tmp file now... | 20260507 03:33:36.988 | INFO | RTMetrics done | 20260507 03:33:37.060 | INFO | Finished a round of Baz IO: Wrote 1.748 GiB in 3.208 seconds | 20260507 03:33:37.061 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.88% / 0.00% / 67.12% | 20260507 03:33:37.061 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.196664% / 0% / 5.96046e-05% | 20260507 03:33:37.061 | INFO | Baz IO overall: | 20260507 03:33:37.061 | INFO | BytesWritten : 3471113977213 | 20260507 03:33:37.061 | INFO | NumEvents : 1667719024516 | 20260507 03:33:38.893 | INFO | Analyzing chunk frames = [6766592,6767104) | 20260507 03:33:38.893 | INFO | MemInfo: { | 20260507 03:33:38.893 | INFO | "gpu" : " 17 GB", | 20260507 03:33:38.893 | INFO | "huge" : "", | 20260507 03:33:38.893 | INFO | "huge_shared" : " 27 GB", | 20260507 03:33:38.893 | INFO | "malloc" : " 14 GB", | 20260507 03:33:38.893 | INFO | "pinned" : " 29 GB" | 20260507 03:33:38.893 | INFO | } | 20260507 03:33:38.893 | INFO | Perf report for the last 5.13825 seconds: | 20260507 03:33:38.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:33:38.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:38.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:38.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:38.893 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 03:33:38.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:38.893 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 03:33:38.893 | INFO | PRE_HQ : 59.4%, 1.8, 55.2ms, 0 | 20260507 03:33:38.893 | INFO | BAZWRITER : 62.0%, 3.3, 104.4ms, 0 | 20260507 03:33:38.893 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:33:38.893 | INFO | Uploaded 36GiB to GPU in 2.0477s (17.5807GiB/s) | 20260507 03:33:38.894 | INFO | Downloaded 38GiB to GPU in 2.1789s (17.44GiB/s) | 20260507 03:33:44.039 | INFO | Analyzing chunk frames = [6767104,6767616) | 20260507 03:33:44.039 | INFO | MemInfo: { | 20260507 03:33:44.039 | INFO | "gpu" : " 17 GB", | 20260507 03:33:44.039 | INFO | "huge" : "", | 20260507 03:33:44.039 | INFO | "huge_shared" : " 27 GB", | 20260507 03:33:44.039 | INFO | "malloc" : " 14 GB", | 20260507 03:33:44.039 | INFO | "pinned" : " 29 GB" | 20260507 03:33:44.039 | INFO | } | 20260507 03:33:44.039 | INFO | Perf report for the last 5.146 seconds: | 20260507 03:33:44.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:33:44.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:44.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:44.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:44.039 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:33:44.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:44.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:44.039 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:33:44.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:44.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:44.040 | INFO | Uploaded 36GiB to GPU in 1.99465s (18.0482GiB/s) | 20260507 03:33:44.040 | INFO | Downloaded 34GiB to GPU in 1.92767s (17.6379GiB/s) | 20260507 03:33:49.179 | INFO | Analyzing chunk frames = [6767616,6768128) | 20260507 03:33:49.179 | INFO | MemInfo: { | 20260507 03:33:49.179 | INFO | "gpu" : " 17 GB", | 20260507 03:33:49.179 | INFO | "huge" : "", | 20260507 03:33:49.179 | INFO | "huge_shared" : " 27 GB", | 20260507 03:33:49.179 | INFO | "malloc" : " 14 GB", | 20260507 03:33:49.179 | INFO | "pinned" : " 29 GB" | 20260507 03:33:49.179 | INFO | } | 20260507 03:33:49.180 | INFO | Perf report for the last 5.14022 seconds: | 20260507 03:33:49.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:33:49.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:49.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:49.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:49.180 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 03:33:49.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:49.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:49.180 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:33:49.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:49.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:49.180 | INFO | Uploaded 36GiB to GPU in 2.01988s (17.8229GiB/s) | 20260507 03:33:49.180 | INFO | Downloaded 34GiB to GPU in 1.93698s (17.5531GiB/s) | 20260507 03:33:54.317 | INFO | Analyzing chunk frames = [6768128,6768640) | 20260507 03:33:54.317 | INFO | MemInfo: { | 20260507 03:33:54.317 | INFO | "gpu" : " 17 GB", | 20260507 03:33:54.317 | INFO | "huge" : "", | 20260507 03:33:54.317 | INFO | "huge_shared" : " 27 GB", | 20260507 03:33:54.317 | INFO | "malloc" : " 14 GB", | 20260507 03:33:54.318 | INFO | "pinned" : " 29 GB" | 20260507 03:33:54.318 | INFO | } | 20260507 03:33:54.318 | INFO | Perf report for the last 5.13804 seconds: | 20260507 03:33:54.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:33:54.318 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:54.318 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:54.318 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:54.318 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 03:33:54.318 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:54.318 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:54.318 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:33:54.318 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:54.318 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:54.318 | INFO | Uploaded 36GiB to GPU in 2.00147s (17.9868GiB/s) | 20260507 03:33:54.318 | INFO | Downloaded 34GiB to GPU in 1.92846s (17.6307GiB/s) | 20260507 03:33:59.451 | INFO | Analyzing chunk frames = [6768640,6769152) | 20260507 03:33:59.451 | INFO | MemInfo: { | 20260507 03:33:59.451 | INFO | "gpu" : " 17 GB", | 20260507 03:33:59.451 | INFO | "huge" : "", | 20260507 03:33:59.451 | INFO | "huge_shared" : " 27 GB", | 20260507 03:33:59.451 | INFO | "malloc" : " 14 GB", | 20260507 03:33:59.451 | INFO | "pinned" : " 29 GB" | 20260507 03:33:59.451 | INFO | } | 20260507 03:33:59.451 | INFO | Perf report for the last 5.13344 seconds: | 20260507 03:33:59.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:33:59.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:59.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:59.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:59.451 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 03:33:59.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:59.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:59.451 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:33:59.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:59.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:33:59.451 | INFO | Uploaded 36GiB to GPU in 2.05734s (17.4983GiB/s) | 20260507 03:33:59.451 | INFO | Downloaded 34GiB to GPU in 1.94657s (17.4666GiB/s) | 20260507 03:34:04.595 | INFO | Analyzing chunk frames = [6769152,6769664) | 20260507 03:34:04.595 | INFO | MemInfo: { | 20260507 03:34:04.595 | INFO | "gpu" : " 17 GB", | 20260507 03:34:04.595 | INFO | "huge" : "", | 20260507 03:34:04.595 | INFO | "huge_shared" : " 27 GB", | 20260507 03:34:04.595 | INFO | "malloc" : " 14 GB", | 20260507 03:34:04.595 | INFO | "pinned" : " 29 GB" | 20260507 03:34:04.595 | INFO | } | 20260507 03:34:04.595 | INFO | Perf report for the last 5.14441 seconds: | 20260507 03:34:04.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:34:04.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:04.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:04.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:04.595 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:34:04.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:04.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:04.595 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 03:34:04.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:04.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:04.596 | INFO | Uploaded 36GiB to GPU in 2.05908s (17.4835GiB/s) | 20260507 03:34:04.596 | INFO | Downloaded 34GiB to GPU in 1.95847s (17.3605GiB/s) | 20260507 03:34:09.741 | INFO | Analyzing chunk frames = [6769664,6770176) | 20260507 03:34:09.741 | INFO | MemInfo: { | 20260507 03:34:09.741 | INFO | "gpu" : " 17 GB", | 20260507 03:34:09.741 | INFO | "huge" : "", | 20260507 03:34:09.741 | INFO | "huge_shared" : " 27 GB", | 20260507 03:34:09.741 | INFO | "malloc" : " 14 GB", | 20260507 03:34:09.742 | INFO | "pinned" : " 29 GB" | 20260507 03:34:09.742 | INFO | } | 20260507 03:34:09.742 | INFO | Perf report for the last 5.14611 seconds: | 20260507 03:34:09.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:34:09.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:09.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:09.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:09.742 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:34:09.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:09.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:09.742 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:34:09.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:09.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:09.742 | INFO | Uploaded 36GiB to GPU in 1.97247s (18.2512GiB/s) | 20260507 03:34:09.742 | INFO | Downloaded 34GiB to GPU in 1.94887s (17.446GiB/s) | 20260507 03:34:14.873 | INFO | Analyzing chunk frames = [6770176,6770688) | 20260507 03:34:14.873 | INFO | MemInfo: { | 20260507 03:34:14.873 | INFO | "gpu" : " 17 GB", | 20260507 03:34:14.873 | INFO | "huge" : "", | 20260507 03:34:14.873 | INFO | "huge_shared" : " 27 GB", | 20260507 03:34:14.873 | INFO | "malloc" : " 14 GB", | 20260507 03:34:14.873 | INFO | "pinned" : " 29 GB" | 20260507 03:34:14.873 | INFO | } | 20260507 03:34:14.873 | INFO | Perf report for the last 5.13168 seconds: | 20260507 03:34:14.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:34:14.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:14.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:14.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:14.873 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 03:34:14.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:14.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:14.873 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 03:34:14.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:14.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:14.874 | INFO | Uploaded 36GiB to GPU in 1.97655s (18.2135GiB/s) | 20260507 03:34:14.874 | INFO | Downloaded 34GiB to GPU in 1.97605s (17.206GiB/s) | 20260507 03:34:14.971 | INFO | Starting a round of Baz IO | 20260507 03:34:18.072 | INFO | Populating RTMetrics report | 20260507 03:34:18.072 | INFO | RTMetrics report populated | 20260507 03:34:18.072 | INFO | Opening tmp RTMetrics json file | 20260507 03:34:18.076 | INFO | Serializing RTMetrics to json | 20260507 03:34:18.091 | INFO | Serialization done, swapping tmp file now... | 20260507 03:34:18.091 | INFO | RTMetrics done | 20260507 03:34:18.163 | INFO | Finished a round of Baz IO: Wrote 1.748 GiB in 3.191 seconds | 20260507 03:34:18.163 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.88% / 0.00% / 67.12% | 20260507 03:34:18.163 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.195575% / 0% / 8.34465e-05% | 20260507 03:34:18.163 | INFO | Baz IO overall: | 20260507 03:34:18.163 | INFO | BytesWritten : 3472991165754 | 20260507 03:34:18.163 | INFO | NumEvents : 1668604424882 | 20260507 03:34:20.019 | INFO | Analyzing chunk frames = [6770688,6771200) | 20260507 03:34:20.019 | INFO | MemInfo: { | 20260507 03:34:20.019 | INFO | "gpu" : " 17 GB", | 20260507 03:34:20.019 | INFO | "huge" : "", | 20260507 03:34:20.019 | INFO | "huge_shared" : " 27 GB", | 20260507 03:34:20.019 | INFO | "malloc" : " 14 GB", | 20260507 03:34:20.019 | INFO | "pinned" : " 29 GB" | 20260507 03:34:20.019 | INFO | } | 20260507 03:34:20.019 | INFO | Perf report for the last 5.14598 seconds: | 20260507 03:34:20.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:34:20.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:20.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:20.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:20.019 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 03:34:20.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:20.019 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:34:20.019 | INFO | PRE_HQ : 59.1%, 1.8, 55.2ms, 0 | 20260507 03:34:20.019 | INFO | BAZWRITER : 61.6%, 3.3, 104.6ms, 0 | 20260507 03:34:20.019 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 03:34:20.020 | INFO | Uploaded 36GiB to GPU in 2.02155s (17.8081GiB/s) | 20260507 03:34:20.020 | INFO | Downloaded 38GiB to GPU in 2.18s (17.4312GiB/s) | 20260507 03:34:25.152 | INFO | Analyzing chunk frames = [6771200,6771712) | 20260507 03:34:25.152 | INFO | MemInfo: { | 20260507 03:34:25.152 | INFO | "gpu" : " 17 GB", | 20260507 03:34:25.152 | INFO | "huge" : "", | 20260507 03:34:25.152 | INFO | "huge_shared" : " 27 GB", | 20260507 03:34:25.152 | INFO | "malloc" : " 14 GB", | 20260507 03:34:25.152 | INFO | "pinned" : " 29 GB" | 20260507 03:34:25.152 | INFO | } | 20260507 03:34:25.152 | INFO | Perf report for the last 5.13288 seconds: | 20260507 03:34:25.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:34:25.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:25.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:25.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:25.152 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 03:34:25.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:25.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:25.152 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:34:25.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:25.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:25.152 | INFO | Uploaded 36GiB to GPU in 1.99725s (18.0248GiB/s) | 20260507 03:34:25.153 | INFO | Downloaded 34GiB to GPU in 1.93449s (17.5757GiB/s) | 20260507 03:34:30.292 | INFO | Analyzing chunk frames = [6771712,6772224) | 20260507 03:34:30.292 | INFO | MemInfo: { | 20260507 03:34:30.292 | INFO | "gpu" : " 17 GB", | 20260507 03:34:30.292 | INFO | "huge" : "", | 20260507 03:34:30.292 | INFO | "huge_shared" : " 27 GB", | 20260507 03:34:30.292 | INFO | "malloc" : " 14 GB", | 20260507 03:34:30.292 | INFO | "pinned" : " 29 GB" | 20260507 03:34:30.293 | INFO | } | 20260507 03:34:30.293 | INFO | Perf report for the last 5.14043 seconds: | 20260507 03:34:30.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:34:30.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:30.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:30.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:30.293 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 03:34:30.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:30.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:30.293 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:34:30.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:30.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:30.293 | INFO | Uploaded 36GiB to GPU in 2.00884s (17.9207GiB/s) | 20260507 03:34:30.293 | INFO | Downloaded 34GiB to GPU in 1.94243s (17.5039GiB/s) | 20260507 03:34:35.429 | INFO | Analyzing chunk frames = [6772224,6772736) | 20260507 03:34:35.429 | INFO | MemInfo: { | 20260507 03:34:35.429 | INFO | "gpu" : " 17 GB", | 20260507 03:34:35.429 | INFO | "huge" : "", | 20260507 03:34:35.429 | INFO | "huge_shared" : " 27 GB", | 20260507 03:34:35.429 | INFO | "malloc" : " 14 GB", | 20260507 03:34:35.429 | INFO | "pinned" : " 29 GB" | 20260507 03:34:35.429 | INFO | } | 20260507 03:34:35.429 | INFO | Perf report for the last 5.13654 seconds: | 20260507 03:34:35.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:34:35.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:35.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:35.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:35.429 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 03:34:35.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:35.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:35.429 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:34:35.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:35.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:35.429 | INFO | Uploaded 36GiB to GPU in 1.97792s (18.2009GiB/s) | 20260507 03:34:35.429 | INFO | Downloaded 34GiB to GPU in 1.92239s (17.6863GiB/s) | 20260507 03:34:40.571 | INFO | Analyzing chunk frames = [6772736,6773248) | 20260507 03:34:40.571 | INFO | MemInfo: { | 20260507 03:34:40.571 | INFO | "gpu" : " 17 GB", | 20260507 03:34:40.571 | INFO | "huge" : "", | 20260507 03:34:40.571 | INFO | "huge_shared" : " 27 GB", | 20260507 03:34:40.571 | INFO | "malloc" : " 14 GB", | 20260507 03:34:40.571 | INFO | "pinned" : " 29 GB" | 20260507 03:34:40.571 | INFO | } | 20260507 03:34:40.571 | INFO | Perf report for the last 5.14162 seconds: | 20260507 03:34:40.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:34:40.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:40.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:40.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:40.571 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 03:34:40.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:40.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:40.571 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:34:40.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:40.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:40.571 | INFO | Uploaded 36GiB to GPU in 2.04594s (17.5958GiB/s) | 20260507 03:34:40.571 | INFO | Downloaded 34GiB to GPU in 1.95131s (17.4242GiB/s) | 20260507 03:34:45.707 | INFO | Analyzing chunk frames = [6773248,6773760) | 20260507 03:34:45.707 | INFO | MemInfo: { | 20260507 03:34:45.707 | INFO | "gpu" : " 17 GB", | 20260507 03:34:45.707 | INFO | "huge" : "", | 20260507 03:34:45.707 | INFO | "huge_shared" : " 27 GB", | 20260507 03:34:45.707 | INFO | "malloc" : " 14 GB", | 20260507 03:34:45.707 | INFO | "pinned" : " 29 GB" | 20260507 03:34:45.707 | INFO | } | 20260507 03:34:45.707 | INFO | Perf report for the last 5.13646 seconds: | 20260507 03:34:45.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:34:45.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:45.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:45.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:45.707 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:34:45.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:45.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:45.707 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:34:45.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:45.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:45.708 | INFO | Uploaded 36GiB to GPU in 2.0473s (17.5841GiB/s) | 20260507 03:34:45.708 | INFO | Downloaded 34GiB to GPU in 1.96668s (17.288GiB/s) | 20260507 03:34:50.853 | INFO | Analyzing chunk frames = [6773760,6774272) | 20260507 03:34:50.853 | INFO | MemInfo: { | 20260507 03:34:50.853 | INFO | "gpu" : " 17 GB", | 20260507 03:34:50.853 | INFO | "huge" : "", | 20260507 03:34:50.853 | INFO | "huge_shared" : " 27 GB", | 20260507 03:34:50.853 | INFO | "malloc" : " 14 GB", | 20260507 03:34:50.853 | INFO | "pinned" : " 29 GB" | 20260507 03:34:50.853 | INFO | } | 20260507 03:34:50.854 | INFO | Perf report for the last 5.14626 seconds: | 20260507 03:34:50.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:34:50.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:50.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:50.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:50.854 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:34:50.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:50.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:50.854 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:34:50.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:50.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:50.854 | INFO | Uploaded 36GiB to GPU in 1.96738s (18.2984GiB/s) | 20260507 03:34:50.854 | INFO | Downloaded 34GiB to GPU in 1.98249s (17.1502GiB/s) | 20260507 03:34:55.992 | INFO | Analyzing chunk frames = [6774272,6774784) | 20260507 03:34:55.992 | INFO | MemInfo: { | 20260507 03:34:55.992 | INFO | "gpu" : " 17 GB", | 20260507 03:34:55.992 | INFO | "huge" : "", | 20260507 03:34:55.992 | INFO | "huge_shared" : " 27 GB", | 20260507 03:34:55.992 | INFO | "malloc" : " 14 GB", | 20260507 03:34:55.992 | INFO | "pinned" : " 29 GB" | 20260507 03:34:55.992 | INFO | } | 20260507 03:34:55.992 | INFO | Perf report for the last 5.13858 seconds: | 20260507 03:34:55.992 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:34:55.992 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:55.992 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:55.992 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:55.992 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:34:55.992 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:55.992 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:55.992 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 03:34:55.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:55.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:34:55.992 | INFO | Uploaded 36GiB to GPU in 1.97393s (18.2377GiB/s) | 20260507 03:34:55.992 | INFO | Downloaded 34GiB to GPU in 1.97508s (17.2145GiB/s) | 20260507 03:34:55.993 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:34:55.993 | INFO | Total mem usage: 18 GiB | 20260507 03:34:55.993 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:34:55.993 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:34:55.993 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:34:55.993 | INFO | High Water Breakdown: | 20260507 03:34:55.993 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:34:55.993 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:34:55.993 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:34:55.993 | INFO | ------------------------------------------ | 20260507 03:34:55.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:34:55.993 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:34:55.993 | INFO | Total mem usage: 26.5 GiB | 20260507 03:34:55.993 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:34:55.993 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:34:55.993 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 03:34:55.993 | INFO | High Water Breakdown: | 20260507 03:34:55.993 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:34:55.993 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:34:55.993 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:34:55.993 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:34:55.993 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:34:55.993 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:34:55.993 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:34:55.993 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:34:55.993 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:34:55.993 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:34:55.993 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:34:55.993 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:34:55.993 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:34:55.993 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:34:55.993 | INFO | ------------------------------------------ | 20260507 03:34:55.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:34:55.993 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:34:55.993 | INFO | Total mem usage: 25.6 GiB | 20260507 03:34:55.993 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:34:55.993 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:34:55.993 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:34:55.993 | INFO | High Water Breakdown: | 20260507 03:34:55.993 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:34:55.993 | INFO | ------------------------------------------ | 20260507 03:34:55.993 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:34:55.993 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:34:55.994 | INFO | Total mem usage: 15.7 GiB | 20260507 03:34:55.994 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:34:55.994 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:34:55.994 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 03:34:55.994 | INFO | High Water Breakdown: | 20260507 03:34:55.994 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:34:55.994 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:34:55.994 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:34:55.994 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:34:55.994 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:34:55.994 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:34:55.994 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:34:55.994 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:34:55.994 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:34:55.994 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:34:55.994 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:34:55.994 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:34:55.994 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:34:55.994 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:34:55.994 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:34:55.994 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:34:55.994 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:34:55.994 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:34:55.994 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:34:55.994 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:34:55.994 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:34:55.994 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:34:55.994 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:34:55.994 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:34:55.994 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:34:55.994 | INFO | ------------------------------------------ | 20260507 03:34:55.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:34:55.994 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:34:55.994 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:34:55.994 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:34:55.994 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.51e+01, 1.34e+01 / 1.51e+00 | 20260507 03:34:55.994 | INFO | PerfReport: 0.29%, AggregateStats -- 9.00e-02 / 1.01e-01, 8.93e-03 / 6.20e-03 | 20260507 03:34:55.994 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.47e+00, 5.12e-01 / 2.52e-01 | 20260507 03:34:55.994 | INFO | PerfReport: 3.22%, Histogram -- 9.90e-01 / 1.46e+00, 1.66e-01 / 1.03e-01 | 20260507 03:34:55.994 | INFO | PerfReport: 16.14%, DME -- 3.97e+01 / 4.45e+01, 3.97e+00 / 1.69e+00 | 20260507 03:34:55.994 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.26e+00 / 1.54e+00 | 20260507 03:34:55.994 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.14e+00 / 2.74e+00, 3.87e-01 / 2.11e-01 | 20260507 03:34:55.994 | INFO | PerfReport: 3.05%, Metrics -- 9.36e-01 / 1.27e+00, 2.48e-01 / 1.09e-01 | 20260507 03:34:55.994 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.16e-01 / 3.72e-01, 4.90e-02 / 2.16e-02 | 20260507 03:34:55.994 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:34:55.994 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:34:55.994 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:34:55.994 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 03:34:55.994 | INFO | PerfReport: 21.75%, Upload -- 2.09e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260507 03:34:55.994 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 03:34:55.994 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 03:34:55.994 | INFO | PerfReport: 1.48%, LockUpload -- 1.43e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 03:34:55.994 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260507 03:34:55.994 | INFO | PerfReport: 22.13%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 03:34:55.994 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260507 03:34:56.089 | INFO | Starting a round of Baz IO | 20260507 03:34:59.190 | INFO | Populating RTMetrics report | 20260507 03:34:59.191 | INFO | RTMetrics report populated | 20260507 03:34:59.191 | INFO | Opening tmp RTMetrics json file | 20260507 03:34:59.209 | INFO | Serializing RTMetrics to json | 20260507 03:34:59.210 | INFO | Serialization done, swapping tmp file now... | 20260507 03:34:59.211 | INFO | RTMetrics done | 20260507 03:34:59.281 | INFO | Finished a round of Baz IO: Wrote 1.748 GiB in 3.192 seconds | 20260507 03:34:59.281 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.88% / 0.00% / 67.12% | 20260507 03:34:59.281 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.195964% / 0% / 7.54992e-05% | 20260507 03:34:59.281 | INFO | Baz IO overall: | 20260507 03:34:59.281 | INFO | BytesWritten : 3474868083091 | 20260507 03:34:59.281 | INFO | NumEvents : 1669489504776 | 20260507 03:35:01.125 | INFO | Analyzing chunk frames = [6774784,6775296) | 20260507 03:35:01.125 | INFO | MemInfo: { | 20260507 03:35:01.125 | INFO | "gpu" : " 17 GB", | 20260507 03:35:01.125 | INFO | "huge" : "", | 20260507 03:35:01.125 | INFO | "huge_shared" : " 27 GB", | 20260507 03:35:01.125 | INFO | "malloc" : " 14 GB", | 20260507 03:35:01.125 | INFO | "pinned" : " 29 GB" | 20260507 03:35:01.125 | INFO | } | 20260507 03:35:01.125 | INFO | Perf report for the last 5.13314 seconds: | 20260507 03:35:01.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:35:01.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:01.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:01.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:01.125 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260507 03:35:01.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:01.125 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 03:35:01.125 | INFO | PRE_HQ : 60.1%, 1.9, 58.2ms, 0 | 20260507 03:35:01.125 | INFO | BAZWRITER : 61.8%, 3.4, 108.6ms, 0 | 20260507 03:35:01.125 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:35:01.126 | INFO | Uploaded 36GiB to GPU in 2.04369s (17.6152GiB/s) | 20260507 03:35:01.126 | INFO | Downloaded 38GiB to GPU in 2.18377s (17.4011GiB/s) | 20260507 03:35:06.261 | INFO | Analyzing chunk frames = [6775296,6775808) | 20260507 03:35:06.261 | INFO | MemInfo: { | 20260507 03:35:06.261 | INFO | "gpu" : " 17 GB", | 20260507 03:35:06.261 | INFO | "huge" : "", | 20260507 03:35:06.261 | INFO | "huge_shared" : " 27 GB", | 20260507 03:35:06.261 | INFO | "malloc" : " 14 GB", | 20260507 03:35:06.261 | INFO | "pinned" : " 29 GB" | 20260507 03:35:06.261 | INFO | } | 20260507 03:35:06.261 | INFO | Perf report for the last 5.13564 seconds: | 20260507 03:35:06.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:35:06.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:06.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:06.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:06.261 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:35:06.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:06.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:06.261 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 03:35:06.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:06.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:06.261 | INFO | Uploaded 36GiB to GPU in 2.04145s (17.6345GiB/s) | 20260507 03:35:06.261 | INFO | Downloaded 34GiB to GPU in 1.94286s (17.5GiB/s) | 20260507 03:35:11.401 | INFO | Analyzing chunk frames = [6775808,6776320) | 20260507 03:35:11.401 | INFO | MemInfo: { | 20260507 03:35:11.401 | INFO | "gpu" : " 17 GB", | 20260507 03:35:11.401 | INFO | "huge" : "", | 20260507 03:35:11.401 | INFO | "huge_shared" : " 27 GB", | 20260507 03:35:11.401 | INFO | "malloc" : " 14 GB", | 20260507 03:35:11.401 | INFO | "pinned" : " 29 GB" | 20260507 03:35:11.401 | INFO | } | 20260507 03:35:11.401 | INFO | Perf report for the last 5.13994 seconds: | 20260507 03:35:11.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:35:11.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:11.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:11.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:11.401 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 03:35:11.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:11.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:11.401 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:35:11.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:11.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:11.401 | INFO | Uploaded 36GiB to GPU in 1.99569s (18.0389GiB/s) | 20260507 03:35:11.401 | INFO | Downloaded 34GiB to GPU in 1.93839s (17.5403GiB/s) | 20260507 03:35:16.545 | INFO | Analyzing chunk frames = [6776320,6776832) | 20260507 03:35:16.545 | INFO | MemInfo: { | 20260507 03:35:16.545 | INFO | "gpu" : " 17 GB", | 20260507 03:35:16.545 | INFO | "huge" : "", | 20260507 03:35:16.545 | INFO | "huge_shared" : " 27 GB", | 20260507 03:35:16.545 | INFO | "malloc" : " 14 GB", | 20260507 03:35:16.545 | INFO | "pinned" : " 29 GB" | 20260507 03:35:16.545 | INFO | } | 20260507 03:35:16.545 | INFO | Perf report for the last 5.14459 seconds: | 20260507 03:35:16.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:35:16.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:16.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:16.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:16.546 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 03:35:16.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:16.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:16.546 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:35:16.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:16.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:16.546 | INFO | Uploaded 36GiB to GPU in 1.99896s (18.0094GiB/s) | 20260507 03:35:16.546 | INFO | Downloaded 34GiB to GPU in 1.91299s (17.7732GiB/s) | 20260507 03:35:21.694 | INFO | Analyzing chunk frames = [6776832,6777344) | 20260507 03:35:21.694 | INFO | MemInfo: { | 20260507 03:35:21.694 | INFO | "gpu" : " 17 GB", | 20260507 03:35:21.694 | INFO | "huge" : "", | 20260507 03:35:21.694 | INFO | "huge_shared" : " 27 GB", | 20260507 03:35:21.694 | INFO | "malloc" : " 14 GB", | 20260507 03:35:21.694 | INFO | "pinned" : " 29 GB" | 20260507 03:35:21.694 | INFO | } | 20260507 03:35:21.694 | INFO | Perf report for the last 5.14887 seconds: | 20260507 03:35:21.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:35:21.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:21.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:21.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:21.694 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 03:35:21.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:21.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:21.694 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:35:21.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:21.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:21.695 | INFO | Uploaded 36GiB to GPU in 2.00812s (17.9272GiB/s) | 20260507 03:35:21.695 | INFO | Downloaded 34GiB to GPU in 1.94121s (17.5149GiB/s) | 20260507 03:35:26.832 | INFO | Analyzing chunk frames = [6777344,6777856) | 20260507 03:35:26.832 | INFO | MemInfo: { | 20260507 03:35:26.832 | INFO | "gpu" : " 17 GB", | 20260507 03:35:26.832 | INFO | "huge" : "", | 20260507 03:35:26.832 | INFO | "huge_shared" : " 27 GB", | 20260507 03:35:26.832 | INFO | "malloc" : " 14 GB", | 20260507 03:35:26.832 | INFO | "pinned" : " 29 GB" | 20260507 03:35:26.832 | INFO | } | 20260507 03:35:26.832 | INFO | Perf report for the last 5.13762 seconds: | 20260507 03:35:26.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:35:26.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:26.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:26.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:26.832 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 03:35:26.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:26.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:26.832 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:35:26.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:26.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:26.832 | INFO | Uploaded 36GiB to GPU in 2.02886s (17.744GiB/s) | 20260507 03:35:26.832 | INFO | Downloaded 34GiB to GPU in 1.90914s (17.8091GiB/s) | 20260507 03:35:31.966 | INFO | Analyzing chunk frames = [6777856,6778368) | 20260507 03:35:31.966 | INFO | MemInfo: { | 20260507 03:35:31.966 | INFO | "gpu" : " 17 GB", | 20260507 03:35:31.966 | INFO | "huge" : "", | 20260507 03:35:31.966 | INFO | "huge_shared" : " 27 GB", | 20260507 03:35:31.966 | INFO | "malloc" : " 14 GB", | 20260507 03:35:31.967 | INFO | "pinned" : " 29 GB" | 20260507 03:35:31.967 | INFO | } | 20260507 03:35:31.967 | INFO | Perf report for the last 5.13462 seconds: | 20260507 03:35:31.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:35:31.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:31.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:31.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:31.967 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 03:35:31.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:31.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:31.967 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:35:31.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:31.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:31.967 | INFO | Uploaded 36GiB to GPU in 1.99266s (18.0663GiB/s) | 20260507 03:35:31.967 | INFO | Downloaded 34GiB to GPU in 1.96441s (17.308GiB/s) | 20260507 03:35:37.106 | INFO | Analyzing chunk frames = [6778368,6778880) | 20260507 03:35:37.106 | INFO | MemInfo: { | 20260507 03:35:37.106 | INFO | "gpu" : " 17 GB", | 20260507 03:35:37.106 | INFO | "huge" : "", | 20260507 03:35:37.106 | INFO | "huge_shared" : " 27 GB", | 20260507 03:35:37.106 | INFO | "malloc" : " 14 GB", | 20260507 03:35:37.106 | INFO | "pinned" : " 29 GB" | 20260507 03:35:37.106 | INFO | } | 20260507 03:35:37.106 | INFO | Perf report for the last 5.13967 seconds: | 20260507 03:35:37.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:35:37.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:37.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:37.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:37.106 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:35:37.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:37.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:37.106 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 03:35:37.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:37.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:37.107 | INFO | Uploaded 36GiB to GPU in 1.96751s (18.2972GiB/s) | 20260507 03:35:37.107 | INFO | Downloaded 34GiB to GPU in 1.94898s (17.4451GiB/s) | 20260507 03:35:37.203 | INFO | Starting a round of Baz IO | 20260507 03:35:40.316 | INFO | Populating RTMetrics report | 20260507 03:35:40.317 | INFO | RTMetrics report populated | 20260507 03:35:40.317 | INFO | Opening tmp RTMetrics json file | 20260507 03:35:40.321 | INFO | Serializing RTMetrics to json | 20260507 03:35:40.335 | INFO | Serialization done, swapping tmp file now... | 20260507 03:35:40.336 | INFO | RTMetrics done | 20260507 03:35:40.408 | INFO | Finished a round of Baz IO: Wrote 1.749 GiB in 3.204 seconds | 20260507 03:35:40.408 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.88% / 0.00% / 67.12% | 20260507 03:35:40.408 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.196505% / 0% / 9.13938e-05% | 20260507 03:35:40.408 | INFO | Baz IO overall: | 20260507 03:35:40.408 | INFO | BytesWritten : 3476745627972 | 20260507 03:35:40.408 | INFO | NumEvents : 1670375131155 | 20260507 03:35:42.239 | INFO | Analyzing chunk frames = [6778880,6779392) | 20260507 03:35:42.239 | INFO | MemInfo: { | 20260507 03:35:42.239 | INFO | "gpu" : " 17 GB", | 20260507 03:35:42.239 | INFO | "huge" : "", | 20260507 03:35:42.239 | INFO | "huge_shared" : " 27 GB", | 20260507 03:35:42.239 | INFO | "malloc" : " 14 GB", | 20260507 03:35:42.239 | INFO | "pinned" : " 29 GB" | 20260507 03:35:42.239 | INFO | } | 20260507 03:35:42.239 | INFO | Perf report for the last 5.13286 seconds: | 20260507 03:35:42.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:35:42.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:42.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:42.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:42.239 | INFO | ANALYSIS : 62.0%, 3.0, 96.4ms, 0 | 20260507 03:35:42.239 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:42.239 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:35:42.239 | INFO | PRE_HQ : 59.3%, 1.8, 55.1ms, 0 | 20260507 03:35:42.239 | INFO | BAZWRITER : 62.0%, 3.3, 104.4ms, 0 | 20260507 03:35:42.239 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:35:42.239 | INFO | Uploaded 36GiB to GPU in 2.03842s (17.6608GiB/s) | 20260507 03:35:42.240 | INFO | Downloaded 38GiB to GPU in 2.17672s (17.4575GiB/s) | 20260507 03:35:47.385 | INFO | Analyzing chunk frames = [6779392,6779904) | 20260507 03:35:47.385 | INFO | MemInfo: { | 20260507 03:35:47.385 | INFO | "gpu" : " 17 GB", | 20260507 03:35:47.385 | INFO | "huge" : "", | 20260507 03:35:47.385 | INFO | "huge_shared" : " 27 GB", | 20260507 03:35:47.385 | INFO | "malloc" : " 14 GB", | 20260507 03:35:47.385 | INFO | "pinned" : " 29 GB" | 20260507 03:35:47.385 | INFO | } | 20260507 03:35:47.385 | INFO | Perf report for the last 5.14568 seconds: | 20260507 03:35:47.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:35:47.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:47.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:47.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:47.385 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 03:35:47.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:47.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:47.385 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:35:47.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:47.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:47.385 | INFO | Uploaded 36GiB to GPU in 2.043s (17.6211GiB/s) | 20260507 03:35:47.385 | INFO | Downloaded 34GiB to GPU in 1.94321s (17.4968GiB/s) | 20260507 03:35:52.518 | INFO | Analyzing chunk frames = [6779904,6780416) | 20260507 03:35:52.518 | INFO | MemInfo: { | 20260507 03:35:52.518 | INFO | "gpu" : " 17 GB", | 20260507 03:35:52.518 | INFO | "huge" : "", | 20260507 03:35:52.518 | INFO | "huge_shared" : " 27 GB", | 20260507 03:35:52.518 | INFO | "malloc" : " 14 GB", | 20260507 03:35:52.518 | INFO | "pinned" : " 29 GB" | 20260507 03:35:52.518 | INFO | } | 20260507 03:35:52.518 | INFO | Perf report for the last 5.13349 seconds: | 20260507 03:35:52.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:35:52.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:52.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:52.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:52.518 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 03:35:52.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:52.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:52.518 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:35:52.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:52.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:52.519 | INFO | Uploaded 36GiB to GPU in 2.01507s (17.8654GiB/s) | 20260507 03:35:52.519 | INFO | Downloaded 34GiB to GPU in 1.9569s (17.3744GiB/s) | 20260507 03:35:57.661 | INFO | Analyzing chunk frames = [6780416,6780928) | 20260507 03:35:57.661 | INFO | MemInfo: { | 20260507 03:35:57.661 | INFO | "gpu" : " 17 GB", | 20260507 03:35:57.661 | INFO | "huge" : "", | 20260507 03:35:57.661 | INFO | "huge_shared" : " 27 GB", | 20260507 03:35:57.661 | INFO | "malloc" : " 14 GB", | 20260507 03:35:57.661 | INFO | "pinned" : " 29 GB" | 20260507 03:35:57.661 | INFO | } | 20260507 03:35:57.661 | INFO | Perf report for the last 5.14275 seconds: | 20260507 03:35:57.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:35:57.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:57.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:57.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:57.661 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 03:35:57.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:57.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:57.661 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 03:35:57.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:57.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:35:57.661 | INFO | Uploaded 36GiB to GPU in 1.96094s (18.3585GiB/s) | 20260507 03:35:57.662 | INFO | Downloaded 34GiB to GPU in 1.92095s (17.6996GiB/s) | 20260507 03:36:02.804 | INFO | Analyzing chunk frames = [6780928,6781440) | 20260507 03:36:02.804 | INFO | MemInfo: { | 20260507 03:36:02.804 | INFO | "gpu" : " 17 GB", | 20260507 03:36:02.804 | INFO | "huge" : "", | 20260507 03:36:02.804 | INFO | "huge_shared" : " 27 GB", | 20260507 03:36:02.804 | INFO | "malloc" : " 14 GB", | 20260507 03:36:02.804 | INFO | "pinned" : " 29 GB" | 20260507 03:36:02.804 | INFO | } | 20260507 03:36:02.804 | INFO | Perf report for the last 5.14306 seconds: | 20260507 03:36:02.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:36:02.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:02.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:02.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:02.804 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 03:36:02.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:02.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:02.804 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:36:02.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:02.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:02.805 | INFO | Uploaded 36GiB to GPU in 2.02289s (17.7963GiB/s) | 20260507 03:36:02.805 | INFO | Downloaded 34GiB to GPU in 1.94254s (17.5029GiB/s) | 20260507 03:36:07.942 | INFO | Analyzing chunk frames = [6781440,6781952) | 20260507 03:36:07.942 | INFO | MemInfo: { | 20260507 03:36:07.942 | INFO | "gpu" : " 17 GB", | 20260507 03:36:07.942 | INFO | "huge" : "", | 20260507 03:36:07.943 | INFO | "huge_shared" : " 27 GB", | 20260507 03:36:07.943 | INFO | "malloc" : " 14 GB", | 20260507 03:36:07.943 | INFO | "pinned" : " 29 GB" | 20260507 03:36:07.943 | INFO | } | 20260507 03:36:07.943 | INFO | Perf report for the last 5.13848 seconds: | 20260507 03:36:07.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:36:07.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:07.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:07.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:07.943 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 03:36:07.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:07.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:07.943 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 03:36:07.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:07.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:07.943 | INFO | Uploaded 36GiB to GPU in 2.0308s (17.727GiB/s) | 20260507 03:36:07.943 | INFO | Downloaded 34GiB to GPU in 1.91468s (17.7575GiB/s) | 20260507 03:36:13.073 | INFO | Analyzing chunk frames = [6781952,6782464) | 20260507 03:36:13.073 | INFO | MemInfo: { | 20260507 03:36:13.073 | INFO | "gpu" : " 17 GB", | 20260507 03:36:13.073 | INFO | "huge" : "", | 20260507 03:36:13.073 | INFO | "huge_shared" : " 27 GB", | 20260507 03:36:13.073 | INFO | "malloc" : " 14 GB", | 20260507 03:36:13.073 | INFO | "pinned" : " 29 GB" | 20260507 03:36:13.073 | INFO | } | 20260507 03:36:13.073 | INFO | Perf report for the last 5.13068 seconds: | 20260507 03:36:13.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:36:13.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:13.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:13.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:13.073 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 03:36:13.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:13.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:13.074 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:36:13.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:13.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:13.074 | INFO | Uploaded 36GiB to GPU in 1.97602s (18.2185GiB/s) | 20260507 03:36:13.074 | INFO | Downloaded 34GiB to GPU in 1.97909s (17.1796GiB/s) | 20260507 03:36:18.211 | INFO | Analyzing chunk frames = [6782464,6782976) | 20260507 03:36:18.211 | INFO | MemInfo: { | 20260507 03:36:18.211 | INFO | "gpu" : " 17 GB", | 20260507 03:36:18.211 | INFO | "huge" : "", | 20260507 03:36:18.211 | INFO | "huge_shared" : " 27 GB", | 20260507 03:36:18.211 | INFO | "malloc" : " 14 GB", | 20260507 03:36:18.211 | INFO | "pinned" : " 29 GB" | 20260507 03:36:18.211 | INFO | } | 20260507 03:36:18.211 | INFO | Perf report for the last 5.138 seconds: | 20260507 03:36:18.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:36:18.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:18.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:18.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:18.211 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:36:18.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:18.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:18.212 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 03:36:18.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:18.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:18.212 | INFO | Uploaded 36GiB to GPU in 1.98198s (18.1637GiB/s) | 20260507 03:36:18.212 | INFO | Downloaded 34GiB to GPU in 1.94964s (17.4391GiB/s) | 20260507 03:36:18.212 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:36:18.212 | INFO | Total mem usage: 18 GiB | 20260507 03:36:18.212 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:36:18.212 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:36:18.212 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:36:18.212 | INFO | High Water Breakdown: | 20260507 03:36:18.212 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:36:18.212 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:36:18.212 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:36:18.212 | INFO | ------------------------------------------ | 20260507 03:36:18.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:36:18.212 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:36:18.212 | INFO | Total mem usage: 26.5 GiB | 20260507 03:36:18.212 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:36:18.212 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:36:18.212 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:36:18.212 | INFO | High Water Breakdown: | 20260507 03:36:18.212 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:36:18.212 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:36:18.212 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:36:18.212 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:36:18.212 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:36:18.212 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:36:18.212 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:36:18.212 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:36:18.212 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:36:18.212 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:36:18.212 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:36:18.212 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:36:18.212 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:36:18.212 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:36:18.212 | INFO | ------------------------------------------ | 20260507 03:36:18.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:36:18.212 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:36:18.212 | INFO | Total mem usage: 25.6 GiB | 20260507 03:36:18.212 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:36:18.212 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:36:18.212 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:36:18.212 | INFO | High Water Breakdown: | 20260507 03:36:18.212 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:36:18.212 | INFO | ------------------------------------------ | 20260507 03:36:18.212 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:36:18.212 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:36:18.212 | INFO | Total mem usage: 15.7 GiB | 20260507 03:36:18.212 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:36:18.212 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:36:18.212 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:36:18.212 | INFO | High Water Breakdown: | 20260507 03:36:18.213 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:36:18.213 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:36:18.213 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:36:18.213 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:36:18.213 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:36:18.213 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:36:18.213 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:36:18.213 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:36:18.213 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:36:18.213 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:36:18.213 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:36:18.213 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:36:18.213 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:36:18.213 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:36:18.213 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:36:18.213 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:36:18.213 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:36:18.213 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:36:18.213 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:36:18.213 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:36:18.213 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:36:18.213 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:36:18.213 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:36:18.213 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:36:18.213 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:36:18.213 | INFO | ------------------------------------------ | 20260507 03:36:18.213 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:36:18.213 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:36:18.213 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:36:18.213 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:36:18.213 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.50e+01, 1.33e+01 / 1.59e+00 | 20260507 03:36:18.213 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 1.01e-01, 9.92e-03 / 4.40e-03 | 20260507 03:36:18.213 | INFO | PerfReport: 11.68%, Baseline -- 3.60e+00 / 4.47e+00, 5.26e-01 / 2.27e-01 | 20260507 03:36:18.213 | INFO | PerfReport: 3.24%, Histogram -- 9.97e-01 / 1.47e+00, 1.72e-01 / 1.13e-01 | 20260507 03:36:18.213 | INFO | PerfReport: 16.11%, DME -- 3.97e+01 / 4.46e+01, 4.02e+00 / 1.99e+00 | 20260507 03:36:18.213 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.78e+01 / 2.01e+01, 1.38e+00 / 1.83e+00 | 20260507 03:36:18.213 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.14e+00 / 2.70e+00, 3.77e-01 / 1.86e-01 | 20260507 03:36:18.213 | INFO | PerfReport: 3.05%, Metrics -- 9.38e-01 / 1.26e+00, 2.52e-01 / 1.68e-01 | 20260507 03:36:18.213 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.17e-01 / 3.72e-01, 4.93e-02 / 1.82e-02 | 20260507 03:36:18.213 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:36:18.213 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:36:18.213 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:36:18.213 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 03:36:18.213 | INFO | PerfReport: 21.75%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260507 03:36:18.213 | INFO | PerfReport: 21.33%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260507 03:36:18.213 | INFO | PerfReport: 32.00%, Compute -- 3.08e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 03:36:18.213 | INFO | PerfReport: 1.37%, LockUpload -- 1.32e+00 / 0.00e+00, 4.69e+00 / 0.00e+00 | 20260507 03:36:18.213 | INFO | PerfReport: 1.37%, LockDownload -- 1.32e+00 / 0.00e+00, 3.17e+00 / 0.00e+00 | 20260507 03:36:18.213 | INFO | PerfReport: 22.17%, LockCompute -- 2.13e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 03:36:18.213 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260507 03:36:18.308 | INFO | Starting a round of Baz IO | 20260507 03:36:21.426 | INFO | Populating RTMetrics report | 20260507 03:36:21.426 | INFO | RTMetrics report populated | 20260507 03:36:21.426 | INFO | Opening tmp RTMetrics json file | 20260507 03:36:21.444 | INFO | Serializing RTMetrics to json | 20260507 03:36:21.445 | INFO | Serialization done, swapping tmp file now... | 20260507 03:36:21.446 | INFO | RTMetrics done | 20260507 03:36:21.516 | INFO | Finished a round of Baz IO: Wrote 1.747 GiB in 3.207 seconds | 20260507 03:36:21.516 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.88% / 0.00% / 67.12% | 20260507 03:36:21.516 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.195344% / 0% / 9.93411e-05% | 20260507 03:36:21.516 | INFO | Baz IO overall: | 20260507 03:36:21.516 | INFO | BytesWritten : 3478621786525 | 20260507 03:36:21.516 | INFO | NumEvents : 1671259784429 | 20260507 03:36:23.358 | INFO | Analyzing chunk frames = [6782976,6783488) | 20260507 03:36:23.358 | INFO | MemInfo: { | 20260507 03:36:23.358 | INFO | "gpu" : " 17 GB", | 20260507 03:36:23.358 | INFO | "huge" : "", | 20260507 03:36:23.358 | INFO | "huge_shared" : " 27 GB", | 20260507 03:36:23.358 | INFO | "malloc" : " 14 GB", | 20260507 03:36:23.358 | INFO | "pinned" : " 29 GB" | 20260507 03:36:23.358 | INFO | } | 20260507 03:36:23.359 | INFO | Perf report for the last 5.1471 seconds: | 20260507 03:36:23.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:36:23.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:23.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:23.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:23.359 | INFO | ANALYSIS : 61.9%, 3.0, 96.5ms, 0 | 20260507 03:36:23.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:23.359 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 03:36:23.359 | INFO | PRE_HQ : 59.7%, 1.8, 55.5ms, 0 | 20260507 03:36:23.359 | INFO | BAZWRITER : 61.9%, 3.3, 104.9ms, 0 | 20260507 03:36:23.359 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:36:23.359 | INFO | Uploaded 36GiB to GPU in 2.04113s (17.6373GiB/s) | 20260507 03:36:23.359 | INFO | Downloaded 38GiB to GPU in 2.17582s (17.4647GiB/s) | 20260507 03:36:28.500 | INFO | Analyzing chunk frames = [6783488,6784000) | 20260507 03:36:28.500 | INFO | MemInfo: { | 20260507 03:36:28.500 | INFO | "gpu" : " 17 GB", | 20260507 03:36:28.500 | INFO | "huge" : "", | 20260507 03:36:28.500 | INFO | "huge_shared" : " 27 GB", | 20260507 03:36:28.500 | INFO | "malloc" : " 14 GB", | 20260507 03:36:28.500 | INFO | "pinned" : " 29 GB" | 20260507 03:36:28.500 | INFO | } | 20260507 03:36:28.500 | INFO | Perf report for the last 5.14184 seconds: | 20260507 03:36:28.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:36:28.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:28.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:28.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:28.500 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:36:28.500 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:28.500 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:28.500 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:36:28.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:28.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:28.501 | INFO | Uploaded 36GiB to GPU in 2.02343s (17.7916GiB/s) | 20260507 03:36:28.501 | INFO | Downloaded 34GiB to GPU in 1.93498s (17.5712GiB/s) | 20260507 03:36:33.639 | INFO | Analyzing chunk frames = [6784000,6784512) | 20260507 03:36:33.639 | INFO | MemInfo: { | 20260507 03:36:33.639 | INFO | "gpu" : " 17 GB", | 20260507 03:36:33.639 | INFO | "huge" : "", | 20260507 03:36:33.639 | INFO | "huge_shared" : " 27 GB", | 20260507 03:36:33.639 | INFO | "malloc" : " 14 GB", | 20260507 03:36:33.640 | INFO | "pinned" : " 29 GB" | 20260507 03:36:33.640 | INFO | } | 20260507 03:36:33.640 | INFO | Perf report for the last 5.13925 seconds: | 20260507 03:36:33.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:36:33.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:33.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:33.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:33.640 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 03:36:33.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:33.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:33.640 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:36:33.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:33.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:33.640 | INFO | Uploaded 36GiB to GPU in 2.02988s (17.735GiB/s) | 20260507 03:36:33.640 | INFO | Downloaded 34GiB to GPU in 1.9491s (17.444GiB/s) | 20260507 03:36:38.776 | INFO | Analyzing chunk frames = [6784512,6785024) | 20260507 03:36:38.776 | INFO | MemInfo: { | 20260507 03:36:38.776 | INFO | "gpu" : " 17 GB", | 20260507 03:36:38.776 | INFO | "huge" : "", | 20260507 03:36:38.776 | INFO | "huge_shared" : " 27 GB", | 20260507 03:36:38.776 | INFO | "malloc" : " 14 GB", | 20260507 03:36:38.776 | INFO | "pinned" : " 29 GB" | 20260507 03:36:38.776 | INFO | } | 20260507 03:36:38.776 | INFO | Perf report for the last 5.13658 seconds: | 20260507 03:36:38.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:36:38.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:38.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:38.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:38.776 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 03:36:38.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:38.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:38.776 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:36:38.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:38.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:38.777 | INFO | Uploaded 36GiB to GPU in 2.02254s (17.7994GiB/s) | 20260507 03:36:38.777 | INFO | Downloaded 34GiB to GPU in 1.94058s (17.5205GiB/s) | 20260507 03:36:43.913 | INFO | Analyzing chunk frames = [6785024,6785536) | 20260507 03:36:43.913 | INFO | MemInfo: { | 20260507 03:36:43.913 | INFO | "gpu" : " 17 GB", | 20260507 03:36:43.913 | INFO | "huge" : "", | 20260507 03:36:43.913 | INFO | "huge_shared" : " 27 GB", | 20260507 03:36:43.913 | INFO | "malloc" : " 14 GB", | 20260507 03:36:43.913 | INFO | "pinned" : " 29 GB" | 20260507 03:36:43.913 | INFO | } | 20260507 03:36:43.913 | INFO | Perf report for the last 5.13717 seconds: | 20260507 03:36:43.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:36:43.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:43.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:43.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:43.913 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 03:36:43.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:43.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:43.913 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:36:43.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:43.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:43.915 | INFO | Uploaded 36GiB to GPU in 2.05788s (17.4937GiB/s) | 20260507 03:36:43.915 | INFO | Downloaded 34GiB to GPU in 1.94629s (17.4692GiB/s) | 20260507 03:36:49.056 | INFO | Analyzing chunk frames = [6785536,6786048) | 20260507 03:36:49.056 | INFO | MemInfo: { | 20260507 03:36:49.056 | INFO | "gpu" : " 17 GB", | 20260507 03:36:49.056 | INFO | "huge" : "", | 20260507 03:36:49.057 | INFO | "huge_shared" : " 27 GB", | 20260507 03:36:49.057 | INFO | "malloc" : " 14 GB", | 20260507 03:36:49.057 | INFO | "pinned" : " 29 GB" | 20260507 03:36:49.057 | INFO | } | 20260507 03:36:49.057 | INFO | Perf report for the last 5.14329 seconds: | 20260507 03:36:49.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:36:49.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:49.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:49.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:49.057 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 03:36:49.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:49.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:49.057 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 03:36:49.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:49.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:49.057 | INFO | Uploaded 36GiB to GPU in 2.03232s (17.7137GiB/s) | 20260507 03:36:49.057 | INFO | Downloaded 34GiB to GPU in 1.93121s (17.6056GiB/s) | 20260507 03:36:54.195 | INFO | Analyzing chunk frames = [6786048,6786560) | 20260507 03:36:54.195 | INFO | MemInfo: { | 20260507 03:36:54.195 | INFO | "gpu" : " 17 GB", | 20260507 03:36:54.195 | INFO | "huge" : "", | 20260507 03:36:54.195 | INFO | "huge_shared" : " 27 GB", | 20260507 03:36:54.195 | INFO | "malloc" : " 14 GB", | 20260507 03:36:54.195 | INFO | "pinned" : " 29 GB" | 20260507 03:36:54.195 | INFO | } | 20260507 03:36:54.196 | INFO | Perf report for the last 5.13881 seconds: | 20260507 03:36:54.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:36:54.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:54.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:54.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:54.196 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:36:54.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:54.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:54.196 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 03:36:54.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:54.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:54.196 | INFO | Uploaded 36GiB to GPU in 1.98029s (18.1792GiB/s) | 20260507 03:36:54.196 | INFO | Downloaded 34GiB to GPU in 1.95759s (17.3683GiB/s) | 20260507 03:36:59.336 | INFO | Analyzing chunk frames = [6786560,6787072) | 20260507 03:36:59.336 | INFO | MemInfo: { | 20260507 03:36:59.336 | INFO | "gpu" : " 17 GB", | 20260507 03:36:59.336 | INFO | "huge" : "", | 20260507 03:36:59.336 | INFO | "huge_shared" : " 27 GB", | 20260507 03:36:59.337 | INFO | "malloc" : " 14 GB", | 20260507 03:36:59.337 | INFO | "pinned" : " 29 GB" | 20260507 03:36:59.337 | INFO | } | 20260507 03:36:59.337 | INFO | Perf report for the last 5.14111 seconds: | 20260507 03:36:59.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:36:59.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:59.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:59.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:59.337 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:36:59.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:59.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:59.337 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:36:59.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:59.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:36:59.337 | INFO | Uploaded 36GiB to GPU in 1.99408s (18.0534GiB/s) | 20260507 03:36:59.337 | INFO | Downloaded 34GiB to GPU in 1.97472s (17.2176GiB/s) | 20260507 03:36:59.434 | INFO | Starting a round of Baz IO | 20260507 03:37:02.538 | INFO | Populating RTMetrics report | 20260507 03:37:02.538 | INFO | RTMetrics report populated | 20260507 03:37:02.538 | INFO | Opening tmp RTMetrics json file | 20260507 03:37:02.541 | INFO | Serializing RTMetrics to json | 20260507 03:37:02.556 | INFO | Serialization done, swapping tmp file now... | 20260507 03:37:02.557 | INFO | RTMetrics done | 20260507 03:37:02.630 | INFO | Finished a round of Baz IO: Wrote 1.748 GiB in 3.196 seconds | 20260507 03:37:02.630 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.87% / 0.00% / 67.12% | 20260507 03:37:02.630 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.194883% / 0% / 8.34465e-05% | 20260507 03:37:02.630 | INFO | Baz IO overall: | 20260507 03:37:02.630 | INFO | BytesWritten : 3480499038459 | 20260507 03:37:02.630 | INFO | NumEvents : 1672145095179 | 20260507 03:37:04.475 | INFO | Analyzing chunk frames = [6787072,6787584) | 20260507 03:37:04.475 | INFO | MemInfo: { | 20260507 03:37:04.475 | INFO | "gpu" : " 17 GB", | 20260507 03:37:04.475 | INFO | "huge" : "", | 20260507 03:37:04.475 | INFO | "huge_shared" : " 27 GB", | 20260507 03:37:04.475 | INFO | "malloc" : " 14 GB", | 20260507 03:37:04.475 | INFO | "pinned" : " 29 GB" | 20260507 03:37:04.475 | INFO | } | 20260507 03:37:04.475 | INFO | Perf report for the last 5.13861 seconds: | 20260507 03:37:04.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:37:04.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:04.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:04.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:04.475 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 03:37:04.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:04.475 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 03:37:04.475 | INFO | PRE_HQ : 59.4%, 1.8, 55.9ms, 0 | 20260507 03:37:04.475 | INFO | BAZWRITER : 61.8%, 3.3, 104.2ms, 0 | 20260507 03:37:04.475 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:37:04.476 | INFO | Uploaded 36GiB to GPU in 2.04532s (17.6012GiB/s) | 20260507 03:37:04.476 | INFO | Downloaded 38GiB to GPU in 2.19153s (17.3395GiB/s) | 20260507 03:37:09.608 | INFO | Analyzing chunk frames = [6787584,6788096) | 20260507 03:37:09.608 | INFO | MemInfo: { | 20260507 03:37:09.608 | INFO | "gpu" : " 17 GB", | 20260507 03:37:09.608 | INFO | "huge" : "", | 20260507 03:37:09.608 | INFO | "huge_shared" : " 27 GB", | 20260507 03:37:09.608 | INFO | "malloc" : " 14 GB", | 20260507 03:37:09.608 | INFO | "pinned" : " 29 GB" | 20260507 03:37:09.608 | INFO | } | 20260507 03:37:09.608 | INFO | Perf report for the last 5.13277 seconds: | 20260507 03:37:09.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:37:09.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:09.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:09.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:09.608 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:37:09.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:09.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:09.608 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:37:09.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:09.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:09.608 | INFO | Uploaded 36GiB to GPU in 2.03851s (17.66GiB/s) | 20260507 03:37:09.608 | INFO | Downloaded 34GiB to GPU in 1.93839s (17.5404GiB/s) | 20260507 03:37:14.751 | INFO | Analyzing chunk frames = [6788096,6788608) | 20260507 03:37:14.751 | INFO | MemInfo: { | 20260507 03:37:14.751 | INFO | "gpu" : " 17 GB", | 20260507 03:37:14.751 | INFO | "huge" : "", | 20260507 03:37:14.752 | INFO | "huge_shared" : " 27 GB", | 20260507 03:37:14.752 | INFO | "malloc" : " 14 GB", | 20260507 03:37:14.752 | INFO | "pinned" : " 29 GB" | 20260507 03:37:14.752 | INFO | } | 20260507 03:37:14.752 | INFO | Perf report for the last 5.14353 seconds: | 20260507 03:37:14.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:37:14.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:14.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:14.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:14.752 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 03:37:14.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:14.752 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:14.752 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:37:14.752 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:14.752 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:14.752 | INFO | Uploaded 36GiB to GPU in 1.97716s (18.2079GiB/s) | 20260507 03:37:14.752 | INFO | Downloaded 34GiB to GPU in 1.94144s (17.5128GiB/s) | 20260507 03:37:19.890 | INFO | Analyzing chunk frames = [6788608,6789120) | 20260507 03:37:19.890 | INFO | MemInfo: { | 20260507 03:37:19.890 | INFO | "gpu" : " 17 GB", | 20260507 03:37:19.890 | INFO | "huge" : "", | 20260507 03:37:19.890 | INFO | "huge_shared" : " 27 GB", | 20260507 03:37:19.890 | INFO | "malloc" : " 14 GB", | 20260507 03:37:19.890 | INFO | "pinned" : " 29 GB" | 20260507 03:37:19.890 | INFO | } | 20260507 03:37:19.890 | INFO | Perf report for the last 5.1389 seconds: | 20260507 03:37:19.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:37:19.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:19.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:19.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:19.891 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 03:37:19.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:19.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:19.891 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:37:19.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:19.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:19.891 | INFO | Uploaded 36GiB to GPU in 1.98134s (18.1695GiB/s) | 20260507 03:37:19.891 | INFO | Downloaded 34GiB to GPU in 1.92376s (17.6737GiB/s) | 20260507 03:37:25.028 | INFO | Analyzing chunk frames = [6789120,6789632) | 20260507 03:37:25.028 | INFO | MemInfo: { | 20260507 03:37:25.028 | INFO | "gpu" : " 17 GB", | 20260507 03:37:25.028 | INFO | "huge" : "", | 20260507 03:37:25.028 | INFO | "huge_shared" : " 27 GB", | 20260507 03:37:25.028 | INFO | "malloc" : " 14 GB", | 20260507 03:37:25.028 | INFO | "pinned" : " 29 GB" | 20260507 03:37:25.028 | INFO | } | 20260507 03:37:25.028 | INFO | Perf report for the last 5.13747 seconds: | 20260507 03:37:25.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:37:25.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:25.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:25.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:25.028 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 03:37:25.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:25.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:25.028 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:37:25.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:25.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:25.028 | INFO | Uploaded 36GiB to GPU in 2.02492s (17.7785GiB/s) | 20260507 03:37:25.028 | INFO | Downloaded 34GiB to GPU in 1.94169s (17.5105GiB/s) | 20260507 03:37:30.166 | INFO | Analyzing chunk frames = [6789632,6790144) | 20260507 03:37:30.167 | INFO | MemInfo: { | 20260507 03:37:30.167 | INFO | "gpu" : " 17 GB", | 20260507 03:37:30.167 | INFO | "huge" : "", | 20260507 03:37:30.167 | INFO | "huge_shared" : " 27 GB", | 20260507 03:37:30.167 | INFO | "malloc" : " 14 GB", | 20260507 03:37:30.167 | INFO | "pinned" : " 29 GB" | 20260507 03:37:30.167 | INFO | } | 20260507 03:37:30.167 | INFO | Perf report for the last 5.1388 seconds: | 20260507 03:37:30.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:37:30.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:30.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:30.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:30.167 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:37:30.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:30.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:30.167 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 03:37:30.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:30.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:30.167 | INFO | Uploaded 36GiB to GPU in 2.0501s (17.5601GiB/s) | 20260507 03:37:30.167 | INFO | Downloaded 34GiB to GPU in 1.96409s (17.3108GiB/s) | 20260507 03:37:35.314 | INFO | Analyzing chunk frames = [6790144,6790656) | 20260507 03:37:35.314 | INFO | MemInfo: { | 20260507 03:37:35.314 | INFO | "gpu" : " 17 GB", | 20260507 03:37:35.314 | INFO | "huge" : "", | 20260507 03:37:35.314 | INFO | "huge_shared" : " 27 GB", | 20260507 03:37:35.314 | INFO | "malloc" : " 14 GB", | 20260507 03:37:35.314 | INFO | "pinned" : " 29 GB" | 20260507 03:37:35.314 | INFO | } | 20260507 03:37:35.314 | INFO | Perf report for the last 5.14738 seconds: | 20260507 03:37:35.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:37:35.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:35.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:35.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:35.314 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:37:35.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:35.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:35.314 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:37:35.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:35.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:35.315 | INFO | Uploaded 36GiB to GPU in 1.96949s (18.2788GiB/s) | 20260507 03:37:35.315 | INFO | Downloaded 34GiB to GPU in 1.99927s (17.0062GiB/s) | 20260507 03:37:40.451 | INFO | Analyzing chunk frames = [6790656,6791168) | 20260507 03:37:40.451 | INFO | MemInfo: { | 20260507 03:37:40.451 | INFO | "gpu" : " 17 GB", | 20260507 03:37:40.451 | INFO | "huge" : "", | 20260507 03:37:40.451 | INFO | "huge_shared" : " 27 GB", | 20260507 03:37:40.451 | INFO | "malloc" : " 14 GB", | 20260507 03:37:40.451 | INFO | "pinned" : " 29 GB" | 20260507 03:37:40.451 | INFO | } | 20260507 03:37:40.451 | INFO | Perf report for the last 5.13658 seconds: | 20260507 03:37:40.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:37:40.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:40.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:40.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:40.451 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 03:37:40.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:40.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:40.451 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 03:37:40.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:40.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:40.451 | INFO | Uploaded 36GiB to GPU in 1.99243s (18.0683GiB/s) | 20260507 03:37:40.451 | INFO | Downloaded 34GiB to GPU in 1.9732s (17.2309GiB/s) | 20260507 03:37:40.451 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:37:40.451 | INFO | Total mem usage: 18 GiB | 20260507 03:37:40.451 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:37:40.451 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:37:40.451 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:37:40.451 | INFO | High Water Breakdown: | 20260507 03:37:40.451 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:37:40.451 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:37:40.451 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:37:40.451 | INFO | ------------------------------------------ | 20260507 03:37:40.451 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:37:40.451 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:37:40.451 | INFO | Total mem usage: 26.5 GiB | 20260507 03:37:40.451 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:37:40.451 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:37:40.451 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:37:40.451 | INFO | High Water Breakdown: | 20260507 03:37:40.451 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:37:40.451 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:37:40.451 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:37:40.451 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:37:40.451 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:37:40.451 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:37:40.451 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:37:40.451 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:37:40.451 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:37:40.451 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:37:40.451 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:37:40.451 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:37:40.451 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:37:40.451 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:37:40.451 | INFO | ------------------------------------------ | 20260507 03:37:40.451 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:37:40.451 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:37:40.451 | INFO | Total mem usage: 25.6 GiB | 20260507 03:37:40.451 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:37:40.452 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:37:40.452 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:37:40.452 | INFO | High Water Breakdown: | 20260507 03:37:40.452 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:37:40.452 | INFO | ------------------------------------------ | 20260507 03:37:40.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:37:40.452 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:37:40.452 | INFO | Total mem usage: 15.7 GiB | 20260507 03:37:40.452 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:37:40.452 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:37:40.452 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260507 03:37:40.452 | INFO | High Water Breakdown: | 20260507 03:37:40.452 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:37:40.452 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:37:40.452 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:37:40.452 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:37:40.452 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:37:40.452 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:37:40.452 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:37:40.452 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:37:40.452 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:37:40.452 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:37:40.452 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:37:40.452 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:37:40.452 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:37:40.452 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:37:40.452 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:37:40.452 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:37:40.452 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:37:40.452 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:37:40.452 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:37:40.452 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:37:40.452 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:37:40.452 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:37:40.452 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:37:40.452 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:37:40.452 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:37:40.452 | INFO | ------------------------------------------ | 20260507 03:37:40.452 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:37:40.452 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:37:40.452 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:37:40.452 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:37:40.452 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.53e+01, 1.32e+01 / 1.90e+00 | 20260507 03:37:40.452 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 9.88e-02, 9.99e-03 / 4.37e-03 | 20260507 03:37:40.452 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.44e+00, 5.04e-01 / 2.62e-01 | 20260507 03:37:40.452 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.49e+00, 1.69e-01 / 1.05e-01 | 20260507 03:37:40.452 | INFO | PerfReport: 16.19%, DME -- 3.98e+01 / 4.50e+01, 4.04e+00 / 2.29e+00 | 20260507 03:37:40.452 | INFO | PerfReport: 57.56%, FrameLabeling -- 1.77e+01 / 1.99e+01, 1.20e+00 / 1.73e+00 | 20260507 03:37:40.452 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.15e+00 / 2.75e+00, 3.84e-01 / 1.90e-01 | 20260507 03:37:40.452 | INFO | PerfReport: 3.05%, Metrics -- 9.36e-01 / 1.25e+00, 2.44e-01 / 1.23e-01 | 20260507 03:37:40.452 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.17e-01 / 3.71e-01, 4.99e-02 / 1.82e-02 | 20260507 03:37:40.452 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:37:40.452 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:37:40.452 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:37:40.452 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 03:37:40.452 | INFO | PerfReport: 21.80%, Upload -- 2.10e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260507 03:37:40.452 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 03:37:40.452 | INFO | PerfReport: 31.89%, Compute -- 3.07e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 03:37:40.452 | INFO | PerfReport: 1.51%, LockUpload -- 1.46e+00 / 0.00e+00, 4.98e+00 / 0.00e+00 | 20260507 03:37:40.452 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260507 03:37:40.452 | INFO | PerfReport: 22.07%, LockCompute -- 2.13e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 03:37:40.452 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260507 03:37:40.547 | INFO | Starting a round of Baz IO | 20260507 03:37:43.650 | INFO | Populating RTMetrics report | 20260507 03:37:43.650 | INFO | RTMetrics report populated | 20260507 03:37:43.650 | INFO | Opening tmp RTMetrics json file | 20260507 03:37:43.668 | INFO | Serializing RTMetrics to json | 20260507 03:37:43.669 | INFO | Serialization done, swapping tmp file now... | 20260507 03:37:43.670 | INFO | RTMetrics done | 20260507 03:37:43.741 | INFO | Finished a round of Baz IO: Wrote 1.749 GiB in 3.193 seconds | 20260507 03:37:43.741 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.87% / 0.00% / 67.13% | 20260507 03:37:43.741 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.194446% / 3.97364e-06% / 8.34465e-05% | 20260507 03:37:43.741 | INFO | Baz IO overall: | 20260507 03:37:43.741 | INFO | BytesWritten : 3482376490029 | 20260507 03:37:43.741 | INFO | NumEvents : 1673030500108 | 20260507 03:37:45.582 | INFO | Analyzing chunk frames = [6791168,6791680) | 20260507 03:37:45.582 | INFO | MemInfo: { | 20260507 03:37:45.582 | INFO | "gpu" : " 17 GB", | 20260507 03:37:45.582 | INFO | "huge" : "", | 20260507 03:37:45.582 | INFO | "huge_shared" : " 27 GB", | 20260507 03:37:45.582 | INFO | "malloc" : " 14 GB", | 20260507 03:37:45.582 | INFO | "pinned" : " 29 GB" | 20260507 03:37:45.582 | INFO | } | 20260507 03:37:45.582 | INFO | Perf report for the last 5.13104 seconds: | 20260507 03:37:45.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:37:45.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:45.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:45.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:45.582 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260507 03:37:45.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:45.582 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:37:45.582 | INFO | PRE_HQ : 59.2%, 1.8, 55.4ms, 0 | 20260507 03:37:45.582 | INFO | BAZWRITER : 61.8%, 3.3, 103.7ms, 0 | 20260507 03:37:45.582 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:37:45.582 | INFO | Uploaded 36GiB to GPU in 2.04463s (17.6071GiB/s) | 20260507 03:37:45.582 | INFO | Downloaded 38GiB to GPU in 2.17226s (17.4933GiB/s) | 20260507 03:37:50.723 | INFO | Analyzing chunk frames = [6791680,6792192) | 20260507 03:37:50.723 | INFO | MemInfo: { | 20260507 03:37:50.723 | INFO | "gpu" : " 17 GB", | 20260507 03:37:50.723 | INFO | "huge" : "", | 20260507 03:37:50.723 | INFO | "huge_shared" : " 27 GB", | 20260507 03:37:50.723 | INFO | "malloc" : " 14 GB", | 20260507 03:37:50.723 | INFO | "pinned" : " 29 GB" | 20260507 03:37:50.724 | INFO | } | 20260507 03:37:50.724 | INFO | Perf report for the last 5.14172 seconds: | 20260507 03:37:50.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:37:50.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:50.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:50.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:50.724 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:37:50.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:50.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:50.724 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:37:50.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:50.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:50.724 | INFO | Uploaded 36GiB to GPU in 2.03094s (17.7258GiB/s) | 20260507 03:37:50.724 | INFO | Downloaded 34GiB to GPU in 1.93956s (17.5298GiB/s) | 20260507 03:37:55.865 | INFO | Analyzing chunk frames = [6792192,6792704) | 20260507 03:37:55.865 | INFO | MemInfo: { | 20260507 03:37:55.865 | INFO | "gpu" : " 17 GB", | 20260507 03:37:55.865 | INFO | "huge" : "", | 20260507 03:37:55.865 | INFO | "huge_shared" : " 27 GB", | 20260507 03:37:55.865 | INFO | "malloc" : " 14 GB", | 20260507 03:37:55.865 | INFO | "pinned" : " 29 GB" | 20260507 03:37:55.865 | INFO | } | 20260507 03:37:55.865 | INFO | Perf report for the last 5.14179 seconds: | 20260507 03:37:55.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:37:55.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:55.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:55.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:55.865 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 03:37:55.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:55.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:55.865 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:37:55.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:55.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:37:55.866 | INFO | Uploaded 36GiB to GPU in 2.00392s (17.9648GiB/s) | 20260507 03:37:55.866 | INFO | Downloaded 34GiB to GPU in 1.94106s (17.5162GiB/s) | 20260507 03:38:01.018 | INFO | Analyzing chunk frames = [6792704,6793216) | 20260507 03:38:01.018 | INFO | MemInfo: { | 20260507 03:38:01.018 | INFO | "gpu" : " 17 GB", | 20260507 03:38:01.018 | INFO | "huge" : "", | 20260507 03:38:01.018 | INFO | "huge_shared" : " 27 GB", | 20260507 03:38:01.018 | INFO | "malloc" : " 14 GB", | 20260507 03:38:01.018 | INFO | "pinned" : " 29 GB" | 20260507 03:38:01.018 | INFO | } | 20260507 03:38:01.018 | INFO | Perf report for the last 5.15288 seconds: | 20260507 03:38:01.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:38:01.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:01.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:01.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:01.018 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 03:38:01.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:01.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:01.018 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 03:38:01.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:01.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:01.019 | INFO | Uploaded 36GiB to GPU in 1.98348s (18.1499GiB/s) | 20260507 03:38:01.019 | INFO | Downloaded 34GiB to GPU in 1.91096s (17.7921GiB/s) | 20260507 03:38:06.147 | INFO | Analyzing chunk frames = [6793216,6793728) | 20260507 03:38:06.147 | INFO | MemInfo: { | 20260507 03:38:06.147 | INFO | "gpu" : " 17 GB", | 20260507 03:38:06.147 | INFO | "huge" : "", | 20260507 03:38:06.148 | INFO | "huge_shared" : " 27 GB", | 20260507 03:38:06.148 | INFO | "malloc" : " 14 GB", | 20260507 03:38:06.148 | INFO | "pinned" : " 29 GB" | 20260507 03:38:06.148 | INFO | } | 20260507 03:38:06.148 | INFO | Perf report for the last 5.1295 seconds: | 20260507 03:38:06.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:38:06.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:06.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:06.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:06.148 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260507 03:38:06.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:06.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:06.148 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:38:06.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:06.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:06.148 | INFO | Uploaded 36GiB to GPU in 2.02739s (17.7568GiB/s) | 20260507 03:38:06.148 | INFO | Downloaded 34GiB to GPU in 1.91499s (17.7547GiB/s) | 20260507 03:38:11.284 | INFO | Analyzing chunk frames = [6793728,6794240) | 20260507 03:38:11.284 | INFO | MemInfo: { | 20260507 03:38:11.284 | INFO | "gpu" : " 17 GB", | 20260507 03:38:11.284 | INFO | "huge" : "", | 20260507 03:38:11.284 | INFO | "huge_shared" : " 27 GB", | 20260507 03:38:11.284 | INFO | "malloc" : " 14 GB", | 20260507 03:38:11.284 | INFO | "pinned" : " 29 GB" | 20260507 03:38:11.284 | INFO | } | 20260507 03:38:11.284 | INFO | Perf report for the last 5.13627 seconds: | 20260507 03:38:11.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:38:11.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:11.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:11.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:11.284 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 03:38:11.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:11.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:11.284 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 03:38:11.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:11.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:11.284 | INFO | Uploaded 36GiB to GPU in 2.0253s (17.7752GiB/s) | 20260507 03:38:11.284 | INFO | Downloaded 34GiB to GPU in 1.89902s (17.9039GiB/s) | 20260507 03:38:16.421 | INFO | Analyzing chunk frames = [6794240,6794752) | 20260507 03:38:16.421 | INFO | MemInfo: { | 20260507 03:38:16.421 | INFO | "gpu" : " 17 GB", | 20260507 03:38:16.421 | INFO | "huge" : "", | 20260507 03:38:16.421 | INFO | "huge_shared" : " 27 GB", | 20260507 03:38:16.421 | INFO | "malloc" : " 14 GB", | 20260507 03:38:16.421 | INFO | "pinned" : " 29 GB" | 20260507 03:38:16.421 | INFO | } | 20260507 03:38:16.422 | INFO | Perf report for the last 5.13747 seconds: | 20260507 03:38:16.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:38:16.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:16.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:16.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:16.422 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:38:16.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:16.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:16.422 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 03:38:16.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:16.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:16.422 | INFO | Uploaded 36GiB to GPU in 1.96758s (18.2966GiB/s) | 20260507 03:38:16.422 | INFO | Downloaded 34GiB to GPU in 1.95761s (17.3682GiB/s) | 20260507 03:38:21.560 | INFO | Analyzing chunk frames = [6794752,6795264) | 20260507 03:38:21.560 | INFO | MemInfo: { | 20260507 03:38:21.560 | INFO | "gpu" : " 17 GB", | 20260507 03:38:21.560 | INFO | "huge" : "", | 20260507 03:38:21.560 | INFO | "huge_shared" : " 27 GB", | 20260507 03:38:21.560 | INFO | "malloc" : " 14 GB", | 20260507 03:38:21.560 | INFO | "pinned" : " 29 GB" | 20260507 03:38:21.560 | INFO | } | 20260507 03:38:21.560 | INFO | Perf report for the last 5.13873 seconds: | 20260507 03:38:21.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:38:21.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:21.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:21.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:21.560 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:38:21.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:21.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:21.560 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 03:38:21.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:21.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:21.560 | INFO | Uploaded 36GiB to GPU in 1.97068s (18.2678GiB/s) | 20260507 03:38:21.561 | INFO | Downloaded 34GiB to GPU in 1.96473s (17.3052GiB/s) | 20260507 03:38:21.659 | INFO | Starting a round of Baz IO | 20260507 03:38:24.774 | INFO | Populating RTMetrics report | 20260507 03:38:24.775 | INFO | RTMetrics report populated | 20260507 03:38:24.775 | INFO | Opening tmp RTMetrics json file | 20260507 03:38:24.776 | INFO | Serializing RTMetrics to json | 20260507 03:38:24.777 | INFO | Serialization done, swapping tmp file now... | 20260507 03:38:24.777 | INFO | RTMetrics done | 20260507 03:38:24.866 | INFO | Finished a round of Baz IO: Wrote 1.748 GiB in 3.207 seconds | 20260507 03:38:24.866 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.87% / 0.00% / 67.13% | 20260507 03:38:24.866 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.194263% / 3.97364e-06% / 9.53674e-05% | 20260507 03:38:24.866 | INFO | Baz IO overall: | 20260507 03:38:24.866 | INFO | BytesWritten : 3484252967634 | 20260507 03:38:24.866 | INFO | NumEvents : 1673915452993 | 20260507 03:38:26.703 | INFO | Analyzing chunk frames = [6795264,6795776) | 20260507 03:38:26.703 | INFO | MemInfo: { | 20260507 03:38:26.703 | INFO | "gpu" : " 17 GB", | 20260507 03:38:26.703 | INFO | "huge" : "", | 20260507 03:38:26.703 | INFO | "huge_shared" : " 27 GB", | 20260507 03:38:26.704 | INFO | "malloc" : " 14 GB", | 20260507 03:38:26.704 | INFO | "pinned" : " 29 GB" | 20260507 03:38:26.704 | INFO | } | 20260507 03:38:26.704 | INFO | Perf report for the last 5.14333 seconds: | 20260507 03:38:26.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:38:26.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:26.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:26.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:26.704 | INFO | ANALYSIS : 62.0%, 3.0, 96.5ms, 0 | 20260507 03:38:26.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:26.704 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 03:38:26.704 | INFO | PRE_HQ : 59.4%, 1.8, 55.4ms, 0 | 20260507 03:38:26.704 | INFO | BAZWRITER : 62.0%, 3.3, 105.3ms, 0 | 20260507 03:38:26.704 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 03:38:26.704 | INFO | Uploaded 36GiB to GPU in 2.04782s (17.5797GiB/s) | 20260507 03:38:26.704 | INFO | Downloaded 38GiB to GPU in 2.17243s (17.4919GiB/s) | 20260507 03:38:31.843 | INFO | Analyzing chunk frames = [6795776,6796288) | 20260507 03:38:31.843 | INFO | MemInfo: { | 20260507 03:38:31.843 | INFO | "gpu" : " 17 GB", | 20260507 03:38:31.843 | INFO | "huge" : "", | 20260507 03:38:31.843 | INFO | "huge_shared" : " 27 GB", | 20260507 03:38:31.844 | INFO | "malloc" : " 14 GB", | 20260507 03:38:31.844 | INFO | "pinned" : " 29 GB" | 20260507 03:38:31.844 | INFO | } | 20260507 03:38:31.844 | INFO | Perf report for the last 5.13993 seconds: | 20260507 03:38:31.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:38:31.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:31.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:31.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:31.844 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 03:38:31.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:31.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:31.844 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:38:31.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:31.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:31.844 | INFO | Uploaded 36GiB to GPU in 2.05227s (17.5415GiB/s) | 20260507 03:38:31.844 | INFO | Downloaded 34GiB to GPU in 1.94272s (17.5013GiB/s) | 20260507 03:38:36.983 | INFO | Analyzing chunk frames = [6796288,6796800) | 20260507 03:38:36.983 | INFO | MemInfo: { | 20260507 03:38:36.983 | INFO | "gpu" : " 17 GB", | 20260507 03:38:36.983 | INFO | "huge" : "", | 20260507 03:38:36.983 | INFO | "huge_shared" : " 27 GB", | 20260507 03:38:36.983 | INFO | "malloc" : " 14 GB", | 20260507 03:38:36.983 | INFO | "pinned" : " 29 GB" | 20260507 03:38:36.983 | INFO | } | 20260507 03:38:36.983 | INFO | Perf report for the last 5.13953 seconds: | 20260507 03:38:36.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:38:36.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:36.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:36.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:36.983 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 03:38:36.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:36.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:36.983 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:38:36.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:36.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:36.983 | INFO | Uploaded 36GiB to GPU in 2.01778s (17.8414GiB/s) | 20260507 03:38:36.983 | INFO | Downloaded 34GiB to GPU in 1.93932s (17.5319GiB/s) | 20260507 03:38:42.127 | INFO | Analyzing chunk frames = [6796800,6797312) | 20260507 03:38:42.127 | INFO | MemInfo: { | 20260507 03:38:42.127 | INFO | "gpu" : " 17 GB", | 20260507 03:38:42.127 | INFO | "huge" : "", | 20260507 03:38:42.127 | INFO | "huge_shared" : " 27 GB", | 20260507 03:38:42.127 | INFO | "malloc" : " 14 GB", | 20260507 03:38:42.127 | INFO | "pinned" : " 29 GB" | 20260507 03:38:42.127 | INFO | } | 20260507 03:38:42.127 | INFO | Perf report for the last 5.14395 seconds: | 20260507 03:38:42.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:38:42.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:42.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:42.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:42.127 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 03:38:42.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:42.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:42.127 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:38:42.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:42.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:42.127 | INFO | Uploaded 36GiB to GPU in 1.98259s (18.1581GiB/s) | 20260507 03:38:42.127 | INFO | Downloaded 34GiB to GPU in 1.91274s (17.7755GiB/s) | 20260507 03:38:47.259 | INFO | Analyzing chunk frames = [6797312,6797824) | 20260507 03:38:47.259 | INFO | MemInfo: { | 20260507 03:38:47.259 | INFO | "gpu" : " 17 GB", | 20260507 03:38:47.259 | INFO | "huge" : "", | 20260507 03:38:47.259 | INFO | "huge_shared" : " 27 GB", | 20260507 03:38:47.259 | INFO | "malloc" : " 14 GB", | 20260507 03:38:47.259 | INFO | "pinned" : " 29 GB" | 20260507 03:38:47.259 | INFO | } | 20260507 03:38:47.259 | INFO | Perf report for the last 5.13204 seconds: | 20260507 03:38:47.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:38:47.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:47.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:47.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:47.259 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 03:38:47.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:47.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:47.259 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 03:38:47.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:47.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:47.259 | INFO | Uploaded 36GiB to GPU in 2.04024s (17.645GiB/s) | 20260507 03:38:47.259 | INFO | Downloaded 34GiB to GPU in 1.94359s (17.4934GiB/s) | 20260507 03:38:52.396 | INFO | Analyzing chunk frames = [6797824,6798336) | 20260507 03:38:52.396 | INFO | MemInfo: { | 20260507 03:38:52.396 | INFO | "gpu" : " 17 GB", | 20260507 03:38:52.396 | INFO | "huge" : "", | 20260507 03:38:52.396 | INFO | "huge_shared" : " 27 GB", | 20260507 03:38:52.396 | INFO | "malloc" : " 14 GB", | 20260507 03:38:52.396 | INFO | "pinned" : " 29 GB" | 20260507 03:38:52.397 | INFO | } | 20260507 03:38:52.397 | INFO | Perf report for the last 5.13758 seconds: | 20260507 03:38:52.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:38:52.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:52.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:52.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:52.397 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 03:38:52.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:52.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:52.397 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 03:38:52.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:52.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:52.397 | INFO | Uploaded 36GiB to GPU in 2.03711s (17.6721GiB/s) | 20260507 03:38:52.397 | INFO | Downloaded 34GiB to GPU in 1.93475s (17.5733GiB/s) | 20260507 03:38:57.533 | INFO | Analyzing chunk frames = [6798336,6798848) | 20260507 03:38:57.533 | INFO | MemInfo: { | 20260507 03:38:57.533 | INFO | "gpu" : " 17 GB", | 20260507 03:38:57.533 | INFO | "huge" : "", | 20260507 03:38:57.533 | INFO | "huge_shared" : " 27 GB", | 20260507 03:38:57.533 | INFO | "malloc" : " 14 GB", | 20260507 03:38:57.533 | INFO | "pinned" : " 29 GB" | 20260507 03:38:57.533 | INFO | } | 20260507 03:38:57.533 | INFO | Perf report for the last 5.1366 seconds: | 20260507 03:38:57.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:38:57.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:57.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:57.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:57.533 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:38:57.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:57.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:57.533 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 03:38:57.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:57.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:38:57.534 | INFO | Uploaded 36GiB to GPU in 1.9652s (18.3187GiB/s) | 20260507 03:38:57.534 | INFO | Downloaded 34GiB to GPU in 1.97837s (17.1859GiB/s) | 20260507 03:39:02.673 | INFO | Analyzing chunk frames = [6798848,6799360) | 20260507 03:39:02.673 | INFO | MemInfo: { | 20260507 03:39:02.673 | INFO | "gpu" : " 17 GB", | 20260507 03:39:02.674 | INFO | "huge" : "", | 20260507 03:39:02.674 | INFO | "huge_shared" : " 27 GB", | 20260507 03:39:02.674 | INFO | "malloc" : " 14 GB", | 20260507 03:39:02.674 | INFO | "pinned" : " 29 GB" | 20260507 03:39:02.674 | INFO | } | 20260507 03:39:02.674 | INFO | Perf report for the last 5.14036 seconds: | 20260507 03:39:02.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:39:02.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:02.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:02.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:02.674 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:39:02.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:02.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:02.674 | INFO | PRE_HQ : 29.6%, 1.0, 15.4ms, 0 | 20260507 03:39:02.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:02.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:02.674 | INFO | Uploaded 36GiB to GPU in 1.98095s (18.1731GiB/s) | 20260507 03:39:02.674 | INFO | Downloaded 34GiB to GPU in 1.94751s (17.4582GiB/s) | 20260507 03:39:02.674 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:39:02.674 | INFO | Total mem usage: 18 GiB | 20260507 03:39:02.674 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:39:02.674 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 03:39:02.674 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:39:02.674 | INFO | High Water Breakdown: | 20260507 03:39:02.674 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:39:02.674 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:39:02.674 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:39:02.674 | INFO | ------------------------------------------ | 20260507 03:39:02.674 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:39:02.674 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:39:02.674 | INFO | Total mem usage: 26.5 GiB | 20260507 03:39:02.674 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:39:02.674 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:39:02.674 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:39:02.674 | INFO | High Water Breakdown: | 20260507 03:39:02.674 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:39:02.674 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:39:02.674 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:39:02.674 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:39:02.674 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:39:02.674 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:39:02.674 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:39:02.674 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:39:02.674 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:39:02.674 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:39:02.674 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:39:02.674 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:39:02.674 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:39:02.674 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:39:02.674 | INFO | ------------------------------------------ | 20260507 03:39:02.674 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:39:02.674 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:39:02.674 | INFO | Total mem usage: 25.6 GiB | 20260507 03:39:02.674 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:39:02.674 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:39:02.674 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:39:02.674 | INFO | High Water Breakdown: | 20260507 03:39:02.674 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:39:02.674 | INFO | ------------------------------------------ | 20260507 03:39:02.674 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:39:02.674 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:39:02.674 | INFO | Total mem usage: 15.7 GiB | 20260507 03:39:02.675 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:39:02.675 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:39:02.675 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:39:02.675 | INFO | High Water Breakdown: | 20260507 03:39:02.675 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:39:02.675 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:39:02.675 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:39:02.675 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:39:02.675 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:39:02.675 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:39:02.675 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:39:02.675 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:39:02.675 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:39:02.675 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:39:02.675 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:39:02.675 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:39:02.675 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:39:02.675 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:39:02.675 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:39:02.675 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:39:02.675 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:39:02.675 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:39:02.675 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:39:02.675 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:39:02.675 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:39:02.675 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:39:02.675 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:39:02.675 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:39:02.675 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:39:02.675 | INFO | ------------------------------------------ | 20260507 03:39:02.675 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:39:02.675 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:39:02.675 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:39:02.675 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:39:02.675 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.51e+01, 1.32e+01 / 1.15e+00 | 20260507 03:39:02.675 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.04e-01, 9.95e-03 / 1.40e-02 | 20260507 03:39:02.675 | INFO | PerfReport: 11.67%, Baseline -- 3.60e+00 / 4.47e+00, 5.22e-01 / 2.10e-01 | 20260507 03:39:02.675 | INFO | PerfReport: 3.23%, Histogram -- 9.94e-01 / 1.47e+00, 1.69e-01 / 1.25e-01 | 20260507 03:39:02.675 | INFO | PerfReport: 16.08%, DME -- 3.96e+01 / 4.42e+01, 3.64e+00 / 1.47e+00 | 20260507 03:39:02.675 | INFO | PerfReport: 57.73%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.39e+00 / 1.04e+00 | 20260507 03:39:02.675 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.68e+00, 3.73e-01 / 1.93e-01 | 20260507 03:39:02.675 | INFO | PerfReport: 3.04%, Metrics -- 9.36e-01 / 1.25e+00, 2.47e-01 / 1.13e-01 | 20260507 03:39:02.675 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.17e-01 / 3.77e-01, 5.07e-02 / 2.53e-02 | 20260507 03:39:02.675 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:39:02.675 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:39:02.675 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:39:02.675 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 03:39:02.675 | INFO | PerfReport: 21.76%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260507 03:39:02.675 | INFO | PerfReport: 21.28%, Download -- 2.05e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 03:39:02.675 | INFO | PerfReport: 32.00%, Compute -- 3.08e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 03:39:02.675 | INFO | PerfReport: 1.48%, LockUpload -- 1.42e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 03:39:02.675 | INFO | PerfReport: 1.24%, LockDownload -- 1.20e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260507 03:39:02.675 | INFO | PerfReport: 22.23%, LockCompute -- 2.14e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 03:39:02.675 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260507 03:39:02.770 | INFO | Starting a round of Baz IO | 20260507 03:39:05.882 | INFO | Populating RTMetrics report | 20260507 03:39:05.882 | INFO | RTMetrics report populated | 20260507 03:39:05.883 | INFO | Opening tmp RTMetrics json file | 20260507 03:39:05.899 | INFO | Serializing RTMetrics to json | 20260507 03:39:05.900 | INFO | Serialization done, swapping tmp file now... | 20260507 03:39:05.901 | INFO | RTMetrics done | 20260507 03:39:05.975 | INFO | Finished a round of Baz IO: Wrote 1.75 GiB in 3.204 seconds | 20260507 03:39:05.975 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.87% / 0.00% / 67.13% | 20260507 03:39:05.975 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.194979% / 3.97364e-06% / 4.76837e-05% | 20260507 03:39:05.975 | INFO | Baz IO overall: | 20260507 03:39:05.975 | INFO | BytesWritten : 3486132397164 | 20260507 03:39:05.975 | INFO | NumEvents : 1674802530929 | 20260507 03:39:07.820 | INFO | Analyzing chunk frames = [6799360,6799872) | 20260507 03:39:07.820 | INFO | MemInfo: { | 20260507 03:39:07.820 | INFO | "gpu" : " 17 GB", | 20260507 03:39:07.820 | INFO | "huge" : "", | 20260507 03:39:07.820 | INFO | "huge_shared" : " 27 GB", | 20260507 03:39:07.820 | INFO | "malloc" : " 14 GB", | 20260507 03:39:07.820 | INFO | "pinned" : " 29 GB" | 20260507 03:39:07.820 | INFO | } | 20260507 03:39:07.820 | INFO | Perf report for the last 5.14599 seconds: | 20260507 03:39:07.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:39:07.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:07.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:07.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:07.820 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260507 03:39:07.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:07.820 | INFO | RT_METRICS : 42.0%, 1.0, 22.1ms, 0 | 20260507 03:39:07.820 | INFO | PRE_HQ : 59.2%, 1.8, 55.7ms, 0 | 20260507 03:39:07.820 | INFO | BAZWRITER : 61.9%, 3.3, 105.0ms, 0 | 20260507 03:39:07.820 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 03:39:07.820 | INFO | Uploaded 36GiB to GPU in 2.0228s (17.7971GiB/s) | 20260507 03:39:07.820 | INFO | Downloaded 38GiB to GPU in 2.17632s (17.4606GiB/s) | 20260507 03:39:12.965 | INFO | Analyzing chunk frames = [6799872,6800384) | 20260507 03:39:12.965 | INFO | MemInfo: { | 20260507 03:39:12.965 | INFO | "gpu" : " 17 GB", | 20260507 03:39:12.965 | INFO | "huge" : "", | 20260507 03:39:12.965 | INFO | "huge_shared" : " 27 GB", | 20260507 03:39:12.965 | INFO | "malloc" : " 14 GB", | 20260507 03:39:12.965 | INFO | "pinned" : " 29 GB" | 20260507 03:39:12.965 | INFO | } | 20260507 03:39:12.965 | INFO | Perf report for the last 5.14559 seconds: | 20260507 03:39:12.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:39:12.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:12.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:12.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:12.965 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 03:39:12.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:12.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:12.965 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:39:12.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:12.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:12.966 | INFO | Uploaded 36GiB to GPU in 2.04879s (17.5713GiB/s) | 20260507 03:39:12.966 | INFO | Downloaded 34GiB to GPU in 1.93788s (17.545GiB/s) | 20260507 03:39:18.100 | INFO | Analyzing chunk frames = [6800384,6800896) | 20260507 03:39:18.100 | INFO | MemInfo: { | 20260507 03:39:18.100 | INFO | "gpu" : " 17 GB", | 20260507 03:39:18.100 | INFO | "huge" : "", | 20260507 03:39:18.100 | INFO | "huge_shared" : " 27 GB", | 20260507 03:39:18.100 | INFO | "malloc" : " 14 GB", | 20260507 03:39:18.100 | INFO | "pinned" : " 29 GB" | 20260507 03:39:18.100 | INFO | } | 20260507 03:39:18.100 | INFO | Perf report for the last 5.1344 seconds: | 20260507 03:39:18.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:39:18.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:18.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:18.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:18.100 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 03:39:18.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:18.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:18.100 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:39:18.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:18.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:18.100 | INFO | Uploaded 36GiB to GPU in 2.05311s (17.5344GiB/s) | 20260507 03:39:18.100 | INFO | Downloaded 34GiB to GPU in 1.9508s (17.4288GiB/s) | 20260507 03:39:23.231 | INFO | Analyzing chunk frames = [6800896,6801408) | 20260507 03:39:23.231 | INFO | MemInfo: { | 20260507 03:39:23.231 | INFO | "gpu" : " 17 GB", | 20260507 03:39:23.231 | INFO | "huge" : "", | 20260507 03:39:23.232 | INFO | "huge_shared" : " 27 GB", | 20260507 03:39:23.232 | INFO | "malloc" : " 14 GB", | 20260507 03:39:23.232 | INFO | "pinned" : " 29 GB" | 20260507 03:39:23.232 | INFO | } | 20260507 03:39:23.232 | INFO | Perf report for the last 5.13191 seconds: | 20260507 03:39:23.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:39:23.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:23.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:23.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:23.232 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260507 03:39:23.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:23.232 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:23.232 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:39:23.232 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:23.232 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:23.232 | INFO | Uploaded 36GiB to GPU in 2.02046s (17.8177GiB/s) | 20260507 03:39:23.232 | INFO | Downloaded 34GiB to GPU in 1.93399s (17.5803GiB/s) | 20260507 03:39:28.375 | INFO | Analyzing chunk frames = [6801408,6801920) | 20260507 03:39:28.375 | INFO | MemInfo: { | 20260507 03:39:28.375 | INFO | "gpu" : " 17 GB", | 20260507 03:39:28.375 | INFO | "huge" : "", | 20260507 03:39:28.375 | INFO | "huge_shared" : " 27 GB", | 20260507 03:39:28.375 | INFO | "malloc" : " 14 GB", | 20260507 03:39:28.375 | INFO | "pinned" : " 29 GB" | 20260507 03:39:28.375 | INFO | } | 20260507 03:39:28.375 | INFO | Perf report for the last 5.14342 seconds: | 20260507 03:39:28.375 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:39:28.375 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:28.375 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:28.375 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:28.375 | INFO | ANALYSIS : 59.9%, 3.0, 93.4ms, 0 | 20260507 03:39:28.375 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:28.375 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:28.375 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:39:28.375 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:28.375 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:28.375 | INFO | Uploaded 36GiB to GPU in 2.00422s (17.9621GiB/s) | 20260507 03:39:28.375 | INFO | Downloaded 34GiB to GPU in 1.93894s (17.5353GiB/s) | 20260507 03:39:33.519 | INFO | Analyzing chunk frames = [6801920,6802432) | 20260507 03:39:33.519 | INFO | MemInfo: { | 20260507 03:39:33.519 | INFO | "gpu" : " 17 GB", | 20260507 03:39:33.519 | INFO | "huge" : "", | 20260507 03:39:33.519 | INFO | "huge_shared" : " 27 GB", | 20260507 03:39:33.519 | INFO | "malloc" : " 14 GB", | 20260507 03:39:33.519 | INFO | "pinned" : " 29 GB" | 20260507 03:39:33.519 | INFO | } | 20260507 03:39:33.519 | INFO | Perf report for the last 5.14368 seconds: | 20260507 03:39:33.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:39:33.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:33.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:33.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:33.519 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:39:33.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:33.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:33.519 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:39:33.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:33.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:33.519 | INFO | Uploaded 36GiB to GPU in 2.05873s (17.4865GiB/s) | 20260507 03:39:33.519 | INFO | Downloaded 34GiB to GPU in 1.95641s (17.3788GiB/s) | 20260507 03:39:38.656 | INFO | Analyzing chunk frames = [6802432,6802944) | 20260507 03:39:38.656 | INFO | MemInfo: { | 20260507 03:39:38.656 | INFO | "gpu" : " 17 GB", | 20260507 03:39:38.656 | INFO | "huge" : "", | 20260507 03:39:38.656 | INFO | "huge_shared" : " 27 GB", | 20260507 03:39:38.656 | INFO | "malloc" : " 14 GB", | 20260507 03:39:38.656 | INFO | "pinned" : " 29 GB" | 20260507 03:39:38.656 | INFO | } | 20260507 03:39:38.656 | INFO | Perf report for the last 5.13721 seconds: | 20260507 03:39:38.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:39:38.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:38.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:38.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:38.656 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 03:39:38.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:38.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:38.656 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:39:38.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:38.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:38.656 | INFO | Uploaded 36GiB to GPU in 1.9881s (18.1077GiB/s) | 20260507 03:39:38.656 | INFO | Downloaded 34GiB to GPU in 1.98327s (17.1434GiB/s) | 20260507 03:39:43.793 | INFO | Analyzing chunk frames = [6802944,6803456) | 20260507 03:39:43.793 | INFO | MemInfo: { | 20260507 03:39:43.793 | INFO | "gpu" : " 17 GB", | 20260507 03:39:43.793 | INFO | "huge" : "", | 20260507 03:39:43.793 | INFO | "huge_shared" : " 27 GB", | 20260507 03:39:43.793 | INFO | "malloc" : " 14 GB", | 20260507 03:39:43.793 | INFO | "pinned" : " 29 GB" | 20260507 03:39:43.793 | INFO | } | 20260507 03:39:43.793 | INFO | Perf report for the last 5.13702 seconds: | 20260507 03:39:43.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:39:43.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:43.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:43.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:43.793 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:39:43.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:43.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:43.793 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:39:43.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:43.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:43.793 | INFO | Uploaded 36GiB to GPU in 1.98033s (18.1788GiB/s) | 20260507 03:39:43.793 | INFO | Downloaded 34GiB to GPU in 1.96199s (17.3294GiB/s) | 20260507 03:39:43.896 | INFO | Starting a round of Baz IO | 20260507 03:39:46.999 | INFO | Populating RTMetrics report | 20260507 03:39:46.999 | INFO | RTMetrics report populated | 20260507 03:39:46.999 | INFO | Opening tmp RTMetrics json file | 20260507 03:39:47.005 | INFO | Serializing RTMetrics to json | 20260507 03:39:47.017 | INFO | Serialization done, swapping tmp file now... | 20260507 03:39:47.018 | INFO | RTMetrics done | 20260507 03:39:47.089 | INFO | Finished a round of Baz IO: Wrote 1.748 GiB in 3.193 seconds | 20260507 03:39:47.089 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.87% / 0.00% / 67.13% | 20260507 03:39:47.089 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.194228% / 3.97364e-06% / 8.34465e-05% | 20260507 03:39:47.089 | INFO | Baz IO overall: | 20260507 03:39:47.089 | INFO | BytesWritten : 3488009205927 | 20260507 03:39:47.089 | INFO | NumEvents : 1675687531449 | 20260507 03:39:48.931 | INFO | Analyzing chunk frames = [6803456,6803968) | 20260507 03:39:48.931 | INFO | MemInfo: { | 20260507 03:39:48.931 | INFO | "gpu" : " 17 GB", | 20260507 03:39:48.931 | INFO | "huge" : "", | 20260507 03:39:48.931 | INFO | "huge_shared" : " 27 GB", | 20260507 03:39:48.931 | INFO | "malloc" : " 14 GB", | 20260507 03:39:48.931 | INFO | "pinned" : " 29 GB" | 20260507 03:39:48.931 | INFO | } | 20260507 03:39:48.931 | INFO | Perf report for the last 5.13768 seconds: | 20260507 03:39:48.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:39:48.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:48.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:48.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:48.931 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 03:39:48.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:48.931 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:39:48.931 | INFO | PRE_HQ : 59.3%, 1.8, 55.4ms, 0 | 20260507 03:39:48.931 | INFO | BAZWRITER : 61.8%, 3.3, 103.6ms, 0 | 20260507 03:39:48.931 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 03:39:48.931 | INFO | Uploaded 36GiB to GPU in 2.04991s (17.5617GiB/s) | 20260507 03:39:48.931 | INFO | Downloaded 38GiB to GPU in 2.18971s (17.3539GiB/s) | 20260507 03:39:54.076 | INFO | Analyzing chunk frames = [6803968,6804480) | 20260507 03:39:54.076 | INFO | MemInfo: { | 20260507 03:39:54.076 | INFO | "gpu" : " 17 GB", | 20260507 03:39:54.076 | INFO | "huge" : "", | 20260507 03:39:54.076 | INFO | "huge_shared" : " 27 GB", | 20260507 03:39:54.076 | INFO | "malloc" : " 14 GB", | 20260507 03:39:54.076 | INFO | "pinned" : " 29 GB" | 20260507 03:39:54.076 | INFO | } | 20260507 03:39:54.076 | INFO | Perf report for the last 5.14537 seconds: | 20260507 03:39:54.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:39:54.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:54.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:54.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:54.076 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 03:39:54.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:54.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:54.076 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:39:54.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:54.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:54.076 | INFO | Uploaded 36GiB to GPU in 2.03894s (17.6563GiB/s) | 20260507 03:39:54.076 | INFO | Downloaded 34GiB to GPU in 1.93641s (17.5582GiB/s) | 20260507 03:39:59.212 | INFO | Analyzing chunk frames = [6804480,6804992) | 20260507 03:39:59.212 | INFO | MemInfo: { | 20260507 03:39:59.212 | INFO | "gpu" : " 17 GB", | 20260507 03:39:59.212 | INFO | "huge" : "", | 20260507 03:39:59.212 | INFO | "huge_shared" : " 27 GB", | 20260507 03:39:59.212 | INFO | "malloc" : " 14 GB", | 20260507 03:39:59.212 | INFO | "pinned" : " 29 GB" | 20260507 03:39:59.212 | INFO | } | 20260507 03:39:59.212 | INFO | Perf report for the last 5.13558 seconds: | 20260507 03:39:59.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:39:59.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:59.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:59.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:59.212 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 03:39:59.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:59.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:59.212 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:39:59.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:59.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:39:59.212 | INFO | Uploaded 36GiB to GPU in 2.01907s (17.83GiB/s) | 20260507 03:39:59.212 | INFO | Downloaded 34GiB to GPU in 1.93864s (17.538GiB/s) | 20260507 03:40:04.347 | INFO | Analyzing chunk frames = [6804992,6805504) | 20260507 03:40:04.347 | INFO | MemInfo: { | 20260507 03:40:04.347 | INFO | "gpu" : " 17 GB", | 20260507 03:40:04.347 | INFO | "huge" : "", | 20260507 03:40:04.347 | INFO | "huge_shared" : " 27 GB", | 20260507 03:40:04.347 | INFO | "malloc" : " 14 GB", | 20260507 03:40:04.347 | INFO | "pinned" : " 29 GB" | 20260507 03:40:04.347 | INFO | } | 20260507 03:40:04.347 | INFO | Perf report for the last 5.13556 seconds: | 20260507 03:40:04.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:40:04.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:04.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:04.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:04.347 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 03:40:04.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:04.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:04.347 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:40:04.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:04.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:04.348 | INFO | Uploaded 36GiB to GPU in 1.98645s (18.1227GiB/s) | 20260507 03:40:04.348 | INFO | Downloaded 34GiB to GPU in 1.91243s (17.7784GiB/s) | 20260507 03:40:09.484 | INFO | Analyzing chunk frames = [6805504,6806016) | 20260507 03:40:09.484 | INFO | MemInfo: { | 20260507 03:40:09.484 | INFO | "gpu" : " 17 GB", | 20260507 03:40:09.484 | INFO | "huge" : "", | 20260507 03:40:09.485 | INFO | "huge_shared" : " 27 GB", | 20260507 03:40:09.485 | INFO | "malloc" : " 14 GB", | 20260507 03:40:09.485 | INFO | "pinned" : " 29 GB" | 20260507 03:40:09.485 | INFO | } | 20260507 03:40:09.485 | INFO | Perf report for the last 5.13739 seconds: | 20260507 03:40:09.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:40:09.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:09.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:09.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:09.485 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 03:40:09.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:09.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:09.485 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:40:09.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:09.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:09.485 | INFO | Uploaded 36GiB to GPU in 2.01374s (17.8772GiB/s) | 20260507 03:40:09.485 | INFO | Downloaded 34GiB to GPU in 1.941s (17.5167GiB/s) | 20260507 03:40:14.633 | INFO | Analyzing chunk frames = [6806016,6806528) | 20260507 03:40:14.633 | INFO | MemInfo: { | 20260507 03:40:14.633 | INFO | "gpu" : " 17 GB", | 20260507 03:40:14.633 | INFO | "huge" : "", | 20260507 03:40:14.633 | INFO | "huge_shared" : " 27 GB", | 20260507 03:40:14.633 | INFO | "malloc" : " 14 GB", | 20260507 03:40:14.633 | INFO | "pinned" : " 29 GB" | 20260507 03:40:14.633 | INFO | } | 20260507 03:40:14.633 | INFO | Perf report for the last 5.14882 seconds: | 20260507 03:40:14.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:40:14.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:14.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:14.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:14.633 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 03:40:14.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:14.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:14.634 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 03:40:14.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:14.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:14.634 | INFO | Uploaded 36GiB to GPU in 2.04191s (17.6305GiB/s) | 20260507 03:40:14.634 | INFO | Downloaded 34GiB to GPU in 1.94618s (17.4701GiB/s) | 20260507 03:40:19.768 | INFO | Analyzing chunk frames = [6806528,6807040) | 20260507 03:40:19.768 | INFO | MemInfo: { | 20260507 03:40:19.768 | INFO | "gpu" : " 17 GB", | 20260507 03:40:19.768 | INFO | "huge" : "", | 20260507 03:40:19.768 | INFO | "huge_shared" : " 27 GB", | 20260507 03:40:19.769 | INFO | "malloc" : " 14 GB", | 20260507 03:40:19.769 | INFO | "pinned" : " 29 GB" | 20260507 03:40:19.769 | INFO | } | 20260507 03:40:19.769 | INFO | Perf report for the last 5.13515 seconds: | 20260507 03:40:19.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:40:19.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:19.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:19.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:19.769 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 03:40:19.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:19.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:19.769 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 03:40:19.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:19.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:19.769 | INFO | Uploaded 36GiB to GPU in 1.97749s (18.2049GiB/s) | 20260507 03:40:19.769 | INFO | Downloaded 34GiB to GPU in 1.96822s (17.2745GiB/s) | 20260507 03:40:24.908 | INFO | Analyzing chunk frames = [6807040,6807552) | 20260507 03:40:24.908 | INFO | MemInfo: { | 20260507 03:40:24.908 | INFO | "gpu" : " 17 GB", | 20260507 03:40:24.908 | INFO | "huge" : "", | 20260507 03:40:24.908 | INFO | "huge_shared" : " 27 GB", | 20260507 03:40:24.908 | INFO | "malloc" : " 14 GB", | 20260507 03:40:24.908 | INFO | "pinned" : " 29 GB" | 20260507 03:40:24.908 | INFO | } | 20260507 03:40:24.908 | INFO | Perf report for the last 5.13958 seconds: | 20260507 03:40:24.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:40:24.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:24.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:24.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:24.908 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:40:24.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:24.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:24.908 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:40:24.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:24.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:24.908 | INFO | Uploaded 36GiB to GPU in 1.98189s (18.1645GiB/s) | 20260507 03:40:24.909 | INFO | Downloaded 34GiB to GPU in 1.96371s (17.3141GiB/s) | 20260507 03:40:24.909 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:40:24.909 | INFO | Total mem usage: 18 GiB | 20260507 03:40:24.909 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:40:24.909 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:40:24.909 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:40:24.909 | INFO | High Water Breakdown: | 20260507 03:40:24.909 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:40:24.909 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:40:24.909 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:40:24.909 | INFO | ------------------------------------------ | 20260507 03:40:24.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:40:24.909 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:40:24.909 | INFO | Total mem usage: 26.5 GiB | 20260507 03:40:24.909 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:40:24.909 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:40:24.909 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:40:24.909 | INFO | High Water Breakdown: | 20260507 03:40:24.909 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:40:24.909 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:40:24.909 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:40:24.909 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:40:24.909 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:40:24.909 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:40:24.909 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:40:24.909 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:40:24.909 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:40:24.909 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:40:24.909 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:40:24.909 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:40:24.909 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:40:24.909 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:40:24.909 | INFO | ------------------------------------------ | 20260507 03:40:24.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:40:24.909 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:40:24.909 | INFO | Total mem usage: 25.6 GiB | 20260507 03:40:24.909 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:40:24.909 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:40:24.909 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:40:24.909 | INFO | High Water Breakdown: | 20260507 03:40:24.909 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:40:24.909 | INFO | ------------------------------------------ | 20260507 03:40:24.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:40:24.909 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:40:24.909 | INFO | Total mem usage: 15.7 GiB | 20260507 03:40:24.909 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:40:24.909 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:40:24.909 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 03:40:24.909 | INFO | High Water Breakdown: | 20260507 03:40:24.909 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:40:24.909 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:40:24.909 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:40:24.909 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:40:24.909 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:40:24.909 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:40:24.909 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:40:24.909 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:40:24.909 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:40:24.909 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:40:24.909 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:40:24.909 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:40:24.909 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:40:24.909 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:40:24.909 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:40:24.909 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:40:24.909 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:40:24.909 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:40:24.909 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:40:24.909 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:40:24.909 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:40:24.909 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:40:24.909 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:40:24.909 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:40:24.909 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:40:24.909 | INFO | ------------------------------------------ | 20260507 03:40:24.909 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:40:24.909 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:40:24.909 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:40:24.909 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:40:24.909 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 1.32e+00 | 20260507 03:40:24.909 | INFO | PerfReport: 0.29%, AggregateStats -- 9.00e-02 / 1.00e-01, 9.17e-03 / 5.92e-03 | 20260507 03:40:24.909 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.46e+00, 5.03e-01 / 2.61e-01 | 20260507 03:40:24.909 | INFO | PerfReport: 3.21%, Histogram -- 9.85e-01 / 1.45e+00, 1.61e-01 / 1.13e-01 | 20260507 03:40:24.909 | INFO | PerfReport: 16.14%, DME -- 3.96e+01 / 4.43e+01, 3.83e+00 / 1.69e+00 | 20260507 03:40:24.909 | INFO | PerfReport: 57.64%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.25e+00 / 1.78e+00 | 20260507 03:40:24.909 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.15e+00 / 2.74e+00, 3.87e-01 / 2.05e-01 | 20260507 03:40:24.909 | INFO | PerfReport: 3.04%, Metrics -- 9.35e-01 / 1.27e+00, 2.46e-01 / 1.24e-01 | 20260507 03:40:24.909 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.16e-01 / 3.74e-01, 4.91e-02 / 2.41e-02 | 20260507 03:40:24.909 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:40:24.909 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:40:24.909 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:40:24.910 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 03:40:24.910 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260507 03:40:24.910 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260507 03:40:24.910 | INFO | PerfReport: 31.88%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 03:40:24.910 | INFO | PerfReport: 1.58%, LockUpload -- 1.52e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 03:40:24.910 | INFO | PerfReport: 1.23%, LockDownload -- 1.19e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260507 03:40:24.910 | INFO | PerfReport: 22.08%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 03:40:24.910 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260507 03:40:25.005 | INFO | Starting a round of Baz IO | 20260507 03:40:28.103 | INFO | Populating RTMetrics report | 20260507 03:40:28.103 | INFO | RTMetrics report populated | 20260507 03:40:28.103 | INFO | Opening tmp RTMetrics json file | 20260507 03:40:28.122 | INFO | Serializing RTMetrics to json | 20260507 03:40:28.123 | INFO | Serialization done, swapping tmp file now... | 20260507 03:40:28.124 | INFO | RTMetrics done | 20260507 03:40:28.191 | INFO | Finished a round of Baz IO: Wrote 1.746 GiB in 3.186 seconds | 20260507 03:40:28.191 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.87% / 0.00% / 67.13% | 20260507 03:40:28.191 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.193183% / 3.97364e-06% / 8.34465e-05% | 20260507 03:40:28.192 | INFO | Baz IO overall: | 20260507 03:40:28.192 | INFO | BytesWritten : 3489884215354 | 20260507 03:40:28.192 | INFO | NumEvents : 1676571403275 | 20260507 03:40:30.046 | INFO | Analyzing chunk frames = [6807552,6808064) | 20260507 03:40:30.046 | INFO | MemInfo: { | 20260507 03:40:30.046 | INFO | "gpu" : " 17 GB", | 20260507 03:40:30.046 | INFO | "huge" : "", | 20260507 03:40:30.046 | INFO | "huge_shared" : " 27 GB", | 20260507 03:40:30.046 | INFO | "malloc" : " 14 GB", | 20260507 03:40:30.046 | INFO | "pinned" : " 29 GB" | 20260507 03:40:30.046 | INFO | } | 20260507 03:40:30.046 | INFO | Perf report for the last 5.13767 seconds: | 20260507 03:40:30.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:40:30.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:30.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:30.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:30.046 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 03:40:30.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:30.046 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:40:30.046 | INFO | PRE_HQ : 59.4%, 1.8, 55.4ms, 0 | 20260507 03:40:30.046 | INFO | BAZWRITER : 61.6%, 3.3, 104.3ms, 0 | 20260507 03:40:30.046 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:40:30.046 | INFO | Uploaded 36GiB to GPU in 2.02744s (17.7564GiB/s) | 20260507 03:40:30.046 | INFO | Downloaded 38GiB to GPU in 2.17302s (17.4872GiB/s) | 20260507 03:40:35.188 | INFO | Analyzing chunk frames = [6808064,6808576) | 20260507 03:40:35.188 | INFO | MemInfo: { | 20260507 03:40:35.188 | INFO | "gpu" : " 17 GB", | 20260507 03:40:35.188 | INFO | "huge" : "", | 20260507 03:40:35.188 | INFO | "huge_shared" : " 27 GB", | 20260507 03:40:35.188 | INFO | "malloc" : " 14 GB", | 20260507 03:40:35.188 | INFO | "pinned" : " 29 GB" | 20260507 03:40:35.188 | INFO | } | 20260507 03:40:35.188 | INFO | Perf report for the last 5.14177 seconds: | 20260507 03:40:35.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:40:35.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:35.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:35.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:35.188 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:40:35.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:35.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:35.188 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:40:35.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:35.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:35.188 | INFO | Uploaded 36GiB to GPU in 1.99536s (18.0419GiB/s) | 20260507 03:40:35.188 | INFO | Downloaded 34GiB to GPU in 1.939s (17.5348GiB/s) | 20260507 03:40:40.328 | INFO | Analyzing chunk frames = [6808576,6809088) | 20260507 03:40:40.328 | INFO | MemInfo: { | 20260507 03:40:40.328 | INFO | "gpu" : " 17 GB", | 20260507 03:40:40.328 | INFO | "huge" : "", | 20260507 03:40:40.328 | INFO | "huge_shared" : " 27 GB", | 20260507 03:40:40.328 | INFO | "malloc" : " 14 GB", | 20260507 03:40:40.328 | INFO | "pinned" : " 29 GB" | 20260507 03:40:40.328 | INFO | } | 20260507 03:40:40.328 | INFO | Perf report for the last 5.14042 seconds: | 20260507 03:40:40.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:40:40.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:40.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:40.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:40.328 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 03:40:40.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:40.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:40.328 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:40:40.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:40.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:40.328 | INFO | Uploaded 36GiB to GPU in 2.01571s (17.8597GiB/s) | 20260507 03:40:40.328 | INFO | Downloaded 34GiB to GPU in 1.9401s (17.5249GiB/s) | 20260507 03:40:45.471 | INFO | Analyzing chunk frames = [6809088,6809600) | 20260507 03:40:45.471 | INFO | MemInfo: { | 20260507 03:40:45.471 | INFO | "gpu" : " 17 GB", | 20260507 03:40:45.471 | INFO | "huge" : "", | 20260507 03:40:45.471 | INFO | "huge_shared" : " 27 GB", | 20260507 03:40:45.471 | INFO | "malloc" : " 14 GB", | 20260507 03:40:45.471 | INFO | "pinned" : " 29 GB" | 20260507 03:40:45.471 | INFO | } | 20260507 03:40:45.471 | INFO | Perf report for the last 5.14317 seconds: | 20260507 03:40:45.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:40:45.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:45.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:45.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:45.471 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:40:45.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:45.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:45.471 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 03:40:45.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:45.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:45.472 | INFO | Uploaded 36GiB to GPU in 2.01048s (17.9062GiB/s) | 20260507 03:40:45.472 | INFO | Downloaded 34GiB to GPU in 1.92793s (17.6355GiB/s) | 20260507 03:40:50.609 | INFO | Analyzing chunk frames = [6809600,6810112) | 20260507 03:40:50.609 | INFO | MemInfo: { | 20260507 03:40:50.609 | INFO | "gpu" : " 17 GB", | 20260507 03:40:50.609 | INFO | "huge" : "", | 20260507 03:40:50.609 | INFO | "huge_shared" : " 27 GB", | 20260507 03:40:50.609 | INFO | "malloc" : " 14 GB", | 20260507 03:40:50.609 | INFO | "pinned" : " 29 GB" | 20260507 03:40:50.609 | INFO | } | 20260507 03:40:50.609 | INFO | Perf report for the last 5.13745 seconds: | 20260507 03:40:50.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:40:50.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:50.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:50.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:50.609 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 03:40:50.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:50.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:50.609 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:40:50.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:50.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:50.609 | INFO | Uploaded 36GiB to GPU in 2.05722s (17.4994GiB/s) | 20260507 03:40:50.609 | INFO | Downloaded 34GiB to GPU in 1.94673s (17.4652GiB/s) | 20260507 03:40:55.742 | INFO | Analyzing chunk frames = [6810112,6810624) | 20260507 03:40:55.742 | INFO | MemInfo: { | 20260507 03:40:55.742 | INFO | "gpu" : " 17 GB", | 20260507 03:40:55.742 | INFO | "huge" : "", | 20260507 03:40:55.742 | INFO | "huge_shared" : " 27 GB", | 20260507 03:40:55.742 | INFO | "malloc" : " 14 GB", | 20260507 03:40:55.742 | INFO | "pinned" : " 29 GB" | 20260507 03:40:55.742 | INFO | } | 20260507 03:40:55.742 | INFO | Perf report for the last 5.13328 seconds: | 20260507 03:40:55.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:40:55.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:55.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:55.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:55.742 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 03:40:55.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:55.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:55.742 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:40:55.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:55.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:40:55.742 | INFO | Uploaded 36GiB to GPU in 2.03509s (17.6897GiB/s) | 20260507 03:40:55.742 | INFO | Downloaded 34GiB to GPU in 1.92051s (17.7036GiB/s) | 20260507 03:41:00.883 | INFO | Analyzing chunk frames = [6810624,6811136) | 20260507 03:41:00.883 | INFO | MemInfo: { | 20260507 03:41:00.883 | INFO | "gpu" : " 17 GB", | 20260507 03:41:00.883 | INFO | "huge" : "", | 20260507 03:41:00.883 | INFO | "huge_shared" : " 27 GB", | 20260507 03:41:00.883 | INFO | "malloc" : " 14 GB", | 20260507 03:41:00.883 | INFO | "pinned" : " 29 GB" | 20260507 03:41:00.883 | INFO | } | 20260507 03:41:00.883 | INFO | Perf report for the last 5.14095 seconds: | 20260507 03:41:00.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:41:00.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:00.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:00.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:00.883 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:41:00.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:00.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:00.883 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:41:00.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:00.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:00.883 | INFO | Uploaded 36GiB to GPU in 1.97695s (18.2099GiB/s) | 20260507 03:41:00.883 | INFO | Downloaded 34GiB to GPU in 1.9611s (17.3372GiB/s) | 20260507 03:41:06.023 | INFO | Analyzing chunk frames = [6811136,6811648) | 20260507 03:41:06.023 | INFO | MemInfo: { | 20260507 03:41:06.023 | INFO | "gpu" : " 17 GB", | 20260507 03:41:06.023 | INFO | "huge" : "", | 20260507 03:41:06.023 | INFO | "huge_shared" : " 27 GB", | 20260507 03:41:06.024 | INFO | "malloc" : " 14 GB", | 20260507 03:41:06.024 | INFO | "pinned" : " 29 GB" | 20260507 03:41:06.024 | INFO | } | 20260507 03:41:06.024 | INFO | Perf report for the last 5.14056 seconds: | 20260507 03:41:06.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:41:06.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:06.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:06.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:06.024 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 03:41:06.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:06.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:06.024 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 03:41:06.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:06.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:06.024 | INFO | Uploaded 36GiB to GPU in 1.96869s (18.2863GiB/s) | 20260507 03:41:06.024 | INFO | Downloaded 34GiB to GPU in 1.95735s (17.3704GiB/s) | 20260507 03:41:06.121 | INFO | Starting a round of Baz IO | 20260507 03:41:09.225 | INFO | Populating RTMetrics report | 20260507 03:41:09.225 | INFO | RTMetrics report populated | 20260507 03:41:09.225 | INFO | Opening tmp RTMetrics json file | 20260507 03:41:09.230 | INFO | Serializing RTMetrics to json | 20260507 03:41:09.244 | INFO | Serialization done, swapping tmp file now... | 20260507 03:41:09.245 | INFO | RTMetrics done | 20260507 03:41:09.317 | INFO | Finished a round of Baz IO: Wrote 1.747 GiB in 3.195 seconds | 20260507 03:41:09.317 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.87% / 0.00% / 67.13% | 20260507 03:41:09.317 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.19298% / 7.94729e-06% / 4.37101e-05% | 20260507 03:41:09.317 | INFO | Baz IO overall: | 20260507 03:41:09.317 | INFO | BytesWritten : 3491759842313 | 20260507 03:41:09.317 | INFO | NumEvents : 1677455506813 | 20260507 03:41:11.171 | INFO | Analyzing chunk frames = [6811648,6812160) | 20260507 03:41:11.171 | INFO | MemInfo: { | 20260507 03:41:11.171 | INFO | "gpu" : " 17 GB", | 20260507 03:41:11.171 | INFO | "huge" : "", | 20260507 03:41:11.171 | INFO | "huge_shared" : " 27 GB", | 20260507 03:41:11.171 | INFO | "malloc" : " 14 GB", | 20260507 03:41:11.171 | INFO | "pinned" : " 29 GB" | 20260507 03:41:11.171 | INFO | } | 20260507 03:41:11.171 | INFO | Perf report for the last 5.14714 seconds: | 20260507 03:41:11.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:41:11.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:11.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:11.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:11.171 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 03:41:11.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:11.171 | INFO | RT_METRICS : 41.6%, 1.0, 21.9ms, 0 | 20260507 03:41:11.171 | INFO | PRE_HQ : 59.0%, 1.8, 55.3ms, 0 | 20260507 03:41:11.171 | INFO | BAZWRITER : 61.7%, 3.3, 105.2ms, 0 | 20260507 03:41:11.171 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:41:11.171 | INFO | Uploaded 36GiB to GPU in 2.03675s (17.6753GiB/s) | 20260507 03:41:11.171 | INFO | Downloaded 38GiB to GPU in 2.18201s (17.4152GiB/s) | 20260507 03:41:16.300 | INFO | Analyzing chunk frames = [6812160,6812672) | 20260507 03:41:16.300 | INFO | MemInfo: { | 20260507 03:41:16.300 | INFO | "gpu" : " 17 GB", | 20260507 03:41:16.300 | INFO | "huge" : "", | 20260507 03:41:16.300 | INFO | "huge_shared" : " 27 GB", | 20260507 03:41:16.300 | INFO | "malloc" : " 14 GB", | 20260507 03:41:16.300 | INFO | "pinned" : " 29 GB" | 20260507 03:41:16.300 | INFO | } | 20260507 03:41:16.300 | INFO | Perf report for the last 5.1297 seconds: | 20260507 03:41:16.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:41:16.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:16.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:16.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:16.300 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 03:41:16.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:16.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:16.301 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:41:16.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:16.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:16.301 | INFO | Uploaded 36GiB to GPU in 1.97263s (18.2497GiB/s) | 20260507 03:41:16.301 | INFO | Downloaded 34GiB to GPU in 1.93134s (17.6044GiB/s) | 20260507 03:41:21.441 | INFO | Analyzing chunk frames = [6812672,6813184) | 20260507 03:41:21.441 | INFO | MemInfo: { | 20260507 03:41:21.441 | INFO | "gpu" : " 17 GB", | 20260507 03:41:21.441 | INFO | "huge" : "", | 20260507 03:41:21.441 | INFO | "huge_shared" : " 27 GB", | 20260507 03:41:21.441 | INFO | "malloc" : " 14 GB", | 20260507 03:41:21.441 | INFO | "pinned" : " 29 GB" | 20260507 03:41:21.441 | INFO | } | 20260507 03:41:21.441 | INFO | Perf report for the last 5.14072 seconds: | 20260507 03:41:21.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:41:21.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:21.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:21.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:21.441 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 03:41:21.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:21.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:21.441 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 03:41:21.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:21.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:21.442 | INFO | Uploaded 36GiB to GPU in 1.98662s (18.1213GiB/s) | 20260507 03:41:21.442 | INFO | Downloaded 34GiB to GPU in 1.94441s (17.486GiB/s) | 20260507 03:41:26.582 | INFO | Analyzing chunk frames = [6813184,6813696) | 20260507 03:41:26.582 | INFO | MemInfo: { | 20260507 03:41:26.582 | INFO | "gpu" : " 17 GB", | 20260507 03:41:26.582 | INFO | "huge" : "", | 20260507 03:41:26.582 | INFO | "huge_shared" : " 27 GB", | 20260507 03:41:26.582 | INFO | "malloc" : " 14 GB", | 20260507 03:41:26.582 | INFO | "pinned" : " 29 GB" | 20260507 03:41:26.582 | INFO | } | 20260507 03:41:26.582 | INFO | Perf report for the last 5.14116 seconds: | 20260507 03:41:26.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:41:26.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:26.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:26.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:26.582 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 03:41:26.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:26.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:26.582 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:41:26.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:26.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:26.583 | INFO | Uploaded 36GiB to GPU in 1.95714s (18.3942GiB/s) | 20260507 03:41:26.583 | INFO | Downloaded 34GiB to GPU in 1.91601s (17.7452GiB/s) | 20260507 03:41:31.719 | INFO | Analyzing chunk frames = [6813696,6814208) | 20260507 03:41:31.719 | INFO | MemInfo: { | 20260507 03:41:31.719 | INFO | "gpu" : " 17 GB", | 20260507 03:41:31.720 | INFO | "huge" : "", | 20260507 03:41:31.720 | INFO | "huge_shared" : " 27 GB", | 20260507 03:41:31.720 | INFO | "malloc" : " 14 GB", | 20260507 03:41:31.720 | INFO | "pinned" : " 29 GB" | 20260507 03:41:31.720 | INFO | } | 20260507 03:41:31.720 | INFO | Perf report for the last 5.1374 seconds: | 20260507 03:41:31.720 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:41:31.720 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:31.720 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:31.720 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:31.720 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 03:41:31.720 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:31.720 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:31.720 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:41:31.720 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:31.720 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:31.720 | INFO | Uploaded 36GiB to GPU in 2.02967s (17.7368GiB/s) | 20260507 03:41:31.720 | INFO | Downloaded 34GiB to GPU in 1.94676s (17.4649GiB/s) | 20260507 03:41:36.854 | INFO | Analyzing chunk frames = [6814208,6814720) | 20260507 03:41:36.854 | INFO | MemInfo: { | 20260507 03:41:36.854 | INFO | "gpu" : " 17 GB", | 20260507 03:41:36.854 | INFO | "huge" : "", | 20260507 03:41:36.854 | INFO | "huge_shared" : " 27 GB", | 20260507 03:41:36.854 | INFO | "malloc" : " 14 GB", | 20260507 03:41:36.854 | INFO | "pinned" : " 29 GB" | 20260507 03:41:36.854 | INFO | } | 20260507 03:41:36.854 | INFO | Perf report for the last 5.13437 seconds: | 20260507 03:41:36.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:41:36.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:36.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:36.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:36.854 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 03:41:36.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:36.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:36.854 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:41:36.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:36.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:36.855 | INFO | Uploaded 36GiB to GPU in 2.03473s (17.6928GiB/s) | 20260507 03:41:36.855 | INFO | Downloaded 34GiB to GPU in 1.92595s (17.6537GiB/s) | 20260507 03:41:41.997 | INFO | Analyzing chunk frames = [6814720,6815232) | 20260507 03:41:41.997 | INFO | MemInfo: { | 20260507 03:41:41.997 | INFO | "gpu" : " 17 GB", | 20260507 03:41:41.997 | INFO | "huge" : "", | 20260507 03:41:41.997 | INFO | "huge_shared" : " 27 GB", | 20260507 03:41:41.997 | INFO | "malloc" : " 14 GB", | 20260507 03:41:41.997 | INFO | "pinned" : " 29 GB" | 20260507 03:41:41.997 | INFO | } | 20260507 03:41:41.997 | INFO | Perf report for the last 5.14311 seconds: | 20260507 03:41:41.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:41:41.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:41.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:41.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:41.997 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:41:41.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:41.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:41.997 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:41:41.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:41.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:41.998 | INFO | Uploaded 36GiB to GPU in 1.98519s (18.1343GiB/s) | 20260507 03:41:41.998 | INFO | Downloaded 34GiB to GPU in 1.98673s (17.1135GiB/s) | 20260507 03:41:47.141 | INFO | Analyzing chunk frames = [6815232,6815744) | 20260507 03:41:47.141 | INFO | MemInfo: { | 20260507 03:41:47.141 | INFO | "gpu" : " 17 GB", | 20260507 03:41:47.141 | INFO | "huge" : "", | 20260507 03:41:47.141 | INFO | "huge_shared" : " 27 GB", | 20260507 03:41:47.141 | INFO | "malloc" : " 14 GB", | 20260507 03:41:47.141 | INFO | "pinned" : " 29 GB" | 20260507 03:41:47.141 | INFO | } | 20260507 03:41:47.141 | INFO | Perf report for the last 5.14355 seconds: | 20260507 03:41:47.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:41:47.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:47.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:47.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:47.141 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 03:41:47.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:47.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:47.141 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:41:47.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:47.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:47.141 | INFO | Uploaded 36GiB to GPU in 1.97083s (18.2664GiB/s) | 20260507 03:41:47.141 | INFO | Downloaded 34GiB to GPU in 1.96903s (17.2674GiB/s) | 20260507 03:41:47.141 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:41:47.141 | INFO | Total mem usage: 18 GiB | 20260507 03:41:47.141 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:41:47.141 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 03:41:47.141 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:41:47.141 | INFO | High Water Breakdown: | 20260507 03:41:47.142 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:41:47.142 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:41:47.142 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:41:47.142 | INFO | ------------------------------------------ | 20260507 03:41:47.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:41:47.142 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:41:47.142 | INFO | Total mem usage: 26.5 GiB | 20260507 03:41:47.142 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:41:47.142 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:41:47.142 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:41:47.142 | INFO | High Water Breakdown: | 20260507 03:41:47.142 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:41:47.142 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:41:47.142 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:41:47.142 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:41:47.142 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:41:47.142 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:41:47.142 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:41:47.142 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:41:47.142 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:41:47.142 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:41:47.142 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:41:47.142 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:41:47.142 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:41:47.142 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:41:47.142 | INFO | ------------------------------------------ | 20260507 03:41:47.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:41:47.142 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:41:47.142 | INFO | Total mem usage: 25.6 GiB | 20260507 03:41:47.142 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:41:47.142 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:41:47.142 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:41:47.142 | INFO | High Water Breakdown: | 20260507 03:41:47.142 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:41:47.142 | INFO | ------------------------------------------ | 20260507 03:41:47.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:41:47.142 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:41:47.142 | INFO | Total mem usage: 15.7 GiB | 20260507 03:41:47.142 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:41:47.142 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:41:47.142 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:41:47.142 | INFO | High Water Breakdown: | 20260507 03:41:47.142 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:41:47.142 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:41:47.142 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:41:47.142 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:41:47.142 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:41:47.142 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:41:47.142 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:41:47.142 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:41:47.142 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:41:47.142 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:41:47.142 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:41:47.142 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:41:47.142 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:41:47.142 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:41:47.142 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:41:47.142 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:41:47.142 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:41:47.142 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:41:47.142 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:41:47.142 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:41:47.142 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:41:47.142 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:41:47.142 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:41:47.142 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:41:47.142 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:41:47.142 | INFO | ------------------------------------------ | 20260507 03:41:47.142 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:41:47.142 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:41:47.142 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:41:47.142 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:41:47.142 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.50e+01, 1.32e+01 / 1.49e+00 | 20260507 03:41:47.142 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.81e-03 / 8.38e-03 | 20260507 03:41:47.142 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.41e+00, 5.17e-01 / 2.63e-01 | 20260507 03:41:47.142 | INFO | PerfReport: 3.24%, Histogram -- 9.99e-01 / 1.46e+00, 1.72e-01 / 1.00e-01 | 20260507 03:41:47.142 | INFO | PerfReport: 16.09%, DME -- 3.97e+01 / 4.43e+01, 3.84e+00 / 1.71e+00 | 20260507 03:41:47.142 | INFO | PerfReport: 57.74%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.40e+00 / 1.42e+00 | 20260507 03:41:47.142 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.13e+00 / 2.67e+00, 3.75e-01 / 2.19e-01 | 20260507 03:41:47.143 | INFO | PerfReport: 3.02%, Metrics -- 9.32e-01 / 1.24e+00, 2.44e-01 / 1.19e-01 | 20260507 03:41:47.143 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.16e-01 / 3.67e-01, 4.93e-02 / 1.66e-02 | 20260507 03:41:47.143 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:41:47.143 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:41:47.143 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:41:47.143 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 03:41:47.143 | INFO | PerfReport: 21.69%, Upload -- 2.09e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260507 03:41:47.143 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260507 03:41:47.143 | INFO | PerfReport: 32.02%, Compute -- 3.08e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 03:41:47.143 | INFO | PerfReport: 1.41%, LockUpload -- 1.36e+00 / 0.00e+00, 4.73e+00 / 0.00e+00 | 20260507 03:41:47.143 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.17e+00 / 0.00e+00 | 20260507 03:41:47.143 | INFO | PerfReport: 22.18%, LockCompute -- 2.13e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 03:41:47.143 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.57e-02 / 0.00e+00 | 20260507 03:41:47.238 | INFO | Starting a round of Baz IO | 20260507 03:41:50.344 | INFO | Populating RTMetrics report | 20260507 03:41:50.345 | INFO | RTMetrics report populated | 20260507 03:41:50.345 | INFO | Opening tmp RTMetrics json file | 20260507 03:41:50.364 | INFO | Serializing RTMetrics to json | 20260507 03:41:50.365 | INFO | Serialization done, swapping tmp file now... | 20260507 03:41:50.365 | INFO | RTMetrics done | 20260507 03:41:50.438 | INFO | Finished a round of Baz IO: Wrote 1.748 GiB in 3.2 seconds | 20260507 03:41:50.438 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.87% / 0.00% / 67.13% | 20260507 03:41:50.438 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.193206% / 3.97364e-06% / 8.34465e-05% | 20260507 03:41:50.438 | INFO | Baz IO overall: | 20260507 03:41:50.438 | INFO | BytesWritten : 3493636729483 | 20260507 03:41:50.438 | INFO | NumEvents : 1678340760945 | 20260507 03:41:52.282 | INFO | Analyzing chunk frames = [6815744,6816256) | 20260507 03:41:52.282 | INFO | MemInfo: { | 20260507 03:41:52.282 | INFO | "gpu" : " 17 GB", | 20260507 03:41:52.282 | INFO | "huge" : "", | 20260507 03:41:52.282 | INFO | "huge_shared" : " 27 GB", | 20260507 03:41:52.282 | INFO | "malloc" : " 14 GB", | 20260507 03:41:52.282 | INFO | "pinned" : " 29 GB" | 20260507 03:41:52.282 | INFO | } | 20260507 03:41:52.282 | INFO | Perf report for the last 5.14074 seconds: | 20260507 03:41:52.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:41:52.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:52.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:52.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:52.282 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260507 03:41:52.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:52.282 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 03:41:52.282 | INFO | PRE_HQ : 59.3%, 1.8, 56.5ms, 0 | 20260507 03:41:52.282 | INFO | BAZWRITER : 61.8%, 3.3, 106.3ms, 0 | 20260507 03:41:52.282 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:41:52.282 | INFO | Uploaded 36GiB to GPU in 2.05103s (17.5522GiB/s) | 20260507 03:41:52.282 | INFO | Downloaded 38GiB to GPU in 2.17982s (17.4326GiB/s) | 20260507 03:41:57.419 | INFO | Analyzing chunk frames = [6816256,6816768) | 20260507 03:41:57.419 | INFO | MemInfo: { | 20260507 03:41:57.419 | INFO | "gpu" : " 17 GB", | 20260507 03:41:57.419 | INFO | "huge" : "", | 20260507 03:41:57.419 | INFO | "huge_shared" : " 27 GB", | 20260507 03:41:57.419 | INFO | "malloc" : " 14 GB", | 20260507 03:41:57.419 | INFO | "pinned" : " 29 GB" | 20260507 03:41:57.419 | INFO | } | 20260507 03:41:57.419 | INFO | Perf report for the last 5.13769 seconds: | 20260507 03:41:57.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:41:57.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:57.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:57.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:57.419 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:41:57.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:57.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:57.419 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 03:41:57.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:57.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:41:57.420 | INFO | Uploaded 36GiB to GPU in 1.99405s (18.0537GiB/s) | 20260507 03:41:57.420 | INFO | Downloaded 34GiB to GPU in 1.92782s (17.6365GiB/s) | 20260507 03:42:02.553 | INFO | Analyzing chunk frames = [6816768,6817280) | 20260507 03:42:02.553 | INFO | MemInfo: { | 20260507 03:42:02.553 | INFO | "gpu" : " 17 GB", | 20260507 03:42:02.553 | INFO | "huge" : "", | 20260507 03:42:02.553 | INFO | "huge_shared" : " 27 GB", | 20260507 03:42:02.553 | INFO | "malloc" : " 14 GB", | 20260507 03:42:02.553 | INFO | "pinned" : " 29 GB" | 20260507 03:42:02.553 | INFO | } | 20260507 03:42:02.553 | INFO | Perf report for the last 5.13409 seconds: | 20260507 03:42:02.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:42:02.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:02.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:02.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:02.553 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 03:42:02.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:02.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:02.554 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:42:02.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:02.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:02.554 | INFO | Uploaded 36GiB to GPU in 2.03345s (17.7039GiB/s) | 20260507 03:42:02.554 | INFO | Downloaded 34GiB to GPU in 1.94714s (17.4615GiB/s) | 20260507 03:42:07.693 | INFO | Analyzing chunk frames = [6817280,6817792) | 20260507 03:42:07.693 | INFO | MemInfo: { | 20260507 03:42:07.693 | INFO | "gpu" : " 17 GB", | 20260507 03:42:07.693 | INFO | "huge" : "", | 20260507 03:42:07.693 | INFO | "huge_shared" : " 27 GB", | 20260507 03:42:07.693 | INFO | "malloc" : " 14 GB", | 20260507 03:42:07.693 | INFO | "pinned" : " 29 GB" | 20260507 03:42:07.693 | INFO | } | 20260507 03:42:07.693 | INFO | Perf report for the last 5.13938 seconds: | 20260507 03:42:07.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:42:07.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:07.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:07.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:07.693 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:42:07.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:07.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:07.693 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:42:07.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:07.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:07.693 | INFO | Uploaded 36GiB to GPU in 1.98486s (18.1373GiB/s) | 20260507 03:42:07.693 | INFO | Downloaded 34GiB to GPU in 1.90828s (17.8171GiB/s) | 20260507 03:42:12.830 | INFO | Analyzing chunk frames = [6817792,6818304) | 20260507 03:42:12.830 | INFO | MemInfo: { | 20260507 03:42:12.830 | INFO | "gpu" : " 17 GB", | 20260507 03:42:12.830 | INFO | "huge" : "", | 20260507 03:42:12.830 | INFO | "huge_shared" : " 27 GB", | 20260507 03:42:12.830 | INFO | "malloc" : " 14 GB", | 20260507 03:42:12.830 | INFO | "pinned" : " 29 GB" | 20260507 03:42:12.830 | INFO | } | 20260507 03:42:12.830 | INFO | Perf report for the last 5.13747 seconds: | 20260507 03:42:12.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:42:12.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:12.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:12.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:12.830 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 03:42:12.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:12.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:12.830 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:42:12.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:12.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:12.831 | INFO | Uploaded 36GiB to GPU in 2.0148s (17.8678GiB/s) | 20260507 03:42:12.831 | INFO | Downloaded 34GiB to GPU in 1.9502s (17.4341GiB/s) | 20260507 03:42:17.978 | INFO | Analyzing chunk frames = [6818304,6818816) | 20260507 03:42:17.978 | INFO | MemInfo: { | 20260507 03:42:17.978 | INFO | "gpu" : " 17 GB", | 20260507 03:42:17.978 | INFO | "huge" : "", | 20260507 03:42:17.978 | INFO | "huge_shared" : " 27 GB", | 20260507 03:42:17.978 | INFO | "malloc" : " 14 GB", | 20260507 03:42:17.978 | INFO | "pinned" : " 29 GB" | 20260507 03:42:17.978 | INFO | } | 20260507 03:42:17.978 | INFO | Perf report for the last 5.14778 seconds: | 20260507 03:42:17.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:42:17.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:17.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:17.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:17.978 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 03:42:17.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:17.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:17.978 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:42:17.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:17.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:17.978 | INFO | Uploaded 36GiB to GPU in 2.0524s (17.5404GiB/s) | 20260507 03:42:17.978 | INFO | Downloaded 34GiB to GPU in 1.95025s (17.4337GiB/s) | 20260507 03:42:23.116 | INFO | Analyzing chunk frames = [6818816,6819328) | 20260507 03:42:23.116 | INFO | MemInfo: { | 20260507 03:42:23.116 | INFO | "gpu" : " 17 GB", | 20260507 03:42:23.116 | INFO | "huge" : "", | 20260507 03:42:23.116 | INFO | "huge_shared" : " 27 GB", | 20260507 03:42:23.116 | INFO | "malloc" : " 14 GB", | 20260507 03:42:23.116 | INFO | "pinned" : " 29 GB" | 20260507 03:42:23.116 | INFO | } | 20260507 03:42:23.116 | INFO | Perf report for the last 5.13837 seconds: | 20260507 03:42:23.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:42:23.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:23.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:23.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:23.117 | INFO | ANALYSIS : 61.2%, 3.0, 95.3ms, 0 | 20260507 03:42:23.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:23.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:23.117 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:42:23.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:23.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:23.117 | INFO | Uploaded 36GiB to GPU in 2.04926s (17.5673GiB/s) | 20260507 03:42:23.117 | INFO | Downloaded 34GiB to GPU in 1.9879s (17.1035GiB/s) | 20260507 03:42:28.249 | INFO | Analyzing chunk frames = [6819328,6819840) | 20260507 03:42:28.249 | INFO | MemInfo: { | 20260507 03:42:28.249 | INFO | "gpu" : " 17 GB", | 20260507 03:42:28.249 | INFO | "huge" : "", | 20260507 03:42:28.249 | INFO | "huge_shared" : " 27 GB", | 20260507 03:42:28.249 | INFO | "malloc" : " 14 GB", | 20260507 03:42:28.249 | INFO | "pinned" : " 29 GB" | 20260507 03:42:28.250 | INFO | } | 20260507 03:42:28.250 | INFO | Perf report for the last 5.13323 seconds: | 20260507 03:42:28.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:42:28.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:28.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:28.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:28.250 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 03:42:28.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:28.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:28.250 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 03:42:28.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:28.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:28.250 | INFO | Uploaded 36GiB to GPU in 1.95639s (18.4012GiB/s) | 20260507 03:42:28.250 | INFO | Downloaded 34GiB to GPU in 1.96254s (17.3244GiB/s) | 20260507 03:42:28.347 | INFO | Starting a round of Baz IO | 20260507 03:42:31.488 | INFO | Populating RTMetrics report | 20260507 03:42:31.488 | INFO | RTMetrics report populated | 20260507 03:42:31.488 | INFO | Opening tmp RTMetrics json file | 20260507 03:42:31.489 | INFO | Serializing RTMetrics to json | 20260507 03:42:31.490 | INFO | Serialization done, swapping tmp file now... | 20260507 03:42:31.491 | INFO | RTMetrics done | 20260507 03:42:31.574 | INFO | Finished a round of Baz IO: Wrote 1.746 GiB in 3.227 seconds | 20260507 03:42:31.574 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.86% / 0.00% / 67.13% | 20260507 03:42:31.574 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.191998% / 0% / 8.34465e-05% | 20260507 03:42:31.575 | INFO | Baz IO overall: | 20260507 03:42:31.575 | INFO | BytesWritten : 3495511550356 | 20260507 03:42:31.575 | INFO | NumEvents : 1679224633561 | 20260507 03:42:33.400 | INFO | Analyzing chunk frames = [6819840,6820352) | 20260507 03:42:33.400 | INFO | MemInfo: { | 20260507 03:42:33.400 | INFO | "gpu" : " 17 GB", | 20260507 03:42:33.400 | INFO | "huge" : "", | 20260507 03:42:33.400 | INFO | "huge_shared" : " 27 GB", | 20260507 03:42:33.400 | INFO | "malloc" : " 14 GB", | 20260507 03:42:33.400 | INFO | "pinned" : " 29 GB" | 20260507 03:42:33.400 | INFO | } | 20260507 03:42:33.400 | INFO | Perf report for the last 5.15025 seconds: | 20260507 03:42:33.400 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:42:33.400 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:33.400 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:33.400 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:33.400 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 03:42:33.400 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:33.400 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260507 03:42:33.400 | INFO | PRE_HQ : 60.3%, 1.9, 60.2ms, 0 | 20260507 03:42:33.400 | INFO | BAZWRITER : 62.3%, 3.5, 110.9ms, 0 | 20260507 03:42:33.400 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 03:42:33.400 | INFO | Uploaded 36GiB to GPU in 2.02725s (17.758GiB/s) | 20260507 03:42:33.400 | INFO | Downloaded 38GiB to GPU in 2.18592s (17.384GiB/s) | 20260507 03:42:38.531 | INFO | Analyzing chunk frames = [6820352,6820864) | 20260507 03:42:38.531 | INFO | MemInfo: { | 20260507 03:42:38.531 | INFO | "gpu" : " 17 GB", | 20260507 03:42:38.531 | INFO | "huge" : "", | 20260507 03:42:38.531 | INFO | "huge_shared" : " 27 GB", | 20260507 03:42:38.531 | INFO | "malloc" : " 14 GB", | 20260507 03:42:38.531 | INFO | "pinned" : " 29 GB" | 20260507 03:42:38.531 | INFO | } | 20260507 03:42:38.532 | INFO | Perf report for the last 5.13153 seconds: | 20260507 03:42:38.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:42:38.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:38.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:38.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:38.532 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 03:42:38.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:38.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:38.532 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:42:38.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:38.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:38.532 | INFO | Uploaded 36GiB to GPU in 1.99837s (18.0146GiB/s) | 20260507 03:42:38.532 | INFO | Downloaded 34GiB to GPU in 1.93427s (17.5777GiB/s) | 20260507 03:42:43.667 | INFO | Analyzing chunk frames = [6820864,6821376) | 20260507 03:42:43.667 | INFO | MemInfo: { | 20260507 03:42:43.667 | INFO | "gpu" : " 17 GB", | 20260507 03:42:43.667 | INFO | "huge" : "", | 20260507 03:42:43.667 | INFO | "huge_shared" : " 27 GB", | 20260507 03:42:43.667 | INFO | "malloc" : " 14 GB", | 20260507 03:42:43.668 | INFO | "pinned" : " 29 GB" | 20260507 03:42:43.668 | INFO | } | 20260507 03:42:43.668 | INFO | Perf report for the last 5.13613 seconds: | 20260507 03:42:43.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:42:43.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:43.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:43.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:43.668 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 03:42:43.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:43.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:43.668 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:42:43.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:43.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:43.668 | INFO | Uploaded 36GiB to GPU in 2.02417s (17.7851GiB/s) | 20260507 03:42:43.668 | INFO | Downloaded 34GiB to GPU in 1.95513s (17.3902GiB/s) | 20260507 03:42:48.807 | INFO | Analyzing chunk frames = [6821376,6821888) | 20260507 03:42:48.807 | INFO | MemInfo: { | 20260507 03:42:48.807 | INFO | "gpu" : " 17 GB", | 20260507 03:42:48.807 | INFO | "huge" : "", | 20260507 03:42:48.807 | INFO | "huge_shared" : " 27 GB", | 20260507 03:42:48.807 | INFO | "malloc" : " 14 GB", | 20260507 03:42:48.807 | INFO | "pinned" : " 29 GB" | 20260507 03:42:48.807 | INFO | } | 20260507 03:42:48.807 | INFO | Perf report for the last 5.13956 seconds: | 20260507 03:42:48.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:42:48.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:48.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:48.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:48.807 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:42:48.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:48.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:48.807 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:42:48.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:48.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:48.807 | INFO | Uploaded 36GiB to GPU in 1.96888s (18.2845GiB/s) | 20260507 03:42:48.808 | INFO | Downloaded 34GiB to GPU in 1.92396s (17.6719GiB/s) | 20260507 03:42:53.950 | INFO | Analyzing chunk frames = [6821888,6822400) | 20260507 03:42:53.950 | INFO | MemInfo: { | 20260507 03:42:53.950 | INFO | "gpu" : " 17 GB", | 20260507 03:42:53.950 | INFO | "huge" : "", | 20260507 03:42:53.950 | INFO | "huge_shared" : " 27 GB", | 20260507 03:42:53.950 | INFO | "malloc" : " 14 GB", | 20260507 03:42:53.950 | INFO | "pinned" : " 29 GB" | 20260507 03:42:53.950 | INFO | } | 20260507 03:42:53.950 | INFO | Perf report for the last 5.14325 seconds: | 20260507 03:42:53.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:42:53.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:53.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:53.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:53.951 | INFO | ANALYSIS : 59.8%, 3.0, 93.1ms, 0 | 20260507 03:42:53.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:53.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:53.951 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:42:53.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:53.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:53.951 | INFO | Uploaded 36GiB to GPU in 1.98139s (18.169GiB/s) | 20260507 03:42:53.951 | INFO | Downloaded 34GiB to GPU in 1.93646s (17.5578GiB/s) | 20260507 03:42:59.092 | INFO | Analyzing chunk frames = [6822400,6822912) | 20260507 03:42:59.092 | INFO | MemInfo: { | 20260507 03:42:59.092 | INFO | "gpu" : " 17 GB", | 20260507 03:42:59.092 | INFO | "huge" : "", | 20260507 03:42:59.092 | INFO | "huge_shared" : " 27 GB", | 20260507 03:42:59.092 | INFO | "malloc" : " 14 GB", | 20260507 03:42:59.092 | INFO | "pinned" : " 29 GB" | 20260507 03:42:59.092 | INFO | } | 20260507 03:42:59.092 | INFO | Perf report for the last 5.1419 seconds: | 20260507 03:42:59.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:42:59.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:59.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:59.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:59.092 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 03:42:59.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:59.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:59.092 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:42:59.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:59.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:42:59.093 | INFO | Uploaded 36GiB to GPU in 2.04388s (17.6135GiB/s) | 20260507 03:42:59.093 | INFO | Downloaded 34GiB to GPU in 1.95985s (17.3482GiB/s) | 20260507 03:43:04.230 | INFO | Analyzing chunk frames = [6822912,6823424) | 20260507 03:43:04.230 | INFO | MemInfo: { | 20260507 03:43:04.230 | INFO | "gpu" : " 17 GB", | 20260507 03:43:04.230 | INFO | "huge" : "", | 20260507 03:43:04.230 | INFO | "huge_shared" : " 27 GB", | 20260507 03:43:04.230 | INFO | "malloc" : " 14 GB", | 20260507 03:43:04.230 | INFO | "pinned" : " 29 GB" | 20260507 03:43:04.230 | INFO | } | 20260507 03:43:04.230 | INFO | Perf report for the last 5.13743 seconds: | 20260507 03:43:04.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:43:04.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:04.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:04.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:04.230 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:43:04.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:04.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:04.230 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:43:04.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:04.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:04.230 | INFO | Uploaded 36GiB to GPU in 1.95951s (18.372GiB/s) | 20260507 03:43:04.230 | INFO | Downloaded 34GiB to GPU in 1.9782s (17.1873GiB/s) | 20260507 03:43:09.366 | INFO | Analyzing chunk frames = [6823424,6823936) | 20260507 03:43:09.366 | INFO | MemInfo: { | 20260507 03:43:09.366 | INFO | "gpu" : " 17 GB", | 20260507 03:43:09.366 | INFO | "huge" : "", | 20260507 03:43:09.366 | INFO | "huge_shared" : " 27 GB", | 20260507 03:43:09.366 | INFO | "malloc" : " 14 GB", | 20260507 03:43:09.366 | INFO | "pinned" : " 29 GB" | 20260507 03:43:09.366 | INFO | } | 20260507 03:43:09.366 | INFO | Perf report for the last 5.13621 seconds: | 20260507 03:43:09.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:43:09.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:09.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:09.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:09.366 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 03:43:09.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:09.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:09.366 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 03:43:09.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:09.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:09.366 | INFO | Uploaded 36GiB to GPU in 1.98799s (18.1088GiB/s) | 20260507 03:43:09.366 | INFO | Downloaded 34GiB to GPU in 1.96081s (17.3398GiB/s) | 20260507 03:43:09.366 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:43:09.366 | INFO | Total mem usage: 18 GiB | 20260507 03:43:09.366 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:43:09.366 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:43:09.366 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:43:09.367 | INFO | High Water Breakdown: | 20260507 03:43:09.367 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:43:09.367 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:43:09.367 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:43:09.367 | INFO | ------------------------------------------ | 20260507 03:43:09.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:43:09.367 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:43:09.367 | INFO | Total mem usage: 26.5 GiB | 20260507 03:43:09.367 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:43:09.367 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:43:09.367 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:43:09.367 | INFO | High Water Breakdown: | 20260507 03:43:09.367 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:43:09.367 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:43:09.367 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:43:09.367 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:43:09.367 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:43:09.367 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:43:09.367 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:43:09.367 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:43:09.367 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:43:09.367 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:43:09.367 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:43:09.367 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:43:09.367 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:43:09.367 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:43:09.367 | INFO | ------------------------------------------ | 20260507 03:43:09.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:43:09.367 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:43:09.367 | INFO | Total mem usage: 25.6 GiB | 20260507 03:43:09.367 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:43:09.367 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:43:09.367 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:43:09.367 | INFO | High Water Breakdown: | 20260507 03:43:09.367 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:43:09.367 | INFO | ------------------------------------------ | 20260507 03:43:09.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:43:09.367 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:43:09.367 | INFO | Total mem usage: 15.7 GiB | 20260507 03:43:09.367 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:43:09.367 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:43:09.367 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:43:09.367 | INFO | High Water Breakdown: | 20260507 03:43:09.367 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:43:09.367 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:43:09.367 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:43:09.367 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:43:09.367 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:43:09.367 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:43:09.367 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:43:09.367 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:43:09.367 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:43:09.367 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:43:09.367 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:43:09.367 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:43:09.367 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:43:09.367 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:43:09.367 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:43:09.367 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:43:09.367 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:43:09.367 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:43:09.367 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:43:09.367 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:43:09.367 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:43:09.367 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:43:09.367 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:43:09.367 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:43:09.367 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:43:09.367 | INFO | ------------------------------------------ | 20260507 03:43:09.367 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:43:09.367 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:43:09.367 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:43:09.367 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:43:09.367 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.51e+01, 1.36e+01 / 1.36e+00 | 20260507 03:43:09.367 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.00e-01, 1.02e-02 / 6.10e-03 | 20260507 03:43:09.367 | INFO | PerfReport: 11.70%, Baseline -- 3.60e+00 / 4.37e+00, 5.26e-01 / 3.49e-01 | 20260507 03:43:09.367 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.46e+00, 1.73e-01 / 1.30e-01 | 20260507 03:43:09.367 | INFO | PerfReport: 16.13%, DME -- 3.97e+01 / 4.43e+01, 3.92e+00 / 1.75e+00 | 20260507 03:43:09.367 | INFO | PerfReport: 57.61%, FrameLabeling -- 1.77e+01 / 2.06e+01, 1.31e+00 / 1.85e+00 | 20260507 03:43:09.367 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.66e+00, 3.78e-01 / 2.57e-01 | 20260507 03:43:09.368 | INFO | PerfReport: 3.04%, Metrics -- 9.36e-01 / 1.26e+00, 2.47e-01 / 1.23e-01 | 20260507 03:43:09.368 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.16e-01 / 3.66e-01, 5.01e-02 / 1.82e-02 | 20260507 03:43:09.368 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:43:09.368 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:43:09.368 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:43:09.368 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 03:43:09.368 | INFO | PerfReport: 21.74%, Upload -- 2.09e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 03:43:09.368 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260507 03:43:09.368 | INFO | PerfReport: 32.01%, Compute -- 3.08e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260507 03:43:09.368 | INFO | PerfReport: 1.36%, LockUpload -- 1.30e+00 / 0.00e+00, 4.68e+00 / 0.00e+00 | 20260507 03:43:09.368 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260507 03:43:09.368 | INFO | PerfReport: 22.14%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 03:43:09.368 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.93e-02 / 0.00e+00 | 20260507 03:43:09.463 | INFO | Starting a round of Baz IO | 20260507 03:43:12.570 | INFO | Populating RTMetrics report | 20260507 03:43:12.570 | INFO | RTMetrics report populated | 20260507 03:43:12.570 | INFO | Opening tmp RTMetrics json file | 20260507 03:43:12.587 | INFO | Serializing RTMetrics to json | 20260507 03:43:12.589 | INFO | Serialization done, swapping tmp file now... | 20260507 03:43:12.589 | INFO | RTMetrics done | 20260507 03:43:12.661 | INFO | Finished a round of Baz IO: Wrote 1.745 GiB in 3.198 seconds | 20260507 03:43:12.662 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.86% / 0.00% / 67.14% | 20260507 03:43:12.662 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.191482% / 0% / 9.53674e-05% | 20260507 03:43:12.662 | INFO | Baz IO overall: | 20260507 03:43:12.662 | INFO | BytesWritten : 3497385417220 | 20260507 03:43:12.662 | INFO | NumEvents : 1680107694063 | 20260507 03:43:14.503 | INFO | Analyzing chunk frames = [6823936,6824448) | 20260507 03:43:14.503 | INFO | MemInfo: { | 20260507 03:43:14.503 | INFO | "gpu" : " 17 GB", | 20260507 03:43:14.503 | INFO | "huge" : "", | 20260507 03:43:14.503 | INFO | "huge_shared" : " 27 GB", | 20260507 03:43:14.503 | INFO | "malloc" : " 14 GB", | 20260507 03:43:14.503 | INFO | "pinned" : " 29 GB" | 20260507 03:43:14.503 | INFO | } | 20260507 03:43:14.503 | INFO | Perf report for the last 5.13702 seconds: | 20260507 03:43:14.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:43:14.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:14.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:14.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:14.503 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 03:43:14.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:14.503 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 03:43:14.503 | INFO | PRE_HQ : 59.6%, 1.9, 57.8ms, 0 | 20260507 03:43:14.503 | INFO | BAZWRITER : 61.9%, 3.4, 107.4ms, 0 | 20260507 03:43:14.503 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:43:14.504 | INFO | Uploaded 36GiB to GPU in 2.039s (17.6557GiB/s) | 20260507 03:43:14.504 | INFO | Downloaded 38GiB to GPU in 2.18622s (17.3816GiB/s) | 20260507 03:43:19.646 | INFO | Analyzing chunk frames = [6824448,6824960) | 20260507 03:43:19.646 | INFO | MemInfo: { | 20260507 03:43:19.646 | INFO | "gpu" : " 17 GB", | 20260507 03:43:19.646 | INFO | "huge" : "", | 20260507 03:43:19.646 | INFO | "huge_shared" : " 27 GB", | 20260507 03:43:19.646 | INFO | "malloc" : " 14 GB", | 20260507 03:43:19.646 | INFO | "pinned" : " 29 GB" | 20260507 03:43:19.646 | INFO | } | 20260507 03:43:19.647 | INFO | Perf report for the last 5.14358 seconds: | 20260507 03:43:19.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:43:19.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:19.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:19.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:19.647 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:43:19.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:19.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:19.647 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 03:43:19.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:19.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:19.647 | INFO | Uploaded 36GiB to GPU in 2.04358s (17.6161GiB/s) | 20260507 03:43:19.647 | INFO | Downloaded 34GiB to GPU in 1.94031s (17.523GiB/s) | 20260507 03:43:24.788 | INFO | Analyzing chunk frames = [6824960,6825472) | 20260507 03:43:24.788 | INFO | MemInfo: { | 20260507 03:43:24.788 | INFO | "gpu" : " 17 GB", | 20260507 03:43:24.788 | INFO | "huge" : "", | 20260507 03:43:24.788 | INFO | "huge_shared" : " 27 GB", | 20260507 03:43:24.788 | INFO | "malloc" : " 14 GB", | 20260507 03:43:24.788 | INFO | "pinned" : " 29 GB" | 20260507 03:43:24.788 | INFO | } | 20260507 03:43:24.788 | INFO | Perf report for the last 5.14165 seconds: | 20260507 03:43:24.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:43:24.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:24.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:24.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:24.788 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 03:43:24.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:24.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:24.788 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:43:24.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:24.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:24.789 | INFO | Uploaded 36GiB to GPU in 2.01121s (17.8997GiB/s) | 20260507 03:43:24.789 | INFO | Downloaded 34GiB to GPU in 1.93819s (17.5421GiB/s) | 20260507 03:43:29.926 | INFO | Analyzing chunk frames = [6825472,6825984) | 20260507 03:43:29.926 | INFO | MemInfo: { | 20260507 03:43:29.926 | INFO | "gpu" : " 17 GB", | 20260507 03:43:29.926 | INFO | "huge" : "", | 20260507 03:43:29.926 | INFO | "huge_shared" : " 27 GB", | 20260507 03:43:29.926 | INFO | "malloc" : " 14 GB", | 20260507 03:43:29.926 | INFO | "pinned" : " 29 GB" | 20260507 03:43:29.926 | INFO | } | 20260507 03:43:29.926 | INFO | Perf report for the last 5.13809 seconds: | 20260507 03:43:29.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:43:29.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:29.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:29.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:29.926 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 03:43:29.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:29.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:29.927 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:43:29.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:29.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:29.927 | INFO | Uploaded 36GiB to GPU in 1.98983s (18.092GiB/s) | 20260507 03:43:29.927 | INFO | Downloaded 34GiB to GPU in 1.92032s (17.7053GiB/s) | 20260507 03:43:35.061 | INFO | Analyzing chunk frames = [6825984,6826496) | 20260507 03:43:35.061 | INFO | MemInfo: { | 20260507 03:43:35.061 | INFO | "gpu" : " 17 GB", | 20260507 03:43:35.061 | INFO | "huge" : "", | 20260507 03:43:35.061 | INFO | "huge_shared" : " 27 GB", | 20260507 03:43:35.061 | INFO | "malloc" : " 14 GB", | 20260507 03:43:35.061 | INFO | "pinned" : " 29 GB" | 20260507 03:43:35.061 | INFO | } | 20260507 03:43:35.061 | INFO | Perf report for the last 5.13499 seconds: | 20260507 03:43:35.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:43:35.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:35.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:35.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:35.062 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 03:43:35.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:35.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:35.062 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:43:35.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:35.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:35.062 | INFO | Uploaded 36GiB to GPU in 2.05516s (17.5169GiB/s) | 20260507 03:43:35.062 | INFO | Downloaded 34GiB to GPU in 1.94569s (17.4745GiB/s) | 20260507 03:43:40.203 | INFO | Analyzing chunk frames = [6826496,6827008) | 20260507 03:43:40.203 | INFO | MemInfo: { | 20260507 03:43:40.203 | INFO | "gpu" : " 17 GB", | 20260507 03:43:40.203 | INFO | "huge" : "", | 20260507 03:43:40.203 | INFO | "huge_shared" : " 27 GB", | 20260507 03:43:40.203 | INFO | "malloc" : " 14 GB", | 20260507 03:43:40.203 | INFO | "pinned" : " 29 GB" | 20260507 03:43:40.203 | INFO | } | 20260507 03:43:40.203 | INFO | Perf report for the last 5.14183 seconds: | 20260507 03:43:40.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:43:40.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:40.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:40.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:40.203 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:43:40.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:40.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:40.203 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:43:40.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:40.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:40.204 | INFO | Uploaded 36GiB to GPU in 2.05847s (17.4887GiB/s) | 20260507 03:43:40.204 | INFO | Downloaded 34GiB to GPU in 1.96335s (17.3173GiB/s) | 20260507 03:43:45.343 | INFO | Analyzing chunk frames = [6827008,6827520) | 20260507 03:43:45.343 | INFO | MemInfo: { | 20260507 03:43:45.343 | INFO | "gpu" : " 17 GB", | 20260507 03:43:45.343 | INFO | "huge" : "", | 20260507 03:43:45.343 | INFO | "huge_shared" : " 27 GB", | 20260507 03:43:45.343 | INFO | "malloc" : " 14 GB", | 20260507 03:43:45.343 | INFO | "pinned" : " 29 GB" | 20260507 03:43:45.343 | INFO | } | 20260507 03:43:45.343 | INFO | Perf report for the last 5.14006 seconds: | 20260507 03:43:45.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:43:45.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:45.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:45.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:45.344 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:43:45.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:45.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:45.344 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:43:45.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:45.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:45.344 | INFO | Uploaded 36GiB to GPU in 1.98025s (18.1795GiB/s) | 20260507 03:43:45.344 | INFO | Downloaded 34GiB to GPU in 1.959s (17.3558GiB/s) | 20260507 03:43:50.485 | INFO | Analyzing chunk frames = [6827520,6828032) | 20260507 03:43:50.485 | INFO | MemInfo: { | 20260507 03:43:50.485 | INFO | "gpu" : " 17 GB", | 20260507 03:43:50.485 | INFO | "huge" : "", | 20260507 03:43:50.485 | INFO | "huge_shared" : " 27 GB", | 20260507 03:43:50.485 | INFO | "malloc" : " 14 GB", | 20260507 03:43:50.485 | INFO | "pinned" : " 29 GB" | 20260507 03:43:50.485 | INFO | } | 20260507 03:43:50.485 | INFO | Perf report for the last 5.14174 seconds: | 20260507 03:43:50.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:43:50.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:50.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:50.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:50.485 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:43:50.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:50.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:50.485 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 03:43:50.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:50.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:50.485 | INFO | Uploaded 36GiB to GPU in 1.96052s (18.3625GiB/s) | 20260507 03:43:50.485 | INFO | Downloaded 34GiB to GPU in 1.97732s (17.195GiB/s) | 20260507 03:43:50.582 | INFO | Starting a round of Baz IO | 20260507 03:43:53.685 | INFO | Populating RTMetrics report | 20260507 03:43:53.685 | INFO | RTMetrics report populated | 20260507 03:43:53.685 | INFO | Opening tmp RTMetrics json file | 20260507 03:43:53.692 | INFO | Serializing RTMetrics to json | 20260507 03:43:53.704 | INFO | Serialization done, swapping tmp file now... | 20260507 03:43:53.704 | INFO | RTMetrics done | 20260507 03:43:53.775 | INFO | Finished a round of Baz IO: Wrote 1.746 GiB in 3.193 seconds | 20260507 03:43:53.775 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.86% / 0.00% / 67.14% | 20260507 03:43:53.775 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.191565% / 0% / 7.15256e-05% | 20260507 03:43:53.775 | INFO | Baz IO overall: | 20260507 03:43:53.775 | INFO | BytesWritten : 3499260040623 | 20260507 03:43:53.775 | INFO | NumEvents : 1680991593416 | 20260507 03:43:55.621 | INFO | Analyzing chunk frames = [6828032,6828544) | 20260507 03:43:55.621 | INFO | MemInfo: { | 20260507 03:43:55.621 | INFO | "gpu" : " 17 GB", | 20260507 03:43:55.621 | INFO | "huge" : "", | 20260507 03:43:55.621 | INFO | "huge_shared" : " 27 GB", | 20260507 03:43:55.621 | INFO | "malloc" : " 14 GB", | 20260507 03:43:55.621 | INFO | "pinned" : " 29 GB" | 20260507 03:43:55.621 | INFO | } | 20260507 03:43:55.621 | INFO | Perf report for the last 5.13636 seconds: | 20260507 03:43:55.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:43:55.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:55.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:55.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:55.621 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260507 03:43:55.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:43:55.622 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:43:55.622 | INFO | PRE_HQ : 59.2%, 1.8, 55.2ms, 0 | 20260507 03:43:55.622 | INFO | BAZWRITER : 61.8%, 3.3, 104.7ms, 0 | 20260507 03:43:55.622 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:43:55.622 | INFO | Uploaded 36GiB to GPU in 2.03726s (17.6708GiB/s) | 20260507 03:43:55.622 | INFO | Downloaded 38GiB to GPU in 2.17867s (17.4418GiB/s) | 20260507 03:44:00.757 | INFO | Analyzing chunk frames = [6828544,6829056) | 20260507 03:44:00.757 | INFO | MemInfo: { | 20260507 03:44:00.757 | INFO | "gpu" : " 17 GB", | 20260507 03:44:00.757 | INFO | "huge" : "", | 20260507 03:44:00.757 | INFO | "huge_shared" : " 27 GB", | 20260507 03:44:00.757 | INFO | "malloc" : " 14 GB", | 20260507 03:44:00.757 | INFO | "pinned" : " 29 GB" | 20260507 03:44:00.757 | INFO | } | 20260507 03:44:00.757 | INFO | Perf report for the last 5.13578 seconds: | 20260507 03:44:00.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:44:00.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:00.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:00.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:00.757 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:44:00.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:00.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:00.757 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:44:00.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:00.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:00.758 | INFO | Uploaded 36GiB to GPU in 2.03334s (17.7049GiB/s) | 20260507 03:44:00.758 | INFO | Downloaded 34GiB to GPU in 1.94121s (17.5148GiB/s) | 20260507 03:44:05.907 | INFO | Analyzing chunk frames = [6829056,6829568) | 20260507 03:44:05.907 | INFO | MemInfo: { | 20260507 03:44:05.907 | INFO | "gpu" : " 17 GB", | 20260507 03:44:05.908 | INFO | "huge" : "", | 20260507 03:44:05.908 | INFO | "huge_shared" : " 27 GB", | 20260507 03:44:05.908 | INFO | "malloc" : " 14 GB", | 20260507 03:44:05.908 | INFO | "pinned" : " 29 GB" | 20260507 03:44:05.908 | INFO | } | 20260507 03:44:05.908 | INFO | Perf report for the last 5.15043 seconds: | 20260507 03:44:05.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:44:05.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:05.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:05.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:05.908 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 03:44:05.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:05.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:05.908 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:44:05.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:05.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:05.908 | INFO | Uploaded 36GiB to GPU in 1.98343s (18.1504GiB/s) | 20260507 03:44:05.908 | INFO | Downloaded 34GiB to GPU in 1.93683s (17.5545GiB/s) | 20260507 03:44:11.045 | INFO | Analyzing chunk frames = [6829568,6830080) | 20260507 03:44:11.045 | INFO | MemInfo: { | 20260507 03:44:11.045 | INFO | "gpu" : " 17 GB", | 20260507 03:44:11.045 | INFO | "huge" : "", | 20260507 03:44:11.045 | INFO | "huge_shared" : " 27 GB", | 20260507 03:44:11.045 | INFO | "malloc" : " 14 GB", | 20260507 03:44:11.045 | INFO | "pinned" : " 29 GB" | 20260507 03:44:11.045 | INFO | } | 20260507 03:44:11.045 | INFO | Perf report for the last 5.13734 seconds: | 20260507 03:44:11.045 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:44:11.045 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:11.045 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:11.045 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:11.045 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:44:11.045 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:11.045 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:11.045 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:44:11.045 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:11.045 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:11.045 | INFO | Uploaded 36GiB to GPU in 1.98557s (18.1308GiB/s) | 20260507 03:44:11.045 | INFO | Downloaded 34GiB to GPU in 1.9384s (17.5403GiB/s) | 20260507 03:44:16.176 | INFO | Analyzing chunk frames = [6830080,6830592) | 20260507 03:44:16.176 | INFO | MemInfo: { | 20260507 03:44:16.176 | INFO | "gpu" : " 17 GB", | 20260507 03:44:16.176 | INFO | "huge" : "", | 20260507 03:44:16.176 | INFO | "huge_shared" : " 27 GB", | 20260507 03:44:16.176 | INFO | "malloc" : " 14 GB", | 20260507 03:44:16.176 | INFO | "pinned" : " 29 GB" | 20260507 03:44:16.176 | INFO | } | 20260507 03:44:16.176 | INFO | Perf report for the last 5.13081 seconds: | 20260507 03:44:16.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:44:16.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:16.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:16.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:16.176 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 03:44:16.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:16.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:16.176 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:44:16.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:16.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:16.176 | INFO | Uploaded 36GiB to GPU in 2.0483s (17.5756GiB/s) | 20260507 03:44:16.176 | INFO | Downloaded 34GiB to GPU in 1.95754s (17.3687GiB/s) | 20260507 03:44:21.313 | INFO | Analyzing chunk frames = [6830592,6831104) | 20260507 03:44:21.313 | INFO | MemInfo: { | 20260507 03:44:21.313 | INFO | "gpu" : " 17 GB", | 20260507 03:44:21.313 | INFO | "huge" : "", | 20260507 03:44:21.313 | INFO | "huge_shared" : " 27 GB", | 20260507 03:44:21.313 | INFO | "malloc" : " 14 GB", | 20260507 03:44:21.313 | INFO | "pinned" : " 29 GB" | 20260507 03:44:21.313 | INFO | } | 20260507 03:44:21.313 | INFO | Perf report for the last 5.13757 seconds: | 20260507 03:44:21.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:44:21.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:21.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:21.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:21.314 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 03:44:21.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:21.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:21.314 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 03:44:21.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:21.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:21.314 | INFO | Uploaded 36GiB to GPU in 2.03949s (17.6514GiB/s) | 20260507 03:44:21.314 | INFO | Downloaded 34GiB to GPU in 1.92742s (17.6402GiB/s) | 20260507 03:44:26.456 | INFO | Analyzing chunk frames = [6831104,6831616) | 20260507 03:44:26.456 | INFO | MemInfo: { | 20260507 03:44:26.456 | INFO | "gpu" : " 17 GB", | 20260507 03:44:26.456 | INFO | "huge" : "", | 20260507 03:44:26.456 | INFO | "huge_shared" : " 27 GB", | 20260507 03:44:26.456 | INFO | "malloc" : " 14 GB", | 20260507 03:44:26.456 | INFO | "pinned" : " 29 GB" | 20260507 03:44:26.457 | INFO | } | 20260507 03:44:26.457 | INFO | Perf report for the last 5.14305 seconds: | 20260507 03:44:26.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:44:26.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:26.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:26.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:26.457 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:44:26.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:26.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:26.457 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:44:26.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:26.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:26.457 | INFO | Uploaded 36GiB to GPU in 1.97752s (18.2046GiB/s) | 20260507 03:44:26.457 | INFO | Downloaded 34GiB to GPU in 1.98671s (17.1137GiB/s) | 20260507 03:44:31.600 | INFO | Analyzing chunk frames = [6831616,6832128) | 20260507 03:44:31.600 | INFO | MemInfo: { | 20260507 03:44:31.600 | INFO | "gpu" : " 17 GB", | 20260507 03:44:31.600 | INFO | "huge" : "", | 20260507 03:44:31.600 | INFO | "huge_shared" : " 27 GB", | 20260507 03:44:31.600 | INFO | "malloc" : " 14 GB", | 20260507 03:44:31.601 | INFO | "pinned" : " 29 GB" | 20260507 03:44:31.601 | INFO | } | 20260507 03:44:31.601 | INFO | Perf report for the last 5.14402 seconds: | 20260507 03:44:31.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:44:31.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:31.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:31.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:31.601 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:44:31.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:31.601 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:31.601 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:44:31.601 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:31.601 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:31.601 | INFO | Uploaded 36GiB to GPU in 1.97315s (18.245GiB/s) | 20260507 03:44:31.601 | INFO | Downloaded 34GiB to GPU in 1.98054s (17.167GiB/s) | 20260507 03:44:31.601 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:44:31.601 | INFO | Total mem usage: 18 GiB | 20260507 03:44:31.601 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:44:31.601 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:44:31.601 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:44:31.601 | INFO | High Water Breakdown: | 20260507 03:44:31.601 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:44:31.601 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:44:31.601 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:44:31.601 | INFO | ------------------------------------------ | 20260507 03:44:31.601 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:44:31.601 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:44:31.601 | INFO | Total mem usage: 26.5 GiB | 20260507 03:44:31.601 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:44:31.601 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:44:31.601 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:44:31.601 | INFO | High Water Breakdown: | 20260507 03:44:31.601 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:44:31.601 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:44:31.601 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:44:31.601 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:44:31.601 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:44:31.601 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:44:31.601 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:44:31.601 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:44:31.601 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:44:31.601 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:44:31.601 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:44:31.601 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:44:31.601 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:44:31.601 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:44:31.601 | INFO | ------------------------------------------ | 20260507 03:44:31.601 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:44:31.601 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:44:31.601 | INFO | Total mem usage: 25.6 GiB | 20260507 03:44:31.602 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:44:31.602 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:44:31.602 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:44:31.602 | INFO | High Water Breakdown: | 20260507 03:44:31.602 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:44:31.602 | INFO | ------------------------------------------ | 20260507 03:44:31.602 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:44:31.602 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:44:31.602 | INFO | Total mem usage: 15.7 GiB | 20260507 03:44:31.602 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:44:31.602 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:44:31.602 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:44:31.602 | INFO | High Water Breakdown: | 20260507 03:44:31.602 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:44:31.602 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:44:31.602 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:44:31.602 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:44:31.602 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:44:31.602 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:44:31.602 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:44:31.602 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:44:31.602 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:44:31.602 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:44:31.602 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:44:31.602 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:44:31.602 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:44:31.602 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:44:31.602 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:44:31.602 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:44:31.602 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:44:31.602 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:44:31.602 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:44:31.602 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:44:31.602 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:44:31.602 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:44:31.602 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:44:31.602 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:44:31.602 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:44:31.602 | INFO | ------------------------------------------ | 20260507 03:44:31.602 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:44:31.602 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:44:31.602 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:44:31.602 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:44:31.602 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.34e+01 / 1.44e+00 | 20260507 03:44:31.602 | INFO | PerfReport: 0.29%, AggregateStats -- 9.03e-02 / 1.01e-01, 9.09e-03 / 5.25e-03 | 20260507 03:44:31.602 | INFO | PerfReport: 11.64%, Baseline -- 3.57e+00 / 4.46e+00, 5.01e-01 / 2.32e-01 | 20260507 03:44:31.602 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.48e+00, 1.67e-01 / 9.27e-02 | 20260507 03:44:31.602 | INFO | PerfReport: 16.11%, DME -- 3.96e+01 / 4.41e+01, 3.74e+00 / 1.66e+00 | 20260507 03:44:31.602 | INFO | PerfReport: 57.69%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.29e+00 / 1.37e+00 | 20260507 03:44:31.602 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.14e+00 / 2.70e+00, 3.83e-01 / 1.96e-01 | 20260507 03:44:31.602 | INFO | PerfReport: 3.05%, Metrics -- 9.36e-01 / 1.26e+00, 2.51e-01 / 1.47e-01 | 20260507 03:44:31.602 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.16e-01 / 3.69e-01, 4.99e-02 / 2.10e-02 | 20260507 03:44:31.602 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:44:31.602 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:44:31.602 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:44:31.602 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 03:44:31.602 | INFO | PerfReport: 21.78%, Upload -- 2.10e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260507 03:44:31.602 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 03:44:31.602 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 03:44:31.602 | INFO | PerfReport: 1.48%, LockUpload -- 1.42e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 03:44:31.602 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260507 03:44:31.602 | INFO | PerfReport: 22.08%, LockCompute -- 2.13e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 03:44:31.602 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260507 03:44:31.697 | INFO | Starting a round of Baz IO | 20260507 03:44:34.799 | INFO | Populating RTMetrics report | 20260507 03:44:34.800 | INFO | RTMetrics report populated | 20260507 03:44:34.800 | INFO | Opening tmp RTMetrics json file | 20260507 03:44:34.801 | INFO | Serializing RTMetrics to json | 20260507 03:44:34.814 | INFO | Serialization done, swapping tmp file now... | 20260507 03:44:34.820 | INFO | RTMetrics done | 20260507 03:44:34.893 | INFO | Finished a round of Baz IO: Wrote 1.746 GiB in 3.195 seconds | 20260507 03:44:34.893 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.86% / 0.00% / 67.14% | 20260507 03:44:34.893 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.191689% / 7.94729e-06% / 7.54992e-05% | 20260507 03:44:34.893 | INFO | Baz IO overall: | 20260507 03:44:34.893 | INFO | BytesWritten : 3501134554958 | 20260507 03:44:34.893 | INFO | NumEvents : 1681875670413 | 20260507 03:44:36.749 | INFO | Analyzing chunk frames = [6832128,6832640) | 20260507 03:44:36.749 | INFO | MemInfo: { | 20260507 03:44:36.749 | INFO | "gpu" : " 17 GB", | 20260507 03:44:36.749 | INFO | "huge" : "", | 20260507 03:44:36.749 | INFO | "huge_shared" : " 27 GB", | 20260507 03:44:36.749 | INFO | "malloc" : " 14 GB", | 20260507 03:44:36.749 | INFO | "pinned" : " 29 GB" | 20260507 03:44:36.749 | INFO | } | 20260507 03:44:36.749 | INFO | Perf report for the last 5.14889 seconds: | 20260507 03:44:36.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:44:36.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:36.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:36.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:36.750 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 03:44:36.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:36.750 | INFO | RT_METRICS : 41.6%, 1.0, 21.9ms, 0 | 20260507 03:44:36.750 | INFO | PRE_HQ : 59.0%, 1.8, 55.1ms, 0 | 20260507 03:44:36.750 | INFO | BAZWRITER : 61.6%, 3.2, 102.6ms, 0 | 20260507 03:44:36.750 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:44:36.750 | INFO | Uploaded 36GiB to GPU in 2.03242s (17.7129GiB/s) | 20260507 03:44:36.750 | INFO | Downloaded 38GiB to GPU in 2.17791s (17.4479GiB/s) | 20260507 03:44:41.876 | INFO | Analyzing chunk frames = [6832640,6833152) | 20260507 03:44:41.876 | INFO | MemInfo: { | 20260507 03:44:41.876 | INFO | "gpu" : " 17 GB", | 20260507 03:44:41.876 | INFO | "huge" : "", | 20260507 03:44:41.876 | INFO | "huge_shared" : " 27 GB", | 20260507 03:44:41.876 | INFO | "malloc" : " 14 GB", | 20260507 03:44:41.877 | INFO | "pinned" : " 29 GB" | 20260507 03:44:41.877 | INFO | } | 20260507 03:44:41.877 | INFO | Perf report for the last 5.12725 seconds: | 20260507 03:44:41.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:44:41.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:41.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:41.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:41.877 | INFO | ANALYSIS : 60.9%, 3.0, 94.5ms, 0 | 20260507 03:44:41.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:41.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:41.877 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:44:41.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:41.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:41.877 | INFO | Uploaded 36GiB to GPU in 1.97911s (18.19GiB/s) | 20260507 03:44:41.877 | INFO | Downloaded 34GiB to GPU in 1.92921s (17.6238GiB/s) | 20260507 03:44:47.013 | INFO | Analyzing chunk frames = [6833152,6833664) | 20260507 03:44:47.013 | INFO | MemInfo: { | 20260507 03:44:47.013 | INFO | "gpu" : " 17 GB", | 20260507 03:44:47.013 | INFO | "huge" : "", | 20260507 03:44:47.013 | INFO | "huge_shared" : " 27 GB", | 20260507 03:44:47.013 | INFO | "malloc" : " 14 GB", | 20260507 03:44:47.013 | INFO | "pinned" : " 29 GB" | 20260507 03:44:47.013 | INFO | } | 20260507 03:44:47.013 | INFO | Perf report for the last 5.13636 seconds: | 20260507 03:44:47.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:44:47.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:47.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:47.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:47.013 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 03:44:47.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:47.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:47.013 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:44:47.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:47.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:47.013 | INFO | Uploaded 36GiB to GPU in 2.01543s (17.8622GiB/s) | 20260507 03:44:47.013 | INFO | Downloaded 34GiB to GPU in 1.94582s (17.4733GiB/s) | 20260507 03:44:52.157 | INFO | Analyzing chunk frames = [6833664,6834176) | 20260507 03:44:52.157 | INFO | MemInfo: { | 20260507 03:44:52.157 | INFO | "gpu" : " 17 GB", | 20260507 03:44:52.157 | INFO | "huge" : "", | 20260507 03:44:52.157 | INFO | "huge_shared" : " 27 GB", | 20260507 03:44:52.157 | INFO | "malloc" : " 14 GB", | 20260507 03:44:52.157 | INFO | "pinned" : " 29 GB" | 20260507 03:44:52.157 | INFO | } | 20260507 03:44:52.157 | INFO | Perf report for the last 5.1436 seconds: | 20260507 03:44:52.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:44:52.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:52.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:52.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:52.157 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260507 03:44:52.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:52.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:52.157 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:44:52.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:52.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:52.157 | INFO | Uploaded 36GiB to GPU in 1.94831s (18.4776GiB/s) | 20260507 03:44:52.157 | INFO | Downloaded 34GiB to GPU in 1.90112s (17.8842GiB/s) | 20260507 03:44:57.297 | INFO | Analyzing chunk frames = [6834176,6834688) | 20260507 03:44:57.297 | INFO | MemInfo: { | 20260507 03:44:57.297 | INFO | "gpu" : " 17 GB", | 20260507 03:44:57.297 | INFO | "huge" : "", | 20260507 03:44:57.297 | INFO | "huge_shared" : " 27 GB", | 20260507 03:44:57.297 | INFO | "malloc" : " 14 GB", | 20260507 03:44:57.297 | INFO | "pinned" : " 29 GB" | 20260507 03:44:57.297 | INFO | } | 20260507 03:44:57.297 | INFO | Perf report for the last 5.14018 seconds: | 20260507 03:44:57.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:44:57.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:57.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:57.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:57.297 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 03:44:57.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:57.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:57.297 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:44:57.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:57.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:44:57.297 | INFO | Uploaded 36GiB to GPU in 2.03387s (17.7002GiB/s) | 20260507 03:44:57.297 | INFO | Downloaded 34GiB to GPU in 1.93046s (17.6124GiB/s) | 20260507 03:45:02.432 | INFO | Analyzing chunk frames = [6834688,6835200) | 20260507 03:45:02.432 | INFO | MemInfo: { | 20260507 03:45:02.432 | INFO | "gpu" : " 17 GB", | 20260507 03:45:02.432 | INFO | "huge" : "", | 20260507 03:45:02.432 | INFO | "huge_shared" : " 27 GB", | 20260507 03:45:02.432 | INFO | "malloc" : " 14 GB", | 20260507 03:45:02.433 | INFO | "pinned" : " 29 GB" | 20260507 03:45:02.433 | INFO | } | 20260507 03:45:02.433 | INFO | Perf report for the last 5.13565 seconds: | 20260507 03:45:02.433 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:45:02.433 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:02.433 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:02.433 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:02.433 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 03:45:02.433 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:02.433 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:02.433 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 03:45:02.433 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:02.433 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:02.433 | INFO | Uploaded 36GiB to GPU in 2.03836s (17.6612GiB/s) | 20260507 03:45:02.433 | INFO | Downloaded 34GiB to GPU in 1.89869s (17.9071GiB/s) | 20260507 03:45:07.571 | INFO | Analyzing chunk frames = [6835200,6835712) | 20260507 03:45:07.571 | INFO | MemInfo: { | 20260507 03:45:07.571 | INFO | "gpu" : " 17 GB", | 20260507 03:45:07.571 | INFO | "huge" : "", | 20260507 03:45:07.571 | INFO | "huge_shared" : " 27 GB", | 20260507 03:45:07.571 | INFO | "malloc" : " 14 GB", | 20260507 03:45:07.571 | INFO | "pinned" : " 29 GB" | 20260507 03:45:07.571 | INFO | } | 20260507 03:45:07.571 | INFO | Perf report for the last 5.13865 seconds: | 20260507 03:45:07.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:45:07.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:07.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:07.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:07.571 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:45:07.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:07.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:07.571 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:45:07.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:07.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:07.571 | INFO | Uploaded 36GiB to GPU in 1.97894s (18.1916GiB/s) | 20260507 03:45:07.571 | INFO | Downloaded 34GiB to GPU in 1.98842s (17.099GiB/s) | 20260507 03:45:12.717 | INFO | Analyzing chunk frames = [6835712,6836224) | 20260507 03:45:12.717 | INFO | MemInfo: { | 20260507 03:45:12.717 | INFO | "gpu" : " 17 GB", | 20260507 03:45:12.717 | INFO | "huge" : "", | 20260507 03:45:12.717 | INFO | "huge_shared" : " 27 GB", | 20260507 03:45:12.717 | INFO | "malloc" : " 14 GB", | 20260507 03:45:12.717 | INFO | "pinned" : " 29 GB" | 20260507 03:45:12.717 | INFO | } | 20260507 03:45:12.717 | INFO | Perf report for the last 5.14562 seconds: | 20260507 03:45:12.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:45:12.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:12.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:12.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:12.717 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:45:12.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:12.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:12.717 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 03:45:12.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:12.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:12.717 | INFO | Uploaded 36GiB to GPU in 1.99778s (18.02GiB/s) | 20260507 03:45:12.717 | INFO | Downloaded 34GiB to GPU in 1.97755s (17.193GiB/s) | 20260507 03:45:12.813 | INFO | Starting a round of Baz IO | 20260507 03:45:15.923 | INFO | Populating RTMetrics report | 20260507 03:45:15.924 | INFO | RTMetrics report populated | 20260507 03:45:15.924 | INFO | Opening tmp RTMetrics json file | 20260507 03:45:15.931 | INFO | Serializing RTMetrics to json | 20260507 03:45:15.941 | INFO | Serialization done, swapping tmp file now... | 20260507 03:45:15.942 | INFO | RTMetrics done | 20260507 03:45:16.014 | INFO | Finished a round of Baz IO: Wrote 1.744 GiB in 3.201 seconds | 20260507 03:45:16.015 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.86% / 0.00% / 67.14% | 20260507 03:45:16.015 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.19081% / 7.94729e-06% / 6.75519e-05% | 20260507 03:45:16.015 | INFO | Baz IO overall: | 20260507 03:45:16.015 | INFO | BytesWritten : 3503007014285 | 20260507 03:45:16.015 | INFO | NumEvents : 1682758117217 | 20260507 03:45:17.853 | INFO | Analyzing chunk frames = [6836224,6836736) | 20260507 03:45:17.853 | INFO | MemInfo: { | 20260507 03:45:17.853 | INFO | "gpu" : " 17 GB", | 20260507 03:45:17.853 | INFO | "huge" : "", | 20260507 03:45:17.853 | INFO | "huge_shared" : " 27 GB", | 20260507 03:45:17.853 | INFO | "malloc" : " 14 GB", | 20260507 03:45:17.853 | INFO | "pinned" : " 29 GB" | 20260507 03:45:17.853 | INFO | } | 20260507 03:45:17.853 | INFO | Perf report for the last 5.13592 seconds: | 20260507 03:45:17.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:45:17.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:17.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:17.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:17.853 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 03:45:17.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:17.853 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 03:45:17.853 | INFO | PRE_HQ : 59.3%, 1.8, 55.3ms, 0 | 20260507 03:45:17.853 | INFO | BAZWRITER : 61.8%, 3.3, 105.5ms, 0 | 20260507 03:45:17.853 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 03:45:17.853 | INFO | Uploaded 36GiB to GPU in 2.03938s (17.6524GiB/s) | 20260507 03:45:17.853 | INFO | Downloaded 38GiB to GPU in 2.17282s (17.4888GiB/s) | 20260507 03:45:22.989 | INFO | Analyzing chunk frames = [6836736,6837248) | 20260507 03:45:22.989 | INFO | MemInfo: { | 20260507 03:45:22.989 | INFO | "gpu" : " 17 GB", | 20260507 03:45:22.989 | INFO | "huge" : "", | 20260507 03:45:22.989 | INFO | "huge_shared" : " 27 GB", | 20260507 03:45:22.989 | INFO | "malloc" : " 14 GB", | 20260507 03:45:22.989 | INFO | "pinned" : " 29 GB" | 20260507 03:45:22.989 | INFO | } | 20260507 03:45:22.989 | INFO | Perf report for the last 5.1362 seconds: | 20260507 03:45:22.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:45:22.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:22.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:22.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:22.989 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 03:45:22.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:22.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:22.989 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:45:22.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:22.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:22.989 | INFO | Uploaded 36GiB to GPU in 2.00018s (17.9983GiB/s) | 20260507 03:45:22.989 | INFO | Downloaded 34GiB to GPU in 1.93486s (17.5723GiB/s) | 20260507 03:45:28.125 | INFO | Analyzing chunk frames = [6837248,6837760) | 20260507 03:45:28.125 | INFO | MemInfo: { | 20260507 03:45:28.125 | INFO | "gpu" : " 17 GB", | 20260507 03:45:28.126 | INFO | "huge" : "", | 20260507 03:45:28.126 | INFO | "huge_shared" : " 27 GB", | 20260507 03:45:28.126 | INFO | "malloc" : " 14 GB", | 20260507 03:45:28.126 | INFO | "pinned" : " 29 GB" | 20260507 03:45:28.126 | INFO | } | 20260507 03:45:28.126 | INFO | Perf report for the last 5.13661 seconds: | 20260507 03:45:28.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:45:28.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:28.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:28.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:28.126 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 03:45:28.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:28.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:28.126 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:45:28.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:28.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:28.126 | INFO | Uploaded 36GiB to GPU in 2.01202s (17.8924GiB/s) | 20260507 03:45:28.126 | INFO | Downloaded 34GiB to GPU in 1.93858s (17.5387GiB/s) | 20260507 03:45:33.266 | INFO | Analyzing chunk frames = [6837760,6838272) | 20260507 03:45:33.266 | INFO | MemInfo: { | 20260507 03:45:33.266 | INFO | "gpu" : " 17 GB", | 20260507 03:45:33.266 | INFO | "huge" : "", | 20260507 03:45:33.266 | INFO | "huge_shared" : " 27 GB", | 20260507 03:45:33.266 | INFO | "malloc" : " 14 GB", | 20260507 03:45:33.266 | INFO | "pinned" : " 29 GB" | 20260507 03:45:33.266 | INFO | } | 20260507 03:45:33.266 | INFO | Perf report for the last 5.1404 seconds: | 20260507 03:45:33.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:45:33.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:33.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:33.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:33.266 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 03:45:33.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:33.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:33.266 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:45:33.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:33.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:33.266 | INFO | Uploaded 36GiB to GPU in 1.97768s (18.2031GiB/s) | 20260507 03:45:33.266 | INFO | Downloaded 34GiB to GPU in 1.90619s (17.8366GiB/s) | 20260507 03:45:38.408 | INFO | Analyzing chunk frames = [6838272,6838784) | 20260507 03:45:38.408 | INFO | MemInfo: { | 20260507 03:45:38.408 | INFO | "gpu" : " 17 GB", | 20260507 03:45:38.408 | INFO | "huge" : "", | 20260507 03:45:38.408 | INFO | "huge_shared" : " 27 GB", | 20260507 03:45:38.408 | INFO | "malloc" : " 14 GB", | 20260507 03:45:38.408 | INFO | "pinned" : " 29 GB" | 20260507 03:45:38.408 | INFO | } | 20260507 03:45:38.408 | INFO | Perf report for the last 5.14203 seconds: | 20260507 03:45:38.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:45:38.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:38.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:38.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:38.408 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 03:45:38.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:38.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:38.408 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:45:38.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:38.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:38.408 | INFO | Uploaded 36GiB to GPU in 2.02329s (17.7928GiB/s) | 20260507 03:45:38.408 | INFO | Downloaded 34GiB to GPU in 1.94557s (17.4756GiB/s) | 20260507 03:45:43.554 | INFO | Analyzing chunk frames = [6838784,6839296) | 20260507 03:45:43.554 | INFO | MemInfo: { | 20260507 03:45:43.554 | INFO | "gpu" : " 17 GB", | 20260507 03:45:43.554 | INFO | "huge" : "", | 20260507 03:45:43.554 | INFO | "huge_shared" : " 27 GB", | 20260507 03:45:43.554 | INFO | "malloc" : " 14 GB", | 20260507 03:45:43.554 | INFO | "pinned" : " 29 GB" | 20260507 03:45:43.554 | INFO | } | 20260507 03:45:43.554 | INFO | Perf report for the last 5.14609 seconds: | 20260507 03:45:43.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:45:43.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:43.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:43.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:43.554 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:45:43.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:43.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:43.554 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 03:45:43.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:43.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:43.554 | INFO | Uploaded 36GiB to GPU in 2.05833s (17.4899GiB/s) | 20260507 03:45:43.555 | INFO | Downloaded 34GiB to GPU in 1.9648s (17.3046GiB/s) | 20260507 03:45:48.699 | INFO | Analyzing chunk frames = [6839296,6839808) | 20260507 03:45:48.699 | INFO | MemInfo: { | 20260507 03:45:48.699 | INFO | "gpu" : " 17 GB", | 20260507 03:45:48.699 | INFO | "huge" : "", | 20260507 03:45:48.699 | INFO | "huge_shared" : " 27 GB", | 20260507 03:45:48.699 | INFO | "malloc" : " 14 GB", | 20260507 03:45:48.699 | INFO | "pinned" : " 29 GB" | 20260507 03:45:48.699 | INFO | } | 20260507 03:45:48.699 | INFO | Perf report for the last 5.14487 seconds: | 20260507 03:45:48.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:45:48.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:48.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:48.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:48.699 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:45:48.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:48.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:48.699 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 03:45:48.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:48.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:48.699 | INFO | Uploaded 36GiB to GPU in 2.03696s (17.6734GiB/s) | 20260507 03:45:48.699 | INFO | Downloaded 34GiB to GPU in 1.99143s (17.0731GiB/s) | 20260507 03:45:53.827 | INFO | Analyzing chunk frames = [6839808,6840320) | 20260507 03:45:53.827 | INFO | MemInfo: { | 20260507 03:45:53.827 | INFO | "gpu" : " 17 GB", | 20260507 03:45:53.827 | INFO | "huge" : "", | 20260507 03:45:53.827 | INFO | "huge_shared" : " 27 GB", | 20260507 03:45:53.827 | INFO | "malloc" : " 14 GB", | 20260507 03:45:53.827 | INFO | "pinned" : " 29 GB" | 20260507 03:45:53.827 | INFO | } | 20260507 03:45:53.827 | INFO | Perf report for the last 5.12765 seconds: | 20260507 03:45:53.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:45:53.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:53.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:53.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:53.827 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 03:45:53.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:53.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:53.827 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 03:45:53.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:53.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:53.827 | INFO | Uploaded 36GiB to GPU in 1.98228s (18.1609GiB/s) | 20260507 03:45:53.827 | INFO | Downloaded 34GiB to GPU in 1.95571s (17.385GiB/s) | 20260507 03:45:53.827 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:45:53.827 | INFO | Total mem usage: 18 GiB | 20260507 03:45:53.827 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:45:53.827 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:45:53.827 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:45:53.827 | INFO | High Water Breakdown: | 20260507 03:45:53.827 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:45:53.827 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:45:53.827 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:45:53.827 | INFO | ------------------------------------------ | 20260507 03:45:53.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:45:53.827 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:45:53.827 | INFO | Total mem usage: 26.5 GiB | 20260507 03:45:53.827 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:45:53.827 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:45:53.827 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:45:53.827 | INFO | High Water Breakdown: | 20260507 03:45:53.827 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:45:53.827 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:45:53.827 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:45:53.827 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:45:53.827 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:45:53.827 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:45:53.827 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:45:53.827 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:45:53.827 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:45:53.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:45:53.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:45:53.827 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:45:53.827 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:45:53.827 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:45:53.827 | INFO | ------------------------------------------ | 20260507 03:45:53.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:45:53.827 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:45:53.827 | INFO | Total mem usage: 25.6 GiB | 20260507 03:45:53.827 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:45:53.827 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:45:53.827 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:45:53.827 | INFO | High Water Breakdown: | 20260507 03:45:53.827 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:45:53.827 | INFO | ------------------------------------------ | 20260507 03:45:53.827 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:45:53.827 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:45:53.827 | INFO | Total mem usage: 15.7 GiB | 20260507 03:45:53.827 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:45:53.827 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:45:53.827 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:45:53.827 | INFO | High Water Breakdown: | 20260507 03:45:53.827 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:45:53.827 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:45:53.828 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:45:53.828 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:45:53.828 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:45:53.828 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:45:53.828 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:45:53.828 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:45:53.828 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:45:53.828 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:45:53.828 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:45:53.828 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:45:53.828 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:45:53.828 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:45:53.828 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:45:53.828 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:45:53.828 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:45:53.828 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:45:53.828 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:45:53.828 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:45:53.828 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:45:53.828 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:45:53.828 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:45:53.828 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:45:53.828 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:45:53.828 | INFO | ------------------------------------------ | 20260507 03:45:53.828 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:45:53.828 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:45:53.828 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:45:53.828 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:45:53.828 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.50e+01, 1.33e+01 / 1.84e+00 | 20260507 03:45:53.828 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.92e-03 / 7.40e-03 | 20260507 03:45:53.828 | INFO | PerfReport: 11.64%, Baseline -- 3.58e+00 / 4.38e+00, 5.12e-01 / 3.12e-01 | 20260507 03:45:53.828 | INFO | PerfReport: 3.23%, Histogram -- 9.95e-01 / 1.46e+00, 1.71e-01 / 1.27e-01 | 20260507 03:45:53.828 | INFO | PerfReport: 16.12%, DME -- 3.97e+01 / 4.44e+01, 3.93e+00 / 1.94e+00 | 20260507 03:45:53.828 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.35e+00 / 1.51e+00 | 20260507 03:45:53.828 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.70e+00, 3.79e-01 / 2.03e-01 | 20260507 03:45:53.828 | INFO | PerfReport: 3.04%, Metrics -- 9.35e-01 / 1.27e+00, 2.48e-01 / 1.63e-01 | 20260507 03:45:53.828 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.16e-01 / 3.65e-01, 5.01e-02 / 1.71e-02 | 20260507 03:45:53.828 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:45:53.828 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:45:53.828 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:45:53.828 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 03:45:53.828 | INFO | PerfReport: 21.75%, Upload -- 2.09e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260507 03:45:53.828 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 03:45:53.828 | INFO | PerfReport: 31.98%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 03:45:53.828 | INFO | PerfReport: 1.43%, LockUpload -- 1.37e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 03:45:53.828 | INFO | PerfReport: 1.27%, LockDownload -- 1.22e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260507 03:45:53.828 | INFO | PerfReport: 22.21%, LockCompute -- 2.14e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 03:45:53.828 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260507 03:45:53.924 | INFO | Starting a round of Baz IO | 20260507 03:45:57.032 | INFO | Populating RTMetrics report | 20260507 03:45:57.033 | INFO | RTMetrics report populated | 20260507 03:45:57.033 | INFO | Opening tmp RTMetrics json file | 20260507 03:45:57.052 | INFO | Serializing RTMetrics to json | 20260507 03:45:57.053 | INFO | Serialization done, swapping tmp file now... | 20260507 03:45:57.054 | INFO | RTMetrics done | 20260507 03:45:57.122 | INFO | Finished a round of Baz IO: Wrote 1.744 GiB in 3.197 seconds | 20260507 03:45:57.122 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.86% / 0.00% / 67.14% | 20260507 03:45:57.122 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.191184% / 3.97364e-06% / 5.5631e-05% | 20260507 03:45:57.122 | INFO | Baz IO overall: | 20260507 03:45:57.122 | INFO | BytesWritten : 3504880129947 | 20260507 03:45:57.122 | INFO | NumEvents : 1683641015225 | 20260507 03:45:58.968 | INFO | Analyzing chunk frames = [6840320,6840832) | 20260507 03:45:58.968 | INFO | MemInfo: { | 20260507 03:45:58.968 | INFO | "gpu" : " 17 GB", | 20260507 03:45:58.968 | INFO | "huge" : "", | 20260507 03:45:58.968 | INFO | "huge_shared" : " 27 GB", | 20260507 03:45:58.968 | INFO | "malloc" : " 14 GB", | 20260507 03:45:58.968 | INFO | "pinned" : " 29 GB" | 20260507 03:45:58.968 | INFO | } | 20260507 03:45:58.968 | INFO | Perf report for the last 5.14104 seconds: | 20260507 03:45:58.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:45:58.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:58.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:58.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:58.968 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 03:45:58.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:45:58.968 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 03:45:58.968 | INFO | PRE_HQ : 59.2%, 1.8, 55.1ms, 0 | 20260507 03:45:58.968 | INFO | BAZWRITER : 61.8%, 3.2, 102.0ms, 0 | 20260507 03:45:58.968 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:45:58.968 | INFO | Uploaded 36GiB to GPU in 2.04844s (17.5744GiB/s) | 20260507 03:45:58.968 | INFO | Downloaded 38GiB to GPU in 2.17704s (17.4549GiB/s) | 20260507 03:46:04.104 | INFO | Analyzing chunk frames = [6840832,6841344) | 20260507 03:46:04.104 | INFO | MemInfo: { | 20260507 03:46:04.104 | INFO | "gpu" : " 17 GB", | 20260507 03:46:04.104 | INFO | "huge" : "", | 20260507 03:46:04.104 | INFO | "huge_shared" : " 27 GB", | 20260507 03:46:04.104 | INFO | "malloc" : " 14 GB", | 20260507 03:46:04.104 | INFO | "pinned" : " 29 GB" | 20260507 03:46:04.104 | INFO | } | 20260507 03:46:04.104 | INFO | Perf report for the last 5.13621 seconds: | 20260507 03:46:04.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:46:04.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:04.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:04.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:04.104 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:46:04.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:04.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:04.104 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:46:04.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:04.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:04.104 | INFO | Uploaded 36GiB to GPU in 2.0356s (17.6852GiB/s) | 20260507 03:46:04.104 | INFO | Downloaded 34GiB to GPU in 1.94089s (17.5177GiB/s) | 20260507 03:46:09.244 | INFO | Analyzing chunk frames = [6841344,6841856) | 20260507 03:46:09.244 | INFO | MemInfo: { | 20260507 03:46:09.244 | INFO | "gpu" : " 17 GB", | 20260507 03:46:09.244 | INFO | "huge" : "", | 20260507 03:46:09.244 | INFO | "huge_shared" : " 27 GB", | 20260507 03:46:09.244 | INFO | "malloc" : " 14 GB", | 20260507 03:46:09.244 | INFO | "pinned" : " 29 GB" | 20260507 03:46:09.244 | INFO | } | 20260507 03:46:09.244 | INFO | Perf report for the last 5.13983 seconds: | 20260507 03:46:09.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:46:09.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:09.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:09.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:09.244 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 03:46:09.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:09.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:09.244 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:46:09.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:09.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:09.244 | INFO | Uploaded 36GiB to GPU in 2.00983s (17.912GiB/s) | 20260507 03:46:09.244 | INFO | Downloaded 34GiB to GPU in 1.93866s (17.5379GiB/s) | 20260507 03:46:14.385 | INFO | Analyzing chunk frames = [6841856,6842368) | 20260507 03:46:14.385 | INFO | MemInfo: { | 20260507 03:46:14.385 | INFO | "gpu" : " 17 GB", | 20260507 03:46:14.387 | INFO | "huge" : "", | 20260507 03:46:14.387 | INFO | "huge_shared" : " 27 GB", | 20260507 03:46:14.387 | INFO | "malloc" : " 14 GB", | 20260507 03:46:14.387 | INFO | "pinned" : " 29 GB" | 20260507 03:46:14.387 | INFO | } | 20260507 03:46:14.387 | INFO | Perf report for the last 5.14279 seconds: | 20260507 03:46:14.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:46:14.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:14.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:14.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:14.387 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:46:14.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:14.387 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:14.387 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:46:14.387 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:14.387 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:14.387 | INFO | Uploaded 36GiB to GPU in 2.01317s (17.8822GiB/s) | 20260507 03:46:14.387 | INFO | Downloaded 34GiB to GPU in 1.93311s (17.5882GiB/s) | 20260507 03:46:19.528 | INFO | Analyzing chunk frames = [6842368,6842880) | 20260507 03:46:19.528 | INFO | MemInfo: { | 20260507 03:46:19.528 | INFO | "gpu" : " 17 GB", | 20260507 03:46:19.528 | INFO | "huge" : "", | 20260507 03:46:19.528 | INFO | "huge_shared" : " 27 GB", | 20260507 03:46:19.528 | INFO | "malloc" : " 14 GB", | 20260507 03:46:19.528 | INFO | "pinned" : " 29 GB" | 20260507 03:46:19.529 | INFO | } | 20260507 03:46:19.529 | INFO | Perf report for the last 5.14186 seconds: | 20260507 03:46:19.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:46:19.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:19.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:19.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:19.529 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 03:46:19.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:19.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:19.529 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:46:19.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:19.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:19.529 | INFO | Uploaded 36GiB to GPU in 2.03538s (17.6871GiB/s) | 20260507 03:46:19.529 | INFO | Downloaded 34GiB to GPU in 1.93892s (17.5355GiB/s) | 20260507 03:46:24.665 | INFO | Analyzing chunk frames = [6842880,6843392) | 20260507 03:46:24.665 | INFO | MemInfo: { | 20260507 03:46:24.665 | INFO | "gpu" : " 17 GB", | 20260507 03:46:24.665 | INFO | "huge" : "", | 20260507 03:46:24.665 | INFO | "huge_shared" : " 27 GB", | 20260507 03:46:24.665 | INFO | "malloc" : " 14 GB", | 20260507 03:46:24.665 | INFO | "pinned" : " 29 GB" | 20260507 03:46:24.665 | INFO | } | 20260507 03:46:24.665 | INFO | Perf report for the last 5.13612 seconds: | 20260507 03:46:24.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:46:24.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:24.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:24.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:24.665 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 03:46:24.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:24.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:24.665 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 03:46:24.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:24.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:24.665 | INFO | Uploaded 36GiB to GPU in 2.02977s (17.736GiB/s) | 20260507 03:46:24.665 | INFO | Downloaded 34GiB to GPU in 1.918s (17.7268GiB/s) | 20260507 03:46:29.801 | INFO | Analyzing chunk frames = [6843392,6843904) | 20260507 03:46:29.801 | INFO | MemInfo: { | 20260507 03:46:29.801 | INFO | "gpu" : " 17 GB", | 20260507 03:46:29.801 | INFO | "huge" : "", | 20260507 03:46:29.802 | INFO | "huge_shared" : " 27 GB", | 20260507 03:46:29.802 | INFO | "malloc" : " 14 GB", | 20260507 03:46:29.802 | INFO | "pinned" : " 29 GB" | 20260507 03:46:29.802 | INFO | } | 20260507 03:46:29.802 | INFO | Perf report for the last 5.13693 seconds: | 20260507 03:46:29.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:46:29.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:29.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:29.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:29.802 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 03:46:29.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:29.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:29.802 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 03:46:29.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:29.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:29.802 | INFO | Uploaded 36GiB to GPU in 1.96707s (18.3014GiB/s) | 20260507 03:46:29.802 | INFO | Downloaded 34GiB to GPU in 1.97092s (17.2508GiB/s) | 20260507 03:46:34.936 | INFO | Analyzing chunk frames = [6843904,6844416) | 20260507 03:46:34.936 | INFO | MemInfo: { | 20260507 03:46:34.936 | INFO | "gpu" : " 17 GB", | 20260507 03:46:34.936 | INFO | "huge" : "", | 20260507 03:46:34.936 | INFO | "huge_shared" : " 27 GB", | 20260507 03:46:34.936 | INFO | "malloc" : " 14 GB", | 20260507 03:46:34.936 | INFO | "pinned" : " 29 GB" | 20260507 03:46:34.936 | INFO | } | 20260507 03:46:34.936 | INFO | Perf report for the last 5.13484 seconds: | 20260507 03:46:34.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:46:34.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:34.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:34.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:34.937 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:46:34.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:34.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:34.937 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 03:46:34.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:34.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:34.937 | INFO | Uploaded 36GiB to GPU in 1.99206s (18.0718GiB/s) | 20260507 03:46:34.937 | INFO | Downloaded 34GiB to GPU in 1.96424s (17.3095GiB/s) | 20260507 03:46:35.033 | INFO | Starting a round of Baz IO | 20260507 03:46:38.172 | INFO | Populating RTMetrics report | 20260507 03:46:38.173 | INFO | RTMetrics report populated | 20260507 03:46:38.173 | INFO | Opening tmp RTMetrics json file | 20260507 03:46:38.173 | INFO | Serializing RTMetrics to json | 20260507 03:46:38.175 | INFO | Serialization done, swapping tmp file now... | 20260507 03:46:38.175 | INFO | RTMetrics done | 20260507 03:46:38.264 | INFO | Finished a round of Baz IO: Wrote 1.743 GiB in 3.23 seconds | 20260507 03:46:38.264 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.86% / 0.00% / 67.14% | 20260507 03:46:38.264 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.189781% / 3.97364e-06% / 6.75519e-05% | 20260507 03:46:38.264 | INFO | Baz IO overall: | 20260507 03:46:38.264 | INFO | BytesWritten : 3506751743498 | 20260507 03:46:38.264 | INFO | NumEvents : 1684522960775 | 20260507 03:46:40.073 | INFO | Analyzing chunk frames = [6844416,6844928) | 20260507 03:46:40.073 | INFO | MemInfo: { | 20260507 03:46:40.073 | INFO | "gpu" : " 17 GB", | 20260507 03:46:40.073 | INFO | "huge" : "", | 20260507 03:46:40.073 | INFO | "huge_shared" : " 27 GB", | 20260507 03:46:40.073 | INFO | "malloc" : " 14 GB", | 20260507 03:46:40.073 | INFO | "pinned" : " 29 GB" | 20260507 03:46:40.073 | INFO | } | 20260507 03:46:40.073 | INFO | Perf report for the last 5.13663 seconds: | 20260507 03:46:40.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:46:40.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:40.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:40.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:40.073 | INFO | ANALYSIS : 62.0%, 3.0, 95.9ms, 0 | 20260507 03:46:40.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:40.073 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260507 03:46:40.073 | INFO | PRE_HQ : 60.5%, 2.0, 61.6ms, 0 | 20260507 03:46:40.073 | INFO | BAZWRITER : 62.5%, 3.4, 110.3ms, 0 | 20260507 03:46:40.073 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 03:46:40.073 | INFO | Uploaded 36GiB to GPU in 2.04264s (17.6243GiB/s) | 20260507 03:46:40.073 | INFO | Downloaded 38GiB to GPU in 2.17931s (17.4367GiB/s) | 20260507 03:46:45.217 | INFO | Analyzing chunk frames = [6844928,6845440) | 20260507 03:46:45.217 | INFO | MemInfo: { | 20260507 03:46:45.217 | INFO | "gpu" : " 17 GB", | 20260507 03:46:45.217 | INFO | "huge" : "", | 20260507 03:46:45.217 | INFO | "huge_shared" : " 27 GB", | 20260507 03:46:45.217 | INFO | "malloc" : " 14 GB", | 20260507 03:46:45.217 | INFO | "pinned" : " 29 GB" | 20260507 03:46:45.217 | INFO | } | 20260507 03:46:45.217 | INFO | Perf report for the last 5.1439 seconds: | 20260507 03:46:45.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:46:45.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:45.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:45.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:45.217 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 03:46:45.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:45.217 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:45.217 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 03:46:45.217 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:45.217 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:45.217 | INFO | Uploaded 36GiB to GPU in 1.9999s (18.0009GiB/s) | 20260507 03:46:45.217 | INFO | Downloaded 34GiB to GPU in 1.93019s (17.6149GiB/s) | 20260507 03:46:50.368 | INFO | Analyzing chunk frames = [6845440,6845952) | 20260507 03:46:50.368 | INFO | MemInfo: { | 20260507 03:46:50.368 | INFO | "gpu" : " 17 GB", | 20260507 03:46:50.368 | INFO | "huge" : "", | 20260507 03:46:50.368 | INFO | "huge_shared" : " 27 GB", | 20260507 03:46:50.368 | INFO | "malloc" : " 14 GB", | 20260507 03:46:50.368 | INFO | "pinned" : " 29 GB" | 20260507 03:46:50.368 | INFO | } | 20260507 03:46:50.368 | INFO | Perf report for the last 5.15081 seconds: | 20260507 03:46:50.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:46:50.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:50.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:50.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:50.368 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 03:46:50.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:50.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:50.368 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:46:50.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:50.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:50.368 | INFO | Uploaded 36GiB to GPU in 1.9987s (18.0117GiB/s) | 20260507 03:46:50.368 | INFO | Downloaded 34GiB to GPU in 1.93551s (17.5664GiB/s) | 20260507 03:46:55.508 | INFO | Analyzing chunk frames = [6845952,6846464) | 20260507 03:46:55.508 | INFO | MemInfo: { | 20260507 03:46:55.508 | INFO | "gpu" : " 17 GB", | 20260507 03:46:55.508 | INFO | "huge" : "", | 20260507 03:46:55.508 | INFO | "huge_shared" : " 27 GB", | 20260507 03:46:55.508 | INFO | "malloc" : " 14 GB", | 20260507 03:46:55.508 | INFO | "pinned" : " 29 GB" | 20260507 03:46:55.508 | INFO | } | 20260507 03:46:55.508 | INFO | Perf report for the last 5.14058 seconds: | 20260507 03:46:55.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:46:55.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:55.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:55.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:55.509 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 03:46:55.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:55.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:55.509 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:46:55.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:55.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:46:55.509 | INFO | Uploaded 36GiB to GPU in 2.01057s (17.9054GiB/s) | 20260507 03:46:55.509 | INFO | Downloaded 34GiB to GPU in 1.94036s (17.5225GiB/s) | 20260507 03:47:00.642 | INFO | Analyzing chunk frames = [6846464,6846976) | 20260507 03:47:00.642 | INFO | MemInfo: { | 20260507 03:47:00.642 | INFO | "gpu" : " 17 GB", | 20260507 03:47:00.642 | INFO | "huge" : "", | 20260507 03:47:00.642 | INFO | "huge_shared" : " 27 GB", | 20260507 03:47:00.642 | INFO | "malloc" : " 14 GB", | 20260507 03:47:00.642 | INFO | "pinned" : " 29 GB" | 20260507 03:47:00.642 | INFO | } | 20260507 03:47:00.642 | INFO | Perf report for the last 5.13352 seconds: | 20260507 03:47:00.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:47:00.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:00.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:00.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:00.642 | INFO | ANALYSIS : 60.3%, 3.0, 93.7ms, 0 | 20260507 03:47:00.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:00.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:00.642 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:47:00.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:00.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:00.642 | INFO | Uploaded 36GiB to GPU in 2.04735s (17.5837GiB/s) | 20260507 03:47:00.642 | INFO | Downloaded 34GiB to GPU in 1.91206s (17.7819GiB/s) | 20260507 03:47:05.777 | INFO | Analyzing chunk frames = [6846976,6847488) | 20260507 03:47:05.777 | INFO | MemInfo: { | 20260507 03:47:05.777 | INFO | "gpu" : " 17 GB", | 20260507 03:47:05.777 | INFO | "huge" : "", | 20260507 03:47:05.777 | INFO | "huge_shared" : " 27 GB", | 20260507 03:47:05.777 | INFO | "malloc" : " 14 GB", | 20260507 03:47:05.777 | INFO | "pinned" : " 29 GB" | 20260507 03:47:05.777 | INFO | } | 20260507 03:47:05.777 | INFO | Perf report for the last 5.13525 seconds: | 20260507 03:47:05.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:47:05.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:05.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:05.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:05.777 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 03:47:05.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:05.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:05.777 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:47:05.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:05.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:05.777 | INFO | Uploaded 36GiB to GPU in 2.0421s (17.6289GiB/s) | 20260507 03:47:05.777 | INFO | Downloaded 34GiB to GPU in 1.91021s (17.7991GiB/s) | 20260507 03:47:10.916 | INFO | Analyzing chunk frames = [6847488,6848000) | 20260507 03:47:10.916 | INFO | MemInfo: { | 20260507 03:47:10.916 | INFO | "gpu" : " 17 GB", | 20260507 03:47:10.916 | INFO | "huge" : "", | 20260507 03:47:10.917 | INFO | "huge_shared" : " 27 GB", | 20260507 03:47:10.917 | INFO | "malloc" : " 14 GB", | 20260507 03:47:10.917 | INFO | "pinned" : " 29 GB" | 20260507 03:47:10.917 | INFO | } | 20260507 03:47:10.917 | INFO | Perf report for the last 5.13936 seconds: | 20260507 03:47:10.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:47:10.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:10.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:10.917 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:10.917 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:47:10.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:10.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:10.917 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 03:47:10.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:10.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:10.917 | INFO | Uploaded 36GiB to GPU in 2.01939s (17.8271GiB/s) | 20260507 03:47:10.917 | INFO | Downloaded 34GiB to GPU in 1.98152s (17.1585GiB/s) | 20260507 03:47:16.055 | INFO | Analyzing chunk frames = [6848000,6848512) | 20260507 03:47:16.055 | INFO | MemInfo: { | 20260507 03:47:16.055 | INFO | "gpu" : " 17 GB", | 20260507 03:47:16.055 | INFO | "huge" : "", | 20260507 03:47:16.055 | INFO | "huge_shared" : " 27 GB", | 20260507 03:47:16.055 | INFO | "malloc" : " 14 GB", | 20260507 03:47:16.055 | INFO | "pinned" : " 29 GB" | 20260507 03:47:16.055 | INFO | } | 20260507 03:47:16.055 | INFO | Perf report for the last 5.13889 seconds: | 20260507 03:47:16.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:47:16.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:16.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:16.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:16.056 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:47:16.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:16.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:16.056 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:47:16.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:16.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:16.056 | INFO | Uploaded 36GiB to GPU in 1.99945s (18.005GiB/s) | 20260507 03:47:16.056 | INFO | Downloaded 34GiB to GPU in 1.97978s (17.1737GiB/s) | 20260507 03:47:16.056 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:47:16.056 | INFO | Total mem usage: 18 GiB | 20260507 03:47:16.056 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:47:16.056 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.6%) | 20260507 03:47:16.056 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:47:16.056 | INFO | High Water Breakdown: | 20260507 03:47:16.056 | INFO | 60.2% : 8.63 GiB ( 75.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:47:16.056 | INFO | 39.6% : 5.68 GiB ( 84.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:47:16.056 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:47:16.056 | INFO | ------------------------------------------ | 20260507 03:47:16.056 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:47:16.056 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:47:16.056 | INFO | Total mem usage: 26.5 GiB | 20260507 03:47:16.056 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:47:16.056 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:47:16.056 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:47:16.056 | INFO | High Water Breakdown: | 20260507 03:47:16.056 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:47:16.056 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:47:16.056 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:47:16.056 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:47:16.056 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:47:16.056 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:47:16.056 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:47:16.056 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:47:16.056 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:47:16.056 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:47:16.056 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:47:16.056 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:47:16.056 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:47:16.056 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:47:16.056 | INFO | ------------------------------------------ | 20260507 03:47:16.056 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:47:16.056 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:47:16.056 | INFO | Total mem usage: 25.6 GiB | 20260507 03:47:16.056 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:47:16.056 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 03:47:16.056 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 03:47:16.056 | INFO | High Water Breakdown: | 20260507 03:47:16.056 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 03:47:16.056 | INFO | ------------------------------------------ | 20260507 03:47:16.056 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:47:16.056 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:47:16.056 | INFO | Total mem usage: 15.7 GiB | 20260507 03:47:16.056 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:47:16.056 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:47:16.056 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:47:16.056 | INFO | High Water Breakdown: | 20260507 03:47:16.056 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:47:16.056 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:47:16.056 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:47:16.056 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:47:16.056 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:47:16.056 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:47:16.056 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:47:16.056 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:47:16.056 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:47:16.056 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:47:16.056 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:47:16.056 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:47:16.056 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:47:16.056 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:47:16.056 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:47:16.056 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:47:16.056 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:47:16.056 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:47:16.056 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:47:16.056 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:47:16.056 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:47:16.056 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:47:16.056 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:47:16.056 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:47:16.056 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:47:16.056 | INFO | ------------------------------------------ | 20260507 03:47:16.056 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:47:16.057 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:47:16.057 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:47:16.057 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:47:16.057 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.46e+01, 1.34e+01 / 9.05e-01 | 20260507 03:47:16.057 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.02e-01, 9.72e-03 / 7.52e-03 | 20260507 03:47:16.057 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.41e+00, 5.09e-01 / 3.16e-01 | 20260507 03:47:16.057 | INFO | PerfReport: 3.23%, Histogram -- 9.94e-01 / 1.45e+00, 1.69e-01 / 1.22e-01 | 20260507 03:47:16.057 | INFO | PerfReport: 16.03%, DME -- 3.95e+01 / 4.37e+01, 3.64e+00 / 1.42e+00 | 20260507 03:47:16.057 | INFO | PerfReport: 57.80%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.43e+00 / 1.79e+00 | 20260507 03:47:16.057 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.66e+00, 3.75e-01 / 2.50e-01 | 20260507 03:47:16.057 | INFO | PerfReport: 3.03%, Metrics -- 9.31e-01 / 1.25e+00, 2.49e-01 / 1.34e-01 | 20260507 03:47:16.057 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.16e-01 / 3.72e-01, 5.03e-02 / 2.21e-02 | 20260507 03:47:16.057 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:47:16.057 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:47:16.057 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:47:16.057 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 03:47:16.057 | INFO | PerfReport: 21.84%, Upload -- 2.10e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260507 03:47:16.057 | INFO | PerfReport: 21.34%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260507 03:47:16.057 | INFO | PerfReport: 31.97%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 03:47:16.057 | INFO | PerfReport: 1.51%, LockUpload -- 1.45e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 03:47:16.057 | INFO | PerfReport: 1.23%, LockDownload -- 1.19e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260507 03:47:16.057 | INFO | PerfReport: 22.10%, LockCompute -- 2.13e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 03:47:16.057 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260507 03:47:16.152 | INFO | Starting a round of Baz IO | 20260507 03:47:19.258 | INFO | Populating RTMetrics report | 20260507 03:47:19.258 | INFO | RTMetrics report populated | 20260507 03:47:19.258 | INFO | Opening tmp RTMetrics json file | 20260507 03:47:19.276 | INFO | Serializing RTMetrics to json | 20260507 03:47:19.278 | INFO | Serialization done, swapping tmp file now... | 20260507 03:47:19.278 | INFO | RTMetrics done | 20260507 03:47:19.345 | INFO | Finished a round of Baz IO: Wrote 1.742 GiB in 3.192 seconds | 20260507 03:47:19.345 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.86% / 0.00% / 67.14% | 20260507 03:47:19.345 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.189396% / 3.97364e-06% / 7.15256e-05% | 20260507 03:47:19.345 | INFO | Baz IO overall: | 20260507 03:47:19.345 | INFO | BytesWritten : 3508621722814 | 20260507 03:47:19.345 | INFO | NumEvents : 1685403863941 | 20260507 03:47:21.192 | INFO | Analyzing chunk frames = [6848512,6849024) | 20260507 03:47:21.192 | INFO | MemInfo: { | 20260507 03:47:21.192 | INFO | "gpu" : " 17 GB", | 20260507 03:47:21.192 | INFO | "huge" : "", | 20260507 03:47:21.192 | INFO | "huge_shared" : " 27 GB", | 20260507 03:47:21.192 | INFO | "malloc" : " 14 GB", | 20260507 03:47:21.192 | INFO | "pinned" : " 29 GB" | 20260507 03:47:21.192 | INFO | } | 20260507 03:47:21.193 | INFO | Perf report for the last 5.13703 seconds: | 20260507 03:47:21.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:47:21.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:21.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:21.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:21.193 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 03:47:21.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:21.193 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 03:47:21.193 | INFO | PRE_HQ : 59.1%, 1.8, 55.1ms, 0 | 20260507 03:47:21.193 | INFO | BAZWRITER : 61.7%, 3.3, 103.1ms, 0 | 20260507 03:47:21.193 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:47:21.193 | INFO | Uploaded 36GiB to GPU in 2.04418s (17.611GiB/s) | 20260507 03:47:21.193 | INFO | Downloaded 38GiB to GPU in 2.18448s (17.3954GiB/s) | 20260507 03:47:26.337 | INFO | Analyzing chunk frames = [6849024,6849536) | 20260507 03:47:26.337 | INFO | MemInfo: { | 20260507 03:47:26.337 | INFO | "gpu" : " 17 GB", | 20260507 03:47:26.337 | INFO | "huge" : "", | 20260507 03:47:26.337 | INFO | "huge_shared" : " 27 GB", | 20260507 03:47:26.337 | INFO | "malloc" : " 14 GB", | 20260507 03:47:26.337 | INFO | "pinned" : " 29 GB" | 20260507 03:47:26.337 | INFO | } | 20260507 03:47:26.337 | INFO | Perf report for the last 5.14424 seconds: | 20260507 03:47:26.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:47:26.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:26.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:26.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:26.337 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:47:26.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:26.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:26.337 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:47:26.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:26.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:26.337 | INFO | Uploaded 36GiB to GPU in 2.03533s (17.6875GiB/s) | 20260507 03:47:26.337 | INFO | Downloaded 34GiB to GPU in 1.94039s (17.5222GiB/s) | 20260507 03:47:31.482 | INFO | Analyzing chunk frames = [6849536,6850048) | 20260507 03:47:31.482 | INFO | MemInfo: { | 20260507 03:47:31.482 | INFO | "gpu" : " 17 GB", | 20260507 03:47:31.482 | INFO | "huge" : "", | 20260507 03:47:31.482 | INFO | "huge_shared" : " 27 GB", | 20260507 03:47:31.482 | INFO | "malloc" : " 14 GB", | 20260507 03:47:31.482 | INFO | "pinned" : " 29 GB" | 20260507 03:47:31.482 | INFO | } | 20260507 03:47:31.482 | INFO | Perf report for the last 5.14499 seconds: | 20260507 03:47:31.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:47:31.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:31.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:31.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:31.482 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 03:47:31.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:31.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:31.482 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:47:31.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:31.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:31.482 | INFO | Uploaded 36GiB to GPU in 1.99091s (18.0822GiB/s) | 20260507 03:47:31.482 | INFO | Downloaded 34GiB to GPU in 1.94241s (17.504GiB/s) | 20260507 03:47:36.620 | INFO | Analyzing chunk frames = [6850048,6850560) | 20260507 03:47:36.620 | INFO | MemInfo: { | 20260507 03:47:36.620 | INFO | "gpu" : " 17 GB", | 20260507 03:47:36.620 | INFO | "huge" : "", | 20260507 03:47:36.620 | INFO | "huge_shared" : " 27 GB", | 20260507 03:47:36.620 | INFO | "malloc" : " 14 GB", | 20260507 03:47:36.620 | INFO | "pinned" : " 29 GB" | 20260507 03:47:36.620 | INFO | } | 20260507 03:47:36.620 | INFO | Perf report for the last 5.13791 seconds: | 20260507 03:47:36.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:47:36.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:36.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:36.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:36.620 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:47:36.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:36.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:36.620 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:47:36.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:36.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:36.620 | INFO | Uploaded 36GiB to GPU in 1.99452s (18.0494GiB/s) | 20260507 03:47:36.620 | INFO | Downloaded 34GiB to GPU in 1.93509s (17.5703GiB/s) | 20260507 03:47:41.749 | INFO | Analyzing chunk frames = [6850560,6851072) | 20260507 03:47:41.749 | INFO | MemInfo: { | 20260507 03:47:41.749 | INFO | "gpu" : " 17 GB", | 20260507 03:47:41.749 | INFO | "huge" : "", | 20260507 03:47:41.749 | INFO | "huge_shared" : " 27 GB", | 20260507 03:47:41.749 | INFO | "malloc" : " 14 GB", | 20260507 03:47:41.749 | INFO | "pinned" : " 29 GB" | 20260507 03:47:41.749 | INFO | } | 20260507 03:47:41.749 | INFO | Perf report for the last 5.1293 seconds: | 20260507 03:47:41.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:47:41.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:41.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:41.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:41.749 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260507 03:47:41.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:41.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:41.749 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:47:41.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:41.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:41.749 | INFO | Uploaded 36GiB to GPU in 2.04905s (17.5691GiB/s) | 20260507 03:47:41.749 | INFO | Downloaded 34GiB to GPU in 1.95023s (17.4339GiB/s) | 20260507 03:47:46.886 | INFO | Analyzing chunk frames = [6851072,6851584) | 20260507 03:47:46.886 | INFO | MemInfo: { | 20260507 03:47:46.886 | INFO | "gpu" : " 17 GB", | 20260507 03:47:46.886 | INFO | "huge" : "", | 20260507 03:47:46.886 | INFO | "huge_shared" : " 27 GB", | 20260507 03:47:46.886 | INFO | "malloc" : " 14 GB", | 20260507 03:47:46.886 | INFO | "pinned" : " 29 GB" | 20260507 03:47:46.886 | INFO | } | 20260507 03:47:46.886 | INFO | Perf report for the last 5.13683 seconds: | 20260507 03:47:46.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:47:46.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:46.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:46.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:46.886 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 03:47:46.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:46.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:46.886 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 03:47:46.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:46.886 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:46.886 | INFO | Uploaded 36GiB to GPU in 2.03107s (17.7247GiB/s) | 20260507 03:47:46.886 | INFO | Downloaded 34GiB to GPU in 1.94024s (17.5236GiB/s) | 20260507 03:47:52.029 | INFO | Analyzing chunk frames = [6851584,6852096) | 20260507 03:47:52.029 | INFO | MemInfo: { | 20260507 03:47:52.029 | INFO | "gpu" : " 17 GB", | 20260507 03:47:52.029 | INFO | "huge" : "", | 20260507 03:47:52.029 | INFO | "huge_shared" : " 27 GB", | 20260507 03:47:52.029 | INFO | "malloc" : " 14 GB", | 20260507 03:47:52.029 | INFO | "pinned" : " 29 GB" | 20260507 03:47:52.029 | INFO | } | 20260507 03:47:52.030 | INFO | Perf report for the last 5.14366 seconds: | 20260507 03:47:52.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:47:52.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:52.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:52.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:52.030 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:47:52.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:52.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:52.030 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 03:47:52.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:52.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:52.030 | INFO | Uploaded 36GiB to GPU in 2.00563s (17.9495GiB/s) | 20260507 03:47:52.030 | INFO | Downloaded 34GiB to GPU in 1.97915s (17.1791GiB/s) | 20260507 03:47:57.169 | INFO | Analyzing chunk frames = [6852096,6852608) | 20260507 03:47:57.169 | INFO | MemInfo: { | 20260507 03:47:57.169 | INFO | "gpu" : " 17 GB", | 20260507 03:47:57.169 | INFO | "huge" : "", | 20260507 03:47:57.169 | INFO | "huge_shared" : " 27 GB", | 20260507 03:47:57.169 | INFO | "malloc" : " 14 GB", | 20260507 03:47:57.169 | INFO | "pinned" : " 29 GB" | 20260507 03:47:57.169 | INFO | } | 20260507 03:47:57.169 | INFO | Perf report for the last 5.13912 seconds: | 20260507 03:47:57.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:47:57.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:57.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:57.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:57.169 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:47:57.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:57.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:57.169 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 03:47:57.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:57.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:47:57.169 | INFO | Uploaded 36GiB to GPU in 1.97551s (18.2232GiB/s) | 20260507 03:47:57.169 | INFO | Downloaded 34GiB to GPU in 1.97381s (17.2256GiB/s) | 20260507 03:47:57.266 | INFO | Starting a round of Baz IO | 20260507 03:48:00.422 | INFO | Populating RTMetrics report | 20260507 03:48:00.422 | INFO | RTMetrics report populated | 20260507 03:48:00.422 | INFO | Opening tmp RTMetrics json file | 20260507 03:48:00.430 | INFO | Serializing RTMetrics to json | 20260507 03:48:00.445 | INFO | Serialization done, swapping tmp file now... | 20260507 03:48:00.446 | INFO | RTMetrics done | 20260507 03:48:00.503 | INFO | Finished a round of Baz IO: Wrote 1.742 GiB in 3.237 seconds | 20260507 03:48:00.503 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.86% / 0.00% / 67.14% | 20260507 03:48:00.503 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.188144% / 7.94729e-06% / 4.76837e-05% | 20260507 03:48:00.504 | INFO | Baz IO overall: | 20260507 03:48:00.504 | INFO | BytesWritten : 3510491695511 | 20260507 03:48:00.504 | INFO | NumEvents : 1686284897036 | 20260507 03:48:02.318 | INFO | Analyzing chunk frames = [6852608,6853120) | 20260507 03:48:02.318 | INFO | MemInfo: { | 20260507 03:48:02.318 | INFO | "gpu" : " 17 GB", | 20260507 03:48:02.318 | INFO | "huge" : "", | 20260507 03:48:02.318 | INFO | "huge_shared" : " 27 GB", | 20260507 03:48:02.318 | INFO | "malloc" : " 14 GB", | 20260507 03:48:02.318 | INFO | "pinned" : " 29 GB" | 20260507 03:48:02.318 | INFO | } | 20260507 03:48:02.318 | INFO | Perf report for the last 5.14978 seconds: | 20260507 03:48:02.318 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:48:02.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:02.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:02.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:02.319 | INFO | ANALYSIS : 62.1%, 2.9, 95.4ms, 0 | 20260507 03:48:02.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:02.319 | INFO | RT_METRICS : 44.8%, 1.0, 23.5ms, 0 | 20260507 03:48:02.319 | INFO | PRE_HQ : 60.1%, 1.9, 59.6ms, 0 | 20260507 03:48:02.319 | INFO | BAZWRITER : 62.5%, 3.7, 118.8ms, 0 | 20260507 03:48:02.319 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 03:48:02.319 | INFO | Uploaded 36GiB to GPU in 2.07127s (17.3806GiB/s) | 20260507 03:48:02.319 | INFO | Downloaded 38GiB to GPU in 2.14996s (17.6747GiB/s) | 20260507 03:48:07.458 | INFO | Analyzing chunk frames = [6853120,6853632) | 20260507 03:48:07.458 | INFO | MemInfo: { | 20260507 03:48:07.458 | INFO | "gpu" : " 17 GB", | 20260507 03:48:07.458 | INFO | "huge" : "", | 20260507 03:48:07.458 | INFO | "huge_shared" : " 27 GB", | 20260507 03:48:07.458 | INFO | "malloc" : " 14 GB", | 20260507 03:48:07.458 | INFO | "pinned" : " 29 GB" | 20260507 03:48:07.458 | INFO | } | 20260507 03:48:07.458 | INFO | Perf report for the last 5.13976 seconds: | 20260507 03:48:07.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:48:07.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:07.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:07.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:07.458 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 03:48:07.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:07.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:07.458 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 03:48:07.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:07.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:07.458 | INFO | Uploaded 36GiB to GPU in 1.99776s (18.0202GiB/s) | 20260507 03:48:07.459 | INFO | Downloaded 34GiB to GPU in 1.92765s (17.6381GiB/s) | 20260507 03:48:12.590 | INFO | Analyzing chunk frames = [6853632,6854144) | 20260507 03:48:12.590 | INFO | MemInfo: { | 20260507 03:48:12.590 | INFO | "gpu" : " 17 GB", | 20260507 03:48:12.590 | INFO | "huge" : "", | 20260507 03:48:12.590 | INFO | "huge_shared" : " 27 GB", | 20260507 03:48:12.590 | INFO | "malloc" : " 14 GB", | 20260507 03:48:12.590 | INFO | "pinned" : " 29 GB" | 20260507 03:48:12.590 | INFO | } | 20260507 03:48:12.590 | INFO | Perf report for the last 5.13189 seconds: | 20260507 03:48:12.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:48:12.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:12.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:12.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:12.590 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 03:48:12.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:12.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:12.590 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:48:12.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:12.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:12.590 | INFO | Uploaded 36GiB to GPU in 2.04888s (17.5706GiB/s) | 20260507 03:48:12.590 | INFO | Downloaded 34GiB to GPU in 1.94997s (17.4361GiB/s) | 20260507 03:48:17.735 | INFO | Analyzing chunk frames = [6854144,6854656) | 20260507 03:48:17.735 | INFO | MemInfo: { | 20260507 03:48:17.735 | INFO | "gpu" : " 17 GB", | 20260507 03:48:17.735 | INFO | "huge" : "", | 20260507 03:48:17.735 | INFO | "huge_shared" : " 27 GB", | 20260507 03:48:17.735 | INFO | "malloc" : " 14 GB", | 20260507 03:48:17.735 | INFO | "pinned" : " 29 GB" | 20260507 03:48:17.735 | INFO | } | 20260507 03:48:17.735 | INFO | Perf report for the last 5.14509 seconds: | 20260507 03:48:17.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:48:17.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:17.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:17.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:17.735 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 03:48:17.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:17.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:17.735 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:48:17.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:17.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:17.736 | INFO | Uploaded 36GiB to GPU in 1.9882s (18.1068GiB/s) | 20260507 03:48:17.736 | INFO | Downloaded 34GiB to GPU in 1.92077s (17.7012GiB/s) | 20260507 03:48:22.867 | INFO | Analyzing chunk frames = [6854656,6855168) | 20260507 03:48:22.867 | INFO | MemInfo: { | 20260507 03:48:22.867 | INFO | "gpu" : " 17 GB", | 20260507 03:48:22.867 | INFO | "huge" : "", | 20260507 03:48:22.867 | INFO | "huge_shared" : " 27 GB", | 20260507 03:48:22.867 | INFO | "malloc" : " 14 GB", | 20260507 03:48:22.867 | INFO | "pinned" : " 29 GB" | 20260507 03:48:22.867 | INFO | } | 20260507 03:48:22.867 | INFO | Perf report for the last 5.13153 seconds: | 20260507 03:48:22.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:48:22.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:22.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:22.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:22.867 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 03:48:22.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:22.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:22.867 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 03:48:22.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:22.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:22.867 | INFO | Uploaded 36GiB to GPU in 2.03434s (17.6961GiB/s) | 20260507 03:48:22.867 | INFO | Downloaded 34GiB to GPU in 1.93873s (17.5373GiB/s) | 20260507 03:48:28.007 | INFO | Analyzing chunk frames = [6855168,6855680) | 20260507 03:48:28.007 | INFO | MemInfo: { | 20260507 03:48:28.007 | INFO | "gpu" : " 17 GB", | 20260507 03:48:28.007 | INFO | "huge" : "", | 20260507 03:48:28.007 | INFO | "huge_shared" : " 27 GB", | 20260507 03:48:28.007 | INFO | "malloc" : " 14 GB", | 20260507 03:48:28.007 | INFO | "pinned" : " 29 GB" | 20260507 03:48:28.007 | INFO | } | 20260507 03:48:28.007 | INFO | Perf report for the last 5.14034 seconds: | 20260507 03:48:28.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:48:28.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:28.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:28.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:28.007 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:48:28.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:28.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:28.007 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:48:28.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:28.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:28.007 | INFO | Uploaded 36GiB to GPU in 2.05696s (17.5016GiB/s) | 20260507 03:48:28.007 | INFO | Downloaded 34GiB to GPU in 1.95512s (17.3902GiB/s) | 20260507 03:48:33.151 | INFO | Analyzing chunk frames = [6855680,6856192) | 20260507 03:48:33.151 | INFO | MemInfo: { | 20260507 03:48:33.151 | INFO | "gpu" : " 17 GB", | 20260507 03:48:33.151 | INFO | "huge" : "", | 20260507 03:48:33.151 | INFO | "huge_shared" : " 27 GB", | 20260507 03:48:33.151 | INFO | "malloc" : " 14 GB", | 20260507 03:48:33.151 | INFO | "pinned" : " 29 GB" | 20260507 03:48:33.151 | INFO | } | 20260507 03:48:33.151 | INFO | Perf report for the last 5.1442 seconds: | 20260507 03:48:33.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:48:33.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:33.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:33.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:33.152 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:48:33.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:33.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:33.152 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:48:33.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:33.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:33.152 | INFO | Uploaded 36GiB to GPU in 1.95703s (18.3952GiB/s) | 20260507 03:48:33.152 | INFO | Downloaded 34GiB to GPU in 1.97252s (17.2368GiB/s) | 20260507 03:48:38.284 | INFO | Analyzing chunk frames = [6856192,6856704) | 20260507 03:48:38.284 | INFO | MemInfo: { | 20260507 03:48:38.284 | INFO | "gpu" : " 17 GB", | 20260507 03:48:38.284 | INFO | "huge" : "", | 20260507 03:48:38.284 | INFO | "huge_shared" : " 27 GB", | 20260507 03:48:38.284 | INFO | "malloc" : " 14 GB", | 20260507 03:48:38.284 | INFO | "pinned" : " 29 GB" | 20260507 03:48:38.284 | INFO | } | 20260507 03:48:38.284 | INFO | Perf report for the last 5.13305 seconds: | 20260507 03:48:38.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:48:38.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:38.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:38.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:38.284 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 03:48:38.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:38.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:38.285 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 03:48:38.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:38.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:38.285 | INFO | Uploaded 36GiB to GPU in 1.95343s (18.4291GiB/s) | 20260507 03:48:38.285 | INFO | Downloaded 34GiB to GPU in 1.94684s (17.4642GiB/s) | 20260507 03:48:38.285 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:48:38.285 | INFO | Total mem usage: 18 GiB | 20260507 03:48:38.285 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:48:38.285 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -21.1%) | 20260507 03:48:38.285 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:48:38.285 | INFO | High Water Breakdown: | 20260507 03:48:38.285 | INFO | 60.8% : 9.04 GiB ( 79.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:48:38.285 | INFO | 39.1% : 5.81 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:48:38.285 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:48:38.285 | INFO | ------------------------------------------ | 20260507 03:48:38.285 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:48:38.285 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:48:38.285 | INFO | Total mem usage: 26.5 GiB | 20260507 03:48:38.285 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:48:38.285 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 03:48:38.285 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260507 03:48:38.285 | INFO | High Water Breakdown: | 20260507 03:48:38.285 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:48:38.285 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:48:38.285 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:48:38.285 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:48:38.285 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:48:38.285 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:48:38.285 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:48:38.285 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:48:38.285 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:48:38.285 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:48:38.285 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:48:38.285 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:48:38.285 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:48:38.285 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:48:38.285 | INFO | ------------------------------------------ | 20260507 03:48:38.285 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:48:38.285 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:48:38.285 | INFO | Total mem usage: 25.6 GiB | 20260507 03:48:38.285 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:48:38.285 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:48:38.285 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:48:38.285 | INFO | High Water Breakdown: | 20260507 03:48:38.285 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:48:38.285 | INFO | ------------------------------------------ | 20260507 03:48:38.285 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:48:38.285 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:48:38.285 | INFO | Total mem usage: 15.7 GiB | 20260507 03:48:38.285 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:48:38.285 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:48:38.285 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:48:38.285 | INFO | High Water Breakdown: | 20260507 03:48:38.285 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:48:38.285 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:48:38.285 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:48:38.285 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:48:38.285 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:48:38.285 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:48:38.285 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:48:38.285 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:48:38.285 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:48:38.285 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:48:38.285 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:48:38.285 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:48:38.285 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:48:38.286 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:48:38.286 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:48:38.286 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:48:38.286 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:48:38.286 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:48:38.286 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:48:38.286 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:48:38.286 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:48:38.286 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:48:38.286 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:48:38.286 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:48:38.286 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:48:38.286 | INFO | ------------------------------------------ | 20260507 03:48:38.286 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:48:38.286 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:48:38.286 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:48:38.286 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:48:38.286 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.32e+01 / 1.21e+00 | 20260507 03:48:38.286 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.02e-01, 9.97e-03 / 8.03e-03 | 20260507 03:48:38.286 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.41e+00, 5.21e-01 / 3.00e-01 | 20260507 03:48:38.286 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.45e+00, 1.67e-01 / 1.23e-01 | 20260507 03:48:38.286 | INFO | PerfReport: 16.14%, DME -- 3.97e+01 / 4.44e+01, 3.96e+00 / 1.51e+00 | 20260507 03:48:38.286 | INFO | PerfReport: 57.65%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.30e+00 / 1.66e+00 | 20260507 03:48:38.286 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.14e+00 / 2.68e+00, 3.84e-01 / 2.31e-01 | 20260507 03:48:38.286 | INFO | PerfReport: 3.03%, Metrics -- 9.32e-01 / 1.25e+00, 2.45e-01 / 1.43e-01 | 20260507 03:48:38.286 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.16e-01 / 3.72e-01, 5.06e-02 / 2.34e-02 | 20260507 03:48:38.286 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:48:38.286 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:48:38.286 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:48:38.286 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 03:48:38.286 | INFO | PerfReport: 21.80%, Upload -- 2.10e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260507 03:48:38.286 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260507 03:48:38.286 | INFO | PerfReport: 31.95%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 03:48:38.286 | INFO | PerfReport: 1.52%, LockUpload -- 1.47e+00 / 0.00e+00, 4.94e+00 / 0.00e+00 | 20260507 03:48:38.286 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260507 03:48:38.286 | INFO | PerfReport: 22.04%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 03:48:38.286 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260507 03:48:38.381 | INFO | Starting a round of Baz IO | 20260507 03:48:41.486 | INFO | Populating RTMetrics report | 20260507 03:48:41.487 | INFO | RTMetrics report populated | 20260507 03:48:41.487 | INFO | Opening tmp RTMetrics json file | 20260507 03:48:41.505 | INFO | Serializing RTMetrics to json | 20260507 03:48:41.506 | INFO | Serialization done, swapping tmp file now... | 20260507 03:48:41.507 | INFO | RTMetrics done | 20260507 03:48:41.577 | INFO | Finished a round of Baz IO: Wrote 1.741 GiB in 3.195 seconds | 20260507 03:48:41.577 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.85% / 0.00% / 67.14% | 20260507 03:48:41.577 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.187695% / 0% / 6.35783e-05% | 20260507 03:48:41.577 | INFO | Baz IO overall: | 20260507 03:48:41.577 | INFO | BytesWritten : 3512360620204 | 20260507 03:48:41.577 | INFO | NumEvents : 1687165270876 | 20260507 03:48:43.427 | INFO | Analyzing chunk frames = [6856704,6857216) | 20260507 03:48:43.427 | INFO | MemInfo: { | 20260507 03:48:43.427 | INFO | "gpu" : " 17 GB", | 20260507 03:48:43.427 | INFO | "huge" : "", | 20260507 03:48:43.427 | INFO | "huge_shared" : " 27 GB", | 20260507 03:48:43.427 | INFO | "malloc" : " 14 GB", | 20260507 03:48:43.427 | INFO | "pinned" : " 29 GB" | 20260507 03:48:43.427 | INFO | } | 20260507 03:48:43.427 | INFO | Perf report for the last 5.14282 seconds: | 20260507 03:48:43.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:48:43.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:43.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:43.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:43.427 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 03:48:43.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:43.428 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:48:43.428 | INFO | PRE_HQ : 59.1%, 1.8, 55.3ms, 0 | 20260507 03:48:43.428 | INFO | BAZWRITER : 61.7%, 3.2, 101.9ms, 0 | 20260507 03:48:43.428 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:48:43.428 | INFO | Uploaded 36GiB to GPU in 2.02297s (17.7956GiB/s) | 20260507 03:48:43.428 | INFO | Downloaded 38GiB to GPU in 2.18634s (17.3807GiB/s) | 20260507 03:48:48.569 | INFO | Analyzing chunk frames = [6857216,6857728) | 20260507 03:48:48.569 | INFO | MemInfo: { | 20260507 03:48:48.570 | INFO | "gpu" : " 17 GB", | 20260507 03:48:48.570 | INFO | "huge" : "", | 20260507 03:48:48.570 | INFO | "huge_shared" : " 27 GB", | 20260507 03:48:48.570 | INFO | "malloc" : " 14 GB", | 20260507 03:48:48.570 | INFO | "pinned" : " 29 GB" | 20260507 03:48:48.570 | INFO | } | 20260507 03:48:48.570 | INFO | Perf report for the last 5.14229 seconds: | 20260507 03:48:48.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:48:48.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:48.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:48.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:48.570 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:48:48.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:48.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:48.570 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:48:48.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:48.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:48.570 | INFO | Uploaded 36GiB to GPU in 1.98856s (18.1035GiB/s) | 20260507 03:48:48.570 | INFO | Downloaded 34GiB to GPU in 1.93679s (17.5548GiB/s) | 20260507 03:48:53.703 | INFO | Analyzing chunk frames = [6857728,6858240) | 20260507 03:48:53.703 | INFO | MemInfo: { | 20260507 03:48:53.703 | INFO | "gpu" : " 17 GB", | 20260507 03:48:53.703 | INFO | "huge" : "", | 20260507 03:48:53.703 | INFO | "huge_shared" : " 27 GB", | 20260507 03:48:53.703 | INFO | "malloc" : " 14 GB", | 20260507 03:48:53.703 | INFO | "pinned" : " 29 GB" | 20260507 03:48:53.703 | INFO | } | 20260507 03:48:53.703 | INFO | Perf report for the last 5.13316 seconds: | 20260507 03:48:53.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:48:53.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:53.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:53.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:53.703 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 03:48:53.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:53.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:53.703 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:48:53.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:53.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:53.703 | INFO | Uploaded 36GiB to GPU in 2.04128s (17.636GiB/s) | 20260507 03:48:53.703 | INFO | Downloaded 34GiB to GPU in 1.95352s (17.4045GiB/s) | 20260507 03:48:58.837 | INFO | Analyzing chunk frames = [6858240,6858752) | 20260507 03:48:58.837 | INFO | MemInfo: { | 20260507 03:48:58.837 | INFO | "gpu" : " 17 GB", | 20260507 03:48:58.837 | INFO | "huge" : "", | 20260507 03:48:58.837 | INFO | "huge_shared" : " 27 GB", | 20260507 03:48:58.837 | INFO | "malloc" : " 14 GB", | 20260507 03:48:58.837 | INFO | "pinned" : " 29 GB" | 20260507 03:48:58.837 | INFO | } | 20260507 03:48:58.837 | INFO | Perf report for the last 5.13441 seconds: | 20260507 03:48:58.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:48:58.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:58.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:58.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:58.837 | INFO | ANALYSIS : 59.7%, 3.0, 92.4ms, 0 | 20260507 03:48:58.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:58.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:58.837 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:48:58.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:58.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:48:58.838 | INFO | Uploaded 36GiB to GPU in 1.96565s (18.3146GiB/s) | 20260507 03:48:58.838 | INFO | Downloaded 34GiB to GPU in 1.92379s (17.6734GiB/s) | 20260507 03:49:03.984 | INFO | Analyzing chunk frames = [6858752,6859264) | 20260507 03:49:03.984 | INFO | MemInfo: { | 20260507 03:49:03.984 | INFO | "gpu" : " 17 GB", | 20260507 03:49:03.984 | INFO | "huge" : "", | 20260507 03:49:03.984 | INFO | "huge_shared" : " 27 GB", | 20260507 03:49:03.984 | INFO | "malloc" : " 14 GB", | 20260507 03:49:03.984 | INFO | "pinned" : " 29 GB" | 20260507 03:49:03.984 | INFO | } | 20260507 03:49:03.984 | INFO | Perf report for the last 5.14724 seconds: | 20260507 03:49:03.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:49:03.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:03.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:03.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:03.984 | INFO | ANALYSIS : 59.8%, 3.0, 93.2ms, 0 | 20260507 03:49:03.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:03.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:03.985 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:49:03.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:03.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:03.985 | INFO | Uploaded 36GiB to GPU in 1.98585s (18.1283GiB/s) | 20260507 03:49:03.985 | INFO | Downloaded 34GiB to GPU in 1.93503s (17.5708GiB/s) | 20260507 03:49:09.129 | INFO | Analyzing chunk frames = [6859264,6859776) | 20260507 03:49:09.129 | INFO | MemInfo: { | 20260507 03:49:09.129 | INFO | "gpu" : " 17 GB", | 20260507 03:49:09.129 | INFO | "huge" : "", | 20260507 03:49:09.129 | INFO | "huge_shared" : " 27 GB", | 20260507 03:49:09.129 | INFO | "malloc" : " 14 GB", | 20260507 03:49:09.129 | INFO | "pinned" : " 29 GB" | 20260507 03:49:09.129 | INFO | } | 20260507 03:49:09.129 | INFO | Perf report for the last 5.1445 seconds: | 20260507 03:49:09.129 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:49:09.129 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:09.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:09.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:09.129 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 03:49:09.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:09.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:09.129 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:49:09.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:09.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:09.129 | INFO | Uploaded 36GiB to GPU in 2.0477s (17.5807GiB/s) | 20260507 03:49:09.129 | INFO | Downloaded 34GiB to GPU in 1.9584s (17.3611GiB/s) | 20260507 03:49:14.263 | INFO | Analyzing chunk frames = [6859776,6860288) | 20260507 03:49:14.263 | INFO | MemInfo: { | 20260507 03:49:14.263 | INFO | "gpu" : " 17 GB", | 20260507 03:49:14.263 | INFO | "huge" : "", | 20260507 03:49:14.263 | INFO | "huge_shared" : " 27 GB", | 20260507 03:49:14.263 | INFO | "malloc" : " 14 GB", | 20260507 03:49:14.263 | INFO | "pinned" : " 29 GB" | 20260507 03:49:14.263 | INFO | } | 20260507 03:49:14.263 | INFO | Perf report for the last 5.13434 seconds: | 20260507 03:49:14.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:49:14.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:14.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:14.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:14.263 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:49:14.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:14.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:14.263 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 03:49:14.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:14.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:14.264 | INFO | Uploaded 36GiB to GPU in 2.02024s (17.8196GiB/s) | 20260507 03:49:14.264 | INFO | Downloaded 34GiB to GPU in 1.97914s (17.1792GiB/s) | 20260507 03:49:19.401 | INFO | Analyzing chunk frames = [6860288,6860800) | 20260507 03:49:19.401 | INFO | MemInfo: { | 20260507 03:49:19.401 | INFO | "gpu" : " 17 GB", | 20260507 03:49:19.401 | INFO | "huge" : "", | 20260507 03:49:19.401 | INFO | "huge_shared" : " 27 GB", | 20260507 03:49:19.401 | INFO | "malloc" : " 14 GB", | 20260507 03:49:19.401 | INFO | "pinned" : " 29 GB" | 20260507 03:49:19.401 | INFO | } | 20260507 03:49:19.401 | INFO | Perf report for the last 5.13792 seconds: | 20260507 03:49:19.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:49:19.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:19.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:19.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:19.401 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 03:49:19.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:19.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:19.401 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:49:19.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:19.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:19.402 | INFO | Uploaded 36GiB to GPU in 1.97199s (18.2556GiB/s) | 20260507 03:49:19.402 | INFO | Downloaded 34GiB to GPU in 1.95626s (17.3801GiB/s) | 20260507 03:49:19.499 | INFO | Starting a round of Baz IO | 20260507 03:49:22.602 | INFO | Populating RTMetrics report | 20260507 03:49:22.603 | INFO | RTMetrics report populated | 20260507 03:49:22.603 | INFO | Opening tmp RTMetrics json file | 20260507 03:49:22.607 | INFO | Serializing RTMetrics to json | 20260507 03:49:22.621 | INFO | Serialization done, swapping tmp file now... | 20260507 03:49:22.621 | INFO | RTMetrics done | 20260507 03:49:22.697 | INFO | Finished a round of Baz IO: Wrote 1.742 GiB in 3.198 seconds | 20260507 03:49:22.697 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.85% / 0.00% / 67.15% | 20260507 03:49:22.697 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.187659% / 3.97364e-06% / 6.35783e-05% | 20260507 03:49:22.697 | INFO | Baz IO overall: | 20260507 03:49:22.697 | INFO | BytesWritten : 3514230650332 | 20260507 03:49:22.697 | INFO | NumEvents : 1688046829028 | 20260507 03:49:24.537 | INFO | Analyzing chunk frames = [6860800,6861312) | 20260507 03:49:24.537 | INFO | MemInfo: { | 20260507 03:49:24.537 | INFO | "gpu" : " 17 GB", | 20260507 03:49:24.537 | INFO | "huge" : "", | 20260507 03:49:24.537 | INFO | "huge_shared" : " 27 GB", | 20260507 03:49:24.537 | INFO | "malloc" : " 14 GB", | 20260507 03:49:24.537 | INFO | "pinned" : " 29 GB" | 20260507 03:49:24.537 | INFO | } | 20260507 03:49:24.537 | INFO | Perf report for the last 5.13618 seconds: | 20260507 03:49:24.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:49:24.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:24.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:24.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:24.537 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 03:49:24.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:24.538 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:49:24.538 | INFO | PRE_HQ : 59.1%, 1.8, 55.1ms, 0 | 20260507 03:49:24.538 | INFO | BAZWRITER : 61.9%, 3.3, 103.4ms, 0 | 20260507 03:49:24.538 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:49:24.538 | INFO | Uploaded 36GiB to GPU in 2.03788s (17.6654GiB/s) | 20260507 03:49:24.538 | INFO | Downloaded 38GiB to GPU in 2.17614s (17.4621GiB/s) | 20260507 03:49:29.679 | INFO | Analyzing chunk frames = [6861312,6861824) | 20260507 03:49:29.679 | INFO | MemInfo: { | 20260507 03:49:29.679 | INFO | "gpu" : " 17 GB", | 20260507 03:49:29.679 | INFO | "huge" : "", | 20260507 03:49:29.680 | INFO | "huge_shared" : " 27 GB", | 20260507 03:49:29.680 | INFO | "malloc" : " 14 GB", | 20260507 03:49:29.680 | INFO | "pinned" : " 29 GB" | 20260507 03:49:29.680 | INFO | } | 20260507 03:49:29.680 | INFO | Perf report for the last 5.1423 seconds: | 20260507 03:49:29.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:49:29.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:29.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:29.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:29.680 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:49:29.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:29.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:29.680 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:49:29.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:29.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:29.680 | INFO | Uploaded 36GiB to GPU in 2.03966s (17.65GiB/s) | 20260507 03:49:29.680 | INFO | Downloaded 34GiB to GPU in 1.93745s (17.5488GiB/s) | 20260507 03:49:34.825 | INFO | Analyzing chunk frames = [6861824,6862336) | 20260507 03:49:34.825 | INFO | MemInfo: { | 20260507 03:49:34.825 | INFO | "gpu" : " 17 GB", | 20260507 03:49:34.825 | INFO | "huge" : "", | 20260507 03:49:34.825 | INFO | "huge_shared" : " 27 GB", | 20260507 03:49:34.825 | INFO | "malloc" : " 14 GB", | 20260507 03:49:34.825 | INFO | "pinned" : " 29 GB" | 20260507 03:49:34.825 | INFO | } | 20260507 03:49:34.825 | INFO | Perf report for the last 5.14561 seconds: | 20260507 03:49:34.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:49:34.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:34.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:34.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:34.825 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 03:49:34.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:34.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:34.825 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:49:34.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:34.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:34.826 | INFO | Uploaded 36GiB to GPU in 2.01084s (17.9029GiB/s) | 20260507 03:49:34.826 | INFO | Downloaded 34GiB to GPU in 1.9398s (17.5276GiB/s) | 20260507 03:49:39.960 | INFO | Analyzing chunk frames = [6862336,6862848) | 20260507 03:49:39.960 | INFO | MemInfo: { | 20260507 03:49:39.960 | INFO | "gpu" : " 17 GB", | 20260507 03:49:39.960 | INFO | "huge" : "", | 20260507 03:49:39.960 | INFO | "huge_shared" : " 27 GB", | 20260507 03:49:39.960 | INFO | "malloc" : " 14 GB", | 20260507 03:49:39.960 | INFO | "pinned" : " 29 GB" | 20260507 03:49:39.960 | INFO | } | 20260507 03:49:39.960 | INFO | Perf report for the last 5.13456 seconds: | 20260507 03:49:39.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:49:39.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:39.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:39.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:39.960 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 03:49:39.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:39.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:39.960 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:49:39.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:39.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:39.960 | INFO | Uploaded 36GiB to GPU in 2.00912s (17.9183GiB/s) | 20260507 03:49:39.960 | INFO | Downloaded 34GiB to GPU in 1.93545s (17.567GiB/s) | 20260507 03:49:45.099 | INFO | Analyzing chunk frames = [6862848,6863360) | 20260507 03:49:45.099 | INFO | MemInfo: { | 20260507 03:49:45.099 | INFO | "gpu" : " 17 GB", | 20260507 03:49:45.099 | INFO | "huge" : "", | 20260507 03:49:45.099 | INFO | "huge_shared" : " 27 GB", | 20260507 03:49:45.099 | INFO | "malloc" : " 14 GB", | 20260507 03:49:45.099 | INFO | "pinned" : " 29 GB" | 20260507 03:49:45.099 | INFO | } | 20260507 03:49:45.099 | INFO | Perf report for the last 5.13887 seconds: | 20260507 03:49:45.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:49:45.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:45.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:45.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:45.099 | INFO | ANALYSIS : 59.9%, 3.0, 93.5ms, 0 | 20260507 03:49:45.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:45.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:45.099 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:49:45.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:45.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:45.099 | INFO | Uploaded 36GiB to GPU in 2.02638s (17.7657GiB/s) | 20260507 03:49:45.099 | INFO | Downloaded 34GiB to GPU in 1.92953s (17.6209GiB/s) | 20260507 03:49:50.240 | INFO | Analyzing chunk frames = [6863360,6863872) | 20260507 03:49:50.240 | INFO | MemInfo: { | 20260507 03:49:50.240 | INFO | "gpu" : " 17 GB", | 20260507 03:49:50.240 | INFO | "huge" : "", | 20260507 03:49:50.240 | INFO | "huge_shared" : " 27 GB", | 20260507 03:49:50.240 | INFO | "malloc" : " 14 GB", | 20260507 03:49:50.240 | INFO | "pinned" : " 29 GB" | 20260507 03:49:50.240 | INFO | } | 20260507 03:49:50.240 | INFO | Perf report for the last 5.14125 seconds: | 20260507 03:49:50.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:49:50.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:50.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:50.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:50.240 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260507 03:49:50.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:50.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:50.240 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 03:49:50.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:50.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:50.240 | INFO | Uploaded 36GiB to GPU in 2.04496s (17.6043GiB/s) | 20260507 03:49:50.240 | INFO | Downloaded 34GiB to GPU in 1.93546s (17.5669GiB/s) | 20260507 03:49:55.381 | INFO | Analyzing chunk frames = [6863872,6864384) | 20260507 03:49:55.381 | INFO | MemInfo: { | 20260507 03:49:55.381 | INFO | "gpu" : " 17 GB", | 20260507 03:49:55.381 | INFO | "huge" : "", | 20260507 03:49:55.381 | INFO | "huge_shared" : " 27 GB", | 20260507 03:49:55.381 | INFO | "malloc" : " 14 GB", | 20260507 03:49:55.381 | INFO | "pinned" : " 29 GB" | 20260507 03:49:55.381 | INFO | } | 20260507 03:49:55.381 | INFO | Perf report for the last 5.14113 seconds: | 20260507 03:49:55.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:49:55.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:55.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:55.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:55.381 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 03:49:55.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:55.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:55.381 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 03:49:55.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:55.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:49:55.382 | INFO | Uploaded 36GiB to GPU in 1.9685s (18.288GiB/s) | 20260507 03:49:55.382 | INFO | Downloaded 34GiB to GPU in 1.97151s (17.2456GiB/s) | 20260507 03:50:00.517 | INFO | Analyzing chunk frames = [6864384,6864896) | 20260507 03:50:00.517 | INFO | MemInfo: { | 20260507 03:50:00.517 | INFO | "gpu" : " 17 GB", | 20260507 03:50:00.518 | INFO | "huge" : "", | 20260507 03:50:00.518 | INFO | "huge_shared" : " 27 GB", | 20260507 03:50:00.518 | INFO | "malloc" : " 14 GB", | 20260507 03:50:00.518 | INFO | "pinned" : " 29 GB" | 20260507 03:50:00.518 | INFO | } | 20260507 03:50:00.518 | INFO | Perf report for the last 5.13646 seconds: | 20260507 03:50:00.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:50:00.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:00.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:00.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:00.518 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 03:50:00.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:00.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:00.518 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 03:50:00.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:00.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:00.518 | INFO | Uploaded 36GiB to GPU in 1.99159s (18.076GiB/s) | 20260507 03:50:00.518 | INFO | Downloaded 34GiB to GPU in 1.97952s (17.1759GiB/s) | 20260507 03:50:00.518 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:50:00.518 | INFO | Total mem usage: 18 GiB | 20260507 03:50:00.518 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:50:00.518 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 03:50:00.518 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:50:00.518 | INFO | High Water Breakdown: | 20260507 03:50:00.518 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:50:00.518 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:50:00.518 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:50:00.518 | INFO | ------------------------------------------ | 20260507 03:50:00.518 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:50:00.518 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:50:00.518 | INFO | Total mem usage: 26.5 GiB | 20260507 03:50:00.518 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:50:00.518 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:50:00.518 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:50:00.518 | INFO | High Water Breakdown: | 20260507 03:50:00.518 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:50:00.518 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:50:00.518 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:50:00.518 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:50:00.518 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:50:00.518 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:50:00.519 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:50:00.519 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:50:00.519 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:50:00.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:50:00.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:50:00.519 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:50:00.519 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:50:00.519 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:50:00.519 | INFO | ------------------------------------------ | 20260507 03:50:00.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:50:00.519 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:50:00.519 | INFO | Total mem usage: 25.6 GiB | 20260507 03:50:00.519 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:50:00.519 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:50:00.519 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:50:00.519 | INFO | High Water Breakdown: | 20260507 03:50:00.519 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:50:00.519 | INFO | ------------------------------------------ | 20260507 03:50:00.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:50:00.519 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:50:00.519 | INFO | Total mem usage: 15.7 GiB | 20260507 03:50:00.519 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:50:00.519 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:50:00.519 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:50:00.519 | INFO | High Water Breakdown: | 20260507 03:50:00.519 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:50:00.519 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:50:00.519 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:50:00.519 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:50:00.519 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:50:00.519 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:50:00.519 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:50:00.519 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:50:00.519 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:50:00.519 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:50:00.519 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:50:00.519 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:50:00.519 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:50:00.519 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:50:00.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:50:00.519 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:50:00.519 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:50:00.519 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:50:00.519 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:50:00.519 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:50:00.519 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:50:00.519 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:50:00.519 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:50:00.519 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:50:00.519 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:50:00.519 | INFO | ------------------------------------------ | 20260507 03:50:00.519 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:50:00.519 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:50:00.519 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:50:00.519 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:50:00.519 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.48e+01, 1.33e+01 / 1.36e+00 | 20260507 03:50:00.519 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.00e-01, 9.98e-03 / 5.82e-03 | 20260507 03:50:00.519 | INFO | PerfReport: 11.69%, Baseline -- 3.60e+00 / 4.41e+00, 5.22e-01 / 2.90e-01 | 20260507 03:50:00.519 | INFO | PerfReport: 3.23%, Histogram -- 9.95e-01 / 1.44e+00, 1.69e-01 / 1.34e-01 | 20260507 03:50:00.519 | INFO | PerfReport: 16.10%, DME -- 3.96e+01 / 4.40e+01, 3.79e+00 / 1.53e+00 | 20260507 03:50:00.519 | INFO | PerfReport: 57.67%, FrameLabeling -- 1.77e+01 / 2.05e+01, 1.32e+00 / 1.46e+00 | 20260507 03:50:00.519 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.66e+00, 3.78e-01 / 2.00e-01 | 20260507 03:50:00.519 | INFO | PerfReport: 3.03%, Metrics -- 9.32e-01 / 1.26e+00, 2.47e-01 / 1.07e-01 | 20260507 03:50:00.519 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.15e-01 / 3.69e-01, 4.93e-02 / 1.76e-02 | 20260507 03:50:00.519 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:50:00.519 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:50:00.519 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:50:00.519 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 03:50:00.519 | INFO | PerfReport: 21.77%, Upload -- 2.09e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260507 03:50:00.519 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260507 03:50:00.519 | INFO | PerfReport: 31.96%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 03:50:00.519 | INFO | PerfReport: 1.42%, LockUpload -- 1.36e+00 / 0.00e+00, 4.75e+00 / 0.00e+00 | 20260507 03:50:00.519 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260507 03:50:00.519 | INFO | PerfReport: 22.15%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 03:50:00.519 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260507 03:50:00.614 | INFO | Starting a round of Baz IO | 20260507 03:50:03.714 | INFO | Populating RTMetrics report | 20260507 03:50:03.714 | INFO | RTMetrics report populated | 20260507 03:50:03.714 | INFO | Opening tmp RTMetrics json file | 20260507 03:50:03.732 | INFO | Serializing RTMetrics to json | 20260507 03:50:03.733 | INFO | Serialization done, swapping tmp file now... | 20260507 03:50:03.734 | INFO | RTMetrics done | 20260507 03:50:03.802 | INFO | Finished a round of Baz IO: Wrote 1.741 GiB in 3.187 seconds | 20260507 03:50:03.802 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.85% / 0.00% / 67.15% | 20260507 03:50:03.802 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.187703% / 3.97364e-06% / 6.75519e-05% | 20260507 03:50:03.802 | INFO | Baz IO overall: | 20260507 03:50:03.802 | INFO | BytesWritten : 3516100273769 | 20260507 03:50:03.802 | INFO | NumEvents : 1688927887000 | 20260507 03:50:05.651 | INFO | Analyzing chunk frames = [6864896,6865408) | 20260507 03:50:05.651 | INFO | MemInfo: { | 20260507 03:50:05.651 | INFO | "gpu" : " 17 GB", | 20260507 03:50:05.651 | INFO | "huge" : "", | 20260507 03:50:05.651 | INFO | "huge_shared" : " 27 GB", | 20260507 03:50:05.651 | INFO | "malloc" : " 14 GB", | 20260507 03:50:05.651 | INFO | "pinned" : " 29 GB" | 20260507 03:50:05.652 | INFO | } | 20260507 03:50:05.652 | INFO | Perf report for the last 5.13386 seconds: | 20260507 03:50:05.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:50:05.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:05.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:05.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:05.652 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 03:50:05.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:05.652 | INFO | RT_METRICS : 41.7%, 1.0, 21.8ms, 0 | 20260507 03:50:05.652 | INFO | PRE_HQ : 59.2%, 1.8, 55.3ms, 0 | 20260507 03:50:05.652 | INFO | BAZWRITER : 61.7%, 3.2, 102.6ms, 0 | 20260507 03:50:05.652 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 03:50:05.652 | INFO | Uploaded 36GiB to GPU in 2.04148s (17.6343GiB/s) | 20260507 03:50:05.652 | INFO | Downloaded 38GiB to GPU in 2.17977s (17.433GiB/s) | 20260507 03:50:10.795 | INFO | Analyzing chunk frames = [6865408,6865920) | 20260507 03:50:10.795 | INFO | MemInfo: { | 20260507 03:50:10.795 | INFO | "gpu" : " 17 GB", | 20260507 03:50:10.795 | INFO | "huge" : "", | 20260507 03:50:10.795 | INFO | "huge_shared" : " 27 GB", | 20260507 03:50:10.795 | INFO | "malloc" : " 14 GB", | 20260507 03:50:10.795 | INFO | "pinned" : " 29 GB" | 20260507 03:50:10.795 | INFO | } | 20260507 03:50:10.795 | INFO | Perf report for the last 5.14338 seconds: | 20260507 03:50:10.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:50:10.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:10.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:10.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:10.795 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:50:10.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:10.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:10.795 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:50:10.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:10.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:10.795 | INFO | Uploaded 36GiB to GPU in 2.02038s (17.8184GiB/s) | 20260507 03:50:10.795 | INFO | Downloaded 34GiB to GPU in 1.93946s (17.5306GiB/s) | 20260507 03:50:15.932 | INFO | Analyzing chunk frames = [6865920,6866432) | 20260507 03:50:15.932 | INFO | MemInfo: { | 20260507 03:50:15.932 | INFO | "gpu" : " 17 GB", | 20260507 03:50:15.932 | INFO | "huge" : "", | 20260507 03:50:15.932 | INFO | "huge_shared" : " 27 GB", | 20260507 03:50:15.932 | INFO | "malloc" : " 14 GB", | 20260507 03:50:15.932 | INFO | "pinned" : " 29 GB" | 20260507 03:50:15.932 | INFO | } | 20260507 03:50:15.932 | INFO | Perf report for the last 5.13695 seconds: | 20260507 03:50:15.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:50:15.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:15.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:15.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:15.932 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 03:50:15.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:15.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:15.932 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:50:15.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:15.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:15.932 | INFO | Uploaded 36GiB to GPU in 1.97763s (18.2036GiB/s) | 20260507 03:50:15.932 | INFO | Downloaded 34GiB to GPU in 1.94039s (17.5223GiB/s) | 20260507 03:50:21.079 | INFO | Analyzing chunk frames = [6866432,6866944) | 20260507 03:50:21.079 | INFO | MemInfo: { | 20260507 03:50:21.079 | INFO | "gpu" : " 17 GB", | 20260507 03:50:21.079 | INFO | "huge" : "", | 20260507 03:50:21.079 | INFO | "huge_shared" : " 27 GB", | 20260507 03:50:21.079 | INFO | "malloc" : " 14 GB", | 20260507 03:50:21.079 | INFO | "pinned" : " 29 GB" | 20260507 03:50:21.079 | INFO | } | 20260507 03:50:21.079 | INFO | Perf report for the last 5.14711 seconds: | 20260507 03:50:21.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:50:21.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:21.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:21.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:21.079 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 03:50:21.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:21.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:21.079 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:50:21.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:21.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:21.079 | INFO | Uploaded 36GiB to GPU in 1.96491s (18.3215GiB/s) | 20260507 03:50:21.079 | INFO | Downloaded 34GiB to GPU in 1.92363s (17.6749GiB/s) | 20260507 03:50:26.216 | INFO | Analyzing chunk frames = [6866944,6867456) | 20260507 03:50:26.216 | INFO | MemInfo: { | 20260507 03:50:26.216 | INFO | "gpu" : " 17 GB", | 20260507 03:50:26.216 | INFO | "huge" : "", | 20260507 03:50:26.216 | INFO | "huge_shared" : " 27 GB", | 20260507 03:50:26.216 | INFO | "malloc" : " 14 GB", | 20260507 03:50:26.216 | INFO | "pinned" : " 29 GB" | 20260507 03:50:26.216 | INFO | } | 20260507 03:50:26.216 | INFO | Perf report for the last 5.13708 seconds: | 20260507 03:50:26.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:50:26.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:26.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:26.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:26.216 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 03:50:26.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:26.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:26.216 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:50:26.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:26.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:26.216 | INFO | Uploaded 36GiB to GPU in 2.05045s (17.5571GiB/s) | 20260507 03:50:26.216 | INFO | Downloaded 34GiB to GPU in 1.95383s (17.4017GiB/s) | 20260507 03:50:31.349 | INFO | Analyzing chunk frames = [6867456,6867968) | 20260507 03:50:31.349 | INFO | MemInfo: { | 20260507 03:50:31.349 | INFO | "gpu" : " 17 GB", | 20260507 03:50:31.349 | INFO | "huge" : "", | 20260507 03:50:31.349 | INFO | "huge_shared" : " 27 GB", | 20260507 03:50:31.349 | INFO | "malloc" : " 14 GB", | 20260507 03:50:31.350 | INFO | "pinned" : " 29 GB" | 20260507 03:50:31.350 | INFO | } | 20260507 03:50:31.350 | INFO | Perf report for the last 5.13354 seconds: | 20260507 03:50:31.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:50:31.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:31.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:31.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:31.350 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 03:50:31.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:31.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:31.350 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:50:31.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:31.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:31.350 | INFO | Uploaded 36GiB to GPU in 2.02541s (17.7742GiB/s) | 20260507 03:50:31.350 | INFO | Downloaded 34GiB to GPU in 1.90342s (17.8626GiB/s) | 20260507 03:50:36.494 | INFO | Analyzing chunk frames = [6867968,6868480) | 20260507 03:50:36.494 | INFO | MemInfo: { | 20260507 03:50:36.494 | INFO | "gpu" : " 17 GB", | 20260507 03:50:36.494 | INFO | "huge" : "", | 20260507 03:50:36.494 | INFO | "huge_shared" : " 27 GB", | 20260507 03:50:36.494 | INFO | "malloc" : " 14 GB", | 20260507 03:50:36.494 | INFO | "pinned" : " 29 GB" | 20260507 03:50:36.494 | INFO | } | 20260507 03:50:36.495 | INFO | Perf report for the last 5.14486 seconds: | 20260507 03:50:36.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:50:36.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:36.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:36.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:36.495 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:50:36.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:36.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:36.495 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:50:36.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:36.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:36.495 | INFO | Uploaded 36GiB to GPU in 1.98026s (18.1794GiB/s) | 20260507 03:50:36.495 | INFO | Downloaded 34GiB to GPU in 1.9997s (17.0025GiB/s) | 20260507 03:50:41.635 | INFO | Analyzing chunk frames = [6868480,6868992) | 20260507 03:50:41.635 | INFO | MemInfo: { | 20260507 03:50:41.635 | INFO | "gpu" : " 17 GB", | 20260507 03:50:41.636 | INFO | "huge" : "", | 20260507 03:50:41.636 | INFO | "huge_shared" : " 27 GB", | 20260507 03:50:41.636 | INFO | "malloc" : " 14 GB", | 20260507 03:50:41.636 | INFO | "pinned" : " 29 GB" | 20260507 03:50:41.636 | INFO | } | 20260507 03:50:41.636 | INFO | Perf report for the last 5.1412 seconds: | 20260507 03:50:41.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:50:41.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:41.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:41.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:41.636 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:50:41.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:41.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:41.636 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 03:50:41.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:41.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:41.636 | INFO | Uploaded 36GiB to GPU in 2.0066s (17.9408GiB/s) | 20260507 03:50:41.636 | INFO | Downloaded 34GiB to GPU in 1.96949s (17.2633GiB/s) | 20260507 03:50:41.732 | INFO | Starting a round of Baz IO | 20260507 03:50:44.833 | INFO | Populating RTMetrics report | 20260507 03:50:44.833 | INFO | RTMetrics report populated | 20260507 03:50:44.833 | INFO | Opening tmp RTMetrics json file | 20260507 03:50:44.838 | INFO | Serializing RTMetrics to json | 20260507 03:50:44.852 | INFO | Serialization done, swapping tmp file now... | 20260507 03:50:44.852 | INFO | RTMetrics done | 20260507 03:50:44.923 | INFO | Finished a round of Baz IO: Wrote 1.739 GiB in 3.19 seconds | 20260507 03:50:44.923 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.85% / 0.00% / 67.15% | 20260507 03:50:44.923 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.186308% / 0% / 7.15256e-05% | 20260507 03:50:44.923 | INFO | Baz IO overall: | 20260507 03:50:44.923 | INFO | BytesWritten : 3517967908058 | 20260507 03:50:44.923 | INFO | NumEvents : 1689807490795 | 20260507 03:50:46.780 | INFO | Analyzing chunk frames = [6868992,6869504) | 20260507 03:50:46.780 | INFO | MemInfo: { | 20260507 03:50:46.780 | INFO | "gpu" : " 17 GB", | 20260507 03:50:46.780 | INFO | "huge" : "", | 20260507 03:50:46.780 | INFO | "huge_shared" : " 27 GB", | 20260507 03:50:46.780 | INFO | "malloc" : " 14 GB", | 20260507 03:50:46.780 | INFO | "pinned" : " 29 GB" | 20260507 03:50:46.780 | INFO | } | 20260507 03:50:46.780 | INFO | Perf report for the last 5.1447 seconds: | 20260507 03:50:46.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:50:46.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:46.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:46.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:46.781 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 03:50:46.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:46.781 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 03:50:46.781 | INFO | PRE_HQ : 59.0%, 1.8, 55.4ms, 0 | 20260507 03:50:46.781 | INFO | BAZWRITER : 61.6%, 3.3, 103.6ms, 0 | 20260507 03:50:46.781 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:50:46.781 | INFO | Uploaded 36GiB to GPU in 2.03927s (17.6534GiB/s) | 20260507 03:50:46.781 | INFO | Downloaded 38GiB to GPU in 2.183s (17.4072GiB/s) | 20260507 03:50:51.909 | INFO | Analyzing chunk frames = [6869504,6870016) | 20260507 03:50:51.909 | INFO | MemInfo: { | 20260507 03:50:51.909 | INFO | "gpu" : " 17 GB", | 20260507 03:50:51.909 | INFO | "huge" : "", | 20260507 03:50:51.910 | INFO | "huge_shared" : " 27 GB", | 20260507 03:50:51.910 | INFO | "malloc" : " 14 GB", | 20260507 03:50:51.910 | INFO | "pinned" : " 29 GB" | 20260507 03:50:51.910 | INFO | } | 20260507 03:50:51.910 | INFO | Perf report for the last 5.1292 seconds: | 20260507 03:50:51.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:50:51.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:51.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:51.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:51.910 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 03:50:51.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:51.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:51.910 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:50:51.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:51.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:51.910 | INFO | Uploaded 36GiB to GPU in 1.988s (18.1086GiB/s) | 20260507 03:50:51.910 | INFO | Downloaded 34GiB to GPU in 1.92702s (17.6438GiB/s) | 20260507 03:50:57.051 | INFO | Analyzing chunk frames = [6870016,6870528) | 20260507 03:50:57.051 | INFO | MemInfo: { | 20260507 03:50:57.051 | INFO | "gpu" : " 17 GB", | 20260507 03:50:57.051 | INFO | "huge" : "", | 20260507 03:50:57.051 | INFO | "huge_shared" : " 27 GB", | 20260507 03:50:57.051 | INFO | "malloc" : " 14 GB", | 20260507 03:50:57.051 | INFO | "pinned" : " 29 GB" | 20260507 03:50:57.051 | INFO | } | 20260507 03:50:57.051 | INFO | Perf report for the last 5.14179 seconds: | 20260507 03:50:57.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:50:57.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:57.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:57.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:57.052 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 03:50:57.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:57.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:57.052 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:50:57.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:57.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:50:57.052 | INFO | Uploaded 36GiB to GPU in 2.00508s (17.9544GiB/s) | 20260507 03:50:57.052 | INFO | Downloaded 34GiB to GPU in 1.93913s (17.5337GiB/s) | 20260507 03:51:02.189 | INFO | Analyzing chunk frames = [6870528,6871040) | 20260507 03:51:02.189 | INFO | MemInfo: { | 20260507 03:51:02.189 | INFO | "gpu" : " 17 GB", | 20260507 03:51:02.189 | INFO | "huge" : "", | 20260507 03:51:02.189 | INFO | "huge_shared" : " 27 GB", | 20260507 03:51:02.189 | INFO | "malloc" : " 14 GB", | 20260507 03:51:02.190 | INFO | "pinned" : " 29 GB" | 20260507 03:51:02.190 | INFO | } | 20260507 03:51:02.190 | INFO | Perf report for the last 5.13808 seconds: | 20260507 03:51:02.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:51:02.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:02.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:02.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:02.190 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 03:51:02.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:02.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:02.190 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:51:02.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:02.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:02.190 | INFO | Uploaded 36GiB to GPU in 1.99085s (18.0827GiB/s) | 20260507 03:51:02.190 | INFO | Downloaded 34GiB to GPU in 1.91811s (17.7258GiB/s) | 20260507 03:51:07.328 | INFO | Analyzing chunk frames = [6871040,6871552) | 20260507 03:51:07.328 | INFO | MemInfo: { | 20260507 03:51:07.328 | INFO | "gpu" : " 17 GB", | 20260507 03:51:07.328 | INFO | "huge" : "", | 20260507 03:51:07.328 | INFO | "huge_shared" : " 27 GB", | 20260507 03:51:07.328 | INFO | "malloc" : " 14 GB", | 20260507 03:51:07.328 | INFO | "pinned" : " 29 GB" | 20260507 03:51:07.328 | INFO | } | 20260507 03:51:07.328 | INFO | Perf report for the last 5.13861 seconds: | 20260507 03:51:07.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:51:07.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:07.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:07.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:07.328 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 03:51:07.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:07.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:07.328 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:51:07.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:07.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:07.329 | INFO | Uploaded 36GiB to GPU in 2.03803s (17.6641GiB/s) | 20260507 03:51:07.329 | INFO | Downloaded 34GiB to GPU in 1.9416s (17.5113GiB/s) | 20260507 03:51:12.468 | INFO | Analyzing chunk frames = [6871552,6872064) | 20260507 03:51:12.468 | INFO | MemInfo: { | 20260507 03:51:12.468 | INFO | "gpu" : " 17 GB", | 20260507 03:51:12.468 | INFO | "huge" : "", | 20260507 03:51:12.468 | INFO | "huge_shared" : " 27 GB", | 20260507 03:51:12.468 | INFO | "malloc" : " 14 GB", | 20260507 03:51:12.468 | INFO | "pinned" : " 29 GB" | 20260507 03:51:12.468 | INFO | } | 20260507 03:51:12.468 | INFO | Perf report for the last 5.14 seconds: | 20260507 03:51:12.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:51:12.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:12.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:12.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:12.468 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 03:51:12.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:12.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:12.468 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 03:51:12.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:12.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:12.468 | INFO | Uploaded 36GiB to GPU in 2.04313s (17.62GiB/s) | 20260507 03:51:12.469 | INFO | Downloaded 34GiB to GPU in 1.91763s (17.7303GiB/s) | 20260507 03:51:17.604 | INFO | Analyzing chunk frames = [6872064,6872576) | 20260507 03:51:17.604 | INFO | MemInfo: { | 20260507 03:51:17.604 | INFO | "gpu" : " 17 GB", | 20260507 03:51:17.604 | INFO | "huge" : "", | 20260507 03:51:17.604 | INFO | "huge_shared" : " 27 GB", | 20260507 03:51:17.604 | INFO | "malloc" : " 14 GB", | 20260507 03:51:17.604 | INFO | "pinned" : " 29 GB" | 20260507 03:51:17.604 | INFO | } | 20260507 03:51:17.604 | INFO | Perf report for the last 5.13567 seconds: | 20260507 03:51:17.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:51:17.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:17.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:17.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:17.604 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 03:51:17.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:17.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:17.604 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 03:51:17.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:17.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:17.604 | INFO | Uploaded 36GiB to GPU in 1.9855s (18.1315GiB/s) | 20260507 03:51:17.604 | INFO | Downloaded 34GiB to GPU in 1.95416s (17.3988GiB/s) | 20260507 03:51:22.753 | INFO | Analyzing chunk frames = [6872576,6873088) | 20260507 03:51:22.753 | INFO | MemInfo: { | 20260507 03:51:22.753 | INFO | "gpu" : " 17 GB", | 20260507 03:51:22.753 | INFO | "huge" : "", | 20260507 03:51:22.754 | INFO | "huge_shared" : " 27 GB", | 20260507 03:51:22.754 | INFO | "malloc" : " 14 GB", | 20260507 03:51:22.754 | INFO | "pinned" : " 29 GB" | 20260507 03:51:22.754 | INFO | } | 20260507 03:51:22.754 | INFO | Perf report for the last 5.14981 seconds: | 20260507 03:51:22.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:51:22.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:22.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:22.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:22.754 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 03:51:22.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:22.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:22.754 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 03:51:22.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:22.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:22.754 | INFO | Uploaded 36GiB to GPU in 1.9789s (18.192GiB/s) | 20260507 03:51:22.754 | INFO | Downloaded 34GiB to GPU in 1.96851s (17.2719GiB/s) | 20260507 03:51:22.754 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:51:22.754 | INFO | Total mem usage: 18 GiB | 20260507 03:51:22.754 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:51:22.754 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 03:51:22.754 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:51:22.754 | INFO | High Water Breakdown: | 20260507 03:51:22.754 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:51:22.754 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:51:22.754 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:51:22.754 | INFO | ------------------------------------------ | 20260507 03:51:22.754 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:51:22.754 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:51:22.754 | INFO | Total mem usage: 26.5 GiB | 20260507 03:51:22.754 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:51:22.754 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:51:22.754 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:51:22.754 | INFO | High Water Breakdown: | 20260507 03:51:22.754 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:51:22.754 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:51:22.754 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:51:22.754 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:51:22.754 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:51:22.754 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:51:22.754 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:51:22.754 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:51:22.754 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:51:22.754 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:51:22.754 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:51:22.754 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:51:22.754 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:51:22.755 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:51:22.755 | INFO | ------------------------------------------ | 20260507 03:51:22.755 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:51:22.755 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:51:22.755 | INFO | Total mem usage: 25.6 GiB | 20260507 03:51:22.755 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:51:22.755 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 03:51:22.755 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 03:51:22.755 | INFO | High Water Breakdown: | 20260507 03:51:22.755 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 03:51:22.755 | INFO | ------------------------------------------ | 20260507 03:51:22.755 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:51:22.755 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:51:22.755 | INFO | Total mem usage: 15.7 GiB | 20260507 03:51:22.755 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:51:22.755 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:51:22.755 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:51:22.755 | INFO | High Water Breakdown: | 20260507 03:51:22.755 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:51:22.755 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:51:22.755 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:51:22.755 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:51:22.755 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:51:22.755 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:51:22.755 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:51:22.755 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:51:22.755 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:51:22.755 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:51:22.755 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:51:22.755 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:51:22.755 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:51:22.755 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:51:22.755 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:51:22.755 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:51:22.755 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:51:22.755 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:51:22.755 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:51:22.755 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:51:22.755 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:51:22.755 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:51:22.755 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:51:22.755 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:51:22.755 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:51:22.755 | INFO | ------------------------------------------ | 20260507 03:51:22.755 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:51:22.755 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:51:22.755 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:51:22.755 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:51:22.755 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.52e+01, 1.34e+01 / 1.62e+00 | 20260507 03:51:22.755 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.02e-01, 9.58e-03 / 3.72e-03 | 20260507 03:51:22.755 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.52e+00, 5.23e-01 / 1.35e-01 | 20260507 03:51:22.755 | INFO | PerfReport: 3.24%, Histogram -- 9.96e-01 / 1.50e+00, 1.72e-01 / 8.51e-02 | 20260507 03:51:22.755 | INFO | PerfReport: 16.13%, DME -- 3.97e+01 / 4.44e+01, 3.97e+00 / 2.33e+00 | 20260507 03:51:22.755 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.30e+00 / 1.54e+00 | 20260507 03:51:22.755 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.14e+00 / 2.69e+00, 3.84e-01 / 1.74e-01 | 20260507 03:51:22.755 | INFO | PerfReport: 3.04%, Metrics -- 9.34e-01 / 1.25e+00, 2.53e-01 / 8.93e-02 | 20260507 03:51:22.755 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.15e-01 / 3.69e-01, 5.06e-02 / 1.60e-02 | 20260507 03:51:22.755 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:51:22.755 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:51:22.755 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:51:22.755 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 03:51:22.755 | INFO | PerfReport: 21.74%, Upload -- 2.09e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260507 03:51:22.755 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 03:51:22.755 | INFO | PerfReport: 31.97%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 03:51:22.755 | INFO | PerfReport: 1.45%, LockUpload -- 1.40e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 03:51:22.755 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260507 03:51:22.755 | INFO | PerfReport: 22.16%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 03:51:22.755 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260507 03:51:22.851 | INFO | Starting a round of Baz IO | 20260507 03:51:25.955 | INFO | Populating RTMetrics report | 20260507 03:51:25.955 | INFO | RTMetrics report populated | 20260507 03:51:25.955 | INFO | Opening tmp RTMetrics json file | 20260507 03:51:25.973 | INFO | Serializing RTMetrics to json | 20260507 03:51:25.974 | INFO | Serialization done, swapping tmp file now... | 20260507 03:51:25.974 | INFO | RTMetrics done | 20260507 03:51:26.045 | INFO | Finished a round of Baz IO: Wrote 1.741 GiB in 3.193 seconds | 20260507 03:51:26.045 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.85% / 0.00% / 67.15% | 20260507 03:51:26.045 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.185843% / 3.97364e-06% / 6.75519e-05% | 20260507 03:51:26.045 | INFO | Baz IO overall: | 20260507 03:51:26.045 | INFO | BytesWritten : 3519836962141 | 20260507 03:51:26.045 | INFO | NumEvents : 1690688391560 | 20260507 03:51:27.892 | INFO | Analyzing chunk frames = [6873088,6873600) | 20260507 03:51:27.892 | INFO | MemInfo: { | 20260507 03:51:27.892 | INFO | "gpu" : " 17 GB", | 20260507 03:51:27.892 | INFO | "huge" : "", | 20260507 03:51:27.892 | INFO | "huge_shared" : " 27 GB", | 20260507 03:51:27.892 | INFO | "malloc" : " 14 GB", | 20260507 03:51:27.892 | INFO | "pinned" : " 29 GB" | 20260507 03:51:27.892 | INFO | } | 20260507 03:51:27.892 | INFO | Perf report for the last 5.13845 seconds: | 20260507 03:51:27.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:51:27.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:27.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:27.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:27.892 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 03:51:27.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:27.892 | INFO | RT_METRICS : 41.7%, 1.0, 21.8ms, 0 | 20260507 03:51:27.892 | INFO | PRE_HQ : 59.0%, 1.8, 55.1ms, 0 | 20260507 03:51:27.892 | INFO | BAZWRITER : 61.7%, 3.3, 103.6ms, 0 | 20260507 03:51:27.892 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:51:27.893 | INFO | Uploaded 36GiB to GPU in 2.04554s (17.5993GiB/s) | 20260507 03:51:27.893 | INFO | Downloaded 38GiB to GPU in 2.1725s (17.4914GiB/s) | 20260507 03:51:33.026 | INFO | Analyzing chunk frames = [6873600,6874112) | 20260507 03:51:33.026 | INFO | MemInfo: { | 20260507 03:51:33.026 | INFO | "gpu" : " 17 GB", | 20260507 03:51:33.026 | INFO | "huge" : "", | 20260507 03:51:33.026 | INFO | "huge_shared" : " 27 GB", | 20260507 03:51:33.026 | INFO | "malloc" : " 14 GB", | 20260507 03:51:33.026 | INFO | "pinned" : " 29 GB" | 20260507 03:51:33.026 | INFO | } | 20260507 03:51:33.026 | INFO | Perf report for the last 5.13371 seconds: | 20260507 03:51:33.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:51:33.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:33.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:33.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:33.026 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:51:33.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:33.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:33.026 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:51:33.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:33.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:33.026 | INFO | Uploaded 36GiB to GPU in 2.00826s (17.926GiB/s) | 20260507 03:51:33.026 | INFO | Downloaded 34GiB to GPU in 1.93444s (17.5761GiB/s) | 20260507 03:51:38.161 | INFO | Analyzing chunk frames = [6874112,6874624) | 20260507 03:51:38.161 | INFO | MemInfo: { | 20260507 03:51:38.161 | INFO | "gpu" : " 17 GB", | 20260507 03:51:38.161 | INFO | "huge" : "", | 20260507 03:51:38.161 | INFO | "huge_shared" : " 27 GB", | 20260507 03:51:38.161 | INFO | "malloc" : " 14 GB", | 20260507 03:51:38.161 | INFO | "pinned" : " 29 GB" | 20260507 03:51:38.161 | INFO | } | 20260507 03:51:38.161 | INFO | Perf report for the last 5.13493 seconds: | 20260507 03:51:38.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:51:38.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:38.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:38.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:38.161 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 03:51:38.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:38.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:38.161 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 03:51:38.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:38.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:38.161 | INFO | Uploaded 36GiB to GPU in 2.02065s (17.8161GiB/s) | 20260507 03:51:38.161 | INFO | Downloaded 34GiB to GPU in 1.93799s (17.5439GiB/s) | 20260507 03:51:43.303 | INFO | Analyzing chunk frames = [6874624,6875136) | 20260507 03:51:43.303 | INFO | MemInfo: { | 20260507 03:51:43.303 | INFO | "gpu" : " 17 GB", | 20260507 03:51:43.303 | INFO | "huge" : "", | 20260507 03:51:43.303 | INFO | "huge_shared" : " 27 GB", | 20260507 03:51:43.303 | INFO | "malloc" : " 14 GB", | 20260507 03:51:43.303 | INFO | "pinned" : " 29 GB" | 20260507 03:51:43.303 | INFO | } | 20260507 03:51:43.303 | INFO | Perf report for the last 5.14198 seconds: | 20260507 03:51:43.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:51:43.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:43.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:43.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:43.303 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:51:43.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:43.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:43.303 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:51:43.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:43.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:43.303 | INFO | Uploaded 36GiB to GPU in 1.97149s (18.2603GiB/s) | 20260507 03:51:43.303 | INFO | Downloaded 34GiB to GPU in 1.90866s (17.8136GiB/s) | 20260507 03:51:48.441 | INFO | Analyzing chunk frames = [6875136,6875648) | 20260507 03:51:48.441 | INFO | MemInfo: { | 20260507 03:51:48.441 | INFO | "gpu" : " 17 GB", | 20260507 03:51:48.441 | INFO | "huge" : "", | 20260507 03:51:48.441 | INFO | "huge_shared" : " 27 GB", | 20260507 03:51:48.441 | INFO | "malloc" : " 14 GB", | 20260507 03:51:48.441 | INFO | "pinned" : " 29 GB" | 20260507 03:51:48.441 | INFO | } | 20260507 03:51:48.442 | INFO | Perf report for the last 5.13866 seconds: | 20260507 03:51:48.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:51:48.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:48.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:48.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:48.442 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 03:51:48.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:48.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:48.442 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:51:48.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:48.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:48.442 | INFO | Uploaded 36GiB to GPU in 2.02049s (17.8175GiB/s) | 20260507 03:51:48.442 | INFO | Downloaded 34GiB to GPU in 1.9383s (17.5411GiB/s) | 20260507 03:51:53.582 | INFO | Analyzing chunk frames = [6875648,6876160) | 20260507 03:51:53.582 | INFO | MemInfo: { | 20260507 03:51:53.582 | INFO | "gpu" : " 17 GB", | 20260507 03:51:53.582 | INFO | "huge" : "", | 20260507 03:51:53.582 | INFO | "huge_shared" : " 27 GB", | 20260507 03:51:53.582 | INFO | "malloc" : " 14 GB", | 20260507 03:51:53.583 | INFO | "pinned" : " 29 GB" | 20260507 03:51:53.583 | INFO | } | 20260507 03:51:53.583 | INFO | Perf report for the last 5.14107 seconds: | 20260507 03:51:53.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:51:53.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:53.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:53.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:53.583 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:51:53.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:53.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:53.583 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 03:51:53.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:53.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:53.583 | INFO | Uploaded 36GiB to GPU in 2.0586s (17.4876GiB/s) | 20260507 03:51:53.583 | INFO | Downloaded 34GiB to GPU in 1.96046s (17.3428GiB/s) | 20260507 03:51:58.729 | INFO | Analyzing chunk frames = [6876160,6876672) | 20260507 03:51:58.729 | INFO | MemInfo: { | 20260507 03:51:58.729 | INFO | "gpu" : " 17 GB", | 20260507 03:51:58.730 | INFO | "huge" : "", | 20260507 03:51:58.730 | INFO | "huge_shared" : " 27 GB", | 20260507 03:51:58.730 | INFO | "malloc" : " 14 GB", | 20260507 03:51:58.730 | INFO | "pinned" : " 29 GB" | 20260507 03:51:58.730 | INFO | } | 20260507 03:51:58.730 | INFO | Perf report for the last 5.14706 seconds: | 20260507 03:51:58.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:51:58.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:58.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:58.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:58.730 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 03:51:58.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:58.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:58.730 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:51:58.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:58.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:51:58.730 | INFO | Uploaded 36GiB to GPU in 1.99244s (18.0683GiB/s) | 20260507 03:51:58.730 | INFO | Downloaded 34GiB to GPU in 1.96613s (17.2929GiB/s) | 20260507 03:52:03.865 | INFO | Analyzing chunk frames = [6876672,6877184) | 20260507 03:52:03.865 | INFO | MemInfo: { | 20260507 03:52:03.865 | INFO | "gpu" : " 17 GB", | 20260507 03:52:03.865 | INFO | "huge" : "", | 20260507 03:52:03.865 | INFO | "huge_shared" : " 27 GB", | 20260507 03:52:03.866 | INFO | "malloc" : " 14 GB", | 20260507 03:52:03.866 | INFO | "pinned" : " 29 GB" | 20260507 03:52:03.866 | INFO | } | 20260507 03:52:03.866 | INFO | Perf report for the last 5.13598 seconds: | 20260507 03:52:03.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:52:03.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:03.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:03.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:03.866 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 03:52:03.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:03.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:03.866 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 03:52:03.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:03.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:03.866 | INFO | Uploaded 36GiB to GPU in 1.97254s (18.2506GiB/s) | 20260507 03:52:03.866 | INFO | Downloaded 34GiB to GPU in 1.95355s (17.4042GiB/s) | 20260507 03:52:03.963 | INFO | Starting a round of Baz IO | 20260507 03:52:07.065 | INFO | Populating RTMetrics report | 20260507 03:52:07.065 | INFO | RTMetrics report populated | 20260507 03:52:07.065 | INFO | Opening tmp RTMetrics json file | 20260507 03:52:07.066 | INFO | Serializing RTMetrics to json | 20260507 03:52:07.080 | INFO | Serialization done, swapping tmp file now... | 20260507 03:52:07.085 | INFO | RTMetrics done | 20260507 03:52:07.154 | INFO | Finished a round of Baz IO: Wrote 1.741 GiB in 3.191 seconds | 20260507 03:52:07.154 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.85% / 0.00% / 67.15% | 20260507 03:52:07.154 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.186145% / 3.97364e-06% / 4.37101e-05% | 20260507 03:52:07.154 | INFO | Baz IO overall: | 20260507 03:52:07.154 | INFO | BytesWritten : 3521706197450 | 20260507 03:52:07.154 | INFO | NumEvents : 1691569294044 | 20260507 03:52:09.008 | INFO | Analyzing chunk frames = [6877184,6877696) | 20260507 03:52:09.008 | INFO | MemInfo: { | 20260507 03:52:09.008 | INFO | "gpu" : " 17 GB", | 20260507 03:52:09.008 | INFO | "huge" : "", | 20260507 03:52:09.008 | INFO | "huge_shared" : " 27 GB", | 20260507 03:52:09.008 | INFO | "malloc" : " 14 GB", | 20260507 03:52:09.008 | INFO | "pinned" : " 29 GB" | 20260507 03:52:09.008 | INFO | } | 20260507 03:52:09.008 | INFO | Perf report for the last 5.14243 seconds: | 20260507 03:52:09.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:52:09.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:09.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:09.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:09.008 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 03:52:09.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:09.008 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 03:52:09.008 | INFO | PRE_HQ : 59.0%, 1.8, 55.2ms, 0 | 20260507 03:52:09.008 | INFO | BAZWRITER : 61.6%, 3.3, 105.0ms, 0 | 20260507 03:52:09.008 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:52:09.008 | INFO | Uploaded 36GiB to GPU in 2.04609s (17.5946GiB/s) | 20260507 03:52:09.008 | INFO | Downloaded 38GiB to GPU in 2.16783s (17.5291GiB/s) | 20260507 03:52:14.140 | INFO | Analyzing chunk frames = [6877696,6878208) | 20260507 03:52:14.141 | INFO | MemInfo: { | 20260507 03:52:14.141 | INFO | "gpu" : " 17 GB", | 20260507 03:52:14.141 | INFO | "huge" : "", | 20260507 03:52:14.141 | INFO | "huge_shared" : " 27 GB", | 20260507 03:52:14.141 | INFO | "malloc" : " 14 GB", | 20260507 03:52:14.141 | INFO | "pinned" : " 29 GB" | 20260507 03:52:14.141 | INFO | } | 20260507 03:52:14.141 | INFO | Perf report for the last 5.13265 seconds: | 20260507 03:52:14.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:52:14.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:14.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:14.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:14.141 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:52:14.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:14.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:14.141 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:52:14.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:14.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:14.141 | INFO | Uploaded 36GiB to GPU in 2.0462s (17.5936GiB/s) | 20260507 03:52:14.141 | INFO | Downloaded 34GiB to GPU in 1.93512s (17.57GiB/s) | 20260507 03:52:19.276 | INFO | Analyzing chunk frames = [6878208,6878720) | 20260507 03:52:19.276 | INFO | MemInfo: { | 20260507 03:52:19.276 | INFO | "gpu" : " 17 GB", | 20260507 03:52:19.276 | INFO | "huge" : "", | 20260507 03:52:19.276 | INFO | "huge_shared" : " 27 GB", | 20260507 03:52:19.276 | INFO | "malloc" : " 14 GB", | 20260507 03:52:19.276 | INFO | "pinned" : " 29 GB" | 20260507 03:52:19.276 | INFO | } | 20260507 03:52:19.276 | INFO | Perf report for the last 5.13556 seconds: | 20260507 03:52:19.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:52:19.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:19.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:19.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:19.276 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 03:52:19.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:19.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:19.276 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:52:19.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:19.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:19.277 | INFO | Uploaded 36GiB to GPU in 2.01132s (17.8987GiB/s) | 20260507 03:52:19.277 | INFO | Downloaded 34GiB to GPU in 1.94128s (17.5142GiB/s) | 20260507 03:52:24.417 | INFO | Analyzing chunk frames = [6878720,6879232) | 20260507 03:52:24.417 | INFO | MemInfo: { | 20260507 03:52:24.417 | INFO | "gpu" : " 17 GB", | 20260507 03:52:24.417 | INFO | "huge" : "", | 20260507 03:52:24.417 | INFO | "huge_shared" : " 27 GB", | 20260507 03:52:24.417 | INFO | "malloc" : " 14 GB", | 20260507 03:52:24.417 | INFO | "pinned" : " 29 GB" | 20260507 03:52:24.417 | INFO | } | 20260507 03:52:24.417 | INFO | Perf report for the last 5.14096 seconds: | 20260507 03:52:24.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:52:24.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:24.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:24.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:24.417 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 03:52:24.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:24.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:24.417 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:52:24.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:24.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:24.418 | INFO | Uploaded 36GiB to GPU in 1.98291s (18.1551GiB/s) | 20260507 03:52:24.418 | INFO | Downloaded 34GiB to GPU in 1.91943s (17.7136GiB/s) | 20260507 03:52:29.559 | INFO | Analyzing chunk frames = [6879232,6879744) | 20260507 03:52:29.559 | INFO | MemInfo: { | 20260507 03:52:29.559 | INFO | "gpu" : " 17 GB", | 20260507 03:52:29.559 | INFO | "huge" : "", | 20260507 03:52:29.559 | INFO | "huge_shared" : " 27 GB", | 20260507 03:52:29.559 | INFO | "malloc" : " 14 GB", | 20260507 03:52:29.559 | INFO | "pinned" : " 29 GB" | 20260507 03:52:29.559 | INFO | } | 20260507 03:52:29.559 | INFO | Perf report for the last 5.14164 seconds: | 20260507 03:52:29.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:52:29.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:29.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:29.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:29.559 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 03:52:29.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:29.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:29.559 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:52:29.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:29.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:29.559 | INFO | Uploaded 36GiB to GPU in 2.0367s (17.6756GiB/s) | 20260507 03:52:29.559 | INFO | Downloaded 34GiB to GPU in 1.93826s (17.5416GiB/s) | 20260507 03:52:34.701 | INFO | Analyzing chunk frames = [6879744,6880256) | 20260507 03:52:34.701 | INFO | MemInfo: { | 20260507 03:52:34.701 | INFO | "gpu" : " 17 GB", | 20260507 03:52:34.701 | INFO | "huge" : "", | 20260507 03:52:34.701 | INFO | "huge_shared" : " 27 GB", | 20260507 03:52:34.701 | INFO | "malloc" : " 14 GB", | 20260507 03:52:34.701 | INFO | "pinned" : " 29 GB" | 20260507 03:52:34.701 | INFO | } | 20260507 03:52:34.701 | INFO | Perf report for the last 5.14195 seconds: | 20260507 03:52:34.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:52:34.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:34.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:34.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:34.701 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:52:34.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:34.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:34.701 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 03:52:34.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:34.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:34.701 | INFO | Uploaded 36GiB to GPU in 2.05592s (17.5104GiB/s) | 20260507 03:52:34.701 | INFO | Downloaded 34GiB to GPU in 1.959s (17.3558GiB/s) | 20260507 03:52:39.845 | INFO | Analyzing chunk frames = [6880256,6880768) | 20260507 03:52:39.845 | INFO | MemInfo: { | 20260507 03:52:39.845 | INFO | "gpu" : " 17 GB", | 20260507 03:52:39.845 | INFO | "huge" : "", | 20260507 03:52:39.845 | INFO | "huge_shared" : " 27 GB", | 20260507 03:52:39.845 | INFO | "malloc" : " 14 GB", | 20260507 03:52:39.845 | INFO | "pinned" : " 29 GB" | 20260507 03:52:39.845 | INFO | } | 20260507 03:52:39.845 | INFO | Perf report for the last 5.14393 seconds: | 20260507 03:52:39.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:52:39.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:39.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:39.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:39.845 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 03:52:39.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:39.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:39.845 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:52:39.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:39.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:39.845 | INFO | Uploaded 36GiB to GPU in 2.02848s (17.7473GiB/s) | 20260507 03:52:39.845 | INFO | Downloaded 34GiB to GPU in 1.99785s (17.0183GiB/s) | 20260507 03:52:44.974 | INFO | Analyzing chunk frames = [6880768,6881280) | 20260507 03:52:44.974 | INFO | MemInfo: { | 20260507 03:52:44.974 | INFO | "gpu" : " 17 GB", | 20260507 03:52:44.974 | INFO | "huge" : "", | 20260507 03:52:44.974 | INFO | "huge_shared" : " 27 GB", | 20260507 03:52:44.974 | INFO | "malloc" : " 14 GB", | 20260507 03:52:44.974 | INFO | "pinned" : " 29 GB" | 20260507 03:52:44.974 | INFO | } | 20260507 03:52:44.974 | INFO | Perf report for the last 5.12941 seconds: | 20260507 03:52:44.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:52:44.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:44.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:44.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:44.974 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 03:52:44.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:44.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:44.974 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 03:52:44.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:44.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:44.975 | INFO | Uploaded 36GiB to GPU in 1.98272s (18.1568GiB/s) | 20260507 03:52:44.975 | INFO | Downloaded 34GiB to GPU in 1.95071s (17.4296GiB/s) | 20260507 03:52:44.975 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:52:44.975 | INFO | Total mem usage: 18 GiB | 20260507 03:52:44.975 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:52:44.975 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:52:44.975 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:52:44.975 | INFO | High Water Breakdown: | 20260507 03:52:44.975 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:52:44.975 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:52:44.975 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:52:44.975 | INFO | ------------------------------------------ | 20260507 03:52:44.975 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:52:44.975 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:52:44.975 | INFO | Total mem usage: 26.5 GiB | 20260507 03:52:44.975 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:52:44.975 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:52:44.975 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 03:52:44.975 | INFO | High Water Breakdown: | 20260507 03:52:44.975 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:52:44.975 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:52:44.975 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:52:44.975 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:52:44.975 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:52:44.975 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:52:44.975 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:52:44.975 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:52:44.975 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:52:44.975 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:52:44.975 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:52:44.975 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:52:44.975 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:52:44.975 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:52:44.975 | INFO | ------------------------------------------ | 20260507 03:52:44.975 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:52:44.975 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:52:44.975 | INFO | Total mem usage: 25.6 GiB | 20260507 03:52:44.975 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:52:44.975 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:52:44.975 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:52:44.975 | INFO | High Water Breakdown: | 20260507 03:52:44.975 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:52:44.975 | INFO | ------------------------------------------ | 20260507 03:52:44.975 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:52:44.975 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:52:44.975 | INFO | Total mem usage: 15.7 GiB | 20260507 03:52:44.975 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:52:44.975 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:52:44.975 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:52:44.975 | INFO | High Water Breakdown: | 20260507 03:52:44.975 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:52:44.975 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:52:44.975 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:52:44.975 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:52:44.975 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:52:44.975 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:52:44.975 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:52:44.975 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:52:44.975 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:52:44.976 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:52:44.976 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:52:44.976 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:52:44.976 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:52:44.976 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:52:44.976 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:52:44.976 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:52:44.976 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:52:44.976 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:52:44.976 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:52:44.976 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:52:44.976 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:52:44.976 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:52:44.976 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:52:44.976 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:52:44.976 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:52:44.976 | INFO | ------------------------------------------ | 20260507 03:52:44.976 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:52:44.976 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:52:44.976 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:52:44.976 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:52:44.976 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.52e+01, 1.34e+01 / 1.44e+00 | 20260507 03:52:44.976 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.02e-01, 1.03e-02 / 7.01e-03 | 20260507 03:52:44.976 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.46e+00, 5.10e-01 / 2.93e-01 | 20260507 03:52:44.976 | INFO | PerfReport: 3.22%, Histogram -- 9.90e-01 / 1.45e+00, 1.65e-01 / 1.08e-01 | 20260507 03:52:44.976 | INFO | PerfReport: 16.14%, DME -- 3.97e+01 / 4.45e+01, 3.94e+00 / 1.83e+00 | 20260507 03:52:44.976 | INFO | PerfReport: 57.68%, FrameLabeling -- 1.77e+01 / 2.05e+01, 1.30e+00 / 1.62e+00 | 20260507 03:52:44.976 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.63e+00, 3.80e-01 / 2.52e-01 | 20260507 03:52:44.976 | INFO | PerfReport: 3.03%, Metrics -- 9.31e-01 / 1.24e+00, 2.46e-01 / 1.29e-01 | 20260507 03:52:44.976 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.15e-01 / 3.66e-01, 5.06e-02 / 2.16e-02 | 20260507 03:52:44.976 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:52:44.976 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:52:44.976 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:52:44.976 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 03:52:44.976 | INFO | PerfReport: 21.83%, Upload -- 2.10e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260507 03:52:44.976 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260507 03:52:44.976 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 03:52:44.976 | INFO | PerfReport: 1.52%, LockUpload -- 1.47e+00 / 0.00e+00, 4.87e+00 / 0.00e+00 | 20260507 03:52:44.976 | INFO | PerfReport: 1.20%, LockDownload -- 1.16e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260507 03:52:44.976 | INFO | PerfReport: 22.14%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 03:52:44.976 | INFO | PerfReport: 0.02%, Misc -- 1.45e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260507 03:52:45.071 | INFO | Starting a round of Baz IO | 20260507 03:52:48.180 | INFO | Populating RTMetrics report | 20260507 03:52:48.180 | INFO | RTMetrics report populated | 20260507 03:52:48.180 | INFO | Opening tmp RTMetrics json file | 20260507 03:52:48.200 | INFO | Serializing RTMetrics to json | 20260507 03:52:48.201 | INFO | Serialization done, swapping tmp file now... | 20260507 03:52:48.202 | INFO | RTMetrics done | 20260507 03:52:48.273 | INFO | Finished a round of Baz IO: Wrote 1.74 GiB in 3.201 seconds | 20260507 03:52:48.273 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.85% / 0.00% / 67.15% | 20260507 03:52:48.273 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.186042% / 3.97364e-06% / 7.94729e-05% | 20260507 03:52:48.273 | INFO | Baz IO overall: | 20260507 03:52:48.273 | INFO | BytesWritten : 3523574831878 | 20260507 03:52:48.273 | INFO | NumEvents : 1692449493413 | 20260507 03:52:50.113 | INFO | Analyzing chunk frames = [6881280,6881792) | 20260507 03:52:50.113 | INFO | MemInfo: { | 20260507 03:52:50.113 | INFO | "gpu" : " 17 GB", | 20260507 03:52:50.113 | INFO | "huge" : "", | 20260507 03:52:50.113 | INFO | "huge_shared" : " 27 GB", | 20260507 03:52:50.113 | INFO | "malloc" : " 14 GB", | 20260507 03:52:50.113 | INFO | "pinned" : " 29 GB" | 20260507 03:52:50.113 | INFO | } | 20260507 03:52:50.113 | INFO | Perf report for the last 5.13882 seconds: | 20260507 03:52:50.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:52:50.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:50.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:50.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:50.113 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 03:52:50.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:50.113 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 03:52:50.113 | INFO | PRE_HQ : 59.2%, 1.8, 55.4ms, 0 | 20260507 03:52:50.113 | INFO | BAZWRITER : 61.9%, 3.3, 104.1ms, 0 | 20260507 03:52:50.113 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:52:50.113 | INFO | Uploaded 36GiB to GPU in 2.0391s (17.6548GiB/s) | 20260507 03:52:50.113 | INFO | Downloaded 38GiB to GPU in 2.18544s (17.3878GiB/s) | 20260507 03:52:55.251 | INFO | Analyzing chunk frames = [6881792,6882304) | 20260507 03:52:55.251 | INFO | MemInfo: { | 20260507 03:52:55.251 | INFO | "gpu" : " 17 GB", | 20260507 03:52:55.251 | INFO | "huge" : "", | 20260507 03:52:55.251 | INFO | "huge_shared" : " 27 GB", | 20260507 03:52:55.251 | INFO | "malloc" : " 14 GB", | 20260507 03:52:55.251 | INFO | "pinned" : " 29 GB" | 20260507 03:52:55.251 | INFO | } | 20260507 03:52:55.251 | INFO | Perf report for the last 5.13771 seconds: | 20260507 03:52:55.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:52:55.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:55.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:55.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:55.251 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:52:55.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:55.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:55.251 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:52:55.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:55.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:52:55.251 | INFO | Uploaded 36GiB to GPU in 2.00241s (17.9783GiB/s) | 20260507 03:52:55.251 | INFO | Downloaded 34GiB to GPU in 1.92691s (17.6448GiB/s) | 20260507 03:53:00.397 | INFO | Analyzing chunk frames = [6882304,6882816) | 20260507 03:53:00.397 | INFO | MemInfo: { | 20260507 03:53:00.397 | INFO | "gpu" : " 17 GB", | 20260507 03:53:00.397 | INFO | "huge" : "", | 20260507 03:53:00.397 | INFO | "huge_shared" : " 27 GB", | 20260507 03:53:00.397 | INFO | "malloc" : " 14 GB", | 20260507 03:53:00.397 | INFO | "pinned" : " 29 GB" | 20260507 03:53:00.397 | INFO | } | 20260507 03:53:00.397 | INFO | Perf report for the last 5.14662 seconds: | 20260507 03:53:00.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:53:00.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:00.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:00.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:00.398 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 03:53:00.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:00.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:00.398 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 03:53:00.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:00.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:00.398 | INFO | Uploaded 36GiB to GPU in 1.99949s (18.0046GiB/s) | 20260507 03:53:00.398 | INFO | Downloaded 34GiB to GPU in 1.93767s (17.5468GiB/s) | 20260507 03:53:05.533 | INFO | Analyzing chunk frames = [6882816,6883328) | 20260507 03:53:05.533 | INFO | MemInfo: { | 20260507 03:53:05.533 | INFO | "gpu" : " 17 GB", | 20260507 03:53:05.533 | INFO | "huge" : "", | 20260507 03:53:05.533 | INFO | "huge_shared" : " 27 GB", | 20260507 03:53:05.533 | INFO | "malloc" : " 14 GB", | 20260507 03:53:05.533 | INFO | "pinned" : " 29 GB" | 20260507 03:53:05.533 | INFO | } | 20260507 03:53:05.533 | INFO | Perf report for the last 5.13552 seconds: | 20260507 03:53:05.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:53:05.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:05.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:05.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:05.533 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 03:53:05.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:05.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:05.533 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:53:05.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:05.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:05.534 | INFO | Uploaded 36GiB to GPU in 2.00871s (17.9219GiB/s) | 20260507 03:53:05.534 | INFO | Downloaded 34GiB to GPU in 1.93528s (17.5685GiB/s) | 20260507 03:53:10.671 | INFO | Analyzing chunk frames = [6883328,6883840) | 20260507 03:53:10.671 | INFO | MemInfo: { | 20260507 03:53:10.671 | INFO | "gpu" : " 17 GB", | 20260507 03:53:10.671 | INFO | "huge" : "", | 20260507 03:53:10.671 | INFO | "huge_shared" : " 27 GB", | 20260507 03:53:10.671 | INFO | "malloc" : " 14 GB", | 20260507 03:53:10.671 | INFO | "pinned" : " 29 GB" | 20260507 03:53:10.671 | INFO | } | 20260507 03:53:10.671 | INFO | Perf report for the last 5.13846 seconds: | 20260507 03:53:10.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:53:10.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:10.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:10.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:10.672 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 03:53:10.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:10.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:10.672 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:53:10.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:10.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:10.672 | INFO | Uploaded 36GiB to GPU in 2.05295s (17.5358GiB/s) | 20260507 03:53:10.672 | INFO | Downloaded 34GiB to GPU in 1.94512s (17.4796GiB/s) | 20260507 03:53:15.813 | INFO | Analyzing chunk frames = [6883840,6884352) | 20260507 03:53:15.813 | INFO | MemInfo: { | 20260507 03:53:15.813 | INFO | "gpu" : " 17 GB", | 20260507 03:53:15.813 | INFO | "huge" : "", | 20260507 03:53:15.813 | INFO | "huge_shared" : " 27 GB", | 20260507 03:53:15.813 | INFO | "malloc" : " 14 GB", | 20260507 03:53:15.813 | INFO | "pinned" : " 29 GB" | 20260507 03:53:15.813 | INFO | } | 20260507 03:53:15.813 | INFO | Perf report for the last 5.14199 seconds: | 20260507 03:53:15.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:53:15.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:15.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:15.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:15.814 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 03:53:15.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:15.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:15.814 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 03:53:15.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:15.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:15.814 | INFO | Uploaded 36GiB to GPU in 2.03736s (17.6699GiB/s) | 20260507 03:53:15.814 | INFO | Downloaded 34GiB to GPU in 1.92136s (17.6958GiB/s) | 20260507 03:53:20.952 | INFO | Analyzing chunk frames = [6884352,6884864) | 20260507 03:53:20.952 | INFO | MemInfo: { | 20260507 03:53:20.952 | INFO | "gpu" : " 17 GB", | 20260507 03:53:20.952 | INFO | "huge" : "", | 20260507 03:53:20.952 | INFO | "huge_shared" : " 27 GB", | 20260507 03:53:20.952 | INFO | "malloc" : " 14 GB", | 20260507 03:53:20.952 | INFO | "pinned" : " 29 GB" | 20260507 03:53:20.952 | INFO | } | 20260507 03:53:20.952 | INFO | Perf report for the last 5.13859 seconds: | 20260507 03:53:20.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:53:20.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:20.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:20.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:20.952 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:53:20.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:20.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:20.952 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 03:53:20.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:20.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:20.952 | INFO | Uploaded 36GiB to GPU in 1.97229s (18.2529GiB/s) | 20260507 03:53:20.952 | INFO | Downloaded 34GiB to GPU in 1.96422s (17.3097GiB/s) | 20260507 03:53:26.099 | INFO | Analyzing chunk frames = [6884864,6885376) | 20260507 03:53:26.099 | INFO | MemInfo: { | 20260507 03:53:26.099 | INFO | "gpu" : " 17 GB", | 20260507 03:53:26.099 | INFO | "huge" : "", | 20260507 03:53:26.099 | INFO | "huge_shared" : " 27 GB", | 20260507 03:53:26.099 | INFO | "malloc" : " 14 GB", | 20260507 03:53:26.099 | INFO | "pinned" : " 29 GB" | 20260507 03:53:26.099 | INFO | } | 20260507 03:53:26.099 | INFO | Perf report for the last 5.14729 seconds: | 20260507 03:53:26.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:53:26.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:26.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:26.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:26.099 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 03:53:26.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:26.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:26.099 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:53:26.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:26.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:26.100 | INFO | Uploaded 36GiB to GPU in 1.973s (18.2463GiB/s) | 20260507 03:53:26.100 | INFO | Downloaded 34GiB to GPU in 1.971s (17.2501GiB/s) | 20260507 03:53:26.196 | INFO | Starting a round of Baz IO | 20260507 03:53:29.296 | INFO | Populating RTMetrics report | 20260507 03:53:29.296 | INFO | RTMetrics report populated | 20260507 03:53:29.296 | INFO | Opening tmp RTMetrics json file | 20260507 03:53:29.302 | INFO | Serializing RTMetrics to json | 20260507 03:53:29.314 | INFO | Serialization done, swapping tmp file now... | 20260507 03:53:29.315 | INFO | RTMetrics done | 20260507 03:53:29.385 | INFO | Finished a round of Baz IO: Wrote 1.74 GiB in 3.188 seconds | 20260507 03:53:29.385 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.85% / 0.00% / 67.15% | 20260507 03:53:29.385 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.185498% / 0% / 7.15256e-05% | 20260507 03:53:29.385 | INFO | Baz IO overall: | 20260507 03:53:29.385 | INFO | BytesWritten : 3525443272936 | 20260507 03:53:29.385 | INFO | NumEvents : 1693329945916 | 20260507 03:53:31.226 | INFO | Analyzing chunk frames = [6885376,6885888) | 20260507 03:53:31.226 | INFO | MemInfo: { | 20260507 03:53:31.226 | INFO | "gpu" : " 17 GB", | 20260507 03:53:31.226 | INFO | "huge" : "", | 20260507 03:53:31.226 | INFO | "huge_shared" : " 27 GB", | 20260507 03:53:31.226 | INFO | "malloc" : " 14 GB", | 20260507 03:53:31.226 | INFO | "pinned" : " 29 GB" | 20260507 03:53:31.226 | INFO | } | 20260507 03:53:31.226 | INFO | Perf report for the last 5.12692 seconds: | 20260507 03:53:31.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:53:31.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:31.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:31.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:31.226 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260507 03:53:31.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:31.226 | INFO | RT_METRICS : 41.9%, 1.0, 21.9ms, 0 | 20260507 03:53:31.226 | INFO | PRE_HQ : 59.3%, 1.8, 55.3ms, 0 | 20260507 03:53:31.226 | INFO | BAZWRITER : 61.8%, 3.3, 103.3ms, 0 | 20260507 03:53:31.226 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 03:53:31.227 | INFO | Uploaded 36GiB to GPU in 2.04633s (17.5924GiB/s) | 20260507 03:53:31.227 | INFO | Downloaded 38GiB to GPU in 2.17996s (17.4315GiB/s) | 20260507 03:53:36.369 | INFO | Analyzing chunk frames = [6885888,6886400) | 20260507 03:53:36.369 | INFO | MemInfo: { | 20260507 03:53:36.369 | INFO | "gpu" : " 17 GB", | 20260507 03:53:36.369 | INFO | "huge" : "", | 20260507 03:53:36.369 | INFO | "huge_shared" : " 27 GB", | 20260507 03:53:36.369 | INFO | "malloc" : " 14 GB", | 20260507 03:53:36.369 | INFO | "pinned" : " 29 GB" | 20260507 03:53:36.369 | INFO | } | 20260507 03:53:36.369 | INFO | Perf report for the last 5.14287 seconds: | 20260507 03:53:36.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:53:36.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:36.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:36.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:36.369 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 03:53:36.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:36.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:36.369 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260507 03:53:36.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:36.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:36.369 | INFO | Uploaded 36GiB to GPU in 2.03872s (17.6581GiB/s) | 20260507 03:53:36.370 | INFO | Downloaded 34GiB to GPU in 1.94131s (17.5139GiB/s) | 20260507 03:53:41.515 | INFO | Analyzing chunk frames = [6886400,6886912) | 20260507 03:53:41.515 | INFO | MemInfo: { | 20260507 03:53:41.515 | INFO | "gpu" : " 17 GB", | 20260507 03:53:41.515 | INFO | "huge" : "", | 20260507 03:53:41.515 | INFO | "huge_shared" : " 27 GB", | 20260507 03:53:41.515 | INFO | "malloc" : " 14 GB", | 20260507 03:53:41.515 | INFO | "pinned" : " 29 GB" | 20260507 03:53:41.515 | INFO | } | 20260507 03:53:41.515 | INFO | Perf report for the last 5.14573 seconds: | 20260507 03:53:41.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:53:41.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:41.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:41.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:41.515 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 03:53:41.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:41.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:41.515 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260507 03:53:41.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:41.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:41.515 | INFO | Uploaded 36GiB to GPU in 2.00835s (17.9251GiB/s) | 20260507 03:53:41.515 | INFO | Downloaded 34GiB to GPU in 1.93518s (17.5695GiB/s) | 20260507 03:53:46.649 | INFO | Analyzing chunk frames = [6886912,6887424) | 20260507 03:53:46.649 | INFO | MemInfo: { | 20260507 03:53:46.649 | INFO | "gpu" : " 17 GB", | 20260507 03:53:46.649 | INFO | "huge" : "", | 20260507 03:53:46.649 | INFO | "huge_shared" : " 27 GB", | 20260507 03:53:46.649 | INFO | "malloc" : " 14 GB", | 20260507 03:53:46.649 | INFO | "pinned" : " 29 GB" | 20260507 03:53:46.649 | INFO | } | 20260507 03:53:46.649 | INFO | Perf report for the last 5.13388 seconds: | 20260507 03:53:46.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:53:46.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:46.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:46.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:46.649 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 03:53:46.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:46.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:46.649 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 03:53:46.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:46.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:46.649 | INFO | Uploaded 36GiB to GPU in 1.99612s (18.035GiB/s) | 20260507 03:53:46.649 | INFO | Downloaded 34GiB to GPU in 1.92622s (17.6511GiB/s) | 20260507 03:53:51.784 | INFO | Analyzing chunk frames = [6887424,6887936) | 20260507 03:53:51.784 | INFO | MemInfo: { | 20260507 03:53:51.784 | INFO | "gpu" : " 17 GB", | 20260507 03:53:51.784 | INFO | "huge" : "", | 20260507 03:53:51.784 | INFO | "huge_shared" : " 27 GB", | 20260507 03:53:51.784 | INFO | "malloc" : " 14 GB", | 20260507 03:53:51.784 | INFO | "pinned" : " 29 GB" | 20260507 03:53:51.784 | INFO | } | 20260507 03:53:51.785 | INFO | Perf report for the last 5.13568 seconds: | 20260507 03:53:51.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:53:51.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:51.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:51.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:51.785 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 03:53:51.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:51.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:51.785 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:53:51.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:51.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:51.785 | INFO | Uploaded 36GiB to GPU in 2.0545s (17.5225GiB/s) | 20260507 03:53:51.785 | INFO | Downloaded 34GiB to GPU in 1.94385s (17.491GiB/s) | 20260507 03:53:56.924 | INFO | Analyzing chunk frames = [6887936,6888448) | 20260507 03:53:56.924 | INFO | MemInfo: { | 20260507 03:53:56.924 | INFO | "gpu" : " 17 GB", | 20260507 03:53:56.924 | INFO | "huge" : "", | 20260507 03:53:56.925 | INFO | "huge_shared" : " 27 GB", | 20260507 03:53:56.925 | INFO | "malloc" : " 14 GB", | 20260507 03:53:56.925 | INFO | "pinned" : " 29 GB" | 20260507 03:53:56.925 | INFO | } | 20260507 03:53:56.925 | INFO | Perf report for the last 5.14014 seconds: | 20260507 03:53:56.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:53:56.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:56.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:56.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:56.925 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 03:53:56.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:56.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:56.925 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:53:56.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:56.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:53:56.925 | INFO | Uploaded 36GiB to GPU in 2.03411s (17.6981GiB/s) | 20260507 03:53:56.925 | INFO | Downloaded 34GiB to GPU in 1.93412s (17.5791GiB/s) | 20260507 03:54:02.064 | INFO | Analyzing chunk frames = [6888448,6888960) | 20260507 03:54:02.064 | INFO | MemInfo: { | 20260507 03:54:02.064 | INFO | "gpu" : " 17 GB", | 20260507 03:54:02.064 | INFO | "huge" : "", | 20260507 03:54:02.064 | INFO | "huge_shared" : " 27 GB", | 20260507 03:54:02.064 | INFO | "malloc" : " 14 GB", | 20260507 03:54:02.064 | INFO | "pinned" : " 29 GB" | 20260507 03:54:02.064 | INFO | } | 20260507 03:54:02.064 | INFO | Perf report for the last 5.13954 seconds: | 20260507 03:54:02.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:54:02.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:02.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:02.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:02.064 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:54:02.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:02.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:02.064 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 03:54:02.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:02.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:02.065 | INFO | Uploaded 36GiB to GPU in 1.99266s (18.0663GiB/s) | 20260507 03:54:02.065 | INFO | Downloaded 34GiB to GPU in 1.97735s (17.1948GiB/s) | 20260507 03:54:07.204 | INFO | Analyzing chunk frames = [6888960,6889472) | 20260507 03:54:07.204 | INFO | MemInfo: { | 20260507 03:54:07.204 | INFO | "gpu" : " 17 GB", | 20260507 03:54:07.204 | INFO | "huge" : "", | 20260507 03:54:07.204 | INFO | "huge_shared" : " 27 GB", | 20260507 03:54:07.204 | INFO | "malloc" : " 14 GB", | 20260507 03:54:07.204 | INFO | "pinned" : " 29 GB" | 20260507 03:54:07.204 | INFO | } | 20260507 03:54:07.204 | INFO | Perf report for the last 5.14008 seconds: | 20260507 03:54:07.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:54:07.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:07.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:07.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:07.204 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 03:54:07.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:07.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:07.204 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 03:54:07.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:07.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:07.205 | INFO | Uploaded 36GiB to GPU in 2.00734s (17.9342GiB/s) | 20260507 03:54:07.205 | INFO | Downloaded 34GiB to GPU in 1.97019s (17.2572GiB/s) | 20260507 03:54:07.205 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:54:07.205 | INFO | Total mem usage: 18 GiB | 20260507 03:54:07.205 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:54:07.205 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:54:07.205 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:54:07.205 | INFO | High Water Breakdown: | 20260507 03:54:07.205 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:54:07.205 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:54:07.205 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:54:07.205 | INFO | ------------------------------------------ | 20260507 03:54:07.205 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:54:07.205 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:54:07.205 | INFO | Total mem usage: 26.5 GiB | 20260507 03:54:07.205 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:54:07.205 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:54:07.205 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:54:07.205 | INFO | High Water Breakdown: | 20260507 03:54:07.205 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:54:07.205 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:54:07.205 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:54:07.205 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:54:07.205 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:54:07.205 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:54:07.205 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:54:07.205 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:54:07.205 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:54:07.205 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:54:07.205 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:54:07.205 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:54:07.205 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:54:07.205 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:54:07.205 | INFO | ------------------------------------------ | 20260507 03:54:07.205 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:54:07.205 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:54:07.205 | INFO | Total mem usage: 25.6 GiB | 20260507 03:54:07.205 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:54:07.205 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 03:54:07.205 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 03:54:07.205 | INFO | High Water Breakdown: | 20260507 03:54:07.205 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 03:54:07.205 | INFO | ------------------------------------------ | 20260507 03:54:07.205 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:54:07.205 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:54:07.205 | INFO | Total mem usage: 15.7 GiB | 20260507 03:54:07.205 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:54:07.205 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:54:07.205 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:54:07.205 | INFO | High Water Breakdown: | 20260507 03:54:07.205 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:54:07.205 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:54:07.205 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:54:07.206 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:54:07.206 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:54:07.206 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:54:07.206 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:54:07.206 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:54:07.206 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:54:07.206 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:54:07.206 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:54:07.206 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:54:07.206 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:54:07.206 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:54:07.206 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:54:07.206 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:54:07.206 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:54:07.206 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:54:07.206 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:54:07.206 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:54:07.206 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:54:07.206 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:54:07.206 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:54:07.206 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:54:07.206 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:54:07.206 | INFO | ------------------------------------------ | 20260507 03:54:07.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:54:07.206 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:54:07.206 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:54:07.206 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:54:07.206 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.50e+01, 1.35e+01 / 1.22e+00 | 20260507 03:54:07.206 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.38e-03 / 4.94e-03 | 20260507 03:54:07.206 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.48e+00, 5.20e-01 / 1.87e-01 | 20260507 03:54:07.206 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.47e+00, 1.69e-01 / 1.24e-01 | 20260507 03:54:07.206 | INFO | PerfReport: 16.11%, DME -- 3.96e+01 / 4.42e+01, 3.82e+00 / 1.37e+00 | 20260507 03:54:07.206 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.31e+00 / 1.61e+00 | 20260507 03:54:07.206 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.67e+00, 3.82e-01 / 2.32e-01 | 20260507 03:54:07.206 | INFO | PerfReport: 3.03%, Metrics -- 9.32e-01 / 1.25e+00, 2.50e-01 / 1.22e-01 | 20260507 03:54:07.206 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.15e-01 / 3.68e-01, 5.09e-02 / 1.64e-02 | 20260507 03:54:07.206 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:54:07.206 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:54:07.206 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:54:07.206 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 03:54:07.206 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260507 03:54:07.206 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260507 03:54:07.206 | INFO | PerfReport: 31.94%, Compute -- 3.08e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 03:54:07.206 | INFO | PerfReport: 1.51%, LockUpload -- 1.45e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 03:54:07.206 | INFO | PerfReport: 1.25%, LockDownload -- 1.20e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260507 03:54:07.206 | INFO | PerfReport: 22.11%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 03:54:07.206 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260507 03:54:07.301 | INFO | Starting a round of Baz IO | 20260507 03:54:10.407 | INFO | Populating RTMetrics report | 20260507 03:54:10.407 | INFO | RTMetrics report populated | 20260507 03:54:10.407 | INFO | Opening tmp RTMetrics json file | 20260507 03:54:10.424 | INFO | Serializing RTMetrics to json | 20260507 03:54:10.425 | INFO | Serialization done, swapping tmp file now... | 20260507 03:54:10.426 | INFO | RTMetrics done | 20260507 03:54:10.503 | INFO | Finished a round of Baz IO: Wrote 1.74 GiB in 3.202 seconds | 20260507 03:54:10.503 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.85% / 0.00% / 67.15% | 20260507 03:54:10.503 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.185136% / 0% / 7.15256e-05% | 20260507 03:54:10.503 | INFO | Baz IO overall: | 20260507 03:54:10.503 | INFO | BytesWritten : 3527311678332 | 20260507 03:54:10.503 | INFO | NumEvents : 1694210761139 | 20260507 03:54:12.346 | INFO | Analyzing chunk frames = [6889472,6889984) | 20260507 03:54:12.347 | INFO | MemInfo: { | 20260507 03:54:12.347 | INFO | "gpu" : " 17 GB", | 20260507 03:54:12.347 | INFO | "huge" : "", | 20260507 03:54:12.347 | INFO | "huge_shared" : " 27 GB", | 20260507 03:54:12.347 | INFO | "malloc" : " 14 GB", | 20260507 03:54:12.347 | INFO | "pinned" : " 29 GB" | 20260507 03:54:12.347 | INFO | } | 20260507 03:54:12.347 | INFO | Perf report for the last 5.14225 seconds: | 20260507 03:54:12.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:54:12.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:12.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:12.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:12.347 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 03:54:12.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:12.347 | INFO | RT_METRICS : 43.4%, 1.0, 22.8ms, 0 | 20260507 03:54:12.347 | INFO | PRE_HQ : 59.6%, 1.9, 57.9ms, 0 | 20260507 03:54:12.347 | INFO | BAZWRITER : 61.9%, 3.3, 106.5ms, 0 | 20260507 03:54:12.347 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:54:12.347 | INFO | Uploaded 36GiB to GPU in 2.0309s (17.7261GiB/s) | 20260507 03:54:12.347 | INFO | Downloaded 38GiB to GPU in 2.18129s (17.4209GiB/s) | 20260507 03:54:17.487 | INFO | Analyzing chunk frames = [6889984,6890496) | 20260507 03:54:17.487 | INFO | MemInfo: { | 20260507 03:54:17.487 | INFO | "gpu" : " 17 GB", | 20260507 03:54:17.487 | INFO | "huge" : "", | 20260507 03:54:17.487 | INFO | "huge_shared" : " 27 GB", | 20260507 03:54:17.488 | INFO | "malloc" : " 14 GB", | 20260507 03:54:17.488 | INFO | "pinned" : " 29 GB" | 20260507 03:54:17.488 | INFO | } | 20260507 03:54:17.488 | INFO | Perf report for the last 5.1411 seconds: | 20260507 03:54:17.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:54:17.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:17.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:17.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:17.488 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:54:17.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:17.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:17.488 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:54:17.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:17.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:17.488 | INFO | Uploaded 36GiB to GPU in 2.00107s (17.9904GiB/s) | 20260507 03:54:17.488 | INFO | Downloaded 34GiB to GPU in 1.93196s (17.5987GiB/s) | 20260507 03:54:22.619 | INFO | Analyzing chunk frames = [6890496,6891008) | 20260507 03:54:22.619 | INFO | MemInfo: { | 20260507 03:54:22.619 | INFO | "gpu" : " 17 GB", | 20260507 03:54:22.619 | INFO | "huge" : "", | 20260507 03:54:22.619 | INFO | "huge_shared" : " 27 GB", | 20260507 03:54:22.619 | INFO | "malloc" : " 14 GB", | 20260507 03:54:22.620 | INFO | "pinned" : " 29 GB" | 20260507 03:54:22.620 | INFO | } | 20260507 03:54:22.620 | INFO | Perf report for the last 5.13189 seconds: | 20260507 03:54:22.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:54:22.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:22.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:22.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:22.620 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 03:54:22.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:22.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:22.620 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:54:22.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:22.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:22.620 | INFO | Uploaded 36GiB to GPU in 2.03212s (17.7155GiB/s) | 20260507 03:54:22.620 | INFO | Downloaded 34GiB to GPU in 1.94963s (17.4392GiB/s) | 20260507 03:54:27.764 | INFO | Analyzing chunk frames = [6891008,6891520) | 20260507 03:54:27.764 | INFO | MemInfo: { | 20260507 03:54:27.764 | INFO | "gpu" : " 17 GB", | 20260507 03:54:27.764 | INFO | "huge" : "", | 20260507 03:54:27.764 | INFO | "huge_shared" : " 27 GB", | 20260507 03:54:27.764 | INFO | "malloc" : " 14 GB", | 20260507 03:54:27.764 | INFO | "pinned" : " 29 GB" | 20260507 03:54:27.764 | INFO | } | 20260507 03:54:27.764 | INFO | Perf report for the last 5.14475 seconds: | 20260507 03:54:27.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:54:27.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:27.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:27.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:27.764 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 03:54:27.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:27.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:27.764 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:54:27.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:27.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:27.765 | INFO | Uploaded 36GiB to GPU in 1.97368s (18.24GiB/s) | 20260507 03:54:27.765 | INFO | Downloaded 34GiB to GPU in 1.90982s (17.8027GiB/s) | 20260507 03:54:32.908 | INFO | Analyzing chunk frames = [6891520,6892032) | 20260507 03:54:32.908 | INFO | MemInfo: { | 20260507 03:54:32.908 | INFO | "gpu" : " 17 GB", | 20260507 03:54:32.908 | INFO | "huge" : "", | 20260507 03:54:32.908 | INFO | "huge_shared" : " 27 GB", | 20260507 03:54:32.908 | INFO | "malloc" : " 14 GB", | 20260507 03:54:32.908 | INFO | "pinned" : " 29 GB" | 20260507 03:54:32.908 | INFO | } | 20260507 03:54:32.908 | INFO | Perf report for the last 5.14376 seconds: | 20260507 03:54:32.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:54:32.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:32.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:32.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:32.908 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 03:54:32.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:32.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:32.908 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:54:32.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:32.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:32.908 | INFO | Uploaded 36GiB to GPU in 2.02752s (17.7557GiB/s) | 20260507 03:54:32.908 | INFO | Downloaded 34GiB to GPU in 1.93198s (17.5985GiB/s) | 20260507 03:54:38.050 | INFO | Analyzing chunk frames = [6892032,6892544) | 20260507 03:54:38.050 | INFO | MemInfo: { | 20260507 03:54:38.050 | INFO | "gpu" : " 17 GB", | 20260507 03:54:38.050 | INFO | "huge" : "", | 20260507 03:54:38.050 | INFO | "huge_shared" : " 27 GB", | 20260507 03:54:38.050 | INFO | "malloc" : " 14 GB", | 20260507 03:54:38.050 | INFO | "pinned" : " 29 GB" | 20260507 03:54:38.050 | INFO | } | 20260507 03:54:38.050 | INFO | Perf report for the last 5.14198 seconds: | 20260507 03:54:38.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:54:38.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:38.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:38.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:38.050 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 03:54:38.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:38.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:38.050 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:54:38.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:38.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:38.050 | INFO | Uploaded 36GiB to GPU in 2.03681s (17.6747GiB/s) | 20260507 03:54:38.050 | INFO | Downloaded 34GiB to GPU in 1.89866s (17.9074GiB/s) | 20260507 03:54:43.185 | INFO | Analyzing chunk frames = [6892544,6893056) | 20260507 03:54:43.185 | INFO | MemInfo: { | 20260507 03:54:43.185 | INFO | "gpu" : " 17 GB", | 20260507 03:54:43.185 | INFO | "huge" : "", | 20260507 03:54:43.185 | INFO | "huge_shared" : " 27 GB", | 20260507 03:54:43.185 | INFO | "malloc" : " 14 GB", | 20260507 03:54:43.185 | INFO | "pinned" : " 29 GB" | 20260507 03:54:43.185 | INFO | } | 20260507 03:54:43.185 | INFO | Perf report for the last 5.13501 seconds: | 20260507 03:54:43.185 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:54:43.185 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:43.185 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:43.185 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:43.185 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:54:43.185 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:43.185 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:43.185 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 03:54:43.185 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:43.185 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:43.185 | INFO | Uploaded 36GiB to GPU in 1.92623s (18.6894GiB/s) | 20260507 03:54:43.186 | INFO | Downloaded 34GiB to GPU in 1.96908s (17.2669GiB/s) | 20260507 03:54:48.322 | INFO | Analyzing chunk frames = [6893056,6893568) | 20260507 03:54:48.322 | INFO | MemInfo: { | 20260507 03:54:48.322 | INFO | "gpu" : " 17 GB", | 20260507 03:54:48.322 | INFO | "huge" : "", | 20260507 03:54:48.322 | INFO | "huge_shared" : " 27 GB", | 20260507 03:54:48.322 | INFO | "malloc" : " 14 GB", | 20260507 03:54:48.322 | INFO | "pinned" : " 29 GB" | 20260507 03:54:48.323 | INFO | } | 20260507 03:54:48.323 | INFO | Perf report for the last 5.13749 seconds: | 20260507 03:54:48.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:54:48.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:48.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:48.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:48.323 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 03:54:48.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:48.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:48.323 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 03:54:48.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:48.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:48.323 | INFO | Uploaded 36GiB to GPU in 1.97685s (18.2108GiB/s) | 20260507 03:54:48.323 | INFO | Downloaded 34GiB to GPU in 1.95231s (17.4153GiB/s) | 20260507 03:54:48.419 | INFO | Starting a round of Baz IO | 20260507 03:54:51.527 | INFO | Populating RTMetrics report | 20260507 03:54:51.527 | INFO | RTMetrics report populated | 20260507 03:54:51.527 | INFO | Opening tmp RTMetrics json file | 20260507 03:54:51.532 | INFO | Serializing RTMetrics to json | 20260507 03:54:51.546 | INFO | Serialization done, swapping tmp file now... | 20260507 03:54:51.546 | INFO | RTMetrics done | 20260507 03:54:51.618 | INFO | Finished a round of Baz IO: Wrote 1.741 GiB in 3.199 seconds | 20260507 03:54:51.619 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.84% / 0.00% / 67.15% | 20260507 03:54:51.619 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.18516% / 3.97364e-06% / 4.37101e-05% | 20260507 03:54:51.619 | INFO | Baz IO overall: | 20260507 03:54:51.619 | INFO | BytesWritten : 3529180703290 | 20260507 03:54:51.619 | INFO | NumEvents : 1695092087513 | 20260507 03:54:53.465 | INFO | Analyzing chunk frames = [6893568,6894080) | 20260507 03:54:53.465 | INFO | MemInfo: { | 20260507 03:54:53.465 | INFO | "gpu" : " 17 GB", | 20260507 03:54:53.465 | INFO | "huge" : "", | 20260507 03:54:53.465 | INFO | "huge_shared" : " 27 GB", | 20260507 03:54:53.465 | INFO | "malloc" : " 14 GB", | 20260507 03:54:53.465 | INFO | "pinned" : " 29 GB" | 20260507 03:54:53.465 | INFO | } | 20260507 03:54:53.465 | INFO | Perf report for the last 5.14263 seconds: | 20260507 03:54:53.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:54:53.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:53.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:53.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:53.465 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 03:54:53.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:53.465 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 03:54:53.465 | INFO | PRE_HQ : 59.3%, 1.8, 55.4ms, 0 | 20260507 03:54:53.465 | INFO | BAZWRITER : 61.8%, 3.3, 103.6ms, 0 | 20260507 03:54:53.465 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:54:53.466 | INFO | Uploaded 36GiB to GPU in 2.04202s (17.6296GiB/s) | 20260507 03:54:53.466 | INFO | Downloaded 38GiB to GPU in 2.18157s (17.4187GiB/s) | 20260507 03:54:58.596 | INFO | Analyzing chunk frames = [6894080,6894592) | 20260507 03:54:58.596 | INFO | MemInfo: { | 20260507 03:54:58.596 | INFO | "gpu" : " 17 GB", | 20260507 03:54:58.596 | INFO | "huge" : "", | 20260507 03:54:58.596 | INFO | "huge_shared" : " 27 GB", | 20260507 03:54:58.596 | INFO | "malloc" : " 14 GB", | 20260507 03:54:58.596 | INFO | "pinned" : " 29 GB" | 20260507 03:54:58.596 | INFO | } | 20260507 03:54:58.597 | INFO | Perf report for the last 5.1313 seconds: | 20260507 03:54:58.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:54:58.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:58.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:58.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:58.597 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260507 03:54:58.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:58.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:58.597 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:54:58.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:58.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:54:58.597 | INFO | Uploaded 36GiB to GPU in 2.05155s (17.5477GiB/s) | 20260507 03:54:58.597 | INFO | Downloaded 34GiB to GPU in 1.93801s (17.5438GiB/s) | 20260507 03:55:03.735 | INFO | Analyzing chunk frames = [6894592,6895104) | 20260507 03:55:03.735 | INFO | MemInfo: { | 20260507 03:55:03.735 | INFO | "gpu" : " 17 GB", | 20260507 03:55:03.735 | INFO | "huge" : "", | 20260507 03:55:03.736 | INFO | "huge_shared" : " 27 GB", | 20260507 03:55:03.736 | INFO | "malloc" : " 14 GB", | 20260507 03:55:03.736 | INFO | "pinned" : " 29 GB" | 20260507 03:55:03.736 | INFO | } | 20260507 03:55:03.736 | INFO | Perf report for the last 5.13913 seconds: | 20260507 03:55:03.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:55:03.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:03.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:03.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:03.736 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 03:55:03.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:03.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:03.736 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 03:55:03.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:03.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:03.736 | INFO | Uploaded 36GiB to GPU in 2.0135s (17.8793GiB/s) | 20260507 03:55:03.736 | INFO | Downloaded 34GiB to GPU in 1.93846s (17.5397GiB/s) | 20260507 03:55:08.878 | INFO | Analyzing chunk frames = [6895104,6895616) | 20260507 03:55:08.878 | INFO | MemInfo: { | 20260507 03:55:08.879 | INFO | "gpu" : " 17 GB", | 20260507 03:55:08.879 | INFO | "huge" : "", | 20260507 03:55:08.879 | INFO | "huge_shared" : " 27 GB", | 20260507 03:55:08.879 | INFO | "malloc" : " 14 GB", | 20260507 03:55:08.879 | INFO | "pinned" : " 29 GB" | 20260507 03:55:08.879 | INFO | } | 20260507 03:55:08.879 | INFO | Perf report for the last 5.14302 seconds: | 20260507 03:55:08.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:55:08.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:08.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:08.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:08.879 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 03:55:08.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:08.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:08.879 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:55:08.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:08.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:08.879 | INFO | Uploaded 36GiB to GPU in 1.96996s (18.2745GiB/s) | 20260507 03:55:08.879 | INFO | Downloaded 34GiB to GPU in 1.89956s (17.8989GiB/s) | 20260507 03:55:14.022 | INFO | Analyzing chunk frames = [6895616,6896128) | 20260507 03:55:14.022 | INFO | MemInfo: { | 20260507 03:55:14.022 | INFO | "gpu" : " 17 GB", | 20260507 03:55:14.022 | INFO | "huge" : "", | 20260507 03:55:14.022 | INFO | "huge_shared" : " 27 GB", | 20260507 03:55:14.022 | INFO | "malloc" : " 14 GB", | 20260507 03:55:14.022 | INFO | "pinned" : " 29 GB" | 20260507 03:55:14.022 | INFO | } | 20260507 03:55:14.022 | INFO | Perf report for the last 5.14369 seconds: | 20260507 03:55:14.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:55:14.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:14.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:14.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:14.022 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 03:55:14.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:14.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:14.023 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:55:14.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:14.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:14.023 | INFO | Uploaded 36GiB to GPU in 2.03529s (17.6879GiB/s) | 20260507 03:55:14.023 | INFO | Downloaded 34GiB to GPU in 1.93261s (17.5928GiB/s) | 20260507 03:55:19.160 | INFO | Analyzing chunk frames = [6896128,6896640) | 20260507 03:55:19.160 | INFO | MemInfo: { | 20260507 03:55:19.160 | INFO | "gpu" : " 17 GB", | 20260507 03:55:19.160 | INFO | "huge" : "", | 20260507 03:55:19.160 | INFO | "huge_shared" : " 27 GB", | 20260507 03:55:19.160 | INFO | "malloc" : " 14 GB", | 20260507 03:55:19.160 | INFO | "pinned" : " 29 GB" | 20260507 03:55:19.160 | INFO | } | 20260507 03:55:19.160 | INFO | Perf report for the last 5.13797 seconds: | 20260507 03:55:19.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:55:19.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:19.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:19.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:19.160 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 03:55:19.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:19.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:19.161 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:55:19.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:19.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:19.161 | INFO | Uploaded 36GiB to GPU in 2.04298s (17.6213GiB/s) | 20260507 03:55:19.161 | INFO | Downloaded 34GiB to GPU in 1.91228s (17.7798GiB/s) | 20260507 03:55:24.296 | INFO | Analyzing chunk frames = [6896640,6897152) | 20260507 03:55:24.296 | INFO | MemInfo: { | 20260507 03:55:24.296 | INFO | "gpu" : " 17 GB", | 20260507 03:55:24.296 | INFO | "huge" : "", | 20260507 03:55:24.296 | INFO | "huge_shared" : " 27 GB", | 20260507 03:55:24.296 | INFO | "malloc" : " 14 GB", | 20260507 03:55:24.296 | INFO | "pinned" : " 29 GB" | 20260507 03:55:24.296 | INFO | } | 20260507 03:55:24.296 | INFO | Perf report for the last 5.13584 seconds: | 20260507 03:55:24.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:55:24.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:24.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:24.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:24.296 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:55:24.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:24.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:24.296 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 03:55:24.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:24.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:24.297 | INFO | Uploaded 36GiB to GPU in 2.0115s (17.8971GiB/s) | 20260507 03:55:24.297 | INFO | Downloaded 34GiB to GPU in 1.98346s (17.1418GiB/s) | 20260507 03:55:29.430 | INFO | Analyzing chunk frames = [6897152,6897664) | 20260507 03:55:29.430 | INFO | MemInfo: { | 20260507 03:55:29.430 | INFO | "gpu" : " 17 GB", | 20260507 03:55:29.430 | INFO | "huge" : "", | 20260507 03:55:29.430 | INFO | "huge_shared" : " 27 GB", | 20260507 03:55:29.430 | INFO | "malloc" : " 14 GB", | 20260507 03:55:29.430 | INFO | "pinned" : " 29 GB" | 20260507 03:55:29.430 | INFO | } | 20260507 03:55:29.430 | INFO | Perf report for the last 5.13398 seconds: | 20260507 03:55:29.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:55:29.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:29.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:29.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:29.430 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 03:55:29.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:29.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:29.430 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:55:29.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:29.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:29.431 | INFO | Uploaded 36GiB to GPU in 1.97433s (18.2341GiB/s) | 20260507 03:55:29.431 | INFO | Downloaded 34GiB to GPU in 1.9626s (17.3239GiB/s) | 20260507 03:55:29.431 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:55:29.431 | INFO | Total mem usage: 18 GiB | 20260507 03:55:29.431 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:55:29.431 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:55:29.431 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:55:29.431 | INFO | High Water Breakdown: | 20260507 03:55:29.431 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:55:29.431 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:55:29.431 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:55:29.431 | INFO | ------------------------------------------ | 20260507 03:55:29.431 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:55:29.431 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:55:29.431 | INFO | Total mem usage: 26.5 GiB | 20260507 03:55:29.431 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:55:29.431 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:55:29.431 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:55:29.431 | INFO | High Water Breakdown: | 20260507 03:55:29.431 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:55:29.431 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:55:29.431 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:55:29.431 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:55:29.431 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:55:29.431 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:55:29.431 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:55:29.431 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:55:29.431 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:55:29.431 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:55:29.431 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:55:29.431 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:55:29.431 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:55:29.431 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:55:29.431 | INFO | ------------------------------------------ | 20260507 03:55:29.431 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:55:29.431 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:55:29.431 | INFO | Total mem usage: 25.6 GiB | 20260507 03:55:29.431 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:55:29.431 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:55:29.431 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:55:29.431 | INFO | High Water Breakdown: | 20260507 03:55:29.431 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:55:29.431 | INFO | ------------------------------------------ | 20260507 03:55:29.431 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:55:29.431 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:55:29.431 | INFO | Total mem usage: 15.7 GiB | 20260507 03:55:29.431 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:55:29.431 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:55:29.431 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:55:29.431 | INFO | High Water Breakdown: | 20260507 03:55:29.431 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:55:29.431 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:55:29.431 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:55:29.431 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:55:29.431 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:55:29.431 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:55:29.431 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:55:29.431 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:55:29.432 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:55:29.432 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:55:29.432 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:55:29.432 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:55:29.432 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:55:29.432 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:55:29.432 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:55:29.432 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:55:29.432 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:55:29.432 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:55:29.432 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:55:29.432 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:55:29.432 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:55:29.432 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:55:29.432 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:55:29.432 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:55:29.432 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:55:29.432 | INFO | ------------------------------------------ | 20260507 03:55:29.432 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:55:29.432 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:55:29.432 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:55:29.432 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:55:29.432 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.49e+01, 1.31e+01 / 1.28e+00 | 20260507 03:55:29.432 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.00e-01, 9.44e-03 / 4.92e-03 | 20260507 03:55:29.432 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.42e+00, 5.17e-01 / 3.01e-01 | 20260507 03:55:29.432 | INFO | PerfReport: 3.24%, Histogram -- 9.96e-01 / 1.45e+00, 1.70e-01 / 1.23e-01 | 20260507 03:55:29.432 | INFO | PerfReport: 16.12%, DME -- 3.97e+01 / 4.44e+01, 4.05e+00 / 1.89e+00 | 20260507 03:55:29.432 | INFO | PerfReport: 57.67%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.32e+00 / 1.79e+00 | 20260507 03:55:29.432 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.65e+00, 3.77e-01 / 2.53e-01 | 20260507 03:55:29.432 | INFO | PerfReport: 3.04%, Metrics -- 9.35e-01 / 1.25e+00, 2.48e-01 / 1.48e-01 | 20260507 03:55:29.432 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.15e-01 / 3.70e-01, 4.98e-02 / 2.41e-02 | 20260507 03:55:29.432 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:55:29.432 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:55:29.432 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:55:29.432 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 03:55:29.432 | INFO | PerfReport: 21.75%, Upload -- 2.09e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260507 03:55:29.432 | INFO | PerfReport: 21.30%, Download -- 2.05e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 03:55:29.432 | INFO | PerfReport: 31.99%, Compute -- 3.08e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 03:55:29.432 | INFO | PerfReport: 1.49%, LockUpload -- 1.44e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 03:55:29.432 | INFO | PerfReport: 1.24%, LockDownload -- 1.19e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260507 03:55:29.432 | INFO | PerfReport: 22.22%, LockCompute -- 2.14e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 03:55:29.432 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260507 03:55:29.527 | INFO | Starting a round of Baz IO | 20260507 03:55:32.633 | INFO | Populating RTMetrics report | 20260507 03:55:32.633 | INFO | RTMetrics report populated | 20260507 03:55:32.633 | INFO | Opening tmp RTMetrics json file | 20260507 03:55:32.634 | INFO | Serializing RTMetrics to json | 20260507 03:55:32.647 | INFO | Serialization done, swapping tmp file now... | 20260507 03:55:32.654 | INFO | RTMetrics done | 20260507 03:55:32.725 | INFO | Finished a round of Baz IO: Wrote 1.74 GiB in 3.197 seconds | 20260507 03:55:32.725 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.84% / 0.00% / 67.16% | 20260507 03:55:32.725 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.184353% / 0% / 6.75519e-05% | 20260507 03:55:32.725 | INFO | Baz IO overall: | 20260507 03:55:32.725 | INFO | BytesWritten : 3531048624465 | 20260507 03:55:32.725 | INFO | NumEvents : 1695972705621 | 20260507 03:55:34.573 | INFO | Analyzing chunk frames = [6897664,6898176) | 20260507 03:55:34.573 | INFO | MemInfo: { | 20260507 03:55:34.573 | INFO | "gpu" : " 17 GB", | 20260507 03:55:34.573 | INFO | "huge" : "", | 20260507 03:55:34.573 | INFO | "huge_shared" : " 27 GB", | 20260507 03:55:34.573 | INFO | "malloc" : " 14 GB", | 20260507 03:55:34.573 | INFO | "pinned" : " 29 GB" | 20260507 03:55:34.573 | INFO | } | 20260507 03:55:34.573 | INFO | Perf report for the last 5.14266 seconds: | 20260507 03:55:34.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:55:34.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:34.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:34.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:34.573 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 03:55:34.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:34.573 | INFO | RT_METRICS : 42.2%, 1.0, 22.1ms, 0 | 20260507 03:55:34.573 | INFO | PRE_HQ : 59.3%, 1.8, 55.8ms, 0 | 20260507 03:55:34.573 | INFO | BAZWRITER : 61.3%, 3.3, 104.8ms, 0 | 20260507 03:55:34.573 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:55:34.573 | INFO | Uploaded 36GiB to GPU in 2.03743s (17.6694GiB/s) | 20260507 03:55:34.573 | INFO | Downloaded 38GiB to GPU in 2.18207s (17.4147GiB/s) | 20260507 03:55:39.714 | INFO | Analyzing chunk frames = [6898176,6898688) | 20260507 03:55:39.714 | INFO | MemInfo: { | 20260507 03:55:39.714 | INFO | "gpu" : " 17 GB", | 20260507 03:55:39.714 | INFO | "huge" : "", | 20260507 03:55:39.714 | INFO | "huge_shared" : " 27 GB", | 20260507 03:55:39.714 | INFO | "malloc" : " 14 GB", | 20260507 03:55:39.714 | INFO | "pinned" : " 29 GB" | 20260507 03:55:39.714 | INFO | } | 20260507 03:55:39.714 | INFO | Perf report for the last 5.14127 seconds: | 20260507 03:55:39.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:55:39.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:39.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:39.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:39.714 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:55:39.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:39.714 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:39.714 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:55:39.714 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:39.714 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:39.715 | INFO | Uploaded 36GiB to GPU in 2.03435s (17.696GiB/s) | 20260507 03:55:39.715 | INFO | Downloaded 34GiB to GPU in 1.94121s (17.5148GiB/s) | 20260507 03:55:44.858 | INFO | Analyzing chunk frames = [6898688,6899200) | 20260507 03:55:44.858 | INFO | MemInfo: { | 20260507 03:55:44.858 | INFO | "gpu" : " 17 GB", | 20260507 03:55:44.858 | INFO | "huge" : "", | 20260507 03:55:44.858 | INFO | "huge_shared" : " 27 GB", | 20260507 03:55:44.858 | INFO | "malloc" : " 14 GB", | 20260507 03:55:44.858 | INFO | "pinned" : " 29 GB" | 20260507 03:55:44.858 | INFO | } | 20260507 03:55:44.858 | INFO | Perf report for the last 5.14421 seconds: | 20260507 03:55:44.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:55:44.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:44.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:44.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:44.859 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 03:55:44.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:44.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:44.859 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:55:44.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:44.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:44.859 | INFO | Uploaded 36GiB to GPU in 2.01966s (17.8248GiB/s) | 20260507 03:55:44.859 | INFO | Downloaded 34GiB to GPU in 1.93676s (17.5551GiB/s) | 20260507 03:55:49.997 | INFO | Analyzing chunk frames = [6899200,6899712) | 20260507 03:55:49.997 | INFO | MemInfo: { | 20260507 03:55:49.997 | INFO | "gpu" : " 17 GB", | 20260507 03:55:49.997 | INFO | "huge" : "", | 20260507 03:55:49.997 | INFO | "huge_shared" : " 27 GB", | 20260507 03:55:49.997 | INFO | "malloc" : " 14 GB", | 20260507 03:55:49.997 | INFO | "pinned" : " 29 GB" | 20260507 03:55:49.997 | INFO | } | 20260507 03:55:49.998 | INFO | Perf report for the last 5.13905 seconds: | 20260507 03:55:49.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:55:49.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:49.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:49.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:49.998 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 03:55:49.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:49.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:49.998 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:55:49.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:49.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:49.998 | INFO | Uploaded 36GiB to GPU in 2.02252s (17.7996GiB/s) | 20260507 03:55:49.998 | INFO | Downloaded 34GiB to GPU in 1.93946s (17.5306GiB/s) | 20260507 03:55:55.135 | INFO | Analyzing chunk frames = [6899712,6900224) | 20260507 03:55:55.135 | INFO | MemInfo: { | 20260507 03:55:55.135 | INFO | "gpu" : " 17 GB", | 20260507 03:55:55.135 | INFO | "huge" : "", | 20260507 03:55:55.135 | INFO | "huge_shared" : " 27 GB", | 20260507 03:55:55.135 | INFO | "malloc" : " 14 GB", | 20260507 03:55:55.135 | INFO | "pinned" : " 29 GB" | 20260507 03:55:55.135 | INFO | } | 20260507 03:55:55.135 | INFO | Perf report for the last 5.13786 seconds: | 20260507 03:55:55.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:55:55.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:55.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:55.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:55.135 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 03:55:55.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:55.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:55.136 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:55:55.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:55.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:55:55.136 | INFO | Uploaded 36GiB to GPU in 2.05624s (17.5076GiB/s) | 20260507 03:55:55.136 | INFO | Downloaded 34GiB to GPU in 1.94806s (17.4532GiB/s) | 20260507 03:56:00.279 | INFO | Analyzing chunk frames = [6900224,6900736) | 20260507 03:56:00.279 | INFO | MemInfo: { | 20260507 03:56:00.279 | INFO | "gpu" : " 17 GB", | 20260507 03:56:00.279 | INFO | "huge" : "", | 20260507 03:56:00.280 | INFO | "huge_shared" : " 27 GB", | 20260507 03:56:00.280 | INFO | "malloc" : " 14 GB", | 20260507 03:56:00.280 | INFO | "pinned" : " 29 GB" | 20260507 03:56:00.280 | INFO | } | 20260507 03:56:00.280 | INFO | Perf report for the last 5.14435 seconds: | 20260507 03:56:00.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:56:00.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:00.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:00.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:00.280 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 03:56:00.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:00.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:00.280 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 03:56:00.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:00.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:00.280 | INFO | Uploaded 36GiB to GPU in 2.0199s (17.8227GiB/s) | 20260507 03:56:00.280 | INFO | Downloaded 34GiB to GPU in 1.91722s (17.734GiB/s) | 20260507 03:56:05.412 | INFO | Analyzing chunk frames = [6900736,6901248) | 20260507 03:56:05.412 | INFO | MemInfo: { | 20260507 03:56:05.412 | INFO | "gpu" : " 17 GB", | 20260507 03:56:05.412 | INFO | "huge" : "", | 20260507 03:56:05.412 | INFO | "huge_shared" : " 27 GB", | 20260507 03:56:05.412 | INFO | "malloc" : " 14 GB", | 20260507 03:56:05.412 | INFO | "pinned" : " 29 GB" | 20260507 03:56:05.412 | INFO | } | 20260507 03:56:05.412 | INFO | Perf report for the last 5.13241 seconds: | 20260507 03:56:05.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:56:05.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:05.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:05.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:05.412 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 03:56:05.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:05.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:05.412 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 03:56:05.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:05.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:05.413 | INFO | Uploaded 36GiB to GPU in 1.97221s (18.2536GiB/s) | 20260507 03:56:05.413 | INFO | Downloaded 34GiB to GPU in 1.96984s (17.2603GiB/s) | 20260507 03:56:10.549 | INFO | Analyzing chunk frames = [6901248,6901760) | 20260507 03:56:10.549 | INFO | MemInfo: { | 20260507 03:56:10.549 | INFO | "gpu" : " 17 GB", | 20260507 03:56:10.549 | INFO | "huge" : "", | 20260507 03:56:10.549 | INFO | "huge_shared" : " 27 GB", | 20260507 03:56:10.549 | INFO | "malloc" : " 14 GB", | 20260507 03:56:10.549 | INFO | "pinned" : " 29 GB" | 20260507 03:56:10.549 | INFO | } | 20260507 03:56:10.550 | INFO | Perf report for the last 5.13731 seconds: | 20260507 03:56:10.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:56:10.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:10.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:10.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:10.550 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:56:10.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:10.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:10.550 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 03:56:10.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:10.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:10.550 | INFO | Uploaded 36GiB to GPU in 2.00825s (17.926GiB/s) | 20260507 03:56:10.550 | INFO | Downloaded 34GiB to GPU in 1.97097s (17.2504GiB/s) | 20260507 03:56:10.647 | INFO | Starting a round of Baz IO | 20260507 03:56:13.744 | INFO | Populating RTMetrics report | 20260507 03:56:13.745 | INFO | RTMetrics report populated | 20260507 03:56:13.745 | INFO | Opening tmp RTMetrics json file | 20260507 03:56:13.750 | INFO | Serializing RTMetrics to json | 20260507 03:56:13.763 | INFO | Serialization done, swapping tmp file now... | 20260507 03:56:13.764 | INFO | RTMetrics done | 20260507 03:56:13.837 | INFO | Finished a round of Baz IO: Wrote 1.739 GiB in 3.189 seconds | 20260507 03:56:13.837 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.84% / 0.00% / 67.16% | 20260507 03:56:13.837 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.184516% / 3.97364e-06% / 7.15256e-05% | 20260507 03:56:13.837 | INFO | Baz IO overall: | 20260507 03:56:13.837 | INFO | BytesWritten : 3532916240990 | 20260507 03:56:13.837 | INFO | NumEvents : 1696852947576 | 20260507 03:56:15.685 | INFO | Analyzing chunk frames = [6901760,6902272) | 20260507 03:56:15.685 | INFO | MemInfo: { | 20260507 03:56:15.685 | INFO | "gpu" : " 17 GB", | 20260507 03:56:15.685 | INFO | "huge" : "", | 20260507 03:56:15.685 | INFO | "huge_shared" : " 27 GB", | 20260507 03:56:15.686 | INFO | "malloc" : " 14 GB", | 20260507 03:56:15.686 | INFO | "pinned" : " 29 GB" | 20260507 03:56:15.686 | INFO | } | 20260507 03:56:15.686 | INFO | Perf report for the last 5.13612 seconds: | 20260507 03:56:15.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:56:15.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:15.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:15.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:15.686 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 03:56:15.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:15.686 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 03:56:15.686 | INFO | PRE_HQ : 59.0%, 1.8, 55.3ms, 0 | 20260507 03:56:15.686 | INFO | BAZWRITER : 61.7%, 3.3, 104.8ms, 0 | 20260507 03:56:15.686 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:56:15.686 | INFO | Uploaded 36GiB to GPU in 2.01879s (17.8325GiB/s) | 20260507 03:56:15.686 | INFO | Downloaded 38GiB to GPU in 2.17475s (17.4733GiB/s) | 20260507 03:56:20.831 | INFO | Analyzing chunk frames = [6902272,6902784) | 20260507 03:56:20.831 | INFO | MemInfo: { | 20260507 03:56:20.831 | INFO | "gpu" : " 17 GB", | 20260507 03:56:20.831 | INFO | "huge" : "", | 20260507 03:56:20.831 | INFO | "huge_shared" : " 27 GB", | 20260507 03:56:20.831 | INFO | "malloc" : " 14 GB", | 20260507 03:56:20.831 | INFO | "pinned" : " 29 GB" | 20260507 03:56:20.831 | INFO | } | 20260507 03:56:20.831 | INFO | Perf report for the last 5.14507 seconds: | 20260507 03:56:20.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:56:20.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:20.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:20.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:20.831 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 03:56:20.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:20.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:20.831 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:56:20.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:20.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:20.831 | INFO | Uploaded 36GiB to GPU in 2.01687s (17.8494GiB/s) | 20260507 03:56:20.832 | INFO | Downloaded 34GiB to GPU in 1.93616s (17.5606GiB/s) | 20260507 03:56:25.971 | INFO | Analyzing chunk frames = [6902784,6903296) | 20260507 03:56:25.971 | INFO | MemInfo: { | 20260507 03:56:25.971 | INFO | "gpu" : " 17 GB", | 20260507 03:56:25.971 | INFO | "huge" : "", | 20260507 03:56:25.971 | INFO | "huge_shared" : " 27 GB", | 20260507 03:56:25.971 | INFO | "malloc" : " 14 GB", | 20260507 03:56:25.971 | INFO | "pinned" : " 29 GB" | 20260507 03:56:25.971 | INFO | } | 20260507 03:56:25.971 | INFO | Perf report for the last 5.14021 seconds: | 20260507 03:56:25.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:56:25.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:25.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:25.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:25.971 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 03:56:25.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:25.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:25.971 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:56:25.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:25.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:25.971 | INFO | Uploaded 36GiB to GPU in 2.00689s (17.9382GiB/s) | 20260507 03:56:25.971 | INFO | Downloaded 34GiB to GPU in 1.93671s (17.5555GiB/s) | 20260507 03:56:31.110 | INFO | Analyzing chunk frames = [6903296,6903808) | 20260507 03:56:31.110 | INFO | MemInfo: { | 20260507 03:56:31.110 | INFO | "gpu" : " 17 GB", | 20260507 03:56:31.110 | INFO | "huge" : "", | 20260507 03:56:31.110 | INFO | "huge_shared" : " 27 GB", | 20260507 03:56:31.110 | INFO | "malloc" : " 14 GB", | 20260507 03:56:31.110 | INFO | "pinned" : " 29 GB" | 20260507 03:56:31.110 | INFO | } | 20260507 03:56:31.110 | INFO | Perf report for the last 5.13921 seconds: | 20260507 03:56:31.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:56:31.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:31.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:31.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:31.110 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 03:56:31.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:31.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:31.110 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:56:31.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:31.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:31.110 | INFO | Uploaded 36GiB to GPU in 2.0106s (17.9051GiB/s) | 20260507 03:56:31.111 | INFO | Downloaded 34GiB to GPU in 1.93899s (17.5349GiB/s) | 20260507 03:56:36.250 | INFO | Analyzing chunk frames = [6903808,6904320) | 20260507 03:56:36.250 | INFO | MemInfo: { | 20260507 03:56:36.250 | INFO | "gpu" : " 17 GB", | 20260507 03:56:36.250 | INFO | "huge" : "", | 20260507 03:56:36.250 | INFO | "huge_shared" : " 27 GB", | 20260507 03:56:36.250 | INFO | "malloc" : " 14 GB", | 20260507 03:56:36.250 | INFO | "pinned" : " 29 GB" | 20260507 03:56:36.250 | INFO | } | 20260507 03:56:36.250 | INFO | Perf report for the last 5.13952 seconds: | 20260507 03:56:36.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:56:36.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:36.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:36.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:36.250 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 03:56:36.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:36.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:36.250 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:56:36.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:36.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:36.250 | INFO | Uploaded 36GiB to GPU in 2.0583s (17.4902GiB/s) | 20260507 03:56:36.250 | INFO | Downloaded 34GiB to GPU in 1.94938s (17.4414GiB/s) | 20260507 03:56:41.381 | INFO | Analyzing chunk frames = [6904320,6904832) | 20260507 03:56:41.381 | INFO | MemInfo: { | 20260507 03:56:41.381 | INFO | "gpu" : " 17 GB", | 20260507 03:56:41.381 | INFO | "huge" : "", | 20260507 03:56:41.381 | INFO | "huge_shared" : " 27 GB", | 20260507 03:56:41.381 | INFO | "malloc" : " 14 GB", | 20260507 03:56:41.381 | INFO | "pinned" : " 29 GB" | 20260507 03:56:41.381 | INFO | } | 20260507 03:56:41.381 | INFO | Perf report for the last 5.13108 seconds: | 20260507 03:56:41.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:56:41.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:41.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:41.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:41.381 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 03:56:41.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:41.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:41.381 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:56:41.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:41.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:41.381 | INFO | Uploaded 36GiB to GPU in 2.05564s (17.5128GiB/s) | 20260507 03:56:41.381 | INFO | Downloaded 34GiB to GPU in 1.9653s (17.3002GiB/s) | 20260507 03:56:46.525 | INFO | Analyzing chunk frames = [6904832,6905344) | 20260507 03:56:46.525 | INFO | MemInfo: { | 20260507 03:56:46.525 | INFO | "gpu" : " 17 GB", | 20260507 03:56:46.525 | INFO | "huge" : "", | 20260507 03:56:46.525 | INFO | "huge_shared" : " 27 GB", | 20260507 03:56:46.525 | INFO | "malloc" : " 14 GB", | 20260507 03:56:46.525 | INFO | "pinned" : " 29 GB" | 20260507 03:56:46.525 | INFO | } | 20260507 03:56:46.525 | INFO | Perf report for the last 5.14459 seconds: | 20260507 03:56:46.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:56:46.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:46.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:46.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:46.525 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 03:56:46.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:46.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:46.526 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 03:56:46.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:46.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:46.526 | INFO | Uploaded 36GiB to GPU in 2.00049s (17.9956GiB/s) | 20260507 03:56:46.526 | INFO | Downloaded 34GiB to GPU in 1.99845s (17.0132GiB/s) | 20260507 03:56:51.669 | INFO | Analyzing chunk frames = [6905344,6905856) | 20260507 03:56:51.669 | INFO | MemInfo: { | 20260507 03:56:51.669 | INFO | "gpu" : " 17 GB", | 20260507 03:56:51.669 | INFO | "huge" : "", | 20260507 03:56:51.669 | INFO | "huge_shared" : " 27 GB", | 20260507 03:56:51.669 | INFO | "malloc" : " 14 GB", | 20260507 03:56:51.669 | INFO | "pinned" : " 29 GB" | 20260507 03:56:51.669 | INFO | } | 20260507 03:56:51.669 | INFO | Perf report for the last 5.14376 seconds: | 20260507 03:56:51.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:56:51.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:51.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:51.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:51.669 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:56:51.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:51.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:51.669 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:56:51.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:51.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:51.669 | INFO | Uploaded 36GiB to GPU in 1.98837s (18.1053GiB/s) | 20260507 03:56:51.669 | INFO | Downloaded 34GiB to GPU in 1.97917s (17.1789GiB/s) | 20260507 03:56:51.670 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:56:51.670 | INFO | Total mem usage: 18 GiB | 20260507 03:56:51.670 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:56:51.670 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:56:51.670 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:56:51.670 | INFO | High Water Breakdown: | 20260507 03:56:51.670 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:56:51.670 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:56:51.670 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:56:51.670 | INFO | ------------------------------------------ | 20260507 03:56:51.670 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:56:51.670 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:56:51.670 | INFO | Total mem usage: 26.5 GiB | 20260507 03:56:51.670 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:56:51.670 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:56:51.670 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:56:51.670 | INFO | High Water Breakdown: | 20260507 03:56:51.670 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:56:51.670 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:56:51.670 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:56:51.670 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:56:51.670 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:56:51.670 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:56:51.670 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:56:51.670 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:56:51.670 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:56:51.670 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:56:51.670 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:56:51.670 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:56:51.670 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:56:51.670 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:56:51.670 | INFO | ------------------------------------------ | 20260507 03:56:51.670 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:56:51.670 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:56:51.670 | INFO | Total mem usage: 25.6 GiB | 20260507 03:56:51.670 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:56:51.670 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 03:56:51.670 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 03:56:51.670 | INFO | High Water Breakdown: | 20260507 03:56:51.670 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 03:56:51.670 | INFO | ------------------------------------------ | 20260507 03:56:51.670 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:56:51.670 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:56:51.670 | INFO | Total mem usage: 15.7 GiB | 20260507 03:56:51.670 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:56:51.670 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:56:51.670 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:56:51.670 | INFO | High Water Breakdown: | 20260507 03:56:51.670 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:56:51.670 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:56:51.670 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:56:51.670 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:56:51.670 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:56:51.670 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:56:51.670 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:56:51.670 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:56:51.670 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:56:51.670 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:56:51.670 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:56:51.670 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:56:51.670 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:56:51.670 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:56:51.670 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:56:51.670 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:56:51.670 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:56:51.670 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:56:51.670 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:56:51.670 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:56:51.670 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:56:51.670 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:56:51.670 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:56:51.670 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:56:51.670 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:56:51.671 | INFO | ------------------------------------------ | 20260507 03:56:51.671 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:56:51.671 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:56:51.671 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:56:51.671 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:56:51.671 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.32e+01 / 1.88e+00 | 20260507 03:56:51.671 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.01e-01, 9.69e-03 / 9.13e-03 | 20260507 03:56:51.671 | INFO | PerfReport: 11.63%, Baseline -- 3.57e+00 / 4.39e+00, 4.96e-01 / 2.84e-01 | 20260507 03:56:51.671 | INFO | PerfReport: 3.21%, Histogram -- 9.85e-01 / 1.44e+00, 1.62e-01 / 1.21e-01 | 20260507 03:56:51.671 | INFO | PerfReport: 16.17%, DME -- 3.97e+01 / 4.45e+01, 4.01e+00 / 2.25e+00 | 20260507 03:56:51.671 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.77e+01 / 2.00e+01, 1.20e+00 / 1.25e+00 | 20260507 03:56:51.671 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.15e+00 / 2.70e+00, 3.86e-01 / 2.25e-01 | 20260507 03:56:51.671 | INFO | PerfReport: 3.04%, Metrics -- 9.33e-01 / 1.23e+00, 2.48e-01 / 1.46e-01 | 20260507 03:56:51.671 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.15e-01 / 3.77e-01, 5.01e-02 / 2.66e-02 | 20260507 03:56:51.671 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:56:51.671 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:56:51.671 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:56:51.671 | INFO | PerfReport: Full Execution -- 9.64e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 03:56:51.671 | INFO | PerfReport: 21.84%, Upload -- 2.10e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260507 03:56:51.671 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260507 03:56:51.671 | INFO | PerfReport: 31.84%, Compute -- 3.07e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 03:56:51.671 | INFO | PerfReport: 1.64%, LockUpload -- 1.58e+00 / 0.00e+00, 5.12e+00 / 0.00e+00 | 20260507 03:56:51.671 | INFO | PerfReport: 1.24%, LockDownload -- 1.19e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260507 03:56:51.671 | INFO | PerfReport: 22.02%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 03:56:51.671 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.15e-02 / 0.00e+00 | 20260507 03:56:51.766 | INFO | Starting a round of Baz IO | 20260507 03:56:54.866 | INFO | Populating RTMetrics report | 20260507 03:56:54.866 | INFO | RTMetrics report populated | 20260507 03:56:54.866 | INFO | Opening tmp RTMetrics json file | 20260507 03:56:54.885 | INFO | Serializing RTMetrics to json | 20260507 03:56:54.886 | INFO | Serialization done, swapping tmp file now... | 20260507 03:56:54.887 | INFO | RTMetrics done | 20260507 03:56:54.957 | INFO | Finished a round of Baz IO: Wrote 1.738 GiB in 3.191 seconds | 20260507 03:56:54.957 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.84% / 0.00% / 67.16% | 20260507 03:56:54.957 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.184111% / 0% / 8.34465e-05% | 20260507 03:56:54.957 | INFO | Baz IO overall: | 20260507 03:56:54.957 | INFO | BytesWritten : 3534782119122 | 20260507 03:56:54.957 | INFO | NumEvents : 1697732051292 | 20260507 03:56:56.808 | INFO | Analyzing chunk frames = [6905856,6906368) | 20260507 03:56:56.808 | INFO | MemInfo: { | 20260507 03:56:56.808 | INFO | "gpu" : " 17 GB", | 20260507 03:56:56.808 | INFO | "huge" : "", | 20260507 03:56:56.808 | INFO | "huge_shared" : " 27 GB", | 20260507 03:56:56.808 | INFO | "malloc" : " 14 GB", | 20260507 03:56:56.808 | INFO | "pinned" : " 29 GB" | 20260507 03:56:56.808 | INFO | } | 20260507 03:56:56.808 | INFO | Perf report for the last 5.13886 seconds: | 20260507 03:56:56.808 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:56:56.808 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:56.808 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:56.808 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:56.808 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 03:56:56.808 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:56:56.808 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 03:56:56.808 | INFO | PRE_HQ : 59.0%, 1.8, 55.0ms, 0 | 20260507 03:56:56.808 | INFO | BAZWRITER : 61.7%, 3.3, 103.4ms, 0 | 20260507 03:56:56.808 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 03:56:56.808 | INFO | Uploaded 36GiB to GPU in 2.03256s (17.7116GiB/s) | 20260507 03:56:56.808 | INFO | Downloaded 38GiB to GPU in 2.1778s (17.4488GiB/s) | 20260507 03:57:01.939 | INFO | Analyzing chunk frames = [6906368,6906880) | 20260507 03:57:01.939 | INFO | MemInfo: { | 20260507 03:57:01.939 | INFO | "gpu" : " 17 GB", | 20260507 03:57:01.939 | INFO | "huge" : "", | 20260507 03:57:01.939 | INFO | "huge_shared" : " 27 GB", | 20260507 03:57:01.939 | INFO | "malloc" : " 14 GB", | 20260507 03:57:01.939 | INFO | "pinned" : " 29 GB" | 20260507 03:57:01.939 | INFO | } | 20260507 03:57:01.939 | INFO | Perf report for the last 5.13135 seconds: | 20260507 03:57:01.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:57:01.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:01.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:01.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:01.939 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 03:57:01.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:01.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:01.940 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:57:01.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:01.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:01.940 | INFO | Uploaded 36GiB to GPU in 1.98759s (18.1124GiB/s) | 20260507 03:57:01.940 | INFO | Downloaded 34GiB to GPU in 1.93003s (17.6163GiB/s) | 20260507 03:57:07.082 | INFO | Analyzing chunk frames = [6906880,6907392) | 20260507 03:57:07.082 | INFO | MemInfo: { | 20260507 03:57:07.083 | INFO | "gpu" : " 17 GB", | 20260507 03:57:07.083 | INFO | "huge" : "", | 20260507 03:57:07.083 | INFO | "huge_shared" : " 27 GB", | 20260507 03:57:07.083 | INFO | "malloc" : " 14 GB", | 20260507 03:57:07.083 | INFO | "pinned" : " 29 GB" | 20260507 03:57:07.083 | INFO | } | 20260507 03:57:07.083 | INFO | Perf report for the last 5.14336 seconds: | 20260507 03:57:07.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:57:07.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:07.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:07.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:07.083 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 03:57:07.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:07.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:07.083 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:57:07.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:07.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:07.083 | INFO | Uploaded 36GiB to GPU in 2.01334s (17.8807GiB/s) | 20260507 03:57:07.083 | INFO | Downloaded 34GiB to GPU in 1.94286s (17.4999GiB/s) | 20260507 03:57:12.226 | INFO | Analyzing chunk frames = [6907392,6907904) | 20260507 03:57:12.226 | INFO | MemInfo: { | 20260507 03:57:12.226 | INFO | "gpu" : " 17 GB", | 20260507 03:57:12.226 | INFO | "huge" : "", | 20260507 03:57:12.226 | INFO | "huge_shared" : " 27 GB", | 20260507 03:57:12.227 | INFO | "malloc" : " 14 GB", | 20260507 03:57:12.227 | INFO | "pinned" : " 29 GB" | 20260507 03:57:12.227 | INFO | } | 20260507 03:57:12.227 | INFO | Perf report for the last 5.1439 seconds: | 20260507 03:57:12.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:57:12.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:12.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:12.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:12.227 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 03:57:12.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:12.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:12.227 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:57:12.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:12.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:12.227 | INFO | Uploaded 36GiB to GPU in 1.9643s (18.3271GiB/s) | 20260507 03:57:12.227 | INFO | Downloaded 34GiB to GPU in 1.92185s (17.6913GiB/s) | 20260507 03:57:17.363 | INFO | Analyzing chunk frames = [6907904,6908416) | 20260507 03:57:17.363 | INFO | MemInfo: { | 20260507 03:57:17.363 | INFO | "gpu" : " 17 GB", | 20260507 03:57:17.363 | INFO | "huge" : "", | 20260507 03:57:17.363 | INFO | "huge_shared" : " 27 GB", | 20260507 03:57:17.363 | INFO | "malloc" : " 14 GB", | 20260507 03:57:17.363 | INFO | "pinned" : " 29 GB" | 20260507 03:57:17.363 | INFO | } | 20260507 03:57:17.363 | INFO | Perf report for the last 5.13642 seconds: | 20260507 03:57:17.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:57:17.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:17.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:17.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:17.363 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 03:57:17.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:17.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:17.363 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:57:17.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:17.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:17.364 | INFO | Uploaded 36GiB to GPU in 2.03729s (17.6706GiB/s) | 20260507 03:57:17.364 | INFO | Downloaded 34GiB to GPU in 1.94405s (17.4892GiB/s) | 20260507 03:57:22.501 | INFO | Analyzing chunk frames = [6908416,6908928) | 20260507 03:57:22.501 | INFO | MemInfo: { | 20260507 03:57:22.501 | INFO | "gpu" : " 17 GB", | 20260507 03:57:22.501 | INFO | "huge" : "", | 20260507 03:57:22.501 | INFO | "huge_shared" : " 27 GB", | 20260507 03:57:22.501 | INFO | "malloc" : " 14 GB", | 20260507 03:57:22.501 | INFO | "pinned" : " 29 GB" | 20260507 03:57:22.501 | INFO | } | 20260507 03:57:22.501 | INFO | Perf report for the last 5.13833 seconds: | 20260507 03:57:22.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:57:22.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:22.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:22.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:22.502 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 03:57:22.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:22.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:22.502 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 03:57:22.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:22.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:22.502 | INFO | Uploaded 36GiB to GPU in 2.02444s (17.7827GiB/s) | 20260507 03:57:22.502 | INFO | Downloaded 34GiB to GPU in 1.90127s (17.8828GiB/s) | 20260507 03:57:27.638 | INFO | Analyzing chunk frames = [6908928,6909440) | 20260507 03:57:27.638 | INFO | MemInfo: { | 20260507 03:57:27.638 | INFO | "gpu" : " 17 GB", | 20260507 03:57:27.638 | INFO | "huge" : "", | 20260507 03:57:27.639 | INFO | "huge_shared" : " 27 GB", | 20260507 03:57:27.639 | INFO | "malloc" : " 14 GB", | 20260507 03:57:27.639 | INFO | "pinned" : " 29 GB" | 20260507 03:57:27.639 | INFO | } | 20260507 03:57:27.639 | INFO | Perf report for the last 5.13725 seconds: | 20260507 03:57:27.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:57:27.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:27.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:27.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:27.639 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 03:57:27.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:27.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:27.639 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 03:57:27.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:27.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:27.639 | INFO | Uploaded 36GiB to GPU in 1.9634s (18.3356GiB/s) | 20260507 03:57:27.639 | INFO | Downloaded 34GiB to GPU in 1.99307s (17.0591GiB/s) | 20260507 03:57:32.786 | INFO | Analyzing chunk frames = [6909440,6909952) | 20260507 03:57:32.786 | INFO | MemInfo: { | 20260507 03:57:32.786 | INFO | "gpu" : " 17 GB", | 20260507 03:57:32.786 | INFO | "huge" : "", | 20260507 03:57:32.786 | INFO | "huge_shared" : " 27 GB", | 20260507 03:57:32.786 | INFO | "malloc" : " 14 GB", | 20260507 03:57:32.786 | INFO | "pinned" : " 29 GB" | 20260507 03:57:32.786 | INFO | } | 20260507 03:57:32.787 | INFO | Perf report for the last 5.14781 seconds: | 20260507 03:57:32.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:57:32.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:32.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:32.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:32.787 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 03:57:32.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:32.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:32.787 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 03:57:32.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:32.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:32.787 | INFO | Uploaded 36GiB to GPU in 2.00804s (17.9279GiB/s) | 20260507 03:57:32.787 | INFO | Downloaded 34GiB to GPU in 1.97658s (17.2014GiB/s) | 20260507 03:57:32.884 | INFO | Starting a round of Baz IO | 20260507 03:57:35.987 | INFO | Populating RTMetrics report | 20260507 03:57:35.987 | INFO | RTMetrics report populated | 20260507 03:57:35.987 | INFO | Opening tmp RTMetrics json file | 20260507 03:57:35.990 | INFO | Serializing RTMetrics to json | 20260507 03:57:36.006 | INFO | Serialization done, swapping tmp file now... | 20260507 03:57:36.006 | INFO | RTMetrics done | 20260507 03:57:36.075 | INFO | Finished a round of Baz IO: Wrote 1.739 GiB in 3.191 seconds | 20260507 03:57:36.075 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.84% / 0.00% / 67.16% | 20260507 03:57:36.075 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.183948% / 0% / 9.53674e-05% | 20260507 03:57:36.075 | INFO | Baz IO overall: | 20260507 03:57:36.075 | INFO | BytesWritten : 3536649052592 | 20260507 03:57:36.075 | INFO | NumEvents : 1698611933125 | 20260507 03:57:37.920 | INFO | Analyzing chunk frames = [6909952,6910464) | 20260507 03:57:37.920 | INFO | MemInfo: { | 20260507 03:57:37.920 | INFO | "gpu" : " 17 GB", | 20260507 03:57:37.920 | INFO | "huge" : "", | 20260507 03:57:37.920 | INFO | "huge_shared" : " 27 GB", | 20260507 03:57:37.920 | INFO | "malloc" : " 14 GB", | 20260507 03:57:37.920 | INFO | "pinned" : " 29 GB" | 20260507 03:57:37.920 | INFO | } | 20260507 03:57:37.920 | INFO | Perf report for the last 5.13365 seconds: | 20260507 03:57:37.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:57:37.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:37.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:37.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:37.920 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260507 03:57:37.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:37.920 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 03:57:37.920 | INFO | PRE_HQ : 59.9%, 1.9, 60.0ms, 0 | 20260507 03:57:37.920 | INFO | BAZWRITER : 61.8%, 3.4, 108.8ms, 0 | 20260507 03:57:37.920 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:57:37.920 | INFO | Uploaded 36GiB to GPU in 2.05312s (17.5343GiB/s) | 20260507 03:57:37.921 | INFO | Downloaded 38GiB to GPU in 2.17832s (17.4447GiB/s) | 20260507 03:57:43.057 | INFO | Analyzing chunk frames = [6910464,6910976) | 20260507 03:57:43.057 | INFO | MemInfo: { | 20260507 03:57:43.057 | INFO | "gpu" : " 17 GB", | 20260507 03:57:43.057 | INFO | "huge" : "", | 20260507 03:57:43.057 | INFO | "huge_shared" : " 27 GB", | 20260507 03:57:43.057 | INFO | "malloc" : " 14 GB", | 20260507 03:57:43.057 | INFO | "pinned" : " 29 GB" | 20260507 03:57:43.057 | INFO | } | 20260507 03:57:43.057 | INFO | Perf report for the last 5.13715 seconds: | 20260507 03:57:43.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:57:43.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:43.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:43.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:43.057 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 03:57:43.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:43.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:43.057 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:57:43.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:43.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:43.058 | INFO | Uploaded 36GiB to GPU in 2.00146s (17.9869GiB/s) | 20260507 03:57:43.058 | INFO | Downloaded 34GiB to GPU in 1.92713s (17.6429GiB/s) | 20260507 03:57:48.200 | INFO | Analyzing chunk frames = [6910976,6911488) | 20260507 03:57:48.200 | INFO | MemInfo: { | 20260507 03:57:48.200 | INFO | "gpu" : " 17 GB", | 20260507 03:57:48.200 | INFO | "huge" : "", | 20260507 03:57:48.200 | INFO | "huge_shared" : " 27 GB", | 20260507 03:57:48.200 | INFO | "malloc" : " 14 GB", | 20260507 03:57:48.200 | INFO | "pinned" : " 29 GB" | 20260507 03:57:48.200 | INFO | } | 20260507 03:57:48.200 | INFO | Perf report for the last 5.14297 seconds: | 20260507 03:57:48.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:57:48.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:48.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:48.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:48.200 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 03:57:48.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:48.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:48.200 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:57:48.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:48.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:48.201 | INFO | Uploaded 36GiB to GPU in 2.01957s (17.8256GiB/s) | 20260507 03:57:48.201 | INFO | Downloaded 34GiB to GPU in 1.94011s (17.5248GiB/s) | 20260507 03:57:53.335 | INFO | Analyzing chunk frames = [6911488,6912000) | 20260507 03:57:53.335 | INFO | MemInfo: { | 20260507 03:57:53.335 | INFO | "gpu" : " 17 GB", | 20260507 03:57:53.335 | INFO | "huge" : "", | 20260507 03:57:53.336 | INFO | "huge_shared" : " 27 GB", | 20260507 03:57:53.336 | INFO | "malloc" : " 14 GB", | 20260507 03:57:53.336 | INFO | "pinned" : " 29 GB" | 20260507 03:57:53.336 | INFO | } | 20260507 03:57:53.336 | INFO | Perf report for the last 5.13535 seconds: | 20260507 03:57:53.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:57:53.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:53.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:53.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:53.336 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 03:57:53.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:53.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:53.336 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:57:53.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:53.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:53.336 | INFO | Uploaded 36GiB to GPU in 1.99589s (18.0371GiB/s) | 20260507 03:57:53.336 | INFO | Downloaded 34GiB to GPU in 1.91994s (17.7089GiB/s) | 20260507 03:57:58.478 | INFO | Analyzing chunk frames = [6912000,6912512) | 20260507 03:57:58.478 | INFO | MemInfo: { | 20260507 03:57:58.478 | INFO | "gpu" : " 17 GB", | 20260507 03:57:58.478 | INFO | "huge" : "", | 20260507 03:57:58.478 | INFO | "huge_shared" : " 27 GB", | 20260507 03:57:58.478 | INFO | "malloc" : " 14 GB", | 20260507 03:57:58.478 | INFO | "pinned" : " 29 GB" | 20260507 03:57:58.478 | INFO | } | 20260507 03:57:58.478 | INFO | Perf report for the last 5.1422 seconds: | 20260507 03:57:58.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:57:58.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:58.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:58.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:58.478 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 03:57:58.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:58.478 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:58.478 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 03:57:58.478 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:58.478 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:57:58.478 | INFO | Uploaded 36GiB to GPU in 2.01689s (17.8492GiB/s) | 20260507 03:57:58.478 | INFO | Downloaded 34GiB to GPU in 1.93973s (17.5282GiB/s) | 20260507 03:58:03.623 | INFO | Analyzing chunk frames = [6912512,6913024) | 20260507 03:58:03.623 | INFO | MemInfo: { | 20260507 03:58:03.623 | INFO | "gpu" : " 17 GB", | 20260507 03:58:03.623 | INFO | "huge" : "", | 20260507 03:58:03.623 | INFO | "huge_shared" : " 27 GB", | 20260507 03:58:03.623 | INFO | "malloc" : " 14 GB", | 20260507 03:58:03.623 | INFO | "pinned" : " 29 GB" | 20260507 03:58:03.623 | INFO | } | 20260507 03:58:03.624 | INFO | Perf report for the last 5.14562 seconds: | 20260507 03:58:03.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:58:03.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:03.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:03.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:03.624 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 03:58:03.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:03.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:03.624 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 03:58:03.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:03.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:03.624 | INFO | Uploaded 36GiB to GPU in 2.04124s (17.6363GiB/s) | 20260507 03:58:03.624 | INFO | Downloaded 34GiB to GPU in 1.93474s (17.5734GiB/s) | 20260507 03:58:08.757 | INFO | Analyzing chunk frames = [6913024,6913536) | 20260507 03:58:08.757 | INFO | MemInfo: { | 20260507 03:58:08.757 | INFO | "gpu" : " 17 GB", | 20260507 03:58:08.757 | INFO | "huge" : "", | 20260507 03:58:08.757 | INFO | "huge_shared" : " 27 GB", | 20260507 03:58:08.757 | INFO | "malloc" : " 14 GB", | 20260507 03:58:08.757 | INFO | "pinned" : " 29 GB" | 20260507 03:58:08.757 | INFO | } | 20260507 03:58:08.757 | INFO | Perf report for the last 5.1333 seconds: | 20260507 03:58:08.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:58:08.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:08.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:08.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:08.757 | INFO | ANALYSIS : 61.0%, 3.0, 94.7ms, 0 | 20260507 03:58:08.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:08.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:08.757 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 03:58:08.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:08.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:08.757 | INFO | Uploaded 36GiB to GPU in 1.96486s (18.322GiB/s) | 20260507 03:58:08.757 | INFO | Downloaded 34GiB to GPU in 1.97868s (17.1832GiB/s) | 20260507 03:58:13.891 | INFO | Analyzing chunk frames = [6913536,6914048) | 20260507 03:58:13.891 | INFO | MemInfo: { | 20260507 03:58:13.891 | INFO | "gpu" : " 17 GB", | 20260507 03:58:13.891 | INFO | "huge" : "", | 20260507 03:58:13.891 | INFO | "huge_shared" : " 27 GB", | 20260507 03:58:13.891 | INFO | "malloc" : " 14 GB", | 20260507 03:58:13.891 | INFO | "pinned" : " 29 GB" | 20260507 03:58:13.891 | INFO | } | 20260507 03:58:13.891 | INFO | Perf report for the last 5.13457 seconds: | 20260507 03:58:13.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:58:13.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:13.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:13.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:13.891 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 03:58:13.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:13.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:13.892 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 03:58:13.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:13.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:13.892 | INFO | Uploaded 36GiB to GPU in 1.9761s (18.2177GiB/s) | 20260507 03:58:13.892 | INFO | Downloaded 34GiB to GPU in 1.96208s (17.3286GiB/s) | 20260507 03:58:13.892 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:58:13.892 | INFO | Total mem usage: 18 GiB | 20260507 03:58:13.892 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:58:13.892 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 03:58:13.892 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:58:13.892 | INFO | High Water Breakdown: | 20260507 03:58:13.892 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:58:13.892 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:58:13.892 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:58:13.892 | INFO | ------------------------------------------ | 20260507 03:58:13.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:58:13.892 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:58:13.892 | INFO | Total mem usage: 26.5 GiB | 20260507 03:58:13.892 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:58:13.892 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 03:58:13.892 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 03:58:13.892 | INFO | High Water Breakdown: | 20260507 03:58:13.892 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:58:13.892 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:58:13.892 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:58:13.892 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:58:13.892 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:58:13.892 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:58:13.892 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:58:13.892 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:58:13.892 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:58:13.892 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:58:13.892 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:58:13.892 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:58:13.892 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:58:13.892 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:58:13.892 | INFO | ------------------------------------------ | 20260507 03:58:13.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:58:13.892 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:58:13.892 | INFO | Total mem usage: 25.6 GiB | 20260507 03:58:13.892 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:58:13.892 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 03:58:13.892 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 03:58:13.892 | INFO | High Water Breakdown: | 20260507 03:58:13.892 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 03:58:13.892 | INFO | ------------------------------------------ | 20260507 03:58:13.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:58:13.892 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:58:13.892 | INFO | Total mem usage: 15.7 GiB | 20260507 03:58:13.892 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:58:13.892 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:58:13.892 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:58:13.892 | INFO | High Water Breakdown: | 20260507 03:58:13.892 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:58:13.892 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:58:13.892 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:58:13.892 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:58:13.892 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:58:13.892 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:58:13.892 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:58:13.892 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:58:13.892 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:58:13.892 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:58:13.892 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:58:13.892 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:58:13.892 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:58:13.892 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:58:13.892 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:58:13.892 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:58:13.892 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:58:13.892 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:58:13.892 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:58:13.892 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:58:13.892 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:58:13.892 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:58:13.892 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:58:13.892 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:58:13.892 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:58:13.892 | INFO | ------------------------------------------ | 20260507 03:58:13.892 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:58:13.893 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:58:13.893 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:58:13.893 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:58:13.893 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.52e+01, 1.33e+01 / 1.57e+00 | 20260507 03:58:13.893 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.92e-03 / 5.58e-03 | 20260507 03:58:13.893 | INFO | PerfReport: 11.69%, Baseline -- 3.60e+00 / 4.40e+00, 5.32e-01 / 2.84e-01 | 20260507 03:58:13.893 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.45e+00, 1.75e-01 / 1.14e-01 | 20260507 03:58:13.893 | INFO | PerfReport: 16.11%, DME -- 3.97e+01 / 4.45e+01, 4.03e+00 / 2.13e+00 | 20260507 03:58:13.893 | INFO | PerfReport: 57.68%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.35e+00 / 1.82e+00 | 20260507 03:58:13.893 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.14e+00 / 2.68e+00, 3.76e-01 / 2.28e-01 | 20260507 03:58:13.893 | INFO | PerfReport: 3.02%, Metrics -- 9.30e-01 / 1.24e+00, 2.44e-01 / 1.35e-01 | 20260507 03:58:13.893 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.16e-01 / 3.67e-01, 5.00e-02 / 1.65e-02 | 20260507 03:58:13.893 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:58:13.893 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:58:13.893 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:58:13.893 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 03:58:13.893 | INFO | PerfReport: 21.72%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260507 03:58:13.893 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260507 03:58:13.893 | INFO | PerfReport: 32.02%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 03:58:13.893 | INFO | PerfReport: 1.42%, LockUpload -- 1.37e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 03:58:13.893 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260507 03:58:13.893 | INFO | PerfReport: 22.18%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 03:58:13.893 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.55e-02 / 0.00e+00 | 20260507 03:58:13.989 | INFO | Starting a round of Baz IO | 20260507 03:58:17.101 | INFO | Populating RTMetrics report | 20260507 03:58:17.102 | INFO | RTMetrics report populated | 20260507 03:58:17.102 | INFO | Opening tmp RTMetrics json file | 20260507 03:58:17.103 | INFO | Serializing RTMetrics to json | 20260507 03:58:17.104 | INFO | Serialization done, swapping tmp file now... | 20260507 03:58:17.104 | INFO | RTMetrics done | 20260507 03:58:17.192 | INFO | Finished a round of Baz IO: Wrote 1.738 GiB in 3.202 seconds | 20260507 03:58:17.192 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.84% / 0.00% / 67.16% | 20260507 03:58:17.192 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.183181% / 0% / 8.34465e-05% | 20260507 03:58:17.192 | INFO | Baz IO overall: | 20260507 03:58:17.192 | INFO | BytesWritten : 3538514684408 | 20260507 03:58:17.192 | INFO | NumEvents : 1699490265466 | 20260507 03:58:19.039 | INFO | Analyzing chunk frames = [6914048,6914560) | 20260507 03:58:19.039 | INFO | MemInfo: { | 20260507 03:58:19.039 | INFO | "gpu" : " 17 GB", | 20260507 03:58:19.039 | INFO | "huge" : "", | 20260507 03:58:19.039 | INFO | "huge_shared" : " 27 GB", | 20260507 03:58:19.039 | INFO | "malloc" : " 14 GB", | 20260507 03:58:19.039 | INFO | "pinned" : " 29 GB" | 20260507 03:58:19.039 | INFO | } | 20260507 03:58:19.039 | INFO | Perf report for the last 5.14736 seconds: | 20260507 03:58:19.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:58:19.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:19.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:19.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:19.039 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260507 03:58:19.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:19.039 | INFO | RT_METRICS : 45.5%, 1.0, 23.9ms, 0 | 20260507 03:58:19.039 | INFO | PRE_HQ : 60.2%, 1.9, 60.8ms, 0 | 20260507 03:58:19.039 | INFO | BAZWRITER : 61.8%, 3.5, 109.8ms, 0 | 20260507 03:58:19.039 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 03:58:19.039 | INFO | Uploaded 36GiB to GPU in 2.03709s (17.6723GiB/s) | 20260507 03:58:19.039 | INFO | Downloaded 38GiB to GPU in 2.18568s (17.3859GiB/s) | 20260507 03:58:24.180 | INFO | Analyzing chunk frames = [6914560,6915072) | 20260507 03:58:24.180 | INFO | MemInfo: { | 20260507 03:58:24.180 | INFO | "gpu" : " 17 GB", | 20260507 03:58:24.180 | INFO | "huge" : "", | 20260507 03:58:24.180 | INFO | "huge_shared" : " 27 GB", | 20260507 03:58:24.180 | INFO | "malloc" : " 14 GB", | 20260507 03:58:24.180 | INFO | "pinned" : " 29 GB" | 20260507 03:58:24.180 | INFO | } | 20260507 03:58:24.180 | INFO | Perf report for the last 5.14097 seconds: | 20260507 03:58:24.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:58:24.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:24.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:24.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:24.180 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 03:58:24.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:24.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:24.180 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 03:58:24.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:24.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:24.180 | INFO | Uploaded 36GiB to GPU in 2.04728s (17.5843GiB/s) | 20260507 03:58:24.180 | INFO | Downloaded 34GiB to GPU in 1.93723s (17.5509GiB/s) | 20260507 03:58:29.315 | INFO | Analyzing chunk frames = [6915072,6915584) | 20260507 03:58:29.315 | INFO | MemInfo: { | 20260507 03:58:29.315 | INFO | "gpu" : " 17 GB", | 20260507 03:58:29.315 | INFO | "huge" : "", | 20260507 03:58:29.315 | INFO | "huge_shared" : " 27 GB", | 20260507 03:58:29.315 | INFO | "malloc" : " 14 GB", | 20260507 03:58:29.315 | INFO | "pinned" : " 29 GB" | 20260507 03:58:29.315 | INFO | } | 20260507 03:58:29.315 | INFO | Perf report for the last 5.13564 seconds: | 20260507 03:58:29.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:58:29.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:29.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:29.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:29.315 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 03:58:29.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:29.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:29.315 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 03:58:29.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:29.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:29.316 | INFO | Uploaded 36GiB to GPU in 2.04215s (17.6285GiB/s) | 20260507 03:58:29.316 | INFO | Downloaded 34GiB to GPU in 1.94749s (17.4584GiB/s) | 20260507 03:58:34.451 | INFO | Analyzing chunk frames = [6915584,6916096) | 20260507 03:58:34.451 | INFO | MemInfo: { | 20260507 03:58:34.451 | INFO | "gpu" : " 17 GB", | 20260507 03:58:34.451 | INFO | "huge" : "", | 20260507 03:58:34.451 | INFO | "huge_shared" : " 27 GB", | 20260507 03:58:34.451 | INFO | "malloc" : " 14 GB", | 20260507 03:58:34.451 | INFO | "pinned" : " 29 GB" | 20260507 03:58:34.451 | INFO | } | 20260507 03:58:34.451 | INFO | Perf report for the last 5.13536 seconds: | 20260507 03:58:34.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:58:34.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:34.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:34.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:34.451 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:58:34.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:34.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:34.451 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:58:34.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:34.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:34.451 | INFO | Uploaded 36GiB to GPU in 2.00527s (17.9527GiB/s) | 20260507 03:58:34.451 | INFO | Downloaded 34GiB to GPU in 1.9315s (17.6029GiB/s) | 20260507 03:58:39.593 | INFO | Analyzing chunk frames = [6916096,6916608) | 20260507 03:58:39.593 | INFO | MemInfo: { | 20260507 03:58:39.593 | INFO | "gpu" : " 17 GB", | 20260507 03:58:39.593 | INFO | "huge" : "", | 20260507 03:58:39.593 | INFO | "huge_shared" : " 27 GB", | 20260507 03:58:39.593 | INFO | "malloc" : " 14 GB", | 20260507 03:58:39.593 | INFO | "pinned" : " 29 GB" | 20260507 03:58:39.593 | INFO | } | 20260507 03:58:39.593 | INFO | Perf report for the last 5.14236 seconds: | 20260507 03:58:39.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:58:39.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:39.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:39.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:39.593 | INFO | ANALYSIS : 59.9%, 3.0, 93.2ms, 0 | 20260507 03:58:39.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:39.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:39.593 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:58:39.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:39.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:39.593 | INFO | Uploaded 36GiB to GPU in 2.0102s (17.9087GiB/s) | 20260507 03:58:39.593 | INFO | Downloaded 34GiB to GPU in 1.94147s (17.5125GiB/s) | 20260507 03:58:44.734 | INFO | Analyzing chunk frames = [6916608,6917120) | 20260507 03:58:44.734 | INFO | MemInfo: { | 20260507 03:58:44.734 | INFO | "gpu" : " 17 GB", | 20260507 03:58:44.734 | INFO | "huge" : "", | 20260507 03:58:44.734 | INFO | "huge_shared" : " 27 GB", | 20260507 03:58:44.734 | INFO | "malloc" : " 14 GB", | 20260507 03:58:44.734 | INFO | "pinned" : " 29 GB" | 20260507 03:58:44.734 | INFO | } | 20260507 03:58:44.734 | INFO | Perf report for the last 5.14111 seconds: | 20260507 03:58:44.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:58:44.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:44.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:44.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:44.735 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:58:44.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:44.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:44.735 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:58:44.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:44.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:44.735 | INFO | Uploaded 36GiB to GPU in 2.05679s (17.503GiB/s) | 20260507 03:58:44.735 | INFO | Downloaded 34GiB to GPU in 1.95374s (17.4025GiB/s) | 20260507 03:58:49.866 | INFO | Analyzing chunk frames = [6917120,6917632) | 20260507 03:58:49.866 | INFO | MemInfo: { | 20260507 03:58:49.866 | INFO | "gpu" : " 17 GB", | 20260507 03:58:49.866 | INFO | "huge" : "", | 20260507 03:58:49.866 | INFO | "huge_shared" : " 27 GB", | 20260507 03:58:49.866 | INFO | "malloc" : " 14 GB", | 20260507 03:58:49.867 | INFO | "pinned" : " 29 GB" | 20260507 03:58:49.867 | INFO | } | 20260507 03:58:49.867 | INFO | Perf report for the last 5.13229 seconds: | 20260507 03:58:49.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:58:49.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:49.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:49.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:49.867 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 03:58:49.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:49.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:49.867 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 03:58:49.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:49.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:49.867 | INFO | Uploaded 36GiB to GPU in 1.96432s (18.327GiB/s) | 20260507 03:58:49.867 | INFO | Downloaded 34GiB to GPU in 1.97968s (17.1745GiB/s) | 20260507 03:58:55.010 | INFO | Analyzing chunk frames = [6917632,6918144) | 20260507 03:58:55.010 | INFO | MemInfo: { | 20260507 03:58:55.010 | INFO | "gpu" : " 17 GB", | 20260507 03:58:55.010 | INFO | "huge" : "", | 20260507 03:58:55.010 | INFO | "huge_shared" : " 27 GB", | 20260507 03:58:55.011 | INFO | "malloc" : " 14 GB", | 20260507 03:58:55.011 | INFO | "pinned" : " 29 GB" | 20260507 03:58:55.011 | INFO | } | 20260507 03:58:55.011 | INFO | Perf report for the last 5.14395 seconds: | 20260507 03:58:55.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:58:55.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:55.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:55.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:55.011 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 03:58:55.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:55.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:55.011 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 03:58:55.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:55.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:58:55.011 | INFO | Uploaded 36GiB to GPU in 1.97196s (18.256GiB/s) | 20260507 03:58:55.011 | INFO | Downloaded 34GiB to GPU in 1.95805s (17.3643GiB/s) | 20260507 03:58:55.117 | INFO | Starting a round of Baz IO | 20260507 03:58:58.212 | INFO | Populating RTMetrics report | 20260507 03:58:58.213 | INFO | RTMetrics report populated | 20260507 03:58:58.213 | INFO | Opening tmp RTMetrics json file | 20260507 03:58:58.214 | INFO | Serializing RTMetrics to json | 20260507 03:58:58.231 | INFO | Serialization done, swapping tmp file now... | 20260507 03:58:58.232 | INFO | RTMetrics done | 20260507 03:58:58.307 | INFO | Finished a round of Baz IO: Wrote 1.738 GiB in 3.19 seconds | 20260507 03:58:58.307 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.84% / 0.00% / 67.16% | 20260507 03:58:58.307 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.182652% / 7.94729e-06% / 5.96046e-05% | 20260507 03:58:58.307 | INFO | Baz IO overall: | 20260507 03:58:58.307 | INFO | BytesWritten : 3540380424546 | 20260507 03:58:58.307 | INFO | NumEvents : 1700369212303 | 20260507 03:59:00.152 | INFO | Analyzing chunk frames = [6918144,6918656) | 20260507 03:59:00.152 | INFO | MemInfo: { | 20260507 03:59:00.152 | INFO | "gpu" : " 17 GB", | 20260507 03:59:00.152 | INFO | "huge" : "", | 20260507 03:59:00.152 | INFO | "huge_shared" : " 27 GB", | 20260507 03:59:00.152 | INFO | "malloc" : " 14 GB", | 20260507 03:59:00.152 | INFO | "pinned" : " 29 GB" | 20260507 03:59:00.152 | INFO | } | 20260507 03:59:00.152 | INFO | Perf report for the last 5.14155 seconds: | 20260507 03:59:00.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:59:00.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:00.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:00.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:00.152 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 03:59:00.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:00.152 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260507 03:59:00.152 | INFO | PRE_HQ : 59.3%, 1.9, 57.6ms, 0 | 20260507 03:59:00.152 | INFO | BAZWRITER : 62.1%, 3.3, 106.5ms, 0 | 20260507 03:59:00.152 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:59:00.153 | INFO | Uploaded 36GiB to GPU in 2.03796s (17.6647GiB/s) | 20260507 03:59:00.153 | INFO | Downloaded 38GiB to GPU in 2.17957s (17.4346GiB/s) | 20260507 03:59:05.291 | INFO | Analyzing chunk frames = [6918656,6919168) | 20260507 03:59:05.291 | INFO | MemInfo: { | 20260507 03:59:05.291 | INFO | "gpu" : " 17 GB", | 20260507 03:59:05.291 | INFO | "huge" : "", | 20260507 03:59:05.291 | INFO | "huge_shared" : " 27 GB", | 20260507 03:59:05.291 | INFO | "malloc" : " 14 GB", | 20260507 03:59:05.291 | INFO | "pinned" : " 29 GB" | 20260507 03:59:05.291 | INFO | } | 20260507 03:59:05.291 | INFO | Perf report for the last 5.13913 seconds: | 20260507 03:59:05.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:59:05.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:05.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:05.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:05.291 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 03:59:05.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:05.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:05.291 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 03:59:05.292 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:05.292 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:05.292 | INFO | Uploaded 36GiB to GPU in 2.04236s (17.6267GiB/s) | 20260507 03:59:05.292 | INFO | Downloaded 34GiB to GPU in 1.94034s (17.5227GiB/s) | 20260507 03:59:10.432 | INFO | Analyzing chunk frames = [6919168,6919680) | 20260507 03:59:10.432 | INFO | MemInfo: { | 20260507 03:59:10.432 | INFO | "gpu" : " 17 GB", | 20260507 03:59:10.432 | INFO | "huge" : "", | 20260507 03:59:10.432 | INFO | "huge_shared" : " 27 GB", | 20260507 03:59:10.432 | INFO | "malloc" : " 14 GB", | 20260507 03:59:10.432 | INFO | "pinned" : " 29 GB" | 20260507 03:59:10.432 | INFO | } | 20260507 03:59:10.432 | INFO | Perf report for the last 5.14067 seconds: | 20260507 03:59:10.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:59:10.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:10.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:10.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:10.432 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260507 03:59:10.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:10.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:10.432 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:59:10.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:10.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:10.432 | INFO | Uploaded 36GiB to GPU in 2.03633s (17.6789GiB/s) | 20260507 03:59:10.433 | INFO | Downloaded 34GiB to GPU in 1.95258s (17.4129GiB/s) | 20260507 03:59:15.567 | INFO | Analyzing chunk frames = [6919680,6920192) | 20260507 03:59:15.567 | INFO | MemInfo: { | 20260507 03:59:15.567 | INFO | "gpu" : " 17 GB", | 20260507 03:59:15.567 | INFO | "huge" : "", | 20260507 03:59:15.567 | INFO | "huge_shared" : " 27 GB", | 20260507 03:59:15.567 | INFO | "malloc" : " 14 GB", | 20260507 03:59:15.567 | INFO | "pinned" : " 29 GB" | 20260507 03:59:15.567 | INFO | } | 20260507 03:59:15.568 | INFO | Perf report for the last 5.13546 seconds: | 20260507 03:59:15.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:59:15.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:15.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:15.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:15.568 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 03:59:15.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:15.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:15.568 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 03:59:15.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:15.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:15.568 | INFO | Uploaded 36GiB to GPU in 2.01128s (17.899GiB/s) | 20260507 03:59:15.568 | INFO | Downloaded 34GiB to GPU in 1.9367s (17.5556GiB/s) | 20260507 03:59:20.702 | INFO | Analyzing chunk frames = [6920192,6920704) | 20260507 03:59:20.702 | INFO | MemInfo: { | 20260507 03:59:20.702 | INFO | "gpu" : " 17 GB", | 20260507 03:59:20.702 | INFO | "huge" : "", | 20260507 03:59:20.702 | INFO | "huge_shared" : " 27 GB", | 20260507 03:59:20.702 | INFO | "malloc" : " 14 GB", | 20260507 03:59:20.702 | INFO | "pinned" : " 29 GB" | 20260507 03:59:20.702 | INFO | } | 20260507 03:59:20.702 | INFO | Perf report for the last 5.1342 seconds: | 20260507 03:59:20.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:59:20.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:20.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:20.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:20.702 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 03:59:20.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:20.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:20.702 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:59:20.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:20.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:20.702 | INFO | Uploaded 36GiB to GPU in 2.04967s (17.5638GiB/s) | 20260507 03:59:20.702 | INFO | Downloaded 34GiB to GPU in 1.94233s (17.5047GiB/s) | 20260507 03:59:25.845 | INFO | Analyzing chunk frames = [6920704,6921216) | 20260507 03:59:25.845 | INFO | MemInfo: { | 20260507 03:59:25.845 | INFO | "gpu" : " 17 GB", | 20260507 03:59:25.845 | INFO | "huge" : "", | 20260507 03:59:25.845 | INFO | "huge_shared" : " 27 GB", | 20260507 03:59:25.845 | INFO | "malloc" : " 14 GB", | 20260507 03:59:25.845 | INFO | "pinned" : " 29 GB" | 20260507 03:59:25.845 | INFO | } | 20260507 03:59:25.845 | INFO | Perf report for the last 5.14322 seconds: | 20260507 03:59:25.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:59:25.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:25.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:25.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:25.845 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 03:59:25.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:25.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:25.845 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 03:59:25.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:25.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:25.845 | INFO | Uploaded 36GiB to GPU in 2.02676s (17.7623GiB/s) | 20260507 03:59:25.845 | INFO | Downloaded 34GiB to GPU in 1.93444s (17.5761GiB/s) | 20260507 03:59:30.986 | INFO | Analyzing chunk frames = [6921216,6921728) | 20260507 03:59:30.986 | INFO | MemInfo: { | 20260507 03:59:30.986 | INFO | "gpu" : " 17 GB", | 20260507 03:59:30.986 | INFO | "huge" : "", | 20260507 03:59:30.986 | INFO | "huge_shared" : " 27 GB", | 20260507 03:59:30.986 | INFO | "malloc" : " 14 GB", | 20260507 03:59:30.986 | INFO | "pinned" : " 29 GB" | 20260507 03:59:30.986 | INFO | } | 20260507 03:59:30.986 | INFO | Perf report for the last 5.14073 seconds: | 20260507 03:59:30.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:59:30.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:30.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:30.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:30.986 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 03:59:30.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:30.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:30.986 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 03:59:30.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:30.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:30.986 | INFO | Uploaded 36GiB to GPU in 1.98053s (18.177GiB/s) | 20260507 03:59:30.986 | INFO | Downloaded 34GiB to GPU in 1.97852s (17.1846GiB/s) | 20260507 03:59:36.127 | INFO | Analyzing chunk frames = [6921728,6922240) | 20260507 03:59:36.127 | INFO | MemInfo: { | 20260507 03:59:36.127 | INFO | "gpu" : " 17 GB", | 20260507 03:59:36.127 | INFO | "huge" : "", | 20260507 03:59:36.127 | INFO | "huge_shared" : " 27 GB", | 20260507 03:59:36.127 | INFO | "malloc" : " 14 GB", | 20260507 03:59:36.127 | INFO | "pinned" : " 29 GB" | 20260507 03:59:36.127 | INFO | } | 20260507 03:59:36.127 | INFO | Perf report for the last 5.14154 seconds: | 20260507 03:59:36.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:59:36.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:36.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:36.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:36.127 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 03:59:36.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:36.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:36.127 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 03:59:36.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:36.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:36.128 | INFO | Uploaded 36GiB to GPU in 1.98715s (18.1164GiB/s) | 20260507 03:59:36.128 | INFO | Downloaded 34GiB to GPU in 1.96923s (17.2656GiB/s) | 20260507 03:59:36.128 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 03:59:36.128 | INFO | Total mem usage: 18 GiB | 20260507 03:59:36.128 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 03:59:36.128 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.8%) | 20260507 03:59:36.128 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 03:59:36.128 | INFO | High Water Breakdown: | 20260507 03:59:36.128 | INFO | 60.2% : 8.62 GiB ( 75.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 03:59:36.128 | INFO | 39.6% : 5.67 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 03:59:36.128 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 03:59:36.128 | INFO | ------------------------------------------ | 20260507 03:59:36.128 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:59:36.128 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 03:59:36.128 | INFO | Total mem usage: 26.5 GiB | 20260507 03:59:36.128 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 03:59:36.128 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 03:59:36.128 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 03:59:36.128 | INFO | High Water Breakdown: | 20260507 03:59:36.128 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:59:36.128 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:59:36.128 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:59:36.128 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:59:36.128 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:59:36.128 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:59:36.128 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 03:59:36.128 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 03:59:36.128 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:59:36.128 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 03:59:36.128 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 03:59:36.128 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 03:59:36.128 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 03:59:36.128 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 03:59:36.128 | INFO | ------------------------------------------ | 20260507 03:59:36.128 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:59:36.128 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 03:59:36.128 | INFO | Total mem usage: 25.6 GiB | 20260507 03:59:36.128 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 03:59:36.128 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 03:59:36.128 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 03:59:36.128 | INFO | High Water Breakdown: | 20260507 03:59:36.128 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 03:59:36.128 | INFO | ------------------------------------------ | 20260507 03:59:36.128 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:59:36.128 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 03:59:36.128 | INFO | Total mem usage: 15.7 GiB | 20260507 03:59:36.128 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 03:59:36.128 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 03:59:36.128 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 03:59:36.128 | INFO | High Water Breakdown: | 20260507 03:59:36.128 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 03:59:36.128 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 03:59:36.128 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 03:59:36.128 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 03:59:36.128 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 03:59:36.128 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 03:59:36.128 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 03:59:36.128 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 03:59:36.128 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 03:59:36.128 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 03:59:36.128 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 03:59:36.128 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 03:59:36.128 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 03:59:36.128 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 03:59:36.128 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 03:59:36.128 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 03:59:36.128 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 03:59:36.128 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 03:59:36.128 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 03:59:36.128 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 03:59:36.128 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 03:59:36.128 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 03:59:36.128 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 03:59:36.128 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 03:59:36.128 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 03:59:36.128 | INFO | ------------------------------------------ | 20260507 03:59:36.128 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 03:59:36.128 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 03:59:36.128 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:59:36.128 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:59:36.128 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.50e+01, 1.34e+01 / 1.78e+00 | 20260507 03:59:36.128 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 9.94e-02, 9.81e-03 / 6.43e-03 | 20260507 03:59:36.129 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.39e+00, 5.14e-01 / 3.20e-01 | 20260507 03:59:36.129 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.44e+00, 1.65e-01 / 1.31e-01 | 20260507 03:59:36.129 | INFO | PerfReport: 16.20%, DME -- 3.98e+01 / 4.47e+01, 3.82e+00 / 2.04e+00 | 20260507 03:59:36.129 | INFO | PerfReport: 57.54%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.17e+00 / 1.28e+00 | 20260507 03:59:36.129 | INFO | PerfReport: 6.99%, PulseAccumulating -- 2.15e+00 / 2.70e+00, 3.87e-01 / 2.06e-01 | 20260507 03:59:36.129 | INFO | PerfReport: 3.03%, Metrics -- 9.32e-01 / 1.23e+00, 2.45e-01 / 1.43e-01 | 20260507 03:59:36.129 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.16e-01 / 3.68e-01, 4.95e-02 / 2.11e-02 | 20260507 03:59:36.129 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 03:59:36.129 | INFO | PerfReport: Gathered 1536 samples. | 20260507 03:59:36.129 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 03:59:36.129 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 03:59:36.129 | INFO | PerfReport: 21.85%, Upload -- 2.10e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260507 03:59:36.129 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260507 03:59:36.129 | INFO | PerfReport: 31.90%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 03:59:36.129 | INFO | PerfReport: 1.55%, LockUpload -- 1.49e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 03:59:36.129 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260507 03:59:36.129 | INFO | PerfReport: 21.99%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 03:59:36.129 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260507 03:59:36.224 | INFO | Starting a round of Baz IO | 20260507 03:59:39.327 | INFO | Populating RTMetrics report | 20260507 03:59:39.328 | INFO | RTMetrics report populated | 20260507 03:59:39.328 | INFO | Opening tmp RTMetrics json file | 20260507 03:59:39.329 | INFO | Serializing RTMetrics to json | 20260507 03:59:39.335 | INFO | Serialization done, swapping tmp file now... | 20260507 03:59:39.348 | INFO | RTMetrics done | 20260507 03:59:39.422 | INFO | Finished a round of Baz IO: Wrote 1.738 GiB in 3.197 seconds | 20260507 03:59:39.422 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.84% / 0.00% / 67.16% | 20260507 03:59:39.422 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.1834% / 0% / 7.15256e-05% | 20260507 03:59:39.422 | INFO | Baz IO overall: | 20260507 03:59:39.422 | INFO | BytesWritten : 3542246295271 | 20260507 03:59:39.422 | INFO | NumEvents : 1701248137182 | 20260507 03:59:41.271 | INFO | Analyzing chunk frames = [6922240,6922752) | 20260507 03:59:41.271 | INFO | MemInfo: { | 20260507 03:59:41.271 | INFO | "gpu" : " 17 GB", | 20260507 03:59:41.271 | INFO | "huge" : "", | 20260507 03:59:41.271 | INFO | "huge_shared" : " 27 GB", | 20260507 03:59:41.271 | INFO | "malloc" : " 14 GB", | 20260507 03:59:41.271 | INFO | "pinned" : " 29 GB" | 20260507 03:59:41.271 | INFO | } | 20260507 03:59:41.271 | INFO | Perf report for the last 5.14345 seconds: | 20260507 03:59:41.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:59:41.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:41.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:41.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:41.271 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 03:59:41.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:41.271 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 03:59:41.271 | INFO | PRE_HQ : 59.1%, 1.8, 54.9ms, 0 | 20260507 03:59:41.271 | INFO | BAZWRITER : 61.8%, 3.3, 104.4ms, 0 | 20260507 03:59:41.271 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 03:59:41.271 | INFO | Uploaded 36GiB to GPU in 2.06509s (17.4326GiB/s) | 20260507 03:59:41.271 | INFO | Downloaded 38GiB to GPU in 2.17841s (17.4439GiB/s) | 20260507 03:59:46.409 | INFO | Analyzing chunk frames = [6922752,6923264) | 20260507 03:59:46.409 | INFO | MemInfo: { | 20260507 03:59:46.409 | INFO | "gpu" : " 17 GB", | 20260507 03:59:46.409 | INFO | "huge" : "", | 20260507 03:59:46.409 | INFO | "huge_shared" : " 27 GB", | 20260507 03:59:46.409 | INFO | "malloc" : " 14 GB", | 20260507 03:59:46.409 | INFO | "pinned" : " 29 GB" | 20260507 03:59:46.409 | INFO | } | 20260507 03:59:46.409 | INFO | Perf report for the last 5.13854 seconds: | 20260507 03:59:46.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:59:46.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:46.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:46.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:46.409 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 03:59:46.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:46.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:46.409 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 03:59:46.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:46.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:46.410 | INFO | Uploaded 36GiB to GPU in 2.013s (17.8838GiB/s) | 20260507 03:59:46.410 | INFO | Downloaded 34GiB to GPU in 1.93678s (17.5549GiB/s) | 20260507 03:59:51.549 | INFO | Analyzing chunk frames = [6923264,6923776) | 20260507 03:59:51.549 | INFO | MemInfo: { | 20260507 03:59:51.549 | INFO | "gpu" : " 17 GB", | 20260507 03:59:51.549 | INFO | "huge" : "", | 20260507 03:59:51.549 | INFO | "huge_shared" : " 27 GB", | 20260507 03:59:51.549 | INFO | "malloc" : " 14 GB", | 20260507 03:59:51.549 | INFO | "pinned" : " 29 GB" | 20260507 03:59:51.549 | INFO | } | 20260507 03:59:51.549 | INFO | Perf report for the last 5.13941 seconds: | 20260507 03:59:51.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:59:51.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:51.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:51.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:51.549 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 03:59:51.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:51.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:51.549 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 03:59:51.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:51.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:51.549 | INFO | Uploaded 36GiB to GPU in 2.01057s (17.9054GiB/s) | 20260507 03:59:51.549 | INFO | Downloaded 34GiB to GPU in 1.94053s (17.521GiB/s) | 20260507 03:59:56.682 | INFO | Analyzing chunk frames = [6923776,6924288) | 20260507 03:59:56.683 | INFO | MemInfo: { | 20260507 03:59:56.683 | INFO | "gpu" : " 17 GB", | 20260507 03:59:56.683 | INFO | "huge" : "", | 20260507 03:59:56.683 | INFO | "huge_shared" : " 27 GB", | 20260507 03:59:56.683 | INFO | "malloc" : " 14 GB", | 20260507 03:59:56.683 | INFO | "pinned" : " 29 GB" | 20260507 03:59:56.683 | INFO | } | 20260507 03:59:56.683 | INFO | Perf report for the last 5.1339 seconds: | 20260507 03:59:56.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 03:59:56.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:56.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:56.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:56.683 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 03:59:56.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:56.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:56.683 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 03:59:56.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:56.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 03:59:56.683 | INFO | Uploaded 36GiB to GPU in 2.00512s (17.954GiB/s) | 20260507 03:59:56.683 | INFO | Downloaded 34GiB to GPU in 1.93223s (17.5962GiB/s) | 20260507 04:00:01.823 | INFO | Analyzing chunk frames = [6924288,6924800) | 20260507 04:00:01.823 | INFO | MemInfo: { | 20260507 04:00:01.823 | INFO | "gpu" : " 17 GB", | 20260507 04:00:01.824 | INFO | "huge" : "", | 20260507 04:00:01.824 | INFO | "huge_shared" : " 27 GB", | 20260507 04:00:01.824 | INFO | "malloc" : " 14 GB", | 20260507 04:00:01.824 | INFO | "pinned" : " 29 GB" | 20260507 04:00:01.824 | INFO | } | 20260507 04:00:01.824 | INFO | Perf report for the last 5.14109 seconds: | 20260507 04:00:01.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:00:01.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:01.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:01.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:01.824 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 04:00:01.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:01.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:01.824 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:00:01.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:01.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:01.824 | INFO | Uploaded 36GiB to GPU in 2.02843s (17.7477GiB/s) | 20260507 04:00:01.824 | INFO | Downloaded 34GiB to GPU in 1.93486s (17.5724GiB/s) | 20260507 04:00:06.956 | INFO | Analyzing chunk frames = [6924800,6925312) | 20260507 04:00:06.956 | INFO | MemInfo: { | 20260507 04:00:06.956 | INFO | "gpu" : " 17 GB", | 20260507 04:00:06.956 | INFO | "huge" : "", | 20260507 04:00:06.956 | INFO | "huge_shared" : " 27 GB", | 20260507 04:00:06.956 | INFO | "malloc" : " 14 GB", | 20260507 04:00:06.956 | INFO | "pinned" : " 29 GB" | 20260507 04:00:06.956 | INFO | } | 20260507 04:00:06.956 | INFO | Perf report for the last 5.13257 seconds: | 20260507 04:00:06.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:00:06.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:06.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:06.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:06.956 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 04:00:06.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:06.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:06.956 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 04:00:06.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:06.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:06.957 | INFO | Uploaded 36GiB to GPU in 2.03185s (17.7179GiB/s) | 20260507 04:00:06.957 | INFO | Downloaded 34GiB to GPU in 1.90699s (17.8291GiB/s) | 20260507 04:00:12.102 | INFO | Analyzing chunk frames = [6925312,6925824) | 20260507 04:00:12.102 | INFO | MemInfo: { | 20260507 04:00:12.102 | INFO | "gpu" : " 17 GB", | 20260507 04:00:12.102 | INFO | "huge" : "", | 20260507 04:00:12.102 | INFO | "huge_shared" : " 27 GB", | 20260507 04:00:12.102 | INFO | "malloc" : " 14 GB", | 20260507 04:00:12.102 | INFO | "pinned" : " 29 GB" | 20260507 04:00:12.102 | INFO | } | 20260507 04:00:12.102 | INFO | Perf report for the last 5.14616 seconds: | 20260507 04:00:12.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:00:12.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:12.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:12.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:12.103 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:00:12.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:12.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:12.103 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:00:12.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:12.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:12.103 | INFO | Uploaded 36GiB to GPU in 1.99147s (18.0771GiB/s) | 20260507 04:00:12.103 | INFO | Downloaded 34GiB to GPU in 1.98568s (17.1226GiB/s) | 20260507 04:00:17.247 | INFO | Analyzing chunk frames = [6925824,6926336) | 20260507 04:00:17.247 | INFO | MemInfo: { | 20260507 04:00:17.247 | INFO | "gpu" : " 17 GB", | 20260507 04:00:17.247 | INFO | "huge" : "", | 20260507 04:00:17.247 | INFO | "huge_shared" : " 27 GB", | 20260507 04:00:17.248 | INFO | "malloc" : " 14 GB", | 20260507 04:00:17.248 | INFO | "pinned" : " 29 GB" | 20260507 04:00:17.248 | INFO | } | 20260507 04:00:17.248 | INFO | Perf report for the last 5.1452 seconds: | 20260507 04:00:17.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:00:17.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:17.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:17.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:17.248 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:00:17.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:17.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:17.248 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 04:00:17.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:17.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:17.248 | INFO | Uploaded 36GiB to GPU in 1.95354s (18.4281GiB/s) | 20260507 04:00:17.248 | INFO | Downloaded 34GiB to GPU in 1.97682s (17.1993GiB/s) | 20260507 04:00:17.345 | INFO | Starting a round of Baz IO | 20260507 04:00:20.444 | INFO | Populating RTMetrics report | 20260507 04:00:20.445 | INFO | RTMetrics report populated | 20260507 04:00:20.445 | INFO | Opening tmp RTMetrics json file | 20260507 04:00:20.451 | INFO | Serializing RTMetrics to json | 20260507 04:00:20.462 | INFO | Serialization done, swapping tmp file now... | 20260507 04:00:20.463 | INFO | RTMetrics done | 20260507 04:00:20.533 | INFO | Finished a round of Baz IO: Wrote 1.737 GiB in 3.187 seconds | 20260507 04:00:20.533 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.83% / 0.00% / 67.16% | 20260507 04:00:20.533 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.182962% / 1.19209e-05% / 5.5631e-05% | 20260507 04:00:20.533 | INFO | Baz IO overall: | 20260507 04:00:20.533 | INFO | BytesWritten : 3544111403642 | 20260507 04:00:20.533 | INFO | NumEvents : 1702126355135 | 20260507 04:00:22.381 | INFO | Analyzing chunk frames = [6926336,6926848) | 20260507 04:00:22.381 | INFO | MemInfo: { | 20260507 04:00:22.381 | INFO | "gpu" : " 17 GB", | 20260507 04:00:22.381 | INFO | "huge" : "", | 20260507 04:00:22.381 | INFO | "huge_shared" : " 27 GB", | 20260507 04:00:22.381 | INFO | "malloc" : " 14 GB", | 20260507 04:00:22.381 | INFO | "pinned" : " 29 GB" | 20260507 04:00:22.381 | INFO | } | 20260507 04:00:22.381 | INFO | Perf report for the last 5.1338 seconds: | 20260507 04:00:22.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:00:22.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:22.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:22.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:22.381 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 04:00:22.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:22.382 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 04:00:22.382 | INFO | PRE_HQ : 59.2%, 1.8, 55.4ms, 0 | 20260507 04:00:22.382 | INFO | BAZWRITER : 61.7%, 3.2, 102.5ms, 0 | 20260507 04:00:22.382 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 04:00:22.382 | INFO | Uploaded 36GiB to GPU in 2.0451s (17.603GiB/s) | 20260507 04:00:22.382 | INFO | Downloaded 38GiB to GPU in 2.18069s (17.4257GiB/s) | 20260507 04:00:27.513 | INFO | Analyzing chunk frames = [6926848,6927360) | 20260507 04:00:27.513 | INFO | MemInfo: { | 20260507 04:00:27.513 | INFO | "gpu" : " 17 GB", | 20260507 04:00:27.513 | INFO | "huge" : "", | 20260507 04:00:27.513 | INFO | "huge_shared" : " 27 GB", | 20260507 04:00:27.513 | INFO | "malloc" : " 14 GB", | 20260507 04:00:27.513 | INFO | "pinned" : " 29 GB" | 20260507 04:00:27.513 | INFO | } | 20260507 04:00:27.513 | INFO | Perf report for the last 5.13128 seconds: | 20260507 04:00:27.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:00:27.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:27.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:27.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:27.513 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 04:00:27.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:27.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:27.513 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 04:00:27.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:27.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:27.513 | INFO | Uploaded 36GiB to GPU in 2.00045s (17.996GiB/s) | 20260507 04:00:27.513 | INFO | Downloaded 34GiB to GPU in 1.9325s (17.5938GiB/s) | 20260507 04:00:32.656 | INFO | Analyzing chunk frames = [6927360,6927872) | 20260507 04:00:32.656 | INFO | MemInfo: { | 20260507 04:00:32.656 | INFO | "gpu" : " 17 GB", | 20260507 04:00:32.656 | INFO | "huge" : "", | 20260507 04:00:32.656 | INFO | "huge_shared" : " 27 GB", | 20260507 04:00:32.656 | INFO | "malloc" : " 14 GB", | 20260507 04:00:32.656 | INFO | "pinned" : " 29 GB" | 20260507 04:00:32.656 | INFO | } | 20260507 04:00:32.656 | INFO | Perf report for the last 5.14368 seconds: | 20260507 04:00:32.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:00:32.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:32.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:32.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:32.657 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 04:00:32.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:32.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:32.657 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:00:32.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:32.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:32.657 | INFO | Uploaded 36GiB to GPU in 2.00227s (17.9796GiB/s) | 20260507 04:00:32.657 | INFO | Downloaded 34GiB to GPU in 1.93957s (17.5296GiB/s) | 20260507 04:00:37.799 | INFO | Analyzing chunk frames = [6927872,6928384) | 20260507 04:00:37.799 | INFO | MemInfo: { | 20260507 04:00:37.799 | INFO | "gpu" : " 17 GB", | 20260507 04:00:37.799 | INFO | "huge" : "", | 20260507 04:00:37.799 | INFO | "huge_shared" : " 27 GB", | 20260507 04:00:37.799 | INFO | "malloc" : " 14 GB", | 20260507 04:00:37.799 | INFO | "pinned" : " 29 GB" | 20260507 04:00:37.799 | INFO | } | 20260507 04:00:37.799 | INFO | Perf report for the last 5.14283 seconds: | 20260507 04:00:37.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:00:37.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:37.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:37.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:37.799 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 04:00:37.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:37.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:37.799 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:00:37.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:37.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:37.800 | INFO | Uploaded 36GiB to GPU in 1.97776s (18.2025GiB/s) | 20260507 04:00:37.800 | INFO | Downloaded 34GiB to GPU in 1.90942s (17.8065GiB/s) | 20260507 04:00:42.938 | INFO | Analyzing chunk frames = [6928384,6928896) | 20260507 04:00:42.938 | INFO | MemInfo: { | 20260507 04:00:42.938 | INFO | "gpu" : " 17 GB", | 20260507 04:00:42.938 | INFO | "huge" : "", | 20260507 04:00:42.938 | INFO | "huge_shared" : " 27 GB", | 20260507 04:00:42.938 | INFO | "malloc" : " 14 GB", | 20260507 04:00:42.938 | INFO | "pinned" : " 29 GB" | 20260507 04:00:42.938 | INFO | } | 20260507 04:00:42.939 | INFO | Perf report for the last 5.13918 seconds: | 20260507 04:00:42.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:00:42.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:42.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:42.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:42.939 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 04:00:42.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:42.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:42.939 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 04:00:42.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:42.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:42.939 | INFO | Uploaded 36GiB to GPU in 2.00342s (17.9693GiB/s) | 20260507 04:00:42.939 | INFO | Downloaded 34GiB to GPU in 1.94176s (17.5099GiB/s) | 20260507 04:00:48.076 | INFO | Analyzing chunk frames = [6928896,6929408) | 20260507 04:00:48.076 | INFO | MemInfo: { | 20260507 04:00:48.076 | INFO | "gpu" : " 17 GB", | 20260507 04:00:48.076 | INFO | "huge" : "", | 20260507 04:00:48.076 | INFO | "huge_shared" : " 27 GB", | 20260507 04:00:48.077 | INFO | "malloc" : " 14 GB", | 20260507 04:00:48.077 | INFO | "pinned" : " 29 GB" | 20260507 04:00:48.077 | INFO | } | 20260507 04:00:48.077 | INFO | Perf report for the last 5.13815 seconds: | 20260507 04:00:48.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:00:48.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:48.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:48.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:48.077 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 04:00:48.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:48.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:48.077 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:00:48.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:48.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:48.077 | INFO | Uploaded 36GiB to GPU in 2.0353s (17.6878GiB/s) | 20260507 04:00:48.077 | INFO | Downloaded 34GiB to GPU in 1.91194s (17.783GiB/s) | 20260507 04:00:53.212 | INFO | Analyzing chunk frames = [6929408,6929920) | 20260507 04:00:53.212 | INFO | MemInfo: { | 20260507 04:00:53.212 | INFO | "gpu" : " 17 GB", | 20260507 04:00:53.212 | INFO | "huge" : "", | 20260507 04:00:53.212 | INFO | "huge_shared" : " 27 GB", | 20260507 04:00:53.213 | INFO | "malloc" : " 14 GB", | 20260507 04:00:53.213 | INFO | "pinned" : " 29 GB" | 20260507 04:00:53.213 | INFO | } | 20260507 04:00:53.213 | INFO | Perf report for the last 5.13598 seconds: | 20260507 04:00:53.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:00:53.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:53.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:53.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:53.213 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:00:53.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:53.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:53.213 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 04:00:53.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:53.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:53.213 | INFO | Uploaded 36GiB to GPU in 1.96844s (18.2886GiB/s) | 20260507 04:00:53.213 | INFO | Downloaded 34GiB to GPU in 1.97549s (17.211GiB/s) | 20260507 04:00:58.357 | INFO | Analyzing chunk frames = [6929920,6930432) | 20260507 04:00:58.357 | INFO | MemInfo: { | 20260507 04:00:58.357 | INFO | "gpu" : " 17 GB", | 20260507 04:00:58.357 | INFO | "huge" : "", | 20260507 04:00:58.357 | INFO | "huge_shared" : " 27 GB", | 20260507 04:00:58.357 | INFO | "malloc" : " 14 GB", | 20260507 04:00:58.357 | INFO | "pinned" : " 29 GB" | 20260507 04:00:58.357 | INFO | } | 20260507 04:00:58.357 | INFO | Perf report for the last 5.14468 seconds: | 20260507 04:00:58.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:00:58.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:58.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:58.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:58.357 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:00:58.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:58.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:58.357 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:00:58.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:58.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:00:58.358 | INFO | Uploaded 36GiB to GPU in 1.96005s (18.3669GiB/s) | 20260507 04:00:58.358 | INFO | Downloaded 34GiB to GPU in 1.97933s (17.1775GiB/s) | 20260507 04:00:58.358 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:00:58.358 | INFO | Total mem usage: 18 GiB | 20260507 04:00:58.358 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:00:58.358 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 04:00:58.358 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:00:58.358 | INFO | High Water Breakdown: | 20260507 04:00:58.358 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:00:58.358 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:00:58.358 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:00:58.358 | INFO | ------------------------------------------ | 20260507 04:00:58.358 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:00:58.358 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:00:58.358 | INFO | Total mem usage: 26.5 GiB | 20260507 04:00:58.358 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:00:58.358 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:00:58.358 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:00:58.358 | INFO | High Water Breakdown: | 20260507 04:00:58.358 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:00:58.358 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:00:58.358 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:00:58.358 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:00:58.358 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:00:58.358 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:00:58.358 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:00:58.358 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:00:58.358 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:00:58.358 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:00:58.358 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:00:58.358 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:00:58.358 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:00:58.358 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:00:58.358 | INFO | ------------------------------------------ | 20260507 04:00:58.358 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:00:58.358 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:00:58.358 | INFO | Total mem usage: 25.6 GiB | 20260507 04:00:58.358 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:00:58.358 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:00:58.358 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:00:58.358 | INFO | High Water Breakdown: | 20260507 04:00:58.358 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:00:58.358 | INFO | ------------------------------------------ | 20260507 04:00:58.358 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:00:58.358 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:00:58.358 | INFO | Total mem usage: 15.7 GiB | 20260507 04:00:58.358 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:00:58.358 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:00:58.358 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:00:58.358 | INFO | High Water Breakdown: | 20260507 04:00:58.358 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:00:58.358 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:00:58.358 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:00:58.358 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:00:58.358 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:00:58.358 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:00:58.358 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:00:58.358 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:00:58.358 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:00:58.358 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:00:58.358 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:00:58.358 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:00:58.358 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:00:58.358 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:00:58.358 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:00:58.358 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:00:58.358 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:00:58.358 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:00:58.358 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:00:58.358 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:00:58.358 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:00:58.358 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:00:58.358 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:00:58.358 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:00:58.358 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:00:58.358 | INFO | ------------------------------------------ | 20260507 04:00:58.358 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:00:58.359 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:00:58.359 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:00:58.359 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:00:58.359 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.51e+01, 1.35e+01 / 1.91e+00 | 20260507 04:00:58.359 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.57e-03 / 4.26e-03 | 20260507 04:00:58.359 | INFO | PerfReport: 11.69%, Baseline -- 3.60e+00 / 4.41e+00, 5.26e-01 / 2.97e-01 | 20260507 04:00:58.359 | INFO | PerfReport: 3.24%, Histogram -- 9.98e-01 / 1.48e+00, 1.72e-01 / 9.29e-02 | 20260507 04:00:58.359 | INFO | PerfReport: 16.12%, DME -- 3.97e+01 / 4.44e+01, 3.98e+00 / 2.15e+00 | 20260507 04:00:58.359 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.34e+00 / 1.72e+00 | 20260507 04:00:58.359 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.14e+00 / 2.69e+00, 3.79e-01 / 2.43e-01 | 20260507 04:00:58.359 | INFO | PerfReport: 3.03%, Metrics -- 9.33e-01 / 1.24e+00, 2.47e-01 / 1.20e-01 | 20260507 04:00:58.359 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.15e-01 / 3.67e-01, 4.97e-02 / 1.75e-02 | 20260507 04:00:58.359 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:00:58.359 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:00:58.359 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:00:58.359 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 04:00:58.359 | INFO | PerfReport: 21.73%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260507 04:00:58.359 | INFO | PerfReport: 21.37%, Download -- 2.05e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 04:00:58.359 | INFO | PerfReport: 32.02%, Compute -- 3.08e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 04:00:58.359 | INFO | PerfReport: 1.40%, LockUpload -- 1.34e+00 / 0.00e+00, 4.71e+00 / 0.00e+00 | 20260507 04:00:58.359 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.10e+00 / 0.00e+00 | 20260507 04:00:58.359 | INFO | PerfReport: 22.15%, LockCompute -- 2.13e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 04:00:58.359 | INFO | PerfReport: 0.02%, Misc -- 1.53e-02 / 0.00e+00, 2.91e-02 / 0.00e+00 | 20260507 04:00:58.454 | INFO | Starting a round of Baz IO | 20260507 04:01:01.557 | INFO | Populating RTMetrics report | 20260507 04:01:01.558 | INFO | RTMetrics report populated | 20260507 04:01:01.558 | INFO | Opening tmp RTMetrics json file | 20260507 04:01:01.559 | INFO | Serializing RTMetrics to json | 20260507 04:01:01.570 | INFO | Serialization done, swapping tmp file now... | 20260507 04:01:01.578 | INFO | RTMetrics done | 20260507 04:01:01.645 | INFO | Finished a round of Baz IO: Wrote 1.737 GiB in 3.191 seconds | 20260507 04:01:01.646 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.83% / 0.00% / 67.17% | 20260507 04:01:01.646 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.18235% / 3.97364e-06% / 5.96046e-05% | 20260507 04:01:01.646 | INFO | Baz IO overall: | 20260507 04:01:01.646 | INFO | BytesWritten : 3545976478808 | 20260507 04:01:01.646 | INFO | NumEvents : 1703004737667 | 20260507 04:01:03.494 | INFO | Analyzing chunk frames = [6930432,6930944) | 20260507 04:01:03.494 | INFO | MemInfo: { | 20260507 04:01:03.494 | INFO | "gpu" : " 17 GB", | 20260507 04:01:03.494 | INFO | "huge" : "", | 20260507 04:01:03.494 | INFO | "huge_shared" : " 27 GB", | 20260507 04:01:03.494 | INFO | "malloc" : " 14 GB", | 20260507 04:01:03.494 | INFO | "pinned" : " 29 GB" | 20260507 04:01:03.494 | INFO | } | 20260507 04:01:03.494 | INFO | Perf report for the last 5.13709 seconds: | 20260507 04:01:03.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:01:03.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:03.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:03.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:03.495 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 04:01:03.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:03.495 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 04:01:03.495 | INFO | PRE_HQ : 59.2%, 1.8, 55.5ms, 0 | 20260507 04:01:03.495 | INFO | BAZWRITER : 61.7%, 3.2, 102.5ms, 0 | 20260507 04:01:03.495 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:01:03.495 | INFO | Uploaded 36GiB to GPU in 2.04043s (17.6434GiB/s) | 20260507 04:01:03.495 | INFO | Downloaded 38GiB to GPU in 2.18399s (17.3994GiB/s) | 20260507 04:01:08.633 | INFO | Analyzing chunk frames = [6930944,6931456) | 20260507 04:01:08.633 | INFO | MemInfo: { | 20260507 04:01:08.633 | INFO | "gpu" : " 17 GB", | 20260507 04:01:08.633 | INFO | "huge" : "", | 20260507 04:01:08.633 | INFO | "huge_shared" : " 27 GB", | 20260507 04:01:08.633 | INFO | "malloc" : " 14 GB", | 20260507 04:01:08.633 | INFO | "pinned" : " 29 GB" | 20260507 04:01:08.633 | INFO | } | 20260507 04:01:08.633 | INFO | Perf report for the last 5.13846 seconds: | 20260507 04:01:08.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:01:08.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:08.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:08.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:08.633 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:01:08.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:08.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:08.633 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:01:08.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:08.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:08.633 | INFO | Uploaded 36GiB to GPU in 1.98952s (18.0948GiB/s) | 20260507 04:01:08.634 | INFO | Downloaded 34GiB to GPU in 1.9303s (17.6138GiB/s) | 20260507 04:01:13.772 | INFO | Analyzing chunk frames = [6931456,6931968) | 20260507 04:01:13.772 | INFO | MemInfo: { | 20260507 04:01:13.772 | INFO | "gpu" : " 17 GB", | 20260507 04:01:13.773 | INFO | "huge" : "", | 20260507 04:01:13.773 | INFO | "huge_shared" : " 27 GB", | 20260507 04:01:13.773 | INFO | "malloc" : " 14 GB", | 20260507 04:01:13.773 | INFO | "pinned" : " 29 GB" | 20260507 04:01:13.773 | INFO | } | 20260507 04:01:13.773 | INFO | Perf report for the last 5.13977 seconds: | 20260507 04:01:13.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:01:13.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:13.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:13.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:13.773 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 04:01:13.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:13.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:13.773 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:01:13.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:13.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:13.773 | INFO | Uploaded 36GiB to GPU in 1.98436s (18.1419GiB/s) | 20260507 04:01:13.773 | INFO | Downloaded 34GiB to GPU in 1.94438s (17.4863GiB/s) | 20260507 04:01:18.912 | INFO | Analyzing chunk frames = [6931968,6932480) | 20260507 04:01:18.912 | INFO | MemInfo: { | 20260507 04:01:18.912 | INFO | "gpu" : " 17 GB", | 20260507 04:01:18.912 | INFO | "huge" : "", | 20260507 04:01:18.912 | INFO | "huge_shared" : " 27 GB", | 20260507 04:01:18.912 | INFO | "malloc" : " 14 GB", | 20260507 04:01:18.912 | INFO | "pinned" : " 29 GB" | 20260507 04:01:18.912 | INFO | } | 20260507 04:01:18.912 | INFO | Perf report for the last 5.13959 seconds: | 20260507 04:01:18.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:01:18.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:18.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:18.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:18.912 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 04:01:18.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:18.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:18.912 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:01:18.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:18.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:18.913 | INFO | Uploaded 36GiB to GPU in 1.9536s (18.4275GiB/s) | 20260507 04:01:18.913 | INFO | Downloaded 34GiB to GPU in 1.90849s (17.8151GiB/s) | 20260507 04:01:24.055 | INFO | Analyzing chunk frames = [6932480,6932992) | 20260507 04:01:24.055 | INFO | MemInfo: { | 20260507 04:01:24.055 | INFO | "gpu" : " 17 GB", | 20260507 04:01:24.055 | INFO | "huge" : "", | 20260507 04:01:24.055 | INFO | "huge_shared" : " 27 GB", | 20260507 04:01:24.055 | INFO | "malloc" : " 14 GB", | 20260507 04:01:24.055 | INFO | "pinned" : " 29 GB" | 20260507 04:01:24.055 | INFO | } | 20260507 04:01:24.055 | INFO | Perf report for the last 5.14314 seconds: | 20260507 04:01:24.055 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:01:24.055 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:24.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:24.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:24.056 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 04:01:24.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:24.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:24.056 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:01:24.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:24.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:24.056 | INFO | Uploaded 36GiB to GPU in 1.99468s (18.048GiB/s) | 20260507 04:01:24.056 | INFO | Downloaded 34GiB to GPU in 1.94434s (17.4867GiB/s) | 20260507 04:01:29.194 | INFO | Analyzing chunk frames = [6932992,6933504) | 20260507 04:01:29.194 | INFO | MemInfo: { | 20260507 04:01:29.194 | INFO | "gpu" : " 17 GB", | 20260507 04:01:29.194 | INFO | "huge" : "", | 20260507 04:01:29.194 | INFO | "huge_shared" : " 27 GB", | 20260507 04:01:29.194 | INFO | "malloc" : " 14 GB", | 20260507 04:01:29.195 | INFO | "pinned" : " 29 GB" | 20260507 04:01:29.195 | INFO | } | 20260507 04:01:29.195 | INFO | Perf report for the last 5.13917 seconds: | 20260507 04:01:29.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:01:29.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:29.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:29.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:29.195 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 04:01:29.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:29.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:29.195 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 04:01:29.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:29.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:29.195 | INFO | Uploaded 36GiB to GPU in 2.02455s (17.7817GiB/s) | 20260507 04:01:29.195 | INFO | Downloaded 34GiB to GPU in 1.90473s (17.8503GiB/s) | 20260507 04:01:34.326 | INFO | Analyzing chunk frames = [6933504,6934016) | 20260507 04:01:34.326 | INFO | MemInfo: { | 20260507 04:01:34.326 | INFO | "gpu" : " 17 GB", | 20260507 04:01:34.326 | INFO | "huge" : "", | 20260507 04:01:34.326 | INFO | "huge_shared" : " 27 GB", | 20260507 04:01:34.326 | INFO | "malloc" : " 14 GB", | 20260507 04:01:34.326 | INFO | "pinned" : " 29 GB" | 20260507 04:01:34.326 | INFO | } | 20260507 04:01:34.326 | INFO | Perf report for the last 5.13112 seconds: | 20260507 04:01:34.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:01:34.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:34.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:34.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:34.326 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260507 04:01:34.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:34.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:34.326 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 04:01:34.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:34.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:34.326 | INFO | Uploaded 36GiB to GPU in 2.03103s (17.725GiB/s) | 20260507 04:01:34.326 | INFO | Downloaded 34GiB to GPU in 1.97191s (17.2422GiB/s) | 20260507 04:01:39.464 | INFO | Analyzing chunk frames = [6934016,6934528) | 20260507 04:01:39.464 | INFO | MemInfo: { | 20260507 04:01:39.464 | INFO | "gpu" : " 17 GB", | 20260507 04:01:39.464 | INFO | "huge" : "", | 20260507 04:01:39.464 | INFO | "huge_shared" : " 27 GB", | 20260507 04:01:39.464 | INFO | "malloc" : " 14 GB", | 20260507 04:01:39.464 | INFO | "pinned" : " 29 GB" | 20260507 04:01:39.464 | INFO | } | 20260507 04:01:39.464 | INFO | Perf report for the last 5.1382 seconds: | 20260507 04:01:39.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:01:39.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:39.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:39.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:39.464 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:01:39.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:39.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:39.464 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 04:01:39.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:39.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:39.464 | INFO | Uploaded 36GiB to GPU in 1.96286s (18.3406GiB/s) | 20260507 04:01:39.465 | INFO | Downloaded 34GiB to GPU in 1.95698s (17.3737GiB/s) | 20260507 04:01:39.561 | INFO | Starting a round of Baz IO | 20260507 04:01:42.677 | INFO | Populating RTMetrics report | 20260507 04:01:42.678 | INFO | RTMetrics report populated | 20260507 04:01:42.678 | INFO | Opening tmp RTMetrics json file | 20260507 04:01:42.682 | INFO | Serializing RTMetrics to json | 20260507 04:01:42.696 | INFO | Serialization done, swapping tmp file now... | 20260507 04:01:42.697 | INFO | RTMetrics done | 20260507 04:01:42.764 | INFO | Finished a round of Baz IO: Wrote 1.735 GiB in 3.203 seconds | 20260507 04:01:42.764 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.83% / 0.00% / 67.17% | 20260507 04:01:42.764 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.182418% / 3.97364e-06% / 7.94729e-05% | 20260507 04:01:42.764 | INFO | Baz IO overall: | 20260507 04:01:42.764 | INFO | BytesWritten : 3547839006372 | 20260507 04:01:42.764 | INFO | NumEvents : 1703881371272 | 20260507 04:01:44.610 | INFO | Analyzing chunk frames = [6934528,6935040) | 20260507 04:01:44.610 | INFO | MemInfo: { | 20260507 04:01:44.610 | INFO | "gpu" : " 17 GB", | 20260507 04:01:44.610 | INFO | "huge" : "", | 20260507 04:01:44.610 | INFO | "huge_shared" : " 27 GB", | 20260507 04:01:44.610 | INFO | "malloc" : " 14 GB", | 20260507 04:01:44.610 | INFO | "pinned" : " 29 GB" | 20260507 04:01:44.610 | INFO | } | 20260507 04:01:44.610 | INFO | Perf report for the last 5.14614 seconds: | 20260507 04:01:44.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:01:44.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:44.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:44.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:44.610 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260507 04:01:44.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:44.610 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260507 04:01:44.610 | INFO | PRE_HQ : 59.6%, 1.9, 58.9ms, 0 | 20260507 04:01:44.610 | INFO | BAZWRITER : 61.8%, 3.4, 107.2ms, 0 | 20260507 04:01:44.610 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 04:01:44.610 | INFO | Uploaded 36GiB to GPU in 2.0221s (17.8033GiB/s) | 20260507 04:01:44.611 | INFO | Downloaded 38GiB to GPU in 2.181s (17.4232GiB/s) | 20260507 04:01:49.749 | INFO | Analyzing chunk frames = [6935040,6935552) | 20260507 04:01:49.749 | INFO | MemInfo: { | 20260507 04:01:49.749 | INFO | "gpu" : " 17 GB", | 20260507 04:01:49.749 | INFO | "huge" : "", | 20260507 04:01:49.749 | INFO | "huge_shared" : " 27 GB", | 20260507 04:01:49.749 | INFO | "malloc" : " 14 GB", | 20260507 04:01:49.749 | INFO | "pinned" : " 29 GB" | 20260507 04:01:49.749 | INFO | } | 20260507 04:01:49.750 | INFO | Perf report for the last 5.13937 seconds: | 20260507 04:01:49.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:01:49.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:49.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:49.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:49.750 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 04:01:49.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:49.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:49.750 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 04:01:49.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:49.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:49.750 | INFO | Uploaded 36GiB to GPU in 2.0449s (17.6048GiB/s) | 20260507 04:01:49.750 | INFO | Downloaded 34GiB to GPU in 1.94087s (17.5179GiB/s) | 20260507 04:01:54.891 | INFO | Analyzing chunk frames = [6935552,6936064) | 20260507 04:01:54.891 | INFO | MemInfo: { | 20260507 04:01:54.891 | INFO | "gpu" : " 17 GB", | 20260507 04:01:54.891 | INFO | "huge" : "", | 20260507 04:01:54.891 | INFO | "huge_shared" : " 27 GB", | 20260507 04:01:54.891 | INFO | "malloc" : " 14 GB", | 20260507 04:01:54.891 | INFO | "pinned" : " 29 GB" | 20260507 04:01:54.891 | INFO | } | 20260507 04:01:54.891 | INFO | Perf report for the last 5.14148 seconds: | 20260507 04:01:54.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:01:54.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:54.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:54.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:54.891 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 04:01:54.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:54.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:54.891 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:01:54.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:54.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:01:54.891 | INFO | Uploaded 36GiB to GPU in 2.0171s (17.8474GiB/s) | 20260507 04:01:54.891 | INFO | Downloaded 34GiB to GPU in 1.93717s (17.5514GiB/s) | 20260507 04:02:00.028 | INFO | Analyzing chunk frames = [6936064,6936576) | 20260507 04:02:00.028 | INFO | MemInfo: { | 20260507 04:02:00.028 | INFO | "gpu" : " 17 GB", | 20260507 04:02:00.028 | INFO | "huge" : "", | 20260507 04:02:00.028 | INFO | "huge_shared" : " 27 GB", | 20260507 04:02:00.028 | INFO | "malloc" : " 14 GB", | 20260507 04:02:00.028 | INFO | "pinned" : " 29 GB" | 20260507 04:02:00.028 | INFO | } | 20260507 04:02:00.028 | INFO | Perf report for the last 5.1373 seconds: | 20260507 04:02:00.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:02:00.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:00.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:00.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:00.028 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 04:02:00.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:00.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:00.028 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:02:00.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:00.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:00.029 | INFO | Uploaded 36GiB to GPU in 2.01928s (17.8281GiB/s) | 20260507 04:02:00.029 | INFO | Downloaded 34GiB to GPU in 1.93576s (17.5642GiB/s) | 20260507 04:02:05.166 | INFO | Analyzing chunk frames = [6936576,6937088) | 20260507 04:02:05.166 | INFO | MemInfo: { | 20260507 04:02:05.166 | INFO | "gpu" : " 17 GB", | 20260507 04:02:05.166 | INFO | "huge" : "", | 20260507 04:02:05.166 | INFO | "huge_shared" : " 27 GB", | 20260507 04:02:05.166 | INFO | "malloc" : " 14 GB", | 20260507 04:02:05.166 | INFO | "pinned" : " 29 GB" | 20260507 04:02:05.166 | INFO | } | 20260507 04:02:05.166 | INFO | Perf report for the last 5.13777 seconds: | 20260507 04:02:05.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:02:05.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:05.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:05.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:05.166 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 04:02:05.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:05.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:05.166 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:02:05.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:05.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:05.166 | INFO | Uploaded 36GiB to GPU in 2.05058s (17.556GiB/s) | 20260507 04:02:05.166 | INFO | Downloaded 34GiB to GPU in 1.94482s (17.4823GiB/s) | 20260507 04:02:10.305 | INFO | Analyzing chunk frames = [6937088,6937600) | 20260507 04:02:10.306 | INFO | MemInfo: { | 20260507 04:02:10.306 | INFO | "gpu" : " 17 GB", | 20260507 04:02:10.306 | INFO | "huge" : "", | 20260507 04:02:10.306 | INFO | "huge_shared" : " 27 GB", | 20260507 04:02:10.306 | INFO | "malloc" : " 14 GB", | 20260507 04:02:10.306 | INFO | "pinned" : " 29 GB" | 20260507 04:02:10.306 | INFO | } | 20260507 04:02:10.306 | INFO | Perf report for the last 5.13965 seconds: | 20260507 04:02:10.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:02:10.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:10.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:10.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:10.306 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:02:10.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:10.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:10.306 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:02:10.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:10.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:10.306 | INFO | Uploaded 36GiB to GPU in 2.05725s (17.4991GiB/s) | 20260507 04:02:10.306 | INFO | Downloaded 34GiB to GPU in 1.95945s (17.3518GiB/s) | 20260507 04:02:15.452 | INFO | Analyzing chunk frames = [6937600,6938112) | 20260507 04:02:15.452 | INFO | MemInfo: { | 20260507 04:02:15.452 | INFO | "gpu" : " 17 GB", | 20260507 04:02:15.452 | INFO | "huge" : "", | 20260507 04:02:15.453 | INFO | "huge_shared" : " 27 GB", | 20260507 04:02:15.453 | INFO | "malloc" : " 14 GB", | 20260507 04:02:15.453 | INFO | "pinned" : " 29 GB" | 20260507 04:02:15.453 | INFO | } | 20260507 04:02:15.453 | INFO | Perf report for the last 5.14684 seconds: | 20260507 04:02:15.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:02:15.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:15.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:15.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:15.453 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 04:02:15.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:15.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:15.453 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:02:15.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:15.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:15.453 | INFO | Uploaded 36GiB to GPU in 1.97698s (18.2096GiB/s) | 20260507 04:02:15.453 | INFO | Downloaded 34GiB to GPU in 1.96793s (17.2771GiB/s) | 20260507 04:02:20.591 | INFO | Analyzing chunk frames = [6938112,6938624) | 20260507 04:02:20.591 | INFO | MemInfo: { | 20260507 04:02:20.591 | INFO | "gpu" : " 17 GB", | 20260507 04:02:20.591 | INFO | "huge" : "", | 20260507 04:02:20.591 | INFO | "huge_shared" : " 27 GB", | 20260507 04:02:20.591 | INFO | "malloc" : " 14 GB", | 20260507 04:02:20.591 | INFO | "pinned" : " 29 GB" | 20260507 04:02:20.591 | INFO | } | 20260507 04:02:20.591 | INFO | Perf report for the last 5.13845 seconds: | 20260507 04:02:20.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:02:20.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:20.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:20.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:20.591 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:02:20.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:20.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:20.591 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 04:02:20.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:20.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:20.591 | INFO | Uploaded 36GiB to GPU in 1.98196s (18.1638GiB/s) | 20260507 04:02:20.591 | INFO | Downloaded 34GiB to GPU in 1.97174s (17.2437GiB/s) | 20260507 04:02:20.592 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:02:20.592 | INFO | Total mem usage: 18 GiB | 20260507 04:02:20.592 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:02:20.592 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 04:02:20.592 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:02:20.592 | INFO | High Water Breakdown: | 20260507 04:02:20.592 | INFO | 60.2% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:02:20.592 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:02:20.592 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:02:20.592 | INFO | ------------------------------------------ | 20260507 04:02:20.592 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:02:20.592 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:02:20.592 | INFO | Total mem usage: 26.5 GiB | 20260507 04:02:20.592 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:02:20.592 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:02:20.592 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:02:20.592 | INFO | High Water Breakdown: | 20260507 04:02:20.592 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:02:20.592 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:02:20.592 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:02:20.592 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:02:20.592 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:02:20.592 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:02:20.592 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:02:20.592 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:02:20.592 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:02:20.592 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:02:20.592 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:02:20.592 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:02:20.592 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:02:20.592 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:02:20.592 | INFO | ------------------------------------------ | 20260507 04:02:20.592 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:02:20.592 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:02:20.592 | INFO | Total mem usage: 25.6 GiB | 20260507 04:02:20.592 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:02:20.592 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:02:20.592 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:02:20.592 | INFO | High Water Breakdown: | 20260507 04:02:20.592 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:02:20.592 | INFO | ------------------------------------------ | 20260507 04:02:20.592 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:02:20.592 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:02:20.592 | INFO | Total mem usage: 15.7 GiB | 20260507 04:02:20.592 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:02:20.592 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:02:20.592 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:02:20.592 | INFO | High Water Breakdown: | 20260507 04:02:20.592 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:02:20.592 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:02:20.592 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:02:20.592 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:02:20.592 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:02:20.592 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:02:20.592 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:02:20.592 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:02:20.592 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:02:20.592 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:02:20.592 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:02:20.592 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:02:20.592 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:02:20.592 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:02:20.592 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:02:20.592 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:02:20.592 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:02:20.592 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:02:20.592 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:02:20.592 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:02:20.592 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:02:20.592 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:02:20.592 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:02:20.592 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:02:20.592 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:02:20.592 | INFO | ------------------------------------------ | 20260507 04:02:20.592 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:02:20.592 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:02:20.592 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:02:20.592 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:02:20.592 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.31e+01 / 1.61e+00 | 20260507 04:02:20.592 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.01e-01, 9.71e-03 / 6.27e-03 | 20260507 04:02:20.592 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.41e+00, 5.14e-01 / 3.51e-01 | 20260507 04:02:20.592 | INFO | PerfReport: 3.22%, Histogram -- 9.91e-01 / 1.46e+00, 1.66e-01 / 1.28e-01 | 20260507 04:02:20.592 | INFO | PerfReport: 16.10%, DME -- 3.96e+01 / 4.41e+01, 3.82e+00 / 1.87e+00 | 20260507 04:02:20.592 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.33e+00 / 1.69e+00 | 20260507 04:02:20.592 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.14e+00 / 2.68e+00, 3.83e-01 / 2.12e-01 | 20260507 04:02:20.592 | INFO | PerfReport: 3.03%, Metrics -- 9.31e-01 / 1.26e+00, 2.45e-01 / 1.43e-01 | 20260507 04:02:20.592 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.15e-01 / 3.69e-01, 4.95e-02 / 2.47e-02 | 20260507 04:02:20.592 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:02:20.592 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:02:20.592 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:02:20.592 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 04:02:20.592 | INFO | PerfReport: 21.75%, Upload -- 2.09e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260507 04:02:20.592 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 04:02:20.592 | INFO | PerfReport: 31.95%, Compute -- 3.07e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 04:02:20.592 | INFO | PerfReport: 1.46%, LockUpload -- 1.41e+00 / 0.00e+00, 4.79e+00 / 0.00e+00 | 20260507 04:02:20.592 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260507 04:02:20.592 | INFO | PerfReport: 22.16%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 04:02:20.592 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.62e-02 / 0.00e+00 | 20260507 04:02:20.688 | INFO | Starting a round of Baz IO | 20260507 04:02:23.787 | INFO | Populating RTMetrics report | 20260507 04:02:23.787 | INFO | RTMetrics report populated | 20260507 04:02:23.787 | INFO | Opening tmp RTMetrics json file | 20260507 04:02:23.789 | INFO | Serializing RTMetrics to json | 20260507 04:02:23.790 | INFO | Serialization done, swapping tmp file now... | 20260507 04:02:23.790 | INFO | RTMetrics done | 20260507 04:02:23.874 | INFO | Finished a round of Baz IO: Wrote 1.738 GiB in 3.186 seconds | 20260507 04:02:23.874 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.83% / 0.00% / 67.17% | 20260507 04:02:23.875 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.18189% / 0% / 6.35783e-05% | 20260507 04:02:23.875 | INFO | Baz IO overall: | 20260507 04:02:23.875 | INFO | BytesWritten : 3549704840839 | 20260507 04:02:23.875 | INFO | NumEvents : 1704760670629 | 20260507 04:02:25.723 | INFO | Analyzing chunk frames = [6938624,6939136) | 20260507 04:02:25.723 | INFO | MemInfo: { | 20260507 04:02:25.723 | INFO | "gpu" : " 17 GB", | 20260507 04:02:25.723 | INFO | "huge" : "", | 20260507 04:02:25.723 | INFO | "huge_shared" : " 27 GB", | 20260507 04:02:25.723 | INFO | "malloc" : " 14 GB", | 20260507 04:02:25.723 | INFO | "pinned" : " 29 GB" | 20260507 04:02:25.723 | INFO | } | 20260507 04:02:25.723 | INFO | Perf report for the last 5.13232 seconds: | 20260507 04:02:25.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:02:25.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:25.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:25.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:25.724 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 04:02:25.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:25.724 | INFO | RT_METRICS : 41.5%, 1.0, 21.7ms, 0 | 20260507 04:02:25.724 | INFO | PRE_HQ : 59.6%, 1.8, 55.6ms, 0 | 20260507 04:02:25.724 | INFO | BAZWRITER : 61.7%, 3.3, 103.5ms, 0 | 20260507 04:02:25.724 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:02:25.724 | INFO | Uploaded 36GiB to GPU in 2.03931s (17.653GiB/s) | 20260507 04:02:25.724 | INFO | Downloaded 38GiB to GPU in 2.17609s (17.4625GiB/s) | 20260507 04:02:30.865 | INFO | Analyzing chunk frames = [6939136,6939648) | 20260507 04:02:30.865 | INFO | MemInfo: { | 20260507 04:02:30.865 | INFO | "gpu" : " 17 GB", | 20260507 04:02:30.865 | INFO | "huge" : "", | 20260507 04:02:30.865 | INFO | "huge_shared" : " 27 GB", | 20260507 04:02:30.865 | INFO | "malloc" : " 14 GB", | 20260507 04:02:30.865 | INFO | "pinned" : " 29 GB" | 20260507 04:02:30.865 | INFO | } | 20260507 04:02:30.865 | INFO | Perf report for the last 5.14167 seconds: | 20260507 04:02:30.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:02:30.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:30.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:30.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:30.865 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:02:30.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:30.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:30.865 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 04:02:30.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:30.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:30.865 | INFO | Uploaded 36GiB to GPU in 2.03579s (17.6836GiB/s) | 20260507 04:02:30.865 | INFO | Downloaded 34GiB to GPU in 1.94063s (17.5201GiB/s) | 20260507 04:02:36.005 | INFO | Analyzing chunk frames = [6939648,6940160) | 20260507 04:02:36.005 | INFO | MemInfo: { | 20260507 04:02:36.005 | INFO | "gpu" : " 17 GB", | 20260507 04:02:36.005 | INFO | "huge" : "", | 20260507 04:02:36.005 | INFO | "huge_shared" : " 27 GB", | 20260507 04:02:36.005 | INFO | "malloc" : " 14 GB", | 20260507 04:02:36.006 | INFO | "pinned" : " 29 GB" | 20260507 04:02:36.006 | INFO | } | 20260507 04:02:36.006 | INFO | Perf report for the last 5.14055 seconds: | 20260507 04:02:36.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:02:36.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:36.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:36.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:36.006 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 04:02:36.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:36.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:36.006 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:02:36.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:36.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:36.006 | INFO | Uploaded 36GiB to GPU in 2.00412s (17.963GiB/s) | 20260507 04:02:36.006 | INFO | Downloaded 34GiB to GPU in 1.94179s (17.5097GiB/s) | 20260507 04:02:41.139 | INFO | Analyzing chunk frames = [6940160,6940672) | 20260507 04:02:41.139 | INFO | MemInfo: { | 20260507 04:02:41.139 | INFO | "gpu" : " 17 GB", | 20260507 04:02:41.139 | INFO | "huge" : "", | 20260507 04:02:41.139 | INFO | "huge_shared" : " 27 GB", | 20260507 04:02:41.139 | INFO | "malloc" : " 14 GB", | 20260507 04:02:41.139 | INFO | "pinned" : " 29 GB" | 20260507 04:02:41.139 | INFO | } | 20260507 04:02:41.139 | INFO | Perf report for the last 5.1332 seconds: | 20260507 04:02:41.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:02:41.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:41.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:41.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:41.139 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 04:02:41.139 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:41.139 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:41.139 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:02:41.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:41.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:41.139 | INFO | Uploaded 36GiB to GPU in 1.99267s (18.0662GiB/s) | 20260507 04:02:41.139 | INFO | Downloaded 34GiB to GPU in 1.93512s (17.5699GiB/s) | 20260507 04:02:46.278 | INFO | Analyzing chunk frames = [6940672,6941184) | 20260507 04:02:46.278 | INFO | MemInfo: { | 20260507 04:02:46.278 | INFO | "gpu" : " 17 GB", | 20260507 04:02:46.278 | INFO | "huge" : "", | 20260507 04:02:46.279 | INFO | "huge_shared" : " 27 GB", | 20260507 04:02:46.279 | INFO | "malloc" : " 14 GB", | 20260507 04:02:46.279 | INFO | "pinned" : " 29 GB" | 20260507 04:02:46.279 | INFO | } | 20260507 04:02:46.279 | INFO | Perf report for the last 5.13981 seconds: | 20260507 04:02:46.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:02:46.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:46.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:46.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:46.279 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 04:02:46.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:46.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:46.279 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:02:46.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:46.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:46.279 | INFO | Uploaded 36GiB to GPU in 2.02862s (17.746GiB/s) | 20260507 04:02:46.279 | INFO | Downloaded 34GiB to GPU in 1.94348s (17.4944GiB/s) | 20260507 04:02:51.421 | INFO | Analyzing chunk frames = [6941184,6941696) | 20260507 04:02:51.421 | INFO | MemInfo: { | 20260507 04:02:51.421 | INFO | "gpu" : " 17 GB", | 20260507 04:02:51.421 | INFO | "huge" : "", | 20260507 04:02:51.421 | INFO | "huge_shared" : " 27 GB", | 20260507 04:02:51.421 | INFO | "malloc" : " 14 GB", | 20260507 04:02:51.421 | INFO | "pinned" : " 29 GB" | 20260507 04:02:51.421 | INFO | } | 20260507 04:02:51.421 | INFO | Perf report for the last 5.142 seconds: | 20260507 04:02:51.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:02:51.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:51.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:51.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:51.421 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:02:51.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:51.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:51.421 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 04:02:51.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:51.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:51.421 | INFO | Uploaded 36GiB to GPU in 2.04981s (17.5626GiB/s) | 20260507 04:02:51.421 | INFO | Downloaded 34GiB to GPU in 1.96408s (17.3109GiB/s) | 20260507 04:02:56.565 | INFO | Analyzing chunk frames = [6941696,6942208) | 20260507 04:02:56.565 | INFO | MemInfo: { | 20260507 04:02:56.565 | INFO | "gpu" : " 17 GB", | 20260507 04:02:56.565 | INFO | "huge" : "", | 20260507 04:02:56.565 | INFO | "huge_shared" : " 27 GB", | 20260507 04:02:56.565 | INFO | "malloc" : " 14 GB", | 20260507 04:02:56.565 | INFO | "pinned" : " 29 GB" | 20260507 04:02:56.565 | INFO | } | 20260507 04:02:56.565 | INFO | Perf report for the last 5.14432 seconds: | 20260507 04:02:56.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:02:56.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:56.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:56.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:56.565 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 04:02:56.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:56.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:56.565 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 04:02:56.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:56.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:02:56.565 | INFO | Uploaded 36GiB to GPU in 1.98439s (18.1416GiB/s) | 20260507 04:02:56.565 | INFO | Downloaded 34GiB to GPU in 1.98581s (17.1215GiB/s) | 20260507 04:03:01.705 | INFO | Analyzing chunk frames = [6942208,6942720) | 20260507 04:03:01.705 | INFO | MemInfo: { | 20260507 04:03:01.705 | INFO | "gpu" : " 17 GB", | 20260507 04:03:01.705 | INFO | "huge" : "", | 20260507 04:03:01.705 | INFO | "huge_shared" : " 27 GB", | 20260507 04:03:01.705 | INFO | "malloc" : " 14 GB", | 20260507 04:03:01.705 | INFO | "pinned" : " 29 GB" | 20260507 04:03:01.705 | INFO | } | 20260507 04:03:01.705 | INFO | Perf report for the last 5.14009 seconds: | 20260507 04:03:01.705 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:03:01.705 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:01.705 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:01.705 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:01.705 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:03:01.705 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:01.705 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:01.705 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:03:01.705 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:01.705 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:01.706 | INFO | Uploaded 36GiB to GPU in 1.97395s (18.2375GiB/s) | 20260507 04:03:01.706 | INFO | Downloaded 34GiB to GPU in 1.96847s (17.2723GiB/s) | 20260507 04:03:01.802 | INFO | Starting a round of Baz IO | 20260507 04:03:04.901 | INFO | Populating RTMetrics report | 20260507 04:03:04.901 | INFO | RTMetrics report populated | 20260507 04:03:04.901 | INFO | Opening tmp RTMetrics json file | 20260507 04:03:04.902 | INFO | Serializing RTMetrics to json | 20260507 04:03:04.911 | INFO | Serialization done, swapping tmp file now... | 20260507 04:03:04.921 | INFO | RTMetrics done | 20260507 04:03:04.989 | INFO | Finished a round of Baz IO: Wrote 1.736 GiB in 3.186 seconds | 20260507 04:03:04.989 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.83% / 0.00% / 67.17% | 20260507 04:03:04.989 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.181468% / 3.97364e-06% / 7.54992e-05% | 20260507 04:03:04.989 | INFO | Baz IO overall: | 20260507 04:03:04.989 | INFO | BytesWritten : 3551568853548 | 20260507 04:03:04.989 | INFO | NumEvents : 1705639032650 | 20260507 04:03:06.837 | INFO | Analyzing chunk frames = [6942720,6943232) | 20260507 04:03:06.837 | INFO | MemInfo: { | 20260507 04:03:06.837 | INFO | "gpu" : " 17 GB", | 20260507 04:03:06.837 | INFO | "huge" : "", | 20260507 04:03:06.837 | INFO | "huge_shared" : " 27 GB", | 20260507 04:03:06.837 | INFO | "malloc" : " 14 GB", | 20260507 04:03:06.837 | INFO | "pinned" : " 29 GB" | 20260507 04:03:06.837 | INFO | } | 20260507 04:03:06.837 | INFO | Perf report for the last 5.13195 seconds: | 20260507 04:03:06.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:03:06.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:06.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:06.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:06.837 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 04:03:06.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:06.837 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:03:06.837 | INFO | PRE_HQ : 59.3%, 1.8, 55.5ms, 0 | 20260507 04:03:06.837 | INFO | BAZWRITER : 61.7%, 3.2, 102.5ms, 0 | 20260507 04:03:06.837 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:03:06.837 | INFO | Uploaded 36GiB to GPU in 2.0453s (17.6014GiB/s) | 20260507 04:03:06.838 | INFO | Downloaded 38GiB to GPU in 2.18119s (17.4217GiB/s) | 20260507 04:03:11.976 | INFO | Analyzing chunk frames = [6943232,6943744) | 20260507 04:03:11.976 | INFO | MemInfo: { | 20260507 04:03:11.976 | INFO | "gpu" : " 17 GB", | 20260507 04:03:11.976 | INFO | "huge" : "", | 20260507 04:03:11.976 | INFO | "huge_shared" : " 27 GB", | 20260507 04:03:11.976 | INFO | "malloc" : " 14 GB", | 20260507 04:03:11.976 | INFO | "pinned" : " 29 GB" | 20260507 04:03:11.976 | INFO | } | 20260507 04:03:11.976 | INFO | Perf report for the last 5.13885 seconds: | 20260507 04:03:11.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:03:11.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:11.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:11.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:11.976 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 04:03:11.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:11.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:11.976 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:03:11.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:11.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:11.976 | INFO | Uploaded 36GiB to GPU in 2.03768s (17.6671GiB/s) | 20260507 04:03:11.976 | INFO | Downloaded 34GiB to GPU in 1.94245s (17.5037GiB/s) | 20260507 04:03:17.118 | INFO | Analyzing chunk frames = [6943744,6944256) | 20260507 04:03:17.118 | INFO | MemInfo: { | 20260507 04:03:17.118 | INFO | "gpu" : " 17 GB", | 20260507 04:03:17.118 | INFO | "huge" : "", | 20260507 04:03:17.118 | INFO | "huge_shared" : " 27 GB", | 20260507 04:03:17.118 | INFO | "malloc" : " 14 GB", | 20260507 04:03:17.118 | INFO | "pinned" : " 29 GB" | 20260507 04:03:17.118 | INFO | } | 20260507 04:03:17.118 | INFO | Perf report for the last 5.1422 seconds: | 20260507 04:03:17.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:03:17.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:17.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:17.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:17.118 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 04:03:17.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:17.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:17.118 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:03:17.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:17.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:17.118 | INFO | Uploaded 36GiB to GPU in 2.00914s (17.9181GiB/s) | 20260507 04:03:17.119 | INFO | Downloaded 34GiB to GPU in 1.9425s (17.5032GiB/s) | 20260507 04:03:22.260 | INFO | Analyzing chunk frames = [6944256,6944768) | 20260507 04:03:22.260 | INFO | MemInfo: { | 20260507 04:03:22.260 | INFO | "gpu" : " 17 GB", | 20260507 04:03:22.260 | INFO | "huge" : "", | 20260507 04:03:22.260 | INFO | "huge_shared" : " 27 GB", | 20260507 04:03:22.260 | INFO | "malloc" : " 14 GB", | 20260507 04:03:22.260 | INFO | "pinned" : " 29 GB" | 20260507 04:03:22.260 | INFO | } | 20260507 04:03:22.260 | INFO | Perf report for the last 5.14167 seconds: | 20260507 04:03:22.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:03:22.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:22.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:22.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:22.260 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:03:22.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:22.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:22.260 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:03:22.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:22.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:22.260 | INFO | Uploaded 36GiB to GPU in 1.97061s (18.2685GiB/s) | 20260507 04:03:22.260 | INFO | Downloaded 34GiB to GPU in 1.93468s (17.574GiB/s) | 20260507 04:03:27.399 | INFO | Analyzing chunk frames = [6944768,6945280) | 20260507 04:03:27.399 | INFO | MemInfo: { | 20260507 04:03:27.399 | INFO | "gpu" : " 17 GB", | 20260507 04:03:27.399 | INFO | "huge" : "", | 20260507 04:03:27.399 | INFO | "huge_shared" : " 27 GB", | 20260507 04:03:27.399 | INFO | "malloc" : " 14 GB", | 20260507 04:03:27.399 | INFO | "pinned" : " 29 GB" | 20260507 04:03:27.399 | INFO | } | 20260507 04:03:27.399 | INFO | Perf report for the last 5.13933 seconds: | 20260507 04:03:27.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:03:27.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:27.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:27.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:27.399 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 04:03:27.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:27.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:27.399 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:03:27.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:27.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:27.399 | INFO | Uploaded 36GiB to GPU in 2.02536s (17.7746GiB/s) | 20260507 04:03:27.400 | INFO | Downloaded 34GiB to GPU in 1.94054s (17.5209GiB/s) | 20260507 04:03:32.540 | INFO | Analyzing chunk frames = [6945280,6945792) | 20260507 04:03:32.540 | INFO | MemInfo: { | 20260507 04:03:32.540 | INFO | "gpu" : " 17 GB", | 20260507 04:03:32.540 | INFO | "huge" : "", | 20260507 04:03:32.541 | INFO | "huge_shared" : " 27 GB", | 20260507 04:03:32.541 | INFO | "malloc" : " 14 GB", | 20260507 04:03:32.541 | INFO | "pinned" : " 29 GB" | 20260507 04:03:32.541 | INFO | } | 20260507 04:03:32.541 | INFO | Perf report for the last 5.14144 seconds: | 20260507 04:03:32.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:03:32.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:32.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:32.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:32.541 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 04:03:32.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:32.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:32.541 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 04:03:32.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:32.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:32.541 | INFO | Uploaded 36GiB to GPU in 2.03012s (17.733GiB/s) | 20260507 04:03:32.541 | INFO | Downloaded 34GiB to GPU in 1.91345s (17.7689GiB/s) | 20260507 04:03:37.679 | INFO | Analyzing chunk frames = [6945792,6946304) | 20260507 04:03:37.679 | INFO | MemInfo: { | 20260507 04:03:37.679 | INFO | "gpu" : " 17 GB", | 20260507 04:03:37.679 | INFO | "huge" : "", | 20260507 04:03:37.679 | INFO | "huge_shared" : " 27 GB", | 20260507 04:03:37.679 | INFO | "malloc" : " 14 GB", | 20260507 04:03:37.679 | INFO | "pinned" : " 29 GB" | 20260507 04:03:37.679 | INFO | } | 20260507 04:03:37.679 | INFO | Perf report for the last 5.13829 seconds: | 20260507 04:03:37.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:03:37.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:37.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:37.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:37.679 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 04:03:37.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:37.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:37.679 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 04:03:37.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:37.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:37.679 | INFO | Uploaded 36GiB to GPU in 2.00712s (17.9362GiB/s) | 20260507 04:03:37.679 | INFO | Downloaded 34GiB to GPU in 1.97846s (17.1851GiB/s) | 20260507 04:03:42.827 | INFO | Analyzing chunk frames = [6946304,6946816) | 20260507 04:03:42.827 | INFO | MemInfo: { | 20260507 04:03:42.827 | INFO | "gpu" : " 17 GB", | 20260507 04:03:42.827 | INFO | "huge" : "", | 20260507 04:03:42.827 | INFO | "huge_shared" : " 27 GB", | 20260507 04:03:42.827 | INFO | "malloc" : " 14 GB", | 20260507 04:03:42.827 | INFO | "pinned" : " 29 GB" | 20260507 04:03:42.827 | INFO | } | 20260507 04:03:42.827 | INFO | Perf report for the last 5.14831 seconds: | 20260507 04:03:42.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:03:42.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:42.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:42.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:42.827 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 04:03:42.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:42.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:42.827 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 04:03:42.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:42.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:42.828 | INFO | Uploaded 36GiB to GPU in 1.95889s (18.3777GiB/s) | 20260507 04:03:42.828 | INFO | Downloaded 34GiB to GPU in 1.95811s (17.3637GiB/s) | 20260507 04:03:42.828 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:03:42.828 | INFO | Total mem usage: 18 GiB | 20260507 04:03:42.828 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:03:42.828 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 04:03:42.828 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:03:42.828 | INFO | High Water Breakdown: | 20260507 04:03:42.828 | INFO | 60.1% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:03:42.828 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:03:42.828 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:03:42.828 | INFO | ------------------------------------------ | 20260507 04:03:42.828 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:03:42.828 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:03:42.828 | INFO | Total mem usage: 26.5 GiB | 20260507 04:03:42.828 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:03:42.828 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:03:42.828 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:03:42.828 | INFO | High Water Breakdown: | 20260507 04:03:42.828 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:03:42.828 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:03:42.828 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:03:42.828 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:03:42.828 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:03:42.828 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:03:42.828 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:03:42.828 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:03:42.828 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:03:42.828 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:03:42.828 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:03:42.828 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:03:42.828 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:03:42.828 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:03:42.828 | INFO | ------------------------------------------ | 20260507 04:03:42.828 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:03:42.828 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:03:42.828 | INFO | Total mem usage: 25.6 GiB | 20260507 04:03:42.828 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:03:42.828 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:03:42.828 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:03:42.828 | INFO | High Water Breakdown: | 20260507 04:03:42.828 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:03:42.828 | INFO | ------------------------------------------ | 20260507 04:03:42.828 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:03:42.828 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:03:42.828 | INFO | Total mem usage: 15.7 GiB | 20260507 04:03:42.829 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:03:42.829 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:03:42.829 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:03:42.829 | INFO | High Water Breakdown: | 20260507 04:03:42.829 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:03:42.829 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:03:42.829 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:03:42.829 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:03:42.829 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:03:42.829 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:03:42.829 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:03:42.829 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:03:42.829 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:03:42.829 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:03:42.829 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:03:42.829 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:03:42.829 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:03:42.829 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:03:42.829 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:03:42.829 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:03:42.829 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:03:42.829 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:03:42.829 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:03:42.829 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:03:42.829 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:03:42.829 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:03:42.829 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:03:42.829 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:03:42.829 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:03:42.829 | INFO | ------------------------------------------ | 20260507 04:03:42.829 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:03:42.829 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:03:42.829 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:03:42.829 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:03:42.829 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.31e+01 / 1.69e+00 | 20260507 04:03:42.829 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.02e-01, 9.70e-03 / 5.81e-03 | 20260507 04:03:42.829 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.42e+00, 5.06e-01 / 2.36e-01 | 20260507 04:03:42.829 | INFO | PerfReport: 3.22%, Histogram -- 9.90e-01 / 1.47e+00, 1.66e-01 / 1.17e-01 | 20260507 04:03:42.829 | INFO | PerfReport: 16.11%, DME -- 3.96e+01 / 4.42e+01, 3.90e+00 / 1.86e+00 | 20260507 04:03:42.829 | INFO | PerfReport: 57.65%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.27e+00 / 1.60e+00 | 20260507 04:03:42.829 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.15e+00 / 2.68e+00, 3.87e-01 / 2.47e-01 | 20260507 04:03:42.829 | INFO | PerfReport: 3.04%, Metrics -- 9.34e-01 / 1.24e+00, 2.50e-01 / 1.26e-01 | 20260507 04:03:42.829 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.16e-01 / 3.70e-01, 4.99e-02 / 2.70e-02 | 20260507 04:03:42.829 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:03:42.829 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:03:42.829 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:03:42.829 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 04:03:42.829 | INFO | PerfReport: 21.79%, Upload -- 2.10e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260507 04:03:42.829 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 04:03:42.829 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 04:03:42.829 | INFO | PerfReport: 1.36%, LockUpload -- 1.31e+00 / 0.00e+00, 4.73e+00 / 0.00e+00 | 20260507 04:03:42.829 | INFO | PerfReport: 1.39%, LockDownload -- 1.34e+00 / 0.00e+00, 3.16e+00 / 0.00e+00 | 20260507 04:03:42.829 | INFO | PerfReport: 22.08%, LockCompute -- 2.12e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 04:03:42.829 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260507 04:03:42.925 | INFO | Starting a round of Baz IO | 20260507 04:03:46.031 | INFO | Populating RTMetrics report | 20260507 04:03:46.031 | INFO | RTMetrics report populated | 20260507 04:03:46.031 | INFO | Opening tmp RTMetrics json file | 20260507 04:03:46.033 | INFO | Serializing RTMetrics to json | 20260507 04:03:46.034 | INFO | Serialization done, swapping tmp file now... | 20260507 04:03:46.034 | INFO | RTMetrics done | 20260507 04:03:46.121 | INFO | Finished a round of Baz IO: Wrote 1.736 GiB in 3.196 seconds | 20260507 04:03:46.121 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.83% / 0.00% / 67.17% | 20260507 04:03:46.121 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.18115% / 3.97364e-06% / 8.74201e-05% | 20260507 04:03:46.121 | INFO | Baz IO overall: | 20260507 04:03:46.121 | INFO | BytesWritten : 3553432402921 | 20260507 04:03:46.121 | INFO | NumEvents : 1706517017576 | 20260507 04:03:47.955 | INFO | Analyzing chunk frames = [6946816,6947328) | 20260507 04:03:47.955 | INFO | MemInfo: { | 20260507 04:03:47.955 | INFO | "gpu" : " 17 GB", | 20260507 04:03:47.955 | INFO | "huge" : "", | 20260507 04:03:47.955 | INFO | "huge_shared" : " 27 GB", | 20260507 04:03:47.955 | INFO | "malloc" : " 14 GB", | 20260507 04:03:47.955 | INFO | "pinned" : " 29 GB" | 20260507 04:03:47.955 | INFO | } | 20260507 04:03:47.956 | INFO | Perf report for the last 5.1282 seconds: | 20260507 04:03:47.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:03:47.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:47.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:47.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:47.956 | INFO | ANALYSIS : 61.8%, 3.0, 95.7ms, 0 | 20260507 04:03:47.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:47.956 | INFO | RT_METRICS : 44.8%, 1.0, 23.4ms, 0 | 20260507 04:03:47.956 | INFO | PRE_HQ : 60.4%, 1.9, 60.3ms, 0 | 20260507 04:03:47.956 | INFO | BAZWRITER : 61.8%, 3.4, 107.6ms, 0 | 20260507 04:03:47.956 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:03:47.956 | INFO | Uploaded 36GiB to GPU in 2.03383s (17.7006GiB/s) | 20260507 04:03:47.956 | INFO | Downloaded 38GiB to GPU in 2.18402s (17.3991GiB/s) | 20260507 04:03:53.092 | INFO | Analyzing chunk frames = [6947328,6947840) | 20260507 04:03:53.092 | INFO | MemInfo: { | 20260507 04:03:53.092 | INFO | "gpu" : " 17 GB", | 20260507 04:03:53.092 | INFO | "huge" : "", | 20260507 04:03:53.092 | INFO | "huge_shared" : " 27 GB", | 20260507 04:03:53.092 | INFO | "malloc" : " 14 GB", | 20260507 04:03:53.092 | INFO | "pinned" : " 29 GB" | 20260507 04:03:53.092 | INFO | } | 20260507 04:03:53.092 | INFO | Perf report for the last 5.13648 seconds: | 20260507 04:03:53.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:03:53.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:53.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:53.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:53.092 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 04:03:53.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:53.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:53.092 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 04:03:53.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:53.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:53.092 | INFO | Uploaded 36GiB to GPU in 2.00034s (17.9969GiB/s) | 20260507 04:03:53.092 | INFO | Downloaded 34GiB to GPU in 1.93569s (17.5648GiB/s) | 20260507 04:03:58.233 | INFO | Analyzing chunk frames = [6947840,6948352) | 20260507 04:03:58.233 | INFO | MemInfo: { | 20260507 04:03:58.233 | INFO | "gpu" : " 17 GB", | 20260507 04:03:58.233 | INFO | "huge" : "", | 20260507 04:03:58.233 | INFO | "huge_shared" : " 27 GB", | 20260507 04:03:58.233 | INFO | "malloc" : " 14 GB", | 20260507 04:03:58.233 | INFO | "pinned" : " 29 GB" | 20260507 04:03:58.233 | INFO | } | 20260507 04:03:58.234 | INFO | Perf report for the last 5.14149 seconds: | 20260507 04:03:58.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:03:58.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:58.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:58.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:58.234 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 04:03:58.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:58.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:58.234 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:03:58.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:58.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:03:58.234 | INFO | Uploaded 36GiB to GPU in 1.9754s (18.2242GiB/s) | 20260507 04:03:58.234 | INFO | Downloaded 34GiB to GPU in 1.94424s (17.4876GiB/s) | 20260507 04:04:03.374 | INFO | Analyzing chunk frames = [6948352,6948864) | 20260507 04:04:03.374 | INFO | MemInfo: { | 20260507 04:04:03.374 | INFO | "gpu" : " 17 GB", | 20260507 04:04:03.374 | INFO | "huge" : "", | 20260507 04:04:03.374 | INFO | "huge_shared" : " 27 GB", | 20260507 04:04:03.374 | INFO | "malloc" : " 14 GB", | 20260507 04:04:03.374 | INFO | "pinned" : " 29 GB" | 20260507 04:04:03.374 | INFO | } | 20260507 04:04:03.374 | INFO | Perf report for the last 5.14039 seconds: | 20260507 04:04:03.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:04:03.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:03.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:03.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:03.374 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:04:03.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:03.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:03.374 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:04:03.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:03.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:03.374 | INFO | Uploaded 36GiB to GPU in 1.97718s (18.2078GiB/s) | 20260507 04:04:03.374 | INFO | Downloaded 34GiB to GPU in 1.9214s (17.6954GiB/s) | 20260507 04:04:08.513 | INFO | Analyzing chunk frames = [6948864,6949376) | 20260507 04:04:08.513 | INFO | MemInfo: { | 20260507 04:04:08.513 | INFO | "gpu" : " 17 GB", | 20260507 04:04:08.513 | INFO | "huge" : "", | 20260507 04:04:08.513 | INFO | "huge_shared" : " 27 GB", | 20260507 04:04:08.513 | INFO | "malloc" : " 14 GB", | 20260507 04:04:08.514 | INFO | "pinned" : " 29 GB" | 20260507 04:04:08.514 | INFO | } | 20260507 04:04:08.514 | INFO | Perf report for the last 5.13967 seconds: | 20260507 04:04:08.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:04:08.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:08.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:08.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:08.514 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 04:04:08.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:08.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:08.514 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:04:08.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:08.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:08.514 | INFO | Uploaded 36GiB to GPU in 2.03072s (17.7277GiB/s) | 20260507 04:04:08.514 | INFO | Downloaded 34GiB to GPU in 1.943s (17.4987GiB/s) | 20260507 04:04:13.652 | INFO | Analyzing chunk frames = [6949376,6949888) | 20260507 04:04:13.652 | INFO | MemInfo: { | 20260507 04:04:13.652 | INFO | "gpu" : " 17 GB", | 20260507 04:04:13.652 | INFO | "huge" : "", | 20260507 04:04:13.652 | INFO | "huge_shared" : " 27 GB", | 20260507 04:04:13.652 | INFO | "malloc" : " 14 GB", | 20260507 04:04:13.652 | INFO | "pinned" : " 29 GB" | 20260507 04:04:13.652 | INFO | } | 20260507 04:04:13.652 | INFO | Perf report for the last 5.13829 seconds: | 20260507 04:04:13.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:04:13.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:13.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:13.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:13.652 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 04:04:13.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:13.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:13.652 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:04:13.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:13.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:13.652 | INFO | Uploaded 36GiB to GPU in 2.03987s (17.6482GiB/s) | 20260507 04:04:13.652 | INFO | Downloaded 34GiB to GPU in 1.89934s (17.901GiB/s) | 20260507 04:04:18.793 | INFO | Analyzing chunk frames = [6949888,6950400) | 20260507 04:04:18.793 | INFO | MemInfo: { | 20260507 04:04:18.793 | INFO | "gpu" : " 17 GB", | 20260507 04:04:18.793 | INFO | "huge" : "", | 20260507 04:04:18.793 | INFO | "huge_shared" : " 27 GB", | 20260507 04:04:18.793 | INFO | "malloc" : " 14 GB", | 20260507 04:04:18.793 | INFO | "pinned" : " 29 GB" | 20260507 04:04:18.793 | INFO | } | 20260507 04:04:18.793 | INFO | Perf report for the last 5.14096 seconds: | 20260507 04:04:18.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:04:18.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:18.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:18.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:18.793 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:04:18.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:18.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:18.793 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 04:04:18.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:18.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:18.793 | INFO | Uploaded 36GiB to GPU in 1.98182s (18.1651GiB/s) | 20260507 04:04:18.793 | INFO | Downloaded 34GiB to GPU in 1.96973s (17.2612GiB/s) | 20260507 04:04:23.931 | INFO | Analyzing chunk frames = [6950400,6950912) | 20260507 04:04:23.931 | INFO | MemInfo: { | 20260507 04:04:23.931 | INFO | "gpu" : " 17 GB", | 20260507 04:04:23.931 | INFO | "huge" : "", | 20260507 04:04:23.931 | INFO | "huge_shared" : " 27 GB", | 20260507 04:04:23.932 | INFO | "malloc" : " 14 GB", | 20260507 04:04:23.932 | INFO | "pinned" : " 29 GB" | 20260507 04:04:23.932 | INFO | } | 20260507 04:04:23.932 | INFO | Perf report for the last 5.13869 seconds: | 20260507 04:04:23.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:04:23.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:23.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:23.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:23.932 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:04:23.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:23.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:23.932 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 04:04:23.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:23.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:23.932 | INFO | Uploaded 36GiB to GPU in 1.96435s (18.3266GiB/s) | 20260507 04:04:23.932 | INFO | Downloaded 34GiB to GPU in 1.95948s (17.3515GiB/s) | 20260507 04:04:24.029 | INFO | Starting a round of Baz IO | 20260507 04:04:27.134 | INFO | Populating RTMetrics report | 20260507 04:04:27.134 | INFO | RTMetrics report populated | 20260507 04:04:27.134 | INFO | Opening tmp RTMetrics json file | 20260507 04:04:27.137 | INFO | Serializing RTMetrics to json | 20260507 04:04:27.152 | INFO | Serialization done, swapping tmp file now... | 20260507 04:04:27.153 | INFO | RTMetrics done | 20260507 04:04:27.223 | INFO | Finished a round of Baz IO: Wrote 1.734 GiB in 3.194 seconds | 20260507 04:04:27.223 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.83% / 0.00% / 67.17% | 20260507 04:04:27.223 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.179668% / 3.97364e-06% / 9.93411e-05% | 20260507 04:04:27.223 | INFO | Baz IO overall: | 20260507 04:04:27.223 | INFO | BytesWritten : 3555294138646 | 20260507 04:04:27.223 | INFO | NumEvents : 1707393811652 | 20260507 04:04:29.068 | INFO | Analyzing chunk frames = [6950912,6951424) | 20260507 04:04:29.068 | INFO | MemInfo: { | 20260507 04:04:29.068 | INFO | "gpu" : " 17 GB", | 20260507 04:04:29.068 | INFO | "huge" : "", | 20260507 04:04:29.068 | INFO | "huge_shared" : " 27 GB", | 20260507 04:04:29.068 | INFO | "malloc" : " 14 GB", | 20260507 04:04:29.068 | INFO | "pinned" : " 29 GB" | 20260507 04:04:29.068 | INFO | } | 20260507 04:04:29.068 | INFO | Perf report for the last 5.13644 seconds: | 20260507 04:04:29.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:04:29.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:29.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:29.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:29.068 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 04:04:29.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:29.068 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 04:04:29.068 | INFO | PRE_HQ : 59.1%, 1.8, 55.2ms, 0 | 20260507 04:04:29.068 | INFO | BAZWRITER : 61.7%, 3.2, 102.0ms, 0 | 20260507 04:04:29.068 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:04:29.068 | INFO | Uploaded 36GiB to GPU in 2.02917s (17.7412GiB/s) | 20260507 04:04:29.068 | INFO | Downloaded 38GiB to GPU in 2.17652s (17.459GiB/s) | 20260507 04:04:34.212 | INFO | Analyzing chunk frames = [6951424,6951936) | 20260507 04:04:34.212 | INFO | MemInfo: { | 20260507 04:04:34.212 | INFO | "gpu" : " 17 GB", | 20260507 04:04:34.212 | INFO | "huge" : "", | 20260507 04:04:34.212 | INFO | "huge_shared" : " 27 GB", | 20260507 04:04:34.212 | INFO | "malloc" : " 14 GB", | 20260507 04:04:34.212 | INFO | "pinned" : " 29 GB" | 20260507 04:04:34.212 | INFO | } | 20260507 04:04:34.212 | INFO | Perf report for the last 5.14417 seconds: | 20260507 04:04:34.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:04:34.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:34.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:34.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:34.212 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:04:34.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:34.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:34.212 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:04:34.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:34.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:34.212 | INFO | Uploaded 36GiB to GPU in 2.04263s (17.6243GiB/s) | 20260507 04:04:34.213 | INFO | Downloaded 34GiB to GPU in 1.93874s (17.5372GiB/s) | 20260507 04:04:39.351 | INFO | Analyzing chunk frames = [6951936,6952448) | 20260507 04:04:39.351 | INFO | MemInfo: { | 20260507 04:04:39.351 | INFO | "gpu" : " 17 GB", | 20260507 04:04:39.351 | INFO | "huge" : "", | 20260507 04:04:39.351 | INFO | "huge_shared" : " 27 GB", | 20260507 04:04:39.351 | INFO | "malloc" : " 14 GB", | 20260507 04:04:39.351 | INFO | "pinned" : " 29 GB" | 20260507 04:04:39.351 | INFO | } | 20260507 04:04:39.351 | INFO | Perf report for the last 5.13858 seconds: | 20260507 04:04:39.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:04:39.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:39.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:39.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:39.351 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 04:04:39.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:39.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:39.351 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:04:39.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:39.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:39.351 | INFO | Uploaded 36GiB to GPU in 2.04411s (17.6115GiB/s) | 20260507 04:04:39.351 | INFO | Downloaded 34GiB to GPU in 1.95117s (17.4254GiB/s) | 20260507 04:04:44.486 | INFO | Analyzing chunk frames = [6952448,6952960) | 20260507 04:04:44.486 | INFO | MemInfo: { | 20260507 04:04:44.486 | INFO | "gpu" : " 17 GB", | 20260507 04:04:44.486 | INFO | "huge" : "", | 20260507 04:04:44.486 | INFO | "huge_shared" : " 27 GB", | 20260507 04:04:44.486 | INFO | "malloc" : " 14 GB", | 20260507 04:04:44.486 | INFO | "pinned" : " 29 GB" | 20260507 04:04:44.486 | INFO | } | 20260507 04:04:44.486 | INFO | Perf report for the last 5.13515 seconds: | 20260507 04:04:44.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:04:44.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:44.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:44.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:44.486 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 04:04:44.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:44.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:44.486 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:04:44.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:44.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:44.486 | INFO | Uploaded 36GiB to GPU in 2.00729s (17.9346GiB/s) | 20260507 04:04:44.486 | INFO | Downloaded 34GiB to GPU in 1.93269s (17.5921GiB/s) | 20260507 04:04:49.629 | INFO | Analyzing chunk frames = [6952960,6953472) | 20260507 04:04:49.629 | INFO | MemInfo: { | 20260507 04:04:49.629 | INFO | "gpu" : " 17 GB", | 20260507 04:04:49.629 | INFO | "huge" : "", | 20260507 04:04:49.629 | INFO | "huge_shared" : " 27 GB", | 20260507 04:04:49.629 | INFO | "malloc" : " 14 GB", | 20260507 04:04:49.629 | INFO | "pinned" : " 29 GB" | 20260507 04:04:49.629 | INFO | } | 20260507 04:04:49.629 | INFO | Perf report for the last 5.14319 seconds: | 20260507 04:04:49.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:04:49.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:49.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:49.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:49.629 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 04:04:49.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:49.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:49.629 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:04:49.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:49.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:49.629 | INFO | Uploaded 36GiB to GPU in 2.03079s (17.7271GiB/s) | 20260507 04:04:49.630 | INFO | Downloaded 34GiB to GPU in 1.932s (17.5983GiB/s) | 20260507 04:04:54.767 | INFO | Analyzing chunk frames = [6953472,6953984) | 20260507 04:04:54.767 | INFO | MemInfo: { | 20260507 04:04:54.767 | INFO | "gpu" : " 17 GB", | 20260507 04:04:54.767 | INFO | "huge" : "", | 20260507 04:04:54.767 | INFO | "huge_shared" : " 27 GB", | 20260507 04:04:54.767 | INFO | "malloc" : " 14 GB", | 20260507 04:04:54.767 | INFO | "pinned" : " 29 GB" | 20260507 04:04:54.767 | INFO | } | 20260507 04:04:54.767 | INFO | Perf report for the last 5.1378 seconds: | 20260507 04:04:54.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:04:54.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:54.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:54.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:54.767 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 04:04:54.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:54.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:54.767 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 04:04:54.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:54.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:54.767 | INFO | Uploaded 36GiB to GPU in 2.0491s (17.5687GiB/s) | 20260507 04:04:54.767 | INFO | Downloaded 34GiB to GPU in 1.92082s (17.7008GiB/s) | 20260507 04:04:59.905 | INFO | Analyzing chunk frames = [6953984,6954496) | 20260507 04:04:59.905 | INFO | MemInfo: { | 20260507 04:04:59.905 | INFO | "gpu" : " 17 GB", | 20260507 04:04:59.905 | INFO | "huge" : "", | 20260507 04:04:59.905 | INFO | "huge_shared" : " 27 GB", | 20260507 04:04:59.905 | INFO | "malloc" : " 14 GB", | 20260507 04:04:59.905 | INFO | "pinned" : " 29 GB" | 20260507 04:04:59.905 | INFO | } | 20260507 04:04:59.905 | INFO | Perf report for the last 5.13825 seconds: | 20260507 04:04:59.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:04:59.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:59.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:59.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:59.905 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:04:59.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:59.905 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:59.905 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:04:59.905 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:59.905 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:04:59.906 | INFO | Uploaded 36GiB to GPU in 1.98336s (18.151GiB/s) | 20260507 04:04:59.906 | INFO | Downloaded 34GiB to GPU in 1.98095s (17.1635GiB/s) | 20260507 04:05:05.041 | INFO | Analyzing chunk frames = [6954496,6955008) | 20260507 04:05:05.041 | INFO | MemInfo: { | 20260507 04:05:05.041 | INFO | "gpu" : " 17 GB", | 20260507 04:05:05.041 | INFO | "huge" : "", | 20260507 04:05:05.041 | INFO | "huge_shared" : " 27 GB", | 20260507 04:05:05.041 | INFO | "malloc" : " 14 GB", | 20260507 04:05:05.041 | INFO | "pinned" : " 29 GB" | 20260507 04:05:05.041 | INFO | } | 20260507 04:05:05.041 | INFO | Perf report for the last 5.1357 seconds: | 20260507 04:05:05.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:05:05.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:05.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:05.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:05.041 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 04:05:05.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:05.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:05.041 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 04:05:05.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:05.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:05.041 | INFO | Uploaded 36GiB to GPU in 1.96979s (18.276GiB/s) | 20260507 04:05:05.041 | INFO | Downloaded 34GiB to GPU in 1.9707s (17.2528GiB/s) | 20260507 04:05:05.041 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:05:05.041 | INFO | Total mem usage: 18 GiB | 20260507 04:05:05.041 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:05:05.041 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 04:05:05.041 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:05:05.042 | INFO | High Water Breakdown: | 20260507 04:05:05.042 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:05:05.042 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:05:05.042 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:05:05.042 | INFO | ------------------------------------------ | 20260507 04:05:05.042 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:05:05.042 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:05:05.042 | INFO | Total mem usage: 26.5 GiB | 20260507 04:05:05.042 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:05:05.042 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 04:05:05.042 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260507 04:05:05.042 | INFO | High Water Breakdown: | 20260507 04:05:05.042 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:05:05.042 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:05:05.042 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:05:05.042 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:05:05.042 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:05:05.042 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:05:05.042 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:05:05.042 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:05:05.042 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:05:05.042 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:05:05.042 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:05:05.042 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:05:05.042 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:05:05.042 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:05:05.042 | INFO | ------------------------------------------ | 20260507 04:05:05.042 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:05:05.042 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:05:05.042 | INFO | Total mem usage: 25.6 GiB | 20260507 04:05:05.042 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:05:05.042 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:05:05.042 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:05:05.042 | INFO | High Water Breakdown: | 20260507 04:05:05.042 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:05:05.042 | INFO | ------------------------------------------ | 20260507 04:05:05.042 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:05:05.042 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:05:05.042 | INFO | Total mem usage: 15.7 GiB | 20260507 04:05:05.042 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:05:05.042 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:05:05.042 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:05:05.042 | INFO | High Water Breakdown: | 20260507 04:05:05.042 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:05:05.042 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:05:05.042 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:05:05.042 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:05:05.042 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:05:05.042 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:05:05.042 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:05:05.042 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:05:05.042 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:05:05.042 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:05:05.042 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:05:05.042 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:05:05.042 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:05:05.042 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:05:05.042 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:05:05.042 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:05:05.042 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:05:05.042 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:05:05.042 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:05:05.042 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:05:05.042 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:05:05.042 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:05:05.042 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:05:05.042 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:05:05.042 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:05:05.042 | INFO | ------------------------------------------ | 20260507 04:05:05.042 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:05:05.042 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:05:05.043 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:05:05.043 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:05:05.043 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.33e+01 / 1.06e+00 | 20260507 04:05:05.043 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 1.01e-01, 9.60e-03 / 6.88e-03 | 20260507 04:05:05.043 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.42e+00, 5.15e-01 / 2.88e-01 | 20260507 04:05:05.043 | INFO | PerfReport: 3.24%, Histogram -- 9.94e-01 / 1.44e+00, 1.68e-01 / 1.28e-01 | 20260507 04:05:05.043 | INFO | PerfReport: 16.11%, DME -- 3.96e+01 / 4.42e+01, 3.87e+00 / 1.74e+00 | 20260507 04:05:05.043 | INFO | PerfReport: 57.64%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.29e+00 / 1.56e+00 | 20260507 04:05:05.043 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.14e+00 / 2.69e+00, 3.82e-01 / 2.17e-01 | 20260507 04:05:05.043 | INFO | PerfReport: 3.04%, Metrics -- 9.33e-01 / 1.24e+00, 2.47e-01 / 1.34e-01 | 20260507 04:05:05.043 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.15e-01 / 3.65e-01, 4.96e-02 / 1.70e-02 | 20260507 04:05:05.043 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:05:05.043 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:05:05.043 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:05:05.043 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 04:05:05.043 | INFO | PerfReport: 21.77%, Upload -- 2.09e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260507 04:05:05.043 | INFO | PerfReport: 21.37%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 04:05:05.043 | INFO | PerfReport: 31.97%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 04:05:05.043 | INFO | PerfReport: 1.46%, LockUpload -- 1.40e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260507 04:05:05.043 | INFO | PerfReport: 1.33%, LockDownload -- 1.27e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260507 04:05:05.043 | INFO | PerfReport: 22.09%, LockCompute -- 2.12e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 04:05:05.043 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.68e-02 / 0.00e+00 | 20260507 04:05:05.138 | INFO | Starting a round of Baz IO | 20260507 04:05:08.256 | INFO | Populating RTMetrics report | 20260507 04:05:08.256 | INFO | RTMetrics report populated | 20260507 04:05:08.256 | INFO | Opening tmp RTMetrics json file | 20260507 04:05:08.273 | INFO | Serializing RTMetrics to json | 20260507 04:05:08.275 | INFO | Serialization done, swapping tmp file now... | 20260507 04:05:08.275 | INFO | RTMetrics done | 20260507 04:05:08.345 | INFO | Finished a round of Baz IO: Wrote 1.735 GiB in 3.207 seconds | 20260507 04:05:08.345 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.83% / 0.00% / 67.17% | 20260507 04:05:08.345 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.180105% / 0% / 9.13938e-05% | 20260507 04:05:08.345 | INFO | Baz IO overall: | 20260507 04:05:08.345 | INFO | BytesWritten : 3557157233415 | 20260507 04:05:08.345 | INFO | NumEvents : 1708271367767 | 20260507 04:05:10.185 | INFO | Analyzing chunk frames = [6955008,6955520) | 20260507 04:05:10.185 | INFO | MemInfo: { | 20260507 04:05:10.185 | INFO | "gpu" : " 17 GB", | 20260507 04:05:10.185 | INFO | "huge" : "", | 20260507 04:05:10.185 | INFO | "huge_shared" : " 27 GB", | 20260507 04:05:10.185 | INFO | "malloc" : " 14 GB", | 20260507 04:05:10.185 | INFO | "pinned" : " 29 GB" | 20260507 04:05:10.185 | INFO | } | 20260507 04:05:10.186 | INFO | Perf report for the last 5.14453 seconds: | 20260507 04:05:10.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:05:10.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:10.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:10.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:10.186 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260507 04:05:10.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:10.186 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 04:05:10.186 | INFO | PRE_HQ : 59.6%, 1.9, 58.2ms, 0 | 20260507 04:05:10.186 | INFO | BAZWRITER : 61.9%, 3.3, 104.1ms, 0 | 20260507 04:05:10.186 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:05:10.186 | INFO | Uploaded 36GiB to GPU in 2.02673s (17.7626GiB/s) | 20260507 04:05:10.186 | INFO | Downloaded 38GiB to GPU in 2.18174s (17.4173GiB/s) | 20260507 04:05:15.327 | INFO | Analyzing chunk frames = [6955520,6956032) | 20260507 04:05:15.327 | INFO | MemInfo: { | 20260507 04:05:15.327 | INFO | "gpu" : " 17 GB", | 20260507 04:05:15.327 | INFO | "huge" : "", | 20260507 04:05:15.327 | INFO | "huge_shared" : " 27 GB", | 20260507 04:05:15.327 | INFO | "malloc" : " 14 GB", | 20260507 04:05:15.327 | INFO | "pinned" : " 29 GB" | 20260507 04:05:15.327 | INFO | } | 20260507 04:05:15.327 | INFO | Perf report for the last 5.14177 seconds: | 20260507 04:05:15.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:05:15.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:15.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:15.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:15.327 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 04:05:15.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:15.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:15.327 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 04:05:15.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:15.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:15.328 | INFO | Uploaded 36GiB to GPU in 1.99018s (18.0889GiB/s) | 20260507 04:05:15.328 | INFO | Downloaded 34GiB to GPU in 1.93761s (17.5474GiB/s) | 20260507 04:05:20.463 | INFO | Analyzing chunk frames = [6956032,6956544) | 20260507 04:05:20.463 | INFO | MemInfo: { | 20260507 04:05:20.463 | INFO | "gpu" : " 17 GB", | 20260507 04:05:20.463 | INFO | "huge" : "", | 20260507 04:05:20.463 | INFO | "huge_shared" : " 27 GB", | 20260507 04:05:20.464 | INFO | "malloc" : " 14 GB", | 20260507 04:05:20.464 | INFO | "pinned" : " 29 GB" | 20260507 04:05:20.464 | INFO | } | 20260507 04:05:20.464 | INFO | Perf report for the last 5.13632 seconds: | 20260507 04:05:20.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:05:20.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:20.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:20.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:20.464 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 04:05:20.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:20.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:20.464 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:05:20.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:20.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:20.464 | INFO | Uploaded 36GiB to GPU in 2.0314s (17.7218GiB/s) | 20260507 04:05:20.464 | INFO | Downloaded 34GiB to GPU in 1.95092s (17.4277GiB/s) | 20260507 04:05:25.603 | INFO | Analyzing chunk frames = [6956544,6957056) | 20260507 04:05:25.603 | INFO | MemInfo: { | 20260507 04:05:25.603 | INFO | "gpu" : " 17 GB", | 20260507 04:05:25.603 | INFO | "huge" : "", | 20260507 04:05:25.603 | INFO | "huge_shared" : " 27 GB", | 20260507 04:05:25.603 | INFO | "malloc" : " 14 GB", | 20260507 04:05:25.603 | INFO | "pinned" : " 29 GB" | 20260507 04:05:25.603 | INFO | } | 20260507 04:05:25.603 | INFO | Perf report for the last 5.13933 seconds: | 20260507 04:05:25.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:05:25.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:25.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:25.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:25.603 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 04:05:25.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:25.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:25.603 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:05:25.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:25.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:25.603 | INFO | Uploaded 36GiB to GPU in 2.00443s (17.9602GiB/s) | 20260507 04:05:25.603 | INFO | Downloaded 34GiB to GPU in 1.9375s (17.5483GiB/s) | 20260507 04:05:30.737 | INFO | Analyzing chunk frames = [6957056,6957568) | 20260507 04:05:30.737 | INFO | MemInfo: { | 20260507 04:05:30.737 | INFO | "gpu" : " 17 GB", | 20260507 04:05:30.737 | INFO | "huge" : "", | 20260507 04:05:30.737 | INFO | "huge_shared" : " 27 GB", | 20260507 04:05:30.737 | INFO | "malloc" : " 14 GB", | 20260507 04:05:30.737 | INFO | "pinned" : " 29 GB" | 20260507 04:05:30.737 | INFO | } | 20260507 04:05:30.738 | INFO | Perf report for the last 5.1346 seconds: | 20260507 04:05:30.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:05:30.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:30.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:30.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:30.738 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 04:05:30.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:30.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:30.738 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:05:30.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:30.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:30.738 | INFO | Uploaded 36GiB to GPU in 2.02162s (17.8075GiB/s) | 20260507 04:05:30.738 | INFO | Downloaded 34GiB to GPU in 1.94242s (17.5039GiB/s) | 20260507 04:05:35.879 | INFO | Analyzing chunk frames = [6957568,6958080) | 20260507 04:05:35.879 | INFO | MemInfo: { | 20260507 04:05:35.879 | INFO | "gpu" : " 17 GB", | 20260507 04:05:35.879 | INFO | "huge" : "", | 20260507 04:05:35.879 | INFO | "huge_shared" : " 27 GB", | 20260507 04:05:35.879 | INFO | "malloc" : " 14 GB", | 20260507 04:05:35.879 | INFO | "pinned" : " 29 GB" | 20260507 04:05:35.879 | INFO | } | 20260507 04:05:35.879 | INFO | Perf report for the last 5.14145 seconds: | 20260507 04:05:35.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:05:35.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:35.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:35.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:35.879 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 04:05:35.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:35.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:35.879 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:05:35.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:35.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:35.879 | INFO | Uploaded 36GiB to GPU in 2.00564s (17.9494GiB/s) | 20260507 04:05:35.879 | INFO | Downloaded 34GiB to GPU in 1.93929s (17.5322GiB/s) | 20260507 04:05:41.020 | INFO | Analyzing chunk frames = [6958080,6958592) | 20260507 04:05:41.020 | INFO | MemInfo: { | 20260507 04:05:41.020 | INFO | "gpu" : " 17 GB", | 20260507 04:05:41.020 | INFO | "huge" : "", | 20260507 04:05:41.020 | INFO | "huge_shared" : " 27 GB", | 20260507 04:05:41.020 | INFO | "malloc" : " 14 GB", | 20260507 04:05:41.020 | INFO | "pinned" : " 29 GB" | 20260507 04:05:41.020 | INFO | } | 20260507 04:05:41.020 | INFO | Perf report for the last 5.14125 seconds: | 20260507 04:05:41.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:05:41.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:41.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:41.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:41.020 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:05:41.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:41.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:41.020 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 04:05:41.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:41.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:41.021 | INFO | Uploaded 36GiB to GPU in 1.97429s (18.2344GiB/s) | 20260507 04:05:41.021 | INFO | Downloaded 34GiB to GPU in 1.98225s (17.1522GiB/s) | 20260507 04:05:46.162 | INFO | Analyzing chunk frames = [6958592,6959104) | 20260507 04:05:46.162 | INFO | MemInfo: { | 20260507 04:05:46.162 | INFO | "gpu" : " 17 GB", | 20260507 04:05:46.162 | INFO | "huge" : "", | 20260507 04:05:46.162 | INFO | "huge_shared" : " 27 GB", | 20260507 04:05:46.162 | INFO | "malloc" : " 14 GB", | 20260507 04:05:46.162 | INFO | "pinned" : " 29 GB" | 20260507 04:05:46.162 | INFO | } | 20260507 04:05:46.162 | INFO | Perf report for the last 5.14218 seconds: | 20260507 04:05:46.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:05:46.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:46.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:46.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:46.163 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 04:05:46.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:46.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:46.163 | INFO | PRE_HQ : 29.3%, 1.0, 15.3ms, 0 | 20260507 04:05:46.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:46.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:46.163 | INFO | Uploaded 36GiB to GPU in 1.98873s (18.102GiB/s) | 20260507 04:05:46.163 | INFO | Downloaded 34GiB to GPU in 1.96821s (17.2745GiB/s) | 20260507 04:05:46.260 | INFO | Starting a round of Baz IO | 20260507 04:05:49.363 | INFO | Populating RTMetrics report | 20260507 04:05:49.363 | INFO | RTMetrics report populated | 20260507 04:05:49.363 | INFO | Opening tmp RTMetrics json file | 20260507 04:05:49.365 | INFO | Serializing RTMetrics to json | 20260507 04:05:49.382 | INFO | Serialization done, swapping tmp file now... | 20260507 04:05:49.382 | INFO | RTMetrics done | 20260507 04:05:49.455 | INFO | Finished a round of Baz IO: Wrote 1.735 GiB in 3.195 seconds | 20260507 04:05:49.455 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.83% / 0.00% / 67.17% | 20260507 04:05:49.455 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.179994% / 7.94729e-06% / 6.75519e-05% | 20260507 04:05:49.455 | INFO | Baz IO overall: | 20260507 04:05:49.455 | INFO | BytesWritten : 3559019952549 | 20260507 04:05:49.455 | INFO | NumEvents : 1709149191175 | 20260507 04:05:51.296 | INFO | Analyzing chunk frames = [6959104,6959616) | 20260507 04:05:51.296 | INFO | MemInfo: { | 20260507 04:05:51.296 | INFO | "gpu" : " 17 GB", | 20260507 04:05:51.296 | INFO | "huge" : "", | 20260507 04:05:51.296 | INFO | "huge_shared" : " 27 GB", | 20260507 04:05:51.296 | INFO | "malloc" : " 14 GB", | 20260507 04:05:51.296 | INFO | "pinned" : " 29 GB" | 20260507 04:05:51.296 | INFO | } | 20260507 04:05:51.296 | INFO | Perf report for the last 5.1334 seconds: | 20260507 04:05:51.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:05:51.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:51.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:51.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:51.296 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 04:05:51.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:51.296 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 04:05:51.296 | INFO | PRE_HQ : 59.2%, 1.8, 55.4ms, 0 | 20260507 04:05:51.296 | INFO | BAZWRITER : 61.8%, 3.2, 102.3ms, 0 | 20260507 04:05:51.296 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:05:51.296 | INFO | Uploaded 36GiB to GPU in 2.04789s (17.5791GiB/s) | 20260507 04:05:51.296 | INFO | Downloaded 38GiB to GPU in 2.17993s (17.4318GiB/s) | 20260507 04:05:56.442 | INFO | Analyzing chunk frames = [6959616,6960128) | 20260507 04:05:56.442 | INFO | MemInfo: { | 20260507 04:05:56.442 | INFO | "gpu" : " 17 GB", | 20260507 04:05:56.442 | INFO | "huge" : "", | 20260507 04:05:56.442 | INFO | "huge_shared" : " 27 GB", | 20260507 04:05:56.442 | INFO | "malloc" : " 14 GB", | 20260507 04:05:56.442 | INFO | "pinned" : " 29 GB" | 20260507 04:05:56.442 | INFO | } | 20260507 04:05:56.442 | INFO | Perf report for the last 5.14588 seconds: | 20260507 04:05:56.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:05:56.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:56.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:56.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:56.442 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 04:05:56.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:56.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:56.442 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260507 04:05:56.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:56.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:05:56.442 | INFO | Uploaded 36GiB to GPU in 2.03717s (17.6715GiB/s) | 20260507 04:05:56.442 | INFO | Downloaded 34GiB to GPU in 1.93737s (17.5496GiB/s) | 20260507 04:06:01.576 | INFO | Analyzing chunk frames = [6960128,6960640) | 20260507 04:06:01.576 | INFO | MemInfo: { | 20260507 04:06:01.576 | INFO | "gpu" : " 17 GB", | 20260507 04:06:01.576 | INFO | "huge" : "", | 20260507 04:06:01.576 | INFO | "huge_shared" : " 27 GB", | 20260507 04:06:01.577 | INFO | "malloc" : " 14 GB", | 20260507 04:06:01.577 | INFO | "pinned" : " 29 GB" | 20260507 04:06:01.577 | INFO | } | 20260507 04:06:01.577 | INFO | Perf report for the last 5.13482 seconds: | 20260507 04:06:01.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:06:01.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:01.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:01.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:01.577 | INFO | ANALYSIS : 60.3%, 3.0, 93.3ms, 0 | 20260507 04:06:01.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:01.577 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:01.577 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 04:06:01.577 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:01.577 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:01.577 | INFO | Uploaded 36GiB to GPU in 2.01986s (17.823GiB/s) | 20260507 04:06:01.577 | INFO | Downloaded 34GiB to GPU in 1.93796s (17.5442GiB/s) | 20260507 04:06:06.717 | INFO | Analyzing chunk frames = [6960640,6961152) | 20260507 04:06:06.717 | INFO | MemInfo: { | 20260507 04:06:06.717 | INFO | "gpu" : " 17 GB", | 20260507 04:06:06.717 | INFO | "huge" : "", | 20260507 04:06:06.717 | INFO | "huge_shared" : " 27 GB", | 20260507 04:06:06.717 | INFO | "malloc" : " 14 GB", | 20260507 04:06:06.717 | INFO | "pinned" : " 29 GB" | 20260507 04:06:06.717 | INFO | } | 20260507 04:06:06.718 | INFO | Perf report for the last 5.14088 seconds: | 20260507 04:06:06.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:06:06.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:06.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:06.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:06.718 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 04:06:06.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:06.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:06.718 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:06:06.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:06.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:06.718 | INFO | Uploaded 36GiB to GPU in 1.97987s (18.183GiB/s) | 20260507 04:06:06.718 | INFO | Downloaded 34GiB to GPU in 1.91989s (17.7093GiB/s) | 20260507 04:06:11.863 | INFO | Analyzing chunk frames = [6961152,6961664) | 20260507 04:06:11.863 | INFO | MemInfo: { | 20260507 04:06:11.863 | INFO | "gpu" : " 17 GB", | 20260507 04:06:11.863 | INFO | "huge" : "", | 20260507 04:06:11.863 | INFO | "huge_shared" : " 27 GB", | 20260507 04:06:11.863 | INFO | "malloc" : " 14 GB", | 20260507 04:06:11.863 | INFO | "pinned" : " 29 GB" | 20260507 04:06:11.863 | INFO | } | 20260507 04:06:11.864 | INFO | Perf report for the last 5.14603 seconds: | 20260507 04:06:11.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:06:11.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:11.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:11.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:11.864 | INFO | ANALYSIS : 60.1%, 3.0, 94.0ms, 0 | 20260507 04:06:11.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:11.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:11.864 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:06:11.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:11.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:11.864 | INFO | Uploaded 36GiB to GPU in 2.05737s (17.4981GiB/s) | 20260507 04:06:11.864 | INFO | Downloaded 34GiB to GPU in 1.94703s (17.4625GiB/s) | 20260507 04:06:16.997 | INFO | Analyzing chunk frames = [6961664,6962176) | 20260507 04:06:16.997 | INFO | MemInfo: { | 20260507 04:06:16.997 | INFO | "gpu" : " 17 GB", | 20260507 04:06:16.997 | INFO | "huge" : "", | 20260507 04:06:16.997 | INFO | "huge_shared" : " 27 GB", | 20260507 04:06:16.997 | INFO | "malloc" : " 14 GB", | 20260507 04:06:16.997 | INFO | "pinned" : " 29 GB" | 20260507 04:06:16.997 | INFO | } | 20260507 04:06:16.997 | INFO | Perf report for the last 5.13361 seconds: | 20260507 04:06:16.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:06:16.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:16.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:16.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:16.997 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 04:06:16.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:16.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:16.997 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:06:16.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:16.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:16.997 | INFO | Uploaded 36GiB to GPU in 2.05332s (17.5325GiB/s) | 20260507 04:06:16.998 | INFO | Downloaded 34GiB to GPU in 1.89571s (17.9352GiB/s) | 20260507 04:06:22.141 | INFO | Analyzing chunk frames = [6962176,6962688) | 20260507 04:06:22.141 | INFO | MemInfo: { | 20260507 04:06:22.141 | INFO | "gpu" : " 17 GB", | 20260507 04:06:22.141 | INFO | "huge" : "", | 20260507 04:06:22.141 | INFO | "huge_shared" : " 27 GB", | 20260507 04:06:22.141 | INFO | "malloc" : " 14 GB", | 20260507 04:06:22.141 | INFO | "pinned" : " 29 GB" | 20260507 04:06:22.141 | INFO | } | 20260507 04:06:22.141 | INFO | Perf report for the last 5.14362 seconds: | 20260507 04:06:22.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:06:22.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:22.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:22.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:22.141 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:06:22.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:22.141 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:22.141 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 04:06:22.141 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:22.141 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:22.141 | INFO | Uploaded 36GiB to GPU in 1.94574s (18.502GiB/s) | 20260507 04:06:22.141 | INFO | Downloaded 34GiB to GPU in 1.9583s (17.362GiB/s) | 20260507 04:06:27.273 | INFO | Analyzing chunk frames = [6962688,6963200) | 20260507 04:06:27.273 | INFO | MemInfo: { | 20260507 04:06:27.273 | INFO | "gpu" : " 17 GB", | 20260507 04:06:27.273 | INFO | "huge" : "", | 20260507 04:06:27.273 | INFO | "huge_shared" : " 27 GB", | 20260507 04:06:27.273 | INFO | "malloc" : " 14 GB", | 20260507 04:06:27.273 | INFO | "pinned" : " 29 GB" | 20260507 04:06:27.273 | INFO | } | 20260507 04:06:27.273 | INFO | Perf report for the last 5.13252 seconds: | 20260507 04:06:27.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:06:27.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:27.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:27.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:27.273 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 04:06:27.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:27.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:27.273 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:06:27.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:27.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:27.274 | INFO | Uploaded 36GiB to GPU in 1.96928s (18.2807GiB/s) | 20260507 04:06:27.274 | INFO | Downloaded 34GiB to GPU in 1.94316s (17.4973GiB/s) | 20260507 04:06:27.274 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:06:27.274 | INFO | Total mem usage: 18 GiB | 20260507 04:06:27.274 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:06:27.274 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 04:06:27.274 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:06:27.274 | INFO | High Water Breakdown: | 20260507 04:06:27.274 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:06:27.274 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:06:27.274 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:06:27.274 | INFO | ------------------------------------------ | 20260507 04:06:27.274 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:06:27.274 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:06:27.274 | INFO | Total mem usage: 26.5 GiB | 20260507 04:06:27.274 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:06:27.274 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:06:27.274 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:06:27.274 | INFO | High Water Breakdown: | 20260507 04:06:27.274 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:06:27.274 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:06:27.274 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:06:27.274 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:06:27.274 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:06:27.274 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:06:27.274 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:06:27.274 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:06:27.274 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:06:27.274 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:06:27.274 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:06:27.274 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:06:27.274 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:06:27.274 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:06:27.274 | INFO | ------------------------------------------ | 20260507 04:06:27.274 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:06:27.274 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:06:27.274 | INFO | Total mem usage: 25.6 GiB | 20260507 04:06:27.274 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:06:27.274 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:06:27.274 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:06:27.274 | INFO | High Water Breakdown: | 20260507 04:06:27.274 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:06:27.274 | INFO | ------------------------------------------ | 20260507 04:06:27.274 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:06:27.274 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:06:27.274 | INFO | Total mem usage: 15.7 GiB | 20260507 04:06:27.274 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:06:27.274 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:06:27.274 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:06:27.274 | INFO | High Water Breakdown: | 20260507 04:06:27.274 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:06:27.274 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:06:27.274 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:06:27.274 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:06:27.274 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:06:27.274 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:06:27.274 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:06:27.274 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:06:27.275 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:06:27.275 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:06:27.275 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:06:27.275 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:06:27.275 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:06:27.275 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:06:27.275 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:06:27.275 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:06:27.275 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:06:27.275 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:06:27.275 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:06:27.275 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:06:27.275 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:06:27.275 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:06:27.275 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:06:27.275 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:06:27.275 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:06:27.275 | INFO | ------------------------------------------ | 20260507 04:06:27.275 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:06:27.275 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:06:27.275 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:06:27.275 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:06:27.275 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.52e+01, 1.34e+01 / 1.56e+00 | 20260507 04:06:27.275 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.02e-01, 9.72e-03 / 5.94e-03 | 20260507 04:06:27.275 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.50e+00, 5.18e-01 / 1.92e-01 | 20260507 04:06:27.275 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.47e+00, 1.68e-01 / 9.63e-02 | 20260507 04:06:27.275 | INFO | PerfReport: 16.18%, DME -- 3.98e+01 / 4.48e+01, 4.06e+00 / 2.06e+00 | 20260507 04:06:27.275 | INFO | PerfReport: 57.59%, FrameLabeling -- 1.77e+01 / 1.99e+01, 1.26e+00 / 1.37e+00 | 20260507 04:06:27.275 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.14e+00 / 2.71e+00, 3.80e-01 / 1.86e-01 | 20260507 04:06:27.275 | INFO | PerfReport: 3.03%, Metrics -- 9.31e-01 / 1.27e+00, 2.44e-01 / 1.45e-01 | 20260507 04:06:27.275 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.15e-01 / 3.71e-01, 4.90e-02 / 1.95e-02 | 20260507 04:06:27.275 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:06:27.275 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:06:27.275 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:06:27.275 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 04:06:27.275 | INFO | PerfReport: 21.75%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260507 04:06:27.275 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 04:06:27.275 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:06:27.275 | INFO | PerfReport: 1.48%, LockUpload -- 1.43e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 04:06:27.275 | INFO | PerfReport: 1.36%, LockDownload -- 1.31e+00 / 0.00e+00, 3.17e+00 / 0.00e+00 | 20260507 04:06:27.275 | INFO | PerfReport: 22.11%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 04:06:27.275 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 04:06:27.370 | INFO | Starting a round of Baz IO | 20260507 04:06:30.472 | INFO | Populating RTMetrics report | 20260507 04:06:30.472 | INFO | RTMetrics report populated | 20260507 04:06:30.472 | INFO | Opening tmp RTMetrics json file | 20260507 04:06:30.490 | INFO | Serializing RTMetrics to json | 20260507 04:06:30.491 | INFO | Serialization done, swapping tmp file now... | 20260507 04:06:30.492 | INFO | RTMetrics done | 20260507 04:06:30.562 | INFO | Finished a round of Baz IO: Wrote 1.733 GiB in 3.191 seconds | 20260507 04:06:30.562 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.82% / 0.00% / 67.17% | 20260507 04:06:30.562 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.179609% / 3.97364e-06% / 8.34465e-05% | 20260507 04:06:30.562 | INFO | Baz IO overall: | 20260507 04:06:30.562 | INFO | BytesWritten : 3560880705818 | 20260507 04:06:30.562 | INFO | NumEvents : 1710025541690 | 20260507 04:06:32.409 | INFO | Analyzing chunk frames = [6963200,6963712) | 20260507 04:06:32.409 | INFO | MemInfo: { | 20260507 04:06:32.409 | INFO | "gpu" : " 17 GB", | 20260507 04:06:32.409 | INFO | "huge" : "", | 20260507 04:06:32.409 | INFO | "huge_shared" : " 27 GB", | 20260507 04:06:32.409 | INFO | "malloc" : " 14 GB", | 20260507 04:06:32.409 | INFO | "pinned" : " 29 GB" | 20260507 04:06:32.409 | INFO | } | 20260507 04:06:32.409 | INFO | Perf report for the last 5.13578 seconds: | 20260507 04:06:32.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:06:32.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:32.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:32.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:32.409 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 04:06:32.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:32.409 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 04:06:32.409 | INFO | PRE_HQ : 59.3%, 1.8, 55.3ms, 0 | 20260507 04:06:32.409 | INFO | BAZWRITER : 61.7%, 3.2, 102.7ms, 0 | 20260507 04:06:32.409 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 04:06:32.410 | INFO | Uploaded 36GiB to GPU in 2.02707s (17.7596GiB/s) | 20260507 04:06:32.410 | INFO | Downloaded 38GiB to GPU in 2.18227s (17.413GiB/s) | 20260507 04:06:37.549 | INFO | Analyzing chunk frames = [6963712,6964224) | 20260507 04:06:37.549 | INFO | MemInfo: { | 20260507 04:06:37.549 | INFO | "gpu" : " 17 GB", | 20260507 04:06:37.549 | INFO | "huge" : "", | 20260507 04:06:37.549 | INFO | "huge_shared" : " 27 GB", | 20260507 04:06:37.549 | INFO | "malloc" : " 14 GB", | 20260507 04:06:37.549 | INFO | "pinned" : " 29 GB" | 20260507 04:06:37.549 | INFO | } | 20260507 04:06:37.549 | INFO | Perf report for the last 5.1402 seconds: | 20260507 04:06:37.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:06:37.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:37.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:37.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:37.549 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:06:37.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:37.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:37.549 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:06:37.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:37.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:37.550 | INFO | Uploaded 36GiB to GPU in 2.04068s (17.6412GiB/s) | 20260507 04:06:37.550 | INFO | Downloaded 34GiB to GPU in 1.94217s (17.5062GiB/s) | 20260507 04:06:42.691 | INFO | Analyzing chunk frames = [6964224,6964736) | 20260507 04:06:42.691 | INFO | MemInfo: { | 20260507 04:06:42.691 | INFO | "gpu" : " 17 GB", | 20260507 04:06:42.691 | INFO | "huge" : "", | 20260507 04:06:42.691 | INFO | "huge_shared" : " 27 GB", | 20260507 04:06:42.691 | INFO | "malloc" : " 14 GB", | 20260507 04:06:42.691 | INFO | "pinned" : " 29 GB" | 20260507 04:06:42.691 | INFO | } | 20260507 04:06:42.691 | INFO | Perf report for the last 5.14167 seconds: | 20260507 04:06:42.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:06:42.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:42.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:42.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:42.691 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 04:06:42.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:42.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:42.691 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:06:42.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:42.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:42.691 | INFO | Uploaded 36GiB to GPU in 2.00261s (17.9766GiB/s) | 20260507 04:06:42.691 | INFO | Downloaded 34GiB to GPU in 1.94141s (17.5131GiB/s) | 20260507 04:06:47.837 | INFO | Analyzing chunk frames = [6964736,6965248) | 20260507 04:06:47.837 | INFO | MemInfo: { | 20260507 04:06:47.837 | INFO | "gpu" : " 17 GB", | 20260507 04:06:47.837 | INFO | "huge" : "", | 20260507 04:06:47.837 | INFO | "huge_shared" : " 27 GB", | 20260507 04:06:47.837 | INFO | "malloc" : " 14 GB", | 20260507 04:06:47.837 | INFO | "pinned" : " 29 GB" | 20260507 04:06:47.837 | INFO | } | 20260507 04:06:47.837 | INFO | Perf report for the last 5.14576 seconds: | 20260507 04:06:47.837 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:06:47.837 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:47.837 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:47.837 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:47.837 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 04:06:47.837 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:47.837 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:47.837 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:06:47.837 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:47.837 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:47.837 | INFO | Uploaded 36GiB to GPU in 1.97567s (18.2217GiB/s) | 20260507 04:06:47.837 | INFO | Downloaded 34GiB to GPU in 1.91117s (17.7902GiB/s) | 20260507 04:06:52.980 | INFO | Analyzing chunk frames = [6965248,6965760) | 20260507 04:06:52.980 | INFO | MemInfo: { | 20260507 04:06:52.980 | INFO | "gpu" : " 17 GB", | 20260507 04:06:52.980 | INFO | "huge" : "", | 20260507 04:06:52.980 | INFO | "huge_shared" : " 27 GB", | 20260507 04:06:52.980 | INFO | "malloc" : " 14 GB", | 20260507 04:06:52.980 | INFO | "pinned" : " 29 GB" | 20260507 04:06:52.980 | INFO | } | 20260507 04:06:52.980 | INFO | Perf report for the last 5.14322 seconds: | 20260507 04:06:52.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:06:52.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:52.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:52.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:52.980 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 04:06:52.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:52.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:52.980 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 04:06:52.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:52.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:52.980 | INFO | Uploaded 36GiB to GPU in 2.0497s (17.5635GiB/s) | 20260507 04:06:52.980 | INFO | Downloaded 34GiB to GPU in 1.95157s (17.4219GiB/s) | 20260507 04:06:58.110 | INFO | Analyzing chunk frames = [6965760,6966272) | 20260507 04:06:58.110 | INFO | MemInfo: { | 20260507 04:06:58.110 | INFO | "gpu" : " 17 GB", | 20260507 04:06:58.110 | INFO | "huge" : "", | 20260507 04:06:58.110 | INFO | "huge_shared" : " 27 GB", | 20260507 04:06:58.110 | INFO | "malloc" : " 14 GB", | 20260507 04:06:58.110 | INFO | "pinned" : " 29 GB" | 20260507 04:06:58.110 | INFO | } | 20260507 04:06:58.110 | INFO | Perf report for the last 5.12978 seconds: | 20260507 04:06:58.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:06:58.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:58.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:58.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:58.110 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:06:58.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:58.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:58.110 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 04:06:58.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:58.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:06:58.110 | INFO | Uploaded 36GiB to GPU in 2.05964s (17.4788GiB/s) | 20260507 04:06:58.110 | INFO | Downloaded 34GiB to GPU in 1.93021s (17.6146GiB/s) | 20260507 04:07:03.247 | INFO | Analyzing chunk frames = [6966272,6966784) | 20260507 04:07:03.247 | INFO | MemInfo: { | 20260507 04:07:03.247 | INFO | "gpu" : " 17 GB", | 20260507 04:07:03.247 | INFO | "huge" : "", | 20260507 04:07:03.247 | INFO | "huge_shared" : " 27 GB", | 20260507 04:07:03.247 | INFO | "malloc" : " 14 GB", | 20260507 04:07:03.247 | INFO | "pinned" : " 29 GB" | 20260507 04:07:03.247 | INFO | } | 20260507 04:07:03.247 | INFO | Perf report for the last 5.1373 seconds: | 20260507 04:07:03.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:07:03.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:03.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:03.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:03.247 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 04:07:03.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:03.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:03.247 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 04:07:03.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:03.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:03.247 | INFO | Uploaded 36GiB to GPU in 1.94158s (18.5416GiB/s) | 20260507 04:07:03.247 | INFO | Downloaded 34GiB to GPU in 1.95523s (17.3892GiB/s) | 20260507 04:07:08.389 | INFO | Analyzing chunk frames = [6966784,6967296) | 20260507 04:07:08.389 | INFO | MemInfo: { | 20260507 04:07:08.389 | INFO | "gpu" : " 17 GB", | 20260507 04:07:08.389 | INFO | "huge" : "", | 20260507 04:07:08.390 | INFO | "huge_shared" : " 27 GB", | 20260507 04:07:08.390 | INFO | "malloc" : " 14 GB", | 20260507 04:07:08.390 | INFO | "pinned" : " 29 GB" | 20260507 04:07:08.390 | INFO | } | 20260507 04:07:08.390 | INFO | Perf report for the last 5.14248 seconds: | 20260507 04:07:08.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:07:08.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:08.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:08.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:08.390 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:07:08.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:08.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:08.390 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:07:08.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:08.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:08.390 | INFO | Uploaded 36GiB to GPU in 1.96674s (18.3044GiB/s) | 20260507 04:07:08.390 | INFO | Downloaded 34GiB to GPU in 1.9538s (17.402GiB/s) | 20260507 04:07:08.487 | INFO | Starting a round of Baz IO | 20260507 04:07:11.591 | INFO | Populating RTMetrics report | 20260507 04:07:11.591 | INFO | RTMetrics report populated | 20260507 04:07:11.591 | INFO | Opening tmp RTMetrics json file | 20260507 04:07:11.600 | INFO | Serializing RTMetrics to json | 20260507 04:07:11.607 | INFO | Serialization done, swapping tmp file now... | 20260507 04:07:11.607 | INFO | RTMetrics done | 20260507 04:07:11.677 | INFO | Finished a round of Baz IO: Wrote 1.734 GiB in 3.19 seconds | 20260507 04:07:11.677 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.82% / 0.00% / 67.18% | 20260507 04:07:11.677 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.179585% / 0% / 8.74201e-05% | 20260507 04:07:11.677 | INFO | Baz IO overall: | 20260507 04:07:11.677 | INFO | BytesWritten : 3562742965920 | 20260507 04:07:11.677 | INFO | NumEvents : 1710902926796 | 20260507 04:07:13.530 | INFO | Analyzing chunk frames = [6967296,6967808) | 20260507 04:07:13.530 | INFO | MemInfo: { | 20260507 04:07:13.530 | INFO | "gpu" : " 17 GB", | 20260507 04:07:13.530 | INFO | "huge" : "", | 20260507 04:07:13.530 | INFO | "huge_shared" : " 27 GB", | 20260507 04:07:13.530 | INFO | "malloc" : " 14 GB", | 20260507 04:07:13.530 | INFO | "pinned" : " 29 GB" | 20260507 04:07:13.530 | INFO | } | 20260507 04:07:13.530 | INFO | Perf report for the last 5.14018 seconds: | 20260507 04:07:13.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:07:13.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:13.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:13.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:13.530 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:07:13.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:13.530 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 04:07:13.530 | INFO | PRE_HQ : 59.1%, 1.8, 55.1ms, 0 | 20260507 04:07:13.530 | INFO | BAZWRITER : 61.7%, 3.2, 102.2ms, 0 | 20260507 04:07:13.530 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 04:07:13.530 | INFO | Uploaded 36GiB to GPU in 2.01929s (17.8281GiB/s) | 20260507 04:07:13.530 | INFO | Downloaded 38GiB to GPU in 2.1808s (17.4248GiB/s) | 20260507 04:07:18.669 | INFO | Analyzing chunk frames = [6967808,6968320) | 20260507 04:07:18.669 | INFO | MemInfo: { | 20260507 04:07:18.669 | INFO | "gpu" : " 17 GB", | 20260507 04:07:18.669 | INFO | "huge" : "", | 20260507 04:07:18.669 | INFO | "huge_shared" : " 27 GB", | 20260507 04:07:18.669 | INFO | "malloc" : " 14 GB", | 20260507 04:07:18.669 | INFO | "pinned" : " 29 GB" | 20260507 04:07:18.669 | INFO | } | 20260507 04:07:18.669 | INFO | Perf report for the last 5.13909 seconds: | 20260507 04:07:18.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:07:18.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:18.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:18.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:18.669 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:07:18.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:18.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:18.669 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:07:18.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:18.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:18.669 | INFO | Uploaded 36GiB to GPU in 1.99385s (18.0555GiB/s) | 20260507 04:07:18.669 | INFO | Downloaded 34GiB to GPU in 1.92665s (17.6472GiB/s) | 20260507 04:07:23.808 | INFO | Analyzing chunk frames = [6968320,6968832) | 20260507 04:07:23.808 | INFO | MemInfo: { | 20260507 04:07:23.808 | INFO | "gpu" : " 17 GB", | 20260507 04:07:23.808 | INFO | "huge" : "", | 20260507 04:07:23.808 | INFO | "huge_shared" : " 27 GB", | 20260507 04:07:23.808 | INFO | "malloc" : " 14 GB", | 20260507 04:07:23.808 | INFO | "pinned" : " 29 GB" | 20260507 04:07:23.808 | INFO | } | 20260507 04:07:23.809 | INFO | Perf report for the last 5.13957 seconds: | 20260507 04:07:23.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:07:23.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:23.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:23.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:23.809 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 04:07:23.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:23.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:23.809 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:07:23.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:23.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:23.809 | INFO | Uploaded 36GiB to GPU in 2.00432s (17.9612GiB/s) | 20260507 04:07:23.809 | INFO | Downloaded 34GiB to GPU in 1.94762s (17.4572GiB/s) | 20260507 04:07:28.955 | INFO | Analyzing chunk frames = [6968832,6969344) | 20260507 04:07:28.955 | INFO | MemInfo: { | 20260507 04:07:28.955 | INFO | "gpu" : " 17 GB", | 20260507 04:07:28.955 | INFO | "huge" : "", | 20260507 04:07:28.955 | INFO | "huge_shared" : " 27 GB", | 20260507 04:07:28.955 | INFO | "malloc" : " 14 GB", | 20260507 04:07:28.955 | INFO | "pinned" : " 29 GB" | 20260507 04:07:28.955 | INFO | } | 20260507 04:07:28.955 | INFO | Perf report for the last 5.1467 seconds: | 20260507 04:07:28.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:07:28.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:28.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:28.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:28.955 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 04:07:28.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:28.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:28.955 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:07:28.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:28.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:28.956 | INFO | Uploaded 36GiB to GPU in 1.99187s (18.0735GiB/s) | 20260507 04:07:28.956 | INFO | Downloaded 34GiB to GPU in 1.93712s (17.5518GiB/s) | 20260507 04:07:34.092 | INFO | Analyzing chunk frames = [6969344,6969856) | 20260507 04:07:34.092 | INFO | MemInfo: { | 20260507 04:07:34.092 | INFO | "gpu" : " 17 GB", | 20260507 04:07:34.092 | INFO | "huge" : "", | 20260507 04:07:34.092 | INFO | "huge_shared" : " 27 GB", | 20260507 04:07:34.092 | INFO | "malloc" : " 14 GB", | 20260507 04:07:34.092 | INFO | "pinned" : " 29 GB" | 20260507 04:07:34.092 | INFO | } | 20260507 04:07:34.092 | INFO | Perf report for the last 5.1372 seconds: | 20260507 04:07:34.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:07:34.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:34.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:34.093 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:34.093 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 04:07:34.093 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:34.093 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:34.093 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:07:34.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:34.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:34.093 | INFO | Uploaded 36GiB to GPU in 2.04859s (17.573GiB/s) | 20260507 04:07:34.093 | INFO | Downloaded 34GiB to GPU in 1.95071s (17.4296GiB/s) | 20260507 04:07:39.227 | INFO | Analyzing chunk frames = [6969856,6970368) | 20260507 04:07:39.227 | INFO | MemInfo: { | 20260507 04:07:39.227 | INFO | "gpu" : " 17 GB", | 20260507 04:07:39.227 | INFO | "huge" : "", | 20260507 04:07:39.227 | INFO | "huge_shared" : " 27 GB", | 20260507 04:07:39.227 | INFO | "malloc" : " 14 GB", | 20260507 04:07:39.227 | INFO | "pinned" : " 29 GB" | 20260507 04:07:39.227 | INFO | } | 20260507 04:07:39.227 | INFO | Perf report for the last 5.13468 seconds: | 20260507 04:07:39.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:07:39.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:39.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:39.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:39.227 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 04:07:39.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:39.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:39.227 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 04:07:39.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:39.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:39.227 | INFO | Uploaded 36GiB to GPU in 2.04044s (17.6433GiB/s) | 20260507 04:07:39.227 | INFO | Downloaded 34GiB to GPU in 1.91378s (17.7659GiB/s) | 20260507 04:07:44.361 | INFO | Analyzing chunk frames = [6970368,6970880) | 20260507 04:07:44.361 | INFO | MemInfo: { | 20260507 04:07:44.361 | INFO | "gpu" : " 17 GB", | 20260507 04:07:44.361 | INFO | "huge" : "", | 20260507 04:07:44.361 | INFO | "huge_shared" : " 27 GB", | 20260507 04:07:44.361 | INFO | "malloc" : " 14 GB", | 20260507 04:07:44.361 | INFO | "pinned" : " 29 GB" | 20260507 04:07:44.361 | INFO | } | 20260507 04:07:44.361 | INFO | Perf report for the last 5.13413 seconds: | 20260507 04:07:44.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:07:44.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:44.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:44.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:44.361 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:07:44.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:44.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:44.361 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:07:44.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:44.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:44.362 | INFO | Uploaded 36GiB to GPU in 1.97983s (18.1834GiB/s) | 20260507 04:07:44.362 | INFO | Downloaded 34GiB to GPU in 1.98404s (17.1367GiB/s) | 20260507 04:07:49.503 | INFO | Analyzing chunk frames = [6970880,6971392) | 20260507 04:07:49.503 | INFO | MemInfo: { | 20260507 04:07:49.503 | INFO | "gpu" : " 17 GB", | 20260507 04:07:49.503 | INFO | "huge" : "", | 20260507 04:07:49.503 | INFO | "huge_shared" : " 27 GB", | 20260507 04:07:49.503 | INFO | "malloc" : " 14 GB", | 20260507 04:07:49.503 | INFO | "pinned" : " 29 GB" | 20260507 04:07:49.503 | INFO | } | 20260507 04:07:49.503 | INFO | Perf report for the last 5.14186 seconds: | 20260507 04:07:49.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:07:49.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:49.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:49.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:49.503 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:07:49.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:49.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:49.503 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:07:49.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:49.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:49.503 | INFO | Uploaded 36GiB to GPU in 1.97646s (18.2144GiB/s) | 20260507 04:07:49.503 | INFO | Downloaded 34GiB to GPU in 1.97274s (17.2349GiB/s) | 20260507 04:07:49.504 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:07:49.504 | INFO | Total mem usage: 18 GiB | 20260507 04:07:49.504 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:07:49.504 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 04:07:49.504 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:07:49.504 | INFO | High Water Breakdown: | 20260507 04:07:49.504 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:07:49.504 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:07:49.504 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:07:49.504 | INFO | ------------------------------------------ | 20260507 04:07:49.504 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:07:49.504 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:07:49.504 | INFO | Total mem usage: 26.5 GiB | 20260507 04:07:49.504 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:07:49.504 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:07:49.504 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:07:49.504 | INFO | High Water Breakdown: | 20260507 04:07:49.504 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:07:49.504 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:07:49.504 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:07:49.504 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:07:49.504 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:07:49.504 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:07:49.504 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:07:49.504 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:07:49.504 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:07:49.504 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:07:49.504 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:07:49.504 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:07:49.504 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:07:49.504 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:07:49.504 | INFO | ------------------------------------------ | 20260507 04:07:49.504 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:07:49.504 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:07:49.504 | INFO | Total mem usage: 25.6 GiB | 20260507 04:07:49.504 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:07:49.504 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:07:49.504 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:07:49.504 | INFO | High Water Breakdown: | 20260507 04:07:49.504 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:07:49.504 | INFO | ------------------------------------------ | 20260507 04:07:49.504 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:07:49.504 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:07:49.504 | INFO | Total mem usage: 15.7 GiB | 20260507 04:07:49.504 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:07:49.504 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:07:49.504 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:07:49.504 | INFO | High Water Breakdown: | 20260507 04:07:49.504 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:07:49.504 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:07:49.504 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:07:49.504 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:07:49.504 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:07:49.504 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:07:49.504 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:07:49.504 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:07:49.504 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:07:49.504 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:07:49.504 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:07:49.504 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:07:49.504 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:07:49.504 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:07:49.504 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:07:49.504 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:07:49.504 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:07:49.504 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:07:49.504 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:07:49.504 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:07:49.504 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:07:49.504 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:07:49.504 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:07:49.504 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:07:49.504 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:07:49.504 | INFO | ------------------------------------------ | 20260507 04:07:49.504 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:07:49.505 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:07:49.505 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:07:49.505 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:07:49.505 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.33e+01 / 1.32e+00 | 20260507 04:07:49.505 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.00e-01, 9.35e-03 / 4.68e-03 | 20260507 04:07:49.505 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.43e+00, 5.09e-01 / 3.07e-01 | 20260507 04:07:49.505 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.45e+00, 1.67e-01 / 1.15e-01 | 20260507 04:07:49.505 | INFO | PerfReport: 16.09%, DME -- 3.96e+01 / 4.40e+01, 3.79e+00 / 1.48e+00 | 20260507 04:07:49.505 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.33e+00 / 1.52e+00 | 20260507 04:07:49.505 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.14e+00 / 2.67e+00, 3.83e-01 / 2.29e-01 | 20260507 04:07:49.505 | INFO | PerfReport: 3.04%, Metrics -- 9.33e-01 / 1.26e+00, 2.47e-01 / 1.09e-01 | 20260507 04:07:49.505 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.15e-01 / 3.70e-01, 4.98e-02 / 1.67e-02 | 20260507 04:07:49.505 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:07:49.505 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:07:49.505 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:07:49.505 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 04:07:49.505 | INFO | PerfReport: 21.74%, Upload -- 2.09e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260507 04:07:49.505 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 04:07:49.505 | INFO | PerfReport: 31.97%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 04:07:49.505 | INFO | PerfReport: 1.43%, LockUpload -- 1.37e+00 / 0.00e+00, 4.77e+00 / 0.00e+00 | 20260507 04:07:49.505 | INFO | PerfReport: 1.35%, LockDownload -- 1.30e+00 / 0.00e+00, 3.18e+00 / 0.00e+00 | 20260507 04:07:49.505 | INFO | PerfReport: 22.12%, LockCompute -- 2.13e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 04:07:49.505 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260507 04:07:49.599 | INFO | Starting a round of Baz IO | 20260507 04:07:52.703 | INFO | Populating RTMetrics report | 20260507 04:07:52.703 | INFO | RTMetrics report populated | 20260507 04:07:52.703 | INFO | Opening tmp RTMetrics json file | 20260507 04:07:52.720 | INFO | Serializing RTMetrics to json | 20260507 04:07:52.721 | INFO | Serialization done, swapping tmp file now... | 20260507 04:07:52.722 | INFO | RTMetrics done | 20260507 04:07:52.793 | INFO | Finished a round of Baz IO: Wrote 1.734 GiB in 3.193 seconds | 20260507 04:07:52.793 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.82% / 0.00% / 67.18% | 20260507 04:07:52.793 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.179366% / 0% / 6.75519e-05% | 20260507 04:07:52.793 | INFO | Baz IO overall: | 20260507 04:07:52.793 | INFO | BytesWritten : 3564604821661 | 20260507 04:07:52.793 | INFO | NumEvents : 1711780423025 | 20260507 04:07:54.646 | INFO | Analyzing chunk frames = [6971392,6971904) | 20260507 04:07:54.646 | INFO | MemInfo: { | 20260507 04:07:54.646 | INFO | "gpu" : " 17 GB", | 20260507 04:07:54.646 | INFO | "huge" : "", | 20260507 04:07:54.646 | INFO | "huge_shared" : " 27 GB", | 20260507 04:07:54.646 | INFO | "malloc" : " 14 GB", | 20260507 04:07:54.646 | INFO | "pinned" : " 29 GB" | 20260507 04:07:54.646 | INFO | } | 20260507 04:07:54.646 | INFO | Perf report for the last 5.14291 seconds: | 20260507 04:07:54.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:07:54.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:54.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:54.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:54.646 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:07:54.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:54.646 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:07:54.646 | INFO | PRE_HQ : 59.0%, 1.8, 55.1ms, 0 | 20260507 04:07:54.646 | INFO | BAZWRITER : 61.7%, 3.2, 102.3ms, 0 | 20260507 04:07:54.646 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:07:54.646 | INFO | Uploaded 36GiB to GPU in 2.02919s (17.741GiB/s) | 20260507 04:07:54.646 | INFO | Downloaded 38GiB to GPU in 2.17703s (17.455GiB/s) | 20260507 04:07:59.785 | INFO | Analyzing chunk frames = [6971904,6972416) | 20260507 04:07:59.785 | INFO | MemInfo: { | 20260507 04:07:59.785 | INFO | "gpu" : " 17 GB", | 20260507 04:07:59.785 | INFO | "huge" : "", | 20260507 04:07:59.785 | INFO | "huge_shared" : " 27 GB", | 20260507 04:07:59.785 | INFO | "malloc" : " 14 GB", | 20260507 04:07:59.785 | INFO | "pinned" : " 29 GB" | 20260507 04:07:59.785 | INFO | } | 20260507 04:07:59.785 | INFO | Perf report for the last 5.13879 seconds: | 20260507 04:07:59.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:07:59.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:59.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:59.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:59.785 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 04:07:59.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:59.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:59.785 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:07:59.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:59.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:07:59.785 | INFO | Uploaded 36GiB to GPU in 2.05103s (17.5522GiB/s) | 20260507 04:07:59.785 | INFO | Downloaded 34GiB to GPU in 1.93943s (17.5309GiB/s) | 20260507 04:08:04.925 | INFO | Analyzing chunk frames = [6972416,6972928) | 20260507 04:08:04.925 | INFO | MemInfo: { | 20260507 04:08:04.925 | INFO | "gpu" : " 17 GB", | 20260507 04:08:04.925 | INFO | "huge" : "", | 20260507 04:08:04.925 | INFO | "huge_shared" : " 27 GB", | 20260507 04:08:04.925 | INFO | "malloc" : " 14 GB", | 20260507 04:08:04.925 | INFO | "pinned" : " 29 GB" | 20260507 04:08:04.925 | INFO | } | 20260507 04:08:04.926 | INFO | Perf report for the last 5.14068 seconds: | 20260507 04:08:04.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:08:04.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:04.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:04.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:04.926 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 04:08:04.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:04.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:04.926 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:08:04.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:04.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:04.926 | INFO | Uploaded 36GiB to GPU in 2.05138s (17.5491GiB/s) | 20260507 04:08:04.926 | INFO | Downloaded 34GiB to GPU in 1.94989s (17.4369GiB/s) | 20260507 04:08:10.057 | INFO | Analyzing chunk frames = [6972928,6973440) | 20260507 04:08:10.057 | INFO | MemInfo: { | 20260507 04:08:10.057 | INFO | "gpu" : " 17 GB", | 20260507 04:08:10.057 | INFO | "huge" : "", | 20260507 04:08:10.057 | INFO | "huge_shared" : " 27 GB", | 20260507 04:08:10.057 | INFO | "malloc" : " 14 GB", | 20260507 04:08:10.057 | INFO | "pinned" : " 29 GB" | 20260507 04:08:10.057 | INFO | } | 20260507 04:08:10.057 | INFO | Perf report for the last 5.13126 seconds: | 20260507 04:08:10.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:08:10.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:10.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:10.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:10.057 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260507 04:08:10.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:10.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:10.057 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:08:10.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:10.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:10.057 | INFO | Uploaded 36GiB to GPU in 2.02277s (17.7974GiB/s) | 20260507 04:08:10.057 | INFO | Downloaded 34GiB to GPU in 1.93946s (17.5307GiB/s) | 20260507 04:08:15.200 | INFO | Analyzing chunk frames = [6973440,6973952) | 20260507 04:08:15.200 | INFO | MemInfo: { | 20260507 04:08:15.200 | INFO | "gpu" : " 17 GB", | 20260507 04:08:15.200 | INFO | "huge" : "", | 20260507 04:08:15.200 | INFO | "huge_shared" : " 27 GB", | 20260507 04:08:15.200 | INFO | "malloc" : " 14 GB", | 20260507 04:08:15.200 | INFO | "pinned" : " 29 GB" | 20260507 04:08:15.200 | INFO | } | 20260507 04:08:15.200 | INFO | Perf report for the last 5.14322 seconds: | 20260507 04:08:15.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:08:15.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:15.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:15.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:15.200 | INFO | ANALYSIS : 59.8%, 3.0, 93.2ms, 0 | 20260507 04:08:15.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:15.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:15.200 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:08:15.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:15.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:15.200 | INFO | Uploaded 36GiB to GPU in 2.0148s (17.8678GiB/s) | 20260507 04:08:15.200 | INFO | Downloaded 34GiB to GPU in 1.94974s (17.4382GiB/s) | 20260507 04:08:20.340 | INFO | Analyzing chunk frames = [6973952,6974464) | 20260507 04:08:20.340 | INFO | MemInfo: { | 20260507 04:08:20.340 | INFO | "gpu" : " 17 GB", | 20260507 04:08:20.340 | INFO | "huge" : "", | 20260507 04:08:20.340 | INFO | "huge_shared" : " 27 GB", | 20260507 04:08:20.340 | INFO | "malloc" : " 14 GB", | 20260507 04:08:20.340 | INFO | "pinned" : " 29 GB" | 20260507 04:08:20.340 | INFO | } | 20260507 04:08:20.340 | INFO | Perf report for the last 5.13973 seconds: | 20260507 04:08:20.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:08:20.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:20.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:20.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:20.340 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 04:08:20.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:20.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:20.340 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 04:08:20.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:20.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:20.340 | INFO | Uploaded 36GiB to GPU in 2.05891s (17.485GiB/s) | 20260507 04:08:20.340 | INFO | Downloaded 34GiB to GPU in 1.95419s (17.3985GiB/s) | 20260507 04:08:25.485 | INFO | Analyzing chunk frames = [6974464,6974976) | 20260507 04:08:25.485 | INFO | MemInfo: { | 20260507 04:08:25.485 | INFO | "gpu" : " 17 GB", | 20260507 04:08:25.485 | INFO | "huge" : "", | 20260507 04:08:25.485 | INFO | "huge_shared" : " 27 GB", | 20260507 04:08:25.485 | INFO | "malloc" : " 14 GB", | 20260507 04:08:25.485 | INFO | "pinned" : " 29 GB" | 20260507 04:08:25.485 | INFO | } | 20260507 04:08:25.485 | INFO | Perf report for the last 5.14523 seconds: | 20260507 04:08:25.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:08:25.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:25.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:25.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:25.485 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 04:08:25.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:25.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:25.485 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 04:08:25.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:25.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:25.485 | INFO | Uploaded 36GiB to GPU in 1.98836s (18.1054GiB/s) | 20260507 04:08:25.485 | INFO | Downloaded 34GiB to GPU in 1.95581s (17.3841GiB/s) | 20260507 04:08:30.617 | INFO | Analyzing chunk frames = [6974976,6975488) | 20260507 04:08:30.617 | INFO | MemInfo: { | 20260507 04:08:30.617 | INFO | "gpu" : " 17 GB", | 20260507 04:08:30.617 | INFO | "huge" : "", | 20260507 04:08:30.617 | INFO | "huge_shared" : " 27 GB", | 20260507 04:08:30.617 | INFO | "malloc" : " 14 GB", | 20260507 04:08:30.617 | INFO | "pinned" : " 29 GB" | 20260507 04:08:30.617 | INFO | } | 20260507 04:08:30.618 | INFO | Perf report for the last 5.13247 seconds: | 20260507 04:08:30.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:08:30.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:30.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:30.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:30.618 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260507 04:08:30.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:30.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:30.618 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 04:08:30.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:30.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:30.618 | INFO | Uploaded 36GiB to GPU in 1.98815s (18.1073GiB/s) | 20260507 04:08:30.618 | INFO | Downloaded 34GiB to GPU in 1.96755s (17.2803GiB/s) | 20260507 04:08:30.715 | INFO | Starting a round of Baz IO | 20260507 04:08:33.816 | INFO | Populating RTMetrics report | 20260507 04:08:33.816 | INFO | RTMetrics report populated | 20260507 04:08:33.816 | INFO | Opening tmp RTMetrics json file | 20260507 04:08:33.821 | INFO | Serializing RTMetrics to json | 20260507 04:08:33.834 | INFO | Serialization done, swapping tmp file now... | 20260507 04:08:33.834 | INFO | RTMetrics done | 20260507 04:08:33.906 | INFO | Finished a round of Baz IO: Wrote 1.732 GiB in 3.191 seconds | 20260507 04:08:33.906 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.82% / 0.00% / 67.18% | 20260507 04:08:33.907 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.178214% / 3.97364e-06% / 5.5631e-05% | 20260507 04:08:33.907 | INFO | Baz IO overall: | 20260507 04:08:33.907 | INFO | BytesWritten : 3566464755128 | 20260507 04:08:33.907 | INFO | NumEvents : 1712656487026 | 20260507 04:08:35.758 | INFO | Analyzing chunk frames = [6975488,6976000) | 20260507 04:08:35.758 | INFO | MemInfo: { | 20260507 04:08:35.758 | INFO | "gpu" : " 17 GB", | 20260507 04:08:35.758 | INFO | "huge" : "", | 20260507 04:08:35.758 | INFO | "huge_shared" : " 27 GB", | 20260507 04:08:35.759 | INFO | "malloc" : " 14 GB", | 20260507 04:08:35.759 | INFO | "pinned" : " 29 GB" | 20260507 04:08:35.759 | INFO | } | 20260507 04:08:35.759 | INFO | Perf report for the last 5.1411 seconds: | 20260507 04:08:35.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:08:35.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:35.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:35.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:35.759 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:08:35.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:35.759 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260507 04:08:35.759 | INFO | PRE_HQ : 60.0%, 1.9, 58.3ms, 0 | 20260507 04:08:35.759 | INFO | BAZWRITER : 61.7%, 3.3, 105.7ms, 0 | 20260507 04:08:35.759 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:08:35.759 | INFO | Uploaded 36GiB to GPU in 2.02751s (17.7558GiB/s) | 20260507 04:08:35.759 | INFO | Downloaded 38GiB to GPU in 2.18421s (17.3976GiB/s) | 20260507 04:08:40.897 | INFO | Analyzing chunk frames = [6976000,6976512) | 20260507 04:08:40.898 | INFO | MemInfo: { | 20260507 04:08:40.898 | INFO | "gpu" : " 17 GB", | 20260507 04:08:40.898 | INFO | "huge" : "", | 20260507 04:08:40.898 | INFO | "huge_shared" : " 27 GB", | 20260507 04:08:40.898 | INFO | "malloc" : " 14 GB", | 20260507 04:08:40.898 | INFO | "pinned" : " 29 GB" | 20260507 04:08:40.898 | INFO | } | 20260507 04:08:40.898 | INFO | Perf report for the last 5.13904 seconds: | 20260507 04:08:40.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:08:40.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:40.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:40.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:40.898 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:08:40.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:40.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:40.898 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:08:40.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:40.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:40.898 | INFO | Uploaded 36GiB to GPU in 2.03706s (17.6726GiB/s) | 20260507 04:08:40.898 | INFO | Downloaded 34GiB to GPU in 1.9437s (17.4924GiB/s) | 20260507 04:08:46.039 | INFO | Analyzing chunk frames = [6976512,6977024) | 20260507 04:08:46.039 | INFO | MemInfo: { | 20260507 04:08:46.039 | INFO | "gpu" : " 17 GB", | 20260507 04:08:46.039 | INFO | "huge" : "", | 20260507 04:08:46.039 | INFO | "huge_shared" : " 27 GB", | 20260507 04:08:46.039 | INFO | "malloc" : " 14 GB", | 20260507 04:08:46.039 | INFO | "pinned" : " 29 GB" | 20260507 04:08:46.039 | INFO | } | 20260507 04:08:46.039 | INFO | Perf report for the last 5.14136 seconds: | 20260507 04:08:46.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:08:46.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:46.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:46.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:46.039 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 04:08:46.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:46.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:46.039 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:08:46.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:46.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:46.039 | INFO | Uploaded 36GiB to GPU in 1.98546s (18.1318GiB/s) | 20260507 04:08:46.039 | INFO | Downloaded 34GiB to GPU in 1.94331s (17.4959GiB/s) | 20260507 04:08:51.182 | INFO | Analyzing chunk frames = [6977024,6977536) | 20260507 04:08:51.182 | INFO | MemInfo: { | 20260507 04:08:51.182 | INFO | "gpu" : " 17 GB", | 20260507 04:08:51.182 | INFO | "huge" : "", | 20260507 04:08:51.182 | INFO | "huge_shared" : " 27 GB", | 20260507 04:08:51.182 | INFO | "malloc" : " 14 GB", | 20260507 04:08:51.182 | INFO | "pinned" : " 29 GB" | 20260507 04:08:51.182 | INFO | } | 20260507 04:08:51.182 | INFO | Perf report for the last 5.14288 seconds: | 20260507 04:08:51.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:08:51.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:51.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:51.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:51.182 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:08:51.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:51.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:51.182 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:08:51.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:51.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:51.182 | INFO | Uploaded 36GiB to GPU in 1.99557s (18.04GiB/s) | 20260507 04:08:51.182 | INFO | Downloaded 34GiB to GPU in 1.93729s (17.5503GiB/s) | 20260507 04:08:56.309 | INFO | Analyzing chunk frames = [6977536,6978048) | 20260507 04:08:56.309 | INFO | MemInfo: { | 20260507 04:08:56.309 | INFO | "gpu" : " 17 GB", | 20260507 04:08:56.309 | INFO | "huge" : "", | 20260507 04:08:56.309 | INFO | "huge_shared" : " 27 GB", | 20260507 04:08:56.309 | INFO | "malloc" : " 14 GB", | 20260507 04:08:56.309 | INFO | "pinned" : " 29 GB" | 20260507 04:08:56.309 | INFO | } | 20260507 04:08:56.309 | INFO | Perf report for the last 5.12682 seconds: | 20260507 04:08:56.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:08:56.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:56.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:56.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:56.309 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260507 04:08:56.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:56.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:56.309 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 04:08:56.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:56.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:08:56.309 | INFO | Uploaded 36GiB to GPU in 2.05066s (17.5553GiB/s) | 20260507 04:08:56.309 | INFO | Downloaded 34GiB to GPU in 1.95015s (17.4346GiB/s) | 20260507 04:09:01.460 | INFO | Analyzing chunk frames = [6978048,6978560) | 20260507 04:09:01.460 | INFO | MemInfo: { | 20260507 04:09:01.460 | INFO | "gpu" : " 17 GB", | 20260507 04:09:01.460 | INFO | "huge" : "", | 20260507 04:09:01.460 | INFO | "huge_shared" : " 27 GB", | 20260507 04:09:01.460 | INFO | "malloc" : " 14 GB", | 20260507 04:09:01.460 | INFO | "pinned" : " 29 GB" | 20260507 04:09:01.460 | INFO | } | 20260507 04:09:01.460 | INFO | Perf report for the last 5.15136 seconds: | 20260507 04:09:01.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:09:01.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:01.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:01.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:01.460 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 04:09:01.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:01.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:01.460 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:09:01.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:01.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:01.460 | INFO | Uploaded 36GiB to GPU in 2.03638s (17.6784GiB/s) | 20260507 04:09:01.460 | INFO | Downloaded 34GiB to GPU in 1.94807s (17.4531GiB/s) | 20260507 04:09:06.602 | INFO | Analyzing chunk frames = [6978560,6979072) | 20260507 04:09:06.602 | INFO | MemInfo: { | 20260507 04:09:06.602 | INFO | "gpu" : " 17 GB", | 20260507 04:09:06.602 | INFO | "huge" : "", | 20260507 04:09:06.602 | INFO | "huge_shared" : " 27 GB", | 20260507 04:09:06.602 | INFO | "malloc" : " 14 GB", | 20260507 04:09:06.602 | INFO | "pinned" : " 29 GB" | 20260507 04:09:06.602 | INFO | } | 20260507 04:09:06.602 | INFO | Perf report for the last 5.1416 seconds: | 20260507 04:09:06.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:09:06.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:06.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:06.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:06.602 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:09:06.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:06.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:06.602 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:09:06.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:06.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:06.602 | INFO | Uploaded 36GiB to GPU in 1.95282s (18.4348GiB/s) | 20260507 04:09:06.602 | INFO | Downloaded 34GiB to GPU in 1.95505s (17.3909GiB/s) | 20260507 04:09:11.732 | INFO | Analyzing chunk frames = [6979072,6979584) | 20260507 04:09:11.732 | INFO | MemInfo: { | 20260507 04:09:11.732 | INFO | "gpu" : " 17 GB", | 20260507 04:09:11.732 | INFO | "huge" : "", | 20260507 04:09:11.732 | INFO | "huge_shared" : " 27 GB", | 20260507 04:09:11.732 | INFO | "malloc" : " 14 GB", | 20260507 04:09:11.732 | INFO | "pinned" : " 29 GB" | 20260507 04:09:11.732 | INFO | } | 20260507 04:09:11.732 | INFO | Perf report for the last 5.1306 seconds: | 20260507 04:09:11.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:09:11.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:11.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:11.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:11.732 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 04:09:11.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:11.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:11.732 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 04:09:11.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:11.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:11.733 | INFO | Uploaded 36GiB to GPU in 1.97646s (18.2144GiB/s) | 20260507 04:09:11.733 | INFO | Downloaded 34GiB to GPU in 1.96435s (17.3085GiB/s) | 20260507 04:09:11.733 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:09:11.733 | INFO | Total mem usage: 18 GiB | 20260507 04:09:11.733 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:09:11.733 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 04:09:11.733 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:09:11.733 | INFO | High Water Breakdown: | 20260507 04:09:11.733 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:09:11.733 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:09:11.733 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:09:11.733 | INFO | ------------------------------------------ | 20260507 04:09:11.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:09:11.733 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:09:11.733 | INFO | Total mem usage: 26.5 GiB | 20260507 04:09:11.733 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:09:11.733 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:09:11.733 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:09:11.733 | INFO | High Water Breakdown: | 20260507 04:09:11.733 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:09:11.733 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:09:11.733 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:09:11.733 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:09:11.733 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:09:11.733 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:09:11.733 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:09:11.733 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:09:11.733 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:09:11.733 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:09:11.733 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:09:11.733 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:09:11.733 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:09:11.733 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:09:11.733 | INFO | ------------------------------------------ | 20260507 04:09:11.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:09:11.733 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:09:11.733 | INFO | Total mem usage: 25.6 GiB | 20260507 04:09:11.733 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:09:11.733 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:09:11.733 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:09:11.733 | INFO | High Water Breakdown: | 20260507 04:09:11.733 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:09:11.733 | INFO | ------------------------------------------ | 20260507 04:09:11.733 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:09:11.733 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:09:11.733 | INFO | Total mem usage: 15.7 GiB | 20260507 04:09:11.733 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:09:11.733 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:09:11.733 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:09:11.733 | INFO | High Water Breakdown: | 20260507 04:09:11.733 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:09:11.733 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:09:11.733 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:09:11.733 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:09:11.733 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:09:11.733 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:09:11.733 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:09:11.733 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:09:11.733 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:09:11.733 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:09:11.733 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:09:11.733 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:09:11.734 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:09:11.734 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:09:11.734 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:09:11.734 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:09:11.734 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:09:11.734 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:09:11.734 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:09:11.734 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:09:11.734 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:09:11.734 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:09:11.734 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:09:11.734 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:09:11.734 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:09:11.734 | INFO | ------------------------------------------ | 20260507 04:09:11.734 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:09:11.734 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:09:11.734 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:09:11.734 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:09:11.734 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.34e+01 / 1.38e+00 | 20260507 04:09:11.734 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.00e-01, 9.73e-03 / 5.15e-03 | 20260507 04:09:11.734 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.37e+00, 5.02e-01 / 3.19e-01 | 20260507 04:09:11.734 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.43e+00, 1.64e-01 / 1.31e-01 | 20260507 04:09:11.734 | INFO | PerfReport: 16.14%, DME -- 3.96e+01 / 4.43e+01, 3.85e+00 / 1.86e+00 | 20260507 04:09:11.734 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.23e+00 / 1.47e+00 | 20260507 04:09:11.734 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.14e+00 / 2.69e+00, 3.84e-01 / 2.35e-01 | 20260507 04:09:11.734 | INFO | PerfReport: 3.04%, Metrics -- 9.32e-01 / 1.24e+00, 2.47e-01 / 9.39e-02 | 20260507 04:09:11.734 | INFO | PerfReport: 1.03%, PulseSerializing -- 3.15e-01 / 3.68e-01, 4.99e-02 / 2.13e-02 | 20260507 04:09:11.734 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:09:11.734 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:09:11.734 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:09:11.734 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 04:09:11.734 | INFO | PerfReport: 21.83%, Upload -- 2.10e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260507 04:09:11.734 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 04:09:11.734 | INFO | PerfReport: 31.89%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:09:11.734 | INFO | PerfReport: 1.52%, LockUpload -- 1.47e+00 / 0.00e+00, 4.89e+00 / 0.00e+00 | 20260507 04:09:11.734 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260507 04:09:11.734 | INFO | PerfReport: 22.04%, LockCompute -- 2.12e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 04:09:11.734 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260507 04:09:11.829 | INFO | Starting a round of Baz IO | 20260507 04:09:14.932 | INFO | Populating RTMetrics report | 20260507 04:09:14.932 | INFO | RTMetrics report populated | 20260507 04:09:14.932 | INFO | Opening tmp RTMetrics json file | 20260507 04:09:14.950 | INFO | Serializing RTMetrics to json | 20260507 04:09:14.952 | INFO | Serialization done, swapping tmp file now... | 20260507 04:09:14.952 | INFO | RTMetrics done | 20260507 04:09:15.022 | INFO | Finished a round of Baz IO: Wrote 1.733 GiB in 3.193 seconds | 20260507 04:09:15.023 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.82% / 0.00% / 67.18% | 20260507 04:09:15.023 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.17819% / 3.97364e-06% / 8.74201e-05% | 20260507 04:09:15.023 | INFO | Baz IO overall: | 20260507 04:09:15.023 | INFO | BytesWritten : 3568325273152 | 20260507 04:09:15.023 | INFO | NumEvents : 1713532819360 | 20260507 04:09:16.868 | INFO | Analyzing chunk frames = [6979584,6980096) | 20260507 04:09:16.868 | INFO | MemInfo: { | 20260507 04:09:16.868 | INFO | "gpu" : " 17 GB", | 20260507 04:09:16.868 | INFO | "huge" : "", | 20260507 04:09:16.868 | INFO | "huge_shared" : " 27 GB", | 20260507 04:09:16.868 | INFO | "malloc" : " 14 GB", | 20260507 04:09:16.868 | INFO | "pinned" : " 29 GB" | 20260507 04:09:16.868 | INFO | } | 20260507 04:09:16.868 | INFO | Perf report for the last 5.13583 seconds: | 20260507 04:09:16.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:09:16.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:16.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:16.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:16.868 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 04:09:16.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:16.868 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 04:09:16.868 | INFO | PRE_HQ : 59.1%, 1.8, 55.3ms, 0 | 20260507 04:09:16.868 | INFO | BAZWRITER : 61.8%, 3.2, 102.0ms, 0 | 20260507 04:09:16.868 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:09:16.869 | INFO | Uploaded 36GiB to GPU in 2.03454s (17.6944GiB/s) | 20260507 04:09:16.869 | INFO | Downloaded 38GiB to GPU in 2.18069s (17.4257GiB/s) | 20260507 04:09:22.012 | INFO | Analyzing chunk frames = [6980096,6980608) | 20260507 04:09:22.012 | INFO | MemInfo: { | 20260507 04:09:22.012 | INFO | "gpu" : " 17 GB", | 20260507 04:09:22.012 | INFO | "huge" : "", | 20260507 04:09:22.012 | INFO | "huge_shared" : " 27 GB", | 20260507 04:09:22.012 | INFO | "malloc" : " 14 GB", | 20260507 04:09:22.012 | INFO | "pinned" : " 29 GB" | 20260507 04:09:22.012 | INFO | } | 20260507 04:09:22.013 | INFO | Perf report for the last 5.14427 seconds: | 20260507 04:09:22.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:09:22.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:22.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:22.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:22.013 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:09:22.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:22.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:22.013 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:09:22.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:22.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:22.013 | INFO | Uploaded 36GiB to GPU in 2.02756s (17.7553GiB/s) | 20260507 04:09:22.013 | INFO | Downloaded 34GiB to GPU in 1.94345s (17.4946GiB/s) | 20260507 04:09:27.152 | INFO | Analyzing chunk frames = [6980608,6981120) | 20260507 04:09:27.152 | INFO | MemInfo: { | 20260507 04:09:27.152 | INFO | "gpu" : " 17 GB", | 20260507 04:09:27.152 | INFO | "huge" : "", | 20260507 04:09:27.152 | INFO | "huge_shared" : " 27 GB", | 20260507 04:09:27.152 | INFO | "malloc" : " 14 GB", | 20260507 04:09:27.152 | INFO | "pinned" : " 29 GB" | 20260507 04:09:27.152 | INFO | } | 20260507 04:09:27.152 | INFO | Perf report for the last 5.13995 seconds: | 20260507 04:09:27.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:09:27.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:27.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:27.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:27.153 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 04:09:27.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:27.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:27.153 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:09:27.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:27.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:27.153 | INFO | Uploaded 36GiB to GPU in 2.00766s (17.9313GiB/s) | 20260507 04:09:27.153 | INFO | Downloaded 34GiB to GPU in 1.941s (17.5168GiB/s) | 20260507 04:09:32.294 | INFO | Analyzing chunk frames = [6981120,6981632) | 20260507 04:09:32.294 | INFO | MemInfo: { | 20260507 04:09:32.294 | INFO | "gpu" : " 17 GB", | 20260507 04:09:32.294 | INFO | "huge" : "", | 20260507 04:09:32.294 | INFO | "huge_shared" : " 27 GB", | 20260507 04:09:32.295 | INFO | "malloc" : " 14 GB", | 20260507 04:09:32.295 | INFO | "pinned" : " 29 GB" | 20260507 04:09:32.295 | INFO | } | 20260507 04:09:32.295 | INFO | Perf report for the last 5.1421 seconds: | 20260507 04:09:32.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:09:32.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:32.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:32.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:32.295 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:09:32.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:32.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:32.295 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:09:32.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:32.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:32.295 | INFO | Uploaded 36GiB to GPU in 1.98838s (18.1052GiB/s) | 20260507 04:09:32.295 | INFO | Downloaded 34GiB to GPU in 1.92261s (17.6843GiB/s) | 20260507 04:09:37.430 | INFO | Analyzing chunk frames = [6981632,6982144) | 20260507 04:09:37.430 | INFO | MemInfo: { | 20260507 04:09:37.430 | INFO | "gpu" : " 17 GB", | 20260507 04:09:37.430 | INFO | "huge" : "", | 20260507 04:09:37.430 | INFO | "huge_shared" : " 27 GB", | 20260507 04:09:37.430 | INFO | "malloc" : " 14 GB", | 20260507 04:09:37.430 | INFO | "pinned" : " 29 GB" | 20260507 04:09:37.430 | INFO | } | 20260507 04:09:37.430 | INFO | Perf report for the last 5.13542 seconds: | 20260507 04:09:37.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:09:37.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:37.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:37.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:37.430 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 04:09:37.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:37.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:37.430 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:09:37.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:37.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:37.430 | INFO | Uploaded 36GiB to GPU in 2.05091s (17.5531GiB/s) | 20260507 04:09:37.430 | INFO | Downloaded 34GiB to GPU in 1.95158s (17.4218GiB/s) | 20260507 04:09:42.567 | INFO | Analyzing chunk frames = [6982144,6982656) | 20260507 04:09:42.567 | INFO | MemInfo: { | 20260507 04:09:42.567 | INFO | "gpu" : " 17 GB", | 20260507 04:09:42.567 | INFO | "huge" : "", | 20260507 04:09:42.567 | INFO | "huge_shared" : " 27 GB", | 20260507 04:09:42.567 | INFO | "malloc" : " 14 GB", | 20260507 04:09:42.567 | INFO | "pinned" : " 29 GB" | 20260507 04:09:42.567 | INFO | } | 20260507 04:09:42.568 | INFO | Perf report for the last 5.13749 seconds: | 20260507 04:09:42.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:09:42.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:42.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:42.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:42.568 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:09:42.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:42.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:42.568 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:09:42.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:42.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:42.568 | INFO | Uploaded 36GiB to GPU in 2.04853s (17.5736GiB/s) | 20260507 04:09:42.568 | INFO | Downloaded 34GiB to GPU in 1.97025s (17.2567GiB/s) | 20260507 04:09:47.710 | INFO | Analyzing chunk frames = [6982656,6983168) | 20260507 04:09:47.710 | INFO | MemInfo: { | 20260507 04:09:47.710 | INFO | "gpu" : " 17 GB", | 20260507 04:09:47.710 | INFO | "huge" : "", | 20260507 04:09:47.710 | INFO | "huge_shared" : " 27 GB", | 20260507 04:09:47.710 | INFO | "malloc" : " 14 GB", | 20260507 04:09:47.710 | INFO | "pinned" : " 29 GB" | 20260507 04:09:47.710 | INFO | } | 20260507 04:09:47.710 | INFO | Perf report for the last 5.14245 seconds: | 20260507 04:09:47.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:09:47.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:47.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:47.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:47.710 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:09:47.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:47.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:47.710 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 04:09:47.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:47.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:47.710 | INFO | Uploaded 36GiB to GPU in 1.97026s (18.2717GiB/s) | 20260507 04:09:47.710 | INFO | Downloaded 34GiB to GPU in 1.96905s (17.2672GiB/s) | 20260507 04:09:52.848 | INFO | Analyzing chunk frames = [6983168,6983680) | 20260507 04:09:52.848 | INFO | MemInfo: { | 20260507 04:09:52.848 | INFO | "gpu" : " 17 GB", | 20260507 04:09:52.848 | INFO | "huge" : "", | 20260507 04:09:52.848 | INFO | "huge_shared" : " 27 GB", | 20260507 04:09:52.848 | INFO | "malloc" : " 14 GB", | 20260507 04:09:52.848 | INFO | "pinned" : " 29 GB" | 20260507 04:09:52.848 | INFO | } | 20260507 04:09:52.848 | INFO | Perf report for the last 5.13796 seconds: | 20260507 04:09:52.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:09:52.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:52.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:52.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:52.848 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:09:52.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:52.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:52.848 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 04:09:52.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:52.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:52.848 | INFO | Uploaded 36GiB to GPU in 1.96159s (18.3524GiB/s) | 20260507 04:09:52.848 | INFO | Downloaded 34GiB to GPU in 1.96383s (17.3131GiB/s) | 20260507 04:09:52.945 | INFO | Starting a round of Baz IO | 20260507 04:09:56.047 | INFO | Populating RTMetrics report | 20260507 04:09:56.047 | INFO | RTMetrics report populated | 20260507 04:09:56.047 | INFO | Opening tmp RTMetrics json file | 20260507 04:09:56.048 | INFO | Serializing RTMetrics to json | 20260507 04:09:56.049 | INFO | Serialization done, swapping tmp file now... | 20260507 04:09:56.050 | INFO | RTMetrics done | 20260507 04:09:56.137 | INFO | Finished a round of Baz IO: Wrote 1.732 GiB in 3.192 seconds | 20260507 04:09:56.137 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.82% / 0.00% / 67.18% | 20260507 04:09:56.137 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.177276% / 0% / 6.75519e-05% | 20260507 04:09:56.138 | INFO | Baz IO overall: | 20260507 04:09:56.138 | INFO | BytesWritten : 3570184610899 | 20260507 04:09:56.138 | INFO | NumEvents : 1714408364181 | 20260507 04:09:57.992 | INFO | Analyzing chunk frames = [6983680,6984192) | 20260507 04:09:57.992 | INFO | MemInfo: { | 20260507 04:09:57.992 | INFO | "gpu" : " 17 GB", | 20260507 04:09:57.992 | INFO | "huge" : "", | 20260507 04:09:57.993 | INFO | "huge_shared" : " 27 GB", | 20260507 04:09:57.993 | INFO | "malloc" : " 14 GB", | 20260507 04:09:57.993 | INFO | "pinned" : " 29 GB" | 20260507 04:09:57.993 | INFO | } | 20260507 04:09:57.993 | INFO | Perf report for the last 5.14472 seconds: | 20260507 04:09:57.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:09:57.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:57.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:57.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:57.993 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 04:09:57.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:09:57.993 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 04:09:57.993 | INFO | PRE_HQ : 58.9%, 1.8, 55.2ms, 0 | 20260507 04:09:57.993 | INFO | BAZWRITER : 61.6%, 3.3, 103.3ms, 0 | 20260507 04:09:57.993 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:09:57.993 | INFO | Uploaded 36GiB to GPU in 2.02994s (17.7345GiB/s) | 20260507 04:09:57.993 | INFO | Downloaded 38GiB to GPU in 2.17818s (17.4458GiB/s) | 20260507 04:10:03.136 | INFO | Analyzing chunk frames = [6984192,6984704) | 20260507 04:10:03.136 | INFO | MemInfo: { | 20260507 04:10:03.136 | INFO | "gpu" : " 17 GB", | 20260507 04:10:03.136 | INFO | "huge" : "", | 20260507 04:10:03.136 | INFO | "huge_shared" : " 27 GB", | 20260507 04:10:03.136 | INFO | "malloc" : " 14 GB", | 20260507 04:10:03.136 | INFO | "pinned" : " 29 GB" | 20260507 04:10:03.136 | INFO | } | 20260507 04:10:03.136 | INFO | Perf report for the last 5.14343 seconds: | 20260507 04:10:03.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:10:03.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:03.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:03.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:03.136 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:10:03.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:03.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:03.136 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:10:03.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:03.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:03.137 | INFO | Uploaded 36GiB to GPU in 1.99567s (18.039GiB/s) | 20260507 04:10:03.137 | INFO | Downloaded 34GiB to GPU in 1.93932s (17.5319GiB/s) | 20260507 04:10:08.269 | INFO | Analyzing chunk frames = [6984704,6985216) | 20260507 04:10:08.269 | INFO | MemInfo: { | 20260507 04:10:08.269 | INFO | "gpu" : " 17 GB", | 20260507 04:10:08.269 | INFO | "huge" : "", | 20260507 04:10:08.269 | INFO | "huge_shared" : " 27 GB", | 20260507 04:10:08.269 | INFO | "malloc" : " 14 GB", | 20260507 04:10:08.269 | INFO | "pinned" : " 29 GB" | 20260507 04:10:08.269 | INFO | } | 20260507 04:10:08.269 | INFO | Perf report for the last 5.13304 seconds: | 20260507 04:10:08.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:10:08.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:08.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:08.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:08.269 | INFO | ANALYSIS : 60.7%, 3.0, 93.8ms, 0 | 20260507 04:10:08.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:08.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:08.269 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:10:08.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:08.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:08.269 | INFO | Uploaded 36GiB to GPU in 2.0424s (17.6263GiB/s) | 20260507 04:10:08.270 | INFO | Downloaded 34GiB to GPU in 1.95329s (17.4065GiB/s) | 20260507 04:10:13.411 | INFO | Analyzing chunk frames = [6985216,6985728) | 20260507 04:10:13.411 | INFO | MemInfo: { | 20260507 04:10:13.411 | INFO | "gpu" : " 17 GB", | 20260507 04:10:13.411 | INFO | "huge" : "", | 20260507 04:10:13.411 | INFO | "huge_shared" : " 27 GB", | 20260507 04:10:13.412 | INFO | "malloc" : " 14 GB", | 20260507 04:10:13.412 | INFO | "pinned" : " 29 GB" | 20260507 04:10:13.412 | INFO | } | 20260507 04:10:13.412 | INFO | Perf report for the last 5.14239 seconds: | 20260507 04:10:13.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:10:13.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:13.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:13.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:13.412 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:10:13.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:13.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:13.412 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:10:13.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:13.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:13.412 | INFO | Uploaded 36GiB to GPU in 1.97305s (18.2458GiB/s) | 20260507 04:10:13.412 | INFO | Downloaded 34GiB to GPU in 1.92055s (17.7033GiB/s) | 20260507 04:10:18.550 | INFO | Analyzing chunk frames = [6985728,6986240) | 20260507 04:10:18.550 | INFO | MemInfo: { | 20260507 04:10:18.550 | INFO | "gpu" : " 17 GB", | 20260507 04:10:18.550 | INFO | "huge" : "", | 20260507 04:10:18.550 | INFO | "huge_shared" : " 27 GB", | 20260507 04:10:18.551 | INFO | "malloc" : " 14 GB", | 20260507 04:10:18.551 | INFO | "pinned" : " 29 GB" | 20260507 04:10:18.551 | INFO | } | 20260507 04:10:18.551 | INFO | Perf report for the last 5.13949 seconds: | 20260507 04:10:18.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:10:18.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:18.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:18.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:18.551 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 04:10:18.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:18.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:18.551 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:10:18.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:18.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:18.551 | INFO | Uploaded 36GiB to GPU in 2.02914s (17.7415GiB/s) | 20260507 04:10:18.551 | INFO | Downloaded 34GiB to GPU in 1.9431s (17.4978GiB/s) | 20260507 04:10:23.681 | INFO | Analyzing chunk frames = [6986240,6986752) | 20260507 04:10:23.681 | INFO | MemInfo: { | 20260507 04:10:23.681 | INFO | "gpu" : " 17 GB", | 20260507 04:10:23.681 | INFO | "huge" : "", | 20260507 04:10:23.682 | INFO | "huge_shared" : " 27 GB", | 20260507 04:10:23.682 | INFO | "malloc" : " 14 GB", | 20260507 04:10:23.682 | INFO | "pinned" : " 29 GB" | 20260507 04:10:23.682 | INFO | } | 20260507 04:10:23.682 | INFO | Perf report for the last 5.13064 seconds: | 20260507 04:10:23.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:10:23.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:23.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:23.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:23.682 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 04:10:23.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:23.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:23.682 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:10:23.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:23.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:23.682 | INFO | Uploaded 36GiB to GPU in 2.02884s (17.7442GiB/s) | 20260507 04:10:23.682 | INFO | Downloaded 34GiB to GPU in 1.90286s (17.8678GiB/s) | 20260507 04:10:28.819 | INFO | Analyzing chunk frames = [6986752,6987264) | 20260507 04:10:28.819 | INFO | MemInfo: { | 20260507 04:10:28.819 | INFO | "gpu" : " 17 GB", | 20260507 04:10:28.819 | INFO | "huge" : "", | 20260507 04:10:28.819 | INFO | "huge_shared" : " 27 GB", | 20260507 04:10:28.819 | INFO | "malloc" : " 14 GB", | 20260507 04:10:28.819 | INFO | "pinned" : " 29 GB" | 20260507 04:10:28.819 | INFO | } | 20260507 04:10:28.819 | INFO | Perf report for the last 5.13768 seconds: | 20260507 04:10:28.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:10:28.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:28.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:28.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:28.819 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:10:28.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:28.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:28.820 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 04:10:28.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:28.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:28.820 | INFO | Uploaded 36GiB to GPU in 1.96704s (18.3016GiB/s) | 20260507 04:10:28.820 | INFO | Downloaded 34GiB to GPU in 1.9718s (17.2432GiB/s) | 20260507 04:10:33.966 | INFO | Analyzing chunk frames = [6987264,6987776) | 20260507 04:10:33.966 | INFO | MemInfo: { | 20260507 04:10:33.966 | INFO | "gpu" : " 17 GB", | 20260507 04:10:33.966 | INFO | "huge" : "", | 20260507 04:10:33.966 | INFO | "huge_shared" : " 27 GB", | 20260507 04:10:33.966 | INFO | "malloc" : " 14 GB", | 20260507 04:10:33.966 | INFO | "pinned" : " 29 GB" | 20260507 04:10:33.966 | INFO | } | 20260507 04:10:33.967 | INFO | Perf report for the last 5.14708 seconds: | 20260507 04:10:33.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:10:33.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:33.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:33.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:33.967 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:10:33.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:33.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:33.967 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 04:10:33.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:33.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:33.967 | INFO | Uploaded 36GiB to GPU in 1.9742s (18.2352GiB/s) | 20260507 04:10:33.967 | INFO | Downloaded 34GiB to GPU in 1.95561s (17.3858GiB/s) | 20260507 04:10:33.967 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:10:33.967 | INFO | Total mem usage: 18 GiB | 20260507 04:10:33.967 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:10:33.967 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 04:10:33.967 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:10:33.967 | INFO | High Water Breakdown: | 20260507 04:10:33.967 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:10:33.967 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:10:33.967 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:10:33.967 | INFO | ------------------------------------------ | 20260507 04:10:33.967 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:10:33.967 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:10:33.967 | INFO | Total mem usage: 26.5 GiB | 20260507 04:10:33.967 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:10:33.967 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:10:33.967 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:10:33.967 | INFO | High Water Breakdown: | 20260507 04:10:33.967 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:10:33.967 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:10:33.967 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:10:33.967 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:10:33.967 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:10:33.967 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:10:33.967 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:10:33.967 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:10:33.967 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:10:33.967 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:10:33.967 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:10:33.967 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:10:33.967 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:10:33.967 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:10:33.967 | INFO | ------------------------------------------ | 20260507 04:10:33.967 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:10:33.967 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:10:33.967 | INFO | Total mem usage: 25.6 GiB | 20260507 04:10:33.967 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:10:33.967 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:10:33.967 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:10:33.968 | INFO | High Water Breakdown: | 20260507 04:10:33.968 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:10:33.968 | INFO | ------------------------------------------ | 20260507 04:10:33.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:10:33.968 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:10:33.968 | INFO | Total mem usage: 15.7 GiB | 20260507 04:10:33.968 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:10:33.968 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:10:33.968 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:10:33.968 | INFO | High Water Breakdown: | 20260507 04:10:33.968 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:10:33.968 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:10:33.968 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:10:33.968 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:10:33.968 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:10:33.968 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:10:33.968 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:10:33.968 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:10:33.968 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:10:33.968 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:10:33.968 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:10:33.968 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:10:33.968 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:10:33.968 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:10:33.968 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:10:33.968 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:10:33.968 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:10:33.968 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:10:33.968 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:10:33.968 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:10:33.968 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:10:33.968 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:10:33.968 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:10:33.968 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:10:33.968 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:10:33.968 | INFO | ------------------------------------------ | 20260507 04:10:33.968 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:10:33.968 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:10:33.968 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:10:33.968 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:10:33.968 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.34e+01 / 9.68e-01 | 20260507 04:10:33.968 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.01e-01, 9.69e-03 / 4.13e-03 | 20260507 04:10:33.968 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.52e+00, 5.13e-01 / 2.00e-01 | 20260507 04:10:33.968 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.48e+00, 1.68e-01 / 1.03e-01 | 20260507 04:10:33.968 | INFO | PerfReport: 16.15%, DME -- 3.97e+01 / 4.44e+01, 3.93e+00 / 1.33e+00 | 20260507 04:10:33.968 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.24e+00 / 1.55e+00 | 20260507 04:10:33.968 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.14e+00 / 2.70e+00, 3.84e-01 / 2.36e-01 | 20260507 04:10:33.968 | INFO | PerfReport: 3.04%, Metrics -- 9.32e-01 / 1.23e+00, 2.46e-01 / 1.28e-01 | 20260507 04:10:33.968 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.14e-01 / 3.63e-01, 4.94e-02 / 1.86e-02 | 20260507 04:10:33.968 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:10:33.968 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:10:33.968 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:10:33.968 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 04:10:33.968 | INFO | PerfReport: 21.74%, Upload -- 2.09e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260507 04:10:33.968 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260507 04:10:33.968 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:10:33.968 | INFO | PerfReport: 1.45%, LockUpload -- 1.40e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 04:10:33.968 | INFO | PerfReport: 1.35%, LockDownload -- 1.30e+00 / 0.00e+00, 3.14e+00 / 0.00e+00 | 20260507 04:10:33.968 | INFO | PerfReport: 22.12%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 04:10:33.968 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.46e-02 / 0.00e+00 | 20260507 04:10:34.063 | INFO | Starting a round of Baz IO | 20260507 04:10:37.167 | INFO | Populating RTMetrics report | 20260507 04:10:37.167 | INFO | RTMetrics report populated | 20260507 04:10:37.167 | INFO | Opening tmp RTMetrics json file | 20260507 04:10:37.186 | INFO | Serializing RTMetrics to json | 20260507 04:10:37.187 | INFO | Serialization done, swapping tmp file now... | 20260507 04:10:37.187 | INFO | RTMetrics done | 20260507 04:10:37.258 | INFO | Finished a round of Baz IO: Wrote 1.732 GiB in 3.194 seconds | 20260507 04:10:37.258 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.82% / 0.00% / 67.18% | 20260507 04:10:37.258 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.177761% / 0% / 8.34465e-05% | 20260507 04:10:37.258 | INFO | Baz IO overall: | 20260507 04:10:37.258 | INFO | BytesWritten : 3572044286150 | 20260507 04:10:37.258 | INFO | NumEvents : 1715284366304 | 20260507 04:10:39.111 | INFO | Analyzing chunk frames = [6987776,6988288) | 20260507 04:10:39.111 | INFO | MemInfo: { | 20260507 04:10:39.111 | INFO | "gpu" : " 17 GB", | 20260507 04:10:39.111 | INFO | "huge" : "", | 20260507 04:10:39.111 | INFO | "huge_shared" : " 27 GB", | 20260507 04:10:39.111 | INFO | "malloc" : " 14 GB", | 20260507 04:10:39.111 | INFO | "pinned" : " 29 GB" | 20260507 04:10:39.111 | INFO | } | 20260507 04:10:39.111 | INFO | Perf report for the last 5.1441 seconds: | 20260507 04:10:39.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:10:39.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:39.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:39.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:39.111 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:10:39.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:39.111 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 04:10:39.111 | INFO | PRE_HQ : 59.2%, 1.8, 56.9ms, 0 | 20260507 04:10:39.111 | INFO | BAZWRITER : 61.7%, 3.3, 105.4ms, 0 | 20260507 04:10:39.111 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 04:10:39.111 | INFO | Uploaded 36GiB to GPU in 2.05183s (17.5453GiB/s) | 20260507 04:10:39.111 | INFO | Downloaded 38GiB to GPU in 2.18347s (17.4035GiB/s) | 20260507 04:10:44.245 | INFO | Analyzing chunk frames = [6988288,6988800) | 20260507 04:10:44.245 | INFO | MemInfo: { | 20260507 04:10:44.245 | INFO | "gpu" : " 17 GB", | 20260507 04:10:44.245 | INFO | "huge" : "", | 20260507 04:10:44.245 | INFO | "huge_shared" : " 27 GB", | 20260507 04:10:44.245 | INFO | "malloc" : " 14 GB", | 20260507 04:10:44.245 | INFO | "pinned" : " 29 GB" | 20260507 04:10:44.245 | INFO | } | 20260507 04:10:44.245 | INFO | Perf report for the last 5.1344 seconds: | 20260507 04:10:44.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:10:44.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:44.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:44.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:44.245 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:10:44.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:44.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:44.245 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:10:44.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:44.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:44.245 | INFO | Uploaded 36GiB to GPU in 1.98414s (18.1439GiB/s) | 20260507 04:10:44.245 | INFO | Downloaded 34GiB to GPU in 1.92722s (17.642GiB/s) | 20260507 04:10:49.382 | INFO | Analyzing chunk frames = [6988800,6989312) | 20260507 04:10:49.382 | INFO | MemInfo: { | 20260507 04:10:49.382 | INFO | "gpu" : " 17 GB", | 20260507 04:10:49.382 | INFO | "huge" : "", | 20260507 04:10:49.382 | INFO | "huge_shared" : " 27 GB", | 20260507 04:10:49.382 | INFO | "malloc" : " 14 GB", | 20260507 04:10:49.382 | INFO | "pinned" : " 29 GB" | 20260507 04:10:49.382 | INFO | } | 20260507 04:10:49.382 | INFO | Perf report for the last 5.13731 seconds: | 20260507 04:10:49.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:10:49.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:49.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:49.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:49.382 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 04:10:49.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:49.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:49.382 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:10:49.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:49.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:49.383 | INFO | Uploaded 36GiB to GPU in 2.03893s (17.6563GiB/s) | 20260507 04:10:49.383 | INFO | Downloaded 34GiB to GPU in 1.95235s (17.4149GiB/s) | 20260507 04:10:54.519 | INFO | Analyzing chunk frames = [6989312,6989824) | 20260507 04:10:54.519 | INFO | MemInfo: { | 20260507 04:10:54.519 | INFO | "gpu" : " 17 GB", | 20260507 04:10:54.519 | INFO | "huge" : "", | 20260507 04:10:54.519 | INFO | "huge_shared" : " 27 GB", | 20260507 04:10:54.519 | INFO | "malloc" : " 14 GB", | 20260507 04:10:54.519 | INFO | "pinned" : " 29 GB" | 20260507 04:10:54.519 | INFO | } | 20260507 04:10:54.519 | INFO | Perf report for the last 5.13704 seconds: | 20260507 04:10:54.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:10:54.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:54.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:54.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:54.519 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 04:10:54.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:54.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:54.520 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:10:54.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:54.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:54.520 | INFO | Uploaded 36GiB to GPU in 1.99248s (18.068GiB/s) | 20260507 04:10:54.520 | INFO | Downloaded 34GiB to GPU in 1.91344s (17.7691GiB/s) | 20260507 04:10:59.658 | INFO | Analyzing chunk frames = [6989824,6990336) | 20260507 04:10:59.658 | INFO | MemInfo: { | 20260507 04:10:59.658 | INFO | "gpu" : " 17 GB", | 20260507 04:10:59.658 | INFO | "huge" : "", | 20260507 04:10:59.658 | INFO | "huge_shared" : " 27 GB", | 20260507 04:10:59.658 | INFO | "malloc" : " 14 GB", | 20260507 04:10:59.658 | INFO | "pinned" : " 29 GB" | 20260507 04:10:59.659 | INFO | } | 20260507 04:10:59.659 | INFO | Perf report for the last 5.13913 seconds: | 20260507 04:10:59.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:10:59.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:59.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:59.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:59.659 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 04:10:59.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:59.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:59.659 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:10:59.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:59.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:10:59.659 | INFO | Uploaded 36GiB to GPU in 2.02071s (17.8155GiB/s) | 20260507 04:10:59.659 | INFO | Downloaded 34GiB to GPU in 1.9334s (17.5856GiB/s) | 20260507 04:11:04.803 | INFO | Analyzing chunk frames = [6990336,6990848) | 20260507 04:11:04.803 | INFO | MemInfo: { | 20260507 04:11:04.803 | INFO | "gpu" : " 17 GB", | 20260507 04:11:04.803 | INFO | "huge" : "", | 20260507 04:11:04.803 | INFO | "huge_shared" : " 27 GB", | 20260507 04:11:04.803 | INFO | "malloc" : " 14 GB", | 20260507 04:11:04.803 | INFO | "pinned" : " 29 GB" | 20260507 04:11:04.803 | INFO | } | 20260507 04:11:04.803 | INFO | Perf report for the last 5.14436 seconds: | 20260507 04:11:04.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:11:04.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:04.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:04.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:04.803 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:11:04.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:04.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:04.803 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:11:04.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:04.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:04.803 | INFO | Uploaded 36GiB to GPU in 2.05747s (17.4972GiB/s) | 20260507 04:11:04.803 | INFO | Downloaded 34GiB to GPU in 1.95407s (17.3996GiB/s) | 20260507 04:11:09.942 | INFO | Analyzing chunk frames = [6990848,6991360) | 20260507 04:11:09.942 | INFO | MemInfo: { | 20260507 04:11:09.942 | INFO | "gpu" : " 17 GB", | 20260507 04:11:09.942 | INFO | "huge" : "", | 20260507 04:11:09.942 | INFO | "huge_shared" : " 27 GB", | 20260507 04:11:09.942 | INFO | "malloc" : " 14 GB", | 20260507 04:11:09.942 | INFO | "pinned" : " 29 GB" | 20260507 04:11:09.942 | INFO | } | 20260507 04:11:09.942 | INFO | Perf report for the last 5.13892 seconds: | 20260507 04:11:09.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:11:09.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:09.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:09.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:09.942 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:11:09.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:09.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:09.942 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:11:09.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:09.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:09.942 | INFO | Uploaded 36GiB to GPU in 1.97133s (18.2618GiB/s) | 20260507 04:11:09.942 | INFO | Downloaded 34GiB to GPU in 1.97702s (17.1976GiB/s) | 20260507 04:11:15.078 | INFO | Analyzing chunk frames = [6991360,6991872) | 20260507 04:11:15.078 | INFO | MemInfo: { | 20260507 04:11:15.078 | INFO | "gpu" : " 17 GB", | 20260507 04:11:15.078 | INFO | "huge" : "", | 20260507 04:11:15.078 | INFO | "huge_shared" : " 27 GB", | 20260507 04:11:15.079 | INFO | "malloc" : " 14 GB", | 20260507 04:11:15.079 | INFO | "pinned" : " 29 GB" | 20260507 04:11:15.079 | INFO | } | 20260507 04:11:15.079 | INFO | Perf report for the last 5.13669 seconds: | 20260507 04:11:15.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:11:15.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:15.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:15.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:15.079 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:11:15.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:15.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:15.079 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 04:11:15.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:15.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:15.079 | INFO | Uploaded 36GiB to GPU in 1.99661s (18.0305GiB/s) | 20260507 04:11:15.079 | INFO | Downloaded 34GiB to GPU in 1.9757s (17.2091GiB/s) | 20260507 04:11:15.175 | INFO | Starting a round of Baz IO | 20260507 04:11:18.309 | INFO | Populating RTMetrics report | 20260507 04:11:18.309 | INFO | RTMetrics report populated | 20260507 04:11:18.309 | INFO | Opening tmp RTMetrics json file | 20260507 04:11:18.310 | INFO | Serializing RTMetrics to json | 20260507 04:11:18.311 | INFO | Serialization done, swapping tmp file now... | 20260507 04:11:18.311 | INFO | RTMetrics done | 20260507 04:11:18.405 | INFO | Finished a round of Baz IO: Wrote 1.731 GiB in 3.23 seconds | 20260507 04:11:18.405 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.82% / 0.00% / 67.18% | 20260507 04:11:18.405 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.176668% / 0% / 8.74201e-05% | 20260507 04:11:18.405 | INFO | Baz IO overall: | 20260507 04:11:18.405 | INFO | BytesWritten : 3573903163494 | 20260507 04:11:18.406 | INFO | NumEvents : 1716159404748 | 20260507 04:11:20.223 | INFO | Analyzing chunk frames = [6991872,6992384) | 20260507 04:11:20.223 | INFO | MemInfo: { | 20260507 04:11:20.223 | INFO | "gpu" : " 17 GB", | 20260507 04:11:20.223 | INFO | "huge" : "", | 20260507 04:11:20.223 | INFO | "huge_shared" : " 27 GB", | 20260507 04:11:20.223 | INFO | "malloc" : " 14 GB", | 20260507 04:11:20.223 | INFO | "pinned" : " 29 GB" | 20260507 04:11:20.223 | INFO | } | 20260507 04:11:20.223 | INFO | Perf report for the last 5.14474 seconds: | 20260507 04:11:20.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:11:20.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:20.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:20.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:20.223 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260507 04:11:20.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:20.223 | INFO | RT_METRICS : 45.7%, 1.0, 24.0ms, 0 | 20260507 04:11:20.223 | INFO | PRE_HQ : 60.2%, 2.0, 61.2ms, 0 | 20260507 04:11:20.223 | INFO | BAZWRITER : 62.4%, 3.4, 108.8ms, 0 | 20260507 04:11:20.223 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 04:11:20.224 | INFO | Uploaded 36GiB to GPU in 2.04426s (17.6103GiB/s) | 20260507 04:11:20.224 | INFO | Downloaded 38GiB to GPU in 2.17747s (17.4515GiB/s) | 20260507 04:11:25.360 | INFO | Analyzing chunk frames = [6992384,6992896) | 20260507 04:11:25.360 | INFO | MemInfo: { | 20260507 04:11:25.360 | INFO | "gpu" : " 17 GB", | 20260507 04:11:25.360 | INFO | "huge" : "", | 20260507 04:11:25.360 | INFO | "huge_shared" : " 27 GB", | 20260507 04:11:25.360 | INFO | "malloc" : " 14 GB", | 20260507 04:11:25.360 | INFO | "pinned" : " 29 GB" | 20260507 04:11:25.360 | INFO | } | 20260507 04:11:25.361 | INFO | Perf report for the last 5.13721 seconds: | 20260507 04:11:25.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:11:25.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:25.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:25.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:25.361 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 04:11:25.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:25.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:25.361 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:11:25.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:25.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:25.361 | INFO | Uploaded 36GiB to GPU in 2.04895s (17.57GiB/s) | 20260507 04:11:25.361 | INFO | Downloaded 34GiB to GPU in 1.93876s (17.537GiB/s) | 20260507 04:11:30.498 | INFO | Analyzing chunk frames = [6992896,6993408) | 20260507 04:11:30.498 | INFO | MemInfo: { | 20260507 04:11:30.498 | INFO | "gpu" : " 17 GB", | 20260507 04:11:30.498 | INFO | "huge" : "", | 20260507 04:11:30.498 | INFO | "huge_shared" : " 27 GB", | 20260507 04:11:30.498 | INFO | "malloc" : " 14 GB", | 20260507 04:11:30.498 | INFO | "pinned" : " 29 GB" | 20260507 04:11:30.498 | INFO | } | 20260507 04:11:30.498 | INFO | Perf report for the last 5.13753 seconds: | 20260507 04:11:30.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:11:30.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:30.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:30.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:30.498 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 04:11:30.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:30.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:30.498 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:11:30.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:30.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:30.498 | INFO | Uploaded 36GiB to GPU in 2.04448s (17.6084GiB/s) | 20260507 04:11:30.498 | INFO | Downloaded 34GiB to GPU in 1.95065s (17.4301GiB/s) | 20260507 04:11:35.637 | INFO | Analyzing chunk frames = [6993408,6993920) | 20260507 04:11:35.637 | INFO | MemInfo: { | 20260507 04:11:35.637 | INFO | "gpu" : " 17 GB", | 20260507 04:11:35.637 | INFO | "huge" : "", | 20260507 04:11:35.637 | INFO | "huge_shared" : " 27 GB", | 20260507 04:11:35.637 | INFO | "malloc" : " 14 GB", | 20260507 04:11:35.637 | INFO | "pinned" : " 29 GB" | 20260507 04:11:35.637 | INFO | } | 20260507 04:11:35.637 | INFO | Perf report for the last 5.13865 seconds: | 20260507 04:11:35.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:11:35.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:35.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:35.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:35.637 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 04:11:35.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:35.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:35.637 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:11:35.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:35.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:35.637 | INFO | Uploaded 36GiB to GPU in 1.99087s (18.0825GiB/s) | 20260507 04:11:35.637 | INFO | Downloaded 34GiB to GPU in 1.91739s (17.7325GiB/s) | 20260507 04:11:40.773 | INFO | Analyzing chunk frames = [6993920,6994432) | 20260507 04:11:40.773 | INFO | MemInfo: { | 20260507 04:11:40.773 | INFO | "gpu" : " 17 GB", | 20260507 04:11:40.773 | INFO | "huge" : "", | 20260507 04:11:40.773 | INFO | "huge_shared" : " 27 GB", | 20260507 04:11:40.773 | INFO | "malloc" : " 14 GB", | 20260507 04:11:40.773 | INFO | "pinned" : " 29 GB" | 20260507 04:11:40.773 | INFO | } | 20260507 04:11:40.773 | INFO | Perf report for the last 5.13619 seconds: | 20260507 04:11:40.773 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:11:40.773 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:40.773 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:40.773 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:40.773 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 04:11:40.773 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:40.773 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:40.773 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:11:40.773 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:40.773 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:40.773 | INFO | Uploaded 36GiB to GPU in 2.03138s (17.7219GiB/s) | 20260507 04:11:40.773 | INFO | Downloaded 34GiB to GPU in 1.93394s (17.5807GiB/s) | 20260507 04:11:45.918 | INFO | Analyzing chunk frames = [6994432,6994944) | 20260507 04:11:45.918 | INFO | MemInfo: { | 20260507 04:11:45.918 | INFO | "gpu" : " 17 GB", | 20260507 04:11:45.918 | INFO | "huge" : "", | 20260507 04:11:45.918 | INFO | "huge_shared" : " 27 GB", | 20260507 04:11:45.918 | INFO | "malloc" : " 14 GB", | 20260507 04:11:45.918 | INFO | "pinned" : " 29 GB" | 20260507 04:11:45.918 | INFO | } | 20260507 04:11:45.918 | INFO | Perf report for the last 5.14544 seconds: | 20260507 04:11:45.918 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:11:45.918 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:45.918 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:45.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:45.918 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:11:45.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:45.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:45.919 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:11:45.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:45.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:45.919 | INFO | Uploaded 36GiB to GPU in 2.05634s (17.5068GiB/s) | 20260507 04:11:45.919 | INFO | Downloaded 34GiB to GPU in 1.95465s (17.3944GiB/s) | 20260507 04:11:51.052 | INFO | Analyzing chunk frames = [6994944,6995456) | 20260507 04:11:51.052 | INFO | MemInfo: { | 20260507 04:11:51.052 | INFO | "gpu" : " 17 GB", | 20260507 04:11:51.052 | INFO | "huge" : "", | 20260507 04:11:51.052 | INFO | "huge_shared" : " 27 GB", | 20260507 04:11:51.052 | INFO | "malloc" : " 14 GB", | 20260507 04:11:51.052 | INFO | "pinned" : " 29 GB" | 20260507 04:11:51.052 | INFO | } | 20260507 04:11:51.052 | INFO | Perf report for the last 5.13365 seconds: | 20260507 04:11:51.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:11:51.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:51.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:51.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:51.052 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 04:11:51.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:51.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:51.052 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 04:11:51.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:51.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:51.052 | INFO | Uploaded 36GiB to GPU in 1.99681s (18.0288GiB/s) | 20260507 04:11:51.052 | INFO | Downloaded 34GiB to GPU in 1.98216s (17.153GiB/s) | 20260507 04:11:56.193 | INFO | Analyzing chunk frames = [6995456,6995968) | 20260507 04:11:56.193 | INFO | MemInfo: { | 20260507 04:11:56.193 | INFO | "gpu" : " 17 GB", | 20260507 04:11:56.193 | INFO | "huge" : "", | 20260507 04:11:56.193 | INFO | "huge_shared" : " 27 GB", | 20260507 04:11:56.193 | INFO | "malloc" : " 14 GB", | 20260507 04:11:56.193 | INFO | "pinned" : " 29 GB" | 20260507 04:11:56.193 | INFO | } | 20260507 04:11:56.193 | INFO | Perf report for the last 5.14121 seconds: | 20260507 04:11:56.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:11:56.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:56.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:56.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:56.193 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:11:56.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:56.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:56.193 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 04:11:56.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:56.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:11:56.194 | INFO | Uploaded 36GiB to GPU in 1.99528s (18.0425GiB/s) | 20260507 04:11:56.194 | INFO | Downloaded 34GiB to GPU in 1.97104s (17.2498GiB/s) | 20260507 04:11:56.194 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:11:56.194 | INFO | Total mem usage: 18 GiB | 20260507 04:11:56.194 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:11:56.194 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 04:11:56.194 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:11:56.194 | INFO | High Water Breakdown: | 20260507 04:11:56.194 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:11:56.194 | INFO | 39.8% : 5.66 GiB ( 84.6% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:11:56.194 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:11:56.194 | INFO | ------------------------------------------ | 20260507 04:11:56.194 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:11:56.194 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:11:56.194 | INFO | Total mem usage: 26.5 GiB | 20260507 04:11:56.194 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:11:56.194 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:11:56.194 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:11:56.194 | INFO | High Water Breakdown: | 20260507 04:11:56.194 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:11:56.194 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:11:56.194 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:11:56.194 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:11:56.194 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:11:56.194 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:11:56.194 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:11:56.194 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:11:56.194 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:11:56.194 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:11:56.194 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:11:56.194 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:11:56.194 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:11:56.194 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:11:56.194 | INFO | ------------------------------------------ | 20260507 04:11:56.194 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:11:56.194 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:11:56.194 | INFO | Total mem usage: 25.6 GiB | 20260507 04:11:56.194 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:11:56.194 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:11:56.194 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:11:56.194 | INFO | High Water Breakdown: | 20260507 04:11:56.194 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:11:56.194 | INFO | ------------------------------------------ | 20260507 04:11:56.194 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:11:56.194 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:11:56.194 | INFO | Total mem usage: 15.7 GiB | 20260507 04:11:56.194 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:11:56.194 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:11:56.194 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:11:56.194 | INFO | High Water Breakdown: | 20260507 04:11:56.194 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:11:56.194 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:11:56.194 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:11:56.194 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:11:56.194 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:11:56.194 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:11:56.194 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:11:56.194 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:11:56.194 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:11:56.194 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:11:56.194 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:11:56.194 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:11:56.194 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:11:56.195 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:11:56.195 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:11:56.195 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:11:56.195 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:11:56.195 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:11:56.195 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:11:56.195 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:11:56.195 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:11:56.195 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:11:56.195 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:11:56.195 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:11:56.195 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:11:56.195 | INFO | ------------------------------------------ | 20260507 04:11:56.195 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:11:56.195 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:11:56.195 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:11:56.195 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:11:56.195 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.50e+01, 1.34e+01 / 1.48e+00 | 20260507 04:11:56.195 | INFO | PerfReport: 0.29%, AggregateStats -- 9.02e-02 / 1.01e-01, 9.27e-03 / 5.19e-03 | 20260507 04:11:56.195 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.46e+00, 5.08e-01 / 2.32e-01 | 20260507 04:11:56.195 | INFO | PerfReport: 3.22%, Histogram -- 9.87e-01 / 1.46e+00, 1.64e-01 / 1.04e-01 | 20260507 04:11:56.195 | INFO | PerfReport: 16.25%, DME -- 3.99e+01 / 4.50e+01, 4.03e+00 / 1.63e+00 | 20260507 04:11:56.195 | INFO | PerfReport: 57.50%, FrameLabeling -- 1.76e+01 / 1.97e+01, 1.13e+00 / 1.09e+00 | 20260507 04:11:56.195 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.15e+00 / 2.68e+00, 3.88e-01 / 2.22e-01 | 20260507 04:11:56.195 | INFO | PerfReport: 3.04%, Metrics -- 9.34e-01 / 1.25e+00, 2.48e-01 / 1.16e-01 | 20260507 04:11:56.195 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.14e-01 / 3.72e-01, 4.88e-02 / 2.35e-02 | 20260507 04:11:56.195 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:11:56.195 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:11:56.195 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:11:56.195 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 04:11:56.195 | INFO | PerfReport: 21.86%, Upload -- 2.10e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260507 04:11:56.195 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 04:11:56.195 | INFO | PerfReport: 31.88%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:11:56.195 | INFO | PerfReport: 1.60%, LockUpload -- 1.54e+00 / 0.00e+00, 5.01e+00 / 0.00e+00 | 20260507 04:11:56.195 | INFO | PerfReport: 1.25%, LockDownload -- 1.21e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260507 04:11:56.195 | INFO | PerfReport: 21.98%, LockCompute -- 2.12e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 04:11:56.195 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260507 04:11:56.290 | INFO | Starting a round of Baz IO | 20260507 04:11:59.408 | INFO | Populating RTMetrics report | 20260507 04:11:59.409 | INFO | RTMetrics report populated | 20260507 04:11:59.409 | INFO | Opening tmp RTMetrics json file | 20260507 04:11:59.410 | INFO | Serializing RTMetrics to json | 20260507 04:11:59.411 | INFO | Serialization done, swapping tmp file now... | 20260507 04:11:59.412 | INFO | RTMetrics done | 20260507 04:11:59.508 | INFO | Finished a round of Baz IO: Wrote 1.732 GiB in 3.218 seconds | 20260507 04:11:59.508 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.82% / 0.00% / 67.18% | 20260507 04:11:59.508 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.176871% / 7.94729e-06% / 8.34465e-05% | 20260507 04:11:59.508 | INFO | Baz IO overall: | 20260507 04:11:59.508 | INFO | BytesWritten : 3575762464174 | 20260507 04:11:59.508 | INFO | NumEvents : 1717034573749 | 20260507 04:12:01.334 | INFO | Analyzing chunk frames = [6995968,6996480) | 20260507 04:12:01.334 | INFO | MemInfo: { | 20260507 04:12:01.334 | INFO | "gpu" : " 17 GB", | 20260507 04:12:01.334 | INFO | "huge" : "", | 20260507 04:12:01.334 | INFO | "huge_shared" : " 27 GB", | 20260507 04:12:01.334 | INFO | "malloc" : " 14 GB", | 20260507 04:12:01.334 | INFO | "pinned" : " 29 GB" | 20260507 04:12:01.334 | INFO | } | 20260507 04:12:01.334 | INFO | Perf report for the last 5.14036 seconds: | 20260507 04:12:01.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:12:01.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:01.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:01.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:01.334 | INFO | ANALYSIS : 61.8%, 2.9, 95.4ms, 0 | 20260507 04:12:01.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:01.334 | INFO | RT_METRICS : 45.0%, 1.0, 23.6ms, 0 | 20260507 04:12:01.334 | INFO | PRE_HQ : 60.0%, 1.9, 60.4ms, 0 | 20260507 04:12:01.334 | INFO | BAZWRITER : 62.2%, 3.4, 108.9ms, 0 | 20260507 04:12:01.334 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 04:12:01.334 | INFO | Uploaded 36GiB to GPU in 2.05301s (17.5352GiB/s) | 20260507 04:12:01.334 | INFO | Downloaded 38GiB to GPU in 2.17269s (17.4899GiB/s) | 20260507 04:12:06.471 | INFO | Analyzing chunk frames = [6996480,6996992) | 20260507 04:12:06.471 | INFO | MemInfo: { | 20260507 04:12:06.471 | INFO | "gpu" : " 17 GB", | 20260507 04:12:06.471 | INFO | "huge" : "", | 20260507 04:12:06.471 | INFO | "huge_shared" : " 27 GB", | 20260507 04:12:06.471 | INFO | "malloc" : " 14 GB", | 20260507 04:12:06.471 | INFO | "pinned" : " 29 GB" | 20260507 04:12:06.471 | INFO | } | 20260507 04:12:06.471 | INFO | Perf report for the last 5.13709 seconds: | 20260507 04:12:06.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:12:06.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:06.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:06.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:06.471 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 04:12:06.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:06.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:06.471 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:12:06.471 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:06.471 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:06.471 | INFO | Uploaded 36GiB to GPU in 2.03072s (17.7277GiB/s) | 20260507 04:12:06.471 | INFO | Downloaded 34GiB to GPU in 1.9374s (17.5493GiB/s) | 20260507 04:12:11.614 | INFO | Analyzing chunk frames = [6996992,6997504) | 20260507 04:12:11.614 | INFO | MemInfo: { | 20260507 04:12:11.614 | INFO | "gpu" : " 17 GB", | 20260507 04:12:11.615 | INFO | "huge" : "", | 20260507 04:12:11.615 | INFO | "huge_shared" : " 27 GB", | 20260507 04:12:11.615 | INFO | "malloc" : " 14 GB", | 20260507 04:12:11.615 | INFO | "pinned" : " 29 GB" | 20260507 04:12:11.615 | INFO | } | 20260507 04:12:11.615 | INFO | Perf report for the last 5.144 seconds: | 20260507 04:12:11.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:12:11.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:11.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:11.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:11.615 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 04:12:11.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:11.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:11.615 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:12:11.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:11.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:11.615 | INFO | Uploaded 36GiB to GPU in 2.0125s (17.8882GiB/s) | 20260507 04:12:11.615 | INFO | Downloaded 34GiB to GPU in 1.93598s (17.5621GiB/s) | 20260507 04:12:16.755 | INFO | Analyzing chunk frames = [6997504,6998016) | 20260507 04:12:16.755 | INFO | MemInfo: { | 20260507 04:12:16.755 | INFO | "gpu" : " 17 GB", | 20260507 04:12:16.755 | INFO | "huge" : "", | 20260507 04:12:16.755 | INFO | "huge_shared" : " 27 GB", | 20260507 04:12:16.756 | INFO | "malloc" : " 14 GB", | 20260507 04:12:16.756 | INFO | "pinned" : " 29 GB" | 20260507 04:12:16.756 | INFO | } | 20260507 04:12:16.756 | INFO | Perf report for the last 5.14086 seconds: | 20260507 04:12:16.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:12:16.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:16.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:16.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:16.756 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 04:12:16.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:16.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:16.756 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:12:16.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:16.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:16.756 | INFO | Uploaded 36GiB to GPU in 1.99322s (18.0612GiB/s) | 20260507 04:12:16.756 | INFO | Downloaded 34GiB to GPU in 1.92441s (17.6677GiB/s) | 20260507 04:12:21.893 | INFO | Analyzing chunk frames = [6998016,6998528) | 20260507 04:12:21.893 | INFO | MemInfo: { | 20260507 04:12:21.893 | INFO | "gpu" : " 17 GB", | 20260507 04:12:21.893 | INFO | "huge" : "", | 20260507 04:12:21.893 | INFO | "huge_shared" : " 27 GB", | 20260507 04:12:21.893 | INFO | "malloc" : " 14 GB", | 20260507 04:12:21.893 | INFO | "pinned" : " 29 GB" | 20260507 04:12:21.894 | INFO | } | 20260507 04:12:21.894 | INFO | Perf report for the last 5.13789 seconds: | 20260507 04:12:21.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:12:21.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:21.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:21.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:21.894 | INFO | ANALYSIS : 60.2%, 3.0, 94.0ms, 0 | 20260507 04:12:21.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:21.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:21.894 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:12:21.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:21.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:21.894 | INFO | Uploaded 36GiB to GPU in 2.05573s (17.512GiB/s) | 20260507 04:12:21.894 | INFO | Downloaded 34GiB to GPU in 1.94832s (17.4509GiB/s) | 20260507 04:12:27.029 | INFO | Analyzing chunk frames = [6998528,6999040) | 20260507 04:12:27.029 | INFO | MemInfo: { | 20260507 04:12:27.029 | INFO | "gpu" : " 17 GB", | 20260507 04:12:27.029 | INFO | "huge" : "", | 20260507 04:12:27.029 | INFO | "huge_shared" : " 27 GB", | 20260507 04:12:27.029 | INFO | "malloc" : " 14 GB", | 20260507 04:12:27.029 | INFO | "pinned" : " 29 GB" | 20260507 04:12:27.029 | INFO | } | 20260507 04:12:27.029 | INFO | Perf report for the last 5.13548 seconds: | 20260507 04:12:27.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:12:27.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:27.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:27.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:27.029 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:12:27.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:27.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:27.029 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:12:27.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:27.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:27.029 | INFO | Uploaded 36GiB to GPU in 2.03765s (17.6674GiB/s) | 20260507 04:12:27.029 | INFO | Downloaded 34GiB to GPU in 1.93399s (17.5803GiB/s) | 20260507 04:12:32.173 | INFO | Analyzing chunk frames = [6999040,6999552) | 20260507 04:12:32.173 | INFO | MemInfo: { | 20260507 04:12:32.173 | INFO | "gpu" : " 17 GB", | 20260507 04:12:32.173 | INFO | "huge" : "", | 20260507 04:12:32.173 | INFO | "huge_shared" : " 27 GB", | 20260507 04:12:32.173 | INFO | "malloc" : " 14 GB", | 20260507 04:12:32.173 | INFO | "pinned" : " 29 GB" | 20260507 04:12:32.173 | INFO | } | 20260507 04:12:32.173 | INFO | Perf report for the last 5.14435 seconds: | 20260507 04:12:32.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:12:32.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:32.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:32.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:32.173 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:12:32.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:32.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:32.174 | INFO | PRE_HQ : 26.6%, 1.0, 13.9ms, 0 | 20260507 04:12:32.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:32.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:32.174 | INFO | Uploaded 36GiB to GPU in 1.97328s (18.2437GiB/s) | 20260507 04:12:32.174 | INFO | Downloaded 34GiB to GPU in 1.95341s (17.4055GiB/s) | 20260507 04:12:37.316 | INFO | Analyzing chunk frames = [6999552,7000064) | 20260507 04:12:37.316 | INFO | MemInfo: { | 20260507 04:12:37.316 | INFO | "gpu" : " 17 GB", | 20260507 04:12:37.316 | INFO | "huge" : "", | 20260507 04:12:37.316 | INFO | "huge_shared" : " 27 GB", | 20260507 04:12:37.316 | INFO | "malloc" : " 14 GB", | 20260507 04:12:37.316 | INFO | "pinned" : " 29 GB" | 20260507 04:12:37.316 | INFO | } | 20260507 04:12:37.316 | INFO | Perf report for the last 5.14244 seconds: | 20260507 04:12:37.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:12:37.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:37.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:37.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:37.316 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:12:37.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:37.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:37.316 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 04:12:37.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:37.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:37.316 | INFO | Uploaded 36GiB to GPU in 1.98551s (18.1313GiB/s) | 20260507 04:12:37.316 | INFO | Downloaded 34GiB to GPU in 1.96562s (17.2974GiB/s) | 20260507 04:12:37.413 | INFO | Starting a round of Baz IO | 20260507 04:12:40.512 | INFO | Populating RTMetrics report | 20260507 04:12:40.512 | INFO | RTMetrics report populated | 20260507 04:12:40.512 | INFO | Opening tmp RTMetrics json file | 20260507 04:12:40.513 | INFO | Serializing RTMetrics to json | 20260507 04:12:40.514 | INFO | Serialization done, swapping tmp file now... | 20260507 04:12:40.514 | INFO | RTMetrics done | 20260507 04:12:40.600 | INFO | Finished a round of Baz IO: Wrote 1.734 GiB in 3.187 seconds | 20260507 04:12:40.600 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.81% / 0.00% / 67.18% | 20260507 04:12:40.600 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.176779% / 3.97364e-06% / 0.000115236% | 20260507 04:12:40.600 | INFO | Baz IO overall: | 20260507 04:12:40.600 | INFO | BytesWritten : 3577623886531 | 20260507 04:12:40.600 | INFO | NumEvents : 1717909728927 | 20260507 04:12:42.451 | INFO | Analyzing chunk frames = [7000064,7000576) | 20260507 04:12:42.451 | INFO | MemInfo: { | 20260507 04:12:42.451 | INFO | "gpu" : " 17 GB", | 20260507 04:12:42.451 | INFO | "huge" : "", | 20260507 04:12:42.451 | INFO | "huge_shared" : " 27 GB", | 20260507 04:12:42.451 | INFO | "malloc" : " 14 GB", | 20260507 04:12:42.451 | INFO | "pinned" : " 29 GB" | 20260507 04:12:42.451 | INFO | } | 20260507 04:12:42.451 | INFO | Perf report for the last 5.13561 seconds: | 20260507 04:12:42.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:12:42.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:42.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:42.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:42.452 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 04:12:42.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:42.452 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 04:12:42.452 | INFO | PRE_HQ : 59.0%, 1.8, 55.0ms, 0 | 20260507 04:12:42.452 | INFO | BAZWRITER : 61.6%, 3.2, 100.8ms, 0 | 20260507 04:12:42.452 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:12:42.452 | INFO | Uploaded 36GiB to GPU in 2.06671s (17.419GiB/s) | 20260507 04:12:42.452 | INFO | Downloaded 38GiB to GPU in 2.16971s (17.5139GiB/s) | 20260507 04:12:47.584 | INFO | Analyzing chunk frames = [7000576,7001088) | 20260507 04:12:47.584 | INFO | MemInfo: { | 20260507 04:12:47.584 | INFO | "gpu" : " 17 GB", | 20260507 04:12:47.584 | INFO | "huge" : "", | 20260507 04:12:47.584 | INFO | "huge_shared" : " 27 GB", | 20260507 04:12:47.584 | INFO | "malloc" : " 14 GB", | 20260507 04:12:47.584 | INFO | "pinned" : " 29 GB" | 20260507 04:12:47.584 | INFO | } | 20260507 04:12:47.584 | INFO | Perf report for the last 5.13283 seconds: | 20260507 04:12:47.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:12:47.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:47.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:47.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:47.584 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 04:12:47.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:47.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:47.584 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:12:47.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:47.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:47.585 | INFO | Uploaded 36GiB to GPU in 1.99032s (18.0876GiB/s) | 20260507 04:12:47.585 | INFO | Downloaded 34GiB to GPU in 1.93295s (17.5897GiB/s) | 20260507 04:12:52.727 | INFO | Analyzing chunk frames = [7001088,7001600) | 20260507 04:12:52.727 | INFO | MemInfo: { | 20260507 04:12:52.727 | INFO | "gpu" : " 17 GB", | 20260507 04:12:52.727 | INFO | "huge" : "", | 20260507 04:12:52.727 | INFO | "huge_shared" : " 27 GB", | 20260507 04:12:52.727 | INFO | "malloc" : " 14 GB", | 20260507 04:12:52.727 | INFO | "pinned" : " 29 GB" | 20260507 04:12:52.727 | INFO | } | 20260507 04:12:52.727 | INFO | Perf report for the last 5.14283 seconds: | 20260507 04:12:52.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:12:52.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:52.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:52.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:52.727 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 04:12:52.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:52.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:52.727 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 04:12:52.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:52.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:52.727 | INFO | Uploaded 36GiB to GPU in 1.99823s (18.0159GiB/s) | 20260507 04:12:52.727 | INFO | Downloaded 34GiB to GPU in 1.93502s (17.5709GiB/s) | 20260507 04:12:57.869 | INFO | Analyzing chunk frames = [7001600,7002112) | 20260507 04:12:57.869 | INFO | MemInfo: { | 20260507 04:12:57.869 | INFO | "gpu" : " 17 GB", | 20260507 04:12:57.869 | INFO | "huge" : "", | 20260507 04:12:57.869 | INFO | "huge_shared" : " 27 GB", | 20260507 04:12:57.869 | INFO | "malloc" : " 14 GB", | 20260507 04:12:57.869 | INFO | "pinned" : " 29 GB" | 20260507 04:12:57.869 | INFO | } | 20260507 04:12:57.869 | INFO | Perf report for the last 5.1419 seconds: | 20260507 04:12:57.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:12:57.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:57.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:57.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:57.869 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:12:57.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:57.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:57.870 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 04:12:57.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:57.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:12:57.870 | INFO | Uploaded 36GiB to GPU in 1.97222s (18.2535GiB/s) | 20260507 04:12:57.870 | INFO | Downloaded 34GiB to GPU in 1.89914s (17.9028GiB/s) | 20260507 04:13:03.001 | INFO | Analyzing chunk frames = [7002112,7002624) | 20260507 04:13:03.001 | INFO | MemInfo: { | 20260507 04:13:03.001 | INFO | "gpu" : " 17 GB", | 20260507 04:13:03.001 | INFO | "huge" : "", | 20260507 04:13:03.001 | INFO | "huge_shared" : " 27 GB", | 20260507 04:13:03.001 | INFO | "malloc" : " 14 GB", | 20260507 04:13:03.001 | INFO | "pinned" : " 29 GB" | 20260507 04:13:03.001 | INFO | } | 20260507 04:13:03.001 | INFO | Perf report for the last 5.13186 seconds: | 20260507 04:13:03.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:13:03.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:03.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:03.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:03.001 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 04:13:03.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:03.001 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:03.001 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:13:03.001 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:03.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:03.001 | INFO | Uploaded 36GiB to GPU in 2.05743s (17.4976GiB/s) | 20260507 04:13:03.001 | INFO | Downloaded 34GiB to GPU in 1.94958s (17.4396GiB/s) | 20260507 04:13:08.143 | INFO | Analyzing chunk frames = [7002624,7003136) | 20260507 04:13:08.143 | INFO | MemInfo: { | 20260507 04:13:08.143 | INFO | "gpu" : " 17 GB", | 20260507 04:13:08.143 | INFO | "huge" : "", | 20260507 04:13:08.143 | INFO | "huge_shared" : " 27 GB", | 20260507 04:13:08.143 | INFO | "malloc" : " 14 GB", | 20260507 04:13:08.143 | INFO | "pinned" : " 29 GB" | 20260507 04:13:08.143 | INFO | } | 20260507 04:13:08.143 | INFO | Perf report for the last 5.14242 seconds: | 20260507 04:13:08.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:13:08.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:08.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:08.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:08.143 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:13:08.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:08.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:08.143 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:13:08.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:08.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:08.144 | INFO | Uploaded 36GiB to GPU in 2.05739s (17.4979GiB/s) | 20260507 04:13:08.144 | INFO | Downloaded 34GiB to GPU in 1.96643s (17.2902GiB/s) | 20260507 04:13:13.287 | INFO | Analyzing chunk frames = [7003136,7003648) | 20260507 04:13:13.287 | INFO | MemInfo: { | 20260507 04:13:13.287 | INFO | "gpu" : " 17 GB", | 20260507 04:13:13.287 | INFO | "huge" : "", | 20260507 04:13:13.287 | INFO | "huge_shared" : " 27 GB", | 20260507 04:13:13.287 | INFO | "malloc" : " 14 GB", | 20260507 04:13:13.287 | INFO | "pinned" : " 29 GB" | 20260507 04:13:13.287 | INFO | } | 20260507 04:13:13.287 | INFO | Perf report for the last 5.14369 seconds: | 20260507 04:13:13.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:13:13.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:13.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:13.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:13.287 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 04:13:13.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:13.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:13.287 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 04:13:13.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:13.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:13.287 | INFO | Uploaded 36GiB to GPU in 1.98535s (18.1328GiB/s) | 20260507 04:13:13.288 | INFO | Downloaded 34GiB to GPU in 1.97216s (17.24GiB/s) | 20260507 04:13:18.426 | INFO | Analyzing chunk frames = [7003648,7004160) | 20260507 04:13:18.426 | INFO | MemInfo: { | 20260507 04:13:18.426 | INFO | "gpu" : " 17 GB", | 20260507 04:13:18.426 | INFO | "huge" : "", | 20260507 04:13:18.426 | INFO | "huge_shared" : " 27 GB", | 20260507 04:13:18.426 | INFO | "malloc" : " 14 GB", | 20260507 04:13:18.426 | INFO | "pinned" : " 29 GB" | 20260507 04:13:18.426 | INFO | } | 20260507 04:13:18.426 | INFO | Perf report for the last 5.13923 seconds: | 20260507 04:13:18.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:13:18.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:18.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:18.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:18.426 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:13:18.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:18.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:18.426 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 04:13:18.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:18.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:18.427 | INFO | Uploaded 36GiB to GPU in 1.97972s (18.1844GiB/s) | 20260507 04:13:18.427 | INFO | Downloaded 34GiB to GPU in 1.9803s (17.1691GiB/s) | 20260507 04:13:18.427 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:13:18.427 | INFO | Total mem usage: 18 GiB | 20260507 04:13:18.427 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:13:18.427 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.6%) | 20260507 04:13:18.427 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:13:18.427 | INFO | High Water Breakdown: | 20260507 04:13:18.427 | INFO | 60.2% : 8.64 GiB ( 75.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:13:18.427 | INFO | 39.6% : 5.68 GiB ( 84.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:13:18.427 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:13:18.427 | INFO | ------------------------------------------ | 20260507 04:13:18.427 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:13:18.427 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:13:18.427 | INFO | Total mem usage: 26.5 GiB | 20260507 04:13:18.427 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:13:18.427 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:13:18.427 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:13:18.427 | INFO | High Water Breakdown: | 20260507 04:13:18.427 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:13:18.427 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:13:18.427 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:13:18.427 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:13:18.427 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:13:18.427 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:13:18.427 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:13:18.427 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:13:18.427 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:13:18.427 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:13:18.427 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:13:18.427 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:13:18.427 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:13:18.427 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:13:18.427 | INFO | ------------------------------------------ | 20260507 04:13:18.427 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:13:18.427 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:13:18.427 | INFO | Total mem usage: 25.6 GiB | 20260507 04:13:18.427 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:13:18.427 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:13:18.427 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:13:18.427 | INFO | High Water Breakdown: | 20260507 04:13:18.427 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:13:18.427 | INFO | ------------------------------------------ | 20260507 04:13:18.427 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:13:18.427 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:13:18.428 | INFO | Total mem usage: 15.7 GiB | 20260507 04:13:18.428 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:13:18.428 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:13:18.428 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:13:18.428 | INFO | High Water Breakdown: | 20260507 04:13:18.428 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:13:18.428 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:13:18.428 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:13:18.428 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:13:18.428 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:13:18.428 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:13:18.428 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:13:18.428 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:13:18.428 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:13:18.428 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:13:18.428 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:13:18.428 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:13:18.428 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:13:18.428 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:13:18.428 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:13:18.428 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:13:18.428 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:13:18.428 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:13:18.428 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:13:18.428 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:13:18.428 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:13:18.428 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:13:18.428 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:13:18.428 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:13:18.428 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:13:18.428 | INFO | ------------------------------------------ | 20260507 04:13:18.428 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:13:18.428 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:13:18.428 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:13:18.428 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:13:18.428 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.34e+01 / 9.42e-01 | 20260507 04:13:18.428 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.02e-01, 9.55e-03 / 4.99e-03 | 20260507 04:13:18.428 | INFO | PerfReport: 11.64%, Baseline -- 3.58e+00 / 4.46e+00, 5.03e-01 / 2.18e-01 | 20260507 04:13:18.428 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.48e+00, 1.68e-01 / 9.44e-02 | 20260507 04:13:18.428 | INFO | PerfReport: 16.12%, DME -- 3.96e+01 / 4.41e+01, 3.67e+00 / 1.43e+00 | 20260507 04:13:18.428 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.33e+00 / 1.73e+00 | 20260507 04:13:18.428 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.14e+00 / 2.66e+00, 3.80e-01 / 2.45e-01 | 20260507 04:13:18.428 | INFO | PerfReport: 3.03%, Metrics -- 9.32e-01 / 1.24e+00, 2.45e-01 / 1.20e-01 | 20260507 04:13:18.428 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.15e-01 / 3.65e-01, 4.96e-02 / 2.11e-02 | 20260507 04:13:18.428 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:13:18.428 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:13:18.428 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:13:18.428 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 04:13:18.428 | INFO | PerfReport: 21.83%, Upload -- 2.10e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260507 04:13:18.428 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260507 04:13:18.428 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 04:13:18.428 | INFO | PerfReport: 1.56%, LockUpload -- 1.50e+00 / 0.00e+00, 4.87e+00 / 0.00e+00 | 20260507 04:13:18.428 | INFO | PerfReport: 1.16%, LockDownload -- 1.11e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260507 04:13:18.428 | INFO | PerfReport: 22.12%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 04:13:18.428 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.58e-02 / 0.00e+00 | 20260507 04:13:18.523 | INFO | Starting a round of Baz IO | 20260507 04:13:21.624 | INFO | Populating RTMetrics report | 20260507 04:13:21.625 | INFO | RTMetrics report populated | 20260507 04:13:21.625 | INFO | Opening tmp RTMetrics json file | 20260507 04:13:21.643 | INFO | Serializing RTMetrics to json | 20260507 04:13:21.644 | INFO | Serialization done, swapping tmp file now... | 20260507 04:13:21.645 | INFO | RTMetrics done | 20260507 04:13:21.713 | INFO | Finished a round of Baz IO: Wrote 1.735 GiB in 3.189 seconds | 20260507 04:13:21.713 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.81% / 0.00% / 67.19% | 20260507 04:13:21.713 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.177848% / 0% / 7.54992e-05% | 20260507 04:13:21.713 | INFO | Baz IO overall: | 20260507 04:13:21.713 | INFO | BytesWritten : 3579487077671 | 20260507 04:13:21.713 | INFO | NumEvents : 1718786023070 | 20260507 04:13:23.569 | INFO | Analyzing chunk frames = [7004160,7004672) | 20260507 04:13:23.569 | INFO | MemInfo: { | 20260507 04:13:23.569 | INFO | "gpu" : " 17 GB", | 20260507 04:13:23.569 | INFO | "huge" : "", | 20260507 04:13:23.569 | INFO | "huge_shared" : " 27 GB", | 20260507 04:13:23.569 | INFO | "malloc" : " 14 GB", | 20260507 04:13:23.569 | INFO | "pinned" : " 29 GB" | 20260507 04:13:23.569 | INFO | } | 20260507 04:13:23.569 | INFO | Perf report for the last 5.14246 seconds: | 20260507 04:13:23.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:13:23.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:23.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:23.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:23.569 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 04:13:23.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:23.569 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:13:23.569 | INFO | PRE_HQ : 58.9%, 1.8, 54.9ms, 0 | 20260507 04:13:23.569 | INFO | BAZWRITER : 61.6%, 3.2, 102.9ms, 0 | 20260507 04:13:23.569 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:13:23.569 | INFO | Uploaded 36GiB to GPU in 2.04473s (17.6062GiB/s) | 20260507 04:13:23.569 | INFO | Downloaded 38GiB to GPU in 2.17166s (17.4981GiB/s) | 20260507 04:13:28.703 | INFO | Analyzing chunk frames = [7004672,7005184) | 20260507 04:13:28.704 | INFO | MemInfo: { | 20260507 04:13:28.704 | INFO | "gpu" : " 17 GB", | 20260507 04:13:28.704 | INFO | "huge" : "", | 20260507 04:13:28.704 | INFO | "huge_shared" : " 27 GB", | 20260507 04:13:28.704 | INFO | "malloc" : " 14 GB", | 20260507 04:13:28.704 | INFO | "pinned" : " 29 GB" | 20260507 04:13:28.704 | INFO | } | 20260507 04:13:28.704 | INFO | Perf report for the last 5.13499 seconds: | 20260507 04:13:28.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:13:28.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:28.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:28.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:28.704 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:13:28.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:28.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:28.704 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:13:28.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:28.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:28.704 | INFO | Uploaded 36GiB to GPU in 1.99805s (18.0176GiB/s) | 20260507 04:13:28.704 | INFO | Downloaded 34GiB to GPU in 1.92591s (17.654GiB/s) | 20260507 04:13:33.838 | INFO | Analyzing chunk frames = [7005184,7005696) | 20260507 04:13:33.838 | INFO | MemInfo: { | 20260507 04:13:33.838 | INFO | "gpu" : " 17 GB", | 20260507 04:13:33.838 | INFO | "huge" : "", | 20260507 04:13:33.838 | INFO | "huge_shared" : " 27 GB", | 20260507 04:13:33.838 | INFO | "malloc" : " 14 GB", | 20260507 04:13:33.838 | INFO | "pinned" : " 29 GB" | 20260507 04:13:33.838 | INFO | } | 20260507 04:13:33.839 | INFO | Perf report for the last 5.13473 seconds: | 20260507 04:13:33.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:13:33.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:33.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:33.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:33.839 | INFO | ANALYSIS : 60.3%, 3.0, 93.2ms, 0 | 20260507 04:13:33.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:33.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:33.839 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:13:33.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:33.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:33.839 | INFO | Uploaded 36GiB to GPU in 2.01349s (17.8794GiB/s) | 20260507 04:13:33.839 | INFO | Downloaded 34GiB to GPU in 1.94015s (17.5244GiB/s) | 20260507 04:13:38.981 | INFO | Analyzing chunk frames = [7005696,7006208) | 20260507 04:13:38.981 | INFO | MemInfo: { | 20260507 04:13:38.981 | INFO | "gpu" : " 17 GB", | 20260507 04:13:38.981 | INFO | "huge" : "", | 20260507 04:13:38.981 | INFO | "huge_shared" : " 27 GB", | 20260507 04:13:38.981 | INFO | "malloc" : " 14 GB", | 20260507 04:13:38.981 | INFO | "pinned" : " 29 GB" | 20260507 04:13:38.981 | INFO | } | 20260507 04:13:38.981 | INFO | Perf report for the last 5.14286 seconds: | 20260507 04:13:38.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:13:38.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:38.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:38.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:38.982 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:13:38.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:38.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:38.982 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:13:38.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:38.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:38.982 | INFO | Uploaded 36GiB to GPU in 1.97249s (18.251GiB/s) | 20260507 04:13:38.982 | INFO | Downloaded 34GiB to GPU in 1.90782s (17.8214GiB/s) | 20260507 04:13:44.118 | INFO | Analyzing chunk frames = [7006208,7006720) | 20260507 04:13:44.118 | INFO | MemInfo: { | 20260507 04:13:44.118 | INFO | "gpu" : " 17 GB", | 20260507 04:13:44.118 | INFO | "huge" : "", | 20260507 04:13:44.118 | INFO | "huge_shared" : " 27 GB", | 20260507 04:13:44.118 | INFO | "malloc" : " 14 GB", | 20260507 04:13:44.118 | INFO | "pinned" : " 29 GB" | 20260507 04:13:44.118 | INFO | } | 20260507 04:13:44.118 | INFO | Perf report for the last 5.13689 seconds: | 20260507 04:13:44.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:13:44.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:44.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:44.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:44.118 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 04:13:44.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:44.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:44.118 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:13:44.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:44.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:44.119 | INFO | Uploaded 36GiB to GPU in 2.03476s (17.6925GiB/s) | 20260507 04:13:44.119 | INFO | Downloaded 34GiB to GPU in 1.93797s (17.5441GiB/s) | 20260507 04:13:49.260 | INFO | Analyzing chunk frames = [7006720,7007232) | 20260507 04:13:49.260 | INFO | MemInfo: { | 20260507 04:13:49.260 | INFO | "gpu" : " 17 GB", | 20260507 04:13:49.260 | INFO | "huge" : "", | 20260507 04:13:49.260 | INFO | "huge_shared" : " 27 GB", | 20260507 04:13:49.260 | INFO | "malloc" : " 14 GB", | 20260507 04:13:49.260 | INFO | "pinned" : " 29 GB" | 20260507 04:13:49.260 | INFO | } | 20260507 04:13:49.260 | INFO | Perf report for the last 5.14206 seconds: | 20260507 04:13:49.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:13:49.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:49.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:49.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:49.260 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 04:13:49.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:49.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:49.261 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 04:13:49.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:49.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:49.261 | INFO | Uploaded 36GiB to GPU in 2.04186s (17.6309GiB/s) | 20260507 04:13:49.261 | INFO | Downloaded 34GiB to GPU in 1.91648s (17.7408GiB/s) | 20260507 04:13:54.402 | INFO | Analyzing chunk frames = [7007232,7007744) | 20260507 04:13:54.402 | INFO | MemInfo: { | 20260507 04:13:54.402 | INFO | "gpu" : " 17 GB", | 20260507 04:13:54.402 | INFO | "huge" : "", | 20260507 04:13:54.402 | INFO | "huge_shared" : " 27 GB", | 20260507 04:13:54.402 | INFO | "malloc" : " 14 GB", | 20260507 04:13:54.402 | INFO | "pinned" : " 29 GB" | 20260507 04:13:54.402 | INFO | } | 20260507 04:13:54.402 | INFO | Perf report for the last 5.14169 seconds: | 20260507 04:13:54.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:13:54.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:54.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:54.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:54.402 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:13:54.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:54.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:54.402 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 04:13:54.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:54.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:54.402 | INFO | Uploaded 36GiB to GPU in 1.96685s (18.3034GiB/s) | 20260507 04:13:54.402 | INFO | Downloaded 34GiB to GPU in 1.97588s (17.2075GiB/s) | 20260507 04:13:59.547 | INFO | Analyzing chunk frames = [7007744,7008256) | 20260507 04:13:59.547 | INFO | MemInfo: { | 20260507 04:13:59.547 | INFO | "gpu" : " 17 GB", | 20260507 04:13:59.547 | INFO | "huge" : "", | 20260507 04:13:59.547 | INFO | "huge_shared" : " 27 GB", | 20260507 04:13:59.548 | INFO | "malloc" : " 14 GB", | 20260507 04:13:59.548 | INFO | "pinned" : " 29 GB" | 20260507 04:13:59.548 | INFO | } | 20260507 04:13:59.548 | INFO | Perf report for the last 5.14551 seconds: | 20260507 04:13:59.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:13:59.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:59.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:59.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:59.548 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:13:59.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:59.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:59.548 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260507 04:13:59.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:59.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:13:59.548 | INFO | Uploaded 36GiB to GPU in 1.96631s (18.3084GiB/s) | 20260507 04:13:59.548 | INFO | Downloaded 34GiB to GPU in 1.95638s (17.379GiB/s) | 20260507 04:13:59.645 | INFO | Starting a round of Baz IO | 20260507 04:14:02.768 | INFO | Populating RTMetrics report | 20260507 04:14:02.768 | INFO | RTMetrics report populated | 20260507 04:14:02.768 | INFO | Opening tmp RTMetrics json file | 20260507 04:14:02.769 | INFO | Serializing RTMetrics to json | 20260507 04:14:02.770 | INFO | Serialization done, swapping tmp file now... | 20260507 04:14:02.771 | INFO | RTMetrics done | 20260507 04:14:02.856 | INFO | Finished a round of Baz IO: Wrote 1.733 GiB in 3.21 seconds | 20260507 04:14:02.856 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.81% / 0.00% / 67.19% | 20260507 04:14:02.856 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.176593% / 0% / 7.94729e-05% | 20260507 04:14:02.856 | INFO | Baz IO overall: | 20260507 04:14:02.856 | INFO | BytesWritten : 3581348079880 | 20260507 04:14:02.856 | INFO | NumEvents : 1719661846431 | 20260507 04:14:04.683 | INFO | Analyzing chunk frames = [7008256,7008768) | 20260507 04:14:04.683 | INFO | MemInfo: { | 20260507 04:14:04.683 | INFO | "gpu" : " 17 GB", | 20260507 04:14:04.683 | INFO | "huge" : "", | 20260507 04:14:04.683 | INFO | "huge_shared" : " 27 GB", | 20260507 04:14:04.683 | INFO | "malloc" : " 14 GB", | 20260507 04:14:04.683 | INFO | "pinned" : " 29 GB" | 20260507 04:14:04.683 | INFO | } | 20260507 04:14:04.684 | INFO | Perf report for the last 5.13585 seconds: | 20260507 04:14:04.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:14:04.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:04.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:04.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:04.684 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260507 04:14:04.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:04.684 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260507 04:14:04.684 | INFO | PRE_HQ : 59.8%, 1.9, 58.9ms, 0 | 20260507 04:14:04.684 | INFO | BAZWRITER : 62.1%, 3.3, 105.4ms, 0 | 20260507 04:14:04.684 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260507 04:14:04.684 | INFO | Uploaded 36GiB to GPU in 2.04055s (17.6423GiB/s) | 20260507 04:14:04.684 | INFO | Downloaded 38GiB to GPU in 2.18239s (17.4121GiB/s) | 20260507 04:14:09.817 | INFO | Analyzing chunk frames = [7008768,7009280) | 20260507 04:14:09.817 | INFO | MemInfo: { | 20260507 04:14:09.817 | INFO | "gpu" : " 17 GB", | 20260507 04:14:09.817 | INFO | "huge" : "", | 20260507 04:14:09.817 | INFO | "huge_shared" : " 27 GB", | 20260507 04:14:09.817 | INFO | "malloc" : " 14 GB", | 20260507 04:14:09.817 | INFO | "pinned" : " 29 GB" | 20260507 04:14:09.817 | INFO | } | 20260507 04:14:09.817 | INFO | Perf report for the last 5.13349 seconds: | 20260507 04:14:09.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:14:09.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:09.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:09.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:09.817 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:14:09.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:09.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:09.817 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:14:09.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:09.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:09.817 | INFO | Uploaded 36GiB to GPU in 1.97782s (18.2019GiB/s) | 20260507 04:14:09.817 | INFO | Downloaded 34GiB to GPU in 1.92911s (17.6247GiB/s) | 20260507 04:14:14.954 | INFO | Analyzing chunk frames = [7009280,7009792) | 20260507 04:14:14.954 | INFO | MemInfo: { | 20260507 04:14:14.954 | INFO | "gpu" : " 17 GB", | 20260507 04:14:14.954 | INFO | "huge" : "", | 20260507 04:14:14.954 | INFO | "huge_shared" : " 27 GB", | 20260507 04:14:14.955 | INFO | "malloc" : " 14 GB", | 20260507 04:14:14.955 | INFO | "pinned" : " 29 GB" | 20260507 04:14:14.955 | INFO | } | 20260507 04:14:14.955 | INFO | Perf report for the last 5.13777 seconds: | 20260507 04:14:14.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:14:14.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:14.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:14.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:14.955 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 04:14:14.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:14.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:14.955 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:14:14.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:14.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:14.955 | INFO | Uploaded 36GiB to GPU in 1.99774s (18.0204GiB/s) | 20260507 04:14:14.955 | INFO | Downloaded 34GiB to GPU in 1.94275s (17.501GiB/s) | 20260507 04:14:20.088 | INFO | Analyzing chunk frames = [7009792,7010304) | 20260507 04:14:20.088 | INFO | MemInfo: { | 20260507 04:14:20.088 | INFO | "gpu" : " 17 GB", | 20260507 04:14:20.088 | INFO | "huge" : "", | 20260507 04:14:20.088 | INFO | "huge_shared" : " 27 GB", | 20260507 04:14:20.088 | INFO | "malloc" : " 14 GB", | 20260507 04:14:20.088 | INFO | "pinned" : " 29 GB" | 20260507 04:14:20.088 | INFO | } | 20260507 04:14:20.088 | INFO | Perf report for the last 5.13349 seconds: | 20260507 04:14:20.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:14:20.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:20.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:20.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:20.088 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 04:14:20.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:20.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:20.088 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:14:20.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:20.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:20.089 | INFO | Uploaded 36GiB to GPU in 1.96001s (18.3673GiB/s) | 20260507 04:14:20.089 | INFO | Downloaded 34GiB to GPU in 1.92499s (17.6625GiB/s) | 20260507 04:14:25.235 | INFO | Analyzing chunk frames = [7010304,7010816) | 20260507 04:14:25.235 | INFO | MemInfo: { | 20260507 04:14:25.235 | INFO | "gpu" : " 17 GB", | 20260507 04:14:25.235 | INFO | "huge" : "", | 20260507 04:14:25.235 | INFO | "huge_shared" : " 27 GB", | 20260507 04:14:25.235 | INFO | "malloc" : " 14 GB", | 20260507 04:14:25.235 | INFO | "pinned" : " 29 GB" | 20260507 04:14:25.235 | INFO | } | 20260507 04:14:25.236 | INFO | Perf report for the last 5.14725 seconds: | 20260507 04:14:25.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:14:25.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:25.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:25.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:25.236 | INFO | ANALYSIS : 59.8%, 3.0, 93.2ms, 0 | 20260507 04:14:25.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:25.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:25.236 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:14:25.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:25.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:25.236 | INFO | Uploaded 36GiB to GPU in 2.01s (17.9104GiB/s) | 20260507 04:14:25.236 | INFO | Downloaded 34GiB to GPU in 1.9529s (17.41GiB/s) | 20260507 04:14:30.381 | INFO | Analyzing chunk frames = [7010816,7011328) | 20260507 04:14:30.381 | INFO | MemInfo: { | 20260507 04:14:30.381 | INFO | "gpu" : " 17 GB", | 20260507 04:14:30.381 | INFO | "huge" : "", | 20260507 04:14:30.381 | INFO | "huge_shared" : " 27 GB", | 20260507 04:14:30.381 | INFO | "malloc" : " 14 GB", | 20260507 04:14:30.381 | INFO | "pinned" : " 29 GB" | 20260507 04:14:30.381 | INFO | } | 20260507 04:14:30.381 | INFO | Perf report for the last 5.14577 seconds: | 20260507 04:14:30.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:14:30.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:30.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:30.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:30.381 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 04:14:30.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:30.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:30.381 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:14:30.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:30.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:30.382 | INFO | Uploaded 36GiB to GPU in 2.05088s (17.5534GiB/s) | 20260507 04:14:30.382 | INFO | Downloaded 34GiB to GPU in 1.96563s (17.2973GiB/s) | 20260507 04:14:35.516 | INFO | Analyzing chunk frames = [7011328,7011840) | 20260507 04:14:35.516 | INFO | MemInfo: { | 20260507 04:14:35.516 | INFO | "gpu" : " 17 GB", | 20260507 04:14:35.516 | INFO | "huge" : "", | 20260507 04:14:35.516 | INFO | "huge_shared" : " 27 GB", | 20260507 04:14:35.516 | INFO | "malloc" : " 14 GB", | 20260507 04:14:35.516 | INFO | "pinned" : " 29 GB" | 20260507 04:14:35.517 | INFO | } | 20260507 04:14:35.517 | INFO | Perf report for the last 5.13522 seconds: | 20260507 04:14:35.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:14:35.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:35.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:35.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:35.517 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 04:14:35.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:35.517 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:35.517 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 04:14:35.517 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:35.517 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:35.517 | INFO | Uploaded 36GiB to GPU in 2.00529s (17.9526GiB/s) | 20260507 04:14:35.517 | INFO | Downloaded 34GiB to GPU in 1.98999s (17.0855GiB/s) | 20260507 04:14:40.653 | INFO | Analyzing chunk frames = [7011840,7012352) | 20260507 04:14:40.653 | INFO | MemInfo: { | 20260507 04:14:40.653 | INFO | "gpu" : " 17 GB", | 20260507 04:14:40.653 | INFO | "huge" : "", | 20260507 04:14:40.653 | INFO | "huge_shared" : " 27 GB", | 20260507 04:14:40.653 | INFO | "malloc" : " 14 GB", | 20260507 04:14:40.653 | INFO | "pinned" : " 29 GB" | 20260507 04:14:40.653 | INFO | } | 20260507 04:14:40.653 | INFO | Perf report for the last 5.13629 seconds: | 20260507 04:14:40.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:14:40.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:40.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:40.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:40.653 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:14:40.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:40.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:40.653 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 04:14:40.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:40.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:40.653 | INFO | Uploaded 36GiB to GPU in 1.98195s (18.1639GiB/s) | 20260507 04:14:40.653 | INFO | Downloaded 34GiB to GPU in 1.95712s (17.3724GiB/s) | 20260507 04:14:40.653 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:14:40.653 | INFO | Total mem usage: 18 GiB | 20260507 04:14:40.653 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:14:40.653 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 04:14:40.653 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:14:40.653 | INFO | High Water Breakdown: | 20260507 04:14:40.653 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:14:40.653 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:14:40.653 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:14:40.653 | INFO | ------------------------------------------ | 20260507 04:14:40.653 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:14:40.653 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:14:40.653 | INFO | Total mem usage: 26.5 GiB | 20260507 04:14:40.653 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:14:40.653 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 04:14:40.653 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 04:14:40.653 | INFO | High Water Breakdown: | 20260507 04:14:40.653 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:14:40.653 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:14:40.653 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:14:40.653 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:14:40.653 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:14:40.654 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:14:40.654 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:14:40.654 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:14:40.654 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:14:40.654 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:14:40.654 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:14:40.654 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:14:40.654 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:14:40.654 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:14:40.654 | INFO | ------------------------------------------ | 20260507 04:14:40.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:14:40.654 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:14:40.654 | INFO | Total mem usage: 25.6 GiB | 20260507 04:14:40.654 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:14:40.654 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:14:40.654 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:14:40.654 | INFO | High Water Breakdown: | 20260507 04:14:40.654 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:14:40.654 | INFO | ------------------------------------------ | 20260507 04:14:40.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:14:40.654 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:14:40.654 | INFO | Total mem usage: 15.7 GiB | 20260507 04:14:40.654 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:14:40.654 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:14:40.654 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:14:40.654 | INFO | High Water Breakdown: | 20260507 04:14:40.654 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:14:40.654 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:14:40.654 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:14:40.654 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:14:40.654 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:14:40.654 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:14:40.654 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:14:40.654 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:14:40.654 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:14:40.654 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:14:40.654 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:14:40.654 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:14:40.654 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:14:40.654 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:14:40.654 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:14:40.654 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:14:40.654 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:14:40.654 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:14:40.654 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:14:40.654 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:14:40.654 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:14:40.654 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:14:40.654 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:14:40.654 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:14:40.654 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:14:40.654 | INFO | ------------------------------------------ | 20260507 04:14:40.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:14:40.654 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:14:40.654 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:14:40.654 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:14:40.654 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.47e+01, 1.32e+01 / 1.19e+00 | 20260507 04:14:40.654 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 1.01e-01, 1.07e-02 / 5.61e-03 | 20260507 04:14:40.654 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.42e+00, 5.23e-01 / 3.08e-01 | 20260507 04:14:40.654 | INFO | PerfReport: 3.24%, Histogram -- 9.97e-01 / 1.43e+00, 1.70e-01 / 1.16e-01 | 20260507 04:14:40.654 | INFO | PerfReport: 16.06%, DME -- 3.95e+01 / 4.37e+01, 3.72e+00 / 1.61e+00 | 20260507 04:14:40.654 | INFO | PerfReport: 57.76%, FrameLabeling -- 1.78e+01 / 2.08e+01, 1.38e+00 / 1.63e+00 | 20260507 04:14:40.654 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.64e+00, 3.77e-01 / 2.44e-01 | 20260507 04:14:40.654 | INFO | PerfReport: 3.02%, Metrics -- 9.29e-01 / 1.22e+00, 2.46e-01 / 1.28e-01 | 20260507 04:14:40.654 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.15e-01 / 3.62e-01, 4.92e-02 / 2.04e-02 | 20260507 04:14:40.654 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:14:40.654 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:14:40.654 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:14:40.654 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 04:14:40.654 | INFO | PerfReport: 21.73%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260507 04:14:40.654 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260507 04:14:40.654 | INFO | PerfReport: 32.03%, Compute -- 3.08e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 04:14:40.654 | INFO | PerfReport: 1.34%, LockUpload -- 1.28e+00 / 0.00e+00, 4.62e+00 / 0.00e+00 | 20260507 04:14:40.654 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.10e+00 / 0.00e+00 | 20260507 04:14:40.654 | INFO | PerfReport: 22.17%, LockCompute -- 2.13e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 04:14:40.654 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 04:14:40.750 | INFO | Starting a round of Baz IO | 20260507 04:14:43.857 | INFO | Populating RTMetrics report | 20260507 04:14:43.857 | INFO | RTMetrics report populated | 20260507 04:14:43.857 | INFO | Opening tmp RTMetrics json file | 20260507 04:14:43.876 | INFO | Serializing RTMetrics to json | 20260507 04:14:43.877 | INFO | Serialization done, swapping tmp file now... | 20260507 04:14:43.877 | INFO | RTMetrics done | 20260507 04:14:43.947 | INFO | Finished a round of Baz IO: Wrote 1.731 GiB in 3.196 seconds | 20260507 04:14:43.947 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.81% / 0.00% / 67.19% | 20260507 04:14:43.947 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.176056% / 0% / 7.15256e-05% | 20260507 04:14:43.947 | INFO | Baz IO overall: | 20260507 04:14:43.947 | INFO | BytesWritten : 3583206754303 | 20260507 04:14:43.947 | INFO | NumEvents : 1720536743580 | 20260507 04:14:45.789 | INFO | Analyzing chunk frames = [7012352,7012864) | 20260507 04:14:45.789 | INFO | MemInfo: { | 20260507 04:14:45.789 | INFO | "gpu" : " 17 GB", | 20260507 04:14:45.789 | INFO | "huge" : "", | 20260507 04:14:45.789 | INFO | "huge_shared" : " 27 GB", | 20260507 04:14:45.789 | INFO | "malloc" : " 14 GB", | 20260507 04:14:45.789 | INFO | "pinned" : " 29 GB" | 20260507 04:14:45.789 | INFO | } | 20260507 04:14:45.789 | INFO | Perf report for the last 5.13618 seconds: | 20260507 04:14:45.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:14:45.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:45.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:45.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:45.789 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260507 04:14:45.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:45.789 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260507 04:14:45.789 | INFO | PRE_HQ : 59.9%, 1.9, 59.3ms, 0 | 20260507 04:14:45.789 | INFO | BAZWRITER : 61.8%, 3.4, 108.0ms, 0 | 20260507 04:14:45.789 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 04:14:45.789 | INFO | Uploaded 36GiB to GPU in 2.03552s (17.6859GiB/s) | 20260507 04:14:45.789 | INFO | Downloaded 38GiB to GPU in 2.1832s (17.4056GiB/s) | 20260507 04:14:50.931 | INFO | Analyzing chunk frames = [7012864,7013376) | 20260507 04:14:50.931 | INFO | MemInfo: { | 20260507 04:14:50.931 | INFO | "gpu" : " 17 GB", | 20260507 04:14:50.931 | INFO | "huge" : "", | 20260507 04:14:50.931 | INFO | "huge_shared" : " 27 GB", | 20260507 04:14:50.931 | INFO | "malloc" : " 14 GB", | 20260507 04:14:50.932 | INFO | "pinned" : " 29 GB" | 20260507 04:14:50.932 | INFO | } | 20260507 04:14:50.932 | INFO | Perf report for the last 5.14271 seconds: | 20260507 04:14:50.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:14:50.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:50.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:50.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:50.932 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 04:14:50.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:50.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:50.932 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 04:14:50.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:50.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:50.932 | INFO | Uploaded 36GiB to GPU in 2.02735s (17.7572GiB/s) | 20260507 04:14:50.932 | INFO | Downloaded 34GiB to GPU in 1.9413s (17.514GiB/s) | 20260507 04:14:56.073 | INFO | Analyzing chunk frames = [7013376,7013888) | 20260507 04:14:56.073 | INFO | MemInfo: { | 20260507 04:14:56.073 | INFO | "gpu" : " 17 GB", | 20260507 04:14:56.073 | INFO | "huge" : "", | 20260507 04:14:56.073 | INFO | "huge_shared" : " 27 GB", | 20260507 04:14:56.074 | INFO | "malloc" : " 14 GB", | 20260507 04:14:56.074 | INFO | "pinned" : " 29 GB" | 20260507 04:14:56.074 | INFO | } | 20260507 04:14:56.074 | INFO | Perf report for the last 5.14182 seconds: | 20260507 04:14:56.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:14:56.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:56.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:56.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:56.074 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 04:14:56.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:56.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:56.074 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 04:14:56.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:56.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:14:56.074 | INFO | Uploaded 36GiB to GPU in 1.98658s (18.1216GiB/s) | 20260507 04:14:56.074 | INFO | Downloaded 34GiB to GPU in 1.94425s (17.4875GiB/s) | 20260507 04:15:01.210 | INFO | Analyzing chunk frames = [7013888,7014400) | 20260507 04:15:01.210 | INFO | MemInfo: { | 20260507 04:15:01.210 | INFO | "gpu" : " 17 GB", | 20260507 04:15:01.210 | INFO | "huge" : "", | 20260507 04:15:01.210 | INFO | "huge_shared" : " 27 GB", | 20260507 04:15:01.210 | INFO | "malloc" : " 14 GB", | 20260507 04:15:01.210 | INFO | "pinned" : " 29 GB" | 20260507 04:15:01.210 | INFO | } | 20260507 04:15:01.210 | INFO | Perf report for the last 5.1364 seconds: | 20260507 04:15:01.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:15:01.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:01.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:01.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:01.210 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 04:15:01.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:01.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:01.210 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:15:01.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:01.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:01.210 | INFO | Uploaded 36GiB to GPU in 2.01039s (17.907GiB/s) | 20260507 04:15:01.210 | INFO | Downloaded 34GiB to GPU in 1.94486s (17.482GiB/s) | 20260507 04:15:06.349 | INFO | Analyzing chunk frames = [7014400,7014912) | 20260507 04:15:06.349 | INFO | MemInfo: { | 20260507 04:15:06.349 | INFO | "gpu" : " 17 GB", | 20260507 04:15:06.349 | INFO | "huge" : "", | 20260507 04:15:06.349 | INFO | "huge_shared" : " 27 GB", | 20260507 04:15:06.349 | INFO | "malloc" : " 14 GB", | 20260507 04:15:06.349 | INFO | "pinned" : " 29 GB" | 20260507 04:15:06.349 | INFO | } | 20260507 04:15:06.349 | INFO | Perf report for the last 5.13935 seconds: | 20260507 04:15:06.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:15:06.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:06.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:06.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:06.349 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 04:15:06.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:06.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:06.349 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:15:06.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:06.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:06.350 | INFO | Uploaded 36GiB to GPU in 2.01905s (17.8302GiB/s) | 20260507 04:15:06.350 | INFO | Downloaded 34GiB to GPU in 1.93809s (17.5431GiB/s) | 20260507 04:15:11.492 | INFO | Analyzing chunk frames = [7014912,7015424) | 20260507 04:15:11.492 | INFO | MemInfo: { | 20260507 04:15:11.492 | INFO | "gpu" : " 17 GB", | 20260507 04:15:11.492 | INFO | "huge" : "", | 20260507 04:15:11.492 | INFO | "huge_shared" : " 27 GB", | 20260507 04:15:11.492 | INFO | "malloc" : " 14 GB", | 20260507 04:15:11.492 | INFO | "pinned" : " 29 GB" | 20260507 04:15:11.492 | INFO | } | 20260507 04:15:11.493 | INFO | Perf report for the last 5.14315 seconds: | 20260507 04:15:11.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:15:11.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:11.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:11.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:11.493 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 04:15:11.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:11.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:11.493 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 04:15:11.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:11.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:11.493 | INFO | Uploaded 36GiB to GPU in 2.04156s (17.6336GiB/s) | 20260507 04:15:11.493 | INFO | Downloaded 34GiB to GPU in 1.91575s (17.7476GiB/s) | 20260507 04:15:16.633 | INFO | Analyzing chunk frames = [7015424,7015936) | 20260507 04:15:16.633 | INFO | MemInfo: { | 20260507 04:15:16.633 | INFO | "gpu" : " 17 GB", | 20260507 04:15:16.633 | INFO | "huge" : "", | 20260507 04:15:16.633 | INFO | "huge_shared" : " 27 GB", | 20260507 04:15:16.633 | INFO | "malloc" : " 14 GB", | 20260507 04:15:16.633 | INFO | "pinned" : " 29 GB" | 20260507 04:15:16.633 | INFO | } | 20260507 04:15:16.633 | INFO | Perf report for the last 5.14044 seconds: | 20260507 04:15:16.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:15:16.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:16.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:16.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:16.633 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:15:16.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:16.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:16.633 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 04:15:16.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:16.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:16.633 | INFO | Uploaded 36GiB to GPU in 1.96973s (18.2766GiB/s) | 20260507 04:15:16.633 | INFO | Downloaded 34GiB to GPU in 1.985s (17.1284GiB/s) | 20260507 04:15:21.763 | INFO | Analyzing chunk frames = [7015936,7016448) | 20260507 04:15:21.763 | INFO | MemInfo: { | 20260507 04:15:21.763 | INFO | "gpu" : " 17 GB", | 20260507 04:15:21.763 | INFO | "huge" : "", | 20260507 04:15:21.763 | INFO | "huge_shared" : " 27 GB", | 20260507 04:15:21.763 | INFO | "malloc" : " 14 GB", | 20260507 04:15:21.764 | INFO | "pinned" : " 29 GB" | 20260507 04:15:21.764 | INFO | } | 20260507 04:15:21.764 | INFO | Perf report for the last 5.1306 seconds: | 20260507 04:15:21.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:15:21.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:21.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:21.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:21.764 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 04:15:21.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:21.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:21.764 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:15:21.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:21.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:21.764 | INFO | Uploaded 36GiB to GPU in 1.99512s (18.044GiB/s) | 20260507 04:15:21.764 | INFO | Downloaded 34GiB to GPU in 1.96549s (17.2985GiB/s) | 20260507 04:15:21.860 | INFO | Starting a round of Baz IO | 20260507 04:15:24.968 | INFO | Populating RTMetrics report | 20260507 04:15:24.968 | INFO | RTMetrics report populated | 20260507 04:15:24.968 | INFO | Opening tmp RTMetrics json file | 20260507 04:15:24.972 | INFO | Serializing RTMetrics to json | 20260507 04:15:24.986 | INFO | Serialization done, swapping tmp file now... | 20260507 04:15:24.986 | INFO | RTMetrics done | 20260507 04:15:25.056 | INFO | Finished a round of Baz IO: Wrote 1.731 GiB in 3.195 seconds | 20260507 04:15:25.056 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.81% / 0.00% / 67.19% | 20260507 04:15:25.056 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.176056% / 3.97364e-06% / 7.54992e-05% | 20260507 04:15:25.056 | INFO | Baz IO overall: | 20260507 04:15:25.056 | INFO | BytesWritten : 3585065710890 | 20260507 04:15:25.056 | INFO | NumEvents : 1721411732887 | 20260507 04:15:26.905 | INFO | Analyzing chunk frames = [7016448,7016960) | 20260507 04:15:26.905 | INFO | MemInfo: { | 20260507 04:15:26.905 | INFO | "gpu" : " 17 GB", | 20260507 04:15:26.905 | INFO | "huge" : "", | 20260507 04:15:26.905 | INFO | "huge_shared" : " 27 GB", | 20260507 04:15:26.905 | INFO | "malloc" : " 14 GB", | 20260507 04:15:26.905 | INFO | "pinned" : " 29 GB" | 20260507 04:15:26.905 | INFO | } | 20260507 04:15:26.905 | INFO | Perf report for the last 5.14184 seconds: | 20260507 04:15:26.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:15:26.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:26.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:26.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:26.906 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 04:15:26.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:26.906 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 04:15:26.906 | INFO | PRE_HQ : 59.2%, 1.8, 55.1ms, 0 | 20260507 04:15:26.906 | INFO | BAZWRITER : 61.7%, 3.2, 102.0ms, 0 | 20260507 04:15:26.906 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:15:26.906 | INFO | Uploaded 36GiB to GPU in 2.04752s (17.5823GiB/s) | 20260507 04:15:26.906 | INFO | Downloaded 38GiB to GPU in 2.18907s (17.3589GiB/s) | 20260507 04:15:32.047 | INFO | Analyzing chunk frames = [7016960,7017472) | 20260507 04:15:32.047 | INFO | MemInfo: { | 20260507 04:15:32.047 | INFO | "gpu" : " 17 GB", | 20260507 04:15:32.047 | INFO | "huge" : "", | 20260507 04:15:32.047 | INFO | "huge_shared" : " 27 GB", | 20260507 04:15:32.047 | INFO | "malloc" : " 14 GB", | 20260507 04:15:32.047 | INFO | "pinned" : " 29 GB" | 20260507 04:15:32.047 | INFO | } | 20260507 04:15:32.047 | INFO | Perf report for the last 5.14168 seconds: | 20260507 04:15:32.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:15:32.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:32.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:32.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:32.047 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 04:15:32.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:32.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:32.047 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:15:32.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:32.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:32.048 | INFO | Uploaded 36GiB to GPU in 2.04319s (17.6195GiB/s) | 20260507 04:15:32.048 | INFO | Downloaded 34GiB to GPU in 1.93926s (17.5324GiB/s) | 20260507 04:15:37.192 | INFO | Analyzing chunk frames = [7017472,7017984) | 20260507 04:15:37.192 | INFO | MemInfo: { | 20260507 04:15:37.192 | INFO | "gpu" : " 17 GB", | 20260507 04:15:37.192 | INFO | "huge" : "", | 20260507 04:15:37.192 | INFO | "huge_shared" : " 27 GB", | 20260507 04:15:37.192 | INFO | "malloc" : " 14 GB", | 20260507 04:15:37.192 | INFO | "pinned" : " 29 GB" | 20260507 04:15:37.192 | INFO | } | 20260507 04:15:37.192 | INFO | Perf report for the last 5.14519 seconds: | 20260507 04:15:37.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:15:37.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:37.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:37.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:37.192 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 04:15:37.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:37.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:37.192 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:15:37.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:37.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:37.193 | INFO | Uploaded 36GiB to GPU in 2.00068s (17.9939GiB/s) | 20260507 04:15:37.193 | INFO | Downloaded 34GiB to GPU in 1.939s (17.5348GiB/s) | 20260507 04:15:42.331 | INFO | Analyzing chunk frames = [7017984,7018496) | 20260507 04:15:42.331 | INFO | MemInfo: { | 20260507 04:15:42.331 | INFO | "gpu" : " 17 GB", | 20260507 04:15:42.331 | INFO | "huge" : "", | 20260507 04:15:42.331 | INFO | "huge_shared" : " 27 GB", | 20260507 04:15:42.331 | INFO | "malloc" : " 14 GB", | 20260507 04:15:42.331 | INFO | "pinned" : " 29 GB" | 20260507 04:15:42.331 | INFO | } | 20260507 04:15:42.331 | INFO | Perf report for the last 5.13869 seconds: | 20260507 04:15:42.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:15:42.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:42.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:42.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:42.331 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 04:15:42.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:42.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:42.331 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:15:42.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:42.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:42.331 | INFO | Uploaded 36GiB to GPU in 2.02027s (17.8194GiB/s) | 20260507 04:15:42.332 | INFO | Downloaded 34GiB to GPU in 1.92058s (17.703GiB/s) | 20260507 04:15:47.463 | INFO | Analyzing chunk frames = [7018496,7019008) | 20260507 04:15:47.463 | INFO | MemInfo: { | 20260507 04:15:47.463 | INFO | "gpu" : " 17 GB", | 20260507 04:15:47.463 | INFO | "huge" : "", | 20260507 04:15:47.463 | INFO | "huge_shared" : " 27 GB", | 20260507 04:15:47.464 | INFO | "malloc" : " 14 GB", | 20260507 04:15:47.464 | INFO | "pinned" : " 29 GB" | 20260507 04:15:47.464 | INFO | } | 20260507 04:15:47.464 | INFO | Perf report for the last 5.1327 seconds: | 20260507 04:15:47.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:15:47.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:47.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:47.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:47.464 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 04:15:47.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:47.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:47.464 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 04:15:47.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:47.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:47.464 | INFO | Uploaded 36GiB to GPU in 2.03609s (17.681GiB/s) | 20260507 04:15:47.464 | INFO | Downloaded 34GiB to GPU in 1.91854s (17.7218GiB/s) | 20260507 04:15:52.600 | INFO | Analyzing chunk frames = [7019008,7019520) | 20260507 04:15:52.600 | INFO | MemInfo: { | 20260507 04:15:52.600 | INFO | "gpu" : " 17 GB", | 20260507 04:15:52.600 | INFO | "huge" : "", | 20260507 04:15:52.600 | INFO | "huge_shared" : " 27 GB", | 20260507 04:15:52.600 | INFO | "malloc" : " 14 GB", | 20260507 04:15:52.600 | INFO | "pinned" : " 29 GB" | 20260507 04:15:52.600 | INFO | } | 20260507 04:15:52.600 | INFO | Perf report for the last 5.13653 seconds: | 20260507 04:15:52.600 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:15:52.600 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:52.600 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:52.600 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:52.600 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 04:15:52.600 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:52.600 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:52.600 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 04:15:52.600 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:52.600 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:52.601 | INFO | Uploaded 36GiB to GPU in 2.04925s (17.5674GiB/s) | 20260507 04:15:52.601 | INFO | Downloaded 34GiB to GPU in 1.91897s (17.7179GiB/s) | 20260507 04:15:57.743 | INFO | Analyzing chunk frames = [7019520,7020032) | 20260507 04:15:57.743 | INFO | MemInfo: { | 20260507 04:15:57.743 | INFO | "gpu" : " 17 GB", | 20260507 04:15:57.743 | INFO | "huge" : "", | 20260507 04:15:57.743 | INFO | "huge_shared" : " 27 GB", | 20260507 04:15:57.743 | INFO | "malloc" : " 14 GB", | 20260507 04:15:57.743 | INFO | "pinned" : " 29 GB" | 20260507 04:15:57.743 | INFO | } | 20260507 04:15:57.743 | INFO | Perf report for the last 5.14293 seconds: | 20260507 04:15:57.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:15:57.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:57.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:57.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:57.743 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:15:57.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:57.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:57.743 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:15:57.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:57.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:15:57.744 | INFO | Uploaded 36GiB to GPU in 1.98038s (18.1784GiB/s) | 20260507 04:15:57.744 | INFO | Downloaded 34GiB to GPU in 1.99759s (17.0205GiB/s) | 20260507 04:16:02.885 | INFO | Analyzing chunk frames = [7020032,7020544) | 20260507 04:16:02.885 | INFO | MemInfo: { | 20260507 04:16:02.885 | INFO | "gpu" : " 17 GB", | 20260507 04:16:02.885 | INFO | "huge" : "", | 20260507 04:16:02.885 | INFO | "huge_shared" : " 27 GB", | 20260507 04:16:02.885 | INFO | "malloc" : " 14 GB", | 20260507 04:16:02.885 | INFO | "pinned" : " 29 GB" | 20260507 04:16:02.885 | INFO | } | 20260507 04:16:02.885 | INFO | Perf report for the last 5.14203 seconds: | 20260507 04:16:02.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:16:02.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:02.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:02.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:02.885 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:16:02.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:02.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:02.885 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:16:02.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:02.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:02.885 | INFO | Uploaded 36GiB to GPU in 1.98682s (18.1194GiB/s) | 20260507 04:16:02.886 | INFO | Downloaded 34GiB to GPU in 1.96853s (17.2718GiB/s) | 20260507 04:16:02.886 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:16:02.886 | INFO | Total mem usage: 18 GiB | 20260507 04:16:02.886 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:16:02.886 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 04:16:02.886 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:16:02.886 | INFO | High Water Breakdown: | 20260507 04:16:02.886 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:16:02.886 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:16:02.886 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:16:02.886 | INFO | ------------------------------------------ | 20260507 04:16:02.886 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:16:02.886 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:16:02.886 | INFO | Total mem usage: 26.5 GiB | 20260507 04:16:02.886 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:16:02.886 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 04:16:02.886 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 04:16:02.886 | INFO | High Water Breakdown: | 20260507 04:16:02.886 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:16:02.886 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:16:02.886 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:16:02.886 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:16:02.886 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:16:02.886 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:16:02.886 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:16:02.886 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:16:02.886 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:16:02.886 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:16:02.886 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:16:02.886 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:16:02.886 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:16:02.886 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:16:02.886 | INFO | ------------------------------------------ | 20260507 04:16:02.886 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:16:02.886 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:16:02.886 | INFO | Total mem usage: 25.6 GiB | 20260507 04:16:02.886 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:16:02.886 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:16:02.886 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:16:02.886 | INFO | High Water Breakdown: | 20260507 04:16:02.886 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:16:02.886 | INFO | ------------------------------------------ | 20260507 04:16:02.886 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:16:02.886 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:16:02.886 | INFO | Total mem usage: 15.7 GiB | 20260507 04:16:02.886 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:16:02.886 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:16:02.886 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 04:16:02.886 | INFO | High Water Breakdown: | 20260507 04:16:02.886 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:16:02.886 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:16:02.886 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:16:02.886 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:16:02.886 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:16:02.886 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:16:02.887 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:16:02.887 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:16:02.887 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:16:02.887 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:16:02.887 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:16:02.887 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:16:02.887 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:16:02.887 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:16:02.887 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:16:02.887 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:16:02.887 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:16:02.887 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:16:02.887 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:16:02.887 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:16:02.887 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:16:02.887 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:16:02.887 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:16:02.887 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:16:02.887 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:16:02.887 | INFO | ------------------------------------------ | 20260507 04:16:02.887 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:16:02.887 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:16:02.887 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:16:02.887 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:16:02.887 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.33e+01 / 1.53e+00 | 20260507 04:16:02.887 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 9.97e-02, 9.42e-03 / 5.36e-03 | 20260507 04:16:02.887 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.45e+00, 5.10e-01 / 2.45e-01 | 20260507 04:16:02.887 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.45e+00, 1.68e-01 / 1.23e-01 | 20260507 04:16:02.887 | INFO | PerfReport: 16.10%, DME -- 3.96e+01 / 4.41e+01, 3.88e+00 / 1.72e+00 | 20260507 04:16:02.887 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.31e+00 / 1.67e+00 | 20260507 04:16:02.887 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.14e+00 / 2.70e+00, 3.84e-01 / 2.39e-01 | 20260507 04:16:02.887 | INFO | PerfReport: 3.03%, Metrics -- 9.32e-01 / 1.26e+00, 2.48e-01 / 1.46e-01 | 20260507 04:16:02.887 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.15e-01 / 3.65e-01, 4.98e-02 / 1.51e-02 | 20260507 04:16:02.887 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:16:02.887 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:16:02.887 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:16:02.887 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 04:16:02.887 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260507 04:16:02.887 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 04:16:02.887 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 04:16:02.887 | INFO | PerfReport: 1.46%, LockUpload -- 1.41e+00 / 0.00e+00, 4.78e+00 / 0.00e+00 | 20260507 04:16:02.887 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.15e+00 / 0.00e+00 | 20260507 04:16:02.887 | INFO | PerfReport: 22.06%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 04:16:02.887 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260507 04:16:02.984 | INFO | Starting a round of Baz IO | 20260507 04:16:06.084 | INFO | Populating RTMetrics report | 20260507 04:16:06.085 | INFO | RTMetrics report populated | 20260507 04:16:06.085 | INFO | Opening tmp RTMetrics json file | 20260507 04:16:06.103 | INFO | Serializing RTMetrics to json | 20260507 04:16:06.104 | INFO | Serialization done, swapping tmp file now... | 20260507 04:16:06.105 | INFO | RTMetrics done | 20260507 04:16:06.172 | INFO | Finished a round of Baz IO: Wrote 1.73 GiB in 3.188 seconds | 20260507 04:16:06.172 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.81% / 0.00% / 67.19% | 20260507 04:16:06.172 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.173903% / 3.97364e-06% / 8.74201e-05% | 20260507 04:16:06.172 | INFO | Baz IO overall: | 20260507 04:16:06.172 | INFO | BytesWritten : 3586923191456 | 20260507 04:16:06.172 | INFO | NumEvents : 1722285936374 | 20260507 04:16:08.026 | INFO | Analyzing chunk frames = [7020544,7021056) | 20260507 04:16:08.026 | INFO | MemInfo: { | 20260507 04:16:08.026 | INFO | "gpu" : " 17 GB", | 20260507 04:16:08.026 | INFO | "huge" : "", | 20260507 04:16:08.026 | INFO | "huge_shared" : " 27 GB", | 20260507 04:16:08.026 | INFO | "malloc" : " 14 GB", | 20260507 04:16:08.026 | INFO | "pinned" : " 29 GB" | 20260507 04:16:08.026 | INFO | } | 20260507 04:16:08.026 | INFO | Perf report for the last 5.14034 seconds: | 20260507 04:16:08.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:16:08.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:08.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:08.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:08.026 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:16:08.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:08.026 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 04:16:08.026 | INFO | PRE_HQ : 59.1%, 1.8, 55.3ms, 0 | 20260507 04:16:08.026 | INFO | BAZWRITER : 61.5%, 3.2, 102.5ms, 0 | 20260507 04:16:08.026 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:16:08.026 | INFO | Uploaded 36GiB to GPU in 2.0463s (17.5927GiB/s) | 20260507 04:16:08.026 | INFO | Downloaded 38GiB to GPU in 2.17504s (17.4709GiB/s) | 20260507 04:16:13.165 | INFO | Analyzing chunk frames = [7021056,7021568) | 20260507 04:16:13.165 | INFO | MemInfo: { | 20260507 04:16:13.165 | INFO | "gpu" : " 17 GB", | 20260507 04:16:13.165 | INFO | "huge" : "", | 20260507 04:16:13.165 | INFO | "huge_shared" : " 27 GB", | 20260507 04:16:13.165 | INFO | "malloc" : " 14 GB", | 20260507 04:16:13.166 | INFO | "pinned" : " 29 GB" | 20260507 04:16:13.166 | INFO | } | 20260507 04:16:13.166 | INFO | Perf report for the last 5.13995 seconds: | 20260507 04:16:13.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:16:13.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:13.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:13.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:13.166 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:16:13.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:13.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:13.166 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:16:13.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:13.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:13.166 | INFO | Uploaded 36GiB to GPU in 2.00404s (17.9637GiB/s) | 20260507 04:16:13.166 | INFO | Downloaded 34GiB to GPU in 1.93204s (17.598GiB/s) | 20260507 04:16:18.303 | INFO | Analyzing chunk frames = [7021568,7022080) | 20260507 04:16:18.303 | INFO | MemInfo: { | 20260507 04:16:18.303 | INFO | "gpu" : " 17 GB", | 20260507 04:16:18.303 | INFO | "huge" : "", | 20260507 04:16:18.303 | INFO | "huge_shared" : " 27 GB", | 20260507 04:16:18.303 | INFO | "malloc" : " 14 GB", | 20260507 04:16:18.303 | INFO | "pinned" : " 29 GB" | 20260507 04:16:18.303 | INFO | } | 20260507 04:16:18.303 | INFO | Perf report for the last 5.13743 seconds: | 20260507 04:16:18.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:16:18.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:18.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:18.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:18.303 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 04:16:18.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:18.303 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:18.303 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:16:18.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:18.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:18.303 | INFO | Uploaded 36GiB to GPU in 2.04037s (17.6439GiB/s) | 20260507 04:16:18.303 | INFO | Downloaded 34GiB to GPU in 1.95249s (17.4137GiB/s) | 20260507 04:16:23.443 | INFO | Analyzing chunk frames = [7022080,7022592) | 20260507 04:16:23.443 | INFO | MemInfo: { | 20260507 04:16:23.443 | INFO | "gpu" : " 17 GB", | 20260507 04:16:23.443 | INFO | "huge" : "", | 20260507 04:16:23.443 | INFO | "huge_shared" : " 27 GB", | 20260507 04:16:23.443 | INFO | "malloc" : " 14 GB", | 20260507 04:16:23.444 | INFO | "pinned" : " 29 GB" | 20260507 04:16:23.444 | INFO | } | 20260507 04:16:23.444 | INFO | Perf report for the last 5.14066 seconds: | 20260507 04:16:23.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:16:23.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:23.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:23.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:23.444 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 04:16:23.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:23.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:23.444 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:16:23.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:23.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:23.444 | INFO | Uploaded 36GiB to GPU in 2.00376s (17.9663GiB/s) | 20260507 04:16:23.444 | INFO | Downloaded 34GiB to GPU in 1.93338s (17.5857GiB/s) | 20260507 04:16:28.583 | INFO | Analyzing chunk frames = [7022592,7023104) | 20260507 04:16:28.583 | INFO | MemInfo: { | 20260507 04:16:28.583 | INFO | "gpu" : " 17 GB", | 20260507 04:16:28.583 | INFO | "huge" : "", | 20260507 04:16:28.583 | INFO | "huge_shared" : " 27 GB", | 20260507 04:16:28.583 | INFO | "malloc" : " 14 GB", | 20260507 04:16:28.583 | INFO | "pinned" : " 29 GB" | 20260507 04:16:28.583 | INFO | } | 20260507 04:16:28.583 | INFO | Perf report for the last 5.13922 seconds: | 20260507 04:16:28.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:16:28.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:28.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:28.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:28.583 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 04:16:28.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:28.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:28.583 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:16:28.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:28.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:28.583 | INFO | Uploaded 36GiB to GPU in 2.05399s (17.5269GiB/s) | 20260507 04:16:28.583 | INFO | Downloaded 34GiB to GPU in 1.94519s (17.479GiB/s) | 20260507 04:16:33.721 | INFO | Analyzing chunk frames = [7023104,7023616) | 20260507 04:16:33.721 | INFO | MemInfo: { | 20260507 04:16:33.721 | INFO | "gpu" : " 17 GB", | 20260507 04:16:33.721 | INFO | "huge" : "", | 20260507 04:16:33.721 | INFO | "huge_shared" : " 27 GB", | 20260507 04:16:33.721 | INFO | "malloc" : " 14 GB", | 20260507 04:16:33.721 | INFO | "pinned" : " 29 GB" | 20260507 04:16:33.721 | INFO | } | 20260507 04:16:33.721 | INFO | Perf report for the last 5.13834 seconds: | 20260507 04:16:33.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:16:33.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:33.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:33.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:33.721 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 04:16:33.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:33.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:33.721 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 04:16:33.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:33.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:33.722 | INFO | Uploaded 36GiB to GPU in 2.03543s (17.6867GiB/s) | 20260507 04:16:33.722 | INFO | Downloaded 34GiB to GPU in 1.91351s (17.7684GiB/s) | 20260507 04:16:38.855 | INFO | Analyzing chunk frames = [7023616,7024128) | 20260507 04:16:38.855 | INFO | MemInfo: { | 20260507 04:16:38.855 | INFO | "gpu" : " 17 GB", | 20260507 04:16:38.855 | INFO | "huge" : "", | 20260507 04:16:38.855 | INFO | "huge_shared" : " 27 GB", | 20260507 04:16:38.855 | INFO | "malloc" : " 14 GB", | 20260507 04:16:38.856 | INFO | "pinned" : " 29 GB" | 20260507 04:16:38.856 | INFO | } | 20260507 04:16:38.856 | INFO | Perf report for the last 5.13442 seconds: | 20260507 04:16:38.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:16:38.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:38.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:38.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:38.856 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:16:38.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:38.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:38.856 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 04:16:38.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:38.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:38.856 | INFO | Uploaded 36GiB to GPU in 1.99295s (18.0637GiB/s) | 20260507 04:16:38.856 | INFO | Downloaded 34GiB to GPU in 1.98268s (17.1485GiB/s) | 20260507 04:16:44.002 | INFO | Analyzing chunk frames = [7024128,7024640) | 20260507 04:16:44.002 | INFO | MemInfo: { | 20260507 04:16:44.002 | INFO | "gpu" : " 17 GB", | 20260507 04:16:44.002 | INFO | "huge" : "", | 20260507 04:16:44.002 | INFO | "huge_shared" : " 27 GB", | 20260507 04:16:44.003 | INFO | "malloc" : " 14 GB", | 20260507 04:16:44.003 | INFO | "pinned" : " 29 GB" | 20260507 04:16:44.003 | INFO | } | 20260507 04:16:44.003 | INFO | Perf report for the last 5.14695 seconds: | 20260507 04:16:44.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:16:44.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:44.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:44.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:44.003 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 04:16:44.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:44.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:44.003 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 04:16:44.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:44.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:44.003 | INFO | Uploaded 36GiB to GPU in 1.97392s (18.2379GiB/s) | 20260507 04:16:44.003 | INFO | Downloaded 34GiB to GPU in 1.95253s (17.4133GiB/s) | 20260507 04:16:44.102 | INFO | Starting a round of Baz IO | 20260507 04:16:47.206 | INFO | Populating RTMetrics report | 20260507 04:16:47.207 | INFO | RTMetrics report populated | 20260507 04:16:47.207 | INFO | Opening tmp RTMetrics json file | 20260507 04:16:47.210 | INFO | Serializing RTMetrics to json | 20260507 04:16:47.225 | INFO | Serialization done, swapping tmp file now... | 20260507 04:16:47.225 | INFO | RTMetrics done | 20260507 04:16:47.295 | INFO | Finished a round of Baz IO: Wrote 1.732 GiB in 3.192 seconds | 20260507 04:16:47.295 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.81% / 0.00% / 67.19% | 20260507 04:16:47.295 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.174193% / 0% / 0.000123183% | 20260507 04:16:47.295 | INFO | Baz IO overall: | 20260507 04:16:47.295 | INFO | BytesWritten : 3588782490612 | 20260507 04:16:47.295 | INFO | NumEvents : 1723161952639 | 20260507 04:16:49.141 | INFO | Analyzing chunk frames = [7024640,7025152) | 20260507 04:16:49.141 | INFO | MemInfo: { | 20260507 04:16:49.141 | INFO | "gpu" : " 17 GB", | 20260507 04:16:49.141 | INFO | "huge" : "", | 20260507 04:16:49.141 | INFO | "huge_shared" : " 27 GB", | 20260507 04:16:49.141 | INFO | "malloc" : " 14 GB", | 20260507 04:16:49.141 | INFO | "pinned" : " 29 GB" | 20260507 04:16:49.141 | INFO | } | 20260507 04:16:49.142 | INFO | Perf report for the last 5.13881 seconds: | 20260507 04:16:49.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:16:49.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:49.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:49.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:49.142 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 04:16:49.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:49.142 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 04:16:49.142 | INFO | PRE_HQ : 59.3%, 1.8, 55.6ms, 0 | 20260507 04:16:49.142 | INFO | BAZWRITER : 61.7%, 3.2, 102.1ms, 0 | 20260507 04:16:49.142 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 04:16:49.142 | INFO | Uploaded 36GiB to GPU in 2.04174s (17.632GiB/s) | 20260507 04:16:49.142 | INFO | Downloaded 38GiB to GPU in 2.17717s (17.4538GiB/s) | 20260507 04:16:54.275 | INFO | Analyzing chunk frames = [7025152,7025664) | 20260507 04:16:54.275 | INFO | MemInfo: { | 20260507 04:16:54.275 | INFO | "gpu" : " 17 GB", | 20260507 04:16:54.275 | INFO | "huge" : "", | 20260507 04:16:54.275 | INFO | "huge_shared" : " 27 GB", | 20260507 04:16:54.275 | INFO | "malloc" : " 14 GB", | 20260507 04:16:54.275 | INFO | "pinned" : " 29 GB" | 20260507 04:16:54.275 | INFO | } | 20260507 04:16:54.276 | INFO | Perf report for the last 5.13402 seconds: | 20260507 04:16:54.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:16:54.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:54.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:54.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:54.276 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:16:54.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:54.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:54.276 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:16:54.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:54.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:54.276 | INFO | Uploaded 36GiB to GPU in 1.99963s (18.0034GiB/s) | 20260507 04:16:54.276 | INFO | Downloaded 34GiB to GPU in 1.91686s (17.7374GiB/s) | 20260507 04:16:59.412 | INFO | Analyzing chunk frames = [7025664,7026176) | 20260507 04:16:59.412 | INFO | MemInfo: { | 20260507 04:16:59.412 | INFO | "gpu" : " 17 GB", | 20260507 04:16:59.412 | INFO | "huge" : "", | 20260507 04:16:59.412 | INFO | "huge_shared" : " 27 GB", | 20260507 04:16:59.412 | INFO | "malloc" : " 14 GB", | 20260507 04:16:59.412 | INFO | "pinned" : " 29 GB" | 20260507 04:16:59.412 | INFO | } | 20260507 04:16:59.412 | INFO | Perf report for the last 5.13641 seconds: | 20260507 04:16:59.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:16:59.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:59.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:59.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:59.412 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 04:16:59.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:59.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:59.412 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:16:59.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:59.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:16:59.412 | INFO | Uploaded 36GiB to GPU in 2.04678s (17.5886GiB/s) | 20260507 04:16:59.412 | INFO | Downloaded 34GiB to GPU in 1.94799s (17.4539GiB/s) | 20260507 04:17:04.555 | INFO | Analyzing chunk frames = [7026176,7026688) | 20260507 04:17:04.555 | INFO | MemInfo: { | 20260507 04:17:04.555 | INFO | "gpu" : " 17 GB", | 20260507 04:17:04.555 | INFO | "huge" : "", | 20260507 04:17:04.555 | INFO | "huge_shared" : " 27 GB", | 20260507 04:17:04.555 | INFO | "malloc" : " 14 GB", | 20260507 04:17:04.555 | INFO | "pinned" : " 29 GB" | 20260507 04:17:04.555 | INFO | } | 20260507 04:17:04.555 | INFO | Perf report for the last 5.14331 seconds: | 20260507 04:17:04.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:17:04.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:04.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:04.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:04.555 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 04:17:04.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:04.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:04.555 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:17:04.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:04.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:04.556 | INFO | Uploaded 36GiB to GPU in 1.98188s (18.1646GiB/s) | 20260507 04:17:04.556 | INFO | Downloaded 34GiB to GPU in 1.91488s (17.7557GiB/s) | 20260507 04:17:09.694 | INFO | Analyzing chunk frames = [7026688,7027200) | 20260507 04:17:09.694 | INFO | MemInfo: { | 20260507 04:17:09.694 | INFO | "gpu" : " 17 GB", | 20260507 04:17:09.694 | INFO | "huge" : "", | 20260507 04:17:09.694 | INFO | "huge_shared" : " 27 GB", | 20260507 04:17:09.694 | INFO | "malloc" : " 14 GB", | 20260507 04:17:09.694 | INFO | "pinned" : " 29 GB" | 20260507 04:17:09.694 | INFO | } | 20260507 04:17:09.694 | INFO | Perf report for the last 5.13908 seconds: | 20260507 04:17:09.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:17:09.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:09.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:09.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:09.694 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 04:17:09.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:09.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:09.694 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:17:09.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:09.695 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:09.695 | INFO | Uploaded 36GiB to GPU in 2.037s (17.673GiB/s) | 20260507 04:17:09.695 | INFO | Downloaded 34GiB to GPU in 1.9332s (17.5874GiB/s) | 20260507 04:17:14.836 | INFO | Analyzing chunk frames = [7027200,7027712) | 20260507 04:17:14.836 | INFO | MemInfo: { | 20260507 04:17:14.836 | INFO | "gpu" : " 17 GB", | 20260507 04:17:14.836 | INFO | "huge" : "", | 20260507 04:17:14.836 | INFO | "huge_shared" : " 27 GB", | 20260507 04:17:14.836 | INFO | "malloc" : " 14 GB", | 20260507 04:17:14.836 | INFO | "pinned" : " 29 GB" | 20260507 04:17:14.836 | INFO | } | 20260507 04:17:14.836 | INFO | Perf report for the last 5.14161 seconds: | 20260507 04:17:14.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:17:14.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:14.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:14.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:14.836 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:17:14.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:14.836 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:14.836 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 04:17:14.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:14.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:14.836 | INFO | Uploaded 36GiB to GPU in 2.05732s (17.4985GiB/s) | 20260507 04:17:14.836 | INFO | Downloaded 34GiB to GPU in 1.96534s (17.2998GiB/s) | 20260507 04:17:19.978 | INFO | Analyzing chunk frames = [7027712,7028224) | 20260507 04:17:19.978 | INFO | MemInfo: { | 20260507 04:17:19.978 | INFO | "gpu" : " 17 GB", | 20260507 04:17:19.978 | INFO | "huge" : "", | 20260507 04:17:19.978 | INFO | "huge_shared" : " 27 GB", | 20260507 04:17:19.978 | INFO | "malloc" : " 14 GB", | 20260507 04:17:19.978 | INFO | "pinned" : " 29 GB" | 20260507 04:17:19.978 | INFO | } | 20260507 04:17:19.978 | INFO | Perf report for the last 5.142 seconds: | 20260507 04:17:19.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:17:19.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:19.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:19.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:19.978 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 04:17:19.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:19.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:19.978 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:17:19.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:19.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:19.978 | INFO | Uploaded 36GiB to GPU in 2.01754s (17.8435GiB/s) | 20260507 04:17:19.978 | INFO | Downloaded 34GiB to GPU in 1.97286s (17.2338GiB/s) | 20260507 04:17:25.110 | INFO | Analyzing chunk frames = [7028224,7028736) | 20260507 04:17:25.110 | INFO | MemInfo: { | 20260507 04:17:25.110 | INFO | "gpu" : " 17 GB", | 20260507 04:17:25.110 | INFO | "huge" : "", | 20260507 04:17:25.110 | INFO | "huge_shared" : " 27 GB", | 20260507 04:17:25.110 | INFO | "malloc" : " 14 GB", | 20260507 04:17:25.110 | INFO | "pinned" : " 29 GB" | 20260507 04:17:25.110 | INFO | } | 20260507 04:17:25.110 | INFO | Perf report for the last 5.13207 seconds: | 20260507 04:17:25.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:17:25.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:25.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:25.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:25.110 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260507 04:17:25.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:25.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:25.110 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 04:17:25.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:25.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:25.111 | INFO | Uploaded 36GiB to GPU in 1.96647s (18.3069GiB/s) | 20260507 04:17:25.111 | INFO | Downloaded 34GiB to GPU in 1.97174s (17.2436GiB/s) | 20260507 04:17:25.111 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:17:25.111 | INFO | Total mem usage: 18 GiB | 20260507 04:17:25.111 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:17:25.111 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 04:17:25.111 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:17:25.111 | INFO | High Water Breakdown: | 20260507 04:17:25.111 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:17:25.111 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:17:25.111 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:17:25.111 | INFO | ------------------------------------------ | 20260507 04:17:25.111 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:17:25.111 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:17:25.111 | INFO | Total mem usage: 26.5 GiB | 20260507 04:17:25.111 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:17:25.111 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:17:25.111 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:17:25.111 | INFO | High Water Breakdown: | 20260507 04:17:25.111 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:17:25.111 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:17:25.111 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:17:25.111 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:17:25.111 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:17:25.111 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:17:25.111 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:17:25.111 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:17:25.111 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:17:25.111 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:17:25.111 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:17:25.111 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:17:25.111 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:17:25.111 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:17:25.111 | INFO | ------------------------------------------ | 20260507 04:17:25.111 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:17:25.111 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:17:25.111 | INFO | Total mem usage: 25.6 GiB | 20260507 04:17:25.111 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:17:25.111 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:17:25.111 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:17:25.111 | INFO | High Water Breakdown: | 20260507 04:17:25.111 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:17:25.111 | INFO | ------------------------------------------ | 20260507 04:17:25.111 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:17:25.111 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:17:25.111 | INFO | Total mem usage: 15.7 GiB | 20260507 04:17:25.111 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:17:25.111 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:17:25.111 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:17:25.111 | INFO | High Water Breakdown: | 20260507 04:17:25.111 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:17:25.111 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:17:25.111 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:17:25.112 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:17:25.112 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:17:25.112 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:17:25.112 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:17:25.112 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:17:25.112 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:17:25.112 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:17:25.112 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:17:25.112 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:17:25.112 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:17:25.112 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:17:25.112 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:17:25.112 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:17:25.112 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:17:25.112 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:17:25.112 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:17:25.112 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:17:25.112 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:17:25.112 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:17:25.112 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:17:25.112 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:17:25.112 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:17:25.112 | INFO | ------------------------------------------ | 20260507 04:17:25.112 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:17:25.112 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:17:25.112 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:17:25.112 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:17:25.112 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.32e+01 / 1.55e+00 | 20260507 04:17:25.112 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.00e-01, 9.79e-03 / 6.35e-03 | 20260507 04:17:25.112 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.44e+00, 5.08e-01 / 3.38e-01 | 20260507 04:17:25.112 | INFO | PerfReport: 3.22%, Histogram -- 9.91e-01 / 1.44e+00, 1.67e-01 / 1.24e-01 | 20260507 04:17:25.112 | INFO | PerfReport: 16.19%, DME -- 3.98e+01 / 4.45e+01, 3.99e+00 / 1.57e+00 | 20260507 04:17:25.112 | INFO | PerfReport: 57.61%, FrameLabeling -- 1.77e+01 / 1.99e+01, 1.26e+00 / 1.42e+00 | 20260507 04:17:25.112 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.14e+00 / 2.69e+00, 3.83e-01 / 2.30e-01 | 20260507 04:17:25.112 | INFO | PerfReport: 3.03%, Metrics -- 9.31e-01 / 1.24e+00, 2.44e-01 / 1.37e-01 | 20260507 04:17:25.112 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.15e-01 / 3.68e-01, 5.00e-02 / 1.82e-02 | 20260507 04:17:25.112 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:17:25.112 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:17:25.112 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:17:25.112 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 04:17:25.112 | INFO | PerfReport: 21.84%, Upload -- 2.10e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260507 04:17:25.112 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260507 04:17:25.112 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 04:17:25.112 | INFO | PerfReport: 1.57%, LockUpload -- 1.51e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 04:17:25.112 | INFO | PerfReport: 1.20%, LockDownload -- 1.16e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260507 04:17:25.112 | INFO | PerfReport: 22.08%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 04:17:25.112 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260507 04:17:25.207 | INFO | Starting a round of Baz IO | 20260507 04:17:28.347 | INFO | Populating RTMetrics report | 20260507 04:17:28.347 | INFO | RTMetrics report populated | 20260507 04:17:28.347 | INFO | Opening tmp RTMetrics json file | 20260507 04:17:28.349 | INFO | Serializing RTMetrics to json | 20260507 04:17:28.350 | INFO | Serialization done, swapping tmp file now... | 20260507 04:17:28.350 | INFO | RTMetrics done | 20260507 04:17:28.442 | INFO | Finished a round of Baz IO: Wrote 1.73 GiB in 3.234 seconds | 20260507 04:17:28.442 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.81% / 0.00% / 67.19% | 20260507 04:17:28.442 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.174061% / 0% / 8.34465e-05% | 20260507 04:17:28.442 | INFO | Baz IO overall: | 20260507 04:17:28.442 | INFO | BytesWritten : 3590640291599 | 20260507 04:17:28.442 | INFO | NumEvents : 1724036978758 | 20260507 04:17:30.254 | INFO | Analyzing chunk frames = [7028736,7029248) | 20260507 04:17:30.254 | INFO | MemInfo: { | 20260507 04:17:30.254 | INFO | "gpu" : " 17 GB", | 20260507 04:17:30.254 | INFO | "huge" : "", | 20260507 04:17:30.255 | INFO | "huge_shared" : " 27 GB", | 20260507 04:17:30.255 | INFO | "malloc" : " 14 GB", | 20260507 04:17:30.255 | INFO | "pinned" : " 29 GB" | 20260507 04:17:30.255 | INFO | } | 20260507 04:17:30.255 | INFO | Perf report for the last 5.14455 seconds: | 20260507 04:17:30.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:17:30.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:30.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:30.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:30.255 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260507 04:17:30.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:30.255 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260507 04:17:30.255 | INFO | PRE_HQ : 60.0%, 1.9, 59.3ms, 0 | 20260507 04:17:30.255 | INFO | BAZWRITER : 62.5%, 3.3, 106.3ms, 0 | 20260507 04:17:30.255 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 04:17:30.255 | INFO | Uploaded 36GiB to GPU in 2.03001s (17.7339GiB/s) | 20260507 04:17:30.255 | INFO | Downloaded 38GiB to GPU in 2.1833s (17.4049GiB/s) | 20260507 04:17:35.392 | INFO | Analyzing chunk frames = [7029248,7029760) | 20260507 04:17:35.392 | INFO | MemInfo: { | 20260507 04:17:35.392 | INFO | "gpu" : " 17 GB", | 20260507 04:17:35.392 | INFO | "huge" : "", | 20260507 04:17:35.392 | INFO | "huge_shared" : " 27 GB", | 20260507 04:17:35.392 | INFO | "malloc" : " 14 GB", | 20260507 04:17:35.392 | INFO | "pinned" : " 29 GB" | 20260507 04:17:35.392 | INFO | } | 20260507 04:17:35.392 | INFO | Perf report for the last 5.13741 seconds: | 20260507 04:17:35.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:17:35.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:35.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:35.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:35.392 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:17:35.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:35.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:35.392 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 04:17:35.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:35.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:35.392 | INFO | Uploaded 36GiB to GPU in 1.99994s (18.0005GiB/s) | 20260507 04:17:35.392 | INFO | Downloaded 34GiB to GPU in 1.93353s (17.5844GiB/s) | 20260507 04:17:40.529 | INFO | Analyzing chunk frames = [7029760,7030272) | 20260507 04:17:40.529 | INFO | MemInfo: { | 20260507 04:17:40.529 | INFO | "gpu" : " 17 GB", | 20260507 04:17:40.529 | INFO | "huge" : "", | 20260507 04:17:40.529 | INFO | "huge_shared" : " 27 GB", | 20260507 04:17:40.530 | INFO | "malloc" : " 14 GB", | 20260507 04:17:40.530 | INFO | "pinned" : " 29 GB" | 20260507 04:17:40.530 | INFO | } | 20260507 04:17:40.530 | INFO | Perf report for the last 5.13753 seconds: | 20260507 04:17:40.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:17:40.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:40.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:40.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:40.530 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 04:17:40.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:40.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:40.530 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:17:40.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:40.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:40.530 | INFO | Uploaded 36GiB to GPU in 2.01941s (17.827GiB/s) | 20260507 04:17:40.530 | INFO | Downloaded 34GiB to GPU in 1.95514s (17.3901GiB/s) | 20260507 04:17:45.673 | INFO | Analyzing chunk frames = [7030272,7030784) | 20260507 04:17:45.673 | INFO | MemInfo: { | 20260507 04:17:45.673 | INFO | "gpu" : " 17 GB", | 20260507 04:17:45.673 | INFO | "huge" : "", | 20260507 04:17:45.673 | INFO | "huge_shared" : " 27 GB", | 20260507 04:17:45.673 | INFO | "malloc" : " 14 GB", | 20260507 04:17:45.673 | INFO | "pinned" : " 29 GB" | 20260507 04:17:45.673 | INFO | } | 20260507 04:17:45.673 | INFO | Perf report for the last 5.14334 seconds: | 20260507 04:17:45.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:17:45.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:45.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:45.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:45.673 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 04:17:45.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:45.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:45.673 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:17:45.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:45.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:45.674 | INFO | Uploaded 36GiB to GPU in 1.99215s (18.071GiB/s) | 20260507 04:17:45.674 | INFO | Downloaded 34GiB to GPU in 1.9327s (17.592GiB/s) | 20260507 04:17:50.814 | INFO | Analyzing chunk frames = [7030784,7031296) | 20260507 04:17:50.814 | INFO | MemInfo: { | 20260507 04:17:50.814 | INFO | "gpu" : " 17 GB", | 20260507 04:17:50.814 | INFO | "huge" : "", | 20260507 04:17:50.814 | INFO | "huge_shared" : " 27 GB", | 20260507 04:17:50.814 | INFO | "malloc" : " 14 GB", | 20260507 04:17:50.814 | INFO | "pinned" : " 29 GB" | 20260507 04:17:50.814 | INFO | } | 20260507 04:17:50.814 | INFO | Perf report for the last 5.14138 seconds: | 20260507 04:17:50.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:17:50.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:50.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:50.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:50.815 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 04:17:50.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:50.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:50.815 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:17:50.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:50.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:50.815 | INFO | Uploaded 36GiB to GPU in 2.02776s (17.7536GiB/s) | 20260507 04:17:50.815 | INFO | Downloaded 34GiB to GPU in 1.94419s (17.488GiB/s) | 20260507 04:17:55.957 | INFO | Analyzing chunk frames = [7031296,7031808) | 20260507 04:17:55.957 | INFO | MemInfo: { | 20260507 04:17:55.957 | INFO | "gpu" : " 17 GB", | 20260507 04:17:55.957 | INFO | "huge" : "", | 20260507 04:17:55.957 | INFO | "huge_shared" : " 27 GB", | 20260507 04:17:55.957 | INFO | "malloc" : " 14 GB", | 20260507 04:17:55.957 | INFO | "pinned" : " 29 GB" | 20260507 04:17:55.958 | INFO | } | 20260507 04:17:55.958 | INFO | Perf report for the last 5.14318 seconds: | 20260507 04:17:55.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:17:55.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:55.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:55.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:55.958 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:17:55.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:55.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:55.958 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 04:17:55.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:55.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:17:55.958 | INFO | Uploaded 36GiB to GPU in 2.05076s (17.5545GiB/s) | 20260507 04:17:55.958 | INFO | Downloaded 34GiB to GPU in 1.96954s (17.2629GiB/s) | 20260507 04:18:01.090 | INFO | Analyzing chunk frames = [7031808,7032320) | 20260507 04:18:01.090 | INFO | MemInfo: { | 20260507 04:18:01.090 | INFO | "gpu" : " 17 GB", | 20260507 04:18:01.090 | INFO | "huge" : "", | 20260507 04:18:01.090 | INFO | "huge_shared" : " 27 GB", | 20260507 04:18:01.090 | INFO | "malloc" : " 14 GB", | 20260507 04:18:01.090 | INFO | "pinned" : " 29 GB" | 20260507 04:18:01.090 | INFO | } | 20260507 04:18:01.090 | INFO | Perf report for the last 5.13271 seconds: | 20260507 04:18:01.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:18:01.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:01.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:01.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:01.090 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 04:18:01.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:01.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:01.090 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 04:18:01.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:01.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:01.091 | INFO | Uploaded 36GiB to GPU in 1.9926s (18.0668GiB/s) | 20260507 04:18:01.091 | INFO | Downloaded 34GiB to GPU in 1.97085s (17.2514GiB/s) | 20260507 04:18:06.224 | INFO | Analyzing chunk frames = [7032320,7032832) | 20260507 04:18:06.224 | INFO | MemInfo: { | 20260507 04:18:06.224 | INFO | "gpu" : " 17 GB", | 20260507 04:18:06.224 | INFO | "huge" : "", | 20260507 04:18:06.224 | INFO | "huge_shared" : " 27 GB", | 20260507 04:18:06.224 | INFO | "malloc" : " 14 GB", | 20260507 04:18:06.224 | INFO | "pinned" : " 29 GB" | 20260507 04:18:06.224 | INFO | } | 20260507 04:18:06.224 | INFO | Perf report for the last 5.13402 seconds: | 20260507 04:18:06.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:18:06.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:06.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:06.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:06.224 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 04:18:06.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:06.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:06.224 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 04:18:06.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:06.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:06.225 | INFO | Uploaded 36GiB to GPU in 1.97773s (18.2027GiB/s) | 20260507 04:18:06.225 | INFO | Downloaded 34GiB to GPU in 1.96383s (17.3131GiB/s) | 20260507 04:18:06.321 | INFO | Starting a round of Baz IO | 20260507 04:18:09.456 | INFO | Populating RTMetrics report | 20260507 04:18:09.457 | INFO | RTMetrics report populated | 20260507 04:18:09.457 | INFO | Opening tmp RTMetrics json file | 20260507 04:18:09.457 | INFO | Serializing RTMetrics to json | 20260507 04:18:09.458 | INFO | Serialization done, swapping tmp file now... | 20260507 04:18:09.459 | INFO | RTMetrics done | 20260507 04:18:09.559 | INFO | Finished a round of Baz IO: Wrote 1.729 GiB in 3.238 seconds | 20260507 04:18:09.559 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.81% / 0.00% / 67.19% | 20260507 04:18:09.559 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.17372% / 0% / 8.34465e-05% | 20260507 04:18:09.559 | INFO | Baz IO overall: | 20260507 04:18:09.560 | INFO | BytesWritten : 3592496727906 | 20260507 04:18:09.560 | INFO | NumEvents : 1724911146649 | 20260507 04:18:11.367 | INFO | Analyzing chunk frames = [7032832,7033344) | 20260507 04:18:11.367 | INFO | MemInfo: { | 20260507 04:18:11.367 | INFO | "gpu" : " 17 GB", | 20260507 04:18:11.367 | INFO | "huge" : "", | 20260507 04:18:11.367 | INFO | "huge_shared" : " 27 GB", | 20260507 04:18:11.367 | INFO | "malloc" : " 14 GB", | 20260507 04:18:11.367 | INFO | "pinned" : " 29 GB" | 20260507 04:18:11.367 | INFO | } | 20260507 04:18:11.367 | INFO | Perf report for the last 5.14295 seconds: | 20260507 04:18:11.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:18:11.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:11.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:11.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:11.367 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260507 04:18:11.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:11.367 | INFO | RT_METRICS : 46.0%, 1.0, 24.1ms, 0 | 20260507 04:18:11.367 | INFO | PRE_HQ : 60.3%, 2.0, 61.2ms, 0 | 20260507 04:18:11.367 | INFO | BAZWRITER : 62.6%, 3.4, 109.6ms, 0 | 20260507 04:18:11.367 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 04:18:11.368 | INFO | Uploaded 36GiB to GPU in 2.04516s (17.6026GiB/s) | 20260507 04:18:11.368 | INFO | Downloaded 38GiB to GPU in 2.17812s (17.4462GiB/s) | 20260507 04:18:16.507 | INFO | Analyzing chunk frames = [7033344,7033856) | 20260507 04:18:16.507 | INFO | MemInfo: { | 20260507 04:18:16.507 | INFO | "gpu" : " 17 GB", | 20260507 04:18:16.507 | INFO | "huge" : "", | 20260507 04:18:16.507 | INFO | "huge_shared" : " 27 GB", | 20260507 04:18:16.508 | INFO | "malloc" : " 14 GB", | 20260507 04:18:16.508 | INFO | "pinned" : " 29 GB" | 20260507 04:18:16.508 | INFO | } | 20260507 04:18:16.508 | INFO | Perf report for the last 5.14026 seconds: | 20260507 04:18:16.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:18:16.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:16.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:16.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:16.508 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 04:18:16.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:16.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:16.508 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:18:16.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:16.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:16.508 | INFO | Uploaded 36GiB to GPU in 2.0382s (17.6627GiB/s) | 20260507 04:18:16.508 | INFO | Downloaded 34GiB to GPU in 1.93871s (17.5374GiB/s) | 20260507 04:18:21.645 | INFO | Analyzing chunk frames = [7033856,7034368) | 20260507 04:18:21.645 | INFO | MemInfo: { | 20260507 04:18:21.645 | INFO | "gpu" : " 17 GB", | 20260507 04:18:21.645 | INFO | "huge" : "", | 20260507 04:18:21.645 | INFO | "huge_shared" : " 27 GB", | 20260507 04:18:21.645 | INFO | "malloc" : " 14 GB", | 20260507 04:18:21.645 | INFO | "pinned" : " 29 GB" | 20260507 04:18:21.645 | INFO | } | 20260507 04:18:21.646 | INFO | Perf report for the last 5.13792 seconds: | 20260507 04:18:21.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:18:21.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:21.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:21.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:21.646 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 04:18:21.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:21.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:21.646 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:18:21.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:21.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:21.646 | INFO | Uploaded 36GiB to GPU in 1.98698s (18.1179GiB/s) | 20260507 04:18:21.646 | INFO | Downloaded 34GiB to GPU in 1.94352s (17.4941GiB/s) | 20260507 04:18:26.788 | INFO | Analyzing chunk frames = [7034368,7034880) | 20260507 04:18:26.788 | INFO | MemInfo: { | 20260507 04:18:26.788 | INFO | "gpu" : " 17 GB", | 20260507 04:18:26.788 | INFO | "huge" : "", | 20260507 04:18:26.788 | INFO | "huge_shared" : " 27 GB", | 20260507 04:18:26.788 | INFO | "malloc" : " 14 GB", | 20260507 04:18:26.788 | INFO | "pinned" : " 29 GB" | 20260507 04:18:26.788 | INFO | } | 20260507 04:18:26.788 | INFO | Perf report for the last 5.14292 seconds: | 20260507 04:18:26.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:18:26.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:26.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:26.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:26.789 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 04:18:26.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:26.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:26.789 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:18:26.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:26.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:26.789 | INFO | Uploaded 36GiB to GPU in 1.9925s (18.0677GiB/s) | 20260507 04:18:26.789 | INFO | Downloaded 34GiB to GPU in 1.93795s (17.5443GiB/s) | 20260507 04:18:31.925 | INFO | Analyzing chunk frames = [7034880,7035392) | 20260507 04:18:31.925 | INFO | MemInfo: { | 20260507 04:18:31.925 | INFO | "gpu" : " 17 GB", | 20260507 04:18:31.925 | INFO | "huge" : "", | 20260507 04:18:31.925 | INFO | "huge_shared" : " 27 GB", | 20260507 04:18:31.925 | INFO | "malloc" : " 14 GB", | 20260507 04:18:31.925 | INFO | "pinned" : " 29 GB" | 20260507 04:18:31.925 | INFO | } | 20260507 04:18:31.925 | INFO | Perf report for the last 5.13656 seconds: | 20260507 04:18:31.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:18:31.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:31.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:31.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:31.925 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 04:18:31.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:31.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:31.925 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:18:31.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:31.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:31.925 | INFO | Uploaded 36GiB to GPU in 2.04337s (17.618GiB/s) | 20260507 04:18:31.926 | INFO | Downloaded 34GiB to GPU in 1.94996s (17.4362GiB/s) | 20260507 04:18:37.064 | INFO | Analyzing chunk frames = [7035392,7035904) | 20260507 04:18:37.064 | INFO | MemInfo: { | 20260507 04:18:37.064 | INFO | "gpu" : " 17 GB", | 20260507 04:18:37.064 | INFO | "huge" : "", | 20260507 04:18:37.064 | INFO | "huge_shared" : " 27 GB", | 20260507 04:18:37.064 | INFO | "malloc" : " 14 GB", | 20260507 04:18:37.064 | INFO | "pinned" : " 29 GB" | 20260507 04:18:37.064 | INFO | } | 20260507 04:18:37.064 | INFO | Perf report for the last 5.13903 seconds: | 20260507 04:18:37.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:18:37.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:37.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:37.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:37.064 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 04:18:37.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:37.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:37.064 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:18:37.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:37.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:37.064 | INFO | Uploaded 36GiB to GPU in 2.02036s (17.8186GiB/s) | 20260507 04:18:37.065 | INFO | Downloaded 34GiB to GPU in 1.92421s (17.6696GiB/s) | 20260507 04:18:42.201 | INFO | Analyzing chunk frames = [7035904,7036416) | 20260507 04:18:42.201 | INFO | MemInfo: { | 20260507 04:18:42.201 | INFO | "gpu" : " 17 GB", | 20260507 04:18:42.201 | INFO | "huge" : "", | 20260507 04:18:42.201 | INFO | "huge_shared" : " 27 GB", | 20260507 04:18:42.201 | INFO | "malloc" : " 14 GB", | 20260507 04:18:42.201 | INFO | "pinned" : " 29 GB" | 20260507 04:18:42.201 | INFO | } | 20260507 04:18:42.201 | INFO | Perf report for the last 5.13655 seconds: | 20260507 04:18:42.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:18:42.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:42.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:42.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:42.201 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 04:18:42.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:42.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:42.201 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 04:18:42.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:42.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:42.201 | INFO | Uploaded 36GiB to GPU in 1.973s (18.2463GiB/s) | 20260507 04:18:42.201 | INFO | Downloaded 34GiB to GPU in 1.96525s (17.3006GiB/s) | 20260507 04:18:47.347 | INFO | Analyzing chunk frames = [7036416,7036928) | 20260507 04:18:47.348 | INFO | MemInfo: { | 20260507 04:18:47.348 | INFO | "gpu" : " 17 GB", | 20260507 04:18:47.348 | INFO | "huge" : "", | 20260507 04:18:47.348 | INFO | "huge_shared" : " 27 GB", | 20260507 04:18:47.348 | INFO | "malloc" : " 14 GB", | 20260507 04:18:47.348 | INFO | "pinned" : " 29 GB" | 20260507 04:18:47.348 | INFO | } | 20260507 04:18:47.348 | INFO | Perf report for the last 5.14696 seconds: | 20260507 04:18:47.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:18:47.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:47.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:47.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:47.348 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:18:47.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:47.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:47.348 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:18:47.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:47.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:47.348 | INFO | Uploaded 36GiB to GPU in 1.96908s (18.2826GiB/s) | 20260507 04:18:47.348 | INFO | Downloaded 34GiB to GPU in 1.95947s (17.3516GiB/s) | 20260507 04:18:47.348 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:18:47.348 | INFO | Total mem usage: 18 GiB | 20260507 04:18:47.348 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:18:47.348 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.6%) | 20260507 04:18:47.348 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:18:47.348 | INFO | High Water Breakdown: | 20260507 04:18:47.348 | INFO | 60.2% : 8.64 GiB ( 75.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:18:47.348 | INFO | 39.6% : 5.68 GiB ( 84.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:18:47.348 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:18:47.348 | INFO | ------------------------------------------ | 20260507 04:18:47.348 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:18:47.348 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:18:47.348 | INFO | Total mem usage: 26.5 GiB | 20260507 04:18:47.348 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:18:47.348 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:18:47.348 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:18:47.348 | INFO | High Water Breakdown: | 20260507 04:18:47.348 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:18:47.348 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:18:47.348 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:18:47.349 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:18:47.349 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:18:47.349 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:18:47.349 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:18:47.349 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:18:47.349 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:18:47.349 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:18:47.349 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:18:47.349 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:18:47.349 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:18:47.349 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:18:47.349 | INFO | ------------------------------------------ | 20260507 04:18:47.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:18:47.349 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:18:47.349 | INFO | Total mem usage: 25.6 GiB | 20260507 04:18:47.349 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:18:47.349 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:18:47.349 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:18:47.349 | INFO | High Water Breakdown: | 20260507 04:18:47.349 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:18:47.349 | INFO | ------------------------------------------ | 20260507 04:18:47.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:18:47.349 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:18:47.349 | INFO | Total mem usage: 15.7 GiB | 20260507 04:18:47.349 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:18:47.349 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:18:47.349 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:18:47.349 | INFO | High Water Breakdown: | 20260507 04:18:47.349 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:18:47.349 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:18:47.349 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:18:47.349 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:18:47.349 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:18:47.349 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:18:47.349 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:18:47.349 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:18:47.349 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:18:47.349 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:18:47.349 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:18:47.349 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:18:47.349 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:18:47.349 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:18:47.349 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:18:47.349 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:18:47.349 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:18:47.349 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:18:47.349 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:18:47.349 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:18:47.349 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:18:47.349 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:18:47.349 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:18:47.349 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:18:47.349 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:18:47.349 | INFO | ------------------------------------------ | 20260507 04:18:47.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:18:47.349 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:18:47.349 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:18:47.349 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:18:47.349 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.31e+01 / 1.42e+00 | 20260507 04:18:47.349 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 1.02e-01, 1.01e-02 / 6.31e-03 | 20260507 04:18:47.349 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.42e+00, 5.15e-01 / 2.96e-01 | 20260507 04:18:47.349 | INFO | PerfReport: 3.24%, Histogram -- 9.95e-01 / 1.47e+00, 1.71e-01 / 1.20e-01 | 20260507 04:18:47.349 | INFO | PerfReport: 16.15%, DME -- 3.97e+01 / 4.45e+01, 3.75e+00 / 2.00e+00 | 20260507 04:18:47.349 | INFO | PerfReport: 57.61%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.26e+00 / 1.72e+00 | 20260507 04:18:47.349 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.14e+00 / 2.69e+00, 3.87e-01 / 2.47e-01 | 20260507 04:18:47.349 | INFO | PerfReport: 3.03%, Metrics -- 9.31e-01 / 1.22e+00, 2.47e-01 / 1.06e-01 | 20260507 04:18:47.349 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.15e-01 / 3.64e-01, 5.03e-02 / 1.90e-02 | 20260507 04:18:47.349 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:18:47.349 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:18:47.349 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:18:47.349 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 04:18:47.349 | INFO | PerfReport: 21.77%, Upload -- 2.09e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260507 04:18:47.350 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.45e+00 / 0.00e+00 | 20260507 04:18:47.350 | INFO | PerfReport: 31.95%, Compute -- 3.07e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 04:18:47.350 | INFO | PerfReport: 1.41%, LockUpload -- 1.36e+00 / 0.00e+00, 4.74e+00 / 0.00e+00 | 20260507 04:18:47.350 | INFO | PerfReport: 1.39%, LockDownload -- 1.34e+00 / 0.00e+00, 3.20e+00 / 0.00e+00 | 20260507 04:18:47.350 | INFO | PerfReport: 22.04%, LockCompute -- 2.12e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 04:18:47.350 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260507 04:18:47.447 | INFO | Starting a round of Baz IO | 20260507 04:18:50.544 | INFO | Populating RTMetrics report | 20260507 04:18:50.544 | INFO | RTMetrics report populated | 20260507 04:18:50.544 | INFO | Opening tmp RTMetrics json file | 20260507 04:18:50.559 | INFO | Serializing RTMetrics to json | 20260507 04:18:50.560 | INFO | Serialization done, swapping tmp file now... | 20260507 04:18:50.560 | INFO | RTMetrics done | 20260507 04:18:50.632 | INFO | Finished a round of Baz IO: Wrote 1.729 GiB in 3.185 seconds | 20260507 04:18:50.632 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.80% / 0.00% / 67.19% | 20260507 04:18:50.633 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.173652% / 0% / 5.5631e-05% | 20260507 04:18:50.633 | INFO | Baz IO overall: | 20260507 04:18:50.633 | INFO | BytesWritten : 3594353169228 | 20260507 04:18:50.633 | INFO | NumEvents : 1725785459391 | 20260507 04:18:52.484 | INFO | Analyzing chunk frames = [7036928,7037440) | 20260507 04:18:52.484 | INFO | MemInfo: { | 20260507 04:18:52.484 | INFO | "gpu" : " 17 GB", | 20260507 04:18:52.484 | INFO | "huge" : "", | 20260507 04:18:52.484 | INFO | "huge_shared" : " 27 GB", | 20260507 04:18:52.484 | INFO | "malloc" : " 14 GB", | 20260507 04:18:52.484 | INFO | "pinned" : " 29 GB" | 20260507 04:18:52.484 | INFO | } | 20260507 04:18:52.484 | INFO | Perf report for the last 5.13621 seconds: | 20260507 04:18:52.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:18:52.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:52.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:52.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:52.484 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 04:18:52.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:52.484 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:18:52.484 | INFO | PRE_HQ : 59.1%, 1.8, 55.7ms, 0 | 20260507 04:18:52.484 | INFO | BAZWRITER : 61.6%, 3.2, 101.7ms, 0 | 20260507 04:18:52.484 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:18:52.484 | INFO | Uploaded 36GiB to GPU in 2.06265s (17.4533GiB/s) | 20260507 04:18:52.484 | INFO | Downloaded 38GiB to GPU in 2.17482s (17.4727GiB/s) | 20260507 04:18:57.624 | INFO | Analyzing chunk frames = [7037440,7037952) | 20260507 04:18:57.624 | INFO | MemInfo: { | 20260507 04:18:57.624 | INFO | "gpu" : " 17 GB", | 20260507 04:18:57.624 | INFO | "huge" : "", | 20260507 04:18:57.624 | INFO | "huge_shared" : " 27 GB", | 20260507 04:18:57.624 | INFO | "malloc" : " 14 GB", | 20260507 04:18:57.624 | INFO | "pinned" : " 29 GB" | 20260507 04:18:57.624 | INFO | } | 20260507 04:18:57.624 | INFO | Perf report for the last 5.14046 seconds: | 20260507 04:18:57.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:18:57.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:57.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:57.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:57.624 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:18:57.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:57.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:57.624 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:18:57.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:57.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:18:57.624 | INFO | Uploaded 36GiB to GPU in 1.99459s (18.0488GiB/s) | 20260507 04:18:57.625 | INFO | Downloaded 34GiB to GPU in 1.93588s (17.563GiB/s) | 20260507 04:19:02.767 | INFO | Analyzing chunk frames = [7037952,7038464) | 20260507 04:19:02.767 | INFO | MemInfo: { | 20260507 04:19:02.767 | INFO | "gpu" : " 17 GB", | 20260507 04:19:02.767 | INFO | "huge" : "", | 20260507 04:19:02.767 | INFO | "huge_shared" : " 27 GB", | 20260507 04:19:02.767 | INFO | "malloc" : " 14 GB", | 20260507 04:19:02.767 | INFO | "pinned" : " 29 GB" | 20260507 04:19:02.767 | INFO | } | 20260507 04:19:02.767 | INFO | Perf report for the last 5.14299 seconds: | 20260507 04:19:02.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:19:02.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:02.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:02.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:02.767 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 04:19:02.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:02.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:02.767 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:19:02.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:02.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:02.768 | INFO | Uploaded 36GiB to GPU in 2.01657s (17.8521GiB/s) | 20260507 04:19:02.768 | INFO | Downloaded 34GiB to GPU in 1.93801s (17.5438GiB/s) | 20260507 04:19:07.906 | INFO | Analyzing chunk frames = [7038464,7038976) | 20260507 04:19:07.906 | INFO | MemInfo: { | 20260507 04:19:07.906 | INFO | "gpu" : " 17 GB", | 20260507 04:19:07.906 | INFO | "huge" : "", | 20260507 04:19:07.906 | INFO | "huge_shared" : " 27 GB", | 20260507 04:19:07.906 | INFO | "malloc" : " 14 GB", | 20260507 04:19:07.906 | INFO | "pinned" : " 29 GB" | 20260507 04:19:07.906 | INFO | } | 20260507 04:19:07.906 | INFO | Perf report for the last 5.13889 seconds: | 20260507 04:19:07.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:19:07.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:07.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:07.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:07.906 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:19:07.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:07.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:07.906 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 04:19:07.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:07.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:07.906 | INFO | Uploaded 36GiB to GPU in 2.00829s (17.9257GiB/s) | 20260507 04:19:07.906 | INFO | Downloaded 34GiB to GPU in 1.93131s (17.6047GiB/s) | 20260507 04:19:13.039 | INFO | Analyzing chunk frames = [7038976,7039488) | 20260507 04:19:13.039 | INFO | MemInfo: { | 20260507 04:19:13.039 | INFO | "gpu" : " 17 GB", | 20260507 04:19:13.039 | INFO | "huge" : "", | 20260507 04:19:13.039 | INFO | "huge_shared" : " 27 GB", | 20260507 04:19:13.039 | INFO | "malloc" : " 14 GB", | 20260507 04:19:13.039 | INFO | "pinned" : " 29 GB" | 20260507 04:19:13.039 | INFO | } | 20260507 04:19:13.039 | INFO | Perf report for the last 5.13321 seconds: | 20260507 04:19:13.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:19:13.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:13.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:13.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:13.039 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260507 04:19:13.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:13.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:13.040 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:19:13.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:13.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:13.040 | INFO | Uploaded 36GiB to GPU in 2.03794s (17.6649GiB/s) | 20260507 04:19:13.040 | INFO | Downloaded 34GiB to GPU in 1.91603s (17.7451GiB/s) | 20260507 04:19:18.174 | INFO | Analyzing chunk frames = [7039488,7040000) | 20260507 04:19:18.174 | INFO | MemInfo: { | 20260507 04:19:18.174 | INFO | "gpu" : " 17 GB", | 20260507 04:19:18.174 | INFO | "huge" : "", | 20260507 04:19:18.174 | INFO | "huge_shared" : " 27 GB", | 20260507 04:19:18.174 | INFO | "malloc" : " 14 GB", | 20260507 04:19:18.174 | INFO | "pinned" : " 29 GB" | 20260507 04:19:18.174 | INFO | } | 20260507 04:19:18.174 | INFO | Perf report for the last 5.13489 seconds: | 20260507 04:19:18.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:19:18.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:18.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:18.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:18.174 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 04:19:18.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:18.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:18.174 | INFO | PRE_HQ : 29.9%, 1.0, 15.5ms, 0 | 20260507 04:19:18.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:18.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:18.175 | INFO | Uploaded 36GiB to GPU in 2.04855s (17.5734GiB/s) | 20260507 04:19:18.175 | INFO | Downloaded 34GiB to GPU in 1.91528s (17.752GiB/s) | 20260507 04:19:23.322 | INFO | Analyzing chunk frames = [7040000,7040512) | 20260507 04:19:23.322 | INFO | MemInfo: { | 20260507 04:19:23.322 | INFO | "gpu" : " 17 GB", | 20260507 04:19:23.322 | INFO | "huge" : "", | 20260507 04:19:23.322 | INFO | "huge_shared" : " 27 GB", | 20260507 04:19:23.322 | INFO | "malloc" : " 14 GB", | 20260507 04:19:23.322 | INFO | "pinned" : " 29 GB" | 20260507 04:19:23.322 | INFO | } | 20260507 04:19:23.322 | INFO | Perf report for the last 5.14733 seconds: | 20260507 04:19:23.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:19:23.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:23.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:23.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:23.322 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:19:23.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:23.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:23.322 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 04:19:23.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:23.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:23.322 | INFO | Uploaded 36GiB to GPU in 1.98468s (18.1389GiB/s) | 20260507 04:19:23.322 | INFO | Downloaded 34GiB to GPU in 2.00012s (16.999GiB/s) | 20260507 04:19:28.460 | INFO | Analyzing chunk frames = [7040512,7041024) | 20260507 04:19:28.460 | INFO | MemInfo: { | 20260507 04:19:28.460 | INFO | "gpu" : " 17 GB", | 20260507 04:19:28.460 | INFO | "huge" : "", | 20260507 04:19:28.460 | INFO | "huge_shared" : " 27 GB", | 20260507 04:19:28.460 | INFO | "malloc" : " 14 GB", | 20260507 04:19:28.460 | INFO | "pinned" : " 29 GB" | 20260507 04:19:28.461 | INFO | } | 20260507 04:19:28.461 | INFO | Perf report for the last 5.13889 seconds: | 20260507 04:19:28.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:19:28.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:28.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:28.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:28.461 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:19:28.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:28.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:28.461 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:19:28.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:28.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:28.461 | INFO | Uploaded 36GiB to GPU in 1.98346s (18.1501GiB/s) | 20260507 04:19:28.461 | INFO | Downloaded 34GiB to GPU in 1.97556s (17.2103GiB/s) | 20260507 04:19:28.557 | INFO | Starting a round of Baz IO | 20260507 04:19:31.656 | INFO | Populating RTMetrics report | 20260507 04:19:31.656 | INFO | RTMetrics report populated | 20260507 04:19:31.656 | INFO | Opening tmp RTMetrics json file | 20260507 04:19:31.661 | INFO | Serializing RTMetrics to json | 20260507 04:19:31.674 | INFO | Serialization done, swapping tmp file now... | 20260507 04:19:31.675 | INFO | RTMetrics done | 20260507 04:19:31.747 | INFO | Finished a round of Baz IO: Wrote 1.728 GiB in 3.189 seconds | 20260507 04:19:31.747 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.80% / 0.00% / 67.20% | 20260507 04:19:31.747 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.173076% / 0% / 6.35783e-05% | 20260507 04:19:31.747 | INFO | Baz IO overall: | 20260507 04:19:31.747 | INFO | BytesWritten : 3596208063590 | 20260507 04:19:31.747 | INFO | NumEvents : 1726658797195 | 20260507 04:19:33.600 | INFO | Analyzing chunk frames = [7041024,7041536) | 20260507 04:19:33.600 | INFO | MemInfo: { | 20260507 04:19:33.600 | INFO | "gpu" : " 17 GB", | 20260507 04:19:33.600 | INFO | "huge" : "", | 20260507 04:19:33.600 | INFO | "huge_shared" : " 27 GB", | 20260507 04:19:33.600 | INFO | "malloc" : " 14 GB", | 20260507 04:19:33.600 | INFO | "pinned" : " 29 GB" | 20260507 04:19:33.600 | INFO | } | 20260507 04:19:33.601 | INFO | Perf report for the last 5.13996 seconds: | 20260507 04:19:33.601 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:19:33.601 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:33.601 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:33.601 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:33.601 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 04:19:33.601 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:33.601 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 04:19:33.601 | INFO | PRE_HQ : 59.2%, 1.8, 55.5ms, 0 | 20260507 04:19:33.601 | INFO | BAZWRITER : 61.6%, 3.2, 102.9ms, 0 | 20260507 04:19:33.601 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:19:33.601 | INFO | Uploaded 36GiB to GPU in 2.05235s (17.5409GiB/s) | 20260507 04:19:33.601 | INFO | Downloaded 38GiB to GPU in 2.18195s (17.4156GiB/s) | 20260507 04:19:38.739 | INFO | Analyzing chunk frames = [7041536,7042048) | 20260507 04:19:38.739 | INFO | MemInfo: { | 20260507 04:19:38.739 | INFO | "gpu" : " 17 GB", | 20260507 04:19:38.739 | INFO | "huge" : "", | 20260507 04:19:38.739 | INFO | "huge_shared" : " 27 GB", | 20260507 04:19:38.740 | INFO | "malloc" : " 14 GB", | 20260507 04:19:38.740 | INFO | "pinned" : " 29 GB" | 20260507 04:19:38.740 | INFO | } | 20260507 04:19:38.740 | INFO | Perf report for the last 5.139 seconds: | 20260507 04:19:38.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:19:38.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:38.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:38.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:38.740 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:19:38.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:38.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:38.740 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:19:38.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:38.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:38.740 | INFO | Uploaded 36GiB to GPU in 1.98189s (18.1645GiB/s) | 20260507 04:19:38.740 | INFO | Downloaded 34GiB to GPU in 1.9312s (17.6057GiB/s) | 20260507 04:19:43.874 | INFO | Analyzing chunk frames = [7042048,7042560) | 20260507 04:19:43.874 | INFO | MemInfo: { | 20260507 04:19:43.874 | INFO | "gpu" : " 17 GB", | 20260507 04:19:43.874 | INFO | "huge" : "", | 20260507 04:19:43.874 | INFO | "huge_shared" : " 27 GB", | 20260507 04:19:43.874 | INFO | "malloc" : " 14 GB", | 20260507 04:19:43.874 | INFO | "pinned" : " 29 GB" | 20260507 04:19:43.874 | INFO | } | 20260507 04:19:43.874 | INFO | Perf report for the last 5.13441 seconds: | 20260507 04:19:43.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:19:43.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:43.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:43.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:43.874 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 04:19:43.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:43.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:43.874 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:19:43.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:43.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:43.874 | INFO | Uploaded 36GiB to GPU in 2.01411s (17.8739GiB/s) | 20260507 04:19:43.874 | INFO | Downloaded 34GiB to GPU in 1.95642s (17.3786GiB/s) | 20260507 04:19:49.015 | INFO | Analyzing chunk frames = [7042560,7043072) | 20260507 04:19:49.015 | INFO | MemInfo: { | 20260507 04:19:49.015 | INFO | "gpu" : " 17 GB", | 20260507 04:19:49.015 | INFO | "huge" : "", | 20260507 04:19:49.016 | INFO | "huge_shared" : " 27 GB", | 20260507 04:19:49.016 | INFO | "malloc" : " 14 GB", | 20260507 04:19:49.016 | INFO | "pinned" : " 29 GB" | 20260507 04:19:49.016 | INFO | } | 20260507 04:19:49.016 | INFO | Perf report for the last 5.1417 seconds: | 20260507 04:19:49.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:19:49.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:49.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:49.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:49.016 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:19:49.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:49.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:49.016 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:19:49.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:49.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:49.016 | INFO | Uploaded 36GiB to GPU in 1.96612s (18.3102GiB/s) | 20260507 04:19:49.016 | INFO | Downloaded 34GiB to GPU in 1.92289s (17.6817GiB/s) | 20260507 04:19:54.153 | INFO | Analyzing chunk frames = [7043072,7043584) | 20260507 04:19:54.153 | INFO | MemInfo: { | 20260507 04:19:54.153 | INFO | "gpu" : " 17 GB", | 20260507 04:19:54.153 | INFO | "huge" : "", | 20260507 04:19:54.153 | INFO | "huge_shared" : " 27 GB", | 20260507 04:19:54.153 | INFO | "malloc" : " 14 GB", | 20260507 04:19:54.153 | INFO | "pinned" : " 29 GB" | 20260507 04:19:54.153 | INFO | } | 20260507 04:19:54.153 | INFO | Perf report for the last 5.13777 seconds: | 20260507 04:19:54.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:19:54.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:54.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:54.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:54.154 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 04:19:54.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:54.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:54.154 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:19:54.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:54.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:54.154 | INFO | Uploaded 36GiB to GPU in 2.00212s (17.9809GiB/s) | 20260507 04:19:54.154 | INFO | Downloaded 34GiB to GPU in 1.94067s (17.5197GiB/s) | 20260507 04:19:59.299 | INFO | Analyzing chunk frames = [7043584,7044096) | 20260507 04:19:59.299 | INFO | MemInfo: { | 20260507 04:19:59.299 | INFO | "gpu" : " 17 GB", | 20260507 04:19:59.299 | INFO | "huge" : "", | 20260507 04:19:59.299 | INFO | "huge_shared" : " 27 GB", | 20260507 04:19:59.299 | INFO | "malloc" : " 14 GB", | 20260507 04:19:59.299 | INFO | "pinned" : " 29 GB" | 20260507 04:19:59.299 | INFO | } | 20260507 04:19:59.299 | INFO | Perf report for the last 5.14557 seconds: | 20260507 04:19:59.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:19:59.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:59.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:59.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:59.299 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:19:59.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:59.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:59.299 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 04:19:59.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:59.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:19:59.299 | INFO | Uploaded 36GiB to GPU in 2.0511s (17.5515GiB/s) | 20260507 04:19:59.299 | INFO | Downloaded 34GiB to GPU in 1.96977s (17.2609GiB/s) | 20260507 04:20:04.431 | INFO | Analyzing chunk frames = [7044096,7044608) | 20260507 04:20:04.431 | INFO | MemInfo: { | 20260507 04:20:04.431 | INFO | "gpu" : " 17 GB", | 20260507 04:20:04.431 | INFO | "huge" : "", | 20260507 04:20:04.431 | INFO | "huge_shared" : " 27 GB", | 20260507 04:20:04.431 | INFO | "malloc" : " 14 GB", | 20260507 04:20:04.431 | INFO | "pinned" : " 29 GB" | 20260507 04:20:04.431 | INFO | } | 20260507 04:20:04.431 | INFO | Perf report for the last 5.132 seconds: | 20260507 04:20:04.431 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:20:04.431 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:04.431 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:04.431 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:04.431 | INFO | ANALYSIS : 61.1%, 3.0, 94.9ms, 0 | 20260507 04:20:04.431 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:04.431 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:04.431 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:20:04.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:04.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:04.431 | INFO | Uploaded 36GiB to GPU in 2.03886s (17.6569GiB/s) | 20260507 04:20:04.431 | INFO | Downloaded 34GiB to GPU in 1.9839s (17.138GiB/s) | 20260507 04:20:09.580 | INFO | Analyzing chunk frames = [7044608,7045120) | 20260507 04:20:09.580 | INFO | MemInfo: { | 20260507 04:20:09.580 | INFO | "gpu" : " 17 GB", | 20260507 04:20:09.580 | INFO | "huge" : "", | 20260507 04:20:09.580 | INFO | "huge_shared" : " 27 GB", | 20260507 04:20:09.580 | INFO | "malloc" : " 14 GB", | 20260507 04:20:09.580 | INFO | "pinned" : " 29 GB" | 20260507 04:20:09.580 | INFO | } | 20260507 04:20:09.580 | INFO | Perf report for the last 5.14886 seconds: | 20260507 04:20:09.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:20:09.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:09.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:09.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:09.580 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 04:20:09.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:09.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:09.580 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:20:09.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:09.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:09.580 | INFO | Uploaded 36GiB to GPU in 1.97523s (18.2257GiB/s) | 20260507 04:20:09.580 | INFO | Downloaded 34GiB to GPU in 1.94855s (17.4489GiB/s) | 20260507 04:20:09.580 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:20:09.580 | INFO | Total mem usage: 18 GiB | 20260507 04:20:09.580 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:20:09.580 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 04:20:09.581 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:20:09.581 | INFO | High Water Breakdown: | 20260507 04:20:09.581 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:20:09.581 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:20:09.581 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:20:09.581 | INFO | ------------------------------------------ | 20260507 04:20:09.581 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:20:09.581 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:20:09.581 | INFO | Total mem usage: 26.5 GiB | 20260507 04:20:09.581 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:20:09.581 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:20:09.581 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:20:09.581 | INFO | High Water Breakdown: | 20260507 04:20:09.581 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:20:09.581 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:20:09.581 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:20:09.581 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:20:09.581 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:20:09.581 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:20:09.581 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:20:09.581 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:20:09.581 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:20:09.581 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:20:09.581 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:20:09.581 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:20:09.581 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:20:09.581 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:20:09.581 | INFO | ------------------------------------------ | 20260507 04:20:09.581 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:20:09.581 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:20:09.581 | INFO | Total mem usage: 25.6 GiB | 20260507 04:20:09.581 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:20:09.581 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:20:09.581 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:20:09.581 | INFO | High Water Breakdown: | 20260507 04:20:09.581 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:20:09.581 | INFO | ------------------------------------------ | 20260507 04:20:09.581 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:20:09.581 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:20:09.581 | INFO | Total mem usage: 15.7 GiB | 20260507 04:20:09.581 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:20:09.581 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:20:09.581 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:20:09.581 | INFO | High Water Breakdown: | 20260507 04:20:09.581 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:20:09.581 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:20:09.581 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:20:09.581 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:20:09.581 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:20:09.581 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:20:09.581 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:20:09.581 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:20:09.581 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:20:09.581 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:20:09.581 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:20:09.581 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:20:09.581 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:20:09.581 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:20:09.581 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:20:09.581 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:20:09.581 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:20:09.581 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:20:09.581 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:20:09.581 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:20:09.581 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:20:09.581 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:20:09.581 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:20:09.581 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:20:09.581 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:20:09.581 | INFO | ------------------------------------------ | 20260507 04:20:09.581 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:20:09.581 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:20:09.581 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:20:09.581 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:20:09.581 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.51e+01, 1.34e+01 / 1.98e+00 | 20260507 04:20:09.581 | INFO | PerfReport: 0.30%, AggregateStats -- 9.13e-02 / 1.01e-01, 9.45e-03 / 5.99e-03 | 20260507 04:20:09.581 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.41e+00, 5.19e-01 / 2.89e-01 | 20260507 04:20:09.581 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.47e+00, 1.74e-01 / 9.41e-02 | 20260507 04:20:09.581 | INFO | PerfReport: 16.17%, DME -- 3.98e+01 / 4.47e+01, 4.09e+00 / 2.46e+00 | 20260507 04:20:09.581 | INFO | PerfReport: 57.64%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.32e+00 / 1.47e+00 | 20260507 04:20:09.581 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.68e+00, 3.76e-01 / 2.29e-01 | 20260507 04:20:09.581 | INFO | PerfReport: 3.02%, Metrics -- 9.29e-01 / 1.25e+00, 2.47e-01 / 9.44e-02 | 20260507 04:20:09.581 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.14e-01 / 3.64e-01, 4.99e-02 / 1.54e-02 | 20260507 04:20:09.581 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:20:09.581 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:20:09.581 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:20:09.581 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 04:20:09.581 | INFO | PerfReport: 21.80%, Upload -- 2.10e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260507 04:20:09.582 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 04:20:09.582 | INFO | PerfReport: 31.99%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:20:09.582 | INFO | PerfReport: 1.42%, LockUpload -- 1.36e+00 / 0.00e+00, 4.73e+00 / 0.00e+00 | 20260507 04:20:09.582 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260507 04:20:09.582 | INFO | PerfReport: 22.09%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 04:20:09.582 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.16e-02 / 0.00e+00 | 20260507 04:20:09.678 | INFO | Starting a round of Baz IO | 20260507 04:20:12.782 | INFO | Populating RTMetrics report | 20260507 04:20:12.782 | INFO | RTMetrics report populated | 20260507 04:20:12.782 | INFO | Opening tmp RTMetrics json file | 20260507 04:20:12.800 | INFO | Serializing RTMetrics to json | 20260507 04:20:12.801 | INFO | Serialization done, swapping tmp file now... | 20260507 04:20:12.802 | INFO | RTMetrics done | 20260507 04:20:12.869 | INFO | Finished a round of Baz IO: Wrote 1.728 GiB in 3.191 seconds | 20260507 04:20:12.869 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.80% / 0.00% / 67.20% | 20260507 04:20:12.869 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.173604% / 0% / 9.13938e-05% | 20260507 04:20:12.869 | INFO | Baz IO overall: | 20260507 04:20:12.869 | INFO | BytesWritten : 3598063320800 | 20260507 04:20:12.869 | INFO | NumEvents : 1727532141846 | 20260507 04:20:14.713 | INFO | Analyzing chunk frames = [7045120,7045632) | 20260507 04:20:14.713 | INFO | MemInfo: { | 20260507 04:20:14.713 | INFO | "gpu" : " 17 GB", | 20260507 04:20:14.713 | INFO | "huge" : "", | 20260507 04:20:14.713 | INFO | "huge_shared" : " 27 GB", | 20260507 04:20:14.713 | INFO | "malloc" : " 14 GB", | 20260507 04:20:14.713 | INFO | "pinned" : " 29 GB" | 20260507 04:20:14.713 | INFO | } | 20260507 04:20:14.713 | INFO | Perf report for the last 5.13331 seconds: | 20260507 04:20:14.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:20:14.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:14.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:14.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:14.713 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 04:20:14.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:14.713 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:20:14.713 | INFO | PRE_HQ : 59.2%, 1.8, 55.2ms, 0 | 20260507 04:20:14.713 | INFO | BAZWRITER : 61.8%, 3.2, 102.9ms, 0 | 20260507 04:20:14.713 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:20:14.713 | INFO | Uploaded 36GiB to GPU in 2.04917s (17.5681GiB/s) | 20260507 04:20:14.714 | INFO | Downloaded 38GiB to GPU in 2.18247s (17.4114GiB/s) | 20260507 04:20:19.848 | INFO | Analyzing chunk frames = [7045632,7046144) | 20260507 04:20:19.848 | INFO | MemInfo: { | 20260507 04:20:19.848 | INFO | "gpu" : " 17 GB", | 20260507 04:20:19.848 | INFO | "huge" : "", | 20260507 04:20:19.848 | INFO | "huge_shared" : " 27 GB", | 20260507 04:20:19.848 | INFO | "malloc" : " 14 GB", | 20260507 04:20:19.848 | INFO | "pinned" : " 29 GB" | 20260507 04:20:19.848 | INFO | } | 20260507 04:20:19.848 | INFO | Perf report for the last 5.13474 seconds: | 20260507 04:20:19.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:20:19.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:19.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:19.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:19.848 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 04:20:19.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:19.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:19.848 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 04:20:19.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:19.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:19.848 | INFO | Uploaded 36GiB to GPU in 2.0053s (17.9525GiB/s) | 20260507 04:20:19.848 | INFO | Downloaded 34GiB to GPU in 1.92916s (17.6242GiB/s) | 20260507 04:20:24.991 | INFO | Analyzing chunk frames = [7046144,7046656) | 20260507 04:20:24.991 | INFO | MemInfo: { | 20260507 04:20:24.991 | INFO | "gpu" : " 17 GB", | 20260507 04:20:24.991 | INFO | "huge" : "", | 20260507 04:20:24.991 | INFO | "huge_shared" : " 27 GB", | 20260507 04:20:24.991 | INFO | "malloc" : " 14 GB", | 20260507 04:20:24.991 | INFO | "pinned" : " 29 GB" | 20260507 04:20:24.991 | INFO | } | 20260507 04:20:24.991 | INFO | Perf report for the last 5.14281 seconds: | 20260507 04:20:24.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:20:24.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:24.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:24.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:24.991 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 04:20:24.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:24.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:24.991 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:20:24.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:24.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:24.991 | INFO | Uploaded 36GiB to GPU in 2.01309s (17.883GiB/s) | 20260507 04:20:24.991 | INFO | Downloaded 34GiB to GPU in 1.94227s (17.5053GiB/s) | 20260507 04:20:30.126 | INFO | Analyzing chunk frames = [7046656,7047168) | 20260507 04:20:30.126 | INFO | MemInfo: { | 20260507 04:20:30.126 | INFO | "gpu" : " 17 GB", | 20260507 04:20:30.126 | INFO | "huge" : "", | 20260507 04:20:30.126 | INFO | "huge_shared" : " 27 GB", | 20260507 04:20:30.126 | INFO | "malloc" : " 14 GB", | 20260507 04:20:30.126 | INFO | "pinned" : " 29 GB" | 20260507 04:20:30.126 | INFO | } | 20260507 04:20:30.126 | INFO | Perf report for the last 5.13509 seconds: | 20260507 04:20:30.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:20:30.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:30.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:30.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:30.126 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 04:20:30.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:30.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:30.126 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:20:30.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:30.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:30.126 | INFO | Uploaded 36GiB to GPU in 1.98631s (18.124GiB/s) | 20260507 04:20:30.126 | INFO | Downloaded 34GiB to GPU in 1.91549s (17.75GiB/s) | 20260507 04:20:35.275 | INFO | Analyzing chunk frames = [7047168,7047680) | 20260507 04:20:35.275 | INFO | MemInfo: { | 20260507 04:20:35.275 | INFO | "gpu" : " 17 GB", | 20260507 04:20:35.276 | INFO | "huge" : "", | 20260507 04:20:35.276 | INFO | "huge_shared" : " 27 GB", | 20260507 04:20:35.276 | INFO | "malloc" : " 14 GB", | 20260507 04:20:35.276 | INFO | "pinned" : " 29 GB" | 20260507 04:20:35.276 | INFO | } | 20260507 04:20:35.276 | INFO | Perf report for the last 5.14973 seconds: | 20260507 04:20:35.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:20:35.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:35.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:35.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:35.276 | INFO | ANALYSIS : 59.8%, 3.0, 93.2ms, 0 | 20260507 04:20:35.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:35.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:35.276 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 04:20:35.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:35.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:35.276 | INFO | Uploaded 36GiB to GPU in 2.01335s (17.8806GiB/s) | 20260507 04:20:35.276 | INFO | Downloaded 34GiB to GPU in 1.95069s (17.4297GiB/s) | 20260507 04:20:40.409 | INFO | Analyzing chunk frames = [7047680,7048192) | 20260507 04:20:40.409 | INFO | MemInfo: { | 20260507 04:20:40.409 | INFO | "gpu" : " 17 GB", | 20260507 04:20:40.409 | INFO | "huge" : "", | 20260507 04:20:40.409 | INFO | "huge_shared" : " 27 GB", | 20260507 04:20:40.410 | INFO | "malloc" : " 14 GB", | 20260507 04:20:40.410 | INFO | "pinned" : " 29 GB" | 20260507 04:20:40.410 | INFO | } | 20260507 04:20:40.410 | INFO | Perf report for the last 5.13431 seconds: | 20260507 04:20:40.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:20:40.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:40.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:40.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:40.410 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 04:20:40.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:40.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:40.410 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 04:20:40.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:40.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:40.410 | INFO | Uploaded 36GiB to GPU in 2.05012s (17.5599GiB/s) | 20260507 04:20:40.410 | INFO | Downloaded 34GiB to GPU in 1.89937s (17.9006GiB/s) | 20260507 04:20:45.547 | INFO | Analyzing chunk frames = [7048192,7048704) | 20260507 04:20:45.547 | INFO | MemInfo: { | 20260507 04:20:45.547 | INFO | "gpu" : " 17 GB", | 20260507 04:20:45.547 | INFO | "huge" : "", | 20260507 04:20:45.547 | INFO | "huge_shared" : " 27 GB", | 20260507 04:20:45.547 | INFO | "malloc" : " 14 GB", | 20260507 04:20:45.547 | INFO | "pinned" : " 29 GB" | 20260507 04:20:45.547 | INFO | } | 20260507 04:20:45.547 | INFO | Perf report for the last 5.13675 seconds: | 20260507 04:20:45.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:20:45.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:45.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:45.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:45.547 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 04:20:45.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:45.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:45.547 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 04:20:45.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:45.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:45.547 | INFO | Uploaded 36GiB to GPU in 2.03142s (17.7216GiB/s) | 20260507 04:20:45.547 | INFO | Downloaded 34GiB to GPU in 1.98247s (17.1503GiB/s) | 20260507 04:20:50.685 | INFO | Analyzing chunk frames = [7048704,7049216) | 20260507 04:20:50.685 | INFO | MemInfo: { | 20260507 04:20:50.685 | INFO | "gpu" : " 17 GB", | 20260507 04:20:50.685 | INFO | "huge" : "", | 20260507 04:20:50.685 | INFO | "huge_shared" : " 27 GB", | 20260507 04:20:50.685 | INFO | "malloc" : " 14 GB", | 20260507 04:20:50.685 | INFO | "pinned" : " 29 GB" | 20260507 04:20:50.685 | INFO | } | 20260507 04:20:50.685 | INFO | Perf report for the last 5.13801 seconds: | 20260507 04:20:50.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:20:50.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:50.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:50.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:50.685 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:20:50.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:50.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:50.685 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:20:50.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:50.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:50.685 | INFO | Uploaded 36GiB to GPU in 1.97065s (18.2681GiB/s) | 20260507 04:20:50.685 | INFO | Downloaded 34GiB to GPU in 1.9576s (17.3682GiB/s) | 20260507 04:20:50.781 | INFO | Starting a round of Baz IO | 20260507 04:20:53.887 | INFO | Populating RTMetrics report | 20260507 04:20:53.888 | INFO | RTMetrics report populated | 20260507 04:20:53.888 | INFO | Opening tmp RTMetrics json file | 20260507 04:20:53.891 | INFO | Serializing RTMetrics to json | 20260507 04:20:53.906 | INFO | Serialization done, swapping tmp file now... | 20260507 04:20:53.906 | INFO | RTMetrics done | 20260507 04:20:53.977 | INFO | Finished a round of Baz IO: Wrote 1.727 GiB in 3.196 seconds | 20260507 04:20:53.977 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.80% / 0.00% / 67.20% | 20260507 04:20:53.977 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.172806% / 0% / 9.53674e-05% | 20260507 04:20:53.977 | INFO | Baz IO overall: | 20260507 04:20:53.977 | INFO | BytesWritten : 3599917832704 | 20260507 04:20:53.978 | INFO | NumEvents : 1728405033696 | 20260507 04:20:55.827 | INFO | Analyzing chunk frames = [7049216,7049728) | 20260507 04:20:55.827 | INFO | MemInfo: { | 20260507 04:20:55.827 | INFO | "gpu" : " 17 GB", | 20260507 04:20:55.827 | INFO | "huge" : "", | 20260507 04:20:55.827 | INFO | "huge_shared" : " 27 GB", | 20260507 04:20:55.827 | INFO | "malloc" : " 14 GB", | 20260507 04:20:55.827 | INFO | "pinned" : " 29 GB" | 20260507 04:20:55.827 | INFO | } | 20260507 04:20:55.827 | INFO | Perf report for the last 5.1419 seconds: | 20260507 04:20:55.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:20:55.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:55.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:55.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:55.827 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 04:20:55.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:20:55.827 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 04:20:55.827 | INFO | PRE_HQ : 59.0%, 1.8, 54.9ms, 0 | 20260507 04:20:55.827 | INFO | BAZWRITER : 61.8%, 3.2, 102.8ms, 0 | 20260507 04:20:55.827 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:20:55.827 | INFO | Uploaded 36GiB to GPU in 2.04161s (17.6331GiB/s) | 20260507 04:20:55.827 | INFO | Downloaded 38GiB to GPU in 2.18184s (17.4165GiB/s) | 20260507 04:21:00.966 | INFO | Analyzing chunk frames = [7049728,7050240) | 20260507 04:21:00.966 | INFO | MemInfo: { | 20260507 04:21:00.966 | INFO | "gpu" : " 17 GB", | 20260507 04:21:00.966 | INFO | "huge" : "", | 20260507 04:21:00.966 | INFO | "huge_shared" : " 27 GB", | 20260507 04:21:00.966 | INFO | "malloc" : " 14 GB", | 20260507 04:21:00.966 | INFO | "pinned" : " 29 GB" | 20260507 04:21:00.966 | INFO | } | 20260507 04:21:00.966 | INFO | Perf report for the last 5.13974 seconds: | 20260507 04:21:00.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:21:00.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:00.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:00.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:00.967 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 04:21:00.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:00.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:00.967 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:21:00.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:00.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:00.967 | INFO | Uploaded 36GiB to GPU in 2.02079s (17.8148GiB/s) | 20260507 04:21:00.967 | INFO | Downloaded 34GiB to GPU in 1.93215s (17.597GiB/s) | 20260507 04:21:06.112 | INFO | Analyzing chunk frames = [7050240,7050752) | 20260507 04:21:06.112 | INFO | MemInfo: { | 20260507 04:21:06.112 | INFO | "gpu" : " 17 GB", | 20260507 04:21:06.112 | INFO | "huge" : "", | 20260507 04:21:06.112 | INFO | "huge_shared" : " 27 GB", | 20260507 04:21:06.113 | INFO | "malloc" : " 14 GB", | 20260507 04:21:06.113 | INFO | "pinned" : " 29 GB" | 20260507 04:21:06.113 | INFO | } | 20260507 04:21:06.113 | INFO | Perf report for the last 5.14616 seconds: | 20260507 04:21:06.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:21:06.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:06.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:06.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:06.113 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 04:21:06.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:06.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:06.113 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:21:06.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:06.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:06.113 | INFO | Uploaded 36GiB to GPU in 2.00801s (17.9282GiB/s) | 20260507 04:21:06.113 | INFO | Downloaded 34GiB to GPU in 1.94206s (17.5072GiB/s) | 20260507 04:21:11.245 | INFO | Analyzing chunk frames = [7050752,7051264) | 20260507 04:21:11.245 | INFO | MemInfo: { | 20260507 04:21:11.245 | INFO | "gpu" : " 17 GB", | 20260507 04:21:11.245 | INFO | "huge" : "", | 20260507 04:21:11.245 | INFO | "huge_shared" : " 27 GB", | 20260507 04:21:11.245 | INFO | "malloc" : " 14 GB", | 20260507 04:21:11.245 | INFO | "pinned" : " 29 GB" | 20260507 04:21:11.245 | INFO | } | 20260507 04:21:11.245 | INFO | Perf report for the last 5.13258 seconds: | 20260507 04:21:11.245 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:21:11.245 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:11.245 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:11.245 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:11.245 | INFO | ANALYSIS : 59.9%, 3.0, 92.7ms, 0 | 20260507 04:21:11.245 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:11.245 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:11.245 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 04:21:11.245 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:11.245 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:11.246 | INFO | Uploaded 36GiB to GPU in 2.01902s (17.8305GiB/s) | 20260507 04:21:11.246 | INFO | Downloaded 34GiB to GPU in 1.94197s (17.508GiB/s) | 20260507 04:21:16.382 | INFO | Analyzing chunk frames = [7051264,7051776) | 20260507 04:21:16.382 | INFO | MemInfo: { | 20260507 04:21:16.382 | INFO | "gpu" : " 17 GB", | 20260507 04:21:16.382 | INFO | "huge" : "", | 20260507 04:21:16.382 | INFO | "huge_shared" : " 27 GB", | 20260507 04:21:16.382 | INFO | "malloc" : " 14 GB", | 20260507 04:21:16.382 | INFO | "pinned" : " 29 GB" | 20260507 04:21:16.382 | INFO | } | 20260507 04:21:16.382 | INFO | Perf report for the last 5.13681 seconds: | 20260507 04:21:16.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:21:16.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:16.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:16.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:16.382 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 04:21:16.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:16.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:16.382 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 04:21:16.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:16.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:16.382 | INFO | Uploaded 36GiB to GPU in 2.04605s (17.5948GiB/s) | 20260507 04:21:16.382 | INFO | Downloaded 34GiB to GPU in 1.94141s (17.5131GiB/s) | 20260507 04:21:21.524 | INFO | Analyzing chunk frames = [7051776,7052288) | 20260507 04:21:21.524 | INFO | MemInfo: { | 20260507 04:21:21.524 | INFO | "gpu" : " 17 GB", | 20260507 04:21:21.524 | INFO | "huge" : "", | 20260507 04:21:21.524 | INFO | "huge_shared" : " 27 GB", | 20260507 04:21:21.524 | INFO | "malloc" : " 14 GB", | 20260507 04:21:21.524 | INFO | "pinned" : " 29 GB" | 20260507 04:21:21.524 | INFO | } | 20260507 04:21:21.524 | INFO | Perf report for the last 5.14179 seconds: | 20260507 04:21:21.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:21:21.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:21.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:21.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:21.524 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 04:21:21.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:21.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:21.524 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 04:21:21.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:21.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:21.524 | INFO | Uploaded 36GiB to GPU in 2.05831s (17.4901GiB/s) | 20260507 04:21:21.524 | INFO | Downloaded 34GiB to GPU in 1.96364s (17.3148GiB/s) | 20260507 04:21:26.662 | INFO | Analyzing chunk frames = [7052288,7052800) | 20260507 04:21:26.662 | INFO | MemInfo: { | 20260507 04:21:26.662 | INFO | "gpu" : " 17 GB", | 20260507 04:21:26.662 | INFO | "huge" : "", | 20260507 04:21:26.662 | INFO | "huge_shared" : " 27 GB", | 20260507 04:21:26.662 | INFO | "malloc" : " 14 GB", | 20260507 04:21:26.662 | INFO | "pinned" : " 29 GB" | 20260507 04:21:26.662 | INFO | } | 20260507 04:21:26.662 | INFO | Perf report for the last 5.13827 seconds: | 20260507 04:21:26.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:21:26.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:26.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:26.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:26.662 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:21:26.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:26.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:26.662 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:21:26.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:26.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:26.662 | INFO | Uploaded 36GiB to GPU in 1.96199s (18.3487GiB/s) | 20260507 04:21:26.662 | INFO | Downloaded 34GiB to GPU in 1.96653s (17.2893GiB/s) | 20260507 04:21:31.798 | INFO | Analyzing chunk frames = [7052800,7053312) | 20260507 04:21:31.798 | INFO | MemInfo: { | 20260507 04:21:31.798 | INFO | "gpu" : " 17 GB", | 20260507 04:21:31.798 | INFO | "huge" : "", | 20260507 04:21:31.798 | INFO | "huge_shared" : " 27 GB", | 20260507 04:21:31.798 | INFO | "malloc" : " 14 GB", | 20260507 04:21:31.798 | INFO | "pinned" : " 29 GB" | 20260507 04:21:31.798 | INFO | } | 20260507 04:21:31.798 | INFO | Perf report for the last 5.13578 seconds: | 20260507 04:21:31.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:21:31.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:31.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:31.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:31.798 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:21:31.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:31.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:31.798 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:21:31.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:31.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:31.798 | INFO | Uploaded 36GiB to GPU in 1.95938s (18.3731GiB/s) | 20260507 04:21:31.798 | INFO | Downloaded 34GiB to GPU in 1.97696s (17.1982GiB/s) | 20260507 04:21:31.798 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:21:31.798 | INFO | Total mem usage: 18 GiB | 20260507 04:21:31.798 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:21:31.798 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 04:21:31.798 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:21:31.798 | INFO | High Water Breakdown: | 20260507 04:21:31.798 | INFO | 60.1% : 8.57 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:21:31.798 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:21:31.798 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:21:31.798 | INFO | ------------------------------------------ | 20260507 04:21:31.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:21:31.798 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:21:31.798 | INFO | Total mem usage: 26.5 GiB | 20260507 04:21:31.798 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:21:31.798 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:21:31.798 | INFO | Unused Cache: 1.78 GiB (Waste: 6.7%) | 20260507 04:21:31.798 | INFO | High Water Breakdown: | 20260507 04:21:31.798 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:21:31.798 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:21:31.798 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:21:31.798 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:21:31.798 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:21:31.798 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:21:31.798 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:21:31.798 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:21:31.798 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:21:31.798 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:21:31.798 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:21:31.798 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:21:31.798 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:21:31.798 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:21:31.798 | INFO | ------------------------------------------ | 20260507 04:21:31.798 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:21:31.799 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:21:31.799 | INFO | Total mem usage: 25.6 GiB | 20260507 04:21:31.799 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:21:31.799 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:21:31.799 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:21:31.799 | INFO | High Water Breakdown: | 20260507 04:21:31.799 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:21:31.799 | INFO | ------------------------------------------ | 20260507 04:21:31.799 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:21:31.799 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:21:31.799 | INFO | Total mem usage: 15.7 GiB | 20260507 04:21:31.799 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:21:31.799 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:21:31.799 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:21:31.799 | INFO | High Water Breakdown: | 20260507 04:21:31.799 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:21:31.799 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:21:31.799 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:21:31.799 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:21:31.799 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:21:31.799 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:21:31.799 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:21:31.799 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:21:31.799 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:21:31.799 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:21:31.799 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:21:31.799 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:21:31.799 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:21:31.799 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:21:31.799 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:21:31.799 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:21:31.799 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:21:31.799 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:21:31.799 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:21:31.799 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:21:31.799 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:21:31.799 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:21:31.799 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:21:31.799 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:21:31.799 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:21:31.799 | INFO | ------------------------------------------ | 20260507 04:21:31.799 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:21:31.799 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:21:31.799 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:21:31.799 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:21:31.799 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.33e+01 / 1.14e+00 | 20260507 04:21:31.799 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 9.24e-03 / 5.46e-03 | 20260507 04:21:31.799 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.36e+00, 5.08e-01 / 3.63e-01 | 20260507 04:21:31.799 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.45e+00, 1.69e-01 / 1.33e-01 | 20260507 04:21:31.799 | INFO | PerfReport: 16.07%, DME -- 3.95e+01 / 4.36e+01, 3.66e+00 / 1.44e+00 | 20260507 04:21:31.799 | INFO | PerfReport: 57.76%, FrameLabeling -- 1.77e+01 / 2.08e+01, 1.31e+00 / 1.55e+00 | 20260507 04:21:31.799 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.13e+00 / 2.66e+00, 3.79e-01 / 2.52e-01 | 20260507 04:21:31.799 | INFO | PerfReport: 3.03%, Metrics -- 9.29e-01 / 1.24e+00, 2.44e-01 / 1.67e-01 | 20260507 04:21:31.799 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.14e-01 / 3.69e-01, 4.91e-02 / 2.67e-02 | 20260507 04:21:31.799 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:21:31.799 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:21:31.799 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:21:31.799 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 04:21:31.799 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260507 04:21:31.799 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260507 04:21:31.799 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 04:21:31.799 | INFO | PerfReport: 1.51%, LockUpload -- 1.45e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 04:21:31.799 | INFO | PerfReport: 1.23%, LockDownload -- 1.18e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260507 04:21:31.799 | INFO | PerfReport: 22.12%, LockCompute -- 2.13e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 04:21:31.799 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260507 04:21:31.895 | INFO | Starting a round of Baz IO | 20260507 04:21:35.009 | INFO | Populating RTMetrics report | 20260507 04:21:35.009 | INFO | RTMetrics report populated | 20260507 04:21:35.009 | INFO | Opening tmp RTMetrics json file | 20260507 04:21:35.026 | INFO | Serializing RTMetrics to json | 20260507 04:21:35.027 | INFO | Serialization done, swapping tmp file now... | 20260507 04:21:35.027 | INFO | RTMetrics done | 20260507 04:21:35.097 | INFO | Finished a round of Baz IO: Wrote 1.725 GiB in 3.201 seconds | 20260507 04:21:35.097 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.80% / 0.00% / 67.20% | 20260507 04:21:35.097 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.171506% / 3.97364e-06% / 0.000103315% | 20260507 04:21:35.097 | INFO | Baz IO overall: | 20260507 04:21:35.097 | INFO | BytesWritten : 3601770150225 | 20260507 04:21:35.097 | INFO | NumEvents : 1729276225025 | 20260507 04:21:36.944 | INFO | Analyzing chunk frames = [7053312,7053824) | 20260507 04:21:36.944 | INFO | MemInfo: { | 20260507 04:21:36.944 | INFO | "gpu" : " 17 GB", | 20260507 04:21:36.944 | INFO | "huge" : "", | 20260507 04:21:36.944 | INFO | "huge_shared" : " 27 GB", | 20260507 04:21:36.944 | INFO | "malloc" : " 14 GB", | 20260507 04:21:36.944 | INFO | "pinned" : " 29 GB" | 20260507 04:21:36.944 | INFO | } | 20260507 04:21:36.944 | INFO | Perf report for the last 5.14606 seconds: | 20260507 04:21:36.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:21:36.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:36.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:36.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:36.944 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260507 04:21:36.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:36.944 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 04:21:36.944 | INFO | PRE_HQ : 59.2%, 1.8, 55.5ms, 0 | 20260507 04:21:36.944 | INFO | BAZWRITER : 61.8%, 3.2, 101.5ms, 0 | 20260507 04:21:36.944 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:21:36.944 | INFO | Uploaded 36GiB to GPU in 2.02831s (17.7487GiB/s) | 20260507 04:21:36.944 | INFO | Downloaded 38GiB to GPU in 2.17982s (17.4326GiB/s) | 20260507 04:21:42.088 | INFO | Analyzing chunk frames = [7053824,7054336) | 20260507 04:21:42.088 | INFO | MemInfo: { | 20260507 04:21:42.088 | INFO | "gpu" : " 17 GB", | 20260507 04:21:42.088 | INFO | "huge" : "", | 20260507 04:21:42.088 | INFO | "huge_shared" : " 27 GB", | 20260507 04:21:42.088 | INFO | "malloc" : " 14 GB", | 20260507 04:21:42.088 | INFO | "pinned" : " 29 GB" | 20260507 04:21:42.088 | INFO | } | 20260507 04:21:42.088 | INFO | Perf report for the last 5.1437 seconds: | 20260507 04:21:42.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:21:42.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:42.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:42.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:42.088 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 04:21:42.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:42.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:42.088 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:21:42.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:42.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:42.088 | INFO | Uploaded 36GiB to GPU in 2.01252s (17.8881GiB/s) | 20260507 04:21:42.088 | INFO | Downloaded 34GiB to GPU in 1.94191s (17.5085GiB/s) | 20260507 04:21:47.226 | INFO | Analyzing chunk frames = [7054336,7054848) | 20260507 04:21:47.226 | INFO | MemInfo: { | 20260507 04:21:47.226 | INFO | "gpu" : " 17 GB", | 20260507 04:21:47.226 | INFO | "huge" : "", | 20260507 04:21:47.226 | INFO | "huge_shared" : " 27 GB", | 20260507 04:21:47.226 | INFO | "malloc" : " 14 GB", | 20260507 04:21:47.226 | INFO | "pinned" : " 29 GB" | 20260507 04:21:47.226 | INFO | } | 20260507 04:21:47.226 | INFO | Perf report for the last 5.13821 seconds: | 20260507 04:21:47.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:21:47.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:47.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:47.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:47.226 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 04:21:47.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:47.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:47.226 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:21:47.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:47.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:47.226 | INFO | Uploaded 36GiB to GPU in 1.99002s (18.0902GiB/s) | 20260507 04:21:47.226 | INFO | Downloaded 34GiB to GPU in 1.94368s (17.4926GiB/s) | 20260507 04:21:52.363 | INFO | Analyzing chunk frames = [7054848,7055360) | 20260507 04:21:52.363 | INFO | MemInfo: { | 20260507 04:21:52.363 | INFO | "gpu" : " 17 GB", | 20260507 04:21:52.363 | INFO | "huge" : "", | 20260507 04:21:52.363 | INFO | "huge_shared" : " 27 GB", | 20260507 04:21:52.363 | INFO | "malloc" : " 14 GB", | 20260507 04:21:52.363 | INFO | "pinned" : " 29 GB" | 20260507 04:21:52.363 | INFO | } | 20260507 04:21:52.363 | INFO | Perf report for the last 5.13716 seconds: | 20260507 04:21:52.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:21:52.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:52.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:52.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:52.363 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:21:52.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:52.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:52.363 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:21:52.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:52.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:52.363 | INFO | Uploaded 36GiB to GPU in 1.99416s (18.0527GiB/s) | 20260507 04:21:52.363 | INFO | Downloaded 34GiB to GPU in 1.93799s (17.5439GiB/s) | 20260507 04:21:57.493 | INFO | Analyzing chunk frames = [7055360,7055872) | 20260507 04:21:57.493 | INFO | MemInfo: { | 20260507 04:21:57.493 | INFO | "gpu" : " 17 GB", | 20260507 04:21:57.493 | INFO | "huge" : "", | 20260507 04:21:57.493 | INFO | "huge_shared" : " 27 GB", | 20260507 04:21:57.493 | INFO | "malloc" : " 14 GB", | 20260507 04:21:57.493 | INFO | "pinned" : " 29 GB" | 20260507 04:21:57.493 | INFO | } | 20260507 04:21:57.493 | INFO | Perf report for the last 5.13004 seconds: | 20260507 04:21:57.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:21:57.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:57.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:57.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:57.493 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 04:21:57.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:57.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:57.493 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 04:21:57.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:57.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:21:57.493 | INFO | Uploaded 36GiB to GPU in 2.04758s (17.5817GiB/s) | 20260507 04:21:57.493 | INFO | Downloaded 34GiB to GPU in 1.94859s (17.4485GiB/s) | 20260507 04:22:02.637 | INFO | Analyzing chunk frames = [7055872,7056384) | 20260507 04:22:02.637 | INFO | MemInfo: { | 20260507 04:22:02.637 | INFO | "gpu" : " 17 GB", | 20260507 04:22:02.637 | INFO | "huge" : "", | 20260507 04:22:02.637 | INFO | "huge_shared" : " 27 GB", | 20260507 04:22:02.637 | INFO | "malloc" : " 14 GB", | 20260507 04:22:02.637 | INFO | "pinned" : " 29 GB" | 20260507 04:22:02.637 | INFO | } | 20260507 04:22:02.637 | INFO | Perf report for the last 5.14391 seconds: | 20260507 04:22:02.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:22:02.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:02.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:02.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:02.637 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260507 04:22:02.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:02.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:02.637 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 04:22:02.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:02.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:02.637 | INFO | Uploaded 36GiB to GPU in 2.03454s (17.6944GiB/s) | 20260507 04:22:02.637 | INFO | Downloaded 34GiB to GPU in 1.94972s (17.4384GiB/s) | 20260507 04:22:07.779 | INFO | Analyzing chunk frames = [7056384,7056896) | 20260507 04:22:07.779 | INFO | MemInfo: { | 20260507 04:22:07.779 | INFO | "gpu" : " 17 GB", | 20260507 04:22:07.779 | INFO | "huge" : "", | 20260507 04:22:07.779 | INFO | "huge_shared" : " 27 GB", | 20260507 04:22:07.779 | INFO | "malloc" : " 14 GB", | 20260507 04:22:07.779 | INFO | "pinned" : " 29 GB" | 20260507 04:22:07.779 | INFO | } | 20260507 04:22:07.779 | INFO | Perf report for the last 5.14219 seconds: | 20260507 04:22:07.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:22:07.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:07.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:07.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:07.779 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 04:22:07.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:07.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:07.779 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 04:22:07.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:07.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:07.780 | INFO | Uploaded 36GiB to GPU in 1.97245s (18.2514GiB/s) | 20260507 04:22:07.780 | INFO | Downloaded 34GiB to GPU in 1.95842s (17.3609GiB/s) | 20260507 04:22:12.924 | INFO | Analyzing chunk frames = [7056896,7057408) | 20260507 04:22:12.924 | INFO | MemInfo: { | 20260507 04:22:12.924 | INFO | "gpu" : " 17 GB", | 20260507 04:22:12.924 | INFO | "huge" : "", | 20260507 04:22:12.924 | INFO | "huge_shared" : " 27 GB", | 20260507 04:22:12.924 | INFO | "malloc" : " 14 GB", | 20260507 04:22:12.924 | INFO | "pinned" : " 29 GB" | 20260507 04:22:12.924 | INFO | } | 20260507 04:22:12.924 | INFO | Perf report for the last 5.14469 seconds: | 20260507 04:22:12.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:22:12.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:12.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:12.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:12.924 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 04:22:12.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:12.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:12.924 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 04:22:12.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:12.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:12.924 | INFO | Uploaded 36GiB to GPU in 1.97693s (18.2101GiB/s) | 20260507 04:22:12.924 | INFO | Downloaded 34GiB to GPU in 1.94713s (17.4616GiB/s) | 20260507 04:22:13.021 | INFO | Starting a round of Baz IO | 20260507 04:22:16.153 | INFO | Populating RTMetrics report | 20260507 04:22:16.153 | INFO | RTMetrics report populated | 20260507 04:22:16.153 | INFO | Opening tmp RTMetrics json file | 20260507 04:22:16.156 | INFO | Serializing RTMetrics to json | 20260507 04:22:16.173 | INFO | Serialization done, swapping tmp file now... | 20260507 04:22:16.173 | INFO | RTMetrics done | 20260507 04:22:16.239 | INFO | Finished a round of Baz IO: Wrote 1.726 GiB in 3.218 seconds | 20260507 04:22:16.239 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.80% / 0.00% / 67.20% | 20260507 04:22:16.239 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.171808% / 0% / 6.75519e-05% | 20260507 04:22:16.239 | INFO | Baz IO overall: | 20260507 04:22:16.239 | INFO | BytesWritten : 3603623229743 | 20260507 04:22:16.239 | INFO | NumEvents : 1730148413777 | 20260507 04:22:18.061 | INFO | Analyzing chunk frames = [7057408,7057920) | 20260507 04:22:18.061 | INFO | MemInfo: { | 20260507 04:22:18.061 | INFO | "gpu" : " 17 GB", | 20260507 04:22:18.061 | INFO | "huge" : "", | 20260507 04:22:18.061 | INFO | "huge_shared" : " 27 GB", | 20260507 04:22:18.061 | INFO | "malloc" : " 14 GB", | 20260507 04:22:18.061 | INFO | "pinned" : " 29 GB" | 20260507 04:22:18.061 | INFO | } | 20260507 04:22:18.061 | INFO | Perf report for the last 5.13735 seconds: | 20260507 04:22:18.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:22:18.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:18.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:18.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:18.061 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 04:22:18.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:18.061 | INFO | RT_METRICS : 45.5%, 1.0, 23.8ms, 0 | 20260507 04:22:18.061 | INFO | PRE_HQ : 60.1%, 1.9, 60.6ms, 0 | 20260507 04:22:18.061 | INFO | BAZWRITER : 62.3%, 3.4, 109.4ms, 0 | 20260507 04:22:18.061 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 04:22:18.062 | INFO | Uploaded 36GiB to GPU in 2.05542s (17.5147GiB/s) | 20260507 04:22:18.062 | INFO | Downloaded 38GiB to GPU in 2.17696s (17.4556GiB/s) | 20260507 04:22:23.196 | INFO | Analyzing chunk frames = [7057920,7058432) | 20260507 04:22:23.196 | INFO | MemInfo: { | 20260507 04:22:23.196 | INFO | "gpu" : " 17 GB", | 20260507 04:22:23.196 | INFO | "huge" : "", | 20260507 04:22:23.196 | INFO | "huge_shared" : " 27 GB", | 20260507 04:22:23.196 | INFO | "malloc" : " 14 GB", | 20260507 04:22:23.196 | INFO | "pinned" : " 29 GB" | 20260507 04:22:23.196 | INFO | } | 20260507 04:22:23.196 | INFO | Perf report for the last 5.13497 seconds: | 20260507 04:22:23.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:22:23.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:23.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:23.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:23.196 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:22:23.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:23.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:23.196 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:22:23.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:23.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:23.197 | INFO | Uploaded 36GiB to GPU in 1.99213s (18.0711GiB/s) | 20260507 04:22:23.197 | INFO | Downloaded 34GiB to GPU in 1.92277s (17.6829GiB/s) | 20260507 04:22:28.334 | INFO | Analyzing chunk frames = [7058432,7058944) | 20260507 04:22:28.334 | INFO | MemInfo: { | 20260507 04:22:28.334 | INFO | "gpu" : " 17 GB", | 20260507 04:22:28.334 | INFO | "huge" : "", | 20260507 04:22:28.334 | INFO | "huge_shared" : " 27 GB", | 20260507 04:22:28.335 | INFO | "malloc" : " 14 GB", | 20260507 04:22:28.335 | INFO | "pinned" : " 29 GB" | 20260507 04:22:28.335 | INFO | } | 20260507 04:22:28.335 | INFO | Perf report for the last 5.13833 seconds: | 20260507 04:22:28.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:22:28.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:28.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:28.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:28.335 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 04:22:28.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:28.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:28.335 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:22:28.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:28.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:28.335 | INFO | Uploaded 36GiB to GPU in 2.01998s (17.822GiB/s) | 20260507 04:22:28.335 | INFO | Downloaded 34GiB to GPU in 1.93929s (17.5322GiB/s) | 20260507 04:22:33.473 | INFO | Analyzing chunk frames = [7058944,7059456) | 20260507 04:22:33.473 | INFO | MemInfo: { | 20260507 04:22:33.473 | INFO | "gpu" : " 17 GB", | 20260507 04:22:33.473 | INFO | "huge" : "", | 20260507 04:22:33.473 | INFO | "huge_shared" : " 27 GB", | 20260507 04:22:33.473 | INFO | "malloc" : " 14 GB", | 20260507 04:22:33.473 | INFO | "pinned" : " 29 GB" | 20260507 04:22:33.473 | INFO | } | 20260507 04:22:33.473 | INFO | Perf report for the last 5.13806 seconds: | 20260507 04:22:33.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:22:33.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:33.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:33.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:33.473 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 04:22:33.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:33.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:33.473 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:22:33.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:33.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:33.473 | INFO | Uploaded 36GiB to GPU in 1.97027s (18.2716GiB/s) | 20260507 04:22:33.473 | INFO | Downloaded 34GiB to GPU in 1.91378s (17.7658GiB/s) | 20260507 04:22:38.617 | INFO | Analyzing chunk frames = [7059456,7059968) | 20260507 04:22:38.617 | INFO | MemInfo: { | 20260507 04:22:38.617 | INFO | "gpu" : " 17 GB", | 20260507 04:22:38.617 | INFO | "huge" : "", | 20260507 04:22:38.617 | INFO | "huge_shared" : " 27 GB", | 20260507 04:22:38.617 | INFO | "malloc" : " 14 GB", | 20260507 04:22:38.617 | INFO | "pinned" : " 29 GB" | 20260507 04:22:38.617 | INFO | } | 20260507 04:22:38.617 | INFO | Perf report for the last 5.14402 seconds: | 20260507 04:22:38.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:22:38.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:38.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:38.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:38.617 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 04:22:38.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:38.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:38.617 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:22:38.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:38.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:38.617 | INFO | Uploaded 36GiB to GPU in 2.04236s (17.6267GiB/s) | 20260507 04:22:38.617 | INFO | Downloaded 34GiB to GPU in 1.94001s (17.5256GiB/s) | 20260507 04:22:43.753 | INFO | Analyzing chunk frames = [7059968,7060480) | 20260507 04:22:43.753 | INFO | MemInfo: { | 20260507 04:22:43.753 | INFO | "gpu" : " 17 GB", | 20260507 04:22:43.753 | INFO | "huge" : "", | 20260507 04:22:43.753 | INFO | "huge_shared" : " 27 GB", | 20260507 04:22:43.753 | INFO | "malloc" : " 14 GB", | 20260507 04:22:43.753 | INFO | "pinned" : " 29 GB" | 20260507 04:22:43.753 | INFO | } | 20260507 04:22:43.753 | INFO | Perf report for the last 5.13587 seconds: | 20260507 04:22:43.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:22:43.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:43.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:43.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:43.753 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 04:22:43.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:43.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:43.753 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:22:43.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:43.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:43.753 | INFO | Uploaded 36GiB to GPU in 2.0534s (17.5319GiB/s) | 20260507 04:22:43.753 | INFO | Downloaded 34GiB to GPU in 1.90478s (17.8498GiB/s) | 20260507 04:22:48.899 | INFO | Analyzing chunk frames = [7060480,7060992) | 20260507 04:22:48.899 | INFO | MemInfo: { | 20260507 04:22:48.899 | INFO | "gpu" : " 17 GB", | 20260507 04:22:48.899 | INFO | "huge" : "", | 20260507 04:22:48.899 | INFO | "huge_shared" : " 27 GB", | 20260507 04:22:48.899 | INFO | "malloc" : " 14 GB", | 20260507 04:22:48.899 | INFO | "pinned" : " 29 GB" | 20260507 04:22:48.899 | INFO | } | 20260507 04:22:48.899 | INFO | Perf report for the last 5.14632 seconds: | 20260507 04:22:48.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:22:48.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:48.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:48.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:48.899 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:22:48.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:48.899 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:48.899 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 04:22:48.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:48.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:48.899 | INFO | Uploaded 36GiB to GPU in 1.98286s (18.1556GiB/s) | 20260507 04:22:48.899 | INFO | Downloaded 34GiB to GPU in 1.9722s (17.2396GiB/s) | 20260507 04:22:54.041 | INFO | Analyzing chunk frames = [7060992,7061504) | 20260507 04:22:54.041 | INFO | MemInfo: { | 20260507 04:22:54.041 | INFO | "gpu" : " 17 GB", | 20260507 04:22:54.041 | INFO | "huge" : "", | 20260507 04:22:54.041 | INFO | "huge_shared" : " 27 GB", | 20260507 04:22:54.041 | INFO | "malloc" : " 14 GB", | 20260507 04:22:54.041 | INFO | "pinned" : " 29 GB" | 20260507 04:22:54.041 | INFO | } | 20260507 04:22:54.041 | INFO | Perf report for the last 5.1418 seconds: | 20260507 04:22:54.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:22:54.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:54.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:54.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:54.041 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 04:22:54.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:54.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:54.041 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:22:54.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:54.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:54.041 | INFO | Uploaded 36GiB to GPU in 1.96957s (18.2781GiB/s) | 20260507 04:22:54.041 | INFO | Downloaded 34GiB to GPU in 1.96503s (17.3025GiB/s) | 20260507 04:22:54.041 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:22:54.041 | INFO | Total mem usage: 18 GiB | 20260507 04:22:54.041 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:22:54.041 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 04:22:54.041 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:22:54.041 | INFO | High Water Breakdown: | 20260507 04:22:54.041 | INFO | 60.2% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:22:54.041 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:22:54.041 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:22:54.041 | INFO | ------------------------------------------ | 20260507 04:22:54.041 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:22:54.041 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:22:54.041 | INFO | Total mem usage: 26.5 GiB | 20260507 04:22:54.041 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:22:54.041 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:22:54.041 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:22:54.042 | INFO | High Water Breakdown: | 20260507 04:22:54.042 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:22:54.042 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:22:54.042 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:22:54.042 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:22:54.042 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:22:54.042 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:22:54.042 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:22:54.042 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:22:54.042 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:22:54.042 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:22:54.042 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:22:54.042 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:22:54.042 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:22:54.042 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:22:54.042 | INFO | ------------------------------------------ | 20260507 04:22:54.042 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:22:54.042 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:22:54.042 | INFO | Total mem usage: 25.6 GiB | 20260507 04:22:54.042 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:22:54.042 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:22:54.042 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:22:54.042 | INFO | High Water Breakdown: | 20260507 04:22:54.042 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:22:54.042 | INFO | ------------------------------------------ | 20260507 04:22:54.042 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:22:54.042 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:22:54.042 | INFO | Total mem usage: 15.7 GiB | 20260507 04:22:54.042 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:22:54.042 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:22:54.042 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:22:54.042 | INFO | High Water Breakdown: | 20260507 04:22:54.042 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:22:54.042 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:22:54.042 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:22:54.042 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:22:54.042 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:22:54.042 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:22:54.042 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:22:54.042 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:22:54.042 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:22:54.042 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:22:54.042 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:22:54.042 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:22:54.042 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:22:54.042 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:22:54.042 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:22:54.042 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:22:54.042 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:22:54.042 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:22:54.042 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:22:54.042 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:22:54.042 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:22:54.042 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:22:54.042 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:22:54.042 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:22:54.042 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:22:54.042 | INFO | ------------------------------------------ | 20260507 04:22:54.042 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:22:54.042 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:22:54.042 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:22:54.042 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:22:54.042 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.50e+01, 1.34e+01 / 1.43e+00 | 20260507 04:22:54.042 | INFO | PerfReport: 0.30%, AggregateStats -- 9.12e-02 / 1.01e-01, 1.01e-02 / 4.83e-03 | 20260507 04:22:54.042 | INFO | PerfReport: 11.69%, Baseline -- 3.60e+00 / 4.45e+00, 5.26e-01 / 3.24e-01 | 20260507 04:22:54.042 | INFO | PerfReport: 3.24%, Histogram -- 9.97e-01 / 1.44e+00, 1.69e-01 / 1.17e-01 | 20260507 04:22:54.042 | INFO | PerfReport: 16.08%, DME -- 3.96e+01 / 4.41e+01, 3.82e+00 / 1.53e+00 | 20260507 04:22:54.042 | INFO | PerfReport: 57.76%, FrameLabeling -- 1.78e+01 / 2.07e+01, 1.40e+00 / 1.48e+00 | 20260507 04:22:54.042 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.60e+00, 3.68e-01 / 2.25e-01 | 20260507 04:22:54.042 | INFO | PerfReport: 3.01%, Metrics -- 9.27e-01 / 1.25e+00, 2.45e-01 / 1.32e-01 | 20260507 04:22:54.042 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.14e-01 / 3.66e-01, 5.03e-02 / 1.71e-02 | 20260507 04:22:54.042 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:22:54.042 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:22:54.042 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:22:54.042 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 04:22:54.042 | INFO | PerfReport: 21.75%, Upload -- 2.09e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260507 04:22:54.042 | INFO | PerfReport: 21.34%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260507 04:22:54.042 | INFO | PerfReport: 31.99%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:22:54.042 | INFO | PerfReport: 1.47%, LockUpload -- 1.41e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260507 04:22:54.042 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260507 04:22:54.042 | INFO | PerfReport: 22.16%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 04:22:54.042 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260507 04:22:54.141 | INFO | Starting a round of Baz IO | 20260507 04:22:57.238 | INFO | Populating RTMetrics report | 20260507 04:22:57.239 | INFO | RTMetrics report populated | 20260507 04:22:57.239 | INFO | Opening tmp RTMetrics json file | 20260507 04:22:57.257 | INFO | Serializing RTMetrics to json | 20260507 04:22:57.258 | INFO | Serialization done, swapping tmp file now... | 20260507 04:22:57.259 | INFO | RTMetrics done | 20260507 04:22:57.324 | INFO | Finished a round of Baz IO: Wrote 1.726 GiB in 3.182 seconds | 20260507 04:22:57.324 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.80% / 0.00% / 67.20% | 20260507 04:22:57.324 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.171967% / 3.97364e-06% / 8.34465e-05% | 20260507 04:22:57.324 | INFO | Baz IO overall: | 20260507 04:22:57.324 | INFO | BytesWritten : 3605477030151 | 20260507 04:22:57.324 | INFO | NumEvents : 1731020802005 | 20260507 04:22:59.177 | INFO | Analyzing chunk frames = [7061504,7062016) | 20260507 04:22:59.177 | INFO | MemInfo: { | 20260507 04:22:59.177 | INFO | "gpu" : " 17 GB", | 20260507 04:22:59.177 | INFO | "huge" : "", | 20260507 04:22:59.177 | INFO | "huge_shared" : " 27 GB", | 20260507 04:22:59.177 | INFO | "malloc" : " 14 GB", | 20260507 04:22:59.177 | INFO | "pinned" : " 29 GB" | 20260507 04:22:59.177 | INFO | } | 20260507 04:22:59.177 | INFO | Perf report for the last 5.13597 seconds: | 20260507 04:22:59.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:22:59.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:59.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:59.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:59.177 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260507 04:22:59.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:22:59.177 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 04:22:59.177 | INFO | PRE_HQ : 59.2%, 1.8, 55.5ms, 0 | 20260507 04:22:59.177 | INFO | BAZWRITER : 61.6%, 3.6, 114.3ms, 0 | 20260507 04:22:59.177 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 04:22:59.177 | INFO | Uploaded 36GiB to GPU in 2.07036s (17.3883GiB/s) | 20260507 04:22:59.177 | INFO | Downloaded 38GiB to GPU in 2.18598s (17.3835GiB/s) | 20260507 04:23:04.311 | INFO | Analyzing chunk frames = [7062016,7062528) | 20260507 04:23:04.311 | INFO | MemInfo: { | 20260507 04:23:04.311 | INFO | "gpu" : " 17 GB", | 20260507 04:23:04.311 | INFO | "huge" : "", | 20260507 04:23:04.311 | INFO | "huge_shared" : " 27 GB", | 20260507 04:23:04.311 | INFO | "malloc" : " 14 GB", | 20260507 04:23:04.311 | INFO | "pinned" : " 29 GB" | 20260507 04:23:04.311 | INFO | } | 20260507 04:23:04.311 | INFO | Perf report for the last 5.13448 seconds: | 20260507 04:23:04.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:23:04.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:04.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:04.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:04.311 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:23:04.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:04.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:04.311 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:23:04.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:04.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:04.312 | INFO | Uploaded 36GiB to GPU in 1.98681s (18.1195GiB/s) | 20260507 04:23:04.312 | INFO | Downloaded 34GiB to GPU in 1.93083s (17.609GiB/s) | 20260507 04:23:09.444 | INFO | Analyzing chunk frames = [7062528,7063040) | 20260507 04:23:09.444 | INFO | MemInfo: { | 20260507 04:23:09.444 | INFO | "gpu" : " 17 GB", | 20260507 04:23:09.444 | INFO | "huge" : "", | 20260507 04:23:09.444 | INFO | "huge_shared" : " 27 GB", | 20260507 04:23:09.444 | INFO | "malloc" : " 14 GB", | 20260507 04:23:09.444 | INFO | "pinned" : " 29 GB" | 20260507 04:23:09.444 | INFO | } | 20260507 04:23:09.444 | INFO | Perf report for the last 5.1332 seconds: | 20260507 04:23:09.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:23:09.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:09.445 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:09.445 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:09.445 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 04:23:09.445 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:09.445 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:09.445 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:23:09.445 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:09.445 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:09.445 | INFO | Uploaded 36GiB to GPU in 1.98828s (18.1061GiB/s) | 20260507 04:23:09.445 | INFO | Downloaded 34GiB to GPU in 1.94314s (17.4975GiB/s) | 20260507 04:23:14.593 | INFO | Analyzing chunk frames = [7063040,7063552) | 20260507 04:23:14.593 | INFO | MemInfo: { | 20260507 04:23:14.593 | INFO | "gpu" : " 17 GB", | 20260507 04:23:14.593 | INFO | "huge" : "", | 20260507 04:23:14.593 | INFO | "huge_shared" : " 27 GB", | 20260507 04:23:14.593 | INFO | "malloc" : " 14 GB", | 20260507 04:23:14.593 | INFO | "pinned" : " 29 GB" | 20260507 04:23:14.593 | INFO | } | 20260507 04:23:14.593 | INFO | Perf report for the last 5.14853 seconds: | 20260507 04:23:14.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:23:14.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:14.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:14.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:14.593 | INFO | ANALYSIS : 59.5%, 3.0, 92.5ms, 0 | 20260507 04:23:14.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:14.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:14.593 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:23:14.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:14.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:14.593 | INFO | Uploaded 36GiB to GPU in 1.97149s (18.2603GiB/s) | 20260507 04:23:14.593 | INFO | Downloaded 34GiB to GPU in 1.90899s (17.8104GiB/s) | 20260507 04:23:19.738 | INFO | Analyzing chunk frames = [7063552,7064064) | 20260507 04:23:19.738 | INFO | MemInfo: { | 20260507 04:23:19.738 | INFO | "gpu" : " 17 GB", | 20260507 04:23:19.738 | INFO | "huge" : "", | 20260507 04:23:19.738 | INFO | "huge_shared" : " 27 GB", | 20260507 04:23:19.738 | INFO | "malloc" : " 14 GB", | 20260507 04:23:19.738 | INFO | "pinned" : " 29 GB" | 20260507 04:23:19.738 | INFO | } | 20260507 04:23:19.738 | INFO | Perf report for the last 5.14541 seconds: | 20260507 04:23:19.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:23:19.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:19.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:19.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:19.739 | INFO | ANALYSIS : 60.1%, 3.0, 94.0ms, 0 | 20260507 04:23:19.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:19.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:19.739 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 04:23:19.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:19.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:19.739 | INFO | Uploaded 36GiB to GPU in 2.04979s (17.5628GiB/s) | 20260507 04:23:19.739 | INFO | Downloaded 34GiB to GPU in 1.94919s (17.4431GiB/s) | 20260507 04:23:24.870 | INFO | Analyzing chunk frames = [7064064,7064576) | 20260507 04:23:24.870 | INFO | MemInfo: { | 20260507 04:23:24.870 | INFO | "gpu" : " 17 GB", | 20260507 04:23:24.870 | INFO | "huge" : "", | 20260507 04:23:24.870 | INFO | "huge_shared" : " 27 GB", | 20260507 04:23:24.870 | INFO | "malloc" : " 14 GB", | 20260507 04:23:24.870 | INFO | "pinned" : " 29 GB" | 20260507 04:23:24.870 | INFO | } | 20260507 04:23:24.870 | INFO | Perf report for the last 5.13127 seconds: | 20260507 04:23:24.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:23:24.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:24.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:24.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:24.870 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:23:24.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:24.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:24.870 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260507 04:23:24.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:24.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:24.870 | INFO | Uploaded 36GiB to GPU in 2.06239s (17.4555GiB/s) | 20260507 04:23:24.870 | INFO | Downloaded 34GiB to GPU in 1.9329s (17.5901GiB/s) | 20260507 04:23:30.008 | INFO | Analyzing chunk frames = [7064576,7065088) | 20260507 04:23:30.008 | INFO | MemInfo: { | 20260507 04:23:30.008 | INFO | "gpu" : " 17 GB", | 20260507 04:23:30.008 | INFO | "huge" : "", | 20260507 04:23:30.009 | INFO | "huge_shared" : " 27 GB", | 20260507 04:23:30.009 | INFO | "malloc" : " 14 GB", | 20260507 04:23:30.009 | INFO | "pinned" : " 29 GB" | 20260507 04:23:30.009 | INFO | } | 20260507 04:23:30.009 | INFO | Perf report for the last 5.13884 seconds: | 20260507 04:23:30.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:23:30.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:30.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:30.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:30.009 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:23:30.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:30.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:30.009 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 04:23:30.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:30.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:30.009 | INFO | Uploaded 36GiB to GPU in 1.92969s (18.6558GiB/s) | 20260507 04:23:30.009 | INFO | Downloaded 34GiB to GPU in 1.97044s (17.2551GiB/s) | 20260507 04:23:35.143 | INFO | Analyzing chunk frames = [7065088,7065600) | 20260507 04:23:35.143 | INFO | MemInfo: { | 20260507 04:23:35.143 | INFO | "gpu" : " 17 GB", | 20260507 04:23:35.143 | INFO | "huge" : "", | 20260507 04:23:35.143 | INFO | "huge_shared" : " 27 GB", | 20260507 04:23:35.143 | INFO | "malloc" : " 14 GB", | 20260507 04:23:35.143 | INFO | "pinned" : " 29 GB" | 20260507 04:23:35.143 | INFO | } | 20260507 04:23:35.143 | INFO | Perf report for the last 5.13427 seconds: | 20260507 04:23:35.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:23:35.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:35.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:35.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:35.143 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 04:23:35.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:35.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:35.143 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:23:35.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:35.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:35.143 | INFO | Uploaded 36GiB to GPU in 1.96498s (18.3208GiB/s) | 20260507 04:23:35.143 | INFO | Downloaded 34GiB to GPU in 1.9434s (17.4952GiB/s) | 20260507 04:23:35.240 | INFO | Starting a round of Baz IO | 20260507 04:23:38.354 | INFO | Populating RTMetrics report | 20260507 04:23:38.355 | INFO | RTMetrics report populated | 20260507 04:23:38.355 | INFO | Opening tmp RTMetrics json file | 20260507 04:23:38.358 | INFO | Serializing RTMetrics to json | 20260507 04:23:38.379 | INFO | Serialization done, swapping tmp file now... | 20260507 04:23:38.381 | INFO | RTMetrics done | 20260507 04:23:38.451 | INFO | Finished a round of Baz IO: Wrote 1.727 GiB in 3.211 seconds | 20260507 04:23:38.451 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.80% / 0.00% / 67.20% | 20260507 04:23:38.451 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.171463% / 0% / 7.15256e-05% | 20260507 04:23:38.451 | INFO | Baz IO overall: | 20260507 04:23:38.451 | INFO | BytesWritten : 3607331159954 | 20260507 04:23:38.451 | INFO | NumEvents : 1731893225938 | 20260507 04:23:40.287 | INFO | Analyzing chunk frames = [7065600,7066112) | 20260507 04:23:40.287 | INFO | MemInfo: { | 20260507 04:23:40.287 | INFO | "gpu" : " 17 GB", | 20260507 04:23:40.287 | INFO | "huge" : "", | 20260507 04:23:40.287 | INFO | "huge_shared" : " 27 GB", | 20260507 04:23:40.287 | INFO | "malloc" : " 14 GB", | 20260507 04:23:40.287 | INFO | "pinned" : " 29 GB" | 20260507 04:23:40.287 | INFO | } | 20260507 04:23:40.287 | INFO | Perf report for the last 5.14444 seconds: | 20260507 04:23:40.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:23:40.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:40.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:40.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:40.287 | INFO | ANALYSIS : 61.9%, 3.0, 96.4ms, 0 | 20260507 04:23:40.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:40.287 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 04:23:40.287 | INFO | PRE_HQ : 59.4%, 1.8, 55.3ms, 0 | 20260507 04:23:40.287 | INFO | BAZWRITER : 62.0%, 3.3, 105.3ms, 0 | 20260507 04:23:40.287 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:23:40.288 | INFO | Uploaded 36GiB to GPU in 2.03479s (17.6922GiB/s) | 20260507 04:23:40.288 | INFO | Downloaded 38GiB to GPU in 2.17966s (17.4339GiB/s) | 20260507 04:23:45.427 | INFO | Analyzing chunk frames = [7066112,7066624) | 20260507 04:23:45.427 | INFO | MemInfo: { | 20260507 04:23:45.427 | INFO | "gpu" : " 17 GB", | 20260507 04:23:45.427 | INFO | "huge" : "", | 20260507 04:23:45.427 | INFO | "huge_shared" : " 27 GB", | 20260507 04:23:45.427 | INFO | "malloc" : " 14 GB", | 20260507 04:23:45.427 | INFO | "pinned" : " 29 GB" | 20260507 04:23:45.427 | INFO | } | 20260507 04:23:45.427 | INFO | Perf report for the last 5.14004 seconds: | 20260507 04:23:45.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:23:45.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:45.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:45.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:45.427 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 04:23:45.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:45.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:45.427 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:23:45.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:45.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:45.428 | INFO | Uploaded 36GiB to GPU in 2.04932s (17.5668GiB/s) | 20260507 04:23:45.428 | INFO | Downloaded 34GiB to GPU in 1.93964s (17.529GiB/s) | 20260507 04:23:50.565 | INFO | Analyzing chunk frames = [7066624,7067136) | 20260507 04:23:50.565 | INFO | MemInfo: { | 20260507 04:23:50.565 | INFO | "gpu" : " 17 GB", | 20260507 04:23:50.565 | INFO | "huge" : "", | 20260507 04:23:50.565 | INFO | "huge_shared" : " 27 GB", | 20260507 04:23:50.565 | INFO | "malloc" : " 14 GB", | 20260507 04:23:50.565 | INFO | "pinned" : " 29 GB" | 20260507 04:23:50.565 | INFO | } | 20260507 04:23:50.565 | INFO | Perf report for the last 5.13738 seconds: | 20260507 04:23:50.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:23:50.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:50.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:50.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:50.565 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 04:23:50.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:50.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:50.565 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:23:50.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:50.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:50.565 | INFO | Uploaded 36GiB to GPU in 2.01009s (17.9097GiB/s) | 20260507 04:23:50.565 | INFO | Downloaded 34GiB to GPU in 1.93888s (17.5359GiB/s) | 20260507 04:23:55.704 | INFO | Analyzing chunk frames = [7067136,7067648) | 20260507 04:23:55.704 | INFO | MemInfo: { | 20260507 04:23:55.704 | INFO | "gpu" : " 17 GB", | 20260507 04:23:55.704 | INFO | "huge" : "", | 20260507 04:23:55.704 | INFO | "huge_shared" : " 27 GB", | 20260507 04:23:55.704 | INFO | "malloc" : " 14 GB", | 20260507 04:23:55.704 | INFO | "pinned" : " 29 GB" | 20260507 04:23:55.704 | INFO | } | 20260507 04:23:55.704 | INFO | Perf report for the last 5.13923 seconds: | 20260507 04:23:55.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:23:55.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:55.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:55.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:55.704 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 04:23:55.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:55.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:55.704 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 04:23:55.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:55.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:23:55.704 | INFO | Uploaded 36GiB to GPU in 1.99111s (18.0804GiB/s) | 20260507 04:23:55.704 | INFO | Downloaded 34GiB to GPU in 1.92783s (17.6364GiB/s) | 20260507 04:24:00.848 | INFO | Analyzing chunk frames = [7067648,7068160) | 20260507 04:24:00.848 | INFO | MemInfo: { | 20260507 04:24:00.848 | INFO | "gpu" : " 17 GB", | 20260507 04:24:00.848 | INFO | "huge" : "", | 20260507 04:24:00.848 | INFO | "huge_shared" : " 27 GB", | 20260507 04:24:00.848 | INFO | "malloc" : " 14 GB", | 20260507 04:24:00.848 | INFO | "pinned" : " 29 GB" | 20260507 04:24:00.848 | INFO | } | 20260507 04:24:00.848 | INFO | Perf report for the last 5.14414 seconds: | 20260507 04:24:00.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:24:00.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:00.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:00.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:00.848 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 04:24:00.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:00.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:00.848 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:24:00.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:00.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:00.848 | INFO | Uploaded 36GiB to GPU in 2.04286s (17.6224GiB/s) | 20260507 04:24:00.848 | INFO | Downloaded 34GiB to GPU in 1.93982s (17.5274GiB/s) | 20260507 04:24:05.988 | INFO | Analyzing chunk frames = [7068160,7068672) | 20260507 04:24:05.988 | INFO | MemInfo: { | 20260507 04:24:05.988 | INFO | "gpu" : " 17 GB", | 20260507 04:24:05.988 | INFO | "huge" : "", | 20260507 04:24:05.988 | INFO | "huge_shared" : " 27 GB", | 20260507 04:24:05.988 | INFO | "malloc" : " 14 GB", | 20260507 04:24:05.988 | INFO | "pinned" : " 29 GB" | 20260507 04:24:05.988 | INFO | } | 20260507 04:24:05.988 | INFO | Perf report for the last 5.13961 seconds: | 20260507 04:24:05.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:24:05.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:05.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:05.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:05.988 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 04:24:05.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:05.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:05.988 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 04:24:05.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:05.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:05.988 | INFO | Uploaded 36GiB to GPU in 2.0382s (17.6626GiB/s) | 20260507 04:24:05.988 | INFO | Downloaded 34GiB to GPU in 1.91453s (17.7589GiB/s) | 20260507 04:24:11.121 | INFO | Analyzing chunk frames = [7068672,7069184) | 20260507 04:24:11.121 | INFO | MemInfo: { | 20260507 04:24:11.121 | INFO | "gpu" : " 17 GB", | 20260507 04:24:11.121 | INFO | "huge" : "", | 20260507 04:24:11.121 | INFO | "huge_shared" : " 27 GB", | 20260507 04:24:11.121 | INFO | "malloc" : " 14 GB", | 20260507 04:24:11.121 | INFO | "pinned" : " 29 GB" | 20260507 04:24:11.121 | INFO | } | 20260507 04:24:11.121 | INFO | Perf report for the last 5.13339 seconds: | 20260507 04:24:11.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:24:11.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:11.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:11.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:11.121 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 04:24:11.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:11.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:11.121 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:24:11.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:11.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:11.121 | INFO | Uploaded 36GiB to GPU in 2.03261s (17.7112GiB/s) | 20260507 04:24:11.122 | INFO | Downloaded 34GiB to GPU in 1.97167s (17.2443GiB/s) | 20260507 04:24:16.265 | INFO | Analyzing chunk frames = [7069184,7069696) | 20260507 04:24:16.265 | INFO | MemInfo: { | 20260507 04:24:16.265 | INFO | "gpu" : " 17 GB", | 20260507 04:24:16.265 | INFO | "huge" : "", | 20260507 04:24:16.265 | INFO | "huge_shared" : " 27 GB", | 20260507 04:24:16.265 | INFO | "malloc" : " 14 GB", | 20260507 04:24:16.265 | INFO | "pinned" : " 29 GB" | 20260507 04:24:16.265 | INFO | } | 20260507 04:24:16.265 | INFO | Perf report for the last 5.14353 seconds: | 20260507 04:24:16.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:24:16.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:16.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:16.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:16.265 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 04:24:16.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:16.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:16.265 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 04:24:16.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:16.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:16.265 | INFO | Uploaded 36GiB to GPU in 1.97541s (18.224GiB/s) | 20260507 04:24:16.265 | INFO | Downloaded 34GiB to GPU in 1.94405s (17.4892GiB/s) | 20260507 04:24:16.265 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:24:16.265 | INFO | Total mem usage: 18 GiB | 20260507 04:24:16.265 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:24:16.265 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -20.2%) | 20260507 04:24:16.265 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:24:16.265 | INFO | High Water Breakdown: | 20260507 04:24:16.265 | INFO | 60.9% : 9.12 GiB ( 79.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:24:16.265 | INFO | 39% : 5.83 GiB ( 87.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:24:16.265 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:24:16.265 | INFO | ------------------------------------------ | 20260507 04:24:16.265 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:24:16.265 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:24:16.265 | INFO | Total mem usage: 26.5 GiB | 20260507 04:24:16.265 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:24:16.265 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:24:16.265 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:24:16.266 | INFO | High Water Breakdown: | 20260507 04:24:16.266 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:24:16.266 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:24:16.266 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:24:16.266 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:24:16.266 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:24:16.266 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:24:16.266 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:24:16.266 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:24:16.266 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:24:16.266 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:24:16.266 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:24:16.266 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:24:16.266 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:24:16.266 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:24:16.266 | INFO | ------------------------------------------ | 20260507 04:24:16.266 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:24:16.266 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:24:16.266 | INFO | Total mem usage: 25.6 GiB | 20260507 04:24:16.266 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:24:16.266 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:24:16.266 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:24:16.266 | INFO | High Water Breakdown: | 20260507 04:24:16.266 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:24:16.266 | INFO | ------------------------------------------ | 20260507 04:24:16.266 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:24:16.266 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:24:16.266 | INFO | Total mem usage: 15.7 GiB | 20260507 04:24:16.266 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:24:16.266 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:24:16.266 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:24:16.266 | INFO | High Water Breakdown: | 20260507 04:24:16.266 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:24:16.266 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:24:16.266 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:24:16.266 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:24:16.266 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:24:16.266 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:24:16.266 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:24:16.266 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:24:16.266 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:24:16.266 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:24:16.266 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:24:16.266 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:24:16.266 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:24:16.266 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:24:16.266 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:24:16.266 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:24:16.266 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:24:16.266 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:24:16.266 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:24:16.266 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:24:16.266 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:24:16.266 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:24:16.266 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:24:16.266 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:24:16.266 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:24:16.266 | INFO | ------------------------------------------ | 20260507 04:24:16.266 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:24:16.266 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:24:16.266 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:24:16.266 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:24:16.266 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.48e+01, 1.33e+01 / 1.44e+00 | 20260507 04:24:16.266 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.03e-01, 9.54e-03 / 5.85e-03 | 20260507 04:24:16.266 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.46e+00, 5.17e-01 / 2.94e-01 | 20260507 04:24:16.266 | INFO | PerfReport: 3.23%, Histogram -- 9.94e-01 / 1.47e+00, 1.69e-01 / 1.11e-01 | 20260507 04:24:16.266 | INFO | PerfReport: 16.08%, DME -- 3.96e+01 / 4.41e+01, 3.75e+00 / 1.64e+00 | 20260507 04:24:16.266 | INFO | PerfReport: 57.74%, FrameLabeling -- 1.78e+01 / 2.03e+01, 1.38e+00 / 1.81e+00 | 20260507 04:24:16.266 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.14e+00 / 2.67e+00, 3.82e-01 / 2.45e-01 | 20260507 04:24:16.266 | INFO | PerfReport: 3.02%, Metrics -- 9.29e-01 / 1.26e+00, 2.44e-01 / 1.80e-01 | 20260507 04:24:16.266 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.14e-01 / 3.75e-01, 5.06e-02 / 3.03e-02 | 20260507 04:24:16.266 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:24:16.266 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:24:16.266 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:24:16.266 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 04:24:16.266 | INFO | PerfReport: 21.79%, Upload -- 2.10e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260507 04:24:16.266 | INFO | PerfReport: 21.33%, Download -- 2.05e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260507 04:24:16.266 | INFO | PerfReport: 31.99%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 04:24:16.266 | INFO | PerfReport: 1.44%, LockUpload -- 1.39e+00 / 0.00e+00, 4.75e+00 / 0.00e+00 | 20260507 04:24:16.266 | INFO | PerfReport: 1.26%, LockDownload -- 1.21e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260507 04:24:16.266 | INFO | PerfReport: 22.17%, LockCompute -- 2.13e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 04:24:16.266 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260507 04:24:16.362 | INFO | Starting a round of Baz IO | 20260507 04:24:19.467 | INFO | Populating RTMetrics report | 20260507 04:24:19.468 | INFO | RTMetrics report populated | 20260507 04:24:19.468 | INFO | Opening tmp RTMetrics json file | 20260507 04:24:19.469 | INFO | Serializing RTMetrics to json | 20260507 04:24:19.476 | INFO | Serialization done, swapping tmp file now... | 20260507 04:24:19.488 | INFO | RTMetrics done | 20260507 04:24:19.569 | INFO | Finished a round of Baz IO: Wrote 1.727 GiB in 3.207 seconds | 20260507 04:24:19.569 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.80% / 0.00% / 67.20% | 20260507 04:24:19.569 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.171645% / 7.94729e-06% / 7.94729e-05% | 20260507 04:24:19.569 | INFO | Baz IO overall: | 20260507 04:24:19.569 | INFO | BytesWritten : 3609185598313 | 20260507 04:24:19.569 | INFO | NumEvents : 1732766115805 | 20260507 04:24:21.401 | INFO | Analyzing chunk frames = [7069696,7070208) | 20260507 04:24:21.401 | INFO | MemInfo: { | 20260507 04:24:21.401 | INFO | "gpu" : " 17 GB", | 20260507 04:24:21.401 | INFO | "huge" : "", | 20260507 04:24:21.401 | INFO | "huge_shared" : " 27 GB", | 20260507 04:24:21.401 | INFO | "malloc" : " 14 GB", | 20260507 04:24:21.401 | INFO | "pinned" : " 29 GB" | 20260507 04:24:21.401 | INFO | } | 20260507 04:24:21.401 | INFO | Perf report for the last 5.13669 seconds: | 20260507 04:24:21.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:24:21.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:21.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:21.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:21.401 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260507 04:24:21.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:21.402 | INFO | RT_METRICS : 45.1%, 1.0, 23.6ms, 0 | 20260507 04:24:21.402 | INFO | PRE_HQ : 60.5%, 1.9, 60.4ms, 0 | 20260507 04:24:21.402 | INFO | BAZWRITER : 62.0%, 3.4, 107.5ms, 0 | 20260507 04:24:21.402 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 04:24:21.402 | INFO | Uploaded 36GiB to GPU in 2.03551s (17.686GiB/s) | 20260507 04:24:21.402 | INFO | Downloaded 38GiB to GPU in 2.17312s (17.4864GiB/s) | 20260507 04:24:26.544 | INFO | Analyzing chunk frames = [7070208,7070720) | 20260507 04:24:26.544 | INFO | MemInfo: { | 20260507 04:24:26.544 | INFO | "gpu" : " 17 GB", | 20260507 04:24:26.544 | INFO | "huge" : "", | 20260507 04:24:26.544 | INFO | "huge_shared" : " 27 GB", | 20260507 04:24:26.544 | INFO | "malloc" : " 14 GB", | 20260507 04:24:26.545 | INFO | "pinned" : " 29 GB" | 20260507 04:24:26.545 | INFO | } | 20260507 04:24:26.545 | INFO | Perf report for the last 5.14311 seconds: | 20260507 04:24:26.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:24:26.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:26.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:26.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:26.545 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:24:26.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:26.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:26.545 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 04:24:26.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:26.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:26.545 | INFO | Uploaded 36GiB to GPU in 2.02941s (17.7392GiB/s) | 20260507 04:24:26.545 | INFO | Downloaded 34GiB to GPU in 1.93647s (17.5577GiB/s) | 20260507 04:24:31.684 | INFO | Analyzing chunk frames = [7070720,7071232) | 20260507 04:24:31.684 | INFO | MemInfo: { | 20260507 04:24:31.684 | INFO | "gpu" : " 17 GB", | 20260507 04:24:31.684 | INFO | "huge" : "", | 20260507 04:24:31.684 | INFO | "huge_shared" : " 27 GB", | 20260507 04:24:31.684 | INFO | "malloc" : " 14 GB", | 20260507 04:24:31.684 | INFO | "pinned" : " 29 GB" | 20260507 04:24:31.684 | INFO | } | 20260507 04:24:31.684 | INFO | Perf report for the last 5.13923 seconds: | 20260507 04:24:31.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:24:31.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:31.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:31.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:31.684 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 04:24:31.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:31.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:31.684 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:24:31.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:31.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:31.684 | INFO | Uploaded 36GiB to GPU in 2.0193s (17.828GiB/s) | 20260507 04:24:31.684 | INFO | Downloaded 34GiB to GPU in 1.94269s (17.5015GiB/s) | 20260507 04:24:36.818 | INFO | Analyzing chunk frames = [7071232,7071744) | 20260507 04:24:36.818 | INFO | MemInfo: { | 20260507 04:24:36.818 | INFO | "gpu" : " 17 GB", | 20260507 04:24:36.818 | INFO | "huge" : "", | 20260507 04:24:36.818 | INFO | "huge_shared" : " 27 GB", | 20260507 04:24:36.818 | INFO | "malloc" : " 14 GB", | 20260507 04:24:36.818 | INFO | "pinned" : " 29 GB" | 20260507 04:24:36.818 | INFO | } | 20260507 04:24:36.818 | INFO | Perf report for the last 5.13398 seconds: | 20260507 04:24:36.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:24:36.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:36.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:36.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:36.818 | INFO | ANALYSIS : 59.9%, 3.0, 92.8ms, 0 | 20260507 04:24:36.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:36.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:36.818 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:24:36.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:36.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:36.818 | INFO | Uploaded 36GiB to GPU in 2.02257s (17.7992GiB/s) | 20260507 04:24:36.818 | INFO | Downloaded 34GiB to GPU in 1.94184s (17.5092GiB/s) | 20260507 04:24:41.956 | INFO | Analyzing chunk frames = [7071744,7072256) | 20260507 04:24:41.956 | INFO | MemInfo: { | 20260507 04:24:41.956 | INFO | "gpu" : " 17 GB", | 20260507 04:24:41.956 | INFO | "huge" : "", | 20260507 04:24:41.956 | INFO | "huge_shared" : " 27 GB", | 20260507 04:24:41.956 | INFO | "malloc" : " 14 GB", | 20260507 04:24:41.956 | INFO | "pinned" : " 29 GB" | 20260507 04:24:41.956 | INFO | } | 20260507 04:24:41.956 | INFO | Perf report for the last 5.13785 seconds: | 20260507 04:24:41.956 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:24:41.956 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:41.956 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:41.956 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:41.956 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 04:24:41.956 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:41.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:41.956 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 04:24:41.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:41.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:41.956 | INFO | Uploaded 36GiB to GPU in 2.03962s (17.6503GiB/s) | 20260507 04:24:41.956 | INFO | Downloaded 34GiB to GPU in 1.93906s (17.5343GiB/s) | 20260507 04:24:47.096 | INFO | Analyzing chunk frames = [7072256,7072768) | 20260507 04:24:47.096 | INFO | MemInfo: { | 20260507 04:24:47.096 | INFO | "gpu" : " 17 GB", | 20260507 04:24:47.096 | INFO | "huge" : "", | 20260507 04:24:47.096 | INFO | "huge_shared" : " 27 GB", | 20260507 04:24:47.096 | INFO | "malloc" : " 14 GB", | 20260507 04:24:47.096 | INFO | "pinned" : " 29 GB" | 20260507 04:24:47.096 | INFO | } | 20260507 04:24:47.096 | INFO | Perf report for the last 5.14042 seconds: | 20260507 04:24:47.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:24:47.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:47.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:47.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:47.096 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:24:47.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:47.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:47.096 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 04:24:47.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:47.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:47.096 | INFO | Uploaded 36GiB to GPU in 2.05835s (17.4898GiB/s) | 20260507 04:24:47.096 | INFO | Downloaded 34GiB to GPU in 1.96396s (17.312GiB/s) | 20260507 04:24:52.238 | INFO | Analyzing chunk frames = [7072768,7073280) | 20260507 04:24:52.238 | INFO | MemInfo: { | 20260507 04:24:52.238 | INFO | "gpu" : " 17 GB", | 20260507 04:24:52.238 | INFO | "huge" : "", | 20260507 04:24:52.238 | INFO | "huge_shared" : " 27 GB", | 20260507 04:24:52.238 | INFO | "malloc" : " 14 GB", | 20260507 04:24:52.238 | INFO | "pinned" : " 29 GB" | 20260507 04:24:52.238 | INFO | } | 20260507 04:24:52.238 | INFO | Perf report for the last 5.14187 seconds: | 20260507 04:24:52.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:24:52.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:52.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:52.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:52.238 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 04:24:52.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:52.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:52.238 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 04:24:52.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:52.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:52.238 | INFO | Uploaded 36GiB to GPU in 2.01993s (17.8224GiB/s) | 20260507 04:24:52.238 | INFO | Downloaded 34GiB to GPU in 1.98345s (17.1418GiB/s) | 20260507 04:24:57.384 | INFO | Analyzing chunk frames = [7073280,7073792) | 20260507 04:24:57.384 | INFO | MemInfo: { | 20260507 04:24:57.384 | INFO | "gpu" : " 17 GB", | 20260507 04:24:57.384 | INFO | "huge" : "", | 20260507 04:24:57.384 | INFO | "huge_shared" : " 27 GB", | 20260507 04:24:57.384 | INFO | "malloc" : " 14 GB", | 20260507 04:24:57.384 | INFO | "pinned" : " 29 GB" | 20260507 04:24:57.384 | INFO | } | 20260507 04:24:57.384 | INFO | Perf report for the last 5.14596 seconds: | 20260507 04:24:57.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:24:57.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:57.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:57.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:57.384 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:24:57.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:57.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:57.384 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 04:24:57.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:57.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:24:57.384 | INFO | Uploaded 36GiB to GPU in 2.00085s (17.9923GiB/s) | 20260507 04:24:57.384 | INFO | Downloaded 34GiB to GPU in 1.97723s (17.1958GiB/s) | 20260507 04:24:57.495 | INFO | Starting a round of Baz IO | 20260507 04:25:00.578 | INFO | Populating RTMetrics report | 20260507 04:25:00.578 | INFO | RTMetrics report populated | 20260507 04:25:00.578 | INFO | Opening tmp RTMetrics json file | 20260507 04:25:00.585 | INFO | Serializing RTMetrics to json | 20260507 04:25:00.595 | INFO | Serialization done, swapping tmp file now... | 20260507 04:25:00.596 | INFO | RTMetrics done | 20260507 04:25:00.664 | INFO | Finished a round of Baz IO: Wrote 1.725 GiB in 3.169 seconds | 20260507 04:25:00.664 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.80% / 0.00% / 67.20% | 20260507 04:25:00.664 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.170243% / 7.94729e-06% / 6.75519e-05% | 20260507 04:25:00.664 | INFO | Baz IO overall: | 20260507 04:25:00.664 | INFO | BytesWritten : 3611037576508 | 20260507 04:25:00.664 | INFO | NumEvents : 1733637316754 | 20260507 04:25:02.518 | INFO | Analyzing chunk frames = [7073792,7074304) | 20260507 04:25:02.518 | INFO | MemInfo: { | 20260507 04:25:02.518 | INFO | "gpu" : " 17 GB", | 20260507 04:25:02.518 | INFO | "huge" : "", | 20260507 04:25:02.518 | INFO | "huge_shared" : " 27 GB", | 20260507 04:25:02.518 | INFO | "malloc" : " 14 GB", | 20260507 04:25:02.518 | INFO | "pinned" : " 29 GB" | 20260507 04:25:02.518 | INFO | } | 20260507 04:25:02.518 | INFO | Perf report for the last 5.1345 seconds: | 20260507 04:25:02.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:25:02.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:02.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:02.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:02.519 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 04:25:02.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:02.519 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 04:25:02.519 | INFO | PRE_HQ : 59.1%, 1.8, 55.6ms, 0 | 20260507 04:25:02.519 | INFO | BAZWRITER : 61.6%, 3.3, 103.2ms, 0 | 20260507 04:25:02.519 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 04:25:02.519 | INFO | Uploaded 36GiB to GPU in 2.05657s (17.5049GiB/s) | 20260507 04:25:02.519 | INFO | Downloaded 38GiB to GPU in 2.17683s (17.4566GiB/s) | 20260507 04:25:07.660 | INFO | Analyzing chunk frames = [7074304,7074816) | 20260507 04:25:07.660 | INFO | MemInfo: { | 20260507 04:25:07.660 | INFO | "gpu" : " 17 GB", | 20260507 04:25:07.660 | INFO | "huge" : "", | 20260507 04:25:07.660 | INFO | "huge_shared" : " 27 GB", | 20260507 04:25:07.660 | INFO | "malloc" : " 14 GB", | 20260507 04:25:07.660 | INFO | "pinned" : " 29 GB" | 20260507 04:25:07.660 | INFO | } | 20260507 04:25:07.660 | INFO | Perf report for the last 5.14148 seconds: | 20260507 04:25:07.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:25:07.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:07.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:07.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:07.660 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:25:07.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:07.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:07.660 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:25:07.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:07.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:07.660 | INFO | Uploaded 36GiB to GPU in 2.00279s (17.9749GiB/s) | 20260507 04:25:07.660 | INFO | Downloaded 34GiB to GPU in 1.92842s (17.631GiB/s) | 20260507 04:25:12.794 | INFO | Analyzing chunk frames = [7074816,7075328) | 20260507 04:25:12.794 | INFO | MemInfo: { | 20260507 04:25:12.794 | INFO | "gpu" : " 17 GB", | 20260507 04:25:12.794 | INFO | "huge" : "", | 20260507 04:25:12.795 | INFO | "huge_shared" : " 27 GB", | 20260507 04:25:12.795 | INFO | "malloc" : " 14 GB", | 20260507 04:25:12.795 | INFO | "pinned" : " 29 GB" | 20260507 04:25:12.795 | INFO | } | 20260507 04:25:12.795 | INFO | Perf report for the last 5.13458 seconds: | 20260507 04:25:12.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:25:12.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:12.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:12.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:12.795 | INFO | ANALYSIS : 60.3%, 3.0, 93.2ms, 0 | 20260507 04:25:12.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:12.795 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:12.795 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:25:12.795 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:12.795 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:12.795 | INFO | Uploaded 36GiB to GPU in 2.02264s (17.7986GiB/s) | 20260507 04:25:12.795 | INFO | Downloaded 34GiB to GPU in 1.93936s (17.5316GiB/s) | 20260507 04:25:17.936 | INFO | Analyzing chunk frames = [7075328,7075840) | 20260507 04:25:17.936 | INFO | MemInfo: { | 20260507 04:25:17.936 | INFO | "gpu" : " 17 GB", | 20260507 04:25:17.936 | INFO | "huge" : "", | 20260507 04:25:17.936 | INFO | "huge_shared" : " 27 GB", | 20260507 04:25:17.936 | INFO | "malloc" : " 14 GB", | 20260507 04:25:17.936 | INFO | "pinned" : " 29 GB" | 20260507 04:25:17.936 | INFO | } | 20260507 04:25:17.936 | INFO | Perf report for the last 5.14188 seconds: | 20260507 04:25:17.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:25:17.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:17.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:17.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:17.936 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 04:25:17.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:17.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:17.937 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:25:17.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:17.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:17.937 | INFO | Uploaded 36GiB to GPU in 2.01392s (17.8756GiB/s) | 20260507 04:25:17.937 | INFO | Downloaded 34GiB to GPU in 1.93499s (17.5712GiB/s) | 20260507 04:25:23.072 | INFO | Analyzing chunk frames = [7075840,7076352) | 20260507 04:25:23.072 | INFO | MemInfo: { | 20260507 04:25:23.072 | INFO | "gpu" : " 17 GB", | 20260507 04:25:23.073 | INFO | "huge" : "", | 20260507 04:25:23.073 | INFO | "huge_shared" : " 27 GB", | 20260507 04:25:23.073 | INFO | "malloc" : " 14 GB", | 20260507 04:25:23.073 | INFO | "pinned" : " 29 GB" | 20260507 04:25:23.073 | INFO | } | 20260507 04:25:23.073 | INFO | Perf report for the last 5.13614 seconds: | 20260507 04:25:23.073 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:25:23.073 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:23.073 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:23.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:23.073 | INFO | ANALYSIS : 60.2%, 3.0, 94.0ms, 0 | 20260507 04:25:23.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:23.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:23.073 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:25:23.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:23.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:23.073 | INFO | Uploaded 36GiB to GPU in 2.05751s (17.4969GiB/s) | 20260507 04:25:23.073 | INFO | Downloaded 34GiB to GPU in 1.94615s (17.4704GiB/s) | 20260507 04:25:28.212 | INFO | Analyzing chunk frames = [7076352,7076864) | 20260507 04:25:28.212 | INFO | MemInfo: { | 20260507 04:25:28.212 | INFO | "gpu" : " 17 GB", | 20260507 04:25:28.212 | INFO | "huge" : "", | 20260507 04:25:28.212 | INFO | "huge_shared" : " 27 GB", | 20260507 04:25:28.212 | INFO | "malloc" : " 14 GB", | 20260507 04:25:28.212 | INFO | "pinned" : " 29 GB" | 20260507 04:25:28.212 | INFO | } | 20260507 04:25:28.212 | INFO | Perf report for the last 5.13953 seconds: | 20260507 04:25:28.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:25:28.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:28.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:28.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:28.212 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 04:25:28.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:28.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:28.212 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 04:25:28.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:28.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:28.212 | INFO | Uploaded 36GiB to GPU in 2.04936s (17.5664GiB/s) | 20260507 04:25:28.212 | INFO | Downloaded 34GiB to GPU in 1.92364s (17.6748GiB/s) | 20260507 04:25:33.362 | INFO | Analyzing chunk frames = [7076864,7077376) | 20260507 04:25:33.362 | INFO | MemInfo: { | 20260507 04:25:33.362 | INFO | "gpu" : " 17 GB", | 20260507 04:25:33.362 | INFO | "huge" : "", | 20260507 04:25:33.362 | INFO | "huge_shared" : " 27 GB", | 20260507 04:25:33.362 | INFO | "malloc" : " 14 GB", | 20260507 04:25:33.362 | INFO | "pinned" : " 29 GB" | 20260507 04:25:33.362 | INFO | } | 20260507 04:25:33.362 | INFO | Perf report for the last 5.15005 seconds: | 20260507 04:25:33.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:25:33.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:33.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:33.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:33.362 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:25:33.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:33.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:33.362 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:25:33.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:33.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:33.362 | INFO | Uploaded 36GiB to GPU in 1.98328s (18.1517GiB/s) | 20260507 04:25:33.363 | INFO | Downloaded 34GiB to GPU in 1.96919s (17.266GiB/s) | 20260507 04:25:38.496 | INFO | Analyzing chunk frames = [7077376,7077888) | 20260507 04:25:38.496 | INFO | MemInfo: { | 20260507 04:25:38.496 | INFO | "gpu" : " 17 GB", | 20260507 04:25:38.496 | INFO | "huge" : "", | 20260507 04:25:38.496 | INFO | "huge_shared" : " 27 GB", | 20260507 04:25:38.497 | INFO | "malloc" : " 14 GB", | 20260507 04:25:38.497 | INFO | "pinned" : " 29 GB" | 20260507 04:25:38.497 | INFO | } | 20260507 04:25:38.497 | INFO | Perf report for the last 5.13433 seconds: | 20260507 04:25:38.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:25:38.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:38.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:38.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:38.497 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:25:38.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:38.497 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:38.497 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 04:25:38.497 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:38.497 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:38.497 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:25:38.497 | INFO | Uploaded 36GiB to GPU in 1.98576s (18.129GiB/s) | 20260507 04:25:38.497 | INFO | Total mem usage: 18 GiB | 20260507 04:25:38.497 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:25:38.497 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 04:25:38.497 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:25:38.497 | INFO | High Water Breakdown: | 20260507 04:25:38.497 | INFO | 60.2% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:25:38.497 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:25:38.497 | INFO | Downloaded 34GiB to GPU in 1.96182s (17.3308GiB/s) | 20260507 04:25:38.497 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:25:38.497 | INFO | ------------------------------------------ | 20260507 04:25:38.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:25:38.497 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:25:38.497 | INFO | Total mem usage: 26.5 GiB | 20260507 04:25:38.497 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:25:38.497 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:25:38.497 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:25:38.497 | INFO | High Water Breakdown: | 20260507 04:25:38.497 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:25:38.497 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:25:38.497 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:25:38.497 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:25:38.497 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:25:38.497 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:25:38.497 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:25:38.497 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:25:38.497 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:25:38.497 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:25:38.497 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:25:38.497 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:25:38.497 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:25:38.497 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:25:38.497 | INFO | ------------------------------------------ | 20260507 04:25:38.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:25:38.497 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:25:38.497 | INFO | Total mem usage: 25.6 GiB | 20260507 04:25:38.497 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:25:38.497 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:25:38.497 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:25:38.497 | INFO | High Water Breakdown: | 20260507 04:25:38.497 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:25:38.497 | INFO | ------------------------------------------ | 20260507 04:25:38.497 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:25:38.497 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:25:38.497 | INFO | Total mem usage: 15.7 GiB | 20260507 04:25:38.497 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:25:38.497 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:25:38.497 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:25:38.497 | INFO | High Water Breakdown: | 20260507 04:25:38.497 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:25:38.497 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:25:38.497 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:25:38.497 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:25:38.497 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:25:38.497 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:25:38.498 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:25:38.498 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:25:38.498 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:25:38.498 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:25:38.498 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:25:38.498 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:25:38.498 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:25:38.498 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:25:38.498 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:25:38.498 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:25:38.498 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:25:38.498 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:25:38.498 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:25:38.498 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:25:38.498 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:25:38.498 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:25:38.498 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:25:38.498 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:25:38.498 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:25:38.498 | INFO | ------------------------------------------ | 20260507 04:25:38.498 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:25:38.498 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:25:38.498 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:25:38.498 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:25:38.498 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.32e+01 / 9.63e-01 | 20260507 04:25:38.498 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.01e-01, 1.06e-02 / 7.18e-03 | 20260507 04:25:38.498 | INFO | PerfReport: 11.63%, Baseline -- 3.57e+00 / 4.42e+00, 5.00e-01 / 3.19e-01 | 20260507 04:25:38.498 | INFO | PerfReport: 3.22%, Histogram -- 9.87e-01 / 1.47e+00, 1.63e-01 / 1.18e-01 | 20260507 04:25:38.498 | INFO | PerfReport: 16.13%, DME -- 3.96e+01 / 4.42e+01, 3.77e+00 / 1.50e+00 | 20260507 04:25:38.498 | INFO | PerfReport: 57.73%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.32e+00 / 1.83e+00 | 20260507 04:25:38.498 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.14e+00 / 2.67e+00, 3.83e-01 / 2.48e-01 | 20260507 04:25:38.498 | INFO | PerfReport: 3.02%, Metrics -- 9.27e-01 / 1.24e+00, 2.46e-01 / 1.28e-01 | 20260507 04:25:38.498 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.13e-01 / 3.71e-01, 5.06e-02 / 2.24e-02 | 20260507 04:25:38.498 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:25:38.498 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:25:38.498 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:25:38.498 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 04:25:38.498 | INFO | PerfReport: 21.91%, Upload -- 2.11e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 04:25:38.498 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260507 04:25:38.498 | INFO | PerfReport: 31.90%, Compute -- 3.07e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 04:25:38.498 | INFO | PerfReport: 1.59%, LockUpload -- 1.53e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 04:25:38.498 | INFO | PerfReport: 1.17%, LockDownload -- 1.13e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260507 04:25:38.498 | INFO | PerfReport: 22.01%, LockCompute -- 2.12e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 04:25:38.498 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260507 04:25:38.593 | INFO | Starting a round of Baz IO | 20260507 04:25:41.701 | INFO | Populating RTMetrics report | 20260507 04:25:41.701 | INFO | RTMetrics report populated | 20260507 04:25:41.701 | INFO | Opening tmp RTMetrics json file | 20260507 04:25:41.702 | INFO | Serializing RTMetrics to json | 20260507 04:25:41.713 | INFO | Serialization done, swapping tmp file now... | 20260507 04:25:41.721 | INFO | RTMetrics done | 20260507 04:25:41.793 | INFO | Finished a round of Baz IO: Wrote 1.726 GiB in 3.199 seconds | 20260507 04:25:41.793 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.79% / 0.00% / 67.20% | 20260507 04:25:41.793 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.170211% / 3.97364e-06% / 8.74201e-05% | 20260507 04:25:41.793 | INFO | Baz IO overall: | 20260507 04:25:41.793 | INFO | BytesWritten : 3612891312701 | 20260507 04:25:41.793 | INFO | NumEvents : 1734509882493 | 20260507 04:25:43.630 | INFO | Analyzing chunk frames = [7077888,7078400) | 20260507 04:25:43.630 | INFO | MemInfo: { | 20260507 04:25:43.630 | INFO | "gpu" : " 17 GB", | 20260507 04:25:43.630 | INFO | "huge" : "", | 20260507 04:25:43.631 | INFO | "huge_shared" : " 27 GB", | 20260507 04:25:43.631 | INFO | "malloc" : " 14 GB", | 20260507 04:25:43.631 | INFO | "pinned" : " 29 GB" | 20260507 04:25:43.631 | INFO | } | 20260507 04:25:43.631 | INFO | Perf report for the last 5.13406 seconds: | 20260507 04:25:43.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:25:43.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:43.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:43.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:43.631 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 04:25:43.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:43.631 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260507 04:25:43.631 | INFO | PRE_HQ : 60.0%, 1.9, 58.9ms, 0 | 20260507 04:25:43.631 | INFO | BAZWRITER : 61.9%, 3.4, 108.4ms, 0 | 20260507 04:25:43.631 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:25:43.631 | INFO | Uploaded 36GiB to GPU in 2.05597s (17.51GiB/s) | 20260507 04:25:43.631 | INFO | Downloaded 38GiB to GPU in 2.18036s (17.4283GiB/s) | 20260507 04:25:48.764 | INFO | Analyzing chunk frames = [7078400,7078912) | 20260507 04:25:48.764 | INFO | MemInfo: { | 20260507 04:25:48.764 | INFO | "gpu" : " 17 GB", | 20260507 04:25:48.764 | INFO | "huge" : "", | 20260507 04:25:48.764 | INFO | "huge_shared" : " 27 GB", | 20260507 04:25:48.764 | INFO | "malloc" : " 14 GB", | 20260507 04:25:48.764 | INFO | "pinned" : " 29 GB" | 20260507 04:25:48.764 | INFO | } | 20260507 04:25:48.765 | INFO | Perf report for the last 5.13386 seconds: | 20260507 04:25:48.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:25:48.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:48.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:48.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:48.765 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 04:25:48.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:48.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:48.765 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 04:25:48.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:48.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:48.765 | INFO | Uploaded 36GiB to GPU in 2.01954s (17.8259GiB/s) | 20260507 04:25:48.765 | INFO | Downloaded 34GiB to GPU in 1.93877s (17.5369GiB/s) | 20260507 04:25:53.909 | INFO | Analyzing chunk frames = [7078912,7079424) | 20260507 04:25:53.909 | INFO | MemInfo: { | 20260507 04:25:53.909 | INFO | "gpu" : " 17 GB", | 20260507 04:25:53.909 | INFO | "huge" : "", | 20260507 04:25:53.909 | INFO | "huge_shared" : " 27 GB", | 20260507 04:25:53.909 | INFO | "malloc" : " 14 GB", | 20260507 04:25:53.909 | INFO | "pinned" : " 29 GB" | 20260507 04:25:53.909 | INFO | } | 20260507 04:25:53.909 | INFO | Perf report for the last 5.1445 seconds: | 20260507 04:25:53.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:25:53.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:53.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:53.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:53.909 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 04:25:53.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:53.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:53.909 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:25:53.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:53.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:53.909 | INFO | Uploaded 36GiB to GPU in 2.00373s (17.9665GiB/s) | 20260507 04:25:53.909 | INFO | Downloaded 34GiB to GPU in 1.93754s (17.548GiB/s) | 20260507 04:25:59.059 | INFO | Analyzing chunk frames = [7079424,7079936) | 20260507 04:25:59.059 | INFO | MemInfo: { | 20260507 04:25:59.059 | INFO | "gpu" : " 17 GB", | 20260507 04:25:59.059 | INFO | "huge" : "", | 20260507 04:25:59.059 | INFO | "huge_shared" : " 27 GB", | 20260507 04:25:59.059 | INFO | "malloc" : " 14 GB", | 20260507 04:25:59.059 | INFO | "pinned" : " 29 GB" | 20260507 04:25:59.059 | INFO | } | 20260507 04:25:59.059 | INFO | Perf report for the last 5.15016 seconds: | 20260507 04:25:59.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:25:59.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:59.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:59.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:59.059 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 04:25:59.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:59.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:59.059 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:25:59.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:59.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:25:59.059 | INFO | Uploaded 36GiB to GPU in 1.99649s (18.0317GiB/s) | 20260507 04:25:59.059 | INFO | Downloaded 34GiB to GPU in 1.9236s (17.6752GiB/s) | 20260507 04:26:04.192 | INFO | Analyzing chunk frames = [7079936,7080448) | 20260507 04:26:04.192 | INFO | MemInfo: { | 20260507 04:26:04.192 | INFO | "gpu" : " 17 GB", | 20260507 04:26:04.192 | INFO | "huge" : "", | 20260507 04:26:04.192 | INFO | "huge_shared" : " 27 GB", | 20260507 04:26:04.192 | INFO | "malloc" : " 14 GB", | 20260507 04:26:04.192 | INFO | "pinned" : " 29 GB" | 20260507 04:26:04.192 | INFO | } | 20260507 04:26:04.192 | INFO | Perf report for the last 5.13289 seconds: | 20260507 04:26:04.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:26:04.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:04.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:04.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:04.192 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 04:26:04.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:04.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:04.192 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:26:04.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:04.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:04.192 | INFO | Uploaded 36GiB to GPU in 2.02504s (17.7775GiB/s) | 20260507 04:26:04.192 | INFO | Downloaded 34GiB to GPU in 1.90541s (17.8439GiB/s) | 20260507 04:26:09.329 | INFO | Analyzing chunk frames = [7080448,7080960) | 20260507 04:26:09.329 | INFO | MemInfo: { | 20260507 04:26:09.329 | INFO | "gpu" : " 17 GB", | 20260507 04:26:09.329 | INFO | "huge" : "", | 20260507 04:26:09.329 | INFO | "huge_shared" : " 27 GB", | 20260507 04:26:09.329 | INFO | "malloc" : " 14 GB", | 20260507 04:26:09.329 | INFO | "pinned" : " 29 GB" | 20260507 04:26:09.329 | INFO | } | 20260507 04:26:09.330 | INFO | Perf report for the last 5.1374 seconds: | 20260507 04:26:09.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:26:09.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:09.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:09.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:09.330 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 04:26:09.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:09.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:09.330 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:26:09.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:09.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:09.330 | INFO | Uploaded 36GiB to GPU in 2.04923s (17.5676GiB/s) | 20260507 04:26:09.330 | INFO | Downloaded 34GiB to GPU in 1.91032s (17.7981GiB/s) | 20260507 04:26:14.469 | INFO | Analyzing chunk frames = [7080960,7081472) | 20260507 04:26:14.469 | INFO | MemInfo: { | 20260507 04:26:14.469 | INFO | "gpu" : " 17 GB", | 20260507 04:26:14.469 | INFO | "huge" : "", | 20260507 04:26:14.469 | INFO | "huge_shared" : " 27 GB", | 20260507 04:26:14.469 | INFO | "malloc" : " 14 GB", | 20260507 04:26:14.469 | INFO | "pinned" : " 29 GB" | 20260507 04:26:14.469 | INFO | } | 20260507 04:26:14.469 | INFO | Perf report for the last 5.13986 seconds: | 20260507 04:26:14.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:26:14.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:14.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:14.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:14.469 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 04:26:14.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:14.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:14.469 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:26:14.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:14.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:14.470 | INFO | Uploaded 36GiB to GPU in 1.97921s (18.189GiB/s) | 20260507 04:26:14.470 | INFO | Downloaded 34GiB to GPU in 1.98451s (17.1327GiB/s) | 20260507 04:26:19.609 | INFO | Analyzing chunk frames = [7081472,7081984) | 20260507 04:26:19.609 | INFO | MemInfo: { | 20260507 04:26:19.609 | INFO | "gpu" : " 17 GB", | 20260507 04:26:19.609 | INFO | "huge" : "", | 20260507 04:26:19.609 | INFO | "huge_shared" : " 27 GB", | 20260507 04:26:19.609 | INFO | "malloc" : " 14 GB", | 20260507 04:26:19.609 | INFO | "pinned" : " 29 GB" | 20260507 04:26:19.609 | INFO | } | 20260507 04:26:19.609 | INFO | Perf report for the last 5.14007 seconds: | 20260507 04:26:19.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:26:19.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:19.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:19.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:19.610 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 04:26:19.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:19.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:19.610 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:26:19.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:19.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:19.610 | INFO | Uploaded 36GiB to GPU in 1.97352s (18.2415GiB/s) | 20260507 04:26:19.610 | INFO | Downloaded 34GiB to GPU in 1.95135s (17.4238GiB/s) | 20260507 04:26:19.707 | INFO | Starting a round of Baz IO | 20260507 04:26:22.813 | INFO | Populating RTMetrics report | 20260507 04:26:22.813 | INFO | RTMetrics report populated | 20260507 04:26:22.813 | INFO | Opening tmp RTMetrics json file | 20260507 04:26:22.815 | INFO | Serializing RTMetrics to json | 20260507 04:26:22.824 | INFO | Serialization done, swapping tmp file now... | 20260507 04:26:22.834 | INFO | RTMetrics done | 20260507 04:26:22.899 | INFO | Finished a round of Baz IO: Wrote 1.724 GiB in 3.192 seconds | 20260507 04:26:22.899 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.79% / 0.00% / 67.21% | 20260507 04:26:22.899 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.169774% / 0% / 0.000103315% | 20260507 04:26:22.899 | INFO | Baz IO overall: | 20260507 04:26:22.899 | INFO | BytesWritten : 3614742720942 | 20260507 04:26:22.899 | INFO | NumEvents : 1735380812508 | 20260507 04:26:24.744 | INFO | Analyzing chunk frames = [7081984,7082496) | 20260507 04:26:24.744 | INFO | MemInfo: { | 20260507 04:26:24.744 | INFO | "gpu" : " 17 GB", | 20260507 04:26:24.744 | INFO | "huge" : "", | 20260507 04:26:24.744 | INFO | "huge_shared" : " 27 GB", | 20260507 04:26:24.744 | INFO | "malloc" : " 14 GB", | 20260507 04:26:24.744 | INFO | "pinned" : " 29 GB" | 20260507 04:26:24.745 | INFO | } | 20260507 04:26:24.745 | INFO | Perf report for the last 5.13506 seconds: | 20260507 04:26:24.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:26:24.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:24.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:24.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:24.745 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 04:26:24.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:24.745 | INFO | RT_METRICS : 41.7%, 1.0, 21.8ms, 0 | 20260507 04:26:24.745 | INFO | PRE_HQ : 59.2%, 1.8, 55.0ms, 0 | 20260507 04:26:24.745 | INFO | BAZWRITER : 61.8%, 3.2, 101.4ms, 0 | 20260507 04:26:24.745 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:26:24.745 | INFO | Uploaded 36GiB to GPU in 2.0447s (17.6065GiB/s) | 20260507 04:26:24.745 | INFO | Downloaded 38GiB to GPU in 2.18229s (17.4129GiB/s) | 20260507 04:26:29.889 | INFO | Analyzing chunk frames = [7082496,7083008) | 20260507 04:26:29.889 | INFO | MemInfo: { | 20260507 04:26:29.889 | INFO | "gpu" : " 17 GB", | 20260507 04:26:29.889 | INFO | "huge" : "", | 20260507 04:26:29.889 | INFO | "huge_shared" : " 27 GB", | 20260507 04:26:29.889 | INFO | "malloc" : " 14 GB", | 20260507 04:26:29.889 | INFO | "pinned" : " 29 GB" | 20260507 04:26:29.889 | INFO | } | 20260507 04:26:29.889 | INFO | Perf report for the last 5.14479 seconds: | 20260507 04:26:29.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:26:29.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:29.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:29.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:29.889 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 04:26:29.889 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:29.889 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:29.889 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:26:29.889 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:29.889 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:29.890 | INFO | Uploaded 36GiB to GPU in 2.03799s (17.6644GiB/s) | 20260507 04:26:29.890 | INFO | Downloaded 34GiB to GPU in 1.94144s (17.5128GiB/s) | 20260507 04:26:35.026 | INFO | Analyzing chunk frames = [7083008,7083520) | 20260507 04:26:35.026 | INFO | MemInfo: { | 20260507 04:26:35.026 | INFO | "gpu" : " 17 GB", | 20260507 04:26:35.026 | INFO | "huge" : "", | 20260507 04:26:35.026 | INFO | "huge_shared" : " 27 GB", | 20260507 04:26:35.026 | INFO | "malloc" : " 14 GB", | 20260507 04:26:35.026 | INFO | "pinned" : " 29 GB" | 20260507 04:26:35.026 | INFO | } | 20260507 04:26:35.026 | INFO | Perf report for the last 5.13708 seconds: | 20260507 04:26:35.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:26:35.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:35.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:35.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:35.026 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 04:26:35.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:35.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:35.027 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:26:35.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:35.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:35.027 | INFO | Uploaded 36GiB to GPU in 2.04148s (17.6342GiB/s) | 20260507 04:26:35.027 | INFO | Downloaded 34GiB to GPU in 1.95489s (17.3923GiB/s) | 20260507 04:26:40.166 | INFO | Analyzing chunk frames = [7083520,7084032) | 20260507 04:26:40.166 | INFO | MemInfo: { | 20260507 04:26:40.166 | INFO | "gpu" : " 17 GB", | 20260507 04:26:40.166 | INFO | "huge" : "", | 20260507 04:26:40.166 | INFO | "huge_shared" : " 27 GB", | 20260507 04:26:40.166 | INFO | "malloc" : " 14 GB", | 20260507 04:26:40.166 | INFO | "pinned" : " 29 GB" | 20260507 04:26:40.166 | INFO | } | 20260507 04:26:40.166 | INFO | Perf report for the last 5.13953 seconds: | 20260507 04:26:40.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:26:40.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:40.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:40.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:40.166 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:26:40.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:40.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:40.166 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:26:40.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:40.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:40.166 | INFO | Uploaded 36GiB to GPU in 1.99054s (18.0856GiB/s) | 20260507 04:26:40.166 | INFO | Downloaded 34GiB to GPU in 1.93034s (17.6135GiB/s) | 20260507 04:26:45.308 | INFO | Analyzing chunk frames = [7084032,7084544) | 20260507 04:26:45.308 | INFO | MemInfo: { | 20260507 04:26:45.308 | INFO | "gpu" : " 17 GB", | 20260507 04:26:45.308 | INFO | "huge" : "", | 20260507 04:26:45.308 | INFO | "huge_shared" : " 27 GB", | 20260507 04:26:45.308 | INFO | "malloc" : " 14 GB", | 20260507 04:26:45.308 | INFO | "pinned" : " 29 GB" | 20260507 04:26:45.308 | INFO | } | 20260507 04:26:45.308 | INFO | Perf report for the last 5.14192 seconds: | 20260507 04:26:45.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:26:45.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:45.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:45.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:45.308 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 04:26:45.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:45.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:45.308 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:26:45.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:45.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:45.308 | INFO | Uploaded 36GiB to GPU in 2.03934s (17.6528GiB/s) | 20260507 04:26:45.308 | INFO | Downloaded 34GiB to GPU in 1.95068s (17.4299GiB/s) | 20260507 04:26:50.446 | INFO | Analyzing chunk frames = [7084544,7085056) | 20260507 04:26:50.446 | INFO | MemInfo: { | 20260507 04:26:50.446 | INFO | "gpu" : " 17 GB", | 20260507 04:26:50.446 | INFO | "huge" : "", | 20260507 04:26:50.446 | INFO | "huge_shared" : " 27 GB", | 20260507 04:26:50.446 | INFO | "malloc" : " 14 GB", | 20260507 04:26:50.446 | INFO | "pinned" : " 29 GB" | 20260507 04:26:50.446 | INFO | } | 20260507 04:26:50.446 | INFO | Perf report for the last 5.13787 seconds: | 20260507 04:26:50.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:26:50.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:50.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:50.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:50.446 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 04:26:50.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:50.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:50.446 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 04:26:50.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:50.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:50.446 | INFO | Uploaded 36GiB to GPU in 2.04233s (17.6269GiB/s) | 20260507 04:26:50.446 | INFO | Downloaded 34GiB to GPU in 1.92102s (17.699GiB/s) | 20260507 04:26:55.579 | INFO | Analyzing chunk frames = [7085056,7085568) | 20260507 04:26:55.579 | INFO | MemInfo: { | 20260507 04:26:55.579 | INFO | "gpu" : " 17 GB", | 20260507 04:26:55.579 | INFO | "huge" : "", | 20260507 04:26:55.579 | INFO | "huge_shared" : " 27 GB", | 20260507 04:26:55.579 | INFO | "malloc" : " 14 GB", | 20260507 04:26:55.579 | INFO | "pinned" : " 29 GB" | 20260507 04:26:55.579 | INFO | } | 20260507 04:26:55.579 | INFO | Perf report for the last 5.13299 seconds: | 20260507 04:26:55.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:26:55.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:55.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:55.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:55.579 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 04:26:55.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:55.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:55.579 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 04:26:55.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:55.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:26:55.579 | INFO | Uploaded 36GiB to GPU in 2.02726s (17.758GiB/s) | 20260507 04:26:55.579 | INFO | Downloaded 34GiB to GPU in 1.98773s (17.1049GiB/s) | 20260507 04:27:00.716 | INFO | Analyzing chunk frames = [7085568,7086080) | 20260507 04:27:00.716 | INFO | MemInfo: { | 20260507 04:27:00.716 | INFO | "gpu" : " 17 GB", | 20260507 04:27:00.716 | INFO | "huge" : "", | 20260507 04:27:00.716 | INFO | "huge_shared" : " 27 GB", | 20260507 04:27:00.716 | INFO | "malloc" : " 14 GB", | 20260507 04:27:00.716 | INFO | "pinned" : " 29 GB" | 20260507 04:27:00.716 | INFO | } | 20260507 04:27:00.716 | INFO | Perf report for the last 5.13747 seconds: | 20260507 04:27:00.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:27:00.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:00.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:00.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:00.716 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 04:27:00.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:00.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:00.716 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:27:00.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:00.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:00.717 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:27:00.717 | INFO | Total mem usage: 18 GiB | 20260507 04:27:00.717 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:27:00.717 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 04:27:00.717 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:27:00.717 | INFO | High Water Breakdown: | 20260507 04:27:00.717 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:27:00.717 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:27:00.717 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:27:00.717 | INFO | ------------------------------------------ | 20260507 04:27:00.717 | INFO | Uploaded 36GiB to GPU in 2.03175s (17.7187GiB/s) | 20260507 04:27:00.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:27:00.717 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:27:00.717 | INFO | Downloaded 34GiB to GPU in 1.9647s (17.3055GiB/s) | 20260507 04:27:00.717 | INFO | Total mem usage: 26.5 GiB | 20260507 04:27:00.717 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:27:00.717 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:27:00.717 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:27:00.717 | INFO | High Water Breakdown: | 20260507 04:27:00.717 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:27:00.717 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:27:00.717 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:27:00.717 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:27:00.717 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:27:00.717 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:27:00.717 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:27:00.717 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:27:00.717 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:27:00.717 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:27:00.717 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:27:00.717 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:27:00.717 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:27:00.717 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:27:00.717 | INFO | ------------------------------------------ | 20260507 04:27:00.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:27:00.717 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:27:00.717 | INFO | Total mem usage: 25.6 GiB | 20260507 04:27:00.717 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:27:00.717 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:27:00.717 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:27:00.717 | INFO | High Water Breakdown: | 20260507 04:27:00.717 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:27:00.717 | INFO | ------------------------------------------ | 20260507 04:27:00.717 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:27:00.717 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:27:00.717 | INFO | Total mem usage: 15.7 GiB | 20260507 04:27:00.717 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:27:00.717 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:27:00.717 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:27:00.717 | INFO | High Water Breakdown: | 20260507 04:27:00.717 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:27:00.717 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:27:00.718 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:27:00.718 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:27:00.718 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:27:00.718 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:27:00.718 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:27:00.718 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:27:00.718 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:27:00.718 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:27:00.718 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:27:00.718 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:27:00.718 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:27:00.718 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:27:00.718 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:27:00.718 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:27:00.718 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:27:00.718 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:27:00.718 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:27:00.718 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:27:00.718 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:27:00.718 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:27:00.718 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:27:00.718 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:27:00.718 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:27:00.718 | INFO | ------------------------------------------ | 20260507 04:27:00.718 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:27:00.718 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:27:00.718 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:27:00.718 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:27:00.718 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.34e+01 / 1.29e+00 | 20260507 04:27:00.718 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.00e-01, 1.03e-02 / 4.61e-03 | 20260507 04:27:00.718 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.48e+00, 5.10e-01 / 2.43e-01 | 20260507 04:27:00.718 | INFO | PerfReport: 3.22%, Histogram -- 9.90e-01 / 1.43e+00, 1.63e-01 / 1.18e-01 | 20260507 04:27:00.718 | INFO | PerfReport: 16.10%, DME -- 3.96e+01 / 4.39e+01, 3.78e+00 / 1.02e+00 | 20260507 04:27:00.718 | INFO | PerfReport: 57.74%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.36e+00 / 1.36e+00 | 20260507 04:27:00.718 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.67e+00, 3.80e-01 / 2.24e-01 | 20260507 04:27:00.718 | INFO | PerfReport: 3.02%, Metrics -- 9.27e-01 / 1.24e+00, 2.46e-01 / 1.29e-01 | 20260507 04:27:00.718 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.14e-01 / 3.61e-01, 5.05e-02 / 1.40e-02 | 20260507 04:27:00.718 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:27:00.718 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:27:00.718 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:27:00.718 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 04:27:00.718 | INFO | PerfReport: 21.88%, Upload -- 2.11e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260507 04:27:00.718 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260507 04:27:00.718 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:27:00.718 | INFO | PerfReport: 1.44%, LockUpload -- 1.39e+00 / 0.00e+00, 4.78e+00 / 0.00e+00 | 20260507 04:27:00.718 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260507 04:27:00.718 | INFO | PerfReport: 22.08%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 04:27:00.718 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.13e-02 / 0.00e+00 | 20260507 04:27:00.815 | INFO | Starting a round of Baz IO | 20260507 04:27:03.925 | INFO | Populating RTMetrics report | 20260507 04:27:03.925 | INFO | RTMetrics report populated | 20260507 04:27:03.925 | INFO | Opening tmp RTMetrics json file | 20260507 04:27:03.944 | INFO | Serializing RTMetrics to json | 20260507 04:27:03.945 | INFO | Serialization done, swapping tmp file now... | 20260507 04:27:03.946 | INFO | RTMetrics done | 20260507 04:27:04.010 | INFO | Finished a round of Baz IO: Wrote 1.725 GiB in 3.195 seconds | 20260507 04:27:04.010 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.79% / 0.00% / 67.21% | 20260507 04:27:04.010 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.169496% / 0% / 9.53674e-05% | 20260507 04:27:04.010 | INFO | Baz IO overall: | 20260507 04:27:04.010 | INFO | BytesWritten : 3616594470004 | 20260507 04:27:04.010 | INFO | NumEvents : 1736251999257 | 20260507 04:27:05.864 | INFO | Analyzing chunk frames = [7086080,7086592) | 20260507 04:27:05.864 | INFO | MemInfo: { | 20260507 04:27:05.864 | INFO | "gpu" : " 17 GB", | 20260507 04:27:05.864 | INFO | "huge" : "", | 20260507 04:27:05.864 | INFO | "huge_shared" : " 27 GB", | 20260507 04:27:05.864 | INFO | "malloc" : " 14 GB", | 20260507 04:27:05.864 | INFO | "pinned" : " 29 GB" | 20260507 04:27:05.864 | INFO | } | 20260507 04:27:05.864 | INFO | Perf report for the last 5.14805 seconds: | 20260507 04:27:05.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:27:05.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:05.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:05.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:05.864 | INFO | ANALYSIS : 61.8%, 3.0, 96.3ms, 0 | 20260507 04:27:05.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:05.864 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 04:27:05.864 | INFO | PRE_HQ : 59.4%, 1.8, 55.4ms, 0 | 20260507 04:27:05.864 | INFO | BAZWRITER : 61.6%, 3.2, 101.4ms, 0 | 20260507 04:27:05.864 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:27:05.865 | INFO | Uploaded 36GiB to GPU in 2.02961s (17.7374GiB/s) | 20260507 04:27:05.865 | INFO | Downloaded 38GiB to GPU in 2.18406s (17.3988GiB/s) | 20260507 04:27:11.004 | INFO | Analyzing chunk frames = [7086592,7087104) | 20260507 04:27:11.004 | INFO | MemInfo: { | 20260507 04:27:11.004 | INFO | "gpu" : " 17 GB", | 20260507 04:27:11.004 | INFO | "huge" : "", | 20260507 04:27:11.004 | INFO | "huge_shared" : " 27 GB", | 20260507 04:27:11.004 | INFO | "malloc" : " 14 GB", | 20260507 04:27:11.004 | INFO | "pinned" : " 29 GB" | 20260507 04:27:11.004 | INFO | } | 20260507 04:27:11.004 | INFO | Perf report for the last 5.13933 seconds: | 20260507 04:27:11.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:27:11.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:11.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:11.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:11.004 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:27:11.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:11.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:11.004 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 04:27:11.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:11.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:11.004 | INFO | Uploaded 36GiB to GPU in 2.0407s (17.641GiB/s) | 20260507 04:27:11.004 | INFO | Downloaded 34GiB to GPU in 1.94312s (17.4976GiB/s) | 20260507 04:27:16.145 | INFO | Analyzing chunk frames = [7087104,7087616) | 20260507 04:27:16.145 | INFO | MemInfo: { | 20260507 04:27:16.145 | INFO | "gpu" : " 17 GB", | 20260507 04:27:16.145 | INFO | "huge" : "", | 20260507 04:27:16.145 | INFO | "huge_shared" : " 27 GB", | 20260507 04:27:16.145 | INFO | "malloc" : " 14 GB", | 20260507 04:27:16.145 | INFO | "pinned" : " 29 GB" | 20260507 04:27:16.145 | INFO | } | 20260507 04:27:16.145 | INFO | Perf report for the last 5.14093 seconds: | 20260507 04:27:16.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:27:16.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:16.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:16.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:16.145 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 04:27:16.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:16.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:16.145 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:27:16.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:16.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:16.145 | INFO | Uploaded 36GiB to GPU in 2.0446s (17.6074GiB/s) | 20260507 04:27:16.145 | INFO | Downloaded 34GiB to GPU in 1.95383s (17.4017GiB/s) | 20260507 04:27:21.278 | INFO | Analyzing chunk frames = [7087616,7088128) | 20260507 04:27:21.278 | INFO | MemInfo: { | 20260507 04:27:21.278 | INFO | "gpu" : " 17 GB", | 20260507 04:27:21.278 | INFO | "huge" : "", | 20260507 04:27:21.278 | INFO | "huge_shared" : " 27 GB", | 20260507 04:27:21.278 | INFO | "malloc" : " 14 GB", | 20260507 04:27:21.278 | INFO | "pinned" : " 29 GB" | 20260507 04:27:21.278 | INFO | } | 20260507 04:27:21.278 | INFO | Perf report for the last 5.13343 seconds: | 20260507 04:27:21.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:27:21.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:21.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:21.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:21.278 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 04:27:21.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:21.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:21.278 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:27:21.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:21.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:21.278 | INFO | Uploaded 36GiB to GPU in 2.00562s (17.9496GiB/s) | 20260507 04:27:21.278 | INFO | Downloaded 34GiB to GPU in 1.94022s (17.5238GiB/s) | 20260507 04:27:26.420 | INFO | Analyzing chunk frames = [7088128,7088640) | 20260507 04:27:26.420 | INFO | MemInfo: { | 20260507 04:27:26.420 | INFO | "gpu" : " 17 GB", | 20260507 04:27:26.420 | INFO | "huge" : "", | 20260507 04:27:26.420 | INFO | "huge_shared" : " 27 GB", | 20260507 04:27:26.420 | INFO | "malloc" : " 14 GB", | 20260507 04:27:26.420 | INFO | "pinned" : " 29 GB" | 20260507 04:27:26.420 | INFO | } | 20260507 04:27:26.420 | INFO | Perf report for the last 5.14217 seconds: | 20260507 04:27:26.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:27:26.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:26.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:26.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:26.420 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 04:27:26.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:26.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:26.420 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:27:26.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:26.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:26.420 | INFO | Uploaded 36GiB to GPU in 2.02088s (17.814GiB/s) | 20260507 04:27:26.420 | INFO | Downloaded 34GiB to GPU in 1.94423s (17.4877GiB/s) | 20260507 04:27:31.559 | INFO | Analyzing chunk frames = [7088640,7089152) | 20260507 04:27:31.559 | INFO | MemInfo: { | 20260507 04:27:31.559 | INFO | "gpu" : " 17 GB", | 20260507 04:27:31.559 | INFO | "huge" : "", | 20260507 04:27:31.559 | INFO | "huge_shared" : " 27 GB", | 20260507 04:27:31.559 | INFO | "malloc" : " 14 GB", | 20260507 04:27:31.559 | INFO | "pinned" : " 29 GB" | 20260507 04:27:31.559 | INFO | } | 20260507 04:27:31.559 | INFO | Perf report for the last 5.13896 seconds: | 20260507 04:27:31.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:27:31.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:31.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:31.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:31.559 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 04:27:31.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:31.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:31.559 | INFO | PRE_HQ : 29.6%, 1.0, 15.3ms, 0 | 20260507 04:27:31.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:31.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:31.559 | INFO | Uploaded 36GiB to GPU in 2.03249s (17.7123GiB/s) | 20260507 04:27:31.559 | INFO | Downloaded 34GiB to GPU in 1.94066s (17.5198GiB/s) | 20260507 04:27:36.699 | INFO | Analyzing chunk frames = [7089152,7089664) | 20260507 04:27:36.699 | INFO | MemInfo: { | 20260507 04:27:36.699 | INFO | "gpu" : " 17 GB", | 20260507 04:27:36.699 | INFO | "huge" : "", | 20260507 04:27:36.699 | INFO | "huge_shared" : " 27 GB", | 20260507 04:27:36.699 | INFO | "malloc" : " 14 GB", | 20260507 04:27:36.699 | INFO | "pinned" : " 29 GB" | 20260507 04:27:36.699 | INFO | } | 20260507 04:27:36.699 | INFO | Perf report for the last 5.13958 seconds: | 20260507 04:27:36.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:27:36.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:36.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:36.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:36.699 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:27:36.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:36.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:36.699 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 04:27:36.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:36.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:36.699 | INFO | Uploaded 36GiB to GPU in 1.9664s (18.3076GiB/s) | 20260507 04:27:36.699 | INFO | Downloaded 34GiB to GPU in 1.96105s (17.3376GiB/s) | 20260507 04:27:41.835 | INFO | Analyzing chunk frames = [7089664,7090176) | 20260507 04:27:41.835 | INFO | MemInfo: { | 20260507 04:27:41.835 | INFO | "gpu" : " 17 GB", | 20260507 04:27:41.835 | INFO | "huge" : "", | 20260507 04:27:41.835 | INFO | "huge_shared" : " 27 GB", | 20260507 04:27:41.835 | INFO | "malloc" : " 14 GB", | 20260507 04:27:41.835 | INFO | "pinned" : " 29 GB" | 20260507 04:27:41.835 | INFO | } | 20260507 04:27:41.835 | INFO | Perf report for the last 5.13645 seconds: | 20260507 04:27:41.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:27:41.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:41.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:41.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:41.835 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:27:41.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:41.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:41.835 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 04:27:41.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:41.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:41.835 | INFO | Uploaded 36GiB to GPU in 1.97576s (18.2209GiB/s) | 20260507 04:27:41.836 | INFO | Downloaded 34GiB to GPU in 1.9692s (17.2659GiB/s) | 20260507 04:27:41.932 | INFO | Starting a round of Baz IO | 20260507 04:27:45.074 | INFO | Populating RTMetrics report | 20260507 04:27:45.074 | INFO | RTMetrics report populated | 20260507 04:27:45.074 | INFO | Opening tmp RTMetrics json file | 20260507 04:27:45.075 | INFO | Serializing RTMetrics to json | 20260507 04:27:45.076 | INFO | Serialization done, swapping tmp file now... | 20260507 04:27:45.077 | INFO | RTMetrics done | 20260507 04:27:45.162 | INFO | Finished a round of Baz IO: Wrote 1.724 GiB in 3.229 seconds | 20260507 04:27:45.162 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.79% / 0.00% / 67.21% | 20260507 04:27:45.162 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.169019% / 3.97364e-06% / 9.13938e-05% | 20260507 04:27:45.162 | INFO | Baz IO overall: | 20260507 04:27:45.162 | INFO | BytesWritten : 3618445103752 | 20260507 04:27:45.162 | INFO | NumEvents : 1737122692544 | 20260507 04:27:46.976 | INFO | Analyzing chunk frames = [7090176,7090688) | 20260507 04:27:46.976 | INFO | MemInfo: { | 20260507 04:27:46.976 | INFO | "gpu" : " 17 GB", | 20260507 04:27:46.976 | INFO | "huge" : "", | 20260507 04:27:46.976 | INFO | "huge_shared" : " 27 GB", | 20260507 04:27:46.976 | INFO | "malloc" : " 14 GB", | 20260507 04:27:46.976 | INFO | "pinned" : " 29 GB" | 20260507 04:27:46.976 | INFO | } | 20260507 04:27:46.976 | INFO | Perf report for the last 5.1409 seconds: | 20260507 04:27:46.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:27:46.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:46.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:46.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:46.976 | INFO | ANALYSIS : 61.8%, 3.0, 95.8ms, 0 | 20260507 04:27:46.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:46.976 | INFO | RT_METRICS : 45.9%, 1.0, 24.1ms, 0 | 20260507 04:27:46.976 | INFO | PRE_HQ : 60.6%, 2.0, 61.4ms, 0 | 20260507 04:27:46.976 | INFO | BAZWRITER : 62.4%, 3.4, 107.6ms, 0 | 20260507 04:27:46.976 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260507 04:27:46.976 | INFO | Uploaded 36GiB to GPU in 2.04588s (17.5963GiB/s) | 20260507 04:27:46.976 | INFO | Downloaded 38GiB to GPU in 2.18621s (17.3817GiB/s) | 20260507 04:27:52.117 | INFO | Analyzing chunk frames = [7090688,7091200) | 20260507 04:27:52.117 | INFO | MemInfo: { | 20260507 04:27:52.117 | INFO | "gpu" : " 17 GB", | 20260507 04:27:52.117 | INFO | "huge" : "", | 20260507 04:27:52.117 | INFO | "huge_shared" : " 27 GB", | 20260507 04:27:52.117 | INFO | "malloc" : " 14 GB", | 20260507 04:27:52.117 | INFO | "pinned" : " 29 GB" | 20260507 04:27:52.117 | INFO | } | 20260507 04:27:52.117 | INFO | Perf report for the last 5.14114 seconds: | 20260507 04:27:52.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:27:52.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:52.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:52.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:52.117 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:27:52.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:52.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:52.117 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 04:27:52.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:52.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:52.117 | INFO | Uploaded 36GiB to GPU in 2.03766s (17.6673GiB/s) | 20260507 04:27:52.117 | INFO | Downloaded 34GiB to GPU in 1.94216s (17.5063GiB/s) | 20260507 04:27:57.259 | INFO | Analyzing chunk frames = [7091200,7091712) | 20260507 04:27:57.259 | INFO | MemInfo: { | 20260507 04:27:57.259 | INFO | "gpu" : " 17 GB", | 20260507 04:27:57.259 | INFO | "huge" : "", | 20260507 04:27:57.259 | INFO | "huge_shared" : " 27 GB", | 20260507 04:27:57.259 | INFO | "malloc" : " 14 GB", | 20260507 04:27:57.259 | INFO | "pinned" : " 29 GB" | 20260507 04:27:57.259 | INFO | } | 20260507 04:27:57.259 | INFO | Perf report for the last 5.14201 seconds: | 20260507 04:27:57.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:27:57.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:57.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:57.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:57.259 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 04:27:57.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:57.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:57.259 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:27:57.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:57.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:27:57.260 | INFO | Uploaded 36GiB to GPU in 2.01439s (17.8714GiB/s) | 20260507 04:27:57.260 | INFO | Downloaded 34GiB to GPU in 1.94465s (17.4839GiB/s) | 20260507 04:28:02.390 | INFO | Analyzing chunk frames = [7091712,7092224) | 20260507 04:28:02.390 | INFO | MemInfo: { | 20260507 04:28:02.390 | INFO | "gpu" : " 17 GB", | 20260507 04:28:02.390 | INFO | "huge" : "", | 20260507 04:28:02.390 | INFO | "huge_shared" : " 27 GB", | 20260507 04:28:02.390 | INFO | "malloc" : " 14 GB", | 20260507 04:28:02.390 | INFO | "pinned" : " 29 GB" | 20260507 04:28:02.390 | INFO | } | 20260507 04:28:02.390 | INFO | Perf report for the last 5.13076 seconds: | 20260507 04:28:02.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:28:02.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:02.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:02.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:02.390 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 04:28:02.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:02.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:02.390 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:28:02.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:02.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:02.390 | INFO | Uploaded 36GiB to GPU in 1.98794s (18.1092GiB/s) | 20260507 04:28:02.390 | INFO | Downloaded 34GiB to GPU in 1.94739s (17.4592GiB/s) | 20260507 04:28:07.533 | INFO | Analyzing chunk frames = [7092224,7092736) | 20260507 04:28:07.533 | INFO | MemInfo: { | 20260507 04:28:07.533 | INFO | "gpu" : " 17 GB", | 20260507 04:28:07.533 | INFO | "huge" : "", | 20260507 04:28:07.533 | INFO | "huge_shared" : " 27 GB", | 20260507 04:28:07.533 | INFO | "malloc" : " 14 GB", | 20260507 04:28:07.533 | INFO | "pinned" : " 29 GB" | 20260507 04:28:07.533 | INFO | } | 20260507 04:28:07.533 | INFO | Perf report for the last 5.1428 seconds: | 20260507 04:28:07.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:28:07.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:07.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:07.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:07.533 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 04:28:07.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:07.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:07.533 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 04:28:07.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:07.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:07.533 | INFO | Uploaded 36GiB to GPU in 2.00306s (17.9725GiB/s) | 20260507 04:28:07.533 | INFO | Downloaded 34GiB to GPU in 1.94356s (17.4937GiB/s) | 20260507 04:28:12.677 | INFO | Analyzing chunk frames = [7092736,7093248) | 20260507 04:28:12.677 | INFO | MemInfo: { | 20260507 04:28:12.677 | INFO | "gpu" : " 17 GB", | 20260507 04:28:12.677 | INFO | "huge" : "", | 20260507 04:28:12.677 | INFO | "huge_shared" : " 27 GB", | 20260507 04:28:12.677 | INFO | "malloc" : " 14 GB", | 20260507 04:28:12.677 | INFO | "pinned" : " 29 GB" | 20260507 04:28:12.677 | INFO | } | 20260507 04:28:12.677 | INFO | Perf report for the last 5.1441 seconds: | 20260507 04:28:12.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:28:12.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:12.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:12.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:12.677 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:28:12.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:12.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:12.677 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 04:28:12.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:12.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:12.677 | INFO | Uploaded 36GiB to GPU in 2.05075s (17.5545GiB/s) | 20260507 04:28:12.677 | INFO | Downloaded 34GiB to GPU in 1.96788s (17.2775GiB/s) | 20260507 04:28:17.814 | INFO | Analyzing chunk frames = [7093248,7093760) | 20260507 04:28:17.814 | INFO | MemInfo: { | 20260507 04:28:17.814 | INFO | "gpu" : " 17 GB", | 20260507 04:28:17.814 | INFO | "huge" : "", | 20260507 04:28:17.814 | INFO | "huge_shared" : " 27 GB", | 20260507 04:28:17.814 | INFO | "malloc" : " 14 GB", | 20260507 04:28:17.814 | INFO | "pinned" : " 29 GB" | 20260507 04:28:17.814 | INFO | } | 20260507 04:28:17.814 | INFO | Perf report for the last 5.13721 seconds: | 20260507 04:28:17.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:28:17.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:17.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:17.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:17.814 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:28:17.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:17.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:17.814 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 04:28:17.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:17.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:17.815 | INFO | Uploaded 36GiB to GPU in 1.97671s (18.212GiB/s) | 20260507 04:28:17.815 | INFO | Downloaded 34GiB to GPU in 1.95772s (17.3672GiB/s) | 20260507 04:28:22.954 | INFO | Analyzing chunk frames = [7093760,7094272) | 20260507 04:28:22.954 | INFO | MemInfo: { | 20260507 04:28:22.954 | INFO | "gpu" : " 17 GB", | 20260507 04:28:22.954 | INFO | "huge" : "", | 20260507 04:28:22.954 | INFO | "huge_shared" : " 27 GB", | 20260507 04:28:22.954 | INFO | "malloc" : " 14 GB", | 20260507 04:28:22.954 | INFO | "pinned" : " 29 GB" | 20260507 04:28:22.954 | INFO | } | 20260507 04:28:22.954 | INFO | Perf report for the last 5.13963 seconds: | 20260507 04:28:22.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:28:22.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:22.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:22.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:22.954 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:28:22.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:22.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:22.954 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:28:22.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:22.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:22.954 | INFO | Uploaded 36GiB to GPU in 1.97977s (18.1839GiB/s) | 20260507 04:28:22.954 | INFO | Downloaded 34GiB to GPU in 1.97835s (17.186GiB/s) | 20260507 04:28:22.954 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:28:22.954 | INFO | Total mem usage: 18 GiB | 20260507 04:28:22.954 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:28:22.954 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.6%) | 20260507 04:28:22.954 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:28:22.954 | INFO | High Water Breakdown: | 20260507 04:28:22.954 | INFO | 60.2% : 8.63 GiB ( 75.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:28:22.954 | INFO | 39.6% : 5.68 GiB ( 84.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:28:22.954 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:28:22.954 | INFO | ------------------------------------------ | 20260507 04:28:22.954 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:28:22.954 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:28:22.954 | INFO | Total mem usage: 26.5 GiB | 20260507 04:28:22.954 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:28:22.954 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:28:22.955 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:28:22.955 | INFO | High Water Breakdown: | 20260507 04:28:22.955 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:28:22.955 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:28:22.955 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:28:22.955 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:28:22.955 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:28:22.955 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:28:22.955 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:28:22.955 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:28:22.955 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:28:22.955 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:28:22.955 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:28:22.955 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:28:22.955 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:28:22.955 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:28:22.955 | INFO | ------------------------------------------ | 20260507 04:28:22.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:28:22.955 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:28:22.955 | INFO | Total mem usage: 25.6 GiB | 20260507 04:28:22.955 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:28:22.955 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:28:22.955 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:28:22.955 | INFO | High Water Breakdown: | 20260507 04:28:22.955 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:28:22.955 | INFO | ------------------------------------------ | 20260507 04:28:22.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:28:22.955 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:28:22.955 | INFO | Total mem usage: 15.7 GiB | 20260507 04:28:22.955 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:28:22.955 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:28:22.955 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 04:28:22.955 | INFO | High Water Breakdown: | 20260507 04:28:22.955 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:28:22.955 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:28:22.955 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:28:22.955 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:28:22.955 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:28:22.955 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:28:22.955 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:28:22.955 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:28:22.955 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:28:22.955 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:28:22.955 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:28:22.955 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:28:22.955 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:28:22.955 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:28:22.955 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:28:22.955 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:28:22.955 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:28:22.955 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:28:22.955 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:28:22.955 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:28:22.955 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:28:22.955 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:28:22.955 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:28:22.955 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:28:22.955 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:28:22.955 | INFO | ------------------------------------------ | 20260507 04:28:22.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:28:22.955 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:28:22.955 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:28:22.955 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:28:22.955 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.50e+01, 1.33e+01 / 1.64e+00 | 20260507 04:28:22.955 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.29e-03 / 4.70e-03 | 20260507 04:28:22.955 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.45e+00, 5.10e-01 / 2.69e-01 | 20260507 04:28:22.955 | INFO | PerfReport: 3.22%, Histogram -- 9.90e-01 / 1.48e+00, 1.65e-01 / 1.09e-01 | 20260507 04:28:22.955 | INFO | PerfReport: 16.18%, DME -- 3.97e+01 / 4.47e+01, 4.03e+00 / 1.97e+00 | 20260507 04:28:22.955 | INFO | PerfReport: 57.62%, FrameLabeling -- 1.77e+01 / 1.99e+01, 1.23e+00 / 1.45e+00 | 20260507 04:28:22.955 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.14e+00 / 2.73e+00, 3.85e-01 / 1.75e-01 | 20260507 04:28:22.955 | INFO | PerfReport: 3.02%, Metrics -- 9.28e-01 / 1.26e+00, 2.47e-01 / 9.86e-02 | 20260507 04:28:22.955 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.13e-01 / 3.71e-01, 5.01e-02 / 1.73e-02 | 20260507 04:28:22.955 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:28:22.955 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:28:22.955 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:28:22.955 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 04:28:22.955 | INFO | PerfReport: 21.78%, Upload -- 2.10e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260507 04:28:22.955 | INFO | PerfReport: 21.44%, Download -- 2.06e+01 / 0.00e+00, 7.46e+00 / 0.00e+00 | 20260507 04:28:22.955 | INFO | PerfReport: 31.89%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 04:28:22.955 | INFO | PerfReport: 1.47%, LockUpload -- 1.42e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 04:28:22.955 | INFO | PerfReport: 1.40%, LockDownload -- 1.35e+00 / 0.00e+00, 3.18e+00 / 0.00e+00 | 20260507 04:28:22.955 | INFO | PerfReport: 22.01%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 04:28:22.955 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260507 04:28:23.051 | INFO | Starting a round of Baz IO | 20260507 04:28:26.152 | INFO | Populating RTMetrics report | 20260507 04:28:26.152 | INFO | RTMetrics report populated | 20260507 04:28:26.152 | INFO | Opening tmp RTMetrics json file | 20260507 04:28:26.158 | INFO | Serializing RTMetrics to json | 20260507 04:28:26.171 | INFO | Serialization done, swapping tmp file now... | 20260507 04:28:26.172 | INFO | RTMetrics done | 20260507 04:28:26.241 | INFO | Finished a round of Baz IO: Wrote 1.722 GiB in 3.19 seconds | 20260507 04:28:26.241 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.79% / 0.00% / 67.21% | 20260507 04:28:26.241 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.168117% / 0% / 8.74201e-05% | 20260507 04:28:26.241 | INFO | Baz IO overall: | 20260507 04:28:26.241 | INFO | BytesWritten : 3620293570212 | 20260507 04:28:26.241 | INFO | NumEvents : 1737991944269 | 20260507 04:28:28.095 | INFO | Analyzing chunk frames = [7094272,7094784) | 20260507 04:28:28.095 | INFO | MemInfo: { | 20260507 04:28:28.095 | INFO | "gpu" : " 17 GB", | 20260507 04:28:28.095 | INFO | "huge" : "", | 20260507 04:28:28.095 | INFO | "huge_shared" : " 27 GB", | 20260507 04:28:28.095 | INFO | "malloc" : " 14 GB", | 20260507 04:28:28.095 | INFO | "pinned" : " 29 GB" | 20260507 04:28:28.095 | INFO | } | 20260507 04:28:28.095 | INFO | Perf report for the last 5.14068 seconds: | 20260507 04:28:28.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:28:28.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:28.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:28.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:28.095 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:28:28.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:28.095 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:28:28.095 | INFO | PRE_HQ : 59.1%, 1.8, 55.0ms, 0 | 20260507 04:28:28.095 | INFO | BAZWRITER : 61.7%, 3.2, 101.3ms, 0 | 20260507 04:28:28.095 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:28:28.095 | INFO | Uploaded 36GiB to GPU in 2.05863s (17.4873GiB/s) | 20260507 04:28:28.095 | INFO | Downloaded 38GiB to GPU in 2.18634s (17.3806GiB/s) | 20260507 04:28:33.227 | INFO | Analyzing chunk frames = [7094784,7095296) | 20260507 04:28:33.227 | INFO | MemInfo: { | 20260507 04:28:33.227 | INFO | "gpu" : " 17 GB", | 20260507 04:28:33.227 | INFO | "huge" : "", | 20260507 04:28:33.227 | INFO | "huge_shared" : " 27 GB", | 20260507 04:28:33.227 | INFO | "malloc" : " 14 GB", | 20260507 04:28:33.227 | INFO | "pinned" : " 29 GB" | 20260507 04:28:33.227 | INFO | } | 20260507 04:28:33.228 | INFO | Perf report for the last 5.1329 seconds: | 20260507 04:28:33.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:28:33.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:33.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:33.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:33.228 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 04:28:33.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:33.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:33.228 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:28:33.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:33.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:33.228 | INFO | Uploaded 36GiB to GPU in 1.99862s (18.0124GiB/s) | 20260507 04:28:33.228 | INFO | Downloaded 34GiB to GPU in 1.93547s (17.5668GiB/s) | 20260507 04:28:38.365 | INFO | Analyzing chunk frames = [7095296,7095808) | 20260507 04:28:38.365 | INFO | MemInfo: { | 20260507 04:28:38.365 | INFO | "gpu" : " 17 GB", | 20260507 04:28:38.365 | INFO | "huge" : "", | 20260507 04:28:38.365 | INFO | "huge_shared" : " 27 GB", | 20260507 04:28:38.365 | INFO | "malloc" : " 14 GB", | 20260507 04:28:38.365 | INFO | "pinned" : " 29 GB" | 20260507 04:28:38.365 | INFO | } | 20260507 04:28:38.365 | INFO | Perf report for the last 5.13759 seconds: | 20260507 04:28:38.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:28:38.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:38.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:38.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:38.365 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 04:28:38.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:38.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:38.365 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:28:38.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:38.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:38.365 | INFO | Uploaded 36GiB to GPU in 2.00042s (17.9962GiB/s) | 20260507 04:28:38.365 | INFO | Downloaded 34GiB to GPU in 1.94017s (17.5242GiB/s) | 20260507 04:28:43.514 | INFO | Analyzing chunk frames = [7095808,7096320) | 20260507 04:28:43.514 | INFO | MemInfo: { | 20260507 04:28:43.514 | INFO | "gpu" : " 17 GB", | 20260507 04:28:43.514 | INFO | "huge" : "", | 20260507 04:28:43.514 | INFO | "huge_shared" : " 27 GB", | 20260507 04:28:43.514 | INFO | "malloc" : " 14 GB", | 20260507 04:28:43.514 | INFO | "pinned" : " 29 GB" | 20260507 04:28:43.514 | INFO | } | 20260507 04:28:43.514 | INFO | Perf report for the last 5.14926 seconds: | 20260507 04:28:43.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:28:43.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:43.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:43.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:43.514 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 04:28:43.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:43.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:43.514 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:28:43.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:43.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:43.515 | INFO | Uploaded 36GiB to GPU in 1.96729s (18.2993GiB/s) | 20260507 04:28:43.515 | INFO | Downloaded 34GiB to GPU in 1.91116s (17.7902GiB/s) | 20260507 04:28:48.650 | INFO | Analyzing chunk frames = [7096320,7096832) | 20260507 04:28:48.650 | INFO | MemInfo: { | 20260507 04:28:48.650 | INFO | "gpu" : " 17 GB", | 20260507 04:28:48.650 | INFO | "huge" : "", | 20260507 04:28:48.650 | INFO | "huge_shared" : " 27 GB", | 20260507 04:28:48.650 | INFO | "malloc" : " 14 GB", | 20260507 04:28:48.650 | INFO | "pinned" : " 29 GB" | 20260507 04:28:48.651 | INFO | } | 20260507 04:28:48.651 | INFO | Perf report for the last 5.13623 seconds: | 20260507 04:28:48.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:28:48.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:48.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:48.651 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:48.651 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 04:28:48.651 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:48.651 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:48.651 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:28:48.651 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:48.651 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:48.651 | INFO | Uploaded 36GiB to GPU in 2.0522s (17.5421GiB/s) | 20260507 04:28:48.651 | INFO | Downloaded 34GiB to GPU in 1.95298s (17.4093GiB/s) | 20260507 04:28:53.786 | INFO | Analyzing chunk frames = [7096832,7097344) | 20260507 04:28:53.786 | INFO | MemInfo: { | 20260507 04:28:53.786 | INFO | "gpu" : " 17 GB", | 20260507 04:28:53.786 | INFO | "huge" : "", | 20260507 04:28:53.786 | INFO | "huge_shared" : " 27 GB", | 20260507 04:28:53.786 | INFO | "malloc" : " 14 GB", | 20260507 04:28:53.786 | INFO | "pinned" : " 29 GB" | 20260507 04:28:53.786 | INFO | } | 20260507 04:28:53.786 | INFO | Perf report for the last 5.13546 seconds: | 20260507 04:28:53.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:28:53.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:53.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:53.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:53.786 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 04:28:53.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:53.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:53.786 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 04:28:53.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:53.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:53.786 | INFO | Uploaded 36GiB to GPU in 2.03062s (17.7286GiB/s) | 20260507 04:28:53.786 | INFO | Downloaded 34GiB to GPU in 1.904s (17.8572GiB/s) | 20260507 04:28:58.929 | INFO | Analyzing chunk frames = [7097344,7097856) | 20260507 04:28:58.929 | INFO | MemInfo: { | 20260507 04:28:58.929 | INFO | "gpu" : " 17 GB", | 20260507 04:28:58.929 | INFO | "huge" : "", | 20260507 04:28:58.929 | INFO | "huge_shared" : " 27 GB", | 20260507 04:28:58.929 | INFO | "malloc" : " 14 GB", | 20260507 04:28:58.929 | INFO | "pinned" : " 29 GB" | 20260507 04:28:58.929 | INFO | } | 20260507 04:28:58.929 | INFO | Perf report for the last 5.14304 seconds: | 20260507 04:28:58.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:28:58.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:58.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:58.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:58.929 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:28:58.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:58.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:58.929 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 04:28:58.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:58.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:28:58.929 | INFO | Uploaded 36GiB to GPU in 1.97698s (18.2096GiB/s) | 20260507 04:28:58.929 | INFO | Downloaded 34GiB to GPU in 1.96353s (17.3158GiB/s) | 20260507 04:29:04.069 | INFO | Analyzing chunk frames = [7097856,7098368) | 20260507 04:29:04.069 | INFO | MemInfo: { | 20260507 04:29:04.069 | INFO | "gpu" : " 17 GB", | 20260507 04:29:04.069 | INFO | "huge" : "", | 20260507 04:29:04.069 | INFO | "huge_shared" : " 27 GB", | 20260507 04:29:04.069 | INFO | "malloc" : " 14 GB", | 20260507 04:29:04.069 | INFO | "pinned" : " 29 GB" | 20260507 04:29:04.069 | INFO | } | 20260507 04:29:04.069 | INFO | Perf report for the last 5.13945 seconds: | 20260507 04:29:04.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:29:04.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:04.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:04.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:04.069 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:29:04.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:04.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:04.069 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:29:04.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:04.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:04.069 | INFO | Uploaded 36GiB to GPU in 1.96526s (18.3182GiB/s) | 20260507 04:29:04.069 | INFO | Downloaded 34GiB to GPU in 1.95238s (17.4146GiB/s) | 20260507 04:29:04.166 | INFO | Starting a round of Baz IO | 20260507 04:29:07.264 | INFO | Populating RTMetrics report | 20260507 04:29:07.264 | INFO | RTMetrics report populated | 20260507 04:29:07.265 | INFO | Opening tmp RTMetrics json file | 20260507 04:29:07.265 | INFO | Serializing RTMetrics to json | 20260507 04:29:07.266 | INFO | Serialization done, swapping tmp file now... | 20260507 04:29:07.267 | INFO | RTMetrics done | 20260507 04:29:07.352 | INFO | Finished a round of Baz IO: Wrote 1.722 GiB in 3.185 seconds | 20260507 04:29:07.352 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.79% / 0.00% / 67.21% | 20260507 04:29:07.352 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.168204% / 0% / 8.34465e-05% | 20260507 04:29:07.352 | INFO | Baz IO overall: | 20260507 04:29:07.352 | INFO | BytesWritten : 3622142696893 | 20260507 04:29:07.352 | INFO | NumEvents : 1738861957036 | 20260507 04:29:09.202 | INFO | Analyzing chunk frames = [7098368,7098880) | 20260507 04:29:09.202 | INFO | MemInfo: { | 20260507 04:29:09.203 | INFO | "gpu" : " 17 GB", | 20260507 04:29:09.203 | INFO | "huge" : "", | 20260507 04:29:09.203 | INFO | "huge_shared" : " 27 GB", | 20260507 04:29:09.203 | INFO | "malloc" : " 14 GB", | 20260507 04:29:09.203 | INFO | "pinned" : " 29 GB" | 20260507 04:29:09.203 | INFO | } | 20260507 04:29:09.203 | INFO | Perf report for the last 5.13401 seconds: | 20260507 04:29:09.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:29:09.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:09.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:09.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:09.203 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 04:29:09.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:09.203 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 04:29:09.203 | INFO | PRE_HQ : 59.1%, 1.8, 55.1ms, 0 | 20260507 04:29:09.203 | INFO | BAZWRITER : 61.5%, 3.2, 101.5ms, 0 | 20260507 04:29:09.203 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:29:09.203 | INFO | Uploaded 36GiB to GPU in 2.04374s (17.6148GiB/s) | 20260507 04:29:09.203 | INFO | Downloaded 38GiB to GPU in 2.17722s (17.4535GiB/s) | 20260507 04:29:14.343 | INFO | Analyzing chunk frames = [7098880,7099392) | 20260507 04:29:14.343 | INFO | MemInfo: { | 20260507 04:29:14.343 | INFO | "gpu" : " 17 GB", | 20260507 04:29:14.343 | INFO | "huge" : "", | 20260507 04:29:14.343 | INFO | "huge_shared" : " 27 GB", | 20260507 04:29:14.343 | INFO | "malloc" : " 14 GB", | 20260507 04:29:14.343 | INFO | "pinned" : " 29 GB" | 20260507 04:29:14.343 | INFO | } | 20260507 04:29:14.343 | INFO | Perf report for the last 5.14017 seconds: | 20260507 04:29:14.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:29:14.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:14.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:14.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:14.343 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 04:29:14.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:14.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:14.343 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:29:14.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:14.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:14.343 | INFO | Uploaded 36GiB to GPU in 2.0488s (17.5713GiB/s) | 20260507 04:29:14.343 | INFO | Downloaded 34GiB to GPU in 1.93739s (17.5494GiB/s) | 20260507 04:29:19.487 | INFO | Analyzing chunk frames = [7099392,7099904) | 20260507 04:29:19.487 | INFO | MemInfo: { | 20260507 04:29:19.487 | INFO | "gpu" : " 17 GB", | 20260507 04:29:19.487 | INFO | "huge" : "", | 20260507 04:29:19.487 | INFO | "huge_shared" : " 27 GB", | 20260507 04:29:19.487 | INFO | "malloc" : " 14 GB", | 20260507 04:29:19.487 | INFO | "pinned" : " 29 GB" | 20260507 04:29:19.487 | INFO | } | 20260507 04:29:19.487 | INFO | Perf report for the last 5.14448 seconds: | 20260507 04:29:19.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:29:19.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:19.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:19.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:19.487 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 04:29:19.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:19.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:19.487 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:29:19.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:19.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:19.488 | INFO | Uploaded 36GiB to GPU in 2.01607s (17.8565GiB/s) | 20260507 04:29:19.488 | INFO | Downloaded 34GiB to GPU in 1.93592s (17.5627GiB/s) | 20260507 04:29:24.633 | INFO | Analyzing chunk frames = [7099904,7100416) | 20260507 04:29:24.633 | INFO | MemInfo: { | 20260507 04:29:24.633 | INFO | "gpu" : " 17 GB", | 20260507 04:29:24.633 | INFO | "huge" : "", | 20260507 04:29:24.633 | INFO | "huge_shared" : " 27 GB", | 20260507 04:29:24.633 | INFO | "malloc" : " 14 GB", | 20260507 04:29:24.633 | INFO | "pinned" : " 29 GB" | 20260507 04:29:24.633 | INFO | } | 20260507 04:29:24.633 | INFO | Perf report for the last 5.14532 seconds: | 20260507 04:29:24.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:29:24.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:24.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:24.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:24.633 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 04:29:24.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:24.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:24.633 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:29:24.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:24.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:24.633 | INFO | Uploaded 36GiB to GPU in 1.98339s (18.1508GiB/s) | 20260507 04:29:24.633 | INFO | Downloaded 34GiB to GPU in 1.90845s (17.8155GiB/s) | 20260507 04:29:29.771 | INFO | Analyzing chunk frames = [7100416,7100928) | 20260507 04:29:29.771 | INFO | MemInfo: { | 20260507 04:29:29.771 | INFO | "gpu" : " 17 GB", | 20260507 04:29:29.771 | INFO | "huge" : "", | 20260507 04:29:29.771 | INFO | "huge_shared" : " 27 GB", | 20260507 04:29:29.771 | INFO | "malloc" : " 14 GB", | 20260507 04:29:29.771 | INFO | "pinned" : " 29 GB" | 20260507 04:29:29.771 | INFO | } | 20260507 04:29:29.771 | INFO | Perf report for the last 5.13863 seconds: | 20260507 04:29:29.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:29:29.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:29.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:29.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:29.771 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 04:29:29.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:29.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:29.771 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:29:29.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:29.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:29.772 | INFO | Uploaded 36GiB to GPU in 2.05663s (17.5044GiB/s) | 20260507 04:29:29.772 | INFO | Downloaded 34GiB to GPU in 1.9516s (17.4216GiB/s) | 20260507 04:29:34.902 | INFO | Analyzing chunk frames = [7100928,7101440) | 20260507 04:29:34.902 | INFO | MemInfo: { | 20260507 04:29:34.902 | INFO | "gpu" : " 17 GB", | 20260507 04:29:34.902 | INFO | "huge" : "", | 20260507 04:29:34.902 | INFO | "huge_shared" : " 27 GB", | 20260507 04:29:34.902 | INFO | "malloc" : " 14 GB", | 20260507 04:29:34.902 | INFO | "pinned" : " 29 GB" | 20260507 04:29:34.902 | INFO | } | 20260507 04:29:34.902 | INFO | Perf report for the last 5.13074 seconds: | 20260507 04:29:34.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:29:34.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:34.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:34.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:34.902 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 04:29:34.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:34.902 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:34.902 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 04:29:34.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:34.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:34.902 | INFO | Uploaded 36GiB to GPU in 2.03729s (17.6706GiB/s) | 20260507 04:29:34.902 | INFO | Downloaded 34GiB to GPU in 1.90748s (17.8246GiB/s) | 20260507 04:29:40.037 | INFO | Analyzing chunk frames = [7101440,7101952) | 20260507 04:29:40.037 | INFO | MemInfo: { | 20260507 04:29:40.037 | INFO | "gpu" : " 17 GB", | 20260507 04:29:40.037 | INFO | "huge" : "", | 20260507 04:29:40.037 | INFO | "huge_shared" : " 27 GB", | 20260507 04:29:40.037 | INFO | "malloc" : " 14 GB", | 20260507 04:29:40.037 | INFO | "pinned" : " 29 GB" | 20260507 04:29:40.037 | INFO | } | 20260507 04:29:40.037 | INFO | Perf report for the last 5.13503 seconds: | 20260507 04:29:40.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:29:40.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:40.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:40.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:40.037 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 04:29:40.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:40.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:40.037 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:29:40.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:40.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:40.037 | INFO | Uploaded 36GiB to GPU in 1.97169s (18.2585GiB/s) | 20260507 04:29:40.037 | INFO | Downloaded 34GiB to GPU in 1.98299s (17.1458GiB/s) | 20260507 04:29:45.178 | INFO | Analyzing chunk frames = [7101952,7102464) | 20260507 04:29:45.178 | INFO | MemInfo: { | 20260507 04:29:45.178 | INFO | "gpu" : " 17 GB", | 20260507 04:29:45.178 | INFO | "huge" : "", | 20260507 04:29:45.178 | INFO | "huge_shared" : " 27 GB", | 20260507 04:29:45.178 | INFO | "malloc" : " 14 GB", | 20260507 04:29:45.178 | INFO | "pinned" : " 29 GB" | 20260507 04:29:45.178 | INFO | } | 20260507 04:29:45.178 | INFO | Perf report for the last 5.14126 seconds: | 20260507 04:29:45.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:29:45.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:45.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:45.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:45.178 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:29:45.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:45.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:45.178 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 04:29:45.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:45.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:45.179 | INFO | Uploaded 36GiB to GPU in 1.98079s (18.1746GiB/s) | 20260507 04:29:45.179 | INFO | Downloaded 34GiB to GPU in 1.96358s (17.3153GiB/s) | 20260507 04:29:45.179 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:29:45.179 | INFO | Total mem usage: 18 GiB | 20260507 04:29:45.179 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:29:45.179 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 04:29:45.179 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:29:45.179 | INFO | High Water Breakdown: | 20260507 04:29:45.179 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:29:45.179 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:29:45.179 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:29:45.179 | INFO | ------------------------------------------ | 20260507 04:29:45.179 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:29:45.179 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:29:45.179 | INFO | Total mem usage: 26.5 GiB | 20260507 04:29:45.179 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:29:45.179 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:29:45.179 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:29:45.179 | INFO | High Water Breakdown: | 20260507 04:29:45.179 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:29:45.179 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:29:45.179 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:29:45.179 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:29:45.179 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:29:45.179 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:29:45.179 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:29:45.179 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:29:45.179 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:29:45.179 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:29:45.179 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:29:45.179 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:29:45.179 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:29:45.179 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:29:45.179 | INFO | ------------------------------------------ | 20260507 04:29:45.179 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:29:45.179 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:29:45.179 | INFO | Total mem usage: 25.6 GiB | 20260507 04:29:45.179 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:29:45.179 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:29:45.179 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:29:45.179 | INFO | High Water Breakdown: | 20260507 04:29:45.179 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:29:45.179 | INFO | ------------------------------------------ | 20260507 04:29:45.179 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:29:45.179 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:29:45.179 | INFO | Total mem usage: 15.7 GiB | 20260507 04:29:45.179 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:29:45.179 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:29:45.179 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:29:45.179 | INFO | High Water Breakdown: | 20260507 04:29:45.179 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:29:45.179 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:29:45.179 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:29:45.179 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:29:45.179 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:29:45.179 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:29:45.179 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:29:45.179 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:29:45.179 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:29:45.179 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:29:45.179 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:29:45.179 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:29:45.179 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:29:45.179 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:29:45.179 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:29:45.179 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:29:45.179 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:29:45.179 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:29:45.179 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:29:45.179 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:29:45.179 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:29:45.179 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:29:45.179 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:29:45.179 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:29:45.179 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:29:45.179 | INFO | ------------------------------------------ | 20260507 04:29:45.179 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:29:45.179 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:29:45.179 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:29:45.179 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:29:45.179 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.33e+01 / 1.28e+00 | 20260507 04:29:45.179 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.01e-01, 9.34e-03 / 5.07e-03 | 20260507 04:29:45.179 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.52e+00, 5.14e-01 / 2.13e-01 | 20260507 04:29:45.179 | INFO | PerfReport: 3.22%, Histogram -- 9.91e-01 / 1.47e+00, 1.68e-01 / 1.11e-01 | 20260507 04:29:45.179 | INFO | PerfReport: 16.10%, DME -- 3.96e+01 / 4.40e+01, 3.74e+00 / 1.32e+00 | 20260507 04:29:45.179 | INFO | PerfReport: 57.71%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.34e+00 / 1.57e+00 | 20260507 04:29:45.179 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.66e+00, 3.82e-01 / 2.48e-01 | 20260507 04:29:45.179 | INFO | PerfReport: 3.03%, Metrics -- 9.30e-01 / 1.24e+00, 2.51e-01 / 1.41e-01 | 20260507 04:29:45.179 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.13e-01 / 3.69e-01, 5.07e-02 / 2.20e-02 | 20260507 04:29:45.180 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:29:45.180 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:29:45.180 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:29:45.180 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 04:29:45.180 | INFO | PerfReport: 21.79%, Upload -- 2.10e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260507 04:29:45.180 | INFO | PerfReport: 21.33%, Download -- 2.05e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260507 04:29:45.180 | INFO | PerfReport: 31.96%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 04:29:45.180 | INFO | PerfReport: 1.49%, LockUpload -- 1.43e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 04:29:45.180 | INFO | PerfReport: 1.27%, LockDownload -- 1.22e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260507 04:29:45.180 | INFO | PerfReport: 22.14%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 04:29:45.180 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260507 04:29:45.276 | INFO | Starting a round of Baz IO | 20260507 04:29:48.383 | INFO | Populating RTMetrics report | 20260507 04:29:48.383 | INFO | RTMetrics report populated | 20260507 04:29:48.383 | INFO | Opening tmp RTMetrics json file | 20260507 04:29:48.400 | INFO | Serializing RTMetrics to json | 20260507 04:29:48.401 | INFO | Serialization done, swapping tmp file now... | 20260507 04:29:48.402 | INFO | RTMetrics done | 20260507 04:29:48.469 | INFO | Finished a round of Baz IO: Wrote 1.724 GiB in 3.193 seconds | 20260507 04:29:48.469 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.79% / 0.00% / 67.21% | 20260507 04:29:48.469 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.168359% / 0% / 7.54992e-05% | 20260507 04:29:48.469 | INFO | Baz IO overall: | 20260507 04:29:48.469 | INFO | BytesWritten : 3623993605532 | 20260507 04:29:48.469 | INFO | NumEvents : 1739733359195 | 20260507 04:29:50.324 | INFO | Analyzing chunk frames = [7102464,7102976) | 20260507 04:29:50.324 | INFO | MemInfo: { | 20260507 04:29:50.324 | INFO | "gpu" : " 17 GB", | 20260507 04:29:50.324 | INFO | "huge" : "", | 20260507 04:29:50.324 | INFO | "huge_shared" : " 27 GB", | 20260507 04:29:50.324 | INFO | "malloc" : " 14 GB", | 20260507 04:29:50.324 | INFO | "pinned" : " 29 GB" | 20260507 04:29:50.324 | INFO | } | 20260507 04:29:50.324 | INFO | Perf report for the last 5.14587 seconds: | 20260507 04:29:50.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:29:50.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:50.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:50.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:50.324 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 04:29:50.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:50.324 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 04:29:50.324 | INFO | PRE_HQ : 59.0%, 1.8, 55.0ms, 0 | 20260507 04:29:50.324 | INFO | BAZWRITER : 61.6%, 3.3, 104.0ms, 0 | 20260507 04:29:50.324 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 04:29:50.324 | INFO | Uploaded 36GiB to GPU in 2.03597s (17.682GiB/s) | 20260507 04:29:50.325 | INFO | Downloaded 38GiB to GPU in 2.17319s (17.4858GiB/s) | 20260507 04:29:55.468 | INFO | Analyzing chunk frames = [7102976,7103488) | 20260507 04:29:55.468 | INFO | MemInfo: { | 20260507 04:29:55.468 | INFO | "gpu" : " 17 GB", | 20260507 04:29:55.468 | INFO | "huge" : "", | 20260507 04:29:55.468 | INFO | "huge_shared" : " 27 GB", | 20260507 04:29:55.468 | INFO | "malloc" : " 14 GB", | 20260507 04:29:55.468 | INFO | "pinned" : " 29 GB" | 20260507 04:29:55.468 | INFO | } | 20260507 04:29:55.468 | INFO | Perf report for the last 5.14396 seconds: | 20260507 04:29:55.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:29:55.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:55.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:55.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:55.468 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:29:55.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:55.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:55.468 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260507 04:29:55.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:55.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:29:55.468 | INFO | Uploaded 36GiB to GPU in 2.04892s (17.5703GiB/s) | 20260507 04:29:55.468 | INFO | Downloaded 34GiB to GPU in 1.94116s (17.5153GiB/s) | 20260507 04:30:00.603 | INFO | Analyzing chunk frames = [7103488,7104000) | 20260507 04:30:00.603 | INFO | MemInfo: { | 20260507 04:30:00.603 | INFO | "gpu" : " 17 GB", | 20260507 04:30:00.603 | INFO | "huge" : "", | 20260507 04:30:00.603 | INFO | "huge_shared" : " 27 GB", | 20260507 04:30:00.603 | INFO | "malloc" : " 14 GB", | 20260507 04:30:00.603 | INFO | "pinned" : " 29 GB" | 20260507 04:30:00.603 | INFO | } | 20260507 04:30:00.603 | INFO | Perf report for the last 5.13453 seconds: | 20260507 04:30:00.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:30:00.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:00.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:00.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:00.603 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 04:30:00.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:00.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:00.603 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:30:00.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:00.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:00.603 | INFO | Uploaded 36GiB to GPU in 2.05852s (17.4883GiB/s) | 20260507 04:30:00.603 | INFO | Downloaded 34GiB to GPU in 1.948s (17.4538GiB/s) | 20260507 04:30:05.736 | INFO | Analyzing chunk frames = [7104000,7104512) | 20260507 04:30:05.736 | INFO | MemInfo: { | 20260507 04:30:05.736 | INFO | "gpu" : " 17 GB", | 20260507 04:30:05.736 | INFO | "huge" : "", | 20260507 04:30:05.736 | INFO | "huge_shared" : " 27 GB", | 20260507 04:30:05.736 | INFO | "malloc" : " 14 GB", | 20260507 04:30:05.736 | INFO | "pinned" : " 29 GB" | 20260507 04:30:05.736 | INFO | } | 20260507 04:30:05.736 | INFO | Perf report for the last 5.13341 seconds: | 20260507 04:30:05.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:30:05.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:05.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:05.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:05.736 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 04:30:05.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:05.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:05.736 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:30:05.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:05.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:05.736 | INFO | Uploaded 36GiB to GPU in 2.01599s (17.8572GiB/s) | 20260507 04:30:05.736 | INFO | Downloaded 34GiB to GPU in 1.93626s (17.5597GiB/s) | 20260507 04:30:10.877 | INFO | Analyzing chunk frames = [7104512,7105024) | 20260507 04:30:10.877 | INFO | MemInfo: { | 20260507 04:30:10.877 | INFO | "gpu" : " 17 GB", | 20260507 04:30:10.877 | INFO | "huge" : "", | 20260507 04:30:10.877 | INFO | "huge_shared" : " 27 GB", | 20260507 04:30:10.877 | INFO | "malloc" : " 14 GB", | 20260507 04:30:10.877 | INFO | "pinned" : " 29 GB" | 20260507 04:30:10.877 | INFO | } | 20260507 04:30:10.877 | INFO | Perf report for the last 5.1405 seconds: | 20260507 04:30:10.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:30:10.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:10.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:10.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:10.877 | INFO | ANALYSIS : 59.9%, 3.0, 93.2ms, 0 | 20260507 04:30:10.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:10.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:10.877 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:30:10.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:10.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:10.877 | INFO | Uploaded 36GiB to GPU in 2.01243s (17.8888GiB/s) | 20260507 04:30:10.877 | INFO | Downloaded 34GiB to GPU in 1.95222s (17.4161GiB/s) | 20260507 04:30:16.015 | INFO | Analyzing chunk frames = [7105024,7105536) | 20260507 04:30:16.015 | INFO | MemInfo: { | 20260507 04:30:16.015 | INFO | "gpu" : " 17 GB", | 20260507 04:30:16.015 | INFO | "huge" : "", | 20260507 04:30:16.015 | INFO | "huge_shared" : " 27 GB", | 20260507 04:30:16.015 | INFO | "malloc" : " 14 GB", | 20260507 04:30:16.015 | INFO | "pinned" : " 29 GB" | 20260507 04:30:16.015 | INFO | } | 20260507 04:30:16.015 | INFO | Perf report for the last 5.13879 seconds: | 20260507 04:30:16.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:30:16.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:16.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:16.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:16.016 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:30:16.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:16.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:16.016 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:30:16.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:16.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:16.016 | INFO | Uploaded 36GiB to GPU in 2.05817s (17.4913GiB/s) | 20260507 04:30:16.016 | INFO | Downloaded 34GiB to GPU in 1.9635s (17.316GiB/s) | 20260507 04:30:21.155 | INFO | Analyzing chunk frames = [7105536,7106048) | 20260507 04:30:21.155 | INFO | MemInfo: { | 20260507 04:30:21.155 | INFO | "gpu" : " 17 GB", | 20260507 04:30:21.155 | INFO | "huge" : "", | 20260507 04:30:21.155 | INFO | "huge_shared" : " 27 GB", | 20260507 04:30:21.155 | INFO | "malloc" : " 14 GB", | 20260507 04:30:21.155 | INFO | "pinned" : " 29 GB" | 20260507 04:30:21.155 | INFO | } | 20260507 04:30:21.155 | INFO | Perf report for the last 5.13935 seconds: | 20260507 04:30:21.155 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:30:21.155 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:21.155 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:21.155 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:21.155 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 04:30:21.155 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:21.155 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:21.155 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 04:30:21.155 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:21.155 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:21.155 | INFO | Uploaded 36GiB to GPU in 1.98498s (18.1362GiB/s) | 20260507 04:30:21.155 | INFO | Downloaded 34GiB to GPU in 1.99304s (17.0593GiB/s) | 20260507 04:30:26.298 | INFO | Analyzing chunk frames = [7106048,7106560) | 20260507 04:30:26.298 | INFO | MemInfo: { | 20260507 04:30:26.298 | INFO | "gpu" : " 17 GB", | 20260507 04:30:26.298 | INFO | "huge" : "", | 20260507 04:30:26.298 | INFO | "huge_shared" : " 27 GB", | 20260507 04:30:26.298 | INFO | "malloc" : " 14 GB", | 20260507 04:30:26.298 | INFO | "pinned" : " 29 GB" | 20260507 04:30:26.298 | INFO | } | 20260507 04:30:26.298 | INFO | Perf report for the last 5.1431 seconds: | 20260507 04:30:26.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:30:26.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:26.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:26.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:26.298 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 04:30:26.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:26.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:26.298 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 04:30:26.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:26.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:26.298 | INFO | Uploaded 36GiB to GPU in 1.96161s (18.3523GiB/s) | 20260507 04:30:26.298 | INFO | Downloaded 34GiB to GPU in 1.95907s (17.3552GiB/s) | 20260507 04:30:26.395 | INFO | Starting a round of Baz IO | 20260507 04:30:29.497 | INFO | Populating RTMetrics report | 20260507 04:30:29.498 | INFO | RTMetrics report populated | 20260507 04:30:29.498 | INFO | Opening tmp RTMetrics json file | 20260507 04:30:29.500 | INFO | Serializing RTMetrics to json | 20260507 04:30:29.516 | INFO | Serialization done, swapping tmp file now... | 20260507 04:30:29.516 | INFO | RTMetrics done | 20260507 04:30:29.587 | INFO | Finished a round of Baz IO: Wrote 1.722 GiB in 3.191 seconds | 20260507 04:30:29.587 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.79% / 0.00% / 67.21% | 20260507 04:30:29.587 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.166972% / 0% / 0.000111262% | 20260507 04:30:29.587 | INFO | Baz IO overall: | 20260507 04:30:29.587 | INFO | BytesWritten : 3625842633907 | 20260507 04:30:29.587 | INFO | NumEvents : 1740603285604 | 20260507 04:30:31.438 | INFO | Analyzing chunk frames = [7106560,7107072) | 20260507 04:30:31.438 | INFO | MemInfo: { | 20260507 04:30:31.438 | INFO | "gpu" : " 17 GB", | 20260507 04:30:31.438 | INFO | "huge" : "", | 20260507 04:30:31.438 | INFO | "huge_shared" : " 27 GB", | 20260507 04:30:31.438 | INFO | "malloc" : " 14 GB", | 20260507 04:30:31.438 | INFO | "pinned" : " 29 GB" | 20260507 04:30:31.438 | INFO | } | 20260507 04:30:31.438 | INFO | Perf report for the last 5.13996 seconds: | 20260507 04:30:31.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:30:31.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:31.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:31.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:31.438 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:30:31.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:31.438 | INFO | RT_METRICS : 41.9%, 1.0, 22.0ms, 0 | 20260507 04:30:31.438 | INFO | PRE_HQ : 59.2%, 1.8, 55.7ms, 0 | 20260507 04:30:31.438 | INFO | BAZWRITER : 61.7%, 3.2, 102.5ms, 0 | 20260507 04:30:31.438 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:30:31.438 | INFO | Uploaded 36GiB to GPU in 2.04478s (17.6058GiB/s) | 20260507 04:30:31.439 | INFO | Downloaded 38GiB to GPU in 2.17769s (17.4497GiB/s) | 20260507 04:30:36.579 | INFO | Analyzing chunk frames = [7107072,7107584) | 20260507 04:30:36.579 | INFO | MemInfo: { | 20260507 04:30:36.579 | INFO | "gpu" : " 17 GB", | 20260507 04:30:36.579 | INFO | "huge" : "", | 20260507 04:30:36.579 | INFO | "huge_shared" : " 27 GB", | 20260507 04:30:36.579 | INFO | "malloc" : " 14 GB", | 20260507 04:30:36.579 | INFO | "pinned" : " 29 GB" | 20260507 04:30:36.579 | INFO | } | 20260507 04:30:36.579 | INFO | Perf report for the last 5.14082 seconds: | 20260507 04:30:36.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:30:36.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:36.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:36.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:36.579 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 04:30:36.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:36.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:36.579 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:30:36.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:36.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:36.579 | INFO | Uploaded 36GiB to GPU in 2.0422s (17.6281GiB/s) | 20260507 04:30:36.579 | INFO | Downloaded 34GiB to GPU in 1.94072s (17.5193GiB/s) | 20260507 04:30:41.716 | INFO | Analyzing chunk frames = [7107584,7108096) | 20260507 04:30:41.716 | INFO | MemInfo: { | 20260507 04:30:41.716 | INFO | "gpu" : " 17 GB", | 20260507 04:30:41.717 | INFO | "huge" : "", | 20260507 04:30:41.717 | INFO | "huge_shared" : " 27 GB", | 20260507 04:30:41.717 | INFO | "malloc" : " 14 GB", | 20260507 04:30:41.717 | INFO | "pinned" : " 29 GB" | 20260507 04:30:41.717 | INFO | } | 20260507 04:30:41.717 | INFO | Perf report for the last 5.13783 seconds: | 20260507 04:30:41.717 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:30:41.717 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:41.717 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:41.717 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:41.717 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 04:30:41.717 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:41.717 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:41.717 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:30:41.717 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:41.717 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:41.717 | INFO | Uploaded 36GiB to GPU in 2.04041s (17.6435GiB/s) | 20260507 04:30:41.717 | INFO | Downloaded 34GiB to GPU in 1.95113s (17.4258GiB/s) | 20260507 04:30:46.851 | INFO | Analyzing chunk frames = [7108096,7108608) | 20260507 04:30:46.851 | INFO | MemInfo: { | 20260507 04:30:46.851 | INFO | "gpu" : " 17 GB", | 20260507 04:30:46.851 | INFO | "huge" : "", | 20260507 04:30:46.851 | INFO | "huge_shared" : " 27 GB", | 20260507 04:30:46.851 | INFO | "malloc" : " 14 GB", | 20260507 04:30:46.851 | INFO | "pinned" : " 29 GB" | 20260507 04:30:46.852 | INFO | } | 20260507 04:30:46.852 | INFO | Perf report for the last 5.135 seconds: | 20260507 04:30:46.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:30:46.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:46.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:46.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:46.852 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 04:30:46.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:46.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:46.852 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:30:46.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:46.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:46.852 | INFO | Uploaded 36GiB to GPU in 1.98456s (18.14GiB/s) | 20260507 04:30:46.852 | INFO | Downloaded 34GiB to GPU in 1.93405s (17.5797GiB/s) | 20260507 04:30:51.990 | INFO | Analyzing chunk frames = [7108608,7109120) | 20260507 04:30:51.990 | INFO | MemInfo: { | 20260507 04:30:51.990 | INFO | "gpu" : " 17 GB", | 20260507 04:30:51.990 | INFO | "huge" : "", | 20260507 04:30:51.990 | INFO | "huge_shared" : " 27 GB", | 20260507 04:30:51.990 | INFO | "malloc" : " 14 GB", | 20260507 04:30:51.990 | INFO | "pinned" : " 29 GB" | 20260507 04:30:51.990 | INFO | } | 20260507 04:30:51.991 | INFO | Perf report for the last 5.13896 seconds: | 20260507 04:30:51.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:30:51.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:51.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:51.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:51.991 | INFO | ANALYSIS : 59.9%, 3.0, 93.5ms, 0 | 20260507 04:30:51.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:51.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:51.991 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:30:51.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:51.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:51.991 | INFO | Uploaded 36GiB to GPU in 2.01838s (17.8361GiB/s) | 20260507 04:30:51.991 | INFO | Downloaded 34GiB to GPU in 1.94423s (17.4876GiB/s) | 20260507 04:30:57.137 | INFO | Analyzing chunk frames = [7109120,7109632) | 20260507 04:30:57.137 | INFO | MemInfo: { | 20260507 04:30:57.137 | INFO | "gpu" : " 17 GB", | 20260507 04:30:57.137 | INFO | "huge" : "", | 20260507 04:30:57.138 | INFO | "huge_shared" : " 27 GB", | 20260507 04:30:57.138 | INFO | "malloc" : " 14 GB", | 20260507 04:30:57.138 | INFO | "pinned" : " 29 GB" | 20260507 04:30:57.138 | INFO | } | 20260507 04:30:57.138 | INFO | Perf report for the last 5.14691 seconds: | 20260507 04:30:57.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:30:57.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:57.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:57.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:57.138 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 04:30:57.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:57.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:57.138 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:30:57.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:57.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:30:57.138 | INFO | Uploaded 36GiB to GPU in 2.03215s (17.7152GiB/s) | 20260507 04:30:57.138 | INFO | Downloaded 34GiB to GPU in 1.95736s (17.3703GiB/s) | 20260507 04:31:02.279 | INFO | Analyzing chunk frames = [7109632,7110144) | 20260507 04:31:02.279 | INFO | MemInfo: { | 20260507 04:31:02.279 | INFO | "gpu" : " 17 GB", | 20260507 04:31:02.279 | INFO | "huge" : "", | 20260507 04:31:02.279 | INFO | "huge_shared" : " 27 GB", | 20260507 04:31:02.279 | INFO | "malloc" : " 14 GB", | 20260507 04:31:02.279 | INFO | "pinned" : " 29 GB" | 20260507 04:31:02.279 | INFO | } | 20260507 04:31:02.279 | INFO | Perf report for the last 5.14148 seconds: | 20260507 04:31:02.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:31:02.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:02.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:02.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:02.279 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:31:02.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:02.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:02.279 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:31:02.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:02.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:02.279 | INFO | Uploaded 36GiB to GPU in 1.97861s (18.1946GiB/s) | 20260507 04:31:02.279 | INFO | Downloaded 34GiB to GPU in 1.98812s (17.1016GiB/s) | 20260507 04:31:07.414 | INFO | Analyzing chunk frames = [7110144,7110656) | 20260507 04:31:07.414 | INFO | MemInfo: { | 20260507 04:31:07.414 | INFO | "gpu" : " 17 GB", | 20260507 04:31:07.414 | INFO | "huge" : "", | 20260507 04:31:07.414 | INFO | "huge_shared" : " 27 GB", | 20260507 04:31:07.414 | INFO | "malloc" : " 14 GB", | 20260507 04:31:07.414 | INFO | "pinned" : " 29 GB" | 20260507 04:31:07.414 | INFO | } | 20260507 04:31:07.414 | INFO | Perf report for the last 5.13543 seconds: | 20260507 04:31:07.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:31:07.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:07.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:07.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:07.415 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 04:31:07.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:07.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:07.415 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:31:07.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:07.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:07.415 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:31:07.415 | INFO | Total mem usage: 18 GiB | 20260507 04:31:07.415 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:31:07.415 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 04:31:07.415 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:31:07.415 | INFO | High Water Breakdown: | 20260507 04:31:07.415 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:31:07.415 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:31:07.415 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:31:07.415 | INFO | Uploaded 36GiB to GPU in 1.98094s (18.1732GiB/s) | 20260507 04:31:07.415 | INFO | Downloaded 34GiB to GPU in 1.97596s (17.2068GiB/s) | 20260507 04:31:07.415 | INFO | ------------------------------------------ | 20260507 04:31:07.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:31:07.415 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:31:07.415 | INFO | Total mem usage: 26.5 GiB | 20260507 04:31:07.415 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:31:07.415 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:31:07.415 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:31:07.415 | INFO | High Water Breakdown: | 20260507 04:31:07.415 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:31:07.415 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:31:07.415 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:31:07.415 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:31:07.415 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:31:07.415 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:31:07.415 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:31:07.415 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:31:07.415 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:31:07.415 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:31:07.415 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:31:07.415 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:31:07.415 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:31:07.415 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:31:07.415 | INFO | ------------------------------------------ | 20260507 04:31:07.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:31:07.415 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:31:07.415 | INFO | Total mem usage: 25.6 GiB | 20260507 04:31:07.415 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:31:07.415 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:31:07.415 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:31:07.415 | INFO | High Water Breakdown: | 20260507 04:31:07.415 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:31:07.415 | INFO | ------------------------------------------ | 20260507 04:31:07.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:31:07.415 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:31:07.415 | INFO | Total mem usage: 15.7 GiB | 20260507 04:31:07.415 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:31:07.415 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:31:07.415 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:31:07.415 | INFO | High Water Breakdown: | 20260507 04:31:07.415 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:31:07.415 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:31:07.415 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:31:07.415 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:31:07.415 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:31:07.415 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:31:07.415 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:31:07.415 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:31:07.415 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:31:07.415 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:31:07.415 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:31:07.415 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:31:07.415 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:31:07.415 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:31:07.415 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:31:07.415 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:31:07.415 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:31:07.415 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:31:07.415 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:31:07.415 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:31:07.415 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:31:07.415 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:31:07.415 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:31:07.415 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:31:07.415 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:31:07.415 | INFO | ------------------------------------------ | 20260507 04:31:07.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:31:07.416 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:31:07.416 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:31:07.416 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:31:07.416 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.47e+01, 1.35e+01 / 1.09e+00 | 20260507 04:31:07.416 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 9.53e-03 / 4.14e-03 | 20260507 04:31:07.416 | INFO | PerfReport: 11.64%, Baseline -- 3.57e+00 / 4.48e+00, 4.90e-01 / 2.31e-01 | 20260507 04:31:07.416 | INFO | PerfReport: 3.21%, Histogram -- 9.82e-01 / 1.46e+00, 1.60e-01 / 1.19e-01 | 20260507 04:31:07.416 | INFO | PerfReport: 16.19%, DME -- 3.97e+01 / 4.42e+01, 3.75e+00 / 1.64e+00 | 20260507 04:31:07.416 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.18e+00 / 1.81e+00 | 20260507 04:31:07.416 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.14e+00 / 2.67e+00, 3.89e-01 / 2.20e-01 | 20260507 04:31:07.416 | INFO | PerfReport: 3.02%, Metrics -- 9.25e-01 / 1.22e+00, 2.42e-01 / 1.15e-01 | 20260507 04:31:07.416 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.12e-01 / 3.67e-01, 5.00e-02 / 2.29e-02 | 20260507 04:31:07.416 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:31:07.416 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:31:07.416 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:31:07.416 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 04:31:07.416 | INFO | PerfReport: 21.85%, Upload -- 2.10e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260507 04:31:07.416 | INFO | PerfReport: 21.47%, Download -- 2.07e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260507 04:31:07.416 | INFO | PerfReport: 31.85%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 04:31:07.416 | INFO | PerfReport: 1.45%, LockUpload -- 1.40e+00 / 0.00e+00, 4.78e+00 / 0.00e+00 | 20260507 04:31:07.416 | INFO | PerfReport: 1.36%, LockDownload -- 1.31e+00 / 0.00e+00, 3.10e+00 / 0.00e+00 | 20260507 04:31:07.416 | INFO | PerfReport: 21.99%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 04:31:07.416 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.35e-02 / 0.00e+00 | 20260507 04:31:07.511 | INFO | Starting a round of Baz IO | 20260507 04:31:10.611 | INFO | Populating RTMetrics report | 20260507 04:31:10.611 | INFO | RTMetrics report populated | 20260507 04:31:10.611 | INFO | Opening tmp RTMetrics json file | 20260507 04:31:10.629 | INFO | Serializing RTMetrics to json | 20260507 04:31:10.631 | INFO | Serialization done, swapping tmp file now... | 20260507 04:31:10.631 | INFO | RTMetrics done | 20260507 04:31:10.698 | INFO | Finished a round of Baz IO: Wrote 1.721 GiB in 3.186 seconds | 20260507 04:31:10.698 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.79% / 0.00% / 67.21% | 20260507 04:31:10.698 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.167429% / 0% / 5.96046e-05% | 20260507 04:31:10.698 | INFO | Baz IO overall: | 20260507 04:31:10.698 | INFO | BytesWritten : 3627690192220 | 20260507 04:31:10.698 | INFO | NumEvents : 1741472339617 | 20260507 04:31:12.546 | INFO | Analyzing chunk frames = [7110656,7111168) | 20260507 04:31:12.546 | INFO | MemInfo: { | 20260507 04:31:12.546 | INFO | "gpu" : " 17 GB", | 20260507 04:31:12.546 | INFO | "huge" : "", | 20260507 04:31:12.546 | INFO | "huge_shared" : " 27 GB", | 20260507 04:31:12.546 | INFO | "malloc" : " 14 GB", | 20260507 04:31:12.546 | INFO | "pinned" : " 29 GB" | 20260507 04:31:12.546 | INFO | } | 20260507 04:31:12.546 | INFO | Perf report for the last 5.13163 seconds: | 20260507 04:31:12.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:31:12.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:12.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:12.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:12.546 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260507 04:31:12.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:12.546 | INFO | RT_METRICS : 41.7%, 1.0, 21.8ms, 0 | 20260507 04:31:12.546 | INFO | PRE_HQ : 59.2%, 1.8, 55.1ms, 0 | 20260507 04:31:12.546 | INFO | BAZWRITER : 61.7%, 3.2, 101.9ms, 0 | 20260507 04:31:12.546 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:31:12.546 | INFO | Uploaded 36GiB to GPU in 2.04926s (17.5673GiB/s) | 20260507 04:31:12.546 | INFO | Downloaded 38GiB to GPU in 2.17662s (17.4582GiB/s) | 20260507 04:31:17.690 | INFO | Analyzing chunk frames = [7111168,7111680) | 20260507 04:31:17.690 | INFO | MemInfo: { | 20260507 04:31:17.690 | INFO | "gpu" : " 17 GB", | 20260507 04:31:17.690 | INFO | "huge" : "", | 20260507 04:31:17.690 | INFO | "huge_shared" : " 27 GB", | 20260507 04:31:17.690 | INFO | "malloc" : " 14 GB", | 20260507 04:31:17.690 | INFO | "pinned" : " 29 GB" | 20260507 04:31:17.690 | INFO | } | 20260507 04:31:17.690 | INFO | Perf report for the last 5.14419 seconds: | 20260507 04:31:17.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:31:17.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:17.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:17.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:17.690 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 04:31:17.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:17.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:17.690 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:31:17.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:17.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:17.691 | INFO | Uploaded 36GiB to GPU in 2.02393s (17.7872GiB/s) | 20260507 04:31:17.691 | INFO | Downloaded 34GiB to GPU in 1.94282s (17.5003GiB/s) | 20260507 04:31:22.834 | INFO | Analyzing chunk frames = [7111680,7112192) | 20260507 04:31:22.834 | INFO | MemInfo: { | 20260507 04:31:22.834 | INFO | "gpu" : " 17 GB", | 20260507 04:31:22.834 | INFO | "huge" : "", | 20260507 04:31:22.834 | INFO | "huge_shared" : " 27 GB", | 20260507 04:31:22.834 | INFO | "malloc" : " 14 GB", | 20260507 04:31:22.834 | INFO | "pinned" : " 29 GB" | 20260507 04:31:22.834 | INFO | } | 20260507 04:31:22.834 | INFO | Perf report for the last 5.1437 seconds: | 20260507 04:31:22.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:31:22.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:22.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:22.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:22.834 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 04:31:22.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:22.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:22.834 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:31:22.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:22.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:22.834 | INFO | Uploaded 36GiB to GPU in 1.99242s (18.0685GiB/s) | 20260507 04:31:22.834 | INFO | Downloaded 34GiB to GPU in 1.94122s (17.5147GiB/s) | 20260507 04:31:27.971 | INFO | Analyzing chunk frames = [7112192,7112704) | 20260507 04:31:27.971 | INFO | MemInfo: { | 20260507 04:31:27.971 | INFO | "gpu" : " 17 GB", | 20260507 04:31:27.971 | INFO | "huge" : "", | 20260507 04:31:27.971 | INFO | "huge_shared" : " 27 GB", | 20260507 04:31:27.971 | INFO | "malloc" : " 14 GB", | 20260507 04:31:27.971 | INFO | "pinned" : " 29 GB" | 20260507 04:31:27.971 | INFO | } | 20260507 04:31:27.971 | INFO | Perf report for the last 5.13676 seconds: | 20260507 04:31:27.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:31:27.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:27.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:27.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:27.971 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 04:31:27.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:27.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:27.971 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 04:31:27.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:27.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:27.971 | INFO | Uploaded 36GiB to GPU in 2.01071s (17.9041GiB/s) | 20260507 04:31:27.971 | INFO | Downloaded 34GiB to GPU in 1.94575s (17.4739GiB/s) | 20260507 04:31:33.111 | INFO | Analyzing chunk frames = [7112704,7113216) | 20260507 04:31:33.111 | INFO | MemInfo: { | 20260507 04:31:33.111 | INFO | "gpu" : " 17 GB", | 20260507 04:31:33.111 | INFO | "huge" : "", | 20260507 04:31:33.111 | INFO | "huge_shared" : " 27 GB", | 20260507 04:31:33.111 | INFO | "malloc" : " 14 GB", | 20260507 04:31:33.111 | INFO | "pinned" : " 29 GB" | 20260507 04:31:33.111 | INFO | } | 20260507 04:31:33.111 | INFO | Perf report for the last 5.13986 seconds: | 20260507 04:31:33.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:31:33.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:33.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:33.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:33.111 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 04:31:33.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:33.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:33.111 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:31:33.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:33.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:33.111 | INFO | Uploaded 36GiB to GPU in 2.05029s (17.5585GiB/s) | 20260507 04:31:33.111 | INFO | Downloaded 34GiB to GPU in 1.95064s (17.4301GiB/s) | 20260507 04:31:38.248 | INFO | Analyzing chunk frames = [7113216,7113728) | 20260507 04:31:38.248 | INFO | MemInfo: { | 20260507 04:31:38.248 | INFO | "gpu" : " 17 GB", | 20260507 04:31:38.248 | INFO | "huge" : "", | 20260507 04:31:38.248 | INFO | "huge_shared" : " 27 GB", | 20260507 04:31:38.248 | INFO | "malloc" : " 14 GB", | 20260507 04:31:38.248 | INFO | "pinned" : " 29 GB" | 20260507 04:31:38.248 | INFO | } | 20260507 04:31:38.248 | INFO | Perf report for the last 5.13704 seconds: | 20260507 04:31:38.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:31:38.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:38.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:38.248 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:38.248 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 04:31:38.248 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:38.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:38.248 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:31:38.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:38.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:38.248 | INFO | Uploaded 36GiB to GPU in 2.03722s (17.6711GiB/s) | 20260507 04:31:38.248 | INFO | Downloaded 34GiB to GPU in 1.92562s (17.6566GiB/s) | 20260507 04:31:43.393 | INFO | Analyzing chunk frames = [7113728,7114240) | 20260507 04:31:43.393 | INFO | MemInfo: { | 20260507 04:31:43.393 | INFO | "gpu" : " 17 GB", | 20260507 04:31:43.393 | INFO | "huge" : "", | 20260507 04:31:43.393 | INFO | "huge_shared" : " 27 GB", | 20260507 04:31:43.393 | INFO | "malloc" : " 14 GB", | 20260507 04:31:43.393 | INFO | "pinned" : " 29 GB" | 20260507 04:31:43.393 | INFO | } | 20260507 04:31:43.393 | INFO | Perf report for the last 5.14519 seconds: | 20260507 04:31:43.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:31:43.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:43.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:43.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:43.393 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:31:43.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:43.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:43.393 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 04:31:43.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:43.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:43.393 | INFO | Uploaded 36GiB to GPU in 1.96922s (18.2813GiB/s) | 20260507 04:31:43.393 | INFO | Downloaded 34GiB to GPU in 1.98366s (17.14GiB/s) | 20260507 04:31:48.526 | INFO | Analyzing chunk frames = [7114240,7114752) | 20260507 04:31:48.526 | INFO | MemInfo: { | 20260507 04:31:48.526 | INFO | "gpu" : " 17 GB", | 20260507 04:31:48.526 | INFO | "huge" : "", | 20260507 04:31:48.526 | INFO | "huge_shared" : " 27 GB", | 20260507 04:31:48.526 | INFO | "malloc" : " 14 GB", | 20260507 04:31:48.526 | INFO | "pinned" : " 29 GB" | 20260507 04:31:48.526 | INFO | } | 20260507 04:31:48.526 | INFO | Perf report for the last 5.13336 seconds: | 20260507 04:31:48.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:31:48.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:48.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:48.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:48.526 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 04:31:48.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:48.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:48.526 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 04:31:48.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:48.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:48.527 | INFO | Uploaded 36GiB to GPU in 1.98657s (18.1217GiB/s) | 20260507 04:31:48.527 | INFO | Downloaded 34GiB to GPU in 1.97288s (17.2337GiB/s) | 20260507 04:31:48.623 | INFO | Starting a round of Baz IO | 20260507 04:31:51.723 | INFO | Populating RTMetrics report | 20260507 04:31:51.724 | INFO | RTMetrics report populated | 20260507 04:31:51.724 | INFO | Opening tmp RTMetrics json file | 20260507 04:31:51.731 | INFO | Serializing RTMetrics to json | 20260507 04:31:51.742 | INFO | Serialization done, swapping tmp file now... | 20260507 04:31:51.742 | INFO | RTMetrics done | 20260507 04:31:51.812 | INFO | Finished a round of Baz IO: Wrote 1.721 GiB in 3.188 seconds | 20260507 04:31:51.812 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.78% / 0.00% / 67.21% | 20260507 04:31:51.812 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.166901% / 3.97364e-06% / 7.15256e-05% | 20260507 04:31:51.812 | INFO | Baz IO overall: | 20260507 04:31:51.812 | INFO | BytesWritten : 3629537907024 | 20260507 04:31:51.812 | INFO | NumEvents : 1742341517989 | 20260507 04:31:53.666 | INFO | Analyzing chunk frames = [7114752,7115264) | 20260507 04:31:53.666 | INFO | MemInfo: { | 20260507 04:31:53.666 | INFO | "gpu" : " 17 GB", | 20260507 04:31:53.666 | INFO | "huge" : "", | 20260507 04:31:53.666 | INFO | "huge_shared" : " 27 GB", | 20260507 04:31:53.666 | INFO | "malloc" : " 14 GB", | 20260507 04:31:53.666 | INFO | "pinned" : " 29 GB" | 20260507 04:31:53.666 | INFO | } | 20260507 04:31:53.667 | INFO | Perf report for the last 5.14022 seconds: | 20260507 04:31:53.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:31:53.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:53.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:53.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:53.667 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 04:31:53.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:53.667 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 04:31:53.667 | INFO | PRE_HQ : 59.2%, 1.8, 55.2ms, 0 | 20260507 04:31:53.667 | INFO | BAZWRITER : 61.6%, 3.2, 102.9ms, 0 | 20260507 04:31:53.667 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:31:53.667 | INFO | Uploaded 36GiB to GPU in 2.03986s (17.6482GiB/s) | 20260507 04:31:53.667 | INFO | Downloaded 38GiB to GPU in 2.17866s (17.4419GiB/s) | 20260507 04:31:58.807 | INFO | Analyzing chunk frames = [7115264,7115776) | 20260507 04:31:58.807 | INFO | MemInfo: { | 20260507 04:31:58.807 | INFO | "gpu" : " 17 GB", | 20260507 04:31:58.807 | INFO | "huge" : "", | 20260507 04:31:58.807 | INFO | "huge_shared" : " 27 GB", | 20260507 04:31:58.807 | INFO | "malloc" : " 14 GB", | 20260507 04:31:58.807 | INFO | "pinned" : " 29 GB" | 20260507 04:31:58.807 | INFO | } | 20260507 04:31:58.807 | INFO | Perf report for the last 5.14028 seconds: | 20260507 04:31:58.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:31:58.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:58.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:58.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:58.807 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:31:58.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:58.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:58.807 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:31:58.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:58.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:31:58.807 | INFO | Uploaded 36GiB to GPU in 1.998s (18.018GiB/s) | 20260507 04:31:58.807 | INFO | Downloaded 34GiB to GPU in 1.92601s (17.6531GiB/s) | 20260507 04:32:03.948 | INFO | Analyzing chunk frames = [7115776,7116288) | 20260507 04:32:03.948 | INFO | MemInfo: { | 20260507 04:32:03.948 | INFO | "gpu" : " 17 GB", | 20260507 04:32:03.948 | INFO | "huge" : "", | 20260507 04:32:03.948 | INFO | "huge_shared" : " 27 GB", | 20260507 04:32:03.948 | INFO | "malloc" : " 14 GB", | 20260507 04:32:03.948 | INFO | "pinned" : " 29 GB" | 20260507 04:32:03.948 | INFO | } | 20260507 04:32:03.948 | INFO | Perf report for the last 5.14105 seconds: | 20260507 04:32:03.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:32:03.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:03.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:03.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:03.948 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 04:32:03.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:03.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:03.948 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:32:03.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:03.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:03.948 | INFO | Uploaded 36GiB to GPU in 2.0136s (17.8785GiB/s) | 20260507 04:32:03.948 | INFO | Downloaded 34GiB to GPU in 1.94249s (17.5033GiB/s) | 20260507 04:32:09.086 | INFO | Analyzing chunk frames = [7116288,7116800) | 20260507 04:32:09.086 | INFO | MemInfo: { | 20260507 04:32:09.086 | INFO | "gpu" : " 17 GB", | 20260507 04:32:09.086 | INFO | "huge" : "", | 20260507 04:32:09.086 | INFO | "huge_shared" : " 27 GB", | 20260507 04:32:09.086 | INFO | "malloc" : " 14 GB", | 20260507 04:32:09.086 | INFO | "pinned" : " 29 GB" | 20260507 04:32:09.086 | INFO | } | 20260507 04:32:09.086 | INFO | Perf report for the last 5.13832 seconds: | 20260507 04:32:09.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:32:09.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:09.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:09.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:09.086 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 04:32:09.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:09.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:09.086 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:32:09.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:09.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:09.087 | INFO | Uploaded 36GiB to GPU in 2.00789s (17.9293GiB/s) | 20260507 04:32:09.087 | INFO | Downloaded 34GiB to GPU in 1.92442s (17.6676GiB/s) | 20260507 04:32:14.222 | INFO | Analyzing chunk frames = [7116800,7117312) | 20260507 04:32:14.222 | INFO | MemInfo: { | 20260507 04:32:14.222 | INFO | "gpu" : " 17 GB", | 20260507 04:32:14.222 | INFO | "huge" : "", | 20260507 04:32:14.222 | INFO | "huge_shared" : " 27 GB", | 20260507 04:32:14.222 | INFO | "malloc" : " 14 GB", | 20260507 04:32:14.222 | INFO | "pinned" : " 29 GB" | 20260507 04:32:14.222 | INFO | } | 20260507 04:32:14.222 | INFO | Perf report for the last 5.13605 seconds: | 20260507 04:32:14.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:32:14.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:14.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:14.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:14.222 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 04:32:14.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:14.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:14.222 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:32:14.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:14.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:14.223 | INFO | Uploaded 36GiB to GPU in 2.04338s (17.6179GiB/s) | 20260507 04:32:14.223 | INFO | Downloaded 34GiB to GPU in 1.93067s (17.6105GiB/s) | 20260507 04:32:19.360 | INFO | Analyzing chunk frames = [7117312,7117824) | 20260507 04:32:19.360 | INFO | MemInfo: { | 20260507 04:32:19.360 | INFO | "gpu" : " 17 GB", | 20260507 04:32:19.360 | INFO | "huge" : "", | 20260507 04:32:19.360 | INFO | "huge_shared" : " 27 GB", | 20260507 04:32:19.360 | INFO | "malloc" : " 14 GB", | 20260507 04:32:19.360 | INFO | "pinned" : " 29 GB" | 20260507 04:32:19.360 | INFO | } | 20260507 04:32:19.360 | INFO | Perf report for the last 5.13818 seconds: | 20260507 04:32:19.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:32:19.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:19.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:19.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:19.361 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 04:32:19.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:19.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:19.361 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 04:32:19.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:19.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:19.361 | INFO | Uploaded 36GiB to GPU in 2.04985s (17.5622GiB/s) | 20260507 04:32:19.361 | INFO | Downloaded 34GiB to GPU in 1.90446s (17.8529GiB/s) | 20260507 04:32:24.497 | INFO | Analyzing chunk frames = [7117824,7118336) | 20260507 04:32:24.497 | INFO | MemInfo: { | 20260507 04:32:24.497 | INFO | "gpu" : " 17 GB", | 20260507 04:32:24.497 | INFO | "huge" : "", | 20260507 04:32:24.497 | INFO | "huge_shared" : " 27 GB", | 20260507 04:32:24.497 | INFO | "malloc" : " 14 GB", | 20260507 04:32:24.497 | INFO | "pinned" : " 29 GB" | 20260507 04:32:24.497 | INFO | } | 20260507 04:32:24.497 | INFO | Perf report for the last 5.13696 seconds: | 20260507 04:32:24.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:32:24.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:24.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:24.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:24.497 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:32:24.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:24.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:24.498 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:32:24.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:24.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:24.498 | INFO | Uploaded 36GiB to GPU in 1.97762s (18.2037GiB/s) | 20260507 04:32:24.498 | INFO | Downloaded 34GiB to GPU in 1.9716s (17.2449GiB/s) | 20260507 04:32:29.643 | INFO | Analyzing chunk frames = [7118336,7118848) | 20260507 04:32:29.643 | INFO | MemInfo: { | 20260507 04:32:29.643 | INFO | "gpu" : " 17 GB", | 20260507 04:32:29.643 | INFO | "huge" : "", | 20260507 04:32:29.643 | INFO | "huge_shared" : " 27 GB", | 20260507 04:32:29.643 | INFO | "malloc" : " 14 GB", | 20260507 04:32:29.643 | INFO | "pinned" : " 29 GB" | 20260507 04:32:29.643 | INFO | } | 20260507 04:32:29.643 | INFO | Perf report for the last 5.14583 seconds: | 20260507 04:32:29.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:32:29.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:29.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:29.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:29.643 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 04:32:29.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:29.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:29.643 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:32:29.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:29.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:29.644 | INFO | Uploaded 36GiB to GPU in 1.96085s (18.3594GiB/s) | 20260507 04:32:29.644 | INFO | Downloaded 34GiB to GPU in 1.9599s (17.3479GiB/s) | 20260507 04:32:29.644 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:32:29.644 | INFO | Total mem usage: 18 GiB | 20260507 04:32:29.644 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:32:29.644 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 04:32:29.644 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:32:29.644 | INFO | High Water Breakdown: | 20260507 04:32:29.644 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:32:29.644 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:32:29.644 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:32:29.644 | INFO | ------------------------------------------ | 20260507 04:32:29.644 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:32:29.644 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:32:29.644 | INFO | Total mem usage: 26.5 GiB | 20260507 04:32:29.644 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:32:29.644 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:32:29.644 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:32:29.644 | INFO | High Water Breakdown: | 20260507 04:32:29.644 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:32:29.644 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:32:29.644 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:32:29.644 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:32:29.644 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:32:29.644 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:32:29.644 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:32:29.644 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:32:29.644 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:32:29.644 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:32:29.644 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:32:29.644 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:32:29.644 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:32:29.644 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:32:29.644 | INFO | ------------------------------------------ | 20260507 04:32:29.644 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:32:29.644 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:32:29.644 | INFO | Total mem usage: 25.6 GiB | 20260507 04:32:29.644 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:32:29.644 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:32:29.644 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:32:29.644 | INFO | High Water Breakdown: | 20260507 04:32:29.644 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:32:29.644 | INFO | ------------------------------------------ | 20260507 04:32:29.644 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:32:29.644 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:32:29.644 | INFO | Total mem usage: 15.7 GiB | 20260507 04:32:29.644 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:32:29.644 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:32:29.644 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:32:29.644 | INFO | High Water Breakdown: | 20260507 04:32:29.644 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:32:29.644 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:32:29.644 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:32:29.644 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:32:29.644 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:32:29.644 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:32:29.644 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:32:29.644 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:32:29.644 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:32:29.644 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:32:29.644 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:32:29.644 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:32:29.644 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:32:29.644 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:32:29.644 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:32:29.644 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:32:29.644 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:32:29.644 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:32:29.644 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:32:29.644 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:32:29.644 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:32:29.644 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:32:29.644 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:32:29.644 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:32:29.644 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:32:29.644 | INFO | ------------------------------------------ | 20260507 04:32:29.644 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:32:29.644 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:32:29.644 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:32:29.644 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:32:29.644 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.34e+01 / 1.37e+00 | 20260507 04:32:29.644 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.00e-01, 9.45e-03 / 4.79e-03 | 20260507 04:32:29.644 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.46e+00, 5.19e-01 / 2.78e-01 | 20260507 04:32:29.644 | INFO | PerfReport: 3.24%, Histogram -- 9.95e-01 / 1.46e+00, 1.70e-01 / 1.14e-01 | 20260507 04:32:29.644 | INFO | PerfReport: 16.12%, DME -- 3.96e+01 / 4.42e+01, 3.86e+00 / 1.74e+00 | 20260507 04:32:29.644 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.33e+00 / 1.64e+00 | 20260507 04:32:29.644 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.65e+00, 3.78e-01 / 2.42e-01 | 20260507 04:32:29.644 | INFO | PerfReport: 3.02%, Metrics -- 9.27e-01 / 1.24e+00, 2.48e-01 / 1.55e-01 | 20260507 04:32:29.644 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.13e-01 / 3.66e-01, 5.00e-02 / 1.94e-02 | 20260507 04:32:29.644 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:32:29.644 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:32:29.644 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:32:29.644 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 04:32:29.644 | INFO | PerfReport: 21.80%, Upload -- 2.10e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260507 04:32:29.644 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 04:32:29.644 | INFO | PerfReport: 31.97%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:32:29.644 | INFO | PerfReport: 1.46%, LockUpload -- 1.40e+00 / 0.00e+00, 4.73e+00 / 0.00e+00 | 20260507 04:32:29.645 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260507 04:32:29.645 | INFO | PerfReport: 22.08%, LockCompute -- 2.12e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 04:32:29.645 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260507 04:32:29.740 | INFO | Starting a round of Baz IO | 20260507 04:32:32.843 | INFO | Populating RTMetrics report | 20260507 04:32:32.843 | INFO | RTMetrics report populated | 20260507 04:32:32.843 | INFO | Opening tmp RTMetrics json file | 20260507 04:32:32.860 | INFO | Serializing RTMetrics to json | 20260507 04:32:32.861 | INFO | Serialization done, swapping tmp file now... | 20260507 04:32:32.861 | INFO | RTMetrics done | 20260507 04:32:32.939 | INFO | Finished a round of Baz IO: Wrote 1.721 GiB in 3.198 seconds | 20260507 04:32:32.939 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.78% / 0.00% / 67.22% | 20260507 04:32:32.939 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.166003% / 3.97364e-06% / 9.13938e-05% | 20260507 04:32:32.939 | INFO | Baz IO overall: | 20260507 04:32:32.939 | INFO | BytesWritten : 3631385419206 | 20260507 04:32:32.939 | INFO | NumEvents : 1743210559837 | 20260507 04:32:34.787 | INFO | Analyzing chunk frames = [7118848,7119360) | 20260507 04:32:34.787 | INFO | MemInfo: { | 20260507 04:32:34.787 | INFO | "gpu" : " 17 GB", | 20260507 04:32:34.787 | INFO | "huge" : "", | 20260507 04:32:34.787 | INFO | "huge_shared" : " 27 GB", | 20260507 04:32:34.787 | INFO | "malloc" : " 14 GB", | 20260507 04:32:34.787 | INFO | "pinned" : " 29 GB" | 20260507 04:32:34.787 | INFO | } | 20260507 04:32:34.787 | INFO | Perf report for the last 5.14385 seconds: | 20260507 04:32:34.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:32:34.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:34.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:34.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:34.787 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:32:34.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:34.787 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:32:34.787 | INFO | PRE_HQ : 59.1%, 1.8, 54.9ms, 0 | 20260507 04:32:34.787 | INFO | BAZWRITER : 61.8%, 3.2, 101.6ms, 0 | 20260507 04:32:34.787 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 04:32:34.787 | INFO | Uploaded 36GiB to GPU in 2.03814s (17.6632GiB/s) | 20260507 04:32:34.787 | INFO | Downloaded 38GiB to GPU in 2.18474s (17.3934GiB/s) | 20260507 04:32:39.922 | INFO | Analyzing chunk frames = [7119360,7119872) | 20260507 04:32:39.922 | INFO | MemInfo: { | 20260507 04:32:39.922 | INFO | "gpu" : " 17 GB", | 20260507 04:32:39.922 | INFO | "huge" : "", | 20260507 04:32:39.922 | INFO | "huge_shared" : " 27 GB", | 20260507 04:32:39.922 | INFO | "malloc" : " 14 GB", | 20260507 04:32:39.922 | INFO | "pinned" : " 29 GB" | 20260507 04:32:39.922 | INFO | } | 20260507 04:32:39.922 | INFO | Perf report for the last 5.13475 seconds: | 20260507 04:32:39.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:32:39.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:39.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:39.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:39.922 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:32:39.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:39.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:39.922 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:32:39.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:39.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:39.922 | INFO | Uploaded 36GiB to GPU in 1.97594s (18.2192GiB/s) | 20260507 04:32:39.922 | INFO | Downloaded 34GiB to GPU in 1.93147s (17.6031GiB/s) | 20260507 04:32:45.056 | INFO | Analyzing chunk frames = [7119872,7120384) | 20260507 04:32:45.056 | INFO | MemInfo: { | 20260507 04:32:45.056 | INFO | "gpu" : " 17 GB", | 20260507 04:32:45.056 | INFO | "huge" : "", | 20260507 04:32:45.056 | INFO | "huge_shared" : " 27 GB", | 20260507 04:32:45.056 | INFO | "malloc" : " 14 GB", | 20260507 04:32:45.056 | INFO | "pinned" : " 29 GB" | 20260507 04:32:45.056 | INFO | } | 20260507 04:32:45.056 | INFO | Perf report for the last 5.1341 seconds: | 20260507 04:32:45.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:32:45.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:45.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:45.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:45.056 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 04:32:45.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:45.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:45.056 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:32:45.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:45.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:45.056 | INFO | Uploaded 36GiB to GPU in 2.04275s (17.6233GiB/s) | 20260507 04:32:45.056 | INFO | Downloaded 34GiB to GPU in 1.95285s (17.4105GiB/s) | 20260507 04:32:50.201 | INFO | Analyzing chunk frames = [7120384,7120896) | 20260507 04:32:50.201 | INFO | MemInfo: { | 20260507 04:32:50.201 | INFO | "gpu" : " 17 GB", | 20260507 04:32:50.201 | INFO | "huge" : "", | 20260507 04:32:50.201 | INFO | "huge_shared" : " 27 GB", | 20260507 04:32:50.201 | INFO | "malloc" : " 14 GB", | 20260507 04:32:50.201 | INFO | "pinned" : " 29 GB" | 20260507 04:32:50.201 | INFO | } | 20260507 04:32:50.201 | INFO | Perf report for the last 5.14517 seconds: | 20260507 04:32:50.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:32:50.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:50.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:50.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:50.201 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:32:50.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:50.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:50.201 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:32:50.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:50.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:50.201 | INFO | Uploaded 36GiB to GPU in 1.95732s (18.3925GiB/s) | 20260507 04:32:50.202 | INFO | Downloaded 34GiB to GPU in 1.925s (17.6623GiB/s) | 20260507 04:32:55.339 | INFO | Analyzing chunk frames = [7120896,7121408) | 20260507 04:32:55.339 | INFO | MemInfo: { | 20260507 04:32:55.339 | INFO | "gpu" : " 17 GB", | 20260507 04:32:55.339 | INFO | "huge" : "", | 20260507 04:32:55.339 | INFO | "huge_shared" : " 27 GB", | 20260507 04:32:55.339 | INFO | "malloc" : " 14 GB", | 20260507 04:32:55.339 | INFO | "pinned" : " 29 GB" | 20260507 04:32:55.339 | INFO | } | 20260507 04:32:55.339 | INFO | Perf report for the last 5.13772 seconds: | 20260507 04:32:55.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:32:55.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:55.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:55.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:55.339 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 04:32:55.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:55.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:55.339 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:32:55.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:55.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:32:55.339 | INFO | Uploaded 36GiB to GPU in 2.02814s (17.7502GiB/s) | 20260507 04:32:55.339 | INFO | Downloaded 34GiB to GPU in 1.94352s (17.494GiB/s) | 20260507 04:33:00.477 | INFO | Analyzing chunk frames = [7121408,7121920) | 20260507 04:33:00.477 | INFO | MemInfo: { | 20260507 04:33:00.477 | INFO | "gpu" : " 17 GB", | 20260507 04:33:00.477 | INFO | "huge" : "", | 20260507 04:33:00.477 | INFO | "huge_shared" : " 27 GB", | 20260507 04:33:00.477 | INFO | "malloc" : " 14 GB", | 20260507 04:33:00.477 | INFO | "pinned" : " 29 GB" | 20260507 04:33:00.477 | INFO | } | 20260507 04:33:00.477 | INFO | Perf report for the last 5.138 seconds: | 20260507 04:33:00.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:33:00.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:00.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:00.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:00.477 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 04:33:00.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:00.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:00.477 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:33:00.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:00.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:00.477 | INFO | Uploaded 36GiB to GPU in 2.03469s (17.6931GiB/s) | 20260507 04:33:00.477 | INFO | Downloaded 34GiB to GPU in 1.92209s (17.6891GiB/s) | 20260507 04:33:05.618 | INFO | Analyzing chunk frames = [7121920,7122432) | 20260507 04:33:05.618 | INFO | MemInfo: { | 20260507 04:33:05.618 | INFO | "gpu" : " 17 GB", | 20260507 04:33:05.618 | INFO | "huge" : "", | 20260507 04:33:05.618 | INFO | "huge_shared" : " 27 GB", | 20260507 04:33:05.618 | INFO | "malloc" : " 14 GB", | 20260507 04:33:05.618 | INFO | "pinned" : " 29 GB" | 20260507 04:33:05.618 | INFO | } | 20260507 04:33:05.618 | INFO | Perf report for the last 5.14079 seconds: | 20260507 04:33:05.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:33:05.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:05.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:05.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:05.618 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:33:05.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:05.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:05.618 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 04:33:05.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:05.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:05.618 | INFO | Uploaded 36GiB to GPU in 1.96418s (18.3282GiB/s) | 20260507 04:33:05.618 | INFO | Downloaded 34GiB to GPU in 1.97558s (17.2101GiB/s) | 20260507 04:33:10.760 | INFO | Analyzing chunk frames = [7122432,7122944) | 20260507 04:33:10.760 | INFO | MemInfo: { | 20260507 04:33:10.760 | INFO | "gpu" : " 17 GB", | 20260507 04:33:10.760 | INFO | "huge" : "", | 20260507 04:33:10.760 | INFO | "huge_shared" : " 27 GB", | 20260507 04:33:10.760 | INFO | "malloc" : " 14 GB", | 20260507 04:33:10.760 | INFO | "pinned" : " 29 GB" | 20260507 04:33:10.760 | INFO | } | 20260507 04:33:10.761 | INFO | Perf report for the last 5.14271 seconds: | 20260507 04:33:10.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:33:10.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:10.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:10.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:10.761 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 04:33:10.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:10.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:10.761 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 04:33:10.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:10.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:10.761 | INFO | Uploaded 36GiB to GPU in 1.96884s (18.2849GiB/s) | 20260507 04:33:10.761 | INFO | Downloaded 34GiB to GPU in 1.95164s (17.4213GiB/s) | 20260507 04:33:10.858 | INFO | Starting a round of Baz IO | 20260507 04:33:13.964 | INFO | Populating RTMetrics report | 20260507 04:33:13.964 | INFO | RTMetrics report populated | 20260507 04:33:13.964 | INFO | Opening tmp RTMetrics json file | 20260507 04:33:13.970 | INFO | Serializing RTMetrics to json | 20260507 04:33:13.982 | INFO | Serialization done, swapping tmp file now... | 20260507 04:33:13.983 | INFO | RTMetrics done | 20260507 04:33:14.050 | INFO | Finished a round of Baz IO: Wrote 1.722 GiB in 3.191 seconds | 20260507 04:33:14.050 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.78% / 0.00% / 67.22% | 20260507 04:33:14.050 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.166198% / 0% / 6.75519e-05% | 20260507 04:33:14.050 | INFO | Baz IO overall: | 20260507 04:33:14.050 | INFO | BytesWritten : 3633234250495 | 20260507 04:33:14.050 | INFO | NumEvents : 1744080710844 | 20260507 04:33:15.899 | INFO | Analyzing chunk frames = [7122944,7123456) | 20260507 04:33:15.899 | INFO | MemInfo: { | 20260507 04:33:15.899 | INFO | "gpu" : " 17 GB", | 20260507 04:33:15.899 | INFO | "huge" : "", | 20260507 04:33:15.899 | INFO | "huge_shared" : " 27 GB", | 20260507 04:33:15.899 | INFO | "malloc" : " 14 GB", | 20260507 04:33:15.899 | INFO | "pinned" : " 29 GB" | 20260507 04:33:15.899 | INFO | } | 20260507 04:33:15.899 | INFO | Perf report for the last 5.13826 seconds: | 20260507 04:33:15.899 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:33:15.899 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:15.899 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:15.899 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:15.899 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:33:15.899 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:15.899 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 04:33:15.899 | INFO | PRE_HQ : 59.0%, 1.8, 55.1ms, 0 | 20260507 04:33:15.899 | INFO | BAZWRITER : 61.7%, 3.2, 102.2ms, 0 | 20260507 04:33:15.899 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:33:15.899 | INFO | Uploaded 36GiB to GPU in 2.05083s (17.5539GiB/s) | 20260507 04:33:15.899 | INFO | Downloaded 38GiB to GPU in 2.1819s (17.416GiB/s) | 20260507 04:33:21.044 | INFO | Analyzing chunk frames = [7123456,7123968) | 20260507 04:33:21.044 | INFO | MemInfo: { | 20260507 04:33:21.044 | INFO | "gpu" : " 17 GB", | 20260507 04:33:21.044 | INFO | "huge" : "", | 20260507 04:33:21.044 | INFO | "huge_shared" : " 27 GB", | 20260507 04:33:21.044 | INFO | "malloc" : " 14 GB", | 20260507 04:33:21.044 | INFO | "pinned" : " 29 GB" | 20260507 04:33:21.044 | INFO | } | 20260507 04:33:21.044 | INFO | Perf report for the last 5.14521 seconds: | 20260507 04:33:21.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:33:21.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:21.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:21.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:21.044 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 04:33:21.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:21.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:21.044 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:33:21.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:21.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:21.044 | INFO | Uploaded 36GiB to GPU in 2.05052s (17.5565GiB/s) | 20260507 04:33:21.044 | INFO | Downloaded 34GiB to GPU in 1.94133s (17.5138GiB/s) | 20260507 04:33:26.173 | INFO | Analyzing chunk frames = [7123968,7124480) | 20260507 04:33:26.173 | INFO | MemInfo: { | 20260507 04:33:26.173 | INFO | "gpu" : " 17 GB", | 20260507 04:33:26.173 | INFO | "huge" : "", | 20260507 04:33:26.173 | INFO | "huge_shared" : " 27 GB", | 20260507 04:33:26.173 | INFO | "malloc" : " 14 GB", | 20260507 04:33:26.173 | INFO | "pinned" : " 29 GB" | 20260507 04:33:26.173 | INFO | } | 20260507 04:33:26.173 | INFO | Perf report for the last 5.1292 seconds: | 20260507 04:33:26.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:33:26.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:26.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:26.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:26.173 | INFO | ANALYSIS : 60.7%, 3.0, 93.8ms, 0 | 20260507 04:33:26.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:26.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:26.173 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 04:33:26.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:26.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:26.173 | INFO | Uploaded 36GiB to GPU in 2.04853s (17.5736GiB/s) | 20260507 04:33:26.174 | INFO | Downloaded 34GiB to GPU in 1.94926s (17.4425GiB/s) | 20260507 04:33:31.308 | INFO | Analyzing chunk frames = [7124480,7124992) | 20260507 04:33:31.308 | INFO | MemInfo: { | 20260507 04:33:31.308 | INFO | "gpu" : " 17 GB", | 20260507 04:33:31.308 | INFO | "huge" : "", | 20260507 04:33:31.308 | INFO | "huge_shared" : " 27 GB", | 20260507 04:33:31.308 | INFO | "malloc" : " 14 GB", | 20260507 04:33:31.308 | INFO | "pinned" : " 29 GB" | 20260507 04:33:31.308 | INFO | } | 20260507 04:33:31.308 | INFO | Perf report for the last 5.13479 seconds: | 20260507 04:33:31.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:33:31.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:31.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:31.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:31.308 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 04:33:31.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:31.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:31.308 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:33:31.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:31.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:31.308 | INFO | Uploaded 36GiB to GPU in 1.99034s (18.0873GiB/s) | 20260507 04:33:31.308 | INFO | Downloaded 34GiB to GPU in 1.91911s (17.7165GiB/s) | 20260507 04:33:36.459 | INFO | Analyzing chunk frames = [7124992,7125504) | 20260507 04:33:36.459 | INFO | MemInfo: { | 20260507 04:33:36.459 | INFO | "gpu" : " 17 GB", | 20260507 04:33:36.459 | INFO | "huge" : "", | 20260507 04:33:36.459 | INFO | "huge_shared" : " 27 GB", | 20260507 04:33:36.459 | INFO | "malloc" : " 14 GB", | 20260507 04:33:36.459 | INFO | "pinned" : " 29 GB" | 20260507 04:33:36.459 | INFO | } | 20260507 04:33:36.459 | INFO | Perf report for the last 5.15133 seconds: | 20260507 04:33:36.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:33:36.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:36.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:36.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:36.459 | INFO | ANALYSIS : 59.9%, 3.0, 93.7ms, 0 | 20260507 04:33:36.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:36.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:36.459 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:33:36.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:36.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:36.460 | INFO | Uploaded 36GiB to GPU in 2.00421s (17.9622GiB/s) | 20260507 04:33:36.460 | INFO | Downloaded 34GiB to GPU in 1.93864s (17.5381GiB/s) | 20260507 04:33:41.594 | INFO | Analyzing chunk frames = [7125504,7126016) | 20260507 04:33:41.594 | INFO | MemInfo: { | 20260507 04:33:41.594 | INFO | "gpu" : " 17 GB", | 20260507 04:33:41.594 | INFO | "huge" : "", | 20260507 04:33:41.594 | INFO | "huge_shared" : " 27 GB", | 20260507 04:33:41.594 | INFO | "malloc" : " 14 GB", | 20260507 04:33:41.594 | INFO | "pinned" : " 29 GB" | 20260507 04:33:41.594 | INFO | } | 20260507 04:33:41.594 | INFO | Perf report for the last 5.1348 seconds: | 20260507 04:33:41.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:33:41.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:41.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:41.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:41.594 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 04:33:41.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:41.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:41.594 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:33:41.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:41.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:41.594 | INFO | Uploaded 36GiB to GPU in 2.03097s (17.7255GiB/s) | 20260507 04:33:41.594 | INFO | Downloaded 34GiB to GPU in 1.91767s (17.7299GiB/s) | 20260507 04:33:46.734 | INFO | Analyzing chunk frames = [7126016,7126528) | 20260507 04:33:46.734 | INFO | MemInfo: { | 20260507 04:33:46.734 | INFO | "gpu" : " 17 GB", | 20260507 04:33:46.734 | INFO | "huge" : "", | 20260507 04:33:46.734 | INFO | "huge_shared" : " 27 GB", | 20260507 04:33:46.734 | INFO | "malloc" : " 14 GB", | 20260507 04:33:46.734 | INFO | "pinned" : " 29 GB" | 20260507 04:33:46.734 | INFO | } | 20260507 04:33:46.735 | INFO | Perf report for the last 5.14032 seconds: | 20260507 04:33:46.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:33:46.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:46.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:46.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:46.735 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:33:46.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:46.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:46.735 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 04:33:46.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:46.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:46.735 | INFO | Uploaded 36GiB to GPU in 2.00179s (17.9839GiB/s) | 20260507 04:33:46.735 | INFO | Downloaded 34GiB to GPU in 1.96442s (17.3079GiB/s) | 20260507 04:33:51.872 | INFO | Analyzing chunk frames = [7126528,7127040) | 20260507 04:33:51.872 | INFO | MemInfo: { | 20260507 04:33:51.872 | INFO | "gpu" : " 17 GB", | 20260507 04:33:51.872 | INFO | "huge" : "", | 20260507 04:33:51.872 | INFO | "huge_shared" : " 27 GB", | 20260507 04:33:51.872 | INFO | "malloc" : " 14 GB", | 20260507 04:33:51.872 | INFO | "pinned" : " 29 GB" | 20260507 04:33:51.872 | INFO | } | 20260507 04:33:51.872 | INFO | Perf report for the last 5.13731 seconds: | 20260507 04:33:51.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:33:51.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:51.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:51.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:51.872 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 04:33:51.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:51.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:51.872 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:33:51.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:51.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:51.872 | INFO | Uploaded 36GiB to GPU in 1.96436s (18.3266GiB/s) | 20260507 04:33:51.872 | INFO | Downloaded 34GiB to GPU in 1.96034s (17.344GiB/s) | 20260507 04:33:51.872 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:33:51.872 | INFO | Total mem usage: 18 GiB | 20260507 04:33:51.872 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:33:51.872 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 04:33:51.872 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:33:51.872 | INFO | High Water Breakdown: | 20260507 04:33:51.872 | INFO | 60.2% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:33:51.872 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:33:51.872 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:33:51.872 | INFO | ------------------------------------------ | 20260507 04:33:51.872 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:33:51.872 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:33:51.872 | INFO | Total mem usage: 26.5 GiB | 20260507 04:33:51.872 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:33:51.872 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:33:51.872 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 04:33:51.872 | INFO | High Water Breakdown: | 20260507 04:33:51.872 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:33:51.872 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:33:51.872 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:33:51.872 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:33:51.872 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:33:51.872 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:33:51.872 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:33:51.872 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:33:51.872 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:33:51.872 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:33:51.872 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:33:51.872 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:33:51.872 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:33:51.872 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:33:51.873 | INFO | ------------------------------------------ | 20260507 04:33:51.873 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:33:51.873 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:33:51.873 | INFO | Total mem usage: 25.6 GiB | 20260507 04:33:51.873 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:33:51.873 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:33:51.873 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:33:51.873 | INFO | High Water Breakdown: | 20260507 04:33:51.873 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:33:51.873 | INFO | ------------------------------------------ | 20260507 04:33:51.873 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:33:51.873 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:33:51.873 | INFO | Total mem usage: 15.7 GiB | 20260507 04:33:51.873 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:33:51.873 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:33:51.873 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:33:51.873 | INFO | High Water Breakdown: | 20260507 04:33:51.873 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:33:51.873 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:33:51.873 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:33:51.873 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:33:51.873 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:33:51.873 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:33:51.873 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:33:51.873 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:33:51.873 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:33:51.873 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:33:51.873 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:33:51.873 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:33:51.873 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:33:51.873 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:33:51.873 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:33:51.873 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:33:51.873 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:33:51.873 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:33:51.873 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:33:51.873 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:33:51.873 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:33:51.873 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:33:51.873 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:33:51.873 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:33:51.873 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:33:51.873 | INFO | ------------------------------------------ | 20260507 04:33:51.873 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:33:51.873 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:33:51.873 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:33:51.873 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:33:51.873 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.50e+01, 1.35e+01 / 1.69e+00 | 20260507 04:33:51.873 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.02e-01, 9.50e-03 / 6.77e-03 | 20260507 04:33:51.873 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.51e+00, 5.11e-01 / 1.40e-01 | 20260507 04:33:51.873 | INFO | PerfReport: 3.23%, Histogram -- 9.94e-01 / 1.47e+00, 1.70e-01 / 1.05e-01 | 20260507 04:33:51.873 | INFO | PerfReport: 16.22%, DME -- 3.99e+01 / 4.48e+01, 4.08e+00 / 1.88e+00 | 20260507 04:33:51.873 | INFO | PerfReport: 57.59%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.28e+00 / 1.46e+00 | 20260507 04:33:51.873 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.75e+00, 3.85e-01 / 1.88e-01 | 20260507 04:33:51.873 | INFO | PerfReport: 3.01%, Metrics -- 9.27e-01 / 1.24e+00, 2.48e-01 / 1.39e-01 | 20260507 04:33:51.873 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.12e-01 / 3.67e-01, 5.02e-02 / 2.06e-02 | 20260507 04:33:51.873 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:33:51.873 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:33:51.873 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:33:51.873 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 04:33:51.873 | INFO | PerfReport: 21.76%, Upload -- 2.09e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260507 04:33:51.873 | INFO | PerfReport: 21.36%, Download -- 2.05e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 04:33:51.873 | INFO | PerfReport: 31.97%, Compute -- 3.08e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 04:33:51.873 | INFO | PerfReport: 1.44%, LockUpload -- 1.39e+00 / 0.00e+00, 4.76e+00 / 0.00e+00 | 20260507 04:33:51.873 | INFO | PerfReport: 1.33%, LockDownload -- 1.28e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260507 04:33:51.873 | INFO | PerfReport: 22.11%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 04:33:51.873 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260507 04:33:51.969 | INFO | Starting a round of Baz IO | 20260507 04:33:55.068 | INFO | Populating RTMetrics report | 20260507 04:33:55.069 | INFO | RTMetrics report populated | 20260507 04:33:55.069 | INFO | Opening tmp RTMetrics json file | 20260507 04:33:55.087 | INFO | Serializing RTMetrics to json | 20260507 04:33:55.089 | INFO | Serialization done, swapping tmp file now... | 20260507 04:33:55.089 | INFO | RTMetrics done | 20260507 04:33:55.157 | INFO | Finished a round of Baz IO: Wrote 1.72 GiB in 3.187 seconds | 20260507 04:33:55.157 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.78% / 0.00% / 67.22% | 20260507 04:33:55.157 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.166305% / 0% / 9.13938e-05% | 20260507 04:33:55.157 | INFO | Baz IO overall: | 20260507 04:33:55.157 | INFO | BytesWritten : 3635081471702 | 20260507 04:33:55.157 | INFO | NumEvents : 1744949668160 | 20260507 04:33:57.011 | INFO | Analyzing chunk frames = [7127040,7127552) | 20260507 04:33:57.011 | INFO | MemInfo: { | 20260507 04:33:57.011 | INFO | "gpu" : " 17 GB", | 20260507 04:33:57.011 | INFO | "huge" : "", | 20260507 04:33:57.011 | INFO | "huge_shared" : " 27 GB", | 20260507 04:33:57.011 | INFO | "malloc" : " 14 GB", | 20260507 04:33:57.011 | INFO | "pinned" : " 29 GB" | 20260507 04:33:57.011 | INFO | } | 20260507 04:33:57.011 | INFO | Perf report for the last 5.13887 seconds: | 20260507 04:33:57.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:33:57.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:57.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:57.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:57.011 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:33:57.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:33:57.011 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 04:33:57.011 | INFO | PRE_HQ : 59.0%, 1.8, 55.1ms, 0 | 20260507 04:33:57.011 | INFO | BAZWRITER : 61.6%, 3.2, 102.3ms, 0 | 20260507 04:33:57.011 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:33:57.011 | INFO | Uploaded 36GiB to GPU in 2.02816s (17.7501GiB/s) | 20260507 04:33:57.011 | INFO | Downloaded 38GiB to GPU in 2.17902s (17.439GiB/s) | 20260507 04:34:02.148 | INFO | Analyzing chunk frames = [7127552,7128064) | 20260507 04:34:02.148 | INFO | MemInfo: { | 20260507 04:34:02.148 | INFO | "gpu" : " 17 GB", | 20260507 04:34:02.148 | INFO | "huge" : "", | 20260507 04:34:02.149 | INFO | "huge_shared" : " 27 GB", | 20260507 04:34:02.149 | INFO | "malloc" : " 14 GB", | 20260507 04:34:02.149 | INFO | "pinned" : " 29 GB" | 20260507 04:34:02.149 | INFO | } | 20260507 04:34:02.149 | INFO | Perf report for the last 5.13786 seconds: | 20260507 04:34:02.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:34:02.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:02.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:02.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:02.149 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 04:34:02.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:02.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:02.149 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:34:02.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:02.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:02.149 | INFO | Uploaded 36GiB to GPU in 2.02914s (17.7415GiB/s) | 20260507 04:34:02.149 | INFO | Downloaded 34GiB to GPU in 1.94021s (17.5238GiB/s) | 20260507 04:34:07.289 | INFO | Analyzing chunk frames = [7128064,7128576) | 20260507 04:34:07.289 | INFO | MemInfo: { | 20260507 04:34:07.289 | INFO | "gpu" : " 17 GB", | 20260507 04:34:07.289 | INFO | "huge" : "", | 20260507 04:34:07.289 | INFO | "huge_shared" : " 27 GB", | 20260507 04:34:07.289 | INFO | "malloc" : " 14 GB", | 20260507 04:34:07.289 | INFO | "pinned" : " 29 GB" | 20260507 04:34:07.289 | INFO | } | 20260507 04:34:07.289 | INFO | Perf report for the last 5.1402 seconds: | 20260507 04:34:07.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:34:07.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:07.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:07.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:07.289 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 04:34:07.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:07.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:07.289 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:34:07.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:07.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:07.289 | INFO | Uploaded 36GiB to GPU in 1.98597s (18.1271GiB/s) | 20260507 04:34:07.289 | INFO | Downloaded 34GiB to GPU in 1.94254s (17.5029GiB/s) | 20260507 04:34:12.432 | INFO | Analyzing chunk frames = [7128576,7129088) | 20260507 04:34:12.432 | INFO | MemInfo: { | 20260507 04:34:12.432 | INFO | "gpu" : " 17 GB", | 20260507 04:34:12.432 | INFO | "huge" : "", | 20260507 04:34:12.432 | INFO | "huge_shared" : " 27 GB", | 20260507 04:34:12.432 | INFO | "malloc" : " 14 GB", | 20260507 04:34:12.432 | INFO | "pinned" : " 29 GB" | 20260507 04:34:12.432 | INFO | } | 20260507 04:34:12.432 | INFO | Perf report for the last 5.14291 seconds: | 20260507 04:34:12.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:34:12.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:12.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:12.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:12.432 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 04:34:12.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:12.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:12.432 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:34:12.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:12.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:12.432 | INFO | Uploaded 36GiB to GPU in 1.97579s (18.2206GiB/s) | 20260507 04:34:12.432 | INFO | Downloaded 34GiB to GPU in 1.92178s (17.692GiB/s) | 20260507 04:34:17.565 | INFO | Analyzing chunk frames = [7129088,7129600) | 20260507 04:34:17.565 | INFO | MemInfo: { | 20260507 04:34:17.565 | INFO | "gpu" : " 17 GB", | 20260507 04:34:17.565 | INFO | "huge" : "", | 20260507 04:34:17.565 | INFO | "huge_shared" : " 27 GB", | 20260507 04:34:17.565 | INFO | "malloc" : " 14 GB", | 20260507 04:34:17.565 | INFO | "pinned" : " 29 GB" | 20260507 04:34:17.565 | INFO | } | 20260507 04:34:17.566 | INFO | Perf report for the last 5.13379 seconds: | 20260507 04:34:17.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:34:17.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:17.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:17.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:17.566 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 04:34:17.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:17.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:17.566 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:34:17.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:17.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:17.566 | INFO | Uploaded 36GiB to GPU in 2.04892s (17.5702GiB/s) | 20260507 04:34:17.566 | INFO | Downloaded 34GiB to GPU in 1.9512s (17.4252GiB/s) | 20260507 04:34:22.711 | INFO | Analyzing chunk frames = [7129600,7130112) | 20260507 04:34:22.711 | INFO | MemInfo: { | 20260507 04:34:22.711 | INFO | "gpu" : " 17 GB", | 20260507 04:34:22.711 | INFO | "huge" : "", | 20260507 04:34:22.711 | INFO | "huge_shared" : " 27 GB", | 20260507 04:34:22.711 | INFO | "malloc" : " 14 GB", | 20260507 04:34:22.711 | INFO | "pinned" : " 29 GB" | 20260507 04:34:22.711 | INFO | } | 20260507 04:34:22.711 | INFO | Perf report for the last 5.14523 seconds: | 20260507 04:34:22.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:34:22.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:22.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:22.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:22.711 | INFO | ANALYSIS : 60.4%, 3.0, 94.4ms, 0 | 20260507 04:34:22.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:22.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:22.711 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 04:34:22.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:22.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:22.711 | INFO | Uploaded 36GiB to GPU in 2.02339s (17.7919GiB/s) | 20260507 04:34:22.711 | INFO | Downloaded 34GiB to GPU in 1.93964s (17.529GiB/s) | 20260507 04:34:27.849 | INFO | Analyzing chunk frames = [7130112,7130624) | 20260507 04:34:27.849 | INFO | MemInfo: { | 20260507 04:34:27.849 | INFO | "gpu" : " 17 GB", | 20260507 04:34:27.849 | INFO | "huge" : "", | 20260507 04:34:27.849 | INFO | "huge_shared" : " 27 GB", | 20260507 04:34:27.850 | INFO | "malloc" : " 14 GB", | 20260507 04:34:27.850 | INFO | "pinned" : " 29 GB" | 20260507 04:34:27.850 | INFO | } | 20260507 04:34:27.850 | INFO | Perf report for the last 5.1388 seconds: | 20260507 04:34:27.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:34:27.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:27.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:27.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:27.850 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:34:27.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:27.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:27.850 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 04:34:27.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:27.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:27.850 | INFO | Uploaded 36GiB to GPU in 1.98213s (18.1623GiB/s) | 20260507 04:34:27.850 | INFO | Downloaded 34GiB to GPU in 1.98178s (17.1563GiB/s) | 20260507 04:34:32.988 | INFO | Analyzing chunk frames = [7130624,7131136) | 20260507 04:34:32.988 | INFO | MemInfo: { | 20260507 04:34:32.988 | INFO | "gpu" : " 17 GB", | 20260507 04:34:32.988 | INFO | "huge" : "", | 20260507 04:34:32.988 | INFO | "huge_shared" : " 27 GB", | 20260507 04:34:32.988 | INFO | "malloc" : " 14 GB", | 20260507 04:34:32.988 | INFO | "pinned" : " 29 GB" | 20260507 04:34:32.988 | INFO | } | 20260507 04:34:32.988 | INFO | Perf report for the last 5.13846 seconds: | 20260507 04:34:32.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:34:32.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:32.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:32.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:32.988 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:34:32.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:32.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:32.988 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 04:34:32.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:32.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:32.988 | INFO | Uploaded 36GiB to GPU in 1.99378s (18.0562GiB/s) | 20260507 04:34:32.988 | INFO | Downloaded 34GiB to GPU in 1.96696s (17.2855GiB/s) | 20260507 04:34:33.085 | INFO | Starting a round of Baz IO | 20260507 04:34:36.186 | INFO | Populating RTMetrics report | 20260507 04:34:36.186 | INFO | RTMetrics report populated | 20260507 04:34:36.186 | INFO | Opening tmp RTMetrics json file | 20260507 04:34:36.191 | INFO | Serializing RTMetrics to json | 20260507 04:34:36.205 | INFO | Serialization done, swapping tmp file now... | 20260507 04:34:36.206 | INFO | RTMetrics done | 20260507 04:34:36.277 | INFO | Finished a round of Baz IO: Wrote 1.722 GiB in 3.192 seconds | 20260507 04:34:36.277 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.78% / 0.00% / 67.22% | 20260507 04:34:36.277 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.166027% / 1.19209e-05% / 0.000123183% | 20260507 04:34:36.277 | INFO | Baz IO overall: | 20260507 04:34:36.277 | INFO | BytesWritten : 3636930493810 | 20260507 04:34:36.277 | INFO | NumEvents : 1745820085507 | 20260507 04:34:38.125 | INFO | Analyzing chunk frames = [7131136,7131648) | 20260507 04:34:38.125 | INFO | MemInfo: { | 20260507 04:34:38.125 | INFO | "gpu" : " 17 GB", | 20260507 04:34:38.125 | INFO | "huge" : "", | 20260507 04:34:38.125 | INFO | "huge_shared" : " 27 GB", | 20260507 04:34:38.125 | INFO | "malloc" : " 14 GB", | 20260507 04:34:38.125 | INFO | "pinned" : " 29 GB" | 20260507 04:34:38.125 | INFO | } | 20260507 04:34:38.125 | INFO | Perf report for the last 5.13741 seconds: | 20260507 04:34:38.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:34:38.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:38.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:38.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:38.126 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:34:38.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:38.126 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:34:38.126 | INFO | PRE_HQ : 59.3%, 1.8, 55.3ms, 0 | 20260507 04:34:38.126 | INFO | BAZWRITER : 61.7%, 3.2, 102.0ms, 0 | 20260507 04:34:38.126 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:34:38.126 | INFO | Uploaded 36GiB to GPU in 2.05515s (17.517GiB/s) | 20260507 04:34:38.126 | INFO | Downloaded 38GiB to GPU in 2.17701s (17.4552GiB/s) | 20260507 04:34:43.260 | INFO | Analyzing chunk frames = [7131648,7132160) | 20260507 04:34:43.260 | INFO | MemInfo: { | 20260507 04:34:43.260 | INFO | "gpu" : " 17 GB", | 20260507 04:34:43.260 | INFO | "huge" : "", | 20260507 04:34:43.260 | INFO | "huge_shared" : " 27 GB", | 20260507 04:34:43.260 | INFO | "malloc" : " 14 GB", | 20260507 04:34:43.260 | INFO | "pinned" : " 29 GB" | 20260507 04:34:43.260 | INFO | } | 20260507 04:34:43.260 | INFO | Perf report for the last 5.1344 seconds: | 20260507 04:34:43.260 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:34:43.260 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:43.260 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:43.260 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:43.260 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 04:34:43.260 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:43.260 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:43.260 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:34:43.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:43.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:43.260 | INFO | Uploaded 36GiB to GPU in 2.04433s (17.6097GiB/s) | 20260507 04:34:43.260 | INFO | Downloaded 34GiB to GPU in 1.93658s (17.5567GiB/s) | 20260507 04:34:48.408 | INFO | Analyzing chunk frames = [7132160,7132672) | 20260507 04:34:48.408 | INFO | MemInfo: { | 20260507 04:34:48.408 | INFO | "gpu" : " 17 GB", | 20260507 04:34:48.408 | INFO | "huge" : "", | 20260507 04:34:48.408 | INFO | "huge_shared" : " 27 GB", | 20260507 04:34:48.408 | INFO | "malloc" : " 14 GB", | 20260507 04:34:48.408 | INFO | "pinned" : " 29 GB" | 20260507 04:34:48.408 | INFO | } | 20260507 04:34:48.408 | INFO | Perf report for the last 5.14803 seconds: | 20260507 04:34:48.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:34:48.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:48.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:48.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:48.408 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 04:34:48.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:48.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:48.408 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:34:48.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:48.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:48.408 | INFO | Uploaded 36GiB to GPU in 1.99631s (18.0333GiB/s) | 20260507 04:34:48.408 | INFO | Downloaded 34GiB to GPU in 1.93588s (17.5631GiB/s) | 20260507 04:34:53.550 | INFO | Analyzing chunk frames = [7132672,7133184) | 20260507 04:34:53.550 | INFO | MemInfo: { | 20260507 04:34:53.550 | INFO | "gpu" : " 17 GB", | 20260507 04:34:53.550 | INFO | "huge" : "", | 20260507 04:34:53.550 | INFO | "huge_shared" : " 27 GB", | 20260507 04:34:53.550 | INFO | "malloc" : " 14 GB", | 20260507 04:34:53.550 | INFO | "pinned" : " 29 GB" | 20260507 04:34:53.550 | INFO | } | 20260507 04:34:53.551 | INFO | Perf report for the last 5.14257 seconds: | 20260507 04:34:53.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:34:53.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:53.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:53.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:53.551 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 04:34:53.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:53.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:53.551 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 04:34:53.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:53.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:53.551 | INFO | Uploaded 36GiB to GPU in 1.99972s (18.0025GiB/s) | 20260507 04:34:53.551 | INFO | Downloaded 34GiB to GPU in 1.93427s (17.5776GiB/s) | 20260507 04:34:58.684 | INFO | Analyzing chunk frames = [7133184,7133696) | 20260507 04:34:58.684 | INFO | MemInfo: { | 20260507 04:34:58.684 | INFO | "gpu" : " 17 GB", | 20260507 04:34:58.684 | INFO | "huge" : "", | 20260507 04:34:58.684 | INFO | "huge_shared" : " 27 GB", | 20260507 04:34:58.684 | INFO | "malloc" : " 14 GB", | 20260507 04:34:58.684 | INFO | "pinned" : " 29 GB" | 20260507 04:34:58.684 | INFO | } | 20260507 04:34:58.684 | INFO | Perf report for the last 5.13362 seconds: | 20260507 04:34:58.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:34:58.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:58.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:58.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:58.684 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 04:34:58.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:58.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:58.684 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 04:34:58.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:58.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:34:58.684 | INFO | Uploaded 36GiB to GPU in 2.03622s (17.6799GiB/s) | 20260507 04:34:58.684 | INFO | Downloaded 34GiB to GPU in 1.91581s (17.747GiB/s) | 20260507 04:35:03.825 | INFO | Analyzing chunk frames = [7133696,7134208) | 20260507 04:35:03.825 | INFO | MemInfo: { | 20260507 04:35:03.825 | INFO | "gpu" : " 17 GB", | 20260507 04:35:03.825 | INFO | "huge" : "", | 20260507 04:35:03.825 | INFO | "huge_shared" : " 27 GB", | 20260507 04:35:03.825 | INFO | "malloc" : " 14 GB", | 20260507 04:35:03.825 | INFO | "pinned" : " 29 GB" | 20260507 04:35:03.825 | INFO | } | 20260507 04:35:03.825 | INFO | Perf report for the last 5.14067 seconds: | 20260507 04:35:03.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:35:03.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:03.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:03.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:03.825 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 04:35:03.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:03.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:03.825 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260507 04:35:03.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:03.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:03.825 | INFO | Uploaded 36GiB to GPU in 2.06512s (17.4324GiB/s) | 20260507 04:35:03.825 | INFO | Downloaded 34GiB to GPU in 1.92237s (17.6865GiB/s) | 20260507 04:35:08.962 | INFO | Analyzing chunk frames = [7134208,7134720) | 20260507 04:35:08.962 | INFO | MemInfo: { | 20260507 04:35:08.962 | INFO | "gpu" : " 17 GB", | 20260507 04:35:08.962 | INFO | "huge" : "", | 20260507 04:35:08.962 | INFO | "huge_shared" : " 27 GB", | 20260507 04:35:08.962 | INFO | "malloc" : " 14 GB", | 20260507 04:35:08.962 | INFO | "pinned" : " 29 GB" | 20260507 04:35:08.962 | INFO | } | 20260507 04:35:08.962 | INFO | Perf report for the last 5.13747 seconds: | 20260507 04:35:08.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:35:08.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:08.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:08.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:08.962 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:35:08.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:08.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:08.962 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 04:35:08.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:08.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:08.962 | INFO | Uploaded 36GiB to GPU in 1.98031s (18.1789GiB/s) | 20260507 04:35:08.963 | INFO | Downloaded 34GiB to GPU in 1.98646s (17.1159GiB/s) | 20260507 04:35:14.102 | INFO | Analyzing chunk frames = [7134720,7135232) | 20260507 04:35:14.102 | INFO | MemInfo: { | 20260507 04:35:14.102 | INFO | "gpu" : " 17 GB", | 20260507 04:35:14.102 | INFO | "huge" : "", | 20260507 04:35:14.102 | INFO | "huge_shared" : " 27 GB", | 20260507 04:35:14.102 | INFO | "malloc" : " 14 GB", | 20260507 04:35:14.102 | INFO | "pinned" : " 29 GB" | 20260507 04:35:14.102 | INFO | } | 20260507 04:35:14.102 | INFO | Perf report for the last 5.14002 seconds: | 20260507 04:35:14.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:35:14.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:14.102 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:14.102 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:14.102 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:35:14.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:14.102 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:14.102 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 04:35:14.102 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:14.102 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:14.103 | INFO | Uploaded 36GiB to GPU in 1.96121s (18.356GiB/s) | 20260507 04:35:14.103 | INFO | Downloaded 34GiB to GPU in 1.97449s (17.2197GiB/s) | 20260507 04:35:14.103 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:35:14.103 | INFO | Total mem usage: 18 GiB | 20260507 04:35:14.103 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:35:14.103 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 04:35:14.103 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:35:14.103 | INFO | High Water Breakdown: | 20260507 04:35:14.103 | INFO | 60.2% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:35:14.103 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:35:14.103 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:35:14.103 | INFO | ------------------------------------------ | 20260507 04:35:14.103 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:35:14.103 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:35:14.103 | INFO | Total mem usage: 26.5 GiB | 20260507 04:35:14.103 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:35:14.103 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:35:14.103 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:35:14.103 | INFO | High Water Breakdown: | 20260507 04:35:14.103 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:35:14.103 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:35:14.103 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:35:14.103 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:35:14.103 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:35:14.103 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:35:14.103 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:35:14.103 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:35:14.103 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:35:14.103 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:35:14.103 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:35:14.103 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:35:14.103 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:35:14.103 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:35:14.103 | INFO | ------------------------------------------ | 20260507 04:35:14.103 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:35:14.103 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:35:14.103 | INFO | Total mem usage: 25.6 GiB | 20260507 04:35:14.103 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:35:14.103 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:35:14.103 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:35:14.103 | INFO | High Water Breakdown: | 20260507 04:35:14.103 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:35:14.103 | INFO | ------------------------------------------ | 20260507 04:35:14.103 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:35:14.103 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:35:14.103 | INFO | Total mem usage: 15.7 GiB | 20260507 04:35:14.103 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:35:14.103 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:35:14.103 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:35:14.103 | INFO | High Water Breakdown: | 20260507 04:35:14.103 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:35:14.103 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:35:14.103 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:35:14.103 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:35:14.103 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:35:14.103 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:35:14.103 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:35:14.103 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:35:14.103 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:35:14.103 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:35:14.103 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:35:14.103 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:35:14.103 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:35:14.103 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:35:14.103 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:35:14.103 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:35:14.103 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:35:14.103 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:35:14.103 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:35:14.103 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:35:14.103 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:35:14.103 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:35:14.103 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:35:14.103 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:35:14.103 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:35:14.103 | INFO | ------------------------------------------ | 20260507 04:35:14.104 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:35:14.104 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:35:14.104 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:35:14.104 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:35:14.104 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.43e+01, 1.32e+01 / 1.21e+00 | 20260507 04:35:14.104 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 9.15e-03 / 5.94e-03 | 20260507 04:35:14.104 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.48e+00, 5.05e-01 / 2.27e-01 | 20260507 04:35:14.104 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.46e+00, 1.65e-01 / 1.09e-01 | 20260507 04:35:14.104 | INFO | PerfReport: 16.07%, DME -- 3.95e+01 / 4.36e+01, 3.55e+00 / 1.57e+00 | 20260507 04:35:14.104 | INFO | PerfReport: 57.73%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.29e+00 / 1.64e+00 | 20260507 04:35:14.104 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.14e+00 / 2.70e+00, 3.88e-01 / 2.36e-01 | 20260507 04:35:14.104 | INFO | PerfReport: 3.03%, Metrics -- 9.31e-01 / 1.22e+00, 2.51e-01 / 1.26e-01 | 20260507 04:35:14.104 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.13e-01 / 3.69e-01, 5.02e-02 / 2.66e-02 | 20260507 04:35:14.104 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:35:14.104 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:35:14.104 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:35:14.104 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 04:35:14.104 | INFO | PerfReport: 21.80%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260507 04:35:14.104 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.43e+00 / 0.00e+00 | 20260507 04:35:14.104 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 04:35:14.104 | INFO | PerfReport: 1.42%, LockUpload -- 1.37e+00 / 0.00e+00, 4.74e+00 / 0.00e+00 | 20260507 04:35:14.104 | INFO | PerfReport: 1.34%, LockDownload -- 1.29e+00 / 0.00e+00, 3.12e+00 / 0.00e+00 | 20260507 04:35:14.104 | INFO | PerfReport: 22.10%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 04:35:14.104 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260507 04:35:14.199 | INFO | Starting a round of Baz IO | 20260507 04:35:17.300 | INFO | Populating RTMetrics report | 20260507 04:35:17.301 | INFO | RTMetrics report populated | 20260507 04:35:17.301 | INFO | Opening tmp RTMetrics json file | 20260507 04:35:17.319 | INFO | Serializing RTMetrics to json | 20260507 04:35:17.320 | INFO | Serialization done, swapping tmp file now... | 20260507 04:35:17.320 | INFO | RTMetrics done | 20260507 04:35:17.389 | INFO | Finished a round of Baz IO: Wrote 1.721 GiB in 3.189 seconds | 20260507 04:35:17.389 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.78% / 0.00% / 67.22% | 20260507 04:35:17.389 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.166659% / 0% / 7.94729e-05% | 20260507 04:35:17.389 | INFO | Baz IO overall: | 20260507 04:35:17.389 | INFO | BytesWritten : 3638778564056 | 20260507 04:35:17.389 | INFO | NumEvents : 1746689998542 | 20260507 04:35:19.239 | INFO | Analyzing chunk frames = [7135232,7135744) | 20260507 04:35:19.239 | INFO | MemInfo: { | 20260507 04:35:19.239 | INFO | "gpu" : " 17 GB", | 20260507 04:35:19.239 | INFO | "huge" : "", | 20260507 04:35:19.239 | INFO | "huge_shared" : " 27 GB", | 20260507 04:35:19.239 | INFO | "malloc" : " 14 GB", | 20260507 04:35:19.239 | INFO | "pinned" : " 29 GB" | 20260507 04:35:19.239 | INFO | } | 20260507 04:35:19.239 | INFO | Perf report for the last 5.13712 seconds: | 20260507 04:35:19.239 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:35:19.239 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:19.239 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:19.239 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:19.239 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:35:19.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:19.240 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:35:19.240 | INFO | PRE_HQ : 59.2%, 1.8, 55.3ms, 0 | 20260507 04:35:19.240 | INFO | BAZWRITER : 61.6%, 3.2, 102.0ms, 0 | 20260507 04:35:19.240 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 04:35:19.240 | INFO | Uploaded 36GiB to GPU in 2.03325s (17.7057GiB/s) | 20260507 04:35:19.240 | INFO | Downloaded 38GiB to GPU in 2.18745s (17.3718GiB/s) | 20260507 04:35:24.382 | INFO | Analyzing chunk frames = [7135744,7136256) | 20260507 04:35:24.382 | INFO | MemInfo: { | 20260507 04:35:24.382 | INFO | "gpu" : " 17 GB", | 20260507 04:35:24.382 | INFO | "huge" : "", | 20260507 04:35:24.382 | INFO | "huge_shared" : " 27 GB", | 20260507 04:35:24.382 | INFO | "malloc" : " 14 GB", | 20260507 04:35:24.382 | INFO | "pinned" : " 29 GB" | 20260507 04:35:24.382 | INFO | } | 20260507 04:35:24.382 | INFO | Perf report for the last 5.14276 seconds: | 20260507 04:35:24.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:35:24.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:24.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:24.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:24.382 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:35:24.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:24.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:24.382 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:35:24.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:24.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:24.383 | INFO | Uploaded 36GiB to GPU in 1.99953s (18.0043GiB/s) | 20260507 04:35:24.383 | INFO | Downloaded 34GiB to GPU in 1.9364s (17.5583GiB/s) | 20260507 04:35:29.519 | INFO | Analyzing chunk frames = [7136256,7136768) | 20260507 04:35:29.519 | INFO | MemInfo: { | 20260507 04:35:29.519 | INFO | "gpu" : " 17 GB", | 20260507 04:35:29.519 | INFO | "huge" : "", | 20260507 04:35:29.519 | INFO | "huge_shared" : " 27 GB", | 20260507 04:35:29.519 | INFO | "malloc" : " 14 GB", | 20260507 04:35:29.519 | INFO | "pinned" : " 29 GB" | 20260507 04:35:29.519 | INFO | } | 20260507 04:35:29.519 | INFO | Perf report for the last 5.13644 seconds: | 20260507 04:35:29.519 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:35:29.519 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:29.519 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:29.519 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:29.519 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 04:35:29.519 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:29.519 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:29.519 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:35:29.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:29.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:29.519 | INFO | Uploaded 36GiB to GPU in 2.02528s (17.7753GiB/s) | 20260507 04:35:29.520 | INFO | Downloaded 34GiB to GPU in 1.95846s (17.3606GiB/s) | 20260507 04:35:34.665 | INFO | Analyzing chunk frames = [7136768,7137280) | 20260507 04:35:34.665 | INFO | MemInfo: { | 20260507 04:35:34.665 | INFO | "gpu" : " 17 GB", | 20260507 04:35:34.665 | INFO | "huge" : "", | 20260507 04:35:34.665 | INFO | "huge_shared" : " 27 GB", | 20260507 04:35:34.665 | INFO | "malloc" : " 14 GB", | 20260507 04:35:34.665 | INFO | "pinned" : " 29 GB" | 20260507 04:35:34.665 | INFO | } | 20260507 04:35:34.665 | INFO | Perf report for the last 5.146 seconds: | 20260507 04:35:34.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:35:34.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:34.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:34.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:34.665 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:35:34.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:34.665 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:34.665 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 04:35:34.665 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:34.665 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:34.665 | INFO | Uploaded 36GiB to GPU in 1.97894s (18.1916GiB/s) | 20260507 04:35:34.665 | INFO | Downloaded 34GiB to GPU in 1.92989s (17.6176GiB/s) | 20260507 04:35:39.796 | INFO | Analyzing chunk frames = [7137280,7137792) | 20260507 04:35:39.796 | INFO | MemInfo: { | 20260507 04:35:39.796 | INFO | "gpu" : " 17 GB", | 20260507 04:35:39.796 | INFO | "huge" : "", | 20260507 04:35:39.796 | INFO | "huge_shared" : " 27 GB", | 20260507 04:35:39.796 | INFO | "malloc" : " 14 GB", | 20260507 04:35:39.796 | INFO | "pinned" : " 29 GB" | 20260507 04:35:39.796 | INFO | } | 20260507 04:35:39.796 | INFO | Perf report for the last 5.1313 seconds: | 20260507 04:35:39.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:35:39.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:39.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:39.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:39.796 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 04:35:39.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:39.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:39.796 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 04:35:39.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:39.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:39.796 | INFO | Uploaded 36GiB to GPU in 2.04462s (17.6072GiB/s) | 20260507 04:35:39.796 | INFO | Downloaded 34GiB to GPU in 1.94979s (17.4378GiB/s) | 20260507 04:35:44.936 | INFO | Analyzing chunk frames = [7137792,7138304) | 20260507 04:35:44.936 | INFO | MemInfo: { | 20260507 04:35:44.936 | INFO | "gpu" : " 17 GB", | 20260507 04:35:44.937 | INFO | "huge" : "", | 20260507 04:35:44.937 | INFO | "huge_shared" : " 27 GB", | 20260507 04:35:44.937 | INFO | "malloc" : " 14 GB", | 20260507 04:35:44.937 | INFO | "pinned" : " 29 GB" | 20260507 04:35:44.937 | INFO | } | 20260507 04:35:44.937 | INFO | Perf report for the last 5.14055 seconds: | 20260507 04:35:44.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:35:44.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:44.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:44.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:44.937 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 04:35:44.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:44.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:44.937 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 04:35:44.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:44.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:44.937 | INFO | Uploaded 36GiB to GPU in 2.03117s (17.7237GiB/s) | 20260507 04:35:44.937 | INFO | Downloaded 34GiB to GPU in 1.9136s (17.7675GiB/s) | 20260507 04:35:50.077 | INFO | Analyzing chunk frames = [7138304,7138816) | 20260507 04:35:50.077 | INFO | MemInfo: { | 20260507 04:35:50.077 | INFO | "gpu" : " 17 GB", | 20260507 04:35:50.077 | INFO | "huge" : "", | 20260507 04:35:50.077 | INFO | "huge_shared" : " 27 GB", | 20260507 04:35:50.077 | INFO | "malloc" : " 14 GB", | 20260507 04:35:50.077 | INFO | "pinned" : " 29 GB" | 20260507 04:35:50.077 | INFO | } | 20260507 04:35:50.078 | INFO | Perf report for the last 5.14094 seconds: | 20260507 04:35:50.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:35:50.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:50.078 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:50.078 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:50.078 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:35:50.078 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:50.078 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:50.078 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:35:50.078 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:50.078 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:50.078 | INFO | Uploaded 36GiB to GPU in 1.96084s (18.3594GiB/s) | 20260507 04:35:50.078 | INFO | Downloaded 34GiB to GPU in 1.95739s (17.3701GiB/s) | 20260507 04:35:55.216 | INFO | Analyzing chunk frames = [7138816,7139328) | 20260507 04:35:55.216 | INFO | MemInfo: { | 20260507 04:35:55.216 | INFO | "gpu" : " 17 GB", | 20260507 04:35:55.216 | INFO | "huge" : "", | 20260507 04:35:55.216 | INFO | "huge_shared" : " 27 GB", | 20260507 04:35:55.216 | INFO | "malloc" : " 14 GB", | 20260507 04:35:55.216 | INFO | "pinned" : " 29 GB" | 20260507 04:35:55.216 | INFO | } | 20260507 04:35:55.216 | INFO | Perf report for the last 5.13858 seconds: | 20260507 04:35:55.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:35:55.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:55.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:55.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:55.216 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:35:55.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:55.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:55.216 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:35:55.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:55.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:35:55.216 | INFO | Uploaded 36GiB to GPU in 1.97761s (18.2038GiB/s) | 20260507 04:35:55.216 | INFO | Downloaded 34GiB to GPU in 1.95422s (17.3982GiB/s) | 20260507 04:35:55.312 | INFO | Starting a round of Baz IO | 20260507 04:35:58.415 | INFO | Populating RTMetrics report | 20260507 04:35:58.415 | INFO | RTMetrics report populated | 20260507 04:35:58.415 | INFO | Opening tmp RTMetrics json file | 20260507 04:35:58.418 | INFO | Serializing RTMetrics to json | 20260507 04:35:58.434 | INFO | Serialization done, swapping tmp file now... | 20260507 04:35:58.434 | INFO | RTMetrics done | 20260507 04:35:58.505 | INFO | Finished a round of Baz IO: Wrote 1.723 GiB in 3.192 seconds | 20260507 04:35:58.505 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.78% / 0.00% / 67.22% | 20260507 04:35:58.506 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.166821% / 0% / 9.13938e-05% | 20260507 04:35:58.506 | INFO | Baz IO overall: | 20260507 04:35:58.506 | INFO | BytesWritten : 3640628163776 | 20260507 04:35:58.506 | INFO | NumEvents : 1747560689927 | 20260507 04:36:00.362 | INFO | Analyzing chunk frames = [7139328,7139840) | 20260507 04:36:00.362 | INFO | MemInfo: { | 20260507 04:36:00.362 | INFO | "gpu" : " 17 GB", | 20260507 04:36:00.362 | INFO | "huge" : "", | 20260507 04:36:00.362 | INFO | "huge_shared" : " 27 GB", | 20260507 04:36:00.363 | INFO | "malloc" : " 14 GB", | 20260507 04:36:00.363 | INFO | "pinned" : " 29 GB" | 20260507 04:36:00.363 | INFO | } | 20260507 04:36:00.363 | INFO | Perf report for the last 5.14651 seconds: | 20260507 04:36:00.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:36:00.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:00.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:00.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:00.363 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 04:36:00.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:00.363 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 04:36:00.363 | INFO | PRE_HQ : 58.9%, 1.8, 55.0ms, 0 | 20260507 04:36:00.363 | INFO | BAZWRITER : 61.6%, 3.2, 102.4ms, 0 | 20260507 04:36:00.363 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:36:00.363 | INFO | Uploaded 36GiB to GPU in 2.02963s (17.7372GiB/s) | 20260507 04:36:00.363 | INFO | Downloaded 38GiB to GPU in 2.17493s (17.4718GiB/s) | 20260507 04:36:05.496 | INFO | Analyzing chunk frames = [7139840,7140352) | 20260507 04:36:05.496 | INFO | MemInfo: { | 20260507 04:36:05.496 | INFO | "gpu" : " 17 GB", | 20260507 04:36:05.496 | INFO | "huge" : "", | 20260507 04:36:05.496 | INFO | "huge_shared" : " 27 GB", | 20260507 04:36:05.496 | INFO | "malloc" : " 14 GB", | 20260507 04:36:05.496 | INFO | "pinned" : " 29 GB" | 20260507 04:36:05.496 | INFO | } | 20260507 04:36:05.496 | INFO | Perf report for the last 5.13339 seconds: | 20260507 04:36:05.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:36:05.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:05.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:05.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:05.496 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 04:36:05.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:05.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:05.496 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 04:36:05.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:05.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:05.496 | INFO | Uploaded 36GiB to GPU in 1.99223s (18.0702GiB/s) | 20260507 04:36:05.496 | INFO | Downloaded 34GiB to GPU in 1.93047s (17.6123GiB/s) | 20260507 04:36:10.632 | INFO | Analyzing chunk frames = [7140352,7140864) | 20260507 04:36:10.632 | INFO | MemInfo: { | 20260507 04:36:10.632 | INFO | "gpu" : " 17 GB", | 20260507 04:36:10.632 | INFO | "huge" : "", | 20260507 04:36:10.632 | INFO | "huge_shared" : " 27 GB", | 20260507 04:36:10.632 | INFO | "malloc" : " 14 GB", | 20260507 04:36:10.632 | INFO | "pinned" : " 29 GB" | 20260507 04:36:10.632 | INFO | } | 20260507 04:36:10.632 | INFO | Perf report for the last 5.13595 seconds: | 20260507 04:36:10.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:36:10.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:10.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:10.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:10.632 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 04:36:10.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:10.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:10.632 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:36:10.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:10.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:10.632 | INFO | Uploaded 36GiB to GPU in 2.04136s (17.6353GiB/s) | 20260507 04:36:10.632 | INFO | Downloaded 34GiB to GPU in 1.94867s (17.4478GiB/s) | 20260507 04:36:15.771 | INFO | Analyzing chunk frames = [7140864,7141376) | 20260507 04:36:15.771 | INFO | MemInfo: { | 20260507 04:36:15.771 | INFO | "gpu" : " 17 GB", | 20260507 04:36:15.771 | INFO | "huge" : "", | 20260507 04:36:15.771 | INFO | "huge_shared" : " 27 GB", | 20260507 04:36:15.771 | INFO | "malloc" : " 14 GB", | 20260507 04:36:15.771 | INFO | "pinned" : " 29 GB" | 20260507 04:36:15.771 | INFO | } | 20260507 04:36:15.771 | INFO | Perf report for the last 5.1388 seconds: | 20260507 04:36:15.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:36:15.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:15.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:15.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:15.771 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:36:15.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:15.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:15.771 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 04:36:15.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:15.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:15.771 | INFO | Uploaded 36GiB to GPU in 1.98071s (18.1753GiB/s) | 20260507 04:36:15.771 | INFO | Downloaded 34GiB to GPU in 1.91192s (17.7832GiB/s) | 20260507 04:36:20.911 | INFO | Analyzing chunk frames = [7141376,7141888) | 20260507 04:36:20.911 | INFO | MemInfo: { | 20260507 04:36:20.911 | INFO | "gpu" : " 17 GB", | 20260507 04:36:20.911 | INFO | "huge" : "", | 20260507 04:36:20.911 | INFO | "huge_shared" : " 27 GB", | 20260507 04:36:20.911 | INFO | "malloc" : " 14 GB", | 20260507 04:36:20.911 | INFO | "pinned" : " 29 GB" | 20260507 04:36:20.911 | INFO | } | 20260507 04:36:20.911 | INFO | Perf report for the last 5.13991 seconds: | 20260507 04:36:20.911 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:36:20.911 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:20.911 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:20.911 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:20.911 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 04:36:20.911 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:20.911 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:20.911 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:36:20.911 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:20.911 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:20.911 | INFO | Uploaded 36GiB to GPU in 2.02031s (17.819GiB/s) | 20260507 04:36:20.911 | INFO | Downloaded 34GiB to GPU in 1.93987s (17.527GiB/s) | 20260507 04:36:26.050 | INFO | Analyzing chunk frames = [7141888,7142400) | 20260507 04:36:26.050 | INFO | MemInfo: { | 20260507 04:36:26.050 | INFO | "gpu" : " 17 GB", | 20260507 04:36:26.050 | INFO | "huge" : "", | 20260507 04:36:26.050 | INFO | "huge_shared" : " 27 GB", | 20260507 04:36:26.050 | INFO | "malloc" : " 14 GB", | 20260507 04:36:26.050 | INFO | "pinned" : " 29 GB" | 20260507 04:36:26.050 | INFO | } | 20260507 04:36:26.050 | INFO | Perf report for the last 5.13938 seconds: | 20260507 04:36:26.050 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:36:26.050 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:26.050 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:26.050 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:26.050 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:36:26.050 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:26.050 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:26.050 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:36:26.050 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:26.050 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:26.050 | INFO | Uploaded 36GiB to GPU in 2.05765s (17.4956GiB/s) | 20260507 04:36:26.050 | INFO | Downloaded 34GiB to GPU in 1.96284s (17.3219GiB/s) | 20260507 04:36:31.192 | INFO | Analyzing chunk frames = [7142400,7142912) | 20260507 04:36:31.192 | INFO | MemInfo: { | 20260507 04:36:31.192 | INFO | "gpu" : " 17 GB", | 20260507 04:36:31.192 | INFO | "huge" : "", | 20260507 04:36:31.192 | INFO | "huge_shared" : " 27 GB", | 20260507 04:36:31.192 | INFO | "malloc" : " 14 GB", | 20260507 04:36:31.192 | INFO | "pinned" : " 29 GB" | 20260507 04:36:31.192 | INFO | } | 20260507 04:36:31.192 | INFO | Perf report for the last 5.14182 seconds: | 20260507 04:36:31.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:36:31.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:31.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:31.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:31.192 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 04:36:31.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:31.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:31.192 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:36:31.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:31.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:31.192 | INFO | Uploaded 36GiB to GPU in 2.03441s (17.6956GiB/s) | 20260507 04:36:31.192 | INFO | Downloaded 34GiB to GPU in 1.98002s (17.1716GiB/s) | 20260507 04:36:36.331 | INFO | Analyzing chunk frames = [7142912,7143424) | 20260507 04:36:36.331 | INFO | MemInfo: { | 20260507 04:36:36.331 | INFO | "gpu" : " 17 GB", | 20260507 04:36:36.331 | INFO | "huge" : "", | 20260507 04:36:36.331 | INFO | "huge_shared" : " 27 GB", | 20260507 04:36:36.331 | INFO | "malloc" : " 14 GB", | 20260507 04:36:36.331 | INFO | "pinned" : " 29 GB" | 20260507 04:36:36.331 | INFO | } | 20260507 04:36:36.331 | INFO | Perf report for the last 5.13907 seconds: | 20260507 04:36:36.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:36:36.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:36.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:36.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:36.331 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 04:36:36.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:36.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:36.331 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:36:36.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:36.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:36.331 | INFO | Uploaded 36GiB to GPU in 1.96629s (18.3086GiB/s) | 20260507 04:36:36.331 | INFO | Downloaded 34GiB to GPU in 1.95868s (17.3586GiB/s) | 20260507 04:36:36.332 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:36:36.332 | INFO | Total mem usage: 18 GiB | 20260507 04:36:36.332 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:36:36.332 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 04:36:36.332 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:36:36.332 | INFO | High Water Breakdown: | 20260507 04:36:36.332 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:36:36.332 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:36:36.332 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:36:36.332 | INFO | ------------------------------------------ | 20260507 04:36:36.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:36:36.332 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:36:36.332 | INFO | Total mem usage: 26.5 GiB | 20260507 04:36:36.332 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:36:36.332 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:36:36.332 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:36:36.332 | INFO | High Water Breakdown: | 20260507 04:36:36.332 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:36:36.332 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:36:36.332 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:36:36.332 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:36:36.332 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:36:36.332 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:36:36.332 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:36:36.332 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:36:36.332 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:36:36.332 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:36:36.332 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:36:36.332 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:36:36.332 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:36:36.332 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:36:36.332 | INFO | ------------------------------------------ | 20260507 04:36:36.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:36:36.332 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:36:36.332 | INFO | Total mem usage: 25.6 GiB | 20260507 04:36:36.332 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:36:36.332 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:36:36.332 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:36:36.332 | INFO | High Water Breakdown: | 20260507 04:36:36.332 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:36:36.332 | INFO | ------------------------------------------ | 20260507 04:36:36.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:36:36.332 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:36:36.332 | INFO | Total mem usage: 15.7 GiB | 20260507 04:36:36.332 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:36:36.332 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:36:36.332 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:36:36.332 | INFO | High Water Breakdown: | 20260507 04:36:36.332 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:36:36.332 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:36:36.332 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:36:36.332 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:36:36.332 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:36:36.332 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:36:36.332 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:36:36.332 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:36:36.332 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:36:36.332 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:36:36.332 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:36:36.332 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:36:36.332 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:36:36.332 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:36:36.332 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:36:36.332 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:36:36.332 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:36:36.332 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:36:36.332 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:36:36.332 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:36:36.332 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:36:36.332 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:36:36.332 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:36:36.332 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:36:36.332 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:36:36.332 | INFO | ------------------------------------------ | 20260507 04:36:36.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:36:36.332 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:36:36.332 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:36:36.332 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:36:36.332 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.34e+01 / 1.46e+00 | 20260507 04:36:36.332 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.02e-01, 9.64e-03 / 6.98e-03 | 20260507 04:36:36.332 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.41e+00, 5.17e-01 / 2.86e-01 | 20260507 04:36:36.332 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.47e+00, 1.69e-01 / 1.19e-01 | 20260507 04:36:36.333 | INFO | PerfReport: 16.21%, DME -- 3.99e+01 / 4.47e+01, 4.01e+00 / 1.85e+00 | 20260507 04:36:36.333 | INFO | PerfReport: 57.58%, FrameLabeling -- 1.77e+01 / 1.98e+01, 1.24e+00 / 1.32e+00 | 20260507 04:36:36.333 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.14e+00 / 2.72e+00, 3.89e-01 / 1.92e-01 | 20260507 04:36:36.333 | INFO | PerfReport: 3.02%, Metrics -- 9.28e-01 / 1.26e+00, 2.47e-01 / 1.44e-01 | 20260507 04:36:36.333 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.13e-01 / 3.65e-01, 4.94e-02 / 1.83e-02 | 20260507 04:36:36.333 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:36:36.333 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:36:36.333 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:36:36.333 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 04:36:36.333 | INFO | PerfReport: 21.77%, Upload -- 2.09e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260507 04:36:36.333 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 04:36:36.333 | INFO | PerfReport: 31.95%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:36:36.333 | INFO | PerfReport: 1.47%, LockUpload -- 1.42e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 04:36:36.333 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260507 04:36:36.333 | INFO | PerfReport: 22.10%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 04:36:36.333 | INFO | PerfReport: 0.02%, Misc -- 1.58e-02 / 0.00e+00, 5.44e-02 / 0.00e+00 | 20260507 04:36:36.429 | INFO | Starting a round of Baz IO | 20260507 04:36:39.537 | INFO | Populating RTMetrics report | 20260507 04:36:39.538 | INFO | RTMetrics report populated | 20260507 04:36:39.538 | INFO | Opening tmp RTMetrics json file | 20260507 04:36:39.561 | INFO | Serializing RTMetrics to json | 20260507 04:36:39.564 | INFO | Serialization done, swapping tmp file now... | 20260507 04:36:39.564 | INFO | RTMetrics done | 20260507 04:36:39.630 | INFO | Finished a round of Baz IO: Wrote 1.72 GiB in 3.201 seconds | 20260507 04:36:39.630 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.78% / 0.00% / 67.22% | 20260507 04:36:39.630 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.165884% / 0% / 7.15256e-05% | 20260507 04:36:39.630 | INFO | Baz IO overall: | 20260507 04:36:39.630 | INFO | BytesWritten : 3642474919547 | 20260507 04:36:39.630 | INFO | NumEvents : 1748429517815 | 20260507 04:36:41.470 | INFO | Analyzing chunk frames = [7143424,7143936) | 20260507 04:36:41.470 | INFO | MemInfo: { | 20260507 04:36:41.470 | INFO | "gpu" : " 17 GB", | 20260507 04:36:41.470 | INFO | "huge" : "", | 20260507 04:36:41.470 | INFO | "huge_shared" : " 27 GB", | 20260507 04:36:41.470 | INFO | "malloc" : " 14 GB", | 20260507 04:36:41.470 | INFO | "pinned" : " 29 GB" | 20260507 04:36:41.470 | INFO | } | 20260507 04:36:41.470 | INFO | Perf report for the last 5.13917 seconds: | 20260507 04:36:41.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:36:41.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:41.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:41.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:41.470 | INFO | ANALYSIS : 61.9%, 3.0, 96.2ms, 0 | 20260507 04:36:41.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:41.470 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 04:36:41.470 | INFO | PRE_HQ : 59.1%, 1.8, 55.1ms, 0 | 20260507 04:36:41.470 | INFO | BAZWRITER : 61.9%, 3.3, 104.2ms, 0 | 20260507 04:36:41.470 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:36:41.471 | INFO | Uploaded 36GiB to GPU in 2.0424s (17.6263GiB/s) | 20260507 04:36:41.471 | INFO | Downloaded 38GiB to GPU in 2.1757s (17.4657GiB/s) | 20260507 04:36:46.609 | INFO | Analyzing chunk frames = [7143936,7144448) | 20260507 04:36:46.609 | INFO | MemInfo: { | 20260507 04:36:46.609 | INFO | "gpu" : " 17 GB", | 20260507 04:36:46.609 | INFO | "huge" : "", | 20260507 04:36:46.609 | INFO | "huge_shared" : " 27 GB", | 20260507 04:36:46.609 | INFO | "malloc" : " 14 GB", | 20260507 04:36:46.609 | INFO | "pinned" : " 29 GB" | 20260507 04:36:46.609 | INFO | } | 20260507 04:36:46.609 | INFO | Perf report for the last 5.13879 seconds: | 20260507 04:36:46.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:36:46.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:46.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:46.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:46.609 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 04:36:46.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:46.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:46.609 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:36:46.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:46.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:46.610 | INFO | Uploaded 36GiB to GPU in 2.04837s (17.5749GiB/s) | 20260507 04:36:46.610 | INFO | Downloaded 34GiB to GPU in 1.93652s (17.5573GiB/s) | 20260507 04:36:51.750 | INFO | Analyzing chunk frames = [7144448,7144960) | 20260507 04:36:51.750 | INFO | MemInfo: { | 20260507 04:36:51.750 | INFO | "gpu" : " 17 GB", | 20260507 04:36:51.750 | INFO | "huge" : "", | 20260507 04:36:51.750 | INFO | "huge_shared" : " 27 GB", | 20260507 04:36:51.750 | INFO | "malloc" : " 14 GB", | 20260507 04:36:51.750 | INFO | "pinned" : " 29 GB" | 20260507 04:36:51.750 | INFO | } | 20260507 04:36:51.750 | INFO | Perf report for the last 5.14123 seconds: | 20260507 04:36:51.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:36:51.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:51.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:51.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:51.750 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 04:36:51.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:51.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:51.750 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:36:51.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:51.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:51.750 | INFO | Uploaded 36GiB to GPU in 2.0526s (17.5387GiB/s) | 20260507 04:36:51.750 | INFO | Downloaded 34GiB to GPU in 1.94795s (17.4543GiB/s) | 20260507 04:36:56.888 | INFO | Analyzing chunk frames = [7144960,7145472) | 20260507 04:36:56.888 | INFO | MemInfo: { | 20260507 04:36:56.888 | INFO | "gpu" : " 17 GB", | 20260507 04:36:56.888 | INFO | "huge" : "", | 20260507 04:36:56.888 | INFO | "huge_shared" : " 27 GB", | 20260507 04:36:56.888 | INFO | "malloc" : " 14 GB", | 20260507 04:36:56.888 | INFO | "pinned" : " 29 GB" | 20260507 04:36:56.888 | INFO | } | 20260507 04:36:56.888 | INFO | Perf report for the last 5.13747 seconds: | 20260507 04:36:56.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:36:56.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:56.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:56.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:56.888 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 04:36:56.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:56.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:56.888 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:36:56.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:56.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:36:56.888 | INFO | Uploaded 36GiB to GPU in 2.01868s (17.8335GiB/s) | 20260507 04:36:56.888 | INFO | Downloaded 34GiB to GPU in 1.93526s (17.5687GiB/s) | 20260507 04:37:02.029 | INFO | Analyzing chunk frames = [7145472,7145984) | 20260507 04:37:02.029 | INFO | MemInfo: { | 20260507 04:37:02.029 | INFO | "gpu" : " 17 GB", | 20260507 04:37:02.029 | INFO | "huge" : "", | 20260507 04:37:02.029 | INFO | "huge_shared" : " 27 GB", | 20260507 04:37:02.029 | INFO | "malloc" : " 14 GB", | 20260507 04:37:02.029 | INFO | "pinned" : " 29 GB" | 20260507 04:37:02.029 | INFO | } | 20260507 04:37:02.029 | INFO | Perf report for the last 5.14135 seconds: | 20260507 04:37:02.029 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:37:02.029 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:02.029 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:02.029 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:02.029 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 04:37:02.029 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:02.029 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:02.029 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:37:02.029 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:02.029 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:02.029 | INFO | Uploaded 36GiB to GPU in 2.03567s (17.6846GiB/s) | 20260507 04:37:02.029 | INFO | Downloaded 34GiB to GPU in 1.93873s (17.5372GiB/s) | 20260507 04:37:07.173 | INFO | Analyzing chunk frames = [7145984,7146496) | 20260507 04:37:07.173 | INFO | MemInfo: { | 20260507 04:37:07.173 | INFO | "gpu" : " 17 GB", | 20260507 04:37:07.173 | INFO | "huge" : "", | 20260507 04:37:07.173 | INFO | "huge_shared" : " 27 GB", | 20260507 04:37:07.173 | INFO | "malloc" : " 14 GB", | 20260507 04:37:07.173 | INFO | "pinned" : " 29 GB" | 20260507 04:37:07.173 | INFO | } | 20260507 04:37:07.173 | INFO | Perf report for the last 5.1439 seconds: | 20260507 04:37:07.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:37:07.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:07.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:07.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:07.173 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 04:37:07.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:07.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:07.173 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 04:37:07.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:07.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:07.173 | INFO | Uploaded 36GiB to GPU in 2.03206s (17.716GiB/s) | 20260507 04:37:07.173 | INFO | Downloaded 34GiB to GPU in 1.91846s (17.7226GiB/s) | 20260507 04:37:12.311 | INFO | Analyzing chunk frames = [7146496,7147008) | 20260507 04:37:12.311 | INFO | MemInfo: { | 20260507 04:37:12.311 | INFO | "gpu" : " 17 GB", | 20260507 04:37:12.311 | INFO | "huge" : "", | 20260507 04:37:12.311 | INFO | "huge_shared" : " 27 GB", | 20260507 04:37:12.311 | INFO | "malloc" : " 14 GB", | 20260507 04:37:12.311 | INFO | "pinned" : " 29 GB" | 20260507 04:37:12.311 | INFO | } | 20260507 04:37:12.311 | INFO | Perf report for the last 5.13839 seconds: | 20260507 04:37:12.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:37:12.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:12.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:12.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:12.311 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 04:37:12.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:12.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:12.312 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 04:37:12.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:12.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:12.312 | INFO | Uploaded 36GiB to GPU in 1.95971s (18.3701GiB/s) | 20260507 04:37:12.312 | INFO | Downloaded 34GiB to GPU in 1.97704s (17.1974GiB/s) | 20260507 04:37:17.445 | INFO | Analyzing chunk frames = [7147008,7147520) | 20260507 04:37:17.445 | INFO | MemInfo: { | 20260507 04:37:17.445 | INFO | "gpu" : " 17 GB", | 20260507 04:37:17.445 | INFO | "huge" : "", | 20260507 04:37:17.446 | INFO | "huge_shared" : " 27 GB", | 20260507 04:37:17.446 | INFO | "malloc" : " 14 GB", | 20260507 04:37:17.446 | INFO | "pinned" : " 29 GB" | 20260507 04:37:17.446 | INFO | } | 20260507 04:37:17.446 | INFO | Perf report for the last 5.13416 seconds: | 20260507 04:37:17.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:37:17.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:17.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:17.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:17.446 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 04:37:17.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:17.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:17.446 | INFO | PRE_HQ : 29.5%, 1.0, 15.3ms, 0 | 20260507 04:37:17.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:17.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:17.446 | INFO | Uploaded 36GiB to GPU in 1.99048s (18.0861GiB/s) | 20260507 04:37:17.446 | INFO | Downloaded 34GiB to GPU in 1.9521s (17.4171GiB/s) | 20260507 04:37:17.543 | INFO | Starting a round of Baz IO | 20260507 04:37:20.641 | INFO | Populating RTMetrics report | 20260507 04:37:20.642 | INFO | RTMetrics report populated | 20260507 04:37:20.642 | INFO | Opening tmp RTMetrics json file | 20260507 04:37:20.642 | INFO | Serializing RTMetrics to json | 20260507 04:37:20.649 | INFO | Serialization done, swapping tmp file now... | 20260507 04:37:20.662 | INFO | RTMetrics done | 20260507 04:37:20.731 | INFO | Finished a round of Baz IO: Wrote 1.723 GiB in 3.188 seconds | 20260507 04:37:20.731 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.78% / 0.00% / 67.22% | 20260507 04:37:20.731 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.165963% / 3.97364e-06% / 6.75519e-05% | 20260507 04:37:20.731 | INFO | Baz IO overall: | 20260507 04:37:20.731 | INFO | BytesWritten : 3644324669670 | 20260507 04:37:20.731 | INFO | NumEvents : 1749300700982 | 20260507 04:37:22.582 | INFO | Analyzing chunk frames = [7147520,7148032) | 20260507 04:37:22.582 | INFO | MemInfo: { | 20260507 04:37:22.582 | INFO | "gpu" : " 17 GB", | 20260507 04:37:22.582 | INFO | "huge" : "", | 20260507 04:37:22.582 | INFO | "huge_shared" : " 27 GB", | 20260507 04:37:22.582 | INFO | "malloc" : " 14 GB", | 20260507 04:37:22.582 | INFO | "pinned" : " 29 GB" | 20260507 04:37:22.582 | INFO | } | 20260507 04:37:22.582 | INFO | Perf report for the last 5.13611 seconds: | 20260507 04:37:22.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:37:22.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:22.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:22.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:22.582 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 04:37:22.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:22.582 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 04:37:22.582 | INFO | PRE_HQ : 59.3%, 1.8, 55.6ms, 0 | 20260507 04:37:22.582 | INFO | BAZWRITER : 61.7%, 3.3, 104.5ms, 0 | 20260507 04:37:22.582 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:37:22.582 | INFO | Uploaded 36GiB to GPU in 2.02153s (17.8083GiB/s) | 20260507 04:37:22.582 | INFO | Downloaded 38GiB to GPU in 2.17108s (17.5028GiB/s) | 20260507 04:37:27.725 | INFO | Analyzing chunk frames = [7148032,7148544) | 20260507 04:37:27.725 | INFO | MemInfo: { | 20260507 04:37:27.725 | INFO | "gpu" : " 17 GB", | 20260507 04:37:27.725 | INFO | "huge" : "", | 20260507 04:37:27.725 | INFO | "huge_shared" : " 27 GB", | 20260507 04:37:27.725 | INFO | "malloc" : " 14 GB", | 20260507 04:37:27.725 | INFO | "pinned" : " 29 GB" | 20260507 04:37:27.725 | INFO | } | 20260507 04:37:27.725 | INFO | Perf report for the last 5.14356 seconds: | 20260507 04:37:27.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:37:27.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:27.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:27.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:27.725 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 04:37:27.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:27.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:27.725 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:37:27.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:27.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:27.726 | INFO | Uploaded 36GiB to GPU in 2.04018s (17.6455GiB/s) | 20260507 04:37:27.726 | INFO | Downloaded 34GiB to GPU in 1.93963s (17.5291GiB/s) | 20260507 04:37:32.875 | INFO | Analyzing chunk frames = [7148544,7149056) | 20260507 04:37:32.875 | INFO | MemInfo: { | 20260507 04:37:32.875 | INFO | "gpu" : " 17 GB", | 20260507 04:37:32.875 | INFO | "huge" : "", | 20260507 04:37:32.875 | INFO | "huge_shared" : " 27 GB", | 20260507 04:37:32.875 | INFO | "malloc" : " 14 GB", | 20260507 04:37:32.875 | INFO | "pinned" : " 29 GB" | 20260507 04:37:32.875 | INFO | } | 20260507 04:37:32.875 | INFO | Perf report for the last 5.1499 seconds: | 20260507 04:37:32.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:37:32.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:32.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:32.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:32.875 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 04:37:32.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:32.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:32.875 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:37:32.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:32.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:32.875 | INFO | Uploaded 36GiB to GPU in 2.00341s (17.9693GiB/s) | 20260507 04:37:32.875 | INFO | Downloaded 34GiB to GPU in 1.9393s (17.5321GiB/s) | 20260507 04:37:38.009 | INFO | Analyzing chunk frames = [7149056,7149568) | 20260507 04:37:38.009 | INFO | MemInfo: { | 20260507 04:37:38.009 | INFO | "gpu" : " 17 GB", | 20260507 04:37:38.009 | INFO | "huge" : "", | 20260507 04:37:38.009 | INFO | "huge_shared" : " 27 GB", | 20260507 04:37:38.009 | INFO | "malloc" : " 14 GB", | 20260507 04:37:38.009 | INFO | "pinned" : " 29 GB" | 20260507 04:37:38.009 | INFO | } | 20260507 04:37:38.009 | INFO | Perf report for the last 5.13391 seconds: | 20260507 04:37:38.009 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:37:38.009 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:38.009 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:38.009 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:38.009 | INFO | ANALYSIS : 60.6%, 2.9, 93.4ms, 0 | 20260507 04:37:38.009 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:38.009 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:38.009 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:37:38.009 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:38.009 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:38.009 | INFO | Uploaded 36GiB to GPU in 2.05231s (17.5412GiB/s) | 20260507 04:37:38.009 | INFO | Downloaded 34GiB to GPU in 1.94283s (17.5003GiB/s) | 20260507 04:37:43.142 | INFO | Analyzing chunk frames = [7149568,7150080) | 20260507 04:37:43.142 | INFO | MemInfo: { | 20260507 04:37:43.142 | INFO | "gpu" : " 17 GB", | 20260507 04:37:43.142 | INFO | "huge" : "", | 20260507 04:37:43.142 | INFO | "huge_shared" : " 27 GB", | 20260507 04:37:43.142 | INFO | "malloc" : " 14 GB", | 20260507 04:37:43.142 | INFO | "pinned" : " 29 GB" | 20260507 04:37:43.142 | INFO | } | 20260507 04:37:43.142 | INFO | Perf report for the last 5.13308 seconds: | 20260507 04:37:43.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:37:43.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:43.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:43.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:43.142 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 04:37:43.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:43.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:43.142 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 04:37:43.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:43.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:43.142 | INFO | Uploaded 36GiB to GPU in 2.03449s (17.6949GiB/s) | 20260507 04:37:43.142 | INFO | Downloaded 34GiB to GPU in 1.91033s (17.798GiB/s) | 20260507 04:37:48.281 | INFO | Analyzing chunk frames = [7150080,7150592) | 20260507 04:37:48.281 | INFO | MemInfo: { | 20260507 04:37:48.281 | INFO | "gpu" : " 17 GB", | 20260507 04:37:48.281 | INFO | "huge" : "", | 20260507 04:37:48.281 | INFO | "huge_shared" : " 27 GB", | 20260507 04:37:48.281 | INFO | "malloc" : " 14 GB", | 20260507 04:37:48.281 | INFO | "pinned" : " 29 GB" | 20260507 04:37:48.282 | INFO | } | 20260507 04:37:48.282 | INFO | Perf report for the last 5.13929 seconds: | 20260507 04:37:48.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:37:48.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:48.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:48.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:48.282 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:37:48.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:48.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:48.282 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:37:48.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:48.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:48.282 | INFO | Uploaded 36GiB to GPU in 2.0578s (17.4944GiB/s) | 20260507 04:37:48.282 | INFO | Downloaded 34GiB to GPU in 1.96546s (17.2988GiB/s) | 20260507 04:37:53.421 | INFO | Analyzing chunk frames = [7150592,7151104) | 20260507 04:37:53.422 | INFO | MemInfo: { | 20260507 04:37:53.422 | INFO | "gpu" : " 17 GB", | 20260507 04:37:53.422 | INFO | "huge" : "", | 20260507 04:37:53.422 | INFO | "huge_shared" : " 27 GB", | 20260507 04:37:53.422 | INFO | "malloc" : " 14 GB", | 20260507 04:37:53.422 | INFO | "pinned" : " 29 GB" | 20260507 04:37:53.422 | INFO | } | 20260507 04:37:53.422 | INFO | Perf report for the last 5.14009 seconds: | 20260507 04:37:53.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:37:53.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:53.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:53.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:53.422 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:37:53.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:53.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:53.422 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:37:53.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:53.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:53.422 | INFO | Uploaded 36GiB to GPU in 1.96686s (18.3033GiB/s) | 20260507 04:37:53.422 | INFO | Downloaded 34GiB to GPU in 1.96855s (17.2716GiB/s) | 20260507 04:37:58.559 | INFO | Analyzing chunk frames = [7151104,7151616) | 20260507 04:37:58.559 | INFO | MemInfo: { | 20260507 04:37:58.559 | INFO | "gpu" : " 17 GB", | 20260507 04:37:58.559 | INFO | "huge" : "", | 20260507 04:37:58.559 | INFO | "huge_shared" : " 27 GB", | 20260507 04:37:58.559 | INFO | "malloc" : " 14 GB", | 20260507 04:37:58.559 | INFO | "pinned" : " 29 GB" | 20260507 04:37:58.559 | INFO | } | 20260507 04:37:58.559 | INFO | Perf report for the last 5.13712 seconds: | 20260507 04:37:58.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:37:58.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:58.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:58.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:58.559 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:37:58.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:58.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:58.559 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:37:58.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:58.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:37:58.559 | INFO | Uploaded 36GiB to GPU in 1.97029s (18.2714GiB/s) | 20260507 04:37:58.559 | INFO | Downloaded 34GiB to GPU in 1.96676s (17.2873GiB/s) | 20260507 04:37:58.559 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:37:58.559 | INFO | Total mem usage: 18 GiB | 20260507 04:37:58.559 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:37:58.559 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 04:37:58.559 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:37:58.559 | INFO | High Water Breakdown: | 20260507 04:37:58.559 | INFO | 60.1% : 8.57 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:37:58.559 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:37:58.559 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:37:58.559 | INFO | ------------------------------------------ | 20260507 04:37:58.559 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:37:58.559 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:37:58.559 | INFO | Total mem usage: 26.5 GiB | 20260507 04:37:58.559 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:37:58.559 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:37:58.559 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:37:58.559 | INFO | High Water Breakdown: | 20260507 04:37:58.559 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:37:58.559 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:37:58.559 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:37:58.559 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:37:58.559 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:37:58.559 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:37:58.559 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:37:58.559 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:37:58.560 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:37:58.560 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:37:58.560 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:37:58.560 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:37:58.560 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:37:58.560 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:37:58.560 | INFO | ------------------------------------------ | 20260507 04:37:58.560 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:37:58.560 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:37:58.560 | INFO | Total mem usage: 25.6 GiB | 20260507 04:37:58.560 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:37:58.560 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:37:58.560 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:37:58.560 | INFO | High Water Breakdown: | 20260507 04:37:58.560 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:37:58.560 | INFO | ------------------------------------------ | 20260507 04:37:58.560 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:37:58.560 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:37:58.560 | INFO | Total mem usage: 15.7 GiB | 20260507 04:37:58.560 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:37:58.560 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:37:58.560 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:37:58.560 | INFO | High Water Breakdown: | 20260507 04:37:58.560 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:37:58.560 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:37:58.560 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:37:58.560 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:37:58.560 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:37:58.560 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:37:58.560 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:37:58.560 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:37:58.560 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:37:58.560 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:37:58.560 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:37:58.560 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:37:58.560 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:37:58.560 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:37:58.560 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:37:58.560 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:37:58.560 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:37:58.560 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:37:58.560 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:37:58.560 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:37:58.560 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:37:58.560 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:37:58.560 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:37:58.560 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:37:58.560 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:37:58.560 | INFO | ------------------------------------------ | 20260507 04:37:58.560 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:37:58.560 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:37:58.560 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:37:58.560 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:37:58.560 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.33e+01 / 1.23e+00 | 20260507 04:37:58.560 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.01e-01, 9.45e-03 / 4.07e-03 | 20260507 04:37:58.560 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.47e+00, 5.05e-01 / 2.34e-01 | 20260507 04:37:58.560 | INFO | PerfReport: 3.21%, Histogram -- 9.86e-01 / 1.48e+00, 1.64e-01 / 1.02e-01 | 20260507 04:37:58.560 | INFO | PerfReport: 16.20%, DME -- 3.98e+01 / 4.46e+01, 3.85e+00 / 1.56e+00 | 20260507 04:37:58.560 | INFO | PerfReport: 57.64%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.23e+00 / 1.51e+00 | 20260507 04:37:58.560 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.14e+00 / 2.73e+00, 3.84e-01 / 1.80e-01 | 20260507 04:37:58.560 | INFO | PerfReport: 3.02%, Metrics -- 9.28e-01 / 1.25e+00, 2.44e-01 / 1.37e-01 | 20260507 04:37:58.560 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.13e-01 / 3.72e-01, 5.01e-02 / 2.39e-02 | 20260507 04:37:58.560 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:37:58.560 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:37:58.560 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:37:58.560 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 04:37:58.560 | INFO | PerfReport: 21.86%, Upload -- 2.10e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260507 04:37:58.560 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 04:37:58.560 | INFO | PerfReport: 31.88%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 04:37:58.560 | INFO | PerfReport: 1.66%, LockUpload -- 1.60e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 04:37:58.560 | INFO | PerfReport: 1.20%, LockDownload -- 1.15e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260507 04:37:58.560 | INFO | PerfReport: 22.03%, LockCompute -- 2.12e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 04:37:58.560 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260507 04:37:58.655 | INFO | Starting a round of Baz IO | 20260507 04:38:01.758 | INFO | Populating RTMetrics report | 20260507 04:38:01.759 | INFO | RTMetrics report populated | 20260507 04:38:01.759 | INFO | Opening tmp RTMetrics json file | 20260507 04:38:01.776 | INFO | Serializing RTMetrics to json | 20260507 04:38:01.777 | INFO | Serialization done, swapping tmp file now... | 20260507 04:38:01.778 | INFO | RTMetrics done | 20260507 04:38:01.846 | INFO | Finished a round of Baz IO: Wrote 1.72 GiB in 3.19 seconds | 20260507 04:38:01.846 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.78% / 0.00% / 67.22% | 20260507 04:38:01.846 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.164874% / 0% / 7.54992e-05% | 20260507 04:38:01.846 | INFO | Baz IO overall: | 20260507 04:38:01.846 | INFO | BytesWritten : 3646171265363 | 20260507 04:38:01.846 | INFO | NumEvents : 1750169578690 | 20260507 04:38:03.698 | INFO | Analyzing chunk frames = [7151616,7152128) | 20260507 04:38:03.698 | INFO | MemInfo: { | 20260507 04:38:03.698 | INFO | "gpu" : " 17 GB", | 20260507 04:38:03.698 | INFO | "huge" : "", | 20260507 04:38:03.699 | INFO | "huge_shared" : " 27 GB", | 20260507 04:38:03.699 | INFO | "malloc" : " 14 GB", | 20260507 04:38:03.699 | INFO | "pinned" : " 29 GB" | 20260507 04:38:03.699 | INFO | } | 20260507 04:38:03.699 | INFO | Perf report for the last 5.13981 seconds: | 20260507 04:38:03.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:38:03.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:03.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:03.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:03.699 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:38:03.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:03.699 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 04:38:03.699 | INFO | PRE_HQ : 59.1%, 1.8, 55.1ms, 0 | 20260507 04:38:03.699 | INFO | BAZWRITER : 61.7%, 3.2, 101.3ms, 0 | 20260507 04:38:03.699 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 04:38:03.699 | INFO | Uploaded 36GiB to GPU in 2.03103s (17.725GiB/s) | 20260507 04:38:03.699 | INFO | Downloaded 38GiB to GPU in 2.17761s (17.4503GiB/s) | 20260507 04:38:08.839 | INFO | Analyzing chunk frames = [7152128,7152640) | 20260507 04:38:08.839 | INFO | MemInfo: { | 20260507 04:38:08.839 | INFO | "gpu" : " 17 GB", | 20260507 04:38:08.839 | INFO | "huge" : "", | 20260507 04:38:08.839 | INFO | "huge_shared" : " 27 GB", | 20260507 04:38:08.839 | INFO | "malloc" : " 14 GB", | 20260507 04:38:08.839 | INFO | "pinned" : " 29 GB" | 20260507 04:38:08.839 | INFO | } | 20260507 04:38:08.839 | INFO | Perf report for the last 5.1403 seconds: | 20260507 04:38:08.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:38:08.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:08.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:08.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:08.839 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:38:08.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:08.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:08.839 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:38:08.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:08.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:08.839 | INFO | Uploaded 36GiB to GPU in 2.03471s (17.693GiB/s) | 20260507 04:38:08.839 | INFO | Downloaded 34GiB to GPU in 1.93754s (17.5481GiB/s) | 20260507 04:38:13.984 | INFO | Analyzing chunk frames = [7152640,7153152) | 20260507 04:38:13.984 | INFO | MemInfo: { | 20260507 04:38:13.984 | INFO | "gpu" : " 17 GB", | 20260507 04:38:13.984 | INFO | "huge" : "", | 20260507 04:38:13.984 | INFO | "huge_shared" : " 27 GB", | 20260507 04:38:13.984 | INFO | "malloc" : " 14 GB", | 20260507 04:38:13.984 | INFO | "pinned" : " 29 GB" | 20260507 04:38:13.984 | INFO | } | 20260507 04:38:13.984 | INFO | Perf report for the last 5.14498 seconds: | 20260507 04:38:13.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:38:13.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:13.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:13.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:13.984 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 04:38:13.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:13.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:13.984 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:38:13.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:13.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:13.984 | INFO | Uploaded 36GiB to GPU in 2.00971s (17.913GiB/s) | 20260507 04:38:13.984 | INFO | Downloaded 34GiB to GPU in 1.94041s (17.5221GiB/s) | 20260507 04:38:19.121 | INFO | Analyzing chunk frames = [7153152,7153664) | 20260507 04:38:19.121 | INFO | MemInfo: { | 20260507 04:38:19.121 | INFO | "gpu" : " 17 GB", | 20260507 04:38:19.121 | INFO | "huge" : "", | 20260507 04:38:19.121 | INFO | "huge_shared" : " 27 GB", | 20260507 04:38:19.121 | INFO | "malloc" : " 14 GB", | 20260507 04:38:19.121 | INFO | "pinned" : " 29 GB" | 20260507 04:38:19.121 | INFO | } | 20260507 04:38:19.121 | INFO | Perf report for the last 5.1374 seconds: | 20260507 04:38:19.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:38:19.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:19.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:19.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:19.121 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 04:38:19.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:19.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:19.121 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:38:19.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:19.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:19.121 | INFO | Uploaded 36GiB to GPU in 2.02177s (17.8061GiB/s) | 20260507 04:38:19.121 | INFO | Downloaded 34GiB to GPU in 1.94182s (17.5094GiB/s) | 20260507 04:38:24.255 | INFO | Analyzing chunk frames = [7153664,7154176) | 20260507 04:38:24.255 | INFO | MemInfo: { | 20260507 04:38:24.255 | INFO | "gpu" : " 17 GB", | 20260507 04:38:24.255 | INFO | "huge" : "", | 20260507 04:38:24.255 | INFO | "huge_shared" : " 27 GB", | 20260507 04:38:24.255 | INFO | "malloc" : " 14 GB", | 20260507 04:38:24.255 | INFO | "pinned" : " 29 GB" | 20260507 04:38:24.255 | INFO | } | 20260507 04:38:24.255 | INFO | Perf report for the last 5.13392 seconds: | 20260507 04:38:24.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:38:24.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:24.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:24.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:24.255 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 04:38:24.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:24.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:24.255 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:38:24.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:24.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:24.256 | INFO | Uploaded 36GiB to GPU in 2.05774s (17.4949GiB/s) | 20260507 04:38:24.256 | INFO | Downloaded 34GiB to GPU in 1.95106s (17.4265GiB/s) | 20260507 04:38:29.397 | INFO | Analyzing chunk frames = [7154176,7154688) | 20260507 04:38:29.397 | INFO | MemInfo: { | 20260507 04:38:29.397 | INFO | "gpu" : " 17 GB", | 20260507 04:38:29.397 | INFO | "huge" : "", | 20260507 04:38:29.397 | INFO | "huge_shared" : " 27 GB", | 20260507 04:38:29.397 | INFO | "malloc" : " 14 GB", | 20260507 04:38:29.397 | INFO | "pinned" : " 29 GB" | 20260507 04:38:29.397 | INFO | } | 20260507 04:38:29.397 | INFO | Perf report for the last 5.14155 seconds: | 20260507 04:38:29.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:38:29.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:29.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:29.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:29.397 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 04:38:29.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:29.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:29.397 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 04:38:29.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:29.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:29.397 | INFO | Uploaded 36GiB to GPU in 2.04794s (17.5787GiB/s) | 20260507 04:38:29.397 | INFO | Downloaded 34GiB to GPU in 1.91462s (17.7581GiB/s) | 20260507 04:38:34.535 | INFO | Analyzing chunk frames = [7154688,7155200) | 20260507 04:38:34.535 | INFO | MemInfo: { | 20260507 04:38:34.535 | INFO | "gpu" : " 17 GB", | 20260507 04:38:34.535 | INFO | "huge" : "", | 20260507 04:38:34.535 | INFO | "huge_shared" : " 27 GB", | 20260507 04:38:34.535 | INFO | "malloc" : " 14 GB", | 20260507 04:38:34.535 | INFO | "pinned" : " 29 GB" | 20260507 04:38:34.535 | INFO | } | 20260507 04:38:34.536 | INFO | Perf report for the last 5.13875 seconds: | 20260507 04:38:34.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:38:34.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:34.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:34.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:34.536 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:38:34.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:34.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:34.536 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:38:34.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:34.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:34.536 | INFO | Uploaded 36GiB to GPU in 1.96116s (18.3564GiB/s) | 20260507 04:38:34.536 | INFO | Downloaded 34GiB to GPU in 1.96612s (17.2929GiB/s) | 20260507 04:38:39.678 | INFO | Analyzing chunk frames = [7155200,7155712) | 20260507 04:38:39.678 | INFO | MemInfo: { | 20260507 04:38:39.678 | INFO | "gpu" : " 17 GB", | 20260507 04:38:39.678 | INFO | "huge" : "", | 20260507 04:38:39.678 | INFO | "huge_shared" : " 27 GB", | 20260507 04:38:39.678 | INFO | "malloc" : " 14 GB", | 20260507 04:38:39.679 | INFO | "pinned" : " 29 GB" | 20260507 04:38:39.679 | INFO | } | 20260507 04:38:39.679 | INFO | Perf report for the last 5.14298 seconds: | 20260507 04:38:39.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:38:39.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:39.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:39.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:39.679 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:38:39.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:39.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:39.679 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:38:39.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:39.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:39.679 | INFO | Uploaded 36GiB to GPU in 1.95016s (18.46GiB/s) | 20260507 04:38:39.679 | INFO | Downloaded 34GiB to GPU in 1.95776s (17.3668GiB/s) | 20260507 04:38:39.776 | INFO | Starting a round of Baz IO | 20260507 04:38:42.878 | INFO | Populating RTMetrics report | 20260507 04:38:42.878 | INFO | RTMetrics report populated | 20260507 04:38:42.878 | INFO | Opening tmp RTMetrics json file | 20260507 04:38:42.883 | INFO | Serializing RTMetrics to json | 20260507 04:38:42.896 | INFO | Serialization done, swapping tmp file now... | 20260507 04:38:42.896 | INFO | RTMetrics done | 20260507 04:38:42.966 | INFO | Finished a round of Baz IO: Wrote 1.718 GiB in 3.19 seconds | 20260507 04:38:42.966 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.77% / 0.00% / 67.22% | 20260507 04:38:42.966 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.164636% / 3.97364e-06% / 7.94729e-05% | 20260507 04:38:42.966 | INFO | Baz IO overall: | 20260507 04:38:42.966 | INFO | BytesWritten : 3648015673849 | 20260507 04:38:42.966 | INFO | NumEvents : 1751036826749 | 20260507 04:38:44.821 | INFO | Analyzing chunk frames = [7155712,7156224) | 20260507 04:38:44.821 | INFO | MemInfo: { | 20260507 04:38:44.821 | INFO | "gpu" : " 17 GB", | 20260507 04:38:44.821 | INFO | "huge" : "", | 20260507 04:38:44.821 | INFO | "huge_shared" : " 27 GB", | 20260507 04:38:44.821 | INFO | "malloc" : " 14 GB", | 20260507 04:38:44.821 | INFO | "pinned" : " 29 GB" | 20260507 04:38:44.821 | INFO | } | 20260507 04:38:44.821 | INFO | Perf report for the last 5.14212 seconds: | 20260507 04:38:44.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:38:44.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:44.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:44.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:44.821 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:38:44.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:44.821 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:38:44.821 | INFO | PRE_HQ : 59.0%, 1.8, 55.2ms, 0 | 20260507 04:38:44.821 | INFO | BAZWRITER : 61.6%, 3.2, 100.9ms, 0 | 20260507 04:38:44.821 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:38:44.821 | INFO | Uploaded 36GiB to GPU in 2.03387s (17.7002GiB/s) | 20260507 04:38:44.821 | INFO | Downloaded 38GiB to GPU in 2.17917s (17.4378GiB/s) | 20260507 04:38:49.953 | INFO | Analyzing chunk frames = [7156224,7156736) | 20260507 04:38:49.953 | INFO | MemInfo: { | 20260507 04:38:49.953 | INFO | "gpu" : " 17 GB", | 20260507 04:38:49.953 | INFO | "huge" : "", | 20260507 04:38:49.953 | INFO | "huge_shared" : " 27 GB", | 20260507 04:38:49.953 | INFO | "malloc" : " 14 GB", | 20260507 04:38:49.953 | INFO | "pinned" : " 29 GB" | 20260507 04:38:49.953 | INFO | } | 20260507 04:38:49.954 | INFO | Perf report for the last 5.1328 seconds: | 20260507 04:38:49.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:38:49.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:49.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:49.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:49.954 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:38:49.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:49.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:49.954 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:38:49.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:49.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:49.954 | INFO | Uploaded 36GiB to GPU in 1.97549s (18.2234GiB/s) | 20260507 04:38:49.954 | INFO | Downloaded 34GiB to GPU in 1.92923s (17.6236GiB/s) | 20260507 04:38:55.094 | INFO | Analyzing chunk frames = [7156736,7157248) | 20260507 04:38:55.094 | INFO | MemInfo: { | 20260507 04:38:55.094 | INFO | "gpu" : " 17 GB", | 20260507 04:38:55.094 | INFO | "huge" : "", | 20260507 04:38:55.094 | INFO | "huge_shared" : " 27 GB", | 20260507 04:38:55.094 | INFO | "malloc" : " 14 GB", | 20260507 04:38:55.094 | INFO | "pinned" : " 29 GB" | 20260507 04:38:55.094 | INFO | } | 20260507 04:38:55.094 | INFO | Perf report for the last 5.1404 seconds: | 20260507 04:38:55.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:38:55.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:55.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:55.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:55.094 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 04:38:55.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:55.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:55.094 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:38:55.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:55.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:38:55.094 | INFO | Uploaded 36GiB to GPU in 2.00788s (17.9294GiB/s) | 20260507 04:38:55.094 | INFO | Downloaded 34GiB to GPU in 1.94391s (17.4905GiB/s) | 20260507 04:39:00.235 | INFO | Analyzing chunk frames = [7157248,7157760) | 20260507 04:39:00.235 | INFO | MemInfo: { | 20260507 04:39:00.235 | INFO | "gpu" : " 17 GB", | 20260507 04:39:00.235 | INFO | "huge" : "", | 20260507 04:39:00.235 | INFO | "huge_shared" : " 27 GB", | 20260507 04:39:00.235 | INFO | "malloc" : " 14 GB", | 20260507 04:39:00.236 | INFO | "pinned" : " 29 GB" | 20260507 04:39:00.236 | INFO | } | 20260507 04:39:00.236 | INFO | Perf report for the last 5.14167 seconds: | 20260507 04:39:00.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:39:00.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:00.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:00.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:00.236 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:39:00.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:00.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:00.236 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:39:00.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:00.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:00.236 | INFO | Uploaded 36GiB to GPU in 1.96534s (18.3174GiB/s) | 20260507 04:39:00.236 | INFO | Downloaded 34GiB to GPU in 1.92373s (17.674GiB/s) | 20260507 04:39:05.373 | INFO | Analyzing chunk frames = [7157760,7158272) | 20260507 04:39:05.373 | INFO | MemInfo: { | 20260507 04:39:05.373 | INFO | "gpu" : " 17 GB", | 20260507 04:39:05.373 | INFO | "huge" : "", | 20260507 04:39:05.373 | INFO | "huge_shared" : " 27 GB", | 20260507 04:39:05.373 | INFO | "malloc" : " 14 GB", | 20260507 04:39:05.373 | INFO | "pinned" : " 29 GB" | 20260507 04:39:05.373 | INFO | } | 20260507 04:39:05.373 | INFO | Perf report for the last 5.13719 seconds: | 20260507 04:39:05.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:39:05.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:05.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:05.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:05.373 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 04:39:05.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:05.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:05.373 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:39:05.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:05.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:05.373 | INFO | Uploaded 36GiB to GPU in 2.01768s (17.8423GiB/s) | 20260507 04:39:05.373 | INFO | Downloaded 34GiB to GPU in 1.93553s (17.5662GiB/s) | 20260507 04:39:10.511 | INFO | Analyzing chunk frames = [7158272,7158784) | 20260507 04:39:10.511 | INFO | MemInfo: { | 20260507 04:39:10.511 | INFO | "gpu" : " 17 GB", | 20260507 04:39:10.511 | INFO | "huge" : "", | 20260507 04:39:10.511 | INFO | "huge_shared" : " 27 GB", | 20260507 04:39:10.511 | INFO | "malloc" : " 14 GB", | 20260507 04:39:10.511 | INFO | "pinned" : " 29 GB" | 20260507 04:39:10.511 | INFO | } | 20260507 04:39:10.511 | INFO | Perf report for the last 5.13801 seconds: | 20260507 04:39:10.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:39:10.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:10.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:10.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:10.511 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 04:39:10.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:10.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:10.511 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 04:39:10.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:10.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:10.511 | INFO | Uploaded 36GiB to GPU in 2.02837s (17.7483GiB/s) | 20260507 04:39:10.511 | INFO | Downloaded 34GiB to GPU in 1.90566s (17.8416GiB/s) | 20260507 04:39:15.652 | INFO | Analyzing chunk frames = [7158784,7159296) | 20260507 04:39:15.652 | INFO | MemInfo: { | 20260507 04:39:15.652 | INFO | "gpu" : " 17 GB", | 20260507 04:39:15.653 | INFO | "huge" : "", | 20260507 04:39:15.653 | INFO | "huge_shared" : " 27 GB", | 20260507 04:39:15.653 | INFO | "malloc" : " 14 GB", | 20260507 04:39:15.653 | INFO | "pinned" : " 29 GB" | 20260507 04:39:15.653 | INFO | } | 20260507 04:39:15.653 | INFO | Perf report for the last 5.14205 seconds: | 20260507 04:39:15.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:39:15.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:15.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:15.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:15.653 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 04:39:15.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:15.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:15.653 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:39:15.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:15.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:15.653 | INFO | Uploaded 36GiB to GPU in 1.97178s (18.2577GiB/s) | 20260507 04:39:15.653 | INFO | Downloaded 34GiB to GPU in 1.96402s (17.3115GiB/s) | 20260507 04:39:20.793 | INFO | Analyzing chunk frames = [7159296,7159808) | 20260507 04:39:20.793 | INFO | MemInfo: { | 20260507 04:39:20.793 | INFO | "gpu" : " 17 GB", | 20260507 04:39:20.793 | INFO | "huge" : "", | 20260507 04:39:20.793 | INFO | "huge_shared" : " 27 GB", | 20260507 04:39:20.793 | INFO | "malloc" : " 14 GB", | 20260507 04:39:20.793 | INFO | "pinned" : " 29 GB" | 20260507 04:39:20.793 | INFO | } | 20260507 04:39:20.793 | INFO | Perf report for the last 5.1402 seconds: | 20260507 04:39:20.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:39:20.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:20.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:20.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:20.793 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:39:20.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:20.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:20.793 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 04:39:20.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:20.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:20.793 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:39:20.793 | INFO | Total mem usage: 18 GiB | 20260507 04:39:20.793 | INFO | Uploaded 36GiB to GPU in 1.95855s (18.381GiB/s) | 20260507 04:39:20.793 | INFO | Downloaded 34GiB to GPU in 1.96202s (17.3291GiB/s) | 20260507 04:39:20.793 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:39:20.793 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.4%) | 20260507 04:39:20.793 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:39:20.793 | INFO | High Water Breakdown: | 20260507 04:39:20.793 | INFO | 60.2% : 8.57 GiB ( 75.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:39:20.793 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:39:20.793 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:39:20.793 | INFO | ------------------------------------------ | 20260507 04:39:20.793 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:39:20.794 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:39:20.794 | INFO | Total mem usage: 26.5 GiB | 20260507 04:39:20.794 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:39:20.794 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:39:20.794 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:39:20.794 | INFO | High Water Breakdown: | 20260507 04:39:20.794 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:39:20.794 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:39:20.794 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:39:20.794 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:39:20.794 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:39:20.794 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:39:20.794 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:39:20.794 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:39:20.794 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:39:20.794 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:39:20.794 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:39:20.794 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:39:20.794 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:39:20.794 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:39:20.794 | INFO | ------------------------------------------ | 20260507 04:39:20.794 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:39:20.794 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:39:20.794 | INFO | Total mem usage: 25.6 GiB | 20260507 04:39:20.794 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:39:20.794 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:39:20.794 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:39:20.794 | INFO | High Water Breakdown: | 20260507 04:39:20.794 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:39:20.794 | INFO | ------------------------------------------ | 20260507 04:39:20.794 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:39:20.794 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:39:20.794 | INFO | Total mem usage: 15.7 GiB | 20260507 04:39:20.794 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:39:20.794 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:39:20.794 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:39:20.794 | INFO | High Water Breakdown: | 20260507 04:39:20.794 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:39:20.794 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:39:20.794 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:39:20.794 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:39:20.794 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:39:20.794 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:39:20.794 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:39:20.794 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:39:20.794 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:39:20.794 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:39:20.794 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:39:20.794 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:39:20.794 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:39:20.794 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:39:20.794 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:39:20.794 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:39:20.794 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:39:20.794 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:39:20.794 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:39:20.794 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:39:20.794 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:39:20.794 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:39:20.794 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:39:20.794 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:39:20.794 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:39:20.794 | INFO | ------------------------------------------ | 20260507 04:39:20.794 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:39:20.794 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:39:20.794 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:39:20.794 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:39:20.794 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.32e+01 / 1.30e+00 | 20260507 04:39:20.794 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.00e-01, 9.49e-03 / 5.86e-03 | 20260507 04:39:20.794 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.46e+00, 5.24e-01 / 3.48e-01 | 20260507 04:39:20.794 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.45e+00, 1.69e-01 / 1.31e-01 | 20260507 04:39:20.794 | INFO | PerfReport: 16.13%, DME -- 3.97e+01 / 4.44e+01, 3.88e+00 / 1.65e+00 | 20260507 04:39:20.794 | INFO | PerfReport: 57.67%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.31e+00 / 1.59e+00 | 20260507 04:39:20.794 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.71e+00, 3.83e-01 / 2.34e-01 | 20260507 04:39:20.794 | INFO | PerfReport: 3.02%, Metrics -- 9.28e-01 / 1.24e+00, 2.49e-01 / 1.59e-01 | 20260507 04:39:20.794 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.12e-01 / 3.68e-01, 4.95e-02 / 2.42e-02 | 20260507 04:39:20.794 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:39:20.794 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:39:20.794 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:39:20.794 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 04:39:20.794 | INFO | PerfReport: 21.70%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260507 04:39:20.794 | INFO | PerfReport: 21.34%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260507 04:39:20.794 | INFO | PerfReport: 31.96%, Compute -- 3.07e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 04:39:20.794 | INFO | PerfReport: 1.49%, LockUpload -- 1.44e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 04:39:20.794 | INFO | PerfReport: 1.33%, LockDownload -- 1.28e+00 / 0.00e+00, 3.15e+00 / 0.00e+00 | 20260507 04:39:20.794 | INFO | PerfReport: 22.16%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 04:39:20.794 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260507 04:39:20.890 | INFO | Starting a round of Baz IO | 20260507 04:39:23.993 | INFO | Populating RTMetrics report | 20260507 04:39:23.993 | INFO | RTMetrics report populated | 20260507 04:39:23.993 | INFO | Opening tmp RTMetrics json file | 20260507 04:39:24.010 | INFO | Serializing RTMetrics to json | 20260507 04:39:24.012 | INFO | Serialization done, swapping tmp file now... | 20260507 04:39:24.012 | INFO | RTMetrics done | 20260507 04:39:24.081 | INFO | Finished a round of Baz IO: Wrote 1.72 GiB in 3.19 seconds | 20260507 04:39:24.081 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.77% / 0.00% / 67.23% | 20260507 04:39:24.081 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.164564% / 0% / 6.75519e-05% | 20260507 04:39:24.081 | INFO | Baz IO overall: | 20260507 04:39:24.081 | INFO | BytesWritten : 3649862321141 | 20260507 04:39:24.081 | INFO | NumEvents : 1751906120854 | 20260507 04:39:25.931 | INFO | Analyzing chunk frames = [7159808,7160320) | 20260507 04:39:25.931 | INFO | MemInfo: { | 20260507 04:39:25.931 | INFO | "gpu" : " 17 GB", | 20260507 04:39:25.931 | INFO | "huge" : "", | 20260507 04:39:25.931 | INFO | "huge_shared" : " 27 GB", | 20260507 04:39:25.931 | INFO | "malloc" : " 14 GB", | 20260507 04:39:25.931 | INFO | "pinned" : " 29 GB" | 20260507 04:39:25.931 | INFO | } | 20260507 04:39:25.931 | INFO | Perf report for the last 5.13749 seconds: | 20260507 04:39:25.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:39:25.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:25.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:25.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:25.931 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:39:25.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:25.931 | INFO | RT_METRICS : 42.8%, 1.0, 22.4ms, 0 | 20260507 04:39:25.931 | INFO | PRE_HQ : 59.5%, 1.8, 56.4ms, 0 | 20260507 04:39:25.931 | INFO | BAZWRITER : 61.7%, 3.2, 102.2ms, 0 | 20260507 04:39:25.931 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:39:25.931 | INFO | Uploaded 36GiB to GPU in 2.03373s (17.7015GiB/s) | 20260507 04:39:25.931 | INFO | Downloaded 38GiB to GPU in 2.17722s (17.4535GiB/s) | 20260507 04:39:31.070 | INFO | Analyzing chunk frames = [7160320,7160832) | 20260507 04:39:31.070 | INFO | MemInfo: { | 20260507 04:39:31.070 | INFO | "gpu" : " 17 GB", | 20260507 04:39:31.070 | INFO | "huge" : "", | 20260507 04:39:31.070 | INFO | "huge_shared" : " 27 GB", | 20260507 04:39:31.070 | INFO | "malloc" : " 14 GB", | 20260507 04:39:31.070 | INFO | "pinned" : " 29 GB" | 20260507 04:39:31.070 | INFO | } | 20260507 04:39:31.070 | INFO | Perf report for the last 5.13942 seconds: | 20260507 04:39:31.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:39:31.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:31.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:31.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:31.070 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:39:31.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:31.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:31.070 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:39:31.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:31.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:31.070 | INFO | Uploaded 36GiB to GPU in 1.98354s (18.1494GiB/s) | 20260507 04:39:31.070 | INFO | Downloaded 34GiB to GPU in 1.93378s (17.5821GiB/s) | 20260507 04:39:36.205 | INFO | Analyzing chunk frames = [7160832,7161344) | 20260507 04:39:36.205 | INFO | MemInfo: { | 20260507 04:39:36.205 | INFO | "gpu" : " 17 GB", | 20260507 04:39:36.205 | INFO | "huge" : "", | 20260507 04:39:36.205 | INFO | "huge_shared" : " 27 GB", | 20260507 04:39:36.205 | INFO | "malloc" : " 14 GB", | 20260507 04:39:36.205 | INFO | "pinned" : " 29 GB" | 20260507 04:39:36.205 | INFO | } | 20260507 04:39:36.205 | INFO | Perf report for the last 5.13497 seconds: | 20260507 04:39:36.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:39:36.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:36.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:36.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:36.205 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 04:39:36.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:36.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:36.205 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:39:36.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:36.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:36.205 | INFO | Uploaded 36GiB to GPU in 2.01674s (17.8506GiB/s) | 20260507 04:39:36.205 | INFO | Downloaded 34GiB to GPU in 1.95683s (17.375GiB/s) | 20260507 04:39:41.347 | INFO | Analyzing chunk frames = [7161344,7161856) | 20260507 04:39:41.347 | INFO | MemInfo: { | 20260507 04:39:41.347 | INFO | "gpu" : " 17 GB", | 20260507 04:39:41.347 | INFO | "huge" : "", | 20260507 04:39:41.348 | INFO | "huge_shared" : " 27 GB", | 20260507 04:39:41.348 | INFO | "malloc" : " 14 GB", | 20260507 04:39:41.348 | INFO | "pinned" : " 29 GB" | 20260507 04:39:41.348 | INFO | } | 20260507 04:39:41.348 | INFO | Perf report for the last 5.14257 seconds: | 20260507 04:39:41.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:39:41.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:41.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:41.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:41.348 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:39:41.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:41.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:41.348 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:39:41.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:41.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:41.348 | INFO | Uploaded 36GiB to GPU in 1.95958s (18.3712GiB/s) | 20260507 04:39:41.348 | INFO | Downloaded 34GiB to GPU in 1.92272s (17.6833GiB/s) | 20260507 04:39:46.489 | INFO | Analyzing chunk frames = [7161856,7162368) | 20260507 04:39:46.489 | INFO | MemInfo: { | 20260507 04:39:46.489 | INFO | "gpu" : " 17 GB", | 20260507 04:39:46.489 | INFO | "huge" : "", | 20260507 04:39:46.489 | INFO | "huge_shared" : " 27 GB", | 20260507 04:39:46.489 | INFO | "malloc" : " 14 GB", | 20260507 04:39:46.489 | INFO | "pinned" : " 29 GB" | 20260507 04:39:46.489 | INFO | } | 20260507 04:39:46.489 | INFO | Perf report for the last 5.14122 seconds: | 20260507 04:39:46.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:39:46.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:46.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:46.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:46.489 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 04:39:46.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:46.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:46.489 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:39:46.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:46.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:46.489 | INFO | Uploaded 36GiB to GPU in 2.01102s (17.9013GiB/s) | 20260507 04:39:46.489 | INFO | Downloaded 34GiB to GPU in 1.94017s (17.5242GiB/s) | 20260507 04:39:51.631 | INFO | Analyzing chunk frames = [7162368,7162880) | 20260507 04:39:51.631 | INFO | MemInfo: { | 20260507 04:39:51.631 | INFO | "gpu" : " 17 GB", | 20260507 04:39:51.631 | INFO | "huge" : "", | 20260507 04:39:51.631 | INFO | "huge_shared" : " 27 GB", | 20260507 04:39:51.631 | INFO | "malloc" : " 14 GB", | 20260507 04:39:51.631 | INFO | "pinned" : " 29 GB" | 20260507 04:39:51.631 | INFO | } | 20260507 04:39:51.631 | INFO | Perf report for the last 5.14194 seconds: | 20260507 04:39:51.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:39:51.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:51.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:51.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:51.631 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:39:51.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:51.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:51.631 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 04:39:51.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:51.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:51.631 | INFO | Uploaded 36GiB to GPU in 2.04983s (17.5624GiB/s) | 20260507 04:39:51.631 | INFO | Downloaded 34GiB to GPU in 1.96663s (17.2884GiB/s) | 20260507 04:39:56.769 | INFO | Analyzing chunk frames = [7162880,7163392) | 20260507 04:39:56.769 | INFO | MemInfo: { | 20260507 04:39:56.770 | INFO | "gpu" : " 17 GB", | 20260507 04:39:56.770 | INFO | "huge" : "", | 20260507 04:39:56.770 | INFO | "huge_shared" : " 27 GB", | 20260507 04:39:56.770 | INFO | "malloc" : " 14 GB", | 20260507 04:39:56.770 | INFO | "pinned" : " 29 GB" | 20260507 04:39:56.770 | INFO | } | 20260507 04:39:56.770 | INFO | Perf report for the last 5.13883 seconds: | 20260507 04:39:56.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:39:56.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:56.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:56.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:56.770 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:39:56.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:56.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:56.770 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:39:56.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:56.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:39:56.770 | INFO | Uploaded 36GiB to GPU in 1.99999s (18.0001GiB/s) | 20260507 04:39:56.770 | INFO | Downloaded 34GiB to GPU in 1.9742s (17.2222GiB/s) | 20260507 04:40:01.900 | INFO | Analyzing chunk frames = [7163392,7163904) | 20260507 04:40:01.900 | INFO | MemInfo: { | 20260507 04:40:01.900 | INFO | "gpu" : " 17 GB", | 20260507 04:40:01.900 | INFO | "huge" : "", | 20260507 04:40:01.900 | INFO | "huge_shared" : " 27 GB", | 20260507 04:40:01.900 | INFO | "malloc" : " 14 GB", | 20260507 04:40:01.900 | INFO | "pinned" : " 29 GB" | 20260507 04:40:01.900 | INFO | } | 20260507 04:40:01.900 | INFO | Perf report for the last 5.13051 seconds: | 20260507 04:40:01.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:40:01.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:01.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:01.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:01.900 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 04:40:01.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:01.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:01.900 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 04:40:01.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:01.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:01.900 | INFO | Uploaded 36GiB to GPU in 2.00265s (17.9762GiB/s) | 20260507 04:40:01.900 | INFO | Downloaded 34GiB to GPU in 1.97052s (17.2543GiB/s) | 20260507 04:40:01.997 | INFO | Starting a round of Baz IO | 20260507 04:40:05.105 | INFO | Populating RTMetrics report | 20260507 04:40:05.105 | INFO | RTMetrics report populated | 20260507 04:40:05.105 | INFO | Opening tmp RTMetrics json file | 20260507 04:40:05.109 | INFO | Serializing RTMetrics to json | 20260507 04:40:05.123 | INFO | Serialization done, swapping tmp file now... | 20260507 04:40:05.124 | INFO | RTMetrics done | 20260507 04:40:05.191 | INFO | Finished a round of Baz IO: Wrote 1.718 GiB in 3.193 seconds | 20260507 04:40:05.191 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.77% / 0.00% / 67.23% | 20260507 04:40:05.191 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.163523% / 0% / 5.5631e-05% | 20260507 04:40:05.191 | INFO | Baz IO overall: | 20260507 04:40:05.191 | INFO | BytesWritten : 3651707110053 | 20260507 04:40:05.191 | INFO | NumEvents : 1752774099303 | 20260507 04:40:07.041 | INFO | Analyzing chunk frames = [7163904,7164416) | 20260507 04:40:07.041 | INFO | MemInfo: { | 20260507 04:40:07.041 | INFO | "gpu" : " 17 GB", | 20260507 04:40:07.042 | INFO | "huge" : "", | 20260507 04:40:07.042 | INFO | "huge_shared" : " 27 GB", | 20260507 04:40:07.042 | INFO | "malloc" : " 14 GB", | 20260507 04:40:07.042 | INFO | "pinned" : " 29 GB" | 20260507 04:40:07.042 | INFO | } | 20260507 04:40:07.042 | INFO | Perf report for the last 5.1414 seconds: | 20260507 04:40:07.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:40:07.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:07.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:07.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:07.042 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 04:40:07.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:07.042 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 04:40:07.042 | INFO | PRE_HQ : 59.1%, 1.8, 55.1ms, 0 | 20260507 04:40:07.042 | INFO | BAZWRITER : 61.7%, 3.2, 101.1ms, 0 | 20260507 04:40:07.042 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 04:40:07.042 | INFO | Uploaded 36GiB to GPU in 2.04636s (17.5922GiB/s) | 20260507 04:40:07.042 | INFO | Downloaded 38GiB to GPU in 2.17964s (17.434GiB/s) | 20260507 04:40:12.190 | INFO | Analyzing chunk frames = [7164416,7164928) | 20260507 04:40:12.190 | INFO | MemInfo: { | 20260507 04:40:12.190 | INFO | "gpu" : " 17 GB", | 20260507 04:40:12.190 | INFO | "huge" : "", | 20260507 04:40:12.190 | INFO | "huge_shared" : " 27 GB", | 20260507 04:40:12.190 | INFO | "malloc" : " 14 GB", | 20260507 04:40:12.190 | INFO | "pinned" : " 29 GB" | 20260507 04:40:12.190 | INFO | } | 20260507 04:40:12.190 | INFO | Perf report for the last 5.14871 seconds: | 20260507 04:40:12.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:40:12.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:12.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:12.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:12.190 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 04:40:12.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:12.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:12.190 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:40:12.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:12.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:12.191 | INFO | Uploaded 36GiB to GPU in 2.03957s (17.6507GiB/s) | 20260507 04:40:12.191 | INFO | Downloaded 34GiB to GPU in 1.94035s (17.5227GiB/s) | 20260507 04:40:17.325 | INFO | Analyzing chunk frames = [7164928,7165440) | 20260507 04:40:17.325 | INFO | MemInfo: { | 20260507 04:40:17.325 | INFO | "gpu" : " 17 GB", | 20260507 04:40:17.325 | INFO | "huge" : "", | 20260507 04:40:17.325 | INFO | "huge_shared" : " 27 GB", | 20260507 04:40:17.325 | INFO | "malloc" : " 14 GB", | 20260507 04:40:17.325 | INFO | "pinned" : " 29 GB" | 20260507 04:40:17.325 | INFO | } | 20260507 04:40:17.325 | INFO | Perf report for the last 5.13481 seconds: | 20260507 04:40:17.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:40:17.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:17.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:17.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:17.325 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 04:40:17.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:17.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:17.325 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:40:17.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:17.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:17.325 | INFO | Uploaded 36GiB to GPU in 2.05013s (17.5599GiB/s) | 20260507 04:40:17.325 | INFO | Downloaded 34GiB to GPU in 1.9484s (17.4502GiB/s) | 20260507 04:40:22.464 | INFO | Analyzing chunk frames = [7165440,7165952) | 20260507 04:40:22.464 | INFO | MemInfo: { | 20260507 04:40:22.464 | INFO | "gpu" : " 17 GB", | 20260507 04:40:22.464 | INFO | "huge" : "", | 20260507 04:40:22.464 | INFO | "huge_shared" : " 27 GB", | 20260507 04:40:22.464 | INFO | "malloc" : " 14 GB", | 20260507 04:40:22.464 | INFO | "pinned" : " 29 GB" | 20260507 04:40:22.464 | INFO | } | 20260507 04:40:22.464 | INFO | Perf report for the last 5.13896 seconds: | 20260507 04:40:22.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:40:22.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:22.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:22.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:22.464 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 04:40:22.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:22.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:22.464 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:40:22.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:22.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:22.464 | INFO | Uploaded 36GiB to GPU in 2.01349s (17.8794GiB/s) | 20260507 04:40:22.464 | INFO | Downloaded 34GiB to GPU in 1.93545s (17.567GiB/s) | 20260507 04:40:27.608 | INFO | Analyzing chunk frames = [7165952,7166464) | 20260507 04:40:27.608 | INFO | MemInfo: { | 20260507 04:40:27.608 | INFO | "gpu" : " 17 GB", | 20260507 04:40:27.608 | INFO | "huge" : "", | 20260507 04:40:27.608 | INFO | "huge_shared" : " 27 GB", | 20260507 04:40:27.608 | INFO | "malloc" : " 14 GB", | 20260507 04:40:27.608 | INFO | "pinned" : " 29 GB" | 20260507 04:40:27.608 | INFO | } | 20260507 04:40:27.608 | INFO | Perf report for the last 5.1438 seconds: | 20260507 04:40:27.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:40:27.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:27.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:27.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:27.608 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 04:40:27.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:27.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:27.608 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:40:27.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:27.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:27.608 | INFO | Uploaded 36GiB to GPU in 2.03817s (17.6629GiB/s) | 20260507 04:40:27.608 | INFO | Downloaded 34GiB to GPU in 1.93969s (17.5285GiB/s) | 20260507 04:40:32.747 | INFO | Analyzing chunk frames = [7166464,7166976) | 20260507 04:40:32.747 | INFO | MemInfo: { | 20260507 04:40:32.747 | INFO | "gpu" : " 17 GB", | 20260507 04:40:32.747 | INFO | "huge" : "", | 20260507 04:40:32.747 | INFO | "huge_shared" : " 27 GB", | 20260507 04:40:32.747 | INFO | "malloc" : " 14 GB", | 20260507 04:40:32.747 | INFO | "pinned" : " 29 GB" | 20260507 04:40:32.747 | INFO | } | 20260507 04:40:32.747 | INFO | Perf report for the last 5.13944 seconds: | 20260507 04:40:32.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:40:32.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:32.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:32.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:32.747 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 04:40:32.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:32.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:32.747 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 04:40:32.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:32.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:32.748 | INFO | Uploaded 36GiB to GPU in 2.04349s (17.617GiB/s) | 20260507 04:40:32.748 | INFO | Downloaded 34GiB to GPU in 1.91242s (17.7785GiB/s) | 20260507 04:40:37.880 | INFO | Analyzing chunk frames = [7166976,7167488) | 20260507 04:40:37.880 | INFO | MemInfo: { | 20260507 04:40:37.880 | INFO | "gpu" : " 17 GB", | 20260507 04:40:37.880 | INFO | "huge" : "", | 20260507 04:40:37.880 | INFO | "huge_shared" : " 27 GB", | 20260507 04:40:37.880 | INFO | "malloc" : " 14 GB", | 20260507 04:40:37.880 | INFO | "pinned" : " 29 GB" | 20260507 04:40:37.880 | INFO | } | 20260507 04:40:37.880 | INFO | Perf report for the last 5.13297 seconds: | 20260507 04:40:37.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:40:37.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:37.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:37.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:37.880 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 04:40:37.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:37.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:37.881 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:40:37.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:37.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:37.881 | INFO | Uploaded 36GiB to GPU in 2.00563s (17.9494GiB/s) | 20260507 04:40:37.881 | INFO | Downloaded 34GiB to GPU in 1.97542s (17.2116GiB/s) | 20260507 04:40:43.021 | INFO | Analyzing chunk frames = [7167488,7168000) | 20260507 04:40:43.021 | INFO | MemInfo: { | 20260507 04:40:43.021 | INFO | "gpu" : " 17 GB", | 20260507 04:40:43.021 | INFO | "huge" : "", | 20260507 04:40:43.021 | INFO | "huge_shared" : " 27 GB", | 20260507 04:40:43.021 | INFO | "malloc" : " 14 GB", | 20260507 04:40:43.021 | INFO | "pinned" : " 29 GB" | 20260507 04:40:43.021 | INFO | } | 20260507 04:40:43.021 | INFO | Perf report for the last 5.14099 seconds: | 20260507 04:40:43.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:40:43.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:43.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:43.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:43.021 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:40:43.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:43.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:43.021 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 04:40:43.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:43.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:43.022 | INFO | Uploaded 36GiB to GPU in 1.98683s (18.1193GiB/s) | 20260507 04:40:43.022 | INFO | Downloaded 34GiB to GPU in 1.95948s (17.3515GiB/s) | 20260507 04:40:43.022 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:40:43.022 | INFO | Total mem usage: 18 GiB | 20260507 04:40:43.022 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:40:43.022 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 04:40:43.022 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:40:43.022 | INFO | High Water Breakdown: | 20260507 04:40:43.022 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:40:43.022 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:40:43.022 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:40:43.022 | INFO | ------------------------------------------ | 20260507 04:40:43.022 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:40:43.022 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:40:43.022 | INFO | Total mem usage: 26.5 GiB | 20260507 04:40:43.022 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:40:43.022 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:40:43.022 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 04:40:43.022 | INFO | High Water Breakdown: | 20260507 04:40:43.022 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:40:43.022 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:40:43.022 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:40:43.022 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:40:43.022 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:40:43.022 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:40:43.022 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:40:43.022 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:40:43.022 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:40:43.022 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:40:43.022 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:40:43.022 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:40:43.022 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:40:43.022 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:40:43.022 | INFO | ------------------------------------------ | 20260507 04:40:43.022 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:40:43.022 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:40:43.022 | INFO | Total mem usage: 25.6 GiB | 20260507 04:40:43.022 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:40:43.022 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:40:43.022 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:40:43.022 | INFO | High Water Breakdown: | 20260507 04:40:43.022 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:40:43.022 | INFO | ------------------------------------------ | 20260507 04:40:43.022 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:40:43.022 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:40:43.022 | INFO | Total mem usage: 15.7 GiB | 20260507 04:40:43.022 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:40:43.022 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:40:43.022 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:40:43.022 | INFO | High Water Breakdown: | 20260507 04:40:43.022 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:40:43.022 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:40:43.022 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:40:43.022 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:40:43.022 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:40:43.022 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:40:43.022 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:40:43.022 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:40:43.022 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:40:43.022 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:40:43.022 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:40:43.022 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:40:43.022 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:40:43.022 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:40:43.022 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:40:43.022 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:40:43.022 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:40:43.022 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:40:43.022 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:40:43.022 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:40:43.022 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:40:43.022 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:40:43.022 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:40:43.022 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:40:43.022 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:40:43.022 | INFO | ------------------------------------------ | 20260507 04:40:43.022 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:40:43.022 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:40:43.022 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:40:43.022 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:40:43.022 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.30e+01 / 1.46e+00 | 20260507 04:40:43.022 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.00e-01, 1.03e-02 / 5.59e-03 | 20260507 04:40:43.022 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.43e+00, 5.09e-01 / 3.51e-01 | 20260507 04:40:43.022 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.47e+00, 1.68e-01 / 1.31e-01 | 20260507 04:40:43.022 | INFO | PerfReport: 16.18%, DME -- 3.98e+01 / 4.44e+01, 3.90e+00 / 1.94e+00 | 20260507 04:40:43.022 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.27e+00 / 1.40e+00 | 20260507 04:40:43.022 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.66e+00, 3.83e-01 / 2.15e-01 | 20260507 04:40:43.022 | INFO | PerfReport: 3.01%, Metrics -- 9.25e-01 / 1.25e+00, 2.46e-01 / 1.42e-01 | 20260507 04:40:43.023 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.13e-01 / 3.72e-01, 5.05e-02 / 2.77e-02 | 20260507 04:40:43.023 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:40:43.023 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:40:43.023 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:40:43.023 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 04:40:43.023 | INFO | PerfReport: 21.83%, Upload -- 2.10e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260507 04:40:43.023 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260507 04:40:43.023 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.30e+01 / 0.00e+00 | 20260507 04:40:43.023 | INFO | PerfReport: 1.50%, LockUpload -- 1.44e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 04:40:43.023 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260507 04:40:43.023 | INFO | PerfReport: 22.05%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 04:40:43.023 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260507 04:40:43.118 | INFO | Starting a round of Baz IO | 20260507 04:40:46.216 | INFO | Populating RTMetrics report | 20260507 04:40:46.216 | INFO | RTMetrics report populated | 20260507 04:40:46.216 | INFO | Opening tmp RTMetrics json file | 20260507 04:40:46.220 | INFO | Serializing RTMetrics to json | 20260507 04:40:46.235 | INFO | Serialization done, swapping tmp file now... | 20260507 04:40:46.236 | INFO | RTMetrics done | 20260507 04:40:46.304 | INFO | Finished a round of Baz IO: Wrote 1.718 GiB in 3.186 seconds | 20260507 04:40:46.304 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.77% / 0.00% / 67.23% | 20260507 04:40:46.304 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.163468% / 0% / 7.94729e-05% | 20260507 04:40:46.304 | INFO | Baz IO overall: | 20260507 04:40:46.304 | INFO | BytesWritten : 3653552104525 | 20260507 04:40:46.304 | INFO | NumEvents : 1753642180053 | 20260507 04:40:48.159 | INFO | Analyzing chunk frames = [7168000,7168512) | 20260507 04:40:48.159 | INFO | MemInfo: { | 20260507 04:40:48.159 | INFO | "gpu" : " 17 GB", | 20260507 04:40:48.159 | INFO | "huge" : "", | 20260507 04:40:48.159 | INFO | "huge_shared" : " 27 GB", | 20260507 04:40:48.159 | INFO | "malloc" : " 14 GB", | 20260507 04:40:48.159 | INFO | "pinned" : " 29 GB" | 20260507 04:40:48.159 | INFO | } | 20260507 04:40:48.159 | INFO | Perf report for the last 5.13776 seconds: | 20260507 04:40:48.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:40:48.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:48.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:48.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:48.159 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 04:40:48.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:48.159 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 04:40:48.159 | INFO | PRE_HQ : 59.2%, 1.8, 55.6ms, 0 | 20260507 04:40:48.159 | INFO | BAZWRITER : 61.5%, 3.2, 100.8ms, 0 | 20260507 04:40:48.159 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:40:48.160 | INFO | Uploaded 36GiB to GPU in 2.05569s (17.5124GiB/s) | 20260507 04:40:48.160 | INFO | Downloaded 38GiB to GPU in 2.17892s (17.4398GiB/s) | 20260507 04:40:53.299 | INFO | Analyzing chunk frames = [7168512,7169024) | 20260507 04:40:53.299 | INFO | MemInfo: { | 20260507 04:40:53.299 | INFO | "gpu" : " 17 GB", | 20260507 04:40:53.299 | INFO | "huge" : "", | 20260507 04:40:53.299 | INFO | "huge_shared" : " 27 GB", | 20260507 04:40:53.299 | INFO | "malloc" : " 14 GB", | 20260507 04:40:53.299 | INFO | "pinned" : " 29 GB" | 20260507 04:40:53.299 | INFO | } | 20260507 04:40:53.299 | INFO | Perf report for the last 5.13957 seconds: | 20260507 04:40:53.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:40:53.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:53.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:53.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:53.299 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:40:53.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:53.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:53.299 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:40:53.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:53.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:53.299 | INFO | Uploaded 36GiB to GPU in 2.04919s (17.568GiB/s) | 20260507 04:40:53.299 | INFO | Downloaded 34GiB to GPU in 1.93847s (17.5396GiB/s) | 20260507 04:40:58.442 | INFO | Analyzing chunk frames = [7169024,7169536) | 20260507 04:40:58.442 | INFO | MemInfo: { | 20260507 04:40:58.442 | INFO | "gpu" : " 17 GB", | 20260507 04:40:58.442 | INFO | "huge" : "", | 20260507 04:40:58.442 | INFO | "huge_shared" : " 27 GB", | 20260507 04:40:58.442 | INFO | "malloc" : " 14 GB", | 20260507 04:40:58.442 | INFO | "pinned" : " 29 GB" | 20260507 04:40:58.442 | INFO | } | 20260507 04:40:58.442 | INFO | Perf report for the last 5.14364 seconds: | 20260507 04:40:58.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:40:58.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:58.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:58.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:58.442 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 04:40:58.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:58.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:58.443 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 04:40:58.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:58.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:40:58.443 | INFO | Uploaded 36GiB to GPU in 2.01063s (17.9049GiB/s) | 20260507 04:40:58.443 | INFO | Downloaded 34GiB to GPU in 1.93929s (17.5322GiB/s) | 20260507 04:41:03.584 | INFO | Analyzing chunk frames = [7169536,7170048) | 20260507 04:41:03.584 | INFO | MemInfo: { | 20260507 04:41:03.584 | INFO | "gpu" : " 17 GB", | 20260507 04:41:03.584 | INFO | "huge" : "", | 20260507 04:41:03.584 | INFO | "huge_shared" : " 27 GB", | 20260507 04:41:03.584 | INFO | "malloc" : " 14 GB", | 20260507 04:41:03.584 | INFO | "pinned" : " 29 GB" | 20260507 04:41:03.584 | INFO | } | 20260507 04:41:03.584 | INFO | Perf report for the last 5.14143 seconds: | 20260507 04:41:03.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:41:03.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:03.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:03.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:03.584 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 04:41:03.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:03.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:03.584 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:41:03.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:03.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:03.584 | INFO | Uploaded 36GiB to GPU in 2.02116s (17.8116GiB/s) | 20260507 04:41:03.584 | INFO | Downloaded 34GiB to GPU in 1.93758s (17.5476GiB/s) | 20260507 04:41:08.716 | INFO | Analyzing chunk frames = [7170048,7170560) | 20260507 04:41:08.716 | INFO | MemInfo: { | 20260507 04:41:08.716 | INFO | "gpu" : " 17 GB", | 20260507 04:41:08.716 | INFO | "huge" : "", | 20260507 04:41:08.716 | INFO | "huge_shared" : " 27 GB", | 20260507 04:41:08.716 | INFO | "malloc" : " 14 GB", | 20260507 04:41:08.716 | INFO | "pinned" : " 29 GB" | 20260507 04:41:08.716 | INFO | } | 20260507 04:41:08.716 | INFO | Perf report for the last 5.13225 seconds: | 20260507 04:41:08.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:41:08.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:08.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:08.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:08.716 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260507 04:41:08.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:08.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:08.716 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 04:41:08.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:08.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:08.716 | INFO | Uploaded 36GiB to GPU in 2.03324s (17.7057GiB/s) | 20260507 04:41:08.716 | INFO | Downloaded 34GiB to GPU in 1.91786s (17.7281GiB/s) | 20260507 04:41:13.855 | INFO | Analyzing chunk frames = [7170560,7171072) | 20260507 04:41:13.855 | INFO | MemInfo: { | 20260507 04:41:13.855 | INFO | "gpu" : " 17 GB", | 20260507 04:41:13.855 | INFO | "huge" : "", | 20260507 04:41:13.855 | INFO | "huge_shared" : " 27 GB", | 20260507 04:41:13.855 | INFO | "malloc" : " 14 GB", | 20260507 04:41:13.855 | INFO | "pinned" : " 29 GB" | 20260507 04:41:13.855 | INFO | } | 20260507 04:41:13.855 | INFO | Perf report for the last 5.1389 seconds: | 20260507 04:41:13.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:41:13.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:13.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:13.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:13.855 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 04:41:13.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:13.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:13.855 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:41:13.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:13.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:13.855 | INFO | Uploaded 36GiB to GPU in 2.03491s (17.6912GiB/s) | 20260507 04:41:13.855 | INFO | Downloaded 34GiB to GPU in 1.93532s (17.5681GiB/s) | 20260507 04:41:18.995 | INFO | Analyzing chunk frames = [7171072,7171584) | 20260507 04:41:18.995 | INFO | MemInfo: { | 20260507 04:41:18.995 | INFO | "gpu" : " 17 GB", | 20260507 04:41:18.995 | INFO | "huge" : "", | 20260507 04:41:18.996 | INFO | "huge_shared" : " 27 GB", | 20260507 04:41:18.996 | INFO | "malloc" : " 14 GB", | 20260507 04:41:18.996 | INFO | "pinned" : " 29 GB" | 20260507 04:41:18.996 | INFO | } | 20260507 04:41:18.996 | INFO | Perf report for the last 5.14058 seconds: | 20260507 04:41:18.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:41:18.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:18.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:18.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:18.996 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:41:18.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:18.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:18.996 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 04:41:18.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:18.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:18.996 | INFO | Uploaded 36GiB to GPU in 1.98104s (18.1723GiB/s) | 20260507 04:41:18.996 | INFO | Downloaded 34GiB to GPU in 1.96502s (17.3026GiB/s) | 20260507 04:41:24.143 | INFO | Analyzing chunk frames = [7171584,7172096) | 20260507 04:41:24.143 | INFO | MemInfo: { | 20260507 04:41:24.143 | INFO | "gpu" : " 17 GB", | 20260507 04:41:24.143 | INFO | "huge" : "", | 20260507 04:41:24.143 | INFO | "huge_shared" : " 27 GB", | 20260507 04:41:24.143 | INFO | "malloc" : " 14 GB", | 20260507 04:41:24.143 | INFO | "pinned" : " 29 GB" | 20260507 04:41:24.143 | INFO | } | 20260507 04:41:24.143 | INFO | Perf report for the last 5.14778 seconds: | 20260507 04:41:24.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:41:24.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:24.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:24.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:24.143 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 04:41:24.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:24.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:24.143 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 04:41:24.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:24.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:24.144 | INFO | Uploaded 36GiB to GPU in 1.97705s (18.2089GiB/s) | 20260507 04:41:24.144 | INFO | Downloaded 34GiB to GPU in 1.97368s (17.2267GiB/s) | 20260507 04:41:24.241 | INFO | Starting a round of Baz IO | 20260507 04:41:27.340 | INFO | Populating RTMetrics report | 20260507 04:41:27.340 | INFO | RTMetrics report populated | 20260507 04:41:27.340 | INFO | Opening tmp RTMetrics json file | 20260507 04:41:27.344 | INFO | Serializing RTMetrics to json | 20260507 04:41:27.358 | INFO | Serialization done, swapping tmp file now... | 20260507 04:41:27.359 | INFO | RTMetrics done | 20260507 04:41:27.427 | INFO | Finished a round of Baz IO: Wrote 1.717 GiB in 3.186 seconds | 20260507 04:41:27.427 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.77% / 0.00% / 67.23% | 20260507 04:41:27.427 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.162983% / 3.97364e-06% / 8.34465e-05% | 20260507 04:41:27.427 | INFO | Baz IO overall: | 20260507 04:41:27.427 | INFO | BytesWritten : 3655396244415 | 20260507 04:41:27.427 | INFO | NumEvents : 1754509668343 | 20260507 04:41:29.277 | INFO | Analyzing chunk frames = [7172096,7172608) | 20260507 04:41:29.277 | INFO | MemInfo: { | 20260507 04:41:29.277 | INFO | "gpu" : " 17 GB", | 20260507 04:41:29.277 | INFO | "huge" : "", | 20260507 04:41:29.277 | INFO | "huge_shared" : " 27 GB", | 20260507 04:41:29.277 | INFO | "malloc" : " 14 GB", | 20260507 04:41:29.277 | INFO | "pinned" : " 29 GB" | 20260507 04:41:29.277 | INFO | } | 20260507 04:41:29.277 | INFO | Perf report for the last 5.13365 seconds: | 20260507 04:41:29.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:41:29.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:29.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:29.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:29.277 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:41:29.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:29.277 | INFO | RT_METRICS : 43.2%, 1.0, 22.6ms, 0 | 20260507 04:41:29.277 | INFO | PRE_HQ : 59.8%, 1.9, 57.6ms, 0 | 20260507 04:41:29.277 | INFO | BAZWRITER : 61.7%, 3.3, 103.5ms, 0 | 20260507 04:41:29.277 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 04:41:29.278 | INFO | Uploaded 36GiB to GPU in 2.06325s (17.4482GiB/s) | 20260507 04:41:29.278 | INFO | Downloaded 38GiB to GPU in 2.17748s (17.4513GiB/s) | 20260507 04:41:34.413 | INFO | Analyzing chunk frames = [7172608,7173120) | 20260507 04:41:34.413 | INFO | MemInfo: { | 20260507 04:41:34.413 | INFO | "gpu" : " 17 GB", | 20260507 04:41:34.413 | INFO | "huge" : "", | 20260507 04:41:34.413 | INFO | "huge_shared" : " 27 GB", | 20260507 04:41:34.413 | INFO | "malloc" : " 14 GB", | 20260507 04:41:34.413 | INFO | "pinned" : " 29 GB" | 20260507 04:41:34.413 | INFO | } | 20260507 04:41:34.413 | INFO | Perf report for the last 5.13569 seconds: | 20260507 04:41:34.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:41:34.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:34.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:34.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:34.413 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 04:41:34.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:34.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:34.413 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:41:34.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:34.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:34.413 | INFO | Uploaded 36GiB to GPU in 1.97124s (18.2626GiB/s) | 20260507 04:41:34.413 | INFO | Downloaded 34GiB to GPU in 1.92332s (17.6778GiB/s) | 20260507 04:41:39.555 | INFO | Analyzing chunk frames = [7173120,7173632) | 20260507 04:41:39.555 | INFO | MemInfo: { | 20260507 04:41:39.555 | INFO | "gpu" : " 17 GB", | 20260507 04:41:39.555 | INFO | "huge" : "", | 20260507 04:41:39.555 | INFO | "huge_shared" : " 27 GB", | 20260507 04:41:39.555 | INFO | "malloc" : " 14 GB", | 20260507 04:41:39.555 | INFO | "pinned" : " 29 GB" | 20260507 04:41:39.555 | INFO | } | 20260507 04:41:39.555 | INFO | Perf report for the last 5.14226 seconds: | 20260507 04:41:39.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:41:39.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:39.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:39.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:39.555 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 04:41:39.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:39.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:39.555 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:41:39.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:39.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:39.555 | INFO | Uploaded 36GiB to GPU in 1.98754s (18.1128GiB/s) | 20260507 04:41:39.555 | INFO | Downloaded 34GiB to GPU in 1.94273s (17.5011GiB/s) | 20260507 04:41:44.693 | INFO | Analyzing chunk frames = [7173632,7174144) | 20260507 04:41:44.693 | INFO | MemInfo: { | 20260507 04:41:44.693 | INFO | "gpu" : " 17 GB", | 20260507 04:41:44.693 | INFO | "huge" : "", | 20260507 04:41:44.693 | INFO | "huge_shared" : " 27 GB", | 20260507 04:41:44.693 | INFO | "malloc" : " 14 GB", | 20260507 04:41:44.693 | INFO | "pinned" : " 29 GB" | 20260507 04:41:44.693 | INFO | } | 20260507 04:41:44.693 | INFO | Perf report for the last 5.13835 seconds: | 20260507 04:41:44.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:41:44.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:44.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:44.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:44.693 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:41:44.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:44.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:44.693 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:41:44.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:44.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:44.694 | INFO | Uploaded 36GiB to GPU in 1.99666s (18.0301GiB/s) | 20260507 04:41:44.694 | INFO | Downloaded 34GiB to GPU in 1.93581s (17.5637GiB/s) | 20260507 04:41:49.831 | INFO | Analyzing chunk frames = [7174144,7174656) | 20260507 04:41:49.831 | INFO | MemInfo: { | 20260507 04:41:49.831 | INFO | "gpu" : " 17 GB", | 20260507 04:41:49.831 | INFO | "huge" : "", | 20260507 04:41:49.831 | INFO | "huge_shared" : " 27 GB", | 20260507 04:41:49.831 | INFO | "malloc" : " 14 GB", | 20260507 04:41:49.831 | INFO | "pinned" : " 29 GB" | 20260507 04:41:49.831 | INFO | } | 20260507 04:41:49.831 | INFO | Perf report for the last 5.13733 seconds: | 20260507 04:41:49.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:41:49.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:49.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:49.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:49.831 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 04:41:49.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:49.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:49.831 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:41:49.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:49.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:49.831 | INFO | Uploaded 36GiB to GPU in 2.02271s (17.7979GiB/s) | 20260507 04:41:49.831 | INFO | Downloaded 34GiB to GPU in 1.94616s (17.4703GiB/s) | 20260507 04:41:54.972 | INFO | Analyzing chunk frames = [7174656,7175168) | 20260507 04:41:54.972 | INFO | MemInfo: { | 20260507 04:41:54.972 | INFO | "gpu" : " 17 GB", | 20260507 04:41:54.972 | INFO | "huge" : "", | 20260507 04:41:54.972 | INFO | "huge_shared" : " 27 GB", | 20260507 04:41:54.972 | INFO | "malloc" : " 14 GB", | 20260507 04:41:54.972 | INFO | "pinned" : " 29 GB" | 20260507 04:41:54.972 | INFO | } | 20260507 04:41:54.972 | INFO | Perf report for the last 5.14164 seconds: | 20260507 04:41:54.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:41:54.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:54.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:54.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:54.972 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:41:54.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:54.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:54.972 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:41:54.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:54.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:41:54.973 | INFO | Uploaded 36GiB to GPU in 2.049s (17.5696GiB/s) | 20260507 04:41:54.973 | INFO | Downloaded 34GiB to GPU in 1.97068s (17.2529GiB/s) | 20260507 04:42:00.111 | INFO | Analyzing chunk frames = [7175168,7175680) | 20260507 04:42:00.111 | INFO | MemInfo: { | 20260507 04:42:00.111 | INFO | "gpu" : " 17 GB", | 20260507 04:42:00.111 | INFO | "huge" : "", | 20260507 04:42:00.111 | INFO | "huge_shared" : " 27 GB", | 20260507 04:42:00.111 | INFO | "malloc" : " 14 GB", | 20260507 04:42:00.111 | INFO | "pinned" : " 29 GB" | 20260507 04:42:00.111 | INFO | } | 20260507 04:42:00.111 | INFO | Perf report for the last 5.13894 seconds: | 20260507 04:42:00.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:42:00.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:00.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:00.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:00.111 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 04:42:00.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:00.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:00.111 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 04:42:00.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:00.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:00.112 | INFO | Uploaded 36GiB to GPU in 1.97499s (18.228GiB/s) | 20260507 04:42:00.112 | INFO | Downloaded 34GiB to GPU in 1.96417s (17.3101GiB/s) | 20260507 04:42:05.251 | INFO | Analyzing chunk frames = [7175680,7176192) | 20260507 04:42:05.251 | INFO | MemInfo: { | 20260507 04:42:05.251 | INFO | "gpu" : " 17 GB", | 20260507 04:42:05.251 | INFO | "huge" : "", | 20260507 04:42:05.251 | INFO | "huge_shared" : " 27 GB", | 20260507 04:42:05.251 | INFO | "malloc" : " 14 GB", | 20260507 04:42:05.251 | INFO | "pinned" : " 29 GB" | 20260507 04:42:05.251 | INFO | } | 20260507 04:42:05.251 | INFO | Perf report for the last 5.13934 seconds: | 20260507 04:42:05.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:42:05.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:05.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:05.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:05.251 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:42:05.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:05.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:05.251 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 04:42:05.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:05.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:05.251 | INFO | Uploaded 36GiB to GPU in 1.97648s (18.2142GiB/s) | 20260507 04:42:05.251 | INFO | Downloaded 34GiB to GPU in 1.96369s (17.3143GiB/s) | 20260507 04:42:05.251 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:42:05.251 | INFO | Total mem usage: 18 GiB | 20260507 04:42:05.251 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:42:05.251 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 04:42:05.251 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:42:05.251 | INFO | High Water Breakdown: | 20260507 04:42:05.251 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:42:05.251 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:42:05.251 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:42:05.251 | INFO | ------------------------------------------ | 20260507 04:42:05.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:42:05.251 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:42:05.251 | INFO | Total mem usage: 26.5 GiB | 20260507 04:42:05.251 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:42:05.251 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:42:05.251 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:42:05.251 | INFO | High Water Breakdown: | 20260507 04:42:05.251 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:42:05.251 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:42:05.251 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:42:05.251 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:42:05.251 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:42:05.251 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:42:05.251 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:42:05.251 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:42:05.251 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:42:05.251 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:42:05.251 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:42:05.251 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:42:05.251 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:42:05.251 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:42:05.251 | INFO | ------------------------------------------ | 20260507 04:42:05.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:42:05.251 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:42:05.251 | INFO | Total mem usage: 25.6 GiB | 20260507 04:42:05.251 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:42:05.251 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:42:05.251 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:42:05.251 | INFO | High Water Breakdown: | 20260507 04:42:05.251 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:42:05.251 | INFO | ------------------------------------------ | 20260507 04:42:05.251 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:42:05.251 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:42:05.251 | INFO | Total mem usage: 15.7 GiB | 20260507 04:42:05.251 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:42:05.251 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:42:05.251 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:42:05.252 | INFO | High Water Breakdown: | 20260507 04:42:05.252 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:42:05.252 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:42:05.252 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:42:05.252 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:42:05.252 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:42:05.252 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:42:05.252 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:42:05.252 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:42:05.252 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:42:05.252 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:42:05.252 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:42:05.252 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:42:05.252 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:42:05.252 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:42:05.252 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:42:05.252 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:42:05.252 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:42:05.252 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:42:05.252 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:42:05.252 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:42:05.252 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:42:05.252 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:42:05.252 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:42:05.252 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:42:05.252 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:42:05.252 | INFO | ------------------------------------------ | 20260507 04:42:05.252 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:42:05.252 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:42:05.252 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:42:05.252 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:42:05.252 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.51e+01, 1.32e+01 / 1.06e+00 | 20260507 04:42:05.252 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.03e-01, 1.03e-02 / 1.20e-02 | 20260507 04:42:05.252 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.42e+00, 5.10e-01 / 2.40e-01 | 20260507 04:42:05.252 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.44e+00, 1.66e-01 / 1.06e-01 | 20260507 04:42:05.252 | INFO | PerfReport: 16.12%, DME -- 3.96e+01 / 4.44e+01, 3.83e+00 / 1.59e+00 | 20260507 04:42:05.252 | INFO | PerfReport: 57.76%, FrameLabeling -- 1.77e+01 / 2.05e+01, 1.34e+00 / 1.78e+00 | 20260507 04:42:05.252 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.66e+00, 3.80e-01 / 2.62e-01 | 20260507 04:42:05.252 | INFO | PerfReport: 3.00%, Metrics -- 9.23e-01 / 1.22e+00, 2.45e-01 / 1.22e-01 | 20260507 04:42:05.252 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.12e-01 / 3.58e-01, 5.00e-02 / 1.51e-02 | 20260507 04:42:05.252 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:42:05.252 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:42:05.252 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:42:05.252 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 04:42:05.252 | INFO | PerfReport: 21.80%, Upload -- 2.10e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260507 04:42:05.252 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260507 04:42:05.252 | INFO | PerfReport: 31.95%, Compute -- 3.07e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 04:42:05.252 | INFO | PerfReport: 1.50%, LockUpload -- 1.45e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 04:42:05.252 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260507 04:42:05.252 | INFO | PerfReport: 22.06%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 04:42:05.252 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260507 04:42:05.348 | INFO | Starting a round of Baz IO | 20260507 04:42:08.447 | INFO | Populating RTMetrics report | 20260507 04:42:08.447 | INFO | RTMetrics report populated | 20260507 04:42:08.447 | INFO | Opening tmp RTMetrics json file | 20260507 04:42:08.463 | INFO | Serializing RTMetrics to json | 20260507 04:42:08.466 | INFO | Serialization done, swapping tmp file now... | 20260507 04:42:08.467 | INFO | RTMetrics done | 20260507 04:42:08.535 | INFO | Finished a round of Baz IO: Wrote 1.717 GiB in 3.187 seconds | 20260507 04:42:08.535 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.77% / 0.00% / 67.23% | 20260507 04:42:08.535 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.162776% / 0% / 6.35783e-05% | 20260507 04:42:08.535 | INFO | Baz IO overall: | 20260507 04:42:08.535 | INFO | BytesWritten : 3657239549386 | 20260507 04:42:08.535 | INFO | NumEvents : 1755376606627 | 20260507 04:42:10.394 | INFO | Analyzing chunk frames = [7176192,7176704) | 20260507 04:42:10.394 | INFO | MemInfo: { | 20260507 04:42:10.394 | INFO | "gpu" : " 17 GB", | 20260507 04:42:10.394 | INFO | "huge" : "", | 20260507 04:42:10.394 | INFO | "huge_shared" : " 27 GB", | 20260507 04:42:10.394 | INFO | "malloc" : " 14 GB", | 20260507 04:42:10.394 | INFO | "pinned" : " 29 GB" | 20260507 04:42:10.394 | INFO | } | 20260507 04:42:10.394 | INFO | Perf report for the last 5.14308 seconds: | 20260507 04:42:10.394 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:42:10.394 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:10.394 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:10.394 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:10.394 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 04:42:10.394 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:10.394 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 04:42:10.394 | INFO | PRE_HQ : 59.4%, 1.8, 56.7ms, 0 | 20260507 04:42:10.394 | INFO | BAZWRITER : 61.6%, 3.2, 102.6ms, 0 | 20260507 04:42:10.394 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:42:10.394 | INFO | Uploaded 36GiB to GPU in 2.04171s (17.6322GiB/s) | 20260507 04:42:10.394 | INFO | Downloaded 38GiB to GPU in 2.17058s (17.5069GiB/s) | 20260507 04:42:15.530 | INFO | Analyzing chunk frames = [7176704,7177216) | 20260507 04:42:15.530 | INFO | MemInfo: { | 20260507 04:42:15.530 | INFO | "gpu" : " 17 GB", | 20260507 04:42:15.530 | INFO | "huge" : "", | 20260507 04:42:15.530 | INFO | "huge_shared" : " 27 GB", | 20260507 04:42:15.530 | INFO | "malloc" : " 14 GB", | 20260507 04:42:15.530 | INFO | "pinned" : " 29 GB" | 20260507 04:42:15.530 | INFO | } | 20260507 04:42:15.530 | INFO | Perf report for the last 5.13638 seconds: | 20260507 04:42:15.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:42:15.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:15.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:15.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:15.530 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:42:15.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:15.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:15.530 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:42:15.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:15.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:15.530 | INFO | Uploaded 36GiB to GPU in 1.99005s (18.09GiB/s) | 20260507 04:42:15.530 | INFO | Downloaded 34GiB to GPU in 1.93111s (17.6065GiB/s) | 20260507 04:42:20.666 | INFO | Analyzing chunk frames = [7177216,7177728) | 20260507 04:42:20.666 | INFO | MemInfo: { | 20260507 04:42:20.666 | INFO | "gpu" : " 17 GB", | 20260507 04:42:20.666 | INFO | "huge" : "", | 20260507 04:42:20.666 | INFO | "huge_shared" : " 27 GB", | 20260507 04:42:20.666 | INFO | "malloc" : " 14 GB", | 20260507 04:42:20.666 | INFO | "pinned" : " 29 GB" | 20260507 04:42:20.666 | INFO | } | 20260507 04:42:20.666 | INFO | Perf report for the last 5.13611 seconds: | 20260507 04:42:20.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:42:20.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:20.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:20.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:20.666 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 04:42:20.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:20.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:20.666 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:42:20.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:20.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:20.666 | INFO | Uploaded 36GiB to GPU in 2.02632s (17.7662GiB/s) | 20260507 04:42:20.667 | INFO | Downloaded 34GiB to GPU in 1.94917s (17.4433GiB/s) | 20260507 04:42:25.807 | INFO | Analyzing chunk frames = [7177728,7178240) | 20260507 04:42:25.807 | INFO | MemInfo: { | 20260507 04:42:25.807 | INFO | "gpu" : " 17 GB", | 20260507 04:42:25.807 | INFO | "huge" : "", | 20260507 04:42:25.807 | INFO | "huge_shared" : " 27 GB", | 20260507 04:42:25.807 | INFO | "malloc" : " 14 GB", | 20260507 04:42:25.807 | INFO | "pinned" : " 29 GB" | 20260507 04:42:25.807 | INFO | } | 20260507 04:42:25.807 | INFO | Perf report for the last 5.14079 seconds: | 20260507 04:42:25.807 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:42:25.807 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:25.807 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:25.807 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:25.807 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 04:42:25.807 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:25.807 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:25.807 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:42:25.807 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:25.807 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:25.808 | INFO | Uploaded 36GiB to GPU in 1.99093s (18.082GiB/s) | 20260507 04:42:25.808 | INFO | Downloaded 34GiB to GPU in 1.91735s (17.7328GiB/s) | 20260507 04:42:30.954 | INFO | Analyzing chunk frames = [7178240,7178752) | 20260507 04:42:30.955 | INFO | MemInfo: { | 20260507 04:42:30.955 | INFO | "gpu" : " 17 GB", | 20260507 04:42:30.955 | INFO | "huge" : "", | 20260507 04:42:30.955 | INFO | "huge_shared" : " 27 GB", | 20260507 04:42:30.955 | INFO | "malloc" : " 14 GB", | 20260507 04:42:30.955 | INFO | "pinned" : " 29 GB" | 20260507 04:42:30.955 | INFO | } | 20260507 04:42:30.955 | INFO | Perf report for the last 5.14753 seconds: | 20260507 04:42:30.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:42:30.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:30.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:30.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:30.955 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 04:42:30.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:30.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:30.955 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 04:42:30.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:30.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:30.955 | INFO | Uploaded 36GiB to GPU in 2.02979s (17.7358GiB/s) | 20260507 04:42:30.955 | INFO | Downloaded 34GiB to GPU in 1.93752s (17.5482GiB/s) | 20260507 04:42:36.089 | INFO | Analyzing chunk frames = [7178752,7179264) | 20260507 04:42:36.089 | INFO | MemInfo: { | 20260507 04:42:36.089 | INFO | "gpu" : " 17 GB", | 20260507 04:42:36.089 | INFO | "huge" : "", | 20260507 04:42:36.089 | INFO | "huge_shared" : " 27 GB", | 20260507 04:42:36.089 | INFO | "malloc" : " 14 GB", | 20260507 04:42:36.089 | INFO | "pinned" : " 29 GB" | 20260507 04:42:36.089 | INFO | } | 20260507 04:42:36.089 | INFO | Perf report for the last 5.13447 seconds: | 20260507 04:42:36.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:42:36.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:36.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:36.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:36.089 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:42:36.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:36.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:36.089 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 04:42:36.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:36.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:36.089 | INFO | Uploaded 36GiB to GPU in 2.06761s (17.4114GiB/s) | 20260507 04:42:36.089 | INFO | Downloaded 34GiB to GPU in 1.92138s (17.6956GiB/s) | 20260507 04:42:41.227 | INFO | Analyzing chunk frames = [7179264,7179776) | 20260507 04:42:41.227 | INFO | MemInfo: { | 20260507 04:42:41.227 | INFO | "gpu" : " 17 GB", | 20260507 04:42:41.227 | INFO | "huge" : "", | 20260507 04:42:41.227 | INFO | "huge_shared" : " 27 GB", | 20260507 04:42:41.227 | INFO | "malloc" : " 14 GB", | 20260507 04:42:41.227 | INFO | "pinned" : " 29 GB" | 20260507 04:42:41.227 | INFO | } | 20260507 04:42:41.227 | INFO | Perf report for the last 5.13832 seconds: | 20260507 04:42:41.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:42:41.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:41.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:41.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:41.227 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 04:42:41.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:41.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:41.228 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:42:41.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:41.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:41.228 | INFO | Uploaded 36GiB to GPU in 2.04472s (17.6063GiB/s) | 20260507 04:42:41.228 | INFO | Downloaded 34GiB to GPU in 1.9868s (17.113GiB/s) | 20260507 04:42:46.362 | INFO | Analyzing chunk frames = [7179776,7180288) | 20260507 04:42:46.363 | INFO | MemInfo: { | 20260507 04:42:46.363 | INFO | "gpu" : " 17 GB", | 20260507 04:42:46.363 | INFO | "huge" : "", | 20260507 04:42:46.363 | INFO | "huge_shared" : " 27 GB", | 20260507 04:42:46.363 | INFO | "malloc" : " 14 GB", | 20260507 04:42:46.363 | INFO | "pinned" : " 29 GB" | 20260507 04:42:46.363 | INFO | } | 20260507 04:42:46.363 | INFO | Perf report for the last 5.13518 seconds: | 20260507 04:42:46.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:42:46.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:46.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:46.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:46.363 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 04:42:46.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:46.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:46.363 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 04:42:46.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:46.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:46.363 | INFO | Uploaded 36GiB to GPU in 1.96076s (18.3603GiB/s) | 20260507 04:42:46.363 | INFO | Downloaded 34GiB to GPU in 1.95518s (17.3897GiB/s) | 20260507 04:42:46.460 | INFO | Starting a round of Baz IO | 20260507 04:42:49.566 | INFO | Populating RTMetrics report | 20260507 04:42:49.566 | INFO | RTMetrics report populated | 20260507 04:42:49.566 | INFO | Opening tmp RTMetrics json file | 20260507 04:42:49.567 | INFO | Serializing RTMetrics to json | 20260507 04:42:49.584 | INFO | Serialization done, swapping tmp file now... | 20260507 04:42:49.585 | INFO | RTMetrics done | 20260507 04:42:49.653 | INFO | Finished a round of Baz IO: Wrote 1.716 GiB in 3.192 seconds | 20260507 04:42:49.653 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.77% / 0.00% / 67.23% | 20260507 04:42:49.653 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.161811% / 0% / 7.15256e-05% | 20260507 04:42:49.653 | INFO | Baz IO overall: | 20260507 04:42:49.653 | INFO | BytesWritten : 3659082612812 | 20260507 04:42:49.653 | INFO | NumEvents : 1756243333130 | 20260507 04:42:51.501 | INFO | Analyzing chunk frames = [7180288,7180800) | 20260507 04:42:51.501 | INFO | MemInfo: { | 20260507 04:42:51.501 | INFO | "gpu" : " 17 GB", | 20260507 04:42:51.501 | INFO | "huge" : "", | 20260507 04:42:51.501 | INFO | "huge_shared" : " 27 GB", | 20260507 04:42:51.501 | INFO | "malloc" : " 14 GB", | 20260507 04:42:51.501 | INFO | "pinned" : " 29 GB" | 20260507 04:42:51.501 | INFO | } | 20260507 04:42:51.501 | INFO | Perf report for the last 5.13807 seconds: | 20260507 04:42:51.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:42:51.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:51.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:51.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:51.501 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 04:42:51.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:51.501 | INFO | RT_METRICS : 42.1%, 1.0, 22.1ms, 0 | 20260507 04:42:51.501 | INFO | PRE_HQ : 59.3%, 1.8, 55.8ms, 0 | 20260507 04:42:51.501 | INFO | BAZWRITER : 61.7%, 3.2, 102.0ms, 0 | 20260507 04:42:51.501 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 04:42:51.501 | INFO | Uploaded 36GiB to GPU in 2.03628s (17.6793GiB/s) | 20260507 04:42:51.501 | INFO | Downloaded 38GiB to GPU in 2.18736s (17.3725GiB/s) | 20260507 04:42:56.644 | INFO | Analyzing chunk frames = [7180800,7181312) | 20260507 04:42:56.644 | INFO | MemInfo: { | 20260507 04:42:56.644 | INFO | "gpu" : " 17 GB", | 20260507 04:42:56.644 | INFO | "huge" : "", | 20260507 04:42:56.644 | INFO | "huge_shared" : " 27 GB", | 20260507 04:42:56.644 | INFO | "malloc" : " 14 GB", | 20260507 04:42:56.644 | INFO | "pinned" : " 29 GB" | 20260507 04:42:56.644 | INFO | } | 20260507 04:42:56.645 | INFO | Perf report for the last 5.14379 seconds: | 20260507 04:42:56.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:42:56.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:56.645 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:56.645 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:56.645 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:42:56.645 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:56.645 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:56.645 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:42:56.645 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:56.645 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:42:56.645 | INFO | Uploaded 36GiB to GPU in 2.03226s (17.7143GiB/s) | 20260507 04:42:56.645 | INFO | Downloaded 34GiB to GPU in 1.94266s (17.5018GiB/s) | 20260507 04:43:01.786 | INFO | Analyzing chunk frames = [7181312,7181824) | 20260507 04:43:01.786 | INFO | MemInfo: { | 20260507 04:43:01.786 | INFO | "gpu" : " 17 GB", | 20260507 04:43:01.786 | INFO | "huge" : "", | 20260507 04:43:01.786 | INFO | "huge_shared" : " 27 GB", | 20260507 04:43:01.786 | INFO | "malloc" : " 14 GB", | 20260507 04:43:01.786 | INFO | "pinned" : " 29 GB" | 20260507 04:43:01.786 | INFO | } | 20260507 04:43:01.786 | INFO | Perf report for the last 5.14166 seconds: | 20260507 04:43:01.786 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:43:01.786 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:01.786 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:01.786 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:01.786 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 04:43:01.786 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:01.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:01.786 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:43:01.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:01.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:01.786 | INFO | Uploaded 36GiB to GPU in 2.01377s (17.877GiB/s) | 20260507 04:43:01.786 | INFO | Downloaded 34GiB to GPU in 1.9452s (17.479GiB/s) | 20260507 04:43:06.920 | INFO | Analyzing chunk frames = [7181824,7182336) | 20260507 04:43:06.920 | INFO | MemInfo: { | 20260507 04:43:06.920 | INFO | "gpu" : " 17 GB", | 20260507 04:43:06.920 | INFO | "huge" : "", | 20260507 04:43:06.920 | INFO | "huge_shared" : " 27 GB", | 20260507 04:43:06.920 | INFO | "malloc" : " 14 GB", | 20260507 04:43:06.920 | INFO | "pinned" : " 29 GB" | 20260507 04:43:06.920 | INFO | } | 20260507 04:43:06.921 | INFO | Perf report for the last 5.13432 seconds: | 20260507 04:43:06.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:43:06.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:06.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:06.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:06.921 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 04:43:06.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:06.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:06.921 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:43:06.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:06.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:06.921 | INFO | Uploaded 36GiB to GPU in 1.98737s (18.1144GiB/s) | 20260507 04:43:06.921 | INFO | Downloaded 34GiB to GPU in 1.94444s (17.4857GiB/s) | 20260507 04:43:12.063 | INFO | Analyzing chunk frames = [7182336,7182848) | 20260507 04:43:12.063 | INFO | MemInfo: { | 20260507 04:43:12.063 | INFO | "gpu" : " 17 GB", | 20260507 04:43:12.063 | INFO | "huge" : "", | 20260507 04:43:12.063 | INFO | "huge_shared" : " 27 GB", | 20260507 04:43:12.063 | INFO | "malloc" : " 14 GB", | 20260507 04:43:12.063 | INFO | "pinned" : " 29 GB" | 20260507 04:43:12.063 | INFO | } | 20260507 04:43:12.063 | INFO | Perf report for the last 5.14237 seconds: | 20260507 04:43:12.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:43:12.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:12.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:12.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:12.063 | INFO | ANALYSIS : 59.9%, 3.0, 93.5ms, 0 | 20260507 04:43:12.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:12.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:12.063 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:43:12.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:12.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:12.063 | INFO | Uploaded 36GiB to GPU in 1.99s (18.0904GiB/s) | 20260507 04:43:12.063 | INFO | Downloaded 34GiB to GPU in 1.94581s (17.4735GiB/s) | 20260507 04:43:17.202 | INFO | Analyzing chunk frames = [7182848,7183360) | 20260507 04:43:17.202 | INFO | MemInfo: { | 20260507 04:43:17.202 | INFO | "gpu" : " 17 GB", | 20260507 04:43:17.202 | INFO | "huge" : "", | 20260507 04:43:17.202 | INFO | "huge_shared" : " 27 GB", | 20260507 04:43:17.202 | INFO | "malloc" : " 14 GB", | 20260507 04:43:17.202 | INFO | "pinned" : " 29 GB" | 20260507 04:43:17.202 | INFO | } | 20260507 04:43:17.202 | INFO | Perf report for the last 5.13935 seconds: | 20260507 04:43:17.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:43:17.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:17.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:17.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:17.202 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 04:43:17.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:17.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:17.202 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:43:17.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:17.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:17.203 | INFO | Uploaded 36GiB to GPU in 2.02549s (17.7735GiB/s) | 20260507 04:43:17.203 | INFO | Downloaded 34GiB to GPU in 1.92962s (17.62GiB/s) | 20260507 04:43:22.338 | INFO | Analyzing chunk frames = [7183360,7183872) | 20260507 04:43:22.338 | INFO | MemInfo: { | 20260507 04:43:22.338 | INFO | "gpu" : " 17 GB", | 20260507 04:43:22.338 | INFO | "huge" : "", | 20260507 04:43:22.338 | INFO | "huge_shared" : " 27 GB", | 20260507 04:43:22.338 | INFO | "malloc" : " 14 GB", | 20260507 04:43:22.338 | INFO | "pinned" : " 29 GB" | 20260507 04:43:22.339 | INFO | } | 20260507 04:43:22.339 | INFO | Perf report for the last 5.13628 seconds: | 20260507 04:43:22.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:43:22.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:22.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:22.339 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:22.339 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 04:43:22.339 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:22.339 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:22.339 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:43:22.339 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:22.339 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:22.339 | INFO | Uploaded 36GiB to GPU in 2.02649s (17.7647GiB/s) | 20260507 04:43:22.339 | INFO | Downloaded 34GiB to GPU in 1.97762s (17.1924GiB/s) | 20260507 04:43:27.481 | INFO | Analyzing chunk frames = [7183872,7184384) | 20260507 04:43:27.481 | INFO | MemInfo: { | 20260507 04:43:27.481 | INFO | "gpu" : " 17 GB", | 20260507 04:43:27.481 | INFO | "huge" : "", | 20260507 04:43:27.481 | INFO | "huge_shared" : " 27 GB", | 20260507 04:43:27.481 | INFO | "malloc" : " 14 GB", | 20260507 04:43:27.481 | INFO | "pinned" : " 29 GB" | 20260507 04:43:27.481 | INFO | } | 20260507 04:43:27.481 | INFO | Perf report for the last 5.14215 seconds: | 20260507 04:43:27.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:43:27.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:27.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:27.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:27.481 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:43:27.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:27.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:27.481 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 04:43:27.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:27.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:27.481 | INFO | Uploaded 36GiB to GPU in 1.97208s (18.2548GiB/s) | 20260507 04:43:27.481 | INFO | Downloaded 34GiB to GPU in 1.96107s (17.3374GiB/s) | 20260507 04:43:27.481 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:43:27.481 | INFO | Total mem usage: 18 GiB | 20260507 04:43:27.481 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:43:27.481 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 04:43:27.481 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:43:27.481 | INFO | High Water Breakdown: | 20260507 04:43:27.481 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:43:27.481 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:43:27.481 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:43:27.481 | INFO | ------------------------------------------ | 20260507 04:43:27.481 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:43:27.481 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:43:27.481 | INFO | Total mem usage: 26.5 GiB | 20260507 04:43:27.481 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:43:27.481 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:43:27.481 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:43:27.481 | INFO | High Water Breakdown: | 20260507 04:43:27.481 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:43:27.481 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:43:27.481 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:43:27.482 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:43:27.482 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:43:27.482 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:43:27.482 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:43:27.482 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:43:27.482 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:43:27.482 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:43:27.482 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:43:27.482 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:43:27.482 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:43:27.482 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:43:27.482 | INFO | ------------------------------------------ | 20260507 04:43:27.482 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:43:27.482 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:43:27.482 | INFO | Total mem usage: 25.6 GiB | 20260507 04:43:27.482 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:43:27.482 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:43:27.482 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:43:27.482 | INFO | High Water Breakdown: | 20260507 04:43:27.482 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:43:27.482 | INFO | ------------------------------------------ | 20260507 04:43:27.482 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:43:27.482 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:43:27.482 | INFO | Total mem usage: 15.7 GiB | 20260507 04:43:27.482 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:43:27.482 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:43:27.482 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:43:27.482 | INFO | High Water Breakdown: | 20260507 04:43:27.482 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:43:27.482 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:43:27.482 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:43:27.482 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:43:27.482 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:43:27.482 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:43:27.482 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:43:27.482 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:43:27.482 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:43:27.482 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:43:27.482 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:43:27.482 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:43:27.482 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:43:27.482 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:43:27.482 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:43:27.482 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:43:27.482 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:43:27.482 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:43:27.482 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:43:27.482 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:43:27.482 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:43:27.482 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:43:27.482 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:43:27.482 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:43:27.482 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:43:27.482 | INFO | ------------------------------------------ | 20260507 04:43:27.482 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:43:27.482 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:43:27.482 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:43:27.482 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:43:27.482 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.34e+01 / 1.50e+00 | 20260507 04:43:27.482 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.01e-01, 9.43e-03 / 5.07e-03 | 20260507 04:43:27.482 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.39e+00, 5.12e-01 / 2.49e-01 | 20260507 04:43:27.482 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.44e+00, 1.67e-01 / 1.10e-01 | 20260507 04:43:27.482 | INFO | PerfReport: 16.14%, DME -- 3.97e+01 / 4.44e+01, 3.96e+00 / 1.50e+00 | 20260507 04:43:27.482 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.33e+00 / 1.32e+00 | 20260507 04:43:27.482 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.65e+00, 3.81e-01 / 1.92e-01 | 20260507 04:43:27.482 | INFO | PerfReport: 3.01%, Metrics -- 9.25e-01 / 1.26e+00, 2.47e-01 / 1.21e-01 | 20260507 04:43:27.482 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.12e-01 / 3.65e-01, 4.98e-02 / 2.09e-02 | 20260507 04:43:27.482 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:43:27.482 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:43:27.482 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:43:27.482 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 04:43:27.482 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260507 04:43:27.482 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 04:43:27.482 | INFO | PerfReport: 31.97%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:43:27.482 | INFO | PerfReport: 1.41%, LockUpload -- 1.36e+00 / 0.00e+00, 4.71e+00 / 0.00e+00 | 20260507 04:43:27.482 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260507 04:43:27.482 | INFO | PerfReport: 22.12%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 04:43:27.482 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260507 04:43:27.578 | INFO | Starting a round of Baz IO | 20260507 04:43:30.685 | INFO | Populating RTMetrics report | 20260507 04:43:30.685 | INFO | RTMetrics report populated | 20260507 04:43:30.685 | INFO | Opening tmp RTMetrics json file | 20260507 04:43:30.699 | INFO | Serializing RTMetrics to json | 20260507 04:43:30.705 | INFO | Serialization done, swapping tmp file now... | 20260507 04:43:30.705 | INFO | RTMetrics done | 20260507 04:43:30.771 | INFO | Finished a round of Baz IO: Wrote 1.718 GiB in 3.193 seconds | 20260507 04:43:30.771 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.77% / 0.00% / 67.23% | 20260507 04:43:30.771 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.162172% / 3.97364e-06% / 8.34465e-05% | 20260507 04:43:30.771 | INFO | Baz IO overall: | 20260507 04:43:30.771 | INFO | BytesWritten : 3660926986317 | 20260507 04:43:30.771 | INFO | NumEvents : 1757111143800 | 20260507 04:43:32.621 | INFO | Analyzing chunk frames = [7184384,7184896) | 20260507 04:43:32.621 | INFO | MemInfo: { | 20260507 04:43:32.621 | INFO | "gpu" : " 17 GB", | 20260507 04:43:32.621 | INFO | "huge" : "", | 20260507 04:43:32.621 | INFO | "huge_shared" : " 27 GB", | 20260507 04:43:32.621 | INFO | "malloc" : " 14 GB", | 20260507 04:43:32.621 | INFO | "pinned" : " 29 GB" | 20260507 04:43:32.621 | INFO | } | 20260507 04:43:32.621 | INFO | Perf report for the last 5.14002 seconds: | 20260507 04:43:32.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:43:32.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:32.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:32.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:32.621 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 04:43:32.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:32.621 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:43:32.621 | INFO | PRE_HQ : 59.2%, 1.8, 55.1ms, 0 | 20260507 04:43:32.621 | INFO | BAZWRITER : 61.7%, 3.2, 100.9ms, 0 | 20260507 04:43:32.621 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:43:32.621 | INFO | Uploaded 36GiB to GPU in 2.05436s (17.5237GiB/s) | 20260507 04:43:32.621 | INFO | Downloaded 38GiB to GPU in 2.18628s (17.3811GiB/s) | 20260507 04:43:37.768 | INFO | Analyzing chunk frames = [7184896,7185408) | 20260507 04:43:37.768 | INFO | MemInfo: { | 20260507 04:43:37.768 | INFO | "gpu" : " 17 GB", | 20260507 04:43:37.768 | INFO | "huge" : "", | 20260507 04:43:37.768 | INFO | "huge_shared" : " 27 GB", | 20260507 04:43:37.768 | INFO | "malloc" : " 14 GB", | 20260507 04:43:37.768 | INFO | "pinned" : " 29 GB" | 20260507 04:43:37.768 | INFO | } | 20260507 04:43:37.768 | INFO | Perf report for the last 5.14746 seconds: | 20260507 04:43:37.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:43:37.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:37.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:37.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:37.768 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260507 04:43:37.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:37.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:37.768 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:43:37.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:37.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:37.768 | INFO | Uploaded 36GiB to GPU in 2.04249s (17.6255GiB/s) | 20260507 04:43:37.768 | INFO | Downloaded 34GiB to GPU in 1.93999s (17.5258GiB/s) | 20260507 04:43:42.903 | INFO | Analyzing chunk frames = [7185408,7185920) | 20260507 04:43:42.903 | INFO | MemInfo: { | 20260507 04:43:42.903 | INFO | "gpu" : " 17 GB", | 20260507 04:43:42.903 | INFO | "huge" : "", | 20260507 04:43:42.903 | INFO | "huge_shared" : " 27 GB", | 20260507 04:43:42.904 | INFO | "malloc" : " 14 GB", | 20260507 04:43:42.904 | INFO | "pinned" : " 29 GB" | 20260507 04:43:42.904 | INFO | } | 20260507 04:43:42.904 | INFO | Perf report for the last 5.13533 seconds: | 20260507 04:43:42.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:43:42.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:42.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:42.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:42.904 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 04:43:42.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:42.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:42.904 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:43:42.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:42.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:42.904 | INFO | Uploaded 36GiB to GPU in 2.04697s (17.587GiB/s) | 20260507 04:43:42.904 | INFO | Downloaded 34GiB to GPU in 1.94829s (17.4512GiB/s) | 20260507 04:43:48.038 | INFO | Analyzing chunk frames = [7185920,7186432) | 20260507 04:43:48.038 | INFO | MemInfo: { | 20260507 04:43:48.038 | INFO | "gpu" : " 17 GB", | 20260507 04:43:48.038 | INFO | "huge" : "", | 20260507 04:43:48.038 | INFO | "huge_shared" : " 27 GB", | 20260507 04:43:48.038 | INFO | "malloc" : " 14 GB", | 20260507 04:43:48.038 | INFO | "pinned" : " 29 GB" | 20260507 04:43:48.038 | INFO | } | 20260507 04:43:48.038 | INFO | Perf report for the last 5.13438 seconds: | 20260507 04:43:48.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:43:48.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:48.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:48.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:48.038 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 04:43:48.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:48.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:48.038 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 04:43:48.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:48.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:48.038 | INFO | Uploaded 36GiB to GPU in 2.01273s (17.8861GiB/s) | 20260507 04:43:48.038 | INFO | Downloaded 34GiB to GPU in 1.9355s (17.5666GiB/s) | 20260507 04:43:53.171 | INFO | Analyzing chunk frames = [7186432,7186944) | 20260507 04:43:53.171 | INFO | MemInfo: { | 20260507 04:43:53.171 | INFO | "gpu" : " 17 GB", | 20260507 04:43:53.171 | INFO | "huge" : "", | 20260507 04:43:53.171 | INFO | "huge_shared" : " 27 GB", | 20260507 04:43:53.171 | INFO | "malloc" : " 14 GB", | 20260507 04:43:53.171 | INFO | "pinned" : " 29 GB" | 20260507 04:43:53.171 | INFO | } | 20260507 04:43:53.171 | INFO | Perf report for the last 5.13292 seconds: | 20260507 04:43:53.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:43:53.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:53.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:53.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:53.171 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 04:43:53.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:53.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:53.171 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:43:53.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:53.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:53.171 | INFO | Uploaded 36GiB to GPU in 2.05812s (17.4917GiB/s) | 20260507 04:43:53.171 | INFO | Downloaded 34GiB to GPU in 1.94684s (17.4642GiB/s) | 20260507 04:43:58.313 | INFO | Analyzing chunk frames = [7186944,7187456) | 20260507 04:43:58.313 | INFO | MemInfo: { | 20260507 04:43:58.313 | INFO | "gpu" : " 17 GB", | 20260507 04:43:58.313 | INFO | "huge" : "", | 20260507 04:43:58.313 | INFO | "huge_shared" : " 27 GB", | 20260507 04:43:58.314 | INFO | "malloc" : " 14 GB", | 20260507 04:43:58.314 | INFO | "pinned" : " 29 GB" | 20260507 04:43:58.314 | INFO | } | 20260507 04:43:58.314 | INFO | Perf report for the last 5.14265 seconds: | 20260507 04:43:58.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:43:58.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:58.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:58.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:58.314 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 04:43:58.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:58.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:58.314 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 04:43:58.314 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:58.314 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:43:58.314 | INFO | Uploaded 36GiB to GPU in 2.02537s (17.7745GiB/s) | 20260507 04:43:58.314 | INFO | Downloaded 34GiB to GPU in 1.92942s (17.6219GiB/s) | 20260507 04:44:03.459 | INFO | Analyzing chunk frames = [7187456,7187968) | 20260507 04:44:03.459 | INFO | MemInfo: { | 20260507 04:44:03.459 | INFO | "gpu" : " 17 GB", | 20260507 04:44:03.459 | INFO | "huge" : "", | 20260507 04:44:03.459 | INFO | "huge_shared" : " 27 GB", | 20260507 04:44:03.459 | INFO | "malloc" : " 14 GB", | 20260507 04:44:03.459 | INFO | "pinned" : " 29 GB" | 20260507 04:44:03.459 | INFO | } | 20260507 04:44:03.459 | INFO | Perf report for the last 5.1457 seconds: | 20260507 04:44:03.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:44:03.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:03.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:03.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:03.459 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 04:44:03.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:03.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:03.459 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:44:03.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:03.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:03.459 | INFO | Uploaded 36GiB to GPU in 1.97193s (18.2563GiB/s) | 20260507 04:44:03.460 | INFO | Downloaded 34GiB to GPU in 1.95432s (17.3974GiB/s) | 20260507 04:44:08.597 | INFO | Analyzing chunk frames = [7187968,7188480) | 20260507 04:44:08.597 | INFO | MemInfo: { | 20260507 04:44:08.597 | INFO | "gpu" : " 17 GB", | 20260507 04:44:08.597 | INFO | "huge" : "", | 20260507 04:44:08.597 | INFO | "huge_shared" : " 27 GB", | 20260507 04:44:08.597 | INFO | "malloc" : " 14 GB", | 20260507 04:44:08.597 | INFO | "pinned" : " 29 GB" | 20260507 04:44:08.597 | INFO | } | 20260507 04:44:08.597 | INFO | Perf report for the last 5.13767 seconds: | 20260507 04:44:08.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:44:08.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:08.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:08.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:08.597 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:44:08.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:08.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:08.597 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 04:44:08.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:08.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:08.597 | INFO | Uploaded 36GiB to GPU in 1.96954s (18.2784GiB/s) | 20260507 04:44:08.597 | INFO | Downloaded 34GiB to GPU in 1.96833s (17.2735GiB/s) | 20260507 04:44:08.694 | INFO | Starting a round of Baz IO | 20260507 04:44:11.790 | INFO | Populating RTMetrics report | 20260507 04:44:11.791 | INFO | RTMetrics report populated | 20260507 04:44:11.791 | INFO | Opening tmp RTMetrics json file | 20260507 04:44:11.799 | INFO | Serializing RTMetrics to json | 20260507 04:44:11.808 | INFO | Serialization done, swapping tmp file now... | 20260507 04:44:11.808 | INFO | RTMetrics done | 20260507 04:44:11.878 | INFO | Finished a round of Baz IO: Wrote 1.716 GiB in 3.183 seconds | 20260507 04:44:11.878 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.77% / 0.00% / 67.23% | 20260507 04:44:11.878 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.161795% / 3.97364e-06% / 5.96046e-05% | 20260507 04:44:11.878 | INFO | Baz IO overall: | 20260507 04:44:11.878 | INFO | BytesWritten : 3662770017859 | 20260507 04:44:11.878 | INFO | NumEvents : 1757978035511 | 20260507 04:44:13.737 | INFO | Analyzing chunk frames = [7188480,7188992) | 20260507 04:44:13.737 | INFO | MemInfo: { | 20260507 04:44:13.737 | INFO | "gpu" : " 17 GB", | 20260507 04:44:13.737 | INFO | "huge" : "", | 20260507 04:44:13.737 | INFO | "huge_shared" : " 27 GB", | 20260507 04:44:13.737 | INFO | "malloc" : " 14 GB", | 20260507 04:44:13.737 | INFO | "pinned" : " 29 GB" | 20260507 04:44:13.737 | INFO | } | 20260507 04:44:13.737 | INFO | Perf report for the last 5.14025 seconds: | 20260507 04:44:13.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:44:13.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:13.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:13.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:13.737 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 04:44:13.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:13.737 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 04:44:13.737 | INFO | PRE_HQ : 59.2%, 1.8, 55.7ms, 0 | 20260507 04:44:13.737 | INFO | BAZWRITER : 61.5%, 3.2, 101.0ms, 0 | 20260507 04:44:13.737 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 04:44:13.737 | INFO | Uploaded 36GiB to GPU in 2.04662s (17.59GiB/s) | 20260507 04:44:13.737 | INFO | Downloaded 38GiB to GPU in 2.17574s (17.4653GiB/s) | 20260507 04:44:18.877 | INFO | Analyzing chunk frames = [7188992,7189504) | 20260507 04:44:18.877 | INFO | MemInfo: { | 20260507 04:44:18.877 | INFO | "gpu" : " 17 GB", | 20260507 04:44:18.877 | INFO | "huge" : "", | 20260507 04:44:18.877 | INFO | "huge_shared" : " 27 GB", | 20260507 04:44:18.877 | INFO | "malloc" : " 14 GB", | 20260507 04:44:18.877 | INFO | "pinned" : " 29 GB" | 20260507 04:44:18.877 | INFO | } | 20260507 04:44:18.877 | INFO | Perf report for the last 5.13987 seconds: | 20260507 04:44:18.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:44:18.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:18.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:18.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:18.877 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:44:18.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:18.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:18.877 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:44:18.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:18.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:18.877 | INFO | Uploaded 36GiB to GPU in 1.98057s (18.1766GiB/s) | 20260507 04:44:18.877 | INFO | Downloaded 34GiB to GPU in 1.93283s (17.5908GiB/s) | 20260507 04:44:24.014 | INFO | Analyzing chunk frames = [7189504,7190016) | 20260507 04:44:24.014 | INFO | MemInfo: { | 20260507 04:44:24.014 | INFO | "gpu" : " 17 GB", | 20260507 04:44:24.014 | INFO | "huge" : "", | 20260507 04:44:24.014 | INFO | "huge_shared" : " 27 GB", | 20260507 04:44:24.014 | INFO | "malloc" : " 14 GB", | 20260507 04:44:24.014 | INFO | "pinned" : " 29 GB" | 20260507 04:44:24.014 | INFO | } | 20260507 04:44:24.014 | INFO | Perf report for the last 5.13719 seconds: | 20260507 04:44:24.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:44:24.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:24.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:24.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:24.014 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 04:44:24.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:24.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:24.014 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:44:24.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:24.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:24.015 | INFO | Uploaded 36GiB to GPU in 1.98838s (18.1052GiB/s) | 20260507 04:44:24.015 | INFO | Downloaded 34GiB to GPU in 1.94238s (17.5043GiB/s) | 20260507 04:44:29.151 | INFO | Analyzing chunk frames = [7190016,7190528) | 20260507 04:44:29.151 | INFO | MemInfo: { | 20260507 04:44:29.151 | INFO | "gpu" : " 17 GB", | 20260507 04:44:29.151 | INFO | "huge" : "", | 20260507 04:44:29.151 | INFO | "huge_shared" : " 27 GB", | 20260507 04:44:29.151 | INFO | "malloc" : " 14 GB", | 20260507 04:44:29.151 | INFO | "pinned" : " 29 GB" | 20260507 04:44:29.151 | INFO | } | 20260507 04:44:29.151 | INFO | Perf report for the last 5.13684 seconds: | 20260507 04:44:29.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:44:29.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:29.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:29.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:29.151 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 04:44:29.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:29.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:29.151 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:44:29.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:29.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:29.151 | INFO | Uploaded 36GiB to GPU in 1.96382s (18.3316GiB/s) | 20260507 04:44:29.151 | INFO | Downloaded 34GiB to GPU in 1.92293s (17.6814GiB/s) | 20260507 04:44:34.291 | INFO | Analyzing chunk frames = [7190528,7191040) | 20260507 04:44:34.291 | INFO | MemInfo: { | 20260507 04:44:34.291 | INFO | "gpu" : " 17 GB", | 20260507 04:44:34.291 | INFO | "huge" : "", | 20260507 04:44:34.291 | INFO | "huge_shared" : " 27 GB", | 20260507 04:44:34.291 | INFO | "malloc" : " 14 GB", | 20260507 04:44:34.291 | INFO | "pinned" : " 29 GB" | 20260507 04:44:34.291 | INFO | } | 20260507 04:44:34.291 | INFO | Perf report for the last 5.14016 seconds: | 20260507 04:44:34.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:44:34.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:34.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:34.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:34.291 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 04:44:34.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:34.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:34.291 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:44:34.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:34.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:34.292 | INFO | Uploaded 36GiB to GPU in 2.05005s (17.5606GiB/s) | 20260507 04:44:34.292 | INFO | Downloaded 34GiB to GPU in 1.94977s (17.438GiB/s) | 20260507 04:44:39.431 | INFO | Analyzing chunk frames = [7191040,7191552) | 20260507 04:44:39.431 | INFO | MemInfo: { | 20260507 04:44:39.431 | INFO | "gpu" : " 17 GB", | 20260507 04:44:39.432 | INFO | "huge" : "", | 20260507 04:44:39.432 | INFO | "huge_shared" : " 27 GB", | 20260507 04:44:39.432 | INFO | "malloc" : " 14 GB", | 20260507 04:44:39.432 | INFO | "pinned" : " 29 GB" | 20260507 04:44:39.432 | INFO | } | 20260507 04:44:39.432 | INFO | Perf report for the last 5.14056 seconds: | 20260507 04:44:39.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:44:39.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:39.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:39.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:39.432 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 04:44:39.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:39.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:39.432 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 04:44:39.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:39.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:39.432 | INFO | Uploaded 36GiB to GPU in 2.03635s (17.6787GiB/s) | 20260507 04:44:39.432 | INFO | Downloaded 34GiB to GPU in 1.92654s (17.6482GiB/s) | 20260507 04:44:44.574 | INFO | Analyzing chunk frames = [7191552,7192064) | 20260507 04:44:44.574 | INFO | MemInfo: { | 20260507 04:44:44.574 | INFO | "gpu" : " 17 GB", | 20260507 04:44:44.574 | INFO | "huge" : "", | 20260507 04:44:44.574 | INFO | "huge_shared" : " 27 GB", | 20260507 04:44:44.574 | INFO | "malloc" : " 14 GB", | 20260507 04:44:44.574 | INFO | "pinned" : " 29 GB" | 20260507 04:44:44.574 | INFO | } | 20260507 04:44:44.574 | INFO | Perf report for the last 5.14257 seconds: | 20260507 04:44:44.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:44:44.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:44.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:44.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:44.575 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:44:44.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:44.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:44.575 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:44:44.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:44.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:44.575 | INFO | Uploaded 36GiB to GPU in 1.96754s (18.2969GiB/s) | 20260507 04:44:44.575 | INFO | Downloaded 34GiB to GPU in 1.97509s (17.2144GiB/s) | 20260507 04:44:49.715 | INFO | Analyzing chunk frames = [7192064,7192576) | 20260507 04:44:49.715 | INFO | MemInfo: { | 20260507 04:44:49.715 | INFO | "gpu" : " 17 GB", | 20260507 04:44:49.715 | INFO | "huge" : "", | 20260507 04:44:49.715 | INFO | "huge_shared" : " 27 GB", | 20260507 04:44:49.715 | INFO | "malloc" : " 14 GB", | 20260507 04:44:49.715 | INFO | "pinned" : " 29 GB" | 20260507 04:44:49.715 | INFO | } | 20260507 04:44:49.715 | INFO | Perf report for the last 5.14023 seconds: | 20260507 04:44:49.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:44:49.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:49.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:49.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:49.715 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:44:49.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:49.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:49.715 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:44:49.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:49.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:49.715 | INFO | Uploaded 36GiB to GPU in 1.97499s (18.2279GiB/s) | 20260507 04:44:49.715 | INFO | Downloaded 34GiB to GPU in 1.97516s (17.2138GiB/s) | 20260507 04:44:49.715 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:44:49.715 | INFO | Total mem usage: 18 GiB | 20260507 04:44:49.715 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:44:49.715 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 04:44:49.715 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:44:49.715 | INFO | High Water Breakdown: | 20260507 04:44:49.715 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:44:49.715 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:44:49.715 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:44:49.715 | INFO | ------------------------------------------ | 20260507 04:44:49.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:44:49.715 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:44:49.715 | INFO | Total mem usage: 26.5 GiB | 20260507 04:44:49.715 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:44:49.715 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:44:49.715 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:44:49.715 | INFO | High Water Breakdown: | 20260507 04:44:49.715 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:44:49.715 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:44:49.715 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:44:49.715 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:44:49.715 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:44:49.715 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:44:49.715 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:44:49.715 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:44:49.715 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:44:49.715 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:44:49.715 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:44:49.715 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:44:49.715 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:44:49.715 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:44:49.715 | INFO | ------------------------------------------ | 20260507 04:44:49.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:44:49.715 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:44:49.715 | INFO | Total mem usage: 25.6 GiB | 20260507 04:44:49.715 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:44:49.715 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:44:49.715 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:44:49.715 | INFO | High Water Breakdown: | 20260507 04:44:49.715 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:44:49.715 | INFO | ------------------------------------------ | 20260507 04:44:49.715 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:44:49.715 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:44:49.715 | INFO | Total mem usage: 15.7 GiB | 20260507 04:44:49.716 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:44:49.716 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:44:49.716 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:44:49.716 | INFO | High Water Breakdown: | 20260507 04:44:49.716 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:44:49.716 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:44:49.716 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:44:49.716 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:44:49.716 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:44:49.716 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:44:49.716 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:44:49.716 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:44:49.716 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:44:49.716 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:44:49.716 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:44:49.716 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:44:49.716 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:44:49.716 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:44:49.716 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:44:49.716 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:44:49.716 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:44:49.716 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:44:49.716 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:44:49.716 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:44:49.716 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:44:49.716 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:44:49.716 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:44:49.716 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:44:49.716 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:44:49.716 | INFO | ------------------------------------------ | 20260507 04:44:49.716 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:44:49.716 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:44:49.716 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:44:49.716 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:44:49.716 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 1.16e+00 | 20260507 04:44:49.716 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.52e-03 / 5.06e-03 | 20260507 04:44:49.716 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.43e+00, 5.06e-01 / 2.98e-01 | 20260507 04:44:49.716 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.46e+00, 1.66e-01 / 1.24e-01 | 20260507 04:44:49.716 | INFO | PerfReport: 16.15%, DME -- 3.97e+01 / 4.43e+01, 3.89e+00 / 1.67e+00 | 20260507 04:44:49.716 | INFO | PerfReport: 57.69%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.29e+00 / 1.69e+00 | 20260507 04:44:49.716 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.69e+00, 3.85e-01 / 2.25e-01 | 20260507 04:44:49.716 | INFO | PerfReport: 3.01%, Metrics -- 9.26e-01 / 1.24e+00, 2.47e-01 / 1.21e-01 | 20260507 04:44:49.716 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.12e-01 / 3.62e-01, 4.94e-02 / 1.69e-02 | 20260507 04:44:49.716 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:44:49.716 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:44:49.716 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:44:49.716 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 04:44:49.716 | INFO | PerfReport: 21.79%, Upload -- 2.10e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260507 04:44:49.716 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260507 04:44:49.716 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:44:49.716 | INFO | PerfReport: 1.50%, LockUpload -- 1.45e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 04:44:49.716 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260507 04:44:49.716 | INFO | PerfReport: 22.06%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 04:44:49.716 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260507 04:44:49.812 | INFO | Starting a round of Baz IO | 20260507 04:44:52.907 | INFO | Populating RTMetrics report | 20260507 04:44:52.908 | INFO | RTMetrics report populated | 20260507 04:44:52.908 | INFO | Opening tmp RTMetrics json file | 20260507 04:44:52.925 | INFO | Serializing RTMetrics to json | 20260507 04:44:52.927 | INFO | Serialization done, swapping tmp file now... | 20260507 04:44:52.927 | INFO | RTMetrics done | 20260507 04:44:52.995 | INFO | Finished a round of Baz IO: Wrote 1.717 GiB in 3.183 seconds | 20260507 04:44:52.995 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.77% / 0.00% / 67.23% | 20260507 04:44:52.995 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.162756% / 0% / 5.16574e-05% | 20260507 04:44:52.996 | INFO | Baz IO overall: | 20260507 04:44:52.996 | INFO | BytesWritten : 3664613308691 | 20260507 04:44:52.996 | INFO | NumEvents : 1758844891853 | 20260507 04:44:54.855 | INFO | Analyzing chunk frames = [7192576,7193088) | 20260507 04:44:54.855 | INFO | MemInfo: { | 20260507 04:44:54.855 | INFO | "gpu" : " 17 GB", | 20260507 04:44:54.855 | INFO | "huge" : "", | 20260507 04:44:54.855 | INFO | "huge_shared" : " 27 GB", | 20260507 04:44:54.855 | INFO | "malloc" : " 14 GB", | 20260507 04:44:54.855 | INFO | "pinned" : " 29 GB" | 20260507 04:44:54.855 | INFO | } | 20260507 04:44:54.855 | INFO | Perf report for the last 5.14037 seconds: | 20260507 04:44:54.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:44:54.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:54.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:54.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:54.855 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 04:44:54.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:54.855 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:44:54.855 | INFO | PRE_HQ : 59.0%, 1.8, 55.1ms, 0 | 20260507 04:44:54.855 | INFO | BAZWRITER : 61.5%, 3.3, 103.2ms, 0 | 20260507 04:44:54.855 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:44:54.855 | INFO | Uploaded 36GiB to GPU in 2.05272s (17.5377GiB/s) | 20260507 04:44:54.856 | INFO | Downloaded 38GiB to GPU in 2.17997s (17.4314GiB/s) | 20260507 04:44:59.983 | INFO | Analyzing chunk frames = [7193088,7193600) | 20260507 04:44:59.983 | INFO | MemInfo: { | 20260507 04:44:59.983 | INFO | "gpu" : " 17 GB", | 20260507 04:44:59.983 | INFO | "huge" : "", | 20260507 04:44:59.983 | INFO | "huge_shared" : " 27 GB", | 20260507 04:44:59.983 | INFO | "malloc" : " 14 GB", | 20260507 04:44:59.983 | INFO | "pinned" : " 29 GB" | 20260507 04:44:59.983 | INFO | } | 20260507 04:44:59.983 | INFO | Perf report for the last 5.12807 seconds: | 20260507 04:44:59.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:44:59.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:59.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:59.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:59.983 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 04:44:59.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:59.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:59.983 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:44:59.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:59.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:44:59.984 | INFO | Uploaded 36GiB to GPU in 1.99052s (18.0857GiB/s) | 20260507 04:44:59.984 | INFO | Downloaded 34GiB to GPU in 1.9265s (17.6486GiB/s) | 20260507 04:45:05.125 | INFO | Analyzing chunk frames = [7193600,7194112) | 20260507 04:45:05.125 | INFO | MemInfo: { | 20260507 04:45:05.125 | INFO | "gpu" : " 17 GB", | 20260507 04:45:05.125 | INFO | "huge" : "", | 20260507 04:45:05.125 | INFO | "huge_shared" : " 27 GB", | 20260507 04:45:05.125 | INFO | "malloc" : " 14 GB", | 20260507 04:45:05.125 | INFO | "pinned" : " 29 GB" | 20260507 04:45:05.125 | INFO | } | 20260507 04:45:05.125 | INFO | Perf report for the last 5.14206 seconds: | 20260507 04:45:05.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:45:05.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:05.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:05.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:05.125 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 04:45:05.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:05.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:05.125 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:45:05.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:05.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:05.125 | INFO | Uploaded 36GiB to GPU in 1.99525s (18.0428GiB/s) | 20260507 04:45:05.125 | INFO | Downloaded 34GiB to GPU in 1.94097s (17.517GiB/s) | 20260507 04:45:10.268 | INFO | Analyzing chunk frames = [7194112,7194624) | 20260507 04:45:10.268 | INFO | MemInfo: { | 20260507 04:45:10.268 | INFO | "gpu" : " 17 GB", | 20260507 04:45:10.268 | INFO | "huge" : "", | 20260507 04:45:10.268 | INFO | "huge_shared" : " 27 GB", | 20260507 04:45:10.268 | INFO | "malloc" : " 14 GB", | 20260507 04:45:10.268 | INFO | "pinned" : " 29 GB" | 20260507 04:45:10.268 | INFO | } | 20260507 04:45:10.268 | INFO | Perf report for the last 5.14254 seconds: | 20260507 04:45:10.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:45:10.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:10.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:10.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:10.268 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:45:10.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:10.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:10.268 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:45:10.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:10.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:10.268 | INFO | Uploaded 36GiB to GPU in 1.96181s (18.3504GiB/s) | 20260507 04:45:10.268 | INFO | Downloaded 34GiB to GPU in 1.92382s (17.6732GiB/s) | 20260507 04:45:15.416 | INFO | Analyzing chunk frames = [7194624,7195136) | 20260507 04:45:15.416 | INFO | MemInfo: { | 20260507 04:45:15.416 | INFO | "gpu" : " 17 GB", | 20260507 04:45:15.416 | INFO | "huge" : "", | 20260507 04:45:15.416 | INFO | "huge_shared" : " 27 GB", | 20260507 04:45:15.417 | INFO | "malloc" : " 14 GB", | 20260507 04:45:15.417 | INFO | "pinned" : " 29 GB" | 20260507 04:45:15.417 | INFO | } | 20260507 04:45:15.417 | INFO | Perf report for the last 5.14876 seconds: | 20260507 04:45:15.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:45:15.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:15.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:15.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:15.417 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 04:45:15.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:15.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:15.417 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:45:15.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:15.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:15.417 | INFO | Uploaded 36GiB to GPU in 2.02802s (17.7513GiB/s) | 20260507 04:45:15.417 | INFO | Downloaded 34GiB to GPU in 1.94316s (17.4972GiB/s) | 20260507 04:45:20.550 | INFO | Analyzing chunk frames = [7195136,7195648) | 20260507 04:45:20.550 | INFO | MemInfo: { | 20260507 04:45:20.550 | INFO | "gpu" : " 17 GB", | 20260507 04:45:20.550 | INFO | "huge" : "", | 20260507 04:45:20.550 | INFO | "huge_shared" : " 27 GB", | 20260507 04:45:20.550 | INFO | "malloc" : " 14 GB", | 20260507 04:45:20.551 | INFO | "pinned" : " 29 GB" | 20260507 04:45:20.551 | INFO | } | 20260507 04:45:20.551 | INFO | Perf report for the last 5.13397 seconds: | 20260507 04:45:20.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:45:20.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:20.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:20.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:20.551 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:45:20.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:20.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:20.551 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:45:20.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:20.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:20.551 | INFO | Uploaded 36GiB to GPU in 2.06668s (17.4192GiB/s) | 20260507 04:45:20.551 | INFO | Downloaded 34GiB to GPU in 1.94718s (17.4611GiB/s) | 20260507 04:45:25.681 | INFO | Analyzing chunk frames = [7195648,7196160) | 20260507 04:45:25.681 | INFO | MemInfo: { | 20260507 04:45:25.681 | INFO | "gpu" : " 17 GB", | 20260507 04:45:25.681 | INFO | "huge" : "", | 20260507 04:45:25.681 | INFO | "huge_shared" : " 27 GB", | 20260507 04:45:25.681 | INFO | "malloc" : " 14 GB", | 20260507 04:45:25.681 | INFO | "pinned" : " 29 GB" | 20260507 04:45:25.681 | INFO | } | 20260507 04:45:25.681 | INFO | Perf report for the last 5.13064 seconds: | 20260507 04:45:25.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:45:25.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:25.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:25.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:25.681 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 04:45:25.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:25.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:25.681 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 04:45:25.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:25.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:25.681 | INFO | Uploaded 36GiB to GPU in 1.93987s (18.5579GiB/s) | 20260507 04:45:25.681 | INFO | Downloaded 34GiB to GPU in 1.9758s (17.2082GiB/s) | 20260507 04:45:30.824 | INFO | Analyzing chunk frames = [7196160,7196672) | 20260507 04:45:30.824 | INFO | MemInfo: { | 20260507 04:45:30.824 | INFO | "gpu" : " 17 GB", | 20260507 04:45:30.824 | INFO | "huge" : "", | 20260507 04:45:30.824 | INFO | "huge_shared" : " 27 GB", | 20260507 04:45:30.824 | INFO | "malloc" : " 14 GB", | 20260507 04:45:30.824 | INFO | "pinned" : " 29 GB" | 20260507 04:45:30.824 | INFO | } | 20260507 04:45:30.824 | INFO | Perf report for the last 5.14264 seconds: | 20260507 04:45:30.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:45:30.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:30.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:30.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:30.824 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 04:45:30.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:30.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:30.824 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:45:30.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:30.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:30.824 | INFO | Uploaded 36GiB to GPU in 1.96291s (18.3401GiB/s) | 20260507 04:45:30.824 | INFO | Downloaded 34GiB to GPU in 1.94485s (17.4821GiB/s) | 20260507 04:45:30.921 | INFO | Starting a round of Baz IO | 20260507 04:45:34.026 | INFO | Populating RTMetrics report | 20260507 04:45:34.026 | INFO | RTMetrics report populated | 20260507 04:45:34.026 | INFO | Opening tmp RTMetrics json file | 20260507 04:45:34.030 | INFO | Serializing RTMetrics to json | 20260507 04:45:34.044 | INFO | Serialization done, swapping tmp file now... | 20260507 04:45:34.044 | INFO | RTMetrics done | 20260507 04:45:34.114 | INFO | Finished a round of Baz IO: Wrote 1.717 GiB in 3.193 seconds | 20260507 04:45:34.114 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.76% / 0.00% / 67.23% | 20260507 04:45:34.114 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.161636% / 3.97364e-06% / 7.54992e-05% | 20260507 04:45:34.114 | INFO | Baz IO overall: | 20260507 04:45:34.114 | INFO | BytesWritten : 3666456527747 | 20260507 04:45:34.114 | INFO | NumEvents : 1759711911268 | 20260507 04:45:35.962 | INFO | Analyzing chunk frames = [7196672,7197184) | 20260507 04:45:35.962 | INFO | MemInfo: { | 20260507 04:45:35.962 | INFO | "gpu" : " 17 GB", | 20260507 04:45:35.962 | INFO | "huge" : "", | 20260507 04:45:35.962 | INFO | "huge_shared" : " 27 GB", | 20260507 04:45:35.962 | INFO | "malloc" : " 14 GB", | 20260507 04:45:35.962 | INFO | "pinned" : " 29 GB" | 20260507 04:45:35.962 | INFO | } | 20260507 04:45:35.962 | INFO | Perf report for the last 5.13849 seconds: | 20260507 04:45:35.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:45:35.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:35.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:35.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:35.962 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 04:45:35.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:35.962 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:45:35.962 | INFO | PRE_HQ : 59.1%, 1.8, 55.0ms, 0 | 20260507 04:45:35.962 | INFO | BAZWRITER : 61.7%, 3.2, 101.7ms, 0 | 20260507 04:45:35.962 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 04:45:35.963 | INFO | Uploaded 36GiB to GPU in 2.04216s (17.6284GiB/s) | 20260507 04:45:35.963 | INFO | Downloaded 38GiB to GPU in 2.17801s (17.4471GiB/s) | 20260507 04:45:41.104 | INFO | Analyzing chunk frames = [7197184,7197696) | 20260507 04:45:41.104 | INFO | MemInfo: { | 20260507 04:45:41.104 | INFO | "gpu" : " 17 GB", | 20260507 04:45:41.104 | INFO | "huge" : "", | 20260507 04:45:41.104 | INFO | "huge_shared" : " 27 GB", | 20260507 04:45:41.104 | INFO | "malloc" : " 14 GB", | 20260507 04:45:41.104 | INFO | "pinned" : " 29 GB" | 20260507 04:45:41.104 | INFO | } | 20260507 04:45:41.104 | INFO | Perf report for the last 5.14153 seconds: | 20260507 04:45:41.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:45:41.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:41.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:41.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:41.104 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:45:41.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:41.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:41.104 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:45:41.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:41.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:41.104 | INFO | Uploaded 36GiB to GPU in 1.98809s (18.1078GiB/s) | 20260507 04:45:41.104 | INFO | Downloaded 34GiB to GPU in 1.92546s (17.6581GiB/s) | 20260507 04:45:46.242 | INFO | Analyzing chunk frames = [7197696,7198208) | 20260507 04:45:46.242 | INFO | MemInfo: { | 20260507 04:45:46.242 | INFO | "gpu" : " 17 GB", | 20260507 04:45:46.242 | INFO | "huge" : "", | 20260507 04:45:46.242 | INFO | "huge_shared" : " 27 GB", | 20260507 04:45:46.242 | INFO | "malloc" : " 14 GB", | 20260507 04:45:46.242 | INFO | "pinned" : " 29 GB" | 20260507 04:45:46.242 | INFO | } | 20260507 04:45:46.242 | INFO | Perf report for the last 5.13815 seconds: | 20260507 04:45:46.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:45:46.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:46.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:46.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:46.242 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 04:45:46.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:46.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:46.242 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:45:46.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:46.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:46.242 | INFO | Uploaded 36GiB to GPU in 2.00871s (17.9219GiB/s) | 20260507 04:45:46.242 | INFO | Downloaded 34GiB to GPU in 1.94088s (17.5178GiB/s) | 20260507 04:45:51.386 | INFO | Analyzing chunk frames = [7198208,7198720) | 20260507 04:45:51.386 | INFO | MemInfo: { | 20260507 04:45:51.386 | INFO | "gpu" : " 17 GB", | 20260507 04:45:51.386 | INFO | "huge" : "", | 20260507 04:45:51.386 | INFO | "huge_shared" : " 27 GB", | 20260507 04:45:51.386 | INFO | "malloc" : " 14 GB", | 20260507 04:45:51.386 | INFO | "pinned" : " 29 GB" | 20260507 04:45:51.386 | INFO | } | 20260507 04:45:51.386 | INFO | Perf report for the last 5.14401 seconds: | 20260507 04:45:51.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:45:51.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:51.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:51.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:51.386 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:45:51.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:51.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:51.386 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:45:51.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:51.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:51.386 | INFO | Uploaded 36GiB to GPU in 1.98035s (18.1786GiB/s) | 20260507 04:45:51.386 | INFO | Downloaded 34GiB to GPU in 1.92086s (17.7004GiB/s) | 20260507 04:45:56.525 | INFO | Analyzing chunk frames = [7198720,7199232) | 20260507 04:45:56.525 | INFO | MemInfo: { | 20260507 04:45:56.525 | INFO | "gpu" : " 17 GB", | 20260507 04:45:56.525 | INFO | "huge" : "", | 20260507 04:45:56.525 | INFO | "huge_shared" : " 27 GB", | 20260507 04:45:56.525 | INFO | "malloc" : " 14 GB", | 20260507 04:45:56.525 | INFO | "pinned" : " 29 GB" | 20260507 04:45:56.525 | INFO | } | 20260507 04:45:56.526 | INFO | Perf report for the last 5.13945 seconds: | 20260507 04:45:56.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:45:56.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:56.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:56.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:56.526 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 04:45:56.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:56.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:56.526 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:45:56.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:56.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:45:56.526 | INFO | Uploaded 36GiB to GPU in 2.04001s (17.647GiB/s) | 20260507 04:45:56.526 | INFO | Downloaded 34GiB to GPU in 1.93166s (17.6014GiB/s) | 20260507 04:46:01.661 | INFO | Analyzing chunk frames = [7199232,7199744) | 20260507 04:46:01.661 | INFO | MemInfo: { | 20260507 04:46:01.661 | INFO | "gpu" : " 17 GB", | 20260507 04:46:01.661 | INFO | "huge" : "", | 20260507 04:46:01.661 | INFO | "huge_shared" : " 27 GB", | 20260507 04:46:01.661 | INFO | "malloc" : " 14 GB", | 20260507 04:46:01.661 | INFO | "pinned" : " 29 GB" | 20260507 04:46:01.661 | INFO | } | 20260507 04:46:01.661 | INFO | Perf report for the last 5.13542 seconds: | 20260507 04:46:01.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:46:01.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:01.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:01.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:01.661 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 04:46:01.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:01.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:01.661 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 04:46:01.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:01.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:01.661 | INFO | Uploaded 36GiB to GPU in 2.03987s (17.6482GiB/s) | 20260507 04:46:01.661 | INFO | Downloaded 34GiB to GPU in 1.90555s (17.8426GiB/s) | 20260507 04:46:06.800 | INFO | Analyzing chunk frames = [7199744,7200256) | 20260507 04:46:06.800 | INFO | MemInfo: { | 20260507 04:46:06.800 | INFO | "gpu" : " 17 GB", | 20260507 04:46:06.800 | INFO | "huge" : "", | 20260507 04:46:06.800 | INFO | "huge_shared" : " 27 GB", | 20260507 04:46:06.800 | INFO | "malloc" : " 14 GB", | 20260507 04:46:06.800 | INFO | "pinned" : " 29 GB" | 20260507 04:46:06.800 | INFO | } | 20260507 04:46:06.800 | INFO | Perf report for the last 5.13882 seconds: | 20260507 04:46:06.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:46:06.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:06.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:06.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:06.800 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 04:46:06.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:06.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:06.800 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:46:06.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:06.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:06.800 | INFO | Uploaded 36GiB to GPU in 2.04525s (17.6018GiB/s) | 20260507 04:46:06.800 | INFO | Downloaded 34GiB to GPU in 1.99179s (17.07GiB/s) | 20260507 04:46:11.935 | INFO | Analyzing chunk frames = [7200256,7200768) | 20260507 04:46:11.935 | INFO | MemInfo: { | 20260507 04:46:11.935 | INFO | "gpu" : " 17 GB", | 20260507 04:46:11.935 | INFO | "huge" : "", | 20260507 04:46:11.935 | INFO | "huge_shared" : " 27 GB", | 20260507 04:46:11.935 | INFO | "malloc" : " 14 GB", | 20260507 04:46:11.935 | INFO | "pinned" : " 29 GB" | 20260507 04:46:11.935 | INFO | } | 20260507 04:46:11.935 | INFO | Perf report for the last 5.1355 seconds: | 20260507 04:46:11.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:46:11.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:11.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:11.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:11.935 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:46:11.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:11.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:11.935 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 04:46:11.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:11.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:11.936 | INFO | Uploaded 36GiB to GPU in 1.97035s (18.2708GiB/s) | 20260507 04:46:11.936 | INFO | Downloaded 34GiB to GPU in 1.96642s (17.2903GiB/s) | 20260507 04:46:11.936 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:46:11.936 | INFO | Total mem usage: 18 GiB | 20260507 04:46:11.936 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:46:11.936 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 04:46:11.936 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:46:11.936 | INFO | High Water Breakdown: | 20260507 04:46:11.936 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:46:11.936 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:46:11.936 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:46:11.936 | INFO | ------------------------------------------ | 20260507 04:46:11.936 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:46:11.936 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:46:11.936 | INFO | Total mem usage: 26.5 GiB | 20260507 04:46:11.936 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:46:11.936 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:46:11.936 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 04:46:11.936 | INFO | High Water Breakdown: | 20260507 04:46:11.936 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:46:11.936 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:46:11.936 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:46:11.936 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:46:11.936 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:46:11.936 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:46:11.936 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:46:11.936 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:46:11.936 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:46:11.936 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:46:11.936 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:46:11.936 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:46:11.936 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:46:11.936 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:46:11.936 | INFO | ------------------------------------------ | 20260507 04:46:11.936 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:46:11.936 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:46:11.936 | INFO | Total mem usage: 25.6 GiB | 20260507 04:46:11.936 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:46:11.936 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:46:11.936 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:46:11.936 | INFO | High Water Breakdown: | 20260507 04:46:11.936 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:46:11.936 | INFO | ------------------------------------------ | 20260507 04:46:11.936 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:46:11.936 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:46:11.936 | INFO | Total mem usage: 15.7 GiB | 20260507 04:46:11.936 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:46:11.936 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:46:11.936 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:46:11.936 | INFO | High Water Breakdown: | 20260507 04:46:11.936 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:46:11.936 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:46:11.936 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:46:11.936 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:46:11.936 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:46:11.936 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:46:11.936 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:46:11.936 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:46:11.936 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:46:11.936 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:46:11.936 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:46:11.936 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:46:11.936 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:46:11.936 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:46:11.936 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:46:11.936 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:46:11.936 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:46:11.936 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:46:11.936 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:46:11.936 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:46:11.936 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:46:11.936 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:46:11.936 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:46:11.936 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:46:11.936 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:46:11.936 | INFO | ------------------------------------------ | 20260507 04:46:11.936 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:46:11.936 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:46:11.936 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:46:11.936 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:46:11.936 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.49e+01, 1.34e+01 / 1.16e+00 | 20260507 04:46:11.936 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.00e-01, 1.02e-02 / 6.14e-03 | 20260507 04:46:11.936 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.37e+00, 5.19e-01 / 3.77e-01 | 20260507 04:46:11.936 | INFO | PerfReport: 3.24%, Histogram -- 9.98e-01 / 1.47e+00, 1.74e-01 / 1.44e-01 | 20260507 04:46:11.936 | INFO | PerfReport: 16.09%, DME -- 3.96e+01 / 4.42e+01, 3.89e+00 / 1.74e+00 | 20260507 04:46:11.936 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.35e+00 / 1.63e+00 | 20260507 04:46:11.936 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.69e+00, 3.82e-01 / 2.34e-01 | 20260507 04:46:11.936 | INFO | PerfReport: 3.01%, Metrics -- 9.26e-01 / 1.27e+00, 2.45e-01 / 1.35e-01 | 20260507 04:46:11.936 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.13e-01 / 3.67e-01, 4.98e-02 / 2.05e-02 | 20260507 04:46:11.937 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:46:11.937 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:46:11.937 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:46:11.937 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 04:46:11.937 | INFO | PerfReport: 21.75%, Upload -- 2.09e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260507 04:46:11.937 | INFO | PerfReport: 21.36%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 04:46:11.937 | INFO | PerfReport: 32.01%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:46:11.937 | INFO | PerfReport: 1.38%, LockUpload -- 1.33e+00 / 0.00e+00, 4.72e+00 / 0.00e+00 | 20260507 04:46:11.937 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260507 04:46:11.937 | INFO | PerfReport: 22.21%, LockCompute -- 2.14e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 04:46:11.937 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260507 04:46:12.032 | INFO | Starting a round of Baz IO | 20260507 04:46:15.138 | INFO | Populating RTMetrics report | 20260507 04:46:15.138 | INFO | RTMetrics report populated | 20260507 04:46:15.138 | INFO | Opening tmp RTMetrics json file | 20260507 04:46:15.155 | INFO | Serializing RTMetrics to json | 20260507 04:46:15.157 | INFO | Serialization done, swapping tmp file now... | 20260507 04:46:15.157 | INFO | RTMetrics done | 20260507 04:46:15.224 | INFO | Finished a round of Baz IO: Wrote 1.716 GiB in 3.191 seconds | 20260507 04:46:15.224 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.76% / 0.00% / 67.24% | 20260507 04:46:15.224 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.161628% / 0% / 0.000103315% | 20260507 04:46:15.224 | INFO | Baz IO overall: | 20260507 04:46:15.224 | INFO | BytesWritten : 3668299424262 | 20260507 04:46:15.224 | INFO | NumEvents : 1760578423623 | 20260507 04:46:17.080 | INFO | Analyzing chunk frames = [7200768,7201280) | 20260507 04:46:17.080 | INFO | MemInfo: { | 20260507 04:46:17.080 | INFO | "gpu" : " 17 GB", | 20260507 04:46:17.080 | INFO | "huge" : "", | 20260507 04:46:17.080 | INFO | "huge_shared" : " 27 GB", | 20260507 04:46:17.080 | INFO | "malloc" : " 14 GB", | 20260507 04:46:17.080 | INFO | "pinned" : " 29 GB" | 20260507 04:46:17.080 | INFO | } | 20260507 04:46:17.080 | INFO | Perf report for the last 5.14425 seconds: | 20260507 04:46:17.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:46:17.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:17.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:17.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:17.080 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 04:46:17.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:17.080 | INFO | RT_METRICS : 41.6%, 1.0, 21.9ms, 0 | 20260507 04:46:17.080 | INFO | PRE_HQ : 59.1%, 1.8, 55.2ms, 0 | 20260507 04:46:17.080 | INFO | BAZWRITER : 61.6%, 3.2, 102.1ms, 0 | 20260507 04:46:17.080 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:46:17.080 | INFO | Uploaded 36GiB to GPU in 2.03291s (17.7086GiB/s) | 20260507 04:46:17.080 | INFO | Downloaded 38GiB to GPU in 2.19177s (17.3376GiB/s) | 20260507 04:46:22.220 | INFO | Analyzing chunk frames = [7201280,7201792) | 20260507 04:46:22.220 | INFO | MemInfo: { | 20260507 04:46:22.220 | INFO | "gpu" : " 17 GB", | 20260507 04:46:22.220 | INFO | "huge" : "", | 20260507 04:46:22.220 | INFO | "huge_shared" : " 27 GB", | 20260507 04:46:22.220 | INFO | "malloc" : " 14 GB", | 20260507 04:46:22.220 | INFO | "pinned" : " 29 GB" | 20260507 04:46:22.220 | INFO | } | 20260507 04:46:22.220 | INFO | Perf report for the last 5.14023 seconds: | 20260507 04:46:22.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:46:22.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:22.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:22.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:22.220 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:46:22.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:22.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:22.220 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:46:22.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:22.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:22.220 | INFO | Uploaded 36GiB to GPU in 1.98469s (18.1389GiB/s) | 20260507 04:46:22.220 | INFO | Downloaded 34GiB to GPU in 1.93335s (17.586GiB/s) | 20260507 04:46:27.363 | INFO | Analyzing chunk frames = [7201792,7202304) | 20260507 04:46:27.363 | INFO | MemInfo: { | 20260507 04:46:27.363 | INFO | "gpu" : " 17 GB", | 20260507 04:46:27.363 | INFO | "huge" : "", | 20260507 04:46:27.363 | INFO | "huge_shared" : " 27 GB", | 20260507 04:46:27.363 | INFO | "malloc" : " 14 GB", | 20260507 04:46:27.363 | INFO | "pinned" : " 29 GB" | 20260507 04:46:27.363 | INFO | } | 20260507 04:46:27.363 | INFO | Perf report for the last 5.1429 seconds: | 20260507 04:46:27.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:46:27.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:27.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:27.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:27.363 | INFO | ANALYSIS : 59.9%, 3.0, 92.9ms, 0 | 20260507 04:46:27.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:27.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:27.363 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:46:27.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:27.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:27.363 | INFO | Uploaded 36GiB to GPU in 1.99604s (18.0357GiB/s) | 20260507 04:46:27.363 | INFO | Downloaded 34GiB to GPU in 1.92542s (17.6585GiB/s) | 20260507 04:46:32.493 | INFO | Analyzing chunk frames = [7202304,7202816) | 20260507 04:46:32.493 | INFO | MemInfo: { | 20260507 04:46:32.493 | INFO | "gpu" : " 17 GB", | 20260507 04:46:32.493 | INFO | "huge" : "", | 20260507 04:46:32.493 | INFO | "huge_shared" : " 27 GB", | 20260507 04:46:32.493 | INFO | "malloc" : " 14 GB", | 20260507 04:46:32.493 | INFO | "pinned" : " 29 GB" | 20260507 04:46:32.493 | INFO | } | 20260507 04:46:32.493 | INFO | Perf report for the last 5.13056 seconds: | 20260507 04:46:32.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:46:32.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:32.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:32.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:32.493 | INFO | ANALYSIS : 60.6%, 2.9, 93.4ms, 0 | 20260507 04:46:32.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:32.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:32.493 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:46:32.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:32.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:32.494 | INFO | Uploaded 36GiB to GPU in 2.04463s (17.6071GiB/s) | 20260507 04:46:32.494 | INFO | Downloaded 34GiB to GPU in 1.94978s (17.4379GiB/s) | 20260507 04:46:37.636 | INFO | Analyzing chunk frames = [7202816,7203328) | 20260507 04:46:37.636 | INFO | MemInfo: { | 20260507 04:46:37.636 | INFO | "gpu" : " 17 GB", | 20260507 04:46:37.636 | INFO | "huge" : "", | 20260507 04:46:37.636 | INFO | "huge_shared" : " 27 GB", | 20260507 04:46:37.636 | INFO | "malloc" : " 14 GB", | 20260507 04:46:37.636 | INFO | "pinned" : " 29 GB" | 20260507 04:46:37.636 | INFO | } | 20260507 04:46:37.636 | INFO | Perf report for the last 5.14299 seconds: | 20260507 04:46:37.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:46:37.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:37.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:37.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:37.636 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 04:46:37.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:37.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:37.637 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:46:37.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:37.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:37.637 | INFO | Uploaded 36GiB to GPU in 2.03091s (17.726GiB/s) | 20260507 04:46:37.637 | INFO | Downloaded 34GiB to GPU in 1.93556s (17.566GiB/s) | 20260507 04:46:42.778 | INFO | Analyzing chunk frames = [7203328,7203840) | 20260507 04:46:42.778 | INFO | MemInfo: { | 20260507 04:46:42.778 | INFO | "gpu" : " 17 GB", | 20260507 04:46:42.778 | INFO | "huge" : "", | 20260507 04:46:42.778 | INFO | "huge_shared" : " 27 GB", | 20260507 04:46:42.778 | INFO | "malloc" : " 14 GB", | 20260507 04:46:42.778 | INFO | "pinned" : " 29 GB" | 20260507 04:46:42.778 | INFO | } | 20260507 04:46:42.778 | INFO | Perf report for the last 5.1419 seconds: | 20260507 04:46:42.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:46:42.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:42.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:42.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:42.778 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:46:42.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:42.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:42.778 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:46:42.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:42.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:42.778 | INFO | Uploaded 36GiB to GPU in 2.05017s (17.5595GiB/s) | 20260507 04:46:42.778 | INFO | Downloaded 34GiB to GPU in 1.96962s (17.2623GiB/s) | 20260507 04:46:47.919 | INFO | Analyzing chunk frames = [7203840,7204352) | 20260507 04:46:47.919 | INFO | MemInfo: { | 20260507 04:46:47.919 | INFO | "gpu" : " 17 GB", | 20260507 04:46:47.919 | INFO | "huge" : "", | 20260507 04:46:47.919 | INFO | "huge_shared" : " 27 GB", | 20260507 04:46:47.919 | INFO | "malloc" : " 14 GB", | 20260507 04:46:47.919 | INFO | "pinned" : " 29 GB" | 20260507 04:46:47.919 | INFO | } | 20260507 04:46:47.919 | INFO | Perf report for the last 5.14109 seconds: | 20260507 04:46:47.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:46:47.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:47.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:47.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:47.919 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:46:47.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:47.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:47.919 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 04:46:47.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:47.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:47.920 | INFO | Uploaded 36GiB to GPU in 1.983s (18.1543GiB/s) | 20260507 04:46:47.920 | INFO | Downloaded 34GiB to GPU in 1.98319s (17.1441GiB/s) | 20260507 04:46:53.057 | INFO | Analyzing chunk frames = [7204352,7204864) | 20260507 04:46:53.057 | INFO | MemInfo: { | 20260507 04:46:53.057 | INFO | "gpu" : " 17 GB", | 20260507 04:46:53.057 | INFO | "huge" : "", | 20260507 04:46:53.057 | INFO | "huge_shared" : " 27 GB", | 20260507 04:46:53.057 | INFO | "malloc" : " 14 GB", | 20260507 04:46:53.057 | INFO | "pinned" : " 29 GB" | 20260507 04:46:53.057 | INFO | } | 20260507 04:46:53.057 | INFO | Perf report for the last 5.13766 seconds: | 20260507 04:46:53.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:46:53.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:53.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:53.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:53.057 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:46:53.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:53.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:53.057 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 04:46:53.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:53.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:53.057 | INFO | Uploaded 36GiB to GPU in 1.99468s (18.048GiB/s) | 20260507 04:46:53.057 | INFO | Downloaded 34GiB to GPU in 1.97477s (17.2172GiB/s) | 20260507 04:46:53.154 | INFO | Starting a round of Baz IO | 20260507 04:46:56.254 | INFO | Populating RTMetrics report | 20260507 04:46:56.255 | INFO | RTMetrics report populated | 20260507 04:46:56.255 | INFO | Opening tmp RTMetrics json file | 20260507 04:46:56.258 | INFO | Serializing RTMetrics to json | 20260507 04:46:56.272 | INFO | Serialization done, swapping tmp file now... | 20260507 04:46:56.273 | INFO | RTMetrics done | 20260507 04:46:56.344 | INFO | Finished a round of Baz IO: Wrote 1.716 GiB in 3.189 seconds | 20260507 04:46:56.344 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.76% / 0.00% / 67.24% | 20260507 04:46:56.344 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.161227% / 0% / 6.75519e-05% | 20260507 04:46:56.344 | INFO | Baz IO overall: | 20260507 04:46:56.344 | INFO | BytesWritten : 3670142184743 | 20260507 04:46:56.344 | INFO | NumEvents : 1761444983437 | 20260507 04:46:58.192 | INFO | Analyzing chunk frames = [7204864,7205376) | 20260507 04:46:58.192 | INFO | MemInfo: { | 20260507 04:46:58.192 | INFO | "gpu" : " 17 GB", | 20260507 04:46:58.192 | INFO | "huge" : "", | 20260507 04:46:58.192 | INFO | "huge_shared" : " 27 GB", | 20260507 04:46:58.192 | INFO | "malloc" : " 14 GB", | 20260507 04:46:58.192 | INFO | "pinned" : " 29 GB" | 20260507 04:46:58.192 | INFO | } | 20260507 04:46:58.192 | INFO | Perf report for the last 5.13511 seconds: | 20260507 04:46:58.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:46:58.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:58.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:58.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:58.192 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 04:46:58.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:46:58.192 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:46:58.192 | INFO | PRE_HQ : 59.3%, 1.8, 55.4ms, 0 | 20260507 04:46:58.192 | INFO | BAZWRITER : 61.7%, 3.2, 101.5ms, 0 | 20260507 04:46:58.192 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:46:58.192 | INFO | Uploaded 36GiB to GPU in 2.04016s (17.6456GiB/s) | 20260507 04:46:58.192 | INFO | Downloaded 38GiB to GPU in 2.16798s (17.5278GiB/s) | 20260507 04:47:03.333 | INFO | Analyzing chunk frames = [7205376,7205888) | 20260507 04:47:03.333 | INFO | MemInfo: { | 20260507 04:47:03.333 | INFO | "gpu" : " 17 GB", | 20260507 04:47:03.333 | INFO | "huge" : "", | 20260507 04:47:03.333 | INFO | "huge_shared" : " 27 GB", | 20260507 04:47:03.333 | INFO | "malloc" : " 14 GB", | 20260507 04:47:03.333 | INFO | "pinned" : " 29 GB" | 20260507 04:47:03.333 | INFO | } | 20260507 04:47:03.334 | INFO | Perf report for the last 5.14134 seconds: | 20260507 04:47:03.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:47:03.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:03.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:03.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:03.334 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:47:03.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:03.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:03.334 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:47:03.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:03.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:03.334 | INFO | Uploaded 36GiB to GPU in 2.04465s (17.6069GiB/s) | 20260507 04:47:03.334 | INFO | Downloaded 34GiB to GPU in 1.94089s (17.5177GiB/s) | 20260507 04:47:08.474 | INFO | Analyzing chunk frames = [7205888,7206400) | 20260507 04:47:08.474 | INFO | MemInfo: { | 20260507 04:47:08.474 | INFO | "gpu" : " 17 GB", | 20260507 04:47:08.474 | INFO | "huge" : "", | 20260507 04:47:08.474 | INFO | "huge_shared" : " 27 GB", | 20260507 04:47:08.474 | INFO | "malloc" : " 14 GB", | 20260507 04:47:08.474 | INFO | "pinned" : " 29 GB" | 20260507 04:47:08.474 | INFO | } | 20260507 04:47:08.474 | INFO | Perf report for the last 5.14017 seconds: | 20260507 04:47:08.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:47:08.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:08.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:08.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:08.474 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 04:47:08.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:08.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:08.474 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:47:08.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:08.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:08.474 | INFO | Uploaded 36GiB to GPU in 2.01292s (17.8845GiB/s) | 20260507 04:47:08.474 | INFO | Downloaded 34GiB to GPU in 1.93911s (17.5338GiB/s) | 20260507 04:47:13.608 | INFO | Analyzing chunk frames = [7206400,7206912) | 20260507 04:47:13.608 | INFO | MemInfo: { | 20260507 04:47:13.608 | INFO | "gpu" : " 17 GB", | 20260507 04:47:13.608 | INFO | "huge" : "", | 20260507 04:47:13.608 | INFO | "huge_shared" : " 27 GB", | 20260507 04:47:13.608 | INFO | "malloc" : " 14 GB", | 20260507 04:47:13.608 | INFO | "pinned" : " 29 GB" | 20260507 04:47:13.608 | INFO | } | 20260507 04:47:13.608 | INFO | Perf report for the last 5.13458 seconds: | 20260507 04:47:13.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:47:13.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:13.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:13.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:13.608 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 04:47:13.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:13.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:13.608 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:47:13.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:13.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:13.608 | INFO | Uploaded 36GiB to GPU in 2.02251s (17.7996GiB/s) | 20260507 04:47:13.608 | INFO | Downloaded 34GiB to GPU in 1.93242s (17.5945GiB/s) | 20260507 04:47:18.753 | INFO | Analyzing chunk frames = [7206912,7207424) | 20260507 04:47:18.753 | INFO | MemInfo: { | 20260507 04:47:18.753 | INFO | "gpu" : " 17 GB", | 20260507 04:47:18.753 | INFO | "huge" : "", | 20260507 04:47:18.753 | INFO | "huge_shared" : " 27 GB", | 20260507 04:47:18.753 | INFO | "malloc" : " 14 GB", | 20260507 04:47:18.753 | INFO | "pinned" : " 29 GB" | 20260507 04:47:18.753 | INFO | } | 20260507 04:47:18.753 | INFO | Perf report for the last 5.14485 seconds: | 20260507 04:47:18.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:47:18.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:18.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:18.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:18.753 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 04:47:18.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:18.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:18.753 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:47:18.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:18.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:18.754 | INFO | Uploaded 36GiB to GPU in 2.04173s (17.6321GiB/s) | 20260507 04:47:18.754 | INFO | Downloaded 34GiB to GPU in 1.94358s (17.4935GiB/s) | 20260507 04:47:23.897 | INFO | Analyzing chunk frames = [7207424,7207936) | 20260507 04:47:23.897 | INFO | MemInfo: { | 20260507 04:47:23.897 | INFO | "gpu" : " 17 GB", | 20260507 04:47:23.897 | INFO | "huge" : "", | 20260507 04:47:23.897 | INFO | "huge_shared" : " 27 GB", | 20260507 04:47:23.897 | INFO | "malloc" : " 14 GB", | 20260507 04:47:23.897 | INFO | "pinned" : " 29 GB" | 20260507 04:47:23.897 | INFO | } | 20260507 04:47:23.897 | INFO | Perf report for the last 5.14353 seconds: | 20260507 04:47:23.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:47:23.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:23.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:23.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:23.897 | INFO | ANALYSIS : 60.4%, 3.0, 94.4ms, 0 | 20260507 04:47:23.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:23.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:23.897 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 04:47:23.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:23.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:23.897 | INFO | Uploaded 36GiB to GPU in 2.04906s (17.569GiB/s) | 20260507 04:47:23.897 | INFO | Downloaded 34GiB to GPU in 1.92468s (17.6653GiB/s) | 20260507 04:47:29.036 | INFO | Analyzing chunk frames = [7207936,7208448) | 20260507 04:47:29.036 | INFO | MemInfo: { | 20260507 04:47:29.036 | INFO | "gpu" : " 17 GB", | 20260507 04:47:29.036 | INFO | "huge" : "", | 20260507 04:47:29.036 | INFO | "huge_shared" : " 27 GB", | 20260507 04:47:29.036 | INFO | "malloc" : " 14 GB", | 20260507 04:47:29.036 | INFO | "pinned" : " 29 GB" | 20260507 04:47:29.036 | INFO | } | 20260507 04:47:29.036 | INFO | Perf report for the last 5.13924 seconds: | 20260507 04:47:29.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:47:29.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:29.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:29.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:29.036 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 04:47:29.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:29.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:29.036 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:47:29.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:29.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:29.036 | INFO | Uploaded 36GiB to GPU in 2.03026s (17.7318GiB/s) | 20260507 04:47:29.036 | INFO | Downloaded 34GiB to GPU in 1.97254s (17.2367GiB/s) | 20260507 04:47:34.173 | INFO | Analyzing chunk frames = [7208448,7208960) | 20260507 04:47:34.173 | INFO | MemInfo: { | 20260507 04:47:34.173 | INFO | "gpu" : " 17 GB", | 20260507 04:47:34.173 | INFO | "huge" : "", | 20260507 04:47:34.173 | INFO | "huge_shared" : " 27 GB", | 20260507 04:47:34.173 | INFO | "malloc" : " 14 GB", | 20260507 04:47:34.173 | INFO | "pinned" : " 29 GB" | 20260507 04:47:34.173 | INFO | } | 20260507 04:47:34.173 | INFO | Perf report for the last 5.13728 seconds: | 20260507 04:47:34.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:47:34.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:34.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:34.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:34.173 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 04:47:34.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:34.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:34.173 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:47:34.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:34.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:34.173 | INFO | Uploaded 36GiB to GPU in 1.95976s (18.3696GiB/s) | 20260507 04:47:34.174 | INFO | Downloaded 34GiB to GPU in 1.95017s (17.4344GiB/s) | 20260507 04:47:34.174 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:47:34.174 | INFO | Total mem usage: 18 GiB | 20260507 04:47:34.174 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:47:34.174 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 04:47:34.174 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:47:34.174 | INFO | High Water Breakdown: | 20260507 04:47:34.174 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:47:34.174 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:47:34.174 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:47:34.174 | INFO | ------------------------------------------ | 20260507 04:47:34.174 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:47:34.174 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:47:34.174 | INFO | Total mem usage: 26.5 GiB | 20260507 04:47:34.174 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:47:34.174 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:47:34.174 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:47:34.174 | INFO | High Water Breakdown: | 20260507 04:47:34.174 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:47:34.174 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:47:34.174 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:47:34.174 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:47:34.174 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:47:34.174 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:47:34.174 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:47:34.174 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:47:34.174 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:47:34.174 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:47:34.174 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:47:34.174 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:47:34.174 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:47:34.174 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:47:34.174 | INFO | ------------------------------------------ | 20260507 04:47:34.174 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:47:34.174 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:47:34.174 | INFO | Total mem usage: 25.6 GiB | 20260507 04:47:34.174 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:47:34.174 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:47:34.174 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:47:34.174 | INFO | High Water Breakdown: | 20260507 04:47:34.174 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:47:34.174 | INFO | ------------------------------------------ | 20260507 04:47:34.174 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:47:34.174 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:47:34.174 | INFO | Total mem usage: 15.7 GiB | 20260507 04:47:34.174 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:47:34.174 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:47:34.174 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:47:34.174 | INFO | High Water Breakdown: | 20260507 04:47:34.174 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:47:34.174 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:47:34.174 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:47:34.174 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:47:34.174 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:47:34.174 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:47:34.174 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:47:34.174 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:47:34.174 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:47:34.174 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:47:34.174 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:47:34.174 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:47:34.174 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:47:34.174 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:47:34.174 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:47:34.174 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:47:34.174 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:47:34.174 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:47:34.174 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:47:34.174 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:47:34.174 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:47:34.174 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:47:34.174 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:47:34.174 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:47:34.174 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:47:34.174 | INFO | ------------------------------------------ | 20260507 04:47:34.174 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:47:34.174 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:47:34.174 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:47:34.174 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:47:34.174 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.34e+01 / 9.64e-01 | 20260507 04:47:34.174 | INFO | PerfReport: 0.29%, AggregateStats -- 9.03e-02 / 1.01e-01, 9.78e-03 / 4.17e-03 | 20260507 04:47:34.174 | INFO | PerfReport: 11.64%, Baseline -- 3.57e+00 / 4.42e+00, 4.98e-01 / 2.38e-01 | 20260507 04:47:34.174 | INFO | PerfReport: 3.21%, Histogram -- 9.87e-01 / 1.48e+00, 1.65e-01 / 9.20e-02 | 20260507 04:47:34.174 | INFO | PerfReport: 16.14%, DME -- 3.97e+01 / 4.42e+01, 3.80e+00 / 1.46e+00 | 20260507 04:47:34.174 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.29e+00 / 1.51e+00 | 20260507 04:47:34.174 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.14e+00 / 2.66e+00, 3.85e-01 / 1.90e-01 | 20260507 04:47:34.174 | INFO | PerfReport: 3.02%, Metrics -- 9.26e-01 / 1.25e+00, 2.46e-01 / 1.07e-01 | 20260507 04:47:34.174 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.12e-01 / 3.68e-01, 4.94e-02 / 2.07e-02 | 20260507 04:47:34.174 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:47:34.174 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:47:34.174 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:47:34.174 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 04:47:34.174 | INFO | PerfReport: 21.86%, Upload -- 2.10e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260507 04:47:34.174 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 04:47:34.174 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:47:34.174 | INFO | PerfReport: 1.49%, LockUpload -- 1.43e+00 / 0.00e+00, 4.79e+00 / 0.00e+00 | 20260507 04:47:34.174 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260507 04:47:34.174 | INFO | PerfReport: 22.04%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 04:47:34.174 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.13e-02 / 0.00e+00 | 20260507 04:47:34.270 | INFO | Starting a round of Baz IO | 20260507 04:47:37.372 | INFO | Populating RTMetrics report | 20260507 04:47:37.372 | INFO | RTMetrics report populated | 20260507 04:47:37.372 | INFO | Opening tmp RTMetrics json file | 20260507 04:47:37.389 | INFO | Serializing RTMetrics to json | 20260507 04:47:37.391 | INFO | Serialization done, swapping tmp file now... | 20260507 04:47:37.391 | INFO | RTMetrics done | 20260507 04:47:37.464 | INFO | Finished a round of Baz IO: Wrote 1.716 GiB in 3.193 seconds | 20260507 04:47:37.464 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.76% / 0.00% / 67.24% | 20260507 04:47:37.464 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.161048% / 3.97364e-06% / 7.94729e-05% | 20260507 04:47:37.464 | INFO | Baz IO overall: | 20260507 04:47:37.464 | INFO | BytesWritten : 3671984959948 | 20260507 04:47:37.464 | INFO | NumEvents : 1762311819381 | 20260507 04:47:39.306 | INFO | Analyzing chunk frames = [7208960,7209472) | 20260507 04:47:39.306 | INFO | MemInfo: { | 20260507 04:47:39.306 | INFO | "gpu" : " 17 GB", | 20260507 04:47:39.306 | INFO | "huge" : "", | 20260507 04:47:39.306 | INFO | "huge_shared" : " 27 GB", | 20260507 04:47:39.306 | INFO | "malloc" : " 14 GB", | 20260507 04:47:39.306 | INFO | "pinned" : " 29 GB" | 20260507 04:47:39.306 | INFO | } | 20260507 04:47:39.306 | INFO | Perf report for the last 5.13253 seconds: | 20260507 04:47:39.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:47:39.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:39.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:39.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:39.306 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260507 04:47:39.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:39.306 | INFO | RT_METRICS : 41.7%, 1.0, 21.8ms, 0 | 20260507 04:47:39.306 | INFO | PRE_HQ : 59.1%, 1.8, 55.0ms, 0 | 20260507 04:47:39.306 | INFO | BAZWRITER : 61.8%, 3.5, 109.9ms, 0 | 20260507 04:47:39.306 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:47:39.306 | INFO | Uploaded 36GiB to GPU in 2.04727s (17.5844GiB/s) | 20260507 04:47:39.306 | INFO | Downloaded 38GiB to GPU in 2.1787s (17.4416GiB/s) | 20260507 04:47:44.443 | INFO | Analyzing chunk frames = [7209472,7209984) | 20260507 04:47:44.443 | INFO | MemInfo: { | 20260507 04:47:44.443 | INFO | "gpu" : " 17 GB", | 20260507 04:47:44.444 | INFO | "huge" : "", | 20260507 04:47:44.444 | INFO | "huge_shared" : " 27 GB", | 20260507 04:47:44.444 | INFO | "malloc" : " 14 GB", | 20260507 04:47:44.444 | INFO | "pinned" : " 29 GB" | 20260507 04:47:44.444 | INFO | } | 20260507 04:47:44.444 | INFO | Perf report for the last 5.13784 seconds: | 20260507 04:47:44.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:47:44.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:44.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:44.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:44.444 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 04:47:44.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:44.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:44.444 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 04:47:44.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:44.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:44.444 | INFO | Uploaded 36GiB to GPU in 2.05233s (17.541GiB/s) | 20260507 04:47:44.444 | INFO | Downloaded 34GiB to GPU in 1.94189s (17.5087GiB/s) | 20260507 04:47:49.588 | INFO | Analyzing chunk frames = [7209984,7210496) | 20260507 04:47:49.588 | INFO | MemInfo: { | 20260507 04:47:49.588 | INFO | "gpu" : " 17 GB", | 20260507 04:47:49.588 | INFO | "huge" : "", | 20260507 04:47:49.589 | INFO | "huge_shared" : " 27 GB", | 20260507 04:47:49.589 | INFO | "malloc" : " 14 GB", | 20260507 04:47:49.589 | INFO | "pinned" : " 29 GB" | 20260507 04:47:49.589 | INFO | } | 20260507 04:47:49.589 | INFO | Perf report for the last 5.14495 seconds: | 20260507 04:47:49.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:47:49.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:49.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:49.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:49.589 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 04:47:49.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:49.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:49.589 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:47:49.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:49.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:49.589 | INFO | Uploaded 36GiB to GPU in 2.01938s (17.8273GiB/s) | 20260507 04:47:49.589 | INFO | Downloaded 34GiB to GPU in 1.94151s (17.5122GiB/s) | 20260507 04:47:54.731 | INFO | Analyzing chunk frames = [7210496,7211008) | 20260507 04:47:54.731 | INFO | MemInfo: { | 20260507 04:47:54.731 | INFO | "gpu" : " 17 GB", | 20260507 04:47:54.731 | INFO | "huge" : "", | 20260507 04:47:54.731 | INFO | "huge_shared" : " 27 GB", | 20260507 04:47:54.731 | INFO | "malloc" : " 14 GB", | 20260507 04:47:54.731 | INFO | "pinned" : " 29 GB" | 20260507 04:47:54.731 | INFO | } | 20260507 04:47:54.731 | INFO | Perf report for the last 5.14223 seconds: | 20260507 04:47:54.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:47:54.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:54.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:54.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:54.731 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:47:54.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:54.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:54.731 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:47:54.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:54.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:54.731 | INFO | Uploaded 36GiB to GPU in 1.99203s (18.072GiB/s) | 20260507 04:47:54.731 | INFO | Downloaded 34GiB to GPU in 1.92613s (17.652GiB/s) | 20260507 04:47:59.875 | INFO | Analyzing chunk frames = [7211008,7211520) | 20260507 04:47:59.875 | INFO | MemInfo: { | 20260507 04:47:59.875 | INFO | "gpu" : " 17 GB", | 20260507 04:47:59.875 | INFO | "huge" : "", | 20260507 04:47:59.875 | INFO | "huge_shared" : " 27 GB", | 20260507 04:47:59.875 | INFO | "malloc" : " 14 GB", | 20260507 04:47:59.875 | INFO | "pinned" : " 29 GB" | 20260507 04:47:59.875 | INFO | } | 20260507 04:47:59.875 | INFO | Perf report for the last 5.14401 seconds: | 20260507 04:47:59.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:47:59.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:59.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:59.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:59.875 | INFO | ANALYSIS : 60.1%, 3.0, 94.0ms, 0 | 20260507 04:47:59.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:59.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:59.875 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:47:59.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:59.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:47:59.875 | INFO | Uploaded 36GiB to GPU in 2.05783s (17.4941GiB/s) | 20260507 04:47:59.875 | INFO | Downloaded 34GiB to GPU in 1.94879s (17.4467GiB/s) | 20260507 04:48:05.005 | INFO | Analyzing chunk frames = [7211520,7212032) | 20260507 04:48:05.005 | INFO | MemInfo: { | 20260507 04:48:05.005 | INFO | "gpu" : " 17 GB", | 20260507 04:48:05.006 | INFO | "huge" : "", | 20260507 04:48:05.006 | INFO | "huge_shared" : " 27 GB", | 20260507 04:48:05.006 | INFO | "malloc" : " 14 GB", | 20260507 04:48:05.006 | INFO | "pinned" : " 29 GB" | 20260507 04:48:05.006 | INFO | } | 20260507 04:48:05.006 | INFO | Perf report for the last 5.13078 seconds: | 20260507 04:48:05.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:48:05.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:05.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:05.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:05.006 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:48:05.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:05.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:05.006 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 04:48:05.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:05.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:05.006 | INFO | Uploaded 36GiB to GPU in 2.0695s (17.3955GiB/s) | 20260507 04:48:05.006 | INFO | Downloaded 34GiB to GPU in 1.93014s (17.6153GiB/s) | 20260507 04:48:10.146 | INFO | Analyzing chunk frames = [7212032,7212544) | 20260507 04:48:10.146 | INFO | MemInfo: { | 20260507 04:48:10.146 | INFO | "gpu" : " 17 GB", | 20260507 04:48:10.146 | INFO | "huge" : "", | 20260507 04:48:10.146 | INFO | "huge_shared" : " 27 GB", | 20260507 04:48:10.146 | INFO | "malloc" : " 14 GB", | 20260507 04:48:10.146 | INFO | "pinned" : " 29 GB" | 20260507 04:48:10.146 | INFO | } | 20260507 04:48:10.146 | INFO | Perf report for the last 5.14084 seconds: | 20260507 04:48:10.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:48:10.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:10.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:10.147 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:10.147 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:48:10.147 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:10.147 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:10.147 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:48:10.147 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:10.147 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:10.147 | INFO | Uploaded 36GiB to GPU in 1.92646s (18.6871GiB/s) | 20260507 04:48:10.147 | INFO | Downloaded 34GiB to GPU in 1.96347s (17.3163GiB/s) | 20260507 04:48:15.286 | INFO | Analyzing chunk frames = [7212544,7213056) | 20260507 04:48:15.286 | INFO | MemInfo: { | 20260507 04:48:15.286 | INFO | "gpu" : " 17 GB", | 20260507 04:48:15.286 | INFO | "huge" : "", | 20260507 04:48:15.286 | INFO | "huge_shared" : " 27 GB", | 20260507 04:48:15.286 | INFO | "malloc" : " 14 GB", | 20260507 04:48:15.286 | INFO | "pinned" : " 29 GB" | 20260507 04:48:15.286 | INFO | } | 20260507 04:48:15.286 | INFO | Perf report for the last 5.13935 seconds: | 20260507 04:48:15.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:48:15.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:15.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:15.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:15.286 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 04:48:15.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:15.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:15.286 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:48:15.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:15.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:15.286 | INFO | Uploaded 36GiB to GPU in 1.96869s (18.2863GiB/s) | 20260507 04:48:15.286 | INFO | Downloaded 34GiB to GPU in 1.94642s (17.4679GiB/s) | 20260507 04:48:15.383 | INFO | Starting a round of Baz IO | 20260507 04:48:18.486 | INFO | Populating RTMetrics report | 20260507 04:48:18.486 | INFO | RTMetrics report populated | 20260507 04:48:18.486 | INFO | Opening tmp RTMetrics json file | 20260507 04:48:18.490 | INFO | Serializing RTMetrics to json | 20260507 04:48:18.506 | INFO | Serialization done, swapping tmp file now... | 20260507 04:48:18.506 | INFO | RTMetrics done | 20260507 04:48:18.578 | INFO | Finished a round of Baz IO: Wrote 1.716 GiB in 3.194 seconds | 20260507 04:48:18.578 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.76% / 0.00% / 67.24% | 20260507 04:48:18.578 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.160718% / 0% / 9.13938e-05% | 20260507 04:48:18.578 | INFO | Baz IO overall: | 20260507 04:48:18.578 | INFO | BytesWritten : 3673827939270 | 20260507 04:48:18.578 | INFO | NumEvents : 1763178427561 | 20260507 04:48:20.426 | INFO | Analyzing chunk frames = [7213056,7213568) | 20260507 04:48:20.426 | INFO | MemInfo: { | 20260507 04:48:20.426 | INFO | "gpu" : " 17 GB", | 20260507 04:48:20.426 | INFO | "huge" : "", | 20260507 04:48:20.426 | INFO | "huge_shared" : " 27 GB", | 20260507 04:48:20.426 | INFO | "malloc" : " 14 GB", | 20260507 04:48:20.426 | INFO | "pinned" : " 29 GB" | 20260507 04:48:20.426 | INFO | } | 20260507 04:48:20.426 | INFO | Perf report for the last 5.13987 seconds: | 20260507 04:48:20.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:48:20.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:20.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:20.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:20.426 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 04:48:20.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:20.426 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 04:48:20.426 | INFO | PRE_HQ : 59.1%, 1.8, 55.0ms, 0 | 20260507 04:48:20.426 | INFO | BAZWRITER : 61.7%, 3.2, 102.8ms, 0 | 20260507 04:48:20.426 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:48:20.426 | INFO | Uploaded 36GiB to GPU in 2.03424s (17.697GiB/s) | 20260507 04:48:20.426 | INFO | Downloaded 38GiB to GPU in 2.17949s (17.4353GiB/s) | 20260507 04:48:25.560 | INFO | Analyzing chunk frames = [7213568,7214080) | 20260507 04:48:25.560 | INFO | MemInfo: { | 20260507 04:48:25.560 | INFO | "gpu" : " 17 GB", | 20260507 04:48:25.560 | INFO | "huge" : "", | 20260507 04:48:25.560 | INFO | "huge_shared" : " 27 GB", | 20260507 04:48:25.560 | INFO | "malloc" : " 14 GB", | 20260507 04:48:25.560 | INFO | "pinned" : " 29 GB" | 20260507 04:48:25.560 | INFO | } | 20260507 04:48:25.560 | INFO | Perf report for the last 5.13449 seconds: | 20260507 04:48:25.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:48:25.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:25.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:25.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:25.560 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 04:48:25.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:25.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:25.560 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:48:25.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:25.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:25.560 | INFO | Uploaded 36GiB to GPU in 1.98251s (18.1588GiB/s) | 20260507 04:48:25.560 | INFO | Downloaded 34GiB to GPU in 1.9286s (17.6293GiB/s) | 20260507 04:48:30.700 | INFO | Analyzing chunk frames = [7214080,7214592) | 20260507 04:48:30.700 | INFO | MemInfo: { | 20260507 04:48:30.700 | INFO | "gpu" : " 17 GB", | 20260507 04:48:30.700 | INFO | "huge" : "", | 20260507 04:48:30.700 | INFO | "huge_shared" : " 27 GB", | 20260507 04:48:30.700 | INFO | "malloc" : " 14 GB", | 20260507 04:48:30.700 | INFO | "pinned" : " 29 GB" | 20260507 04:48:30.700 | INFO | } | 20260507 04:48:30.700 | INFO | Perf report for the last 5.13957 seconds: | 20260507 04:48:30.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:48:30.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:30.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:30.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:30.700 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 04:48:30.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:30.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:30.700 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:48:30.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:30.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:30.700 | INFO | Uploaded 36GiB to GPU in 1.99045s (18.0864GiB/s) | 20260507 04:48:30.700 | INFO | Downloaded 34GiB to GPU in 1.94199s (17.5078GiB/s) | 20260507 04:48:35.849 | INFO | Analyzing chunk frames = [7214592,7215104) | 20260507 04:48:35.849 | INFO | MemInfo: { | 20260507 04:48:35.849 | INFO | "gpu" : " 17 GB", | 20260507 04:48:35.849 | INFO | "huge" : "", | 20260507 04:48:35.849 | INFO | "huge_shared" : " 27 GB", | 20260507 04:48:35.849 | INFO | "malloc" : " 14 GB", | 20260507 04:48:35.849 | INFO | "pinned" : " 29 GB" | 20260507 04:48:35.849 | INFO | } | 20260507 04:48:35.849 | INFO | Perf report for the last 5.14912 seconds: | 20260507 04:48:35.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:48:35.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:35.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:35.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:35.849 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 04:48:35.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:35.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:35.849 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:48:35.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:35.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:35.849 | INFO | Uploaded 36GiB to GPU in 1.96556s (18.3154GiB/s) | 20260507 04:48:35.849 | INFO | Downloaded 34GiB to GPU in 1.92552s (17.6575GiB/s) | 20260507 04:48:40.984 | INFO | Analyzing chunk frames = [7215104,7215616) | 20260507 04:48:40.984 | INFO | MemInfo: { | 20260507 04:48:40.984 | INFO | "gpu" : " 17 GB", | 20260507 04:48:40.984 | INFO | "huge" : "", | 20260507 04:48:40.984 | INFO | "huge_shared" : " 27 GB", | 20260507 04:48:40.984 | INFO | "malloc" : " 14 GB", | 20260507 04:48:40.984 | INFO | "pinned" : " 29 GB" | 20260507 04:48:40.984 | INFO | } | 20260507 04:48:40.984 | INFO | Perf report for the last 5.13552 seconds: | 20260507 04:48:40.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:48:40.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:40.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:40.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:40.985 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 04:48:40.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:40.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:40.985 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:48:40.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:40.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:40.985 | INFO | Uploaded 36GiB to GPU in 2.05031s (17.5583GiB/s) | 20260507 04:48:40.985 | INFO | Downloaded 34GiB to GPU in 1.95345s (17.4051GiB/s) | 20260507 04:48:46.125 | INFO | Analyzing chunk frames = [7215616,7216128) | 20260507 04:48:46.125 | INFO | MemInfo: { | 20260507 04:48:46.125 | INFO | "gpu" : " 17 GB", | 20260507 04:48:46.125 | INFO | "huge" : "", | 20260507 04:48:46.125 | INFO | "huge_shared" : " 27 GB", | 20260507 04:48:46.125 | INFO | "malloc" : " 14 GB", | 20260507 04:48:46.125 | INFO | "pinned" : " 29 GB" | 20260507 04:48:46.125 | INFO | } | 20260507 04:48:46.125 | INFO | Perf report for the last 5.14036 seconds: | 20260507 04:48:46.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:48:46.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:46.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:46.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:46.125 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 04:48:46.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:46.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:46.125 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 04:48:46.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:46.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:46.125 | INFO | Uploaded 36GiB to GPU in 2.05971s (17.4782GiB/s) | 20260507 04:48:46.125 | INFO | Downloaded 34GiB to GPU in 1.92611s (17.6522GiB/s) | 20260507 04:48:51.255 | INFO | Analyzing chunk frames = [7216128,7216640) | 20260507 04:48:51.255 | INFO | MemInfo: { | 20260507 04:48:51.255 | INFO | "gpu" : " 17 GB", | 20260507 04:48:51.255 | INFO | "huge" : "", | 20260507 04:48:51.256 | INFO | "huge_shared" : " 27 GB", | 20260507 04:48:51.256 | INFO | "malloc" : " 14 GB", | 20260507 04:48:51.256 | INFO | "pinned" : " 29 GB" | 20260507 04:48:51.256 | INFO | } | 20260507 04:48:51.256 | INFO | Perf report for the last 5.13073 seconds: | 20260507 04:48:51.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:48:51.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:51.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:51.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:51.256 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 04:48:51.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:51.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:51.256 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:48:51.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:51.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:51.256 | INFO | Uploaded 36GiB to GPU in 2.03147s (17.7211GiB/s) | 20260507 04:48:51.256 | INFO | Downloaded 34GiB to GPU in 1.97697s (17.1981GiB/s) | 20260507 04:48:56.397 | INFO | Analyzing chunk frames = [7216640,7217152) | 20260507 04:48:56.397 | INFO | MemInfo: { | 20260507 04:48:56.397 | INFO | "gpu" : " 17 GB", | 20260507 04:48:56.397 | INFO | "huge" : "", | 20260507 04:48:56.397 | INFO | "huge_shared" : " 27 GB", | 20260507 04:48:56.397 | INFO | "malloc" : " 14 GB", | 20260507 04:48:56.397 | INFO | "pinned" : " 29 GB" | 20260507 04:48:56.397 | INFO | } | 20260507 04:48:56.397 | INFO | Perf report for the last 5.14169 seconds: | 20260507 04:48:56.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:48:56.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:56.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:56.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:56.397 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:48:56.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:56.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:56.397 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 04:48:56.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:56.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:48:56.398 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:48:56.398 | INFO | Total mem usage: 18 GiB | 20260507 04:48:56.398 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:48:56.398 | INFO | Recent High Water Marks Sum: 14.4 GiB (Savings: -24.9%) | 20260507 04:48:56.398 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:48:56.398 | INFO | High Water Breakdown: | 20260507 04:48:56.398 | INFO | 60.3% : 8.69 GiB ( 76.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:48:56.398 | INFO | 39.6% : 5.71 GiB ( 85.3% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:48:56.398 | INFO | 0.138% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:48:56.398 | INFO | ------------------------------------------ | 20260507 04:48:56.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:48:56.398 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:48:56.398 | INFO | Total mem usage: 26.5 GiB | 20260507 04:48:56.398 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:48:56.398 | INFO | Uploaded 36GiB to GPU in 1.96615s (18.3099GiB/s) | 20260507 04:48:56.398 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:48:56.398 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 04:48:56.398 | INFO | Downloaded 34GiB to GPU in 1.95318s (17.4075GiB/s) | 20260507 04:48:56.398 | INFO | High Water Breakdown: | 20260507 04:48:56.398 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:48:56.398 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:48:56.398 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:48:56.398 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:48:56.398 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:48:56.398 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:48:56.398 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:48:56.398 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:48:56.398 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:48:56.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:48:56.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:48:56.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:48:56.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:48:56.398 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:48:56.398 | INFO | ------------------------------------------ | 20260507 04:48:56.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:48:56.398 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:48:56.398 | INFO | Total mem usage: 25.6 GiB | 20260507 04:48:56.398 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:48:56.398 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:48:56.398 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:48:56.398 | INFO | High Water Breakdown: | 20260507 04:48:56.399 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:48:56.399 | INFO | ------------------------------------------ | 20260507 04:48:56.399 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:48:56.399 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:48:56.399 | INFO | Total mem usage: 15.7 GiB | 20260507 04:48:56.399 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:48:56.399 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:48:56.399 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:48:56.399 | INFO | High Water Breakdown: | 20260507 04:48:56.399 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:48:56.399 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:48:56.399 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:48:56.399 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:48:56.399 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:48:56.399 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:48:56.399 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:48:56.399 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:48:56.399 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:48:56.399 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:48:56.399 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:48:56.399 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:48:56.399 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:48:56.399 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:48:56.399 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:48:56.399 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:48:56.399 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:48:56.399 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:48:56.399 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:48:56.399 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:48:56.399 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:48:56.399 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:48:56.399 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:48:56.399 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:48:56.399 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:48:56.399 | INFO | ------------------------------------------ | 20260507 04:48:56.399 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:48:56.399 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:48:56.399 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:48:56.399 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:48:56.399 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.44e+01, 1.34e+01 / 1.25e+00 | 20260507 04:48:56.399 | INFO | PerfReport: 0.30%, AggregateStats -- 9.06e-02 / 1.01e-01, 1.28e-02 / 6.58e-03 | 20260507 04:48:56.399 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.48e+00, 5.10e-01 / 2.22e-01 | 20260507 04:48:56.399 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.45e+00, 1.65e-01 / 1.35e-01 | 20260507 04:48:56.399 | INFO | PerfReport: 16.08%, DME -- 3.95e+01 / 4.38e+01, 3.64e+00 / 1.26e+00 | 20260507 04:48:56.399 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.30e+00 / 1.57e+00 | 20260507 04:48:56.399 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.14e+00 / 2.68e+00, 3.85e-01 / 2.12e-01 | 20260507 04:48:56.399 | INFO | PerfReport: 3.01%, Metrics -- 9.25e-01 / 1.22e+00, 2.43e-01 / 1.10e-01 | 20260507 04:48:56.399 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.12e-01 / 3.70e-01, 4.94e-02 / 2.02e-02 | 20260507 04:48:56.399 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:48:56.399 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:48:56.399 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:48:56.399 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.34e+01 / 0.00e+00 | 20260507 04:48:56.399 | INFO | PerfReport: 21.80%, Upload -- 2.10e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 04:48:56.399 | INFO | PerfReport: 21.36%, Download -- 2.05e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 04:48:56.399 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:48:56.399 | INFO | PerfReport: 1.50%, LockUpload -- 1.44e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 04:48:56.399 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260507 04:48:56.399 | INFO | PerfReport: 22.11%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 04:48:56.399 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260507 04:48:56.495 | INFO | Starting a round of Baz IO | 20260507 04:48:59.597 | INFO | Populating RTMetrics report | 20260507 04:48:59.597 | INFO | RTMetrics report populated | 20260507 04:48:59.597 | INFO | Opening tmp RTMetrics json file | 20260507 04:48:59.598 | INFO | Serializing RTMetrics to json | 20260507 04:48:59.615 | INFO | Serialization done, swapping tmp file now... | 20260507 04:48:59.617 | INFO | RTMetrics done | 20260507 04:48:59.688 | INFO | Finished a round of Baz IO: Wrote 1.715 GiB in 3.192 seconds | 20260507 04:48:59.688 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.76% / 0.00% / 67.24% | 20260507 04:48:59.688 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.160285% / 0% / 9.13938e-05% | 20260507 04:48:59.688 | INFO | Baz IO overall: | 20260507 04:48:59.688 | INFO | BytesWritten : 3675669585364 | 20260507 04:48:59.688 | INFO | NumEvents : 1764044295619 | 20260507 04:49:01.543 | INFO | Analyzing chunk frames = [7217152,7217664) | 20260507 04:49:01.543 | INFO | MemInfo: { | 20260507 04:49:01.543 | INFO | "gpu" : " 17 GB", | 20260507 04:49:01.543 | INFO | "huge" : "", | 20260507 04:49:01.543 | INFO | "huge_shared" : " 27 GB", | 20260507 04:49:01.543 | INFO | "malloc" : " 14 GB", | 20260507 04:49:01.543 | INFO | "pinned" : " 29 GB" | 20260507 04:49:01.543 | INFO | } | 20260507 04:49:01.543 | INFO | Perf report for the last 5.14588 seconds: | 20260507 04:49:01.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:49:01.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:01.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:01.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:01.543 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 04:49:01.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:01.543 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 04:49:01.543 | INFO | PRE_HQ : 58.8%, 1.8, 55.2ms, 0 | 20260507 04:49:01.543 | INFO | BAZWRITER : 61.6%, 3.3, 104.0ms, 0 | 20260507 04:49:01.543 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:49:01.543 | INFO | Uploaded 36GiB to GPU in 2.03668s (17.6758GiB/s) | 20260507 04:49:01.543 | INFO | Downloaded 38GiB to GPU in 2.17513s (17.4702GiB/s) | 20260507 04:49:06.682 | INFO | Analyzing chunk frames = [7217664,7218176) | 20260507 04:49:06.682 | INFO | MemInfo: { | 20260507 04:49:06.682 | INFO | "gpu" : " 17 GB", | 20260507 04:49:06.682 | INFO | "huge" : "", | 20260507 04:49:06.682 | INFO | "huge_shared" : " 27 GB", | 20260507 04:49:06.682 | INFO | "malloc" : " 14 GB", | 20260507 04:49:06.682 | INFO | "pinned" : " 29 GB" | 20260507 04:49:06.682 | INFO | } | 20260507 04:49:06.682 | INFO | Perf report for the last 5.13865 seconds: | 20260507 04:49:06.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:49:06.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:06.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:06.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:06.682 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 04:49:06.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:06.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:06.682 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:49:06.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:06.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:06.682 | INFO | Uploaded 36GiB to GPU in 1.99949s (18.0046GiB/s) | 20260507 04:49:06.682 | INFO | Downloaded 34GiB to GPU in 1.93039s (17.613GiB/s) | 20260507 04:49:11.822 | INFO | Analyzing chunk frames = [7218176,7218688) | 20260507 04:49:11.822 | INFO | MemInfo: { | 20260507 04:49:11.822 | INFO | "gpu" : " 17 GB", | 20260507 04:49:11.822 | INFO | "huge" : "", | 20260507 04:49:11.822 | INFO | "huge_shared" : " 27 GB", | 20260507 04:49:11.822 | INFO | "malloc" : " 14 GB", | 20260507 04:49:11.822 | INFO | "pinned" : " 29 GB" | 20260507 04:49:11.822 | INFO | } | 20260507 04:49:11.822 | INFO | Perf report for the last 5.13985 seconds: | 20260507 04:49:11.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:49:11.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:11.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:11.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:11.822 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 04:49:11.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:11.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:11.822 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:49:11.822 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:11.822 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:11.822 | INFO | Uploaded 36GiB to GPU in 2.04975s (17.5631GiB/s) | 20260507 04:49:11.822 | INFO | Downloaded 34GiB to GPU in 1.94881s (17.4466GiB/s) | 20260507 04:49:16.955 | INFO | Analyzing chunk frames = [7218688,7219200) | 20260507 04:49:16.955 | INFO | MemInfo: { | 20260507 04:49:16.955 | INFO | "gpu" : " 17 GB", | 20260507 04:49:16.955 | INFO | "huge" : "", | 20260507 04:49:16.955 | INFO | "huge_shared" : " 27 GB", | 20260507 04:49:16.955 | INFO | "malloc" : " 14 GB", | 20260507 04:49:16.955 | INFO | "pinned" : " 29 GB" | 20260507 04:49:16.955 | INFO | } | 20260507 04:49:16.955 | INFO | Perf report for the last 5.13375 seconds: | 20260507 04:49:16.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:49:16.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:16.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:16.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:16.955 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 04:49:16.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:16.956 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:16.956 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:49:16.956 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:16.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:16.956 | INFO | Uploaded 36GiB to GPU in 2.01156s (17.8965GiB/s) | 20260507 04:49:16.956 | INFO | Downloaded 34GiB to GPU in 1.93756s (17.5479GiB/s) | 20260507 04:49:22.102 | INFO | Analyzing chunk frames = [7219200,7219712) | 20260507 04:49:22.102 | INFO | MemInfo: { | 20260507 04:49:22.102 | INFO | "gpu" : " 17 GB", | 20260507 04:49:22.102 | INFO | "huge" : "", | 20260507 04:49:22.102 | INFO | "huge_shared" : " 27 GB", | 20260507 04:49:22.102 | INFO | "malloc" : " 14 GB", | 20260507 04:49:22.102 | INFO | "pinned" : " 29 GB" | 20260507 04:49:22.102 | INFO | } | 20260507 04:49:22.102 | INFO | Perf report for the last 5.14669 seconds: | 20260507 04:49:22.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:49:22.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:22.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:22.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:22.103 | INFO | ANALYSIS : 59.9%, 3.0, 93.7ms, 0 | 20260507 04:49:22.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:22.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:22.103 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:49:22.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:22.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:22.103 | INFO | Uploaded 36GiB to GPU in 2.02104s (17.8126GiB/s) | 20260507 04:49:22.103 | INFO | Downloaded 34GiB to GPU in 1.93422s (17.5782GiB/s) | 20260507 04:49:27.237 | INFO | Analyzing chunk frames = [7219712,7220224) | 20260507 04:49:27.237 | INFO | MemInfo: { | 20260507 04:49:27.237 | INFO | "gpu" : " 17 GB", | 20260507 04:49:27.237 | INFO | "huge" : "", | 20260507 04:49:27.237 | INFO | "huge_shared" : " 27 GB", | 20260507 04:49:27.237 | INFO | "malloc" : " 14 GB", | 20260507 04:49:27.237 | INFO | "pinned" : " 29 GB" | 20260507 04:49:27.237 | INFO | } | 20260507 04:49:27.237 | INFO | Perf report for the last 5.13519 seconds: | 20260507 04:49:27.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:49:27.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:27.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:27.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:27.237 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 04:49:27.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:27.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:27.237 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:49:27.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:27.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:27.238 | INFO | Uploaded 36GiB to GPU in 2.04204s (17.6294GiB/s) | 20260507 04:49:27.238 | INFO | Downloaded 34GiB to GPU in 1.91153s (17.7868GiB/s) | 20260507 04:49:32.379 | INFO | Analyzing chunk frames = [7220224,7220736) | 20260507 04:49:32.379 | INFO | MemInfo: { | 20260507 04:49:32.379 | INFO | "gpu" : " 17 GB", | 20260507 04:49:32.380 | INFO | "huge" : "", | 20260507 04:49:32.380 | INFO | "huge_shared" : " 27 GB", | 20260507 04:49:32.380 | INFO | "malloc" : " 14 GB", | 20260507 04:49:32.380 | INFO | "pinned" : " 29 GB" | 20260507 04:49:32.380 | INFO | } | 20260507 04:49:32.380 | INFO | Perf report for the last 5.14222 seconds: | 20260507 04:49:32.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:49:32.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:32.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:32.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:32.380 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 04:49:32.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:32.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:32.380 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:49:32.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:32.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:32.380 | INFO | Uploaded 36GiB to GPU in 1.96539s (18.3169GiB/s) | 20260507 04:49:32.380 | INFO | Downloaded 34GiB to GPU in 1.98479s (17.1303GiB/s) | 20260507 04:49:37.513 | INFO | Analyzing chunk frames = [7220736,7221248) | 20260507 04:49:37.513 | INFO | MemInfo: { | 20260507 04:49:37.513 | INFO | "gpu" : " 17 GB", | 20260507 04:49:37.513 | INFO | "huge" : "", | 20260507 04:49:37.513 | INFO | "huge_shared" : " 27 GB", | 20260507 04:49:37.513 | INFO | "malloc" : " 14 GB", | 20260507 04:49:37.513 | INFO | "pinned" : " 29 GB" | 20260507 04:49:37.513 | INFO | } | 20260507 04:49:37.513 | INFO | Perf report for the last 5.13356 seconds: | 20260507 04:49:37.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:49:37.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:37.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:37.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:37.513 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 04:49:37.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:37.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:37.513 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:49:37.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:37.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:37.513 | INFO | Uploaded 36GiB to GPU in 1.98205s (18.163GiB/s) | 20260507 04:49:37.513 | INFO | Downloaded 34GiB to GPU in 1.96948s (17.2635GiB/s) | 20260507 04:49:37.610 | INFO | Starting a round of Baz IO | 20260507 04:49:40.709 | INFO | Populating RTMetrics report | 20260507 04:49:40.710 | INFO | RTMetrics report populated | 20260507 04:49:40.710 | INFO | Opening tmp RTMetrics json file | 20260507 04:49:40.718 | INFO | Serializing RTMetrics to json | 20260507 04:49:40.726 | INFO | Serialization done, swapping tmp file now... | 20260507 04:49:40.726 | INFO | RTMetrics done | 20260507 04:49:40.802 | INFO | Finished a round of Baz IO: Wrote 1.715 GiB in 3.191 seconds | 20260507 04:49:40.802 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.76% / 0.00% / 67.24% | 20260507 04:49:40.802 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.160162% / 0% / 6.35783e-05% | 20260507 04:49:40.802 | INFO | Baz IO overall: | 20260507 04:49:40.802 | INFO | BytesWritten : 3677511548865 | 20260507 04:49:40.802 | INFO | NumEvents : 1764911082430 | 20260507 04:49:42.654 | INFO | Analyzing chunk frames = [7221248,7221760) | 20260507 04:49:42.654 | INFO | MemInfo: { | 20260507 04:49:42.654 | INFO | "gpu" : " 17 GB", | 20260507 04:49:42.654 | INFO | "huge" : "", | 20260507 04:49:42.655 | INFO | "huge_shared" : " 27 GB", | 20260507 04:49:42.655 | INFO | "malloc" : " 14 GB", | 20260507 04:49:42.655 | INFO | "pinned" : " 29 GB" | 20260507 04:49:42.655 | INFO | } | 20260507 04:49:42.655 | INFO | Perf report for the last 5.14143 seconds: | 20260507 04:49:42.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:49:42.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:42.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:42.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:42.655 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 04:49:42.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:42.655 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 04:49:42.655 | INFO | PRE_HQ : 58.9%, 1.8, 54.8ms, 0 | 20260507 04:49:42.655 | INFO | BAZWRITER : 61.6%, 3.3, 103.3ms, 0 | 20260507 04:49:42.655 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:49:42.655 | INFO | Uploaded 36GiB to GPU in 2.02895s (17.7432GiB/s) | 20260507 04:49:42.655 | INFO | Downloaded 38GiB to GPU in 2.17489s (17.4721GiB/s) | 20260507 04:49:47.793 | INFO | Analyzing chunk frames = [7221760,7222272) | 20260507 04:49:47.793 | INFO | MemInfo: { | 20260507 04:49:47.793 | INFO | "gpu" : " 17 GB", | 20260507 04:49:47.793 | INFO | "huge" : "", | 20260507 04:49:47.793 | INFO | "huge_shared" : " 27 GB", | 20260507 04:49:47.793 | INFO | "malloc" : " 14 GB", | 20260507 04:49:47.793 | INFO | "pinned" : " 29 GB" | 20260507 04:49:47.793 | INFO | } | 20260507 04:49:47.793 | INFO | Perf report for the last 5.1388 seconds: | 20260507 04:49:47.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:49:47.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:47.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:47.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:47.793 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 04:49:47.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:47.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:47.793 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:49:47.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:47.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:47.794 | INFO | Uploaded 36GiB to GPU in 2.01199s (17.8928GiB/s) | 20260507 04:49:47.794 | INFO | Downloaded 34GiB to GPU in 1.9387s (17.5375GiB/s) | 20260507 04:49:52.934 | INFO | Analyzing chunk frames = [7222272,7222784) | 20260507 04:49:52.934 | INFO | MemInfo: { | 20260507 04:49:52.934 | INFO | "gpu" : " 17 GB", | 20260507 04:49:52.934 | INFO | "huge" : "", | 20260507 04:49:52.934 | INFO | "huge_shared" : " 27 GB", | 20260507 04:49:52.934 | INFO | "malloc" : " 14 GB", | 20260507 04:49:52.934 | INFO | "pinned" : " 29 GB" | 20260507 04:49:52.934 | INFO | } | 20260507 04:49:52.934 | INFO | Perf report for the last 5.14094 seconds: | 20260507 04:49:52.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:49:52.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:52.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:52.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:52.934 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 04:49:52.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:52.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:52.934 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:49:52.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:52.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:52.935 | INFO | Uploaded 36GiB to GPU in 2.00619s (17.9444GiB/s) | 20260507 04:49:52.935 | INFO | Downloaded 34GiB to GPU in 1.9568s (17.3753GiB/s) | 20260507 04:49:58.072 | INFO | Analyzing chunk frames = [7222784,7223296) | 20260507 04:49:58.072 | INFO | MemInfo: { | 20260507 04:49:58.072 | INFO | "gpu" : " 17 GB", | 20260507 04:49:58.072 | INFO | "huge" : "", | 20260507 04:49:58.072 | INFO | "huge_shared" : " 27 GB", | 20260507 04:49:58.072 | INFO | "malloc" : " 14 GB", | 20260507 04:49:58.072 | INFO | "pinned" : " 29 GB" | 20260507 04:49:58.072 | INFO | } | 20260507 04:49:58.072 | INFO | Perf report for the last 5.13727 seconds: | 20260507 04:49:58.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:49:58.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:58.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:58.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:58.072 | INFO | ANALYSIS : 60.3%, 2.9, 93.1ms, 0 | 20260507 04:49:58.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:58.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:58.072 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:49:58.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:58.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:49:58.072 | INFO | Uploaded 36GiB to GPU in 2.01256s (17.8876GiB/s) | 20260507 04:49:58.072 | INFO | Downloaded 34GiB to GPU in 1.95533s (17.3884GiB/s) | 20260507 04:50:03.208 | INFO | Analyzing chunk frames = [7223296,7223808) | 20260507 04:50:03.208 | INFO | MemInfo: { | 20260507 04:50:03.208 | INFO | "gpu" : " 17 GB", | 20260507 04:50:03.208 | INFO | "huge" : "", | 20260507 04:50:03.208 | INFO | "huge_shared" : " 27 GB", | 20260507 04:50:03.208 | INFO | "malloc" : " 14 GB", | 20260507 04:50:03.208 | INFO | "pinned" : " 29 GB" | 20260507 04:50:03.208 | INFO | } | 20260507 04:50:03.208 | INFO | Perf report for the last 5.13603 seconds: | 20260507 04:50:03.208 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:50:03.208 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:03.208 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:03.208 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:03.208 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 04:50:03.208 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:03.208 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:03.208 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:50:03.208 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:03.208 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:03.208 | INFO | Uploaded 36GiB to GPU in 2.03315s (17.7065GiB/s) | 20260507 04:50:03.208 | INFO | Downloaded 34GiB to GPU in 1.94425s (17.4875GiB/s) | 20260507 04:50:08.354 | INFO | Analyzing chunk frames = [7223808,7224320) | 20260507 04:50:08.354 | INFO | MemInfo: { | 20260507 04:50:08.354 | INFO | "gpu" : " 17 GB", | 20260507 04:50:08.354 | INFO | "huge" : "", | 20260507 04:50:08.354 | INFO | "huge_shared" : " 27 GB", | 20260507 04:50:08.354 | INFO | "malloc" : " 14 GB", | 20260507 04:50:08.354 | INFO | "pinned" : " 29 GB" | 20260507 04:50:08.354 | INFO | } | 20260507 04:50:08.354 | INFO | Perf report for the last 5.14641 seconds: | 20260507 04:50:08.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:50:08.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:08.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:08.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:08.354 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 04:50:08.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:08.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:08.354 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 04:50:08.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:08.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:08.354 | INFO | Uploaded 36GiB to GPU in 2.02271s (17.7979GiB/s) | 20260507 04:50:08.354 | INFO | Downloaded 34GiB to GPU in 1.92097s (17.6994GiB/s) | 20260507 04:50:13.496 | INFO | Analyzing chunk frames = [7224320,7224832) | 20260507 04:50:13.496 | INFO | MemInfo: { | 20260507 04:50:13.496 | INFO | "gpu" : " 17 GB", | 20260507 04:50:13.496 | INFO | "huge" : "", | 20260507 04:50:13.496 | INFO | "huge_shared" : " 27 GB", | 20260507 04:50:13.496 | INFO | "malloc" : " 14 GB", | 20260507 04:50:13.496 | INFO | "pinned" : " 29 GB" | 20260507 04:50:13.496 | INFO | } | 20260507 04:50:13.496 | INFO | Perf report for the last 5.14209 seconds: | 20260507 04:50:13.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:50:13.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:13.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:13.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:13.496 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:50:13.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:13.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:13.496 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:50:13.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:13.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:13.497 | INFO | Uploaded 36GiB to GPU in 1.97355s (18.2412GiB/s) | 20260507 04:50:13.497 | INFO | Downloaded 34GiB to GPU in 1.97369s (17.2266GiB/s) | 20260507 04:50:18.629 | INFO | Analyzing chunk frames = [7224832,7225344) | 20260507 04:50:18.629 | INFO | MemInfo: { | 20260507 04:50:18.629 | INFO | "gpu" : " 17 GB", | 20260507 04:50:18.629 | INFO | "huge" : "", | 20260507 04:50:18.629 | INFO | "huge_shared" : " 27 GB", | 20260507 04:50:18.629 | INFO | "malloc" : " 14 GB", | 20260507 04:50:18.629 | INFO | "pinned" : " 29 GB" | 20260507 04:50:18.629 | INFO | } | 20260507 04:50:18.629 | INFO | Perf report for the last 5.13308 seconds: | 20260507 04:50:18.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:50:18.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:18.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:18.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:18.629 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260507 04:50:18.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:18.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:18.629 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 04:50:18.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:18.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:18.630 | INFO | Uploaded 36GiB to GPU in 1.98322s (18.1523GiB/s) | 20260507 04:50:18.630 | INFO | Downloaded 34GiB to GPU in 1.97201s (17.2413GiB/s) | 20260507 04:50:18.630 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:50:18.630 | INFO | Total mem usage: 18 GiB | 20260507 04:50:18.630 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:50:18.630 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 04:50:18.630 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:50:18.630 | INFO | High Water Breakdown: | 20260507 04:50:18.630 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:50:18.630 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:50:18.630 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:50:18.630 | INFO | ------------------------------------------ | 20260507 04:50:18.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:50:18.630 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:50:18.630 | INFO | Total mem usage: 26.5 GiB | 20260507 04:50:18.630 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:50:18.630 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:50:18.630 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:50:18.630 | INFO | High Water Breakdown: | 20260507 04:50:18.630 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:50:18.630 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:50:18.630 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:50:18.630 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:50:18.630 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:50:18.630 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:50:18.630 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:50:18.630 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:50:18.630 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:50:18.630 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:50:18.630 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:50:18.630 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:50:18.630 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:50:18.630 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:50:18.630 | INFO | ------------------------------------------ | 20260507 04:50:18.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:50:18.630 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:50:18.630 | INFO | Total mem usage: 25.6 GiB | 20260507 04:50:18.630 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:50:18.630 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:50:18.630 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:50:18.630 | INFO | High Water Breakdown: | 20260507 04:50:18.630 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:50:18.630 | INFO | ------------------------------------------ | 20260507 04:50:18.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:50:18.630 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:50:18.630 | INFO | Total mem usage: 15.7 GiB | 20260507 04:50:18.630 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:50:18.630 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:50:18.630 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:50:18.630 | INFO | High Water Breakdown: | 20260507 04:50:18.630 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:50:18.630 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:50:18.630 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:50:18.630 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:50:18.630 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:50:18.630 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:50:18.630 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:50:18.630 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:50:18.630 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:50:18.630 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:50:18.630 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:50:18.630 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:50:18.630 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:50:18.630 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:50:18.630 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:50:18.630 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:50:18.630 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:50:18.630 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:50:18.630 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:50:18.630 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:50:18.630 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:50:18.630 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:50:18.630 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:50:18.630 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:50:18.630 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:50:18.630 | INFO | ------------------------------------------ | 20260507 04:50:18.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:50:18.630 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:50:18.630 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:50:18.630 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:50:18.630 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.33e+01 / 1.38e+00 | 20260507 04:50:18.630 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 9.91e-02, 9.42e-03 / 5.10e-03 | 20260507 04:50:18.630 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.47e+00, 5.20e-01 / 3.08e-01 | 20260507 04:50:18.630 | INFO | PerfReport: 3.24%, Histogram -- 9.96e-01 / 1.45e+00, 1.69e-01 / 1.22e-01 | 20260507 04:50:18.630 | INFO | PerfReport: 16.25%, DME -- 4.00e+01 / 4.49e+01, 4.06e+00 / 1.59e+00 | 20260507 04:50:18.630 | INFO | PerfReport: 57.56%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.23e+00 / 1.25e+00 | 20260507 04:50:18.630 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.72e+00, 3.84e-01 / 2.04e-01 | 20260507 04:50:18.630 | INFO | PerfReport: 3.01%, Metrics -- 9.25e-01 / 1.26e+00, 2.45e-01 / 1.23e-01 | 20260507 04:50:18.630 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.11e-01 / 3.67e-01, 4.99e-02 / 2.06e-02 | 20260507 04:50:18.631 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:50:18.631 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:50:18.631 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:50:18.631 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 04:50:18.631 | INFO | PerfReport: 21.77%, Upload -- 2.10e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260507 04:50:18.631 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 04:50:18.631 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 04:50:18.631 | INFO | PerfReport: 1.49%, LockUpload -- 1.44e+00 / 0.00e+00, 4.79e+00 / 0.00e+00 | 20260507 04:50:18.631 | INFO | PerfReport: 1.38%, LockDownload -- 1.33e+00 / 0.00e+00, 3.21e+00 / 0.00e+00 | 20260507 04:50:18.631 | INFO | PerfReport: 22.01%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 04:50:18.631 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260507 04:50:18.726 | INFO | Starting a round of Baz IO | 20260507 04:50:21.823 | INFO | Populating RTMetrics report | 20260507 04:50:21.823 | INFO | RTMetrics report populated | 20260507 04:50:21.823 | INFO | Opening tmp RTMetrics json file | 20260507 04:50:21.841 | INFO | Serializing RTMetrics to json | 20260507 04:50:21.843 | INFO | Serialization done, swapping tmp file now... | 20260507 04:50:21.843 | INFO | RTMetrics done | 20260507 04:50:21.913 | INFO | Finished a round of Baz IO: Wrote 1.713 GiB in 3.187 seconds | 20260507 04:50:21.913 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.76% / 0.00% / 67.24% | 20260507 04:50:21.913 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.159164% / 7.94729e-06% / 6.35783e-05% | 20260507 04:50:21.913 | INFO | Baz IO overall: | 20260507 04:50:21.913 | INFO | BytesWritten : 3679351328050 | 20260507 04:50:21.913 | INFO | NumEvents : 1765776261043 | 20260507 04:50:23.765 | INFO | Analyzing chunk frames = [7225344,7225856) | 20260507 04:50:23.765 | INFO | MemInfo: { | 20260507 04:50:23.765 | INFO | "gpu" : " 17 GB", | 20260507 04:50:23.765 | INFO | "huge" : "", | 20260507 04:50:23.765 | INFO | "huge_shared" : " 27 GB", | 20260507 04:50:23.765 | INFO | "malloc" : " 14 GB", | 20260507 04:50:23.765 | INFO | "pinned" : " 29 GB" | 20260507 04:50:23.765 | INFO | } | 20260507 04:50:23.765 | INFO | Perf report for the last 5.13595 seconds: | 20260507 04:50:23.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:50:23.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:23.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:23.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:23.765 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 04:50:23.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:23.765 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:50:23.765 | INFO | PRE_HQ : 59.2%, 1.8, 55.2ms, 0 | 20260507 04:50:23.765 | INFO | BAZWRITER : 61.7%, 3.4, 107.1ms, 0 | 20260507 04:50:23.765 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 04:50:23.765 | INFO | Uploaded 36GiB to GPU in 2.04159s (17.6333GiB/s) | 20260507 04:50:23.765 | INFO | Downloaded 38GiB to GPU in 2.18164s (17.4181GiB/s) | 20260507 04:50:28.910 | INFO | Analyzing chunk frames = [7225856,7226368) | 20260507 04:50:28.910 | INFO | MemInfo: { | 20260507 04:50:28.910 | INFO | "gpu" : " 17 GB", | 20260507 04:50:28.910 | INFO | "huge" : "", | 20260507 04:50:28.910 | INFO | "huge_shared" : " 27 GB", | 20260507 04:50:28.910 | INFO | "malloc" : " 14 GB", | 20260507 04:50:28.910 | INFO | "pinned" : " 29 GB" | 20260507 04:50:28.910 | INFO | } | 20260507 04:50:28.910 | INFO | Perf report for the last 5.14454 seconds: | 20260507 04:50:28.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:50:28.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:28.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:28.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:28.910 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 04:50:28.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:28.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:28.910 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:50:28.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:28.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:28.910 | INFO | Uploaded 36GiB to GPU in 2.03585s (17.6831GiB/s) | 20260507 04:50:28.910 | INFO | Downloaded 34GiB to GPU in 1.94099s (17.5168GiB/s) | 20260507 04:50:34.049 | INFO | Analyzing chunk frames = [7226368,7226880) | 20260507 04:50:34.049 | INFO | MemInfo: { | 20260507 04:50:34.049 | INFO | "gpu" : " 17 GB", | 20260507 04:50:34.049 | INFO | "huge" : "", | 20260507 04:50:34.049 | INFO | "huge_shared" : " 27 GB", | 20260507 04:50:34.049 | INFO | "malloc" : " 14 GB", | 20260507 04:50:34.049 | INFO | "pinned" : " 29 GB" | 20260507 04:50:34.049 | INFO | } | 20260507 04:50:34.049 | INFO | Perf report for the last 5.13929 seconds: | 20260507 04:50:34.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:50:34.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:34.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:34.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:34.049 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 04:50:34.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:34.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:34.049 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:50:34.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:34.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:34.049 | INFO | Uploaded 36GiB to GPU in 2.01447s (17.8707GiB/s) | 20260507 04:50:34.049 | INFO | Downloaded 34GiB to GPU in 1.94545s (17.4767GiB/s) | 20260507 04:50:39.186 | INFO | Analyzing chunk frames = [7226880,7227392) | 20260507 04:50:39.186 | INFO | MemInfo: { | 20260507 04:50:39.186 | INFO | "gpu" : " 17 GB", | 20260507 04:50:39.186 | INFO | "huge" : "", | 20260507 04:50:39.186 | INFO | "huge_shared" : " 27 GB", | 20260507 04:50:39.186 | INFO | "malloc" : " 14 GB", | 20260507 04:50:39.186 | INFO | "pinned" : " 29 GB" | 20260507 04:50:39.186 | INFO | } | 20260507 04:50:39.186 | INFO | Perf report for the last 5.13695 seconds: | 20260507 04:50:39.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:50:39.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:39.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:39.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:39.186 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 04:50:39.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:39.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:39.186 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:50:39.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:39.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:39.186 | INFO | Uploaded 36GiB to GPU in 1.9863s (18.1242GiB/s) | 20260507 04:50:39.186 | INFO | Downloaded 34GiB to GPU in 1.93353s (17.5844GiB/s) | 20260507 04:50:44.326 | INFO | Analyzing chunk frames = [7227392,7227904) | 20260507 04:50:44.326 | INFO | MemInfo: { | 20260507 04:50:44.326 | INFO | "gpu" : " 17 GB", | 20260507 04:50:44.326 | INFO | "huge" : "", | 20260507 04:50:44.327 | INFO | "huge_shared" : " 27 GB", | 20260507 04:50:44.327 | INFO | "malloc" : " 14 GB", | 20260507 04:50:44.327 | INFO | "pinned" : " 29 GB" | 20260507 04:50:44.327 | INFO | } | 20260507 04:50:44.327 | INFO | Perf report for the last 5.1405 seconds: | 20260507 04:50:44.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:50:44.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:44.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:44.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:44.327 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 04:50:44.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:44.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:44.327 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:50:44.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:44.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:44.327 | INFO | Uploaded 36GiB to GPU in 2.04963s (17.5641GiB/s) | 20260507 04:50:44.327 | INFO | Downloaded 34GiB to GPU in 1.95284s (17.4105GiB/s) | 20260507 04:50:49.463 | INFO | Analyzing chunk frames = [7227904,7228416) | 20260507 04:50:49.463 | INFO | MemInfo: { | 20260507 04:50:49.463 | INFO | "gpu" : " 17 GB", | 20260507 04:50:49.463 | INFO | "huge" : "", | 20260507 04:50:49.463 | INFO | "huge_shared" : " 27 GB", | 20260507 04:50:49.463 | INFO | "malloc" : " 14 GB", | 20260507 04:50:49.463 | INFO | "pinned" : " 29 GB" | 20260507 04:50:49.463 | INFO | } | 20260507 04:50:49.463 | INFO | Perf report for the last 5.13644 seconds: | 20260507 04:50:49.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:50:49.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:49.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:49.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:49.463 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:50:49.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:49.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:49.463 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 04:50:49.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:49.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:49.463 | INFO | Uploaded 36GiB to GPU in 2.05201s (17.5438GiB/s) | 20260507 04:50:49.463 | INFO | Downloaded 34GiB to GPU in 1.96911s (17.2667GiB/s) | 20260507 04:50:54.608 | INFO | Analyzing chunk frames = [7228416,7228928) | 20260507 04:50:54.608 | INFO | MemInfo: { | 20260507 04:50:54.608 | INFO | "gpu" : " 17 GB", | 20260507 04:50:54.608 | INFO | "huge" : "", | 20260507 04:50:54.608 | INFO | "huge_shared" : " 27 GB", | 20260507 04:50:54.608 | INFO | "malloc" : " 14 GB", | 20260507 04:50:54.608 | INFO | "pinned" : " 29 GB" | 20260507 04:50:54.608 | INFO | } | 20260507 04:50:54.608 | INFO | Perf report for the last 5.14534 seconds: | 20260507 04:50:54.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:50:54.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:54.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:54.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:54.608 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:50:54.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:54.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:54.608 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:50:54.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:54.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:54.609 | INFO | Uploaded 36GiB to GPU in 1.97206s (18.255GiB/s) | 20260507 04:50:54.609 | INFO | Downloaded 34GiB to GPU in 1.97445s (17.22GiB/s) | 20260507 04:50:59.743 | INFO | Analyzing chunk frames = [7228928,7229440) | 20260507 04:50:59.743 | INFO | MemInfo: { | 20260507 04:50:59.743 | INFO | "gpu" : " 17 GB", | 20260507 04:50:59.743 | INFO | "huge" : "", | 20260507 04:50:59.743 | INFO | "huge_shared" : " 27 GB", | 20260507 04:50:59.743 | INFO | "malloc" : " 14 GB", | 20260507 04:50:59.744 | INFO | "pinned" : " 29 GB" | 20260507 04:50:59.744 | INFO | } | 20260507 04:50:59.744 | INFO | Perf report for the last 5.13515 seconds: | 20260507 04:50:59.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:50:59.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:59.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:59.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:59.744 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:50:59.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:59.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:59.744 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:50:59.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:59.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:50:59.744 | INFO | Uploaded 36GiB to GPU in 1.98463s (18.1394GiB/s) | 20260507 04:50:59.744 | INFO | Downloaded 34GiB to GPU in 1.96448s (17.3074GiB/s) | 20260507 04:50:59.840 | INFO | Starting a round of Baz IO | 20260507 04:51:02.940 | INFO | Populating RTMetrics report | 20260507 04:51:02.940 | INFO | RTMetrics report populated | 20260507 04:51:02.940 | INFO | Opening tmp RTMetrics json file | 20260507 04:51:02.941 | INFO | Serializing RTMetrics to json | 20260507 04:51:02.942 | INFO | Serialization done, swapping tmp file now... | 20260507 04:51:02.945 | INFO | RTMetrics done | 20260507 04:51:03.027 | INFO | Finished a round of Baz IO: Wrote 1.714 GiB in 3.186 seconds | 20260507 04:51:03.027 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.76% / 0.00% / 67.24% | 20260507 04:51:03.027 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.159558% / 0% / 6.35783e-05% | 20260507 04:51:03.027 | INFO | Baz IO overall: | 20260507 04:51:03.027 | INFO | BytesWritten : 3681191837959 | 20260507 04:51:03.027 | INFO | NumEvents : 1766642064732 | 20260507 04:51:04.883 | INFO | Analyzing chunk frames = [7229440,7229952) | 20260507 04:51:04.883 | INFO | MemInfo: { | 20260507 04:51:04.883 | INFO | "gpu" : " 17 GB", | 20260507 04:51:04.883 | INFO | "huge" : "", | 20260507 04:51:04.883 | INFO | "huge_shared" : " 27 GB", | 20260507 04:51:04.883 | INFO | "malloc" : " 14 GB", | 20260507 04:51:04.883 | INFO | "pinned" : " 29 GB" | 20260507 04:51:04.883 | INFO | } | 20260507 04:51:04.883 | INFO | Perf report for the last 5.13969 seconds: | 20260507 04:51:04.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:51:04.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:04.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:04.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:04.883 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 04:51:04.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:04.883 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 04:51:04.883 | INFO | PRE_HQ : 59.1%, 1.8, 55.4ms, 0 | 20260507 04:51:04.883 | INFO | BAZWRITER : 61.6%, 3.2, 102.1ms, 0 | 20260507 04:51:04.883 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:51:04.883 | INFO | Uploaded 36GiB to GPU in 2.05576s (17.5118GiB/s) | 20260507 04:51:04.884 | INFO | Downloaded 38GiB to GPU in 2.18108s (17.4226GiB/s) | 20260507 04:51:10.024 | INFO | Analyzing chunk frames = [7229952,7230464) | 20260507 04:51:10.024 | INFO | MemInfo: { | 20260507 04:51:10.024 | INFO | "gpu" : " 17 GB", | 20260507 04:51:10.024 | INFO | "huge" : "", | 20260507 04:51:10.024 | INFO | "huge_shared" : " 27 GB", | 20260507 04:51:10.024 | INFO | "malloc" : " 14 GB", | 20260507 04:51:10.024 | INFO | "pinned" : " 29 GB" | 20260507 04:51:10.024 | INFO | } | 20260507 04:51:10.024 | INFO | Perf report for the last 5.14117 seconds: | 20260507 04:51:10.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:51:10.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:10.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:10.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:10.024 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:51:10.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:10.025 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:10.025 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:51:10.025 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:10.025 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:10.025 | INFO | Uploaded 36GiB to GPU in 2.00187s (17.9832GiB/s) | 20260507 04:51:10.025 | INFO | Downloaded 34GiB to GPU in 1.92935s (17.6225GiB/s) | 20260507 04:51:15.162 | INFO | Analyzing chunk frames = [7230464,7230976) | 20260507 04:51:15.162 | INFO | MemInfo: { | 20260507 04:51:15.162 | INFO | "gpu" : " 17 GB", | 20260507 04:51:15.162 | INFO | "huge" : "", | 20260507 04:51:15.162 | INFO | "huge_shared" : " 27 GB", | 20260507 04:51:15.162 | INFO | "malloc" : " 14 GB", | 20260507 04:51:15.162 | INFO | "pinned" : " 29 GB" | 20260507 04:51:15.162 | INFO | } | 20260507 04:51:15.162 | INFO | Perf report for the last 5.13766 seconds: | 20260507 04:51:15.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:51:15.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:15.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:15.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:15.162 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 04:51:15.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:15.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:15.162 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:51:15.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:15.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:15.162 | INFO | Uploaded 36GiB to GPU in 2.01084s (17.9029GiB/s) | 20260507 04:51:15.162 | INFO | Downloaded 34GiB to GPU in 1.94044s (17.5218GiB/s) | 20260507 04:51:20.302 | INFO | Analyzing chunk frames = [7230976,7231488) | 20260507 04:51:20.302 | INFO | MemInfo: { | 20260507 04:51:20.302 | INFO | "gpu" : " 17 GB", | 20260507 04:51:20.302 | INFO | "huge" : "", | 20260507 04:51:20.302 | INFO | "huge_shared" : " 27 GB", | 20260507 04:51:20.302 | INFO | "malloc" : " 14 GB", | 20260507 04:51:20.302 | INFO | "pinned" : " 29 GB" | 20260507 04:51:20.302 | INFO | } | 20260507 04:51:20.302 | INFO | Perf report for the last 5.13985 seconds: | 20260507 04:51:20.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:51:20.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:20.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:20.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:20.302 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:51:20.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:20.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:20.302 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:51:20.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:20.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:20.302 | INFO | Uploaded 36GiB to GPU in 1.98097s (18.1729GiB/s) | 20260507 04:51:20.302 | INFO | Downloaded 34GiB to GPU in 1.91716s (17.7346GiB/s) | 20260507 04:51:25.449 | INFO | Analyzing chunk frames = [7231488,7232000) | 20260507 04:51:25.449 | INFO | MemInfo: { | 20260507 04:51:25.449 | INFO | "gpu" : " 17 GB", | 20260507 04:51:25.449 | INFO | "huge" : "", | 20260507 04:51:25.449 | INFO | "huge_shared" : " 27 GB", | 20260507 04:51:25.449 | INFO | "malloc" : " 14 GB", | 20260507 04:51:25.449 | INFO | "pinned" : " 29 GB" | 20260507 04:51:25.449 | INFO | } | 20260507 04:51:25.449 | INFO | Perf report for the last 5.14715 seconds: | 20260507 04:51:25.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:51:25.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:25.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:25.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:25.449 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 04:51:25.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:25.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:25.449 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:51:25.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:25.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:25.449 | INFO | Uploaded 36GiB to GPU in 2.05681s (17.5029GiB/s) | 20260507 04:51:25.449 | INFO | Downloaded 34GiB to GPU in 1.94462s (17.4842GiB/s) | 20260507 04:51:30.581 | INFO | Analyzing chunk frames = [7232000,7232512) | 20260507 04:51:30.581 | INFO | MemInfo: { | 20260507 04:51:30.581 | INFO | "gpu" : " 17 GB", | 20260507 04:51:30.581 | INFO | "huge" : "", | 20260507 04:51:30.582 | INFO | "huge_shared" : " 27 GB", | 20260507 04:51:30.582 | INFO | "malloc" : " 14 GB", | 20260507 04:51:30.582 | INFO | "pinned" : " 29 GB" | 20260507 04:51:30.582 | INFO | } | 20260507 04:51:30.582 | INFO | Perf report for the last 5.13249 seconds: | 20260507 04:51:30.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:51:30.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:30.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:30.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:30.582 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 04:51:30.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:30.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:30.582 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 04:51:30.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:30.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:30.583 | INFO | Uploaded 36GiB to GPU in 2.06939s (17.3965GiB/s) | 20260507 04:51:30.583 | INFO | Downloaded 34GiB to GPU in 1.93269s (17.5921GiB/s) | 20260507 04:51:35.716 | INFO | Analyzing chunk frames = [7232512,7233024) | 20260507 04:51:35.716 | INFO | MemInfo: { | 20260507 04:51:35.716 | INFO | "gpu" : " 17 GB", | 20260507 04:51:35.716 | INFO | "huge" : "", | 20260507 04:51:35.716 | INFO | "huge_shared" : " 27 GB", | 20260507 04:51:35.716 | INFO | "malloc" : " 14 GB", | 20260507 04:51:35.716 | INFO | "pinned" : " 29 GB" | 20260507 04:51:35.716 | INFO | } | 20260507 04:51:35.716 | INFO | Perf report for the last 5.13409 seconds: | 20260507 04:51:35.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:51:35.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:35.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:35.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:35.716 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:51:35.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:35.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:35.716 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 04:51:35.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:35.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:35.716 | INFO | Uploaded 36GiB to GPU in 1.95885s (18.3782GiB/s) | 20260507 04:51:35.716 | INFO | Downloaded 34GiB to GPU in 1.97223s (17.2394GiB/s) | 20260507 04:51:40.866 | INFO | Analyzing chunk frames = [7233024,7233536) | 20260507 04:51:40.866 | INFO | MemInfo: { | 20260507 04:51:40.866 | INFO | "gpu" : " 17 GB", | 20260507 04:51:40.866 | INFO | "huge" : "", | 20260507 04:51:40.866 | INFO | "huge_shared" : " 27 GB", | 20260507 04:51:40.866 | INFO | "malloc" : " 14 GB", | 20260507 04:51:40.866 | INFO | "pinned" : " 29 GB" | 20260507 04:51:40.866 | INFO | } | 20260507 04:51:40.866 | INFO | Perf report for the last 5.14994 seconds: | 20260507 04:51:40.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:51:40.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:40.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:40.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:40.866 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 04:51:40.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:40.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:40.866 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:51:40.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:40.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:40.866 | INFO | Uploaded 36GiB to GPU in 1.95199s (18.4427GiB/s) | 20260507 04:51:40.866 | INFO | Downloaded 34GiB to GPU in 1.95024s (17.4338GiB/s) | 20260507 04:51:40.866 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:51:40.866 | INFO | Total mem usage: 18 GiB | 20260507 04:51:40.866 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:51:40.866 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 04:51:40.866 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:51:40.866 | INFO | High Water Breakdown: | 20260507 04:51:40.866 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:51:40.866 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:51:40.866 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:51:40.866 | INFO | ------------------------------------------ | 20260507 04:51:40.866 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:51:40.866 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:51:40.866 | INFO | Total mem usage: 26.5 GiB | 20260507 04:51:40.866 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:51:40.866 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:51:40.866 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:51:40.866 | INFO | High Water Breakdown: | 20260507 04:51:40.866 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:51:40.866 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:51:40.866 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:51:40.866 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:51:40.866 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:51:40.866 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:51:40.866 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:51:40.866 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:51:40.866 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:51:40.866 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:51:40.866 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:51:40.866 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:51:40.866 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:51:40.866 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:51:40.866 | INFO | ------------------------------------------ | 20260507 04:51:40.866 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:51:40.867 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:51:40.867 | INFO | Total mem usage: 25.6 GiB | 20260507 04:51:40.867 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:51:40.867 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:51:40.867 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:51:40.867 | INFO | High Water Breakdown: | 20260507 04:51:40.867 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:51:40.867 | INFO | ------------------------------------------ | 20260507 04:51:40.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:51:40.867 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:51:40.867 | INFO | Total mem usage: 15.7 GiB | 20260507 04:51:40.867 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:51:40.867 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:51:40.867 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:51:40.867 | INFO | High Water Breakdown: | 20260507 04:51:40.867 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:51:40.867 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:51:40.867 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:51:40.867 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:51:40.867 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:51:40.867 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:51:40.867 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:51:40.867 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:51:40.867 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:51:40.867 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:51:40.867 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:51:40.867 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:51:40.867 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:51:40.867 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:51:40.867 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:51:40.867 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:51:40.867 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:51:40.867 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:51:40.867 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:51:40.867 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:51:40.867 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:51:40.867 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:51:40.867 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:51:40.867 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:51:40.867 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:51:40.867 | INFO | ------------------------------------------ | 20260507 04:51:40.867 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:51:40.867 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:51:40.867 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:51:40.867 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:51:40.867 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 1.14e+00 | 20260507 04:51:40.867 | INFO | PerfReport: 0.29%, AggregateStats -- 9.03e-02 / 1.01e-01, 9.02e-03 / 7.32e-03 | 20260507 04:51:40.867 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.50e+00, 5.06e-01 / 2.53e-01 | 20260507 04:51:40.867 | INFO | PerfReport: 3.21%, Histogram -- 9.85e-01 / 1.47e+00, 1.63e-01 / 1.10e-01 | 20260507 04:51:40.867 | INFO | PerfReport: 16.14%, DME -- 3.96e+01 / 4.39e+01, 3.75e+00 / 1.88e+00 | 20260507 04:51:40.867 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.26e+00 / 1.76e+00 | 20260507 04:51:40.867 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.14e+00 / 2.68e+00, 3.84e-01 / 2.31e-01 | 20260507 04:51:40.867 | INFO | PerfReport: 3.02%, Metrics -- 9.25e-01 / 1.23e+00, 2.45e-01 / 1.34e-01 | 20260507 04:51:40.867 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.12e-01 / 3.65e-01, 4.94e-02 / 2.23e-02 | 20260507 04:51:40.867 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:51:40.867 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:51:40.867 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:51:40.867 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 04:51:40.867 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260507 04:51:40.867 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260507 04:51:40.867 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:51:40.867 | INFO | PerfReport: 1.48%, LockUpload -- 1.42e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260507 04:51:40.867 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260507 04:51:40.867 | INFO | PerfReport: 22.06%, LockCompute -- 2.12e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 04:51:40.867 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260507 04:51:40.963 | INFO | Starting a round of Baz IO | 20260507 04:51:44.062 | INFO | Populating RTMetrics report | 20260507 04:51:44.062 | INFO | RTMetrics report populated | 20260507 04:51:44.062 | INFO | Opening tmp RTMetrics json file | 20260507 04:51:44.080 | INFO | Serializing RTMetrics to json | 20260507 04:51:44.081 | INFO | Serialization done, swapping tmp file now... | 20260507 04:51:44.082 | INFO | RTMetrics done | 20260507 04:51:44.148 | INFO | Finished a round of Baz IO: Wrote 1.716 GiB in 3.185 seconds | 20260507 04:51:44.148 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.75% / 0.00% / 67.24% | 20260507 04:51:44.148 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.159339% / 0% / 7.94729e-05% | 20260507 04:51:44.148 | INFO | Baz IO overall: | 20260507 04:51:44.148 | INFO | BytesWritten : 3683034202451 | 20260507 04:51:44.148 | INFO | NumEvents : 1767509267500 | 20260507 04:51:46.004 | INFO | Analyzing chunk frames = [7233536,7234048) | 20260507 04:51:46.004 | INFO | MemInfo: { | 20260507 04:51:46.004 | INFO | "gpu" : " 17 GB", | 20260507 04:51:46.004 | INFO | "huge" : "", | 20260507 04:51:46.004 | INFO | "huge_shared" : " 27 GB", | 20260507 04:51:46.004 | INFO | "malloc" : " 14 GB", | 20260507 04:51:46.004 | INFO | "pinned" : " 29 GB" | 20260507 04:51:46.004 | INFO | } | 20260507 04:51:46.004 | INFO | Perf report for the last 5.13801 seconds: | 20260507 04:51:46.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:51:46.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:46.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:46.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:46.004 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:51:46.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:46.004 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:51:46.004 | INFO | PRE_HQ : 59.2%, 1.8, 54.9ms, 0 | 20260507 04:51:46.004 | INFO | BAZWRITER : 61.6%, 3.2, 101.5ms, 0 | 20260507 04:51:46.004 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:51:46.004 | INFO | Uploaded 36GiB to GPU in 2.04738s (17.5834GiB/s) | 20260507 04:51:46.004 | INFO | Downloaded 38GiB to GPU in 2.18268s (17.4098GiB/s) | 20260507 04:51:51.142 | INFO | Analyzing chunk frames = [7234048,7234560) | 20260507 04:51:51.142 | INFO | MemInfo: { | 20260507 04:51:51.142 | INFO | "gpu" : " 17 GB", | 20260507 04:51:51.142 | INFO | "huge" : "", | 20260507 04:51:51.142 | INFO | "huge_shared" : " 27 GB", | 20260507 04:51:51.142 | INFO | "malloc" : " 14 GB", | 20260507 04:51:51.142 | INFO | "pinned" : " 29 GB" | 20260507 04:51:51.143 | INFO | } | 20260507 04:51:51.143 | INFO | Perf report for the last 5.13882 seconds: | 20260507 04:51:51.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:51:51.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:51.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:51.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:51.143 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:51:51.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:51.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:51.143 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:51:51.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:51.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:51.143 | INFO | Uploaded 36GiB to GPU in 1.98576s (18.1291GiB/s) | 20260507 04:51:51.143 | INFO | Downloaded 34GiB to GPU in 1.93215s (17.597GiB/s) | 20260507 04:51:56.275 | INFO | Analyzing chunk frames = [7234560,7235072) | 20260507 04:51:56.275 | INFO | MemInfo: { | 20260507 04:51:56.275 | INFO | "gpu" : " 17 GB", | 20260507 04:51:56.275 | INFO | "huge" : "", | 20260507 04:51:56.275 | INFO | "huge_shared" : " 27 GB", | 20260507 04:51:56.275 | INFO | "malloc" : " 14 GB", | 20260507 04:51:56.275 | INFO | "pinned" : " 29 GB" | 20260507 04:51:56.275 | INFO | } | 20260507 04:51:56.276 | INFO | Perf report for the last 5.13299 seconds: | 20260507 04:51:56.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:51:56.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:56.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:56.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:56.276 | INFO | ANALYSIS : 60.7%, 3.0, 93.8ms, 0 | 20260507 04:51:56.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:56.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:56.276 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:51:56.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:56.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:51:56.276 | INFO | Uploaded 36GiB to GPU in 2.04872s (17.5719GiB/s) | 20260507 04:51:56.276 | INFO | Downloaded 34GiB to GPU in 1.95469s (17.3941GiB/s) | 20260507 04:52:01.421 | INFO | Analyzing chunk frames = [7235072,7235584) | 20260507 04:52:01.421 | INFO | MemInfo: { | 20260507 04:52:01.421 | INFO | "gpu" : " 17 GB", | 20260507 04:52:01.421 | INFO | "huge" : "", | 20260507 04:52:01.421 | INFO | "huge_shared" : " 27 GB", | 20260507 04:52:01.421 | INFO | "malloc" : " 14 GB", | 20260507 04:52:01.421 | INFO | "pinned" : " 29 GB" | 20260507 04:52:01.421 | INFO | } | 20260507 04:52:01.421 | INFO | Perf report for the last 5.14557 seconds: | 20260507 04:52:01.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:52:01.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:01.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:01.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:01.421 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 04:52:01.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:01.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:01.421 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:52:01.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:01.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:01.421 | INFO | Uploaded 36GiB to GPU in 1.97827s (18.1977GiB/s) | 20260507 04:52:01.421 | INFO | Downloaded 34GiB to GPU in 1.91187s (17.7836GiB/s) | 20260507 04:52:06.557 | INFO | Analyzing chunk frames = [7235584,7236096) | 20260507 04:52:06.557 | INFO | MemInfo: { | 20260507 04:52:06.557 | INFO | "gpu" : " 17 GB", | 20260507 04:52:06.557 | INFO | "huge" : "", | 20260507 04:52:06.557 | INFO | "huge_shared" : " 27 GB", | 20260507 04:52:06.557 | INFO | "malloc" : " 14 GB", | 20260507 04:52:06.557 | INFO | "pinned" : " 29 GB" | 20260507 04:52:06.557 | INFO | } | 20260507 04:52:06.558 | INFO | Perf report for the last 5.13639 seconds: | 20260507 04:52:06.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:52:06.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:06.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:06.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:06.558 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 04:52:06.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:06.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:06.558 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:52:06.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:06.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:06.558 | INFO | Uploaded 36GiB to GPU in 2.05145s (17.5486GiB/s) | 20260507 04:52:06.558 | INFO | Downloaded 34GiB to GPU in 1.95512s (17.3902GiB/s) | 20260507 04:52:11.691 | INFO | Analyzing chunk frames = [7236096,7236608) | 20260507 04:52:11.691 | INFO | MemInfo: { | 20260507 04:52:11.691 | INFO | "gpu" : " 17 GB", | 20260507 04:52:11.691 | INFO | "huge" : "", | 20260507 04:52:11.691 | INFO | "huge_shared" : " 27 GB", | 20260507 04:52:11.691 | INFO | "malloc" : " 14 GB", | 20260507 04:52:11.691 | INFO | "pinned" : " 29 GB" | 20260507 04:52:11.691 | INFO | } | 20260507 04:52:11.691 | INFO | Perf report for the last 5.13394 seconds: | 20260507 04:52:11.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:52:11.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:11.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:11.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:11.692 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 04:52:11.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:11.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:11.692 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 04:52:11.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:11.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:11.692 | INFO | Uploaded 36GiB to GPU in 2.03831s (17.6617GiB/s) | 20260507 04:52:11.692 | INFO | Downloaded 34GiB to GPU in 1.9185s (17.7222GiB/s) | 20260507 04:52:16.833 | INFO | Analyzing chunk frames = [7236608,7237120) | 20260507 04:52:16.833 | INFO | MemInfo: { | 20260507 04:52:16.833 | INFO | "gpu" : " 17 GB", | 20260507 04:52:16.833 | INFO | "huge" : "", | 20260507 04:52:16.833 | INFO | "huge_shared" : " 27 GB", | 20260507 04:52:16.833 | INFO | "malloc" : " 14 GB", | 20260507 04:52:16.833 | INFO | "pinned" : " 29 GB" | 20260507 04:52:16.833 | INFO | } | 20260507 04:52:16.833 | INFO | Perf report for the last 5.14164 seconds: | 20260507 04:52:16.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:52:16.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:16.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:16.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:16.833 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:52:16.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:16.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:16.833 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:52:16.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:16.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:16.833 | INFO | Uploaded 36GiB to GPU in 1.96376s (18.3322GiB/s) | 20260507 04:52:16.833 | INFO | Downloaded 34GiB to GPU in 1.97061s (17.2535GiB/s) | 20260507 04:52:21.973 | INFO | Analyzing chunk frames = [7237120,7237632) | 20260507 04:52:21.973 | INFO | MemInfo: { | 20260507 04:52:21.973 | INFO | "gpu" : " 17 GB", | 20260507 04:52:21.973 | INFO | "huge" : "", | 20260507 04:52:21.973 | INFO | "huge_shared" : " 27 GB", | 20260507 04:52:21.973 | INFO | "malloc" : " 14 GB", | 20260507 04:52:21.973 | INFO | "pinned" : " 29 GB" | 20260507 04:52:21.973 | INFO | } | 20260507 04:52:21.973 | INFO | Perf report for the last 5.14017 seconds: | 20260507 04:52:21.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:52:21.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:21.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:21.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:21.973 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:52:21.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:21.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:21.973 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 04:52:21.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:21.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:21.974 | INFO | Uploaded 36GiB to GPU in 1.97557s (18.2226GiB/s) | 20260507 04:52:21.974 | INFO | Downloaded 34GiB to GPU in 1.96864s (17.2708GiB/s) | 20260507 04:52:22.070 | INFO | Starting a round of Baz IO | 20260507 04:52:25.175 | INFO | Populating RTMetrics report | 20260507 04:52:25.175 | INFO | RTMetrics report populated | 20260507 04:52:25.175 | INFO | Opening tmp RTMetrics json file | 20260507 04:52:25.176 | INFO | Serializing RTMetrics to json | 20260507 04:52:25.177 | INFO | Serialization done, swapping tmp file now... | 20260507 04:52:25.177 | INFO | RTMetrics done | 20260507 04:52:25.262 | INFO | Finished a round of Baz IO: Wrote 1.715 GiB in 3.192 seconds | 20260507 04:52:25.262 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.75% / 0.00% / 67.25% | 20260507 04:52:25.262 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.159359% / 3.97364e-06% / 9.13938e-05% | 20260507 04:52:25.262 | INFO | Baz IO overall: | 20260507 04:52:25.262 | INFO | BytesWritten : 3684875754697 | 20260507 04:52:25.262 | INFO | NumEvents : 1768376009015 | 20260507 04:52:27.121 | INFO | Analyzing chunk frames = [7237632,7238144) | 20260507 04:52:27.121 | INFO | MemInfo: { | 20260507 04:52:27.121 | INFO | "gpu" : " 17 GB", | 20260507 04:52:27.121 | INFO | "huge" : "", | 20260507 04:52:27.121 | INFO | "huge_shared" : " 27 GB", | 20260507 04:52:27.121 | INFO | "malloc" : " 14 GB", | 20260507 04:52:27.121 | INFO | "pinned" : " 29 GB" | 20260507 04:52:27.121 | INFO | } | 20260507 04:52:27.121 | INFO | Perf report for the last 5.14749 seconds: | 20260507 04:52:27.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:52:27.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:27.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:27.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:27.121 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 04:52:27.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:27.121 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 04:52:27.121 | INFO | PRE_HQ : 59.3%, 1.8, 55.4ms, 0 | 20260507 04:52:27.121 | INFO | BAZWRITER : 61.6%, 3.2, 101.8ms, 0 | 20260507 04:52:27.121 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:52:27.121 | INFO | Uploaded 36GiB to GPU in 2.03271s (17.7103GiB/s) | 20260507 04:52:27.121 | INFO | Downloaded 38GiB to GPU in 2.18073s (17.4253GiB/s) | 20260507 04:52:32.258 | INFO | Analyzing chunk frames = [7238144,7238656) | 20260507 04:52:32.258 | INFO | MemInfo: { | 20260507 04:52:32.258 | INFO | "gpu" : " 17 GB", | 20260507 04:52:32.258 | INFO | "huge" : "", | 20260507 04:52:32.258 | INFO | "huge_shared" : " 27 GB", | 20260507 04:52:32.258 | INFO | "malloc" : " 14 GB", | 20260507 04:52:32.258 | INFO | "pinned" : " 29 GB" | 20260507 04:52:32.258 | INFO | } | 20260507 04:52:32.258 | INFO | Perf report for the last 5.13734 seconds: | 20260507 04:52:32.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:52:32.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:32.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:32.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:32.258 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:52:32.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:32.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:32.258 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:52:32.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:32.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:32.258 | INFO | Uploaded 36GiB to GPU in 1.9753s (18.2251GiB/s) | 20260507 04:52:32.259 | INFO | Downloaded 34GiB to GPU in 1.92478s (17.6644GiB/s) | 20260507 04:52:37.396 | INFO | Analyzing chunk frames = [7238656,7239168) | 20260507 04:52:37.396 | INFO | MemInfo: { | 20260507 04:52:37.396 | INFO | "gpu" : " 17 GB", | 20260507 04:52:37.396 | INFO | "huge" : "", | 20260507 04:52:37.396 | INFO | "huge_shared" : " 27 GB", | 20260507 04:52:37.396 | INFO | "malloc" : " 14 GB", | 20260507 04:52:37.396 | INFO | "pinned" : " 29 GB" | 20260507 04:52:37.396 | INFO | } | 20260507 04:52:37.396 | INFO | Perf report for the last 5.13748 seconds: | 20260507 04:52:37.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:52:37.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:37.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:37.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:37.396 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 04:52:37.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:37.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:37.396 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:52:37.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:37.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:37.396 | INFO | Uploaded 36GiB to GPU in 2.05833s (17.4899GiB/s) | 20260507 04:52:37.396 | INFO | Downloaded 34GiB to GPU in 1.94805s (17.4533GiB/s) | 20260507 04:52:42.529 | INFO | Analyzing chunk frames = [7239168,7239680) | 20260507 04:52:42.529 | INFO | MemInfo: { | 20260507 04:52:42.529 | INFO | "gpu" : " 17 GB", | 20260507 04:52:42.529 | INFO | "huge" : "", | 20260507 04:52:42.529 | INFO | "huge_shared" : " 27 GB", | 20260507 04:52:42.529 | INFO | "malloc" : " 14 GB", | 20260507 04:52:42.529 | INFO | "pinned" : " 29 GB" | 20260507 04:52:42.529 | INFO | } | 20260507 04:52:42.529 | INFO | Perf report for the last 5.13364 seconds: | 20260507 04:52:42.529 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:52:42.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:42.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:42.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:42.529 | INFO | ANALYSIS : 60.6%, 2.9, 93.4ms, 0 | 20260507 04:52:42.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:42.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:42.529 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:52:42.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:42.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:42.530 | INFO | Uploaded 36GiB to GPU in 2.04034s (17.6442GiB/s) | 20260507 04:52:42.530 | INFO | Downloaded 34GiB to GPU in 1.94795s (17.4543GiB/s) | 20260507 04:52:47.668 | INFO | Analyzing chunk frames = [7239680,7240192) | 20260507 04:52:47.668 | INFO | MemInfo: { | 20260507 04:52:47.668 | INFO | "gpu" : " 17 GB", | 20260507 04:52:47.668 | INFO | "huge" : "", | 20260507 04:52:47.668 | INFO | "huge_shared" : " 27 GB", | 20260507 04:52:47.668 | INFO | "malloc" : " 14 GB", | 20260507 04:52:47.668 | INFO | "pinned" : " 29 GB" | 20260507 04:52:47.668 | INFO | } | 20260507 04:52:47.668 | INFO | Perf report for the last 5.13904 seconds: | 20260507 04:52:47.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:52:47.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:47.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:47.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:47.668 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 04:52:47.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:47.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:47.669 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:52:47.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:47.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:47.669 | INFO | Uploaded 36GiB to GPU in 2.03894s (17.6562GiB/s) | 20260507 04:52:47.669 | INFO | Downloaded 34GiB to GPU in 1.94198s (17.5079GiB/s) | 20260507 04:52:52.817 | INFO | Analyzing chunk frames = [7240192,7240704) | 20260507 04:52:52.817 | INFO | MemInfo: { | 20260507 04:52:52.817 | INFO | "gpu" : " 17 GB", | 20260507 04:52:52.817 | INFO | "huge" : "", | 20260507 04:52:52.817 | INFO | "huge_shared" : " 27 GB", | 20260507 04:52:52.817 | INFO | "malloc" : " 14 GB", | 20260507 04:52:52.817 | INFO | "pinned" : " 29 GB" | 20260507 04:52:52.817 | INFO | } | 20260507 04:52:52.817 | INFO | Perf report for the last 5.1489 seconds: | 20260507 04:52:52.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:52:52.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:52.817 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:52.817 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:52.817 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:52:52.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:52.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:52.817 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 04:52:52.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:52.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:52.818 | INFO | Uploaded 36GiB to GPU in 2.05759s (17.4962GiB/s) | 20260507 04:52:52.818 | INFO | Downloaded 34GiB to GPU in 1.95556s (17.3863GiB/s) | 20260507 04:52:57.951 | INFO | Analyzing chunk frames = [7240704,7241216) | 20260507 04:52:57.951 | INFO | MemInfo: { | 20260507 04:52:57.951 | INFO | "gpu" : " 17 GB", | 20260507 04:52:57.951 | INFO | "huge" : "", | 20260507 04:52:57.951 | INFO | "huge_shared" : " 27 GB", | 20260507 04:52:57.951 | INFO | "malloc" : " 14 GB", | 20260507 04:52:57.951 | INFO | "pinned" : " 29 GB" | 20260507 04:52:57.951 | INFO | } | 20260507 04:52:57.951 | INFO | Perf report for the last 5.13363 seconds: | 20260507 04:52:57.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:52:57.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:57.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:57.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:57.951 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 04:52:57.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:57.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:57.951 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 04:52:57.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:57.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:52:57.951 | INFO | Uploaded 36GiB to GPU in 1.93162s (18.6372GiB/s) | 20260507 04:52:57.951 | INFO | Downloaded 34GiB to GPU in 1.97816s (17.1877GiB/s) | 20260507 04:53:03.092 | INFO | Analyzing chunk frames = [7241216,7241728) | 20260507 04:53:03.092 | INFO | MemInfo: { | 20260507 04:53:03.092 | INFO | "gpu" : " 17 GB", | 20260507 04:53:03.092 | INFO | "huge" : "", | 20260507 04:53:03.092 | INFO | "huge_shared" : " 27 GB", | 20260507 04:53:03.092 | INFO | "malloc" : " 14 GB", | 20260507 04:53:03.092 | INFO | "pinned" : " 29 GB" | 20260507 04:53:03.092 | INFO | } | 20260507 04:53:03.092 | INFO | Perf report for the last 5.14107 seconds: | 20260507 04:53:03.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:53:03.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:03.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:03.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:03.092 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 04:53:03.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:03.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:03.092 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:53:03.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:03.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:03.092 | INFO | Uploaded 36GiB to GPU in 1.97172s (18.2581GiB/s) | 20260507 04:53:03.092 | INFO | Downloaded 34GiB to GPU in 1.9643s (17.309GiB/s) | 20260507 04:53:03.093 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:53:03.093 | INFO | Total mem usage: 18 GiB | 20260507 04:53:03.093 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:53:03.093 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 04:53:03.093 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:53:03.093 | INFO | High Water Breakdown: | 20260507 04:53:03.093 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:53:03.093 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:53:03.093 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:53:03.093 | INFO | ------------------------------------------ | 20260507 04:53:03.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:53:03.093 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:53:03.093 | INFO | Total mem usage: 26.5 GiB | 20260507 04:53:03.093 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:53:03.093 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:53:03.093 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:53:03.093 | INFO | High Water Breakdown: | 20260507 04:53:03.093 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:53:03.093 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:53:03.093 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:53:03.093 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:53:03.093 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:53:03.093 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:53:03.093 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:53:03.093 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:53:03.093 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:53:03.093 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:53:03.093 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:53:03.093 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:53:03.093 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:53:03.093 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:53:03.093 | INFO | ------------------------------------------ | 20260507 04:53:03.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:53:03.093 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:53:03.093 | INFO | Total mem usage: 25.6 GiB | 20260507 04:53:03.093 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:53:03.093 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:53:03.093 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:53:03.093 | INFO | High Water Breakdown: | 20260507 04:53:03.093 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:53:03.093 | INFO | ------------------------------------------ | 20260507 04:53:03.093 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:53:03.093 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:53:03.093 | INFO | Total mem usage: 15.7 GiB | 20260507 04:53:03.093 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:53:03.093 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:53:03.093 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:53:03.093 | INFO | High Water Breakdown: | 20260507 04:53:03.093 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:53:03.093 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:53:03.093 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:53:03.093 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:53:03.093 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:53:03.093 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:53:03.093 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:53:03.093 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:53:03.093 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:53:03.093 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:53:03.093 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:53:03.093 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:53:03.093 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:53:03.093 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:53:03.093 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:53:03.093 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:53:03.093 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:53:03.093 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:53:03.093 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:53:03.094 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:53:03.094 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:53:03.094 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:53:03.094 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:53:03.094 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:53:03.094 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:53:03.094 | INFO | ------------------------------------------ | 20260507 04:53:03.094 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:53:03.094 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:53:03.094 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:53:03.094 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:53:03.094 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.50e+01, 1.34e+01 / 1.03e+00 | 20260507 04:53:03.094 | INFO | PerfReport: 0.29%, AggregateStats -- 9.03e-02 / 1.02e-01, 9.75e-03 / 5.51e-03 | 20260507 04:53:03.094 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.47e+00, 5.08e-01 / 2.07e-01 | 20260507 04:53:03.094 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.50e+00, 1.68e-01 / 8.61e-02 | 20260507 04:53:03.094 | INFO | PerfReport: 16.29%, DME -- 4.00e+01 / 4.52e+01, 3.91e+00 / 1.33e+00 | 20260507 04:53:03.094 | INFO | PerfReport: 57.51%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.16e+00 / 8.05e-01 | 20260507 04:53:03.094 | INFO | PerfReport: 6.98%, PulseAccumulating -- 2.14e+00 / 2.71e+00, 3.90e-01 / 1.66e-01 | 20260507 04:53:03.094 | INFO | PerfReport: 3.02%, Metrics -- 9.28e-01 / 1.25e+00, 2.47e-01 / 1.44e-01 | 20260507 04:53:03.094 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.12e-01 / 3.75e-01, 4.98e-02 / 2.71e-02 | 20260507 04:53:03.094 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:53:03.094 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:53:03.094 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:53:03.094 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 04:53:03.094 | INFO | PerfReport: 21.77%, Upload -- 2.10e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260507 04:53:03.094 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260507 04:53:03.094 | INFO | PerfReport: 31.89%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:53:03.094 | INFO | PerfReport: 1.55%, LockUpload -- 1.49e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260507 04:53:03.094 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260507 04:53:03.094 | INFO | PerfReport: 22.10%, LockCompute -- 2.13e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 04:53:03.094 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260507 04:53:03.189 | INFO | Starting a round of Baz IO | 20260507 04:53:06.284 | INFO | Populating RTMetrics report | 20260507 04:53:06.284 | INFO | RTMetrics report populated | 20260507 04:53:06.284 | INFO | Opening tmp RTMetrics json file | 20260507 04:53:06.301 | INFO | Serializing RTMetrics to json | 20260507 04:53:06.302 | INFO | Serialization done, swapping tmp file now... | 20260507 04:53:06.303 | INFO | RTMetrics done | 20260507 04:53:06.367 | INFO | Finished a round of Baz IO: Wrote 1.714 GiB in 3.178 seconds | 20260507 04:53:06.367 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.75% / 0.00% / 67.25% | 20260507 04:53:06.367 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.158179% / 3.97364e-06% / 7.15256e-05% | 20260507 04:53:06.367 | INFO | Baz IO overall: | 20260507 04:53:06.367 | INFO | BytesWritten : 3686716178579 | 20260507 04:53:06.367 | INFO | NumEvents : 1769241656656 | 20260507 04:53:08.228 | INFO | Analyzing chunk frames = [7241728,7242240) | 20260507 04:53:08.228 | INFO | MemInfo: { | 20260507 04:53:08.228 | INFO | "gpu" : " 17 GB", | 20260507 04:53:08.228 | INFO | "huge" : "", | 20260507 04:53:08.228 | INFO | "huge_shared" : " 27 GB", | 20260507 04:53:08.228 | INFO | "malloc" : " 14 GB", | 20260507 04:53:08.228 | INFO | "pinned" : " 29 GB" | 20260507 04:53:08.228 | INFO | } | 20260507 04:53:08.228 | INFO | Perf report for the last 5.13629 seconds: | 20260507 04:53:08.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:53:08.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:08.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:08.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:08.229 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 04:53:08.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:08.229 | INFO | RT_METRICS : 41.9%, 1.0, 21.9ms, 0 | 20260507 04:53:08.229 | INFO | PRE_HQ : 58.9%, 1.8, 54.8ms, 0 | 20260507 04:53:08.229 | INFO | BAZWRITER : 61.5%, 3.2, 99.7ms, 0 | 20260507 04:53:08.229 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:53:08.229 | INFO | Uploaded 36GiB to GPU in 2.02847s (17.7474GiB/s) | 20260507 04:53:08.229 | INFO | Downloaded 38GiB to GPU in 2.16977s (17.5134GiB/s) | 20260507 04:53:13.367 | INFO | Analyzing chunk frames = [7242240,7242752) | 20260507 04:53:13.367 | INFO | MemInfo: { | 20260507 04:53:13.367 | INFO | "gpu" : " 17 GB", | 20260507 04:53:13.367 | INFO | "huge" : "", | 20260507 04:53:13.367 | INFO | "huge_shared" : " 27 GB", | 20260507 04:53:13.367 | INFO | "malloc" : " 14 GB", | 20260507 04:53:13.367 | INFO | "pinned" : " 29 GB" | 20260507 04:53:13.367 | INFO | } | 20260507 04:53:13.367 | INFO | Perf report for the last 5.13906 seconds: | 20260507 04:53:13.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:53:13.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:13.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:13.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:13.367 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:53:13.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:13.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:13.367 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 04:53:13.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:13.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:13.368 | INFO | Uploaded 36GiB to GPU in 2.04281s (17.6228GiB/s) | 20260507 04:53:13.368 | INFO | Downloaded 34GiB to GPU in 1.94114s (17.5154GiB/s) | 20260507 04:53:18.509 | INFO | Analyzing chunk frames = [7242752,7243264) | 20260507 04:53:18.509 | INFO | MemInfo: { | 20260507 04:53:18.509 | INFO | "gpu" : " 17 GB", | 20260507 04:53:18.509 | INFO | "huge" : "", | 20260507 04:53:18.509 | INFO | "huge_shared" : " 27 GB", | 20260507 04:53:18.509 | INFO | "malloc" : " 14 GB", | 20260507 04:53:18.509 | INFO | "pinned" : " 29 GB" | 20260507 04:53:18.509 | INFO | } | 20260507 04:53:18.509 | INFO | Perf report for the last 5.14137 seconds: | 20260507 04:53:18.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:53:18.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:18.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:18.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:18.509 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 04:53:18.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:18.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:18.509 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:53:18.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:18.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:18.510 | INFO | Uploaded 36GiB to GPU in 2.03215s (17.7152GiB/s) | 20260507 04:53:18.510 | INFO | Downloaded 34GiB to GPU in 1.95742s (17.3698GiB/s) | 20260507 04:53:23.647 | INFO | Analyzing chunk frames = [7243264,7243776) | 20260507 04:53:23.647 | INFO | MemInfo: { | 20260507 04:53:23.647 | INFO | "gpu" : " 17 GB", | 20260507 04:53:23.647 | INFO | "huge" : "", | 20260507 04:53:23.647 | INFO | "huge_shared" : " 27 GB", | 20260507 04:53:23.647 | INFO | "malloc" : " 14 GB", | 20260507 04:53:23.647 | INFO | "pinned" : " 29 GB" | 20260507 04:53:23.647 | INFO | } | 20260507 04:53:23.647 | INFO | Perf report for the last 5.13846 seconds: | 20260507 04:53:23.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:53:23.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:23.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:23.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:23.647 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 04:53:23.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:23.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:23.647 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:53:23.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:23.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:23.647 | INFO | Uploaded 36GiB to GPU in 2.00419s (17.9623GiB/s) | 20260507 04:53:23.648 | INFO | Downloaded 34GiB to GPU in 1.93543s (17.5671GiB/s) | 20260507 04:53:28.782 | INFO | Analyzing chunk frames = [7243776,7244288) | 20260507 04:53:28.782 | INFO | MemInfo: { | 20260507 04:53:28.782 | INFO | "gpu" : " 17 GB", | 20260507 04:53:28.782 | INFO | "huge" : "", | 20260507 04:53:28.782 | INFO | "huge_shared" : " 27 GB", | 20260507 04:53:28.782 | INFO | "malloc" : " 14 GB", | 20260507 04:53:28.782 | INFO | "pinned" : " 29 GB" | 20260507 04:53:28.782 | INFO | } | 20260507 04:53:28.782 | INFO | Perf report for the last 5.13475 seconds: | 20260507 04:53:28.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:53:28.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:28.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:28.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:28.782 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 04:53:28.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:28.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:28.782 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:53:28.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:28.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:28.782 | INFO | Uploaded 36GiB to GPU in 2.04496s (17.6042GiB/s) | 20260507 04:53:28.782 | INFO | Downloaded 34GiB to GPU in 1.94663s (17.4661GiB/s) | 20260507 04:53:33.927 | INFO | Analyzing chunk frames = [7244288,7244800) | 20260507 04:53:33.927 | INFO | MemInfo: { | 20260507 04:53:33.927 | INFO | "gpu" : " 17 GB", | 20260507 04:53:33.927 | INFO | "huge" : "", | 20260507 04:53:33.927 | INFO | "huge_shared" : " 27 GB", | 20260507 04:53:33.927 | INFO | "malloc" : " 14 GB", | 20260507 04:53:33.927 | INFO | "pinned" : " 29 GB" | 20260507 04:53:33.927 | INFO | } | 20260507 04:53:33.927 | INFO | Perf report for the last 5.14472 seconds: | 20260507 04:53:33.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:53:33.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:33.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:33.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:33.927 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 04:53:33.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:33.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:33.927 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 04:53:33.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:33.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:33.927 | INFO | Uploaded 36GiB to GPU in 2.03615s (17.6804GiB/s) | 20260507 04:53:33.927 | INFO | Downloaded 34GiB to GPU in 1.93908s (17.5341GiB/s) | 20260507 04:53:39.071 | INFO | Analyzing chunk frames = [7244800,7245312) | 20260507 04:53:39.071 | INFO | MemInfo: { | 20260507 04:53:39.071 | INFO | "gpu" : " 17 GB", | 20260507 04:53:39.071 | INFO | "huge" : "", | 20260507 04:53:39.071 | INFO | "huge_shared" : " 27 GB", | 20260507 04:53:39.071 | INFO | "malloc" : " 14 GB", | 20260507 04:53:39.071 | INFO | "pinned" : " 29 GB" | 20260507 04:53:39.071 | INFO | } | 20260507 04:53:39.071 | INFO | Perf report for the last 5.14472 seconds: | 20260507 04:53:39.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:53:39.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:39.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:39.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:39.072 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:53:39.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:39.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:39.072 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:53:39.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:39.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:39.072 | INFO | Uploaded 36GiB to GPU in 1.97901s (18.1909GiB/s) | 20260507 04:53:39.072 | INFO | Downloaded 34GiB to GPU in 1.97983s (17.1732GiB/s) | 20260507 04:53:44.205 | INFO | Analyzing chunk frames = [7245312,7245824) | 20260507 04:53:44.205 | INFO | MemInfo: { | 20260507 04:53:44.205 | INFO | "gpu" : " 17 GB", | 20260507 04:53:44.205 | INFO | "huge" : "", | 20260507 04:53:44.205 | INFO | "huge_shared" : " 27 GB", | 20260507 04:53:44.205 | INFO | "malloc" : " 14 GB", | 20260507 04:53:44.205 | INFO | "pinned" : " 29 GB" | 20260507 04:53:44.205 | INFO | } | 20260507 04:53:44.205 | INFO | Perf report for the last 5.13387 seconds: | 20260507 04:53:44.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:53:44.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:44.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:44.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:44.205 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:53:44.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:44.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:44.205 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:53:44.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:44.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:44.206 | INFO | Uploaded 36GiB to GPU in 1.98932s (18.0967GiB/s) | 20260507 04:53:44.206 | INFO | Downloaded 34GiB to GPU in 1.95845s (17.3607GiB/s) | 20260507 04:53:44.302 | INFO | Starting a round of Baz IO | 20260507 04:53:47.400 | INFO | Populating RTMetrics report | 20260507 04:53:47.401 | INFO | RTMetrics report populated | 20260507 04:53:47.401 | INFO | Opening tmp RTMetrics json file | 20260507 04:53:47.405 | INFO | Serializing RTMetrics to json | 20260507 04:53:47.419 | INFO | Serialization done, swapping tmp file now... | 20260507 04:53:47.419 | INFO | RTMetrics done | 20260507 04:53:47.487 | INFO | Finished a round of Baz IO: Wrote 1.713 GiB in 3.185 seconds | 20260507 04:53:47.487 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.75% / 0.00% / 67.25% | 20260507 04:53:47.487 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.158834% / 3.97364e-06% / 6.75519e-05% | 20260507 04:53:47.487 | INFO | Baz IO overall: | 20260507 04:53:47.487 | INFO | BytesWritten : 3688555703103 | 20260507 04:53:47.487 | INFO | NumEvents : 1770106938021 | 20260507 04:53:49.340 | INFO | Analyzing chunk frames = [7245824,7246336) | 20260507 04:53:49.340 | INFO | MemInfo: { | 20260507 04:53:49.340 | INFO | "gpu" : " 17 GB", | 20260507 04:53:49.340 | INFO | "huge" : "", | 20260507 04:53:49.340 | INFO | "huge_shared" : " 27 GB", | 20260507 04:53:49.340 | INFO | "malloc" : " 14 GB", | 20260507 04:53:49.340 | INFO | "pinned" : " 29 GB" | 20260507 04:53:49.340 | INFO | } | 20260507 04:53:49.340 | INFO | Perf report for the last 5.13485 seconds: | 20260507 04:53:49.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:53:49.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:49.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:49.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:49.340 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 04:53:49.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:49.340 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:53:49.340 | INFO | PRE_HQ : 59.1%, 1.8, 55.1ms, 0 | 20260507 04:53:49.340 | INFO | BAZWRITER : 61.6%, 3.2, 100.9ms, 0 | 20260507 04:53:49.340 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 04:53:49.340 | INFO | Uploaded 36GiB to GPU in 2.05868s (17.4869GiB/s) | 20260507 04:53:49.341 | INFO | Downloaded 38GiB to GPU in 2.17777s (17.449GiB/s) | 20260507 04:53:54.482 | INFO | Analyzing chunk frames = [7246336,7246848) | 20260507 04:53:54.482 | INFO | MemInfo: { | 20260507 04:53:54.482 | INFO | "gpu" : " 17 GB", | 20260507 04:53:54.482 | INFO | "huge" : "", | 20260507 04:53:54.482 | INFO | "huge_shared" : " 27 GB", | 20260507 04:53:54.482 | INFO | "malloc" : " 14 GB", | 20260507 04:53:54.482 | INFO | "pinned" : " 29 GB" | 20260507 04:53:54.482 | INFO | } | 20260507 04:53:54.482 | INFO | Perf report for the last 5.14227 seconds: | 20260507 04:53:54.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:53:54.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:54.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:54.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:54.483 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:53:54.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:54.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:54.483 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:53:54.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:54.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:54.483 | INFO | Uploaded 36GiB to GPU in 2.04346s (17.6172GiB/s) | 20260507 04:53:54.483 | INFO | Downloaded 34GiB to GPU in 1.93811s (17.5429GiB/s) | 20260507 04:53:59.622 | INFO | Analyzing chunk frames = [7246848,7247360) | 20260507 04:53:59.622 | INFO | MemInfo: { | 20260507 04:53:59.622 | INFO | "gpu" : " 17 GB", | 20260507 04:53:59.622 | INFO | "huge" : "", | 20260507 04:53:59.622 | INFO | "huge_shared" : " 27 GB", | 20260507 04:53:59.622 | INFO | "malloc" : " 14 GB", | 20260507 04:53:59.622 | INFO | "pinned" : " 29 GB" | 20260507 04:53:59.622 | INFO | } | 20260507 04:53:59.622 | INFO | Perf report for the last 5.13926 seconds: | 20260507 04:53:59.622 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:53:59.622 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:59.622 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:59.622 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:59.622 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 04:53:59.622 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:59.622 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:59.622 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:53:59.622 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:59.622 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:53:59.622 | INFO | Uploaded 36GiB to GPU in 2.01045s (17.9064GiB/s) | 20260507 04:53:59.622 | INFO | Downloaded 34GiB to GPU in 1.9416s (17.5113GiB/s) | 20260507 04:54:04.766 | INFO | Analyzing chunk frames = [7247360,7247872) | 20260507 04:54:04.766 | INFO | MemInfo: { | 20260507 04:54:04.766 | INFO | "gpu" : " 17 GB", | 20260507 04:54:04.766 | INFO | "huge" : "", | 20260507 04:54:04.766 | INFO | "huge_shared" : " 27 GB", | 20260507 04:54:04.766 | INFO | "malloc" : " 14 GB", | 20260507 04:54:04.766 | INFO | "pinned" : " 29 GB" | 20260507 04:54:04.766 | INFO | } | 20260507 04:54:04.766 | INFO | Perf report for the last 5.14454 seconds: | 20260507 04:54:04.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:54:04.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:04.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:04.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:04.766 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:54:04.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:04.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:04.766 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 04:54:04.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:04.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:04.767 | INFO | Uploaded 36GiB to GPU in 1.99316s (18.0618GiB/s) | 20260507 04:54:04.767 | INFO | Downloaded 34GiB to GPU in 1.92107s (17.6985GiB/s) | 20260507 04:54:09.903 | INFO | Analyzing chunk frames = [7247872,7248384) | 20260507 04:54:09.903 | INFO | MemInfo: { | 20260507 04:54:09.903 | INFO | "gpu" : " 17 GB", | 20260507 04:54:09.903 | INFO | "huge" : "", | 20260507 04:54:09.904 | INFO | "huge_shared" : " 27 GB", | 20260507 04:54:09.904 | INFO | "malloc" : " 14 GB", | 20260507 04:54:09.904 | INFO | "pinned" : " 29 GB" | 20260507 04:54:09.904 | INFO | } | 20260507 04:54:09.904 | INFO | Perf report for the last 5.13728 seconds: | 20260507 04:54:09.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:54:09.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:09.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:09.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:09.904 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260507 04:54:09.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:09.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:09.904 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:54:09.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:09.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:09.904 | INFO | Uploaded 36GiB to GPU in 2.04518s (17.6023GiB/s) | 20260507 04:54:09.904 | INFO | Downloaded 34GiB to GPU in 1.90895s (17.8108GiB/s) | 20260507 04:54:15.038 | INFO | Analyzing chunk frames = [7248384,7248896) | 20260507 04:54:15.038 | INFO | MemInfo: { | 20260507 04:54:15.038 | INFO | "gpu" : " 17 GB", | 20260507 04:54:15.038 | INFO | "huge" : "", | 20260507 04:54:15.038 | INFO | "huge_shared" : " 27 GB", | 20260507 04:54:15.038 | INFO | "malloc" : " 14 GB", | 20260507 04:54:15.038 | INFO | "pinned" : " 29 GB" | 20260507 04:54:15.038 | INFO | } | 20260507 04:54:15.038 | INFO | Perf report for the last 5.13407 seconds: | 20260507 04:54:15.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:54:15.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:15.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:15.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:15.038 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 04:54:15.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:15.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:15.038 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:54:15.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:15.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:15.038 | INFO | Uploaded 36GiB to GPU in 2.05077s (17.5544GiB/s) | 20260507 04:54:15.038 | INFO | Downloaded 34GiB to GPU in 1.9115s (17.7871GiB/s) | 20260507 04:54:20.186 | INFO | Analyzing chunk frames = [7248896,7249408) | 20260507 04:54:20.186 | INFO | MemInfo: { | 20260507 04:54:20.186 | INFO | "gpu" : " 17 GB", | 20260507 04:54:20.186 | INFO | "huge" : "", | 20260507 04:54:20.186 | INFO | "huge_shared" : " 27 GB", | 20260507 04:54:20.186 | INFO | "malloc" : " 14 GB", | 20260507 04:54:20.186 | INFO | "pinned" : " 29 GB" | 20260507 04:54:20.186 | INFO | } | 20260507 04:54:20.186 | INFO | Perf report for the last 5.14825 seconds: | 20260507 04:54:20.186 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:54:20.186 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:20.186 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:20.186 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:20.186 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 04:54:20.186 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:20.186 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:20.186 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 04:54:20.186 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:20.186 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:20.186 | INFO | Uploaded 36GiB to GPU in 1.96018s (18.3656GiB/s) | 20260507 04:54:20.186 | INFO | Downloaded 34GiB to GPU in 1.96989s (17.2599GiB/s) | 20260507 04:54:25.317 | INFO | Analyzing chunk frames = [7249408,7249920) | 20260507 04:54:25.317 | INFO | MemInfo: { | 20260507 04:54:25.317 | INFO | "gpu" : " 17 GB", | 20260507 04:54:25.317 | INFO | "huge" : "", | 20260507 04:54:25.317 | INFO | "huge_shared" : " 27 GB", | 20260507 04:54:25.317 | INFO | "malloc" : " 14 GB", | 20260507 04:54:25.317 | INFO | "pinned" : " 29 GB" | 20260507 04:54:25.317 | INFO | } | 20260507 04:54:25.317 | INFO | Perf report for the last 5.13086 seconds: | 20260507 04:54:25.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:54:25.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:25.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:25.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:25.317 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:54:25.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:25.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:25.317 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 04:54:25.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:25.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:25.317 | INFO | Uploaded 36GiB to GPU in 1.97936s (18.1877GiB/s) | 20260507 04:54:25.317 | INFO | Downloaded 34GiB to GPU in 1.96566s (17.297GiB/s) | 20260507 04:54:25.317 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:54:25.317 | INFO | Total mem usage: 18 GiB | 20260507 04:54:25.317 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:54:25.317 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 04:54:25.317 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:54:25.317 | INFO | High Water Breakdown: | 20260507 04:54:25.317 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:54:25.317 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:54:25.317 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:54:25.317 | INFO | ------------------------------------------ | 20260507 04:54:25.317 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:54:25.317 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:54:25.317 | INFO | Total mem usage: 26.5 GiB | 20260507 04:54:25.317 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:54:25.317 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:54:25.317 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 04:54:25.317 | INFO | High Water Breakdown: | 20260507 04:54:25.317 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:54:25.317 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:54:25.317 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:54:25.317 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:54:25.317 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:54:25.317 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:54:25.317 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:54:25.317 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:54:25.317 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:54:25.317 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:54:25.317 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:54:25.317 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:54:25.317 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:54:25.317 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:54:25.317 | INFO | ------------------------------------------ | 20260507 04:54:25.317 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:54:25.318 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:54:25.318 | INFO | Total mem usage: 25.6 GiB | 20260507 04:54:25.318 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:54:25.318 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:54:25.318 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:54:25.318 | INFO | High Water Breakdown: | 20260507 04:54:25.318 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:54:25.318 | INFO | ------------------------------------------ | 20260507 04:54:25.318 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:54:25.318 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:54:25.318 | INFO | Total mem usage: 15.7 GiB | 20260507 04:54:25.318 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:54:25.318 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:54:25.318 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:54:25.318 | INFO | High Water Breakdown: | 20260507 04:54:25.318 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:54:25.318 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:54:25.318 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:54:25.318 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:54:25.318 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:54:25.318 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:54:25.318 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:54:25.318 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:54:25.318 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:54:25.318 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:54:25.318 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:54:25.318 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:54:25.318 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:54:25.318 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:54:25.318 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:54:25.318 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:54:25.318 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:54:25.318 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:54:25.318 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:54:25.318 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:54:25.318 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:54:25.318 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:54:25.318 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:54:25.318 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:54:25.318 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:54:25.318 | INFO | ------------------------------------------ | 20260507 04:54:25.318 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:54:25.318 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:54:25.318 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:54:25.318 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:54:25.318 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.33e+01 / 1.12e+00 | 20260507 04:54:25.318 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.02e-01, 9.55e-03 / 5.27e-03 | 20260507 04:54:25.318 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.47e+00, 5.10e-01 / 2.69e-01 | 20260507 04:54:25.318 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.45e+00, 1.65e-01 / 1.19e-01 | 20260507 04:54:25.318 | INFO | PerfReport: 16.14%, DME -- 3.97e+01 / 4.43e+01, 3.77e+00 / 1.45e+00 | 20260507 04:54:25.318 | INFO | PerfReport: 57.68%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.28e+00 / 1.23e+00 | 20260507 04:54:25.318 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.69e+00, 3.80e-01 / 2.29e-01 | 20260507 04:54:25.318 | INFO | PerfReport: 3.02%, Metrics -- 9.28e-01 / 1.24e+00, 2.50e-01 / 1.24e-01 | 20260507 04:54:25.318 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.12e-01 / 3.68e-01, 4.99e-02 / 1.81e-02 | 20260507 04:54:25.318 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:54:25.318 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:54:25.318 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:54:25.318 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 04:54:25.318 | INFO | PerfReport: 21.86%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260507 04:54:25.318 | INFO | PerfReport: 21.36%, Download -- 2.05e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 04:54:25.318 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 04:54:25.318 | INFO | PerfReport: 1.48%, LockUpload -- 1.42e+00 / 0.00e+00, 4.79e+00 / 0.00e+00 | 20260507 04:54:25.318 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260507 04:54:25.318 | INFO | PerfReport: 22.05%, LockCompute -- 2.12e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 04:54:25.318 | INFO | PerfReport: 0.02%, Misc -- 1.49e-02 / 0.00e+00, 2.39e-02 / 0.00e+00 | 20260507 04:54:25.414 | INFO | Starting a round of Baz IO | 20260507 04:54:28.514 | INFO | Populating RTMetrics report | 20260507 04:54:28.514 | INFO | RTMetrics report populated | 20260507 04:54:28.514 | INFO | Opening tmp RTMetrics json file | 20260507 04:54:28.533 | INFO | Serializing RTMetrics to json | 20260507 04:54:28.534 | INFO | Serialization done, swapping tmp file now... | 20260507 04:54:28.534 | INFO | RTMetrics done | 20260507 04:54:28.601 | INFO | Finished a round of Baz IO: Wrote 1.715 GiB in 3.187 seconds | 20260507 04:54:28.601 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.75% / 0.00% / 67.25% | 20260507 04:54:28.601 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.15833% / 3.97364e-06% / 4.37101e-05% | 20260507 04:54:28.602 | INFO | Baz IO overall: | 20260507 04:54:28.602 | INFO | BytesWritten : 3690396678924 | 20260507 04:54:28.602 | INFO | NumEvents : 1770973006310 | 20260507 04:54:30.458 | INFO | Analyzing chunk frames = [7249920,7250432) | 20260507 04:54:30.458 | INFO | MemInfo: { | 20260507 04:54:30.458 | INFO | "gpu" : " 17 GB", | 20260507 04:54:30.458 | INFO | "huge" : "", | 20260507 04:54:30.458 | INFO | "huge_shared" : " 27 GB", | 20260507 04:54:30.458 | INFO | "malloc" : " 14 GB", | 20260507 04:54:30.458 | INFO | "pinned" : " 29 GB" | 20260507 04:54:30.458 | INFO | } | 20260507 04:54:30.458 | INFO | Perf report for the last 5.14124 seconds: | 20260507 04:54:30.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:54:30.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:30.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:30.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:30.458 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 04:54:30.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:30.458 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:54:30.458 | INFO | PRE_HQ : 59.0%, 1.8, 55.0ms, 0 | 20260507 04:54:30.458 | INFO | BAZWRITER : 61.6%, 3.1, 99.6ms, 0 | 20260507 04:54:30.458 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:54:30.458 | INFO | Uploaded 36GiB to GPU in 2.0402s (17.6453GiB/s) | 20260507 04:54:30.458 | INFO | Downloaded 38GiB to GPU in 2.18201s (17.4152GiB/s) | 20260507 04:54:35.598 | INFO | Analyzing chunk frames = [7250432,7250944) | 20260507 04:54:35.598 | INFO | MemInfo: { | 20260507 04:54:35.598 | INFO | "gpu" : " 17 GB", | 20260507 04:54:35.598 | INFO | "huge" : "", | 20260507 04:54:35.598 | INFO | "huge_shared" : " 27 GB", | 20260507 04:54:35.598 | INFO | "malloc" : " 14 GB", | 20260507 04:54:35.598 | INFO | "pinned" : " 29 GB" | 20260507 04:54:35.598 | INFO | } | 20260507 04:54:35.598 | INFO | Perf report for the last 5.14011 seconds: | 20260507 04:54:35.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:54:35.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:35.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:35.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:35.598 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:54:35.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:35.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:35.598 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:54:35.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:35.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:35.598 | INFO | Uploaded 36GiB to GPU in 1.99392s (18.0549GiB/s) | 20260507 04:54:35.598 | INFO | Downloaded 34GiB to GPU in 1.93325s (17.5869GiB/s) | 20260507 04:54:40.736 | INFO | Analyzing chunk frames = [7250944,7251456) | 20260507 04:54:40.736 | INFO | MemInfo: { | 20260507 04:54:40.736 | INFO | "gpu" : " 17 GB", | 20260507 04:54:40.736 | INFO | "huge" : "", | 20260507 04:54:40.736 | INFO | "huge_shared" : " 27 GB", | 20260507 04:54:40.736 | INFO | "malloc" : " 14 GB", | 20260507 04:54:40.736 | INFO | "pinned" : " 29 GB" | 20260507 04:54:40.736 | INFO | } | 20260507 04:54:40.736 | INFO | Perf report for the last 5.13807 seconds: | 20260507 04:54:40.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:54:40.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:40.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:40.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:40.736 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 04:54:40.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:40.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:40.736 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:54:40.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:40.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:40.737 | INFO | Uploaded 36GiB to GPU in 2.0045s (17.9596GiB/s) | 20260507 04:54:40.737 | INFO | Downloaded 34GiB to GPU in 1.9423s (17.5051GiB/s) | 20260507 04:54:45.879 | INFO | Analyzing chunk frames = [7251456,7251968) | 20260507 04:54:45.879 | INFO | MemInfo: { | 20260507 04:54:45.879 | INFO | "gpu" : " 17 GB", | 20260507 04:54:45.879 | INFO | "huge" : "", | 20260507 04:54:45.879 | INFO | "huge_shared" : " 27 GB", | 20260507 04:54:45.879 | INFO | "malloc" : " 14 GB", | 20260507 04:54:45.879 | INFO | "pinned" : " 29 GB" | 20260507 04:54:45.879 | INFO | } | 20260507 04:54:45.879 | INFO | Perf report for the last 5.14282 seconds: | 20260507 04:54:45.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:54:45.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:45.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:45.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:45.879 | INFO | ANALYSIS : 59.4%, 3.0, 92.3ms, 0 | 20260507 04:54:45.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:45.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:45.879 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:54:45.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:45.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:45.879 | INFO | Uploaded 36GiB to GPU in 1.97291s (18.2471GiB/s) | 20260507 04:54:45.879 | INFO | Downloaded 34GiB to GPU in 1.90811s (17.8186GiB/s) | 20260507 04:54:51.015 | INFO | Analyzing chunk frames = [7251968,7252480) | 20260507 04:54:51.015 | INFO | MemInfo: { | 20260507 04:54:51.015 | INFO | "gpu" : " 17 GB", | 20260507 04:54:51.015 | INFO | "huge" : "", | 20260507 04:54:51.015 | INFO | "huge_shared" : " 27 GB", | 20260507 04:54:51.015 | INFO | "malloc" : " 14 GB", | 20260507 04:54:51.015 | INFO | "pinned" : " 29 GB" | 20260507 04:54:51.016 | INFO | } | 20260507 04:54:51.016 | INFO | Perf report for the last 5.13643 seconds: | 20260507 04:54:51.016 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:54:51.016 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:51.016 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:51.016 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:51.016 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 04:54:51.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:51.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:51.016 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:54:51.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:51.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:51.016 | INFO | Uploaded 36GiB to GPU in 2.05001s (17.5609GiB/s) | 20260507 04:54:51.016 | INFO | Downloaded 34GiB to GPU in 1.95242s (17.4143GiB/s) | 20260507 04:54:56.153 | INFO | Analyzing chunk frames = [7252480,7252992) | 20260507 04:54:56.153 | INFO | MemInfo: { | 20260507 04:54:56.153 | INFO | "gpu" : " 17 GB", | 20260507 04:54:56.153 | INFO | "huge" : "", | 20260507 04:54:56.153 | INFO | "huge_shared" : " 27 GB", | 20260507 04:54:56.153 | INFO | "malloc" : " 14 GB", | 20260507 04:54:56.154 | INFO | "pinned" : " 29 GB" | 20260507 04:54:56.154 | INFO | } | 20260507 04:54:56.154 | INFO | Perf report for the last 5.138 seconds: | 20260507 04:54:56.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:54:56.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:56.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:56.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:56.154 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 04:54:56.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:56.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:56.154 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:54:56.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:56.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:54:56.154 | INFO | Uploaded 36GiB to GPU in 2.03094s (17.7258GiB/s) | 20260507 04:54:56.154 | INFO | Downloaded 34GiB to GPU in 1.91343s (17.7691GiB/s) | 20260507 04:55:01.289 | INFO | Analyzing chunk frames = [7252992,7253504) | 20260507 04:55:01.289 | INFO | MemInfo: { | 20260507 04:55:01.289 | INFO | "gpu" : " 17 GB", | 20260507 04:55:01.289 | INFO | "huge" : "", | 20260507 04:55:01.289 | INFO | "huge_shared" : " 27 GB", | 20260507 04:55:01.289 | INFO | "malloc" : " 14 GB", | 20260507 04:55:01.289 | INFO | "pinned" : " 29 GB" | 20260507 04:55:01.289 | INFO | } | 20260507 04:55:01.289 | INFO | Perf report for the last 5.13575 seconds: | 20260507 04:55:01.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:55:01.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:01.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:01.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:01.289 | INFO | ANALYSIS : 61.0%, 3.0, 94.8ms, 0 | 20260507 04:55:01.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:01.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:01.289 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 04:55:01.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:01.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:01.290 | INFO | Uploaded 36GiB to GPU in 1.9822s (18.1617GiB/s) | 20260507 04:55:01.290 | INFO | Downloaded 34GiB to GPU in 1.98936s (17.0909GiB/s) | 20260507 04:55:06.436 | INFO | Analyzing chunk frames = [7253504,7254016) | 20260507 04:55:06.436 | INFO | MemInfo: { | 20260507 04:55:06.436 | INFO | "gpu" : " 17 GB", | 20260507 04:55:06.436 | INFO | "huge" : "", | 20260507 04:55:06.436 | INFO | "huge_shared" : " 27 GB", | 20260507 04:55:06.436 | INFO | "malloc" : " 14 GB", | 20260507 04:55:06.436 | INFO | "pinned" : " 29 GB" | 20260507 04:55:06.436 | INFO | } | 20260507 04:55:06.436 | INFO | Perf report for the last 5.14713 seconds: | 20260507 04:55:06.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:55:06.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:06.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:06.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:06.436 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:55:06.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:06.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:06.437 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 04:55:06.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:06.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:06.437 | INFO | Uploaded 36GiB to GPU in 1.96859s (18.2872GiB/s) | 20260507 04:55:06.437 | INFO | Downloaded 34GiB to GPU in 1.95206s (17.4175GiB/s) | 20260507 04:55:06.533 | INFO | Starting a round of Baz IO | 20260507 04:55:09.642 | INFO | Populating RTMetrics report | 20260507 04:55:09.643 | INFO | RTMetrics report populated | 20260507 04:55:09.643 | INFO | Opening tmp RTMetrics json file | 20260507 04:55:09.650 | INFO | Serializing RTMetrics to json | 20260507 04:55:09.659 | INFO | Serialization done, swapping tmp file now... | 20260507 04:55:09.660 | INFO | RTMetrics done | 20260507 04:55:09.730 | INFO | Finished a round of Baz IO: Wrote 1.712 GiB in 3.196 seconds | 20260507 04:55:09.730 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.75% / 0.00% / 67.25% | 20260507 04:55:09.730 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.157718% / 3.97364e-06% / 4.37101e-05% | 20260507 04:55:09.730 | INFO | Baz IO overall: | 20260507 04:55:09.730 | INFO | BytesWritten : 3692235278584 | 20260507 04:55:09.730 | INFO | NumEvents : 1771837624026 | 20260507 04:55:11.577 | INFO | Analyzing chunk frames = [7254016,7254528) | 20260507 04:55:11.577 | INFO | MemInfo: { | 20260507 04:55:11.577 | INFO | "gpu" : " 17 GB", | 20260507 04:55:11.577 | INFO | "huge" : "", | 20260507 04:55:11.577 | INFO | "huge_shared" : " 27 GB", | 20260507 04:55:11.577 | INFO | "malloc" : " 14 GB", | 20260507 04:55:11.577 | INFO | "pinned" : " 29 GB" | 20260507 04:55:11.577 | INFO | } | 20260507 04:55:11.577 | INFO | Perf report for the last 5.14038 seconds: | 20260507 04:55:11.577 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:55:11.577 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:11.577 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:11.577 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:11.577 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 04:55:11.577 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:11.577 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 04:55:11.577 | INFO | PRE_HQ : 59.2%, 1.8, 55.3ms, 0 | 20260507 04:55:11.577 | INFO | BAZWRITER : 61.7%, 3.2, 101.8ms, 0 | 20260507 04:55:11.577 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:55:11.578 | INFO | Uploaded 36GiB to GPU in 2.05573s (17.512GiB/s) | 20260507 04:55:11.578 | INFO | Downloaded 38GiB to GPU in 2.18015s (17.43GiB/s) | 20260507 04:55:16.718 | INFO | Analyzing chunk frames = [7254528,7255040) | 20260507 04:55:16.718 | INFO | MemInfo: { | 20260507 04:55:16.718 | INFO | "gpu" : " 17 GB", | 20260507 04:55:16.718 | INFO | "huge" : "", | 20260507 04:55:16.719 | INFO | "huge_shared" : " 27 GB", | 20260507 04:55:16.719 | INFO | "malloc" : " 14 GB", | 20260507 04:55:16.719 | INFO | "pinned" : " 29 GB" | 20260507 04:55:16.719 | INFO | } | 20260507 04:55:16.719 | INFO | Perf report for the last 5.14173 seconds: | 20260507 04:55:16.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:55:16.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:16.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:16.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:16.719 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:55:16.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:16.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:16.719 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:55:16.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:16.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:16.719 | INFO | Uploaded 36GiB to GPU in 1.99048s (18.086GiB/s) | 20260507 04:55:16.719 | INFO | Downloaded 34GiB to GPU in 1.92887s (17.6269GiB/s) | 20260507 04:55:21.850 | INFO | Analyzing chunk frames = [7255040,7255552) | 20260507 04:55:21.850 | INFO | MemInfo: { | 20260507 04:55:21.850 | INFO | "gpu" : " 17 GB", | 20260507 04:55:21.850 | INFO | "huge" : "", | 20260507 04:55:21.850 | INFO | "huge_shared" : " 27 GB", | 20260507 04:55:21.850 | INFO | "malloc" : " 14 GB", | 20260507 04:55:21.850 | INFO | "pinned" : " 29 GB" | 20260507 04:55:21.850 | INFO | } | 20260507 04:55:21.850 | INFO | Perf report for the last 5.13163 seconds: | 20260507 04:55:21.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:55:21.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:21.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:21.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:21.850 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 04:55:21.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:21.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:21.850 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:55:21.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:21.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:21.850 | INFO | Uploaded 36GiB to GPU in 2.05039s (17.5577GiB/s) | 20260507 04:55:21.850 | INFO | Downloaded 34GiB to GPU in 1.9477s (17.4565GiB/s) | 20260507 04:55:26.991 | INFO | Analyzing chunk frames = [7255552,7256064) | 20260507 04:55:26.991 | INFO | MemInfo: { | 20260507 04:55:26.991 | INFO | "gpu" : " 17 GB", | 20260507 04:55:26.991 | INFO | "huge" : "", | 20260507 04:55:26.991 | INFO | "huge_shared" : " 27 GB", | 20260507 04:55:26.991 | INFO | "malloc" : " 14 GB", | 20260507 04:55:26.991 | INFO | "pinned" : " 29 GB" | 20260507 04:55:26.991 | INFO | } | 20260507 04:55:26.991 | INFO | Perf report for the last 5.14084 seconds: | 20260507 04:55:26.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:55:26.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:26.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:26.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:26.991 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 04:55:26.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:26.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:26.991 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 04:55:26.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:26.991 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:26.991 | INFO | Uploaded 36GiB to GPU in 1.98498s (18.1362GiB/s) | 20260507 04:55:26.992 | INFO | Downloaded 34GiB to GPU in 1.92109s (17.6983GiB/s) | 20260507 04:55:32.132 | INFO | Analyzing chunk frames = [7256064,7256576) | 20260507 04:55:32.132 | INFO | MemInfo: { | 20260507 04:55:32.132 | INFO | "gpu" : " 17 GB", | 20260507 04:55:32.132 | INFO | "huge" : "", | 20260507 04:55:32.132 | INFO | "huge_shared" : " 27 GB", | 20260507 04:55:32.132 | INFO | "malloc" : " 14 GB", | 20260507 04:55:32.132 | INFO | "pinned" : " 29 GB" | 20260507 04:55:32.132 | INFO | } | 20260507 04:55:32.132 | INFO | Perf report for the last 5.14074 seconds: | 20260507 04:55:32.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:55:32.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:32.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:32.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:32.132 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 04:55:32.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:32.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:32.132 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 04:55:32.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:32.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:32.132 | INFO | Uploaded 36GiB to GPU in 2.03939s (17.6524GiB/s) | 20260507 04:55:32.132 | INFO | Downloaded 34GiB to GPU in 1.94009s (17.5249GiB/s) | 20260507 04:55:37.272 | INFO | Analyzing chunk frames = [7256576,7257088) | 20260507 04:55:37.272 | INFO | MemInfo: { | 20260507 04:55:37.272 | INFO | "gpu" : " 17 GB", | 20260507 04:55:37.272 | INFO | "huge" : "", | 20260507 04:55:37.273 | INFO | "huge_shared" : " 27 GB", | 20260507 04:55:37.273 | INFO | "malloc" : " 14 GB", | 20260507 04:55:37.273 | INFO | "pinned" : " 29 GB" | 20260507 04:55:37.273 | INFO | } | 20260507 04:55:37.273 | INFO | Perf report for the last 5.14074 seconds: | 20260507 04:55:37.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:55:37.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:37.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:37.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:37.273 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 04:55:37.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:37.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:37.273 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:55:37.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:37.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:37.273 | INFO | Uploaded 36GiB to GPU in 2.04284s (17.6226GiB/s) | 20260507 04:55:37.273 | INFO | Downloaded 34GiB to GPU in 1.91712s (17.735GiB/s) | 20260507 04:55:42.411 | INFO | Analyzing chunk frames = [7257088,7257600) | 20260507 04:55:42.412 | INFO | MemInfo: { | 20260507 04:55:42.412 | INFO | "gpu" : " 17 GB", | 20260507 04:55:42.412 | INFO | "huge" : "", | 20260507 04:55:42.412 | INFO | "huge_shared" : " 27 GB", | 20260507 04:55:42.412 | INFO | "malloc" : " 14 GB", | 20260507 04:55:42.412 | INFO | "pinned" : " 29 GB" | 20260507 04:55:42.412 | INFO | } | 20260507 04:55:42.412 | INFO | Perf report for the last 5.13906 seconds: | 20260507 04:55:42.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:55:42.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:42.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:42.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:42.412 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:55:42.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:42.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:42.412 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 04:55:42.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:42.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:42.412 | INFO | Uploaded 36GiB to GPU in 2.0362s (17.68GiB/s) | 20260507 04:55:42.412 | INFO | Downloaded 34GiB to GPU in 1.98813s (17.1015GiB/s) | 20260507 04:55:47.549 | INFO | Analyzing chunk frames = [7257600,7258112) | 20260507 04:55:47.549 | INFO | MemInfo: { | 20260507 04:55:47.549 | INFO | "gpu" : " 17 GB", | 20260507 04:55:47.549 | INFO | "huge" : "", | 20260507 04:55:47.549 | INFO | "huge_shared" : " 27 GB", | 20260507 04:55:47.549 | INFO | "malloc" : " 14 GB", | 20260507 04:55:47.549 | INFO | "pinned" : " 29 GB" | 20260507 04:55:47.549 | INFO | } | 20260507 04:55:47.549 | INFO | Perf report for the last 5.13753 seconds: | 20260507 04:55:47.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:55:47.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:47.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:47.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:47.549 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:55:47.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:47.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:47.549 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:55:47.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:47.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:47.549 | INFO | Uploaded 36GiB to GPU in 1.96598s (18.3115GiB/s) | 20260507 04:55:47.549 | INFO | Downloaded 34GiB to GPU in 1.96319s (17.3187GiB/s) | 20260507 04:55:47.550 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:55:47.550 | INFO | Total mem usage: 18 GiB | 20260507 04:55:47.550 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:55:47.550 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 04:55:47.550 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:55:47.550 | INFO | High Water Breakdown: | 20260507 04:55:47.550 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:55:47.550 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:55:47.550 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:55:47.550 | INFO | ------------------------------------------ | 20260507 04:55:47.550 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:55:47.550 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:55:47.550 | INFO | Total mem usage: 26.5 GiB | 20260507 04:55:47.550 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:55:47.550 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:55:47.550 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:55:47.550 | INFO | High Water Breakdown: | 20260507 04:55:47.550 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:55:47.550 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:55:47.550 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:55:47.550 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:55:47.550 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:55:47.550 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:55:47.550 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:55:47.550 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:55:47.550 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:55:47.550 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:55:47.550 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:55:47.550 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:55:47.550 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:55:47.550 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:55:47.550 | INFO | ------------------------------------------ | 20260507 04:55:47.550 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:55:47.550 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:55:47.550 | INFO | Total mem usage: 25.6 GiB | 20260507 04:55:47.550 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:55:47.551 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:55:47.551 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:55:47.551 | INFO | High Water Breakdown: | 20260507 04:55:47.551 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:55:47.551 | INFO | ------------------------------------------ | 20260507 04:55:47.551 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:55:47.551 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:55:47.551 | INFO | Total mem usage: 15.7 GiB | 20260507 04:55:47.551 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:55:47.551 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:55:47.551 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:55:47.551 | INFO | High Water Breakdown: | 20260507 04:55:47.551 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:55:47.551 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:55:47.551 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:55:47.551 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:55:47.551 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:55:47.551 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:55:47.551 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:55:47.551 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:55:47.551 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:55:47.551 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:55:47.551 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:55:47.551 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:55:47.551 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:55:47.551 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:55:47.551 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:55:47.551 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:55:47.551 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:55:47.551 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:55:47.551 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:55:47.551 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:55:47.551 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:55:47.551 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:55:47.551 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:55:47.551 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:55:47.551 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:55:47.551 | INFO | ------------------------------------------ | 20260507 04:55:47.551 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:55:47.551 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:55:47.551 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:55:47.551 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:55:47.551 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.34e+01 / 1.23e+00 | 20260507 04:55:47.551 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.85e-03 / 5.71e-03 | 20260507 04:55:47.551 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.43e+00, 5.16e-01 / 2.69e-01 | 20260507 04:55:47.551 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.46e+00, 1.70e-01 / 1.32e-01 | 20260507 04:55:47.551 | INFO | PerfReport: 16.13%, DME -- 3.97e+01 / 4.40e+01, 3.72e+00 / 1.75e+00 | 20260507 04:55:47.551 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.33e+00 / 1.89e+00 | 20260507 04:55:47.551 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.66e+00, 3.80e-01 / 2.50e-01 | 20260507 04:55:47.551 | INFO | PerfReport: 3.01%, Metrics -- 9.25e-01 / 1.24e+00, 2.49e-01 / 1.23e-01 | 20260507 04:55:47.551 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.11e-01 / 3.64e-01, 5.08e-02 / 2.11e-02 | 20260507 04:55:47.551 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:55:47.551 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:55:47.551 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:55:47.551 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 04:55:47.551 | INFO | PerfReport: 21.80%, Upload -- 2.10e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260507 04:55:47.551 | INFO | PerfReport: 21.36%, Download -- 2.05e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 04:55:47.551 | INFO | PerfReport: 31.97%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:55:47.551 | INFO | PerfReport: 1.43%, LockUpload -- 1.38e+00 / 0.00e+00, 4.75e+00 / 0.00e+00 | 20260507 04:55:47.551 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260507 04:55:47.551 | INFO | PerfReport: 22.13%, LockCompute -- 2.13e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 04:55:47.551 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260507 04:55:47.650 | INFO | Starting a round of Baz IO | 20260507 04:55:50.749 | INFO | Populating RTMetrics report | 20260507 04:55:50.749 | INFO | RTMetrics report populated | 20260507 04:55:50.749 | INFO | Opening tmp RTMetrics json file | 20260507 04:55:50.767 | INFO | Serializing RTMetrics to json | 20260507 04:55:50.768 | INFO | Serialization done, swapping tmp file now... | 20260507 04:55:50.768 | INFO | RTMetrics done | 20260507 04:55:50.836 | INFO | Finished a round of Baz IO: Wrote 1.712 GiB in 3.186 seconds | 20260507 04:55:50.836 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.75% / 0.00% / 67.25% | 20260507 04:55:50.836 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.157952% / 3.97364e-06% / 5.16574e-05% | 20260507 04:55:50.836 | INFO | Baz IO overall: | 20260507 04:55:50.836 | INFO | BytesWritten : 3694073392581 | 20260507 04:55:50.836 | INFO | NumEvents : 1772702285310 | 20260507 04:55:52.690 | INFO | Analyzing chunk frames = [7258112,7258624) | 20260507 04:55:52.690 | INFO | MemInfo: { | 20260507 04:55:52.690 | INFO | "gpu" : " 17 GB", | 20260507 04:55:52.690 | INFO | "huge" : "", | 20260507 04:55:52.690 | INFO | "huge_shared" : " 27 GB", | 20260507 04:55:52.690 | INFO | "malloc" : " 14 GB", | 20260507 04:55:52.690 | INFO | "pinned" : " 29 GB" | 20260507 04:55:52.690 | INFO | } | 20260507 04:55:52.690 | INFO | Perf report for the last 5.1411 seconds: | 20260507 04:55:52.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:55:52.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:52.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:52.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:52.690 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 04:55:52.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:52.690 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 04:55:52.690 | INFO | PRE_HQ : 59.2%, 1.8, 55.3ms, 0 | 20260507 04:55:52.690 | INFO | BAZWRITER : 61.6%, 3.2, 101.6ms, 0 | 20260507 04:55:52.690 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:55:52.691 | INFO | Uploaded 36GiB to GPU in 2.04123s (17.6364GiB/s) | 20260507 04:55:52.691 | INFO | Downloaded 38GiB to GPU in 2.17588s (17.4642GiB/s) | 20260507 04:55:57.828 | INFO | Analyzing chunk frames = [7258624,7259136) | 20260507 04:55:57.828 | INFO | MemInfo: { | 20260507 04:55:57.828 | INFO | "gpu" : " 17 GB", | 20260507 04:55:57.828 | INFO | "huge" : "", | 20260507 04:55:57.828 | INFO | "huge_shared" : " 27 GB", | 20260507 04:55:57.828 | INFO | "malloc" : " 14 GB", | 20260507 04:55:57.828 | INFO | "pinned" : " 29 GB" | 20260507 04:55:57.828 | INFO | } | 20260507 04:55:57.828 | INFO | Perf report for the last 5.1381 seconds: | 20260507 04:55:57.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:55:57.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:57.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:57.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:57.829 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:55:57.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:57.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:57.829 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:55:57.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:57.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:55:57.829 | INFO | Uploaded 36GiB to GPU in 2.00263s (17.9763GiB/s) | 20260507 04:55:57.829 | INFO | Downloaded 34GiB to GPU in 1.92781s (17.6366GiB/s) | 20260507 04:56:02.964 | INFO | Analyzing chunk frames = [7259136,7259648) | 20260507 04:56:02.964 | INFO | MemInfo: { | 20260507 04:56:02.964 | INFO | "gpu" : " 17 GB", | 20260507 04:56:02.964 | INFO | "huge" : "", | 20260507 04:56:02.964 | INFO | "huge_shared" : " 27 GB", | 20260507 04:56:02.964 | INFO | "malloc" : " 14 GB", | 20260507 04:56:02.964 | INFO | "pinned" : " 29 GB" | 20260507 04:56:02.964 | INFO | } | 20260507 04:56:02.964 | INFO | Perf report for the last 5.13567 seconds: | 20260507 04:56:02.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:56:02.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:02.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:02.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:02.964 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 04:56:02.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:02.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:02.964 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:56:02.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:02.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:02.964 | INFO | Uploaded 36GiB to GPU in 2.03896s (17.6561GiB/s) | 20260507 04:56:02.964 | INFO | Downloaded 34GiB to GPU in 1.95229s (17.4154GiB/s) | 20260507 04:56:08.105 | INFO | Analyzing chunk frames = [7259648,7260160) | 20260507 04:56:08.105 | INFO | MemInfo: { | 20260507 04:56:08.105 | INFO | "gpu" : " 17 GB", | 20260507 04:56:08.105 | INFO | "huge" : "", | 20260507 04:56:08.105 | INFO | "huge_shared" : " 27 GB", | 20260507 04:56:08.105 | INFO | "malloc" : " 14 GB", | 20260507 04:56:08.105 | INFO | "pinned" : " 29 GB" | 20260507 04:56:08.105 | INFO | } | 20260507 04:56:08.105 | INFO | Perf report for the last 5.14094 seconds: | 20260507 04:56:08.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:56:08.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:08.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:08.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:08.105 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 04:56:08.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:08.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:08.105 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:56:08.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:08.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:08.105 | INFO | Uploaded 36GiB to GPU in 2.01247s (17.8885GiB/s) | 20260507 04:56:08.105 | INFO | Downloaded 34GiB to GPU in 1.93287s (17.5905GiB/s) | 20260507 04:56:13.244 | INFO | Analyzing chunk frames = [7260160,7260672) | 20260507 04:56:13.244 | INFO | MemInfo: { | 20260507 04:56:13.244 | INFO | "gpu" : " 17 GB", | 20260507 04:56:13.244 | INFO | "huge" : "", | 20260507 04:56:13.244 | INFO | "huge_shared" : " 27 GB", | 20260507 04:56:13.244 | INFO | "malloc" : " 14 GB", | 20260507 04:56:13.244 | INFO | "pinned" : " 29 GB" | 20260507 04:56:13.244 | INFO | } | 20260507 04:56:13.244 | INFO | Perf report for the last 5.13881 seconds: | 20260507 04:56:13.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:56:13.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:13.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:13.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:13.244 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 04:56:13.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:13.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:13.244 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:56:13.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:13.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:13.244 | INFO | Uploaded 36GiB to GPU in 2.04361s (17.6159GiB/s) | 20260507 04:56:13.244 | INFO | Downloaded 34GiB to GPU in 1.94023s (17.5237GiB/s) | 20260507 04:56:18.391 | INFO | Analyzing chunk frames = [7260672,7261184) | 20260507 04:56:18.391 | INFO | MemInfo: { | 20260507 04:56:18.391 | INFO | "gpu" : " 17 GB", | 20260507 04:56:18.391 | INFO | "huge" : "", | 20260507 04:56:18.391 | INFO | "huge_shared" : " 27 GB", | 20260507 04:56:18.391 | INFO | "malloc" : " 14 GB", | 20260507 04:56:18.391 | INFO | "pinned" : " 29 GB" | 20260507 04:56:18.391 | INFO | } | 20260507 04:56:18.391 | INFO | Perf report for the last 5.14753 seconds: | 20260507 04:56:18.391 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:56:18.391 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:18.391 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:18.391 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:18.392 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 04:56:18.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:18.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:18.392 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 04:56:18.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:18.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:18.392 | INFO | Uploaded 36GiB to GPU in 2.04023s (17.6451GiB/s) | 20260507 04:56:18.392 | INFO | Downloaded 34GiB to GPU in 1.9343s (17.5774GiB/s) | 20260507 04:56:23.531 | INFO | Analyzing chunk frames = [7261184,7261696) | 20260507 04:56:23.531 | INFO | MemInfo: { | 20260507 04:56:23.531 | INFO | "gpu" : " 17 GB", | 20260507 04:56:23.531 | INFO | "huge" : "", | 20260507 04:56:23.532 | INFO | "huge_shared" : " 27 GB", | 20260507 04:56:23.532 | INFO | "malloc" : " 14 GB", | 20260507 04:56:23.532 | INFO | "pinned" : " 29 GB" | 20260507 04:56:23.532 | INFO | } | 20260507 04:56:23.532 | INFO | Perf report for the last 5.14012 seconds: | 20260507 04:56:23.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:56:23.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:23.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:23.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:23.532 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 04:56:23.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:23.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:23.532 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 04:56:23.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:23.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:23.532 | INFO | Uploaded 36GiB to GPU in 2.04555s (17.5991GiB/s) | 20260507 04:56:23.532 | INFO | Downloaded 34GiB to GPU in 1.99796s (17.0174GiB/s) | 20260507 04:56:28.664 | INFO | Analyzing chunk frames = [7261696,7262208) | 20260507 04:56:28.664 | INFO | MemInfo: { | 20260507 04:56:28.664 | INFO | "gpu" : " 17 GB", | 20260507 04:56:28.664 | INFO | "huge" : "", | 20260507 04:56:28.664 | INFO | "huge_shared" : " 27 GB", | 20260507 04:56:28.664 | INFO | "malloc" : " 14 GB", | 20260507 04:56:28.664 | INFO | "pinned" : " 29 GB" | 20260507 04:56:28.664 | INFO | } | 20260507 04:56:28.664 | INFO | Perf report for the last 5.13255 seconds: | 20260507 04:56:28.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:56:28.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:28.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:28.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:28.664 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:56:28.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:28.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:28.664 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 04:56:28.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:28.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:28.665 | INFO | Uploaded 36GiB to GPU in 1.98205s (18.163GiB/s) | 20260507 04:56:28.665 | INFO | Downloaded 34GiB to GPU in 1.9653s (17.3002GiB/s) | 20260507 04:56:28.761 | INFO | Starting a round of Baz IO | 20260507 04:56:31.857 | INFO | Populating RTMetrics report | 20260507 04:56:31.857 | INFO | RTMetrics report populated | 20260507 04:56:31.857 | INFO | Opening tmp RTMetrics json file | 20260507 04:56:31.860 | INFO | Serializing RTMetrics to json | 20260507 04:56:31.876 | INFO | Serialization done, swapping tmp file now... | 20260507 04:56:31.876 | INFO | RTMetrics done | 20260507 04:56:31.942 | INFO | Finished a round of Baz IO: Wrote 1.711 GiB in 3.181 seconds | 20260507 04:56:31.942 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.75% / 0.00% / 67.25% | 20260507 04:56:31.943 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.156828% / 3.97364e-06% / 5.96046e-05% | 20260507 04:56:31.943 | INFO | Baz IO overall: | 20260507 04:56:31.943 | INFO | BytesWritten : 3695910599906 | 20260507 04:56:31.943 | INFO | NumEvents : 1773566249183 | 20260507 04:56:33.797 | INFO | Analyzing chunk frames = [7262208,7262720) | 20260507 04:56:33.797 | INFO | MemInfo: { | 20260507 04:56:33.797 | INFO | "gpu" : " 17 GB", | 20260507 04:56:33.797 | INFO | "huge" : "", | 20260507 04:56:33.797 | INFO | "huge_shared" : " 27 GB", | 20260507 04:56:33.797 | INFO | "malloc" : " 14 GB", | 20260507 04:56:33.797 | INFO | "pinned" : " 29 GB" | 20260507 04:56:33.797 | INFO | } | 20260507 04:56:33.797 | INFO | Perf report for the last 5.13281 seconds: | 20260507 04:56:33.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:56:33.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:33.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:33.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:33.797 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260507 04:56:33.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:33.797 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260507 04:56:33.797 | INFO | PRE_HQ : 59.7%, 1.9, 59.0ms, 0 | 20260507 04:56:33.797 | INFO | BAZWRITER : 61.6%, 3.4, 106.0ms, 0 | 20260507 04:56:33.797 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:56:33.797 | INFO | Uploaded 36GiB to GPU in 2.0212s (17.8112GiB/s) | 20260507 04:56:33.797 | INFO | Downloaded 38GiB to GPU in 2.17469s (17.4737GiB/s) | 20260507 04:56:38.941 | INFO | Analyzing chunk frames = [7262720,7263232) | 20260507 04:56:38.941 | INFO | MemInfo: { | 20260507 04:56:38.941 | INFO | "gpu" : " 17 GB", | 20260507 04:56:38.941 | INFO | "huge" : "", | 20260507 04:56:38.941 | INFO | "huge_shared" : " 27 GB", | 20260507 04:56:38.941 | INFO | "malloc" : " 14 GB", | 20260507 04:56:38.941 | INFO | "pinned" : " 29 GB" | 20260507 04:56:38.941 | INFO | } | 20260507 04:56:38.941 | INFO | Perf report for the last 5.14394 seconds: | 20260507 04:56:38.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:56:38.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:38.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:38.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:38.941 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:56:38.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:38.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:38.941 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 04:56:38.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:38.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:38.941 | INFO | Uploaded 36GiB to GPU in 2.025s (17.7778GiB/s) | 20260507 04:56:38.941 | INFO | Downloaded 34GiB to GPU in 1.93769s (17.5467GiB/s) | 20260507 04:56:44.084 | INFO | Analyzing chunk frames = [7263232,7263744) | 20260507 04:56:44.084 | INFO | MemInfo: { | 20260507 04:56:44.084 | INFO | "gpu" : " 17 GB", | 20260507 04:56:44.084 | INFO | "huge" : "", | 20260507 04:56:44.084 | INFO | "huge_shared" : " 27 GB", | 20260507 04:56:44.084 | INFO | "malloc" : " 14 GB", | 20260507 04:56:44.084 | INFO | "pinned" : " 29 GB" | 20260507 04:56:44.084 | INFO | } | 20260507 04:56:44.084 | INFO | Perf report for the last 5.14287 seconds: | 20260507 04:56:44.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:56:44.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:44.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:44.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:44.084 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 04:56:44.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:44.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:44.084 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:56:44.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:44.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:44.084 | INFO | Uploaded 36GiB to GPU in 1.97906s (18.1905GiB/s) | 20260507 04:56:44.084 | INFO | Downloaded 34GiB to GPU in 1.94206s (17.5072GiB/s) | 20260507 04:56:49.229 | INFO | Analyzing chunk frames = [7263744,7264256) | 20260507 04:56:49.229 | INFO | MemInfo: { | 20260507 04:56:49.229 | INFO | "gpu" : " 17 GB", | 20260507 04:56:49.229 | INFO | "huge" : "", | 20260507 04:56:49.229 | INFO | "huge_shared" : " 27 GB", | 20260507 04:56:49.229 | INFO | "malloc" : " 14 GB", | 20260507 04:56:49.229 | INFO | "pinned" : " 29 GB" | 20260507 04:56:49.229 | INFO | } | 20260507 04:56:49.229 | INFO | Perf report for the last 5.14496 seconds: | 20260507 04:56:49.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:56:49.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:49.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:49.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:49.229 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 04:56:49.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:49.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:49.229 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:56:49.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:49.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:49.229 | INFO | Uploaded 36GiB to GPU in 2.00482s (17.9567GiB/s) | 20260507 04:56:49.229 | INFO | Downloaded 34GiB to GPU in 1.94639s (17.4682GiB/s) | 20260507 04:56:54.364 | INFO | Analyzing chunk frames = [7264256,7264768) | 20260507 04:56:54.364 | INFO | MemInfo: { | 20260507 04:56:54.364 | INFO | "gpu" : " 17 GB", | 20260507 04:56:54.364 | INFO | "huge" : "", | 20260507 04:56:54.364 | INFO | "huge_shared" : " 27 GB", | 20260507 04:56:54.364 | INFO | "malloc" : " 14 GB", | 20260507 04:56:54.364 | INFO | "pinned" : " 29 GB" | 20260507 04:56:54.364 | INFO | } | 20260507 04:56:54.364 | INFO | Perf report for the last 5.13564 seconds: | 20260507 04:56:54.364 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:56:54.364 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:54.364 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:54.364 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:54.364 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 04:56:54.364 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:54.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:54.365 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 04:56:54.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:54.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:54.365 | INFO | Uploaded 36GiB to GPU in 2.04187s (17.6309GiB/s) | 20260507 04:56:54.365 | INFO | Downloaded 34GiB to GPU in 1.91308s (17.7724GiB/s) | 20260507 04:56:59.503 | INFO | Analyzing chunk frames = [7264768,7265280) | 20260507 04:56:59.504 | INFO | MemInfo: { | 20260507 04:56:59.504 | INFO | "gpu" : " 17 GB", | 20260507 04:56:59.504 | INFO | "huge" : "", | 20260507 04:56:59.504 | INFO | "huge_shared" : " 27 GB", | 20260507 04:56:59.504 | INFO | "malloc" : " 14 GB", | 20260507 04:56:59.504 | INFO | "pinned" : " 29 GB" | 20260507 04:56:59.504 | INFO | } | 20260507 04:56:59.504 | INFO | Perf report for the last 5.13922 seconds: | 20260507 04:56:59.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:56:59.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:59.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:59.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:59.504 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 04:56:59.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:59.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:59.504 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:56:59.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:59.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:56:59.504 | INFO | Uploaded 36GiB to GPU in 2.03746s (17.6691GiB/s) | 20260507 04:56:59.504 | INFO | Downloaded 34GiB to GPU in 1.92193s (17.6906GiB/s) | 20260507 04:57:04.639 | INFO | Analyzing chunk frames = [7265280,7265792) | 20260507 04:57:04.639 | INFO | MemInfo: { | 20260507 04:57:04.639 | INFO | "gpu" : " 17 GB", | 20260507 04:57:04.639 | INFO | "huge" : "", | 20260507 04:57:04.639 | INFO | "huge_shared" : " 27 GB", | 20260507 04:57:04.639 | INFO | "malloc" : " 14 GB", | 20260507 04:57:04.639 | INFO | "pinned" : " 29 GB" | 20260507 04:57:04.639 | INFO | } | 20260507 04:57:04.639 | INFO | Perf report for the last 5.13556 seconds: | 20260507 04:57:04.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:57:04.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:04.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:04.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:04.639 | INFO | ANALYSIS : 61.2%, 3.0, 95.2ms, 0 | 20260507 04:57:04.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:04.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:04.639 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 04:57:04.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:04.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:04.640 | INFO | Uploaded 36GiB to GPU in 2.04538s (17.6006GiB/s) | 20260507 04:57:04.640 | INFO | Downloaded 34GiB to GPU in 1.99218s (17.0667GiB/s) | 20260507 04:57:09.778 | INFO | Analyzing chunk frames = [7265792,7266304) | 20260507 04:57:09.778 | INFO | MemInfo: { | 20260507 04:57:09.778 | INFO | "gpu" : " 17 GB", | 20260507 04:57:09.778 | INFO | "huge" : "", | 20260507 04:57:09.778 | INFO | "huge_shared" : " 27 GB", | 20260507 04:57:09.778 | INFO | "malloc" : " 14 GB", | 20260507 04:57:09.778 | INFO | "pinned" : " 29 GB" | 20260507 04:57:09.778 | INFO | } | 20260507 04:57:09.778 | INFO | Perf report for the last 5.13879 seconds: | 20260507 04:57:09.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:57:09.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:09.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:09.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:09.778 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 04:57:09.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:09.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:09.778 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 04:57:09.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:09.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:09.778 | INFO | Uploaded 36GiB to GPU in 1.97309s (18.2455GiB/s) | 20260507 04:57:09.778 | INFO | Downloaded 34GiB to GPU in 1.96129s (17.3355GiB/s) | 20260507 04:57:09.779 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:57:09.779 | INFO | Total mem usage: 18 GiB | 20260507 04:57:09.779 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:57:09.779 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 04:57:09.779 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:57:09.779 | INFO | High Water Breakdown: | 20260507 04:57:09.779 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:57:09.779 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:57:09.779 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:57:09.779 | INFO | ------------------------------------------ | 20260507 04:57:09.779 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:57:09.779 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:57:09.779 | INFO | Total mem usage: 26.5 GiB | 20260507 04:57:09.779 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:57:09.779 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:57:09.779 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:57:09.779 | INFO | High Water Breakdown: | 20260507 04:57:09.779 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:57:09.779 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:57:09.779 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:57:09.779 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:57:09.779 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:57:09.779 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:57:09.779 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:57:09.779 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:57:09.779 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:57:09.779 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:57:09.779 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:57:09.779 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:57:09.779 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:57:09.779 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:57:09.779 | INFO | ------------------------------------------ | 20260507 04:57:09.779 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:57:09.779 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:57:09.779 | INFO | Total mem usage: 25.6 GiB | 20260507 04:57:09.779 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:57:09.779 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:57:09.779 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:57:09.779 | INFO | High Water Breakdown: | 20260507 04:57:09.779 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:57:09.779 | INFO | ------------------------------------------ | 20260507 04:57:09.779 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:57:09.779 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:57:09.779 | INFO | Total mem usage: 15.7 GiB | 20260507 04:57:09.779 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:57:09.779 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:57:09.779 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:57:09.779 | INFO | High Water Breakdown: | 20260507 04:57:09.779 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:57:09.779 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:57:09.779 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:57:09.779 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:57:09.779 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:57:09.779 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:57:09.779 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:57:09.779 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:57:09.779 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:57:09.779 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:57:09.779 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:57:09.779 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:57:09.779 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:57:09.779 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:57:09.779 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:57:09.779 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:57:09.779 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:57:09.779 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:57:09.779 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:57:09.779 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:57:09.780 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:57:09.780 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:57:09.780 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:57:09.780 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:57:09.780 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:57:09.780 | INFO | ------------------------------------------ | 20260507 04:57:09.780 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:57:09.780 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:57:09.780 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:57:09.780 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:57:09.780 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.48e+01, 1.28e+01 / 1.18e+00 | 20260507 04:57:09.780 | INFO | PerfReport: 0.30%, AggregateStats -- 9.12e-02 / 1.02e-01, 1.01e-02 / 7.54e-03 | 20260507 04:57:09.780 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.35e+00, 5.15e-01 / 3.08e-01 | 20260507 04:57:09.780 | INFO | PerfReport: 3.23%, Histogram -- 9.95e-01 / 1.46e+00, 1.70e-01 / 1.35e-01 | 20260507 04:57:09.780 | INFO | PerfReport: 16.08%, DME -- 3.96e+01 / 4.40e+01, 3.79e+00 / 1.49e+00 | 20260507 04:57:09.780 | INFO | PerfReport: 57.82%, FrameLabeling -- 1.78e+01 / 2.06e+01, 1.44e+00 / 1.52e+00 | 20260507 04:57:09.780 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.63e+00, 3.77e-01 / 2.51e-01 | 20260507 04:57:09.780 | INFO | PerfReport: 2.99%, Metrics -- 9.21e-01 / 1.25e+00, 2.47e-01 / 1.09e-01 | 20260507 04:57:09.780 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.11e-01 / 3.71e-01, 5.15e-02 / 2.63e-02 | 20260507 04:57:09.780 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:57:09.780 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:57:09.780 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:57:09.780 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 04:57:09.780 | INFO | PerfReport: 21.88%, Upload -- 2.11e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260507 04:57:09.780 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.42e+00 / 0.00e+00 | 20260507 04:57:09.780 | INFO | PerfReport: 32.00%, Compute -- 3.08e+01 / 0.00e+00, 1.28e+01 / 0.00e+00 | 20260507 04:57:09.780 | INFO | PerfReport: 1.40%, LockUpload -- 1.35e+00 / 0.00e+00, 4.67e+00 / 0.00e+00 | 20260507 04:57:09.780 | INFO | PerfReport: 1.24%, LockDownload -- 1.19e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260507 04:57:09.780 | INFO | PerfReport: 22.08%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 04:57:09.780 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260507 04:57:09.875 | INFO | Starting a round of Baz IO | 20260507 04:57:12.978 | INFO | Populating RTMetrics report | 20260507 04:57:12.978 | INFO | RTMetrics report populated | 20260507 04:57:12.978 | INFO | Opening tmp RTMetrics json file | 20260507 04:57:12.979 | INFO | Serializing RTMetrics to json | 20260507 04:57:12.994 | INFO | Serialization done, swapping tmp file now... | 20260507 04:57:12.998 | INFO | RTMetrics done | 20260507 04:57:13.066 | INFO | Finished a round of Baz IO: Wrote 1.712 GiB in 3.19 seconds | 20260507 04:57:13.066 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.75% / 0.00% / 67.25% | 20260507 04:57:13.066 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.156669% / 0% / 7.54992e-05% | 20260507 04:57:13.066 | INFO | Baz IO overall: | 20260507 04:57:13.066 | INFO | BytesWritten : 3697748393411 | 20260507 04:57:13.066 | INFO | NumEvents : 1774430645480 | 20260507 04:57:14.916 | INFO | Analyzing chunk frames = [7266304,7266816) | 20260507 04:57:14.916 | INFO | MemInfo: { | 20260507 04:57:14.916 | INFO | "gpu" : " 17 GB", | 20260507 04:57:14.916 | INFO | "huge" : "", | 20260507 04:57:14.916 | INFO | "huge_shared" : " 27 GB", | 20260507 04:57:14.916 | INFO | "malloc" : " 14 GB", | 20260507 04:57:14.916 | INFO | "pinned" : " 29 GB" | 20260507 04:57:14.916 | INFO | } | 20260507 04:57:14.916 | INFO | Perf report for the last 5.13785 seconds: | 20260507 04:57:14.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:57:14.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:14.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:14.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:14.916 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 04:57:14.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:14.916 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 04:57:14.916 | INFO | PRE_HQ : 59.1%, 1.8, 55.0ms, 0 | 20260507 04:57:14.916 | INFO | BAZWRITER : 61.6%, 3.2, 101.3ms, 0 | 20260507 04:57:14.916 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:57:14.916 | INFO | Uploaded 36GiB to GPU in 2.04287s (17.6223GiB/s) | 20260507 04:57:14.916 | INFO | Downloaded 38GiB to GPU in 2.17972s (17.4334GiB/s) | 20260507 04:57:20.054 | INFO | Analyzing chunk frames = [7266816,7267328) | 20260507 04:57:20.054 | INFO | MemInfo: { | 20260507 04:57:20.054 | INFO | "gpu" : " 17 GB", | 20260507 04:57:20.054 | INFO | "huge" : "", | 20260507 04:57:20.054 | INFO | "huge_shared" : " 27 GB", | 20260507 04:57:20.054 | INFO | "malloc" : " 14 GB", | 20260507 04:57:20.054 | INFO | "pinned" : " 29 GB" | 20260507 04:57:20.054 | INFO | } | 20260507 04:57:20.054 | INFO | Perf report for the last 5.1381 seconds: | 20260507 04:57:20.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:57:20.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:20.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:20.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:20.054 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 04:57:20.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:20.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:20.054 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:57:20.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:20.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:20.054 | INFO | Uploaded 36GiB to GPU in 1.98914s (18.0983GiB/s) | 20260507 04:57:20.054 | INFO | Downloaded 34GiB to GPU in 1.93498s (17.5713GiB/s) | 20260507 04:57:25.198 | INFO | Analyzing chunk frames = [7267328,7267840) | 20260507 04:57:25.198 | INFO | MemInfo: { | 20260507 04:57:25.198 | INFO | "gpu" : " 17 GB", | 20260507 04:57:25.198 | INFO | "huge" : "", | 20260507 04:57:25.198 | INFO | "huge_shared" : " 27 GB", | 20260507 04:57:25.198 | INFO | "malloc" : " 14 GB", | 20260507 04:57:25.198 | INFO | "pinned" : " 29 GB" | 20260507 04:57:25.198 | INFO | } | 20260507 04:57:25.198 | INFO | Perf report for the last 5.14386 seconds: | 20260507 04:57:25.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:57:25.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:25.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:25.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:25.198 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 04:57:25.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:25.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:25.198 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:57:25.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:25.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:25.198 | INFO | Uploaded 36GiB to GPU in 1.9883s (18.1059GiB/s) | 20260507 04:57:25.198 | INFO | Downloaded 34GiB to GPU in 1.94389s (17.4907GiB/s) | 20260507 04:57:30.343 | INFO | Analyzing chunk frames = [7267840,7268352) | 20260507 04:57:30.343 | INFO | MemInfo: { | 20260507 04:57:30.343 | INFO | "gpu" : " 17 GB", | 20260507 04:57:30.343 | INFO | "huge" : "", | 20260507 04:57:30.343 | INFO | "huge_shared" : " 27 GB", | 20260507 04:57:30.343 | INFO | "malloc" : " 14 GB", | 20260507 04:57:30.343 | INFO | "pinned" : " 29 GB" | 20260507 04:57:30.343 | INFO | } | 20260507 04:57:30.343 | INFO | Perf report for the last 5.14495 seconds: | 20260507 04:57:30.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:57:30.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:30.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:30.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:30.343 | INFO | ANALYSIS : 59.5%, 3.0, 92.5ms, 0 | 20260507 04:57:30.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:30.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:30.343 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 04:57:30.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:30.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:30.343 | INFO | Uploaded 36GiB to GPU in 1.98164s (18.1667GiB/s) | 20260507 04:57:30.343 | INFO | Downloaded 34GiB to GPU in 1.93069s (17.6103GiB/s) | 20260507 04:57:35.473 | INFO | Analyzing chunk frames = [7268352,7268864) | 20260507 04:57:35.473 | INFO | MemInfo: { | 20260507 04:57:35.474 | INFO | "gpu" : " 17 GB", | 20260507 04:57:35.474 | INFO | "huge" : "", | 20260507 04:57:35.474 | INFO | "huge_shared" : " 27 GB", | 20260507 04:57:35.474 | INFO | "malloc" : " 14 GB", | 20260507 04:57:35.474 | INFO | "pinned" : " 29 GB" | 20260507 04:57:35.474 | INFO | } | 20260507 04:57:35.474 | INFO | Perf report for the last 5.13079 seconds: | 20260507 04:57:35.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:57:35.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:35.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:35.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:35.474 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 04:57:35.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:35.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:35.474 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:57:35.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:35.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:35.474 | INFO | Uploaded 36GiB to GPU in 2.03695s (17.6735GiB/s) | 20260507 04:57:35.474 | INFO | Downloaded 34GiB to GPU in 1.91108s (17.791GiB/s) | 20260507 04:57:40.610 | INFO | Analyzing chunk frames = [7268864,7269376) | 20260507 04:57:40.610 | INFO | MemInfo: { | 20260507 04:57:40.610 | INFO | "gpu" : " 17 GB", | 20260507 04:57:40.610 | INFO | "huge" : "", | 20260507 04:57:40.610 | INFO | "huge_shared" : " 27 GB", | 20260507 04:57:40.610 | INFO | "malloc" : " 14 GB", | 20260507 04:57:40.610 | INFO | "pinned" : " 29 GB" | 20260507 04:57:40.610 | INFO | } | 20260507 04:57:40.610 | INFO | Perf report for the last 5.13613 seconds: | 20260507 04:57:40.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:57:40.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:40.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:40.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:40.610 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 04:57:40.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:40.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:40.610 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:57:40.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:40.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:40.610 | INFO | Uploaded 36GiB to GPU in 2.03848s (17.6602GiB/s) | 20260507 04:57:40.610 | INFO | Downloaded 34GiB to GPU in 1.92216s (17.6884GiB/s) | 20260507 04:57:45.756 | INFO | Analyzing chunk frames = [7269376,7269888) | 20260507 04:57:45.756 | INFO | MemInfo: { | 20260507 04:57:45.756 | INFO | "gpu" : " 17 GB", | 20260507 04:57:45.756 | INFO | "huge" : "", | 20260507 04:57:45.756 | INFO | "huge_shared" : " 27 GB", | 20260507 04:57:45.756 | INFO | "malloc" : " 14 GB", | 20260507 04:57:45.756 | INFO | "pinned" : " 29 GB" | 20260507 04:57:45.756 | INFO | } | 20260507 04:57:45.756 | INFO | Perf report for the last 5.14634 seconds: | 20260507 04:57:45.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:57:45.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:45.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:45.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:45.756 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 04:57:45.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:45.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:45.756 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 04:57:45.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:45.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:45.756 | INFO | Uploaded 36GiB to GPU in 1.96477s (18.3227GiB/s) | 20260507 04:57:45.756 | INFO | Downloaded 34GiB to GPU in 1.96581s (17.2957GiB/s) | 20260507 04:57:50.894 | INFO | Analyzing chunk frames = [7269888,7270400) | 20260507 04:57:50.894 | INFO | MemInfo: { | 20260507 04:57:50.894 | INFO | "gpu" : " 17 GB", | 20260507 04:57:50.894 | INFO | "huge" : "", | 20260507 04:57:50.894 | INFO | "huge_shared" : " 27 GB", | 20260507 04:57:50.894 | INFO | "malloc" : " 14 GB", | 20260507 04:57:50.894 | INFO | "pinned" : " 29 GB" | 20260507 04:57:50.894 | INFO | } | 20260507 04:57:50.894 | INFO | Perf report for the last 5.13764 seconds: | 20260507 04:57:50.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:57:50.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:50.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:50.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:50.894 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:57:50.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:50.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:50.894 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 04:57:50.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:50.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:50.894 | INFO | Uploaded 36GiB to GPU in 1.96895s (18.2838GiB/s) | 20260507 04:57:50.894 | INFO | Downloaded 34GiB to GPU in 1.962s (17.3292GiB/s) | 20260507 04:57:50.991 | INFO | Starting a round of Baz IO | 20260507 04:57:54.088 | INFO | Populating RTMetrics report | 20260507 04:57:54.088 | INFO | RTMetrics report populated | 20260507 04:57:54.088 | INFO | Opening tmp RTMetrics json file | 20260507 04:57:54.089 | INFO | Serializing RTMetrics to json | 20260507 04:57:54.090 | INFO | Serialization done, swapping tmp file now... | 20260507 04:57:54.091 | INFO | RTMetrics done | 20260507 04:57:54.175 | INFO | Finished a round of Baz IO: Wrote 1.71 GiB in 3.184 seconds | 20260507 04:57:54.175 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.75% / 0.00% / 67.25% | 20260507 04:57:54.175 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.155481% / 0% / 5.16574e-05% | 20260507 04:57:54.175 | INFO | Baz IO overall: | 20260507 04:57:54.175 | INFO | BytesWritten : 3699584339020 | 20260507 04:57:54.175 | INFO | NumEvents : 1775293862471 | 20260507 04:57:56.038 | INFO | Analyzing chunk frames = [7270400,7270912) | 20260507 04:57:56.038 | INFO | MemInfo: { | 20260507 04:57:56.038 | INFO | "gpu" : " 17 GB", | 20260507 04:57:56.038 | INFO | "huge" : "", | 20260507 04:57:56.038 | INFO | "huge_shared" : " 27 GB", | 20260507 04:57:56.038 | INFO | "malloc" : " 14 GB", | 20260507 04:57:56.038 | INFO | "pinned" : " 29 GB" | 20260507 04:57:56.038 | INFO | } | 20260507 04:57:56.038 | INFO | Perf report for the last 5.14443 seconds: | 20260507 04:57:56.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:57:56.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:56.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:56.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:56.038 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 04:57:56.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:57:56.038 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 04:57:56.038 | INFO | PRE_HQ : 59.0%, 1.8, 55.3ms, 0 | 20260507 04:57:56.038 | INFO | BAZWRITER : 61.4%, 3.2, 101.5ms, 0 | 20260507 04:57:56.038 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:57:56.038 | INFO | Uploaded 36GiB to GPU in 2.04604s (17.595GiB/s) | 20260507 04:57:56.038 | INFO | Downloaded 38GiB to GPU in 2.17219s (17.4939GiB/s) | 20260507 04:58:01.173 | INFO | Analyzing chunk frames = [7270912,7271424) | 20260507 04:58:01.173 | INFO | MemInfo: { | 20260507 04:58:01.173 | INFO | "gpu" : " 17 GB", | 20260507 04:58:01.173 | INFO | "huge" : "", | 20260507 04:58:01.173 | INFO | "huge_shared" : " 27 GB", | 20260507 04:58:01.173 | INFO | "malloc" : " 14 GB", | 20260507 04:58:01.173 | INFO | "pinned" : " 29 GB" | 20260507 04:58:01.173 | INFO | } | 20260507 04:58:01.173 | INFO | Perf report for the last 5.13517 seconds: | 20260507 04:58:01.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:58:01.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:01.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:01.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:01.173 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 04:58:01.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:01.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:01.173 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 04:58:01.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:01.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:01.174 | INFO | Uploaded 36GiB to GPU in 1.9888s (18.1014GiB/s) | 20260507 04:58:01.174 | INFO | Downloaded 34GiB to GPU in 1.93187s (17.5995GiB/s) | 20260507 04:58:06.309 | INFO | Analyzing chunk frames = [7271424,7271936) | 20260507 04:58:06.309 | INFO | MemInfo: { | 20260507 04:58:06.309 | INFO | "gpu" : " 17 GB", | 20260507 04:58:06.309 | INFO | "huge" : "", | 20260507 04:58:06.309 | INFO | "huge_shared" : " 27 GB", | 20260507 04:58:06.309 | INFO | "malloc" : " 14 GB", | 20260507 04:58:06.309 | INFO | "pinned" : " 29 GB" | 20260507 04:58:06.309 | INFO | } | 20260507 04:58:06.309 | INFO | Perf report for the last 5.13564 seconds: | 20260507 04:58:06.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:58:06.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:06.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:06.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:06.309 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 04:58:06.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:06.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:06.309 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:58:06.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:06.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:06.309 | INFO | Uploaded 36GiB to GPU in 2.05249s (17.5397GiB/s) | 20260507 04:58:06.309 | INFO | Downloaded 34GiB to GPU in 1.94883s (17.4464GiB/s) | 20260507 04:58:11.451 | INFO | Analyzing chunk frames = [7271936,7272448) | 20260507 04:58:11.451 | INFO | MemInfo: { | 20260507 04:58:11.451 | INFO | "gpu" : " 17 GB", | 20260507 04:58:11.451 | INFO | "huge" : "", | 20260507 04:58:11.451 | INFO | "huge_shared" : " 27 GB", | 20260507 04:58:11.451 | INFO | "malloc" : " 14 GB", | 20260507 04:58:11.451 | INFO | "pinned" : " 29 GB" | 20260507 04:58:11.451 | INFO | } | 20260507 04:58:11.451 | INFO | Perf report for the last 5.14193 seconds: | 20260507 04:58:11.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:58:11.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:11.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:11.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:11.451 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 04:58:11.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:11.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:11.451 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:58:11.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:11.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:11.451 | INFO | Uploaded 36GiB to GPU in 1.99256s (18.0672GiB/s) | 20260507 04:58:11.451 | INFO | Downloaded 34GiB to GPU in 1.92294s (17.6813GiB/s) | 20260507 04:58:16.590 | INFO | Analyzing chunk frames = [7272448,7272960) | 20260507 04:58:16.590 | INFO | MemInfo: { | 20260507 04:58:16.590 | INFO | "gpu" : " 17 GB", | 20260507 04:58:16.590 | INFO | "huge" : "", | 20260507 04:58:16.590 | INFO | "huge_shared" : " 27 GB", | 20260507 04:58:16.590 | INFO | "malloc" : " 14 GB", | 20260507 04:58:16.590 | INFO | "pinned" : " 29 GB" | 20260507 04:58:16.590 | INFO | } | 20260507 04:58:16.590 | INFO | Perf report for the last 5.1391 seconds: | 20260507 04:58:16.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:58:16.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:16.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:16.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:16.590 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 04:58:16.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:16.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:16.590 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 04:58:16.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:16.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:16.590 | INFO | Uploaded 36GiB to GPU in 2.04088s (17.6395GiB/s) | 20260507 04:58:16.590 | INFO | Downloaded 34GiB to GPU in 1.93678s (17.5549GiB/s) | 20260507 04:58:21.732 | INFO | Analyzing chunk frames = [7272960,7273472) | 20260507 04:58:21.732 | INFO | MemInfo: { | 20260507 04:58:21.732 | INFO | "gpu" : " 17 GB", | 20260507 04:58:21.732 | INFO | "huge" : "", | 20260507 04:58:21.732 | INFO | "huge_shared" : " 27 GB", | 20260507 04:58:21.732 | INFO | "malloc" : " 14 GB", | 20260507 04:58:21.732 | INFO | "pinned" : " 29 GB" | 20260507 04:58:21.732 | INFO | } | 20260507 04:58:21.732 | INFO | Perf report for the last 5.14183 seconds: | 20260507 04:58:21.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:58:21.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:21.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:21.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:21.732 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 04:58:21.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:21.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:21.732 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 04:58:21.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:21.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:21.732 | INFO | Uploaded 36GiB to GPU in 2.03978s (17.649GiB/s) | 20260507 04:58:21.732 | INFO | Downloaded 34GiB to GPU in 1.89905s (17.9036GiB/s) | 20260507 04:58:26.868 | INFO | Analyzing chunk frames = [7273472,7273984) | 20260507 04:58:26.868 | INFO | MemInfo: { | 20260507 04:58:26.868 | INFO | "gpu" : " 17 GB", | 20260507 04:58:26.868 | INFO | "huge" : "", | 20260507 04:58:26.868 | INFO | "huge_shared" : " 27 GB", | 20260507 04:58:26.868 | INFO | "malloc" : " 14 GB", | 20260507 04:58:26.868 | INFO | "pinned" : " 29 GB" | 20260507 04:58:26.868 | INFO | } | 20260507 04:58:26.868 | INFO | Perf report for the last 5.13571 seconds: | 20260507 04:58:26.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:58:26.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:26.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:26.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:26.868 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 04:58:26.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:26.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:26.868 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:58:26.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:26.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:26.868 | INFO | Uploaded 36GiB to GPU in 1.93134s (18.6399GiB/s) | 20260507 04:58:26.868 | INFO | Downloaded 34GiB to GPU in 1.97236s (17.2382GiB/s) | 20260507 04:58:32.008 | INFO | Analyzing chunk frames = [7273984,7274496) | 20260507 04:58:32.008 | INFO | MemInfo: { | 20260507 04:58:32.008 | INFO | "gpu" : " 17 GB", | 20260507 04:58:32.008 | INFO | "huge" : "", | 20260507 04:58:32.008 | INFO | "huge_shared" : " 27 GB", | 20260507 04:58:32.008 | INFO | "malloc" : " 14 GB", | 20260507 04:58:32.008 | INFO | "pinned" : " 29 GB" | 20260507 04:58:32.008 | INFO | } | 20260507 04:58:32.008 | INFO | Perf report for the last 5.14 seconds: | 20260507 04:58:32.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:58:32.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:32.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:32.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:32.008 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 04:58:32.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:32.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:32.008 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 04:58:32.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:32.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:32.008 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:58:32.008 | INFO | Total mem usage: 18 GiB | 20260507 04:58:32.008 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:58:32.008 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 04:58:32.008 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:58:32.008 | INFO | High Water Breakdown: | 20260507 04:58:32.008 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:58:32.008 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:58:32.008 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:58:32.008 | INFO | ------------------------------------------ | 20260507 04:58:32.008 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:58:32.008 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:58:32.008 | INFO | Total mem usage: 26.5 GiB | 20260507 04:58:32.008 | INFO | Uploaded 36GiB to GPU in 1.96001s (18.3672GiB/s) | 20260507 04:58:32.008 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:58:32.008 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:58:32.008 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:58:32.008 | INFO | High Water Breakdown: | 20260507 04:58:32.008 | INFO | Downloaded 34GiB to GPU in 1.95617s (17.3809GiB/s) | 20260507 04:58:32.008 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:58:32.008 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:58:32.008 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:58:32.008 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:58:32.008 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:58:32.008 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:58:32.008 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:58:32.008 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:58:32.008 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:58:32.008 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:58:32.008 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:58:32.008 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:58:32.008 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:58:32.008 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:58:32.008 | INFO | ------------------------------------------ | 20260507 04:58:32.008 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:58:32.008 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:58:32.008 | INFO | Total mem usage: 25.6 GiB | 20260507 04:58:32.008 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:58:32.008 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:58:32.008 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:58:32.008 | INFO | High Water Breakdown: | 20260507 04:58:32.008 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:58:32.008 | INFO | ------------------------------------------ | 20260507 04:58:32.009 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:58:32.009 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:58:32.009 | INFO | Total mem usage: 15.7 GiB | 20260507 04:58:32.009 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:58:32.009 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:58:32.009 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:58:32.009 | INFO | High Water Breakdown: | 20260507 04:58:32.009 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:58:32.009 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:58:32.009 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:58:32.009 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:58:32.009 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:58:32.009 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:58:32.009 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:58:32.009 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:58:32.009 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:58:32.009 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:58:32.009 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:58:32.009 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:58:32.009 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:58:32.009 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:58:32.009 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:58:32.009 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:58:32.009 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:58:32.009 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:58:32.009 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:58:32.009 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:58:32.009 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:58:32.009 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:58:32.009 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:58:32.009 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:58:32.009 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:58:32.009 | INFO | ------------------------------------------ | 20260507 04:58:32.009 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:58:32.009 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:58:32.009 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:58:32.009 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:58:32.009 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.49e+01, 1.34e+01 / 1.13e+00 | 20260507 04:58:32.009 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.00e-01, 9.78e-03 / 4.70e-03 | 20260507 04:58:32.009 | INFO | PerfReport: 11.69%, Baseline -- 3.60e+00 / 4.45e+00, 5.28e-01 / 2.45e-01 | 20260507 04:58:32.009 | INFO | PerfReport: 3.25%, Histogram -- 9.99e-01 / 1.48e+00, 1.74e-01 / 1.03e-01 | 20260507 04:58:32.009 | INFO | PerfReport: 16.13%, DME -- 3.97e+01 / 4.44e+01, 3.91e+00 / 1.64e+00 | 20260507 04:58:32.009 | INFO | PerfReport: 57.69%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.35e+00 / 1.52e+00 | 20260507 04:58:32.009 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.13e+00 / 2.67e+00, 3.82e-01 / 2.35e-01 | 20260507 04:58:32.009 | INFO | PerfReport: 3.00%, Metrics -- 9.24e-01 / 1.22e+00, 2.43e-01 / 1.30e-01 | 20260507 04:58:32.009 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.11e-01 / 3.62e-01, 4.99e-02 / 1.91e-02 | 20260507 04:58:32.009 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:58:32.009 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:58:32.009 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:58:32.009 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 04:58:32.009 | INFO | PerfReport: 21.71%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260507 04:58:32.009 | INFO | PerfReport: 21.33%, Download -- 2.05e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 04:58:32.009 | INFO | PerfReport: 32.03%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 04:58:32.009 | INFO | PerfReport: 1.46%, LockUpload -- 1.41e+00 / 0.00e+00, 4.76e+00 / 0.00e+00 | 20260507 04:58:32.009 | INFO | PerfReport: 1.25%, LockDownload -- 1.20e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260507 04:58:32.009 | INFO | PerfReport: 22.20%, LockCompute -- 2.13e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 04:58:32.009 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260507 04:58:32.105 | INFO | Starting a round of Baz IO | 20260507 04:58:35.208 | INFO | Populating RTMetrics report | 20260507 04:58:35.209 | INFO | RTMetrics report populated | 20260507 04:58:35.209 | INFO | Opening tmp RTMetrics json file | 20260507 04:58:35.227 | INFO | Serializing RTMetrics to json | 20260507 04:58:35.228 | INFO | Serialization done, swapping tmp file now... | 20260507 04:58:35.229 | INFO | RTMetrics done | 20260507 04:58:35.296 | INFO | Finished a round of Baz IO: Wrote 1.71 GiB in 3.19 seconds | 20260507 04:58:35.296 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.74% / 0.00% / 67.25% | 20260507 04:58:35.296 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.155699% / 0% / 6.35783e-05% | 20260507 04:58:35.296 | INFO | Baz IO overall: | 20260507 04:58:35.296 | INFO | BytesWritten : 3701420195548 | 20260507 04:58:35.296 | INFO | NumEvents : 1776157047497 | 20260507 04:58:37.150 | INFO | Analyzing chunk frames = [7274496,7275008) | 20260507 04:58:37.150 | INFO | MemInfo: { | 20260507 04:58:37.150 | INFO | "gpu" : " 17 GB", | 20260507 04:58:37.150 | INFO | "huge" : "", | 20260507 04:58:37.150 | INFO | "huge_shared" : " 27 GB", | 20260507 04:58:37.150 | INFO | "malloc" : " 14 GB", | 20260507 04:58:37.150 | INFO | "pinned" : " 29 GB" | 20260507 04:58:37.150 | INFO | } | 20260507 04:58:37.150 | INFO | Perf report for the last 5.14259 seconds: | 20260507 04:58:37.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:58:37.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:37.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:37.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:37.150 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 04:58:37.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:37.150 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 04:58:37.150 | INFO | PRE_HQ : 59.0%, 1.8, 55.0ms, 0 | 20260507 04:58:37.150 | INFO | BAZWRITER : 61.6%, 3.2, 101.5ms, 0 | 20260507 04:58:37.150 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:58:37.151 | INFO | Uploaded 36GiB to GPU in 2.03761s (17.6677GiB/s) | 20260507 04:58:37.151 | INFO | Downloaded 38GiB to GPU in 2.17724s (17.4533GiB/s) | 20260507 04:58:42.287 | INFO | Analyzing chunk frames = [7275008,7275520) | 20260507 04:58:42.287 | INFO | MemInfo: { | 20260507 04:58:42.287 | INFO | "gpu" : " 17 GB", | 20260507 04:58:42.287 | INFO | "huge" : "", | 20260507 04:58:42.287 | INFO | "huge_shared" : " 27 GB", | 20260507 04:58:42.287 | INFO | "malloc" : " 14 GB", | 20260507 04:58:42.287 | INFO | "pinned" : " 29 GB" | 20260507 04:58:42.287 | INFO | } | 20260507 04:58:42.287 | INFO | Perf report for the last 5.13634 seconds: | 20260507 04:58:42.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:58:42.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:42.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:42.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:42.287 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 04:58:42.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:42.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:42.287 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 04:58:42.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:42.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:42.287 | INFO | Uploaded 36GiB to GPU in 1.99128s (18.0789GiB/s) | 20260507 04:58:42.287 | INFO | Downloaded 34GiB to GPU in 1.92907s (17.6251GiB/s) | 20260507 04:58:47.426 | INFO | Analyzing chunk frames = [7275520,7276032) | 20260507 04:58:47.426 | INFO | MemInfo: { | 20260507 04:58:47.426 | INFO | "gpu" : " 17 GB", | 20260507 04:58:47.426 | INFO | "huge" : "", | 20260507 04:58:47.426 | INFO | "huge_shared" : " 27 GB", | 20260507 04:58:47.426 | INFO | "malloc" : " 14 GB", | 20260507 04:58:47.426 | INFO | "pinned" : " 29 GB" | 20260507 04:58:47.426 | INFO | } | 20260507 04:58:47.426 | INFO | Perf report for the last 5.13967 seconds: | 20260507 04:58:47.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:58:47.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:47.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:47.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:47.426 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 04:58:47.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:47.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:47.426 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 04:58:47.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:47.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:47.427 | INFO | Uploaded 36GiB to GPU in 2.05109s (17.5516GiB/s) | 20260507 04:58:47.427 | INFO | Downloaded 34GiB to GPU in 1.95011s (17.4349GiB/s) | 20260507 04:58:52.564 | INFO | Analyzing chunk frames = [7276032,7276544) | 20260507 04:58:52.564 | INFO | MemInfo: { | 20260507 04:58:52.564 | INFO | "gpu" : " 17 GB", | 20260507 04:58:52.564 | INFO | "huge" : "", | 20260507 04:58:52.564 | INFO | "huge_shared" : " 27 GB", | 20260507 04:58:52.564 | INFO | "malloc" : " 14 GB", | 20260507 04:58:52.564 | INFO | "pinned" : " 29 GB" | 20260507 04:58:52.564 | INFO | } | 20260507 04:58:52.564 | INFO | Perf report for the last 5.13743 seconds: | 20260507 04:58:52.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:58:52.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:52.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:52.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:52.564 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 04:58:52.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:52.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:52.564 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 04:58:52.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:52.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:52.564 | INFO | Uploaded 36GiB to GPU in 2.00593s (17.9468GiB/s) | 20260507 04:58:52.564 | INFO | Downloaded 34GiB to GPU in 1.92969s (17.6194GiB/s) | 20260507 04:58:57.710 | INFO | Analyzing chunk frames = [7276544,7277056) | 20260507 04:58:57.710 | INFO | MemInfo: { | 20260507 04:58:57.710 | INFO | "gpu" : " 17 GB", | 20260507 04:58:57.710 | INFO | "huge" : "", | 20260507 04:58:57.710 | INFO | "huge_shared" : " 27 GB", | 20260507 04:58:57.710 | INFO | "malloc" : " 14 GB", | 20260507 04:58:57.710 | INFO | "pinned" : " 29 GB" | 20260507 04:58:57.710 | INFO | } | 20260507 04:58:57.710 | INFO | Perf report for the last 5.14622 seconds: | 20260507 04:58:57.710 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:58:57.710 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:57.710 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:57.710 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:57.710 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 04:58:57.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:57.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:57.710 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 04:58:57.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:57.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:58:57.710 | INFO | Uploaded 36GiB to GPU in 2.03464s (17.6936GiB/s) | 20260507 04:58:57.710 | INFO | Downloaded 34GiB to GPU in 1.93396s (17.5805GiB/s) | 20260507 04:59:02.853 | INFO | Analyzing chunk frames = [7277056,7277568) | 20260507 04:59:02.853 | INFO | MemInfo: { | 20260507 04:59:02.853 | INFO | "gpu" : " 17 GB", | 20260507 04:59:02.853 | INFO | "huge" : "", | 20260507 04:59:02.853 | INFO | "huge_shared" : " 27 GB", | 20260507 04:59:02.853 | INFO | "malloc" : " 14 GB", | 20260507 04:59:02.853 | INFO | "pinned" : " 29 GB" | 20260507 04:59:02.853 | INFO | } | 20260507 04:59:02.853 | INFO | Perf report for the last 5.14278 seconds: | 20260507 04:59:02.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:59:02.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:02.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:02.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:02.853 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 04:59:02.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:02.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:02.853 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 04:59:02.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:02.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:02.853 | INFO | Uploaded 36GiB to GPU in 2.05352s (17.5309GiB/s) | 20260507 04:59:02.853 | INFO | Downloaded 34GiB to GPU in 1.91404s (17.7635GiB/s) | 20260507 04:59:07.983 | INFO | Analyzing chunk frames = [7277568,7278080) | 20260507 04:59:07.983 | INFO | MemInfo: { | 20260507 04:59:07.983 | INFO | "gpu" : " 17 GB", | 20260507 04:59:07.983 | INFO | "huge" : "", | 20260507 04:59:07.983 | INFO | "huge_shared" : " 27 GB", | 20260507 04:59:07.983 | INFO | "malloc" : " 14 GB", | 20260507 04:59:07.983 | INFO | "pinned" : " 29 GB" | 20260507 04:59:07.983 | INFO | } | 20260507 04:59:07.983 | INFO | Perf report for the last 5.12985 seconds: | 20260507 04:59:07.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:59:07.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:07.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:07.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:07.983 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:59:07.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:07.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:07.983 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 04:59:07.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:07.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:07.983 | INFO | Uploaded 36GiB to GPU in 1.91712s (18.7782GiB/s) | 20260507 04:59:07.983 | INFO | Downloaded 34GiB to GPU in 1.97255s (17.2366GiB/s) | 20260507 04:59:13.124 | INFO | Analyzing chunk frames = [7278080,7278592) | 20260507 04:59:13.124 | INFO | MemInfo: { | 20260507 04:59:13.124 | INFO | "gpu" : " 17 GB", | 20260507 04:59:13.124 | INFO | "huge" : "", | 20260507 04:59:13.124 | INFO | "huge_shared" : " 27 GB", | 20260507 04:59:13.124 | INFO | "malloc" : " 14 GB", | 20260507 04:59:13.124 | INFO | "pinned" : " 29 GB" | 20260507 04:59:13.124 | INFO | } | 20260507 04:59:13.124 | INFO | Perf report for the last 5.14175 seconds: | 20260507 04:59:13.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:59:13.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:13.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:13.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:13.124 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:59:13.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:13.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:13.125 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260507 04:59:13.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:13.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:13.125 | INFO | Uploaded 36GiB to GPU in 1.9709s (18.2658GiB/s) | 20260507 04:59:13.125 | INFO | Downloaded 34GiB to GPU in 1.95814s (17.3634GiB/s) | 20260507 04:59:13.221 | INFO | Starting a round of Baz IO | 20260507 04:59:16.317 | INFO | Populating RTMetrics report | 20260507 04:59:16.317 | INFO | RTMetrics report populated | 20260507 04:59:16.317 | INFO | Opening tmp RTMetrics json file | 20260507 04:59:16.324 | INFO | Serializing RTMetrics to json | 20260507 04:59:16.334 | INFO | Serialization done, swapping tmp file now... | 20260507 04:59:16.335 | INFO | RTMetrics done | 20260507 04:59:16.407 | INFO | Finished a round of Baz IO: Wrote 1.712 GiB in 3.185 seconds | 20260507 04:59:16.407 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.74% / 0.00% / 67.26% | 20260507 04:59:16.407 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.156188% / 0% / 7.94729e-05% | 20260507 04:59:16.407 | INFO | Baz IO overall: | 20260507 04:59:16.407 | INFO | BytesWritten : 3703257904941 | 20260507 04:59:16.407 | INFO | NumEvents : 1777021662236 | 20260507 04:59:18.259 | INFO | Analyzing chunk frames = [7278592,7279104) | 20260507 04:59:18.259 | INFO | MemInfo: { | 20260507 04:59:18.259 | INFO | "gpu" : " 17 GB", | 20260507 04:59:18.259 | INFO | "huge" : "", | 20260507 04:59:18.259 | INFO | "huge_shared" : " 27 GB", | 20260507 04:59:18.259 | INFO | "malloc" : " 14 GB", | 20260507 04:59:18.259 | INFO | "pinned" : " 29 GB" | 20260507 04:59:18.259 | INFO | } | 20260507 04:59:18.259 | INFO | Perf report for the last 5.13467 seconds: | 20260507 04:59:18.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:59:18.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:18.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:18.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:18.259 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260507 04:59:18.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:18.259 | INFO | RT_METRICS : 41.7%, 1.0, 21.8ms, 0 | 20260507 04:59:18.259 | INFO | PRE_HQ : 59.3%, 1.8, 55.3ms, 0 | 20260507 04:59:18.259 | INFO | BAZWRITER : 61.5%, 3.2, 102.0ms, 0 | 20260507 04:59:18.259 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:59:18.259 | INFO | Uploaded 36GiB to GPU in 2.02507s (17.7771GiB/s) | 20260507 04:59:18.259 | INFO | Downloaded 38GiB to GPU in 2.18689s (17.3763GiB/s) | 20260507 04:59:23.405 | INFO | Analyzing chunk frames = [7279104,7279616) | 20260507 04:59:23.405 | INFO | MemInfo: { | 20260507 04:59:23.405 | INFO | "gpu" : " 17 GB", | 20260507 04:59:23.405 | INFO | "huge" : "", | 20260507 04:59:23.405 | INFO | "huge_shared" : " 27 GB", | 20260507 04:59:23.405 | INFO | "malloc" : " 14 GB", | 20260507 04:59:23.405 | INFO | "pinned" : " 29 GB" | 20260507 04:59:23.405 | INFO | } | 20260507 04:59:23.405 | INFO | Perf report for the last 5.14602 seconds: | 20260507 04:59:23.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:59:23.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:23.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:23.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:23.405 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 04:59:23.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:23.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:23.405 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 04:59:23.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:23.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:23.405 | INFO | Uploaded 36GiB to GPU in 2.03028s (17.7315GiB/s) | 20260507 04:59:23.405 | INFO | Downloaded 34GiB to GPU in 1.93848s (17.5395GiB/s) | 20260507 04:59:28.544 | INFO | Analyzing chunk frames = [7279616,7280128) | 20260507 04:59:28.544 | INFO | MemInfo: { | 20260507 04:59:28.544 | INFO | "gpu" : " 17 GB", | 20260507 04:59:28.544 | INFO | "huge" : "", | 20260507 04:59:28.544 | INFO | "huge_shared" : " 27 GB", | 20260507 04:59:28.544 | INFO | "malloc" : " 14 GB", | 20260507 04:59:28.544 | INFO | "pinned" : " 29 GB" | 20260507 04:59:28.544 | INFO | } | 20260507 04:59:28.544 | INFO | Perf report for the last 5.13924 seconds: | 20260507 04:59:28.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:59:28.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:28.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:28.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:28.544 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 04:59:28.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:28.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:28.544 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 04:59:28.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:28.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:28.545 | INFO | Uploaded 36GiB to GPU in 2.04161s (17.6332GiB/s) | 20260507 04:59:28.545 | INFO | Downloaded 34GiB to GPU in 1.95171s (17.4207GiB/s) | 20260507 04:59:33.683 | INFO | Analyzing chunk frames = [7280128,7280640) | 20260507 04:59:33.683 | INFO | MemInfo: { | 20260507 04:59:33.683 | INFO | "gpu" : " 17 GB", | 20260507 04:59:33.683 | INFO | "huge" : "", | 20260507 04:59:33.683 | INFO | "huge_shared" : " 27 GB", | 20260507 04:59:33.683 | INFO | "malloc" : " 14 GB", | 20260507 04:59:33.683 | INFO | "pinned" : " 29 GB" | 20260507 04:59:33.683 | INFO | } | 20260507 04:59:33.683 | INFO | Perf report for the last 5.13849 seconds: | 20260507 04:59:33.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:59:33.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:33.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:33.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:33.683 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 04:59:33.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:33.683 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:33.683 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:59:33.683 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:33.683 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:33.683 | INFO | Uploaded 36GiB to GPU in 2.02742s (17.7565GiB/s) | 20260507 04:59:33.683 | INFO | Downloaded 34GiB to GPU in 1.95365s (17.4033GiB/s) | 20260507 04:59:38.815 | INFO | Analyzing chunk frames = [7280640,7281152) | 20260507 04:59:38.815 | INFO | MemInfo: { | 20260507 04:59:38.815 | INFO | "gpu" : " 17 GB", | 20260507 04:59:38.815 | INFO | "huge" : "", | 20260507 04:59:38.815 | INFO | "huge_shared" : " 27 GB", | 20260507 04:59:38.815 | INFO | "malloc" : " 14 GB", | 20260507 04:59:38.815 | INFO | "pinned" : " 29 GB" | 20260507 04:59:38.815 | INFO | } | 20260507 04:59:38.815 | INFO | Perf report for the last 5.13202 seconds: | 20260507 04:59:38.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:59:38.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:38.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:38.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:38.815 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 04:59:38.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:38.815 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:38.815 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 04:59:38.815 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:38.815 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:38.815 | INFO | Uploaded 36GiB to GPU in 2.04987s (17.5621GiB/s) | 20260507 04:59:38.815 | INFO | Downloaded 34GiB to GPU in 1.95448s (17.3959GiB/s) | 20260507 04:59:43.961 | INFO | Analyzing chunk frames = [7281152,7281664) | 20260507 04:59:43.961 | INFO | MemInfo: { | 20260507 04:59:43.961 | INFO | "gpu" : " 17 GB", | 20260507 04:59:43.961 | INFO | "huge" : "", | 20260507 04:59:43.961 | INFO | "huge_shared" : " 27 GB", | 20260507 04:59:43.961 | INFO | "malloc" : " 14 GB", | 20260507 04:59:43.961 | INFO | "pinned" : " 29 GB" | 20260507 04:59:43.961 | INFO | } | 20260507 04:59:43.961 | INFO | Perf report for the last 5.14596 seconds: | 20260507 04:59:43.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:59:43.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:43.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:43.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:43.961 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 04:59:43.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:43.961 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:43.961 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 04:59:43.961 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:43.961 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:43.961 | INFO | Uploaded 36GiB to GPU in 2.04935s (17.5666GiB/s) | 20260507 04:59:43.961 | INFO | Downloaded 34GiB to GPU in 1.96091s (17.3389GiB/s) | 20260507 04:59:49.100 | INFO | Analyzing chunk frames = [7281664,7282176) | 20260507 04:59:49.100 | INFO | MemInfo: { | 20260507 04:59:49.100 | INFO | "gpu" : " 17 GB", | 20260507 04:59:49.100 | INFO | "huge" : "", | 20260507 04:59:49.100 | INFO | "huge_shared" : " 27 GB", | 20260507 04:59:49.100 | INFO | "malloc" : " 14 GB", | 20260507 04:59:49.100 | INFO | "pinned" : " 29 GB" | 20260507 04:59:49.100 | INFO | } | 20260507 04:59:49.100 | INFO | Perf report for the last 5.13928 seconds: | 20260507 04:59:49.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:59:49.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:49.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:49.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:49.100 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 04:59:49.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:49.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:49.100 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 04:59:49.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:49.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:49.101 | INFO | Uploaded 36GiB to GPU in 1.97181s (18.2573GiB/s) | 20260507 04:59:49.101 | INFO | Downloaded 34GiB to GPU in 1.98961s (17.0888GiB/s) | 20260507 04:59:54.235 | INFO | Analyzing chunk frames = [7282176,7282688) | 20260507 04:59:54.235 | INFO | MemInfo: { | 20260507 04:59:54.235 | INFO | "gpu" : " 17 GB", | 20260507 04:59:54.235 | INFO | "huge" : "", | 20260507 04:59:54.235 | INFO | "huge_shared" : " 27 GB", | 20260507 04:59:54.235 | INFO | "malloc" : " 14 GB", | 20260507 04:59:54.235 | INFO | "pinned" : " 29 GB" | 20260507 04:59:54.235 | INFO | } | 20260507 04:59:54.235 | INFO | Perf report for the last 5.13512 seconds: | 20260507 04:59:54.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:59:54.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:54.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:54.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:54.235 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 04:59:54.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:54.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:54.235 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 04:59:54.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:54.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:54.236 | INFO | Uploaded 36GiB to GPU in 1.98579s (18.1288GiB/s) | 20260507 04:59:54.236 | INFO | Downloaded 34GiB to GPU in 1.97445s (17.2199GiB/s) | 20260507 04:59:54.236 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 04:59:54.236 | INFO | Total mem usage: 18 GiB | 20260507 04:59:54.236 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 04:59:54.236 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 04:59:54.236 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 04:59:54.236 | INFO | High Water Breakdown: | 20260507 04:59:54.236 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 04:59:54.236 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 04:59:54.236 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 04:59:54.236 | INFO | ------------------------------------------ | 20260507 04:59:54.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:59:54.236 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 04:59:54.236 | INFO | Total mem usage: 26.5 GiB | 20260507 04:59:54.236 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 04:59:54.236 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 04:59:54.236 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 04:59:54.236 | INFO | High Water Breakdown: | 20260507 04:59:54.236 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:59:54.236 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:59:54.236 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:59:54.236 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:59:54.236 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:59:54.236 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:59:54.236 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 04:59:54.236 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 04:59:54.236 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:59:54.236 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 04:59:54.236 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 04:59:54.236 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 04:59:54.236 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 04:59:54.236 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 04:59:54.236 | INFO | ------------------------------------------ | 20260507 04:59:54.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:59:54.236 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 04:59:54.236 | INFO | Total mem usage: 25.6 GiB | 20260507 04:59:54.236 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 04:59:54.236 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 04:59:54.236 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 04:59:54.236 | INFO | High Water Breakdown: | 20260507 04:59:54.236 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 04:59:54.236 | INFO | ------------------------------------------ | 20260507 04:59:54.236 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:59:54.236 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 04:59:54.236 | INFO | Total mem usage: 15.7 GiB | 20260507 04:59:54.237 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 04:59:54.237 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 04:59:54.237 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 04:59:54.237 | INFO | High Water Breakdown: | 20260507 04:59:54.237 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 04:59:54.237 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 04:59:54.237 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 04:59:54.237 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 04:59:54.237 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 04:59:54.237 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 04:59:54.237 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 04:59:54.237 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 04:59:54.237 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 04:59:54.237 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 04:59:54.237 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 04:59:54.237 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 04:59:54.237 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 04:59:54.237 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 04:59:54.237 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 04:59:54.237 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 04:59:54.237 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 04:59:54.237 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 04:59:54.237 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 04:59:54.237 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 04:59:54.237 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 04:59:54.237 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 04:59:54.237 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 04:59:54.237 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 04:59:54.237 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 04:59:54.237 | INFO | ------------------------------------------ | 20260507 04:59:54.237 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 04:59:54.237 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 04:59:54.237 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:59:54.237 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:59:54.237 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.50e+01, 1.35e+01 / 1.26e+00 | 20260507 04:59:54.237 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 9.32e-03 / 3.93e-03 | 20260507 04:59:54.237 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.44e+00, 5.00e-01 / 2.62e-01 | 20260507 04:59:54.237 | INFO | PerfReport: 3.20%, Histogram -- 9.82e-01 / 1.47e+00, 1.61e-01 / 1.13e-01 | 20260507 04:59:54.237 | INFO | PerfReport: 16.29%, DME -- 4.00e+01 / 4.52e+01, 4.10e+00 / 1.64e+00 | 20260507 04:59:54.237 | INFO | PerfReport: 57.50%, FrameLabeling -- 1.76e+01 / 1.94e+01, 1.12e+00 / 1.04e+00 | 20260507 04:59:54.237 | INFO | PerfReport: 7.00%, PulseAccumulating -- 2.15e+00 / 2.75e+00, 3.95e-01 / 1.68e-01 | 20260507 04:59:54.237 | INFO | PerfReport: 3.02%, Metrics -- 9.26e-01 / 1.23e+00, 2.43e-01 / 1.02e-01 | 20260507 04:59:54.237 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.12e-01 / 3.71e-01, 4.96e-02 / 2.18e-02 | 20260507 04:59:54.237 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 04:59:54.237 | INFO | PerfReport: Gathered 1536 samples. | 20260507 04:59:54.237 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 04:59:54.237 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 04:59:54.237 | INFO | PerfReport: 21.81%, Upload -- 2.10e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260507 04:59:54.237 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260507 04:59:54.237 | INFO | PerfReport: 31.87%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 04:59:54.237 | INFO | PerfReport: 1.49%, LockUpload -- 1.44e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 04:59:54.237 | INFO | PerfReport: 1.35%, LockDownload -- 1.30e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260507 04:59:54.237 | INFO | PerfReport: 22.03%, LockCompute -- 2.12e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 04:59:54.237 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.34e-02 / 0.00e+00 | 20260507 04:59:54.334 | INFO | Starting a round of Baz IO | 20260507 04:59:57.433 | INFO | Populating RTMetrics report | 20260507 04:59:57.433 | INFO | RTMetrics report populated | 20260507 04:59:57.433 | INFO | Opening tmp RTMetrics json file | 20260507 04:59:57.434 | INFO | Serializing RTMetrics to json | 20260507 04:59:57.435 | INFO | Serialization done, swapping tmp file now... | 20260507 04:59:57.438 | INFO | RTMetrics done | 20260507 04:59:57.523 | INFO | Finished a round of Baz IO: Wrote 1.711 GiB in 3.189 seconds | 20260507 04:59:57.523 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.74% / 0.00% / 67.26% | 20260507 04:59:57.523 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.155592% / 0% / 7.54992e-05% | 20260507 04:59:57.523 | INFO | Baz IO overall: | 20260507 04:59:57.523 | INFO | BytesWritten : 3705095156135 | 20260507 04:59:57.523 | INFO | NumEvents : 1777885930850 | 20260507 04:59:59.381 | INFO | Analyzing chunk frames = [7282688,7283200) | 20260507 04:59:59.381 | INFO | MemInfo: { | 20260507 04:59:59.381 | INFO | "gpu" : " 17 GB", | 20260507 04:59:59.381 | INFO | "huge" : "", | 20260507 04:59:59.381 | INFO | "huge_shared" : " 27 GB", | 20260507 04:59:59.381 | INFO | "malloc" : " 14 GB", | 20260507 04:59:59.381 | INFO | "pinned" : " 29 GB" | 20260507 04:59:59.381 | INFO | } | 20260507 04:59:59.381 | INFO | Perf report for the last 5.14541 seconds: | 20260507 04:59:59.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 04:59:59.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:59.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:59.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:59.381 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 04:59:59.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 04:59:59.381 | INFO | RT_METRICS : 41.2%, 1.0, 21.7ms, 0 | 20260507 04:59:59.381 | INFO | PRE_HQ : 59.2%, 1.8, 55.4ms, 0 | 20260507 04:59:59.381 | INFO | BAZWRITER : 61.6%, 3.2, 102.2ms, 0 | 20260507 04:59:59.381 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 04:59:59.381 | INFO | Uploaded 36GiB to GPU in 2.04619s (17.5937GiB/s) | 20260507 04:59:59.381 | INFO | Downloaded 38GiB to GPU in 2.1804s (17.428GiB/s) | 20260507 05:00:04.518 | INFO | Analyzing chunk frames = [7283200,7283712) | 20260507 05:00:04.518 | INFO | MemInfo: { | 20260507 05:00:04.518 | INFO | "gpu" : " 17 GB", | 20260507 05:00:04.518 | INFO | "huge" : "", | 20260507 05:00:04.518 | INFO | "huge_shared" : " 27 GB", | 20260507 05:00:04.518 | INFO | "malloc" : " 14 GB", | 20260507 05:00:04.518 | INFO | "pinned" : " 29 GB" | 20260507 05:00:04.518 | INFO | } | 20260507 05:00:04.518 | INFO | Perf report for the last 5.13759 seconds: | 20260507 05:00:04.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:00:04.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:04.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:04.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:04.518 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 05:00:04.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:04.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:04.518 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:00:04.519 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:04.519 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:04.519 | INFO | Uploaded 36GiB to GPU in 2.04792s (17.5788GiB/s) | 20260507 05:00:04.519 | INFO | Downloaded 34GiB to GPU in 1.94611s (17.4708GiB/s) | 20260507 05:00:09.661 | INFO | Analyzing chunk frames = [7283712,7284224) | 20260507 05:00:09.661 | INFO | MemInfo: { | 20260507 05:00:09.661 | INFO | "gpu" : " 17 GB", | 20260507 05:00:09.661 | INFO | "huge" : "", | 20260507 05:00:09.661 | INFO | "huge_shared" : " 27 GB", | 20260507 05:00:09.661 | INFO | "malloc" : " 14 GB", | 20260507 05:00:09.661 | INFO | "pinned" : " 29 GB" | 20260507 05:00:09.661 | INFO | } | 20260507 05:00:09.661 | INFO | Perf report for the last 5.14299 seconds: | 20260507 05:00:09.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:00:09.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:09.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:09.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:09.661 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 05:00:09.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:09.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:09.661 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:00:09.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:09.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:09.662 | INFO | Uploaded 36GiB to GPU in 2.02187s (17.8053GiB/s) | 20260507 05:00:09.662 | INFO | Downloaded 34GiB to GPU in 1.95512s (17.3902GiB/s) | 20260507 05:00:14.799 | INFO | Analyzing chunk frames = [7284224,7284736) | 20260507 05:00:14.799 | INFO | MemInfo: { | 20260507 05:00:14.799 | INFO | "gpu" : " 17 GB", | 20260507 05:00:14.799 | INFO | "huge" : "", | 20260507 05:00:14.799 | INFO | "huge_shared" : " 27 GB", | 20260507 05:00:14.799 | INFO | "malloc" : " 14 GB", | 20260507 05:00:14.799 | INFO | "pinned" : " 29 GB" | 20260507 05:00:14.799 | INFO | } | 20260507 05:00:14.799 | INFO | Perf report for the last 5.13734 seconds: | 20260507 05:00:14.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:00:14.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:14.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:14.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:14.799 | INFO | ANALYSIS : 60.3%, 2.9, 93.2ms, 0 | 20260507 05:00:14.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:14.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:14.799 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:00:14.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:14.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:14.799 | INFO | Uploaded 36GiB to GPU in 2.01096s (17.9019GiB/s) | 20260507 05:00:14.799 | INFO | Downloaded 34GiB to GPU in 1.95484s (17.3927GiB/s) | 20260507 05:00:19.934 | INFO | Analyzing chunk frames = [7284736,7285248) | 20260507 05:00:19.934 | INFO | MemInfo: { | 20260507 05:00:19.934 | INFO | "gpu" : " 17 GB", | 20260507 05:00:19.934 | INFO | "huge" : "", | 20260507 05:00:19.934 | INFO | "huge_shared" : " 27 GB", | 20260507 05:00:19.934 | INFO | "malloc" : " 14 GB", | 20260507 05:00:19.934 | INFO | "pinned" : " 29 GB" | 20260507 05:00:19.934 | INFO | } | 20260507 05:00:19.934 | INFO | Perf report for the last 5.13527 seconds: | 20260507 05:00:19.934 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:00:19.934 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:19.934 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:19.934 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:19.934 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 05:00:19.934 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:19.934 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:19.934 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:00:19.934 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:19.934 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:19.934 | INFO | Uploaded 36GiB to GPU in 2.03952s (17.6512GiB/s) | 20260507 05:00:19.934 | INFO | Downloaded 34GiB to GPU in 1.91729s (17.7334GiB/s) | 20260507 05:00:25.071 | INFO | Analyzing chunk frames = [7285248,7285760) | 20260507 05:00:25.071 | INFO | MemInfo: { | 20260507 05:00:25.071 | INFO | "gpu" : " 17 GB", | 20260507 05:00:25.071 | INFO | "huge" : "", | 20260507 05:00:25.071 | INFO | "huge_shared" : " 27 GB", | 20260507 05:00:25.071 | INFO | "malloc" : " 14 GB", | 20260507 05:00:25.071 | INFO | "pinned" : " 29 GB" | 20260507 05:00:25.071 | INFO | } | 20260507 05:00:25.071 | INFO | Perf report for the last 5.13716 seconds: | 20260507 05:00:25.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:00:25.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:25.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:25.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:25.071 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 05:00:25.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:25.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:25.071 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:00:25.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:25.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:25.071 | INFO | Uploaded 36GiB to GPU in 2.04816s (17.5767GiB/s) | 20260507 05:00:25.072 | INFO | Downloaded 34GiB to GPU in 1.9127s (17.7759GiB/s) | 20260507 05:00:30.212 | INFO | Analyzing chunk frames = [7285760,7286272) | 20260507 05:00:30.212 | INFO | MemInfo: { | 20260507 05:00:30.212 | INFO | "gpu" : " 17 GB", | 20260507 05:00:30.212 | INFO | "huge" : "", | 20260507 05:00:30.212 | INFO | "huge_shared" : " 27 GB", | 20260507 05:00:30.212 | INFO | "malloc" : " 14 GB", | 20260507 05:00:30.212 | INFO | "pinned" : " 29 GB" | 20260507 05:00:30.212 | INFO | } | 20260507 05:00:30.212 | INFO | Perf report for the last 5.14087 seconds: | 20260507 05:00:30.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:00:30.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:30.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:30.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:30.212 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 05:00:30.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:30.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:30.212 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 05:00:30.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:30.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:30.212 | INFO | Uploaded 36GiB to GPU in 2.0143s (17.8722GiB/s) | 20260507 05:00:30.212 | INFO | Downloaded 34GiB to GPU in 1.9924s (17.0648GiB/s) | 20260507 05:00:35.359 | INFO | Analyzing chunk frames = [7286272,7286784) | 20260507 05:00:35.359 | INFO | MemInfo: { | 20260507 05:00:35.359 | INFO | "gpu" : " 17 GB", | 20260507 05:00:35.359 | INFO | "huge" : "", | 20260507 05:00:35.359 | INFO | "huge_shared" : " 27 GB", | 20260507 05:00:35.359 | INFO | "malloc" : " 14 GB", | 20260507 05:00:35.359 | INFO | "pinned" : " 29 GB" | 20260507 05:00:35.359 | INFO | } | 20260507 05:00:35.359 | INFO | Perf report for the last 5.14704 seconds: | 20260507 05:00:35.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:00:35.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:35.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:35.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:35.359 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 05:00:35.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:35.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:35.359 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 05:00:35.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:35.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:35.359 | INFO | Uploaded 36GiB to GPU in 2.02004s (17.8214GiB/s) | 20260507 05:00:35.359 | INFO | Downloaded 34GiB to GPU in 1.97841s (17.1855GiB/s) | 20260507 05:00:35.456 | INFO | Starting a round of Baz IO | 20260507 05:00:38.559 | INFO | Populating RTMetrics report | 20260507 05:00:38.559 | INFO | RTMetrics report populated | 20260507 05:00:38.559 | INFO | Opening tmp RTMetrics json file | 20260507 05:00:38.566 | INFO | Serializing RTMetrics to json | 20260507 05:00:38.576 | INFO | Serialization done, swapping tmp file now... | 20260507 05:00:38.577 | INFO | RTMetrics done | 20260507 05:00:38.649 | INFO | Finished a round of Baz IO: Wrote 1.709 GiB in 3.193 seconds | 20260507 05:00:38.649 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.74% / 0.00% / 67.26% | 20260507 05:00:38.649 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.155342% / 3.97364e-06% / 6.35783e-05% | 20260507 05:00:38.649 | INFO | Baz IO overall: | 20260507 05:00:38.649 | INFO | BytesWritten : 3706930025335 | 20260507 05:00:38.649 | INFO | NumEvents : 1778749105884 | 20260507 05:00:40.499 | INFO | Analyzing chunk frames = [7286784,7287296) | 20260507 05:00:40.499 | INFO | MemInfo: { | 20260507 05:00:40.499 | INFO | "gpu" : " 17 GB", | 20260507 05:00:40.499 | INFO | "huge" : "", | 20260507 05:00:40.499 | INFO | "huge_shared" : " 27 GB", | 20260507 05:00:40.499 | INFO | "malloc" : " 14 GB", | 20260507 05:00:40.499 | INFO | "pinned" : " 29 GB" | 20260507 05:00:40.499 | INFO | } | 20260507 05:00:40.499 | INFO | Perf report for the last 5.13964 seconds: | 20260507 05:00:40.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:00:40.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:40.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:40.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:40.499 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:00:40.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:40.499 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260507 05:00:40.499 | INFO | PRE_HQ : 59.4%, 1.9, 57.7ms, 0 | 20260507 05:00:40.499 | INFO | BAZWRITER : 61.7%, 3.2, 102.8ms, 0 | 20260507 05:00:40.499 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 05:00:40.499 | INFO | Uploaded 36GiB to GPU in 2.05659s (17.5047GiB/s) | 20260507 05:00:40.499 | INFO | Downloaded 38GiB to GPU in 2.17651s (17.4591GiB/s) | 20260507 05:00:45.630 | INFO | Analyzing chunk frames = [7287296,7287808) | 20260507 05:00:45.630 | INFO | MemInfo: { | 20260507 05:00:45.630 | INFO | "gpu" : " 17 GB", | 20260507 05:00:45.630 | INFO | "huge" : "", | 20260507 05:00:45.631 | INFO | "huge_shared" : " 27 GB", | 20260507 05:00:45.631 | INFO | "malloc" : " 14 GB", | 20260507 05:00:45.631 | INFO | "pinned" : " 29 GB" | 20260507 05:00:45.631 | INFO | } | 20260507 05:00:45.631 | INFO | Perf report for the last 5.13184 seconds: | 20260507 05:00:45.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:00:45.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:45.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:45.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:45.631 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:00:45.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:45.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:45.631 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 05:00:45.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:45.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:45.631 | INFO | Uploaded 36GiB to GPU in 1.9966s (18.0307GiB/s) | 20260507 05:00:45.631 | INFO | Downloaded 34GiB to GPU in 1.93591s (17.5628GiB/s) | 20260507 05:00:50.772 | INFO | Analyzing chunk frames = [7287808,7288320) | 20260507 05:00:50.772 | INFO | MemInfo: { | 20260507 05:00:50.772 | INFO | "gpu" : " 17 GB", | 20260507 05:00:50.772 | INFO | "huge" : "", | 20260507 05:00:50.772 | INFO | "huge_shared" : " 27 GB", | 20260507 05:00:50.772 | INFO | "malloc" : " 14 GB", | 20260507 05:00:50.772 | INFO | "pinned" : " 29 GB" | 20260507 05:00:50.772 | INFO | } | 20260507 05:00:50.772 | INFO | Perf report for the last 5.14142 seconds: | 20260507 05:00:50.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:00:50.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:50.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:50.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:50.772 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 05:00:50.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:50.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:50.772 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:00:50.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:50.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:50.772 | INFO | Uploaded 36GiB to GPU in 2.0357s (17.6843GiB/s) | 20260507 05:00:50.772 | INFO | Downloaded 34GiB to GPU in 1.95125s (17.4247GiB/s) | 20260507 05:00:55.913 | INFO | Analyzing chunk frames = [7288320,7288832) | 20260507 05:00:55.913 | INFO | MemInfo: { | 20260507 05:00:55.913 | INFO | "gpu" : " 17 GB", | 20260507 05:00:55.913 | INFO | "huge" : "", | 20260507 05:00:55.913 | INFO | "huge_shared" : " 27 GB", | 20260507 05:00:55.913 | INFO | "malloc" : " 14 GB", | 20260507 05:00:55.913 | INFO | "pinned" : " 29 GB" | 20260507 05:00:55.913 | INFO | } | 20260507 05:00:55.913 | INFO | Perf report for the last 5.14063 seconds: | 20260507 05:00:55.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:00:55.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:55.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:55.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:55.913 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 05:00:55.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:55.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:55.913 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:00:55.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:55.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:00:55.913 | INFO | Uploaded 36GiB to GPU in 1.99537s (18.0418GiB/s) | 20260507 05:00:55.913 | INFO | Downloaded 34GiB to GPU in 1.91891s (17.7184GiB/s) | 20260507 05:01:01.049 | INFO | Analyzing chunk frames = [7288832,7289344) | 20260507 05:01:01.049 | INFO | MemInfo: { | 20260507 05:01:01.049 | INFO | "gpu" : " 17 GB", | 20260507 05:01:01.049 | INFO | "huge" : "", | 20260507 05:01:01.049 | INFO | "huge_shared" : " 27 GB", | 20260507 05:01:01.049 | INFO | "malloc" : " 14 GB", | 20260507 05:01:01.049 | INFO | "pinned" : " 29 GB" | 20260507 05:01:01.049 | INFO | } | 20260507 05:01:01.049 | INFO | Perf report for the last 5.13621 seconds: | 20260507 05:01:01.049 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:01:01.049 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:01.049 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:01.049 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:01.049 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 05:01:01.049 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:01.049 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:01.049 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:01:01.049 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:01.049 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:01.049 | INFO | Uploaded 36GiB to GPU in 2.05878s (17.4861GiB/s) | 20260507 05:01:01.049 | INFO | Downloaded 34GiB to GPU in 1.94922s (17.4429GiB/s) | 20260507 05:01:06.188 | INFO | Analyzing chunk frames = [7289344,7289856) | 20260507 05:01:06.188 | INFO | MemInfo: { | 20260507 05:01:06.188 | INFO | "gpu" : " 17 GB", | 20260507 05:01:06.188 | INFO | "huge" : "", | 20260507 05:01:06.188 | INFO | "huge_shared" : " 27 GB", | 20260507 05:01:06.188 | INFO | "malloc" : " 14 GB", | 20260507 05:01:06.188 | INFO | "pinned" : " 29 GB" | 20260507 05:01:06.188 | INFO | } | 20260507 05:01:06.188 | INFO | Perf report for the last 5.13914 seconds: | 20260507 05:01:06.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:01:06.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:06.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:06.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:06.188 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 05:01:06.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:06.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:06.188 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:01:06.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:06.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:06.188 | INFO | Uploaded 36GiB to GPU in 2.03282s (17.7094GiB/s) | 20260507 05:01:06.189 | INFO | Downloaded 34GiB to GPU in 1.90087s (17.8865GiB/s) | 20260507 05:01:11.327 | INFO | Analyzing chunk frames = [7289856,7290368) | 20260507 05:01:11.327 | INFO | MemInfo: { | 20260507 05:01:11.327 | INFO | "gpu" : " 17 GB", | 20260507 05:01:11.327 | INFO | "huge" : "", | 20260507 05:01:11.327 | INFO | "huge_shared" : " 27 GB", | 20260507 05:01:11.327 | INFO | "malloc" : " 14 GB", | 20260507 05:01:11.327 | INFO | "pinned" : " 29 GB" | 20260507 05:01:11.327 | INFO | } | 20260507 05:01:11.327 | INFO | Perf report for the last 5.13936 seconds: | 20260507 05:01:11.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:01:11.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:11.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:11.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:11.327 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 05:01:11.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:11.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:11.328 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:01:11.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:11.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:11.328 | INFO | Uploaded 36GiB to GPU in 1.98918s (18.0979GiB/s) | 20260507 05:01:11.328 | INFO | Downloaded 34GiB to GPU in 1.9803s (17.1691GiB/s) | 20260507 05:01:16.474 | INFO | Analyzing chunk frames = [7290368,7290880) | 20260507 05:01:16.474 | INFO | MemInfo: { | 20260507 05:01:16.474 | INFO | "gpu" : " 17 GB", | 20260507 05:01:16.474 | INFO | "huge" : "", | 20260507 05:01:16.475 | INFO | "huge_shared" : " 27 GB", | 20260507 05:01:16.475 | INFO | "malloc" : " 14 GB", | 20260507 05:01:16.475 | INFO | "pinned" : " 29 GB" | 20260507 05:01:16.475 | INFO | } | 20260507 05:01:16.475 | INFO | Perf report for the last 5.14724 seconds: | 20260507 05:01:16.475 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:01:16.475 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:16.475 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:16.475 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:16.475 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 05:01:16.475 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:16.475 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:16.475 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 05:01:16.475 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:16.475 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:16.475 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:01:16.475 | INFO | Uploaded 36GiB to GPU in 2.00954s (17.9145GiB/s) | 20260507 05:01:16.475 | INFO | Total mem usage: 18 GiB | 20260507 05:01:16.475 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:01:16.475 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 05:01:16.475 | INFO | Downloaded 34GiB to GPU in 1.97122s (17.2482GiB/s) | 20260507 05:01:16.475 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:01:16.475 | INFO | High Water Breakdown: | 20260507 05:01:16.475 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:01:16.475 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:01:16.475 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:01:16.475 | INFO | ------------------------------------------ | 20260507 05:01:16.475 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:01:16.475 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:01:16.475 | INFO | Total mem usage: 26.5 GiB | 20260507 05:01:16.475 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:01:16.475 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:01:16.475 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:01:16.475 | INFO | High Water Breakdown: | 20260507 05:01:16.475 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:01:16.475 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:01:16.475 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:01:16.475 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:01:16.475 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:01:16.475 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:01:16.475 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:01:16.476 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:01:16.476 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:01:16.476 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:01:16.476 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:01:16.476 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:01:16.476 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:01:16.476 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:01:16.476 | INFO | ------------------------------------------ | 20260507 05:01:16.476 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:01:16.476 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:01:16.476 | INFO | Total mem usage: 25.6 GiB | 20260507 05:01:16.476 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:01:16.476 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 05:01:16.476 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 05:01:16.476 | INFO | High Water Breakdown: | 20260507 05:01:16.476 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 05:01:16.476 | INFO | ------------------------------------------ | 20260507 05:01:16.476 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:01:16.476 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:01:16.476 | INFO | Total mem usage: 15.7 GiB | 20260507 05:01:16.476 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:01:16.476 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:01:16.476 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:01:16.476 | INFO | High Water Breakdown: | 20260507 05:01:16.476 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:01:16.476 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:01:16.476 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:01:16.476 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:01:16.476 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:01:16.476 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:01:16.476 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:01:16.476 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:01:16.476 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:01:16.476 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:01:16.476 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:01:16.476 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:01:16.476 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:01:16.476 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:01:16.476 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:01:16.476 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:01:16.476 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:01:16.476 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:01:16.476 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:01:16.476 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:01:16.476 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:01:16.476 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:01:16.476 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:01:16.476 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:01:16.476 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:01:16.476 | INFO | ------------------------------------------ | 20260507 05:01:16.476 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:01:16.476 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:01:16.476 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:01:16.476 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:01:16.476 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.51e+01, 1.34e+01 / 1.85e+00 | 20260507 05:01:16.476 | INFO | PerfReport: 0.30%, AggregateStats -- 9.12e-02 / 1.01e-01, 1.06e-02 / 4.50e-03 | 20260507 05:01:16.476 | INFO | PerfReport: 11.66%, Baseline -- 3.59e+00 / 4.47e+00, 5.05e-01 / 2.06e-01 | 20260507 05:01:16.476 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.49e+00, 1.69e-01 / 8.53e-02 | 20260507 05:01:16.476 | INFO | PerfReport: 16.27%, DME -- 4.00e+01 / 4.52e+01, 4.20e+00 / 1.92e+00 | 20260507 05:01:16.476 | INFO | PerfReport: 57.59%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.26e+00 / 9.99e-01 | 20260507 05:01:16.476 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.68e+00, 3.81e-01 / 2.01e-01 | 20260507 05:01:16.476 | INFO | PerfReport: 3.00%, Metrics -- 9.22e-01 / 1.21e+00, 2.45e-01 / 1.19e-01 | 20260507 05:01:16.476 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.11e-01 / 3.68e-01, 4.96e-02 / 2.27e-02 | 20260507 05:01:16.476 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:01:16.476 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:01:16.476 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:01:16.476 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 05:01:16.476 | INFO | PerfReport: 21.91%, Upload -- 2.11e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260507 05:01:16.476 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 05:01:16.476 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 05:01:16.476 | INFO | PerfReport: 1.56%, LockUpload -- 1.50e+00 / 0.00e+00, 5.07e+00 / 0.00e+00 | 20260507 05:01:16.476 | INFO | PerfReport: 1.29%, LockDownload -- 1.25e+00 / 0.00e+00, 3.00e+00 / 0.00e+00 | 20260507 05:01:16.476 | INFO | PerfReport: 21.94%, LockCompute -- 2.11e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 05:01:16.476 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260507 05:01:16.571 | INFO | Starting a round of Baz IO | 20260507 05:01:19.700 | INFO | Populating RTMetrics report | 20260507 05:01:19.701 | INFO | RTMetrics report populated | 20260507 05:01:19.701 | INFO | Opening tmp RTMetrics json file | 20260507 05:01:19.702 | INFO | Serializing RTMetrics to json | 20260507 05:01:19.714 | INFO | Serialization done, swapping tmp file now... | 20260507 05:01:19.720 | INFO | RTMetrics done | 20260507 05:01:19.803 | INFO | Finished a round of Baz IO: Wrote 1.709 GiB in 3.231 seconds | 20260507 05:01:19.803 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.74% / 0.00% / 67.26% | 20260507 05:01:19.803 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.154587% / 3.97364e-06% / 7.15256e-05% | 20260507 05:01:19.803 | INFO | Baz IO overall: | 20260507 05:01:19.803 | INFO | BytesWritten : 3708765225718 | 20260507 05:01:19.803 | INFO | NumEvents : 1779612140022 | 20260507 05:01:21.607 | INFO | Analyzing chunk frames = [7290880,7291392) | 20260507 05:01:21.607 | INFO | MemInfo: { | 20260507 05:01:21.607 | INFO | "gpu" : " 17 GB", | 20260507 05:01:21.607 | INFO | "huge" : "", | 20260507 05:01:21.607 | INFO | "huge_shared" : " 27 GB", | 20260507 05:01:21.607 | INFO | "malloc" : " 14 GB", | 20260507 05:01:21.607 | INFO | "pinned" : " 29 GB" | 20260507 05:01:21.607 | INFO | } | 20260507 05:01:21.607 | INFO | Perf report for the last 5.13265 seconds: | 20260507 05:01:21.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:01:21.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:21.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:21.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:21.607 | INFO | ANALYSIS : 61.8%, 3.0, 95.6ms, 0 | 20260507 05:01:21.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:21.608 | INFO | RT_METRICS : 45.8%, 1.0, 24.0ms, 0 | 20260507 05:01:21.608 | INFO | PRE_HQ : 60.3%, 2.0, 61.1ms, 0 | 20260507 05:01:21.608 | INFO | BAZWRITER : 62.6%, 3.4, 109.2ms, 0 | 20260507 05:01:21.608 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 05:01:21.608 | INFO | Uploaded 36GiB to GPU in 2.07148s (17.3788GiB/s) | 20260507 05:01:21.608 | INFO | Downloaded 38GiB to GPU in 2.16929s (17.5172GiB/s) | 20260507 05:01:26.746 | INFO | Analyzing chunk frames = [7291392,7291904) | 20260507 05:01:26.746 | INFO | MemInfo: { | 20260507 05:01:26.746 | INFO | "gpu" : " 17 GB", | 20260507 05:01:26.746 | INFO | "huge" : "", | 20260507 05:01:26.746 | INFO | "huge_shared" : " 27 GB", | 20260507 05:01:26.746 | INFO | "malloc" : " 14 GB", | 20260507 05:01:26.746 | INFO | "pinned" : " 29 GB" | 20260507 05:01:26.746 | INFO | } | 20260507 05:01:26.746 | INFO | Perf report for the last 5.13902 seconds: | 20260507 05:01:26.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:01:26.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:26.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:26.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:26.746 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 05:01:26.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:26.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:26.746 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 05:01:26.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:26.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:26.747 | INFO | Uploaded 36GiB to GPU in 1.99394s (18.0547GiB/s) | 20260507 05:01:26.747 | INFO | Downloaded 34GiB to GPU in 1.92854s (17.6299GiB/s) | 20260507 05:01:31.882 | INFO | Analyzing chunk frames = [7291904,7292416) | 20260507 05:01:31.882 | INFO | MemInfo: { | 20260507 05:01:31.882 | INFO | "gpu" : " 17 GB", | 20260507 05:01:31.882 | INFO | "huge" : "", | 20260507 05:01:31.882 | INFO | "huge_shared" : " 27 GB", | 20260507 05:01:31.882 | INFO | "malloc" : " 14 GB", | 20260507 05:01:31.882 | INFO | "pinned" : " 29 GB" | 20260507 05:01:31.882 | INFO | } | 20260507 05:01:31.882 | INFO | Perf report for the last 5.13589 seconds: | 20260507 05:01:31.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:01:31.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:31.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:31.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:31.882 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 05:01:31.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:31.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:31.882 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:01:31.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:31.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:31.883 | INFO | Uploaded 36GiB to GPU in 2.04346s (17.6172GiB/s) | 20260507 05:01:31.883 | INFO | Downloaded 34GiB to GPU in 1.94875s (17.4471GiB/s) | 20260507 05:01:37.025 | INFO | Analyzing chunk frames = [7292416,7292928) | 20260507 05:01:37.025 | INFO | MemInfo: { | 20260507 05:01:37.025 | INFO | "gpu" : " 17 GB", | 20260507 05:01:37.025 | INFO | "huge" : "", | 20260507 05:01:37.025 | INFO | "huge_shared" : " 27 GB", | 20260507 05:01:37.025 | INFO | "malloc" : " 14 GB", | 20260507 05:01:37.025 | INFO | "pinned" : " 29 GB" | 20260507 05:01:37.025 | INFO | } | 20260507 05:01:37.026 | INFO | Perf report for the last 5.14323 seconds: | 20260507 05:01:37.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:01:37.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:37.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:37.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:37.026 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:01:37.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:37.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:37.026 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:01:37.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:37.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:37.026 | INFO | Uploaded 36GiB to GPU in 1.9922s (18.0705GiB/s) | 20260507 05:01:37.026 | INFO | Downloaded 34GiB to GPU in 1.92084s (17.7006GiB/s) | 20260507 05:01:42.168 | INFO | Analyzing chunk frames = [7292928,7293440) | 20260507 05:01:42.168 | INFO | MemInfo: { | 20260507 05:01:42.168 | INFO | "gpu" : " 17 GB", | 20260507 05:01:42.168 | INFO | "huge" : "", | 20260507 05:01:42.168 | INFO | "huge_shared" : " 27 GB", | 20260507 05:01:42.168 | INFO | "malloc" : " 14 GB", | 20260507 05:01:42.168 | INFO | "pinned" : " 29 GB" | 20260507 05:01:42.168 | INFO | } | 20260507 05:01:42.168 | INFO | Perf report for the last 5.14287 seconds: | 20260507 05:01:42.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:01:42.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:42.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:42.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:42.169 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 05:01:42.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:42.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:42.169 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:01:42.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:42.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:42.169 | INFO | Uploaded 36GiB to GPU in 2.03203s (17.7163GiB/s) | 20260507 05:01:42.169 | INFO | Downloaded 34GiB to GPU in 1.94048s (17.5214GiB/s) | 20260507 05:01:47.306 | INFO | Analyzing chunk frames = [7293440,7293952) | 20260507 05:01:47.306 | INFO | MemInfo: { | 20260507 05:01:47.306 | INFO | "gpu" : " 17 GB", | 20260507 05:01:47.306 | INFO | "huge" : "", | 20260507 05:01:47.306 | INFO | "huge_shared" : " 27 GB", | 20260507 05:01:47.306 | INFO | "malloc" : " 14 GB", | 20260507 05:01:47.306 | INFO | "pinned" : " 29 GB" | 20260507 05:01:47.306 | INFO | } | 20260507 05:01:47.306 | INFO | Perf report for the last 5.1377 seconds: | 20260507 05:01:47.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:01:47.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:47.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:47.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:47.306 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 05:01:47.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:47.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:47.306 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:01:47.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:47.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:47.306 | INFO | Uploaded 36GiB to GPU in 2.05543s (17.5145GiB/s) | 20260507 05:01:47.307 | INFO | Downloaded 34GiB to GPU in 1.95906s (17.3553GiB/s) | 20260507 05:01:52.446 | INFO | Analyzing chunk frames = [7293952,7294464) | 20260507 05:01:52.446 | INFO | MemInfo: { | 20260507 05:01:52.446 | INFO | "gpu" : " 17 GB", | 20260507 05:01:52.446 | INFO | "huge" : "", | 20260507 05:01:52.446 | INFO | "huge_shared" : " 27 GB", | 20260507 05:01:52.446 | INFO | "malloc" : " 14 GB", | 20260507 05:01:52.446 | INFO | "pinned" : " 29 GB" | 20260507 05:01:52.446 | INFO | } | 20260507 05:01:52.446 | INFO | Perf report for the last 5.13987 seconds: | 20260507 05:01:52.446 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:01:52.446 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:52.446 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:52.446 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:52.446 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:01:52.446 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:52.446 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:52.446 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 05:01:52.446 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:52.446 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:52.446 | INFO | Uploaded 36GiB to GPU in 1.92713s (18.6806GiB/s) | 20260507 05:01:52.446 | INFO | Downloaded 34GiB to GPU in 1.97057s (17.2539GiB/s) | 20260507 05:01:57.579 | INFO | Analyzing chunk frames = [7294464,7294976) | 20260507 05:01:57.579 | INFO | MemInfo: { | 20260507 05:01:57.579 | INFO | "gpu" : " 17 GB", | 20260507 05:01:57.579 | INFO | "huge" : "", | 20260507 05:01:57.579 | INFO | "huge_shared" : " 27 GB", | 20260507 05:01:57.579 | INFO | "malloc" : " 14 GB", | 20260507 05:01:57.579 | INFO | "pinned" : " 29 GB" | 20260507 05:01:57.579 | INFO | } | 20260507 05:01:57.579 | INFO | Perf report for the last 5.13313 seconds: | 20260507 05:01:57.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:01:57.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:57.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:57.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:57.580 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 05:01:57.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:57.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:57.580 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:01:57.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:57.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:01:57.580 | INFO | Uploaded 36GiB to GPU in 1.97029s (18.2714GiB/s) | 20260507 05:01:57.580 | INFO | Downloaded 34GiB to GPU in 1.96027s (17.3445GiB/s) | 20260507 05:01:57.677 | INFO | Starting a round of Baz IO | 20260507 05:02:00.784 | INFO | Populating RTMetrics report | 20260507 05:02:00.784 | INFO | RTMetrics report populated | 20260507 05:02:00.784 | INFO | Opening tmp RTMetrics json file | 20260507 05:02:00.789 | INFO | Serializing RTMetrics to json | 20260507 05:02:00.801 | INFO | Serialization done, swapping tmp file now... | 20260507 05:02:00.802 | INFO | RTMetrics done | 20260507 05:02:00.871 | INFO | Finished a round of Baz IO: Wrote 1.71 GiB in 3.193 seconds | 20260507 05:02:00.871 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.74% / 0.00% / 67.26% | 20260507 05:02:00.871 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.154475% / 3.97364e-06% / 6.35783e-05% | 20260507 05:02:00.871 | INFO | Baz IO overall: | 20260507 05:02:00.871 | INFO | BytesWritten : 3710601268134 | 20260507 05:02:00.871 | INFO | NumEvents : 1780475755275 | 20260507 05:02:02.725 | INFO | Analyzing chunk frames = [7294976,7295488) | 20260507 05:02:02.725 | INFO | MemInfo: { | 20260507 05:02:02.725 | INFO | "gpu" : " 17 GB", | 20260507 05:02:02.725 | INFO | "huge" : "", | 20260507 05:02:02.725 | INFO | "huge_shared" : " 27 GB", | 20260507 05:02:02.725 | INFO | "malloc" : " 14 GB", | 20260507 05:02:02.725 | INFO | "pinned" : " 29 GB" | 20260507 05:02:02.725 | INFO | } | 20260507 05:02:02.725 | INFO | Perf report for the last 5.1455 seconds: | 20260507 05:02:02.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:02:02.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:02.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:02.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:02.725 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260507 05:02:02.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:02.725 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 05:02:02.725 | INFO | PRE_HQ : 59.0%, 1.8, 55.1ms, 0 | 20260507 05:02:02.725 | INFO | BAZWRITER : 61.7%, 3.2, 100.9ms, 0 | 20260507 05:02:02.725 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:02:02.725 | INFO | Uploaded 36GiB to GPU in 2.04032s (17.6443GiB/s) | 20260507 05:02:02.725 | INFO | Downloaded 38GiB to GPU in 2.17702s (17.4551GiB/s) | 20260507 05:02:07.867 | INFO | Analyzing chunk frames = [7295488,7296000) | 20260507 05:02:07.867 | INFO | MemInfo: { | 20260507 05:02:07.867 | INFO | "gpu" : " 17 GB", | 20260507 05:02:07.867 | INFO | "huge" : "", | 20260507 05:02:07.867 | INFO | "huge_shared" : " 27 GB", | 20260507 05:02:07.867 | INFO | "malloc" : " 14 GB", | 20260507 05:02:07.867 | INFO | "pinned" : " 29 GB" | 20260507 05:02:07.867 | INFO | } | 20260507 05:02:07.867 | INFO | Perf report for the last 5.14256 seconds: | 20260507 05:02:07.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:02:07.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:07.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:07.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:07.867 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:02:07.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:07.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:07.867 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:02:07.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:07.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:07.867 | INFO | Uploaded 36GiB to GPU in 1.9911s (18.0804GiB/s) | 20260507 05:02:07.868 | INFO | Downloaded 34GiB to GPU in 1.92851s (17.6302GiB/s) | 20260507 05:02:13.002 | INFO | Analyzing chunk frames = [7296000,7296512) | 20260507 05:02:13.002 | INFO | MemInfo: { | 20260507 05:02:13.002 | INFO | "gpu" : " 17 GB", | 20260507 05:02:13.002 | INFO | "huge" : "", | 20260507 05:02:13.002 | INFO | "huge_shared" : " 27 GB", | 20260507 05:02:13.002 | INFO | "malloc" : " 14 GB", | 20260507 05:02:13.002 | INFO | "pinned" : " 29 GB" | 20260507 05:02:13.002 | INFO | } | 20260507 05:02:13.002 | INFO | Perf report for the last 5.13484 seconds: | 20260507 05:02:13.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:02:13.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:13.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:13.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:13.002 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 05:02:13.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:13.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:13.002 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 05:02:13.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:13.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:13.003 | INFO | Uploaded 36GiB to GPU in 2.05268s (17.5381GiB/s) | 20260507 05:02:13.003 | INFO | Downloaded 34GiB to GPU in 1.95004s (17.4355GiB/s) | 20260507 05:02:18.139 | INFO | Analyzing chunk frames = [7296512,7297024) | 20260507 05:02:18.139 | INFO | MemInfo: { | 20260507 05:02:18.139 | INFO | "gpu" : " 17 GB", | 20260507 05:02:18.139 | INFO | "huge" : "", | 20260507 05:02:18.139 | INFO | "huge_shared" : " 27 GB", | 20260507 05:02:18.139 | INFO | "malloc" : " 14 GB", | 20260507 05:02:18.139 | INFO | "pinned" : " 29 GB" | 20260507 05:02:18.139 | INFO | } | 20260507 05:02:18.140 | INFO | Perf report for the last 5.13739 seconds: | 20260507 05:02:18.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:02:18.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:18.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:18.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:18.140 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 05:02:18.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:18.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:18.140 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:02:18.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:18.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:18.140 | INFO | Uploaded 36GiB to GPU in 1.9941s (18.0532GiB/s) | 20260507 05:02:18.140 | INFO | Downloaded 34GiB to GPU in 1.92155s (17.694GiB/s) | 20260507 05:02:23.284 | INFO | Analyzing chunk frames = [7297024,7297536) | 20260507 05:02:23.284 | INFO | MemInfo: { | 20260507 05:02:23.284 | INFO | "gpu" : " 17 GB", | 20260507 05:02:23.284 | INFO | "huge" : "", | 20260507 05:02:23.284 | INFO | "huge_shared" : " 27 GB", | 20260507 05:02:23.284 | INFO | "malloc" : " 14 GB", | 20260507 05:02:23.284 | INFO | "pinned" : " 29 GB" | 20260507 05:02:23.284 | INFO | } | 20260507 05:02:23.284 | INFO | Perf report for the last 5.14474 seconds: | 20260507 05:02:23.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:02:23.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:23.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:23.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:23.285 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 05:02:23.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:23.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:23.285 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 05:02:23.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:23.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:23.285 | INFO | Uploaded 36GiB to GPU in 2.03921s (17.6539GiB/s) | 20260507 05:02:23.285 | INFO | Downloaded 34GiB to GPU in 1.93977s (17.5278GiB/s) | 20260507 05:02:28.419 | INFO | Analyzing chunk frames = [7297536,7298048) | 20260507 05:02:28.419 | INFO | MemInfo: { | 20260507 05:02:28.420 | INFO | "gpu" : " 17 GB", | 20260507 05:02:28.420 | INFO | "huge" : "", | 20260507 05:02:28.420 | INFO | "huge_shared" : " 27 GB", | 20260507 05:02:28.420 | INFO | "malloc" : " 14 GB", | 20260507 05:02:28.420 | INFO | "pinned" : " 29 GB" | 20260507 05:02:28.420 | INFO | } | 20260507 05:02:28.420 | INFO | Perf report for the last 5.13542 seconds: | 20260507 05:02:28.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:02:28.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:28.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:28.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:28.420 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 05:02:28.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:28.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:28.420 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 05:02:28.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:28.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:28.420 | INFO | Uploaded 36GiB to GPU in 2.04343s (17.6174GiB/s) | 20260507 05:02:28.420 | INFO | Downloaded 34GiB to GPU in 1.91387s (17.7651GiB/s) | 20260507 05:02:33.561 | INFO | Analyzing chunk frames = [7298048,7298560) | 20260507 05:02:33.561 | INFO | MemInfo: { | 20260507 05:02:33.561 | INFO | "gpu" : " 17 GB", | 20260507 05:02:33.561 | INFO | "huge" : "", | 20260507 05:02:33.561 | INFO | "huge_shared" : " 27 GB", | 20260507 05:02:33.561 | INFO | "malloc" : " 14 GB", | 20260507 05:02:33.561 | INFO | "pinned" : " 29 GB" | 20260507 05:02:33.561 | INFO | } | 20260507 05:02:33.561 | INFO | Perf report for the last 5.14145 seconds: | 20260507 05:02:33.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:02:33.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:33.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:33.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:33.561 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:02:33.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:33.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:33.561 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 05:02:33.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:33.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:33.561 | INFO | Uploaded 36GiB to GPU in 1.97304s (18.246GiB/s) | 20260507 05:02:33.562 | INFO | Downloaded 34GiB to GPU in 1.98442s (17.1335GiB/s) | 20260507 05:02:38.696 | INFO | Analyzing chunk frames = [7298560,7299072) | 20260507 05:02:38.696 | INFO | MemInfo: { | 20260507 05:02:38.696 | INFO | "gpu" : " 17 GB", | 20260507 05:02:38.696 | INFO | "huge" : "", | 20260507 05:02:38.696 | INFO | "huge_shared" : " 27 GB", | 20260507 05:02:38.696 | INFO | "malloc" : " 14 GB", | 20260507 05:02:38.696 | INFO | "pinned" : " 29 GB" | 20260507 05:02:38.697 | INFO | } | 20260507 05:02:38.697 | INFO | Perf report for the last 5.13559 seconds: | 20260507 05:02:38.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:02:38.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:38.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:38.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:38.697 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:02:38.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:38.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:38.697 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:02:38.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:38.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:38.697 | INFO | Uploaded 36GiB to GPU in 1.99414s (18.0529GiB/s) | 20260507 05:02:38.697 | INFO | Downloaded 34GiB to GPU in 1.9585s (17.3602GiB/s) | 20260507 05:02:38.697 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:02:38.697 | INFO | Total mem usage: 18 GiB | 20260507 05:02:38.697 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:02:38.697 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:02:38.697 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:02:38.697 | INFO | High Water Breakdown: | 20260507 05:02:38.697 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:02:38.697 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:02:38.697 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:02:38.697 | INFO | ------------------------------------------ | 20260507 05:02:38.697 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:02:38.697 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:02:38.697 | INFO | Total mem usage: 26.5 GiB | 20260507 05:02:38.697 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:02:38.697 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:02:38.697 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:02:38.697 | INFO | High Water Breakdown: | 20260507 05:02:38.697 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:02:38.697 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:02:38.697 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:02:38.697 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:02:38.697 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:02:38.697 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:02:38.697 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:02:38.697 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:02:38.697 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:02:38.697 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:02:38.697 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:02:38.697 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:02:38.697 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:02:38.697 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:02:38.697 | INFO | ------------------------------------------ | 20260507 05:02:38.697 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:02:38.697 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:02:38.697 | INFO | Total mem usage: 25.6 GiB | 20260507 05:02:38.697 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:02:38.697 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:02:38.697 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:02:38.697 | INFO | High Water Breakdown: | 20260507 05:02:38.697 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:02:38.697 | INFO | ------------------------------------------ | 20260507 05:02:38.698 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:02:38.698 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:02:38.698 | INFO | Total mem usage: 15.7 GiB | 20260507 05:02:38.698 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:02:38.698 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:02:38.698 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:02:38.698 | INFO | High Water Breakdown: | 20260507 05:02:38.698 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:02:38.698 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:02:38.698 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:02:38.698 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:02:38.698 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:02:38.698 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:02:38.698 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:02:38.698 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:02:38.698 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:02:38.698 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:02:38.698 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:02:38.698 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:02:38.698 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:02:38.698 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:02:38.698 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:02:38.698 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:02:38.698 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:02:38.698 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:02:38.698 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:02:38.698 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:02:38.698 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:02:38.698 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:02:38.698 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:02:38.698 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:02:38.698 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:02:38.698 | INFO | ------------------------------------------ | 20260507 05:02:38.698 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:02:38.698 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:02:38.698 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:02:38.698 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:02:38.698 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.33e+01 / 1.29e+00 | 20260507 05:02:38.698 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 9.94e-02, 9.75e-03 / 5.70e-03 | 20260507 05:02:38.698 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.40e+00, 5.15e-01 / 2.83e-01 | 20260507 05:02:38.698 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.45e+00, 1.66e-01 / 1.22e-01 | 20260507 05:02:38.698 | INFO | PerfReport: 16.24%, DME -- 3.99e+01 / 4.49e+01, 4.11e+00 / 1.49e+00 | 20260507 05:02:38.698 | INFO | PerfReport: 57.58%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.23e+00 / 1.12e+00 | 20260507 05:02:38.698 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.69e+00, 3.85e-01 / 2.17e-01 | 20260507 05:02:38.698 | INFO | PerfReport: 3.00%, Metrics -- 9.22e-01 / 1.24e+00, 2.43e-01 / 1.07e-01 | 20260507 05:02:38.698 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.11e-01 / 3.65e-01, 4.95e-02 / 2.13e-02 | 20260507 05:02:38.698 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:02:38.698 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:02:38.698 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:02:38.698 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 05:02:38.698 | INFO | PerfReport: 21.81%, Upload -- 2.10e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260507 05:02:38.698 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 05:02:38.698 | INFO | PerfReport: 31.95%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 05:02:38.698 | INFO | PerfReport: 1.54%, LockUpload -- 1.49e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 05:02:38.698 | INFO | PerfReport: 1.23%, LockDownload -- 1.18e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260507 05:02:38.698 | INFO | PerfReport: 22.08%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 05:02:38.698 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.51e-02 / 0.00e+00 | 20260507 05:02:38.793 | INFO | Starting a round of Baz IO | 20260507 05:02:41.892 | INFO | Populating RTMetrics report | 20260507 05:02:41.892 | INFO | RTMetrics report populated | 20260507 05:02:41.892 | INFO | Opening tmp RTMetrics json file | 20260507 05:02:41.893 | INFO | Serializing RTMetrics to json | 20260507 05:02:41.898 | INFO | Serialization done, swapping tmp file now... | 20260507 05:02:41.908 | INFO | RTMetrics done | 20260507 05:02:41.978 | INFO | Finished a round of Baz IO: Wrote 1.709 GiB in 3.184 seconds | 20260507 05:02:41.978 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.74% / 0.00% / 67.26% | 20260507 05:02:41.978 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.154785% / 0% / 7.15256e-05% | 20260507 05:02:41.978 | INFO | Baz IO overall: | 20260507 05:02:41.978 | INFO | BytesWritten : 3712435969148 | 20260507 05:02:41.979 | INFO | NumEvents : 1781338493442 | 20260507 05:02:43.834 | INFO | Analyzing chunk frames = [7299072,7299584) | 20260507 05:02:43.834 | INFO | MemInfo: { | 20260507 05:02:43.834 | INFO | "gpu" : " 17 GB", | 20260507 05:02:43.834 | INFO | "huge" : "", | 20260507 05:02:43.834 | INFO | "huge_shared" : " 27 GB", | 20260507 05:02:43.834 | INFO | "malloc" : " 14 GB", | 20260507 05:02:43.834 | INFO | "pinned" : " 29 GB" | 20260507 05:02:43.834 | INFO | } | 20260507 05:02:43.834 | INFO | Perf report for the last 5.13699 seconds: | 20260507 05:02:43.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:02:43.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:43.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:43.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:43.834 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 05:02:43.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:43.834 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 05:02:43.834 | INFO | PRE_HQ : 59.0%, 1.8, 54.9ms, 0 | 20260507 05:02:43.834 | INFO | BAZWRITER : 61.6%, 3.2, 101.7ms, 0 | 20260507 05:02:43.834 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:02:43.834 | INFO | Uploaded 36GiB to GPU in 2.04129s (17.6359GiB/s) | 20260507 05:02:43.834 | INFO | Downloaded 38GiB to GPU in 2.17824s (17.4453GiB/s) | 20260507 05:02:48.975 | INFO | Analyzing chunk frames = [7299584,7300096) | 20260507 05:02:48.975 | INFO | MemInfo: { | 20260507 05:02:48.975 | INFO | "gpu" : " 17 GB", | 20260507 05:02:48.975 | INFO | "huge" : "", | 20260507 05:02:48.975 | INFO | "huge_shared" : " 27 GB", | 20260507 05:02:48.975 | INFO | "malloc" : " 14 GB", | 20260507 05:02:48.975 | INFO | "pinned" : " 29 GB" | 20260507 05:02:48.975 | INFO | } | 20260507 05:02:48.975 | INFO | Perf report for the last 5.14093 seconds: | 20260507 05:02:48.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:02:48.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:48.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:48.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:48.975 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 05:02:48.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:48.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:48.975 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:02:48.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:48.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:48.975 | INFO | Uploaded 36GiB to GPU in 2.04968s (17.5637GiB/s) | 20260507 05:02:48.975 | INFO | Downloaded 34GiB to GPU in 1.94102s (17.5165GiB/s) | 20260507 05:02:54.123 | INFO | Analyzing chunk frames = [7300096,7300608) | 20260507 05:02:54.123 | INFO | MemInfo: { | 20260507 05:02:54.123 | INFO | "gpu" : " 17 GB", | 20260507 05:02:54.123 | INFO | "huge" : "", | 20260507 05:02:54.123 | INFO | "huge_shared" : " 27 GB", | 20260507 05:02:54.123 | INFO | "malloc" : " 14 GB", | 20260507 05:02:54.123 | INFO | "pinned" : " 29 GB" | 20260507 05:02:54.123 | INFO | } | 20260507 05:02:54.123 | INFO | Perf report for the last 5.14857 seconds: | 20260507 05:02:54.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:02:54.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:54.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:54.123 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:54.124 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 05:02:54.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:54.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:54.124 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260507 05:02:54.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:54.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:54.124 | INFO | Uploaded 36GiB to GPU in 2.02013s (17.8206GiB/s) | 20260507 05:02:54.124 | INFO | Downloaded 34GiB to GPU in 1.93804s (17.5435GiB/s) | 20260507 05:02:59.255 | INFO | Analyzing chunk frames = [7300608,7301120) | 20260507 05:02:59.255 | INFO | MemInfo: { | 20260507 05:02:59.255 | INFO | "gpu" : " 17 GB", | 20260507 05:02:59.255 | INFO | "huge" : "", | 20260507 05:02:59.255 | INFO | "huge_shared" : " 27 GB", | 20260507 05:02:59.255 | INFO | "malloc" : " 14 GB", | 20260507 05:02:59.255 | INFO | "pinned" : " 29 GB" | 20260507 05:02:59.255 | INFO | } | 20260507 05:02:59.255 | INFO | Perf report for the last 5.13184 seconds: | 20260507 05:02:59.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:02:59.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:59.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:59.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:59.255 | INFO | ANALYSIS : 60.6%, 2.9, 93.5ms, 0 | 20260507 05:02:59.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:59.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:59.255 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:02:59.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:59.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:02:59.255 | INFO | Uploaded 36GiB to GPU in 2.05632s (17.507GiB/s) | 20260507 05:02:59.255 | INFO | Downloaded 34GiB to GPU in 1.9493s (17.4422GiB/s) | 20260507 05:03:04.389 | INFO | Analyzing chunk frames = [7301120,7301632) | 20260507 05:03:04.389 | INFO | MemInfo: { | 20260507 05:03:04.389 | INFO | "gpu" : " 17 GB", | 20260507 05:03:04.389 | INFO | "huge" : "", | 20260507 05:03:04.389 | INFO | "huge_shared" : " 27 GB", | 20260507 05:03:04.389 | INFO | "malloc" : " 14 GB", | 20260507 05:03:04.389 | INFO | "pinned" : " 29 GB" | 20260507 05:03:04.389 | INFO | } | 20260507 05:03:04.389 | INFO | Perf report for the last 5.13406 seconds: | 20260507 05:03:04.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:03:04.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:04.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:04.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:04.389 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 05:03:04.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:04.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:04.389 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:03:04.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:04.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:04.390 | INFO | Uploaded 36GiB to GPU in 2.03753s (17.6685GiB/s) | 20260507 05:03:04.390 | INFO | Downloaded 34GiB to GPU in 1.91085s (17.7931GiB/s) | 20260507 05:03:09.536 | INFO | Analyzing chunk frames = [7301632,7302144) | 20260507 05:03:09.536 | INFO | MemInfo: { | 20260507 05:03:09.536 | INFO | "gpu" : " 17 GB", | 20260507 05:03:09.536 | INFO | "huge" : "", | 20260507 05:03:09.536 | INFO | "huge_shared" : " 27 GB", | 20260507 05:03:09.536 | INFO | "malloc" : " 14 GB", | 20260507 05:03:09.536 | INFO | "pinned" : " 29 GB" | 20260507 05:03:09.536 | INFO | } | 20260507 05:03:09.536 | INFO | Perf report for the last 5.14681 seconds: | 20260507 05:03:09.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:03:09.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:09.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:09.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:09.536 | INFO | ANALYSIS : 60.5%, 3.0, 94.6ms, 0 | 20260507 05:03:09.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:09.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:09.536 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 05:03:09.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:09.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:09.536 | INFO | Uploaded 36GiB to GPU in 2.04034s (17.6441GiB/s) | 20260507 05:03:09.536 | INFO | Downloaded 34GiB to GPU in 1.93294s (17.5897GiB/s) | 20260507 05:03:14.677 | INFO | Analyzing chunk frames = [7302144,7302656) | 20260507 05:03:14.677 | INFO | MemInfo: { | 20260507 05:03:14.677 | INFO | "gpu" : " 17 GB", | 20260507 05:03:14.677 | INFO | "huge" : "", | 20260507 05:03:14.677 | INFO | "huge_shared" : " 27 GB", | 20260507 05:03:14.677 | INFO | "malloc" : " 14 GB", | 20260507 05:03:14.677 | INFO | "pinned" : " 29 GB" | 20260507 05:03:14.677 | INFO | } | 20260507 05:03:14.677 | INFO | Perf report for the last 5.14088 seconds: | 20260507 05:03:14.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:03:14.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:14.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:14.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:14.677 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:03:14.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:14.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:14.677 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:03:14.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:14.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:14.677 | INFO | Uploaded 36GiB to GPU in 1.97982s (18.1835GiB/s) | 20260507 05:03:14.677 | INFO | Downloaded 34GiB to GPU in 1.96389s (17.3126GiB/s) | 20260507 05:03:19.820 | INFO | Analyzing chunk frames = [7302656,7303168) | 20260507 05:03:19.820 | INFO | MemInfo: { | 20260507 05:03:19.820 | INFO | "gpu" : " 17 GB", | 20260507 05:03:19.820 | INFO | "huge" : "", | 20260507 05:03:19.820 | INFO | "huge_shared" : " 27 GB", | 20260507 05:03:19.820 | INFO | "malloc" : " 14 GB", | 20260507 05:03:19.820 | INFO | "pinned" : " 29 GB" | 20260507 05:03:19.820 | INFO | } | 20260507 05:03:19.820 | INFO | Perf report for the last 5.14295 seconds: | 20260507 05:03:19.820 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:03:19.820 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:19.820 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:19.820 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:19.820 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:03:19.820 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:19.820 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:19.820 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 05:03:19.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:19.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:19.820 | INFO | Uploaded 36GiB to GPU in 1.98948s (18.0952GiB/s) | 20260507 05:03:19.820 | INFO | Downloaded 34GiB to GPU in 1.96591s (17.2948GiB/s) | 20260507 05:03:19.917 | INFO | Starting a round of Baz IO | 20260507 05:03:23.015 | INFO | Populating RTMetrics report | 20260507 05:03:23.015 | INFO | RTMetrics report populated | 20260507 05:03:23.015 | INFO | Opening tmp RTMetrics json file | 20260507 05:03:23.021 | INFO | Serializing RTMetrics to json | 20260507 05:03:23.033 | INFO | Serialization done, swapping tmp file now... | 20260507 05:03:23.034 | INFO | RTMetrics done | 20260507 05:03:23.104 | INFO | Finished a round of Baz IO: Wrote 1.708 GiB in 3.187 seconds | 20260507 05:03:23.104 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.74% / 0.00% / 67.26% | 20260507 05:03:23.104 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.154209% / 3.97364e-06% / 6.35783e-05% | 20260507 05:03:23.104 | INFO | Baz IO overall: | 20260507 05:03:23.104 | INFO | BytesWritten : 3714269539666 | 20260507 05:03:23.104 | INFO | NumEvents : 1782200076691 | 20260507 05:03:24.950 | INFO | Analyzing chunk frames = [7303168,7303680) | 20260507 05:03:24.950 | INFO | MemInfo: { | 20260507 05:03:24.950 | INFO | "gpu" : " 17 GB", | 20260507 05:03:24.950 | INFO | "huge" : "", | 20260507 05:03:24.950 | INFO | "huge_shared" : " 27 GB", | 20260507 05:03:24.950 | INFO | "malloc" : " 14 GB", | 20260507 05:03:24.950 | INFO | "pinned" : " 29 GB" | 20260507 05:03:24.950 | INFO | } | 20260507 05:03:24.950 | INFO | Perf report for the last 5.13052 seconds: | 20260507 05:03:24.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:03:24.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:24.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:24.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:24.950 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 05:03:24.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:24.950 | INFO | RT_METRICS : 41.5%, 1.0, 21.7ms, 0 | 20260507 05:03:24.950 | INFO | PRE_HQ : 59.2%, 1.8, 54.9ms, 0 | 20260507 05:03:24.950 | INFO | BAZWRITER : 61.7%, 3.2, 101.0ms, 0 | 20260507 05:03:24.951 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:03:24.951 | INFO | Uploaded 36GiB to GPU in 2.07684s (17.334GiB/s) | 20260507 05:03:24.951 | INFO | Downloaded 38GiB to GPU in 2.18019s (17.4297GiB/s) | 20260507 05:03:30.091 | INFO | Analyzing chunk frames = [7303680,7304192) | 20260507 05:03:30.091 | INFO | MemInfo: { | 20260507 05:03:30.091 | INFO | "gpu" : " 17 GB", | 20260507 05:03:30.091 | INFO | "huge" : "", | 20260507 05:03:30.091 | INFO | "huge_shared" : " 27 GB", | 20260507 05:03:30.091 | INFO | "malloc" : " 14 GB", | 20260507 05:03:30.091 | INFO | "pinned" : " 29 GB" | 20260507 05:03:30.091 | INFO | } | 20260507 05:03:30.091 | INFO | Perf report for the last 5.141 seconds: | 20260507 05:03:30.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:03:30.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:30.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:30.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:30.091 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 05:03:30.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:30.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:30.092 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:03:30.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:30.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:30.092 | INFO | Uploaded 36GiB to GPU in 2.0048s (17.9569GiB/s) | 20260507 05:03:30.092 | INFO | Downloaded 34GiB to GPU in 1.93038s (17.6131GiB/s) | 20260507 05:03:35.233 | INFO | Analyzing chunk frames = [7304192,7304704) | 20260507 05:03:35.233 | INFO | MemInfo: { | 20260507 05:03:35.233 | INFO | "gpu" : " 17 GB", | 20260507 05:03:35.233 | INFO | "huge" : "", | 20260507 05:03:35.233 | INFO | "huge_shared" : " 27 GB", | 20260507 05:03:35.233 | INFO | "malloc" : " 14 GB", | 20260507 05:03:35.233 | INFO | "pinned" : " 29 GB" | 20260507 05:03:35.233 | INFO | } | 20260507 05:03:35.233 | INFO | Perf report for the last 5.14154 seconds: | 20260507 05:03:35.233 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:03:35.233 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:35.233 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:35.233 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:35.233 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 05:03:35.233 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:35.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:35.233 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:03:35.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:35.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:35.233 | INFO | Uploaded 36GiB to GPU in 2.01745s (17.8443GiB/s) | 20260507 05:03:35.233 | INFO | Downloaded 34GiB to GPU in 1.93882s (17.5365GiB/s) | 20260507 05:03:40.371 | INFO | Analyzing chunk frames = [7304704,7305216) | 20260507 05:03:40.371 | INFO | MemInfo: { | 20260507 05:03:40.371 | INFO | "gpu" : " 17 GB", | 20260507 05:03:40.371 | INFO | "huge" : "", | 20260507 05:03:40.371 | INFO | "huge_shared" : " 27 GB", | 20260507 05:03:40.371 | INFO | "malloc" : " 14 GB", | 20260507 05:03:40.371 | INFO | "pinned" : " 29 GB" | 20260507 05:03:40.371 | INFO | } | 20260507 05:03:40.371 | INFO | Perf report for the last 5.13848 seconds: | 20260507 05:03:40.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:03:40.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:40.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:40.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:40.372 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:03:40.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:40.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:40.372 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:03:40.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:40.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:40.372 | INFO | Uploaded 36GiB to GPU in 2.00902s (17.9192GiB/s) | 20260507 05:03:40.372 | INFO | Downloaded 34GiB to GPU in 1.93659s (17.5566GiB/s) | 20260507 05:03:45.506 | INFO | Analyzing chunk frames = [7305216,7305728) | 20260507 05:03:45.506 | INFO | MemInfo: { | 20260507 05:03:45.506 | INFO | "gpu" : " 17 GB", | 20260507 05:03:45.506 | INFO | "huge" : "", | 20260507 05:03:45.506 | INFO | "huge_shared" : " 27 GB", | 20260507 05:03:45.506 | INFO | "malloc" : " 14 GB", | 20260507 05:03:45.506 | INFO | "pinned" : " 29 GB" | 20260507 05:03:45.506 | INFO | } | 20260507 05:03:45.506 | INFO | Perf report for the last 5.13475 seconds: | 20260507 05:03:45.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:03:45.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:45.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:45.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:45.506 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 05:03:45.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:45.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:45.506 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 05:03:45.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:45.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:45.506 | INFO | Uploaded 36GiB to GPU in 2.05801s (17.4926GiB/s) | 20260507 05:03:45.507 | INFO | Downloaded 34GiB to GPU in 1.94988s (17.437GiB/s) | 20260507 05:03:50.650 | INFO | Analyzing chunk frames = [7305728,7306240) | 20260507 05:03:50.650 | INFO | MemInfo: { | 20260507 05:03:50.650 | INFO | "gpu" : " 17 GB", | 20260507 05:03:50.650 | INFO | "huge" : "", | 20260507 05:03:50.650 | INFO | "huge_shared" : " 27 GB", | 20260507 05:03:50.650 | INFO | "malloc" : " 14 GB", | 20260507 05:03:50.650 | INFO | "pinned" : " 29 GB" | 20260507 05:03:50.650 | INFO | } | 20260507 05:03:50.650 | INFO | Perf report for the last 5.14398 seconds: | 20260507 05:03:50.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:03:50.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:50.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:50.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:50.650 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 05:03:50.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:50.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:50.650 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:03:50.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:50.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:50.650 | INFO | Uploaded 36GiB to GPU in 2.05611s (17.5088GiB/s) | 20260507 05:03:50.651 | INFO | Downloaded 34GiB to GPU in 1.96591s (17.2948GiB/s) | 20260507 05:03:55.790 | INFO | Analyzing chunk frames = [7306240,7306752) | 20260507 05:03:55.790 | INFO | MemInfo: { | 20260507 05:03:55.790 | INFO | "gpu" : " 17 GB", | 20260507 05:03:55.790 | INFO | "huge" : "", | 20260507 05:03:55.790 | INFO | "huge_shared" : " 27 GB", | 20260507 05:03:55.790 | INFO | "malloc" : " 14 GB", | 20260507 05:03:55.790 | INFO | "pinned" : " 29 GB" | 20260507 05:03:55.790 | INFO | } | 20260507 05:03:55.790 | INFO | Perf report for the last 5.13994 seconds: | 20260507 05:03:55.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:03:55.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:55.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:55.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:55.790 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:03:55.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:55.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:55.790 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 05:03:55.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:55.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:03:55.790 | INFO | Uploaded 36GiB to GPU in 1.96806s (18.2921GiB/s) | 20260507 05:03:55.791 | INFO | Downloaded 34GiB to GPU in 1.9838s (17.1389GiB/s) | 20260507 05:04:00.928 | INFO | Analyzing chunk frames = [7306752,7307264) | 20260507 05:04:00.928 | INFO | MemInfo: { | 20260507 05:04:00.928 | INFO | "gpu" : " 17 GB", | 20260507 05:04:00.928 | INFO | "huge" : "", | 20260507 05:04:00.928 | INFO | "huge_shared" : " 27 GB", | 20260507 05:04:00.928 | INFO | "malloc" : " 14 GB", | 20260507 05:04:00.928 | INFO | "pinned" : " 29 GB" | 20260507 05:04:00.928 | INFO | } | 20260507 05:04:00.928 | INFO | Perf report for the last 5.13827 seconds: | 20260507 05:04:00.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:04:00.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:00.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:00.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:00.929 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:04:00.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:00.929 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:00.929 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:04:00.929 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:00.929 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:00.929 | INFO | Uploaded 36GiB to GPU in 1.97084s (18.2664GiB/s) | 20260507 05:04:00.929 | INFO | Downloaded 34GiB to GPU in 1.97404s (17.2236GiB/s) | 20260507 05:04:00.929 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:04:00.929 | INFO | Total mem usage: 18 GiB | 20260507 05:04:00.929 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:04:00.929 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:04:00.929 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:04:00.929 | INFO | High Water Breakdown: | 20260507 05:04:00.929 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:04:00.929 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:04:00.929 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:04:00.929 | INFO | ------------------------------------------ | 20260507 05:04:00.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:04:00.929 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:04:00.929 | INFO | Total mem usage: 26.5 GiB | 20260507 05:04:00.929 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:04:00.929 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:04:00.929 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 05:04:00.929 | INFO | High Water Breakdown: | 20260507 05:04:00.929 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:04:00.929 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:04:00.929 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:04:00.929 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:04:00.929 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:04:00.929 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:04:00.929 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:04:00.929 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:04:00.929 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:04:00.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:04:00.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:04:00.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:04:00.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:04:00.929 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:04:00.929 | INFO | ------------------------------------------ | 20260507 05:04:00.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:04:00.929 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:04:00.929 | INFO | Total mem usage: 25.6 GiB | 20260507 05:04:00.929 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:04:00.929 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:04:00.929 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:04:00.929 | INFO | High Water Breakdown: | 20260507 05:04:00.929 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:04:00.929 | INFO | ------------------------------------------ | 20260507 05:04:00.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:04:00.929 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:04:00.929 | INFO | Total mem usage: 15.7 GiB | 20260507 05:04:00.929 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:04:00.929 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:04:00.929 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:04:00.929 | INFO | High Water Breakdown: | 20260507 05:04:00.929 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:04:00.929 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:04:00.929 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:04:00.929 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:04:00.929 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:04:00.929 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:04:00.929 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:04:00.929 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:04:00.929 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:04:00.929 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:04:00.929 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:04:00.929 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:04:00.929 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:04:00.929 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:04:00.929 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:04:00.929 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:04:00.929 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:04:00.929 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:04:00.930 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:04:00.930 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:04:00.930 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:04:00.930 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:04:00.930 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:04:00.930 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:04:00.930 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:04:00.930 | INFO | ------------------------------------------ | 20260507 05:04:00.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:04:00.930 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:04:00.930 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:04:00.930 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:04:00.930 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.32e+01 / 1.09e+00 | 20260507 05:04:00.930 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 9.99e-02, 9.67e-03 / 5.60e-03 | 20260507 05:04:00.930 | INFO | PerfReport: 11.63%, Baseline -- 3.57e+00 / 4.43e+00, 4.93e-01 / 2.93e-01 | 20260507 05:04:00.930 | INFO | PerfReport: 3.21%, Histogram -- 9.85e-01 / 1.45e+00, 1.60e-01 / 1.16e-01 | 20260507 05:04:00.930 | INFO | PerfReport: 16.16%, DME -- 3.97e+01 / 4.42e+01, 3.62e+00 / 1.33e+00 | 20260507 05:04:00.930 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.27e+00 / 1.10e+00 | 20260507 05:04:00.930 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.14e+00 / 2.69e+00, 3.80e-01 / 2.12e-01 | 20260507 05:04:00.930 | INFO | PerfReport: 3.01%, Metrics -- 9.23e-01 / 1.24e+00, 2.43e-01 / 1.32e-01 | 20260507 05:04:00.930 | INFO | PerfReport: 1.02%, PulseSerializing -- 3.12e-01 / 3.70e-01, 4.96e-02 / 2.39e-02 | 20260507 05:04:00.930 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:04:00.930 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:04:00.930 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:04:00.930 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 05:04:00.930 | INFO | PerfReport: 21.90%, Upload -- 2.11e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260507 05:04:00.930 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 05:04:00.930 | INFO | PerfReport: 31.89%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 05:04:00.930 | INFO | PerfReport: 1.61%, LockUpload -- 1.55e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 05:04:00.930 | INFO | PerfReport: 1.20%, LockDownload -- 1.15e+00 / 0.00e+00, 2.84e+00 / 0.00e+00 | 20260507 05:04:00.930 | INFO | PerfReport: 21.98%, LockCompute -- 2.12e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 05:04:00.930 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260507 05:04:01.027 | INFO | Starting a round of Baz IO | 20260507 05:04:04.124 | INFO | Populating RTMetrics report | 20260507 05:04:04.124 | INFO | RTMetrics report populated | 20260507 05:04:04.124 | INFO | Opening tmp RTMetrics json file | 20260507 05:04:04.143 | INFO | Serializing RTMetrics to json | 20260507 05:04:04.144 | INFO | Serialization done, swapping tmp file now... | 20260507 05:04:04.144 | INFO | RTMetrics done | 20260507 05:04:04.207 | INFO | Finished a round of Baz IO: Wrote 1.709 GiB in 3.18 seconds | 20260507 05:04:04.207 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.74% / 0.00% / 67.26% | 20260507 05:04:04.207 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.15345% / 1.19209e-05% / 5.16574e-05% | 20260507 05:04:04.207 | INFO | Baz IO overall: | 20260507 05:04:04.207 | INFO | BytesWritten : 3716104423955 | 20260507 05:04:04.207 | INFO | NumEvents : 1783062608257 | 20260507 05:04:06.068 | INFO | Analyzing chunk frames = [7307264,7307776) | 20260507 05:04:06.068 | INFO | MemInfo: { | 20260507 05:04:06.068 | INFO | "gpu" : " 17 GB", | 20260507 05:04:06.068 | INFO | "huge" : "", | 20260507 05:04:06.068 | INFO | "huge_shared" : " 27 GB", | 20260507 05:04:06.068 | INFO | "malloc" : " 14 GB", | 20260507 05:04:06.068 | INFO | "pinned" : " 29 GB" | 20260507 05:04:06.068 | INFO | } | 20260507 05:04:06.068 | INFO | Perf report for the last 5.13964 seconds: | 20260507 05:04:06.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:04:06.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:06.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:06.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:06.068 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:04:06.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:06.068 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 05:04:06.068 | INFO | PRE_HQ : 59.1%, 1.8, 55.3ms, 0 | 20260507 05:04:06.068 | INFO | BAZWRITER : 61.5%, 3.2, 99.7ms, 0 | 20260507 05:04:06.068 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 05:04:06.068 | INFO | Uploaded 36GiB to GPU in 2.03944s (17.6519GiB/s) | 20260507 05:04:06.069 | INFO | Downloaded 38GiB to GPU in 2.18059s (17.4265GiB/s) | 20260507 05:04:11.206 | INFO | Analyzing chunk frames = [7307776,7308288) | 20260507 05:04:11.206 | INFO | MemInfo: { | 20260507 05:04:11.206 | INFO | "gpu" : " 17 GB", | 20260507 05:04:11.206 | INFO | "huge" : "", | 20260507 05:04:11.206 | INFO | "huge_shared" : " 27 GB", | 20260507 05:04:11.206 | INFO | "malloc" : " 14 GB", | 20260507 05:04:11.206 | INFO | "pinned" : " 29 GB" | 20260507 05:04:11.206 | INFO | } | 20260507 05:04:11.206 | INFO | Perf report for the last 5.13798 seconds: | 20260507 05:04:11.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:04:11.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:11.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:11.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:11.206 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 05:04:11.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:11.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:11.206 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:04:11.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:11.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:11.206 | INFO | Uploaded 36GiB to GPU in 1.97928s (18.1885GiB/s) | 20260507 05:04:11.206 | INFO | Downloaded 34GiB to GPU in 1.92877s (17.6279GiB/s) | 20260507 05:04:16.342 | INFO | Analyzing chunk frames = [7308288,7308800) | 20260507 05:04:16.342 | INFO | MemInfo: { | 20260507 05:04:16.342 | INFO | "gpu" : " 17 GB", | 20260507 05:04:16.342 | INFO | "huge" : "", | 20260507 05:04:16.342 | INFO | "huge_shared" : " 27 GB", | 20260507 05:04:16.342 | INFO | "malloc" : " 14 GB", | 20260507 05:04:16.342 | INFO | "pinned" : " 29 GB" | 20260507 05:04:16.342 | INFO | } | 20260507 05:04:16.342 | INFO | Perf report for the last 5.1361 seconds: | 20260507 05:04:16.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:04:16.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:16.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:16.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:16.342 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 05:04:16.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:16.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:16.342 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:04:16.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:16.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:16.343 | INFO | Uploaded 36GiB to GPU in 2.03248s (17.7124GiB/s) | 20260507 05:04:16.343 | INFO | Downloaded 34GiB to GPU in 1.9547s (17.394GiB/s) | 20260507 05:04:21.489 | INFO | Analyzing chunk frames = [7308800,7309312) | 20260507 05:04:21.489 | INFO | MemInfo: { | 20260507 05:04:21.489 | INFO | "gpu" : " 17 GB", | 20260507 05:04:21.489 | INFO | "huge" : "", | 20260507 05:04:21.489 | INFO | "huge_shared" : " 27 GB", | 20260507 05:04:21.489 | INFO | "malloc" : " 14 GB", | 20260507 05:04:21.489 | INFO | "pinned" : " 29 GB" | 20260507 05:04:21.489 | INFO | } | 20260507 05:04:21.489 | INFO | Perf report for the last 5.14672 seconds: | 20260507 05:04:21.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:04:21.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:21.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:21.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:21.489 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 05:04:21.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:21.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:21.489 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:04:21.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:21.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:21.489 | INFO | Uploaded 36GiB to GPU in 1.98302s (18.1542GiB/s) | 20260507 05:04:21.489 | INFO | Downloaded 34GiB to GPU in 1.92294s (17.6812GiB/s) | 20260507 05:04:26.630 | INFO | Analyzing chunk frames = [7309312,7309824) | 20260507 05:04:26.630 | INFO | MemInfo: { | 20260507 05:04:26.630 | INFO | "gpu" : " 17 GB", | 20260507 05:04:26.630 | INFO | "huge" : "", | 20260507 05:04:26.630 | INFO | "huge_shared" : " 27 GB", | 20260507 05:04:26.630 | INFO | "malloc" : " 14 GB", | 20260507 05:04:26.630 | INFO | "pinned" : " 29 GB" | 20260507 05:04:26.630 | INFO | } | 20260507 05:04:26.630 | INFO | Perf report for the last 5.14097 seconds: | 20260507 05:04:26.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:04:26.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:26.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:26.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:26.630 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 05:04:26.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:26.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:26.630 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:04:26.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:26.630 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:26.630 | INFO | Uploaded 36GiB to GPU in 2.05032s (17.5583GiB/s) | 20260507 05:04:26.630 | INFO | Downloaded 34GiB to GPU in 1.95345s (17.4051GiB/s) | 20260507 05:04:31.764 | INFO | Analyzing chunk frames = [7309824,7310336) | 20260507 05:04:31.764 | INFO | MemInfo: { | 20260507 05:04:31.764 | INFO | "gpu" : " 17 GB", | 20260507 05:04:31.764 | INFO | "huge" : "", | 20260507 05:04:31.764 | INFO | "huge_shared" : " 27 GB", | 20260507 05:04:31.764 | INFO | "malloc" : " 14 GB", | 20260507 05:04:31.764 | INFO | "pinned" : " 29 GB" | 20260507 05:04:31.764 | INFO | } | 20260507 05:04:31.764 | INFO | Perf report for the last 5.13447 seconds: | 20260507 05:04:31.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:04:31.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:31.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:31.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:31.764 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 05:04:31.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:31.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:31.764 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260507 05:04:31.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:31.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:31.765 | INFO | Uploaded 36GiB to GPU in 2.06556s (17.4287GiB/s) | 20260507 05:04:31.765 | INFO | Downloaded 34GiB to GPU in 1.93515s (17.5697GiB/s) | 20260507 05:04:36.900 | INFO | Analyzing chunk frames = [7310336,7310848) | 20260507 05:04:36.900 | INFO | MemInfo: { | 20260507 05:04:36.900 | INFO | "gpu" : " 17 GB", | 20260507 05:04:36.900 | INFO | "huge" : "", | 20260507 05:04:36.900 | INFO | "huge_shared" : " 27 GB", | 20260507 05:04:36.900 | INFO | "malloc" : " 14 GB", | 20260507 05:04:36.900 | INFO | "pinned" : " 29 GB" | 20260507 05:04:36.900 | INFO | } | 20260507 05:04:36.901 | INFO | Perf report for the last 5.13621 seconds: | 20260507 05:04:36.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:04:36.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:36.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:36.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:36.901 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 05:04:36.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:36.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:36.901 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 05:04:36.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:36.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:36.901 | INFO | Uploaded 36GiB to GPU in 2.02445s (17.7826GiB/s) | 20260507 05:04:36.901 | INFO | Downloaded 34GiB to GPU in 1.98466s (17.1314GiB/s) | 20260507 05:04:42.043 | INFO | Analyzing chunk frames = [7310848,7311360) | 20260507 05:04:42.043 | INFO | MemInfo: { | 20260507 05:04:42.043 | INFO | "gpu" : " 17 GB", | 20260507 05:04:42.043 | INFO | "huge" : "", | 20260507 05:04:42.043 | INFO | "huge_shared" : " 27 GB", | 20260507 05:04:42.043 | INFO | "malloc" : " 14 GB", | 20260507 05:04:42.043 | INFO | "pinned" : " 29 GB" | 20260507 05:04:42.043 | INFO | } | 20260507 05:04:42.043 | INFO | Perf report for the last 5.14231 seconds: | 20260507 05:04:42.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:04:42.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:42.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:42.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:42.043 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 05:04:42.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:42.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:42.043 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 05:04:42.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:42.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:42.043 | INFO | Uploaded 36GiB to GPU in 1.96674s (18.3044GiB/s) | 20260507 05:04:42.043 | INFO | Downloaded 34GiB to GPU in 1.9473s (17.4601GiB/s) | 20260507 05:04:42.140 | INFO | Starting a round of Baz IO | 20260507 05:04:45.246 | INFO | Populating RTMetrics report | 20260507 05:04:45.246 | INFO | RTMetrics report populated | 20260507 05:04:45.246 | INFO | Opening tmp RTMetrics json file | 20260507 05:04:45.247 | INFO | Serializing RTMetrics to json | 20260507 05:04:45.248 | INFO | Serialization done, swapping tmp file now... | 20260507 05:04:45.248 | INFO | RTMetrics done | 20260507 05:04:45.333 | INFO | Finished a round of Baz IO: Wrote 1.707 GiB in 3.193 seconds | 20260507 05:04:45.333 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.74% / 0.00% / 67.26% | 20260507 05:04:45.333 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.153406% / 0% / 5.16574e-05% | 20260507 05:04:45.333 | INFO | Baz IO overall: | 20260507 05:04:45.333 | INFO | BytesWritten : 3717937336630 | 20260507 05:04:45.333 | INFO | NumEvents : 1783924022506 | 20260507 05:04:47.183 | INFO | Analyzing chunk frames = [7311360,7311872) | 20260507 05:04:47.184 | INFO | MemInfo: { | 20260507 05:04:47.184 | INFO | "gpu" : " 17 GB", | 20260507 05:04:47.184 | INFO | "huge" : "", | 20260507 05:04:47.184 | INFO | "huge_shared" : " 27 GB", | 20260507 05:04:47.184 | INFO | "malloc" : " 14 GB", | 20260507 05:04:47.184 | INFO | "pinned" : " 29 GB" | 20260507 05:04:47.184 | INFO | } | 20260507 05:04:47.184 | INFO | Perf report for the last 5.14074 seconds: | 20260507 05:04:47.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:04:47.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:47.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:47.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:47.184 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 05:04:47.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:47.184 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 05:04:47.184 | INFO | PRE_HQ : 59.3%, 1.8, 55.2ms, 0 | 20260507 05:04:47.184 | INFO | BAZWRITER : 61.7%, 3.2, 101.0ms, 0 | 20260507 05:04:47.184 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:04:47.184 | INFO | Uploaded 36GiB to GPU in 2.03851s (17.66GiB/s) | 20260507 05:04:47.184 | INFO | Downloaded 38GiB to GPU in 2.18213s (17.4142GiB/s) | 20260507 05:04:52.321 | INFO | Analyzing chunk frames = [7311872,7312384) | 20260507 05:04:52.321 | INFO | MemInfo: { | 20260507 05:04:52.321 | INFO | "gpu" : " 17 GB", | 20260507 05:04:52.321 | INFO | "huge" : "", | 20260507 05:04:52.321 | INFO | "huge_shared" : " 27 GB", | 20260507 05:04:52.321 | INFO | "malloc" : " 14 GB", | 20260507 05:04:52.321 | INFO | "pinned" : " 29 GB" | 20260507 05:04:52.321 | INFO | } | 20260507 05:04:52.321 | INFO | Perf report for the last 5.13731 seconds: | 20260507 05:04:52.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:04:52.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:52.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:52.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:52.321 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:04:52.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:52.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:52.321 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:04:52.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:52.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:52.321 | INFO | Uploaded 36GiB to GPU in 1.99218s (18.0707GiB/s) | 20260507 05:04:52.321 | INFO | Downloaded 34GiB to GPU in 1.93285s (17.5906GiB/s) | 20260507 05:04:57.460 | INFO | Analyzing chunk frames = [7312384,7312896) | 20260507 05:04:57.460 | INFO | MemInfo: { | 20260507 05:04:57.460 | INFO | "gpu" : " 17 GB", | 20260507 05:04:57.460 | INFO | "huge" : "", | 20260507 05:04:57.460 | INFO | "huge_shared" : " 27 GB", | 20260507 05:04:57.461 | INFO | "malloc" : " 14 GB", | 20260507 05:04:57.461 | INFO | "pinned" : " 29 GB" | 20260507 05:04:57.461 | INFO | } | 20260507 05:04:57.461 | INFO | Perf report for the last 5.13965 seconds: | 20260507 05:04:57.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:04:57.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:57.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:57.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:57.461 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 05:04:57.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:57.461 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:57.461 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:04:57.461 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:57.461 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:04:57.461 | INFO | Uploaded 36GiB to GPU in 2.04491s (17.6047GiB/s) | 20260507 05:04:57.461 | INFO | Downloaded 34GiB to GPU in 1.95258s (17.4128GiB/s) | 20260507 05:05:02.598 | INFO | Analyzing chunk frames = [7312896,7313408) | 20260507 05:05:02.598 | INFO | MemInfo: { | 20260507 05:05:02.598 | INFO | "gpu" : " 17 GB", | 20260507 05:05:02.598 | INFO | "huge" : "", | 20260507 05:05:02.598 | INFO | "huge_shared" : " 27 GB", | 20260507 05:05:02.598 | INFO | "malloc" : " 14 GB", | 20260507 05:05:02.598 | INFO | "pinned" : " 29 GB" | 20260507 05:05:02.598 | INFO | } | 20260507 05:05:02.598 | INFO | Perf report for the last 5.13773 seconds: | 20260507 05:05:02.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:05:02.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:02.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:02.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:02.598 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 05:05:02.599 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:02.599 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:02.599 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:05:02.599 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:02.599 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:02.599 | INFO | Uploaded 36GiB to GPU in 2.0009s (17.9919GiB/s) | 20260507 05:05:02.599 | INFO | Downloaded 34GiB to GPU in 1.93507s (17.5704GiB/s) | 20260507 05:05:07.742 | INFO | Analyzing chunk frames = [7313408,7313920) | 20260507 05:05:07.742 | INFO | MemInfo: { | 20260507 05:05:07.742 | INFO | "gpu" : " 17 GB", | 20260507 05:05:07.742 | INFO | "huge" : "", | 20260507 05:05:07.742 | INFO | "huge_shared" : " 27 GB", | 20260507 05:05:07.742 | INFO | "malloc" : " 14 GB", | 20260507 05:05:07.742 | INFO | "pinned" : " 29 GB" | 20260507 05:05:07.742 | INFO | } | 20260507 05:05:07.742 | INFO | Perf report for the last 5.14336 seconds: | 20260507 05:05:07.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:05:07.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:07.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:07.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:07.742 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 05:05:07.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:07.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:07.742 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 05:05:07.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:07.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:07.742 | INFO | Uploaded 36GiB to GPU in 2.0267s (17.7629GiB/s) | 20260507 05:05:07.742 | INFO | Downloaded 34GiB to GPU in 1.93629s (17.5594GiB/s) | 20260507 05:05:12.880 | INFO | Analyzing chunk frames = [7313920,7314432) | 20260507 05:05:12.880 | INFO | MemInfo: { | 20260507 05:05:12.880 | INFO | "gpu" : " 17 GB", | 20260507 05:05:12.880 | INFO | "huge" : "", | 20260507 05:05:12.880 | INFO | "huge_shared" : " 27 GB", | 20260507 05:05:12.880 | INFO | "malloc" : " 14 GB", | 20260507 05:05:12.880 | INFO | "pinned" : " 29 GB" | 20260507 05:05:12.880 | INFO | } | 20260507 05:05:12.880 | INFO | Perf report for the last 5.13853 seconds: | 20260507 05:05:12.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:05:12.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:12.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:12.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:12.880 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 05:05:12.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:12.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:12.880 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 05:05:12.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:12.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:12.881 | INFO | Uploaded 36GiB to GPU in 2.02648s (17.7648GiB/s) | 20260507 05:05:12.881 | INFO | Downloaded 34GiB to GPU in 1.91313s (17.772GiB/s) | 20260507 05:05:18.012 | INFO | Analyzing chunk frames = [7314432,7314944) | 20260507 05:05:18.012 | INFO | MemInfo: { | 20260507 05:05:18.012 | INFO | "gpu" : " 17 GB", | 20260507 05:05:18.012 | INFO | "huge" : "", | 20260507 05:05:18.012 | INFO | "huge_shared" : " 27 GB", | 20260507 05:05:18.012 | INFO | "malloc" : " 14 GB", | 20260507 05:05:18.012 | INFO | "pinned" : " 29 GB" | 20260507 05:05:18.012 | INFO | } | 20260507 05:05:18.012 | INFO | Perf report for the last 5.13195 seconds: | 20260507 05:05:18.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:05:18.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:18.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:18.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:18.012 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 05:05:18.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:18.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:18.012 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 05:05:18.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:18.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:18.013 | INFO | Uploaded 36GiB to GPU in 1.98907s (18.0989GiB/s) | 20260507 05:05:18.013 | INFO | Downloaded 34GiB to GPU in 1.98153s (17.1585GiB/s) | 20260507 05:05:23.160 | INFO | Analyzing chunk frames = [7314944,7315456) | 20260507 05:05:23.160 | INFO | MemInfo: { | 20260507 05:05:23.160 | INFO | "gpu" : " 17 GB", | 20260507 05:05:23.160 | INFO | "huge" : "", | 20260507 05:05:23.160 | INFO | "huge_shared" : " 27 GB", | 20260507 05:05:23.160 | INFO | "malloc" : " 14 GB", | 20260507 05:05:23.160 | INFO | "pinned" : " 29 GB" | 20260507 05:05:23.160 | INFO | } | 20260507 05:05:23.160 | INFO | Perf report for the last 5.14764 seconds: | 20260507 05:05:23.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:05:23.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:23.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:23.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:23.160 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:05:23.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:23.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:23.160 | INFO | PRE_HQ : 29.7%, 1.0, 15.4ms, 0 | 20260507 05:05:23.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:23.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:23.160 | INFO | Uploaded 36GiB to GPU in 1.97596s (18.219GiB/s) | 20260507 05:05:23.160 | INFO | Downloaded 34GiB to GPU in 1.95083s (17.4285GiB/s) | 20260507 05:05:23.161 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:05:23.161 | INFO | Total mem usage: 18 GiB | 20260507 05:05:23.161 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:05:23.161 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 05:05:23.161 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:05:23.161 | INFO | High Water Breakdown: | 20260507 05:05:23.161 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:05:23.161 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:05:23.161 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:05:23.161 | INFO | ------------------------------------------ | 20260507 05:05:23.161 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:05:23.161 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:05:23.161 | INFO | Total mem usage: 26.5 GiB | 20260507 05:05:23.161 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:05:23.161 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:05:23.161 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:05:23.161 | INFO | High Water Breakdown: | 20260507 05:05:23.161 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:05:23.161 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:05:23.161 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:05:23.161 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:05:23.161 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:05:23.161 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:05:23.161 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:05:23.161 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:05:23.161 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:05:23.161 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:05:23.161 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:05:23.161 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:05:23.161 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:05:23.161 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:05:23.161 | INFO | ------------------------------------------ | 20260507 05:05:23.161 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:05:23.161 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:05:23.161 | INFO | Total mem usage: 25.6 GiB | 20260507 05:05:23.161 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:05:23.161 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:05:23.161 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:05:23.161 | INFO | High Water Breakdown: | 20260507 05:05:23.161 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:05:23.161 | INFO | ------------------------------------------ | 20260507 05:05:23.161 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:05:23.161 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:05:23.161 | INFO | Total mem usage: 15.7 GiB | 20260507 05:05:23.161 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:05:23.161 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:05:23.161 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:05:23.161 | INFO | High Water Breakdown: | 20260507 05:05:23.161 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:05:23.161 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:05:23.161 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:05:23.161 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:05:23.161 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:05:23.161 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:05:23.161 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:05:23.161 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:05:23.161 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:05:23.161 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:05:23.161 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:05:23.161 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:05:23.161 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:05:23.161 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:05:23.161 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:05:23.161 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:05:23.161 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:05:23.161 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:05:23.161 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:05:23.161 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:05:23.161 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:05:23.161 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:05:23.161 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:05:23.161 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:05:23.161 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:05:23.161 | INFO | ------------------------------------------ | 20260507 05:05:23.161 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:05:23.161 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:05:23.161 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:05:23.161 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:05:23.161 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.32e+01 / 1.45e+00 | 20260507 05:05:23.161 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 1.02e-01, 9.63e-03 / 5.94e-03 | 20260507 05:05:23.161 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.43e+00, 5.10e-01 / 2.35e-01 | 20260507 05:05:23.161 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.47e+00, 1.70e-01 / 1.21e-01 | 20260507 05:05:23.161 | INFO | PerfReport: 16.20%, DME -- 3.98e+01 / 4.45e+01, 3.94e+00 / 1.75e+00 | 20260507 05:05:23.161 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.77e+01 / 1.99e+01, 1.28e+00 / 1.42e+00 | 20260507 05:05:23.161 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.69e+00, 3.85e-01 / 2.09e-01 | 20260507 05:05:23.161 | INFO | PerfReport: 3.01%, Metrics -- 9.24e-01 / 1.23e+00, 2.46e-01 / 1.31e-01 | 20260507 05:05:23.161 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.12e-01 / 3.63e-01, 4.91e-02 / 2.05e-02 | 20260507 05:05:23.161 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:05:23.161 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:05:23.161 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:05:23.161 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 05:05:23.161 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260507 05:05:23.161 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 05:05:23.161 | INFO | PerfReport: 31.96%, Compute -- 3.07e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 05:05:23.161 | INFO | PerfReport: 1.39%, LockUpload -- 1.34e+00 / 0.00e+00, 4.76e+00 / 0.00e+00 | 20260507 05:05:23.161 | INFO | PerfReport: 1.35%, LockDownload -- 1.29e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260507 05:05:23.161 | INFO | PerfReport: 22.09%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 05:05:23.161 | INFO | PerfReport: 0.01%, Misc -- 1.28e-02 / 0.00e+00, 1.37e-02 / 0.00e+00 | 20260507 05:05:23.257 | INFO | Starting a round of Baz IO | 20260507 05:05:26.354 | INFO | Populating RTMetrics report | 20260507 05:05:26.354 | INFO | RTMetrics report populated | 20260507 05:05:26.354 | INFO | Opening tmp RTMetrics json file | 20260507 05:05:26.372 | INFO | Serializing RTMetrics to json | 20260507 05:05:26.374 | INFO | Serialization done, swapping tmp file now... | 20260507 05:05:26.374 | INFO | RTMetrics done | 20260507 05:05:26.446 | INFO | Finished a round of Baz IO: Wrote 1.706 GiB in 3.188 seconds | 20260507 05:05:26.446 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.73% / 0.00% / 67.26% | 20260507 05:05:26.446 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.153251% / 0% / 7.94729e-05% | 20260507 05:05:26.446 | INFO | Baz IO overall: | 20260507 05:05:26.446 | INFO | BytesWritten : 3719769475428 | 20260507 05:05:26.446 | INFO | NumEvents : 1784784608471 | 20260507 05:05:28.300 | INFO | Analyzing chunk frames = [7315456,7315968) | 20260507 05:05:28.300 | INFO | MemInfo: { | 20260507 05:05:28.300 | INFO | "gpu" : " 17 GB", | 20260507 05:05:28.300 | INFO | "huge" : "", | 20260507 05:05:28.300 | INFO | "huge_shared" : " 27 GB", | 20260507 05:05:28.300 | INFO | "malloc" : " 14 GB", | 20260507 05:05:28.300 | INFO | "pinned" : " 29 GB" | 20260507 05:05:28.300 | INFO | } | 20260507 05:05:28.300 | INFO | Perf report for the last 5.14017 seconds: | 20260507 05:05:28.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:05:28.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:28.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:28.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:28.300 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:05:28.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:28.300 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 05:05:28.300 | INFO | PRE_HQ : 59.0%, 1.8, 55.0ms, 0 | 20260507 05:05:28.300 | INFO | BAZWRITER : 61.6%, 3.2, 101.8ms, 0 | 20260507 05:05:28.300 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:05:28.301 | INFO | Uploaded 36GiB to GPU in 2.0351s (17.6896GiB/s) | 20260507 05:05:28.301 | INFO | Downloaded 38GiB to GPU in 2.17878s (17.441GiB/s) | 20260507 05:05:33.447 | INFO | Analyzing chunk frames = [7315968,7316480) | 20260507 05:05:33.447 | INFO | MemInfo: { | 20260507 05:05:33.447 | INFO | "gpu" : " 17 GB", | 20260507 05:05:33.447 | INFO | "huge" : "", | 20260507 05:05:33.447 | INFO | "huge_shared" : " 27 GB", | 20260507 05:05:33.447 | INFO | "malloc" : " 14 GB", | 20260507 05:05:33.447 | INFO | "pinned" : " 29 GB" | 20260507 05:05:33.447 | INFO | } | 20260507 05:05:33.447 | INFO | Perf report for the last 5.14678 seconds: | 20260507 05:05:33.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:05:33.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:33.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:33.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:33.447 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 05:05:33.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:33.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:33.447 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:05:33.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:33.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:33.447 | INFO | Uploaded 36GiB to GPU in 1.99341s (18.0595GiB/s) | 20260507 05:05:33.447 | INFO | Downloaded 34GiB to GPU in 1.93146s (17.6032GiB/s) | 20260507 05:05:38.579 | INFO | Analyzing chunk frames = [7316480,7316992) | 20260507 05:05:38.579 | INFO | MemInfo: { | 20260507 05:05:38.579 | INFO | "gpu" : " 17 GB", | 20260507 05:05:38.579 | INFO | "huge" : "", | 20260507 05:05:38.579 | INFO | "huge_shared" : " 27 GB", | 20260507 05:05:38.579 | INFO | "malloc" : " 14 GB", | 20260507 05:05:38.579 | INFO | "pinned" : " 29 GB" | 20260507 05:05:38.579 | INFO | } | 20260507 05:05:38.579 | INFO | Perf report for the last 5.1319 seconds: | 20260507 05:05:38.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:05:38.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:38.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:38.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:38.579 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260507 05:05:38.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:38.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:38.579 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 05:05:38.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:38.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:38.579 | INFO | Uploaded 36GiB to GPU in 2.03612s (17.6807GiB/s) | 20260507 05:05:38.579 | INFO | Downloaded 34GiB to GPU in 1.91822s (17.7248GiB/s) | 20260507 05:05:43.712 | INFO | Analyzing chunk frames = [7316992,7317504) | 20260507 05:05:43.712 | INFO | MemInfo: { | 20260507 05:05:43.712 | INFO | "gpu" : " 17 GB", | 20260507 05:05:43.712 | INFO | "huge" : "", | 20260507 05:05:43.712 | INFO | "huge_shared" : " 27 GB", | 20260507 05:05:43.712 | INFO | "malloc" : " 14 GB", | 20260507 05:05:43.712 | INFO | "pinned" : " 29 GB" | 20260507 05:05:43.712 | INFO | } | 20260507 05:05:43.712 | INFO | Perf report for the last 5.13326 seconds: | 20260507 05:05:43.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:05:43.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:43.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:43.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:43.712 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 05:05:43.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:43.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:43.712 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:05:43.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:43.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:43.712 | INFO | Uploaded 36GiB to GPU in 2.01899s (17.8307GiB/s) | 20260507 05:05:43.712 | INFO | Downloaded 34GiB to GPU in 1.94035s (17.5226GiB/s) | 20260507 05:05:48.852 | INFO | Analyzing chunk frames = [7317504,7318016) | 20260507 05:05:48.852 | INFO | MemInfo: { | 20260507 05:05:48.852 | INFO | "gpu" : " 17 GB", | 20260507 05:05:48.852 | INFO | "huge" : "", | 20260507 05:05:48.852 | INFO | "huge_shared" : " 27 GB", | 20260507 05:05:48.853 | INFO | "malloc" : " 14 GB", | 20260507 05:05:48.853 | INFO | "pinned" : " 29 GB" | 20260507 05:05:48.853 | INFO | } | 20260507 05:05:48.853 | INFO | Perf report for the last 5.14057 seconds: | 20260507 05:05:48.853 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:05:48.853 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:48.853 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:48.853 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:48.853 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 05:05:48.853 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:48.853 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:48.853 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:05:48.853 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:48.853 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:48.853 | INFO | Uploaded 36GiB to GPU in 2.03751s (17.6686GiB/s) | 20260507 05:05:48.853 | INFO | Downloaded 34GiB to GPU in 1.9392s (17.533GiB/s) | 20260507 05:05:53.993 | INFO | Analyzing chunk frames = [7318016,7318528) | 20260507 05:05:53.993 | INFO | MemInfo: { | 20260507 05:05:53.993 | INFO | "gpu" : " 17 GB", | 20260507 05:05:53.993 | INFO | "huge" : "", | 20260507 05:05:53.993 | INFO | "huge_shared" : " 27 GB", | 20260507 05:05:53.993 | INFO | "malloc" : " 14 GB", | 20260507 05:05:53.993 | INFO | "pinned" : " 29 GB" | 20260507 05:05:53.993 | INFO | } | 20260507 05:05:53.993 | INFO | Perf report for the last 5.14055 seconds: | 20260507 05:05:53.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:05:53.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:53.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:53.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:53.993 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:05:53.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:53.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:53.993 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:05:53.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:53.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:53.994 | INFO | Uploaded 36GiB to GPU in 2.05507s (17.5177GiB/s) | 20260507 05:05:53.994 | INFO | Downloaded 34GiB to GPU in 1.96222s (17.3273GiB/s) | 20260507 05:05:59.140 | INFO | Analyzing chunk frames = [7318528,7319040) | 20260507 05:05:59.140 | INFO | MemInfo: { | 20260507 05:05:59.140 | INFO | "gpu" : " 17 GB", | 20260507 05:05:59.140 | INFO | "huge" : "", | 20260507 05:05:59.140 | INFO | "huge_shared" : " 27 GB", | 20260507 05:05:59.140 | INFO | "malloc" : " 14 GB", | 20260507 05:05:59.140 | INFO | "pinned" : " 29 GB" | 20260507 05:05:59.140 | INFO | } | 20260507 05:05:59.140 | INFO | Perf report for the last 5.1467 seconds: | 20260507 05:05:59.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:05:59.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:59.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:59.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:59.140 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:05:59.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:59.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:59.140 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 05:05:59.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:59.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:05:59.140 | INFO | Uploaded 36GiB to GPU in 1.96138s (18.3544GiB/s) | 20260507 05:05:59.140 | INFO | Downloaded 34GiB to GPU in 1.97748s (17.1936GiB/s) | 20260507 05:06:04.276 | INFO | Analyzing chunk frames = [7319040,7319552) | 20260507 05:06:04.276 | INFO | MemInfo: { | 20260507 05:06:04.276 | INFO | "gpu" : " 17 GB", | 20260507 05:06:04.276 | INFO | "huge" : "", | 20260507 05:06:04.276 | INFO | "huge_shared" : " 27 GB", | 20260507 05:06:04.276 | INFO | "malloc" : " 14 GB", | 20260507 05:06:04.276 | INFO | "pinned" : " 29 GB" | 20260507 05:06:04.276 | INFO | } | 20260507 05:06:04.276 | INFO | Perf report for the last 5.13612 seconds: | 20260507 05:06:04.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:06:04.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:04.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:04.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:04.276 | INFO | ANALYSIS : 60.7%, 3.0, 93.5ms, 0 | 20260507 05:06:04.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:04.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:04.276 | INFO | PRE_HQ : 25.1%, 1.0, 12.9ms, 0 | 20260507 05:06:04.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:04.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:04.276 | INFO | Uploaded 36GiB to GPU in 1.99397s (18.0544GiB/s) | 20260507 05:06:04.276 | INFO | Downloaded 34GiB to GPU in 1.95373s (17.4026GiB/s) | 20260507 05:06:04.373 | INFO | Starting a round of Baz IO | 20260507 05:06:07.467 | INFO | Populating RTMetrics report | 20260507 05:06:07.467 | INFO | RTMetrics report populated | 20260507 05:06:07.467 | INFO | Opening tmp RTMetrics json file | 20260507 05:06:07.473 | INFO | Serializing RTMetrics to json | 20260507 05:06:07.484 | INFO | Serialization done, swapping tmp file now... | 20260507 05:06:07.485 | INFO | RTMetrics done | 20260507 05:06:07.552 | INFO | Finished a round of Baz IO: Wrote 1.705 GiB in 3.179 seconds | 20260507 05:06:07.552 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.73% / 0.00% / 67.27% | 20260507 05:06:07.552 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.152735% / 0% / 6.75519e-05% | 20260507 05:06:07.552 | INFO | Baz IO overall: | 20260507 05:06:07.552 | INFO | BytesWritten : 3721600292929 | 20260507 05:06:07.552 | INFO | NumEvents : 1785644391734 | 20260507 05:06:09.415 | INFO | Analyzing chunk frames = [7319552,7320064) | 20260507 05:06:09.415 | INFO | MemInfo: { | 20260507 05:06:09.415 | INFO | "gpu" : " 17 GB", | 20260507 05:06:09.415 | INFO | "huge" : "", | 20260507 05:06:09.415 | INFO | "huge_shared" : " 27 GB", | 20260507 05:06:09.415 | INFO | "malloc" : " 14 GB", | 20260507 05:06:09.415 | INFO | "pinned" : " 29 GB" | 20260507 05:06:09.415 | INFO | } | 20260507 05:06:09.415 | INFO | Perf report for the last 5.13893 seconds: | 20260507 05:06:09.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:06:09.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:09.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:09.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:09.415 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 05:06:09.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:09.415 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 05:06:09.415 | INFO | PRE_HQ : 59.1%, 1.8, 54.8ms, 0 | 20260507 05:06:09.415 | INFO | BAZWRITER : 61.5%, 3.2, 100.4ms, 0 | 20260507 05:06:09.415 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 05:06:09.415 | INFO | Uploaded 36GiB to GPU in 2.05547s (17.5142GiB/s) | 20260507 05:06:09.415 | INFO | Downloaded 38GiB to GPU in 2.175s (17.4713GiB/s) | 20260507 05:06:14.552 | INFO | Analyzing chunk frames = [7320064,7320576) | 20260507 05:06:14.552 | INFO | MemInfo: { | 20260507 05:06:14.552 | INFO | "gpu" : " 17 GB", | 20260507 05:06:14.552 | INFO | "huge" : "", | 20260507 05:06:14.552 | INFO | "huge_shared" : " 27 GB", | 20260507 05:06:14.552 | INFO | "malloc" : " 14 GB", | 20260507 05:06:14.552 | INFO | "pinned" : " 29 GB" | 20260507 05:06:14.552 | INFO | } | 20260507 05:06:14.552 | INFO | Perf report for the last 5.1374 seconds: | 20260507 05:06:14.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:06:14.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:14.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:14.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:14.553 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:06:14.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:14.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:14.553 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:06:14.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:14.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:14.553 | INFO | Uploaded 36GiB to GPU in 1.99799s (18.0182GiB/s) | 20260507 05:06:14.553 | INFO | Downloaded 34GiB to GPU in 1.92596s (17.6536GiB/s) | 20260507 05:06:19.690 | INFO | Analyzing chunk frames = [7320576,7321088) | 20260507 05:06:19.690 | INFO | MemInfo: { | 20260507 05:06:19.690 | INFO | "gpu" : " 17 GB", | 20260507 05:06:19.690 | INFO | "huge" : "", | 20260507 05:06:19.690 | INFO | "huge_shared" : " 27 GB", | 20260507 05:06:19.690 | INFO | "malloc" : " 14 GB", | 20260507 05:06:19.690 | INFO | "pinned" : " 29 GB" | 20260507 05:06:19.690 | INFO | } | 20260507 05:06:19.690 | INFO | Perf report for the last 5.13775 seconds: | 20260507 05:06:19.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:06:19.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:19.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:19.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:19.690 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 05:06:19.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:19.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:19.690 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:06:19.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:19.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:19.691 | INFO | Uploaded 36GiB to GPU in 2.01435s (17.8717GiB/s) | 20260507 05:06:19.691 | INFO | Downloaded 34GiB to GPU in 1.941s (17.5168GiB/s) | 20260507 05:06:24.827 | INFO | Analyzing chunk frames = [7321088,7321600) | 20260507 05:06:24.828 | INFO | MemInfo: { | 20260507 05:06:24.828 | INFO | "gpu" : " 17 GB", | 20260507 05:06:24.828 | INFO | "huge" : "", | 20260507 05:06:24.828 | INFO | "huge_shared" : " 27 GB", | 20260507 05:06:24.828 | INFO | "malloc" : " 14 GB", | 20260507 05:06:24.828 | INFO | "pinned" : " 29 GB" | 20260507 05:06:24.828 | INFO | } | 20260507 05:06:24.828 | INFO | Perf report for the last 5.13756 seconds: | 20260507 05:06:24.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:06:24.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:24.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:24.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:24.828 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 05:06:24.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:24.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:24.828 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:06:24.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:24.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:24.828 | INFO | Uploaded 36GiB to GPU in 2.01127s (17.8992GiB/s) | 20260507 05:06:24.828 | INFO | Downloaded 34GiB to GPU in 1.93765s (17.547GiB/s) | 20260507 05:06:29.966 | INFO | Analyzing chunk frames = [7321600,7322112) | 20260507 05:06:29.966 | INFO | MemInfo: { | 20260507 05:06:29.966 | INFO | "gpu" : " 17 GB", | 20260507 05:06:29.966 | INFO | "huge" : "", | 20260507 05:06:29.966 | INFO | "huge_shared" : " 27 GB", | 20260507 05:06:29.966 | INFO | "malloc" : " 14 GB", | 20260507 05:06:29.966 | INFO | "pinned" : " 29 GB" | 20260507 05:06:29.966 | INFO | } | 20260507 05:06:29.966 | INFO | Perf report for the last 5.1386 seconds: | 20260507 05:06:29.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:06:29.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:29.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:29.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:29.966 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 05:06:29.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:29.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:29.967 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:06:29.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:29.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:29.967 | INFO | Uploaded 36GiB to GPU in 2.05761s (17.496GiB/s) | 20260507 05:06:29.967 | INFO | Downloaded 34GiB to GPU in 1.94764s (17.457GiB/s) | 20260507 05:06:35.107 | INFO | Analyzing chunk frames = [7322112,7322624) | 20260507 05:06:35.107 | INFO | MemInfo: { | 20260507 05:06:35.107 | INFO | "gpu" : " 17 GB", | 20260507 05:06:35.107 | INFO | "huge" : "", | 20260507 05:06:35.107 | INFO | "huge_shared" : " 27 GB", | 20260507 05:06:35.107 | INFO | "malloc" : " 14 GB", | 20260507 05:06:35.107 | INFO | "pinned" : " 29 GB" | 20260507 05:06:35.107 | INFO | } | 20260507 05:06:35.107 | INFO | Perf report for the last 5.14073 seconds: | 20260507 05:06:35.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:06:35.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:35.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:35.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:35.107 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 05:06:35.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:35.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:35.107 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 05:06:35.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:35.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:35.107 | INFO | Uploaded 36GiB to GPU in 2.04921s (17.5678GiB/s) | 20260507 05:06:35.108 | INFO | Downloaded 34GiB to GPU in 1.92438s (17.668GiB/s) | 20260507 05:06:40.256 | INFO | Analyzing chunk frames = [7322624,7323136) | 20260507 05:06:40.256 | INFO | MemInfo: { | 20260507 05:06:40.256 | INFO | "gpu" : " 17 GB", | 20260507 05:06:40.256 | INFO | "huge" : "", | 20260507 05:06:40.256 | INFO | "huge_shared" : " 27 GB", | 20260507 05:06:40.256 | INFO | "malloc" : " 14 GB", | 20260507 05:06:40.256 | INFO | "pinned" : " 29 GB" | 20260507 05:06:40.256 | INFO | } | 20260507 05:06:40.256 | INFO | Perf report for the last 5.14897 seconds: | 20260507 05:06:40.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:06:40.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:40.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:40.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:40.256 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 05:06:40.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:40.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:40.256 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 05:06:40.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:40.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:40.256 | INFO | Uploaded 36GiB to GPU in 1.96923s (18.2813GiB/s) | 20260507 05:06:40.256 | INFO | Downloaded 34GiB to GPU in 1.9584s (17.3611GiB/s) | 20260507 05:06:45.397 | INFO | Analyzing chunk frames = [7323136,7323648) | 20260507 05:06:45.397 | INFO | MemInfo: { | 20260507 05:06:45.397 | INFO | "gpu" : " 17 GB", | 20260507 05:06:45.397 | INFO | "huge" : "", | 20260507 05:06:45.397 | INFO | "huge_shared" : " 27 GB", | 20260507 05:06:45.397 | INFO | "malloc" : " 14 GB", | 20260507 05:06:45.397 | INFO | "pinned" : " 29 GB" | 20260507 05:06:45.397 | INFO | } | 20260507 05:06:45.398 | INFO | Perf report for the last 5.14137 seconds: | 20260507 05:06:45.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:06:45.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:45.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:45.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:45.398 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:06:45.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:45.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:45.398 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:06:45.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:45.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:45.398 | INFO | Uploaded 36GiB to GPU in 1.99805s (18.0175GiB/s) | 20260507 05:06:45.398 | INFO | Downloaded 34GiB to GPU in 1.96013s (17.3458GiB/s) | 20260507 05:06:45.398 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:06:45.398 | INFO | Total mem usage: 18 GiB | 20260507 05:06:45.398 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:06:45.398 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:06:45.398 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:06:45.398 | INFO | High Water Breakdown: | 20260507 05:06:45.398 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:06:45.398 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:06:45.398 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:06:45.398 | INFO | ------------------------------------------ | 20260507 05:06:45.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:06:45.398 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:06:45.398 | INFO | Total mem usage: 26.5 GiB | 20260507 05:06:45.398 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:06:45.398 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:06:45.398 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:06:45.398 | INFO | High Water Breakdown: | 20260507 05:06:45.398 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:06:45.398 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:06:45.398 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:06:45.398 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:06:45.398 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:06:45.398 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:06:45.398 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:06:45.398 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:06:45.398 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:06:45.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:06:45.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:06:45.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:06:45.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:06:45.398 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:06:45.398 | INFO | ------------------------------------------ | 20260507 05:06:45.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:06:45.398 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:06:45.398 | INFO | Total mem usage: 25.6 GiB | 20260507 05:06:45.398 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:06:45.398 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:06:45.398 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:06:45.399 | INFO | High Water Breakdown: | 20260507 05:06:45.399 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:06:45.399 | INFO | ------------------------------------------ | 20260507 05:06:45.399 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:06:45.399 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:06:45.399 | INFO | Total mem usage: 15.7 GiB | 20260507 05:06:45.399 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:06:45.399 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:06:45.399 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:06:45.399 | INFO | High Water Breakdown: | 20260507 05:06:45.399 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:06:45.399 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:06:45.399 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:06:45.399 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:06:45.399 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:06:45.399 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:06:45.399 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:06:45.399 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:06:45.399 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:06:45.399 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:06:45.399 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:06:45.399 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:06:45.399 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:06:45.399 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:06:45.399 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:06:45.399 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:06:45.399 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:06:45.399 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:06:45.399 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:06:45.399 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:06:45.399 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:06:45.399 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:06:45.399 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:06:45.399 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:06:45.399 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:06:45.399 | INFO | ------------------------------------------ | 20260507 05:06:45.399 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:06:45.399 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:06:45.399 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:06:45.399 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:06:45.399 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 9.87e-01 | 20260507 05:06:45.399 | INFO | PerfReport: 0.29%, AggregateStats -- 9.02e-02 / 9.86e-02, 9.15e-03 / 5.51e-03 | 20260507 05:06:45.399 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.42e+00, 5.02e-01 / 3.41e-01 | 20260507 05:06:45.399 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.46e+00, 1.65e-01 / 1.14e-01 | 20260507 05:06:45.399 | INFO | PerfReport: 16.15%, DME -- 3.97e+01 / 4.44e+01, 3.87e+00 / 1.41e+00 | 20260507 05:06:45.399 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.77e+01 / 2.00e+01, 1.28e+00 / 1.52e+00 | 20260507 05:06:45.399 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.70e+00, 3.82e-01 / 2.05e-01 | 20260507 05:06:45.399 | INFO | PerfReport: 3.00%, Metrics -- 9.22e-01 / 1.25e+00, 2.45e-01 / 1.05e-01 | 20260507 05:06:45.399 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.11e-01 / 3.63e-01, 5.00e-02 / 1.79e-02 | 20260507 05:06:45.399 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:06:45.399 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:06:45.399 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:06:45.399 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 05:06:45.399 | INFO | PerfReport: 21.85%, Upload -- 2.10e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260507 05:06:45.399 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 05:06:45.399 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 05:06:45.399 | INFO | PerfReport: 1.61%, LockUpload -- 1.55e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260507 05:06:45.399 | INFO | PerfReport: 1.22%, LockDownload -- 1.17e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260507 05:06:45.399 | INFO | PerfReport: 22.01%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 05:06:45.399 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260507 05:06:45.495 | INFO | Starting a round of Baz IO | 20260507 05:06:48.587 | INFO | Populating RTMetrics report | 20260507 05:06:48.587 | INFO | RTMetrics report populated | 20260507 05:06:48.587 | INFO | Opening tmp RTMetrics json file | 20260507 05:06:48.605 | INFO | Serializing RTMetrics to json | 20260507 05:06:48.607 | INFO | Serialization done, swapping tmp file now... | 20260507 05:06:48.607 | INFO | RTMetrics done | 20260507 05:06:48.672 | INFO | Finished a round of Baz IO: Wrote 1.705 GiB in 3.177 seconds | 20260507 05:06:48.672 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.73% / 0.00% / 67.27% | 20260507 05:06:48.672 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.151996% / 0% / 7.15256e-05% | 20260507 05:06:48.672 | INFO | Baz IO overall: | 20260507 05:06:48.672 | INFO | BytesWritten : 3723430495732 | 20260507 05:06:48.672 | INFO | NumEvents : 1786503931752 | 20260507 05:06:50.528 | INFO | Analyzing chunk frames = [7323648,7324160) | 20260507 05:06:50.528 | INFO | MemInfo: { | 20260507 05:06:50.528 | INFO | "gpu" : " 17 GB", | 20260507 05:06:50.528 | INFO | "huge" : "", | 20260507 05:06:50.528 | INFO | "huge_shared" : " 27 GB", | 20260507 05:06:50.528 | INFO | "malloc" : " 14 GB", | 20260507 05:06:50.528 | INFO | "pinned" : " 29 GB" | 20260507 05:06:50.528 | INFO | } | 20260507 05:06:50.528 | INFO | Perf report for the last 5.1304 seconds: | 20260507 05:06:50.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:06:50.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:50.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:50.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:50.528 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 05:06:50.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:50.528 | INFO | RT_METRICS : 41.5%, 1.0, 21.7ms, 0 | 20260507 05:06:50.528 | INFO | PRE_HQ : 58.9%, 1.8, 54.8ms, 0 | 20260507 05:06:50.528 | INFO | BAZWRITER : 61.5%, 3.2, 100.6ms, 0 | 20260507 05:06:50.528 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:06:50.528 | INFO | Uploaded 36GiB to GPU in 2.06835s (17.4052GiB/s) | 20260507 05:06:50.528 | INFO | Downloaded 38GiB to GPU in 2.17191s (17.4961GiB/s) | 20260507 05:06:55.660 | INFO | Analyzing chunk frames = [7324160,7324672) | 20260507 05:06:55.660 | INFO | MemInfo: { | 20260507 05:06:55.660 | INFO | "gpu" : " 17 GB", | 20260507 05:06:55.660 | INFO | "huge" : "", | 20260507 05:06:55.660 | INFO | "huge_shared" : " 27 GB", | 20260507 05:06:55.660 | INFO | "malloc" : " 14 GB", | 20260507 05:06:55.660 | INFO | "pinned" : " 29 GB" | 20260507 05:06:55.660 | INFO | } | 20260507 05:06:55.661 | INFO | Perf report for the last 5.13272 seconds: | 20260507 05:06:55.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:06:55.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:55.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:55.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:55.661 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 05:06:55.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:55.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:55.661 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:06:55.661 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:55.661 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:06:55.661 | INFO | Uploaded 36GiB to GPU in 2.00303s (17.9728GiB/s) | 20260507 05:06:55.661 | INFO | Downloaded 34GiB to GPU in 1.93344s (17.5852GiB/s) | 20260507 05:07:00.805 | INFO | Analyzing chunk frames = [7324672,7325184) | 20260507 05:07:00.805 | INFO | MemInfo: { | 20260507 05:07:00.805 | INFO | "gpu" : " 17 GB", | 20260507 05:07:00.805 | INFO | "huge" : "", | 20260507 05:07:00.805 | INFO | "huge_shared" : " 27 GB", | 20260507 05:07:00.805 | INFO | "malloc" : " 14 GB", | 20260507 05:07:00.805 | INFO | "pinned" : " 29 GB" | 20260507 05:07:00.805 | INFO | } | 20260507 05:07:00.805 | INFO | Perf report for the last 5.14462 seconds: | 20260507 05:07:00.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:07:00.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:00.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:00.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:00.805 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 05:07:00.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:00.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:00.805 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:07:00.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:00.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:00.806 | INFO | Uploaded 36GiB to GPU in 2.00563s (17.9495GiB/s) | 20260507 05:07:00.806 | INFO | Downloaded 34GiB to GPU in 1.93825s (17.5416GiB/s) | 20260507 05:07:05.948 | INFO | Analyzing chunk frames = [7325184,7325696) | 20260507 05:07:05.948 | INFO | MemInfo: { | 20260507 05:07:05.948 | INFO | "gpu" : " 17 GB", | 20260507 05:07:05.948 | INFO | "huge" : "", | 20260507 05:07:05.948 | INFO | "huge_shared" : " 27 GB", | 20260507 05:07:05.948 | INFO | "malloc" : " 14 GB", | 20260507 05:07:05.948 | INFO | "pinned" : " 29 GB" | 20260507 05:07:05.948 | INFO | } | 20260507 05:07:05.948 | INFO | Perf report for the last 5.14294 seconds: | 20260507 05:07:05.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:07:05.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:05.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:05.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:05.948 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 05:07:05.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:05.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:05.948 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:07:05.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:05.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:05.949 | INFO | Uploaded 36GiB to GPU in 2.00398s (17.9642GiB/s) | 20260507 05:07:05.949 | INFO | Downloaded 34GiB to GPU in 1.93106s (17.6069GiB/s) | 20260507 05:07:11.085 | INFO | Analyzing chunk frames = [7325696,7326208) | 20260507 05:07:11.085 | INFO | MemInfo: { | 20260507 05:07:11.085 | INFO | "gpu" : " 17 GB", | 20260507 05:07:11.085 | INFO | "huge" : "", | 20260507 05:07:11.085 | INFO | "huge_shared" : " 27 GB", | 20260507 05:07:11.085 | INFO | "malloc" : " 14 GB", | 20260507 05:07:11.085 | INFO | "pinned" : " 29 GB" | 20260507 05:07:11.085 | INFO | } | 20260507 05:07:11.085 | INFO | Perf report for the last 5.13688 seconds: | 20260507 05:07:11.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:07:11.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:11.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:11.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:11.085 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 05:07:11.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:11.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:11.085 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:07:11.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:11.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:11.085 | INFO | Uploaded 36GiB to GPU in 2.05804s (17.4923GiB/s) | 20260507 05:07:11.085 | INFO | Downloaded 34GiB to GPU in 1.94809s (17.453GiB/s) | 20260507 05:07:16.225 | INFO | Analyzing chunk frames = [7326208,7326720) | 20260507 05:07:16.225 | INFO | MemInfo: { | 20260507 05:07:16.225 | INFO | "gpu" : " 17 GB", | 20260507 05:07:16.225 | INFO | "huge" : "", | 20260507 05:07:16.225 | INFO | "huge_shared" : " 27 GB", | 20260507 05:07:16.225 | INFO | "malloc" : " 14 GB", | 20260507 05:07:16.225 | INFO | "pinned" : " 29 GB" | 20260507 05:07:16.225 | INFO | } | 20260507 05:07:16.225 | INFO | Perf report for the last 5.14028 seconds: | 20260507 05:07:16.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:07:16.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:16.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:16.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:16.226 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 05:07:16.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:16.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:16.226 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 05:07:16.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:16.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:16.226 | INFO | Uploaded 36GiB to GPU in 2.07098s (17.3831GiB/s) | 20260507 05:07:16.226 | INFO | Downloaded 34GiB to GPU in 1.92837s (17.6315GiB/s) | 20260507 05:07:21.360 | INFO | Analyzing chunk frames = [7326720,7327232) | 20260507 05:07:21.360 | INFO | MemInfo: { | 20260507 05:07:21.360 | INFO | "gpu" : " 17 GB", | 20260507 05:07:21.360 | INFO | "huge" : "", | 20260507 05:07:21.360 | INFO | "huge_shared" : " 27 GB", | 20260507 05:07:21.360 | INFO | "malloc" : " 14 GB", | 20260507 05:07:21.360 | INFO | "pinned" : " 29 GB" | 20260507 05:07:21.360 | INFO | } | 20260507 05:07:21.360 | INFO | Perf report for the last 5.13441 seconds: | 20260507 05:07:21.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:07:21.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:21.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:21.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:21.360 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 05:07:21.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:21.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:21.360 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:07:21.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:21.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:21.360 | INFO | Uploaded 36GiB to GPU in 1.97268s (18.2492GiB/s) | 20260507 05:07:21.360 | INFO | Downloaded 34GiB to GPU in 1.97995s (17.1722GiB/s) | 20260507 05:07:26.504 | INFO | Analyzing chunk frames = [7327232,7327744) | 20260507 05:07:26.504 | INFO | MemInfo: { | 20260507 05:07:26.504 | INFO | "gpu" : " 17 GB", | 20260507 05:07:26.504 | INFO | "huge" : "", | 20260507 05:07:26.504 | INFO | "huge_shared" : " 27 GB", | 20260507 05:07:26.504 | INFO | "malloc" : " 14 GB", | 20260507 05:07:26.504 | INFO | "pinned" : " 29 GB" | 20260507 05:07:26.504 | INFO | } | 20260507 05:07:26.504 | INFO | Perf report for the last 5.1442 seconds: | 20260507 05:07:26.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:07:26.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:26.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:26.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:26.504 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 05:07:26.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:26.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:26.504 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:07:26.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:26.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:26.505 | INFO | Uploaded 36GiB to GPU in 1.95465s (18.4176GiB/s) | 20260507 05:07:26.505 | INFO | Downloaded 34GiB to GPU in 1.96177s (17.3313GiB/s) | 20260507 05:07:26.601 | INFO | Starting a round of Baz IO | 20260507 05:07:29.700 | INFO | Populating RTMetrics report | 20260507 05:07:29.700 | INFO | RTMetrics report populated | 20260507 05:07:29.700 | INFO | Opening tmp RTMetrics json file | 20260507 05:07:29.703 | INFO | Serializing RTMetrics to json | 20260507 05:07:29.718 | INFO | Serialization done, swapping tmp file now... | 20260507 05:07:29.718 | INFO | RTMetrics done | 20260507 05:07:29.787 | INFO | Finished a round of Baz IO: Wrote 1.703 GiB in 3.186 seconds | 20260507 05:07:29.787 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.73% / 0.00% / 67.27% | 20260507 05:07:29.787 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.151245% / 1.19209e-05% / 8.74201e-05% | 20260507 05:07:29.787 | INFO | Baz IO overall: | 20260507 05:07:29.787 | INFO | BytesWritten : 3725258910155 | 20260507 05:07:29.788 | INFO | NumEvents : 1787362381822 | 20260507 05:07:31.640 | INFO | Analyzing chunk frames = [7327744,7328256) | 20260507 05:07:31.640 | INFO | MemInfo: { | 20260507 05:07:31.640 | INFO | "gpu" : " 17 GB", | 20260507 05:07:31.640 | INFO | "huge" : "", | 20260507 05:07:31.640 | INFO | "huge_shared" : " 27 GB", | 20260507 05:07:31.640 | INFO | "malloc" : " 14 GB", | 20260507 05:07:31.640 | INFO | "pinned" : " 29 GB" | 20260507 05:07:31.640 | INFO | } | 20260507 05:07:31.640 | INFO | Perf report for the last 5.13602 seconds: | 20260507 05:07:31.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:07:31.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:31.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:31.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:31.640 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 05:07:31.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:31.640 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 05:07:31.640 | INFO | PRE_HQ : 59.1%, 1.8, 55.1ms, 0 | 20260507 05:07:31.640 | INFO | BAZWRITER : 61.6%, 3.2, 100.2ms, 0 | 20260507 05:07:31.640 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:07:31.640 | INFO | Uploaded 36GiB to GPU in 2.03404s (17.6988GiB/s) | 20260507 05:07:31.641 | INFO | Downloaded 38GiB to GPU in 2.18694s (17.3759GiB/s) | 20260507 05:07:36.778 | INFO | Analyzing chunk frames = [7328256,7328768) | 20260507 05:07:36.778 | INFO | MemInfo: { | 20260507 05:07:36.778 | INFO | "gpu" : " 17 GB", | 20260507 05:07:36.778 | INFO | "huge" : "", | 20260507 05:07:36.778 | INFO | "huge_shared" : " 27 GB", | 20260507 05:07:36.778 | INFO | "malloc" : " 14 GB", | 20260507 05:07:36.778 | INFO | "pinned" : " 29 GB" | 20260507 05:07:36.778 | INFO | } | 20260507 05:07:36.778 | INFO | Perf report for the last 5.1384 seconds: | 20260507 05:07:36.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:07:36.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:36.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:36.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:36.778 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:07:36.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:36.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:36.779 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 05:07:36.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:36.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:36.779 | INFO | Uploaded 36GiB to GPU in 1.98337s (18.1509GiB/s) | 20260507 05:07:36.779 | INFO | Downloaded 34GiB to GPU in 1.93881s (17.5365GiB/s) | 20260507 05:07:41.916 | INFO | Analyzing chunk frames = [7328768,7329280) | 20260507 05:07:41.916 | INFO | MemInfo: { | 20260507 05:07:41.916 | INFO | "gpu" : " 17 GB", | 20260507 05:07:41.916 | INFO | "huge" : "", | 20260507 05:07:41.916 | INFO | "huge_shared" : " 27 GB", | 20260507 05:07:41.916 | INFO | "malloc" : " 14 GB", | 20260507 05:07:41.916 | INFO | "pinned" : " 29 GB" | 20260507 05:07:41.916 | INFO | } | 20260507 05:07:41.916 | INFO | Perf report for the last 5.13793 seconds: | 20260507 05:07:41.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:07:41.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:41.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:41.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:41.917 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 05:07:41.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:41.917 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:41.917 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:07:41.917 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:41.917 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:41.917 | INFO | Uploaded 36GiB to GPU in 2.01058s (17.9053GiB/s) | 20260507 05:07:41.917 | INFO | Downloaded 34GiB to GPU in 1.94502s (17.4805GiB/s) | 20260507 05:07:47.064 | INFO | Analyzing chunk frames = [7329280,7329792) | 20260507 05:07:47.064 | INFO | MemInfo: { | 20260507 05:07:47.064 | INFO | "gpu" : " 17 GB", | 20260507 05:07:47.064 | INFO | "huge" : "", | 20260507 05:07:47.064 | INFO | "huge_shared" : " 27 GB", | 20260507 05:07:47.064 | INFO | "malloc" : " 14 GB", | 20260507 05:07:47.064 | INFO | "pinned" : " 29 GB" | 20260507 05:07:47.064 | INFO | } | 20260507 05:07:47.064 | INFO | Perf report for the last 5.14742 seconds: | 20260507 05:07:47.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:07:47.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:47.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:47.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:47.064 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 05:07:47.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:47.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:47.064 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 05:07:47.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:47.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:47.064 | INFO | Uploaded 36GiB to GPU in 1.98944s (18.0955GiB/s) | 20260507 05:07:47.064 | INFO | Downloaded 34GiB to GPU in 1.92372s (17.6741GiB/s) | 20260507 05:07:52.204 | INFO | Analyzing chunk frames = [7329792,7330304) | 20260507 05:07:52.204 | INFO | MemInfo: { | 20260507 05:07:52.204 | INFO | "gpu" : " 17 GB", | 20260507 05:07:52.204 | INFO | "huge" : "", | 20260507 05:07:52.204 | INFO | "huge_shared" : " 27 GB", | 20260507 05:07:52.204 | INFO | "malloc" : " 14 GB", | 20260507 05:07:52.204 | INFO | "pinned" : " 29 GB" | 20260507 05:07:52.204 | INFO | } | 20260507 05:07:52.204 | INFO | Perf report for the last 5.14047 seconds: | 20260507 05:07:52.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:07:52.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:52.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:52.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:52.204 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 05:07:52.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:52.204 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:52.204 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:07:52.204 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:52.204 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:52.205 | INFO | Uploaded 36GiB to GPU in 2.04932s (17.5668GiB/s) | 20260507 05:07:52.205 | INFO | Downloaded 34GiB to GPU in 1.95033s (17.433GiB/s) | 20260507 05:07:57.339 | INFO | Analyzing chunk frames = [7330304,7330816) | 20260507 05:07:57.339 | INFO | MemInfo: { | 20260507 05:07:57.339 | INFO | "gpu" : " 17 GB", | 20260507 05:07:57.339 | INFO | "huge" : "", | 20260507 05:07:57.339 | INFO | "huge_shared" : " 27 GB", | 20260507 05:07:57.339 | INFO | "malloc" : " 14 GB", | 20260507 05:07:57.339 | INFO | "pinned" : " 29 GB" | 20260507 05:07:57.339 | INFO | } | 20260507 05:07:57.340 | INFO | Perf report for the last 5.13528 seconds: | 20260507 05:07:57.340 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:07:57.340 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:57.340 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:57.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:57.340 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 05:07:57.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:57.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:57.340 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 05:07:57.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:57.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:07:57.340 | INFO | Uploaded 36GiB to GPU in 2.06635s (17.4221GiB/s) | 20260507 05:07:57.340 | INFO | Downloaded 34GiB to GPU in 1.94129s (17.5141GiB/s) | 20260507 05:08:02.476 | INFO | Analyzing chunk frames = [7330816,7331328) | 20260507 05:08:02.476 | INFO | MemInfo: { | 20260507 05:08:02.476 | INFO | "gpu" : " 17 GB", | 20260507 05:08:02.476 | INFO | "huge" : "", | 20260507 05:08:02.476 | INFO | "huge_shared" : " 27 GB", | 20260507 05:08:02.476 | INFO | "malloc" : " 14 GB", | 20260507 05:08:02.476 | INFO | "pinned" : " 29 GB" | 20260507 05:08:02.476 | INFO | } | 20260507 05:08:02.476 | INFO | Perf report for the last 5.13621 seconds: | 20260507 05:08:02.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:08:02.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:02.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:02.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:02.476 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:08:02.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:02.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:02.476 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:08:02.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:02.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:02.476 | INFO | Uploaded 36GiB to GPU in 1.9148s (18.8009GiB/s) | 20260507 05:08:02.476 | INFO | Downloaded 34GiB to GPU in 1.9794s (17.177GiB/s) | 20260507 05:08:07.618 | INFO | Analyzing chunk frames = [7331328,7331840) | 20260507 05:08:07.618 | INFO | MemInfo: { | 20260507 05:08:07.618 | INFO | "gpu" : " 17 GB", | 20260507 05:08:07.618 | INFO | "huge" : "", | 20260507 05:08:07.618 | INFO | "huge_shared" : " 27 GB", | 20260507 05:08:07.618 | INFO | "malloc" : " 14 GB", | 20260507 05:08:07.618 | INFO | "pinned" : " 29 GB" | 20260507 05:08:07.618 | INFO | } | 20260507 05:08:07.618 | INFO | Perf report for the last 5.14233 seconds: | 20260507 05:08:07.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:08:07.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:07.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:07.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:07.618 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 05:08:07.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:07.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:07.618 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 05:08:07.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:07.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:07.618 | INFO | Uploaded 36GiB to GPU in 1.93976s (18.559GiB/s) | 20260507 05:08:07.618 | INFO | Downloaded 34GiB to GPU in 1.95286s (17.4103GiB/s) | 20260507 05:08:07.619 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:08:07.619 | INFO | Total mem usage: 18 GiB | 20260507 05:08:07.619 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:08:07.619 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 05:08:07.619 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:08:07.619 | INFO | High Water Breakdown: | 20260507 05:08:07.619 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:08:07.619 | INFO | 39.7% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:08:07.619 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:08:07.619 | INFO | ------------------------------------------ | 20260507 05:08:07.619 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:08:07.619 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:08:07.619 | INFO | Total mem usage: 26.5 GiB | 20260507 05:08:07.619 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:08:07.619 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:08:07.619 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 05:08:07.619 | INFO | High Water Breakdown: | 20260507 05:08:07.619 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:08:07.619 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:08:07.619 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:08:07.619 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:08:07.619 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:08:07.619 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:08:07.619 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:08:07.619 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:08:07.619 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:08:07.619 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:08:07.619 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:08:07.619 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:08:07.619 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:08:07.619 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:08:07.619 | INFO | ------------------------------------------ | 20260507 05:08:07.619 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:08:07.619 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:08:07.619 | INFO | Total mem usage: 25.6 GiB | 20260507 05:08:07.619 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:08:07.619 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:08:07.619 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:08:07.619 | INFO | High Water Breakdown: | 20260507 05:08:07.619 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:08:07.619 | INFO | ------------------------------------------ | 20260507 05:08:07.619 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:08:07.619 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:08:07.619 | INFO | Total mem usage: 15.7 GiB | 20260507 05:08:07.619 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:08:07.619 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:08:07.619 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:08:07.619 | INFO | High Water Breakdown: | 20260507 05:08:07.619 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:08:07.619 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:08:07.619 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:08:07.619 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:08:07.619 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:08:07.619 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:08:07.619 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:08:07.619 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:08:07.619 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:08:07.619 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:08:07.619 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:08:07.619 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:08:07.619 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:08:07.619 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:08:07.619 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:08:07.619 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:08:07.619 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:08:07.619 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:08:07.619 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:08:07.619 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:08:07.619 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:08:07.620 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:08:07.620 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:08:07.620 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:08:07.620 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:08:07.620 | INFO | ------------------------------------------ | 20260507 05:08:07.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:08:07.620 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:08:07.620 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:08:07.620 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:08:07.620 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.35e+01 / 1.22e+00 | 20260507 05:08:07.620 | INFO | PerfReport: 0.30%, AggregateStats -- 9.06e-02 / 1.02e-01, 9.79e-03 / 6.86e-03 | 20260507 05:08:07.620 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.42e+00, 5.08e-01 / 3.34e-01 | 20260507 05:08:07.620 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.44e+00, 1.67e-01 / 1.31e-01 | 20260507 05:08:07.620 | INFO | PerfReport: 16.11%, DME -- 3.96e+01 / 4.40e+01, 3.80e+00 / 1.24e+00 | 20260507 05:08:07.620 | INFO | PerfReport: 57.74%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.33e+00 / 1.47e+00 | 20260507 05:08:07.620 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.67e+00, 3.85e-01 / 2.18e-01 | 20260507 05:08:07.620 | INFO | PerfReport: 3.00%, Metrics -- 9.22e-01 / 1.23e+00, 2.41e-01 / 1.38e-01 | 20260507 05:08:07.620 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.11e-01 / 3.66e-01, 4.91e-02 / 1.91e-02 | 20260507 05:08:07.620 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:08:07.620 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:08:07.620 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:08:07.620 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 05:08:07.620 | INFO | PerfReport: 21.76%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260507 05:08:07.620 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 05:08:07.620 | INFO | PerfReport: 31.97%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 05:08:07.620 | INFO | PerfReport: 1.44%, LockUpload -- 1.38e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 05:08:07.620 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260507 05:08:07.620 | INFO | PerfReport: 22.10%, LockCompute -- 2.12e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 05:08:07.620 | INFO | PerfReport: 0.02%, Misc -- 1.45e-02 / 0.00e+00, 2.45e-02 / 0.00e+00 | 20260507 05:08:07.715 | INFO | Starting a round of Baz IO | 20260507 05:08:10.815 | INFO | Populating RTMetrics report | 20260507 05:08:10.816 | INFO | RTMetrics report populated | 20260507 05:08:10.816 | INFO | Opening tmp RTMetrics json file | 20260507 05:08:10.833 | INFO | Serializing RTMetrics to json | 20260507 05:08:10.834 | INFO | Serialization done, swapping tmp file now... | 20260507 05:08:10.835 | INFO | RTMetrics done | 20260507 05:08:10.902 | INFO | Finished a round of Baz IO: Wrote 1.704 GiB in 3.187 seconds | 20260507 05:08:10.903 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.73% / 0.00% / 67.27% | 20260507 05:08:10.903 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.150959% / 0% / 7.54992e-05% | 20260507 05:08:10.903 | INFO | Baz IO overall: | 20260507 05:08:10.903 | INFO | BytesWritten : 3727088605700 | 20260507 05:08:10.903 | INFO | NumEvents : 1788221906744 | 20260507 05:08:12.758 | INFO | Analyzing chunk frames = [7331840,7332352) | 20260507 05:08:12.758 | INFO | MemInfo: { | 20260507 05:08:12.758 | INFO | "gpu" : " 17 GB", | 20260507 05:08:12.758 | INFO | "huge" : "", | 20260507 05:08:12.758 | INFO | "huge_shared" : " 27 GB", | 20260507 05:08:12.758 | INFO | "malloc" : " 14 GB", | 20260507 05:08:12.758 | INFO | "pinned" : " 29 GB" | 20260507 05:08:12.758 | INFO | } | 20260507 05:08:12.758 | INFO | Perf report for the last 5.14007 seconds: | 20260507 05:08:12.758 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:08:12.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:12.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:12.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:12.759 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 05:08:12.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:12.759 | INFO | RT_METRICS : 41.5%, 1.0, 21.7ms, 0 | 20260507 05:08:12.759 | INFO | PRE_HQ : 59.1%, 1.8, 55.1ms, 0 | 20260507 05:08:12.759 | INFO | BAZWRITER : 61.6%, 3.2, 100.7ms, 0 | 20260507 05:08:12.759 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 05:08:12.759 | INFO | Uploaded 36GiB to GPU in 2.02869s (17.7454GiB/s) | 20260507 05:08:12.759 | INFO | Downloaded 38GiB to GPU in 2.18592s (17.384GiB/s) | 20260507 05:08:17.903 | INFO | Analyzing chunk frames = [7332352,7332864) | 20260507 05:08:17.903 | INFO | MemInfo: { | 20260507 05:08:17.903 | INFO | "gpu" : " 17 GB", | 20260507 05:08:17.903 | INFO | "huge" : "", | 20260507 05:08:17.903 | INFO | "huge_shared" : " 27 GB", | 20260507 05:08:17.903 | INFO | "malloc" : " 14 GB", | 20260507 05:08:17.903 | INFO | "pinned" : " 29 GB" | 20260507 05:08:17.903 | INFO | } | 20260507 05:08:17.903 | INFO | Perf report for the last 5.14505 seconds: | 20260507 05:08:17.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:08:17.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:17.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:17.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:17.904 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 05:08:17.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:17.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:17.904 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:08:17.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:17.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:17.904 | INFO | Uploaded 36GiB to GPU in 1.99032s (18.0876GiB/s) | 20260507 05:08:17.904 | INFO | Downloaded 34GiB to GPU in 1.93333s (17.5863GiB/s) | 20260507 05:08:23.038 | INFO | Analyzing chunk frames = [7332864,7333376) | 20260507 05:08:23.038 | INFO | MemInfo: { | 20260507 05:08:23.038 | INFO | "gpu" : " 17 GB", | 20260507 05:08:23.038 | INFO | "huge" : "", | 20260507 05:08:23.038 | INFO | "huge_shared" : " 27 GB", | 20260507 05:08:23.038 | INFO | "malloc" : " 14 GB", | 20260507 05:08:23.039 | INFO | "pinned" : " 29 GB" | 20260507 05:08:23.039 | INFO | } | 20260507 05:08:23.039 | INFO | Perf report for the last 5.13532 seconds: | 20260507 05:08:23.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:08:23.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:23.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:23.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:23.039 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 05:08:23.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:23.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:23.039 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:08:23.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:23.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:23.039 | INFO | Uploaded 36GiB to GPU in 2.04799s (17.5783GiB/s) | 20260507 05:08:23.039 | INFO | Downloaded 34GiB to GPU in 1.95313s (17.408GiB/s) | 20260507 05:08:28.172 | INFO | Analyzing chunk frames = [7333376,7333888) | 20260507 05:08:28.172 | INFO | MemInfo: { | 20260507 05:08:28.172 | INFO | "gpu" : " 17 GB", | 20260507 05:08:28.172 | INFO | "huge" : "", | 20260507 05:08:28.172 | INFO | "huge_shared" : " 27 GB", | 20260507 05:08:28.172 | INFO | "malloc" : " 14 GB", | 20260507 05:08:28.172 | INFO | "pinned" : " 29 GB" | 20260507 05:08:28.172 | INFO | } | 20260507 05:08:28.172 | INFO | Perf report for the last 5.13287 seconds: | 20260507 05:08:28.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:08:28.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:28.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:28.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:28.172 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 05:08:28.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:28.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:28.172 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:08:28.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:28.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:28.172 | INFO | Uploaded 36GiB to GPU in 2.01233s (17.8897GiB/s) | 20260507 05:08:28.172 | INFO | Downloaded 34GiB to GPU in 1.94567s (17.4747GiB/s) | 20260507 05:08:33.314 | INFO | Analyzing chunk frames = [7333888,7334400) | 20260507 05:08:33.314 | INFO | MemInfo: { | 20260507 05:08:33.314 | INFO | "gpu" : " 17 GB", | 20260507 05:08:33.314 | INFO | "huge" : "", | 20260507 05:08:33.314 | INFO | "huge_shared" : " 27 GB", | 20260507 05:08:33.314 | INFO | "malloc" : " 14 GB", | 20260507 05:08:33.314 | INFO | "pinned" : " 29 GB" | 20260507 05:08:33.314 | INFO | } | 20260507 05:08:33.315 | INFO | Perf report for the last 5.14304 seconds: | 20260507 05:08:33.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:08:33.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:33.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:33.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:33.315 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 05:08:33.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:33.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:33.315 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 05:08:33.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:33.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:33.315 | INFO | Uploaded 36GiB to GPU in 2.04364s (17.6157GiB/s) | 20260507 05:08:33.315 | INFO | Downloaded 34GiB to GPU in 1.94788s (17.4549GiB/s) | 20260507 05:08:38.452 | INFO | Analyzing chunk frames = [7334400,7334912) | 20260507 05:08:38.453 | INFO | MemInfo: { | 20260507 05:08:38.453 | INFO | "gpu" : " 17 GB", | 20260507 05:08:38.453 | INFO | "huge" : "", | 20260507 05:08:38.453 | INFO | "huge_shared" : " 27 GB", | 20260507 05:08:38.453 | INFO | "malloc" : " 14 GB", | 20260507 05:08:38.453 | INFO | "pinned" : " 29 GB" | 20260507 05:08:38.453 | INFO | } | 20260507 05:08:38.453 | INFO | Perf report for the last 5.13816 seconds: | 20260507 05:08:38.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:08:38.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:38.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:38.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:38.453 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 05:08:38.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:38.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:38.453 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:08:38.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:38.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:38.453 | INFO | Uploaded 36GiB to GPU in 2.03121s (17.7234GiB/s) | 20260507 05:08:38.453 | INFO | Downloaded 34GiB to GPU in 1.91596s (17.7457GiB/s) | 20260507 05:08:43.594 | INFO | Analyzing chunk frames = [7334912,7335424) | 20260507 05:08:43.594 | INFO | MemInfo: { | 20260507 05:08:43.594 | INFO | "gpu" : " 17 GB", | 20260507 05:08:43.594 | INFO | "huge" : "", | 20260507 05:08:43.594 | INFO | "huge_shared" : " 27 GB", | 20260507 05:08:43.594 | INFO | "malloc" : " 14 GB", | 20260507 05:08:43.594 | INFO | "pinned" : " 29 GB" | 20260507 05:08:43.594 | INFO | } | 20260507 05:08:43.594 | INFO | Perf report for the last 5.14154 seconds: | 20260507 05:08:43.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:08:43.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:43.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:43.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:43.594 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 05:08:43.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:43.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:43.594 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 05:08:43.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:43.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:43.595 | INFO | Uploaded 36GiB to GPU in 2.01955s (17.8258GiB/s) | 20260507 05:08:43.595 | INFO | Downloaded 34GiB to GPU in 1.97849s (17.1848GiB/s) | 20260507 05:08:48.727 | INFO | Analyzing chunk frames = [7335424,7335936) | 20260507 05:08:48.727 | INFO | MemInfo: { | 20260507 05:08:48.727 | INFO | "gpu" : " 17 GB", | 20260507 05:08:48.727 | INFO | "huge" : "", | 20260507 05:08:48.727 | INFO | "huge_shared" : " 27 GB", | 20260507 05:08:48.727 | INFO | "malloc" : " 14 GB", | 20260507 05:08:48.728 | INFO | "pinned" : " 29 GB" | 20260507 05:08:48.728 | INFO | } | 20260507 05:08:48.728 | INFO | Perf report for the last 5.13321 seconds: | 20260507 05:08:48.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:08:48.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:48.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:48.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:48.728 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:08:48.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:48.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:48.728 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260507 05:08:48.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:48.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:48.728 | INFO | Uploaded 36GiB to GPU in 1.98404s (18.1448GiB/s) | 20260507 05:08:48.728 | INFO | Downloaded 34GiB to GPU in 1.96856s (17.2715GiB/s) | 20260507 05:08:48.825 | INFO | Starting a round of Baz IO | 20260507 05:08:51.927 | INFO | Populating RTMetrics report | 20260507 05:08:51.927 | INFO | RTMetrics report populated | 20260507 05:08:51.927 | INFO | Opening tmp RTMetrics json file | 20260507 05:08:51.936 | INFO | Serializing RTMetrics to json | 20260507 05:08:51.943 | INFO | Serialization done, swapping tmp file now... | 20260507 05:08:51.944 | INFO | RTMetrics done | 20260507 05:08:52.015 | INFO | Finished a round of Baz IO: Wrote 1.705 GiB in 3.19 seconds | 20260507 05:08:52.016 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.73% / 0.00% / 67.27% | 20260507 05:08:52.016 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.150975% / 0% / 6.75519e-05% | 20260507 05:08:52.016 | INFO | Baz IO overall: | 20260507 05:08:52.016 | INFO | BytesWritten : 3728919165588 | 20260507 05:08:52.016 | INFO | NumEvents : 1789082026810 | 20260507 05:08:53.869 | INFO | Analyzing chunk frames = [7335936,7336448) | 20260507 05:08:53.869 | INFO | MemInfo: { | 20260507 05:08:53.869 | INFO | "gpu" : " 17 GB", | 20260507 05:08:53.869 | INFO | "huge" : "", | 20260507 05:08:53.869 | INFO | "huge_shared" : " 27 GB", | 20260507 05:08:53.869 | INFO | "malloc" : " 14 GB", | 20260507 05:08:53.869 | INFO | "pinned" : " 29 GB" | 20260507 05:08:53.869 | INFO | } | 20260507 05:08:53.869 | INFO | Perf report for the last 5.14139 seconds: | 20260507 05:08:53.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:08:53.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:53.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:53.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:53.869 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 05:08:53.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:53.869 | INFO | RT_METRICS : 44.4%, 1.0, 23.3ms, 0 | 20260507 05:08:53.869 | INFO | PRE_HQ : 59.7%, 1.9, 59.5ms, 0 | 20260507 05:08:53.869 | INFO | BAZWRITER : 61.6%, 3.3, 104.1ms, 0 | 20260507 05:08:53.869 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:08:53.869 | INFO | Uploaded 36GiB to GPU in 2.0221s (17.8032GiB/s) | 20260507 05:08:53.869 | INFO | Downloaded 38GiB to GPU in 2.18279s (17.4089GiB/s) | 20260507 05:08:59.015 | INFO | Analyzing chunk frames = [7336448,7336960) | 20260507 05:08:59.015 | INFO | MemInfo: { | 20260507 05:08:59.015 | INFO | "gpu" : " 17 GB", | 20260507 05:08:59.015 | INFO | "huge" : "", | 20260507 05:08:59.015 | INFO | "huge_shared" : " 27 GB", | 20260507 05:08:59.015 | INFO | "malloc" : " 14 GB", | 20260507 05:08:59.015 | INFO | "pinned" : " 29 GB" | 20260507 05:08:59.015 | INFO | } | 20260507 05:08:59.015 | INFO | Perf report for the last 5.14646 seconds: | 20260507 05:08:59.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:08:59.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:59.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:59.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:59.015 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 05:08:59.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:59.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:59.016 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 05:08:59.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:59.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:08:59.016 | INFO | Uploaded 36GiB to GPU in 2.04618s (17.5937GiB/s) | 20260507 05:08:59.016 | INFO | Downloaded 34GiB to GPU in 1.94581s (17.4734GiB/s) | 20260507 05:09:04.154 | INFO | Analyzing chunk frames = [7336960,7337472) | 20260507 05:09:04.154 | INFO | MemInfo: { | 20260507 05:09:04.154 | INFO | "gpu" : " 17 GB", | 20260507 05:09:04.154 | INFO | "huge" : "", | 20260507 05:09:04.154 | INFO | "huge_shared" : " 27 GB", | 20260507 05:09:04.154 | INFO | "malloc" : " 14 GB", | 20260507 05:09:04.154 | INFO | "pinned" : " 29 GB" | 20260507 05:09:04.154 | INFO | } | 20260507 05:09:04.154 | INFO | Perf report for the last 5.13866 seconds: | 20260507 05:09:04.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:09:04.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:04.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:04.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:04.154 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 05:09:04.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:04.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:04.154 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:09:04.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:04.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:04.154 | INFO | Uploaded 36GiB to GPU in 2.04356s (17.6163GiB/s) | 20260507 05:09:04.154 | INFO | Downloaded 34GiB to GPU in 1.95097s (17.4272GiB/s) | 20260507 05:09:09.284 | INFO | Analyzing chunk frames = [7337472,7337984) | 20260507 05:09:09.284 | INFO | MemInfo: { | 20260507 05:09:09.284 | INFO | "gpu" : " 17 GB", | 20260507 05:09:09.284 | INFO | "huge" : "", | 20260507 05:09:09.284 | INFO | "huge_shared" : " 27 GB", | 20260507 05:09:09.284 | INFO | "malloc" : " 14 GB", | 20260507 05:09:09.284 | INFO | "pinned" : " 29 GB" | 20260507 05:09:09.284 | INFO | } | 20260507 05:09:09.284 | INFO | Perf report for the last 5.13031 seconds: | 20260507 05:09:09.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:09:09.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:09.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:09.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:09.285 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 05:09:09.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:09.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:09.285 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:09:09.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:09.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:09.285 | INFO | Uploaded 36GiB to GPU in 2.04157s (17.6335GiB/s) | 20260507 05:09:09.285 | INFO | Downloaded 34GiB to GPU in 1.94795s (17.4542GiB/s) | 20260507 05:09:14.427 | INFO | Analyzing chunk frames = [7337984,7338496) | 20260507 05:09:14.427 | INFO | MemInfo: { | 20260507 05:09:14.427 | INFO | "gpu" : " 17 GB", | 20260507 05:09:14.427 | INFO | "huge" : "", | 20260507 05:09:14.427 | INFO | "huge_shared" : " 27 GB", | 20260507 05:09:14.427 | INFO | "malloc" : " 14 GB", | 20260507 05:09:14.427 | INFO | "pinned" : " 29 GB" | 20260507 05:09:14.427 | INFO | } | 20260507 05:09:14.427 | INFO | Perf report for the last 5.14221 seconds: | 20260507 05:09:14.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:09:14.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:14.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:14.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:14.427 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 05:09:14.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:14.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:14.427 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:09:14.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:14.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:14.427 | INFO | Uploaded 36GiB to GPU in 2.04261s (17.6245GiB/s) | 20260507 05:09:14.427 | INFO | Downloaded 34GiB to GPU in 1.94768s (17.4567GiB/s) | 20260507 05:09:19.573 | INFO | Analyzing chunk frames = [7338496,7339008) | 20260507 05:09:19.573 | INFO | MemInfo: { | 20260507 05:09:19.573 | INFO | "gpu" : " 17 GB", | 20260507 05:09:19.573 | INFO | "huge" : "", | 20260507 05:09:19.573 | INFO | "huge_shared" : " 27 GB", | 20260507 05:09:19.573 | INFO | "malloc" : " 14 GB", | 20260507 05:09:19.573 | INFO | "pinned" : " 29 GB" | 20260507 05:09:19.573 | INFO | } | 20260507 05:09:19.573 | INFO | Perf report for the last 5.1465 seconds: | 20260507 05:09:19.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:09:19.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:19.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:19.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:19.573 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 05:09:19.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:19.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:19.573 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:09:19.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:19.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:19.573 | INFO | Uploaded 36GiB to GPU in 2.02839s (17.7481GiB/s) | 20260507 05:09:19.573 | INFO | Downloaded 34GiB to GPU in 1.91825s (17.7244GiB/s) | 20260507 05:09:24.709 | INFO | Analyzing chunk frames = [7339008,7339520) | 20260507 05:09:24.709 | INFO | MemInfo: { | 20260507 05:09:24.709 | INFO | "gpu" : " 17 GB", | 20260507 05:09:24.709 | INFO | "huge" : "", | 20260507 05:09:24.709 | INFO | "huge_shared" : " 27 GB", | 20260507 05:09:24.709 | INFO | "malloc" : " 14 GB", | 20260507 05:09:24.709 | INFO | "pinned" : " 29 GB" | 20260507 05:09:24.709 | INFO | } | 20260507 05:09:24.709 | INFO | Perf report for the last 5.13587 seconds: | 20260507 05:09:24.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:09:24.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:24.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:24.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:24.709 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:09:24.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:24.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:24.709 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:09:24.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:24.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:24.709 | INFO | Uploaded 36GiB to GPU in 1.99339s (18.0597GiB/s) | 20260507 05:09:24.709 | INFO | Downloaded 34GiB to GPU in 1.97504s (17.2149GiB/s) | 20260507 05:09:29.851 | INFO | Analyzing chunk frames = [7339520,7340032) | 20260507 05:09:29.851 | INFO | MemInfo: { | 20260507 05:09:29.851 | INFO | "gpu" : " 17 GB", | 20260507 05:09:29.851 | INFO | "huge" : "", | 20260507 05:09:29.851 | INFO | "huge_shared" : " 27 GB", | 20260507 05:09:29.851 | INFO | "malloc" : " 14 GB", | 20260507 05:09:29.851 | INFO | "pinned" : " 29 GB" | 20260507 05:09:29.851 | INFO | } | 20260507 05:09:29.851 | INFO | Perf report for the last 5.1417 seconds: | 20260507 05:09:29.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:09:29.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:29.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:29.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:29.851 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:09:29.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:29.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:29.851 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 05:09:29.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:29.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:29.851 | INFO | Uploaded 36GiB to GPU in 1.9814s (18.169GiB/s) | 20260507 05:09:29.851 | INFO | Downloaded 34GiB to GPU in 1.95286s (17.4104GiB/s) | 20260507 05:09:29.851 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:09:29.851 | INFO | Total mem usage: 18 GiB | 20260507 05:09:29.851 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:09:29.851 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:09:29.851 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:09:29.851 | INFO | High Water Breakdown: | 20260507 05:09:29.851 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:09:29.851 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:09:29.851 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:09:29.851 | INFO | ------------------------------------------ | 20260507 05:09:29.851 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:09:29.851 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:09:29.851 | INFO | Total mem usage: 26.5 GiB | 20260507 05:09:29.851 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:09:29.851 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:09:29.851 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:09:29.851 | INFO | High Water Breakdown: | 20260507 05:09:29.851 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:09:29.852 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:09:29.852 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:09:29.852 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:09:29.852 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:09:29.852 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:09:29.852 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:09:29.852 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:09:29.852 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:09:29.852 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:09:29.852 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:09:29.852 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:09:29.852 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:09:29.852 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:09:29.852 | INFO | ------------------------------------------ | 20260507 05:09:29.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:09:29.852 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:09:29.852 | INFO | Total mem usage: 25.6 GiB | 20260507 05:09:29.852 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:09:29.852 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:09:29.852 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:09:29.852 | INFO | High Water Breakdown: | 20260507 05:09:29.852 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:09:29.852 | INFO | ------------------------------------------ | 20260507 05:09:29.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:09:29.852 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:09:29.852 | INFO | Total mem usage: 15.7 GiB | 20260507 05:09:29.852 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:09:29.852 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:09:29.852 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 05:09:29.852 | INFO | High Water Breakdown: | 20260507 05:09:29.852 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:09:29.852 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:09:29.852 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:09:29.852 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:09:29.852 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:09:29.852 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:09:29.852 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:09:29.852 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:09:29.852 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:09:29.852 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:09:29.852 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:09:29.852 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:09:29.852 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:09:29.852 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:09:29.852 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:09:29.852 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:09:29.852 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:09:29.852 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:09:29.852 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:09:29.852 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:09:29.852 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:09:29.852 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:09:29.852 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:09:29.852 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:09:29.852 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:09:29.852 | INFO | ------------------------------------------ | 20260507 05:09:29.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:09:29.852 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:09:29.852 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:09:29.852 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:09:29.852 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.33e+01 / 1.31e+00 | 20260507 05:09:29.852 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.00e-01, 9.97e-03 / 6.28e-03 | 20260507 05:09:29.852 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.43e+00, 5.03e-01 / 3.14e-01 | 20260507 05:09:29.852 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.44e+00, 1.63e-01 / 1.33e-01 | 20260507 05:09:29.852 | INFO | PerfReport: 16.25%, DME -- 3.99e+01 / 4.48e+01, 3.95e+00 / 1.46e+00 | 20260507 05:09:29.852 | INFO | PerfReport: 57.59%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.21e+00 / 1.18e+00 | 20260507 05:09:29.852 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.14e+00 / 2.74e+00, 3.87e-01 / 1.60e-01 | 20260507 05:09:29.852 | INFO | PerfReport: 3.00%, Metrics -- 9.21e-01 / 1.24e+00, 2.44e-01 / 1.00e-01 | 20260507 05:09:29.852 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.11e-01 / 3.61e-01, 5.00e-02 / 1.83e-02 | 20260507 05:09:29.852 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:09:29.852 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:09:29.852 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:09:29.852 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 05:09:29.852 | INFO | PerfReport: 21.88%, Upload -- 2.11e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260507 05:09:29.852 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 05:09:29.852 | INFO | PerfReport: 31.90%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 05:09:29.852 | INFO | PerfReport: 1.45%, LockUpload -- 1.40e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 05:09:29.852 | INFO | PerfReport: 1.36%, LockDownload -- 1.30e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260507 05:09:29.852 | INFO | PerfReport: 21.99%, LockCompute -- 2.12e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 05:09:29.852 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260507 05:09:29.950 | INFO | Starting a round of Baz IO | 20260507 05:09:33.043 | INFO | Populating RTMetrics report | 20260507 05:09:33.043 | INFO | RTMetrics report populated | 20260507 05:09:33.043 | INFO | Opening tmp RTMetrics json file | 20260507 05:09:33.061 | INFO | Serializing RTMetrics to json | 20260507 05:09:33.062 | INFO | Serialization done, swapping tmp file now... | 20260507 05:09:33.062 | INFO | RTMetrics done | 20260507 05:09:33.132 | INFO | Finished a round of Baz IO: Wrote 1.706 GiB in 3.181 seconds | 20260507 05:09:33.132 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.73% / 0.00% / 67.27% | 20260507 05:09:33.132 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.15138% / 3.97364e-06% / 8.34465e-05% | 20260507 05:09:33.132 | INFO | Baz IO overall: | 20260507 05:09:33.132 | INFO | BytesWritten : 3730750611447 | 20260507 05:09:33.132 | INFO | NumEvents : 1789942633958 | 20260507 05:09:34.985 | INFO | Analyzing chunk frames = [7340032,7340544) | 20260507 05:09:34.985 | INFO | MemInfo: { | 20260507 05:09:34.985 | INFO | "gpu" : " 17 GB", | 20260507 05:09:34.985 | INFO | "huge" : "", | 20260507 05:09:34.985 | INFO | "huge_shared" : " 27 GB", | 20260507 05:09:34.985 | INFO | "malloc" : " 14 GB", | 20260507 05:09:34.985 | INFO | "pinned" : " 29 GB" | 20260507 05:09:34.985 | INFO | } | 20260507 05:09:34.985 | INFO | Perf report for the last 5.13423 seconds: | 20260507 05:09:34.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:09:34.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:34.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:34.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:34.985 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 05:09:34.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:34.985 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 05:09:34.985 | INFO | PRE_HQ : 59.0%, 1.8, 55.0ms, 0 | 20260507 05:09:34.985 | INFO | BAZWRITER : 61.6%, 3.2, 100.8ms, 0 | 20260507 05:09:34.985 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:09:34.985 | INFO | Uploaded 36GiB to GPU in 2.05784s (17.4941GiB/s) | 20260507 05:09:34.985 | INFO | Downloaded 38GiB to GPU in 2.17717s (17.4539GiB/s) | 20260507 05:09:40.126 | INFO | Analyzing chunk frames = [7340544,7341056) | 20260507 05:09:40.126 | INFO | MemInfo: { | 20260507 05:09:40.126 | INFO | "gpu" : " 17 GB", | 20260507 05:09:40.126 | INFO | "huge" : "", | 20260507 05:09:40.126 | INFO | "huge_shared" : " 27 GB", | 20260507 05:09:40.126 | INFO | "malloc" : " 14 GB", | 20260507 05:09:40.126 | INFO | "pinned" : " 29 GB" | 20260507 05:09:40.126 | INFO | } | 20260507 05:09:40.126 | INFO | Perf report for the last 5.14111 seconds: | 20260507 05:09:40.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:09:40.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:40.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:40.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:40.126 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 05:09:40.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:40.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:40.126 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:09:40.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:40.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:40.126 | INFO | Uploaded 36GiB to GPU in 1.99624s (18.0339GiB/s) | 20260507 05:09:40.126 | INFO | Downloaded 34GiB to GPU in 1.92981s (17.6183GiB/s) | 20260507 05:09:45.268 | INFO | Analyzing chunk frames = [7341056,7341568) | 20260507 05:09:45.268 | INFO | MemInfo: { | 20260507 05:09:45.268 | INFO | "gpu" : " 17 GB", | 20260507 05:09:45.268 | INFO | "huge" : "", | 20260507 05:09:45.268 | INFO | "huge_shared" : " 27 GB", | 20260507 05:09:45.268 | INFO | "malloc" : " 14 GB", | 20260507 05:09:45.268 | INFO | "pinned" : " 29 GB" | 20260507 05:09:45.268 | INFO | } | 20260507 05:09:45.269 | INFO | Perf report for the last 5.14239 seconds: | 20260507 05:09:45.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:09:45.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:45.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:45.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:45.269 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 05:09:45.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:45.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:45.269 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:09:45.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:45.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:45.269 | INFO | Uploaded 36GiB to GPU in 2.02012s (17.8207GiB/s) | 20260507 05:09:45.269 | INFO | Downloaded 34GiB to GPU in 1.94265s (17.5018GiB/s) | 20260507 05:09:50.409 | INFO | Analyzing chunk frames = [7341568,7342080) | 20260507 05:09:50.409 | INFO | MemInfo: { | 20260507 05:09:50.409 | INFO | "gpu" : " 17 GB", | 20260507 05:09:50.409 | INFO | "huge" : "", | 20260507 05:09:50.409 | INFO | "huge_shared" : " 27 GB", | 20260507 05:09:50.409 | INFO | "malloc" : " 14 GB", | 20260507 05:09:50.409 | INFO | "pinned" : " 29 GB" | 20260507 05:09:50.409 | INFO | } | 20260507 05:09:50.409 | INFO | Perf report for the last 5.14091 seconds: | 20260507 05:09:50.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:09:50.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:50.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:50.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:50.410 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 05:09:50.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:50.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:50.410 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:09:50.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:50.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:50.410 | INFO | Uploaded 36GiB to GPU in 2.02327s (17.793GiB/s) | 20260507 05:09:50.410 | INFO | Downloaded 34GiB to GPU in 1.9434s (17.4951GiB/s) | 20260507 05:09:55.549 | INFO | Analyzing chunk frames = [7342080,7342592) | 20260507 05:09:55.549 | INFO | MemInfo: { | 20260507 05:09:55.549 | INFO | "gpu" : " 17 GB", | 20260507 05:09:55.549 | INFO | "huge" : "", | 20260507 05:09:55.549 | INFO | "huge_shared" : " 27 GB", | 20260507 05:09:55.549 | INFO | "malloc" : " 14 GB", | 20260507 05:09:55.549 | INFO | "pinned" : " 29 GB" | 20260507 05:09:55.549 | INFO | } | 20260507 05:09:55.549 | INFO | Perf report for the last 5.13983 seconds: | 20260507 05:09:55.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:09:55.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:55.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:55.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:55.549 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 05:09:55.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:55.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:55.549 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:09:55.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:55.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:09:55.550 | INFO | Uploaded 36GiB to GPU in 2.03881s (17.6573GiB/s) | 20260507 05:09:55.550 | INFO | Downloaded 34GiB to GPU in 1.91221s (17.7805GiB/s) | 20260507 05:10:00.677 | INFO | Analyzing chunk frames = [7342592,7343104) | 20260507 05:10:00.677 | INFO | MemInfo: { | 20260507 05:10:00.677 | INFO | "gpu" : " 17 GB", | 20260507 05:10:00.677 | INFO | "huge" : "", | 20260507 05:10:00.677 | INFO | "huge_shared" : " 27 GB", | 20260507 05:10:00.677 | INFO | "malloc" : " 14 GB", | 20260507 05:10:00.677 | INFO | "pinned" : " 29 GB" | 20260507 05:10:00.677 | INFO | } | 20260507 05:10:00.677 | INFO | Perf report for the last 5.12769 seconds: | 20260507 05:10:00.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:10:00.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:00.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:00.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:00.677 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 05:10:00.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:00.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:00.677 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 05:10:00.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:00.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:00.677 | INFO | Uploaded 36GiB to GPU in 2.03499s (17.6905GiB/s) | 20260507 05:10:00.677 | INFO | Downloaded 34GiB to GPU in 1.89926s (17.9017GiB/s) | 20260507 05:10:05.826 | INFO | Analyzing chunk frames = [7343104,7343616) | 20260507 05:10:05.826 | INFO | MemInfo: { | 20260507 05:10:05.826 | INFO | "gpu" : " 17 GB", | 20260507 05:10:05.826 | INFO | "huge" : "", | 20260507 05:10:05.826 | INFO | "huge_shared" : " 27 GB", | 20260507 05:10:05.826 | INFO | "malloc" : " 14 GB", | 20260507 05:10:05.826 | INFO | "pinned" : " 29 GB" | 20260507 05:10:05.826 | INFO | } | 20260507 05:10:05.826 | INFO | Perf report for the last 5.14947 seconds: | 20260507 05:10:05.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:10:05.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:05.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:05.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:05.827 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 05:10:05.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:05.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:05.827 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 05:10:05.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:05.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:05.827 | INFO | Uploaded 36GiB to GPU in 1.98128s (18.17GiB/s) | 20260507 05:10:05.827 | INFO | Downloaded 34GiB to GPU in 1.9868s (17.1129GiB/s) | 20260507 05:10:10.960 | INFO | Analyzing chunk frames = [7343616,7344128) | 20260507 05:10:10.960 | INFO | MemInfo: { | 20260507 05:10:10.960 | INFO | "gpu" : " 17 GB", | 20260507 05:10:10.960 | INFO | "huge" : "", | 20260507 05:10:10.960 | INFO | "huge_shared" : " 27 GB", | 20260507 05:10:10.960 | INFO | "malloc" : " 14 GB", | 20260507 05:10:10.960 | INFO | "pinned" : " 29 GB" | 20260507 05:10:10.960 | INFO | } | 20260507 05:10:10.960 | INFO | Perf report for the last 5.13336 seconds: | 20260507 05:10:10.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:10:10.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:10.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:10.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:10.960 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 05:10:10.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:10.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:10.960 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 05:10:10.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:10.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:10.960 | INFO | Uploaded 36GiB to GPU in 1.98104s (18.1723GiB/s) | 20260507 05:10:10.960 | INFO | Downloaded 34GiB to GPU in 1.96962s (17.2622GiB/s) | 20260507 05:10:11.057 | INFO | Starting a round of Baz IO | 20260507 05:10:14.163 | INFO | Populating RTMetrics report | 20260507 05:10:14.163 | INFO | RTMetrics report populated | 20260507 05:10:14.163 | INFO | Opening tmp RTMetrics json file | 20260507 05:10:14.164 | INFO | Serializing RTMetrics to json | 20260507 05:10:14.165 | INFO | Serialization done, swapping tmp file now... | 20260507 05:10:14.166 | INFO | RTMetrics done | 20260507 05:10:14.252 | INFO | Finished a round of Baz IO: Wrote 1.703 GiB in 3.195 seconds | 20260507 05:10:14.252 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.73% / 0.00% / 67.27% | 20260507 05:10:14.252 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.150299% / 0% / 5.16574e-05% | 20260507 05:10:14.252 | INFO | Baz IO overall: | 20260507 05:10:14.252 | INFO | BytesWritten : 3732579536611 | 20260507 05:10:14.253 | INFO | NumEvents : 1790801146839 | 20260507 05:10:16.101 | INFO | Analyzing chunk frames = [7344128,7344640) | 20260507 05:10:16.101 | INFO | MemInfo: { | 20260507 05:10:16.101 | INFO | "gpu" : " 17 GB", | 20260507 05:10:16.101 | INFO | "huge" : "", | 20260507 05:10:16.101 | INFO | "huge_shared" : " 27 GB", | 20260507 05:10:16.101 | INFO | "malloc" : " 14 GB", | 20260507 05:10:16.101 | INFO | "pinned" : " 29 GB" | 20260507 05:10:16.101 | INFO | } | 20260507 05:10:16.101 | INFO | Perf report for the last 5.14156 seconds: | 20260507 05:10:16.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:10:16.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:16.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:16.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:16.102 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:10:16.102 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:16.102 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 05:10:16.102 | INFO | PRE_HQ : 59.6%, 1.9, 58.5ms, 0 | 20260507 05:10:16.102 | INFO | BAZWRITER : 61.7%, 3.3, 104.0ms, 0 | 20260507 05:10:16.102 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:10:16.102 | INFO | Uploaded 36GiB to GPU in 2.03479s (17.6922GiB/s) | 20260507 05:10:16.102 | INFO | Downloaded 38GiB to GPU in 2.17077s (17.5053GiB/s) | 20260507 05:10:21.235 | INFO | Analyzing chunk frames = [7344640,7345152) | 20260507 05:10:21.235 | INFO | MemInfo: { | 20260507 05:10:21.235 | INFO | "gpu" : " 17 GB", | 20260507 05:10:21.235 | INFO | "huge" : "", | 20260507 05:10:21.235 | INFO | "huge_shared" : " 27 GB", | 20260507 05:10:21.235 | INFO | "malloc" : " 14 GB", | 20260507 05:10:21.235 | INFO | "pinned" : " 29 GB" | 20260507 05:10:21.235 | INFO | } | 20260507 05:10:21.235 | INFO | Perf report for the last 5.13343 seconds: | 20260507 05:10:21.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:10:21.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:21.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:21.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:21.235 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:10:21.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:21.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:21.235 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 05:10:21.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:21.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:21.235 | INFO | Uploaded 36GiB to GPU in 1.9785s (18.1956GiB/s) | 20260507 05:10:21.235 | INFO | Downloaded 34GiB to GPU in 1.935s (17.5711GiB/s) | 20260507 05:10:26.378 | INFO | Analyzing chunk frames = [7345152,7345664) | 20260507 05:10:26.378 | INFO | MemInfo: { | 20260507 05:10:26.378 | INFO | "gpu" : " 17 GB", | 20260507 05:10:26.378 | INFO | "huge" : "", | 20260507 05:10:26.378 | INFO | "huge_shared" : " 27 GB", | 20260507 05:10:26.378 | INFO | "malloc" : " 14 GB", | 20260507 05:10:26.378 | INFO | "pinned" : " 29 GB" | 20260507 05:10:26.378 | INFO | } | 20260507 05:10:26.378 | INFO | Perf report for the last 5.14355 seconds: | 20260507 05:10:26.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:10:26.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:26.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:26.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:26.379 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 05:10:26.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:26.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:26.379 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:10:26.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:26.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:26.379 | INFO | Uploaded 36GiB to GPU in 1.98205s (18.163GiB/s) | 20260507 05:10:26.379 | INFO | Downloaded 34GiB to GPU in 1.94476s (17.4828GiB/s) | 20260507 05:10:31.523 | INFO | Analyzing chunk frames = [7345664,7346176) | 20260507 05:10:31.523 | INFO | MemInfo: { | 20260507 05:10:31.523 | INFO | "gpu" : " 17 GB", | 20260507 05:10:31.523 | INFO | "huge" : "", | 20260507 05:10:31.523 | INFO | "huge_shared" : " 27 GB", | 20260507 05:10:31.523 | INFO | "malloc" : " 14 GB", | 20260507 05:10:31.523 | INFO | "pinned" : " 29 GB" | 20260507 05:10:31.523 | INFO | } | 20260507 05:10:31.523 | INFO | Perf report for the last 5.14496 seconds: | 20260507 05:10:31.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:10:31.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:31.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:31.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:31.524 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 05:10:31.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:31.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:31.524 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 05:10:31.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:31.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:31.524 | INFO | Uploaded 36GiB to GPU in 1.96341s (18.3354GiB/s) | 20260507 05:10:31.524 | INFO | Downloaded 34GiB to GPU in 1.92387s (17.6727GiB/s) | 20260507 05:10:36.662 | INFO | Analyzing chunk frames = [7346176,7346688) | 20260507 05:10:36.662 | INFO | MemInfo: { | 20260507 05:10:36.662 | INFO | "gpu" : " 17 GB", | 20260507 05:10:36.662 | INFO | "huge" : "", | 20260507 05:10:36.662 | INFO | "huge_shared" : " 27 GB", | 20260507 05:10:36.662 | INFO | "malloc" : " 14 GB", | 20260507 05:10:36.662 | INFO | "pinned" : " 29 GB" | 20260507 05:10:36.662 | INFO | } | 20260507 05:10:36.662 | INFO | Perf report for the last 5.13887 seconds: | 20260507 05:10:36.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:10:36.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:36.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:36.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:36.662 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 05:10:36.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:36.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:36.662 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:10:36.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:36.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:36.663 | INFO | Uploaded 36GiB to GPU in 2.01894s (17.8312GiB/s) | 20260507 05:10:36.663 | INFO | Downloaded 34GiB to GPU in 1.91535s (17.7513GiB/s) | 20260507 05:10:41.802 | INFO | Analyzing chunk frames = [7346688,7347200) | 20260507 05:10:41.802 | INFO | MemInfo: { | 20260507 05:10:41.802 | INFO | "gpu" : " 17 GB", | 20260507 05:10:41.802 | INFO | "huge" : "", | 20260507 05:10:41.802 | INFO | "huge_shared" : " 27 GB", | 20260507 05:10:41.802 | INFO | "malloc" : " 14 GB", | 20260507 05:10:41.802 | INFO | "pinned" : " 29 GB" | 20260507 05:10:41.802 | INFO | } | 20260507 05:10:41.802 | INFO | Perf report for the last 5.13986 seconds: | 20260507 05:10:41.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:10:41.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:41.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:41.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:41.802 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 05:10:41.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:41.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:41.802 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:10:41.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:41.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:41.802 | INFO | Uploaded 36GiB to GPU in 2.05007s (17.5603GiB/s) | 20260507 05:10:41.802 | INFO | Downloaded 34GiB to GPU in 1.91981s (17.7101GiB/s) | 20260507 05:10:46.931 | INFO | Analyzing chunk frames = [7347200,7347712) | 20260507 05:10:46.931 | INFO | MemInfo: { | 20260507 05:10:46.931 | INFO | "gpu" : " 17 GB", | 20260507 05:10:46.931 | INFO | "huge" : "", | 20260507 05:10:46.931 | INFO | "huge_shared" : " 27 GB", | 20260507 05:10:46.931 | INFO | "malloc" : " 14 GB", | 20260507 05:10:46.931 | INFO | "pinned" : " 29 GB" | 20260507 05:10:46.931 | INFO | } | 20260507 05:10:46.932 | INFO | Perf report for the last 5.12934 seconds: | 20260507 05:10:46.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:10:46.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:46.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:46.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:46.932 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 05:10:46.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:46.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:46.932 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 05:10:46.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:46.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:46.932 | INFO | Uploaded 36GiB to GPU in 1.93344s (18.6196GiB/s) | 20260507 05:10:46.932 | INFO | Downloaded 34GiB to GPU in 1.96638s (17.2906GiB/s) | 20260507 05:10:52.074 | INFO | Analyzing chunk frames = [7347712,7348224) | 20260507 05:10:52.074 | INFO | MemInfo: { | 20260507 05:10:52.074 | INFO | "gpu" : " 17 GB", | 20260507 05:10:52.074 | INFO | "huge" : "", | 20260507 05:10:52.074 | INFO | "huge_shared" : " 27 GB", | 20260507 05:10:52.074 | INFO | "malloc" : " 14 GB", | 20260507 05:10:52.074 | INFO | "pinned" : " 29 GB" | 20260507 05:10:52.074 | INFO | } | 20260507 05:10:52.074 | INFO | Perf report for the last 5.14246 seconds: | 20260507 05:10:52.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:10:52.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:52.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:52.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:52.074 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:10:52.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:52.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:52.074 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 05:10:52.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:52.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:52.074 | INFO | Uploaded 36GiB to GPU in 1.97235s (18.2524GiB/s) | 20260507 05:10:52.074 | INFO | Downloaded 34GiB to GPU in 1.95292s (17.4098GiB/s) | 20260507 05:10:52.074 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:10:52.074 | INFO | Total mem usage: 18 GiB | 20260507 05:10:52.074 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:10:52.074 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:10:52.074 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:10:52.074 | INFO | High Water Breakdown: | 20260507 05:10:52.074 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:10:52.075 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:10:52.075 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:10:52.075 | INFO | ------------------------------------------ | 20260507 05:10:52.075 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:10:52.075 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:10:52.075 | INFO | Total mem usage: 26.5 GiB | 20260507 05:10:52.075 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:10:52.075 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:10:52.075 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:10:52.075 | INFO | High Water Breakdown: | 20260507 05:10:52.075 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:10:52.075 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:10:52.075 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:10:52.075 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:10:52.075 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:10:52.075 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:10:52.075 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:10:52.075 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:10:52.075 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:10:52.075 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:10:52.075 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:10:52.075 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:10:52.075 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:10:52.075 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:10:52.075 | INFO | ------------------------------------------ | 20260507 05:10:52.075 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:10:52.075 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:10:52.075 | INFO | Total mem usage: 25.6 GiB | 20260507 05:10:52.075 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:10:52.075 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 05:10:52.075 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 05:10:52.075 | INFO | High Water Breakdown: | 20260507 05:10:52.075 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 05:10:52.075 | INFO | ------------------------------------------ | 20260507 05:10:52.075 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:10:52.075 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:10:52.075 | INFO | Total mem usage: 15.7 GiB | 20260507 05:10:52.075 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:10:52.075 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:10:52.075 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:10:52.075 | INFO | High Water Breakdown: | 20260507 05:10:52.075 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:10:52.075 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:10:52.075 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:10:52.075 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:10:52.075 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:10:52.075 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:10:52.075 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:10:52.075 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:10:52.075 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:10:52.076 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:10:52.076 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:10:52.076 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:10:52.076 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:10:52.076 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:10:52.076 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:10:52.076 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:10:52.076 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:10:52.076 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:10:52.076 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:10:52.076 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:10:52.076 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:10:52.076 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:10:52.076 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:10:52.076 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:10:52.076 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:10:52.076 | INFO | ------------------------------------------ | 20260507 05:10:52.076 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:10:52.076 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:10:52.076 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:10:52.076 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:10:52.076 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.48e+01, 1.32e+01 / 1.60e+00 | 20260507 05:10:52.076 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.33e-03 / 6.07e-03 | 20260507 05:10:52.076 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.39e+00, 5.18e-01 / 3.61e-01 | 20260507 05:10:52.076 | INFO | PerfReport: 3.25%, Histogram -- 1.00e+00 / 1.46e+00, 1.73e-01 / 1.31e-01 | 20260507 05:10:52.076 | INFO | PerfReport: 16.10%, DME -- 3.96e+01 / 4.42e+01, 3.93e+00 / 1.99e+00 | 20260507 05:10:52.076 | INFO | PerfReport: 57.77%, FrameLabeling -- 1.78e+01 / 2.04e+01, 1.38e+00 / 1.84e+00 | 20260507 05:10:52.076 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.67e+00, 3.77e-01 / 2.47e-01 | 20260507 05:10:52.076 | INFO | PerfReport: 3.00%, Metrics -- 9.23e-01 / 1.24e+00, 2.46e-01 / 1.42e-01 | 20260507 05:10:52.076 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.12e-01 / 3.66e-01, 5.09e-02 / 2.35e-02 | 20260507 05:10:52.076 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:10:52.076 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:10:52.076 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:10:52.076 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 05:10:52.076 | INFO | PerfReport: 21.73%, Upload -- 2.09e+01 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260507 05:10:52.076 | INFO | PerfReport: 21.34%, Download -- 2.05e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 05:10:52.076 | INFO | PerfReport: 32.04%, Compute -- 3.08e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 05:10:52.076 | INFO | PerfReport: 1.43%, LockUpload -- 1.37e+00 / 0.00e+00, 4.75e+00 / 0.00e+00 | 20260507 05:10:52.076 | INFO | PerfReport: 1.31%, LockDownload -- 1.25e+00 / 0.00e+00, 3.14e+00 / 0.00e+00 | 20260507 05:10:52.076 | INFO | PerfReport: 22.14%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 05:10:52.076 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260507 05:10:52.171 | INFO | Starting a round of Baz IO | 20260507 05:10:55.273 | INFO | Populating RTMetrics report | 20260507 05:10:55.274 | INFO | RTMetrics report populated | 20260507 05:10:55.274 | INFO | Opening tmp RTMetrics json file | 20260507 05:10:55.293 | INFO | Serializing RTMetrics to json | 20260507 05:10:55.294 | INFO | Serialization done, swapping tmp file now... | 20260507 05:10:55.294 | INFO | RTMetrics done | 20260507 05:10:55.361 | INFO | Finished a round of Baz IO: Wrote 1.705 GiB in 3.189 seconds | 20260507 05:10:55.361 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.73% / 0.00% / 67.27% | 20260507 05:10:55.361 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.150379% / 0% / 4.76837e-05% | 20260507 05:10:55.361 | INFO | Baz IO overall: | 20260507 05:10:55.361 | INFO | BytesWritten : 3734409896870 | 20260507 05:10:55.361 | INFO | NumEvents : 1791661041591 | 20260507 05:10:57.219 | INFO | Analyzing chunk frames = [7348224,7348736) | 20260507 05:10:57.219 | INFO | MemInfo: { | 20260507 05:10:57.219 | INFO | "gpu" : " 17 GB", | 20260507 05:10:57.219 | INFO | "huge" : "", | 20260507 05:10:57.219 | INFO | "huge_shared" : " 27 GB", | 20260507 05:10:57.219 | INFO | "malloc" : " 14 GB", | 20260507 05:10:57.219 | INFO | "pinned" : " 29 GB" | 20260507 05:10:57.219 | INFO | } | 20260507 05:10:57.219 | INFO | Perf report for the last 5.14518 seconds: | 20260507 05:10:57.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:10:57.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:57.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:57.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:57.219 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 05:10:57.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:10:57.219 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 05:10:57.219 | INFO | PRE_HQ : 59.0%, 1.8, 55.4ms, 0 | 20260507 05:10:57.219 | INFO | BAZWRITER : 61.6%, 3.2, 100.5ms, 0 | 20260507 05:10:57.219 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 05:10:57.220 | INFO | Uploaded 36GiB to GPU in 2.03482s (17.692GiB/s) | 20260507 05:10:57.220 | INFO | Downloaded 38GiB to GPU in 2.17837s (17.4442GiB/s) | 20260507 05:11:02.357 | INFO | Analyzing chunk frames = [7348736,7349248) | 20260507 05:11:02.357 | INFO | MemInfo: { | 20260507 05:11:02.357 | INFO | "gpu" : " 17 GB", | 20260507 05:11:02.357 | INFO | "huge" : "", | 20260507 05:11:02.357 | INFO | "huge_shared" : " 27 GB", | 20260507 05:11:02.357 | INFO | "malloc" : " 14 GB", | 20260507 05:11:02.357 | INFO | "pinned" : " 29 GB" | 20260507 05:11:02.357 | INFO | } | 20260507 05:11:02.357 | INFO | Perf report for the last 5.13805 seconds: | 20260507 05:11:02.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:11:02.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:02.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:02.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:02.357 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:11:02.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:02.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:02.358 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:11:02.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:02.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:02.358 | INFO | Uploaded 36GiB to GPU in 1.99477s (18.0472GiB/s) | 20260507 05:11:02.358 | INFO | Downloaded 34GiB to GPU in 1.92541s (17.6585GiB/s) | 20260507 05:11:07.492 | INFO | Analyzing chunk frames = [7349248,7349760) | 20260507 05:11:07.492 | INFO | MemInfo: { | 20260507 05:11:07.492 | INFO | "gpu" : " 17 GB", | 20260507 05:11:07.492 | INFO | "huge" : "", | 20260507 05:11:07.492 | INFO | "huge_shared" : " 27 GB", | 20260507 05:11:07.492 | INFO | "malloc" : " 14 GB", | 20260507 05:11:07.492 | INFO | "pinned" : " 29 GB" | 20260507 05:11:07.492 | INFO | } | 20260507 05:11:07.492 | INFO | Perf report for the last 5.13483 seconds: | 20260507 05:11:07.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:11:07.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:07.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:07.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:07.492 | INFO | ANALYSIS : 60.4%, 3.0, 93.4ms, 0 | 20260507 05:11:07.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:07.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:07.492 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:11:07.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:07.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:07.492 | INFO | Uploaded 36GiB to GPU in 2.02502s (17.7776GiB/s) | 20260507 05:11:07.492 | INFO | Downloaded 34GiB to GPU in 1.93855s (17.5389GiB/s) | 20260507 05:11:12.635 | INFO | Analyzing chunk frames = [7349760,7350272) | 20260507 05:11:12.635 | INFO | MemInfo: { | 20260507 05:11:12.635 | INFO | "gpu" : " 17 GB", | 20260507 05:11:12.635 | INFO | "huge" : "", | 20260507 05:11:12.635 | INFO | "huge_shared" : " 27 GB", | 20260507 05:11:12.635 | INFO | "malloc" : " 14 GB", | 20260507 05:11:12.635 | INFO | "pinned" : " 29 GB" | 20260507 05:11:12.635 | INFO | } | 20260507 05:11:12.635 | INFO | Perf report for the last 5.14296 seconds: | 20260507 05:11:12.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:11:12.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:12.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:12.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:12.635 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:11:12.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:12.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:12.635 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 05:11:12.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:12.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:12.635 | INFO | Uploaded 36GiB to GPU in 2.01327s (17.8814GiB/s) | 20260507 05:11:12.635 | INFO | Downloaded 34GiB to GPU in 1.9344s (17.5765GiB/s) | 20260507 05:11:17.774 | INFO | Analyzing chunk frames = [7350272,7350784) | 20260507 05:11:17.774 | INFO | MemInfo: { | 20260507 05:11:17.774 | INFO | "gpu" : " 17 GB", | 20260507 05:11:17.774 | INFO | "huge" : "", | 20260507 05:11:17.774 | INFO | "huge_shared" : " 27 GB", | 20260507 05:11:17.774 | INFO | "malloc" : " 14 GB", | 20260507 05:11:17.774 | INFO | "pinned" : " 29 GB" | 20260507 05:11:17.774 | INFO | } | 20260507 05:11:17.774 | INFO | Perf report for the last 5.13925 seconds: | 20260507 05:11:17.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:11:17.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:17.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:17.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:17.774 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 05:11:17.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:17.774 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:17.775 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:11:17.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:17.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:17.775 | INFO | Uploaded 36GiB to GPU in 2.03764s (17.6675GiB/s) | 20260507 05:11:17.775 | INFO | Downloaded 34GiB to GPU in 1.94038s (17.5223GiB/s) | 20260507 05:11:22.914 | INFO | Analyzing chunk frames = [7350784,7351296) | 20260507 05:11:22.914 | INFO | MemInfo: { | 20260507 05:11:22.914 | INFO | "gpu" : " 17 GB", | 20260507 05:11:22.914 | INFO | "huge" : "", | 20260507 05:11:22.914 | INFO | "huge_shared" : " 27 GB", | 20260507 05:11:22.914 | INFO | "malloc" : " 14 GB", | 20260507 05:11:22.914 | INFO | "pinned" : " 29 GB" | 20260507 05:11:22.914 | INFO | } | 20260507 05:11:22.914 | INFO | Perf report for the last 5.14002 seconds: | 20260507 05:11:22.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:11:22.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:22.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:22.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:22.915 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 05:11:22.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:22.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:22.915 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 05:11:22.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:22.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:22.915 | INFO | Uploaded 36GiB to GPU in 2.04402s (17.6123GiB/s) | 20260507 05:11:22.915 | INFO | Downloaded 34GiB to GPU in 1.91579s (17.7472GiB/s) | 20260507 05:11:28.053 | INFO | Analyzing chunk frames = [7351296,7351808) | 20260507 05:11:28.053 | INFO | MemInfo: { | 20260507 05:11:28.053 | INFO | "gpu" : " 17 GB", | 20260507 05:11:28.053 | INFO | "huge" : "", | 20260507 05:11:28.053 | INFO | "huge_shared" : " 27 GB", | 20260507 05:11:28.053 | INFO | "malloc" : " 14 GB", | 20260507 05:11:28.053 | INFO | "pinned" : " 29 GB" | 20260507 05:11:28.053 | INFO | } | 20260507 05:11:28.053 | INFO | Perf report for the last 5.13905 seconds: | 20260507 05:11:28.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:11:28.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:28.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:28.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:28.054 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:11:28.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:28.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:28.054 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 05:11:28.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:28.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:28.054 | INFO | Uploaded 36GiB to GPU in 1.9259s (18.6926GiB/s) | 20260507 05:11:28.054 | INFO | Downloaded 34GiB to GPU in 1.96056s (17.342GiB/s) | 20260507 05:11:33.188 | INFO | Analyzing chunk frames = [7351808,7352320) | 20260507 05:11:33.188 | INFO | MemInfo: { | 20260507 05:11:33.188 | INFO | "gpu" : " 17 GB", | 20260507 05:11:33.188 | INFO | "huge" : "", | 20260507 05:11:33.188 | INFO | "huge_shared" : " 27 GB", | 20260507 05:11:33.188 | INFO | "malloc" : " 14 GB", | 20260507 05:11:33.188 | INFO | "pinned" : " 29 GB" | 20260507 05:11:33.188 | INFO | } | 20260507 05:11:33.188 | INFO | Perf report for the last 5.1346 seconds: | 20260507 05:11:33.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:11:33.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:33.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:33.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:33.188 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:11:33.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:33.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:33.188 | INFO | PRE_HQ : 29.2%, 1.0, 15.2ms, 0 | 20260507 05:11:33.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:33.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:33.189 | INFO | Uploaded 36GiB to GPU in 1.97311s (18.2453GiB/s) | 20260507 05:11:33.189 | INFO | Downloaded 34GiB to GPU in 1.952s (17.4181GiB/s) | 20260507 05:11:33.285 | INFO | Starting a round of Baz IO | 20260507 05:11:36.388 | INFO | Populating RTMetrics report | 20260507 05:11:36.388 | INFO | RTMetrics report populated | 20260507 05:11:36.388 | INFO | Opening tmp RTMetrics json file | 20260507 05:11:36.392 | INFO | Serializing RTMetrics to json | 20260507 05:11:36.406 | INFO | Serialization done, swapping tmp file now... | 20260507 05:11:36.406 | INFO | RTMetrics done | 20260507 05:11:36.472 | INFO | Finished a round of Baz IO: Wrote 1.703 GiB in 3.186 seconds | 20260507 05:11:36.472 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.73% / 0.00% / 67.27% | 20260507 05:11:36.472 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.149747% / 3.97364e-06% / 6.75519e-05% | 20260507 05:11:36.472 | INFO | Baz IO overall: | 20260507 05:11:36.472 | INFO | BytesWritten : 3736238607351 | 20260507 05:11:36.472 | INFO | NumEvents : 1792520039487 | 20260507 05:11:38.332 | INFO | Analyzing chunk frames = [7352320,7352832) | 20260507 05:11:38.332 | INFO | MemInfo: { | 20260507 05:11:38.332 | INFO | "gpu" : " 17 GB", | 20260507 05:11:38.332 | INFO | "huge" : "", | 20260507 05:11:38.332 | INFO | "huge_shared" : " 27 GB", | 20260507 05:11:38.332 | INFO | "malloc" : " 14 GB", | 20260507 05:11:38.332 | INFO | "pinned" : " 29 GB" | 20260507 05:11:38.332 | INFO | } | 20260507 05:11:38.332 | INFO | Perf report for the last 5.14375 seconds: | 20260507 05:11:38.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:11:38.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:38.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:38.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:38.332 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 05:11:38.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:38.332 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 05:11:38.332 | INFO | PRE_HQ : 59.1%, 1.8, 55.5ms, 0 | 20260507 05:11:38.332 | INFO | BAZWRITER : 61.5%, 3.2, 100.6ms, 0 | 20260507 05:11:38.332 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 05:11:38.332 | INFO | Uploaded 36GiB to GPU in 2.0306s (17.7287GiB/s) | 20260507 05:11:38.332 | INFO | Downloaded 38GiB to GPU in 2.1765s (17.4592GiB/s) | 20260507 05:11:43.472 | INFO | Analyzing chunk frames = [7352832,7353344) | 20260507 05:11:43.472 | INFO | MemInfo: { | 20260507 05:11:43.472 | INFO | "gpu" : " 17 GB", | 20260507 05:11:43.472 | INFO | "huge" : "", | 20260507 05:11:43.472 | INFO | "huge_shared" : " 27 GB", | 20260507 05:11:43.472 | INFO | "malloc" : " 14 GB", | 20260507 05:11:43.472 | INFO | "pinned" : " 29 GB" | 20260507 05:11:43.472 | INFO | } | 20260507 05:11:43.472 | INFO | Perf report for the last 5.14014 seconds: | 20260507 05:11:43.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:11:43.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:43.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:43.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:43.472 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 05:11:43.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:43.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:43.472 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:11:43.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:43.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:43.472 | INFO | Uploaded 36GiB to GPU in 2.05047s (17.5569GiB/s) | 20260507 05:11:43.472 | INFO | Downloaded 34GiB to GPU in 1.93901s (17.5347GiB/s) | 20260507 05:11:48.609 | INFO | Analyzing chunk frames = [7353344,7353856) | 20260507 05:11:48.609 | INFO | MemInfo: { | 20260507 05:11:48.609 | INFO | "gpu" : " 17 GB", | 20260507 05:11:48.609 | INFO | "huge" : "", | 20260507 05:11:48.609 | INFO | "huge_shared" : " 27 GB", | 20260507 05:11:48.609 | INFO | "malloc" : " 14 GB", | 20260507 05:11:48.609 | INFO | "pinned" : " 29 GB" | 20260507 05:11:48.609 | INFO | } | 20260507 05:11:48.609 | INFO | Perf report for the last 5.13701 seconds: | 20260507 05:11:48.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:11:48.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:48.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:48.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:48.609 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 05:11:48.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:48.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:48.609 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:11:48.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:48.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:48.609 | INFO | Uploaded 36GiB to GPU in 2.05336s (17.5322GiB/s) | 20260507 05:11:48.609 | INFO | Downloaded 34GiB to GPU in 1.95178s (17.42GiB/s) | 20260507 05:11:53.748 | INFO | Analyzing chunk frames = [7353856,7354368) | 20260507 05:11:53.748 | INFO | MemInfo: { | 20260507 05:11:53.748 | INFO | "gpu" : " 17 GB", | 20260507 05:11:53.748 | INFO | "huge" : "", | 20260507 05:11:53.748 | INFO | "huge_shared" : " 27 GB", | 20260507 05:11:53.748 | INFO | "malloc" : " 14 GB", | 20260507 05:11:53.748 | INFO | "pinned" : " 29 GB" | 20260507 05:11:53.748 | INFO | } | 20260507 05:11:53.748 | INFO | Perf report for the last 5.13897 seconds: | 20260507 05:11:53.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:11:53.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:53.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:53.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:53.748 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 05:11:53.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:53.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:53.748 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:11:53.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:53.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:53.748 | INFO | Uploaded 36GiB to GPU in 2.00402s (17.9639GiB/s) | 20260507 05:11:53.748 | INFO | Downloaded 34GiB to GPU in 1.91749s (17.7315GiB/s) | 20260507 05:11:58.893 | INFO | Analyzing chunk frames = [7354368,7354880) | 20260507 05:11:58.893 | INFO | MemInfo: { | 20260507 05:11:58.893 | INFO | "gpu" : " 17 GB", | 20260507 05:11:58.893 | INFO | "huge" : "", | 20260507 05:11:58.893 | INFO | "huge_shared" : " 27 GB", | 20260507 05:11:58.893 | INFO | "malloc" : " 14 GB", | 20260507 05:11:58.893 | INFO | "pinned" : " 29 GB" | 20260507 05:11:58.893 | INFO | } | 20260507 05:11:58.893 | INFO | Perf report for the last 5.14485 seconds: | 20260507 05:11:58.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:11:58.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:58.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:58.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:58.893 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 05:11:58.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:58.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:58.893 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 05:11:58.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:58.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:11:58.893 | INFO | Uploaded 36GiB to GPU in 2.05527s (17.516GiB/s) | 20260507 05:11:58.893 | INFO | Downloaded 34GiB to GPU in 1.94403s (17.4895GiB/s) | 20260507 05:12:04.030 | INFO | Analyzing chunk frames = [7354880,7355392) | 20260507 05:12:04.030 | INFO | MemInfo: { | 20260507 05:12:04.030 | INFO | "gpu" : " 17 GB", | 20260507 05:12:04.030 | INFO | "huge" : "", | 20260507 05:12:04.030 | INFO | "huge_shared" : " 27 GB", | 20260507 05:12:04.030 | INFO | "malloc" : " 14 GB", | 20260507 05:12:04.030 | INFO | "pinned" : " 29 GB" | 20260507 05:12:04.030 | INFO | } | 20260507 05:12:04.030 | INFO | Perf report for the last 5.13717 seconds: | 20260507 05:12:04.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:12:04.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:04.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:04.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:04.030 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 05:12:04.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:04.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:04.030 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 05:12:04.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:04.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:04.030 | INFO | Uploaded 36GiB to GPU in 2.03821s (17.6626GiB/s) | 20260507 05:12:04.030 | INFO | Downloaded 34GiB to GPU in 1.90117s (17.8838GiB/s) | 20260507 05:12:09.176 | INFO | Analyzing chunk frames = [7355392,7355904) | 20260507 05:12:09.176 | INFO | MemInfo: { | 20260507 05:12:09.176 | INFO | "gpu" : " 17 GB", | 20260507 05:12:09.176 | INFO | "huge" : "", | 20260507 05:12:09.176 | INFO | "huge_shared" : " 27 GB", | 20260507 05:12:09.176 | INFO | "malloc" : " 14 GB", | 20260507 05:12:09.176 | INFO | "pinned" : " 29 GB" | 20260507 05:12:09.176 | INFO | } | 20260507 05:12:09.177 | INFO | Perf report for the last 5.14651 seconds: | 20260507 05:12:09.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:12:09.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:09.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:09.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:09.177 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 05:12:09.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:09.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:09.177 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 05:12:09.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:09.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:09.177 | INFO | Uploaded 36GiB to GPU in 2.039s (17.6557GiB/s) | 20260507 05:12:09.177 | INFO | Downloaded 34GiB to GPU in 1.978s (17.1891GiB/s) | 20260507 05:12:14.305 | INFO | Analyzing chunk frames = [7355904,7356416) | 20260507 05:12:14.305 | INFO | MemInfo: { | 20260507 05:12:14.305 | INFO | "gpu" : " 17 GB", | 20260507 05:12:14.305 | INFO | "huge" : "", | 20260507 05:12:14.305 | INFO | "huge_shared" : " 27 GB", | 20260507 05:12:14.305 | INFO | "malloc" : " 14 GB", | 20260507 05:12:14.305 | INFO | "pinned" : " 29 GB" | 20260507 05:12:14.305 | INFO | } | 20260507 05:12:14.305 | INFO | Perf report for the last 5.12892 seconds: | 20260507 05:12:14.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:12:14.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:14.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:14.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:14.306 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 05:12:14.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:14.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:14.306 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:12:14.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:14.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:14.306 | INFO | Uploaded 36GiB to GPU in 1.99959s (18.0037GiB/s) | 20260507 05:12:14.306 | INFO | Downloaded 34GiB to GPU in 1.95886s (17.357GiB/s) | 20260507 05:12:14.306 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:12:14.306 | INFO | Total mem usage: 18 GiB | 20260507 05:12:14.306 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:12:14.306 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:12:14.306 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:12:14.306 | INFO | High Water Breakdown: | 20260507 05:12:14.306 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:12:14.306 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:12:14.306 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:12:14.306 | INFO | ------------------------------------------ | 20260507 05:12:14.306 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:12:14.306 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:12:14.306 | INFO | Total mem usage: 26.5 GiB | 20260507 05:12:14.306 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:12:14.306 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:12:14.306 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:12:14.306 | INFO | High Water Breakdown: | 20260507 05:12:14.306 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:12:14.306 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:12:14.306 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:12:14.306 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:12:14.306 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:12:14.306 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:12:14.306 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:12:14.306 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:12:14.306 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:12:14.306 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:12:14.306 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:12:14.306 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:12:14.306 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:12:14.306 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:12:14.306 | INFO | ------------------------------------------ | 20260507 05:12:14.306 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:12:14.306 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:12:14.306 | INFO | Total mem usage: 25.6 GiB | 20260507 05:12:14.306 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:12:14.306 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:12:14.306 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:12:14.306 | INFO | High Water Breakdown: | 20260507 05:12:14.306 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:12:14.306 | INFO | ------------------------------------------ | 20260507 05:12:14.306 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:12:14.306 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:12:14.306 | INFO | Total mem usage: 15.7 GiB | 20260507 05:12:14.306 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:12:14.306 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:12:14.306 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:12:14.306 | INFO | High Water Breakdown: | 20260507 05:12:14.307 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:12:14.307 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:12:14.307 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:12:14.307 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:12:14.307 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:12:14.307 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:12:14.307 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:12:14.307 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:12:14.307 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:12:14.307 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:12:14.307 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:12:14.307 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:12:14.307 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:12:14.307 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:12:14.307 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:12:14.307 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:12:14.307 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:12:14.307 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:12:14.307 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:12:14.307 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:12:14.307 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:12:14.307 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:12:14.307 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:12:14.307 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:12:14.307 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:12:14.307 | INFO | ------------------------------------------ | 20260507 05:12:14.307 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:12:14.307 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:12:14.307 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:12:14.307 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:12:14.307 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.33e+01 / 1.39e+00 | 20260507 05:12:14.307 | INFO | PerfReport: 0.30%, AggregateStats -- 9.06e-02 / 1.01e-01, 9.94e-03 / 5.39e-03 | 20260507 05:12:14.307 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.41e+00, 5.08e-01 / 2.95e-01 | 20260507 05:12:14.307 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.46e+00, 1.66e-01 / 1.25e-01 | 20260507 05:12:14.307 | INFO | PerfReport: 16.21%, DME -- 3.98e+01 / 4.45e+01, 3.90e+00 / 1.57e+00 | 20260507 05:12:14.307 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.77e+01 / 1.99e+01, 1.27e+00 / 1.31e+00 | 20260507 05:12:14.307 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.13e+00 / 2.72e+00, 3.85e-01 / 2.15e-01 | 20260507 05:12:14.307 | INFO | PerfReport: 3.00%, Metrics -- 9.21e-01 / 1.25e+00, 2.44e-01 / 1.35e-01 | 20260507 05:12:14.307 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.11e-01 / 3.71e-01, 4.94e-02 / 2.43e-02 | 20260507 05:12:14.307 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:12:14.307 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:12:14.307 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:12:14.307 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 05:12:14.307 | INFO | PerfReport: 21.87%, Upload -- 2.10e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260507 05:12:14.307 | INFO | PerfReport: 21.33%, Download -- 2.05e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260507 05:12:14.307 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 05:12:14.307 | INFO | PerfReport: 1.58%, LockUpload -- 1.52e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 05:12:14.307 | INFO | PerfReport: 1.19%, LockDownload -- 1.14e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260507 05:12:14.307 | INFO | PerfReport: 22.09%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 05:12:14.307 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260507 05:12:14.404 | INFO | Starting a round of Baz IO | 20260507 05:12:17.502 | INFO | Populating RTMetrics report | 20260507 05:12:17.502 | INFO | RTMetrics report populated | 20260507 05:12:17.502 | INFO | Opening tmp RTMetrics json file | 20260507 05:12:17.503 | INFO | Serializing RTMetrics to json | 20260507 05:12:17.504 | INFO | Serialization done, swapping tmp file now... | 20260507 05:12:17.505 | INFO | RTMetrics done | 20260507 05:12:17.589 | INFO | Finished a round of Baz IO: Wrote 1.704 GiB in 3.185 seconds | 20260507 05:12:17.589 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.73% / 0.00% / 67.27% | 20260507 05:12:17.590 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.149957% / 0% / 6.35783e-05% | 20260507 05:12:17.590 | INFO | Baz IO overall: | 20260507 05:12:17.590 | INFO | BytesWritten : 3738068133000 | 20260507 05:12:17.590 | INFO | NumEvents : 1793379676533 | 20260507 05:12:19.442 | INFO | Analyzing chunk frames = [7356416,7356928) | 20260507 05:12:19.442 | INFO | MemInfo: { | 20260507 05:12:19.442 | INFO | "gpu" : " 17 GB", | 20260507 05:12:19.442 | INFO | "huge" : "", | 20260507 05:12:19.442 | INFO | "huge_shared" : " 27 GB", | 20260507 05:12:19.442 | INFO | "malloc" : " 14 GB", | 20260507 05:12:19.442 | INFO | "pinned" : " 29 GB" | 20260507 05:12:19.442 | INFO | } | 20260507 05:12:19.442 | INFO | Perf report for the last 5.13657 seconds: | 20260507 05:12:19.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:12:19.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:19.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:19.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:19.442 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 05:12:19.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:19.442 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 05:12:19.442 | INFO | PRE_HQ : 59.2%, 1.8, 55.3ms, 0 | 20260507 05:12:19.442 | INFO | BAZWRITER : 61.6%, 3.2, 101.1ms, 0 | 20260507 05:12:19.442 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:12:19.442 | INFO | Uploaded 36GiB to GPU in 2.02635s (17.766GiB/s) | 20260507 05:12:19.442 | INFO | Downloaded 38GiB to GPU in 2.16783s (17.5291GiB/s) | 20260507 05:12:24.587 | INFO | Analyzing chunk frames = [7356928,7357440) | 20260507 05:12:24.587 | INFO | MemInfo: { | 20260507 05:12:24.587 | INFO | "gpu" : " 17 GB", | 20260507 05:12:24.587 | INFO | "huge" : "", | 20260507 05:12:24.587 | INFO | "huge_shared" : " 27 GB", | 20260507 05:12:24.587 | INFO | "malloc" : " 14 GB", | 20260507 05:12:24.587 | INFO | "pinned" : " 29 GB" | 20260507 05:12:24.587 | INFO | } | 20260507 05:12:24.587 | INFO | Perf report for the last 5.14516 seconds: | 20260507 05:12:24.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:12:24.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:24.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:24.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:24.587 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 05:12:24.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:24.587 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:24.587 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:12:24.587 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:24.587 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:24.587 | INFO | Uploaded 36GiB to GPU in 2.04977s (17.563GiB/s) | 20260507 05:12:24.587 | INFO | Downloaded 34GiB to GPU in 1.94175s (17.51GiB/s) | 20260507 05:12:29.728 | INFO | Analyzing chunk frames = [7357440,7357952) | 20260507 05:12:29.728 | INFO | MemInfo: { | 20260507 05:12:29.728 | INFO | "gpu" : " 17 GB", | 20260507 05:12:29.728 | INFO | "huge" : "", | 20260507 05:12:29.728 | INFO | "huge_shared" : " 27 GB", | 20260507 05:12:29.729 | INFO | "malloc" : " 14 GB", | 20260507 05:12:29.729 | INFO | "pinned" : " 29 GB" | 20260507 05:12:29.729 | INFO | } | 20260507 05:12:29.729 | INFO | Perf report for the last 5.14133 seconds: | 20260507 05:12:29.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:12:29.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:29.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:29.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:29.729 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 05:12:29.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:29.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:29.729 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:12:29.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:29.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:29.729 | INFO | Uploaded 36GiB to GPU in 2.03738s (17.6698GiB/s) | 20260507 05:12:29.729 | INFO | Downloaded 34GiB to GPU in 1.95014s (17.4347GiB/s) | 20260507 05:12:34.866 | INFO | Analyzing chunk frames = [7357952,7358464) | 20260507 05:12:34.866 | INFO | MemInfo: { | 20260507 05:12:34.866 | INFO | "gpu" : " 17 GB", | 20260507 05:12:34.866 | INFO | "huge" : "", | 20260507 05:12:34.866 | INFO | "huge_shared" : " 27 GB", | 20260507 05:12:34.866 | INFO | "malloc" : " 14 GB", | 20260507 05:12:34.866 | INFO | "pinned" : " 29 GB" | 20260507 05:12:34.866 | INFO | } | 20260507 05:12:34.866 | INFO | Perf report for the last 5.1378 seconds: | 20260507 05:12:34.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:12:34.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:34.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:34.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:34.866 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 05:12:34.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:34.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:34.866 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:12:34.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:34.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:34.867 | INFO | Uploaded 36GiB to GPU in 2.04252s (17.6253GiB/s) | 20260507 05:12:34.867 | INFO | Downloaded 34GiB to GPU in 1.94954s (17.44GiB/s) | 20260507 05:12:40.001 | INFO | Analyzing chunk frames = [7358464,7358976) | 20260507 05:12:40.001 | INFO | MemInfo: { | 20260507 05:12:40.001 | INFO | "gpu" : " 17 GB", | 20260507 05:12:40.001 | INFO | "huge" : "", | 20260507 05:12:40.001 | INFO | "huge_shared" : " 27 GB", | 20260507 05:12:40.001 | INFO | "malloc" : " 14 GB", | 20260507 05:12:40.002 | INFO | "pinned" : " 29 GB" | 20260507 05:12:40.002 | INFO | } | 20260507 05:12:40.002 | INFO | Perf report for the last 5.13521 seconds: | 20260507 05:12:40.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:12:40.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:40.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:40.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:40.002 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 05:12:40.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:40.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:40.002 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:12:40.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:40.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:40.002 | INFO | Uploaded 36GiB to GPU in 2.03817s (17.6629GiB/s) | 20260507 05:12:40.002 | INFO | Downloaded 34GiB to GPU in 1.91733s (17.733GiB/s) | 20260507 05:12:45.148 | INFO | Analyzing chunk frames = [7358976,7359488) | 20260507 05:12:45.148 | INFO | MemInfo: { | 20260507 05:12:45.148 | INFO | "gpu" : " 17 GB", | 20260507 05:12:45.148 | INFO | "huge" : "", | 20260507 05:12:45.148 | INFO | "huge_shared" : " 27 GB", | 20260507 05:12:45.148 | INFO | "malloc" : " 14 GB", | 20260507 05:12:45.148 | INFO | "pinned" : " 29 GB" | 20260507 05:12:45.148 | INFO | } | 20260507 05:12:45.148 | INFO | Perf report for the last 5.1468 seconds: | 20260507 05:12:45.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:12:45.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:45.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:45.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:45.148 | INFO | ANALYSIS : 60.4%, 3.0, 94.4ms, 0 | 20260507 05:12:45.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:45.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:45.148 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 05:12:45.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:45.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:45.149 | INFO | Uploaded 36GiB to GPU in 2.05097s (17.5526GiB/s) | 20260507 05:12:45.149 | INFO | Downloaded 34GiB to GPU in 1.91869s (17.7204GiB/s) | 20260507 05:12:50.282 | INFO | Analyzing chunk frames = [7359488,7360000) | 20260507 05:12:50.282 | INFO | MemInfo: { | 20260507 05:12:50.282 | INFO | "gpu" : " 17 GB", | 20260507 05:12:50.282 | INFO | "huge" : "", | 20260507 05:12:50.282 | INFO | "huge_shared" : " 27 GB", | 20260507 05:12:50.282 | INFO | "malloc" : " 14 GB", | 20260507 05:12:50.282 | INFO | "pinned" : " 29 GB" | 20260507 05:12:50.282 | INFO | } | 20260507 05:12:50.282 | INFO | Perf report for the last 5.13375 seconds: | 20260507 05:12:50.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:12:50.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:50.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:50.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:50.282 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 05:12:50.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:50.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:50.282 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 05:12:50.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:50.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:50.282 | INFO | Uploaded 36GiB to GPU in 2.0327s (17.7104GiB/s) | 20260507 05:12:50.283 | INFO | Downloaded 34GiB to GPU in 1.97905s (17.1799GiB/s) | 20260507 05:12:55.420 | INFO | Analyzing chunk frames = [7360000,7360512) | 20260507 05:12:55.420 | INFO | MemInfo: { | 20260507 05:12:55.420 | INFO | "gpu" : " 17 GB", | 20260507 05:12:55.420 | INFO | "huge" : "", | 20260507 05:12:55.420 | INFO | "huge_shared" : " 27 GB", | 20260507 05:12:55.420 | INFO | "malloc" : " 14 GB", | 20260507 05:12:55.420 | INFO | "pinned" : " 29 GB" | 20260507 05:12:55.420 | INFO | } | 20260507 05:12:55.420 | INFO | Perf report for the last 5.13784 seconds: | 20260507 05:12:55.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:12:55.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:55.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:55.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:55.420 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 05:12:55.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:55.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:55.420 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:12:55.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:55.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:12:55.420 | INFO | Uploaded 36GiB to GPU in 1.96547s (18.3162GiB/s) | 20260507 05:12:55.420 | INFO | Downloaded 34GiB to GPU in 1.95133s (17.424GiB/s) | 20260507 05:12:55.517 | INFO | Starting a round of Baz IO | 20260507 05:12:58.612 | INFO | Populating RTMetrics report | 20260507 05:12:58.613 | INFO | RTMetrics report populated | 20260507 05:12:58.613 | INFO | Opening tmp RTMetrics json file | 20260507 05:12:58.615 | INFO | Serializing RTMetrics to json | 20260507 05:12:58.631 | INFO | Serialization done, swapping tmp file now... | 20260507 05:12:58.631 | INFO | RTMetrics done | 20260507 05:12:58.704 | INFO | Finished a round of Baz IO: Wrote 1.705 GiB in 3.186 seconds | 20260507 05:12:58.704 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.72% / 0.00% / 67.27% | 20260507 05:12:58.704 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.149779% / 0% / 5.96046e-05% | 20260507 05:12:58.704 | INFO | Baz IO overall: | 20260507 05:12:58.704 | INFO | BytesWritten : 3739898629727 | 20260507 05:12:58.704 | INFO | NumEvents : 1794239580490 | 20260507 05:13:00.560 | INFO | Analyzing chunk frames = [7360512,7361024) | 20260507 05:13:00.560 | INFO | MemInfo: { | 20260507 05:13:00.560 | INFO | "gpu" : " 17 GB", | 20260507 05:13:00.560 | INFO | "huge" : "", | 20260507 05:13:00.560 | INFO | "huge_shared" : " 27 GB", | 20260507 05:13:00.560 | INFO | "malloc" : " 14 GB", | 20260507 05:13:00.560 | INFO | "pinned" : " 29 GB" | 20260507 05:13:00.560 | INFO | } | 20260507 05:13:00.560 | INFO | Perf report for the last 5.13996 seconds: | 20260507 05:13:00.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:13:00.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:00.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:00.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:00.560 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 05:13:00.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:00.560 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 05:13:00.560 | INFO | PRE_HQ : 58.9%, 1.8, 54.8ms, 0 | 20260507 05:13:00.560 | INFO | BAZWRITER : 61.6%, 3.2, 100.8ms, 0 | 20260507 05:13:00.560 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:13:00.560 | INFO | Uploaded 36GiB to GPU in 2.04388s (17.6135GiB/s) | 20260507 05:13:00.560 | INFO | Downloaded 38GiB to GPU in 2.17547s (17.4675GiB/s) | 20260507 05:13:05.699 | INFO | Analyzing chunk frames = [7361024,7361536) | 20260507 05:13:05.699 | INFO | MemInfo: { | 20260507 05:13:05.699 | INFO | "gpu" : " 17 GB", | 20260507 05:13:05.699 | INFO | "huge" : "", | 20260507 05:13:05.699 | INFO | "huge_shared" : " 27 GB", | 20260507 05:13:05.699 | INFO | "malloc" : " 14 GB", | 20260507 05:13:05.699 | INFO | "pinned" : " 29 GB" | 20260507 05:13:05.699 | INFO | } | 20260507 05:13:05.699 | INFO | Perf report for the last 5.13883 seconds: | 20260507 05:13:05.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:13:05.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:05.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:05.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:05.699 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 05:13:05.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:05.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:05.699 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:13:05.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:05.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:05.699 | INFO | Uploaded 36GiB to GPU in 2.04669s (17.5893GiB/s) | 20260507 05:13:05.699 | INFO | Downloaded 34GiB to GPU in 1.94631s (17.469GiB/s) | 20260507 05:13:10.843 | INFO | Analyzing chunk frames = [7361536,7362048) | 20260507 05:13:10.843 | INFO | MemInfo: { | 20260507 05:13:10.843 | INFO | "gpu" : " 17 GB", | 20260507 05:13:10.843 | INFO | "huge" : "", | 20260507 05:13:10.843 | INFO | "huge_shared" : " 27 GB", | 20260507 05:13:10.843 | INFO | "malloc" : " 14 GB", | 20260507 05:13:10.843 | INFO | "pinned" : " 29 GB" | 20260507 05:13:10.843 | INFO | } | 20260507 05:13:10.843 | INFO | Perf report for the last 5.14423 seconds: | 20260507 05:13:10.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:13:10.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:10.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:10.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:10.843 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 05:13:10.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:10.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:10.843 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:13:10.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:10.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:10.844 | INFO | Uploaded 36GiB to GPU in 2.0078s (17.9301GiB/s) | 20260507 05:13:10.844 | INFO | Downloaded 34GiB to GPU in 1.94599s (17.4718GiB/s) | 20260507 05:13:15.983 | INFO | Analyzing chunk frames = [7362048,7362560) | 20260507 05:13:15.983 | INFO | MemInfo: { | 20260507 05:13:15.983 | INFO | "gpu" : " 17 GB", | 20260507 05:13:15.983 | INFO | "huge" : "", | 20260507 05:13:15.983 | INFO | "huge_shared" : " 27 GB", | 20260507 05:13:15.983 | INFO | "malloc" : " 14 GB", | 20260507 05:13:15.983 | INFO | "pinned" : " 29 GB" | 20260507 05:13:15.983 | INFO | } | 20260507 05:13:15.983 | INFO | Perf report for the last 5.14017 seconds: | 20260507 05:13:15.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:13:15.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:15.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:15.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:15.984 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:13:15.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:15.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:15.984 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:13:15.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:15.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:15.984 | INFO | Uploaded 36GiB to GPU in 1.99079s (18.0833GiB/s) | 20260507 05:13:15.984 | INFO | Downloaded 34GiB to GPU in 1.94681s (17.4645GiB/s) | 20260507 05:13:21.122 | INFO | Analyzing chunk frames = [7362560,7363072) | 20260507 05:13:21.122 | INFO | MemInfo: { | 20260507 05:13:21.122 | INFO | "gpu" : " 17 GB", | 20260507 05:13:21.122 | INFO | "huge" : "", | 20260507 05:13:21.122 | INFO | "huge_shared" : " 27 GB", | 20260507 05:13:21.122 | INFO | "malloc" : " 14 GB", | 20260507 05:13:21.122 | INFO | "pinned" : " 29 GB" | 20260507 05:13:21.122 | INFO | } | 20260507 05:13:21.122 | INFO | Perf report for the last 5.13878 seconds: | 20260507 05:13:21.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:13:21.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:21.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:21.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:21.122 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 05:13:21.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:21.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:21.122 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:13:21.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:21.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:21.122 | INFO | Uploaded 36GiB to GPU in 2.03204s (17.7162GiB/s) | 20260507 05:13:21.122 | INFO | Downloaded 34GiB to GPU in 1.9114s (17.788GiB/s) | 20260507 05:13:26.250 | INFO | Analyzing chunk frames = [7363072,7363584) | 20260507 05:13:26.250 | INFO | MemInfo: { | 20260507 05:13:26.250 | INFO | "gpu" : " 17 GB", | 20260507 05:13:26.250 | INFO | "huge" : "", | 20260507 05:13:26.250 | INFO | "huge_shared" : " 27 GB", | 20260507 05:13:26.250 | INFO | "malloc" : " 14 GB", | 20260507 05:13:26.250 | INFO | "pinned" : " 29 GB" | 20260507 05:13:26.250 | INFO | } | 20260507 05:13:26.250 | INFO | Perf report for the last 5.12816 seconds: | 20260507 05:13:26.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:13:26.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:26.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:26.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:26.250 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:13:26.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:26.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:26.250 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 05:13:26.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:26.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:26.250 | INFO | Uploaded 36GiB to GPU in 2.05559s (17.5132GiB/s) | 20260507 05:13:26.250 | INFO | Downloaded 34GiB to GPU in 1.92473s (17.6648GiB/s) | 20260507 05:13:31.396 | INFO | Analyzing chunk frames = [7363584,7364096) | 20260507 05:13:31.396 | INFO | MemInfo: { | 20260507 05:13:31.396 | INFO | "gpu" : " 17 GB", | 20260507 05:13:31.396 | INFO | "huge" : "", | 20260507 05:13:31.396 | INFO | "huge_shared" : " 27 GB", | 20260507 05:13:31.397 | INFO | "malloc" : " 14 GB", | 20260507 05:13:31.397 | INFO | "pinned" : " 29 GB" | 20260507 05:13:31.397 | INFO | } | 20260507 05:13:31.397 | INFO | Perf report for the last 5.14633 seconds: | 20260507 05:13:31.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:13:31.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:31.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:31.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:31.397 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 05:13:31.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:31.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:31.397 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:13:31.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:31.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:31.397 | INFO | Uploaded 36GiB to GPU in 1.9765s (18.214GiB/s) | 20260507 05:13:31.397 | INFO | Downloaded 34GiB to GPU in 1.98812s (17.1016GiB/s) | 20260507 05:13:36.540 | INFO | Analyzing chunk frames = [7364096,7364608) | 20260507 05:13:36.540 | INFO | MemInfo: { | 20260507 05:13:36.540 | INFO | "gpu" : " 17 GB", | 20260507 05:13:36.540 | INFO | "huge" : "", | 20260507 05:13:36.540 | INFO | "huge_shared" : " 27 GB", | 20260507 05:13:36.540 | INFO | "malloc" : " 14 GB", | 20260507 05:13:36.540 | INFO | "pinned" : " 29 GB" | 20260507 05:13:36.540 | INFO | } | 20260507 05:13:36.540 | INFO | Perf report for the last 5.14331 seconds: | 20260507 05:13:36.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:13:36.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:36.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:36.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:36.540 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:13:36.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:36.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:36.540 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:13:36.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:36.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:36.540 | INFO | Uploaded 36GiB to GPU in 1.95317s (18.4316GiB/s) | 20260507 05:13:36.540 | INFO | Downloaded 34GiB to GPU in 1.96693s (17.2858GiB/s) | 20260507 05:13:36.540 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:13:36.540 | INFO | Total mem usage: 18 GiB | 20260507 05:13:36.540 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:13:36.540 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:13:36.540 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:13:36.540 | INFO | High Water Breakdown: | 20260507 05:13:36.541 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:13:36.541 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:13:36.541 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:13:36.541 | INFO | ------------------------------------------ | 20260507 05:13:36.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:13:36.541 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:13:36.541 | INFO | Total mem usage: 26.5 GiB | 20260507 05:13:36.541 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:13:36.541 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:13:36.541 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:13:36.541 | INFO | High Water Breakdown: | 20260507 05:13:36.541 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:13:36.541 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:13:36.541 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:13:36.541 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:13:36.541 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:13:36.541 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:13:36.541 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:13:36.541 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:13:36.541 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:13:36.541 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:13:36.541 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:13:36.541 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:13:36.541 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:13:36.541 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:13:36.541 | INFO | ------------------------------------------ | 20260507 05:13:36.541 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:13:36.541 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:13:36.541 | INFO | Total mem usage: 25.6 GiB | 20260507 05:13:36.541 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:13:36.541 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:13:36.541 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:13:36.541 | INFO | High Water Breakdown: | 20260507 05:13:36.541 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:13:36.541 | INFO | ------------------------------------------ | 20260507 05:13:36.542 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:13:36.542 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:13:36.542 | INFO | Total mem usage: 15.7 GiB | 20260507 05:13:36.542 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:13:36.542 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:13:36.542 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:13:36.542 | INFO | High Water Breakdown: | 20260507 05:13:36.542 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:13:36.542 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:13:36.542 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:13:36.542 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:13:36.542 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:13:36.542 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:13:36.542 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:13:36.542 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:13:36.542 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:13:36.542 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:13:36.542 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:13:36.542 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:13:36.542 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:13:36.542 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:13:36.542 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:13:36.542 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:13:36.542 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:13:36.542 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:13:36.542 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:13:36.542 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:13:36.542 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:13:36.542 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:13:36.542 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:13:36.542 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:13:36.542 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:13:36.542 | INFO | ------------------------------------------ | 20260507 05:13:36.542 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:13:36.542 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:13:36.542 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:13:36.542 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:13:36.542 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.33e+01 / 1.71e+00 | 20260507 05:13:36.542 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.00e-01, 9.46e-03 / 5.48e-03 | 20260507 05:13:36.542 | INFO | PerfReport: 11.64%, Baseline -- 3.58e+00 / 4.50e+00, 5.02e-01 / 2.71e-01 | 20260507 05:13:36.542 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.46e+00, 1.67e-01 / 1.24e-01 | 20260507 05:13:36.542 | INFO | PerfReport: 16.16%, DME -- 3.97e+01 / 4.43e+01, 3.85e+00 / 2.04e+00 | 20260507 05:13:36.542 | INFO | PerfReport: 57.74%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.32e+00 / 1.49e+00 | 20260507 05:13:36.542 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.71e+00, 3.83e-01 / 2.31e-01 | 20260507 05:13:36.542 | INFO | PerfReport: 2.99%, Metrics -- 9.19e-01 / 1.21e+00, 2.47e-01 / 1.12e-01 | 20260507 05:13:36.542 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.11e-01 / 3.59e-01, 5.18e-02 / 1.59e-02 | 20260507 05:13:36.542 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:13:36.542 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:13:36.542 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:13:36.542 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 05:13:36.542 | INFO | PerfReport: 21.88%, Upload -- 2.11e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260507 05:13:36.542 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 05:13:36.542 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 05:13:36.542 | INFO | PerfReport: 1.53%, LockUpload -- 1.47e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260507 05:13:36.542 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260507 05:13:36.542 | INFO | PerfReport: 22.01%, LockCompute -- 2.12e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 05:13:36.542 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260507 05:13:36.638 | INFO | Starting a round of Baz IO | 20260507 05:13:39.735 | INFO | Populating RTMetrics report | 20260507 05:13:39.736 | INFO | RTMetrics report populated | 20260507 05:13:39.736 | INFO | Opening tmp RTMetrics json file | 20260507 05:13:39.753 | INFO | Serializing RTMetrics to json | 20260507 05:13:39.754 | INFO | Serialization done, swapping tmp file now... | 20260507 05:13:39.755 | INFO | RTMetrics done | 20260507 05:13:39.827 | INFO | Finished a round of Baz IO: Wrote 1.702 GiB in 3.189 seconds | 20260507 05:13:39.827 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.72% / 0.00% / 67.28% | 20260507 05:13:39.827 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.148288% / 0% / 7.15256e-05% | 20260507 05:13:39.827 | INFO | Baz IO overall: | 20260507 05:13:39.827 | INFO | BytesWritten : 3741725784595 | 20260507 05:13:39.827 | INFO | NumEvents : 1795097296379 | 20260507 05:13:41.677 | INFO | Analyzing chunk frames = [7364608,7365120) | 20260507 05:13:41.677 | INFO | MemInfo: { | 20260507 05:13:41.677 | INFO | "gpu" : " 17 GB", | 20260507 05:13:41.677 | INFO | "huge" : "", | 20260507 05:13:41.677 | INFO | "huge_shared" : " 27 GB", | 20260507 05:13:41.677 | INFO | "malloc" : " 14 GB", | 20260507 05:13:41.677 | INFO | "pinned" : " 29 GB" | 20260507 05:13:41.677 | INFO | } | 20260507 05:13:41.678 | INFO | Perf report for the last 5.13765 seconds: | 20260507 05:13:41.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:13:41.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:41.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:41.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:41.678 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 05:13:41.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:41.678 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 05:13:41.678 | INFO | PRE_HQ : 59.0%, 1.8, 54.9ms, 0 | 20260507 05:13:41.678 | INFO | BAZWRITER : 61.7%, 3.2, 101.7ms, 0 | 20260507 05:13:41.678 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:13:41.678 | INFO | Uploaded 36GiB to GPU in 2.0417s (17.6324GiB/s) | 20260507 05:13:41.678 | INFO | Downloaded 38GiB to GPU in 2.17321s (17.4856GiB/s) | 20260507 05:13:46.811 | INFO | Analyzing chunk frames = [7365120,7365632) | 20260507 05:13:46.811 | INFO | MemInfo: { | 20260507 05:13:46.811 | INFO | "gpu" : " 17 GB", | 20260507 05:13:46.811 | INFO | "huge" : "", | 20260507 05:13:46.811 | INFO | "huge_shared" : " 27 GB", | 20260507 05:13:46.811 | INFO | "malloc" : " 14 GB", | 20260507 05:13:46.811 | INFO | "pinned" : " 29 GB" | 20260507 05:13:46.811 | INFO | } | 20260507 05:13:46.811 | INFO | Perf report for the last 5.13345 seconds: | 20260507 05:13:46.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:13:46.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:46.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:46.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:46.811 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:13:46.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:46.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:46.811 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:13:46.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:46.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:46.811 | INFO | Uploaded 36GiB to GPU in 1.97657s (18.2134GiB/s) | 20260507 05:13:46.812 | INFO | Downloaded 34GiB to GPU in 1.92549s (17.6578GiB/s) | 20260507 05:13:51.950 | INFO | Analyzing chunk frames = [7365632,7366144) | 20260507 05:13:51.950 | INFO | MemInfo: { | 20260507 05:13:51.950 | INFO | "gpu" : " 17 GB", | 20260507 05:13:51.950 | INFO | "huge" : "", | 20260507 05:13:51.950 | INFO | "huge_shared" : " 27 GB", | 20260507 05:13:51.950 | INFO | "malloc" : " 14 GB", | 20260507 05:13:51.950 | INFO | "pinned" : " 29 GB" | 20260507 05:13:51.950 | INFO | } | 20260507 05:13:51.950 | INFO | Perf report for the last 5.13889 seconds: | 20260507 05:13:51.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:13:51.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:51.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:51.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:51.950 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 05:13:51.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:51.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:51.950 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 05:13:51.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:51.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:51.950 | INFO | Uploaded 36GiB to GPU in 2.01021s (17.9086GiB/s) | 20260507 05:13:51.950 | INFO | Downloaded 34GiB to GPU in 1.94528s (17.4782GiB/s) | 20260507 05:13:57.096 | INFO | Analyzing chunk frames = [7366144,7366656) | 20260507 05:13:57.096 | INFO | MemInfo: { | 20260507 05:13:57.096 | INFO | "gpu" : " 17 GB", | 20260507 05:13:57.096 | INFO | "huge" : "", | 20260507 05:13:57.096 | INFO | "huge_shared" : " 27 GB", | 20260507 05:13:57.096 | INFO | "malloc" : " 14 GB", | 20260507 05:13:57.096 | INFO | "pinned" : " 29 GB" | 20260507 05:13:57.096 | INFO | } | 20260507 05:13:57.096 | INFO | Perf report for the last 5.14596 seconds: | 20260507 05:13:57.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:13:57.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:57.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:57.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:57.096 | INFO | ANALYSIS : 59.3%, 3.0, 92.2ms, 0 | 20260507 05:13:57.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:57.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:57.096 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:13:57.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:57.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:13:57.096 | INFO | Uploaded 36GiB to GPU in 1.94884s (18.4725GiB/s) | 20260507 05:13:57.096 | INFO | Downloaded 34GiB to GPU in 1.89694s (17.9236GiB/s) | 20260507 05:14:02.235 | INFO | Analyzing chunk frames = [7366656,7367168) | 20260507 05:14:02.235 | INFO | MemInfo: { | 20260507 05:14:02.235 | INFO | "gpu" : " 17 GB", | 20260507 05:14:02.235 | INFO | "huge" : "", | 20260507 05:14:02.235 | INFO | "huge_shared" : " 27 GB", | 20260507 05:14:02.235 | INFO | "malloc" : " 14 GB", | 20260507 05:14:02.235 | INFO | "pinned" : " 29 GB" | 20260507 05:14:02.235 | INFO | } | 20260507 05:14:02.235 | INFO | Perf report for the last 5.13919 seconds: | 20260507 05:14:02.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:14:02.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:02.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:02.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:02.235 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 05:14:02.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:02.235 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:02.235 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:14:02.235 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:02.235 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:02.235 | INFO | Uploaded 36GiB to GPU in 2.02047s (17.8176GiB/s) | 20260507 05:14:02.235 | INFO | Downloaded 34GiB to GPU in 1.94779s (17.4557GiB/s) | 20260507 05:14:07.373 | INFO | Analyzing chunk frames = [7367168,7367680) | 20260507 05:14:07.373 | INFO | MemInfo: { | 20260507 05:14:07.373 | INFO | "gpu" : " 17 GB", | 20260507 05:14:07.373 | INFO | "huge" : "", | 20260507 05:14:07.374 | INFO | "huge_shared" : " 27 GB", | 20260507 05:14:07.374 | INFO | "malloc" : " 14 GB", | 20260507 05:14:07.374 | INFO | "pinned" : " 29 GB" | 20260507 05:14:07.374 | INFO | } | 20260507 05:14:07.374 | INFO | Perf report for the last 5.13846 seconds: | 20260507 05:14:07.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:14:07.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:07.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:07.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:07.374 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 05:14:07.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:07.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:07.374 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 05:14:07.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:07.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:07.374 | INFO | Uploaded 36GiB to GPU in 2.04894s (17.5701GiB/s) | 20260507 05:14:07.374 | INFO | Downloaded 34GiB to GPU in 1.92606s (17.6526GiB/s) | 20260507 05:14:12.511 | INFO | Analyzing chunk frames = [7367680,7368192) | 20260507 05:14:12.511 | INFO | MemInfo: { | 20260507 05:14:12.511 | INFO | "gpu" : " 17 GB", | 20260507 05:14:12.511 | INFO | "huge" : "", | 20260507 05:14:12.511 | INFO | "huge_shared" : " 27 GB", | 20260507 05:14:12.511 | INFO | "malloc" : " 14 GB", | 20260507 05:14:12.511 | INFO | "pinned" : " 29 GB" | 20260507 05:14:12.511 | INFO | } | 20260507 05:14:12.511 | INFO | Perf report for the last 5.13755 seconds: | 20260507 05:14:12.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:14:12.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:12.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:12.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:12.511 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:14:12.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:12.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:12.511 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:14:12.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:12.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:12.511 | INFO | Uploaded 36GiB to GPU in 1.95046s (18.4572GiB/s) | 20260507 05:14:12.511 | INFO | Downloaded 34GiB to GPU in 1.97987s (17.1728GiB/s) | 20260507 05:14:17.653 | INFO | Analyzing chunk frames = [7368192,7368704) | 20260507 05:14:17.653 | INFO | MemInfo: { | 20260507 05:14:17.653 | INFO | "gpu" : " 17 GB", | 20260507 05:14:17.653 | INFO | "huge" : "", | 20260507 05:14:17.653 | INFO | "huge_shared" : " 27 GB", | 20260507 05:14:17.653 | INFO | "malloc" : " 14 GB", | 20260507 05:14:17.653 | INFO | "pinned" : " 29 GB" | 20260507 05:14:17.653 | INFO | } | 20260507 05:14:17.653 | INFO | Perf report for the last 5.14228 seconds: | 20260507 05:14:17.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:14:17.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:17.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:17.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:17.654 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:14:17.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:17.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:17.654 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 05:14:17.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:17.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:17.654 | INFO | Uploaded 36GiB to GPU in 1.97289s (18.2474GiB/s) | 20260507 05:14:17.654 | INFO | Downloaded 34GiB to GPU in 1.96881s (17.2693GiB/s) | 20260507 05:14:17.751 | INFO | Starting a round of Baz IO | 20260507 05:14:20.880 | INFO | Populating RTMetrics report | 20260507 05:14:20.881 | INFO | RTMetrics report populated | 20260507 05:14:20.881 | INFO | Opening tmp RTMetrics json file | 20260507 05:14:20.882 | INFO | Serializing RTMetrics to json | 20260507 05:14:20.883 | INFO | Serialization done, swapping tmp file now... | 20260507 05:14:20.884 | INFO | RTMetrics done | 20260507 05:14:20.978 | INFO | Finished a round of Baz IO: Wrote 1.702 GiB in 3.226 seconds | 20260507 05:14:20.978 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.72% / 0.00% / 67.28% | 20260507 05:14:20.978 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.148733% / 7.94729e-06% / 9.13938e-05% | 20260507 05:14:20.978 | INFO | Baz IO overall: | 20260507 05:14:20.978 | INFO | BytesWritten : 3743553367591 | 20260507 05:14:20.978 | INFO | NumEvents : 1795955791088 | 20260507 05:14:22.793 | INFO | Analyzing chunk frames = [7368704,7369216) | 20260507 05:14:22.793 | INFO | MemInfo: { | 20260507 05:14:22.793 | INFO | "gpu" : " 17 GB", | 20260507 05:14:22.793 | INFO | "huge" : "", | 20260507 05:14:22.793 | INFO | "huge_shared" : " 27 GB", | 20260507 05:14:22.793 | INFO | "malloc" : " 14 GB", | 20260507 05:14:22.793 | INFO | "pinned" : " 29 GB" | 20260507 05:14:22.793 | INFO | } | 20260507 05:14:22.793 | INFO | Perf report for the last 5.14 seconds: | 20260507 05:14:22.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:14:22.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:22.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:22.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:22.793 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 05:14:22.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:22.794 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 05:14:22.794 | INFO | PRE_HQ : 59.4%, 1.8, 56.9ms, 0 | 20260507 05:14:22.794 | INFO | BAZWRITER : 62.4%, 3.2, 103.3ms, 0 | 20260507 05:14:22.794 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 05:14:22.794 | INFO | Uploaded 36GiB to GPU in 2.04357s (17.6162GiB/s) | 20260507 05:14:22.794 | INFO | Downloaded 38GiB to GPU in 2.17984s (17.4325GiB/s) | 20260507 05:14:27.931 | INFO | Analyzing chunk frames = [7369216,7369728) | 20260507 05:14:27.931 | INFO | MemInfo: { | 20260507 05:14:27.931 | INFO | "gpu" : " 17 GB", | 20260507 05:14:27.931 | INFO | "huge" : "", | 20260507 05:14:27.931 | INFO | "huge_shared" : " 27 GB", | 20260507 05:14:27.931 | INFO | "malloc" : " 14 GB", | 20260507 05:14:27.931 | INFO | "pinned" : " 29 GB" | 20260507 05:14:27.931 | INFO | } | 20260507 05:14:27.931 | INFO | Perf report for the last 5.13786 seconds: | 20260507 05:14:27.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:14:27.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:27.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:27.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:27.931 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 05:14:27.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:27.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:27.931 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:14:27.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:27.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:27.932 | INFO | Uploaded 36GiB to GPU in 1.96754s (18.297GiB/s) | 20260507 05:14:27.932 | INFO | Downloaded 34GiB to GPU in 1.93278s (17.5913GiB/s) | 20260507 05:14:33.072 | INFO | Analyzing chunk frames = [7369728,7370240) | 20260507 05:14:33.072 | INFO | MemInfo: { | 20260507 05:14:33.072 | INFO | "gpu" : " 17 GB", | 20260507 05:14:33.072 | INFO | "huge" : "", | 20260507 05:14:33.072 | INFO | "huge_shared" : " 27 GB", | 20260507 05:14:33.072 | INFO | "malloc" : " 14 GB", | 20260507 05:14:33.072 | INFO | "pinned" : " 29 GB" | 20260507 05:14:33.072 | INFO | } | 20260507 05:14:33.072 | INFO | Perf report for the last 5.14115 seconds: | 20260507 05:14:33.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:14:33.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:33.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:33.073 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:33.073 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 05:14:33.073 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:33.073 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:33.073 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:14:33.073 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:33.073 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:33.073 | INFO | Uploaded 36GiB to GPU in 2.04858s (17.5731GiB/s) | 20260507 05:14:33.073 | INFO | Downloaded 34GiB to GPU in 1.95577s (17.3844GiB/s) | 20260507 05:14:38.206 | INFO | Analyzing chunk frames = [7370240,7370752) | 20260507 05:14:38.206 | INFO | MemInfo: { | 20260507 05:14:38.206 | INFO | "gpu" : " 17 GB", | 20260507 05:14:38.206 | INFO | "huge" : "", | 20260507 05:14:38.206 | INFO | "huge_shared" : " 27 GB", | 20260507 05:14:38.206 | INFO | "malloc" : " 14 GB", | 20260507 05:14:38.206 | INFO | "pinned" : " 29 GB" | 20260507 05:14:38.206 | INFO | } | 20260507 05:14:38.206 | INFO | Perf report for the last 5.13366 seconds: | 20260507 05:14:38.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:14:38.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:38.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:38.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:38.206 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:14:38.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:38.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:38.206 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:14:38.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:38.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:38.206 | INFO | Uploaded 36GiB to GPU in 2.00919s (17.9176GiB/s) | 20260507 05:14:38.206 | INFO | Downloaded 34GiB to GPU in 1.94027s (17.5233GiB/s) | 20260507 05:14:43.351 | INFO | Analyzing chunk frames = [7370752,7371264) | 20260507 05:14:43.351 | INFO | MemInfo: { | 20260507 05:14:43.351 | INFO | "gpu" : " 17 GB", | 20260507 05:14:43.351 | INFO | "huge" : "", | 20260507 05:14:43.351 | INFO | "huge_shared" : " 27 GB", | 20260507 05:14:43.351 | INFO | "malloc" : " 14 GB", | 20260507 05:14:43.351 | INFO | "pinned" : " 29 GB" | 20260507 05:14:43.351 | INFO | } | 20260507 05:14:43.351 | INFO | Perf report for the last 5.1451 seconds: | 20260507 05:14:43.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:14:43.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:43.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:43.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:43.351 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 05:14:43.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:43.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:43.351 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:14:43.351 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:43.351 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:43.352 | INFO | Uploaded 36GiB to GPU in 2.0293s (17.7401GiB/s) | 20260507 05:14:43.352 | INFO | Downloaded 34GiB to GPU in 1.94255s (17.5027GiB/s) | 20260507 05:14:48.493 | INFO | Analyzing chunk frames = [7371264,7371776) | 20260507 05:14:48.493 | INFO | MemInfo: { | 20260507 05:14:48.493 | INFO | "gpu" : " 17 GB", | 20260507 05:14:48.493 | INFO | "huge" : "", | 20260507 05:14:48.493 | INFO | "huge_shared" : " 27 GB", | 20260507 05:14:48.493 | INFO | "malloc" : " 14 GB", | 20260507 05:14:48.493 | INFO | "pinned" : " 29 GB" | 20260507 05:14:48.493 | INFO | } | 20260507 05:14:48.493 | INFO | Perf report for the last 5.14156 seconds: | 20260507 05:14:48.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:14:48.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:48.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:48.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:48.493 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 05:14:48.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:48.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:48.493 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:14:48.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:48.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:48.493 | INFO | Uploaded 36GiB to GPU in 2.02978s (17.736GiB/s) | 20260507 05:14:48.493 | INFO | Downloaded 34GiB to GPU in 1.91631s (17.7425GiB/s) | 20260507 05:14:53.628 | INFO | Analyzing chunk frames = [7371776,7372288) | 20260507 05:14:53.628 | INFO | MemInfo: { | 20260507 05:14:53.628 | INFO | "gpu" : " 17 GB", | 20260507 05:14:53.629 | INFO | "huge" : "", | 20260507 05:14:53.629 | INFO | "huge_shared" : " 27 GB", | 20260507 05:14:53.629 | INFO | "malloc" : " 14 GB", | 20260507 05:14:53.629 | INFO | "pinned" : " 29 GB" | 20260507 05:14:53.629 | INFO | } | 20260507 05:14:53.629 | INFO | Perf report for the last 5.13576 seconds: | 20260507 05:14:53.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:14:53.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:53.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:53.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:53.629 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:14:53.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:53.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:53.629 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:14:53.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:53.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:53.629 | INFO | Uploaded 36GiB to GPU in 1.97755s (18.2043GiB/s) | 20260507 05:14:53.629 | INFO | Downloaded 34GiB to GPU in 1.97692s (17.1985GiB/s) | 20260507 05:14:58.759 | INFO | Analyzing chunk frames = [7372288,7372800) | 20260507 05:14:58.759 | INFO | MemInfo: { | 20260507 05:14:58.759 | INFO | "gpu" : " 17 GB", | 20260507 05:14:58.759 | INFO | "huge" : "", | 20260507 05:14:58.759 | INFO | "huge_shared" : " 27 GB", | 20260507 05:14:58.759 | INFO | "malloc" : " 14 GB", | 20260507 05:14:58.759 | INFO | "pinned" : " 29 GB" | 20260507 05:14:58.759 | INFO | } | 20260507 05:14:58.759 | INFO | Perf report for the last 5.13076 seconds: | 20260507 05:14:58.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:14:58.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:58.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:58.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:58.760 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 05:14:58.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:58.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:58.760 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 05:14:58.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:58.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:14:58.760 | INFO | Uploaded 36GiB to GPU in 1.98769s (18.1115GiB/s) | 20260507 05:14:58.760 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:14:58.760 | INFO | Total mem usage: 18 GiB | 20260507 05:14:58.760 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:14:58.760 | INFO | Downloaded 34GiB to GPU in 1.95601s (17.3823GiB/s) | 20260507 05:14:58.760 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -27%) | 20260507 05:14:58.760 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:14:58.760 | INFO | High Water Breakdown: | 20260507 05:14:58.760 | INFO | 60.1% : 8.53 GiB ( 74.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:14:58.760 | INFO | 39.7% : 5.63 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:14:58.760 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:14:58.760 | INFO | ------------------------------------------ | 20260507 05:14:58.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:14:58.760 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:14:58.760 | INFO | Total mem usage: 26.5 GiB | 20260507 05:14:58.760 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:14:58.760 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:14:58.760 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:14:58.760 | INFO | High Water Breakdown: | 20260507 05:14:58.760 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:14:58.760 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:14:58.760 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:14:58.760 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:14:58.760 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:14:58.760 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:14:58.760 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:14:58.760 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:14:58.760 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:14:58.760 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:14:58.760 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:14:58.760 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:14:58.760 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:14:58.760 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:14:58.760 | INFO | ------------------------------------------ | 20260507 05:14:58.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:14:58.760 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:14:58.760 | INFO | Total mem usage: 25.6 GiB | 20260507 05:14:58.760 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:14:58.760 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:14:58.760 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:14:58.760 | INFO | High Water Breakdown: | 20260507 05:14:58.760 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:14:58.760 | INFO | ------------------------------------------ | 20260507 05:14:58.760 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:14:58.760 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:14:58.760 | INFO | Total mem usage: 15.7 GiB | 20260507 05:14:58.760 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:14:58.760 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:14:58.760 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:14:58.760 | INFO | High Water Breakdown: | 20260507 05:14:58.760 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:14:58.760 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:14:58.761 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:14:58.761 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:14:58.761 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:14:58.761 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:14:58.761 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:14:58.761 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:14:58.761 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:14:58.761 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:14:58.761 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:14:58.761 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:14:58.761 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:14:58.761 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:14:58.761 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:14:58.761 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:14:58.761 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:14:58.761 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:14:58.761 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:14:58.761 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:14:58.761 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:14:58.761 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:14:58.761 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:14:58.761 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:14:58.761 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:14:58.761 | INFO | ------------------------------------------ | 20260507 05:14:58.761 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:14:58.761 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:14:58.761 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:14:58.761 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:14:58.761 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.31e+01 / 1.48e+00 | 20260507 05:14:58.761 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.04e-01, 1.04e-02 / 1.10e-02 | 20260507 05:14:58.761 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.50e+00, 5.08e-01 / 2.07e-01 | 20260507 05:14:58.761 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.46e+00, 1.66e-01 / 1.21e-01 | 20260507 05:14:58.761 | INFO | PerfReport: 16.19%, DME -- 3.98e+01 / 4.45e+01, 3.89e+00 / 1.63e+00 | 20260507 05:14:58.761 | INFO | PerfReport: 57.65%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.27e+00 / 1.44e+00 | 20260507 05:14:58.761 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.14e+00 / 2.72e+00, 3.83e-01 / 1.94e-01 | 20260507 05:14:58.761 | INFO | PerfReport: 3.00%, Metrics -- 9.23e-01 / 1.22e+00, 2.51e-01 / 1.36e-01 | 20260507 05:14:58.761 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.11e-01 / 3.66e-01, 5.06e-02 / 2.28e-02 | 20260507 05:14:58.761 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:14:58.761 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:14:58.761 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:14:58.761 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.35e+01 / 0.00e+00 | 20260507 05:14:58.761 | INFO | PerfReport: 21.73%, Upload -- 2.09e+01 / 0.00e+00, 6.94e+00 / 0.00e+00 | 20260507 05:14:58.761 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 05:14:58.761 | INFO | PerfReport: 31.99%, Compute -- 3.07e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 05:14:58.761 | INFO | PerfReport: 1.35%, LockUpload -- 1.30e+00 / 0.00e+00, 4.69e+00 / 0.00e+00 | 20260507 05:14:58.761 | INFO | PerfReport: 1.41%, LockDownload -- 1.35e+00 / 0.00e+00, 3.24e+00 / 0.00e+00 | 20260507 05:14:58.761 | INFO | PerfReport: 22.12%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 05:14:58.761 | INFO | PerfReport: 0.02%, Misc -- 1.45e-02 / 0.00e+00, 2.26e-02 / 0.00e+00 | 20260507 05:14:58.863 | INFO | Starting a round of Baz IO | 20260507 05:15:01.963 | INFO | Populating RTMetrics report | 20260507 05:15:01.963 | INFO | RTMetrics report populated | 20260507 05:15:01.963 | INFO | Opening tmp RTMetrics json file | 20260507 05:15:01.981 | INFO | Serializing RTMetrics to json | 20260507 05:15:01.982 | INFO | Serialization done, swapping tmp file now... | 20260507 05:15:01.982 | INFO | RTMetrics done | 20260507 05:15:02.054 | INFO | Finished a round of Baz IO: Wrote 1.701 GiB in 3.19 seconds | 20260507 05:15:02.054 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.72% / 0.00% / 67.28% | 20260507 05:15:02.054 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.14834% / 3.97364e-06% / 7.94729e-05% | 20260507 05:15:02.054 | INFO | Baz IO overall: | 20260507 05:15:02.054 | INFO | BytesWritten : 3745380019375 | 20260507 05:15:02.054 | INFO | NumEvents : 1796813650309 | 20260507 05:15:03.908 | INFO | Analyzing chunk frames = [7372800,7373312) | 20260507 05:15:03.908 | INFO | MemInfo: { | 20260507 05:15:03.908 | INFO | "gpu" : " 17 GB", | 20260507 05:15:03.908 | INFO | "huge" : "", | 20260507 05:15:03.908 | INFO | "huge_shared" : " 27 GB", | 20260507 05:15:03.908 | INFO | "malloc" : " 14 GB", | 20260507 05:15:03.908 | INFO | "pinned" : " 29 GB" | 20260507 05:15:03.908 | INFO | } | 20260507 05:15:03.908 | INFO | Perf report for the last 5.14864 seconds: | 20260507 05:15:03.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:15:03.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:03.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:03.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:03.908 | INFO | ANALYSIS : 61.7%, 3.0, 96.2ms, 0 | 20260507 05:15:03.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:03.908 | INFO | RT_METRICS : 43.0%, 1.0, 22.6ms, 0 | 20260507 05:15:03.908 | INFO | PRE_HQ : 59.4%, 1.8, 57.1ms, 0 | 20260507 05:15:03.908 | INFO | BAZWRITER : 61.7%, 3.3, 104.0ms, 0 | 20260507 05:15:03.908 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:15:03.908 | INFO | Uploaded 36GiB to GPU in 2.0441s (17.6117GiB/s) | 20260507 05:15:03.909 | INFO | Downloaded 38GiB to GPU in 2.18552s (17.3872GiB/s) | 20260507 05:15:09.051 | INFO | Analyzing chunk frames = [7373312,7373824) | 20260507 05:15:09.051 | INFO | MemInfo: { | 20260507 05:15:09.051 | INFO | "gpu" : " 17 GB", | 20260507 05:15:09.051 | INFO | "huge" : "", | 20260507 05:15:09.051 | INFO | "huge_shared" : " 27 GB", | 20260507 05:15:09.051 | INFO | "malloc" : " 14 GB", | 20260507 05:15:09.051 | INFO | "pinned" : " 29 GB" | 20260507 05:15:09.051 | INFO | } | 20260507 05:15:09.051 | INFO | Perf report for the last 5.14271 seconds: | 20260507 05:15:09.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:15:09.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:09.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:09.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:09.051 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 05:15:09.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:09.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:09.051 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 05:15:09.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:09.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:09.051 | INFO | Uploaded 36GiB to GPU in 2.00231s (17.9792GiB/s) | 20260507 05:15:09.051 | INFO | Downloaded 34GiB to GPU in 1.92923s (17.6236GiB/s) | 20260507 05:15:14.190 | INFO | Analyzing chunk frames = [7373824,7374336) | 20260507 05:15:14.190 | INFO | MemInfo: { | 20260507 05:15:14.190 | INFO | "gpu" : " 17 GB", | 20260507 05:15:14.190 | INFO | "huge" : "", | 20260507 05:15:14.190 | INFO | "huge_shared" : " 27 GB", | 20260507 05:15:14.190 | INFO | "malloc" : " 14 GB", | 20260507 05:15:14.190 | INFO | "pinned" : " 29 GB" | 20260507 05:15:14.190 | INFO | } | 20260507 05:15:14.190 | INFO | Perf report for the last 5.13914 seconds: | 20260507 05:15:14.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:15:14.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:14.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:14.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:14.190 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260507 05:15:14.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:14.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:14.190 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:15:14.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:14.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:14.190 | INFO | Uploaded 36GiB to GPU in 2.05395s (17.5272GiB/s) | 20260507 05:15:14.190 | INFO | Downloaded 34GiB to GPU in 1.94848s (17.4495GiB/s) | 20260507 05:15:19.323 | INFO | Analyzing chunk frames = [7374336,7374848) | 20260507 05:15:19.323 | INFO | MemInfo: { | 20260507 05:15:19.323 | INFO | "gpu" : " 17 GB", | 20260507 05:15:19.323 | INFO | "huge" : "", | 20260507 05:15:19.323 | INFO | "huge_shared" : " 27 GB", | 20260507 05:15:19.323 | INFO | "malloc" : " 14 GB", | 20260507 05:15:19.323 | INFO | "pinned" : " 29 GB" | 20260507 05:15:19.323 | INFO | } | 20260507 05:15:19.323 | INFO | Perf report for the last 5.13318 seconds: | 20260507 05:15:19.323 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:15:19.323 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:19.323 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:19.323 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:19.323 | INFO | ANALYSIS : 60.4%, 2.9, 93.1ms, 0 | 20260507 05:15:19.323 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:19.323 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:19.323 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:15:19.323 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:19.323 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:19.323 | INFO | Uploaded 36GiB to GPU in 2.03514s (17.6892GiB/s) | 20260507 05:15:19.323 | INFO | Downloaded 34GiB to GPU in 1.94996s (17.4363GiB/s) | 20260507 05:15:24.461 | INFO | Analyzing chunk frames = [7374848,7375360) | 20260507 05:15:24.461 | INFO | MemInfo: { | 20260507 05:15:24.461 | INFO | "gpu" : " 17 GB", | 20260507 05:15:24.461 | INFO | "huge" : "", | 20260507 05:15:24.461 | INFO | "huge_shared" : " 27 GB", | 20260507 05:15:24.461 | INFO | "malloc" : " 14 GB", | 20260507 05:15:24.461 | INFO | "pinned" : " 29 GB" | 20260507 05:15:24.461 | INFO | } | 20260507 05:15:24.461 | INFO | Perf report for the last 5.13833 seconds: | 20260507 05:15:24.461 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:15:24.461 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:24.461 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:24.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:24.461 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 05:15:24.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:24.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:24.462 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:15:24.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:24.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:24.462 | INFO | Uploaded 36GiB to GPU in 2.0419s (17.6307GiB/s) | 20260507 05:15:24.462 | INFO | Downloaded 34GiB to GPU in 1.93973s (17.5282GiB/s) | 20260507 05:15:29.603 | INFO | Analyzing chunk frames = [7375360,7375872) | 20260507 05:15:29.603 | INFO | MemInfo: { | 20260507 05:15:29.603 | INFO | "gpu" : " 17 GB", | 20260507 05:15:29.603 | INFO | "huge" : "", | 20260507 05:15:29.603 | INFO | "huge_shared" : " 27 GB", | 20260507 05:15:29.603 | INFO | "malloc" : " 14 GB", | 20260507 05:15:29.603 | INFO | "pinned" : " 29 GB" | 20260507 05:15:29.603 | INFO | } | 20260507 05:15:29.603 | INFO | Perf report for the last 5.14147 seconds: | 20260507 05:15:29.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:15:29.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:29.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:29.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:29.603 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 05:15:29.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:29.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:29.603 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:15:29.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:29.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:29.603 | INFO | Uploaded 36GiB to GPU in 2.03391s (17.6999GiB/s) | 20260507 05:15:29.603 | INFO | Downloaded 34GiB to GPU in 1.92411s (17.6705GiB/s) | 20260507 05:15:34.741 | INFO | Analyzing chunk frames = [7375872,7376384) | 20260507 05:15:34.741 | INFO | MemInfo: { | 20260507 05:15:34.741 | INFO | "gpu" : " 17 GB", | 20260507 05:15:34.741 | INFO | "huge" : "", | 20260507 05:15:34.741 | INFO | "huge_shared" : " 27 GB", | 20260507 05:15:34.741 | INFO | "malloc" : " 14 GB", | 20260507 05:15:34.741 | INFO | "pinned" : " 29 GB" | 20260507 05:15:34.741 | INFO | } | 20260507 05:15:34.741 | INFO | Perf report for the last 5.13844 seconds: | 20260507 05:15:34.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:15:34.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:34.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:34.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:34.742 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 05:15:34.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:34.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:34.742 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 05:15:34.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:34.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:34.742 | INFO | Uploaded 36GiB to GPU in 2.00981s (17.9121GiB/s) | 20260507 05:15:34.742 | INFO | Downloaded 34GiB to GPU in 1.97684s (17.1992GiB/s) | 20260507 05:15:39.882 | INFO | Analyzing chunk frames = [7376384,7376896) | 20260507 05:15:39.882 | INFO | MemInfo: { | 20260507 05:15:39.882 | INFO | "gpu" : " 17 GB", | 20260507 05:15:39.882 | INFO | "huge" : "", | 20260507 05:15:39.882 | INFO | "huge_shared" : " 27 GB", | 20260507 05:15:39.882 | INFO | "malloc" : " 14 GB", | 20260507 05:15:39.882 | INFO | "pinned" : " 29 GB" | 20260507 05:15:39.882 | INFO | } | 20260507 05:15:39.882 | INFO | Perf report for the last 5.14091 seconds: | 20260507 05:15:39.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:15:39.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:39.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:39.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:39.883 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 05:15:39.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:39.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:39.883 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 05:15:39.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:39.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:39.883 | INFO | Uploaded 36GiB to GPU in 1.96084s (18.3594GiB/s) | 20260507 05:15:39.883 | INFO | Downloaded 34GiB to GPU in 1.95977s (17.349GiB/s) | 20260507 05:15:39.981 | INFO | Starting a round of Baz IO | 20260507 05:15:43.078 | INFO | Populating RTMetrics report | 20260507 05:15:43.079 | INFO | RTMetrics report populated | 20260507 05:15:43.079 | INFO | Opening tmp RTMetrics json file | 20260507 05:15:43.079 | INFO | Serializing RTMetrics to json | 20260507 05:15:43.080 | INFO | Serialization done, swapping tmp file now... | 20260507 05:15:43.080 | INFO | RTMetrics done | 20260507 05:15:43.168 | INFO | Finished a round of Baz IO: Wrote 1.703 GiB in 3.187 seconds | 20260507 05:15:43.168 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.72% / 0.00% / 67.28% | 20260507 05:15:43.168 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.148924% / 0% / 9.93411e-05% | 20260507 05:15:43.168 | INFO | Baz IO overall: | 20260507 05:15:43.168 | INFO | BytesWritten : 3747208745794 | 20260507 05:15:43.168 | INFO | NumEvents : 1797673000358 | 20260507 05:15:45.019 | INFO | Analyzing chunk frames = [7376896,7377408) | 20260507 05:15:45.019 | INFO | MemInfo: { | 20260507 05:15:45.019 | INFO | "gpu" : " 17 GB", | 20260507 05:15:45.019 | INFO | "huge" : "", | 20260507 05:15:45.019 | INFO | "huge_shared" : " 27 GB", | 20260507 05:15:45.019 | INFO | "malloc" : " 14 GB", | 20260507 05:15:45.019 | INFO | "pinned" : " 29 GB" | 20260507 05:15:45.019 | INFO | } | 20260507 05:15:45.019 | INFO | Perf report for the last 5.13712 seconds: | 20260507 05:15:45.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:15:45.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:45.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:45.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:45.020 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:15:45.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:45.020 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 05:15:45.020 | INFO | PRE_HQ : 59.5%, 1.9, 58.9ms, 0 | 20260507 05:15:45.020 | INFO | BAZWRITER : 61.7%, 3.3, 105.7ms, 0 | 20260507 05:15:45.020 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 05:15:45.020 | INFO | Uploaded 36GiB to GPU in 2.03802s (17.6642GiB/s) | 20260507 05:15:45.020 | INFO | Downloaded 38GiB to GPU in 2.1797s (17.4336GiB/s) | 20260507 05:15:50.162 | INFO | Analyzing chunk frames = [7377408,7377920) | 20260507 05:15:50.162 | INFO | MemInfo: { | 20260507 05:15:50.162 | INFO | "gpu" : " 17 GB", | 20260507 05:15:50.162 | INFO | "huge" : "", | 20260507 05:15:50.162 | INFO | "huge_shared" : " 27 GB", | 20260507 05:15:50.162 | INFO | "malloc" : " 14 GB", | 20260507 05:15:50.163 | INFO | "pinned" : " 29 GB" | 20260507 05:15:50.163 | INFO | } | 20260507 05:15:50.163 | INFO | Perf report for the last 5.14324 seconds: | 20260507 05:15:50.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:15:50.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:50.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:50.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:50.163 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 05:15:50.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:50.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:50.163 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 05:15:50.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:50.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:50.163 | INFO | Uploaded 36GiB to GPU in 2.00122s (17.989GiB/s) | 20260507 05:15:50.163 | INFO | Downloaded 34GiB to GPU in 1.93691s (17.5537GiB/s) | 20260507 05:15:55.301 | INFO | Analyzing chunk frames = [7377920,7378432) | 20260507 05:15:55.301 | INFO | MemInfo: { | 20260507 05:15:55.301 | INFO | "gpu" : " 17 GB", | 20260507 05:15:55.301 | INFO | "huge" : "", | 20260507 05:15:55.301 | INFO | "huge_shared" : " 27 GB", | 20260507 05:15:55.301 | INFO | "malloc" : " 14 GB", | 20260507 05:15:55.301 | INFO | "pinned" : " 29 GB" | 20260507 05:15:55.301 | INFO | } | 20260507 05:15:55.301 | INFO | Perf report for the last 5.13801 seconds: | 20260507 05:15:55.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:15:55.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:55.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:55.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:55.301 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 05:15:55.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:55.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:55.301 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:15:55.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:55.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:15:55.301 | INFO | Uploaded 36GiB to GPU in 2.03366s (17.7021GiB/s) | 20260507 05:15:55.301 | INFO | Downloaded 34GiB to GPU in 1.95303s (17.4088GiB/s) | 20260507 05:16:00.441 | INFO | Analyzing chunk frames = [7378432,7378944) | 20260507 05:16:00.441 | INFO | MemInfo: { | 20260507 05:16:00.441 | INFO | "gpu" : " 17 GB", | 20260507 05:16:00.441 | INFO | "huge" : "", | 20260507 05:16:00.441 | INFO | "huge_shared" : " 27 GB", | 20260507 05:16:00.441 | INFO | "malloc" : " 14 GB", | 20260507 05:16:00.441 | INFO | "pinned" : " 29 GB" | 20260507 05:16:00.441 | INFO | } | 20260507 05:16:00.441 | INFO | Perf report for the last 5.14049 seconds: | 20260507 05:16:00.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:16:00.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:00.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:00.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:00.441 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 05:16:00.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:00.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:00.441 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:16:00.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:00.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:00.441 | INFO | Uploaded 36GiB to GPU in 2.00395s (17.9645GiB/s) | 20260507 05:16:00.441 | INFO | Downloaded 34GiB to GPU in 1.93955s (17.5299GiB/s) | 20260507 05:16:05.574 | INFO | Analyzing chunk frames = [7378944,7379456) | 20260507 05:16:05.574 | INFO | MemInfo: { | 20260507 05:16:05.574 | INFO | "gpu" : " 17 GB", | 20260507 05:16:05.574 | INFO | "huge" : "", | 20260507 05:16:05.574 | INFO | "huge_shared" : " 27 GB", | 20260507 05:16:05.574 | INFO | "malloc" : " 14 GB", | 20260507 05:16:05.574 | INFO | "pinned" : " 29 GB" | 20260507 05:16:05.574 | INFO | } | 20260507 05:16:05.574 | INFO | Perf report for the last 5.13263 seconds: | 20260507 05:16:05.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:16:05.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:05.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:05.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:05.574 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260507 05:16:05.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:05.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:05.574 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:16:05.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:05.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:05.574 | INFO | Uploaded 36GiB to GPU in 2.02942s (17.7391GiB/s) | 20260507 05:16:05.574 | INFO | Downloaded 34GiB to GPU in 1.9221s (17.6889GiB/s) | 20260507 05:16:10.715 | INFO | Analyzing chunk frames = [7379456,7379968) | 20260507 05:16:10.715 | INFO | MemInfo: { | 20260507 05:16:10.715 | INFO | "gpu" : " 17 GB", | 20260507 05:16:10.715 | INFO | "huge" : "", | 20260507 05:16:10.715 | INFO | "huge_shared" : " 27 GB", | 20260507 05:16:10.715 | INFO | "malloc" : " 14 GB", | 20260507 05:16:10.715 | INFO | "pinned" : " 29 GB" | 20260507 05:16:10.715 | INFO | } | 20260507 05:16:10.715 | INFO | Perf report for the last 5.14123 seconds: | 20260507 05:16:10.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:16:10.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:10.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:10.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:10.715 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 05:16:10.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:10.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:10.715 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:16:10.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:10.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:10.715 | INFO | Uploaded 36GiB to GPU in 2.02702s (17.7601GiB/s) | 20260507 05:16:10.715 | INFO | Downloaded 34GiB to GPU in 1.93352s (17.5845GiB/s) | 20260507 05:16:15.862 | INFO | Analyzing chunk frames = [7379968,7380480) | 20260507 05:16:15.862 | INFO | MemInfo: { | 20260507 05:16:15.862 | INFO | "gpu" : " 17 GB", | 20260507 05:16:15.862 | INFO | "huge" : "", | 20260507 05:16:15.862 | INFO | "huge_shared" : " 27 GB", | 20260507 05:16:15.862 | INFO | "malloc" : " 14 GB", | 20260507 05:16:15.862 | INFO | "pinned" : " 29 GB" | 20260507 05:16:15.862 | INFO | } | 20260507 05:16:15.862 | INFO | Perf report for the last 5.14669 seconds: | 20260507 05:16:15.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:16:15.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:15.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:15.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:15.862 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 05:16:15.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:15.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:15.862 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 05:16:15.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:15.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:15.862 | INFO | Uploaded 36GiB to GPU in 1.95616s (18.4034GiB/s) | 20260507 05:16:15.862 | INFO | Downloaded 34GiB to GPU in 1.95985s (17.3482GiB/s) | 20260507 05:16:21.003 | INFO | Analyzing chunk frames = [7380480,7380992) | 20260507 05:16:21.003 | INFO | MemInfo: { | 20260507 05:16:21.003 | INFO | "gpu" : " 17 GB", | 20260507 05:16:21.003 | INFO | "huge" : "", | 20260507 05:16:21.003 | INFO | "huge_shared" : " 27 GB", | 20260507 05:16:21.003 | INFO | "malloc" : " 14 GB", | 20260507 05:16:21.003 | INFO | "pinned" : " 29 GB" | 20260507 05:16:21.003 | INFO | } | 20260507 05:16:21.004 | INFO | Perf report for the last 5.14175 seconds: | 20260507 05:16:21.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:16:21.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:21.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:21.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:21.004 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:16:21.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:21.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:21.004 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:16:21.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:21.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:21.004 | INFO | Uploaded 36GiB to GPU in 1.98498s (18.1362GiB/s) | 20260507 05:16:21.004 | INFO | Downloaded 34GiB to GPU in 1.96096s (17.3385GiB/s) | 20260507 05:16:21.004 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:16:21.004 | INFO | Total mem usage: 18 GiB | 20260507 05:16:21.004 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:16:21.004 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:16:21.004 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:16:21.004 | INFO | High Water Breakdown: | 20260507 05:16:21.004 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:16:21.004 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:16:21.004 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:16:21.004 | INFO | ------------------------------------------ | 20260507 05:16:21.004 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:16:21.004 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:16:21.004 | INFO | Total mem usage: 26.5 GiB | 20260507 05:16:21.004 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:16:21.004 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:16:21.004 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:16:21.004 | INFO | High Water Breakdown: | 20260507 05:16:21.004 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:16:21.004 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:16:21.004 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:16:21.004 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:16:21.004 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:16:21.004 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:16:21.004 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:16:21.004 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:16:21.004 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:16:21.004 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:16:21.004 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:16:21.004 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:16:21.004 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:16:21.004 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:16:21.004 | INFO | ------------------------------------------ | 20260507 05:16:21.004 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:16:21.004 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:16:21.004 | INFO | Total mem usage: 25.6 GiB | 20260507 05:16:21.004 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:16:21.004 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:16:21.004 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:16:21.004 | INFO | High Water Breakdown: | 20260507 05:16:21.004 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:16:21.004 | INFO | ------------------------------------------ | 20260507 05:16:21.004 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:16:21.005 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:16:21.005 | INFO | Total mem usage: 15.7 GiB | 20260507 05:16:21.005 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:16:21.005 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:16:21.005 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:16:21.005 | INFO | High Water Breakdown: | 20260507 05:16:21.005 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:16:21.005 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:16:21.005 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:16:21.005 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:16:21.005 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:16:21.005 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:16:21.005 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:16:21.005 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:16:21.005 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:16:21.005 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:16:21.005 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:16:21.005 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:16:21.005 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:16:21.005 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:16:21.005 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:16:21.005 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:16:21.005 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:16:21.005 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:16:21.005 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:16:21.005 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:16:21.005 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:16:21.005 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:16:21.005 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:16:21.005 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:16:21.005 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:16:21.005 | INFO | ------------------------------------------ | 20260507 05:16:21.005 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:16:21.005 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:16:21.005 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:16:21.005 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:16:21.005 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.50e+01, 1.35e+01 / 1.18e+00 | 20260507 05:16:21.005 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 9.99e-02, 9.85e-03 / 6.11e-03 | 20260507 05:16:21.005 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.40e+00, 5.05e-01 / 2.86e-01 | 20260507 05:16:21.005 | INFO | PerfReport: 3.22%, Histogram -- 9.91e-01 / 1.44e+00, 1.66e-01 / 1.30e-01 | 20260507 05:16:21.005 | INFO | PerfReport: 16.24%, DME -- 3.99e+01 / 4.49e+01, 4.00e+00 / 1.42e+00 | 20260507 05:16:21.005 | INFO | PerfReport: 57.64%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.29e+00 / 1.31e+00 | 20260507 05:16:21.005 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.72e+00, 3.84e-01 / 1.69e-01 | 20260507 05:16:21.005 | INFO | PerfReport: 2.98%, Metrics -- 9.17e-01 / 1.24e+00, 2.43e-01 / 1.22e-01 | 20260507 05:16:21.005 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.11e-01 / 3.64e-01, 5.09e-02 / 2.01e-02 | 20260507 05:16:21.005 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:16:21.005 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:16:21.005 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:16:21.005 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 05:16:21.005 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260507 05:16:21.005 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 05:16:21.005 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 05:16:21.005 | INFO | PerfReport: 1.54%, LockUpload -- 1.48e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 05:16:21.005 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260507 05:16:21.005 | INFO | PerfReport: 22.03%, LockCompute -- 2.12e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 05:16:21.005 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.21e-02 / 0.00e+00 | 20260507 05:16:21.101 | INFO | Starting a round of Baz IO | 20260507 05:16:24.205 | INFO | Populating RTMetrics report | 20260507 05:16:24.206 | INFO | RTMetrics report populated | 20260507 05:16:24.206 | INFO | Opening tmp RTMetrics json file | 20260507 05:16:24.221 | INFO | Serializing RTMetrics to json | 20260507 05:16:24.223 | INFO | Serialization done, swapping tmp file now... | 20260507 05:16:24.224 | INFO | RTMetrics done | 20260507 05:16:24.293 | INFO | Finished a round of Baz IO: Wrote 1.701 GiB in 3.192 seconds | 20260507 05:16:24.293 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.72% / 0.00% / 67.28% | 20260507 05:16:24.293 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.148102% / 0% / 6.35783e-05% | 20260507 05:16:24.293 | INFO | Baz IO overall: | 20260507 05:16:24.294 | INFO | BytesWritten : 3749034649786 | 20260507 05:16:24.294 | INFO | NumEvents : 1798530346751 | 20260507 05:16:26.140 | INFO | Analyzing chunk frames = [7380992,7381504) | 20260507 05:16:26.140 | INFO | MemInfo: { | 20260507 05:16:26.140 | INFO | "gpu" : " 17 GB", | 20260507 05:16:26.140 | INFO | "huge" : "", | 20260507 05:16:26.140 | INFO | "huge_shared" : " 27 GB", | 20260507 05:16:26.140 | INFO | "malloc" : " 14 GB", | 20260507 05:16:26.140 | INFO | "pinned" : " 29 GB" | 20260507 05:16:26.140 | INFO | } | 20260507 05:16:26.140 | INFO | Perf report for the last 5.13656 seconds: | 20260507 05:16:26.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:16:26.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:26.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:26.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:26.140 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 05:16:26.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:26.140 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 05:16:26.140 | INFO | PRE_HQ : 59.0%, 1.8, 55.0ms, 0 | 20260507 05:16:26.140 | INFO | BAZWRITER : 61.7%, 3.2, 101.7ms, 0 | 20260507 05:16:26.140 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:16:26.140 | INFO | Uploaded 36GiB to GPU in 2.05388s (17.5278GiB/s) | 20260507 05:16:26.140 | INFO | Downloaded 38GiB to GPU in 2.16935s (17.5168GiB/s) | 20260507 05:16:31.274 | INFO | Analyzing chunk frames = [7381504,7382016) | 20260507 05:16:31.274 | INFO | MemInfo: { | 20260507 05:16:31.274 | INFO | "gpu" : " 17 GB", | 20260507 05:16:31.274 | INFO | "huge" : "", | 20260507 05:16:31.274 | INFO | "huge_shared" : " 27 GB", | 20260507 05:16:31.274 | INFO | "malloc" : " 14 GB", | 20260507 05:16:31.274 | INFO | "pinned" : " 29 GB" | 20260507 05:16:31.274 | INFO | } | 20260507 05:16:31.274 | INFO | Perf report for the last 5.13401 seconds: | 20260507 05:16:31.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:16:31.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:31.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:31.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:31.274 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:16:31.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:31.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:31.274 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:16:31.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:31.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:31.274 | INFO | Uploaded 36GiB to GPU in 1.99266s (18.0663GiB/s) | 20260507 05:16:31.275 | INFO | Downloaded 34GiB to GPU in 1.92325s (17.6784GiB/s) | 20260507 05:16:36.413 | INFO | Analyzing chunk frames = [7382016,7382528) | 20260507 05:16:36.413 | INFO | MemInfo: { | 20260507 05:16:36.413 | INFO | "gpu" : " 17 GB", | 20260507 05:16:36.413 | INFO | "huge" : "", | 20260507 05:16:36.413 | INFO | "huge_shared" : " 27 GB", | 20260507 05:16:36.413 | INFO | "malloc" : " 14 GB", | 20260507 05:16:36.413 | INFO | "pinned" : " 29 GB" | 20260507 05:16:36.413 | INFO | } | 20260507 05:16:36.413 | INFO | Perf report for the last 5.13874 seconds: | 20260507 05:16:36.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:16:36.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:36.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:36.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:36.413 | INFO | ANALYSIS : 60.2%, 3.0, 93.1ms, 0 | 20260507 05:16:36.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:36.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:36.413 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:16:36.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:36.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:36.413 | INFO | Uploaded 36GiB to GPU in 2.01692s (17.849GiB/s) | 20260507 05:16:36.413 | INFO | Downloaded 34GiB to GPU in 1.94205s (17.5073GiB/s) | 20260507 05:16:41.551 | INFO | Analyzing chunk frames = [7382528,7383040) | 20260507 05:16:41.551 | INFO | MemInfo: { | 20260507 05:16:41.551 | INFO | "gpu" : " 17 GB", | 20260507 05:16:41.551 | INFO | "huge" : "", | 20260507 05:16:41.551 | INFO | "huge_shared" : " 27 GB", | 20260507 05:16:41.551 | INFO | "malloc" : " 14 GB", | 20260507 05:16:41.551 | INFO | "pinned" : " 29 GB" | 20260507 05:16:41.551 | INFO | } | 20260507 05:16:41.552 | INFO | Perf report for the last 5.13865 seconds: | 20260507 05:16:41.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:16:41.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:41.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:41.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:41.552 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:16:41.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:41.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:41.552 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:16:41.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:41.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:41.552 | INFO | Uploaded 36GiB to GPU in 1.99232s (18.0694GiB/s) | 20260507 05:16:41.552 | INFO | Downloaded 34GiB to GPU in 1.92322s (17.6786GiB/s) | 20260507 05:16:46.693 | INFO | Analyzing chunk frames = [7383040,7383552) | 20260507 05:16:46.693 | INFO | MemInfo: { | 20260507 05:16:46.693 | INFO | "gpu" : " 17 GB", | 20260507 05:16:46.693 | INFO | "huge" : "", | 20260507 05:16:46.693 | INFO | "huge_shared" : " 27 GB", | 20260507 05:16:46.693 | INFO | "malloc" : " 14 GB", | 20260507 05:16:46.693 | INFO | "pinned" : " 29 GB" | 20260507 05:16:46.693 | INFO | } | 20260507 05:16:46.693 | INFO | Perf report for the last 5.14141 seconds: | 20260507 05:16:46.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:16:46.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:46.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:46.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:46.693 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 05:16:46.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:46.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:46.693 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 05:16:46.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:46.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:46.693 | INFO | Uploaded 36GiB to GPU in 2.05581s (17.5113GiB/s) | 20260507 05:16:46.693 | INFO | Downloaded 34GiB to GPU in 1.94291s (17.4995GiB/s) | 20260507 05:16:51.830 | INFO | Analyzing chunk frames = [7383552,7384064) | 20260507 05:16:51.830 | INFO | MemInfo: { | 20260507 05:16:51.830 | INFO | "gpu" : " 17 GB", | 20260507 05:16:51.830 | INFO | "huge" : "", | 20260507 05:16:51.830 | INFO | "huge_shared" : " 27 GB", | 20260507 05:16:51.830 | INFO | "malloc" : " 14 GB", | 20260507 05:16:51.830 | INFO | "pinned" : " 29 GB" | 20260507 05:16:51.830 | INFO | } | 20260507 05:16:51.830 | INFO | Perf report for the last 5.13741 seconds: | 20260507 05:16:51.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:16:51.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:51.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:51.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:51.831 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 05:16:51.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:51.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:51.831 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:16:51.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:51.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:51.831 | INFO | Uploaded 36GiB to GPU in 2.03187s (17.7177GiB/s) | 20260507 05:16:51.831 | INFO | Downloaded 34GiB to GPU in 1.89895s (17.9046GiB/s) | 20260507 05:16:56.969 | INFO | Analyzing chunk frames = [7384064,7384576) | 20260507 05:16:56.969 | INFO | MemInfo: { | 20260507 05:16:56.969 | INFO | "gpu" : " 17 GB", | 20260507 05:16:56.969 | INFO | "huge" : "", | 20260507 05:16:56.969 | INFO | "huge_shared" : " 27 GB", | 20260507 05:16:56.969 | INFO | "malloc" : " 14 GB", | 20260507 05:16:56.969 | INFO | "pinned" : " 29 GB" | 20260507 05:16:56.969 | INFO | } | 20260507 05:16:56.969 | INFO | Perf report for the last 5.13846 seconds: | 20260507 05:16:56.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:16:56.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:56.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:56.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:56.969 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 05:16:56.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:56.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:56.969 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 05:16:56.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:56.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:16:56.969 | INFO | Uploaded 36GiB to GPU in 1.98382s (18.1468GiB/s) | 20260507 05:16:56.969 | INFO | Downloaded 34GiB to GPU in 1.96883s (17.2691GiB/s) | 20260507 05:17:02.116 | INFO | Analyzing chunk frames = [7384576,7385088) | 20260507 05:17:02.116 | INFO | MemInfo: { | 20260507 05:17:02.116 | INFO | "gpu" : " 17 GB", | 20260507 05:17:02.116 | INFO | "huge" : "", | 20260507 05:17:02.116 | INFO | "huge_shared" : " 27 GB", | 20260507 05:17:02.116 | INFO | "malloc" : " 14 GB", | 20260507 05:17:02.116 | INFO | "pinned" : " 29 GB" | 20260507 05:17:02.116 | INFO | } | 20260507 05:17:02.116 | INFO | Perf report for the last 5.14749 seconds: | 20260507 05:17:02.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:17:02.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:02.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:02.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:02.117 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 05:17:02.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:02.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:02.117 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 05:17:02.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:02.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:02.117 | INFO | Uploaded 36GiB to GPU in 1.95774s (18.3886GiB/s) | 20260507 05:17:02.117 | INFO | Downloaded 34GiB to GPU in 1.95466s (17.3943GiB/s) | 20260507 05:17:02.213 | INFO | Starting a round of Baz IO | 20260507 05:17:05.312 | INFO | Populating RTMetrics report | 20260507 05:17:05.313 | INFO | RTMetrics report populated | 20260507 05:17:05.313 | INFO | Opening tmp RTMetrics json file | 20260507 05:17:05.318 | INFO | Serializing RTMetrics to json | 20260507 05:17:05.330 | INFO | Serialization done, swapping tmp file now... | 20260507 05:17:05.331 | INFO | RTMetrics done | 20260507 05:17:05.405 | INFO | Finished a round of Baz IO: Wrote 1.7 GiB in 3.191 seconds | 20260507 05:17:05.405 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.72% / 0.00% / 67.28% | 20260507 05:17:05.405 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.14774% / 0% / 4.37101e-05% | 20260507 05:17:05.405 | INFO | Baz IO overall: | 20260507 05:17:05.405 | INFO | BytesWritten : 3750859731795 | 20260507 05:17:05.405 | INFO | NumEvents : 1799387138688 | 20260507 05:17:07.254 | INFO | Analyzing chunk frames = [7385088,7385600) | 20260507 05:17:07.255 | INFO | MemInfo: { | 20260507 05:17:07.255 | INFO | "gpu" : " 17 GB", | 20260507 05:17:07.255 | INFO | "huge" : "", | 20260507 05:17:07.255 | INFO | "huge_shared" : " 27 GB", | 20260507 05:17:07.255 | INFO | "malloc" : " 14 GB", | 20260507 05:17:07.255 | INFO | "pinned" : " 29 GB" | 20260507 05:17:07.255 | INFO | } | 20260507 05:17:07.255 | INFO | Perf report for the last 5.13837 seconds: | 20260507 05:17:07.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:17:07.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:07.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:07.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:07.255 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 05:17:07.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:07.255 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260507 05:17:07.255 | INFO | PRE_HQ : 59.6%, 1.9, 59.4ms, 0 | 20260507 05:17:07.255 | INFO | BAZWRITER : 61.7%, 3.3, 105.5ms, 0 | 20260507 05:17:07.255 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 05:17:07.255 | INFO | Uploaded 36GiB to GPU in 2.05704s (17.5009GiB/s) | 20260507 05:17:07.255 | INFO | Downloaded 38GiB to GPU in 2.18898s (17.3597GiB/s) | 20260507 05:17:12.389 | INFO | Analyzing chunk frames = [7385600,7386112) | 20260507 05:17:12.389 | INFO | MemInfo: { | 20260507 05:17:12.389 | INFO | "gpu" : " 17 GB", | 20260507 05:17:12.389 | INFO | "huge" : "", | 20260507 05:17:12.389 | INFO | "huge_shared" : " 27 GB", | 20260507 05:17:12.389 | INFO | "malloc" : " 14 GB", | 20260507 05:17:12.389 | INFO | "pinned" : " 29 GB" | 20260507 05:17:12.389 | INFO | } | 20260507 05:17:12.389 | INFO | Perf report for the last 5.1341 seconds: | 20260507 05:17:12.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:17:12.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:12.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:12.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:12.389 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:17:12.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:12.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:12.389 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 05:17:12.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:12.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:12.389 | INFO | Uploaded 36GiB to GPU in 1.98908s (18.0988GiB/s) | 20260507 05:17:12.389 | INFO | Downloaded 34GiB to GPU in 1.94163s (17.511GiB/s) | 20260507 05:17:17.525 | INFO | Analyzing chunk frames = [7386112,7386624) | 20260507 05:17:17.525 | INFO | MemInfo: { | 20260507 05:17:17.525 | INFO | "gpu" : " 17 GB", | 20260507 05:17:17.525 | INFO | "huge" : "", | 20260507 05:17:17.525 | INFO | "huge_shared" : " 27 GB", | 20260507 05:17:17.525 | INFO | "malloc" : " 14 GB", | 20260507 05:17:17.525 | INFO | "pinned" : " 29 GB" | 20260507 05:17:17.525 | INFO | } | 20260507 05:17:17.526 | INFO | Perf report for the last 5.13669 seconds: | 20260507 05:17:17.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:17:17.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:17.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:17.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:17.526 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 05:17:17.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:17.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:17.526 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:17:17.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:17.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:17.526 | INFO | Uploaded 36GiB to GPU in 2.02561s (17.7724GiB/s) | 20260507 05:17:17.526 | INFO | Downloaded 34GiB to GPU in 1.9634s (17.3169GiB/s) | 20260507 05:17:22.668 | INFO | Analyzing chunk frames = [7386624,7387136) | 20260507 05:17:22.668 | INFO | MemInfo: { | 20260507 05:17:22.668 | INFO | "gpu" : " 17 GB", | 20260507 05:17:22.668 | INFO | "huge" : "", | 20260507 05:17:22.668 | INFO | "huge_shared" : " 27 GB", | 20260507 05:17:22.668 | INFO | "malloc" : " 14 GB", | 20260507 05:17:22.668 | INFO | "pinned" : " 29 GB" | 20260507 05:17:22.668 | INFO | } | 20260507 05:17:22.668 | INFO | Perf report for the last 5.14239 seconds: | 20260507 05:17:22.668 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:17:22.668 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:22.668 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:22.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:22.668 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 05:17:22.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:22.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:22.668 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:17:22.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:22.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:22.668 | INFO | Uploaded 36GiB to GPU in 1.98881s (18.1013GiB/s) | 20260507 05:17:22.668 | INFO | Downloaded 34GiB to GPU in 1.93239s (17.5948GiB/s) | 20260507 05:17:27.809 | INFO | Analyzing chunk frames = [7387136,7387648) | 20260507 05:17:27.809 | INFO | MemInfo: { | 20260507 05:17:27.809 | INFO | "gpu" : " 17 GB", | 20260507 05:17:27.809 | INFO | "huge" : "", | 20260507 05:17:27.809 | INFO | "huge_shared" : " 27 GB", | 20260507 05:17:27.809 | INFO | "malloc" : " 14 GB", | 20260507 05:17:27.809 | INFO | "pinned" : " 29 GB" | 20260507 05:17:27.809 | INFO | } | 20260507 05:17:27.809 | INFO | Perf report for the last 5.14149 seconds: | 20260507 05:17:27.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:17:27.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:27.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:27.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:27.810 | INFO | ANALYSIS : 59.9%, 3.0, 93.4ms, 0 | 20260507 05:17:27.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:27.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:27.810 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:17:27.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:27.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:27.810 | INFO | Uploaded 36GiB to GPU in 2.02664s (17.7634GiB/s) | 20260507 05:17:27.810 | INFO | Downloaded 34GiB to GPU in 1.93076s (17.6096GiB/s) | 20260507 05:17:32.951 | INFO | Analyzing chunk frames = [7387648,7388160) | 20260507 05:17:32.951 | INFO | MemInfo: { | 20260507 05:17:32.951 | INFO | "gpu" : " 17 GB", | 20260507 05:17:32.951 | INFO | "huge" : "", | 20260507 05:17:32.951 | INFO | "huge_shared" : " 27 GB", | 20260507 05:17:32.951 | INFO | "malloc" : " 14 GB", | 20260507 05:17:32.951 | INFO | "pinned" : " 29 GB" | 20260507 05:17:32.951 | INFO | } | 20260507 05:17:32.952 | INFO | Perf report for the last 5.14207 seconds: | 20260507 05:17:32.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:17:32.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:32.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:32.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:32.952 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 05:17:32.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:32.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:32.952 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:17:32.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:32.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:32.952 | INFO | Uploaded 36GiB to GPU in 2.05543s (17.5146GiB/s) | 20260507 05:17:32.952 | INFO | Downloaded 34GiB to GPU in 1.92332s (17.6778GiB/s) | 20260507 05:17:38.086 | INFO | Analyzing chunk frames = [7388160,7388672) | 20260507 05:17:38.086 | INFO | MemInfo: { | 20260507 05:17:38.086 | INFO | "gpu" : " 17 GB", | 20260507 05:17:38.086 | INFO | "huge" : "", | 20260507 05:17:38.086 | INFO | "huge_shared" : " 27 GB", | 20260507 05:17:38.086 | INFO | "malloc" : " 14 GB", | 20260507 05:17:38.086 | INFO | "pinned" : " 29 GB" | 20260507 05:17:38.086 | INFO | } | 20260507 05:17:38.086 | INFO | Perf report for the last 5.13456 seconds: | 20260507 05:17:38.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:17:38.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:38.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:38.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:38.086 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:17:38.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:38.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:38.086 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:17:38.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:38.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:38.086 | INFO | Uploaded 36GiB to GPU in 1.92153s (18.735GiB/s) | 20260507 05:17:38.086 | INFO | Downloaded 34GiB to GPU in 1.97894s (17.1809GiB/s) | 20260507 05:17:43.229 | INFO | Analyzing chunk frames = [7388672,7389184) | 20260507 05:17:43.229 | INFO | MemInfo: { | 20260507 05:17:43.229 | INFO | "gpu" : " 17 GB", | 20260507 05:17:43.229 | INFO | "huge" : "", | 20260507 05:17:43.229 | INFO | "huge_shared" : " 27 GB", | 20260507 05:17:43.229 | INFO | "malloc" : " 14 GB", | 20260507 05:17:43.229 | INFO | "pinned" : " 29 GB" | 20260507 05:17:43.229 | INFO | } | 20260507 05:17:43.229 | INFO | Perf report for the last 5.1433 seconds: | 20260507 05:17:43.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:17:43.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:43.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:43.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:43.229 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 05:17:43.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:43.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:43.230 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:17:43.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:43.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:43.230 | INFO | Uploaded 36GiB to GPU in 1.96517s (18.319GiB/s) | 20260507 05:17:43.230 | INFO | Downloaded 34GiB to GPU in 1.96405s (17.3112GiB/s) | 20260507 05:17:43.230 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:17:43.230 | INFO | Total mem usage: 18 GiB | 20260507 05:17:43.230 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:17:43.230 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.6%) | 20260507 05:17:43.230 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:17:43.230 | INFO | High Water Breakdown: | 20260507 05:17:43.230 | INFO | 60.2% : 8.63 GiB ( 75.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:17:43.230 | INFO | 39.6% : 5.68 GiB ( 84.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:17:43.230 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:17:43.230 | INFO | ------------------------------------------ | 20260507 05:17:43.230 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:17:43.230 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:17:43.230 | INFO | Total mem usage: 26.5 GiB | 20260507 05:17:43.230 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:17:43.230 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:17:43.230 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:17:43.230 | INFO | High Water Breakdown: | 20260507 05:17:43.230 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:17:43.230 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:17:43.230 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:17:43.230 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:17:43.230 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:17:43.230 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:17:43.230 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:17:43.230 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:17:43.230 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:17:43.230 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:17:43.230 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:17:43.230 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:17:43.230 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:17:43.230 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:17:43.230 | INFO | ------------------------------------------ | 20260507 05:17:43.230 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:17:43.230 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:17:43.230 | INFO | Total mem usage: 25.6 GiB | 20260507 05:17:43.230 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:17:43.230 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:17:43.230 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:17:43.230 | INFO | High Water Breakdown: | 20260507 05:17:43.230 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:17:43.230 | INFO | ------------------------------------------ | 20260507 05:17:43.230 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:17:43.230 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:17:43.230 | INFO | Total mem usage: 15.7 GiB | 20260507 05:17:43.230 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:17:43.230 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:17:43.231 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:17:43.231 | INFO | High Water Breakdown: | 20260507 05:17:43.231 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:17:43.231 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:17:43.231 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:17:43.231 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:17:43.231 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:17:43.231 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:17:43.231 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:17:43.231 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:17:43.231 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:17:43.231 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:17:43.231 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:17:43.231 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:17:43.231 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:17:43.231 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:17:43.231 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:17:43.231 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:17:43.231 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:17:43.231 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:17:43.231 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:17:43.231 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:17:43.231 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:17:43.231 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:17:43.231 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:17:43.231 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:17:43.231 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:17:43.231 | INFO | ------------------------------------------ | 20260507 05:17:43.231 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:17:43.231 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:17:43.231 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:17:43.231 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:17:43.231 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.34e+01 / 1.52e+00 | 20260507 05:17:43.231 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.02e-01, 9.86e-03 / 4.29e-03 | 20260507 05:17:43.231 | INFO | PerfReport: 11.69%, Baseline -- 3.60e+00 / 4.40e+00, 5.20e-01 / 2.80e-01 | 20260507 05:17:43.231 | INFO | PerfReport: 3.23%, Histogram -- 9.95e-01 / 1.48e+00, 1.71e-01 / 1.06e-01 | 20260507 05:17:43.231 | INFO | PerfReport: 16.16%, DME -- 3.97e+01 / 4.45e+01, 3.88e+00 / 1.65e+00 | 20260507 05:17:43.231 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.35e+00 / 1.42e+00 | 20260507 05:17:43.231 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.13e+00 / 2.66e+00, 3.78e-01 / 2.34e-01 | 20260507 05:17:43.231 | INFO | PerfReport: 2.99%, Metrics -- 9.20e-01 / 1.23e+00, 2.45e-01 / 1.25e-01 | 20260507 05:17:43.231 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.10e-01 / 3.63e-01, 5.06e-02 / 1.85e-02 | 20260507 05:17:43.231 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:17:43.231 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:17:43.231 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:17:43.231 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 05:17:43.231 | INFO | PerfReport: 21.76%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260507 05:17:43.231 | INFO | PerfReport: 21.38%, Download -- 2.05e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 05:17:43.231 | INFO | PerfReport: 32.00%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 05:17:43.231 | INFO | PerfReport: 1.43%, LockUpload -- 1.37e+00 / 0.00e+00, 4.78e+00 / 0.00e+00 | 20260507 05:17:43.231 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260507 05:17:43.231 | INFO | PerfReport: 22.11%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 05:17:43.231 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260507 05:17:43.326 | INFO | Starting a round of Baz IO | 20260507 05:17:46.424 | INFO | Populating RTMetrics report | 20260507 05:17:46.424 | INFO | RTMetrics report populated | 20260507 05:17:46.424 | INFO | Opening tmp RTMetrics json file | 20260507 05:17:46.442 | INFO | Serializing RTMetrics to json | 20260507 05:17:46.444 | INFO | Serialization done, swapping tmp file now... | 20260507 05:17:46.444 | INFO | RTMetrics done | 20260507 05:17:46.515 | INFO | Finished a round of Baz IO: Wrote 1.701 GiB in 3.188 seconds | 20260507 05:17:46.515 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.72% / 0.00% / 67.28% | 20260507 05:17:46.515 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.147665% / 0% / 5.16574e-05% | 20260507 05:17:46.515 | INFO | Baz IO overall: | 20260507 05:17:46.515 | INFO | BytesWritten : 3752685641867 | 20260507 05:17:46.515 | INFO | NumEvents : 1800244329885 | 20260507 05:17:48.369 | INFO | Analyzing chunk frames = [7389184,7389696) | 20260507 05:17:48.369 | INFO | MemInfo: { | 20260507 05:17:48.369 | INFO | "gpu" : " 17 GB", | 20260507 05:17:48.369 | INFO | "huge" : "", | 20260507 05:17:48.369 | INFO | "huge_shared" : " 27 GB", | 20260507 05:17:48.369 | INFO | "malloc" : " 14 GB", | 20260507 05:17:48.369 | INFO | "pinned" : " 29 GB" | 20260507 05:17:48.369 | INFO | } | 20260507 05:17:48.369 | INFO | Perf report for the last 5.13974 seconds: | 20260507 05:17:48.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:17:48.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:48.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:48.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:48.369 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:17:48.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:48.369 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 05:17:48.369 | INFO | PRE_HQ : 59.3%, 1.8, 55.7ms, 0 | 20260507 05:17:48.369 | INFO | BAZWRITER : 61.6%, 3.2, 101.8ms, 0 | 20260507 05:17:48.369 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 05:17:48.370 | INFO | Uploaded 36GiB to GPU in 2.0484s (17.5747GiB/s) | 20260507 05:17:48.370 | INFO | Downloaded 38GiB to GPU in 2.18044s (17.4277GiB/s) | 20260507 05:17:53.507 | INFO | Analyzing chunk frames = [7389696,7390208) | 20260507 05:17:53.507 | INFO | MemInfo: { | 20260507 05:17:53.507 | INFO | "gpu" : " 17 GB", | 20260507 05:17:53.508 | INFO | "huge" : "", | 20260507 05:17:53.508 | INFO | "huge_shared" : " 27 GB", | 20260507 05:17:53.508 | INFO | "malloc" : " 14 GB", | 20260507 05:17:53.508 | INFO | "pinned" : " 29 GB" | 20260507 05:17:53.508 | INFO | } | 20260507 05:17:53.508 | INFO | Perf report for the last 5.13857 seconds: | 20260507 05:17:53.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:17:53.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:53.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:53.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:53.508 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:17:53.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:53.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:53.508 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:17:53.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:53.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:53.508 | INFO | Uploaded 36GiB to GPU in 1.99702s (18.0269GiB/s) | 20260507 05:17:53.508 | INFO | Downloaded 34GiB to GPU in 1.93357s (17.5841GiB/s) | 20260507 05:17:58.644 | INFO | Analyzing chunk frames = [7390208,7390720) | 20260507 05:17:58.644 | INFO | MemInfo: { | 20260507 05:17:58.644 | INFO | "gpu" : " 17 GB", | 20260507 05:17:58.644 | INFO | "huge" : "", | 20260507 05:17:58.644 | INFO | "huge_shared" : " 27 GB", | 20260507 05:17:58.644 | INFO | "malloc" : " 14 GB", | 20260507 05:17:58.644 | INFO | "pinned" : " 29 GB" | 20260507 05:17:58.644 | INFO | } | 20260507 05:17:58.644 | INFO | Perf report for the last 5.13642 seconds: | 20260507 05:17:58.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:17:58.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:58.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:58.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:58.644 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 05:17:58.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:58.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:58.644 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:17:58.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:58.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:17:58.645 | INFO | Uploaded 36GiB to GPU in 2.05149s (17.5482GiB/s) | 20260507 05:17:58.645 | INFO | Downloaded 34GiB to GPU in 1.95094s (17.4275GiB/s) | 20260507 05:18:03.784 | INFO | Analyzing chunk frames = [7390720,7391232) | 20260507 05:18:03.784 | INFO | MemInfo: { | 20260507 05:18:03.784 | INFO | "gpu" : " 17 GB", | 20260507 05:18:03.784 | INFO | "huge" : "", | 20260507 05:18:03.785 | INFO | "huge_shared" : " 27 GB", | 20260507 05:18:03.785 | INFO | "malloc" : " 14 GB", | 20260507 05:18:03.785 | INFO | "pinned" : " 29 GB" | 20260507 05:18:03.785 | INFO | } | 20260507 05:18:03.785 | INFO | Perf report for the last 5.14043 seconds: | 20260507 05:18:03.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:18:03.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:03.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:03.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:03.785 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 05:18:03.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:03.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:03.785 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:18:03.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:03.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:03.785 | INFO | Uploaded 36GiB to GPU in 2.00086s (17.9922GiB/s) | 20260507 05:18:03.785 | INFO | Downloaded 34GiB to GPU in 1.93651s (17.5574GiB/s) | 20260507 05:18:08.923 | INFO | Analyzing chunk frames = [7391232,7391744) | 20260507 05:18:08.923 | INFO | MemInfo: { | 20260507 05:18:08.923 | INFO | "gpu" : " 17 GB", | 20260507 05:18:08.923 | INFO | "huge" : "", | 20260507 05:18:08.923 | INFO | "huge_shared" : " 27 GB", | 20260507 05:18:08.923 | INFO | "malloc" : " 14 GB", | 20260507 05:18:08.923 | INFO | "pinned" : " 29 GB" | 20260507 05:18:08.923 | INFO | } | 20260507 05:18:08.923 | INFO | Perf report for the last 5.13876 seconds: | 20260507 05:18:08.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:18:08.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:08.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:08.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:08.924 | INFO | ANALYSIS : 60.2%, 3.0, 94.0ms, 0 | 20260507 05:18:08.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:08.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:08.924 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:18:08.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:08.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:08.924 | INFO | Uploaded 36GiB to GPU in 2.05761s (17.496GiB/s) | 20260507 05:18:08.924 | INFO | Downloaded 34GiB to GPU in 1.9481s (17.4529GiB/s) | 20260507 05:18:14.064 | INFO | Analyzing chunk frames = [7391744,7392256) | 20260507 05:18:14.064 | INFO | MemInfo: { | 20260507 05:18:14.064 | INFO | "gpu" : " 17 GB", | 20260507 05:18:14.064 | INFO | "huge" : "", | 20260507 05:18:14.064 | INFO | "huge_shared" : " 27 GB", | 20260507 05:18:14.064 | INFO | "malloc" : " 14 GB", | 20260507 05:18:14.064 | INFO | "pinned" : " 29 GB" | 20260507 05:18:14.064 | INFO | } | 20260507 05:18:14.064 | INFO | Perf report for the last 5.14073 seconds: | 20260507 05:18:14.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:18:14.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:14.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:14.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:14.064 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 05:18:14.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:14.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:14.064 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 05:18:14.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:14.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:14.064 | INFO | Uploaded 36GiB to GPU in 2.02566s (17.772GiB/s) | 20260507 05:18:14.065 | INFO | Downloaded 34GiB to GPU in 1.9047s (17.8506GiB/s) | 20260507 05:18:19.201 | INFO | Analyzing chunk frames = [7392256,7392768) | 20260507 05:18:19.201 | INFO | MemInfo: { | 20260507 05:18:19.201 | INFO | "gpu" : " 17 GB", | 20260507 05:18:19.201 | INFO | "huge" : "", | 20260507 05:18:19.201 | INFO | "huge_shared" : " 27 GB", | 20260507 05:18:19.201 | INFO | "malloc" : " 14 GB", | 20260507 05:18:19.201 | INFO | "pinned" : " 29 GB" | 20260507 05:18:19.201 | INFO | } | 20260507 05:18:19.202 | INFO | Perf report for the last 5.13738 seconds: | 20260507 05:18:19.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:18:19.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:19.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:19.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:19.202 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:18:19.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:19.202 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:19.202 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 05:18:19.202 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:19.202 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:19.202 | INFO | Uploaded 36GiB to GPU in 1.91575s (18.7916GiB/s) | 20260507 05:18:19.202 | INFO | Downloaded 34GiB to GPU in 1.96934s (17.2646GiB/s) | 20260507 05:18:24.337 | INFO | Analyzing chunk frames = [7392768,7393280) | 20260507 05:18:24.337 | INFO | MemInfo: { | 20260507 05:18:24.337 | INFO | "gpu" : " 17 GB", | 20260507 05:18:24.337 | INFO | "huge" : "", | 20260507 05:18:24.337 | INFO | "huge_shared" : " 27 GB", | 20260507 05:18:24.337 | INFO | "malloc" : " 14 GB", | 20260507 05:18:24.337 | INFO | "pinned" : " 29 GB" | 20260507 05:18:24.337 | INFO | } | 20260507 05:18:24.337 | INFO | Perf report for the last 5.13578 seconds: | 20260507 05:18:24.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:18:24.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:24.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:24.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:24.338 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 05:18:24.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:24.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:24.338 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:18:24.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:24.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:24.338 | INFO | Uploaded 36GiB to GPU in 1.97909s (18.1902GiB/s) | 20260507 05:18:24.338 | INFO | Downloaded 34GiB to GPU in 1.96481s (17.3044GiB/s) | 20260507 05:18:24.435 | INFO | Starting a round of Baz IO | 20260507 05:18:27.534 | INFO | Populating RTMetrics report | 20260507 05:18:27.534 | INFO | RTMetrics report populated | 20260507 05:18:27.534 | INFO | Opening tmp RTMetrics json file | 20260507 05:18:27.537 | INFO | Serializing RTMetrics to json | 20260507 05:18:27.552 | INFO | Serialization done, swapping tmp file now... | 20260507 05:18:27.552 | INFO | RTMetrics done | 20260507 05:18:27.620 | INFO | Finished a round of Baz IO: Wrote 1.7 GiB in 3.185 seconds | 20260507 05:18:27.620 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.72% / 0.00% / 67.28% | 20260507 05:18:27.620 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.146834% / 0% / 4.76837e-05% | 20260507 05:18:27.620 | INFO | Baz IO overall: | 20260507 05:18:27.620 | INFO | BytesWritten : 3754511058366 | 20260507 05:18:27.620 | INFO | NumEvents : 1801101596388 | 20260507 05:18:29.478 | INFO | Analyzing chunk frames = [7393280,7393792) | 20260507 05:18:29.478 | INFO | MemInfo: { | 20260507 05:18:29.478 | INFO | "gpu" : " 17 GB", | 20260507 05:18:29.478 | INFO | "huge" : "", | 20260507 05:18:29.478 | INFO | "huge_shared" : " 27 GB", | 20260507 05:18:29.478 | INFO | "malloc" : " 14 GB", | 20260507 05:18:29.478 | INFO | "pinned" : " 29 GB" | 20260507 05:18:29.478 | INFO | } | 20260507 05:18:29.478 | INFO | Perf report for the last 5.14082 seconds: | 20260507 05:18:29.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:18:29.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:29.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:29.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:29.478 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:18:29.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:29.478 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 05:18:29.478 | INFO | PRE_HQ : 59.0%, 1.8, 55.0ms, 0 | 20260507 05:18:29.478 | INFO | BAZWRITER : 61.5%, 3.2, 100.5ms, 0 | 20260507 05:18:29.478 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 05:18:29.478 | INFO | Uploaded 36GiB to GPU in 2.03221s (17.7147GiB/s) | 20260507 05:18:29.479 | INFO | Downloaded 38GiB to GPU in 2.18061s (17.4263GiB/s) | 20260507 05:18:34.619 | INFO | Analyzing chunk frames = [7393792,7394304) | 20260507 05:18:34.619 | INFO | MemInfo: { | 20260507 05:18:34.619 | INFO | "gpu" : " 17 GB", | 20260507 05:18:34.619 | INFO | "huge" : "", | 20260507 05:18:34.619 | INFO | "huge_shared" : " 27 GB", | 20260507 05:18:34.619 | INFO | "malloc" : " 14 GB", | 20260507 05:18:34.619 | INFO | "pinned" : " 29 GB" | 20260507 05:18:34.619 | INFO | } | 20260507 05:18:34.619 | INFO | Perf report for the last 5.14101 seconds: | 20260507 05:18:34.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:18:34.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:34.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:34.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:34.619 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 05:18:34.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:34.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:34.619 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:18:34.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:34.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:34.620 | INFO | Uploaded 36GiB to GPU in 2.03478s (17.6924GiB/s) | 20260507 05:18:34.620 | INFO | Downloaded 34GiB to GPU in 1.94154s (17.5119GiB/s) | 20260507 05:18:39.768 | INFO | Analyzing chunk frames = [7394304,7394816) | 20260507 05:18:39.768 | INFO | MemInfo: { | 20260507 05:18:39.768 | INFO | "gpu" : " 17 GB", | 20260507 05:18:39.768 | INFO | "huge" : "", | 20260507 05:18:39.768 | INFO | "huge_shared" : " 27 GB", | 20260507 05:18:39.768 | INFO | "malloc" : " 14 GB", | 20260507 05:18:39.768 | INFO | "pinned" : " 29 GB" | 20260507 05:18:39.768 | INFO | } | 20260507 05:18:39.768 | INFO | Perf report for the last 5.14883 seconds: | 20260507 05:18:39.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:18:39.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:39.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:39.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:39.768 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 05:18:39.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:39.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:39.768 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:18:39.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:39.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:39.768 | INFO | Uploaded 36GiB to GPU in 2.02627s (17.7667GiB/s) | 20260507 05:18:39.768 | INFO | Downloaded 34GiB to GPU in 1.96114s (17.3369GiB/s) | 20260507 05:18:44.898 | INFO | Analyzing chunk frames = [7394816,7395328) | 20260507 05:18:44.898 | INFO | MemInfo: { | 20260507 05:18:44.898 | INFO | "gpu" : " 17 GB", | 20260507 05:18:44.898 | INFO | "huge" : "", | 20260507 05:18:44.898 | INFO | "huge_shared" : " 27 GB", | 20260507 05:18:44.898 | INFO | "malloc" : " 14 GB", | 20260507 05:18:44.898 | INFO | "pinned" : " 29 GB" | 20260507 05:18:44.898 | INFO | } | 20260507 05:18:44.898 | INFO | Perf report for the last 5.13012 seconds: | 20260507 05:18:44.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:18:44.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:44.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:44.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:44.898 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 05:18:44.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:44.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:44.898 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:18:44.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:44.898 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:44.899 | INFO | Uploaded 36GiB to GPU in 2.045s (17.6039GiB/s) | 20260507 05:18:44.899 | INFO | Downloaded 34GiB to GPU in 1.94963s (17.4392GiB/s) | 20260507 05:18:50.031 | INFO | Analyzing chunk frames = [7395328,7395840) | 20260507 05:18:50.031 | INFO | MemInfo: { | 20260507 05:18:50.031 | INFO | "gpu" : " 17 GB", | 20260507 05:18:50.031 | INFO | "huge" : "", | 20260507 05:18:50.031 | INFO | "huge_shared" : " 27 GB", | 20260507 05:18:50.031 | INFO | "malloc" : " 14 GB", | 20260507 05:18:50.031 | INFO | "pinned" : " 29 GB" | 20260507 05:18:50.031 | INFO | } | 20260507 05:18:50.031 | INFO | Perf report for the last 5.13283 seconds: | 20260507 05:18:50.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:18:50.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:50.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:50.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:50.031 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 05:18:50.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:50.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:50.031 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:18:50.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:50.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:50.031 | INFO | Uploaded 36GiB to GPU in 2.02656s (17.7641GiB/s) | 20260507 05:18:50.031 | INFO | Downloaded 34GiB to GPU in 1.91632s (17.7423GiB/s) | 20260507 05:18:55.174 | INFO | Analyzing chunk frames = [7395840,7396352) | 20260507 05:18:55.174 | INFO | MemInfo: { | 20260507 05:18:55.174 | INFO | "gpu" : " 17 GB", | 20260507 05:18:55.174 | INFO | "huge" : "", | 20260507 05:18:55.174 | INFO | "huge_shared" : " 27 GB", | 20260507 05:18:55.174 | INFO | "malloc" : " 14 GB", | 20260507 05:18:55.174 | INFO | "pinned" : " 29 GB" | 20260507 05:18:55.174 | INFO | } | 20260507 05:18:55.174 | INFO | Perf report for the last 5.14273 seconds: | 20260507 05:18:55.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:18:55.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:55.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:55.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:55.174 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 05:18:55.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:55.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:55.174 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 05:18:55.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:55.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:18:55.174 | INFO | Uploaded 36GiB to GPU in 2.05259s (17.5388GiB/s) | 20260507 05:18:55.174 | INFO | Downloaded 34GiB to GPU in 1.97114s (17.2489GiB/s) | 20260507 05:19:00.317 | INFO | Analyzing chunk frames = [7396352,7396864) | 20260507 05:19:00.317 | INFO | MemInfo: { | 20260507 05:19:00.317 | INFO | "gpu" : " 17 GB", | 20260507 05:19:00.317 | INFO | "huge" : "", | 20260507 05:19:00.317 | INFO | "huge_shared" : " 27 GB", | 20260507 05:19:00.317 | INFO | "malloc" : " 14 GB", | 20260507 05:19:00.317 | INFO | "pinned" : " 29 GB" | 20260507 05:19:00.317 | INFO | } | 20260507 05:19:00.317 | INFO | Perf report for the last 5.14307 seconds: | 20260507 05:19:00.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:19:00.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:00.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:00.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:00.317 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:19:00.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:00.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:00.317 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 05:19:00.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:00.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:00.317 | INFO | Uploaded 36GiB to GPU in 1.96014s (18.366GiB/s) | 20260507 05:19:00.317 | INFO | Downloaded 34GiB to GPU in 1.97779s (17.1909GiB/s) | 20260507 05:19:05.453 | INFO | Analyzing chunk frames = [7396864,7397376) | 20260507 05:19:05.453 | INFO | MemInfo: { | 20260507 05:19:05.453 | INFO | "gpu" : " 17 GB", | 20260507 05:19:05.453 | INFO | "huge" : "", | 20260507 05:19:05.453 | INFO | "huge_shared" : " 27 GB", | 20260507 05:19:05.453 | INFO | "malloc" : " 14 GB", | 20260507 05:19:05.453 | INFO | "pinned" : " 29 GB" | 20260507 05:19:05.453 | INFO | } | 20260507 05:19:05.453 | INFO | Perf report for the last 5.13592 seconds: | 20260507 05:19:05.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:19:05.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:05.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:05.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:05.453 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:19:05.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:05.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:05.453 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:19:05.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:05.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:05.453 | INFO | Uploaded 36GiB to GPU in 1.97905s (18.1906GiB/s) | 20260507 05:19:05.453 | INFO | Downloaded 34GiB to GPU in 1.97422s (17.222GiB/s) | 20260507 05:19:05.453 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:19:05.453 | INFO | Total mem usage: 18 GiB | 20260507 05:19:05.453 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:19:05.453 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 05:19:05.453 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:19:05.453 | INFO | High Water Breakdown: | 20260507 05:19:05.453 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:19:05.453 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:19:05.453 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:19:05.453 | INFO | ------------------------------------------ | 20260507 05:19:05.453 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:19:05.453 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:19:05.453 | INFO | Total mem usage: 26.5 GiB | 20260507 05:19:05.453 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:19:05.453 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:19:05.453 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:19:05.453 | INFO | High Water Breakdown: | 20260507 05:19:05.453 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:19:05.453 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:19:05.453 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:19:05.453 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:19:05.453 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:19:05.453 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:19:05.453 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:19:05.453 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:19:05.453 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:19:05.453 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:19:05.453 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:19:05.453 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:19:05.453 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:19:05.453 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:19:05.453 | INFO | ------------------------------------------ | 20260507 05:19:05.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:19:05.454 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:19:05.454 | INFO | Total mem usage: 25.6 GiB | 20260507 05:19:05.454 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:19:05.454 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:19:05.454 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:19:05.454 | INFO | High Water Breakdown: | 20260507 05:19:05.454 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:19:05.454 | INFO | ------------------------------------------ | 20260507 05:19:05.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:19:05.454 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:19:05.454 | INFO | Total mem usage: 15.7 GiB | 20260507 05:19:05.454 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:19:05.454 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:19:05.454 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:19:05.454 | INFO | High Water Breakdown: | 20260507 05:19:05.454 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:19:05.454 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:19:05.454 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:19:05.454 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:19:05.454 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:19:05.454 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:19:05.454 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:19:05.454 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:19:05.454 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:19:05.454 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:19:05.454 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:19:05.454 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:19:05.454 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:19:05.454 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:19:05.454 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:19:05.454 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:19:05.454 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:19:05.454 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:19:05.454 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:19:05.454 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:19:05.454 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:19:05.454 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:19:05.454 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:19:05.454 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:19:05.454 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:19:05.454 | INFO | ------------------------------------------ | 20260507 05:19:05.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:19:05.454 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:19:05.454 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:19:05.454 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:19:05.454 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.34e+01 / 1.21e+00 | 20260507 05:19:05.454 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.01e-01, 9.53e-03 / 3.41e-03 | 20260507 05:19:05.454 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.48e+00, 5.05e-01 / 2.72e-01 | 20260507 05:19:05.454 | INFO | PerfReport: 3.21%, Histogram -- 9.86e-01 / 1.48e+00, 1.64e-01 / 1.07e-01 | 20260507 05:19:05.454 | INFO | PerfReport: 16.27%, DME -- 3.99e+01 / 4.50e+01, 3.90e+00 / 1.46e+00 | 20260507 05:19:05.454 | INFO | PerfReport: 57.58%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.18e+00 / 1.31e+00 | 20260507 05:19:05.454 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.14e+00 / 2.75e+00, 3.88e-01 / 1.58e-01 | 20260507 05:19:05.454 | INFO | PerfReport: 2.99%, Metrics -- 9.18e-01 / 1.23e+00, 2.37e-01 / 1.04e-01 | 20260507 05:19:05.454 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.10e-01 / 3.70e-01, 5.00e-02 / 2.07e-02 | 20260507 05:19:05.454 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:19:05.454 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:19:05.454 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:19:05.454 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 05:19:05.454 | INFO | PerfReport: 21.80%, Upload -- 2.10e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260507 05:19:05.454 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 05:19:05.454 | INFO | PerfReport: 31.89%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 05:19:05.454 | INFO | PerfReport: 1.54%, LockUpload -- 1.48e+00 / 0.00e+00, 4.89e+00 / 0.00e+00 | 20260507 05:19:05.454 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260507 05:19:05.454 | INFO | PerfReport: 22.02%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 05:19:05.454 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.26e-02 / 0.00e+00 | 20260507 05:19:05.550 | INFO | Starting a round of Baz IO | 20260507 05:19:08.646 | INFO | Populating RTMetrics report | 20260507 05:19:08.646 | INFO | RTMetrics report populated | 20260507 05:19:08.647 | INFO | Opening tmp RTMetrics json file | 20260507 05:19:08.648 | INFO | Serializing RTMetrics to json | 20260507 05:19:08.657 | INFO | Serialization done, swapping tmp file now... | 20260507 05:19:08.667 | INFO | RTMetrics done | 20260507 05:19:08.733 | INFO | Finished a round of Baz IO: Wrote 1.699 GiB in 3.183 seconds | 20260507 05:19:08.733 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.72% / 0.00% / 67.28% | 20260507 05:19:08.733 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.147029% / 3.97364e-06% / 7.54992e-05% | 20260507 05:19:08.733 | INFO | Baz IO overall: | 20260507 05:19:08.733 | INFO | BytesWritten : 3756335868545 | 20260507 05:19:08.733 | INFO | NumEvents : 1801958745346 | 20260507 05:19:10.595 | INFO | Analyzing chunk frames = [7397376,7397888) | 20260507 05:19:10.595 | INFO | MemInfo: { | 20260507 05:19:10.595 | INFO | "gpu" : " 17 GB", | 20260507 05:19:10.595 | INFO | "huge" : "", | 20260507 05:19:10.595 | INFO | "huge_shared" : " 27 GB", | 20260507 05:19:10.595 | INFO | "malloc" : " 14 GB", | 20260507 05:19:10.595 | INFO | "pinned" : " 29 GB" | 20260507 05:19:10.595 | INFO | } | 20260507 05:19:10.595 | INFO | Perf report for the last 5.14232 seconds: | 20260507 05:19:10.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:19:10.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:10.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:10.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:10.595 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:19:10.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:10.595 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 05:19:10.595 | INFO | PRE_HQ : 59.0%, 1.8, 54.9ms, 0 | 20260507 05:19:10.595 | INFO | BAZWRITER : 61.5%, 3.2, 100.1ms, 0 | 20260507 05:19:10.595 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:19:10.595 | INFO | Uploaded 36GiB to GPU in 2.03562s (17.685GiB/s) | 20260507 05:19:10.595 | INFO | Downloaded 38GiB to GPU in 2.1805s (17.4272GiB/s) | 20260507 05:19:15.733 | INFO | Analyzing chunk frames = [7397888,7398400) | 20260507 05:19:15.733 | INFO | MemInfo: { | 20260507 05:19:15.733 | INFO | "gpu" : " 17 GB", | 20260507 05:19:15.733 | INFO | "huge" : "", | 20260507 05:19:15.733 | INFO | "huge_shared" : " 27 GB", | 20260507 05:19:15.733 | INFO | "malloc" : " 14 GB", | 20260507 05:19:15.733 | INFO | "pinned" : " 29 GB" | 20260507 05:19:15.733 | INFO | } | 20260507 05:19:15.734 | INFO | Perf report for the last 5.13844 seconds: | 20260507 05:19:15.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:19:15.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:15.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:15.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:15.734 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:19:15.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:15.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:15.734 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:19:15.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:15.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:15.734 | INFO | Uploaded 36GiB to GPU in 1.99249s (18.0678GiB/s) | 20260507 05:19:15.734 | INFO | Downloaded 34GiB to GPU in 1.93422s (17.5781GiB/s) | 20260507 05:19:20.876 | INFO | Analyzing chunk frames = [7398400,7398912) | 20260507 05:19:20.876 | INFO | MemInfo: { | 20260507 05:19:20.876 | INFO | "gpu" : " 17 GB", | 20260507 05:19:20.876 | INFO | "huge" : "", | 20260507 05:19:20.876 | INFO | "huge_shared" : " 27 GB", | 20260507 05:19:20.876 | INFO | "malloc" : " 14 GB", | 20260507 05:19:20.876 | INFO | "pinned" : " 29 GB" | 20260507 05:19:20.876 | INFO | } | 20260507 05:19:20.876 | INFO | Perf report for the last 5.14255 seconds: | 20260507 05:19:20.876 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:19:20.876 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:20.876 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:20.876 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:20.876 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 05:19:20.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:20.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:20.876 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 05:19:20.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:20.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:20.876 | INFO | Uploaded 36GiB to GPU in 2.02426s (17.7843GiB/s) | 20260507 05:19:20.876 | INFO | Downloaded 34GiB to GPU in 1.95707s (17.3729GiB/s) | 20260507 05:19:26.014 | INFO | Analyzing chunk frames = [7398912,7399424) | 20260507 05:19:26.014 | INFO | MemInfo: { | 20260507 05:19:26.014 | INFO | "gpu" : " 17 GB", | 20260507 05:19:26.014 | INFO | "huge" : "", | 20260507 05:19:26.014 | INFO | "huge_shared" : " 27 GB", | 20260507 05:19:26.014 | INFO | "malloc" : " 14 GB", | 20260507 05:19:26.014 | INFO | "pinned" : " 29 GB" | 20260507 05:19:26.014 | INFO | } | 20260507 05:19:26.014 | INFO | Perf report for the last 5.13827 seconds: | 20260507 05:19:26.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:19:26.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:26.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:26.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:26.014 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:19:26.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:26.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:26.014 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:19:26.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:26.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:26.015 | INFO | Uploaded 36GiB to GPU in 2.00256s (17.977GiB/s) | 20260507 05:19:26.015 | INFO | Downloaded 34GiB to GPU in 1.94221s (17.5058GiB/s) | 20260507 05:19:31.152 | INFO | Analyzing chunk frames = [7399424,7399936) | 20260507 05:19:31.152 | INFO | MemInfo: { | 20260507 05:19:31.152 | INFO | "gpu" : " 17 GB", | 20260507 05:19:31.152 | INFO | "huge" : "", | 20260507 05:19:31.152 | INFO | "huge_shared" : " 27 GB", | 20260507 05:19:31.152 | INFO | "malloc" : " 14 GB", | 20260507 05:19:31.152 | INFO | "pinned" : " 29 GB" | 20260507 05:19:31.152 | INFO | } | 20260507 05:19:31.152 | INFO | Perf report for the last 5.13764 seconds: | 20260507 05:19:31.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:19:31.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:31.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:31.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:31.152 | INFO | ANALYSIS : 60.2%, 3.0, 94.0ms, 0 | 20260507 05:19:31.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:31.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:31.152 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:19:31.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:31.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:31.152 | INFO | Uploaded 36GiB to GPU in 2.0509s (17.5532GiB/s) | 20260507 05:19:31.152 | INFO | Downloaded 34GiB to GPU in 1.9529s (17.41GiB/s) | 20260507 05:19:36.289 | INFO | Analyzing chunk frames = [7399936,7400448) | 20260507 05:19:36.289 | INFO | MemInfo: { | 20260507 05:19:36.289 | INFO | "gpu" : " 17 GB", | 20260507 05:19:36.289 | INFO | "huge" : "", | 20260507 05:19:36.289 | INFO | "huge_shared" : " 27 GB", | 20260507 05:19:36.289 | INFO | "malloc" : " 14 GB", | 20260507 05:19:36.289 | INFO | "pinned" : " 29 GB" | 20260507 05:19:36.289 | INFO | } | 20260507 05:19:36.289 | INFO | Perf report for the last 5.13688 seconds: | 20260507 05:19:36.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:19:36.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:36.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:36.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:36.289 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 05:19:36.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:36.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:36.289 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 05:19:36.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:36.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:36.289 | INFO | Uploaded 36GiB to GPU in 2.02701s (17.7602GiB/s) | 20260507 05:19:36.289 | INFO | Downloaded 34GiB to GPU in 1.90941s (17.8065GiB/s) | 20260507 05:19:41.437 | INFO | Analyzing chunk frames = [7400448,7400960) | 20260507 05:19:41.437 | INFO | MemInfo: { | 20260507 05:19:41.437 | INFO | "gpu" : " 17 GB", | 20260507 05:19:41.437 | INFO | "huge" : "", | 20260507 05:19:41.437 | INFO | "huge_shared" : " 27 GB", | 20260507 05:19:41.437 | INFO | "malloc" : " 14 GB", | 20260507 05:19:41.437 | INFO | "pinned" : " 29 GB" | 20260507 05:19:41.437 | INFO | } | 20260507 05:19:41.438 | INFO | Perf report for the last 5.14859 seconds: | 20260507 05:19:41.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:19:41.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:41.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:41.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:41.438 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 05:19:41.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:41.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:41.438 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 05:19:41.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:41.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:41.438 | INFO | Uploaded 36GiB to GPU in 1.97669s (18.2123GiB/s) | 20260507 05:19:41.438 | INFO | Downloaded 34GiB to GPU in 1.97625s (17.2043GiB/s) | 20260507 05:19:46.576 | INFO | Analyzing chunk frames = [7400960,7401472) | 20260507 05:19:46.576 | INFO | MemInfo: { | 20260507 05:19:46.576 | INFO | "gpu" : " 17 GB", | 20260507 05:19:46.576 | INFO | "huge" : "", | 20260507 05:19:46.576 | INFO | "huge_shared" : " 27 GB", | 20260507 05:19:46.576 | INFO | "malloc" : " 14 GB", | 20260507 05:19:46.576 | INFO | "pinned" : " 29 GB" | 20260507 05:19:46.576 | INFO | } | 20260507 05:19:46.576 | INFO | Perf report for the last 5.13844 seconds: | 20260507 05:19:46.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:19:46.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:46.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:46.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:46.576 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:19:46.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:46.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:46.576 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 05:19:46.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:46.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:46.576 | INFO | Uploaded 36GiB to GPU in 1.98267s (18.1573GiB/s) | 20260507 05:19:46.576 | INFO | Downloaded 34GiB to GPU in 1.95859s (17.3595GiB/s) | 20260507 05:19:46.673 | INFO | Starting a round of Baz IO | 20260507 05:19:49.769 | INFO | Populating RTMetrics report | 20260507 05:19:49.770 | INFO | RTMetrics report populated | 20260507 05:19:49.770 | INFO | Opening tmp RTMetrics json file | 20260507 05:19:49.771 | INFO | Serializing RTMetrics to json | 20260507 05:19:49.788 | INFO | Serialization done, swapping tmp file now... | 20260507 05:19:49.789 | INFO | RTMetrics done | 20260507 05:19:49.855 | INFO | Finished a round of Baz IO: Wrote 1.699 GiB in 3.181 seconds | 20260507 05:19:49.855 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.71% / 0.00% / 67.28% | 20260507 05:19:49.855 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.146763% / 7.94729e-06% / 4.76837e-05% | 20260507 05:19:49.855 | INFO | Baz IO overall: | 20260507 05:19:49.855 | INFO | BytesWritten : 3758160691917 | 20260507 05:19:49.855 | INFO | NumEvents : 1802815818330 | 20260507 05:19:51.713 | INFO | Analyzing chunk frames = [7401472,7401984) | 20260507 05:19:51.713 | INFO | MemInfo: { | 20260507 05:19:51.713 | INFO | "gpu" : " 17 GB", | 20260507 05:19:51.713 | INFO | "huge" : "", | 20260507 05:19:51.713 | INFO | "huge_shared" : " 27 GB", | 20260507 05:19:51.713 | INFO | "malloc" : " 14 GB", | 20260507 05:19:51.713 | INFO | "pinned" : " 29 GB" | 20260507 05:19:51.713 | INFO | } | 20260507 05:19:51.713 | INFO | Perf report for the last 5.13737 seconds: | 20260507 05:19:51.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:19:51.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:51.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:51.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:51.713 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:19:51.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:51.713 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 05:19:51.713 | INFO | PRE_HQ : 59.1%, 1.8, 55.3ms, 0 | 20260507 05:19:51.713 | INFO | BAZWRITER : 61.5%, 3.2, 100.3ms, 0 | 20260507 05:19:51.713 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 05:19:51.714 | INFO | Uploaded 36GiB to GPU in 2.05431s (17.5241GiB/s) | 20260507 05:19:51.714 | INFO | Downloaded 38GiB to GPU in 2.17117s (17.502GiB/s) | 20260507 05:19:56.849 | INFO | Analyzing chunk frames = [7401984,7402496) | 20260507 05:19:56.849 | INFO | MemInfo: { | 20260507 05:19:56.849 | INFO | "gpu" : " 17 GB", | 20260507 05:19:56.849 | INFO | "huge" : "", | 20260507 05:19:56.849 | INFO | "huge_shared" : " 27 GB", | 20260507 05:19:56.849 | INFO | "malloc" : " 14 GB", | 20260507 05:19:56.849 | INFO | "pinned" : " 29 GB" | 20260507 05:19:56.849 | INFO | } | 20260507 05:19:56.849 | INFO | Perf report for the last 5.13567 seconds: | 20260507 05:19:56.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:19:56.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:56.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:56.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:56.849 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:19:56.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:56.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:56.849 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:19:56.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:56.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:19:56.849 | INFO | Uploaded 36GiB to GPU in 1.994s (18.0542GiB/s) | 20260507 05:19:56.849 | INFO | Downloaded 34GiB to GPU in 1.92855s (17.6298GiB/s) | 20260507 05:20:01.984 | INFO | Analyzing chunk frames = [7402496,7403008) | 20260507 05:20:01.984 | INFO | MemInfo: { | 20260507 05:20:01.984 | INFO | "gpu" : " 17 GB", | 20260507 05:20:01.984 | INFO | "huge" : "", | 20260507 05:20:01.984 | INFO | "huge_shared" : " 27 GB", | 20260507 05:20:01.984 | INFO | "malloc" : " 14 GB", | 20260507 05:20:01.984 | INFO | "pinned" : " 29 GB" | 20260507 05:20:01.984 | INFO | } | 20260507 05:20:01.985 | INFO | Perf report for the last 5.13548 seconds: | 20260507 05:20:01.985 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:20:01.985 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:01.985 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:01.985 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:01.985 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 05:20:01.985 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:01.985 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:01.985 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 05:20:01.985 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:01.985 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:01.985 | INFO | Uploaded 36GiB to GPU in 2.01296s (17.8841GiB/s) | 20260507 05:20:01.985 | INFO | Downloaded 34GiB to GPU in 1.93945s (17.5307GiB/s) | 20260507 05:20:07.127 | INFO | Analyzing chunk frames = [7403008,7403520) | 20260507 05:20:07.127 | INFO | MemInfo: { | 20260507 05:20:07.127 | INFO | "gpu" : " 17 GB", | 20260507 05:20:07.127 | INFO | "huge" : "", | 20260507 05:20:07.127 | INFO | "huge_shared" : " 27 GB", | 20260507 05:20:07.128 | INFO | "malloc" : " 14 GB", | 20260507 05:20:07.128 | INFO | "pinned" : " 29 GB" | 20260507 05:20:07.128 | INFO | } | 20260507 05:20:07.128 | INFO | Perf report for the last 5.14306 seconds: | 20260507 05:20:07.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:20:07.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:07.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:07.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:07.128 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 05:20:07.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:07.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:07.128 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:20:07.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:07.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:07.128 | INFO | Uploaded 36GiB to GPU in 1.97218s (18.2539GiB/s) | 20260507 05:20:07.128 | INFO | Downloaded 34GiB to GPU in 1.9093s (17.8075GiB/s) | 20260507 05:20:12.273 | INFO | Analyzing chunk frames = [7403520,7404032) | 20260507 05:20:12.273 | INFO | MemInfo: { | 20260507 05:20:12.273 | INFO | "gpu" : " 17 GB", | 20260507 05:20:12.273 | INFO | "huge" : "", | 20260507 05:20:12.273 | INFO | "huge_shared" : " 27 GB", | 20260507 05:20:12.273 | INFO | "malloc" : " 14 GB", | 20260507 05:20:12.273 | INFO | "pinned" : " 29 GB" | 20260507 05:20:12.273 | INFO | } | 20260507 05:20:12.273 | INFO | Perf report for the last 5.14526 seconds: | 20260507 05:20:12.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:20:12.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:12.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:12.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:12.273 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 05:20:12.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:12.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:12.273 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:20:12.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:12.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:12.273 | INFO | Uploaded 36GiB to GPU in 2.05649s (17.5056GiB/s) | 20260507 05:20:12.273 | INFO | Downloaded 34GiB to GPU in 1.94775s (17.4561GiB/s) | 20260507 05:20:17.418 | INFO | Analyzing chunk frames = [7404032,7404544) | 20260507 05:20:17.418 | INFO | MemInfo: { | 20260507 05:20:17.418 | INFO | "gpu" : " 17 GB", | 20260507 05:20:17.418 | INFO | "huge" : "", | 20260507 05:20:17.418 | INFO | "huge_shared" : " 27 GB", | 20260507 05:20:17.418 | INFO | "malloc" : " 14 GB", | 20260507 05:20:17.418 | INFO | "pinned" : " 29 GB" | 20260507 05:20:17.418 | INFO | } | 20260507 05:20:17.418 | INFO | Perf report for the last 5.14541 seconds: | 20260507 05:20:17.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:20:17.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:17.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:17.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:17.418 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 05:20:17.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:17.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:17.418 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 05:20:17.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:17.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:17.418 | INFO | Uploaded 36GiB to GPU in 2.07003s (17.391GiB/s) | 20260507 05:20:17.419 | INFO | Downloaded 34GiB to GPU in 1.93509s (17.5702GiB/s) | 20260507 05:20:22.545 | INFO | Analyzing chunk frames = [7404544,7405056) | 20260507 05:20:22.545 | INFO | MemInfo: { | 20260507 05:20:22.545 | INFO | "gpu" : " 17 GB", | 20260507 05:20:22.545 | INFO | "huge" : "", | 20260507 05:20:22.545 | INFO | "huge_shared" : " 27 GB", | 20260507 05:20:22.545 | INFO | "malloc" : " 14 GB", | 20260507 05:20:22.545 | INFO | "pinned" : " 29 GB" | 20260507 05:20:22.545 | INFO | } | 20260507 05:20:22.545 | INFO | Perf report for the last 5.12641 seconds: | 20260507 05:20:22.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:20:22.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:22.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:22.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:22.545 | INFO | ANALYSIS : 60.5%, 3.0, 94.0ms, 0 | 20260507 05:20:22.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:22.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:22.545 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 05:20:22.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:22.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:22.545 | INFO | Uploaded 36GiB to GPU in 1.90568s (18.8909GiB/s) | 20260507 05:20:22.545 | INFO | Downloaded 34GiB to GPU in 1.95589s (17.3834GiB/s) | 20260507 05:20:27.684 | INFO | Analyzing chunk frames = [7405056,7405568) | 20260507 05:20:27.684 | INFO | MemInfo: { | 20260507 05:20:27.684 | INFO | "gpu" : " 17 GB", | 20260507 05:20:27.684 | INFO | "huge" : "", | 20260507 05:20:27.684 | INFO | "huge_shared" : " 27 GB", | 20260507 05:20:27.684 | INFO | "malloc" : " 14 GB", | 20260507 05:20:27.684 | INFO | "pinned" : " 29 GB" | 20260507 05:20:27.684 | INFO | } | 20260507 05:20:27.684 | INFO | Perf report for the last 5.13966 seconds: | 20260507 05:20:27.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:20:27.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:27.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:27.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:27.684 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:20:27.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:27.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:27.685 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 05:20:27.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:27.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:27.685 | INFO | Uploaded 36GiB to GPU in 1.95552s (18.4094GiB/s) | 20260507 05:20:27.685 | INFO | Downloaded 34GiB to GPU in 1.96187s (17.3304GiB/s) | 20260507 05:20:27.685 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:20:27.685 | INFO | Total mem usage: 18 GiB | 20260507 05:20:27.685 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:20:27.685 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:20:27.685 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:20:27.685 | INFO | High Water Breakdown: | 20260507 05:20:27.685 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:20:27.685 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:20:27.685 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:20:27.685 | INFO | ------------------------------------------ | 20260507 05:20:27.685 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:20:27.685 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:20:27.685 | INFO | Total mem usage: 26.5 GiB | 20260507 05:20:27.685 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:20:27.685 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:20:27.685 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:20:27.685 | INFO | High Water Breakdown: | 20260507 05:20:27.685 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:20:27.685 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:20:27.685 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:20:27.685 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:20:27.685 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:20:27.685 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:20:27.685 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:20:27.685 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:20:27.685 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:20:27.685 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:20:27.685 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:20:27.685 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:20:27.685 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:20:27.685 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:20:27.685 | INFO | ------------------------------------------ | 20260507 05:20:27.685 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:20:27.685 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:20:27.685 | INFO | Total mem usage: 25.6 GiB | 20260507 05:20:27.685 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:20:27.685 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:20:27.685 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:20:27.685 | INFO | High Water Breakdown: | 20260507 05:20:27.685 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:20:27.685 | INFO | ------------------------------------------ | 20260507 05:20:27.685 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:20:27.685 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:20:27.685 | INFO | Total mem usage: 15.7 GiB | 20260507 05:20:27.685 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:20:27.685 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:20:27.685 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:20:27.685 | INFO | High Water Breakdown: | 20260507 05:20:27.685 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:20:27.685 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:20:27.685 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:20:27.685 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:20:27.685 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:20:27.685 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:20:27.685 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:20:27.685 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:20:27.685 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:20:27.685 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:20:27.685 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:20:27.685 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:20:27.685 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:20:27.686 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:20:27.686 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:20:27.686 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:20:27.686 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:20:27.686 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:20:27.686 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:20:27.686 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:20:27.686 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:20:27.686 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:20:27.686 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:20:27.686 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:20:27.686 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:20:27.686 | INFO | ------------------------------------------ | 20260507 05:20:27.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:20:27.686 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:20:27.686 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:20:27.686 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:20:27.686 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 1.40e+00 | 20260507 05:20:27.686 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.01e-01, 9.60e-03 / 4.43e-03 | 20260507 05:20:27.686 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.42e+00, 5.15e-01 / 3.11e-01 | 20260507 05:20:27.686 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.47e+00, 1.69e-01 / 1.15e-01 | 20260507 05:20:27.686 | INFO | PerfReport: 16.18%, DME -- 3.97e+01 / 4.44e+01, 3.90e+00 / 1.66e+00 | 20260507 05:20:27.686 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.77e+01 / 2.00e+01, 1.27e+00 / 1.60e+00 | 20260507 05:20:27.686 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.67e+00, 3.84e-01 / 2.06e-01 | 20260507 05:20:27.686 | INFO | PerfReport: 3.00%, Metrics -- 9.20e-01 / 1.24e+00, 2.46e-01 / 1.40e-01 | 20260507 05:20:27.686 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.10e-01 / 3.66e-01, 5.06e-02 / 1.77e-02 | 20260507 05:20:27.686 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:20:27.686 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:20:27.686 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:20:27.686 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 05:20:27.686 | INFO | PerfReport: 21.75%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260507 05:20:27.686 | INFO | PerfReport: 21.38%, Download -- 2.05e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260507 05:20:27.686 | INFO | PerfReport: 31.95%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 05:20:27.686 | INFO | PerfReport: 1.48%, LockUpload -- 1.42e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 05:20:27.686 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.14e+00 / 0.00e+00 | 20260507 05:20:27.686 | INFO | PerfReport: 22.11%, LockCompute -- 2.13e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 05:20:27.686 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.01e-02 / 0.00e+00 | 20260507 05:20:27.781 | INFO | Starting a round of Baz IO | 20260507 05:20:30.888 | INFO | Populating RTMetrics report | 20260507 05:20:30.889 | INFO | RTMetrics report populated | 20260507 05:20:30.889 | INFO | Opening tmp RTMetrics json file | 20260507 05:20:30.906 | INFO | Serializing RTMetrics to json | 20260507 05:20:30.907 | INFO | Serialization done, swapping tmp file now... | 20260507 05:20:30.908 | INFO | RTMetrics done | 20260507 05:20:30.975 | INFO | Finished a round of Baz IO: Wrote 1.699 GiB in 3.193 seconds | 20260507 05:20:30.975 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.71% / 0.00% / 67.29% | 20260507 05:20:30.975 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.146357% / 0% / 5.5631e-05% | 20260507 05:20:30.975 | INFO | Baz IO overall: | 20260507 05:20:30.975 | INFO | BytesWritten : 3759984945164 | 20260507 05:20:30.975 | INFO | NumEvents : 1803673089665 | 20260507 05:20:32.823 | INFO | Analyzing chunk frames = [7405568,7406080) | 20260507 05:20:32.823 | INFO | MemInfo: { | 20260507 05:20:32.823 | INFO | "gpu" : " 17 GB", | 20260507 05:20:32.823 | INFO | "huge" : "", | 20260507 05:20:32.823 | INFO | "huge_shared" : " 27 GB", | 20260507 05:20:32.823 | INFO | "malloc" : " 14 GB", | 20260507 05:20:32.823 | INFO | "pinned" : " 29 GB" | 20260507 05:20:32.823 | INFO | } | 20260507 05:20:32.823 | INFO | Perf report for the last 5.13863 seconds: | 20260507 05:20:32.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:20:32.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:32.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:32.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:32.823 | INFO | ANALYSIS : 61.8%, 3.0, 96.2ms, 0 | 20260507 05:20:32.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:32.823 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 05:20:32.823 | INFO | PRE_HQ : 59.0%, 1.8, 55.1ms, 0 | 20260507 05:20:32.823 | INFO | BAZWRITER : 61.7%, 3.1, 99.8ms, 0 | 20260507 05:20:32.823 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:20:32.823 | INFO | Uploaded 36GiB to GPU in 2.05103s (17.5522GiB/s) | 20260507 05:20:32.823 | INFO | Downloaded 38GiB to GPU in 2.18463s (17.3942GiB/s) | 20260507 05:20:37.960 | INFO | Analyzing chunk frames = [7406080,7406592) | 20260507 05:20:37.960 | INFO | MemInfo: { | 20260507 05:20:37.960 | INFO | "gpu" : " 17 GB", | 20260507 05:20:37.960 | INFO | "huge" : "", | 20260507 05:20:37.960 | INFO | "huge_shared" : " 27 GB", | 20260507 05:20:37.960 | INFO | "malloc" : " 14 GB", | 20260507 05:20:37.960 | INFO | "pinned" : " 29 GB" | 20260507 05:20:37.960 | INFO | } | 20260507 05:20:37.960 | INFO | Perf report for the last 5.13726 seconds: | 20260507 05:20:37.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:20:37.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:37.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:37.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:37.960 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:20:37.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:37.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:37.960 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:20:37.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:37.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:37.961 | INFO | Uploaded 36GiB to GPU in 1.9865s (18.1223GiB/s) | 20260507 05:20:37.961 | INFO | Downloaded 34GiB to GPU in 1.93023s (17.6145GiB/s) | 20260507 05:20:43.100 | INFO | Analyzing chunk frames = [7406592,7407104) | 20260507 05:20:43.100 | INFO | MemInfo: { | 20260507 05:20:43.100 | INFO | "gpu" : " 17 GB", | 20260507 05:20:43.100 | INFO | "huge" : "", | 20260507 05:20:43.100 | INFO | "huge_shared" : " 27 GB", | 20260507 05:20:43.100 | INFO | "malloc" : " 14 GB", | 20260507 05:20:43.100 | INFO | "pinned" : " 29 GB" | 20260507 05:20:43.100 | INFO | } | 20260507 05:20:43.100 | INFO | Perf report for the last 5.13967 seconds: | 20260507 05:20:43.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:20:43.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:43.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:43.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:43.100 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 05:20:43.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:43.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:43.100 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:20:43.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:43.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:43.100 | INFO | Uploaded 36GiB to GPU in 2.01445s (17.8709GiB/s) | 20260507 05:20:43.100 | INFO | Downloaded 34GiB to GPU in 1.94471s (17.4833GiB/s) | 20260507 05:20:48.242 | INFO | Analyzing chunk frames = [7407104,7407616) | 20260507 05:20:48.242 | INFO | MemInfo: { | 20260507 05:20:48.242 | INFO | "gpu" : " 17 GB", | 20260507 05:20:48.242 | INFO | "huge" : "", | 20260507 05:20:48.242 | INFO | "huge_shared" : " 27 GB", | 20260507 05:20:48.242 | INFO | "malloc" : " 14 GB", | 20260507 05:20:48.242 | INFO | "pinned" : " 29 GB" | 20260507 05:20:48.242 | INFO | } | 20260507 05:20:48.242 | INFO | Perf report for the last 5.14201 seconds: | 20260507 05:20:48.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:20:48.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:48.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:48.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:48.242 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 05:20:48.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:48.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:48.242 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:20:48.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:48.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:48.242 | INFO | Uploaded 36GiB to GPU in 1.98338s (18.1509GiB/s) | 20260507 05:20:48.242 | INFO | Downloaded 34GiB to GPU in 1.92488s (17.6634GiB/s) | 20260507 05:20:53.389 | INFO | Analyzing chunk frames = [7407616,7408128) | 20260507 05:20:53.389 | INFO | MemInfo: { | 20260507 05:20:53.389 | INFO | "gpu" : " 17 GB", | 20260507 05:20:53.389 | INFO | "huge" : "", | 20260507 05:20:53.389 | INFO | "huge_shared" : " 27 GB", | 20260507 05:20:53.389 | INFO | "malloc" : " 14 GB", | 20260507 05:20:53.389 | INFO | "pinned" : " 29 GB" | 20260507 05:20:53.389 | INFO | } | 20260507 05:20:53.389 | INFO | Perf report for the last 5.14742 seconds: | 20260507 05:20:53.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:20:53.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:53.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:53.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:53.389 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 05:20:53.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:53.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:53.389 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:20:53.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:53.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:53.390 | INFO | Uploaded 36GiB to GPU in 2.02659s (17.7638GiB/s) | 20260507 05:20:53.390 | INFO | Downloaded 34GiB to GPU in 1.94439s (17.4862GiB/s) | 20260507 05:20:58.527 | INFO | Analyzing chunk frames = [7408128,7408640) | 20260507 05:20:58.527 | INFO | MemInfo: { | 20260507 05:20:58.527 | INFO | "gpu" : " 17 GB", | 20260507 05:20:58.527 | INFO | "huge" : "", | 20260507 05:20:58.527 | INFO | "huge_shared" : " 27 GB", | 20260507 05:20:58.527 | INFO | "malloc" : " 14 GB", | 20260507 05:20:58.527 | INFO | "pinned" : " 29 GB" | 20260507 05:20:58.527 | INFO | } | 20260507 05:20:58.527 | INFO | Perf report for the last 5.13735 seconds: | 20260507 05:20:58.527 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:20:58.527 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:58.527 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:58.527 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:58.527 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 05:20:58.527 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:58.527 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:58.527 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:20:58.527 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:58.527 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:20:58.527 | INFO | Uploaded 36GiB to GPU in 2.06522s (17.4316GiB/s) | 20260507 05:20:58.527 | INFO | Downloaded 34GiB to GPU in 1.94059s (17.5205GiB/s) | 20260507 05:21:03.657 | INFO | Analyzing chunk frames = [7408640,7409152) | 20260507 05:21:03.657 | INFO | MemInfo: { | 20260507 05:21:03.657 | INFO | "gpu" : " 17 GB", | 20260507 05:21:03.657 | INFO | "huge" : "", | 20260507 05:21:03.657 | INFO | "huge_shared" : " 27 GB", | 20260507 05:21:03.657 | INFO | "malloc" : " 14 GB", | 20260507 05:21:03.657 | INFO | "pinned" : " 29 GB" | 20260507 05:21:03.657 | INFO | } | 20260507 05:21:03.657 | INFO | Perf report for the last 5.13029 seconds: | 20260507 05:21:03.657 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:21:03.657 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:03.657 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:03.657 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:03.657 | INFO | ANALYSIS : 60.5%, 3.0, 93.9ms, 0 | 20260507 05:21:03.657 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:03.657 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:03.657 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 05:21:03.657 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:03.657 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:03.657 | INFO | Uploaded 36GiB to GPU in 1.88105s (19.1383GiB/s) | 20260507 05:21:03.657 | INFO | Downloaded 34GiB to GPU in 1.97509s (17.2144GiB/s) | 20260507 05:21:08.797 | INFO | Analyzing chunk frames = [7409152,7409664) | 20260507 05:21:08.797 | INFO | MemInfo: { | 20260507 05:21:08.797 | INFO | "gpu" : " 17 GB", | 20260507 05:21:08.797 | INFO | "huge" : "", | 20260507 05:21:08.797 | INFO | "huge_shared" : " 27 GB", | 20260507 05:21:08.797 | INFO | "malloc" : " 14 GB", | 20260507 05:21:08.797 | INFO | "pinned" : " 29 GB" | 20260507 05:21:08.797 | INFO | } | 20260507 05:21:08.797 | INFO | Perf report for the last 5.13971 seconds: | 20260507 05:21:08.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:21:08.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:08.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:08.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:08.797 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 05:21:08.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:08.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:08.797 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:21:08.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:08.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:08.797 | INFO | Uploaded 36GiB to GPU in 1.96911s (18.2823GiB/s) | 20260507 05:21:08.797 | INFO | Downloaded 34GiB to GPU in 1.95144s (17.423GiB/s) | 20260507 05:21:08.894 | INFO | Starting a round of Baz IO | 20260507 05:21:11.997 | INFO | Populating RTMetrics report | 20260507 05:21:11.997 | INFO | RTMetrics report populated | 20260507 05:21:11.997 | INFO | Opening tmp RTMetrics json file | 20260507 05:21:12.003 | INFO | Serializing RTMetrics to json | 20260507 05:21:12.015 | INFO | Serialization done, swapping tmp file now... | 20260507 05:21:12.015 | INFO | RTMetrics done | 20260507 05:21:12.082 | INFO | Finished a round of Baz IO: Wrote 1.699 GiB in 3.188 seconds | 20260507 05:21:12.082 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.71% / 0.00% / 67.29% | 20260507 05:21:12.082 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.14567% / 3.97364e-06% / 5.5631e-05% | 20260507 05:21:12.082 | INFO | Baz IO overall: | 20260507 05:21:12.083 | INFO | BytesWritten : 3761809403408 | 20260507 05:21:12.083 | INFO | NumEvents : 1804530042452 | 20260507 05:21:13.938 | INFO | Analyzing chunk frames = [7409664,7410176) | 20260507 05:21:13.938 | INFO | MemInfo: { | 20260507 05:21:13.938 | INFO | "gpu" : " 17 GB", | 20260507 05:21:13.938 | INFO | "huge" : "", | 20260507 05:21:13.938 | INFO | "huge_shared" : " 27 GB", | 20260507 05:21:13.938 | INFO | "malloc" : " 14 GB", | 20260507 05:21:13.938 | INFO | "pinned" : " 29 GB" | 20260507 05:21:13.938 | INFO | } | 20260507 05:21:13.938 | INFO | Perf report for the last 5.141 seconds: | 20260507 05:21:13.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:21:13.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:13.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:13.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:13.938 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 05:21:13.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:13.938 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 05:21:13.938 | INFO | PRE_HQ : 59.0%, 1.8, 54.8ms, 0 | 20260507 05:21:13.938 | INFO | BAZWRITER : 61.5%, 3.1, 99.3ms, 0 | 20260507 05:21:13.938 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 05:21:13.938 | INFO | Uploaded 36GiB to GPU in 2.04161s (17.6331GiB/s) | 20260507 05:21:13.938 | INFO | Downloaded 38GiB to GPU in 2.18039s (17.4281GiB/s) | 20260507 05:21:19.083 | INFO | Analyzing chunk frames = [7410176,7410688) | 20260507 05:21:19.083 | INFO | MemInfo: { | 20260507 05:21:19.083 | INFO | "gpu" : " 17 GB", | 20260507 05:21:19.083 | INFO | "huge" : "", | 20260507 05:21:19.083 | INFO | "huge_shared" : " 27 GB", | 20260507 05:21:19.083 | INFO | "malloc" : " 14 GB", | 20260507 05:21:19.083 | INFO | "pinned" : " 29 GB" | 20260507 05:21:19.083 | INFO | } | 20260507 05:21:19.083 | INFO | Perf report for the last 5.14495 seconds: | 20260507 05:21:19.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:21:19.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:19.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:19.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:19.083 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 05:21:19.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:19.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:19.083 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:21:19.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:19.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:19.083 | INFO | Uploaded 36GiB to GPU in 2.04937s (17.5664GiB/s) | 20260507 05:21:19.083 | INFO | Downloaded 34GiB to GPU in 1.93879s (17.5368GiB/s) | 20260507 05:21:24.221 | INFO | Analyzing chunk frames = [7410688,7411200) | 20260507 05:21:24.221 | INFO | MemInfo: { | 20260507 05:21:24.221 | INFO | "gpu" : " 17 GB", | 20260507 05:21:24.221 | INFO | "huge" : "", | 20260507 05:21:24.221 | INFO | "huge_shared" : " 27 GB", | 20260507 05:21:24.221 | INFO | "malloc" : " 14 GB", | 20260507 05:21:24.221 | INFO | "pinned" : " 29 GB" | 20260507 05:21:24.221 | INFO | } | 20260507 05:21:24.221 | INFO | Perf report for the last 5.13834 seconds: | 20260507 05:21:24.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:21:24.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:24.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:24.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:24.221 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 05:21:24.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:24.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:24.221 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:21:24.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:24.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:24.221 | INFO | Uploaded 36GiB to GPU in 2.05132s (17.5497GiB/s) | 20260507 05:21:24.221 | INFO | Downloaded 34GiB to GPU in 1.94979s (17.4378GiB/s) | 20260507 05:21:29.357 | INFO | Analyzing chunk frames = [7411200,7411712) | 20260507 05:21:29.357 | INFO | MemInfo: { | 20260507 05:21:29.357 | INFO | "gpu" : " 17 GB", | 20260507 05:21:29.357 | INFO | "huge" : "", | 20260507 05:21:29.357 | INFO | "huge_shared" : " 27 GB", | 20260507 05:21:29.357 | INFO | "malloc" : " 14 GB", | 20260507 05:21:29.357 | INFO | "pinned" : " 29 GB" | 20260507 05:21:29.357 | INFO | } | 20260507 05:21:29.357 | INFO | Perf report for the last 5.13552 seconds: | 20260507 05:21:29.357 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:21:29.357 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:29.357 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:29.357 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:29.357 | INFO | ANALYSIS : 60.6%, 2.9, 93.4ms, 0 | 20260507 05:21:29.357 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:29.357 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:29.357 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 05:21:29.357 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:29.357 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:29.357 | INFO | Uploaded 36GiB to GPU in 2.05295s (17.5357GiB/s) | 20260507 05:21:29.357 | INFO | Downloaded 34GiB to GPU in 1.94389s (17.4907GiB/s) | 20260507 05:21:34.498 | INFO | Analyzing chunk frames = [7411712,7412224) | 20260507 05:21:34.498 | INFO | MemInfo: { | 20260507 05:21:34.498 | INFO | "gpu" : " 17 GB", | 20260507 05:21:34.498 | INFO | "huge" : "", | 20260507 05:21:34.498 | INFO | "huge_shared" : " 27 GB", | 20260507 05:21:34.498 | INFO | "malloc" : " 14 GB", | 20260507 05:21:34.498 | INFO | "pinned" : " 29 GB" | 20260507 05:21:34.498 | INFO | } | 20260507 05:21:34.498 | INFO | Perf report for the last 5.14135 seconds: | 20260507 05:21:34.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:21:34.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:34.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:34.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:34.498 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 05:21:34.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:34.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:34.498 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:21:34.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:34.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:34.498 | INFO | Uploaded 36GiB to GPU in 2.05157s (17.5476GiB/s) | 20260507 05:21:34.498 | INFO | Downloaded 34GiB to GPU in 1.94348s (17.4944GiB/s) | 20260507 05:21:39.637 | INFO | Analyzing chunk frames = [7412224,7412736) | 20260507 05:21:39.637 | INFO | MemInfo: { | 20260507 05:21:39.637 | INFO | "gpu" : " 17 GB", | 20260507 05:21:39.637 | INFO | "huge" : "", | 20260507 05:21:39.638 | INFO | "huge_shared" : " 27 GB", | 20260507 05:21:39.638 | INFO | "malloc" : " 14 GB", | 20260507 05:21:39.638 | INFO | "pinned" : " 29 GB" | 20260507 05:21:39.638 | INFO | } | 20260507 05:21:39.638 | INFO | Perf report for the last 5.13955 seconds: | 20260507 05:21:39.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:21:39.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:39.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:39.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:39.638 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 05:21:39.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:39.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:39.638 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 05:21:39.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:39.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:39.638 | INFO | Uploaded 36GiB to GPU in 2.04321s (17.6194GiB/s) | 20260507 05:21:39.638 | INFO | Downloaded 34GiB to GPU in 1.91042s (17.7971GiB/s) | 20260507 05:21:44.776 | INFO | Analyzing chunk frames = [7412736,7413248) | 20260507 05:21:44.776 | INFO | MemInfo: { | 20260507 05:21:44.776 | INFO | "gpu" : " 17 GB", | 20260507 05:21:44.776 | INFO | "huge" : "", | 20260507 05:21:44.776 | INFO | "huge_shared" : " 27 GB", | 20260507 05:21:44.776 | INFO | "malloc" : " 14 GB", | 20260507 05:21:44.776 | INFO | "pinned" : " 29 GB" | 20260507 05:21:44.776 | INFO | } | 20260507 05:21:44.776 | INFO | Perf report for the last 5.1382 seconds: | 20260507 05:21:44.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:21:44.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:44.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:44.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:44.776 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 05:21:44.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:44.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:44.776 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 05:21:44.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:44.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:44.776 | INFO | Uploaded 36GiB to GPU in 2.00369s (17.9669GiB/s) | 20260507 05:21:44.776 | INFO | Downloaded 34GiB to GPU in 1.97641s (17.2029GiB/s) | 20260507 05:21:49.916 | INFO | Analyzing chunk frames = [7413248,7413760) | 20260507 05:21:49.916 | INFO | MemInfo: { | 20260507 05:21:49.916 | INFO | "gpu" : " 17 GB", | 20260507 05:21:49.916 | INFO | "huge" : "", | 20260507 05:21:49.916 | INFO | "huge_shared" : " 27 GB", | 20260507 05:21:49.916 | INFO | "malloc" : " 14 GB", | 20260507 05:21:49.916 | INFO | "pinned" : " 29 GB" | 20260507 05:21:49.916 | INFO | } | 20260507 05:21:49.916 | INFO | Perf report for the last 5.14046 seconds: | 20260507 05:21:49.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:21:49.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:49.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:49.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:49.916 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 05:21:49.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:49.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:49.916 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:21:49.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:49.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:49.917 | INFO | Uploaded 36GiB to GPU in 1.9663s (18.3085GiB/s) | 20260507 05:21:49.917 | INFO | Downloaded 34GiB to GPU in 1.95535s (17.3882GiB/s) | 20260507 05:21:49.917 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:21:49.917 | INFO | Total mem usage: 18 GiB | 20260507 05:21:49.917 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:21:49.917 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:21:49.917 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:21:49.917 | INFO | High Water Breakdown: | 20260507 05:21:49.917 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:21:49.917 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:21:49.917 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:21:49.917 | INFO | ------------------------------------------ | 20260507 05:21:49.917 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:21:49.917 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:21:49.917 | INFO | Total mem usage: 26.5 GiB | 20260507 05:21:49.917 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:21:49.917 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:21:49.917 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:21:49.917 | INFO | High Water Breakdown: | 20260507 05:21:49.917 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:21:49.917 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:21:49.917 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:21:49.917 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:21:49.917 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:21:49.917 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:21:49.917 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:21:49.917 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:21:49.917 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:21:49.917 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:21:49.917 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:21:49.917 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:21:49.917 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:21:49.917 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:21:49.917 | INFO | ------------------------------------------ | 20260507 05:21:49.917 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:21:49.917 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:21:49.917 | INFO | Total mem usage: 25.6 GiB | 20260507 05:21:49.917 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:21:49.917 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:21:49.917 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:21:49.917 | INFO | High Water Breakdown: | 20260507 05:21:49.917 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:21:49.917 | INFO | ------------------------------------------ | 20260507 05:21:49.917 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:21:49.917 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:21:49.917 | INFO | Total mem usage: 15.7 GiB | 20260507 05:21:49.917 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:21:49.917 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:21:49.917 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:21:49.917 | INFO | High Water Breakdown: | 20260507 05:21:49.917 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:21:49.917 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:21:49.917 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:21:49.917 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:21:49.917 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:21:49.917 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:21:49.917 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:21:49.917 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:21:49.917 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:21:49.917 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:21:49.917 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:21:49.917 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:21:49.917 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:21:49.917 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:21:49.917 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:21:49.917 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:21:49.917 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:21:49.917 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:21:49.917 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:21:49.917 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:21:49.917 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:21:49.917 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:21:49.917 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:21:49.917 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:21:49.917 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:21:49.917 | INFO | ------------------------------------------ | 20260507 05:21:49.917 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:21:49.918 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:21:49.918 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:21:49.918 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:21:49.918 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.44e+01, 1.33e+01 / 1.54e+00 | 20260507 05:21:49.918 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.02e-01, 9.23e-03 / 7.63e-03 | 20260507 05:21:49.918 | INFO | PerfReport: 11.64%, Baseline -- 3.57e+00 / 4.39e+00, 4.97e-01 / 3.52e-01 | 20260507 05:21:49.918 | INFO | PerfReport: 3.22%, Histogram -- 9.87e-01 / 1.46e+00, 1.64e-01 / 1.32e-01 | 20260507 05:21:49.918 | INFO | PerfReport: 16.20%, DME -- 3.98e+01 / 4.43e+01, 3.90e+00 / 1.75e+00 | 20260507 05:21:49.918 | INFO | PerfReport: 57.67%, FrameLabeling -- 1.77e+01 / 1.98e+01, 1.24e+00 / 1.60e+00 | 20260507 05:21:49.918 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.13e+00 / 2.68e+00, 3.87e-01 / 2.45e-01 | 20260507 05:21:49.918 | INFO | PerfReport: 3.00%, Metrics -- 9.20e-01 / 1.24e+00, 2.45e-01 / 1.29e-01 | 20260507 05:21:49.918 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.09e-01 / 3.63e-01, 5.07e-02 / 2.20e-02 | 20260507 05:21:49.918 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:21:49.918 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:21:49.918 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:21:49.918 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 05:21:49.918 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260507 05:21:49.918 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 05:21:49.918 | INFO | PerfReport: 31.89%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 05:21:49.918 | INFO | PerfReport: 1.55%, LockUpload -- 1.49e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 05:21:49.918 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260507 05:21:49.918 | INFO | PerfReport: 22.04%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 05:21:49.918 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.36e-02 / 0.00e+00 | 20260507 05:21:50.013 | INFO | Starting a round of Baz IO | 20260507 05:21:53.117 | INFO | Populating RTMetrics report | 20260507 05:21:53.118 | INFO | RTMetrics report populated | 20260507 05:21:53.118 | INFO | Opening tmp RTMetrics json file | 20260507 05:21:53.135 | INFO | Serializing RTMetrics to json | 20260507 05:21:53.137 | INFO | Serialization done, swapping tmp file now... | 20260507 05:21:53.137 | INFO | RTMetrics done | 20260507 05:21:53.204 | INFO | Finished a round of Baz IO: Wrote 1.698 GiB in 3.19 seconds | 20260507 05:21:53.204 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.71% / 0.00% / 67.29% | 20260507 05:21:53.204 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.145992% / 3.97364e-06% / 7.15256e-05% | 20260507 05:21:53.204 | INFO | Baz IO overall: | 20260507 05:21:53.204 | INFO | BytesWritten : 3763633050541 | 20260507 05:21:53.204 | INFO | NumEvents : 1805386517929 | 20260507 05:21:55.050 | INFO | Analyzing chunk frames = [7413760,7414272) | 20260507 05:21:55.050 | INFO | MemInfo: { | 20260507 05:21:55.050 | INFO | "gpu" : " 17 GB", | 20260507 05:21:55.050 | INFO | "huge" : "", | 20260507 05:21:55.050 | INFO | "huge_shared" : " 27 GB", | 20260507 05:21:55.050 | INFO | "malloc" : " 14 GB", | 20260507 05:21:55.050 | INFO | "pinned" : " 29 GB" | 20260507 05:21:55.050 | INFO | } | 20260507 05:21:55.051 | INFO | Perf report for the last 5.13424 seconds: | 20260507 05:21:55.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:21:55.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:55.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:55.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:55.051 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 05:21:55.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:21:55.051 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 05:21:55.051 | INFO | PRE_HQ : 59.2%, 1.8, 55.0ms, 0 | 20260507 05:21:55.051 | INFO | BAZWRITER : 61.7%, 3.2, 100.8ms, 0 | 20260507 05:21:55.051 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:21:55.051 | INFO | Uploaded 36GiB to GPU in 2.05015s (17.5597GiB/s) | 20260507 05:21:55.051 | INFO | Downloaded 38GiB to GPU in 2.17337s (17.4844GiB/s) | 20260507 05:22:00.193 | INFO | Analyzing chunk frames = [7414272,7414784) | 20260507 05:22:00.193 | INFO | MemInfo: { | 20260507 05:22:00.193 | INFO | "gpu" : " 17 GB", | 20260507 05:22:00.193 | INFO | "huge" : "", | 20260507 05:22:00.193 | INFO | "huge_shared" : " 27 GB", | 20260507 05:22:00.193 | INFO | "malloc" : " 14 GB", | 20260507 05:22:00.193 | INFO | "pinned" : " 29 GB" | 20260507 05:22:00.193 | INFO | } | 20260507 05:22:00.193 | INFO | Perf report for the last 5.14296 seconds: | 20260507 05:22:00.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:22:00.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:00.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:00.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:00.194 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 05:22:00.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:00.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:00.194 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:22:00.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:00.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:00.194 | INFO | Uploaded 36GiB to GPU in 2.02693s (17.7608GiB/s) | 20260507 05:22:00.194 | INFO | Downloaded 34GiB to GPU in 1.93647s (17.5577GiB/s) | 20260507 05:22:05.332 | INFO | Analyzing chunk frames = [7414784,7415296) | 20260507 05:22:05.332 | INFO | MemInfo: { | 20260507 05:22:05.332 | INFO | "gpu" : " 17 GB", | 20260507 05:22:05.333 | INFO | "huge" : "", | 20260507 05:22:05.333 | INFO | "huge_shared" : " 27 GB", | 20260507 05:22:05.333 | INFO | "malloc" : " 14 GB", | 20260507 05:22:05.333 | INFO | "pinned" : " 29 GB" | 20260507 05:22:05.333 | INFO | } | 20260507 05:22:05.333 | INFO | Perf report for the last 5.13911 seconds: | 20260507 05:22:05.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:22:05.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:05.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:05.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:05.333 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 05:22:05.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:05.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:05.333 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:22:05.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:05.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:05.333 | INFO | Uploaded 36GiB to GPU in 2.02757s (17.7553GiB/s) | 20260507 05:22:05.333 | INFO | Downloaded 34GiB to GPU in 1.95325s (17.4069GiB/s) | 20260507 05:22:10.473 | INFO | Analyzing chunk frames = [7415296,7415808) | 20260507 05:22:10.474 | INFO | MemInfo: { | 20260507 05:22:10.474 | INFO | "gpu" : " 17 GB", | 20260507 05:22:10.474 | INFO | "huge" : "", | 20260507 05:22:10.474 | INFO | "huge_shared" : " 27 GB", | 20260507 05:22:10.474 | INFO | "malloc" : " 14 GB", | 20260507 05:22:10.474 | INFO | "pinned" : " 29 GB" | 20260507 05:22:10.474 | INFO | } | 20260507 05:22:10.474 | INFO | Perf report for the last 5.14105 seconds: | 20260507 05:22:10.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:22:10.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:10.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:10.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:10.474 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 05:22:10.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:10.474 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:10.474 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:22:10.474 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:10.474 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:10.474 | INFO | Uploaded 36GiB to GPU in 2.01395s (17.8753GiB/s) | 20260507 05:22:10.474 | INFO | Downloaded 34GiB to GPU in 1.93646s (17.5578GiB/s) | 20260507 05:22:15.609 | INFO | Analyzing chunk frames = [7415808,7416320) | 20260507 05:22:15.609 | INFO | MemInfo: { | 20260507 05:22:15.609 | INFO | "gpu" : " 17 GB", | 20260507 05:22:15.609 | INFO | "huge" : "", | 20260507 05:22:15.609 | INFO | "huge_shared" : " 27 GB", | 20260507 05:22:15.609 | INFO | "malloc" : " 14 GB", | 20260507 05:22:15.609 | INFO | "pinned" : " 29 GB" | 20260507 05:22:15.609 | INFO | } | 20260507 05:22:15.609 | INFO | Perf report for the last 5.13498 seconds: | 20260507 05:22:15.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:22:15.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:15.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:15.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:15.609 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 05:22:15.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:15.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:15.609 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 05:22:15.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:15.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:15.609 | INFO | Uploaded 36GiB to GPU in 2.058s (17.4927GiB/s) | 20260507 05:22:15.609 | INFO | Downloaded 34GiB to GPU in 1.95142s (17.4232GiB/s) | 20260507 05:22:20.753 | INFO | Analyzing chunk frames = [7416320,7416832) | 20260507 05:22:20.753 | INFO | MemInfo: { | 20260507 05:22:20.753 | INFO | "gpu" : " 17 GB", | 20260507 05:22:20.753 | INFO | "huge" : "", | 20260507 05:22:20.753 | INFO | "huge_shared" : " 27 GB", | 20260507 05:22:20.753 | INFO | "malloc" : " 14 GB", | 20260507 05:22:20.753 | INFO | "pinned" : " 29 GB" | 20260507 05:22:20.753 | INFO | } | 20260507 05:22:20.753 | INFO | Perf report for the last 5.14403 seconds: | 20260507 05:22:20.753 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:22:20.753 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:20.753 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:20.753 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:20.753 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 05:22:20.753 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:20.753 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:20.753 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 05:22:20.753 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:20.753 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:20.753 | INFO | Uploaded 36GiB to GPU in 2.03967s (17.6499GiB/s) | 20260507 05:22:20.753 | INFO | Downloaded 34GiB to GPU in 1.9352s (17.5692GiB/s) | 20260507 05:22:25.900 | INFO | Analyzing chunk frames = [7416832,7417344) | 20260507 05:22:25.900 | INFO | MemInfo: { | 20260507 05:22:25.900 | INFO | "gpu" : " 17 GB", | 20260507 05:22:25.900 | INFO | "huge" : "", | 20260507 05:22:25.900 | INFO | "huge_shared" : " 27 GB", | 20260507 05:22:25.900 | INFO | "malloc" : " 14 GB", | 20260507 05:22:25.900 | INFO | "pinned" : " 29 GB" | 20260507 05:22:25.900 | INFO | } | 20260507 05:22:25.900 | INFO | Perf report for the last 5.14721 seconds: | 20260507 05:22:25.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:22:25.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:25.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:25.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:25.900 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 05:22:25.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:25.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:25.900 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 05:22:25.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:25.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:25.900 | INFO | Uploaded 36GiB to GPU in 2.01441s (17.8713GiB/s) | 20260507 05:22:25.900 | INFO | Downloaded 34GiB to GPU in 1.97516s (17.2138GiB/s) | 20260507 05:22:31.034 | INFO | Analyzing chunk frames = [7417344,7417856) | 20260507 05:22:31.034 | INFO | MemInfo: { | 20260507 05:22:31.034 | INFO | "gpu" : " 17 GB", | 20260507 05:22:31.034 | INFO | "huge" : "", | 20260507 05:22:31.034 | INFO | "huge_shared" : " 27 GB", | 20260507 05:22:31.034 | INFO | "malloc" : " 14 GB", | 20260507 05:22:31.034 | INFO | "pinned" : " 29 GB" | 20260507 05:22:31.034 | INFO | } | 20260507 05:22:31.034 | INFO | Perf report for the last 5.13388 seconds: | 20260507 05:22:31.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:22:31.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:31.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:31.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:31.034 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 05:22:31.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:31.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:31.034 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:22:31.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:31.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:31.034 | INFO | Uploaded 36GiB to GPU in 1.99436s (18.0509GiB/s) | 20260507 05:22:31.034 | INFO | Downloaded 34GiB to GPU in 1.96276s (17.3225GiB/s) | 20260507 05:22:31.131 | INFO | Starting a round of Baz IO | 20260507 05:22:34.230 | INFO | Populating RTMetrics report | 20260507 05:22:34.230 | INFO | RTMetrics report populated | 20260507 05:22:34.230 | INFO | Opening tmp RTMetrics json file | 20260507 05:22:34.233 | INFO | Serializing RTMetrics to json | 20260507 05:22:34.255 | INFO | Serialization done, swapping tmp file now... | 20260507 05:22:34.257 | INFO | RTMetrics done | 20260507 05:22:34.328 | INFO | Finished a round of Baz IO: Wrote 1.698 GiB in 3.196 seconds | 20260507 05:22:34.328 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.71% / 0.00% / 67.29% | 20260507 05:22:34.328 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.146437% / 0% / 7.54992e-05% | 20260507 05:22:34.328 | INFO | Baz IO overall: | 20260507 05:22:34.328 | INFO | BytesWritten : 3765456578252 | 20260507 05:22:34.328 | INFO | NumEvents : 1806243186617 | 20260507 05:22:36.174 | INFO | Analyzing chunk frames = [7417856,7418368) | 20260507 05:22:36.174 | INFO | MemInfo: { | 20260507 05:22:36.174 | INFO | "gpu" : " 17 GB", | 20260507 05:22:36.174 | INFO | "huge" : "", | 20260507 05:22:36.174 | INFO | "huge_shared" : " 27 GB", | 20260507 05:22:36.174 | INFO | "malloc" : " 14 GB", | 20260507 05:22:36.174 | INFO | "pinned" : " 29 GB" | 20260507 05:22:36.174 | INFO | } | 20260507 05:22:36.174 | INFO | Perf report for the last 5.13983 seconds: | 20260507 05:22:36.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:22:36.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:36.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:36.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:36.174 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 05:22:36.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:36.174 | INFO | RT_METRICS : 41.7%, 1.0, 21.9ms, 0 | 20260507 05:22:36.174 | INFO | PRE_HQ : 59.1%, 1.8, 54.9ms, 0 | 20260507 05:22:36.174 | INFO | BAZWRITER : 61.8%, 3.2, 103.1ms, 0 | 20260507 05:22:36.174 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:22:36.174 | INFO | Uploaded 36GiB to GPU in 2.06909s (17.399GiB/s) | 20260507 05:22:36.174 | INFO | Downloaded 38GiB to GPU in 2.18515s (17.3901GiB/s) | 20260507 05:22:41.303 | INFO | Analyzing chunk frames = [7418368,7418880) | 20260507 05:22:41.303 | INFO | MemInfo: { | 20260507 05:22:41.303 | INFO | "gpu" : " 17 GB", | 20260507 05:22:41.303 | INFO | "huge" : "", | 20260507 05:22:41.303 | INFO | "huge_shared" : " 27 GB", | 20260507 05:22:41.303 | INFO | "malloc" : " 14 GB", | 20260507 05:22:41.303 | INFO | "pinned" : " 29 GB" | 20260507 05:22:41.303 | INFO | } | 20260507 05:22:41.303 | INFO | Perf report for the last 5.12974 seconds: | 20260507 05:22:41.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:22:41.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:41.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:41.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:41.303 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 05:22:41.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:41.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:41.304 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:22:41.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:41.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:41.304 | INFO | Uploaded 36GiB to GPU in 2.00535s (17.9519GiB/s) | 20260507 05:22:41.304 | INFO | Downloaded 34GiB to GPU in 1.92389s (17.6725GiB/s) | 20260507 05:22:46.452 | INFO | Analyzing chunk frames = [7418880,7419392) | 20260507 05:22:46.452 | INFO | MemInfo: { | 20260507 05:22:46.452 | INFO | "gpu" : " 17 GB", | 20260507 05:22:46.452 | INFO | "huge" : "", | 20260507 05:22:46.452 | INFO | "huge_shared" : " 27 GB", | 20260507 05:22:46.452 | INFO | "malloc" : " 14 GB", | 20260507 05:22:46.452 | INFO | "pinned" : " 29 GB" | 20260507 05:22:46.452 | INFO | } | 20260507 05:22:46.452 | INFO | Perf report for the last 5.14869 seconds: | 20260507 05:22:46.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:22:46.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:46.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:46.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:46.452 | INFO | ANALYSIS : 60.1%, 3.0, 93.3ms, 0 | 20260507 05:22:46.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:46.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:46.452 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:22:46.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:46.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:46.452 | INFO | Uploaded 36GiB to GPU in 2.02233s (17.8013GiB/s) | 20260507 05:22:46.452 | INFO | Downloaded 34GiB to GPU in 1.94089s (17.5177GiB/s) | 20260507 05:22:51.596 | INFO | Analyzing chunk frames = [7419392,7419904) | 20260507 05:22:51.596 | INFO | MemInfo: { | 20260507 05:22:51.596 | INFO | "gpu" : " 17 GB", | 20260507 05:22:51.596 | INFO | "huge" : "", | 20260507 05:22:51.596 | INFO | "huge_shared" : " 27 GB", | 20260507 05:22:51.596 | INFO | "malloc" : " 14 GB", | 20260507 05:22:51.596 | INFO | "pinned" : " 29 GB" | 20260507 05:22:51.596 | INFO | } | 20260507 05:22:51.596 | INFO | Perf report for the last 5.14373 seconds: | 20260507 05:22:51.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:22:51.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:51.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:51.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:51.596 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 05:22:51.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:51.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:51.596 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:22:51.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:51.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:51.596 | INFO | Uploaded 36GiB to GPU in 2.01698s (17.8485GiB/s) | 20260507 05:22:51.596 | INFO | Downloaded 34GiB to GPU in 1.9316s (17.6019GiB/s) | 20260507 05:22:56.732 | INFO | Analyzing chunk frames = [7419904,7420416) | 20260507 05:22:56.732 | INFO | MemInfo: { | 20260507 05:22:56.732 | INFO | "gpu" : " 17 GB", | 20260507 05:22:56.732 | INFO | "huge" : "", | 20260507 05:22:56.732 | INFO | "huge_shared" : " 27 GB", | 20260507 05:22:56.732 | INFO | "malloc" : " 14 GB", | 20260507 05:22:56.732 | INFO | "pinned" : " 29 GB" | 20260507 05:22:56.732 | INFO | } | 20260507 05:22:56.732 | INFO | Perf report for the last 5.13642 seconds: | 20260507 05:22:56.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:22:56.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:56.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:56.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:56.732 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260507 05:22:56.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:56.732 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:56.732 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 05:22:56.732 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:56.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:22:56.732 | INFO | Uploaded 36GiB to GPU in 2.05777s (17.4947GiB/s) | 20260507 05:22:56.732 | INFO | Downloaded 34GiB to GPU in 1.9056s (17.8422GiB/s) | 20260507 05:23:01.866 | INFO | Analyzing chunk frames = [7420416,7420928) | 20260507 05:23:01.867 | INFO | MemInfo: { | 20260507 05:23:01.867 | INFO | "gpu" : " 17 GB", | 20260507 05:23:01.867 | INFO | "huge" : "", | 20260507 05:23:01.867 | INFO | "huge_shared" : " 27 GB", | 20260507 05:23:01.867 | INFO | "malloc" : " 14 GB", | 20260507 05:23:01.867 | INFO | "pinned" : " 29 GB" | 20260507 05:23:01.867 | INFO | } | 20260507 05:23:01.867 | INFO | Perf report for the last 5.13434 seconds: | 20260507 05:23:01.867 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:23:01.867 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:01.867 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:01.867 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:01.867 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 05:23:01.867 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:01.867 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:01.867 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:23:01.867 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:01.867 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:01.867 | INFO | Uploaded 36GiB to GPU in 2.06895s (17.4001GiB/s) | 20260507 05:23:01.867 | INFO | Downloaded 34GiB to GPU in 1.9262s (17.6513GiB/s) | 20260507 05:23:07.006 | INFO | Analyzing chunk frames = [7420928,7421440) | 20260507 05:23:07.006 | INFO | MemInfo: { | 20260507 05:23:07.006 | INFO | "gpu" : " 17 GB", | 20260507 05:23:07.006 | INFO | "huge" : "", | 20260507 05:23:07.006 | INFO | "huge_shared" : " 27 GB", | 20260507 05:23:07.006 | INFO | "malloc" : " 14 GB", | 20260507 05:23:07.006 | INFO | "pinned" : " 29 GB" | 20260507 05:23:07.006 | INFO | } | 20260507 05:23:07.006 | INFO | Perf report for the last 5.13942 seconds: | 20260507 05:23:07.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:23:07.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:07.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:07.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:07.006 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:23:07.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:07.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:07.006 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:23:07.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:07.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:07.007 | INFO | Uploaded 36GiB to GPU in 1.97453s (18.2321GiB/s) | 20260507 05:23:07.007 | INFO | Downloaded 34GiB to GPU in 1.98119s (17.1614GiB/s) | 20260507 05:23:12.148 | INFO | Analyzing chunk frames = [7421440,7421952) | 20260507 05:23:12.149 | INFO | MemInfo: { | 20260507 05:23:12.149 | INFO | "gpu" : " 17 GB", | 20260507 05:23:12.149 | INFO | "huge" : "", | 20260507 05:23:12.149 | INFO | "huge_shared" : " 27 GB", | 20260507 05:23:12.149 | INFO | "malloc" : " 14 GB", | 20260507 05:23:12.149 | INFO | "pinned" : " 29 GB" | 20260507 05:23:12.149 | INFO | } | 20260507 05:23:12.149 | INFO | Perf report for the last 5.14265 seconds: | 20260507 05:23:12.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:23:12.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:12.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:12.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:12.149 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:23:12.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:12.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:12.149 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:23:12.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:12.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:12.149 | INFO | Uploaded 36GiB to GPU in 1.96126s (18.3555GiB/s) | 20260507 05:23:12.149 | INFO | Downloaded 34GiB to GPU in 1.96615s (17.2927GiB/s) | 20260507 05:23:12.149 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:23:12.149 | INFO | Total mem usage: 18 GiB | 20260507 05:23:12.149 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:23:12.149 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 05:23:12.149 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:23:12.149 | INFO | High Water Breakdown: | 20260507 05:23:12.149 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:23:12.149 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:23:12.149 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:23:12.149 | INFO | ------------------------------------------ | 20260507 05:23:12.149 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:23:12.149 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:23:12.149 | INFO | Total mem usage: 26.5 GiB | 20260507 05:23:12.149 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:23:12.149 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:23:12.149 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 05:23:12.149 | INFO | High Water Breakdown: | 20260507 05:23:12.149 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:23:12.150 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:23:12.150 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:23:12.150 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:23:12.150 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:23:12.150 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:23:12.150 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:23:12.150 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:23:12.150 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:23:12.150 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:23:12.150 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:23:12.150 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:23:12.150 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:23:12.150 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:23:12.150 | INFO | ------------------------------------------ | 20260507 05:23:12.150 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:23:12.150 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:23:12.150 | INFO | Total mem usage: 25.6 GiB | 20260507 05:23:12.150 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:23:12.150 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:23:12.150 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:23:12.150 | INFO | High Water Breakdown: | 20260507 05:23:12.150 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:23:12.150 | INFO | ------------------------------------------ | 20260507 05:23:12.150 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:23:12.150 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:23:12.150 | INFO | Total mem usage: 15.7 GiB | 20260507 05:23:12.150 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:23:12.150 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:23:12.150 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:23:12.150 | INFO | High Water Breakdown: | 20260507 05:23:12.150 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:23:12.150 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:23:12.150 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:23:12.150 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:23:12.150 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:23:12.150 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:23:12.150 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:23:12.150 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:23:12.150 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:23:12.150 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:23:12.150 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:23:12.150 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:23:12.150 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:23:12.150 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:23:12.150 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:23:12.150 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:23:12.150 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:23:12.150 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:23:12.150 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:23:12.150 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:23:12.150 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:23:12.150 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:23:12.150 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:23:12.150 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:23:12.150 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:23:12.150 | INFO | ------------------------------------------ | 20260507 05:23:12.150 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:23:12.150 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:23:12.150 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:23:12.150 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:23:12.150 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.44e+01, 1.34e+01 / 1.50e+00 | 20260507 05:23:12.150 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.87e-03 / 6.14e-03 | 20260507 05:23:12.150 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.43e+00, 5.06e-01 / 3.36e-01 | 20260507 05:23:12.150 | INFO | PerfReport: 3.22%, Histogram -- 9.87e-01 / 1.45e+00, 1.64e-01 / 1.39e-01 | 20260507 05:23:12.150 | INFO | PerfReport: 16.13%, DME -- 3.96e+01 / 4.38e+01, 3.69e+00 / 1.79e+00 | 20260507 05:23:12.150 | INFO | PerfReport: 57.77%, FrameLabeling -- 1.77e+01 / 2.03e+01, 1.32e+00 / 1.59e+00 | 20260507 05:23:12.150 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.67e+00, 3.80e-01 / 2.49e-01 | 20260507 05:23:12.150 | INFO | PerfReport: 2.99%, Metrics -- 9.17e-01 / 1.24e+00, 2.45e-01 / 1.71e-01 | 20260507 05:23:12.150 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.09e-01 / 3.63e-01, 5.00e-02 / 1.82e-02 | 20260507 05:23:12.150 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:23:12.150 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:23:12.150 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:23:12.150 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 05:23:12.150 | INFO | PerfReport: 21.93%, Upload -- 2.11e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260507 05:23:12.150 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260507 05:23:12.150 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 05:23:12.150 | INFO | PerfReport: 1.57%, LockUpload -- 1.51e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 05:23:12.150 | INFO | PerfReport: 1.20%, LockDownload -- 1.15e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260507 05:23:12.150 | INFO | PerfReport: 22.00%, LockCompute -- 2.12e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 05:23:12.150 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260507 05:23:12.246 | INFO | Starting a round of Baz IO | 20260507 05:23:15.343 | INFO | Populating RTMetrics report | 20260507 05:23:15.344 | INFO | RTMetrics report populated | 20260507 05:23:15.344 | INFO | Opening tmp RTMetrics json file | 20260507 05:23:15.345 | INFO | Serializing RTMetrics to json | 20260507 05:23:15.363 | INFO | Serialization done, swapping tmp file now... | 20260507 05:23:15.363 | INFO | RTMetrics done | 20260507 05:23:15.427 | INFO | Finished a round of Baz IO: Wrote 1.698 GiB in 3.181 seconds | 20260507 05:23:15.428 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.71% / 0.00% / 67.29% | 20260507 05:23:15.428 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.145515% / 7.94729e-06% / 7.54992e-05% | 20260507 05:23:15.428 | INFO | Baz IO overall: | 20260507 05:23:15.428 | INFO | BytesWritten : 3767279746365 | 20260507 05:23:15.428 | INFO | NumEvents : 1807099923941 | 20260507 05:23:17.282 | INFO | Analyzing chunk frames = [7421952,7422464) | 20260507 05:23:17.282 | INFO | MemInfo: { | 20260507 05:23:17.282 | INFO | "gpu" : " 17 GB", | 20260507 05:23:17.282 | INFO | "huge" : "", | 20260507 05:23:17.282 | INFO | "huge_shared" : " 27 GB", | 20260507 05:23:17.282 | INFO | "malloc" : " 14 GB", | 20260507 05:23:17.282 | INFO | "pinned" : " 29 GB" | 20260507 05:23:17.282 | INFO | } | 20260507 05:23:17.282 | INFO | Perf report for the last 5.13317 seconds: | 20260507 05:23:17.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:23:17.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:17.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:17.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:17.282 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 05:23:17.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:17.282 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 05:23:17.282 | INFO | PRE_HQ : 59.1%, 1.8, 55.0ms, 0 | 20260507 05:23:17.282 | INFO | BAZWRITER : 61.6%, 3.1, 99.0ms, 0 | 20260507 05:23:17.282 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:23:17.282 | INFO | Uploaded 36GiB to GPU in 2.02987s (17.7351GiB/s) | 20260507 05:23:17.282 | INFO | Downloaded 38GiB to GPU in 2.17443s (17.4759GiB/s) | 20260507 05:23:22.427 | INFO | Analyzing chunk frames = [7422464,7422976) | 20260507 05:23:22.427 | INFO | MemInfo: { | 20260507 05:23:22.427 | INFO | "gpu" : " 17 GB", | 20260507 05:23:22.427 | INFO | "huge" : "", | 20260507 05:23:22.427 | INFO | "huge_shared" : " 27 GB", | 20260507 05:23:22.427 | INFO | "malloc" : " 14 GB", | 20260507 05:23:22.427 | INFO | "pinned" : " 29 GB" | 20260507 05:23:22.427 | INFO | } | 20260507 05:23:22.427 | INFO | Perf report for the last 5.14484 seconds: | 20260507 05:23:22.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:23:22.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:22.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:22.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:22.427 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 05:23:22.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:22.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:22.427 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:23:22.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:22.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:22.427 | INFO | Uploaded 36GiB to GPU in 1.99442s (18.0504GiB/s) | 20260507 05:23:22.427 | INFO | Downloaded 34GiB to GPU in 1.9299s (17.6175GiB/s) | 20260507 05:23:27.561 | INFO | Analyzing chunk frames = [7422976,7423488) | 20260507 05:23:27.561 | INFO | MemInfo: { | 20260507 05:23:27.561 | INFO | "gpu" : " 17 GB", | 20260507 05:23:27.561 | INFO | "huge" : "", | 20260507 05:23:27.561 | INFO | "huge_shared" : " 27 GB", | 20260507 05:23:27.561 | INFO | "malloc" : " 14 GB", | 20260507 05:23:27.561 | INFO | "pinned" : " 29 GB" | 20260507 05:23:27.561 | INFO | } | 20260507 05:23:27.561 | INFO | Perf report for the last 5.13431 seconds: | 20260507 05:23:27.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:23:27.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:27.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:27.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:27.561 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 05:23:27.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:27.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:27.561 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:23:27.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:27.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:27.561 | INFO | Uploaded 36GiB to GPU in 2.04423s (17.6105GiB/s) | 20260507 05:23:27.561 | INFO | Downloaded 34GiB to GPU in 1.95641s (17.3787GiB/s) | 20260507 05:23:32.707 | INFO | Analyzing chunk frames = [7423488,7424000) | 20260507 05:23:32.707 | INFO | MemInfo: { | 20260507 05:23:32.707 | INFO | "gpu" : " 17 GB", | 20260507 05:23:32.707 | INFO | "huge" : "", | 20260507 05:23:32.708 | INFO | "huge_shared" : " 27 GB", | 20260507 05:23:32.708 | INFO | "malloc" : " 14 GB", | 20260507 05:23:32.708 | INFO | "pinned" : " 29 GB" | 20260507 05:23:32.708 | INFO | } | 20260507 05:23:32.708 | INFO | Perf report for the last 5.14656 seconds: | 20260507 05:23:32.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:23:32.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:32.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:32.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:32.708 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 05:23:32.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:32.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:32.708 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:23:32.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:32.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:32.708 | INFO | Uploaded 36GiB to GPU in 1.99119s (18.0797GiB/s) | 20260507 05:23:32.708 | INFO | Downloaded 34GiB to GPU in 1.93438s (17.5767GiB/s) | 20260507 05:23:37.842 | INFO | Analyzing chunk frames = [7424000,7424512) | 20260507 05:23:37.842 | INFO | MemInfo: { | 20260507 05:23:37.842 | INFO | "gpu" : " 17 GB", | 20260507 05:23:37.842 | INFO | "huge" : "", | 20260507 05:23:37.842 | INFO | "huge_shared" : " 27 GB", | 20260507 05:23:37.842 | INFO | "malloc" : " 14 GB", | 20260507 05:23:37.842 | INFO | "pinned" : " 29 GB" | 20260507 05:23:37.842 | INFO | } | 20260507 05:23:37.843 | INFO | Perf report for the last 5.13485 seconds: | 20260507 05:23:37.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:23:37.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:37.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:37.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:37.843 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 05:23:37.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:37.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:37.843 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:23:37.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:37.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:37.843 | INFO | Uploaded 36GiB to GPU in 2.0514s (17.549GiB/s) | 20260507 05:23:37.843 | INFO | Downloaded 34GiB to GPU in 1.95384s (17.4016GiB/s) | 20260507 05:23:42.981 | INFO | Analyzing chunk frames = [7424512,7425024) | 20260507 05:23:42.981 | INFO | MemInfo: { | 20260507 05:23:42.981 | INFO | "gpu" : " 17 GB", | 20260507 05:23:42.981 | INFO | "huge" : "", | 20260507 05:23:42.981 | INFO | "huge_shared" : " 27 GB", | 20260507 05:23:42.981 | INFO | "malloc" : " 14 GB", | 20260507 05:23:42.981 | INFO | "pinned" : " 29 GB" | 20260507 05:23:42.981 | INFO | } | 20260507 05:23:42.982 | INFO | Perf report for the last 5.13901 seconds: | 20260507 05:23:42.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:23:42.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:42.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:42.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:42.982 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 05:23:42.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:42.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:42.982 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 05:23:42.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:42.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:42.982 | INFO | Uploaded 36GiB to GPU in 2.03432s (17.6964GiB/s) | 20260507 05:23:42.982 | INFO | Downloaded 34GiB to GPU in 1.91495s (17.755GiB/s) | 20260507 05:23:48.119 | INFO | Analyzing chunk frames = [7425024,7425536) | 20260507 05:23:48.119 | INFO | MemInfo: { | 20260507 05:23:48.119 | INFO | "gpu" : " 17 GB", | 20260507 05:23:48.119 | INFO | "huge" : "", | 20260507 05:23:48.119 | INFO | "huge_shared" : " 27 GB", | 20260507 05:23:48.119 | INFO | "malloc" : " 14 GB", | 20260507 05:23:48.119 | INFO | "pinned" : " 29 GB" | 20260507 05:23:48.119 | INFO | } | 20260507 05:23:48.119 | INFO | Perf report for the last 5.13786 seconds: | 20260507 05:23:48.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:23:48.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:48.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:48.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:48.119 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 05:23:48.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:48.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:48.119 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 05:23:48.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:48.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:48.120 | INFO | Uploaded 36GiB to GPU in 2.02843s (17.7478GiB/s) | 20260507 05:23:48.120 | INFO | Downloaded 34GiB to GPU in 1.99744s (17.0218GiB/s) | 20260507 05:23:53.257 | INFO | Analyzing chunk frames = [7425536,7426048) | 20260507 05:23:53.257 | INFO | MemInfo: { | 20260507 05:23:53.257 | INFO | "gpu" : " 17 GB", | 20260507 05:23:53.257 | INFO | "huge" : "", | 20260507 05:23:53.257 | INFO | "huge_shared" : " 27 GB", | 20260507 05:23:53.257 | INFO | "malloc" : " 14 GB", | 20260507 05:23:53.257 | INFO | "pinned" : " 29 GB" | 20260507 05:23:53.257 | INFO | } | 20260507 05:23:53.257 | INFO | Perf report for the last 5.13758 seconds: | 20260507 05:23:53.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:23:53.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:53.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:53.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:53.257 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:23:53.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:53.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:53.257 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:23:53.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:53.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:53.257 | INFO | Uploaded 36GiB to GPU in 1.97102s (18.2647GiB/s) | 20260507 05:23:53.257 | INFO | Downloaded 34GiB to GPU in 1.95568s (17.3853GiB/s) | 20260507 05:23:53.354 | INFO | Starting a round of Baz IO | 20260507 05:23:56.454 | INFO | Populating RTMetrics report | 20260507 05:23:56.455 | INFO | RTMetrics report populated | 20260507 05:23:56.455 | INFO | Opening tmp RTMetrics json file | 20260507 05:23:56.459 | INFO | Serializing RTMetrics to json | 20260507 05:23:56.473 | INFO | Serialization done, swapping tmp file now... | 20260507 05:23:56.474 | INFO | RTMetrics done | 20260507 05:23:56.539 | INFO | Finished a round of Baz IO: Wrote 1.697 GiB in 3.185 seconds | 20260507 05:23:56.539 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.71% / 0.00% / 67.29% | 20260507 05:23:56.539 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.145157% / 3.97364e-06% / 7.54992e-05% | 20260507 05:23:56.539 | INFO | Baz IO overall: | 20260507 05:23:56.539 | INFO | BytesWritten : 3769101719010 | 20260507 05:23:56.539 | INFO | NumEvents : 1807955563894 | 20260507 05:23:58.403 | INFO | Analyzing chunk frames = [7426048,7426560) | 20260507 05:23:58.403 | INFO | MemInfo: { | 20260507 05:23:58.403 | INFO | "gpu" : " 17 GB", | 20260507 05:23:58.403 | INFO | "huge" : "", | 20260507 05:23:58.403 | INFO | "huge_shared" : " 27 GB", | 20260507 05:23:58.403 | INFO | "malloc" : " 14 GB", | 20260507 05:23:58.403 | INFO | "pinned" : " 29 GB" | 20260507 05:23:58.403 | INFO | } | 20260507 05:23:58.403 | INFO | Perf report for the last 5.14619 seconds: | 20260507 05:23:58.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:23:58.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:58.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:58.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:58.403 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 05:23:58.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:23:58.403 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 05:23:58.403 | INFO | PRE_HQ : 58.8%, 1.8, 54.4ms, 0 | 20260507 05:23:58.403 | INFO | BAZWRITER : 61.5%, 3.1, 99.5ms, 0 | 20260507 05:23:58.403 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 05:23:58.403 | INFO | Uploaded 36GiB to GPU in 2.03505s (17.69GiB/s) | 20260507 05:23:58.404 | INFO | Downloaded 38GiB to GPU in 2.18133s (17.4206GiB/s) | 20260507 05:24:03.547 | INFO | Analyzing chunk frames = [7426560,7427072) | 20260507 05:24:03.547 | INFO | MemInfo: { | 20260507 05:24:03.547 | INFO | "gpu" : " 17 GB", | 20260507 05:24:03.547 | INFO | "huge" : "", | 20260507 05:24:03.547 | INFO | "huge_shared" : " 27 GB", | 20260507 05:24:03.547 | INFO | "malloc" : " 14 GB", | 20260507 05:24:03.547 | INFO | "pinned" : " 29 GB" | 20260507 05:24:03.547 | INFO | } | 20260507 05:24:03.547 | INFO | Perf report for the last 5.14383 seconds: | 20260507 05:24:03.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:24:03.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:03.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:03.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:03.547 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 05:24:03.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:03.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:03.547 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:24:03.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:03.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:03.547 | INFO | Uploaded 36GiB to GPU in 1.99238s (18.0688GiB/s) | 20260507 05:24:03.547 | INFO | Downloaded 34GiB to GPU in 1.93101s (17.6074GiB/s) | 20260507 05:24:08.676 | INFO | Analyzing chunk frames = [7427072,7427584) | 20260507 05:24:08.676 | INFO | MemInfo: { | 20260507 05:24:08.676 | INFO | "gpu" : " 17 GB", | 20260507 05:24:08.676 | INFO | "huge" : "", | 20260507 05:24:08.676 | INFO | "huge_shared" : " 27 GB", | 20260507 05:24:08.677 | INFO | "malloc" : " 14 GB", | 20260507 05:24:08.677 | INFO | "pinned" : " 29 GB" | 20260507 05:24:08.677 | INFO | } | 20260507 05:24:08.677 | INFO | Perf report for the last 5.12965 seconds: | 20260507 05:24:08.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:24:08.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:08.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:08.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:08.677 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 05:24:08.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:08.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:08.677 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:24:08.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:08.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:08.677 | INFO | Uploaded 36GiB to GPU in 2.0403s (17.6445GiB/s) | 20260507 05:24:08.677 | INFO | Downloaded 34GiB to GPU in 1.91578s (17.7473GiB/s) | 20260507 05:24:13.814 | INFO | Analyzing chunk frames = [7427584,7428096) | 20260507 05:24:13.814 | INFO | MemInfo: { | 20260507 05:24:13.814 | INFO | "gpu" : " 17 GB", | 20260507 05:24:13.814 | INFO | "huge" : "", | 20260507 05:24:13.814 | INFO | "huge_shared" : " 27 GB", | 20260507 05:24:13.814 | INFO | "malloc" : " 14 GB", | 20260507 05:24:13.814 | INFO | "pinned" : " 29 GB" | 20260507 05:24:13.814 | INFO | } | 20260507 05:24:13.814 | INFO | Perf report for the last 5.13733 seconds: | 20260507 05:24:13.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:24:13.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:13.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:13.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:13.814 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:24:13.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:13.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:13.814 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:24:13.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:13.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:13.814 | INFO | Uploaded 36GiB to GPU in 2.0144s (17.8713GiB/s) | 20260507 05:24:13.814 | INFO | Downloaded 34GiB to GPU in 1.93367s (17.5831GiB/s) | 20260507 05:24:18.958 | INFO | Analyzing chunk frames = [7428096,7428608) | 20260507 05:24:18.958 | INFO | MemInfo: { | 20260507 05:24:18.958 | INFO | "gpu" : " 17 GB", | 20260507 05:24:18.958 | INFO | "huge" : "", | 20260507 05:24:18.958 | INFO | "huge_shared" : " 27 GB", | 20260507 05:24:18.958 | INFO | "malloc" : " 14 GB", | 20260507 05:24:18.958 | INFO | "pinned" : " 29 GB" | 20260507 05:24:18.958 | INFO | } | 20260507 05:24:18.958 | INFO | Perf report for the last 5.14383 seconds: | 20260507 05:24:18.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:24:18.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:18.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:18.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:18.958 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 05:24:18.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:18.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:18.958 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:24:18.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:18.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:18.958 | INFO | Uploaded 36GiB to GPU in 2.04314s (17.6199GiB/s) | 20260507 05:24:18.958 | INFO | Downloaded 34GiB to GPU in 1.93513s (17.5698GiB/s) | 20260507 05:24:24.100 | INFO | Analyzing chunk frames = [7428608,7429120) | 20260507 05:24:24.100 | INFO | MemInfo: { | 20260507 05:24:24.100 | INFO | "gpu" : " 17 GB", | 20260507 05:24:24.100 | INFO | "huge" : "", | 20260507 05:24:24.100 | INFO | "huge_shared" : " 27 GB", | 20260507 05:24:24.100 | INFO | "malloc" : " 14 GB", | 20260507 05:24:24.100 | INFO | "pinned" : " 29 GB" | 20260507 05:24:24.100 | INFO | } | 20260507 05:24:24.100 | INFO | Perf report for the last 5.14196 seconds: | 20260507 05:24:24.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:24:24.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:24.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:24.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:24.100 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 05:24:24.100 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:24.100 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:24.100 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 05:24:24.100 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:24.100 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:24.100 | INFO | Uploaded 36GiB to GPU in 2.04565s (17.5983GiB/s) | 20260507 05:24:24.100 | INFO | Downloaded 34GiB to GPU in 1.90787s (17.8209GiB/s) | 20260507 05:24:29.233 | INFO | Analyzing chunk frames = [7429120,7429632) | 20260507 05:24:29.233 | INFO | MemInfo: { | 20260507 05:24:29.233 | INFO | "gpu" : " 17 GB", | 20260507 05:24:29.233 | INFO | "huge" : "", | 20260507 05:24:29.233 | INFO | "huge_shared" : " 27 GB", | 20260507 05:24:29.233 | INFO | "malloc" : " 14 GB", | 20260507 05:24:29.233 | INFO | "pinned" : " 29 GB" | 20260507 05:24:29.233 | INFO | } | 20260507 05:24:29.233 | INFO | Perf report for the last 5.13361 seconds: | 20260507 05:24:29.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:24:29.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:29.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:29.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:29.234 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 05:24:29.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:29.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:29.234 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 05:24:29.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:29.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:29.234 | INFO | Uploaded 36GiB to GPU in 1.92338s (18.717GiB/s) | 20260507 05:24:29.234 | INFO | Downloaded 34GiB to GPU in 1.96947s (17.2635GiB/s) | 20260507 05:24:34.372 | INFO | Analyzing chunk frames = [7429632,7430144) | 20260507 05:24:34.372 | INFO | MemInfo: { | 20260507 05:24:34.372 | INFO | "gpu" : " 17 GB", | 20260507 05:24:34.372 | INFO | "huge" : "", | 20260507 05:24:34.372 | INFO | "huge_shared" : " 27 GB", | 20260507 05:24:34.372 | INFO | "malloc" : " 14 GB", | 20260507 05:24:34.372 | INFO | "pinned" : " 29 GB" | 20260507 05:24:34.372 | INFO | } | 20260507 05:24:34.372 | INFO | Perf report for the last 5.13812 seconds: | 20260507 05:24:34.372 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:24:34.372 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:34.372 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:34.372 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:34.372 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:24:34.372 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:34.372 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:34.372 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:24:34.372 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:34.372 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:34.372 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:24:34.372 | INFO | Total mem usage: 18 GiB | 20260507 05:24:34.372 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:24:34.372 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:24:34.372 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:24:34.372 | INFO | High Water Breakdown: | 20260507 05:24:34.372 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:24:34.372 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:24:34.372 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:24:34.372 | INFO | ------------------------------------------ | 20260507 05:24:34.372 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:24:34.372 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:24:34.372 | INFO | Total mem usage: 26.5 GiB | 20260507 05:24:34.372 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:24:34.372 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:24:34.372 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:24:34.372 | INFO | High Water Breakdown: | 20260507 05:24:34.372 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:24:34.372 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:24:34.372 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:24:34.372 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:24:34.372 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:24:34.372 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:24:34.372 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:24:34.372 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:24:34.372 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:24:34.372 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:24:34.372 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:24:34.372 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:24:34.372 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:24:34.372 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:24:34.372 | INFO | ------------------------------------------ | 20260507 05:24:34.372 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:24:34.372 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:24:34.372 | INFO | Total mem usage: 25.6 GiB | 20260507 05:24:34.372 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:24:34.372 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:24:34.372 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:24:34.372 | INFO | High Water Breakdown: | 20260507 05:24:34.372 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:24:34.372 | INFO | ------------------------------------------ | 20260507 05:24:34.372 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:24:34.372 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:24:34.372 | INFO | Total mem usage: 15.7 GiB | 20260507 05:24:34.372 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:24:34.372 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:24:34.372 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:24:34.372 | INFO | High Water Breakdown: | 20260507 05:24:34.372 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:24:34.372 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:24:34.372 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:24:34.372 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:24:34.372 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:24:34.372 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:24:34.372 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:24:34.372 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:24:34.372 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:24:34.372 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:24:34.372 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:24:34.372 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:24:34.372 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:24:34.372 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:24:34.372 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:24:34.372 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:24:34.372 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:24:34.372 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:24:34.372 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:24:34.372 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:24:34.372 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:24:34.372 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:24:34.372 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:24:34.372 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:24:34.372 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:24:34.372 | INFO | ------------------------------------------ | 20260507 05:24:34.372 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:24:34.372 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:24:34.372 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:24:34.372 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:24:34.372 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.35e+01 / 1.07e+00 | 20260507 05:24:34.372 | INFO | PerfReport: 0.29%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.95e-03 / 6.15e-03 | 20260507 05:24:34.372 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.42e+00, 5.12e-01 / 2.65e-01 | 20260507 05:24:34.372 | INFO | PerfReport: 3.22%, Histogram -- 9.91e-01 / 1.47e+00, 1.68e-01 / 1.17e-01 | 20260507 05:24:34.372 | INFO | PerfReport: 16.19%, DME -- 3.98e+01 / 4.45e+01, 4.02e+00 / 1.64e+00 | 20260507 05:24:34.372 | INFO | PerfReport: 57.68%, FrameLabeling -- 1.77e+01 / 1.99e+01, 1.32e+00 / 1.50e+00 | 20260507 05:24:34.372 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.70e+00, 3.85e-01 / 1.96e-01 | 20260507 05:24:34.372 | INFO | PerfReport: 2.99%, Metrics -- 9.19e-01 / 1.24e+00, 2.43e-01 / 1.36e-01 | 20260507 05:24:34.372 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.09e-01 / 3.62e-01, 5.07e-02 / 2.21e-02 | 20260507 05:24:34.372 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:24:34.372 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:24:34.372 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:24:34.372 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 05:24:34.372 | INFO | PerfReport: 21.79%, Upload -- 2.10e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260507 05:24:34.372 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 05:24:34.372 | INFO | PerfReport: 31.96%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 05:24:34.372 | INFO | PerfReport: 1.50%, LockUpload -- 1.45e+00 / 0.00e+00, 4.87e+00 / 0.00e+00 | 20260507 05:24:34.372 | INFO | PerfReport: 1.25%, LockDownload -- 1.21e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260507 05:24:34.372 | INFO | PerfReport: 22.12%, LockCompute -- 2.13e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 05:24:34.372 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260507 05:24:34.372 | INFO | Uploaded 36GiB to GPU in 1.96526s (18.3182GiB/s) | 20260507 05:24:34.373 | INFO | Downloaded 34GiB to GPU in 1.95741s (17.3699GiB/s) | 20260507 05:24:34.470 | INFO | Starting a round of Baz IO | 20260507 05:24:37.569 | INFO | Populating RTMetrics report | 20260507 05:24:37.569 | INFO | RTMetrics report populated | 20260507 05:24:37.569 | INFO | Opening tmp RTMetrics json file | 20260507 05:24:37.586 | INFO | Serializing RTMetrics to json | 20260507 05:24:37.587 | INFO | Serialization done, swapping tmp file now... | 20260507 05:24:37.588 | INFO | RTMetrics done | 20260507 05:24:37.655 | INFO | Finished a round of Baz IO: Wrote 1.697 GiB in 3.185 seconds | 20260507 05:24:37.655 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.71% / 0.00% / 67.29% | 20260507 05:24:37.655 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.145682% / 0% / 4.37101e-05% | 20260507 05:24:37.655 | INFO | Baz IO overall: | 20260507 05:24:37.655 | INFO | BytesWritten : 3770923856898 | 20260507 05:24:37.655 | INFO | NumEvents : 1808811514930 | 20260507 05:24:39.517 | INFO | Analyzing chunk frames = [7430144,7430656) | 20260507 05:24:39.517 | INFO | MemInfo: { | 20260507 05:24:39.517 | INFO | "gpu" : " 17 GB", | 20260507 05:24:39.517 | INFO | "huge" : "", | 20260507 05:24:39.517 | INFO | "huge_shared" : " 27 GB", | 20260507 05:24:39.517 | INFO | "malloc" : " 14 GB", | 20260507 05:24:39.517 | INFO | "pinned" : " 29 GB" | 20260507 05:24:39.517 | INFO | } | 20260507 05:24:39.517 | INFO | Perf report for the last 5.145 seconds: | 20260507 05:24:39.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:24:39.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:39.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:39.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:39.517 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:24:39.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:39.517 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 05:24:39.517 | INFO | PRE_HQ : 58.9%, 1.8, 54.6ms, 0 | 20260507 05:24:39.517 | INFO | BAZWRITER : 61.5%, 3.2, 100.0ms, 0 | 20260507 05:24:39.517 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:24:39.517 | INFO | Uploaded 36GiB to GPU in 2.02926s (17.7404GiB/s) | 20260507 05:24:39.517 | INFO | Downloaded 38GiB to GPU in 2.18261s (17.4104GiB/s) | 20260507 05:24:44.655 | INFO | Analyzing chunk frames = [7430656,7431168) | 20260507 05:24:44.655 | INFO | MemInfo: { | 20260507 05:24:44.655 | INFO | "gpu" : " 17 GB", | 20260507 05:24:44.655 | INFO | "huge" : "", | 20260507 05:24:44.655 | INFO | "huge_shared" : " 27 GB", | 20260507 05:24:44.655 | INFO | "malloc" : " 14 GB", | 20260507 05:24:44.655 | INFO | "pinned" : " 29 GB" | 20260507 05:24:44.655 | INFO | } | 20260507 05:24:44.655 | INFO | Perf report for the last 5.13823 seconds: | 20260507 05:24:44.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:24:44.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:44.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:44.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:44.655 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 05:24:44.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:44.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:44.655 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:24:44.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:44.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:44.655 | INFO | Uploaded 36GiB to GPU in 2.00603s (17.9459GiB/s) | 20260507 05:24:44.655 | INFO | Downloaded 34GiB to GPU in 1.92627s (17.6507GiB/s) | 20260507 05:24:49.792 | INFO | Analyzing chunk frames = [7431168,7431680) | 20260507 05:24:49.792 | INFO | MemInfo: { | 20260507 05:24:49.792 | INFO | "gpu" : " 17 GB", | 20260507 05:24:49.792 | INFO | "huge" : "", | 20260507 05:24:49.792 | INFO | "huge_shared" : " 27 GB", | 20260507 05:24:49.792 | INFO | "malloc" : " 14 GB", | 20260507 05:24:49.792 | INFO | "pinned" : " 29 GB" | 20260507 05:24:49.792 | INFO | } | 20260507 05:24:49.792 | INFO | Perf report for the last 5.13706 seconds: | 20260507 05:24:49.792 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:24:49.792 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:49.792 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:49.792 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:49.792 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 05:24:49.792 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:49.792 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:49.792 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:24:49.792 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:49.792 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:49.792 | INFO | Uploaded 36GiB to GPU in 2.02778s (17.7534GiB/s) | 20260507 05:24:49.792 | INFO | Downloaded 34GiB to GPU in 1.94061s (17.5203GiB/s) | 20260507 05:24:54.934 | INFO | Analyzing chunk frames = [7431680,7432192) | 20260507 05:24:54.934 | INFO | MemInfo: { | 20260507 05:24:54.934 | INFO | "gpu" : " 17 GB", | 20260507 05:24:54.934 | INFO | "huge" : "", | 20260507 05:24:54.934 | INFO | "huge_shared" : " 27 GB", | 20260507 05:24:54.934 | INFO | "malloc" : " 14 GB", | 20260507 05:24:54.934 | INFO | "pinned" : " 29 GB" | 20260507 05:24:54.934 | INFO | } | 20260507 05:24:54.935 | INFO | Perf report for the last 5.14256 seconds: | 20260507 05:24:54.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:24:54.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:54.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:54.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:54.935 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 05:24:54.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:54.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:54.935 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:24:54.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:54.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:24:54.935 | INFO | Uploaded 36GiB to GPU in 2.04236s (17.6267GiB/s) | 20260507 05:24:54.935 | INFO | Downloaded 34GiB to GPU in 1.94913s (17.4437GiB/s) | 20260507 05:25:00.073 | INFO | Analyzing chunk frames = [7432192,7432704) | 20260507 05:25:00.073 | INFO | MemInfo: { | 20260507 05:25:00.073 | INFO | "gpu" : " 17 GB", | 20260507 05:25:00.073 | INFO | "huge" : "", | 20260507 05:25:00.073 | INFO | "huge_shared" : " 27 GB", | 20260507 05:25:00.074 | INFO | "malloc" : " 14 GB", | 20260507 05:25:00.074 | INFO | "pinned" : " 29 GB" | 20260507 05:25:00.074 | INFO | } | 20260507 05:25:00.074 | INFO | Perf report for the last 5.13915 seconds: | 20260507 05:25:00.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:25:00.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:00.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:00.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:00.074 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 05:25:00.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:00.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:00.074 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:25:00.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:00.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:00.074 | INFO | Uploaded 36GiB to GPU in 2.05662s (17.5044GiB/s) | 20260507 05:25:00.074 | INFO | Downloaded 34GiB to GPU in 1.94791s (17.4546GiB/s) | 20260507 05:25:05.212 | INFO | Analyzing chunk frames = [7432704,7433216) | 20260507 05:25:05.212 | INFO | MemInfo: { | 20260507 05:25:05.212 | INFO | "gpu" : " 17 GB", | 20260507 05:25:05.212 | INFO | "huge" : "", | 20260507 05:25:05.212 | INFO | "huge_shared" : " 27 GB", | 20260507 05:25:05.212 | INFO | "malloc" : " 14 GB", | 20260507 05:25:05.212 | INFO | "pinned" : " 29 GB" | 20260507 05:25:05.212 | INFO | } | 20260507 05:25:05.212 | INFO | Perf report for the last 5.13864 seconds: | 20260507 05:25:05.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:25:05.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:05.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:05.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:05.212 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 05:25:05.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:05.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:05.213 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 05:25:05.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:05.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:05.213 | INFO | Uploaded 36GiB to GPU in 2.0415s (17.6341GiB/s) | 20260507 05:25:05.213 | INFO | Downloaded 34GiB to GPU in 1.91119s (17.7899GiB/s) | 20260507 05:25:10.354 | INFO | Analyzing chunk frames = [7433216,7433728) | 20260507 05:25:10.354 | INFO | MemInfo: { | 20260507 05:25:10.354 | INFO | "gpu" : " 17 GB", | 20260507 05:25:10.354 | INFO | "huge" : "", | 20260507 05:25:10.354 | INFO | "huge_shared" : " 27 GB", | 20260507 05:25:10.354 | INFO | "malloc" : " 14 GB", | 20260507 05:25:10.354 | INFO | "pinned" : " 29 GB" | 20260507 05:25:10.354 | INFO | } | 20260507 05:25:10.354 | INFO | Perf report for the last 5.14178 seconds: | 20260507 05:25:10.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:25:10.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:10.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:10.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:10.354 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 05:25:10.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:10.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:10.354 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:25:10.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:10.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:10.354 | INFO | Uploaded 36GiB to GPU in 1.98135s (18.1694GiB/s) | 20260507 05:25:10.354 | INFO | Downloaded 34GiB to GPU in 1.95626s (17.3801GiB/s) | 20260507 05:25:15.492 | INFO | Analyzing chunk frames = [7433728,7434240) | 20260507 05:25:15.492 | INFO | MemInfo: { | 20260507 05:25:15.492 | INFO | "gpu" : " 17 GB", | 20260507 05:25:15.492 | INFO | "huge" : "", | 20260507 05:25:15.492 | INFO | "huge_shared" : " 27 GB", | 20260507 05:25:15.492 | INFO | "malloc" : " 14 GB", | 20260507 05:25:15.492 | INFO | "pinned" : " 29 GB" | 20260507 05:25:15.492 | INFO | } | 20260507 05:25:15.492 | INFO | Perf report for the last 5.13785 seconds: | 20260507 05:25:15.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:25:15.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:15.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:15.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:15.492 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:25:15.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:15.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:15.492 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:25:15.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:15.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:15.492 | INFO | Uploaded 36GiB to GPU in 1.98475s (18.1383GiB/s) | 20260507 05:25:15.492 | INFO | Downloaded 34GiB to GPU in 1.96996s (17.2592GiB/s) | 20260507 05:25:15.589 | INFO | Starting a round of Baz IO | 20260507 05:25:18.684 | INFO | Populating RTMetrics report | 20260507 05:25:18.685 | INFO | RTMetrics report populated | 20260507 05:25:18.685 | INFO | Opening tmp RTMetrics json file | 20260507 05:25:18.689 | INFO | Serializing RTMetrics to json | 20260507 05:25:18.702 | INFO | Serialization done, swapping tmp file now... | 20260507 05:25:18.703 | INFO | RTMetrics done | 20260507 05:25:18.769 | INFO | Finished a round of Baz IO: Wrote 1.697 GiB in 3.179 seconds | 20260507 05:25:18.769 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.71% / 0.00% / 67.29% | 20260507 05:25:18.769 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.144283% / 0% / 6.75519e-05% | 20260507 05:25:18.769 | INFO | Baz IO overall: | 20260507 05:25:18.769 | INFO | BytesWritten : 3772745487736 | 20260507 05:25:18.769 | INFO | NumEvents : 1809667093435 | 20260507 05:25:20.623 | INFO | Analyzing chunk frames = [7434240,7434752) | 20260507 05:25:20.623 | INFO | MemInfo: { | 20260507 05:25:20.623 | INFO | "gpu" : " 17 GB", | 20260507 05:25:20.623 | INFO | "huge" : "", | 20260507 05:25:20.623 | INFO | "huge_shared" : " 27 GB", | 20260507 05:25:20.624 | INFO | "malloc" : " 14 GB", | 20260507 05:25:20.624 | INFO | "pinned" : " 29 GB" | 20260507 05:25:20.624 | INFO | } | 20260507 05:25:20.624 | INFO | Perf report for the last 5.13161 seconds: | 20260507 05:25:20.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:25:20.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:20.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:20.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:20.624 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260507 05:25:20.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:20.624 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 05:25:20.624 | INFO | PRE_HQ : 59.1%, 1.8, 54.8ms, 0 | 20260507 05:25:20.624 | INFO | BAZWRITER : 61.6%, 3.1, 99.3ms, 0 | 20260507 05:25:20.624 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:25:20.624 | INFO | Uploaded 36GiB to GPU in 2.0541s (17.5259GiB/s) | 20260507 05:25:20.624 | INFO | Downloaded 38GiB to GPU in 2.18053s (17.427GiB/s) | 20260507 05:25:25.770 | INFO | Analyzing chunk frames = [7434752,7435264) | 20260507 05:25:25.770 | INFO | MemInfo: { | 20260507 05:25:25.770 | INFO | "gpu" : " 17 GB", | 20260507 05:25:25.770 | INFO | "huge" : "", | 20260507 05:25:25.770 | INFO | "huge_shared" : " 27 GB", | 20260507 05:25:25.770 | INFO | "malloc" : " 14 GB", | 20260507 05:25:25.770 | INFO | "pinned" : " 29 GB" | 20260507 05:25:25.770 | INFO | } | 20260507 05:25:25.770 | INFO | Perf report for the last 5.14666 seconds: | 20260507 05:25:25.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:25:25.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:25.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:25.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:25.770 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 05:25:25.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:25.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:25.770 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:25:25.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:25.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:25.771 | INFO | Uploaded 36GiB to GPU in 2.04249s (17.6256GiB/s) | 20260507 05:25:25.771 | INFO | Downloaded 34GiB to GPU in 1.9447s (17.4834GiB/s) | 20260507 05:25:30.913 | INFO | Analyzing chunk frames = [7435264,7435776) | 20260507 05:25:30.913 | INFO | MemInfo: { | 20260507 05:25:30.913 | INFO | "gpu" : " 17 GB", | 20260507 05:25:30.913 | INFO | "huge" : "", | 20260507 05:25:30.913 | INFO | "huge_shared" : " 27 GB", | 20260507 05:25:30.913 | INFO | "malloc" : " 14 GB", | 20260507 05:25:30.913 | INFO | "pinned" : " 29 GB" | 20260507 05:25:30.913 | INFO | } | 20260507 05:25:30.913 | INFO | Perf report for the last 5.14296 seconds: | 20260507 05:25:30.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:25:30.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:30.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:30.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:30.913 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 05:25:30.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:30.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:30.913 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:25:30.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:30.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:30.914 | INFO | Uploaded 36GiB to GPU in 2.02562s (17.7723GiB/s) | 20260507 05:25:30.914 | INFO | Downloaded 34GiB to GPU in 1.9577s (17.3673GiB/s) | 20260507 05:25:36.053 | INFO | Analyzing chunk frames = [7435776,7436288) | 20260507 05:25:36.053 | INFO | MemInfo: { | 20260507 05:25:36.053 | INFO | "gpu" : " 17 GB", | 20260507 05:25:36.053 | INFO | "huge" : "", | 20260507 05:25:36.053 | INFO | "huge_shared" : " 27 GB", | 20260507 05:25:36.053 | INFO | "malloc" : " 14 GB", | 20260507 05:25:36.053 | INFO | "pinned" : " 29 GB" | 20260507 05:25:36.054 | INFO | } | 20260507 05:25:36.054 | INFO | Perf report for the last 5.14032 seconds: | 20260507 05:25:36.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:25:36.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:36.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:36.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:36.054 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 05:25:36.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:36.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:36.054 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:25:36.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:36.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:36.054 | INFO | Uploaded 36GiB to GPU in 1.9912s (18.0796GiB/s) | 20260507 05:25:36.054 | INFO | Downloaded 34GiB to GPU in 1.94589s (17.4727GiB/s) | 20260507 05:25:41.189 | INFO | Analyzing chunk frames = [7436288,7436800) | 20260507 05:25:41.189 | INFO | MemInfo: { | 20260507 05:25:41.189 | INFO | "gpu" : " 17 GB", | 20260507 05:25:41.189 | INFO | "huge" : "", | 20260507 05:25:41.189 | INFO | "huge_shared" : " 27 GB", | 20260507 05:25:41.189 | INFO | "malloc" : " 14 GB", | 20260507 05:25:41.189 | INFO | "pinned" : " 29 GB" | 20260507 05:25:41.189 | INFO | } | 20260507 05:25:41.189 | INFO | Perf report for the last 5.13571 seconds: | 20260507 05:25:41.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:25:41.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:41.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:41.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:41.189 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 05:25:41.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:41.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:41.189 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:25:41.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:41.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:41.190 | INFO | Uploaded 36GiB to GPU in 2.0287s (17.7453GiB/s) | 20260507 05:25:41.190 | INFO | Downloaded 34GiB to GPU in 1.91221s (17.7805GiB/s) | 20260507 05:25:46.324 | INFO | Analyzing chunk frames = [7436800,7437312) | 20260507 05:25:46.324 | INFO | MemInfo: { | 20260507 05:25:46.324 | INFO | "gpu" : " 17 GB", | 20260507 05:25:46.324 | INFO | "huge" : "", | 20260507 05:25:46.324 | INFO | "huge_shared" : " 27 GB", | 20260507 05:25:46.324 | INFO | "malloc" : " 14 GB", | 20260507 05:25:46.324 | INFO | "pinned" : " 29 GB" | 20260507 05:25:46.324 | INFO | } | 20260507 05:25:46.324 | INFO | Perf report for the last 5.13464 seconds: | 20260507 05:25:46.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:25:46.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:46.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:46.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:46.324 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 05:25:46.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:46.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:46.324 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 05:25:46.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:46.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:46.324 | INFO | Uploaded 36GiB to GPU in 2.03366s (17.7021GiB/s) | 20260507 05:25:46.324 | INFO | Downloaded 34GiB to GPU in 1.92037s (17.7049GiB/s) | 20260507 05:25:51.469 | INFO | Analyzing chunk frames = [7437312,7437824) | 20260507 05:25:51.469 | INFO | MemInfo: { | 20260507 05:25:51.469 | INFO | "gpu" : " 17 GB", | 20260507 05:25:51.469 | INFO | "huge" : "", | 20260507 05:25:51.469 | INFO | "huge_shared" : " 27 GB", | 20260507 05:25:51.469 | INFO | "malloc" : " 14 GB", | 20260507 05:25:51.469 | INFO | "pinned" : " 29 GB" | 20260507 05:25:51.469 | INFO | } | 20260507 05:25:51.469 | INFO | Perf report for the last 5.14509 seconds: | 20260507 05:25:51.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:25:51.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:51.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:51.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:51.469 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 05:25:51.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:51.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:51.469 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 05:25:51.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:51.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:51.469 | INFO | Uploaded 36GiB to GPU in 1.96231s (18.3457GiB/s) | 20260507 05:25:51.469 | INFO | Downloaded 34GiB to GPU in 1.97574s (17.2088GiB/s) | 20260507 05:25:56.606 | INFO | Analyzing chunk frames = [7437824,7438336) | 20260507 05:25:56.606 | INFO | MemInfo: { | 20260507 05:25:56.606 | INFO | "gpu" : " 17 GB", | 20260507 05:25:56.606 | INFO | "huge" : "", | 20260507 05:25:56.606 | INFO | "huge_shared" : " 27 GB", | 20260507 05:25:56.607 | INFO | "malloc" : " 14 GB", | 20260507 05:25:56.607 | INFO | "pinned" : " 29 GB" | 20260507 05:25:56.607 | INFO | } | 20260507 05:25:56.607 | INFO | Perf report for the last 5.13763 seconds: | 20260507 05:25:56.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:25:56.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:56.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:56.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:56.607 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:25:56.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:56.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:56.607 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 05:25:56.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:56.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:25:56.607 | INFO | Uploaded 36GiB to GPU in 1.97826s (18.1978GiB/s) | 20260507 05:25:56.607 | INFO | Downloaded 34GiB to GPU in 1.96966s (17.2619GiB/s) | 20260507 05:25:56.607 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:25:56.607 | INFO | Total mem usage: 18 GiB | 20260507 05:25:56.607 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:25:56.607 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:25:56.607 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:25:56.607 | INFO | High Water Breakdown: | 20260507 05:25:56.607 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:25:56.607 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:25:56.607 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:25:56.607 | INFO | ------------------------------------------ | 20260507 05:25:56.608 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:25:56.608 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:25:56.608 | INFO | Total mem usage: 26.5 GiB | 20260507 05:25:56.608 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:25:56.608 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:25:56.608 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 05:25:56.608 | INFO | High Water Breakdown: | 20260507 05:25:56.608 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:25:56.608 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:25:56.608 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:25:56.608 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:25:56.608 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:25:56.608 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:25:56.608 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:25:56.608 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:25:56.608 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:25:56.608 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:25:56.608 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:25:56.608 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:25:56.608 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:25:56.608 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:25:56.608 | INFO | ------------------------------------------ | 20260507 05:25:56.608 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:25:56.608 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:25:56.608 | INFO | Total mem usage: 25.6 GiB | 20260507 05:25:56.608 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:25:56.608 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:25:56.608 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:25:56.608 | INFO | High Water Breakdown: | 20260507 05:25:56.608 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:25:56.608 | INFO | ------------------------------------------ | 20260507 05:25:56.608 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:25:56.608 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:25:56.608 | INFO | Total mem usage: 15.7 GiB | 20260507 05:25:56.608 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:25:56.608 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:25:56.608 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:25:56.608 | INFO | High Water Breakdown: | 20260507 05:25:56.608 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:25:56.608 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:25:56.608 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:25:56.608 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:25:56.608 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:25:56.608 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:25:56.608 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:25:56.608 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:25:56.608 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:25:56.608 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:25:56.608 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:25:56.608 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:25:56.608 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:25:56.608 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:25:56.608 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:25:56.608 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:25:56.608 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:25:56.608 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:25:56.608 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:25:56.608 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:25:56.608 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:25:56.608 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:25:56.608 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:25:56.608 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:25:56.608 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:25:56.608 | INFO | ------------------------------------------ | 20260507 05:25:56.608 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:25:56.608 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:25:56.608 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:25:56.608 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:25:56.608 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.35e+01 / 1.43e+00 | 20260507 05:25:56.608 | INFO | PerfReport: 0.30%, AggregateStats -- 9.06e-02 / 1.01e-01, 9.53e-03 / 3.94e-03 | 20260507 05:25:56.608 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.42e+00, 5.08e-01 / 2.85e-01 | 20260507 05:25:56.608 | INFO | PerfReport: 3.23%, Histogram -- 9.90e-01 / 1.47e+00, 1.67e-01 / 1.10e-01 | 20260507 05:25:56.608 | INFO | PerfReport: 16.25%, DME -- 3.99e+01 / 4.48e+01, 3.96e+00 / 1.70e+00 | 20260507 05:25:56.608 | INFO | PerfReport: 57.62%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.23e+00 / 1.14e+00 | 20260507 05:25:56.608 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.13e+00 / 2.71e+00, 3.89e-01 / 2.07e-01 | 20260507 05:25:56.608 | INFO | PerfReport: 2.99%, Metrics -- 9.18e-01 / 1.24e+00, 2.43e-01 / 1.32e-01 | 20260507 05:25:56.608 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.09e-01 / 3.62e-01, 5.07e-02 / 2.10e-02 | 20260507 05:25:56.609 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:25:56.609 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:25:56.609 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:25:56.609 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 05:25:56.609 | INFO | PerfReport: 21.84%, Upload -- 2.10e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260507 05:25:56.609 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 05:25:56.609 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 05:25:56.609 | INFO | PerfReport: 1.56%, LockUpload -- 1.50e+00 / 0.00e+00, 4.86e+00 / 0.00e+00 | 20260507 05:25:56.609 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260507 05:25:56.609 | INFO | PerfReport: 21.97%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 05:25:56.609 | INFO | PerfReport: 0.02%, Misc -- 1.55e-02 / 0.00e+00, 4.90e-02 / 0.00e+00 | 20260507 05:25:56.703 | INFO | Starting a round of Baz IO | 20260507 05:25:59.801 | INFO | Populating RTMetrics report | 20260507 05:25:59.802 | INFO | RTMetrics report populated | 20260507 05:25:59.802 | INFO | Opening tmp RTMetrics json file | 20260507 05:25:59.803 | INFO | Serializing RTMetrics to json | 20260507 05:25:59.816 | INFO | Serialization done, swapping tmp file now... | 20260507 05:25:59.821 | INFO | RTMetrics done | 20260507 05:25:59.890 | INFO | Finished a round of Baz IO: Wrote 1.698 GiB in 3.186 seconds | 20260507 05:25:59.890 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.71% / 0.00% / 67.29% | 20260507 05:25:59.890 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.144486% / 0% / 6.75519e-05% | 20260507 05:25:59.890 | INFO | Baz IO overall: | 20260507 05:25:59.890 | INFO | BytesWritten : 3774568659616 | 20260507 05:25:59.890 | INFO | NumEvents : 1810523634580 | 20260507 05:26:01.746 | INFO | Analyzing chunk frames = [7438336,7438848) | 20260507 05:26:01.746 | INFO | MemInfo: { | 20260507 05:26:01.746 | INFO | "gpu" : " 17 GB", | 20260507 05:26:01.746 | INFO | "huge" : "", | 20260507 05:26:01.746 | INFO | "huge_shared" : " 27 GB", | 20260507 05:26:01.746 | INFO | "malloc" : " 14 GB", | 20260507 05:26:01.746 | INFO | "pinned" : " 29 GB" | 20260507 05:26:01.746 | INFO | } | 20260507 05:26:01.746 | INFO | Perf report for the last 5.13945 seconds: | 20260507 05:26:01.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:26:01.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:01.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:01.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:01.746 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:26:01.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:01.746 | INFO | RT_METRICS : 41.7%, 1.0, 21.8ms, 0 | 20260507 05:26:01.746 | INFO | PRE_HQ : 59.1%, 1.8, 55.1ms, 0 | 20260507 05:26:01.746 | INFO | BAZWRITER : 61.6%, 3.2, 100.5ms, 0 | 20260507 05:26:01.746 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:26:01.747 | INFO | Uploaded 36GiB to GPU in 2.05137s (17.5493GiB/s) | 20260507 05:26:01.747 | INFO | Downloaded 38GiB to GPU in 2.18075s (17.4252GiB/s) | 20260507 05:26:06.880 | INFO | Analyzing chunk frames = [7438848,7439360) | 20260507 05:26:06.880 | INFO | MemInfo: { | 20260507 05:26:06.880 | INFO | "gpu" : " 17 GB", | 20260507 05:26:06.880 | INFO | "huge" : "", | 20260507 05:26:06.880 | INFO | "huge_shared" : " 27 GB", | 20260507 05:26:06.880 | INFO | "malloc" : " 14 GB", | 20260507 05:26:06.880 | INFO | "pinned" : " 29 GB" | 20260507 05:26:06.880 | INFO | } | 20260507 05:26:06.880 | INFO | Perf report for the last 5.13375 seconds: | 20260507 05:26:06.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:26:06.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:06.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:06.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:06.880 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 05:26:06.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:06.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:06.880 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:26:06.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:06.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:06.880 | INFO | Uploaded 36GiB to GPU in 1.99841s (18.0143GiB/s) | 20260507 05:26:06.880 | INFO | Downloaded 34GiB to GPU in 1.93959s (17.5295GiB/s) | 20260507 05:26:12.024 | INFO | Analyzing chunk frames = [7439360,7439872) | 20260507 05:26:12.024 | INFO | MemInfo: { | 20260507 05:26:12.024 | INFO | "gpu" : " 17 GB", | 20260507 05:26:12.024 | INFO | "huge" : "", | 20260507 05:26:12.024 | INFO | "huge_shared" : " 27 GB", | 20260507 05:26:12.024 | INFO | "malloc" : " 14 GB", | 20260507 05:26:12.024 | INFO | "pinned" : " 29 GB" | 20260507 05:26:12.024 | INFO | } | 20260507 05:26:12.024 | INFO | Perf report for the last 5.14416 seconds: | 20260507 05:26:12.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:26:12.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:12.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:12.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:12.024 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 05:26:12.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:12.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:12.024 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:26:12.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:12.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:12.024 | INFO | Uploaded 36GiB to GPU in 1.99159s (18.076GiB/s) | 20260507 05:26:12.024 | INFO | Downloaded 34GiB to GPU in 1.94277s (17.5008GiB/s) | 20260507 05:26:17.172 | INFO | Analyzing chunk frames = [7439872,7440384) | 20260507 05:26:17.172 | INFO | MemInfo: { | 20260507 05:26:17.172 | INFO | "gpu" : " 17 GB", | 20260507 05:26:17.172 | INFO | "huge" : "", | 20260507 05:26:17.172 | INFO | "huge_shared" : " 27 GB", | 20260507 05:26:17.172 | INFO | "malloc" : " 14 GB", | 20260507 05:26:17.172 | INFO | "pinned" : " 29 GB" | 20260507 05:26:17.172 | INFO | } | 20260507 05:26:17.172 | INFO | Perf report for the last 5.14784 seconds: | 20260507 05:26:17.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:26:17.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:17.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:17.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:17.172 | INFO | ANALYSIS : 59.3%, 3.0, 92.2ms, 0 | 20260507 05:26:17.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:17.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:17.172 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:26:17.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:17.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:17.172 | INFO | Uploaded 36GiB to GPU in 1.97178s (18.2576GiB/s) | 20260507 05:26:17.172 | INFO | Downloaded 34GiB to GPU in 1.9171s (17.7351GiB/s) | 20260507 05:26:22.305 | INFO | Analyzing chunk frames = [7440384,7440896) | 20260507 05:26:22.305 | INFO | MemInfo: { | 20260507 05:26:22.305 | INFO | "gpu" : " 17 GB", | 20260507 05:26:22.305 | INFO | "huge" : "", | 20260507 05:26:22.305 | INFO | "huge_shared" : " 27 GB", | 20260507 05:26:22.306 | INFO | "malloc" : " 14 GB", | 20260507 05:26:22.306 | INFO | "pinned" : " 29 GB" | 20260507 05:26:22.306 | INFO | } | 20260507 05:26:22.306 | INFO | Perf report for the last 5.13392 seconds: | 20260507 05:26:22.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:26:22.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:22.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:22.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:22.306 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 05:26:22.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:22.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:22.306 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:26:22.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:22.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:22.306 | INFO | Uploaded 36GiB to GPU in 2.02543s (17.774GiB/s) | 20260507 05:26:22.306 | INFO | Downloaded 34GiB to GPU in 1.90026s (17.8923GiB/s) | 20260507 05:26:27.439 | INFO | Analyzing chunk frames = [7440896,7441408) | 20260507 05:26:27.439 | INFO | MemInfo: { | 20260507 05:26:27.439 | INFO | "gpu" : " 17 GB", | 20260507 05:26:27.439 | INFO | "huge" : "", | 20260507 05:26:27.439 | INFO | "huge_shared" : " 27 GB", | 20260507 05:26:27.439 | INFO | "malloc" : " 14 GB", | 20260507 05:26:27.440 | INFO | "pinned" : " 29 GB" | 20260507 05:26:27.440 | INFO | } | 20260507 05:26:27.440 | INFO | Perf report for the last 5.13379 seconds: | 20260507 05:26:27.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:26:27.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:27.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:27.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:27.440 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 05:26:27.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:27.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:27.440 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:26:27.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:27.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:27.440 | INFO | Uploaded 36GiB to GPU in 2.05841s (17.4892GiB/s) | 20260507 05:26:27.440 | INFO | Downloaded 34GiB to GPU in 1.92674s (17.6464GiB/s) | 20260507 05:26:32.576 | INFO | Analyzing chunk frames = [7441408,7441920) | 20260507 05:26:32.576 | INFO | MemInfo: { | 20260507 05:26:32.576 | INFO | "gpu" : " 17 GB", | 20260507 05:26:32.576 | INFO | "huge" : "", | 20260507 05:26:32.576 | INFO | "huge_shared" : " 27 GB", | 20260507 05:26:32.576 | INFO | "malloc" : " 14 GB", | 20260507 05:26:32.576 | INFO | "pinned" : " 29 GB" | 20260507 05:26:32.576 | INFO | } | 20260507 05:26:32.576 | INFO | Perf report for the last 5.13645 seconds: | 20260507 05:26:32.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:26:32.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:32.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:32.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:32.576 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:26:32.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:32.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:32.576 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 05:26:32.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:32.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:32.576 | INFO | Uploaded 36GiB to GPU in 1.96282s (18.341GiB/s) | 20260507 05:26:32.577 | INFO | Downloaded 34GiB to GPU in 1.9799s (17.1726GiB/s) | 20260507 05:26:37.721 | INFO | Analyzing chunk frames = [7441920,7442432) | 20260507 05:26:37.721 | INFO | MemInfo: { | 20260507 05:26:37.721 | INFO | "gpu" : " 17 GB", | 20260507 05:26:37.721 | INFO | "huge" : "", | 20260507 05:26:37.721 | INFO | "huge_shared" : " 27 GB", | 20260507 05:26:37.721 | INFO | "malloc" : " 14 GB", | 20260507 05:26:37.721 | INFO | "pinned" : " 29 GB" | 20260507 05:26:37.721 | INFO | } | 20260507 05:26:37.721 | INFO | Perf report for the last 5.14482 seconds: | 20260507 05:26:37.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:26:37.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:37.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:37.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:37.721 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 05:26:37.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:37.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:37.721 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 05:26:37.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:37.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:37.721 | INFO | Uploaded 36GiB to GPU in 1.9755s (18.2232GiB/s) | 20260507 05:26:37.721 | INFO | Downloaded 34GiB to GPU in 1.95895s (17.3563GiB/s) | 20260507 05:26:37.819 | INFO | Starting a round of Baz IO | 20260507 05:26:40.917 | INFO | Populating RTMetrics report | 20260507 05:26:40.917 | INFO | RTMetrics report populated | 20260507 05:26:40.917 | INFO | Opening tmp RTMetrics json file | 20260507 05:26:40.924 | INFO | Serializing RTMetrics to json | 20260507 05:26:40.934 | INFO | Serialization done, swapping tmp file now... | 20260507 05:26:40.934 | INFO | RTMetrics done | 20260507 05:26:41.003 | INFO | Finished a round of Baz IO: Wrote 1.696 GiB in 3.183 seconds | 20260507 05:26:41.003 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.71% / 0.00% / 67.29% | 20260507 05:26:41.003 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.14478% / 0% / 6.35783e-05% | 20260507 05:26:41.003 | INFO | Baz IO overall: | 20260507 05:26:41.003 | INFO | BytesWritten : 3776390222965 | 20260507 05:26:41.003 | INFO | NumEvents : 1811378826472 | 20260507 05:26:42.863 | INFO | Analyzing chunk frames = [7442432,7442944) | 20260507 05:26:42.863 | INFO | MemInfo: { | 20260507 05:26:42.863 | INFO | "gpu" : " 17 GB", | 20260507 05:26:42.863 | INFO | "huge" : "", | 20260507 05:26:42.863 | INFO | "huge_shared" : " 27 GB", | 20260507 05:26:42.863 | INFO | "malloc" : " 14 GB", | 20260507 05:26:42.863 | INFO | "pinned" : " 29 GB" | 20260507 05:26:42.863 | INFO | } | 20260507 05:26:42.863 | INFO | Perf report for the last 5.14191 seconds: | 20260507 05:26:42.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:26:42.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:42.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:42.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:42.863 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:26:42.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:42.863 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 05:26:42.863 | INFO | PRE_HQ : 58.9%, 1.8, 54.8ms, 0 | 20260507 05:26:42.863 | INFO | BAZWRITER : 61.5%, 3.1, 99.5ms, 0 | 20260507 05:26:42.863 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 05:26:42.863 | INFO | Uploaded 36GiB to GPU in 2.04382s (17.614GiB/s) | 20260507 05:26:42.863 | INFO | Downloaded 38GiB to GPU in 2.17338s (17.4843GiB/s) | 20260507 05:26:47.998 | INFO | Analyzing chunk frames = [7442944,7443456) | 20260507 05:26:47.998 | INFO | MemInfo: { | 20260507 05:26:47.998 | INFO | "gpu" : " 17 GB", | 20260507 05:26:47.998 | INFO | "huge" : "", | 20260507 05:26:47.998 | INFO | "huge_shared" : " 27 GB", | 20260507 05:26:47.998 | INFO | "malloc" : " 14 GB", | 20260507 05:26:47.998 | INFO | "pinned" : " 29 GB" | 20260507 05:26:47.998 | INFO | } | 20260507 05:26:47.998 | INFO | Perf report for the last 5.13507 seconds: | 20260507 05:26:47.998 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:26:47.998 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:47.998 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:47.998 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:47.998 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:26:47.998 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:47.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:47.999 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:26:47.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:47.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:47.999 | INFO | Uploaded 36GiB to GPU in 1.98258s (18.1582GiB/s) | 20260507 05:26:47.999 | INFO | Downloaded 34GiB to GPU in 1.93418s (17.5785GiB/s) | 20260507 05:26:53.138 | INFO | Analyzing chunk frames = [7443456,7443968) | 20260507 05:26:53.138 | INFO | MemInfo: { | 20260507 05:26:53.138 | INFO | "gpu" : " 17 GB", | 20260507 05:26:53.138 | INFO | "huge" : "", | 20260507 05:26:53.138 | INFO | "huge_shared" : " 27 GB", | 20260507 05:26:53.138 | INFO | "malloc" : " 14 GB", | 20260507 05:26:53.138 | INFO | "pinned" : " 29 GB" | 20260507 05:26:53.138 | INFO | } | 20260507 05:26:53.138 | INFO | Perf report for the last 5.14006 seconds: | 20260507 05:26:53.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:26:53.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:53.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:53.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:53.138 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 05:26:53.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:53.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:53.138 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:26:53.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:53.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:53.138 | INFO | Uploaded 36GiB to GPU in 2.04992s (17.5617GiB/s) | 20260507 05:26:53.138 | INFO | Downloaded 34GiB to GPU in 1.93985s (17.5272GiB/s) | 20260507 05:26:58.280 | INFO | Analyzing chunk frames = [7443968,7444480) | 20260507 05:26:58.280 | INFO | MemInfo: { | 20260507 05:26:58.280 | INFO | "gpu" : " 17 GB", | 20260507 05:26:58.280 | INFO | "huge" : "", | 20260507 05:26:58.280 | INFO | "huge_shared" : " 27 GB", | 20260507 05:26:58.280 | INFO | "malloc" : " 14 GB", | 20260507 05:26:58.280 | INFO | "pinned" : " 29 GB" | 20260507 05:26:58.280 | INFO | } | 20260507 05:26:58.280 | INFO | Perf report for the last 5.14222 seconds: | 20260507 05:26:58.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:26:58.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:58.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:58.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:58.280 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:26:58.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:58.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:58.280 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:26:58.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:58.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:26:58.281 | INFO | Uploaded 36GiB to GPU in 2.01748s (17.8441GiB/s) | 20260507 05:26:58.281 | INFO | Downloaded 34GiB to GPU in 1.94186s (17.5089GiB/s) | 20260507 05:27:03.415 | INFO | Analyzing chunk frames = [7444480,7444992) | 20260507 05:27:03.415 | INFO | MemInfo: { | 20260507 05:27:03.415 | INFO | "gpu" : " 17 GB", | 20260507 05:27:03.415 | INFO | "huge" : "", | 20260507 05:27:03.415 | INFO | "huge_shared" : " 27 GB", | 20260507 05:27:03.415 | INFO | "malloc" : " 14 GB", | 20260507 05:27:03.415 | INFO | "pinned" : " 29 GB" | 20260507 05:27:03.415 | INFO | } | 20260507 05:27:03.415 | INFO | Perf report for the last 5.13481 seconds: | 20260507 05:27:03.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:27:03.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:03.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:03.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:03.415 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 05:27:03.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:03.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:03.415 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:27:03.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:03.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:03.416 | INFO | Uploaded 36GiB to GPU in 2.03852s (17.6599GiB/s) | 20260507 05:27:03.416 | INFO | Downloaded 34GiB to GPU in 1.93737s (17.5496GiB/s) | 20260507 05:27:08.554 | INFO | Analyzing chunk frames = [7444992,7445504) | 20260507 05:27:08.554 | INFO | MemInfo: { | 20260507 05:27:08.554 | INFO | "gpu" : " 17 GB", | 20260507 05:27:08.554 | INFO | "huge" : "", | 20260507 05:27:08.554 | INFO | "huge_shared" : " 27 GB", | 20260507 05:27:08.554 | INFO | "malloc" : " 14 GB", | 20260507 05:27:08.554 | INFO | "pinned" : " 29 GB" | 20260507 05:27:08.554 | INFO | } | 20260507 05:27:08.554 | INFO | Perf report for the last 5.13909 seconds: | 20260507 05:27:08.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:27:08.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:08.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:08.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:08.554 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 05:27:08.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:08.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:08.554 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 05:27:08.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:08.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:08.555 | INFO | Uploaded 36GiB to GPU in 2.03149s (17.721GiB/s) | 20260507 05:27:08.555 | INFO | Downloaded 34GiB to GPU in 1.89615s (17.9311GiB/s) | 20260507 05:27:13.697 | INFO | Analyzing chunk frames = [7445504,7446016) | 20260507 05:27:13.697 | INFO | MemInfo: { | 20260507 05:27:13.697 | INFO | "gpu" : " 17 GB", | 20260507 05:27:13.697 | INFO | "huge" : "", | 20260507 05:27:13.697 | INFO | "huge_shared" : " 27 GB", | 20260507 05:27:13.697 | INFO | "malloc" : " 14 GB", | 20260507 05:27:13.697 | INFO | "pinned" : " 29 GB" | 20260507 05:27:13.697 | INFO | } | 20260507 05:27:13.697 | INFO | Perf report for the last 5.14284 seconds: | 20260507 05:27:13.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:27:13.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:13.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:13.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:13.697 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:27:13.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:13.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:13.697 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:27:13.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:13.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:13.698 | INFO | Uploaded 36GiB to GPU in 1.96598s (18.3115GiB/s) | 20260507 05:27:13.698 | INFO | Downloaded 34GiB to GPU in 1.97533s (17.2123GiB/s) | 20260507 05:27:18.832 | INFO | Analyzing chunk frames = [7446016,7446528) | 20260507 05:27:18.832 | INFO | MemInfo: { | 20260507 05:27:18.832 | INFO | "gpu" : " 17 GB", | 20260507 05:27:18.832 | INFO | "huge" : "", | 20260507 05:27:18.832 | INFO | "huge_shared" : " 27 GB", | 20260507 05:27:18.833 | INFO | "malloc" : " 14 GB", | 20260507 05:27:18.833 | INFO | "pinned" : " 29 GB" | 20260507 05:27:18.833 | INFO | } | 20260507 05:27:18.833 | INFO | Perf report for the last 5.1356 seconds: | 20260507 05:27:18.833 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:27:18.833 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:18.833 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:18.833 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:18.833 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:27:18.833 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:18.833 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:18.833 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:27:18.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:18.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:18.833 | INFO | Uploaded 36GiB to GPU in 1.96296s (18.3396GiB/s) | 20260507 05:27:18.833 | INFO | Downloaded 34GiB to GPU in 1.96229s (17.3267GiB/s) | 20260507 05:27:18.833 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:27:18.833 | INFO | Total mem usage: 18 GiB | 20260507 05:27:18.833 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:27:18.833 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:27:18.833 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:27:18.833 | INFO | High Water Breakdown: | 20260507 05:27:18.833 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:27:18.833 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:27:18.833 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:27:18.834 | INFO | ------------------------------------------ | 20260507 05:27:18.834 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:27:18.834 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:27:18.834 | INFO | Total mem usage: 26.5 GiB | 20260507 05:27:18.834 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:27:18.834 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:27:18.834 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:27:18.834 | INFO | High Water Breakdown: | 20260507 05:27:18.834 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:27:18.834 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:27:18.834 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:27:18.834 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:27:18.834 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:27:18.834 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:27:18.834 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:27:18.834 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:27:18.834 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:27:18.834 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:27:18.834 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:27:18.834 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:27:18.834 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:27:18.834 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:27:18.834 | INFO | ------------------------------------------ | 20260507 05:27:18.834 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:27:18.834 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:27:18.834 | INFO | Total mem usage: 25.6 GiB | 20260507 05:27:18.834 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:27:18.834 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:27:18.834 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:27:18.834 | INFO | High Water Breakdown: | 20260507 05:27:18.834 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:27:18.834 | INFO | ------------------------------------------ | 20260507 05:27:18.834 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:27:18.834 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:27:18.834 | INFO | Total mem usage: 15.7 GiB | 20260507 05:27:18.834 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:27:18.834 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:27:18.834 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:27:18.834 | INFO | High Water Breakdown: | 20260507 05:27:18.834 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:27:18.834 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:27:18.834 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:27:18.834 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:27:18.834 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:27:18.834 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:27:18.834 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:27:18.834 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:27:18.834 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:27:18.834 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:27:18.834 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:27:18.834 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:27:18.834 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:27:18.834 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:27:18.834 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:27:18.834 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:27:18.834 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:27:18.834 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:27:18.834 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:27:18.834 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:27:18.834 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:27:18.834 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:27:18.834 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:27:18.834 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:27:18.834 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:27:18.834 | INFO | ------------------------------------------ | 20260507 05:27:18.834 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:27:18.834 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:27:18.834 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:27:18.834 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:27:18.834 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.44e+01, 1.33e+01 / 1.14e+00 | 20260507 05:27:18.834 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.02e-01, 9.75e-03 / 5.62e-03 | 20260507 05:27:18.834 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.43e+00, 5.12e-01 / 3.07e-01 | 20260507 05:27:18.834 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.46e+00, 1.68e-01 / 1.14e-01 | 20260507 05:27:18.834 | INFO | PerfReport: 16.15%, DME -- 3.97e+01 / 4.42e+01, 3.75e+00 / 1.43e+00 | 20260507 05:27:18.834 | INFO | PerfReport: 57.71%, FrameLabeling -- 1.77e+01 / 1.99e+01, 1.30e+00 / 1.21e+00 | 20260507 05:27:18.834 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.13e+00 / 2.71e+00, 3.87e-01 / 1.75e-01 | 20260507 05:27:18.835 | INFO | PerfReport: 2.99%, Metrics -- 9.19e-01 / 1.25e+00, 2.46e-01 / 1.34e-01 | 20260507 05:27:18.835 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.09e-01 / 3.65e-01, 5.02e-02 / 1.63e-02 | 20260507 05:27:18.835 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:27:18.835 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:27:18.835 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:27:18.835 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 05:27:18.835 | INFO | PerfReport: 21.77%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260507 05:27:18.835 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 05:27:18.835 | INFO | PerfReport: 31.97%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 05:27:18.835 | INFO | PerfReport: 1.48%, LockUpload -- 1.42e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 05:27:18.835 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260507 05:27:18.835 | INFO | PerfReport: 22.09%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 05:27:18.835 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260507 05:27:18.930 | INFO | Starting a round of Baz IO | 20260507 05:27:22.023 | INFO | Populating RTMetrics report | 20260507 05:27:22.024 | INFO | RTMetrics report populated | 20260507 05:27:22.024 | INFO | Opening tmp RTMetrics json file | 20260507 05:27:22.041 | INFO | Serializing RTMetrics to json | 20260507 05:27:22.042 | INFO | Serialization done, swapping tmp file now... | 20260507 05:27:22.043 | INFO | RTMetrics done | 20260507 05:27:22.110 | INFO | Finished a round of Baz IO: Wrote 1.695 GiB in 3.18 seconds | 20260507 05:27:22.110 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.70% / 0.00% / 67.29% | 20260507 05:27:22.110 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.14476% / 0% / 4.37101e-05% | 20260507 05:27:22.110 | INFO | Baz IO overall: | 20260507 05:27:22.110 | INFO | BytesWritten : 3778210471393 | 20260507 05:27:22.110 | INFO | NumEvents : 1812233074055 | 20260507 05:27:23.977 | INFO | Analyzing chunk frames = [7446528,7447040) | 20260507 05:27:23.977 | INFO | MemInfo: { | 20260507 05:27:23.977 | INFO | "gpu" : " 17 GB", | 20260507 05:27:23.977 | INFO | "huge" : "", | 20260507 05:27:23.977 | INFO | "huge_shared" : " 27 GB", | 20260507 05:27:23.977 | INFO | "malloc" : " 14 GB", | 20260507 05:27:23.977 | INFO | "pinned" : " 29 GB" | 20260507 05:27:23.977 | INFO | } | 20260507 05:27:23.977 | INFO | Perf report for the last 5.14466 seconds: | 20260507 05:27:23.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:27:23.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:23.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:23.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:23.977 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 05:27:23.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:23.978 | INFO | RT_METRICS : 41.4%, 1.0, 21.8ms, 0 | 20260507 05:27:23.978 | INFO | PRE_HQ : 58.9%, 1.8, 55.3ms, 0 | 20260507 05:27:23.978 | INFO | BAZWRITER : 61.4%, 3.2, 99.9ms, 0 | 20260507 05:27:23.978 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:27:23.978 | INFO | Uploaded 36GiB to GPU in 2.03813s (17.6632GiB/s) | 20260507 05:27:23.978 | INFO | Downloaded 38GiB to GPU in 2.1881s (17.3667GiB/s) | 20260507 05:27:29.116 | INFO | Analyzing chunk frames = [7447040,7447552) | 20260507 05:27:29.116 | INFO | MemInfo: { | 20260507 05:27:29.116 | INFO | "gpu" : " 17 GB", | 20260507 05:27:29.116 | INFO | "huge" : "", | 20260507 05:27:29.116 | INFO | "huge_shared" : " 27 GB", | 20260507 05:27:29.116 | INFO | "malloc" : " 14 GB", | 20260507 05:27:29.116 | INFO | "pinned" : " 29 GB" | 20260507 05:27:29.116 | INFO | } | 20260507 05:27:29.116 | INFO | Perf report for the last 5.13904 seconds: | 20260507 05:27:29.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:27:29.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:29.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:29.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:29.116 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:27:29.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:29.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:29.117 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:27:29.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:29.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:29.117 | INFO | Uploaded 36GiB to GPU in 1.99824s (18.0159GiB/s) | 20260507 05:27:29.117 | INFO | Downloaded 34GiB to GPU in 1.93194s (17.5989GiB/s) | 20260507 05:27:34.252 | INFO | Analyzing chunk frames = [7447552,7448064) | 20260507 05:27:34.252 | INFO | MemInfo: { | 20260507 05:27:34.252 | INFO | "gpu" : " 17 GB", | 20260507 05:27:34.252 | INFO | "huge" : "", | 20260507 05:27:34.252 | INFO | "huge_shared" : " 27 GB", | 20260507 05:27:34.252 | INFO | "malloc" : " 14 GB", | 20260507 05:27:34.252 | INFO | "pinned" : " 29 GB" | 20260507 05:27:34.252 | INFO | } | 20260507 05:27:34.252 | INFO | Perf report for the last 5.13579 seconds: | 20260507 05:27:34.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:27:34.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:34.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:34.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:34.252 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 05:27:34.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:34.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:34.252 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:27:34.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:34.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:34.253 | INFO | Uploaded 36GiB to GPU in 2.01876s (17.8327GiB/s) | 20260507 05:27:34.253 | INFO | Downloaded 34GiB to GPU in 1.92226s (17.6875GiB/s) | 20260507 05:27:39.385 | INFO | Analyzing chunk frames = [7448064,7448576) | 20260507 05:27:39.385 | INFO | MemInfo: { | 20260507 05:27:39.385 | INFO | "gpu" : " 17 GB", | 20260507 05:27:39.385 | INFO | "huge" : "", | 20260507 05:27:39.385 | INFO | "huge_shared" : " 27 GB", | 20260507 05:27:39.385 | INFO | "malloc" : " 14 GB", | 20260507 05:27:39.385 | INFO | "pinned" : " 29 GB" | 20260507 05:27:39.385 | INFO | } | 20260507 05:27:39.385 | INFO | Perf report for the last 5.13314 seconds: | 20260507 05:27:39.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:27:39.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:39.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:39.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:39.385 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 05:27:39.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:39.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:39.386 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:27:39.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:39.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:39.386 | INFO | Uploaded 36GiB to GPU in 2.00303s (17.9728GiB/s) | 20260507 05:27:39.386 | INFO | Downloaded 34GiB to GPU in 1.9452s (17.4789GiB/s) | 20260507 05:27:44.531 | INFO | Analyzing chunk frames = [7448576,7449088) | 20260507 05:27:44.531 | INFO | MemInfo: { | 20260507 05:27:44.531 | INFO | "gpu" : " 17 GB", | 20260507 05:27:44.531 | INFO | "huge" : "", | 20260507 05:27:44.531 | INFO | "huge_shared" : " 27 GB", | 20260507 05:27:44.531 | INFO | "malloc" : " 14 GB", | 20260507 05:27:44.532 | INFO | "pinned" : " 29 GB" | 20260507 05:27:44.532 | INFO | } | 20260507 05:27:44.532 | INFO | Perf report for the last 5.14617 seconds: | 20260507 05:27:44.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:27:44.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:44.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:44.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:44.532 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 05:27:44.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:44.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:44.532 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 05:27:44.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:44.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:44.532 | INFO | Uploaded 36GiB to GPU in 2.0044s (17.9605GiB/s) | 20260507 05:27:44.532 | INFO | Downloaded 34GiB to GPU in 1.95182s (17.4196GiB/s) | 20260507 05:27:49.677 | INFO | Analyzing chunk frames = [7449088,7449600) | 20260507 05:27:49.677 | INFO | MemInfo: { | 20260507 05:27:49.677 | INFO | "gpu" : " 17 GB", | 20260507 05:27:49.677 | INFO | "huge" : "", | 20260507 05:27:49.677 | INFO | "huge_shared" : " 27 GB", | 20260507 05:27:49.677 | INFO | "malloc" : " 14 GB", | 20260507 05:27:49.677 | INFO | "pinned" : " 29 GB" | 20260507 05:27:49.677 | INFO | } | 20260507 05:27:49.677 | INFO | Perf report for the last 5.14558 seconds: | 20260507 05:27:49.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:27:49.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:49.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:49.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:49.677 | INFO | ANALYSIS : 60.3%, 3.0, 94.3ms, 0 | 20260507 05:27:49.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:49.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:49.677 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 05:27:49.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:49.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:49.677 | INFO | Uploaded 36GiB to GPU in 2.02509s (17.777GiB/s) | 20260507 05:27:49.678 | INFO | Downloaded 34GiB to GPU in 1.92879s (17.6276GiB/s) | 20260507 05:27:54.813 | INFO | Analyzing chunk frames = [7449600,7450112) | 20260507 05:27:54.814 | INFO | MemInfo: { | 20260507 05:27:54.814 | INFO | "gpu" : " 17 GB", | 20260507 05:27:54.814 | INFO | "huge" : "", | 20260507 05:27:54.814 | INFO | "huge_shared" : " 27 GB", | 20260507 05:27:54.814 | INFO | "malloc" : " 14 GB", | 20260507 05:27:54.814 | INFO | "pinned" : " 29 GB" | 20260507 05:27:54.814 | INFO | } | 20260507 05:27:54.814 | INFO | Perf report for the last 5.13659 seconds: | 20260507 05:27:54.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:27:54.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:54.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:54.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:54.814 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 05:27:54.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:54.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:54.814 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:27:54.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:54.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:54.814 | INFO | Uploaded 36GiB to GPU in 2.00694s (17.9378GiB/s) | 20260507 05:27:54.814 | INFO | Downloaded 34GiB to GPU in 1.99141s (17.0734GiB/s) | 20260507 05:27:59.953 | INFO | Analyzing chunk frames = [7450112,7450624) | 20260507 05:27:59.953 | INFO | MemInfo: { | 20260507 05:27:59.953 | INFO | "gpu" : " 17 GB", | 20260507 05:27:59.953 | INFO | "huge" : "", | 20260507 05:27:59.953 | INFO | "huge_shared" : " 27 GB", | 20260507 05:27:59.953 | INFO | "malloc" : " 14 GB", | 20260507 05:27:59.953 | INFO | "pinned" : " 29 GB" | 20260507 05:27:59.953 | INFO | } | 20260507 05:27:59.953 | INFO | Perf report for the last 5.13951 seconds: | 20260507 05:27:59.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:27:59.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:59.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:59.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:59.953 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:27:59.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:59.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:59.953 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:27:59.953 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:59.953 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:27:59.954 | INFO | Uploaded 36GiB to GPU in 1.98062s (18.1761GiB/s) | 20260507 05:27:59.954 | INFO | Downloaded 34GiB to GPU in 1.96499s (17.3029GiB/s) | 20260507 05:28:00.050 | INFO | Starting a round of Baz IO | 20260507 05:28:03.143 | INFO | Populating RTMetrics report | 20260507 05:28:03.143 | INFO | RTMetrics report populated | 20260507 05:28:03.143 | INFO | Opening tmp RTMetrics json file | 20260507 05:28:03.150 | INFO | Serializing RTMetrics to json | 20260507 05:28:03.165 | INFO | Serialization done, swapping tmp file now... | 20260507 05:28:03.165 | INFO | RTMetrics done | 20260507 05:28:03.236 | INFO | Finished a round of Baz IO: Wrote 1.697 GiB in 3.186 seconds | 20260507 05:28:03.237 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.70% / 0.00% / 67.30% | 20260507 05:28:03.237 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.144025% / 0% / 6.75519e-05% | 20260507 05:28:03.237 | INFO | Baz IO overall: | 20260507 05:28:03.237 | INFO | BytesWritten : 3780032505241 | 20260507 05:28:03.237 | INFO | NumEvents : 1813088633142 | 20260507 05:28:05.085 | INFO | Analyzing chunk frames = [7450624,7451136) | 20260507 05:28:05.085 | INFO | MemInfo: { | 20260507 05:28:05.085 | INFO | "gpu" : " 17 GB", | 20260507 05:28:05.085 | INFO | "huge" : "", | 20260507 05:28:05.085 | INFO | "huge_shared" : " 27 GB", | 20260507 05:28:05.085 | INFO | "malloc" : " 14 GB", | 20260507 05:28:05.085 | INFO | "pinned" : " 29 GB" | 20260507 05:28:05.085 | INFO | } | 20260507 05:28:05.085 | INFO | Perf report for the last 5.13172 seconds: | 20260507 05:28:05.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:28:05.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:05.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:05.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:05.085 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 05:28:05.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:05.085 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 05:28:05.085 | INFO | PRE_HQ : 59.3%, 1.8, 55.4ms, 0 | 20260507 05:28:05.085 | INFO | BAZWRITER : 61.7%, 3.3, 103.4ms, 0 | 20260507 05:28:05.085 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:28:05.085 | INFO | Uploaded 36GiB to GPU in 2.02984s (17.7353GiB/s) | 20260507 05:28:05.085 | INFO | Downloaded 38GiB to GPU in 2.17655s (17.4588GiB/s) | 20260507 05:28:10.228 | INFO | Analyzing chunk frames = [7451136,7451648) | 20260507 05:28:10.228 | INFO | MemInfo: { | 20260507 05:28:10.228 | INFO | "gpu" : " 17 GB", | 20260507 05:28:10.228 | INFO | "huge" : "", | 20260507 05:28:10.228 | INFO | "huge_shared" : " 27 GB", | 20260507 05:28:10.228 | INFO | "malloc" : " 14 GB", | 20260507 05:28:10.228 | INFO | "pinned" : " 29 GB" | 20260507 05:28:10.228 | INFO | } | 20260507 05:28:10.228 | INFO | Perf report for the last 5.14294 seconds: | 20260507 05:28:10.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:28:10.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:10.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:10.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:10.228 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 05:28:10.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:10.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:10.228 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:28:10.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:10.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:10.228 | INFO | Uploaded 36GiB to GPU in 2.03604s (17.6814GiB/s) | 20260507 05:28:10.228 | INFO | Downloaded 34GiB to GPU in 1.94269s (17.5015GiB/s) | 20260507 05:28:15.367 | INFO | Analyzing chunk frames = [7451648,7452160) | 20260507 05:28:15.367 | INFO | MemInfo: { | 20260507 05:28:15.367 | INFO | "gpu" : " 17 GB", | 20260507 05:28:15.367 | INFO | "huge" : "", | 20260507 05:28:15.367 | INFO | "huge_shared" : " 27 GB", | 20260507 05:28:15.367 | INFO | "malloc" : " 14 GB", | 20260507 05:28:15.367 | INFO | "pinned" : " 29 GB" | 20260507 05:28:15.367 | INFO | } | 20260507 05:28:15.367 | INFO | Perf report for the last 5.13902 seconds: | 20260507 05:28:15.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:28:15.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:15.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:15.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:15.367 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 05:28:15.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:15.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:15.367 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:28:15.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:15.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:15.367 | INFO | Uploaded 36GiB to GPU in 1.98784s (18.1101GiB/s) | 20260507 05:28:15.367 | INFO | Downloaded 34GiB to GPU in 1.94486s (17.4819GiB/s) | 20260507 05:28:20.506 | INFO | Analyzing chunk frames = [7452160,7452672) | 20260507 05:28:20.506 | INFO | MemInfo: { | 20260507 05:28:20.506 | INFO | "gpu" : " 17 GB", | 20260507 05:28:20.506 | INFO | "huge" : "", | 20260507 05:28:20.506 | INFO | "huge_shared" : " 27 GB", | 20260507 05:28:20.506 | INFO | "malloc" : " 14 GB", | 20260507 05:28:20.506 | INFO | "pinned" : " 29 GB" | 20260507 05:28:20.506 | INFO | } | 20260507 05:28:20.506 | INFO | Perf report for the last 5.13912 seconds: | 20260507 05:28:20.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:28:20.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:20.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:20.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:20.506 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 05:28:20.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:20.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:20.506 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:28:20.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:20.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:20.506 | INFO | Uploaded 36GiB to GPU in 2.00288s (17.9741GiB/s) | 20260507 05:28:20.506 | INFO | Downloaded 34GiB to GPU in 1.93793s (17.5445GiB/s) | 20260507 05:28:25.642 | INFO | Analyzing chunk frames = [7452672,7453184) | 20260507 05:28:25.642 | INFO | MemInfo: { | 20260507 05:28:25.642 | INFO | "gpu" : " 17 GB", | 20260507 05:28:25.642 | INFO | "huge" : "", | 20260507 05:28:25.642 | INFO | "huge_shared" : " 27 GB", | 20260507 05:28:25.642 | INFO | "malloc" : " 14 GB", | 20260507 05:28:25.642 | INFO | "pinned" : " 29 GB" | 20260507 05:28:25.642 | INFO | } | 20260507 05:28:25.643 | INFO | Perf report for the last 5.13638 seconds: | 20260507 05:28:25.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:28:25.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:25.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:25.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:25.643 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 05:28:25.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:25.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:25.643 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:28:25.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:25.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:25.643 | INFO | Uploaded 36GiB to GPU in 2.04501s (17.6038GiB/s) | 20260507 05:28:25.643 | INFO | Downloaded 34GiB to GPU in 1.94387s (17.4908GiB/s) | 20260507 05:28:30.787 | INFO | Analyzing chunk frames = [7453184,7453696) | 20260507 05:28:30.788 | INFO | MemInfo: { | 20260507 05:28:30.788 | INFO | "gpu" : " 17 GB", | 20260507 05:28:30.788 | INFO | "huge" : "", | 20260507 05:28:30.788 | INFO | "huge_shared" : " 27 GB", | 20260507 05:28:30.788 | INFO | "malloc" : " 14 GB", | 20260507 05:28:30.788 | INFO | "pinned" : " 29 GB" | 20260507 05:28:30.788 | INFO | } | 20260507 05:28:30.788 | INFO | Perf report for the last 5.14519 seconds: | 20260507 05:28:30.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:28:30.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:30.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:30.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:30.788 | INFO | ANALYSIS : 60.3%, 3.0, 94.3ms, 0 | 20260507 05:28:30.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:30.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:30.788 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 05:28:30.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:30.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:30.788 | INFO | Uploaded 36GiB to GPU in 2.028s (17.7515GiB/s) | 20260507 05:28:30.788 | INFO | Downloaded 34GiB to GPU in 1.92345s (17.6766GiB/s) | 20260507 05:28:35.928 | INFO | Analyzing chunk frames = [7453696,7454208) | 20260507 05:28:35.928 | INFO | MemInfo: { | 20260507 05:28:35.928 | INFO | "gpu" : " 17 GB", | 20260507 05:28:35.928 | INFO | "huge" : "", | 20260507 05:28:35.928 | INFO | "huge_shared" : " 27 GB", | 20260507 05:28:35.928 | INFO | "malloc" : " 14 GB", | 20260507 05:28:35.928 | INFO | "pinned" : " 29 GB" | 20260507 05:28:35.928 | INFO | } | 20260507 05:28:35.928 | INFO | Perf report for the last 5.14032 seconds: | 20260507 05:28:35.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:28:35.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:35.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:35.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:35.928 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 05:28:35.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:35.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:35.928 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 05:28:35.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:35.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:35.928 | INFO | Uploaded 36GiB to GPU in 1.9729s (18.2473GiB/s) | 20260507 05:28:35.929 | INFO | Downloaded 34GiB to GPU in 1.97834s (17.1861GiB/s) | 20260507 05:28:41.063 | INFO | Analyzing chunk frames = [7454208,7454720) | 20260507 05:28:41.063 | INFO | MemInfo: { | 20260507 05:28:41.063 | INFO | "gpu" : " 17 GB", | 20260507 05:28:41.063 | INFO | "huge" : "", | 20260507 05:28:41.063 | INFO | "huge_shared" : " 27 GB", | 20260507 05:28:41.063 | INFO | "malloc" : " 14 GB", | 20260507 05:28:41.063 | INFO | "pinned" : " 29 GB" | 20260507 05:28:41.063 | INFO | } | 20260507 05:28:41.063 | INFO | Perf report for the last 5.13511 seconds: | 20260507 05:28:41.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:28:41.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:41.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:41.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:41.063 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 05:28:41.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:41.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:41.063 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 05:28:41.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:41.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:41.064 | INFO | Uploaded 36GiB to GPU in 1.98961s (18.094GiB/s) | 20260507 05:28:41.064 | INFO | Downloaded 34GiB to GPU in 1.97791s (17.1899GiB/s) | 20260507 05:28:41.064 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:28:41.064 | INFO | Total mem usage: 18 GiB | 20260507 05:28:41.064 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:28:41.064 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:28:41.064 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:28:41.064 | INFO | High Water Breakdown: | 20260507 05:28:41.064 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:28:41.064 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:28:41.064 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:28:41.064 | INFO | ------------------------------------------ | 20260507 05:28:41.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:28:41.064 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:28:41.064 | INFO | Total mem usage: 26.5 GiB | 20260507 05:28:41.064 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:28:41.064 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:28:41.064 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:28:41.064 | INFO | High Water Breakdown: | 20260507 05:28:41.064 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:28:41.064 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:28:41.064 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:28:41.064 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:28:41.064 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:28:41.064 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:28:41.064 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:28:41.064 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:28:41.064 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:28:41.064 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:28:41.064 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:28:41.064 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:28:41.064 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:28:41.064 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:28:41.064 | INFO | ------------------------------------------ | 20260507 05:28:41.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:28:41.064 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:28:41.064 | INFO | Total mem usage: 25.6 GiB | 20260507 05:28:41.064 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:28:41.064 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:28:41.064 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:28:41.064 | INFO | High Water Breakdown: | 20260507 05:28:41.064 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:28:41.064 | INFO | ------------------------------------------ | 20260507 05:28:41.064 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:28:41.064 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:28:41.064 | INFO | Total mem usage: 15.7 GiB | 20260507 05:28:41.064 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:28:41.065 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:28:41.065 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:28:41.065 | INFO | High Water Breakdown: | 20260507 05:28:41.065 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:28:41.065 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:28:41.065 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:28:41.065 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:28:41.065 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:28:41.065 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:28:41.065 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:28:41.065 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:28:41.065 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:28:41.065 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:28:41.065 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:28:41.065 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:28:41.065 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:28:41.065 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:28:41.065 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:28:41.065 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:28:41.065 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:28:41.065 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:28:41.065 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:28:41.065 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:28:41.065 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:28:41.065 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:28:41.065 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:28:41.065 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:28:41.065 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:28:41.065 | INFO | ------------------------------------------ | 20260507 05:28:41.065 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:28:41.065 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:28:41.065 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:28:41.065 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:28:41.065 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.35e+01 / 1.66e+00 | 20260507 05:28:41.065 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.02e-01, 1.01e-02 / 8.51e-03 | 20260507 05:28:41.065 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.36e+00, 5.15e-01 / 2.89e-01 | 20260507 05:28:41.065 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.46e+00, 1.68e-01 / 1.15e-01 | 20260507 05:28:41.065 | INFO | PerfReport: 16.16%, DME -- 3.97e+01 / 4.46e+01, 3.87e+00 / 1.80e+00 | 20260507 05:28:41.065 | INFO | PerfReport: 57.74%, FrameLabeling -- 1.77e+01 / 2.00e+01, 1.34e+00 / 1.35e+00 | 20260507 05:28:41.065 | INFO | PerfReport: 6.91%, PulseAccumulating -- 2.12e+00 / 2.69e+00, 3.79e-01 / 1.96e-01 | 20260507 05:28:41.065 | INFO | PerfReport: 2.98%, Metrics -- 9.16e-01 / 1.23e+00, 2.44e-01 / 1.25e-01 | 20260507 05:28:41.065 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.09e-01 / 3.59e-01, 5.10e-02 / 1.12e-02 | 20260507 05:28:41.065 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:28:41.065 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:28:41.065 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:28:41.065 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 05:28:41.065 | INFO | PerfReport: 21.78%, Upload -- 2.09e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260507 05:28:41.065 | INFO | PerfReport: 21.43%, Download -- 2.06e+01 / 0.00e+00, 7.44e+00 / 0.00e+00 | 20260507 05:28:41.065 | INFO | PerfReport: 31.97%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 05:28:41.065 | INFO | PerfReport: 1.37%, LockUpload -- 1.32e+00 / 0.00e+00, 4.67e+00 / 0.00e+00 | 20260507 05:28:41.065 | INFO | PerfReport: 1.38%, LockDownload -- 1.32e+00 / 0.00e+00, 3.22e+00 / 0.00e+00 | 20260507 05:28:41.065 | INFO | PerfReport: 22.05%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 05:28:41.065 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260507 05:28:41.160 | INFO | Starting a round of Baz IO | 20260507 05:28:44.257 | INFO | Populating RTMetrics report | 20260507 05:28:44.257 | INFO | RTMetrics report populated | 20260507 05:28:44.257 | INFO | Opening tmp RTMetrics json file | 20260507 05:28:44.272 | INFO | Serializing RTMetrics to json | 20260507 05:28:44.273 | INFO | Serialization done, swapping tmp file now... | 20260507 05:28:44.274 | INFO | RTMetrics done | 20260507 05:28:44.344 | INFO | Finished a round of Baz IO: Wrote 1.697 GiB in 3.184 seconds | 20260507 05:28:44.344 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.70% / 0.00% / 67.30% | 20260507 05:28:44.344 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.144621% / 3.97364e-06% / 5.96046e-05% | 20260507 05:28:44.344 | INFO | Baz IO overall: | 20260507 05:28:44.344 | INFO | BytesWritten : 3781854771574 | 20260507 05:28:44.344 | INFO | NumEvents : 1813944556818 | 20260507 05:28:46.202 | INFO | Analyzing chunk frames = [7454720,7455232) | 20260507 05:28:46.202 | INFO | MemInfo: { | 20260507 05:28:46.202 | INFO | "gpu" : " 17 GB", | 20260507 05:28:46.202 | INFO | "huge" : "", | 20260507 05:28:46.202 | INFO | "huge_shared" : " 27 GB", | 20260507 05:28:46.202 | INFO | "malloc" : " 14 GB", | 20260507 05:28:46.202 | INFO | "pinned" : " 29 GB" | 20260507 05:28:46.202 | INFO | } | 20260507 05:28:46.202 | INFO | Perf report for the last 5.13884 seconds: | 20260507 05:28:46.202 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:28:46.202 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:46.202 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:46.202 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:46.202 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:28:46.202 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:46.202 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 05:28:46.202 | INFO | PRE_HQ : 59.2%, 1.8, 54.9ms, 0 | 20260507 05:28:46.202 | INFO | BAZWRITER : 61.4%, 3.2, 100.5ms, 0 | 20260507 05:28:46.202 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 05:28:46.202 | INFO | Uploaded 36GiB to GPU in 2.05862s (17.4875GiB/s) | 20260507 05:28:46.202 | INFO | Downloaded 38GiB to GPU in 2.18794s (17.368GiB/s) | 20260507 05:28:51.343 | INFO | Analyzing chunk frames = [7455232,7455744) | 20260507 05:28:51.343 | INFO | MemInfo: { | 20260507 05:28:51.343 | INFO | "gpu" : " 17 GB", | 20260507 05:28:51.343 | INFO | "huge" : "", | 20260507 05:28:51.343 | INFO | "huge_shared" : " 27 GB", | 20260507 05:28:51.343 | INFO | "malloc" : " 14 GB", | 20260507 05:28:51.344 | INFO | "pinned" : " 29 GB" | 20260507 05:28:51.344 | INFO | } | 20260507 05:28:51.344 | INFO | Perf report for the last 5.14166 seconds: | 20260507 05:28:51.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:28:51.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:51.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:51.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:51.344 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 05:28:51.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:51.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:51.344 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:28:51.344 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:51.344 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:51.344 | INFO | Uploaded 36GiB to GPU in 2.04226s (17.6276GiB/s) | 20260507 05:28:51.344 | INFO | Downloaded 34GiB to GPU in 1.94068s (17.5196GiB/s) | 20260507 05:28:56.489 | INFO | Analyzing chunk frames = [7455744,7456256) | 20260507 05:28:56.489 | INFO | MemInfo: { | 20260507 05:28:56.489 | INFO | "gpu" : " 17 GB", | 20260507 05:28:56.489 | INFO | "huge" : "", | 20260507 05:28:56.489 | INFO | "huge_shared" : " 27 GB", | 20260507 05:28:56.489 | INFO | "malloc" : " 14 GB", | 20260507 05:28:56.489 | INFO | "pinned" : " 29 GB" | 20260507 05:28:56.489 | INFO | } | 20260507 05:28:56.489 | INFO | Perf report for the last 5.14547 seconds: | 20260507 05:28:56.489 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:28:56.489 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:56.489 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:56.489 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:56.489 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 05:28:56.489 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:56.489 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:56.489 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:28:56.489 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:56.489 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:28:56.489 | INFO | Uploaded 36GiB to GPU in 1.98798s (18.1088GiB/s) | 20260507 05:28:56.490 | INFO | Downloaded 34GiB to GPU in 1.94513s (17.4795GiB/s) | 20260507 05:29:01.623 | INFO | Analyzing chunk frames = [7456256,7456768) | 20260507 05:29:01.623 | INFO | MemInfo: { | 20260507 05:29:01.623 | INFO | "gpu" : " 17 GB", | 20260507 05:29:01.623 | INFO | "huge" : "", | 20260507 05:29:01.623 | INFO | "huge_shared" : " 27 GB", | 20260507 05:29:01.623 | INFO | "malloc" : " 14 GB", | 20260507 05:29:01.623 | INFO | "pinned" : " 29 GB" | 20260507 05:29:01.623 | INFO | } | 20260507 05:29:01.623 | INFO | Perf report for the last 5.13415 seconds: | 20260507 05:29:01.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:29:01.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:01.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:01.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:01.623 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 05:29:01.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:01.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:01.623 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:29:01.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:01.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:01.624 | INFO | Uploaded 36GiB to GPU in 2.00854s (17.9235GiB/s) | 20260507 05:29:01.624 | INFO | Downloaded 34GiB to GPU in 1.94586s (17.473GiB/s) | 20260507 05:29:06.762 | INFO | Analyzing chunk frames = [7456768,7457280) | 20260507 05:29:06.762 | INFO | MemInfo: { | 20260507 05:29:06.762 | INFO | "gpu" : " 17 GB", | 20260507 05:29:06.762 | INFO | "huge" : "", | 20260507 05:29:06.762 | INFO | "huge_shared" : " 27 GB", | 20260507 05:29:06.762 | INFO | "malloc" : " 14 GB", | 20260507 05:29:06.762 | INFO | "pinned" : " 29 GB" | 20260507 05:29:06.762 | INFO | } | 20260507 05:29:06.763 | INFO | Perf report for the last 5.13914 seconds: | 20260507 05:29:06.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:29:06.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:06.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:06.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:06.763 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 05:29:06.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:06.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:06.763 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:29:06.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:06.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:06.763 | INFO | Uploaded 36GiB to GPU in 2.01805s (17.839GiB/s) | 20260507 05:29:06.763 | INFO | Downloaded 34GiB to GPU in 1.91717s (17.7345GiB/s) | 20260507 05:29:11.895 | INFO | Analyzing chunk frames = [7457280,7457792) | 20260507 05:29:11.895 | INFO | MemInfo: { | 20260507 05:29:11.895 | INFO | "gpu" : " 17 GB", | 20260507 05:29:11.895 | INFO | "huge" : "", | 20260507 05:29:11.895 | INFO | "huge_shared" : " 27 GB", | 20260507 05:29:11.895 | INFO | "malloc" : " 14 GB", | 20260507 05:29:11.895 | INFO | "pinned" : " 29 GB" | 20260507 05:29:11.895 | INFO | } | 20260507 05:29:11.895 | INFO | Perf report for the last 5.13238 seconds: | 20260507 05:29:11.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:29:11.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:11.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:11.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:11.895 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 05:29:11.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:11.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:11.895 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:29:11.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:11.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:11.895 | INFO | Uploaded 36GiB to GPU in 2.03902s (17.6555GiB/s) | 20260507 05:29:11.895 | INFO | Downloaded 34GiB to GPU in 1.92175s (17.6922GiB/s) | 20260507 05:29:17.040 | INFO | Analyzing chunk frames = [7457792,7458304) | 20260507 05:29:17.040 | INFO | MemInfo: { | 20260507 05:29:17.040 | INFO | "gpu" : " 17 GB", | 20260507 05:29:17.040 | INFO | "huge" : "", | 20260507 05:29:17.040 | INFO | "huge_shared" : " 27 GB", | 20260507 05:29:17.040 | INFO | "malloc" : " 14 GB", | 20260507 05:29:17.040 | INFO | "pinned" : " 29 GB" | 20260507 05:29:17.040 | INFO | } | 20260507 05:29:17.040 | INFO | Perf report for the last 5.14524 seconds: | 20260507 05:29:17.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:29:17.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:17.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:17.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:17.040 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 05:29:17.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:17.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:17.040 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 05:29:17.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:17.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:17.041 | INFO | Uploaded 36GiB to GPU in 1.96713s (18.3007GiB/s) | 20260507 05:29:17.041 | INFO | Downloaded 34GiB to GPU in 1.97231s (17.2386GiB/s) | 20260507 05:29:22.182 | INFO | Analyzing chunk frames = [7458304,7458816) | 20260507 05:29:22.182 | INFO | MemInfo: { | 20260507 05:29:22.182 | INFO | "gpu" : " 17 GB", | 20260507 05:29:22.182 | INFO | "huge" : "", | 20260507 05:29:22.182 | INFO | "huge_shared" : " 27 GB", | 20260507 05:29:22.183 | INFO | "malloc" : " 14 GB", | 20260507 05:29:22.183 | INFO | "pinned" : " 29 GB" | 20260507 05:29:22.183 | INFO | } | 20260507 05:29:22.183 | INFO | Perf report for the last 5.14241 seconds: | 20260507 05:29:22.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:29:22.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:22.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:22.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:22.183 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 05:29:22.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:22.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:22.183 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:29:22.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:22.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:22.183 | INFO | Uploaded 36GiB to GPU in 1.96106s (18.3574GiB/s) | 20260507 05:29:22.183 | INFO | Downloaded 34GiB to GPU in 1.95967s (17.3499GiB/s) | 20260507 05:29:22.280 | INFO | Starting a round of Baz IO | 20260507 05:29:25.374 | INFO | Populating RTMetrics report | 20260507 05:29:25.374 | INFO | RTMetrics report populated | 20260507 05:29:25.374 | INFO | Opening tmp RTMetrics json file | 20260507 05:29:25.379 | INFO | Serializing RTMetrics to json | 20260507 05:29:25.391 | INFO | Serialization done, swapping tmp file now... | 20260507 05:29:25.392 | INFO | RTMetrics done | 20260507 05:29:25.463 | INFO | Finished a round of Baz IO: Wrote 1.696 GiB in 3.183 seconds | 20260507 05:29:25.463 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.70% / 0.00% / 67.30% | 20260507 05:29:25.463 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.143921% / 0% / 5.5631e-05% | 20260507 05:29:25.463 | INFO | Baz IO overall: | 20260507 05:29:25.463 | INFO | BytesWritten : 3783675311090 | 20260507 05:29:25.463 | INFO | NumEvents : 1814798925776 | 20260507 05:29:27.327 | INFO | Analyzing chunk frames = [7458816,7459328) | 20260507 05:29:27.327 | INFO | MemInfo: { | 20260507 05:29:27.327 | INFO | "gpu" : " 17 GB", | 20260507 05:29:27.327 | INFO | "huge" : "", | 20260507 05:29:27.327 | INFO | "huge_shared" : " 27 GB", | 20260507 05:29:27.327 | INFO | "malloc" : " 14 GB", | 20260507 05:29:27.327 | INFO | "pinned" : " 29 GB" | 20260507 05:29:27.327 | INFO | } | 20260507 05:29:27.327 | INFO | Perf report for the last 5.14413 seconds: | 20260507 05:29:27.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:29:27.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:27.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:27.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:27.327 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 05:29:27.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:27.327 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 05:29:27.327 | INFO | PRE_HQ : 58.8%, 1.8, 54.8ms, 0 | 20260507 05:29:27.327 | INFO | BAZWRITER : 61.3%, 3.2, 100.9ms, 0 | 20260507 05:29:27.327 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:29:27.327 | INFO | Uploaded 36GiB to GPU in 2.03812s (17.6633GiB/s) | 20260507 05:29:27.327 | INFO | Downloaded 38GiB to GPU in 2.17752s (17.4511GiB/s) | 20260507 05:29:32.462 | INFO | Analyzing chunk frames = [7459328,7459840) | 20260507 05:29:32.462 | INFO | MemInfo: { | 20260507 05:29:32.462 | INFO | "gpu" : " 17 GB", | 20260507 05:29:32.462 | INFO | "huge" : "", | 20260507 05:29:32.462 | INFO | "huge_shared" : " 27 GB", | 20260507 05:29:32.462 | INFO | "malloc" : " 14 GB", | 20260507 05:29:32.462 | INFO | "pinned" : " 29 GB" | 20260507 05:29:32.462 | INFO | } | 20260507 05:29:32.462 | INFO | Perf report for the last 5.13529 seconds: | 20260507 05:29:32.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:29:32.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:32.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:32.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:32.462 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:29:32.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:32.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:32.462 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:29:32.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:32.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:32.462 | INFO | Uploaded 36GiB to GPU in 1.99063s (18.0847GiB/s) | 20260507 05:29:32.462 | INFO | Downloaded 34GiB to GPU in 1.93145s (17.6033GiB/s) | 20260507 05:29:37.595 | INFO | Analyzing chunk frames = [7459840,7460352) | 20260507 05:29:37.595 | INFO | MemInfo: { | 20260507 05:29:37.595 | INFO | "gpu" : " 17 GB", | 20260507 05:29:37.595 | INFO | "huge" : "", | 20260507 05:29:37.595 | INFO | "huge_shared" : " 27 GB", | 20260507 05:29:37.596 | INFO | "malloc" : " 14 GB", | 20260507 05:29:37.596 | INFO | "pinned" : " 29 GB" | 20260507 05:29:37.596 | INFO | } | 20260507 05:29:37.596 | INFO | Perf report for the last 5.13352 seconds: | 20260507 05:29:37.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:29:37.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:37.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:37.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:37.596 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 05:29:37.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:37.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:37.596 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:29:37.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:37.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:37.596 | INFO | Uploaded 36GiB to GPU in 2.04465s (17.6069GiB/s) | 20260507 05:29:37.596 | INFO | Downloaded 34GiB to GPU in 1.95316s (17.4077GiB/s) | 20260507 05:29:42.740 | INFO | Analyzing chunk frames = [7460352,7460864) | 20260507 05:29:42.740 | INFO | MemInfo: { | 20260507 05:29:42.740 | INFO | "gpu" : " 17 GB", | 20260507 05:29:42.740 | INFO | "huge" : "", | 20260507 05:29:42.740 | INFO | "huge_shared" : " 27 GB", | 20260507 05:29:42.740 | INFO | "malloc" : " 14 GB", | 20260507 05:29:42.740 | INFO | "pinned" : " 29 GB" | 20260507 05:29:42.740 | INFO | } | 20260507 05:29:42.740 | INFO | Perf report for the last 5.14426 seconds: | 20260507 05:29:42.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:29:42.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:42.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:42.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:42.740 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 05:29:42.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:42.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:42.740 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:29:42.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:42.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:42.740 | INFO | Uploaded 36GiB to GPU in 1.98243s (18.1595GiB/s) | 20260507 05:29:42.740 | INFO | Downloaded 34GiB to GPU in 1.91963s (17.7117GiB/s) | 20260507 05:29:47.877 | INFO | Analyzing chunk frames = [7460864,7461376) | 20260507 05:29:47.877 | INFO | MemInfo: { | 20260507 05:29:47.877 | INFO | "gpu" : " 17 GB", | 20260507 05:29:47.877 | INFO | "huge" : "", | 20260507 05:29:47.877 | INFO | "huge_shared" : " 27 GB", | 20260507 05:29:47.877 | INFO | "malloc" : " 14 GB", | 20260507 05:29:47.877 | INFO | "pinned" : " 29 GB" | 20260507 05:29:47.877 | INFO | } | 20260507 05:29:47.877 | INFO | Perf report for the last 5.13748 seconds: | 20260507 05:29:47.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:29:47.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:47.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:47.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:47.877 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 05:29:47.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:47.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:47.877 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:29:47.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:47.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:47.878 | INFO | Uploaded 36GiB to GPU in 2.05069s (17.5551GiB/s) | 20260507 05:29:47.878 | INFO | Downloaded 34GiB to GPU in 1.95065s (17.4301GiB/s) | 20260507 05:29:53.017 | INFO | Analyzing chunk frames = [7461376,7461888) | 20260507 05:29:53.017 | INFO | MemInfo: { | 20260507 05:29:53.017 | INFO | "gpu" : " 17 GB", | 20260507 05:29:53.017 | INFO | "huge" : "", | 20260507 05:29:53.017 | INFO | "huge_shared" : " 27 GB", | 20260507 05:29:53.017 | INFO | "malloc" : " 14 GB", | 20260507 05:29:53.017 | INFO | "pinned" : " 29 GB" | 20260507 05:29:53.017 | INFO | } | 20260507 05:29:53.017 | INFO | Perf report for the last 5.13955 seconds: | 20260507 05:29:53.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:29:53.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:53.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:53.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:53.017 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 05:29:53.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:53.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:53.017 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 05:29:53.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:53.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:53.017 | INFO | Uploaded 36GiB to GPU in 2.05961s (17.479GiB/s) | 20260507 05:29:53.017 | INFO | Downloaded 34GiB to GPU in 1.92893s (17.6263GiB/s) | 20260507 05:29:58.151 | INFO | Analyzing chunk frames = [7461888,7462400) | 20260507 05:29:58.151 | INFO | MemInfo: { | 20260507 05:29:58.151 | INFO | "gpu" : " 17 GB", | 20260507 05:29:58.151 | INFO | "huge" : "", | 20260507 05:29:58.151 | INFO | "huge_shared" : " 27 GB", | 20260507 05:29:58.151 | INFO | "malloc" : " 14 GB", | 20260507 05:29:58.151 | INFO | "pinned" : " 29 GB" | 20260507 05:29:58.151 | INFO | } | 20260507 05:29:58.151 | INFO | Perf report for the last 5.13428 seconds: | 20260507 05:29:58.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:29:58.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:58.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:58.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:58.151 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 05:29:58.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:58.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:58.151 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:29:58.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:58.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:29:58.151 | INFO | Uploaded 36GiB to GPU in 2.02795s (17.7519GiB/s) | 20260507 05:29:58.152 | INFO | Downloaded 34GiB to GPU in 1.97888s (17.1815GiB/s) | 20260507 05:30:03.297 | INFO | Analyzing chunk frames = [7462400,7462912) | 20260507 05:30:03.297 | INFO | MemInfo: { | 20260507 05:30:03.297 | INFO | "gpu" : " 17 GB", | 20260507 05:30:03.297 | INFO | "huge" : "", | 20260507 05:30:03.297 | INFO | "huge_shared" : " 27 GB", | 20260507 05:30:03.297 | INFO | "malloc" : " 14 GB", | 20260507 05:30:03.297 | INFO | "pinned" : " 29 GB" | 20260507 05:30:03.298 | INFO | } | 20260507 05:30:03.298 | INFO | Perf report for the last 5.14633 seconds: | 20260507 05:30:03.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:30:03.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:03.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:03.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:03.298 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 05:30:03.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:03.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:03.298 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:30:03.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:03.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:03.298 | INFO | Uploaded 36GiB to GPU in 1.96667s (18.305GiB/s) | 20260507 05:30:03.298 | INFO | Downloaded 34GiB to GPU in 1.94533s (17.4777GiB/s) | 20260507 05:30:03.298 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:30:03.298 | INFO | Total mem usage: 18 GiB | 20260507 05:30:03.298 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:30:03.298 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 05:30:03.298 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:30:03.298 | INFO | High Water Breakdown: | 20260507 05:30:03.298 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:30:03.298 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:30:03.298 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:30:03.298 | INFO | ------------------------------------------ | 20260507 05:30:03.298 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:30:03.298 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:30:03.298 | INFO | Total mem usage: 26.5 GiB | 20260507 05:30:03.298 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:30:03.298 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:30:03.298 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:30:03.298 | INFO | High Water Breakdown: | 20260507 05:30:03.298 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:30:03.299 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:30:03.299 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:30:03.299 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:30:03.299 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:30:03.299 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:30:03.299 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:30:03.299 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:30:03.299 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:30:03.299 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:30:03.299 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:30:03.299 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:30:03.299 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:30:03.299 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:30:03.299 | INFO | ------------------------------------------ | 20260507 05:30:03.299 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:30:03.299 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:30:03.299 | INFO | Total mem usage: 25.6 GiB | 20260507 05:30:03.299 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:30:03.299 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:30:03.299 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:30:03.299 | INFO | High Water Breakdown: | 20260507 05:30:03.299 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:30:03.299 | INFO | ------------------------------------------ | 20260507 05:30:03.299 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:30:03.299 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:30:03.299 | INFO | Total mem usage: 15.7 GiB | 20260507 05:30:03.299 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:30:03.299 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:30:03.299 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:30:03.299 | INFO | High Water Breakdown: | 20260507 05:30:03.299 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:30:03.299 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:30:03.299 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:30:03.299 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:30:03.299 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:30:03.299 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:30:03.299 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:30:03.299 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:30:03.299 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:30:03.299 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:30:03.299 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:30:03.299 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:30:03.299 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:30:03.299 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:30:03.299 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:30:03.299 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:30:03.299 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:30:03.299 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:30:03.299 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:30:03.299 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:30:03.299 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:30:03.299 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:30:03.299 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:30:03.299 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:30:03.299 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:30:03.299 | INFO | ------------------------------------------ | 20260507 05:30:03.299 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:30:03.299 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:30:03.299 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:30:03.299 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:30:03.299 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.44e+01, 1.35e+01 / 9.48e-01 | 20260507 05:30:03.299 | INFO | PerfReport: 0.29%, AggregateStats -- 9.01e-02 / 1.01e-01, 9.27e-03 / 5.23e-03 | 20260507 05:30:03.299 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.49e+00, 5.05e-01 / 2.40e-01 | 20260507 05:30:03.299 | INFO | PerfReport: 3.22%, Histogram -- 9.87e-01 / 1.48e+00, 1.63e-01 / 1.03e-01 | 20260507 05:30:03.299 | INFO | PerfReport: 16.14%, DME -- 3.96e+01 / 4.40e+01, 3.57e+00 / 1.04e+00 | 20260507 05:30:03.299 | INFO | PerfReport: 57.73%, FrameLabeling -- 1.77e+01 / 2.00e+01, 1.28e+00 / 1.48e+00 | 20260507 05:30:03.300 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.13e+00 / 2.71e+00, 3.84e-01 / 1.88e-01 | 20260507 05:30:03.300 | INFO | PerfReport: 2.99%, Metrics -- 9.19e-01 / 1.24e+00, 2.44e-01 / 1.22e-01 | 20260507 05:30:03.300 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.09e-01 / 3.66e-01, 5.07e-02 / 1.90e-02 | 20260507 05:30:03.300 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:30:03.300 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:30:03.300 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:30:03.300 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 05:30:03.300 | INFO | PerfReport: 21.84%, Upload -- 2.10e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260507 05:30:03.300 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 05:30:03.300 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 05:30:03.300 | INFO | PerfReport: 1.44%, LockUpload -- 1.38e+00 / 0.00e+00, 4.78e+00 / 0.00e+00 | 20260507 05:30:03.300 | INFO | PerfReport: 1.34%, LockDownload -- 1.29e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260507 05:30:03.300 | INFO | PerfReport: 22.04%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 05:30:03.300 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260507 05:30:03.394 | INFO | Starting a round of Baz IO | 20260507 05:30:06.490 | INFO | Populating RTMetrics report | 20260507 05:30:06.490 | INFO | RTMetrics report populated | 20260507 05:30:06.490 | INFO | Opening tmp RTMetrics json file | 20260507 05:30:06.508 | INFO | Serializing RTMetrics to json | 20260507 05:30:06.509 | INFO | Serialization done, swapping tmp file now... | 20260507 05:30:06.510 | INFO | RTMetrics done | 20260507 05:30:06.574 | INFO | Finished a round of Baz IO: Wrote 1.695 GiB in 3.18 seconds | 20260507 05:30:06.574 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.70% / 0.00% / 67.30% | 20260507 05:30:06.574 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.143242% / 0% / 5.5631e-05% | 20260507 05:30:06.574 | INFO | Baz IO overall: | 20260507 05:30:06.575 | INFO | BytesWritten : 3785495462867 | 20260507 05:30:06.575 | INFO | NumEvents : 1815653252203 | 20260507 05:30:08.438 | INFO | Analyzing chunk frames = [7462912,7463424) | 20260507 05:30:08.438 | INFO | MemInfo: { | 20260507 05:30:08.438 | INFO | "gpu" : " 17 GB", | 20260507 05:30:08.438 | INFO | "huge" : "", | 20260507 05:30:08.438 | INFO | "huge_shared" : " 27 GB", | 20260507 05:30:08.438 | INFO | "malloc" : " 14 GB", | 20260507 05:30:08.438 | INFO | "pinned" : " 29 GB" | 20260507 05:30:08.438 | INFO | } | 20260507 05:30:08.438 | INFO | Perf report for the last 5.14046 seconds: | 20260507 05:30:08.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:30:08.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:08.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:08.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:08.438 | INFO | ANALYSIS : 61.6%, 3.0, 95.3ms, 0 | 20260507 05:30:08.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:08.438 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 05:30:08.438 | INFO | PRE_HQ : 58.9%, 1.8, 54.8ms, 0 | 20260507 05:30:08.438 | INFO | BAZWRITER : 61.5%, 3.6, 114.7ms, 0 | 20260507 05:30:08.438 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:30:08.438 | INFO | Uploaded 36GiB to GPU in 2.05318s (17.5338GiB/s) | 20260507 05:30:08.438 | INFO | Downloaded 38GiB to GPU in 2.1869s (17.3762GiB/s) | 20260507 05:30:13.581 | INFO | Analyzing chunk frames = [7463424,7463936) | 20260507 05:30:13.581 | INFO | MemInfo: { | 20260507 05:30:13.581 | INFO | "gpu" : " 17 GB", | 20260507 05:30:13.581 | INFO | "huge" : "", | 20260507 05:30:13.581 | INFO | "huge_shared" : " 27 GB", | 20260507 05:30:13.581 | INFO | "malloc" : " 14 GB", | 20260507 05:30:13.581 | INFO | "pinned" : " 29 GB" | 20260507 05:30:13.581 | INFO | } | 20260507 05:30:13.581 | INFO | Perf report for the last 5.14309 seconds: | 20260507 05:30:13.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:30:13.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:13.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:13.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:13.581 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 05:30:13.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:13.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:13.581 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260507 05:30:13.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:13.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:13.581 | INFO | Uploaded 36GiB to GPU in 1.97027s (18.2716GiB/s) | 20260507 05:30:13.581 | INFO | Downloaded 34GiB to GPU in 1.92841s (17.6311GiB/s) | 20260507 05:30:18.707 | INFO | Analyzing chunk frames = [7463936,7464448) | 20260507 05:30:18.707 | INFO | MemInfo: { | 20260507 05:30:18.707 | INFO | "gpu" : " 17 GB", | 20260507 05:30:18.707 | INFO | "huge" : "", | 20260507 05:30:18.707 | INFO | "huge_shared" : " 27 GB", | 20260507 05:30:18.707 | INFO | "malloc" : " 14 GB", | 20260507 05:30:18.707 | INFO | "pinned" : " 29 GB" | 20260507 05:30:18.707 | INFO | } | 20260507 05:30:18.707 | INFO | Perf report for the last 5.12609 seconds: | 20260507 05:30:18.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:30:18.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:18.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:18.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:18.707 | INFO | ANALYSIS : 60.7%, 3.0, 93.7ms, 0 | 20260507 05:30:18.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:18.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:18.707 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 05:30:18.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:18.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:18.708 | INFO | Uploaded 36GiB to GPU in 2.04461s (17.6072GiB/s) | 20260507 05:30:18.708 | INFO | Downloaded 34GiB to GPU in 1.95428s (17.3978GiB/s) | 20260507 05:30:23.850 | INFO | Analyzing chunk frames = [7464448,7464960) | 20260507 05:30:23.850 | INFO | MemInfo: { | 20260507 05:30:23.850 | INFO | "gpu" : " 17 GB", | 20260507 05:30:23.850 | INFO | "huge" : "", | 20260507 05:30:23.850 | INFO | "huge_shared" : " 27 GB", | 20260507 05:30:23.850 | INFO | "malloc" : " 14 GB", | 20260507 05:30:23.850 | INFO | "pinned" : " 29 GB" | 20260507 05:30:23.850 | INFO | } | 20260507 05:30:23.850 | INFO | Perf report for the last 5.1425 seconds: | 20260507 05:30:23.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:30:23.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:23.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:23.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:23.850 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 05:30:23.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:23.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:23.850 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:30:23.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:23.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:23.850 | INFO | Uploaded 36GiB to GPU in 1.95747s (18.3911GiB/s) | 20260507 05:30:23.850 | INFO | Downloaded 34GiB to GPU in 1.92432s (17.6685GiB/s) | 20260507 05:30:28.995 | INFO | Analyzing chunk frames = [7464960,7465472) | 20260507 05:30:28.995 | INFO | MemInfo: { | 20260507 05:30:28.995 | INFO | "gpu" : " 17 GB", | 20260507 05:30:28.995 | INFO | "huge" : "", | 20260507 05:30:28.995 | INFO | "huge_shared" : " 27 GB", | 20260507 05:30:28.995 | INFO | "malloc" : " 14 GB", | 20260507 05:30:28.995 | INFO | "pinned" : " 29 GB" | 20260507 05:30:28.995 | INFO | } | 20260507 05:30:28.995 | INFO | Perf report for the last 5.14528 seconds: | 20260507 05:30:28.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:30:28.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:28.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:28.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:28.995 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 05:30:28.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:28.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:28.995 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:30:28.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:28.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:28.995 | INFO | Uploaded 36GiB to GPU in 2.02208s (17.8035GiB/s) | 20260507 05:30:28.995 | INFO | Downloaded 34GiB to GPU in 1.94387s (17.4909GiB/s) | 20260507 05:30:34.135 | INFO | Analyzing chunk frames = [7465472,7465984) | 20260507 05:30:34.135 | INFO | MemInfo: { | 20260507 05:30:34.135 | INFO | "gpu" : " 17 GB", | 20260507 05:30:34.135 | INFO | "huge" : "", | 20260507 05:30:34.135 | INFO | "huge_shared" : " 27 GB", | 20260507 05:30:34.135 | INFO | "malloc" : " 14 GB", | 20260507 05:30:34.135 | INFO | "pinned" : " 29 GB" | 20260507 05:30:34.135 | INFO | } | 20260507 05:30:34.135 | INFO | Perf report for the last 5.14034 seconds: | 20260507 05:30:34.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:30:34.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:34.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:34.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:34.135 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 05:30:34.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:34.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:34.136 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 05:30:34.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:34.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:34.136 | INFO | Uploaded 36GiB to GPU in 2.05798s (17.4929GiB/s) | 20260507 05:30:34.136 | INFO | Downloaded 34GiB to GPU in 1.92046s (17.7041GiB/s) | 20260507 05:30:39.275 | INFO | Analyzing chunk frames = [7465984,7466496) | 20260507 05:30:39.275 | INFO | MemInfo: { | 20260507 05:30:39.275 | INFO | "gpu" : " 17 GB", | 20260507 05:30:39.275 | INFO | "huge" : "", | 20260507 05:30:39.275 | INFO | "huge_shared" : " 27 GB", | 20260507 05:30:39.275 | INFO | "malloc" : " 14 GB", | 20260507 05:30:39.275 | INFO | "pinned" : " 29 GB" | 20260507 05:30:39.275 | INFO | } | 20260507 05:30:39.275 | INFO | Perf report for the last 5.13945 seconds: | 20260507 05:30:39.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:30:39.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:39.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:39.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:39.275 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 05:30:39.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:39.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:39.275 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 05:30:39.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:39.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:39.275 | INFO | Uploaded 36GiB to GPU in 1.98105s (18.1722GiB/s) | 20260507 05:30:39.275 | INFO | Downloaded 34GiB to GPU in 1.99207s (17.0677GiB/s) | 20260507 05:30:44.404 | INFO | Analyzing chunk frames = [7466496,7467008) | 20260507 05:30:44.404 | INFO | MemInfo: { | 20260507 05:30:44.404 | INFO | "gpu" : " 17 GB", | 20260507 05:30:44.404 | INFO | "huge" : "", | 20260507 05:30:44.404 | INFO | "huge_shared" : " 27 GB", | 20260507 05:30:44.404 | INFO | "malloc" : " 14 GB", | 20260507 05:30:44.404 | INFO | "pinned" : " 29 GB" | 20260507 05:30:44.404 | INFO | } | 20260507 05:30:44.404 | INFO | Perf report for the last 5.12955 seconds: | 20260507 05:30:44.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:30:44.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:44.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:44.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:44.404 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 05:30:44.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:44.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:44.405 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:30:44.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:44.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:44.405 | INFO | Uploaded 36GiB to GPU in 1.96274s (18.3417GiB/s) | 20260507 05:30:44.405 | INFO | Downloaded 34GiB to GPU in 1.95772s (17.3671GiB/s) | 20260507 05:30:44.501 | INFO | Starting a round of Baz IO | 20260507 05:30:47.606 | INFO | Populating RTMetrics report | 20260507 05:30:47.607 | INFO | RTMetrics report populated | 20260507 05:30:47.607 | INFO | Opening tmp RTMetrics json file | 20260507 05:30:47.614 | INFO | Serializing RTMetrics to json | 20260507 05:30:47.624 | INFO | Serialization done, swapping tmp file now... | 20260507 05:30:47.624 | INFO | RTMetrics done | 20260507 05:30:47.695 | INFO | Finished a round of Baz IO: Wrote 1.696 GiB in 3.193 seconds | 20260507 05:30:47.695 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.70% / 0.00% / 67.30% | 20260507 05:30:47.695 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.143019% / 0% / 6.35783e-05% | 20260507 05:30:47.695 | INFO | Baz IO overall: | 20260507 05:30:47.695 | INFO | BytesWritten : 3787316981937 | 20260507 05:30:47.695 | INFO | NumEvents : 1816508628701 | 20260507 05:30:49.551 | INFO | Analyzing chunk frames = [7467008,7467520) | 20260507 05:30:49.551 | INFO | MemInfo: { | 20260507 05:30:49.551 | INFO | "gpu" : " 17 GB", | 20260507 05:30:49.551 | INFO | "huge" : "", | 20260507 05:30:49.551 | INFO | "huge_shared" : " 27 GB", | 20260507 05:30:49.551 | INFO | "malloc" : " 14 GB", | 20260507 05:30:49.551 | INFO | "pinned" : " 29 GB" | 20260507 05:30:49.551 | INFO | } | 20260507 05:30:49.551 | INFO | Perf report for the last 5.14642 seconds: | 20260507 05:30:49.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:30:49.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:49.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:49.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:49.551 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 05:30:49.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:49.551 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 05:30:49.551 | INFO | PRE_HQ : 59.0%, 1.8, 54.9ms, 0 | 20260507 05:30:49.551 | INFO | BAZWRITER : 61.6%, 3.2, 100.5ms, 0 | 20260507 05:30:49.551 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 05:30:49.551 | INFO | Uploaded 36GiB to GPU in 2.04299s (17.6212GiB/s) | 20260507 05:30:49.551 | INFO | Downloaded 38GiB to GPU in 2.18645s (17.3798GiB/s) | 20260507 05:30:54.689 | INFO | Analyzing chunk frames = [7467520,7468032) | 20260507 05:30:54.689 | INFO | MemInfo: { | 20260507 05:30:54.689 | INFO | "gpu" : " 17 GB", | 20260507 05:30:54.689 | INFO | "huge" : "", | 20260507 05:30:54.689 | INFO | "huge_shared" : " 27 GB", | 20260507 05:30:54.689 | INFO | "malloc" : " 14 GB", | 20260507 05:30:54.689 | INFO | "pinned" : " 29 GB" | 20260507 05:30:54.689 | INFO | } | 20260507 05:30:54.689 | INFO | Perf report for the last 5.13815 seconds: | 20260507 05:30:54.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:30:54.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:54.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:54.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:54.689 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:30:54.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:54.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:54.689 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:30:54.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:54.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:54.689 | INFO | Uploaded 36GiB to GPU in 1.99454s (18.0493GiB/s) | 20260507 05:30:54.689 | INFO | Downloaded 34GiB to GPU in 1.93273s (17.5917GiB/s) | 20260507 05:30:59.825 | INFO | Analyzing chunk frames = [7468032,7468544) | 20260507 05:30:59.825 | INFO | MemInfo: { | 20260507 05:30:59.825 | INFO | "gpu" : " 17 GB", | 20260507 05:30:59.825 | INFO | "huge" : "", | 20260507 05:30:59.825 | INFO | "huge_shared" : " 27 GB", | 20260507 05:30:59.825 | INFO | "malloc" : " 14 GB", | 20260507 05:30:59.825 | INFO | "pinned" : " 29 GB" | 20260507 05:30:59.825 | INFO | } | 20260507 05:30:59.825 | INFO | Perf report for the last 5.13639 seconds: | 20260507 05:30:59.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:30:59.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:59.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:59.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:59.825 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 05:30:59.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:59.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:59.826 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:30:59.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:59.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:30:59.826 | INFO | Uploaded 36GiB to GPU in 2.04604s (17.595GiB/s) | 20260507 05:30:59.826 | INFO | Downloaded 34GiB to GPU in 1.95258s (17.4128GiB/s) | 20260507 05:31:04.963 | INFO | Analyzing chunk frames = [7468544,7469056) | 20260507 05:31:04.963 | INFO | MemInfo: { | 20260507 05:31:04.963 | INFO | "gpu" : " 17 GB", | 20260507 05:31:04.963 | INFO | "huge" : "", | 20260507 05:31:04.963 | INFO | "huge_shared" : " 27 GB", | 20260507 05:31:04.963 | INFO | "malloc" : " 14 GB", | 20260507 05:31:04.964 | INFO | "pinned" : " 29 GB" | 20260507 05:31:04.964 | INFO | } | 20260507 05:31:04.964 | INFO | Perf report for the last 5.13829 seconds: | 20260507 05:31:04.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:31:04.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:04.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:04.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:04.964 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 05:31:04.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:04.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:04.964 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 05:31:04.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:04.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:04.964 | INFO | Uploaded 36GiB to GPU in 2.00027s (17.9976GiB/s) | 20260507 05:31:04.964 | INFO | Downloaded 34GiB to GPU in 1.94478s (17.4827GiB/s) | 20260507 05:31:10.107 | INFO | Analyzing chunk frames = [7469056,7469568) | 20260507 05:31:10.107 | INFO | MemInfo: { | 20260507 05:31:10.107 | INFO | "gpu" : " 17 GB", | 20260507 05:31:10.107 | INFO | "huge" : "", | 20260507 05:31:10.107 | INFO | "huge_shared" : " 27 GB", | 20260507 05:31:10.107 | INFO | "malloc" : " 14 GB", | 20260507 05:31:10.107 | INFO | "pinned" : " 29 GB" | 20260507 05:31:10.107 | INFO | } | 20260507 05:31:10.107 | INFO | Perf report for the last 5.1433 seconds: | 20260507 05:31:10.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:31:10.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:10.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:10.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:10.107 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 05:31:10.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:10.107 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:10.107 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:31:10.107 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:10.107 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:10.107 | INFO | Uploaded 36GiB to GPU in 2.0334s (17.7044GiB/s) | 20260507 05:31:10.107 | INFO | Downloaded 34GiB to GPU in 1.94479s (17.4826GiB/s) | 20260507 05:31:15.249 | INFO | Analyzing chunk frames = [7469568,7470080) | 20260507 05:31:15.249 | INFO | MemInfo: { | 20260507 05:31:15.249 | INFO | "gpu" : " 17 GB", | 20260507 05:31:15.249 | INFO | "huge" : "", | 20260507 05:31:15.249 | INFO | "huge_shared" : " 27 GB", | 20260507 05:31:15.249 | INFO | "malloc" : " 14 GB", | 20260507 05:31:15.249 | INFO | "pinned" : " 29 GB" | 20260507 05:31:15.249 | INFO | } | 20260507 05:31:15.249 | INFO | Perf report for the last 5.14242 seconds: | 20260507 05:31:15.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:31:15.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:15.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:15.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:15.250 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 05:31:15.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:15.250 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:15.250 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:31:15.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:15.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:15.250 | INFO | Uploaded 36GiB to GPU in 2.02546s (17.7738GiB/s) | 20260507 05:31:15.250 | INFO | Downloaded 34GiB to GPU in 1.92241s (17.6861GiB/s) | 20260507 05:31:20.388 | INFO | Analyzing chunk frames = [7470080,7470592) | 20260507 05:31:20.388 | INFO | MemInfo: { | 20260507 05:31:20.388 | INFO | "gpu" : " 17 GB", | 20260507 05:31:20.388 | INFO | "huge" : "", | 20260507 05:31:20.388 | INFO | "huge_shared" : " 27 GB", | 20260507 05:31:20.388 | INFO | "malloc" : " 14 GB", | 20260507 05:31:20.388 | INFO | "pinned" : " 29 GB" | 20260507 05:31:20.388 | INFO | } | 20260507 05:31:20.388 | INFO | Perf report for the last 5.13892 seconds: | 20260507 05:31:20.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:31:20.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:20.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:20.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:20.388 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 05:31:20.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:20.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:20.389 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:31:20.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:20.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:20.389 | INFO | Uploaded 36GiB to GPU in 2.03061s (17.7287GiB/s) | 20260507 05:31:20.389 | INFO | Downloaded 34GiB to GPU in 1.98984s (17.0868GiB/s) | 20260507 05:31:25.526 | INFO | Analyzing chunk frames = [7470592,7471104) | 20260507 05:31:25.526 | INFO | MemInfo: { | 20260507 05:31:25.526 | INFO | "gpu" : " 17 GB", | 20260507 05:31:25.526 | INFO | "huge" : "", | 20260507 05:31:25.526 | INFO | "huge_shared" : " 27 GB", | 20260507 05:31:25.526 | INFO | "malloc" : " 14 GB", | 20260507 05:31:25.526 | INFO | "pinned" : " 29 GB" | 20260507 05:31:25.526 | INFO | } | 20260507 05:31:25.526 | INFO | Perf report for the last 5.13744 seconds: | 20260507 05:31:25.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:31:25.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:25.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:25.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:25.526 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:31:25.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:25.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:25.526 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:31:25.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:25.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:25.526 | INFO | Uploaded 36GiB to GPU in 1.99314s (18.062GiB/s) | 20260507 05:31:25.526 | INFO | Downloaded 34GiB to GPU in 1.96111s (17.3371GiB/s) | 20260507 05:31:25.526 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:31:25.526 | INFO | Total mem usage: 18 GiB | 20260507 05:31:25.526 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:31:25.526 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -21%) | 20260507 05:31:25.526 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:31:25.526 | INFO | High Water Breakdown: | 20260507 05:31:25.526 | INFO | 60.9% : 9.06 GiB ( 79.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:31:25.526 | INFO | 39% : 5.81 GiB ( 86.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:31:25.526 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:31:25.526 | INFO | ------------------------------------------ | 20260507 05:31:25.526 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:31:25.526 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:31:25.526 | INFO | Total mem usage: 26.5 GiB | 20260507 05:31:25.526 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:31:25.526 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:31:25.526 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:31:25.527 | INFO | High Water Breakdown: | 20260507 05:31:25.527 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:31:25.527 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:31:25.527 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:31:25.527 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:31:25.527 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:31:25.527 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:31:25.527 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:31:25.527 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:31:25.527 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:31:25.527 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:31:25.527 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:31:25.527 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:31:25.527 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:31:25.527 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:31:25.527 | INFO | ------------------------------------------ | 20260507 05:31:25.527 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:31:25.527 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:31:25.527 | INFO | Total mem usage: 25.6 GiB | 20260507 05:31:25.527 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:31:25.527 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:31:25.527 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:31:25.527 | INFO | High Water Breakdown: | 20260507 05:31:25.527 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:31:25.527 | INFO | ------------------------------------------ | 20260507 05:31:25.527 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:31:25.527 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:31:25.527 | INFO | Total mem usage: 15.7 GiB | 20260507 05:31:25.527 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:31:25.527 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:31:25.527 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:31:25.527 | INFO | High Water Breakdown: | 20260507 05:31:25.527 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:31:25.527 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:31:25.527 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:31:25.527 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:31:25.527 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:31:25.527 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:31:25.527 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:31:25.527 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:31:25.527 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:31:25.527 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:31:25.527 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:31:25.527 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:31:25.527 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:31:25.527 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:31:25.527 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:31:25.527 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:31:25.527 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:31:25.527 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:31:25.527 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:31:25.527 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:31:25.527 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:31:25.527 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:31:25.527 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:31:25.527 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:31:25.527 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:31:25.527 | INFO | ------------------------------------------ | 20260507 05:31:25.527 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:31:25.527 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:31:25.527 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:31:25.527 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:31:25.527 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.33e+01 / 1.17e+00 | 20260507 05:31:25.527 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.00e-01, 9.31e-03 / 4.58e-03 | 20260507 05:31:25.527 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.41e+00, 5.10e-01 / 2.54e-01 | 20260507 05:31:25.527 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.44e+00, 1.67e-01 / 1.03e-01 | 20260507 05:31:25.527 | INFO | PerfReport: 16.20%, DME -- 3.98e+01 / 4.46e+01, 3.95e+00 / 1.42e+00 | 20260507 05:31:25.527 | INFO | PerfReport: 57.67%, FrameLabeling -- 1.77e+01 / 2.00e+01, 1.30e+00 / 1.14e+00 | 20260507 05:31:25.527 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.68e+00, 3.83e-01 / 1.43e-01 | 20260507 05:31:25.527 | INFO | PerfReport: 2.99%, Metrics -- 9.18e-01 / 1.24e+00, 2.46e-01 / 1.24e-01 | 20260507 05:31:25.527 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.09e-01 / 3.62e-01, 5.16e-02 / 1.94e-02 | 20260507 05:31:25.527 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:31:25.527 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:31:25.527 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:31:25.527 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 05:31:25.527 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 05:31:25.527 | INFO | PerfReport: 21.44%, Download -- 2.06e+01 / 0.00e+00, 7.41e+00 / 0.00e+00 | 20260507 05:31:25.527 | INFO | PerfReport: 31.97%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 05:31:25.527 | INFO | PerfReport: 1.37%, LockUpload -- 1.32e+00 / 0.00e+00, 4.72e+00 / 0.00e+00 | 20260507 05:31:25.527 | INFO | PerfReport: 1.36%, LockDownload -- 1.31e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260507 05:31:25.527 | INFO | PerfReport: 22.03%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 05:31:25.527 | INFO | PerfReport: 0.01%, Misc -- 1.26e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260507 05:31:25.623 | INFO | Starting a round of Baz IO | 20260507 05:31:28.720 | INFO | Populating RTMetrics report | 20260507 05:31:28.721 | INFO | RTMetrics report populated | 20260507 05:31:28.721 | INFO | Opening tmp RTMetrics json file | 20260507 05:31:28.737 | INFO | Serializing RTMetrics to json | 20260507 05:31:28.739 | INFO | Serialization done, swapping tmp file now... | 20260507 05:31:28.739 | INFO | RTMetrics done | 20260507 05:31:28.810 | INFO | Finished a round of Baz IO: Wrote 1.696 GiB in 3.187 seconds | 20260507 05:31:28.810 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.70% / 0.00% / 67.30% | 20260507 05:31:28.810 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.14269% / 3.97364e-06% / 5.5631e-05% | 20260507 05:31:28.810 | INFO | Baz IO overall: | 20260507 05:31:28.810 | INFO | BytesWritten : 3789137897283 | 20260507 05:31:28.810 | INFO | NumEvents : 1817363779373 | 20260507 05:31:30.656 | INFO | Analyzing chunk frames = [7471104,7471616) | 20260507 05:31:30.656 | INFO | MemInfo: { | 20260507 05:31:30.656 | INFO | "gpu" : " 17 GB", | 20260507 05:31:30.656 | INFO | "huge" : "", | 20260507 05:31:30.656 | INFO | "huge_shared" : " 27 GB", | 20260507 05:31:30.656 | INFO | "malloc" : " 14 GB", | 20260507 05:31:30.656 | INFO | "pinned" : " 29 GB" | 20260507 05:31:30.656 | INFO | } | 20260507 05:31:30.656 | INFO | Perf report for the last 5.12987 seconds: | 20260507 05:31:30.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:31:30.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:30.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:30.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:30.656 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 05:31:30.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:30.656 | INFO | RT_METRICS : 41.5%, 1.0, 21.7ms, 0 | 20260507 05:31:30.656 | INFO | PRE_HQ : 59.1%, 1.8, 54.9ms, 0 | 20260507 05:31:30.656 | INFO | BAZWRITER : 61.7%, 3.2, 100.8ms, 0 | 20260507 05:31:30.656 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:31:30.656 | INFO | Uploaded 36GiB to GPU in 2.05732s (17.4985GiB/s) | 20260507 05:31:30.656 | INFO | Downloaded 38GiB to GPU in 2.17126s (17.5013GiB/s) | 20260507 05:31:35.803 | INFO | Analyzing chunk frames = [7471616,7472128) | 20260507 05:31:35.803 | INFO | MemInfo: { | 20260507 05:31:35.803 | INFO | "gpu" : " 17 GB", | 20260507 05:31:35.803 | INFO | "huge" : "", | 20260507 05:31:35.803 | INFO | "huge_shared" : " 27 GB", | 20260507 05:31:35.803 | INFO | "malloc" : " 14 GB", | 20260507 05:31:35.803 | INFO | "pinned" : " 29 GB" | 20260507 05:31:35.803 | INFO | } | 20260507 05:31:35.803 | INFO | Perf report for the last 5.14731 seconds: | 20260507 05:31:35.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:31:35.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:35.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:35.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:35.803 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260507 05:31:35.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:35.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:35.803 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:31:35.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:35.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:35.803 | INFO | Uploaded 36GiB to GPU in 2.04326s (17.6189GiB/s) | 20260507 05:31:35.804 | INFO | Downloaded 34GiB to GPU in 1.93917s (17.5333GiB/s) | 20260507 05:31:40.945 | INFO | Analyzing chunk frames = [7472128,7472640) | 20260507 05:31:40.945 | INFO | MemInfo: { | 20260507 05:31:40.945 | INFO | "gpu" : " 17 GB", | 20260507 05:31:40.945 | INFO | "huge" : "", | 20260507 05:31:40.945 | INFO | "huge_shared" : " 27 GB", | 20260507 05:31:40.945 | INFO | "malloc" : " 14 GB", | 20260507 05:31:40.945 | INFO | "pinned" : " 29 GB" | 20260507 05:31:40.945 | INFO | } | 20260507 05:31:40.945 | INFO | Perf report for the last 5.14185 seconds: | 20260507 05:31:40.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:31:40.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:40.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:40.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:40.945 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 05:31:40.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:40.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:40.945 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 05:31:40.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:40.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:40.945 | INFO | Uploaded 36GiB to GPU in 2.01975s (17.8239GiB/s) | 20260507 05:31:40.945 | INFO | Downloaded 34GiB to GPU in 1.94049s (17.5214GiB/s) | 20260507 05:31:46.085 | INFO | Analyzing chunk frames = [7472640,7473152) | 20260507 05:31:46.085 | INFO | MemInfo: { | 20260507 05:31:46.085 | INFO | "gpu" : " 17 GB", | 20260507 05:31:46.085 | INFO | "huge" : "", | 20260507 05:31:46.085 | INFO | "huge_shared" : " 27 GB", | 20260507 05:31:46.085 | INFO | "malloc" : " 14 GB", | 20260507 05:31:46.085 | INFO | "pinned" : " 29 GB" | 20260507 05:31:46.085 | INFO | } | 20260507 05:31:46.085 | INFO | Perf report for the last 5.14031 seconds: | 20260507 05:31:46.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:31:46.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:46.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:46.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:46.085 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 05:31:46.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:46.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:46.085 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:31:46.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:46.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:46.085 | INFO | Uploaded 36GiB to GPU in 2.04291s (17.6219GiB/s) | 20260507 05:31:46.086 | INFO | Downloaded 34GiB to GPU in 1.94973s (17.4383GiB/s) | 20260507 05:31:51.224 | INFO | Analyzing chunk frames = [7473152,7473664) | 20260507 05:31:51.224 | INFO | MemInfo: { | 20260507 05:31:51.224 | INFO | "gpu" : " 17 GB", | 20260507 05:31:51.224 | INFO | "huge" : "", | 20260507 05:31:51.224 | INFO | "huge_shared" : " 27 GB", | 20260507 05:31:51.224 | INFO | "malloc" : " 14 GB", | 20260507 05:31:51.224 | INFO | "pinned" : " 29 GB" | 20260507 05:31:51.224 | INFO | } | 20260507 05:31:51.224 | INFO | Perf report for the last 5.13883 seconds: | 20260507 05:31:51.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:31:51.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:51.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:51.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:51.224 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 05:31:51.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:51.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:51.224 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:31:51.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:51.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:51.224 | INFO | Uploaded 36GiB to GPU in 2.03879s (17.6576GiB/s) | 20260507 05:31:51.224 | INFO | Downloaded 34GiB to GPU in 1.9165s (17.7407GiB/s) | 20260507 05:31:56.370 | INFO | Analyzing chunk frames = [7473664,7474176) | 20260507 05:31:56.370 | INFO | MemInfo: { | 20260507 05:31:56.370 | INFO | "gpu" : " 17 GB", | 20260507 05:31:56.370 | INFO | "huge" : "", | 20260507 05:31:56.370 | INFO | "huge_shared" : " 27 GB", | 20260507 05:31:56.370 | INFO | "malloc" : " 14 GB", | 20260507 05:31:56.370 | INFO | "pinned" : " 29 GB" | 20260507 05:31:56.370 | INFO | } | 20260507 05:31:56.370 | INFO | Perf report for the last 5.14613 seconds: | 20260507 05:31:56.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:31:56.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:56.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:56.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:56.370 | INFO | ANALYSIS : 60.3%, 3.0, 94.3ms, 0 | 20260507 05:31:56.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:56.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:56.370 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:31:56.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:56.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:31:56.370 | INFO | Uploaded 36GiB to GPU in 2.05481s (17.5199GiB/s) | 20260507 05:31:56.371 | INFO | Downloaded 34GiB to GPU in 1.90851s (17.815GiB/s) | 20260507 05:32:01.503 | INFO | Analyzing chunk frames = [7474176,7474688) | 20260507 05:32:01.503 | INFO | MemInfo: { | 20260507 05:32:01.503 | INFO | "gpu" : " 17 GB", | 20260507 05:32:01.503 | INFO | "huge" : "", | 20260507 05:32:01.503 | INFO | "huge_shared" : " 27 GB", | 20260507 05:32:01.503 | INFO | "malloc" : " 14 GB", | 20260507 05:32:01.503 | INFO | "pinned" : " 29 GB" | 20260507 05:32:01.503 | INFO | } | 20260507 05:32:01.503 | INFO | Perf report for the last 5.13298 seconds: | 20260507 05:32:01.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:32:01.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:01.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:01.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:01.503 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 05:32:01.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:01.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:01.503 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 05:32:01.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:01.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:01.503 | INFO | Uploaded 36GiB to GPU in 1.93555s (18.5993GiB/s) | 20260507 05:32:01.504 | INFO | Downloaded 34GiB to GPU in 1.96616s (17.2926GiB/s) | 20260507 05:32:06.639 | INFO | Analyzing chunk frames = [7474688,7475200) | 20260507 05:32:06.639 | INFO | MemInfo: { | 20260507 05:32:06.639 | INFO | "gpu" : " 17 GB", | 20260507 05:32:06.639 | INFO | "huge" : "", | 20260507 05:32:06.639 | INFO | "huge_shared" : " 27 GB", | 20260507 05:32:06.639 | INFO | "malloc" : " 14 GB", | 20260507 05:32:06.639 | INFO | "pinned" : " 29 GB" | 20260507 05:32:06.639 | INFO | } | 20260507 05:32:06.639 | INFO | Perf report for the last 5.13621 seconds: | 20260507 05:32:06.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:32:06.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:06.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:06.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:06.640 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 05:32:06.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:06.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:06.640 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:32:06.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:06.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:06.640 | INFO | Uploaded 36GiB to GPU in 1.97274s (18.2487GiB/s) | 20260507 05:32:06.640 | INFO | Downloaded 34GiB to GPU in 1.96734s (17.2822GiB/s) | 20260507 05:32:06.737 | INFO | Starting a round of Baz IO | 20260507 05:32:09.830 | INFO | Populating RTMetrics report | 20260507 05:32:09.830 | INFO | RTMetrics report populated | 20260507 05:32:09.830 | INFO | Opening tmp RTMetrics json file | 20260507 05:32:09.835 | INFO | Serializing RTMetrics to json | 20260507 05:32:09.848 | INFO | Serialization done, swapping tmp file now... | 20260507 05:32:09.848 | INFO | RTMetrics done | 20260507 05:32:09.913 | INFO | Finished a round of Baz IO: Wrote 1.693 GiB in 3.176 seconds | 20260507 05:32:09.913 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.70% / 0.00% / 67.30% | 20260507 05:32:09.913 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.14205% / 0% / 6.75519e-05% | 20260507 05:32:09.913 | INFO | Baz IO overall: | 20260507 05:32:09.913 | INFO | BytesWritten : 3790955882727 | 20260507 05:32:09.913 | INFO | NumEvents : 1818216950673 | 20260507 05:32:11.775 | INFO | Analyzing chunk frames = [7475200,7475712) | 20260507 05:32:11.775 | INFO | MemInfo: { | 20260507 05:32:11.775 | INFO | "gpu" : " 17 GB", | 20260507 05:32:11.775 | INFO | "huge" : "", | 20260507 05:32:11.775 | INFO | "huge_shared" : " 27 GB", | 20260507 05:32:11.775 | INFO | "malloc" : " 14 GB", | 20260507 05:32:11.775 | INFO | "pinned" : " 29 GB" | 20260507 05:32:11.775 | INFO | } | 20260507 05:32:11.775 | INFO | Perf report for the last 5.13565 seconds: | 20260507 05:32:11.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:32:11.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:11.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:11.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:11.775 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 05:32:11.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:11.775 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 05:32:11.775 | INFO | PRE_HQ : 59.0%, 1.8, 54.8ms, 0 | 20260507 05:32:11.775 | INFO | BAZWRITER : 61.4%, 3.2, 101.4ms, 0 | 20260507 05:32:11.775 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:32:11.775 | INFO | Uploaded 36GiB to GPU in 2.05119s (17.5508GiB/s) | 20260507 05:32:11.775 | INFO | Downloaded 38GiB to GPU in 2.17988s (17.4322GiB/s) | 20260507 05:32:16.912 | INFO | Analyzing chunk frames = [7475712,7476224) | 20260507 05:32:16.912 | INFO | MemInfo: { | 20260507 05:32:16.912 | INFO | "gpu" : " 17 GB", | 20260507 05:32:16.912 | INFO | "huge" : "", | 20260507 05:32:16.912 | INFO | "huge_shared" : " 27 GB", | 20260507 05:32:16.912 | INFO | "malloc" : " 14 GB", | 20260507 05:32:16.912 | INFO | "pinned" : " 29 GB" | 20260507 05:32:16.912 | INFO | } | 20260507 05:32:16.912 | INFO | Perf report for the last 5.13732 seconds: | 20260507 05:32:16.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:32:16.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:16.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:16.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:16.913 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 05:32:16.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:16.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:16.913 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:32:16.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:16.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:16.913 | INFO | Uploaded 36GiB to GPU in 2.03959s (17.6506GiB/s) | 20260507 05:32:16.913 | INFO | Downloaded 34GiB to GPU in 1.94213s (17.5065GiB/s) | 20260507 05:32:22.054 | INFO | Analyzing chunk frames = [7476224,7476736) | 20260507 05:32:22.054 | INFO | MemInfo: { | 20260507 05:32:22.054 | INFO | "gpu" : " 17 GB", | 20260507 05:32:22.054 | INFO | "huge" : "", | 20260507 05:32:22.054 | INFO | "huge_shared" : " 27 GB", | 20260507 05:32:22.054 | INFO | "malloc" : " 14 GB", | 20260507 05:32:22.054 | INFO | "pinned" : " 29 GB" | 20260507 05:32:22.054 | INFO | } | 20260507 05:32:22.054 | INFO | Perf report for the last 5.14199 seconds: | 20260507 05:32:22.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:32:22.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:22.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:22.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:22.055 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 05:32:22.055 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:22.055 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:22.055 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:32:22.055 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:22.055 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:22.055 | INFO | Uploaded 36GiB to GPU in 2.00707s (17.9366GiB/s) | 20260507 05:32:22.055 | INFO | Downloaded 34GiB to GPU in 1.94366s (17.4928GiB/s) | 20260507 05:32:27.202 | INFO | Analyzing chunk frames = [7476736,7477248) | 20260507 05:32:27.202 | INFO | MemInfo: { | 20260507 05:32:27.202 | INFO | "gpu" : " 17 GB", | 20260507 05:32:27.202 | INFO | "huge" : "", | 20260507 05:32:27.202 | INFO | "huge_shared" : " 27 GB", | 20260507 05:32:27.202 | INFO | "malloc" : " 14 GB", | 20260507 05:32:27.202 | INFO | "pinned" : " 29 GB" | 20260507 05:32:27.203 | INFO | } | 20260507 05:32:27.203 | INFO | Perf report for the last 5.1481 seconds: | 20260507 05:32:27.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:32:27.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:27.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:27.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:27.203 | INFO | ANALYSIS : 59.5%, 3.0, 92.5ms, 0 | 20260507 05:32:27.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:27.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:27.203 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:32:27.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:27.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:27.203 | INFO | Uploaded 36GiB to GPU in 1.97931s (18.1882GiB/s) | 20260507 05:32:27.203 | INFO | Downloaded 34GiB to GPU in 1.93217s (17.5968GiB/s) | 20260507 05:32:32.341 | INFO | Analyzing chunk frames = [7477248,7477760) | 20260507 05:32:32.341 | INFO | MemInfo: { | 20260507 05:32:32.341 | INFO | "gpu" : " 17 GB", | 20260507 05:32:32.341 | INFO | "huge" : "", | 20260507 05:32:32.341 | INFO | "huge_shared" : " 27 GB", | 20260507 05:32:32.341 | INFO | "malloc" : " 14 GB", | 20260507 05:32:32.341 | INFO | "pinned" : " 29 GB" | 20260507 05:32:32.341 | INFO | } | 20260507 05:32:32.341 | INFO | Perf report for the last 5.1386 seconds: | 20260507 05:32:32.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:32:32.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:32.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:32.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:32.341 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 05:32:32.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:32.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:32.341 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:32:32.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:32.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:32.342 | INFO | Uploaded 36GiB to GPU in 2.05172s (17.5462GiB/s) | 20260507 05:32:32.342 | INFO | Downloaded 34GiB to GPU in 1.90867s (17.8134GiB/s) | 20260507 05:32:37.471 | INFO | Analyzing chunk frames = [7477760,7478272) | 20260507 05:32:37.471 | INFO | MemInfo: { | 20260507 05:32:37.471 | INFO | "gpu" : " 17 GB", | 20260507 05:32:37.471 | INFO | "huge" : "", | 20260507 05:32:37.471 | INFO | "huge_shared" : " 27 GB", | 20260507 05:32:37.471 | INFO | "malloc" : " 14 GB", | 20260507 05:32:37.471 | INFO | "pinned" : " 29 GB" | 20260507 05:32:37.471 | INFO | } | 20260507 05:32:37.471 | INFO | Perf report for the last 5.13015 seconds: | 20260507 05:32:37.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:32:37.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:37.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:37.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:37.471 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:32:37.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:37.471 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:37.471 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 05:32:37.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:37.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:37.472 | INFO | Uploaded 36GiB to GPU in 2.06747s (17.4126GiB/s) | 20260507 05:32:37.472 | INFO | Downloaded 34GiB to GPU in 1.9387s (17.5375GiB/s) | 20260507 05:32:42.611 | INFO | Analyzing chunk frames = [7478272,7478784) | 20260507 05:32:42.611 | INFO | MemInfo: { | 20260507 05:32:42.611 | INFO | "gpu" : " 17 GB", | 20260507 05:32:42.611 | INFO | "huge" : "", | 20260507 05:32:42.611 | INFO | "huge_shared" : " 27 GB", | 20260507 05:32:42.611 | INFO | "malloc" : " 14 GB", | 20260507 05:32:42.611 | INFO | "pinned" : " 29 GB" | 20260507 05:32:42.611 | INFO | } | 20260507 05:32:42.612 | INFO | Perf report for the last 5.14021 seconds: | 20260507 05:32:42.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:32:42.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:42.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:42.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:42.612 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 05:32:42.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:42.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:42.612 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 05:32:42.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:42.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:42.612 | INFO | Uploaded 36GiB to GPU in 2.00059s (17.9947GiB/s) | 20260507 05:32:42.612 | INFO | Downloaded 34GiB to GPU in 1.98799s (17.1027GiB/s) | 20260507 05:32:47.754 | INFO | Analyzing chunk frames = [7478784,7479296) | 20260507 05:32:47.754 | INFO | MemInfo: { | 20260507 05:32:47.754 | INFO | "gpu" : " 17 GB", | 20260507 05:32:47.754 | INFO | "huge" : "", | 20260507 05:32:47.754 | INFO | "huge_shared" : " 27 GB", | 20260507 05:32:47.754 | INFO | "malloc" : " 14 GB", | 20260507 05:32:47.754 | INFO | "pinned" : " 29 GB" | 20260507 05:32:47.754 | INFO | } | 20260507 05:32:47.754 | INFO | Perf report for the last 5.14272 seconds: | 20260507 05:32:47.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:32:47.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:47.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:47.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:47.755 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:32:47.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:47.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:47.755 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 05:32:47.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:47.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:47.755 | INFO | Uploaded 36GiB to GPU in 1.96061s (18.3617GiB/s) | 20260507 05:32:47.755 | INFO | Downloaded 34GiB to GPU in 1.94753s (17.458GiB/s) | 20260507 05:32:47.755 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:32:47.755 | INFO | Total mem usage: 18 GiB | 20260507 05:32:47.755 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:32:47.755 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 05:32:47.755 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:32:47.755 | INFO | High Water Breakdown: | 20260507 05:32:47.755 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:32:47.755 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:32:47.755 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:32:47.755 | INFO | ------------------------------------------ | 20260507 05:32:47.755 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:32:47.755 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:32:47.755 | INFO | Total mem usage: 26.5 GiB | 20260507 05:32:47.755 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:32:47.755 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:32:47.755 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 05:32:47.755 | INFO | High Water Breakdown: | 20260507 05:32:47.755 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:32:47.755 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:32:47.755 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:32:47.755 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:32:47.755 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:32:47.755 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:32:47.755 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:32:47.755 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:32:47.755 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:32:47.755 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:32:47.755 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:32:47.755 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:32:47.755 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:32:47.755 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:32:47.755 | INFO | ------------------------------------------ | 20260507 05:32:47.755 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:32:47.755 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:32:47.755 | INFO | Total mem usage: 25.6 GiB | 20260507 05:32:47.755 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:32:47.755 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:32:47.755 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:32:47.755 | INFO | High Water Breakdown: | 20260507 05:32:47.756 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:32:47.756 | INFO | ------------------------------------------ | 20260507 05:32:47.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:32:47.756 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:32:47.756 | INFO | Total mem usage: 15.7 GiB | 20260507 05:32:47.756 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:32:47.756 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:32:47.756 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:32:47.756 | INFO | High Water Breakdown: | 20260507 05:32:47.756 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:32:47.756 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:32:47.756 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:32:47.756 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:32:47.756 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:32:47.756 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:32:47.756 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:32:47.756 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:32:47.756 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:32:47.756 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:32:47.756 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:32:47.756 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:32:47.756 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:32:47.756 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:32:47.756 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:32:47.756 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:32:47.756 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:32:47.756 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:32:47.756 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:32:47.756 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:32:47.756 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:32:47.756 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:32:47.756 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:32:47.756 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:32:47.756 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:32:47.756 | INFO | ------------------------------------------ | 20260507 05:32:47.756 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:32:47.756 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:32:47.756 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:32:47.756 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:32:47.756 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.34e+01 / 1.31e+00 | 20260507 05:32:47.756 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.02e-01, 9.48e-03 / 4.65e-03 | 20260507 05:32:47.756 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.47e+00, 5.02e-01 / 2.20e-01 | 20260507 05:32:47.756 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.48e+00, 1.68e-01 / 1.11e-01 | 20260507 05:32:47.756 | INFO | PerfReport: 16.15%, DME -- 3.97e+01 / 4.40e+01, 3.72e+00 / 1.45e+00 | 20260507 05:32:47.756 | INFO | PerfReport: 57.75%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.30e+00 / 1.47e+00 | 20260507 05:32:47.756 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.68e+00, 3.84e-01 / 2.26e-01 | 20260507 05:32:47.756 | INFO | PerfReport: 2.99%, Metrics -- 9.19e-01 / 1.24e+00, 2.48e-01 / 1.50e-01 | 20260507 05:32:47.756 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.09e-01 / 3.66e-01, 5.14e-02 / 1.95e-02 | 20260507 05:32:47.757 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:32:47.757 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:32:47.757 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:32:47.757 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 05:32:47.757 | INFO | PerfReport: 21.88%, Upload -- 2.10e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260507 05:32:47.757 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 05:32:47.757 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 05:32:47.757 | INFO | PerfReport: 1.50%, LockUpload -- 1.44e+00 / 0.00e+00, 4.83e+00 / 0.00e+00 | 20260507 05:32:47.757 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260507 05:32:47.757 | INFO | PerfReport: 22.05%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 05:32:47.757 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260507 05:32:47.851 | INFO | Starting a round of Baz IO | 20260507 05:32:50.950 | INFO | Populating RTMetrics report | 20260507 05:32:50.950 | INFO | RTMetrics report populated | 20260507 05:32:50.950 | INFO | Opening tmp RTMetrics json file | 20260507 05:32:50.967 | INFO | Serializing RTMetrics to json | 20260507 05:32:50.968 | INFO | Serialization done, swapping tmp file now... | 20260507 05:32:50.969 | INFO | RTMetrics done | 20260507 05:32:51.037 | INFO | Finished a round of Baz IO: Wrote 1.693 GiB in 3.185 seconds | 20260507 05:32:51.037 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.70% / 0.00% / 67.30% | 20260507 05:32:51.037 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.142105% / 0% / 7.54992e-05% | 20260507 05:32:51.037 | INFO | Baz IO overall: | 20260507 05:32:51.037 | INFO | BytesWritten : 3792773901312 | 20260507 05:32:51.037 | INFO | NumEvents : 1819070053835 | 20260507 05:32:52.901 | INFO | Analyzing chunk frames = [7479296,7479808) | 20260507 05:32:52.901 | INFO | MemInfo: { | 20260507 05:32:52.901 | INFO | "gpu" : " 17 GB", | 20260507 05:32:52.901 | INFO | "huge" : "", | 20260507 05:32:52.901 | INFO | "huge_shared" : " 27 GB", | 20260507 05:32:52.901 | INFO | "malloc" : " 14 GB", | 20260507 05:32:52.901 | INFO | "pinned" : " 29 GB" | 20260507 05:32:52.901 | INFO | } | 20260507 05:32:52.901 | INFO | Perf report for the last 5.1471 seconds: | 20260507 05:32:52.902 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:32:52.902 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:52.902 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:52.902 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:52.902 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260507 05:32:52.902 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:52.902 | INFO | RT_METRICS : 44.1%, 1.0, 23.2ms, 0 | 20260507 05:32:52.902 | INFO | PRE_HQ : 59.6%, 1.9, 59.2ms, 0 | 20260507 05:32:52.902 | INFO | BAZWRITER : 61.5%, 3.3, 104.9ms, 0 | 20260507 05:32:52.902 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 05:32:52.902 | INFO | Uploaded 36GiB to GPU in 2.03521s (17.6886GiB/s) | 20260507 05:32:52.902 | INFO | Downloaded 38GiB to GPU in 2.1724s (17.4921GiB/s) | 20260507 05:32:58.032 | INFO | Analyzing chunk frames = [7479808,7480320) | 20260507 05:32:58.032 | INFO | MemInfo: { | 20260507 05:32:58.032 | INFO | "gpu" : " 17 GB", | 20260507 05:32:58.032 | INFO | "huge" : "", | 20260507 05:32:58.032 | INFO | "huge_shared" : " 27 GB", | 20260507 05:32:58.032 | INFO | "malloc" : " 14 GB", | 20260507 05:32:58.032 | INFO | "pinned" : " 29 GB" | 20260507 05:32:58.032 | INFO | } | 20260507 05:32:58.032 | INFO | Perf report for the last 5.13037 seconds: | 20260507 05:32:58.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:32:58.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:58.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:58.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:58.032 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 05:32:58.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:58.032 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:58.032 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:32:58.032 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:58.032 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:32:58.032 | INFO | Uploaded 36GiB to GPU in 2.00589s (17.9472GiB/s) | 20260507 05:32:58.032 | INFO | Downloaded 34GiB to GPU in 1.94358s (17.4935GiB/s) | 20260507 05:33:03.167 | INFO | Analyzing chunk frames = [7480320,7480832) | 20260507 05:33:03.167 | INFO | MemInfo: { | 20260507 05:33:03.167 | INFO | "gpu" : " 17 GB", | 20260507 05:33:03.167 | INFO | "huge" : "", | 20260507 05:33:03.167 | INFO | "huge_shared" : " 27 GB", | 20260507 05:33:03.167 | INFO | "malloc" : " 14 GB", | 20260507 05:33:03.167 | INFO | "pinned" : " 29 GB" | 20260507 05:33:03.167 | INFO | } | 20260507 05:33:03.167 | INFO | Perf report for the last 5.1351 seconds: | 20260507 05:33:03.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:33:03.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:03.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:03.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:03.167 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 05:33:03.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:03.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:03.167 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:33:03.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:03.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:03.167 | INFO | Uploaded 36GiB to GPU in 2.01601s (17.857GiB/s) | 20260507 05:33:03.167 | INFO | Downloaded 34GiB to GPU in 1.94723s (17.4607GiB/s) | 20260507 05:33:08.316 | INFO | Analyzing chunk frames = [7480832,7481344) | 20260507 05:33:08.316 | INFO | MemInfo: { | 20260507 05:33:08.316 | INFO | "gpu" : " 17 GB", | 20260507 05:33:08.316 | INFO | "huge" : "", | 20260507 05:33:08.316 | INFO | "huge_shared" : " 27 GB", | 20260507 05:33:08.316 | INFO | "malloc" : " 14 GB", | 20260507 05:33:08.316 | INFO | "pinned" : " 29 GB" | 20260507 05:33:08.316 | INFO | } | 20260507 05:33:08.316 | INFO | Perf report for the last 5.14903 seconds: | 20260507 05:33:08.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:33:08.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:08.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:08.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:08.316 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 05:33:08.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:08.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:08.316 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:33:08.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:08.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:08.316 | INFO | Uploaded 36GiB to GPU in 1.98143s (18.1687GiB/s) | 20260507 05:33:08.316 | INFO | Downloaded 34GiB to GPU in 1.9252s (17.6605GiB/s) | 20260507 05:33:13.452 | INFO | Analyzing chunk frames = [7481344,7481856) | 20260507 05:33:13.452 | INFO | MemInfo: { | 20260507 05:33:13.452 | INFO | "gpu" : " 17 GB", | 20260507 05:33:13.452 | INFO | "huge" : "", | 20260507 05:33:13.452 | INFO | "huge_shared" : " 27 GB", | 20260507 05:33:13.452 | INFO | "malloc" : " 14 GB", | 20260507 05:33:13.452 | INFO | "pinned" : " 29 GB" | 20260507 05:33:13.452 | INFO | } | 20260507 05:33:13.452 | INFO | Perf report for the last 5.13638 seconds: | 20260507 05:33:13.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:33:13.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:13.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:13.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:13.452 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 05:33:13.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:13.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:13.452 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:33:13.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:13.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:13.453 | INFO | Uploaded 36GiB to GPU in 2.04997s (17.5612GiB/s) | 20260507 05:33:13.453 | INFO | Downloaded 34GiB to GPU in 1.95109s (17.4261GiB/s) | 20260507 05:33:18.584 | INFO | Analyzing chunk frames = [7481856,7482368) | 20260507 05:33:18.584 | INFO | MemInfo: { | 20260507 05:33:18.584 | INFO | "gpu" : " 17 GB", | 20260507 05:33:18.584 | INFO | "huge" : "", | 20260507 05:33:18.584 | INFO | "huge_shared" : " 27 GB", | 20260507 05:33:18.584 | INFO | "malloc" : " 14 GB", | 20260507 05:33:18.584 | INFO | "pinned" : " 29 GB" | 20260507 05:33:18.584 | INFO | } | 20260507 05:33:18.584 | INFO | Perf report for the last 5.13174 seconds: | 20260507 05:33:18.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:33:18.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:18.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:18.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:18.584 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 05:33:18.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:18.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:18.584 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 05:33:18.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:18.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:18.584 | INFO | Uploaded 36GiB to GPU in 2.02297s (17.7956GiB/s) | 20260507 05:33:18.584 | INFO | Downloaded 34GiB to GPU in 1.91315s (17.7718GiB/s) | 20260507 05:33:23.729 | INFO | Analyzing chunk frames = [7482368,7482880) | 20260507 05:33:23.729 | INFO | MemInfo: { | 20260507 05:33:23.729 | INFO | "gpu" : " 17 GB", | 20260507 05:33:23.729 | INFO | "huge" : "", | 20260507 05:33:23.729 | INFO | "huge_shared" : " 27 GB", | 20260507 05:33:23.729 | INFO | "malloc" : " 14 GB", | 20260507 05:33:23.729 | INFO | "pinned" : " 29 GB" | 20260507 05:33:23.729 | INFO | } | 20260507 05:33:23.729 | INFO | Perf report for the last 5.14449 seconds: | 20260507 05:33:23.729 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:33:23.729 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:23.729 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:23.729 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:23.729 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:33:23.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:23.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:23.729 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 05:33:23.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:23.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:23.729 | INFO | Uploaded 36GiB to GPU in 1.9643s (18.3272GiB/s) | 20260507 05:33:23.729 | INFO | Downloaded 34GiB to GPU in 1.97439s (17.2205GiB/s) | 20260507 05:33:28.870 | INFO | Analyzing chunk frames = [7482880,7483392) | 20260507 05:33:28.870 | INFO | MemInfo: { | 20260507 05:33:28.870 | INFO | "gpu" : " 17 GB", | 20260507 05:33:28.870 | INFO | "huge" : "", | 20260507 05:33:28.870 | INFO | "huge_shared" : " 27 GB", | 20260507 05:33:28.870 | INFO | "malloc" : " 14 GB", | 20260507 05:33:28.870 | INFO | "pinned" : " 29 GB" | 20260507 05:33:28.870 | INFO | } | 20260507 05:33:28.870 | INFO | Perf report for the last 5.14126 seconds: | 20260507 05:33:28.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:33:28.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:28.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:28.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:28.870 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:33:28.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:28.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:28.870 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 05:33:28.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:28.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:28.870 | INFO | Uploaded 36GiB to GPU in 1.95855s (18.381GiB/s) | 20260507 05:33:28.870 | INFO | Downloaded 34GiB to GPU in 1.95789s (17.3656GiB/s) | 20260507 05:33:28.974 | INFO | Starting a round of Baz IO | 20260507 05:33:32.074 | INFO | Populating RTMetrics report | 20260507 05:33:32.074 | INFO | RTMetrics report populated | 20260507 05:33:32.074 | INFO | Opening tmp RTMetrics json file | 20260507 05:33:32.084 | INFO | Serializing RTMetrics to json | 20260507 05:33:32.090 | INFO | Serialization done, swapping tmp file now... | 20260507 05:33:32.090 | INFO | RTMetrics done | 20260507 05:33:32.166 | INFO | Finished a round of Baz IO: Wrote 1.693 GiB in 3.191 seconds | 20260507 05:33:32.166 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.70% / 0.00% / 67.30% | 20260507 05:33:32.166 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.141366% / 7.94729e-06% / 4.37101e-05% | 20260507 05:33:32.166 | INFO | Baz IO overall: | 20260507 05:33:32.166 | INFO | BytesWritten : 3794591838909 | 20260507 05:33:32.166 | INFO | NumEvents : 1819923412838 | 20260507 05:33:34.010 | INFO | Analyzing chunk frames = [7483392,7483904) | 20260507 05:33:34.010 | INFO | MemInfo: { | 20260507 05:33:34.010 | INFO | "gpu" : " 17 GB", | 20260507 05:33:34.010 | INFO | "huge" : "", | 20260507 05:33:34.010 | INFO | "huge_shared" : " 27 GB", | 20260507 05:33:34.010 | INFO | "malloc" : " 14 GB", | 20260507 05:33:34.010 | INFO | "pinned" : " 29 GB" | 20260507 05:33:34.010 | INFO | } | 20260507 05:33:34.010 | INFO | Perf report for the last 5.14031 seconds: | 20260507 05:33:34.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:33:34.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:34.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:34.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:34.010 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 05:33:34.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:34.010 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 05:33:34.010 | INFO | PRE_HQ : 59.7%, 1.9, 58.1ms, 0 | 20260507 05:33:34.010 | INFO | BAZWRITER : 61.8%, 3.3, 103.6ms, 0 | 20260507 05:33:34.010 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:33:34.010 | INFO | Uploaded 36GiB to GPU in 2.01864s (17.8338GiB/s) | 20260507 05:33:34.010 | INFO | Downloaded 38GiB to GPU in 2.16799s (17.5278GiB/s) | 20260507 05:33:39.151 | INFO | Analyzing chunk frames = [7483904,7484416) | 20260507 05:33:39.151 | INFO | MemInfo: { | 20260507 05:33:39.151 | INFO | "gpu" : " 17 GB", | 20260507 05:33:39.151 | INFO | "huge" : "", | 20260507 05:33:39.151 | INFO | "huge_shared" : " 27 GB", | 20260507 05:33:39.151 | INFO | "malloc" : " 14 GB", | 20260507 05:33:39.151 | INFO | "pinned" : " 29 GB" | 20260507 05:33:39.151 | INFO | } | 20260507 05:33:39.151 | INFO | Perf report for the last 5.14065 seconds: | 20260507 05:33:39.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:33:39.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:39.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:39.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:39.151 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:33:39.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:39.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:39.151 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 05:33:39.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:39.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:39.151 | INFO | Uploaded 36GiB to GPU in 2.00384s (17.9655GiB/s) | 20260507 05:33:39.151 | INFO | Downloaded 34GiB to GPU in 1.93648s (17.5576GiB/s) | 20260507 05:33:44.285 | INFO | Analyzing chunk frames = [7484416,7484928) | 20260507 05:33:44.285 | INFO | MemInfo: { | 20260507 05:33:44.285 | INFO | "gpu" : " 17 GB", | 20260507 05:33:44.285 | INFO | "huge" : "", | 20260507 05:33:44.285 | INFO | "huge_shared" : " 27 GB", | 20260507 05:33:44.285 | INFO | "malloc" : " 14 GB", | 20260507 05:33:44.285 | INFO | "pinned" : " 29 GB" | 20260507 05:33:44.285 | INFO | } | 20260507 05:33:44.285 | INFO | Perf report for the last 5.13431 seconds: | 20260507 05:33:44.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:33:44.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:44.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:44.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:44.285 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 05:33:44.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:44.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:44.285 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:33:44.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:44.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:44.285 | INFO | Uploaded 36GiB to GPU in 2.02054s (17.817GiB/s) | 20260507 05:33:44.285 | INFO | Downloaded 34GiB to GPU in 1.92336s (17.6774GiB/s) | 20260507 05:33:49.422 | INFO | Analyzing chunk frames = [7484928,7485440) | 20260507 05:33:49.422 | INFO | MemInfo: { | 20260507 05:33:49.422 | INFO | "gpu" : " 17 GB", | 20260507 05:33:49.422 | INFO | "huge" : "", | 20260507 05:33:49.422 | INFO | "huge_shared" : " 27 GB", | 20260507 05:33:49.423 | INFO | "malloc" : " 14 GB", | 20260507 05:33:49.423 | INFO | "pinned" : " 29 GB" | 20260507 05:33:49.423 | INFO | } | 20260507 05:33:49.423 | INFO | Perf report for the last 5.13743 seconds: | 20260507 05:33:49.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:33:49.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:49.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:49.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:49.423 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 05:33:49.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:49.423 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:49.423 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:33:49.423 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:49.423 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:49.423 | INFO | Uploaded 36GiB to GPU in 2.0115s (17.8971GiB/s) | 20260507 05:33:49.423 | INFO | Downloaded 34GiB to GPU in 1.93775s (17.5461GiB/s) | 20260507 05:33:54.567 | INFO | Analyzing chunk frames = [7485440,7485952) | 20260507 05:33:54.567 | INFO | MemInfo: { | 20260507 05:33:54.567 | INFO | "gpu" : " 17 GB", | 20260507 05:33:54.567 | INFO | "huge" : "", | 20260507 05:33:54.567 | INFO | "huge_shared" : " 27 GB", | 20260507 05:33:54.567 | INFO | "malloc" : " 14 GB", | 20260507 05:33:54.567 | INFO | "pinned" : " 29 GB" | 20260507 05:33:54.567 | INFO | } | 20260507 05:33:54.567 | INFO | Perf report for the last 5.14409 seconds: | 20260507 05:33:54.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:33:54.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:54.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:54.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:54.567 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 05:33:54.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:54.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:54.567 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:33:54.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:54.567 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:54.567 | INFO | Uploaded 36GiB to GPU in 2.02033s (17.8189GiB/s) | 20260507 05:33:54.567 | INFO | Downloaded 34GiB to GPU in 1.94076s (17.519GiB/s) | 20260507 05:33:59.708 | INFO | Analyzing chunk frames = [7485952,7486464) | 20260507 05:33:59.708 | INFO | MemInfo: { | 20260507 05:33:59.708 | INFO | "gpu" : " 17 GB", | 20260507 05:33:59.708 | INFO | "huge" : "", | 20260507 05:33:59.708 | INFO | "huge_shared" : " 27 GB", | 20260507 05:33:59.708 | INFO | "malloc" : " 14 GB", | 20260507 05:33:59.708 | INFO | "pinned" : " 29 GB" | 20260507 05:33:59.708 | INFO | } | 20260507 05:33:59.709 | INFO | Perf report for the last 5.14226 seconds: | 20260507 05:33:59.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:33:59.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:59.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:59.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:59.709 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 05:33:59.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:59.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:59.709 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:33:59.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:59.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:33:59.709 | INFO | Uploaded 36GiB to GPU in 2.03468s (17.6932GiB/s) | 20260507 05:33:59.709 | INFO | Downloaded 34GiB to GPU in 1.92581s (17.6549GiB/s) | 20260507 05:34:04.849 | INFO | Analyzing chunk frames = [7486464,7486976) | 20260507 05:34:04.849 | INFO | MemInfo: { | 20260507 05:34:04.849 | INFO | "gpu" : " 17 GB", | 20260507 05:34:04.849 | INFO | "huge" : "", | 20260507 05:34:04.849 | INFO | "huge_shared" : " 27 GB", | 20260507 05:34:04.849 | INFO | "malloc" : " 14 GB", | 20260507 05:34:04.849 | INFO | "pinned" : " 29 GB" | 20260507 05:34:04.849 | INFO | } | 20260507 05:34:04.849 | INFO | Perf report for the last 5.14 seconds: | 20260507 05:34:04.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:34:04.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:04.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:04.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:04.849 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:34:04.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:04.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:04.849 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 05:34:04.849 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:04.849 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:04.849 | INFO | Uploaded 36GiB to GPU in 1.90749s (18.873GiB/s) | 20260507 05:34:04.849 | INFO | Downloaded 34GiB to GPU in 1.97339s (17.2293GiB/s) | 20260507 05:34:09.978 | INFO | Analyzing chunk frames = [7486976,7487488) | 20260507 05:34:09.978 | INFO | MemInfo: { | 20260507 05:34:09.978 | INFO | "gpu" : " 17 GB", | 20260507 05:34:09.978 | INFO | "huge" : "", | 20260507 05:34:09.978 | INFO | "huge_shared" : " 27 GB", | 20260507 05:34:09.978 | INFO | "malloc" : " 14 GB", | 20260507 05:34:09.978 | INFO | "pinned" : " 29 GB" | 20260507 05:34:09.978 | INFO | } | 20260507 05:34:09.978 | INFO | Perf report for the last 5.12893 seconds: | 20260507 05:34:09.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:34:09.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:09.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:09.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:09.978 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 05:34:09.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:09.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:09.978 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:34:09.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:09.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:09.978 | INFO | Uploaded 36GiB to GPU in 1.99199s (18.0724GiB/s) | 20260507 05:34:09.978 | INFO | Downloaded 34GiB to GPU in 1.95661s (17.377GiB/s) | 20260507 05:34:09.978 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:34:09.978 | INFO | Total mem usage: 18 GiB | 20260507 05:34:09.978 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:34:09.978 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:34:09.978 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:34:09.978 | INFO | High Water Breakdown: | 20260507 05:34:09.978 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:34:09.978 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:34:09.978 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:34:09.978 | INFO | ------------------------------------------ | 20260507 05:34:09.979 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:34:09.979 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:34:09.979 | INFO | Total mem usage: 26.5 GiB | 20260507 05:34:09.979 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:34:09.979 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:34:09.979 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:34:09.979 | INFO | High Water Breakdown: | 20260507 05:34:09.979 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:34:09.979 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:34:09.979 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:34:09.979 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:34:09.979 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:34:09.979 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:34:09.979 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:34:09.979 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:34:09.979 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:34:09.979 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:34:09.979 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:34:09.979 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:34:09.979 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:34:09.979 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:34:09.979 | INFO | ------------------------------------------ | 20260507 05:34:09.979 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:34:09.979 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:34:09.979 | INFO | Total mem usage: 25.6 GiB | 20260507 05:34:09.979 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:34:09.979 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:34:09.979 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:34:09.979 | INFO | High Water Breakdown: | 20260507 05:34:09.979 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:34:09.979 | INFO | ------------------------------------------ | 20260507 05:34:09.979 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:34:09.979 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:34:09.979 | INFO | Total mem usage: 15.7 GiB | 20260507 05:34:09.979 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:34:09.979 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:34:09.979 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:34:09.979 | INFO | High Water Breakdown: | 20260507 05:34:09.979 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:34:09.979 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:34:09.979 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:34:09.979 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:34:09.979 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:34:09.979 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:34:09.979 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:34:09.979 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:34:09.979 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:34:09.979 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:34:09.979 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:34:09.979 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:34:09.979 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:34:09.979 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:34:09.979 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:34:09.979 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:34:09.979 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:34:09.979 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:34:09.979 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:34:09.979 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:34:09.979 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:34:09.979 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:34:09.979 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:34:09.979 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:34:09.979 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:34:09.979 | INFO | ------------------------------------------ | 20260507 05:34:09.979 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:34:09.979 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:34:09.979 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:34:09.979 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:34:09.979 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.33e+01 / 1.33e+00 | 20260507 05:34:09.979 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.01e-01, 9.47e-03 / 5.68e-03 | 20260507 05:34:09.979 | INFO | PerfReport: 11.73%, Baseline -- 3.60e+00 / 4.49e+00, 5.26e-01 / 2.29e-01 | 20260507 05:34:09.979 | INFO | PerfReport: 3.24%, Histogram -- 9.96e-01 / 1.49e+00, 1.72e-01 / 1.07e-01 | 20260507 05:34:09.979 | INFO | PerfReport: 16.20%, DME -- 3.98e+01 / 4.46e+01, 3.94e+00 / 1.48e+00 | 20260507 05:34:09.979 | INFO | PerfReport: 57.61%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.27e+00 / 1.53e+00 | 20260507 05:34:09.979 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.73e+00, 3.84e-01 / 1.97e-01 | 20260507 05:34:09.979 | INFO | PerfReport: 2.99%, Metrics -- 9.18e-01 / 1.23e+00, 2.46e-01 / 1.10e-01 | 20260507 05:34:09.979 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.09e-01 / 3.62e-01, 5.12e-02 / 1.78e-02 | 20260507 05:34:09.979 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:34:09.979 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:34:09.979 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:34:09.979 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 05:34:09.979 | INFO | PerfReport: 21.71%, Upload -- 2.09e+01 / 0.00e+00, 6.97e+00 / 0.00e+00 | 20260507 05:34:09.979 | INFO | PerfReport: 21.37%, Download -- 2.05e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260507 05:34:09.979 | INFO | PerfReport: 31.99%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 05:34:09.979 | INFO | PerfReport: 1.40%, LockUpload -- 1.35e+00 / 0.00e+00, 4.74e+00 / 0.00e+00 | 20260507 05:34:09.979 | INFO | PerfReport: 1.40%, LockDownload -- 1.35e+00 / 0.00e+00, 3.28e+00 / 0.00e+00 | 20260507 05:34:09.979 | INFO | PerfReport: 22.11%, LockCompute -- 2.13e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 05:34:09.979 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.45e-02 / 0.00e+00 | 20260507 05:34:10.074 | INFO | Starting a round of Baz IO | 20260507 05:34:13.175 | INFO | Populating RTMetrics report | 20260507 05:34:13.176 | INFO | RTMetrics report populated | 20260507 05:34:13.176 | INFO | Opening tmp RTMetrics json file | 20260507 05:34:13.193 | INFO | Serializing RTMetrics to json | 20260507 05:34:13.194 | INFO | Serialization done, swapping tmp file now... | 20260507 05:34:13.194 | INFO | RTMetrics done | 20260507 05:34:13.263 | INFO | Finished a round of Baz IO: Wrote 1.693 GiB in 3.188 seconds | 20260507 05:34:13.263 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.70% / 0.00% / 67.30% | 20260507 05:34:13.263 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.141311% / 3.97364e-06% / 5.96046e-05% | 20260507 05:34:13.263 | INFO | Baz IO overall: | 20260507 05:34:13.263 | INFO | BytesWritten : 3796409370870 | 20260507 05:34:13.263 | INFO | NumEvents : 1820776281667 | 20260507 05:34:15.122 | INFO | Analyzing chunk frames = [7487488,7488000) | 20260507 05:34:15.122 | INFO | MemInfo: { | 20260507 05:34:15.122 | INFO | "gpu" : " 17 GB", | 20260507 05:34:15.122 | INFO | "huge" : "", | 20260507 05:34:15.122 | INFO | "huge_shared" : " 27 GB", | 20260507 05:34:15.122 | INFO | "malloc" : " 14 GB", | 20260507 05:34:15.122 | INFO | "pinned" : " 29 GB" | 20260507 05:34:15.122 | INFO | } | 20260507 05:34:15.122 | INFO | Perf report for the last 5.14405 seconds: | 20260507 05:34:15.122 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:34:15.122 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:15.122 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:15.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:15.122 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 05:34:15.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:15.122 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 05:34:15.122 | INFO | PRE_HQ : 59.0%, 1.8, 55.0ms, 0 | 20260507 05:34:15.122 | INFO | BAZWRITER : 61.6%, 3.2, 101.7ms, 0 | 20260507 05:34:15.122 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:34:15.122 | INFO | Uploaded 36GiB to GPU in 2.02887s (17.7439GiB/s) | 20260507 05:34:15.122 | INFO | Downloaded 38GiB to GPU in 2.17877s (17.441GiB/s) | 20260507 05:34:20.266 | INFO | Analyzing chunk frames = [7488000,7488512) | 20260507 05:34:20.266 | INFO | MemInfo: { | 20260507 05:34:20.266 | INFO | "gpu" : " 17 GB", | 20260507 05:34:20.266 | INFO | "huge" : "", | 20260507 05:34:20.266 | INFO | "huge_shared" : " 27 GB", | 20260507 05:34:20.266 | INFO | "malloc" : " 14 GB", | 20260507 05:34:20.266 | INFO | "pinned" : " 29 GB" | 20260507 05:34:20.266 | INFO | } | 20260507 05:34:20.266 | INFO | Perf report for the last 5.14397 seconds: | 20260507 05:34:20.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:34:20.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:20.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:20.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:20.266 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 05:34:20.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:20.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:20.266 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:34:20.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:20.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:20.266 | INFO | Uploaded 36GiB to GPU in 2.04215s (17.6284GiB/s) | 20260507 05:34:20.266 | INFO | Downloaded 34GiB to GPU in 1.93716s (17.5514GiB/s) | 20260507 05:34:25.404 | INFO | Analyzing chunk frames = [7488512,7489024) | 20260507 05:34:25.404 | INFO | MemInfo: { | 20260507 05:34:25.404 | INFO | "gpu" : " 17 GB", | 20260507 05:34:25.404 | INFO | "huge" : "", | 20260507 05:34:25.404 | INFO | "huge_shared" : " 27 GB", | 20260507 05:34:25.404 | INFO | "malloc" : " 14 GB", | 20260507 05:34:25.404 | INFO | "pinned" : " 29 GB" | 20260507 05:34:25.404 | INFO | } | 20260507 05:34:25.404 | INFO | Perf report for the last 5.13788 seconds: | 20260507 05:34:25.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:34:25.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:25.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:25.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:25.404 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 05:34:25.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:25.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:25.404 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 05:34:25.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:25.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:25.404 | INFO | Uploaded 36GiB to GPU in 2.04884s (17.5709GiB/s) | 20260507 05:34:25.404 | INFO | Downloaded 34GiB to GPU in 1.94819s (17.4521GiB/s) | 20260507 05:34:30.539 | INFO | Analyzing chunk frames = [7489024,7489536) | 20260507 05:34:30.539 | INFO | MemInfo: { | 20260507 05:34:30.539 | INFO | "gpu" : " 17 GB", | 20260507 05:34:30.539 | INFO | "huge" : "", | 20260507 05:34:30.539 | INFO | "huge_shared" : " 27 GB", | 20260507 05:34:30.539 | INFO | "malloc" : " 14 GB", | 20260507 05:34:30.539 | INFO | "pinned" : " 29 GB" | 20260507 05:34:30.539 | INFO | } | 20260507 05:34:30.539 | INFO | Perf report for the last 5.13542 seconds: | 20260507 05:34:30.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:34:30.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:30.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:30.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:30.539 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 05:34:30.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:30.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:30.539 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:34:30.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:30.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:30.540 | INFO | Uploaded 36GiB to GPU in 2.04162s (17.633GiB/s) | 20260507 05:34:30.540 | INFO | Downloaded 34GiB to GPU in 1.94753s (17.458GiB/s) | 20260507 05:34:35.682 | INFO | Analyzing chunk frames = [7489536,7490048) | 20260507 05:34:35.682 | INFO | MemInfo: { | 20260507 05:34:35.682 | INFO | "gpu" : " 17 GB", | 20260507 05:34:35.682 | INFO | "huge" : "", | 20260507 05:34:35.682 | INFO | "huge_shared" : " 27 GB", | 20260507 05:34:35.682 | INFO | "malloc" : " 14 GB", | 20260507 05:34:35.682 | INFO | "pinned" : " 29 GB" | 20260507 05:34:35.682 | INFO | } | 20260507 05:34:35.682 | INFO | Perf report for the last 5.14278 seconds: | 20260507 05:34:35.682 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:34:35.682 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:35.682 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:35.682 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:35.682 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 05:34:35.682 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:35.682 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:35.682 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:34:35.682 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:35.682 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:35.682 | INFO | Uploaded 36GiB to GPU in 2.04046s (17.6431GiB/s) | 20260507 05:34:35.682 | INFO | Downloaded 34GiB to GPU in 1.94078s (17.5187GiB/s) | 20260507 05:34:40.822 | INFO | Analyzing chunk frames = [7490048,7490560) | 20260507 05:34:40.822 | INFO | MemInfo: { | 20260507 05:34:40.822 | INFO | "gpu" : " 17 GB", | 20260507 05:34:40.822 | INFO | "huge" : "", | 20260507 05:34:40.822 | INFO | "huge_shared" : " 27 GB", | 20260507 05:34:40.822 | INFO | "malloc" : " 14 GB", | 20260507 05:34:40.822 | INFO | "pinned" : " 29 GB" | 20260507 05:34:40.822 | INFO | } | 20260507 05:34:40.822 | INFO | Perf report for the last 5.14028 seconds: | 20260507 05:34:40.822 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:34:40.822 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:40.822 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:40.822 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:40.822 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 05:34:40.822 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:40.822 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:40.822 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 05:34:40.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:40.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:40.823 | INFO | Uploaded 36GiB to GPU in 2.04032s (17.6443GiB/s) | 20260507 05:34:40.823 | INFO | Downloaded 34GiB to GPU in 1.90948s (17.8059GiB/s) | 20260507 05:34:45.963 | INFO | Analyzing chunk frames = [7490560,7491072) | 20260507 05:34:45.963 | INFO | MemInfo: { | 20260507 05:34:45.963 | INFO | "gpu" : " 17 GB", | 20260507 05:34:45.963 | INFO | "huge" : "", | 20260507 05:34:45.963 | INFO | "huge_shared" : " 27 GB", | 20260507 05:34:45.963 | INFO | "malloc" : " 14 GB", | 20260507 05:34:45.963 | INFO | "pinned" : " 29 GB" | 20260507 05:34:45.963 | INFO | } | 20260507 05:34:45.963 | INFO | Perf report for the last 5.14097 seconds: | 20260507 05:34:45.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:34:45.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:45.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:45.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:45.963 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:34:45.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:45.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:45.963 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:34:45.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:45.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:45.964 | INFO | Uploaded 36GiB to GPU in 1.99676s (18.0292GiB/s) | 20260507 05:34:45.964 | INFO | Downloaded 34GiB to GPU in 1.97309s (17.2319GiB/s) | 20260507 05:34:51.097 | INFO | Analyzing chunk frames = [7491072,7491584) | 20260507 05:34:51.097 | INFO | MemInfo: { | 20260507 05:34:51.097 | INFO | "gpu" : " 17 GB", | 20260507 05:34:51.097 | INFO | "huge" : "", | 20260507 05:34:51.097 | INFO | "huge_shared" : " 27 GB", | 20260507 05:34:51.097 | INFO | "malloc" : " 14 GB", | 20260507 05:34:51.097 | INFO | "pinned" : " 29 GB" | 20260507 05:34:51.097 | INFO | } | 20260507 05:34:51.097 | INFO | Perf report for the last 5.13382 seconds: | 20260507 05:34:51.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:34:51.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:51.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:51.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:51.097 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 05:34:51.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:51.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:51.097 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:34:51.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:51.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:51.097 | INFO | Uploaded 36GiB to GPU in 1.98354s (18.1493GiB/s) | 20260507 05:34:51.098 | INFO | Downloaded 34GiB to GPU in 1.96357s (17.3154GiB/s) | 20260507 05:34:51.195 | INFO | Starting a round of Baz IO | 20260507 05:34:54.289 | INFO | Populating RTMetrics report | 20260507 05:34:54.290 | INFO | RTMetrics report populated | 20260507 05:34:54.290 | INFO | Opening tmp RTMetrics json file | 20260507 05:34:54.299 | INFO | Serializing RTMetrics to json | 20260507 05:34:54.306 | INFO | Serialization done, swapping tmp file now... | 20260507 05:34:54.306 | INFO | RTMetrics done | 20260507 05:34:54.377 | INFO | Finished a round of Baz IO: Wrote 1.692 GiB in 3.182 seconds | 20260507 05:34:54.378 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.69% / 0.00% / 67.30% | 20260507 05:34:54.378 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.141076% / 0% / 7.15256e-05% | 20260507 05:34:54.378 | INFO | Baz IO overall: | 20260507 05:34:54.378 | INFO | BytesWritten : 3798226226808 | 20260507 05:34:54.378 | INFO | NumEvents : 1821629055096 | 20260507 05:34:56.236 | INFO | Analyzing chunk frames = [7491584,7492096) | 20260507 05:34:56.236 | INFO | MemInfo: { | 20260507 05:34:56.236 | INFO | "gpu" : " 17 GB", | 20260507 05:34:56.236 | INFO | "huge" : "", | 20260507 05:34:56.236 | INFO | "huge_shared" : " 27 GB", | 20260507 05:34:56.236 | INFO | "malloc" : " 14 GB", | 20260507 05:34:56.236 | INFO | "pinned" : " 29 GB" | 20260507 05:34:56.236 | INFO | } | 20260507 05:34:56.236 | INFO | Perf report for the last 5.13859 seconds: | 20260507 05:34:56.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:34:56.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:56.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:56.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:56.236 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 05:34:56.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:34:56.236 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 05:34:56.236 | INFO | PRE_HQ : 58.9%, 1.8, 54.8ms, 0 | 20260507 05:34:56.236 | INFO | BAZWRITER : 61.5%, 3.2, 101.1ms, 0 | 20260507 05:34:56.236 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:34:56.236 | INFO | Uploaded 36GiB to GPU in 2.03732s (17.6703GiB/s) | 20260507 05:34:56.236 | INFO | Downloaded 38GiB to GPU in 2.18215s (17.414GiB/s) | 20260507 05:35:01.374 | INFO | Analyzing chunk frames = [7492096,7492608) | 20260507 05:35:01.374 | INFO | MemInfo: { | 20260507 05:35:01.374 | INFO | "gpu" : " 17 GB", | 20260507 05:35:01.374 | INFO | "huge" : "", | 20260507 05:35:01.374 | INFO | "huge_shared" : " 27 GB", | 20260507 05:35:01.374 | INFO | "malloc" : " 14 GB", | 20260507 05:35:01.374 | INFO | "pinned" : " 29 GB" | 20260507 05:35:01.374 | INFO | } | 20260507 05:35:01.374 | INFO | Perf report for the last 5.13809 seconds: | 20260507 05:35:01.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:35:01.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:01.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:01.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:01.374 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:35:01.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:01.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:01.374 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:35:01.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:01.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:01.374 | INFO | Uploaded 36GiB to GPU in 1.99127s (18.0789GiB/s) | 20260507 05:35:01.374 | INFO | Downloaded 34GiB to GPU in 1.93611s (17.561GiB/s) | 20260507 05:35:06.523 | INFO | Analyzing chunk frames = [7492608,7493120) | 20260507 05:35:06.523 | INFO | MemInfo: { | 20260507 05:35:06.523 | INFO | "gpu" : " 17 GB", | 20260507 05:35:06.523 | INFO | "huge" : "", | 20260507 05:35:06.523 | INFO | "huge_shared" : " 27 GB", | 20260507 05:35:06.523 | INFO | "malloc" : " 14 GB", | 20260507 05:35:06.523 | INFO | "pinned" : " 29 GB" | 20260507 05:35:06.523 | INFO | } | 20260507 05:35:06.523 | INFO | Perf report for the last 5.14899 seconds: | 20260507 05:35:06.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:35:06.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:06.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:06.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:06.523 | INFO | ANALYSIS : 59.8%, 3.0, 92.9ms, 0 | 20260507 05:35:06.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:06.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:06.523 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:35:06.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:06.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:06.523 | INFO | Uploaded 36GiB to GPU in 1.9976s (18.0216GiB/s) | 20260507 05:35:06.523 | INFO | Downloaded 34GiB to GPU in 1.9399s (17.5266GiB/s) | 20260507 05:35:11.666 | INFO | Analyzing chunk frames = [7493120,7493632) | 20260507 05:35:11.666 | INFO | MemInfo: { | 20260507 05:35:11.666 | INFO | "gpu" : " 17 GB", | 20260507 05:35:11.666 | INFO | "huge" : "", | 20260507 05:35:11.666 | INFO | "huge_shared" : " 27 GB", | 20260507 05:35:11.666 | INFO | "malloc" : " 14 GB", | 20260507 05:35:11.666 | INFO | "pinned" : " 29 GB" | 20260507 05:35:11.666 | INFO | } | 20260507 05:35:11.666 | INFO | Perf report for the last 5.14359 seconds: | 20260507 05:35:11.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:35:11.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:11.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:11.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:11.667 | INFO | ANALYSIS : 60.3%, 2.9, 93.2ms, 0 | 20260507 05:35:11.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:11.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:11.667 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:35:11.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:11.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:11.667 | INFO | Uploaded 36GiB to GPU in 2.03746s (17.669GiB/s) | 20260507 05:35:11.667 | INFO | Downloaded 34GiB to GPU in 1.95032s (17.4331GiB/s) | 20260507 05:35:16.797 | INFO | Analyzing chunk frames = [7493632,7494144) | 20260507 05:35:16.797 | INFO | MemInfo: { | 20260507 05:35:16.797 | INFO | "gpu" : " 17 GB", | 20260507 05:35:16.797 | INFO | "huge" : "", | 20260507 05:35:16.797 | INFO | "huge_shared" : " 27 GB", | 20260507 05:35:16.798 | INFO | "malloc" : " 14 GB", | 20260507 05:35:16.798 | INFO | "pinned" : " 29 GB" | 20260507 05:35:16.798 | INFO | } | 20260507 05:35:16.798 | INFO | Perf report for the last 5.1311 seconds: | 20260507 05:35:16.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:35:16.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:16.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:16.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:16.798 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 05:35:16.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:16.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:16.798 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:35:16.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:16.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:16.798 | INFO | Uploaded 36GiB to GPU in 2.04497s (17.6042GiB/s) | 20260507 05:35:16.798 | INFO | Downloaded 34GiB to GPU in 1.90173s (17.8784GiB/s) | 20260507 05:35:21.930 | INFO | Analyzing chunk frames = [7494144,7494656) | 20260507 05:35:21.930 | INFO | MemInfo: { | 20260507 05:35:21.930 | INFO | "gpu" : " 17 GB", | 20260507 05:35:21.930 | INFO | "huge" : "", | 20260507 05:35:21.930 | INFO | "huge_shared" : " 27 GB", | 20260507 05:35:21.930 | INFO | "malloc" : " 14 GB", | 20260507 05:35:21.930 | INFO | "pinned" : " 29 GB" | 20260507 05:35:21.930 | INFO | } | 20260507 05:35:21.930 | INFO | Perf report for the last 5.13224 seconds: | 20260507 05:35:21.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:35:21.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:21.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:21.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:21.930 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 05:35:21.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:21.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:21.930 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 05:35:21.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:21.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:21.930 | INFO | Uploaded 36GiB to GPU in 2.03879s (17.6575GiB/s) | 20260507 05:35:21.930 | INFO | Downloaded 34GiB to GPU in 1.92154s (17.6941GiB/s) | 20260507 05:35:27.074 | INFO | Analyzing chunk frames = [7494656,7495168) | 20260507 05:35:27.074 | INFO | MemInfo: { | 20260507 05:35:27.074 | INFO | "gpu" : " 17 GB", | 20260507 05:35:27.074 | INFO | "huge" : "", | 20260507 05:35:27.074 | INFO | "huge_shared" : " 27 GB", | 20260507 05:35:27.074 | INFO | "malloc" : " 14 GB", | 20260507 05:35:27.074 | INFO | "pinned" : " 29 GB" | 20260507 05:35:27.074 | INFO | } | 20260507 05:35:27.074 | INFO | Perf report for the last 5.14433 seconds: | 20260507 05:35:27.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:35:27.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:27.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:27.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:27.074 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 05:35:27.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:27.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:27.074 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 05:35:27.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:27.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:27.075 | INFO | Uploaded 36GiB to GPU in 1.98879s (18.1014GiB/s) | 20260507 05:35:27.075 | INFO | Downloaded 34GiB to GPU in 1.98738s (17.108GiB/s) | 20260507 05:35:32.220 | INFO | Analyzing chunk frames = [7495168,7495680) | 20260507 05:35:32.220 | INFO | MemInfo: { | 20260507 05:35:32.220 | INFO | "gpu" : " 17 GB", | 20260507 05:35:32.220 | INFO | "huge" : "", | 20260507 05:35:32.220 | INFO | "huge_shared" : " 27 GB", | 20260507 05:35:32.220 | INFO | "malloc" : " 14 GB", | 20260507 05:35:32.220 | INFO | "pinned" : " 29 GB" | 20260507 05:35:32.220 | INFO | } | 20260507 05:35:32.220 | INFO | Perf report for the last 5.14595 seconds: | 20260507 05:35:32.220 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:35:32.220 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:32.220 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:32.220 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:32.220 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 05:35:32.220 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:32.220 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:32.220 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:35:32.220 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:32.220 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:32.220 | INFO | Uploaded 36GiB to GPU in 1.99386s (18.0554GiB/s) | 20260507 05:35:32.220 | INFO | Downloaded 34GiB to GPU in 1.96832s (17.2736GiB/s) | 20260507 05:35:32.221 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:35:32.221 | INFO | Total mem usage: 18 GiB | 20260507 05:35:32.221 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:35:32.221 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 05:35:32.221 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:35:32.221 | INFO | High Water Breakdown: | 20260507 05:35:32.221 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:35:32.221 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:35:32.221 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:35:32.221 | INFO | ------------------------------------------ | 20260507 05:35:32.221 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:35:32.221 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:35:32.221 | INFO | Total mem usage: 26.5 GiB | 20260507 05:35:32.221 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:35:32.221 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:35:32.221 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:35:32.221 | INFO | High Water Breakdown: | 20260507 05:35:32.221 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:35:32.221 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:35:32.221 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:35:32.221 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:35:32.221 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:35:32.221 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:35:32.221 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:35:32.221 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:35:32.221 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:35:32.221 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:35:32.221 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:35:32.221 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:35:32.221 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:35:32.221 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:35:32.221 | INFO | ------------------------------------------ | 20260507 05:35:32.221 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:35:32.221 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:35:32.221 | INFO | Total mem usage: 25.6 GiB | 20260507 05:35:32.221 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:35:32.221 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:35:32.221 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:35:32.221 | INFO | High Water Breakdown: | 20260507 05:35:32.221 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:35:32.221 | INFO | ------------------------------------------ | 20260507 05:35:32.221 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:35:32.221 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:35:32.221 | INFO | Total mem usage: 15.7 GiB | 20260507 05:35:32.221 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:35:32.221 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:35:32.221 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:35:32.221 | INFO | High Water Breakdown: | 20260507 05:35:32.221 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:35:32.221 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:35:32.221 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:35:32.221 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:35:32.221 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:35:32.221 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:35:32.221 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:35:32.221 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:35:32.221 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:35:32.221 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:35:32.221 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:35:32.221 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:35:32.221 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:35:32.221 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:35:32.221 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:35:32.221 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:35:32.221 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:35:32.221 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:35:32.221 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:35:32.221 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:35:32.222 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:35:32.222 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:35:32.222 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:35:32.222 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:35:32.222 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:35:32.222 | INFO | ------------------------------------------ | 20260507 05:35:32.222 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:35:32.222 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:35:32.222 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:35:32.222 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:35:32.222 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.33e+01 / 1.55e+00 | 20260507 05:35:32.222 | INFO | PerfReport: 0.29%, AggregateStats -- 9.01e-02 / 9.95e-02, 9.17e-03 / 4.99e-03 | 20260507 05:35:32.222 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.44e+00, 5.07e-01 / 2.84e-01 | 20260507 05:35:32.222 | INFO | PerfReport: 3.21%, Histogram -- 9.86e-01 / 1.48e+00, 1.65e-01 / 1.10e-01 | 20260507 05:35:32.222 | INFO | PerfReport: 16.27%, DME -- 3.99e+01 / 4.51e+01, 4.07e+00 / 1.87e+00 | 20260507 05:35:32.222 | INFO | PerfReport: 57.61%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.21e+00 / 1.06e+00 | 20260507 05:35:32.222 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.13e+00 / 2.74e+00, 3.90e-01 / 1.60e-01 | 20260507 05:35:32.222 | INFO | PerfReport: 2.98%, Metrics -- 9.16e-01 / 1.23e+00, 2.45e-01 / 1.28e-01 | 20260507 05:35:32.222 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.08e-01 / 3.63e-01, 5.02e-02 / 2.23e-02 | 20260507 05:35:32.222 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:35:32.222 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:35:32.222 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:35:32.222 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 05:35:32.222 | INFO | PerfReport: 21.89%, Upload -- 2.11e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260507 05:35:32.222 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 05:35:32.222 | INFO | PerfReport: 31.90%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 05:35:32.222 | INFO | PerfReport: 1.55%, LockUpload -- 1.49e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260507 05:35:32.222 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260507 05:35:32.222 | INFO | PerfReport: 21.96%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 05:35:32.222 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260507 05:35:32.317 | INFO | Starting a round of Baz IO | 20260507 05:35:35.412 | INFO | Populating RTMetrics report | 20260507 05:35:35.413 | INFO | RTMetrics report populated | 20260507 05:35:35.413 | INFO | Opening tmp RTMetrics json file | 20260507 05:35:35.430 | INFO | Serializing RTMetrics to json | 20260507 05:35:35.432 | INFO | Serialization done, swapping tmp file now... | 20260507 05:35:35.432 | INFO | RTMetrics done | 20260507 05:35:35.501 | INFO | Finished a round of Baz IO: Wrote 1.692 GiB in 3.183 seconds | 20260507 05:35:35.501 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.69% / 0.00% / 67.31% | 20260507 05:35:35.501 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.140905% / 3.97364e-06% / 4.76837e-05% | 20260507 05:35:35.501 | INFO | Baz IO overall: | 20260507 05:35:35.501 | INFO | BytesWritten : 3800042466446 | 20260507 05:35:35.501 | INFO | NumEvents : 1822481931668 | 20260507 05:35:37.353 | INFO | Analyzing chunk frames = [7495680,7496192) | 20260507 05:35:37.353 | INFO | MemInfo: { | 20260507 05:35:37.353 | INFO | "gpu" : " 17 GB", | 20260507 05:35:37.353 | INFO | "huge" : "", | 20260507 05:35:37.353 | INFO | "huge_shared" : " 27 GB", | 20260507 05:35:37.353 | INFO | "malloc" : " 14 GB", | 20260507 05:35:37.353 | INFO | "pinned" : " 29 GB" | 20260507 05:35:37.353 | INFO | } | 20260507 05:35:37.353 | INFO | Perf report for the last 5.13244 seconds: | 20260507 05:35:37.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:35:37.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:37.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:37.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:37.353 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260507 05:35:37.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:37.353 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 05:35:37.353 | INFO | PRE_HQ : 58.9%, 1.8, 54.7ms, 0 | 20260507 05:35:37.353 | INFO | BAZWRITER : 61.6%, 3.3, 102.9ms, 0 | 20260507 05:35:37.353 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 05:35:37.353 | INFO | Uploaded 36GiB to GPU in 2.07437s (17.3546GiB/s) | 20260507 05:35:37.353 | INFO | Downloaded 38GiB to GPU in 2.17638s (17.4602GiB/s) | 20260507 05:35:42.494 | INFO | Analyzing chunk frames = [7496192,7496704) | 20260507 05:35:42.494 | INFO | MemInfo: { | 20260507 05:35:42.494 | INFO | "gpu" : " 17 GB", | 20260507 05:35:42.494 | INFO | "huge" : "", | 20260507 05:35:42.494 | INFO | "huge_shared" : " 27 GB", | 20260507 05:35:42.494 | INFO | "malloc" : " 14 GB", | 20260507 05:35:42.494 | INFO | "pinned" : " 29 GB" | 20260507 05:35:42.494 | INFO | } | 20260507 05:35:42.494 | INFO | Perf report for the last 5.14176 seconds: | 20260507 05:35:42.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:35:42.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:42.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:42.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:42.494 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 05:35:42.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:42.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:42.494 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:35:42.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:42.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:42.495 | INFO | Uploaded 36GiB to GPU in 1.9916s (18.076GiB/s) | 20260507 05:35:42.495 | INFO | Downloaded 34GiB to GPU in 1.92558s (17.657GiB/s) | 20260507 05:35:47.633 | INFO | Analyzing chunk frames = [7496704,7497216) | 20260507 05:35:47.633 | INFO | MemInfo: { | 20260507 05:35:47.633 | INFO | "gpu" : " 17 GB", | 20260507 05:35:47.633 | INFO | "huge" : "", | 20260507 05:35:47.633 | INFO | "huge_shared" : " 27 GB", | 20260507 05:35:47.633 | INFO | "malloc" : " 14 GB", | 20260507 05:35:47.633 | INFO | "pinned" : " 29 GB" | 20260507 05:35:47.633 | INFO | } | 20260507 05:35:47.633 | INFO | Perf report for the last 5.13835 seconds: | 20260507 05:35:47.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:35:47.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:47.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:47.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:47.633 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 05:35:47.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:47.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:47.633 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 05:35:47.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:47.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:47.633 | INFO | Uploaded 36GiB to GPU in 2.05361s (17.5301GiB/s) | 20260507 05:35:47.633 | INFO | Downloaded 34GiB to GPU in 1.94893s (17.4454GiB/s) | 20260507 05:35:52.770 | INFO | Analyzing chunk frames = [7497216,7497728) | 20260507 05:35:52.770 | INFO | MemInfo: { | 20260507 05:35:52.770 | INFO | "gpu" : " 17 GB", | 20260507 05:35:52.770 | INFO | "huge" : "", | 20260507 05:35:52.770 | INFO | "huge_shared" : " 27 GB", | 20260507 05:35:52.770 | INFO | "malloc" : " 14 GB", | 20260507 05:35:52.770 | INFO | "pinned" : " 29 GB" | 20260507 05:35:52.770 | INFO | } | 20260507 05:35:52.770 | INFO | Perf report for the last 5.13759 seconds: | 20260507 05:35:52.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:35:52.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:52.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:52.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:52.770 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:35:52.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:52.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:52.770 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:35:52.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:52.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:52.771 | INFO | Uploaded 36GiB to GPU in 2.01639s (17.8536GiB/s) | 20260507 05:35:52.771 | INFO | Downloaded 34GiB to GPU in 1.93221s (17.5964GiB/s) | 20260507 05:35:57.907 | INFO | Analyzing chunk frames = [7497728,7498240) | 20260507 05:35:57.907 | INFO | MemInfo: { | 20260507 05:35:57.907 | INFO | "gpu" : " 17 GB", | 20260507 05:35:57.907 | INFO | "huge" : "", | 20260507 05:35:57.907 | INFO | "huge_shared" : " 27 GB", | 20260507 05:35:57.907 | INFO | "malloc" : " 14 GB", | 20260507 05:35:57.907 | INFO | "pinned" : " 29 GB" | 20260507 05:35:57.907 | INFO | } | 20260507 05:35:57.907 | INFO | Perf report for the last 5.13651 seconds: | 20260507 05:35:57.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:35:57.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:57.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:57.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:57.907 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 05:35:57.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:57.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:57.907 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:35:57.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:57.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:35:57.907 | INFO | Uploaded 36GiB to GPU in 2.05882s (17.4858GiB/s) | 20260507 05:35:57.907 | INFO | Downloaded 34GiB to GPU in 1.94618s (17.4701GiB/s) | 20260507 05:36:03.045 | INFO | Analyzing chunk frames = [7498240,7498752) | 20260507 05:36:03.045 | INFO | MemInfo: { | 20260507 05:36:03.045 | INFO | "gpu" : " 17 GB", | 20260507 05:36:03.045 | INFO | "huge" : "", | 20260507 05:36:03.045 | INFO | "huge_shared" : " 27 GB", | 20260507 05:36:03.045 | INFO | "malloc" : " 14 GB", | 20260507 05:36:03.045 | INFO | "pinned" : " 29 GB" | 20260507 05:36:03.046 | INFO | } | 20260507 05:36:03.046 | INFO | Perf report for the last 5.13919 seconds: | 20260507 05:36:03.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:36:03.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:03.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:03.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:03.046 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 05:36:03.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:03.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:03.046 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:36:03.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:03.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:03.047 | INFO | Uploaded 36GiB to GPU in 2.0408s (17.6401GiB/s) | 20260507 05:36:03.047 | INFO | Downloaded 34GiB to GPU in 1.90101s (17.8853GiB/s) | 20260507 05:36:08.187 | INFO | Analyzing chunk frames = [7498752,7499264) | 20260507 05:36:08.187 | INFO | MemInfo: { | 20260507 05:36:08.187 | INFO | "gpu" : " 17 GB", | 20260507 05:36:08.187 | INFO | "huge" : "", | 20260507 05:36:08.187 | INFO | "huge_shared" : " 27 GB", | 20260507 05:36:08.187 | INFO | "malloc" : " 14 GB", | 20260507 05:36:08.187 | INFO | "pinned" : " 29 GB" | 20260507 05:36:08.187 | INFO | } | 20260507 05:36:08.188 | INFO | Perf report for the last 5.14141 seconds: | 20260507 05:36:08.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:36:08.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:08.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:08.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:08.188 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 05:36:08.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:08.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:08.188 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 05:36:08.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:08.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:08.188 | INFO | Uploaded 36GiB to GPU in 1.96502s (18.3204GiB/s) | 20260507 05:36:08.188 | INFO | Downloaded 34GiB to GPU in 1.97279s (17.2345GiB/s) | 20260507 05:36:13.331 | INFO | Analyzing chunk frames = [7499264,7499776) | 20260507 05:36:13.331 | INFO | MemInfo: { | 20260507 05:36:13.331 | INFO | "gpu" : " 17 GB", | 20260507 05:36:13.331 | INFO | "huge" : "", | 20260507 05:36:13.331 | INFO | "huge_shared" : " 27 GB", | 20260507 05:36:13.331 | INFO | "malloc" : " 14 GB", | 20260507 05:36:13.331 | INFO | "pinned" : " 29 GB" | 20260507 05:36:13.331 | INFO | } | 20260507 05:36:13.331 | INFO | Perf report for the last 5.14391 seconds: | 20260507 05:36:13.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:36:13.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:13.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:13.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:13.331 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:36:13.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:13.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:13.331 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:36:13.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:13.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:13.332 | INFO | Uploaded 36GiB to GPU in 1.97308s (18.2456GiB/s) | 20260507 05:36:13.332 | INFO | Downloaded 34GiB to GPU in 1.95098s (17.4271GiB/s) | 20260507 05:36:13.428 | INFO | Starting a round of Baz IO | 20260507 05:36:16.525 | INFO | Populating RTMetrics report | 20260507 05:36:16.526 | INFO | RTMetrics report populated | 20260507 05:36:16.526 | INFO | Opening tmp RTMetrics json file | 20260507 05:36:16.533 | INFO | Serializing RTMetrics to json | 20260507 05:36:16.542 | INFO | Serialization done, swapping tmp file now... | 20260507 05:36:16.543 | INFO | RTMetrics done | 20260507 05:36:16.610 | INFO | Finished a round of Baz IO: Wrote 1.691 GiB in 3.181 seconds | 20260507 05:36:16.610 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.69% / 0.00% / 67.31% | 20260507 05:36:16.610 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.140738% / 0% / 5.5631e-05% | 20260507 05:36:16.610 | INFO | Baz IO overall: | 20260507 05:36:16.610 | INFO | BytesWritten : 3801857769945 | 20260507 05:36:16.610 | INFO | NumEvents : 1823334044467 | 20260507 05:36:18.477 | INFO | Analyzing chunk frames = [7499776,7500288) | 20260507 05:36:18.477 | INFO | MemInfo: { | 20260507 05:36:18.477 | INFO | "gpu" : " 17 GB", | 20260507 05:36:18.477 | INFO | "huge" : "", | 20260507 05:36:18.477 | INFO | "huge_shared" : " 27 GB", | 20260507 05:36:18.477 | INFO | "malloc" : " 14 GB", | 20260507 05:36:18.477 | INFO | "pinned" : " 29 GB" | 20260507 05:36:18.477 | INFO | } | 20260507 05:36:18.478 | INFO | Perf report for the last 5.14606 seconds: | 20260507 05:36:18.478 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:36:18.478 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:18.478 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:18.478 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:18.478 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260507 05:36:18.478 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:18.478 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 05:36:18.478 | INFO | PRE_HQ : 59.0%, 1.8, 55.0ms, 0 | 20260507 05:36:18.478 | INFO | BAZWRITER : 61.4%, 3.2, 100.8ms, 0 | 20260507 05:36:18.478 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 05:36:18.478 | INFO | Uploaded 36GiB to GPU in 2.05067s (17.5552GiB/s) | 20260507 05:36:18.478 | INFO | Downloaded 38GiB to GPU in 2.17403s (17.4791GiB/s) | 20260507 05:36:23.609 | INFO | Analyzing chunk frames = [7500288,7500800) | 20260507 05:36:23.609 | INFO | MemInfo: { | 20260507 05:36:23.609 | INFO | "gpu" : " 17 GB", | 20260507 05:36:23.609 | INFO | "huge" : "", | 20260507 05:36:23.609 | INFO | "huge_shared" : " 27 GB", | 20260507 05:36:23.609 | INFO | "malloc" : " 14 GB", | 20260507 05:36:23.609 | INFO | "pinned" : " 29 GB" | 20260507 05:36:23.609 | INFO | } | 20260507 05:36:23.609 | INFO | Perf report for the last 5.13153 seconds: | 20260507 05:36:23.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:36:23.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:23.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:23.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:23.609 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 05:36:23.609 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:23.609 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:23.609 | INFO | PRE_HQ : 27.7%, 1.0, 14.3ms, 0 | 20260507 05:36:23.609 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:23.609 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:23.609 | INFO | Uploaded 36GiB to GPU in 2.02046s (17.8177GiB/s) | 20260507 05:36:23.609 | INFO | Downloaded 34GiB to GPU in 1.94485s (17.482GiB/s) | 20260507 05:36:28.741 | INFO | Analyzing chunk frames = [7500800,7501312) | 20260507 05:36:28.741 | INFO | MemInfo: { | 20260507 05:36:28.741 | INFO | "gpu" : " 17 GB", | 20260507 05:36:28.742 | INFO | "huge" : "", | 20260507 05:36:28.742 | INFO | "huge_shared" : " 27 GB", | 20260507 05:36:28.742 | INFO | "malloc" : " 14 GB", | 20260507 05:36:28.742 | INFO | "pinned" : " 29 GB" | 20260507 05:36:28.742 | INFO | } | 20260507 05:36:28.742 | INFO | Perf report for the last 5.13272 seconds: | 20260507 05:36:28.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:36:28.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:28.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:28.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:28.742 | INFO | ANALYSIS : 60.7%, 3.0, 93.8ms, 0 | 20260507 05:36:28.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:28.742 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:28.742 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 05:36:28.742 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:28.742 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:28.742 | INFO | Uploaded 36GiB to GPU in 2.05764s (17.4957GiB/s) | 20260507 05:36:28.742 | INFO | Downloaded 34GiB to GPU in 1.94876s (17.447GiB/s) | 20260507 05:36:33.882 | INFO | Analyzing chunk frames = [7501312,7501824) | 20260507 05:36:33.882 | INFO | MemInfo: { | 20260507 05:36:33.882 | INFO | "gpu" : " 17 GB", | 20260507 05:36:33.882 | INFO | "huge" : "", | 20260507 05:36:33.882 | INFO | "huge_shared" : " 27 GB", | 20260507 05:36:33.882 | INFO | "malloc" : " 14 GB", | 20260507 05:36:33.882 | INFO | "pinned" : " 29 GB" | 20260507 05:36:33.882 | INFO | } | 20260507 05:36:33.882 | INFO | Perf report for the last 5.14025 seconds: | 20260507 05:36:33.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:36:33.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:33.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:33.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:33.882 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 05:36:33.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:33.882 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:33.882 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:36:33.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:33.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:33.882 | INFO | Uploaded 36GiB to GPU in 1.99908s (18.0083GiB/s) | 20260507 05:36:33.883 | INFO | Downloaded 34GiB to GPU in 1.93032s (17.6136GiB/s) | 20260507 05:36:39.024 | INFO | Analyzing chunk frames = [7501824,7502336) | 20260507 05:36:39.024 | INFO | MemInfo: { | 20260507 05:36:39.024 | INFO | "gpu" : " 17 GB", | 20260507 05:36:39.024 | INFO | "huge" : "", | 20260507 05:36:39.024 | INFO | "huge_shared" : " 27 GB", | 20260507 05:36:39.024 | INFO | "malloc" : " 14 GB", | 20260507 05:36:39.024 | INFO | "pinned" : " 29 GB" | 20260507 05:36:39.024 | INFO | } | 20260507 05:36:39.024 | INFO | Perf report for the last 5.14209 seconds: | 20260507 05:36:39.024 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:36:39.024 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:39.024 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:39.024 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:39.024 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 05:36:39.024 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:39.024 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:39.024 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:36:39.024 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:39.024 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:39.024 | INFO | Uploaded 36GiB to GPU in 2.0333s (17.7052GiB/s) | 20260507 05:36:39.024 | INFO | Downloaded 34GiB to GPU in 1.9349s (17.5719GiB/s) | 20260507 05:36:44.169 | INFO | Analyzing chunk frames = [7502336,7502848) | 20260507 05:36:44.169 | INFO | MemInfo: { | 20260507 05:36:44.169 | INFO | "gpu" : " 17 GB", | 20260507 05:36:44.169 | INFO | "huge" : "", | 20260507 05:36:44.169 | INFO | "huge_shared" : " 27 GB", | 20260507 05:36:44.169 | INFO | "malloc" : " 14 GB", | 20260507 05:36:44.169 | INFO | "pinned" : " 29 GB" | 20260507 05:36:44.169 | INFO | } | 20260507 05:36:44.170 | INFO | Perf report for the last 5.14536 seconds: | 20260507 05:36:44.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:36:44.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:44.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:44.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:44.170 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 05:36:44.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:44.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:44.170 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:36:44.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:44.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:44.170 | INFO | Uploaded 36GiB to GPU in 2.04907s (17.5689GiB/s) | 20260507 05:36:44.170 | INFO | Downloaded 34GiB to GPU in 1.91383s (17.7654GiB/s) | 20260507 05:36:49.305 | INFO | Analyzing chunk frames = [7502848,7503360) | 20260507 05:36:49.305 | INFO | MemInfo: { | 20260507 05:36:49.305 | INFO | "gpu" : " 17 GB", | 20260507 05:36:49.305 | INFO | "huge" : "", | 20260507 05:36:49.305 | INFO | "huge_shared" : " 27 GB", | 20260507 05:36:49.305 | INFO | "malloc" : " 14 GB", | 20260507 05:36:49.305 | INFO | "pinned" : " 29 GB" | 20260507 05:36:49.305 | INFO | } | 20260507 05:36:49.305 | INFO | Perf report for the last 5.1359 seconds: | 20260507 05:36:49.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:36:49.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:49.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:49.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:49.306 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 05:36:49.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:49.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:49.306 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 05:36:49.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:49.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:49.306 | INFO | Uploaded 36GiB to GPU in 1.90565s (18.8912GiB/s) | 20260507 05:36:49.306 | INFO | Downloaded 34GiB to GPU in 1.97652s (17.202GiB/s) | 20260507 05:36:54.442 | INFO | Analyzing chunk frames = [7503360,7503872) | 20260507 05:36:54.442 | INFO | MemInfo: { | 20260507 05:36:54.442 | INFO | "gpu" : " 17 GB", | 20260507 05:36:54.442 | INFO | "huge" : "", | 20260507 05:36:54.442 | INFO | "huge_shared" : " 27 GB", | 20260507 05:36:54.442 | INFO | "malloc" : " 14 GB", | 20260507 05:36:54.442 | INFO | "pinned" : " 29 GB" | 20260507 05:36:54.442 | INFO | } | 20260507 05:36:54.442 | INFO | Perf report for the last 5.13617 seconds: | 20260507 05:36:54.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:36:54.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:54.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:54.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:54.442 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 05:36:54.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:54.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:54.442 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:36:54.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:54.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:54.442 | INFO | Uploaded 36GiB to GPU in 1.97559s (18.2224GiB/s) | 20260507 05:36:54.442 | INFO | Downloaded 34GiB to GPU in 1.9562s (17.3807GiB/s) | 20260507 05:36:54.442 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:36:54.442 | INFO | Total mem usage: 18 GiB | 20260507 05:36:54.442 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:36:54.443 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 05:36:54.443 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:36:54.443 | INFO | High Water Breakdown: | 20260507 05:36:54.443 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:36:54.443 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:36:54.443 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:36:54.443 | INFO | ------------------------------------------ | 20260507 05:36:54.443 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:36:54.443 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:36:54.443 | INFO | Total mem usage: 26.5 GiB | 20260507 05:36:54.443 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:36:54.443 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:36:54.443 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:36:54.443 | INFO | High Water Breakdown: | 20260507 05:36:54.443 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:36:54.443 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:36:54.443 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:36:54.443 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:36:54.443 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:36:54.443 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:36:54.443 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:36:54.443 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:36:54.443 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:36:54.443 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:36:54.443 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:36:54.443 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:36:54.443 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:36:54.443 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:36:54.443 | INFO | ------------------------------------------ | 20260507 05:36:54.443 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:36:54.443 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:36:54.443 | INFO | Total mem usage: 25.6 GiB | 20260507 05:36:54.443 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:36:54.443 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:36:54.443 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:36:54.443 | INFO | High Water Breakdown: | 20260507 05:36:54.443 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:36:54.443 | INFO | ------------------------------------------ | 20260507 05:36:54.443 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:36:54.443 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:36:54.443 | INFO | Total mem usage: 15.7 GiB | 20260507 05:36:54.443 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:36:54.443 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:36:54.443 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:36:54.443 | INFO | High Water Breakdown: | 20260507 05:36:54.443 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:36:54.443 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:36:54.443 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:36:54.443 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:36:54.443 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:36:54.443 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:36:54.443 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:36:54.443 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:36:54.443 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:36:54.443 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:36:54.443 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:36:54.443 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:36:54.443 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:36:54.443 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:36:54.443 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:36:54.443 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:36:54.443 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:36:54.443 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:36:54.443 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:36:54.443 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:36:54.443 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:36:54.443 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:36:54.443 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:36:54.443 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:36:54.443 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:36:54.444 | INFO | ------------------------------------------ | 20260507 05:36:54.444 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:36:54.444 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:36:54.444 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:36:54.444 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:36:54.444 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 1.07e+00 | 20260507 05:36:54.444 | INFO | PerfReport: 0.30%, AggregateStats -- 9.06e-02 / 1.03e-01, 9.34e-03 / 4.55e-03 | 20260507 05:36:54.444 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.50e+00, 5.12e-01 / 1.82e-01 | 20260507 05:36:54.444 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.50e+00, 1.69e-01 / 1.02e-01 | 20260507 05:36:54.444 | INFO | PerfReport: 16.23%, DME -- 3.99e+01 / 4.46e+01, 3.69e+00 / 1.23e+00 | 20260507 05:36:54.444 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.77e+01 / 1.98e+01, 1.26e+00 / 1.28e+00 | 20260507 05:36:54.444 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.13e+00 / 2.70e+00, 3.87e-01 / 2.13e-01 | 20260507 05:36:54.444 | INFO | PerfReport: 2.99%, Metrics -- 9.19e-01 / 1.23e+00, 2.47e-01 / 1.16e-01 | 20260507 05:36:54.444 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.08e-01 / 3.65e-01, 5.05e-02 / 2.35e-02 | 20260507 05:36:54.444 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:36:54.444 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:36:54.444 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:36:54.444 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 05:36:54.444 | INFO | PerfReport: 21.85%, Upload -- 2.10e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260507 05:36:54.444 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 05:36:54.444 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 05:36:54.444 | INFO | PerfReport: 1.59%, LockUpload -- 1.52e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 05:36:54.444 | INFO | PerfReport: 1.22%, LockDownload -- 1.17e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260507 05:36:54.444 | INFO | PerfReport: 22.04%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 05:36:54.444 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260507 05:36:54.539 | INFO | Starting a round of Baz IO | 20260507 05:36:57.643 | INFO | Populating RTMetrics report | 20260507 05:36:57.644 | INFO | RTMetrics report populated | 20260507 05:36:57.644 | INFO | Opening tmp RTMetrics json file | 20260507 05:36:57.645 | INFO | Serializing RTMetrics to json | 20260507 05:36:57.647 | INFO | Serialization done, swapping tmp file now... | 20260507 05:36:57.651 | INFO | RTMetrics done | 20260507 05:36:57.729 | INFO | Finished a round of Baz IO: Wrote 1.691 GiB in 3.19 seconds | 20260507 05:36:57.729 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.69% / 0.00% / 67.31% | 20260507 05:36:57.729 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.140186% / 0% / 5.5631e-05% | 20260507 05:36:57.729 | INFO | Baz IO overall: | 20260507 05:36:57.729 | INFO | BytesWritten : 3803673599410 | 20260507 05:36:57.729 | INFO | NumEvents : 1824186526213 | 20260507 05:36:59.587 | INFO | Analyzing chunk frames = [7503872,7504384) | 20260507 05:36:59.587 | INFO | MemInfo: { | 20260507 05:36:59.587 | INFO | "gpu" : " 17 GB", | 20260507 05:36:59.587 | INFO | "huge" : "", | 20260507 05:36:59.587 | INFO | "huge_shared" : " 27 GB", | 20260507 05:36:59.587 | INFO | "malloc" : " 14 GB", | 20260507 05:36:59.587 | INFO | "pinned" : " 29 GB" | 20260507 05:36:59.587 | INFO | } | 20260507 05:36:59.587 | INFO | Perf report for the last 5.145 seconds: | 20260507 05:36:59.587 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:36:59.587 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:59.587 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:59.587 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:59.587 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 05:36:59.587 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:36:59.587 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 05:36:59.587 | INFO | PRE_HQ : 58.9%, 1.8, 54.9ms, 0 | 20260507 05:36:59.587 | INFO | BAZWRITER : 61.6%, 3.2, 100.6ms, 0 | 20260507 05:36:59.587 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:36:59.587 | INFO | Uploaded 36GiB to GPU in 2.05242s (17.5403GiB/s) | 20260507 05:36:59.587 | INFO | Downloaded 38GiB to GPU in 2.18246s (17.4116GiB/s) | 20260507 05:37:04.717 | INFO | Analyzing chunk frames = [7504384,7504896) | 20260507 05:37:04.717 | INFO | MemInfo: { | 20260507 05:37:04.717 | INFO | "gpu" : " 17 GB", | 20260507 05:37:04.717 | INFO | "huge" : "", | 20260507 05:37:04.717 | INFO | "huge_shared" : " 27 GB", | 20260507 05:37:04.717 | INFO | "malloc" : " 14 GB", | 20260507 05:37:04.717 | INFO | "pinned" : " 29 GB" | 20260507 05:37:04.718 | INFO | } | 20260507 05:37:04.718 | INFO | Perf report for the last 5.13093 seconds: | 20260507 05:37:04.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:37:04.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:04.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:04.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:04.718 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 05:37:04.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:04.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:04.718 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:37:04.718 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:04.718 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:04.718 | INFO | Uploaded 36GiB to GPU in 2.00072s (17.9935GiB/s) | 20260507 05:37:04.718 | INFO | Downloaded 34GiB to GPU in 1.92953s (17.6209GiB/s) | 20260507 05:37:09.862 | INFO | Analyzing chunk frames = [7504896,7505408) | 20260507 05:37:09.862 | INFO | MemInfo: { | 20260507 05:37:09.862 | INFO | "gpu" : " 17 GB", | 20260507 05:37:09.862 | INFO | "huge" : "", | 20260507 05:37:09.862 | INFO | "huge_shared" : " 27 GB", | 20260507 05:37:09.862 | INFO | "malloc" : " 14 GB", | 20260507 05:37:09.862 | INFO | "pinned" : " 29 GB" | 20260507 05:37:09.862 | INFO | } | 20260507 05:37:09.862 | INFO | Perf report for the last 5.1446 seconds: | 20260507 05:37:09.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:37:09.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:09.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:09.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:09.862 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 05:37:09.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:09.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:09.862 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:37:09.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:09.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:09.862 | INFO | Uploaded 36GiB to GPU in 2.01619s (17.8555GiB/s) | 20260507 05:37:09.862 | INFO | Downloaded 34GiB to GPU in 1.94361s (17.4933GiB/s) | 20260507 05:37:14.997 | INFO | Analyzing chunk frames = [7505408,7505920) | 20260507 05:37:14.997 | INFO | MemInfo: { | 20260507 05:37:14.997 | INFO | "gpu" : " 17 GB", | 20260507 05:37:14.997 | INFO | "huge" : "", | 20260507 05:37:14.997 | INFO | "huge_shared" : " 27 GB", | 20260507 05:37:14.997 | INFO | "malloc" : " 14 GB", | 20260507 05:37:14.997 | INFO | "pinned" : " 29 GB" | 20260507 05:37:14.997 | INFO | } | 20260507 05:37:14.997 | INFO | Perf report for the last 5.13526 seconds: | 20260507 05:37:14.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:37:14.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:14.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:14.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:14.997 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 05:37:14.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:14.998 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:14.998 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:37:14.998 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:14.998 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:14.998 | INFO | Uploaded 36GiB to GPU in 2.01135s (17.8985GiB/s) | 20260507 05:37:14.998 | INFO | Downloaded 34GiB to GPU in 1.93702s (17.5528GiB/s) | 20260507 05:37:20.139 | INFO | Analyzing chunk frames = [7505920,7506432) | 20260507 05:37:20.139 | INFO | MemInfo: { | 20260507 05:37:20.139 | INFO | "gpu" : " 17 GB", | 20260507 05:37:20.139 | INFO | "huge" : "", | 20260507 05:37:20.140 | INFO | "huge_shared" : " 27 GB", | 20260507 05:37:20.140 | INFO | "malloc" : " 14 GB", | 20260507 05:37:20.140 | INFO | "pinned" : " 29 GB" | 20260507 05:37:20.140 | INFO | } | 20260507 05:37:20.140 | INFO | Perf report for the last 5.1422 seconds: | 20260507 05:37:20.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:37:20.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:20.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:20.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:20.140 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 05:37:20.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:20.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:20.140 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:37:20.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:20.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:20.140 | INFO | Uploaded 36GiB to GPU in 2.03965s (17.6501GiB/s) | 20260507 05:37:20.140 | INFO | Downloaded 34GiB to GPU in 1.93993s (17.5264GiB/s) | 20260507 05:37:25.283 | INFO | Analyzing chunk frames = [7506432,7506944) | 20260507 05:37:25.283 | INFO | MemInfo: { | 20260507 05:37:25.283 | INFO | "gpu" : " 17 GB", | 20260507 05:37:25.283 | INFO | "huge" : "", | 20260507 05:37:25.283 | INFO | "huge_shared" : " 27 GB", | 20260507 05:37:25.283 | INFO | "malloc" : " 14 GB", | 20260507 05:37:25.283 | INFO | "pinned" : " 29 GB" | 20260507 05:37:25.283 | INFO | } | 20260507 05:37:25.283 | INFO | Perf report for the last 5.14364 seconds: | 20260507 05:37:25.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:37:25.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:25.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:25.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:25.283 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 05:37:25.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:25.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:25.283 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 05:37:25.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:25.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:25.284 | INFO | Uploaded 36GiB to GPU in 2.0346s (17.6939GiB/s) | 20260507 05:37:25.284 | INFO | Downloaded 34GiB to GPU in 1.9019s (17.8769GiB/s) | 20260507 05:37:30.419 | INFO | Analyzing chunk frames = [7506944,7507456) | 20260507 05:37:30.419 | INFO | MemInfo: { | 20260507 05:37:30.419 | INFO | "gpu" : " 17 GB", | 20260507 05:37:30.419 | INFO | "huge" : "", | 20260507 05:37:30.420 | INFO | "huge_shared" : " 27 GB", | 20260507 05:37:30.420 | INFO | "malloc" : " 14 GB", | 20260507 05:37:30.420 | INFO | "pinned" : " 29 GB" | 20260507 05:37:30.420 | INFO | } | 20260507 05:37:30.420 | INFO | Perf report for the last 5.13628 seconds: | 20260507 05:37:30.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:37:30.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:30.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:30.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:30.420 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:37:30.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:30.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:30.420 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:37:30.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:30.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:30.420 | INFO | Uploaded 36GiB to GPU in 1.90748s (18.873GiB/s) | 20260507 05:37:30.420 | INFO | Downloaded 34GiB to GPU in 1.96599s (17.2941GiB/s) | 20260507 05:37:35.556 | INFO | Analyzing chunk frames = [7507456,7507968) | 20260507 05:37:35.556 | INFO | MemInfo: { | 20260507 05:37:35.556 | INFO | "gpu" : " 17 GB", | 20260507 05:37:35.556 | INFO | "huge" : "", | 20260507 05:37:35.557 | INFO | "huge_shared" : " 27 GB", | 20260507 05:37:35.557 | INFO | "malloc" : " 14 GB", | 20260507 05:37:35.557 | INFO | "pinned" : " 29 GB" | 20260507 05:37:35.557 | INFO | } | 20260507 05:37:35.557 | INFO | Perf report for the last 5.137 seconds: | 20260507 05:37:35.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:37:35.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:35.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:35.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:35.557 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:37:35.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:35.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:35.557 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 05:37:35.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:35.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:35.557 | INFO | Uploaded 36GiB to GPU in 1.98945s (18.0955GiB/s) | 20260507 05:37:35.557 | INFO | Downloaded 34GiB to GPU in 1.95247s (17.4138GiB/s) | 20260507 05:37:35.654 | INFO | Starting a round of Baz IO | 20260507 05:37:38.751 | INFO | Populating RTMetrics report | 20260507 05:37:38.752 | INFO | RTMetrics report populated | 20260507 05:37:38.752 | INFO | Opening tmp RTMetrics json file | 20260507 05:37:38.754 | INFO | Serializing RTMetrics to json | 20260507 05:37:38.770 | INFO | Serialization done, swapping tmp file now... | 20260507 05:37:38.770 | INFO | RTMetrics done | 20260507 05:37:38.839 | INFO | Finished a round of Baz IO: Wrote 1.69 GiB in 3.184 seconds | 20260507 05:37:38.839 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.69% / 0.00% / 67.31% | 20260507 05:37:38.839 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.140361% / 0% / 7.54992e-05% | 20260507 05:37:38.839 | INFO | Baz IO overall: | 20260507 05:37:38.839 | INFO | BytesWritten : 3805488141454 | 20260507 05:37:38.839 | INFO | NumEvents : 1825037947797 | 20260507 05:37:40.692 | INFO | Analyzing chunk frames = [7507968,7508480) | 20260507 05:37:40.692 | INFO | MemInfo: { | 20260507 05:37:40.692 | INFO | "gpu" : " 17 GB", | 20260507 05:37:40.692 | INFO | "huge" : "", | 20260507 05:37:40.692 | INFO | "huge_shared" : " 27 GB", | 20260507 05:37:40.692 | INFO | "malloc" : " 14 GB", | 20260507 05:37:40.692 | INFO | "pinned" : " 29 GB" | 20260507 05:37:40.692 | INFO | } | 20260507 05:37:40.692 | INFO | Perf report for the last 5.13533 seconds: | 20260507 05:37:40.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:37:40.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:40.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:40.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:40.692 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 05:37:40.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:40.692 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 05:37:40.692 | INFO | PRE_HQ : 59.2%, 1.8, 55.2ms, 0 | 20260507 05:37:40.692 | INFO | BAZWRITER : 61.5%, 3.2, 100.9ms, 0 | 20260507 05:37:40.692 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:37:40.692 | INFO | Uploaded 36GiB to GPU in 2.02796s (17.7518GiB/s) | 20260507 05:37:40.692 | INFO | Downloaded 38GiB to GPU in 2.17863s (17.4421GiB/s) | 20260507 05:37:45.845 | INFO | Analyzing chunk frames = [7508480,7508992) | 20260507 05:37:45.845 | INFO | MemInfo: { | 20260507 05:37:45.845 | INFO | "gpu" : " 17 GB", | 20260507 05:37:45.845 | INFO | "huge" : "", | 20260507 05:37:45.845 | INFO | "huge_shared" : " 27 GB", | 20260507 05:37:45.845 | INFO | "malloc" : " 14 GB", | 20260507 05:37:45.845 | INFO | "pinned" : " 29 GB" | 20260507 05:37:45.845 | INFO | } | 20260507 05:37:45.845 | INFO | Perf report for the last 5.15286 seconds: | 20260507 05:37:45.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:37:45.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:45.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:45.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:45.845 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260507 05:37:45.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:45.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:45.845 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:37:45.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:45.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:45.845 | INFO | Uploaded 36GiB to GPU in 2.04371s (17.6151GiB/s) | 20260507 05:37:45.845 | INFO | Downloaded 34GiB to GPU in 1.94081s (17.5184GiB/s) | 20260507 05:37:50.979 | INFO | Analyzing chunk frames = [7508992,7509504) | 20260507 05:37:50.979 | INFO | MemInfo: { | 20260507 05:37:50.979 | INFO | "gpu" : " 17 GB", | 20260507 05:37:50.979 | INFO | "huge" : "", | 20260507 05:37:50.979 | INFO | "huge_shared" : " 27 GB", | 20260507 05:37:50.979 | INFO | "malloc" : " 14 GB", | 20260507 05:37:50.979 | INFO | "pinned" : " 29 GB" | 20260507 05:37:50.979 | INFO | } | 20260507 05:37:50.979 | INFO | Perf report for the last 5.1339 seconds: | 20260507 05:37:50.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:37:50.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:50.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:50.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:50.979 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 05:37:50.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:50.979 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:50.979 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:37:50.979 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:50.979 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:50.979 | INFO | Uploaded 36GiB to GPU in 2.04792s (17.5788GiB/s) | 20260507 05:37:50.979 | INFO | Downloaded 34GiB to GPU in 1.9483s (17.4511GiB/s) | 20260507 05:37:56.120 | INFO | Analyzing chunk frames = [7509504,7510016) | 20260507 05:37:56.120 | INFO | MemInfo: { | 20260507 05:37:56.120 | INFO | "gpu" : " 17 GB", | 20260507 05:37:56.120 | INFO | "huge" : "", | 20260507 05:37:56.120 | INFO | "huge_shared" : " 27 GB", | 20260507 05:37:56.120 | INFO | "malloc" : " 14 GB", | 20260507 05:37:56.121 | INFO | "pinned" : " 29 GB" | 20260507 05:37:56.121 | INFO | } | 20260507 05:37:56.121 | INFO | Perf report for the last 5.14196 seconds: | 20260507 05:37:56.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:37:56.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:56.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:56.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:56.121 | INFO | ANALYSIS : 60.3%, 2.9, 93.2ms, 0 | 20260507 05:37:56.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:56.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:56.121 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:37:56.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:56.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:37:56.121 | INFO | Uploaded 36GiB to GPU in 2.03725s (17.6709GiB/s) | 20260507 05:37:56.121 | INFO | Downloaded 34GiB to GPU in 1.94815s (17.4525GiB/s) | 20260507 05:38:01.252 | INFO | Analyzing chunk frames = [7510016,7510528) | 20260507 05:38:01.252 | INFO | MemInfo: { | 20260507 05:38:01.252 | INFO | "gpu" : " 17 GB", | 20260507 05:38:01.252 | INFO | "huge" : "", | 20260507 05:38:01.252 | INFO | "huge_shared" : " 27 GB", | 20260507 05:38:01.252 | INFO | "malloc" : " 14 GB", | 20260507 05:38:01.252 | INFO | "pinned" : " 29 GB" | 20260507 05:38:01.252 | INFO | } | 20260507 05:38:01.252 | INFO | Perf report for the last 5.13158 seconds: | 20260507 05:38:01.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:38:01.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:01.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:01.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:01.252 | INFO | ANALYSIS : 60.3%, 3.0, 93.7ms, 0 | 20260507 05:38:01.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:01.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:01.252 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 05:38:01.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:01.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:01.253 | INFO | Uploaded 36GiB to GPU in 2.04683s (17.5881GiB/s) | 20260507 05:38:01.253 | INFO | Downloaded 34GiB to GPU in 1.91237s (17.779GiB/s) | 20260507 05:38:06.396 | INFO | Analyzing chunk frames = [7510528,7511040) | 20260507 05:38:06.396 | INFO | MemInfo: { | 20260507 05:38:06.396 | INFO | "gpu" : " 17 GB", | 20260507 05:38:06.396 | INFO | "huge" : "", | 20260507 05:38:06.396 | INFO | "huge_shared" : " 27 GB", | 20260507 05:38:06.396 | INFO | "malloc" : " 14 GB", | 20260507 05:38:06.396 | INFO | "pinned" : " 29 GB" | 20260507 05:38:06.396 | INFO | } | 20260507 05:38:06.396 | INFO | Perf report for the last 5.14369 seconds: | 20260507 05:38:06.396 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:38:06.396 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:06.396 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:06.396 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:06.396 | INFO | ANALYSIS : 60.4%, 3.0, 94.4ms, 0 | 20260507 05:38:06.396 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:06.396 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:06.396 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:38:06.396 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:06.396 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:06.396 | INFO | Uploaded 36GiB to GPU in 2.03213s (17.7154GiB/s) | 20260507 05:38:06.396 | INFO | Downloaded 34GiB to GPU in 1.92816s (17.6334GiB/s) | 20260507 05:38:11.533 | INFO | Analyzing chunk frames = [7511040,7511552) | 20260507 05:38:11.533 | INFO | MemInfo: { | 20260507 05:38:11.533 | INFO | "gpu" : " 17 GB", | 20260507 05:38:11.533 | INFO | "huge" : "", | 20260507 05:38:11.533 | INFO | "huge_shared" : " 27 GB", | 20260507 05:38:11.534 | INFO | "malloc" : " 14 GB", | 20260507 05:38:11.534 | INFO | "pinned" : " 29 GB" | 20260507 05:38:11.534 | INFO | } | 20260507 05:38:11.534 | INFO | Perf report for the last 5.1376 seconds: | 20260507 05:38:11.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:38:11.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:11.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:11.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:11.534 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 05:38:11.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:11.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:11.534 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 05:38:11.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:11.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:11.534 | INFO | Uploaded 36GiB to GPU in 1.95026s (18.459GiB/s) | 20260507 05:38:11.534 | INFO | Downloaded 34GiB to GPU in 1.98031s (17.1691GiB/s) | 20260507 05:38:16.676 | INFO | Analyzing chunk frames = [7511552,7512064) | 20260507 05:38:16.676 | INFO | MemInfo: { | 20260507 05:38:16.676 | INFO | "gpu" : " 17 GB", | 20260507 05:38:16.676 | INFO | "huge" : "", | 20260507 05:38:16.676 | INFO | "huge_shared" : " 27 GB", | 20260507 05:38:16.676 | INFO | "malloc" : " 14 GB", | 20260507 05:38:16.676 | INFO | "pinned" : " 29 GB" | 20260507 05:38:16.676 | INFO | } | 20260507 05:38:16.676 | INFO | Perf report for the last 5.14203 seconds: | 20260507 05:38:16.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:38:16.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:16.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:16.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:16.676 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:38:16.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:16.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:16.676 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 05:38:16.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:16.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:16.676 | INFO | Uploaded 36GiB to GPU in 1.98053s (18.177GiB/s) | 20260507 05:38:16.676 | INFO | Downloaded 34GiB to GPU in 1.95402s (17.4001GiB/s) | 20260507 05:38:16.676 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:38:16.676 | INFO | Total mem usage: 18 GiB | 20260507 05:38:16.676 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:38:16.676 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:38:16.676 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:38:16.676 | INFO | High Water Breakdown: | 20260507 05:38:16.676 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:38:16.676 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:38:16.676 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:38:16.676 | INFO | ------------------------------------------ | 20260507 05:38:16.676 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:38:16.676 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:38:16.676 | INFO | Total mem usage: 26.5 GiB | 20260507 05:38:16.676 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:38:16.676 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:38:16.676 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:38:16.676 | INFO | High Water Breakdown: | 20260507 05:38:16.676 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:38:16.677 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:38:16.677 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:38:16.677 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:38:16.677 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:38:16.677 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:38:16.677 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:38:16.677 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:38:16.677 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:38:16.677 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:38:16.677 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:38:16.677 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:38:16.677 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:38:16.677 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:38:16.677 | INFO | ------------------------------------------ | 20260507 05:38:16.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:38:16.677 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:38:16.677 | INFO | Total mem usage: 25.6 GiB | 20260507 05:38:16.677 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:38:16.677 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:38:16.677 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:38:16.677 | INFO | High Water Breakdown: | 20260507 05:38:16.677 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:38:16.677 | INFO | ------------------------------------------ | 20260507 05:38:16.677 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:38:16.677 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:38:16.677 | INFO | Total mem usage: 15.7 GiB | 20260507 05:38:16.677 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:38:16.677 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:38:16.677 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:38:16.677 | INFO | High Water Breakdown: | 20260507 05:38:16.677 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:38:16.677 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:38:16.677 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:38:16.677 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:38:16.677 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:38:16.677 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:38:16.677 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:38:16.677 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:38:16.677 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:38:16.677 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:38:16.677 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:38:16.677 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:38:16.677 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:38:16.677 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:38:16.677 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:38:16.677 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:38:16.677 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:38:16.677 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:38:16.677 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:38:16.677 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:38:16.677 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:38:16.677 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:38:16.677 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:38:16.677 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:38:16.677 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:38:16.678 | INFO | ------------------------------------------ | 20260507 05:38:16.678 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:38:16.678 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:38:16.678 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:38:16.678 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:38:16.678 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.33e+01 / 1.25e+00 | 20260507 05:38:16.678 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 9.96e-02, 9.55e-03 / 5.04e-03 | 20260507 05:38:16.678 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.43e+00, 5.10e-01 / 2.80e-01 | 20260507 05:38:16.678 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.48e+00, 1.67e-01 / 1.21e-01 | 20260507 05:38:16.678 | INFO | PerfReport: 16.22%, DME -- 3.99e+01 / 4.49e+01, 4.02e+00 / 1.85e+00 | 20260507 05:38:16.678 | INFO | PerfReport: 57.65%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.27e+00 / 1.60e+00 | 20260507 05:38:16.678 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.72e+00, 3.84e-01 / 1.76e-01 | 20260507 05:38:16.678 | INFO | PerfReport: 2.98%, Metrics -- 9.17e-01 / 1.22e+00, 2.44e-01 / 8.89e-02 | 20260507 05:38:16.678 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.09e-01 / 3.62e-01, 5.11e-02 / 1.70e-02 | 20260507 05:38:16.678 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:38:16.678 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:38:16.678 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:38:16.678 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 05:38:16.678 | INFO | PerfReport: 21.80%, Upload -- 2.10e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 05:38:16.678 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260507 05:38:16.678 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 05:38:16.678 | INFO | PerfReport: 1.58%, LockUpload -- 1.52e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 05:38:16.678 | INFO | PerfReport: 1.23%, LockDownload -- 1.19e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260507 05:38:16.678 | INFO | PerfReport: 22.08%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 05:38:16.678 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260507 05:38:16.772 | INFO | Starting a round of Baz IO | 20260507 05:38:19.866 | INFO | Populating RTMetrics report | 20260507 05:38:19.866 | INFO | RTMetrics report populated | 20260507 05:38:19.866 | INFO | Opening tmp RTMetrics json file | 20260507 05:38:19.884 | INFO | Serializing RTMetrics to json | 20260507 05:38:19.885 | INFO | Serialization done, swapping tmp file now... | 20260507 05:38:19.885 | INFO | RTMetrics done | 20260507 05:38:19.953 | INFO | Finished a round of Baz IO: Wrote 1.689 GiB in 3.18 seconds | 20260507 05:38:19.953 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.69% / 0.00% / 67.31% | 20260507 05:38:19.953 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.140278% / 3.97364e-06% / 7.15256e-05% | 20260507 05:38:19.953 | INFO | Baz IO overall: | 20260507 05:38:19.953 | INFO | BytesWritten : 3807302222519 | 20260507 05:38:19.953 | INFO | NumEvents : 1825889253751 | 20260507 05:38:21.816 | INFO | Analyzing chunk frames = [7512064,7512576) | 20260507 05:38:21.816 | INFO | MemInfo: { | 20260507 05:38:21.816 | INFO | "gpu" : " 17 GB", | 20260507 05:38:21.816 | INFO | "huge" : "", | 20260507 05:38:21.816 | INFO | "huge_shared" : " 27 GB", | 20260507 05:38:21.816 | INFO | "malloc" : " 14 GB", | 20260507 05:38:21.816 | INFO | "pinned" : " 29 GB" | 20260507 05:38:21.816 | INFO | } | 20260507 05:38:21.816 | INFO | Perf report for the last 5.14022 seconds: | 20260507 05:38:21.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:38:21.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:21.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:21.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:21.816 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 05:38:21.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:21.816 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 05:38:21.816 | INFO | PRE_HQ : 59.1%, 1.8, 55.5ms, 0 | 20260507 05:38:21.816 | INFO | BAZWRITER : 61.3%, 3.2, 100.6ms, 0 | 20260507 05:38:21.816 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 05:38:21.816 | INFO | Uploaded 36GiB to GPU in 2.0629s (17.4512GiB/s) | 20260507 05:38:21.816 | INFO | Downloaded 38GiB to GPU in 2.18185s (17.4164GiB/s) | 20260507 05:38:26.950 | INFO | Analyzing chunk frames = [7512576,7513088) | 20260507 05:38:26.950 | INFO | MemInfo: { | 20260507 05:38:26.950 | INFO | "gpu" : " 17 GB", | 20260507 05:38:26.950 | INFO | "huge" : "", | 20260507 05:38:26.950 | INFO | "huge_shared" : " 27 GB", | 20260507 05:38:26.950 | INFO | "malloc" : " 14 GB", | 20260507 05:38:26.950 | INFO | "pinned" : " 29 GB" | 20260507 05:38:26.950 | INFO | } | 20260507 05:38:26.950 | INFO | Perf report for the last 5.13382 seconds: | 20260507 05:38:26.950 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:38:26.950 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:26.950 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:26.950 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:26.950 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 05:38:26.950 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:26.950 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:26.950 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:38:26.950 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:26.950 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:26.950 | INFO | Uploaded 36GiB to GPU in 2.00476s (17.9573GiB/s) | 20260507 05:38:26.950 | INFO | Downloaded 34GiB to GPU in 1.93s (17.6166GiB/s) | 20260507 05:38:32.091 | INFO | Analyzing chunk frames = [7513088,7513600) | 20260507 05:38:32.091 | INFO | MemInfo: { | 20260507 05:38:32.091 | INFO | "gpu" : " 17 GB", | 20260507 05:38:32.091 | INFO | "huge" : "", | 20260507 05:38:32.091 | INFO | "huge_shared" : " 27 GB", | 20260507 05:38:32.091 | INFO | "malloc" : " 14 GB", | 20260507 05:38:32.092 | INFO | "pinned" : " 29 GB" | 20260507 05:38:32.092 | INFO | } | 20260507 05:38:32.092 | INFO | Perf report for the last 5.14189 seconds: | 20260507 05:38:32.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:38:32.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:32.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:32.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:32.092 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 05:38:32.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:32.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:32.092 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:38:32.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:32.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:32.092 | INFO | Uploaded 36GiB to GPU in 2.04442s (17.6089GiB/s) | 20260507 05:38:32.092 | INFO | Downloaded 34GiB to GPU in 1.95408s (17.3995GiB/s) | 20260507 05:38:37.231 | INFO | Analyzing chunk frames = [7513600,7514112) | 20260507 05:38:37.231 | INFO | MemInfo: { | 20260507 05:38:37.231 | INFO | "gpu" : " 17 GB", | 20260507 05:38:37.231 | INFO | "huge" : "", | 20260507 05:38:37.231 | INFO | "huge_shared" : " 27 GB", | 20260507 05:38:37.231 | INFO | "malloc" : " 14 GB", | 20260507 05:38:37.231 | INFO | "pinned" : " 29 GB" | 20260507 05:38:37.231 | INFO | } | 20260507 05:38:37.231 | INFO | Perf report for the last 5.13954 seconds: | 20260507 05:38:37.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:38:37.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:37.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:37.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:37.231 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:38:37.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:37.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:37.231 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:38:37.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:37.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:37.232 | INFO | Uploaded 36GiB to GPU in 2.01373s (17.8772GiB/s) | 20260507 05:38:37.232 | INFO | Downloaded 34GiB to GPU in 1.93154s (17.6026GiB/s) | 20260507 05:38:42.367 | INFO | Analyzing chunk frames = [7514112,7514624) | 20260507 05:38:42.367 | INFO | MemInfo: { | 20260507 05:38:42.367 | INFO | "gpu" : " 17 GB", | 20260507 05:38:42.367 | INFO | "huge" : "", | 20260507 05:38:42.367 | INFO | "huge_shared" : " 27 GB", | 20260507 05:38:42.367 | INFO | "malloc" : " 14 GB", | 20260507 05:38:42.367 | INFO | "pinned" : " 29 GB" | 20260507 05:38:42.367 | INFO | } | 20260507 05:38:42.367 | INFO | Perf report for the last 5.13591 seconds: | 20260507 05:38:42.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:38:42.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:42.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:42.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:42.367 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 05:38:42.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:42.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:42.367 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:38:42.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:42.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:42.367 | INFO | Uploaded 36GiB to GPU in 2.05497s (17.5185GiB/s) | 20260507 05:38:42.368 | INFO | Downloaded 34GiB to GPU in 1.94821s (17.4519GiB/s) | 20260507 05:38:47.502 | INFO | Analyzing chunk frames = [7514624,7515136) | 20260507 05:38:47.502 | INFO | MemInfo: { | 20260507 05:38:47.502 | INFO | "gpu" : " 17 GB", | 20260507 05:38:47.502 | INFO | "huge" : "", | 20260507 05:38:47.502 | INFO | "huge_shared" : " 27 GB", | 20260507 05:38:47.502 | INFO | "malloc" : " 14 GB", | 20260507 05:38:47.502 | INFO | "pinned" : " 29 GB" | 20260507 05:38:47.502 | INFO | } | 20260507 05:38:47.502 | INFO | Perf report for the last 5.13517 seconds: | 20260507 05:38:47.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:38:47.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:47.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:47.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:47.502 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 05:38:47.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:47.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:47.503 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:38:47.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:47.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:47.503 | INFO | Uploaded 36GiB to GPU in 2.03378s (17.701GiB/s) | 20260507 05:38:47.503 | INFO | Downloaded 34GiB to GPU in 1.90657s (17.8331GiB/s) | 20260507 05:38:52.648 | INFO | Analyzing chunk frames = [7515136,7515648) | 20260507 05:38:52.648 | INFO | MemInfo: { | 20260507 05:38:52.648 | INFO | "gpu" : " 17 GB", | 20260507 05:38:52.648 | INFO | "huge" : "", | 20260507 05:38:52.648 | INFO | "huge_shared" : " 27 GB", | 20260507 05:38:52.648 | INFO | "malloc" : " 14 GB", | 20260507 05:38:52.648 | INFO | "pinned" : " 29 GB" | 20260507 05:38:52.648 | INFO | } | 20260507 05:38:52.648 | INFO | Perf report for the last 5.14578 seconds: | 20260507 05:38:52.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:38:52.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:52.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:52.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:52.648 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 05:38:52.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:52.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:52.648 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:38:52.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:52.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:52.648 | INFO | Uploaded 36GiB to GPU in 1.97434s (18.2339GiB/s) | 20260507 05:38:52.649 | INFO | Downloaded 34GiB to GPU in 1.97617s (17.205GiB/s) | 20260507 05:38:57.796 | INFO | Analyzing chunk frames = [7515648,7516160) | 20260507 05:38:57.796 | INFO | MemInfo: { | 20260507 05:38:57.796 | INFO | "gpu" : " 17 GB", | 20260507 05:38:57.796 | INFO | "huge" : "", | 20260507 05:38:57.796 | INFO | "huge_shared" : " 27 GB", | 20260507 05:38:57.796 | INFO | "malloc" : " 14 GB", | 20260507 05:38:57.796 | INFO | "pinned" : " 29 GB" | 20260507 05:38:57.797 | INFO | } | 20260507 05:38:57.797 | INFO | Perf report for the last 5.14849 seconds: | 20260507 05:38:57.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:38:57.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:57.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:57.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:57.797 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 05:38:57.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:57.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:57.797 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:38:57.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:57.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:38:57.797 | INFO | Uploaded 36GiB to GPU in 1.95995s (18.3678GiB/s) | 20260507 05:38:57.797 | INFO | Downloaded 34GiB to GPU in 1.95589s (17.3834GiB/s) | 20260507 05:38:57.894 | INFO | Starting a round of Baz IO | 20260507 05:39:00.985 | INFO | Populating RTMetrics report | 20260507 05:39:00.985 | INFO | RTMetrics report populated | 20260507 05:39:00.985 | INFO | Opening tmp RTMetrics json file | 20260507 05:39:00.990 | INFO | Serializing RTMetrics to json | 20260507 05:39:01.003 | INFO | Serialization done, swapping tmp file now... | 20260507 05:39:01.003 | INFO | RTMetrics done | 20260507 05:39:01.072 | INFO | Finished a round of Baz IO: Wrote 1.689 GiB in 3.178 seconds | 20260507 05:39:01.072 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.69% / 0.00% / 67.31% | 20260507 05:39:01.072 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.139757% / 7.94729e-06% / 5.96046e-05% | 20260507 05:39:01.072 | INFO | Baz IO overall: | 20260507 05:39:01.072 | INFO | BytesWritten : 3809116288012 | 20260507 05:39:01.072 | INFO | NumEvents : 1826740770541 | 20260507 05:39:02.936 | INFO | Analyzing chunk frames = [7516160,7516672) | 20260507 05:39:02.936 | INFO | MemInfo: { | 20260507 05:39:02.936 | INFO | "gpu" : " 17 GB", | 20260507 05:39:02.936 | INFO | "huge" : "", | 20260507 05:39:02.936 | INFO | "huge_shared" : " 27 GB", | 20260507 05:39:02.936 | INFO | "malloc" : " 14 GB", | 20260507 05:39:02.936 | INFO | "pinned" : " 29 GB" | 20260507 05:39:02.936 | INFO | } | 20260507 05:39:02.936 | INFO | Perf report for the last 5.13986 seconds: | 20260507 05:39:02.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:39:02.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:02.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:02.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:02.936 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 05:39:02.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:02.936 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 05:39:02.936 | INFO | PRE_HQ : 58.9%, 1.8, 55.0ms, 0 | 20260507 05:39:02.936 | INFO | BAZWRITER : 61.4%, 3.2, 99.9ms, 0 | 20260507 05:39:02.937 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:39:02.937 | INFO | Uploaded 36GiB to GPU in 2.05209s (17.543GiB/s) | 20260507 05:39:02.937 | INFO | Downloaded 38GiB to GPU in 2.18514s (17.3902GiB/s) | 20260507 05:39:08.063 | INFO | Analyzing chunk frames = [7516672,7517184) | 20260507 05:39:08.063 | INFO | MemInfo: { | 20260507 05:39:08.063 | INFO | "gpu" : " 17 GB", | 20260507 05:39:08.063 | INFO | "huge" : "", | 20260507 05:39:08.063 | INFO | "huge_shared" : " 27 GB", | 20260507 05:39:08.063 | INFO | "malloc" : " 14 GB", | 20260507 05:39:08.063 | INFO | "pinned" : " 29 GB" | 20260507 05:39:08.063 | INFO | } | 20260507 05:39:08.063 | INFO | Perf report for the last 5.12662 seconds: | 20260507 05:39:08.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:39:08.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:08.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:08.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:08.063 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260507 05:39:08.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:08.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:08.063 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:39:08.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:08.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:08.063 | INFO | Uploaded 36GiB to GPU in 2.0093s (17.9167GiB/s) | 20260507 05:39:08.063 | INFO | Downloaded 34GiB to GPU in 1.94554s (17.4759GiB/s) | 20260507 05:39:13.206 | INFO | Analyzing chunk frames = [7517184,7517696) | 20260507 05:39:13.206 | INFO | MemInfo: { | 20260507 05:39:13.206 | INFO | "gpu" : " 17 GB", | 20260507 05:39:13.206 | INFO | "huge" : "", | 20260507 05:39:13.206 | INFO | "huge_shared" : " 27 GB", | 20260507 05:39:13.206 | INFO | "malloc" : " 14 GB", | 20260507 05:39:13.206 | INFO | "pinned" : " 29 GB" | 20260507 05:39:13.206 | INFO | } | 20260507 05:39:13.206 | INFO | Perf report for the last 5.14306 seconds: | 20260507 05:39:13.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:39:13.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:13.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:13.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:13.206 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 05:39:13.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:13.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:13.206 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:39:13.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:13.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:13.206 | INFO | Uploaded 36GiB to GPU in 1.98623s (18.1248GiB/s) | 20260507 05:39:13.207 | INFO | Downloaded 34GiB to GPU in 1.94379s (17.4916GiB/s) | 20260507 05:39:18.342 | INFO | Analyzing chunk frames = [7517696,7518208) | 20260507 05:39:18.342 | INFO | MemInfo: { | 20260507 05:39:18.342 | INFO | "gpu" : " 17 GB", | 20260507 05:39:18.342 | INFO | "huge" : "", | 20260507 05:39:18.342 | INFO | "huge_shared" : " 27 GB", | 20260507 05:39:18.342 | INFO | "malloc" : " 14 GB", | 20260507 05:39:18.342 | INFO | "pinned" : " 29 GB" | 20260507 05:39:18.342 | INFO | } | 20260507 05:39:18.342 | INFO | Perf report for the last 5.13589 seconds: | 20260507 05:39:18.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:39:18.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:18.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:18.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:18.342 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 05:39:18.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:18.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:18.342 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:39:18.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:18.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:18.342 | INFO | Uploaded 36GiB to GPU in 1.98442s (18.1413GiB/s) | 20260507 05:39:18.342 | INFO | Downloaded 34GiB to GPU in 1.9353s (17.5683GiB/s) | 20260507 05:39:23.488 | INFO | Analyzing chunk frames = [7518208,7518720) | 20260507 05:39:23.488 | INFO | MemInfo: { | 20260507 05:39:23.488 | INFO | "gpu" : " 17 GB", | 20260507 05:39:23.488 | INFO | "huge" : "", | 20260507 05:39:23.488 | INFO | "huge_shared" : " 27 GB", | 20260507 05:39:23.488 | INFO | "malloc" : " 14 GB", | 20260507 05:39:23.488 | INFO | "pinned" : " 29 GB" | 20260507 05:39:23.488 | INFO | } | 20260507 05:39:23.488 | INFO | Perf report for the last 5.14564 seconds: | 20260507 05:39:23.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:39:23.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:23.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:23.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:23.488 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 05:39:23.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:23.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:23.488 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 05:39:23.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:23.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:23.488 | INFO | Uploaded 36GiB to GPU in 2.01194s (17.8932GiB/s) | 20260507 05:39:23.489 | INFO | Downloaded 34GiB to GPU in 1.94252s (17.503GiB/s) | 20260507 05:39:28.628 | INFO | Analyzing chunk frames = [7518720,7519232) | 20260507 05:39:28.628 | INFO | MemInfo: { | 20260507 05:39:28.628 | INFO | "gpu" : " 17 GB", | 20260507 05:39:28.628 | INFO | "huge" : "", | 20260507 05:39:28.628 | INFO | "huge_shared" : " 27 GB", | 20260507 05:39:28.628 | INFO | "malloc" : " 14 GB", | 20260507 05:39:28.628 | INFO | "pinned" : " 29 GB" | 20260507 05:39:28.628 | INFO | } | 20260507 05:39:28.628 | INFO | Perf report for the last 5.14061 seconds: | 20260507 05:39:28.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:39:28.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:28.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:28.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:28.628 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 05:39:28.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:28.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:28.628 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 05:39:28.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:28.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:28.628 | INFO | Uploaded 36GiB to GPU in 2.06491s (17.4342GiB/s) | 20260507 05:39:28.629 | INFO | Downloaded 34GiB to GPU in 1.93544s (17.567GiB/s) | 20260507 05:39:33.759 | INFO | Analyzing chunk frames = [7519232,7519744) | 20260507 05:39:33.759 | INFO | MemInfo: { | 20260507 05:39:33.759 | INFO | "gpu" : " 17 GB", | 20260507 05:39:33.759 | INFO | "huge" : "", | 20260507 05:39:33.759 | INFO | "huge_shared" : " 27 GB", | 20260507 05:39:33.759 | INFO | "malloc" : " 14 GB", | 20260507 05:39:33.759 | INFO | "pinned" : " 29 GB" | 20260507 05:39:33.759 | INFO | } | 20260507 05:39:33.760 | INFO | Perf report for the last 5.13125 seconds: | 20260507 05:39:33.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:39:33.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:33.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:33.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:33.760 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260507 05:39:33.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:33.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:33.760 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:39:33.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:33.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:33.760 | INFO | Uploaded 36GiB to GPU in 1.92391s (18.7119GiB/s) | 20260507 05:39:33.760 | INFO | Downloaded 34GiB to GPU in 1.97405s (17.2235GiB/s) | 20260507 05:39:38.906 | INFO | Analyzing chunk frames = [7519744,7520256) | 20260507 05:39:38.906 | INFO | MemInfo: { | 20260507 05:39:38.906 | INFO | "gpu" : " 17 GB", | 20260507 05:39:38.906 | INFO | "huge" : "", | 20260507 05:39:38.906 | INFO | "huge_shared" : " 27 GB", | 20260507 05:39:38.906 | INFO | "malloc" : " 14 GB", | 20260507 05:39:38.906 | INFO | "pinned" : " 29 GB" | 20260507 05:39:38.906 | INFO | } | 20260507 05:39:38.906 | INFO | Perf report for the last 5.14662 seconds: | 20260507 05:39:38.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:39:38.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:38.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:38.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:38.906 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 05:39:38.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:38.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:38.906 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:39:38.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:38.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:38.906 | INFO | Uploaded 36GiB to GPU in 1.97s (18.2741GiB/s) | 20260507 05:39:38.906 | INFO | Downloaded 34GiB to GPU in 1.96103s (17.3379GiB/s) | 20260507 05:39:38.906 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:39:38.906 | INFO | Total mem usage: 18 GiB | 20260507 05:39:38.906 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:39:38.906 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:39:38.907 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:39:38.907 | INFO | High Water Breakdown: | 20260507 05:39:38.907 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:39:38.907 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:39:38.907 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:39:38.907 | INFO | ------------------------------------------ | 20260507 05:39:38.907 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:39:38.907 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:39:38.907 | INFO | Total mem usage: 26.5 GiB | 20260507 05:39:38.907 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:39:38.907 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:39:38.907 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:39:38.907 | INFO | High Water Breakdown: | 20260507 05:39:38.907 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:39:38.907 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:39:38.907 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:39:38.907 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:39:38.907 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:39:38.907 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:39:38.907 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:39:38.907 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:39:38.907 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:39:38.907 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:39:38.907 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:39:38.907 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:39:38.907 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:39:38.907 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:39:38.907 | INFO | ------------------------------------------ | 20260507 05:39:38.907 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:39:38.907 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:39:38.907 | INFO | Total mem usage: 25.6 GiB | 20260507 05:39:38.907 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:39:38.907 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:39:38.907 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:39:38.907 | INFO | High Water Breakdown: | 20260507 05:39:38.907 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:39:38.907 | INFO | ------------------------------------------ | 20260507 05:39:38.907 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:39:38.907 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:39:38.907 | INFO | Total mem usage: 15.7 GiB | 20260507 05:39:38.907 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:39:38.907 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:39:38.907 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:39:38.907 | INFO | High Water Breakdown: | 20260507 05:39:38.907 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:39:38.907 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:39:38.907 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:39:38.907 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:39:38.907 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:39:38.907 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:39:38.907 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:39:38.907 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:39:38.907 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:39:38.907 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:39:38.907 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:39:38.907 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:39:38.907 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:39:38.907 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:39:38.907 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:39:38.907 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:39:38.907 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:39:38.907 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:39:38.907 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:39:38.907 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:39:38.907 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:39:38.907 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:39:38.907 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:39:38.907 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:39:38.907 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:39:38.907 | INFO | ------------------------------------------ | 20260507 05:39:38.907 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:39:38.907 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:39:38.907 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:39:38.907 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:39:38.907 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.33e+01 / 1.42e+00 | 20260507 05:39:38.907 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.01e-01, 9.65e-03 / 5.05e-03 | 20260507 05:39:38.907 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.43e+00, 5.21e-01 / 3.09e-01 | 20260507 05:39:38.907 | INFO | PerfReport: 3.24%, Histogram -- 9.94e-01 / 1.49e+00, 1.71e-01 / 1.17e-01 | 20260507 05:39:38.907 | INFO | PerfReport: 16.17%, DME -- 3.97e+01 / 4.42e+01, 3.77e+00 / 1.27e+00 | 20260507 05:39:38.907 | INFO | PerfReport: 57.68%, FrameLabeling -- 1.77e+01 / 2.00e+01, 1.28e+00 / 1.40e+00 | 20260507 05:39:38.907 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.13e+00 / 2.71e+00, 3.88e-01 / 2.30e-01 | 20260507 05:39:38.907 | INFO | PerfReport: 2.98%, Metrics -- 9.16e-01 / 1.24e+00, 2.44e-01 / 1.10e-01 | 20260507 05:39:38.907 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.07e-01 / 3.60e-01, 5.01e-02 / 1.94e-02 | 20260507 05:39:38.907 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:39:38.907 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:39:38.907 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:39:38.907 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 05:39:38.907 | INFO | PerfReport: 21.79%, Upload -- 2.09e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 05:39:38.907 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260507 05:39:38.907 | INFO | PerfReport: 31.96%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 05:39:38.907 | INFO | PerfReport: 1.45%, LockUpload -- 1.39e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 05:39:38.907 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.10e+00 / 0.00e+00 | 20260507 05:39:38.907 | INFO | PerfReport: 22.05%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 05:39:38.907 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.40e-02 / 0.00e+00 | 20260507 05:39:39.004 | INFO | Starting a round of Baz IO | 20260507 05:39:42.097 | INFO | Populating RTMetrics report | 20260507 05:39:42.097 | INFO | RTMetrics report populated | 20260507 05:39:42.097 | INFO | Opening tmp RTMetrics json file | 20260507 05:39:42.114 | INFO | Serializing RTMetrics to json | 20260507 05:39:42.115 | INFO | Serialization done, swapping tmp file now... | 20260507 05:39:42.115 | INFO | RTMetrics done | 20260507 05:39:42.181 | INFO | Finished a round of Baz IO: Wrote 1.689 GiB in 3.177 seconds | 20260507 05:39:42.181 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.69% / 0.00% / 67.31% | 20260507 05:39:42.181 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.138462% / 3.97364e-06% / 4.37101e-05% | 20260507 05:39:42.181 | INFO | Baz IO overall: | 20260507 05:39:42.181 | INFO | BytesWritten : 3810929395221 | 20260507 05:39:42.181 | INFO | NumEvents : 1827591419115 | 20260507 05:39:44.042 | INFO | Analyzing chunk frames = [7520256,7520768) | 20260507 05:39:44.043 | INFO | MemInfo: { | 20260507 05:39:44.043 | INFO | "gpu" : " 17 GB", | 20260507 05:39:44.043 | INFO | "huge" : "", | 20260507 05:39:44.043 | INFO | "huge_shared" : " 27 GB", | 20260507 05:39:44.043 | INFO | "malloc" : " 14 GB", | 20260507 05:39:44.043 | INFO | "pinned" : " 29 GB" | 20260507 05:39:44.043 | INFO | } | 20260507 05:39:44.043 | INFO | Perf report for the last 5.13643 seconds: | 20260507 05:39:44.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:39:44.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:44.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:44.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:44.043 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 05:39:44.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:44.043 | INFO | RT_METRICS : 41.7%, 1.0, 21.8ms, 0 | 20260507 05:39:44.043 | INFO | PRE_HQ : 58.9%, 1.8, 54.9ms, 0 | 20260507 05:39:44.043 | INFO | BAZWRITER : 61.5%, 3.2, 99.8ms, 0 | 20260507 05:39:44.043 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 05:39:44.043 | INFO | Uploaded 36GiB to GPU in 2.03458s (17.6941GiB/s) | 20260507 05:39:44.043 | INFO | Downloaded 38GiB to GPU in 2.17207s (17.4948GiB/s) | 20260507 05:39:49.180 | INFO | Analyzing chunk frames = [7520768,7521280) | 20260507 05:39:49.180 | INFO | MemInfo: { | 20260507 05:39:49.180 | INFO | "gpu" : " 17 GB", | 20260507 05:39:49.180 | INFO | "huge" : "", | 20260507 05:39:49.180 | INFO | "huge_shared" : " 27 GB", | 20260507 05:39:49.180 | INFO | "malloc" : " 14 GB", | 20260507 05:39:49.180 | INFO | "pinned" : " 29 GB" | 20260507 05:39:49.180 | INFO | } | 20260507 05:39:49.180 | INFO | Perf report for the last 5.13776 seconds: | 20260507 05:39:49.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:39:49.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:49.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:49.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:49.180 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:39:49.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:49.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:49.180 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:39:49.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:49.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:49.181 | INFO | Uploaded 36GiB to GPU in 1.99585s (18.0375GiB/s) | 20260507 05:39:49.181 | INFO | Downloaded 34GiB to GPU in 1.92915s (17.6244GiB/s) | 20260507 05:39:54.317 | INFO | Analyzing chunk frames = [7521280,7521792) | 20260507 05:39:54.317 | INFO | MemInfo: { | 20260507 05:39:54.317 | INFO | "gpu" : " 17 GB", | 20260507 05:39:54.317 | INFO | "huge" : "", | 20260507 05:39:54.317 | INFO | "huge_shared" : " 27 GB", | 20260507 05:39:54.317 | INFO | "malloc" : " 14 GB", | 20260507 05:39:54.317 | INFO | "pinned" : " 29 GB" | 20260507 05:39:54.317 | INFO | } | 20260507 05:39:54.317 | INFO | Perf report for the last 5.13678 seconds: | 20260507 05:39:54.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:39:54.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:54.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:54.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:54.317 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 05:39:54.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:54.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:54.317 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:39:54.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:54.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:54.317 | INFO | Uploaded 36GiB to GPU in 2.05032s (17.5582GiB/s) | 20260507 05:39:54.317 | INFO | Downloaded 34GiB to GPU in 1.94896s (17.4452GiB/s) | 20260507 05:39:59.458 | INFO | Analyzing chunk frames = [7521792,7522304) | 20260507 05:39:59.458 | INFO | MemInfo: { | 20260507 05:39:59.458 | INFO | "gpu" : " 17 GB", | 20260507 05:39:59.458 | INFO | "huge" : "", | 20260507 05:39:59.458 | INFO | "huge_shared" : " 27 GB", | 20260507 05:39:59.458 | INFO | "malloc" : " 14 GB", | 20260507 05:39:59.458 | INFO | "pinned" : " 29 GB" | 20260507 05:39:59.458 | INFO | } | 20260507 05:39:59.459 | INFO | Perf report for the last 5.14128 seconds: | 20260507 05:39:59.459 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:39:59.459 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:59.459 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:59.459 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:59.459 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 05:39:59.459 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:59.459 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:59.459 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:39:59.459 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:59.459 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:39:59.459 | INFO | Uploaded 36GiB to GPU in 1.9939s (18.055GiB/s) | 20260507 05:39:59.459 | INFO | Downloaded 34GiB to GPU in 1.91719s (17.7343GiB/s) | 20260507 05:40:04.603 | INFO | Analyzing chunk frames = [7522304,7522816) | 20260507 05:40:04.603 | INFO | MemInfo: { | 20260507 05:40:04.603 | INFO | "gpu" : " 17 GB", | 20260507 05:40:04.603 | INFO | "huge" : "", | 20260507 05:40:04.603 | INFO | "huge_shared" : " 27 GB", | 20260507 05:40:04.603 | INFO | "malloc" : " 14 GB", | 20260507 05:40:04.603 | INFO | "pinned" : " 29 GB" | 20260507 05:40:04.603 | INFO | } | 20260507 05:40:04.603 | INFO | Perf report for the last 5.14434 seconds: | 20260507 05:40:04.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:40:04.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:04.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:04.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:04.603 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 05:40:04.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:04.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:04.603 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:40:04.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:04.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:04.603 | INFO | Uploaded 36GiB to GPU in 2.05369s (17.5294GiB/s) | 20260507 05:40:04.603 | INFO | Downloaded 34GiB to GPU in 1.94268s (17.5016GiB/s) | 20260507 05:40:09.744 | INFO | Analyzing chunk frames = [7522816,7523328) | 20260507 05:40:09.744 | INFO | MemInfo: { | 20260507 05:40:09.744 | INFO | "gpu" : " 17 GB", | 20260507 05:40:09.744 | INFO | "huge" : "", | 20260507 05:40:09.744 | INFO | "huge_shared" : " 27 GB", | 20260507 05:40:09.744 | INFO | "malloc" : " 14 GB", | 20260507 05:40:09.744 | INFO | "pinned" : " 29 GB" | 20260507 05:40:09.744 | INFO | } | 20260507 05:40:09.744 | INFO | Perf report for the last 5.14115 seconds: | 20260507 05:40:09.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:40:09.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:09.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:09.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:09.744 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 05:40:09.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:09.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:09.744 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 05:40:09.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:09.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:09.744 | INFO | Uploaded 36GiB to GPU in 2.05746s (17.4973GiB/s) | 20260507 05:40:09.744 | INFO | Downloaded 34GiB to GPU in 1.91805s (17.7264GiB/s) | 20260507 05:40:14.880 | INFO | Analyzing chunk frames = [7523328,7523840) | 20260507 05:40:14.880 | INFO | MemInfo: { | 20260507 05:40:14.880 | INFO | "gpu" : " 17 GB", | 20260507 05:40:14.880 | INFO | "huge" : "", | 20260507 05:40:14.880 | INFO | "huge_shared" : " 27 GB", | 20260507 05:40:14.880 | INFO | "malloc" : " 14 GB", | 20260507 05:40:14.880 | INFO | "pinned" : " 29 GB" | 20260507 05:40:14.880 | INFO | } | 20260507 05:40:14.880 | INFO | Perf report for the last 5.13612 seconds: | 20260507 05:40:14.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:40:14.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:14.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:14.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:14.880 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260507 05:40:14.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:14.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:14.880 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 05:40:14.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:14.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:14.880 | INFO | Uploaded 36GiB to GPU in 1.89241s (19.0234GiB/s) | 20260507 05:40:14.880 | INFO | Downloaded 34GiB to GPU in 1.96669s (17.2879GiB/s) | 20260507 05:40:20.011 | INFO | Analyzing chunk frames = [7523840,7524352) | 20260507 05:40:20.011 | INFO | MemInfo: { | 20260507 05:40:20.011 | INFO | "gpu" : " 17 GB", | 20260507 05:40:20.011 | INFO | "huge" : "", | 20260507 05:40:20.011 | INFO | "huge_shared" : " 27 GB", | 20260507 05:40:20.011 | INFO | "malloc" : " 14 GB", | 20260507 05:40:20.011 | INFO | "pinned" : " 29 GB" | 20260507 05:40:20.011 | INFO | } | 20260507 05:40:20.011 | INFO | Perf report for the last 5.13051 seconds: | 20260507 05:40:20.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:40:20.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:20.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:20.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:20.011 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 05:40:20.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:20.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:20.011 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:40:20.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:20.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:20.011 | INFO | Uploaded 36GiB to GPU in 1.9797s (18.1846GiB/s) | 20260507 05:40:20.011 | INFO | Downloaded 34GiB to GPU in 1.95957s (17.3507GiB/s) | 20260507 05:40:20.108 | INFO | Starting a round of Baz IO | 20260507 05:40:23.206 | INFO | Populating RTMetrics report | 20260507 05:40:23.206 | INFO | RTMetrics report populated | 20260507 05:40:23.206 | INFO | Opening tmp RTMetrics json file | 20260507 05:40:23.207 | INFO | Serializing RTMetrics to json | 20260507 05:40:23.208 | INFO | Serialization done, swapping tmp file now... | 20260507 05:40:23.211 | INFO | RTMetrics done | 20260507 05:40:23.291 | INFO | Finished a round of Baz IO: Wrote 1.689 GiB in 3.182 seconds | 20260507 05:40:23.291 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.69% / 0.00% / 67.31% | 20260507 05:40:23.291 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.139117% / 0% / 6.75519e-05% | 20260507 05:40:23.291 | INFO | Baz IO overall: | 20260507 05:40:23.291 | INFO | BytesWritten : 3812742442503 | 20260507 05:40:23.291 | INFO | NumEvents : 1828441896725 | 20260507 05:40:25.161 | INFO | Analyzing chunk frames = [7524352,7524864) | 20260507 05:40:25.161 | INFO | MemInfo: { | 20260507 05:40:25.161 | INFO | "gpu" : " 17 GB", | 20260507 05:40:25.161 | INFO | "huge" : "", | 20260507 05:40:25.161 | INFO | "huge_shared" : " 27 GB", | 20260507 05:40:25.161 | INFO | "malloc" : " 14 GB", | 20260507 05:40:25.161 | INFO | "pinned" : " 29 GB" | 20260507 05:40:25.161 | INFO | } | 20260507 05:40:25.161 | INFO | Perf report for the last 5.15069 seconds: | 20260507 05:40:25.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:40:25.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:25.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:25.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:25.161 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260507 05:40:25.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:25.161 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 05:40:25.161 | INFO | PRE_HQ : 58.9%, 1.8, 54.9ms, 0 | 20260507 05:40:25.161 | INFO | BAZWRITER : 61.4%, 3.2, 100.2ms, 0 | 20260507 05:40:25.161 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:40:25.162 | INFO | Uploaded 36GiB to GPU in 2.03556s (17.6856GiB/s) | 20260507 05:40:25.162 | INFO | Downloaded 38GiB to GPU in 2.18137s (17.4202GiB/s) | 20260507 05:40:30.300 | INFO | Analyzing chunk frames = [7524864,7525376) | 20260507 05:40:30.300 | INFO | MemInfo: { | 20260507 05:40:30.300 | INFO | "gpu" : " 17 GB", | 20260507 05:40:30.300 | INFO | "huge" : "", | 20260507 05:40:30.300 | INFO | "huge_shared" : " 27 GB", | 20260507 05:40:30.300 | INFO | "malloc" : " 14 GB", | 20260507 05:40:30.300 | INFO | "pinned" : " 29 GB" | 20260507 05:40:30.300 | INFO | } | 20260507 05:40:30.300 | INFO | Perf report for the last 5.13906 seconds: | 20260507 05:40:30.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:40:30.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:30.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:30.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:30.301 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:40:30.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:30.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:30.301 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:40:30.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:30.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:30.301 | INFO | Uploaded 36GiB to GPU in 1.99029s (18.0879GiB/s) | 20260507 05:40:30.301 | INFO | Downloaded 34GiB to GPU in 1.93417s (17.5786GiB/s) | 20260507 05:40:35.442 | INFO | Analyzing chunk frames = [7525376,7525888) | 20260507 05:40:35.442 | INFO | MemInfo: { | 20260507 05:40:35.442 | INFO | "gpu" : " 17 GB", | 20260507 05:40:35.442 | INFO | "huge" : "", | 20260507 05:40:35.442 | INFO | "huge_shared" : " 27 GB", | 20260507 05:40:35.442 | INFO | "malloc" : " 14 GB", | 20260507 05:40:35.442 | INFO | "pinned" : " 29 GB" | 20260507 05:40:35.442 | INFO | } | 20260507 05:40:35.442 | INFO | Perf report for the last 5.14203 seconds: | 20260507 05:40:35.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:40:35.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:35.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:35.443 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:35.443 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 05:40:35.443 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:35.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:35.443 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:40:35.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:35.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:35.443 | INFO | Uploaded 36GiB to GPU in 2.04362s (17.6158GiB/s) | 20260507 05:40:35.443 | INFO | Downloaded 34GiB to GPU in 1.95156s (17.4219GiB/s) | 20260507 05:40:40.578 | INFO | Analyzing chunk frames = [7525888,7526400) | 20260507 05:40:40.578 | INFO | MemInfo: { | 20260507 05:40:40.578 | INFO | "gpu" : " 17 GB", | 20260507 05:40:40.578 | INFO | "huge" : "", | 20260507 05:40:40.578 | INFO | "huge_shared" : " 27 GB", | 20260507 05:40:40.578 | INFO | "malloc" : " 14 GB", | 20260507 05:40:40.578 | INFO | "pinned" : " 29 GB" | 20260507 05:40:40.578 | INFO | } | 20260507 05:40:40.578 | INFO | Perf report for the last 5.13536 seconds: | 20260507 05:40:40.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:40:40.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:40.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:40.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:40.578 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 05:40:40.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:40.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:40.578 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:40:40.578 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:40.578 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:40.578 | INFO | Uploaded 36GiB to GPU in 2.04497s (17.6042GiB/s) | 20260507 05:40:40.578 | INFO | Downloaded 34GiB to GPU in 1.9487s (17.4475GiB/s) | 20260507 05:40:45.716 | INFO | Analyzing chunk frames = [7526400,7526912) | 20260507 05:40:45.716 | INFO | MemInfo: { | 20260507 05:40:45.716 | INFO | "gpu" : " 17 GB", | 20260507 05:40:45.716 | INFO | "huge" : "", | 20260507 05:40:45.716 | INFO | "huge_shared" : " 27 GB", | 20260507 05:40:45.716 | INFO | "malloc" : " 14 GB", | 20260507 05:40:45.716 | INFO | "pinned" : " 29 GB" | 20260507 05:40:45.716 | INFO | } | 20260507 05:40:45.716 | INFO | Perf report for the last 5.13781 seconds: | 20260507 05:40:45.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:40:45.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:45.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:45.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:45.716 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260507 05:40:45.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:45.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:45.716 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 05:40:45.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:45.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:45.716 | INFO | Uploaded 36GiB to GPU in 2.02418s (17.7849GiB/s) | 20260507 05:40:45.716 | INFO | Downloaded 34GiB to GPU in 1.91993s (17.709GiB/s) | 20260507 05:40:50.855 | INFO | Analyzing chunk frames = [7526912,7527424) | 20260507 05:40:50.855 | INFO | MemInfo: { | 20260507 05:40:50.855 | INFO | "gpu" : " 17 GB", | 20260507 05:40:50.855 | INFO | "huge" : "", | 20260507 05:40:50.855 | INFO | "huge_shared" : " 27 GB", | 20260507 05:40:50.855 | INFO | "malloc" : " 14 GB", | 20260507 05:40:50.855 | INFO | "pinned" : " 29 GB" | 20260507 05:40:50.855 | INFO | } | 20260507 05:40:50.855 | INFO | Perf report for the last 5.13959 seconds: | 20260507 05:40:50.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:40:50.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:50.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:50.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:50.855 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 05:40:50.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:50.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:50.855 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:40:50.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:50.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:50.855 | INFO | Uploaded 36GiB to GPU in 2.03625s (17.6795GiB/s) | 20260507 05:40:50.855 | INFO | Downloaded 34GiB to GPU in 1.90859s (17.8142GiB/s) | 20260507 05:40:55.999 | INFO | Analyzing chunk frames = [7527424,7527936) | 20260507 05:40:55.999 | INFO | MemInfo: { | 20260507 05:40:55.999 | INFO | "gpu" : " 17 GB", | 20260507 05:40:55.999 | INFO | "huge" : "", | 20260507 05:40:55.999 | INFO | "huge_shared" : " 27 GB", | 20260507 05:40:55.999 | INFO | "malloc" : " 14 GB", | 20260507 05:40:55.999 | INFO | "pinned" : " 29 GB" | 20260507 05:40:55.999 | INFO | } | 20260507 05:40:55.999 | INFO | Perf report for the last 5.14352 seconds: | 20260507 05:40:55.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:40:55.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:55.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:55.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:55.999 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:40:55.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:55.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:55.999 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 05:40:55.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:55.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:40:55.999 | INFO | Uploaded 36GiB to GPU in 1.96372s (18.3325GiB/s) | 20260507 05:40:55.999 | INFO | Downloaded 34GiB to GPU in 1.98291s (17.1465GiB/s) | 20260507 05:41:01.133 | INFO | Analyzing chunk frames = [7527936,7528448) | 20260507 05:41:01.133 | INFO | MemInfo: { | 20260507 05:41:01.133 | INFO | "gpu" : " 17 GB", | 20260507 05:41:01.133 | INFO | "huge" : "", | 20260507 05:41:01.133 | INFO | "huge_shared" : " 27 GB", | 20260507 05:41:01.133 | INFO | "malloc" : " 14 GB", | 20260507 05:41:01.133 | INFO | "pinned" : " 29 GB" | 20260507 05:41:01.133 | INFO | } | 20260507 05:41:01.133 | INFO | Perf report for the last 5.13426 seconds: | 20260507 05:41:01.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:41:01.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:01.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:01.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:01.133 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:41:01.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:01.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:01.133 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 05:41:01.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:01.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:01.133 | INFO | Uploaded 36GiB to GPU in 1.99489s (18.0461GiB/s) | 20260507 05:41:01.133 | INFO | Downloaded 34GiB to GPU in 1.95668s (17.3764GiB/s) | 20260507 05:41:01.133 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:41:01.133 | INFO | Total mem usage: 18 GiB | 20260507 05:41:01.133 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:41:01.133 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 05:41:01.133 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:41:01.133 | INFO | High Water Breakdown: | 20260507 05:41:01.133 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:41:01.133 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:41:01.133 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:41:01.133 | INFO | ------------------------------------------ | 20260507 05:41:01.134 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:41:01.134 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:41:01.134 | INFO | Total mem usage: 26.5 GiB | 20260507 05:41:01.134 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:41:01.134 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:41:01.134 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:41:01.134 | INFO | High Water Breakdown: | 20260507 05:41:01.134 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:41:01.134 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:41:01.134 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:41:01.134 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:41:01.134 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:41:01.134 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:41:01.134 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:41:01.134 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:41:01.134 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:41:01.134 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:41:01.134 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:41:01.134 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:41:01.134 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:41:01.134 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:41:01.134 | INFO | ------------------------------------------ | 20260507 05:41:01.134 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:41:01.134 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:41:01.134 | INFO | Total mem usage: 25.6 GiB | 20260507 05:41:01.134 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:41:01.134 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:41:01.134 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:41:01.134 | INFO | High Water Breakdown: | 20260507 05:41:01.134 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:41:01.134 | INFO | ------------------------------------------ | 20260507 05:41:01.134 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:41:01.134 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:41:01.134 | INFO | Total mem usage: 15.7 GiB | 20260507 05:41:01.134 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:41:01.134 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:41:01.134 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:41:01.134 | INFO | High Water Breakdown: | 20260507 05:41:01.134 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:41:01.134 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:41:01.134 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:41:01.134 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:41:01.134 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:41:01.134 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:41:01.134 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:41:01.134 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:41:01.134 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:41:01.134 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:41:01.134 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:41:01.134 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:41:01.134 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:41:01.134 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:41:01.134 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:41:01.134 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:41:01.134 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:41:01.134 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:41:01.134 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:41:01.134 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:41:01.134 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:41:01.134 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:41:01.134 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:41:01.134 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:41:01.134 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:41:01.134 | INFO | ------------------------------------------ | 20260507 05:41:01.134 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:41:01.134 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:41:01.134 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:41:01.134 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:41:01.134 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.33e+01 / 9.66e-01 | 20260507 05:41:01.134 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.02e-01, 9.42e-03 / 3.86e-03 | 20260507 05:41:01.134 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.48e+00, 5.07e-01 / 2.24e-01 | 20260507 05:41:01.134 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.50e+00, 1.68e-01 / 8.63e-02 | 20260507 05:41:01.134 | INFO | PerfReport: 16.26%, DME -- 4.00e+01 / 4.50e+01, 3.91e+00 / 1.25e+00 | 20260507 05:41:01.134 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.26e+00 / 1.35e+00 | 20260507 05:41:01.134 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.72e+00, 3.85e-01 / 1.58e-01 | 20260507 05:41:01.134 | INFO | PerfReport: 2.98%, Metrics -- 9.16e-01 / 1.22e+00, 2.42e-01 / 1.06e-01 | 20260507 05:41:01.134 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.09e-01 / 3.65e-01, 5.08e-02 / 1.53e-02 | 20260507 05:41:01.134 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:41:01.134 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:41:01.134 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:41:01.134 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 05:41:01.134 | INFO | PerfReport: 21.80%, Upload -- 2.10e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260507 05:41:01.134 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 05:41:01.134 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 05:41:01.134 | INFO | PerfReport: 1.54%, LockUpload -- 1.48e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 05:41:01.134 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260507 05:41:01.134 | INFO | PerfReport: 22.06%, LockCompute -- 2.12e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 05:41:01.134 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260507 05:41:01.230 | INFO | Starting a round of Baz IO | 20260507 05:41:04.326 | INFO | Populating RTMetrics report | 20260507 05:41:04.326 | INFO | RTMetrics report populated | 20260507 05:41:04.326 | INFO | Opening tmp RTMetrics json file | 20260507 05:41:04.349 | INFO | Serializing RTMetrics to json | 20260507 05:41:04.352 | INFO | Serialization done, swapping tmp file now... | 20260507 05:41:04.352 | INFO | RTMetrics done | 20260507 05:41:04.427 | INFO | Finished a round of Baz IO: Wrote 1.689 GiB in 3.196 seconds | 20260507 05:41:04.427 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.69% / 0.00% / 67.31% | 20260507 05:41:04.427 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.138334% / 0% / 7.94729e-05% | 20260507 05:41:04.427 | INFO | Baz IO overall: | 20260507 05:41:04.427 | INFO | BytesWritten : 3814555587260 | 20260507 05:41:04.427 | INFO | NumEvents : 1829292287221 | 20260507 05:41:06.265 | INFO | Analyzing chunk frames = [7528448,7528960) | 20260507 05:41:06.265 | INFO | MemInfo: { | 20260507 05:41:06.265 | INFO | "gpu" : " 17 GB", | 20260507 05:41:06.265 | INFO | "huge" : "", | 20260507 05:41:06.265 | INFO | "huge_shared" : " 27 GB", | 20260507 05:41:06.265 | INFO | "malloc" : " 14 GB", | 20260507 05:41:06.265 | INFO | "pinned" : " 29 GB" | 20260507 05:41:06.265 | INFO | } | 20260507 05:41:06.265 | INFO | Perf report for the last 5.13208 seconds: | 20260507 05:41:06.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:41:06.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:06.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:06.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:06.265 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260507 05:41:06.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:06.265 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 05:41:06.265 | INFO | PRE_HQ : 59.2%, 1.8, 55.3ms, 0 | 20260507 05:41:06.265 | INFO | BAZWRITER : 61.9%, 3.3, 103.7ms, 0 | 20260507 05:41:06.265 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:41:06.265 | INFO | Uploaded 36GiB to GPU in 2.05204s (17.5436GiB/s) | 20260507 05:41:06.265 | INFO | Downloaded 38GiB to GPU in 2.17549s (17.4673GiB/s) | 20260507 05:41:11.410 | INFO | Analyzing chunk frames = [7528960,7529472) | 20260507 05:41:11.410 | INFO | MemInfo: { | 20260507 05:41:11.410 | INFO | "gpu" : " 17 GB", | 20260507 05:41:11.411 | INFO | "huge" : "", | 20260507 05:41:11.411 | INFO | "huge_shared" : " 27 GB", | 20260507 05:41:11.411 | INFO | "malloc" : " 14 GB", | 20260507 05:41:11.411 | INFO | "pinned" : " 29 GB" | 20260507 05:41:11.411 | INFO | } | 20260507 05:41:11.411 | INFO | Perf report for the last 5.14543 seconds: | 20260507 05:41:11.411 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:41:11.411 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:11.411 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:11.411 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:11.411 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 05:41:11.411 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:11.411 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:11.411 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:41:11.411 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:11.411 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:11.411 | INFO | Uploaded 36GiB to GPU in 2.03697s (17.6733GiB/s) | 20260507 05:41:11.411 | INFO | Downloaded 34GiB to GPU in 1.93488s (17.5722GiB/s) | 20260507 05:41:16.553 | INFO | Analyzing chunk frames = [7529472,7529984) | 20260507 05:41:16.553 | INFO | MemInfo: { | 20260507 05:41:16.553 | INFO | "gpu" : " 17 GB", | 20260507 05:41:16.553 | INFO | "huge" : "", | 20260507 05:41:16.553 | INFO | "huge_shared" : " 27 GB", | 20260507 05:41:16.553 | INFO | "malloc" : " 14 GB", | 20260507 05:41:16.553 | INFO | "pinned" : " 29 GB" | 20260507 05:41:16.553 | INFO | } | 20260507 05:41:16.553 | INFO | Perf report for the last 5.14222 seconds: | 20260507 05:41:16.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:41:16.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:16.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:16.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:16.553 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 05:41:16.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:16.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:16.553 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:41:16.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:16.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:16.553 | INFO | Uploaded 36GiB to GPU in 2.01971s (17.8243GiB/s) | 20260507 05:41:16.553 | INFO | Downloaded 34GiB to GPU in 1.93931s (17.532GiB/s) | 20260507 05:41:21.691 | INFO | Analyzing chunk frames = [7529984,7530496) | 20260507 05:41:21.691 | INFO | MemInfo: { | 20260507 05:41:21.691 | INFO | "gpu" : " 17 GB", | 20260507 05:41:21.691 | INFO | "huge" : "", | 20260507 05:41:21.691 | INFO | "huge_shared" : " 27 GB", | 20260507 05:41:21.691 | INFO | "malloc" : " 14 GB", | 20260507 05:41:21.691 | INFO | "pinned" : " 29 GB" | 20260507 05:41:21.691 | INFO | } | 20260507 05:41:21.691 | INFO | Perf report for the last 5.13827 seconds: | 20260507 05:41:21.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:41:21.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:21.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:21.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:21.691 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 05:41:21.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:21.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:21.691 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:41:21.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:21.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:21.691 | INFO | Uploaded 36GiB to GPU in 2.03475s (17.6926GiB/s) | 20260507 05:41:21.691 | INFO | Downloaded 34GiB to GPU in 1.95205s (17.4176GiB/s) | 20260507 05:41:26.824 | INFO | Analyzing chunk frames = [7530496,7531008) | 20260507 05:41:26.824 | INFO | MemInfo: { | 20260507 05:41:26.824 | INFO | "gpu" : " 17 GB", | 20260507 05:41:26.824 | INFO | "huge" : "", | 20260507 05:41:26.825 | INFO | "huge_shared" : " 27 GB", | 20260507 05:41:26.825 | INFO | "malloc" : " 14 GB", | 20260507 05:41:26.825 | INFO | "pinned" : " 29 GB" | 20260507 05:41:26.825 | INFO | } | 20260507 05:41:26.825 | INFO | Perf report for the last 5.13346 seconds: | 20260507 05:41:26.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:41:26.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:26.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:26.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:26.825 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 05:41:26.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:26.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:26.825 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:41:26.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:26.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:26.825 | INFO | Uploaded 36GiB to GPU in 2.03235s (17.7135GiB/s) | 20260507 05:41:26.825 | INFO | Downloaded 34GiB to GPU in 1.91684s (17.7375GiB/s) | 20260507 05:41:31.969 | INFO | Analyzing chunk frames = [7531008,7531520) | 20260507 05:41:31.969 | INFO | MemInfo: { | 20260507 05:41:31.969 | INFO | "gpu" : " 17 GB", | 20260507 05:41:31.969 | INFO | "huge" : "", | 20260507 05:41:31.969 | INFO | "huge_shared" : " 27 GB", | 20260507 05:41:31.969 | INFO | "malloc" : " 14 GB", | 20260507 05:41:31.969 | INFO | "pinned" : " 29 GB" | 20260507 05:41:31.969 | INFO | } | 20260507 05:41:31.969 | INFO | Perf report for the last 5.14418 seconds: | 20260507 05:41:31.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:41:31.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:31.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:31.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:31.969 | INFO | ANALYSIS : 60.4%, 3.0, 94.4ms, 0 | 20260507 05:41:31.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:31.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:31.969 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 05:41:31.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:31.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:31.969 | INFO | Uploaded 36GiB to GPU in 2.04461s (17.6073GiB/s) | 20260507 05:41:31.969 | INFO | Downloaded 34GiB to GPU in 1.91503s (17.7543GiB/s) | 20260507 05:41:37.112 | INFO | Analyzing chunk frames = [7531520,7532032) | 20260507 05:41:37.112 | INFO | MemInfo: { | 20260507 05:41:37.112 | INFO | "gpu" : " 17 GB", | 20260507 05:41:37.112 | INFO | "huge" : "", | 20260507 05:41:37.112 | INFO | "huge_shared" : " 27 GB", | 20260507 05:41:37.112 | INFO | "malloc" : " 14 GB", | 20260507 05:41:37.112 | INFO | "pinned" : " 29 GB" | 20260507 05:41:37.112 | INFO | } | 20260507 05:41:37.112 | INFO | Perf report for the last 5.14354 seconds: | 20260507 05:41:37.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:41:37.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:37.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:37.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:37.112 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:41:37.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:37.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:37.112 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 05:41:37.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:37.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:37.113 | INFO | Uploaded 36GiB to GPU in 1.96568s (18.3143GiB/s) | 20260507 05:41:37.113 | INFO | Downloaded 34GiB to GPU in 1.97996s (17.1721GiB/s) | 20260507 05:41:42.253 | INFO | Analyzing chunk frames = [7532032,7532544) | 20260507 05:41:42.253 | INFO | MemInfo: { | 20260507 05:41:42.253 | INFO | "gpu" : " 17 GB", | 20260507 05:41:42.253 | INFO | "huge" : "", | 20260507 05:41:42.253 | INFO | "huge_shared" : " 27 GB", | 20260507 05:41:42.253 | INFO | "malloc" : " 14 GB", | 20260507 05:41:42.253 | INFO | "pinned" : " 29 GB" | 20260507 05:41:42.253 | INFO | } | 20260507 05:41:42.253 | INFO | Perf report for the last 5.14102 seconds: | 20260507 05:41:42.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:41:42.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:42.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:42.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:42.253 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:41:42.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:42.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:42.253 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 05:41:42.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:42.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:42.254 | INFO | Uploaded 36GiB to GPU in 1.99253s (18.0675GiB/s) | 20260507 05:41:42.254 | INFO | Downloaded 34GiB to GPU in 1.95811s (17.3637GiB/s) | 20260507 05:41:42.350 | INFO | Starting a round of Baz IO | 20260507 05:41:45.443 | INFO | Populating RTMetrics report | 20260507 05:41:45.444 | INFO | RTMetrics report populated | 20260507 05:41:45.444 | INFO | Opening tmp RTMetrics json file | 20260507 05:41:45.444 | INFO | Serializing RTMetrics to json | 20260507 05:41:45.454 | INFO | Serialization done, swapping tmp file now... | 20260507 05:41:45.463 | INFO | RTMetrics done | 20260507 05:41:45.529 | INFO | Finished a round of Baz IO: Wrote 1.69 GiB in 3.178 seconds | 20260507 05:41:45.529 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.68% / 0.00% / 67.31% | 20260507 05:41:45.529 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.138879% / 3.97364e-06% / 8.34465e-05% | 20260507 05:41:45.529 | INFO | Baz IO overall: | 20260507 05:41:45.529 | INFO | BytesWritten : 3816370738378 | 20260507 05:41:45.529 | INFO | NumEvents : 1830143978181 | 20260507 05:41:47.387 | INFO | Analyzing chunk frames = [7532544,7533056) | 20260507 05:41:47.387 | INFO | MemInfo: { | 20260507 05:41:47.387 | INFO | "gpu" : " 17 GB", | 20260507 05:41:47.387 | INFO | "huge" : "", | 20260507 05:41:47.387 | INFO | "huge_shared" : " 27 GB", | 20260507 05:41:47.387 | INFO | "malloc" : " 14 GB", | 20260507 05:41:47.387 | INFO | "pinned" : " 29 GB" | 20260507 05:41:47.387 | INFO | } | 20260507 05:41:47.387 | INFO | Perf report for the last 5.13358 seconds: | 20260507 05:41:47.387 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:41:47.387 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:47.387 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:47.387 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:47.387 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 05:41:47.387 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:47.387 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 05:41:47.387 | INFO | PRE_HQ : 59.3%, 1.8, 55.5ms, 0 | 20260507 05:41:47.387 | INFO | BAZWRITER : 61.5%, 3.2, 101.7ms, 0 | 20260507 05:41:47.387 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:41:47.387 | INFO | Uploaded 36GiB to GPU in 2.04867s (17.5724GiB/s) | 20260507 05:41:47.387 | INFO | Downloaded 38GiB to GPU in 2.17512s (17.4703GiB/s) | 20260507 05:41:52.521 | INFO | Analyzing chunk frames = [7533056,7533568) | 20260507 05:41:52.521 | INFO | MemInfo: { | 20260507 05:41:52.521 | INFO | "gpu" : " 17 GB", | 20260507 05:41:52.521 | INFO | "huge" : "", | 20260507 05:41:52.521 | INFO | "huge_shared" : " 27 GB", | 20260507 05:41:52.521 | INFO | "malloc" : " 14 GB", | 20260507 05:41:52.521 | INFO | "pinned" : " 29 GB" | 20260507 05:41:52.521 | INFO | } | 20260507 05:41:52.521 | INFO | Perf report for the last 5.13388 seconds: | 20260507 05:41:52.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:41:52.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:52.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:52.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:52.521 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:41:52.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:52.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:52.521 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:41:52.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:52.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:52.521 | INFO | Uploaded 36GiB to GPU in 1.99215s (18.0709GiB/s) | 20260507 05:41:52.521 | INFO | Downloaded 34GiB to GPU in 1.92369s (17.6744GiB/s) | 20260507 05:41:57.663 | INFO | Analyzing chunk frames = [7533568,7534080) | 20260507 05:41:57.663 | INFO | MemInfo: { | 20260507 05:41:57.663 | INFO | "gpu" : " 17 GB", | 20260507 05:41:57.663 | INFO | "huge" : "", | 20260507 05:41:57.663 | INFO | "huge_shared" : " 27 GB", | 20260507 05:41:57.663 | INFO | "malloc" : " 14 GB", | 20260507 05:41:57.663 | INFO | "pinned" : " 29 GB" | 20260507 05:41:57.663 | INFO | } | 20260507 05:41:57.663 | INFO | Perf report for the last 5.14226 seconds: | 20260507 05:41:57.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:41:57.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:57.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:57.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:57.663 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 05:41:57.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:57.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:57.663 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:41:57.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:57.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:41:57.663 | INFO | Uploaded 36GiB to GPU in 2.02385s (17.7878GiB/s) | 20260507 05:41:57.663 | INFO | Downloaded 34GiB to GPU in 1.94134s (17.5137GiB/s) | 20260507 05:42:02.802 | INFO | Analyzing chunk frames = [7534080,7534592) | 20260507 05:42:02.802 | INFO | MemInfo: { | 20260507 05:42:02.802 | INFO | "gpu" : " 17 GB", | 20260507 05:42:02.802 | INFO | "huge" : "", | 20260507 05:42:02.802 | INFO | "huge_shared" : " 27 GB", | 20260507 05:42:02.802 | INFO | "malloc" : " 14 GB", | 20260507 05:42:02.802 | INFO | "pinned" : " 29 GB" | 20260507 05:42:02.802 | INFO | } | 20260507 05:42:02.802 | INFO | Perf report for the last 5.13904 seconds: | 20260507 05:42:02.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:42:02.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:02.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:02.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:02.802 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:42:02.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:02.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:02.802 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:42:02.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:02.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:02.802 | INFO | Uploaded 36GiB to GPU in 2.01265s (17.8868GiB/s) | 20260507 05:42:02.802 | INFO | Downloaded 34GiB to GPU in 1.9325s (17.5938GiB/s) | 20260507 05:42:07.948 | INFO | Analyzing chunk frames = [7534592,7535104) | 20260507 05:42:07.948 | INFO | MemInfo: { | 20260507 05:42:07.948 | INFO | "gpu" : " 17 GB", | 20260507 05:42:07.948 | INFO | "huge" : "", | 20260507 05:42:07.948 | INFO | "huge_shared" : " 27 GB", | 20260507 05:42:07.948 | INFO | "malloc" : " 14 GB", | 20260507 05:42:07.948 | INFO | "pinned" : " 29 GB" | 20260507 05:42:07.948 | INFO | } | 20260507 05:42:07.948 | INFO | Perf report for the last 5.14597 seconds: | 20260507 05:42:07.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:42:07.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:07.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:07.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:07.948 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 05:42:07.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:07.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:07.948 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 05:42:07.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:07.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:07.948 | INFO | Uploaded 36GiB to GPU in 2.05778s (17.4946GiB/s) | 20260507 05:42:07.948 | INFO | Downloaded 34GiB to GPU in 1.94843s (17.45GiB/s) | 20260507 05:42:13.084 | INFO | Analyzing chunk frames = [7535104,7535616) | 20260507 05:42:13.084 | INFO | MemInfo: { | 20260507 05:42:13.084 | INFO | "gpu" : " 17 GB", | 20260507 05:42:13.084 | INFO | "huge" : "", | 20260507 05:42:13.084 | INFO | "huge_shared" : " 27 GB", | 20260507 05:42:13.084 | INFO | "malloc" : " 14 GB", | 20260507 05:42:13.084 | INFO | "pinned" : " 29 GB" | 20260507 05:42:13.084 | INFO | } | 20260507 05:42:13.084 | INFO | Perf report for the last 5.13609 seconds: | 20260507 05:42:13.084 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:42:13.084 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:13.084 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:13.084 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:13.084 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 05:42:13.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:13.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:13.084 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 05:42:13.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:13.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:13.084 | INFO | Uploaded 36GiB to GPU in 2.07274s (17.3683GiB/s) | 20260507 05:42:13.085 | INFO | Downloaded 34GiB to GPU in 1.934s (17.5802GiB/s) | 20260507 05:42:18.222 | INFO | Analyzing chunk frames = [7535616,7536128) | 20260507 05:42:18.222 | INFO | MemInfo: { | 20260507 05:42:18.222 | INFO | "gpu" : " 17 GB", | 20260507 05:42:18.222 | INFO | "huge" : "", | 20260507 05:42:18.222 | INFO | "huge_shared" : " 27 GB", | 20260507 05:42:18.222 | INFO | "malloc" : " 14 GB", | 20260507 05:42:18.222 | INFO | "pinned" : " 29 GB" | 20260507 05:42:18.222 | INFO | } | 20260507 05:42:18.222 | INFO | Perf report for the last 5.138 seconds: | 20260507 05:42:18.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:42:18.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:18.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:18.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:18.222 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:42:18.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:18.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:18.222 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 05:42:18.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:18.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:18.223 | INFO | Uploaded 36GiB to GPU in 1.97597s (18.2189GiB/s) | 20260507 05:42:18.223 | INFO | Downloaded 34GiB to GPU in 1.97122s (17.2482GiB/s) | 20260507 05:42:23.367 | INFO | Analyzing chunk frames = [7536128,7536640) | 20260507 05:42:23.367 | INFO | MemInfo: { | 20260507 05:42:23.367 | INFO | "gpu" : " 17 GB", | 20260507 05:42:23.367 | INFO | "huge" : "", | 20260507 05:42:23.367 | INFO | "huge_shared" : " 27 GB", | 20260507 05:42:23.367 | INFO | "malloc" : " 14 GB", | 20260507 05:42:23.367 | INFO | "pinned" : " 29 GB" | 20260507 05:42:23.367 | INFO | } | 20260507 05:42:23.367 | INFO | Perf report for the last 5.14482 seconds: | 20260507 05:42:23.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:42:23.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:23.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:23.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:23.367 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 05:42:23.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:23.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:23.367 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 05:42:23.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:23.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:23.367 | INFO | Uploaded 36GiB to GPU in 1.99135s (18.0781GiB/s) | 20260507 05:42:23.367 | INFO | Downloaded 34GiB to GPU in 1.96449s (17.3073GiB/s) | 20260507 05:42:23.367 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:42:23.367 | INFO | Total mem usage: 18 GiB | 20260507 05:42:23.367 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:42:23.367 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 05:42:23.367 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:42:23.368 | INFO | High Water Breakdown: | 20260507 05:42:23.368 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:42:23.368 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:42:23.368 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:42:23.368 | INFO | ------------------------------------------ | 20260507 05:42:23.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:42:23.368 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:42:23.368 | INFO | Total mem usage: 26.5 GiB | 20260507 05:42:23.368 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:42:23.368 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:42:23.368 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:42:23.368 | INFO | High Water Breakdown: | 20260507 05:42:23.368 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:42:23.368 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:42:23.368 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:42:23.368 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:42:23.368 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:42:23.368 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:42:23.368 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:42:23.368 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:42:23.368 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:42:23.368 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:42:23.368 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:42:23.368 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:42:23.368 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:42:23.368 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:42:23.368 | INFO | ------------------------------------------ | 20260507 05:42:23.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:42:23.368 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:42:23.368 | INFO | Total mem usage: 25.6 GiB | 20260507 05:42:23.368 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:42:23.368 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:42:23.368 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:42:23.368 | INFO | High Water Breakdown: | 20260507 05:42:23.368 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:42:23.368 | INFO | ------------------------------------------ | 20260507 05:42:23.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:42:23.368 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:42:23.368 | INFO | Total mem usage: 15.7 GiB | 20260507 05:42:23.368 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:42:23.368 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:42:23.368 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:42:23.368 | INFO | High Water Breakdown: | 20260507 05:42:23.368 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:42:23.368 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:42:23.368 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:42:23.368 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:42:23.368 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:42:23.368 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:42:23.368 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:42:23.368 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:42:23.368 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:42:23.368 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:42:23.368 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:42:23.368 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:42:23.368 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:42:23.368 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:42:23.368 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:42:23.368 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:42:23.368 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:42:23.368 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:42:23.368 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:42:23.368 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:42:23.368 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:42:23.368 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:42:23.368 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:42:23.368 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:42:23.368 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:42:23.368 | INFO | ------------------------------------------ | 20260507 05:42:23.368 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:42:23.368 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:42:23.368 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:42:23.368 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:42:23.368 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.33e+01 / 1.26e+00 | 20260507 05:42:23.368 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.01e-01, 9.67e-03 / 4.83e-03 | 20260507 05:42:23.368 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.45e+00, 5.12e-01 / 2.66e-01 | 20260507 05:42:23.368 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.46e+00, 1.67e-01 / 1.14e-01 | 20260507 05:42:23.368 | INFO | PerfReport: 16.16%, DME -- 3.97e+01 / 4.43e+01, 3.81e+00 / 1.49e+00 | 20260507 05:42:23.368 | INFO | PerfReport: 57.69%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.30e+00 / 1.24e+00 | 20260507 05:42:23.368 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.69e+00, 3.84e-01 / 1.99e-01 | 20260507 05:42:23.368 | INFO | PerfReport: 2.99%, Metrics -- 9.19e-01 / 1.25e+00, 2.47e-01 / 1.50e-01 | 20260507 05:42:23.368 | INFO | PerfReport: 1.01%, PulseSerializing -- 3.09e-01 / 3.61e-01, 5.10e-02 / 1.85e-02 | 20260507 05:42:23.368 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:42:23.368 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:42:23.368 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:42:23.368 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 05:42:23.368 | INFO | PerfReport: 21.90%, Upload -- 2.11e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260507 05:42:23.368 | INFO | PerfReport: 21.37%, Download -- 2.05e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 05:42:23.368 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 05:42:23.368 | INFO | PerfReport: 1.57%, LockUpload -- 1.51e+00 / 0.00e+00, 4.89e+00 / 0.00e+00 | 20260507 05:42:23.368 | INFO | PerfReport: 1.22%, LockDownload -- 1.17e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260507 05:42:23.368 | INFO | PerfReport: 22.00%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 05:42:23.368 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.58e-02 / 0.00e+00 | 20260507 05:42:23.464 | INFO | Starting a round of Baz IO | 20260507 05:42:26.559 | INFO | Populating RTMetrics report | 20260507 05:42:26.559 | INFO | RTMetrics report populated | 20260507 05:42:26.559 | INFO | Opening tmp RTMetrics json file | 20260507 05:42:26.578 | INFO | Serializing RTMetrics to json | 20260507 05:42:26.580 | INFO | Serialization done, swapping tmp file now... | 20260507 05:42:26.580 | INFO | RTMetrics done | 20260507 05:42:26.648 | INFO | Finished a round of Baz IO: Wrote 1.689 GiB in 3.183 seconds | 20260507 05:42:26.648 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.68% / 0.00% / 67.32% | 20260507 05:42:26.648 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.138338% / 3.97364e-06% / 8.74201e-05% | 20260507 05:42:26.648 | INFO | Baz IO overall: | 20260507 05:42:26.648 | INFO | BytesWritten : 3818184309921 | 20260507 05:42:26.648 | INFO | NumEvents : 1830994753712 | 20260507 05:42:28.501 | INFO | Analyzing chunk frames = [7536640,7537152) | 20260507 05:42:28.502 | INFO | MemInfo: { | 20260507 05:42:28.502 | INFO | "gpu" : " 17 GB", | 20260507 05:42:28.502 | INFO | "huge" : "", | 20260507 05:42:28.502 | INFO | "huge_shared" : " 27 GB", | 20260507 05:42:28.502 | INFO | "malloc" : " 14 GB", | 20260507 05:42:28.502 | INFO | "pinned" : " 29 GB" | 20260507 05:42:28.502 | INFO | } | 20260507 05:42:28.502 | INFO | Perf report for the last 5.13442 seconds: | 20260507 05:42:28.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:42:28.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:28.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:28.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:28.502 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 05:42:28.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:28.502 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 05:42:28.502 | INFO | PRE_HQ : 59.2%, 1.8, 55.2ms, 0 | 20260507 05:42:28.502 | INFO | BAZWRITER : 61.6%, 3.2, 101.5ms, 0 | 20260507 05:42:28.502 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:42:28.502 | INFO | Uploaded 36GiB to GPU in 2.05451s (17.5224GiB/s) | 20260507 05:42:28.502 | INFO | Downloaded 38GiB to GPU in 2.17558s (17.4666GiB/s) | 20260507 05:42:33.641 | INFO | Analyzing chunk frames = [7537152,7537664) | 20260507 05:42:33.641 | INFO | MemInfo: { | 20260507 05:42:33.641 | INFO | "gpu" : " 17 GB", | 20260507 05:42:33.641 | INFO | "huge" : "", | 20260507 05:42:33.641 | INFO | "huge_shared" : " 27 GB", | 20260507 05:42:33.641 | INFO | "malloc" : " 14 GB", | 20260507 05:42:33.641 | INFO | "pinned" : " 29 GB" | 20260507 05:42:33.641 | INFO | } | 20260507 05:42:33.641 | INFO | Perf report for the last 5.13904 seconds: | 20260507 05:42:33.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:42:33.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:33.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:33.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:33.641 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:42:33.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:33.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:33.641 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:42:33.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:33.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:33.641 | INFO | Uploaded 36GiB to GPU in 1.98549s (18.1315GiB/s) | 20260507 05:42:33.641 | INFO | Downloaded 34GiB to GPU in 1.93871s (17.5375GiB/s) | 20260507 05:42:38.775 | INFO | Analyzing chunk frames = [7537664,7538176) | 20260507 05:42:38.775 | INFO | MemInfo: { | 20260507 05:42:38.775 | INFO | "gpu" : " 17 GB", | 20260507 05:42:38.775 | INFO | "huge" : "", | 20260507 05:42:38.775 | INFO | "huge_shared" : " 27 GB", | 20260507 05:42:38.775 | INFO | "malloc" : " 14 GB", | 20260507 05:42:38.775 | INFO | "pinned" : " 29 GB" | 20260507 05:42:38.775 | INFO | } | 20260507 05:42:38.775 | INFO | Perf report for the last 5.13477 seconds: | 20260507 05:42:38.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:42:38.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:38.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:38.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:38.775 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 05:42:38.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:38.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:38.776 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:42:38.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:38.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:38.776 | INFO | Uploaded 36GiB to GPU in 2.05019s (17.5593GiB/s) | 20260507 05:42:38.776 | INFO | Downloaded 34GiB to GPU in 1.94897s (17.4451GiB/s) | 20260507 05:42:43.919 | INFO | Analyzing chunk frames = [7538176,7538688) | 20260507 05:42:43.919 | INFO | MemInfo: { | 20260507 05:42:43.919 | INFO | "gpu" : " 17 GB", | 20260507 05:42:43.919 | INFO | "huge" : "", | 20260507 05:42:43.919 | INFO | "huge_shared" : " 27 GB", | 20260507 05:42:43.919 | INFO | "malloc" : " 14 GB", | 20260507 05:42:43.919 | INFO | "pinned" : " 29 GB" | 20260507 05:42:43.919 | INFO | } | 20260507 05:42:43.919 | INFO | Perf report for the last 5.14399 seconds: | 20260507 05:42:43.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:42:43.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:43.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:43.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:43.919 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 05:42:43.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:43.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:43.919 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260507 05:42:43.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:43.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:43.920 | INFO | Uploaded 36GiB to GPU in 1.99781s (18.0198GiB/s) | 20260507 05:42:43.920 | INFO | Downloaded 34GiB to GPU in 1.92742s (17.6401GiB/s) | 20260507 05:42:49.063 | INFO | Analyzing chunk frames = [7538688,7539200) | 20260507 05:42:49.063 | INFO | MemInfo: { | 20260507 05:42:49.063 | INFO | "gpu" : " 17 GB", | 20260507 05:42:49.063 | INFO | "huge" : "", | 20260507 05:42:49.063 | INFO | "huge_shared" : " 27 GB", | 20260507 05:42:49.063 | INFO | "malloc" : " 14 GB", | 20260507 05:42:49.063 | INFO | "pinned" : " 29 GB" | 20260507 05:42:49.063 | INFO | } | 20260507 05:42:49.063 | INFO | Perf report for the last 5.14384 seconds: | 20260507 05:42:49.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:42:49.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:49.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:49.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:49.063 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 05:42:49.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:49.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:49.064 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:42:49.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:49.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:49.064 | INFO | Uploaded 36GiB to GPU in 2.05487s (17.5193GiB/s) | 20260507 05:42:49.064 | INFO | Downloaded 34GiB to GPU in 1.94364s (17.493GiB/s) | 20260507 05:42:54.210 | INFO | Analyzing chunk frames = [7539200,7539712) | 20260507 05:42:54.210 | INFO | MemInfo: { | 20260507 05:42:54.210 | INFO | "gpu" : " 17 GB", | 20260507 05:42:54.210 | INFO | "huge" : "", | 20260507 05:42:54.210 | INFO | "huge_shared" : " 27 GB", | 20260507 05:42:54.210 | INFO | "malloc" : " 14 GB", | 20260507 05:42:54.210 | INFO | "pinned" : " 29 GB" | 20260507 05:42:54.210 | INFO | } | 20260507 05:42:54.211 | INFO | Perf report for the last 5.14736 seconds: | 20260507 05:42:54.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:42:54.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:54.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:54.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:54.211 | INFO | ANALYSIS : 60.3%, 3.0, 94.3ms, 0 | 20260507 05:42:54.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:54.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:54.211 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:42:54.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:54.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:54.211 | INFO | Uploaded 36GiB to GPU in 2.0693s (17.3972GiB/s) | 20260507 05:42:54.211 | INFO | Downloaded 34GiB to GPU in 1.92266s (17.6838GiB/s) | 20260507 05:42:59.336 | INFO | Analyzing chunk frames = [7539712,7540224) | 20260507 05:42:59.336 | INFO | MemInfo: { | 20260507 05:42:59.336 | INFO | "gpu" : " 17 GB", | 20260507 05:42:59.336 | INFO | "huge" : "", | 20260507 05:42:59.336 | INFO | "huge_shared" : " 27 GB", | 20260507 05:42:59.337 | INFO | "malloc" : " 14 GB", | 20260507 05:42:59.337 | INFO | "pinned" : " 29 GB" | 20260507 05:42:59.337 | INFO | } | 20260507 05:42:59.337 | INFO | Perf report for the last 5.12602 seconds: | 20260507 05:42:59.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:42:59.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:59.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:59.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:59.337 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 05:42:59.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:59.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:59.337 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 05:42:59.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:59.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:42:59.337 | INFO | Uploaded 36GiB to GPU in 1.97534s (18.2247GiB/s) | 20260507 05:42:59.337 | INFO | Downloaded 34GiB to GPU in 1.97927s (17.178GiB/s) | 20260507 05:43:04.476 | INFO | Analyzing chunk frames = [7540224,7540736) | 20260507 05:43:04.476 | INFO | MemInfo: { | 20260507 05:43:04.476 | INFO | "gpu" : " 17 GB", | 20260507 05:43:04.476 | INFO | "huge" : "", | 20260507 05:43:04.476 | INFO | "huge_shared" : " 27 GB", | 20260507 05:43:04.476 | INFO | "malloc" : " 14 GB", | 20260507 05:43:04.476 | INFO | "pinned" : " 29 GB" | 20260507 05:43:04.476 | INFO | } | 20260507 05:43:04.476 | INFO | Perf report for the last 5.1395 seconds: | 20260507 05:43:04.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:43:04.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:04.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:04.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:04.476 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:43:04.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:04.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:04.476 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:43:04.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:04.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:04.476 | INFO | Uploaded 36GiB to GPU in 1.9688s (18.2852GiB/s) | 20260507 05:43:04.476 | INFO | Downloaded 34GiB to GPU in 1.95804s (17.3643GiB/s) | 20260507 05:43:04.573 | INFO | Starting a round of Baz IO | 20260507 05:43:07.672 | INFO | Populating RTMetrics report | 20260507 05:43:07.672 | INFO | RTMetrics report populated | 20260507 05:43:07.672 | INFO | Opening tmp RTMetrics json file | 20260507 05:43:07.678 | INFO | Serializing RTMetrics to json | 20260507 05:43:07.689 | INFO | Serialization done, swapping tmp file now... | 20260507 05:43:07.690 | INFO | RTMetrics done | 20260507 05:43:07.758 | INFO | Finished a round of Baz IO: Wrote 1.688 GiB in 3.184 seconds | 20260507 05:43:07.758 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.68% / 0.00% / 67.32% | 20260507 05:43:07.758 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.137687% / 0% / 7.54992e-05% | 20260507 05:43:07.758 | INFO | Baz IO overall: | 20260507 05:43:07.758 | INFO | BytesWritten : 3819997146773 | 20260507 05:43:07.758 | INFO | NumEvents : 1831845116208 | 20260507 05:43:09.615 | INFO | Analyzing chunk frames = [7540736,7541248) | 20260507 05:43:09.615 | INFO | MemInfo: { | 20260507 05:43:09.615 | INFO | "gpu" : " 17 GB", | 20260507 05:43:09.615 | INFO | "huge" : "", | 20260507 05:43:09.615 | INFO | "huge_shared" : " 27 GB", | 20260507 05:43:09.615 | INFO | "malloc" : " 14 GB", | 20260507 05:43:09.615 | INFO | "pinned" : " 29 GB" | 20260507 05:43:09.615 | INFO | } | 20260507 05:43:09.615 | INFO | Perf report for the last 5.13917 seconds: | 20260507 05:43:09.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:43:09.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:09.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:09.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:09.615 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:43:09.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:09.615 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 05:43:09.615 | INFO | PRE_HQ : 58.9%, 1.8, 54.9ms, 0 | 20260507 05:43:09.615 | INFO | BAZWRITER : 61.6%, 3.2, 100.1ms, 0 | 20260507 05:43:09.615 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 05:43:09.616 | INFO | Uploaded 36GiB to GPU in 2.03109s (17.7245GiB/s) | 20260507 05:43:09.616 | INFO | Downloaded 38GiB to GPU in 2.18388s (17.4002GiB/s) | 20260507 05:43:14.761 | INFO | Analyzing chunk frames = [7541248,7541760) | 20260507 05:43:14.761 | INFO | MemInfo: { | 20260507 05:43:14.761 | INFO | "gpu" : " 17 GB", | 20260507 05:43:14.761 | INFO | "huge" : "", | 20260507 05:43:14.761 | INFO | "huge_shared" : " 27 GB", | 20260507 05:43:14.762 | INFO | "malloc" : " 14 GB", | 20260507 05:43:14.762 | INFO | "pinned" : " 29 GB" | 20260507 05:43:14.762 | INFO | } | 20260507 05:43:14.762 | INFO | Perf report for the last 5.14633 seconds: | 20260507 05:43:14.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:43:14.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:14.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:14.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:14.762 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 05:43:14.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:14.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:14.762 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:43:14.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:14.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:14.762 | INFO | Uploaded 36GiB to GPU in 2.04338s (17.6179GiB/s) | 20260507 05:43:14.762 | INFO | Downloaded 34GiB to GPU in 1.94364s (17.4929GiB/s) | 20260507 05:43:19.895 | INFO | Analyzing chunk frames = [7541760,7542272) | 20260507 05:43:19.895 | INFO | MemInfo: { | 20260507 05:43:19.895 | INFO | "gpu" : " 17 GB", | 20260507 05:43:19.895 | INFO | "huge" : "", | 20260507 05:43:19.895 | INFO | "huge_shared" : " 27 GB", | 20260507 05:43:19.895 | INFO | "malloc" : " 14 GB", | 20260507 05:43:19.895 | INFO | "pinned" : " 29 GB" | 20260507 05:43:19.895 | INFO | } | 20260507 05:43:19.895 | INFO | Perf report for the last 5.13362 seconds: | 20260507 05:43:19.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:43:19.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:19.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:19.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:19.895 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 05:43:19.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:19.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:19.895 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 05:43:19.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:19.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:19.896 | INFO | Uploaded 36GiB to GPU in 2.02512s (17.7767GiB/s) | 20260507 05:43:19.896 | INFO | Downloaded 34GiB to GPU in 1.92216s (17.6884GiB/s) | 20260507 05:43:25.028 | INFO | Analyzing chunk frames = [7542272,7542784) | 20260507 05:43:25.028 | INFO | MemInfo: { | 20260507 05:43:25.028 | INFO | "gpu" : " 17 GB", | 20260507 05:43:25.028 | INFO | "huge" : "", | 20260507 05:43:25.028 | INFO | "huge_shared" : " 27 GB", | 20260507 05:43:25.028 | INFO | "malloc" : " 14 GB", | 20260507 05:43:25.028 | INFO | "pinned" : " 29 GB" | 20260507 05:43:25.028 | INFO | } | 20260507 05:43:25.028 | INFO | Perf report for the last 5.13289 seconds: | 20260507 05:43:25.028 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:43:25.028 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:25.028 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:25.028 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:25.028 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:43:25.028 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:25.028 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:25.028 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:43:25.028 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:25.028 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:25.029 | INFO | Uploaded 36GiB to GPU in 2.00627s (17.9438GiB/s) | 20260507 05:43:25.029 | INFO | Downloaded 34GiB to GPU in 1.94644s (17.4678GiB/s) | 20260507 05:43:30.176 | INFO | Analyzing chunk frames = [7542784,7543296) | 20260507 05:43:30.176 | INFO | MemInfo: { | 20260507 05:43:30.176 | INFO | "gpu" : " 17 GB", | 20260507 05:43:30.177 | INFO | "huge" : "", | 20260507 05:43:30.177 | INFO | "huge_shared" : " 27 GB", | 20260507 05:43:30.177 | INFO | "malloc" : " 14 GB", | 20260507 05:43:30.177 | INFO | "pinned" : " 29 GB" | 20260507 05:43:30.177 | INFO | } | 20260507 05:43:30.177 | INFO | Perf report for the last 5.1485 seconds: | 20260507 05:43:30.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:43:30.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:30.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:30.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:30.177 | INFO | ANALYSIS : 59.9%, 3.0, 93.7ms, 0 | 20260507 05:43:30.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:30.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:30.177 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:43:30.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:30.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:30.177 | INFO | Uploaded 36GiB to GPU in 2.03215s (17.7152GiB/s) | 20260507 05:43:30.177 | INFO | Downloaded 34GiB to GPU in 1.9437s (17.4924GiB/s) | 20260507 05:43:35.317 | INFO | Analyzing chunk frames = [7543296,7543808) | 20260507 05:43:35.317 | INFO | MemInfo: { | 20260507 05:43:35.317 | INFO | "gpu" : " 17 GB", | 20260507 05:43:35.317 | INFO | "huge" : "", | 20260507 05:43:35.317 | INFO | "huge_shared" : " 27 GB", | 20260507 05:43:35.317 | INFO | "malloc" : " 14 GB", | 20260507 05:43:35.317 | INFO | "pinned" : " 29 GB" | 20260507 05:43:35.317 | INFO | } | 20260507 05:43:35.317 | INFO | Perf report for the last 5.14034 seconds: | 20260507 05:43:35.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:43:35.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:35.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:35.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:35.317 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 05:43:35.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:35.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:35.317 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 05:43:35.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:35.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:35.317 | INFO | Uploaded 36GiB to GPU in 2.03475s (17.6926GiB/s) | 20260507 05:43:35.317 | INFO | Downloaded 34GiB to GPU in 1.91702s (17.7359GiB/s) | 20260507 05:43:40.453 | INFO | Analyzing chunk frames = [7543808,7544320) | 20260507 05:43:40.453 | INFO | MemInfo: { | 20260507 05:43:40.453 | INFO | "gpu" : " 17 GB", | 20260507 05:43:40.454 | INFO | "huge" : "", | 20260507 05:43:40.454 | INFO | "huge_shared" : " 27 GB", | 20260507 05:43:40.454 | INFO | "malloc" : " 14 GB", | 20260507 05:43:40.454 | INFO | "pinned" : " 29 GB" | 20260507 05:43:40.454 | INFO | } | 20260507 05:43:40.454 | INFO | Perf report for the last 5.13667 seconds: | 20260507 05:43:40.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:43:40.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:40.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:40.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:40.454 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:43:40.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:40.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:40.454 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:43:40.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:40.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:40.454 | INFO | Uploaded 36GiB to GPU in 1.9193s (18.7568GiB/s) | 20260507 05:43:40.454 | INFO | Downloaded 34GiB to GPU in 1.97314s (17.2314GiB/s) | 20260507 05:43:45.588 | INFO | Analyzing chunk frames = [7544320,7544832) | 20260507 05:43:45.588 | INFO | MemInfo: { | 20260507 05:43:45.588 | INFO | "gpu" : " 17 GB", | 20260507 05:43:45.588 | INFO | "huge" : "", | 20260507 05:43:45.588 | INFO | "huge_shared" : " 27 GB", | 20260507 05:43:45.588 | INFO | "malloc" : " 14 GB", | 20260507 05:43:45.588 | INFO | "pinned" : " 29 GB" | 20260507 05:43:45.588 | INFO | } | 20260507 05:43:45.588 | INFO | Perf report for the last 5.13466 seconds: | 20260507 05:43:45.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:43:45.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:45.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:45.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:45.589 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:43:45.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:45.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:45.589 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 05:43:45.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:45.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:45.589 | INFO | Uploaded 36GiB to GPU in 1.97723s (18.2073GiB/s) | 20260507 05:43:45.589 | INFO | Downloaded 34GiB to GPU in 1.9543s (17.3976GiB/s) | 20260507 05:43:45.589 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:43:45.589 | INFO | Total mem usage: 18 GiB | 20260507 05:43:45.589 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:43:45.589 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:43:45.589 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:43:45.589 | INFO | High Water Breakdown: | 20260507 05:43:45.589 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:43:45.589 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:43:45.589 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:43:45.589 | INFO | ------------------------------------------ | 20260507 05:43:45.589 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:43:45.589 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:43:45.589 | INFO | Total mem usage: 26.5 GiB | 20260507 05:43:45.589 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:43:45.589 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:43:45.589 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:43:45.589 | INFO | High Water Breakdown: | 20260507 05:43:45.589 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:43:45.589 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:43:45.589 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:43:45.589 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:43:45.589 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:43:45.589 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:43:45.589 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:43:45.589 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:43:45.589 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:43:45.589 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:43:45.589 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:43:45.589 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:43:45.589 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:43:45.589 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:43:45.589 | INFO | ------------------------------------------ | 20260507 05:43:45.589 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:43:45.589 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:43:45.589 | INFO | Total mem usage: 25.6 GiB | 20260507 05:43:45.589 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:43:45.589 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 05:43:45.589 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 05:43:45.589 | INFO | High Water Breakdown: | 20260507 05:43:45.589 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 05:43:45.590 | INFO | ------------------------------------------ | 20260507 05:43:45.590 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:43:45.590 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:43:45.590 | INFO | Total mem usage: 15.7 GiB | 20260507 05:43:45.590 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:43:45.590 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:43:45.590 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:43:45.590 | INFO | High Water Breakdown: | 20260507 05:43:45.590 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:43:45.590 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:43:45.590 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:43:45.590 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:43:45.590 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:43:45.590 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:43:45.590 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:43:45.590 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:43:45.590 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:43:45.590 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:43:45.590 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:43:45.590 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:43:45.590 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:43:45.590 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:43:45.590 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:43:45.590 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:43:45.590 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:43:45.590 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:43:45.590 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:43:45.590 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:43:45.590 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:43:45.590 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:43:45.590 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:43:45.590 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:43:45.590 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:43:45.590 | INFO | ------------------------------------------ | 20260507 05:43:45.590 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:43:45.590 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:43:45.590 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:43:45.590 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:43:45.590 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.34e+01 / 1.25e+00 | 20260507 05:43:45.590 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.02e-01, 9.43e-03 / 3.79e-03 | 20260507 05:43:45.590 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.50e+00, 5.10e-01 / 2.26e-01 | 20260507 05:43:45.590 | INFO | PerfReport: 3.22%, Histogram -- 9.90e-01 / 1.50e+00, 1.68e-01 / 9.45e-02 | 20260507 05:43:45.590 | INFO | PerfReport: 16.23%, DME -- 3.99e+01 / 4.49e+01, 3.97e+00 / 1.58e+00 | 20260507 05:43:45.590 | INFO | PerfReport: 57.64%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.25e+00 / 1.18e+00 | 20260507 05:43:45.590 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.13e+00 / 2.68e+00, 3.88e-01 / 1.71e-01 | 20260507 05:43:45.590 | INFO | PerfReport: 2.99%, Metrics -- 9.18e-01 / 1.23e+00, 2.46e-01 / 1.28e-01 | 20260507 05:43:45.590 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.08e-01 / 3.63e-01, 5.13e-02 / 2.07e-02 | 20260507 05:43:45.590 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:43:45.590 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:43:45.590 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:43:45.590 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 05:43:45.590 | INFO | PerfReport: 21.81%, Upload -- 2.10e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260507 05:43:45.590 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 05:43:45.590 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 05:43:45.590 | INFO | PerfReport: 1.53%, LockUpload -- 1.47e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 05:43:45.590 | INFO | PerfReport: 1.26%, LockDownload -- 1.21e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260507 05:43:45.590 | INFO | PerfReport: 22.08%, LockCompute -- 2.12e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 05:43:45.590 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.09e-02 / 0.00e+00 | 20260507 05:43:45.686 | INFO | Starting a round of Baz IO | 20260507 05:43:48.785 | INFO | Populating RTMetrics report | 20260507 05:43:48.785 | INFO | RTMetrics report populated | 20260507 05:43:48.786 | INFO | Opening tmp RTMetrics json file | 20260507 05:43:48.802 | INFO | Serializing RTMetrics to json | 20260507 05:43:48.803 | INFO | Serialization done, swapping tmp file now... | 20260507 05:43:48.804 | INFO | RTMetrics done | 20260507 05:43:48.871 | INFO | Finished a round of Baz IO: Wrote 1.685 GiB in 3.185 seconds | 20260507 05:43:48.871 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.68% / 0.00% / 67.32% | 20260507 05:43:48.871 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.137472% / 0% / 6.35783e-05% | 20260507 05:43:48.871 | INFO | Baz IO overall: | 20260507 05:43:48.871 | INFO | BytesWritten : 3821806867081 | 20260507 05:43:48.871 | INFO | NumEvents : 1832693533279 | 20260507 05:43:50.731 | INFO | Analyzing chunk frames = [7544832,7545344) | 20260507 05:43:50.731 | INFO | MemInfo: { | 20260507 05:43:50.731 | INFO | "gpu" : " 17 GB", | 20260507 05:43:50.731 | INFO | "huge" : "", | 20260507 05:43:50.732 | INFO | "huge_shared" : " 27 GB", | 20260507 05:43:50.732 | INFO | "malloc" : " 14 GB", | 20260507 05:43:50.732 | INFO | "pinned" : " 29 GB" | 20260507 05:43:50.732 | INFO | } | 20260507 05:43:50.732 | INFO | Perf report for the last 5.14311 seconds: | 20260507 05:43:50.732 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:43:50.732 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:50.732 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:50.732 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:50.732 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 05:43:50.732 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:50.732 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 05:43:50.732 | INFO | PRE_HQ : 59.0%, 1.8, 55.0ms, 0 | 20260507 05:43:50.732 | INFO | BAZWRITER : 61.5%, 3.2, 100.1ms, 0 | 20260507 05:43:50.732 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:43:50.732 | INFO | Uploaded 36GiB to GPU in 2.03505s (17.69GiB/s) | 20260507 05:43:50.732 | INFO | Downloaded 38GiB to GPU in 2.18816s (17.3662GiB/s) | 20260507 05:43:55.872 | INFO | Analyzing chunk frames = [7545344,7545856) | 20260507 05:43:55.872 | INFO | MemInfo: { | 20260507 05:43:55.872 | INFO | "gpu" : " 17 GB", | 20260507 05:43:55.872 | INFO | "huge" : "", | 20260507 05:43:55.872 | INFO | "huge_shared" : " 27 GB", | 20260507 05:43:55.872 | INFO | "malloc" : " 14 GB", | 20260507 05:43:55.872 | INFO | "pinned" : " 29 GB" | 20260507 05:43:55.872 | INFO | } | 20260507 05:43:55.872 | INFO | Perf report for the last 5.14067 seconds: | 20260507 05:43:55.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:43:55.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:55.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:55.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:55.872 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:43:55.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:55.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:55.872 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:43:55.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:55.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:43:55.873 | INFO | Uploaded 36GiB to GPU in 2.00539s (17.9516GiB/s) | 20260507 05:43:55.873 | INFO | Downloaded 34GiB to GPU in 1.92812s (17.6338GiB/s) | 20260507 05:44:01.012 | INFO | Analyzing chunk frames = [7545856,7546368) | 20260507 05:44:01.012 | INFO | MemInfo: { | 20260507 05:44:01.012 | INFO | "gpu" : " 17 GB", | 20260507 05:44:01.012 | INFO | "huge" : "", | 20260507 05:44:01.012 | INFO | "huge_shared" : " 27 GB", | 20260507 05:44:01.012 | INFO | "malloc" : " 14 GB", | 20260507 05:44:01.012 | INFO | "pinned" : " 29 GB" | 20260507 05:44:01.012 | INFO | } | 20260507 05:44:01.012 | INFO | Perf report for the last 5.13998 seconds: | 20260507 05:44:01.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:44:01.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:01.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:01.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:01.012 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260507 05:44:01.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:01.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:01.012 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:44:01.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:01.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:01.012 | INFO | Uploaded 36GiB to GPU in 2.05147s (17.5484GiB/s) | 20260507 05:44:01.012 | INFO | Downloaded 34GiB to GPU in 1.94975s (17.4381GiB/s) | 20260507 05:44:06.150 | INFO | Analyzing chunk frames = [7546368,7546880) | 20260507 05:44:06.150 | INFO | MemInfo: { | 20260507 05:44:06.150 | INFO | "gpu" : " 17 GB", | 20260507 05:44:06.150 | INFO | "huge" : "", | 20260507 05:44:06.151 | INFO | "huge_shared" : " 27 GB", | 20260507 05:44:06.151 | INFO | "malloc" : " 14 GB", | 20260507 05:44:06.151 | INFO | "pinned" : " 29 GB" | 20260507 05:44:06.151 | INFO | } | 20260507 05:44:06.151 | INFO | Perf report for the last 5.13842 seconds: | 20260507 05:44:06.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:44:06.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:06.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:06.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:06.151 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 05:44:06.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:06.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:06.151 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:44:06.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:06.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:06.151 | INFO | Uploaded 36GiB to GPU in 2.05309s (17.5345GiB/s) | 20260507 05:44:06.151 | INFO | Downloaded 34GiB to GPU in 1.94666s (17.4658GiB/s) | 20260507 05:44:11.283 | INFO | Analyzing chunk frames = [7546880,7547392) | 20260507 05:44:11.283 | INFO | MemInfo: { | 20260507 05:44:11.283 | INFO | "gpu" : " 17 GB", | 20260507 05:44:11.283 | INFO | "huge" : "", | 20260507 05:44:11.283 | INFO | "huge_shared" : " 27 GB", | 20260507 05:44:11.283 | INFO | "malloc" : " 14 GB", | 20260507 05:44:11.283 | INFO | "pinned" : " 29 GB" | 20260507 05:44:11.283 | INFO | } | 20260507 05:44:11.283 | INFO | Perf report for the last 5.1328 seconds: | 20260507 05:44:11.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:44:11.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:11.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:11.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:11.283 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 05:44:11.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:11.284 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:11.284 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:44:11.284 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:11.284 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:11.284 | INFO | Uploaded 36GiB to GPU in 2.0484s (17.5747GiB/s) | 20260507 05:44:11.284 | INFO | Downloaded 34GiB to GPU in 1.90675s (17.8314GiB/s) | 20260507 05:44:16.429 | INFO | Analyzing chunk frames = [7547392,7547904) | 20260507 05:44:16.429 | INFO | MemInfo: { | 20260507 05:44:16.429 | INFO | "gpu" : " 17 GB", | 20260507 05:44:16.429 | INFO | "huge" : "", | 20260507 05:44:16.429 | INFO | "huge_shared" : " 27 GB", | 20260507 05:44:16.429 | INFO | "malloc" : " 14 GB", | 20260507 05:44:16.429 | INFO | "pinned" : " 29 GB" | 20260507 05:44:16.429 | INFO | } | 20260507 05:44:16.429 | INFO | Perf report for the last 5.14554 seconds: | 20260507 05:44:16.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:44:16.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:16.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:16.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:16.429 | INFO | ANALYSIS : 60.3%, 3.0, 94.3ms, 0 | 20260507 05:44:16.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:16.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:16.429 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:44:16.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:16.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:16.429 | INFO | Uploaded 36GiB to GPU in 2.0338s (17.7009GiB/s) | 20260507 05:44:16.429 | INFO | Downloaded 34GiB to GPU in 1.92504s (17.662GiB/s) | 20260507 05:44:21.567 | INFO | Analyzing chunk frames = [7547904,7548416) | 20260507 05:44:21.567 | INFO | MemInfo: { | 20260507 05:44:21.567 | INFO | "gpu" : " 17 GB", | 20260507 05:44:21.567 | INFO | "huge" : "", | 20260507 05:44:21.567 | INFO | "huge_shared" : " 27 GB", | 20260507 05:44:21.567 | INFO | "malloc" : " 14 GB", | 20260507 05:44:21.567 | INFO | "pinned" : " 29 GB" | 20260507 05:44:21.567 | INFO | } | 20260507 05:44:21.567 | INFO | Perf report for the last 5.1383 seconds: | 20260507 05:44:21.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:44:21.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:21.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:21.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:21.567 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:44:21.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:21.567 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:21.567 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 05:44:21.567 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:21.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:21.568 | INFO | Uploaded 36GiB to GPU in 1.97285s (18.2477GiB/s) | 20260507 05:44:21.568 | INFO | Downloaded 34GiB to GPU in 1.97638s (17.2031GiB/s) | 20260507 05:44:26.711 | INFO | Analyzing chunk frames = [7548416,7548928) | 20260507 05:44:26.711 | INFO | MemInfo: { | 20260507 05:44:26.711 | INFO | "gpu" : " 17 GB", | 20260507 05:44:26.711 | INFO | "huge" : "", | 20260507 05:44:26.711 | INFO | "huge_shared" : " 27 GB", | 20260507 05:44:26.711 | INFO | "malloc" : " 14 GB", | 20260507 05:44:26.711 | INFO | "pinned" : " 29 GB" | 20260507 05:44:26.711 | INFO | } | 20260507 05:44:26.711 | INFO | Perf report for the last 5.14356 seconds: | 20260507 05:44:26.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:44:26.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:26.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:26.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:26.711 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:44:26.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:26.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:26.711 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 05:44:26.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:26.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:26.711 | INFO | Uploaded 36GiB to GPU in 1.99691s (18.0279GiB/s) | 20260507 05:44:26.711 | INFO | Downloaded 34GiB to GPU in 1.96752s (17.2806GiB/s) | 20260507 05:44:26.808 | INFO | Starting a round of Baz IO | 20260507 05:44:29.903 | INFO | Populating RTMetrics report | 20260507 05:44:29.903 | INFO | RTMetrics report populated | 20260507 05:44:29.903 | INFO | Opening tmp RTMetrics json file | 20260507 05:44:29.912 | INFO | Serializing RTMetrics to json | 20260507 05:44:29.919 | INFO | Serialization done, swapping tmp file now... | 20260507 05:44:29.920 | INFO | RTMetrics done | 20260507 05:44:29.992 | INFO | Finished a round of Baz IO: Wrote 1.686 GiB in 3.184 seconds | 20260507 05:44:29.992 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.68% / 0.00% / 67.32% | 20260507 05:44:29.992 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.13688% / 0% / 7.15256e-05% | 20260507 05:44:29.992 | INFO | Baz IO overall: | 20260507 05:44:29.992 | INFO | BytesWritten : 3823617303106 | 20260507 05:44:29.992 | INFO | NumEvents : 1833542696497 | 20260507 05:44:31.845 | INFO | Analyzing chunk frames = [7548928,7549440) | 20260507 05:44:31.845 | INFO | MemInfo: { | 20260507 05:44:31.845 | INFO | "gpu" : " 17 GB", | 20260507 05:44:31.845 | INFO | "huge" : "", | 20260507 05:44:31.845 | INFO | "huge_shared" : " 27 GB", | 20260507 05:44:31.845 | INFO | "malloc" : " 14 GB", | 20260507 05:44:31.845 | INFO | "pinned" : " 29 GB" | 20260507 05:44:31.845 | INFO | } | 20260507 05:44:31.845 | INFO | Perf report for the last 5.13447 seconds: | 20260507 05:44:31.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:44:31.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:31.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:31.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:31.845 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 05:44:31.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:31.846 | INFO | RT_METRICS : 42.0%, 1.0, 22.0ms, 0 | 20260507 05:44:31.846 | INFO | PRE_HQ : 59.2%, 1.8, 55.5ms, 0 | 20260507 05:44:31.846 | INFO | BAZWRITER : 61.6%, 3.2, 100.8ms, 0 | 20260507 05:44:31.846 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:44:31.846 | INFO | Uploaded 36GiB to GPU in 2.06324s (17.4483GiB/s) | 20260507 05:44:31.846 | INFO | Downloaded 38GiB to GPU in 2.18738s (17.3724GiB/s) | 20260507 05:44:36.980 | INFO | Analyzing chunk frames = [7549440,7549952) | 20260507 05:44:36.980 | INFO | MemInfo: { | 20260507 05:44:36.980 | INFO | "gpu" : " 17 GB", | 20260507 05:44:36.980 | INFO | "huge" : "", | 20260507 05:44:36.980 | INFO | "huge_shared" : " 27 GB", | 20260507 05:44:36.980 | INFO | "malloc" : " 14 GB", | 20260507 05:44:36.980 | INFO | "pinned" : " 29 GB" | 20260507 05:44:36.980 | INFO | } | 20260507 05:44:36.980 | INFO | Perf report for the last 5.13483 seconds: | 20260507 05:44:36.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:44:36.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:36.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:36.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:36.980 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:44:36.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:36.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:36.980 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:44:36.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:36.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:36.980 | INFO | Uploaded 36GiB to GPU in 2.00327s (17.9706GiB/s) | 20260507 05:44:36.980 | INFO | Downloaded 34GiB to GPU in 1.9283s (17.6322GiB/s) | 20260507 05:44:42.126 | INFO | Analyzing chunk frames = [7549952,7550464) | 20260507 05:44:42.126 | INFO | MemInfo: { | 20260507 05:44:42.126 | INFO | "gpu" : " 17 GB", | 20260507 05:44:42.126 | INFO | "huge" : "", | 20260507 05:44:42.126 | INFO | "huge_shared" : " 27 GB", | 20260507 05:44:42.126 | INFO | "malloc" : " 14 GB", | 20260507 05:44:42.126 | INFO | "pinned" : " 29 GB" | 20260507 05:44:42.126 | INFO | } | 20260507 05:44:42.126 | INFO | Perf report for the last 5.14614 seconds: | 20260507 05:44:42.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:44:42.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:42.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:42.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:42.126 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 05:44:42.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:42.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:42.126 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:44:42.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:42.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:42.127 | INFO | Uploaded 36GiB to GPU in 2.0092s (17.9176GiB/s) | 20260507 05:44:42.127 | INFO | Downloaded 34GiB to GPU in 1.94116s (17.5153GiB/s) | 20260507 05:44:47.267 | INFO | Analyzing chunk frames = [7550464,7550976) | 20260507 05:44:47.267 | INFO | MemInfo: { | 20260507 05:44:47.267 | INFO | "gpu" : " 17 GB", | 20260507 05:44:47.267 | INFO | "huge" : "", | 20260507 05:44:47.267 | INFO | "huge_shared" : " 27 GB", | 20260507 05:44:47.267 | INFO | "malloc" : " 14 GB", | 20260507 05:44:47.267 | INFO | "pinned" : " 29 GB" | 20260507 05:44:47.267 | INFO | } | 20260507 05:44:47.267 | INFO | Perf report for the last 5.14094 seconds: | 20260507 05:44:47.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:44:47.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:47.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:47.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:47.267 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 05:44:47.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:47.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:47.267 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:44:47.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:47.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:47.268 | INFO | Uploaded 36GiB to GPU in 2.01272s (17.8863GiB/s) | 20260507 05:44:47.268 | INFO | Downloaded 34GiB to GPU in 1.94204s (17.5074GiB/s) | 20260507 05:44:52.397 | INFO | Analyzing chunk frames = [7550976,7551488) | 20260507 05:44:52.397 | INFO | MemInfo: { | 20260507 05:44:52.397 | INFO | "gpu" : " 17 GB", | 20260507 05:44:52.397 | INFO | "huge" : "", | 20260507 05:44:52.397 | INFO | "huge_shared" : " 27 GB", | 20260507 05:44:52.398 | INFO | "malloc" : " 14 GB", | 20260507 05:44:52.398 | INFO | "pinned" : " 29 GB" | 20260507 05:44:52.398 | INFO | } | 20260507 05:44:52.398 | INFO | Perf report for the last 5.13022 seconds: | 20260507 05:44:52.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:44:52.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:52.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:52.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:52.398 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 05:44:52.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:52.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:52.398 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:44:52.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:52.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:52.398 | INFO | Uploaded 36GiB to GPU in 2.03591s (17.6825GiB/s) | 20260507 05:44:52.398 | INFO | Downloaded 34GiB to GPU in 1.91176s (17.7847GiB/s) | 20260507 05:44:57.535 | INFO | Analyzing chunk frames = [7551488,7552000) | 20260507 05:44:57.535 | INFO | MemInfo: { | 20260507 05:44:57.535 | INFO | "gpu" : " 17 GB", | 20260507 05:44:57.535 | INFO | "huge" : "", | 20260507 05:44:57.535 | INFO | "huge_shared" : " 27 GB", | 20260507 05:44:57.535 | INFO | "malloc" : " 14 GB", | 20260507 05:44:57.535 | INFO | "pinned" : " 29 GB" | 20260507 05:44:57.535 | INFO | } | 20260507 05:44:57.536 | INFO | Perf report for the last 5.13791 seconds: | 20260507 05:44:57.536 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:44:57.536 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:57.536 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:57.536 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:57.536 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:44:57.536 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:57.536 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:57.536 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:44:57.536 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:57.536 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:44:57.536 | INFO | Uploaded 36GiB to GPU in 2.05694s (17.5018GiB/s) | 20260507 05:44:57.536 | INFO | Downloaded 34GiB to GPU in 1.95923s (17.3538GiB/s) | 20260507 05:45:02.688 | INFO | Analyzing chunk frames = [7552000,7552512) | 20260507 05:45:02.688 | INFO | MemInfo: { | 20260507 05:45:02.688 | INFO | "gpu" : " 17 GB", | 20260507 05:45:02.688 | INFO | "huge" : "", | 20260507 05:45:02.689 | INFO | "huge_shared" : " 27 GB", | 20260507 05:45:02.689 | INFO | "malloc" : " 14 GB", | 20260507 05:45:02.689 | INFO | "pinned" : " 29 GB" | 20260507 05:45:02.689 | INFO | } | 20260507 05:45:02.689 | INFO | Perf report for the last 5.1531 seconds: | 20260507 05:45:02.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:45:02.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:02.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:02.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:02.689 | INFO | ANALYSIS : 60.6%, 3.0, 94.7ms, 0 | 20260507 05:45:02.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:02.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:02.689 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 05:45:02.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:02.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:02.689 | INFO | Uploaded 36GiB to GPU in 1.97257s (18.2503GiB/s) | 20260507 05:45:02.689 | INFO | Downloaded 34GiB to GPU in 1.96121s (17.3363GiB/s) | 20260507 05:45:07.824 | INFO | Analyzing chunk frames = [7552512,7553024) | 20260507 05:45:07.824 | INFO | MemInfo: { | 20260507 05:45:07.824 | INFO | "gpu" : " 17 GB", | 20260507 05:45:07.824 | INFO | "huge" : "", | 20260507 05:45:07.824 | INFO | "huge_shared" : " 27 GB", | 20260507 05:45:07.824 | INFO | "malloc" : " 14 GB", | 20260507 05:45:07.824 | INFO | "pinned" : " 29 GB" | 20260507 05:45:07.824 | INFO | } | 20260507 05:45:07.825 | INFO | Perf report for the last 5.13588 seconds: | 20260507 05:45:07.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:45:07.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:07.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:07.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:07.825 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:45:07.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:07.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:07.825 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:45:07.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:07.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:07.825 | INFO | Uploaded 36GiB to GPU in 1.99091s (18.0821GiB/s) | 20260507 05:45:07.825 | INFO | Downloaded 34GiB to GPU in 1.95775s (17.3669GiB/s) | 20260507 05:45:07.825 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:45:07.825 | INFO | Total mem usage: 18 GiB | 20260507 05:45:07.825 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:45:07.825 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 05:45:07.825 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:45:07.825 | INFO | High Water Breakdown: | 20260507 05:45:07.825 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:45:07.825 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:45:07.825 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:45:07.825 | INFO | ------------------------------------------ | 20260507 05:45:07.825 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:45:07.825 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:45:07.825 | INFO | Total mem usage: 26.5 GiB | 20260507 05:45:07.825 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:45:07.825 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:45:07.825 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:45:07.825 | INFO | High Water Breakdown: | 20260507 05:45:07.825 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:45:07.825 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:45:07.825 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:45:07.825 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:45:07.825 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:45:07.825 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:45:07.825 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:45:07.825 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:45:07.825 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:45:07.826 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:45:07.826 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:45:07.826 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:45:07.826 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:45:07.826 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:45:07.826 | INFO | ------------------------------------------ | 20260507 05:45:07.826 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:45:07.826 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:45:07.826 | INFO | Total mem usage: 25.6 GiB | 20260507 05:45:07.826 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:45:07.826 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:45:07.826 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:45:07.826 | INFO | High Water Breakdown: | 20260507 05:45:07.826 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:45:07.826 | INFO | ------------------------------------------ | 20260507 05:45:07.826 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:45:07.826 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:45:07.826 | INFO | Total mem usage: 15.7 GiB | 20260507 05:45:07.826 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:45:07.826 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:45:07.826 | INFO | Unused Cache: 858 MiB (Waste: 5.35%) | 20260507 05:45:07.826 | INFO | High Water Breakdown: | 20260507 05:45:07.826 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:45:07.826 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:45:07.826 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:45:07.826 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:45:07.826 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:45:07.826 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:45:07.826 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:45:07.826 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:45:07.826 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:45:07.826 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:45:07.826 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:45:07.826 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:45:07.826 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:45:07.826 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:45:07.826 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:45:07.826 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:45:07.826 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:45:07.826 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:45:07.826 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:45:07.826 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:45:07.826 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:45:07.826 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:45:07.826 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:45:07.826 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:45:07.826 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:45:07.826 | INFO | ------------------------------------------ | 20260507 05:45:07.826 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:45:07.826 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:45:07.826 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:45:07.826 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:45:07.826 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.33e+01 / 1.02e+00 | 20260507 05:45:07.826 | INFO | PerfReport: 0.29%, AggregateStats -- 9.02e-02 / 9.97e-02, 1.04e-02 / 5.07e-03 | 20260507 05:45:07.826 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.40e+00, 5.03e-01 / 3.32e-01 | 20260507 05:45:07.826 | INFO | PerfReport: 3.22%, Histogram -- 9.87e-01 / 1.45e+00, 1.65e-01 / 1.39e-01 | 20260507 05:45:07.826 | INFO | PerfReport: 16.19%, DME -- 3.98e+01 / 4.45e+01, 3.74e+00 / 1.46e+00 | 20260507 05:45:07.826 | INFO | PerfReport: 57.75%, FrameLabeling -- 1.77e+01 / 1.98e+01, 1.31e+00 / 1.38e+00 | 20260507 05:45:07.826 | INFO | PerfReport: 6.91%, PulseAccumulating -- 2.12e+00 / 2.70e+00, 3.76e-01 / 1.97e-01 | 20260507 05:45:07.826 | INFO | PerfReport: 2.97%, Metrics -- 9.12e-01 / 1.25e+00, 2.43e-01 / 1.29e-01 | 20260507 05:45:07.826 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.08e-01 / 3.65e-01, 5.10e-02 / 1.96e-02 | 20260507 05:45:07.826 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:45:07.826 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:45:07.826 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:45:07.826 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 05:45:07.826 | INFO | PerfReport: 21.89%, Upload -- 2.11e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260507 05:45:07.826 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 05:45:07.826 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 05:45:07.826 | INFO | PerfReport: 1.63%, LockUpload -- 1.57e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 05:45:07.826 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260507 05:45:07.826 | INFO | PerfReport: 21.96%, LockCompute -- 2.11e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 05:45:07.826 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.40e-02 / 0.00e+00 | 20260507 05:45:07.922 | INFO | Starting a round of Baz IO | 20260507 05:45:11.018 | INFO | Populating RTMetrics report | 20260507 05:45:11.018 | INFO | RTMetrics report populated | 20260507 05:45:11.018 | INFO | Opening tmp RTMetrics json file | 20260507 05:45:11.035 | INFO | Serializing RTMetrics to json | 20260507 05:45:11.036 | INFO | Serialization done, swapping tmp file now... | 20260507 05:45:11.036 | INFO | RTMetrics done | 20260507 05:45:11.103 | INFO | Finished a round of Baz IO: Wrote 1.687 GiB in 3.181 seconds | 20260507 05:45:11.103 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.68% / 0.00% / 67.32% | 20260507 05:45:11.103 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.137353% / 0% / 8.74201e-05% | 20260507 05:45:11.103 | INFO | Baz IO overall: | 20260507 05:45:11.103 | INFO | BytesWritten : 3825428985393 | 20260507 05:45:11.103 | INFO | NumEvents : 1834392898458 | 20260507 05:45:12.963 | INFO | Analyzing chunk frames = [7553024,7553536) | 20260507 05:45:12.963 | INFO | MemInfo: { | 20260507 05:45:12.963 | INFO | "gpu" : " 17 GB", | 20260507 05:45:12.963 | INFO | "huge" : "", | 20260507 05:45:12.963 | INFO | "huge_shared" : " 27 GB", | 20260507 05:45:12.963 | INFO | "malloc" : " 14 GB", | 20260507 05:45:12.963 | INFO | "pinned" : " 29 GB" | 20260507 05:45:12.963 | INFO | } | 20260507 05:45:12.963 | INFO | Perf report for the last 5.13889 seconds: | 20260507 05:45:12.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:45:12.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:12.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:12.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:12.964 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:45:12.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:12.964 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 05:45:12.964 | INFO | PRE_HQ : 59.1%, 1.8, 55.1ms, 0 | 20260507 05:45:12.964 | INFO | BAZWRITER : 61.5%, 3.2, 100.4ms, 0 | 20260507 05:45:12.964 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 05:45:12.964 | INFO | Uploaded 36GiB to GPU in 2.05534s (17.5154GiB/s) | 20260507 05:45:12.964 | INFO | Downloaded 38GiB to GPU in 2.17198s (17.4955GiB/s) | 20260507 05:45:18.095 | INFO | Analyzing chunk frames = [7553536,7554048) | 20260507 05:45:18.095 | INFO | MemInfo: { | 20260507 05:45:18.095 | INFO | "gpu" : " 17 GB", | 20260507 05:45:18.095 | INFO | "huge" : "", | 20260507 05:45:18.095 | INFO | "huge_shared" : " 27 GB", | 20260507 05:45:18.095 | INFO | "malloc" : " 14 GB", | 20260507 05:45:18.095 | INFO | "pinned" : " 29 GB" | 20260507 05:45:18.095 | INFO | } | 20260507 05:45:18.095 | INFO | Perf report for the last 5.13137 seconds: | 20260507 05:45:18.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:45:18.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:18.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:18.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:18.095 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 05:45:18.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:18.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:18.095 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:45:18.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:18.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:18.095 | INFO | Uploaded 36GiB to GPU in 1.99607s (18.0354GiB/s) | 20260507 05:45:18.095 | INFO | Downloaded 34GiB to GPU in 1.9291s (17.6248GiB/s) | 20260507 05:45:23.237 | INFO | Analyzing chunk frames = [7554048,7554560) | 20260507 05:45:23.237 | INFO | MemInfo: { | 20260507 05:45:23.237 | INFO | "gpu" : " 17 GB", | 20260507 05:45:23.237 | INFO | "huge" : "", | 20260507 05:45:23.237 | INFO | "huge_shared" : " 27 GB", | 20260507 05:45:23.237 | INFO | "malloc" : " 14 GB", | 20260507 05:45:23.237 | INFO | "pinned" : " 29 GB" | 20260507 05:45:23.237 | INFO | } | 20260507 05:45:23.237 | INFO | Perf report for the last 5.14233 seconds: | 20260507 05:45:23.237 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:45:23.237 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:23.237 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:23.237 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:23.237 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 05:45:23.237 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:23.237 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:23.237 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:45:23.237 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:23.237 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:23.238 | INFO | Uploaded 36GiB to GPU in 2.02105s (17.8125GiB/s) | 20260507 05:45:23.238 | INFO | Downloaded 34GiB to GPU in 1.93724s (17.5508GiB/s) | 20260507 05:45:28.377 | INFO | Analyzing chunk frames = [7554560,7555072) | 20260507 05:45:28.377 | INFO | MemInfo: { | 20260507 05:45:28.377 | INFO | "gpu" : " 17 GB", | 20260507 05:45:28.377 | INFO | "huge" : "", | 20260507 05:45:28.377 | INFO | "huge_shared" : " 27 GB", | 20260507 05:45:28.377 | INFO | "malloc" : " 14 GB", | 20260507 05:45:28.377 | INFO | "pinned" : " 29 GB" | 20260507 05:45:28.377 | INFO | } | 20260507 05:45:28.377 | INFO | Perf report for the last 5.1399 seconds: | 20260507 05:45:28.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:45:28.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:28.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:28.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:28.377 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 05:45:28.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:28.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:28.377 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:45:28.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:28.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:28.377 | INFO | Uploaded 36GiB to GPU in 2.01952s (17.8261GiB/s) | 20260507 05:45:28.377 | INFO | Downloaded 34GiB to GPU in 1.93649s (17.5575GiB/s) | 20260507 05:45:33.523 | INFO | Analyzing chunk frames = [7555072,7555584) | 20260507 05:45:33.523 | INFO | MemInfo: { | 20260507 05:45:33.523 | INFO | "gpu" : " 17 GB", | 20260507 05:45:33.523 | INFO | "huge" : "", | 20260507 05:45:33.523 | INFO | "huge_shared" : " 27 GB", | 20260507 05:45:33.523 | INFO | "malloc" : " 14 GB", | 20260507 05:45:33.523 | INFO | "pinned" : " 29 GB" | 20260507 05:45:33.523 | INFO | } | 20260507 05:45:33.523 | INFO | Perf report for the last 5.14626 seconds: | 20260507 05:45:33.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:45:33.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:33.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:33.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:33.523 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 05:45:33.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:33.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:33.523 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:45:33.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:33.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:33.524 | INFO | Uploaded 36GiB to GPU in 2.05353s (17.5308GiB/s) | 20260507 05:45:33.524 | INFO | Downloaded 34GiB to GPU in 1.93453s (17.5753GiB/s) | 20260507 05:45:38.659 | INFO | Analyzing chunk frames = [7555584,7556096) | 20260507 05:45:38.659 | INFO | MemInfo: { | 20260507 05:45:38.659 | INFO | "gpu" : " 17 GB", | 20260507 05:45:38.659 | INFO | "huge" : "", | 20260507 05:45:38.659 | INFO | "huge_shared" : " 27 GB", | 20260507 05:45:38.659 | INFO | "malloc" : " 14 GB", | 20260507 05:45:38.659 | INFO | "pinned" : " 29 GB" | 20260507 05:45:38.659 | INFO | } | 20260507 05:45:38.659 | INFO | Perf report for the last 5.1361 seconds: | 20260507 05:45:38.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:45:38.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:38.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:38.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:38.659 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 05:45:38.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:38.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:38.659 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 05:45:38.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:38.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:38.660 | INFO | Uploaded 36GiB to GPU in 2.07675s (17.3348GiB/s) | 20260507 05:45:38.660 | INFO | Downloaded 34GiB to GPU in 1.94553s (17.476GiB/s) | 20260507 05:45:43.797 | INFO | Analyzing chunk frames = [7556096,7556608) | 20260507 05:45:43.797 | INFO | MemInfo: { | 20260507 05:45:43.797 | INFO | "gpu" : " 17 GB", | 20260507 05:45:43.797 | INFO | "huge" : "", | 20260507 05:45:43.797 | INFO | "huge_shared" : " 27 GB", | 20260507 05:45:43.797 | INFO | "malloc" : " 14 GB", | 20260507 05:45:43.797 | INFO | "pinned" : " 29 GB" | 20260507 05:45:43.798 | INFO | } | 20260507 05:45:43.798 | INFO | Perf report for the last 5.13814 seconds: | 20260507 05:45:43.798 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:45:43.798 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:43.798 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:43.798 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:43.798 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 05:45:43.798 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:43.798 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:43.798 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 05:45:43.798 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:43.798 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:43.798 | INFO | Uploaded 36GiB to GPU in 2.03964s (17.6502GiB/s) | 20260507 05:45:43.798 | INFO | Downloaded 34GiB to GPU in 1.99374s (17.0534GiB/s) | 20260507 05:45:48.943 | INFO | Analyzing chunk frames = [7556608,7557120) | 20260507 05:45:48.943 | INFO | MemInfo: { | 20260507 05:45:48.943 | INFO | "gpu" : " 17 GB", | 20260507 05:45:48.943 | INFO | "huge" : "", | 20260507 05:45:48.943 | INFO | "huge_shared" : " 27 GB", | 20260507 05:45:48.943 | INFO | "malloc" : " 14 GB", | 20260507 05:45:48.943 | INFO | "pinned" : " 29 GB" | 20260507 05:45:48.943 | INFO | } | 20260507 05:45:48.943 | INFO | Perf report for the last 5.14541 seconds: | 20260507 05:45:48.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:45:48.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:48.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:48.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:48.943 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 05:45:48.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:48.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:48.943 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:45:48.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:48.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:48.943 | INFO | Uploaded 36GiB to GPU in 1.97932s (18.1881GiB/s) | 20260507 05:45:48.943 | INFO | Downloaded 34GiB to GPU in 1.96191s (17.3301GiB/s) | 20260507 05:45:49.040 | INFO | Starting a round of Baz IO | 20260507 05:45:52.136 | INFO | Populating RTMetrics report | 20260507 05:45:52.136 | INFO | RTMetrics report populated | 20260507 05:45:52.136 | INFO | Opening tmp RTMetrics json file | 20260507 05:45:52.143 | INFO | Serializing RTMetrics to json | 20260507 05:45:52.153 | INFO | Serialization done, swapping tmp file now... | 20260507 05:45:52.153 | INFO | RTMetrics done | 20260507 05:45:52.220 | INFO | Finished a round of Baz IO: Wrote 1.687 GiB in 3.179 seconds | 20260507 05:45:52.220 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.68% / 0.00% / 67.32% | 20260507 05:45:52.220 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.136364% / 3.97364e-06% / 5.96046e-05% | 20260507 05:45:52.220 | INFO | Baz IO overall: | 20260507 05:45:52.220 | INFO | BytesWritten : 3827240454938 | 20260507 05:45:52.220 | INFO | NumEvents : 1835242617131 | 20260507 05:45:54.078 | INFO | Analyzing chunk frames = [7557120,7557632) | 20260507 05:45:54.078 | INFO | MemInfo: { | 20260507 05:45:54.078 | INFO | "gpu" : " 17 GB", | 20260507 05:45:54.078 | INFO | "huge" : "", | 20260507 05:45:54.078 | INFO | "huge_shared" : " 27 GB", | 20260507 05:45:54.078 | INFO | "malloc" : " 14 GB", | 20260507 05:45:54.078 | INFO | "pinned" : " 29 GB" | 20260507 05:45:54.078 | INFO | } | 20260507 05:45:54.078 | INFO | Perf report for the last 5.13544 seconds: | 20260507 05:45:54.078 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:45:54.078 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:54.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:54.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:54.079 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 05:45:54.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:54.079 | INFO | RT_METRICS : 41.6%, 1.0, 21.8ms, 0 | 20260507 05:45:54.079 | INFO | PRE_HQ : 58.9%, 1.8, 54.8ms, 0 | 20260507 05:45:54.079 | INFO | BAZWRITER : 61.5%, 3.2, 100.4ms, 0 | 20260507 05:45:54.079 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 05:45:54.079 | INFO | Uploaded 36GiB to GPU in 2.04518s (17.6024GiB/s) | 20260507 05:45:54.079 | INFO | Downloaded 38GiB to GPU in 2.1804s (17.428GiB/s) | 20260507 05:45:59.221 | INFO | Analyzing chunk frames = [7557632,7558144) | 20260507 05:45:59.221 | INFO | MemInfo: { | 20260507 05:45:59.221 | INFO | "gpu" : " 17 GB", | 20260507 05:45:59.221 | INFO | "huge" : "", | 20260507 05:45:59.221 | INFO | "huge_shared" : " 27 GB", | 20260507 05:45:59.221 | INFO | "malloc" : " 14 GB", | 20260507 05:45:59.221 | INFO | "pinned" : " 29 GB" | 20260507 05:45:59.221 | INFO | } | 20260507 05:45:59.221 | INFO | Perf report for the last 5.14269 seconds: | 20260507 05:45:59.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:45:59.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:59.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:59.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:59.221 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:45:59.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:59.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:59.221 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:45:59.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:59.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:45:59.221 | INFO | Uploaded 36GiB to GPU in 1.98954s (18.0946GiB/s) | 20260507 05:45:59.221 | INFO | Downloaded 34GiB to GPU in 1.9334s (17.5856GiB/s) | 20260507 05:46:04.349 | INFO | Analyzing chunk frames = [7558144,7558656) | 20260507 05:46:04.349 | INFO | MemInfo: { | 20260507 05:46:04.349 | INFO | "gpu" : " 17 GB", | 20260507 05:46:04.349 | INFO | "huge" : "", | 20260507 05:46:04.349 | INFO | "huge_shared" : " 27 GB", | 20260507 05:46:04.349 | INFO | "malloc" : " 14 GB", | 20260507 05:46:04.349 | INFO | "pinned" : " 29 GB" | 20260507 05:46:04.349 | INFO | } | 20260507 05:46:04.349 | INFO | Perf report for the last 5.12765 seconds: | 20260507 05:46:04.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:46:04.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:04.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:04.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:04.349 | INFO | ANALYSIS : 60.6%, 3.0, 93.6ms, 0 | 20260507 05:46:04.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:04.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:04.349 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:46:04.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:04.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:04.349 | INFO | Uploaded 36GiB to GPU in 2.04947s (17.5655GiB/s) | 20260507 05:46:04.349 | INFO | Downloaded 34GiB to GPU in 1.95227s (17.4157GiB/s) | 20260507 05:46:09.493 | INFO | Analyzing chunk frames = [7558656,7559168) | 20260507 05:46:09.493 | INFO | MemInfo: { | 20260507 05:46:09.493 | INFO | "gpu" : " 17 GB", | 20260507 05:46:09.493 | INFO | "huge" : "", | 20260507 05:46:09.493 | INFO | "huge_shared" : " 27 GB", | 20260507 05:46:09.493 | INFO | "malloc" : " 14 GB", | 20260507 05:46:09.493 | INFO | "pinned" : " 29 GB" | 20260507 05:46:09.494 | INFO | } | 20260507 05:46:09.494 | INFO | Perf report for the last 5.14475 seconds: | 20260507 05:46:09.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:46:09.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:09.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:09.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:09.494 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 05:46:09.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:09.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:09.494 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:46:09.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:09.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:09.494 | INFO | Uploaded 36GiB to GPU in 1.95762s (18.3897GiB/s) | 20260507 05:46:09.494 | INFO | Downloaded 34GiB to GPU in 1.92474s (17.6647GiB/s) | 20260507 05:46:14.633 | INFO | Analyzing chunk frames = [7559168,7559680) | 20260507 05:46:14.633 | INFO | MemInfo: { | 20260507 05:46:14.633 | INFO | "gpu" : " 17 GB", | 20260507 05:46:14.633 | INFO | "huge" : "", | 20260507 05:46:14.633 | INFO | "huge_shared" : " 27 GB", | 20260507 05:46:14.633 | INFO | "malloc" : " 14 GB", | 20260507 05:46:14.633 | INFO | "pinned" : " 29 GB" | 20260507 05:46:14.633 | INFO | } | 20260507 05:46:14.633 | INFO | Perf report for the last 5.13965 seconds: | 20260507 05:46:14.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:46:14.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:14.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:14.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:14.633 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 05:46:14.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:14.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:14.633 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:46:14.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:14.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:14.634 | INFO | Uploaded 36GiB to GPU in 2.03445s (17.6952GiB/s) | 20260507 05:46:14.634 | INFO | Downloaded 34GiB to GPU in 1.94295s (17.4991GiB/s) | 20260507 05:46:19.775 | INFO | Analyzing chunk frames = [7559680,7560192) | 20260507 05:46:19.775 | INFO | MemInfo: { | 20260507 05:46:19.775 | INFO | "gpu" : " 17 GB", | 20260507 05:46:19.775 | INFO | "huge" : "", | 20260507 05:46:19.775 | INFO | "huge_shared" : " 27 GB", | 20260507 05:46:19.775 | INFO | "malloc" : " 14 GB", | 20260507 05:46:19.775 | INFO | "pinned" : " 29 GB" | 20260507 05:46:19.775 | INFO | } | 20260507 05:46:19.775 | INFO | Perf report for the last 5.14155 seconds: | 20260507 05:46:19.775 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:46:19.775 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:19.775 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:19.775 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:19.775 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 05:46:19.775 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:19.775 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:19.775 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:46:19.775 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:19.775 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:19.775 | INFO | Uploaded 36GiB to GPU in 2.03541s (17.6868GiB/s) | 20260507 05:46:19.775 | INFO | Downloaded 34GiB to GPU in 1.92914s (17.6244GiB/s) | 20260507 05:46:24.908 | INFO | Analyzing chunk frames = [7560192,7560704) | 20260507 05:46:24.908 | INFO | MemInfo: { | 20260507 05:46:24.908 | INFO | "gpu" : " 17 GB", | 20260507 05:46:24.908 | INFO | "huge" : "", | 20260507 05:46:24.909 | INFO | "huge_shared" : " 27 GB", | 20260507 05:46:24.909 | INFO | "malloc" : " 14 GB", | 20260507 05:46:24.909 | INFO | "pinned" : " 29 GB" | 20260507 05:46:24.909 | INFO | } | 20260507 05:46:24.909 | INFO | Perf report for the last 5.1338 seconds: | 20260507 05:46:24.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:46:24.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:24.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:24.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:24.909 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 05:46:24.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:24.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:24.909 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:46:24.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:24.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:24.909 | INFO | Uploaded 36GiB to GPU in 1.92325s (18.7183GiB/s) | 20260507 05:46:24.909 | INFO | Downloaded 34GiB to GPU in 1.98671s (17.1137GiB/s) | 20260507 05:46:30.043 | INFO | Analyzing chunk frames = [7560704,7561216) | 20260507 05:46:30.043 | INFO | MemInfo: { | 20260507 05:46:30.043 | INFO | "gpu" : " 17 GB", | 20260507 05:46:30.043 | INFO | "huge" : "", | 20260507 05:46:30.043 | INFO | "huge_shared" : " 27 GB", | 20260507 05:46:30.043 | INFO | "malloc" : " 14 GB", | 20260507 05:46:30.043 | INFO | "pinned" : " 29 GB" | 20260507 05:46:30.043 | INFO | } | 20260507 05:46:30.043 | INFO | Perf report for the last 5.13469 seconds: | 20260507 05:46:30.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:46:30.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:30.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:30.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:30.043 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 05:46:30.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:30.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:30.043 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:46:30.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:30.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:30.044 | INFO | Uploaded 36GiB to GPU in 1.95961s (18.371GiB/s) | 20260507 05:46:30.044 | INFO | Downloaded 34GiB to GPU in 1.95545s (17.3873GiB/s) | 20260507 05:46:30.044 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:46:30.044 | INFO | Total mem usage: 18 GiB | 20260507 05:46:30.044 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:46:30.044 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 05:46:30.044 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:46:30.044 | INFO | High Water Breakdown: | 20260507 05:46:30.044 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:46:30.044 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:46:30.044 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:46:30.044 | INFO | ------------------------------------------ | 20260507 05:46:30.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:46:30.044 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:46:30.044 | INFO | Total mem usage: 26.5 GiB | 20260507 05:46:30.044 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:46:30.044 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:46:30.044 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:46:30.044 | INFO | High Water Breakdown: | 20260507 05:46:30.044 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:46:30.044 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:46:30.044 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:46:30.044 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:46:30.044 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:46:30.044 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:46:30.044 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:46:30.044 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:46:30.044 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:46:30.044 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:46:30.044 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:46:30.044 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:46:30.044 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:46:30.044 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:46:30.044 | INFO | ------------------------------------------ | 20260507 05:46:30.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:46:30.044 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:46:30.044 | INFO | Total mem usage: 25.6 GiB | 20260507 05:46:30.044 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:46:30.044 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:46:30.044 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:46:30.044 | INFO | High Water Breakdown: | 20260507 05:46:30.044 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:46:30.044 | INFO | ------------------------------------------ | 20260507 05:46:30.044 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:46:30.044 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:46:30.044 | INFO | Total mem usage: 15.7 GiB | 20260507 05:46:30.044 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:46:30.044 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:46:30.044 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:46:30.044 | INFO | High Water Breakdown: | 20260507 05:46:30.044 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:46:30.044 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:46:30.044 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:46:30.044 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:46:30.044 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:46:30.045 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:46:30.045 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:46:30.045 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:46:30.045 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:46:30.045 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:46:30.045 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:46:30.045 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:46:30.045 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:46:30.045 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:46:30.045 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:46:30.045 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:46:30.045 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:46:30.045 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:46:30.045 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:46:30.045 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:46:30.045 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:46:30.045 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:46:30.045 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:46:30.045 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:46:30.045 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:46:30.045 | INFO | ------------------------------------------ | 20260507 05:46:30.045 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:46:30.045 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:46:30.045 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:46:30.045 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:46:30.045 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 9.47e-01 | 20260507 05:46:30.045 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.02e-01, 1.07e-02 / 4.66e-03 | 20260507 05:46:30.045 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.51e+00, 5.13e-01 / 1.95e-01 | 20260507 05:46:30.045 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.47e+00, 1.68e-01 / 8.77e-02 | 20260507 05:46:30.045 | INFO | PerfReport: 16.16%, DME -- 3.97e+01 / 4.42e+01, 3.67e+00 / 1.33e+00 | 20260507 05:46:30.045 | INFO | PerfReport: 57.73%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.34e+00 / 1.38e+00 | 20260507 05:46:30.045 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.66e+00, 3.85e-01 / 2.11e-01 | 20260507 05:46:30.045 | INFO | PerfReport: 2.98%, Metrics -- 9.14e-01 / 1.24e+00, 2.48e-01 / 1.29e-01 | 20260507 05:46:30.045 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.08e-01 / 3.60e-01, 5.09e-02 / 1.81e-02 | 20260507 05:46:30.045 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:46:30.045 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:46:30.045 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:46:30.045 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 05:46:30.045 | INFO | PerfReport: 21.84%, Upload -- 2.10e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260507 05:46:30.045 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 05:46:30.045 | INFO | PerfReport: 31.97%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 05:46:30.045 | INFO | PerfReport: 1.42%, LockUpload -- 1.36e+00 / 0.00e+00, 4.76e+00 / 0.00e+00 | 20260507 05:46:30.045 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260507 05:46:30.045 | INFO | PerfReport: 22.05%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 05:46:30.045 | INFO | PerfReport: 0.02%, Misc -- 1.57e-02 / 0.00e+00, 4.14e-02 / 0.00e+00 | 20260507 05:46:30.141 | INFO | Starting a round of Baz IO | 20260507 05:46:33.240 | INFO | Populating RTMetrics report | 20260507 05:46:33.240 | INFO | RTMetrics report populated | 20260507 05:46:33.240 | INFO | Opening tmp RTMetrics json file | 20260507 05:46:33.258 | INFO | Serializing RTMetrics to json | 20260507 05:46:33.259 | INFO | Serialization done, swapping tmp file now... | 20260507 05:46:33.259 | INFO | RTMetrics done | 20260507 05:46:33.328 | INFO | Finished a round of Baz IO: Wrote 1.685 GiB in 3.186 seconds | 20260507 05:46:33.328 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.68% / 0.00% / 67.32% | 20260507 05:46:33.328 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.13576% / 3.97364e-06% / 9.13938e-05% | 20260507 05:46:33.328 | INFO | Baz IO overall: | 20260507 05:46:33.328 | INFO | BytesWritten : 3829049517391 | 20260507 05:46:33.328 | INFO | NumEvents : 1836090752453 | 20260507 05:46:35.192 | INFO | Analyzing chunk frames = [7561216,7561728) | 20260507 05:46:35.192 | INFO | MemInfo: { | 20260507 05:46:35.192 | INFO | "gpu" : " 17 GB", | 20260507 05:46:35.192 | INFO | "huge" : "", | 20260507 05:46:35.192 | INFO | "huge_shared" : " 27 GB", | 20260507 05:46:35.192 | INFO | "malloc" : " 14 GB", | 20260507 05:46:35.192 | INFO | "pinned" : " 29 GB" | 20260507 05:46:35.192 | INFO | } | 20260507 05:46:35.192 | INFO | Perf report for the last 5.14843 seconds: | 20260507 05:46:35.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:46:35.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:35.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:35.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:35.192 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 05:46:35.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:35.192 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 05:46:35.192 | INFO | PRE_HQ : 58.9%, 1.8, 54.7ms, 0 | 20260507 05:46:35.192 | INFO | BAZWRITER : 61.5%, 3.2, 100.8ms, 0 | 20260507 05:46:35.192 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 05:46:35.192 | INFO | Uploaded 36GiB to GPU in 2.02293s (17.796GiB/s) | 20260507 05:46:35.192 | INFO | Downloaded 38GiB to GPU in 2.1897s (17.354GiB/s) | 20260507 05:46:40.335 | INFO | Analyzing chunk frames = [7561728,7562240) | 20260507 05:46:40.335 | INFO | MemInfo: { | 20260507 05:46:40.335 | INFO | "gpu" : " 17 GB", | 20260507 05:46:40.335 | INFO | "huge" : "", | 20260507 05:46:40.335 | INFO | "huge_shared" : " 27 GB", | 20260507 05:46:40.335 | INFO | "malloc" : " 14 GB", | 20260507 05:46:40.335 | INFO | "pinned" : " 29 GB" | 20260507 05:46:40.336 | INFO | } | 20260507 05:46:40.336 | INFO | Perf report for the last 5.14381 seconds: | 20260507 05:46:40.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:46:40.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:40.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:40.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:40.336 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 05:46:40.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:40.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:40.336 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 05:46:40.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:40.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:40.336 | INFO | Uploaded 36GiB to GPU in 1.98224s (18.1613GiB/s) | 20260507 05:46:40.336 | INFO | Downloaded 34GiB to GPU in 1.93464s (17.5743GiB/s) | 20260507 05:46:45.475 | INFO | Analyzing chunk frames = [7562240,7562752) | 20260507 05:46:45.475 | INFO | MemInfo: { | 20260507 05:46:45.475 | INFO | "gpu" : " 17 GB", | 20260507 05:46:45.475 | INFO | "huge" : "", | 20260507 05:46:45.475 | INFO | "huge_shared" : " 27 GB", | 20260507 05:46:45.475 | INFO | "malloc" : " 14 GB", | 20260507 05:46:45.475 | INFO | "pinned" : " 29 GB" | 20260507 05:46:45.475 | INFO | } | 20260507 05:46:45.476 | INFO | Perf report for the last 5.13993 seconds: | 20260507 05:46:45.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:46:45.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:45.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:45.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:45.476 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 05:46:45.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:45.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:45.476 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 05:46:45.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:45.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:45.476 | INFO | Uploaded 36GiB to GPU in 2.02086s (17.8142GiB/s) | 20260507 05:46:45.476 | INFO | Downloaded 34GiB to GPU in 1.91956s (17.7124GiB/s) | 20260507 05:46:50.603 | INFO | Analyzing chunk frames = [7562752,7563264) | 20260507 05:46:50.603 | INFO | MemInfo: { | 20260507 05:46:50.603 | INFO | "gpu" : " 17 GB", | 20260507 05:46:50.603 | INFO | "huge" : "", | 20260507 05:46:50.603 | INFO | "huge_shared" : " 27 GB", | 20260507 05:46:50.603 | INFO | "malloc" : " 14 GB", | 20260507 05:46:50.603 | INFO | "pinned" : " 29 GB" | 20260507 05:46:50.603 | INFO | } | 20260507 05:46:50.603 | INFO | Perf report for the last 5.12784 seconds: | 20260507 05:46:50.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:46:50.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:50.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:50.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:50.603 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 05:46:50.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:50.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:50.603 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:46:50.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:50.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:50.604 | INFO | Uploaded 36GiB to GPU in 2.05049s (17.5568GiB/s) | 20260507 05:46:50.604 | INFO | Downloaded 34GiB to GPU in 1.95264s (17.4123GiB/s) | 20260507 05:46:55.746 | INFO | Analyzing chunk frames = [7563264,7563776) | 20260507 05:46:55.746 | INFO | MemInfo: { | 20260507 05:46:55.746 | INFO | "gpu" : " 17 GB", | 20260507 05:46:55.746 | INFO | "huge" : "", | 20260507 05:46:55.746 | INFO | "huge_shared" : " 27 GB", | 20260507 05:46:55.746 | INFO | "malloc" : " 14 GB", | 20260507 05:46:55.746 | INFO | "pinned" : " 29 GB" | 20260507 05:46:55.746 | INFO | } | 20260507 05:46:55.746 | INFO | Perf report for the last 5.14294 seconds: | 20260507 05:46:55.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:46:55.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:55.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:55.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:55.746 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 05:46:55.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:55.746 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:55.746 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:46:55.746 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:55.746 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:46:55.747 | INFO | Uploaded 36GiB to GPU in 2.02323s (17.7934GiB/s) | 20260507 05:46:55.747 | INFO | Downloaded 34GiB to GPU in 1.94406s (17.4892GiB/s) | 20260507 05:47:00.886 | INFO | Analyzing chunk frames = [7563776,7564288) | 20260507 05:47:00.886 | INFO | MemInfo: { | 20260507 05:47:00.886 | INFO | "gpu" : " 17 GB", | 20260507 05:47:00.886 | INFO | "huge" : "", | 20260507 05:47:00.886 | INFO | "huge_shared" : " 27 GB", | 20260507 05:47:00.886 | INFO | "malloc" : " 14 GB", | 20260507 05:47:00.886 | INFO | "pinned" : " 29 GB" | 20260507 05:47:00.886 | INFO | } | 20260507 05:47:00.886 | INFO | Perf report for the last 5.14 seconds: | 20260507 05:47:00.886 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:47:00.886 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:00.886 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:00.886 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:00.886 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:47:00.886 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:00.886 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:00.886 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:47:00.886 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:00.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:00.887 | INFO | Uploaded 36GiB to GPU in 2.05063s (17.5556GiB/s) | 20260507 05:47:00.887 | INFO | Downloaded 34GiB to GPU in 1.97315s (17.2313GiB/s) | 20260507 05:47:06.023 | INFO | Analyzing chunk frames = [7564288,7564800) | 20260507 05:47:06.023 | INFO | MemInfo: { | 20260507 05:47:06.023 | INFO | "gpu" : " 17 GB", | 20260507 05:47:06.023 | INFO | "huge" : "", | 20260507 05:47:06.023 | INFO | "huge_shared" : " 27 GB", | 20260507 05:47:06.023 | INFO | "malloc" : " 14 GB", | 20260507 05:47:06.023 | INFO | "pinned" : " 29 GB" | 20260507 05:47:06.023 | INFO | } | 20260507 05:47:06.023 | INFO | Perf report for the last 5.13674 seconds: | 20260507 05:47:06.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:47:06.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:06.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:06.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:06.023 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:47:06.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:06.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:06.023 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:47:06.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:06.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:06.023 | INFO | Uploaded 36GiB to GPU in 1.9979s (18.0189GiB/s) | 20260507 05:47:06.023 | INFO | Downloaded 34GiB to GPU in 1.97196s (17.2417GiB/s) | 20260507 05:47:11.167 | INFO | Analyzing chunk frames = [7564800,7565312) | 20260507 05:47:11.167 | INFO | MemInfo: { | 20260507 05:47:11.167 | INFO | "gpu" : " 17 GB", | 20260507 05:47:11.167 | INFO | "huge" : "", | 20260507 05:47:11.168 | INFO | "huge_shared" : " 27 GB", | 20260507 05:47:11.168 | INFO | "malloc" : " 14 GB", | 20260507 05:47:11.168 | INFO | "pinned" : " 29 GB" | 20260507 05:47:11.168 | INFO | } | 20260507 05:47:11.168 | INFO | Perf report for the last 5.14455 seconds: | 20260507 05:47:11.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:47:11.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:11.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:11.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:11.168 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 05:47:11.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:11.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:11.168 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 05:47:11.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:11.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:11.168 | INFO | Uploaded 36GiB to GPU in 1.96507s (18.3199GiB/s) | 20260507 05:47:11.168 | INFO | Downloaded 34GiB to GPU in 1.95842s (17.361GiB/s) | 20260507 05:47:11.265 | INFO | Starting a round of Baz IO | 20260507 05:47:14.363 | INFO | Populating RTMetrics report | 20260507 05:47:14.364 | INFO | RTMetrics report populated | 20260507 05:47:14.364 | INFO | Opening tmp RTMetrics json file | 20260507 05:47:14.372 | INFO | Serializing RTMetrics to json | 20260507 05:47:14.380 | INFO | Serialization done, swapping tmp file now... | 20260507 05:47:14.380 | INFO | RTMetrics done | 20260507 05:47:14.457 | INFO | Finished a round of Baz IO: Wrote 1.685 GiB in 3.191 seconds | 20260507 05:47:14.457 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.68% / 0.00% / 67.32% | 20260507 05:47:14.457 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.135986% / 0% / 8.34465e-05% | 20260507 05:47:14.457 | INFO | Baz IO overall: | 20260507 05:47:14.457 | INFO | BytesWritten : 3830858279918 | 20260507 05:47:14.457 | INFO | NumEvents : 1836939124705 | 20260507 05:47:16.306 | INFO | Analyzing chunk frames = [7565312,7565824) | 20260507 05:47:16.306 | INFO | MemInfo: { | 20260507 05:47:16.306 | INFO | "gpu" : " 17 GB", | 20260507 05:47:16.306 | INFO | "huge" : "", | 20260507 05:47:16.306 | INFO | "huge_shared" : " 27 GB", | 20260507 05:47:16.306 | INFO | "malloc" : " 14 GB", | 20260507 05:47:16.306 | INFO | "pinned" : " 29 GB" | 20260507 05:47:16.306 | INFO | } | 20260507 05:47:16.306 | INFO | Perf report for the last 5.13879 seconds: | 20260507 05:47:16.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:47:16.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:16.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:16.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:16.307 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:47:16.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:16.307 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 05:47:16.307 | INFO | PRE_HQ : 59.8%, 1.9, 57.6ms, 0 | 20260507 05:47:16.307 | INFO | BAZWRITER : 61.7%, 3.3, 103.5ms, 0 | 20260507 05:47:16.307 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:47:16.307 | INFO | Uploaded 36GiB to GPU in 2.05214s (17.5427GiB/s) | 20260507 05:47:16.307 | INFO | Downloaded 38GiB to GPU in 2.17977s (17.4331GiB/s) | 20260507 05:47:21.449 | INFO | Analyzing chunk frames = [7565824,7566336) | 20260507 05:47:21.449 | INFO | MemInfo: { | 20260507 05:47:21.449 | INFO | "gpu" : " 17 GB", | 20260507 05:47:21.449 | INFO | "huge" : "", | 20260507 05:47:21.449 | INFO | "huge_shared" : " 27 GB", | 20260507 05:47:21.449 | INFO | "malloc" : " 14 GB", | 20260507 05:47:21.449 | INFO | "pinned" : " 29 GB" | 20260507 05:47:21.449 | INFO | } | 20260507 05:47:21.449 | INFO | Perf report for the last 5.14272 seconds: | 20260507 05:47:21.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:47:21.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:21.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:21.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:21.449 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 05:47:21.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:21.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:21.449 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:47:21.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:21.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:21.449 | INFO | Uploaded 36GiB to GPU in 1.98948s (18.0952GiB/s) | 20260507 05:47:21.450 | INFO | Downloaded 34GiB to GPU in 1.93261s (17.5928GiB/s) | 20260507 05:47:26.585 | INFO | Analyzing chunk frames = [7566336,7566848) | 20260507 05:47:26.585 | INFO | MemInfo: { | 20260507 05:47:26.585 | INFO | "gpu" : " 17 GB", | 20260507 05:47:26.585 | INFO | "huge" : "", | 20260507 05:47:26.585 | INFO | "huge_shared" : " 27 GB", | 20260507 05:47:26.585 | INFO | "malloc" : " 14 GB", | 20260507 05:47:26.585 | INFO | "pinned" : " 29 GB" | 20260507 05:47:26.585 | INFO | } | 20260507 05:47:26.585 | INFO | Perf report for the last 5.13551 seconds: | 20260507 05:47:26.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:47:26.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:26.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:26.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:26.585 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 05:47:26.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:26.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:26.585 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:47:26.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:26.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:26.585 | INFO | Uploaded 36GiB to GPU in 2.04507s (17.6033GiB/s) | 20260507 05:47:26.585 | INFO | Downloaded 34GiB to GPU in 1.95554s (17.3865GiB/s) | 20260507 05:47:31.722 | INFO | Analyzing chunk frames = [7566848,7567360) | 20260507 05:47:31.722 | INFO | MemInfo: { | 20260507 05:47:31.722 | INFO | "gpu" : " 17 GB", | 20260507 05:47:31.722 | INFO | "huge" : "", | 20260507 05:47:31.722 | INFO | "huge_shared" : " 27 GB", | 20260507 05:47:31.722 | INFO | "malloc" : " 14 GB", | 20260507 05:47:31.722 | INFO | "pinned" : " 29 GB" | 20260507 05:47:31.722 | INFO | } | 20260507 05:47:31.722 | INFO | Perf report for the last 5.13695 seconds: | 20260507 05:47:31.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:47:31.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:31.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:31.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:31.722 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 05:47:31.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:31.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:31.722 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:47:31.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:31.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:31.722 | INFO | Uploaded 36GiB to GPU in 2.02075s (17.8152GiB/s) | 20260507 05:47:31.722 | INFO | Downloaded 34GiB to GPU in 1.95789s (17.3656GiB/s) | 20260507 05:47:36.856 | INFO | Analyzing chunk frames = [7567360,7567872) | 20260507 05:47:36.856 | INFO | MemInfo: { | 20260507 05:47:36.856 | INFO | "gpu" : " 17 GB", | 20260507 05:47:36.856 | INFO | "huge" : "", | 20260507 05:47:36.856 | INFO | "huge_shared" : " 27 GB", | 20260507 05:47:36.856 | INFO | "malloc" : " 14 GB", | 20260507 05:47:36.856 | INFO | "pinned" : " 29 GB" | 20260507 05:47:36.856 | INFO | } | 20260507 05:47:36.856 | INFO | Perf report for the last 5.13438 seconds: | 20260507 05:47:36.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:47:36.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:36.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:36.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:36.856 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260507 05:47:36.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:36.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:36.856 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:47:36.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:36.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:36.856 | INFO | Uploaded 36GiB to GPU in 2.02735s (17.7572GiB/s) | 20260507 05:47:36.856 | INFO | Downloaded 34GiB to GPU in 1.91918s (17.7159GiB/s) | 20260507 05:47:41.999 | INFO | Analyzing chunk frames = [7567872,7568384) | 20260507 05:47:41.999 | INFO | MemInfo: { | 20260507 05:47:41.999 | INFO | "gpu" : " 17 GB", | 20260507 05:47:41.999 | INFO | "huge" : "", | 20260507 05:47:41.999 | INFO | "huge_shared" : " 27 GB", | 20260507 05:47:41.999 | INFO | "malloc" : " 14 GB", | 20260507 05:47:41.999 | INFO | "pinned" : " 29 GB" | 20260507 05:47:41.999 | INFO | } | 20260507 05:47:41.999 | INFO | Perf report for the last 5.14312 seconds: | 20260507 05:47:41.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:47:41.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:41.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:41.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:41.999 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 05:47:41.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:41.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:41.999 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 05:47:41.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:41.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:41.999 | INFO | Uploaded 36GiB to GPU in 2.05023s (17.559GiB/s) | 20260507 05:47:42.000 | INFO | Downloaded 34GiB to GPU in 1.96772s (17.2789GiB/s) | 20260507 05:47:47.143 | INFO | Analyzing chunk frames = [7568384,7568896) | 20260507 05:47:47.143 | INFO | MemInfo: { | 20260507 05:47:47.143 | INFO | "gpu" : " 17 GB", | 20260507 05:47:47.143 | INFO | "huge" : "", | 20260507 05:47:47.143 | INFO | "huge_shared" : " 27 GB", | 20260507 05:47:47.143 | INFO | "malloc" : " 14 GB", | 20260507 05:47:47.143 | INFO | "pinned" : " 29 GB" | 20260507 05:47:47.143 | INFO | } | 20260507 05:47:47.143 | INFO | Perf report for the last 5.14392 seconds: | 20260507 05:47:47.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:47:47.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:47.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:47.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:47.143 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 05:47:47.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:47.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:47.143 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 05:47:47.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:47.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:47.144 | INFO | Uploaded 36GiB to GPU in 1.98158s (18.1673GiB/s) | 20260507 05:47:47.144 | INFO | Downloaded 34GiB to GPU in 1.98276s (17.1478GiB/s) | 20260507 05:47:52.286 | INFO | Analyzing chunk frames = [7568896,7569408) | 20260507 05:47:52.286 | INFO | MemInfo: { | 20260507 05:47:52.286 | INFO | "gpu" : " 17 GB", | 20260507 05:47:52.286 | INFO | "huge" : "", | 20260507 05:47:52.286 | INFO | "huge_shared" : " 27 GB", | 20260507 05:47:52.286 | INFO | "malloc" : " 14 GB", | 20260507 05:47:52.286 | INFO | "pinned" : " 29 GB" | 20260507 05:47:52.286 | INFO | } | 20260507 05:47:52.286 | INFO | Perf report for the last 5.14312 seconds: | 20260507 05:47:52.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:47:52.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:52.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:52.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:52.286 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 05:47:52.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:52.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:52.286 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 05:47:52.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:52.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:52.287 | INFO | Uploaded 36GiB to GPU in 2.00497s (17.9554GiB/s) | 20260507 05:47:52.287 | INFO | Downloaded 34GiB to GPU in 1.9645s (17.3072GiB/s) | 20260507 05:47:52.287 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:47:52.287 | INFO | Total mem usage: 18 GiB | 20260507 05:47:52.287 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:47:52.287 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 05:47:52.287 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:47:52.287 | INFO | High Water Breakdown: | 20260507 05:47:52.287 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:47:52.287 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:47:52.287 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:47:52.287 | INFO | ------------------------------------------ | 20260507 05:47:52.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:47:52.287 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:47:52.287 | INFO | Total mem usage: 26.5 GiB | 20260507 05:47:52.287 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:47:52.287 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:47:52.287 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:47:52.287 | INFO | High Water Breakdown: | 20260507 05:47:52.287 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:47:52.287 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:47:52.287 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:47:52.287 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:47:52.287 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:47:52.287 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:47:52.287 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:47:52.287 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:47:52.287 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:47:52.287 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:47:52.287 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:47:52.287 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:47:52.287 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:47:52.287 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:47:52.287 | INFO | ------------------------------------------ | 20260507 05:47:52.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:47:52.287 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:47:52.287 | INFO | Total mem usage: 25.6 GiB | 20260507 05:47:52.287 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:47:52.287 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:47:52.287 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:47:52.287 | INFO | High Water Breakdown: | 20260507 05:47:52.287 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:47:52.287 | INFO | ------------------------------------------ | 20260507 05:47:52.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:47:52.287 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:47:52.287 | INFO | Total mem usage: 15.7 GiB | 20260507 05:47:52.287 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:47:52.287 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:47:52.287 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:47:52.287 | INFO | High Water Breakdown: | 20260507 05:47:52.287 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:47:52.287 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:47:52.287 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:47:52.287 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:47:52.287 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:47:52.287 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:47:52.287 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:47:52.287 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:47:52.287 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:47:52.287 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:47:52.287 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:47:52.287 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:47:52.287 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:47:52.287 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:47:52.287 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:47:52.287 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:47:52.287 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:47:52.287 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:47:52.287 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:47:52.287 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:47:52.287 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:47:52.287 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:47:52.287 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:47:52.287 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:47:52.287 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:47:52.287 | INFO | ------------------------------------------ | 20260507 05:47:52.287 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:47:52.287 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:47:52.287 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:47:52.287 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:47:52.287 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.50e+01, 1.34e+01 / 1.26e+00 | 20260507 05:47:52.287 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.83e-03 / 6.35e-03 | 20260507 05:47:52.287 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.44e+00, 4.97e-01 / 2.45e-01 | 20260507 05:47:52.288 | INFO | PerfReport: 3.21%, Histogram -- 9.85e-01 / 1.47e+00, 1.63e-01 / 1.14e-01 | 20260507 05:47:52.288 | INFO | PerfReport: 16.31%, DME -- 4.00e+01 / 4.52e+01, 4.03e+00 / 1.42e+00 | 20260507 05:47:52.288 | INFO | PerfReport: 57.61%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.20e+00 / 1.33e+00 | 20260507 05:47:52.288 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.13e+00 / 2.71e+00, 3.87e-01 / 2.01e-01 | 20260507 05:47:52.288 | INFO | PerfReport: 2.97%, Metrics -- 9.10e-01 / 1.23e+00, 2.39e-01 / 1.38e-01 | 20260507 05:47:52.288 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.08e-01 / 3.62e-01, 4.99e-02 / 2.00e-02 | 20260507 05:47:52.288 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:47:52.288 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:47:52.288 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:47:52.288 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 05:47:52.288 | INFO | PerfReport: 21.83%, Upload -- 2.10e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260507 05:47:52.288 | INFO | PerfReport: 21.43%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 05:47:52.288 | INFO | PerfReport: 31.86%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 05:47:52.288 | INFO | PerfReport: 1.54%, LockUpload -- 1.49e+00 / 0.00e+00, 4.96e+00 / 0.00e+00 | 20260507 05:47:52.288 | INFO | PerfReport: 1.38%, LockDownload -- 1.33e+00 / 0.00e+00, 3.19e+00 / 0.00e+00 | 20260507 05:47:52.288 | INFO | PerfReport: 21.94%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 05:47:52.288 | INFO | PerfReport: 0.02%, Misc -- 1.54e-02 / 0.00e+00, 2.84e-02 / 0.00e+00 | 20260507 05:47:52.383 | INFO | Starting a round of Baz IO | 20260507 05:47:55.476 | INFO | Populating RTMetrics report | 20260507 05:47:55.477 | INFO | RTMetrics report populated | 20260507 05:47:55.477 | INFO | Opening tmp RTMetrics json file | 20260507 05:47:55.494 | INFO | Serializing RTMetrics to json | 20260507 05:47:55.495 | INFO | Serialization done, swapping tmp file now... | 20260507 05:47:55.495 | INFO | RTMetrics done | 20260507 05:47:55.566 | INFO | Finished a round of Baz IO: Wrote 1.686 GiB in 3.183 seconds | 20260507 05:47:55.567 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.68% / 0.00% / 67.32% | 20260507 05:47:55.567 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.135291% / 0% / 6.35783e-05% | 20260507 05:47:55.567 | INFO | Baz IO overall: | 20260507 05:47:55.567 | INFO | BytesWritten : 3832668112379 | 20260507 05:47:55.567 | INFO | NumEvents : 1837787795955 | 20260507 05:47:57.420 | INFO | Analyzing chunk frames = [7569408,7569920) | 20260507 05:47:57.420 | INFO | MemInfo: { | 20260507 05:47:57.420 | INFO | "gpu" : " 17 GB", | 20260507 05:47:57.420 | INFO | "huge" : "", | 20260507 05:47:57.420 | INFO | "huge_shared" : " 27 GB", | 20260507 05:47:57.420 | INFO | "malloc" : " 14 GB", | 20260507 05:47:57.420 | INFO | "pinned" : " 29 GB" | 20260507 05:47:57.420 | INFO | } | 20260507 05:47:57.420 | INFO | Perf report for the last 5.13394 seconds: | 20260507 05:47:57.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:47:57.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:57.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:57.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:57.420 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 05:47:57.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:47:57.420 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 05:47:57.420 | INFO | PRE_HQ : 58.9%, 1.8, 54.5ms, 0 | 20260507 05:47:57.420 | INFO | BAZWRITER : 61.6%, 3.2, 100.2ms, 0 | 20260507 05:47:57.420 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:47:57.421 | INFO | Uploaded 36GiB to GPU in 2.07302s (17.366GiB/s) | 20260507 05:47:57.421 | INFO | Downloaded 38GiB to GPU in 2.17366s (17.4821GiB/s) | 20260507 05:48:02.554 | INFO | Analyzing chunk frames = [7569920,7570432) | 20260507 05:48:02.554 | INFO | MemInfo: { | 20260507 05:48:02.554 | INFO | "gpu" : " 17 GB", | 20260507 05:48:02.554 | INFO | "huge" : "", | 20260507 05:48:02.554 | INFO | "huge_shared" : " 27 GB", | 20260507 05:48:02.554 | INFO | "malloc" : " 14 GB", | 20260507 05:48:02.554 | INFO | "pinned" : " 29 GB" | 20260507 05:48:02.554 | INFO | } | 20260507 05:48:02.555 | INFO | Perf report for the last 5.13429 seconds: | 20260507 05:48:02.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:48:02.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:02.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:02.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:02.555 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:48:02.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:02.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:02.555 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 05:48:02.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:02.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:02.555 | INFO | Uploaded 36GiB to GPU in 1.9938s (18.056GiB/s) | 20260507 05:48:02.555 | INFO | Downloaded 34GiB to GPU in 1.92959s (17.6203GiB/s) | 20260507 05:48:07.700 | INFO | Analyzing chunk frames = [7570432,7570944) | 20260507 05:48:07.700 | INFO | MemInfo: { | 20260507 05:48:07.700 | INFO | "gpu" : " 17 GB", | 20260507 05:48:07.700 | INFO | "huge" : "", | 20260507 05:48:07.700 | INFO | "huge_shared" : " 27 GB", | 20260507 05:48:07.700 | INFO | "malloc" : " 14 GB", | 20260507 05:48:07.701 | INFO | "pinned" : " 29 GB" | 20260507 05:48:07.701 | INFO | } | 20260507 05:48:07.701 | INFO | Perf report for the last 5.1461 seconds: | 20260507 05:48:07.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:48:07.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:07.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:07.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:07.701 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 05:48:07.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:07.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:07.701 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:48:07.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:07.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:07.701 | INFO | Uploaded 36GiB to GPU in 2.0144s (17.8713GiB/s) | 20260507 05:48:07.701 | INFO | Downloaded 34GiB to GPU in 1.93954s (17.53GiB/s) | 20260507 05:48:12.841 | INFO | Analyzing chunk frames = [7570944,7571456) | 20260507 05:48:12.841 | INFO | MemInfo: { | 20260507 05:48:12.841 | INFO | "gpu" : " 17 GB", | 20260507 05:48:12.841 | INFO | "huge" : "", | 20260507 05:48:12.841 | INFO | "huge_shared" : " 27 GB", | 20260507 05:48:12.841 | INFO | "malloc" : " 14 GB", | 20260507 05:48:12.841 | INFO | "pinned" : " 29 GB" | 20260507 05:48:12.841 | INFO | } | 20260507 05:48:12.841 | INFO | Perf report for the last 5.14033 seconds: | 20260507 05:48:12.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:48:12.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:12.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:12.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:12.841 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:48:12.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:12.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:12.841 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:48:12.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:12.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:12.841 | INFO | Uploaded 36GiB to GPU in 2.00887s (17.9205GiB/s) | 20260507 05:48:12.841 | INFO | Downloaded 34GiB to GPU in 1.93215s (17.597GiB/s) | 20260507 05:48:17.983 | INFO | Analyzing chunk frames = [7571456,7571968) | 20260507 05:48:17.983 | INFO | MemInfo: { | 20260507 05:48:17.983 | INFO | "gpu" : " 17 GB", | 20260507 05:48:17.983 | INFO | "huge" : "", | 20260507 05:48:17.983 | INFO | "huge_shared" : " 27 GB", | 20260507 05:48:17.983 | INFO | "malloc" : " 14 GB", | 20260507 05:48:17.983 | INFO | "pinned" : " 29 GB" | 20260507 05:48:17.983 | INFO | } | 20260507 05:48:17.983 | INFO | Perf report for the last 5.14229 seconds: | 20260507 05:48:17.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:48:17.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:17.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:17.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:17.983 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 05:48:17.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:17.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:17.983 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 05:48:17.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:17.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:17.984 | INFO | Uploaded 36GiB to GPU in 2.03978s (17.649GiB/s) | 20260507 05:48:17.984 | INFO | Downloaded 34GiB to GPU in 1.91646s (17.741GiB/s) | 20260507 05:48:23.114 | INFO | Analyzing chunk frames = [7571968,7572480) | 20260507 05:48:23.114 | INFO | MemInfo: { | 20260507 05:48:23.114 | INFO | "gpu" : " 17 GB", | 20260507 05:48:23.114 | INFO | "huge" : "", | 20260507 05:48:23.114 | INFO | "huge_shared" : " 27 GB", | 20260507 05:48:23.114 | INFO | "malloc" : " 14 GB", | 20260507 05:48:23.114 | INFO | "pinned" : " 29 GB" | 20260507 05:48:23.114 | INFO | } | 20260507 05:48:23.114 | INFO | Perf report for the last 5.13055 seconds: | 20260507 05:48:23.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:48:23.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:23.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:23.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:23.114 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:48:23.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:23.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:23.114 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 05:48:23.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:23.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:23.114 | INFO | Uploaded 36GiB to GPU in 2.07361s (17.3611GiB/s) | 20260507 05:48:23.114 | INFO | Downloaded 34GiB to GPU in 1.94341s (17.495GiB/s) | 20260507 05:48:28.257 | INFO | Analyzing chunk frames = [7572480,7572992) | 20260507 05:48:28.257 | INFO | MemInfo: { | 20260507 05:48:28.257 | INFO | "gpu" : " 17 GB", | 20260507 05:48:28.257 | INFO | "huge" : "", | 20260507 05:48:28.257 | INFO | "huge_shared" : " 27 GB", | 20260507 05:48:28.257 | INFO | "malloc" : " 14 GB", | 20260507 05:48:28.257 | INFO | "pinned" : " 29 GB" | 20260507 05:48:28.257 | INFO | } | 20260507 05:48:28.257 | INFO | Perf report for the last 5.14325 seconds: | 20260507 05:48:28.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:48:28.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:28.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:28.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:28.257 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 05:48:28.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:28.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:28.257 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 05:48:28.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:28.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:28.257 | INFO | Uploaded 36GiB to GPU in 2.00028s (17.9975GiB/s) | 20260507 05:48:28.257 | INFO | Downloaded 34GiB to GPU in 1.96722s (17.2833GiB/s) | 20260507 05:48:33.395 | INFO | Analyzing chunk frames = [7572992,7573504) | 20260507 05:48:33.395 | INFO | MemInfo: { | 20260507 05:48:33.395 | INFO | "gpu" : " 17 GB", | 20260507 05:48:33.395 | INFO | "huge" : "", | 20260507 05:48:33.395 | INFO | "huge_shared" : " 27 GB", | 20260507 05:48:33.395 | INFO | "malloc" : " 14 GB", | 20260507 05:48:33.395 | INFO | "pinned" : " 29 GB" | 20260507 05:48:33.395 | INFO | } | 20260507 05:48:33.395 | INFO | Perf report for the last 5.13818 seconds: | 20260507 05:48:33.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:48:33.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:33.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:33.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:33.395 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:48:33.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:33.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:33.395 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:48:33.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:33.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:33.396 | INFO | Uploaded 36GiB to GPU in 1.9558s (18.4068GiB/s) | 20260507 05:48:33.396 | INFO | Downloaded 34GiB to GPU in 1.96286s (17.3217GiB/s) | 20260507 05:48:33.492 | INFO | Starting a round of Baz IO | 20260507 05:48:36.588 | INFO | Populating RTMetrics report | 20260507 05:48:36.588 | INFO | RTMetrics report populated | 20260507 05:48:36.588 | INFO | Opening tmp RTMetrics json file | 20260507 05:48:36.596 | INFO | Serializing RTMetrics to json | 20260507 05:48:36.604 | INFO | Serialization done, swapping tmp file now... | 20260507 05:48:36.605 | INFO | RTMetrics done | 20260507 05:48:36.680 | INFO | Finished a round of Baz IO: Wrote 1.685 GiB in 3.187 seconds | 20260507 05:48:36.680 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.68% / 0.00% / 67.32% | 20260507 05:48:36.680 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.135744% / 3.97364e-06% / 8.34465e-05% | 20260507 05:48:36.680 | INFO | Baz IO overall: | 20260507 05:48:36.680 | INFO | BytesWritten : 3834477469899 | 20260507 05:48:36.680 | INFO | NumEvents : 1838636200932 | 20260507 05:48:38.534 | INFO | Analyzing chunk frames = [7573504,7574016) | 20260507 05:48:38.534 | INFO | MemInfo: { | 20260507 05:48:38.534 | INFO | "gpu" : " 17 GB", | 20260507 05:48:38.534 | INFO | "huge" : "", | 20260507 05:48:38.534 | INFO | "huge_shared" : " 27 GB", | 20260507 05:48:38.534 | INFO | "malloc" : " 14 GB", | 20260507 05:48:38.534 | INFO | "pinned" : " 29 GB" | 20260507 05:48:38.534 | INFO | } | 20260507 05:48:38.534 | INFO | Perf report for the last 5.13859 seconds: | 20260507 05:48:38.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:48:38.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:38.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:38.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:38.534 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 05:48:38.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:38.534 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 05:48:38.534 | INFO | PRE_HQ : 58.9%, 1.8, 54.6ms, 0 | 20260507 05:48:38.534 | INFO | BAZWRITER : 61.6%, 3.2, 100.8ms, 0 | 20260507 05:48:38.534 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:48:38.534 | INFO | Uploaded 36GiB to GPU in 2.03855s (17.6596GiB/s) | 20260507 05:48:38.534 | INFO | Downloaded 38GiB to GPU in 2.18043s (17.4278GiB/s) | 20260507 05:48:43.673 | INFO | Analyzing chunk frames = [7574016,7574528) | 20260507 05:48:43.673 | INFO | MemInfo: { | 20260507 05:48:43.673 | INFO | "gpu" : " 17 GB", | 20260507 05:48:43.673 | INFO | "huge" : "", | 20260507 05:48:43.674 | INFO | "huge_shared" : " 27 GB", | 20260507 05:48:43.674 | INFO | "malloc" : " 14 GB", | 20260507 05:48:43.674 | INFO | "pinned" : " 29 GB" | 20260507 05:48:43.674 | INFO | } | 20260507 05:48:43.674 | INFO | Perf report for the last 5.13979 seconds: | 20260507 05:48:43.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:48:43.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:43.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:43.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:43.674 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:48:43.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:43.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:43.674 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:48:43.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:43.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:43.674 | INFO | Uploaded 36GiB to GPU in 1.98265s (18.1575GiB/s) | 20260507 05:48:43.674 | INFO | Downloaded 34GiB to GPU in 1.92793s (17.6355GiB/s) | 20260507 05:48:48.810 | INFO | Analyzing chunk frames = [7574528,7575040) | 20260507 05:48:48.810 | INFO | MemInfo: { | 20260507 05:48:48.810 | INFO | "gpu" : " 17 GB", | 20260507 05:48:48.810 | INFO | "huge" : "", | 20260507 05:48:48.810 | INFO | "huge_shared" : " 27 GB", | 20260507 05:48:48.810 | INFO | "malloc" : " 14 GB", | 20260507 05:48:48.810 | INFO | "pinned" : " 29 GB" | 20260507 05:48:48.810 | INFO | } | 20260507 05:48:48.810 | INFO | Perf report for the last 5.13613 seconds: | 20260507 05:48:48.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:48:48.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:48.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:48.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:48.810 | INFO | ANALYSIS : 60.5%, 3.0, 93.5ms, 0 | 20260507 05:48:48.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:48.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:48.810 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:48:48.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:48.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:48.810 | INFO | Uploaded 36GiB to GPU in 2.03097s (17.7255GiB/s) | 20260507 05:48:48.810 | INFO | Downloaded 34GiB to GPU in 1.95213s (17.4169GiB/s) | 20260507 05:48:53.955 | INFO | Analyzing chunk frames = [7575040,7575552) | 20260507 05:48:53.955 | INFO | MemInfo: { | 20260507 05:48:53.955 | INFO | "gpu" : " 17 GB", | 20260507 05:48:53.955 | INFO | "huge" : "", | 20260507 05:48:53.955 | INFO | "huge_shared" : " 27 GB", | 20260507 05:48:53.955 | INFO | "malloc" : " 14 GB", | 20260507 05:48:53.955 | INFO | "pinned" : " 29 GB" | 20260507 05:48:53.955 | INFO | } | 20260507 05:48:53.955 | INFO | Perf report for the last 5.14489 seconds: | 20260507 05:48:53.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:48:53.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:53.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:53.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:53.955 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 05:48:53.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:53.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:53.955 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 05:48:53.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:53.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:53.955 | INFO | Uploaded 36GiB to GPU in 1.99175s (18.0745GiB/s) | 20260507 05:48:53.955 | INFO | Downloaded 34GiB to GPU in 1.93648s (17.5576GiB/s) | 20260507 05:48:59.091 | INFO | Analyzing chunk frames = [7575552,7576064) | 20260507 05:48:59.091 | INFO | MemInfo: { | 20260507 05:48:59.091 | INFO | "gpu" : " 17 GB", | 20260507 05:48:59.091 | INFO | "huge" : "", | 20260507 05:48:59.091 | INFO | "huge_shared" : " 27 GB", | 20260507 05:48:59.091 | INFO | "malloc" : " 14 GB", | 20260507 05:48:59.091 | INFO | "pinned" : " 29 GB" | 20260507 05:48:59.091 | INFO | } | 20260507 05:48:59.091 | INFO | Perf report for the last 5.13663 seconds: | 20260507 05:48:59.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:48:59.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:59.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:59.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:59.091 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 05:48:59.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:59.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:59.092 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:48:59.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:59.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:48:59.092 | INFO | Uploaded 36GiB to GPU in 2.05149s (17.5482GiB/s) | 20260507 05:48:59.092 | INFO | Downloaded 34GiB to GPU in 1.9508s (17.4288GiB/s) | 20260507 05:49:04.229 | INFO | Analyzing chunk frames = [7576064,7576576) | 20260507 05:49:04.229 | INFO | MemInfo: { | 20260507 05:49:04.229 | INFO | "gpu" : " 17 GB", | 20260507 05:49:04.229 | INFO | "huge" : "", | 20260507 05:49:04.229 | INFO | "huge_shared" : " 27 GB", | 20260507 05:49:04.229 | INFO | "malloc" : " 14 GB", | 20260507 05:49:04.229 | INFO | "pinned" : " 29 GB" | 20260507 05:49:04.229 | INFO | } | 20260507 05:49:04.229 | INFO | Perf report for the last 5.13776 seconds: | 20260507 05:49:04.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:49:04.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:04.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:04.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:04.229 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 05:49:04.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:04.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:04.229 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 05:49:04.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:04.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:04.229 | INFO | Uploaded 36GiB to GPU in 2.06702s (17.4164GiB/s) | 20260507 05:49:04.230 | INFO | Downloaded 34GiB to GPU in 1.94235s (17.5045GiB/s) | 20260507 05:49:09.362 | INFO | Analyzing chunk frames = [7576576,7577088) | 20260507 05:49:09.362 | INFO | MemInfo: { | 20260507 05:49:09.362 | INFO | "gpu" : " 17 GB", | 20260507 05:49:09.362 | INFO | "huge" : "", | 20260507 05:49:09.362 | INFO | "huge_shared" : " 27 GB", | 20260507 05:49:09.362 | INFO | "malloc" : " 14 GB", | 20260507 05:49:09.362 | INFO | "pinned" : " 29 GB" | 20260507 05:49:09.362 | INFO | } | 20260507 05:49:09.362 | INFO | Perf report for the last 5.13324 seconds: | 20260507 05:49:09.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:49:09.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:09.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:09.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:09.362 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:49:09.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:09.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:09.363 | INFO | PRE_HQ : 25.4%, 1.0, 13.1ms, 0 | 20260507 05:49:09.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:09.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:09.363 | INFO | Uploaded 36GiB to GPU in 2.00559s (17.9498GiB/s) | 20260507 05:49:09.363 | INFO | Downloaded 34GiB to GPU in 1.98058s (17.1667GiB/s) | 20260507 05:49:14.508 | INFO | Analyzing chunk frames = [7577088,7577600) | 20260507 05:49:14.508 | INFO | MemInfo: { | 20260507 05:49:14.508 | INFO | "gpu" : " 17 GB", | 20260507 05:49:14.508 | INFO | "huge" : "", | 20260507 05:49:14.508 | INFO | "huge_shared" : " 27 GB", | 20260507 05:49:14.508 | INFO | "malloc" : " 14 GB", | 20260507 05:49:14.508 | INFO | "pinned" : " 29 GB" | 20260507 05:49:14.508 | INFO | } | 20260507 05:49:14.508 | INFO | Perf report for the last 5.14537 seconds: | 20260507 05:49:14.508 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:49:14.508 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:14.508 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:14.508 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:14.508 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:49:14.508 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:14.508 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:14.508 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 05:49:14.508 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:14.508 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:14.508 | INFO | Uploaded 36GiB to GPU in 1.96965s (18.2774GiB/s) | 20260507 05:49:14.508 | INFO | Downloaded 34GiB to GPU in 1.95437s (17.3969GiB/s) | 20260507 05:49:14.508 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:49:14.508 | INFO | Total mem usage: 18 GiB | 20260507 05:49:14.508 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:49:14.508 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:49:14.508 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:49:14.508 | INFO | High Water Breakdown: | 20260507 05:49:14.508 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:49:14.508 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:49:14.508 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:49:14.508 | INFO | ------------------------------------------ | 20260507 05:49:14.508 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:49:14.508 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:49:14.508 | INFO | Total mem usage: 26.5 GiB | 20260507 05:49:14.508 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:49:14.508 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:49:14.508 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:49:14.508 | INFO | High Water Breakdown: | 20260507 05:49:14.508 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:49:14.508 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:49:14.508 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:49:14.508 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:49:14.508 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:49:14.509 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:49:14.509 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:49:14.509 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:49:14.509 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:49:14.509 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:49:14.509 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:49:14.509 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:49:14.509 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:49:14.509 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:49:14.509 | INFO | ------------------------------------------ | 20260507 05:49:14.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:49:14.509 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:49:14.509 | INFO | Total mem usage: 25.6 GiB | 20260507 05:49:14.509 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:49:14.509 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:49:14.509 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:49:14.509 | INFO | High Water Breakdown: | 20260507 05:49:14.509 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:49:14.509 | INFO | ------------------------------------------ | 20260507 05:49:14.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:49:14.509 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:49:14.509 | INFO | Total mem usage: 15.7 GiB | 20260507 05:49:14.509 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:49:14.509 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:49:14.509 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:49:14.509 | INFO | High Water Breakdown: | 20260507 05:49:14.509 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:49:14.509 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:49:14.509 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:49:14.509 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:49:14.509 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:49:14.509 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:49:14.509 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:49:14.509 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:49:14.509 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:49:14.509 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:49:14.509 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:49:14.509 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:49:14.509 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:49:14.509 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:49:14.509 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:49:14.509 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:49:14.509 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:49:14.509 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:49:14.509 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:49:14.509 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:49:14.509 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:49:14.509 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:49:14.509 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:49:14.509 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:49:14.509 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:49:14.509 | INFO | ------------------------------------------ | 20260507 05:49:14.509 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:49:14.509 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:49:14.509 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:49:14.509 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:49:14.509 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.43e+01, 1.32e+01 / 1.39e+00 | 20260507 05:49:14.509 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.03e-01, 9.86e-03 / 6.97e-03 | 20260507 05:49:14.509 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.41e+00, 5.01e-01 / 3.18e-01 | 20260507 05:49:14.509 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.44e+00, 1.65e-01 / 1.34e-01 | 20260507 05:49:14.509 | INFO | PerfReport: 16.17%, DME -- 3.97e+01 / 4.40e+01, 3.84e+00 / 1.66e+00 | 20260507 05:49:14.509 | INFO | PerfReport: 57.74%, FrameLabeling -- 1.77e+01 / 2.00e+01, 1.30e+00 / 1.58e+00 | 20260507 05:49:14.509 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.66e+00, 3.85e-01 / 2.39e-01 | 20260507 05:49:14.509 | INFO | PerfReport: 2.98%, Metrics -- 9.15e-01 / 1.22e+00, 2.44e-01 / 1.42e-01 | 20260507 05:49:14.509 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.08e-01 / 3.57e-01, 5.13e-02 / 1.99e-02 | 20260507 05:49:14.509 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:49:14.509 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:49:14.509 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:49:14.509 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 05:49:14.509 | INFO | PerfReport: 21.87%, Upload -- 2.10e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260507 05:49:14.509 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 05:49:14.509 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 05:49:14.509 | INFO | PerfReport: 1.47%, LockUpload -- 1.42e+00 / 0.00e+00, 4.86e+00 / 0.00e+00 | 20260507 05:49:14.509 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260507 05:49:14.509 | INFO | PerfReport: 22.03%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 05:49:14.509 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.73e-02 / 0.00e+00 | 20260507 05:49:14.605 | INFO | Starting a round of Baz IO | 20260507 05:49:17.701 | INFO | Populating RTMetrics report | 20260507 05:49:17.702 | INFO | RTMetrics report populated | 20260507 05:49:17.702 | INFO | Opening tmp RTMetrics json file | 20260507 05:49:17.720 | INFO | Serializing RTMetrics to json | 20260507 05:49:17.721 | INFO | Serialization done, swapping tmp file now... | 20260507 05:49:17.721 | INFO | RTMetrics done | 20260507 05:49:17.787 | INFO | Finished a round of Baz IO: Wrote 1.685 GiB in 3.182 seconds | 20260507 05:49:17.787 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.67% / 0.00% / 67.32% | 20260507 05:49:17.787 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.134838% / 0% / 6.75519e-05% | 20260507 05:49:17.787 | INFO | Baz IO overall: | 20260507 05:49:17.787 | INFO | BytesWritten : 3836287157163 | 20260507 05:49:17.787 | INFO | NumEvents : 1839485229104 | 20260507 05:49:19.655 | INFO | Analyzing chunk frames = [7577600,7578112) | 20260507 05:49:19.655 | INFO | MemInfo: { | 20260507 05:49:19.655 | INFO | "gpu" : " 17 GB", | 20260507 05:49:19.655 | INFO | "huge" : "", | 20260507 05:49:19.655 | INFO | "huge_shared" : " 27 GB", | 20260507 05:49:19.655 | INFO | "malloc" : " 14 GB", | 20260507 05:49:19.655 | INFO | "pinned" : " 29 GB" | 20260507 05:49:19.655 | INFO | } | 20260507 05:49:19.655 | INFO | Perf report for the last 5.14676 seconds: | 20260507 05:49:19.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:49:19.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:19.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:19.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:19.655 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260507 05:49:19.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:19.655 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 05:49:19.655 | INFO | PRE_HQ : 58.8%, 1.8, 54.7ms, 0 | 20260507 05:49:19.655 | INFO | BAZWRITER : 61.4%, 3.2, 100.2ms, 0 | 20260507 05:49:19.655 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:49:19.655 | INFO | Uploaded 36GiB to GPU in 2.04336s (17.618GiB/s) | 20260507 05:49:19.655 | INFO | Downloaded 38GiB to GPU in 2.18004s (17.4309GiB/s) | 20260507 05:49:24.799 | INFO | Analyzing chunk frames = [7578112,7578624) | 20260507 05:49:24.799 | INFO | MemInfo: { | 20260507 05:49:24.799 | INFO | "gpu" : " 17 GB", | 20260507 05:49:24.799 | INFO | "huge" : "", | 20260507 05:49:24.799 | INFO | "huge_shared" : " 27 GB", | 20260507 05:49:24.799 | INFO | "malloc" : " 14 GB", | 20260507 05:49:24.799 | INFO | "pinned" : " 29 GB" | 20260507 05:49:24.799 | INFO | } | 20260507 05:49:24.800 | INFO | Perf report for the last 5.14493 seconds: | 20260507 05:49:24.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:49:24.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:24.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:24.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:24.800 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 05:49:24.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:24.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:24.800 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:49:24.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:24.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:24.800 | INFO | Uploaded 36GiB to GPU in 1.9961s (18.0352GiB/s) | 20260507 05:49:24.800 | INFO | Downloaded 34GiB to GPU in 1.9371s (17.552GiB/s) | 20260507 05:49:29.930 | INFO | Analyzing chunk frames = [7578624,7579136) | 20260507 05:49:29.930 | INFO | MemInfo: { | 20260507 05:49:29.930 | INFO | "gpu" : " 17 GB", | 20260507 05:49:29.930 | INFO | "huge" : "", | 20260507 05:49:29.930 | INFO | "huge_shared" : " 27 GB", | 20260507 05:49:29.930 | INFO | "malloc" : " 14 GB", | 20260507 05:49:29.930 | INFO | "pinned" : " 29 GB" | 20260507 05:49:29.930 | INFO | } | 20260507 05:49:29.930 | INFO | Perf report for the last 5.13062 seconds: | 20260507 05:49:29.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:49:29.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:29.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:29.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:29.930 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 05:49:29.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:29.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:29.930 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:49:29.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:29.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:29.931 | INFO | Uploaded 36GiB to GPU in 2.031s (17.7252GiB/s) | 20260507 05:49:29.931 | INFO | Downloaded 34GiB to GPU in 1.91622s (17.7432GiB/s) | 20260507 05:49:35.063 | INFO | Analyzing chunk frames = [7579136,7579648) | 20260507 05:49:35.063 | INFO | MemInfo: { | 20260507 05:49:35.063 | INFO | "gpu" : " 17 GB", | 20260507 05:49:35.063 | INFO | "huge" : "", | 20260507 05:49:35.063 | INFO | "huge_shared" : " 27 GB", | 20260507 05:49:35.063 | INFO | "malloc" : " 14 GB", | 20260507 05:49:35.063 | INFO | "pinned" : " 29 GB" | 20260507 05:49:35.063 | INFO | } | 20260507 05:49:35.063 | INFO | Perf report for the last 5.13269 seconds: | 20260507 05:49:35.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:49:35.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:35.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:35.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:35.063 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 05:49:35.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:35.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:35.063 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:49:35.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:35.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:35.063 | INFO | Uploaded 36GiB to GPU in 2.01406s (17.8743GiB/s) | 20260507 05:49:35.063 | INFO | Downloaded 34GiB to GPU in 1.9396s (17.5294GiB/s) | 20260507 05:49:40.205 | INFO | Analyzing chunk frames = [7579648,7580160) | 20260507 05:49:40.205 | INFO | MemInfo: { | 20260507 05:49:40.205 | INFO | "gpu" : " 17 GB", | 20260507 05:49:40.205 | INFO | "huge" : "", | 20260507 05:49:40.205 | INFO | "huge_shared" : " 27 GB", | 20260507 05:49:40.205 | INFO | "malloc" : " 14 GB", | 20260507 05:49:40.205 | INFO | "pinned" : " 29 GB" | 20260507 05:49:40.205 | INFO | } | 20260507 05:49:40.205 | INFO | Perf report for the last 5.14255 seconds: | 20260507 05:49:40.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:49:40.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:40.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:40.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:40.205 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 05:49:40.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:40.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:40.206 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:49:40.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:40.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:40.206 | INFO | Uploaded 36GiB to GPU in 2.04406s (17.612GiB/s) | 20260507 05:49:40.206 | INFO | Downloaded 34GiB to GPU in 1.93414s (17.5789GiB/s) | 20260507 05:49:45.345 | INFO | Analyzing chunk frames = [7580160,7580672) | 20260507 05:49:45.345 | INFO | MemInfo: { | 20260507 05:49:45.345 | INFO | "gpu" : " 17 GB", | 20260507 05:49:45.345 | INFO | "huge" : "", | 20260507 05:49:45.345 | INFO | "huge_shared" : " 27 GB", | 20260507 05:49:45.345 | INFO | "malloc" : " 14 GB", | 20260507 05:49:45.345 | INFO | "pinned" : " 29 GB" | 20260507 05:49:45.345 | INFO | } | 20260507 05:49:45.345 | INFO | Perf report for the last 5.13954 seconds: | 20260507 05:49:45.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:49:45.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:45.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:45.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:45.345 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 05:49:45.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:45.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:45.345 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:49:45.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:45.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:45.345 | INFO | Uploaded 36GiB to GPU in 2.03652s (17.6772GiB/s) | 20260507 05:49:45.345 | INFO | Downloaded 34GiB to GPU in 1.90315s (17.8652GiB/s) | 20260507 05:49:50.486 | INFO | Analyzing chunk frames = [7580672,7581184) | 20260507 05:49:50.486 | INFO | MemInfo: { | 20260507 05:49:50.486 | INFO | "gpu" : " 17 GB", | 20260507 05:49:50.486 | INFO | "huge" : "", | 20260507 05:49:50.486 | INFO | "huge_shared" : " 27 GB", | 20260507 05:49:50.486 | INFO | "malloc" : " 14 GB", | 20260507 05:49:50.486 | INFO | "pinned" : " 29 GB" | 20260507 05:49:50.486 | INFO | } | 20260507 05:49:50.486 | INFO | Perf report for the last 5.141 seconds: | 20260507 05:49:50.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:49:50.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:50.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:50.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:50.486 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 05:49:50.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:50.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:50.486 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 05:49:50.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:50.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:50.486 | INFO | Uploaded 36GiB to GPU in 2.04087s (17.6395GiB/s) | 20260507 05:49:50.487 | INFO | Downloaded 34GiB to GPU in 1.98605s (17.1194GiB/s) | 20260507 05:49:55.624 | INFO | Analyzing chunk frames = [7581184,7581696) | 20260507 05:49:55.624 | INFO | MemInfo: { | 20260507 05:49:55.624 | INFO | "gpu" : " 17 GB", | 20260507 05:49:55.624 | INFO | "huge" : "", | 20260507 05:49:55.624 | INFO | "huge_shared" : " 27 GB", | 20260507 05:49:55.624 | INFO | "malloc" : " 14 GB", | 20260507 05:49:55.624 | INFO | "pinned" : " 29 GB" | 20260507 05:49:55.624 | INFO | } | 20260507 05:49:55.625 | INFO | Perf report for the last 5.13897 seconds: | 20260507 05:49:55.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:49:55.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:55.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:55.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:55.625 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:49:55.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:55.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:55.625 | INFO | PRE_HQ : 28.8%, 1.0, 14.9ms, 0 | 20260507 05:49:55.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:55.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:49:55.625 | INFO | Uploaded 36GiB to GPU in 1.99133s (18.0783GiB/s) | 20260507 05:49:55.625 | INFO | Downloaded 34GiB to GPU in 1.96883s (17.2691GiB/s) | 20260507 05:49:55.722 | INFO | Starting a round of Baz IO | 20260507 05:49:58.815 | INFO | Populating RTMetrics report | 20260507 05:49:58.815 | INFO | RTMetrics report populated | 20260507 05:49:58.815 | INFO | Opening tmp RTMetrics json file | 20260507 05:49:58.821 | INFO | Serializing RTMetrics to json | 20260507 05:49:58.832 | INFO | Serialization done, swapping tmp file now... | 20260507 05:49:58.833 | INFO | RTMetrics done | 20260507 05:49:58.904 | INFO | Finished a round of Baz IO: Wrote 1.684 GiB in 3.182 seconds | 20260507 05:49:58.905 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.67% / 0.00% / 67.33% | 20260507 05:49:58.905 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.135223% / 0% / 7.54992e-05% | 20260507 05:49:58.905 | INFO | Baz IO overall: | 20260507 05:49:58.905 | INFO | BytesWritten : 3838095799872 | 20260507 05:49:58.905 | INFO | NumEvents : 1840333657136 | 20260507 05:50:00.771 | INFO | Analyzing chunk frames = [7581696,7582208) | 20260507 05:50:00.771 | INFO | MemInfo: { | 20260507 05:50:00.771 | INFO | "gpu" : " 17 GB", | 20260507 05:50:00.771 | INFO | "huge" : "", | 20260507 05:50:00.771 | INFO | "huge_shared" : " 27 GB", | 20260507 05:50:00.771 | INFO | "malloc" : " 14 GB", | 20260507 05:50:00.771 | INFO | "pinned" : " 29 GB" | 20260507 05:50:00.771 | INFO | } | 20260507 05:50:00.771 | INFO | Perf report for the last 5.14581 seconds: | 20260507 05:50:00.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:50:00.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:00.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:00.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:00.771 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 05:50:00.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:00.771 | INFO | RT_METRICS : 41.8%, 1.0, 21.9ms, 0 | 20260507 05:50:00.771 | INFO | PRE_HQ : 59.1%, 1.8, 55.8ms, 0 | 20260507 05:50:00.771 | INFO | BAZWRITER : 61.4%, 3.2, 101.8ms, 0 | 20260507 05:50:00.771 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 05:50:00.771 | INFO | Uploaded 36GiB to GPU in 2.04217s (17.6283GiB/s) | 20260507 05:50:00.771 | INFO | Downloaded 38GiB to GPU in 2.17819s (17.4457GiB/s) | 20260507 05:50:05.905 | INFO | Analyzing chunk frames = [7582208,7582720) | 20260507 05:50:05.905 | INFO | MemInfo: { | 20260507 05:50:05.905 | INFO | "gpu" : " 17 GB", | 20260507 05:50:05.905 | INFO | "huge" : "", | 20260507 05:50:05.905 | INFO | "huge_shared" : " 27 GB", | 20260507 05:50:05.905 | INFO | "malloc" : " 14 GB", | 20260507 05:50:05.905 | INFO | "pinned" : " 29 GB" | 20260507 05:50:05.905 | INFO | } | 20260507 05:50:05.905 | INFO | Perf report for the last 5.13465 seconds: | 20260507 05:50:05.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:50:05.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:05.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:05.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:05.905 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 05:50:05.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:05.906 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:05.906 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:50:05.906 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:05.906 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:05.906 | INFO | Uploaded 36GiB to GPU in 1.98342s (18.1505GiB/s) | 20260507 05:50:05.906 | INFO | Downloaded 34GiB to GPU in 1.92959s (17.6203GiB/s) | 20260507 05:50:11.039 | INFO | Analyzing chunk frames = [7582720,7583232) | 20260507 05:50:11.039 | INFO | MemInfo: { | 20260507 05:50:11.039 | INFO | "gpu" : " 17 GB", | 20260507 05:50:11.039 | INFO | "huge" : "", | 20260507 05:50:11.039 | INFO | "huge_shared" : " 27 GB", | 20260507 05:50:11.039 | INFO | "malloc" : " 14 GB", | 20260507 05:50:11.039 | INFO | "pinned" : " 29 GB" | 20260507 05:50:11.039 | INFO | } | 20260507 05:50:11.039 | INFO | Perf report for the last 5.13401 seconds: | 20260507 05:50:11.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:50:11.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:11.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:11.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:11.040 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 05:50:11.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:11.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:11.040 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:50:11.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:11.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:11.040 | INFO | Uploaded 36GiB to GPU in 2.04018s (17.6455GiB/s) | 20260507 05:50:11.040 | INFO | Downloaded 34GiB to GPU in 1.95259s (17.4128GiB/s) | 20260507 05:50:16.177 | INFO | Analyzing chunk frames = [7583232,7583744) | 20260507 05:50:16.177 | INFO | MemInfo: { | 20260507 05:50:16.177 | INFO | "gpu" : " 17 GB", | 20260507 05:50:16.177 | INFO | "huge" : "", | 20260507 05:50:16.177 | INFO | "huge_shared" : " 27 GB", | 20260507 05:50:16.177 | INFO | "malloc" : " 14 GB", | 20260507 05:50:16.177 | INFO | "pinned" : " 29 GB" | 20260507 05:50:16.177 | INFO | } | 20260507 05:50:16.177 | INFO | Perf report for the last 5.13739 seconds: | 20260507 05:50:16.177 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:50:16.177 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:16.177 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:16.177 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:16.177 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:50:16.177 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:16.177 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:16.177 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:50:16.177 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:16.177 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:16.177 | INFO | Uploaded 36GiB to GPU in 2.01587s (17.8583GiB/s) | 20260507 05:50:16.177 | INFO | Downloaded 34GiB to GPU in 1.94321s (17.4968GiB/s) | 20260507 05:50:21.314 | INFO | Analyzing chunk frames = [7583744,7584256) | 20260507 05:50:21.314 | INFO | MemInfo: { | 20260507 05:50:21.314 | INFO | "gpu" : " 17 GB", | 20260507 05:50:21.314 | INFO | "huge" : "", | 20260507 05:50:21.314 | INFO | "huge_shared" : " 27 GB", | 20260507 05:50:21.314 | INFO | "malloc" : " 14 GB", | 20260507 05:50:21.314 | INFO | "pinned" : " 29 GB" | 20260507 05:50:21.315 | INFO | } | 20260507 05:50:21.315 | INFO | Perf report for the last 5.13765 seconds: | 20260507 05:50:21.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:50:21.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:21.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:21.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:21.315 | INFO | ANALYSIS : 60.2%, 3.0, 92.9ms, 0 | 20260507 05:50:21.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:21.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:21.315 | INFO | PRE_HQ : 24.7%, 1.0, 12.7ms, 0 | 20260507 05:50:21.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:21.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:21.315 | INFO | Uploaded 36GiB to GPU in 2.0491s (17.5687GiB/s) | 20260507 05:50:21.315 | INFO | Downloaded 34GiB to GPU in 1.94939s (17.4413GiB/s) | 20260507 05:50:26.460 | INFO | Analyzing chunk frames = [7584256,7584768) | 20260507 05:50:26.460 | INFO | MemInfo: { | 20260507 05:50:26.460 | INFO | "gpu" : " 17 GB", | 20260507 05:50:26.460 | INFO | "huge" : "", | 20260507 05:50:26.460 | INFO | "huge_shared" : " 27 GB", | 20260507 05:50:26.460 | INFO | "malloc" : " 14 GB", | 20260507 05:50:26.460 | INFO | "pinned" : " 29 GB" | 20260507 05:50:26.460 | INFO | } | 20260507 05:50:26.460 | INFO | Perf report for the last 5.14563 seconds: | 20260507 05:50:26.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:50:26.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:26.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:26.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:26.460 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:50:26.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:26.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:26.460 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:50:26.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:26.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:26.460 | INFO | Uploaded 36GiB to GPU in 2.0503s (17.5584GiB/s) | 20260507 05:50:26.460 | INFO | Downloaded 34GiB to GPU in 1.96774s (17.2787GiB/s) | 20260507 05:50:31.603 | INFO | Analyzing chunk frames = [7584768,7585280) | 20260507 05:50:31.603 | INFO | MemInfo: { | 20260507 05:50:31.603 | INFO | "gpu" : " 17 GB", | 20260507 05:50:31.603 | INFO | "huge" : "", | 20260507 05:50:31.603 | INFO | "huge_shared" : " 27 GB", | 20260507 05:50:31.603 | INFO | "malloc" : " 14 GB", | 20260507 05:50:31.603 | INFO | "pinned" : " 29 GB" | 20260507 05:50:31.604 | INFO | } | 20260507 05:50:31.604 | INFO | Perf report for the last 5.14338 seconds: | 20260507 05:50:31.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:50:31.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:31.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:31.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:31.604 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 05:50:31.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:31.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:31.604 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 05:50:31.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:31.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:31.604 | INFO | Uploaded 36GiB to GPU in 1.98885s (18.1009GiB/s) | 20260507 05:50:31.604 | INFO | Downloaded 34GiB to GPU in 1.96578s (17.2959GiB/s) | 20260507 05:50:36.743 | INFO | Analyzing chunk frames = [7585280,7585792) | 20260507 05:50:36.743 | INFO | MemInfo: { | 20260507 05:50:36.743 | INFO | "gpu" : " 17 GB", | 20260507 05:50:36.743 | INFO | "huge" : "", | 20260507 05:50:36.743 | INFO | "huge_shared" : " 27 GB", | 20260507 05:50:36.743 | INFO | "malloc" : " 14 GB", | 20260507 05:50:36.743 | INFO | "pinned" : " 29 GB" | 20260507 05:50:36.743 | INFO | } | 20260507 05:50:36.743 | INFO | Perf report for the last 5.13922 seconds: | 20260507 05:50:36.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:50:36.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:36.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:36.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:36.743 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:50:36.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:36.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:36.743 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 05:50:36.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:36.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:36.743 | INFO | Uploaded 36GiB to GPU in 1.99925s (18.0068GiB/s) | 20260507 05:50:36.743 | INFO | Downloaded 34GiB to GPU in 1.9612s (17.3363GiB/s) | 20260507 05:50:36.743 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:50:36.743 | INFO | Total mem usage: 18 GiB | 20260507 05:50:36.743 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:50:36.743 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:50:36.743 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:50:36.743 | INFO | High Water Breakdown: | 20260507 05:50:36.743 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:50:36.743 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:50:36.743 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:50:36.743 | INFO | ------------------------------------------ | 20260507 05:50:36.743 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:50:36.743 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:50:36.743 | INFO | Total mem usage: 26.5 GiB | 20260507 05:50:36.743 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:50:36.743 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:50:36.743 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:50:36.743 | INFO | High Water Breakdown: | 20260507 05:50:36.743 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:50:36.743 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:50:36.743 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:50:36.743 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:50:36.743 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:50:36.743 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:50:36.743 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:50:36.743 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:50:36.743 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:50:36.743 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:50:36.743 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:50:36.743 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:50:36.744 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:50:36.744 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:50:36.744 | INFO | ------------------------------------------ | 20260507 05:50:36.744 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:50:36.744 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:50:36.744 | INFO | Total mem usage: 25.6 GiB | 20260507 05:50:36.744 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:50:36.744 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:50:36.744 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:50:36.744 | INFO | High Water Breakdown: | 20260507 05:50:36.744 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:50:36.744 | INFO | ------------------------------------------ | 20260507 05:50:36.744 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:50:36.744 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:50:36.744 | INFO | Total mem usage: 15.7 GiB | 20260507 05:50:36.744 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:50:36.744 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:50:36.744 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:50:36.744 | INFO | High Water Breakdown: | 20260507 05:50:36.744 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:50:36.744 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:50:36.744 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:50:36.744 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:50:36.744 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:50:36.744 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:50:36.744 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:50:36.744 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:50:36.744 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:50:36.744 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:50:36.744 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:50:36.744 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:50:36.744 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:50:36.744 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:50:36.744 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:50:36.744 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:50:36.744 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:50:36.744 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:50:36.744 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:50:36.744 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:50:36.744 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:50:36.744 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:50:36.744 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:50:36.744 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:50:36.744 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:50:36.744 | INFO | ------------------------------------------ | 20260507 05:50:36.744 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:50:36.744 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:50:36.744 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:50:36.744 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:50:36.744 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.34e+01 / 1.13e+00 | 20260507 05:50:36.744 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.03e-01, 1.02e-02 / 5.14e-03 | 20260507 05:50:36.744 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.48e+00, 4.99e-01 / 2.09e-01 | 20260507 05:50:36.744 | INFO | PerfReport: 3.21%, Histogram -- 9.86e-01 / 1.48e+00, 1.64e-01 / 1.12e-01 | 20260507 05:50:36.744 | INFO | PerfReport: 16.22%, DME -- 3.98e+01 / 4.45e+01, 3.90e+00 / 1.19e+00 | 20260507 05:50:36.744 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.77e+01 / 1.99e+01, 1.23e+00 / 1.07e+00 | 20260507 05:50:36.744 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.13e+00 / 2.71e+00, 3.85e-01 / 1.85e-01 | 20260507 05:50:36.744 | INFO | PerfReport: 2.98%, Metrics -- 9.14e-01 / 1.22e+00, 2.46e-01 / 1.29e-01 | 20260507 05:50:36.744 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.08e-01 / 3.64e-01, 5.07e-02 / 2.10e-02 | 20260507 05:50:36.744 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:50:36.744 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:50:36.744 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:50:36.744 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 05:50:36.744 | INFO | PerfReport: 21.91%, Upload -- 2.11e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260507 05:50:36.744 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260507 05:50:36.744 | INFO | PerfReport: 31.90%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 05:50:36.744 | INFO | PerfReport: 1.53%, LockUpload -- 1.47e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 05:50:36.744 | INFO | PerfReport: 1.27%, LockDownload -- 1.22e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260507 05:50:36.744 | INFO | PerfReport: 21.97%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 05:50:36.744 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260507 05:50:36.840 | INFO | Starting a round of Baz IO | 20260507 05:50:39.944 | INFO | Populating RTMetrics report | 20260507 05:50:39.945 | INFO | RTMetrics report populated | 20260507 05:50:39.945 | INFO | Opening tmp RTMetrics json file | 20260507 05:50:39.946 | INFO | Serializing RTMetrics to json | 20260507 05:50:39.947 | INFO | Serialization done, swapping tmp file now... | 20260507 05:50:39.947 | INFO | RTMetrics done | 20260507 05:50:40.039 | INFO | Finished a round of Baz IO: Wrote 1.684 GiB in 3.198 seconds | 20260507 05:50:40.039 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.67% / 0.00% / 67.33% | 20260507 05:50:40.039 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.135088% / 1.19209e-05% / 7.94729e-05% | 20260507 05:50:40.039 | INFO | Baz IO overall: | 20260507 05:50:40.039 | INFO | BytesWritten : 3839904272834 | 20260507 05:50:40.039 | INFO | NumEvents : 1841181762468 | 20260507 05:50:41.873 | INFO | Analyzing chunk frames = [7585792,7586304) | 20260507 05:50:41.873 | INFO | MemInfo: { | 20260507 05:50:41.873 | INFO | "gpu" : " 17 GB", | 20260507 05:50:41.873 | INFO | "huge" : "", | 20260507 05:50:41.873 | INFO | "huge_shared" : " 27 GB", | 20260507 05:50:41.873 | INFO | "malloc" : " 14 GB", | 20260507 05:50:41.873 | INFO | "pinned" : " 29 GB" | 20260507 05:50:41.873 | INFO | } | 20260507 05:50:41.873 | INFO | Perf report for the last 5.12984 seconds: | 20260507 05:50:41.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:50:41.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:41.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:41.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:41.873 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260507 05:50:41.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:41.873 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260507 05:50:41.873 | INFO | PRE_HQ : 60.1%, 1.9, 59.2ms, 0 | 20260507 05:50:41.873 | INFO | BAZWRITER : 62.0%, 3.3, 105.3ms, 0 | 20260507 05:50:41.873 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 05:50:41.873 | INFO | Uploaded 36GiB to GPU in 2.0684s (17.4048GiB/s) | 20260507 05:50:41.873 | INFO | Downloaded 38GiB to GPU in 2.17776s (17.4491GiB/s) | 20260507 05:50:47.016 | INFO | Analyzing chunk frames = [7586304,7586816) | 20260507 05:50:47.016 | INFO | MemInfo: { | 20260507 05:50:47.016 | INFO | "gpu" : " 17 GB", | 20260507 05:50:47.016 | INFO | "huge" : "", | 20260507 05:50:47.016 | INFO | "huge_shared" : " 27 GB", | 20260507 05:50:47.016 | INFO | "malloc" : " 14 GB", | 20260507 05:50:47.016 | INFO | "pinned" : " 29 GB" | 20260507 05:50:47.017 | INFO | } | 20260507 05:50:47.017 | INFO | Perf report for the last 5.14398 seconds: | 20260507 05:50:47.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:50:47.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:47.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:47.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:47.017 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 05:50:47.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:47.017 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:47.017 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:50:47.017 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:47.017 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:47.017 | INFO | Uploaded 36GiB to GPU in 2.04951s (17.5652GiB/s) | 20260507 05:50:47.017 | INFO | Downloaded 34GiB to GPU in 1.93883s (17.5363GiB/s) | 20260507 05:50:52.159 | INFO | Analyzing chunk frames = [7586816,7587328) | 20260507 05:50:52.159 | INFO | MemInfo: { | 20260507 05:50:52.159 | INFO | "gpu" : " 17 GB", | 20260507 05:50:52.159 | INFO | "huge" : "", | 20260507 05:50:52.159 | INFO | "huge_shared" : " 27 GB", | 20260507 05:50:52.159 | INFO | "malloc" : " 14 GB", | 20260507 05:50:52.159 | INFO | "pinned" : " 29 GB" | 20260507 05:50:52.159 | INFO | } | 20260507 05:50:52.159 | INFO | Perf report for the last 5.1428 seconds: | 20260507 05:50:52.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:50:52.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:52.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:52.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:52.160 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 05:50:52.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:52.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:52.160 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:50:52.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:52.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:52.160 | INFO | Uploaded 36GiB to GPU in 2.01333s (17.8808GiB/s) | 20260507 05:50:52.160 | INFO | Downloaded 34GiB to GPU in 1.94325s (17.4964GiB/s) | 20260507 05:50:57.301 | INFO | Analyzing chunk frames = [7587328,7587840) | 20260507 05:50:57.301 | INFO | MemInfo: { | 20260507 05:50:57.301 | INFO | "gpu" : " 17 GB", | 20260507 05:50:57.301 | INFO | "huge" : "", | 20260507 05:50:57.301 | INFO | "huge_shared" : " 27 GB", | 20260507 05:50:57.301 | INFO | "malloc" : " 14 GB", | 20260507 05:50:57.302 | INFO | "pinned" : " 29 GB" | 20260507 05:50:57.302 | INFO | } | 20260507 05:50:57.302 | INFO | Perf report for the last 5.14213 seconds: | 20260507 05:50:57.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:50:57.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:57.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:57.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:57.302 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 05:50:57.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:57.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:57.302 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:50:57.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:57.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:50:57.302 | INFO | Uploaded 36GiB to GPU in 2.01229s (17.8901GiB/s) | 20260507 05:50:57.302 | INFO | Downloaded 34GiB to GPU in 1.94141s (17.513GiB/s) | 20260507 05:51:02.434 | INFO | Analyzing chunk frames = [7587840,7588352) | 20260507 05:51:02.434 | INFO | MemInfo: { | 20260507 05:51:02.434 | INFO | "gpu" : " 17 GB", | 20260507 05:51:02.434 | INFO | "huge" : "", | 20260507 05:51:02.434 | INFO | "huge_shared" : " 27 GB", | 20260507 05:51:02.434 | INFO | "malloc" : " 14 GB", | 20260507 05:51:02.434 | INFO | "pinned" : " 29 GB" | 20260507 05:51:02.434 | INFO | } | 20260507 05:51:02.435 | INFO | Perf report for the last 5.133 seconds: | 20260507 05:51:02.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:51:02.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:02.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:02.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:02.435 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 05:51:02.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:02.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:02.435 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:51:02.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:02.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:02.435 | INFO | Uploaded 36GiB to GPU in 2.0575s (17.497GiB/s) | 20260507 05:51:02.435 | INFO | Downloaded 34GiB to GPU in 1.90725s (17.8267GiB/s) | 20260507 05:51:07.575 | INFO | Analyzing chunk frames = [7588352,7588864) | 20260507 05:51:07.575 | INFO | MemInfo: { | 20260507 05:51:07.575 | INFO | "gpu" : " 17 GB", | 20260507 05:51:07.575 | INFO | "huge" : "", | 20260507 05:51:07.575 | INFO | "huge_shared" : " 27 GB", | 20260507 05:51:07.575 | INFO | "malloc" : " 14 GB", | 20260507 05:51:07.575 | INFO | "pinned" : " 29 GB" | 20260507 05:51:07.575 | INFO | } | 20260507 05:51:07.575 | INFO | Perf report for the last 5.14019 seconds: | 20260507 05:51:07.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:51:07.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:07.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:07.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:07.575 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 05:51:07.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:07.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:07.575 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:51:07.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:07.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:07.575 | INFO | Uploaded 36GiB to GPU in 2.04396s (17.6129GiB/s) | 20260507 05:51:07.575 | INFO | Downloaded 34GiB to GPU in 1.90085s (17.8868GiB/s) | 20260507 05:51:12.715 | INFO | Analyzing chunk frames = [7588864,7589376) | 20260507 05:51:12.715 | INFO | MemInfo: { | 20260507 05:51:12.715 | INFO | "gpu" : " 17 GB", | 20260507 05:51:12.715 | INFO | "huge" : "", | 20260507 05:51:12.715 | INFO | "huge_shared" : " 27 GB", | 20260507 05:51:12.715 | INFO | "malloc" : " 14 GB", | 20260507 05:51:12.715 | INFO | "pinned" : " 29 GB" | 20260507 05:51:12.715 | INFO | } | 20260507 05:51:12.716 | INFO | Perf report for the last 5.14064 seconds: | 20260507 05:51:12.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:51:12.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:12.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:12.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:12.716 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:51:12.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:12.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:12.716 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:51:12.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:12.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:12.716 | INFO | Uploaded 36GiB to GPU in 2.00451s (17.9595GiB/s) | 20260507 05:51:12.716 | INFO | Downloaded 34GiB to GPU in 1.98384s (17.1385GiB/s) | 20260507 05:51:17.856 | INFO | Analyzing chunk frames = [7589376,7589888) | 20260507 05:51:17.856 | INFO | MemInfo: { | 20260507 05:51:17.856 | INFO | "gpu" : " 17 GB", | 20260507 05:51:17.856 | INFO | "huge" : "", | 20260507 05:51:17.856 | INFO | "huge_shared" : " 27 GB", | 20260507 05:51:17.856 | INFO | "malloc" : " 14 GB", | 20260507 05:51:17.856 | INFO | "pinned" : " 29 GB" | 20260507 05:51:17.856 | INFO | } | 20260507 05:51:17.856 | INFO | Perf report for the last 5.14033 seconds: | 20260507 05:51:17.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:51:17.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:17.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:17.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:17.856 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:51:17.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:17.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:17.856 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 05:51:17.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:17.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:17.856 | INFO | Uploaded 36GiB to GPU in 1.9806s (18.1763GiB/s) | 20260507 05:51:17.856 | INFO | Downloaded 34GiB to GPU in 1.9575s (17.3691GiB/s) | 20260507 05:51:17.954 | INFO | Starting a round of Baz IO | 20260507 05:51:21.049 | INFO | Populating RTMetrics report | 20260507 05:51:21.049 | INFO | RTMetrics report populated | 20260507 05:51:21.049 | INFO | Opening tmp RTMetrics json file | 20260507 05:51:21.050 | INFO | Serializing RTMetrics to json | 20260507 05:51:21.051 | INFO | Serialization done, swapping tmp file now... | 20260507 05:51:21.051 | INFO | RTMetrics done | 20260507 05:51:21.134 | INFO | Finished a round of Baz IO: Wrote 1.685 GiB in 3.179 seconds | 20260507 05:51:21.134 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.67% / 0.00% / 67.33% | 20260507 05:51:21.134 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.134945% / 0% / 6.75519e-05% | 20260507 05:51:21.134 | INFO | Baz IO overall: | 20260507 05:51:21.134 | INFO | BytesWritten : 3841713111279 | 20260507 05:51:21.134 | INFO | NumEvents : 1842030303136 | 20260507 05:51:22.993 | INFO | Analyzing chunk frames = [7589888,7590400) | 20260507 05:51:22.993 | INFO | MemInfo: { | 20260507 05:51:22.993 | INFO | "gpu" : " 17 GB", | 20260507 05:51:22.993 | INFO | "huge" : "", | 20260507 05:51:22.993 | INFO | "huge_shared" : " 27 GB", | 20260507 05:51:22.993 | INFO | "malloc" : " 14 GB", | 20260507 05:51:22.993 | INFO | "pinned" : " 29 GB" | 20260507 05:51:22.993 | INFO | } | 20260507 05:51:22.993 | INFO | Perf report for the last 5.13742 seconds: | 20260507 05:51:22.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:51:22.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:22.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:22.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:22.993 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:51:22.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:22.993 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 05:51:22.993 | INFO | PRE_HQ : 59.5%, 1.9, 58.5ms, 0 | 20260507 05:51:22.993 | INFO | BAZWRITER : 61.4%, 3.3, 103.5ms, 0 | 20260507 05:51:22.993 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:51:22.994 | INFO | Uploaded 36GiB to GPU in 2.05636s (17.5067GiB/s) | 20260507 05:51:22.994 | INFO | Downloaded 38GiB to GPU in 2.17716s (17.4539GiB/s) | 20260507 05:51:28.128 | INFO | Analyzing chunk frames = [7590400,7590912) | 20260507 05:51:28.128 | INFO | MemInfo: { | 20260507 05:51:28.128 | INFO | "gpu" : " 17 GB", | 20260507 05:51:28.128 | INFO | "huge" : "", | 20260507 05:51:28.128 | INFO | "huge_shared" : " 27 GB", | 20260507 05:51:28.128 | INFO | "malloc" : " 14 GB", | 20260507 05:51:28.128 | INFO | "pinned" : " 29 GB" | 20260507 05:51:28.128 | INFO | } | 20260507 05:51:28.128 | INFO | Perf report for the last 5.13456 seconds: | 20260507 05:51:28.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:51:28.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:28.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:28.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:28.128 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 05:51:28.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:28.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:28.128 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 05:51:28.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:28.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:28.128 | INFO | Uploaded 36GiB to GPU in 1.98595s (18.1274GiB/s) | 20260507 05:51:28.128 | INFO | Downloaded 34GiB to GPU in 1.92679s (17.646GiB/s) | 20260507 05:51:33.274 | INFO | Analyzing chunk frames = [7590912,7591424) | 20260507 05:51:33.274 | INFO | MemInfo: { | 20260507 05:51:33.274 | INFO | "gpu" : " 17 GB", | 20260507 05:51:33.274 | INFO | "huge" : "", | 20260507 05:51:33.274 | INFO | "huge_shared" : " 27 GB", | 20260507 05:51:33.274 | INFO | "malloc" : " 14 GB", | 20260507 05:51:33.274 | INFO | "pinned" : " 29 GB" | 20260507 05:51:33.274 | INFO | } | 20260507 05:51:33.274 | INFO | Perf report for the last 5.14616 seconds: | 20260507 05:51:33.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:51:33.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:33.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:33.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:33.274 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 05:51:33.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:33.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:33.274 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:51:33.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:33.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:33.274 | INFO | Uploaded 36GiB to GPU in 2.01476s (17.8682GiB/s) | 20260507 05:51:33.275 | INFO | Downloaded 34GiB to GPU in 1.94187s (17.5089GiB/s) | 20260507 05:51:38.411 | INFO | Analyzing chunk frames = [7591424,7591936) | 20260507 05:51:38.411 | INFO | MemInfo: { | 20260507 05:51:38.411 | INFO | "gpu" : " 17 GB", | 20260507 05:51:38.412 | INFO | "huge" : "", | 20260507 05:51:38.412 | INFO | "huge_shared" : " 27 GB", | 20260507 05:51:38.412 | INFO | "malloc" : " 14 GB", | 20260507 05:51:38.412 | INFO | "pinned" : " 29 GB" | 20260507 05:51:38.412 | INFO | } | 20260507 05:51:38.412 | INFO | Perf report for the last 5.13753 seconds: | 20260507 05:51:38.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:51:38.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:38.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:38.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:38.412 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:51:38.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:38.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:38.412 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:51:38.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:38.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:38.412 | INFO | Uploaded 36GiB to GPU in 2.01806s (17.8389GiB/s) | 20260507 05:51:38.412 | INFO | Downloaded 34GiB to GPU in 1.9358s (17.5638GiB/s) | 20260507 05:51:43.552 | INFO | Analyzing chunk frames = [7591936,7592448) | 20260507 05:51:43.552 | INFO | MemInfo: { | 20260507 05:51:43.552 | INFO | "gpu" : " 17 GB", | 20260507 05:51:43.552 | INFO | "huge" : "", | 20260507 05:51:43.552 | INFO | "huge_shared" : " 27 GB", | 20260507 05:51:43.552 | INFO | "malloc" : " 14 GB", | 20260507 05:51:43.552 | INFO | "pinned" : " 29 GB" | 20260507 05:51:43.552 | INFO | } | 20260507 05:51:43.552 | INFO | Perf report for the last 5.14047 seconds: | 20260507 05:51:43.552 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:51:43.552 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:43.552 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:43.552 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:43.552 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 05:51:43.552 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:43.552 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:43.552 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:51:43.552 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:43.552 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:43.552 | INFO | Uploaded 36GiB to GPU in 2.05521s (17.5164GiB/s) | 20260507 05:51:43.552 | INFO | Downloaded 34GiB to GPU in 1.94829s (17.4512GiB/s) | 20260507 05:51:48.684 | INFO | Analyzing chunk frames = [7592448,7592960) | 20260507 05:51:48.684 | INFO | MemInfo: { | 20260507 05:51:48.684 | INFO | "gpu" : " 17 GB", | 20260507 05:51:48.684 | INFO | "huge" : "", | 20260507 05:51:48.684 | INFO | "huge_shared" : " 27 GB", | 20260507 05:51:48.684 | INFO | "malloc" : " 14 GB", | 20260507 05:51:48.684 | INFO | "pinned" : " 29 GB" | 20260507 05:51:48.684 | INFO | } | 20260507 05:51:48.684 | INFO | Perf report for the last 5.13226 seconds: | 20260507 05:51:48.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:51:48.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:48.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:48.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:48.685 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 05:51:48.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:48.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:48.685 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:51:48.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:48.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:48.685 | INFO | Uploaded 36GiB to GPU in 2.06723s (17.4146GiB/s) | 20260507 05:51:48.685 | INFO | Downloaded 34GiB to GPU in 1.92129s (17.6964GiB/s) | 20260507 05:51:53.828 | INFO | Analyzing chunk frames = [7592960,7593472) | 20260507 05:51:53.828 | INFO | MemInfo: { | 20260507 05:51:53.828 | INFO | "gpu" : " 17 GB", | 20260507 05:51:53.828 | INFO | "huge" : "", | 20260507 05:51:53.828 | INFO | "huge_shared" : " 27 GB", | 20260507 05:51:53.828 | INFO | "malloc" : " 14 GB", | 20260507 05:51:53.828 | INFO | "pinned" : " 29 GB" | 20260507 05:51:53.828 | INFO | } | 20260507 05:51:53.828 | INFO | Perf report for the last 5.14414 seconds: | 20260507 05:51:53.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:51:53.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:53.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:53.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:53.829 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:51:53.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:53.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:53.829 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:51:53.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:53.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:53.829 | INFO | Uploaded 36GiB to GPU in 1.99223s (18.0702GiB/s) | 20260507 05:51:53.829 | INFO | Downloaded 34GiB to GPU in 1.9838s (17.1388GiB/s) | 20260507 05:51:58.972 | INFO | Analyzing chunk frames = [7593472,7593984) | 20260507 05:51:58.972 | INFO | MemInfo: { | 20260507 05:51:58.972 | INFO | "gpu" : " 17 GB", | 20260507 05:51:58.972 | INFO | "huge" : "", | 20260507 05:51:58.972 | INFO | "huge_shared" : " 27 GB", | 20260507 05:51:58.972 | INFO | "malloc" : " 14 GB", | 20260507 05:51:58.972 | INFO | "pinned" : " 29 GB" | 20260507 05:51:58.972 | INFO | } | 20260507 05:51:58.972 | INFO | Perf report for the last 5.14379 seconds: | 20260507 05:51:58.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:51:58.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:58.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:58.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:58.972 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 05:51:58.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:58.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:58.972 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:51:58.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:58.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:51:58.973 | INFO | Uploaded 36GiB to GPU in 1.97379s (18.239GiB/s) | 20260507 05:51:58.973 | INFO | Downloaded 34GiB to GPU in 1.96596s (17.2944GiB/s) | 20260507 05:51:58.973 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:51:58.973 | INFO | Total mem usage: 18 GiB | 20260507 05:51:58.973 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:51:58.973 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 05:51:58.973 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:51:58.973 | INFO | High Water Breakdown: | 20260507 05:51:58.973 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:51:58.973 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:51:58.973 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:51:58.973 | INFO | ------------------------------------------ | 20260507 05:51:58.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:51:58.973 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:51:58.973 | INFO | Total mem usage: 26.5 GiB | 20260507 05:51:58.973 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:51:58.973 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:51:58.973 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:51:58.973 | INFO | High Water Breakdown: | 20260507 05:51:58.973 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:51:58.973 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:51:58.973 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:51:58.973 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:51:58.973 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:51:58.973 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:51:58.973 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:51:58.973 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:51:58.973 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:51:58.973 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:51:58.973 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:51:58.973 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:51:58.973 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:51:58.973 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:51:58.973 | INFO | ------------------------------------------ | 20260507 05:51:58.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:51:58.973 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:51:58.973 | INFO | Total mem usage: 25.6 GiB | 20260507 05:51:58.973 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:51:58.973 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:51:58.973 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:51:58.973 | INFO | High Water Breakdown: | 20260507 05:51:58.973 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:51:58.973 | INFO | ------------------------------------------ | 20260507 05:51:58.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:51:58.973 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:51:58.973 | INFO | Total mem usage: 15.7 GiB | 20260507 05:51:58.973 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:51:58.973 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:51:58.973 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:51:58.973 | INFO | High Water Breakdown: | 20260507 05:51:58.973 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:51:58.973 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:51:58.973 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:51:58.973 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:51:58.973 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:51:58.973 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:51:58.973 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:51:58.973 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:51:58.973 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:51:58.973 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:51:58.973 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:51:58.973 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:51:58.973 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:51:58.973 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:51:58.973 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:51:58.974 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:51:58.974 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:51:58.974 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:51:58.974 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:51:58.974 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:51:58.974 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:51:58.974 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:51:58.974 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:51:58.974 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:51:58.974 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:51:58.974 | INFO | ------------------------------------------ | 20260507 05:51:58.974 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:51:58.974 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:51:58.974 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:51:58.974 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:51:58.974 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.42e+01, 1.31e+01 / 1.04e+00 | 20260507 05:51:58.974 | INFO | PerfReport: 0.30%, AggregateStats -- 9.05e-02 / 1.01e-01, 1.01e-02 / 4.50e-03 | 20260507 05:51:58.974 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.46e+00, 5.06e-01 / 2.92e-01 | 20260507 05:51:58.974 | INFO | PerfReport: 3.22%, Histogram -- 9.87e-01 / 1.46e+00, 1.63e-01 / 1.18e-01 | 20260507 05:51:58.974 | INFO | PerfReport: 16.13%, DME -- 3.95e+01 / 4.38e+01, 3.62e+00 / 1.12e+00 | 20260507 05:51:58.974 | INFO | PerfReport: 57.74%, FrameLabeling -- 1.77e+01 / 2.02e+01, 1.27e+00 / 1.41e+00 | 20260507 05:51:58.974 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.13e+00 / 2.66e+00, 3.86e-01 / 2.20e-01 | 20260507 05:51:58.974 | INFO | PerfReport: 2.99%, Metrics -- 9.16e-01 / 1.21e+00, 2.47e-01 / 1.14e-01 | 20260507 05:51:58.974 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.07e-01 / 3.62e-01, 5.05e-02 / 1.88e-02 | 20260507 05:51:58.974 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:51:58.974 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:51:58.974 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:51:58.974 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 05:51:58.974 | INFO | PerfReport: 21.95%, Upload -- 2.11e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260507 05:51:58.974 | INFO | PerfReport: 21.38%, Download -- 2.05e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260507 05:51:58.974 | INFO | PerfReport: 31.89%, Compute -- 3.06e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 05:51:58.974 | INFO | PerfReport: 1.59%, LockUpload -- 1.53e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260507 05:51:58.974 | INFO | PerfReport: 1.20%, LockDownload -- 1.16e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260507 05:51:58.974 | INFO | PerfReport: 21.97%, LockCompute -- 2.11e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 05:51:58.974 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.63e-02 / 0.00e+00 | 20260507 05:51:59.070 | INFO | Starting a round of Baz IO | 20260507 05:52:02.164 | INFO | Populating RTMetrics report | 20260507 05:52:02.164 | INFO | RTMetrics report populated | 20260507 05:52:02.164 | INFO | Opening tmp RTMetrics json file | 20260507 05:52:02.181 | INFO | Serializing RTMetrics to json | 20260507 05:52:02.182 | INFO | Serialization done, swapping tmp file now... | 20260507 05:52:02.182 | INFO | RTMetrics done | 20260507 05:52:02.250 | INFO | Finished a round of Baz IO: Wrote 1.684 GiB in 3.18 seconds | 20260507 05:52:02.250 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.67% / 0.00% / 67.33% | 20260507 05:52:02.250 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.134373% / 0% / 8.34465e-05% | 20260507 05:52:02.250 | INFO | Baz IO overall: | 20260507 05:52:02.250 | INFO | BytesWritten : 3843521238699 | 20260507 05:52:02.250 | INFO | NumEvents : 1842878285351 | 20260507 05:52:04.111 | INFO | Analyzing chunk frames = [7593984,7594496) | 20260507 05:52:04.111 | INFO | MemInfo: { | 20260507 05:52:04.111 | INFO | "gpu" : " 17 GB", | 20260507 05:52:04.111 | INFO | "huge" : "", | 20260507 05:52:04.111 | INFO | "huge_shared" : " 27 GB", | 20260507 05:52:04.111 | INFO | "malloc" : " 14 GB", | 20260507 05:52:04.111 | INFO | "pinned" : " 29 GB" | 20260507 05:52:04.111 | INFO | } | 20260507 05:52:04.111 | INFO | Perf report for the last 5.13881 seconds: | 20260507 05:52:04.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:52:04.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:04.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:04.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:04.111 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 05:52:04.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:04.111 | INFO | RT_METRICS : 41.7%, 1.0, 21.8ms, 0 | 20260507 05:52:04.111 | INFO | PRE_HQ : 59.0%, 1.8, 54.9ms, 0 | 20260507 05:52:04.111 | INFO | BAZWRITER : 61.5%, 3.2, 100.5ms, 0 | 20260507 05:52:04.111 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 05:52:04.111 | INFO | Uploaded 36GiB to GPU in 2.04766s (17.581GiB/s) | 20260507 05:52:04.111 | INFO | Downloaded 38GiB to GPU in 2.17935s (17.4364GiB/s) | 20260507 05:52:09.251 | INFO | Analyzing chunk frames = [7594496,7595008) | 20260507 05:52:09.251 | INFO | MemInfo: { | 20260507 05:52:09.251 | INFO | "gpu" : " 17 GB", | 20260507 05:52:09.251 | INFO | "huge" : "", | 20260507 05:52:09.251 | INFO | "huge_shared" : " 27 GB", | 20260507 05:52:09.251 | INFO | "malloc" : " 14 GB", | 20260507 05:52:09.251 | INFO | "pinned" : " 29 GB" | 20260507 05:52:09.251 | INFO | } | 20260507 05:52:09.251 | INFO | Perf report for the last 5.14021 seconds: | 20260507 05:52:09.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:52:09.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:09.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:09.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:09.251 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:52:09.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:09.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:09.251 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 05:52:09.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:09.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:09.252 | INFO | Uploaded 36GiB to GPU in 1.98191s (18.1643GiB/s) | 20260507 05:52:09.252 | INFO | Downloaded 34GiB to GPU in 1.92863s (17.6291GiB/s) | 20260507 05:52:14.380 | INFO | Analyzing chunk frames = [7595008,7595520) | 20260507 05:52:14.380 | INFO | MemInfo: { | 20260507 05:52:14.381 | INFO | "gpu" : " 17 GB", | 20260507 05:52:14.381 | INFO | "huge" : "", | 20260507 05:52:14.381 | INFO | "huge_shared" : " 27 GB", | 20260507 05:52:14.381 | INFO | "malloc" : " 14 GB", | 20260507 05:52:14.381 | INFO | "pinned" : " 29 GB" | 20260507 05:52:14.381 | INFO | } | 20260507 05:52:14.381 | INFO | Perf report for the last 5.12939 seconds: | 20260507 05:52:14.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:52:14.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:14.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:14.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:14.381 | INFO | ANALYSIS : 60.7%, 3.0, 93.8ms, 0 | 20260507 05:52:14.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:14.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:14.381 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:52:14.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:14.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:14.381 | INFO | Uploaded 36GiB to GPU in 2.0496s (17.5644GiB/s) | 20260507 05:52:14.381 | INFO | Downloaded 34GiB to GPU in 1.95029s (17.4333GiB/s) | 20260507 05:52:19.525 | INFO | Analyzing chunk frames = [7595520,7596032) | 20260507 05:52:19.525 | INFO | MemInfo: { | 20260507 05:52:19.525 | INFO | "gpu" : " 17 GB", | 20260507 05:52:19.525 | INFO | "huge" : "", | 20260507 05:52:19.525 | INFO | "huge_shared" : " 27 GB", | 20260507 05:52:19.525 | INFO | "malloc" : " 14 GB", | 20260507 05:52:19.525 | INFO | "pinned" : " 29 GB" | 20260507 05:52:19.525 | INFO | } | 20260507 05:52:19.526 | INFO | Perf report for the last 5.14492 seconds: | 20260507 05:52:19.526 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:52:19.526 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:19.526 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:19.526 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:19.526 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 05:52:19.526 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:19.526 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:19.526 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:52:19.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:19.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:19.526 | INFO | Uploaded 36GiB to GPU in 1.99112s (18.0802GiB/s) | 20260507 05:52:19.526 | INFO | Downloaded 34GiB to GPU in 1.90799s (17.8198GiB/s) | 20260507 05:52:24.666 | INFO | Analyzing chunk frames = [7596032,7596544) | 20260507 05:52:24.666 | INFO | MemInfo: { | 20260507 05:52:24.666 | INFO | "gpu" : " 17 GB", | 20260507 05:52:24.666 | INFO | "huge" : "", | 20260507 05:52:24.666 | INFO | "huge_shared" : " 27 GB", | 20260507 05:52:24.666 | INFO | "malloc" : " 14 GB", | 20260507 05:52:24.666 | INFO | "pinned" : " 29 GB" | 20260507 05:52:24.666 | INFO | } | 20260507 05:52:24.666 | INFO | Perf report for the last 5.14046 seconds: | 20260507 05:52:24.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:52:24.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:24.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:24.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:24.666 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 05:52:24.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:24.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:24.666 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:52:24.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:24.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:24.666 | INFO | Uploaded 36GiB to GPU in 2.05338s (17.5321GiB/s) | 20260507 05:52:24.666 | INFO | Downloaded 34GiB to GPU in 1.94331s (17.4959GiB/s) | 20260507 05:52:29.810 | INFO | Analyzing chunk frames = [7596544,7597056) | 20260507 05:52:29.810 | INFO | MemInfo: { | 20260507 05:52:29.810 | INFO | "gpu" : " 17 GB", | 20260507 05:52:29.810 | INFO | "huge" : "", | 20260507 05:52:29.810 | INFO | "huge_shared" : " 27 GB", | 20260507 05:52:29.810 | INFO | "malloc" : " 14 GB", | 20260507 05:52:29.810 | INFO | "pinned" : " 29 GB" | 20260507 05:52:29.810 | INFO | } | 20260507 05:52:29.810 | INFO | Perf report for the last 5.14361 seconds: | 20260507 05:52:29.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:52:29.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:29.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:29.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:29.810 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 05:52:29.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:29.810 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:29.810 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 05:52:29.810 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:29.810 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:29.810 | INFO | Uploaded 36GiB to GPU in 2.05708s (17.5006GiB/s) | 20260507 05:52:29.810 | INFO | Downloaded 34GiB to GPU in 1.9154s (17.7508GiB/s) | 20260507 05:52:34.948 | INFO | Analyzing chunk frames = [7597056,7597568) | 20260507 05:52:34.948 | INFO | MemInfo: { | 20260507 05:52:34.948 | INFO | "gpu" : " 17 GB", | 20260507 05:52:34.948 | INFO | "huge" : "", | 20260507 05:52:34.948 | INFO | "huge_shared" : " 27 GB", | 20260507 05:52:34.948 | INFO | "malloc" : " 14 GB", | 20260507 05:52:34.948 | INFO | "pinned" : " 29 GB" | 20260507 05:52:34.948 | INFO | } | 20260507 05:52:34.948 | INFO | Perf report for the last 5.1381 seconds: | 20260507 05:52:34.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:52:34.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:34.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:34.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:34.948 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 05:52:34.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:34.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:34.948 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 05:52:34.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:34.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:34.948 | INFO | Uploaded 36GiB to GPU in 1.90854s (18.8626GiB/s) | 20260507 05:52:34.948 | INFO | Downloaded 34GiB to GPU in 1.97627s (17.2042GiB/s) | 20260507 05:52:40.085 | INFO | Analyzing chunk frames = [7597568,7598080) | 20260507 05:52:40.085 | INFO | MemInfo: { | 20260507 05:52:40.085 | INFO | "gpu" : " 17 GB", | 20260507 05:52:40.085 | INFO | "huge" : "", | 20260507 05:52:40.085 | INFO | "huge_shared" : " 27 GB", | 20260507 05:52:40.085 | INFO | "malloc" : " 14 GB", | 20260507 05:52:40.086 | INFO | "pinned" : " 29 GB" | 20260507 05:52:40.086 | INFO | } | 20260507 05:52:40.086 | INFO | Perf report for the last 5.13769 seconds: | 20260507 05:52:40.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:52:40.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:40.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:40.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:40.086 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 05:52:40.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:40.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:40.086 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:52:40.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:40.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:40.086 | INFO | Uploaded 36GiB to GPU in 1.97842s (18.1963GiB/s) | 20260507 05:52:40.086 | INFO | Downloaded 34GiB to GPU in 1.96309s (17.3196GiB/s) | 20260507 05:52:40.183 | INFO | Starting a round of Baz IO | 20260507 05:52:43.279 | INFO | Populating RTMetrics report | 20260507 05:52:43.279 | INFO | RTMetrics report populated | 20260507 05:52:43.279 | INFO | Opening tmp RTMetrics json file | 20260507 05:52:43.284 | INFO | Serializing RTMetrics to json | 20260507 05:52:43.297 | INFO | Serialization done, swapping tmp file now... | 20260507 05:52:43.298 | INFO | RTMetrics done | 20260507 05:52:43.368 | INFO | Finished a round of Baz IO: Wrote 1.683 GiB in 3.184 seconds | 20260507 05:52:43.368 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.67% / 0.00% / 67.33% | 20260507 05:52:43.368 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.133582% / 3.97364e-06% / 8.74201e-05% | 20260507 05:52:43.368 | INFO | Baz IO overall: | 20260507 05:52:43.368 | INFO | BytesWritten : 3845327830919 | 20260507 05:52:43.368 | INFO | NumEvents : 1843725312242 | 20260507 05:52:45.224 | INFO | Analyzing chunk frames = [7598080,7598592) | 20260507 05:52:45.224 | INFO | MemInfo: { | 20260507 05:52:45.224 | INFO | "gpu" : " 17 GB", | 20260507 05:52:45.224 | INFO | "huge" : "", | 20260507 05:52:45.224 | INFO | "huge_shared" : " 27 GB", | 20260507 05:52:45.224 | INFO | "malloc" : " 14 GB", | 20260507 05:52:45.224 | INFO | "pinned" : " 29 GB" | 20260507 05:52:45.224 | INFO | } | 20260507 05:52:45.224 | INFO | Perf report for the last 5.1383 seconds: | 20260507 05:52:45.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:52:45.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:45.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:45.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:45.224 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:52:45.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:45.224 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 05:52:45.224 | INFO | PRE_HQ : 58.9%, 1.8, 54.8ms, 0 | 20260507 05:52:45.224 | INFO | BAZWRITER : 61.5%, 3.2, 100.1ms, 0 | 20260507 05:52:45.224 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 05:52:45.224 | INFO | Uploaded 36GiB to GPU in 2.04351s (17.6168GiB/s) | 20260507 05:52:45.224 | INFO | Downloaded 38GiB to GPU in 2.18576s (17.3853GiB/s) | 20260507 05:52:50.361 | INFO | Analyzing chunk frames = [7598592,7599104) | 20260507 05:52:50.361 | INFO | MemInfo: { | 20260507 05:52:50.361 | INFO | "gpu" : " 17 GB", | 20260507 05:52:50.361 | INFO | "huge" : "", | 20260507 05:52:50.362 | INFO | "huge_shared" : " 27 GB", | 20260507 05:52:50.362 | INFO | "malloc" : " 14 GB", | 20260507 05:52:50.362 | INFO | "pinned" : " 29 GB" | 20260507 05:52:50.362 | INFO | } | 20260507 05:52:50.362 | INFO | Perf report for the last 5.13775 seconds: | 20260507 05:52:50.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:52:50.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:50.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:50.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:50.362 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:52:50.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:50.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:50.362 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:52:50.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:50.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:50.362 | INFO | Uploaded 36GiB to GPU in 2.00092s (17.9917GiB/s) | 20260507 05:52:50.362 | INFO | Downloaded 34GiB to GPU in 1.93556s (17.566GiB/s) | 20260507 05:52:55.501 | INFO | Analyzing chunk frames = [7599104,7599616) | 20260507 05:52:55.501 | INFO | MemInfo: { | 20260507 05:52:55.501 | INFO | "gpu" : " 17 GB", | 20260507 05:52:55.501 | INFO | "huge" : "", | 20260507 05:52:55.501 | INFO | "huge_shared" : " 27 GB", | 20260507 05:52:55.501 | INFO | "malloc" : " 14 GB", | 20260507 05:52:55.501 | INFO | "pinned" : " 29 GB" | 20260507 05:52:55.501 | INFO | } | 20260507 05:52:55.501 | INFO | Perf report for the last 5.13922 seconds: | 20260507 05:52:55.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:52:55.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:55.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:55.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:55.501 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 05:52:55.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:55.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:55.501 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:52:55.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:55.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:52:55.501 | INFO | Uploaded 36GiB to GPU in 2.03035s (17.7309GiB/s) | 20260507 05:52:55.501 | INFO | Downloaded 34GiB to GPU in 1.95181s (17.4197GiB/s) | 20260507 05:53:00.634 | INFO | Analyzing chunk frames = [7599616,7600128) | 20260507 05:53:00.634 | INFO | MemInfo: { | 20260507 05:53:00.634 | INFO | "gpu" : " 17 GB", | 20260507 05:53:00.634 | INFO | "huge" : "", | 20260507 05:53:00.634 | INFO | "huge_shared" : " 27 GB", | 20260507 05:53:00.634 | INFO | "malloc" : " 14 GB", | 20260507 05:53:00.634 | INFO | "pinned" : " 29 GB" | 20260507 05:53:00.634 | INFO | } | 20260507 05:53:00.634 | INFO | Perf report for the last 5.13333 seconds: | 20260507 05:53:00.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:53:00.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:00.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:00.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:00.634 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:53:00.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:00.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:00.634 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:53:00.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:00.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:00.634 | INFO | Uploaded 36GiB to GPU in 2.00954s (17.9146GiB/s) | 20260507 05:53:00.635 | INFO | Downloaded 34GiB to GPU in 1.9453s (17.478GiB/s) | 20260507 05:53:05.783 | INFO | Analyzing chunk frames = [7600128,7600640) | 20260507 05:53:05.783 | INFO | MemInfo: { | 20260507 05:53:05.783 | INFO | "gpu" : " 17 GB", | 20260507 05:53:05.783 | INFO | "huge" : "", | 20260507 05:53:05.783 | INFO | "huge_shared" : " 27 GB", | 20260507 05:53:05.783 | INFO | "malloc" : " 14 GB", | 20260507 05:53:05.783 | INFO | "pinned" : " 29 GB" | 20260507 05:53:05.783 | INFO | } | 20260507 05:53:05.783 | INFO | Perf report for the last 5.14922 seconds: | 20260507 05:53:05.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:53:05.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:05.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:05.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:05.784 | INFO | ANALYSIS : 59.9%, 3.0, 93.7ms, 0 | 20260507 05:53:05.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:05.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:05.784 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:53:05.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:05.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:05.784 | INFO | Uploaded 36GiB to GPU in 2.03374s (17.7014GiB/s) | 20260507 05:53:05.784 | INFO | Downloaded 34GiB to GPU in 1.94344s (17.4948GiB/s) | 20260507 05:53:10.925 | INFO | Analyzing chunk frames = [7600640,7601152) | 20260507 05:53:10.925 | INFO | MemInfo: { | 20260507 05:53:10.925 | INFO | "gpu" : " 17 GB", | 20260507 05:53:10.925 | INFO | "huge" : "", | 20260507 05:53:10.925 | INFO | "huge_shared" : " 27 GB", | 20260507 05:53:10.925 | INFO | "malloc" : " 14 GB", | 20260507 05:53:10.925 | INFO | "pinned" : " 29 GB" | 20260507 05:53:10.925 | INFO | } | 20260507 05:53:10.925 | INFO | Perf report for the last 5.14152 seconds: | 20260507 05:53:10.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:53:10.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:10.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:10.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:10.925 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 05:53:10.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:10.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:10.925 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 05:53:10.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:10.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:10.925 | INFO | Uploaded 36GiB to GPU in 2.03711s (17.6721GiB/s) | 20260507 05:53:10.925 | INFO | Downloaded 34GiB to GPU in 1.92048s (17.7039GiB/s) | 20260507 05:53:16.062 | INFO | Analyzing chunk frames = [7601152,7601664) | 20260507 05:53:16.062 | INFO | MemInfo: { | 20260507 05:53:16.062 | INFO | "gpu" : " 17 GB", | 20260507 05:53:16.062 | INFO | "huge" : "", | 20260507 05:53:16.062 | INFO | "huge_shared" : " 27 GB", | 20260507 05:53:16.062 | INFO | "malloc" : " 14 GB", | 20260507 05:53:16.062 | INFO | "pinned" : " 29 GB" | 20260507 05:53:16.062 | INFO | } | 20260507 05:53:16.062 | INFO | Perf report for the last 5.13676 seconds: | 20260507 05:53:16.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:53:16.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:16.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:16.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:16.062 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:53:16.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:16.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:16.062 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 05:53:16.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:16.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:16.062 | INFO | Uploaded 36GiB to GPU in 1.8989s (18.9583GiB/s) | 20260507 05:53:16.062 | INFO | Downloaded 34GiB to GPU in 1.97856s (17.1842GiB/s) | 20260507 05:53:21.191 | INFO | Analyzing chunk frames = [7601664,7602176) | 20260507 05:53:21.191 | INFO | MemInfo: { | 20260507 05:53:21.191 | INFO | "gpu" : " 17 GB", | 20260507 05:53:21.191 | INFO | "huge" : "", | 20260507 05:53:21.191 | INFO | "huge_shared" : " 27 GB", | 20260507 05:53:21.191 | INFO | "malloc" : " 14 GB", | 20260507 05:53:21.191 | INFO | "pinned" : " 29 GB" | 20260507 05:53:21.191 | INFO | } | 20260507 05:53:21.191 | INFO | Perf report for the last 5.12972 seconds: | 20260507 05:53:21.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:53:21.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:21.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:21.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:21.192 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 05:53:21.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:21.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:21.192 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:53:21.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:21.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:21.192 | INFO | Uploaded 36GiB to GPU in 1.99404s (18.0538GiB/s) | 20260507 05:53:21.192 | INFO | Downloaded 34GiB to GPU in 1.95818s (17.3631GiB/s) | 20260507 05:53:21.192 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:53:21.192 | INFO | Total mem usage: 18 GiB | 20260507 05:53:21.192 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:53:21.192 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:53:21.192 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:53:21.192 | INFO | High Water Breakdown: | 20260507 05:53:21.192 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:53:21.192 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:53:21.192 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:53:21.192 | INFO | ------------------------------------------ | 20260507 05:53:21.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:53:21.192 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:53:21.192 | INFO | Total mem usage: 26.5 GiB | 20260507 05:53:21.192 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:53:21.192 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:53:21.192 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:53:21.192 | INFO | High Water Breakdown: | 20260507 05:53:21.192 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:53:21.192 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:53:21.192 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:53:21.192 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:53:21.192 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:53:21.192 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:53:21.192 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:53:21.192 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:53:21.192 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:53:21.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:53:21.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:53:21.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:53:21.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:53:21.192 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:53:21.192 | INFO | ------------------------------------------ | 20260507 05:53:21.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:53:21.192 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:53:21.192 | INFO | Total mem usage: 25.6 GiB | 20260507 05:53:21.192 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:53:21.192 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:53:21.192 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:53:21.192 | INFO | High Water Breakdown: | 20260507 05:53:21.192 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:53:21.192 | INFO | ------------------------------------------ | 20260507 05:53:21.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:53:21.192 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:53:21.192 | INFO | Total mem usage: 15.7 GiB | 20260507 05:53:21.193 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:53:21.193 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:53:21.193 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:53:21.193 | INFO | High Water Breakdown: | 20260507 05:53:21.193 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:53:21.193 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:53:21.193 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:53:21.193 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:53:21.193 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:53:21.193 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:53:21.193 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:53:21.193 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:53:21.193 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:53:21.193 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:53:21.193 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:53:21.193 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:53:21.193 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:53:21.193 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:53:21.193 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:53:21.193 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:53:21.193 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:53:21.193 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:53:21.193 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:53:21.193 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:53:21.193 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:53:21.193 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:53:21.193 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:53:21.193 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:53:21.193 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:53:21.193 | INFO | ------------------------------------------ | 20260507 05:53:21.193 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:53:21.193 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:53:21.193 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:53:21.193 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:53:21.193 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.51e+01, 1.34e+01 / 1.14e+00 | 20260507 05:53:21.193 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.92e-03 / 4.64e-03 | 20260507 05:53:21.193 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.42e+00, 5.18e-01 / 2.81e-01 | 20260507 05:53:21.193 | INFO | PerfReport: 3.24%, Histogram -- 9.95e-01 / 1.49e+00, 1.71e-01 / 9.58e-02 | 20260507 05:53:21.193 | INFO | PerfReport: 16.27%, DME -- 4.00e+01 / 4.52e+01, 3.99e+00 / 1.48e+00 | 20260507 05:53:21.193 | INFO | PerfReport: 57.59%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.24e+00 / 1.34e+00 | 20260507 05:53:21.193 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.70e+00, 3.86e-01 / 1.67e-01 | 20260507 05:53:21.193 | INFO | PerfReport: 2.98%, Metrics -- 9.14e-01 / 1.21e+00, 2.45e-01 / 1.15e-01 | 20260507 05:53:21.193 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.07e-01 / 3.59e-01, 5.10e-02 / 1.66e-02 | 20260507 05:53:21.193 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:53:21.193 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:53:21.193 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:53:21.193 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 05:53:21.193 | INFO | PerfReport: 21.76%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260507 05:53:21.193 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 05:53:21.193 | INFO | PerfReport: 31.97%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 05:53:21.193 | INFO | PerfReport: 1.44%, LockUpload -- 1.38e+00 / 0.00e+00, 4.78e+00 / 0.00e+00 | 20260507 05:53:21.193 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260507 05:53:21.193 | INFO | PerfReport: 22.09%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 05:53:21.193 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.38e-02 / 0.00e+00 | 20260507 05:53:21.289 | INFO | Starting a round of Baz IO | 20260507 05:53:24.388 | INFO | Populating RTMetrics report | 20260507 05:53:24.388 | INFO | RTMetrics report populated | 20260507 05:53:24.388 | INFO | Opening tmp RTMetrics json file | 20260507 05:53:24.390 | INFO | Serializing RTMetrics to json | 20260507 05:53:24.400 | INFO | Serialization done, swapping tmp file now... | 20260507 05:53:24.409 | INFO | RTMetrics done | 20260507 05:53:24.473 | INFO | Finished a round of Baz IO: Wrote 1.683 GiB in 3.184 seconds | 20260507 05:53:24.473 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.67% / 0.00% / 67.33% | 20260507 05:53:24.474 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.133554% / 0% / 8.34465e-05% | 20260507 05:53:24.474 | INFO | Baz IO overall: | 20260507 05:53:24.474 | INFO | BytesWritten : 3847134974291 | 20260507 05:53:24.474 | INFO | NumEvents : 1844573004963 | 20260507 05:53:26.331 | INFO | Analyzing chunk frames = [7602176,7602688) | 20260507 05:53:26.331 | INFO | MemInfo: { | 20260507 05:53:26.331 | INFO | "gpu" : " 17 GB", | 20260507 05:53:26.331 | INFO | "huge" : "", | 20260507 05:53:26.331 | INFO | "huge_shared" : " 27 GB", | 20260507 05:53:26.331 | INFO | "malloc" : " 14 GB", | 20260507 05:53:26.331 | INFO | "pinned" : " 29 GB" | 20260507 05:53:26.331 | INFO | } | 20260507 05:53:26.331 | INFO | Perf report for the last 5.14001 seconds: | 20260507 05:53:26.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:53:26.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:26.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:26.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:26.332 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 05:53:26.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:26.332 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 05:53:26.332 | INFO | PRE_HQ : 58.9%, 1.8, 54.7ms, 0 | 20260507 05:53:26.332 | INFO | BAZWRITER : 61.5%, 3.1, 99.4ms, 0 | 20260507 05:53:26.332 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 05:53:26.332 | INFO | Uploaded 36GiB to GPU in 2.04939s (17.5662GiB/s) | 20260507 05:53:26.332 | INFO | Downloaded 38GiB to GPU in 2.17936s (17.4363GiB/s) | 20260507 05:53:31.483 | INFO | Analyzing chunk frames = [7602688,7603200) | 20260507 05:53:31.483 | INFO | MemInfo: { | 20260507 05:53:31.483 | INFO | "gpu" : " 17 GB", | 20260507 05:53:31.483 | INFO | "huge" : "", | 20260507 05:53:31.483 | INFO | "huge_shared" : " 27 GB", | 20260507 05:53:31.483 | INFO | "malloc" : " 14 GB", | 20260507 05:53:31.483 | INFO | "pinned" : " 29 GB" | 20260507 05:53:31.483 | INFO | } | 20260507 05:53:31.483 | INFO | Perf report for the last 5.15122 seconds: | 20260507 05:53:31.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:53:31.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:31.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:31.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:31.483 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260507 05:53:31.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:31.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:31.483 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:53:31.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:31.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:31.483 | INFO | Uploaded 36GiB to GPU in 2.04689s (17.5877GiB/s) | 20260507 05:53:31.483 | INFO | Downloaded 34GiB to GPU in 1.93817s (17.5423GiB/s) | 20260507 05:53:36.617 | INFO | Analyzing chunk frames = [7603200,7603712) | 20260507 05:53:36.617 | INFO | MemInfo: { | 20260507 05:53:36.617 | INFO | "gpu" : " 17 GB", | 20260507 05:53:36.617 | INFO | "huge" : "", | 20260507 05:53:36.617 | INFO | "huge_shared" : " 27 GB", | 20260507 05:53:36.617 | INFO | "malloc" : " 14 GB", | 20260507 05:53:36.617 | INFO | "pinned" : " 29 GB" | 20260507 05:53:36.617 | INFO | } | 20260507 05:53:36.617 | INFO | Perf report for the last 5.13476 seconds: | 20260507 05:53:36.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:53:36.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:36.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:36.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:36.618 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 05:53:36.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:36.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:36.618 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:53:36.618 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:36.618 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:36.618 | INFO | Uploaded 36GiB to GPU in 2.05513s (17.5172GiB/s) | 20260507 05:53:36.618 | INFO | Downloaded 34GiB to GPU in 1.95024s (17.4338GiB/s) | 20260507 05:53:41.759 | INFO | Analyzing chunk frames = [7603712,7604224) | 20260507 05:53:41.759 | INFO | MemInfo: { | 20260507 05:53:41.759 | INFO | "gpu" : " 17 GB", | 20260507 05:53:41.759 | INFO | "huge" : "", | 20260507 05:53:41.759 | INFO | "huge_shared" : " 27 GB", | 20260507 05:53:41.759 | INFO | "malloc" : " 14 GB", | 20260507 05:53:41.759 | INFO | "pinned" : " 29 GB" | 20260507 05:53:41.759 | INFO | } | 20260507 05:53:41.759 | INFO | Perf report for the last 5.14164 seconds: | 20260507 05:53:41.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:53:41.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:41.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:41.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:41.759 | INFO | ANALYSIS : 60.4%, 2.9, 93.4ms, 0 | 20260507 05:53:41.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:41.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:41.759 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:53:41.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:41.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:41.760 | INFO | Uploaded 36GiB to GPU in 2.0516s (17.5473GiB/s) | 20260507 05:53:41.760 | INFO | Downloaded 34GiB to GPU in 1.94668s (17.4656GiB/s) | 20260507 05:53:46.894 | INFO | Analyzing chunk frames = [7604224,7604736) | 20260507 05:53:46.894 | INFO | MemInfo: { | 20260507 05:53:46.894 | INFO | "gpu" : " 17 GB", | 20260507 05:53:46.894 | INFO | "huge" : "", | 20260507 05:53:46.894 | INFO | "huge_shared" : " 27 GB", | 20260507 05:53:46.894 | INFO | "malloc" : " 14 GB", | 20260507 05:53:46.894 | INFO | "pinned" : " 29 GB" | 20260507 05:53:46.894 | INFO | } | 20260507 05:53:46.894 | INFO | Perf report for the last 5.13504 seconds: | 20260507 05:53:46.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:53:46.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:46.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:46.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:46.894 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 05:53:46.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:46.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:46.894 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 05:53:46.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:46.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:46.894 | INFO | Uploaded 36GiB to GPU in 2.04696s (17.5871GiB/s) | 20260507 05:53:46.894 | INFO | Downloaded 34GiB to GPU in 1.90985s (17.8024GiB/s) | 20260507 05:53:52.034 | INFO | Analyzing chunk frames = [7604736,7605248) | 20260507 05:53:52.034 | INFO | MemInfo: { | 20260507 05:53:52.034 | INFO | "gpu" : " 17 GB", | 20260507 05:53:52.034 | INFO | "huge" : "", | 20260507 05:53:52.034 | INFO | "huge_shared" : " 27 GB", | 20260507 05:53:52.034 | INFO | "malloc" : " 14 GB", | 20260507 05:53:52.034 | INFO | "pinned" : " 29 GB" | 20260507 05:53:52.034 | INFO | } | 20260507 05:53:52.034 | INFO | Perf report for the last 5.14016 seconds: | 20260507 05:53:52.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:53:52.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:52.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:52.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:52.034 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 05:53:52.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:52.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:52.034 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 05:53:52.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:52.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:52.035 | INFO | Uploaded 36GiB to GPU in 2.04897s (17.5698GiB/s) | 20260507 05:53:52.035 | INFO | Downloaded 34GiB to GPU in 1.91112s (17.7907GiB/s) | 20260507 05:53:57.169 | INFO | Analyzing chunk frames = [7605248,7605760) | 20260507 05:53:57.169 | INFO | MemInfo: { | 20260507 05:53:57.169 | INFO | "gpu" : " 17 GB", | 20260507 05:53:57.169 | INFO | "huge" : "", | 20260507 05:53:57.169 | INFO | "huge_shared" : " 27 GB", | 20260507 05:53:57.170 | INFO | "malloc" : " 14 GB", | 20260507 05:53:57.170 | INFO | "pinned" : " 29 GB" | 20260507 05:53:57.170 | INFO | } | 20260507 05:53:57.170 | INFO | Perf report for the last 5.13549 seconds: | 20260507 05:53:57.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:53:57.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:57.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:57.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:57.170 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:53:57.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:57.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:57.170 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 05:53:57.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:57.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:53:57.170 | INFO | Uploaded 36GiB to GPU in 1.98086s (18.1739GiB/s) | 20260507 05:53:57.170 | INFO | Downloaded 34GiB to GPU in 1.98434s (17.1342GiB/s) | 20260507 05:54:02.315 | INFO | Analyzing chunk frames = [7605760,7606272) | 20260507 05:54:02.315 | INFO | MemInfo: { | 20260507 05:54:02.315 | INFO | "gpu" : " 17 GB", | 20260507 05:54:02.315 | INFO | "huge" : "", | 20260507 05:54:02.315 | INFO | "huge_shared" : " 27 GB", | 20260507 05:54:02.315 | INFO | "malloc" : " 14 GB", | 20260507 05:54:02.315 | INFO | "pinned" : " 29 GB" | 20260507 05:54:02.315 | INFO | } | 20260507 05:54:02.315 | INFO | Perf report for the last 5.14485 seconds: | 20260507 05:54:02.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:54:02.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:02.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:02.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:02.315 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:54:02.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:02.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:02.315 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 05:54:02.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:02.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:02.315 | INFO | Uploaded 36GiB to GPU in 1.97607s (18.218GiB/s) | 20260507 05:54:02.316 | INFO | Downloaded 34GiB to GPU in 1.9778s (17.1908GiB/s) | 20260507 05:54:02.412 | INFO | Starting a round of Baz IO | 20260507 05:54:05.513 | INFO | Populating RTMetrics report | 20260507 05:54:05.513 | INFO | RTMetrics report populated | 20260507 05:54:05.513 | INFO | Opening tmp RTMetrics json file | 20260507 05:54:05.518 | INFO | Serializing RTMetrics to json | 20260507 05:54:05.530 | INFO | Serialization done, swapping tmp file now... | 20260507 05:54:05.531 | INFO | RTMetrics done | 20260507 05:54:05.601 | INFO | Finished a round of Baz IO: Wrote 1.681 GiB in 3.188 seconds | 20260507 05:54:05.601 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.67% / 0.00% / 67.33% | 20260507 05:54:05.601 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.133483% / 3.97364e-06% / 9.13938e-05% | 20260507 05:54:05.601 | INFO | Baz IO overall: | 20260507 05:54:05.601 | INFO | BytesWritten : 3848940129522 | 20260507 05:54:05.601 | INFO | NumEvents : 1845419004244 | 20260507 05:54:07.450 | INFO | Analyzing chunk frames = [7606272,7606784) | 20260507 05:54:07.450 | INFO | MemInfo: { | 20260507 05:54:07.450 | INFO | "gpu" : " 17 GB", | 20260507 05:54:07.450 | INFO | "huge" : "", | 20260507 05:54:07.450 | INFO | "huge_shared" : " 27 GB", | 20260507 05:54:07.450 | INFO | "malloc" : " 14 GB", | 20260507 05:54:07.450 | INFO | "pinned" : " 29 GB" | 20260507 05:54:07.450 | INFO | } | 20260507 05:54:07.450 | INFO | Perf report for the last 5.13531 seconds: | 20260507 05:54:07.450 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:54:07.450 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:07.450 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:07.450 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:07.450 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260507 05:54:07.450 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:07.450 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260507 05:54:07.450 | INFO | PRE_HQ : 59.8%, 2.0, 60.7ms, 0 | 20260507 05:54:07.450 | INFO | BAZWRITER : 61.7%, 3.3, 105.9ms, 0 | 20260507 05:54:07.450 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:54:07.450 | INFO | Uploaded 36GiB to GPU in 2.0592s (17.4825GiB/s) | 20260507 05:54:07.450 | INFO | Downloaded 38GiB to GPU in 2.17625s (17.4612GiB/s) | 20260507 05:54:12.589 | INFO | Analyzing chunk frames = [7606784,7607296) | 20260507 05:54:12.589 | INFO | MemInfo: { | 20260507 05:54:12.589 | INFO | "gpu" : " 17 GB", | 20260507 05:54:12.589 | INFO | "huge" : "", | 20260507 05:54:12.589 | INFO | "huge_shared" : " 27 GB", | 20260507 05:54:12.589 | INFO | "malloc" : " 14 GB", | 20260507 05:54:12.589 | INFO | "pinned" : " 29 GB" | 20260507 05:54:12.589 | INFO | } | 20260507 05:54:12.589 | INFO | Perf report for the last 5.13924 seconds: | 20260507 05:54:12.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:54:12.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:12.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:12.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:12.589 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:54:12.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:12.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:12.589 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 05:54:12.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:12.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:12.590 | INFO | Uploaded 36GiB to GPU in 1.99374s (18.0566GiB/s) | 20260507 05:54:12.590 | INFO | Downloaded 34GiB to GPU in 1.9306s (17.6111GiB/s) | 20260507 05:54:17.728 | INFO | Analyzing chunk frames = [7607296,7607808) | 20260507 05:54:17.728 | INFO | MemInfo: { | 20260507 05:54:17.728 | INFO | "gpu" : " 17 GB", | 20260507 05:54:17.728 | INFO | "huge" : "", | 20260507 05:54:17.728 | INFO | "huge_shared" : " 27 GB", | 20260507 05:54:17.728 | INFO | "malloc" : " 14 GB", | 20260507 05:54:17.728 | INFO | "pinned" : " 29 GB" | 20260507 05:54:17.728 | INFO | } | 20260507 05:54:17.728 | INFO | Perf report for the last 5.13902 seconds: | 20260507 05:54:17.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:54:17.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:17.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:17.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:17.729 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 05:54:17.729 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:17.729 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:17.729 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:54:17.729 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:17.729 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:17.729 | INFO | Uploaded 36GiB to GPU in 2.01297s (17.8841GiB/s) | 20260507 05:54:17.729 | INFO | Downloaded 34GiB to GPU in 1.94471s (17.4833GiB/s) | 20260507 05:54:22.873 | INFO | Analyzing chunk frames = [7607808,7608320) | 20260507 05:54:22.873 | INFO | MemInfo: { | 20260507 05:54:22.873 | INFO | "gpu" : " 17 GB", | 20260507 05:54:22.873 | INFO | "huge" : "", | 20260507 05:54:22.873 | INFO | "huge_shared" : " 27 GB", | 20260507 05:54:22.873 | INFO | "malloc" : " 14 GB", | 20260507 05:54:22.873 | INFO | "pinned" : " 29 GB" | 20260507 05:54:22.873 | INFO | } | 20260507 05:54:22.873 | INFO | Perf report for the last 5.14452 seconds: | 20260507 05:54:22.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:54:22.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:22.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:22.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:22.873 | INFO | ANALYSIS : 59.5%, 3.0, 92.5ms, 0 | 20260507 05:54:22.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:22.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:22.873 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:54:22.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:22.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:22.874 | INFO | Uploaded 36GiB to GPU in 2.00571s (17.9488GiB/s) | 20260507 05:54:22.874 | INFO | Downloaded 34GiB to GPU in 1.93504s (17.5707GiB/s) | 20260507 05:54:28.005 | INFO | Analyzing chunk frames = [7608320,7608832) | 20260507 05:54:28.005 | INFO | MemInfo: { | 20260507 05:54:28.005 | INFO | "gpu" : " 17 GB", | 20260507 05:54:28.005 | INFO | "huge" : "", | 20260507 05:54:28.005 | INFO | "huge_shared" : " 27 GB", | 20260507 05:54:28.005 | INFO | "malloc" : " 14 GB", | 20260507 05:54:28.005 | INFO | "pinned" : " 29 GB" | 20260507 05:54:28.005 | INFO | } | 20260507 05:54:28.005 | INFO | Perf report for the last 5.13252 seconds: | 20260507 05:54:28.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:54:28.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:28.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:28.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:28.006 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 05:54:28.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:28.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:28.006 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 05:54:28.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:28.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:28.006 | INFO | Uploaded 36GiB to GPU in 2.04018s (17.6455GiB/s) | 20260507 05:54:28.006 | INFO | Downloaded 34GiB to GPU in 1.91145s (17.7875GiB/s) | 20260507 05:54:33.143 | INFO | Analyzing chunk frames = [7608832,7609344) | 20260507 05:54:33.143 | INFO | MemInfo: { | 20260507 05:54:33.143 | INFO | "gpu" : " 17 GB", | 20260507 05:54:33.143 | INFO | "huge" : "", | 20260507 05:54:33.143 | INFO | "huge_shared" : " 27 GB", | 20260507 05:54:33.143 | INFO | "malloc" : " 14 GB", | 20260507 05:54:33.143 | INFO | "pinned" : " 29 GB" | 20260507 05:54:33.143 | INFO | } | 20260507 05:54:33.143 | INFO | Perf report for the last 5.13768 seconds: | 20260507 05:54:33.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:54:33.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:33.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:33.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:33.143 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 05:54:33.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:33.143 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:33.143 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:54:33.143 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:33.143 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:33.143 | INFO | Uploaded 36GiB to GPU in 2.02633s (17.7661GiB/s) | 20260507 05:54:33.143 | INFO | Downloaded 34GiB to GPU in 1.91926s (17.7152GiB/s) | 20260507 05:54:38.289 | INFO | Analyzing chunk frames = [7609344,7609856) | 20260507 05:54:38.289 | INFO | MemInfo: { | 20260507 05:54:38.289 | INFO | "gpu" : " 17 GB", | 20260507 05:54:38.289 | INFO | "huge" : "", | 20260507 05:54:38.289 | INFO | "huge_shared" : " 27 GB", | 20260507 05:54:38.289 | INFO | "malloc" : " 14 GB", | 20260507 05:54:38.289 | INFO | "pinned" : " 29 GB" | 20260507 05:54:38.289 | INFO | } | 20260507 05:54:38.289 | INFO | Perf report for the last 5.14639 seconds: | 20260507 05:54:38.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:54:38.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:38.290 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:38.290 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:38.290 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 05:54:38.290 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:38.290 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:38.290 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:54:38.290 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:38.290 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:38.290 | INFO | Uploaded 36GiB to GPU in 1.97729s (18.2068GiB/s) | 20260507 05:54:38.290 | INFO | Downloaded 34GiB to GPU in 1.97513s (17.214GiB/s) | 20260507 05:54:43.436 | INFO | Analyzing chunk frames = [7609856,7610368) | 20260507 05:54:43.436 | INFO | MemInfo: { | 20260507 05:54:43.436 | INFO | "gpu" : " 17 GB", | 20260507 05:54:43.436 | INFO | "huge" : "", | 20260507 05:54:43.436 | INFO | "huge_shared" : " 27 GB", | 20260507 05:54:43.436 | INFO | "malloc" : " 14 GB", | 20260507 05:54:43.436 | INFO | "pinned" : " 29 GB" | 20260507 05:54:43.436 | INFO | } | 20260507 05:54:43.436 | INFO | Perf report for the last 5.14677 seconds: | 20260507 05:54:43.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:54:43.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:43.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:43.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:43.436 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:54:43.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:43.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:43.436 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 05:54:43.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:43.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:43.437 | INFO | Uploaded 36GiB to GPU in 1.97965s (18.185GiB/s) | 20260507 05:54:43.437 | INFO | Downloaded 34GiB to GPU in 1.96068s (17.3409GiB/s) | 20260507 05:54:43.437 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:54:43.437 | INFO | Total mem usage: 18 GiB | 20260507 05:54:43.437 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:54:43.437 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.7%) | 20260507 05:54:43.437 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:54:43.437 | INFO | High Water Breakdown: | 20260507 05:54:43.437 | INFO | 60.2% : 8.63 GiB ( 75.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:54:43.437 | INFO | 39.6% : 5.68 GiB ( 84.9% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:54:43.437 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:54:43.437 | INFO | ------------------------------------------ | 20260507 05:54:43.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:54:43.437 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:54:43.437 | INFO | Total mem usage: 26.5 GiB | 20260507 05:54:43.437 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:54:43.437 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:54:43.437 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:54:43.437 | INFO | High Water Breakdown: | 20260507 05:54:43.437 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:54:43.437 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:54:43.437 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:54:43.437 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:54:43.437 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:54:43.437 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:54:43.437 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:54:43.437 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:54:43.437 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:54:43.437 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:54:43.437 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:54:43.437 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:54:43.437 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:54:43.437 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:54:43.437 | INFO | ------------------------------------------ | 20260507 05:54:43.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:54:43.437 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:54:43.437 | INFO | Total mem usage: 25.6 GiB | 20260507 05:54:43.437 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:54:43.437 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:54:43.437 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:54:43.437 | INFO | High Water Breakdown: | 20260507 05:54:43.437 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:54:43.437 | INFO | ------------------------------------------ | 20260507 05:54:43.437 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:54:43.437 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:54:43.437 | INFO | Total mem usage: 15.7 GiB | 20260507 05:54:43.437 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:54:43.437 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:54:43.437 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:54:43.437 | INFO | High Water Breakdown: | 20260507 05:54:43.437 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:54:43.437 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:54:43.437 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:54:43.437 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:54:43.437 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:54:43.437 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:54:43.437 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:54:43.437 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:54:43.438 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:54:43.438 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:54:43.438 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:54:43.438 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:54:43.438 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:54:43.438 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:54:43.438 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:54:43.438 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:54:43.438 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:54:43.438 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:54:43.438 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:54:43.438 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:54:43.438 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:54:43.438 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:54:43.438 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:54:43.438 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:54:43.438 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:54:43.438 | INFO | ------------------------------------------ | 20260507 05:54:43.438 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:54:43.438 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:54:43.438 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:54:43.438 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:54:43.438 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.43e+01, 1.32e+01 / 9.01e-01 | 20260507 05:54:43.438 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.00e-01, 9.52e-03 / 5.98e-03 | 20260507 05:54:43.438 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.44e+00, 5.05e-01 / 2.92e-01 | 20260507 05:54:43.438 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.47e+00, 1.65e-01 / 1.17e-01 | 20260507 05:54:43.438 | INFO | PerfReport: 16.17%, DME -- 3.97e+01 / 4.41e+01, 3.67e+00 / 1.25e+00 | 20260507 05:54:43.438 | INFO | PerfReport: 57.75%, FrameLabeling -- 1.77e+01 / 1.99e+01, 1.30e+00 / 1.45e+00 | 20260507 05:54:43.438 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.12e+00 / 2.68e+00, 3.80e-01 / 2.20e-01 | 20260507 05:54:43.438 | INFO | PerfReport: 2.97%, Metrics -- 9.12e-01 / 1.21e+00, 2.41e-01 / 1.10e-01 | 20260507 05:54:43.438 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.07e-01 / 3.58e-01, 5.07e-02 / 1.79e-02 | 20260507 05:54:43.438 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:54:43.438 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:54:43.438 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:54:43.438 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 05:54:43.438 | INFO | PerfReport: 21.91%, Upload -- 2.11e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260507 05:54:43.438 | INFO | PerfReport: 21.36%, Download -- 2.05e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260507 05:54:43.438 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 05:54:43.438 | INFO | PerfReport: 1.56%, LockUpload -- 1.50e+00 / 0.00e+00, 4.87e+00 / 0.00e+00 | 20260507 05:54:43.438 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260507 05:54:43.438 | INFO | PerfReport: 21.94%, LockCompute -- 2.11e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 05:54:43.438 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.10e-02 / 0.00e+00 | 20260507 05:54:43.534 | INFO | Starting a round of Baz IO | 20260507 05:54:46.629 | INFO | Populating RTMetrics report | 20260507 05:54:46.629 | INFO | RTMetrics report populated | 20260507 05:54:46.629 | INFO | Opening tmp RTMetrics json file | 20260507 05:54:46.645 | INFO | Serializing RTMetrics to json | 20260507 05:54:46.647 | INFO | Serialization done, swapping tmp file now... | 20260507 05:54:46.647 | INFO | RTMetrics done | 20260507 05:54:46.720 | INFO | Finished a round of Baz IO: Wrote 1.684 GiB in 3.185 seconds | 20260507 05:54:46.720 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.67% / 0.00% / 67.33% | 20260507 05:54:46.720 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.134134% / 0% / 9.13938e-05% | 20260507 05:54:46.720 | INFO | Baz IO overall: | 20260507 05:54:46.720 | INFO | BytesWritten : 3850748080323 | 20260507 05:54:46.720 | INFO | NumEvents : 1846267029156 | 20260507 05:54:48.573 | INFO | Analyzing chunk frames = [7610368,7610880) | 20260507 05:54:48.573 | INFO | MemInfo: { | 20260507 05:54:48.573 | INFO | "gpu" : " 17 GB", | 20260507 05:54:48.573 | INFO | "huge" : "", | 20260507 05:54:48.573 | INFO | "huge_shared" : " 27 GB", | 20260507 05:54:48.573 | INFO | "malloc" : " 14 GB", | 20260507 05:54:48.573 | INFO | "pinned" : " 29 GB" | 20260507 05:54:48.573 | INFO | } | 20260507 05:54:48.573 | INFO | Perf report for the last 5.13708 seconds: | 20260507 05:54:48.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:54:48.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:48.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:48.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:48.574 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 05:54:48.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:48.574 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260507 05:54:48.574 | INFO | PRE_HQ : 59.8%, 1.9, 58.8ms, 0 | 20260507 05:54:48.574 | INFO | BAZWRITER : 61.6%, 3.3, 104.4ms, 0 | 20260507 05:54:48.574 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 05:54:48.574 | INFO | Uploaded 36GiB to GPU in 2.07098s (17.3831GiB/s) | 20260507 05:54:48.574 | INFO | Downloaded 38GiB to GPU in 2.17464s (17.4742GiB/s) | 20260507 05:54:53.703 | INFO | Analyzing chunk frames = [7610880,7611392) | 20260507 05:54:53.703 | INFO | MemInfo: { | 20260507 05:54:53.703 | INFO | "gpu" : " 17 GB", | 20260507 05:54:53.703 | INFO | "huge" : "", | 20260507 05:54:53.703 | INFO | "huge_shared" : " 27 GB", | 20260507 05:54:53.703 | INFO | "malloc" : " 14 GB", | 20260507 05:54:53.703 | INFO | "pinned" : " 29 GB" | 20260507 05:54:53.703 | INFO | } | 20260507 05:54:53.703 | INFO | Perf report for the last 5.13001 seconds: | 20260507 05:54:53.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:54:53.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:53.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:53.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:53.703 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 05:54:53.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:53.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:53.704 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 05:54:53.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:53.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:53.704 | INFO | Uploaded 36GiB to GPU in 1.9951s (18.0442GiB/s) | 20260507 05:54:53.704 | INFO | Downloaded 34GiB to GPU in 1.93617s (17.5605GiB/s) | 20260507 05:54:58.844 | INFO | Analyzing chunk frames = [7611392,7611904) | 20260507 05:54:58.844 | INFO | MemInfo: { | 20260507 05:54:58.844 | INFO | "gpu" : " 17 GB", | 20260507 05:54:58.844 | INFO | "huge" : "", | 20260507 05:54:58.844 | INFO | "huge_shared" : " 27 GB", | 20260507 05:54:58.844 | INFO | "malloc" : " 14 GB", | 20260507 05:54:58.844 | INFO | "pinned" : " 29 GB" | 20260507 05:54:58.844 | INFO | } | 20260507 05:54:58.844 | INFO | Perf report for the last 5.14107 seconds: | 20260507 05:54:58.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:54:58.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:58.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:58.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:58.845 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 05:54:58.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:58.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:58.845 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 05:54:58.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:58.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:54:58.845 | INFO | Uploaded 36GiB to GPU in 2.03398s (17.6993GiB/s) | 20260507 05:54:58.845 | INFO | Downloaded 34GiB to GPU in 1.95185s (17.4193GiB/s) | 20260507 05:55:03.987 | INFO | Analyzing chunk frames = [7611904,7612416) | 20260507 05:55:03.987 | INFO | MemInfo: { | 20260507 05:55:03.987 | INFO | "gpu" : " 17 GB", | 20260507 05:55:03.987 | INFO | "huge" : "", | 20260507 05:55:03.987 | INFO | "huge_shared" : " 27 GB", | 20260507 05:55:03.987 | INFO | "malloc" : " 14 GB", | 20260507 05:55:03.987 | INFO | "pinned" : " 29 GB" | 20260507 05:55:03.987 | INFO | } | 20260507 05:55:03.987 | INFO | Perf report for the last 5.14208 seconds: | 20260507 05:55:03.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:55:03.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:03.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:03.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:03.987 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 05:55:03.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:03.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:03.987 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:55:03.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:03.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:03.987 | INFO | Uploaded 36GiB to GPU in 2.00015s (17.9986GiB/s) | 20260507 05:55:03.987 | INFO | Downloaded 34GiB to GPU in 1.93193s (17.599GiB/s) | 20260507 05:55:09.127 | INFO | Analyzing chunk frames = [7612416,7612928) | 20260507 05:55:09.127 | INFO | MemInfo: { | 20260507 05:55:09.127 | INFO | "gpu" : " 17 GB", | 20260507 05:55:09.127 | INFO | "huge" : "", | 20260507 05:55:09.127 | INFO | "huge_shared" : " 27 GB", | 20260507 05:55:09.127 | INFO | "malloc" : " 14 GB", | 20260507 05:55:09.127 | INFO | "pinned" : " 29 GB" | 20260507 05:55:09.127 | INFO | } | 20260507 05:55:09.127 | INFO | Perf report for the last 5.14073 seconds: | 20260507 05:55:09.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:55:09.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:09.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:09.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:09.127 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 05:55:09.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:09.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:09.127 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:55:09.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:09.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:09.128 | INFO | Uploaded 36GiB to GPU in 2.05808s (17.492GiB/s) | 20260507 05:55:09.128 | INFO | Downloaded 34GiB to GPU in 1.9489s (17.4457GiB/s) | 20260507 05:55:14.266 | INFO | Analyzing chunk frames = [7612928,7613440) | 20260507 05:55:14.266 | INFO | MemInfo: { | 20260507 05:55:14.266 | INFO | "gpu" : " 17 GB", | 20260507 05:55:14.266 | INFO | "huge" : "", | 20260507 05:55:14.266 | INFO | "huge_shared" : " 27 GB", | 20260507 05:55:14.266 | INFO | "malloc" : " 14 GB", | 20260507 05:55:14.266 | INFO | "pinned" : " 29 GB" | 20260507 05:55:14.266 | INFO | } | 20260507 05:55:14.266 | INFO | Perf report for the last 5.13866 seconds: | 20260507 05:55:14.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:55:14.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:14.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:14.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:14.266 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 05:55:14.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:14.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:14.266 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:55:14.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:14.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:14.266 | INFO | Uploaded 36GiB to GPU in 2.06899s (17.3998GiB/s) | 20260507 05:55:14.266 | INFO | Downloaded 34GiB to GPU in 1.92193s (17.6906GiB/s) | 20260507 05:55:19.401 | INFO | Analyzing chunk frames = [7613440,7613952) | 20260507 05:55:19.401 | INFO | MemInfo: { | 20260507 05:55:19.401 | INFO | "gpu" : " 17 GB", | 20260507 05:55:19.401 | INFO | "huge" : "", | 20260507 05:55:19.401 | INFO | "huge_shared" : " 27 GB", | 20260507 05:55:19.401 | INFO | "malloc" : " 14 GB", | 20260507 05:55:19.401 | INFO | "pinned" : " 29 GB" | 20260507 05:55:19.401 | INFO | } | 20260507 05:55:19.401 | INFO | Perf report for the last 5.13538 seconds: | 20260507 05:55:19.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:55:19.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:19.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:19.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:19.401 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 05:55:19.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:19.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:19.401 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:55:19.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:19.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:19.402 | INFO | Uploaded 36GiB to GPU in 2.04668s (17.5895GiB/s) | 20260507 05:55:19.402 | INFO | Downloaded 34GiB to GPU in 1.99174s (17.0705GiB/s) | 20260507 05:55:24.540 | INFO | Analyzing chunk frames = [7613952,7614464) | 20260507 05:55:24.540 | INFO | MemInfo: { | 20260507 05:55:24.540 | INFO | "gpu" : " 17 GB", | 20260507 05:55:24.540 | INFO | "huge" : "", | 20260507 05:55:24.540 | INFO | "huge_shared" : " 27 GB", | 20260507 05:55:24.541 | INFO | "malloc" : " 14 GB", | 20260507 05:55:24.541 | INFO | "pinned" : " 29 GB" | 20260507 05:55:24.541 | INFO | } | 20260507 05:55:24.541 | INFO | Perf report for the last 5.13919 seconds: | 20260507 05:55:24.541 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:55:24.541 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:24.541 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:24.541 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:24.541 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:55:24.541 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:24.541 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:24.541 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:55:24.541 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:24.541 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:24.541 | INFO | Uploaded 36GiB to GPU in 1.96863s (18.2868GiB/s) | 20260507 05:55:24.541 | INFO | Downloaded 34GiB to GPU in 1.96011s (17.3459GiB/s) | 20260507 05:55:24.637 | INFO | Starting a round of Baz IO | 20260507 05:55:27.734 | INFO | Populating RTMetrics report | 20260507 05:55:27.735 | INFO | RTMetrics report populated | 20260507 05:55:27.735 | INFO | Opening tmp RTMetrics json file | 20260507 05:55:27.743 | INFO | Serializing RTMetrics to json | 20260507 05:55:27.751 | INFO | Serialization done, swapping tmp file now... | 20260507 05:55:27.751 | INFO | RTMetrics done | 20260507 05:55:27.823 | INFO | Finished a round of Baz IO: Wrote 1.681 GiB in 3.185 seconds | 20260507 05:55:27.823 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.67% / 0.00% / 67.33% | 20260507 05:55:27.823 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.133304% / 7.94729e-06% / 7.15256e-05% | 20260507 05:55:27.823 | INFO | Baz IO overall: | 20260507 05:55:27.823 | INFO | BytesWritten : 3852552997654 | 20260507 05:55:27.823 | INFO | NumEvents : 1847113020015 | 20260507 05:55:29.683 | INFO | Analyzing chunk frames = [7614464,7614976) | 20260507 05:55:29.683 | INFO | MemInfo: { | 20260507 05:55:29.683 | INFO | "gpu" : " 17 GB", | 20260507 05:55:29.683 | INFO | "huge" : "", | 20260507 05:55:29.683 | INFO | "huge_shared" : " 27 GB", | 20260507 05:55:29.683 | INFO | "malloc" : " 14 GB", | 20260507 05:55:29.683 | INFO | "pinned" : " 29 GB" | 20260507 05:55:29.683 | INFO | } | 20260507 05:55:29.683 | INFO | Perf report for the last 5.14284 seconds: | 20260507 05:55:29.683 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:55:29.683 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:29.683 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:29.683 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:29.683 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 05:55:29.683 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:29.683 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 05:55:29.683 | INFO | PRE_HQ : 58.8%, 1.8, 54.5ms, 0 | 20260507 05:55:29.683 | INFO | BAZWRITER : 61.5%, 3.2, 100.4ms, 0 | 20260507 05:55:29.684 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:55:29.684 | INFO | Uploaded 36GiB to GPU in 2.04054s (17.6424GiB/s) | 20260507 05:55:29.684 | INFO | Downloaded 38GiB to GPU in 2.1761s (17.4624GiB/s) | 20260507 05:55:34.818 | INFO | Analyzing chunk frames = [7614976,7615488) | 20260507 05:55:34.818 | INFO | MemInfo: { | 20260507 05:55:34.818 | INFO | "gpu" : " 17 GB", | 20260507 05:55:34.818 | INFO | "huge" : "", | 20260507 05:55:34.818 | INFO | "huge_shared" : " 27 GB", | 20260507 05:55:34.818 | INFO | "malloc" : " 14 GB", | 20260507 05:55:34.818 | INFO | "pinned" : " 29 GB" | 20260507 05:55:34.818 | INFO | } | 20260507 05:55:34.818 | INFO | Perf report for the last 5.13489 seconds: | 20260507 05:55:34.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:55:34.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:34.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:34.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:34.818 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:55:34.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:34.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:34.818 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:55:34.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:34.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:34.819 | INFO | Uploaded 36GiB to GPU in 1.98916s (18.0981GiB/s) | 20260507 05:55:34.819 | INFO | Downloaded 34GiB to GPU in 1.92703s (17.6437GiB/s) | 20260507 05:55:39.955 | INFO | Analyzing chunk frames = [7615488,7616000) | 20260507 05:55:39.955 | INFO | MemInfo: { | 20260507 05:55:39.955 | INFO | "gpu" : " 17 GB", | 20260507 05:55:39.955 | INFO | "huge" : "", | 20260507 05:55:39.955 | INFO | "huge_shared" : " 27 GB", | 20260507 05:55:39.955 | INFO | "malloc" : " 14 GB", | 20260507 05:55:39.955 | INFO | "pinned" : " 29 GB" | 20260507 05:55:39.955 | INFO | } | 20260507 05:55:39.955 | INFO | Perf report for the last 5.13686 seconds: | 20260507 05:55:39.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:55:39.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:39.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:39.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:39.955 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 05:55:39.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:39.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:39.955 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:55:39.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:39.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:39.956 | INFO | Uploaded 36GiB to GPU in 2.05331s (17.5327GiB/s) | 20260507 05:55:39.956 | INFO | Downloaded 34GiB to GPU in 1.94896s (17.4452GiB/s) | 20260507 05:55:45.100 | INFO | Analyzing chunk frames = [7616000,7616512) | 20260507 05:55:45.100 | INFO | MemInfo: { | 20260507 05:55:45.100 | INFO | "gpu" : " 17 GB", | 20260507 05:55:45.100 | INFO | "huge" : "", | 20260507 05:55:45.100 | INFO | "huge_shared" : " 27 GB", | 20260507 05:55:45.100 | INFO | "malloc" : " 14 GB", | 20260507 05:55:45.100 | INFO | "pinned" : " 29 GB" | 20260507 05:55:45.100 | INFO | } | 20260507 05:55:45.100 | INFO | Perf report for the last 5.14527 seconds: | 20260507 05:55:45.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:55:45.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:45.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:45.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:45.101 | INFO | ANALYSIS : 59.4%, 3.0, 92.3ms, 0 | 20260507 05:55:45.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:45.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:45.101 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:55:45.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:45.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:45.101 | INFO | Uploaded 36GiB to GPU in 1.98635s (18.1237GiB/s) | 20260507 05:55:45.101 | INFO | Downloaded 34GiB to GPU in 1.91329s (17.7704GiB/s) | 20260507 05:55:50.242 | INFO | Analyzing chunk frames = [7616512,7617024) | 20260507 05:55:50.242 | INFO | MemInfo: { | 20260507 05:55:50.242 | INFO | "gpu" : " 17 GB", | 20260507 05:55:50.242 | INFO | "huge" : "", | 20260507 05:55:50.242 | INFO | "huge_shared" : " 27 GB", | 20260507 05:55:50.242 | INFO | "malloc" : " 14 GB", | 20260507 05:55:50.242 | INFO | "pinned" : " 29 GB" | 20260507 05:55:50.242 | INFO | } | 20260507 05:55:50.242 | INFO | Perf report for the last 5.14184 seconds: | 20260507 05:55:50.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:55:50.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:50.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:50.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:50.242 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 05:55:50.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:50.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:50.242 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:55:50.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:50.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:50.243 | INFO | Uploaded 36GiB to GPU in 2.0568s (17.5029GiB/s) | 20260507 05:55:50.243 | INFO | Downloaded 34GiB to GPU in 1.94635s (17.4686GiB/s) | 20260507 05:55:55.383 | INFO | Analyzing chunk frames = [7617024,7617536) | 20260507 05:55:55.383 | INFO | MemInfo: { | 20260507 05:55:55.383 | INFO | "gpu" : " 17 GB", | 20260507 05:55:55.383 | INFO | "huge" : "", | 20260507 05:55:55.384 | INFO | "huge_shared" : " 27 GB", | 20260507 05:55:55.384 | INFO | "malloc" : " 14 GB", | 20260507 05:55:55.384 | INFO | "pinned" : " 29 GB" | 20260507 05:55:55.384 | INFO | } | 20260507 05:55:55.384 | INFO | Perf report for the last 5.14122 seconds: | 20260507 05:55:55.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:55:55.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:55.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:55.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:55.384 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 05:55:55.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:55.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:55.384 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 05:55:55.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:55.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:55:55.384 | INFO | Uploaded 36GiB to GPU in 2.05714s (17.5GiB/s) | 20260507 05:55:55.384 | INFO | Downloaded 34GiB to GPU in 1.91754s (17.7311GiB/s) | 20260507 05:56:00.520 | INFO | Analyzing chunk frames = [7617536,7618048) | 20260507 05:56:00.520 | INFO | MemInfo: { | 20260507 05:56:00.520 | INFO | "gpu" : " 17 GB", | 20260507 05:56:00.521 | INFO | "huge" : "", | 20260507 05:56:00.521 | INFO | "huge_shared" : " 27 GB", | 20260507 05:56:00.521 | INFO | "malloc" : " 14 GB", | 20260507 05:56:00.521 | INFO | "pinned" : " 29 GB" | 20260507 05:56:00.521 | INFO | } | 20260507 05:56:00.521 | INFO | Perf report for the last 5.13702 seconds: | 20260507 05:56:00.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:56:00.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:00.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:00.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:00.521 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 05:56:00.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:00.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:00.521 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:56:00.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:00.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:00.521 | INFO | Uploaded 36GiB to GPU in 1.99597s (18.0364GiB/s) | 20260507 05:56:00.521 | INFO | Downloaded 34GiB to GPU in 1.98755s (17.1065GiB/s) | 20260507 05:56:05.654 | INFO | Analyzing chunk frames = [7618048,7618560) | 20260507 05:56:05.654 | INFO | MemInfo: { | 20260507 05:56:05.654 | INFO | "gpu" : " 17 GB", | 20260507 05:56:05.654 | INFO | "huge" : "", | 20260507 05:56:05.654 | INFO | "huge_shared" : " 27 GB", | 20260507 05:56:05.654 | INFO | "malloc" : " 14 GB", | 20260507 05:56:05.654 | INFO | "pinned" : " 29 GB" | 20260507 05:56:05.654 | INFO | } | 20260507 05:56:05.654 | INFO | Perf report for the last 5.13317 seconds: | 20260507 05:56:05.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:56:05.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:05.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:05.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:05.654 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:56:05.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:05.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:05.654 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 05:56:05.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:05.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:05.654 | INFO | Uploaded 36GiB to GPU in 1.99339s (18.0597GiB/s) | 20260507 05:56:05.654 | INFO | Downloaded 34GiB to GPU in 1.95841s (17.361GiB/s) | 20260507 05:56:05.654 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:56:05.654 | INFO | Total mem usage: 18 GiB | 20260507 05:56:05.654 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:56:05.654 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:56:05.654 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:56:05.654 | INFO | High Water Breakdown: | 20260507 05:56:05.654 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:56:05.654 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:56:05.654 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:56:05.654 | INFO | ------------------------------------------ | 20260507 05:56:05.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:56:05.654 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:56:05.654 | INFO | Total mem usage: 26.5 GiB | 20260507 05:56:05.654 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:56:05.654 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:56:05.654 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:56:05.654 | INFO | High Water Breakdown: | 20260507 05:56:05.654 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:56:05.654 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:56:05.654 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:56:05.654 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:56:05.654 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:56:05.654 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:56:05.654 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:56:05.654 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:56:05.654 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:56:05.654 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:56:05.654 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:56:05.654 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:56:05.654 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:56:05.654 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:56:05.654 | INFO | ------------------------------------------ | 20260507 05:56:05.654 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:56:05.654 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:56:05.654 | INFO | Total mem usage: 25.6 GiB | 20260507 05:56:05.654 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:56:05.654 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:56:05.654 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:56:05.654 | INFO | High Water Breakdown: | 20260507 05:56:05.654 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:56:05.655 | INFO | ------------------------------------------ | 20260507 05:56:05.655 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:56:05.655 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:56:05.655 | INFO | Total mem usage: 15.7 GiB | 20260507 05:56:05.655 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:56:05.655 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:56:05.655 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:56:05.655 | INFO | High Water Breakdown: | 20260507 05:56:05.655 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:56:05.655 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:56:05.655 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:56:05.655 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:56:05.655 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:56:05.655 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:56:05.655 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:56:05.655 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:56:05.655 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:56:05.655 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:56:05.655 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:56:05.655 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:56:05.655 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:56:05.655 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:56:05.655 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:56:05.655 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:56:05.655 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:56:05.655 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:56:05.655 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:56:05.655 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:56:05.655 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:56:05.655 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:56:05.655 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:56:05.655 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:56:05.655 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:56:05.655 | INFO | ------------------------------------------ | 20260507 05:56:05.655 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:56:05.655 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:56:05.655 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:56:05.655 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:56:05.655 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.34e+01 / 1.17e+00 | 20260507 05:56:05.655 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.03e-01, 9.77e-03 / 9.33e-03 | 20260507 05:56:05.655 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.45e+00, 5.03e-01 / 2.55e-01 | 20260507 05:56:05.655 | INFO | PerfReport: 3.23%, Histogram -- 9.90e-01 / 1.49e+00, 1.67e-01 / 1.14e-01 | 20260507 05:56:05.655 | INFO | PerfReport: 16.19%, DME -- 3.98e+01 / 4.43e+01, 3.78e+00 / 1.53e+00 | 20260507 05:56:05.655 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.77e+01 / 2.00e+01, 1.29e+00 / 1.47e+00 | 20260507 05:56:05.655 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.64e+00, 3.85e-01 / 2.40e-01 | 20260507 05:56:05.655 | INFO | PerfReport: 2.97%, Metrics -- 9.12e-01 / 1.25e+00, 2.46e-01 / 1.76e-01 | 20260507 05:56:05.655 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.07e-01 / 3.67e-01, 5.21e-02 / 2.86e-02 | 20260507 05:56:05.655 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:56:05.655 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:56:05.655 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:56:05.655 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 05:56:05.655 | INFO | PerfReport: 21.94%, Upload -- 2.11e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 05:56:05.655 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260507 05:56:05.655 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 05:56:05.655 | INFO | PerfReport: 1.57%, LockUpload -- 1.51e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260507 05:56:05.655 | INFO | PerfReport: 1.16%, LockDownload -- 1.12e+00 / 0.00e+00, 2.75e+00 / 0.00e+00 | 20260507 05:56:05.655 | INFO | PerfReport: 22.00%, LockCompute -- 2.12e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 05:56:05.655 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260507 05:56:05.750 | INFO | Starting a round of Baz IO | 20260507 05:56:08.854 | INFO | Populating RTMetrics report | 20260507 05:56:08.854 | INFO | RTMetrics report populated | 20260507 05:56:08.854 | INFO | Opening tmp RTMetrics json file | 20260507 05:56:08.870 | INFO | Serializing RTMetrics to json | 20260507 05:56:08.872 | INFO | Serialization done, swapping tmp file now... | 20260507 05:56:08.872 | INFO | RTMetrics done | 20260507 05:56:08.941 | INFO | Finished a round of Baz IO: Wrote 1.68 GiB in 3.19 seconds | 20260507 05:56:08.941 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.67% / 0.00% / 67.33% | 20260507 05:56:08.941 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.133014% / 3.97364e-06% / 6.75519e-05% | 20260507 05:56:08.941 | INFO | Baz IO overall: | 20260507 05:56:08.941 | INFO | BytesWritten : 3854356869714 | 20260507 05:56:08.941 | INFO | NumEvents : 1847958537371 | 20260507 05:56:10.795 | INFO | Analyzing chunk frames = [7618560,7619072) | 20260507 05:56:10.795 | INFO | MemInfo: { | 20260507 05:56:10.795 | INFO | "gpu" : " 17 GB", | 20260507 05:56:10.795 | INFO | "huge" : "", | 20260507 05:56:10.795 | INFO | "huge_shared" : " 27 GB", | 20260507 05:56:10.795 | INFO | "malloc" : " 14 GB", | 20260507 05:56:10.795 | INFO | "pinned" : " 29 GB" | 20260507 05:56:10.795 | INFO | } | 20260507 05:56:10.795 | INFO | Perf report for the last 5.14134 seconds: | 20260507 05:56:10.795 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:56:10.795 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:10.795 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:10.795 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:10.795 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 05:56:10.795 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:10.795 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 05:56:10.795 | INFO | PRE_HQ : 59.1%, 1.8, 54.7ms, 0 | 20260507 05:56:10.795 | INFO | BAZWRITER : 61.6%, 3.2, 99.9ms, 0 | 20260507 05:56:10.795 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 05:56:10.795 | INFO | Uploaded 36GiB to GPU in 2.03905s (17.6553GiB/s) | 20260507 05:56:10.795 | INFO | Downloaded 38GiB to GPU in 2.18402s (17.3991GiB/s) | 20260507 05:56:15.939 | INFO | Analyzing chunk frames = [7619072,7619584) | 20260507 05:56:15.939 | INFO | MemInfo: { | 20260507 05:56:15.939 | INFO | "gpu" : " 17 GB", | 20260507 05:56:15.939 | INFO | "huge" : "", | 20260507 05:56:15.939 | INFO | "huge_shared" : " 27 GB", | 20260507 05:56:15.939 | INFO | "malloc" : " 14 GB", | 20260507 05:56:15.939 | INFO | "pinned" : " 29 GB" | 20260507 05:56:15.939 | INFO | } | 20260507 05:56:15.939 | INFO | Perf report for the last 5.14378 seconds: | 20260507 05:56:15.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:56:15.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:15.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:15.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:15.939 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 05:56:15.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:15.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:15.939 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:56:15.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:15.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:15.939 | INFO | Uploaded 36GiB to GPU in 2.00396s (17.9644GiB/s) | 20260507 05:56:15.939 | INFO | Downloaded 34GiB to GPU in 1.92824s (17.6326GiB/s) | 20260507 05:56:21.081 | INFO | Analyzing chunk frames = [7619584,7620096) | 20260507 05:56:21.081 | INFO | MemInfo: { | 20260507 05:56:21.081 | INFO | "gpu" : " 17 GB", | 20260507 05:56:21.081 | INFO | "huge" : "", | 20260507 05:56:21.081 | INFO | "huge_shared" : " 27 GB", | 20260507 05:56:21.081 | INFO | "malloc" : " 14 GB", | 20260507 05:56:21.081 | INFO | "pinned" : " 29 GB" | 20260507 05:56:21.081 | INFO | } | 20260507 05:56:21.081 | INFO | Perf report for the last 5.14231 seconds: | 20260507 05:56:21.081 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:56:21.081 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:21.081 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:21.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:21.081 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 05:56:21.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:21.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:21.081 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:56:21.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:21.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:21.082 | INFO | Uploaded 36GiB to GPU in 2.05489s (17.5192GiB/s) | 20260507 05:56:21.082 | INFO | Downloaded 34GiB to GPU in 1.95104s (17.4266GiB/s) | 20260507 05:56:26.215 | INFO | Analyzing chunk frames = [7620096,7620608) | 20260507 05:56:26.215 | INFO | MemInfo: { | 20260507 05:56:26.215 | INFO | "gpu" : " 17 GB", | 20260507 05:56:26.215 | INFO | "huge" : "", | 20260507 05:56:26.215 | INFO | "huge_shared" : " 27 GB", | 20260507 05:56:26.216 | INFO | "malloc" : " 14 GB", | 20260507 05:56:26.216 | INFO | "pinned" : " 29 GB" | 20260507 05:56:26.216 | INFO | } | 20260507 05:56:26.216 | INFO | Perf report for the last 5.13429 seconds: | 20260507 05:56:26.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:56:26.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:26.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:26.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:26.216 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 05:56:26.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:26.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:26.216 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 05:56:26.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:26.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:26.216 | INFO | Uploaded 36GiB to GPU in 2.04339s (17.6178GiB/s) | 20260507 05:56:26.216 | INFO | Downloaded 34GiB to GPU in 1.9514s (17.4234GiB/s) | 20260507 05:56:31.354 | INFO | Analyzing chunk frames = [7620608,7621120) | 20260507 05:56:31.354 | INFO | MemInfo: { | 20260507 05:56:31.354 | INFO | "gpu" : " 17 GB", | 20260507 05:56:31.354 | INFO | "huge" : "", | 20260507 05:56:31.354 | INFO | "huge_shared" : " 27 GB", | 20260507 05:56:31.354 | INFO | "malloc" : " 14 GB", | 20260507 05:56:31.354 | INFO | "pinned" : " 29 GB" | 20260507 05:56:31.354 | INFO | } | 20260507 05:56:31.354 | INFO | Perf report for the last 5.13869 seconds: | 20260507 05:56:31.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:56:31.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:31.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:31.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:31.354 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 05:56:31.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:31.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:31.354 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:56:31.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:31.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:31.354 | INFO | Uploaded 36GiB to GPU in 2.05896s (17.4845GiB/s) | 20260507 05:56:31.355 | INFO | Downloaded 34GiB to GPU in 1.95028s (17.4334GiB/s) | 20260507 05:56:36.495 | INFO | Analyzing chunk frames = [7621120,7621632) | 20260507 05:56:36.495 | INFO | MemInfo: { | 20260507 05:56:36.495 | INFO | "gpu" : " 17 GB", | 20260507 05:56:36.495 | INFO | "huge" : "", | 20260507 05:56:36.495 | INFO | "huge_shared" : " 27 GB", | 20260507 05:56:36.495 | INFO | "malloc" : " 14 GB", | 20260507 05:56:36.495 | INFO | "pinned" : " 29 GB" | 20260507 05:56:36.495 | INFO | } | 20260507 05:56:36.495 | INFO | Perf report for the last 5.14122 seconds: | 20260507 05:56:36.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:56:36.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:36.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:36.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:36.496 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 05:56:36.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:36.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:36.496 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:56:36.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:36.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:36.496 | INFO | Uploaded 36GiB to GPU in 2.03525s (17.6882GiB/s) | 20260507 05:56:36.496 | INFO | Downloaded 34GiB to GPU in 1.92352s (17.6759GiB/s) | 20260507 05:56:41.635 | INFO | Analyzing chunk frames = [7621632,7622144) | 20260507 05:56:41.635 | INFO | MemInfo: { | 20260507 05:56:41.635 | INFO | "gpu" : " 17 GB", | 20260507 05:56:41.635 | INFO | "huge" : "", | 20260507 05:56:41.635 | INFO | "huge_shared" : " 27 GB", | 20260507 05:56:41.635 | INFO | "malloc" : " 14 GB", | 20260507 05:56:41.635 | INFO | "pinned" : " 29 GB" | 20260507 05:56:41.635 | INFO | } | 20260507 05:56:41.635 | INFO | Perf report for the last 5.13955 seconds: | 20260507 05:56:41.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:56:41.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:41.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:41.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:41.635 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 05:56:41.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:41.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:41.635 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:56:41.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:41.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:41.635 | INFO | Uploaded 36GiB to GPU in 1.93s (18.6529GiB/s) | 20260507 05:56:41.635 | INFO | Downloaded 34GiB to GPU in 1.96765s (17.2795GiB/s) | 20260507 05:56:46.767 | INFO | Analyzing chunk frames = [7622144,7622656) | 20260507 05:56:46.767 | INFO | MemInfo: { | 20260507 05:56:46.767 | INFO | "gpu" : " 17 GB", | 20260507 05:56:46.767 | INFO | "huge" : "", | 20260507 05:56:46.767 | INFO | "huge_shared" : " 27 GB", | 20260507 05:56:46.767 | INFO | "malloc" : " 14 GB", | 20260507 05:56:46.767 | INFO | "pinned" : " 29 GB" | 20260507 05:56:46.767 | INFO | } | 20260507 05:56:46.767 | INFO | Perf report for the last 5.13177 seconds: | 20260507 05:56:46.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:56:46.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:46.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:46.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:46.767 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 05:56:46.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:46.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:46.767 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:56:46.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:46.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:46.767 | INFO | Uploaded 36GiB to GPU in 1.99153s (18.0765GiB/s) | 20260507 05:56:46.767 | INFO | Downloaded 34GiB to GPU in 1.9624s (17.3258GiB/s) | 20260507 05:56:46.864 | INFO | Starting a round of Baz IO | 20260507 05:56:49.960 | INFO | Populating RTMetrics report | 20260507 05:56:49.960 | INFO | RTMetrics report populated | 20260507 05:56:49.960 | INFO | Opening tmp RTMetrics json file | 20260507 05:56:49.968 | INFO | Serializing RTMetrics to json | 20260507 05:56:49.976 | INFO | Serialization done, swapping tmp file now... | 20260507 05:56:49.977 | INFO | RTMetrics done | 20260507 05:56:50.047 | INFO | Finished a round of Baz IO: Wrote 1.68 GiB in 3.182 seconds | 20260507 05:56:50.047 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.66% / 0.00% / 67.33% | 20260507 05:56:50.047 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.132831% / 0% / 7.15256e-05% | 20260507 05:56:50.047 | INFO | Baz IO overall: | 20260507 05:56:50.047 | INFO | BytesWritten : 3856160575053 | 20260507 05:56:50.047 | INFO | NumEvents : 1848804102523 | 20260507 05:56:51.907 | INFO | Analyzing chunk frames = [7622656,7623168) | 20260507 05:56:51.907 | INFO | MemInfo: { | 20260507 05:56:51.907 | INFO | "gpu" : " 17 GB", | 20260507 05:56:51.907 | INFO | "huge" : "", | 20260507 05:56:51.907 | INFO | "huge_shared" : " 27 GB", | 20260507 05:56:51.907 | INFO | "malloc" : " 14 GB", | 20260507 05:56:51.907 | INFO | "pinned" : " 29 GB" | 20260507 05:56:51.907 | INFO | } | 20260507 05:56:51.907 | INFO | Perf report for the last 5.14014 seconds: | 20260507 05:56:51.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:56:51.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:51.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:51.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:51.907 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 05:56:51.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:51.907 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 05:56:51.907 | INFO | PRE_HQ : 59.1%, 1.8, 54.7ms, 0 | 20260507 05:56:51.907 | INFO | BAZWRITER : 61.5%, 3.2, 100.0ms, 0 | 20260507 05:56:51.907 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 05:56:51.907 | INFO | Uploaded 36GiB to GPU in 2.04703s (17.5865GiB/s) | 20260507 05:56:51.907 | INFO | Downloaded 38GiB to GPU in 2.17303s (17.4871GiB/s) | 20260507 05:56:57.048 | INFO | Analyzing chunk frames = [7623168,7623680) | 20260507 05:56:57.048 | INFO | MemInfo: { | 20260507 05:56:57.048 | INFO | "gpu" : " 17 GB", | 20260507 05:56:57.048 | INFO | "huge" : "", | 20260507 05:56:57.048 | INFO | "huge_shared" : " 27 GB", | 20260507 05:56:57.048 | INFO | "malloc" : " 14 GB", | 20260507 05:56:57.048 | INFO | "pinned" : " 29 GB" | 20260507 05:56:57.048 | INFO | } | 20260507 05:56:57.048 | INFO | Perf report for the last 5.14077 seconds: | 20260507 05:56:57.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:56:57.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:57.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:57.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:57.048 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 05:56:57.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:57.048 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:57.048 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:56:57.048 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:57.048 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:56:57.048 | INFO | Uploaded 36GiB to GPU in 2.04413s (17.6114GiB/s) | 20260507 05:56:57.048 | INFO | Downloaded 34GiB to GPU in 1.94121s (17.5149GiB/s) | 20260507 05:57:02.200 | INFO | Analyzing chunk frames = [7623680,7624192) | 20260507 05:57:02.200 | INFO | MemInfo: { | 20260507 05:57:02.200 | INFO | "gpu" : " 17 GB", | 20260507 05:57:02.200 | INFO | "huge" : "", | 20260507 05:57:02.200 | INFO | "huge_shared" : " 27 GB", | 20260507 05:57:02.200 | INFO | "malloc" : " 14 GB", | 20260507 05:57:02.200 | INFO | "pinned" : " 29 GB" | 20260507 05:57:02.200 | INFO | } | 20260507 05:57:02.200 | INFO | Perf report for the last 5.15216 seconds: | 20260507 05:57:02.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:57:02.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:02.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:02.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:02.200 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 05:57:02.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:02.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:02.200 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 05:57:02.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:02.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:02.200 | INFO | Uploaded 36GiB to GPU in 2.01818s (17.8378GiB/s) | 20260507 05:57:02.200 | INFO | Downloaded 34GiB to GPU in 1.94073s (17.5192GiB/s) | 20260507 05:57:07.334 | INFO | Analyzing chunk frames = [7624192,7624704) | 20260507 05:57:07.334 | INFO | MemInfo: { | 20260507 05:57:07.334 | INFO | "gpu" : " 17 GB", | 20260507 05:57:07.334 | INFO | "huge" : "", | 20260507 05:57:07.334 | INFO | "huge_shared" : " 27 GB", | 20260507 05:57:07.334 | INFO | "malloc" : " 14 GB", | 20260507 05:57:07.334 | INFO | "pinned" : " 29 GB" | 20260507 05:57:07.334 | INFO | } | 20260507 05:57:07.334 | INFO | Perf report for the last 5.13414 seconds: | 20260507 05:57:07.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:57:07.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:07.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:07.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:07.334 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 05:57:07.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:07.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:07.334 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:57:07.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:07.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:07.334 | INFO | Uploaded 36GiB to GPU in 2.05976s (17.4778GiB/s) | 20260507 05:57:07.334 | INFO | Downloaded 34GiB to GPU in 1.94926s (17.4425GiB/s) | 20260507 05:57:12.467 | INFO | Analyzing chunk frames = [7624704,7625216) | 20260507 05:57:12.467 | INFO | MemInfo: { | 20260507 05:57:12.467 | INFO | "gpu" : " 17 GB", | 20260507 05:57:12.467 | INFO | "huge" : "", | 20260507 05:57:12.467 | INFO | "huge_shared" : " 27 GB", | 20260507 05:57:12.467 | INFO | "malloc" : " 14 GB", | 20260507 05:57:12.467 | INFO | "pinned" : " 29 GB" | 20260507 05:57:12.467 | INFO | } | 20260507 05:57:12.467 | INFO | Perf report for the last 5.13322 seconds: | 20260507 05:57:12.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:57:12.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:12.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:12.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:12.467 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 05:57:12.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:12.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:12.467 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:57:12.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:12.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:12.468 | INFO | Uploaded 36GiB to GPU in 2.05237s (17.5407GiB/s) | 20260507 05:57:12.468 | INFO | Downloaded 34GiB to GPU in 1.90581s (17.8401GiB/s) | 20260507 05:57:17.604 | INFO | Analyzing chunk frames = [7625216,7625728) | 20260507 05:57:17.604 | INFO | MemInfo: { | 20260507 05:57:17.604 | INFO | "gpu" : " 17 GB", | 20260507 05:57:17.604 | INFO | "huge" : "", | 20260507 05:57:17.604 | INFO | "huge_shared" : " 27 GB", | 20260507 05:57:17.604 | INFO | "malloc" : " 14 GB", | 20260507 05:57:17.604 | INFO | "pinned" : " 29 GB" | 20260507 05:57:17.604 | INFO | } | 20260507 05:57:17.604 | INFO | Perf report for the last 5.13708 seconds: | 20260507 05:57:17.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:57:17.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:17.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:17.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:17.604 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 05:57:17.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:17.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:17.605 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 05:57:17.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:17.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:17.605 | INFO | Uploaded 36GiB to GPU in 2.03833s (17.6616GiB/s) | 20260507 05:57:17.605 | INFO | Downloaded 34GiB to GPU in 1.90993s (17.8017GiB/s) | 20260507 05:57:22.747 | INFO | Analyzing chunk frames = [7625728,7626240) | 20260507 05:57:22.747 | INFO | MemInfo: { | 20260507 05:57:22.747 | INFO | "gpu" : " 17 GB", | 20260507 05:57:22.747 | INFO | "huge" : "", | 20260507 05:57:22.747 | INFO | "huge_shared" : " 27 GB", | 20260507 05:57:22.747 | INFO | "malloc" : " 14 GB", | 20260507 05:57:22.747 | INFO | "pinned" : " 29 GB" | 20260507 05:57:22.747 | INFO | } | 20260507 05:57:22.747 | INFO | Perf report for the last 5.1425 seconds: | 20260507 05:57:22.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:57:22.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:22.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:22.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:22.747 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 05:57:22.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:22.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:22.747 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 05:57:22.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:22.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:22.747 | INFO | Uploaded 36GiB to GPU in 1.97218s (18.254GiB/s) | 20260507 05:57:22.747 | INFO | Downloaded 34GiB to GPU in 1.9599s (17.3479GiB/s) | 20260507 05:57:27.889 | INFO | Analyzing chunk frames = [7626240,7626752) | 20260507 05:57:27.889 | INFO | MemInfo: { | 20260507 05:57:27.890 | INFO | "gpu" : " 17 GB", | 20260507 05:57:27.890 | INFO | "huge" : "", | 20260507 05:57:27.890 | INFO | "huge_shared" : " 27 GB", | 20260507 05:57:27.890 | INFO | "malloc" : " 14 GB", | 20260507 05:57:27.890 | INFO | "pinned" : " 29 GB" | 20260507 05:57:27.890 | INFO | } | 20260507 05:57:27.890 | INFO | Perf report for the last 5.14271 seconds: | 20260507 05:57:27.890 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:57:27.890 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:27.890 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:27.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:27.890 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:57:27.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:27.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:27.890 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 05:57:27.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:27.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:27.890 | INFO | Uploaded 36GiB to GPU in 1.99471s (18.0477GiB/s) | 20260507 05:57:27.890 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:57:27.890 | INFO | Downloaded 34GiB to GPU in 1.96958s (17.2626GiB/s) | 20260507 05:57:27.890 | INFO | Total mem usage: 18 GiB | 20260507 05:57:27.890 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:57:27.890 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 05:57:27.890 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:57:27.890 | INFO | High Water Breakdown: | 20260507 05:57:27.890 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:57:27.890 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:57:27.890 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:57:27.890 | INFO | ------------------------------------------ | 20260507 05:57:27.890 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:57:27.890 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:57:27.890 | INFO | Total mem usage: 26.5 GiB | 20260507 05:57:27.890 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:57:27.890 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:57:27.890 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 05:57:27.890 | INFO | High Water Breakdown: | 20260507 05:57:27.890 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:57:27.890 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:57:27.890 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:57:27.890 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:57:27.890 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:57:27.890 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:57:27.890 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:57:27.890 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:57:27.890 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:57:27.890 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:57:27.890 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:57:27.890 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:57:27.890 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:57:27.890 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:57:27.890 | INFO | ------------------------------------------ | 20260507 05:57:27.890 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:57:27.890 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:57:27.890 | INFO | Total mem usage: 25.6 GiB | 20260507 05:57:27.890 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:57:27.890 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:57:27.890 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:57:27.890 | INFO | High Water Breakdown: | 20260507 05:57:27.890 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:57:27.890 | INFO | ------------------------------------------ | 20260507 05:57:27.890 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:57:27.891 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:57:27.891 | INFO | Total mem usage: 15.7 GiB | 20260507 05:57:27.891 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:57:27.891 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:57:27.891 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:57:27.891 | INFO | High Water Breakdown: | 20260507 05:57:27.891 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:57:27.891 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:57:27.891 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:57:27.891 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:57:27.891 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:57:27.891 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:57:27.891 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:57:27.891 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:57:27.891 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:57:27.891 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:57:27.891 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:57:27.891 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:57:27.891 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:57:27.891 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:57:27.891 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:57:27.891 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:57:27.891 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:57:27.891 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:57:27.891 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:57:27.891 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:57:27.891 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:57:27.891 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:57:27.891 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:57:27.891 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:57:27.891 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:57:27.891 | INFO | ------------------------------------------ | 20260507 05:57:27.891 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:57:27.891 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:57:27.891 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:57:27.891 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:57:27.891 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.33e+01 / 1.31e+00 | 20260507 05:57:27.891 | INFO | PerfReport: 0.29%, AggregateStats -- 9.03e-02 / 1.02e-01, 9.30e-03 / 6.44e-03 | 20260507 05:57:27.891 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.44e+00, 4.97e-01 / 2.46e-01 | 20260507 05:57:27.891 | INFO | PerfReport: 3.21%, Histogram -- 9.84e-01 / 1.48e+00, 1.62e-01 / 9.63e-02 | 20260507 05:57:27.891 | INFO | PerfReport: 16.27%, DME -- 3.99e+01 / 4.47e+01, 3.86e+00 / 1.67e+00 | 20260507 05:57:27.891 | INFO | PerfReport: 57.67%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.24e+00 / 9.88e-01 | 20260507 05:57:27.891 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.73e+00, 3.89e-01 / 1.84e-01 | 20260507 05:57:27.891 | INFO | PerfReport: 2.97%, Metrics -- 9.11e-01 / 1.22e+00, 2.41e-01 / 1.05e-01 | 20260507 05:57:27.891 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.07e-01 / 3.66e-01, 5.13e-02 / 1.92e-02 | 20260507 05:57:27.891 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:57:27.891 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:57:27.891 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:57:27.891 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 05:57:27.891 | INFO | PerfReport: 21.90%, Upload -- 2.11e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 05:57:27.891 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260507 05:57:27.891 | INFO | PerfReport: 31.87%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 05:57:27.891 | INFO | PerfReport: 1.67%, LockUpload -- 1.61e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 05:57:27.891 | INFO | PerfReport: 1.20%, LockDownload -- 1.15e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260507 05:57:27.891 | INFO | PerfReport: 21.99%, LockCompute -- 2.12e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 05:57:27.891 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260507 05:57:27.986 | INFO | Starting a round of Baz IO | 20260507 05:57:31.093 | INFO | Populating RTMetrics report | 20260507 05:57:31.094 | INFO | RTMetrics report populated | 20260507 05:57:31.094 | INFO | Opening tmp RTMetrics json file | 20260507 05:57:31.095 | INFO | Serializing RTMetrics to json | 20260507 05:57:31.097 | INFO | Serialization done, swapping tmp file now... | 20260507 05:57:31.097 | INFO | RTMetrics done | 20260507 05:57:31.201 | INFO | Finished a round of Baz IO: Wrote 1.68 GiB in 3.214 seconds | 20260507 05:57:31.201 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.66% / 0.00% / 67.34% | 20260507 05:57:31.201 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.131802% / 3.97364e-06% / 7.94729e-05% | 20260507 05:57:31.201 | INFO | Baz IO overall: | 20260507 05:57:31.201 | INFO | BytesWritten : 3857964128035 | 20260507 05:57:31.201 | INFO | NumEvents : 1849649744798 | 20260507 05:57:33.023 | INFO | Analyzing chunk frames = [7626752,7627264) | 20260507 05:57:33.023 | INFO | MemInfo: { | 20260507 05:57:33.023 | INFO | "gpu" : " 17 GB", | 20260507 05:57:33.023 | INFO | "huge" : "", | 20260507 05:57:33.023 | INFO | "huge_shared" : " 27 GB", | 20260507 05:57:33.023 | INFO | "malloc" : " 14 GB", | 20260507 05:57:33.023 | INFO | "pinned" : " 29 GB" | 20260507 05:57:33.023 | INFO | } | 20260507 05:57:33.023 | INFO | Perf report for the last 5.13324 seconds: | 20260507 05:57:33.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:57:33.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:33.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:33.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:33.023 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 05:57:33.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:33.023 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 05:57:33.023 | INFO | PRE_HQ : 60.1%, 1.9, 58.7ms, 0 | 20260507 05:57:33.023 | INFO | BAZWRITER : 62.2%, 3.2, 103.4ms, 0 | 20260507 05:57:33.023 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 05:57:33.023 | INFO | Uploaded 36GiB to GPU in 2.07183s (17.376GiB/s) | 20260507 05:57:33.023 | INFO | Downloaded 38GiB to GPU in 2.17718s (17.4538GiB/s) | 20260507 05:57:38.165 | INFO | Analyzing chunk frames = [7627264,7627776) | 20260507 05:57:38.165 | INFO | MemInfo: { | 20260507 05:57:38.165 | INFO | "gpu" : " 17 GB", | 20260507 05:57:38.165 | INFO | "huge" : "", | 20260507 05:57:38.165 | INFO | "huge_shared" : " 27 GB", | 20260507 05:57:38.165 | INFO | "malloc" : " 14 GB", | 20260507 05:57:38.165 | INFO | "pinned" : " 29 GB" | 20260507 05:57:38.165 | INFO | } | 20260507 05:57:38.165 | INFO | Perf report for the last 5.14187 seconds: | 20260507 05:57:38.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:57:38.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:38.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:38.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:38.165 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 05:57:38.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:38.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:38.165 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:57:38.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:38.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:38.165 | INFO | Uploaded 36GiB to GPU in 2.00851s (17.9237GiB/s) | 20260507 05:57:38.165 | INFO | Downloaded 34GiB to GPU in 1.93321s (17.5873GiB/s) | 20260507 05:57:43.310 | INFO | Analyzing chunk frames = [7627776,7628288) | 20260507 05:57:43.310 | INFO | MemInfo: { | 20260507 05:57:43.310 | INFO | "gpu" : " 17 GB", | 20260507 05:57:43.310 | INFO | "huge" : "", | 20260507 05:57:43.310 | INFO | "huge_shared" : " 27 GB", | 20260507 05:57:43.310 | INFO | "malloc" : " 14 GB", | 20260507 05:57:43.310 | INFO | "pinned" : " 29 GB" | 20260507 05:57:43.310 | INFO | } | 20260507 05:57:43.311 | INFO | Perf report for the last 5.14571 seconds: | 20260507 05:57:43.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:57:43.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:43.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:43.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:43.311 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 05:57:43.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:43.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:43.311 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:57:43.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:43.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:43.311 | INFO | Uploaded 36GiB to GPU in 2.05319s (17.5337GiB/s) | 20260507 05:57:43.311 | INFO | Downloaded 34GiB to GPU in 1.95086s (17.4282GiB/s) | 20260507 05:57:48.446 | INFO | Analyzing chunk frames = [7628288,7628800) | 20260507 05:57:48.447 | INFO | MemInfo: { | 20260507 05:57:48.447 | INFO | "gpu" : " 17 GB", | 20260507 05:57:48.447 | INFO | "huge" : "", | 20260507 05:57:48.447 | INFO | "huge_shared" : " 27 GB", | 20260507 05:57:48.447 | INFO | "malloc" : " 14 GB", | 20260507 05:57:48.447 | INFO | "pinned" : " 29 GB" | 20260507 05:57:48.447 | INFO | } | 20260507 05:57:48.447 | INFO | Perf report for the last 5.13619 seconds: | 20260507 05:57:48.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:57:48.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:48.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:48.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:48.447 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 05:57:48.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:48.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:48.447 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:57:48.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:48.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:48.447 | INFO | Uploaded 36GiB to GPU in 2.00806s (17.9278GiB/s) | 20260507 05:57:48.447 | INFO | Downloaded 34GiB to GPU in 1.94286s (17.5GiB/s) | 20260507 05:57:53.580 | INFO | Analyzing chunk frames = [7628800,7629312) | 20260507 05:57:53.580 | INFO | MemInfo: { | 20260507 05:57:53.580 | INFO | "gpu" : " 17 GB", | 20260507 05:57:53.580 | INFO | "huge" : "", | 20260507 05:57:53.580 | INFO | "huge_shared" : " 27 GB", | 20260507 05:57:53.580 | INFO | "malloc" : " 14 GB", | 20260507 05:57:53.580 | INFO | "pinned" : " 29 GB" | 20260507 05:57:53.580 | INFO | } | 20260507 05:57:53.580 | INFO | Perf report for the last 5.13362 seconds: | 20260507 05:57:53.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:57:53.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:53.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:53.580 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:53.580 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 05:57:53.580 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:53.580 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:53.580 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:57:53.580 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:53.580 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:53.581 | INFO | Uploaded 36GiB to GPU in 2.03526s (17.6881GiB/s) | 20260507 05:57:53.581 | INFO | Downloaded 34GiB to GPU in 1.91185s (17.7838GiB/s) | 20260507 05:57:58.718 | INFO | Analyzing chunk frames = [7629312,7629824) | 20260507 05:57:58.718 | INFO | MemInfo: { | 20260507 05:57:58.718 | INFO | "gpu" : " 17 GB", | 20260507 05:57:58.718 | INFO | "huge" : "", | 20260507 05:57:58.718 | INFO | "huge_shared" : " 27 GB", | 20260507 05:57:58.719 | INFO | "malloc" : " 14 GB", | 20260507 05:57:58.719 | INFO | "pinned" : " 29 GB" | 20260507 05:57:58.719 | INFO | } | 20260507 05:57:58.719 | INFO | Perf report for the last 5.13821 seconds: | 20260507 05:57:58.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:57:58.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:58.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:58.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:58.719 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 05:57:58.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:58.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:58.719 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 05:57:58.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:58.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:57:58.719 | INFO | Uploaded 36GiB to GPU in 2.04535s (17.6009GiB/s) | 20260507 05:57:58.719 | INFO | Downloaded 34GiB to GPU in 1.91341s (17.7693GiB/s) | 20260507 05:58:03.861 | INFO | Analyzing chunk frames = [7629824,7630336) | 20260507 05:58:03.861 | INFO | MemInfo: { | 20260507 05:58:03.862 | INFO | "gpu" : " 17 GB", | 20260507 05:58:03.862 | INFO | "huge" : "", | 20260507 05:58:03.862 | INFO | "huge_shared" : " 27 GB", | 20260507 05:58:03.862 | INFO | "malloc" : " 14 GB", | 20260507 05:58:03.862 | INFO | "pinned" : " 29 GB" | 20260507 05:58:03.862 | INFO | } | 20260507 05:58:03.862 | INFO | Perf report for the last 5.14306 seconds: | 20260507 05:58:03.862 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:58:03.862 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:03.862 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:03.862 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:03.862 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 05:58:03.862 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:03.862 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:03.862 | INFO | PRE_HQ : 25.8%, 1.0, 13.5ms, 0 | 20260507 05:58:03.862 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:03.862 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:03.862 | INFO | Uploaded 36GiB to GPU in 1.97145s (18.2607GiB/s) | 20260507 05:58:03.862 | INFO | Downloaded 34GiB to GPU in 1.97783s (17.1905GiB/s) | 20260507 05:58:09.003 | INFO | Analyzing chunk frames = [7630336,7630848) | 20260507 05:58:09.003 | INFO | MemInfo: { | 20260507 05:58:09.003 | INFO | "gpu" : " 17 GB", | 20260507 05:58:09.003 | INFO | "huge" : "", | 20260507 05:58:09.003 | INFO | "huge_shared" : " 27 GB", | 20260507 05:58:09.003 | INFO | "malloc" : " 14 GB", | 20260507 05:58:09.003 | INFO | "pinned" : " 29 GB" | 20260507 05:58:09.003 | INFO | } | 20260507 05:58:09.003 | INFO | Perf report for the last 5.14128 seconds: | 20260507 05:58:09.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:58:09.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:09.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:09.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:09.003 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:58:09.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:09.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:09.003 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 05:58:09.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:09.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:09.003 | INFO | Uploaded 36GiB to GPU in 1.98388s (18.1462GiB/s) | 20260507 05:58:09.003 | INFO | Downloaded 34GiB to GPU in 1.95685s (17.3749GiB/s) | 20260507 05:58:09.100 | INFO | Starting a round of Baz IO | 20260507 05:58:12.201 | INFO | Populating RTMetrics report | 20260507 05:58:12.201 | INFO | RTMetrics report populated | 20260507 05:58:12.201 | INFO | Opening tmp RTMetrics json file | 20260507 05:58:12.207 | INFO | Serializing RTMetrics to json | 20260507 05:58:12.218 | INFO | Serialization done, swapping tmp file now... | 20260507 05:58:12.219 | INFO | RTMetrics done | 20260507 05:58:12.287 | INFO | Finished a round of Baz IO: Wrote 1.678 GiB in 3.187 seconds | 20260507 05:58:12.287 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.66% / 0.00% / 67.34% | 20260507 05:58:12.287 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.131027% / 3.97364e-06% / 7.94729e-05% | 20260507 05:58:12.287 | INFO | Baz IO overall: | 20260507 05:58:12.287 | INFO | BytesWritten : 3859765961531 | 20260507 05:58:12.287 | INFO | NumEvents : 1850494170821 | 20260507 05:58:14.143 | INFO | Analyzing chunk frames = [7630848,7631360) | 20260507 05:58:14.143 | INFO | MemInfo: { | 20260507 05:58:14.143 | INFO | "gpu" : " 17 GB", | 20260507 05:58:14.143 | INFO | "huge" : "", | 20260507 05:58:14.143 | INFO | "huge_shared" : " 27 GB", | 20260507 05:58:14.143 | INFO | "malloc" : " 14 GB", | 20260507 05:58:14.143 | INFO | "pinned" : " 29 GB" | 20260507 05:58:14.143 | INFO | } | 20260507 05:58:14.143 | INFO | Perf report for the last 5.13986 seconds: | 20260507 05:58:14.143 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:58:14.143 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:14.143 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:14.143 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:14.143 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 05:58:14.143 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:14.143 | INFO | RT_METRICS : 43.2%, 1.0, 22.7ms, 0 | 20260507 05:58:14.143 | INFO | PRE_HQ : 59.7%, 1.9, 57.6ms, 0 | 20260507 05:58:14.143 | INFO | BAZWRITER : 61.6%, 3.2, 102.9ms, 0 | 20260507 05:58:14.143 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:58:14.143 | INFO | Uploaded 36GiB to GPU in 2.06223s (17.4568GiB/s) | 20260507 05:58:14.143 | INFO | Downloaded 38GiB to GPU in 2.17755s (17.4508GiB/s) | 20260507 05:58:19.283 | INFO | Analyzing chunk frames = [7631360,7631872) | 20260507 05:58:19.283 | INFO | MemInfo: { | 20260507 05:58:19.283 | INFO | "gpu" : " 17 GB", | 20260507 05:58:19.283 | INFO | "huge" : "", | 20260507 05:58:19.283 | INFO | "huge_shared" : " 27 GB", | 20260507 05:58:19.283 | INFO | "malloc" : " 14 GB", | 20260507 05:58:19.283 | INFO | "pinned" : " 29 GB" | 20260507 05:58:19.283 | INFO | } | 20260507 05:58:19.283 | INFO | Perf report for the last 5.14039 seconds: | 20260507 05:58:19.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:58:19.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:19.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:19.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:19.283 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 05:58:19.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:19.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:19.283 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:58:19.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:19.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:19.283 | INFO | Uploaded 36GiB to GPU in 1.98925s (18.0973GiB/s) | 20260507 05:58:19.284 | INFO | Downloaded 34GiB to GPU in 1.93418s (17.5785GiB/s) | 20260507 05:58:24.415 | INFO | Analyzing chunk frames = [7631872,7632384) | 20260507 05:58:24.415 | INFO | MemInfo: { | 20260507 05:58:24.415 | INFO | "gpu" : " 17 GB", | 20260507 05:58:24.415 | INFO | "huge" : "", | 20260507 05:58:24.415 | INFO | "huge_shared" : " 27 GB", | 20260507 05:58:24.415 | INFO | "malloc" : " 14 GB", | 20260507 05:58:24.415 | INFO | "pinned" : " 29 GB" | 20260507 05:58:24.415 | INFO | } | 20260507 05:58:24.415 | INFO | Perf report for the last 5.13219 seconds: | 20260507 05:58:24.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:58:24.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:24.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:24.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:24.415 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 05:58:24.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:24.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:24.415 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 05:58:24.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:24.416 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:24.416 | INFO | Uploaded 36GiB to GPU in 2.02693s (17.7608GiB/s) | 20260507 05:58:24.416 | INFO | Downloaded 34GiB to GPU in 1.92182s (17.6915GiB/s) | 20260507 05:58:29.562 | INFO | Analyzing chunk frames = [7632384,7632896) | 20260507 05:58:29.562 | INFO | MemInfo: { | 20260507 05:58:29.562 | INFO | "gpu" : " 17 GB", | 20260507 05:58:29.562 | INFO | "huge" : "", | 20260507 05:58:29.562 | INFO | "huge_shared" : " 27 GB", | 20260507 05:58:29.562 | INFO | "malloc" : " 14 GB", | 20260507 05:58:29.562 | INFO | "pinned" : " 29 GB" | 20260507 05:58:29.562 | INFO | } | 20260507 05:58:29.562 | INFO | Perf report for the last 5.147 seconds: | 20260507 05:58:29.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:58:29.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:29.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:29.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:29.562 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 05:58:29.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:29.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:29.563 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 05:58:29.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:29.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:29.563 | INFO | Uploaded 36GiB to GPU in 2.00706s (17.9367GiB/s) | 20260507 05:58:29.563 | INFO | Downloaded 34GiB to GPU in 1.92647s (17.6489GiB/s) | 20260507 05:58:34.698 | INFO | Analyzing chunk frames = [7632896,7633408) | 20260507 05:58:34.698 | INFO | MemInfo: { | 20260507 05:58:34.698 | INFO | "gpu" : " 17 GB", | 20260507 05:58:34.698 | INFO | "huge" : "", | 20260507 05:58:34.698 | INFO | "huge_shared" : " 27 GB", | 20260507 05:58:34.698 | INFO | "malloc" : " 14 GB", | 20260507 05:58:34.698 | INFO | "pinned" : " 29 GB" | 20260507 05:58:34.698 | INFO | } | 20260507 05:58:34.698 | INFO | Perf report for the last 5.13601 seconds: | 20260507 05:58:34.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:58:34.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:34.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:34.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:34.698 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 05:58:34.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:34.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:34.699 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 05:58:34.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:34.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:34.699 | INFO | Uploaded 36GiB to GPU in 2.05951s (17.4799GiB/s) | 20260507 05:58:34.699 | INFO | Downloaded 34GiB to GPU in 1.94824s (17.4517GiB/s) | 20260507 05:58:39.842 | INFO | Analyzing chunk frames = [7633408,7633920) | 20260507 05:58:39.842 | INFO | MemInfo: { | 20260507 05:58:39.842 | INFO | "gpu" : " 17 GB", | 20260507 05:58:39.842 | INFO | "huge" : "", | 20260507 05:58:39.843 | INFO | "huge_shared" : " 27 GB", | 20260507 05:58:39.843 | INFO | "malloc" : " 14 GB", | 20260507 05:58:39.843 | INFO | "pinned" : " 29 GB" | 20260507 05:58:39.843 | INFO | } | 20260507 05:58:39.843 | INFO | Perf report for the last 5.14413 seconds: | 20260507 05:58:39.843 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:58:39.843 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:39.843 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:39.843 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:39.843 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 05:58:39.843 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:39.843 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:39.843 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 05:58:39.843 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:39.843 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:39.843 | INFO | Uploaded 36GiB to GPU in 2.04664s (17.5898GiB/s) | 20260507 05:58:39.843 | INFO | Downloaded 34GiB to GPU in 1.90913s (17.8091GiB/s) | 20260507 05:58:44.973 | INFO | Analyzing chunk frames = [7633920,7634432) | 20260507 05:58:44.973 | INFO | MemInfo: { | 20260507 05:58:44.973 | INFO | "gpu" : " 17 GB", | 20260507 05:58:44.973 | INFO | "huge" : "", | 20260507 05:58:44.973 | INFO | "huge_shared" : " 27 GB", | 20260507 05:58:44.973 | INFO | "malloc" : " 14 GB", | 20260507 05:58:44.973 | INFO | "pinned" : " 29 GB" | 20260507 05:58:44.973 | INFO | } | 20260507 05:58:44.973 | INFO | Perf report for the last 5.13055 seconds: | 20260507 05:58:44.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:58:44.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:44.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:44.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:44.973 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260507 05:58:44.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:44.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:44.973 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 05:58:44.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:44.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:44.973 | INFO | Uploaded 36GiB to GPU in 1.89623s (18.985GiB/s) | 20260507 05:58:44.974 | INFO | Downloaded 34GiB to GPU in 1.97213s (17.2402GiB/s) | 20260507 05:58:50.113 | INFO | Analyzing chunk frames = [7634432,7634944) | 20260507 05:58:50.113 | INFO | MemInfo: { | 20260507 05:58:50.113 | INFO | "gpu" : " 17 GB", | 20260507 05:58:50.113 | INFO | "huge" : "", | 20260507 05:58:50.113 | INFO | "huge_shared" : " 27 GB", | 20260507 05:58:50.113 | INFO | "malloc" : " 14 GB", | 20260507 05:58:50.113 | INFO | "pinned" : " 29 GB" | 20260507 05:58:50.113 | INFO | } | 20260507 05:58:50.113 | INFO | Perf report for the last 5.13999 seconds: | 20260507 05:58:50.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:58:50.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:50.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:50.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:50.113 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 05:58:50.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:50.113 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:50.113 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 05:58:50.113 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:50.113 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:50.113 | INFO | Uploaded 36GiB to GPU in 1.99758s (18.0218GiB/s) | 20260507 05:58:50.113 | INFO | Downloaded 34GiB to GPU in 1.95054s (17.4311GiB/s) | 20260507 05:58:50.114 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 05:58:50.114 | INFO | Total mem usage: 18 GiB | 20260507 05:58:50.114 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 05:58:50.114 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 05:58:50.114 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 05:58:50.114 | INFO | High Water Breakdown: | 20260507 05:58:50.114 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 05:58:50.114 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 05:58:50.114 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 05:58:50.114 | INFO | ------------------------------------------ | 20260507 05:58:50.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:58:50.114 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 05:58:50.114 | INFO | Total mem usage: 26.5 GiB | 20260507 05:58:50.114 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 05:58:50.114 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 05:58:50.114 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 05:58:50.114 | INFO | High Water Breakdown: | 20260507 05:58:50.114 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:58:50.114 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:58:50.114 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:58:50.114 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:58:50.114 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:58:50.114 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:58:50.114 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 05:58:50.114 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 05:58:50.114 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:58:50.114 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 05:58:50.114 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 05:58:50.114 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 05:58:50.114 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 05:58:50.114 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 05:58:50.114 | INFO | ------------------------------------------ | 20260507 05:58:50.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:58:50.114 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 05:58:50.114 | INFO | Total mem usage: 25.6 GiB | 20260507 05:58:50.114 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 05:58:50.114 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 05:58:50.114 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 05:58:50.114 | INFO | High Water Breakdown: | 20260507 05:58:50.114 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 05:58:50.114 | INFO | ------------------------------------------ | 20260507 05:58:50.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:58:50.114 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 05:58:50.114 | INFO | Total mem usage: 15.7 GiB | 20260507 05:58:50.114 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 05:58:50.114 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 05:58:50.114 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 05:58:50.114 | INFO | High Water Breakdown: | 20260507 05:58:50.114 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 05:58:50.114 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 05:58:50.114 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 05:58:50.114 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 05:58:50.114 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 05:58:50.114 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 05:58:50.114 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 05:58:50.114 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 05:58:50.114 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 05:58:50.114 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 05:58:50.114 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 05:58:50.114 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 05:58:50.114 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 05:58:50.114 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 05:58:50.114 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 05:58:50.114 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 05:58:50.114 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 05:58:50.114 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 05:58:50.114 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 05:58:50.114 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 05:58:50.114 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 05:58:50.114 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 05:58:50.114 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 05:58:50.114 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 05:58:50.114 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 05:58:50.114 | INFO | ------------------------------------------ | 20260507 05:58:50.114 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 05:58:50.114 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 05:58:50.114 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:58:50.114 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:58:50.114 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.50e+01, 1.34e+01 / 1.18e+00 | 20260507 05:58:50.114 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.02e-01, 9.78e-03 / 4.45e-03 | 20260507 05:58:50.114 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.45e+00, 5.16e-01 / 2.57e-01 | 20260507 05:58:50.114 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.48e+00, 1.70e-01 / 1.22e-01 | 20260507 05:58:50.114 | INFO | PerfReport: 16.19%, DME -- 3.98e+01 / 4.46e+01, 4.00e+00 / 1.37e+00 | 20260507 05:58:50.114 | INFO | PerfReport: 57.73%, FrameLabeling -- 1.78e+01 / 2.00e+01, 1.37e+00 / 1.48e+00 | 20260507 05:58:50.115 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.68e+00, 3.83e-01 / 2.00e-01 | 20260507 05:58:50.115 | INFO | PerfReport: 2.97%, Metrics -- 9.12e-01 / 1.24e+00, 2.45e-01 / 1.32e-01 | 20260507 05:58:50.115 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.07e-01 / 3.63e-01, 5.11e-02 / 1.74e-02 | 20260507 05:58:50.115 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 05:58:50.115 | INFO | PerfReport: Gathered 1536 samples. | 20260507 05:58:50.115 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 05:58:50.115 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 05:58:50.115 | INFO | PerfReport: 21.84%, Upload -- 2.10e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260507 05:58:50.115 | INFO | PerfReport: 21.33%, Download -- 2.05e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 05:58:50.115 | INFO | PerfReport: 31.98%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 05:58:50.115 | INFO | PerfReport: 1.60%, LockUpload -- 1.54e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 05:58:50.115 | INFO | PerfReport: 1.18%, LockDownload -- 1.14e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260507 05:58:50.115 | INFO | PerfReport: 22.05%, LockCompute -- 2.12e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 05:58:50.115 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260507 05:58:50.210 | INFO | Starting a round of Baz IO | 20260507 05:58:53.306 | INFO | Populating RTMetrics report | 20260507 05:58:53.306 | INFO | RTMetrics report populated | 20260507 05:58:53.306 | INFO | Opening tmp RTMetrics json file | 20260507 05:58:53.307 | INFO | Serializing RTMetrics to json | 20260507 05:58:53.321 | INFO | Serialization done, swapping tmp file now... | 20260507 05:58:53.332 | INFO | RTMetrics done | 20260507 05:58:53.401 | INFO | Finished a round of Baz IO: Wrote 1.681 GiB in 3.19 seconds | 20260507 05:58:53.401 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.66% / 0.00% / 67.34% | 20260507 05:58:53.401 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.130975% / 3.97364e-06% / 7.15256e-05% | 20260507 05:58:53.401 | INFO | Baz IO overall: | 20260507 05:58:53.401 | INFO | BytesWritten : 3861570785743 | 20260507 05:58:53.401 | INFO | NumEvents : 1851340596773 | 20260507 05:58:55.255 | INFO | Analyzing chunk frames = [7634944,7635456) | 20260507 05:58:55.255 | INFO | MemInfo: { | 20260507 05:58:55.255 | INFO | "gpu" : " 17 GB", | 20260507 05:58:55.255 | INFO | "huge" : "", | 20260507 05:58:55.256 | INFO | "huge_shared" : " 27 GB", | 20260507 05:58:55.256 | INFO | "malloc" : " 14 GB", | 20260507 05:58:55.256 | INFO | "pinned" : " 29 GB" | 20260507 05:58:55.256 | INFO | } | 20260507 05:58:55.256 | INFO | Perf report for the last 5.14241 seconds: | 20260507 05:58:55.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:58:55.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:55.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:55.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:55.256 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 05:58:55.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:58:55.256 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 05:58:55.256 | INFO | PRE_HQ : 60.2%, 1.9, 58.4ms, 0 | 20260507 05:58:55.256 | INFO | BAZWRITER : 61.6%, 3.3, 104.2ms, 0 | 20260507 05:58:55.256 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:58:55.256 | INFO | Uploaded 36GiB to GPU in 2.02695s (17.7607GiB/s) | 20260507 05:58:55.256 | INFO | Downloaded 38GiB to GPU in 2.17509s (17.4705GiB/s) | 20260507 05:59:00.392 | INFO | Analyzing chunk frames = [7635456,7635968) | 20260507 05:59:00.392 | INFO | MemInfo: { | 20260507 05:59:00.392 | INFO | "gpu" : " 17 GB", | 20260507 05:59:00.392 | INFO | "huge" : "", | 20260507 05:59:00.392 | INFO | "huge_shared" : " 27 GB", | 20260507 05:59:00.392 | INFO | "malloc" : " 14 GB", | 20260507 05:59:00.393 | INFO | "pinned" : " 29 GB" | 20260507 05:59:00.393 | INFO | } | 20260507 05:59:00.393 | INFO | Perf report for the last 5.13693 seconds: | 20260507 05:59:00.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:59:00.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:00.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:00.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:00.393 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 05:59:00.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:00.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:00.393 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 05:59:00.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:00.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:00.393 | INFO | Uploaded 36GiB to GPU in 2.0006s (17.9946GiB/s) | 20260507 05:59:00.393 | INFO | Downloaded 34GiB to GPU in 1.92402s (17.6713GiB/s) | 20260507 05:59:05.533 | INFO | Analyzing chunk frames = [7635968,7636480) | 20260507 05:59:05.533 | INFO | MemInfo: { | 20260507 05:59:05.533 | INFO | "gpu" : " 17 GB", | 20260507 05:59:05.533 | INFO | "huge" : "", | 20260507 05:59:05.533 | INFO | "huge_shared" : " 27 GB", | 20260507 05:59:05.533 | INFO | "malloc" : " 14 GB", | 20260507 05:59:05.533 | INFO | "pinned" : " 29 GB" | 20260507 05:59:05.533 | INFO | } | 20260507 05:59:05.533 | INFO | Perf report for the last 5.14082 seconds: | 20260507 05:59:05.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:59:05.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:05.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:05.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:05.533 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 05:59:05.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:05.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:05.533 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:59:05.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:05.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:05.534 | INFO | Uploaded 36GiB to GPU in 2.05211s (17.543GiB/s) | 20260507 05:59:05.534 | INFO | Downloaded 34GiB to GPU in 1.94881s (17.4466GiB/s) | 20260507 05:59:10.667 | INFO | Analyzing chunk frames = [7636480,7636992) | 20260507 05:59:10.667 | INFO | MemInfo: { | 20260507 05:59:10.667 | INFO | "gpu" : " 17 GB", | 20260507 05:59:10.667 | INFO | "huge" : "", | 20260507 05:59:10.667 | INFO | "huge_shared" : " 27 GB", | 20260507 05:59:10.667 | INFO | "malloc" : " 14 GB", | 20260507 05:59:10.667 | INFO | "pinned" : " 29 GB" | 20260507 05:59:10.667 | INFO | } | 20260507 05:59:10.667 | INFO | Perf report for the last 5.1341 seconds: | 20260507 05:59:10.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:59:10.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:10.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:10.668 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:10.668 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 05:59:10.668 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:10.668 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:10.668 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 05:59:10.668 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:10.668 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:10.668 | INFO | Uploaded 36GiB to GPU in 2.01501s (17.8659GiB/s) | 20260507 05:59:10.668 | INFO | Downloaded 34GiB to GPU in 1.93619s (17.5602GiB/s) | 20260507 05:59:15.812 | INFO | Analyzing chunk frames = [7636992,7637504) | 20260507 05:59:15.812 | INFO | MemInfo: { | 20260507 05:59:15.812 | INFO | "gpu" : " 17 GB", | 20260507 05:59:15.812 | INFO | "huge" : "", | 20260507 05:59:15.812 | INFO | "huge_shared" : " 27 GB", | 20260507 05:59:15.812 | INFO | "malloc" : " 14 GB", | 20260507 05:59:15.812 | INFO | "pinned" : " 29 GB" | 20260507 05:59:15.812 | INFO | } | 20260507 05:59:15.812 | INFO | Perf report for the last 5.14437 seconds: | 20260507 05:59:15.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:59:15.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:15.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:15.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:15.812 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 05:59:15.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:15.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:15.812 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 05:59:15.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:15.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:15.812 | INFO | Uploaded 36GiB to GPU in 2.04265s (17.6242GiB/s) | 20260507 05:59:15.812 | INFO | Downloaded 34GiB to GPU in 1.93796s (17.5442GiB/s) | 20260507 05:59:20.954 | INFO | Analyzing chunk frames = [7637504,7638016) | 20260507 05:59:20.954 | INFO | MemInfo: { | 20260507 05:59:20.954 | INFO | "gpu" : " 17 GB", | 20260507 05:59:20.954 | INFO | "huge" : "", | 20260507 05:59:20.954 | INFO | "huge_shared" : " 27 GB", | 20260507 05:59:20.954 | INFO | "malloc" : " 14 GB", | 20260507 05:59:20.954 | INFO | "pinned" : " 29 GB" | 20260507 05:59:20.954 | INFO | } | 20260507 05:59:20.954 | INFO | Perf report for the last 5.1421 seconds: | 20260507 05:59:20.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:59:20.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:20.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:20.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:20.954 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 05:59:20.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:20.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:20.954 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 05:59:20.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:20.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:20.954 | INFO | Uploaded 36GiB to GPU in 2.04208s (17.6291GiB/s) | 20260507 05:59:20.954 | INFO | Downloaded 34GiB to GPU in 1.9007s (17.8881GiB/s) | 20260507 05:59:26.091 | INFO | Analyzing chunk frames = [7638016,7638528) | 20260507 05:59:26.091 | INFO | MemInfo: { | 20260507 05:59:26.091 | INFO | "gpu" : " 17 GB", | 20260507 05:59:26.091 | INFO | "huge" : "", | 20260507 05:59:26.091 | INFO | "huge_shared" : " 27 GB", | 20260507 05:59:26.091 | INFO | "malloc" : " 14 GB", | 20260507 05:59:26.091 | INFO | "pinned" : " 29 GB" | 20260507 05:59:26.091 | INFO | } | 20260507 05:59:26.091 | INFO | Perf report for the last 5.1373 seconds: | 20260507 05:59:26.091 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:59:26.091 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:26.091 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:26.091 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:26.091 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 05:59:26.091 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:26.091 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:26.091 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 05:59:26.091 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:26.091 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:26.092 | INFO | Uploaded 36GiB to GPU in 1.92357s (18.7152GiB/s) | 20260507 05:59:26.092 | INFO | Downloaded 34GiB to GPU in 1.97173s (17.2438GiB/s) | 20260507 05:59:31.224 | INFO | Analyzing chunk frames = [7638528,7639040) | 20260507 05:59:31.224 | INFO | MemInfo: { | 20260507 05:59:31.224 | INFO | "gpu" : " 17 GB", | 20260507 05:59:31.224 | INFO | "huge" : "", | 20260507 05:59:31.224 | INFO | "huge_shared" : " 27 GB", | 20260507 05:59:31.224 | INFO | "malloc" : " 14 GB", | 20260507 05:59:31.224 | INFO | "pinned" : " 29 GB" | 20260507 05:59:31.224 | INFO | } | 20260507 05:59:31.224 | INFO | Perf report for the last 5.1324 seconds: | 20260507 05:59:31.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:59:31.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:31.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:31.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:31.224 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 05:59:31.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:31.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:31.224 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 05:59:31.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:31.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:31.224 | INFO | Uploaded 36GiB to GPU in 1.97872s (18.1936GiB/s) | 20260507 05:59:31.224 | INFO | Downloaded 34GiB to GPU in 1.95376s (17.4024GiB/s) | 20260507 05:59:31.321 | INFO | Starting a round of Baz IO | 20260507 05:59:34.417 | INFO | Populating RTMetrics report | 20260507 05:59:34.418 | INFO | RTMetrics report populated | 20260507 05:59:34.418 | INFO | Opening tmp RTMetrics json file | 20260507 05:59:34.428 | INFO | Serializing RTMetrics to json | 20260507 05:59:34.436 | INFO | Serialization done, swapping tmp file now... | 20260507 05:59:34.437 | INFO | RTMetrics done | 20260507 05:59:34.509 | INFO | Finished a round of Baz IO: Wrote 1.679 GiB in 3.188 seconds | 20260507 05:59:34.509 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.66% / 0.00% / 67.34% | 20260507 05:59:34.509 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.131265% / 0% / 7.94729e-05% | 20260507 05:59:34.509 | INFO | Baz IO overall: | 20260507 05:59:34.509 | INFO | BytesWritten : 3863373456731 | 20260507 05:59:34.509 | INFO | NumEvents : 1852185655170 | 20260507 05:59:36.369 | INFO | Analyzing chunk frames = [7639040,7639552) | 20260507 05:59:36.369 | INFO | MemInfo: { | 20260507 05:59:36.369 | INFO | "gpu" : " 17 GB", | 20260507 05:59:36.369 | INFO | "huge" : "", | 20260507 05:59:36.369 | INFO | "huge_shared" : " 27 GB", | 20260507 05:59:36.369 | INFO | "malloc" : " 14 GB", | 20260507 05:59:36.369 | INFO | "pinned" : " 29 GB" | 20260507 05:59:36.370 | INFO | } | 20260507 05:59:36.370 | INFO | Perf report for the last 5.14581 seconds: | 20260507 05:59:36.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:59:36.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:36.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:36.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:36.370 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 05:59:36.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:36.370 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 05:59:36.370 | INFO | PRE_HQ : 58.8%, 1.8, 54.6ms, 0 | 20260507 05:59:36.370 | INFO | BAZWRITER : 61.5%, 3.2, 102.9ms, 0 | 20260507 05:59:36.370 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 05:59:36.370 | INFO | Uploaded 36GiB to GPU in 2.04117s (17.637GiB/s) | 20260507 05:59:36.370 | INFO | Downloaded 38GiB to GPU in 2.18382s (17.4007GiB/s) | 20260507 05:59:41.518 | INFO | Analyzing chunk frames = [7639552,7640064) | 20260507 05:59:41.518 | INFO | MemInfo: { | 20260507 05:59:41.518 | INFO | "gpu" : " 17 GB", | 20260507 05:59:41.518 | INFO | "huge" : "", | 20260507 05:59:41.518 | INFO | "huge_shared" : " 27 GB", | 20260507 05:59:41.518 | INFO | "malloc" : " 14 GB", | 20260507 05:59:41.518 | INFO | "pinned" : " 29 GB" | 20260507 05:59:41.518 | INFO | } | 20260507 05:59:41.518 | INFO | Perf report for the last 5.14844 seconds: | 20260507 05:59:41.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:59:41.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:41.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:41.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:41.518 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 05:59:41.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:41.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:41.518 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 05:59:41.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:41.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:41.518 | INFO | Uploaded 36GiB to GPU in 2.00337s (17.9697GiB/s) | 20260507 05:59:41.518 | INFO | Downloaded 34GiB to GPU in 1.93215s (17.597GiB/s) | 20260507 05:59:46.651 | INFO | Analyzing chunk frames = [7640064,7640576) | 20260507 05:59:46.651 | INFO | MemInfo: { | 20260507 05:59:46.651 | INFO | "gpu" : " 17 GB", | 20260507 05:59:46.651 | INFO | "huge" : "", | 20260507 05:59:46.651 | INFO | "huge_shared" : " 27 GB", | 20260507 05:59:46.651 | INFO | "malloc" : " 14 GB", | 20260507 05:59:46.651 | INFO | "pinned" : " 29 GB" | 20260507 05:59:46.651 | INFO | } | 20260507 05:59:46.652 | INFO | Perf report for the last 5.13349 seconds: | 20260507 05:59:46.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:59:46.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:46.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:46.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:46.652 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 05:59:46.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:46.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:46.652 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 05:59:46.652 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:46.652 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:46.652 | INFO | Uploaded 36GiB to GPU in 2.03888s (17.6567GiB/s) | 20260507 05:59:46.652 | INFO | Downloaded 34GiB to GPU in 1.91741s (17.7322GiB/s) | 20260507 05:59:51.787 | INFO | Analyzing chunk frames = [7640576,7641088) | 20260507 05:59:51.787 | INFO | MemInfo: { | 20260507 05:59:51.787 | INFO | "gpu" : " 17 GB", | 20260507 05:59:51.787 | INFO | "huge" : "", | 20260507 05:59:51.787 | INFO | "huge_shared" : " 27 GB", | 20260507 05:59:51.787 | INFO | "malloc" : " 14 GB", | 20260507 05:59:51.787 | INFO | "pinned" : " 29 GB" | 20260507 05:59:51.787 | INFO | } | 20260507 05:59:51.787 | INFO | Perf report for the last 5.13541 seconds: | 20260507 05:59:51.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:59:51.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:51.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:51.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:51.787 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 05:59:51.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:51.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:51.787 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 05:59:51.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:51.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:51.787 | INFO | Uploaded 36GiB to GPU in 2.0477s (17.5807GiB/s) | 20260507 05:59:51.787 | INFO | Downloaded 34GiB to GPU in 1.94541s (17.4771GiB/s) | 20260507 05:59:56.924 | INFO | Analyzing chunk frames = [7641088,7641600) | 20260507 05:59:56.924 | INFO | MemInfo: { | 20260507 05:59:56.924 | INFO | "gpu" : " 17 GB", | 20260507 05:59:56.924 | INFO | "huge" : "", | 20260507 05:59:56.924 | INFO | "huge_shared" : " 27 GB", | 20260507 05:59:56.924 | INFO | "malloc" : " 14 GB", | 20260507 05:59:56.924 | INFO | "pinned" : " 29 GB" | 20260507 05:59:56.924 | INFO | } | 20260507 05:59:56.924 | INFO | Perf report for the last 5.13689 seconds: | 20260507 05:59:56.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 05:59:56.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:56.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:56.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:56.924 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 05:59:56.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:56.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:56.924 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 05:59:56.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:56.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 05:59:56.924 | INFO | Uploaded 36GiB to GPU in 2.05707s (17.5006GiB/s) | 20260507 05:59:56.924 | INFO | Downloaded 34GiB to GPU in 1.94721s (17.4609GiB/s) | 20260507 06:00:02.066 | INFO | Analyzing chunk frames = [7641600,7642112) | 20260507 06:00:02.066 | INFO | MemInfo: { | 20260507 06:00:02.066 | INFO | "gpu" : " 17 GB", | 20260507 06:00:02.066 | INFO | "huge" : "", | 20260507 06:00:02.066 | INFO | "huge_shared" : " 27 GB", | 20260507 06:00:02.066 | INFO | "malloc" : " 14 GB", | 20260507 06:00:02.066 | INFO | "pinned" : " 29 GB" | 20260507 06:00:02.066 | INFO | } | 20260507 06:00:02.066 | INFO | Perf report for the last 5.14216 seconds: | 20260507 06:00:02.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:00:02.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:02.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:02.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:02.066 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:00:02.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:02.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:02.066 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 06:00:02.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:02.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:02.066 | INFO | Uploaded 36GiB to GPU in 2.03764s (17.6675GiB/s) | 20260507 06:00:02.066 | INFO | Downloaded 34GiB to GPU in 1.90418s (17.8554GiB/s) | 20260507 06:00:07.205 | INFO | Analyzing chunk frames = [7642112,7642624) | 20260507 06:00:07.205 | INFO | MemInfo: { | 20260507 06:00:07.205 | INFO | "gpu" : " 17 GB", | 20260507 06:00:07.205 | INFO | "huge" : "", | 20260507 06:00:07.205 | INFO | "huge_shared" : " 27 GB", | 20260507 06:00:07.205 | INFO | "malloc" : " 14 GB", | 20260507 06:00:07.205 | INFO | "pinned" : " 29 GB" | 20260507 06:00:07.205 | INFO | } | 20260507 06:00:07.205 | INFO | Perf report for the last 5.13915 seconds: | 20260507 06:00:07.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:00:07.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:07.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:07.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:07.205 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 06:00:07.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:07.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:07.205 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:00:07.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:07.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:07.205 | INFO | Uploaded 36GiB to GPU in 2.01647s (17.853GiB/s) | 20260507 06:00:07.205 | INFO | Downloaded 34GiB to GPU in 1.98511s (17.1276GiB/s) | 20260507 06:00:12.351 | INFO | Analyzing chunk frames = [7642624,7643136) | 20260507 06:00:12.352 | INFO | MemInfo: { | 20260507 06:00:12.352 | INFO | "gpu" : " 17 GB", | 20260507 06:00:12.352 | INFO | "huge" : "", | 20260507 06:00:12.352 | INFO | "huge_shared" : " 27 GB", | 20260507 06:00:12.352 | INFO | "malloc" : " 14 GB", | 20260507 06:00:12.352 | INFO | "pinned" : " 29 GB" | 20260507 06:00:12.352 | INFO | } | 20260507 06:00:12.352 | INFO | Perf report for the last 5.14651 seconds: | 20260507 06:00:12.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:00:12.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:12.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:12.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:12.352 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:00:12.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:12.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:12.352 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:00:12.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:12.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:12.352 | INFO | Uploaded 36GiB to GPU in 1.99369s (18.057GiB/s) | 20260507 06:00:12.352 | INFO | Downloaded 34GiB to GPU in 1.95667s (17.3765GiB/s) | 20260507 06:00:12.352 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:00:12.352 | INFO | Total mem usage: 18 GiB | 20260507 06:00:12.352 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:00:12.352 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:00:12.352 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:00:12.352 | INFO | High Water Breakdown: | 20260507 06:00:12.352 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:00:12.352 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:00:12.352 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:00:12.352 | INFO | ------------------------------------------ | 20260507 06:00:12.352 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:00:12.352 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:00:12.352 | INFO | Total mem usage: 26.5 GiB | 20260507 06:00:12.352 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:00:12.352 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:00:12.352 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:00:12.352 | INFO | High Water Breakdown: | 20260507 06:00:12.352 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:00:12.352 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:00:12.352 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:00:12.352 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:00:12.352 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:00:12.352 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:00:12.352 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:00:12.352 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:00:12.352 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:00:12.352 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:00:12.352 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:00:12.352 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:00:12.352 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:00:12.352 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:00:12.352 | INFO | ------------------------------------------ | 20260507 06:00:12.352 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:00:12.352 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:00:12.352 | INFO | Total mem usage: 25.6 GiB | 20260507 06:00:12.352 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:00:12.352 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:00:12.352 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:00:12.352 | INFO | High Water Breakdown: | 20260507 06:00:12.352 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:00:12.352 | INFO | ------------------------------------------ | 20260507 06:00:12.352 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:00:12.353 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:00:12.353 | INFO | Total mem usage: 15.7 GiB | 20260507 06:00:12.353 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:00:12.353 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:00:12.353 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:00:12.353 | INFO | High Water Breakdown: | 20260507 06:00:12.353 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:00:12.353 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:00:12.353 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:00:12.353 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:00:12.353 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:00:12.353 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:00:12.353 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:00:12.353 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:00:12.353 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:00:12.353 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:00:12.353 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:00:12.353 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:00:12.353 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:00:12.353 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:00:12.353 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:00:12.353 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:00:12.353 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:00:12.353 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:00:12.353 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:00:12.353 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:00:12.353 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:00:12.353 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:00:12.353 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:00:12.353 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:00:12.353 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:00:12.353 | INFO | ------------------------------------------ | 20260507 06:00:12.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:00:12.353 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:00:12.353 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:00:12.353 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:00:12.353 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.34e+01 / 1.13e+00 | 20260507 06:00:12.353 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.02e-01, 9.29e-03 / 5.88e-03 | 20260507 06:00:12.353 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.43e+00, 5.14e-01 / 2.86e-01 | 20260507 06:00:12.353 | INFO | PerfReport: 3.22%, Histogram -- 9.90e-01 / 1.47e+00, 1.66e-01 / 1.04e-01 | 20260507 06:00:12.353 | INFO | PerfReport: 16.26%, DME -- 3.99e+01 / 4.50e+01, 4.01e+00 / 1.62e+00 | 20260507 06:00:12.353 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.26e+00 / 1.47e+00 | 20260507 06:00:12.353 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.73e+00, 3.85e-01 / 1.77e-01 | 20260507 06:00:12.353 | INFO | PerfReport: 2.97%, Metrics -- 9.11e-01 / 1.25e+00, 2.43e-01 / 1.33e-01 | 20260507 06:00:12.353 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.07e-01 / 3.65e-01, 5.08e-02 / 1.94e-02 | 20260507 06:00:12.353 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:00:12.353 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:00:12.353 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:00:12.353 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 06:00:12.353 | INFO | PerfReport: 21.87%, Upload -- 2.10e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260507 06:00:12.353 | INFO | PerfReport: 21.33%, Download -- 2.05e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260507 06:00:12.353 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:00:12.353 | INFO | PerfReport: 1.64%, LockUpload -- 1.58e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 06:00:12.353 | INFO | PerfReport: 1.18%, LockDownload -- 1.13e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260507 06:00:12.353 | INFO | PerfReport: 22.03%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 06:00:12.353 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.51e-02 / 0.00e+00 | 20260507 06:00:12.449 | INFO | Starting a round of Baz IO | 20260507 06:00:15.545 | INFO | Populating RTMetrics report | 20260507 06:00:15.545 | INFO | RTMetrics report populated | 20260507 06:00:15.545 | INFO | Opening tmp RTMetrics json file | 20260507 06:00:15.561 | INFO | Serializing RTMetrics to json | 20260507 06:00:15.563 | INFO | Serialization done, swapping tmp file now... | 20260507 06:00:15.563 | INFO | RTMetrics done | 20260507 06:00:15.630 | INFO | Finished a round of Baz IO: Wrote 1.679 GiB in 3.18 seconds | 20260507 06:00:15.630 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.66% / 0.00% / 67.34% | 20260507 06:00:15.630 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.130884% / 0% / 7.94729e-05% | 20260507 06:00:15.630 | INFO | Baz IO overall: | 20260507 06:00:15.630 | INFO | BytesWritten : 3865175850627 | 20260507 06:00:15.630 | INFO | NumEvents : 1853030331006 | 20260507 06:00:17.483 | INFO | Analyzing chunk frames = [7643136,7643648) | 20260507 06:00:17.483 | INFO | MemInfo: { | 20260507 06:00:17.483 | INFO | "gpu" : " 17 GB", | 20260507 06:00:17.483 | INFO | "huge" : "", | 20260507 06:00:17.483 | INFO | "huge_shared" : " 27 GB", | 20260507 06:00:17.483 | INFO | "malloc" : " 14 GB", | 20260507 06:00:17.483 | INFO | "pinned" : " 29 GB" | 20260507 06:00:17.483 | INFO | } | 20260507 06:00:17.483 | INFO | Perf report for the last 5.13129 seconds: | 20260507 06:00:17.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:00:17.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:17.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:17.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:17.483 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 06:00:17.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:17.483 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 06:00:17.483 | INFO | PRE_HQ : 59.0%, 1.8, 54.8ms, 0 | 20260507 06:00:17.483 | INFO | BAZWRITER : 61.6%, 3.2, 99.7ms, 0 | 20260507 06:00:17.483 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:00:17.483 | INFO | Uploaded 36GiB to GPU in 2.06972s (17.3937GiB/s) | 20260507 06:00:17.483 | INFO | Downloaded 38GiB to GPU in 2.17465s (17.4741GiB/s) | 20260507 06:00:22.623 | INFO | Analyzing chunk frames = [7643648,7644160) | 20260507 06:00:22.623 | INFO | MemInfo: { | 20260507 06:00:22.623 | INFO | "gpu" : " 17 GB", | 20260507 06:00:22.623 | INFO | "huge" : "", | 20260507 06:00:22.623 | INFO | "huge_shared" : " 27 GB", | 20260507 06:00:22.623 | INFO | "malloc" : " 14 GB", | 20260507 06:00:22.623 | INFO | "pinned" : " 29 GB" | 20260507 06:00:22.623 | INFO | } | 20260507 06:00:22.623 | INFO | Perf report for the last 5.14031 seconds: | 20260507 06:00:22.623 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:00:22.623 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:22.623 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:22.623 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:22.623 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 06:00:22.623 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:22.623 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:22.623 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:00:22.623 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:22.623 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:22.624 | INFO | Uploaded 36GiB to GPU in 1.99814s (18.0167GiB/s) | 20260507 06:00:22.624 | INFO | Downloaded 34GiB to GPU in 1.93249s (17.5939GiB/s) | 20260507 06:00:27.761 | INFO | Analyzing chunk frames = [7644160,7644672) | 20260507 06:00:27.761 | INFO | MemInfo: { | 20260507 06:00:27.761 | INFO | "gpu" : " 17 GB", | 20260507 06:00:27.761 | INFO | "huge" : "", | 20260507 06:00:27.761 | INFO | "huge_shared" : " 27 GB", | 20260507 06:00:27.761 | INFO | "malloc" : " 14 GB", | 20260507 06:00:27.761 | INFO | "pinned" : " 29 GB" | 20260507 06:00:27.761 | INFO | } | 20260507 06:00:27.761 | INFO | Perf report for the last 5.13758 seconds: | 20260507 06:00:27.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:00:27.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:27.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:27.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:27.761 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 06:00:27.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:27.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:27.761 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:00:27.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:27.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:27.761 | INFO | Uploaded 36GiB to GPU in 2.01449s (17.8705GiB/s) | 20260507 06:00:27.761 | INFO | Downloaded 34GiB to GPU in 1.93857s (17.5387GiB/s) | 20260507 06:00:32.903 | INFO | Analyzing chunk frames = [7644672,7645184) | 20260507 06:00:32.903 | INFO | MemInfo: { | 20260507 06:00:32.903 | INFO | "gpu" : " 17 GB", | 20260507 06:00:32.903 | INFO | "huge" : "", | 20260507 06:00:32.903 | INFO | "huge_shared" : " 27 GB", | 20260507 06:00:32.903 | INFO | "malloc" : " 14 GB", | 20260507 06:00:32.903 | INFO | "pinned" : " 29 GB" | 20260507 06:00:32.903 | INFO | } | 20260507 06:00:32.903 | INFO | Perf report for the last 5.14244 seconds: | 20260507 06:00:32.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:00:32.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:32.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:32.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:32.903 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 06:00:32.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:32.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:32.903 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:00:32.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:32.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:32.904 | INFO | Uploaded 36GiB to GPU in 2.00395s (17.9645GiB/s) | 20260507 06:00:32.904 | INFO | Downloaded 34GiB to GPU in 1.93618s (17.5604GiB/s) | 20260507 06:00:38.038 | INFO | Analyzing chunk frames = [7645184,7645696) | 20260507 06:00:38.038 | INFO | MemInfo: { | 20260507 06:00:38.038 | INFO | "gpu" : " 17 GB", | 20260507 06:00:38.038 | INFO | "huge" : "", | 20260507 06:00:38.038 | INFO | "huge_shared" : " 27 GB", | 20260507 06:00:38.038 | INFO | "malloc" : " 14 GB", | 20260507 06:00:38.038 | INFO | "pinned" : " 29 GB" | 20260507 06:00:38.038 | INFO | } | 20260507 06:00:38.038 | INFO | Perf report for the last 5.13494 seconds: | 20260507 06:00:38.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:00:38.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:38.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:38.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:38.038 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 06:00:38.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:38.038 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:38.038 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:00:38.038 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:38.038 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:38.039 | INFO | Uploaded 36GiB to GPU in 2.03756s (17.6682GiB/s) | 20260507 06:00:38.039 | INFO | Downloaded 34GiB to GPU in 1.91638s (17.7418GiB/s) | 20260507 06:00:43.178 | INFO | Analyzing chunk frames = [7645696,7646208) | 20260507 06:00:43.178 | INFO | MemInfo: { | 20260507 06:00:43.178 | INFO | "gpu" : " 17 GB", | 20260507 06:00:43.178 | INFO | "huge" : "", | 20260507 06:00:43.178 | INFO | "huge_shared" : " 27 GB", | 20260507 06:00:43.178 | INFO | "malloc" : " 14 GB", | 20260507 06:00:43.178 | INFO | "pinned" : " 29 GB" | 20260507 06:00:43.178 | INFO | } | 20260507 06:00:43.178 | INFO | Perf report for the last 5.14005 seconds: | 20260507 06:00:43.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:00:43.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:43.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:43.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:43.178 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:00:43.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:43.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:43.178 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 06:00:43.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:43.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:43.179 | INFO | Uploaded 36GiB to GPU in 2.05295s (17.5357GiB/s) | 20260507 06:00:43.179 | INFO | Downloaded 34GiB to GPU in 1.90552s (17.8429GiB/s) | 20260507 06:00:48.322 | INFO | Analyzing chunk frames = [7646208,7646720) | 20260507 06:00:48.322 | INFO | MemInfo: { | 20260507 06:00:48.322 | INFO | "gpu" : " 17 GB", | 20260507 06:00:48.322 | INFO | "huge" : "", | 20260507 06:00:48.322 | INFO | "huge_shared" : " 27 GB", | 20260507 06:00:48.322 | INFO | "malloc" : " 14 GB", | 20260507 06:00:48.322 | INFO | "pinned" : " 29 GB" | 20260507 06:00:48.322 | INFO | } | 20260507 06:00:48.322 | INFO | Perf report for the last 5.14329 seconds: | 20260507 06:00:48.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:00:48.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:48.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:48.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:48.322 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:00:48.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:48.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:48.322 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 06:00:48.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:48.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:48.322 | INFO | Uploaded 36GiB to GPU in 1.96747s (18.2976GiB/s) | 20260507 06:00:48.322 | INFO | Downloaded 34GiB to GPU in 1.97013s (17.2577GiB/s) | 20260507 06:00:53.469 | INFO | Analyzing chunk frames = [7646720,7647232) | 20260507 06:00:53.469 | INFO | MemInfo: { | 20260507 06:00:53.469 | INFO | "gpu" : " 17 GB", | 20260507 06:00:53.469 | INFO | "huge" : "", | 20260507 06:00:53.469 | INFO | "huge_shared" : " 27 GB", | 20260507 06:00:53.469 | INFO | "malloc" : " 14 GB", | 20260507 06:00:53.469 | INFO | "pinned" : " 29 GB" | 20260507 06:00:53.469 | INFO | } | 20260507 06:00:53.469 | INFO | Perf report for the last 5.14735 seconds: | 20260507 06:00:53.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:00:53.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:53.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:53.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:53.469 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 06:00:53.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:53.469 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:53.469 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:00:53.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:53.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:53.469 | INFO | Uploaded 36GiB to GPU in 1.96957s (18.2781GiB/s) | 20260507 06:00:53.469 | INFO | Downloaded 34GiB to GPU in 1.95234s (17.415GiB/s) | 20260507 06:00:53.566 | INFO | Starting a round of Baz IO | 20260507 06:00:56.654 | INFO | Populating RTMetrics report | 20260507 06:00:56.655 | INFO | RTMetrics report populated | 20260507 06:00:56.655 | INFO | Opening tmp RTMetrics json file | 20260507 06:00:56.664 | INFO | Serializing RTMetrics to json | 20260507 06:00:56.670 | INFO | Serialization done, swapping tmp file now... | 20260507 06:00:56.670 | INFO | RTMetrics done | 20260507 06:00:56.741 | INFO | Finished a round of Baz IO: Wrote 1.679 GiB in 3.174 seconds | 20260507 06:00:56.741 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.66% / 0.00% / 67.34% | 20260507 06:00:56.741 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.131214% / 0% / 5.5631e-05% | 20260507 06:00:56.741 | INFO | Baz IO overall: | 20260507 06:00:56.741 | INFO | BytesWritten : 3866979009693 | 20260507 06:00:56.741 | INFO | NumEvents : 1853875679039 | 20260507 06:00:58.603 | INFO | Analyzing chunk frames = [7647232,7647744) | 20260507 06:00:58.603 | INFO | MemInfo: { | 20260507 06:00:58.603 | INFO | "gpu" : " 17 GB", | 20260507 06:00:58.603 | INFO | "huge" : "", | 20260507 06:00:58.603 | INFO | "huge_shared" : " 27 GB", | 20260507 06:00:58.603 | INFO | "malloc" : " 14 GB", | 20260507 06:00:58.603 | INFO | "pinned" : " 29 GB" | 20260507 06:00:58.603 | INFO | } | 20260507 06:00:58.603 | INFO | Perf report for the last 5.13408 seconds: | 20260507 06:00:58.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:00:58.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:58.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:58.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:58.603 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260507 06:00:58.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:00:58.603 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:00:58.603 | INFO | PRE_HQ : 59.1%, 1.8, 54.7ms, 0 | 20260507 06:00:58.603 | INFO | BAZWRITER : 61.4%, 3.1, 99.1ms, 0 | 20260507 06:00:58.603 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:00:58.603 | INFO | Uploaded 36GiB to GPU in 2.06058s (17.4708GiB/s) | 20260507 06:00:58.603 | INFO | Downloaded 38GiB to GPU in 2.16945s (17.5159GiB/s) | 20260507 06:01:03.735 | INFO | Analyzing chunk frames = [7647744,7648256) | 20260507 06:01:03.735 | INFO | MemInfo: { | 20260507 06:01:03.735 | INFO | "gpu" : " 17 GB", | 20260507 06:01:03.735 | INFO | "huge" : "", | 20260507 06:01:03.735 | INFO | "huge_shared" : " 27 GB", | 20260507 06:01:03.735 | INFO | "malloc" : " 14 GB", | 20260507 06:01:03.735 | INFO | "pinned" : " 29 GB" | 20260507 06:01:03.735 | INFO | } | 20260507 06:01:03.735 | INFO | Perf report for the last 5.13197 seconds: | 20260507 06:01:03.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:01:03.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:03.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:03.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:03.735 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 06:01:03.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:03.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:03.735 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 06:01:03.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:03.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:03.735 | INFO | Uploaded 36GiB to GPU in 1.99486s (18.0464GiB/s) | 20260507 06:01:03.735 | INFO | Downloaded 34GiB to GPU in 1.93032s (17.6137GiB/s) | 20260507 06:01:08.880 | INFO | Analyzing chunk frames = [7648256,7648768) | 20260507 06:01:08.880 | INFO | MemInfo: { | 20260507 06:01:08.880 | INFO | "gpu" : " 17 GB", | 20260507 06:01:08.880 | INFO | "huge" : "", | 20260507 06:01:08.880 | INFO | "huge_shared" : " 27 GB", | 20260507 06:01:08.880 | INFO | "malloc" : " 14 GB", | 20260507 06:01:08.880 | INFO | "pinned" : " 29 GB" | 20260507 06:01:08.880 | INFO | } | 20260507 06:01:08.880 | INFO | Perf report for the last 5.1453 seconds: | 20260507 06:01:08.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:01:08.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:08.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:08.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:08.881 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 06:01:08.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:08.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:08.881 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:01:08.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:08.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:08.881 | INFO | Uploaded 36GiB to GPU in 2.04742s (17.5831GiB/s) | 20260507 06:01:08.881 | INFO | Downloaded 34GiB to GPU in 1.95339s (17.4056GiB/s) | 20260507 06:01:14.014 | INFO | Analyzing chunk frames = [7648768,7649280) | 20260507 06:01:14.014 | INFO | MemInfo: { | 20260507 06:01:14.014 | INFO | "gpu" : " 17 GB", | 20260507 06:01:14.014 | INFO | "huge" : "", | 20260507 06:01:14.014 | INFO | "huge_shared" : " 27 GB", | 20260507 06:01:14.014 | INFO | "malloc" : " 14 GB", | 20260507 06:01:14.014 | INFO | "pinned" : " 29 GB" | 20260507 06:01:14.014 | INFO | } | 20260507 06:01:14.014 | INFO | Perf report for the last 5.13394 seconds: | 20260507 06:01:14.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:01:14.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:14.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:14.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:14.014 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 06:01:14.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:14.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:14.014 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:01:14.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:14.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:14.015 | INFO | Uploaded 36GiB to GPU in 2.00979s (17.9123GiB/s) | 20260507 06:01:14.015 | INFO | Downloaded 34GiB to GPU in 1.93376s (17.5823GiB/s) | 20260507 06:01:19.160 | INFO | Analyzing chunk frames = [7649280,7649792) | 20260507 06:01:19.160 | INFO | MemInfo: { | 20260507 06:01:19.160 | INFO | "gpu" : " 17 GB", | 20260507 06:01:19.160 | INFO | "huge" : "", | 20260507 06:01:19.160 | INFO | "huge_shared" : " 27 GB", | 20260507 06:01:19.160 | INFO | "malloc" : " 14 GB", | 20260507 06:01:19.160 | INFO | "pinned" : " 29 GB" | 20260507 06:01:19.160 | INFO | } | 20260507 06:01:19.160 | INFO | Perf report for the last 5.14543 seconds: | 20260507 06:01:19.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:01:19.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:19.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:19.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:19.160 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 06:01:19.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:19.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:19.160 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:01:19.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:19.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:19.160 | INFO | Uploaded 36GiB to GPU in 2.0441s (17.6116GiB/s) | 20260507 06:01:19.160 | INFO | Downloaded 34GiB to GPU in 1.9154s (17.7509GiB/s) | 20260507 06:01:24.299 | INFO | Analyzing chunk frames = [7649792,7650304) | 20260507 06:01:24.299 | INFO | MemInfo: { | 20260507 06:01:24.299 | INFO | "gpu" : " 17 GB", | 20260507 06:01:24.299 | INFO | "huge" : "", | 20260507 06:01:24.299 | INFO | "huge_shared" : " 27 GB", | 20260507 06:01:24.299 | INFO | "malloc" : " 14 GB", | 20260507 06:01:24.299 | INFO | "pinned" : " 29 GB" | 20260507 06:01:24.299 | INFO | } | 20260507 06:01:24.299 | INFO | Perf report for the last 5.13925 seconds: | 20260507 06:01:24.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:01:24.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:24.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:24.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:24.299 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 06:01:24.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:24.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:24.299 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 06:01:24.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:24.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:24.299 | INFO | Uploaded 36GiB to GPU in 2.07226s (17.3723GiB/s) | 20260507 06:01:24.299 | INFO | Downloaded 34GiB to GPU in 1.93402s (17.5799GiB/s) | 20260507 06:01:29.437 | INFO | Analyzing chunk frames = [7650304,7650816) | 20260507 06:01:29.437 | INFO | MemInfo: { | 20260507 06:01:29.437 | INFO | "gpu" : " 17 GB", | 20260507 06:01:29.437 | INFO | "huge" : "", | 20260507 06:01:29.437 | INFO | "huge_shared" : " 27 GB", | 20260507 06:01:29.437 | INFO | "malloc" : " 14 GB", | 20260507 06:01:29.437 | INFO | "pinned" : " 29 GB" | 20260507 06:01:29.437 | INFO | } | 20260507 06:01:29.437 | INFO | Perf report for the last 5.13809 seconds: | 20260507 06:01:29.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:01:29.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:29.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:29.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:29.437 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 06:01:29.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:29.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:29.437 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 06:01:29.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:29.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:29.437 | INFO | Uploaded 36GiB to GPU in 1.9064s (18.8837GiB/s) | 20260507 06:01:29.437 | INFO | Downloaded 34GiB to GPU in 1.96771s (17.2789GiB/s) | 20260507 06:01:34.575 | INFO | Analyzing chunk frames = [7650816,7651328) | 20260507 06:01:34.575 | INFO | MemInfo: { | 20260507 06:01:34.575 | INFO | "gpu" : " 17 GB", | 20260507 06:01:34.575 | INFO | "huge" : "", | 20260507 06:01:34.575 | INFO | "huge_shared" : " 27 GB", | 20260507 06:01:34.575 | INFO | "malloc" : " 14 GB", | 20260507 06:01:34.575 | INFO | "pinned" : " 29 GB" | 20260507 06:01:34.575 | INFO | } | 20260507 06:01:34.575 | INFO | Perf report for the last 5.13768 seconds: | 20260507 06:01:34.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:01:34.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:34.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:34.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:34.575 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:01:34.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:34.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:34.575 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:01:34.575 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:34.575 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:34.575 | INFO | Uploaded 36GiB to GPU in 1.98725s (18.1155GiB/s) | 20260507 06:01:34.575 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:01:34.576 | INFO | Total mem usage: 18 GiB | 20260507 06:01:34.576 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:01:34.576 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 06:01:34.575 | INFO | Downloaded 34GiB to GPU in 1.95862s (17.3592GiB/s) | 20260507 06:01:34.576 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:01:34.576 | INFO | High Water Breakdown: | 20260507 06:01:34.576 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:01:34.576 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:01:34.576 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:01:34.576 | INFO | ------------------------------------------ | 20260507 06:01:34.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:01:34.576 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:01:34.576 | INFO | Total mem usage: 26.5 GiB | 20260507 06:01:34.576 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:01:34.576 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:01:34.576 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 06:01:34.576 | INFO | High Water Breakdown: | 20260507 06:01:34.576 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:01:34.576 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:01:34.576 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:01:34.576 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:01:34.576 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:01:34.576 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:01:34.576 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:01:34.576 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:01:34.576 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:01:34.576 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:01:34.576 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:01:34.576 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:01:34.576 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:01:34.576 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:01:34.576 | INFO | ------------------------------------------ | 20260507 06:01:34.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:01:34.576 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:01:34.576 | INFO | Total mem usage: 25.6 GiB | 20260507 06:01:34.576 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:01:34.576 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:01:34.576 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:01:34.576 | INFO | High Water Breakdown: | 20260507 06:01:34.576 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:01:34.576 | INFO | ------------------------------------------ | 20260507 06:01:34.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:01:34.576 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:01:34.576 | INFO | Total mem usage: 15.7 GiB | 20260507 06:01:34.576 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:01:34.576 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:01:34.576 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:01:34.576 | INFO | High Water Breakdown: | 20260507 06:01:34.576 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:01:34.576 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:01:34.576 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:01:34.576 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:01:34.576 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:01:34.576 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:01:34.576 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:01:34.576 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:01:34.576 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:01:34.576 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:01:34.576 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:01:34.576 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:01:34.576 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:01:34.576 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:01:34.576 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:01:34.576 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:01:34.576 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:01:34.576 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:01:34.576 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:01:34.576 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:01:34.576 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:01:34.576 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:01:34.576 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:01:34.576 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:01:34.576 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:01:34.576 | INFO | ------------------------------------------ | 20260507 06:01:34.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:01:34.576 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:01:34.576 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:01:34.576 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:01:34.576 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.44e+01, 1.33e+01 / 1.44e+00 | 20260507 06:01:34.576 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.02e-01, 9.69e-03 / 4.63e-03 | 20260507 06:01:34.576 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.38e+00, 5.13e-01 / 3.00e-01 | 20260507 06:01:34.576 | INFO | PerfReport: 3.24%, Histogram -- 9.97e-01 / 1.46e+00, 1.71e-01 / 1.09e-01 | 20260507 06:01:34.576 | INFO | PerfReport: 16.17%, DME -- 3.98e+01 / 4.42e+01, 3.86e+00 / 1.61e+00 | 20260507 06:01:34.576 | INFO | PerfReport: 57.73%, FrameLabeling -- 1.77e+01 / 2.00e+01, 1.34e+00 / 1.53e+00 | 20260507 06:01:34.576 | INFO | PerfReport: 6.91%, PulseAccumulating -- 2.13e+00 / 2.66e+00, 3.85e-01 / 2.17e-01 | 20260507 06:01:34.576 | INFO | PerfReport: 2.96%, Metrics -- 9.11e-01 / 1.21e+00, 2.41e-01 / 1.23e-01 | 20260507 06:01:34.576 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.08e-01 / 3.56e-01, 5.09e-02 / 1.38e-02 | 20260507 06:01:34.577 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:01:34.577 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:01:34.577 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:01:34.577 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 06:01:34.577 | INFO | PerfReport: 21.84%, Upload -- 2.10e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260507 06:01:34.577 | INFO | PerfReport: 21.34%, Download -- 2.05e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 06:01:34.577 | INFO | PerfReport: 31.99%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 06:01:34.577 | INFO | PerfReport: 1.57%, LockUpload -- 1.51e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 06:01:34.577 | INFO | PerfReport: 1.20%, LockDownload -- 1.15e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260507 06:01:34.577 | INFO | PerfReport: 22.05%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 06:01:34.577 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260507 06:01:34.672 | INFO | Starting a round of Baz IO | 20260507 06:01:37.772 | INFO | Populating RTMetrics report | 20260507 06:01:37.772 | INFO | RTMetrics report populated | 20260507 06:01:37.772 | INFO | Opening tmp RTMetrics json file | 20260507 06:01:37.788 | INFO | Serializing RTMetrics to json | 20260507 06:01:37.789 | INFO | Serialization done, swapping tmp file now... | 20260507 06:01:37.789 | INFO | RTMetrics done | 20260507 06:01:37.857 | INFO | Finished a round of Baz IO: Wrote 1.678 GiB in 3.184 seconds | 20260507 06:01:37.857 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.66% / 0.00% / 67.34% | 20260507 06:01:37.857 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.130773% / 0% / 7.15256e-05% | 20260507 06:01:37.857 | INFO | Baz IO overall: | 20260507 06:01:37.857 | INFO | BytesWritten : 3868781098413 | 20260507 06:01:37.857 | INFO | NumEvents : 1854720489304 | 20260507 06:01:39.721 | INFO | Analyzing chunk frames = [7651328,7651840) | 20260507 06:01:39.721 | INFO | MemInfo: { | 20260507 06:01:39.721 | INFO | "gpu" : " 17 GB", | 20260507 06:01:39.721 | INFO | "huge" : "", | 20260507 06:01:39.721 | INFO | "huge_shared" : " 27 GB", | 20260507 06:01:39.721 | INFO | "malloc" : " 14 GB", | 20260507 06:01:39.721 | INFO | "pinned" : " 29 GB" | 20260507 06:01:39.721 | INFO | } | 20260507 06:01:39.721 | INFO | Perf report for the last 5.14597 seconds: | 20260507 06:01:39.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:01:39.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:39.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:39.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:39.721 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 06:01:39.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:39.721 | INFO | RT_METRICS : 41.2%, 1.0, 21.7ms, 0 | 20260507 06:01:39.721 | INFO | PRE_HQ : 59.1%, 1.8, 55.0ms, 0 | 20260507 06:01:39.721 | INFO | BAZWRITER : 61.5%, 3.1, 98.8ms, 0 | 20260507 06:01:39.721 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:01:39.721 | INFO | Uploaded 36GiB to GPU in 2.04416s (17.6111GiB/s) | 20260507 06:01:39.721 | INFO | Downloaded 38GiB to GPU in 2.17894s (17.4397GiB/s) | 20260507 06:01:44.851 | INFO | Analyzing chunk frames = [7651840,7652352) | 20260507 06:01:44.851 | INFO | MemInfo: { | 20260507 06:01:44.851 | INFO | "gpu" : " 17 GB", | 20260507 06:01:44.851 | INFO | "huge" : "", | 20260507 06:01:44.851 | INFO | "huge_shared" : " 27 GB", | 20260507 06:01:44.851 | INFO | "malloc" : " 14 GB", | 20260507 06:01:44.851 | INFO | "pinned" : " 29 GB" | 20260507 06:01:44.851 | INFO | } | 20260507 06:01:44.851 | INFO | Perf report for the last 5.13027 seconds: | 20260507 06:01:44.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:01:44.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:44.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:44.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:44.851 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 06:01:44.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:44.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:44.851 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 06:01:44.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:44.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:44.851 | INFO | Uploaded 36GiB to GPU in 1.99868s (18.0119GiB/s) | 20260507 06:01:44.851 | INFO | Downloaded 34GiB to GPU in 1.93174s (17.6007GiB/s) | 20260507 06:01:49.990 | INFO | Analyzing chunk frames = [7652352,7652864) | 20260507 06:01:49.990 | INFO | MemInfo: { | 20260507 06:01:49.990 | INFO | "gpu" : " 17 GB", | 20260507 06:01:49.990 | INFO | "huge" : "", | 20260507 06:01:49.990 | INFO | "huge_shared" : " 27 GB", | 20260507 06:01:49.990 | INFO | "malloc" : " 14 GB", | 20260507 06:01:49.990 | INFO | "pinned" : " 29 GB" | 20260507 06:01:49.990 | INFO | } | 20260507 06:01:49.990 | INFO | Perf report for the last 5.13863 seconds: | 20260507 06:01:49.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:01:49.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:49.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:49.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:49.990 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:01:49.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:49.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:49.990 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:01:49.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:49.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:49.990 | INFO | Uploaded 36GiB to GPU in 2.04211s (17.6288GiB/s) | 20260507 06:01:49.990 | INFO | Downloaded 34GiB to GPU in 1.94931s (17.4421GiB/s) | 20260507 06:01:55.132 | INFO | Analyzing chunk frames = [7652864,7653376) | 20260507 06:01:55.132 | INFO | MemInfo: { | 20260507 06:01:55.132 | INFO | "gpu" : " 17 GB", | 20260507 06:01:55.132 | INFO | "huge" : "", | 20260507 06:01:55.132 | INFO | "huge_shared" : " 27 GB", | 20260507 06:01:55.132 | INFO | "malloc" : " 14 GB", | 20260507 06:01:55.132 | INFO | "pinned" : " 29 GB" | 20260507 06:01:55.132 | INFO | } | 20260507 06:01:55.132 | INFO | Perf report for the last 5.14201 seconds: | 20260507 06:01:55.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:01:55.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:55.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:55.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:55.132 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 06:01:55.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:55.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:55.132 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:01:55.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:55.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:01:55.132 | INFO | Uploaded 36GiB to GPU in 2.00582s (17.9478GiB/s) | 20260507 06:01:55.132 | INFO | Downloaded 34GiB to GPU in 1.93503s (17.5708GiB/s) | 20260507 06:02:00.269 | INFO | Analyzing chunk frames = [7653376,7653888) | 20260507 06:02:00.269 | INFO | MemInfo: { | 20260507 06:02:00.269 | INFO | "gpu" : " 17 GB", | 20260507 06:02:00.269 | INFO | "huge" : "", | 20260507 06:02:00.269 | INFO | "huge_shared" : " 27 GB", | 20260507 06:02:00.269 | INFO | "malloc" : " 14 GB", | 20260507 06:02:00.270 | INFO | "pinned" : " 29 GB" | 20260507 06:02:00.270 | INFO | } | 20260507 06:02:00.270 | INFO | Perf report for the last 5.13779 seconds: | 20260507 06:02:00.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:02:00.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:00.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:00.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:00.270 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 06:02:00.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:00.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:00.270 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:02:00.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:00.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:00.270 | INFO | Uploaded 36GiB to GPU in 2.05449s (17.5226GiB/s) | 20260507 06:02:00.270 | INFO | Downloaded 34GiB to GPU in 1.93867s (17.5378GiB/s) | 20260507 06:02:05.409 | INFO | Analyzing chunk frames = [7653888,7654400) | 20260507 06:02:05.409 | INFO | MemInfo: { | 20260507 06:02:05.409 | INFO | "gpu" : " 17 GB", | 20260507 06:02:05.409 | INFO | "huge" : "", | 20260507 06:02:05.409 | INFO | "huge_shared" : " 27 GB", | 20260507 06:02:05.409 | INFO | "malloc" : " 14 GB", | 20260507 06:02:05.409 | INFO | "pinned" : " 29 GB" | 20260507 06:02:05.409 | INFO | } | 20260507 06:02:05.409 | INFO | Perf report for the last 5.13943 seconds: | 20260507 06:02:05.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:02:05.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:05.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:05.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:05.409 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:02:05.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:05.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:05.409 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 06:02:05.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:05.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:05.409 | INFO | Uploaded 36GiB to GPU in 2.04357s (17.6162GiB/s) | 20260507 06:02:05.409 | INFO | Downloaded 34GiB to GPU in 1.91219s (17.7806GiB/s) | 20260507 06:02:10.549 | INFO | Analyzing chunk frames = [7654400,7654912) | 20260507 06:02:10.549 | INFO | MemInfo: { | 20260507 06:02:10.549 | INFO | "gpu" : " 17 GB", | 20260507 06:02:10.549 | INFO | "huge" : "", | 20260507 06:02:10.549 | INFO | "huge_shared" : " 27 GB", | 20260507 06:02:10.549 | INFO | "malloc" : " 14 GB", | 20260507 06:02:10.549 | INFO | "pinned" : " 29 GB" | 20260507 06:02:10.549 | INFO | } | 20260507 06:02:10.550 | INFO | Perf report for the last 5.14048 seconds: | 20260507 06:02:10.550 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:02:10.550 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:10.550 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:10.550 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:10.550 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:02:10.550 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:10.550 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:10.550 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 06:02:10.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:10.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:10.550 | INFO | Uploaded 36GiB to GPU in 1.92415s (18.7096GiB/s) | 20260507 06:02:10.550 | INFO | Downloaded 34GiB to GPU in 1.97293s (17.2332GiB/s) | 20260507 06:02:15.688 | INFO | Analyzing chunk frames = [7654912,7655424) | 20260507 06:02:15.688 | INFO | MemInfo: { | 20260507 06:02:15.688 | INFO | "gpu" : " 17 GB", | 20260507 06:02:15.688 | INFO | "huge" : "", | 20260507 06:02:15.688 | INFO | "huge_shared" : " 27 GB", | 20260507 06:02:15.688 | INFO | "malloc" : " 14 GB", | 20260507 06:02:15.688 | INFO | "pinned" : " 29 GB" | 20260507 06:02:15.688 | INFO | } | 20260507 06:02:15.688 | INFO | Perf report for the last 5.13833 seconds: | 20260507 06:02:15.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:02:15.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:15.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:15.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:15.688 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:02:15.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:15.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:15.688 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:02:15.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:15.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:15.688 | INFO | Uploaded 36GiB to GPU in 1.98057s (18.1766GiB/s) | 20260507 06:02:15.688 | INFO | Downloaded 34GiB to GPU in 1.95708s (17.3728GiB/s) | 20260507 06:02:15.785 | INFO | Starting a round of Baz IO | 20260507 06:02:18.887 | INFO | Populating RTMetrics report | 20260507 06:02:18.887 | INFO | RTMetrics report populated | 20260507 06:02:18.887 | INFO | Opening tmp RTMetrics json file | 20260507 06:02:18.897 | INFO | Serializing RTMetrics to json | 20260507 06:02:18.903 | INFO | Serialization done, swapping tmp file now... | 20260507 06:02:18.903 | INFO | RTMetrics done | 20260507 06:02:18.969 | INFO | Finished a round of Baz IO: Wrote 1.678 GiB in 3.184 seconds | 20260507 06:02:18.969 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.66% / 0.00% / 67.34% | 20260507 06:02:18.969 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.130546% / 0% / 0.000103315% | 20260507 06:02:18.969 | INFO | Baz IO overall: | 20260507 06:02:18.969 | INFO | BytesWritten : 3870582547299 | 20260507 06:02:18.969 | INFO | NumEvents : 1855565064124 | 20260507 06:02:20.828 | INFO | Analyzing chunk frames = [7655424,7655936) | 20260507 06:02:20.828 | INFO | MemInfo: { | 20260507 06:02:20.828 | INFO | "gpu" : " 17 GB", | 20260507 06:02:20.828 | INFO | "huge" : "", | 20260507 06:02:20.828 | INFO | "huge_shared" : " 27 GB", | 20260507 06:02:20.828 | INFO | "malloc" : " 14 GB", | 20260507 06:02:20.828 | INFO | "pinned" : " 29 GB" | 20260507 06:02:20.828 | INFO | } | 20260507 06:02:20.828 | INFO | Perf report for the last 5.14017 seconds: | 20260507 06:02:20.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:02:20.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:20.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:20.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:20.828 | INFO | ANALYSIS : 61.7%, 3.0, 96.1ms, 0 | 20260507 06:02:20.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:20.828 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:02:20.828 | INFO | PRE_HQ : 59.0%, 1.8, 54.6ms, 0 | 20260507 06:02:20.828 | INFO | BAZWRITER : 61.5%, 3.1, 99.1ms, 0 | 20260507 06:02:20.828 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:02:20.828 | INFO | Uploaded 36GiB to GPU in 2.05429s (17.5243GiB/s) | 20260507 06:02:20.828 | INFO | Downloaded 38GiB to GPU in 2.18411s (17.3984GiB/s) | 20260507 06:02:25.970 | INFO | Analyzing chunk frames = [7655936,7656448) | 20260507 06:02:25.970 | INFO | MemInfo: { | 20260507 06:02:25.970 | INFO | "gpu" : " 17 GB", | 20260507 06:02:25.970 | INFO | "huge" : "", | 20260507 06:02:25.970 | INFO | "huge_shared" : " 27 GB", | 20260507 06:02:25.970 | INFO | "malloc" : " 14 GB", | 20260507 06:02:25.971 | INFO | "pinned" : " 29 GB" | 20260507 06:02:25.971 | INFO | } | 20260507 06:02:25.971 | INFO | Perf report for the last 5.14255 seconds: | 20260507 06:02:25.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:02:25.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:25.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:25.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:25.971 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 06:02:25.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:25.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:25.971 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:02:25.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:25.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:25.971 | INFO | Uploaded 36GiB to GPU in 2.00055s (17.9951GiB/s) | 20260507 06:02:25.971 | INFO | Downloaded 34GiB to GPU in 1.92935s (17.6225GiB/s) | 20260507 06:02:31.112 | INFO | Analyzing chunk frames = [7656448,7656960) | 20260507 06:02:31.112 | INFO | MemInfo: { | 20260507 06:02:31.112 | INFO | "gpu" : " 17 GB", | 20260507 06:02:31.112 | INFO | "huge" : "", | 20260507 06:02:31.112 | INFO | "huge_shared" : " 27 GB", | 20260507 06:02:31.112 | INFO | "malloc" : " 14 GB", | 20260507 06:02:31.112 | INFO | "pinned" : " 29 GB" | 20260507 06:02:31.112 | INFO | } | 20260507 06:02:31.112 | INFO | Perf report for the last 5.14144 seconds: | 20260507 06:02:31.112 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:02:31.112 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:31.112 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:31.112 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:31.112 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 06:02:31.112 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:31.112 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:31.112 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:02:31.112 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:31.112 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:31.112 | INFO | Uploaded 36GiB to GPU in 2.05497s (17.5185GiB/s) | 20260507 06:02:31.112 | INFO | Downloaded 34GiB to GPU in 1.9511s (17.426GiB/s) | 20260507 06:02:36.244 | INFO | Analyzing chunk frames = [7656960,7657472) | 20260507 06:02:36.244 | INFO | MemInfo: { | 20260507 06:02:36.244 | INFO | "gpu" : " 17 GB", | 20260507 06:02:36.244 | INFO | "huge" : "", | 20260507 06:02:36.244 | INFO | "huge_shared" : " 27 GB", | 20260507 06:02:36.244 | INFO | "malloc" : " 14 GB", | 20260507 06:02:36.244 | INFO | "pinned" : " 29 GB" | 20260507 06:02:36.244 | INFO | } | 20260507 06:02:36.244 | INFO | Perf report for the last 5.13173 seconds: | 20260507 06:02:36.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:02:36.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:36.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:36.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:36.244 | INFO | ANALYSIS : 60.3%, 2.9, 93.0ms, 0 | 20260507 06:02:36.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:36.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:36.244 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:02:36.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:36.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:36.244 | INFO | Uploaded 36GiB to GPU in 2.00775s (17.9305GiB/s) | 20260507 06:02:36.244 | INFO | Downloaded 34GiB to GPU in 1.94102s (17.5165GiB/s) | 20260507 06:02:41.385 | INFO | Analyzing chunk frames = [7657472,7657984) | 20260507 06:02:41.385 | INFO | MemInfo: { | 20260507 06:02:41.385 | INFO | "gpu" : " 17 GB", | 20260507 06:02:41.385 | INFO | "huge" : "", | 20260507 06:02:41.385 | INFO | "huge_shared" : " 27 GB", | 20260507 06:02:41.385 | INFO | "malloc" : " 14 GB", | 20260507 06:02:41.385 | INFO | "pinned" : " 29 GB" | 20260507 06:02:41.385 | INFO | } | 20260507 06:02:41.385 | INFO | Perf report for the last 5.14088 seconds: | 20260507 06:02:41.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:02:41.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:41.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:41.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:41.385 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 06:02:41.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:41.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:41.385 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:02:41.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:41.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:41.385 | INFO | Uploaded 36GiB to GPU in 2.05553s (17.5137GiB/s) | 20260507 06:02:41.385 | INFO | Downloaded 34GiB to GPU in 1.94814s (17.4526GiB/s) | 20260507 06:02:46.524 | INFO | Analyzing chunk frames = [7657984,7658496) | 20260507 06:02:46.524 | INFO | MemInfo: { | 20260507 06:02:46.524 | INFO | "gpu" : " 17 GB", | 20260507 06:02:46.524 | INFO | "huge" : "", | 20260507 06:02:46.524 | INFO | "huge_shared" : " 27 GB", | 20260507 06:02:46.524 | INFO | "malloc" : " 14 GB", | 20260507 06:02:46.524 | INFO | "pinned" : " 29 GB" | 20260507 06:02:46.524 | INFO | } | 20260507 06:02:46.524 | INFO | Perf report for the last 5.13939 seconds: | 20260507 06:02:46.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:02:46.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:46.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:46.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:46.524 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 06:02:46.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:46.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:46.524 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 06:02:46.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:46.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:46.524 | INFO | Uploaded 36GiB to GPU in 2.05169s (17.5465GiB/s) | 20260507 06:02:46.524 | INFO | Downloaded 34GiB to GPU in 1.91731s (17.7331GiB/s) | 20260507 06:02:51.671 | INFO | Analyzing chunk frames = [7658496,7659008) | 20260507 06:02:51.671 | INFO | MemInfo: { | 20260507 06:02:51.671 | INFO | "gpu" : " 17 GB", | 20260507 06:02:51.671 | INFO | "huge" : "", | 20260507 06:02:51.671 | INFO | "huge_shared" : " 27 GB", | 20260507 06:02:51.671 | INFO | "malloc" : " 14 GB", | 20260507 06:02:51.671 | INFO | "pinned" : " 29 GB" | 20260507 06:02:51.671 | INFO | } | 20260507 06:02:51.671 | INFO | Perf report for the last 5.14705 seconds: | 20260507 06:02:51.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:02:51.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:51.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:51.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:51.671 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 06:02:51.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:51.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:51.671 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 06:02:51.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:51.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:51.671 | INFO | Uploaded 36GiB to GPU in 2.05089s (17.5533GiB/s) | 20260507 06:02:51.671 | INFO | Downloaded 34GiB to GPU in 1.9741s (17.223GiB/s) | 20260507 06:02:56.800 | INFO | Analyzing chunk frames = [7659008,7659520) | 20260507 06:02:56.800 | INFO | MemInfo: { | 20260507 06:02:56.800 | INFO | "gpu" : " 17 GB", | 20260507 06:02:56.800 | INFO | "huge" : "", | 20260507 06:02:56.800 | INFO | "huge_shared" : " 27 GB", | 20260507 06:02:56.800 | INFO | "malloc" : " 14 GB", | 20260507 06:02:56.800 | INFO | "pinned" : " 29 GB" | 20260507 06:02:56.800 | INFO | } | 20260507 06:02:56.800 | INFO | Perf report for the last 5.12897 seconds: | 20260507 06:02:56.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:02:56.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:56.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:56.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:56.800 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 06:02:56.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:56.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:56.800 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 06:02:56.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:56.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:02:56.800 | INFO | Uploaded 36GiB to GPU in 1.99496s (18.0454GiB/s) | 20260507 06:02:56.800 | INFO | Downloaded 34GiB to GPU in 1.95779s (17.3665GiB/s) | 20260507 06:02:56.801 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:02:56.801 | INFO | Total mem usage: 18 GiB | 20260507 06:02:56.801 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:02:56.801 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:02:56.801 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:02:56.801 | INFO | High Water Breakdown: | 20260507 06:02:56.801 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:02:56.801 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:02:56.801 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:02:56.801 | INFO | ------------------------------------------ | 20260507 06:02:56.801 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:02:56.801 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:02:56.801 | INFO | Total mem usage: 26.5 GiB | 20260507 06:02:56.801 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:02:56.801 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:02:56.801 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 06:02:56.801 | INFO | High Water Breakdown: | 20260507 06:02:56.801 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:02:56.801 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:02:56.801 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:02:56.801 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:02:56.801 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:02:56.801 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:02:56.801 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:02:56.801 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:02:56.801 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:02:56.801 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:02:56.801 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:02:56.801 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:02:56.801 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:02:56.801 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:02:56.801 | INFO | ------------------------------------------ | 20260507 06:02:56.801 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:02:56.801 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:02:56.801 | INFO | Total mem usage: 25.6 GiB | 20260507 06:02:56.801 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:02:56.801 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:02:56.801 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:02:56.801 | INFO | High Water Breakdown: | 20260507 06:02:56.801 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:02:56.801 | INFO | ------------------------------------------ | 20260507 06:02:56.801 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:02:56.801 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:02:56.801 | INFO | Total mem usage: 15.7 GiB | 20260507 06:02:56.801 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:02:56.801 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:02:56.801 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:02:56.801 | INFO | High Water Breakdown: | 20260507 06:02:56.801 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:02:56.801 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:02:56.801 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:02:56.801 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:02:56.801 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:02:56.801 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:02:56.801 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:02:56.801 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:02:56.801 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:02:56.801 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:02:56.801 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:02:56.801 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:02:56.801 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:02:56.801 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:02:56.801 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:02:56.801 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:02:56.801 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:02:56.801 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:02:56.801 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:02:56.801 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:02:56.801 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:02:56.801 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:02:56.801 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:02:56.801 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:02:56.801 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:02:56.801 | INFO | ------------------------------------------ | 20260507 06:02:56.801 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:02:56.801 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:02:56.801 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:02:56.801 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:02:56.801 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.34e+01 / 1.02e+00 | 20260507 06:02:56.801 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.74e-03 / 4.72e-03 | 20260507 06:02:56.801 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.45e+00, 5.09e-01 / 2.45e-01 | 20260507 06:02:56.801 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.48e+00, 1.65e-01 / 1.03e-01 | 20260507 06:02:56.801 | INFO | PerfReport: 16.27%, DME -- 4.00e+01 / 4.49e+01, 4.02e+00 / 1.45e+00 | 20260507 06:02:56.801 | INFO | PerfReport: 57.62%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.24e+00 / 1.38e+00 | 20260507 06:02:56.801 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.73e+00, 3.86e-01 / 1.79e-01 | 20260507 06:02:56.801 | INFO | PerfReport: 2.97%, Metrics -- 9.11e-01 / 1.22e+00, 2.42e-01 / 9.26e-02 | 20260507 06:02:56.801 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.07e-01 / 3.64e-01, 5.02e-02 / 2.24e-02 | 20260507 06:02:56.801 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:02:56.801 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:02:56.801 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:02:56.801 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 06:02:56.801 | INFO | PerfReport: 21.90%, Upload -- 2.11e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260507 06:02:56.801 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 06:02:56.802 | INFO | PerfReport: 31.90%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:02:56.802 | INFO | PerfReport: 1.59%, LockUpload -- 1.53e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 06:02:56.802 | INFO | PerfReport: 1.21%, LockDownload -- 1.16e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260507 06:02:56.802 | INFO | PerfReport: 22.02%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 06:02:56.802 | INFO | PerfReport: 0.02%, Misc -- 1.45e-02 / 0.00e+00, 3.14e-02 / 0.00e+00 | 20260507 06:02:56.897 | INFO | Starting a round of Baz IO | 20260507 06:02:59.995 | INFO | Populating RTMetrics report | 20260507 06:02:59.995 | INFO | RTMetrics report populated | 20260507 06:02:59.995 | INFO | Opening tmp RTMetrics json file | 20260507 06:03:00.011 | INFO | Serializing RTMetrics to json | 20260507 06:03:00.012 | INFO | Serialization done, swapping tmp file now... | 20260507 06:03:00.012 | INFO | RTMetrics done | 20260507 06:03:00.082 | INFO | Finished a round of Baz IO: Wrote 1.68 GiB in 3.184 seconds | 20260507 06:03:00.082 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.66% / 0.00% / 67.34% | 20260507 06:03:00.082 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.130423% / 0% / 9.13938e-05% | 20260507 06:03:00.082 | INFO | Baz IO overall: | 20260507 06:03:00.082 | INFO | BytesWritten : 3872386957269 | 20260507 06:03:00.082 | INFO | NumEvents : 1856411771055 | 20260507 06:03:01.939 | INFO | Analyzing chunk frames = [7659520,7660032) | 20260507 06:03:01.939 | INFO | MemInfo: { | 20260507 06:03:01.939 | INFO | "gpu" : " 17 GB", | 20260507 06:03:01.939 | INFO | "huge" : "", | 20260507 06:03:01.939 | INFO | "huge_shared" : " 27 GB", | 20260507 06:03:01.939 | INFO | "malloc" : " 14 GB", | 20260507 06:03:01.939 | INFO | "pinned" : " 29 GB" | 20260507 06:03:01.939 | INFO | } | 20260507 06:03:01.939 | INFO | Perf report for the last 5.1392 seconds: | 20260507 06:03:01.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:03:01.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:01.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:01.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:01.940 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 06:03:01.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:01.940 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 06:03:01.940 | INFO | PRE_HQ : 59.2%, 1.8, 55.0ms, 0 | 20260507 06:03:01.940 | INFO | BAZWRITER : 61.6%, 3.2, 101.4ms, 0 | 20260507 06:03:01.940 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:03:01.940 | INFO | Uploaded 36GiB to GPU in 2.04105s (17.638GiB/s) | 20260507 06:03:01.940 | INFO | Downloaded 38GiB to GPU in 2.18121s (17.4215GiB/s) | 20260507 06:03:07.079 | INFO | Analyzing chunk frames = [7660032,7660544) | 20260507 06:03:07.079 | INFO | MemInfo: { | 20260507 06:03:07.079 | INFO | "gpu" : " 17 GB", | 20260507 06:03:07.079 | INFO | "huge" : "", | 20260507 06:03:07.079 | INFO | "huge_shared" : " 27 GB", | 20260507 06:03:07.080 | INFO | "malloc" : " 14 GB", | 20260507 06:03:07.080 | INFO | "pinned" : " 29 GB" | 20260507 06:03:07.080 | INFO | } | 20260507 06:03:07.080 | INFO | Perf report for the last 5.14024 seconds: | 20260507 06:03:07.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:03:07.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:07.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:07.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:07.080 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 06:03:07.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:07.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:07.080 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:03:07.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:07.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:07.080 | INFO | Uploaded 36GiB to GPU in 2.00594s (17.9467GiB/s) | 20260507 06:03:07.080 | INFO | Downloaded 34GiB to GPU in 1.93284s (17.5907GiB/s) | 20260507 06:03:12.224 | INFO | Analyzing chunk frames = [7660544,7661056) | 20260507 06:03:12.224 | INFO | MemInfo: { | 20260507 06:03:12.224 | INFO | "gpu" : " 17 GB", | 20260507 06:03:12.224 | INFO | "huge" : "", | 20260507 06:03:12.224 | INFO | "huge_shared" : " 27 GB", | 20260507 06:03:12.224 | INFO | "malloc" : " 14 GB", | 20260507 06:03:12.224 | INFO | "pinned" : " 29 GB" | 20260507 06:03:12.224 | INFO | } | 20260507 06:03:12.224 | INFO | Perf report for the last 5.14455 seconds: | 20260507 06:03:12.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:03:12.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:12.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:12.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:12.224 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 06:03:12.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:12.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:12.224 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:03:12.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:12.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:12.224 | INFO | Uploaded 36GiB to GPU in 2.04708s (17.586GiB/s) | 20260507 06:03:12.224 | INFO | Downloaded 34GiB to GPU in 1.95355s (17.4043GiB/s) | 20260507 06:03:17.358 | INFO | Analyzing chunk frames = [7661056,7661568) | 20260507 06:03:17.358 | INFO | MemInfo: { | 20260507 06:03:17.358 | INFO | "gpu" : " 17 GB", | 20260507 06:03:17.358 | INFO | "huge" : "", | 20260507 06:03:17.358 | INFO | "huge_shared" : " 27 GB", | 20260507 06:03:17.358 | INFO | "malloc" : " 14 GB", | 20260507 06:03:17.358 | INFO | "pinned" : " 29 GB" | 20260507 06:03:17.358 | INFO | } | 20260507 06:03:17.358 | INFO | Perf report for the last 5.13415 seconds: | 20260507 06:03:17.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:03:17.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:17.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:17.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:17.358 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 06:03:17.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:17.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:17.358 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:03:17.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:17.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:17.359 | INFO | Uploaded 36GiB to GPU in 2.03604s (17.6814GiB/s) | 20260507 06:03:17.359 | INFO | Downloaded 34GiB to GPU in 1.94779s (17.4557GiB/s) | 20260507 06:03:22.494 | INFO | Analyzing chunk frames = [7661568,7662080) | 20260507 06:03:22.494 | INFO | MemInfo: { | 20260507 06:03:22.494 | INFO | "gpu" : " 17 GB", | 20260507 06:03:22.494 | INFO | "huge" : "", | 20260507 06:03:22.494 | INFO | "huge_shared" : " 27 GB", | 20260507 06:03:22.494 | INFO | "malloc" : " 14 GB", | 20260507 06:03:22.494 | INFO | "pinned" : " 29 GB" | 20260507 06:03:22.494 | INFO | } | 20260507 06:03:22.494 | INFO | Perf report for the last 5.13597 seconds: | 20260507 06:03:22.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:03:22.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:22.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:22.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:22.494 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 06:03:22.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:22.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:22.494 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:03:22.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:22.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:22.495 | INFO | Uploaded 36GiB to GPU in 2.05715s (17.4999GiB/s) | 20260507 06:03:22.495 | INFO | Downloaded 34GiB to GPU in 1.94924s (17.4427GiB/s) | 20260507 06:03:27.643 | INFO | Analyzing chunk frames = [7662080,7662592) | 20260507 06:03:27.643 | INFO | MemInfo: { | 20260507 06:03:27.643 | INFO | "gpu" : " 17 GB", | 20260507 06:03:27.643 | INFO | "huge" : "", | 20260507 06:03:27.643 | INFO | "huge_shared" : " 27 GB", | 20260507 06:03:27.643 | INFO | "malloc" : " 14 GB", | 20260507 06:03:27.643 | INFO | "pinned" : " 29 GB" | 20260507 06:03:27.643 | INFO | } | 20260507 06:03:27.643 | INFO | Perf report for the last 5.14852 seconds: | 20260507 06:03:27.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:03:27.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:27.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:27.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:27.643 | INFO | ANALYSIS : 60.3%, 3.0, 94.3ms, 0 | 20260507 06:03:27.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:27.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:27.643 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 06:03:27.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:27.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:27.643 | INFO | Uploaded 36GiB to GPU in 2.04685s (17.588GiB/s) | 20260507 06:03:27.643 | INFO | Downloaded 34GiB to GPU in 1.91428s (17.7613GiB/s) | 20260507 06:03:32.785 | INFO | Analyzing chunk frames = [7662592,7663104) | 20260507 06:03:32.785 | INFO | MemInfo: { | 20260507 06:03:32.785 | INFO | "gpu" : " 17 GB", | 20260507 06:03:32.785 | INFO | "huge" : "", | 20260507 06:03:32.785 | INFO | "huge_shared" : " 27 GB", | 20260507 06:03:32.785 | INFO | "malloc" : " 14 GB", | 20260507 06:03:32.785 | INFO | "pinned" : " 29 GB" | 20260507 06:03:32.785 | INFO | } | 20260507 06:03:32.785 | INFO | Perf report for the last 5.14257 seconds: | 20260507 06:03:32.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:03:32.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:32.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:32.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:32.785 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 06:03:32.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:32.786 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:32.786 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 06:03:32.786 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:32.786 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:32.786 | INFO | Uploaded 36GiB to GPU in 1.99736s (18.0238GiB/s) | 20260507 06:03:32.786 | INFO | Downloaded 34GiB to GPU in 1.97089s (17.2511GiB/s) | 20260507 06:03:37.925 | INFO | Analyzing chunk frames = [7663104,7663616) | 20260507 06:03:37.925 | INFO | MemInfo: { | 20260507 06:03:37.925 | INFO | "gpu" : " 17 GB", | 20260507 06:03:37.925 | INFO | "huge" : "", | 20260507 06:03:37.925 | INFO | "huge_shared" : " 27 GB", | 20260507 06:03:37.925 | INFO | "malloc" : " 14 GB", | 20260507 06:03:37.925 | INFO | "pinned" : " 29 GB" | 20260507 06:03:37.925 | INFO | } | 20260507 06:03:37.925 | INFO | Perf report for the last 5.13922 seconds: | 20260507 06:03:37.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:03:37.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:37.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:37.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:37.925 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:03:37.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:37.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:37.925 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:03:37.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:37.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:37.925 | INFO | Uploaded 36GiB to GPU in 1.99281s (18.065GiB/s) | 20260507 06:03:37.925 | INFO | Downloaded 34GiB to GPU in 1.95699s (17.3736GiB/s) | 20260507 06:03:38.022 | INFO | Starting a round of Baz IO | 20260507 06:03:41.119 | INFO | Populating RTMetrics report | 20260507 06:03:41.120 | INFO | RTMetrics report populated | 20260507 06:03:41.120 | INFO | Opening tmp RTMetrics json file | 20260507 06:03:41.128 | INFO | Serializing RTMetrics to json | 20260507 06:03:41.136 | INFO | Serialization done, swapping tmp file now... | 20260507 06:03:41.136 | INFO | RTMetrics done | 20260507 06:03:41.209 | INFO | Finished a round of Baz IO: Wrote 1.678 GiB in 3.187 seconds | 20260507 06:03:41.209 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.66% / 0.00% / 67.34% | 20260507 06:03:41.209 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.130447% / 0% / 8.74201e-05% | 20260507 06:03:41.209 | INFO | Baz IO overall: | 20260507 06:03:41.209 | INFO | BytesWritten : 3874188389956 | 20260507 06:03:41.209 | INFO | NumEvents : 1857256201179 | 20260507 06:03:43.061 | INFO | Analyzing chunk frames = [7663616,7664128) | 20260507 06:03:43.061 | INFO | MemInfo: { | 20260507 06:03:43.061 | INFO | "gpu" : " 17 GB", | 20260507 06:03:43.061 | INFO | "huge" : "", | 20260507 06:03:43.061 | INFO | "huge_shared" : " 27 GB", | 20260507 06:03:43.061 | INFO | "malloc" : " 14 GB", | 20260507 06:03:43.061 | INFO | "pinned" : " 29 GB" | 20260507 06:03:43.061 | INFO | } | 20260507 06:03:43.061 | INFO | Perf report for the last 5.13617 seconds: | 20260507 06:03:43.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:03:43.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:43.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:43.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:43.061 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 06:03:43.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:43.061 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 06:03:43.061 | INFO | PRE_HQ : 59.0%, 1.8, 54.6ms, 0 | 20260507 06:03:43.061 | INFO | BAZWRITER : 61.6%, 3.2, 100.6ms, 0 | 20260507 06:03:43.061 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:03:43.061 | INFO | Uploaded 36GiB to GPU in 2.07301s (17.3661GiB/s) | 20260507 06:03:43.061 | INFO | Downloaded 38GiB to GPU in 2.18067s (17.4258GiB/s) | 20260507 06:03:48.199 | INFO | Analyzing chunk frames = [7664128,7664640) | 20260507 06:03:48.199 | INFO | MemInfo: { | 20260507 06:03:48.199 | INFO | "gpu" : " 17 GB", | 20260507 06:03:48.199 | INFO | "huge" : "", | 20260507 06:03:48.199 | INFO | "huge_shared" : " 27 GB", | 20260507 06:03:48.199 | INFO | "malloc" : " 14 GB", | 20260507 06:03:48.200 | INFO | "pinned" : " 29 GB" | 20260507 06:03:48.200 | INFO | } | 20260507 06:03:48.200 | INFO | Perf report for the last 5.1387 seconds: | 20260507 06:03:48.200 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:03:48.200 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:48.200 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:48.200 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:48.200 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:03:48.200 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:48.200 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:48.200 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:03:48.200 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:48.200 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:48.200 | INFO | Uploaded 36GiB to GPU in 1.99152s (18.0766GiB/s) | 20260507 06:03:48.200 | INFO | Downloaded 34GiB to GPU in 1.92645s (17.6491GiB/s) | 20260507 06:03:53.334 | INFO | Analyzing chunk frames = [7664640,7665152) | 20260507 06:03:53.334 | INFO | MemInfo: { | 20260507 06:03:53.334 | INFO | "gpu" : " 17 GB", | 20260507 06:03:53.334 | INFO | "huge" : "", | 20260507 06:03:53.334 | INFO | "huge_shared" : " 27 GB", | 20260507 06:03:53.334 | INFO | "malloc" : " 14 GB", | 20260507 06:03:53.334 | INFO | "pinned" : " 29 GB" | 20260507 06:03:53.334 | INFO | } | 20260507 06:03:53.334 | INFO | Perf report for the last 5.13406 seconds: | 20260507 06:03:53.334 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:03:53.334 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:53.334 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:53.334 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:53.334 | INFO | ANALYSIS : 60.7%, 3.0, 93.8ms, 0 | 20260507 06:03:53.334 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:53.334 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:53.334 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:03:53.334 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:53.334 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:53.334 | INFO | Uploaded 36GiB to GPU in 2.05135s (17.5494GiB/s) | 20260507 06:03:53.334 | INFO | Downloaded 34GiB to GPU in 1.95039s (17.4324GiB/s) | 20260507 06:03:58.479 | INFO | Analyzing chunk frames = [7665152,7665664) | 20260507 06:03:58.479 | INFO | MemInfo: { | 20260507 06:03:58.479 | INFO | "gpu" : " 17 GB", | 20260507 06:03:58.480 | INFO | "huge" : "", | 20260507 06:03:58.480 | INFO | "huge_shared" : " 27 GB", | 20260507 06:03:58.480 | INFO | "malloc" : " 14 GB", | 20260507 06:03:58.480 | INFO | "pinned" : " 29 GB" | 20260507 06:03:58.480 | INFO | } | 20260507 06:03:58.480 | INFO | Perf report for the last 5.14603 seconds: | 20260507 06:03:58.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:03:58.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:58.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:58.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:58.480 | INFO | ANALYSIS : 59.4%, 3.0, 92.3ms, 0 | 20260507 06:03:58.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:58.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:58.480 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:03:58.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:58.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:03:58.480 | INFO | Uploaded 36GiB to GPU in 1.9841s (18.1442GiB/s) | 20260507 06:03:58.480 | INFO | Downloaded 34GiB to GPU in 1.91393s (17.7645GiB/s) | 20260507 06:04:03.617 | INFO | Analyzing chunk frames = [7665664,7666176) | 20260507 06:04:03.617 | INFO | MemInfo: { | 20260507 06:04:03.617 | INFO | "gpu" : " 17 GB", | 20260507 06:04:03.617 | INFO | "huge" : "", | 20260507 06:04:03.617 | INFO | "huge_shared" : " 27 GB", | 20260507 06:04:03.617 | INFO | "malloc" : " 14 GB", | 20260507 06:04:03.617 | INFO | "pinned" : " 29 GB" | 20260507 06:04:03.617 | INFO | } | 20260507 06:04:03.617 | INFO | Perf report for the last 5.1374 seconds: | 20260507 06:04:03.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:04:03.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:03.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:03.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:03.617 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 06:04:03.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:03.617 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:03.617 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 06:04:03.617 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:03.617 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:03.617 | INFO | Uploaded 36GiB to GPU in 2.0452s (17.6022GiB/s) | 20260507 06:04:03.617 | INFO | Downloaded 34GiB to GPU in 1.90786s (17.821GiB/s) | 20260507 06:04:08.756 | INFO | Analyzing chunk frames = [7666176,7666688) | 20260507 06:04:08.756 | INFO | MemInfo: { | 20260507 06:04:08.756 | INFO | "gpu" : " 17 GB", | 20260507 06:04:08.756 | INFO | "huge" : "", | 20260507 06:04:08.756 | INFO | "huge_shared" : " 27 GB", | 20260507 06:04:08.756 | INFO | "malloc" : " 14 GB", | 20260507 06:04:08.756 | INFO | "pinned" : " 29 GB" | 20260507 06:04:08.756 | INFO | } | 20260507 06:04:08.756 | INFO | Perf report for the last 5.13902 seconds: | 20260507 06:04:08.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:04:08.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:08.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:08.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:08.756 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:04:08.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:08.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:08.756 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 06:04:08.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:08.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:08.756 | INFO | Uploaded 36GiB to GPU in 2.04921s (17.5678GiB/s) | 20260507 06:04:08.756 | INFO | Downloaded 34GiB to GPU in 1.91307s (17.7725GiB/s) | 20260507 06:04:13.895 | INFO | Analyzing chunk frames = [7666688,7667200) | 20260507 06:04:13.896 | INFO | MemInfo: { | 20260507 06:04:13.896 | INFO | "gpu" : " 17 GB", | 20260507 06:04:13.896 | INFO | "huge" : "", | 20260507 06:04:13.896 | INFO | "huge_shared" : " 27 GB", | 20260507 06:04:13.896 | INFO | "malloc" : " 14 GB", | 20260507 06:04:13.896 | INFO | "pinned" : " 29 GB" | 20260507 06:04:13.896 | INFO | } | 20260507 06:04:13.896 | INFO | Perf report for the last 5.13956 seconds: | 20260507 06:04:13.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:04:13.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:13.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:13.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:13.896 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:04:13.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:13.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:13.896 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 06:04:13.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:13.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:13.896 | INFO | Uploaded 36GiB to GPU in 1.92863s (18.6661GiB/s) | 20260507 06:04:13.896 | INFO | Downloaded 34GiB to GPU in 1.97241s (17.2378GiB/s) | 20260507 06:04:19.034 | INFO | Analyzing chunk frames = [7667200,7667712) | 20260507 06:04:19.034 | INFO | MemInfo: { | 20260507 06:04:19.034 | INFO | "gpu" : " 17 GB", | 20260507 06:04:19.034 | INFO | "huge" : "", | 20260507 06:04:19.034 | INFO | "huge_shared" : " 27 GB", | 20260507 06:04:19.034 | INFO | "malloc" : " 14 GB", | 20260507 06:04:19.034 | INFO | "pinned" : " 29 GB" | 20260507 06:04:19.034 | INFO | } | 20260507 06:04:19.034 | INFO | Perf report for the last 5.13853 seconds: | 20260507 06:04:19.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:04:19.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:19.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:19.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:19.034 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:04:19.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:19.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:19.034 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 06:04:19.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:19.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:19.034 | INFO | Uploaded 36GiB to GPU in 1.95945s (18.3725GiB/s) | 20260507 06:04:19.035 | INFO | Downloaded 34GiB to GPU in 1.95597s (17.3827GiB/s) | 20260507 06:04:19.035 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:04:19.035 | INFO | Total mem usage: 18 GiB | 20260507 06:04:19.035 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:04:19.035 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 06:04:19.035 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:04:19.035 | INFO | High Water Breakdown: | 20260507 06:04:19.035 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:04:19.035 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:04:19.035 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:04:19.035 | INFO | ------------------------------------------ | 20260507 06:04:19.035 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:04:19.035 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:04:19.035 | INFO | Total mem usage: 26.5 GiB | 20260507 06:04:19.035 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:04:19.035 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:04:19.035 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:04:19.035 | INFO | High Water Breakdown: | 20260507 06:04:19.035 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:04:19.035 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:04:19.035 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:04:19.035 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:04:19.035 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:04:19.035 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:04:19.035 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:04:19.035 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:04:19.035 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:04:19.035 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:04:19.035 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:04:19.035 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:04:19.035 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:04:19.035 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:04:19.035 | INFO | ------------------------------------------ | 20260507 06:04:19.035 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:04:19.035 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:04:19.035 | INFO | Total mem usage: 25.6 GiB | 20260507 06:04:19.035 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:04:19.035 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:04:19.035 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:04:19.035 | INFO | High Water Breakdown: | 20260507 06:04:19.035 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:04:19.035 | INFO | ------------------------------------------ | 20260507 06:04:19.035 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:04:19.035 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:04:19.035 | INFO | Total mem usage: 15.7 GiB | 20260507 06:04:19.035 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:04:19.035 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:04:19.035 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 06:04:19.035 | INFO | High Water Breakdown: | 20260507 06:04:19.035 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:04:19.035 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:04:19.035 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:04:19.035 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:04:19.035 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:04:19.035 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:04:19.035 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:04:19.035 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:04:19.035 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:04:19.035 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:04:19.035 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:04:19.035 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:04:19.035 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:04:19.035 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:04:19.035 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:04:19.035 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:04:19.035 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:04:19.035 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:04:19.035 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:04:19.035 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:04:19.035 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:04:19.035 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:04:19.035 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:04:19.035 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:04:19.035 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:04:19.035 | INFO | ------------------------------------------ | 20260507 06:04:19.035 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:04:19.036 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:04:19.036 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:04:19.036 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:04:19.036 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.34e+01 / 1.42e+00 | 20260507 06:04:19.036 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.59e-03 / 6.57e-03 | 20260507 06:04:19.036 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.45e+00, 5.12e-01 / 2.45e-01 | 20260507 06:04:19.036 | INFO | PerfReport: 3.22%, Histogram -- 9.90e-01 / 1.48e+00, 1.67e-01 / 1.21e-01 | 20260507 06:04:19.036 | INFO | PerfReport: 16.28%, DME -- 4.00e+01 / 4.51e+01, 4.01e+00 / 1.75e+00 | 20260507 06:04:19.036 | INFO | PerfReport: 57.62%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.25e+00 / 1.00e+00 | 20260507 06:04:19.036 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.13e+00 / 2.72e+00, 3.86e-01 / 1.74e-01 | 20260507 06:04:19.036 | INFO | PerfReport: 2.97%, Metrics -- 9.14e-01 / 1.23e+00, 2.46e-01 / 1.19e-01 | 20260507 06:04:19.036 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.07e-01 / 3.64e-01, 5.09e-02 / 2.06e-02 | 20260507 06:04:19.036 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:04:19.036 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:04:19.036 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:04:19.036 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 06:04:19.036 | INFO | PerfReport: 21.86%, Upload -- 2.10e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260507 06:04:19.036 | INFO | PerfReport: 21.33%, Download -- 2.05e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 06:04:19.036 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:04:19.036 | INFO | PerfReport: 1.59%, LockUpload -- 1.53e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260507 06:04:19.036 | INFO | PerfReport: 1.20%, LockDownload -- 1.16e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260507 06:04:19.036 | INFO | PerfReport: 22.06%, LockCompute -- 2.12e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 06:04:19.036 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.08e-02 / 0.00e+00 | 20260507 06:04:19.131 | INFO | Starting a round of Baz IO | 20260507 06:04:22.227 | INFO | Populating RTMetrics report | 20260507 06:04:22.227 | INFO | RTMetrics report populated | 20260507 06:04:22.227 | INFO | Opening tmp RTMetrics json file | 20260507 06:04:22.243 | INFO | Serializing RTMetrics to json | 20260507 06:04:22.244 | INFO | Serialization done, swapping tmp file now... | 20260507 06:04:22.244 | INFO | RTMetrics done | 20260507 06:04:22.319 | INFO | Finished a round of Baz IO: Wrote 1.677 GiB in 3.188 seconds | 20260507 06:04:22.319 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.66% / 0.00% / 67.34% | 20260507 06:04:22.319 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.130097% / 3.97364e-06% / 6.35783e-05% | 20260507 06:04:22.319 | INFO | Baz IO overall: | 20260507 06:04:22.319 | INFO | BytesWritten : 3875988817336 | 20260507 06:04:22.319 | INFO | NumEvents : 1858100147030 | 20260507 06:04:24.173 | INFO | Analyzing chunk frames = [7667712,7668224) | 20260507 06:04:24.173 | INFO | MemInfo: { | 20260507 06:04:24.173 | INFO | "gpu" : " 17 GB", | 20260507 06:04:24.173 | INFO | "huge" : "", | 20260507 06:04:24.173 | INFO | "huge_shared" : " 27 GB", | 20260507 06:04:24.173 | INFO | "malloc" : " 14 GB", | 20260507 06:04:24.173 | INFO | "pinned" : " 29 GB" | 20260507 06:04:24.173 | INFO | } | 20260507 06:04:24.173 | INFO | Perf report for the last 5.13866 seconds: | 20260507 06:04:24.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:04:24.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:24.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:24.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:24.173 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 06:04:24.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:24.173 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260507 06:04:24.173 | INFO | PRE_HQ : 60.1%, 1.9, 59.3ms, 0 | 20260507 06:04:24.173 | INFO | BAZWRITER : 61.6%, 3.4, 106.2ms, 0 | 20260507 06:04:24.173 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 06:04:24.173 | INFO | Uploaded 36GiB to GPU in 2.04187s (17.6309GiB/s) | 20260507 06:04:24.173 | INFO | Downloaded 38GiB to GPU in 2.17477s (17.4731GiB/s) | 20260507 06:04:29.307 | INFO | Analyzing chunk frames = [7668224,7668736) | 20260507 06:04:29.307 | INFO | MemInfo: { | 20260507 06:04:29.307 | INFO | "gpu" : " 17 GB", | 20260507 06:04:29.307 | INFO | "huge" : "", | 20260507 06:04:29.307 | INFO | "huge_shared" : " 27 GB", | 20260507 06:04:29.307 | INFO | "malloc" : " 14 GB", | 20260507 06:04:29.307 | INFO | "pinned" : " 29 GB" | 20260507 06:04:29.307 | INFO | } | 20260507 06:04:29.307 | INFO | Perf report for the last 5.13418 seconds: | 20260507 06:04:29.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:04:29.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:29.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:29.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:29.307 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 06:04:29.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:29.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:29.307 | INFO | PRE_HQ : 26.7%, 1.0, 13.8ms, 0 | 20260507 06:04:29.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:29.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:29.307 | INFO | Uploaded 36GiB to GPU in 1.9902s (18.0886GiB/s) | 20260507 06:04:29.307 | INFO | Downloaded 34GiB to GPU in 1.93088s (17.6085GiB/s) | 20260507 06:04:34.449 | INFO | Analyzing chunk frames = [7668736,7669248) | 20260507 06:04:34.449 | INFO | MemInfo: { | 20260507 06:04:34.449 | INFO | "gpu" : " 17 GB", | 20260507 06:04:34.449 | INFO | "huge" : "", | 20260507 06:04:34.449 | INFO | "huge_shared" : " 27 GB", | 20260507 06:04:34.449 | INFO | "malloc" : " 14 GB", | 20260507 06:04:34.449 | INFO | "pinned" : " 29 GB" | 20260507 06:04:34.449 | INFO | } | 20260507 06:04:34.449 | INFO | Perf report for the last 5.14232 seconds: | 20260507 06:04:34.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:04:34.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:34.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:34.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:34.449 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 06:04:34.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:34.450 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:34.450 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 06:04:34.450 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:34.450 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:34.450 | INFO | Uploaded 36GiB to GPU in 2.04718s (17.5852GiB/s) | 20260507 06:04:34.450 | INFO | Downloaded 34GiB to GPU in 1.9534s (17.4056GiB/s) | 20260507 06:04:39.589 | INFO | Analyzing chunk frames = [7669248,7669760) | 20260507 06:04:39.589 | INFO | MemInfo: { | 20260507 06:04:39.589 | INFO | "gpu" : " 17 GB", | 20260507 06:04:39.589 | INFO | "huge" : "", | 20260507 06:04:39.589 | INFO | "huge_shared" : " 27 GB", | 20260507 06:04:39.589 | INFO | "malloc" : " 14 GB", | 20260507 06:04:39.589 | INFO | "pinned" : " 29 GB" | 20260507 06:04:39.589 | INFO | } | 20260507 06:04:39.589 | INFO | Perf report for the last 5.13989 seconds: | 20260507 06:04:39.589 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:04:39.589 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:39.589 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:39.589 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:39.589 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 06:04:39.589 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:39.589 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:39.589 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:04:39.589 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:39.589 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:39.590 | INFO | Uploaded 36GiB to GPU in 2.01474s (17.8684GiB/s) | 20260507 06:04:39.590 | INFO | Downloaded 34GiB to GPU in 1.93372s (17.5827GiB/s) | 20260507 06:04:44.735 | INFO | Analyzing chunk frames = [7669760,7670272) | 20260507 06:04:44.735 | INFO | MemInfo: { | 20260507 06:04:44.735 | INFO | "gpu" : " 17 GB", | 20260507 06:04:44.735 | INFO | "huge" : "", | 20260507 06:04:44.735 | INFO | "huge_shared" : " 27 GB", | 20260507 06:04:44.735 | INFO | "malloc" : " 14 GB", | 20260507 06:04:44.735 | INFO | "pinned" : " 29 GB" | 20260507 06:04:44.735 | INFO | } | 20260507 06:04:44.735 | INFO | Perf report for the last 5.14607 seconds: | 20260507 06:04:44.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:04:44.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:44.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:44.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:44.735 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 06:04:44.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:44.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:44.735 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:04:44.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:44.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:44.736 | INFO | Uploaded 36GiB to GPU in 2.05662s (17.5045GiB/s) | 20260507 06:04:44.736 | INFO | Downloaded 34GiB to GPU in 1.94699s (17.4629GiB/s) | 20260507 06:04:49.874 | INFO | Analyzing chunk frames = [7670272,7670784) | 20260507 06:04:49.874 | INFO | MemInfo: { | 20260507 06:04:49.874 | INFO | "gpu" : " 17 GB", | 20260507 06:04:49.874 | INFO | "huge" : "", | 20260507 06:04:49.874 | INFO | "huge_shared" : " 27 GB", | 20260507 06:04:49.874 | INFO | "malloc" : " 14 GB", | 20260507 06:04:49.874 | INFO | "pinned" : " 29 GB" | 20260507 06:04:49.874 | INFO | } | 20260507 06:04:49.874 | INFO | Perf report for the last 5.13875 seconds: | 20260507 06:04:49.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:04:49.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:49.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:49.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:49.874 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:04:49.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:49.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:49.874 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:04:49.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:49.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:49.874 | INFO | Uploaded 36GiB to GPU in 2.07293s (17.3667GiB/s) | 20260507 06:04:49.874 | INFO | Downloaded 34GiB to GPU in 1.93901s (17.5347GiB/s) | 20260507 06:04:55.006 | INFO | Analyzing chunk frames = [7670784,7671296) | 20260507 06:04:55.006 | INFO | MemInfo: { | 20260507 06:04:55.006 | INFO | "gpu" : " 17 GB", | 20260507 06:04:55.006 | INFO | "huge" : "", | 20260507 06:04:55.006 | INFO | "huge_shared" : " 27 GB", | 20260507 06:04:55.006 | INFO | "malloc" : " 14 GB", | 20260507 06:04:55.006 | INFO | "pinned" : " 29 GB" | 20260507 06:04:55.006 | INFO | } | 20260507 06:04:55.006 | INFO | Perf report for the last 5.13183 seconds: | 20260507 06:04:55.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:04:55.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:55.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:55.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:55.006 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 06:04:55.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:55.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:55.006 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 06:04:55.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:55.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:04:55.007 | INFO | Uploaded 36GiB to GPU in 1.89679s (18.9795GiB/s) | 20260507 06:04:55.007 | INFO | Downloaded 34GiB to GPU in 1.97315s (17.2313GiB/s) | 20260507 06:05:00.145 | INFO | Analyzing chunk frames = [7671296,7671808) | 20260507 06:05:00.145 | INFO | MemInfo: { | 20260507 06:05:00.145 | INFO | "gpu" : " 17 GB", | 20260507 06:05:00.145 | INFO | "huge" : "", | 20260507 06:05:00.145 | INFO | "huge_shared" : " 27 GB", | 20260507 06:05:00.145 | INFO | "malloc" : " 14 GB", | 20260507 06:05:00.145 | INFO | "pinned" : " 29 GB" | 20260507 06:05:00.145 | INFO | } | 20260507 06:05:00.145 | INFO | Perf report for the last 5.13925 seconds: | 20260507 06:05:00.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:05:00.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:00.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:00.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:00.145 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:05:00.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:00.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:00.145 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:05:00.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:00.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:00.145 | INFO | Uploaded 36GiB to GPU in 1.96208s (18.3479GiB/s) | 20260507 06:05:00.146 | INFO | Downloaded 34GiB to GPU in 1.9566s (17.3771GiB/s) | 20260507 06:05:00.242 | INFO | Starting a round of Baz IO | 20260507 06:05:03.344 | INFO | Populating RTMetrics report | 20260507 06:05:03.344 | INFO | RTMetrics report populated | 20260507 06:05:03.344 | INFO | Opening tmp RTMetrics json file | 20260507 06:05:03.352 | INFO | Serializing RTMetrics to json | 20260507 06:05:03.360 | INFO | Serialization done, swapping tmp file now... | 20260507 06:05:03.361 | INFO | RTMetrics done | 20260507 06:05:03.429 | INFO | Finished a round of Baz IO: Wrote 1.676 GiB in 3.186 seconds | 20260507 06:05:03.429 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.65% / 0.00% / 67.34% | 20260507 06:05:03.429 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.129314% / 0% / 5.5631e-05% | 20260507 06:05:03.429 | INFO | Baz IO overall: | 20260507 06:05:03.429 | INFO | BytesWritten : 3877788108490 | 20260507 06:05:03.429 | INFO | NumEvents : 1858943114270 | 20260507 06:05:05.295 | INFO | Analyzing chunk frames = [7671808,7672320) | 20260507 06:05:05.295 | INFO | MemInfo: { | 20260507 06:05:05.295 | INFO | "gpu" : " 17 GB", | 20260507 06:05:05.295 | INFO | "huge" : "", | 20260507 06:05:05.295 | INFO | "huge_shared" : " 27 GB", | 20260507 06:05:05.295 | INFO | "malloc" : " 14 GB", | 20260507 06:05:05.295 | INFO | "pinned" : " 29 GB" | 20260507 06:05:05.295 | INFO | } | 20260507 06:05:05.295 | INFO | Perf report for the last 5.14999 seconds: | 20260507 06:05:05.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:05:05.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:05.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:05.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:05.295 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 06:05:05.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:05.295 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:05:05.295 | INFO | PRE_HQ : 58.9%, 1.8, 55.1ms, 0 | 20260507 06:05:05.295 | INFO | BAZWRITER : 61.5%, 3.2, 100.1ms, 0 | 20260507 06:05:05.295 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:05:05.296 | INFO | Uploaded 36GiB to GPU in 2.04171s (17.6323GiB/s) | 20260507 06:05:05.296 | INFO | Downloaded 38GiB to GPU in 2.17696s (17.4555GiB/s) | 20260507 06:05:10.428 | INFO | Analyzing chunk frames = [7672320,7672832) | 20260507 06:05:10.428 | INFO | MemInfo: { | 20260507 06:05:10.428 | INFO | "gpu" : " 17 GB", | 20260507 06:05:10.428 | INFO | "huge" : "", | 20260507 06:05:10.428 | INFO | "huge_shared" : " 27 GB", | 20260507 06:05:10.428 | INFO | "malloc" : " 14 GB", | 20260507 06:05:10.428 | INFO | "pinned" : " 29 GB" | 20260507 06:05:10.428 | INFO | } | 20260507 06:05:10.428 | INFO | Perf report for the last 5.13289 seconds: | 20260507 06:05:10.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:05:10.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:10.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:10.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:10.428 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:05:10.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:10.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:10.428 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:05:10.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:10.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:10.428 | INFO | Uploaded 36GiB to GPU in 1.98882s (18.1012GiB/s) | 20260507 06:05:10.429 | INFO | Downloaded 34GiB to GPU in 1.92853s (17.63GiB/s) | 20260507 06:05:15.564 | INFO | Analyzing chunk frames = [7672832,7673344) | 20260507 06:05:15.564 | INFO | MemInfo: { | 20260507 06:05:15.564 | INFO | "gpu" : " 17 GB", | 20260507 06:05:15.564 | INFO | "huge" : "", | 20260507 06:05:15.564 | INFO | "huge_shared" : " 27 GB", | 20260507 06:05:15.564 | INFO | "malloc" : " 14 GB", | 20260507 06:05:15.564 | INFO | "pinned" : " 29 GB" | 20260507 06:05:15.564 | INFO | } | 20260507 06:05:15.564 | INFO | Perf report for the last 5.13554 seconds: | 20260507 06:05:15.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:05:15.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:15.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:15.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:15.564 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 06:05:15.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:15.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:15.564 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:05:15.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:15.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:15.564 | INFO | Uploaded 36GiB to GPU in 2.0545s (17.5225GiB/s) | 20260507 06:05:15.564 | INFO | Downloaded 34GiB to GPU in 1.95067s (17.4299GiB/s) | 20260507 06:05:20.706 | INFO | Analyzing chunk frames = [7673344,7673856) | 20260507 06:05:20.706 | INFO | MemInfo: { | 20260507 06:05:20.706 | INFO | "gpu" : " 17 GB", | 20260507 06:05:20.706 | INFO | "huge" : "", | 20260507 06:05:20.707 | INFO | "huge_shared" : " 27 GB", | 20260507 06:05:20.707 | INFO | "malloc" : " 14 GB", | 20260507 06:05:20.707 | INFO | "pinned" : " 29 GB" | 20260507 06:05:20.707 | INFO | } | 20260507 06:05:20.707 | INFO | Perf report for the last 5.14293 seconds: | 20260507 06:05:20.707 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:05:20.707 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:20.707 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:20.707 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:20.707 | INFO | ANALYSIS : 59.7%, 3.0, 92.7ms, 0 | 20260507 06:05:20.707 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:20.707 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:20.707 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 06:05:20.707 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:20.707 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:20.707 | INFO | Uploaded 36GiB to GPU in 2.01756s (17.8433GiB/s) | 20260507 06:05:20.707 | INFO | Downloaded 34GiB to GPU in 1.93712s (17.5518GiB/s) | 20260507 06:05:25.849 | INFO | Analyzing chunk frames = [7673856,7674368) | 20260507 06:05:25.849 | INFO | MemInfo: { | 20260507 06:05:25.849 | INFO | "gpu" : " 17 GB", | 20260507 06:05:25.849 | INFO | "huge" : "", | 20260507 06:05:25.849 | INFO | "huge_shared" : " 27 GB", | 20260507 06:05:25.849 | INFO | "malloc" : " 14 GB", | 20260507 06:05:25.849 | INFO | "pinned" : " 29 GB" | 20260507 06:05:25.849 | INFO | } | 20260507 06:05:25.849 | INFO | Perf report for the last 5.14265 seconds: | 20260507 06:05:25.849 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:05:25.849 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:25.849 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:25.849 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:25.849 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 06:05:25.849 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:25.849 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:25.849 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:05:25.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:25.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:25.850 | INFO | Uploaded 36GiB to GPU in 2.05068s (17.5552GiB/s) | 20260507 06:05:25.850 | INFO | Downloaded 34GiB to GPU in 1.94324s (17.4966GiB/s) | 20260507 06:05:30.983 | INFO | Analyzing chunk frames = [7674368,7674880) | 20260507 06:05:30.983 | INFO | MemInfo: { | 20260507 06:05:30.983 | INFO | "gpu" : " 17 GB", | 20260507 06:05:30.983 | INFO | "huge" : "", | 20260507 06:05:30.983 | INFO | "huge_shared" : " 27 GB", | 20260507 06:05:30.983 | INFO | "malloc" : " 14 GB", | 20260507 06:05:30.983 | INFO | "pinned" : " 29 GB" | 20260507 06:05:30.983 | INFO | } | 20260507 06:05:30.983 | INFO | Perf report for the last 5.13343 seconds: | 20260507 06:05:30.983 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:05:30.983 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:30.983 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:30.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:30.983 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:05:30.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:30.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:30.983 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 06:05:30.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:30.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:30.983 | INFO | Uploaded 36GiB to GPU in 2.06999s (17.3914GiB/s) | 20260507 06:05:30.983 | INFO | Downloaded 34GiB to GPU in 1.93154s (17.6026GiB/s) | 20260507 06:05:36.118 | INFO | Analyzing chunk frames = [7674880,7675392) | 20260507 06:05:36.118 | INFO | MemInfo: { | 20260507 06:05:36.118 | INFO | "gpu" : " 17 GB", | 20260507 06:05:36.118 | INFO | "huge" : "", | 20260507 06:05:36.118 | INFO | "huge_shared" : " 27 GB", | 20260507 06:05:36.118 | INFO | "malloc" : " 14 GB", | 20260507 06:05:36.118 | INFO | "pinned" : " 29 GB" | 20260507 06:05:36.118 | INFO | } | 20260507 06:05:36.119 | INFO | Perf report for the last 5.1357 seconds: | 20260507 06:05:36.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:05:36.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:36.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:36.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:36.119 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 06:05:36.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:36.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:36.119 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 06:05:36.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:36.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:36.119 | INFO | Uploaded 36GiB to GPU in 2.04665s (17.5897GiB/s) | 20260507 06:05:36.119 | INFO | Downloaded 34GiB to GPU in 1.9956s (17.0375GiB/s) | 20260507 06:05:41.255 | INFO | Analyzing chunk frames = [7675392,7675904) | 20260507 06:05:41.255 | INFO | MemInfo: { | 20260507 06:05:41.255 | INFO | "gpu" : " 17 GB", | 20260507 06:05:41.255 | INFO | "huge" : "", | 20260507 06:05:41.255 | INFO | "huge_shared" : " 27 GB", | 20260507 06:05:41.255 | INFO | "malloc" : " 14 GB", | 20260507 06:05:41.255 | INFO | "pinned" : " 29 GB" | 20260507 06:05:41.255 | INFO | } | 20260507 06:05:41.255 | INFO | Perf report for the last 5.13637 seconds: | 20260507 06:05:41.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:05:41.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:41.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:41.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:41.255 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:05:41.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:41.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:41.255 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 06:05:41.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:41.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:41.255 | INFO | Uploaded 36GiB to GPU in 1.97318s (18.2447GiB/s) | 20260507 06:05:41.255 | INFO | Downloaded 34GiB to GPU in 1.94469s (17.4835GiB/s) | 20260507 06:05:41.255 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:05:41.255 | INFO | Total mem usage: 18 GiB | 20260507 06:05:41.255 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:05:41.255 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:05:41.255 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:05:41.255 | INFO | High Water Breakdown: | 20260507 06:05:41.255 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:05:41.255 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:05:41.255 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:05:41.255 | INFO | ------------------------------------------ | 20260507 06:05:41.255 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:05:41.255 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:05:41.255 | INFO | Total mem usage: 26.5 GiB | 20260507 06:05:41.256 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:05:41.256 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 06:05:41.256 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 06:05:41.256 | INFO | High Water Breakdown: | 20260507 06:05:41.256 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:05:41.256 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:05:41.256 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:05:41.256 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:05:41.256 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:05:41.256 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:05:41.256 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:05:41.256 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:05:41.256 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:05:41.256 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:05:41.256 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:05:41.256 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:05:41.256 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:05:41.256 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:05:41.256 | INFO | ------------------------------------------ | 20260507 06:05:41.256 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:05:41.256 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:05:41.256 | INFO | Total mem usage: 25.6 GiB | 20260507 06:05:41.256 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:05:41.256 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:05:41.256 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:05:41.256 | INFO | High Water Breakdown: | 20260507 06:05:41.256 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:05:41.256 | INFO | ------------------------------------------ | 20260507 06:05:41.256 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:05:41.256 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:05:41.256 | INFO | Total mem usage: 15.7 GiB | 20260507 06:05:41.256 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:05:41.256 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:05:41.256 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:05:41.256 | INFO | High Water Breakdown: | 20260507 06:05:41.256 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:05:41.256 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:05:41.256 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:05:41.256 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:05:41.256 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:05:41.256 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:05:41.256 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:05:41.256 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:05:41.256 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:05:41.256 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:05:41.256 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:05:41.256 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:05:41.256 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:05:41.256 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:05:41.256 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:05:41.256 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:05:41.256 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:05:41.256 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:05:41.256 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:05:41.256 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:05:41.256 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:05:41.256 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:05:41.256 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:05:41.256 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:05:41.256 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:05:41.256 | INFO | ------------------------------------------ | 20260507 06:05:41.256 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:05:41.256 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:05:41.256 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:05:41.256 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:05:41.256 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.44e+01, 1.34e+01 / 1.40e+00 | 20260507 06:05:41.256 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.02e-01, 9.50e-03 / 4.78e-03 | 20260507 06:05:41.256 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.43e+00, 4.97e-01 / 2.59e-01 | 20260507 06:05:41.256 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.48e+00, 1.65e-01 / 9.98e-02 | 20260507 06:05:41.256 | INFO | PerfReport: 16.23%, DME -- 3.98e+01 / 4.44e+01, 3.89e+00 / 1.57e+00 | 20260507 06:05:41.256 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.77e+01 / 1.98e+01, 1.20e+00 / 1.03e+00 | 20260507 06:05:41.256 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.13e+00 / 2.73e+00, 3.91e-01 / 1.37e-01 | 20260507 06:05:41.256 | INFO | PerfReport: 2.97%, Metrics -- 9.12e-01 / 1.23e+00, 2.43e-01 / 1.09e-01 | 20260507 06:05:41.256 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.06e-01 / 3.61e-01, 5.01e-02 / 1.73e-02 | 20260507 06:05:41.256 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:05:41.256 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:05:41.256 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:05:41.256 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 06:05:41.256 | INFO | PerfReport: 21.88%, Upload -- 2.10e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260507 06:05:41.256 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 06:05:41.256 | INFO | PerfReport: 31.88%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:05:41.256 | INFO | PerfReport: 1.59%, LockUpload -- 1.53e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 06:05:41.256 | INFO | PerfReport: 1.21%, LockDownload -- 1.17e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260507 06:05:41.257 | INFO | PerfReport: 22.02%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 06:05:41.257 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260507 06:05:41.356 | INFO | Starting a round of Baz IO | 20260507 06:05:44.449 | INFO | Populating RTMetrics report | 20260507 06:05:44.450 | INFO | RTMetrics report populated | 20260507 06:05:44.450 | INFO | Opening tmp RTMetrics json file | 20260507 06:05:44.468 | INFO | Serializing RTMetrics to json | 20260507 06:05:44.470 | INFO | Serialization done, swapping tmp file now... | 20260507 06:05:44.470 | INFO | RTMetrics done | 20260507 06:05:44.540 | INFO | Finished a round of Baz IO: Wrote 1.677 GiB in 3.184 seconds | 20260507 06:05:44.540 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.65% / 0.00% / 67.35% | 20260507 06:05:44.540 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.128953% / 3.97364e-06% / 8.74201e-05% | 20260507 06:05:44.540 | INFO | Baz IO overall: | 20260507 06:05:44.540 | INFO | BytesWritten : 3879588842586 | 20260507 06:05:44.540 | INFO | NumEvents : 1859787341628 | 20260507 06:05:46.401 | INFO | Analyzing chunk frames = [7675904,7676416) | 20260507 06:05:46.401 | INFO | MemInfo: { | 20260507 06:05:46.401 | INFO | "gpu" : " 17 GB", | 20260507 06:05:46.401 | INFO | "huge" : "", | 20260507 06:05:46.401 | INFO | "huge_shared" : " 27 GB", | 20260507 06:05:46.401 | INFO | "malloc" : " 14 GB", | 20260507 06:05:46.401 | INFO | "pinned" : " 29 GB" | 20260507 06:05:46.401 | INFO | } | 20260507 06:05:46.401 | INFO | Perf report for the last 5.14653 seconds: | 20260507 06:05:46.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:05:46.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:46.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:46.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:46.401 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260507 06:05:46.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:46.402 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 06:05:46.402 | INFO | PRE_HQ : 58.8%, 1.8, 55.1ms, 0 | 20260507 06:05:46.402 | INFO | BAZWRITER : 61.5%, 3.2, 100.4ms, 0 | 20260507 06:05:46.402 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:05:46.402 | INFO | Uploaded 36GiB to GPU in 2.03163s (17.7197GiB/s) | 20260507 06:05:46.402 | INFO | Downloaded 38GiB to GPU in 2.18256s (17.4107GiB/s) | 20260507 06:05:51.542 | INFO | Analyzing chunk frames = [7676416,7676928) | 20260507 06:05:51.542 | INFO | MemInfo: { | 20260507 06:05:51.542 | INFO | "gpu" : " 17 GB", | 20260507 06:05:51.542 | INFO | "huge" : "", | 20260507 06:05:51.542 | INFO | "huge_shared" : " 27 GB", | 20260507 06:05:51.542 | INFO | "malloc" : " 14 GB", | 20260507 06:05:51.542 | INFO | "pinned" : " 29 GB" | 20260507 06:05:51.542 | INFO | } | 20260507 06:05:51.543 | INFO | Perf report for the last 5.14109 seconds: | 20260507 06:05:51.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:05:51.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:51.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:51.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:51.543 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 06:05:51.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:51.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:51.543 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:05:51.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:51.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:51.543 | INFO | Uploaded 36GiB to GPU in 2.0519s (17.5447GiB/s) | 20260507 06:05:51.543 | INFO | Downloaded 34GiB to GPU in 1.9398s (17.5276GiB/s) | 20260507 06:05:56.681 | INFO | Analyzing chunk frames = [7676928,7677440) | 20260507 06:05:56.681 | INFO | MemInfo: { | 20260507 06:05:56.681 | INFO | "gpu" : " 17 GB", | 20260507 06:05:56.681 | INFO | "huge" : "", | 20260507 06:05:56.681 | INFO | "huge_shared" : " 27 GB", | 20260507 06:05:56.681 | INFO | "malloc" : " 14 GB", | 20260507 06:05:56.681 | INFO | "pinned" : " 29 GB" | 20260507 06:05:56.681 | INFO | } | 20260507 06:05:56.681 | INFO | Perf report for the last 5.13833 seconds: | 20260507 06:05:56.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:05:56.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:56.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:56.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:56.681 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 06:05:56.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:56.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:56.681 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:05:56.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:56.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:05:56.681 | INFO | Uploaded 36GiB to GPU in 2.05309s (17.5346GiB/s) | 20260507 06:05:56.681 | INFO | Downloaded 34GiB to GPU in 1.94905s (17.4444GiB/s) | 20260507 06:06:01.818 | INFO | Analyzing chunk frames = [7677440,7677952) | 20260507 06:06:01.818 | INFO | MemInfo: { | 20260507 06:06:01.818 | INFO | "gpu" : " 17 GB", | 20260507 06:06:01.818 | INFO | "huge" : "", | 20260507 06:06:01.818 | INFO | "huge_shared" : " 27 GB", | 20260507 06:06:01.818 | INFO | "malloc" : " 14 GB", | 20260507 06:06:01.818 | INFO | "pinned" : " 29 GB" | 20260507 06:06:01.818 | INFO | } | 20260507 06:06:01.818 | INFO | Perf report for the last 5.13727 seconds: | 20260507 06:06:01.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:06:01.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:01.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:01.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:01.818 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 06:06:01.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:01.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:01.818 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:06:01.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:01.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:01.818 | INFO | Uploaded 36GiB to GPU in 2.05261s (17.5386GiB/s) | 20260507 06:06:01.818 | INFO | Downloaded 34GiB to GPU in 1.94322s (17.4968GiB/s) | 20260507 06:06:06.956 | INFO | Analyzing chunk frames = [7677952,7678464) | 20260507 06:06:06.956 | INFO | MemInfo: { | 20260507 06:06:06.956 | INFO | "gpu" : " 17 GB", | 20260507 06:06:06.956 | INFO | "huge" : "", | 20260507 06:06:06.956 | INFO | "huge_shared" : " 27 GB", | 20260507 06:06:06.956 | INFO | "malloc" : " 14 GB", | 20260507 06:06:06.957 | INFO | "pinned" : " 29 GB" | 20260507 06:06:06.957 | INFO | } | 20260507 06:06:06.957 | INFO | Perf report for the last 5.13851 seconds: | 20260507 06:06:06.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:06:06.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:06.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:06.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:06.957 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 06:06:06.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:06.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:06.957 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:06:06.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:06.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:06.957 | INFO | Uploaded 36GiB to GPU in 2.05651s (17.5054GiB/s) | 20260507 06:06:06.957 | INFO | Downloaded 34GiB to GPU in 1.94942s (17.4411GiB/s) | 20260507 06:06:12.097 | INFO | Analyzing chunk frames = [7678464,7678976) | 20260507 06:06:12.097 | INFO | MemInfo: { | 20260507 06:06:12.097 | INFO | "gpu" : " 17 GB", | 20260507 06:06:12.097 | INFO | "huge" : "", | 20260507 06:06:12.097 | INFO | "huge_shared" : " 27 GB", | 20260507 06:06:12.097 | INFO | "malloc" : " 14 GB", | 20260507 06:06:12.097 | INFO | "pinned" : " 29 GB" | 20260507 06:06:12.097 | INFO | } | 20260507 06:06:12.097 | INFO | Perf report for the last 5.14034 seconds: | 20260507 06:06:12.097 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:06:12.097 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:12.097 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:12.097 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:12.097 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:06:12.097 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:12.097 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:12.097 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 06:06:12.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:12.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:12.097 | INFO | Uploaded 36GiB to GPU in 2.03586s (17.683GiB/s) | 20260507 06:06:12.097 | INFO | Downloaded 34GiB to GPU in 1.90074s (17.8877GiB/s) | 20260507 06:06:17.247 | INFO | Analyzing chunk frames = [7678976,7679488) | 20260507 06:06:17.247 | INFO | MemInfo: { | 20260507 06:06:17.247 | INFO | "gpu" : " 17 GB", | 20260507 06:06:17.247 | INFO | "huge" : "", | 20260507 06:06:17.247 | INFO | "huge_shared" : " 27 GB", | 20260507 06:06:17.247 | INFO | "malloc" : " 14 GB", | 20260507 06:06:17.247 | INFO | "pinned" : " 29 GB" | 20260507 06:06:17.247 | INFO | } | 20260507 06:06:17.247 | INFO | Perf report for the last 5.15015 seconds: | 20260507 06:06:17.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:06:17.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:17.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:17.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:17.247 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 06:06:17.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:17.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:17.247 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 06:06:17.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:17.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:17.247 | INFO | Uploaded 36GiB to GPU in 1.98165s (18.1667GiB/s) | 20260507 06:06:17.247 | INFO | Downloaded 34GiB to GPU in 1.97192s (17.2421GiB/s) | 20260507 06:06:22.380 | INFO | Analyzing chunk frames = [7679488,7680000) | 20260507 06:06:22.380 | INFO | MemInfo: { | 20260507 06:06:22.380 | INFO | "gpu" : " 17 GB", | 20260507 06:06:22.380 | INFO | "huge" : "", | 20260507 06:06:22.380 | INFO | "huge_shared" : " 27 GB", | 20260507 06:06:22.380 | INFO | "malloc" : " 14 GB", | 20260507 06:06:22.380 | INFO | "pinned" : " 29 GB" | 20260507 06:06:22.380 | INFO | } | 20260507 06:06:22.380 | INFO | Perf report for the last 5.1333 seconds: | 20260507 06:06:22.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:06:22.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:22.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:22.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:22.381 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:06:22.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:22.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:22.381 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:06:22.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:22.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:22.381 | INFO | Uploaded 36GiB to GPU in 2.00184s (17.9835GiB/s) | 20260507 06:06:22.381 | INFO | Downloaded 34GiB to GPU in 1.96241s (17.3256GiB/s) | 20260507 06:06:22.477 | INFO | Starting a round of Baz IO | 20260507 06:06:25.572 | INFO | Populating RTMetrics report | 20260507 06:06:25.572 | INFO | RTMetrics report populated | 20260507 06:06:25.572 | INFO | Opening tmp RTMetrics json file | 20260507 06:06:25.581 | INFO | Serializing RTMetrics to json | 20260507 06:06:25.589 | INFO | Serialization done, swapping tmp file now... | 20260507 06:06:25.589 | INFO | RTMetrics done | 20260507 06:06:25.658 | INFO | Finished a round of Baz IO: Wrote 1.678 GiB in 3.181 seconds | 20260507 06:06:25.658 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.65% / 0.00% / 67.35% | 20260507 06:06:25.658 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.129934% / 0% / 7.54992e-05% | 20260507 06:06:25.658 | INFO | Baz IO overall: | 20260507 06:06:25.658 | INFO | BytesWritten : 3881390549058 | 20260507 06:06:25.659 | INFO | NumEvents : 1860632139942 | 20260507 06:06:27.512 | INFO | Analyzing chunk frames = [7680000,7680512) | 20260507 06:06:27.512 | INFO | MemInfo: { | 20260507 06:06:27.512 | INFO | "gpu" : " 17 GB", | 20260507 06:06:27.512 | INFO | "huge" : "", | 20260507 06:06:27.512 | INFO | "huge_shared" : " 27 GB", | 20260507 06:06:27.512 | INFO | "malloc" : " 14 GB", | 20260507 06:06:27.512 | INFO | "pinned" : " 29 GB" | 20260507 06:06:27.512 | INFO | } | 20260507 06:06:27.512 | INFO | Perf report for the last 5.13163 seconds: | 20260507 06:06:27.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:06:27.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:27.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:27.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:27.512 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260507 06:06:27.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:27.512 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 06:06:27.512 | INFO | PRE_HQ : 59.0%, 1.8, 54.8ms, 0 | 20260507 06:06:27.512 | INFO | BAZWRITER : 61.5%, 3.2, 100.0ms, 0 | 20260507 06:06:27.512 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 06:06:27.512 | INFO | Uploaded 36GiB to GPU in 2.07517s (17.348GiB/s) | 20260507 06:06:27.512 | INFO | Downloaded 38GiB to GPU in 2.18126s (17.4211GiB/s) | 20260507 06:06:32.655 | INFO | Analyzing chunk frames = [7680512,7681024) | 20260507 06:06:32.655 | INFO | MemInfo: { | 20260507 06:06:32.655 | INFO | "gpu" : " 17 GB", | 20260507 06:06:32.655 | INFO | "huge" : "", | 20260507 06:06:32.655 | INFO | "huge_shared" : " 27 GB", | 20260507 06:06:32.655 | INFO | "malloc" : " 14 GB", | 20260507 06:06:32.655 | INFO | "pinned" : " 29 GB" | 20260507 06:06:32.655 | INFO | } | 20260507 06:06:32.655 | INFO | Perf report for the last 5.14278 seconds: | 20260507 06:06:32.655 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:06:32.655 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:32.655 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:32.655 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:32.655 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 06:06:32.655 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:32.655 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:32.655 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:06:32.655 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:32.655 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:32.655 | INFO | Uploaded 36GiB to GPU in 2.00036s (17.9968GiB/s) | 20260507 06:06:32.655 | INFO | Downloaded 34GiB to GPU in 1.93193s (17.5989GiB/s) | 20260507 06:06:37.799 | INFO | Analyzing chunk frames = [7681024,7681536) | 20260507 06:06:37.799 | INFO | MemInfo: { | 20260507 06:06:37.799 | INFO | "gpu" : " 17 GB", | 20260507 06:06:37.799 | INFO | "huge" : "", | 20260507 06:06:37.799 | INFO | "huge_shared" : " 27 GB", | 20260507 06:06:37.799 | INFO | "malloc" : " 14 GB", | 20260507 06:06:37.799 | INFO | "pinned" : " 29 GB" | 20260507 06:06:37.799 | INFO | } | 20260507 06:06:37.799 | INFO | Perf report for the last 5.1438 seconds: | 20260507 06:06:37.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:06:37.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:37.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:37.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:37.799 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 06:06:37.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:37.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:37.799 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:06:37.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:37.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:37.799 | INFO | Uploaded 36GiB to GPU in 2.0193s (17.828GiB/s) | 20260507 06:06:37.799 | INFO | Downloaded 34GiB to GPU in 1.93931s (17.532GiB/s) | 20260507 06:06:42.937 | INFO | Analyzing chunk frames = [7681536,7682048) | 20260507 06:06:42.937 | INFO | MemInfo: { | 20260507 06:06:42.937 | INFO | "gpu" : " 17 GB", | 20260507 06:06:42.937 | INFO | "huge" : "", | 20260507 06:06:42.937 | INFO | "huge_shared" : " 27 GB", | 20260507 06:06:42.937 | INFO | "malloc" : " 14 GB", | 20260507 06:06:42.937 | INFO | "pinned" : " 29 GB" | 20260507 06:06:42.937 | INFO | } | 20260507 06:06:42.937 | INFO | Perf report for the last 5.13824 seconds: | 20260507 06:06:42.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:06:42.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:42.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:42.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:42.937 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 06:06:42.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:42.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:42.937 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:06:42.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:42.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:42.937 | INFO | Uploaded 36GiB to GPU in 2.05102s (17.5522GiB/s) | 20260507 06:06:42.937 | INFO | Downloaded 34GiB to GPU in 1.9455s (17.4762GiB/s) | 20260507 06:06:48.068 | INFO | Analyzing chunk frames = [7682048,7682560) | 20260507 06:06:48.068 | INFO | MemInfo: { | 20260507 06:06:48.068 | INFO | "gpu" : " 17 GB", | 20260507 06:06:48.068 | INFO | "huge" : "", | 20260507 06:06:48.068 | INFO | "huge_shared" : " 27 GB", | 20260507 06:06:48.068 | INFO | "malloc" : " 14 GB", | 20260507 06:06:48.069 | INFO | "pinned" : " 29 GB" | 20260507 06:06:48.069 | INFO | } | 20260507 06:06:48.069 | INFO | Perf report for the last 5.13176 seconds: | 20260507 06:06:48.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:06:48.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:48.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:48.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:48.069 | INFO | ANALYSIS : 60.1%, 3.0, 93.4ms, 0 | 20260507 06:06:48.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:48.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:48.069 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:06:48.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:48.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:48.069 | INFO | Uploaded 36GiB to GPU in 2.03041s (17.7304GiB/s) | 20260507 06:06:48.070 | INFO | Downloaded 34GiB to GPU in 1.91515s (17.7532GiB/s) | 20260507 06:06:53.213 | INFO | Analyzing chunk frames = [7682560,7683072) | 20260507 06:06:53.213 | INFO | MemInfo: { | 20260507 06:06:53.213 | INFO | "gpu" : " 17 GB", | 20260507 06:06:53.213 | INFO | "huge" : "", | 20260507 06:06:53.213 | INFO | "huge_shared" : " 27 GB", | 20260507 06:06:53.214 | INFO | "malloc" : " 14 GB", | 20260507 06:06:53.214 | INFO | "pinned" : " 29 GB" | 20260507 06:06:53.214 | INFO | } | 20260507 06:06:53.214 | INFO | Perf report for the last 5.14492 seconds: | 20260507 06:06:53.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:06:53.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:53.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:53.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:53.214 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 06:06:53.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:53.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:53.214 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 06:06:53.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:53.214 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:53.214 | INFO | Uploaded 36GiB to GPU in 2.03319s (17.7061GiB/s) | 20260507 06:06:53.214 | INFO | Downloaded 34GiB to GPU in 1.90608s (17.8377GiB/s) | 20260507 06:06:58.354 | INFO | Analyzing chunk frames = [7683072,7683584) | 20260507 06:06:58.354 | INFO | MemInfo: { | 20260507 06:06:58.354 | INFO | "gpu" : " 17 GB", | 20260507 06:06:58.354 | INFO | "huge" : "", | 20260507 06:06:58.354 | INFO | "huge_shared" : " 27 GB", | 20260507 06:06:58.354 | INFO | "malloc" : " 14 GB", | 20260507 06:06:58.354 | INFO | "pinned" : " 29 GB" | 20260507 06:06:58.354 | INFO | } | 20260507 06:06:58.354 | INFO | Perf report for the last 5.14041 seconds: | 20260507 06:06:58.354 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:06:58.354 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:58.354 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:58.354 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:58.354 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:06:58.354 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:58.354 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:58.354 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:06:58.354 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:58.354 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:06:58.354 | INFO | Uploaded 36GiB to GPU in 1.97943s (18.187GiB/s) | 20260507 06:06:58.354 | INFO | Downloaded 34GiB to GPU in 1.97208s (17.2407GiB/s) | 20260507 06:07:03.495 | INFO | Analyzing chunk frames = [7683584,7684096) | 20260507 06:07:03.495 | INFO | MemInfo: { | 20260507 06:07:03.495 | INFO | "gpu" : " 17 GB", | 20260507 06:07:03.495 | INFO | "huge" : "", | 20260507 06:07:03.495 | INFO | "huge_shared" : " 27 GB", | 20260507 06:07:03.495 | INFO | "malloc" : " 14 GB", | 20260507 06:07:03.495 | INFO | "pinned" : " 29 GB" | 20260507 06:07:03.495 | INFO | } | 20260507 06:07:03.495 | INFO | Perf report for the last 5.1411 seconds: | 20260507 06:07:03.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:07:03.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:03.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:03.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:03.495 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:07:03.495 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:03.495 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:03.495 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 06:07:03.495 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:03.495 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:03.496 | INFO | Uploaded 36GiB to GPU in 1.99767s (18.021GiB/s) | 20260507 06:07:03.496 | INFO | Downloaded 34GiB to GPU in 1.96321s (17.3185GiB/s) | 20260507 06:07:03.496 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:07:03.496 | INFO | Total mem usage: 18 GiB | 20260507 06:07:03.496 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:07:03.496 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:07:03.496 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:07:03.496 | INFO | High Water Breakdown: | 20260507 06:07:03.496 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:07:03.496 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:07:03.496 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:07:03.496 | INFO | ------------------------------------------ | 20260507 06:07:03.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:07:03.496 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:07:03.496 | INFO | Total mem usage: 26.5 GiB | 20260507 06:07:03.496 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:07:03.496 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:07:03.496 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:07:03.496 | INFO | High Water Breakdown: | 20260507 06:07:03.496 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:07:03.496 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:07:03.496 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:07:03.496 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:07:03.496 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:07:03.496 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:07:03.496 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:07:03.496 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:07:03.496 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:07:03.496 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:07:03.496 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:07:03.496 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:07:03.496 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:07:03.496 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:07:03.496 | INFO | ------------------------------------------ | 20260507 06:07:03.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:07:03.496 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:07:03.496 | INFO | Total mem usage: 25.6 GiB | 20260507 06:07:03.496 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:07:03.496 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:07:03.496 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:07:03.496 | INFO | High Water Breakdown: | 20260507 06:07:03.496 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:07:03.496 | INFO | ------------------------------------------ | 20260507 06:07:03.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:07:03.496 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:07:03.496 | INFO | Total mem usage: 15.7 GiB | 20260507 06:07:03.496 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:07:03.496 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:07:03.496 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:07:03.496 | INFO | High Water Breakdown: | 20260507 06:07:03.496 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:07:03.496 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:07:03.496 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:07:03.496 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:07:03.496 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:07:03.496 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:07:03.496 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:07:03.496 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:07:03.496 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:07:03.496 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:07:03.496 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:07:03.496 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:07:03.496 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:07:03.496 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:07:03.496 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:07:03.496 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:07:03.496 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:07:03.496 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:07:03.496 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:07:03.496 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:07:03.496 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:07:03.496 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:07:03.496 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:07:03.496 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:07:03.496 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:07:03.496 | INFO | ------------------------------------------ | 20260507 06:07:03.496 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:07:03.497 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:07:03.497 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:07:03.497 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:07:03.497 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.35e+01 / 1.16e+00 | 20260507 06:07:03.497 | INFO | PerfReport: 0.30%, AggregateStats -- 9.05e-02 / 1.01e-01, 9.65e-03 / 4.25e-03 | 20260507 06:07:03.497 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.44e+00, 4.95e-01 / 2.59e-01 | 20260507 06:07:03.497 | INFO | PerfReport: 3.21%, Histogram -- 9.83e-01 / 1.46e+00, 1.62e-01 / 1.20e-01 | 20260507 06:07:03.497 | INFO | PerfReport: 16.26%, DME -- 3.99e+01 / 4.47e+01, 3.82e+00 / 1.50e+00 | 20260507 06:07:03.497 | INFO | PerfReport: 57.68%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.23e+00 / 1.37e+00 | 20260507 06:07:03.497 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.69e+00, 3.85e-01 / 2.24e-01 | 20260507 06:07:03.497 | INFO | PerfReport: 2.97%, Metrics -- 9.12e-01 / 1.25e+00, 2.44e-01 / 1.45e-01 | 20260507 06:07:03.497 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.07e-01 / 3.67e-01, 5.02e-02 / 1.93e-02 | 20260507 06:07:03.497 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:07:03.497 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:07:03.497 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:07:03.497 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 06:07:03.497 | INFO | PerfReport: 21.95%, Upload -- 2.11e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 06:07:03.497 | INFO | PerfReport: 21.34%, Download -- 2.05e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 06:07:03.497 | INFO | PerfReport: 31.85%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 06:07:03.497 | INFO | PerfReport: 1.75%, LockUpload -- 1.68e+00 / 0.00e+00, 5.12e+00 / 0.00e+00 | 20260507 06:07:03.497 | INFO | PerfReport: 1.18%, LockDownload -- 1.14e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260507 06:07:03.497 | INFO | PerfReport: 21.91%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 06:07:03.497 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260507 06:07:03.592 | INFO | Starting a round of Baz IO | 20260507 06:07:06.686 | INFO | Populating RTMetrics report | 20260507 06:07:06.686 | INFO | RTMetrics report populated | 20260507 06:07:06.686 | INFO | Opening tmp RTMetrics json file | 20260507 06:07:06.702 | INFO | Serializing RTMetrics to json | 20260507 06:07:06.703 | INFO | Serialization done, swapping tmp file now... | 20260507 06:07:06.703 | INFO | RTMetrics done | 20260507 06:07:06.776 | INFO | Finished a round of Baz IO: Wrote 1.675 GiB in 3.183 seconds | 20260507 06:07:06.776 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.65% / 0.00% / 67.35% | 20260507 06:07:06.776 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.128857% / 0% / 5.5631e-05% | 20260507 06:07:06.776 | INFO | Baz IO overall: | 20260507 06:07:06.776 | INFO | BytesWritten : 3883189443221 | 20260507 06:07:06.776 | INFO | NumEvents : 1861475078662 | 20260507 06:07:08.628 | INFO | Analyzing chunk frames = [7684096,7684608) | 20260507 06:07:08.628 | INFO | MemInfo: { | 20260507 06:07:08.628 | INFO | "gpu" : " 17 GB", | 20260507 06:07:08.628 | INFO | "huge" : "", | 20260507 06:07:08.628 | INFO | "huge_shared" : " 27 GB", | 20260507 06:07:08.628 | INFO | "malloc" : " 14 GB", | 20260507 06:07:08.628 | INFO | "pinned" : " 29 GB" | 20260507 06:07:08.628 | INFO | } | 20260507 06:07:08.628 | INFO | Perf report for the last 5.13237 seconds: | 20260507 06:07:08.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:07:08.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:08.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:08.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:08.628 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 06:07:08.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:08.628 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 06:07:08.628 | INFO | PRE_HQ : 59.2%, 1.8, 55.1ms, 0 | 20260507 06:07:08.628 | INFO | BAZWRITER : 61.6%, 3.2, 99.7ms, 0 | 20260507 06:07:08.628 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:07:08.628 | INFO | Uploaded 36GiB to GPU in 2.06232s (17.4561GiB/s) | 20260507 06:07:08.628 | INFO | Downloaded 38GiB to GPU in 2.17659s (17.4585GiB/s) | 20260507 06:07:13.765 | INFO | Analyzing chunk frames = [7684608,7685120) | 20260507 06:07:13.765 | INFO | MemInfo: { | 20260507 06:07:13.765 | INFO | "gpu" : " 17 GB", | 20260507 06:07:13.765 | INFO | "huge" : "", | 20260507 06:07:13.765 | INFO | "huge_shared" : " 27 GB", | 20260507 06:07:13.765 | INFO | "malloc" : " 14 GB", | 20260507 06:07:13.765 | INFO | "pinned" : " 29 GB" | 20260507 06:07:13.765 | INFO | } | 20260507 06:07:13.765 | INFO | Perf report for the last 5.13745 seconds: | 20260507 06:07:13.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:07:13.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:13.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:13.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:13.765 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 06:07:13.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:13.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:13.765 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:07:13.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:13.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:13.765 | INFO | Uploaded 36GiB to GPU in 2.00203s (17.9817GiB/s) | 20260507 06:07:13.766 | INFO | Downloaded 34GiB to GPU in 1.93019s (17.6148GiB/s) | 20260507 06:07:18.910 | INFO | Analyzing chunk frames = [7685120,7685632) | 20260507 06:07:18.910 | INFO | MemInfo: { | 20260507 06:07:18.910 | INFO | "gpu" : " 17 GB", | 20260507 06:07:18.910 | INFO | "huge" : "", | 20260507 06:07:18.910 | INFO | "huge_shared" : " 27 GB", | 20260507 06:07:18.910 | INFO | "malloc" : " 14 GB", | 20260507 06:07:18.910 | INFO | "pinned" : " 29 GB" | 20260507 06:07:18.910 | INFO | } | 20260507 06:07:18.910 | INFO | Perf report for the last 5.14522 seconds: | 20260507 06:07:18.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:07:18.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:18.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:18.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:18.910 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 06:07:18.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:18.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:18.910 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:07:18.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:18.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:18.911 | INFO | Uploaded 36GiB to GPU in 2.01809s (17.8387GiB/s) | 20260507 06:07:18.911 | INFO | Downloaded 34GiB to GPU in 1.94231s (17.5049GiB/s) | 20260507 06:07:24.054 | INFO | Analyzing chunk frames = [7685632,7686144) | 20260507 06:07:24.054 | INFO | MemInfo: { | 20260507 06:07:24.054 | INFO | "gpu" : " 17 GB", | 20260507 06:07:24.054 | INFO | "huge" : "", | 20260507 06:07:24.054 | INFO | "huge_shared" : " 27 GB", | 20260507 06:07:24.054 | INFO | "malloc" : " 14 GB", | 20260507 06:07:24.054 | INFO | "pinned" : " 29 GB" | 20260507 06:07:24.054 | INFO | } | 20260507 06:07:24.054 | INFO | Perf report for the last 5.14377 seconds: | 20260507 06:07:24.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:07:24.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:24.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:24.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:24.054 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 06:07:24.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:24.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:24.054 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:07:24.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:24.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:24.054 | INFO | Uploaded 36GiB to GPU in 2.00513s (17.954GiB/s) | 20260507 06:07:24.054 | INFO | Downloaded 34GiB to GPU in 1.93637s (17.5586GiB/s) | 20260507 06:07:29.197 | INFO | Analyzing chunk frames = [7686144,7686656) | 20260507 06:07:29.197 | INFO | MemInfo: { | 20260507 06:07:29.197 | INFO | "gpu" : " 17 GB", | 20260507 06:07:29.197 | INFO | "huge" : "", | 20260507 06:07:29.197 | INFO | "huge_shared" : " 27 GB", | 20260507 06:07:29.197 | INFO | "malloc" : " 14 GB", | 20260507 06:07:29.198 | INFO | "pinned" : " 29 GB" | 20260507 06:07:29.198 | INFO | } | 20260507 06:07:29.198 | INFO | Perf report for the last 5.14362 seconds: | 20260507 06:07:29.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:07:29.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:29.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:29.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:29.198 | INFO | ANALYSIS : 59.9%, 3.0, 93.5ms, 0 | 20260507 06:07:29.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:29.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:29.198 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:07:29.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:29.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:29.198 | INFO | Uploaded 36GiB to GPU in 2.04313s (17.62GiB/s) | 20260507 06:07:29.198 | INFO | Downloaded 34GiB to GPU in 1.90023s (17.8926GiB/s) | 20260507 06:07:34.327 | INFO | Analyzing chunk frames = [7686656,7687168) | 20260507 06:07:34.327 | INFO | MemInfo: { | 20260507 06:07:34.327 | INFO | "gpu" : " 17 GB", | 20260507 06:07:34.327 | INFO | "huge" : "", | 20260507 06:07:34.327 | INFO | "huge_shared" : " 27 GB", | 20260507 06:07:34.327 | INFO | "malloc" : " 14 GB", | 20260507 06:07:34.327 | INFO | "pinned" : " 29 GB" | 20260507 06:07:34.327 | INFO | } | 20260507 06:07:34.327 | INFO | Perf report for the last 5.12966 seconds: | 20260507 06:07:34.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:07:34.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:34.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:34.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:34.327 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 06:07:34.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:34.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:34.327 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:07:34.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:34.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:34.328 | INFO | Uploaded 36GiB to GPU in 2.04357s (17.6162GiB/s) | 20260507 06:07:34.328 | INFO | Downloaded 34GiB to GPU in 1.92622s (17.6511GiB/s) | 20260507 06:07:39.466 | INFO | Analyzing chunk frames = [7687168,7687680) | 20260507 06:07:39.466 | INFO | MemInfo: { | 20260507 06:07:39.466 | INFO | "gpu" : " 17 GB", | 20260507 06:07:39.466 | INFO | "huge" : "", | 20260507 06:07:39.466 | INFO | "huge_shared" : " 27 GB", | 20260507 06:07:39.466 | INFO | "malloc" : " 14 GB", | 20260507 06:07:39.466 | INFO | "pinned" : " 29 GB" | 20260507 06:07:39.466 | INFO | } | 20260507 06:07:39.466 | INFO | Perf report for the last 5.13885 seconds: | 20260507 06:07:39.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:07:39.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:39.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:39.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:39.467 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 06:07:39.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:39.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:39.467 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:07:39.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:39.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:39.467 | INFO | Uploaded 36GiB to GPU in 2.04404s (17.6122GiB/s) | 20260507 06:07:39.467 | INFO | Downloaded 34GiB to GPU in 1.98537s (17.1253GiB/s) | 20260507 06:07:44.603 | INFO | Analyzing chunk frames = [7687680,7688192) | 20260507 06:07:44.603 | INFO | MemInfo: { | 20260507 06:07:44.603 | INFO | "gpu" : " 17 GB", | 20260507 06:07:44.603 | INFO | "huge" : "", | 20260507 06:07:44.603 | INFO | "huge_shared" : " 27 GB", | 20260507 06:07:44.603 | INFO | "malloc" : " 14 GB", | 20260507 06:07:44.603 | INFO | "pinned" : " 29 GB" | 20260507 06:07:44.603 | INFO | } | 20260507 06:07:44.603 | INFO | Perf report for the last 5.13699 seconds: | 20260507 06:07:44.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:07:44.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:44.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:44.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:44.603 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:07:44.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:44.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:44.603 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:07:44.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:44.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:44.604 | INFO | Uploaded 36GiB to GPU in 1.9707s (18.2677GiB/s) | 20260507 06:07:44.604 | INFO | Downloaded 34GiB to GPU in 1.95587s (17.3836GiB/s) | 20260507 06:07:44.700 | INFO | Starting a round of Baz IO | 20260507 06:07:47.800 | INFO | Populating RTMetrics report | 20260507 06:07:47.800 | INFO | RTMetrics report populated | 20260507 06:07:47.800 | INFO | Opening tmp RTMetrics json file | 20260507 06:07:47.807 | INFO | Serializing RTMetrics to json | 20260507 06:07:47.818 | INFO | Serialization done, swapping tmp file now... | 20260507 06:07:47.818 | INFO | RTMetrics done | 20260507 06:07:47.889 | INFO | Finished a round of Baz IO: Wrote 1.677 GiB in 3.188 seconds | 20260507 06:07:47.889 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.65% / 0.00% / 67.35% | 20260507 06:07:47.889 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.129135% / 3.97364e-06% / 5.5631e-05% | 20260507 06:07:47.889 | INFO | Baz IO overall: | 20260507 06:07:47.889 | INFO | BytesWritten : 3884990023540 | 20260507 06:07:47.889 | INFO | NumEvents : 1862319305539 | 20260507 06:07:49.746 | INFO | Analyzing chunk frames = [7688192,7688704) | 20260507 06:07:49.746 | INFO | MemInfo: { | 20260507 06:07:49.746 | INFO | "gpu" : " 17 GB", | 20260507 06:07:49.746 | INFO | "huge" : "", | 20260507 06:07:49.746 | INFO | "huge_shared" : " 27 GB", | 20260507 06:07:49.746 | INFO | "malloc" : " 14 GB", | 20260507 06:07:49.746 | INFO | "pinned" : " 29 GB" | 20260507 06:07:49.746 | INFO | } | 20260507 06:07:49.746 | INFO | Perf report for the last 5.14278 seconds: | 20260507 06:07:49.746 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:07:49.746 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:49.746 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:49.746 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:49.746 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 06:07:49.746 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:49.746 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:07:49.746 | INFO | PRE_HQ : 59.1%, 1.8, 54.6ms, 0 | 20260507 06:07:49.746 | INFO | BAZWRITER : 61.6%, 3.2, 99.9ms, 0 | 20260507 06:07:49.746 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:07:49.746 | INFO | Uploaded 36GiB to GPU in 2.04733s (17.5839GiB/s) | 20260507 06:07:49.746 | INFO | Downloaded 38GiB to GPU in 2.18153s (17.4189GiB/s) | 20260507 06:07:54.889 | INFO | Analyzing chunk frames = [7688704,7689216) | 20260507 06:07:54.889 | INFO | MemInfo: { | 20260507 06:07:54.889 | INFO | "gpu" : " 17 GB", | 20260507 06:07:54.889 | INFO | "huge" : "", | 20260507 06:07:54.889 | INFO | "huge_shared" : " 27 GB", | 20260507 06:07:54.889 | INFO | "malloc" : " 14 GB", | 20260507 06:07:54.889 | INFO | "pinned" : " 29 GB" | 20260507 06:07:54.889 | INFO | } | 20260507 06:07:54.889 | INFO | Perf report for the last 5.14334 seconds: | 20260507 06:07:54.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:07:54.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:54.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:54.889 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:54.890 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 06:07:54.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:54.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:54.890 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 06:07:54.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:54.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:07:54.890 | INFO | Uploaded 36GiB to GPU in 1.98432s (18.1422GiB/s) | 20260507 06:07:54.890 | INFO | Downloaded 34GiB to GPU in 1.9245s (17.6669GiB/s) | 20260507 06:08:00.026 | INFO | Analyzing chunk frames = [7689216,7689728) | 20260507 06:08:00.026 | INFO | MemInfo: { | 20260507 06:08:00.026 | INFO | "gpu" : " 17 GB", | 20260507 06:08:00.026 | INFO | "huge" : "", | 20260507 06:08:00.026 | INFO | "huge_shared" : " 27 GB", | 20260507 06:08:00.026 | INFO | "malloc" : " 14 GB", | 20260507 06:08:00.026 | INFO | "pinned" : " 29 GB" | 20260507 06:08:00.026 | INFO | } | 20260507 06:08:00.026 | INFO | Perf report for the last 5.13702 seconds: | 20260507 06:08:00.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:08:00.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:00.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:00.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:00.026 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 06:08:00.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:00.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:00.026 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:08:00.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:00.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:00.027 | INFO | Uploaded 36GiB to GPU in 2.05533s (17.5154GiB/s) | 20260507 06:08:00.027 | INFO | Downloaded 34GiB to GPU in 1.95057s (17.4308GiB/s) | 20260507 06:08:05.165 | INFO | Analyzing chunk frames = [7689728,7690240) | 20260507 06:08:05.165 | INFO | MemInfo: { | 20260507 06:08:05.165 | INFO | "gpu" : " 17 GB", | 20260507 06:08:05.165 | INFO | "huge" : "", | 20260507 06:08:05.165 | INFO | "huge_shared" : " 27 GB", | 20260507 06:08:05.165 | INFO | "malloc" : " 14 GB", | 20260507 06:08:05.165 | INFO | "pinned" : " 29 GB" | 20260507 06:08:05.165 | INFO | } | 20260507 06:08:05.165 | INFO | Perf report for the last 5.13843 seconds: | 20260507 06:08:05.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:08:05.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:05.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:05.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:05.165 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 06:08:05.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:05.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:05.165 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:08:05.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:05.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:05.165 | INFO | Uploaded 36GiB to GPU in 2.0094s (17.9158GiB/s) | 20260507 06:08:05.165 | INFO | Downloaded 34GiB to GPU in 1.94269s (17.5015GiB/s) | 20260507 06:08:10.305 | INFO | Analyzing chunk frames = [7690240,7690752) | 20260507 06:08:10.305 | INFO | MemInfo: { | 20260507 06:08:10.305 | INFO | "gpu" : " 17 GB", | 20260507 06:08:10.305 | INFO | "huge" : "", | 20260507 06:08:10.305 | INFO | "huge_shared" : " 27 GB", | 20260507 06:08:10.305 | INFO | "malloc" : " 14 GB", | 20260507 06:08:10.305 | INFO | "pinned" : " 29 GB" | 20260507 06:08:10.305 | INFO | } | 20260507 06:08:10.305 | INFO | Perf report for the last 5.14047 seconds: | 20260507 06:08:10.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:08:10.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:10.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:10.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:10.305 | INFO | ANALYSIS : 60.2%, 3.0, 94.0ms, 0 | 20260507 06:08:10.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:10.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:10.306 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:08:10.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:10.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:10.306 | INFO | Uploaded 36GiB to GPU in 2.05791s (17.4935GiB/s) | 20260507 06:08:10.306 | INFO | Downloaded 34GiB to GPU in 1.95191s (17.4188GiB/s) | 20260507 06:08:15.439 | INFO | Analyzing chunk frames = [7690752,7691264) | 20260507 06:08:15.439 | INFO | MemInfo: { | 20260507 06:08:15.439 | INFO | "gpu" : " 17 GB", | 20260507 06:08:15.439 | INFO | "huge" : "", | 20260507 06:08:15.439 | INFO | "huge_shared" : " 27 GB", | 20260507 06:08:15.439 | INFO | "malloc" : " 14 GB", | 20260507 06:08:15.439 | INFO | "pinned" : " 29 GB" | 20260507 06:08:15.440 | INFO | } | 20260507 06:08:15.440 | INFO | Perf report for the last 5.13424 seconds: | 20260507 06:08:15.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:08:15.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:15.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:15.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:15.440 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 06:08:15.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:15.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:15.440 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 06:08:15.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:15.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:15.440 | INFO | Uploaded 36GiB to GPU in 2.07181s (17.3761GiB/s) | 20260507 06:08:15.440 | INFO | Downloaded 34GiB to GPU in 1.92599s (17.6533GiB/s) | 20260507 06:08:20.575 | INFO | Analyzing chunk frames = [7691264,7691776) | 20260507 06:08:20.575 | INFO | MemInfo: { | 20260507 06:08:20.575 | INFO | "gpu" : " 17 GB", | 20260507 06:08:20.575 | INFO | "huge" : "", | 20260507 06:08:20.575 | INFO | "huge_shared" : " 27 GB", | 20260507 06:08:20.575 | INFO | "malloc" : " 14 GB", | 20260507 06:08:20.575 | INFO | "pinned" : " 29 GB" | 20260507 06:08:20.575 | INFO | } | 20260507 06:08:20.575 | INFO | Perf report for the last 5.13582 seconds: | 20260507 06:08:20.575 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:08:20.575 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:20.575 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:20.575 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:20.575 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 06:08:20.575 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:20.575 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:20.575 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:08:20.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:20.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:20.576 | INFO | Uploaded 36GiB to GPU in 2.03953s (17.6512GiB/s) | 20260507 06:08:20.576 | INFO | Downloaded 34GiB to GPU in 1.97685s (17.1991GiB/s) | 20260507 06:08:25.721 | INFO | Analyzing chunk frames = [7691776,7692288) | 20260507 06:08:25.721 | INFO | MemInfo: { | 20260507 06:08:25.721 | INFO | "gpu" : " 17 GB", | 20260507 06:08:25.721 | INFO | "huge" : "", | 20260507 06:08:25.721 | INFO | "huge_shared" : " 27 GB", | 20260507 06:08:25.721 | INFO | "malloc" : " 14 GB", | 20260507 06:08:25.721 | INFO | "pinned" : " 29 GB" | 20260507 06:08:25.721 | INFO | } | 20260507 06:08:25.721 | INFO | Perf report for the last 5.14564 seconds: | 20260507 06:08:25.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:08:25.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:25.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:25.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:25.721 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 06:08:25.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:25.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:25.721 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:08:25.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:25.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:25.721 | INFO | Uploaded 36GiB to GPU in 1.96873s (18.2859GiB/s) | 20260507 06:08:25.721 | INFO | Downloaded 34GiB to GPU in 1.95858s (17.3595GiB/s) | 20260507 06:08:25.721 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:08:25.721 | INFO | Total mem usage: 18 GiB | 20260507 06:08:25.722 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:08:25.722 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:08:25.722 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:08:25.722 | INFO | High Water Breakdown: | 20260507 06:08:25.722 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:08:25.722 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:08:25.722 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:08:25.722 | INFO | ------------------------------------------ | 20260507 06:08:25.722 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:08:25.722 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:08:25.722 | INFO | Total mem usage: 26.5 GiB | 20260507 06:08:25.722 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:08:25.722 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:08:25.722 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:08:25.722 | INFO | High Water Breakdown: | 20260507 06:08:25.722 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:08:25.722 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:08:25.722 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:08:25.722 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:08:25.722 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:08:25.722 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:08:25.722 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:08:25.722 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:08:25.722 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:08:25.722 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:08:25.722 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:08:25.722 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:08:25.722 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:08:25.722 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:08:25.722 | INFO | ------------------------------------------ | 20260507 06:08:25.722 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:08:25.722 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:08:25.722 | INFO | Total mem usage: 25.6 GiB | 20260507 06:08:25.722 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:08:25.722 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:08:25.722 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:08:25.722 | INFO | High Water Breakdown: | 20260507 06:08:25.722 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:08:25.722 | INFO | ------------------------------------------ | 20260507 06:08:25.722 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:08:25.722 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:08:25.722 | INFO | Total mem usage: 15.7 GiB | 20260507 06:08:25.722 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:08:25.722 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:08:25.722 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:08:25.722 | INFO | High Water Breakdown: | 20260507 06:08:25.722 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:08:25.722 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:08:25.722 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:08:25.722 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:08:25.722 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:08:25.722 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:08:25.722 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:08:25.722 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:08:25.722 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:08:25.722 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:08:25.722 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:08:25.722 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:08:25.722 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:08:25.722 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:08:25.722 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:08:25.722 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:08:25.722 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:08:25.722 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:08:25.722 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:08:25.722 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:08:25.722 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:08:25.722 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:08:25.722 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:08:25.722 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:08:25.722 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:08:25.722 | INFO | ------------------------------------------ | 20260507 06:08:25.722 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:08:25.722 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:08:25.722 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:08:25.722 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:08:25.722 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.44e+01, 1.32e+01 / 1.08e+00 | 20260507 06:08:25.722 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.00e-01, 9.35e-03 / 5.40e-03 | 20260507 06:08:25.722 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.46e+00, 4.99e-01 / 2.82e-01 | 20260507 06:08:25.722 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.45e+00, 1.65e-01 / 1.39e-01 | 20260507 06:08:25.722 | INFO | PerfReport: 16.14%, DME -- 3.96e+01 / 4.39e+01, 3.58e+00 / 1.15e+00 | 20260507 06:08:25.722 | INFO | PerfReport: 57.80%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.32e+00 / 1.34e+00 | 20260507 06:08:25.722 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.72e+00, 3.84e-01 / 1.95e-01 | 20260507 06:08:25.722 | INFO | PerfReport: 2.96%, Metrics -- 9.09e-01 / 1.24e+00, 2.41e-01 / 1.38e-01 | 20260507 06:08:25.722 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.06e-01 / 3.64e-01, 5.02e-02 / 2.06e-02 | 20260507 06:08:25.722 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:08:25.722 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:08:25.722 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:08:25.722 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 06:08:25.722 | INFO | PerfReport: 21.95%, Upload -- 2.11e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 06:08:25.722 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 06:08:25.722 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 06:08:25.723 | INFO | PerfReport: 1.60%, LockUpload -- 1.54e+00 / 0.00e+00, 4.89e+00 / 0.00e+00 | 20260507 06:08:25.723 | INFO | PerfReport: 1.16%, LockDownload -- 1.11e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260507 06:08:25.723 | INFO | PerfReport: 22.00%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 06:08:25.723 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.11e-02 / 0.00e+00 | 20260507 06:08:25.818 | INFO | Starting a round of Baz IO | 20260507 06:08:28.913 | INFO | Populating RTMetrics report | 20260507 06:08:28.913 | INFO | RTMetrics report populated | 20260507 06:08:28.913 | INFO | Opening tmp RTMetrics json file | 20260507 06:08:28.914 | INFO | Serializing RTMetrics to json | 20260507 06:08:28.915 | INFO | Serialization done, swapping tmp file now... | 20260507 06:08:28.916 | INFO | RTMetrics done | 20260507 06:08:29.005 | INFO | Finished a round of Baz IO: Wrote 1.677 GiB in 3.187 seconds | 20260507 06:08:29.005 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.65% / 0.00% / 67.35% | 20260507 06:08:29.005 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.129557% / 3.97364e-06% / 5.96046e-05% | 20260507 06:08:29.005 | INFO | Baz IO overall: | 20260507 06:08:29.005 | INFO | BytesWritten : 3886790400493 | 20260507 06:08:29.005 | INFO | NumEvents : 1863163336447 | 20260507 06:08:30.866 | INFO | Analyzing chunk frames = [7692288,7692800) | 20260507 06:08:30.866 | INFO | MemInfo: { | 20260507 06:08:30.866 | INFO | "gpu" : " 17 GB", | 20260507 06:08:30.866 | INFO | "huge" : "", | 20260507 06:08:30.866 | INFO | "huge_shared" : " 27 GB", | 20260507 06:08:30.866 | INFO | "malloc" : " 14 GB", | 20260507 06:08:30.866 | INFO | "pinned" : " 29 GB" | 20260507 06:08:30.866 | INFO | } | 20260507 06:08:30.866 | INFO | Perf report for the last 5.14529 seconds: | 20260507 06:08:30.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:08:30.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:30.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:30.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:30.866 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 06:08:30.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:30.866 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 06:08:30.867 | INFO | PRE_HQ : 58.8%, 1.8, 54.6ms, 0 | 20260507 06:08:30.867 | INFO | BAZWRITER : 61.5%, 3.1, 99.0ms, 0 | 20260507 06:08:30.867 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:08:30.867 | INFO | Uploaded 36GiB to GPU in 2.03848s (17.6602GiB/s) | 20260507 06:08:30.867 | INFO | Downloaded 38GiB to GPU in 2.17632s (17.4606GiB/s) | 20260507 06:08:36.008 | INFO | Analyzing chunk frames = [7692800,7693312) | 20260507 06:08:36.008 | INFO | MemInfo: { | 20260507 06:08:36.008 | INFO | "gpu" : " 17 GB", | 20260507 06:08:36.008 | INFO | "huge" : "", | 20260507 06:08:36.008 | INFO | "huge_shared" : " 27 GB", | 20260507 06:08:36.008 | INFO | "malloc" : " 14 GB", | 20260507 06:08:36.008 | INFO | "pinned" : " 29 GB" | 20260507 06:08:36.008 | INFO | } | 20260507 06:08:36.008 | INFO | Perf report for the last 5.14172 seconds: | 20260507 06:08:36.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:08:36.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:36.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:36.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:36.008 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:08:36.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:36.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:36.008 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:08:36.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:36.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:36.008 | INFO | Uploaded 36GiB to GPU in 1.97813s (18.199GiB/s) | 20260507 06:08:36.008 | INFO | Downloaded 34GiB to GPU in 1.92954s (17.6208GiB/s) | 20260507 06:08:41.142 | INFO | Analyzing chunk frames = [7693312,7693824) | 20260507 06:08:41.142 | INFO | MemInfo: { | 20260507 06:08:41.142 | INFO | "gpu" : " 17 GB", | 20260507 06:08:41.142 | INFO | "huge" : "", | 20260507 06:08:41.142 | INFO | "huge_shared" : " 27 GB", | 20260507 06:08:41.142 | INFO | "malloc" : " 14 GB", | 20260507 06:08:41.142 | INFO | "pinned" : " 29 GB" | 20260507 06:08:41.142 | INFO | } | 20260507 06:08:41.142 | INFO | Perf report for the last 5.13364 seconds: | 20260507 06:08:41.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:08:41.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:41.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:41.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:41.142 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 06:08:41.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:41.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:41.142 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:08:41.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:41.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:41.142 | INFO | Uploaded 36GiB to GPU in 2.03262s (17.7111GiB/s) | 20260507 06:08:41.142 | INFO | Downloaded 34GiB to GPU in 1.91692s (17.7368GiB/s) | 20260507 06:08:46.276 | INFO | Analyzing chunk frames = [7693824,7694336) | 20260507 06:08:46.276 | INFO | MemInfo: { | 20260507 06:08:46.276 | INFO | "gpu" : " 17 GB", | 20260507 06:08:46.276 | INFO | "huge" : "", | 20260507 06:08:46.276 | INFO | "huge_shared" : " 27 GB", | 20260507 06:08:46.276 | INFO | "malloc" : " 14 GB", | 20260507 06:08:46.276 | INFO | "pinned" : " 29 GB" | 20260507 06:08:46.277 | INFO | } | 20260507 06:08:46.277 | INFO | Perf report for the last 5.13475 seconds: | 20260507 06:08:46.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:08:46.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:46.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:46.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:46.277 | INFO | ANALYSIS : 60.6%, 2.9, 93.4ms, 0 | 20260507 06:08:46.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:46.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:46.277 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:08:46.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:46.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:46.277 | INFO | Uploaded 36GiB to GPU in 2.0527s (17.5379GiB/s) | 20260507 06:08:46.277 | INFO | Downloaded 34GiB to GPU in 1.94465s (17.4839GiB/s) | 20260507 06:08:51.418 | INFO | Analyzing chunk frames = [7694336,7694848) | 20260507 06:08:51.418 | INFO | MemInfo: { | 20260507 06:08:51.418 | INFO | "gpu" : " 17 GB", | 20260507 06:08:51.418 | INFO | "huge" : "", | 20260507 06:08:51.418 | INFO | "huge_shared" : " 27 GB", | 20260507 06:08:51.418 | INFO | "malloc" : " 14 GB", | 20260507 06:08:51.418 | INFO | "pinned" : " 29 GB" | 20260507 06:08:51.418 | INFO | } | 20260507 06:08:51.418 | INFO | Perf report for the last 5.1414 seconds: | 20260507 06:08:51.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:08:51.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:51.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:51.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:51.418 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 06:08:51.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:51.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:51.418 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:08:51.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:51.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:51.418 | INFO | Uploaded 36GiB to GPU in 2.05735s (17.4982GiB/s) | 20260507 06:08:51.418 | INFO | Downloaded 34GiB to GPU in 1.94756s (17.4578GiB/s) | 20260507 06:08:56.558 | INFO | Analyzing chunk frames = [7694848,7695360) | 20260507 06:08:56.558 | INFO | MemInfo: { | 20260507 06:08:56.558 | INFO | "gpu" : " 17 GB", | 20260507 06:08:56.558 | INFO | "huge" : "", | 20260507 06:08:56.558 | INFO | "huge_shared" : " 27 GB", | 20260507 06:08:56.558 | INFO | "malloc" : " 14 GB", | 20260507 06:08:56.558 | INFO | "pinned" : " 29 GB" | 20260507 06:08:56.558 | INFO | } | 20260507 06:08:56.558 | INFO | Perf report for the last 5.13977 seconds: | 20260507 06:08:56.558 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:08:56.558 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:56.558 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:56.558 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:56.558 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:08:56.558 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:56.558 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:56.558 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 06:08:56.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:56.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:08:56.558 | INFO | Uploaded 36GiB to GPU in 2.03865s (17.6587GiB/s) | 20260507 06:08:56.558 | INFO | Downloaded 34GiB to GPU in 1.90041s (17.8909GiB/s) | 20260507 06:09:01.699 | INFO | Analyzing chunk frames = [7695360,7695872) | 20260507 06:09:01.699 | INFO | MemInfo: { | 20260507 06:09:01.699 | INFO | "gpu" : " 17 GB", | 20260507 06:09:01.699 | INFO | "huge" : "", | 20260507 06:09:01.699 | INFO | "huge_shared" : " 27 GB", | 20260507 06:09:01.699 | INFO | "malloc" : " 14 GB", | 20260507 06:09:01.699 | INFO | "pinned" : " 29 GB" | 20260507 06:09:01.699 | INFO | } | 20260507 06:09:01.699 | INFO | Perf report for the last 5.14136 seconds: | 20260507 06:09:01.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:09:01.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:01.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:01.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:01.699 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:09:01.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:01.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:01.699 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:09:01.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:01.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:01.699 | INFO | Uploaded 36GiB to GPU in 1.93254s (18.6283GiB/s) | 20260507 06:09:01.699 | INFO | Downloaded 34GiB to GPU in 1.97099s (17.2502GiB/s) | 20260507 06:09:06.838 | INFO | Analyzing chunk frames = [7695872,7696384) | 20260507 06:09:06.838 | INFO | MemInfo: { | 20260507 06:09:06.838 | INFO | "gpu" : " 17 GB", | 20260507 06:09:06.838 | INFO | "huge" : "", | 20260507 06:09:06.838 | INFO | "huge_shared" : " 27 GB", | 20260507 06:09:06.838 | INFO | "malloc" : " 14 GB", | 20260507 06:09:06.838 | INFO | "pinned" : " 29 GB" | 20260507 06:09:06.838 | INFO | } | 20260507 06:09:06.838 | INFO | Perf report for the last 5.13893 seconds: | 20260507 06:09:06.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:09:06.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:06.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:06.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:06.838 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:09:06.838 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:06.838 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:06.838 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:09:06.838 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:06.838 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:06.838 | INFO | Uploaded 36GiB to GPU in 1.99439s (18.0506GiB/s) | 20260507 06:09:06.838 | INFO | Downloaded 34GiB to GPU in 1.96165s (17.3324GiB/s) | 20260507 06:09:06.935 | INFO | Starting a round of Baz IO | 20260507 06:09:10.030 | INFO | Populating RTMetrics report | 20260507 06:09:10.030 | INFO | RTMetrics report populated | 20260507 06:09:10.030 | INFO | Opening tmp RTMetrics json file | 20260507 06:09:10.035 | INFO | Serializing RTMetrics to json | 20260507 06:09:10.048 | INFO | Serialization done, swapping tmp file now... | 20260507 06:09:10.048 | INFO | RTMetrics done | 20260507 06:09:10.119 | INFO | Finished a round of Baz IO: Wrote 1.676 GiB in 3.184 seconds | 20260507 06:09:10.120 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.65% / 0.00% / 67.35% | 20260507 06:09:10.120 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.128714% / 3.97364e-06% / 8.34465e-05% | 20260507 06:09:10.120 | INFO | Baz IO overall: | 20260507 06:09:10.120 | INFO | BytesWritten : 3888589686984 | 20260507 06:09:10.120 | INFO | NumEvents : 1864006350075 | 20260507 06:09:11.980 | INFO | Analyzing chunk frames = [7696384,7696896) | 20260507 06:09:11.980 | INFO | MemInfo: { | 20260507 06:09:11.980 | INFO | "gpu" : " 17 GB", | 20260507 06:09:11.980 | INFO | "huge" : "", | 20260507 06:09:11.980 | INFO | "huge_shared" : " 27 GB", | 20260507 06:09:11.980 | INFO | "malloc" : " 14 GB", | 20260507 06:09:11.980 | INFO | "pinned" : " 29 GB" | 20260507 06:09:11.980 | INFO | } | 20260507 06:09:11.980 | INFO | Perf report for the last 5.14237 seconds: | 20260507 06:09:11.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:09:11.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:11.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:11.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:11.980 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 06:09:11.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:11.980 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 06:09:11.980 | INFO | PRE_HQ : 58.9%, 1.8, 54.5ms, 0 | 20260507 06:09:11.981 | INFO | BAZWRITER : 61.4%, 3.2, 100.4ms, 0 | 20260507 06:09:11.981 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:09:11.981 | INFO | Uploaded 36GiB to GPU in 2.07218s (17.373GiB/s) | 20260507 06:09:11.981 | INFO | Downloaded 38GiB to GPU in 2.18019s (17.4297GiB/s) | 20260507 06:09:17.115 | INFO | Analyzing chunk frames = [7696896,7697408) | 20260507 06:09:17.115 | INFO | MemInfo: { | 20260507 06:09:17.115 | INFO | "gpu" : " 17 GB", | 20260507 06:09:17.115 | INFO | "huge" : "", | 20260507 06:09:17.115 | INFO | "huge_shared" : " 27 GB", | 20260507 06:09:17.115 | INFO | "malloc" : " 14 GB", | 20260507 06:09:17.115 | INFO | "pinned" : " 29 GB" | 20260507 06:09:17.115 | INFO | } | 20260507 06:09:17.115 | INFO | Perf report for the last 5.13466 seconds: | 20260507 06:09:17.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:09:17.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:17.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:17.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:17.115 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 06:09:17.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:17.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:17.115 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:09:17.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:17.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:17.115 | INFO | Uploaded 36GiB to GPU in 1.98915s (18.0982GiB/s) | 20260507 06:09:17.115 | INFO | Downloaded 34GiB to GPU in 1.92952s (17.6209GiB/s) | 20260507 06:09:22.254 | INFO | Analyzing chunk frames = [7697408,7697920) | 20260507 06:09:22.254 | INFO | MemInfo: { | 20260507 06:09:22.254 | INFO | "gpu" : " 17 GB", | 20260507 06:09:22.254 | INFO | "huge" : "", | 20260507 06:09:22.254 | INFO | "huge_shared" : " 27 GB", | 20260507 06:09:22.254 | INFO | "malloc" : " 14 GB", | 20260507 06:09:22.254 | INFO | "pinned" : " 29 GB" | 20260507 06:09:22.254 | INFO | } | 20260507 06:09:22.254 | INFO | Perf report for the last 5.13895 seconds: | 20260507 06:09:22.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:09:22.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:22.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:22.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:22.254 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 06:09:22.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:22.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:22.254 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 06:09:22.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:22.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:22.254 | INFO | Uploaded 36GiB to GPU in 2.05152s (17.5479GiB/s) | 20260507 06:09:22.254 | INFO | Downloaded 34GiB to GPU in 1.95152s (17.4223GiB/s) | 20260507 06:09:27.388 | INFO | Analyzing chunk frames = [7697920,7698432) | 20260507 06:09:27.388 | INFO | MemInfo: { | 20260507 06:09:27.388 | INFO | "gpu" : " 17 GB", | 20260507 06:09:27.388 | INFO | "huge" : "", | 20260507 06:09:27.388 | INFO | "huge_shared" : " 27 GB", | 20260507 06:09:27.388 | INFO | "malloc" : " 14 GB", | 20260507 06:09:27.388 | INFO | "pinned" : " 29 GB" | 20260507 06:09:27.388 | INFO | } | 20260507 06:09:27.389 | INFO | Perf report for the last 5.13445 seconds: | 20260507 06:09:27.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:09:27.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:27.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:27.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:27.389 | INFO | ANALYSIS : 59.8%, 3.0, 92.7ms, 0 | 20260507 06:09:27.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:27.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:27.389 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:09:27.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:27.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:27.389 | INFO | Uploaded 36GiB to GPU in 2.02432s (17.7837GiB/s) | 20260507 06:09:27.389 | INFO | Downloaded 34GiB to GPU in 1.93956s (17.5298GiB/s) | 20260507 06:09:32.528 | INFO | Analyzing chunk frames = [7698432,7698944) | 20260507 06:09:32.528 | INFO | MemInfo: { | 20260507 06:09:32.528 | INFO | "gpu" : " 17 GB", | 20260507 06:09:32.528 | INFO | "huge" : "", | 20260507 06:09:32.528 | INFO | "huge_shared" : " 27 GB", | 20260507 06:09:32.528 | INFO | "malloc" : " 14 GB", | 20260507 06:09:32.528 | INFO | "pinned" : " 29 GB" | 20260507 06:09:32.528 | INFO | } | 20260507 06:09:32.528 | INFO | Perf report for the last 5.13961 seconds: | 20260507 06:09:32.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:09:32.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:32.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:32.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:32.528 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 06:09:32.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:32.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:32.528 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:09:32.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:32.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:32.529 | INFO | Uploaded 36GiB to GPU in 2.03358s (17.7028GiB/s) | 20260507 06:09:32.529 | INFO | Downloaded 34GiB to GPU in 1.94033s (17.5228GiB/s) | 20260507 06:09:37.672 | INFO | Analyzing chunk frames = [7698944,7699456) | 20260507 06:09:37.673 | INFO | MemInfo: { | 20260507 06:09:37.673 | INFO | "gpu" : " 17 GB", | 20260507 06:09:37.673 | INFO | "huge" : "", | 20260507 06:09:37.673 | INFO | "huge_shared" : " 27 GB", | 20260507 06:09:37.673 | INFO | "malloc" : " 14 GB", | 20260507 06:09:37.673 | INFO | "pinned" : " 29 GB" | 20260507 06:09:37.673 | INFO | } | 20260507 06:09:37.673 | INFO | Perf report for the last 5.14444 seconds: | 20260507 06:09:37.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:09:37.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:37.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:37.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:37.673 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 06:09:37.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:37.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:37.673 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 06:09:37.673 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:37.673 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:37.673 | INFO | Uploaded 36GiB to GPU in 2.03355s (17.703GiB/s) | 20260507 06:09:37.673 | INFO | Downloaded 34GiB to GPU in 1.90897s (17.8107GiB/s) | 20260507 06:09:42.819 | INFO | Analyzing chunk frames = [7699456,7699968) | 20260507 06:09:42.819 | INFO | MemInfo: { | 20260507 06:09:42.819 | INFO | "gpu" : " 17 GB", | 20260507 06:09:42.819 | INFO | "huge" : "", | 20260507 06:09:42.819 | INFO | "huge_shared" : " 27 GB", | 20260507 06:09:42.819 | INFO | "malloc" : " 14 GB", | 20260507 06:09:42.819 | INFO | "pinned" : " 29 GB" | 20260507 06:09:42.819 | INFO | } | 20260507 06:09:42.819 | INFO | Perf report for the last 5.14639 seconds: | 20260507 06:09:42.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:09:42.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:42.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:42.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:42.819 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 06:09:42.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:42.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:42.819 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:09:42.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:42.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:42.819 | INFO | Uploaded 36GiB to GPU in 1.97318s (18.2446GiB/s) | 20260507 06:09:42.819 | INFO | Downloaded 34GiB to GPU in 1.98749s (17.107GiB/s) | 20260507 06:09:47.954 | INFO | Analyzing chunk frames = [7699968,7700480) | 20260507 06:09:47.954 | INFO | MemInfo: { | 20260507 06:09:47.954 | INFO | "gpu" : " 17 GB", | 20260507 06:09:47.954 | INFO | "huge" : "", | 20260507 06:09:47.955 | INFO | "huge_shared" : " 27 GB", | 20260507 06:09:47.955 | INFO | "malloc" : " 14 GB", | 20260507 06:09:47.955 | INFO | "pinned" : " 29 GB" | 20260507 06:09:47.955 | INFO | } | 20260507 06:09:47.955 | INFO | Perf report for the last 5.13588 seconds: | 20260507 06:09:47.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:09:47.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:47.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:47.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:47.955 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 06:09:47.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:47.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:47.955 | INFO | PRE_HQ : 29.4%, 1.0, 15.3ms, 0 | 20260507 06:09:47.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:47.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:47.955 | INFO | Uploaded 36GiB to GPU in 2.00869s (17.9221GiB/s) | 20260507 06:09:47.955 | INFO | Downloaded 34GiB to GPU in 1.97429s (17.2214GiB/s) | 20260507 06:09:47.955 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:09:47.955 | INFO | Total mem usage: 18 GiB | 20260507 06:09:47.955 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:09:47.955 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:09:47.955 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:09:47.955 | INFO | High Water Breakdown: | 20260507 06:09:47.955 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:09:47.955 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:09:47.955 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:09:47.955 | INFO | ------------------------------------------ | 20260507 06:09:47.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:09:47.955 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:09:47.955 | INFO | Total mem usage: 26.5 GiB | 20260507 06:09:47.955 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:09:47.955 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:09:47.955 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 06:09:47.955 | INFO | High Water Breakdown: | 20260507 06:09:47.955 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:09:47.955 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:09:47.955 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:09:47.955 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:09:47.955 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:09:47.955 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:09:47.955 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:09:47.955 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:09:47.955 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:09:47.955 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:09:47.955 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:09:47.955 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:09:47.955 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:09:47.955 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:09:47.955 | INFO | ------------------------------------------ | 20260507 06:09:47.955 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:09:47.955 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:09:47.955 | INFO | Total mem usage: 25.6 GiB | 20260507 06:09:47.956 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:09:47.956 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:09:47.956 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:09:47.956 | INFO | High Water Breakdown: | 20260507 06:09:47.956 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:09:47.956 | INFO | ------------------------------------------ | 20260507 06:09:47.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:09:47.956 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:09:47.956 | INFO | Total mem usage: 15.7 GiB | 20260507 06:09:47.956 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:09:47.956 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:09:47.956 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 06:09:47.956 | INFO | High Water Breakdown: | 20260507 06:09:47.956 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:09:47.956 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:09:47.956 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:09:47.956 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:09:47.956 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:09:47.956 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:09:47.956 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:09:47.956 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:09:47.956 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:09:47.956 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:09:47.956 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:09:47.956 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:09:47.956 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:09:47.956 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:09:47.956 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:09:47.956 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:09:47.956 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:09:47.956 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:09:47.956 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:09:47.956 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:09:47.956 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:09:47.956 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:09:47.956 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:09:47.956 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:09:47.956 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:09:47.956 | INFO | ------------------------------------------ | 20260507 06:09:47.956 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:09:47.956 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:09:47.956 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:09:47.956 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:09:47.956 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 1.23e+00 | 20260507 06:09:47.956 | INFO | PerfReport: 0.30%, AggregateStats -- 9.05e-02 / 1.01e-01, 9.70e-03 / 3.82e-03 | 20260507 06:09:47.956 | INFO | PerfReport: 11.70%, Baseline -- 3.59e+00 / 4.50e+00, 5.09e-01 / 2.27e-01 | 20260507 06:09:47.956 | INFO | PerfReport: 3.22%, Histogram -- 9.87e-01 / 1.49e+00, 1.65e-01 / 9.52e-02 | 20260507 06:09:47.956 | INFO | PerfReport: 16.30%, DME -- 4.00e+01 / 4.51e+01, 4.04e+00 / 1.38e+00 | 20260507 06:09:47.956 | INFO | PerfReport: 57.56%, FrameLabeling -- 1.77e+01 / 1.92e+01, 1.15e+00 / 7.35e-01 | 20260507 06:09:47.956 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.13e+00 / 2.75e+00, 3.90e-01 / 1.46e-01 | 20260507 06:09:47.956 | INFO | PerfReport: 2.97%, Metrics -- 9.11e-01 / 1.24e+00, 2.46e-01 / 1.13e-01 | 20260507 06:09:47.956 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.07e-01 / 3.63e-01, 5.16e-02 / 1.74e-02 | 20260507 06:09:47.956 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:09:47.956 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:09:47.956 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:09:47.956 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 06:09:47.956 | INFO | PerfReport: 21.86%, Upload -- 2.10e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260507 06:09:47.956 | INFO | PerfReport: 21.36%, Download -- 2.05e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 06:09:47.956 | INFO | PerfReport: 31.88%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:09:47.956 | INFO | PerfReport: 1.68%, LockUpload -- 1.61e+00 / 0.00e+00, 5.06e+00 / 0.00e+00 | 20260507 06:09:47.956 | INFO | PerfReport: 1.23%, LockDownload -- 1.18e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260507 06:09:47.956 | INFO | PerfReport: 21.98%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 06:09:47.956 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260507 06:09:48.052 | INFO | Starting a round of Baz IO | 20260507 06:09:51.154 | INFO | Populating RTMetrics report | 20260507 06:09:51.155 | INFO | RTMetrics report populated | 20260507 06:09:51.155 | INFO | Opening tmp RTMetrics json file | 20260507 06:09:51.155 | INFO | Serializing RTMetrics to json | 20260507 06:09:51.157 | INFO | Serialization done, swapping tmp file now... | 20260507 06:09:51.157 | INFO | RTMetrics done | 20260507 06:09:51.243 | INFO | Finished a round of Baz IO: Wrote 1.676 GiB in 3.191 seconds | 20260507 06:09:51.243 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.65% / 0.00% / 67.35% | 20260507 06:09:51.243 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.12846% / 3.97364e-06% / 0.000103315% | 20260507 06:09:51.243 | INFO | Baz IO overall: | 20260507 06:09:51.243 | INFO | BytesWritten : 3890389749828 | 20260507 06:09:51.243 | INFO | NumEvents : 1864850024952 | 20260507 06:09:53.082 | INFO | Analyzing chunk frames = [7700480,7700992) | 20260507 06:09:53.082 | INFO | MemInfo: { | 20260507 06:09:53.082 | INFO | "gpu" : " 17 GB", | 20260507 06:09:53.082 | INFO | "huge" : "", | 20260507 06:09:53.082 | INFO | "huge_shared" : " 27 GB", | 20260507 06:09:53.082 | INFO | "malloc" : " 14 GB", | 20260507 06:09:53.082 | INFO | "pinned" : " 29 GB" | 20260507 06:09:53.082 | INFO | } | 20260507 06:09:53.082 | INFO | Perf report for the last 5.12739 seconds: | 20260507 06:09:53.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:09:53.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:53.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:53.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:53.082 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260507 06:09:53.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:53.082 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260507 06:09:53.082 | INFO | PRE_HQ : 60.1%, 1.9, 59.6ms, 0 | 20260507 06:09:53.082 | INFO | BAZWRITER : 61.8%, 3.3, 104.6ms, 0 | 20260507 06:09:53.082 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 06:09:53.083 | INFO | Uploaded 36GiB to GPU in 2.06976s (17.3933GiB/s) | 20260507 06:09:53.083 | INFO | Downloaded 38GiB to GPU in 2.17624s (17.4613GiB/s) | 20260507 06:09:58.226 | INFO | Analyzing chunk frames = [7700992,7701504) | 20260507 06:09:58.226 | INFO | MemInfo: { | 20260507 06:09:58.226 | INFO | "gpu" : " 17 GB", | 20260507 06:09:58.226 | INFO | "huge" : "", | 20260507 06:09:58.226 | INFO | "huge_shared" : " 27 GB", | 20260507 06:09:58.226 | INFO | "malloc" : " 14 GB", | 20260507 06:09:58.226 | INFO | "pinned" : " 29 GB" | 20260507 06:09:58.226 | INFO | } | 20260507 06:09:58.226 | INFO | Perf report for the last 5.1435 seconds: | 20260507 06:09:58.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:09:58.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:58.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:58.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:58.226 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 06:09:58.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:58.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:58.226 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:09:58.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:58.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:09:58.226 | INFO | Uploaded 36GiB to GPU in 2.00471s (17.9577GiB/s) | 20260507 06:09:58.226 | INFO | Downloaded 34GiB to GPU in 1.92845s (17.6307GiB/s) | 20260507 06:10:03.366 | INFO | Analyzing chunk frames = [7701504,7702016) | 20260507 06:10:03.366 | INFO | MemInfo: { | 20260507 06:10:03.366 | INFO | "gpu" : " 17 GB", | 20260507 06:10:03.366 | INFO | "huge" : "", | 20260507 06:10:03.366 | INFO | "huge_shared" : " 27 GB", | 20260507 06:10:03.366 | INFO | "malloc" : " 14 GB", | 20260507 06:10:03.366 | INFO | "pinned" : " 29 GB" | 20260507 06:10:03.366 | INFO | } | 20260507 06:10:03.366 | INFO | Perf report for the last 5.14035 seconds: | 20260507 06:10:03.366 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:10:03.366 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:03.366 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:03.366 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:03.366 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 06:10:03.366 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:03.366 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:03.366 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:10:03.366 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:03.366 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:03.366 | INFO | Uploaded 36GiB to GPU in 2.01042s (17.9067GiB/s) | 20260507 06:10:03.366 | INFO | Downloaded 34GiB to GPU in 1.94159s (17.5115GiB/s) | 20260507 06:10:08.514 | INFO | Analyzing chunk frames = [7702016,7702528) | 20260507 06:10:08.514 | INFO | MemInfo: { | 20260507 06:10:08.514 | INFO | "gpu" : " 17 GB", | 20260507 06:10:08.514 | INFO | "huge" : "", | 20260507 06:10:08.514 | INFO | "huge_shared" : " 27 GB", | 20260507 06:10:08.514 | INFO | "malloc" : " 14 GB", | 20260507 06:10:08.514 | INFO | "pinned" : " 29 GB" | 20260507 06:10:08.514 | INFO | } | 20260507 06:10:08.515 | INFO | Perf report for the last 5.14835 seconds: | 20260507 06:10:08.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:10:08.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:08.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:08.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:08.515 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 06:10:08.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:08.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:08.515 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:10:08.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:08.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:08.515 | INFO | Uploaded 36GiB to GPU in 1.98484s (18.1375GiB/s) | 20260507 06:10:08.515 | INFO | Downloaded 34GiB to GPU in 1.91298s (17.7733GiB/s) | 20260507 06:10:13.645 | INFO | Analyzing chunk frames = [7702528,7703040) | 20260507 06:10:13.645 | INFO | MemInfo: { | 20260507 06:10:13.645 | INFO | "gpu" : " 17 GB", | 20260507 06:10:13.646 | INFO | "huge" : "", | 20260507 06:10:13.646 | INFO | "huge_shared" : " 27 GB", | 20260507 06:10:13.646 | INFO | "malloc" : " 14 GB", | 20260507 06:10:13.646 | INFO | "pinned" : " 29 GB" | 20260507 06:10:13.646 | INFO | } | 20260507 06:10:13.646 | INFO | Perf report for the last 5.1311 seconds: | 20260507 06:10:13.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:10:13.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:13.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:13.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:13.646 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 06:10:13.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:13.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:13.646 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:10:13.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:13.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:13.646 | INFO | Uploaded 36GiB to GPU in 2.05121s (17.5506GiB/s) | 20260507 06:10:13.646 | INFO | Downloaded 34GiB to GPU in 1.90202s (17.8757GiB/s) | 20260507 06:10:18.781 | INFO | Analyzing chunk frames = [7703040,7703552) | 20260507 06:10:18.781 | INFO | MemInfo: { | 20260507 06:10:18.781 | INFO | "gpu" : " 17 GB", | 20260507 06:10:18.781 | INFO | "huge" : "", | 20260507 06:10:18.781 | INFO | "huge_shared" : " 27 GB", | 20260507 06:10:18.781 | INFO | "malloc" : " 14 GB", | 20260507 06:10:18.781 | INFO | "pinned" : " 29 GB" | 20260507 06:10:18.781 | INFO | } | 20260507 06:10:18.782 | INFO | Perf report for the last 5.13587 seconds: | 20260507 06:10:18.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:10:18.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:18.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:18.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:18.782 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:10:18.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:18.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:18.782 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:10:18.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:18.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:18.782 | INFO | Uploaded 36GiB to GPU in 2.06344s (17.4466GiB/s) | 20260507 06:10:18.782 | INFO | Downloaded 34GiB to GPU in 1.91681s (17.7378GiB/s) | 20260507 06:10:23.925 | INFO | Analyzing chunk frames = [7703552,7704064) | 20260507 06:10:23.925 | INFO | MemInfo: { | 20260507 06:10:23.925 | INFO | "gpu" : " 17 GB", | 20260507 06:10:23.925 | INFO | "huge" : "", | 20260507 06:10:23.926 | INFO | "huge_shared" : " 27 GB", | 20260507 06:10:23.926 | INFO | "malloc" : " 14 GB", | 20260507 06:10:23.926 | INFO | "pinned" : " 29 GB" | 20260507 06:10:23.926 | INFO | } | 20260507 06:10:23.926 | INFO | Perf report for the last 5.14411 seconds: | 20260507 06:10:23.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:10:23.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:23.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:23.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:23.926 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:10:23.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:23.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:23.926 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 06:10:23.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:23.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:23.926 | INFO | Uploaded 36GiB to GPU in 1.97711s (18.2084GiB/s) | 20260507 06:10:23.926 | INFO | Downloaded 34GiB to GPU in 1.98193s (17.155GiB/s) | 20260507 06:10:29.063 | INFO | Analyzing chunk frames = [7704064,7704576) | 20260507 06:10:29.063 | INFO | MemInfo: { | 20260507 06:10:29.063 | INFO | "gpu" : " 17 GB", | 20260507 06:10:29.063 | INFO | "huge" : "", | 20260507 06:10:29.063 | INFO | "huge_shared" : " 27 GB", | 20260507 06:10:29.063 | INFO | "malloc" : " 14 GB", | 20260507 06:10:29.063 | INFO | "pinned" : " 29 GB" | 20260507 06:10:29.063 | INFO | } | 20260507 06:10:29.063 | INFO | Perf report for the last 5.13743 seconds: | 20260507 06:10:29.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:10:29.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:29.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:29.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:29.063 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:10:29.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:29.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:29.063 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 06:10:29.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:29.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:29.063 | INFO | Uploaded 36GiB to GPU in 1.98465s (18.1392GiB/s) | 20260507 06:10:29.063 | INFO | Downloaded 34GiB to GPU in 1.96193s (17.3299GiB/s) | 20260507 06:10:29.160 | INFO | Starting a round of Baz IO | 20260507 06:10:32.258 | INFO | Populating RTMetrics report | 20260507 06:10:32.258 | INFO | RTMetrics report populated | 20260507 06:10:32.258 | INFO | Opening tmp RTMetrics json file | 20260507 06:10:32.265 | INFO | Serializing RTMetrics to json | 20260507 06:10:32.274 | INFO | Serialization done, swapping tmp file now... | 20260507 06:10:32.275 | INFO | RTMetrics done | 20260507 06:10:32.342 | INFO | Finished a round of Baz IO: Wrote 1.675 GiB in 3.182 seconds | 20260507 06:10:32.342 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.65% / 0.00% / 67.35% | 20260507 06:10:32.342 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.12813% / 0% / 7.54992e-05% | 20260507 06:10:32.343 | INFO | Baz IO overall: | 20260507 06:10:32.343 | INFO | BytesWritten : 3892188559582 | 20260507 06:10:32.343 | INFO | NumEvents : 1865692889980 | 20260507 06:10:34.204 | INFO | Analyzing chunk frames = [7704576,7705088) | 20260507 06:10:34.204 | INFO | MemInfo: { | 20260507 06:10:34.204 | INFO | "gpu" : " 17 GB", | 20260507 06:10:34.204 | INFO | "huge" : "", | 20260507 06:10:34.204 | INFO | "huge_shared" : " 27 GB", | 20260507 06:10:34.204 | INFO | "malloc" : " 14 GB", | 20260507 06:10:34.204 | INFO | "pinned" : " 29 GB" | 20260507 06:10:34.204 | INFO | } | 20260507 06:10:34.204 | INFO | Perf report for the last 5.14126 seconds: | 20260507 06:10:34.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:10:34.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:34.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:34.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:34.204 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 06:10:34.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:34.204 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:10:34.204 | INFO | PRE_HQ : 58.9%, 1.8, 54.6ms, 0 | 20260507 06:10:34.204 | INFO | BAZWRITER : 61.5%, 3.1, 98.6ms, 0 | 20260507 06:10:34.205 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:10:34.205 | INFO | Uploaded 36GiB to GPU in 2.06229s (17.4563GiB/s) | 20260507 06:10:34.205 | INFO | Downloaded 38GiB to GPU in 2.17599s (17.4633GiB/s) | 20260507 06:10:39.341 | INFO | Analyzing chunk frames = [7705088,7705600) | 20260507 06:10:39.341 | INFO | MemInfo: { | 20260507 06:10:39.341 | INFO | "gpu" : " 17 GB", | 20260507 06:10:39.341 | INFO | "huge" : "", | 20260507 06:10:39.341 | INFO | "huge_shared" : " 27 GB", | 20260507 06:10:39.341 | INFO | "malloc" : " 14 GB", | 20260507 06:10:39.341 | INFO | "pinned" : " 29 GB" | 20260507 06:10:39.341 | INFO | } | 20260507 06:10:39.341 | INFO | Perf report for the last 5.1369 seconds: | 20260507 06:10:39.341 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:10:39.341 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:39.341 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:39.341 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:39.341 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 06:10:39.341 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:39.341 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:39.341 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:10:39.341 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:39.341 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:39.342 | INFO | Uploaded 36GiB to GPU in 2.00131s (17.9882GiB/s) | 20260507 06:10:39.342 | INFO | Downloaded 34GiB to GPU in 1.92907s (17.6251GiB/s) | 20260507 06:10:44.484 | INFO | Analyzing chunk frames = [7705600,7706112) | 20260507 06:10:44.484 | INFO | MemInfo: { | 20260507 06:10:44.484 | INFO | "gpu" : " 17 GB", | 20260507 06:10:44.484 | INFO | "huge" : "", | 20260507 06:10:44.484 | INFO | "huge_shared" : " 27 GB", | 20260507 06:10:44.484 | INFO | "malloc" : " 14 GB", | 20260507 06:10:44.484 | INFO | "pinned" : " 29 GB" | 20260507 06:10:44.484 | INFO | } | 20260507 06:10:44.484 | INFO | Perf report for the last 5.14245 seconds: | 20260507 06:10:44.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:10:44.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:44.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:44.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:44.484 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:10:44.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:44.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:44.484 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:10:44.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:44.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:44.484 | INFO | Uploaded 36GiB to GPU in 2.03819s (17.6627GiB/s) | 20260507 06:10:44.484 | INFO | Downloaded 34GiB to GPU in 1.95229s (17.4154GiB/s) | 20260507 06:10:49.621 | INFO | Analyzing chunk frames = [7706112,7706624) | 20260507 06:10:49.621 | INFO | MemInfo: { | 20260507 06:10:49.621 | INFO | "gpu" : " 17 GB", | 20260507 06:10:49.621 | INFO | "huge" : "", | 20260507 06:10:49.621 | INFO | "huge_shared" : " 27 GB", | 20260507 06:10:49.621 | INFO | "malloc" : " 14 GB", | 20260507 06:10:49.621 | INFO | "pinned" : " 29 GB" | 20260507 06:10:49.621 | INFO | } | 20260507 06:10:49.621 | INFO | Perf report for the last 5.13763 seconds: | 20260507 06:10:49.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:10:49.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:49.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:49.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:49.621 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 06:10:49.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:49.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:49.621 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:10:49.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:49.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:49.622 | INFO | Uploaded 36GiB to GPU in 2.00935s (17.9163GiB/s) | 20260507 06:10:49.622 | INFO | Downloaded 34GiB to GPU in 1.92857s (17.6296GiB/s) | 20260507 06:10:54.767 | INFO | Analyzing chunk frames = [7706624,7707136) | 20260507 06:10:54.767 | INFO | MemInfo: { | 20260507 06:10:54.767 | INFO | "gpu" : " 17 GB", | 20260507 06:10:54.767 | INFO | "huge" : "", | 20260507 06:10:54.767 | INFO | "huge_shared" : " 27 GB", | 20260507 06:10:54.767 | INFO | "malloc" : " 14 GB", | 20260507 06:10:54.767 | INFO | "pinned" : " 29 GB" | 20260507 06:10:54.767 | INFO | } | 20260507 06:10:54.767 | INFO | Perf report for the last 5.14537 seconds: | 20260507 06:10:54.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:10:54.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:54.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:54.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:54.767 | INFO | ANALYSIS : 60.2%, 3.0, 94.0ms, 0 | 20260507 06:10:54.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:54.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:54.767 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:10:54.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:54.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:54.767 | INFO | Uploaded 36GiB to GPU in 2.0584s (17.4893GiB/s) | 20260507 06:10:54.767 | INFO | Downloaded 34GiB to GPU in 1.94862s (17.4483GiB/s) | 20260507 06:10:59.900 | INFO | Analyzing chunk frames = [7707136,7707648) | 20260507 06:10:59.900 | INFO | MemInfo: { | 20260507 06:10:59.900 | INFO | "gpu" : " 17 GB", | 20260507 06:10:59.900 | INFO | "huge" : "", | 20260507 06:10:59.900 | INFO | "huge_shared" : " 27 GB", | 20260507 06:10:59.900 | INFO | "malloc" : " 14 GB", | 20260507 06:10:59.900 | INFO | "pinned" : " 29 GB" | 20260507 06:10:59.900 | INFO | } | 20260507 06:10:59.900 | INFO | Perf report for the last 5.13355 seconds: | 20260507 06:10:59.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:10:59.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:59.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:59.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:59.900 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:10:59.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:59.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:59.900 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 06:10:59.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:59.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:10:59.901 | INFO | Uploaded 36GiB to GPU in 2.04787s (17.5792GiB/s) | 20260507 06:10:59.901 | INFO | Downloaded 34GiB to GPU in 1.93035s (17.6134GiB/s) | 20260507 06:11:05.036 | INFO | Analyzing chunk frames = [7707648,7708160) | 20260507 06:11:05.036 | INFO | MemInfo: { | 20260507 06:11:05.036 | INFO | "gpu" : " 17 GB", | 20260507 06:11:05.036 | INFO | "huge" : "", | 20260507 06:11:05.036 | INFO | "huge_shared" : " 27 GB", | 20260507 06:11:05.036 | INFO | "malloc" : " 14 GB", | 20260507 06:11:05.036 | INFO | "pinned" : " 29 GB" | 20260507 06:11:05.036 | INFO | } | 20260507 06:11:05.036 | INFO | Perf report for the last 5.13593 seconds: | 20260507 06:11:05.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:11:05.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:05.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:05.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:05.036 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 06:11:05.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:05.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:05.036 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 06:11:05.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:05.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:05.036 | INFO | Uploaded 36GiB to GPU in 2.04689s (17.5877GiB/s) | 20260507 06:11:05.036 | INFO | Downloaded 34GiB to GPU in 1.99392s (17.0518GiB/s) | 20260507 06:11:10.180 | INFO | Analyzing chunk frames = [7708160,7708672) | 20260507 06:11:10.180 | INFO | MemInfo: { | 20260507 06:11:10.180 | INFO | "gpu" : " 17 GB", | 20260507 06:11:10.180 | INFO | "huge" : "", | 20260507 06:11:10.180 | INFO | "huge_shared" : " 27 GB", | 20260507 06:11:10.180 | INFO | "malloc" : " 14 GB", | 20260507 06:11:10.180 | INFO | "pinned" : " 29 GB" | 20260507 06:11:10.180 | INFO | } | 20260507 06:11:10.180 | INFO | Perf report for the last 5.14419 seconds: | 20260507 06:11:10.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:11:10.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:10.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:10.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:10.181 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 06:11:10.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:10.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:10.181 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:11:10.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:10.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:10.181 | INFO | Uploaded 36GiB to GPU in 1.96385s (18.3313GiB/s) | 20260507 06:11:10.181 | INFO | Downloaded 34GiB to GPU in 1.95706s (17.373GiB/s) | 20260507 06:11:10.181 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:11:10.181 | INFO | Total mem usage: 18 GiB | 20260507 06:11:10.181 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:11:10.181 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:11:10.181 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:11:10.181 | INFO | High Water Breakdown: | 20260507 06:11:10.181 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:11:10.181 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:11:10.181 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:11:10.181 | INFO | ------------------------------------------ | 20260507 06:11:10.181 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:11:10.181 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:11:10.181 | INFO | Total mem usage: 26.5 GiB | 20260507 06:11:10.181 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:11:10.181 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:11:10.181 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:11:10.181 | INFO | High Water Breakdown: | 20260507 06:11:10.181 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:11:10.181 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:11:10.181 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:11:10.181 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:11:10.181 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:11:10.181 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:11:10.181 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:11:10.181 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:11:10.181 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:11:10.181 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:11:10.181 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:11:10.181 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:11:10.181 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:11:10.181 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:11:10.181 | INFO | ------------------------------------------ | 20260507 06:11:10.181 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:11:10.181 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:11:10.181 | INFO | Total mem usage: 25.6 GiB | 20260507 06:11:10.181 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:11:10.181 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:11:10.181 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:11:10.181 | INFO | High Water Breakdown: | 20260507 06:11:10.181 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:11:10.181 | INFO | ------------------------------------------ | 20260507 06:11:10.181 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:11:10.181 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:11:10.181 | INFO | Total mem usage: 15.7 GiB | 20260507 06:11:10.181 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:11:10.181 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:11:10.181 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:11:10.181 | INFO | High Water Breakdown: | 20260507 06:11:10.181 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:11:10.181 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:11:10.181 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:11:10.181 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:11:10.181 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:11:10.181 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:11:10.181 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:11:10.181 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:11:10.181 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:11:10.181 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:11:10.181 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:11:10.181 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:11:10.181 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:11:10.181 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:11:10.181 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:11:10.181 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:11:10.181 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:11:10.181 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:11:10.181 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:11:10.181 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:11:10.181 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:11:10.181 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:11:10.181 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:11:10.181 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:11:10.181 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:11:10.181 | INFO | ------------------------------------------ | 20260507 06:11:10.181 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:11:10.182 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:11:10.182 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:11:10.182 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:11:10.182 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.47e+01, 1.33e+01 / 1.05e+00 | 20260507 06:11:10.182 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.02e-01, 9.90e-03 / 8.17e-03 | 20260507 06:11:10.182 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.40e+00, 5.20e-01 / 3.55e-01 | 20260507 06:11:10.182 | INFO | PerfReport: 3.24%, Histogram -- 9.96e-01 / 1.44e+00, 1.70e-01 / 1.43e-01 | 20260507 06:11:10.182 | INFO | PerfReport: 16.12%, DME -- 3.97e+01 / 4.41e+01, 3.82e+00 / 1.48e+00 | 20260507 06:11:10.182 | INFO | PerfReport: 57.84%, FrameLabeling -- 1.78e+01 / 2.05e+01, 1.43e+00 / 1.70e+00 | 20260507 06:11:10.182 | INFO | PerfReport: 6.87%, PulseAccumulating -- 2.11e+00 / 2.61e+00, 3.72e-01 / 2.56e-01 | 20260507 06:11:10.182 | INFO | PerfReport: 2.95%, Metrics -- 9.08e-01 / 1.23e+00, 2.40e-01 / 1.41e-01 | 20260507 06:11:10.182 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.06e-01 / 3.68e-01, 5.11e-02 / 2.36e-02 | 20260507 06:11:10.182 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:11:10.182 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:11:10.182 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:11:10.182 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 06:11:10.182 | INFO | PerfReport: 21.92%, Upload -- 2.11e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 06:11:10.182 | INFO | PerfReport: 21.36%, Download -- 2.05e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 06:11:10.182 | INFO | PerfReport: 31.99%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 06:11:10.182 | INFO | PerfReport: 1.53%, LockUpload -- 1.47e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 06:11:10.182 | INFO | PerfReport: 1.14%, LockDownload -- 1.09e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260507 06:11:10.182 | INFO | PerfReport: 22.04%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 06:11:10.182 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260507 06:11:10.278 | INFO | Starting a round of Baz IO | 20260507 06:11:13.377 | INFO | Populating RTMetrics report | 20260507 06:11:13.378 | INFO | RTMetrics report populated | 20260507 06:11:13.378 | INFO | Opening tmp RTMetrics json file | 20260507 06:11:13.394 | INFO | Serializing RTMetrics to json | 20260507 06:11:13.395 | INFO | Serialization done, swapping tmp file now... | 20260507 06:11:13.395 | INFO | RTMetrics done | 20260507 06:11:13.463 | INFO | Finished a round of Baz IO: Wrote 1.677 GiB in 3.185 seconds | 20260507 06:11:13.463 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.65% / 0.00% / 67.35% | 20260507 06:11:13.463 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.128531% / 0% / 9.13938e-05% | 20260507 06:11:13.463 | INFO | Baz IO overall: | 20260507 06:11:13.463 | INFO | BytesWritten : 3893988814809 | 20260507 06:11:13.463 | INFO | NumEvents : 1866536911982 | 20260507 06:11:15.327 | INFO | Analyzing chunk frames = [7708672,7709184) | 20260507 06:11:15.327 | INFO | MemInfo: { | 20260507 06:11:15.327 | INFO | "gpu" : " 17 GB", | 20260507 06:11:15.327 | INFO | "huge" : "", | 20260507 06:11:15.327 | INFO | "huge_shared" : " 27 GB", | 20260507 06:11:15.327 | INFO | "malloc" : " 14 GB", | 20260507 06:11:15.327 | INFO | "pinned" : " 29 GB" | 20260507 06:11:15.327 | INFO | } | 20260507 06:11:15.327 | INFO | Perf report for the last 5.1466 seconds: | 20260507 06:11:15.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:11:15.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:15.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:15.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:15.327 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 06:11:15.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:15.327 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:11:15.327 | INFO | PRE_HQ : 58.9%, 1.8, 54.7ms, 0 | 20260507 06:11:15.327 | INFO | BAZWRITER : 61.5%, 3.2, 101.2ms, 0 | 20260507 06:11:15.327 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:11:15.327 | INFO | Uploaded 36GiB to GPU in 2.0467s (17.5893GiB/s) | 20260507 06:11:15.327 | INFO | Downloaded 38GiB to GPU in 2.17581s (17.4648GiB/s) | 20260507 06:11:20.466 | INFO | Analyzing chunk frames = [7709184,7709696) | 20260507 06:11:20.466 | INFO | MemInfo: { | 20260507 06:11:20.466 | INFO | "gpu" : " 17 GB", | 20260507 06:11:20.467 | INFO | "huge" : "", | 20260507 06:11:20.467 | INFO | "huge_shared" : " 27 GB", | 20260507 06:11:20.467 | INFO | "malloc" : " 14 GB", | 20260507 06:11:20.467 | INFO | "pinned" : " 29 GB" | 20260507 06:11:20.467 | INFO | } | 20260507 06:11:20.467 | INFO | Perf report for the last 5.13954 seconds: | 20260507 06:11:20.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:11:20.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:20.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:20.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:20.467 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:11:20.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:20.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:20.467 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:11:20.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:20.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:20.467 | INFO | Uploaded 36GiB to GPU in 2.00133s (17.9881GiB/s) | 20260507 06:11:20.467 | INFO | Downloaded 34GiB to GPU in 1.94212s (17.5067GiB/s) | 20260507 06:11:25.593 | INFO | Analyzing chunk frames = [7709696,7710208) | 20260507 06:11:25.593 | INFO | MemInfo: { | 20260507 06:11:25.593 | INFO | "gpu" : " 17 GB", | 20260507 06:11:25.593 | INFO | "huge" : "", | 20260507 06:11:25.593 | INFO | "huge_shared" : " 27 GB", | 20260507 06:11:25.593 | INFO | "malloc" : " 14 GB", | 20260507 06:11:25.594 | INFO | "pinned" : " 29 GB" | 20260507 06:11:25.594 | INFO | } | 20260507 06:11:25.594 | INFO | Perf report for the last 5.12703 seconds: | 20260507 06:11:25.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:11:25.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:25.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:25.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:25.594 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 06:11:25.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:25.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:25.594 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:11:25.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:25.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:25.594 | INFO | Uploaded 36GiB to GPU in 2.02631s (17.7663GiB/s) | 20260507 06:11:25.594 | INFO | Downloaded 34GiB to GPU in 1.91645s (17.7412GiB/s) | 20260507 06:11:30.738 | INFO | Analyzing chunk frames = [7710208,7710720) | 20260507 06:11:30.738 | INFO | MemInfo: { | 20260507 06:11:30.738 | INFO | "gpu" : " 17 GB", | 20260507 06:11:30.738 | INFO | "huge" : "", | 20260507 06:11:30.738 | INFO | "huge_shared" : " 27 GB", | 20260507 06:11:30.738 | INFO | "malloc" : " 14 GB", | 20260507 06:11:30.738 | INFO | "pinned" : " 29 GB" | 20260507 06:11:30.738 | INFO | } | 20260507 06:11:30.738 | INFO | Perf report for the last 5.14429 seconds: | 20260507 06:11:30.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:11:30.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:30.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:30.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:30.738 | INFO | ANALYSIS : 59.5%, 3.0, 92.5ms, 0 | 20260507 06:11:30.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:30.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:30.738 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:11:30.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:30.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:30.738 | INFO | Uploaded 36GiB to GPU in 1.98256s (18.1584GiB/s) | 20260507 06:11:30.738 | INFO | Downloaded 34GiB to GPU in 1.91501s (17.7545GiB/s) | 20260507 06:11:35.874 | INFO | Analyzing chunk frames = [7710720,7711232) | 20260507 06:11:35.874 | INFO | MemInfo: { | 20260507 06:11:35.874 | INFO | "gpu" : " 17 GB", | 20260507 06:11:35.874 | INFO | "huge" : "", | 20260507 06:11:35.874 | INFO | "huge_shared" : " 27 GB", | 20260507 06:11:35.874 | INFO | "malloc" : " 14 GB", | 20260507 06:11:35.874 | INFO | "pinned" : " 29 GB" | 20260507 06:11:35.874 | INFO | } | 20260507 06:11:35.874 | INFO | Perf report for the last 5.13592 seconds: | 20260507 06:11:35.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:11:35.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:35.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:35.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:35.874 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 06:11:35.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:35.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:35.874 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:11:35.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:35.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:35.874 | INFO | Uploaded 36GiB to GPU in 2.0546s (17.5217GiB/s) | 20260507 06:11:35.874 | INFO | Downloaded 34GiB to GPU in 1.94224s (17.5055GiB/s) | 20260507 06:11:41.020 | INFO | Analyzing chunk frames = [7711232,7711744) | 20260507 06:11:41.020 | INFO | MemInfo: { | 20260507 06:11:41.020 | INFO | "gpu" : " 17 GB", | 20260507 06:11:41.020 | INFO | "huge" : "", | 20260507 06:11:41.020 | INFO | "huge_shared" : " 27 GB", | 20260507 06:11:41.020 | INFO | "malloc" : " 14 GB", | 20260507 06:11:41.021 | INFO | "pinned" : " 29 GB" | 20260507 06:11:41.021 | INFO | } | 20260507 06:11:41.021 | INFO | Perf report for the last 5.14668 seconds: | 20260507 06:11:41.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:11:41.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:41.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:41.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:41.021 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 06:11:41.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:41.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:41.021 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260507 06:11:41.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:41.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:41.021 | INFO | Uploaded 36GiB to GPU in 2.04778s (17.58GiB/s) | 20260507 06:11:41.021 | INFO | Downloaded 34GiB to GPU in 1.90367s (17.8602GiB/s) | 20260507 06:11:46.157 | INFO | Analyzing chunk frames = [7711744,7712256) | 20260507 06:11:46.157 | INFO | MemInfo: { | 20260507 06:11:46.157 | INFO | "gpu" : " 17 GB", | 20260507 06:11:46.157 | INFO | "huge" : "", | 20260507 06:11:46.157 | INFO | "huge_shared" : " 27 GB", | 20260507 06:11:46.157 | INFO | "malloc" : " 14 GB", | 20260507 06:11:46.157 | INFO | "pinned" : " 29 GB" | 20260507 06:11:46.157 | INFO | } | 20260507 06:11:46.157 | INFO | Perf report for the last 5.13673 seconds: | 20260507 06:11:46.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:11:46.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:46.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:46.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:46.157 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:11:46.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:46.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:46.157 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:11:46.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:46.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:46.158 | INFO | Uploaded 36GiB to GPU in 1.90259s (18.9216GiB/s) | 20260507 06:11:46.158 | INFO | Downloaded 34GiB to GPU in 1.97177s (17.2434GiB/s) | 20260507 06:11:51.294 | INFO | Analyzing chunk frames = [7712256,7712768) | 20260507 06:11:51.294 | INFO | MemInfo: { | 20260507 06:11:51.294 | INFO | "gpu" : " 17 GB", | 20260507 06:11:51.294 | INFO | "huge" : "", | 20260507 06:11:51.294 | INFO | "huge_shared" : " 27 GB", | 20260507 06:11:51.294 | INFO | "malloc" : " 14 GB", | 20260507 06:11:51.294 | INFO | "pinned" : " 29 GB" | 20260507 06:11:51.294 | INFO | } | 20260507 06:11:51.294 | INFO | Perf report for the last 5.1364 seconds: | 20260507 06:11:51.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:11:51.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:51.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:51.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:51.294 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:11:51.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:51.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:51.294 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:11:51.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:51.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:51.294 | INFO | Uploaded 36GiB to GPU in 1.97454s (18.2321GiB/s) | 20260507 06:11:51.294 | INFO | Downloaded 34GiB to GPU in 1.95604s (17.3821GiB/s) | 20260507 06:11:51.391 | INFO | Starting a round of Baz IO | 20260507 06:11:54.483 | INFO | Populating RTMetrics report | 20260507 06:11:54.483 | INFO | RTMetrics report populated | 20260507 06:11:54.483 | INFO | Opening tmp RTMetrics json file | 20260507 06:11:54.490 | INFO | Serializing RTMetrics to json | 20260507 06:11:54.500 | INFO | Serialization done, swapping tmp file now... | 20260507 06:11:54.500 | INFO | RTMetrics done | 20260507 06:11:54.569 | INFO | Finished a round of Baz IO: Wrote 1.675 GiB in 3.177 seconds | 20260507 06:11:54.569 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.65% / 0.00% / 67.35% | 20260507 06:11:54.569 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.127776% / 0% / 7.15256e-05% | 20260507 06:11:54.569 | INFO | Baz IO overall: | 20260507 06:11:54.569 | INFO | BytesWritten : 3895786980231 | 20260507 06:11:54.569 | INFO | NumEvents : 1867379538697 | 20260507 06:11:56.437 | INFO | Analyzing chunk frames = [7712768,7713280) | 20260507 06:11:56.437 | INFO | MemInfo: { | 20260507 06:11:56.437 | INFO | "gpu" : " 17 GB", | 20260507 06:11:56.437 | INFO | "huge" : "", | 20260507 06:11:56.437 | INFO | "huge_shared" : " 27 GB", | 20260507 06:11:56.437 | INFO | "malloc" : " 14 GB", | 20260507 06:11:56.437 | INFO | "pinned" : " 29 GB" | 20260507 06:11:56.437 | INFO | } | 20260507 06:11:56.437 | INFO | Perf report for the last 5.14374 seconds: | 20260507 06:11:56.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:11:56.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:56.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:56.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:56.438 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 06:11:56.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:11:56.438 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:11:56.438 | INFO | PRE_HQ : 58.8%, 1.8, 54.7ms, 0 | 20260507 06:11:56.438 | INFO | BAZWRITER : 61.4%, 3.1, 99.1ms, 0 | 20260507 06:11:56.438 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:11:56.438 | INFO | Uploaded 36GiB to GPU in 2.02841s (17.7479GiB/s) | 20260507 06:11:56.438 | INFO | Downloaded 38GiB to GPU in 2.16867s (17.5223GiB/s) | 20260507 06:12:01.573 | INFO | Analyzing chunk frames = [7713280,7713792) | 20260507 06:12:01.573 | INFO | MemInfo: { | 20260507 06:12:01.573 | INFO | "gpu" : " 17 GB", | 20260507 06:12:01.573 | INFO | "huge" : "", | 20260507 06:12:01.573 | INFO | "huge_shared" : " 27 GB", | 20260507 06:12:01.573 | INFO | "malloc" : " 14 GB", | 20260507 06:12:01.573 | INFO | "pinned" : " 29 GB" | 20260507 06:12:01.573 | INFO | } | 20260507 06:12:01.573 | INFO | Perf report for the last 5.13566 seconds: | 20260507 06:12:01.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:12:01.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:01.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:01.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:01.573 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 06:12:01.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:01.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:01.573 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 06:12:01.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:01.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:01.574 | INFO | Uploaded 36GiB to GPU in 1.99577s (18.0381GiB/s) | 20260507 06:12:01.574 | INFO | Downloaded 34GiB to GPU in 1.93541s (17.5673GiB/s) | 20260507 06:12:06.712 | INFO | Analyzing chunk frames = [7713792,7714304) | 20260507 06:12:06.712 | INFO | MemInfo: { | 20260507 06:12:06.712 | INFO | "gpu" : " 17 GB", | 20260507 06:12:06.712 | INFO | "huge" : "", | 20260507 06:12:06.712 | INFO | "huge_shared" : " 27 GB", | 20260507 06:12:06.712 | INFO | "malloc" : " 14 GB", | 20260507 06:12:06.712 | INFO | "pinned" : " 29 GB" | 20260507 06:12:06.712 | INFO | } | 20260507 06:12:06.712 | INFO | Perf report for the last 5.13923 seconds: | 20260507 06:12:06.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:12:06.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:06.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:06.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:06.712 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 06:12:06.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:06.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:06.712 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:12:06.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:06.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:06.713 | INFO | Uploaded 36GiB to GPU in 2.05078s (17.5543GiB/s) | 20260507 06:12:06.713 | INFO | Downloaded 34GiB to GPU in 1.94873s (17.4472GiB/s) | 20260507 06:12:11.845 | INFO | Analyzing chunk frames = [7714304,7714816) | 20260507 06:12:11.845 | INFO | MemInfo: { | 20260507 06:12:11.845 | INFO | "gpu" : " 17 GB", | 20260507 06:12:11.845 | INFO | "huge" : "", | 20260507 06:12:11.845 | INFO | "huge_shared" : " 27 GB", | 20260507 06:12:11.845 | INFO | "malloc" : " 14 GB", | 20260507 06:12:11.845 | INFO | "pinned" : " 29 GB" | 20260507 06:12:11.845 | INFO | } | 20260507 06:12:11.845 | INFO | Perf report for the last 5.13219 seconds: | 20260507 06:12:11.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:12:11.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:11.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:11.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:11.845 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 06:12:11.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:11.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:11.845 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:12:11.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:11.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:11.845 | INFO | Uploaded 36GiB to GPU in 2.01939s (17.8271GiB/s) | 20260507 06:12:11.845 | INFO | Downloaded 34GiB to GPU in 1.94246s (17.5036GiB/s) | 20260507 06:12:16.989 | INFO | Analyzing chunk frames = [7714816,7715328) | 20260507 06:12:16.989 | INFO | MemInfo: { | 20260507 06:12:16.989 | INFO | "gpu" : " 17 GB", | 20260507 06:12:16.989 | INFO | "huge" : "", | 20260507 06:12:16.989 | INFO | "huge_shared" : " 27 GB", | 20260507 06:12:16.989 | INFO | "malloc" : " 14 GB", | 20260507 06:12:16.989 | INFO | "pinned" : " 29 GB" | 20260507 06:12:16.989 | INFO | } | 20260507 06:12:16.989 | INFO | Perf report for the last 5.14481 seconds: | 20260507 06:12:16.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:12:16.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:16.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:16.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:16.990 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 06:12:16.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:16.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:16.990 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:12:16.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:16.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:16.990 | INFO | Uploaded 36GiB to GPU in 2.0439s (17.6133GiB/s) | 20260507 06:12:16.990 | INFO | Downloaded 34GiB to GPU in 1.93077s (17.6096GiB/s) | 20260507 06:12:22.132 | INFO | Analyzing chunk frames = [7715328,7715840) | 20260507 06:12:22.132 | INFO | MemInfo: { | 20260507 06:12:22.132 | INFO | "gpu" : " 17 GB", | 20260507 06:12:22.132 | INFO | "huge" : "", | 20260507 06:12:22.132 | INFO | "huge_shared" : " 27 GB", | 20260507 06:12:22.132 | INFO | "malloc" : " 14 GB", | 20260507 06:12:22.132 | INFO | "pinned" : " 29 GB" | 20260507 06:12:22.132 | INFO | } | 20260507 06:12:22.132 | INFO | Perf report for the last 5.14272 seconds: | 20260507 06:12:22.132 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:12:22.132 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:22.132 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:22.132 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:22.132 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:12:22.132 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:22.132 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:22.132 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 06:12:22.132 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:22.132 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:22.132 | INFO | Uploaded 36GiB to GPU in 2.04725s (17.5845GiB/s) | 20260507 06:12:22.132 | INFO | Downloaded 34GiB to GPU in 1.9104s (17.7973GiB/s) | 20260507 06:12:27.276 | INFO | Analyzing chunk frames = [7715840,7716352) | 20260507 06:12:27.276 | INFO | MemInfo: { | 20260507 06:12:27.276 | INFO | "gpu" : " 17 GB", | 20260507 06:12:27.276 | INFO | "huge" : "", | 20260507 06:12:27.276 | INFO | "huge_shared" : " 27 GB", | 20260507 06:12:27.276 | INFO | "malloc" : " 14 GB", | 20260507 06:12:27.276 | INFO | "pinned" : " 29 GB" | 20260507 06:12:27.276 | INFO | } | 20260507 06:12:27.276 | INFO | Perf report for the last 5.14358 seconds: | 20260507 06:12:27.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:12:27.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:27.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:27.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:27.276 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:12:27.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:27.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:27.276 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:12:27.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:27.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:27.276 | INFO | Uploaded 36GiB to GPU in 1.94575s (18.5019GiB/s) | 20260507 06:12:27.276 | INFO | Downloaded 34GiB to GPU in 1.97355s (17.2278GiB/s) | 20260507 06:12:32.409 | INFO | Analyzing chunk frames = [7716352,7716864) | 20260507 06:12:32.409 | INFO | MemInfo: { | 20260507 06:12:32.409 | INFO | "gpu" : " 17 GB", | 20260507 06:12:32.410 | INFO | "huge" : "", | 20260507 06:12:32.410 | INFO | "huge_shared" : " 27 GB", | 20260507 06:12:32.410 | INFO | "malloc" : " 14 GB", | 20260507 06:12:32.410 | INFO | "pinned" : " 29 GB" | 20260507 06:12:32.410 | INFO | } | 20260507 06:12:32.410 | INFO | Perf report for the last 5.13383 seconds: | 20260507 06:12:32.410 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:12:32.410 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:32.410 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:32.410 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:32.410 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:12:32.410 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:32.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:32.410 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:12:32.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:32.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:32.410 | INFO | Uploaded 36GiB to GPU in 1.99197s (18.0726GiB/s) | 20260507 06:12:32.410 | INFO | Downloaded 34GiB to GPU in 1.95411s (17.3992GiB/s) | 20260507 06:12:32.410 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:12:32.410 | INFO | Total mem usage: 18 GiB | 20260507 06:12:32.410 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:12:32.410 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 06:12:32.410 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:12:32.410 | INFO | High Water Breakdown: | 20260507 06:12:32.410 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:12:32.410 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:12:32.410 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:12:32.410 | INFO | ------------------------------------------ | 20260507 06:12:32.410 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:12:32.410 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:12:32.410 | INFO | Total mem usage: 26.5 GiB | 20260507 06:12:32.410 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:12:32.410 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:12:32.410 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 06:12:32.410 | INFO | High Water Breakdown: | 20260507 06:12:32.410 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:12:32.410 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:12:32.410 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:12:32.410 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:12:32.410 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:12:32.410 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:12:32.410 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:12:32.410 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:12:32.410 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:12:32.410 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:12:32.410 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:12:32.410 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:12:32.410 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:12:32.410 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:12:32.410 | INFO | ------------------------------------------ | 20260507 06:12:32.410 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:12:32.410 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:12:32.410 | INFO | Total mem usage: 25.6 GiB | 20260507 06:12:32.410 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:12:32.410 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:12:32.410 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:12:32.410 | INFO | High Water Breakdown: | 20260507 06:12:32.410 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:12:32.410 | INFO | ------------------------------------------ | 20260507 06:12:32.410 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:12:32.410 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:12:32.410 | INFO | Total mem usage: 15.7 GiB | 20260507 06:12:32.410 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:12:32.410 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:12:32.410 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:12:32.410 | INFO | High Water Breakdown: | 20260507 06:12:32.410 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:12:32.410 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:12:32.410 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:12:32.410 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:12:32.410 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:12:32.410 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:12:32.410 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:12:32.410 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:12:32.410 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:12:32.410 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:12:32.410 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:12:32.411 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:12:32.411 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:12:32.411 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:12:32.411 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:12:32.411 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:12:32.411 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:12:32.411 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:12:32.411 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:12:32.411 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:12:32.411 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:12:32.411 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:12:32.411 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:12:32.411 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:12:32.411 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:12:32.411 | INFO | ------------------------------------------ | 20260507 06:12:32.411 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:12:32.411 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:12:32.411 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:12:32.411 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:12:32.411 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 1.17e+00 | 20260507 06:12:32.411 | INFO | PerfReport: 0.29%, AggregateStats -- 9.03e-02 / 1.02e-01, 9.16e-03 / 3.95e-03 | 20260507 06:12:32.411 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.47e+00, 5.19e-01 / 2.11e-01 | 20260507 06:12:32.411 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.49e+00, 1.71e-01 / 9.05e-02 | 20260507 06:12:32.411 | INFO | PerfReport: 16.21%, DME -- 3.98e+01 / 4.46e+01, 3.92e+00 / 1.54e+00 | 20260507 06:12:32.411 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.27e+00 / 1.18e+00 | 20260507 06:12:32.411 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.72e+00, 3.86e-01 / 1.75e-01 | 20260507 06:12:32.411 | INFO | PerfReport: 2.97%, Metrics -- 9.13e-01 / 1.24e+00, 2.45e-01 / 1.19e-01 | 20260507 06:12:32.411 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.06e-01 / 3.59e-01, 5.01e-02 / 1.68e-02 | 20260507 06:12:32.411 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:12:32.411 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:12:32.411 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:12:32.411 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 06:12:32.411 | INFO | PerfReport: 21.79%, Upload -- 2.09e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260507 06:12:32.411 | INFO | PerfReport: 21.33%, Download -- 2.05e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 06:12:32.411 | INFO | PerfReport: 31.96%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:12:32.411 | INFO | PerfReport: 1.57%, LockUpload -- 1.51e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 06:12:32.411 | INFO | PerfReport: 1.21%, LockDownload -- 1.17e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260507 06:12:32.411 | INFO | PerfReport: 22.13%, LockCompute -- 2.13e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 06:12:32.411 | INFO | PerfReport: 0.02%, Misc -- 1.46e-02 / 0.00e+00, 2.76e-02 / 0.00e+00 | 20260507 06:12:32.509 | INFO | Starting a round of Baz IO | 20260507 06:12:35.600 | INFO | Populating RTMetrics report | 20260507 06:12:35.600 | INFO | RTMetrics report populated | 20260507 06:12:35.600 | INFO | Opening tmp RTMetrics json file | 20260507 06:12:35.601 | INFO | Serializing RTMetrics to json | 20260507 06:12:35.603 | INFO | Serialization done, swapping tmp file now... | 20260507 06:12:35.603 | INFO | RTMetrics done | 20260507 06:12:35.687 | INFO | Finished a round of Baz IO: Wrote 1.678 GiB in 3.178 seconds | 20260507 06:12:35.687 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.64% / 0.00% / 67.35% | 20260507 06:12:35.688 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.128043% / 0% / 7.54992e-05% | 20260507 06:12:35.688 | INFO | Baz IO overall: | 20260507 06:12:35.688 | INFO | BytesWritten : 3897589208855 | 20260507 06:12:35.688 | INFO | NumEvents : 1868223526595 | 20260507 06:12:37.548 | INFO | Analyzing chunk frames = [7716864,7717376) | 20260507 06:12:37.548 | INFO | MemInfo: { | 20260507 06:12:37.548 | INFO | "gpu" : " 17 GB", | 20260507 06:12:37.548 | INFO | "huge" : "", | 20260507 06:12:37.548 | INFO | "huge_shared" : " 27 GB", | 20260507 06:12:37.548 | INFO | "malloc" : " 14 GB", | 20260507 06:12:37.548 | INFO | "pinned" : " 29 GB" | 20260507 06:12:37.549 | INFO | } | 20260507 06:12:37.549 | INFO | Perf report for the last 5.1389 seconds: | 20260507 06:12:37.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:12:37.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:37.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:37.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:37.549 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 06:12:37.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:37.549 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260507 06:12:37.549 | INFO | PRE_HQ : 59.3%, 1.8, 55.3ms, 0 | 20260507 06:12:37.549 | INFO | BAZWRITER : 61.5%, 3.1, 99.4ms, 0 | 20260507 06:12:37.549 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:12:37.549 | INFO | Uploaded 36GiB to GPU in 2.06184s (17.4601GiB/s) | 20260507 06:12:37.549 | INFO | Downloaded 38GiB to GPU in 2.18472s (17.3936GiB/s) | 20260507 06:12:42.688 | INFO | Analyzing chunk frames = [7717376,7717888) | 20260507 06:12:42.688 | INFO | MemInfo: { | 20260507 06:12:42.688 | INFO | "gpu" : " 17 GB", | 20260507 06:12:42.688 | INFO | "huge" : "", | 20260507 06:12:42.688 | INFO | "huge_shared" : " 27 GB", | 20260507 06:12:42.688 | INFO | "malloc" : " 14 GB", | 20260507 06:12:42.688 | INFO | "pinned" : " 29 GB" | 20260507 06:12:42.688 | INFO | } | 20260507 06:12:42.688 | INFO | Perf report for the last 5.1392 seconds: | 20260507 06:12:42.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:12:42.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:42.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:42.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:42.688 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 06:12:42.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:42.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:42.688 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:12:42.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:42.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:42.688 | INFO | Uploaded 36GiB to GPU in 2.00255s (17.9771GiB/s) | 20260507 06:12:42.688 | INFO | Downloaded 34GiB to GPU in 1.92752s (17.6392GiB/s) | 20260507 06:12:47.826 | INFO | Analyzing chunk frames = [7717888,7718400) | 20260507 06:12:47.826 | INFO | MemInfo: { | 20260507 06:12:47.826 | INFO | "gpu" : " 17 GB", | 20260507 06:12:47.826 | INFO | "huge" : "", | 20260507 06:12:47.826 | INFO | "huge_shared" : " 27 GB", | 20260507 06:12:47.826 | INFO | "malloc" : " 14 GB", | 20260507 06:12:47.826 | INFO | "pinned" : " 29 GB" | 20260507 06:12:47.826 | INFO | } | 20260507 06:12:47.826 | INFO | Perf report for the last 5.13822 seconds: | 20260507 06:12:47.826 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:12:47.826 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:47.826 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:47.826 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:47.826 | INFO | ANALYSIS : 60.2%, 3.0, 93.2ms, 0 | 20260507 06:12:47.826 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:47.826 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:47.826 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:12:47.826 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:47.826 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:47.826 | INFO | Uploaded 36GiB to GPU in 2.01116s (17.9001GiB/s) | 20260507 06:12:47.826 | INFO | Downloaded 34GiB to GPU in 1.94096s (17.5171GiB/s) | 20260507 06:12:52.975 | INFO | Analyzing chunk frames = [7718400,7718912) | 20260507 06:12:52.975 | INFO | MemInfo: { | 20260507 06:12:52.975 | INFO | "gpu" : " 17 GB", | 20260507 06:12:52.975 | INFO | "huge" : "", | 20260507 06:12:52.975 | INFO | "huge_shared" : " 27 GB", | 20260507 06:12:52.975 | INFO | "malloc" : " 14 GB", | 20260507 06:12:52.975 | INFO | "pinned" : " 29 GB" | 20260507 06:12:52.975 | INFO | } | 20260507 06:12:52.975 | INFO | Perf report for the last 5.14922 seconds: | 20260507 06:12:52.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:12:52.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:52.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:52.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:52.975 | INFO | ANALYSIS : 60.3%, 2.9, 93.3ms, 0 | 20260507 06:12:52.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:52.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:52.975 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 06:12:52.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:52.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:52.975 | INFO | Uploaded 36GiB to GPU in 2.04287s (17.6222GiB/s) | 20260507 06:12:52.976 | INFO | Downloaded 34GiB to GPU in 1.95015s (17.4346GiB/s) | 20260507 06:12:58.103 | INFO | Analyzing chunk frames = [7718912,7719424) | 20260507 06:12:58.103 | INFO | MemInfo: { | 20260507 06:12:58.103 | INFO | "gpu" : " 17 GB", | 20260507 06:12:58.103 | INFO | "huge" : "", | 20260507 06:12:58.103 | INFO | "huge_shared" : " 27 GB", | 20260507 06:12:58.103 | INFO | "malloc" : " 14 GB", | 20260507 06:12:58.103 | INFO | "pinned" : " 29 GB" | 20260507 06:12:58.103 | INFO | } | 20260507 06:12:58.103 | INFO | Perf report for the last 5.12758 seconds: | 20260507 06:12:58.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:12:58.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:58.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:58.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:58.103 | INFO | ANALYSIS : 60.2%, 3.0, 93.5ms, 0 | 20260507 06:12:58.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:58.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:58.103 | INFO | PRE_HQ : 25.4%, 1.0, 13.1ms, 0 | 20260507 06:12:58.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:58.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:12:58.103 | INFO | Uploaded 36GiB to GPU in 2.05188s (17.5449GiB/s) | 20260507 06:12:58.103 | INFO | Downloaded 34GiB to GPU in 1.90155s (17.8802GiB/s) | 20260507 06:13:03.246 | INFO | Analyzing chunk frames = [7719424,7719936) | 20260507 06:13:03.246 | INFO | MemInfo: { | 20260507 06:13:03.246 | INFO | "gpu" : " 17 GB", | 20260507 06:13:03.247 | INFO | "huge" : "", | 20260507 06:13:03.247 | INFO | "huge_shared" : " 27 GB", | 20260507 06:13:03.247 | INFO | "malloc" : " 14 GB", | 20260507 06:13:03.247 | INFO | "pinned" : " 29 GB" | 20260507 06:13:03.247 | INFO | } | 20260507 06:13:03.247 | INFO | Perf report for the last 5.14381 seconds: | 20260507 06:13:03.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:13:03.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:03.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:03.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:03.247 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 06:13:03.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:03.247 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:03.247 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 06:13:03.247 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:03.247 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:03.247 | INFO | Uploaded 36GiB to GPU in 2.03336s (17.7047GiB/s) | 20260507 06:13:03.247 | INFO | Downloaded 34GiB to GPU in 1.90785s (17.8211GiB/s) | 20260507 06:13:08.385 | INFO | Analyzing chunk frames = [7719936,7720448) | 20260507 06:13:08.385 | INFO | MemInfo: { | 20260507 06:13:08.385 | INFO | "gpu" : " 17 GB", | 20260507 06:13:08.385 | INFO | "huge" : "", | 20260507 06:13:08.385 | INFO | "huge_shared" : " 27 GB", | 20260507 06:13:08.385 | INFO | "malloc" : " 14 GB", | 20260507 06:13:08.385 | INFO | "pinned" : " 29 GB" | 20260507 06:13:08.385 | INFO | } | 20260507 06:13:08.385 | INFO | Perf report for the last 5.13836 seconds: | 20260507 06:13:08.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:13:08.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:08.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:08.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:08.385 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:13:08.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:08.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:08.385 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 06:13:08.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:08.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:08.385 | INFO | Uploaded 36GiB to GPU in 1.98944s (18.0955GiB/s) | 20260507 06:13:08.385 | INFO | Downloaded 34GiB to GPU in 1.98035s (17.1687GiB/s) | 20260507 06:13:13.525 | INFO | Analyzing chunk frames = [7720448,7720960) | 20260507 06:13:13.525 | INFO | MemInfo: { | 20260507 06:13:13.525 | INFO | "gpu" : " 17 GB", | 20260507 06:13:13.525 | INFO | "huge" : "", | 20260507 06:13:13.525 | INFO | "huge_shared" : " 27 GB", | 20260507 06:13:13.525 | INFO | "malloc" : " 14 GB", | 20260507 06:13:13.525 | INFO | "pinned" : " 29 GB" | 20260507 06:13:13.525 | INFO | } | 20260507 06:13:13.525 | INFO | Perf report for the last 5.13979 seconds: | 20260507 06:13:13.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:13:13.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:13.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:13.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:13.525 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:13:13.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:13.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:13.525 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:13:13.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:13.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:13.525 | INFO | Uploaded 36GiB to GPU in 1.98075s (18.1749GiB/s) | 20260507 06:13:13.525 | INFO | Downloaded 34GiB to GPU in 1.96182s (17.3309GiB/s) | 20260507 06:13:13.622 | INFO | Starting a round of Baz IO | 20260507 06:13:16.712 | INFO | Populating RTMetrics report | 20260507 06:13:16.712 | INFO | RTMetrics report populated | 20260507 06:13:16.712 | INFO | Opening tmp RTMetrics json file | 20260507 06:13:16.713 | INFO | Serializing RTMetrics to json | 20260507 06:13:16.730 | INFO | Serialization done, swapping tmp file now... | 20260507 06:13:16.731 | INFO | RTMetrics done | 20260507 06:13:16.802 | INFO | Finished a round of Baz IO: Wrote 1.681 GiB in 3.18 seconds | 20260507 06:13:16.802 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.64% / 0.00% / 67.36% | 20260507 06:13:16.802 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.12664% / 0% / 5.96046e-05% | 20260507 06:13:16.802 | INFO | Baz IO overall: | 20260507 06:13:16.802 | INFO | BytesWritten : 3899393990020 | 20260507 06:13:16.802 | INFO | NumEvents : 1869067114863 | 20260507 06:13:18.660 | INFO | Analyzing chunk frames = [7720960,7721472) | 20260507 06:13:18.660 | INFO | MemInfo: { | 20260507 06:13:18.660 | INFO | "gpu" : " 17 GB", | 20260507 06:13:18.660 | INFO | "huge" : "", | 20260507 06:13:18.660 | INFO | "huge_shared" : " 27 GB", | 20260507 06:13:18.660 | INFO | "malloc" : " 14 GB", | 20260507 06:13:18.660 | INFO | "pinned" : " 29 GB" | 20260507 06:13:18.660 | INFO | } | 20260507 06:13:18.660 | INFO | Perf report for the last 5.13532 seconds: | 20260507 06:13:18.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:13:18.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:18.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:18.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:18.660 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 06:13:18.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:18.660 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:13:18.660 | INFO | PRE_HQ : 59.0%, 1.8, 55.0ms, 0 | 20260507 06:13:18.660 | INFO | BAZWRITER : 61.5%, 3.2, 99.9ms, 0 | 20260507 06:13:18.660 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:13:18.660 | INFO | Uploaded 36GiB to GPU in 2.06619s (17.4233GiB/s) | 20260507 06:13:18.660 | INFO | Downloaded 38GiB to GPU in 2.17809s (17.4465GiB/s) | 20260507 06:13:23.799 | INFO | Analyzing chunk frames = [7721472,7721984) | 20260507 06:13:23.799 | INFO | MemInfo: { | 20260507 06:13:23.799 | INFO | "gpu" : " 17 GB", | 20260507 06:13:23.799 | INFO | "huge" : "", | 20260507 06:13:23.799 | INFO | "huge_shared" : " 27 GB", | 20260507 06:13:23.799 | INFO | "malloc" : " 14 GB", | 20260507 06:13:23.799 | INFO | "pinned" : " 29 GB" | 20260507 06:13:23.799 | INFO | } | 20260507 06:13:23.799 | INFO | Perf report for the last 5.1388 seconds: | 20260507 06:13:23.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:13:23.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:23.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:23.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:23.799 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 06:13:23.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:23.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:23.799 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 06:13:23.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:23.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:23.799 | INFO | Uploaded 36GiB to GPU in 2.01655s (17.8522GiB/s) | 20260507 06:13:23.799 | INFO | Downloaded 34GiB to GPU in 1.93366s (17.5832GiB/s) | 20260507 06:13:28.942 | INFO | Analyzing chunk frames = [7721984,7722496) | 20260507 06:13:28.942 | INFO | MemInfo: { | 20260507 06:13:28.942 | INFO | "gpu" : " 17 GB", | 20260507 06:13:28.942 | INFO | "huge" : "", | 20260507 06:13:28.942 | INFO | "huge_shared" : " 27 GB", | 20260507 06:13:28.942 | INFO | "malloc" : " 14 GB", | 20260507 06:13:28.942 | INFO | "pinned" : " 29 GB" | 20260507 06:13:28.942 | INFO | } | 20260507 06:13:28.943 | INFO | Perf report for the last 5.14359 seconds: | 20260507 06:13:28.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:13:28.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:28.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:28.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:28.943 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 06:13:28.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:28.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:28.943 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:13:28.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:28.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:28.943 | INFO | Uploaded 36GiB to GPU in 2.02469s (17.7805GiB/s) | 20260507 06:13:28.943 | INFO | Downloaded 34GiB to GPU in 1.93818s (17.5422GiB/s) | 20260507 06:13:34.083 | INFO | Analyzing chunk frames = [7722496,7723008) | 20260507 06:13:34.083 | INFO | MemInfo: { | 20260507 06:13:34.083 | INFO | "gpu" : " 17 GB", | 20260507 06:13:34.083 | INFO | "huge" : "", | 20260507 06:13:34.083 | INFO | "huge_shared" : " 27 GB", | 20260507 06:13:34.083 | INFO | "malloc" : " 14 GB", | 20260507 06:13:34.083 | INFO | "pinned" : " 29 GB" | 20260507 06:13:34.083 | INFO | } | 20260507 06:13:34.083 | INFO | Perf report for the last 5.14085 seconds: | 20260507 06:13:34.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:13:34.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:34.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:34.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:34.083 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 06:13:34.084 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:34.084 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:34.084 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:13:34.084 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:34.084 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:34.084 | INFO | Uploaded 36GiB to GPU in 1.99809s (18.0172GiB/s) | 20260507 06:13:34.084 | INFO | Downloaded 34GiB to GPU in 1.91979s (17.7103GiB/s) | 20260507 06:13:39.223 | INFO | Analyzing chunk frames = [7723008,7723520) | 20260507 06:13:39.223 | INFO | MemInfo: { | 20260507 06:13:39.223 | INFO | "gpu" : " 17 GB", | 20260507 06:13:39.223 | INFO | "huge" : "", | 20260507 06:13:39.223 | INFO | "huge_shared" : " 27 GB", | 20260507 06:13:39.223 | INFO | "malloc" : " 14 GB", | 20260507 06:13:39.223 | INFO | "pinned" : " 29 GB" | 20260507 06:13:39.223 | INFO | } | 20260507 06:13:39.223 | INFO | Perf report for the last 5.13986 seconds: | 20260507 06:13:39.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:13:39.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:39.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:39.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:39.223 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 06:13:39.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:39.223 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:39.223 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:13:39.223 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:39.223 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:39.224 | INFO | Uploaded 36GiB to GPU in 2.02009s (17.821GiB/s) | 20260507 06:13:39.224 | INFO | Downloaded 34GiB to GPU in 1.90339s (17.8629GiB/s) | 20260507 06:13:44.355 | INFO | Analyzing chunk frames = [7723520,7724032) | 20260507 06:13:44.355 | INFO | MemInfo: { | 20260507 06:13:44.355 | INFO | "gpu" : " 17 GB", | 20260507 06:13:44.355 | INFO | "huge" : "", | 20260507 06:13:44.355 | INFO | "huge_shared" : " 27 GB", | 20260507 06:13:44.355 | INFO | "malloc" : " 14 GB", | 20260507 06:13:44.355 | INFO | "pinned" : " 29 GB" | 20260507 06:13:44.355 | INFO | } | 20260507 06:13:44.356 | INFO | Perf report for the last 5.13225 seconds: | 20260507 06:13:44.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:13:44.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:44.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:44.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:44.356 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:13:44.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:44.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:44.356 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 06:13:44.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:44.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:44.356 | INFO | Uploaded 36GiB to GPU in 2.03528s (17.688GiB/s) | 20260507 06:13:44.356 | INFO | Downloaded 34GiB to GPU in 1.91829s (17.7241GiB/s) | 20260507 06:13:49.500 | INFO | Analyzing chunk frames = [7724032,7724544) | 20260507 06:13:49.500 | INFO | MemInfo: { | 20260507 06:13:49.500 | INFO | "gpu" : " 17 GB", | 20260507 06:13:49.500 | INFO | "huge" : "", | 20260507 06:13:49.500 | INFO | "huge_shared" : " 27 GB", | 20260507 06:13:49.500 | INFO | "malloc" : " 14 GB", | 20260507 06:13:49.500 | INFO | "pinned" : " 29 GB" | 20260507 06:13:49.500 | INFO | } | 20260507 06:13:49.500 | INFO | Perf report for the last 5.14486 seconds: | 20260507 06:13:49.500 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:13:49.500 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:49.500 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:49.500 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:49.501 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:13:49.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:49.501 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:49.501 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:13:49.501 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:49.501 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:49.501 | INFO | Uploaded 36GiB to GPU in 1.98731s (18.1149GiB/s) | 20260507 06:13:49.501 | INFO | Downloaded 34GiB to GPU in 1.97363s (17.2271GiB/s) | 20260507 06:13:54.638 | INFO | Analyzing chunk frames = [7724544,7725056) | 20260507 06:13:54.638 | INFO | MemInfo: { | 20260507 06:13:54.638 | INFO | "gpu" : " 17 GB", | 20260507 06:13:54.638 | INFO | "huge" : "", | 20260507 06:13:54.638 | INFO | "huge_shared" : " 27 GB", | 20260507 06:13:54.639 | INFO | "malloc" : " 14 GB", | 20260507 06:13:54.639 | INFO | "pinned" : " 29 GB" | 20260507 06:13:54.639 | INFO | } | 20260507 06:13:54.639 | INFO | Perf report for the last 5.13812 seconds: | 20260507 06:13:54.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:13:54.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:54.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:54.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:54.639 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:13:54.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:54.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:54.639 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 06:13:54.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:54.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:54.639 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:13:54.639 | INFO | Total mem usage: 18 GiB | 20260507 06:13:54.639 | INFO | Uploaded 36GiB to GPU in 2.0008s (17.9928GiB/s) | 20260507 06:13:54.639 | INFO | Downloaded 34GiB to GPU in 1.96403s (17.3113GiB/s) | 20260507 06:13:54.639 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:13:54.639 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 06:13:54.639 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:13:54.639 | INFO | High Water Breakdown: | 20260507 06:13:54.639 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:13:54.639 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:13:54.639 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:13:54.639 | INFO | ------------------------------------------ | 20260507 06:13:54.639 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:13:54.639 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:13:54.639 | INFO | Total mem usage: 26.5 GiB | 20260507 06:13:54.639 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:13:54.639 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:13:54.639 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:13:54.639 | INFO | High Water Breakdown: | 20260507 06:13:54.639 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:13:54.639 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:13:54.639 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:13:54.639 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:13:54.639 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:13:54.640 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:13:54.640 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:13:54.640 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:13:54.640 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:13:54.640 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:13:54.640 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:13:54.640 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:13:54.640 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:13:54.640 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:13:54.640 | INFO | ------------------------------------------ | 20260507 06:13:54.640 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:13:54.640 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:13:54.640 | INFO | Total mem usage: 25.6 GiB | 20260507 06:13:54.640 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:13:54.640 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:13:54.640 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:13:54.640 | INFO | High Water Breakdown: | 20260507 06:13:54.640 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:13:54.640 | INFO | ------------------------------------------ | 20260507 06:13:54.640 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:13:54.640 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:13:54.640 | INFO | Total mem usage: 15.7 GiB | 20260507 06:13:54.640 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:13:54.640 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:13:54.640 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:13:54.640 | INFO | High Water Breakdown: | 20260507 06:13:54.640 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:13:54.640 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:13:54.640 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:13:54.640 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:13:54.640 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:13:54.640 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:13:54.640 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:13:54.640 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:13:54.640 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:13:54.640 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:13:54.640 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:13:54.640 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:13:54.640 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:13:54.640 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:13:54.640 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:13:54.640 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:13:54.640 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:13:54.640 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:13:54.640 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:13:54.640 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:13:54.640 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:13:54.640 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:13:54.640 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:13:54.640 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:13:54.640 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:13:54.640 | INFO | ------------------------------------------ | 20260507 06:13:54.640 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:13:54.640 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:13:54.640 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:13:54.640 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:13:54.640 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.44e+01, 1.31e+01 / 1.07e+00 | 20260507 06:13:54.640 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.02e-01, 9.48e-03 / 6.33e-03 | 20260507 06:13:54.640 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.41e+00, 5.15e-01 / 3.56e-01 | 20260507 06:13:54.640 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.46e+00, 1.68e-01 / 1.35e-01 | 20260507 06:13:54.640 | INFO | PerfReport: 16.16%, DME -- 3.97e+01 / 4.41e+01, 3.68e+00 / 1.60e+00 | 20260507 06:13:54.640 | INFO | PerfReport: 57.76%, FrameLabeling -- 1.78e+01 / 2.01e+01, 1.38e+00 / 1.71e+00 | 20260507 06:13:54.640 | INFO | PerfReport: 6.91%, PulseAccumulating -- 2.12e+00 / 2.67e+00, 3.84e-01 / 2.41e-01 | 20260507 06:13:54.640 | INFO | PerfReport: 2.96%, Metrics -- 9.10e-01 / 1.23e+00, 2.43e-01 / 1.01e-01 | 20260507 06:13:54.640 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.06e-01 / 3.55e-01, 5.03e-02 / 1.35e-02 | 20260507 06:13:54.640 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:13:54.640 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:13:54.640 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:13:54.640 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 06:13:54.640 | INFO | PerfReport: 21.90%, Upload -- 2.10e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260507 06:13:54.640 | INFO | PerfReport: 21.33%, Download -- 2.05e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 06:13:54.640 | INFO | PerfReport: 31.99%, Compute -- 3.07e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 06:13:54.640 | INFO | PerfReport: 1.52%, LockUpload -- 1.46e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 06:13:54.640 | INFO | PerfReport: 1.23%, LockDownload -- 1.18e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260507 06:13:54.640 | INFO | PerfReport: 22.01%, LockCompute -- 2.11e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 06:13:54.640 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.59e-02 / 0.00e+00 | 20260507 06:13:54.736 | INFO | Starting a round of Baz IO | 20260507 06:13:57.834 | INFO | Populating RTMetrics report | 20260507 06:13:57.835 | INFO | RTMetrics report populated | 20260507 06:13:57.835 | INFO | Opening tmp RTMetrics json file | 20260507 06:13:57.836 | INFO | Serializing RTMetrics to json | 20260507 06:13:57.852 | INFO | Serialization done, swapping tmp file now... | 20260507 06:13:57.854 | INFO | RTMetrics done | 20260507 06:13:57.933 | INFO | Finished a round of Baz IO: Wrote 1.68 GiB in 3.196 seconds | 20260507 06:13:57.933 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.64% / 0.00% / 67.36% | 20260507 06:13:57.933 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.12662% / 0% / 9.93411e-05% | 20260507 06:13:57.933 | INFO | Baz IO overall: | 20260507 06:13:57.933 | INFO | BytesWritten : 3901198310211 | 20260507 06:13:57.933 | INFO | NumEvents : 1869910756377 | 20260507 06:13:59.779 | INFO | Analyzing chunk frames = [7725056,7725568) | 20260507 06:13:59.779 | INFO | MemInfo: { | 20260507 06:13:59.779 | INFO | "gpu" : " 17 GB", | 20260507 06:13:59.779 | INFO | "huge" : "", | 20260507 06:13:59.779 | INFO | "huge_shared" : " 27 GB", | 20260507 06:13:59.779 | INFO | "malloc" : " 14 GB", | 20260507 06:13:59.779 | INFO | "pinned" : " 29 GB" | 20260507 06:13:59.779 | INFO | } | 20260507 06:13:59.779 | INFO | Perf report for the last 5.14089 seconds: | 20260507 06:13:59.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:13:59.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:59.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:59.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:59.779 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 06:13:59.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:13:59.779 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260507 06:13:59.779 | INFO | PRE_HQ : 60.0%, 1.9, 59.5ms, 0 | 20260507 06:13:59.780 | INFO | BAZWRITER : 61.8%, 3.3, 104.2ms, 0 | 20260507 06:13:59.780 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:13:59.780 | INFO | Uploaded 36GiB to GPU in 2.05211s (17.5429GiB/s) | 20260507 06:13:59.780 | INFO | Downloaded 38GiB to GPU in 2.1793s (17.4368GiB/s) | 20260507 06:14:04.920 | INFO | Analyzing chunk frames = [7725568,7726080) | 20260507 06:14:04.920 | INFO | MemInfo: { | 20260507 06:14:04.920 | INFO | "gpu" : " 17 GB", | 20260507 06:14:04.920 | INFO | "huge" : "", | 20260507 06:14:04.920 | INFO | "huge_shared" : " 27 GB", | 20260507 06:14:04.920 | INFO | "malloc" : " 14 GB", | 20260507 06:14:04.920 | INFO | "pinned" : " 29 GB" | 20260507 06:14:04.920 | INFO | } | 20260507 06:14:04.920 | INFO | Perf report for the last 5.14023 seconds: | 20260507 06:14:04.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:14:04.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:04.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:04.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:04.920 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:14:04.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:04.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:04.920 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 06:14:04.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:04.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:04.920 | INFO | Uploaded 36GiB to GPU in 1.99687s (18.0282GiB/s) | 20260507 06:14:04.920 | INFO | Downloaded 34GiB to GPU in 1.93743s (17.549GiB/s) | 20260507 06:14:10.052 | INFO | Analyzing chunk frames = [7726080,7726592) | 20260507 06:14:10.052 | INFO | MemInfo: { | 20260507 06:14:10.052 | INFO | "gpu" : " 17 GB", | 20260507 06:14:10.052 | INFO | "huge" : "", | 20260507 06:14:10.052 | INFO | "huge_shared" : " 27 GB", | 20260507 06:14:10.052 | INFO | "malloc" : " 14 GB", | 20260507 06:14:10.052 | INFO | "pinned" : " 29 GB" | 20260507 06:14:10.052 | INFO | } | 20260507 06:14:10.052 | INFO | Perf report for the last 5.13272 seconds: | 20260507 06:14:10.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:14:10.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:10.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:10.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:10.052 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 06:14:10.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:10.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:10.052 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:14:10.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:10.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:10.053 | INFO | Uploaded 36GiB to GPU in 2.05055s (17.5563GiB/s) | 20260507 06:14:10.053 | INFO | Downloaded 34GiB to GPU in 1.94925s (17.4426GiB/s) | 20260507 06:14:15.199 | INFO | Analyzing chunk frames = [7726592,7727104) | 20260507 06:14:15.199 | INFO | MemInfo: { | 20260507 06:14:15.199 | INFO | "gpu" : " 17 GB", | 20260507 06:14:15.199 | INFO | "huge" : "", | 20260507 06:14:15.199 | INFO | "huge_shared" : " 27 GB", | 20260507 06:14:15.199 | INFO | "malloc" : " 14 GB", | 20260507 06:14:15.199 | INFO | "pinned" : " 29 GB" | 20260507 06:14:15.199 | INFO | } | 20260507 06:14:15.199 | INFO | Perf report for the last 5.14653 seconds: | 20260507 06:14:15.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:14:15.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:15.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:15.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:15.199 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 06:14:15.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:15.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:15.199 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:14:15.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:15.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:15.199 | INFO | Uploaded 36GiB to GPU in 2.01631s (17.8544GiB/s) | 20260507 06:14:15.199 | INFO | Downloaded 34GiB to GPU in 1.93374s (17.5825GiB/s) | 20260507 06:14:20.332 | INFO | Analyzing chunk frames = [7727104,7727616) | 20260507 06:14:20.332 | INFO | MemInfo: { | 20260507 06:14:20.332 | INFO | "gpu" : " 17 GB", | 20260507 06:14:20.332 | INFO | "huge" : "", | 20260507 06:14:20.332 | INFO | "huge_shared" : " 27 GB", | 20260507 06:14:20.332 | INFO | "malloc" : " 14 GB", | 20260507 06:14:20.333 | INFO | "pinned" : " 29 GB" | 20260507 06:14:20.333 | INFO | } | 20260507 06:14:20.333 | INFO | Perf report for the last 5.13358 seconds: | 20260507 06:14:20.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:14:20.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:20.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:20.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:20.333 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 06:14:20.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:20.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:20.333 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 06:14:20.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:20.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:20.333 | INFO | Uploaded 36GiB to GPU in 2.05804s (17.4923GiB/s) | 20260507 06:14:20.333 | INFO | Downloaded 34GiB to GPU in 1.95222s (17.4161GiB/s) | 20260507 06:14:25.476 | INFO | Analyzing chunk frames = [7727616,7728128) | 20260507 06:14:25.476 | INFO | MemInfo: { | 20260507 06:14:25.476 | INFO | "gpu" : " 17 GB", | 20260507 06:14:25.476 | INFO | "huge" : "", | 20260507 06:14:25.476 | INFO | "huge_shared" : " 27 GB", | 20260507 06:14:25.476 | INFO | "malloc" : " 14 GB", | 20260507 06:14:25.476 | INFO | "pinned" : " 29 GB" | 20260507 06:14:25.476 | INFO | } | 20260507 06:14:25.477 | INFO | Perf report for the last 5.14394 seconds: | 20260507 06:14:25.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:14:25.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:25.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:25.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:25.477 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 06:14:25.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:25.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:25.477 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:14:25.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:25.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:25.477 | INFO | Uploaded 36GiB to GPU in 2.03431s (17.6965GiB/s) | 20260507 06:14:25.477 | INFO | Downloaded 34GiB to GPU in 1.90843s (17.8157GiB/s) | 20260507 06:14:30.610 | INFO | Analyzing chunk frames = [7728128,7728640) | 20260507 06:14:30.610 | INFO | MemInfo: { | 20260507 06:14:30.610 | INFO | "gpu" : " 17 GB", | 20260507 06:14:30.610 | INFO | "huge" : "", | 20260507 06:14:30.610 | INFO | "huge_shared" : " 27 GB", | 20260507 06:14:30.610 | INFO | "malloc" : " 14 GB", | 20260507 06:14:30.610 | INFO | "pinned" : " 29 GB" | 20260507 06:14:30.610 | INFO | } | 20260507 06:14:30.610 | INFO | Perf report for the last 5.13363 seconds: | 20260507 06:14:30.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:14:30.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:30.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:30.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:30.610 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:14:30.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:30.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:30.610 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 06:14:30.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:30.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:30.610 | INFO | Uploaded 36GiB to GPU in 1.93515s (18.6032GiB/s) | 20260507 06:14:30.610 | INFO | Downloaded 34GiB to GPU in 1.98404s (17.1367GiB/s) | 20260507 06:14:35.750 | INFO | Analyzing chunk frames = [7728640,7729152) | 20260507 06:14:35.750 | INFO | MemInfo: { | 20260507 06:14:35.750 | INFO | "gpu" : " 17 GB", | 20260507 06:14:35.750 | INFO | "huge" : "", | 20260507 06:14:35.750 | INFO | "huge_shared" : " 27 GB", | 20260507 06:14:35.750 | INFO | "malloc" : " 14 GB", | 20260507 06:14:35.750 | INFO | "pinned" : " 29 GB" | 20260507 06:14:35.750 | INFO | } | 20260507 06:14:35.750 | INFO | Perf report for the last 5.14011 seconds: | 20260507 06:14:35.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:14:35.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:35.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:35.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:35.750 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:14:35.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:35.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:35.750 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:14:35.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:35.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:35.750 | INFO | Uploaded 36GiB to GPU in 1.9647s (18.3234GiB/s) | 20260507 06:14:35.751 | INFO | Downloaded 34GiB to GPU in 1.94943s (17.441GiB/s) | 20260507 06:14:35.847 | INFO | Starting a round of Baz IO | 20260507 06:14:38.949 | INFO | Populating RTMetrics report | 20260507 06:14:38.949 | INFO | RTMetrics report populated | 20260507 06:14:38.949 | INFO | Opening tmp RTMetrics json file | 20260507 06:14:38.952 | INFO | Serializing RTMetrics to json | 20260507 06:14:38.966 | INFO | Serialization done, swapping tmp file now... | 20260507 06:14:38.967 | INFO | RTMetrics done | 20260507 06:14:39.036 | INFO | Finished a round of Baz IO: Wrote 1.678 GiB in 3.188 seconds | 20260507 06:14:39.036 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.64% / 0.00% / 67.36% | 20260507 06:14:39.036 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.126938% / 0% / 7.94729e-05% | 20260507 06:14:39.036 | INFO | Baz IO overall: | 20260507 06:14:39.036 | INFO | BytesWritten : 3902999550845 | 20260507 06:14:39.036 | INFO | NumEvents : 1870753100113 | 20260507 06:14:40.891 | INFO | Analyzing chunk frames = [7729152,7729664) | 20260507 06:14:40.891 | INFO | MemInfo: { | 20260507 06:14:40.891 | INFO | "gpu" : " 17 GB", | 20260507 06:14:40.891 | INFO | "huge" : "", | 20260507 06:14:40.891 | INFO | "huge_shared" : " 27 GB", | 20260507 06:14:40.891 | INFO | "malloc" : " 14 GB", | 20260507 06:14:40.891 | INFO | "pinned" : " 29 GB" | 20260507 06:14:40.892 | INFO | } | 20260507 06:14:40.892 | INFO | Perf report for the last 5.14127 seconds: | 20260507 06:14:40.892 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:14:40.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:40.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:40.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:40.892 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 06:14:40.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:40.892 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 06:14:40.892 | INFO | PRE_HQ : 59.1%, 1.8, 55.2ms, 0 | 20260507 06:14:40.892 | INFO | BAZWRITER : 61.5%, 3.2, 100.1ms, 0 | 20260507 06:14:40.892 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:14:40.892 | INFO | Uploaded 36GiB to GPU in 2.04046s (17.643GiB/s) | 20260507 06:14:40.892 | INFO | Downloaded 38GiB to GPU in 2.17418s (17.4779GiB/s) | 20260507 06:14:46.034 | INFO | Analyzing chunk frames = [7729664,7730176) | 20260507 06:14:46.034 | INFO | MemInfo: { | 20260507 06:14:46.034 | INFO | "gpu" : " 17 GB", | 20260507 06:14:46.034 | INFO | "huge" : "", | 20260507 06:14:46.034 | INFO | "huge_shared" : " 27 GB", | 20260507 06:14:46.034 | INFO | "malloc" : " 14 GB", | 20260507 06:14:46.034 | INFO | "pinned" : " 29 GB" | 20260507 06:14:46.034 | INFO | } | 20260507 06:14:46.034 | INFO | Perf report for the last 5.14252 seconds: | 20260507 06:14:46.034 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:14:46.034 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:46.034 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:46.034 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:46.034 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 06:14:46.034 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:46.034 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:46.034 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:14:46.034 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:46.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:46.034 | INFO | Uploaded 36GiB to GPU in 2.00712s (17.9361GiB/s) | 20260507 06:14:46.034 | INFO | Downloaded 34GiB to GPU in 1.93564s (17.5652GiB/s) | 20260507 06:14:51.170 | INFO | Analyzing chunk frames = [7730176,7730688) | 20260507 06:14:51.170 | INFO | MemInfo: { | 20260507 06:14:51.170 | INFO | "gpu" : " 17 GB", | 20260507 06:14:51.170 | INFO | "huge" : "", | 20260507 06:14:51.170 | INFO | "huge_shared" : " 27 GB", | 20260507 06:14:51.170 | INFO | "malloc" : " 14 GB", | 20260507 06:14:51.170 | INFO | "pinned" : " 29 GB" | 20260507 06:14:51.170 | INFO | } | 20260507 06:14:51.170 | INFO | Perf report for the last 5.13564 seconds: | 20260507 06:14:51.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:14:51.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:51.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:51.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:51.170 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260507 06:14:51.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:51.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:51.170 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:14:51.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:51.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:51.170 | INFO | Uploaded 36GiB to GPU in 2.03229s (17.714GiB/s) | 20260507 06:14:51.170 | INFO | Downloaded 34GiB to GPU in 1.91958s (17.7122GiB/s) | 20260507 06:14:56.304 | INFO | Analyzing chunk frames = [7730688,7731200) | 20260507 06:14:56.304 | INFO | MemInfo: { | 20260507 06:14:56.304 | INFO | "gpu" : " 17 GB", | 20260507 06:14:56.304 | INFO | "huge" : "", | 20260507 06:14:56.304 | INFO | "huge_shared" : " 27 GB", | 20260507 06:14:56.304 | INFO | "malloc" : " 14 GB", | 20260507 06:14:56.304 | INFO | "pinned" : " 29 GB" | 20260507 06:14:56.304 | INFO | } | 20260507 06:14:56.304 | INFO | Perf report for the last 5.13408 seconds: | 20260507 06:14:56.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:14:56.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:56.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:56.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:56.304 | INFO | ANALYSIS : 59.8%, 3.0, 92.6ms, 0 | 20260507 06:14:56.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:56.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:56.304 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 06:14:56.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:56.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:14:56.304 | INFO | Uploaded 36GiB to GPU in 2.02546s (17.7738GiB/s) | 20260507 06:14:56.304 | INFO | Downloaded 34GiB to GPU in 1.93914s (17.5335GiB/s) | 20260507 06:15:01.452 | INFO | Analyzing chunk frames = [7731200,7731712) | 20260507 06:15:01.452 | INFO | MemInfo: { | 20260507 06:15:01.452 | INFO | "gpu" : " 17 GB", | 20260507 06:15:01.452 | INFO | "huge" : "", | 20260507 06:15:01.452 | INFO | "huge_shared" : " 27 GB", | 20260507 06:15:01.452 | INFO | "malloc" : " 14 GB", | 20260507 06:15:01.452 | INFO | "pinned" : " 29 GB" | 20260507 06:15:01.452 | INFO | } | 20260507 06:15:01.452 | INFO | Perf report for the last 5.14843 seconds: | 20260507 06:15:01.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:15:01.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:01.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:01.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:01.452 | INFO | ANALYSIS : 59.9%, 3.0, 93.7ms, 0 | 20260507 06:15:01.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:01.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:01.452 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:15:01.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:01.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:01.453 | INFO | Uploaded 36GiB to GPU in 2.03857s (17.6594GiB/s) | 20260507 06:15:01.453 | INFO | Downloaded 34GiB to GPU in 1.94052s (17.5211GiB/s) | 20260507 06:15:06.596 | INFO | Analyzing chunk frames = [7731712,7732224) | 20260507 06:15:06.596 | INFO | MemInfo: { | 20260507 06:15:06.596 | INFO | "gpu" : " 17 GB", | 20260507 06:15:06.596 | INFO | "huge" : "", | 20260507 06:15:06.596 | INFO | "huge_shared" : " 27 GB", | 20260507 06:15:06.596 | INFO | "malloc" : " 14 GB", | 20260507 06:15:06.596 | INFO | "pinned" : " 29 GB" | 20260507 06:15:06.596 | INFO | } | 20260507 06:15:06.596 | INFO | Perf report for the last 5.14336 seconds: | 20260507 06:15:06.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:15:06.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:06.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:06.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:06.596 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 06:15:06.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:06.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:06.596 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 06:15:06.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:06.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:06.596 | INFO | Uploaded 36GiB to GPU in 2.04217s (17.6283GiB/s) | 20260507 06:15:06.596 | INFO | Downloaded 34GiB to GPU in 1.90487s (17.849GiB/s) | 20260507 06:15:11.733 | INFO | Analyzing chunk frames = [7732224,7732736) | 20260507 06:15:11.733 | INFO | MemInfo: { | 20260507 06:15:11.733 | INFO | "gpu" : " 17 GB", | 20260507 06:15:11.733 | INFO | "huge" : "", | 20260507 06:15:11.733 | INFO | "huge_shared" : " 27 GB", | 20260507 06:15:11.733 | INFO | "malloc" : " 14 GB", | 20260507 06:15:11.733 | INFO | "pinned" : " 29 GB" | 20260507 06:15:11.733 | INFO | } | 20260507 06:15:11.733 | INFO | Perf report for the last 5.13723 seconds: | 20260507 06:15:11.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:15:11.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:11.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:11.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:11.733 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 06:15:11.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:11.733 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:11.733 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 06:15:11.733 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:11.733 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:11.733 | INFO | Uploaded 36GiB to GPU in 1.96993s (18.2748GiB/s) | 20260507 06:15:11.733 | INFO | Downloaded 34GiB to GPU in 1.9697s (17.2615GiB/s) | 20260507 06:15:16.865 | INFO | Analyzing chunk frames = [7732736,7733248) | 20260507 06:15:16.865 | INFO | MemInfo: { | 20260507 06:15:16.865 | INFO | "gpu" : " 17 GB", | 20260507 06:15:16.865 | INFO | "huge" : "", | 20260507 06:15:16.865 | INFO | "huge_shared" : " 27 GB", | 20260507 06:15:16.865 | INFO | "malloc" : " 14 GB", | 20260507 06:15:16.865 | INFO | "pinned" : " 29 GB" | 20260507 06:15:16.865 | INFO | } | 20260507 06:15:16.865 | INFO | Perf report for the last 5.13179 seconds: | 20260507 06:15:16.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:15:16.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:16.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:16.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:16.865 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:15:16.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:16.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:16.865 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 06:15:16.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:16.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:16.865 | INFO | Uploaded 36GiB to GPU in 1.98973s (18.0929GiB/s) | 20260507 06:15:16.865 | INFO | Downloaded 34GiB to GPU in 1.95601s (17.3823GiB/s) | 20260507 06:15:16.865 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:15:16.865 | INFO | Total mem usage: 18 GiB | 20260507 06:15:16.865 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:15:16.865 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:15:16.865 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:15:16.865 | INFO | High Water Breakdown: | 20260507 06:15:16.865 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:15:16.865 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:15:16.865 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:15:16.865 | INFO | ------------------------------------------ | 20260507 06:15:16.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:15:16.865 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:15:16.865 | INFO | Total mem usage: 26.5 GiB | 20260507 06:15:16.865 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:15:16.865 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:15:16.865 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:15:16.865 | INFO | High Water Breakdown: | 20260507 06:15:16.865 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:15:16.865 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:15:16.865 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:15:16.865 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:15:16.865 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:15:16.865 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:15:16.865 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:15:16.865 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:15:16.865 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:15:16.865 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:15:16.865 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:15:16.865 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:15:16.865 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:15:16.865 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:15:16.865 | INFO | ------------------------------------------ | 20260507 06:15:16.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:15:16.865 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:15:16.865 | INFO | Total mem usage: 25.6 GiB | 20260507 06:15:16.865 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:15:16.865 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:15:16.865 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:15:16.865 | INFO | High Water Breakdown: | 20260507 06:15:16.865 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:15:16.865 | INFO | ------------------------------------------ | 20260507 06:15:16.865 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:15:16.865 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:15:16.865 | INFO | Total mem usage: 15.7 GiB | 20260507 06:15:16.865 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:15:16.865 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:15:16.865 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:15:16.865 | INFO | High Water Breakdown: | 20260507 06:15:16.865 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:15:16.865 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:15:16.866 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:15:16.866 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:15:16.866 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:15:16.866 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:15:16.866 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:15:16.866 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:15:16.866 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:15:16.866 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:15:16.866 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:15:16.866 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:15:16.866 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:15:16.866 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:15:16.866 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:15:16.866 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:15:16.866 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:15:16.866 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:15:16.866 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:15:16.866 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:15:16.866 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:15:16.866 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:15:16.866 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:15:16.866 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:15:16.866 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:15:16.866 | INFO | ------------------------------------------ | 20260507 06:15:16.866 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:15:16.866 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:15:16.866 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:15:16.866 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:15:16.866 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 9.96e-01 | 20260507 06:15:16.866 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.02e-01, 9.66e-03 / 4.66e-03 | 20260507 06:15:16.866 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.45e+00, 5.12e-01 / 2.57e-01 | 20260507 06:15:16.866 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.49e+00, 1.69e-01 / 1.04e-01 | 20260507 06:15:16.866 | INFO | PerfReport: 16.22%, DME -- 3.99e+01 / 4.46e+01, 3.98e+00 / 1.42e+00 | 20260507 06:15:16.866 | INFO | PerfReport: 57.68%, FrameLabeling -- 1.77e+01 / 1.98e+01, 1.32e+00 / 1.39e+00 | 20260507 06:15:16.866 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.13e+00 / 2.68e+00, 3.84e-01 / 1.80e-01 | 20260507 06:15:16.866 | INFO | PerfReport: 2.98%, Metrics -- 9.14e-01 / 1.24e+00, 2.47e-01 / 1.26e-01 | 20260507 06:15:16.866 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.07e-01 / 3.68e-01, 5.07e-02 / 2.76e-02 | 20260507 06:15:16.866 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:15:16.866 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:15:16.866 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:15:16.866 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 06:15:16.866 | INFO | PerfReport: 21.84%, Upload -- 2.10e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260507 06:15:16.866 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260507 06:15:16.866 | INFO | PerfReport: 31.96%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:15:16.866 | INFO | PerfReport: 1.58%, LockUpload -- 1.52e+00 / 0.00e+00, 4.98e+00 / 0.00e+00 | 20260507 06:15:16.866 | INFO | PerfReport: 1.19%, LockDownload -- 1.14e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260507 06:15:16.866 | INFO | PerfReport: 22.06%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 06:15:16.866 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260507 06:15:16.961 | INFO | Starting a round of Baz IO | 20260507 06:15:20.065 | INFO | Populating RTMetrics report | 20260507 06:15:20.065 | INFO | RTMetrics report populated | 20260507 06:15:20.065 | INFO | Opening tmp RTMetrics json file | 20260507 06:15:20.083 | INFO | Serializing RTMetrics to json | 20260507 06:15:20.084 | INFO | Serialization done, swapping tmp file now... | 20260507 06:15:20.085 | INFO | RTMetrics done | 20260507 06:15:20.153 | INFO | Finished a round of Baz IO: Wrote 1.677 GiB in 3.192 seconds | 20260507 06:15:20.153 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.64% / 0.00% / 67.36% | 20260507 06:15:20.153 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.126767% / 3.97364e-06% / 9.13938e-05% | 20260507 06:15:20.153 | INFO | Baz IO overall: | 20260507 06:15:20.153 | INFO | BytesWritten : 3904800313087 | 20260507 06:15:20.153 | INFO | NumEvents : 1871595692733 | 20260507 06:15:22.004 | INFO | Analyzing chunk frames = [7733248,7733760) | 20260507 06:15:22.004 | INFO | MemInfo: { | 20260507 06:15:22.004 | INFO | "gpu" : " 17 GB", | 20260507 06:15:22.004 | INFO | "huge" : "", | 20260507 06:15:22.004 | INFO | "huge_shared" : " 27 GB", | 20260507 06:15:22.004 | INFO | "malloc" : " 14 GB", | 20260507 06:15:22.004 | INFO | "pinned" : " 29 GB" | 20260507 06:15:22.004 | INFO | } | 20260507 06:15:22.004 | INFO | Perf report for the last 5.13908 seconds: | 20260507 06:15:22.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:15:22.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:22.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:22.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:22.004 | INFO | ANALYSIS : 61.8%, 3.0, 96.1ms, 0 | 20260507 06:15:22.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:22.004 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 06:15:22.004 | INFO | PRE_HQ : 59.3%, 1.8, 55.2ms, 0 | 20260507 06:15:22.004 | INFO | BAZWRITER : 61.6%, 3.1, 99.1ms, 0 | 20260507 06:15:22.004 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:15:22.004 | INFO | Uploaded 36GiB to GPU in 2.05392s (17.5275GiB/s) | 20260507 06:15:22.004 | INFO | Downloaded 38GiB to GPU in 2.18218s (17.4138GiB/s) | 20260507 06:15:27.148 | INFO | Analyzing chunk frames = [7733760,7734272) | 20260507 06:15:27.148 | INFO | MemInfo: { | 20260507 06:15:27.148 | INFO | "gpu" : " 17 GB", | 20260507 06:15:27.148 | INFO | "huge" : "", | 20260507 06:15:27.148 | INFO | "huge_shared" : " 27 GB", | 20260507 06:15:27.148 | INFO | "malloc" : " 14 GB", | 20260507 06:15:27.148 | INFO | "pinned" : " 29 GB" | 20260507 06:15:27.148 | INFO | } | 20260507 06:15:27.148 | INFO | Perf report for the last 5.14421 seconds: | 20260507 06:15:27.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:15:27.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:27.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:27.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:27.148 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 06:15:27.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:27.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:27.148 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:15:27.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:27.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:27.148 | INFO | Uploaded 36GiB to GPU in 2.04243s (17.6261GiB/s) | 20260507 06:15:27.148 | INFO | Downloaded 34GiB to GPU in 1.93844s (17.5399GiB/s) | 20260507 06:15:32.287 | INFO | Analyzing chunk frames = [7734272,7734784) | 20260507 06:15:32.287 | INFO | MemInfo: { | 20260507 06:15:32.287 | INFO | "gpu" : " 17 GB", | 20260507 06:15:32.287 | INFO | "huge" : "", | 20260507 06:15:32.287 | INFO | "huge_shared" : " 27 GB", | 20260507 06:15:32.287 | INFO | "malloc" : " 14 GB", | 20260507 06:15:32.287 | INFO | "pinned" : " 29 GB" | 20260507 06:15:32.288 | INFO | } | 20260507 06:15:32.288 | INFO | Perf report for the last 5.13957 seconds: | 20260507 06:15:32.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:15:32.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:32.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:32.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:32.288 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 06:15:32.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:32.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:32.288 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:15:32.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:32.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:32.288 | INFO | Uploaded 36GiB to GPU in 2.05651s (17.5054GiB/s) | 20260507 06:15:32.288 | INFO | Downloaded 34GiB to GPU in 1.9505s (17.4314GiB/s) | 20260507 06:15:37.429 | INFO | Analyzing chunk frames = [7734784,7735296) | 20260507 06:15:37.429 | INFO | MemInfo: { | 20260507 06:15:37.429 | INFO | "gpu" : " 17 GB", | 20260507 06:15:37.429 | INFO | "huge" : "", | 20260507 06:15:37.429 | INFO | "huge_shared" : " 27 GB", | 20260507 06:15:37.429 | INFO | "malloc" : " 14 GB", | 20260507 06:15:37.429 | INFO | "pinned" : " 29 GB" | 20260507 06:15:37.429 | INFO | } | 20260507 06:15:37.430 | INFO | Perf report for the last 5.14191 seconds: | 20260507 06:15:37.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:15:37.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:37.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:37.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:37.430 | INFO | ANALYSIS : 60.4%, 2.9, 93.4ms, 0 | 20260507 06:15:37.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:37.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:37.430 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:15:37.430 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:37.430 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:37.430 | INFO | Uploaded 36GiB to GPU in 2.05271s (17.5378GiB/s) | 20260507 06:15:37.430 | INFO | Downloaded 34GiB to GPU in 1.94445s (17.4857GiB/s) | 20260507 06:15:42.563 | INFO | Analyzing chunk frames = [7735296,7735808) | 20260507 06:15:42.563 | INFO | MemInfo: { | 20260507 06:15:42.563 | INFO | "gpu" : " 17 GB", | 20260507 06:15:42.563 | INFO | "huge" : "", | 20260507 06:15:42.563 | INFO | "huge_shared" : " 27 GB", | 20260507 06:15:42.563 | INFO | "malloc" : " 14 GB", | 20260507 06:15:42.563 | INFO | "pinned" : " 29 GB" | 20260507 06:15:42.563 | INFO | } | 20260507 06:15:42.563 | INFO | Perf report for the last 5.13335 seconds: | 20260507 06:15:42.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:15:42.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:42.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:42.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:42.563 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 06:15:42.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:42.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:42.563 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 06:15:42.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:42.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:42.563 | INFO | Uploaded 36GiB to GPU in 2.05582s (17.5113GiB/s) | 20260507 06:15:42.563 | INFO | Downloaded 34GiB to GPU in 1.95179s (17.4199GiB/s) | 20260507 06:15:47.705 | INFO | Analyzing chunk frames = [7735808,7736320) | 20260507 06:15:47.705 | INFO | MemInfo: { | 20260507 06:15:47.705 | INFO | "gpu" : " 17 GB", | 20260507 06:15:47.705 | INFO | "huge" : "", | 20260507 06:15:47.705 | INFO | "huge_shared" : " 27 GB", | 20260507 06:15:47.705 | INFO | "malloc" : " 14 GB", | 20260507 06:15:47.705 | INFO | "pinned" : " 29 GB" | 20260507 06:15:47.705 | INFO | } | 20260507 06:15:47.706 | INFO | Perf report for the last 5.14269 seconds: | 20260507 06:15:47.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:15:47.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:47.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:47.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:47.706 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:15:47.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:47.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:47.706 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:15:47.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:47.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:47.706 | INFO | Uploaded 36GiB to GPU in 2.05319s (17.5337GiB/s) | 20260507 06:15:47.706 | INFO | Downloaded 34GiB to GPU in 1.90883s (17.8119GiB/s) | 20260507 06:15:52.842 | INFO | Analyzing chunk frames = [7736320,7736832) | 20260507 06:15:52.842 | INFO | MemInfo: { | 20260507 06:15:52.842 | INFO | "gpu" : " 17 GB", | 20260507 06:15:52.842 | INFO | "huge" : "", | 20260507 06:15:52.842 | INFO | "huge_shared" : " 27 GB", | 20260507 06:15:52.842 | INFO | "malloc" : " 14 GB", | 20260507 06:15:52.842 | INFO | "pinned" : " 29 GB" | 20260507 06:15:52.842 | INFO | } | 20260507 06:15:52.842 | INFO | Perf report for the last 5.13657 seconds: | 20260507 06:15:52.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:15:52.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:52.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:52.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:52.842 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 06:15:52.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:52.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:52.842 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 06:15:52.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:52.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:52.842 | INFO | Uploaded 36GiB to GPU in 2.03441s (17.6956GiB/s) | 20260507 06:15:52.842 | INFO | Downloaded 34GiB to GPU in 1.97304s (17.2322GiB/s) | 20260507 06:15:57.981 | INFO | Analyzing chunk frames = [7736832,7737344) | 20260507 06:15:57.981 | INFO | MemInfo: { | 20260507 06:15:57.981 | INFO | "gpu" : " 17 GB", | 20260507 06:15:57.981 | INFO | "huge" : "", | 20260507 06:15:57.981 | INFO | "huge_shared" : " 27 GB", | 20260507 06:15:57.981 | INFO | "malloc" : " 14 GB", | 20260507 06:15:57.981 | INFO | "pinned" : " 29 GB" | 20260507 06:15:57.981 | INFO | } | 20260507 06:15:57.981 | INFO | Perf report for the last 5.13878 seconds: | 20260507 06:15:57.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:15:57.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:57.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:57.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:57.981 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:15:57.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:57.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:57.981 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 06:15:57.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:57.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:15:57.981 | INFO | Uploaded 36GiB to GPU in 1.99291s (18.064GiB/s) | 20260507 06:15:57.981 | INFO | Downloaded 34GiB to GPU in 1.95796s (17.3651GiB/s) | 20260507 06:15:58.078 | INFO | Starting a round of Baz IO | 20260507 06:16:01.207 | INFO | Populating RTMetrics report | 20260507 06:16:01.207 | INFO | RTMetrics report populated | 20260507 06:16:01.207 | INFO | Opening tmp RTMetrics json file | 20260507 06:16:01.209 | INFO | Serializing RTMetrics to json | 20260507 06:16:01.212 | INFO | Serialization done, swapping tmp file now... | 20260507 06:16:01.215 | INFO | RTMetrics done | 20260507 06:16:01.307 | INFO | Finished a round of Baz IO: Wrote 1.676 GiB in 3.229 seconds | 20260507 06:16:01.307 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.64% / 0.00% / 67.36% | 20260507 06:16:01.307 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.125909% / 0% / 9.13938e-05% | 20260507 06:16:01.307 | INFO | Baz IO overall: | 20260507 06:16:01.307 | INFO | BytesWritten : 3906599758154 | 20260507 06:16:01.307 | INFO | NumEvents : 1872438061430 | 20260507 06:16:03.114 | INFO | Analyzing chunk frames = [7737344,7737856) | 20260507 06:16:03.114 | INFO | MemInfo: { | 20260507 06:16:03.114 | INFO | "gpu" : " 17 GB", | 20260507 06:16:03.114 | INFO | "huge" : "", | 20260507 06:16:03.114 | INFO | "huge_shared" : " 27 GB", | 20260507 06:16:03.114 | INFO | "malloc" : " 14 GB", | 20260507 06:16:03.114 | INFO | "pinned" : " 29 GB" | 20260507 06:16:03.114 | INFO | } | 20260507 06:16:03.114 | INFO | Perf report for the last 5.13304 seconds: | 20260507 06:16:03.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:16:03.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:03.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:03.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:03.114 | INFO | ANALYSIS : 61.8%, 3.0, 95.6ms, 0 | 20260507 06:16:03.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:03.114 | INFO | RT_METRICS : 45.7%, 1.0, 23.9ms, 0 | 20260507 06:16:03.114 | INFO | PRE_HQ : 60.7%, 2.0, 61.7ms, 0 | 20260507 06:16:03.114 | INFO | BAZWRITER : 62.5%, 3.3, 107.5ms, 0 | 20260507 06:16:03.114 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 06:16:03.114 | INFO | Uploaded 36GiB to GPU in 2.05936s (17.4811GiB/s) | 20260507 06:16:03.114 | INFO | Downloaded 38GiB to GPU in 2.17719s (17.4537GiB/s) | 20260507 06:16:08.259 | INFO | Analyzing chunk frames = [7737856,7738368) | 20260507 06:16:08.259 | INFO | MemInfo: { | 20260507 06:16:08.259 | INFO | "gpu" : " 17 GB", | 20260507 06:16:08.259 | INFO | "huge" : "", | 20260507 06:16:08.259 | INFO | "huge_shared" : " 27 GB", | 20260507 06:16:08.259 | INFO | "malloc" : " 14 GB", | 20260507 06:16:08.259 | INFO | "pinned" : " 29 GB" | 20260507 06:16:08.259 | INFO | } | 20260507 06:16:08.259 | INFO | Perf report for the last 5.14471 seconds: | 20260507 06:16:08.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:16:08.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:08.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:08.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:08.259 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 06:16:08.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:08.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:08.259 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:16:08.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:08.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:08.259 | INFO | Uploaded 36GiB to GPU in 2.00807s (17.9277GiB/s) | 20260507 06:16:08.259 | INFO | Downloaded 34GiB to GPU in 1.9351s (17.5702GiB/s) | 20260507 06:16:13.405 | INFO | Analyzing chunk frames = [7738368,7738880) | 20260507 06:16:13.405 | INFO | MemInfo: { | 20260507 06:16:13.405 | INFO | "gpu" : " 17 GB", | 20260507 06:16:13.405 | INFO | "huge" : "", | 20260507 06:16:13.405 | INFO | "huge_shared" : " 27 GB", | 20260507 06:16:13.405 | INFO | "malloc" : " 14 GB", | 20260507 06:16:13.405 | INFO | "pinned" : " 29 GB" | 20260507 06:16:13.405 | INFO | } | 20260507 06:16:13.405 | INFO | Perf report for the last 5.14604 seconds: | 20260507 06:16:13.405 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:16:13.405 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:13.405 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:13.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:13.405 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 06:16:13.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:13.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:13.405 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:16:13.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:13.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:13.405 | INFO | Uploaded 36GiB to GPU in 2.02215s (17.8028GiB/s) | 20260507 06:16:13.405 | INFO | Downloaded 34GiB to GPU in 1.95404s (17.3999GiB/s) | 20260507 06:16:18.543 | INFO | Analyzing chunk frames = [7738880,7739392) | 20260507 06:16:18.543 | INFO | MemInfo: { | 20260507 06:16:18.543 | INFO | "gpu" : " 17 GB", | 20260507 06:16:18.543 | INFO | "huge" : "", | 20260507 06:16:18.543 | INFO | "huge_shared" : " 27 GB", | 20260507 06:16:18.543 | INFO | "malloc" : " 14 GB", | 20260507 06:16:18.543 | INFO | "pinned" : " 29 GB" | 20260507 06:16:18.543 | INFO | } | 20260507 06:16:18.543 | INFO | Perf report for the last 5.13816 seconds: | 20260507 06:16:18.543 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:16:18.543 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:18.543 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:18.543 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:18.543 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 06:16:18.543 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:18.543 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:18.543 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:16:18.543 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:18.543 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:18.543 | INFO | Uploaded 36GiB to GPU in 2.04505s (17.6035GiB/s) | 20260507 06:16:18.543 | INFO | Downloaded 34GiB to GPU in 1.95109s (17.4262GiB/s) | 20260507 06:16:23.678 | INFO | Analyzing chunk frames = [7739392,7739904) | 20260507 06:16:23.678 | INFO | MemInfo: { | 20260507 06:16:23.678 | INFO | "gpu" : " 17 GB", | 20260507 06:16:23.678 | INFO | "huge" : "", | 20260507 06:16:23.678 | INFO | "huge_shared" : " 27 GB", | 20260507 06:16:23.678 | INFO | "malloc" : " 14 GB", | 20260507 06:16:23.678 | INFO | "pinned" : " 29 GB" | 20260507 06:16:23.678 | INFO | } | 20260507 06:16:23.678 | INFO | Perf report for the last 5.13482 seconds: | 20260507 06:16:23.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:16:23.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:23.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:23.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:23.678 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 06:16:23.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:23.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:23.678 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:16:23.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:23.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:23.678 | INFO | Uploaded 36GiB to GPU in 2.05209s (17.5431GiB/s) | 20260507 06:16:23.678 | INFO | Downloaded 34GiB to GPU in 1.90407s (17.8565GiB/s) | 20260507 06:16:28.817 | INFO | Analyzing chunk frames = [7739904,7740416) | 20260507 06:16:28.817 | INFO | MemInfo: { | 20260507 06:16:28.817 | INFO | "gpu" : " 17 GB", | 20260507 06:16:28.817 | INFO | "huge" : "", | 20260507 06:16:28.817 | INFO | "huge_shared" : " 27 GB", | 20260507 06:16:28.817 | INFO | "malloc" : " 14 GB", | 20260507 06:16:28.818 | INFO | "pinned" : " 29 GB" | 20260507 06:16:28.818 | INFO | } | 20260507 06:16:28.818 | INFO | Perf report for the last 5.13978 seconds: | 20260507 06:16:28.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:16:28.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:28.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:28.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:28.818 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 06:16:28.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:28.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:28.818 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:16:28.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:28.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:28.818 | INFO | Uploaded 36GiB to GPU in 2.05363s (17.53GiB/s) | 20260507 06:16:28.818 | INFO | Downloaded 34GiB to GPU in 1.91585s (17.7467GiB/s) | 20260507 06:16:33.955 | INFO | Analyzing chunk frames = [7740416,7740928) | 20260507 06:16:33.955 | INFO | MemInfo: { | 20260507 06:16:33.955 | INFO | "gpu" : " 17 GB", | 20260507 06:16:33.955 | INFO | "huge" : "", | 20260507 06:16:33.955 | INFO | "huge_shared" : " 27 GB", | 20260507 06:16:33.955 | INFO | "malloc" : " 14 GB", | 20260507 06:16:33.955 | INFO | "pinned" : " 29 GB" | 20260507 06:16:33.955 | INFO | } | 20260507 06:16:33.955 | INFO | Perf report for the last 5.13786 seconds: | 20260507 06:16:33.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:16:33.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:33.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:33.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:33.955 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:16:33.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:33.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:33.955 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:16:33.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:33.956 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:33.956 | INFO | Uploaded 36GiB to GPU in 1.97026s (18.2717GiB/s) | 20260507 06:16:33.956 | INFO | Downloaded 34GiB to GPU in 1.98075s (17.1652GiB/s) | 20260507 06:16:39.099 | INFO | Analyzing chunk frames = [7740928,7741440) | 20260507 06:16:39.099 | INFO | MemInfo: { | 20260507 06:16:39.099 | INFO | "gpu" : " 17 GB", | 20260507 06:16:39.099 | INFO | "huge" : "", | 20260507 06:16:39.099 | INFO | "huge_shared" : " 27 GB", | 20260507 06:16:39.099 | INFO | "malloc" : " 14 GB", | 20260507 06:16:39.099 | INFO | "pinned" : " 29 GB" | 20260507 06:16:39.099 | INFO | } | 20260507 06:16:39.099 | INFO | Perf report for the last 5.14374 seconds: | 20260507 06:16:39.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:16:39.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:39.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:39.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:39.099 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:16:39.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:39.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:39.099 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 06:16:39.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:39.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:39.099 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:16:39.099 | INFO | Total mem usage: 18 GiB | 20260507 06:16:39.099 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:16:39.099 | INFO | Uploaded 36GiB to GPU in 1.97626s (18.2162GiB/s) | 20260507 06:16:39.099 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:16:39.099 | INFO | Downloaded 34GiB to GPU in 1.96344s (17.3165GiB/s) | 20260507 06:16:39.100 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:16:39.100 | INFO | High Water Breakdown: | 20260507 06:16:39.100 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:16:39.100 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:16:39.100 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:16:39.100 | INFO | ------------------------------------------ | 20260507 06:16:39.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:16:39.100 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:16:39.100 | INFO | Total mem usage: 26.5 GiB | 20260507 06:16:39.100 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:16:39.100 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 06:16:39.100 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 06:16:39.100 | INFO | High Water Breakdown: | 20260507 06:16:39.100 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:16:39.100 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:16:39.100 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:16:39.100 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:16:39.100 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:16:39.100 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:16:39.100 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:16:39.100 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:16:39.100 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:16:39.100 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:16:39.100 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:16:39.100 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:16:39.100 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:16:39.100 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:16:39.100 | INFO | ------------------------------------------ | 20260507 06:16:39.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:16:39.100 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:16:39.100 | INFO | Total mem usage: 25.6 GiB | 20260507 06:16:39.100 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:16:39.100 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:16:39.100 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:16:39.100 | INFO | High Water Breakdown: | 20260507 06:16:39.100 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:16:39.100 | INFO | ------------------------------------------ | 20260507 06:16:39.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:16:39.100 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:16:39.100 | INFO | Total mem usage: 15.7 GiB | 20260507 06:16:39.100 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:16:39.100 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:16:39.100 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:16:39.100 | INFO | High Water Breakdown: | 20260507 06:16:39.100 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:16:39.100 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:16:39.100 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:16:39.100 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:16:39.100 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:16:39.100 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:16:39.100 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:16:39.100 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:16:39.100 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:16:39.100 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:16:39.100 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:16:39.100 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:16:39.100 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:16:39.100 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:16:39.100 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:16:39.100 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:16:39.100 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:16:39.100 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:16:39.100 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:16:39.100 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:16:39.100 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:16:39.100 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:16:39.100 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:16:39.100 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:16:39.100 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:16:39.100 | INFO | ------------------------------------------ | 20260507 06:16:39.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:16:39.100 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:16:39.100 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:16:39.100 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:16:39.100 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.30e+01 / 1.60e+00 | 20260507 06:16:39.100 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 9.18e-03 / 5.85e-03 | 20260507 06:16:39.100 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.43e+00, 4.91e-01 / 2.81e-01 | 20260507 06:16:39.100 | INFO | PerfReport: 3.21%, Histogram -- 9.83e-01 / 1.48e+00, 1.60e-01 / 1.23e-01 | 20260507 06:16:39.100 | INFO | PerfReport: 16.31%, DME -- 4.00e+01 / 4.48e+01, 3.99e+00 / 1.75e+00 | 20260507 06:16:39.100 | INFO | PerfReport: 57.65%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.21e+00 / 1.04e+00 | 20260507 06:16:39.100 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.71e+00, 3.83e-01 / 1.31e-01 | 20260507 06:16:39.100 | INFO | PerfReport: 2.96%, Metrics -- 9.07e-01 / 1.23e+00, 2.39e-01 / 1.10e-01 | 20260507 06:16:39.100 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.06e-01 / 3.63e-01, 5.00e-02 / 1.98e-02 | 20260507 06:16:39.100 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:16:39.100 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:16:39.100 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:16:39.100 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 06:16:39.100 | INFO | PerfReport: 21.99%, Upload -- 2.12e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 06:16:39.100 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260507 06:16:39.100 | INFO | PerfReport: 31.84%, Compute -- 3.07e+01 / 0.00e+00, 1.30e+01 / 0.00e+00 | 20260507 06:16:39.100 | INFO | PerfReport: 1.71%, LockUpload -- 1.65e+00 / 0.00e+00, 5.03e+00 / 0.00e+00 | 20260507 06:16:39.100 | INFO | PerfReport: 1.19%, LockDownload -- 1.15e+00 / 0.00e+00, 2.81e+00 / 0.00e+00 | 20260507 06:16:39.100 | INFO | PerfReport: 21.89%, LockCompute -- 2.11e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 06:16:39.100 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260507 06:16:39.196 | INFO | Starting a round of Baz IO | 20260507 06:16:42.291 | INFO | Populating RTMetrics report | 20260507 06:16:42.291 | INFO | RTMetrics report populated | 20260507 06:16:42.291 | INFO | Opening tmp RTMetrics json file | 20260507 06:16:42.292 | INFO | Serializing RTMetrics to json | 20260507 06:16:42.293 | INFO | Serialization done, swapping tmp file now... | 20260507 06:16:42.294 | INFO | RTMetrics done | 20260507 06:16:42.377 | INFO | Finished a round of Baz IO: Wrote 1.675 GiB in 3.181 seconds | 20260507 06:16:42.377 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.64% / 0.00% / 67.36% | 20260507 06:16:42.377 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.126394% / 0% / 9.13938e-05% | 20260507 06:16:42.377 | INFO | Baz IO overall: | 20260507 06:16:42.377 | INFO | BytesWritten : 3908398437225 | 20260507 06:16:42.377 | INFO | NumEvents : 1873280022174 | 20260507 06:16:44.243 | INFO | Analyzing chunk frames = [7741440,7741952) | 20260507 06:16:44.243 | INFO | MemInfo: { | 20260507 06:16:44.243 | INFO | "gpu" : " 17 GB", | 20260507 06:16:44.243 | INFO | "huge" : "", | 20260507 06:16:44.243 | INFO | "huge_shared" : " 27 GB", | 20260507 06:16:44.243 | INFO | "malloc" : " 14 GB", | 20260507 06:16:44.243 | INFO | "pinned" : " 29 GB" | 20260507 06:16:44.243 | INFO | } | 20260507 06:16:44.243 | INFO | Perf report for the last 5.14339 seconds: | 20260507 06:16:44.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:16:44.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:44.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:44.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:44.243 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 06:16:44.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:44.243 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 06:16:44.243 | INFO | PRE_HQ : 59.0%, 1.8, 54.7ms, 0 | 20260507 06:16:44.243 | INFO | BAZWRITER : 61.4%, 3.2, 99.9ms, 0 | 20260507 06:16:44.243 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:16:44.243 | INFO | Uploaded 36GiB to GPU in 2.06151s (17.4629GiB/s) | 20260507 06:16:44.243 | INFO | Downloaded 38GiB to GPU in 2.18146s (17.4195GiB/s) | 20260507 06:16:49.370 | INFO | Analyzing chunk frames = [7741952,7742464) | 20260507 06:16:49.370 | INFO | MemInfo: { | 20260507 06:16:49.370 | INFO | "gpu" : " 17 GB", | 20260507 06:16:49.370 | INFO | "huge" : "", | 20260507 06:16:49.370 | INFO | "huge_shared" : " 27 GB", | 20260507 06:16:49.370 | INFO | "malloc" : " 14 GB", | 20260507 06:16:49.370 | INFO | "pinned" : " 29 GB" | 20260507 06:16:49.370 | INFO | } | 20260507 06:16:49.370 | INFO | Perf report for the last 5.12726 seconds: | 20260507 06:16:49.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:16:49.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:49.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:49.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:49.370 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 06:16:49.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:49.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:49.370 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 06:16:49.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:49.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:49.370 | INFO | Uploaded 36GiB to GPU in 1.99994s (18.0006GiB/s) | 20260507 06:16:49.370 | INFO | Downloaded 34GiB to GPU in 1.9406s (17.5204GiB/s) | 20260507 06:16:54.512 | INFO | Analyzing chunk frames = [7742464,7742976) | 20260507 06:16:54.512 | INFO | MemInfo: { | 20260507 06:16:54.512 | INFO | "gpu" : " 17 GB", | 20260507 06:16:54.512 | INFO | "huge" : "", | 20260507 06:16:54.512 | INFO | "huge_shared" : " 27 GB", | 20260507 06:16:54.512 | INFO | "malloc" : " 14 GB", | 20260507 06:16:54.512 | INFO | "pinned" : " 29 GB" | 20260507 06:16:54.512 | INFO | } | 20260507 06:16:54.513 | INFO | Perf report for the last 5.14265 seconds: | 20260507 06:16:54.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:16:54.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:54.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:54.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:54.513 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 06:16:54.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:54.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:54.513 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:16:54.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:54.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:54.513 | INFO | Uploaded 36GiB to GPU in 2.02383s (17.7881GiB/s) | 20260507 06:16:54.513 | INFO | Downloaded 34GiB to GPU in 1.93995s (17.5263GiB/s) | 20260507 06:16:59.653 | INFO | Analyzing chunk frames = [7742976,7743488) | 20260507 06:16:59.653 | INFO | MemInfo: { | 20260507 06:16:59.653 | INFO | "gpu" : " 17 GB", | 20260507 06:16:59.653 | INFO | "huge" : "", | 20260507 06:16:59.653 | INFO | "huge_shared" : " 27 GB", | 20260507 06:16:59.653 | INFO | "malloc" : " 14 GB", | 20260507 06:16:59.653 | INFO | "pinned" : " 29 GB" | 20260507 06:16:59.653 | INFO | } | 20260507 06:16:59.653 | INFO | Perf report for the last 5.14088 seconds: | 20260507 06:16:59.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:16:59.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:59.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:59.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:59.653 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 06:16:59.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:59.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:59.654 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 06:16:59.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:59.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:16:59.654 | INFO | Uploaded 36GiB to GPU in 2.01295s (17.8842GiB/s) | 20260507 06:16:59.654 | INFO | Downloaded 34GiB to GPU in 1.93394s (17.5807GiB/s) | 20260507 06:17:04.791 | INFO | Analyzing chunk frames = [7743488,7744000) | 20260507 06:17:04.791 | INFO | MemInfo: { | 20260507 06:17:04.791 | INFO | "gpu" : " 17 GB", | 20260507 06:17:04.791 | INFO | "huge" : "", | 20260507 06:17:04.791 | INFO | "huge_shared" : " 27 GB", | 20260507 06:17:04.791 | INFO | "malloc" : " 14 GB", | 20260507 06:17:04.791 | INFO | "pinned" : " 29 GB" | 20260507 06:17:04.791 | INFO | } | 20260507 06:17:04.791 | INFO | Perf report for the last 5.13763 seconds: | 20260507 06:17:04.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:17:04.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:04.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:04.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:04.791 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 06:17:04.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:04.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:04.791 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:17:04.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:04.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:04.791 | INFO | Uploaded 36GiB to GPU in 2.05731s (17.4986GiB/s) | 20260507 06:17:04.791 | INFO | Downloaded 34GiB to GPU in 1.94829s (17.4512GiB/s) | 20260507 06:17:09.931 | INFO | Analyzing chunk frames = [7744000,7744512) | 20260507 06:17:09.931 | INFO | MemInfo: { | 20260507 06:17:09.931 | INFO | "gpu" : " 17 GB", | 20260507 06:17:09.931 | INFO | "huge" : "", | 20260507 06:17:09.931 | INFO | "huge_shared" : " 27 GB", | 20260507 06:17:09.931 | INFO | "malloc" : " 14 GB", | 20260507 06:17:09.931 | INFO | "pinned" : " 29 GB" | 20260507 06:17:09.931 | INFO | } | 20260507 06:17:09.931 | INFO | Perf report for the last 5.1402 seconds: | 20260507 06:17:09.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:17:09.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:09.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:09.931 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:09.931 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 06:17:09.931 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:09.931 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:09.931 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:17:09.931 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:09.931 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:09.932 | INFO | Uploaded 36GiB to GPU in 2.04263s (17.6243GiB/s) | 20260507 06:17:09.932 | INFO | Downloaded 34GiB to GPU in 1.9085s (17.815GiB/s) | 20260507 06:17:15.068 | INFO | Analyzing chunk frames = [7744512,7745024) | 20260507 06:17:15.068 | INFO | MemInfo: { | 20260507 06:17:15.068 | INFO | "gpu" : " 17 GB", | 20260507 06:17:15.068 | INFO | "huge" : "", | 20260507 06:17:15.068 | INFO | "huge_shared" : " 27 GB", | 20260507 06:17:15.068 | INFO | "malloc" : " 14 GB", | 20260507 06:17:15.068 | INFO | "pinned" : " 29 GB" | 20260507 06:17:15.068 | INFO | } | 20260507 06:17:15.068 | INFO | Perf report for the last 5.13692 seconds: | 20260507 06:17:15.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:17:15.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:15.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:15.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:15.068 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 06:17:15.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:15.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:15.068 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 06:17:15.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:15.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:15.069 | INFO | Uploaded 36GiB to GPU in 2.03977s (17.649GiB/s) | 20260507 06:17:15.069 | INFO | Downloaded 34GiB to GPU in 1.97317s (17.2311GiB/s) | 20260507 06:17:20.213 | INFO | Analyzing chunk frames = [7745024,7745536) | 20260507 06:17:20.213 | INFO | MemInfo: { | 20260507 06:17:20.213 | INFO | "gpu" : " 17 GB", | 20260507 06:17:20.213 | INFO | "huge" : "", | 20260507 06:17:20.213 | INFO | "huge_shared" : " 27 GB", | 20260507 06:17:20.213 | INFO | "malloc" : " 14 GB", | 20260507 06:17:20.213 | INFO | "pinned" : " 29 GB" | 20260507 06:17:20.213 | INFO | } | 20260507 06:17:20.213 | INFO | Perf report for the last 5.14494 seconds: | 20260507 06:17:20.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:17:20.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:20.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:20.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:20.213 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:17:20.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:20.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:20.213 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 06:17:20.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:20.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:20.213 | INFO | Uploaded 36GiB to GPU in 1.96509s (18.3198GiB/s) | 20260507 06:17:20.213 | INFO | Downloaded 34GiB to GPU in 1.95545s (17.3873GiB/s) | 20260507 06:17:20.318 | INFO | Starting a round of Baz IO | 20260507 06:17:23.402 | INFO | Populating RTMetrics report | 20260507 06:17:23.402 | INFO | RTMetrics report populated | 20260507 06:17:23.402 | INFO | Opening tmp RTMetrics json file | 20260507 06:17:23.403 | INFO | Serializing RTMetrics to json | 20260507 06:17:23.407 | INFO | Serialization done, swapping tmp file now... | 20260507 06:17:23.414 | INFO | RTMetrics done | 20260507 06:17:23.485 | INFO | Finished a round of Baz IO: Wrote 1.675 GiB in 3.167 seconds | 20260507 06:17:23.485 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.64% / 0.00% / 67.36% | 20260507 06:17:23.485 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.125794% / 0% / 9.93411e-05% | 20260507 06:17:23.485 | INFO | Baz IO overall: | 20260507 06:17:23.485 | INFO | BytesWritten : 3910197088109 | 20260507 06:17:23.485 | INFO | NumEvents : 1874122425904 | 20260507 06:17:25.352 | INFO | Analyzing chunk frames = [7745536,7746048) | 20260507 06:17:25.352 | INFO | MemInfo: { | 20260507 06:17:25.352 | INFO | "gpu" : " 17 GB", | 20260507 06:17:25.352 | INFO | "huge" : "", | 20260507 06:17:25.352 | INFO | "huge_shared" : " 27 GB", | 20260507 06:17:25.352 | INFO | "malloc" : " 14 GB", | 20260507 06:17:25.352 | INFO | "pinned" : " 29 GB" | 20260507 06:17:25.352 | INFO | } | 20260507 06:17:25.352 | INFO | Perf report for the last 5.13904 seconds: | 20260507 06:17:25.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:17:25.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:25.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:25.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:25.352 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 06:17:25.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:25.352 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:17:25.352 | INFO | PRE_HQ : 59.0%, 1.8, 54.9ms, 0 | 20260507 06:17:25.352 | INFO | BAZWRITER : 61.6%, 3.1, 99.1ms, 0 | 20260507 06:17:25.352 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 06:17:25.352 | INFO | Uploaded 36GiB to GPU in 2.04672s (17.5891GiB/s) | 20260507 06:17:25.352 | INFO | Downloaded 38GiB to GPU in 2.17278s (17.4891GiB/s) | 20260507 06:17:30.487 | INFO | Analyzing chunk frames = [7746048,7746560) | 20260507 06:17:30.487 | INFO | MemInfo: { | 20260507 06:17:30.487 | INFO | "gpu" : " 17 GB", | 20260507 06:17:30.487 | INFO | "huge" : "", | 20260507 06:17:30.487 | INFO | "huge_shared" : " 27 GB", | 20260507 06:17:30.487 | INFO | "malloc" : " 14 GB", | 20260507 06:17:30.487 | INFO | "pinned" : " 29 GB" | 20260507 06:17:30.487 | INFO | } | 20260507 06:17:30.487 | INFO | Perf report for the last 5.13483 seconds: | 20260507 06:17:30.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:17:30.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:30.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:30.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:30.487 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:17:30.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:30.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:30.487 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:17:30.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:30.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:30.487 | INFO | Uploaded 36GiB to GPU in 1.99103s (18.081GiB/s) | 20260507 06:17:30.487 | INFO | Downloaded 34GiB to GPU in 1.93056s (17.6114GiB/s) | 20260507 06:17:35.623 | INFO | Analyzing chunk frames = [7746560,7747072) | 20260507 06:17:35.623 | INFO | MemInfo: { | 20260507 06:17:35.623 | INFO | "gpu" : " 17 GB", | 20260507 06:17:35.623 | INFO | "huge" : "", | 20260507 06:17:35.624 | INFO | "huge_shared" : " 27 GB", | 20260507 06:17:35.624 | INFO | "malloc" : " 14 GB", | 20260507 06:17:35.624 | INFO | "pinned" : " 29 GB" | 20260507 06:17:35.624 | INFO | } | 20260507 06:17:35.624 | INFO | Perf report for the last 5.13658 seconds: | 20260507 06:17:35.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:17:35.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:35.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:35.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:35.624 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 06:17:35.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:35.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:35.624 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:17:35.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:35.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:35.624 | INFO | Uploaded 36GiB to GPU in 2.05279s (17.5371GiB/s) | 20260507 06:17:35.624 | INFO | Downloaded 34GiB to GPU in 1.95075s (17.4292GiB/s) | 20260507 06:17:40.768 | INFO | Analyzing chunk frames = [7747072,7747584) | 20260507 06:17:40.768 | INFO | MemInfo: { | 20260507 06:17:40.768 | INFO | "gpu" : " 17 GB", | 20260507 06:17:40.768 | INFO | "huge" : "", | 20260507 06:17:40.768 | INFO | "huge_shared" : " 27 GB", | 20260507 06:17:40.768 | INFO | "malloc" : " 14 GB", | 20260507 06:17:40.768 | INFO | "pinned" : " 29 GB" | 20260507 06:17:40.768 | INFO | } | 20260507 06:17:40.768 | INFO | Perf report for the last 5.14481 seconds: | 20260507 06:17:40.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:17:40.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:40.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:40.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:40.768 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 06:17:40.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:40.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:40.769 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:17:40.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:40.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:40.769 | INFO | Uploaded 36GiB to GPU in 2.01004s (17.9101GiB/s) | 20260507 06:17:40.769 | INFO | Downloaded 34GiB to GPU in 1.92689s (17.645GiB/s) | 20260507 06:17:45.912 | INFO | Analyzing chunk frames = [7747584,7748096) | 20260507 06:17:45.912 | INFO | MemInfo: { | 20260507 06:17:45.912 | INFO | "gpu" : " 17 GB", | 20260507 06:17:45.912 | INFO | "huge" : "", | 20260507 06:17:45.912 | INFO | "huge_shared" : " 27 GB", | 20260507 06:17:45.912 | INFO | "malloc" : " 14 GB", | 20260507 06:17:45.912 | INFO | "pinned" : " 29 GB" | 20260507 06:17:45.912 | INFO | } | 20260507 06:17:45.912 | INFO | Perf report for the last 5.144 seconds: | 20260507 06:17:45.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:17:45.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:45.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:45.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:45.912 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 06:17:45.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:45.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:45.913 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 06:17:45.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:45.913 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:45.913 | INFO | Uploaded 36GiB to GPU in 2.05798s (17.4929GiB/s) | 20260507 06:17:45.913 | INFO | Downloaded 34GiB to GPU in 1.94722s (17.4608GiB/s) | 20260507 06:17:51.053 | INFO | Analyzing chunk frames = [7748096,7748608) | 20260507 06:17:51.053 | INFO | MemInfo: { | 20260507 06:17:51.053 | INFO | "gpu" : " 17 GB", | 20260507 06:17:51.053 | INFO | "huge" : "", | 20260507 06:17:51.053 | INFO | "huge_shared" : " 27 GB", | 20260507 06:17:51.053 | INFO | "malloc" : " 14 GB", | 20260507 06:17:51.053 | INFO | "pinned" : " 29 GB" | 20260507 06:17:51.053 | INFO | } | 20260507 06:17:51.054 | INFO | Perf report for the last 5.14105 seconds: | 20260507 06:17:51.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:17:51.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:51.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:51.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:51.054 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:17:51.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:51.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:51.054 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:17:51.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:51.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:51.054 | INFO | Uploaded 36GiB to GPU in 2.06969s (17.3939GiB/s) | 20260507 06:17:51.054 | INFO | Downloaded 34GiB to GPU in 1.92368s (17.6745GiB/s) | 20260507 06:17:56.190 | INFO | Analyzing chunk frames = [7748608,7749120) | 20260507 06:17:56.190 | INFO | MemInfo: { | 20260507 06:17:56.190 | INFO | "gpu" : " 17 GB", | 20260507 06:17:56.190 | INFO | "huge" : "", | 20260507 06:17:56.190 | INFO | "huge_shared" : " 27 GB", | 20260507 06:17:56.190 | INFO | "malloc" : " 14 GB", | 20260507 06:17:56.190 | INFO | "pinned" : " 29 GB" | 20260507 06:17:56.190 | INFO | } | 20260507 06:17:56.190 | INFO | Perf report for the last 5.13685 seconds: | 20260507 06:17:56.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:17:56.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:56.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:56.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:56.190 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 06:17:56.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:56.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:56.190 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 06:17:56.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:56.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:17:56.191 | INFO | Uploaded 36GiB to GPU in 1.91534s (18.7956GiB/s) | 20260507 06:17:56.191 | INFO | Downloaded 34GiB to GPU in 1.96716s (17.2838GiB/s) | 20260507 06:18:01.324 | INFO | Analyzing chunk frames = [7749120,7749632) | 20260507 06:18:01.324 | INFO | MemInfo: { | 20260507 06:18:01.324 | INFO | "gpu" : " 17 GB", | 20260507 06:18:01.324 | INFO | "huge" : "", | 20260507 06:18:01.324 | INFO | "huge_shared" : " 27 GB", | 20260507 06:18:01.324 | INFO | "malloc" : " 14 GB", | 20260507 06:18:01.324 | INFO | "pinned" : " 29 GB" | 20260507 06:18:01.324 | INFO | } | 20260507 06:18:01.324 | INFO | Perf report for the last 5.13371 seconds: | 20260507 06:18:01.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:18:01.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:01.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:01.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:01.324 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 06:18:01.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:01.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:01.324 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 06:18:01.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:01.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:01.324 | INFO | Uploaded 36GiB to GPU in 1.97281s (18.2481GiB/s) | 20260507 06:18:01.324 | INFO | Downloaded 34GiB to GPU in 1.95266s (17.4122GiB/s) | 20260507 06:18:01.324 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:18:01.324 | INFO | Total mem usage: 18 GiB | 20260507 06:18:01.324 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:18:01.324 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:18:01.324 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:18:01.324 | INFO | High Water Breakdown: | 20260507 06:18:01.324 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:18:01.325 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:18:01.325 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:18:01.325 | INFO | ------------------------------------------ | 20260507 06:18:01.325 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:18:01.325 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:18:01.325 | INFO | Total mem usage: 26.5 GiB | 20260507 06:18:01.325 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:18:01.325 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:18:01.325 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 06:18:01.325 | INFO | High Water Breakdown: | 20260507 06:18:01.325 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:18:01.325 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:18:01.325 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:18:01.325 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:18:01.325 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:18:01.325 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:18:01.325 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:18:01.325 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:18:01.325 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:18:01.325 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:18:01.325 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:18:01.325 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:18:01.325 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:18:01.325 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:18:01.325 | INFO | ------------------------------------------ | 20260507 06:18:01.325 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:18:01.325 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:18:01.325 | INFO | Total mem usage: 25.6 GiB | 20260507 06:18:01.325 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:18:01.325 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:18:01.325 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:18:01.325 | INFO | High Water Breakdown: | 20260507 06:18:01.325 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:18:01.325 | INFO | ------------------------------------------ | 20260507 06:18:01.325 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:18:01.325 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:18:01.325 | INFO | Total mem usage: 15.7 GiB | 20260507 06:18:01.325 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:18:01.325 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:18:01.325 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:18:01.325 | INFO | High Water Breakdown: | 20260507 06:18:01.325 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:18:01.325 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:18:01.325 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:18:01.325 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:18:01.325 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:18:01.325 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:18:01.325 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:18:01.325 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:18:01.325 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:18:01.325 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:18:01.325 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:18:01.325 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:18:01.325 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:18:01.325 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:18:01.325 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:18:01.325 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:18:01.325 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:18:01.325 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:18:01.325 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:18:01.325 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:18:01.325 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:18:01.325 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:18:01.325 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:18:01.325 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:18:01.325 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:18:01.325 | INFO | ------------------------------------------ | 20260507 06:18:01.325 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:18:01.325 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:18:01.325 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:18:01.325 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:18:01.325 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.33e+01 / 1.10e+00 | 20260507 06:18:01.325 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.03e-01, 9.98e-03 / 7.59e-03 | 20260507 06:18:01.325 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.48e+00, 5.01e-01 / 2.06e-01 | 20260507 06:18:01.325 | INFO | PerfReport: 3.23%, Histogram -- 9.90e-01 / 1.48e+00, 1.67e-01 / 9.21e-02 | 20260507 06:18:01.325 | INFO | PerfReport: 16.21%, DME -- 3.97e+01 / 4.41e+01, 3.72e+00 / 1.31e+00 | 20260507 06:18:01.325 | INFO | PerfReport: 57.69%, FrameLabeling -- 1.77e+01 / 2.00e+01, 1.25e+00 / 1.47e+00 | 20260507 06:18:01.325 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.13e+00 / 2.71e+00, 3.91e-01 / 1.88e-01 | 20260507 06:18:01.325 | INFO | PerfReport: 2.98%, Metrics -- 9.13e-01 / 1.23e+00, 2.47e-01 / 1.16e-01 | 20260507 06:18:01.325 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.06e-01 / 3.66e-01, 5.04e-02 / 2.95e-02 | 20260507 06:18:01.325 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:18:01.325 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:18:01.325 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:18:01.325 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 06:18:01.325 | INFO | PerfReport: 21.90%, Upload -- 2.10e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260507 06:18:01.325 | INFO | PerfReport: 21.38%, Download -- 2.05e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260507 06:18:01.325 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 06:18:01.325 | INFO | PerfReport: 1.57%, LockUpload -- 1.51e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 06:18:01.325 | INFO | PerfReport: 1.21%, LockDownload -- 1.16e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260507 06:18:01.325 | INFO | PerfReport: 22.03%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 06:18:01.325 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260507 06:18:01.421 | INFO | Starting a round of Baz IO | 20260507 06:18:04.518 | INFO | Populating RTMetrics report | 20260507 06:18:04.518 | INFO | RTMetrics report populated | 20260507 06:18:04.518 | INFO | Opening tmp RTMetrics json file | 20260507 06:18:04.534 | INFO | Serializing RTMetrics to json | 20260507 06:18:04.535 | INFO | Serialization done, swapping tmp file now... | 20260507 06:18:04.536 | INFO | RTMetrics done | 20260507 06:18:04.604 | INFO | Finished a round of Baz IO: Wrote 1.672 GiB in 3.183 seconds | 20260507 06:18:04.604 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.64% / 0.00% / 67.36% | 20260507 06:18:04.604 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.125468% / 0% / 7.54992e-05% | 20260507 06:18:04.604 | INFO | Baz IO overall: | 20260507 06:18:04.604 | INFO | BytesWritten : 3911992363225 | 20260507 06:18:04.604 | INFO | NumEvents : 1874962837210 | 20260507 06:18:06.464 | INFO | Analyzing chunk frames = [7749632,7750144) | 20260507 06:18:06.464 | INFO | MemInfo: { | 20260507 06:18:06.464 | INFO | "gpu" : " 17 GB", | 20260507 06:18:06.464 | INFO | "huge" : "", | 20260507 06:18:06.464 | INFO | "huge_shared" : " 27 GB", | 20260507 06:18:06.464 | INFO | "malloc" : " 14 GB", | 20260507 06:18:06.464 | INFO | "pinned" : " 29 GB" | 20260507 06:18:06.464 | INFO | } | 20260507 06:18:06.465 | INFO | Perf report for the last 5.1404 seconds: | 20260507 06:18:06.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:18:06.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:06.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:06.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:06.465 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 06:18:06.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:06.465 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 06:18:06.465 | INFO | PRE_HQ : 59.0%, 1.8, 54.7ms, 0 | 20260507 06:18:06.465 | INFO | BAZWRITER : 61.5%, 3.1, 99.1ms, 0 | 20260507 06:18:06.465 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:18:06.465 | INFO | Uploaded 36GiB to GPU in 2.04026s (17.6448GiB/s) | 20260507 06:18:06.465 | INFO | Downloaded 38GiB to GPU in 2.18392s (17.3999GiB/s) | 20260507 06:18:11.608 | INFO | Analyzing chunk frames = [7750144,7750656) | 20260507 06:18:11.608 | INFO | MemInfo: { | 20260507 06:18:11.608 | INFO | "gpu" : " 17 GB", | 20260507 06:18:11.608 | INFO | "huge" : "", | 20260507 06:18:11.608 | INFO | "huge_shared" : " 27 GB", | 20260507 06:18:11.608 | INFO | "malloc" : " 14 GB", | 20260507 06:18:11.608 | INFO | "pinned" : " 29 GB" | 20260507 06:18:11.608 | INFO | } | 20260507 06:18:11.608 | INFO | Perf report for the last 5.14381 seconds: | 20260507 06:18:11.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:18:11.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:11.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:11.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:11.608 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 06:18:11.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:11.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:11.608 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 06:18:11.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:11.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:11.609 | INFO | Uploaded 36GiB to GPU in 2.00021s (17.9982GiB/s) | 20260507 06:18:11.609 | INFO | Downloaded 34GiB to GPU in 1.92957s (17.6205GiB/s) | 20260507 06:18:16.747 | INFO | Analyzing chunk frames = [7750656,7751168) | 20260507 06:18:16.747 | INFO | MemInfo: { | 20260507 06:18:16.747 | INFO | "gpu" : " 17 GB", | 20260507 06:18:16.747 | INFO | "huge" : "", | 20260507 06:18:16.747 | INFO | "huge_shared" : " 27 GB", | 20260507 06:18:16.747 | INFO | "malloc" : " 14 GB", | 20260507 06:18:16.747 | INFO | "pinned" : " 29 GB" | 20260507 06:18:16.747 | INFO | } | 20260507 06:18:16.747 | INFO | Perf report for the last 5.13908 seconds: | 20260507 06:18:16.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:18:16.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:16.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:16.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:16.747 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260507 06:18:16.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:16.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:16.748 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:18:16.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:16.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:16.748 | INFO | Uploaded 36GiB to GPU in 2.03591s (17.6826GiB/s) | 20260507 06:18:16.748 | INFO | Downloaded 34GiB to GPU in 1.9184s (17.7231GiB/s) | 20260507 06:18:21.879 | INFO | Analyzing chunk frames = [7751168,7751680) | 20260507 06:18:21.879 | INFO | MemInfo: { | 20260507 06:18:21.879 | INFO | "gpu" : " 17 GB", | 20260507 06:18:21.879 | INFO | "huge" : "", | 20260507 06:18:21.879 | INFO | "huge_shared" : " 27 GB", | 20260507 06:18:21.879 | INFO | "malloc" : " 14 GB", | 20260507 06:18:21.879 | INFO | "pinned" : " 29 GB" | 20260507 06:18:21.879 | INFO | } | 20260507 06:18:21.879 | INFO | Perf report for the last 5.13132 seconds: | 20260507 06:18:21.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:18:21.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:21.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:21.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:21.879 | INFO | ANALYSIS : 60.4%, 2.9, 93.1ms, 0 | 20260507 06:18:21.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:21.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:21.879 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:18:21.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:21.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:21.879 | INFO | Uploaded 36GiB to GPU in 2.00525s (17.9529GiB/s) | 20260507 06:18:21.879 | INFO | Downloaded 34GiB to GPU in 1.93358s (17.584GiB/s) | 20260507 06:18:27.022 | INFO | Analyzing chunk frames = [7751680,7752192) | 20260507 06:18:27.022 | INFO | MemInfo: { | 20260507 06:18:27.022 | INFO | "gpu" : " 17 GB", | 20260507 06:18:27.022 | INFO | "huge" : "", | 20260507 06:18:27.022 | INFO | "huge_shared" : " 27 GB", | 20260507 06:18:27.022 | INFO | "malloc" : " 14 GB", | 20260507 06:18:27.022 | INFO | "pinned" : " 29 GB" | 20260507 06:18:27.022 | INFO | } | 20260507 06:18:27.022 | INFO | Perf report for the last 5.14302 seconds: | 20260507 06:18:27.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:18:27.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:27.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:27.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:27.022 | INFO | ANALYSIS : 60.2%, 3.0, 94.0ms, 0 | 20260507 06:18:27.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:27.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:27.022 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:18:27.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:27.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:27.022 | INFO | Uploaded 36GiB to GPU in 2.05832s (17.49GiB/s) | 20260507 06:18:27.022 | INFO | Downloaded 34GiB to GPU in 1.94754s (17.4579GiB/s) | 20260507 06:18:32.163 | INFO | Analyzing chunk frames = [7752192,7752704) | 20260507 06:18:32.163 | INFO | MemInfo: { | 20260507 06:18:32.163 | INFO | "gpu" : " 17 GB", | 20260507 06:18:32.163 | INFO | "huge" : "", | 20260507 06:18:32.163 | INFO | "huge_shared" : " 27 GB", | 20260507 06:18:32.163 | INFO | "malloc" : " 14 GB", | 20260507 06:18:32.163 | INFO | "pinned" : " 29 GB" | 20260507 06:18:32.163 | INFO | } | 20260507 06:18:32.163 | INFO | Perf report for the last 5.14098 seconds: | 20260507 06:18:32.163 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:18:32.163 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:32.163 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:32.163 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:32.163 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 06:18:32.163 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:32.163 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:32.163 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 06:18:32.163 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:32.163 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:32.163 | INFO | Uploaded 36GiB to GPU in 2.03673s (17.6754GiB/s) | 20260507 06:18:32.163 | INFO | Downloaded 34GiB to GPU in 1.92085s (17.7005GiB/s) | 20260507 06:18:37.301 | INFO | Analyzing chunk frames = [7752704,7753216) | 20260507 06:18:37.301 | INFO | MemInfo: { | 20260507 06:18:37.301 | INFO | "gpu" : " 17 GB", | 20260507 06:18:37.301 | INFO | "huge" : "", | 20260507 06:18:37.301 | INFO | "huge_shared" : " 27 GB", | 20260507 06:18:37.301 | INFO | "malloc" : " 14 GB", | 20260507 06:18:37.301 | INFO | "pinned" : " 29 GB" | 20260507 06:18:37.301 | INFO | } | 20260507 06:18:37.301 | INFO | Perf report for the last 5.13801 seconds: | 20260507 06:18:37.301 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:18:37.301 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:37.301 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:37.301 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:37.301 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 06:18:37.301 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:37.301 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:37.301 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:18:37.301 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:37.301 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:37.301 | INFO | Uploaded 36GiB to GPU in 2.04291s (17.6219GiB/s) | 20260507 06:18:37.301 | INFO | Downloaded 34GiB to GPU in 1.99692s (17.0262GiB/s) | 20260507 06:18:42.435 | INFO | Analyzing chunk frames = [7753216,7753728) | 20260507 06:18:42.435 | INFO | MemInfo: { | 20260507 06:18:42.435 | INFO | "gpu" : " 17 GB", | 20260507 06:18:42.435 | INFO | "huge" : "", | 20260507 06:18:42.435 | INFO | "huge_shared" : " 27 GB", | 20260507 06:18:42.435 | INFO | "malloc" : " 14 GB", | 20260507 06:18:42.435 | INFO | "pinned" : " 29 GB" | 20260507 06:18:42.435 | INFO | } | 20260507 06:18:42.435 | INFO | Perf report for the last 5.13409 seconds: | 20260507 06:18:42.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:18:42.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:42.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:42.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:42.435 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 06:18:42.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:42.435 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:42.435 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:18:42.435 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:42.435 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:42.435 | INFO | Uploaded 36GiB to GPU in 1.98925s (18.0973GiB/s) | 20260507 06:18:42.435 | INFO | Downloaded 34GiB to GPU in 1.95612s (17.3814GiB/s) | 20260507 06:18:42.532 | INFO | Starting a round of Baz IO | 20260507 06:18:45.630 | INFO | Populating RTMetrics report | 20260507 06:18:45.631 | INFO | RTMetrics report populated | 20260507 06:18:45.631 | INFO | Opening tmp RTMetrics json file | 20260507 06:18:45.637 | INFO | Serializing RTMetrics to json | 20260507 06:18:45.648 | INFO | Serialization done, swapping tmp file now... | 20260507 06:18:45.648 | INFO | RTMetrics done | 20260507 06:18:45.713 | INFO | Finished a round of Baz IO: Wrote 1.671 GiB in 3.18 seconds | 20260507 06:18:45.713 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.64% / 0.00% / 67.36% | 20260507 06:18:45.713 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.124264% / 0% / 9.13938e-05% | 20260507 06:18:45.713 | INFO | Baz IO overall: | 20260507 06:18:45.713 | INFO | BytesWritten : 3913786190074 | 20260507 06:18:45.713 | INFO | NumEvents : 1875802661166 | 20260507 06:18:47.579 | INFO | Analyzing chunk frames = [7753728,7754240) | 20260507 06:18:47.579 | INFO | MemInfo: { | 20260507 06:18:47.579 | INFO | "gpu" : " 17 GB", | 20260507 06:18:47.579 | INFO | "huge" : "", | 20260507 06:18:47.579 | INFO | "huge_shared" : " 27 GB", | 20260507 06:18:47.579 | INFO | "malloc" : " 14 GB", | 20260507 06:18:47.579 | INFO | "pinned" : " 29 GB" | 20260507 06:18:47.579 | INFO | } | 20260507 06:18:47.579 | INFO | Perf report for the last 5.14392 seconds: | 20260507 06:18:47.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:18:47.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:47.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:47.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:47.579 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 06:18:47.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:47.579 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 06:18:47.579 | INFO | PRE_HQ : 58.8%, 1.8, 54.9ms, 0 | 20260507 06:18:47.579 | INFO | BAZWRITER : 61.4%, 3.2, 100.2ms, 0 | 20260507 06:18:47.579 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 06:18:47.579 | INFO | Uploaded 36GiB to GPU in 2.04052s (17.6426GiB/s) | 20260507 06:18:47.579 | INFO | Downloaded 38GiB to GPU in 2.18005s (17.4308GiB/s) | 20260507 06:18:52.721 | INFO | Analyzing chunk frames = [7754240,7754752) | 20260507 06:18:52.721 | INFO | MemInfo: { | 20260507 06:18:52.721 | INFO | "gpu" : " 17 GB", | 20260507 06:18:52.721 | INFO | "huge" : "", | 20260507 06:18:52.721 | INFO | "huge_shared" : " 27 GB", | 20260507 06:18:52.721 | INFO | "malloc" : " 14 GB", | 20260507 06:18:52.721 | INFO | "pinned" : " 29 GB" | 20260507 06:18:52.721 | INFO | } | 20260507 06:18:52.721 | INFO | Perf report for the last 5.1422 seconds: | 20260507 06:18:52.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:18:52.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:52.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:52.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:52.721 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 06:18:52.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:52.721 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:52.721 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:18:52.721 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:52.721 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:52.721 | INFO | Uploaded 36GiB to GPU in 2.00424s (17.9619GiB/s) | 20260507 06:18:52.721 | INFO | Downloaded 34GiB to GPU in 1.93781s (17.5456GiB/s) | 20260507 06:18:57.863 | INFO | Analyzing chunk frames = [7754752,7755264) | 20260507 06:18:57.863 | INFO | MemInfo: { | 20260507 06:18:57.863 | INFO | "gpu" : " 17 GB", | 20260507 06:18:57.863 | INFO | "huge" : "", | 20260507 06:18:57.863 | INFO | "huge_shared" : " 27 GB", | 20260507 06:18:57.863 | INFO | "malloc" : " 14 GB", | 20260507 06:18:57.863 | INFO | "pinned" : " 29 GB" | 20260507 06:18:57.863 | INFO | } | 20260507 06:18:57.863 | INFO | Perf report for the last 5.14205 seconds: | 20260507 06:18:57.863 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:18:57.863 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:57.863 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:57.863 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:57.863 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:18:57.863 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:57.863 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:57.863 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:18:57.863 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:57.863 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:18:57.863 | INFO | Uploaded 36GiB to GPU in 2.04896s (17.5699GiB/s) | 20260507 06:18:57.863 | INFO | Downloaded 34GiB to GPU in 1.9423s (17.505GiB/s) | 20260507 06:19:03.002 | INFO | Analyzing chunk frames = [7755264,7755776) | 20260507 06:19:03.002 | INFO | MemInfo: { | 20260507 06:19:03.002 | INFO | "gpu" : " 17 GB", | 20260507 06:19:03.002 | INFO | "huge" : "", | 20260507 06:19:03.002 | INFO | "huge_shared" : " 27 GB", | 20260507 06:19:03.002 | INFO | "malloc" : " 14 GB", | 20260507 06:19:03.002 | INFO | "pinned" : " 29 GB" | 20260507 06:19:03.002 | INFO | } | 20260507 06:19:03.002 | INFO | Perf report for the last 5.13898 seconds: | 20260507 06:19:03.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:19:03.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:03.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:03.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:03.002 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 06:19:03.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:03.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:03.002 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:19:03.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:03.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:03.002 | INFO | Uploaded 36GiB to GPU in 2.05948s (17.4801GiB/s) | 20260507 06:19:03.002 | INFO | Downloaded 34GiB to GPU in 1.95101s (17.4268GiB/s) | 20260507 06:19:08.137 | INFO | Analyzing chunk frames = [7755776,7756288) | 20260507 06:19:08.137 | INFO | MemInfo: { | 20260507 06:19:08.137 | INFO | "gpu" : " 17 GB", | 20260507 06:19:08.137 | INFO | "huge" : "", | 20260507 06:19:08.137 | INFO | "huge_shared" : " 27 GB", | 20260507 06:19:08.137 | INFO | "malloc" : " 14 GB", | 20260507 06:19:08.137 | INFO | "pinned" : " 29 GB" | 20260507 06:19:08.137 | INFO | } | 20260507 06:19:08.137 | INFO | Perf report for the last 5.13503 seconds: | 20260507 06:19:08.137 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:19:08.137 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:08.137 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:08.137 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:08.137 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 06:19:08.137 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:08.137 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:08.137 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 06:19:08.137 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:08.137 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:08.138 | INFO | Uploaded 36GiB to GPU in 2.05173s (17.5461GiB/s) | 20260507 06:19:08.138 | INFO | Downloaded 34GiB to GPU in 1.91341s (17.7693GiB/s) | 20260507 06:19:13.276 | INFO | Analyzing chunk frames = [7756288,7756800) | 20260507 06:19:13.276 | INFO | MemInfo: { | 20260507 06:19:13.276 | INFO | "gpu" : " 17 GB", | 20260507 06:19:13.276 | INFO | "huge" : "", | 20260507 06:19:13.276 | INFO | "huge_shared" : " 27 GB", | 20260507 06:19:13.276 | INFO | "malloc" : " 14 GB", | 20260507 06:19:13.276 | INFO | "pinned" : " 29 GB" | 20260507 06:19:13.276 | INFO | } | 20260507 06:19:13.276 | INFO | Perf report for the last 5.13917 seconds: | 20260507 06:19:13.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:19:13.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:13.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:13.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:13.276 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 06:19:13.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:13.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:13.276 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 06:19:13.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:13.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:13.277 | INFO | Uploaded 36GiB to GPU in 2.07005s (17.3909GiB/s) | 20260507 06:19:13.277 | INFO | Downloaded 34GiB to GPU in 1.92576s (17.6553GiB/s) | 20260507 06:19:18.414 | INFO | Analyzing chunk frames = [7756800,7757312) | 20260507 06:19:18.414 | INFO | MemInfo: { | 20260507 06:19:18.414 | INFO | "gpu" : " 17 GB", | 20260507 06:19:18.414 | INFO | "huge" : "", | 20260507 06:19:18.414 | INFO | "huge_shared" : " 27 GB", | 20260507 06:19:18.414 | INFO | "malloc" : " 14 GB", | 20260507 06:19:18.414 | INFO | "pinned" : " 29 GB" | 20260507 06:19:18.414 | INFO | } | 20260507 06:19:18.414 | INFO | Perf report for the last 5.13797 seconds: | 20260507 06:19:18.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:19:18.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:18.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:18.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:18.414 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:19:18.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:18.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:18.414 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 06:19:18.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:18.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:18.415 | INFO | Uploaded 36GiB to GPU in 1.98663s (18.1211GiB/s) | 20260507 06:19:18.415 | INFO | Downloaded 34GiB to GPU in 1.98213s (17.1532GiB/s) | 20260507 06:19:23.564 | INFO | Analyzing chunk frames = [7757312,7757824) | 20260507 06:19:23.564 | INFO | MemInfo: { | 20260507 06:19:23.564 | INFO | "gpu" : " 17 GB", | 20260507 06:19:23.564 | INFO | "huge" : "", | 20260507 06:19:23.564 | INFO | "huge_shared" : " 27 GB", | 20260507 06:19:23.564 | INFO | "malloc" : " 14 GB", | 20260507 06:19:23.564 | INFO | "pinned" : " 29 GB" | 20260507 06:19:23.564 | INFO | } | 20260507 06:19:23.564 | INFO | Perf report for the last 5.14944 seconds: | 20260507 06:19:23.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:19:23.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:23.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:23.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:23.564 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:19:23.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:23.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:23.564 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 06:19:23.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:23.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:23.564 | INFO | Uploaded 36GiB to GPU in 1.98657s (18.1217GiB/s) | 20260507 06:19:23.564 | INFO | Downloaded 34GiB to GPU in 1.95647s (17.3782GiB/s) | 20260507 06:19:23.564 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:19:23.564 | INFO | Total mem usage: 18 GiB | 20260507 06:19:23.564 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:19:23.564 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:19:23.564 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:19:23.564 | INFO | High Water Breakdown: | 20260507 06:19:23.564 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:19:23.564 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:19:23.564 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:19:23.564 | INFO | ------------------------------------------ | 20260507 06:19:23.564 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:19:23.564 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:19:23.564 | INFO | Total mem usage: 26.5 GiB | 20260507 06:19:23.564 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:19:23.564 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:19:23.564 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:19:23.564 | INFO | High Water Breakdown: | 20260507 06:19:23.564 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:19:23.564 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:19:23.564 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:19:23.564 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:19:23.564 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:19:23.564 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:19:23.564 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:19:23.564 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:19:23.564 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:19:23.564 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:19:23.564 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:19:23.564 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:19:23.564 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:19:23.564 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:19:23.564 | INFO | ------------------------------------------ | 20260507 06:19:23.564 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:19:23.564 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:19:23.564 | INFO | Total mem usage: 25.6 GiB | 20260507 06:19:23.564 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:19:23.565 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:19:23.565 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:19:23.565 | INFO | High Water Breakdown: | 20260507 06:19:23.565 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:19:23.565 | INFO | ------------------------------------------ | 20260507 06:19:23.565 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:19:23.565 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:19:23.565 | INFO | Total mem usage: 15.7 GiB | 20260507 06:19:23.565 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:19:23.565 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:19:23.565 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:19:23.565 | INFO | High Water Breakdown: | 20260507 06:19:23.565 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:19:23.565 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:19:23.565 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:19:23.565 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:19:23.565 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:19:23.565 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:19:23.565 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:19:23.565 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:19:23.565 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:19:23.565 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:19:23.565 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:19:23.565 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:19:23.565 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:19:23.565 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:19:23.565 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:19:23.565 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:19:23.565 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:19:23.565 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:19:23.565 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:19:23.565 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:19:23.565 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:19:23.565 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:19:23.565 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:19:23.565 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:19:23.565 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:19:23.565 | INFO | ------------------------------------------ | 20260507 06:19:23.565 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:19:23.565 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:19:23.565 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:19:23.565 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:19:23.565 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.35e+01 / 1.10e+00 | 20260507 06:19:23.565 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.01e-01, 9.30e-03 / 5.45e-03 | 20260507 06:19:23.565 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.43e+00, 4.98e-01 / 2.82e-01 | 20260507 06:19:23.565 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.49e+00, 1.66e-01 / 1.04e-01 | 20260507 06:19:23.565 | INFO | PerfReport: 16.25%, DME -- 3.99e+01 / 4.45e+01, 3.81e+00 / 1.18e+00 | 20260507 06:19:23.565 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.30e+00 / 1.31e+00 | 20260507 06:19:23.565 | INFO | PerfReport: 6.91%, PulseAccumulating -- 2.12e+00 / 2.71e+00, 3.82e-01 / 1.87e-01 | 20260507 06:19:23.565 | INFO | PerfReport: 2.95%, Metrics -- 9.06e-01 / 1.24e+00, 2.41e-01 / 1.29e-01 | 20260507 06:19:23.565 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.05e-01 / 3.62e-01, 5.01e-02 / 1.84e-02 | 20260507 06:19:23.565 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:19:23.565 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:19:23.565 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:19:23.565 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 06:19:23.565 | INFO | PerfReport: 21.95%, Upload -- 2.11e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 06:19:23.565 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 06:19:23.565 | INFO | PerfReport: 31.89%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 06:19:23.565 | INFO | PerfReport: 1.66%, LockUpload -- 1.60e+00 / 0.00e+00, 5.00e+00 / 0.00e+00 | 20260507 06:19:23.565 | INFO | PerfReport: 1.18%, LockDownload -- 1.13e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260507 06:19:23.565 | INFO | PerfReport: 21.96%, LockCompute -- 2.11e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260507 06:19:23.565 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260507 06:19:23.661 | INFO | Starting a round of Baz IO | 20260507 06:19:26.749 | INFO | Populating RTMetrics report | 20260507 06:19:26.749 | INFO | RTMetrics report populated | 20260507 06:19:26.749 | INFO | Opening tmp RTMetrics json file | 20260507 06:19:26.767 | INFO | Serializing RTMetrics to json | 20260507 06:19:26.768 | INFO | Serialization done, swapping tmp file now... | 20260507 06:19:26.768 | INFO | RTMetrics done | 20260507 06:19:26.841 | INFO | Finished a round of Baz IO: Wrote 1.671 GiB in 3.179 seconds | 20260507 06:19:26.841 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.64% / 0.00% / 67.36% | 20260507 06:19:26.841 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.124741% / 3.97364e-06% / 5.96046e-05% | 20260507 06:19:26.841 | INFO | Baz IO overall: | 20260507 06:19:26.841 | INFO | BytesWritten : 3915580794784 | 20260507 06:19:26.841 | INFO | NumEvents : 1876642815326 | 20260507 06:19:28.691 | INFO | Analyzing chunk frames = [7757824,7758336) | 20260507 06:19:28.691 | INFO | MemInfo: { | 20260507 06:19:28.691 | INFO | "gpu" : " 17 GB", | 20260507 06:19:28.691 | INFO | "huge" : "", | 20260507 06:19:28.692 | INFO | "huge_shared" : " 27 GB", | 20260507 06:19:28.692 | INFO | "malloc" : " 14 GB", | 20260507 06:19:28.692 | INFO | "pinned" : " 29 GB" | 20260507 06:19:28.692 | INFO | } | 20260507 06:19:28.692 | INFO | Perf report for the last 5.12779 seconds: | 20260507 06:19:28.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:19:28.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:28.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:28.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:28.692 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260507 06:19:28.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:28.692 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 06:19:28.692 | INFO | PRE_HQ : 58.9%, 1.8, 54.8ms, 0 | 20260507 06:19:28.692 | INFO | BAZWRITER : 61.4%, 3.2, 99.7ms, 0 | 20260507 06:19:28.692 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:19:28.692 | INFO | Uploaded 36GiB to GPU in 2.06923s (17.3978GiB/s) | 20260507 06:19:28.692 | INFO | Downloaded 38GiB to GPU in 2.17491s (17.472GiB/s) | 20260507 06:19:33.834 | INFO | Analyzing chunk frames = [7758336,7758848) | 20260507 06:19:33.834 | INFO | MemInfo: { | 20260507 06:19:33.834 | INFO | "gpu" : " 17 GB", | 20260507 06:19:33.834 | INFO | "huge" : "", | 20260507 06:19:33.834 | INFO | "huge_shared" : " 27 GB", | 20260507 06:19:33.834 | INFO | "malloc" : " 14 GB", | 20260507 06:19:33.834 | INFO | "pinned" : " 29 GB" | 20260507 06:19:33.834 | INFO | } | 20260507 06:19:33.834 | INFO | Perf report for the last 5.14233 seconds: | 20260507 06:19:33.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:19:33.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:33.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:33.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:33.834 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 06:19:33.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:33.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:33.834 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:19:33.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:33.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:33.834 | INFO | Uploaded 36GiB to GPU in 1.98526s (18.1336GiB/s) | 20260507 06:19:33.834 | INFO | Downloaded 34GiB to GPU in 1.9293s (17.623GiB/s) | 20260507 06:19:38.975 | INFO | Analyzing chunk frames = [7758848,7759360) | 20260507 06:19:38.975 | INFO | MemInfo: { | 20260507 06:19:38.975 | INFO | "gpu" : " 17 GB", | 20260507 06:19:38.975 | INFO | "huge" : "", | 20260507 06:19:38.975 | INFO | "huge_shared" : " 27 GB", | 20260507 06:19:38.976 | INFO | "malloc" : " 14 GB", | 20260507 06:19:38.976 | INFO | "pinned" : " 29 GB" | 20260507 06:19:38.976 | INFO | } | 20260507 06:19:38.976 | INFO | Perf report for the last 5.14165 seconds: | 20260507 06:19:38.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:19:38.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:38.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:38.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:38.976 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 06:19:38.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:38.976 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:38.976 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:19:38.976 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:38.976 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:38.976 | INFO | Uploaded 36GiB to GPU in 2.01466s (17.869GiB/s) | 20260507 06:19:38.976 | INFO | Downloaded 34GiB to GPU in 1.94104s (17.5164GiB/s) | 20260507 06:19:44.115 | INFO | Analyzing chunk frames = [7759360,7759872) | 20260507 06:19:44.115 | INFO | MemInfo: { | 20260507 06:19:44.115 | INFO | "gpu" : " 17 GB", | 20260507 06:19:44.115 | INFO | "huge" : "", | 20260507 06:19:44.115 | INFO | "huge_shared" : " 27 GB", | 20260507 06:19:44.115 | INFO | "malloc" : " 14 GB", | 20260507 06:19:44.115 | INFO | "pinned" : " 29 GB" | 20260507 06:19:44.115 | INFO | } | 20260507 06:19:44.115 | INFO | Perf report for the last 5.13917 seconds: | 20260507 06:19:44.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:19:44.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:44.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:44.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:44.115 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 06:19:44.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:44.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:44.115 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:19:44.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:44.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:44.115 | INFO | Uploaded 36GiB to GPU in 2.03335s (17.7048GiB/s) | 20260507 06:19:44.115 | INFO | Downloaded 34GiB to GPU in 1.94979s (17.4377GiB/s) | 20260507 06:19:49.251 | INFO | Analyzing chunk frames = [7759872,7760384) | 20260507 06:19:49.251 | INFO | MemInfo: { | 20260507 06:19:49.251 | INFO | "gpu" : " 17 GB", | 20260507 06:19:49.251 | INFO | "huge" : "", | 20260507 06:19:49.252 | INFO | "huge_shared" : " 27 GB", | 20260507 06:19:49.252 | INFO | "malloc" : " 14 GB", | 20260507 06:19:49.252 | INFO | "pinned" : " 29 GB" | 20260507 06:19:49.252 | INFO | } | 20260507 06:19:49.252 | INFO | Perf report for the last 5.13683 seconds: | 20260507 06:19:49.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:19:49.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:49.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:49.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:49.252 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 06:19:49.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:49.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:49.252 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 06:19:49.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:49.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:49.252 | INFO | Uploaded 36GiB to GPU in 2.0536s (17.5302GiB/s) | 20260507 06:19:49.252 | INFO | Downloaded 34GiB to GPU in 1.90453s (17.8521GiB/s) | 20260507 06:19:54.385 | INFO | Analyzing chunk frames = [7760384,7760896) | 20260507 06:19:54.385 | INFO | MemInfo: { | 20260507 06:19:54.385 | INFO | "gpu" : " 17 GB", | 20260507 06:19:54.385 | INFO | "huge" : "", | 20260507 06:19:54.385 | INFO | "huge_shared" : " 27 GB", | 20260507 06:19:54.385 | INFO | "malloc" : " 14 GB", | 20260507 06:19:54.385 | INFO | "pinned" : " 29 GB" | 20260507 06:19:54.385 | INFO | } | 20260507 06:19:54.385 | INFO | Perf report for the last 5.13356 seconds: | 20260507 06:19:54.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:19:54.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:54.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:54.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:54.385 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 06:19:54.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:54.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:54.385 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:19:54.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:54.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:54.385 | INFO | Uploaded 36GiB to GPU in 2.04814s (17.5769GiB/s) | 20260507 06:19:54.386 | INFO | Downloaded 34GiB to GPU in 1.91259s (17.7769GiB/s) | 20260507 06:19:59.531 | INFO | Analyzing chunk frames = [7760896,7761408) | 20260507 06:19:59.531 | INFO | MemInfo: { | 20260507 06:19:59.531 | INFO | "gpu" : " 17 GB", | 20260507 06:19:59.531 | INFO | "huge" : "", | 20260507 06:19:59.531 | INFO | "huge_shared" : " 27 GB", | 20260507 06:19:59.531 | INFO | "malloc" : " 14 GB", | 20260507 06:19:59.531 | INFO | "pinned" : " 29 GB" | 20260507 06:19:59.531 | INFO | } | 20260507 06:19:59.531 | INFO | Perf report for the last 5.14569 seconds: | 20260507 06:19:59.531 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:19:59.531 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:59.531 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:59.531 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:59.531 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 06:19:59.531 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:59.531 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:59.531 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:19:59.531 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:59.531 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:19:59.531 | INFO | Uploaded 36GiB to GPU in 2.00282s (17.9746GiB/s) | 20260507 06:19:59.531 | INFO | Downloaded 34GiB to GPU in 1.99288s (17.0607GiB/s) | 20260507 06:20:04.670 | INFO | Analyzing chunk frames = [7761408,7761920) | 20260507 06:20:04.670 | INFO | MemInfo: { | 20260507 06:20:04.670 | INFO | "gpu" : " 17 GB", | 20260507 06:20:04.670 | INFO | "huge" : "", | 20260507 06:20:04.670 | INFO | "huge_shared" : " 27 GB", | 20260507 06:20:04.670 | INFO | "malloc" : " 14 GB", | 20260507 06:20:04.670 | INFO | "pinned" : " 29 GB" | 20260507 06:20:04.670 | INFO | } | 20260507 06:20:04.670 | INFO | Perf report for the last 5.13928 seconds: | 20260507 06:20:04.670 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:20:04.670 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:04.670 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:04.670 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:04.670 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 06:20:04.670 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:04.670 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:04.670 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 06:20:04.670 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:04.670 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:04.670 | INFO | Uploaded 36GiB to GPU in 1.99194s (18.0729GiB/s) | 20260507 06:20:04.670 | INFO | Downloaded 34GiB to GPU in 1.97851s (17.1847GiB/s) | 20260507 06:20:04.770 | INFO | Starting a round of Baz IO | 20260507 06:20:07.861 | INFO | Populating RTMetrics report | 20260507 06:20:07.862 | INFO | RTMetrics report populated | 20260507 06:20:07.862 | INFO | Opening tmp RTMetrics json file | 20260507 06:20:07.868 | INFO | Serializing RTMetrics to json | 20260507 06:20:07.880 | INFO | Serialization done, swapping tmp file now... | 20260507 06:20:07.880 | INFO | RTMetrics done | 20260507 06:20:07.952 | INFO | Finished a round of Baz IO: Wrote 1.672 GiB in 3.181 seconds | 20260507 06:20:07.952 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.64% / 0.00% / 67.36% | 20260507 06:20:07.952 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.12509% / 0% / 6.35783e-05% | 20260507 06:20:07.952 | INFO | Baz IO overall: | 20260507 06:20:07.952 | INFO | BytesWritten : 3917376156386 | 20260507 06:20:07.952 | INFO | NumEvents : 1877483677418 | 20260507 06:20:09.811 | INFO | Analyzing chunk frames = [7761920,7762432) | 20260507 06:20:09.811 | INFO | MemInfo: { | 20260507 06:20:09.811 | INFO | "gpu" : " 17 GB", | 20260507 06:20:09.811 | INFO | "huge" : "", | 20260507 06:20:09.811 | INFO | "huge_shared" : " 27 GB", | 20260507 06:20:09.811 | INFO | "malloc" : " 14 GB", | 20260507 06:20:09.811 | INFO | "pinned" : " 29 GB" | 20260507 06:20:09.811 | INFO | } | 20260507 06:20:09.811 | INFO | Perf report for the last 5.14102 seconds: | 20260507 06:20:09.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:20:09.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:09.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:09.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:09.811 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 06:20:09.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:09.811 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 06:20:09.811 | INFO | PRE_HQ : 59.3%, 1.9, 57.3ms, 0 | 20260507 06:20:09.811 | INFO | BAZWRITER : 61.5%, 3.2, 102.2ms, 0 | 20260507 06:20:09.811 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:20:09.812 | INFO | Uploaded 36GiB to GPU in 2.05595s (17.5102GiB/s) | 20260507 06:20:09.812 | INFO | Downloaded 38GiB to GPU in 2.18128s (17.421GiB/s) | 20260507 06:20:14.943 | INFO | Analyzing chunk frames = [7762432,7762944) | 20260507 06:20:14.943 | INFO | MemInfo: { | 20260507 06:20:14.943 | INFO | "gpu" : " 17 GB", | 20260507 06:20:14.943 | INFO | "huge" : "", | 20260507 06:20:14.943 | INFO | "huge_shared" : " 27 GB", | 20260507 06:20:14.943 | INFO | "malloc" : " 14 GB", | 20260507 06:20:14.943 | INFO | "pinned" : " 29 GB" | 20260507 06:20:14.943 | INFO | } | 20260507 06:20:14.943 | INFO | Perf report for the last 5.13188 seconds: | 20260507 06:20:14.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:20:14.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:14.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:14.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:14.943 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:20:14.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:14.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:14.943 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:20:14.943 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:14.943 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:14.943 | INFO | Uploaded 36GiB to GPU in 1.97905s (18.1906GiB/s) | 20260507 06:20:14.943 | INFO | Downloaded 34GiB to GPU in 1.94105s (17.5163GiB/s) | 20260507 06:20:20.086 | INFO | Analyzing chunk frames = [7762944,7763456) | 20260507 06:20:20.086 | INFO | MemInfo: { | 20260507 06:20:20.086 | INFO | "gpu" : " 17 GB", | 20260507 06:20:20.086 | INFO | "huge" : "", | 20260507 06:20:20.086 | INFO | "huge_shared" : " 27 GB", | 20260507 06:20:20.086 | INFO | "malloc" : " 14 GB", | 20260507 06:20:20.086 | INFO | "pinned" : " 29 GB" | 20260507 06:20:20.086 | INFO | } | 20260507 06:20:20.086 | INFO | Perf report for the last 5.1432 seconds: | 20260507 06:20:20.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:20:20.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:20.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:20.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:20.086 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:20:20.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:20.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:20.086 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:20:20.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:20.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:20.087 | INFO | Uploaded 36GiB to GPU in 2.02834s (17.7485GiB/s) | 20260507 06:20:20.087 | INFO | Downloaded 34GiB to GPU in 1.95637s (17.3791GiB/s) | 20260507 06:20:25.228 | INFO | Analyzing chunk frames = [7763456,7763968) | 20260507 06:20:25.228 | INFO | MemInfo: { | 20260507 06:20:25.228 | INFO | "gpu" : " 17 GB", | 20260507 06:20:25.228 | INFO | "huge" : "", | 20260507 06:20:25.228 | INFO | "huge_shared" : " 27 GB", | 20260507 06:20:25.228 | INFO | "malloc" : " 14 GB", | 20260507 06:20:25.228 | INFO | "pinned" : " 29 GB" | 20260507 06:20:25.228 | INFO | } | 20260507 06:20:25.228 | INFO | Perf report for the last 5.14208 seconds: | 20260507 06:20:25.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:20:25.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:25.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:25.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:25.228 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 06:20:25.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:25.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:25.228 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:20:25.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:25.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:25.229 | INFO | Uploaded 36GiB to GPU in 1.99165s (18.0754GiB/s) | 20260507 06:20:25.229 | INFO | Downloaded 34GiB to GPU in 1.93635s (17.5588GiB/s) | 20260507 06:20:30.370 | INFO | Analyzing chunk frames = [7763968,7764480) | 20260507 06:20:30.370 | INFO | MemInfo: { | 20260507 06:20:30.370 | INFO | "gpu" : " 17 GB", | 20260507 06:20:30.370 | INFO | "huge" : "", | 20260507 06:20:30.370 | INFO | "huge_shared" : " 27 GB", | 20260507 06:20:30.370 | INFO | "malloc" : " 14 GB", | 20260507 06:20:30.370 | INFO | "pinned" : " 29 GB" | 20260507 06:20:30.370 | INFO | } | 20260507 06:20:30.370 | INFO | Perf report for the last 5.14143 seconds: | 20260507 06:20:30.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:20:30.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:30.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:30.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:30.370 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 06:20:30.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:30.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:30.370 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:20:30.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:30.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:30.370 | INFO | Uploaded 36GiB to GPU in 2.05166s (17.5468GiB/s) | 20260507 06:20:30.370 | INFO | Downloaded 34GiB to GPU in 1.95023s (17.4338GiB/s) | 20260507 06:20:35.513 | INFO | Analyzing chunk frames = [7764480,7764992) | 20260507 06:20:35.513 | INFO | MemInfo: { | 20260507 06:20:35.513 | INFO | "gpu" : " 17 GB", | 20260507 06:20:35.513 | INFO | "huge" : "", | 20260507 06:20:35.513 | INFO | "huge_shared" : " 27 GB", | 20260507 06:20:35.513 | INFO | "malloc" : " 14 GB", | 20260507 06:20:35.513 | INFO | "pinned" : " 29 GB" | 20260507 06:20:35.513 | INFO | } | 20260507 06:20:35.513 | INFO | Perf report for the last 5.14345 seconds: | 20260507 06:20:35.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:20:35.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:35.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:35.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:35.513 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:20:35.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:35.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:35.513 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:20:35.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:35.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:35.514 | INFO | Uploaded 36GiB to GPU in 2.06326s (17.4481GiB/s) | 20260507 06:20:35.514 | INFO | Downloaded 34GiB to GPU in 1.92667s (17.6471GiB/s) | 20260507 06:20:40.641 | INFO | Analyzing chunk frames = [7764992,7765504) | 20260507 06:20:40.641 | INFO | MemInfo: { | 20260507 06:20:40.641 | INFO | "gpu" : " 17 GB", | 20260507 06:20:40.641 | INFO | "huge" : "", | 20260507 06:20:40.641 | INFO | "huge_shared" : " 27 GB", | 20260507 06:20:40.641 | INFO | "malloc" : " 14 GB", | 20260507 06:20:40.641 | INFO | "pinned" : " 29 GB" | 20260507 06:20:40.641 | INFO | } | 20260507 06:20:40.641 | INFO | Perf report for the last 5.12759 seconds: | 20260507 06:20:40.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:20:40.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:40.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:40.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:40.641 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260507 06:20:40.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:40.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:40.641 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 06:20:40.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:40.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:40.641 | INFO | Uploaded 36GiB to GPU in 1.88411s (19.1072GiB/s) | 20260507 06:20:40.641 | INFO | Downloaded 34GiB to GPU in 1.96908s (17.267GiB/s) | 20260507 06:20:45.784 | INFO | Analyzing chunk frames = [7765504,7766016) | 20260507 06:20:45.784 | INFO | MemInfo: { | 20260507 06:20:45.784 | INFO | "gpu" : " 17 GB", | 20260507 06:20:45.784 | INFO | "huge" : "", | 20260507 06:20:45.784 | INFO | "huge_shared" : " 27 GB", | 20260507 06:20:45.784 | INFO | "malloc" : " 14 GB", | 20260507 06:20:45.784 | INFO | "pinned" : " 29 GB" | 20260507 06:20:45.784 | INFO | } | 20260507 06:20:45.784 | INFO | Perf report for the last 5.14294 seconds: | 20260507 06:20:45.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:20:45.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:45.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:45.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:45.784 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 06:20:45.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:45.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:45.784 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:20:45.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:45.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:45.784 | INFO | Uploaded 36GiB to GPU in 1.96004s (18.367GiB/s) | 20260507 06:20:45.784 | INFO | Downloaded 34GiB to GPU in 1.95117s (17.4255GiB/s) | 20260507 06:20:45.784 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:20:45.784 | INFO | Total mem usage: 18 GiB | 20260507 06:20:45.784 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:20:45.784 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.6%) | 20260507 06:20:45.784 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:20:45.784 | INFO | High Water Breakdown: | 20260507 06:20:45.784 | INFO | 60.2% : 8.63 GiB ( 75.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:20:45.784 | INFO | 39.6% : 5.68 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:20:45.784 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:20:45.784 | INFO | ------------------------------------------ | 20260507 06:20:45.784 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:20:45.784 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:20:45.784 | INFO | Total mem usage: 26.5 GiB | 20260507 06:20:45.784 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:20:45.784 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:20:45.784 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:20:45.784 | INFO | High Water Breakdown: | 20260507 06:20:45.784 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:20:45.784 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:20:45.784 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:20:45.784 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:20:45.785 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:20:45.785 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:20:45.785 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:20:45.785 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:20:45.785 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:20:45.785 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:20:45.785 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:20:45.785 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:20:45.785 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:20:45.785 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:20:45.785 | INFO | ------------------------------------------ | 20260507 06:20:45.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:20:45.785 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:20:45.785 | INFO | Total mem usage: 25.6 GiB | 20260507 06:20:45.785 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:20:45.785 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:20:45.785 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:20:45.785 | INFO | High Water Breakdown: | 20260507 06:20:45.785 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:20:45.785 | INFO | ------------------------------------------ | 20260507 06:20:45.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:20:45.785 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:20:45.785 | INFO | Total mem usage: 15.7 GiB | 20260507 06:20:45.785 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:20:45.785 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:20:45.785 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:20:45.785 | INFO | High Water Breakdown: | 20260507 06:20:45.785 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:20:45.785 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:20:45.785 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:20:45.785 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:20:45.785 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:20:45.785 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:20:45.785 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:20:45.785 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:20:45.785 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:20:45.785 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:20:45.785 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:20:45.785 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:20:45.785 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:20:45.785 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:20:45.785 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:20:45.785 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:20:45.785 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:20:45.785 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:20:45.785 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:20:45.785 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:20:45.785 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:20:45.785 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:20:45.785 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:20:45.785 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:20:45.785 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:20:45.785 | INFO | ------------------------------------------ | 20260507 06:20:45.785 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:20:45.785 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:20:45.785 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:20:45.785 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:20:45.785 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.33e+01 / 1.77e+00 | 20260507 06:20:45.785 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 9.64e-03 / 4.72e-03 | 20260507 06:20:45.785 | INFO | PerfReport: 11.68%, Baseline -- 3.58e+00 / 4.42e+00, 5.10e-01 / 2.76e-01 | 20260507 06:20:45.785 | INFO | PerfReport: 3.22%, Histogram -- 9.90e-01 / 1.50e+00, 1.68e-01 / 9.31e-02 | 20260507 06:20:45.785 | INFO | PerfReport: 16.27%, DME -- 4.00e+01 / 4.49e+01, 4.05e+00 / 1.99e+00 | 20260507 06:20:45.785 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.24e+00 / 1.09e+00 | 20260507 06:20:45.785 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.68e+00, 3.89e-01 / 2.06e-01 | 20260507 06:20:45.785 | INFO | PerfReport: 2.97%, Metrics -- 9.11e-01 / 1.23e+00, 2.46e-01 / 1.32e-01 | 20260507 06:20:45.785 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.06e-01 / 3.60e-01, 5.14e-02 / 2.23e-02 | 20260507 06:20:45.785 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:20:45.785 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:20:45.785 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:20:45.785 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 06:20:45.785 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 06:20:45.785 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 06:20:45.785 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 06:20:45.785 | INFO | PerfReport: 1.50%, LockUpload -- 1.44e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 06:20:45.785 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260507 06:20:45.785 | INFO | PerfReport: 22.01%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 06:20:45.785 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.04e-02 / 0.00e+00 | 20260507 06:20:45.881 | INFO | Starting a round of Baz IO | 20260507 06:20:48.979 | INFO | Populating RTMetrics report | 20260507 06:20:48.979 | INFO | RTMetrics report populated | 20260507 06:20:48.979 | INFO | Opening tmp RTMetrics json file | 20260507 06:20:48.997 | INFO | Serializing RTMetrics to json | 20260507 06:20:48.998 | INFO | Serialization done, swapping tmp file now... | 20260507 06:20:48.998 | INFO | RTMetrics done | 20260507 06:20:49.072 | INFO | Finished a round of Baz IO: Wrote 1.672 GiB in 3.19 seconds | 20260507 06:20:49.072 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.63% / 0.00% / 67.36% | 20260507 06:20:49.072 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.125106% / 3.97364e-06% / 7.94729e-05% | 20260507 06:20:49.072 | INFO | Baz IO overall: | 20260507 06:20:49.072 | INFO | BytesWritten : 3919171156435 | 20260507 06:20:49.072 | INFO | NumEvents : 1878324271797 | 20260507 06:20:50.923 | INFO | Analyzing chunk frames = [7766016,7766528) | 20260507 06:20:50.923 | INFO | MemInfo: { | 20260507 06:20:50.923 | INFO | "gpu" : " 17 GB", | 20260507 06:20:50.923 | INFO | "huge" : "", | 20260507 06:20:50.923 | INFO | "huge_shared" : " 27 GB", | 20260507 06:20:50.923 | INFO | "malloc" : " 14 GB", | 20260507 06:20:50.923 | INFO | "pinned" : " 29 GB" | 20260507 06:20:50.923 | INFO | } | 20260507 06:20:50.923 | INFO | Perf report for the last 5.13904 seconds: | 20260507 06:20:50.923 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:20:50.923 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:50.923 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:50.923 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:50.923 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 06:20:50.923 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:50.923 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 06:20:50.923 | INFO | PRE_HQ : 59.0%, 1.8, 55.0ms, 0 | 20260507 06:20:50.923 | INFO | BAZWRITER : 61.7%, 3.1, 99.7ms, 0 | 20260507 06:20:50.923 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:20:50.923 | INFO | Uploaded 36GiB to GPU in 2.04624s (17.5933GiB/s) | 20260507 06:20:50.923 | INFO | Downloaded 38GiB to GPU in 2.17241s (17.4921GiB/s) | 20260507 06:20:56.066 | INFO | Analyzing chunk frames = [7766528,7767040) | 20260507 06:20:56.066 | INFO | MemInfo: { | 20260507 06:20:56.066 | INFO | "gpu" : " 17 GB", | 20260507 06:20:56.066 | INFO | "huge" : "", | 20260507 06:20:56.066 | INFO | "huge_shared" : " 27 GB", | 20260507 06:20:56.066 | INFO | "malloc" : " 14 GB", | 20260507 06:20:56.066 | INFO | "pinned" : " 29 GB" | 20260507 06:20:56.066 | INFO | } | 20260507 06:20:56.066 | INFO | Perf report for the last 5.14329 seconds: | 20260507 06:20:56.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:20:56.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:56.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:56.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:56.066 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:20:56.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:56.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:56.066 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:20:56.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:56.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:20:56.066 | INFO | Uploaded 36GiB to GPU in 1.98787s (18.1098GiB/s) | 20260507 06:20:56.066 | INFO | Downloaded 34GiB to GPU in 1.92441s (17.6678GiB/s) | 20260507 06:21:01.202 | INFO | Analyzing chunk frames = [7767040,7767552) | 20260507 06:21:01.202 | INFO | MemInfo: { | 20260507 06:21:01.202 | INFO | "gpu" : " 17 GB", | 20260507 06:21:01.202 | INFO | "huge" : "", | 20260507 06:21:01.203 | INFO | "huge_shared" : " 27 GB", | 20260507 06:21:01.203 | INFO | "malloc" : " 14 GB", | 20260507 06:21:01.203 | INFO | "pinned" : " 29 GB" | 20260507 06:21:01.203 | INFO | } | 20260507 06:21:01.203 | INFO | Perf report for the last 5.13642 seconds: | 20260507 06:21:01.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:21:01.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:01.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:01.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:01.203 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 06:21:01.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:01.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:01.203 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:21:01.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:01.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:01.203 | INFO | Uploaded 36GiB to GPU in 2.05572s (17.5121GiB/s) | 20260507 06:21:01.203 | INFO | Downloaded 34GiB to GPU in 1.95107s (17.4263GiB/s) | 20260507 06:21:06.347 | INFO | Analyzing chunk frames = [7767552,7768064) | 20260507 06:21:06.347 | INFO | MemInfo: { | 20260507 06:21:06.347 | INFO | "gpu" : " 17 GB", | 20260507 06:21:06.347 | INFO | "huge" : "", | 20260507 06:21:06.348 | INFO | "huge_shared" : " 27 GB", | 20260507 06:21:06.348 | INFO | "malloc" : " 14 GB", | 20260507 06:21:06.348 | INFO | "pinned" : " 29 GB" | 20260507 06:21:06.348 | INFO | } | 20260507 06:21:06.348 | INFO | Perf report for the last 5.14493 seconds: | 20260507 06:21:06.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:21:06.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:06.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:06.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:06.348 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 06:21:06.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:06.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:06.348 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:21:06.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:06.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:06.348 | INFO | Uploaded 36GiB to GPU in 2.01251s (17.8881GiB/s) | 20260507 06:21:06.348 | INFO | Downloaded 34GiB to GPU in 1.9416s (17.5113GiB/s) | 20260507 06:21:11.484 | INFO | Analyzing chunk frames = [7768064,7768576) | 20260507 06:21:11.484 | INFO | MemInfo: { | 20260507 06:21:11.484 | INFO | "gpu" : " 17 GB", | 20260507 06:21:11.484 | INFO | "huge" : "", | 20260507 06:21:11.484 | INFO | "huge_shared" : " 27 GB", | 20260507 06:21:11.484 | INFO | "malloc" : " 14 GB", | 20260507 06:21:11.484 | INFO | "pinned" : " 29 GB" | 20260507 06:21:11.484 | INFO | } | 20260507 06:21:11.484 | INFO | Perf report for the last 5.13613 seconds: | 20260507 06:21:11.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:21:11.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:11.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:11.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:11.484 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260507 06:21:11.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:11.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:11.484 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:21:11.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:11.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:11.484 | INFO | Uploaded 36GiB to GPU in 2.03921s (17.6539GiB/s) | 20260507 06:21:11.484 | INFO | Downloaded 34GiB to GPU in 1.92126s (17.6967GiB/s) | 20260507 06:21:16.621 | INFO | Analyzing chunk frames = [7768576,7769088) | 20260507 06:21:16.621 | INFO | MemInfo: { | 20260507 06:21:16.621 | INFO | "gpu" : " 17 GB", | 20260507 06:21:16.621 | INFO | "huge" : "", | 20260507 06:21:16.621 | INFO | "huge_shared" : " 27 GB", | 20260507 06:21:16.621 | INFO | "malloc" : " 14 GB", | 20260507 06:21:16.621 | INFO | "pinned" : " 29 GB" | 20260507 06:21:16.621 | INFO | } | 20260507 06:21:16.621 | INFO | Perf report for the last 5.13708 seconds: | 20260507 06:21:16.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:21:16.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:16.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:16.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:16.621 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 06:21:16.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:16.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:16.621 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 06:21:16.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:16.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:16.621 | INFO | Uploaded 36GiB to GPU in 2.06594s (17.4255GiB/s) | 20260507 06:21:16.621 | INFO | Downloaded 34GiB to GPU in 1.91888s (17.7186GiB/s) | 20260507 06:21:21.757 | INFO | Analyzing chunk frames = [7769088,7769600) | 20260507 06:21:21.757 | INFO | MemInfo: { | 20260507 06:21:21.757 | INFO | "gpu" : " 17 GB", | 20260507 06:21:21.757 | INFO | "huge" : "", | 20260507 06:21:21.757 | INFO | "huge_shared" : " 27 GB", | 20260507 06:21:21.757 | INFO | "malloc" : " 14 GB", | 20260507 06:21:21.757 | INFO | "pinned" : " 29 GB" | 20260507 06:21:21.757 | INFO | } | 20260507 06:21:21.757 | INFO | Perf report for the last 5.13612 seconds: | 20260507 06:21:21.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:21:21.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:21.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:21.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:21.757 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:21:21.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:21.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:21.757 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 06:21:21.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:21.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:21.757 | INFO | Uploaded 36GiB to GPU in 1.92301s (18.7206GiB/s) | 20260507 06:21:21.757 | INFO | Downloaded 34GiB to GPU in 1.97777s (17.1911GiB/s) | 20260507 06:21:26.893 | INFO | Analyzing chunk frames = [7769600,7770112) | 20260507 06:21:26.893 | INFO | MemInfo: { | 20260507 06:21:26.893 | INFO | "gpu" : " 17 GB", | 20260507 06:21:26.893 | INFO | "huge" : "", | 20260507 06:21:26.893 | INFO | "huge_shared" : " 27 GB", | 20260507 06:21:26.893 | INFO | "malloc" : " 14 GB", | 20260507 06:21:26.893 | INFO | "pinned" : " 29 GB" | 20260507 06:21:26.893 | INFO | } | 20260507 06:21:26.893 | INFO | Perf report for the last 5.13647 seconds: | 20260507 06:21:26.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:21:26.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:26.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:26.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:26.893 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:21:26.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:26.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:26.894 | INFO | PRE_HQ : 29.2%, 1.0, 15.1ms, 0 | 20260507 06:21:26.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:26.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:26.894 | INFO | Uploaded 36GiB to GPU in 1.96474s (18.3231GiB/s) | 20260507 06:21:26.894 | INFO | Downloaded 34GiB to GPU in 1.95846s (17.3605GiB/s) | 20260507 06:21:26.990 | INFO | Starting a round of Baz IO | 20260507 06:21:30.087 | INFO | Populating RTMetrics report | 20260507 06:21:30.088 | INFO | RTMetrics report populated | 20260507 06:21:30.088 | INFO | Opening tmp RTMetrics json file | 20260507 06:21:30.093 | INFO | Serializing RTMetrics to json | 20260507 06:21:30.106 | INFO | Serialization done, swapping tmp file now... | 20260507 06:21:30.106 | INFO | RTMetrics done | 20260507 06:21:30.174 | INFO | Finished a round of Baz IO: Wrote 1.671 GiB in 3.183 seconds | 20260507 06:21:30.174 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.63% / 0.00% / 67.37% | 20260507 06:21:30.174 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.124518% / 0% / 8.74201e-05% | 20260507 06:21:30.174 | INFO | Baz IO overall: | 20260507 06:21:30.174 | INFO | BytesWritten : 3920964919981 | 20260507 06:21:30.174 | INFO | NumEvents : 1879163997730 | 20260507 06:21:32.038 | INFO | Analyzing chunk frames = [7770112,7770624) | 20260507 06:21:32.038 | INFO | MemInfo: { | 20260507 06:21:32.038 | INFO | "gpu" : " 17 GB", | 20260507 06:21:32.038 | INFO | "huge" : "", | 20260507 06:21:32.038 | INFO | "huge_shared" : " 27 GB", | 20260507 06:21:32.038 | INFO | "malloc" : " 14 GB", | 20260507 06:21:32.038 | INFO | "pinned" : " 29 GB" | 20260507 06:21:32.038 | INFO | } | 20260507 06:21:32.038 | INFO | Perf report for the last 5.14458 seconds: | 20260507 06:21:32.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:21:32.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:32.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:32.038 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:32.038 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 06:21:32.038 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:32.038 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 06:21:32.038 | INFO | PRE_HQ : 59.0%, 1.8, 55.0ms, 0 | 20260507 06:21:32.038 | INFO | BAZWRITER : 61.5%, 3.2, 99.6ms, 0 | 20260507 06:21:32.038 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:21:32.038 | INFO | Uploaded 36GiB to GPU in 2.03146s (17.7213GiB/s) | 20260507 06:21:32.038 | INFO | Downloaded 38GiB to GPU in 2.18432s (17.3967GiB/s) | 20260507 06:21:37.183 | INFO | Analyzing chunk frames = [7770624,7771136) | 20260507 06:21:37.183 | INFO | MemInfo: { | 20260507 06:21:37.183 | INFO | "gpu" : " 17 GB", | 20260507 06:21:37.183 | INFO | "huge" : "", | 20260507 06:21:37.183 | INFO | "huge_shared" : " 27 GB", | 20260507 06:21:37.183 | INFO | "malloc" : " 14 GB", | 20260507 06:21:37.183 | INFO | "pinned" : " 29 GB" | 20260507 06:21:37.183 | INFO | } | 20260507 06:21:37.183 | INFO | Perf report for the last 5.14498 seconds: | 20260507 06:21:37.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:21:37.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:37.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:37.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:37.183 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 06:21:37.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:37.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:37.183 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:21:37.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:37.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:37.183 | INFO | Uploaded 36GiB to GPU in 1.99386s (18.0554GiB/s) | 20260507 06:21:37.183 | INFO | Downloaded 34GiB to GPU in 1.9348s (17.5729GiB/s) | 20260507 06:21:42.325 | INFO | Analyzing chunk frames = [7771136,7771648) | 20260507 06:21:42.325 | INFO | MemInfo: { | 20260507 06:21:42.325 | INFO | "gpu" : " 17 GB", | 20260507 06:21:42.325 | INFO | "huge" : "", | 20260507 06:21:42.325 | INFO | "huge_shared" : " 27 GB", | 20260507 06:21:42.325 | INFO | "malloc" : " 14 GB", | 20260507 06:21:42.325 | INFO | "pinned" : " 29 GB" | 20260507 06:21:42.325 | INFO | } | 20260507 06:21:42.325 | INFO | Perf report for the last 5.14183 seconds: | 20260507 06:21:42.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:21:42.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:42.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:42.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:42.325 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 06:21:42.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:42.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:42.325 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:21:42.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:42.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:42.325 | INFO | Uploaded 36GiB to GPU in 2.03654s (17.6771GiB/s) | 20260507 06:21:42.325 | INFO | Downloaded 34GiB to GPU in 1.92235s (17.6867GiB/s) | 20260507 06:21:47.458 | INFO | Analyzing chunk frames = [7771648,7772160) | 20260507 06:21:47.458 | INFO | MemInfo: { | 20260507 06:21:47.458 | INFO | "gpu" : " 17 GB", | 20260507 06:21:47.458 | INFO | "huge" : "", | 20260507 06:21:47.458 | INFO | "huge_shared" : " 27 GB", | 20260507 06:21:47.458 | INFO | "malloc" : " 14 GB", | 20260507 06:21:47.458 | INFO | "pinned" : " 29 GB" | 20260507 06:21:47.458 | INFO | } | 20260507 06:21:47.458 | INFO | Perf report for the last 5.13322 seconds: | 20260507 06:21:47.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:21:47.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:47.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:47.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:47.458 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 06:21:47.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:47.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:47.458 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:21:47.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:47.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:47.458 | INFO | Uploaded 36GiB to GPU in 2.05403s (17.5265GiB/s) | 20260507 06:21:47.458 | INFO | Downloaded 34GiB to GPU in 1.95091s (17.4278GiB/s) | 20260507 06:21:52.594 | INFO | Analyzing chunk frames = [7772160,7772672) | 20260507 06:21:52.594 | INFO | MemInfo: { | 20260507 06:21:52.594 | INFO | "gpu" : " 17 GB", | 20260507 06:21:52.594 | INFO | "huge" : "", | 20260507 06:21:52.594 | INFO | "huge_shared" : " 27 GB", | 20260507 06:21:52.594 | INFO | "malloc" : " 14 GB", | 20260507 06:21:52.594 | INFO | "pinned" : " 29 GB" | 20260507 06:21:52.594 | INFO | } | 20260507 06:21:52.594 | INFO | Perf report for the last 5.1356 seconds: | 20260507 06:21:52.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:21:52.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:52.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:52.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:52.594 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 06:21:52.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:52.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:52.594 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:21:52.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:52.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:52.594 | INFO | Uploaded 36GiB to GPU in 2.02457s (17.7815GiB/s) | 20260507 06:21:52.594 | INFO | Downloaded 34GiB to GPU in 1.91453s (17.7589GiB/s) | 20260507 06:21:57.731 | INFO | Analyzing chunk frames = [7772672,7773184) | 20260507 06:21:57.731 | INFO | MemInfo: { | 20260507 06:21:57.731 | INFO | "gpu" : " 17 GB", | 20260507 06:21:57.731 | INFO | "huge" : "", | 20260507 06:21:57.731 | INFO | "huge_shared" : " 27 GB", | 20260507 06:21:57.731 | INFO | "malloc" : " 14 GB", | 20260507 06:21:57.731 | INFO | "pinned" : " 29 GB" | 20260507 06:21:57.731 | INFO | } | 20260507 06:21:57.731 | INFO | Perf report for the last 5.13762 seconds: | 20260507 06:21:57.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:21:57.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:57.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:57.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:57.731 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:21:57.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:57.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:57.731 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:21:57.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:57.732 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:21:57.732 | INFO | Uploaded 36GiB to GPU in 2.04967s (17.5638GiB/s) | 20260507 06:21:57.732 | INFO | Downloaded 34GiB to GPU in 1.9618s (17.3311GiB/s) | 20260507 06:22:02.876 | INFO | Analyzing chunk frames = [7773184,7773696) | 20260507 06:22:02.876 | INFO | MemInfo: { | 20260507 06:22:02.876 | INFO | "gpu" : " 17 GB", | 20260507 06:22:02.876 | INFO | "huge" : "", | 20260507 06:22:02.876 | INFO | "huge_shared" : " 27 GB", | 20260507 06:22:02.876 | INFO | "malloc" : " 14 GB", | 20260507 06:22:02.876 | INFO | "pinned" : " 29 GB" | 20260507 06:22:02.877 | INFO | } | 20260507 06:22:02.877 | INFO | Perf report for the last 5.14517 seconds: | 20260507 06:22:02.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:22:02.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:02.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:02.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:02.877 | INFO | ANALYSIS : 61.2%, 3.0, 95.4ms, 0 | 20260507 06:22:02.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:02.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:02.877 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 06:22:02.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:02.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:02.877 | INFO | Uploaded 36GiB to GPU in 2.04373s (17.6149GiB/s) | 20260507 06:22:02.877 | INFO | Downloaded 34GiB to GPU in 1.99038s (17.0822GiB/s) | 20260507 06:22:08.012 | INFO | Analyzing chunk frames = [7773696,7774208) | 20260507 06:22:08.012 | INFO | MemInfo: { | 20260507 06:22:08.012 | INFO | "gpu" : " 17 GB", | 20260507 06:22:08.012 | INFO | "huge" : "", | 20260507 06:22:08.013 | INFO | "huge_shared" : " 27 GB", | 20260507 06:22:08.013 | INFO | "malloc" : " 14 GB", | 20260507 06:22:08.013 | INFO | "pinned" : " 29 GB" | 20260507 06:22:08.013 | INFO | } | 20260507 06:22:08.013 | INFO | Perf report for the last 5.13611 seconds: | 20260507 06:22:08.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:22:08.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:08.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:08.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:08.013 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 06:22:08.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:08.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:08.013 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:22:08.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:08.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:08.013 | INFO | Uploaded 36GiB to GPU in 1.99309s (18.0624GiB/s) | 20260507 06:22:08.013 | INFO | Downloaded 34GiB to GPU in 1.95611s (17.3814GiB/s) | 20260507 06:22:08.013 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:22:08.013 | INFO | Total mem usage: 18 GiB | 20260507 06:22:08.013 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:22:08.013 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:22:08.013 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:22:08.013 | INFO | High Water Breakdown: | 20260507 06:22:08.013 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:22:08.013 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:22:08.013 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:22:08.013 | INFO | ------------------------------------------ | 20260507 06:22:08.013 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:22:08.013 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:22:08.013 | INFO | Total mem usage: 26.5 GiB | 20260507 06:22:08.013 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:22:08.013 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:22:08.013 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:22:08.013 | INFO | High Water Breakdown: | 20260507 06:22:08.013 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:22:08.013 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:22:08.013 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:22:08.013 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:22:08.013 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:22:08.013 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:22:08.013 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:22:08.013 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:22:08.013 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:22:08.013 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:22:08.013 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:22:08.013 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:22:08.013 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:22:08.013 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:22:08.013 | INFO | ------------------------------------------ | 20260507 06:22:08.013 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:22:08.013 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:22:08.013 | INFO | Total mem usage: 25.6 GiB | 20260507 06:22:08.013 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:22:08.013 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 06:22:08.013 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 06:22:08.013 | INFO | High Water Breakdown: | 20260507 06:22:08.013 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 06:22:08.013 | INFO | ------------------------------------------ | 20260507 06:22:08.013 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:22:08.013 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:22:08.014 | INFO | Total mem usage: 15.7 GiB | 20260507 06:22:08.014 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:22:08.014 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:22:08.014 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:22:08.014 | INFO | High Water Breakdown: | 20260507 06:22:08.014 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:22:08.014 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:22:08.014 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:22:08.014 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:22:08.014 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:22:08.014 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:22:08.014 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:22:08.014 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:22:08.014 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:22:08.014 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:22:08.014 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:22:08.014 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:22:08.014 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:22:08.014 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:22:08.014 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:22:08.014 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:22:08.014 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:22:08.014 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:22:08.014 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:22:08.014 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:22:08.014 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:22:08.014 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:22:08.014 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:22:08.014 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:22:08.014 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:22:08.014 | INFO | ------------------------------------------ | 20260507 06:22:08.014 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:22:08.014 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:22:08.014 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:22:08.014 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:22:08.014 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.35e+01 / 1.20e+00 | 20260507 06:22:08.014 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.02e-01, 9.41e-03 / 5.45e-03 | 20260507 06:22:08.014 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.40e+00, 5.07e-01 / 2.87e-01 | 20260507 06:22:08.014 | INFO | PerfReport: 3.22%, Histogram -- 9.90e-01 / 1.50e+00, 1.68e-01 / 9.95e-02 | 20260507 06:22:08.014 | INFO | PerfReport: 16.24%, DME -- 3.99e+01 / 4.47e+01, 3.84e+00 / 1.29e+00 | 20260507 06:22:08.014 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.77e+01 / 1.98e+01, 1.33e+00 / 1.31e+00 | 20260507 06:22:08.014 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.13e+00 / 2.71e+00, 3.86e-01 / 1.82e-01 | 20260507 06:22:08.014 | INFO | PerfReport: 2.96%, Metrics -- 9.09e-01 / 1.24e+00, 2.42e-01 / 9.33e-02 | 20260507 06:22:08.014 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.07e-01 / 3.62e-01, 5.09e-02 / 1.72e-02 | 20260507 06:22:08.014 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:22:08.014 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:22:08.014 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:22:08.014 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 06:22:08.014 | INFO | PerfReport: 21.86%, Upload -- 2.10e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260507 06:22:08.014 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260507 06:22:08.014 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 06:22:08.014 | INFO | PerfReport: 1.58%, LockUpload -- 1.52e+00 / 0.00e+00, 4.98e+00 / 0.00e+00 | 20260507 06:22:08.014 | INFO | PerfReport: 1.21%, LockDownload -- 1.17e+00 / 0.00e+00, 2.88e+00 / 0.00e+00 | 20260507 06:22:08.014 | INFO | PerfReport: 22.05%, LockCompute -- 2.12e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 06:22:08.014 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.65e-02 / 0.00e+00 | 20260507 06:22:08.110 | INFO | Starting a round of Baz IO | 20260507 06:22:11.207 | INFO | Populating RTMetrics report | 20260507 06:22:11.207 | INFO | RTMetrics report populated | 20260507 06:22:11.207 | INFO | Opening tmp RTMetrics json file | 20260507 06:22:11.224 | INFO | Serializing RTMetrics to json | 20260507 06:22:11.226 | INFO | Serialization done, swapping tmp file now... | 20260507 06:22:11.226 | INFO | RTMetrics done | 20260507 06:22:11.295 | INFO | Finished a round of Baz IO: Wrote 1.67 GiB in 3.185 seconds | 20260507 06:22:11.295 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.63% / 0.00% / 67.37% | 20260507 06:22:11.295 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.124753% / 3.97364e-06% / 0.000103315% | 20260507 06:22:11.295 | INFO | Baz IO overall: | 20260507 06:22:11.295 | INFO | BytesWritten : 3922757911931 | 20260507 06:22:11.295 | INFO | NumEvents : 1880003135967 | 20260507 06:22:13.150 | INFO | Analyzing chunk frames = [7774208,7774720) | 20260507 06:22:13.150 | INFO | MemInfo: { | 20260507 06:22:13.150 | INFO | "gpu" : " 17 GB", | 20260507 06:22:13.150 | INFO | "huge" : "", | 20260507 06:22:13.150 | INFO | "huge_shared" : " 27 GB", | 20260507 06:22:13.150 | INFO | "malloc" : " 14 GB", | 20260507 06:22:13.150 | INFO | "pinned" : " 29 GB" | 20260507 06:22:13.150 | INFO | } | 20260507 06:22:13.150 | INFO | Perf report for the last 5.13778 seconds: | 20260507 06:22:13.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:22:13.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:13.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:13.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:13.151 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 06:22:13.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:13.151 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:22:13.151 | INFO | PRE_HQ : 58.9%, 1.8, 54.5ms, 0 | 20260507 06:22:13.151 | INFO | BAZWRITER : 61.6%, 3.1, 99.2ms, 0 | 20260507 06:22:13.151 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:22:13.151 | INFO | Uploaded 36GiB to GPU in 2.0618s (17.4604GiB/s) | 20260507 06:22:13.151 | INFO | Downloaded 38GiB to GPU in 2.18009s (17.4305GiB/s) | 20260507 06:22:18.294 | INFO | Analyzing chunk frames = [7774720,7775232) | 20260507 06:22:18.295 | INFO | MemInfo: { | 20260507 06:22:18.295 | INFO | "gpu" : " 17 GB", | 20260507 06:22:18.295 | INFO | "huge" : "", | 20260507 06:22:18.295 | INFO | "huge_shared" : " 27 GB", | 20260507 06:22:18.295 | INFO | "malloc" : " 14 GB", | 20260507 06:22:18.295 | INFO | "pinned" : " 29 GB" | 20260507 06:22:18.295 | INFO | } | 20260507 06:22:18.295 | INFO | Perf report for the last 5.14416 seconds: | 20260507 06:22:18.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:22:18.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:18.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:18.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:18.295 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 06:22:18.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:18.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:18.295 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:22:18.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:18.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:18.295 | INFO | Uploaded 36GiB to GPU in 2.00208s (17.9813GiB/s) | 20260507 06:22:18.295 | INFO | Downloaded 34GiB to GPU in 1.93282s (17.5909GiB/s) | 20260507 06:22:23.434 | INFO | Analyzing chunk frames = [7775232,7775744) | 20260507 06:22:23.434 | INFO | MemInfo: { | 20260507 06:22:23.434 | INFO | "gpu" : " 17 GB", | 20260507 06:22:23.434 | INFO | "huge" : "", | 20260507 06:22:23.434 | INFO | "huge_shared" : " 27 GB", | 20260507 06:22:23.434 | INFO | "malloc" : " 14 GB", | 20260507 06:22:23.434 | INFO | "pinned" : " 29 GB" | 20260507 06:22:23.434 | INFO | } | 20260507 06:22:23.434 | INFO | Perf report for the last 5.13966 seconds: | 20260507 06:22:23.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:22:23.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:23.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:23.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:23.434 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 06:22:23.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:23.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:23.434 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:22:23.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:23.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:23.435 | INFO | Uploaded 36GiB to GPU in 2.04659s (17.5903GiB/s) | 20260507 06:22:23.435 | INFO | Downloaded 34GiB to GPU in 1.9499s (17.4368GiB/s) | 20260507 06:22:28.568 | INFO | Analyzing chunk frames = [7775744,7776256) | 20260507 06:22:28.568 | INFO | MemInfo: { | 20260507 06:22:28.568 | INFO | "gpu" : " 17 GB", | 20260507 06:22:28.568 | INFO | "huge" : "", | 20260507 06:22:28.569 | INFO | "huge_shared" : " 27 GB", | 20260507 06:22:28.569 | INFO | "malloc" : " 14 GB", | 20260507 06:22:28.569 | INFO | "pinned" : " 29 GB" | 20260507 06:22:28.569 | INFO | } | 20260507 06:22:28.569 | INFO | Perf report for the last 5.13439 seconds: | 20260507 06:22:28.569 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:22:28.569 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:28.569 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:28.569 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:28.569 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 06:22:28.569 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:28.569 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:28.569 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:22:28.569 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:28.569 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:28.569 | INFO | Uploaded 36GiB to GPU in 2.04005s (17.6467GiB/s) | 20260507 06:22:28.569 | INFO | Downloaded 34GiB to GPU in 1.95025s (17.4337GiB/s) | 20260507 06:22:33.702 | INFO | Analyzing chunk frames = [7776256,7776768) | 20260507 06:22:33.702 | INFO | MemInfo: { | 20260507 06:22:33.702 | INFO | "gpu" : " 17 GB", | 20260507 06:22:33.702 | INFO | "huge" : "", | 20260507 06:22:33.703 | INFO | "huge_shared" : " 27 GB", | 20260507 06:22:33.703 | INFO | "malloc" : " 14 GB", | 20260507 06:22:33.703 | INFO | "pinned" : " 29 GB" | 20260507 06:22:33.703 | INFO | } | 20260507 06:22:33.703 | INFO | Perf report for the last 5.13395 seconds: | 20260507 06:22:33.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:22:33.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:33.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:33.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:33.703 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 06:22:33.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:33.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:33.703 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:22:33.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:33.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:33.703 | INFO | Uploaded 36GiB to GPU in 2.01333s (17.8808GiB/s) | 20260507 06:22:33.703 | INFO | Downloaded 34GiB to GPU in 1.91038s (17.7975GiB/s) | 20260507 06:22:38.850 | INFO | Analyzing chunk frames = [7776768,7777280) | 20260507 06:22:38.850 | INFO | MemInfo: { | 20260507 06:22:38.850 | INFO | "gpu" : " 17 GB", | 20260507 06:22:38.850 | INFO | "huge" : "", | 20260507 06:22:38.851 | INFO | "huge_shared" : " 27 GB", | 20260507 06:22:38.851 | INFO | "malloc" : " 14 GB", | 20260507 06:22:38.851 | INFO | "pinned" : " 29 GB" | 20260507 06:22:38.851 | INFO | } | 20260507 06:22:38.851 | INFO | Perf report for the last 5.14801 seconds: | 20260507 06:22:38.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:22:38.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:38.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:38.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:38.851 | INFO | ANALYSIS : 60.3%, 3.0, 94.3ms, 0 | 20260507 06:22:38.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:38.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:38.851 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 06:22:38.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:38.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:38.851 | INFO | Uploaded 36GiB to GPU in 2.05577s (17.5117GiB/s) | 20260507 06:22:38.851 | INFO | Downloaded 34GiB to GPU in 1.91838s (17.7233GiB/s) | 20260507 06:22:43.991 | INFO | Analyzing chunk frames = [7777280,7777792) | 20260507 06:22:43.991 | INFO | MemInfo: { | 20260507 06:22:43.991 | INFO | "gpu" : " 17 GB", | 20260507 06:22:43.991 | INFO | "huge" : "", | 20260507 06:22:43.991 | INFO | "huge_shared" : " 27 GB", | 20260507 06:22:43.991 | INFO | "malloc" : " 14 GB", | 20260507 06:22:43.991 | INFO | "pinned" : " 29 GB" | 20260507 06:22:43.991 | INFO | } | 20260507 06:22:43.991 | INFO | Perf report for the last 5.14073 seconds: | 20260507 06:22:43.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:22:43.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:43.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:43.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:43.991 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 06:22:43.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:43.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:43.992 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 06:22:43.992 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:43.992 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:43.992 | INFO | Uploaded 36GiB to GPU in 2.04015s (17.6458GiB/s) | 20260507 06:22:43.992 | INFO | Downloaded 34GiB to GPU in 1.9981s (17.0162GiB/s) | 20260507 06:22:49.135 | INFO | Analyzing chunk frames = [7777792,7778304) | 20260507 06:22:49.135 | INFO | MemInfo: { | 20260507 06:22:49.135 | INFO | "gpu" : " 17 GB", | 20260507 06:22:49.135 | INFO | "huge" : "", | 20260507 06:22:49.135 | INFO | "huge_shared" : " 27 GB", | 20260507 06:22:49.135 | INFO | "malloc" : " 14 GB", | 20260507 06:22:49.135 | INFO | "pinned" : " 29 GB" | 20260507 06:22:49.135 | INFO | } | 20260507 06:22:49.135 | INFO | Perf report for the last 5.14319 seconds: | 20260507 06:22:49.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:22:49.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:49.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:49.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:49.135 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:22:49.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:49.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:49.135 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:22:49.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:49.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:49.135 | INFO | Uploaded 36GiB to GPU in 1.98377s (18.1473GiB/s) | 20260507 06:22:49.135 | INFO | Downloaded 34GiB to GPU in 1.95968s (17.3498GiB/s) | 20260507 06:22:49.232 | INFO | Starting a round of Baz IO | 20260507 06:22:52.343 | INFO | Populating RTMetrics report | 20260507 06:22:52.343 | INFO | RTMetrics report populated | 20260507 06:22:52.343 | INFO | Opening tmp RTMetrics json file | 20260507 06:22:52.358 | INFO | Serializing RTMetrics to json | 20260507 06:22:52.364 | INFO | Serialization done, swapping tmp file now... | 20260507 06:22:52.364 | INFO | RTMetrics done | 20260507 06:22:52.424 | INFO | Finished a round of Baz IO: Wrote 1.669 GiB in 3.192 seconds | 20260507 06:22:52.424 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.63% / 0.00% / 67.37% | 20260507 06:22:52.424 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.123926% / 3.97364e-06% / 7.54992e-05% | 20260507 06:22:52.424 | INFO | Baz IO overall: | 20260507 06:22:52.424 | INFO | BytesWritten : 3924550272314 | 20260507 06:22:52.424 | INFO | NumEvents : 1880841761709 | 20260507 06:22:54.268 | INFO | Analyzing chunk frames = [7778304,7778816) | 20260507 06:22:54.268 | INFO | MemInfo: { | 20260507 06:22:54.268 | INFO | "gpu" : " 17 GB", | 20260507 06:22:54.268 | INFO | "huge" : "", | 20260507 06:22:54.268 | INFO | "huge_shared" : " 27 GB", | 20260507 06:22:54.268 | INFO | "malloc" : " 14 GB", | 20260507 06:22:54.268 | INFO | "pinned" : " 29 GB" | 20260507 06:22:54.268 | INFO | } | 20260507 06:22:54.268 | INFO | Perf report for the last 5.1335 seconds: | 20260507 06:22:54.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:22:54.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:54.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:54.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:54.268 | INFO | ANALYSIS : 61.4%, 3.0, 95.9ms, 0 | 20260507 06:22:54.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:54.268 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 06:22:54.268 | INFO | PRE_HQ : 58.7%, 1.8, 55.0ms, 0 | 20260507 06:22:54.268 | INFO | BAZWRITER : 61.8%, 3.2, 100.2ms, 0 | 20260507 06:22:54.268 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 06:22:54.268 | INFO | Uploaded 36GiB to GPU in 2.02784s (17.7529GiB/s) | 20260507 06:22:54.268 | INFO | Downloaded 38GiB to GPU in 2.16511s (17.5511GiB/s) | 20260507 06:22:59.406 | INFO | Analyzing chunk frames = [7778816,7779328) | 20260507 06:22:59.406 | INFO | MemInfo: { | 20260507 06:22:59.406 | INFO | "gpu" : " 17 GB", | 20260507 06:22:59.406 | INFO | "huge" : "", | 20260507 06:22:59.406 | INFO | "huge_shared" : " 27 GB", | 20260507 06:22:59.406 | INFO | "malloc" : " 14 GB", | 20260507 06:22:59.406 | INFO | "pinned" : " 29 GB" | 20260507 06:22:59.406 | INFO | } | 20260507 06:22:59.406 | INFO | Perf report for the last 5.13783 seconds: | 20260507 06:22:59.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:22:59.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:59.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:59.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:59.406 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:22:59.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:59.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:59.406 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:22:59.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:59.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:22:59.406 | INFO | Uploaded 36GiB to GPU in 1.98574s (18.1292GiB/s) | 20260507 06:22:59.406 | INFO | Downloaded 34GiB to GPU in 1.93867s (17.5378GiB/s) | 20260507 06:23:04.548 | INFO | Analyzing chunk frames = [7779328,7779840) | 20260507 06:23:04.548 | INFO | MemInfo: { | 20260507 06:23:04.548 | INFO | "gpu" : " 17 GB", | 20260507 06:23:04.548 | INFO | "huge" : "", | 20260507 06:23:04.549 | INFO | "huge_shared" : " 27 GB", | 20260507 06:23:04.549 | INFO | "malloc" : " 14 GB", | 20260507 06:23:04.549 | INFO | "pinned" : " 29 GB" | 20260507 06:23:04.549 | INFO | } | 20260507 06:23:04.549 | INFO | Perf report for the last 5.14263 seconds: | 20260507 06:23:04.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:23:04.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:04.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:04.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:04.549 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 06:23:04.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:04.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:04.549 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:23:04.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:04.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:04.549 | INFO | Uploaded 36GiB to GPU in 2.03428s (17.6966GiB/s) | 20260507 06:23:04.549 | INFO | Downloaded 34GiB to GPU in 1.95584s (17.3838GiB/s) | 20260507 06:23:09.687 | INFO | Analyzing chunk frames = [7779840,7780352) | 20260507 06:23:09.687 | INFO | MemInfo: { | 20260507 06:23:09.687 | INFO | "gpu" : " 17 GB", | 20260507 06:23:09.687 | INFO | "huge" : "", | 20260507 06:23:09.687 | INFO | "huge_shared" : " 27 GB", | 20260507 06:23:09.687 | INFO | "malloc" : " 14 GB", | 20260507 06:23:09.687 | INFO | "pinned" : " 29 GB" | 20260507 06:23:09.687 | INFO | } | 20260507 06:23:09.687 | INFO | Perf report for the last 5.13879 seconds: | 20260507 06:23:09.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:23:09.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:09.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:09.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:09.687 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 06:23:09.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:09.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:09.688 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:23:09.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:09.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:09.688 | INFO | Uploaded 36GiB to GPU in 2.01128s (17.8991GiB/s) | 20260507 06:23:09.688 | INFO | Downloaded 34GiB to GPU in 1.94567s (17.4747GiB/s) | 20260507 06:23:14.831 | INFO | Analyzing chunk frames = [7780352,7780864) | 20260507 06:23:14.831 | INFO | MemInfo: { | 20260507 06:23:14.831 | INFO | "gpu" : " 17 GB", | 20260507 06:23:14.831 | INFO | "huge" : "", | 20260507 06:23:14.831 | INFO | "huge_shared" : " 27 GB", | 20260507 06:23:14.831 | INFO | "malloc" : " 14 GB", | 20260507 06:23:14.831 | INFO | "pinned" : " 29 GB" | 20260507 06:23:14.831 | INFO | } | 20260507 06:23:14.831 | INFO | Perf report for the last 5.14377 seconds: | 20260507 06:23:14.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:23:14.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:14.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:14.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:14.831 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 06:23:14.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:14.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:14.831 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 06:23:14.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:14.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:14.832 | INFO | Uploaded 36GiB to GPU in 2.02211s (17.8032GiB/s) | 20260507 06:23:14.832 | INFO | Downloaded 34GiB to GPU in 1.92s (17.7083GiB/s) | 20260507 06:23:19.967 | INFO | Analyzing chunk frames = [7780864,7781376) | 20260507 06:23:19.967 | INFO | MemInfo: { | 20260507 06:23:19.967 | INFO | "gpu" : " 17 GB", | 20260507 06:23:19.967 | INFO | "huge" : "", | 20260507 06:23:19.967 | INFO | "huge_shared" : " 27 GB", | 20260507 06:23:19.967 | INFO | "malloc" : " 14 GB", | 20260507 06:23:19.967 | INFO | "pinned" : " 29 GB" | 20260507 06:23:19.967 | INFO | } | 20260507 06:23:19.967 | INFO | Perf report for the last 5.13604 seconds: | 20260507 06:23:19.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:23:19.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:19.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:19.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:19.967 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 06:23:19.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:19.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:19.967 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 06:23:19.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:19.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:19.968 | INFO | Uploaded 36GiB to GPU in 2.07047s (17.3873GiB/s) | 20260507 06:23:19.968 | INFO | Downloaded 34GiB to GPU in 1.94363s (17.4931GiB/s) | 20260507 06:23:25.103 | INFO | Analyzing chunk frames = [7781376,7781888) | 20260507 06:23:25.103 | INFO | MemInfo: { | 20260507 06:23:25.103 | INFO | "gpu" : " 17 GB", | 20260507 06:23:25.103 | INFO | "huge" : "", | 20260507 06:23:25.103 | INFO | "huge_shared" : " 27 GB", | 20260507 06:23:25.103 | INFO | "malloc" : " 14 GB", | 20260507 06:23:25.103 | INFO | "pinned" : " 29 GB" | 20260507 06:23:25.103 | INFO | } | 20260507 06:23:25.103 | INFO | Perf report for the last 5.13556 seconds: | 20260507 06:23:25.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:23:25.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:25.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:25.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:25.103 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:23:25.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:25.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:25.103 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 06:23:25.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:25.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:25.103 | INFO | Uploaded 36GiB to GPU in 1.93357s (18.6184GiB/s) | 20260507 06:23:25.103 | INFO | Downloaded 34GiB to GPU in 1.97667s (17.2007GiB/s) | 20260507 06:23:30.241 | INFO | Analyzing chunk frames = [7781888,7782400) | 20260507 06:23:30.241 | INFO | MemInfo: { | 20260507 06:23:30.241 | INFO | "gpu" : " 17 GB", | 20260507 06:23:30.241 | INFO | "huge" : "", | 20260507 06:23:30.241 | INFO | "huge_shared" : " 27 GB", | 20260507 06:23:30.241 | INFO | "malloc" : " 14 GB", | 20260507 06:23:30.241 | INFO | "pinned" : " 29 GB" | 20260507 06:23:30.241 | INFO | } | 20260507 06:23:30.241 | INFO | Perf report for the last 5.13837 seconds: | 20260507 06:23:30.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:23:30.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:30.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:30.241 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:30.241 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:23:30.241 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:30.241 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:30.241 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 06:23:30.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:30.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:30.242 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:23:30.242 | INFO | Total mem usage: 18 GiB | 20260507 06:23:30.242 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:23:30.242 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 06:23:30.242 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:23:30.242 | INFO | High Water Breakdown: | 20260507 06:23:30.242 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:23:30.242 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:23:30.242 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:23:30.242 | INFO | ------------------------------------------ | 20260507 06:23:30.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:23:30.242 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:23:30.242 | INFO | Total mem usage: 26.5 GiB | 20260507 06:23:30.242 | INFO | Uploaded 36GiB to GPU in 1.96349s (18.3347GiB/s) | 20260507 06:23:30.242 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:23:30.242 | INFO | Downloaded 34GiB to GPU in 1.96039s (17.3435GiB/s) | 20260507 06:23:30.242 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:23:30.242 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:23:30.242 | INFO | High Water Breakdown: | 20260507 06:23:30.242 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:23:30.242 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:23:30.242 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:23:30.242 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:23:30.242 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:23:30.242 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:23:30.242 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:23:30.242 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:23:30.242 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:23:30.242 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:23:30.242 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:23:30.242 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:23:30.242 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:23:30.242 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:23:30.242 | INFO | ------------------------------------------ | 20260507 06:23:30.242 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:23:30.242 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:23:30.242 | INFO | Total mem usage: 25.6 GiB | 20260507 06:23:30.242 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:23:30.242 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:23:30.242 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:23:30.242 | INFO | High Water Breakdown: | 20260507 06:23:30.242 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:23:30.243 | INFO | ------------------------------------------ | 20260507 06:23:30.243 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:23:30.243 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:23:30.243 | INFO | Total mem usage: 15.7 GiB | 20260507 06:23:30.243 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:23:30.243 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:23:30.243 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:23:30.243 | INFO | High Water Breakdown: | 20260507 06:23:30.243 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:23:30.243 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:23:30.243 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:23:30.243 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:23:30.243 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:23:30.243 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:23:30.243 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:23:30.243 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:23:30.243 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:23:30.243 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:23:30.243 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:23:30.243 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:23:30.243 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:23:30.243 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:23:30.243 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:23:30.243 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:23:30.243 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:23:30.243 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:23:30.243 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:23:30.243 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:23:30.243 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:23:30.243 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:23:30.243 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:23:30.243 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:23:30.243 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:23:30.243 | INFO | ------------------------------------------ | 20260507 06:23:30.243 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:23:30.243 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:23:30.243 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:23:30.243 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:23:30.243 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.49e+01, 1.34e+01 / 1.28e+00 | 20260507 06:23:30.243 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.00e-01, 9.62e-03 / 6.18e-03 | 20260507 06:23:30.243 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.36e+00, 5.15e-01 / 3.33e-01 | 20260507 06:23:30.243 | INFO | PerfReport: 3.24%, Histogram -- 9.96e-01 / 1.47e+00, 1.72e-01 / 1.17e-01 | 20260507 06:23:30.243 | INFO | PerfReport: 16.25%, DME -- 4.00e+01 / 4.49e+01, 4.01e+00 / 1.76e+00 | 20260507 06:23:30.243 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.78e+01 / 1.98e+01, 1.37e+00 / 1.45e+00 | 20260507 06:23:30.243 | INFO | PerfReport: 6.88%, PulseAccumulating -- 2.12e+00 / 2.69e+00, 3.80e-01 / 1.87e-01 | 20260507 06:23:30.243 | INFO | PerfReport: 2.95%, Metrics -- 9.06e-01 / 1.25e+00, 2.43e-01 / 1.32e-01 | 20260507 06:23:30.243 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.06e-01 / 3.60e-01, 4.98e-02 / 1.75e-02 | 20260507 06:23:30.243 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:23:30.243 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:23:30.243 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:23:30.243 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 06:23:30.243 | INFO | PerfReport: 21.86%, Upload -- 2.10e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260507 06:23:30.243 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 06:23:30.243 | INFO | PerfReport: 31.99%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:23:30.243 | INFO | PerfReport: 1.48%, LockUpload -- 1.43e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 06:23:30.243 | INFO | PerfReport: 1.24%, LockDownload -- 1.19e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260507 06:23:30.243 | INFO | PerfReport: 22.03%, LockCompute -- 2.12e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 06:23:30.243 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260507 06:23:30.339 | INFO | Starting a round of Baz IO | 20260507 06:23:33.436 | INFO | Populating RTMetrics report | 20260507 06:23:33.436 | INFO | RTMetrics report populated | 20260507 06:23:33.436 | INFO | Opening tmp RTMetrics json file | 20260507 06:23:33.455 | INFO | Serializing RTMetrics to json | 20260507 06:23:33.456 | INFO | Serialization done, swapping tmp file now... | 20260507 06:23:33.456 | INFO | RTMetrics done | 20260507 06:23:33.523 | INFO | Finished a round of Baz IO: Wrote 1.67 GiB in 3.184 seconds | 20260507 06:23:33.523 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.63% / 0.00% / 67.37% | 20260507 06:23:33.523 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.123099% / 0% / 9.93411e-05% | 20260507 06:23:33.523 | INFO | Baz IO overall: | 20260507 06:23:33.523 | INFO | BytesWritten : 3926343699859 | 20260507 06:23:33.523 | INFO | NumEvents : 1881681389679 | 20260507 06:23:35.381 | INFO | Analyzing chunk frames = [7782400,7782912) | 20260507 06:23:35.381 | INFO | MemInfo: { | 20260507 06:23:35.381 | INFO | "gpu" : " 17 GB", | 20260507 06:23:35.381 | INFO | "huge" : "", | 20260507 06:23:35.381 | INFO | "huge_shared" : " 27 GB", | 20260507 06:23:35.381 | INFO | "malloc" : " 14 GB", | 20260507 06:23:35.381 | INFO | "pinned" : " 29 GB" | 20260507 06:23:35.381 | INFO | } | 20260507 06:23:35.381 | INFO | Perf report for the last 5.13938 seconds: | 20260507 06:23:35.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:23:35.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:35.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:35.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:35.381 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 06:23:35.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:35.381 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 06:23:35.381 | INFO | PRE_HQ : 59.6%, 1.9, 57.7ms, 0 | 20260507 06:23:35.381 | INFO | BAZWRITER : 61.5%, 3.2, 102.6ms, 0 | 20260507 06:23:35.381 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:23:35.381 | INFO | Uploaded 36GiB to GPU in 2.04331s (17.6185GiB/s) | 20260507 06:23:35.381 | INFO | Downloaded 38GiB to GPU in 2.17779s (17.4489GiB/s) | 20260507 06:23:40.521 | INFO | Analyzing chunk frames = [7782912,7783424) | 20260507 06:23:40.521 | INFO | MemInfo: { | 20260507 06:23:40.521 | INFO | "gpu" : " 17 GB", | 20260507 06:23:40.521 | INFO | "huge" : "", | 20260507 06:23:40.521 | INFO | "huge_shared" : " 27 GB", | 20260507 06:23:40.521 | INFO | "malloc" : " 14 GB", | 20260507 06:23:40.521 | INFO | "pinned" : " 29 GB" | 20260507 06:23:40.522 | INFO | } | 20260507 06:23:40.522 | INFO | Perf report for the last 5.14088 seconds: | 20260507 06:23:40.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:23:40.522 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:40.522 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:40.522 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:40.522 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 06:23:40.522 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:40.522 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:40.522 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 06:23:40.522 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:40.522 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:40.522 | INFO | Uploaded 36GiB to GPU in 1.99071s (18.084GiB/s) | 20260507 06:23:40.522 | INFO | Downloaded 34GiB to GPU in 1.93397s (17.5804GiB/s) | 20260507 06:23:45.656 | INFO | Analyzing chunk frames = [7783424,7783936) | 20260507 06:23:45.656 | INFO | MemInfo: { | 20260507 06:23:45.656 | INFO | "gpu" : " 17 GB", | 20260507 06:23:45.656 | INFO | "huge" : "", | 20260507 06:23:45.656 | INFO | "huge_shared" : " 27 GB", | 20260507 06:23:45.656 | INFO | "malloc" : " 14 GB", | 20260507 06:23:45.656 | INFO | "pinned" : " 29 GB" | 20260507 06:23:45.656 | INFO | } | 20260507 06:23:45.656 | INFO | Perf report for the last 5.1346 seconds: | 20260507 06:23:45.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:23:45.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:45.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:45.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:45.656 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 06:23:45.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:45.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:45.656 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 06:23:45.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:45.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:45.656 | INFO | Uploaded 36GiB to GPU in 2.05149s (17.5483GiB/s) | 20260507 06:23:45.657 | INFO | Downloaded 34GiB to GPU in 1.94919s (17.4431GiB/s) | 20260507 06:23:50.799 | INFO | Analyzing chunk frames = [7783936,7784448) | 20260507 06:23:50.799 | INFO | MemInfo: { | 20260507 06:23:50.799 | INFO | "gpu" : " 17 GB", | 20260507 06:23:50.799 | INFO | "huge" : "", | 20260507 06:23:50.799 | INFO | "huge_shared" : " 27 GB", | 20260507 06:23:50.799 | INFO | "malloc" : " 14 GB", | 20260507 06:23:50.799 | INFO | "pinned" : " 29 GB" | 20260507 06:23:50.799 | INFO | } | 20260507 06:23:50.799 | INFO | Perf report for the last 5.14275 seconds: | 20260507 06:23:50.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:23:50.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:50.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:50.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:50.799 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 06:23:50.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:50.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:50.799 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:23:50.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:50.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:50.799 | INFO | Uploaded 36GiB to GPU in 2.00035s (17.9969GiB/s) | 20260507 06:23:50.799 | INFO | Downloaded 34GiB to GPU in 1.92299s (17.6808GiB/s) | 20260507 06:23:55.942 | INFO | Analyzing chunk frames = [7784448,7784960) | 20260507 06:23:55.942 | INFO | MemInfo: { | 20260507 06:23:55.942 | INFO | "gpu" : " 17 GB", | 20260507 06:23:55.942 | INFO | "huge" : "", | 20260507 06:23:55.942 | INFO | "huge_shared" : " 27 GB", | 20260507 06:23:55.942 | INFO | "malloc" : " 14 GB", | 20260507 06:23:55.942 | INFO | "pinned" : " 29 GB" | 20260507 06:23:55.942 | INFO | } | 20260507 06:23:55.942 | INFO | Perf report for the last 5.143 seconds: | 20260507 06:23:55.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:23:55.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:55.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:55.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:55.942 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 06:23:55.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:55.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:55.942 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:23:55.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:55.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:23:55.942 | INFO | Uploaded 36GiB to GPU in 2.05703s (17.5009GiB/s) | 20260507 06:23:55.942 | INFO | Downloaded 34GiB to GPU in 1.9465s (17.4673GiB/s) | 20260507 06:24:01.078 | INFO | Analyzing chunk frames = [7784960,7785472) | 20260507 06:24:01.079 | INFO | MemInfo: { | 20260507 06:24:01.079 | INFO | "gpu" : " 17 GB", | 20260507 06:24:01.079 | INFO | "huge" : "", | 20260507 06:24:01.079 | INFO | "huge_shared" : " 27 GB", | 20260507 06:24:01.079 | INFO | "malloc" : " 14 GB", | 20260507 06:24:01.079 | INFO | "pinned" : " 29 GB" | 20260507 06:24:01.079 | INFO | } | 20260507 06:24:01.079 | INFO | Perf report for the last 5.13676 seconds: | 20260507 06:24:01.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:24:01.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:01.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:01.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:01.079 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 06:24:01.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:01.079 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:01.079 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:24:01.079 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:01.079 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:01.079 | INFO | Uploaded 36GiB to GPU in 2.07597s (17.3413GiB/s) | 20260507 06:24:01.079 | INFO | Downloaded 34GiB to GPU in 1.94416s (17.4883GiB/s) | 20260507 06:24:06.212 | INFO | Analyzing chunk frames = [7785472,7785984) | 20260507 06:24:06.212 | INFO | MemInfo: { | 20260507 06:24:06.212 | INFO | "gpu" : " 17 GB", | 20260507 06:24:06.212 | INFO | "huge" : "", | 20260507 06:24:06.212 | INFO | "huge_shared" : " 27 GB", | 20260507 06:24:06.212 | INFO | "malloc" : " 14 GB", | 20260507 06:24:06.212 | INFO | "pinned" : " 29 GB" | 20260507 06:24:06.212 | INFO | } | 20260507 06:24:06.212 | INFO | Perf report for the last 5.13369 seconds: | 20260507 06:24:06.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:24:06.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:06.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:06.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:06.212 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 06:24:06.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:06.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:06.213 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:24:06.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:06.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:06.213 | INFO | Uploaded 36GiB to GPU in 1.96198s (18.3488GiB/s) | 20260507 06:24:06.213 | INFO | Downloaded 34GiB to GPU in 1.98307s (17.1451GiB/s) | 20260507 06:24:11.356 | INFO | Analyzing chunk frames = [7785984,7786496) | 20260507 06:24:11.356 | INFO | MemInfo: { | 20260507 06:24:11.356 | INFO | "gpu" : " 17 GB", | 20260507 06:24:11.356 | INFO | "huge" : "", | 20260507 06:24:11.356 | INFO | "huge_shared" : " 27 GB", | 20260507 06:24:11.356 | INFO | "malloc" : " 14 GB", | 20260507 06:24:11.356 | INFO | "pinned" : " 29 GB" | 20260507 06:24:11.356 | INFO | } | 20260507 06:24:11.356 | INFO | Perf report for the last 5.14357 seconds: | 20260507 06:24:11.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:24:11.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:11.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:11.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:11.356 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:24:11.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:11.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:11.356 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:24:11.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:11.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:11.356 | INFO | Uploaded 36GiB to GPU in 1.96104s (18.3576GiB/s) | 20260507 06:24:11.356 | INFO | Downloaded 34GiB to GPU in 1.9624s (17.3258GiB/s) | 20260507 06:24:11.453 | INFO | Starting a round of Baz IO | 20260507 06:24:14.548 | INFO | Populating RTMetrics report | 20260507 06:24:14.548 | INFO | RTMetrics report populated | 20260507 06:24:14.548 | INFO | Opening tmp RTMetrics json file | 20260507 06:24:14.556 | INFO | Serializing RTMetrics to json | 20260507 06:24:14.565 | INFO | Serialization done, swapping tmp file now... | 20260507 06:24:14.565 | INFO | RTMetrics done | 20260507 06:24:14.636 | INFO | Finished a round of Baz IO: Wrote 1.669 GiB in 3.182 seconds | 20260507 06:24:14.636 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.63% / 0.00% / 67.37% | 20260507 06:24:14.636 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.123711% / 0% / 8.74201e-05% | 20260507 06:24:14.636 | INFO | Baz IO overall: | 20260507 06:24:14.636 | INFO | BytesWritten : 3928136144579 | 20260507 06:24:14.636 | INFO | NumEvents : 1882520223127 | 20260507 06:24:16.504 | INFO | Analyzing chunk frames = [7786496,7787008) | 20260507 06:24:16.504 | INFO | MemInfo: { | 20260507 06:24:16.504 | INFO | "gpu" : " 17 GB", | 20260507 06:24:16.504 | INFO | "huge" : "", | 20260507 06:24:16.504 | INFO | "huge_shared" : " 27 GB", | 20260507 06:24:16.505 | INFO | "malloc" : " 14 GB", | 20260507 06:24:16.505 | INFO | "pinned" : " 29 GB" | 20260507 06:24:16.505 | INFO | } | 20260507 06:24:16.505 | INFO | Perf report for the last 5.14864 seconds: | 20260507 06:24:16.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:24:16.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:16.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:16.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:16.505 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 06:24:16.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:16.505 | INFO | RT_METRICS : 41.5%, 1.0, 21.8ms, 0 | 20260507 06:24:16.505 | INFO | PRE_HQ : 58.8%, 1.8, 54.6ms, 0 | 20260507 06:24:16.505 | INFO | BAZWRITER : 61.4%, 3.1, 99.5ms, 0 | 20260507 06:24:16.505 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:24:16.505 | INFO | Uploaded 36GiB to GPU in 2.0232s (17.7936GiB/s) | 20260507 06:24:16.505 | INFO | Downloaded 38GiB to GPU in 2.18362s (17.4023GiB/s) | 20260507 06:24:21.641 | INFO | Analyzing chunk frames = [7787008,7787520) | 20260507 06:24:21.641 | INFO | MemInfo: { | 20260507 06:24:21.641 | INFO | "gpu" : " 17 GB", | 20260507 06:24:21.641 | INFO | "huge" : "", | 20260507 06:24:21.641 | INFO | "huge_shared" : " 27 GB", | 20260507 06:24:21.641 | INFO | "malloc" : " 14 GB", | 20260507 06:24:21.641 | INFO | "pinned" : " 29 GB" | 20260507 06:24:21.641 | INFO | } | 20260507 06:24:21.641 | INFO | Perf report for the last 5.13646 seconds: | 20260507 06:24:21.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:24:21.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:21.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:21.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:21.641 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:24:21.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:21.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:21.641 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:24:21.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:21.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:21.641 | INFO | Uploaded 36GiB to GPU in 1.99495s (18.0456GiB/s) | 20260507 06:24:21.641 | INFO | Downloaded 34GiB to GPU in 1.9382s (17.5421GiB/s) | 20260507 06:24:26.776 | INFO | Analyzing chunk frames = [7787520,7788032) | 20260507 06:24:26.776 | INFO | MemInfo: { | 20260507 06:24:26.776 | INFO | "gpu" : " 17 GB", | 20260507 06:24:26.776 | INFO | "huge" : "", | 20260507 06:24:26.776 | INFO | "huge_shared" : " 27 GB", | 20260507 06:24:26.776 | INFO | "malloc" : " 14 GB", | 20260507 06:24:26.776 | INFO | "pinned" : " 29 GB" | 20260507 06:24:26.776 | INFO | } | 20260507 06:24:26.776 | INFO | Perf report for the last 5.13473 seconds: | 20260507 06:24:26.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:24:26.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:26.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:26.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:26.776 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:24:26.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:26.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:26.776 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:24:26.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:26.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:26.776 | INFO | Uploaded 36GiB to GPU in 2.06038s (17.4725GiB/s) | 20260507 06:24:26.776 | INFO | Downloaded 34GiB to GPU in 1.96276s (17.3225GiB/s) | 20260507 06:24:31.913 | INFO | Analyzing chunk frames = [7788032,7788544) | 20260507 06:24:31.913 | INFO | MemInfo: { | 20260507 06:24:31.913 | INFO | "gpu" : " 17 GB", | 20260507 06:24:31.913 | INFO | "huge" : "", | 20260507 06:24:31.913 | INFO | "huge_shared" : " 27 GB", | 20260507 06:24:31.913 | INFO | "malloc" : " 14 GB", | 20260507 06:24:31.913 | INFO | "pinned" : " 29 GB" | 20260507 06:24:31.913 | INFO | } | 20260507 06:24:31.913 | INFO | Perf report for the last 5.13751 seconds: | 20260507 06:24:31.913 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:24:31.913 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:31.913 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:31.913 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:31.913 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 06:24:31.913 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:31.913 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:31.913 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:24:31.913 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:31.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:31.914 | INFO | Uploaded 36GiB to GPU in 2.01066s (17.9046GiB/s) | 20260507 06:24:31.914 | INFO | Downloaded 34GiB to GPU in 1.94867s (17.4478GiB/s) | 20260507 06:24:37.054 | INFO | Analyzing chunk frames = [7788544,7789056) | 20260507 06:24:37.054 | INFO | MemInfo: { | 20260507 06:24:37.054 | INFO | "gpu" : " 17 GB", | 20260507 06:24:37.054 | INFO | "huge" : "", | 20260507 06:24:37.054 | INFO | "huge_shared" : " 27 GB", | 20260507 06:24:37.054 | INFO | "malloc" : " 14 GB", | 20260507 06:24:37.054 | INFO | "pinned" : " 29 GB" | 20260507 06:24:37.054 | INFO | } | 20260507 06:24:37.054 | INFO | Perf report for the last 5.1409 seconds: | 20260507 06:24:37.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:24:37.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:37.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:37.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:37.054 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 06:24:37.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:37.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:37.054 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:24:37.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:37.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:37.055 | INFO | Uploaded 36GiB to GPU in 2.04986s (17.5622GiB/s) | 20260507 06:24:37.055 | INFO | Downloaded 34GiB to GPU in 1.95024s (17.4337GiB/s) | 20260507 06:24:42.195 | INFO | Analyzing chunk frames = [7789056,7789568) | 20260507 06:24:42.195 | INFO | MemInfo: { | 20260507 06:24:42.195 | INFO | "gpu" : " 17 GB", | 20260507 06:24:42.195 | INFO | "huge" : "", | 20260507 06:24:42.195 | INFO | "huge_shared" : " 27 GB", | 20260507 06:24:42.195 | INFO | "malloc" : " 14 GB", | 20260507 06:24:42.195 | INFO | "pinned" : " 29 GB" | 20260507 06:24:42.195 | INFO | } | 20260507 06:24:42.195 | INFO | Perf report for the last 5.14107 seconds: | 20260507 06:24:42.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:24:42.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:42.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:42.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:42.195 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 06:24:42.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:42.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:42.196 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 06:24:42.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:42.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:42.196 | INFO | Uploaded 36GiB to GPU in 2.02292s (17.7961GiB/s) | 20260507 06:24:42.196 | INFO | Downloaded 34GiB to GPU in 1.91113s (17.7905GiB/s) | 20260507 06:24:47.333 | INFO | Analyzing chunk frames = [7789568,7790080) | 20260507 06:24:47.333 | INFO | MemInfo: { | 20260507 06:24:47.333 | INFO | "gpu" : " 17 GB", | 20260507 06:24:47.333 | INFO | "huge" : "", | 20260507 06:24:47.333 | INFO | "huge_shared" : " 27 GB", | 20260507 06:24:47.333 | INFO | "malloc" : " 14 GB", | 20260507 06:24:47.333 | INFO | "pinned" : " 29 GB" | 20260507 06:24:47.333 | INFO | } | 20260507 06:24:47.333 | INFO | Perf report for the last 5.13726 seconds: | 20260507 06:24:47.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:24:47.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:47.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:47.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:47.333 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:24:47.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:47.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:47.333 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 06:24:47.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:47.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:47.333 | INFO | Uploaded 36GiB to GPU in 1.92419s (18.7091GiB/s) | 20260507 06:24:47.333 | INFO | Downloaded 34GiB to GPU in 1.97227s (17.239GiB/s) | 20260507 06:24:52.472 | INFO | Analyzing chunk frames = [7790080,7790592) | 20260507 06:24:52.472 | INFO | MemInfo: { | 20260507 06:24:52.472 | INFO | "gpu" : " 17 GB", | 20260507 06:24:52.472 | INFO | "huge" : "", | 20260507 06:24:52.472 | INFO | "huge_shared" : " 27 GB", | 20260507 06:24:52.472 | INFO | "malloc" : " 14 GB", | 20260507 06:24:52.472 | INFO | "pinned" : " 29 GB" | 20260507 06:24:52.472 | INFO | } | 20260507 06:24:52.472 | INFO | Perf report for the last 5.13929 seconds: | 20260507 06:24:52.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:24:52.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:52.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:52.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:52.472 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:24:52.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:52.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:52.472 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:24:52.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:52.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:52.472 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:24:52.472 | INFO | Total mem usage: 18 GiB | 20260507 06:24:52.472 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:24:52.472 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:24:52.472 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:24:52.472 | INFO | High Water Breakdown: | 20260507 06:24:52.472 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:24:52.472 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:24:52.472 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:24:52.472 | INFO | ------------------------------------------ | 20260507 06:24:52.472 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:24:52.472 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:24:52.472 | INFO | Total mem usage: 26.5 GiB | 20260507 06:24:52.472 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:24:52.472 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:24:52.472 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:24:52.472 | INFO | High Water Breakdown: | 20260507 06:24:52.472 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:24:52.472 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:24:52.472 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:24:52.472 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:24:52.472 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:24:52.472 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:24:52.472 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:24:52.472 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:24:52.472 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:24:52.472 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:24:52.472 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:24:52.472 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:24:52.472 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:24:52.472 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:24:52.472 | INFO | ------------------------------------------ | 20260507 06:24:52.472 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:24:52.472 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:24:52.472 | INFO | Total mem usage: 25.6 GiB | 20260507 06:24:52.472 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:24:52.472 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:24:52.472 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:24:52.472 | INFO | High Water Breakdown: | 20260507 06:24:52.472 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:24:52.472 | INFO | ------------------------------------------ | 20260507 06:24:52.472 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:24:52.472 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:24:52.472 | INFO | Total mem usage: 15.7 GiB | 20260507 06:24:52.472 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:24:52.472 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:24:52.472 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:24:52.472 | INFO | High Water Breakdown: | 20260507 06:24:52.472 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:24:52.472 | INFO | Uploaded 36GiB to GPU in 1.98629s (18.1242GiB/s) | 20260507 06:24:52.472 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:24:52.473 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:24:52.473 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:24:52.472 | INFO | Downloaded 34GiB to GPU in 1.9564s (17.3788GiB/s) | 20260507 06:24:52.473 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:24:52.473 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:24:52.473 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:24:52.473 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:24:52.473 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:24:52.473 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:24:52.473 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:24:52.473 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:24:52.473 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:24:52.473 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:24:52.473 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:24:52.473 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:24:52.473 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:24:52.473 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:24:52.473 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:24:52.473 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:24:52.473 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:24:52.473 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:24:52.473 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:24:52.473 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:24:52.473 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:24:52.473 | INFO | ------------------------------------------ | 20260507 06:24:52.473 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:24:52.473 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:24:52.473 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:24:52.473 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:24:52.473 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.33e+01 / 1.25e+00 | 20260507 06:24:52.474 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 9.37e-03 / 4.22e-03 | 20260507 06:24:52.474 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.47e+00, 5.04e-01 / 2.18e-01 | 20260507 06:24:52.474 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.49e+00, 1.66e-01 / 1.01e-01 | 20260507 06:24:52.474 | INFO | PerfReport: 16.27%, DME -- 3.99e+01 / 4.47e+01, 3.88e+00 / 1.63e+00 | 20260507 06:24:52.474 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.20e+00 / 1.17e+00 | 20260507 06:24:52.474 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.13e+00 / 2.73e+00, 3.89e-01 / 1.75e-01 | 20260507 06:24:52.474 | INFO | PerfReport: 2.97%, Metrics -- 9.11e-01 / 1.24e+00, 2.44e-01 / 1.55e-01 | 20260507 06:24:52.474 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.06e-01 / 3.64e-01, 5.04e-02 / 1.64e-02 | 20260507 06:24:52.474 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:24:52.474 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:24:52.474 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:24:52.474 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 06:24:52.474 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260507 06:24:52.474 | INFO | PerfReport: 21.43%, Download -- 2.06e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260507 06:24:52.474 | INFO | PerfReport: 31.90%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 06:24:52.474 | INFO | PerfReport: 1.50%, LockUpload -- 1.44e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 06:24:52.474 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 3.04e+00 / 0.00e+00 | 20260507 06:24:52.474 | INFO | PerfReport: 22.04%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 06:24:52.474 | INFO | PerfReport: 0.02%, Misc -- 1.53e-02 / 0.00e+00, 3.85e-02 / 0.00e+00 | 20260507 06:24:52.569 | INFO | Starting a round of Baz IO | 20260507 06:24:55.665 | INFO | Populating RTMetrics report | 20260507 06:24:55.665 | INFO | RTMetrics report populated | 20260507 06:24:55.665 | INFO | Opening tmp RTMetrics json file | 20260507 06:24:55.680 | INFO | Serializing RTMetrics to json | 20260507 06:24:55.681 | INFO | Serialization done, swapping tmp file now... | 20260507 06:24:55.681 | INFO | RTMetrics done | 20260507 06:24:55.751 | INFO | Finished a round of Baz IO: Wrote 1.67 GiB in 3.182 seconds | 20260507 06:24:55.751 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.63% / 0.00% / 67.37% | 20260507 06:24:55.751 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.123493% / 0% / 7.15256e-05% | 20260507 06:24:55.751 | INFO | Baz IO overall: | 20260507 06:24:55.751 | INFO | BytesWritten : 3929929247102 | 20260507 06:24:55.751 | INFO | NumEvents : 1883359984174 | 20260507 06:24:57.610 | INFO | Analyzing chunk frames = [7790592,7791104) | 20260507 06:24:57.610 | INFO | MemInfo: { | 20260507 06:24:57.610 | INFO | "gpu" : " 17 GB", | 20260507 06:24:57.610 | INFO | "huge" : "", | 20260507 06:24:57.610 | INFO | "huge_shared" : " 27 GB", | 20260507 06:24:57.610 | INFO | "malloc" : " 14 GB", | 20260507 06:24:57.610 | INFO | "pinned" : " 29 GB" | 20260507 06:24:57.610 | INFO | } | 20260507 06:24:57.610 | INFO | Perf report for the last 5.13836 seconds: | 20260507 06:24:57.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:24:57.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:57.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:57.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:57.611 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 06:24:57.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:24:57.611 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 06:24:57.611 | INFO | PRE_HQ : 58.9%, 1.8, 54.4ms, 0 | 20260507 06:24:57.611 | INFO | BAZWRITER : 61.5%, 3.2, 99.6ms, 0 | 20260507 06:24:57.611 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:24:57.611 | INFO | Uploaded 36GiB to GPU in 2.05506s (17.5177GiB/s) | 20260507 06:24:57.611 | INFO | Downloaded 38GiB to GPU in 2.17191s (17.4961GiB/s) | 20260507 06:25:02.747 | INFO | Analyzing chunk frames = [7791104,7791616) | 20260507 06:25:02.747 | INFO | MemInfo: { | 20260507 06:25:02.747 | INFO | "gpu" : " 17 GB", | 20260507 06:25:02.747 | INFO | "huge" : "", | 20260507 06:25:02.747 | INFO | "huge_shared" : " 27 GB", | 20260507 06:25:02.747 | INFO | "malloc" : " 14 GB", | 20260507 06:25:02.747 | INFO | "pinned" : " 29 GB" | 20260507 06:25:02.747 | INFO | } | 20260507 06:25:02.747 | INFO | Perf report for the last 5.13714 seconds: | 20260507 06:25:02.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:25:02.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:02.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:02.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:02.747 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 06:25:02.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:02.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:02.748 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:25:02.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:02.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:02.748 | INFO | Uploaded 36GiB to GPU in 2.05173s (17.5461GiB/s) | 20260507 06:25:02.748 | INFO | Downloaded 34GiB to GPU in 1.94046s (17.5217GiB/s) | 20260507 06:25:07.891 | INFO | Analyzing chunk frames = [7791616,7792128) | 20260507 06:25:07.891 | INFO | MemInfo: { | 20260507 06:25:07.891 | INFO | "gpu" : " 17 GB", | 20260507 06:25:07.891 | INFO | "huge" : "", | 20260507 06:25:07.891 | INFO | "huge_shared" : " 27 GB", | 20260507 06:25:07.891 | INFO | "malloc" : " 14 GB", | 20260507 06:25:07.891 | INFO | "pinned" : " 29 GB" | 20260507 06:25:07.891 | INFO | } | 20260507 06:25:07.891 | INFO | Perf report for the last 5.14391 seconds: | 20260507 06:25:07.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:25:07.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:07.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:07.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:07.891 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 06:25:07.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:07.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:07.891 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:25:07.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:07.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:07.892 | INFO | Uploaded 36GiB to GPU in 2.0448s (17.6056GiB/s) | 20260507 06:25:07.892 | INFO | Downloaded 34GiB to GPU in 1.95037s (17.4326GiB/s) | 20260507 06:25:13.026 | INFO | Analyzing chunk frames = [7792128,7792640) | 20260507 06:25:13.026 | INFO | MemInfo: { | 20260507 06:25:13.026 | INFO | "gpu" : " 17 GB", | 20260507 06:25:13.026 | INFO | "huge" : "", | 20260507 06:25:13.026 | INFO | "huge_shared" : " 27 GB", | 20260507 06:25:13.026 | INFO | "malloc" : " 14 GB", | 20260507 06:25:13.026 | INFO | "pinned" : " 29 GB" | 20260507 06:25:13.026 | INFO | } | 20260507 06:25:13.026 | INFO | Perf report for the last 5.13447 seconds: | 20260507 06:25:13.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:25:13.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:13.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:13.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:13.026 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 06:25:13.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:13.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:13.026 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:25:13.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:13.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:13.026 | INFO | Uploaded 36GiB to GPU in 2.05107s (17.5518GiB/s) | 20260507 06:25:13.026 | INFO | Downloaded 34GiB to GPU in 1.94827s (17.4514GiB/s) | 20260507 06:25:18.167 | INFO | Analyzing chunk frames = [7792640,7793152) | 20260507 06:25:18.167 | INFO | MemInfo: { | 20260507 06:25:18.167 | INFO | "gpu" : " 17 GB", | 20260507 06:25:18.167 | INFO | "huge" : "", | 20260507 06:25:18.167 | INFO | "huge_shared" : " 27 GB", | 20260507 06:25:18.167 | INFO | "malloc" : " 14 GB", | 20260507 06:25:18.167 | INFO | "pinned" : " 29 GB" | 20260507 06:25:18.167 | INFO | } | 20260507 06:25:18.168 | INFO | Perf report for the last 5.14174 seconds: | 20260507 06:25:18.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:25:18.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:18.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:18.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:18.168 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 06:25:18.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:18.168 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:18.168 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:25:18.168 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:18.168 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:18.168 | INFO | Uploaded 36GiB to GPU in 2.05902s (17.4841GiB/s) | 20260507 06:25:18.168 | INFO | Downloaded 34GiB to GPU in 1.95334s (17.4061GiB/s) | 20260507 06:25:23.309 | INFO | Analyzing chunk frames = [7793152,7793664) | 20260507 06:25:23.309 | INFO | MemInfo: { | 20260507 06:25:23.309 | INFO | "gpu" : " 17 GB", | 20260507 06:25:23.309 | INFO | "huge" : "", | 20260507 06:25:23.309 | INFO | "huge_shared" : " 27 GB", | 20260507 06:25:23.309 | INFO | "malloc" : " 14 GB", | 20260507 06:25:23.309 | INFO | "pinned" : " 29 GB" | 20260507 06:25:23.309 | INFO | } | 20260507 06:25:23.309 | INFO | Perf report for the last 5.14116 seconds: | 20260507 06:25:23.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:25:23.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:23.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:23.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:23.309 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 06:25:23.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:23.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:23.309 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 06:25:23.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:23.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:23.309 | INFO | Uploaded 36GiB to GPU in 2.03465s (17.6935GiB/s) | 20260507 06:25:23.309 | INFO | Downloaded 34GiB to GPU in 1.89646s (17.9282GiB/s) | 20260507 06:25:28.460 | INFO | Analyzing chunk frames = [7793664,7794176) | 20260507 06:25:28.460 | INFO | MemInfo: { | 20260507 06:25:28.460 | INFO | "gpu" : " 17 GB", | 20260507 06:25:28.460 | INFO | "huge" : "", | 20260507 06:25:28.460 | INFO | "huge_shared" : " 27 GB", | 20260507 06:25:28.460 | INFO | "malloc" : " 14 GB", | 20260507 06:25:28.460 | INFO | "pinned" : " 29 GB" | 20260507 06:25:28.460 | INFO | } | 20260507 06:25:28.460 | INFO | Perf report for the last 5.1513 seconds: | 20260507 06:25:28.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:25:28.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:28.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:28.460 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:28.460 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260507 06:25:28.460 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:28.460 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:28.460 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:25:28.460 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:28.460 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:28.461 | INFO | Uploaded 36GiB to GPU in 2.03283s (17.7093GiB/s) | 20260507 06:25:28.461 | INFO | Downloaded 34GiB to GPU in 1.98773s (17.1049GiB/s) | 20260507 06:25:33.593 | INFO | Analyzing chunk frames = [7794176,7794688) | 20260507 06:25:33.593 | INFO | MemInfo: { | 20260507 06:25:33.593 | INFO | "gpu" : " 17 GB", | 20260507 06:25:33.593 | INFO | "huge" : "", | 20260507 06:25:33.593 | INFO | "huge_shared" : " 27 GB", | 20260507 06:25:33.593 | INFO | "malloc" : " 14 GB", | 20260507 06:25:33.593 | INFO | "pinned" : " 29 GB" | 20260507 06:25:33.593 | INFO | } | 20260507 06:25:33.593 | INFO | Perf report for the last 5.13274 seconds: | 20260507 06:25:33.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:25:33.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:33.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:33.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:33.593 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:25:33.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:33.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:33.593 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 06:25:33.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:33.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:33.593 | INFO | Uploaded 36GiB to GPU in 1.99594s (18.0366GiB/s) | 20260507 06:25:33.593 | INFO | Downloaded 34GiB to GPU in 1.96748s (17.281GiB/s) | 20260507 06:25:33.690 | INFO | Starting a round of Baz IO | 20260507 06:25:36.789 | INFO | Populating RTMetrics report | 20260507 06:25:36.789 | INFO | RTMetrics report populated | 20260507 06:25:36.789 | INFO | Opening tmp RTMetrics json file | 20260507 06:25:36.797 | INFO | Serializing RTMetrics to json | 20260507 06:25:36.805 | INFO | Serialization done, swapping tmp file now... | 20260507 06:25:36.806 | INFO | RTMetrics done | 20260507 06:25:36.878 | INFO | Finished a round of Baz IO: Wrote 1.67 GiB in 3.188 seconds | 20260507 06:25:36.878 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.63% / 0.00% / 67.37% | 20260507 06:25:36.878 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.123366% / 3.97364e-06% / 8.34465e-05% | 20260507 06:25:36.878 | INFO | Baz IO overall: | 20260507 06:25:36.878 | INFO | BytesWritten : 3931722240875 | 20260507 06:25:36.879 | INFO | NumEvents : 1884199427414 | 20260507 06:25:38.722 | INFO | Analyzing chunk frames = [7794688,7795200) | 20260507 06:25:38.722 | INFO | MemInfo: { | 20260507 06:25:38.722 | INFO | "gpu" : " 17 GB", | 20260507 06:25:38.722 | INFO | "huge" : "", | 20260507 06:25:38.722 | INFO | "huge_shared" : " 27 GB", | 20260507 06:25:38.722 | INFO | "malloc" : " 14 GB", | 20260507 06:25:38.722 | INFO | "pinned" : " 29 GB" | 20260507 06:25:38.722 | INFO | } | 20260507 06:25:38.722 | INFO | Perf report for the last 5.12939 seconds: | 20260507 06:25:38.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:25:38.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:38.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:38.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:38.722 | INFO | ANALYSIS : 61.8%, 3.0, 96.0ms, 0 | 20260507 06:25:38.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:38.722 | INFO | RT_METRICS : 44.0%, 1.0, 23.0ms, 0 | 20260507 06:25:38.722 | INFO | PRE_HQ : 59.8%, 1.9, 58.7ms, 0 | 20260507 06:25:38.722 | INFO | BAZWRITER : 61.7%, 3.3, 103.8ms, 0 | 20260507 06:25:38.722 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 06:25:38.722 | INFO | Uploaded 36GiB to GPU in 2.05294s (17.5358GiB/s) | 20260507 06:25:38.722 | INFO | Downloaded 38GiB to GPU in 2.17203s (17.4951GiB/s) | 20260507 06:25:43.864 | INFO | Analyzing chunk frames = [7795200,7795712) | 20260507 06:25:43.864 | INFO | MemInfo: { | 20260507 06:25:43.864 | INFO | "gpu" : " 17 GB", | 20260507 06:25:43.864 | INFO | "huge" : "", | 20260507 06:25:43.864 | INFO | "huge_shared" : " 27 GB", | 20260507 06:25:43.864 | INFO | "malloc" : " 14 GB", | 20260507 06:25:43.864 | INFO | "pinned" : " 29 GB" | 20260507 06:25:43.864 | INFO | } | 20260507 06:25:43.864 | INFO | Perf report for the last 5.14211 seconds: | 20260507 06:25:43.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:25:43.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:43.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:43.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:43.865 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 06:25:43.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:43.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:43.865 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:25:43.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:43.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:43.865 | INFO | Uploaded 36GiB to GPU in 2.04504s (17.6036GiB/s) | 20260507 06:25:43.865 | INFO | Downloaded 34GiB to GPU in 1.94524s (17.4786GiB/s) | 20260507 06:25:49.004 | INFO | Analyzing chunk frames = [7795712,7796224) | 20260507 06:25:49.004 | INFO | MemInfo: { | 20260507 06:25:49.004 | INFO | "gpu" : " 17 GB", | 20260507 06:25:49.004 | INFO | "huge" : "", | 20260507 06:25:49.004 | INFO | "huge_shared" : " 27 GB", | 20260507 06:25:49.004 | INFO | "malloc" : " 14 GB", | 20260507 06:25:49.004 | INFO | "pinned" : " 29 GB" | 20260507 06:25:49.004 | INFO | } | 20260507 06:25:49.004 | INFO | Perf report for the last 5.1392 seconds: | 20260507 06:25:49.004 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:25:49.004 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:49.004 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:49.004 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:49.004 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 06:25:49.004 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:49.004 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:49.004 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:25:49.004 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:49.004 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:49.004 | INFO | Uploaded 36GiB to GPU in 2.00218s (17.9804GiB/s) | 20260507 06:25:49.004 | INFO | Downloaded 34GiB to GPU in 1.94852s (17.4492GiB/s) | 20260507 06:25:54.144 | INFO | Analyzing chunk frames = [7796224,7796736) | 20260507 06:25:54.144 | INFO | MemInfo: { | 20260507 06:25:54.144 | INFO | "gpu" : " 17 GB", | 20260507 06:25:54.144 | INFO | "huge" : "", | 20260507 06:25:54.144 | INFO | "huge_shared" : " 27 GB", | 20260507 06:25:54.144 | INFO | "malloc" : " 14 GB", | 20260507 06:25:54.144 | INFO | "pinned" : " 29 GB" | 20260507 06:25:54.144 | INFO | } | 20260507 06:25:54.144 | INFO | Perf report for the last 5.14051 seconds: | 20260507 06:25:54.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:25:54.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:54.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:54.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:54.144 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 06:25:54.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:54.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:54.144 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:25:54.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:54.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:54.144 | INFO | Uploaded 36GiB to GPU in 2.0218s (17.8059GiB/s) | 20260507 06:25:54.144 | INFO | Downloaded 34GiB to GPU in 1.94004s (17.5254GiB/s) | 20260507 06:25:59.285 | INFO | Analyzing chunk frames = [7796736,7797248) | 20260507 06:25:59.285 | INFO | MemInfo: { | 20260507 06:25:59.285 | INFO | "gpu" : " 17 GB", | 20260507 06:25:59.285 | INFO | "huge" : "", | 20260507 06:25:59.285 | INFO | "huge_shared" : " 27 GB", | 20260507 06:25:59.285 | INFO | "malloc" : " 14 GB", | 20260507 06:25:59.285 | INFO | "pinned" : " 29 GB" | 20260507 06:25:59.285 | INFO | } | 20260507 06:25:59.286 | INFO | Perf report for the last 5.14138 seconds: | 20260507 06:25:59.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:25:59.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:59.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:59.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:59.286 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 06:25:59.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:59.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:59.286 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:25:59.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:59.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:25:59.286 | INFO | Uploaded 36GiB to GPU in 2.03321s (17.706GiB/s) | 20260507 06:25:59.286 | INFO | Downloaded 34GiB to GPU in 1.92302s (17.6805GiB/s) | 20260507 06:26:04.422 | INFO | Analyzing chunk frames = [7797248,7797760) | 20260507 06:26:04.422 | INFO | MemInfo: { | 20260507 06:26:04.422 | INFO | "gpu" : " 17 GB", | 20260507 06:26:04.422 | INFO | "huge" : "", | 20260507 06:26:04.422 | INFO | "huge_shared" : " 27 GB", | 20260507 06:26:04.422 | INFO | "malloc" : " 14 GB", | 20260507 06:26:04.422 | INFO | "pinned" : " 29 GB" | 20260507 06:26:04.422 | INFO | } | 20260507 06:26:04.422 | INFO | Perf report for the last 5.13632 seconds: | 20260507 06:26:04.422 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:26:04.422 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:04.422 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:04.422 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:04.422 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 06:26:04.422 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:04.422 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:04.422 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:26:04.422 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:04.422 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:04.422 | INFO | Uploaded 36GiB to GPU in 2.06221s (17.457GiB/s) | 20260507 06:26:04.422 | INFO | Downloaded 34GiB to GPU in 1.93364s (17.5834GiB/s) | 20260507 06:26:09.562 | INFO | Analyzing chunk frames = [7797760,7798272) | 20260507 06:26:09.562 | INFO | MemInfo: { | 20260507 06:26:09.562 | INFO | "gpu" : " 17 GB", | 20260507 06:26:09.562 | INFO | "huge" : "", | 20260507 06:26:09.562 | INFO | "huge_shared" : " 27 GB", | 20260507 06:26:09.562 | INFO | "malloc" : " 14 GB", | 20260507 06:26:09.562 | INFO | "pinned" : " 29 GB" | 20260507 06:26:09.562 | INFO | } | 20260507 06:26:09.562 | INFO | Perf report for the last 5.14022 seconds: | 20260507 06:26:09.562 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:26:09.562 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:09.562 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:09.562 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:09.562 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:26:09.562 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:09.562 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:09.562 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:26:09.562 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:09.562 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:09.562 | INFO | Uploaded 36GiB to GPU in 1.99816s (18.0166GiB/s) | 20260507 06:26:09.562 | INFO | Downloaded 34GiB to GPU in 1.98972s (17.0878GiB/s) | 20260507 06:26:14.708 | INFO | Analyzing chunk frames = [7798272,7798784) | 20260507 06:26:14.708 | INFO | MemInfo: { | 20260507 06:26:14.708 | INFO | "gpu" : " 17 GB", | 20260507 06:26:14.708 | INFO | "huge" : "", | 20260507 06:26:14.708 | INFO | "huge_shared" : " 27 GB", | 20260507 06:26:14.708 | INFO | "malloc" : " 14 GB", | 20260507 06:26:14.708 | INFO | "pinned" : " 29 GB" | 20260507 06:26:14.708 | INFO | } | 20260507 06:26:14.709 | INFO | Perf report for the last 5.14649 seconds: | 20260507 06:26:14.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:26:14.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:14.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:14.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:14.709 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 06:26:14.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:14.709 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:14.709 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 06:26:14.709 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:14.709 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:14.709 | INFO | Uploaded 36GiB to GPU in 1.96355s (18.3341GiB/s) | 20260507 06:26:14.709 | INFO | Downloaded 34GiB to GPU in 1.96304s (17.32GiB/s) | 20260507 06:26:14.709 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:26:14.709 | INFO | Total mem usage: 18 GiB | 20260507 06:26:14.709 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:26:14.709 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 06:26:14.709 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:26:14.709 | INFO | High Water Breakdown: | 20260507 06:26:14.709 | INFO | 60.1% : 8.56 GiB ( 75% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:26:14.709 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:26:14.709 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:26:14.709 | INFO | ------------------------------------------ | 20260507 06:26:14.709 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:26:14.709 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:26:14.709 | INFO | Total mem usage: 26.5 GiB | 20260507 06:26:14.709 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:26:14.709 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:26:14.709 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:26:14.709 | INFO | High Water Breakdown: | 20260507 06:26:14.709 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:26:14.709 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:26:14.709 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:26:14.709 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:26:14.709 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:26:14.709 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:26:14.709 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:26:14.709 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:26:14.709 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:26:14.709 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:26:14.709 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:26:14.709 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:26:14.709 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:26:14.709 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:26:14.709 | INFO | ------------------------------------------ | 20260507 06:26:14.709 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:26:14.709 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:26:14.709 | INFO | Total mem usage: 25.6 GiB | 20260507 06:26:14.710 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:26:14.710 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:26:14.710 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:26:14.710 | INFO | High Water Breakdown: | 20260507 06:26:14.710 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:26:14.710 | INFO | ------------------------------------------ | 20260507 06:26:14.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:26:14.710 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:26:14.710 | INFO | Total mem usage: 15.7 GiB | 20260507 06:26:14.710 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:26:14.710 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:26:14.710 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:26:14.710 | INFO | High Water Breakdown: | 20260507 06:26:14.710 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:26:14.710 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:26:14.710 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:26:14.710 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:26:14.710 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:26:14.710 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:26:14.710 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:26:14.710 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:26:14.710 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:26:14.710 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:26:14.710 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:26:14.710 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:26:14.710 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:26:14.710 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:26:14.710 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:26:14.710 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:26:14.710 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:26:14.710 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:26:14.710 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:26:14.710 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:26:14.710 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:26:14.710 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:26:14.710 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:26:14.710 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:26:14.710 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:26:14.710 | INFO | ------------------------------------------ | 20260507 06:26:14.710 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:26:14.710 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:26:14.710 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:26:14.710 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:26:14.710 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.43e+01, 1.34e+01 / 1.45e+00 | 20260507 06:26:14.710 | INFO | PerfReport: 0.30%, AggregateStats -- 9.05e-02 / 1.01e-01, 9.70e-03 / 6.40e-03 | 20260507 06:26:14.710 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.45e+00, 4.93e-01 / 2.55e-01 | 20260507 06:26:14.710 | INFO | PerfReport: 3.20%, Histogram -- 9.81e-01 / 1.47e+00, 1.59e-01 / 1.16e-01 | 20260507 06:26:14.710 | INFO | PerfReport: 16.22%, DME -- 3.98e+01 / 4.42e+01, 3.72e+00 / 1.82e+00 | 20260507 06:26:14.710 | INFO | PerfReport: 57.73%, FrameLabeling -- 1.77e+01 / 1.98e+01, 1.24e+00 / 1.66e+00 | 20260507 06:26:14.710 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.68e+00, 3.86e-01 / 2.19e-01 | 20260507 06:26:14.710 | INFO | PerfReport: 2.96%, Metrics -- 9.08e-01 / 1.20e+00, 2.42e-01 / 1.20e-01 | 20260507 06:26:14.710 | INFO | PerfReport: 1.00%, PulseSerializing -- 3.06e-01 / 3.62e-01, 5.07e-02 / 2.22e-02 | 20260507 06:26:14.710 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:26:14.710 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:26:14.710 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:26:14.710 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 06:26:14.710 | INFO | PerfReport: 22.00%, Upload -- 2.12e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 06:26:14.710 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.38e+00 / 0.00e+00 | 20260507 06:26:14.710 | INFO | PerfReport: 31.85%, Compute -- 3.06e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:26:14.710 | INFO | PerfReport: 1.58%, LockUpload -- 1.52e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 06:26:14.710 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260507 06:26:14.710 | INFO | PerfReport: 21.93%, LockCompute -- 2.11e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 06:26:14.710 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.97e-02 / 0.00e+00 | 20260507 06:26:14.806 | INFO | Starting a round of Baz IO | 20260507 06:26:17.896 | INFO | Populating RTMetrics report | 20260507 06:26:17.896 | INFO | RTMetrics report populated | 20260507 06:26:17.896 | INFO | Opening tmp RTMetrics json file | 20260507 06:26:17.913 | INFO | Serializing RTMetrics to json | 20260507 06:26:17.915 | INFO | Serialization done, swapping tmp file now... | 20260507 06:26:17.915 | INFO | RTMetrics done | 20260507 06:26:17.983 | INFO | Finished a round of Baz IO: Wrote 1.671 GiB in 3.177 seconds | 20260507 06:26:17.983 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.63% / 0.00% / 67.37% | 20260507 06:26:17.983 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.123044% / 0% / 8.74201e-05% | 20260507 06:26:17.983 | INFO | Baz IO overall: | 20260507 06:26:17.983 | INFO | BytesWritten : 3933516178347 | 20260507 06:26:17.983 | INFO | NumEvents : 1885039969078 | 20260507 06:26:19.839 | INFO | Analyzing chunk frames = [7798784,7799296) | 20260507 06:26:19.839 | INFO | MemInfo: { | 20260507 06:26:19.839 | INFO | "gpu" : " 17 GB", | 20260507 06:26:19.839 | INFO | "huge" : "", | 20260507 06:26:19.839 | INFO | "huge_shared" : " 27 GB", | 20260507 06:26:19.839 | INFO | "malloc" : " 14 GB", | 20260507 06:26:19.839 | INFO | "pinned" : " 29 GB" | 20260507 06:26:19.839 | INFO | } | 20260507 06:26:19.839 | INFO | Perf report for the last 5.13035 seconds: | 20260507 06:26:19.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:26:19.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:19.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:19.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:19.839 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 06:26:19.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:19.839 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260507 06:26:19.839 | INFO | PRE_HQ : 59.6%, 1.9, 58.6ms, 0 | 20260507 06:26:19.839 | INFO | BAZWRITER : 61.5%, 3.3, 103.1ms, 0 | 20260507 06:26:19.839 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:26:19.839 | INFO | Uploaded 36GiB to GPU in 2.05868s (17.4869GiB/s) | 20260507 06:26:19.839 | INFO | Downloaded 38GiB to GPU in 2.17541s (17.468GiB/s) | 20260507 06:26:24.974 | INFO | Analyzing chunk frames = [7799296,7799808) | 20260507 06:26:24.974 | INFO | MemInfo: { | 20260507 06:26:24.974 | INFO | "gpu" : " 17 GB", | 20260507 06:26:24.974 | INFO | "huge" : "", | 20260507 06:26:24.974 | INFO | "huge_shared" : " 27 GB", | 20260507 06:26:24.974 | INFO | "malloc" : " 14 GB", | 20260507 06:26:24.974 | INFO | "pinned" : " 29 GB" | 20260507 06:26:24.974 | INFO | } | 20260507 06:26:24.974 | INFO | Perf report for the last 5.13536 seconds: | 20260507 06:26:24.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:26:24.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:24.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:24.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:24.974 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 06:26:24.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:24.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:24.974 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:26:24.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:24.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:24.975 | INFO | Uploaded 36GiB to GPU in 1.97085s (18.2662GiB/s) | 20260507 06:26:24.975 | INFO | Downloaded 34GiB to GPU in 1.92615s (17.6518GiB/s) | 20260507 06:26:30.119 | INFO | Analyzing chunk frames = [7799808,7800320) | 20260507 06:26:30.119 | INFO | MemInfo: { | 20260507 06:26:30.119 | INFO | "gpu" : " 17 GB", | 20260507 06:26:30.119 | INFO | "huge" : "", | 20260507 06:26:30.119 | INFO | "huge_shared" : " 27 GB", | 20260507 06:26:30.119 | INFO | "malloc" : " 14 GB", | 20260507 06:26:30.119 | INFO | "pinned" : " 29 GB" | 20260507 06:26:30.119 | INFO | } | 20260507 06:26:30.119 | INFO | Perf report for the last 5.14467 seconds: | 20260507 06:26:30.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:26:30.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:30.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:30.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:30.119 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 06:26:30.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:30.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:30.119 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:26:30.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:30.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:30.119 | INFO | Uploaded 36GiB to GPU in 2.00431s (17.9613GiB/s) | 20260507 06:26:30.119 | INFO | Downloaded 34GiB to GPU in 1.94514s (17.4795GiB/s) | 20260507 06:26:35.259 | INFO | Analyzing chunk frames = [7800320,7800832) | 20260507 06:26:35.259 | INFO | MemInfo: { | 20260507 06:26:35.259 | INFO | "gpu" : " 17 GB", | 20260507 06:26:35.259 | INFO | "huge" : "", | 20260507 06:26:35.259 | INFO | "huge_shared" : " 27 GB", | 20260507 06:26:35.259 | INFO | "malloc" : " 14 GB", | 20260507 06:26:35.259 | INFO | "pinned" : " 29 GB" | 20260507 06:26:35.259 | INFO | } | 20260507 06:26:35.259 | INFO | Perf report for the last 5.14041 seconds: | 20260507 06:26:35.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:26:35.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:35.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:35.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:35.259 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 06:26:35.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:35.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:35.260 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:26:35.260 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:35.260 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:35.260 | INFO | Uploaded 36GiB to GPU in 1.9971s (18.0261GiB/s) | 20260507 06:26:35.260 | INFO | Downloaded 34GiB to GPU in 1.93688s (17.554GiB/s) | 20260507 06:26:40.404 | INFO | Analyzing chunk frames = [7800832,7801344) | 20260507 06:26:40.404 | INFO | MemInfo: { | 20260507 06:26:40.404 | INFO | "gpu" : " 17 GB", | 20260507 06:26:40.404 | INFO | "huge" : "", | 20260507 06:26:40.404 | INFO | "huge_shared" : " 27 GB", | 20260507 06:26:40.404 | INFO | "malloc" : " 14 GB", | 20260507 06:26:40.404 | INFO | "pinned" : " 29 GB" | 20260507 06:26:40.404 | INFO | } | 20260507 06:26:40.404 | INFO | Perf report for the last 5.14505 seconds: | 20260507 06:26:40.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:26:40.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:40.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:40.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:40.404 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 06:26:40.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:40.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:40.404 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:26:40.404 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:40.404 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:40.405 | INFO | Uploaded 36GiB to GPU in 2.02945s (17.7388GiB/s) | 20260507 06:26:40.406 | INFO | Downloaded 34GiB to GPU in 1.9188s (17.7194GiB/s) | 20260507 06:26:45.537 | INFO | Analyzing chunk frames = [7801344,7801856) | 20260507 06:26:45.537 | INFO | MemInfo: { | 20260507 06:26:45.537 | INFO | "gpu" : " 17 GB", | 20260507 06:26:45.538 | INFO | "huge" : "", | 20260507 06:26:45.538 | INFO | "huge_shared" : " 27 GB", | 20260507 06:26:45.538 | INFO | "malloc" : " 14 GB", | 20260507 06:26:45.538 | INFO | "pinned" : " 29 GB" | 20260507 06:26:45.538 | INFO | } | 20260507 06:26:45.538 | INFO | Perf report for the last 5.13307 seconds: | 20260507 06:26:45.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:26:45.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:45.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:45.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:45.538 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 06:26:45.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:45.538 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:45.538 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 06:26:45.538 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:45.538 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:45.538 | INFO | Uploaded 36GiB to GPU in 2.02449s (17.7822GiB/s) | 20260507 06:26:45.538 | INFO | Downloaded 34GiB to GPU in 1.9175s (17.7314GiB/s) | 20260507 06:26:50.678 | INFO | Analyzing chunk frames = [7801856,7802368) | 20260507 06:26:50.678 | INFO | MemInfo: { | 20260507 06:26:50.678 | INFO | "gpu" : " 17 GB", | 20260507 06:26:50.678 | INFO | "huge" : "", | 20260507 06:26:50.678 | INFO | "huge_shared" : " 27 GB", | 20260507 06:26:50.678 | INFO | "malloc" : " 14 GB", | 20260507 06:26:50.678 | INFO | "pinned" : " 29 GB" | 20260507 06:26:50.678 | INFO | } | 20260507 06:26:50.678 | INFO | Perf report for the last 5.14049 seconds: | 20260507 06:26:50.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:26:50.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:50.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:50.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:50.678 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 06:26:50.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:50.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:50.678 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 06:26:50.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:50.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:50.678 | INFO | Uploaded 36GiB to GPU in 2.03046s (17.73GiB/s) | 20260507 06:26:50.678 | INFO | Downloaded 34GiB to GPU in 1.98595s (17.1203GiB/s) | 20260507 06:26:55.814 | INFO | Analyzing chunk frames = [7802368,7802880) | 20260507 06:26:55.814 | INFO | MemInfo: { | 20260507 06:26:55.814 | INFO | "gpu" : " 17 GB", | 20260507 06:26:55.814 | INFO | "huge" : "", | 20260507 06:26:55.814 | INFO | "huge_shared" : " 27 GB", | 20260507 06:26:55.814 | INFO | "malloc" : " 14 GB", | 20260507 06:26:55.814 | INFO | "pinned" : " 29 GB" | 20260507 06:26:55.814 | INFO | } | 20260507 06:26:55.814 | INFO | Perf report for the last 5.13599 seconds: | 20260507 06:26:55.814 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:26:55.814 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:55.814 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:55.814 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:55.814 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:26:55.814 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:55.814 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:55.814 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 06:26:55.814 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:55.814 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:26:55.815 | INFO | Uploaded 36GiB to GPU in 1.98643s (18.123GiB/s) | 20260507 06:26:55.815 | INFO | Downloaded 34GiB to GPU in 1.96046s (17.3429GiB/s) | 20260507 06:26:55.912 | INFO | Starting a round of Baz IO | 20260507 06:26:59.041 | INFO | Populating RTMetrics report | 20260507 06:26:59.042 | INFO | RTMetrics report populated | 20260507 06:26:59.042 | INFO | Opening tmp RTMetrics json file | 20260507 06:26:59.042 | INFO | Serializing RTMetrics to json | 20260507 06:26:59.043 | INFO | Serialization done, swapping tmp file now... | 20260507 06:26:59.044 | INFO | RTMetrics done | 20260507 06:26:59.134 | INFO | Finished a round of Baz IO: Wrote 1.669 GiB in 3.222 seconds | 20260507 06:26:59.134 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.63% / 0.00% / 67.37% | 20260507 06:26:59.134 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.122718% / 0% / 7.94729e-05% | 20260507 06:26:59.134 | INFO | Baz IO overall: | 20260507 06:26:59.134 | INFO | BytesWritten : 3935308212648 | 20260507 06:26:59.134 | INFO | NumEvents : 1885878832112 | 20260507 06:27:00.959 | INFO | Analyzing chunk frames = [7802880,7803392) | 20260507 06:27:00.959 | INFO | MemInfo: { | 20260507 06:27:00.959 | INFO | "gpu" : " 17 GB", | 20260507 06:27:00.959 | INFO | "huge" : "", | 20260507 06:27:00.959 | INFO | "huge_shared" : " 27 GB", | 20260507 06:27:00.959 | INFO | "malloc" : " 14 GB", | 20260507 06:27:00.959 | INFO | "pinned" : " 29 GB" | 20260507 06:27:00.959 | INFO | } | 20260507 06:27:00.959 | INFO | Perf report for the last 5.14475 seconds: | 20260507 06:27:00.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:27:00.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:00.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:00.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:00.959 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260507 06:27:00.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:00.959 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260507 06:27:00.959 | INFO | PRE_HQ : 60.1%, 1.9, 60.4ms, 0 | 20260507 06:27:00.959 | INFO | BAZWRITER : 62.2%, 3.3, 106.7ms, 0 | 20260507 06:27:00.959 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 06:27:00.959 | INFO | Uploaded 36GiB to GPU in 2.04631s (17.5927GiB/s) | 20260507 06:27:00.959 | INFO | Downloaded 38GiB to GPU in 2.17563s (17.4662GiB/s) | 20260507 06:27:06.095 | INFO | Analyzing chunk frames = [7803392,7803904) | 20260507 06:27:06.095 | INFO | MemInfo: { | 20260507 06:27:06.095 | INFO | "gpu" : " 17 GB", | 20260507 06:27:06.095 | INFO | "huge" : "", | 20260507 06:27:06.095 | INFO | "huge_shared" : " 27 GB", | 20260507 06:27:06.095 | INFO | "malloc" : " 14 GB", | 20260507 06:27:06.095 | INFO | "pinned" : " 29 GB" | 20260507 06:27:06.095 | INFO | } | 20260507 06:27:06.095 | INFO | Perf report for the last 5.13645 seconds: | 20260507 06:27:06.095 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:27:06.095 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:06.095 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:06.095 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:06.095 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:27:06.095 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:06.095 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:06.095 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:27:06.095 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:06.095 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:06.095 | INFO | Uploaded 36GiB to GPU in 1.98949s (18.095GiB/s) | 20260507 06:27:06.096 | INFO | Downloaded 34GiB to GPU in 1.9244s (17.6678GiB/s) | 20260507 06:27:11.230 | INFO | Analyzing chunk frames = [7803904,7804416) | 20260507 06:27:11.230 | INFO | MemInfo: { | 20260507 06:27:11.230 | INFO | "gpu" : " 17 GB", | 20260507 06:27:11.230 | INFO | "huge" : "", | 20260507 06:27:11.230 | INFO | "huge_shared" : " 27 GB", | 20260507 06:27:11.230 | INFO | "malloc" : " 14 GB", | 20260507 06:27:11.230 | INFO | "pinned" : " 29 GB" | 20260507 06:27:11.230 | INFO | } | 20260507 06:27:11.230 | INFO | Perf report for the last 5.13439 seconds: | 20260507 06:27:11.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:27:11.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:11.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:11.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:11.230 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 06:27:11.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:11.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:11.230 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 06:27:11.230 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:11.230 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:11.230 | INFO | Uploaded 36GiB to GPU in 2.04627s (17.593GiB/s) | 20260507 06:27:11.230 | INFO | Downloaded 34GiB to GPU in 1.94851s (17.4492GiB/s) | 20260507 06:27:16.370 | INFO | Analyzing chunk frames = [7804416,7804928) | 20260507 06:27:16.370 | INFO | MemInfo: { | 20260507 06:27:16.370 | INFO | "gpu" : " 17 GB", | 20260507 06:27:16.370 | INFO | "huge" : "", | 20260507 06:27:16.370 | INFO | "huge_shared" : " 27 GB", | 20260507 06:27:16.370 | INFO | "malloc" : " 14 GB", | 20260507 06:27:16.370 | INFO | "pinned" : " 29 GB" | 20260507 06:27:16.370 | INFO | } | 20260507 06:27:16.370 | INFO | Perf report for the last 5.14056 seconds: | 20260507 06:27:16.370 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:27:16.370 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:16.370 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:16.370 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:16.370 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 06:27:16.370 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:16.370 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:16.370 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:27:16.370 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:16.370 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:16.370 | INFO | Uploaded 36GiB to GPU in 2.01587s (17.8583GiB/s) | 20260507 06:27:16.371 | INFO | Downloaded 34GiB to GPU in 1.9384s (17.5402GiB/s) | 20260507 06:27:21.514 | INFO | Analyzing chunk frames = [7804928,7805440) | 20260507 06:27:21.514 | INFO | MemInfo: { | 20260507 06:27:21.514 | INFO | "gpu" : " 17 GB", | 20260507 06:27:21.514 | INFO | "huge" : "", | 20260507 06:27:21.514 | INFO | "huge_shared" : " 27 GB", | 20260507 06:27:21.514 | INFO | "malloc" : " 14 GB", | 20260507 06:27:21.514 | INFO | "pinned" : " 29 GB" | 20260507 06:27:21.514 | INFO | } | 20260507 06:27:21.514 | INFO | Perf report for the last 5.14414 seconds: | 20260507 06:27:21.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:27:21.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:21.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:21.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:21.514 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 06:27:21.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:21.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:21.515 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:27:21.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:21.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:21.515 | INFO | Uploaded 36GiB to GPU in 2.05914s (17.483GiB/s) | 20260507 06:27:21.515 | INFO | Downloaded 34GiB to GPU in 1.94772s (17.4563GiB/s) | 20260507 06:27:26.658 | INFO | Analyzing chunk frames = [7805440,7805952) | 20260507 06:27:26.658 | INFO | MemInfo: { | 20260507 06:27:26.658 | INFO | "gpu" : " 17 GB", | 20260507 06:27:26.658 | INFO | "huge" : "", | 20260507 06:27:26.658 | INFO | "huge_shared" : " 27 GB", | 20260507 06:27:26.658 | INFO | "malloc" : " 14 GB", | 20260507 06:27:26.658 | INFO | "pinned" : " 29 GB" | 20260507 06:27:26.658 | INFO | } | 20260507 06:27:26.658 | INFO | Perf report for the last 5.14399 seconds: | 20260507 06:27:26.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:27:26.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:26.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:26.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:26.659 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 06:27:26.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:26.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:26.659 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:27:26.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:26.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:26.659 | INFO | Uploaded 36GiB to GPU in 2.06014s (17.4746GiB/s) | 20260507 06:27:26.659 | INFO | Downloaded 34GiB to GPU in 1.91974s (17.7107GiB/s) | 20260507 06:27:31.785 | INFO | Analyzing chunk frames = [7805952,7806464) | 20260507 06:27:31.785 | INFO | MemInfo: { | 20260507 06:27:31.785 | INFO | "gpu" : " 17 GB", | 20260507 06:27:31.785 | INFO | "huge" : "", | 20260507 06:27:31.785 | INFO | "huge_shared" : " 27 GB", | 20260507 06:27:31.785 | INFO | "malloc" : " 14 GB", | 20260507 06:27:31.785 | INFO | "pinned" : " 29 GB" | 20260507 06:27:31.785 | INFO | } | 20260507 06:27:31.785 | INFO | Perf report for the last 5.12616 seconds: | 20260507 06:27:31.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:27:31.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:31.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:31.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:31.785 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260507 06:27:31.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:31.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:31.785 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 06:27:31.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:31.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:31.785 | INFO | Uploaded 36GiB to GPU in 1.89953s (18.9521GiB/s) | 20260507 06:27:31.785 | INFO | Downloaded 34GiB to GPU in 1.97482s (17.2168GiB/s) | 20260507 06:27:36.928 | INFO | Analyzing chunk frames = [7806464,7806976) | 20260507 06:27:36.928 | INFO | MemInfo: { | 20260507 06:27:36.928 | INFO | "gpu" : " 17 GB", | 20260507 06:27:36.928 | INFO | "huge" : "", | 20260507 06:27:36.928 | INFO | "huge_shared" : " 27 GB", | 20260507 06:27:36.928 | INFO | "malloc" : " 14 GB", | 20260507 06:27:36.928 | INFO | "pinned" : " 29 GB" | 20260507 06:27:36.928 | INFO | } | 20260507 06:27:36.928 | INFO | Perf report for the last 5.14349 seconds: | 20260507 06:27:36.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:27:36.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:36.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:36.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:36.928 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 06:27:36.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:36.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:36.928 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 06:27:36.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:36.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:36.928 | INFO | Uploaded 36GiB to GPU in 1.95876s (18.3789GiB/s) | 20260507 06:27:36.928 | INFO | Downloaded 34GiB to GPU in 1.95262s (17.4125GiB/s) | 20260507 06:27:36.928 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:27:36.928 | INFO | Total mem usage: 18 GiB | 20260507 06:27:36.928 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:27:36.928 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.5%) | 20260507 06:27:36.928 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:27:36.928 | INFO | High Water Breakdown: | 20260507 06:27:36.928 | INFO | 60% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:27:36.929 | INFO | 39.8% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:27:36.929 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:27:36.929 | INFO | ------------------------------------------ | 20260507 06:27:36.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:27:36.929 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:27:36.929 | INFO | Total mem usage: 26.5 GiB | 20260507 06:27:36.929 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:27:36.929 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:27:36.929 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:27:36.929 | INFO | High Water Breakdown: | 20260507 06:27:36.929 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:27:36.929 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:27:36.929 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:27:36.929 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:27:36.929 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:27:36.929 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:27:36.929 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:27:36.929 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:27:36.929 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:27:36.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:27:36.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:27:36.929 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:27:36.929 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:27:36.929 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:27:36.929 | INFO | ------------------------------------------ | 20260507 06:27:36.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:27:36.929 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:27:36.929 | INFO | Total mem usage: 25.6 GiB | 20260507 06:27:36.929 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:27:36.929 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:27:36.929 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:27:36.929 | INFO | High Water Breakdown: | 20260507 06:27:36.929 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:27:36.929 | INFO | ------------------------------------------ | 20260507 06:27:36.929 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:27:36.929 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:27:36.929 | INFO | Total mem usage: 15.7 GiB | 20260507 06:27:36.929 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:27:36.929 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:27:36.929 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:27:36.929 | INFO | High Water Breakdown: | 20260507 06:27:36.929 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:27:36.929 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:27:36.929 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:27:36.929 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:27:36.929 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:27:36.929 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:27:36.929 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:27:36.930 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:27:36.930 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:27:36.930 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:27:36.930 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:27:36.930 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:27:36.930 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:27:36.930 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:27:36.930 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:27:36.930 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:27:36.930 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:27:36.930 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:27:36.930 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:27:36.930 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:27:36.930 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:27:36.930 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:27:36.930 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:27:36.930 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:27:36.930 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:27:36.930 | INFO | ------------------------------------------ | 20260507 06:27:36.930 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:27:36.930 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:27:36.930 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:27:36.930 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:27:36.930 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.33e+01 / 1.25e+00 | 20260507 06:27:36.930 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.02e-01, 9.90e-03 / 8.21e-03 | 20260507 06:27:36.930 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.45e+00, 5.20e-01 / 2.51e-01 | 20260507 06:27:36.930 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.45e+00, 1.68e-01 / 1.43e-01 | 20260507 06:27:36.930 | INFO | PerfReport: 16.18%, DME -- 3.98e+01 / 4.45e+01, 3.79e+00 / 1.42e+00 | 20260507 06:27:36.930 | INFO | PerfReport: 57.76%, FrameLabeling -- 1.78e+01 / 2.01e+01, 1.38e+00 / 1.57e+00 | 20260507 06:27:36.930 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.68e+00, 3.82e-01 / 2.25e-01 | 20260507 06:27:36.930 | INFO | PerfReport: 2.95%, Metrics -- 9.06e-01 / 1.23e+00, 2.41e-01 / 1.50e-01 | 20260507 06:27:36.930 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.05e-01 / 3.54e-01, 5.04e-02 / 2.00e-02 | 20260507 06:27:36.930 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:27:36.930 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:27:36.930 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:27:36.930 | INFO | PerfReport: Full Execution -- 9.60e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 06:27:36.930 | INFO | PerfReport: 21.82%, Upload -- 2.09e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 06:27:36.930 | INFO | PerfReport: 21.39%, Download -- 2.05e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 06:27:36.930 | INFO | PerfReport: 32.01%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 06:27:36.930 | INFO | PerfReport: 1.45%, LockUpload -- 1.39e+00 / 0.00e+00, 4.79e+00 / 0.00e+00 | 20260507 06:27:36.930 | INFO | PerfReport: 1.25%, LockDownload -- 1.20e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260507 06:27:36.930 | INFO | PerfReport: 22.06%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 06:27:36.930 | INFO | PerfReport: 0.02%, Misc -- 1.63e-02 / 0.00e+00, 4.38e-02 / 0.00e+00 | 20260507 06:27:37.031 | INFO | Starting a round of Baz IO | 20260507 06:27:40.126 | INFO | Populating RTMetrics report | 20260507 06:27:40.127 | INFO | RTMetrics report populated | 20260507 06:27:40.127 | INFO | Opening tmp RTMetrics json file | 20260507 06:27:40.142 | INFO | Serializing RTMetrics to json | 20260507 06:27:40.143 | INFO | Serialization done, swapping tmp file now... | 20260507 06:27:40.144 | INFO | RTMetrics done | 20260507 06:27:40.217 | INFO | Finished a round of Baz IO: Wrote 1.668 GiB in 3.185 seconds | 20260507 06:27:40.217 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.63% / 0.00% / 67.37% | 20260507 06:27:40.217 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.122178% / 0% / 8.34465e-05% | 20260507 06:27:40.217 | INFO | Baz IO overall: | 20260507 06:27:40.217 | INFO | BytesWritten : 3937099552668 | 20260507 06:27:40.217 | INFO | NumEvents : 1886717380959 | 20260507 06:27:42.072 | INFO | Analyzing chunk frames = [7806976,7807488) | 20260507 06:27:42.072 | INFO | MemInfo: { | 20260507 06:27:42.072 | INFO | "gpu" : " 17 GB", | 20260507 06:27:42.072 | INFO | "huge" : "", | 20260507 06:27:42.072 | INFO | "huge_shared" : " 27 GB", | 20260507 06:27:42.072 | INFO | "malloc" : " 14 GB", | 20260507 06:27:42.072 | INFO | "pinned" : " 29 GB" | 20260507 06:27:42.072 | INFO | } | 20260507 06:27:42.072 | INFO | Perf report for the last 5.14383 seconds: | 20260507 06:27:42.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:27:42.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:42.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:42.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:42.072 | INFO | ANALYSIS : 61.7%, 3.0, 96.0ms, 0 | 20260507 06:27:42.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:42.072 | INFO | RT_METRICS : 43.7%, 1.0, 23.0ms, 0 | 20260507 06:27:42.072 | INFO | PRE_HQ : 59.5%, 1.9, 58.5ms, 0 | 20260507 06:27:42.072 | INFO | BAZWRITER : 61.6%, 3.3, 103.1ms, 0 | 20260507 06:27:42.072 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 06:27:42.072 | INFO | Uploaded 36GiB to GPU in 2.04535s (17.6009GiB/s) | 20260507 06:27:42.072 | INFO | Downloaded 38GiB to GPU in 2.18317s (17.4059GiB/s) | 20260507 06:27:47.212 | INFO | Analyzing chunk frames = [7807488,7808000) | 20260507 06:27:47.212 | INFO | MemInfo: { | 20260507 06:27:47.212 | INFO | "gpu" : " 17 GB", | 20260507 06:27:47.212 | INFO | "huge" : "", | 20260507 06:27:47.212 | INFO | "huge_shared" : " 27 GB", | 20260507 06:27:47.212 | INFO | "malloc" : " 14 GB", | 20260507 06:27:47.212 | INFO | "pinned" : " 29 GB" | 20260507 06:27:47.212 | INFO | } | 20260507 06:27:47.212 | INFO | Perf report for the last 5.13984 seconds: | 20260507 06:27:47.212 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:27:47.212 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:47.212 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:47.212 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:47.212 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 06:27:47.212 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:47.212 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:47.212 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 06:27:47.212 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:47.212 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:47.212 | INFO | Uploaded 36GiB to GPU in 2.01092s (17.9023GiB/s) | 20260507 06:27:47.212 | INFO | Downloaded 34GiB to GPU in 1.93608s (17.5613GiB/s) | 20260507 06:27:52.348 | INFO | Analyzing chunk frames = [7808000,7808512) | 20260507 06:27:52.348 | INFO | MemInfo: { | 20260507 06:27:52.348 | INFO | "gpu" : " 17 GB", | 20260507 06:27:52.348 | INFO | "huge" : "", | 20260507 06:27:52.348 | INFO | "huge_shared" : " 27 GB", | 20260507 06:27:52.348 | INFO | "malloc" : " 14 GB", | 20260507 06:27:52.348 | INFO | "pinned" : " 29 GB" | 20260507 06:27:52.348 | INFO | } | 20260507 06:27:52.348 | INFO | Perf report for the last 5.13619 seconds: | 20260507 06:27:52.348 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:27:52.348 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:52.348 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:52.348 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:52.348 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:27:52.348 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:52.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:52.348 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:27:52.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:52.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:52.348 | INFO | Uploaded 36GiB to GPU in 2.02271s (17.7979GiB/s) | 20260507 06:27:52.349 | INFO | Downloaded 34GiB to GPU in 1.91324s (17.7709GiB/s) | 20260507 06:27:57.486 | INFO | Analyzing chunk frames = [7808512,7809024) | 20260507 06:27:57.486 | INFO | MemInfo: { | 20260507 06:27:57.486 | INFO | "gpu" : " 17 GB", | 20260507 06:27:57.486 | INFO | "huge" : "", | 20260507 06:27:57.486 | INFO | "huge_shared" : " 27 GB", | 20260507 06:27:57.486 | INFO | "malloc" : " 14 GB", | 20260507 06:27:57.487 | INFO | "pinned" : " 29 GB" | 20260507 06:27:57.487 | INFO | } | 20260507 06:27:57.487 | INFO | Perf report for the last 5.13858 seconds: | 20260507 06:27:57.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:27:57.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:57.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:57.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:57.487 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 06:27:57.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:57.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:57.487 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:27:57.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:57.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:27:57.487 | INFO | Uploaded 36GiB to GPU in 2.04307s (17.6205GiB/s) | 20260507 06:27:57.487 | INFO | Downloaded 34GiB to GPU in 1.94935s (17.4417GiB/s) | 20260507 06:28:02.629 | INFO | Analyzing chunk frames = [7809024,7809536) | 20260507 06:28:02.629 | INFO | MemInfo: { | 20260507 06:28:02.629 | INFO | "gpu" : " 17 GB", | 20260507 06:28:02.629 | INFO | "huge" : "", | 20260507 06:28:02.629 | INFO | "huge_shared" : " 27 GB", | 20260507 06:28:02.629 | INFO | "malloc" : " 14 GB", | 20260507 06:28:02.629 | INFO | "pinned" : " 29 GB" | 20260507 06:28:02.629 | INFO | } | 20260507 06:28:02.629 | INFO | Perf report for the last 5.14273 seconds: | 20260507 06:28:02.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:28:02.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:02.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:02.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:02.629 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 06:28:02.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:02.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:02.629 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:28:02.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:02.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:02.630 | INFO | Uploaded 36GiB to GPU in 2.05895s (17.4847GiB/s) | 20260507 06:28:02.630 | INFO | Downloaded 34GiB to GPU in 1.9507s (17.4297GiB/s) | 20260507 06:28:07.767 | INFO | Analyzing chunk frames = [7809536,7810048) | 20260507 06:28:07.767 | INFO | MemInfo: { | 20260507 06:28:07.767 | INFO | "gpu" : " 17 GB", | 20260507 06:28:07.767 | INFO | "huge" : "", | 20260507 06:28:07.768 | INFO | "huge_shared" : " 27 GB", | 20260507 06:28:07.768 | INFO | "malloc" : " 14 GB", | 20260507 06:28:07.768 | INFO | "pinned" : " 29 GB" | 20260507 06:28:07.768 | INFO | } | 20260507 06:28:07.768 | INFO | Perf report for the last 5.13837 seconds: | 20260507 06:28:07.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:28:07.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:07.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:07.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:07.768 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 06:28:07.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:07.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:07.768 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:28:07.768 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:07.768 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:07.768 | INFO | Uploaded 36GiB to GPU in 2.04477s (17.6059GiB/s) | 20260507 06:28:07.768 | INFO | Downloaded 34GiB to GPU in 1.90015s (17.8934GiB/s) | 20260507 06:28:12.910 | INFO | Analyzing chunk frames = [7810048,7810560) | 20260507 06:28:12.910 | INFO | MemInfo: { | 20260507 06:28:12.910 | INFO | "gpu" : " 17 GB", | 20260507 06:28:12.910 | INFO | "huge" : "", | 20260507 06:28:12.910 | INFO | "huge_shared" : " 27 GB", | 20260507 06:28:12.910 | INFO | "malloc" : " 14 GB", | 20260507 06:28:12.910 | INFO | "pinned" : " 29 GB" | 20260507 06:28:12.910 | INFO | } | 20260507 06:28:12.910 | INFO | Perf report for the last 5.14225 seconds: | 20260507 06:28:12.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:28:12.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:12.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:12.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:12.910 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:28:12.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:12.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:12.910 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 06:28:12.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:12.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:12.910 | INFO | Uploaded 36GiB to GPU in 1.95809s (18.3853GiB/s) | 20260507 06:28:12.910 | INFO | Downloaded 34GiB to GPU in 1.98018s (17.1702GiB/s) | 20260507 06:28:18.044 | INFO | Analyzing chunk frames = [7810560,7811072) | 20260507 06:28:18.044 | INFO | MemInfo: { | 20260507 06:28:18.044 | INFO | "gpu" : " 17 GB", | 20260507 06:28:18.044 | INFO | "huge" : "", | 20260507 06:28:18.044 | INFO | "huge_shared" : " 27 GB", | 20260507 06:28:18.044 | INFO | "malloc" : " 14 GB", | 20260507 06:28:18.044 | INFO | "pinned" : " 29 GB" | 20260507 06:28:18.044 | INFO | } | 20260507 06:28:18.044 | INFO | Perf report for the last 5.13399 seconds: | 20260507 06:28:18.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:28:18.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:18.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:18.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:18.044 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 06:28:18.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:18.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:18.044 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:28:18.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:18.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:18.044 | INFO | Uploaded 36GiB to GPU in 1.9823s (18.1607GiB/s) | 20260507 06:28:18.044 | INFO | Downloaded 34GiB to GPU in 1.95974s (17.3493GiB/s) | 20260507 06:28:18.140 | INFO | Starting a round of Baz IO | 20260507 06:28:21.238 | INFO | Populating RTMetrics report | 20260507 06:28:21.238 | INFO | RTMetrics report populated | 20260507 06:28:21.238 | INFO | Opening tmp RTMetrics json file | 20260507 06:28:21.250 | INFO | Serializing RTMetrics to json | 20260507 06:28:21.253 | INFO | Serialization done, swapping tmp file now... | 20260507 06:28:21.253 | INFO | RTMetrics done | 20260507 06:28:21.320 | INFO | Finished a round of Baz IO: Wrote 1.668 GiB in 3.179 seconds | 20260507 06:28:21.320 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.63% / 0.00% / 67.37% | 20260507 06:28:21.320 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.122154% / 0% / 8.34465e-05% | 20260507 06:28:21.320 | INFO | Baz IO overall: | 20260507 06:28:21.320 | INFO | BytesWritten : 3938890487607 | 20260507 06:28:21.320 | INFO | NumEvents : 1887555984942 | 20260507 06:28:23.187 | INFO | Analyzing chunk frames = [7811072,7811584) | 20260507 06:28:23.187 | INFO | MemInfo: { | 20260507 06:28:23.187 | INFO | "gpu" : " 17 GB", | 20260507 06:28:23.187 | INFO | "huge" : "", | 20260507 06:28:23.187 | INFO | "huge_shared" : " 27 GB", | 20260507 06:28:23.187 | INFO | "malloc" : " 14 GB", | 20260507 06:28:23.187 | INFO | "pinned" : " 29 GB" | 20260507 06:28:23.187 | INFO | } | 20260507 06:28:23.187 | INFO | Perf report for the last 5.14309 seconds: | 20260507 06:28:23.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:28:23.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:23.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:23.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:23.187 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 06:28:23.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:23.187 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:28:23.187 | INFO | PRE_HQ : 58.9%, 1.8, 54.7ms, 0 | 20260507 06:28:23.187 | INFO | BAZWRITER : 61.4%, 3.1, 99.0ms, 0 | 20260507 06:28:23.187 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 06:28:23.187 | INFO | Uploaded 36GiB to GPU in 2.04931s (17.5669GiB/s) | 20260507 06:28:23.187 | INFO | Downloaded 38GiB to GPU in 2.17824s (17.4453GiB/s) | 20260507 06:28:28.322 | INFO | Analyzing chunk frames = [7811584,7812096) | 20260507 06:28:28.322 | INFO | MemInfo: { | 20260507 06:28:28.322 | INFO | "gpu" : " 17 GB", | 20260507 06:28:28.322 | INFO | "huge" : "", | 20260507 06:28:28.322 | INFO | "huge_shared" : " 27 GB", | 20260507 06:28:28.322 | INFO | "malloc" : " 14 GB", | 20260507 06:28:28.322 | INFO | "pinned" : " 29 GB" | 20260507 06:28:28.322 | INFO | } | 20260507 06:28:28.322 | INFO | Perf report for the last 5.13471 seconds: | 20260507 06:28:28.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:28:28.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:28.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:28.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:28.322 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:28:28.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:28.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:28.322 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:28:28.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:28.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:28.322 | INFO | Uploaded 36GiB to GPU in 2.00274s (17.9754GiB/s) | 20260507 06:28:28.322 | INFO | Downloaded 34GiB to GPU in 1.92716s (17.6425GiB/s) | 20260507 06:28:33.471 | INFO | Analyzing chunk frames = [7812096,7812608) | 20260507 06:28:33.471 | INFO | MemInfo: { | 20260507 06:28:33.471 | INFO | "gpu" : " 17 GB", | 20260507 06:28:33.471 | INFO | "huge" : "", | 20260507 06:28:33.471 | INFO | "huge_shared" : " 27 GB", | 20260507 06:28:33.472 | INFO | "malloc" : " 14 GB", | 20260507 06:28:33.472 | INFO | "pinned" : " 29 GB" | 20260507 06:28:33.472 | INFO | } | 20260507 06:28:33.472 | INFO | Perf report for the last 5.14977 seconds: | 20260507 06:28:33.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:28:33.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:33.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:33.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:33.472 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 06:28:33.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:33.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:33.472 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:28:33.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:33.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:33.472 | INFO | Uploaded 36GiB to GPU in 2.04412s (17.6115GiB/s) | 20260507 06:28:33.472 | INFO | Downloaded 34GiB to GPU in 1.95153s (17.4222GiB/s) | 20260507 06:28:38.613 | INFO | Analyzing chunk frames = [7812608,7813120) | 20260507 06:28:38.613 | INFO | MemInfo: { | 20260507 06:28:38.613 | INFO | "gpu" : " 17 GB", | 20260507 06:28:38.613 | INFO | "huge" : "", | 20260507 06:28:38.613 | INFO | "huge_shared" : " 27 GB", | 20260507 06:28:38.613 | INFO | "malloc" : " 14 GB", | 20260507 06:28:38.613 | INFO | "pinned" : " 29 GB" | 20260507 06:28:38.613 | INFO | } | 20260507 06:28:38.613 | INFO | Perf report for the last 5.14157 seconds: | 20260507 06:28:38.613 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:28:38.613 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:38.613 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:38.613 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:38.613 | INFO | ANALYSIS : 60.4%, 2.9, 93.4ms, 0 | 20260507 06:28:38.613 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:38.613 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:38.613 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:28:38.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:38.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:38.613 | INFO | Uploaded 36GiB to GPU in 2.05953s (17.4798GiB/s) | 20260507 06:28:38.614 | INFO | Downloaded 34GiB to GPU in 1.94994s (17.4365GiB/s) | 20260507 06:28:43.736 | INFO | Analyzing chunk frames = [7813120,7813632) | 20260507 06:28:43.736 | INFO | MemInfo: { | 20260507 06:28:43.736 | INFO | "gpu" : " 17 GB", | 20260507 06:28:43.736 | INFO | "huge" : "", | 20260507 06:28:43.736 | INFO | "huge_shared" : " 27 GB", | 20260507 06:28:43.736 | INFO | "malloc" : " 14 GB", | 20260507 06:28:43.736 | INFO | "pinned" : " 29 GB" | 20260507 06:28:43.736 | INFO | } | 20260507 06:28:43.736 | INFO | Perf report for the last 5.12282 seconds: | 20260507 06:28:43.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:28:43.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:43.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:43.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:43.736 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 06:28:43.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:43.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:43.736 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:28:43.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:43.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:43.736 | INFO | Uploaded 36GiB to GPU in 2.05713s (17.5001GiB/s) | 20260507 06:28:43.736 | INFO | Downloaded 34GiB to GPU in 1.9051s (17.8468GiB/s) | 20260507 06:28:48.881 | INFO | Analyzing chunk frames = [7813632,7814144) | 20260507 06:28:48.881 | INFO | MemInfo: { | 20260507 06:28:48.881 | INFO | "gpu" : " 17 GB", | 20260507 06:28:48.881 | INFO | "huge" : "", | 20260507 06:28:48.881 | INFO | "huge_shared" : " 27 GB", | 20260507 06:28:48.881 | INFO | "malloc" : " 14 GB", | 20260507 06:28:48.881 | INFO | "pinned" : " 29 GB" | 20260507 06:28:48.881 | INFO | } | 20260507 06:28:48.881 | INFO | Perf report for the last 5.14487 seconds: | 20260507 06:28:48.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:28:48.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:48.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:48.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:48.881 | INFO | ANALYSIS : 60.4%, 3.0, 94.4ms, 0 | 20260507 06:28:48.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:48.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:48.881 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 06:28:48.881 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:48.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:48.881 | INFO | Uploaded 36GiB to GPU in 2.05165s (17.5469GiB/s) | 20260507 06:28:48.881 | INFO | Downloaded 34GiB to GPU in 1.91767s (17.7298GiB/s) | 20260507 06:28:54.022 | INFO | Analyzing chunk frames = [7814144,7814656) | 20260507 06:28:54.022 | INFO | MemInfo: { | 20260507 06:28:54.022 | INFO | "gpu" : " 17 GB", | 20260507 06:28:54.022 | INFO | "huge" : "", | 20260507 06:28:54.022 | INFO | "huge_shared" : " 27 GB", | 20260507 06:28:54.022 | INFO | "malloc" : " 14 GB", | 20260507 06:28:54.022 | INFO | "pinned" : " 29 GB" | 20260507 06:28:54.022 | INFO | } | 20260507 06:28:54.022 | INFO | Perf report for the last 5.14112 seconds: | 20260507 06:28:54.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:28:54.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:54.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:54.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:54.022 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:28:54.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:54.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:54.022 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:28:54.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:54.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:54.022 | INFO | Uploaded 36GiB to GPU in 1.96854s (18.2877GiB/s) | 20260507 06:28:54.022 | INFO | Downloaded 34GiB to GPU in 1.98442s (17.1335GiB/s) | 20260507 06:28:59.161 | INFO | Analyzing chunk frames = [7814656,7815168) | 20260507 06:28:59.161 | INFO | MemInfo: { | 20260507 06:28:59.161 | INFO | "gpu" : " 17 GB", | 20260507 06:28:59.161 | INFO | "huge" : "", | 20260507 06:28:59.161 | INFO | "huge_shared" : " 27 GB", | 20260507 06:28:59.161 | INFO | "malloc" : " 14 GB", | 20260507 06:28:59.161 | INFO | "pinned" : " 29 GB" | 20260507 06:28:59.161 | INFO | } | 20260507 06:28:59.161 | INFO | Perf report for the last 5.13926 seconds: | 20260507 06:28:59.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:28:59.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:59.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:59.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:59.161 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 06:28:59.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:59.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:59.161 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:28:59.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:59.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:28:59.162 | INFO | Uploaded 36GiB to GPU in 1.96596s (18.3117GiB/s) | 20260507 06:28:59.162 | INFO | Downloaded 34GiB to GPU in 1.96382s (17.3132GiB/s) | 20260507 06:28:59.162 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:28:59.162 | INFO | Total mem usage: 18 GiB | 20260507 06:28:59.162 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:28:59.162 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 06:28:59.162 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:28:59.162 | INFO | High Water Breakdown: | 20260507 06:28:59.162 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:28:59.162 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:28:59.162 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:28:59.162 | INFO | ------------------------------------------ | 20260507 06:28:59.162 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:28:59.162 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:28:59.162 | INFO | Total mem usage: 26.5 GiB | 20260507 06:28:59.162 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:28:59.162 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:28:59.162 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:28:59.162 | INFO | High Water Breakdown: | 20260507 06:28:59.162 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:28:59.162 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:28:59.162 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:28:59.162 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:28:59.162 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:28:59.162 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:28:59.162 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:28:59.162 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:28:59.162 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:28:59.162 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:28:59.162 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:28:59.162 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:28:59.162 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:28:59.162 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:28:59.162 | INFO | ------------------------------------------ | 20260507 06:28:59.162 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:28:59.162 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:28:59.162 | INFO | Total mem usage: 25.6 GiB | 20260507 06:28:59.162 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:28:59.162 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:28:59.162 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:28:59.162 | INFO | High Water Breakdown: | 20260507 06:28:59.162 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:28:59.162 | INFO | ------------------------------------------ | 20260507 06:28:59.162 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:28:59.162 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:28:59.162 | INFO | Total mem usage: 15.7 GiB | 20260507 06:28:59.162 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:28:59.162 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:28:59.162 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:28:59.162 | INFO | High Water Breakdown: | 20260507 06:28:59.162 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:28:59.162 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:28:59.162 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:28:59.162 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:28:59.162 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:28:59.163 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:28:59.163 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:28:59.163 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:28:59.163 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:28:59.163 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:28:59.163 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:28:59.163 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:28:59.163 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:28:59.163 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:28:59.163 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:28:59.163 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:28:59.163 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:28:59.163 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:28:59.163 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:28:59.163 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:28:59.163 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:28:59.163 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:28:59.163 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:28:59.163 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:28:59.163 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:28:59.163 | INFO | ------------------------------------------ | 20260507 06:28:59.163 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:28:59.163 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:28:59.163 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:28:59.163 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:28:59.163 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.35e+01 / 1.23e+00 | 20260507 06:28:59.163 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.02e-01, 9.42e-03 / 4.62e-03 | 20260507 06:28:59.163 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.46e+00, 5.06e-01 / 2.48e-01 | 20260507 06:28:59.163 | INFO | PerfReport: 3.21%, Histogram -- 9.86e-01 / 1.49e+00, 1.65e-01 / 1.02e-01 | 20260507 06:28:59.163 | INFO | PerfReport: 16.31%, DME -- 4.01e+01 / 4.50e+01, 3.96e+00 / 1.61e+00 | 20260507 06:28:59.163 | INFO | PerfReport: 57.64%, FrameLabeling -- 1.77e+01 / 1.93e+01, 1.27e+00 / 1.16e+00 | 20260507 06:28:59.163 | INFO | PerfReport: 6.91%, PulseAccumulating -- 2.12e+00 / 2.72e+00, 3.85e-01 / 1.49e-01 | 20260507 06:28:59.163 | INFO | PerfReport: 2.96%, Metrics -- 9.07e-01 / 1.23e+00, 2.41e-01 / 1.20e-01 | 20260507 06:28:59.163 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.05e-01 / 3.64e-01, 5.04e-02 / 2.16e-02 | 20260507 06:28:59.163 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:28:59.163 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:28:59.163 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:28:59.163 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 06:28:59.163 | INFO | PerfReport: 21.90%, Upload -- 2.11e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260507 06:28:59.163 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 06:28:59.163 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 06:28:59.163 | INFO | PerfReport: 1.66%, LockUpload -- 1.60e+00 / 0.00e+00, 5.01e+00 / 0.00e+00 | 20260507 06:28:59.163 | INFO | PerfReport: 1.20%, LockDownload -- 1.15e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260507 06:28:59.163 | INFO | PerfReport: 21.98%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 06:28:59.163 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.52e-02 / 0.00e+00 | 20260507 06:28:59.258 | INFO | Starting a round of Baz IO | 20260507 06:29:02.351 | INFO | Populating RTMetrics report | 20260507 06:29:02.351 | INFO | RTMetrics report populated | 20260507 06:29:02.351 | INFO | Opening tmp RTMetrics json file | 20260507 06:29:02.367 | INFO | Serializing RTMetrics to json | 20260507 06:29:02.369 | INFO | Serialization done, swapping tmp file now... | 20260507 06:29:02.369 | INFO | RTMetrics done | 20260507 06:29:02.440 | INFO | Finished a round of Baz IO: Wrote 1.668 GiB in 3.181 seconds | 20260507 06:29:02.440 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.62% / 0.00% / 67.37% | 20260507 06:29:02.440 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.121355% / 0% / 7.54992e-05% | 20260507 06:29:02.440 | INFO | Baz IO overall: | 20260507 06:29:02.440 | INFO | BytesWritten : 3940681125669 | 20260507 06:29:02.440 | INFO | NumEvents : 1888394146505 | 20260507 06:29:04.291 | INFO | Analyzing chunk frames = [7815168,7815680) | 20260507 06:29:04.291 | INFO | MemInfo: { | 20260507 06:29:04.291 | INFO | "gpu" : " 17 GB", | 20260507 06:29:04.291 | INFO | "huge" : "", | 20260507 06:29:04.291 | INFO | "huge_shared" : " 27 GB", | 20260507 06:29:04.291 | INFO | "malloc" : " 14 GB", | 20260507 06:29:04.291 | INFO | "pinned" : " 29 GB" | 20260507 06:29:04.291 | INFO | } | 20260507 06:29:04.291 | INFO | Perf report for the last 5.13016 seconds: | 20260507 06:29:04.292 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:29:04.292 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:04.292 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:04.292 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:04.292 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 06:29:04.292 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:04.292 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 06:29:04.292 | INFO | PRE_HQ : 59.2%, 1.8, 54.8ms, 0 | 20260507 06:29:04.292 | INFO | BAZWRITER : 61.6%, 3.1, 99.0ms, 0 | 20260507 06:29:04.292 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:29:04.292 | INFO | Uploaded 36GiB to GPU in 2.04658s (17.5904GiB/s) | 20260507 06:29:04.292 | INFO | Downloaded 38GiB to GPU in 2.17471s (17.4736GiB/s) | 20260507 06:29:09.438 | INFO | Analyzing chunk frames = [7815680,7816192) | 20260507 06:29:09.438 | INFO | MemInfo: { | 20260507 06:29:09.438 | INFO | "gpu" : " 17 GB", | 20260507 06:29:09.438 | INFO | "huge" : "", | 20260507 06:29:09.438 | INFO | "huge_shared" : " 27 GB", | 20260507 06:29:09.438 | INFO | "malloc" : " 14 GB", | 20260507 06:29:09.438 | INFO | "pinned" : " 29 GB" | 20260507 06:29:09.438 | INFO | } | 20260507 06:29:09.438 | INFO | Perf report for the last 5.14662 seconds: | 20260507 06:29:09.438 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:29:09.438 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:09.438 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:09.438 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:09.438 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 06:29:09.438 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:09.438 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:09.438 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 06:29:09.438 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:09.438 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:09.438 | INFO | Uploaded 36GiB to GPU in 2.04296s (17.6215GiB/s) | 20260507 06:29:09.438 | INFO | Downloaded 34GiB to GPU in 1.94126s (17.5144GiB/s) | 20260507 06:29:14.578 | INFO | Analyzing chunk frames = [7816192,7816704) | 20260507 06:29:14.578 | INFO | MemInfo: { | 20260507 06:29:14.578 | INFO | "gpu" : " 17 GB", | 20260507 06:29:14.578 | INFO | "huge" : "", | 20260507 06:29:14.578 | INFO | "huge_shared" : " 27 GB", | 20260507 06:29:14.578 | INFO | "malloc" : " 14 GB", | 20260507 06:29:14.578 | INFO | "pinned" : " 29 GB" | 20260507 06:29:14.579 | INFO | } | 20260507 06:29:14.579 | INFO | Perf report for the last 5.14045 seconds: | 20260507 06:29:14.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:29:14.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:14.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:14.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:14.579 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 06:29:14.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:14.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:14.579 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:29:14.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:14.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:14.579 | INFO | Uploaded 36GiB to GPU in 2.01131s (17.8988GiB/s) | 20260507 06:29:14.579 | INFO | Downloaded 34GiB to GPU in 1.95723s (17.3715GiB/s) | 20260507 06:29:19.725 | INFO | Analyzing chunk frames = [7816704,7817216) | 20260507 06:29:19.725 | INFO | MemInfo: { | 20260507 06:29:19.725 | INFO | "gpu" : " 17 GB", | 20260507 06:29:19.725 | INFO | "huge" : "", | 20260507 06:29:19.725 | INFO | "huge_shared" : " 27 GB", | 20260507 06:29:19.725 | INFO | "malloc" : " 14 GB", | 20260507 06:29:19.725 | INFO | "pinned" : " 29 GB" | 20260507 06:29:19.725 | INFO | } | 20260507 06:29:19.726 | INFO | Perf report for the last 5.14699 seconds: | 20260507 06:29:19.726 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:29:19.726 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:19.726 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:19.726 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:19.726 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 06:29:19.726 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:19.726 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:19.726 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:29:19.726 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:19.726 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:19.726 | INFO | Uploaded 36GiB to GPU in 1.99414s (18.0529GiB/s) | 20260507 06:29:19.726 | INFO | Downloaded 34GiB to GPU in 1.93621s (17.56GiB/s) | 20260507 06:29:24.858 | INFO | Analyzing chunk frames = [7817216,7817728) | 20260507 06:29:24.858 | INFO | MemInfo: { | 20260507 06:29:24.858 | INFO | "gpu" : " 17 GB", | 20260507 06:29:24.858 | INFO | "huge" : "", | 20260507 06:29:24.858 | INFO | "huge_shared" : " 27 GB", | 20260507 06:29:24.858 | INFO | "malloc" : " 14 GB", | 20260507 06:29:24.858 | INFO | "pinned" : " 29 GB" | 20260507 06:29:24.858 | INFO | } | 20260507 06:29:24.858 | INFO | Perf report for the last 5.13268 seconds: | 20260507 06:29:24.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:29:24.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:24.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:24.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:24.858 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 06:29:24.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:24.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:24.858 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 06:29:24.858 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:24.858 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:24.859 | INFO | Uploaded 36GiB to GPU in 2.05339s (17.532GiB/s) | 20260507 06:29:24.859 | INFO | Downloaded 34GiB to GPU in 1.90919s (17.8086GiB/s) | 20260507 06:29:29.993 | INFO | Analyzing chunk frames = [7817728,7818240) | 20260507 06:29:29.993 | INFO | MemInfo: { | 20260507 06:29:29.993 | INFO | "gpu" : " 17 GB", | 20260507 06:29:29.993 | INFO | "huge" : "", | 20260507 06:29:29.993 | INFO | "huge_shared" : " 27 GB", | 20260507 06:29:29.993 | INFO | "malloc" : " 14 GB", | 20260507 06:29:29.993 | INFO | "pinned" : " 29 GB" | 20260507 06:29:29.993 | INFO | } | 20260507 06:29:29.993 | INFO | Perf report for the last 5.13517 seconds: | 20260507 06:29:29.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:29:29.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:29.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:29.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:29.994 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 06:29:29.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:29.994 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:29.994 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 06:29:29.994 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:29.994 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:29.994 | INFO | Uploaded 36GiB to GPU in 2.03068s (17.7281GiB/s) | 20260507 06:29:29.994 | INFO | Downloaded 34GiB to GPU in 1.91935s (17.7144GiB/s) | 20260507 06:29:35.138 | INFO | Analyzing chunk frames = [7818240,7818752) | 20260507 06:29:35.138 | INFO | MemInfo: { | 20260507 06:29:35.138 | INFO | "gpu" : " 17 GB", | 20260507 06:29:35.138 | INFO | "huge" : "", | 20260507 06:29:35.138 | INFO | "huge_shared" : " 27 GB", | 20260507 06:29:35.138 | INFO | "malloc" : " 14 GB", | 20260507 06:29:35.138 | INFO | "pinned" : " 29 GB" | 20260507 06:29:35.138 | INFO | } | 20260507 06:29:35.138 | INFO | Perf report for the last 5.14491 seconds: | 20260507 06:29:35.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:29:35.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:35.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:35.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:35.138 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 06:29:35.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:35.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:35.138 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 06:29:35.139 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:35.139 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:35.139 | INFO | Uploaded 36GiB to GPU in 1.9637s (18.3328GiB/s) | 20260507 06:29:35.139 | INFO | Downloaded 34GiB to GPU in 1.98745s (17.1073GiB/s) | 20260507 06:29:40.270 | INFO | Analyzing chunk frames = [7818752,7819264) | 20260507 06:29:40.270 | INFO | MemInfo: { | 20260507 06:29:40.270 | INFO | "gpu" : " 17 GB", | 20260507 06:29:40.270 | INFO | "huge" : "", | 20260507 06:29:40.270 | INFO | "huge_shared" : " 27 GB", | 20260507 06:29:40.270 | INFO | "malloc" : " 14 GB", | 20260507 06:29:40.270 | INFO | "pinned" : " 29 GB" | 20260507 06:29:40.270 | INFO | } | 20260507 06:29:40.270 | INFO | Perf report for the last 5.13167 seconds: | 20260507 06:29:40.270 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:29:40.270 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:40.270 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:40.270 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:40.270 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:29:40.270 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:40.270 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:40.270 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 06:29:40.270 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:40.270 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:40.270 | INFO | Uploaded 36GiB to GPU in 1.98033s (18.1788GiB/s) | 20260507 06:29:40.270 | INFO | Downloaded 34GiB to GPU in 1.96732s (17.2824GiB/s) | 20260507 06:29:40.367 | INFO | Starting a round of Baz IO | 20260507 06:29:43.468 | INFO | Populating RTMetrics report | 20260507 06:29:43.469 | INFO | RTMetrics report populated | 20260507 06:29:43.469 | INFO | Opening tmp RTMetrics json file | 20260507 06:29:43.469 | INFO | Serializing RTMetrics to json | 20260507 06:29:43.471 | INFO | Serialization done, swapping tmp file now... | 20260507 06:29:43.471 | INFO | RTMetrics done | 20260507 06:29:43.558 | INFO | Finished a round of Baz IO: Wrote 1.666 GiB in 3.191 seconds | 20260507 06:29:43.558 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.62% / 0.00% / 67.38% | 20260507 06:29:43.558 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.121645% / 0% / 7.54992e-05% | 20260507 06:29:43.558 | INFO | Baz IO overall: | 20260507 06:29:43.558 | INFO | BytesWritten : 3942470310274 | 20260507 06:29:43.559 | INFO | NumEvents : 1889231436392 | 20260507 06:29:45.421 | INFO | Analyzing chunk frames = [7819264,7819776) | 20260507 06:29:45.421 | INFO | MemInfo: { | 20260507 06:29:45.421 | INFO | "gpu" : " 17 GB", | 20260507 06:29:45.421 | INFO | "huge" : "", | 20260507 06:29:45.421 | INFO | "huge_shared" : " 27 GB", | 20260507 06:29:45.421 | INFO | "malloc" : " 14 GB", | 20260507 06:29:45.421 | INFO | "pinned" : " 29 GB" | 20260507 06:29:45.421 | INFO | } | 20260507 06:29:45.421 | INFO | Perf report for the last 5.15093 seconds: | 20260507 06:29:45.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:29:45.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:45.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:45.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:45.421 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 06:29:45.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:45.421 | INFO | RT_METRICS : 44.3%, 1.0, 23.3ms, 0 | 20260507 06:29:45.421 | INFO | PRE_HQ : 59.6%, 1.9, 59.7ms, 0 | 20260507 06:29:45.421 | INFO | BAZWRITER : 61.6%, 3.3, 104.5ms, 0 | 20260507 06:29:45.421 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:29:45.422 | INFO | Uploaded 36GiB to GPU in 2.04872s (17.5719GiB/s) | 20260507 06:29:45.422 | INFO | Downloaded 38GiB to GPU in 2.18072s (17.4255GiB/s) | 20260507 06:29:50.557 | INFO | Analyzing chunk frames = [7819776,7820288) | 20260507 06:29:50.557 | INFO | MemInfo: { | 20260507 06:29:50.557 | INFO | "gpu" : " 17 GB", | 20260507 06:29:50.557 | INFO | "huge" : "", | 20260507 06:29:50.557 | INFO | "huge_shared" : " 27 GB", | 20260507 06:29:50.557 | INFO | "malloc" : " 14 GB", | 20260507 06:29:50.557 | INFO | "pinned" : " 29 GB" | 20260507 06:29:50.557 | INFO | } | 20260507 06:29:50.557 | INFO | Perf report for the last 5.13638 seconds: | 20260507 06:29:50.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:29:50.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:50.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:50.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:50.557 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:29:50.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:50.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:50.558 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 06:29:50.558 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:50.558 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:50.558 | INFO | Uploaded 36GiB to GPU in 2.00847s (17.9241GiB/s) | 20260507 06:29:50.558 | INFO | Downloaded 34GiB to GPU in 1.94528s (17.4782GiB/s) | 20260507 06:29:55.693 | INFO | Analyzing chunk frames = [7820288,7820800) | 20260507 06:29:55.693 | INFO | MemInfo: { | 20260507 06:29:55.693 | INFO | "gpu" : " 17 GB", | 20260507 06:29:55.693 | INFO | "huge" : "", | 20260507 06:29:55.693 | INFO | "huge_shared" : " 27 GB", | 20260507 06:29:55.693 | INFO | "malloc" : " 14 GB", | 20260507 06:29:55.693 | INFO | "pinned" : " 29 GB" | 20260507 06:29:55.693 | INFO | } | 20260507 06:29:55.693 | INFO | Perf report for the last 5.13566 seconds: | 20260507 06:29:55.693 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:29:55.693 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:55.693 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:55.693 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:55.693 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 06:29:55.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:55.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:55.693 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:29:55.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:55.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:29:55.693 | INFO | Uploaded 36GiB to GPU in 2.04356s (17.6163GiB/s) | 20260507 06:29:55.693 | INFO | Downloaded 34GiB to GPU in 1.95376s (17.4024GiB/s) | 20260507 06:30:00.834 | INFO | Analyzing chunk frames = [7820800,7821312) | 20260507 06:30:00.834 | INFO | MemInfo: { | 20260507 06:30:00.834 | INFO | "gpu" : " 17 GB", | 20260507 06:30:00.834 | INFO | "huge" : "", | 20260507 06:30:00.834 | INFO | "huge_shared" : " 27 GB", | 20260507 06:30:00.834 | INFO | "malloc" : " 14 GB", | 20260507 06:30:00.834 | INFO | "pinned" : " 29 GB" | 20260507 06:30:00.834 | INFO | } | 20260507 06:30:00.834 | INFO | Perf report for the last 5.14057 seconds: | 20260507 06:30:00.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:30:00.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:00.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:00.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:00.834 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 06:30:00.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:00.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:00.834 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:30:00.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:00.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:00.834 | INFO | Uploaded 36GiB to GPU in 2.0143s (17.8722GiB/s) | 20260507 06:30:00.834 | INFO | Downloaded 34GiB to GPU in 1.94475s (17.483GiB/s) | 20260507 06:30:05.972 | INFO | Analyzing chunk frames = [7821312,7821824) | 20260507 06:30:05.972 | INFO | MemInfo: { | 20260507 06:30:05.972 | INFO | "gpu" : " 17 GB", | 20260507 06:30:05.972 | INFO | "huge" : "", | 20260507 06:30:05.972 | INFO | "huge_shared" : " 27 GB", | 20260507 06:30:05.972 | INFO | "malloc" : " 14 GB", | 20260507 06:30:05.972 | INFO | "pinned" : " 29 GB" | 20260507 06:30:05.972 | INFO | } | 20260507 06:30:05.972 | INFO | Perf report for the last 5.13875 seconds: | 20260507 06:30:05.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:30:05.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:05.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:05.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:05.972 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 06:30:05.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:05.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:05.972 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:30:05.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:05.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:05.973 | INFO | Uploaded 36GiB to GPU in 2.05144s (17.5486GiB/s) | 20260507 06:30:05.973 | INFO | Downloaded 34GiB to GPU in 1.952s (17.418GiB/s) | 20260507 06:30:11.109 | INFO | Analyzing chunk frames = [7821824,7822336) | 20260507 06:30:11.109 | INFO | MemInfo: { | 20260507 06:30:11.109 | INFO | "gpu" : " 17 GB", | 20260507 06:30:11.109 | INFO | "huge" : "", | 20260507 06:30:11.109 | INFO | "huge_shared" : " 27 GB", | 20260507 06:30:11.109 | INFO | "malloc" : " 14 GB", | 20260507 06:30:11.109 | INFO | "pinned" : " 29 GB" | 20260507 06:30:11.109 | INFO | } | 20260507 06:30:11.109 | INFO | Perf report for the last 5.13684 seconds: | 20260507 06:30:11.109 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:30:11.109 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:11.109 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:11.109 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:11.109 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 06:30:11.109 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:11.109 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:11.109 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 06:30:11.109 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:11.109 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:11.110 | INFO | Uploaded 36GiB to GPU in 2.0623s (17.4563GiB/s) | 20260507 06:30:11.110 | INFO | Downloaded 34GiB to GPU in 1.92557s (17.6571GiB/s) | 20260507 06:30:16.243 | INFO | Analyzing chunk frames = [7822336,7822848) | 20260507 06:30:16.243 | INFO | MemInfo: { | 20260507 06:30:16.243 | INFO | "gpu" : " 17 GB", | 20260507 06:30:16.243 | INFO | "huge" : "", | 20260507 06:30:16.243 | INFO | "huge_shared" : " 27 GB", | 20260507 06:30:16.243 | INFO | "malloc" : " 14 GB", | 20260507 06:30:16.243 | INFO | "pinned" : " 29 GB" | 20260507 06:30:16.243 | INFO | } | 20260507 06:30:16.243 | INFO | Perf report for the last 5.13419 seconds: | 20260507 06:30:16.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:30:16.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:16.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:16.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:16.244 | INFO | ANALYSIS : 61.2%, 3.0, 95.1ms, 0 | 20260507 06:30:16.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:16.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:16.244 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 06:30:16.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:16.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:16.244 | INFO | Uploaded 36GiB to GPU in 2.03431s (17.6965GiB/s) | 20260507 06:30:16.244 | INFO | Downloaded 34GiB to GPU in 1.99529s (17.0402GiB/s) | 20260507 06:30:21.392 | INFO | Analyzing chunk frames = [7822848,7823360) | 20260507 06:30:21.392 | INFO | MemInfo: { | 20260507 06:30:21.392 | INFO | "gpu" : " 17 GB", | 20260507 06:30:21.392 | INFO | "huge" : "", | 20260507 06:30:21.392 | INFO | "huge_shared" : " 27 GB", | 20260507 06:30:21.392 | INFO | "malloc" : " 14 GB", | 20260507 06:30:21.392 | INFO | "pinned" : " 29 GB" | 20260507 06:30:21.392 | INFO | } | 20260507 06:30:21.393 | INFO | Perf report for the last 5.14907 seconds: | 20260507 06:30:21.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:30:21.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:21.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:21.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:21.393 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 06:30:21.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:21.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:21.393 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 06:30:21.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:21.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:21.393 | INFO | Uploaded 36GiB to GPU in 1.96219s (18.3469GiB/s) | 20260507 06:30:21.393 | INFO | Downloaded 34GiB to GPU in 1.95976s (17.3491GiB/s) | 20260507 06:30:21.393 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:30:21.393 | INFO | Total mem usage: 18 GiB | 20260507 06:30:21.393 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:30:21.393 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:30:21.393 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:30:21.393 | INFO | High Water Breakdown: | 20260507 06:30:21.393 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:30:21.393 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:30:21.393 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:30:21.393 | INFO | ------------------------------------------ | 20260507 06:30:21.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:30:21.393 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:30:21.393 | INFO | Total mem usage: 26.5 GiB | 20260507 06:30:21.393 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:30:21.393 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.44%) | 20260507 06:30:21.393 | INFO | Unused Cache: 1.6 GiB (Waste: 6.05%) | 20260507 06:30:21.393 | INFO | High Water Breakdown: | 20260507 06:30:21.393 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:30:21.393 | INFO | 36.2% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:30:21.393 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:30:21.393 | INFO | 2% : 525 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:30:21.393 | INFO | 1.46% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:30:21.393 | INFO | 0.695% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:30:21.393 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:30:21.393 | INFO | 9.34e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:30:21.393 | INFO | 8.72e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:30:21.393 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:30:21.393 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:30:21.393 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:30:21.393 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:30:21.393 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:30:21.393 | INFO | ------------------------------------------ | 20260507 06:30:21.394 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:30:21.394 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:30:21.394 | INFO | Total mem usage: 25.6 GiB | 20260507 06:30:21.394 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:30:21.394 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:30:21.394 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:30:21.394 | INFO | High Water Breakdown: | 20260507 06:30:21.394 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:30:21.394 | INFO | ------------------------------------------ | 20260507 06:30:21.394 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:30:21.394 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:30:21.394 | INFO | Total mem usage: 15.7 GiB | 20260507 06:30:21.394 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:30:21.394 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:30:21.394 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:30:21.394 | INFO | High Water Breakdown: | 20260507 06:30:21.394 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:30:21.394 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:30:21.394 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:30:21.394 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:30:21.394 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:30:21.394 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:30:21.394 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:30:21.394 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:30:21.394 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:30:21.394 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:30:21.394 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:30:21.394 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:30:21.394 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:30:21.394 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:30:21.394 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:30:21.394 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:30:21.394 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:30:21.394 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:30:21.394 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:30:21.394 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:30:21.394 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:30:21.394 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:30:21.394 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:30:21.394 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:30:21.394 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:30:21.394 | INFO | ------------------------------------------ | 20260507 06:30:21.394 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:30:21.394 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:30:21.394 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:30:21.394 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:30:21.394 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.43e+01, 1.34e+01 / 1.19e+00 | 20260507 06:30:21.394 | INFO | PerfReport: 0.30%, AggregateStats -- 9.06e-02 / 1.04e-01, 9.44e-03 / 5.90e-03 | 20260507 06:30:21.394 | INFO | PerfReport: 11.66%, Baseline -- 3.57e+00 / 4.48e+00, 4.91e-01 / 1.91e-01 | 20260507 06:30:21.394 | INFO | PerfReport: 3.22%, Histogram -- 9.86e-01 / 1.49e+00, 1.63e-01 / 8.63e-02 | 20260507 06:30:21.394 | INFO | PerfReport: 16.20%, DME -- 3.97e+01 / 4.41e+01, 3.79e+00 / 1.57e+00 | 20260507 06:30:21.394 | INFO | PerfReport: 57.73%, FrameLabeling -- 1.77e+01 / 1.99e+01, 1.26e+00 / 1.54e+00 | 20260507 06:30:21.394 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.70e+00, 3.85e-01 / 2.12e-01 | 20260507 06:30:21.394 | INFO | PerfReport: 2.96%, Metrics -- 9.07e-01 / 1.19e+00, 2.44e-01 / 1.20e-01 | 20260507 06:30:21.394 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.05e-01 / 3.57e-01, 5.01e-02 / 1.45e-02 | 20260507 06:30:21.394 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:30:21.394 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:30:21.394 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:30:21.394 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 06:30:21.394 | INFO | PerfReport: 21.91%, Upload -- 2.11e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260507 06:30:21.394 | INFO | PerfReport: 21.44%, Download -- 2.06e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 06:30:21.394 | INFO | PerfReport: 31.89%, Compute -- 3.06e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:30:21.394 | INFO | PerfReport: 1.43%, LockUpload -- 1.38e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260507 06:30:21.394 | INFO | PerfReport: 1.38%, LockDownload -- 1.32e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260507 06:30:21.394 | INFO | PerfReport: 21.94%, LockCompute -- 2.11e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 06:30:21.394 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.92e-02 / 0.00e+00 | 20260507 06:30:21.490 | INFO | Starting a round of Baz IO | 20260507 06:30:24.583 | INFO | Populating RTMetrics report | 20260507 06:30:24.583 | INFO | RTMetrics report populated | 20260507 06:30:24.584 | INFO | Opening tmp RTMetrics json file | 20260507 06:30:24.599 | INFO | Serializing RTMetrics to json | 20260507 06:30:24.600 | INFO | Serialization done, swapping tmp file now... | 20260507 06:30:24.600 | INFO | RTMetrics done | 20260507 06:30:24.666 | INFO | Finished a round of Baz IO: Wrote 1.668 GiB in 3.176 seconds | 20260507 06:30:24.666 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.62% / 0.00% / 67.38% | 20260507 06:30:24.666 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.121963% / 3.97364e-06% / 7.15256e-05% | 20260507 06:30:24.666 | INFO | Baz IO overall: | 20260507 06:30:24.666 | INFO | BytesWritten : 3944261220774 | 20260507 06:30:24.667 | INFO | NumEvents : 1890069946112 | 20260507 06:30:26.532 | INFO | Analyzing chunk frames = [7823360,7823872) | 20260507 06:30:26.532 | INFO | MemInfo: { | 20260507 06:30:26.532 | INFO | "gpu" : " 17 GB", | 20260507 06:30:26.532 | INFO | "huge" : "", | 20260507 06:30:26.532 | INFO | "huge_shared" : " 27 GB", | 20260507 06:30:26.532 | INFO | "malloc" : " 14 GB", | 20260507 06:30:26.532 | INFO | "pinned" : " 29 GB" | 20260507 06:30:26.532 | INFO | } | 20260507 06:30:26.532 | INFO | Perf report for the last 5.13953 seconds: | 20260507 06:30:26.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:30:26.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:26.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:26.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:26.532 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 06:30:26.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:26.532 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:30:26.532 | INFO | PRE_HQ : 58.9%, 1.8, 54.7ms, 0 | 20260507 06:30:26.532 | INFO | BAZWRITER : 61.4%, 3.1, 98.9ms, 0 | 20260507 06:30:26.532 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:30:26.532 | INFO | Uploaded 36GiB to GPU in 2.03702s (17.6728GiB/s) | 20260507 06:30:26.532 | INFO | Downloaded 38GiB to GPU in 2.18172s (17.4175GiB/s) | 20260507 06:30:31.672 | INFO | Analyzing chunk frames = [7823872,7824384) | 20260507 06:30:31.672 | INFO | MemInfo: { | 20260507 06:30:31.672 | INFO | "gpu" : " 17 GB", | 20260507 06:30:31.672 | INFO | "huge" : "", | 20260507 06:30:31.672 | INFO | "huge_shared" : " 27 GB", | 20260507 06:30:31.672 | INFO | "malloc" : " 14 GB", | 20260507 06:30:31.672 | INFO | "pinned" : " 29 GB" | 20260507 06:30:31.672 | INFO | } | 20260507 06:30:31.672 | INFO | Perf report for the last 5.14007 seconds: | 20260507 06:30:31.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:30:31.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:31.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:31.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:31.672 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:30:31.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:31.672 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:31.672 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:30:31.672 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:31.672 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:31.672 | INFO | Uploaded 36GiB to GPU in 1.99121s (18.0795GiB/s) | 20260507 06:30:31.672 | INFO | Downloaded 34GiB to GPU in 1.93886s (17.5361GiB/s) | 20260507 06:30:36.802 | INFO | Analyzing chunk frames = [7824384,7824896) | 20260507 06:30:36.802 | INFO | MemInfo: { | 20260507 06:30:36.802 | INFO | "gpu" : " 17 GB", | 20260507 06:30:36.802 | INFO | "huge" : "", | 20260507 06:30:36.802 | INFO | "huge_shared" : " 27 GB", | 20260507 06:30:36.802 | INFO | "malloc" : " 14 GB", | 20260507 06:30:36.802 | INFO | "pinned" : " 29 GB" | 20260507 06:30:36.802 | INFO | } | 20260507 06:30:36.803 | INFO | Perf report for the last 5.13043 seconds: | 20260507 06:30:36.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:30:36.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:36.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:36.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:36.803 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:30:36.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:36.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:36.803 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:30:36.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:36.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:36.803 | INFO | Uploaded 36GiB to GPU in 2.02678s (17.7622GiB/s) | 20260507 06:30:36.803 | INFO | Downloaded 34GiB to GPU in 1.92145s (17.695GiB/s) | 20260507 06:30:41.944 | INFO | Analyzing chunk frames = [7824896,7825408) | 20260507 06:30:41.945 | INFO | MemInfo: { | 20260507 06:30:41.945 | INFO | "gpu" : " 17 GB", | 20260507 06:30:41.945 | INFO | "huge" : "", | 20260507 06:30:41.945 | INFO | "huge_shared" : " 27 GB", | 20260507 06:30:41.945 | INFO | "malloc" : " 14 GB", | 20260507 06:30:41.945 | INFO | "pinned" : " 29 GB" | 20260507 06:30:41.945 | INFO | } | 20260507 06:30:41.945 | INFO | Perf report for the last 5.14218 seconds: | 20260507 06:30:41.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:30:41.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:41.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:41.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:41.945 | INFO | ANALYSIS : 59.5%, 3.0, 92.5ms, 0 | 20260507 06:30:41.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:41.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:41.945 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:30:41.945 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:41.945 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:41.945 | INFO | Uploaded 36GiB to GPU in 1.98638s (18.1234GiB/s) | 20260507 06:30:41.945 | INFO | Downloaded 34GiB to GPU in 1.93966s (17.5289GiB/s) | 20260507 06:30:47.089 | INFO | Analyzing chunk frames = [7825408,7825920) | 20260507 06:30:47.089 | INFO | MemInfo: { | 20260507 06:30:47.089 | INFO | "gpu" : " 17 GB", | 20260507 06:30:47.089 | INFO | "huge" : "", | 20260507 06:30:47.089 | INFO | "huge_shared" : " 27 GB", | 20260507 06:30:47.089 | INFO | "malloc" : " 14 GB", | 20260507 06:30:47.089 | INFO | "pinned" : " 29 GB" | 20260507 06:30:47.089 | INFO | } | 20260507 06:30:47.090 | INFO | Perf report for the last 5.14473 seconds: | 20260507 06:30:47.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:30:47.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:47.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:47.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:47.090 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 06:30:47.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:47.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:47.090 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:30:47.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:47.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:47.090 | INFO | Uploaded 36GiB to GPU in 2.04167s (17.6326GiB/s) | 20260507 06:30:47.090 | INFO | Downloaded 34GiB to GPU in 1.94578s (17.4737GiB/s) | 20260507 06:30:52.226 | INFO | Analyzing chunk frames = [7825920,7826432) | 20260507 06:30:52.226 | INFO | MemInfo: { | 20260507 06:30:52.226 | INFO | "gpu" : " 17 GB", | 20260507 06:30:52.226 | INFO | "huge" : "", | 20260507 06:30:52.226 | INFO | "huge_shared" : " 27 GB", | 20260507 06:30:52.226 | INFO | "malloc" : " 14 GB", | 20260507 06:30:52.226 | INFO | "pinned" : " 29 GB" | 20260507 06:30:52.226 | INFO | } | 20260507 06:30:52.226 | INFO | Perf report for the last 5.13639 seconds: | 20260507 06:30:52.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:30:52.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:52.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:52.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:52.226 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 06:30:52.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:52.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:52.226 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:30:52.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:52.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:52.226 | INFO | Uploaded 36GiB to GPU in 2.02503s (17.7776GiB/s) | 20260507 06:30:52.226 | INFO | Downloaded 34GiB to GPU in 1.91421s (17.7619GiB/s) | 20260507 06:30:57.368 | INFO | Analyzing chunk frames = [7826432,7826944) | 20260507 06:30:57.368 | INFO | MemInfo: { | 20260507 06:30:57.368 | INFO | "gpu" : " 17 GB", | 20260507 06:30:57.368 | INFO | "huge" : "", | 20260507 06:30:57.368 | INFO | "huge_shared" : " 27 GB", | 20260507 06:30:57.368 | INFO | "malloc" : " 14 GB", | 20260507 06:30:57.368 | INFO | "pinned" : " 29 GB" | 20260507 06:30:57.368 | INFO | } | 20260507 06:30:57.368 | INFO | Perf report for the last 5.14226 seconds: | 20260507 06:30:57.368 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:30:57.368 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:57.368 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:57.368 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:57.368 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 06:30:57.368 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:57.368 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:57.368 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 06:30:57.368 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:57.368 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:30:57.368 | INFO | Uploaded 36GiB to GPU in 1.88408s (19.1074GiB/s) | 20260507 06:30:57.368 | INFO | Downloaded 34GiB to GPU in 1.97721s (17.1959GiB/s) | 20260507 06:31:02.505 | INFO | Analyzing chunk frames = [7826944,7827456) | 20260507 06:31:02.505 | INFO | MemInfo: { | 20260507 06:31:02.505 | INFO | "gpu" : " 17 GB", | 20260507 06:31:02.505 | INFO | "huge" : "", | 20260507 06:31:02.505 | INFO | "huge_shared" : " 27 GB", | 20260507 06:31:02.505 | INFO | "malloc" : " 14 GB", | 20260507 06:31:02.505 | INFO | "pinned" : " 29 GB" | 20260507 06:31:02.505 | INFO | } | 20260507 06:31:02.505 | INFO | Perf report for the last 5.13717 seconds: | 20260507 06:31:02.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:31:02.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:02.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:02.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:02.505 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:31:02.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:02.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:02.506 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 06:31:02.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:02.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:02.506 | INFO | Uploaded 36GiB to GPU in 1.98031s (18.179GiB/s) | 20260507 06:31:02.506 | INFO | Downloaded 34GiB to GPU in 1.96054s (17.3421GiB/s) | 20260507 06:31:02.602 | INFO | Starting a round of Baz IO | 20260507 06:31:05.697 | INFO | Populating RTMetrics report | 20260507 06:31:05.697 | INFO | RTMetrics report populated | 20260507 06:31:05.697 | INFO | Opening tmp RTMetrics json file | 20260507 06:31:05.705 | INFO | Serializing RTMetrics to json | 20260507 06:31:05.714 | INFO | Serialization done, swapping tmp file now... | 20260507 06:31:05.714 | INFO | RTMetrics done | 20260507 06:31:05.782 | INFO | Finished a round of Baz IO: Wrote 1.666 GiB in 3.179 seconds | 20260507 06:31:05.782 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.62% / 0.00% / 67.38% | 20260507 06:31:05.782 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.12145% / 7.94729e-06% / 5.96046e-05% | 20260507 06:31:05.782 | INFO | Baz IO overall: | 20260507 06:31:05.782 | INFO | BytesWritten : 3946049634769 | 20260507 06:31:05.782 | INFO | NumEvents : 1890906662672 | 20260507 06:31:07.642 | INFO | Analyzing chunk frames = [7827456,7827968) | 20260507 06:31:07.642 | INFO | MemInfo: { | 20260507 06:31:07.642 | INFO | "gpu" : " 17 GB", | 20260507 06:31:07.642 | INFO | "huge" : "", | 20260507 06:31:07.642 | INFO | "huge_shared" : " 27 GB", | 20260507 06:31:07.642 | INFO | "malloc" : " 14 GB", | 20260507 06:31:07.642 | INFO | "pinned" : " 29 GB" | 20260507 06:31:07.642 | INFO | } | 20260507 06:31:07.642 | INFO | Perf report for the last 5.13674 seconds: | 20260507 06:31:07.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:31:07.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:07.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:07.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:07.642 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 06:31:07.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:07.642 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 06:31:07.642 | INFO | PRE_HQ : 58.9%, 1.8, 54.7ms, 0 | 20260507 06:31:07.642 | INFO | BAZWRITER : 61.5%, 3.1, 98.2ms, 0 | 20260507 06:31:07.642 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:31:07.642 | INFO | Uploaded 36GiB to GPU in 2.05551s (17.5139GiB/s) | 20260507 06:31:07.642 | INFO | Downloaded 38GiB to GPU in 2.18339s (17.4041GiB/s) | 20260507 06:31:12.782 | INFO | Analyzing chunk frames = [7827968,7828480) | 20260507 06:31:12.782 | INFO | MemInfo: { | 20260507 06:31:12.782 | INFO | "gpu" : " 17 GB", | 20260507 06:31:12.782 | INFO | "huge" : "", | 20260507 06:31:12.782 | INFO | "huge_shared" : " 27 GB", | 20260507 06:31:12.782 | INFO | "malloc" : " 14 GB", | 20260507 06:31:12.782 | INFO | "pinned" : " 29 GB" | 20260507 06:31:12.782 | INFO | } | 20260507 06:31:12.782 | INFO | Perf report for the last 5.14028 seconds: | 20260507 06:31:12.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:31:12.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:12.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:12.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:12.783 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:31:12.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:12.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:12.783 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 06:31:12.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:12.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:12.783 | INFO | Uploaded 36GiB to GPU in 1.9802s (18.18GiB/s) | 20260507 06:31:12.783 | INFO | Downloaded 34GiB to GPU in 1.93524s (17.5689GiB/s) | 20260507 06:31:17.926 | INFO | Analyzing chunk frames = [7828480,7828992) | 20260507 06:31:17.926 | INFO | MemInfo: { | 20260507 06:31:17.926 | INFO | "gpu" : " 17 GB", | 20260507 06:31:17.926 | INFO | "huge" : "", | 20260507 06:31:17.926 | INFO | "huge_shared" : " 27 GB", | 20260507 06:31:17.926 | INFO | "malloc" : " 14 GB", | 20260507 06:31:17.926 | INFO | "pinned" : " 29 GB" | 20260507 06:31:17.926 | INFO | } | 20260507 06:31:17.926 | INFO | Perf report for the last 5.14357 seconds: | 20260507 06:31:17.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:31:17.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:17.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:17.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:17.926 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 06:31:17.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:17.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:17.926 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:31:17.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:17.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:17.926 | INFO | Uploaded 36GiB to GPU in 2.04057s (17.6421GiB/s) | 20260507 06:31:17.926 | INFO | Downloaded 34GiB to GPU in 1.95384s (17.4016GiB/s) | 20260507 06:31:23.056 | INFO | Analyzing chunk frames = [7828992,7829504) | 20260507 06:31:23.056 | INFO | MemInfo: { | 20260507 06:31:23.056 | INFO | "gpu" : " 17 GB", | 20260507 06:31:23.056 | INFO | "huge" : "", | 20260507 06:31:23.056 | INFO | "huge_shared" : " 27 GB", | 20260507 06:31:23.056 | INFO | "malloc" : " 14 GB", | 20260507 06:31:23.056 | INFO | "pinned" : " 29 GB" | 20260507 06:31:23.056 | INFO | } | 20260507 06:31:23.057 | INFO | Perf report for the last 5.13054 seconds: | 20260507 06:31:23.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:31:23.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:23.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:23.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:23.057 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 06:31:23.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:23.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:23.057 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:31:23.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:23.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:23.057 | INFO | Uploaded 36GiB to GPU in 2.04687s (17.5879GiB/s) | 20260507 06:31:23.057 | INFO | Downloaded 34GiB to GPU in 1.95451s (17.3957GiB/s) | 20260507 06:31:28.196 | INFO | Analyzing chunk frames = [7829504,7830016) | 20260507 06:31:28.196 | INFO | MemInfo: { | 20260507 06:31:28.196 | INFO | "gpu" : " 17 GB", | 20260507 06:31:28.196 | INFO | "huge" : "", | 20260507 06:31:28.196 | INFO | "huge_shared" : " 27 GB", | 20260507 06:31:28.196 | INFO | "malloc" : " 14 GB", | 20260507 06:31:28.196 | INFO | "pinned" : " 29 GB" | 20260507 06:31:28.196 | INFO | } | 20260507 06:31:28.196 | INFO | Perf report for the last 5.13923 seconds: | 20260507 06:31:28.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:31:28.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:28.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:28.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:28.196 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 06:31:28.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:28.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:28.196 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:31:28.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:28.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:28.196 | INFO | Uploaded 36GiB to GPU in 2.04869s (17.5722GiB/s) | 20260507 06:31:28.196 | INFO | Downloaded 34GiB to GPU in 1.94574s (17.4741GiB/s) | 20260507 06:31:33.338 | INFO | Analyzing chunk frames = [7830016,7830528) | 20260507 06:31:33.338 | INFO | MemInfo: { | 20260507 06:31:33.338 | INFO | "gpu" : " 17 GB", | 20260507 06:31:33.338 | INFO | "huge" : "", | 20260507 06:31:33.338 | INFO | "huge_shared" : " 27 GB", | 20260507 06:31:33.338 | INFO | "malloc" : " 14 GB", | 20260507 06:31:33.338 | INFO | "pinned" : " 29 GB" | 20260507 06:31:33.338 | INFO | } | 20260507 06:31:33.338 | INFO | Perf report for the last 5.14206 seconds: | 20260507 06:31:33.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:31:33.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:33.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:33.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:33.338 | INFO | ANALYSIS : 60.4%, 3.0, 94.4ms, 0 | 20260507 06:31:33.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:33.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:33.338 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:31:33.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:33.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:33.338 | INFO | Uploaded 36GiB to GPU in 2.02817s (17.75GiB/s) | 20260507 06:31:33.338 | INFO | Downloaded 34GiB to GPU in 1.9354s (17.5675GiB/s) | 20260507 06:31:38.485 | INFO | Analyzing chunk frames = [7830528,7831040) | 20260507 06:31:38.485 | INFO | MemInfo: { | 20260507 06:31:38.485 | INFO | "gpu" : " 17 GB", | 20260507 06:31:38.485 | INFO | "huge" : "", | 20260507 06:31:38.485 | INFO | "huge_shared" : " 27 GB", | 20260507 06:31:38.485 | INFO | "malloc" : " 14 GB", | 20260507 06:31:38.485 | INFO | "pinned" : " 29 GB" | 20260507 06:31:38.485 | INFO | } | 20260507 06:31:38.485 | INFO | Perf report for the last 5.14741 seconds: | 20260507 06:31:38.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:31:38.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:38.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:38.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:38.485 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 06:31:38.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:38.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:38.485 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 06:31:38.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:38.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:38.486 | INFO | Uploaded 36GiB to GPU in 1.98143s (18.1687GiB/s) | 20260507 06:31:38.486 | INFO | Downloaded 34GiB to GPU in 1.98908s (17.0934GiB/s) | 20260507 06:31:43.618 | INFO | Analyzing chunk frames = [7831040,7831552) | 20260507 06:31:43.618 | INFO | MemInfo: { | 20260507 06:31:43.618 | INFO | "gpu" : " 17 GB", | 20260507 06:31:43.618 | INFO | "huge" : "", | 20260507 06:31:43.618 | INFO | "huge_shared" : " 27 GB", | 20260507 06:31:43.618 | INFO | "malloc" : " 14 GB", | 20260507 06:31:43.618 | INFO | "pinned" : " 29 GB" | 20260507 06:31:43.618 | INFO | } | 20260507 06:31:43.618 | INFO | Perf report for the last 5.13307 seconds: | 20260507 06:31:43.618 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:31:43.618 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:43.618 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:43.618 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:43.618 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:31:43.618 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:43.618 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:43.618 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:31:43.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:43.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:43.619 | INFO | Uploaded 36GiB to GPU in 1.99686s (18.0283GiB/s) | 20260507 06:31:43.619 | INFO | Downloaded 34GiB to GPU in 1.96997s (17.2591GiB/s) | 20260507 06:31:43.619 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:31:43.619 | INFO | Total mem usage: 18 GiB | 20260507 06:31:43.619 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:31:43.619 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 06:31:43.619 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:31:43.619 | INFO | High Water Breakdown: | 20260507 06:31:43.619 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:31:43.619 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:31:43.619 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:31:43.619 | INFO | ------------------------------------------ | 20260507 06:31:43.619 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:31:43.619 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:31:43.619 | INFO | Total mem usage: 26.5 GiB | 20260507 06:31:43.619 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:31:43.619 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:31:43.619 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:31:43.619 | INFO | High Water Breakdown: | 20260507 06:31:43.619 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:31:43.619 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:31:43.619 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:31:43.619 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:31:43.619 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:31:43.619 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:31:43.619 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:31:43.619 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:31:43.619 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:31:43.619 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:31:43.619 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:31:43.619 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:31:43.619 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:31:43.619 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:31:43.619 | INFO | ------------------------------------------ | 20260507 06:31:43.619 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:31:43.619 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:31:43.619 | INFO | Total mem usage: 25.6 GiB | 20260507 06:31:43.619 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:31:43.619 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:31:43.619 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:31:43.619 | INFO | High Water Breakdown: | 20260507 06:31:43.620 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:31:43.620 | INFO | ------------------------------------------ | 20260507 06:31:43.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:31:43.620 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:31:43.620 | INFO | Total mem usage: 15.7 GiB | 20260507 06:31:43.620 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:31:43.620 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:31:43.620 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:31:43.620 | INFO | High Water Breakdown: | 20260507 06:31:43.620 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:31:43.620 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:31:43.620 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:31:43.620 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:31:43.620 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:31:43.620 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:31:43.620 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:31:43.620 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:31:43.620 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:31:43.620 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:31:43.620 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:31:43.620 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:31:43.620 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:31:43.620 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:31:43.620 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:31:43.620 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:31:43.620 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:31:43.620 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:31:43.620 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:31:43.620 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:31:43.620 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:31:43.620 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:31:43.620 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:31:43.620 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:31:43.620 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:31:43.620 | INFO | ------------------------------------------ | 20260507 06:31:43.620 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:31:43.620 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:31:43.620 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:31:43.620 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:31:43.620 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.33e+01 / 1.20e+00 | 20260507 06:31:43.620 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.02e-01, 1.01e-02 / 3.80e-03 | 20260507 06:31:43.620 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.48e+00, 5.11e-01 / 2.14e-01 | 20260507 06:31:43.620 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.51e+00, 1.68e-01 / 9.33e-02 | 20260507 06:31:43.620 | INFO | PerfReport: 16.28%, DME -- 4.00e+01 / 4.49e+01, 4.02e+00 / 1.34e+00 | 20260507 06:31:43.620 | INFO | PerfReport: 57.60%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.19e+00 / 1.19e+00 | 20260507 06:31:43.620 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.13e+00 / 2.74e+00, 3.89e-01 / 1.32e-01 | 20260507 06:31:43.620 | INFO | PerfReport: 2.96%, Metrics -- 9.09e-01 / 1.25e+00, 2.46e-01 / 1.18e-01 | 20260507 06:31:43.620 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.05e-01 / 3.58e-01, 5.02e-02 / 1.49e-02 | 20260507 06:31:43.620 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:31:43.620 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:31:43.620 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:31:43.620 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 06:31:43.620 | INFO | PerfReport: 21.78%, Upload -- 2.09e+01 / 0.00e+00, 7.05e+00 / 0.00e+00 | 20260507 06:31:43.620 | INFO | PerfReport: 21.43%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260507 06:31:43.620 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 06:31:43.620 | INFO | PerfReport: 1.44%, LockUpload -- 1.38e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 06:31:43.620 | INFO | PerfReport: 1.43%, LockDownload -- 1.37e+00 / 0.00e+00, 3.26e+00 / 0.00e+00 | 20260507 06:31:43.620 | INFO | PerfReport: 21.99%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 06:31:43.620 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260507 06:31:43.716 | INFO | Starting a round of Baz IO | 20260507 06:31:46.811 | INFO | Populating RTMetrics report | 20260507 06:31:46.811 | INFO | RTMetrics report populated | 20260507 06:31:46.811 | INFO | Opening tmp RTMetrics json file | 20260507 06:31:46.829 | INFO | Serializing RTMetrics to json | 20260507 06:31:46.830 | INFO | Serialization done, swapping tmp file now... | 20260507 06:31:46.830 | INFO | RTMetrics done | 20260507 06:31:46.897 | INFO | Finished a round of Baz IO: Wrote 1.665 GiB in 3.181 seconds | 20260507 06:31:46.897 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.62% / 0.00% / 67.38% | 20260507 06:31:46.897 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.12085% / 0% / 7.15256e-05% | 20260507 06:31:46.897 | INFO | Baz IO overall: | 20260507 06:31:46.897 | INFO | BytesWritten : 3947837294994 | 20260507 06:31:46.898 | INFO | NumEvents : 1891742735721 | 20260507 06:31:48.751 | INFO | Analyzing chunk frames = [7831552,7832064) | 20260507 06:31:48.751 | INFO | MemInfo: { | 20260507 06:31:48.751 | INFO | "gpu" : " 17 GB", | 20260507 06:31:48.751 | INFO | "huge" : "", | 20260507 06:31:48.751 | INFO | "huge_shared" : " 27 GB", | 20260507 06:31:48.751 | INFO | "malloc" : " 14 GB", | 20260507 06:31:48.751 | INFO | "pinned" : " 29 GB" | 20260507 06:31:48.751 | INFO | } | 20260507 06:31:48.752 | INFO | Perf report for the last 5.13316 seconds: | 20260507 06:31:48.752 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:31:48.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:48.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:48.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:48.752 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 06:31:48.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:48.752 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 06:31:48.752 | INFO | PRE_HQ : 58.9%, 1.8, 54.6ms, 0 | 20260507 06:31:48.752 | INFO | BAZWRITER : 61.6%, 3.1, 99.0ms, 0 | 20260507 06:31:48.752 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:31:48.752 | INFO | Uploaded 36GiB to GPU in 2.05722s (17.4994GiB/s) | 20260507 06:31:48.752 | INFO | Downloaded 38GiB to GPU in 2.17541s (17.468GiB/s) | 20260507 06:31:53.898 | INFO | Analyzing chunk frames = [7832064,7832576) | 20260507 06:31:53.898 | INFO | MemInfo: { | 20260507 06:31:53.898 | INFO | "gpu" : " 17 GB", | 20260507 06:31:53.898 | INFO | "huge" : "", | 20260507 06:31:53.898 | INFO | "huge_shared" : " 27 GB", | 20260507 06:31:53.898 | INFO | "malloc" : " 14 GB", | 20260507 06:31:53.898 | INFO | "pinned" : " 29 GB" | 20260507 06:31:53.898 | INFO | } | 20260507 06:31:53.898 | INFO | Perf report for the last 5.1468 seconds: | 20260507 06:31:53.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:31:53.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:53.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:53.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:53.898 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:31:53.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:53.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:53.898 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 06:31:53.899 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:53.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:53.899 | INFO | Uploaded 36GiB to GPU in 1.98686s (18.1191GiB/s) | 20260507 06:31:53.899 | INFO | Downloaded 34GiB to GPU in 1.93151s (17.6028GiB/s) | 20260507 06:31:59.044 | INFO | Analyzing chunk frames = [7832576,7833088) | 20260507 06:31:59.044 | INFO | MemInfo: { | 20260507 06:31:59.044 | INFO | "gpu" : " 17 GB", | 20260507 06:31:59.044 | INFO | "huge" : "", | 20260507 06:31:59.044 | INFO | "huge_shared" : " 27 GB", | 20260507 06:31:59.044 | INFO | "malloc" : " 14 GB", | 20260507 06:31:59.044 | INFO | "pinned" : " 29 GB" | 20260507 06:31:59.044 | INFO | } | 20260507 06:31:59.044 | INFO | Perf report for the last 5.14529 seconds: | 20260507 06:31:59.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:31:59.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:59.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:59.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:59.044 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 06:31:59.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:59.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:59.044 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:31:59.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:59.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:31:59.044 | INFO | Uploaded 36GiB to GPU in 2.04608s (17.5946GiB/s) | 20260507 06:31:59.044 | INFO | Downloaded 34GiB to GPU in 1.95207s (17.4174GiB/s) | 20260507 06:32:04.177 | INFO | Analyzing chunk frames = [7833088,7833600) | 20260507 06:32:04.177 | INFO | MemInfo: { | 20260507 06:32:04.177 | INFO | "gpu" : " 17 GB", | 20260507 06:32:04.177 | INFO | "huge" : "", | 20260507 06:32:04.177 | INFO | "huge_shared" : " 27 GB", | 20260507 06:32:04.177 | INFO | "malloc" : " 14 GB", | 20260507 06:32:04.178 | INFO | "pinned" : " 29 GB" | 20260507 06:32:04.178 | INFO | } | 20260507 06:32:04.178 | INFO | Perf report for the last 5.13393 seconds: | 20260507 06:32:04.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:32:04.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:04.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:04.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:04.178 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 06:32:04.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:04.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:04.178 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:32:04.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:04.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:04.178 | INFO | Uploaded 36GiB to GPU in 2.04716s (17.5853GiB/s) | 20260507 06:32:04.178 | INFO | Downloaded 34GiB to GPU in 1.94778s (17.4557GiB/s) | 20260507 06:32:09.312 | INFO | Analyzing chunk frames = [7833600,7834112) | 20260507 06:32:09.312 | INFO | MemInfo: { | 20260507 06:32:09.312 | INFO | "gpu" : " 17 GB", | 20260507 06:32:09.312 | INFO | "huge" : "", | 20260507 06:32:09.312 | INFO | "huge_shared" : " 27 GB", | 20260507 06:32:09.312 | INFO | "malloc" : " 14 GB", | 20260507 06:32:09.312 | INFO | "pinned" : " 29 GB" | 20260507 06:32:09.312 | INFO | } | 20260507 06:32:09.312 | INFO | Perf report for the last 5.13483 seconds: | 20260507 06:32:09.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:32:09.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:09.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:09.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:09.313 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 06:32:09.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:09.313 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:09.313 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:32:09.313 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:09.313 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:09.313 | INFO | Uploaded 36GiB to GPU in 2.03872s (17.6581GiB/s) | 20260507 06:32:09.313 | INFO | Downloaded 34GiB to GPU in 1.91886s (17.7188GiB/s) | 20260507 06:32:14.457 | INFO | Analyzing chunk frames = [7834112,7834624) | 20260507 06:32:14.457 | INFO | MemInfo: { | 20260507 06:32:14.457 | INFO | "gpu" : " 17 GB", | 20260507 06:32:14.457 | INFO | "huge" : "", | 20260507 06:32:14.457 | INFO | "huge_shared" : " 27 GB", | 20260507 06:32:14.457 | INFO | "malloc" : " 14 GB", | 20260507 06:32:14.457 | INFO | "pinned" : " 29 GB" | 20260507 06:32:14.457 | INFO | } | 20260507 06:32:14.457 | INFO | Perf report for the last 5.14439 seconds: | 20260507 06:32:14.457 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:32:14.457 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:14.457 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:14.457 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:14.457 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 06:32:14.457 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:14.457 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:14.457 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 06:32:14.457 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:14.457 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:14.457 | INFO | Uploaded 36GiB to GPU in 2.03318s (17.7062GiB/s) | 20260507 06:32:14.457 | INFO | Downloaded 34GiB to GPU in 1.90601s (17.8383GiB/s) | 20260507 06:32:19.593 | INFO | Analyzing chunk frames = [7834624,7835136) | 20260507 06:32:19.593 | INFO | MemInfo: { | 20260507 06:32:19.593 | INFO | "gpu" : " 17 GB", | 20260507 06:32:19.594 | INFO | "huge" : "", | 20260507 06:32:19.594 | INFO | "huge_shared" : " 27 GB", | 20260507 06:32:19.594 | INFO | "malloc" : " 14 GB", | 20260507 06:32:19.594 | INFO | "pinned" : " 29 GB" | 20260507 06:32:19.594 | INFO | } | 20260507 06:32:19.594 | INFO | Perf report for the last 5.13675 seconds: | 20260507 06:32:19.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:32:19.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:19.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:19.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:19.594 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 06:32:19.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:19.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:19.594 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:32:19.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:19.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:19.594 | INFO | Uploaded 36GiB to GPU in 2.05019s (17.5594GiB/s) | 20260507 06:32:19.594 | INFO | Downloaded 34GiB to GPU in 1.98896s (17.0944GiB/s) | 20260507 06:32:24.735 | INFO | Analyzing chunk frames = [7835136,7835648) | 20260507 06:32:24.735 | INFO | MemInfo: { | 20260507 06:32:24.735 | INFO | "gpu" : " 17 GB", | 20260507 06:32:24.735 | INFO | "huge" : "", | 20260507 06:32:24.735 | INFO | "huge_shared" : " 27 GB", | 20260507 06:32:24.735 | INFO | "malloc" : " 14 GB", | 20260507 06:32:24.735 | INFO | "pinned" : " 29 GB" | 20260507 06:32:24.735 | INFO | } | 20260507 06:32:24.735 | INFO | Perf report for the last 5.14117 seconds: | 20260507 06:32:24.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:32:24.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:24.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:24.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:24.735 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:32:24.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:24.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:24.735 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 06:32:24.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:24.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:24.735 | INFO | Uploaded 36GiB to GPU in 1.98095s (18.1731GiB/s) | 20260507 06:32:24.735 | INFO | Downloaded 34GiB to GPU in 1.96431s (17.3088GiB/s) | 20260507 06:32:24.832 | INFO | Starting a round of Baz IO | 20260507 06:32:27.927 | INFO | Populating RTMetrics report | 20260507 06:32:27.927 | INFO | RTMetrics report populated | 20260507 06:32:27.927 | INFO | Opening tmp RTMetrics json file | 20260507 06:32:27.931 | INFO | Serializing RTMetrics to json | 20260507 06:32:27.945 | INFO | Serialization done, swapping tmp file now... | 20260507 06:32:27.945 | INFO | RTMetrics done | 20260507 06:32:28.011 | INFO | Finished a round of Baz IO: Wrote 1.664 GiB in 3.179 seconds | 20260507 06:32:28.011 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.62% / 0.00% / 67.38% | 20260507 06:32:28.011 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.120548% / 0% / 9.53674e-05% | 20260507 06:32:28.011 | INFO | Baz IO overall: | 20260507 06:32:28.011 | INFO | BytesWritten : 3949624053310 | 20260507 06:32:28.011 | INFO | NumEvents : 1892578374022 | 20260507 06:32:29.882 | INFO | Analyzing chunk frames = [7835648,7836160) | 20260507 06:32:29.882 | INFO | MemInfo: { | 20260507 06:32:29.882 | INFO | "gpu" : " 17 GB", | 20260507 06:32:29.882 | INFO | "huge" : "", | 20260507 06:32:29.882 | INFO | "huge_shared" : " 27 GB", | 20260507 06:32:29.882 | INFO | "malloc" : " 14 GB", | 20260507 06:32:29.882 | INFO | "pinned" : " 29 GB" | 20260507 06:32:29.882 | INFO | } | 20260507 06:32:29.882 | INFO | Perf report for the last 5.14728 seconds: | 20260507 06:32:29.882 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:32:29.882 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:29.882 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:29.882 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:29.882 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 06:32:29.882 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:29.882 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260507 06:32:29.882 | INFO | PRE_HQ : 59.8%, 1.9, 58.9ms, 0 | 20260507 06:32:29.882 | INFO | BAZWRITER : 61.4%, 3.3, 103.3ms, 0 | 20260507 06:32:29.882 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 06:32:29.882 | INFO | Uploaded 36GiB to GPU in 2.06107s (17.4667GiB/s) | 20260507 06:32:29.883 | INFO | Downloaded 38GiB to GPU in 2.18271s (17.4096GiB/s) | 20260507 06:32:35.011 | INFO | Analyzing chunk frames = [7836160,7836672) | 20260507 06:32:35.011 | INFO | MemInfo: { | 20260507 06:32:35.011 | INFO | "gpu" : " 17 GB", | 20260507 06:32:35.011 | INFO | "huge" : "", | 20260507 06:32:35.011 | INFO | "huge_shared" : " 27 GB", | 20260507 06:32:35.011 | INFO | "malloc" : " 14 GB", | 20260507 06:32:35.011 | INFO | "pinned" : " 29 GB" | 20260507 06:32:35.011 | INFO | } | 20260507 06:32:35.011 | INFO | Perf report for the last 5.12898 seconds: | 20260507 06:32:35.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:32:35.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:35.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:35.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:35.011 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 06:32:35.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:35.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:35.011 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:32:35.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:35.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:35.011 | INFO | Uploaded 36GiB to GPU in 1.97514s (18.2266GiB/s) | 20260507 06:32:35.011 | INFO | Downloaded 34GiB to GPU in 1.93998s (17.5259GiB/s) | 20260507 06:32:40.149 | INFO | Analyzing chunk frames = [7836672,7837184) | 20260507 06:32:40.149 | INFO | MemInfo: { | 20260507 06:32:40.149 | INFO | "gpu" : " 17 GB", | 20260507 06:32:40.149 | INFO | "huge" : "", | 20260507 06:32:40.149 | INFO | "huge_shared" : " 27 GB", | 20260507 06:32:40.149 | INFO | "malloc" : " 14 GB", | 20260507 06:32:40.149 | INFO | "pinned" : " 29 GB" | 20260507 06:32:40.149 | INFO | } | 20260507 06:32:40.149 | INFO | Perf report for the last 5.13814 seconds: | 20260507 06:32:40.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:32:40.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:40.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:40.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:40.149 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:32:40.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:40.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:40.149 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:32:40.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:40.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:40.150 | INFO | Uploaded 36GiB to GPU in 2.04344s (17.6174GiB/s) | 20260507 06:32:40.150 | INFO | Downloaded 34GiB to GPU in 1.95395s (17.4006GiB/s) | 20260507 06:32:45.288 | INFO | Analyzing chunk frames = [7837184,7837696) | 20260507 06:32:45.288 | INFO | MemInfo: { | 20260507 06:32:45.288 | INFO | "gpu" : " 17 GB", | 20260507 06:32:45.288 | INFO | "huge" : "", | 20260507 06:32:45.288 | INFO | "huge_shared" : " 27 GB", | 20260507 06:32:45.288 | INFO | "malloc" : " 14 GB", | 20260507 06:32:45.288 | INFO | "pinned" : " 29 GB" | 20260507 06:32:45.288 | INFO | } | 20260507 06:32:45.288 | INFO | Perf report for the last 5.13858 seconds: | 20260507 06:32:45.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:32:45.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:45.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:45.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:45.288 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 06:32:45.288 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:45.288 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:45.288 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:32:45.288 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:45.288 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:45.288 | INFO | Uploaded 36GiB to GPU in 1.9938s (18.056GiB/s) | 20260507 06:32:45.288 | INFO | Downloaded 34GiB to GPU in 1.93864s (17.5381GiB/s) | 20260507 06:32:50.432 | INFO | Analyzing chunk frames = [7837696,7838208) | 20260507 06:32:50.432 | INFO | MemInfo: { | 20260507 06:32:50.432 | INFO | "gpu" : " 17 GB", | 20260507 06:32:50.432 | INFO | "huge" : "", | 20260507 06:32:50.432 | INFO | "huge_shared" : " 27 GB", | 20260507 06:32:50.432 | INFO | "malloc" : " 14 GB", | 20260507 06:32:50.432 | INFO | "pinned" : " 29 GB" | 20260507 06:32:50.432 | INFO | } | 20260507 06:32:50.432 | INFO | Perf report for the last 5.14415 seconds: | 20260507 06:32:50.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:32:50.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:50.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:50.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:50.432 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 06:32:50.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:50.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:50.432 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:32:50.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:50.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:50.432 | INFO | Uploaded 36GiB to GPU in 2.02445s (17.7826GiB/s) | 20260507 06:32:50.432 | INFO | Downloaded 34GiB to GPU in 1.91855s (17.7218GiB/s) | 20260507 06:32:55.560 | INFO | Analyzing chunk frames = [7838208,7838720) | 20260507 06:32:55.560 | INFO | MemInfo: { | 20260507 06:32:55.560 | INFO | "gpu" : " 17 GB", | 20260507 06:32:55.560 | INFO | "huge" : "", | 20260507 06:32:55.561 | INFO | "huge_shared" : " 27 GB", | 20260507 06:32:55.561 | INFO | "malloc" : " 14 GB", | 20260507 06:32:55.561 | INFO | "pinned" : " 29 GB" | 20260507 06:32:55.561 | INFO | } | 20260507 06:32:55.561 | INFO | Perf report for the last 5.12863 seconds: | 20260507 06:32:55.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:32:55.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:55.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:55.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:55.561 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 06:32:55.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:55.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:55.561 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:32:55.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:55.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:32:55.561 | INFO | Uploaded 36GiB to GPU in 2.06253s (17.4543GiB/s) | 20260507 06:32:55.561 | INFO | Downloaded 34GiB to GPU in 1.93646s (17.5578GiB/s) | 20260507 06:33:00.709 | INFO | Analyzing chunk frames = [7838720,7839232) | 20260507 06:33:00.709 | INFO | MemInfo: { | 20260507 06:33:00.709 | INFO | "gpu" : " 17 GB", | 20260507 06:33:00.709 | INFO | "huge" : "", | 20260507 06:33:00.709 | INFO | "huge_shared" : " 27 GB", | 20260507 06:33:00.709 | INFO | "malloc" : " 14 GB", | 20260507 06:33:00.709 | INFO | "pinned" : " 29 GB" | 20260507 06:33:00.709 | INFO | } | 20260507 06:33:00.709 | INFO | Perf report for the last 5.14878 seconds: | 20260507 06:33:00.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:33:00.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:00.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:00.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:00.709 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 06:33:00.710 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:00.710 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:00.710 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:33:00.710 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:00.710 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:00.710 | INFO | Uploaded 36GiB to GPU in 1.9675s (18.2973GiB/s) | 20260507 06:33:00.710 | INFO | Downloaded 34GiB to GPU in 1.98296s (17.1461GiB/s) | 20260507 06:33:05.848 | INFO | Analyzing chunk frames = [7839232,7839744) | 20260507 06:33:05.848 | INFO | MemInfo: { | 20260507 06:33:05.848 | INFO | "gpu" : " 17 GB", | 20260507 06:33:05.848 | INFO | "huge" : "", | 20260507 06:33:05.848 | INFO | "huge_shared" : " 27 GB", | 20260507 06:33:05.848 | INFO | "malloc" : " 14 GB", | 20260507 06:33:05.848 | INFO | "pinned" : " 29 GB" | 20260507 06:33:05.848 | INFO | } | 20260507 06:33:05.848 | INFO | Perf report for the last 5.13884 seconds: | 20260507 06:33:05.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:33:05.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:05.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:05.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:05.848 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:33:05.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:05.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:05.848 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:33:05.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:05.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:05.849 | INFO | Uploaded 36GiB to GPU in 1.95998s (18.3676GiB/s) | 20260507 06:33:05.849 | INFO | Downloaded 34GiB to GPU in 1.94932s (17.4419GiB/s) | 20260507 06:33:05.849 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:33:05.849 | INFO | Total mem usage: 18 GiB | 20260507 06:33:05.849 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:33:05.849 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:33:05.849 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:33:05.849 | INFO | High Water Breakdown: | 20260507 06:33:05.849 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:33:05.849 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:33:05.849 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:33:05.849 | INFO | ------------------------------------------ | 20260507 06:33:05.849 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:33:05.849 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:33:05.849 | INFO | Total mem usage: 26.5 GiB | 20260507 06:33:05.849 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:33:05.849 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:33:05.849 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:33:05.849 | INFO | High Water Breakdown: | 20260507 06:33:05.849 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:33:05.849 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:33:05.849 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:33:05.849 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:33:05.849 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:33:05.849 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:33:05.849 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:33:05.849 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:33:05.849 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:33:05.849 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:33:05.849 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:33:05.849 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:33:05.849 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:33:05.849 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:33:05.849 | INFO | ------------------------------------------ | 20260507 06:33:05.849 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:33:05.849 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:33:05.849 | INFO | Total mem usage: 25.6 GiB | 20260507 06:33:05.849 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:33:05.849 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:33:05.849 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:33:05.849 | INFO | High Water Breakdown: | 20260507 06:33:05.849 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:33:05.849 | INFO | ------------------------------------------ | 20260507 06:33:05.849 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:33:05.849 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:33:05.849 | INFO | Total mem usage: 15.7 GiB | 20260507 06:33:05.849 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:33:05.849 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:33:05.849 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:33:05.849 | INFO | High Water Breakdown: | 20260507 06:33:05.849 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:33:05.849 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:33:05.849 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:33:05.849 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:33:05.849 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:33:05.849 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:33:05.849 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:33:05.849 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:33:05.849 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:33:05.849 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:33:05.849 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:33:05.849 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:33:05.849 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:33:05.849 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:33:05.849 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:33:05.849 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:33:05.849 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:33:05.849 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:33:05.849 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:33:05.849 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:33:05.849 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:33:05.850 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:33:05.850 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:33:05.850 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:33:05.850 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:33:05.850 | INFO | ------------------------------------------ | 20260507 06:33:05.850 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:33:05.850 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:33:05.850 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:33:05.850 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:33:05.850 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.34e+01 / 1.05e+00 | 20260507 06:33:05.850 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 9.50e-03 / 5.01e-03 | 20260507 06:33:05.850 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.48e+00, 4.99e-01 / 1.96e-01 | 20260507 06:33:05.850 | INFO | PerfReport: 3.21%, Histogram -- 9.86e-01 / 1.47e+00, 1.63e-01 / 1.13e-01 | 20260507 06:33:05.850 | INFO | PerfReport: 16.24%, DME -- 3.99e+01 / 4.45e+01, 3.83e+00 / 1.24e+00 | 20260507 06:33:05.850 | INFO | PerfReport: 57.73%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.30e+00 / 1.51e+00 | 20260507 06:33:05.850 | INFO | PerfReport: 6.91%, PulseAccumulating -- 2.12e+00 / 2.67e+00, 3.80e-01 / 2.41e-01 | 20260507 06:33:05.850 | INFO | PerfReport: 2.95%, Metrics -- 9.07e-01 / 1.22e+00, 2.41e-01 / 1.31e-01 | 20260507 06:33:05.850 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.05e-01 / 3.60e-01, 5.03e-02 / 2.14e-02 | 20260507 06:33:05.850 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:33:05.850 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:33:05.850 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:33:05.850 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 06:33:05.850 | INFO | PerfReport: 21.89%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260507 06:33:05.850 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 06:33:05.850 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:33:05.850 | INFO | PerfReport: 1.48%, LockUpload -- 1.42e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 06:33:05.850 | INFO | PerfReport: 1.33%, LockDownload -- 1.28e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260507 06:33:05.850 | INFO | PerfReport: 21.97%, LockCompute -- 2.11e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 06:33:05.850 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260507 06:33:05.945 | INFO | Starting a round of Baz IO | 20260507 06:33:09.037 | INFO | Populating RTMetrics report | 20260507 06:33:09.037 | INFO | RTMetrics report populated | 20260507 06:33:09.037 | INFO | Opening tmp RTMetrics json file | 20260507 06:33:09.054 | INFO | Serializing RTMetrics to json | 20260507 06:33:09.055 | INFO | Serialization done, swapping tmp file now... | 20260507 06:33:09.056 | INFO | RTMetrics done | 20260507 06:33:09.124 | INFO | Finished a round of Baz IO: Wrote 1.666 GiB in 3.178 seconds | 20260507 06:33:09.124 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.62% / 0.00% / 67.38% | 20260507 06:33:09.124 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.121319% / 3.97364e-06% / 9.53674e-05% | 20260507 06:33:09.124 | INFO | Baz IO overall: | 20260507 06:33:09.124 | INFO | BytesWritten : 3951412666578 | 20260507 06:33:09.124 | INFO | NumEvents : 1893415155475 | 20260507 06:33:10.994 | INFO | Analyzing chunk frames = [7839744,7840256) | 20260507 06:33:10.994 | INFO | MemInfo: { | 20260507 06:33:10.994 | INFO | "gpu" : " 17 GB", | 20260507 06:33:10.994 | INFO | "huge" : "", | 20260507 06:33:10.994 | INFO | "huge_shared" : " 27 GB", | 20260507 06:33:10.994 | INFO | "malloc" : " 14 GB", | 20260507 06:33:10.994 | INFO | "pinned" : " 29 GB" | 20260507 06:33:10.994 | INFO | } | 20260507 06:33:10.995 | INFO | Perf report for the last 5.14626 seconds: | 20260507 06:33:10.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:33:10.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:10.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:10.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:10.995 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260507 06:33:10.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:10.995 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:33:10.995 | INFO | PRE_HQ : 58.8%, 1.8, 54.4ms, 0 | 20260507 06:33:10.995 | INFO | BAZWRITER : 61.3%, 3.1, 98.0ms, 0 | 20260507 06:33:10.995 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:33:10.995 | INFO | Uploaded 36GiB to GPU in 2.04509s (17.6031GiB/s) | 20260507 06:33:10.995 | INFO | Downloaded 38GiB to GPU in 2.17366s (17.482GiB/s) | 20260507 06:33:16.128 | INFO | Analyzing chunk frames = [7840256,7840768) | 20260507 06:33:16.128 | INFO | MemInfo: { | 20260507 06:33:16.128 | INFO | "gpu" : " 17 GB", | 20260507 06:33:16.128 | INFO | "huge" : "", | 20260507 06:33:16.128 | INFO | "huge_shared" : " 27 GB", | 20260507 06:33:16.128 | INFO | "malloc" : " 14 GB", | 20260507 06:33:16.128 | INFO | "pinned" : " 29 GB" | 20260507 06:33:16.128 | INFO | } | 20260507 06:33:16.128 | INFO | Perf report for the last 5.13391 seconds: | 20260507 06:33:16.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:33:16.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:16.129 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:16.129 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:16.129 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 06:33:16.129 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:16.129 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:16.129 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 06:33:16.129 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:16.129 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:16.129 | INFO | Uploaded 36GiB to GPU in 2.02435s (17.7835GiB/s) | 20260507 06:33:16.129 | INFO | Downloaded 34GiB to GPU in 1.95036s (17.4326GiB/s) | 20260507 06:33:21.261 | INFO | Analyzing chunk frames = [7840768,7841280) | 20260507 06:33:21.261 | INFO | MemInfo: { | 20260507 06:33:21.261 | INFO | "gpu" : " 17 GB", | 20260507 06:33:21.261 | INFO | "huge" : "", | 20260507 06:33:21.261 | INFO | "huge_shared" : " 27 GB", | 20260507 06:33:21.261 | INFO | "malloc" : " 14 GB", | 20260507 06:33:21.261 | INFO | "pinned" : " 29 GB" | 20260507 06:33:21.261 | INFO | } | 20260507 06:33:21.261 | INFO | Perf report for the last 5.13282 seconds: | 20260507 06:33:21.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:33:21.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:21.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:21.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:21.261 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 06:33:21.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:21.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:21.261 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:33:21.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:21.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:21.262 | INFO | Uploaded 36GiB to GPU in 2.04432s (17.6098GiB/s) | 20260507 06:33:21.262 | INFO | Downloaded 34GiB to GPU in 1.95409s (17.3994GiB/s) | 20260507 06:33:26.409 | INFO | Analyzing chunk frames = [7841280,7841792) | 20260507 06:33:26.409 | INFO | MemInfo: { | 20260507 06:33:26.409 | INFO | "gpu" : " 17 GB", | 20260507 06:33:26.409 | INFO | "huge" : "", | 20260507 06:33:26.409 | INFO | "huge_shared" : " 27 GB", | 20260507 06:33:26.409 | INFO | "malloc" : " 14 GB", | 20260507 06:33:26.409 | INFO | "pinned" : " 29 GB" | 20260507 06:33:26.409 | INFO | } | 20260507 06:33:26.409 | INFO | Perf report for the last 5.14767 seconds: | 20260507 06:33:26.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:33:26.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:26.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:26.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:26.409 | INFO | ANALYSIS : 59.5%, 3.0, 92.5ms, 0 | 20260507 06:33:26.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:26.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:26.409 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:33:26.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:26.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:26.409 | INFO | Uploaded 36GiB to GPU in 1.99478s (18.0471GiB/s) | 20260507 06:33:26.409 | INFO | Downloaded 34GiB to GPU in 1.93483s (17.5726GiB/s) | 20260507 06:33:31.545 | INFO | Analyzing chunk frames = [7841792,7842304) | 20260507 06:33:31.545 | INFO | MemInfo: { | 20260507 06:33:31.545 | INFO | "gpu" : " 17 GB", | 20260507 06:33:31.545 | INFO | "huge" : "", | 20260507 06:33:31.546 | INFO | "huge_shared" : " 27 GB", | 20260507 06:33:31.546 | INFO | "malloc" : " 14 GB", | 20260507 06:33:31.546 | INFO | "pinned" : " 29 GB" | 20260507 06:33:31.546 | INFO | } | 20260507 06:33:31.546 | INFO | Perf report for the last 5.13676 seconds: | 20260507 06:33:31.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:33:31.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:31.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:31.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:31.546 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 06:33:31.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:31.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:31.546 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:33:31.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:31.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:31.546 | INFO | Uploaded 36GiB to GPU in 2.05245s (17.54GiB/s) | 20260507 06:33:31.546 | INFO | Downloaded 34GiB to GPU in 1.95254s (17.4132GiB/s) | 20260507 06:33:36.685 | INFO | Analyzing chunk frames = [7842304,7842816) | 20260507 06:33:36.685 | INFO | MemInfo: { | 20260507 06:33:36.685 | INFO | "gpu" : " 17 GB", | 20260507 06:33:36.685 | INFO | "huge" : "", | 20260507 06:33:36.685 | INFO | "huge_shared" : " 27 GB", | 20260507 06:33:36.685 | INFO | "malloc" : " 14 GB", | 20260507 06:33:36.685 | INFO | "pinned" : " 29 GB" | 20260507 06:33:36.685 | INFO | } | 20260507 06:33:36.685 | INFO | Perf report for the last 5.13934 seconds: | 20260507 06:33:36.685 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:33:36.685 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:36.685 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:36.685 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:36.685 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 06:33:36.685 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:36.685 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:36.685 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 06:33:36.685 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:36.685 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:36.686 | INFO | Uploaded 36GiB to GPU in 2.06051s (17.4714GiB/s) | 20260507 06:33:36.686 | INFO | Downloaded 34GiB to GPU in 1.93233s (17.5953GiB/s) | 20260507 06:33:41.821 | INFO | Analyzing chunk frames = [7842816,7843328) | 20260507 06:33:41.821 | INFO | MemInfo: { | 20260507 06:33:41.821 | INFO | "gpu" : " 17 GB", | 20260507 06:33:41.821 | INFO | "huge" : "", | 20260507 06:33:41.821 | INFO | "huge_shared" : " 27 GB", | 20260507 06:33:41.821 | INFO | "malloc" : " 14 GB", | 20260507 06:33:41.821 | INFO | "pinned" : " 29 GB" | 20260507 06:33:41.821 | INFO | } | 20260507 06:33:41.821 | INFO | Perf report for the last 5.1362 seconds: | 20260507 06:33:41.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:33:41.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:41.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:41.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:41.821 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:33:41.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:41.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:41.821 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:33:41.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:41.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:41.822 | INFO | Uploaded 36GiB to GPU in 1.92061s (18.7441GiB/s) | 20260507 06:33:41.822 | INFO | Downloaded 34GiB to GPU in 1.97245s (17.2374GiB/s) | 20260507 06:33:46.965 | INFO | Analyzing chunk frames = [7843328,7843840) | 20260507 06:33:46.965 | INFO | MemInfo: { | 20260507 06:33:46.965 | INFO | "gpu" : " 17 GB", | 20260507 06:33:46.965 | INFO | "huge" : "", | 20260507 06:33:46.965 | INFO | "huge_shared" : " 27 GB", | 20260507 06:33:46.965 | INFO | "malloc" : " 14 GB", | 20260507 06:33:46.965 | INFO | "pinned" : " 29 GB" | 20260507 06:33:46.965 | INFO | } | 20260507 06:33:46.965 | INFO | Perf report for the last 5.14367 seconds: | 20260507 06:33:46.965 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:33:46.965 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:46.965 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:46.965 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:46.965 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 06:33:46.965 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:46.965 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:46.965 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:33:46.965 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:46.965 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:46.965 | INFO | Uploaded 36GiB to GPU in 1.96679s (18.3039GiB/s) | 20260507 06:33:46.965 | INFO | Downloaded 34GiB to GPU in 1.9614s (17.3346GiB/s) | 20260507 06:33:47.062 | INFO | Starting a round of Baz IO | 20260507 06:33:50.155 | INFO | Populating RTMetrics report | 20260507 06:33:50.155 | INFO | RTMetrics report populated | 20260507 06:33:50.155 | INFO | Opening tmp RTMetrics json file | 20260507 06:33:50.164 | INFO | Serializing RTMetrics to json | 20260507 06:33:50.171 | INFO | Serialization done, swapping tmp file now... | 20260507 06:33:50.172 | INFO | RTMetrics done | 20260507 06:33:50.240 | INFO | Finished a round of Baz IO: Wrote 1.665 GiB in 3.177 seconds | 20260507 06:33:50.240 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.62% / 0.00% / 67.38% | 20260507 06:33:50.240 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.120815% / 0% / 7.54992e-05% | 20260507 06:33:50.240 | INFO | Baz IO overall: | 20260507 06:33:50.240 | INFO | BytesWritten : 3953200174836 | 20260507 06:33:50.240 | INFO | NumEvents : 1894251082744 | 20260507 06:33:52.100 | INFO | Analyzing chunk frames = [7843840,7844352) | 20260507 06:33:52.100 | INFO | MemInfo: { | 20260507 06:33:52.100 | INFO | "gpu" : " 17 GB", | 20260507 06:33:52.100 | INFO | "huge" : "", | 20260507 06:33:52.101 | INFO | "huge_shared" : " 27 GB", | 20260507 06:33:52.101 | INFO | "malloc" : " 14 GB", | 20260507 06:33:52.101 | INFO | "pinned" : " 29 GB" | 20260507 06:33:52.101 | INFO | } | 20260507 06:33:52.101 | INFO | Perf report for the last 5.13559 seconds: | 20260507 06:33:52.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:33:52.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:52.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:52.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:52.101 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 06:33:52.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:52.101 | INFO | RT_METRICS : 41.5%, 1.0, 21.7ms, 0 | 20260507 06:33:52.101 | INFO | PRE_HQ : 58.8%, 1.8, 54.5ms, 0 | 20260507 06:33:52.101 | INFO | BAZWRITER : 61.5%, 3.1, 98.7ms, 0 | 20260507 06:33:52.101 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:33:52.101 | INFO | Uploaded 36GiB to GPU in 2.05404s (17.5264GiB/s) | 20260507 06:33:52.101 | INFO | Downloaded 38GiB to GPU in 2.17648s (17.4594GiB/s) | 20260507 06:33:57.242 | INFO | Analyzing chunk frames = [7844352,7844864) | 20260507 06:33:57.242 | INFO | MemInfo: { | 20260507 06:33:57.242 | INFO | "gpu" : " 17 GB", | 20260507 06:33:57.242 | INFO | "huge" : "", | 20260507 06:33:57.242 | INFO | "huge_shared" : " 27 GB", | 20260507 06:33:57.242 | INFO | "malloc" : " 14 GB", | 20260507 06:33:57.242 | INFO | "pinned" : " 29 GB" | 20260507 06:33:57.242 | INFO | } | 20260507 06:33:57.242 | INFO | Perf report for the last 5.14155 seconds: | 20260507 06:33:57.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:33:57.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:57.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:57.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:57.242 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 06:33:57.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:57.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:57.242 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 06:33:57.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:57.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:33:57.242 | INFO | Uploaded 36GiB to GPU in 1.989s (18.0995GiB/s) | 20260507 06:33:57.243 | INFO | Downloaded 34GiB to GPU in 1.93235s (17.5952GiB/s) | 20260507 06:34:02.376 | INFO | Analyzing chunk frames = [7844864,7845376) | 20260507 06:34:02.376 | INFO | MemInfo: { | 20260507 06:34:02.376 | INFO | "gpu" : " 17 GB", | 20260507 06:34:02.376 | INFO | "huge" : "", | 20260507 06:34:02.376 | INFO | "huge_shared" : " 27 GB", | 20260507 06:34:02.376 | INFO | "malloc" : " 14 GB", | 20260507 06:34:02.376 | INFO | "pinned" : " 29 GB" | 20260507 06:34:02.376 | INFO | } | 20260507 06:34:02.376 | INFO | Perf report for the last 5.13365 seconds: | 20260507 06:34:02.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:34:02.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:02.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:02.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:02.376 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 06:34:02.376 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:02.376 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:02.376 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:34:02.376 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:02.376 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:02.376 | INFO | Uploaded 36GiB to GPU in 2.0442s (17.6108GiB/s) | 20260507 06:34:02.376 | INFO | Downloaded 34GiB to GPU in 1.95228s (17.4156GiB/s) | 20260507 06:34:07.512 | INFO | Analyzing chunk frames = [7845376,7845888) | 20260507 06:34:07.512 | INFO | MemInfo: { | 20260507 06:34:07.512 | INFO | "gpu" : " 17 GB", | 20260507 06:34:07.513 | INFO | "huge" : "", | 20260507 06:34:07.513 | INFO | "huge_shared" : " 27 GB", | 20260507 06:34:07.513 | INFO | "malloc" : " 14 GB", | 20260507 06:34:07.513 | INFO | "pinned" : " 29 GB" | 20260507 06:34:07.513 | INFO | } | 20260507 06:34:07.513 | INFO | Perf report for the last 5.13688 seconds: | 20260507 06:34:07.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:34:07.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:07.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:07.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:07.513 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 06:34:07.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:07.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:07.513 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:34:07.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:07.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:07.513 | INFO | Uploaded 36GiB to GPU in 2.00126s (17.9886GiB/s) | 20260507 06:34:07.513 | INFO | Downloaded 34GiB to GPU in 1.93808s (17.5431GiB/s) | 20260507 06:34:12.657 | INFO | Analyzing chunk frames = [7845888,7846400) | 20260507 06:34:12.657 | INFO | MemInfo: { | 20260507 06:34:12.657 | INFO | "gpu" : " 17 GB", | 20260507 06:34:12.657 | INFO | "huge" : "", | 20260507 06:34:12.658 | INFO | "huge_shared" : " 27 GB", | 20260507 06:34:12.658 | INFO | "malloc" : " 14 GB", | 20260507 06:34:12.658 | INFO | "pinned" : " 29 GB" | 20260507 06:34:12.658 | INFO | } | 20260507 06:34:12.658 | INFO | Perf report for the last 5.14509 seconds: | 20260507 06:34:12.658 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:34:12.658 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:12.658 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:12.658 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:12.658 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 06:34:12.658 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:12.658 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:12.658 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:34:12.658 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:12.658 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:12.658 | INFO | Uploaded 36GiB to GPU in 2.02664s (17.7634GiB/s) | 20260507 06:34:12.658 | INFO | Downloaded 34GiB to GPU in 1.93466s (17.5741GiB/s) | 20260507 06:34:17.798 | INFO | Analyzing chunk frames = [7846400,7846912) | 20260507 06:34:17.798 | INFO | MemInfo: { | 20260507 06:34:17.798 | INFO | "gpu" : " 17 GB", | 20260507 06:34:17.798 | INFO | "huge" : "", | 20260507 06:34:17.798 | INFO | "huge_shared" : " 27 GB", | 20260507 06:34:17.799 | INFO | "malloc" : " 14 GB", | 20260507 06:34:17.799 | INFO | "pinned" : " 29 GB" | 20260507 06:34:17.799 | INFO | } | 20260507 06:34:17.799 | INFO | Perf report for the last 5.14089 seconds: | 20260507 06:34:17.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:34:17.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:17.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:17.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:17.799 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:34:17.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:17.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:17.799 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 06:34:17.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:17.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:17.799 | INFO | Uploaded 36GiB to GPU in 2.04578s (17.5972GiB/s) | 20260507 06:34:17.799 | INFO | Downloaded 34GiB to GPU in 1.92945s (17.6216GiB/s) | 20260507 06:34:22.939 | INFO | Analyzing chunk frames = [7846912,7847424) | 20260507 06:34:22.939 | INFO | MemInfo: { | 20260507 06:34:22.939 | INFO | "gpu" : " 17 GB", | 20260507 06:34:22.939 | INFO | "huge" : "", | 20260507 06:34:22.939 | INFO | "huge_shared" : " 27 GB", | 20260507 06:34:22.939 | INFO | "malloc" : " 14 GB", | 20260507 06:34:22.939 | INFO | "pinned" : " 29 GB" | 20260507 06:34:22.939 | INFO | } | 20260507 06:34:22.940 | INFO | Perf report for the last 5.14077 seconds: | 20260507 06:34:22.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:34:22.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:22.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:22.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:22.940 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:34:22.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:22.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:22.940 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:34:22.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:22.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:22.940 | INFO | Uploaded 36GiB to GPU in 1.92149s (18.7355GiB/s) | 20260507 06:34:22.940 | INFO | Downloaded 34GiB to GPU in 1.9669s (17.2861GiB/s) | 20260507 06:34:28.075 | INFO | Analyzing chunk frames = [7847424,7847936) | 20260507 06:34:28.075 | INFO | MemInfo: { | 20260507 06:34:28.075 | INFO | "gpu" : " 17 GB", | 20260507 06:34:28.075 | INFO | "huge" : "", | 20260507 06:34:28.075 | INFO | "huge_shared" : " 27 GB", | 20260507 06:34:28.075 | INFO | "malloc" : " 14 GB", | 20260507 06:34:28.076 | INFO | "pinned" : " 29 GB" | 20260507 06:34:28.076 | INFO | } | 20260507 06:34:28.076 | INFO | Perf report for the last 5.13618 seconds: | 20260507 06:34:28.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:34:28.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:28.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:28.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:28.076 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:34:28.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:28.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:28.076 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:34:28.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:28.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:28.076 | INFO | Uploaded 36GiB to GPU in 1.9965s (18.0316GiB/s) | 20260507 06:34:28.076 | INFO | Downloaded 34GiB to GPU in 1.96092s (17.3388GiB/s) | 20260507 06:34:28.076 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:34:28.076 | INFO | Total mem usage: 18 GiB | 20260507 06:34:28.076 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:34:28.076 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 06:34:28.076 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:34:28.076 | INFO | High Water Breakdown: | 20260507 06:34:28.076 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:34:28.076 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:34:28.076 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:34:28.076 | INFO | ------------------------------------------ | 20260507 06:34:28.076 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:34:28.076 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:34:28.076 | INFO | Total mem usage: 26.5 GiB | 20260507 06:34:28.076 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:34:28.076 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:34:28.076 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 06:34:28.076 | INFO | High Water Breakdown: | 20260507 06:34:28.076 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:34:28.076 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:34:28.076 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:34:28.076 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:34:28.076 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:34:28.076 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:34:28.077 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:34:28.077 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:34:28.077 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:34:28.077 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:34:28.077 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:34:28.077 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:34:28.077 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:34:28.077 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:34:28.077 | INFO | ------------------------------------------ | 20260507 06:34:28.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:34:28.077 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:34:28.077 | INFO | Total mem usage: 25.6 GiB | 20260507 06:34:28.077 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:34:28.077 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:34:28.077 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:34:28.077 | INFO | High Water Breakdown: | 20260507 06:34:28.077 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:34:28.077 | INFO | ------------------------------------------ | 20260507 06:34:28.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:34:28.077 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:34:28.077 | INFO | Total mem usage: 15.7 GiB | 20260507 06:34:28.077 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:34:28.077 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:34:28.077 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:34:28.077 | INFO | High Water Breakdown: | 20260507 06:34:28.077 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:34:28.077 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:34:28.077 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:34:28.077 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:34:28.077 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:34:28.077 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:34:28.077 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:34:28.077 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:34:28.077 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:34:28.077 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:34:28.077 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:34:28.077 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:34:28.077 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:34:28.077 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:34:28.077 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:34:28.077 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:34:28.077 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:34:28.077 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:34:28.077 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:34:28.077 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:34:28.077 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:34:28.077 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:34:28.077 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:34:28.077 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:34:28.077 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:34:28.077 | INFO | ------------------------------------------ | 20260507 06:34:28.077 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:34:28.077 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:34:28.077 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:34:28.077 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:34:28.077 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.35e+01 / 1.04e+00 | 20260507 06:34:28.077 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.02e-01, 1.02e-02 / 3.19e-03 | 20260507 06:34:28.077 | INFO | PerfReport: 11.68%, Baseline -- 3.58e+00 / 4.48e+00, 5.03e-01 / 1.88e-01 | 20260507 06:34:28.077 | INFO | PerfReport: 3.23%, Histogram -- 9.90e-01 / 1.50e+00, 1.67e-01 / 7.25e-02 | 20260507 06:34:28.077 | INFO | PerfReport: 16.27%, DME -- 3.99e+01 / 4.48e+01, 3.89e+00 / 1.42e+00 | 20260507 06:34:28.077 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.22e+00 / 1.41e+00 | 20260507 06:34:28.077 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.72e+00, 3.88e-01 / 1.49e-01 | 20260507 06:34:28.077 | INFO | PerfReport: 2.96%, Metrics -- 9.07e-01 / 1.21e+00, 2.42e-01 / 1.20e-01 | 20260507 06:34:28.077 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.05e-01 / 3.57e-01, 5.01e-02 / 1.75e-02 | 20260507 06:34:28.077 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:34:28.077 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:34:28.077 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:34:28.077 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 06:34:28.077 | INFO | PerfReport: 21.81%, Upload -- 2.10e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260507 06:34:28.077 | INFO | PerfReport: 21.43%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 06:34:28.077 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 06:34:28.077 | INFO | PerfReport: 1.42%, LockUpload -- 1.36e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 06:34:28.077 | INFO | PerfReport: 1.37%, LockDownload -- 1.32e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260507 06:34:28.077 | INFO | PerfReport: 22.04%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 06:34:28.077 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260507 06:34:28.173 | INFO | Starting a round of Baz IO | 20260507 06:34:31.265 | INFO | Populating RTMetrics report | 20260507 06:34:31.265 | INFO | RTMetrics report populated | 20260507 06:34:31.265 | INFO | Opening tmp RTMetrics json file | 20260507 06:34:31.281 | INFO | Serializing RTMetrics to json | 20260507 06:34:31.282 | INFO | Serialization done, swapping tmp file now... | 20260507 06:34:31.282 | INFO | RTMetrics done | 20260507 06:34:31.348 | INFO | Finished a round of Baz IO: Wrote 1.663 GiB in 3.175 seconds | 20260507 06:34:31.348 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.62% / 0.00% / 67.38% | 20260507 06:34:31.348 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.119857% / 3.97364e-06% / 8.34465e-05% | 20260507 06:34:31.348 | INFO | Baz IO overall: | 20260507 06:34:31.349 | INFO | BytesWritten : 3954985578997 | 20260507 06:34:31.349 | INFO | NumEvents : 1895085527813 | 20260507 06:34:33.216 | INFO | Analyzing chunk frames = [7847936,7848448) | 20260507 06:34:33.216 | INFO | MemInfo: { | 20260507 06:34:33.216 | INFO | "gpu" : " 17 GB", | 20260507 06:34:33.216 | INFO | "huge" : "", | 20260507 06:34:33.216 | INFO | "huge_shared" : " 27 GB", | 20260507 06:34:33.216 | INFO | "malloc" : " 14 GB", | 20260507 06:34:33.216 | INFO | "pinned" : " 29 GB" | 20260507 06:34:33.216 | INFO | } | 20260507 06:34:33.216 | INFO | Perf report for the last 5.14071 seconds: | 20260507 06:34:33.217 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:34:33.217 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:33.217 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:33.217 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:33.217 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 06:34:33.217 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:33.217 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 06:34:33.217 | INFO | PRE_HQ : 59.0%, 1.8, 54.9ms, 0 | 20260507 06:34:33.217 | INFO | BAZWRITER : 61.4%, 3.1, 99.0ms, 0 | 20260507 06:34:33.217 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:34:33.217 | INFO | Uploaded 36GiB to GPU in 2.04783s (17.5796GiB/s) | 20260507 06:34:33.217 | INFO | Downloaded 38GiB to GPU in 2.17568s (17.4658GiB/s) | 20260507 06:34:38.355 | INFO | Analyzing chunk frames = [7848448,7848960) | 20260507 06:34:38.355 | INFO | MemInfo: { | 20260507 06:34:38.355 | INFO | "gpu" : " 17 GB", | 20260507 06:34:38.355 | INFO | "huge" : "", | 20260507 06:34:38.355 | INFO | "huge_shared" : " 27 GB", | 20260507 06:34:38.355 | INFO | "malloc" : " 14 GB", | 20260507 06:34:38.355 | INFO | "pinned" : " 29 GB" | 20260507 06:34:38.355 | INFO | } | 20260507 06:34:38.355 | INFO | Perf report for the last 5.13879 seconds: | 20260507 06:34:38.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:34:38.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:38.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:38.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:38.355 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 06:34:38.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:38.355 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:38.356 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:34:38.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:38.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:38.356 | INFO | Uploaded 36GiB to GPU in 2.00346s (17.9689GiB/s) | 20260507 06:34:38.356 | INFO | Downloaded 34GiB to GPU in 1.934s (17.5801GiB/s) | 20260507 06:34:43.501 | INFO | Analyzing chunk frames = [7848960,7849472) | 20260507 06:34:43.501 | INFO | MemInfo: { | 20260507 06:34:43.501 | INFO | "gpu" : " 17 GB", | 20260507 06:34:43.501 | INFO | "huge" : "", | 20260507 06:34:43.501 | INFO | "huge_shared" : " 27 GB", | 20260507 06:34:43.501 | INFO | "malloc" : " 14 GB", | 20260507 06:34:43.502 | INFO | "pinned" : " 29 GB" | 20260507 06:34:43.502 | INFO | } | 20260507 06:34:43.502 | INFO | Perf report for the last 5.14644 seconds: | 20260507 06:34:43.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:34:43.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:43.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:43.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:43.502 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 06:34:43.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:43.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:43.502 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:34:43.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:43.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:43.502 | INFO | Uploaded 36GiB to GPU in 2.05084s (17.5538GiB/s) | 20260507 06:34:43.502 | INFO | Downloaded 34GiB to GPU in 1.94801s (17.4537GiB/s) | 20260507 06:34:48.637 | INFO | Analyzing chunk frames = [7849472,7849984) | 20260507 06:34:48.637 | INFO | MemInfo: { | 20260507 06:34:48.637 | INFO | "gpu" : " 17 GB", | 20260507 06:34:48.637 | INFO | "huge" : "", | 20260507 06:34:48.637 | INFO | "huge_shared" : " 27 GB", | 20260507 06:34:48.637 | INFO | "malloc" : " 14 GB", | 20260507 06:34:48.637 | INFO | "pinned" : " 29 GB" | 20260507 06:34:48.637 | INFO | } | 20260507 06:34:48.637 | INFO | Perf report for the last 5.13522 seconds: | 20260507 06:34:48.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:34:48.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:48.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:48.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:48.637 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 06:34:48.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:48.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:48.637 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:34:48.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:48.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:48.637 | INFO | Uploaded 36GiB to GPU in 2.05539s (17.5149GiB/s) | 20260507 06:34:48.637 | INFO | Downloaded 34GiB to GPU in 1.94958s (17.4397GiB/s) | 20260507 06:34:53.770 | INFO | Analyzing chunk frames = [7849984,7850496) | 20260507 06:34:53.770 | INFO | MemInfo: { | 20260507 06:34:53.770 | INFO | "gpu" : " 17 GB", | 20260507 06:34:53.770 | INFO | "huge" : "", | 20260507 06:34:53.770 | INFO | "huge_shared" : " 27 GB", | 20260507 06:34:53.770 | INFO | "malloc" : " 14 GB", | 20260507 06:34:53.770 | INFO | "pinned" : " 29 GB" | 20260507 06:34:53.771 | INFO | } | 20260507 06:34:53.771 | INFO | Perf report for the last 5.13366 seconds: | 20260507 06:34:53.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:34:53.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:53.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:53.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:53.771 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260507 06:34:53.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:53.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:53.771 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:34:53.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:53.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:53.771 | INFO | Uploaded 36GiB to GPU in 2.05833s (17.4899GiB/s) | 20260507 06:34:53.771 | INFO | Downloaded 34GiB to GPU in 1.9519s (17.419GiB/s) | 20260507 06:34:58.911 | INFO | Analyzing chunk frames = [7850496,7851008) | 20260507 06:34:58.911 | INFO | MemInfo: { | 20260507 06:34:58.911 | INFO | "gpu" : " 17 GB", | 20260507 06:34:58.912 | INFO | "huge" : "", | 20260507 06:34:58.912 | INFO | "huge_shared" : " 27 GB", | 20260507 06:34:58.912 | INFO | "malloc" : " 14 GB", | 20260507 06:34:58.912 | INFO | "pinned" : " 29 GB" | 20260507 06:34:58.912 | INFO | } | 20260507 06:34:58.912 | INFO | Perf report for the last 5.14111 seconds: | 20260507 06:34:58.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:34:58.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:58.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:58.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:58.912 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 06:34:58.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:58.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:58.912 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 06:34:58.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:58.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:34:58.912 | INFO | Uploaded 36GiB to GPU in 2.0449s (17.6048GiB/s) | 20260507 06:34:58.912 | INFO | Downloaded 34GiB to GPU in 1.91631s (17.7424GiB/s) | 20260507 06:35:04.057 | INFO | Analyzing chunk frames = [7851008,7851520) | 20260507 06:35:04.057 | INFO | MemInfo: { | 20260507 06:35:04.057 | INFO | "gpu" : " 17 GB", | 20260507 06:35:04.057 | INFO | "huge" : "", | 20260507 06:35:04.057 | INFO | "huge_shared" : " 27 GB", | 20260507 06:35:04.057 | INFO | "malloc" : " 14 GB", | 20260507 06:35:04.057 | INFO | "pinned" : " 29 GB" | 20260507 06:35:04.057 | INFO | } | 20260507 06:35:04.057 | INFO | Perf report for the last 5.14536 seconds: | 20260507 06:35:04.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:35:04.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:04.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:04.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:04.057 | INFO | ANALYSIS : 60.9%, 3.0, 95.0ms, 0 | 20260507 06:35:04.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:04.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:04.057 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 06:35:04.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:04.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:04.057 | INFO | Uploaded 36GiB to GPU in 2.03991s (17.6478GiB/s) | 20260507 06:35:04.057 | INFO | Downloaded 34GiB to GPU in 1.97079s (17.252GiB/s) | 20260507 06:35:09.197 | INFO | Analyzing chunk frames = [7851520,7852032) | 20260507 06:35:09.197 | INFO | MemInfo: { | 20260507 06:35:09.197 | INFO | "gpu" : " 17 GB", | 20260507 06:35:09.197 | INFO | "huge" : "", | 20260507 06:35:09.197 | INFO | "huge_shared" : " 27 GB", | 20260507 06:35:09.197 | INFO | "malloc" : " 14 GB", | 20260507 06:35:09.197 | INFO | "pinned" : " 29 GB" | 20260507 06:35:09.197 | INFO | } | 20260507 06:35:09.197 | INFO | Perf report for the last 5.14042 seconds: | 20260507 06:35:09.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:35:09.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:09.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:09.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:09.198 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:35:09.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:09.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:09.198 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:35:09.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:09.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:09.198 | INFO | Uploaded 36GiB to GPU in 1.99877s (18.011GiB/s) | 20260507 06:35:09.198 | INFO | Downloaded 34GiB to GPU in 1.95679s (17.3754GiB/s) | 20260507 06:35:09.294 | INFO | Starting a round of Baz IO | 20260507 06:35:12.391 | INFO | Populating RTMetrics report | 20260507 06:35:12.391 | INFO | RTMetrics report populated | 20260507 06:35:12.391 | INFO | Opening tmp RTMetrics json file | 20260507 06:35:12.404 | INFO | Serializing RTMetrics to json | 20260507 06:35:12.405 | INFO | Serialization done, swapping tmp file now... | 20260507 06:35:12.406 | INFO | RTMetrics done | 20260507 06:35:12.471 | INFO | Finished a round of Baz IO: Wrote 1.665 GiB in 3.176 seconds | 20260507 06:35:12.471 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.62% / 0.00% / 67.38% | 20260507 06:35:12.471 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.119901% / 0% / 5.96046e-05% | 20260507 06:35:12.471 | INFO | Baz IO overall: | 20260507 06:35:12.471 | INFO | BytesWritten : 3956773553182 | 20260507 06:35:12.471 | INFO | NumEvents : 1895922285515 | 20260507 06:35:14.326 | INFO | Analyzing chunk frames = [7852032,7852544) | 20260507 06:35:14.326 | INFO | MemInfo: { | 20260507 06:35:14.326 | INFO | "gpu" : " 17 GB", | 20260507 06:35:14.326 | INFO | "huge" : "", | 20260507 06:35:14.326 | INFO | "huge_shared" : " 27 GB", | 20260507 06:35:14.326 | INFO | "malloc" : " 14 GB", | 20260507 06:35:14.326 | INFO | "pinned" : " 29 GB" | 20260507 06:35:14.326 | INFO | } | 20260507 06:35:14.326 | INFO | Perf report for the last 5.12864 seconds: | 20260507 06:35:14.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:35:14.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:14.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:14.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:14.326 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260507 06:35:14.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:14.326 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 06:35:14.326 | INFO | PRE_HQ : 59.0%, 1.8, 54.9ms, 0 | 20260507 06:35:14.326 | INFO | BAZWRITER : 61.5%, 3.1, 99.2ms, 0 | 20260507 06:35:14.326 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:35:14.326 | INFO | Uploaded 36GiB to GPU in 2.07471s (17.3518GiB/s) | 20260507 06:35:14.326 | INFO | Downloaded 38GiB to GPU in 2.17605s (17.4628GiB/s) | 20260507 06:35:19.467 | INFO | Analyzing chunk frames = [7852544,7853056) | 20260507 06:35:19.467 | INFO | MemInfo: { | 20260507 06:35:19.467 | INFO | "gpu" : " 17 GB", | 20260507 06:35:19.467 | INFO | "huge" : "", | 20260507 06:35:19.468 | INFO | "huge_shared" : " 27 GB", | 20260507 06:35:19.468 | INFO | "malloc" : " 14 GB", | 20260507 06:35:19.468 | INFO | "pinned" : " 29 GB" | 20260507 06:35:19.468 | INFO | } | 20260507 06:35:19.468 | INFO | Perf report for the last 5.1415 seconds: | 20260507 06:35:19.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:35:19.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:19.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:19.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:19.468 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 06:35:19.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:19.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:19.468 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 06:35:19.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:19.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:19.468 | INFO | Uploaded 36GiB to GPU in 2.00412s (17.963GiB/s) | 20260507 06:35:19.468 | INFO | Downloaded 34GiB to GPU in 1.93256s (17.5932GiB/s) | 20260507 06:35:24.612 | INFO | Analyzing chunk frames = [7853056,7853568) | 20260507 06:35:24.612 | INFO | MemInfo: { | 20260507 06:35:24.612 | INFO | "gpu" : " 17 GB", | 20260507 06:35:24.612 | INFO | "huge" : "", | 20260507 06:35:24.612 | INFO | "huge_shared" : " 27 GB", | 20260507 06:35:24.612 | INFO | "malloc" : " 14 GB", | 20260507 06:35:24.612 | INFO | "pinned" : " 29 GB" | 20260507 06:35:24.612 | INFO | } | 20260507 06:35:24.612 | INFO | Perf report for the last 5.14468 seconds: | 20260507 06:35:24.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:35:24.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:24.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:24.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:24.612 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 06:35:24.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:24.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:24.612 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 06:35:24.613 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:24.613 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:24.613 | INFO | Uploaded 36GiB to GPU in 2.01316s (17.8823GiB/s) | 20260507 06:35:24.613 | INFO | Downloaded 34GiB to GPU in 1.93874s (17.5372GiB/s) | 20260507 06:35:29.755 | INFO | Analyzing chunk frames = [7853568,7854080) | 20260507 06:35:29.755 | INFO | MemInfo: { | 20260507 06:35:29.755 | INFO | "gpu" : " 17 GB", | 20260507 06:35:29.755 | INFO | "huge" : "", | 20260507 06:35:29.755 | INFO | "huge_shared" : " 27 GB", | 20260507 06:35:29.755 | INFO | "malloc" : " 14 GB", | 20260507 06:35:29.756 | INFO | "pinned" : " 29 GB" | 20260507 06:35:29.756 | INFO | } | 20260507 06:35:29.756 | INFO | Perf report for the last 5.14319 seconds: | 20260507 06:35:29.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:35:29.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:29.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:29.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:29.756 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 06:35:29.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:29.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:29.756 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:35:29.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:29.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:29.756 | INFO | Uploaded 36GiB to GPU in 2.02491s (17.7786GiB/s) | 20260507 06:35:29.756 | INFO | Downloaded 34GiB to GPU in 1.93715s (17.5516GiB/s) | 20260507 06:35:34.883 | INFO | Analyzing chunk frames = [7854080,7854592) | 20260507 06:35:34.883 | INFO | MemInfo: { | 20260507 06:35:34.883 | INFO | "gpu" : " 17 GB", | 20260507 06:35:34.883 | INFO | "huge" : "", | 20260507 06:35:34.883 | INFO | "huge_shared" : " 27 GB", | 20260507 06:35:34.883 | INFO | "malloc" : " 14 GB", | 20260507 06:35:34.884 | INFO | "pinned" : " 29 GB" | 20260507 06:35:34.884 | INFO | } | 20260507 06:35:34.884 | INFO | Perf report for the last 5.12804 seconds: | 20260507 06:35:34.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:35:34.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:34.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:34.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:34.884 | INFO | ANALYSIS : 60.3%, 3.0, 93.7ms, 0 | 20260507 06:35:34.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:34.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:34.884 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 06:35:34.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:34.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:34.884 | INFO | Uploaded 36GiB to GPU in 2.05941s (17.4808GiB/s) | 20260507 06:35:34.884 | INFO | Downloaded 34GiB to GPU in 1.90815s (17.8183GiB/s) | 20260507 06:35:40.026 | INFO | Analyzing chunk frames = [7854592,7855104) | 20260507 06:35:40.026 | INFO | MemInfo: { | 20260507 06:35:40.026 | INFO | "gpu" : " 17 GB", | 20260507 06:35:40.026 | INFO | "huge" : "", | 20260507 06:35:40.026 | INFO | "huge_shared" : " 27 GB", | 20260507 06:35:40.026 | INFO | "malloc" : " 14 GB", | 20260507 06:35:40.026 | INFO | "pinned" : " 29 GB" | 20260507 06:35:40.026 | INFO | } | 20260507 06:35:40.026 | INFO | Perf report for the last 5.14256 seconds: | 20260507 06:35:40.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:35:40.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:40.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:40.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:40.026 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 06:35:40.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:40.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:40.026 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 06:35:40.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:40.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:40.026 | INFO | Uploaded 36GiB to GPU in 2.03581s (17.6834GiB/s) | 20260507 06:35:40.027 | INFO | Downloaded 34GiB to GPU in 1.91026s (17.7986GiB/s) | 20260507 06:35:45.166 | INFO | Analyzing chunk frames = [7855104,7855616) | 20260507 06:35:45.166 | INFO | MemInfo: { | 20260507 06:35:45.166 | INFO | "gpu" : " 17 GB", | 20260507 06:35:45.166 | INFO | "huge" : "", | 20260507 06:35:45.166 | INFO | "huge_shared" : " 27 GB", | 20260507 06:35:45.166 | INFO | "malloc" : " 14 GB", | 20260507 06:35:45.166 | INFO | "pinned" : " 29 GB" | 20260507 06:35:45.166 | INFO | } | 20260507 06:35:45.166 | INFO | Perf report for the last 5.13954 seconds: | 20260507 06:35:45.166 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:35:45.166 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:45.166 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:45.166 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:45.166 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 06:35:45.166 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:45.166 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:45.166 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:35:45.166 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:45.166 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:45.166 | INFO | Uploaded 36GiB to GPU in 2.01044s (17.9065GiB/s) | 20260507 06:35:45.166 | INFO | Downloaded 34GiB to GPU in 1.97012s (17.2578GiB/s) | 20260507 06:35:50.309 | INFO | Analyzing chunk frames = [7855616,7856128) | 20260507 06:35:50.309 | INFO | MemInfo: { | 20260507 06:35:50.309 | INFO | "gpu" : " 17 GB", | 20260507 06:35:50.309 | INFO | "huge" : "", | 20260507 06:35:50.309 | INFO | "huge_shared" : " 27 GB", | 20260507 06:35:50.310 | INFO | "malloc" : " 14 GB", | 20260507 06:35:50.310 | INFO | "pinned" : " 29 GB" | 20260507 06:35:50.310 | INFO | } | 20260507 06:35:50.310 | INFO | Perf report for the last 5.14381 seconds: | 20260507 06:35:50.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:35:50.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:50.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:50.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:50.310 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:35:50.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:50.310 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:50.310 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 06:35:50.310 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:50.310 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:50.310 | INFO | Uploaded 36GiB to GPU in 1.97495s (18.2283GiB/s) | 20260507 06:35:50.310 | INFO | Downloaded 34GiB to GPU in 1.96148s (17.3339GiB/s) | 20260507 06:35:50.310 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:35:50.310 | INFO | Total mem usage: 18 GiB | 20260507 06:35:50.310 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:35:50.310 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:35:50.310 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:35:50.310 | INFO | High Water Breakdown: | 20260507 06:35:50.310 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:35:50.310 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:35:50.310 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:35:50.310 | INFO | ------------------------------------------ | 20260507 06:35:50.310 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:35:50.310 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:35:50.310 | INFO | Total mem usage: 26.5 GiB | 20260507 06:35:50.310 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:35:50.310 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:35:50.310 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:35:50.310 | INFO | High Water Breakdown: | 20260507 06:35:50.310 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:35:50.310 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:35:50.310 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:35:50.310 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:35:50.310 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:35:50.310 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:35:50.310 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:35:50.310 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:35:50.310 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:35:50.310 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:35:50.310 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:35:50.310 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:35:50.310 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:35:50.310 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:35:50.310 | INFO | ------------------------------------------ | 20260507 06:35:50.310 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:35:50.310 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:35:50.310 | INFO | Total mem usage: 25.6 GiB | 20260507 06:35:50.310 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:35:50.310 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:35:50.310 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:35:50.310 | INFO | High Water Breakdown: | 20260507 06:35:50.310 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:35:50.310 | INFO | ------------------------------------------ | 20260507 06:35:50.310 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:35:50.310 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:35:50.310 | INFO | Total mem usage: 15.7 GiB | 20260507 06:35:50.310 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:35:50.310 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:35:50.311 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:35:50.311 | INFO | High Water Breakdown: | 20260507 06:35:50.311 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:35:50.311 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:35:50.311 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:35:50.311 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:35:50.311 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:35:50.311 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:35:50.311 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:35:50.311 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:35:50.311 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:35:50.311 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:35:50.311 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:35:50.311 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:35:50.311 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:35:50.311 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:35:50.311 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:35:50.311 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:35:50.311 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:35:50.311 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:35:50.311 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:35:50.311 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:35:50.311 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:35:50.311 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:35:50.311 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:35:50.311 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:35:50.311 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:35:50.311 | INFO | ------------------------------------------ | 20260507 06:35:50.311 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:35:50.311 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:35:50.311 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:35:50.311 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:35:50.311 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.33e+01 / 1.01e+00 | 20260507 06:35:50.311 | INFO | PerfReport: 0.30%, AggregateStats -- 9.06e-02 / 1.02e-01, 9.34e-03 / 7.94e-03 | 20260507 06:35:50.311 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.45e+00, 4.96e-01 / 2.65e-01 | 20260507 06:35:50.311 | INFO | PerfReport: 3.22%, Histogram -- 9.87e-01 / 1.45e+00, 1.63e-01 / 1.37e-01 | 20260507 06:35:50.311 | INFO | PerfReport: 16.20%, DME -- 3.98e+01 / 4.43e+01, 3.70e+00 / 1.20e+00 | 20260507 06:35:50.311 | INFO | PerfReport: 57.79%, FrameLabeling -- 1.77e+01 / 1.99e+01, 1.34e+00 / 1.43e+00 | 20260507 06:35:50.311 | INFO | PerfReport: 6.89%, PulseAccumulating -- 2.12e+00 / 2.67e+00, 3.78e-01 / 2.17e-01 | 20260507 06:35:50.311 | INFO | PerfReport: 2.95%, Metrics -- 9.07e-01 / 1.21e+00, 2.41e-01 / 1.09e-01 | 20260507 06:35:50.311 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.05e-01 / 3.60e-01, 5.09e-02 / 2.22e-02 | 20260507 06:35:50.311 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:35:50.311 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:35:50.311 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:35:50.311 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 06:35:50.311 | INFO | PerfReport: 21.99%, Upload -- 2.12e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 06:35:50.311 | INFO | PerfReport: 21.34%, Download -- 2.05e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260507 06:35:50.311 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 06:35:50.311 | INFO | PerfReport: 1.65%, LockUpload -- 1.59e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 06:35:50.311 | INFO | PerfReport: 1.13%, LockDownload -- 1.09e+00 / 0.00e+00, 2.68e+00 / 0.00e+00 | 20260507 06:35:50.311 | INFO | PerfReport: 21.96%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 06:35:50.311 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260507 06:35:50.407 | INFO | Starting a round of Baz IO | 20260507 06:35:53.498 | INFO | Populating RTMetrics report | 20260507 06:35:53.498 | INFO | RTMetrics report populated | 20260507 06:35:53.498 | INFO | Opening tmp RTMetrics json file | 20260507 06:35:53.500 | INFO | Serializing RTMetrics to json | 20260507 06:35:53.501 | INFO | Serialization done, swapping tmp file now... | 20260507 06:35:53.503 | INFO | RTMetrics done | 20260507 06:35:53.587 | INFO | Finished a round of Baz IO: Wrote 1.666 GiB in 3.18 seconds | 20260507 06:35:53.587 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.62% / 0.00% / 67.38% | 20260507 06:35:53.587 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.120294% / 0% / 6.35783e-05% | 20260507 06:35:53.587 | INFO | Baz IO overall: | 20260507 06:35:53.587 | INFO | BytesWritten : 3958562505781 | 20260507 06:35:53.587 | INFO | NumEvents : 1896759895682 | 20260507 06:35:55.448 | INFO | Analyzing chunk frames = [7856128,7856640) | 20260507 06:35:55.448 | INFO | MemInfo: { | 20260507 06:35:55.448 | INFO | "gpu" : " 17 GB", | 20260507 06:35:55.448 | INFO | "huge" : "", | 20260507 06:35:55.448 | INFO | "huge_shared" : " 27 GB", | 20260507 06:35:55.448 | INFO | "malloc" : " 14 GB", | 20260507 06:35:55.448 | INFO | "pinned" : " 29 GB" | 20260507 06:35:55.448 | INFO | } | 20260507 06:35:55.448 | INFO | Perf report for the last 5.13813 seconds: | 20260507 06:35:55.448 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:35:55.448 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:55.448 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:55.448 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:55.448 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 06:35:55.448 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:35:55.448 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:35:55.448 | INFO | PRE_HQ : 58.8%, 1.8, 55.0ms, 0 | 20260507 06:35:55.448 | INFO | BAZWRITER : 61.5%, 3.1, 99.2ms, 0 | 20260507 06:35:55.448 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:35:55.448 | INFO | Uploaded 36GiB to GPU in 2.06551s (17.4291GiB/s) | 20260507 06:35:55.448 | INFO | Downloaded 38GiB to GPU in 2.17146s (17.4998GiB/s) | 20260507 06:36:00.581 | INFO | Analyzing chunk frames = [7856640,7857152) | 20260507 06:36:00.581 | INFO | MemInfo: { | 20260507 06:36:00.581 | INFO | "gpu" : " 17 GB", | 20260507 06:36:00.581 | INFO | "huge" : "", | 20260507 06:36:00.581 | INFO | "huge_shared" : " 27 GB", | 20260507 06:36:00.581 | INFO | "malloc" : " 14 GB", | 20260507 06:36:00.581 | INFO | "pinned" : " 29 GB" | 20260507 06:36:00.581 | INFO | } | 20260507 06:36:00.581 | INFO | Perf report for the last 5.133 seconds: | 20260507 06:36:00.581 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:36:00.581 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:00.581 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:00.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:00.581 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 06:36:00.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:00.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:00.581 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:36:00.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:00.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:00.581 | INFO | Uploaded 36GiB to GPU in 2.00741s (17.9335GiB/s) | 20260507 06:36:00.581 | INFO | Downloaded 34GiB to GPU in 1.93751s (17.5483GiB/s) | 20260507 06:36:05.728 | INFO | Analyzing chunk frames = [7857152,7857664) | 20260507 06:36:05.728 | INFO | MemInfo: { | 20260507 06:36:05.728 | INFO | "gpu" : " 17 GB", | 20260507 06:36:05.728 | INFO | "huge" : "", | 20260507 06:36:05.728 | INFO | "huge_shared" : " 27 GB", | 20260507 06:36:05.728 | INFO | "malloc" : " 14 GB", | 20260507 06:36:05.728 | INFO | "pinned" : " 29 GB" | 20260507 06:36:05.728 | INFO | } | 20260507 06:36:05.728 | INFO | Perf report for the last 5.14717 seconds: | 20260507 06:36:05.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:36:05.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:05.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:05.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:05.728 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 06:36:05.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:05.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:05.728 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 06:36:05.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:05.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:05.728 | INFO | Uploaded 36GiB to GPU in 2.03759s (17.6679GiB/s) | 20260507 06:36:05.728 | INFO | Downloaded 34GiB to GPU in 1.95173s (17.4205GiB/s) | 20260507 06:36:10.860 | INFO | Analyzing chunk frames = [7857664,7858176) | 20260507 06:36:10.860 | INFO | MemInfo: { | 20260507 06:36:10.860 | INFO | "gpu" : " 17 GB", | 20260507 06:36:10.860 | INFO | "huge" : "", | 20260507 06:36:10.860 | INFO | "huge_shared" : " 27 GB", | 20260507 06:36:10.860 | INFO | "malloc" : " 14 GB", | 20260507 06:36:10.860 | INFO | "pinned" : " 29 GB" | 20260507 06:36:10.860 | INFO | } | 20260507 06:36:10.860 | INFO | Perf report for the last 5.13213 seconds: | 20260507 06:36:10.860 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:36:10.860 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:10.860 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:10.860 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:10.860 | INFO | ANALYSIS : 59.8%, 3.0, 92.5ms, 0 | 20260507 06:36:10.860 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:10.860 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:10.860 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:36:10.860 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:10.860 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:10.860 | INFO | Uploaded 36GiB to GPU in 2.02302s (17.7951GiB/s) | 20260507 06:36:10.860 | INFO | Downloaded 34GiB to GPU in 1.93627s (17.5596GiB/s) | 20260507 06:36:16.006 | INFO | Analyzing chunk frames = [7858176,7858688) | 20260507 06:36:16.006 | INFO | MemInfo: { | 20260507 06:36:16.006 | INFO | "gpu" : " 17 GB", | 20260507 06:36:16.006 | INFO | "huge" : "", | 20260507 06:36:16.006 | INFO | "huge_shared" : " 27 GB", | 20260507 06:36:16.006 | INFO | "malloc" : " 14 GB", | 20260507 06:36:16.006 | INFO | "pinned" : " 29 GB" | 20260507 06:36:16.006 | INFO | } | 20260507 06:36:16.006 | INFO | Perf report for the last 5.14617 seconds: | 20260507 06:36:16.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:36:16.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:16.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:16.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:16.006 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 06:36:16.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:16.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:16.006 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:36:16.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:16.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:16.007 | INFO | Uploaded 36GiB to GPU in 2.03892s (17.6564GiB/s) | 20260507 06:36:16.007 | INFO | Downloaded 34GiB to GPU in 1.91379s (17.7658GiB/s) | 20260507 06:36:21.148 | INFO | Analyzing chunk frames = [7858688,7859200) | 20260507 06:36:21.148 | INFO | MemInfo: { | 20260507 06:36:21.148 | INFO | "gpu" : " 17 GB", | 20260507 06:36:21.148 | INFO | "huge" : "", | 20260507 06:36:21.148 | INFO | "huge_shared" : " 27 GB", | 20260507 06:36:21.148 | INFO | "malloc" : " 14 GB", | 20260507 06:36:21.148 | INFO | "pinned" : " 29 GB" | 20260507 06:36:21.148 | INFO | } | 20260507 06:36:21.148 | INFO | Perf report for the last 5.14208 seconds: | 20260507 06:36:21.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:36:21.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:21.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:21.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:21.148 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:36:21.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:21.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:21.148 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:36:21.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:21.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:21.149 | INFO | Uploaded 36GiB to GPU in 2.07551s (17.3452GiB/s) | 20260507 06:36:21.149 | INFO | Downloaded 34GiB to GPU in 1.93817s (17.5424GiB/s) | 20260507 06:36:26.279 | INFO | Analyzing chunk frames = [7859200,7859712) | 20260507 06:36:26.279 | INFO | MemInfo: { | 20260507 06:36:26.279 | INFO | "gpu" : " 17 GB", | 20260507 06:36:26.279 | INFO | "huge" : "", | 20260507 06:36:26.279 | INFO | "huge_shared" : " 27 GB", | 20260507 06:36:26.279 | INFO | "malloc" : " 14 GB", | 20260507 06:36:26.279 | INFO | "pinned" : " 29 GB" | 20260507 06:36:26.279 | INFO | } | 20260507 06:36:26.279 | INFO | Perf report for the last 5.13081 seconds: | 20260507 06:36:26.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:36:26.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:26.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:26.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:26.279 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260507 06:36:26.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:26.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:26.279 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 06:36:26.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:26.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:26.279 | INFO | Uploaded 36GiB to GPU in 1.89991s (18.9483GiB/s) | 20260507 06:36:26.280 | INFO | Downloaded 34GiB to GPU in 1.97402s (17.2238GiB/s) | 20260507 06:36:31.421 | INFO | Analyzing chunk frames = [7859712,7860224) | 20260507 06:36:31.421 | INFO | MemInfo: { | 20260507 06:36:31.421 | INFO | "gpu" : " 17 GB", | 20260507 06:36:31.421 | INFO | "huge" : "", | 20260507 06:36:31.421 | INFO | "huge_shared" : " 27 GB", | 20260507 06:36:31.421 | INFO | "malloc" : " 14 GB", | 20260507 06:36:31.421 | INFO | "pinned" : " 29 GB" | 20260507 06:36:31.421 | INFO | } | 20260507 06:36:31.421 | INFO | Perf report for the last 5.14187 seconds: | 20260507 06:36:31.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:36:31.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:31.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:31.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:31.421 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:36:31.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:31.421 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:31.421 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 06:36:31.421 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:31.421 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:31.421 | INFO | Uploaded 36GiB to GPU in 1.96241s (18.3448GiB/s) | 20260507 06:36:31.421 | INFO | Downloaded 34GiB to GPU in 1.95537s (17.388GiB/s) | 20260507 06:36:31.518 | INFO | Starting a round of Baz IO | 20260507 06:36:34.612 | INFO | Populating RTMetrics report | 20260507 06:36:34.613 | INFO | RTMetrics report populated | 20260507 06:36:34.613 | INFO | Opening tmp RTMetrics json file | 20260507 06:36:34.622 | INFO | Serializing RTMetrics to json | 20260507 06:36:34.628 | INFO | Serialization done, swapping tmp file now... | 20260507 06:36:34.629 | INFO | RTMetrics done | 20260507 06:36:34.695 | INFO | Finished a round of Baz IO: Wrote 1.663 GiB in 3.176 seconds | 20260507 06:36:34.695 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.62% / 0.00% / 67.38% | 20260507 06:36:34.695 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.119623% / 3.97364e-06% / 7.15256e-05% | 20260507 06:36:34.695 | INFO | Baz IO overall: | 20260507 06:36:34.695 | INFO | BytesWritten : 3960347690990 | 20260507 06:36:34.695 | INFO | NumEvents : 1897594653004 | 20260507 06:36:36.567 | INFO | Analyzing chunk frames = [7860224,7860736) | 20260507 06:36:36.567 | INFO | MemInfo: { | 20260507 06:36:36.567 | INFO | "gpu" : " 17 GB", | 20260507 06:36:36.567 | INFO | "huge" : "", | 20260507 06:36:36.567 | INFO | "huge_shared" : " 27 GB", | 20260507 06:36:36.567 | INFO | "malloc" : " 14 GB", | 20260507 06:36:36.567 | INFO | "pinned" : " 29 GB" | 20260507 06:36:36.567 | INFO | } | 20260507 06:36:36.567 | INFO | Perf report for the last 5.14593 seconds: | 20260507 06:36:36.567 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:36:36.567 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:36.567 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:36.567 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:36.567 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 06:36:36.567 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:36.567 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 06:36:36.567 | INFO | PRE_HQ : 59.0%, 1.8, 55.1ms, 0 | 20260507 06:36:36.567 | INFO | BAZWRITER : 61.3%, 3.1, 97.7ms, 0 | 20260507 06:36:36.567 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:36:36.567 | INFO | Uploaded 36GiB to GPU in 2.04382s (17.6141GiB/s) | 20260507 06:36:36.567 | INFO | Downloaded 38GiB to GPU in 2.17231s (17.4929GiB/s) | 20260507 06:36:41.697 | INFO | Analyzing chunk frames = [7860736,7861248) | 20260507 06:36:41.697 | INFO | MemInfo: { | 20260507 06:36:41.697 | INFO | "gpu" : " 17 GB", | 20260507 06:36:41.697 | INFO | "huge" : "", | 20260507 06:36:41.698 | INFO | "huge_shared" : " 27 GB", | 20260507 06:36:41.698 | INFO | "malloc" : " 14 GB", | 20260507 06:36:41.698 | INFO | "pinned" : " 29 GB" | 20260507 06:36:41.698 | INFO | } | 20260507 06:36:41.698 | INFO | Perf report for the last 5.13068 seconds: | 20260507 06:36:41.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:36:41.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:41.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:41.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:41.698 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 06:36:41.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:41.698 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:41.698 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 06:36:41.698 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:41.698 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:41.698 | INFO | Uploaded 36GiB to GPU in 2.00085s (17.9923GiB/s) | 20260507 06:36:41.698 | INFO | Downloaded 34GiB to GPU in 1.93977s (17.5278GiB/s) | 20260507 06:36:46.832 | INFO | Analyzing chunk frames = [7861248,7861760) | 20260507 06:36:46.832 | INFO | MemInfo: { | 20260507 06:36:46.832 | INFO | "gpu" : " 17 GB", | 20260507 06:36:46.832 | INFO | "huge" : "", | 20260507 06:36:46.832 | INFO | "huge_shared" : " 27 GB", | 20260507 06:36:46.832 | INFO | "malloc" : " 14 GB", | 20260507 06:36:46.832 | INFO | "pinned" : " 29 GB" | 20260507 06:36:46.832 | INFO | } | 20260507 06:36:46.832 | INFO | Perf report for the last 5.1343 seconds: | 20260507 06:36:46.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:36:46.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:46.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:46.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:46.832 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 06:36:46.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:46.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:46.832 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:36:46.832 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:46.832 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:46.832 | INFO | Uploaded 36GiB to GPU in 2.04403s (17.6123GiB/s) | 20260507 06:36:46.832 | INFO | Downloaded 34GiB to GPU in 1.94967s (17.4389GiB/s) | 20260507 06:36:51.974 | INFO | Analyzing chunk frames = [7861760,7862272) | 20260507 06:36:51.974 | INFO | MemInfo: { | 20260507 06:36:51.974 | INFO | "gpu" : " 17 GB", | 20260507 06:36:51.974 | INFO | "huge" : "", | 20260507 06:36:51.974 | INFO | "huge_shared" : " 27 GB", | 20260507 06:36:51.974 | INFO | "malloc" : " 14 GB", | 20260507 06:36:51.974 | INFO | "pinned" : " 29 GB" | 20260507 06:36:51.974 | INFO | } | 20260507 06:36:51.975 | INFO | Perf report for the last 5.14256 seconds: | 20260507 06:36:51.975 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:36:51.975 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:51.975 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:51.975 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:51.975 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 06:36:51.975 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:51.975 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:51.975 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:36:51.975 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:51.975 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:51.975 | INFO | Uploaded 36GiB to GPU in 2.00511s (17.9541GiB/s) | 20260507 06:36:51.975 | INFO | Downloaded 34GiB to GPU in 1.93392s (17.5808GiB/s) | 20260507 06:36:57.120 | INFO | Analyzing chunk frames = [7862272,7862784) | 20260507 06:36:57.120 | INFO | MemInfo: { | 20260507 06:36:57.120 | INFO | "gpu" : " 17 GB", | 20260507 06:36:57.120 | INFO | "huge" : "", | 20260507 06:36:57.120 | INFO | "huge_shared" : " 27 GB", | 20260507 06:36:57.120 | INFO | "malloc" : " 14 GB", | 20260507 06:36:57.120 | INFO | "pinned" : " 29 GB" | 20260507 06:36:57.120 | INFO | } | 20260507 06:36:57.120 | INFO | Perf report for the last 5.1452 seconds: | 20260507 06:36:57.120 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:36:57.120 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:57.120 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:57.120 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:57.120 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 06:36:57.120 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:57.120 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:57.120 | INFO | PRE_HQ : 24.7%, 1.0, 12.9ms, 0 | 20260507 06:36:57.120 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:57.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:36:57.120 | INFO | Uploaded 36GiB to GPU in 2.03776s (17.6665GiB/s) | 20260507 06:36:57.120 | INFO | Downloaded 34GiB to GPU in 1.93586s (17.5633GiB/s) | 20260507 06:37:02.258 | INFO | Analyzing chunk frames = [7862784,7863296) | 20260507 06:37:02.258 | INFO | MemInfo: { | 20260507 06:37:02.258 | INFO | "gpu" : " 17 GB", | 20260507 06:37:02.259 | INFO | "huge" : "", | 20260507 06:37:02.259 | INFO | "huge_shared" : " 27 GB", | 20260507 06:37:02.259 | INFO | "malloc" : " 14 GB", | 20260507 06:37:02.259 | INFO | "pinned" : " 29 GB" | 20260507 06:37:02.259 | INFO | } | 20260507 06:37:02.259 | INFO | Perf report for the last 5.139 seconds: | 20260507 06:37:02.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:37:02.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:02.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:02.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:02.259 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:37:02.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:02.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:02.259 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:37:02.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:02.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:02.259 | INFO | Uploaded 36GiB to GPU in 2.07173s (17.3767GiB/s) | 20260507 06:37:02.259 | INFO | Downloaded 34GiB to GPU in 1.9239s (17.6724GiB/s) | 20260507 06:37:07.397 | INFO | Analyzing chunk frames = [7863296,7863808) | 20260507 06:37:07.397 | INFO | MemInfo: { | 20260507 06:37:07.397 | INFO | "gpu" : " 17 GB", | 20260507 06:37:07.397 | INFO | "huge" : "", | 20260507 06:37:07.397 | INFO | "huge_shared" : " 27 GB", | 20260507 06:37:07.397 | INFO | "malloc" : " 14 GB", | 20260507 06:37:07.397 | INFO | "pinned" : " 29 GB" | 20260507 06:37:07.397 | INFO | } | 20260507 06:37:07.397 | INFO | Perf report for the last 5.13859 seconds: | 20260507 06:37:07.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:37:07.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:07.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:07.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:07.398 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 06:37:07.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:07.398 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:07.398 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 06:37:07.398 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:07.398 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:07.398 | INFO | Uploaded 36GiB to GPU in 1.90077s (18.9397GiB/s) | 20260507 06:37:07.398 | INFO | Downloaded 34GiB to GPU in 1.97425s (17.2217GiB/s) | 20260507 06:37:12.537 | INFO | Analyzing chunk frames = [7863808,7864320) | 20260507 06:37:12.537 | INFO | MemInfo: { | 20260507 06:37:12.537 | INFO | "gpu" : " 17 GB", | 20260507 06:37:12.537 | INFO | "huge" : "", | 20260507 06:37:12.537 | INFO | "huge_shared" : " 27 GB", | 20260507 06:37:12.537 | INFO | "malloc" : " 14 GB", | 20260507 06:37:12.537 | INFO | "pinned" : " 29 GB" | 20260507 06:37:12.537 | INFO | } | 20260507 06:37:12.537 | INFO | Perf report for the last 5.13987 seconds: | 20260507 06:37:12.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:37:12.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:12.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:12.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:12.537 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:37:12.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:12.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:12.537 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:37:12.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:12.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:12.538 | INFO | Uploaded 36GiB to GPU in 1.98793s (18.1093GiB/s) | 20260507 06:37:12.538 | INFO | Downloaded 34GiB to GPU in 1.95942s (17.3521GiB/s) | 20260507 06:37:12.538 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:37:12.538 | INFO | Total mem usage: 18 GiB | 20260507 06:37:12.538 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:37:12.538 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 06:37:12.538 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:37:12.538 | INFO | High Water Breakdown: | 20260507 06:37:12.538 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:37:12.538 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:37:12.538 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:37:12.538 | INFO | ------------------------------------------ | 20260507 06:37:12.538 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:37:12.538 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:37:12.538 | INFO | Total mem usage: 26.5 GiB | 20260507 06:37:12.538 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:37:12.538 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:37:12.538 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:37:12.538 | INFO | High Water Breakdown: | 20260507 06:37:12.538 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:37:12.538 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:37:12.538 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:37:12.538 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:37:12.538 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:37:12.538 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:37:12.538 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:37:12.538 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:37:12.538 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:37:12.538 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:37:12.538 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:37:12.538 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:37:12.538 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:37:12.538 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:37:12.538 | INFO | ------------------------------------------ | 20260507 06:37:12.538 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:37:12.538 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:37:12.538 | INFO | Total mem usage: 25.6 GiB | 20260507 06:37:12.538 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:37:12.538 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:37:12.538 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:37:12.538 | INFO | High Water Breakdown: | 20260507 06:37:12.538 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:37:12.538 | INFO | ------------------------------------------ | 20260507 06:37:12.538 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:37:12.538 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:37:12.538 | INFO | Total mem usage: 15.7 GiB | 20260507 06:37:12.538 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:37:12.538 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:37:12.538 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:37:12.538 | INFO | High Water Breakdown: | 20260507 06:37:12.538 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:37:12.538 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:37:12.538 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:37:12.538 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:37:12.538 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:37:12.538 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:37:12.538 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:37:12.538 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:37:12.538 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:37:12.539 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:37:12.539 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:37:12.539 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:37:12.539 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:37:12.539 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:37:12.539 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:37:12.539 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:37:12.539 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:37:12.539 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:37:12.539 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:37:12.539 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:37:12.539 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:37:12.539 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:37:12.539 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:37:12.539 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:37:12.539 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:37:12.539 | INFO | ------------------------------------------ | 20260507 06:37:12.539 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:37:12.539 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:37:12.539 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:37:12.539 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:37:12.539 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.35e+01 / 1.19e+00 | 20260507 06:37:12.539 | INFO | PerfReport: 0.30%, AggregateStats -- 9.06e-02 / 1.02e-01, 9.48e-03 / 3.51e-03 | 20260507 06:37:12.539 | INFO | PerfReport: 11.71%, Baseline -- 3.59e+00 / 4.43e+00, 5.20e-01 / 2.52e-01 | 20260507 06:37:12.539 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.48e+00, 1.69e-01 / 1.29e-01 | 20260507 06:37:12.539 | INFO | PerfReport: 16.26%, DME -- 3.99e+01 / 4.48e+01, 3.95e+00 / 1.43e+00 | 20260507 06:37:12.539 | INFO | PerfReport: 57.62%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.24e+00 / 1.61e+00 | 20260507 06:37:12.539 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.71e+00, 3.89e-01 / 1.98e-01 | 20260507 06:37:12.539 | INFO | PerfReport: 2.96%, Metrics -- 9.09e-01 / 1.22e+00, 2.47e-01 / 1.43e-01 | 20260507 06:37:12.539 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.05e-01 / 3.59e-01, 5.07e-02 / 1.95e-02 | 20260507 06:37:12.539 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:37:12.539 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:37:12.539 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:37:12.539 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 06:37:12.539 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260507 06:37:12.539 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 06:37:12.539 | INFO | PerfReport: 31.95%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:37:12.539 | INFO | PerfReport: 1.53%, LockUpload -- 1.47e+00 / 0.00e+00, 4.89e+00 / 0.00e+00 | 20260507 06:37:12.539 | INFO | PerfReport: 1.24%, LockDownload -- 1.19e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260507 06:37:12.539 | INFO | PerfReport: 22.04%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 06:37:12.539 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.28e-02 / 0.00e+00 | 20260507 06:37:12.634 | INFO | Starting a round of Baz IO | 20260507 06:37:15.727 | INFO | Populating RTMetrics report | 20260507 06:37:15.727 | INFO | RTMetrics report populated | 20260507 06:37:15.727 | INFO | Opening tmp RTMetrics json file | 20260507 06:37:15.744 | INFO | Serializing RTMetrics to json | 20260507 06:37:15.745 | INFO | Serialization done, swapping tmp file now... | 20260507 06:37:15.745 | INFO | RTMetrics done | 20260507 06:37:15.813 | INFO | Finished a round of Baz IO: Wrote 1.664 GiB in 3.178 seconds | 20260507 06:37:15.813 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.61% / 0.00% / 67.38% | 20260507 06:37:15.813 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.119233% / 7.94729e-06% / 4.76837e-05% | 20260507 06:37:15.813 | INFO | Baz IO overall: | 20260507 06:37:15.813 | INFO | BytesWritten : 3962134266785 | 20260507 06:37:15.813 | INFO | NumEvents : 1898430666179 | 20260507 06:37:17.676 | INFO | Analyzing chunk frames = [7864320,7864832) | 20260507 06:37:17.676 | INFO | MemInfo: { | 20260507 06:37:17.676 | INFO | "gpu" : " 17 GB", | 20260507 06:37:17.676 | INFO | "huge" : "", | 20260507 06:37:17.676 | INFO | "huge_shared" : " 27 GB", | 20260507 06:37:17.676 | INFO | "malloc" : " 14 GB", | 20260507 06:37:17.676 | INFO | "pinned" : " 29 GB" | 20260507 06:37:17.676 | INFO | } | 20260507 06:37:17.676 | INFO | Perf report for the last 5.13858 seconds: | 20260507 06:37:17.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:37:17.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:17.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:17.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:17.676 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 06:37:17.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:17.676 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 06:37:17.676 | INFO | PRE_HQ : 59.0%, 1.8, 54.7ms, 0 | 20260507 06:37:17.676 | INFO | BAZWRITER : 61.4%, 3.1, 98.8ms, 0 | 20260507 06:37:17.676 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:37:17.676 | INFO | Uploaded 36GiB to GPU in 2.05027s (17.5587GiB/s) | 20260507 06:37:17.676 | INFO | Downloaded 38GiB to GPU in 2.1759s (17.4641GiB/s) | 20260507 06:37:22.817 | INFO | Analyzing chunk frames = [7864832,7865344) | 20260507 06:37:22.817 | INFO | MemInfo: { | 20260507 06:37:22.817 | INFO | "gpu" : " 17 GB", | 20260507 06:37:22.817 | INFO | "huge" : "", | 20260507 06:37:22.817 | INFO | "huge_shared" : " 27 GB", | 20260507 06:37:22.817 | INFO | "malloc" : " 14 GB", | 20260507 06:37:22.817 | INFO | "pinned" : " 29 GB" | 20260507 06:37:22.817 | INFO | } | 20260507 06:37:22.817 | INFO | Perf report for the last 5.14161 seconds: | 20260507 06:37:22.817 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:37:22.817 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:22.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:22.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:22.818 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 06:37:22.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:22.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:22.818 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 06:37:22.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:22.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:22.818 | INFO | Uploaded 36GiB to GPU in 1.99904s (18.0086GiB/s) | 20260507 06:37:22.818 | INFO | Downloaded 34GiB to GPU in 1.93087s (17.6086GiB/s) | 20260507 06:37:27.956 | INFO | Analyzing chunk frames = [7865344,7865856) | 20260507 06:37:27.956 | INFO | MemInfo: { | 20260507 06:37:27.956 | INFO | "gpu" : " 17 GB", | 20260507 06:37:27.956 | INFO | "huge" : "", | 20260507 06:37:27.956 | INFO | "huge_shared" : " 27 GB", | 20260507 06:37:27.956 | INFO | "malloc" : " 14 GB", | 20260507 06:37:27.956 | INFO | "pinned" : " 29 GB" | 20260507 06:37:27.956 | INFO | } | 20260507 06:37:27.957 | INFO | Perf report for the last 5.13914 seconds: | 20260507 06:37:27.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:37:27.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:27.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:27.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:27.957 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 06:37:27.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:27.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:27.957 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:37:27.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:27.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:27.957 | INFO | Uploaded 36GiB to GPU in 2.05181s (17.5455GiB/s) | 20260507 06:37:27.957 | INFO | Downloaded 34GiB to GPU in 1.95017s (17.4344GiB/s) | 20260507 06:37:33.092 | INFO | Analyzing chunk frames = [7865856,7866368) | 20260507 06:37:33.092 | INFO | MemInfo: { | 20260507 06:37:33.092 | INFO | "gpu" : " 17 GB", | 20260507 06:37:33.092 | INFO | "huge" : "", | 20260507 06:37:33.092 | INFO | "huge_shared" : " 27 GB", | 20260507 06:37:33.092 | INFO | "malloc" : " 14 GB", | 20260507 06:37:33.092 | INFO | "pinned" : " 29 GB" | 20260507 06:37:33.092 | INFO | } | 20260507 06:37:33.092 | INFO | Perf report for the last 5.1356 seconds: | 20260507 06:37:33.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:37:33.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:33.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:33.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:33.092 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 06:37:33.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:33.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:33.092 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:37:33.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:33.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:33.093 | INFO | Uploaded 36GiB to GPU in 2.05274s (17.5376GiB/s) | 20260507 06:37:33.093 | INFO | Downloaded 34GiB to GPU in 1.94349s (17.4943GiB/s) | 20260507 06:37:38.230 | INFO | Analyzing chunk frames = [7866368,7866880) | 20260507 06:37:38.230 | INFO | MemInfo: { | 20260507 06:37:38.230 | INFO | "gpu" : " 17 GB", | 20260507 06:37:38.230 | INFO | "huge" : "", | 20260507 06:37:38.230 | INFO | "huge_shared" : " 27 GB", | 20260507 06:37:38.230 | INFO | "malloc" : " 14 GB", | 20260507 06:37:38.230 | INFO | "pinned" : " 29 GB" | 20260507 06:37:38.230 | INFO | } | 20260507 06:37:38.230 | INFO | Perf report for the last 5.13811 seconds: | 20260507 06:37:38.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:37:38.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:38.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:38.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:38.230 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 06:37:38.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:38.230 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:38.230 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:37:38.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:38.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:38.231 | INFO | Uploaded 36GiB to GPU in 2.05891s (17.485GiB/s) | 20260507 06:37:38.231 | INFO | Downloaded 34GiB to GPU in 1.95125s (17.4247GiB/s) | 20260507 06:37:43.374 | INFO | Analyzing chunk frames = [7866880,7867392) | 20260507 06:37:43.374 | INFO | MemInfo: { | 20260507 06:37:43.374 | INFO | "gpu" : " 17 GB", | 20260507 06:37:43.374 | INFO | "huge" : "", | 20260507 06:37:43.374 | INFO | "huge_shared" : " 27 GB", | 20260507 06:37:43.374 | INFO | "malloc" : " 14 GB", | 20260507 06:37:43.374 | INFO | "pinned" : " 29 GB" | 20260507 06:37:43.374 | INFO | } | 20260507 06:37:43.374 | INFO | Perf report for the last 5.14355 seconds: | 20260507 06:37:43.374 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:37:43.374 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:43.374 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:43.374 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:43.374 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:37:43.374 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:43.374 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:43.374 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 06:37:43.374 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:43.374 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:43.374 | INFO | Uploaded 36GiB to GPU in 2.05697s (17.5014GiB/s) | 20260507 06:37:43.374 | INFO | Downloaded 34GiB to GPU in 1.91099s (17.7918GiB/s) | 20260507 06:37:48.513 | INFO | Analyzing chunk frames = [7867392,7867904) | 20260507 06:37:48.513 | INFO | MemInfo: { | 20260507 06:37:48.513 | INFO | "gpu" : " 17 GB", | 20260507 06:37:48.513 | INFO | "huge" : "", | 20260507 06:37:48.513 | INFO | "huge_shared" : " 27 GB", | 20260507 06:37:48.513 | INFO | "malloc" : " 14 GB", | 20260507 06:37:48.513 | INFO | "pinned" : " 29 GB" | 20260507 06:37:48.513 | INFO | } | 20260507 06:37:48.514 | INFO | Perf report for the last 5.13964 seconds: | 20260507 06:37:48.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:37:48.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:48.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:48.514 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:48.514 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:37:48.514 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:48.514 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:48.514 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 06:37:48.514 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:48.514 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:48.514 | INFO | Uploaded 36GiB to GPU in 1.9341s (18.6133GiB/s) | 20260507 06:37:48.514 | INFO | Downloaded 34GiB to GPU in 1.96443s (17.3078GiB/s) | 20260507 06:37:53.646 | INFO | Analyzing chunk frames = [7867904,7868416) | 20260507 06:37:53.646 | INFO | MemInfo: { | 20260507 06:37:53.646 | INFO | "gpu" : " 17 GB", | 20260507 06:37:53.646 | INFO | "huge" : "", | 20260507 06:37:53.646 | INFO | "huge_shared" : " 27 GB", | 20260507 06:37:53.646 | INFO | "malloc" : " 14 GB", | 20260507 06:37:53.646 | INFO | "pinned" : " 29 GB" | 20260507 06:37:53.646 | INFO | } | 20260507 06:37:53.646 | INFO | Perf report for the last 5.13273 seconds: | 20260507 06:37:53.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:37:53.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:53.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:53.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:53.647 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 06:37:53.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:53.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:53.647 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:37:53.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:53.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:53.647 | INFO | Uploaded 36GiB to GPU in 1.99403s (18.0539GiB/s) | 20260507 06:37:53.647 | INFO | Downloaded 34GiB to GPU in 1.95557s (17.3862GiB/s) | 20260507 06:37:53.743 | INFO | Starting a round of Baz IO | 20260507 06:37:56.837 | INFO | Populating RTMetrics report | 20260507 06:37:56.838 | INFO | RTMetrics report populated | 20260507 06:37:56.838 | INFO | Opening tmp RTMetrics json file | 20260507 06:37:56.846 | INFO | Serializing RTMetrics to json | 20260507 06:37:56.854 | INFO | Serialization done, swapping tmp file now... | 20260507 06:37:56.854 | INFO | RTMetrics done | 20260507 06:37:56.921 | INFO | Finished a round of Baz IO: Wrote 1.664 GiB in 3.177 seconds | 20260507 06:37:56.921 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.61% / 0.00% / 67.39% | 20260507 06:37:56.921 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.119436% / 0% / 3.97364e-05% | 20260507 06:37:56.921 | INFO | Baz IO overall: | 20260507 06:37:56.921 | INFO | BytesWritten : 3963921142430 | 20260507 06:37:56.921 | INFO | NumEvents : 1899266695169 | 20260507 06:37:58.785 | INFO | Analyzing chunk frames = [7868416,7868928) | 20260507 06:37:58.785 | INFO | MemInfo: { | 20260507 06:37:58.785 | INFO | "gpu" : " 17 GB", | 20260507 06:37:58.785 | INFO | "huge" : "", | 20260507 06:37:58.785 | INFO | "huge_shared" : " 27 GB", | 20260507 06:37:58.785 | INFO | "malloc" : " 14 GB", | 20260507 06:37:58.785 | INFO | "pinned" : " 29 GB" | 20260507 06:37:58.785 | INFO | } | 20260507 06:37:58.785 | INFO | Perf report for the last 5.13898 seconds: | 20260507 06:37:58.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:37:58.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:58.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:58.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:58.785 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 06:37:58.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:37:58.785 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260507 06:37:58.785 | INFO | PRE_HQ : 58.9%, 1.8, 54.6ms, 0 | 20260507 06:37:58.785 | INFO | BAZWRITER : 61.4%, 3.1, 98.2ms, 0 | 20260507 06:37:58.785 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:37:58.786 | INFO | Uploaded 36GiB to GPU in 2.05657s (17.5049GiB/s) | 20260507 06:37:58.786 | INFO | Downloaded 38GiB to GPU in 2.17759s (17.4505GiB/s) | 20260507 06:38:03.931 | INFO | Analyzing chunk frames = [7868928,7869440) | 20260507 06:38:03.931 | INFO | MemInfo: { | 20260507 06:38:03.931 | INFO | "gpu" : " 17 GB", | 20260507 06:38:03.931 | INFO | "huge" : "", | 20260507 06:38:03.931 | INFO | "huge_shared" : " 27 GB", | 20260507 06:38:03.931 | INFO | "malloc" : " 14 GB", | 20260507 06:38:03.931 | INFO | "pinned" : " 29 GB" | 20260507 06:38:03.932 | INFO | } | 20260507 06:38:03.932 | INFO | Perf report for the last 5.14625 seconds: | 20260507 06:38:03.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:38:03.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:03.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:03.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:03.932 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 06:38:03.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:03.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:03.932 | INFO | PRE_HQ : 24.3%, 1.0, 12.7ms, 0 | 20260507 06:38:03.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:03.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:03.932 | INFO | Uploaded 36GiB to GPU in 2.04428s (17.6101GiB/s) | 20260507 06:38:03.932 | INFO | Downloaded 34GiB to GPU in 1.94283s (17.5003GiB/s) | 20260507 06:38:09.070 | INFO | Analyzing chunk frames = [7869440,7869952) | 20260507 06:38:09.070 | INFO | MemInfo: { | 20260507 06:38:09.070 | INFO | "gpu" : " 17 GB", | 20260507 06:38:09.070 | INFO | "huge" : "", | 20260507 06:38:09.070 | INFO | "huge_shared" : " 27 GB", | 20260507 06:38:09.070 | INFO | "malloc" : " 14 GB", | 20260507 06:38:09.070 | INFO | "pinned" : " 29 GB" | 20260507 06:38:09.070 | INFO | } | 20260507 06:38:09.070 | INFO | Perf report for the last 5.13811 seconds: | 20260507 06:38:09.070 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:38:09.070 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:09.070 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:09.070 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:09.070 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 06:38:09.070 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:09.070 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:09.070 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:38:09.070 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:09.070 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:09.070 | INFO | Uploaded 36GiB to GPU in 2.04511s (17.603GiB/s) | 20260507 06:38:09.070 | INFO | Downloaded 34GiB to GPU in 1.94856s (17.4488GiB/s) | 20260507 06:38:14.206 | INFO | Analyzing chunk frames = [7869952,7870464) | 20260507 06:38:14.206 | INFO | MemInfo: { | 20260507 06:38:14.206 | INFO | "gpu" : " 17 GB", | 20260507 06:38:14.206 | INFO | "huge" : "", | 20260507 06:38:14.206 | INFO | "huge_shared" : " 27 GB", | 20260507 06:38:14.206 | INFO | "malloc" : " 14 GB", | 20260507 06:38:14.206 | INFO | "pinned" : " 29 GB" | 20260507 06:38:14.206 | INFO | } | 20260507 06:38:14.206 | INFO | Perf report for the last 5.13588 seconds: | 20260507 06:38:14.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:38:14.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:14.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:14.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:14.206 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 06:38:14.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:14.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:14.206 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:38:14.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:14.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:14.206 | INFO | Uploaded 36GiB to GPU in 2.05503s (17.518GiB/s) | 20260507 06:38:14.206 | INFO | Downloaded 34GiB to GPU in 1.94737s (17.4594GiB/s) | 20260507 06:38:19.344 | INFO | Analyzing chunk frames = [7870464,7870976) | 20260507 06:38:19.344 | INFO | MemInfo: { | 20260507 06:38:19.344 | INFO | "gpu" : " 17 GB", | 20260507 06:38:19.344 | INFO | "huge" : "", | 20260507 06:38:19.344 | INFO | "huge_shared" : " 27 GB", | 20260507 06:38:19.344 | INFO | "malloc" : " 14 GB", | 20260507 06:38:19.344 | INFO | "pinned" : " 29 GB" | 20260507 06:38:19.344 | INFO | } | 20260507 06:38:19.344 | INFO | Perf report for the last 5.13855 seconds: | 20260507 06:38:19.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:38:19.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:19.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:19.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:19.345 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 06:38:19.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:19.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:19.345 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:38:19.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:19.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:19.345 | INFO | Uploaded 36GiB to GPU in 2.02708s (17.7596GiB/s) | 20260507 06:38:19.345 | INFO | Downloaded 34GiB to GPU in 1.90895s (17.8109GiB/s) | 20260507 06:38:24.486 | INFO | Analyzing chunk frames = [7870976,7871488) | 20260507 06:38:24.486 | INFO | MemInfo: { | 20260507 06:38:24.486 | INFO | "gpu" : " 17 GB", | 20260507 06:38:24.486 | INFO | "huge" : "", | 20260507 06:38:24.486 | INFO | "huge_shared" : " 27 GB", | 20260507 06:38:24.486 | INFO | "malloc" : " 14 GB", | 20260507 06:38:24.486 | INFO | "pinned" : " 29 GB" | 20260507 06:38:24.486 | INFO | } | 20260507 06:38:24.486 | INFO | Perf report for the last 5.14229 seconds: | 20260507 06:38:24.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:38:24.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:24.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:24.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:24.487 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:38:24.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:24.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:24.487 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 06:38:24.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:24.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:24.487 | INFO | Uploaded 36GiB to GPU in 2.05448s (17.5227GiB/s) | 20260507 06:38:24.487 | INFO | Downloaded 34GiB to GPU in 1.91177s (17.7846GiB/s) | 20260507 06:38:29.629 | INFO | Analyzing chunk frames = [7871488,7872000) | 20260507 06:38:29.629 | INFO | MemInfo: { | 20260507 06:38:29.629 | INFO | "gpu" : " 17 GB", | 20260507 06:38:29.629 | INFO | "huge" : "", | 20260507 06:38:29.629 | INFO | "huge_shared" : " 27 GB", | 20260507 06:38:29.629 | INFO | "malloc" : " 14 GB", | 20260507 06:38:29.629 | INFO | "pinned" : " 29 GB" | 20260507 06:38:29.629 | INFO | } | 20260507 06:38:29.629 | INFO | Perf report for the last 5.14229 seconds: | 20260507 06:38:29.629 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:38:29.629 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:29.629 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:29.629 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:29.629 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:38:29.629 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:29.629 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:29.629 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 06:38:29.629 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:29.629 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:29.629 | INFO | Uploaded 36GiB to GPU in 2.01163s (17.8959GiB/s) | 20260507 06:38:29.629 | INFO | Downloaded 34GiB to GPU in 1.98515s (17.1272GiB/s) | 20260507 06:38:34.767 | INFO | Analyzing chunk frames = [7872000,7872512) | 20260507 06:38:34.767 | INFO | MemInfo: { | 20260507 06:38:34.767 | INFO | "gpu" : " 17 GB", | 20260507 06:38:34.767 | INFO | "huge" : "", | 20260507 06:38:34.767 | INFO | "huge_shared" : " 27 GB", | 20260507 06:38:34.767 | INFO | "malloc" : " 14 GB", | 20260507 06:38:34.767 | INFO | "pinned" : " 29 GB" | 20260507 06:38:34.767 | INFO | } | 20260507 06:38:34.767 | INFO | Perf report for the last 5.13821 seconds: | 20260507 06:38:34.767 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:38:34.767 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:34.767 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:34.767 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:34.767 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 06:38:34.767 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:34.767 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:34.767 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:38:34.767 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:34.767 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:34.767 | INFO | Uploaded 36GiB to GPU in 1.99241s (18.0686GiB/s) | 20260507 06:38:34.767 | INFO | Downloaded 34GiB to GPU in 1.95883s (17.3573GiB/s) | 20260507 06:38:34.767 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:38:34.767 | INFO | Total mem usage: 18 GiB | 20260507 06:38:34.767 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:38:34.767 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 06:38:34.767 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:38:34.767 | INFO | High Water Breakdown: | 20260507 06:38:34.767 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:38:34.767 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:38:34.767 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:38:34.767 | INFO | ------------------------------------------ | 20260507 06:38:34.768 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:38:34.768 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:38:34.768 | INFO | Total mem usage: 26.5 GiB | 20260507 06:38:34.768 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:38:34.768 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:38:34.768 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:38:34.768 | INFO | High Water Breakdown: | 20260507 06:38:34.768 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:38:34.768 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:38:34.768 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:38:34.768 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:38:34.768 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:38:34.768 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:38:34.768 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:38:34.768 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:38:34.768 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:38:34.768 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:38:34.768 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:38:34.768 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:38:34.768 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:38:34.768 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:38:34.768 | INFO | ------------------------------------------ | 20260507 06:38:34.768 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:38:34.768 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:38:34.768 | INFO | Total mem usage: 25.6 GiB | 20260507 06:38:34.768 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:38:34.768 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:38:34.768 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:38:34.768 | INFO | High Water Breakdown: | 20260507 06:38:34.768 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:38:34.768 | INFO | ------------------------------------------ | 20260507 06:38:34.768 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:38:34.768 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:38:34.768 | INFO | Total mem usage: 15.7 GiB | 20260507 06:38:34.768 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:38:34.768 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:38:34.768 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:38:34.768 | INFO | High Water Breakdown: | 20260507 06:38:34.768 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:38:34.768 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:38:34.768 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:38:34.768 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:38:34.768 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:38:34.768 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:38:34.768 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:38:34.768 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:38:34.768 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:38:34.768 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:38:34.768 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:38:34.768 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:38:34.768 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:38:34.768 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:38:34.768 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:38:34.768 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:38:34.768 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:38:34.768 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:38:34.768 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:38:34.768 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:38:34.768 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:38:34.768 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:38:34.768 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:38:34.768 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:38:34.768 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:38:34.768 | INFO | ------------------------------------------ | 20260507 06:38:34.768 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:38:34.768 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:38:34.768 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:38:34.768 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:38:34.769 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.47e+01, 1.33e+01 / 1.16e+00 | 20260507 06:38:34.769 | INFO | PerfReport: 0.29%, AggregateStats -- 9.01e-02 / 1.01e-01, 8.91e-03 / 4.34e-03 | 20260507 06:38:34.769 | INFO | PerfReport: 11.62%, Baseline -- 3.56e+00 / 4.50e+00, 4.83e-01 / 1.67e-01 | 20260507 06:38:34.769 | INFO | PerfReport: 3.20%, Histogram -- 9.81e-01 / 1.51e+00, 1.62e-01 / 8.73e-02 | 20260507 06:38:34.769 | INFO | PerfReport: 16.34%, DME -- 4.00e+01 / 4.50e+01, 3.95e+00 / 1.37e+00 | 20260507 06:38:34.769 | INFO | PerfReport: 57.64%, FrameLabeling -- 1.77e+01 / 1.93e+01, 1.18e+00 / 1.03e+00 | 20260507 06:38:34.769 | INFO | PerfReport: 6.95%, PulseAccumulating -- 2.13e+00 / 2.73e+00, 3.90e-01 / 1.69e-01 | 20260507 06:38:34.769 | INFO | PerfReport: 2.96%, Metrics -- 9.07e-01 / 1.21e+00, 2.44e-01 / 1.04e-01 | 20260507 06:38:34.769 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.62e-01, 5.03e-02 / 2.05e-02 | 20260507 06:38:34.769 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:38:34.769 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:38:34.769 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:38:34.769 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 06:38:34.769 | INFO | PerfReport: 21.97%, Upload -- 2.11e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 06:38:34.769 | INFO | PerfReport: 21.35%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 06:38:34.769 | INFO | PerfReport: 31.82%, Compute -- 3.06e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 06:38:34.769 | INFO | PerfReport: 1.73%, LockUpload -- 1.66e+00 / 0.00e+00, 5.10e+00 / 0.00e+00 | 20260507 06:38:34.769 | INFO | PerfReport: 1.16%, LockDownload -- 1.11e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260507 06:38:34.769 | INFO | PerfReport: 21.96%, LockCompute -- 2.11e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 06:38:34.769 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.27e-02 / 0.00e+00 | 20260507 06:38:34.864 | INFO | Starting a round of Baz IO | 20260507 06:38:37.962 | INFO | Populating RTMetrics report | 20260507 06:38:37.962 | INFO | RTMetrics report populated | 20260507 06:38:37.962 | INFO | Opening tmp RTMetrics json file | 20260507 06:38:37.978 | INFO | Serializing RTMetrics to json | 20260507 06:38:37.979 | INFO | Serialization done, swapping tmp file now... | 20260507 06:38:37.980 | INFO | RTMetrics done | 20260507 06:38:38.047 | INFO | Finished a round of Baz IO: Wrote 1.664 GiB in 3.183 seconds | 20260507 06:38:38.047 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.61% / 0.00% / 67.39% | 20260507 06:38:38.047 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.119483% / 3.97364e-06% / 3.57628e-05% | 20260507 06:38:38.047 | INFO | Baz IO overall: | 20260507 06:38:38.048 | INFO | BytesWritten : 3965708169605 | 20260507 06:38:38.048 | INFO | NumEvents : 1900103040716 | 20260507 06:38:39.904 | INFO | Analyzing chunk frames = [7872512,7873024) | 20260507 06:38:39.905 | INFO | MemInfo: { | 20260507 06:38:39.905 | INFO | "gpu" : " 17 GB", | 20260507 06:38:39.905 | INFO | "huge" : "", | 20260507 06:38:39.905 | INFO | "huge_shared" : " 27 GB", | 20260507 06:38:39.905 | INFO | "malloc" : " 14 GB", | 20260507 06:38:39.905 | INFO | "pinned" : " 29 GB" | 20260507 06:38:39.905 | INFO | } | 20260507 06:38:39.905 | INFO | Perf report for the last 5.13757 seconds: | 20260507 06:38:39.905 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:38:39.905 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:39.905 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:39.905 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:39.905 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 06:38:39.905 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:39.905 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 06:38:39.905 | INFO | PRE_HQ : 59.2%, 1.8, 54.6ms, 0 | 20260507 06:38:39.905 | INFO | BAZWRITER : 61.6%, 3.1, 98.7ms, 0 | 20260507 06:38:39.905 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:38:39.905 | INFO | Uploaded 36GiB to GPU in 2.06078s (17.4691GiB/s) | 20260507 06:38:39.905 | INFO | Downloaded 38GiB to GPU in 2.17413s (17.4782GiB/s) | 20260507 06:38:45.041 | INFO | Analyzing chunk frames = [7873024,7873536) | 20260507 06:38:45.041 | INFO | MemInfo: { | 20260507 06:38:45.041 | INFO | "gpu" : " 17 GB", | 20260507 06:38:45.041 | INFO | "huge" : "", | 20260507 06:38:45.041 | INFO | "huge_shared" : " 27 GB", | 20260507 06:38:45.041 | INFO | "malloc" : " 14 GB", | 20260507 06:38:45.041 | INFO | "pinned" : " 29 GB" | 20260507 06:38:45.041 | INFO | } | 20260507 06:38:45.042 | INFO | Perf report for the last 5.13697 seconds: | 20260507 06:38:45.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:38:45.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:45.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:45.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:45.042 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:38:45.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:45.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:45.042 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:38:45.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:45.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:45.042 | INFO | Uploaded 36GiB to GPU in 2.00308s (17.9723GiB/s) | 20260507 06:38:45.042 | INFO | Downloaded 34GiB to GPU in 1.9343s (17.5774GiB/s) | 20260507 06:38:50.183 | INFO | Analyzing chunk frames = [7873536,7874048) | 20260507 06:38:50.183 | INFO | MemInfo: { | 20260507 06:38:50.183 | INFO | "gpu" : " 17 GB", | 20260507 06:38:50.183 | INFO | "huge" : "", | 20260507 06:38:50.183 | INFO | "huge_shared" : " 27 GB", | 20260507 06:38:50.183 | INFO | "malloc" : " 14 GB", | 20260507 06:38:50.183 | INFO | "pinned" : " 29 GB" | 20260507 06:38:50.183 | INFO | } | 20260507 06:38:50.183 | INFO | Perf report for the last 5.14191 seconds: | 20260507 06:38:50.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:38:50.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:50.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:50.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:50.183 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:38:50.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:50.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:50.184 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 06:38:50.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:50.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:50.184 | INFO | Uploaded 36GiB to GPU in 2.03576s (17.6838GiB/s) | 20260507 06:38:50.184 | INFO | Downloaded 34GiB to GPU in 1.95028s (17.4334GiB/s) | 20260507 06:38:55.324 | INFO | Analyzing chunk frames = [7874048,7874560) | 20260507 06:38:55.324 | INFO | MemInfo: { | 20260507 06:38:55.324 | INFO | "gpu" : " 17 GB", | 20260507 06:38:55.324 | INFO | "huge" : "", | 20260507 06:38:55.324 | INFO | "huge_shared" : " 27 GB", | 20260507 06:38:55.324 | INFO | "malloc" : " 14 GB", | 20260507 06:38:55.324 | INFO | "pinned" : " 29 GB" | 20260507 06:38:55.324 | INFO | } | 20260507 06:38:55.324 | INFO | Perf report for the last 5.14082 seconds: | 20260507 06:38:55.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:38:55.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:55.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:55.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:55.324 | INFO | ANALYSIS : 59.5%, 3.0, 92.3ms, 0 | 20260507 06:38:55.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:55.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:55.324 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:38:55.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:55.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:38:55.325 | INFO | Uploaded 36GiB to GPU in 1.99404s (18.0538GiB/s) | 20260507 06:38:55.325 | INFO | Downloaded 34GiB to GPU in 1.91851s (17.7221GiB/s) | 20260507 06:39:00.468 | INFO | Analyzing chunk frames = [7874560,7875072) | 20260507 06:39:00.468 | INFO | MemInfo: { | 20260507 06:39:00.468 | INFO | "gpu" : " 17 GB", | 20260507 06:39:00.468 | INFO | "huge" : "", | 20260507 06:39:00.468 | INFO | "huge_shared" : " 27 GB", | 20260507 06:39:00.468 | INFO | "malloc" : " 14 GB", | 20260507 06:39:00.468 | INFO | "pinned" : " 29 GB" | 20260507 06:39:00.468 | INFO | } | 20260507 06:39:00.468 | INFO | Perf report for the last 5.1439 seconds: | 20260507 06:39:00.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:39:00.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:00.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:00.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:00.468 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 06:39:00.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:00.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:00.468 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:39:00.469 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:00.469 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:00.469 | INFO | Uploaded 36GiB to GPU in 2.03411s (17.6981GiB/s) | 20260507 06:39:00.469 | INFO | Downloaded 34GiB to GPU in 1.91607s (17.7447GiB/s) | 20260507 06:39:05.594 | INFO | Analyzing chunk frames = [7875072,7875584) | 20260507 06:39:05.594 | INFO | MemInfo: { | 20260507 06:39:05.594 | INFO | "gpu" : " 17 GB", | 20260507 06:39:05.594 | INFO | "huge" : "", | 20260507 06:39:05.594 | INFO | "huge_shared" : " 27 GB", | 20260507 06:39:05.594 | INFO | "malloc" : " 14 GB", | 20260507 06:39:05.594 | INFO | "pinned" : " 29 GB" | 20260507 06:39:05.594 | INFO | } | 20260507 06:39:05.594 | INFO | Perf report for the last 5.12595 seconds: | 20260507 06:39:05.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:39:05.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:05.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:05.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:05.594 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:39:05.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:05.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:05.594 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 06:39:05.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:05.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:05.594 | INFO | Uploaded 36GiB to GPU in 2.07651s (17.3368GiB/s) | 20260507 06:39:05.595 | INFO | Downloaded 34GiB to GPU in 1.94506s (17.4802GiB/s) | 20260507 06:39:10.740 | INFO | Analyzing chunk frames = [7875584,7876096) | 20260507 06:39:10.740 | INFO | MemInfo: { | 20260507 06:39:10.740 | INFO | "gpu" : " 17 GB", | 20260507 06:39:10.740 | INFO | "huge" : "", | 20260507 06:39:10.741 | INFO | "huge_shared" : " 27 GB", | 20260507 06:39:10.741 | INFO | "malloc" : " 14 GB", | 20260507 06:39:10.741 | INFO | "pinned" : " 29 GB" | 20260507 06:39:10.741 | INFO | } | 20260507 06:39:10.741 | INFO | Perf report for the last 5.14657 seconds: | 20260507 06:39:10.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:39:10.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:10.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:10.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:10.741 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 06:39:10.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:10.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:10.741 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:39:10.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:10.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:10.741 | INFO | Uploaded 36GiB to GPU in 1.98034s (18.1787GiB/s) | 20260507 06:39:10.741 | INFO | Downloaded 34GiB to GPU in 1.98715s (17.1099GiB/s) | 20260507 06:39:15.880 | INFO | Analyzing chunk frames = [7876096,7876608) | 20260507 06:39:15.880 | INFO | MemInfo: { | 20260507 06:39:15.880 | INFO | "gpu" : " 17 GB", | 20260507 06:39:15.880 | INFO | "huge" : "", | 20260507 06:39:15.880 | INFO | "huge_shared" : " 27 GB", | 20260507 06:39:15.880 | INFO | "malloc" : " 14 GB", | 20260507 06:39:15.880 | INFO | "pinned" : " 29 GB" | 20260507 06:39:15.880 | INFO | } | 20260507 06:39:15.880 | INFO | Perf report for the last 5.13902 seconds: | 20260507 06:39:15.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:39:15.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:15.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:15.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:15.880 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:39:15.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:15.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:15.880 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:39:15.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:15.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:15.880 | INFO | Uploaded 36GiB to GPU in 1.96494s (18.3211GiB/s) | 20260507 06:39:15.880 | INFO | Downloaded 34GiB to GPU in 1.9502s (17.4341GiB/s) | 20260507 06:39:15.977 | INFO | Starting a round of Baz IO | 20260507 06:39:19.068 | INFO | Populating RTMetrics report | 20260507 06:39:19.068 | INFO | RTMetrics report populated | 20260507 06:39:19.068 | INFO | Opening tmp RTMetrics json file | 20260507 06:39:19.073 | INFO | Serializing RTMetrics to json | 20260507 06:39:19.086 | INFO | Serialization done, swapping tmp file now... | 20260507 06:39:19.087 | INFO | RTMetrics done | 20260507 06:39:19.158 | INFO | Finished a round of Baz IO: Wrote 1.662 GiB in 3.181 seconds | 20260507 06:39:19.158 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.61% / 0.00% / 67.39% | 20260507 06:39:19.158 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.118423% / 3.97364e-06% / 8.74201e-05% | 20260507 06:39:19.158 | INFO | Baz IO overall: | 20260507 06:39:19.158 | INFO | BytesWritten : 3967493163388 | 20260507 06:39:19.158 | INFO | NumEvents : 1900937921110 | 20260507 06:39:21.016 | INFO | Analyzing chunk frames = [7876608,7877120) | 20260507 06:39:21.016 | INFO | MemInfo: { | 20260507 06:39:21.016 | INFO | "gpu" : " 17 GB", | 20260507 06:39:21.016 | INFO | "huge" : "", | 20260507 06:39:21.016 | INFO | "huge_shared" : " 27 GB", | 20260507 06:39:21.017 | INFO | "malloc" : " 14 GB", | 20260507 06:39:21.017 | INFO | "pinned" : " 29 GB" | 20260507 06:39:21.017 | INFO | } | 20260507 06:39:21.017 | INFO | Perf report for the last 5.13682 seconds: | 20260507 06:39:21.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:39:21.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:21.017 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:21.017 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:21.017 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 06:39:21.017 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:21.017 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 06:39:21.017 | INFO | PRE_HQ : 58.9%, 1.8, 54.7ms, 0 | 20260507 06:39:21.017 | INFO | BAZWRITER : 61.5%, 3.1, 98.7ms, 0 | 20260507 06:39:21.017 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:39:21.017 | INFO | Uploaded 36GiB to GPU in 2.05531s (17.5156GiB/s) | 20260507 06:39:21.017 | INFO | Downloaded 38GiB to GPU in 2.17312s (17.4864GiB/s) | 20260507 06:39:26.154 | INFO | Analyzing chunk frames = [7877120,7877632) | 20260507 06:39:26.154 | INFO | MemInfo: { | 20260507 06:39:26.154 | INFO | "gpu" : " 17 GB", | 20260507 06:39:26.154 | INFO | "huge" : "", | 20260507 06:39:26.154 | INFO | "huge_shared" : " 27 GB", | 20260507 06:39:26.154 | INFO | "malloc" : " 14 GB", | 20260507 06:39:26.154 | INFO | "pinned" : " 29 GB" | 20260507 06:39:26.154 | INFO | } | 20260507 06:39:26.154 | INFO | Perf report for the last 5.13737 seconds: | 20260507 06:39:26.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:39:26.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:26.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:26.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:26.154 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:39:26.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:26.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:26.154 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 06:39:26.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:26.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:26.154 | INFO | Uploaded 36GiB to GPU in 1.99217s (18.0707GiB/s) | 20260507 06:39:26.155 | INFO | Downloaded 34GiB to GPU in 1.92899s (17.6258GiB/s) | 20260507 06:39:31.293 | INFO | Analyzing chunk frames = [7877632,7878144) | 20260507 06:39:31.293 | INFO | MemInfo: { | 20260507 06:39:31.293 | INFO | "gpu" : " 17 GB", | 20260507 06:39:31.293 | INFO | "huge" : "", | 20260507 06:39:31.293 | INFO | "huge_shared" : " 27 GB", | 20260507 06:39:31.293 | INFO | "malloc" : " 14 GB", | 20260507 06:39:31.293 | INFO | "pinned" : " 29 GB" | 20260507 06:39:31.293 | INFO | } | 20260507 06:39:31.293 | INFO | Perf report for the last 5.13937 seconds: | 20260507 06:39:31.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:39:31.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:31.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:31.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:31.294 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 06:39:31.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:31.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:31.294 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:39:31.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:31.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:31.294 | INFO | Uploaded 36GiB to GPU in 2.05292s (17.536GiB/s) | 20260507 06:39:31.294 | INFO | Downloaded 34GiB to GPU in 1.95028s (17.4334GiB/s) | 20260507 06:39:36.439 | INFO | Analyzing chunk frames = [7878144,7878656) | 20260507 06:39:36.439 | INFO | MemInfo: { | 20260507 06:39:36.439 | INFO | "gpu" : " 17 GB", | 20260507 06:39:36.439 | INFO | "huge" : "", | 20260507 06:39:36.439 | INFO | "huge_shared" : " 27 GB", | 20260507 06:39:36.439 | INFO | "malloc" : " 14 GB", | 20260507 06:39:36.439 | INFO | "pinned" : " 29 GB" | 20260507 06:39:36.439 | INFO | } | 20260507 06:39:36.440 | INFO | Perf report for the last 5.14613 seconds: | 20260507 06:39:36.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:39:36.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:36.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:36.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:36.440 | INFO | ANALYSIS : 59.5%, 3.0, 92.5ms, 0 | 20260507 06:39:36.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:36.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:36.440 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:39:36.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:36.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:36.440 | INFO | Uploaded 36GiB to GPU in 2.01165s (17.8957GiB/s) | 20260507 06:39:36.440 | INFO | Downloaded 34GiB to GPU in 1.92972s (17.6191GiB/s) | 20260507 06:39:41.584 | INFO | Analyzing chunk frames = [7878656,7879168) | 20260507 06:39:41.584 | INFO | MemInfo: { | 20260507 06:39:41.584 | INFO | "gpu" : " 17 GB", | 20260507 06:39:41.584 | INFO | "huge" : "", | 20260507 06:39:41.584 | INFO | "huge_shared" : " 27 GB", | 20260507 06:39:41.584 | INFO | "malloc" : " 14 GB", | 20260507 06:39:41.584 | INFO | "pinned" : " 29 GB" | 20260507 06:39:41.584 | INFO | } | 20260507 06:39:41.584 | INFO | Perf report for the last 5.14483 seconds: | 20260507 06:39:41.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:39:41.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:41.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:41.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:41.585 | INFO | ANALYSIS : 60.1%, 3.0, 94.0ms, 0 | 20260507 06:39:41.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:41.585 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:41.585 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:39:41.585 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:41.585 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:41.585 | INFO | Uploaded 36GiB to GPU in 2.05798s (17.4928GiB/s) | 20260507 06:39:41.585 | INFO | Downloaded 34GiB to GPU in 1.94993s (17.4365GiB/s) | 20260507 06:39:46.723 | INFO | Analyzing chunk frames = [7879168,7879680) | 20260507 06:39:46.723 | INFO | MemInfo: { | 20260507 06:39:46.723 | INFO | "gpu" : " 17 GB", | 20260507 06:39:46.723 | INFO | "huge" : "", | 20260507 06:39:46.723 | INFO | "huge_shared" : " 27 GB", | 20260507 06:39:46.723 | INFO | "malloc" : " 14 GB", | 20260507 06:39:46.723 | INFO | "pinned" : " 29 GB" | 20260507 06:39:46.723 | INFO | } | 20260507 06:39:46.724 | INFO | Perf report for the last 5.13934 seconds: | 20260507 06:39:46.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:39:46.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:46.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:46.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:46.724 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:39:46.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:46.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:46.724 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 06:39:46.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:46.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:46.724 | INFO | Uploaded 36GiB to GPU in 2.04135s (17.6354GiB/s) | 20260507 06:39:46.724 | INFO | Downloaded 34GiB to GPU in 1.92488s (17.6635GiB/s) | 20260507 06:39:51.851 | INFO | Analyzing chunk frames = [7879680,7880192) | 20260507 06:39:51.851 | INFO | MemInfo: { | 20260507 06:39:51.851 | INFO | "gpu" : " 17 GB", | 20260507 06:39:51.851 | INFO | "huge" : "", | 20260507 06:39:51.851 | INFO | "huge_shared" : " 27 GB", | 20260507 06:39:51.851 | INFO | "malloc" : " 14 GB", | 20260507 06:39:51.851 | INFO | "pinned" : " 29 GB" | 20260507 06:39:51.851 | INFO | } | 20260507 06:39:51.851 | INFO | Perf report for the last 5.12746 seconds: | 20260507 06:39:51.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:39:51.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:51.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:51.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:51.851 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 06:39:51.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:51.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:51.851 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:39:51.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:51.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:51.852 | INFO | Uploaded 36GiB to GPU in 1.98488s (18.1371GiB/s) | 20260507 06:39:51.852 | INFO | Downloaded 34GiB to GPU in 2s (17GiB/s) | 20260507 06:39:56.993 | INFO | Analyzing chunk frames = [7880192,7880704) | 20260507 06:39:56.993 | INFO | MemInfo: { | 20260507 06:39:56.993 | INFO | "gpu" : " 17 GB", | 20260507 06:39:56.993 | INFO | "huge" : "", | 20260507 06:39:56.993 | INFO | "huge_shared" : " 27 GB", | 20260507 06:39:56.993 | INFO | "malloc" : " 14 GB", | 20260507 06:39:56.993 | INFO | "pinned" : " 29 GB" | 20260507 06:39:56.993 | INFO | } | 20260507 06:39:56.993 | INFO | Perf report for the last 5.14166 seconds: | 20260507 06:39:56.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:39:56.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:56.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:56.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:56.993 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 06:39:56.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:56.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:56.993 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:39:56.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:56.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:39:56.993 | INFO | Uploaded 36GiB to GPU in 1.95806s (18.3855GiB/s) | 20260507 06:39:56.993 | INFO | Downloaded 34GiB to GPU in 1.95116s (17.4255GiB/s) | 20260507 06:39:56.993 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:39:56.993 | INFO | Total mem usage: 18 GiB | 20260507 06:39:56.993 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:39:56.993 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 06:39:56.993 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:39:56.993 | INFO | High Water Breakdown: | 20260507 06:39:56.994 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:39:56.994 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:39:56.994 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:39:56.994 | INFO | ------------------------------------------ | 20260507 06:39:56.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:39:56.994 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:39:56.994 | INFO | Total mem usage: 26.5 GiB | 20260507 06:39:56.994 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:39:56.994 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:39:56.994 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 06:39:56.994 | INFO | High Water Breakdown: | 20260507 06:39:56.994 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:39:56.994 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:39:56.994 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:39:56.994 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:39:56.994 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:39:56.994 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:39:56.994 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:39:56.994 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:39:56.994 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:39:56.994 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:39:56.994 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:39:56.994 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:39:56.994 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:39:56.994 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:39:56.994 | INFO | ------------------------------------------ | 20260507 06:39:56.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:39:56.994 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:39:56.994 | INFO | Total mem usage: 25.6 GiB | 20260507 06:39:56.994 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:39:56.994 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:39:56.994 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:39:56.994 | INFO | High Water Breakdown: | 20260507 06:39:56.994 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:39:56.994 | INFO | ------------------------------------------ | 20260507 06:39:56.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:39:56.994 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:39:56.994 | INFO | Total mem usage: 15.7 GiB | 20260507 06:39:56.994 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:39:56.994 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:39:56.994 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:39:56.994 | INFO | High Water Breakdown: | 20260507 06:39:56.994 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:39:56.994 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:39:56.994 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:39:56.994 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:39:56.994 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:39:56.994 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:39:56.994 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:39:56.994 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:39:56.994 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:39:56.994 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:39:56.994 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:39:56.994 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:39:56.994 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:39:56.994 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:39:56.994 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:39:56.994 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:39:56.994 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:39:56.994 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:39:56.994 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:39:56.994 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:39:56.994 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:39:56.994 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:39:56.994 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:39:56.994 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:39:56.994 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:39:56.994 | INFO | ------------------------------------------ | 20260507 06:39:56.994 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:39:56.994 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:39:56.994 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:39:56.994 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:39:56.994 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 1.37e+00 | 20260507 06:39:56.994 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.01e-01, 9.17e-03 / 3.18e-03 | 20260507 06:39:56.995 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.47e+00, 5.04e-01 / 2.29e-01 | 20260507 06:39:56.995 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.49e+00, 1.68e-01 / 9.84e-02 | 20260507 06:39:56.995 | INFO | PerfReport: 16.20%, DME -- 3.98e+01 / 4.43e+01, 3.89e+00 / 1.52e+00 | 20260507 06:39:56.995 | INFO | PerfReport: 57.78%, FrameLabeling -- 1.77e+01 / 2.01e+01, 1.34e+00 / 1.46e+00 | 20260507 06:39:56.995 | INFO | PerfReport: 6.89%, PulseAccumulating -- 2.12e+00 / 2.66e+00, 3.78e-01 / 2.20e-01 | 20260507 06:39:56.995 | INFO | PerfReport: 2.95%, Metrics -- 9.05e-01 / 1.23e+00, 2.42e-01 / 1.60e-01 | 20260507 06:39:56.995 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.05e-01 / 3.60e-01, 5.03e-02 / 2.08e-02 | 20260507 06:39:56.995 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:39:56.995 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:39:56.995 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:39:56.995 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 06:39:56.995 | INFO | PerfReport: 21.88%, Upload -- 2.10e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260507 06:39:56.995 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 06:39:56.995 | INFO | PerfReport: 31.96%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:39:56.995 | INFO | PerfReport: 1.55%, LockUpload -- 1.49e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 06:39:56.995 | INFO | PerfReport: 1.20%, LockDownload -- 1.15e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260507 06:39:56.995 | INFO | PerfReport: 22.00%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 06:39:56.995 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260507 06:39:57.090 | INFO | Starting a round of Baz IO | 20260507 06:40:00.188 | INFO | Populating RTMetrics report | 20260507 06:40:00.188 | INFO | RTMetrics report populated | 20260507 06:40:00.188 | INFO | Opening tmp RTMetrics json file | 20260507 06:40:00.205 | INFO | Serializing RTMetrics to json | 20260507 06:40:00.206 | INFO | Serialization done, swapping tmp file now... | 20260507 06:40:00.207 | INFO | RTMetrics done | 20260507 06:40:00.274 | INFO | Finished a round of Baz IO: Wrote 1.662 GiB in 3.183 seconds | 20260507 06:40:00.274 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.61% / 0.00% / 67.39% | 20260507 06:40:00.274 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.118192% / 3.97364e-06% / 5.16574e-05% | 20260507 06:40:00.274 | INFO | Baz IO overall: | 20260507 06:40:00.274 | INFO | BytesWritten : 3969277420177 | 20260507 06:40:00.274 | INFO | NumEvents : 1901772727352 | 20260507 06:40:02.135 | INFO | Analyzing chunk frames = [7880704,7881216) | 20260507 06:40:02.135 | INFO | MemInfo: { | 20260507 06:40:02.135 | INFO | "gpu" : " 17 GB", | 20260507 06:40:02.135 | INFO | "huge" : "", | 20260507 06:40:02.135 | INFO | "huge_shared" : " 27 GB", | 20260507 06:40:02.135 | INFO | "malloc" : " 14 GB", | 20260507 06:40:02.135 | INFO | "pinned" : " 29 GB" | 20260507 06:40:02.135 | INFO | } | 20260507 06:40:02.135 | INFO | Perf report for the last 5.14248 seconds: | 20260507 06:40:02.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:40:02.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:02.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:02.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:02.136 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 06:40:02.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:02.136 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 06:40:02.136 | INFO | PRE_HQ : 58.8%, 1.8, 54.8ms, 0 | 20260507 06:40:02.136 | INFO | BAZWRITER : 61.5%, 3.1, 99.1ms, 0 | 20260507 06:40:02.136 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 06:40:02.136 | INFO | Uploaded 36GiB to GPU in 2.03235s (17.7135GiB/s) | 20260507 06:40:02.136 | INFO | Downloaded 38GiB to GPU in 2.17531s (17.4687GiB/s) | 20260507 06:40:07.273 | INFO | Analyzing chunk frames = [7881216,7881728) | 20260507 06:40:07.273 | INFO | MemInfo: { | 20260507 06:40:07.273 | INFO | "gpu" : " 17 GB", | 20260507 06:40:07.273 | INFO | "huge" : "", | 20260507 06:40:07.273 | INFO | "huge_shared" : " 27 GB", | 20260507 06:40:07.273 | INFO | "malloc" : " 14 GB", | 20260507 06:40:07.273 | INFO | "pinned" : " 29 GB" | 20260507 06:40:07.273 | INFO | } | 20260507 06:40:07.273 | INFO | Perf report for the last 5.13806 seconds: | 20260507 06:40:07.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:40:07.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:07.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:07.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:07.274 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:40:07.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:07.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:07.274 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:40:07.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:07.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:07.274 | INFO | Uploaded 36GiB to GPU in 1.96975s (18.2764GiB/s) | 20260507 06:40:07.274 | INFO | Downloaded 34GiB to GPU in 1.92955s (17.6206GiB/s) | 20260507 06:40:12.411 | INFO | Analyzing chunk frames = [7881728,7882240) | 20260507 06:40:12.411 | INFO | MemInfo: { | 20260507 06:40:12.411 | INFO | "gpu" : " 17 GB", | 20260507 06:40:12.411 | INFO | "huge" : "", | 20260507 06:40:12.411 | INFO | "huge_shared" : " 27 GB", | 20260507 06:40:12.411 | INFO | "malloc" : " 14 GB", | 20260507 06:40:12.412 | INFO | "pinned" : " 29 GB" | 20260507 06:40:12.412 | INFO | } | 20260507 06:40:12.412 | INFO | Perf report for the last 5.13823 seconds: | 20260507 06:40:12.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:40:12.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:12.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:12.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:12.412 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:40:12.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:12.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:12.412 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:40:12.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:12.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:12.412 | INFO | Uploaded 36GiB to GPU in 2.05006s (17.5604GiB/s) | 20260507 06:40:12.412 | INFO | Downloaded 34GiB to GPU in 1.95542s (17.3876GiB/s) | 20260507 06:40:17.549 | INFO | Analyzing chunk frames = [7882240,7882752) | 20260507 06:40:17.549 | INFO | MemInfo: { | 20260507 06:40:17.549 | INFO | "gpu" : " 17 GB", | 20260507 06:40:17.549 | INFO | "huge" : "", | 20260507 06:40:17.549 | INFO | "huge_shared" : " 27 GB", | 20260507 06:40:17.549 | INFO | "malloc" : " 14 GB", | 20260507 06:40:17.549 | INFO | "pinned" : " 29 GB" | 20260507 06:40:17.549 | INFO | } | 20260507 06:40:17.549 | INFO | Perf report for the last 5.13768 seconds: | 20260507 06:40:17.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:40:17.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:17.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:17.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:17.549 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 06:40:17.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:17.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:17.549 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 06:40:17.550 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:17.550 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:17.550 | INFO | Uploaded 36GiB to GPU in 2.01892s (17.8313GiB/s) | 20260507 06:40:17.550 | INFO | Downloaded 34GiB to GPU in 1.956s (17.3824GiB/s) | 20260507 06:40:22.696 | INFO | Analyzing chunk frames = [7882752,7883264) | 20260507 06:40:22.696 | INFO | MemInfo: { | 20260507 06:40:22.696 | INFO | "gpu" : " 17 GB", | 20260507 06:40:22.696 | INFO | "huge" : "", | 20260507 06:40:22.696 | INFO | "huge_shared" : " 27 GB", | 20260507 06:40:22.696 | INFO | "malloc" : " 14 GB", | 20260507 06:40:22.696 | INFO | "pinned" : " 29 GB" | 20260507 06:40:22.696 | INFO | } | 20260507 06:40:22.696 | INFO | Perf report for the last 5.14638 seconds: | 20260507 06:40:22.696 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:40:22.696 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:22.696 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:22.696 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:22.696 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 06:40:22.696 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:22.696 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:22.696 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:40:22.696 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:22.696 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:22.696 | INFO | Uploaded 36GiB to GPU in 2.0499s (17.5618GiB/s) | 20260507 06:40:22.696 | INFO | Downloaded 34GiB to GPU in 1.95229s (17.4154GiB/s) | 20260507 06:40:27.832 | INFO | Analyzing chunk frames = [7883264,7883776) | 20260507 06:40:27.832 | INFO | MemInfo: { | 20260507 06:40:27.832 | INFO | "gpu" : " 17 GB", | 20260507 06:40:27.832 | INFO | "huge" : "", | 20260507 06:40:27.832 | INFO | "huge_shared" : " 27 GB", | 20260507 06:40:27.832 | INFO | "malloc" : " 14 GB", | 20260507 06:40:27.832 | INFO | "pinned" : " 29 GB" | 20260507 06:40:27.832 | INFO | } | 20260507 06:40:27.832 | INFO | Perf report for the last 5.13648 seconds: | 20260507 06:40:27.832 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:40:27.832 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:27.832 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:27.832 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:27.832 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 06:40:27.832 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:27.832 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:27.833 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 06:40:27.833 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:27.833 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:27.833 | INFO | Uploaded 36GiB to GPU in 2.05798s (17.4929GiB/s) | 20260507 06:40:27.833 | INFO | Downloaded 34GiB to GPU in 1.92713s (17.6429GiB/s) | 20260507 06:40:32.963 | INFO | Analyzing chunk frames = [7883776,7884288) | 20260507 06:40:32.963 | INFO | MemInfo: { | 20260507 06:40:32.963 | INFO | "gpu" : " 17 GB", | 20260507 06:40:32.963 | INFO | "huge" : "", | 20260507 06:40:32.963 | INFO | "huge_shared" : " 27 GB", | 20260507 06:40:32.963 | INFO | "malloc" : " 14 GB", | 20260507 06:40:32.963 | INFO | "pinned" : " 29 GB" | 20260507 06:40:32.963 | INFO | } | 20260507 06:40:32.963 | INFO | Perf report for the last 5.13038 seconds: | 20260507 06:40:32.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:40:32.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:32.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:32.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:32.963 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 06:40:32.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:32.963 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:32.963 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 06:40:32.963 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:32.963 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:32.963 | INFO | Uploaded 36GiB to GPU in 1.91219s (18.8266GiB/s) | 20260507 06:40:32.963 | INFO | Downloaded 34GiB to GPU in 1.98235s (17.1513GiB/s) | 20260507 06:40:38.100 | INFO | Analyzing chunk frames = [7884288,7884800) | 20260507 06:40:38.100 | INFO | MemInfo: { | 20260507 06:40:38.100 | INFO | "gpu" : " 17 GB", | 20260507 06:40:38.100 | INFO | "huge" : "", | 20260507 06:40:38.100 | INFO | "huge_shared" : " 27 GB", | 20260507 06:40:38.100 | INFO | "malloc" : " 14 GB", | 20260507 06:40:38.100 | INFO | "pinned" : " 29 GB" | 20260507 06:40:38.100 | INFO | } | 20260507 06:40:38.100 | INFO | Perf report for the last 5.13777 seconds: | 20260507 06:40:38.100 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:40:38.100 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:38.100 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:38.100 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:38.100 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:40:38.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:38.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:38.101 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:40:38.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:38.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:38.101 | INFO | Uploaded 36GiB to GPU in 1.94593s (18.5001GiB/s) | 20260507 06:40:38.101 | INFO | Downloaded 34GiB to GPU in 1.95978s (17.3489GiB/s) | 20260507 06:40:38.197 | INFO | Starting a round of Baz IO | 20260507 06:40:41.300 | INFO | Populating RTMetrics report | 20260507 06:40:41.300 | INFO | RTMetrics report populated | 20260507 06:40:41.300 | INFO | Opening tmp RTMetrics json file | 20260507 06:40:41.307 | INFO | Serializing RTMetrics to json | 20260507 06:40:41.317 | INFO | Serialization done, swapping tmp file now... | 20260507 06:40:41.318 | INFO | RTMetrics done | 20260507 06:40:41.384 | INFO | Finished a round of Baz IO: Wrote 1.662 GiB in 3.187 seconds | 20260507 06:40:41.384 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.61% / 0.00% / 67.39% | 20260507 06:40:41.384 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.118983% / 0% / 6.35783e-05% | 20260507 06:40:41.384 | INFO | Baz IO overall: | 20260507 06:40:41.384 | INFO | BytesWritten : 3971062343800 | 20260507 06:40:41.384 | INFO | NumEvents : 1902608263442 | 20260507 06:40:43.250 | INFO | Analyzing chunk frames = [7884800,7885312) | 20260507 06:40:43.250 | INFO | MemInfo: { | 20260507 06:40:43.250 | INFO | "gpu" : " 17 GB", | 20260507 06:40:43.250 | INFO | "huge" : "", | 20260507 06:40:43.250 | INFO | "huge_shared" : " 27 GB", | 20260507 06:40:43.250 | INFO | "malloc" : " 14 GB", | 20260507 06:40:43.250 | INFO | "pinned" : " 29 GB" | 20260507 06:40:43.250 | INFO | } | 20260507 06:40:43.250 | INFO | Perf report for the last 5.14966 seconds: | 20260507 06:40:43.250 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:40:43.250 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:43.250 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:43.250 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:43.250 | INFO | ANALYSIS : 61.6%, 3.0, 96.1ms, 0 | 20260507 06:40:43.250 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:43.250 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 06:40:43.250 | INFO | PRE_HQ : 58.9%, 1.8, 54.5ms, 0 | 20260507 06:40:43.250 | INFO | BAZWRITER : 61.5%, 3.1, 97.2ms, 0 | 20260507 06:40:43.250 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:40:43.250 | INFO | Uploaded 36GiB to GPU in 2.04357s (17.6162GiB/s) | 20260507 06:40:43.250 | INFO | Downloaded 38GiB to GPU in 2.17898s (17.4393GiB/s) | 20260507 06:40:48.388 | INFO | Analyzing chunk frames = [7885312,7885824) | 20260507 06:40:48.388 | INFO | MemInfo: { | 20260507 06:40:48.388 | INFO | "gpu" : " 17 GB", | 20260507 06:40:48.388 | INFO | "huge" : "", | 20260507 06:40:48.388 | INFO | "huge_shared" : " 27 GB", | 20260507 06:40:48.388 | INFO | "malloc" : " 14 GB", | 20260507 06:40:48.388 | INFO | "pinned" : " 29 GB" | 20260507 06:40:48.388 | INFO | } | 20260507 06:40:48.388 | INFO | Perf report for the last 5.13819 seconds: | 20260507 06:40:48.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:40:48.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:48.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:48.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:48.388 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:40:48.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:48.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:48.388 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:40:48.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:48.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:48.389 | INFO | Uploaded 36GiB to GPU in 1.99076s (18.0835GiB/s) | 20260507 06:40:48.389 | INFO | Downloaded 34GiB to GPU in 1.93132s (17.6045GiB/s) | 20260507 06:40:53.530 | INFO | Analyzing chunk frames = [7885824,7886336) | 20260507 06:40:53.530 | INFO | MemInfo: { | 20260507 06:40:53.530 | INFO | "gpu" : " 17 GB", | 20260507 06:40:53.530 | INFO | "huge" : "", | 20260507 06:40:53.530 | INFO | "huge_shared" : " 27 GB", | 20260507 06:40:53.530 | INFO | "malloc" : " 14 GB", | 20260507 06:40:53.530 | INFO | "pinned" : " 29 GB" | 20260507 06:40:53.530 | INFO | } | 20260507 06:40:53.530 | INFO | Perf report for the last 5.14156 seconds: | 20260507 06:40:53.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:40:53.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:53.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:53.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:53.530 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 06:40:53.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:53.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:53.530 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:40:53.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:53.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:53.531 | INFO | Uploaded 36GiB to GPU in 2.04404s (17.6122GiB/s) | 20260507 06:40:53.531 | INFO | Downloaded 34GiB to GPU in 1.95193s (17.4186GiB/s) | 20260507 06:40:58.663 | INFO | Analyzing chunk frames = [7886336,7886848) | 20260507 06:40:58.663 | INFO | MemInfo: { | 20260507 06:40:58.663 | INFO | "gpu" : " 17 GB", | 20260507 06:40:58.663 | INFO | "huge" : "", | 20260507 06:40:58.664 | INFO | "huge_shared" : " 27 GB", | 20260507 06:40:58.664 | INFO | "malloc" : " 14 GB", | 20260507 06:40:58.664 | INFO | "pinned" : " 29 GB" | 20260507 06:40:58.664 | INFO | } | 20260507 06:40:58.664 | INFO | Perf report for the last 5.13381 seconds: | 20260507 06:40:58.664 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:40:58.664 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:58.664 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:58.664 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:58.664 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 06:40:58.664 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:58.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:58.664 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:40:58.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:58.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:40:58.664 | INFO | Uploaded 36GiB to GPU in 2.05206s (17.5434GiB/s) | 20260507 06:40:58.664 | INFO | Downloaded 34GiB to GPU in 1.95773s (17.3671GiB/s) | 20260507 06:41:03.806 | INFO | Analyzing chunk frames = [7886848,7887360) | 20260507 06:41:03.806 | INFO | MemInfo: { | 20260507 06:41:03.806 | INFO | "gpu" : " 17 GB", | 20260507 06:41:03.806 | INFO | "huge" : "", | 20260507 06:41:03.806 | INFO | "huge_shared" : " 27 GB", | 20260507 06:41:03.806 | INFO | "malloc" : " 14 GB", | 20260507 06:41:03.806 | INFO | "pinned" : " 29 GB" | 20260507 06:41:03.806 | INFO | } | 20260507 06:41:03.806 | INFO | Perf report for the last 5.14259 seconds: | 20260507 06:41:03.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:41:03.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:03.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:03.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:03.806 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 06:41:03.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:03.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:03.806 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:41:03.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:03.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:03.807 | INFO | Uploaded 36GiB to GPU in 2.04989s (17.562GiB/s) | 20260507 06:41:03.807 | INFO | Downloaded 34GiB to GPU in 1.95648s (17.3781GiB/s) | 20260507 06:41:08.943 | INFO | Analyzing chunk frames = [7887360,7887872) | 20260507 06:41:08.943 | INFO | MemInfo: { | 20260507 06:41:08.943 | INFO | "gpu" : " 17 GB", | 20260507 06:41:08.943 | INFO | "huge" : "", | 20260507 06:41:08.943 | INFO | "huge_shared" : " 27 GB", | 20260507 06:41:08.943 | INFO | "malloc" : " 14 GB", | 20260507 06:41:08.943 | INFO | "pinned" : " 29 GB" | 20260507 06:41:08.943 | INFO | } | 20260507 06:41:08.943 | INFO | Perf report for the last 5.1371 seconds: | 20260507 06:41:08.943 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:41:08.943 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:08.943 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:08.943 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:08.943 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 06:41:08.943 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:08.943 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:08.944 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 06:41:08.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:08.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:08.944 | INFO | Uploaded 36GiB to GPU in 2.02933s (17.7398GiB/s) | 20260507 06:41:08.944 | INFO | Downloaded 34GiB to GPU in 1.91192s (17.7832GiB/s) | 20260507 06:41:14.089 | INFO | Analyzing chunk frames = [7887872,7888384) | 20260507 06:41:14.089 | INFO | MemInfo: { | 20260507 06:41:14.089 | INFO | "gpu" : " 17 GB", | 20260507 06:41:14.089 | INFO | "huge" : "", | 20260507 06:41:14.089 | INFO | "huge_shared" : " 27 GB", | 20260507 06:41:14.089 | INFO | "malloc" : " 14 GB", | 20260507 06:41:14.089 | INFO | "pinned" : " 29 GB" | 20260507 06:41:14.089 | INFO | } | 20260507 06:41:14.089 | INFO | Perf report for the last 5.14562 seconds: | 20260507 06:41:14.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:41:14.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:14.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:14.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:14.089 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:41:14.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:14.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:14.089 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 06:41:14.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:14.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:14.089 | INFO | Uploaded 36GiB to GPU in 1.96484s (18.3221GiB/s) | 20260507 06:41:14.090 | INFO | Downloaded 34GiB to GPU in 1.97508s (17.2145GiB/s) | 20260507 06:41:19.226 | INFO | Analyzing chunk frames = [7888384,7888896) | 20260507 06:41:19.226 | INFO | MemInfo: { | 20260507 06:41:19.226 | INFO | "gpu" : " 17 GB", | 20260507 06:41:19.226 | INFO | "huge" : "", | 20260507 06:41:19.226 | INFO | "huge_shared" : " 27 GB", | 20260507 06:41:19.226 | INFO | "malloc" : " 14 GB", | 20260507 06:41:19.226 | INFO | "pinned" : " 29 GB" | 20260507 06:41:19.226 | INFO | } | 20260507 06:41:19.226 | INFO | Perf report for the last 5.13667 seconds: | 20260507 06:41:19.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:41:19.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:19.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:19.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:19.226 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 06:41:19.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:19.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:19.226 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 06:41:19.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:19.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:19.226 | INFO | Uploaded 36GiB to GPU in 2.00454s (17.9592GiB/s) | 20260507 06:41:19.226 | INFO | Downloaded 34GiB to GPU in 1.96542s (17.2991GiB/s) | 20260507 06:41:19.226 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:41:19.226 | INFO | Total mem usage: 18 GiB | 20260507 06:41:19.226 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:41:19.226 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -27%) | 20260507 06:41:19.226 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:41:19.226 | INFO | High Water Breakdown: | 20260507 06:41:19.226 | INFO | 60.1% : 8.53 GiB ( 74.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:41:19.226 | INFO | 39.7% : 5.63 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:41:19.226 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:41:19.226 | INFO | ------------------------------------------ | 20260507 06:41:19.226 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:41:19.227 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:41:19.227 | INFO | Total mem usage: 26.5 GiB | 20260507 06:41:19.227 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:41:19.227 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:41:19.227 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:41:19.227 | INFO | High Water Breakdown: | 20260507 06:41:19.227 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:41:19.227 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:41:19.227 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:41:19.227 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:41:19.227 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:41:19.227 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:41:19.227 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:41:19.227 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:41:19.227 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:41:19.227 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:41:19.227 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:41:19.227 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:41:19.227 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:41:19.227 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:41:19.227 | INFO | ------------------------------------------ | 20260507 06:41:19.227 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:41:19.227 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:41:19.227 | INFO | Total mem usage: 25.6 GiB | 20260507 06:41:19.227 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:41:19.227 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:41:19.227 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:41:19.227 | INFO | High Water Breakdown: | 20260507 06:41:19.227 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:41:19.227 | INFO | ------------------------------------------ | 20260507 06:41:19.227 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:41:19.227 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:41:19.227 | INFO | Total mem usage: 15.7 GiB | 20260507 06:41:19.227 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:41:19.227 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:41:19.227 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:41:19.227 | INFO | High Water Breakdown: | 20260507 06:41:19.227 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:41:19.227 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:41:19.227 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:41:19.227 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:41:19.227 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:41:19.227 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:41:19.227 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:41:19.227 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:41:19.227 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:41:19.227 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:41:19.227 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:41:19.227 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:41:19.227 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:41:19.227 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:41:19.227 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:41:19.227 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:41:19.227 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:41:19.227 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:41:19.227 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:41:19.227 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:41:19.227 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:41:19.227 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:41:19.227 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:41:19.227 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:41:19.227 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:41:19.227 | INFO | ------------------------------------------ | 20260507 06:41:19.227 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:41:19.228 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:41:19.228 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:41:19.228 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:41:19.228 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.47e+01, 1.34e+01 / 1.34e+00 | 20260507 06:41:19.228 | INFO | PerfReport: 0.30%, AggregateStats -- 9.06e-02 / 1.01e-01, 9.93e-03 / 3.86e-03 | 20260507 06:41:19.228 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.47e+00, 4.96e-01 / 2.19e-01 | 20260507 06:41:19.228 | INFO | PerfReport: 3.20%, Histogram -- 9.81e-01 / 1.49e+00, 1.61e-01 / 9.23e-02 | 20260507 06:41:19.228 | INFO | PerfReport: 16.37%, DME -- 4.01e+01 / 4.52e+01, 4.03e+00 / 1.69e+00 | 20260507 06:41:19.228 | INFO | PerfReport: 57.53%, FrameLabeling -- 1.76e+01 / 1.91e+01, 1.12e+00 / 1.08e+00 | 20260507 06:41:19.228 | INFO | PerfReport: 6.96%, PulseAccumulating -- 2.13e+00 / 2.71e+00, 3.95e-01 / 2.01e-01 | 20260507 06:41:19.228 | INFO | PerfReport: 2.96%, Metrics -- 9.06e-01 / 1.21e+00, 2.40e-01 / 1.06e-01 | 20260507 06:41:19.228 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.05e-01 / 3.64e-01, 5.04e-02 / 2.15e-02 | 20260507 06:41:19.228 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:41:19.228 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:41:19.228 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:41:19.228 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 06:41:19.228 | INFO | PerfReport: 21.80%, Upload -- 2.10e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260507 06:41:19.228 | INFO | PerfReport: 21.43%, Download -- 2.06e+01 / 0.00e+00, 7.40e+00 / 0.00e+00 | 20260507 06:41:19.228 | INFO | PerfReport: 31.84%, Compute -- 3.06e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:41:19.228 | INFO | PerfReport: 1.54%, LockUpload -- 1.48e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 06:41:19.228 | INFO | PerfReport: 1.40%, LockDownload -- 1.34e+00 / 0.00e+00, 3.19e+00 / 0.00e+00 | 20260507 06:41:19.228 | INFO | PerfReport: 21.99%, LockCompute -- 2.12e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 06:41:19.228 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260507 06:41:19.322 | INFO | Starting a round of Baz IO | 20260507 06:41:22.418 | INFO | Populating RTMetrics report | 20260507 06:41:22.418 | INFO | RTMetrics report populated | 20260507 06:41:22.418 | INFO | Opening tmp RTMetrics json file | 20260507 06:41:22.436 | INFO | Serializing RTMetrics to json | 20260507 06:41:22.437 | INFO | Serialization done, swapping tmp file now... | 20260507 06:41:22.437 | INFO | RTMetrics done | 20260507 06:41:22.507 | INFO | Finished a round of Baz IO: Wrote 1.662 GiB in 3.184 seconds | 20260507 06:41:22.507 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.61% / 0.00% / 67.39% | 20260507 06:41:22.507 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.118383% / 3.97364e-06% / 4.37101e-05% | 20260507 06:41:22.507 | INFO | Baz IO overall: | 20260507 06:41:22.507 | INFO | BytesWritten : 3972847396449 | 20260507 06:41:22.507 | INFO | NumEvents : 1903444113400 | 20260507 06:41:24.360 | INFO | Analyzing chunk frames = [7888896,7889408) | 20260507 06:41:24.360 | INFO | MemInfo: { | 20260507 06:41:24.360 | INFO | "gpu" : " 17 GB", | 20260507 06:41:24.360 | INFO | "huge" : "", | 20260507 06:41:24.360 | INFO | "huge_shared" : " 27 GB", | 20260507 06:41:24.360 | INFO | "malloc" : " 14 GB", | 20260507 06:41:24.360 | INFO | "pinned" : " 29 GB" | 20260507 06:41:24.360 | INFO | } | 20260507 06:41:24.360 | INFO | Perf report for the last 5.13403 seconds: | 20260507 06:41:24.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:41:24.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:24.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:24.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:24.360 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 06:41:24.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:24.360 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 06:41:24.360 | INFO | PRE_HQ : 59.0%, 1.8, 55.1ms, 0 | 20260507 06:41:24.360 | INFO | BAZWRITER : 61.5%, 3.2, 99.5ms, 0 | 20260507 06:41:24.360 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:41:24.360 | INFO | Uploaded 36GiB to GPU in 2.07087s (17.384GiB/s) | 20260507 06:41:24.360 | INFO | Downloaded 38GiB to GPU in 2.17973s (17.4334GiB/s) | 20260507 06:41:29.503 | INFO | Analyzing chunk frames = [7889408,7889920) | 20260507 06:41:29.503 | INFO | MemInfo: { | 20260507 06:41:29.503 | INFO | "gpu" : " 17 GB", | 20260507 06:41:29.503 | INFO | "huge" : "", | 20260507 06:41:29.503 | INFO | "huge_shared" : " 27 GB", | 20260507 06:41:29.503 | INFO | "malloc" : " 14 GB", | 20260507 06:41:29.504 | INFO | "pinned" : " 29 GB" | 20260507 06:41:29.504 | INFO | } | 20260507 06:41:29.504 | INFO | Perf report for the last 5.1438 seconds: | 20260507 06:41:29.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:41:29.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:29.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:29.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:29.504 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:41:29.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:29.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:29.504 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:41:29.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:29.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:29.504 | INFO | Uploaded 36GiB to GPU in 2.00473s (17.9575GiB/s) | 20260507 06:41:29.504 | INFO | Downloaded 34GiB to GPU in 1.93048s (17.6122GiB/s) | 20260507 06:41:34.645 | INFO | Analyzing chunk frames = [7889920,7890432) | 20260507 06:41:34.645 | INFO | MemInfo: { | 20260507 06:41:34.645 | INFO | "gpu" : " 17 GB", | 20260507 06:41:34.645 | INFO | "huge" : "", | 20260507 06:41:34.645 | INFO | "huge_shared" : " 27 GB", | 20260507 06:41:34.645 | INFO | "malloc" : " 14 GB", | 20260507 06:41:34.645 | INFO | "pinned" : " 29 GB" | 20260507 06:41:34.645 | INFO | } | 20260507 06:41:34.645 | INFO | Perf report for the last 5.14185 seconds: | 20260507 06:41:34.645 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:41:34.645 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:34.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:34.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:34.646 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 06:41:34.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:34.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:34.646 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:41:34.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:34.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:34.646 | INFO | Uploaded 36GiB to GPU in 2.04415s (17.6113GiB/s) | 20260507 06:41:34.646 | INFO | Downloaded 34GiB to GPU in 1.94975s (17.4381GiB/s) | 20260507 06:41:39.786 | INFO | Analyzing chunk frames = [7890432,7890944) | 20260507 06:41:39.786 | INFO | MemInfo: { | 20260507 06:41:39.786 | INFO | "gpu" : " 17 GB", | 20260507 06:41:39.786 | INFO | "huge" : "", | 20260507 06:41:39.786 | INFO | "huge_shared" : " 27 GB", | 20260507 06:41:39.786 | INFO | "malloc" : " 14 GB", | 20260507 06:41:39.786 | INFO | "pinned" : " 29 GB" | 20260507 06:41:39.786 | INFO | } | 20260507 06:41:39.787 | INFO | Perf report for the last 5.14111 seconds: | 20260507 06:41:39.787 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:41:39.787 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:39.787 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:39.787 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:39.787 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 06:41:39.787 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:39.787 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:39.787 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:41:39.787 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:39.787 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:39.787 | INFO | Uploaded 36GiB to GPU in 2.05225s (17.5417GiB/s) | 20260507 06:41:39.787 | INFO | Downloaded 34GiB to GPU in 1.94491s (17.4816GiB/s) | 20260507 06:41:44.919 | INFO | Analyzing chunk frames = [7890944,7891456) | 20260507 06:41:44.919 | INFO | MemInfo: { | 20260507 06:41:44.919 | INFO | "gpu" : " 17 GB", | 20260507 06:41:44.919 | INFO | "huge" : "", | 20260507 06:41:44.919 | INFO | "huge_shared" : " 27 GB", | 20260507 06:41:44.919 | INFO | "malloc" : " 14 GB", | 20260507 06:41:44.919 | INFO | "pinned" : " 29 GB" | 20260507 06:41:44.919 | INFO | } | 20260507 06:41:44.919 | INFO | Perf report for the last 5.13223 seconds: | 20260507 06:41:44.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:41:44.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:44.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:44.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:44.919 | INFO | ANALYSIS : 60.2%, 3.0, 93.5ms, 0 | 20260507 06:41:44.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:44.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:44.919 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:41:44.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:44.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:44.919 | INFO | Uploaded 36GiB to GPU in 2.04552s (17.5995GiB/s) | 20260507 06:41:44.919 | INFO | Downloaded 34GiB to GPU in 1.90736s (17.8256GiB/s) | 20260507 06:41:50.056 | INFO | Analyzing chunk frames = [7891456,7891968) | 20260507 06:41:50.056 | INFO | MemInfo: { | 20260507 06:41:50.056 | INFO | "gpu" : " 17 GB", | 20260507 06:41:50.056 | INFO | "huge" : "", | 20260507 06:41:50.056 | INFO | "huge_shared" : " 27 GB", | 20260507 06:41:50.056 | INFO | "malloc" : " 14 GB", | 20260507 06:41:50.056 | INFO | "pinned" : " 29 GB" | 20260507 06:41:50.057 | INFO | } | 20260507 06:41:50.057 | INFO | Perf report for the last 5.13779 seconds: | 20260507 06:41:50.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:41:50.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:50.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:50.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:50.057 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 06:41:50.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:50.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:50.057 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:41:50.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:50.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:50.057 | INFO | Uploaded 36GiB to GPU in 2.04561s (17.5987GiB/s) | 20260507 06:41:50.057 | INFO | Downloaded 34GiB to GPU in 1.90034s (17.8915GiB/s) | 20260507 06:41:55.197 | INFO | Analyzing chunk frames = [7891968,7892480) | 20260507 06:41:55.197 | INFO | MemInfo: { | 20260507 06:41:55.197 | INFO | "gpu" : " 17 GB", | 20260507 06:41:55.197 | INFO | "huge" : "", | 20260507 06:41:55.197 | INFO | "huge_shared" : " 27 GB", | 20260507 06:41:55.197 | INFO | "malloc" : " 14 GB", | 20260507 06:41:55.197 | INFO | "pinned" : " 29 GB" | 20260507 06:41:55.197 | INFO | } | 20260507 06:41:55.197 | INFO | Perf report for the last 5.1402 seconds: | 20260507 06:41:55.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:41:55.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:55.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:55.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:55.197 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:41:55.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:55.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:55.197 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 06:41:55.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:55.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:41:55.197 | INFO | Uploaded 36GiB to GPU in 1.95607s (18.4042GiB/s) | 20260507 06:41:55.197 | INFO | Downloaded 34GiB to GPU in 1.96972s (17.2613GiB/s) | 20260507 06:42:00.342 | INFO | Analyzing chunk frames = [7892480,7892992) | 20260507 06:42:00.342 | INFO | MemInfo: { | 20260507 06:42:00.342 | INFO | "gpu" : " 17 GB", | 20260507 06:42:00.342 | INFO | "huge" : "", | 20260507 06:42:00.342 | INFO | "huge_shared" : " 27 GB", | 20260507 06:42:00.342 | INFO | "malloc" : " 14 GB", | 20260507 06:42:00.342 | INFO | "pinned" : " 29 GB" | 20260507 06:42:00.342 | INFO | } | 20260507 06:42:00.342 | INFO | Perf report for the last 5.14527 seconds: | 20260507 06:42:00.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:42:00.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:00.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:00.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:00.342 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:42:00.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:00.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:00.342 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 06:42:00.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:00.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:00.342 | INFO | Uploaded 36GiB to GPU in 1.9555s (18.4096GiB/s) | 20260507 06:42:00.342 | INFO | Downloaded 34GiB to GPU in 1.95671s (17.3761GiB/s) | 20260507 06:42:00.439 | INFO | Starting a round of Baz IO | 20260507 06:42:03.534 | INFO | Populating RTMetrics report | 20260507 06:42:03.534 | INFO | RTMetrics report populated | 20260507 06:42:03.534 | INFO | Opening tmp RTMetrics json file | 20260507 06:42:03.541 | INFO | Serializing RTMetrics to json | 20260507 06:42:03.550 | INFO | Serialization done, swapping tmp file now... | 20260507 06:42:03.551 | INFO | RTMetrics done | 20260507 06:42:03.618 | INFO | Finished a round of Baz IO: Wrote 1.66 GiB in 3.179 seconds | 20260507 06:42:03.618 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.61% / 0.00% / 67.39% | 20260507 06:42:03.618 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.117771% / 0% / 5.96046e-05% | 20260507 06:42:03.618 | INFO | Baz IO overall: | 20260507 06:42:03.618 | INFO | BytesWritten : 3974630341532 | 20260507 06:42:03.619 | INFO | NumEvents : 1904278332082 | 20260507 06:42:05.473 | INFO | Analyzing chunk frames = [7892992,7893504) | 20260507 06:42:05.473 | INFO | MemInfo: { | 20260507 06:42:05.473 | INFO | "gpu" : " 17 GB", | 20260507 06:42:05.474 | INFO | "huge" : "", | 20260507 06:42:05.474 | INFO | "huge_shared" : " 27 GB", | 20260507 06:42:05.474 | INFO | "malloc" : " 14 GB", | 20260507 06:42:05.474 | INFO | "pinned" : " 29 GB" | 20260507 06:42:05.474 | INFO | } | 20260507 06:42:05.474 | INFO | Perf report for the last 5.13163 seconds: | 20260507 06:42:05.474 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:42:05.474 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:05.474 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:05.474 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:05.474 | INFO | ANALYSIS : 61.7%, 3.0, 95.5ms, 0 | 20260507 06:42:05.474 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:05.474 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260507 06:42:05.474 | INFO | PRE_HQ : 59.4%, 1.9, 57.9ms, 0 | 20260507 06:42:05.474 | INFO | BAZWRITER : 61.5%, 3.2, 100.5ms, 0 | 20260507 06:42:05.474 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:42:05.474 | INFO | Uploaded 36GiB to GPU in 2.06641s (17.4215GiB/s) | 20260507 06:42:05.474 | INFO | Downloaded 38GiB to GPU in 2.18522s (17.3896GiB/s) | 20260507 06:42:10.610 | INFO | Analyzing chunk frames = [7893504,7894016) | 20260507 06:42:10.610 | INFO | MemInfo: { | 20260507 06:42:10.610 | INFO | "gpu" : " 17 GB", | 20260507 06:42:10.610 | INFO | "huge" : "", | 20260507 06:42:10.610 | INFO | "huge_shared" : " 27 GB", | 20260507 06:42:10.610 | INFO | "malloc" : " 14 GB", | 20260507 06:42:10.610 | INFO | "pinned" : " 29 GB" | 20260507 06:42:10.610 | INFO | } | 20260507 06:42:10.610 | INFO | Perf report for the last 5.13671 seconds: | 20260507 06:42:10.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:42:10.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:10.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:10.611 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:10.611 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 06:42:10.611 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:10.611 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:10.611 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 06:42:10.611 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:10.611 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:10.611 | INFO | Uploaded 36GiB to GPU in 1.98537s (18.1327GiB/s) | 20260507 06:42:10.611 | INFO | Downloaded 34GiB to GPU in 1.93596s (17.5624GiB/s) | 20260507 06:42:15.754 | INFO | Analyzing chunk frames = [7894016,7894528) | 20260507 06:42:15.754 | INFO | MemInfo: { | 20260507 06:42:15.754 | INFO | "gpu" : " 17 GB", | 20260507 06:42:15.754 | INFO | "huge" : "", | 20260507 06:42:15.755 | INFO | "huge_shared" : " 27 GB", | 20260507 06:42:15.755 | INFO | "malloc" : " 14 GB", | 20260507 06:42:15.755 | INFO | "pinned" : " 29 GB" | 20260507 06:42:15.755 | INFO | } | 20260507 06:42:15.755 | INFO | Perf report for the last 5.14425 seconds: | 20260507 06:42:15.755 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:42:15.755 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:15.755 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:15.755 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:15.755 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 06:42:15.755 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:15.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:15.755 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:42:15.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:15.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:15.755 | INFO | Uploaded 36GiB to GPU in 2.02664s (17.7634GiB/s) | 20260507 06:42:15.755 | INFO | Downloaded 34GiB to GPU in 1.95606s (17.3818GiB/s) | 20260507 06:42:20.901 | INFO | Analyzing chunk frames = [7894528,7895040) | 20260507 06:42:20.901 | INFO | MemInfo: { | 20260507 06:42:20.901 | INFO | "gpu" : " 17 GB", | 20260507 06:42:20.901 | INFO | "huge" : "", | 20260507 06:42:20.901 | INFO | "huge_shared" : " 27 GB", | 20260507 06:42:20.901 | INFO | "malloc" : " 14 GB", | 20260507 06:42:20.901 | INFO | "pinned" : " 29 GB" | 20260507 06:42:20.901 | INFO | } | 20260507 06:42:20.901 | INFO | Perf report for the last 5.1462 seconds: | 20260507 06:42:20.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:42:20.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:20.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:20.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:20.901 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 06:42:20.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:20.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:20.901 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:42:20.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:20.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:20.901 | INFO | Uploaded 36GiB to GPU in 2.0084s (17.9247GiB/s) | 20260507 06:42:20.901 | INFO | Downloaded 34GiB to GPU in 1.93448s (17.5758GiB/s) | 20260507 06:42:26.040 | INFO | Analyzing chunk frames = [7895040,7895552) | 20260507 06:42:26.040 | INFO | MemInfo: { | 20260507 06:42:26.040 | INFO | "gpu" : " 17 GB", | 20260507 06:42:26.040 | INFO | "huge" : "", | 20260507 06:42:26.040 | INFO | "huge_shared" : " 27 GB", | 20260507 06:42:26.040 | INFO | "malloc" : " 14 GB", | 20260507 06:42:26.040 | INFO | "pinned" : " 29 GB" | 20260507 06:42:26.040 | INFO | } | 20260507 06:42:26.040 | INFO | Perf report for the last 5.13933 seconds: | 20260507 06:42:26.040 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:42:26.040 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:26.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:26.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:26.040 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 06:42:26.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:26.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:26.040 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:42:26.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:26.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:26.041 | INFO | Uploaded 36GiB to GPU in 2.0506s (17.5558GiB/s) | 20260507 06:42:26.041 | INFO | Downloaded 34GiB to GPU in 1.90425s (17.8548GiB/s) | 20260507 06:42:31.169 | INFO | Analyzing chunk frames = [7895552,7896064) | 20260507 06:42:31.169 | INFO | MemInfo: { | 20260507 06:42:31.169 | INFO | "gpu" : " 17 GB", | 20260507 06:42:31.169 | INFO | "huge" : "", | 20260507 06:42:31.170 | INFO | "huge_shared" : " 27 GB", | 20260507 06:42:31.170 | INFO | "malloc" : " 14 GB", | 20260507 06:42:31.170 | INFO | "pinned" : " 29 GB" | 20260507 06:42:31.170 | INFO | } | 20260507 06:42:31.170 | INFO | Perf report for the last 5.12941 seconds: | 20260507 06:42:31.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:42:31.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:31.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:31.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:31.170 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:42:31.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:31.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:31.170 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:42:31.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:31.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:31.170 | INFO | Uploaded 36GiB to GPU in 2.03518s (17.6889GiB/s) | 20260507 06:42:31.170 | INFO | Downloaded 34GiB to GPU in 1.93019s (17.6148GiB/s) | 20260507 06:42:36.311 | INFO | Analyzing chunk frames = [7896064,7896576) | 20260507 06:42:36.312 | INFO | MemInfo: { | 20260507 06:42:36.312 | INFO | "gpu" : " 17 GB", | 20260507 06:42:36.312 | INFO | "huge" : "", | 20260507 06:42:36.312 | INFO | "huge_shared" : " 27 GB", | 20260507 06:42:36.312 | INFO | "malloc" : " 14 GB", | 20260507 06:42:36.312 | INFO | "pinned" : " 29 GB" | 20260507 06:42:36.312 | INFO | } | 20260507 06:42:36.312 | INFO | Perf report for the last 5.14207 seconds: | 20260507 06:42:36.312 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:42:36.312 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:36.312 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:36.312 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:36.312 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 06:42:36.312 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:36.312 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:36.312 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:42:36.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:36.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:36.312 | INFO | Uploaded 36GiB to GPU in 2.037s (17.673GiB/s) | 20260507 06:42:36.312 | INFO | Downloaded 34GiB to GPU in 1.98594s (17.1203GiB/s) | 20260507 06:42:41.452 | INFO | Analyzing chunk frames = [7896576,7897088) | 20260507 06:42:41.452 | INFO | MemInfo: { | 20260507 06:42:41.452 | INFO | "gpu" : " 17 GB", | 20260507 06:42:41.452 | INFO | "huge" : "", | 20260507 06:42:41.452 | INFO | "huge_shared" : " 27 GB", | 20260507 06:42:41.452 | INFO | "malloc" : " 14 GB", | 20260507 06:42:41.452 | INFO | "pinned" : " 29 GB" | 20260507 06:42:41.452 | INFO | } | 20260507 06:42:41.452 | INFO | Perf report for the last 5.14047 seconds: | 20260507 06:42:41.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:42:41.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:41.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:41.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:41.452 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:42:41.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:41.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:41.452 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 06:42:41.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:41.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:41.453 | INFO | Uploaded 36GiB to GPU in 1.96122s (18.3559GiB/s) | 20260507 06:42:41.453 | INFO | Downloaded 34GiB to GPU in 1.95694s (17.3741GiB/s) | 20260507 06:42:41.453 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:42:41.453 | INFO | Total mem usage: 18 GiB | 20260507 06:42:41.453 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:42:41.453 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:42:41.453 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:42:41.453 | INFO | High Water Breakdown: | 20260507 06:42:41.453 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:42:41.453 | INFO | 39.8% : 5.66 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:42:41.453 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:42:41.453 | INFO | ------------------------------------------ | 20260507 06:42:41.453 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:42:41.453 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:42:41.453 | INFO | Total mem usage: 26.5 GiB | 20260507 06:42:41.453 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:42:41.453 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:42:41.453 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:42:41.453 | INFO | High Water Breakdown: | 20260507 06:42:41.453 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:42:41.453 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:42:41.453 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:42:41.453 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:42:41.453 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:42:41.453 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:42:41.453 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:42:41.453 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:42:41.453 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:42:41.453 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:42:41.453 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:42:41.453 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:42:41.453 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:42:41.453 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:42:41.453 | INFO | ------------------------------------------ | 20260507 06:42:41.453 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:42:41.453 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:42:41.453 | INFO | Total mem usage: 25.6 GiB | 20260507 06:42:41.453 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:42:41.453 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:42:41.453 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:42:41.453 | INFO | High Water Breakdown: | 20260507 06:42:41.453 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:42:41.453 | INFO | ------------------------------------------ | 20260507 06:42:41.453 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:42:41.453 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:42:41.453 | INFO | Total mem usage: 15.7 GiB | 20260507 06:42:41.453 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:42:41.453 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:42:41.453 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:42:41.453 | INFO | High Water Breakdown: | 20260507 06:42:41.453 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:42:41.453 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:42:41.453 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:42:41.453 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:42:41.453 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:42:41.453 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:42:41.453 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:42:41.453 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:42:41.453 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:42:41.453 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:42:41.453 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:42:41.453 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:42:41.453 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:42:41.453 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:42:41.453 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:42:41.453 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:42:41.453 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:42:41.453 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:42:41.453 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:42:41.453 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:42:41.453 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:42:41.453 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:42:41.453 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:42:41.454 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:42:41.454 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:42:41.454 | INFO | ------------------------------------------ | 20260507 06:42:41.454 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:42:41.454 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:42:41.454 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:42:41.454 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:42:41.454 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.33e+01 / 1.30e+00 | 20260507 06:42:41.454 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.61e-03 / 4.59e-03 | 20260507 06:42:41.454 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.45e+00, 5.10e-01 / 2.91e-01 | 20260507 06:42:41.454 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.46e+00, 1.66e-01 / 1.37e-01 | 20260507 06:42:41.454 | INFO | PerfReport: 16.23%, DME -- 3.99e+01 / 4.45e+01, 3.89e+00 / 1.62e+00 | 20260507 06:42:41.454 | INFO | PerfReport: 57.77%, FrameLabeling -- 1.78e+01 / 1.98e+01, 1.36e+00 / 1.50e+00 | 20260507 06:42:41.454 | INFO | PerfReport: 6.87%, PulseAccumulating -- 2.11e+00 / 2.66e+00, 3.77e-01 / 2.48e-01 | 20260507 06:42:41.454 | INFO | PerfReport: 2.94%, Metrics -- 9.05e-01 / 1.23e+00, 2.43e-01 / 1.14e-01 | 20260507 06:42:41.454 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.64e-01, 5.06e-02 / 2.32e-02 | 20260507 06:42:41.454 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:42:41.454 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:42:41.454 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:42:41.454 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 06:42:41.454 | INFO | PerfReport: 21.90%, Upload -- 2.11e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260507 06:42:41.454 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 06:42:41.454 | INFO | PerfReport: 31.97%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 06:42:41.454 | INFO | PerfReport: 1.57%, LockUpload -- 1.51e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260507 06:42:41.454 | INFO | PerfReport: 1.25%, LockDownload -- 1.20e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260507 06:42:41.454 | INFO | PerfReport: 21.95%, LockCompute -- 2.11e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 06:42:41.454 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260507 06:42:41.549 | INFO | Starting a round of Baz IO | 20260507 06:42:44.648 | INFO | Populating RTMetrics report | 20260507 06:42:44.648 | INFO | RTMetrics report populated | 20260507 06:42:44.648 | INFO | Opening tmp RTMetrics json file | 20260507 06:42:44.665 | INFO | Serializing RTMetrics to json | 20260507 06:42:44.666 | INFO | Serialization done, swapping tmp file now... | 20260507 06:42:44.667 | INFO | RTMetrics done | 20260507 06:42:44.736 | INFO | Finished a round of Baz IO: Wrote 1.657 GiB in 3.186 seconds | 20260507 06:42:44.736 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.61% / 0.00% / 67.39% | 20260507 06:42:44.736 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.117731% / 3.97364e-06% / 5.96046e-05% | 20260507 06:42:44.736 | INFO | Baz IO overall: | 20260507 06:42:44.736 | INFO | BytesWritten : 3976409939548 | 20260507 06:42:44.736 | INFO | NumEvents : 1905110148785 | 20260507 06:42:46.595 | INFO | Analyzing chunk frames = [7897088,7897600) | 20260507 06:42:46.595 | INFO | MemInfo: { | 20260507 06:42:46.595 | INFO | "gpu" : " 17 GB", | 20260507 06:42:46.595 | INFO | "huge" : "", | 20260507 06:42:46.595 | INFO | "huge_shared" : " 27 GB", | 20260507 06:42:46.595 | INFO | "malloc" : " 14 GB", | 20260507 06:42:46.595 | INFO | "pinned" : " 29 GB" | 20260507 06:42:46.595 | INFO | } | 20260507 06:42:46.595 | INFO | Perf report for the last 5.14234 seconds: | 20260507 06:42:46.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:42:46.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:46.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:46.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:46.595 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 06:42:46.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:46.595 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 06:42:46.595 | INFO | PRE_HQ : 59.5%, 1.8, 57.2ms, 0 | 20260507 06:42:46.595 | INFO | BAZWRITER : 61.6%, 3.2, 101.1ms, 0 | 20260507 06:42:46.595 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:42:46.595 | INFO | Uploaded 36GiB to GPU in 2.04655s (17.5906GiB/s) | 20260507 06:42:46.595 | INFO | Downloaded 38GiB to GPU in 2.17176s (17.4973GiB/s) | 20260507 06:42:51.733 | INFO | Analyzing chunk frames = [7897600,7898112) | 20260507 06:42:51.734 | INFO | MemInfo: { | 20260507 06:42:51.734 | INFO | "gpu" : " 17 GB", | 20260507 06:42:51.734 | INFO | "huge" : "", | 20260507 06:42:51.734 | INFO | "huge_shared" : " 27 GB", | 20260507 06:42:51.734 | INFO | "malloc" : " 14 GB", | 20260507 06:42:51.734 | INFO | "pinned" : " 29 GB" | 20260507 06:42:51.734 | INFO | } | 20260507 06:42:51.734 | INFO | Perf report for the last 5.13913 seconds: | 20260507 06:42:51.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:42:51.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:51.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:51.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:51.734 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:42:51.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:51.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:51.734 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 06:42:51.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:51.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:51.734 | INFO | Uploaded 36GiB to GPU in 1.99495s (18.0456GiB/s) | 20260507 06:42:51.734 | INFO | Downloaded 34GiB to GPU in 1.94099s (17.5168GiB/s) | 20260507 06:42:56.875 | INFO | Analyzing chunk frames = [7898112,7898624) | 20260507 06:42:56.875 | INFO | MemInfo: { | 20260507 06:42:56.875 | INFO | "gpu" : " 17 GB", | 20260507 06:42:56.875 | INFO | "huge" : "", | 20260507 06:42:56.875 | INFO | "huge_shared" : " 27 GB", | 20260507 06:42:56.875 | INFO | "malloc" : " 14 GB", | 20260507 06:42:56.875 | INFO | "pinned" : " 29 GB" | 20260507 06:42:56.875 | INFO | } | 20260507 06:42:56.875 | INFO | Perf report for the last 5.14169 seconds: | 20260507 06:42:56.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:42:56.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:56.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:56.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:56.876 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:42:56.876 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:56.876 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:56.876 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:42:56.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:56.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:42:56.876 | INFO | Uploaded 36GiB to GPU in 2.06812s (17.4071GiB/s) | 20260507 06:42:56.876 | INFO | Downloaded 34GiB to GPU in 1.95475s (17.3935GiB/s) | 20260507 06:43:02.014 | INFO | Analyzing chunk frames = [7898624,7899136) | 20260507 06:43:02.014 | INFO | MemInfo: { | 20260507 06:43:02.014 | INFO | "gpu" : " 17 GB", | 20260507 06:43:02.014 | INFO | "huge" : "", | 20260507 06:43:02.014 | INFO | "huge_shared" : " 27 GB", | 20260507 06:43:02.014 | INFO | "malloc" : " 14 GB", | 20260507 06:43:02.014 | INFO | "pinned" : " 29 GB" | 20260507 06:43:02.014 | INFO | } | 20260507 06:43:02.014 | INFO | Perf report for the last 5.13847 seconds: | 20260507 06:43:02.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:43:02.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:02.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:02.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:02.014 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 06:43:02.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:02.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:02.014 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:43:02.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:02.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:02.014 | INFO | Uploaded 36GiB to GPU in 2.04346s (17.6172GiB/s) | 20260507 06:43:02.014 | INFO | Downloaded 34GiB to GPU in 1.95011s (17.4349GiB/s) | 20260507 06:43:07.146 | INFO | Analyzing chunk frames = [7899136,7899648) | 20260507 06:43:07.146 | INFO | MemInfo: { | 20260507 06:43:07.146 | INFO | "gpu" : " 17 GB", | 20260507 06:43:07.146 | INFO | "huge" : "", | 20260507 06:43:07.146 | INFO | "huge_shared" : " 27 GB", | 20260507 06:43:07.146 | INFO | "malloc" : " 14 GB", | 20260507 06:43:07.146 | INFO | "pinned" : " 29 GB" | 20260507 06:43:07.146 | INFO | } | 20260507 06:43:07.146 | INFO | Perf report for the last 5.13238 seconds: | 20260507 06:43:07.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:43:07.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:07.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:07.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:07.146 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260507 06:43:07.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:07.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:07.146 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:43:07.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:07.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:07.147 | INFO | Uploaded 36GiB to GPU in 2.03765s (17.6674GiB/s) | 20260507 06:43:07.147 | INFO | Downloaded 34GiB to GPU in 1.9186s (17.7212GiB/s) | 20260507 06:43:12.288 | INFO | Analyzing chunk frames = [7899648,7900160) | 20260507 06:43:12.288 | INFO | MemInfo: { | 20260507 06:43:12.288 | INFO | "gpu" : " 17 GB", | 20260507 06:43:12.288 | INFO | "huge" : "", | 20260507 06:43:12.288 | INFO | "huge_shared" : " 27 GB", | 20260507 06:43:12.288 | INFO | "malloc" : " 14 GB", | 20260507 06:43:12.288 | INFO | "pinned" : " 29 GB" | 20260507 06:43:12.288 | INFO | } | 20260507 06:43:12.288 | INFO | Perf report for the last 5.1421 seconds: | 20260507 06:43:12.288 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:43:12.288 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:12.288 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:12.288 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:12.288 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 06:43:12.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:12.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:12.289 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:43:12.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:12.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:12.289 | INFO | Uploaded 36GiB to GPU in 2.04233s (17.6269GiB/s) | 20260507 06:43:12.289 | INFO | Downloaded 34GiB to GPU in 1.8983s (17.9107GiB/s) | 20260507 06:43:17.425 | INFO | Analyzing chunk frames = [7900160,7900672) | 20260507 06:43:17.425 | INFO | MemInfo: { | 20260507 06:43:17.425 | INFO | "gpu" : " 17 GB", | 20260507 06:43:17.425 | INFO | "huge" : "", | 20260507 06:43:17.425 | INFO | "huge_shared" : " 27 GB", | 20260507 06:43:17.425 | INFO | "malloc" : " 14 GB", | 20260507 06:43:17.425 | INFO | "pinned" : " 29 GB" | 20260507 06:43:17.425 | INFO | } | 20260507 06:43:17.425 | INFO | Perf report for the last 5.13661 seconds: | 20260507 06:43:17.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:43:17.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:17.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:17.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:17.425 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:43:17.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:17.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:17.425 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:43:17.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:17.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:17.425 | INFO | Uploaded 36GiB to GPU in 1.93134s (18.6399GiB/s) | 20260507 06:43:17.426 | INFO | Downloaded 34GiB to GPU in 1.97563s (17.2097GiB/s) | 20260507 06:43:22.564 | INFO | Analyzing chunk frames = [7900672,7901184) | 20260507 06:43:22.564 | INFO | MemInfo: { | 20260507 06:43:22.564 | INFO | "gpu" : " 17 GB", | 20260507 06:43:22.564 | INFO | "huge" : "", | 20260507 06:43:22.564 | INFO | "huge_shared" : " 27 GB", | 20260507 06:43:22.564 | INFO | "malloc" : " 14 GB", | 20260507 06:43:22.564 | INFO | "pinned" : " 29 GB" | 20260507 06:43:22.564 | INFO | } | 20260507 06:43:22.564 | INFO | Perf report for the last 5.13908 seconds: | 20260507 06:43:22.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:43:22.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:22.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:22.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:22.564 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 06:43:22.564 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:22.564 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:22.564 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:43:22.564 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:22.564 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:22.564 | INFO | Uploaded 36GiB to GPU in 1.96407s (18.3293GiB/s) | 20260507 06:43:22.565 | INFO | Downloaded 34GiB to GPU in 1.96215s (17.328GiB/s) | 20260507 06:43:22.661 | INFO | Starting a round of Baz IO | 20260507 06:43:25.758 | INFO | Populating RTMetrics report | 20260507 06:43:25.759 | INFO | RTMetrics report populated | 20260507 06:43:25.759 | INFO | Opening tmp RTMetrics json file | 20260507 06:43:25.766 | INFO | Serializing RTMetrics to json | 20260507 06:43:25.775 | INFO | Serialization done, swapping tmp file now... | 20260507 06:43:25.776 | INFO | RTMetrics done | 20260507 06:43:25.842 | INFO | Finished a round of Baz IO: Wrote 1.659 GiB in 3.181 seconds | 20260507 06:43:25.842 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.61% / 0.00% / 67.39% | 20260507 06:43:25.842 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.117807% / 3.97364e-06% / 7.54992e-05% | 20260507 06:43:25.842 | INFO | Baz IO overall: | 20260507 06:43:25.842 | INFO | BytesWritten : 3978191387189 | 20260507 06:43:25.842 | INFO | NumEvents : 1905942991351 | 20260507 06:43:27.708 | INFO | Analyzing chunk frames = [7901184,7901696) | 20260507 06:43:27.708 | INFO | MemInfo: { | 20260507 06:43:27.708 | INFO | "gpu" : " 17 GB", | 20260507 06:43:27.709 | INFO | "huge" : "", | 20260507 06:43:27.709 | INFO | "huge_shared" : " 27 GB", | 20260507 06:43:27.709 | INFO | "malloc" : " 14 GB", | 20260507 06:43:27.709 | INFO | "pinned" : " 29 GB" | 20260507 06:43:27.709 | INFO | } | 20260507 06:43:27.709 | INFO | Perf report for the last 5.14458 seconds: | 20260507 06:43:27.709 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:43:27.709 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:27.709 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:27.709 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:27.709 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 06:43:27.709 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:27.709 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 06:43:27.709 | INFO | PRE_HQ : 59.0%, 1.8, 55.1ms, 0 | 20260507 06:43:27.709 | INFO | BAZWRITER : 61.4%, 3.1, 98.5ms, 0 | 20260507 06:43:27.709 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 06:43:27.709 | INFO | Uploaded 36GiB to GPU in 2.03345s (17.7039GiB/s) | 20260507 06:43:27.709 | INFO | Downloaded 38GiB to GPU in 2.18051s (17.4271GiB/s) | 20260507 06:43:32.850 | INFO | Analyzing chunk frames = [7901696,7902208) | 20260507 06:43:32.850 | INFO | MemInfo: { | 20260507 06:43:32.850 | INFO | "gpu" : " 17 GB", | 20260507 06:43:32.850 | INFO | "huge" : "", | 20260507 06:43:32.850 | INFO | "huge_shared" : " 27 GB", | 20260507 06:43:32.850 | INFO | "malloc" : " 14 GB", | 20260507 06:43:32.850 | INFO | "pinned" : " 29 GB" | 20260507 06:43:32.850 | INFO | } | 20260507 06:43:32.850 | INFO | Perf report for the last 5.14135 seconds: | 20260507 06:43:32.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:43:32.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:32.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:32.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:32.850 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:43:32.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:32.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:32.850 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:43:32.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:32.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:32.850 | INFO | Uploaded 36GiB to GPU in 1.97113s (18.2637GiB/s) | 20260507 06:43:32.851 | INFO | Downloaded 34GiB to GPU in 1.93463s (17.5744GiB/s) | 20260507 06:43:37.982 | INFO | Analyzing chunk frames = [7902208,7902720) | 20260507 06:43:37.982 | INFO | MemInfo: { | 20260507 06:43:37.982 | INFO | "gpu" : " 17 GB", | 20260507 06:43:37.982 | INFO | "huge" : "", | 20260507 06:43:37.982 | INFO | "huge_shared" : " 27 GB", | 20260507 06:43:37.982 | INFO | "malloc" : " 14 GB", | 20260507 06:43:37.982 | INFO | "pinned" : " 29 GB" | 20260507 06:43:37.982 | INFO | } | 20260507 06:43:37.982 | INFO | Perf report for the last 5.13232 seconds: | 20260507 06:43:37.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:43:37.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:37.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:37.983 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:37.983 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:43:37.983 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:37.983 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:37.983 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:43:37.983 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:37.983 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:37.983 | INFO | Uploaded 36GiB to GPU in 2.02252s (17.7995GiB/s) | 20260507 06:43:37.983 | INFO | Downloaded 34GiB to GPU in 1.92264s (17.684GiB/s) | 20260507 06:43:43.121 | INFO | Analyzing chunk frames = [7902720,7903232) | 20260507 06:43:43.121 | INFO | MemInfo: { | 20260507 06:43:43.121 | INFO | "gpu" : " 17 GB", | 20260507 06:43:43.121 | INFO | "huge" : "", | 20260507 06:43:43.121 | INFO | "huge_shared" : " 27 GB", | 20260507 06:43:43.121 | INFO | "malloc" : " 14 GB", | 20260507 06:43:43.121 | INFO | "pinned" : " 29 GB" | 20260507 06:43:43.121 | INFO | } | 20260507 06:43:43.121 | INFO | Perf report for the last 5.13844 seconds: | 20260507 06:43:43.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:43:43.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:43.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:43.121 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:43.121 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 06:43:43.121 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:43.121 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:43.121 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:43:43.121 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:43.121 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:43.121 | INFO | Uploaded 36GiB to GPU in 2.00567s (17.9492GiB/s) | 20260507 06:43:43.121 | INFO | Downloaded 34GiB to GPU in 1.93841s (17.5402GiB/s) | 20260507 06:43:48.265 | INFO | Analyzing chunk frames = [7903232,7903744) | 20260507 06:43:48.265 | INFO | MemInfo: { | 20260507 06:43:48.265 | INFO | "gpu" : " 17 GB", | 20260507 06:43:48.265 | INFO | "huge" : "", | 20260507 06:43:48.265 | INFO | "huge_shared" : " 27 GB", | 20260507 06:43:48.265 | INFO | "malloc" : " 14 GB", | 20260507 06:43:48.265 | INFO | "pinned" : " 29 GB" | 20260507 06:43:48.265 | INFO | } | 20260507 06:43:48.265 | INFO | Perf report for the last 5.1442 seconds: | 20260507 06:43:48.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:43:48.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:48.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:48.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:48.265 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 06:43:48.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:48.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:48.265 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:43:48.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:48.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:48.265 | INFO | Uploaded 36GiB to GPU in 2.04891s (17.5703GiB/s) | 20260507 06:43:48.266 | INFO | Downloaded 34GiB to GPU in 1.95109s (17.4262GiB/s) | 20260507 06:43:53.409 | INFO | Analyzing chunk frames = [7903744,7904256) | 20260507 06:43:53.409 | INFO | MemInfo: { | 20260507 06:43:53.409 | INFO | "gpu" : " 17 GB", | 20260507 06:43:53.409 | INFO | "huge" : "", | 20260507 06:43:53.409 | INFO | "huge_shared" : " 27 GB", | 20260507 06:43:53.409 | INFO | "malloc" : " 14 GB", | 20260507 06:43:53.409 | INFO | "pinned" : " 29 GB" | 20260507 06:43:53.409 | INFO | } | 20260507 06:43:53.409 | INFO | Perf report for the last 5.14375 seconds: | 20260507 06:43:53.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:43:53.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:53.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:53.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:53.409 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 06:43:53.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:53.409 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:53.409 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 06:43:53.409 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:53.409 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:53.409 | INFO | Uploaded 36GiB to GPU in 2.03385s (17.7004GiB/s) | 20260507 06:43:53.409 | INFO | Downloaded 34GiB to GPU in 1.91708s (17.7353GiB/s) | 20260507 06:43:58.547 | INFO | Analyzing chunk frames = [7904256,7904768) | 20260507 06:43:58.547 | INFO | MemInfo: { | 20260507 06:43:58.547 | INFO | "gpu" : " 17 GB", | 20260507 06:43:58.547 | INFO | "huge" : "", | 20260507 06:43:58.547 | INFO | "huge_shared" : " 27 GB", | 20260507 06:43:58.547 | INFO | "malloc" : " 14 GB", | 20260507 06:43:58.547 | INFO | "pinned" : " 29 GB" | 20260507 06:43:58.547 | INFO | } | 20260507 06:43:58.547 | INFO | Perf report for the last 5.13838 seconds: | 20260507 06:43:58.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:43:58.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:58.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:58.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:58.547 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 06:43:58.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:58.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:58.547 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:43:58.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:58.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:43:58.548 | INFO | Uploaded 36GiB to GPU in 1.88573s (19.0908GiB/s) | 20260507 06:43:58.548 | INFO | Downloaded 34GiB to GPU in 1.97254s (17.2366GiB/s) | 20260507 06:44:03.678 | INFO | Analyzing chunk frames = [7904768,7905280) | 20260507 06:44:03.678 | INFO | MemInfo: { | 20260507 06:44:03.678 | INFO | "gpu" : " 17 GB", | 20260507 06:44:03.678 | INFO | "huge" : "", | 20260507 06:44:03.678 | INFO | "huge_shared" : " 27 GB", | 20260507 06:44:03.678 | INFO | "malloc" : " 14 GB", | 20260507 06:44:03.678 | INFO | "pinned" : " 29 GB" | 20260507 06:44:03.678 | INFO | } | 20260507 06:44:03.679 | INFO | Perf report for the last 5.13122 seconds: | 20260507 06:44:03.679 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:44:03.679 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:03.679 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:03.679 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:03.679 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260507 06:44:03.679 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:03.679 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:03.679 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:44:03.679 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:03.679 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:03.679 | INFO | Uploaded 36GiB to GPU in 1.98763s (18.1121GiB/s) | 20260507 06:44:03.679 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:44:03.679 | INFO | Total mem usage: 18 GiB | 20260507 06:44:03.679 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:44:03.679 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 06:44:03.679 | INFO | Downloaded 34GiB to GPU in 1.95766s (17.3676GiB/s) | 20260507 06:44:03.679 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:44:03.679 | INFO | High Water Breakdown: | 20260507 06:44:03.679 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:44:03.679 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:44:03.679 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:44:03.679 | INFO | ------------------------------------------ | 20260507 06:44:03.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:44:03.679 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:44:03.679 | INFO | Total mem usage: 26.5 GiB | 20260507 06:44:03.679 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:44:03.679 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:44:03.679 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:44:03.679 | INFO | High Water Breakdown: | 20260507 06:44:03.679 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:44:03.679 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:44:03.679 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:44:03.679 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:44:03.679 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:44:03.679 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:44:03.679 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:44:03.679 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:44:03.679 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:44:03.679 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:44:03.679 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:44:03.679 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:44:03.679 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:44:03.679 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:44:03.679 | INFO | ------------------------------------------ | 20260507 06:44:03.679 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:44:03.679 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:44:03.679 | INFO | Total mem usage: 25.6 GiB | 20260507 06:44:03.679 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:44:03.679 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:44:03.679 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:44:03.680 | INFO | High Water Breakdown: | 20260507 06:44:03.680 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:44:03.680 | INFO | ------------------------------------------ | 20260507 06:44:03.680 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:44:03.680 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:44:03.680 | INFO | Total mem usage: 15.7 GiB | 20260507 06:44:03.680 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:44:03.680 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:44:03.680 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:44:03.680 | INFO | High Water Breakdown: | 20260507 06:44:03.680 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:44:03.680 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:44:03.680 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:44:03.680 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:44:03.680 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:44:03.680 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:44:03.680 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:44:03.680 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:44:03.680 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:44:03.680 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:44:03.680 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:44:03.680 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:44:03.680 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:44:03.680 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:44:03.680 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:44:03.680 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:44:03.680 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:44:03.680 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:44:03.680 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:44:03.680 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:44:03.680 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:44:03.680 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:44:03.680 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:44:03.680 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:44:03.680 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:44:03.680 | INFO | ------------------------------------------ | 20260507 06:44:03.680 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:44:03.680 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:44:03.680 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:44:03.680 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:44:03.680 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.34e+01 / 1.24e+00 | 20260507 06:44:03.680 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.03e-01, 9.21e-03 / 4.09e-03 | 20260507 06:44:03.680 | INFO | PerfReport: 11.70%, Baseline -- 3.59e+00 / 4.47e+00, 5.14e-01 / 1.81e-01 | 20260507 06:44:03.680 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.50e+00, 1.70e-01 / 9.03e-02 | 20260507 06:44:03.680 | INFO | PerfReport: 16.28%, DME -- 4.00e+01 / 4.50e+01, 4.01e+00 / 1.48e+00 | 20260507 06:44:03.680 | INFO | PerfReport: 57.61%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.23e+00 / 9.00e-01 | 20260507 06:44:03.680 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.73e+00, 3.91e-01 / 1.61e-01 | 20260507 06:44:03.680 | INFO | PerfReport: 2.96%, Metrics -- 9.07e-01 / 1.23e+00, 2.44e-01 / 1.47e-01 | 20260507 06:44:03.680 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.59e-01, 5.10e-02 / 1.94e-02 | 20260507 06:44:03.680 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:44:03.680 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:44:03.680 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:44:03.680 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 06:44:03.680 | INFO | PerfReport: 21.75%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260507 06:44:03.680 | INFO | PerfReport: 21.37%, Download -- 2.05e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 06:44:03.680 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:44:03.680 | INFO | PerfReport: 1.52%, LockUpload -- 1.46e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 06:44:03.680 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.14e+00 / 0.00e+00 | 20260507 06:44:03.680 | INFO | PerfReport: 22.08%, LockCompute -- 2.12e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 06:44:03.680 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.23e-02 / 0.00e+00 | 20260507 06:44:03.788 | INFO | Starting a round of Baz IO | 20260507 06:44:06.872 | INFO | Populating RTMetrics report | 20260507 06:44:06.872 | INFO | RTMetrics report populated | 20260507 06:44:06.872 | INFO | Opening tmp RTMetrics json file | 20260507 06:44:06.887 | INFO | Serializing RTMetrics to json | 20260507 06:44:06.888 | INFO | Serialization done, swapping tmp file now... | 20260507 06:44:06.889 | INFO | RTMetrics done | 20260507 06:44:06.962 | INFO | Finished a round of Baz IO: Wrote 1.662 GiB in 3.173 seconds | 20260507 06:44:06.962 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.61% / 0.00% / 67.39% | 20260507 06:44:06.962 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.117783% / 0% / 7.54992e-05% | 20260507 06:44:06.962 | INFO | Baz IO overall: | 20260507 06:44:06.962 | INFO | BytesWritten : 3979975441805 | 20260507 06:44:06.962 | INFO | NumEvents : 1906777613577 | 20260507 06:44:08.821 | INFO | Analyzing chunk frames = [7905280,7905792) | 20260507 06:44:08.821 | INFO | MemInfo: { | 20260507 06:44:08.821 | INFO | "gpu" : " 17 GB", | 20260507 06:44:08.821 | INFO | "huge" : "", | 20260507 06:44:08.821 | INFO | "huge_shared" : " 27 GB", | 20260507 06:44:08.821 | INFO | "malloc" : " 14 GB", | 20260507 06:44:08.821 | INFO | "pinned" : " 29 GB" | 20260507 06:44:08.821 | INFO | } | 20260507 06:44:08.821 | INFO | Perf report for the last 5.14251 seconds: | 20260507 06:44:08.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:44:08.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:08.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:08.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:08.821 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 06:44:08.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:08.821 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:44:08.821 | INFO | PRE_HQ : 59.3%, 1.8, 54.9ms, 0 | 20260507 06:44:08.821 | INFO | BAZWRITER : 61.5%, 3.1, 98.9ms, 0 | 20260507 06:44:08.821 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:44:08.821 | INFO | Uploaded 36GiB to GPU in 2.06171s (17.4612GiB/s) | 20260507 06:44:08.822 | INFO | Downloaded 38GiB to GPU in 2.18146s (17.4195GiB/s) | 20260507 06:44:13.959 | INFO | Analyzing chunk frames = [7905792,7906304) | 20260507 06:44:13.959 | INFO | MemInfo: { | 20260507 06:44:13.959 | INFO | "gpu" : " 17 GB", | 20260507 06:44:13.959 | INFO | "huge" : "", | 20260507 06:44:13.959 | INFO | "huge_shared" : " 27 GB", | 20260507 06:44:13.959 | INFO | "malloc" : " 14 GB", | 20260507 06:44:13.959 | INFO | "pinned" : " 29 GB" | 20260507 06:44:13.959 | INFO | } | 20260507 06:44:13.959 | INFO | Perf report for the last 5.1379 seconds: | 20260507 06:44:13.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:44:13.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:13.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:13.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:13.959 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:44:13.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:13.959 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:13.959 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 06:44:13.959 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:13.959 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:13.959 | INFO | Uploaded 36GiB to GPU in 2.01275s (17.886GiB/s) | 20260507 06:44:13.959 | INFO | Downloaded 34GiB to GPU in 1.9335s (17.5847GiB/s) | 20260507 06:44:19.101 | INFO | Analyzing chunk frames = [7906304,7906816) | 20260507 06:44:19.101 | INFO | MemInfo: { | 20260507 06:44:19.101 | INFO | "gpu" : " 17 GB", | 20260507 06:44:19.101 | INFO | "huge" : "", | 20260507 06:44:19.101 | INFO | "huge_shared" : " 27 GB", | 20260507 06:44:19.101 | INFO | "malloc" : " 14 GB", | 20260507 06:44:19.101 | INFO | "pinned" : " 29 GB" | 20260507 06:44:19.101 | INFO | } | 20260507 06:44:19.101 | INFO | Perf report for the last 5.14227 seconds: | 20260507 06:44:19.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:44:19.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:19.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:19.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:19.101 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 06:44:19.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:19.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:19.101 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:44:19.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:19.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:19.102 | INFO | Uploaded 36GiB to GPU in 2.03748s (17.6689GiB/s) | 20260507 06:44:19.102 | INFO | Downloaded 34GiB to GPU in 1.9484s (17.4502GiB/s) | 20260507 06:44:24.241 | INFO | Analyzing chunk frames = [7906816,7907328) | 20260507 06:44:24.241 | INFO | MemInfo: { | 20260507 06:44:24.241 | INFO | "gpu" : " 17 GB", | 20260507 06:44:24.241 | INFO | "huge" : "", | 20260507 06:44:24.241 | INFO | "huge_shared" : " 27 GB", | 20260507 06:44:24.241 | INFO | "malloc" : " 14 GB", | 20260507 06:44:24.241 | INFO | "pinned" : " 29 GB" | 20260507 06:44:24.241 | INFO | } | 20260507 06:44:24.241 | INFO | Perf report for the last 5.14013 seconds: | 20260507 06:44:24.241 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:44:24.241 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:24.241 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:24.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:24.242 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 06:44:24.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:24.242 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:24.242 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:44:24.242 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:24.242 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:24.242 | INFO | Uploaded 36GiB to GPU in 2.04383s (17.614GiB/s) | 20260507 06:44:24.242 | INFO | Downloaded 34GiB to GPU in 1.94882s (17.4465GiB/s) | 20260507 06:44:29.376 | INFO | Analyzing chunk frames = [7907328,7907840) | 20260507 06:44:29.376 | INFO | MemInfo: { | 20260507 06:44:29.376 | INFO | "gpu" : " 17 GB", | 20260507 06:44:29.376 | INFO | "huge" : "", | 20260507 06:44:29.376 | INFO | "huge_shared" : " 27 GB", | 20260507 06:44:29.376 | INFO | "malloc" : " 14 GB", | 20260507 06:44:29.376 | INFO | "pinned" : " 29 GB" | 20260507 06:44:29.376 | INFO | } | 20260507 06:44:29.376 | INFO | Perf report for the last 5.13502 seconds: | 20260507 06:44:29.376 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:44:29.376 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:29.376 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:29.376 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:29.377 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 06:44:29.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:29.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:29.377 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 06:44:29.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:29.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:29.377 | INFO | Uploaded 36GiB to GPU in 2.05117s (17.5509GiB/s) | 20260507 06:44:29.377 | INFO | Downloaded 34GiB to GPU in 1.90404s (17.8567GiB/s) | 20260507 06:44:34.514 | INFO | Analyzing chunk frames = [7907840,7908352) | 20260507 06:44:34.514 | INFO | MemInfo: { | 20260507 06:44:34.514 | INFO | "gpu" : " 17 GB", | 20260507 06:44:34.514 | INFO | "huge" : "", | 20260507 06:44:34.514 | INFO | "huge_shared" : " 27 GB", | 20260507 06:44:34.515 | INFO | "malloc" : " 14 GB", | 20260507 06:44:34.515 | INFO | "pinned" : " 29 GB" | 20260507 06:44:34.515 | INFO | } | 20260507 06:44:34.515 | INFO | Perf report for the last 5.13818 seconds: | 20260507 06:44:34.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:44:34.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:34.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:34.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:34.515 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 06:44:34.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:34.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:34.515 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:44:34.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:34.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:34.515 | INFO | Uploaded 36GiB to GPU in 2.04228s (17.6273GiB/s) | 20260507 06:44:34.515 | INFO | Downloaded 34GiB to GPU in 1.90318s (17.8649GiB/s) | 20260507 06:44:39.656 | INFO | Analyzing chunk frames = [7908352,7908864) | 20260507 06:44:39.656 | INFO | MemInfo: { | 20260507 06:44:39.656 | INFO | "gpu" : " 17 GB", | 20260507 06:44:39.656 | INFO | "huge" : "", | 20260507 06:44:39.656 | INFO | "huge_shared" : " 27 GB", | 20260507 06:44:39.656 | INFO | "malloc" : " 14 GB", | 20260507 06:44:39.656 | INFO | "pinned" : " 29 GB" | 20260507 06:44:39.656 | INFO | } | 20260507 06:44:39.656 | INFO | Perf report for the last 5.14165 seconds: | 20260507 06:44:39.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:44:39.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:39.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:39.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:39.656 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:44:39.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:39.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:39.656 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 06:44:39.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:39.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:39.657 | INFO | Uploaded 36GiB to GPU in 1.97367s (18.2401GiB/s) | 20260507 06:44:39.657 | INFO | Downloaded 34GiB to GPU in 1.96812s (17.2754GiB/s) | 20260507 06:44:44.793 | INFO | Analyzing chunk frames = [7908864,7909376) | 20260507 06:44:44.793 | INFO | MemInfo: { | 20260507 06:44:44.793 | INFO | "gpu" : " 17 GB", | 20260507 06:44:44.793 | INFO | "huge" : "", | 20260507 06:44:44.793 | INFO | "huge_shared" : " 27 GB", | 20260507 06:44:44.794 | INFO | "malloc" : " 14 GB", | 20260507 06:44:44.794 | INFO | "pinned" : " 29 GB" | 20260507 06:44:44.794 | INFO | } | 20260507 06:44:44.794 | INFO | Perf report for the last 5.13731 seconds: | 20260507 06:44:44.794 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:44:44.794 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:44.794 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:44.794 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:44.794 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:44:44.794 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:44.794 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:44.794 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 06:44:44.794 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:44.794 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:44.794 | INFO | Uploaded 36GiB to GPU in 1.9866s (18.1214GiB/s) | 20260507 06:44:44.794 | INFO | Downloaded 34GiB to GPU in 1.96664s (17.2884GiB/s) | 20260507 06:44:44.891 | INFO | Starting a round of Baz IO | 20260507 06:44:47.983 | INFO | Populating RTMetrics report | 20260507 06:44:47.983 | INFO | RTMetrics report populated | 20260507 06:44:47.983 | INFO | Opening tmp RTMetrics json file | 20260507 06:44:47.990 | INFO | Serializing RTMetrics to json | 20260507 06:44:48.000 | INFO | Serialization done, swapping tmp file now... | 20260507 06:44:48.000 | INFO | RTMetrics done | 20260507 06:44:48.069 | INFO | Finished a round of Baz IO: Wrote 1.66 GiB in 3.178 seconds | 20260507 06:44:48.069 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.61% / 0.00% / 67.39% | 20260507 06:44:48.069 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.117226% / 3.97364e-06% / 6.75519e-05% | 20260507 06:44:48.069 | INFO | Baz IO overall: | 20260507 06:44:48.069 | INFO | BytesWritten : 3981757753709 | 20260507 06:44:48.069 | INFO | NumEvents : 1907611215133 | 20260507 06:44:49.929 | INFO | Analyzing chunk frames = [7909376,7909888) | 20260507 06:44:49.929 | INFO | MemInfo: { | 20260507 06:44:49.929 | INFO | "gpu" : " 17 GB", | 20260507 06:44:49.929 | INFO | "huge" : "", | 20260507 06:44:49.929 | INFO | "huge_shared" : " 27 GB", | 20260507 06:44:49.929 | INFO | "malloc" : " 14 GB", | 20260507 06:44:49.929 | INFO | "pinned" : " 29 GB" | 20260507 06:44:49.929 | INFO | } | 20260507 06:44:49.929 | INFO | Perf report for the last 5.13547 seconds: | 20260507 06:44:49.929 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:44:49.929 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:49.929 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:49.929 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:49.929 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 06:44:49.929 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:49.929 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:44:49.929 | INFO | PRE_HQ : 59.0%, 1.8, 54.7ms, 0 | 20260507 06:44:49.929 | INFO | BAZWRITER : 61.5%, 3.1, 98.3ms, 0 | 20260507 06:44:49.929 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:44:49.929 | INFO | Uploaded 36GiB to GPU in 2.04673s (17.589GiB/s) | 20260507 06:44:49.929 | INFO | Downloaded 38GiB to GPU in 2.18063s (17.4262GiB/s) | 20260507 06:44:55.075 | INFO | Analyzing chunk frames = [7909888,7910400) | 20260507 06:44:55.075 | INFO | MemInfo: { | 20260507 06:44:55.075 | INFO | "gpu" : " 17 GB", | 20260507 06:44:55.075 | INFO | "huge" : "", | 20260507 06:44:55.075 | INFO | "huge_shared" : " 27 GB", | 20260507 06:44:55.075 | INFO | "malloc" : " 14 GB", | 20260507 06:44:55.075 | INFO | "pinned" : " 29 GB" | 20260507 06:44:55.075 | INFO | } | 20260507 06:44:55.075 | INFO | Perf report for the last 5.14611 seconds: | 20260507 06:44:55.075 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:44:55.075 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:55.075 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:55.075 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:55.075 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 06:44:55.075 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:55.075 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:55.075 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 06:44:55.075 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:55.075 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:44:55.076 | INFO | Uploaded 36GiB to GPU in 1.99372s (18.0567GiB/s) | 20260507 06:44:55.076 | INFO | Downloaded 34GiB to GPU in 1.93616s (17.5605GiB/s) | 20260507 06:45:00.221 | INFO | Analyzing chunk frames = [7910400,7910912) | 20260507 06:45:00.221 | INFO | MemInfo: { | 20260507 06:45:00.221 | INFO | "gpu" : " 17 GB", | 20260507 06:45:00.221 | INFO | "huge" : "", | 20260507 06:45:00.221 | INFO | "huge_shared" : " 27 GB", | 20260507 06:45:00.221 | INFO | "malloc" : " 14 GB", | 20260507 06:45:00.221 | INFO | "pinned" : " 29 GB" | 20260507 06:45:00.221 | INFO | } | 20260507 06:45:00.221 | INFO | Perf report for the last 5.14584 seconds: | 20260507 06:45:00.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:45:00.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:00.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:00.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:00.221 | INFO | ANALYSIS : 60.1%, 3.0, 93.1ms, 0 | 20260507 06:45:00.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:00.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:00.221 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:45:00.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:00.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:00.221 | INFO | Uploaded 36GiB to GPU in 2.01538s (17.8627GiB/s) | 20260507 06:45:00.221 | INFO | Downloaded 34GiB to GPU in 1.94585s (17.4731GiB/s) | 20260507 06:45:05.362 | INFO | Analyzing chunk frames = [7910912,7911424) | 20260507 06:45:05.362 | INFO | MemInfo: { | 20260507 06:45:05.362 | INFO | "gpu" : " 17 GB", | 20260507 06:45:05.362 | INFO | "huge" : "", | 20260507 06:45:05.362 | INFO | "huge_shared" : " 27 GB", | 20260507 06:45:05.362 | INFO | "malloc" : " 14 GB", | 20260507 06:45:05.363 | INFO | "pinned" : " 29 GB" | 20260507 06:45:05.363 | INFO | } | 20260507 06:45:05.363 | INFO | Perf report for the last 5.14143 seconds: | 20260507 06:45:05.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:45:05.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:05.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:05.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:05.363 | INFO | ANALYSIS : 60.1%, 2.9, 92.8ms, 0 | 20260507 06:45:05.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:05.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:05.363 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:45:05.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:05.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:05.363 | INFO | Uploaded 36GiB to GPU in 1.97777s (18.2023GiB/s) | 20260507 06:45:05.363 | INFO | Downloaded 34GiB to GPU in 1.9432s (17.4969GiB/s) | 20260507 06:45:10.491 | INFO | Analyzing chunk frames = [7911424,7911936) | 20260507 06:45:10.491 | INFO | MemInfo: { | 20260507 06:45:10.491 | INFO | "gpu" : " 17 GB", | 20260507 06:45:10.491 | INFO | "huge" : "", | 20260507 06:45:10.491 | INFO | "huge_shared" : " 27 GB", | 20260507 06:45:10.491 | INFO | "malloc" : " 14 GB", | 20260507 06:45:10.491 | INFO | "pinned" : " 29 GB" | 20260507 06:45:10.491 | INFO | } | 20260507 06:45:10.491 | INFO | Perf report for the last 5.12871 seconds: | 20260507 06:45:10.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:45:10.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:10.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:10.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:10.491 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 06:45:10.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:10.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:10.491 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 06:45:10.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:10.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:10.492 | INFO | Uploaded 36GiB to GPU in 2.0595s (17.4799GiB/s) | 20260507 06:45:10.492 | INFO | Downloaded 34GiB to GPU in 1.91462s (17.7581GiB/s) | 20260507 06:45:15.626 | INFO | Analyzing chunk frames = [7911936,7912448) | 20260507 06:45:15.626 | INFO | MemInfo: { | 20260507 06:45:15.626 | INFO | "gpu" : " 17 GB", | 20260507 06:45:15.626 | INFO | "huge" : "", | 20260507 06:45:15.626 | INFO | "huge_shared" : " 27 GB", | 20260507 06:45:15.626 | INFO | "malloc" : " 14 GB", | 20260507 06:45:15.626 | INFO | "pinned" : " 29 GB" | 20260507 06:45:15.626 | INFO | } | 20260507 06:45:15.626 | INFO | Perf report for the last 5.13517 seconds: | 20260507 06:45:15.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:45:15.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:15.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:15.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:15.627 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 06:45:15.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:15.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:15.627 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:45:15.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:15.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:15.627 | INFO | Uploaded 36GiB to GPU in 2.04022s (17.6452GiB/s) | 20260507 06:45:15.627 | INFO | Downloaded 34GiB to GPU in 1.921s (17.6991GiB/s) | 20260507 06:45:20.770 | INFO | Analyzing chunk frames = [7912448,7912960) | 20260507 06:45:20.770 | INFO | MemInfo: { | 20260507 06:45:20.770 | INFO | "gpu" : " 17 GB", | 20260507 06:45:20.770 | INFO | "huge" : "", | 20260507 06:45:20.770 | INFO | "huge_shared" : " 27 GB", | 20260507 06:45:20.770 | INFO | "malloc" : " 14 GB", | 20260507 06:45:20.770 | INFO | "pinned" : " 29 GB" | 20260507 06:45:20.770 | INFO | } | 20260507 06:45:20.770 | INFO | Perf report for the last 5.14382 seconds: | 20260507 06:45:20.770 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:45:20.770 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:20.770 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:20.770 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:20.770 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 06:45:20.770 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:20.770 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:20.770 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 06:45:20.770 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:20.770 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:20.771 | INFO | Uploaded 36GiB to GPU in 1.97811s (18.1992GiB/s) | 20260507 06:45:20.771 | INFO | Downloaded 34GiB to GPU in 1.96936s (17.2645GiB/s) | 20260507 06:45:25.914 | INFO | Analyzing chunk frames = [7912960,7913472) | 20260507 06:45:25.914 | INFO | MemInfo: { | 20260507 06:45:25.914 | INFO | "gpu" : " 17 GB", | 20260507 06:45:25.914 | INFO | "huge" : "", | 20260507 06:45:25.914 | INFO | "huge_shared" : " 27 GB", | 20260507 06:45:25.914 | INFO | "malloc" : " 14 GB", | 20260507 06:45:25.914 | INFO | "pinned" : " 29 GB" | 20260507 06:45:25.914 | INFO | } | 20260507 06:45:25.914 | INFO | Perf report for the last 5.14372 seconds: | 20260507 06:45:25.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:45:25.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:25.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:25.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:25.914 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:45:25.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:25.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:25.914 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:45:25.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:25.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:25.914 | INFO | Uploaded 36GiB to GPU in 1.98798s (18.1088GiB/s) | 20260507 06:45:25.914 | INFO | Downloaded 34GiB to GPU in 1.96671s (17.2878GiB/s) | 20260507 06:45:25.915 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:45:25.915 | INFO | Total mem usage: 18 GiB | 20260507 06:45:25.915 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:45:25.915 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 06:45:25.915 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:45:25.915 | INFO | High Water Breakdown: | 20260507 06:45:25.915 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:45:25.915 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:45:25.915 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:45:25.915 | INFO | ------------------------------------------ | 20260507 06:45:25.915 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:45:25.915 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:45:25.915 | INFO | Total mem usage: 26.5 GiB | 20260507 06:45:25.915 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:45:25.915 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:45:25.915 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 06:45:25.915 | INFO | High Water Breakdown: | 20260507 06:45:25.915 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:45:25.915 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:45:25.915 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:45:25.915 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:45:25.915 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:45:25.915 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:45:25.915 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:45:25.915 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:45:25.915 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:45:25.915 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:45:25.915 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:45:25.915 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:45:25.915 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:45:25.915 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:45:25.915 | INFO | ------------------------------------------ | 20260507 06:45:25.915 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:45:25.915 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:45:25.915 | INFO | Total mem usage: 25.6 GiB | 20260507 06:45:25.915 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:45:25.915 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:45:25.915 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:45:25.915 | INFO | High Water Breakdown: | 20260507 06:45:25.915 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:45:25.915 | INFO | ------------------------------------------ | 20260507 06:45:25.915 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:45:25.915 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:45:25.915 | INFO | Total mem usage: 15.7 GiB | 20260507 06:45:25.915 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:45:25.915 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:45:25.915 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:45:25.915 | INFO | High Water Breakdown: | 20260507 06:45:25.915 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:45:25.915 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:45:25.915 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:45:25.915 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:45:25.915 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:45:25.915 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:45:25.915 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:45:25.915 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:45:25.915 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:45:25.915 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:45:25.915 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:45:25.915 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:45:25.915 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:45:25.915 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:45:25.915 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:45:25.915 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:45:25.915 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:45:25.915 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:45:25.915 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:45:25.915 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:45:25.915 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:45:25.915 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:45:25.916 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:45:25.916 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:45:25.916 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:45:25.916 | INFO | ------------------------------------------ | 20260507 06:45:25.916 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:45:25.916 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:45:25.916 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:45:25.916 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:45:25.916 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.33e+01 / 1.01e+00 | 20260507 06:45:25.916 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.02e-01, 9.54e-03 / 6.71e-03 | 20260507 06:45:25.916 | INFO | PerfReport: 11.70%, Baseline -- 3.59e+00 / 4.46e+00, 5.16e-01 / 2.66e-01 | 20260507 06:45:25.916 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.48e+00, 1.69e-01 / 1.20e-01 | 20260507 06:45:25.916 | INFO | PerfReport: 16.26%, DME -- 3.99e+01 / 4.45e+01, 3.79e+00 / 1.50e+00 | 20260507 06:45:25.916 | INFO | PerfReport: 57.67%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.28e+00 / 1.38e+00 | 20260507 06:45:25.916 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.71e+00, 3.81e-01 / 1.95e-01 | 20260507 06:45:25.916 | INFO | PerfReport: 2.95%, Metrics -- 9.05e-01 / 1.21e+00, 2.44e-01 / 1.24e-01 | 20260507 06:45:25.916 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.58e-01, 5.00e-02 / 2.07e-02 | 20260507 06:45:25.916 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:45:25.916 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:45:25.916 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:45:25.916 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 06:45:25.916 | INFO | PerfReport: 21.88%, Upload -- 2.10e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260507 06:45:25.916 | INFO | PerfReport: 21.36%, Download -- 2.05e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 06:45:25.916 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 06:45:25.916 | INFO | PerfReport: 1.57%, LockUpload -- 1.51e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 06:45:25.916 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260507 06:45:25.916 | INFO | PerfReport: 21.96%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 06:45:25.916 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260507 06:45:26.011 | INFO | Starting a round of Baz IO | 20260507 06:45:29.105 | INFO | Populating RTMetrics report | 20260507 06:45:29.106 | INFO | RTMetrics report populated | 20260507 06:45:29.106 | INFO | Opening tmp RTMetrics json file | 20260507 06:45:29.122 | INFO | Serializing RTMetrics to json | 20260507 06:45:29.123 | INFO | Serialization done, swapping tmp file now... | 20260507 06:45:29.124 | INFO | RTMetrics done | 20260507 06:45:29.191 | INFO | Finished a round of Baz IO: Wrote 1.658 GiB in 3.18 seconds | 20260507 06:45:29.191 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.61% / 0.00% / 67.39% | 20260507 06:45:29.191 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.11636% / 3.97364e-06% / 7.54992e-05% | 20260507 06:45:29.191 | INFO | Baz IO overall: | 20260507 06:45:29.191 | INFO | BytesWritten : 3983537579721 | 20260507 06:45:29.191 | INFO | NumEvents : 1908443574058 | 20260507 06:45:31.051 | INFO | Analyzing chunk frames = [7913472,7913984) | 20260507 06:45:31.051 | INFO | MemInfo: { | 20260507 06:45:31.051 | INFO | "gpu" : " 17 GB", | 20260507 06:45:31.051 | INFO | "huge" : "", | 20260507 06:45:31.051 | INFO | "huge_shared" : " 27 GB", | 20260507 06:45:31.051 | INFO | "malloc" : " 14 GB", | 20260507 06:45:31.051 | INFO | "pinned" : " 29 GB" | 20260507 06:45:31.051 | INFO | } | 20260507 06:45:31.052 | INFO | Perf report for the last 5.13748 seconds: | 20260507 06:45:31.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:45:31.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:31.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:31.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:31.052 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 06:45:31.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:31.052 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:45:31.052 | INFO | PRE_HQ : 58.9%, 1.8, 54.7ms, 0 | 20260507 06:45:31.052 | INFO | BAZWRITER : 61.5%, 3.1, 97.7ms, 0 | 20260507 06:45:31.052 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:45:31.052 | INFO | Uploaded 36GiB to GPU in 2.06647s (17.421GiB/s) | 20260507 06:45:31.052 | INFO | Downloaded 38GiB to GPU in 2.17746s (17.4515GiB/s) | 20260507 06:45:36.190 | INFO | Analyzing chunk frames = [7913984,7914496) | 20260507 06:45:36.190 | INFO | MemInfo: { | 20260507 06:45:36.190 | INFO | "gpu" : " 17 GB", | 20260507 06:45:36.190 | INFO | "huge" : "", | 20260507 06:45:36.190 | INFO | "huge_shared" : " 27 GB", | 20260507 06:45:36.190 | INFO | "malloc" : " 14 GB", | 20260507 06:45:36.190 | INFO | "pinned" : " 29 GB" | 20260507 06:45:36.190 | INFO | } | 20260507 06:45:36.190 | INFO | Perf report for the last 5.1383 seconds: | 20260507 06:45:36.190 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:45:36.190 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:36.190 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:36.190 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:36.190 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:45:36.190 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:36.190 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:36.190 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 06:45:36.190 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:36.190 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:36.190 | INFO | Uploaded 36GiB to GPU in 1.98416s (18.1437GiB/s) | 20260507 06:45:36.190 | INFO | Downloaded 34GiB to GPU in 1.92979s (17.6185GiB/s) | 20260507 06:45:41.328 | INFO | Analyzing chunk frames = [7914496,7915008) | 20260507 06:45:41.328 | INFO | MemInfo: { | 20260507 06:45:41.328 | INFO | "gpu" : " 17 GB", | 20260507 06:45:41.328 | INFO | "huge" : "", | 20260507 06:45:41.328 | INFO | "huge_shared" : " 27 GB", | 20260507 06:45:41.328 | INFO | "malloc" : " 14 GB", | 20260507 06:45:41.328 | INFO | "pinned" : " 29 GB" | 20260507 06:45:41.328 | INFO | } | 20260507 06:45:41.328 | INFO | Perf report for the last 5.13818 seconds: | 20260507 06:45:41.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:45:41.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:41.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:41.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:41.328 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:45:41.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:41.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:41.328 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:45:41.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:41.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:41.328 | INFO | Uploaded 36GiB to GPU in 2.0124s (17.8891GiB/s) | 20260507 06:45:41.328 | INFO | Downloaded 34GiB to GPU in 1.91342s (17.7692GiB/s) | 20260507 06:45:46.467 | INFO | Analyzing chunk frames = [7915008,7915520) | 20260507 06:45:46.467 | INFO | MemInfo: { | 20260507 06:45:46.467 | INFO | "gpu" : " 17 GB", | 20260507 06:45:46.467 | INFO | "huge" : "", | 20260507 06:45:46.467 | INFO | "huge_shared" : " 27 GB", | 20260507 06:45:46.467 | INFO | "malloc" : " 14 GB", | 20260507 06:45:46.467 | INFO | "pinned" : " 29 GB" | 20260507 06:45:46.467 | INFO | } | 20260507 06:45:46.467 | INFO | Perf report for the last 5.13876 seconds: | 20260507 06:45:46.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:45:46.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:46.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:46.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:46.467 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 06:45:46.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:46.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:46.467 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:45:46.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:46.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:46.467 | INFO | Uploaded 36GiB to GPU in 2.02174s (17.8065GiB/s) | 20260507 06:45:46.467 | INFO | Downloaded 34GiB to GPU in 1.93557s (17.5659GiB/s) | 20260507 06:45:51.609 | INFO | Analyzing chunk frames = [7915520,7916032) | 20260507 06:45:51.609 | INFO | MemInfo: { | 20260507 06:45:51.609 | INFO | "gpu" : " 17 GB", | 20260507 06:45:51.609 | INFO | "huge" : "", | 20260507 06:45:51.609 | INFO | "huge_shared" : " 27 GB", | 20260507 06:45:51.609 | INFO | "malloc" : " 14 GB", | 20260507 06:45:51.609 | INFO | "pinned" : " 29 GB" | 20260507 06:45:51.609 | INFO | } | 20260507 06:45:51.609 | INFO | Perf report for the last 5.14262 seconds: | 20260507 06:45:51.609 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:45:51.609 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:51.609 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:51.609 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:51.610 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 06:45:51.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:51.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:51.610 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:45:51.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:51.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:51.610 | INFO | Uploaded 36GiB to GPU in 2.0536s (17.5302GiB/s) | 20260507 06:45:51.610 | INFO | Downloaded 34GiB to GPU in 1.94709s (17.462GiB/s) | 20260507 06:45:56.747 | INFO | Analyzing chunk frames = [7916032,7916544) | 20260507 06:45:56.747 | INFO | MemInfo: { | 20260507 06:45:56.747 | INFO | "gpu" : " 17 GB", | 20260507 06:45:56.747 | INFO | "huge" : "", | 20260507 06:45:56.747 | INFO | "huge_shared" : " 27 GB", | 20260507 06:45:56.747 | INFO | "malloc" : " 14 GB", | 20260507 06:45:56.747 | INFO | "pinned" : " 29 GB" | 20260507 06:45:56.747 | INFO | } | 20260507 06:45:56.747 | INFO | Perf report for the last 5.13749 seconds: | 20260507 06:45:56.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:45:56.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:56.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:56.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:56.747 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:45:56.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:56.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:56.747 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 06:45:56.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:56.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:45:56.747 | INFO | Uploaded 36GiB to GPU in 2.06567s (17.4278GiB/s) | 20260507 06:45:56.747 | INFO | Downloaded 34GiB to GPU in 1.92453s (17.6667GiB/s) | 20260507 06:46:01.882 | INFO | Analyzing chunk frames = [7916544,7917056) | 20260507 06:46:01.882 | INFO | MemInfo: { | 20260507 06:46:01.882 | INFO | "gpu" : " 17 GB", | 20260507 06:46:01.882 | INFO | "huge" : "", | 20260507 06:46:01.883 | INFO | "huge_shared" : " 27 GB", | 20260507 06:46:01.883 | INFO | "malloc" : " 14 GB", | 20260507 06:46:01.883 | INFO | "pinned" : " 29 GB" | 20260507 06:46:01.883 | INFO | } | 20260507 06:46:01.883 | INFO | Perf report for the last 5.13581 seconds: | 20260507 06:46:01.883 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:46:01.883 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:01.883 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:01.883 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:01.883 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 06:46:01.883 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:01.883 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:01.883 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:46:01.883 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:01.883 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:01.883 | INFO | Uploaded 36GiB to GPU in 1.98796s (18.1091GiB/s) | 20260507 06:46:01.883 | INFO | Downloaded 34GiB to GPU in 1.98608s (17.1192GiB/s) | 20260507 06:46:07.025 | INFO | Analyzing chunk frames = [7917056,7917568) | 20260507 06:46:07.025 | INFO | MemInfo: { | 20260507 06:46:07.025 | INFO | "gpu" : " 17 GB", | 20260507 06:46:07.025 | INFO | "huge" : "", | 20260507 06:46:07.025 | INFO | "huge_shared" : " 27 GB", | 20260507 06:46:07.025 | INFO | "malloc" : " 14 GB", | 20260507 06:46:07.025 | INFO | "pinned" : " 29 GB" | 20260507 06:46:07.025 | INFO | } | 20260507 06:46:07.026 | INFO | Perf report for the last 5.14277 seconds: | 20260507 06:46:07.026 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:46:07.026 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:07.026 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:07.026 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:07.026 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:46:07.026 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:07.026 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:07.026 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:46:07.026 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:07.026 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:07.026 | INFO | Uploaded 36GiB to GPU in 1.96648s (18.3068GiB/s) | 20260507 06:46:07.026 | INFO | Downloaded 34GiB to GPU in 1.95037s (17.4326GiB/s) | 20260507 06:46:07.123 | INFO | Starting a round of Baz IO | 20260507 06:46:10.218 | INFO | Populating RTMetrics report | 20260507 06:46:10.218 | INFO | RTMetrics report populated | 20260507 06:46:10.218 | INFO | Opening tmp RTMetrics json file | 20260507 06:46:10.225 | INFO | Serializing RTMetrics to json | 20260507 06:46:10.234 | INFO | Serialization done, swapping tmp file now... | 20260507 06:46:10.235 | INFO | RTMetrics done | 20260507 06:46:10.303 | INFO | Finished a round of Baz IO: Wrote 1.659 GiB in 3.18 seconds | 20260507 06:46:10.303 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.60% / 0.00% / 67.39% | 20260507 06:46:10.303 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.116487% / 0% / 5.5631e-05% | 20260507 06:46:10.303 | INFO | Baz IO overall: | 20260507 06:46:10.303 | INFO | BytesWritten : 3985319171587 | 20260507 06:46:10.303 | INFO | NumEvents : 1909277107114 | 20260507 06:46:12.168 | INFO | Analyzing chunk frames = [7917568,7918080) | 20260507 06:46:12.168 | INFO | MemInfo: { | 20260507 06:46:12.168 | INFO | "gpu" : " 17 GB", | 20260507 06:46:12.168 | INFO | "huge" : "", | 20260507 06:46:12.168 | INFO | "huge_shared" : " 27 GB", | 20260507 06:46:12.168 | INFO | "malloc" : " 14 GB", | 20260507 06:46:12.168 | INFO | "pinned" : " 29 GB" | 20260507 06:46:12.168 | INFO | } | 20260507 06:46:12.168 | INFO | Perf report for the last 5.14276 seconds: | 20260507 06:46:12.168 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:46:12.168 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:12.168 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:12.168 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:12.168 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260507 06:46:12.168 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:12.168 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 06:46:12.168 | INFO | PRE_HQ : 58.9%, 1.8, 55.0ms, 0 | 20260507 06:46:12.168 | INFO | BAZWRITER : 61.4%, 3.1, 97.4ms, 0 | 20260507 06:46:12.168 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:46:12.169 | INFO | Uploaded 36GiB to GPU in 2.0498s (17.5627GiB/s) | 20260507 06:46:12.169 | INFO | Downloaded 38GiB to GPU in 2.17035s (17.5087GiB/s) | 20260507 06:46:17.305 | INFO | Analyzing chunk frames = [7918080,7918592) | 20260507 06:46:17.305 | INFO | MemInfo: { | 20260507 06:46:17.305 | INFO | "gpu" : " 17 GB", | 20260507 06:46:17.305 | INFO | "huge" : "", | 20260507 06:46:17.305 | INFO | "huge_shared" : " 27 GB", | 20260507 06:46:17.305 | INFO | "malloc" : " 14 GB", | 20260507 06:46:17.305 | INFO | "pinned" : " 29 GB" | 20260507 06:46:17.305 | INFO | } | 20260507 06:46:17.305 | INFO | Perf report for the last 5.13681 seconds: | 20260507 06:46:17.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:46:17.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:17.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:17.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:17.305 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:46:17.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:17.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:17.305 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:46:17.305 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:17.305 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:17.305 | INFO | Uploaded 36GiB to GPU in 1.99409s (18.0533GiB/s) | 20260507 06:46:17.305 | INFO | Downloaded 34GiB to GPU in 1.94016s (17.5244GiB/s) | 20260507 06:46:22.440 | INFO | Analyzing chunk frames = [7918592,7919104) | 20260507 06:46:22.440 | INFO | MemInfo: { | 20260507 06:46:22.440 | INFO | "gpu" : " 17 GB", | 20260507 06:46:22.440 | INFO | "huge" : "", | 20260507 06:46:22.440 | INFO | "huge_shared" : " 27 GB", | 20260507 06:46:22.440 | INFO | "malloc" : " 14 GB", | 20260507 06:46:22.440 | INFO | "pinned" : " 29 GB" | 20260507 06:46:22.440 | INFO | } | 20260507 06:46:22.440 | INFO | Perf report for the last 5.13475 seconds: | 20260507 06:46:22.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:46:22.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:22.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:22.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:22.440 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260507 06:46:22.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:22.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:22.440 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:46:22.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:22.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:22.440 | INFO | Uploaded 36GiB to GPU in 2.03221s (17.7147GiB/s) | 20260507 06:46:22.440 | INFO | Downloaded 34GiB to GPU in 1.91914s (17.7163GiB/s) | 20260507 06:46:27.583 | INFO | Analyzing chunk frames = [7919104,7919616) | 20260507 06:46:27.583 | INFO | MemInfo: { | 20260507 06:46:27.583 | INFO | "gpu" : " 17 GB", | 20260507 06:46:27.583 | INFO | "huge" : "", | 20260507 06:46:27.583 | INFO | "huge_shared" : " 27 GB", | 20260507 06:46:27.583 | INFO | "malloc" : " 14 GB", | 20260507 06:46:27.583 | INFO | "pinned" : " 29 GB" | 20260507 06:46:27.583 | INFO | } | 20260507 06:46:27.583 | INFO | Perf report for the last 5.14342 seconds: | 20260507 06:46:27.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:46:27.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:27.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:27.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:27.583 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 06:46:27.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:27.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:27.583 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:46:27.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:27.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:27.584 | INFO | Uploaded 36GiB to GPU in 2.0189s (17.8315GiB/s) | 20260507 06:46:27.584 | INFO | Downloaded 34GiB to GPU in 1.93312s (17.5882GiB/s) | 20260507 06:46:32.724 | INFO | Analyzing chunk frames = [7919616,7920128) | 20260507 06:46:32.724 | INFO | MemInfo: { | 20260507 06:46:32.724 | INFO | "gpu" : " 17 GB", | 20260507 06:46:32.724 | INFO | "huge" : "", | 20260507 06:46:32.724 | INFO | "huge_shared" : " 27 GB", | 20260507 06:46:32.724 | INFO | "malloc" : " 14 GB", | 20260507 06:46:32.724 | INFO | "pinned" : " 29 GB" | 20260507 06:46:32.724 | INFO | } | 20260507 06:46:32.724 | INFO | Perf report for the last 5.14077 seconds: | 20260507 06:46:32.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:46:32.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:32.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:32.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:32.724 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 06:46:32.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:32.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:32.724 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:46:32.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:32.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:32.724 | INFO | Uploaded 36GiB to GPU in 2.05635s (17.5067GiB/s) | 20260507 06:46:32.724 | INFO | Downloaded 34GiB to GPU in 1.94437s (17.4864GiB/s) | 20260507 06:46:37.867 | INFO | Analyzing chunk frames = [7920128,7920640) | 20260507 06:46:37.867 | INFO | MemInfo: { | 20260507 06:46:37.867 | INFO | "gpu" : " 17 GB", | 20260507 06:46:37.867 | INFO | "huge" : "", | 20260507 06:46:37.867 | INFO | "huge_shared" : " 27 GB", | 20260507 06:46:37.867 | INFO | "malloc" : " 14 GB", | 20260507 06:46:37.867 | INFO | "pinned" : " 29 GB" | 20260507 06:46:37.867 | INFO | } | 20260507 06:46:37.868 | INFO | Perf report for the last 5.14343 seconds: | 20260507 06:46:37.868 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:46:37.868 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:37.868 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:37.868 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:37.868 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 06:46:37.868 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:37.868 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:37.868 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 06:46:37.868 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:37.868 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:37.868 | INFO | Uploaded 36GiB to GPU in 2.06009s (17.4749GiB/s) | 20260507 06:46:37.868 | INFO | Downloaded 34GiB to GPU in 1.91867s (17.7206GiB/s) | 20260507 06:46:43.002 | INFO | Analyzing chunk frames = [7920640,7921152) | 20260507 06:46:43.002 | INFO | MemInfo: { | 20260507 06:46:43.002 | INFO | "gpu" : " 17 GB", | 20260507 06:46:43.003 | INFO | "huge" : "", | 20260507 06:46:43.003 | INFO | "huge_shared" : " 27 GB", | 20260507 06:46:43.003 | INFO | "malloc" : " 14 GB", | 20260507 06:46:43.003 | INFO | "pinned" : " 29 GB" | 20260507 06:46:43.003 | INFO | } | 20260507 06:46:43.003 | INFO | Perf report for the last 5.13517 seconds: | 20260507 06:46:43.003 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:46:43.003 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:43.003 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:43.003 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:43.003 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 06:46:43.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:43.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:43.003 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 06:46:43.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:43.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:43.003 | INFO | Uploaded 36GiB to GPU in 1.90217s (18.9257GiB/s) | 20260507 06:46:43.003 | INFO | Downloaded 34GiB to GPU in 1.97395s (17.2244GiB/s) | 20260507 06:46:48.140 | INFO | Analyzing chunk frames = [7921152,7921664) | 20260507 06:46:48.140 | INFO | MemInfo: { | 20260507 06:46:48.140 | INFO | "gpu" : " 17 GB", | 20260507 06:46:48.140 | INFO | "huge" : "", | 20260507 06:46:48.140 | INFO | "huge_shared" : " 27 GB", | 20260507 06:46:48.140 | INFO | "malloc" : " 14 GB", | 20260507 06:46:48.140 | INFO | "pinned" : " 29 GB" | 20260507 06:46:48.140 | INFO | } | 20260507 06:46:48.140 | INFO | Perf report for the last 5.13707 seconds: | 20260507 06:46:48.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:46:48.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:48.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:48.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:48.140 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 06:46:48.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:48.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:48.140 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:46:48.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:48.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:48.140 | INFO | Uploaded 36GiB to GPU in 1.97857s (18.195GiB/s) | 20260507 06:46:48.140 | INFO | Downloaded 34GiB to GPU in 1.96116s (17.3367GiB/s) | 20260507 06:46:48.140 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:46:48.140 | INFO | Total mem usage: 18 GiB | 20260507 06:46:48.140 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:46:48.140 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:46:48.140 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:46:48.140 | INFO | High Water Breakdown: | 20260507 06:46:48.140 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:46:48.140 | INFO | 39.7% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:46:48.140 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:46:48.140 | INFO | ------------------------------------------ | 20260507 06:46:48.140 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:46:48.140 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:46:48.140 | INFO | Total mem usage: 26.5 GiB | 20260507 06:46:48.141 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:46:48.141 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:46:48.141 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:46:48.141 | INFO | High Water Breakdown: | 20260507 06:46:48.141 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:46:48.141 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:46:48.141 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:46:48.141 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:46:48.141 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:46:48.141 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:46:48.141 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:46:48.141 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:46:48.141 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:46:48.141 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:46:48.141 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:46:48.141 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:46:48.141 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:46:48.141 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:46:48.141 | INFO | ------------------------------------------ | 20260507 06:46:48.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:46:48.141 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:46:48.141 | INFO | Total mem usage: 25.6 GiB | 20260507 06:46:48.141 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:46:48.141 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:46:48.141 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:46:48.141 | INFO | High Water Breakdown: | 20260507 06:46:48.141 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:46:48.141 | INFO | ------------------------------------------ | 20260507 06:46:48.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:46:48.141 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:46:48.141 | INFO | Total mem usage: 15.7 GiB | 20260507 06:46:48.141 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:46:48.141 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:46:48.141 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:46:48.141 | INFO | High Water Breakdown: | 20260507 06:46:48.141 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:46:48.141 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:46:48.141 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:46:48.141 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:46:48.141 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:46:48.141 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:46:48.141 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:46:48.141 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:46:48.141 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:46:48.141 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:46:48.141 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:46:48.141 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:46:48.141 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:46:48.141 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:46:48.141 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:46:48.141 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:46:48.141 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:46:48.141 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:46:48.141 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:46:48.141 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:46:48.141 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:46:48.141 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:46:48.141 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:46:48.141 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:46:48.141 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:46:48.141 | INFO | ------------------------------------------ | 20260507 06:46:48.141 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:46:48.141 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:46:48.141 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:46:48.141 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:46:48.141 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.34e+01 / 1.06e+00 | 20260507 06:46:48.141 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.02e-01, 9.93e-03 / 3.48e-03 | 20260507 06:46:48.141 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.47e+00, 5.12e-01 / 1.74e-01 | 20260507 06:46:48.141 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.48e+00, 1.68e-01 / 9.49e-02 | 20260507 06:46:48.141 | INFO | PerfReport: 16.22%, DME -- 3.98e+01 / 4.46e+01, 4.01e+00 / 1.63e+00 | 20260507 06:46:48.141 | INFO | PerfReport: 57.71%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.30e+00 / 1.50e+00 | 20260507 06:46:48.141 | INFO | PerfReport: 6.91%, PulseAccumulating -- 2.12e+00 / 2.68e+00, 3.85e-01 / 1.97e-01 | 20260507 06:46:48.141 | INFO | PerfReport: 2.94%, Metrics -- 9.04e-01 / 1.21e+00, 2.41e-01 / 1.24e-01 | 20260507 06:46:48.141 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.57e-01, 5.02e-02 / 1.93e-02 | 20260507 06:46:48.141 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:46:48.141 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:46:48.141 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:46:48.141 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 06:46:48.141 | INFO | PerfReport: 21.85%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260507 06:46:48.141 | INFO | PerfReport: 21.36%, Download -- 2.05e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260507 06:46:48.141 | INFO | PerfReport: 31.96%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:46:48.141 | INFO | PerfReport: 1.59%, LockUpload -- 1.53e+00 / 0.00e+00, 4.96e+00 / 0.00e+00 | 20260507 06:46:48.141 | INFO | PerfReport: 1.19%, LockDownload -- 1.15e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260507 06:46:48.141 | INFO | PerfReport: 22.04%, LockCompute -- 2.12e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 06:46:48.141 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.86e-02 / 0.00e+00 | 20260507 06:46:48.237 | INFO | Starting a round of Baz IO | 20260507 06:46:51.331 | INFO | Populating RTMetrics report | 20260507 06:46:51.331 | INFO | RTMetrics report populated | 20260507 06:46:51.331 | INFO | Opening tmp RTMetrics json file | 20260507 06:46:51.349 | INFO | Serializing RTMetrics to json | 20260507 06:46:51.350 | INFO | Serialization done, swapping tmp file now... | 20260507 06:46:51.351 | INFO | RTMetrics done | 20260507 06:46:51.418 | INFO | Finished a round of Baz IO: Wrote 1.658 GiB in 3.181 seconds | 20260507 06:46:51.418 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.60% / 0.00% / 67.40% | 20260507 06:46:51.418 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.11605% / 7.94729e-06% / 8.74201e-05% | 20260507 06:46:51.418 | INFO | Baz IO overall: | 20260507 06:46:51.418 | INFO | BytesWritten : 3987099678037 | 20260507 06:46:51.418 | INFO | NumEvents : 1910109584848 | 20260507 06:46:53.274 | INFO | Analyzing chunk frames = [7921664,7922176) | 20260507 06:46:53.274 | INFO | MemInfo: { | 20260507 06:46:53.274 | INFO | "gpu" : " 17 GB", | 20260507 06:46:53.274 | INFO | "huge" : "", | 20260507 06:46:53.274 | INFO | "huge_shared" : " 27 GB", | 20260507 06:46:53.274 | INFO | "malloc" : " 14 GB", | 20260507 06:46:53.274 | INFO | "pinned" : " 29 GB" | 20260507 06:46:53.274 | INFO | } | 20260507 06:46:53.274 | INFO | Perf report for the last 5.13447 seconds: | 20260507 06:46:53.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:46:53.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:53.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:53.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:53.274 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 06:46:53.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:53.274 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:46:53.274 | INFO | PRE_HQ : 59.1%, 1.8, 54.8ms, 0 | 20260507 06:46:53.275 | INFO | BAZWRITER : 61.5%, 3.1, 98.0ms, 0 | 20260507 06:46:53.275 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:46:53.275 | INFO | Uploaded 36GiB to GPU in 2.04123s (17.6364GiB/s) | 20260507 06:46:53.275 | INFO | Downloaded 38GiB to GPU in 2.16963s (17.5145GiB/s) | 20260507 06:46:58.426 | INFO | Analyzing chunk frames = [7922176,7922688) | 20260507 06:46:58.426 | INFO | MemInfo: { | 20260507 06:46:58.426 | INFO | "gpu" : " 17 GB", | 20260507 06:46:58.426 | INFO | "huge" : "", | 20260507 06:46:58.426 | INFO | "huge_shared" : " 27 GB", | 20260507 06:46:58.426 | INFO | "malloc" : " 14 GB", | 20260507 06:46:58.426 | INFO | "pinned" : " 29 GB" | 20260507 06:46:58.426 | INFO | } | 20260507 06:46:58.426 | INFO | Perf report for the last 5.1515 seconds: | 20260507 06:46:58.426 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:46:58.426 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:58.426 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:58.426 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:58.426 | INFO | ANALYSIS : 60.8%, 3.0, 94.9ms, 0 | 20260507 06:46:58.426 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:58.426 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:58.426 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 06:46:58.426 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:58.426 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:46:58.426 | INFO | Uploaded 36GiB to GPU in 2.04839s (17.5748GiB/s) | 20260507 06:46:58.426 | INFO | Downloaded 34GiB to GPU in 1.94598s (17.4719GiB/s) | 20260507 06:47:03.564 | INFO | Analyzing chunk frames = [7922688,7923200) | 20260507 06:47:03.564 | INFO | MemInfo: { | 20260507 06:47:03.564 | INFO | "gpu" : " 17 GB", | 20260507 06:47:03.564 | INFO | "huge" : "", | 20260507 06:47:03.564 | INFO | "huge_shared" : " 27 GB", | 20260507 06:47:03.564 | INFO | "malloc" : " 14 GB", | 20260507 06:47:03.564 | INFO | "pinned" : " 29 GB" | 20260507 06:47:03.564 | INFO | } | 20260507 06:47:03.564 | INFO | Perf report for the last 5.13864 seconds: | 20260507 06:47:03.564 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:47:03.564 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:03.564 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:03.564 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:03.565 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:47:03.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:03.565 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:03.565 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:47:03.565 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:03.565 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:03.565 | INFO | Uploaded 36GiB to GPU in 2.02914s (17.7415GiB/s) | 20260507 06:47:03.565 | INFO | Downloaded 34GiB to GPU in 1.91979s (17.7102GiB/s) | 20260507 06:47:08.692 | INFO | Analyzing chunk frames = [7923200,7923712) | 20260507 06:47:08.692 | INFO | MemInfo: { | 20260507 06:47:08.692 | INFO | "gpu" : " 17 GB", | 20260507 06:47:08.692 | INFO | "huge" : "", | 20260507 06:47:08.692 | INFO | "huge_shared" : " 27 GB", | 20260507 06:47:08.692 | INFO | "malloc" : " 14 GB", | 20260507 06:47:08.692 | INFO | "pinned" : " 29 GB" | 20260507 06:47:08.692 | INFO | } | 20260507 06:47:08.692 | INFO | Perf report for the last 5.12791 seconds: | 20260507 06:47:08.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:47:08.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:08.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:08.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:08.693 | INFO | ANALYSIS : 60.6%, 2.9, 93.3ms, 0 | 20260507 06:47:08.693 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:08.693 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:08.693 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 06:47:08.693 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:08.693 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:08.693 | INFO | Uploaded 36GiB to GPU in 2.04663s (17.5899GiB/s) | 20260507 06:47:08.693 | INFO | Downloaded 34GiB to GPU in 1.94817s (17.4523GiB/s) | 20260507 06:47:13.834 | INFO | Analyzing chunk frames = [7923712,7924224) | 20260507 06:47:13.835 | INFO | MemInfo: { | 20260507 06:47:13.835 | INFO | "gpu" : " 17 GB", | 20260507 06:47:13.835 | INFO | "huge" : "", | 20260507 06:47:13.835 | INFO | "huge_shared" : " 27 GB", | 20260507 06:47:13.835 | INFO | "malloc" : " 14 GB", | 20260507 06:47:13.835 | INFO | "pinned" : " 29 GB" | 20260507 06:47:13.835 | INFO | } | 20260507 06:47:13.835 | INFO | Perf report for the last 5.14236 seconds: | 20260507 06:47:13.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:47:13.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:13.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:13.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:13.835 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 06:47:13.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:13.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:13.835 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:47:13.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:13.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:13.835 | INFO | Uploaded 36GiB to GPU in 2.02868s (17.7455GiB/s) | 20260507 06:47:13.835 | INFO | Downloaded 34GiB to GPU in 1.94222s (17.5058GiB/s) | 20260507 06:47:18.981 | INFO | Analyzing chunk frames = [7924224,7924736) | 20260507 06:47:18.981 | INFO | MemInfo: { | 20260507 06:47:18.981 | INFO | "gpu" : " 17 GB", | 20260507 06:47:18.981 | INFO | "huge" : "", | 20260507 06:47:18.981 | INFO | "huge_shared" : " 27 GB", | 20260507 06:47:18.981 | INFO | "malloc" : " 14 GB", | 20260507 06:47:18.981 | INFO | "pinned" : " 29 GB" | 20260507 06:47:18.981 | INFO | } | 20260507 06:47:18.981 | INFO | Perf report for the last 5.1464 seconds: | 20260507 06:47:18.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:47:18.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:18.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:18.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:18.981 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 06:47:18.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:18.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:18.981 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 06:47:18.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:18.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:18.981 | INFO | Uploaded 36GiB to GPU in 2.02332s (17.7925GiB/s) | 20260507 06:47:18.981 | INFO | Downloaded 34GiB to GPU in 1.91287s (17.7744GiB/s) | 20260507 06:47:24.114 | INFO | Analyzing chunk frames = [7924736,7925248) | 20260507 06:47:24.114 | INFO | MemInfo: { | 20260507 06:47:24.114 | INFO | "gpu" : " 17 GB", | 20260507 06:47:24.114 | INFO | "huge" : "", | 20260507 06:47:24.114 | INFO | "huge_shared" : " 27 GB", | 20260507 06:47:24.114 | INFO | "malloc" : " 14 GB", | 20260507 06:47:24.114 | INFO | "pinned" : " 29 GB" | 20260507 06:47:24.114 | INFO | } | 20260507 06:47:24.114 | INFO | Perf report for the last 5.13279 seconds: | 20260507 06:47:24.114 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:47:24.114 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:24.114 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:24.114 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:24.114 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 06:47:24.114 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:24.114 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:24.114 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 06:47:24.114 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:24.114 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:24.114 | INFO | Uploaded 36GiB to GPU in 1.91603s (18.7888GiB/s) | 20260507 06:47:24.114 | INFO | Downloaded 34GiB to GPU in 1.97863s (17.1836GiB/s) | 20260507 06:47:29.249 | INFO | Analyzing chunk frames = [7925248,7925760) | 20260507 06:47:29.249 | INFO | MemInfo: { | 20260507 06:47:29.249 | INFO | "gpu" : " 17 GB", | 20260507 06:47:29.249 | INFO | "huge" : "", | 20260507 06:47:29.249 | INFO | "huge_shared" : " 27 GB", | 20260507 06:47:29.249 | INFO | "malloc" : " 14 GB", | 20260507 06:47:29.249 | INFO | "pinned" : " 29 GB" | 20260507 06:47:29.249 | INFO | } | 20260507 06:47:29.249 | INFO | Perf report for the last 5.13542 seconds: | 20260507 06:47:29.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:47:29.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:29.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:29.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:29.249 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:47:29.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:29.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:29.250 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 06:47:29.250 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:29.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:29.250 | INFO | Uploaded 36GiB to GPU in 1.99289s (18.0642GiB/s) | 20260507 06:47:29.250 | INFO | Downloaded 34GiB to GPU in 1.96204s (17.3289GiB/s) | 20260507 06:47:29.346 | INFO | Starting a round of Baz IO | 20260507 06:47:32.441 | INFO | Populating RTMetrics report | 20260507 06:47:32.441 | INFO | RTMetrics report populated | 20260507 06:47:32.441 | INFO | Opening tmp RTMetrics json file | 20260507 06:47:32.446 | INFO | Serializing RTMetrics to json | 20260507 06:47:32.458 | INFO | Serialization done, swapping tmp file now... | 20260507 06:47:32.459 | INFO | RTMetrics done | 20260507 06:47:32.529 | INFO | Finished a round of Baz IO: Wrote 1.658 GiB in 3.182 seconds | 20260507 06:47:32.529 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.60% / 0.00% / 67.40% | 20260507 06:47:32.529 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.116324% / 3.97364e-06% / 7.15256e-05% | 20260507 06:47:32.529 | INFO | Baz IO overall: | 20260507 06:47:32.529 | INFO | BytesWritten : 3988880243435 | 20260507 06:47:32.529 | INFO | NumEvents : 1910941945747 | 20260507 06:47:34.393 | INFO | Analyzing chunk frames = [7925760,7926272) | 20260507 06:47:34.393 | INFO | MemInfo: { | 20260507 06:47:34.393 | INFO | "gpu" : " 17 GB", | 20260507 06:47:34.393 | INFO | "huge" : "", | 20260507 06:47:34.393 | INFO | "huge_shared" : " 27 GB", | 20260507 06:47:34.393 | INFO | "malloc" : " 14 GB", | 20260507 06:47:34.393 | INFO | "pinned" : " 29 GB" | 20260507 06:47:34.393 | INFO | } | 20260507 06:47:34.393 | INFO | Perf report for the last 5.14374 seconds: | 20260507 06:47:34.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:47:34.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:34.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:34.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:34.393 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 06:47:34.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:34.393 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 06:47:34.393 | INFO | PRE_HQ : 59.0%, 1.8, 55.0ms, 0 | 20260507 06:47:34.393 | INFO | BAZWRITER : 61.5%, 3.1, 98.2ms, 0 | 20260507 06:47:34.393 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:47:34.394 | INFO | Uploaded 36GiB to GPU in 2.05827s (17.4905GiB/s) | 20260507 06:47:34.394 | INFO | Downloaded 38GiB to GPU in 2.17599s (17.4633GiB/s) | 20260507 06:47:39.534 | INFO | Analyzing chunk frames = [7926272,7926784) | 20260507 06:47:39.534 | INFO | MemInfo: { | 20260507 06:47:39.534 | INFO | "gpu" : " 17 GB", | 20260507 06:47:39.534 | INFO | "huge" : "", | 20260507 06:47:39.534 | INFO | "huge_shared" : " 27 GB", | 20260507 06:47:39.534 | INFO | "malloc" : " 14 GB", | 20260507 06:47:39.534 | INFO | "pinned" : " 29 GB" | 20260507 06:47:39.534 | INFO | } | 20260507 06:47:39.534 | INFO | Perf report for the last 5.14117 seconds: | 20260507 06:47:39.534 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:47:39.534 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:39.534 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:39.534 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:39.534 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 06:47:39.534 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:39.534 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:39.534 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:47:39.534 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:39.534 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:39.535 | INFO | Uploaded 36GiB to GPU in 2.01153s (17.8969GiB/s) | 20260507 06:47:39.535 | INFO | Downloaded 34GiB to GPU in 1.93427s (17.5777GiB/s) | 20260507 06:47:44.677 | INFO | Analyzing chunk frames = [7926784,7927296) | 20260507 06:47:44.677 | INFO | MemInfo: { | 20260507 06:47:44.677 | INFO | "gpu" : " 17 GB", | 20260507 06:47:44.677 | INFO | "huge" : "", | 20260507 06:47:44.677 | INFO | "huge_shared" : " 27 GB", | 20260507 06:47:44.678 | INFO | "malloc" : " 14 GB", | 20260507 06:47:44.678 | INFO | "pinned" : " 29 GB" | 20260507 06:47:44.678 | INFO | } | 20260507 06:47:44.678 | INFO | Perf report for the last 5.14335 seconds: | 20260507 06:47:44.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:47:44.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:44.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:44.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:44.678 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:47:44.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:44.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:44.678 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:47:44.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:44.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:44.678 | INFO | Uploaded 36GiB to GPU in 2.05187s (17.5449GiB/s) | 20260507 06:47:44.678 | INFO | Downloaded 34GiB to GPU in 1.95179s (17.4199GiB/s) | 20260507 06:47:49.812 | INFO | Analyzing chunk frames = [7927296,7927808) | 20260507 06:47:49.812 | INFO | MemInfo: { | 20260507 06:47:49.812 | INFO | "gpu" : " 17 GB", | 20260507 06:47:49.812 | INFO | "huge" : "", | 20260507 06:47:49.812 | INFO | "huge_shared" : " 27 GB", | 20260507 06:47:49.813 | INFO | "malloc" : " 14 GB", | 20260507 06:47:49.813 | INFO | "pinned" : " 29 GB" | 20260507 06:47:49.813 | INFO | } | 20260507 06:47:49.813 | INFO | Perf report for the last 5.13494 seconds: | 20260507 06:47:49.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:47:49.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:49.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:49.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:49.813 | INFO | ANALYSIS : 60.6%, 2.9, 93.4ms, 0 | 20260507 06:47:49.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:49.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:49.813 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:47:49.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:49.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:49.813 | INFO | Uploaded 36GiB to GPU in 2.05582s (17.5113GiB/s) | 20260507 06:47:49.813 | INFO | Downloaded 34GiB to GPU in 1.94949s (17.4405GiB/s) | 20260507 06:47:54.947 | INFO | Analyzing chunk frames = [7927808,7928320) | 20260507 06:47:54.947 | INFO | MemInfo: { | 20260507 06:47:54.947 | INFO | "gpu" : " 17 GB", | 20260507 06:47:54.947 | INFO | "huge" : "", | 20260507 06:47:54.947 | INFO | "huge_shared" : " 27 GB", | 20260507 06:47:54.947 | INFO | "malloc" : " 14 GB", | 20260507 06:47:54.947 | INFO | "pinned" : " 29 GB" | 20260507 06:47:54.947 | INFO | } | 20260507 06:47:54.947 | INFO | Perf report for the last 5.13422 seconds: | 20260507 06:47:54.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:47:54.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:54.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:54.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:54.947 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 06:47:54.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:54.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:54.947 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:47:54.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:54.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:47:54.947 | INFO | Uploaded 36GiB to GPU in 2.05729s (17.4987GiB/s) | 20260507 06:47:54.947 | INFO | Downloaded 34GiB to GPU in 1.94744s (17.4589GiB/s) | 20260507 06:48:00.092 | INFO | Analyzing chunk frames = [7928320,7928832) | 20260507 06:48:00.092 | INFO | MemInfo: { | 20260507 06:48:00.092 | INFO | "gpu" : " 17 GB", | 20260507 06:48:00.092 | INFO | "huge" : "", | 20260507 06:48:00.092 | INFO | "huge_shared" : " 27 GB", | 20260507 06:48:00.092 | INFO | "malloc" : " 14 GB", | 20260507 06:48:00.092 | INFO | "pinned" : " 29 GB" | 20260507 06:48:00.092 | INFO | } | 20260507 06:48:00.092 | INFO | Perf report for the last 5.14548 seconds: | 20260507 06:48:00.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:48:00.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:00.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:00.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:00.092 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:48:00.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:00.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:00.092 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 06:48:00.093 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:00.093 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:00.093 | INFO | Uploaded 36GiB to GPU in 2.04845s (17.5742GiB/s) | 20260507 06:48:00.093 | INFO | Downloaded 34GiB to GPU in 1.91937s (17.7142GiB/s) | 20260507 06:48:05.229 | INFO | Analyzing chunk frames = [7928832,7929344) | 20260507 06:48:05.229 | INFO | MemInfo: { | 20260507 06:48:05.229 | INFO | "gpu" : " 17 GB", | 20260507 06:48:05.229 | INFO | "huge" : "", | 20260507 06:48:05.229 | INFO | "huge_shared" : " 27 GB", | 20260507 06:48:05.229 | INFO | "malloc" : " 14 GB", | 20260507 06:48:05.229 | INFO | "pinned" : " 29 GB" | 20260507 06:48:05.229 | INFO | } | 20260507 06:48:05.229 | INFO | Perf report for the last 5.13644 seconds: | 20260507 06:48:05.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:48:05.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:05.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:05.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:05.229 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 06:48:05.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:05.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:05.229 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:48:05.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:05.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:05.229 | INFO | Uploaded 36GiB to GPU in 1.99613s (18.0349GiB/s) | 20260507 06:48:05.229 | INFO | Downloaded 34GiB to GPU in 1.97317s (17.2311GiB/s) | 20260507 06:48:10.376 | INFO | Analyzing chunk frames = [7929344,7929856) | 20260507 06:48:10.376 | INFO | MemInfo: { | 20260507 06:48:10.376 | INFO | "gpu" : " 17 GB", | 20260507 06:48:10.376 | INFO | "huge" : "", | 20260507 06:48:10.376 | INFO | "huge_shared" : " 27 GB", | 20260507 06:48:10.376 | INFO | "malloc" : " 14 GB", | 20260507 06:48:10.376 | INFO | "pinned" : " 29 GB" | 20260507 06:48:10.376 | INFO | } | 20260507 06:48:10.377 | INFO | Perf report for the last 5.1477 seconds: | 20260507 06:48:10.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:48:10.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:10.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:10.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:10.377 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:48:10.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:10.377 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:10.377 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 06:48:10.377 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:10.377 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:10.377 | INFO | Uploaded 36GiB to GPU in 1.99373s (18.0566GiB/s) | 20260507 06:48:10.377 | INFO | Downloaded 34GiB to GPU in 1.96228s (17.3268GiB/s) | 20260507 06:48:10.377 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:48:10.377 | INFO | Total mem usage: 18 GiB | 20260507 06:48:10.377 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:48:10.377 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -27.5%) | 20260507 06:48:10.377 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:48:10.377 | INFO | High Water Breakdown: | 20260507 06:48:10.377 | INFO | 60% : 8.47 GiB ( 74.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:48:10.377 | INFO | 39.9% : 5.63 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:48:10.377 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:48:10.377 | INFO | ------------------------------------------ | 20260507 06:48:10.377 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:48:10.377 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:48:10.377 | INFO | Total mem usage: 26.5 GiB | 20260507 06:48:10.377 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:48:10.377 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:48:10.377 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:48:10.377 | INFO | High Water Breakdown: | 20260507 06:48:10.377 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:48:10.377 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:48:10.377 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:48:10.377 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:48:10.377 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:48:10.377 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:48:10.377 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:48:10.377 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:48:10.377 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:48:10.378 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:48:10.378 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:48:10.378 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:48:10.378 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:48:10.378 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:48:10.378 | INFO | ------------------------------------------ | 20260507 06:48:10.378 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:48:10.378 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:48:10.378 | INFO | Total mem usage: 25.6 GiB | 20260507 06:48:10.378 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:48:10.378 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:48:10.378 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:48:10.378 | INFO | High Water Breakdown: | 20260507 06:48:10.378 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:48:10.378 | INFO | ------------------------------------------ | 20260507 06:48:10.378 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:48:10.378 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:48:10.378 | INFO | Total mem usage: 15.7 GiB | 20260507 06:48:10.378 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:48:10.378 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:48:10.378 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:48:10.378 | INFO | High Water Breakdown: | 20260507 06:48:10.378 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:48:10.378 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:48:10.378 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:48:10.378 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:48:10.378 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:48:10.378 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:48:10.378 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:48:10.378 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:48:10.378 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:48:10.378 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:48:10.378 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:48:10.378 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:48:10.378 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:48:10.378 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:48:10.378 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:48:10.378 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:48:10.378 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:48:10.378 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:48:10.378 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:48:10.378 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:48:10.378 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:48:10.378 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:48:10.378 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:48:10.378 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:48:10.378 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:48:10.378 | INFO | ------------------------------------------ | 20260507 06:48:10.378 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:48:10.378 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:48:10.378 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:48:10.378 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:48:10.378 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.50e+01, 1.34e+01 / 1.10e+00 | 20260507 06:48:10.378 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.00e-01, 1.11e-02 / 3.34e-03 | 20260507 06:48:10.378 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.48e+00, 5.00e-01 / 1.97e-01 | 20260507 06:48:10.378 | INFO | PerfReport: 3.21%, Histogram -- 9.85e-01 / 1.48e+00, 1.63e-01 / 1.14e-01 | 20260507 06:48:10.378 | INFO | PerfReport: 16.34%, DME -- 4.01e+01 / 4.52e+01, 4.05e+00 / 1.41e+00 | 20260507 06:48:10.378 | INFO | PerfReport: 57.61%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.18e+00 / 1.08e+00 | 20260507 06:48:10.378 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.12e+00 / 2.72e+00, 3.87e-01 / 1.66e-01 | 20260507 06:48:10.378 | INFO | PerfReport: 2.95%, Metrics -- 9.05e-01 / 1.22e+00, 2.42e-01 / 1.15e-01 | 20260507 06:48:10.378 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.56e-01, 5.00e-02 / 1.76e-02 | 20260507 06:48:10.378 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:48:10.378 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:48:10.378 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:48:10.378 | INFO | PerfReport: Full Execution -- 9.63e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260507 06:48:10.378 | INFO | PerfReport: 21.91%, Upload -- 2.11e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 06:48:10.378 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 06:48:10.378 | INFO | PerfReport: 31.85%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:48:10.378 | INFO | PerfReport: 1.65%, LockUpload -- 1.58e+00 / 0.00e+00, 5.02e+00 / 0.00e+00 | 20260507 06:48:10.378 | INFO | PerfReport: 1.24%, LockDownload -- 1.20e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260507 06:48:10.378 | INFO | PerfReport: 21.97%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 06:48:10.378 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260507 06:48:10.474 | INFO | Starting a round of Baz IO | 20260507 06:48:13.564 | INFO | Populating RTMetrics report | 20260507 06:48:13.565 | INFO | RTMetrics report populated | 20260507 06:48:13.565 | INFO | Opening tmp RTMetrics json file | 20260507 06:48:13.565 | INFO | Serializing RTMetrics to json | 20260507 06:48:13.567 | INFO | Serialization done, swapping tmp file now... | 20260507 06:48:13.567 | INFO | RTMetrics done | 20260507 06:48:13.649 | INFO | Finished a round of Baz IO: Wrote 1.658 GiB in 3.174 seconds | 20260507 06:48:13.649 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.60% / 0.00% / 67.40% | 20260507 06:48:13.649 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.115808% / 0% / 7.94729e-05% | 20260507 06:48:13.649 | INFO | Baz IO overall: | 20260507 06:48:13.649 | INFO | BytesWritten : 3990660512021 | 20260507 06:48:13.649 | INFO | NumEvents : 1911774246653 | 20260507 06:48:15.516 | INFO | Analyzing chunk frames = [7929856,7930368) | 20260507 06:48:15.516 | INFO | MemInfo: { | 20260507 06:48:15.516 | INFO | "gpu" : " 17 GB", | 20260507 06:48:15.516 | INFO | "huge" : "", | 20260507 06:48:15.516 | INFO | "huge_shared" : " 27 GB", | 20260507 06:48:15.516 | INFO | "malloc" : " 14 GB", | 20260507 06:48:15.516 | INFO | "pinned" : " 29 GB" | 20260507 06:48:15.516 | INFO | } | 20260507 06:48:15.516 | INFO | Perf report for the last 5.13924 seconds: | 20260507 06:48:15.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:48:15.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:15.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:15.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:15.516 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260507 06:48:15.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:15.516 | INFO | RT_METRICS : 44.1%, 1.0, 23.1ms, 0 | 20260507 06:48:15.516 | INFO | PRE_HQ : 59.8%, 1.9, 59.9ms, 0 | 20260507 06:48:15.516 | INFO | BAZWRITER : 61.4%, 3.3, 103.6ms, 0 | 20260507 06:48:15.516 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:48:15.516 | INFO | Uploaded 36GiB to GPU in 2.06249s (17.4547GiB/s) | 20260507 06:48:15.516 | INFO | Downloaded 38GiB to GPU in 2.17059s (17.5068GiB/s) | 20260507 06:48:20.647 | INFO | Analyzing chunk frames = [7930368,7930880) | 20260507 06:48:20.647 | INFO | MemInfo: { | 20260507 06:48:20.647 | INFO | "gpu" : " 17 GB", | 20260507 06:48:20.647 | INFO | "huge" : "", | 20260507 06:48:20.647 | INFO | "huge_shared" : " 27 GB", | 20260507 06:48:20.647 | INFO | "malloc" : " 14 GB", | 20260507 06:48:20.647 | INFO | "pinned" : " 29 GB" | 20260507 06:48:20.647 | INFO | } | 20260507 06:48:20.647 | INFO | Perf report for the last 5.13095 seconds: | 20260507 06:48:20.647 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:48:20.647 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:20.647 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:20.647 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:20.647 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260507 06:48:20.647 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:20.647 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:20.647 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:48:20.647 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:20.647 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:20.647 | INFO | Uploaded 36GiB to GPU in 2.00414s (17.9628GiB/s) | 20260507 06:48:20.647 | INFO | Downloaded 34GiB to GPU in 1.94147s (17.5125GiB/s) | 20260507 06:48:25.784 | INFO | Analyzing chunk frames = [7930880,7931392) | 20260507 06:48:25.784 | INFO | MemInfo: { | 20260507 06:48:25.784 | INFO | "gpu" : " 17 GB", | 20260507 06:48:25.784 | INFO | "huge" : "", | 20260507 06:48:25.784 | INFO | "huge_shared" : " 27 GB", | 20260507 06:48:25.784 | INFO | "malloc" : " 14 GB", | 20260507 06:48:25.784 | INFO | "pinned" : " 29 GB" | 20260507 06:48:25.784 | INFO | } | 20260507 06:48:25.784 | INFO | Perf report for the last 5.13773 seconds: | 20260507 06:48:25.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:48:25.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:25.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:25.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:25.785 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:48:25.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:25.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:25.785 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:48:25.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:25.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:25.785 | INFO | Uploaded 36GiB to GPU in 2.03589s (17.6827GiB/s) | 20260507 06:48:25.785 | INFO | Downloaded 34GiB to GPU in 1.95506s (17.3907GiB/s) | 20260507 06:48:30.930 | INFO | Analyzing chunk frames = [7931392,7931904) | 20260507 06:48:30.930 | INFO | MemInfo: { | 20260507 06:48:30.930 | INFO | "gpu" : " 17 GB", | 20260507 06:48:30.930 | INFO | "huge" : "", | 20260507 06:48:30.930 | INFO | "huge_shared" : " 27 GB", | 20260507 06:48:30.930 | INFO | "malloc" : " 14 GB", | 20260507 06:48:30.930 | INFO | "pinned" : " 29 GB" | 20260507 06:48:30.930 | INFO | } | 20260507 06:48:30.930 | INFO | Perf report for the last 5.14585 seconds: | 20260507 06:48:30.930 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:48:30.930 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:30.930 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:30.930 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:30.930 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 06:48:30.930 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:30.930 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:30.930 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:48:30.930 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:30.930 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:30.931 | INFO | Uploaded 36GiB to GPU in 2.00796s (17.9287GiB/s) | 20260507 06:48:30.931 | INFO | Downloaded 34GiB to GPU in 1.93387s (17.5813GiB/s) | 20260507 06:48:36.064 | INFO | Analyzing chunk frames = [7931904,7932416) | 20260507 06:48:36.064 | INFO | MemInfo: { | 20260507 06:48:36.064 | INFO | "gpu" : " 17 GB", | 20260507 06:48:36.064 | INFO | "huge" : "", | 20260507 06:48:36.064 | INFO | "huge_shared" : " 27 GB", | 20260507 06:48:36.064 | INFO | "malloc" : " 14 GB", | 20260507 06:48:36.064 | INFO | "pinned" : " 29 GB" | 20260507 06:48:36.064 | INFO | } | 20260507 06:48:36.064 | INFO | Perf report for the last 5.13359 seconds: | 20260507 06:48:36.064 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:48:36.064 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:36.064 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:36.064 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:36.064 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 06:48:36.064 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:36.064 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:36.064 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:48:36.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:36.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:36.064 | INFO | Uploaded 36GiB to GPU in 2.03448s (17.695GiB/s) | 20260507 06:48:36.064 | INFO | Downloaded 34GiB to GPU in 1.91777s (17.7289GiB/s) | 20260507 06:48:41.199 | INFO | Analyzing chunk frames = [7932416,7932928) | 20260507 06:48:41.199 | INFO | MemInfo: { | 20260507 06:48:41.199 | INFO | "gpu" : " 17 GB", | 20260507 06:48:41.199 | INFO | "huge" : "", | 20260507 06:48:41.199 | INFO | "huge_shared" : " 27 GB", | 20260507 06:48:41.199 | INFO | "malloc" : " 14 GB", | 20260507 06:48:41.199 | INFO | "pinned" : " 29 GB" | 20260507 06:48:41.199 | INFO | } | 20260507 06:48:41.199 | INFO | Perf report for the last 5.13537 seconds: | 20260507 06:48:41.199 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:48:41.199 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:41.199 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:41.199 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:41.199 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 06:48:41.199 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:41.199 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:41.199 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:48:41.199 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:41.199 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:41.200 | INFO | Uploaded 36GiB to GPU in 2.03781s (17.666GiB/s) | 20260507 06:48:41.200 | INFO | Downloaded 34GiB to GPU in 1.90945s (17.8062GiB/s) | 20260507 06:48:46.344 | INFO | Analyzing chunk frames = [7932928,7933440) | 20260507 06:48:46.344 | INFO | MemInfo: { | 20260507 06:48:46.344 | INFO | "gpu" : " 17 GB", | 20260507 06:48:46.344 | INFO | "huge" : "", | 20260507 06:48:46.344 | INFO | "huge_shared" : " 27 GB", | 20260507 06:48:46.344 | INFO | "malloc" : " 14 GB", | 20260507 06:48:46.344 | INFO | "pinned" : " 29 GB" | 20260507 06:48:46.344 | INFO | } | 20260507 06:48:46.344 | INFO | Perf report for the last 5.14503 seconds: | 20260507 06:48:46.344 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:48:46.344 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:46.344 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:46.344 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:46.344 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:48:46.344 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:46.344 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:46.344 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 06:48:46.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:46.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:46.345 | INFO | Uploaded 36GiB to GPU in 1.97642s (18.2148GiB/s) | 20260507 06:48:46.345 | INFO | Downloaded 34GiB to GPU in 1.9805s (17.1674GiB/s) | 20260507 06:48:51.488 | INFO | Analyzing chunk frames = [7933440,7933952) | 20260507 06:48:51.488 | INFO | MemInfo: { | 20260507 06:48:51.488 | INFO | "gpu" : " 17 GB", | 20260507 06:48:51.488 | INFO | "huge" : "", | 20260507 06:48:51.488 | INFO | "huge_shared" : " 27 GB", | 20260507 06:48:51.488 | INFO | "malloc" : " 14 GB", | 20260507 06:48:51.488 | INFO | "pinned" : " 29 GB" | 20260507 06:48:51.488 | INFO | } | 20260507 06:48:51.488 | INFO | Perf report for the last 5.14346 seconds: | 20260507 06:48:51.488 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:48:51.488 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:51.488 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:51.488 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:51.488 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 06:48:51.488 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:51.488 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:51.488 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 06:48:51.488 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:51.488 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:51.488 | INFO | Uploaded 36GiB to GPU in 1.97694s (18.21GiB/s) | 20260507 06:48:51.488 | INFO | Downloaded 34GiB to GPU in 1.96327s (17.3181GiB/s) | 20260507 06:48:51.587 | INFO | Starting a round of Baz IO | 20260507 06:48:54.677 | INFO | Populating RTMetrics report | 20260507 06:48:54.678 | INFO | RTMetrics report populated | 20260507 06:48:54.678 | INFO | Opening tmp RTMetrics json file | 20260507 06:48:54.685 | INFO | Serializing RTMetrics to json | 20260507 06:48:54.693 | INFO | Serialization done, swapping tmp file now... | 20260507 06:48:54.694 | INFO | RTMetrics done | 20260507 06:48:54.765 | INFO | Finished a round of Baz IO: Wrote 1.659 GiB in 3.177 seconds | 20260507 06:48:54.765 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.60% / 0.00% / 67.40% | 20260507 06:48:54.765 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.11611% / 3.97364e-06% / 6.75519e-05% | 20260507 06:48:54.765 | INFO | Baz IO overall: | 20260507 06:48:54.765 | INFO | BytesWritten : 3992441722187 | 20260507 06:48:54.765 | INFO | NumEvents : 1912607042445 | 20260507 06:48:56.625 | INFO | Analyzing chunk frames = [7933952,7934464) | 20260507 06:48:56.625 | INFO | MemInfo: { | 20260507 06:48:56.625 | INFO | "gpu" : " 17 GB", | 20260507 06:48:56.625 | INFO | "huge" : "", | 20260507 06:48:56.625 | INFO | "huge_shared" : " 27 GB", | 20260507 06:48:56.625 | INFO | "malloc" : " 14 GB", | 20260507 06:48:56.625 | INFO | "pinned" : " 29 GB" | 20260507 06:48:56.625 | INFO | } | 20260507 06:48:56.626 | INFO | Perf report for the last 5.13763 seconds: | 20260507 06:48:56.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:48:56.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:56.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:56.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:56.626 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 06:48:56.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:48:56.626 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 06:48:56.626 | INFO | PRE_HQ : 59.2%, 1.8, 55.4ms, 0 | 20260507 06:48:56.626 | INFO | BAZWRITER : 61.5%, 3.1, 99.1ms, 0 | 20260507 06:48:56.626 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:48:56.626 | INFO | Uploaded 36GiB to GPU in 2.06507s (17.4328GiB/s) | 20260507 06:48:56.626 | INFO | Downloaded 38GiB to GPU in 2.17982s (17.4327GiB/s) | 20260507 06:49:01.759 | INFO | Analyzing chunk frames = [7934464,7934976) | 20260507 06:49:01.759 | INFO | MemInfo: { | 20260507 06:49:01.759 | INFO | "gpu" : " 17 GB", | 20260507 06:49:01.759 | INFO | "huge" : "", | 20260507 06:49:01.759 | INFO | "huge_shared" : " 27 GB", | 20260507 06:49:01.759 | INFO | "malloc" : " 14 GB", | 20260507 06:49:01.759 | INFO | "pinned" : " 29 GB" | 20260507 06:49:01.759 | INFO | } | 20260507 06:49:01.759 | INFO | Perf report for the last 5.13375 seconds: | 20260507 06:49:01.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:49:01.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:01.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:01.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:01.759 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:49:01.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:01.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:01.759 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:49:01.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:01.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:01.760 | INFO | Uploaded 36GiB to GPU in 1.98574s (18.1292GiB/s) | 20260507 06:49:01.760 | INFO | Downloaded 34GiB to GPU in 1.93643s (17.558GiB/s) | 20260507 06:49:06.901 | INFO | Analyzing chunk frames = [7934976,7935488) | 20260507 06:49:06.901 | INFO | MemInfo: { | 20260507 06:49:06.901 | INFO | "gpu" : " 17 GB", | 20260507 06:49:06.901 | INFO | "huge" : "", | 20260507 06:49:06.901 | INFO | "huge_shared" : " 27 GB", | 20260507 06:49:06.901 | INFO | "malloc" : " 14 GB", | 20260507 06:49:06.901 | INFO | "pinned" : " 29 GB" | 20260507 06:49:06.901 | INFO | } | 20260507 06:49:06.901 | INFO | Perf report for the last 5.14187 seconds: | 20260507 06:49:06.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:49:06.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:06.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:06.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:06.901 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:49:06.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:06.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:06.901 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:49:06.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:06.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:06.902 | INFO | Uploaded 36GiB to GPU in 2.04419s (17.6109GiB/s) | 20260507 06:49:06.902 | INFO | Downloaded 34GiB to GPU in 1.95179s (17.4199GiB/s) | 20260507 06:49:12.041 | INFO | Analyzing chunk frames = [7935488,7936000) | 20260507 06:49:12.041 | INFO | MemInfo: { | 20260507 06:49:12.041 | INFO | "gpu" : " 17 GB", | 20260507 06:49:12.042 | INFO | "huge" : "", | 20260507 06:49:12.042 | INFO | "huge_shared" : " 27 GB", | 20260507 06:49:12.042 | INFO | "malloc" : " 14 GB", | 20260507 06:49:12.042 | INFO | "pinned" : " 29 GB" | 20260507 06:49:12.042 | INFO | } | 20260507 06:49:12.042 | INFO | Perf report for the last 5.14056 seconds: | 20260507 06:49:12.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:49:12.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:12.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:12.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:12.042 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 06:49:12.042 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:12.042 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:12.042 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:49:12.042 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:12.042 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:12.042 | INFO | Uploaded 36GiB to GPU in 1.99126s (18.079GiB/s) | 20260507 06:49:12.042 | INFO | Downloaded 34GiB to GPU in 1.93889s (17.5358GiB/s) | 20260507 06:49:17.183 | INFO | Analyzing chunk frames = [7936000,7936512) | 20260507 06:49:17.183 | INFO | MemInfo: { | 20260507 06:49:17.183 | INFO | "gpu" : " 17 GB", | 20260507 06:49:17.183 | INFO | "huge" : "", | 20260507 06:49:17.183 | INFO | "huge_shared" : " 27 GB", | 20260507 06:49:17.183 | INFO | "malloc" : " 14 GB", | 20260507 06:49:17.183 | INFO | "pinned" : " 29 GB" | 20260507 06:49:17.183 | INFO | } | 20260507 06:49:17.183 | INFO | Perf report for the last 5.14127 seconds: | 20260507 06:49:17.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:49:17.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:17.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:17.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:17.183 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 06:49:17.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:17.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:17.183 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:49:17.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:17.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:17.183 | INFO | Uploaded 36GiB to GPU in 2.03314s (17.7066GiB/s) | 20260507 06:49:17.183 | INFO | Downloaded 34GiB to GPU in 1.92251s (17.6852GiB/s) | 20260507 06:49:22.323 | INFO | Analyzing chunk frames = [7936512,7937024) | 20260507 06:49:22.323 | INFO | MemInfo: { | 20260507 06:49:22.323 | INFO | "gpu" : " 17 GB", | 20260507 06:49:22.323 | INFO | "huge" : "", | 20260507 06:49:22.323 | INFO | "huge_shared" : " 27 GB", | 20260507 06:49:22.324 | INFO | "malloc" : " 14 GB", | 20260507 06:49:22.324 | INFO | "pinned" : " 29 GB" | 20260507 06:49:22.324 | INFO | } | 20260507 06:49:22.324 | INFO | Perf report for the last 5.14072 seconds: | 20260507 06:49:22.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:49:22.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:22.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:22.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:22.324 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:49:22.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:22.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:22.324 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:49:22.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:22.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:22.324 | INFO | Uploaded 36GiB to GPU in 2.06326s (17.4481GiB/s) | 20260507 06:49:22.324 | INFO | Downloaded 34GiB to GPU in 1.92922s (17.6237GiB/s) | 20260507 06:49:27.456 | INFO | Analyzing chunk frames = [7937024,7937536) | 20260507 06:49:27.456 | INFO | MemInfo: { | 20260507 06:49:27.456 | INFO | "gpu" : " 17 GB", | 20260507 06:49:27.456 | INFO | "huge" : "", | 20260507 06:49:27.456 | INFO | "huge_shared" : " 27 GB", | 20260507 06:49:27.456 | INFO | "malloc" : " 14 GB", | 20260507 06:49:27.456 | INFO | "pinned" : " 29 GB" | 20260507 06:49:27.456 | INFO | } | 20260507 06:49:27.456 | INFO | Perf report for the last 5.13261 seconds: | 20260507 06:49:27.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:49:27.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:27.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:27.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:27.456 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 06:49:27.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:27.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:27.456 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 06:49:27.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:27.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:27.457 | INFO | Uploaded 36GiB to GPU in 1.88721s (19.0758GiB/s) | 20260507 06:49:27.457 | INFO | Downloaded 34GiB to GPU in 1.9776s (17.1926GiB/s) | 20260507 06:49:32.596 | INFO | Analyzing chunk frames = [7937536,7938048) | 20260507 06:49:32.596 | INFO | MemInfo: { | 20260507 06:49:32.596 | INFO | "gpu" : " 17 GB", | 20260507 06:49:32.596 | INFO | "huge" : "", | 20260507 06:49:32.596 | INFO | "huge_shared" : " 27 GB", | 20260507 06:49:32.596 | INFO | "malloc" : " 14 GB", | 20260507 06:49:32.596 | INFO | "pinned" : " 29 GB" | 20260507 06:49:32.596 | INFO | } | 20260507 06:49:32.596 | INFO | Perf report for the last 5.1399 seconds: | 20260507 06:49:32.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:49:32.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:32.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:32.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:32.596 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:49:32.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:32.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:32.596 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:49:32.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:32.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:32.597 | INFO | Uploaded 36GiB to GPU in 1.96953s (18.2784GiB/s) | 20260507 06:49:32.597 | INFO | Downloaded 34GiB to GPU in 1.96507s (17.3022GiB/s) | 20260507 06:49:32.597 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:49:32.597 | INFO | Total mem usage: 18 GiB | 20260507 06:49:32.597 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:49:32.597 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 06:49:32.597 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:49:32.597 | INFO | High Water Breakdown: | 20260507 06:49:32.597 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:49:32.597 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:49:32.597 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:49:32.597 | INFO | ------------------------------------------ | 20260507 06:49:32.597 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:49:32.597 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:49:32.597 | INFO | Total mem usage: 26.5 GiB | 20260507 06:49:32.597 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:49:32.597 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:49:32.597 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:49:32.597 | INFO | High Water Breakdown: | 20260507 06:49:32.597 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:49:32.597 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:49:32.597 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:49:32.597 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:49:32.597 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:49:32.597 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:49:32.597 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:49:32.597 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:49:32.597 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:49:32.597 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:49:32.597 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:49:32.597 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:49:32.597 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:49:32.597 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:49:32.597 | INFO | ------------------------------------------ | 20260507 06:49:32.597 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:49:32.597 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:49:32.597 | INFO | Total mem usage: 25.6 GiB | 20260507 06:49:32.597 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:49:32.597 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:49:32.597 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:49:32.597 | INFO | High Water Breakdown: | 20260507 06:49:32.597 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:49:32.597 | INFO | ------------------------------------------ | 20260507 06:49:32.597 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:49:32.597 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:49:32.597 | INFO | Total mem usage: 15.7 GiB | 20260507 06:49:32.597 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:49:32.597 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:49:32.597 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:49:32.597 | INFO | High Water Breakdown: | 20260507 06:49:32.597 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:49:32.597 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:49:32.597 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:49:32.597 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:49:32.597 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:49:32.597 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:49:32.597 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:49:32.597 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:49:32.597 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:49:32.597 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:49:32.597 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:49:32.597 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:49:32.597 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:49:32.597 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:49:32.597 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:49:32.597 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:49:32.597 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:49:32.597 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:49:32.597 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:49:32.597 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:49:32.597 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:49:32.597 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:49:32.597 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:49:32.597 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:49:32.597 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:49:32.597 | INFO | ------------------------------------------ | 20260507 06:49:32.597 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:49:32.597 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:49:32.597 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:49:32.597 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:49:32.597 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.44e+01, 1.33e+01 / 1.45e+00 | 20260507 06:49:32.597 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.02e-01, 9.68e-03 / 4.21e-03 | 20260507 06:49:32.597 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.46e+00, 5.08e-01 / 2.26e-01 | 20260507 06:49:32.597 | INFO | PerfReport: 3.24%, Histogram -- 9.93e-01 / 1.48e+00, 1.70e-01 / 1.24e-01 | 20260507 06:49:32.597 | INFO | PerfReport: 16.21%, DME -- 3.98e+01 / 4.44e+01, 3.88e+00 / 1.72e+00 | 20260507 06:49:32.597 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.77e+01 / 1.98e+01, 1.31e+00 / 1.43e+00 | 20260507 06:49:32.598 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.12e+00 / 2.69e+00, 3.86e-01 / 1.89e-01 | 20260507 06:49:32.598 | INFO | PerfReport: 2.95%, Metrics -- 9.07e-01 / 1.22e+00, 2.44e-01 / 1.24e-01 | 20260507 06:49:32.598 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.05e-01 / 3.61e-01, 5.02e-02 / 1.90e-02 | 20260507 06:49:32.598 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:49:32.598 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:49:32.598 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:49:32.598 | INFO | PerfReport: Full Execution -- 9.60e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 06:49:32.598 | INFO | PerfReport: 21.81%, Upload -- 2.09e+01 / 0.00e+00, 7.07e+00 / 0.00e+00 | 20260507 06:49:32.598 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 06:49:32.598 | INFO | PerfReport: 31.96%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 06:49:32.598 | INFO | PerfReport: 1.48%, LockUpload -- 1.42e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 06:49:32.598 | INFO | PerfReport: 1.34%, LockDownload -- 1.29e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260507 06:49:32.598 | INFO | PerfReport: 21.98%, LockCompute -- 2.11e+01 / 0.00e+00, 1.81e+01 / 0.00e+00 | 20260507 06:49:32.598 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.99e-02 / 0.00e+00 | 20260507 06:49:32.693 | INFO | Starting a round of Baz IO | 20260507 06:49:35.789 | INFO | Populating RTMetrics report | 20260507 06:49:35.790 | INFO | RTMetrics report populated | 20260507 06:49:35.790 | INFO | Opening tmp RTMetrics json file | 20260507 06:49:35.808 | INFO | Serializing RTMetrics to json | 20260507 06:49:35.809 | INFO | Serialization done, swapping tmp file now... | 20260507 06:49:35.809 | INFO | RTMetrics done | 20260507 06:49:35.877 | INFO | Finished a round of Baz IO: Wrote 1.659 GiB in 3.183 seconds | 20260507 06:49:35.877 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.60% / 0.00% / 67.40% | 20260507 06:49:35.877 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.116074% / 3.97364e-06% / 7.54992e-05% | 20260507 06:49:35.877 | INFO | Baz IO overall: | 20260507 06:49:35.877 | INFO | BytesWritten : 3994223257419 | 20260507 06:49:35.877 | INFO | NumEvents : 1913440544387 | 20260507 06:49:37.739 | INFO | Analyzing chunk frames = [7938048,7938560) | 20260507 06:49:37.739 | INFO | MemInfo: { | 20260507 06:49:37.739 | INFO | "gpu" : " 17 GB", | 20260507 06:49:37.739 | INFO | "huge" : "", | 20260507 06:49:37.739 | INFO | "huge_shared" : " 27 GB", | 20260507 06:49:37.739 | INFO | "malloc" : " 14 GB", | 20260507 06:49:37.739 | INFO | "pinned" : " 29 GB" | 20260507 06:49:37.739 | INFO | } | 20260507 06:49:37.739 | INFO | Perf report for the last 5.14271 seconds: | 20260507 06:49:37.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:49:37.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:37.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:37.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:37.739 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 06:49:37.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:37.739 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 06:49:37.739 | INFO | PRE_HQ : 58.8%, 1.8, 54.7ms, 0 | 20260507 06:49:37.739 | INFO | BAZWRITER : 61.5%, 3.1, 97.9ms, 0 | 20260507 06:49:37.739 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:49:37.739 | INFO | Uploaded 36GiB to GPU in 2.04011s (17.6461GiB/s) | 20260507 06:49:37.740 | INFO | Downloaded 38GiB to GPU in 2.18281s (17.4088GiB/s) | 20260507 06:49:42.878 | INFO | Analyzing chunk frames = [7938560,7939072) | 20260507 06:49:42.878 | INFO | MemInfo: { | 20260507 06:49:42.878 | INFO | "gpu" : " 17 GB", | 20260507 06:49:42.878 | INFO | "huge" : "", | 20260507 06:49:42.878 | INFO | "huge_shared" : " 27 GB", | 20260507 06:49:42.878 | INFO | "malloc" : " 14 GB", | 20260507 06:49:42.878 | INFO | "pinned" : " 29 GB" | 20260507 06:49:42.878 | INFO | } | 20260507 06:49:42.878 | INFO | Perf report for the last 5.13948 seconds: | 20260507 06:49:42.879 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:49:42.879 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:42.879 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:42.879 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:42.879 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:49:42.879 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:42.879 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:42.879 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:49:42.879 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:42.879 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:42.879 | INFO | Uploaded 36GiB to GPU in 1.98532s (18.1331GiB/s) | 20260507 06:49:42.879 | INFO | Downloaded 34GiB to GPU in 1.9329s (17.5902GiB/s) | 20260507 06:49:48.014 | INFO | Analyzing chunk frames = [7939072,7939584) | 20260507 06:49:48.014 | INFO | MemInfo: { | 20260507 06:49:48.014 | INFO | "gpu" : " 17 GB", | 20260507 06:49:48.014 | INFO | "huge" : "", | 20260507 06:49:48.014 | INFO | "huge_shared" : " 27 GB", | 20260507 06:49:48.014 | INFO | "malloc" : " 14 GB", | 20260507 06:49:48.014 | INFO | "pinned" : " 29 GB" | 20260507 06:49:48.014 | INFO | } | 20260507 06:49:48.015 | INFO | Perf report for the last 5.13607 seconds: | 20260507 06:49:48.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:49:48.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:48.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:48.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:48.015 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 06:49:48.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:48.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:48.015 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:49:48.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:48.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:48.015 | INFO | Uploaded 36GiB to GPU in 2.02247s (17.8GiB/s) | 20260507 06:49:48.015 | INFO | Downloaded 34GiB to GPU in 1.92122s (17.6971GiB/s) | 20260507 06:49:53.153 | INFO | Analyzing chunk frames = [7939584,7940096) | 20260507 06:49:53.153 | INFO | MemInfo: { | 20260507 06:49:53.153 | INFO | "gpu" : " 17 GB", | 20260507 06:49:53.153 | INFO | "huge" : "", | 20260507 06:49:53.153 | INFO | "huge_shared" : " 27 GB", | 20260507 06:49:53.153 | INFO | "malloc" : " 14 GB", | 20260507 06:49:53.153 | INFO | "pinned" : " 29 GB" | 20260507 06:49:53.153 | INFO | } | 20260507 06:49:53.153 | INFO | Perf report for the last 5.13859 seconds: | 20260507 06:49:53.153 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:49:53.153 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:53.153 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:53.153 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:53.153 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 06:49:53.153 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:53.153 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:53.153 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:49:53.153 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:53.153 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:53.153 | INFO | Uploaded 36GiB to GPU in 2.00042s (17.9962GiB/s) | 20260507 06:49:53.153 | INFO | Downloaded 34GiB to GPU in 1.94499s (17.4808GiB/s) | 20260507 06:49:58.293 | INFO | Analyzing chunk frames = [7940096,7940608) | 20260507 06:49:58.293 | INFO | MemInfo: { | 20260507 06:49:58.293 | INFO | "gpu" : " 17 GB", | 20260507 06:49:58.293 | INFO | "huge" : "", | 20260507 06:49:58.293 | INFO | "huge_shared" : " 27 GB", | 20260507 06:49:58.293 | INFO | "malloc" : " 14 GB", | 20260507 06:49:58.293 | INFO | "pinned" : " 29 GB" | 20260507 06:49:58.293 | INFO | } | 20260507 06:49:58.294 | INFO | Perf report for the last 5.14034 seconds: | 20260507 06:49:58.294 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:49:58.294 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:58.294 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:58.294 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:58.294 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 06:49:58.294 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:58.294 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:58.294 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:49:58.294 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:58.294 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:49:58.294 | INFO | Uploaded 36GiB to GPU in 2.04341s (17.6176GiB/s) | 20260507 06:49:58.294 | INFO | Downloaded 34GiB to GPU in 1.94836s (17.4505GiB/s) | 20260507 06:50:03.434 | INFO | Analyzing chunk frames = [7940608,7941120) | 20260507 06:50:03.434 | INFO | MemInfo: { | 20260507 06:50:03.434 | INFO | "gpu" : " 17 GB", | 20260507 06:50:03.434 | INFO | "huge" : "", | 20260507 06:50:03.434 | INFO | "huge_shared" : " 27 GB", | 20260507 06:50:03.434 | INFO | "malloc" : " 14 GB", | 20260507 06:50:03.434 | INFO | "pinned" : " 29 GB" | 20260507 06:50:03.434 | INFO | } | 20260507 06:50:03.434 | INFO | Perf report for the last 5.14068 seconds: | 20260507 06:50:03.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:50:03.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:03.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:03.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:03.434 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 06:50:03.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:03.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:03.434 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 06:50:03.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:03.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:03.435 | INFO | Uploaded 36GiB to GPU in 2.02549s (17.7735GiB/s) | 20260507 06:50:03.435 | INFO | Downloaded 34GiB to GPU in 1.91491s (17.7554GiB/s) | 20260507 06:50:08.572 | INFO | Analyzing chunk frames = [7941120,7941632) | 20260507 06:50:08.572 | INFO | MemInfo: { | 20260507 06:50:08.572 | INFO | "gpu" : " 17 GB", | 20260507 06:50:08.572 | INFO | "huge" : "", | 20260507 06:50:08.572 | INFO | "huge_shared" : " 27 GB", | 20260507 06:50:08.572 | INFO | "malloc" : " 14 GB", | 20260507 06:50:08.572 | INFO | "pinned" : " 29 GB" | 20260507 06:50:08.572 | INFO | } | 20260507 06:50:08.572 | INFO | Perf report for the last 5.13797 seconds: | 20260507 06:50:08.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:50:08.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:08.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:08.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:08.572 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:50:08.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:08.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:08.572 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:50:08.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:08.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:08.572 | INFO | Uploaded 36GiB to GPU in 1.8849s (19.0991GiB/s) | 20260507 06:50:08.572 | INFO | Downloaded 34GiB to GPU in 1.97261s (17.236GiB/s) | 20260507 06:50:13.711 | INFO | Analyzing chunk frames = [7941632,7942144) | 20260507 06:50:13.711 | INFO | MemInfo: { | 20260507 06:50:13.711 | INFO | "gpu" : " 17 GB", | 20260507 06:50:13.711 | INFO | "huge" : "", | 20260507 06:50:13.711 | INFO | "huge_shared" : " 27 GB", | 20260507 06:50:13.711 | INFO | "malloc" : " 14 GB", | 20260507 06:50:13.711 | INFO | "pinned" : " 29 GB" | 20260507 06:50:13.711 | INFO | } | 20260507 06:50:13.711 | INFO | Perf report for the last 5.13901 seconds: | 20260507 06:50:13.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:50:13.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:13.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:13.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:13.711 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:50:13.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:13.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:13.711 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 06:50:13.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:13.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:13.712 | INFO | Uploaded 36GiB to GPU in 1.98554s (18.1311GiB/s) | 20260507 06:50:13.712 | INFO | Downloaded 34GiB to GPU in 1.96039s (17.3435GiB/s) | 20260507 06:50:13.809 | INFO | Starting a round of Baz IO | 20260507 06:50:16.902 | INFO | Populating RTMetrics report | 20260507 06:50:16.902 | INFO | RTMetrics report populated | 20260507 06:50:16.902 | INFO | Opening tmp RTMetrics json file | 20260507 06:50:16.912 | INFO | Serializing RTMetrics to json | 20260507 06:50:16.918 | INFO | Serialization done, swapping tmp file now... | 20260507 06:50:16.919 | INFO | RTMetrics done | 20260507 06:50:16.991 | INFO | Finished a round of Baz IO: Wrote 1.656 GiB in 3.182 seconds | 20260507 06:50:16.991 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.60% / 0.00% / 67.40% | 20260507 06:50:16.991 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.115486% / 1.19209e-05% / 9.13938e-05% | 20260507 06:50:16.991 | INFO | Baz IO overall: | 20260507 06:50:16.991 | INFO | BytesWritten : 3996001049627 | 20260507 06:50:16.991 | INFO | NumEvents : 1914271439461 | 20260507 06:50:18.855 | INFO | Analyzing chunk frames = [7942144,7942656) | 20260507 06:50:18.855 | INFO | MemInfo: { | 20260507 06:50:18.855 | INFO | "gpu" : " 17 GB", | 20260507 06:50:18.855 | INFO | "huge" : "", | 20260507 06:50:18.855 | INFO | "huge_shared" : " 27 GB", | 20260507 06:50:18.855 | INFO | "malloc" : " 14 GB", | 20260507 06:50:18.855 | INFO | "pinned" : " 29 GB" | 20260507 06:50:18.855 | INFO | } | 20260507 06:50:18.855 | INFO | Perf report for the last 5.14422 seconds: | 20260507 06:50:18.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:50:18.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:18.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:18.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:18.855 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 06:50:18.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:18.855 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 06:50:18.855 | INFO | PRE_HQ : 59.1%, 1.8, 55.1ms, 0 | 20260507 06:50:18.855 | INFO | BAZWRITER : 61.4%, 3.1, 99.0ms, 0 | 20260507 06:50:18.855 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:50:18.856 | INFO | Uploaded 36GiB to GPU in 2.04013s (17.6459GiB/s) | 20260507 06:50:18.856 | INFO | Downloaded 38GiB to GPU in 2.17772s (17.4494GiB/s) | 20260507 06:50:23.996 | INFO | Analyzing chunk frames = [7942656,7943168) | 20260507 06:50:23.997 | INFO | MemInfo: { | 20260507 06:50:23.997 | INFO | "gpu" : " 17 GB", | 20260507 06:50:23.997 | INFO | "huge" : "", | 20260507 06:50:23.997 | INFO | "huge_shared" : " 27 GB", | 20260507 06:50:23.997 | INFO | "malloc" : " 14 GB", | 20260507 06:50:23.997 | INFO | "pinned" : " 29 GB" | 20260507 06:50:23.997 | INFO | } | 20260507 06:50:23.997 | INFO | Perf report for the last 5.14137 seconds: | 20260507 06:50:23.997 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:50:23.997 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:23.997 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:23.997 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:23.997 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:50:23.997 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:23.997 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:23.997 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 06:50:23.997 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:23.997 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:23.997 | INFO | Uploaded 36GiB to GPU in 1.9831s (18.1534GiB/s) | 20260507 06:50:23.997 | INFO | Downloaded 34GiB to GPU in 1.93061s (17.611GiB/s) | 20260507 06:50:29.135 | INFO | Analyzing chunk frames = [7943168,7943680) | 20260507 06:50:29.135 | INFO | MemInfo: { | 20260507 06:50:29.135 | INFO | "gpu" : " 17 GB", | 20260507 06:50:29.135 | INFO | "huge" : "", | 20260507 06:50:29.135 | INFO | "huge_shared" : " 27 GB", | 20260507 06:50:29.135 | INFO | "malloc" : " 14 GB", | 20260507 06:50:29.135 | INFO | "pinned" : " 29 GB" | 20260507 06:50:29.135 | INFO | } | 20260507 06:50:29.135 | INFO | Perf report for the last 5.13835 seconds: | 20260507 06:50:29.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:50:29.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:29.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:29.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:29.135 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 06:50:29.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:29.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:29.135 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:50:29.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:29.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:29.135 | INFO | Uploaded 36GiB to GPU in 2.02891s (17.7435GiB/s) | 20260507 06:50:29.135 | INFO | Downloaded 34GiB to GPU in 1.91989s (17.7094GiB/s) | 20260507 06:50:34.268 | INFO | Analyzing chunk frames = [7943680,7944192) | 20260507 06:50:34.268 | INFO | MemInfo: { | 20260507 06:50:34.268 | INFO | "gpu" : " 17 GB", | 20260507 06:50:34.268 | INFO | "huge" : "", | 20260507 06:50:34.268 | INFO | "huge_shared" : " 27 GB", | 20260507 06:50:34.268 | INFO | "malloc" : " 14 GB", | 20260507 06:50:34.268 | INFO | "pinned" : " 29 GB" | 20260507 06:50:34.268 | INFO | } | 20260507 06:50:34.268 | INFO | Perf report for the last 5.13314 seconds: | 20260507 06:50:34.268 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:50:34.268 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:34.268 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:34.268 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:34.268 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 06:50:34.268 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:34.268 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:34.268 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:50:34.268 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:34.268 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:34.269 | INFO | Uploaded 36GiB to GPU in 2.04586s (17.5965GiB/s) | 20260507 06:50:34.269 | INFO | Downloaded 34GiB to GPU in 1.95095s (17.4274GiB/s) | 20260507 06:50:39.407 | INFO | Analyzing chunk frames = [7944192,7944704) | 20260507 06:50:39.407 | INFO | MemInfo: { | 20260507 06:50:39.407 | INFO | "gpu" : " 17 GB", | 20260507 06:50:39.407 | INFO | "huge" : "", | 20260507 06:50:39.407 | INFO | "huge_shared" : " 27 GB", | 20260507 06:50:39.407 | INFO | "malloc" : " 14 GB", | 20260507 06:50:39.407 | INFO | "pinned" : " 29 GB" | 20260507 06:50:39.407 | INFO | } | 20260507 06:50:39.407 | INFO | Perf report for the last 5.13853 seconds: | 20260507 06:50:39.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:50:39.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:39.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:39.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:39.407 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 06:50:39.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:39.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:39.407 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:50:39.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:39.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:39.407 | INFO | Uploaded 36GiB to GPU in 2.05124s (17.5503GiB/s) | 20260507 06:50:39.407 | INFO | Downloaded 34GiB to GPU in 1.95175s (17.4202GiB/s) | 20260507 06:50:44.548 | INFO | Analyzing chunk frames = [7944704,7945216) | 20260507 06:50:44.548 | INFO | MemInfo: { | 20260507 06:50:44.548 | INFO | "gpu" : " 17 GB", | 20260507 06:50:44.548 | INFO | "huge" : "", | 20260507 06:50:44.548 | INFO | "huge_shared" : " 27 GB", | 20260507 06:50:44.548 | INFO | "malloc" : " 14 GB", | 20260507 06:50:44.548 | INFO | "pinned" : " 29 GB" | 20260507 06:50:44.548 | INFO | } | 20260507 06:50:44.548 | INFO | Perf report for the last 5.14115 seconds: | 20260507 06:50:44.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:50:44.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:44.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:44.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:44.548 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 06:50:44.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:44.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:44.548 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:50:44.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:44.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:44.548 | INFO | Uploaded 36GiB to GPU in 2.02598s (17.7692GiB/s) | 20260507 06:50:44.548 | INFO | Downloaded 34GiB to GPU in 1.90352s (17.8617GiB/s) | 20260507 06:50:49.690 | INFO | Analyzing chunk frames = [7945216,7945728) | 20260507 06:50:49.690 | INFO | MemInfo: { | 20260507 06:50:49.690 | INFO | "gpu" : " 17 GB", | 20260507 06:50:49.690 | INFO | "huge" : "", | 20260507 06:50:49.690 | INFO | "huge_shared" : " 27 GB", | 20260507 06:50:49.690 | INFO | "malloc" : " 14 GB", | 20260507 06:50:49.690 | INFO | "pinned" : " 29 GB" | 20260507 06:50:49.690 | INFO | } | 20260507 06:50:49.690 | INFO | Perf report for the last 5.14198 seconds: | 20260507 06:50:49.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:50:49.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:49.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:49.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:49.690 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:50:49.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:49.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:49.690 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:50:49.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:49.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:49.690 | INFO | Uploaded 36GiB to GPU in 1.95258s (18.4371GiB/s) | 20260507 06:50:49.691 | INFO | Downloaded 34GiB to GPU in 1.97695s (17.1982GiB/s) | 20260507 06:50:54.829 | INFO | Analyzing chunk frames = [7945728,7946240) | 20260507 06:50:54.829 | INFO | MemInfo: { | 20260507 06:50:54.829 | INFO | "gpu" : " 17 GB", | 20260507 06:50:54.829 | INFO | "huge" : "", | 20260507 06:50:54.829 | INFO | "huge_shared" : " 27 GB", | 20260507 06:50:54.829 | INFO | "malloc" : " 14 GB", | 20260507 06:50:54.829 | INFO | "pinned" : " 29 GB" | 20260507 06:50:54.829 | INFO | } | 20260507 06:50:54.829 | INFO | Perf report for the last 5.13935 seconds: | 20260507 06:50:54.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:50:54.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:54.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:54.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:54.830 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 06:50:54.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:54.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:54.830 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:50:54.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:54.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:54.830 | INFO | Uploaded 36GiB to GPU in 1.97933s (18.188GiB/s) | 20260507 06:50:54.830 | INFO | Downloaded 34GiB to GPU in 1.96009s (17.3462GiB/s) | 20260507 06:50:54.830 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:50:54.830 | INFO | Total mem usage: 18 GiB | 20260507 06:50:54.830 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:50:54.830 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 06:50:54.830 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:50:54.830 | INFO | High Water Breakdown: | 20260507 06:50:54.830 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:50:54.830 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:50:54.830 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:50:54.830 | INFO | ------------------------------------------ | 20260507 06:50:54.830 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:50:54.830 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:50:54.830 | INFO | Total mem usage: 26.5 GiB | 20260507 06:50:54.830 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:50:54.830 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:50:54.830 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:50:54.830 | INFO | High Water Breakdown: | 20260507 06:50:54.830 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:50:54.830 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:50:54.830 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:50:54.830 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:50:54.830 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:50:54.830 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:50:54.830 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:50:54.830 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:50:54.830 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:50:54.830 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:50:54.830 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:50:54.830 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:50:54.830 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:50:54.831 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:50:54.831 | INFO | ------------------------------------------ | 20260507 06:50:54.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:50:54.831 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:50:54.831 | INFO | Total mem usage: 25.6 GiB | 20260507 06:50:54.831 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:50:54.831 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 06:50:54.831 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 06:50:54.831 | INFO | High Water Breakdown: | 20260507 06:50:54.831 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 06:50:54.831 | INFO | ------------------------------------------ | 20260507 06:50:54.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:50:54.831 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:50:54.831 | INFO | Total mem usage: 15.7 GiB | 20260507 06:50:54.831 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:50:54.831 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:50:54.831 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:50:54.831 | INFO | High Water Breakdown: | 20260507 06:50:54.831 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:50:54.831 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:50:54.831 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:50:54.831 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:50:54.831 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:50:54.831 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:50:54.831 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:50:54.831 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:50:54.831 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:50:54.831 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:50:54.831 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:50:54.831 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:50:54.831 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:50:54.831 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:50:54.831 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:50:54.831 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:50:54.831 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:50:54.831 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:50:54.831 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:50:54.831 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:50:54.831 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:50:54.831 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:50:54.831 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:50:54.831 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:50:54.831 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:50:54.831 | INFO | ------------------------------------------ | 20260507 06:50:54.831 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:50:54.831 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:50:54.831 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:50:54.831 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:50:54.831 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.30e+01 / 1.26e+00 | 20260507 06:50:54.831 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.01e-01, 1.10e-02 / 7.17e-03 | 20260507 06:50:54.831 | INFO | PerfReport: 11.70%, Baseline -- 3.59e+00 / 4.49e+00, 5.13e-01 / 2.49e-01 | 20260507 06:50:54.831 | INFO | PerfReport: 3.23%, Histogram -- 9.90e-01 / 1.49e+00, 1.66e-01 / 8.33e-02 | 20260507 06:50:54.831 | INFO | PerfReport: 16.26%, DME -- 3.99e+01 / 4.48e+01, 4.04e+00 / 1.41e+00 | 20260507 06:50:54.831 | INFO | PerfReport: 57.64%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.24e+00 / 1.03e+00 | 20260507 06:50:54.831 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.12e+00 / 2.68e+00, 4.17e-01 / 2.22e-01 | 20260507 06:50:54.831 | INFO | PerfReport: 2.95%, Metrics -- 9.05e-01 / 1.22e+00, 2.41e-01 / 1.28e-01 | 20260507 06:50:54.831 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.59e-01, 5.03e-02 / 2.13e-02 | 20260507 06:50:54.831 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:50:54.831 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:50:54.831 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:50:54.831 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 06:50:54.831 | INFO | PerfReport: 21.74%, Upload -- 2.09e+01 / 0.00e+00, 7.04e+00 / 0.00e+00 | 20260507 06:50:54.831 | INFO | PerfReport: 21.37%, Download -- 2.05e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 06:50:54.831 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.30e+01 / 0.00e+00 | 20260507 06:50:54.831 | INFO | PerfReport: 1.51%, LockUpload -- 1.45e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 06:50:54.831 | INFO | PerfReport: 1.40%, LockDownload -- 1.34e+00 / 0.00e+00, 3.23e+00 / 0.00e+00 | 20260507 06:50:54.831 | INFO | PerfReport: 22.04%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 06:50:54.831 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.37e-02 / 0.00e+00 | 20260507 06:50:54.927 | INFO | Starting a round of Baz IO | 20260507 06:50:58.014 | INFO | Populating RTMetrics report | 20260507 06:50:58.014 | INFO | RTMetrics report populated | 20260507 06:50:58.014 | INFO | Opening tmp RTMetrics json file | 20260507 06:50:58.030 | INFO | Serializing RTMetrics to json | 20260507 06:50:58.031 | INFO | Serialization done, swapping tmp file now... | 20260507 06:50:58.031 | INFO | RTMetrics done | 20260507 06:50:58.098 | INFO | Finished a round of Baz IO: Wrote 1.656 GiB in 3.171 seconds | 20260507 06:50:58.098 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.60% / 0.00% / 67.40% | 20260507 06:50:58.098 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.115204% / 3.97364e-06% / 6.35783e-05% | 20260507 06:50:58.099 | INFO | Baz IO overall: | 20260507 06:50:58.099 | INFO | BytesWritten : 3997779641301 | 20260507 06:50:58.099 | INFO | NumEvents : 1915103401539 | 20260507 06:50:59.963 | INFO | Analyzing chunk frames = [7946240,7946752) | 20260507 06:50:59.963 | INFO | MemInfo: { | 20260507 06:50:59.963 | INFO | "gpu" : " 17 GB", | 20260507 06:50:59.963 | INFO | "huge" : "", | 20260507 06:50:59.963 | INFO | "huge_shared" : " 27 GB", | 20260507 06:50:59.963 | INFO | "malloc" : " 14 GB", | 20260507 06:50:59.963 | INFO | "pinned" : " 29 GB" | 20260507 06:50:59.963 | INFO | } | 20260507 06:50:59.963 | INFO | Perf report for the last 5.13389 seconds: | 20260507 06:50:59.963 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:50:59.963 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:59.963 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:59.963 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:59.963 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260507 06:50:59.963 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:50:59.963 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 06:50:59.964 | INFO | PRE_HQ : 58.9%, 1.8, 55.2ms, 0 | 20260507 06:50:59.964 | INFO | BAZWRITER : 61.4%, 3.1, 98.5ms, 0 | 20260507 06:50:59.964 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:50:59.964 | INFO | Uploaded 36GiB to GPU in 2.07075s (17.385GiB/s) | 20260507 06:50:59.964 | INFO | Downloaded 38GiB to GPU in 2.17016s (17.5103GiB/s) | 20260507 06:51:05.103 | INFO | Analyzing chunk frames = [7946752,7947264) | 20260507 06:51:05.103 | INFO | MemInfo: { | 20260507 06:51:05.103 | INFO | "gpu" : " 17 GB", | 20260507 06:51:05.103 | INFO | "huge" : "", | 20260507 06:51:05.103 | INFO | "huge_shared" : " 27 GB", | 20260507 06:51:05.103 | INFO | "malloc" : " 14 GB", | 20260507 06:51:05.103 | INFO | "pinned" : " 29 GB" | 20260507 06:51:05.103 | INFO | } | 20260507 06:51:05.103 | INFO | Perf report for the last 5.14016 seconds: | 20260507 06:51:05.103 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:51:05.103 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:05.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:05.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:05.104 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 06:51:05.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:05.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:05.104 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:51:05.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:05.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:05.104 | INFO | Uploaded 36GiB to GPU in 2.00336s (17.9698GiB/s) | 20260507 06:51:05.104 | INFO | Downloaded 34GiB to GPU in 1.93412s (17.5791GiB/s) | 20260507 06:51:10.247 | INFO | Analyzing chunk frames = [7947264,7947776) | 20260507 06:51:10.247 | INFO | MemInfo: { | 20260507 06:51:10.247 | INFO | "gpu" : " 17 GB", | 20260507 06:51:10.247 | INFO | "huge" : "", | 20260507 06:51:10.247 | INFO | "huge_shared" : " 27 GB", | 20260507 06:51:10.247 | INFO | "malloc" : " 14 GB", | 20260507 06:51:10.247 | INFO | "pinned" : " 29 GB" | 20260507 06:51:10.247 | INFO | } | 20260507 06:51:10.247 | INFO | Perf report for the last 5.14376 seconds: | 20260507 06:51:10.247 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:51:10.247 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:10.247 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:10.247 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:10.247 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 06:51:10.247 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:10.248 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:10.248 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:51:10.248 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:10.248 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:10.248 | INFO | Uploaded 36GiB to GPU in 2.02187s (17.8053GiB/s) | 20260507 06:51:10.248 | INFO | Downloaded 34GiB to GPU in 1.93826s (17.5415GiB/s) | 20260507 06:51:15.385 | INFO | Analyzing chunk frames = [7947776,7948288) | 20260507 06:51:15.385 | INFO | MemInfo: { | 20260507 06:51:15.385 | INFO | "gpu" : " 17 GB", | 20260507 06:51:15.385 | INFO | "huge" : "", | 20260507 06:51:15.385 | INFO | "huge_shared" : " 27 GB", | 20260507 06:51:15.385 | INFO | "malloc" : " 14 GB", | 20260507 06:51:15.385 | INFO | "pinned" : " 29 GB" | 20260507 06:51:15.385 | INFO | } | 20260507 06:51:15.385 | INFO | Perf report for the last 5.13803 seconds: | 20260507 06:51:15.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:51:15.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:15.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:15.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:15.385 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 06:51:15.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:15.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:15.385 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:51:15.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:15.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:15.386 | INFO | Uploaded 36GiB to GPU in 2.04586s (17.5965GiB/s) | 20260507 06:51:15.386 | INFO | Downloaded 34GiB to GPU in 1.94832s (17.451GiB/s) | 20260507 06:51:20.524 | INFO | Analyzing chunk frames = [7948288,7948800) | 20260507 06:51:20.524 | INFO | MemInfo: { | 20260507 06:51:20.524 | INFO | "gpu" : " 17 GB", | 20260507 06:51:20.524 | INFO | "huge" : "", | 20260507 06:51:20.524 | INFO | "huge_shared" : " 27 GB", | 20260507 06:51:20.524 | INFO | "malloc" : " 14 GB", | 20260507 06:51:20.524 | INFO | "pinned" : " 29 GB" | 20260507 06:51:20.524 | INFO | } | 20260507 06:51:20.524 | INFO | Perf report for the last 5.13858 seconds: | 20260507 06:51:20.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:51:20.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:20.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:20.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:20.524 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 06:51:20.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:20.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:20.524 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:51:20.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:20.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:20.524 | INFO | Uploaded 36GiB to GPU in 2.05456s (17.522GiB/s) | 20260507 06:51:20.525 | INFO | Downloaded 34GiB to GPU in 1.90583s (17.84GiB/s) | 20260507 06:51:25.661 | INFO | Analyzing chunk frames = [7948800,7949312) | 20260507 06:51:25.661 | INFO | MemInfo: { | 20260507 06:51:25.661 | INFO | "gpu" : " 17 GB", | 20260507 06:51:25.661 | INFO | "huge" : "", | 20260507 06:51:25.661 | INFO | "huge_shared" : " 27 GB", | 20260507 06:51:25.661 | INFO | "malloc" : " 14 GB", | 20260507 06:51:25.661 | INFO | "pinned" : " 29 GB" | 20260507 06:51:25.661 | INFO | } | 20260507 06:51:25.661 | INFO | Perf report for the last 5.13751 seconds: | 20260507 06:51:25.661 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:51:25.661 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:25.661 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:25.661 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:25.661 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 06:51:25.661 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:25.661 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:25.662 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 06:51:25.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:25.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:25.662 | INFO | Uploaded 36GiB to GPU in 2.07263s (17.3692GiB/s) | 20260507 06:51:25.662 | INFO | Downloaded 34GiB to GPU in 1.9269s (17.6449GiB/s) | 20260507 06:51:30.809 | INFO | Analyzing chunk frames = [7949312,7949824) | 20260507 06:51:30.809 | INFO | MemInfo: { | 20260507 06:51:30.809 | INFO | "gpu" : " 17 GB", | 20260507 06:51:30.809 | INFO | "huge" : "", | 20260507 06:51:30.809 | INFO | "huge_shared" : " 27 GB", | 20260507 06:51:30.809 | INFO | "malloc" : " 14 GB", | 20260507 06:51:30.809 | INFO | "pinned" : " 29 GB" | 20260507 06:51:30.809 | INFO | } | 20260507 06:51:30.809 | INFO | Perf report for the last 5.14765 seconds: | 20260507 06:51:30.809 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:51:30.809 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:30.809 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:30.809 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:30.809 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:51:30.809 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:30.809 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:30.809 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 06:51:30.809 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:30.809 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:30.810 | INFO | Uploaded 36GiB to GPU in 1.96934s (18.2802GiB/s) | 20260507 06:51:30.810 | INFO | Downloaded 34GiB to GPU in 1.97889s (17.1813GiB/s) | 20260507 06:51:35.946 | INFO | Analyzing chunk frames = [7949824,7950336) | 20260507 06:51:35.946 | INFO | MemInfo: { | 20260507 06:51:35.946 | INFO | "gpu" : " 17 GB", | 20260507 06:51:35.946 | INFO | "huge" : "", | 20260507 06:51:35.946 | INFO | "huge_shared" : " 27 GB", | 20260507 06:51:35.946 | INFO | "malloc" : " 14 GB", | 20260507 06:51:35.946 | INFO | "pinned" : " 29 GB" | 20260507 06:51:35.947 | INFO | } | 20260507 06:51:35.947 | INFO | Perf report for the last 5.13756 seconds: | 20260507 06:51:35.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:51:35.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:35.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:35.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:35.947 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:51:35.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:35.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:35.947 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:51:35.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:35.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:35.947 | INFO | Uploaded 36GiB to GPU in 1.99067s (18.0844GiB/s) | 20260507 06:51:35.947 | INFO | Downloaded 34GiB to GPU in 1.95272s (17.4116GiB/s) | 20260507 06:51:36.044 | INFO | Starting a round of Baz IO | 20260507 06:51:39.167 | INFO | Populating RTMetrics report | 20260507 06:51:39.167 | INFO | RTMetrics report populated | 20260507 06:51:39.167 | INFO | Opening tmp RTMetrics json file | 20260507 06:51:39.168 | INFO | Serializing RTMetrics to json | 20260507 06:51:39.170 | INFO | Serialization done, swapping tmp file now... | 20260507 06:51:39.170 | INFO | RTMetrics done | 20260507 06:51:39.251 | INFO | Finished a round of Baz IO: Wrote 1.657 GiB in 3.207 seconds | 20260507 06:51:39.251 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.60% / 0.00% / 67.40% | 20260507 06:51:39.251 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.115069% / 0% / 5.5631e-05% | 20260507 06:51:39.251 | INFO | Baz IO overall: | 20260507 06:51:39.251 | INFO | BytesWritten : 3999558578233 | 20260507 06:51:39.251 | INFO | NumEvents : 1915935635002 | 20260507 06:51:41.090 | INFO | Analyzing chunk frames = [7950336,7950848) | 20260507 06:51:41.090 | INFO | MemInfo: { | 20260507 06:51:41.090 | INFO | "gpu" : " 17 GB", | 20260507 06:51:41.090 | INFO | "huge" : "", | 20260507 06:51:41.090 | INFO | "huge_shared" : " 27 GB", | 20260507 06:51:41.090 | INFO | "malloc" : " 14 GB", | 20260507 06:51:41.090 | INFO | "pinned" : " 29 GB" | 20260507 06:51:41.090 | INFO | } | 20260507 06:51:41.090 | INFO | Perf report for the last 5.14366 seconds: | 20260507 06:51:41.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:51:41.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:41.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:41.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:41.090 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 06:51:41.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:41.090 | INFO | RT_METRICS : 43.6%, 1.0, 22.9ms, 0 | 20260507 06:51:41.090 | INFO | PRE_HQ : 59.7%, 1.9, 58.7ms, 0 | 20260507 06:51:41.090 | INFO | BAZWRITER : 61.9%, 3.2, 102.8ms, 0 | 20260507 06:51:41.090 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260507 06:51:41.091 | INFO | Uploaded 36GiB to GPU in 2.07539s (17.3461GiB/s) | 20260507 06:51:41.091 | INFO | Downloaded 38GiB to GPU in 2.17458s (17.4746GiB/s) | 20260507 06:51:46.215 | INFO | Analyzing chunk frames = [7950848,7951360) | 20260507 06:51:46.215 | INFO | MemInfo: { | 20260507 06:51:46.215 | INFO | "gpu" : " 17 GB", | 20260507 06:51:46.215 | INFO | "huge" : "", | 20260507 06:51:46.215 | INFO | "huge_shared" : " 27 GB", | 20260507 06:51:46.215 | INFO | "malloc" : " 14 GB", | 20260507 06:51:46.215 | INFO | "pinned" : " 29 GB" | 20260507 06:51:46.215 | INFO | } | 20260507 06:51:46.215 | INFO | Perf report for the last 5.12475 seconds: | 20260507 06:51:46.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:51:46.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:46.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:46.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:46.215 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260507 06:51:46.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:46.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:46.215 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:51:46.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:46.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:46.215 | INFO | Uploaded 36GiB to GPU in 2.0008s (17.9928GiB/s) | 20260507 06:51:46.215 | INFO | Downloaded 34GiB to GPU in 1.94277s (17.5008GiB/s) | 20260507 06:51:51.358 | INFO | Analyzing chunk frames = [7951360,7951872) | 20260507 06:51:51.358 | INFO | MemInfo: { | 20260507 06:51:51.358 | INFO | "gpu" : " 17 GB", | 20260507 06:51:51.358 | INFO | "huge" : "", | 20260507 06:51:51.358 | INFO | "huge_shared" : " 27 GB", | 20260507 06:51:51.358 | INFO | "malloc" : " 14 GB", | 20260507 06:51:51.358 | INFO | "pinned" : " 29 GB" | 20260507 06:51:51.358 | INFO | } | 20260507 06:51:51.358 | INFO | Perf report for the last 5.14308 seconds: | 20260507 06:51:51.358 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:51:51.358 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:51.358 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:51.358 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:51.358 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260507 06:51:51.358 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:51.358 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:51.358 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:51:51.358 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:51.358 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:51.358 | INFO | Uploaded 36GiB to GPU in 2.01882s (17.8322GiB/s) | 20260507 06:51:51.358 | INFO | Downloaded 34GiB to GPU in 1.93682s (17.5546GiB/s) | 20260507 06:51:56.498 | INFO | Analyzing chunk frames = [7951872,7952384) | 20260507 06:51:56.498 | INFO | MemInfo: { | 20260507 06:51:56.498 | INFO | "gpu" : " 17 GB", | 20260507 06:51:56.498 | INFO | "huge" : "", | 20260507 06:51:56.498 | INFO | "huge_shared" : " 27 GB", | 20260507 06:51:56.498 | INFO | "malloc" : " 14 GB", | 20260507 06:51:56.499 | INFO | "pinned" : " 29 GB" | 20260507 06:51:56.499 | INFO | } | 20260507 06:51:56.499 | INFO | Perf report for the last 5.14055 seconds: | 20260507 06:51:56.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:51:56.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:56.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:56.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:56.499 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 06:51:56.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:56.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:56.499 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:51:56.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:56.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:51:56.499 | INFO | Uploaded 36GiB to GPU in 2.01456s (17.87GiB/s) | 20260507 06:51:56.499 | INFO | Downloaded 34GiB to GPU in 1.93373s (17.5826GiB/s) | 20260507 06:52:01.640 | INFO | Analyzing chunk frames = [7952384,7952896) | 20260507 06:52:01.640 | INFO | MemInfo: { | 20260507 06:52:01.640 | INFO | "gpu" : " 17 GB", | 20260507 06:52:01.640 | INFO | "huge" : "", | 20260507 06:52:01.640 | INFO | "huge_shared" : " 27 GB", | 20260507 06:52:01.640 | INFO | "malloc" : " 14 GB", | 20260507 06:52:01.640 | INFO | "pinned" : " 29 GB" | 20260507 06:52:01.641 | INFO | } | 20260507 06:52:01.641 | INFO | Perf report for the last 5.14215 seconds: | 20260507 06:52:01.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:52:01.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:01.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:01.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:01.641 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 06:52:01.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:01.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:01.641 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:52:01.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:01.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:01.641 | INFO | Uploaded 36GiB to GPU in 2.05873s (17.4865GiB/s) | 20260507 06:52:01.641 | INFO | Downloaded 34GiB to GPU in 1.95306s (17.4086GiB/s) | 20260507 06:52:06.777 | INFO | Analyzing chunk frames = [7952896,7953408) | 20260507 06:52:06.777 | INFO | MemInfo: { | 20260507 06:52:06.777 | INFO | "gpu" : " 17 GB", | 20260507 06:52:06.777 | INFO | "huge" : "", | 20260507 06:52:06.777 | INFO | "huge_shared" : " 27 GB", | 20260507 06:52:06.777 | INFO | "malloc" : " 14 GB", | 20260507 06:52:06.777 | INFO | "pinned" : " 29 GB" | 20260507 06:52:06.777 | INFO | } | 20260507 06:52:06.777 | INFO | Perf report for the last 5.13644 seconds: | 20260507 06:52:06.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:52:06.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:06.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:06.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:06.777 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 06:52:06.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:06.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:06.777 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:52:06.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:06.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:06.778 | INFO | Uploaded 36GiB to GPU in 2.07195s (17.3749GiB/s) | 20260507 06:52:06.778 | INFO | Downloaded 34GiB to GPU in 1.9369s (17.5538GiB/s) | 20260507 06:52:11.915 | INFO | Analyzing chunk frames = [7953408,7953920) | 20260507 06:52:11.915 | INFO | MemInfo: { | 20260507 06:52:11.915 | INFO | "gpu" : " 17 GB", | 20260507 06:52:11.915 | INFO | "huge" : "", | 20260507 06:52:11.915 | INFO | "huge_shared" : " 27 GB", | 20260507 06:52:11.915 | INFO | "malloc" : " 14 GB", | 20260507 06:52:11.915 | INFO | "pinned" : " 29 GB" | 20260507 06:52:11.915 | INFO | } | 20260507 06:52:11.915 | INFO | Perf report for the last 5.13783 seconds: | 20260507 06:52:11.915 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:52:11.915 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:11.915 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:11.915 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:11.915 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:52:11.915 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:11.915 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:11.915 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:52:11.915 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:11.915 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:11.915 | INFO | Uploaded 36GiB to GPU in 1.93105s (18.6427GiB/s) | 20260507 06:52:11.915 | INFO | Downloaded 34GiB to GPU in 1.97406s (17.2234GiB/s) | 20260507 06:52:17.060 | INFO | Analyzing chunk frames = [7953920,7954432) | 20260507 06:52:17.060 | INFO | MemInfo: { | 20260507 06:52:17.060 | INFO | "gpu" : " 17 GB", | 20260507 06:52:17.060 | INFO | "huge" : "", | 20260507 06:52:17.060 | INFO | "huge_shared" : " 27 GB", | 20260507 06:52:17.060 | INFO | "malloc" : " 14 GB", | 20260507 06:52:17.060 | INFO | "pinned" : " 29 GB" | 20260507 06:52:17.060 | INFO | } | 20260507 06:52:17.060 | INFO | Perf report for the last 5.14485 seconds: | 20260507 06:52:17.060 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:52:17.060 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:17.060 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:17.060 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:17.060 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:52:17.060 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:17.060 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:17.060 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 06:52:17.060 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:17.060 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:17.060 | INFO | Uploaded 36GiB to GPU in 1.95644s (18.4008GiB/s) | 20260507 06:52:17.060 | INFO | Downloaded 34GiB to GPU in 1.96843s (17.2727GiB/s) | 20260507 06:52:17.060 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:52:17.060 | INFO | Total mem usage: 18 GiB | 20260507 06:52:17.060 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:52:17.060 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 06:52:17.060 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:52:17.060 | INFO | High Water Breakdown: | 20260507 06:52:17.060 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:52:17.060 | INFO | 39.8% : 5.65 GiB ( 84.5% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:52:17.060 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:52:17.061 | INFO | ------------------------------------------ | 20260507 06:52:17.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:52:17.061 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:52:17.061 | INFO | Total mem usage: 26.5 GiB | 20260507 06:52:17.061 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:52:17.061 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 06:52:17.061 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 06:52:17.061 | INFO | High Water Breakdown: | 20260507 06:52:17.061 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:52:17.061 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:52:17.061 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:52:17.061 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:52:17.061 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:52:17.061 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:52:17.061 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:52:17.061 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:52:17.061 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:52:17.061 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:52:17.061 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:52:17.061 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:52:17.061 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:52:17.061 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:52:17.061 | INFO | ------------------------------------------ | 20260507 06:52:17.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:52:17.061 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:52:17.061 | INFO | Total mem usage: 25.6 GiB | 20260507 06:52:17.061 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:52:17.061 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:52:17.061 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:52:17.061 | INFO | High Water Breakdown: | 20260507 06:52:17.061 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:52:17.061 | INFO | ------------------------------------------ | 20260507 06:52:17.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:52:17.061 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:52:17.061 | INFO | Total mem usage: 15.7 GiB | 20260507 06:52:17.061 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:52:17.061 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:52:17.061 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:52:17.061 | INFO | High Water Breakdown: | 20260507 06:52:17.061 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:52:17.061 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:52:17.061 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:52:17.061 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:52:17.061 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:52:17.061 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:52:17.061 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:52:17.061 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:52:17.061 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:52:17.061 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:52:17.061 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:52:17.061 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:52:17.061 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:52:17.061 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:52:17.061 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:52:17.061 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:52:17.061 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:52:17.061 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:52:17.061 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:52:17.061 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:52:17.061 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:52:17.061 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:52:17.061 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:52:17.061 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:52:17.061 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:52:17.061 | INFO | ------------------------------------------ | 20260507 06:52:17.061 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:52:17.061 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:52:17.061 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:52:17.061 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:52:17.061 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.42e+01, 1.33e+01 / 1.41e+00 | 20260507 06:52:17.061 | INFO | PerfReport: 0.30%, AggregateStats -- 9.05e-02 / 1.01e-01, 9.34e-03 / 5.44e-03 | 20260507 06:52:17.061 | INFO | PerfReport: 11.66%, Baseline -- 3.57e+00 / 4.44e+00, 5.09e-01 / 2.80e-01 | 20260507 06:52:17.061 | INFO | PerfReport: 3.22%, Histogram -- 9.86e-01 / 1.45e+00, 1.63e-01 / 1.23e-01 | 20260507 06:52:17.061 | INFO | PerfReport: 16.19%, DME -- 3.97e+01 / 4.41e+01, 3.75e+00 / 1.56e+00 | 20260507 06:52:17.061 | INFO | PerfReport: 57.79%, FrameLabeling -- 1.77e+01 / 1.99e+01, 1.31e+00 / 1.15e+00 | 20260507 06:52:17.061 | INFO | PerfReport: 6.91%, PulseAccumulating -- 2.12e+00 / 2.67e+00, 3.84e-01 / 2.07e-01 | 20260507 06:52:17.061 | INFO | PerfReport: 2.95%, Metrics -- 9.04e-01 / 1.21e+00, 2.43e-01 / 1.24e-01 | 20260507 06:52:17.061 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.61e-01, 5.09e-02 / 2.36e-02 | 20260507 06:52:17.061 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:52:17.061 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:52:17.061 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:52:17.061 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 06:52:17.061 | INFO | PerfReport: 21.93%, Upload -- 2.11e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 06:52:17.061 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260507 06:52:17.061 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 06:52:17.061 | INFO | PerfReport: 1.58%, LockUpload -- 1.52e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 06:52:17.061 | INFO | PerfReport: 1.20%, LockDownload -- 1.15e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260507 06:52:17.062 | INFO | PerfReport: 21.96%, LockCompute -- 2.11e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 06:52:17.062 | INFO | PerfReport: 0.02%, Misc -- 1.45e-02 / 0.00e+00, 2.95e-02 / 0.00e+00 | 20260507 06:52:17.157 | INFO | Starting a round of Baz IO | 20260507 06:52:20.282 | INFO | Populating RTMetrics report | 20260507 06:52:20.283 | INFO | RTMetrics report populated | 20260507 06:52:20.283 | INFO | Opening tmp RTMetrics json file | 20260507 06:52:20.284 | INFO | Serializing RTMetrics to json | 20260507 06:52:20.285 | INFO | Serialization done, swapping tmp file now... | 20260507 06:52:20.285 | INFO | RTMetrics done | 20260507 06:52:20.370 | INFO | Finished a round of Baz IO: Wrote 1.656 GiB in 3.212 seconds | 20260507 06:52:20.370 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.60% / 0.00% / 67.40% | 20260507 06:52:20.370 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.114199% / 3.97364e-06% / 5.16574e-05% | 20260507 06:52:20.370 | INFO | Baz IO overall: | 20260507 06:52:20.370 | INFO | BytesWritten : 4001336849586 | 20260507 06:52:20.370 | INFO | NumEvents : 1916767349427 | 20260507 06:52:22.196 | INFO | Analyzing chunk frames = [7954432,7954944) | 20260507 06:52:22.196 | INFO | MemInfo: { | 20260507 06:52:22.196 | INFO | "gpu" : " 17 GB", | 20260507 06:52:22.196 | INFO | "huge" : "", | 20260507 06:52:22.196 | INFO | "huge_shared" : " 27 GB", | 20260507 06:52:22.196 | INFO | "malloc" : " 14 GB", | 20260507 06:52:22.196 | INFO | "pinned" : " 29 GB" | 20260507 06:52:22.196 | INFO | } | 20260507 06:52:22.196 | INFO | Perf report for the last 5.13634 seconds: | 20260507 06:52:22.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:52:22.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:22.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:22.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:22.196 | INFO | ANALYSIS : 61.6%, 3.0, 95.3ms, 0 | 20260507 06:52:22.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:22.196 | INFO | RT_METRICS : 44.6%, 1.0, 23.4ms, 0 | 20260507 06:52:22.196 | INFO | PRE_HQ : 59.7%, 1.9, 59.9ms, 0 | 20260507 06:52:22.197 | INFO | BAZWRITER : 62.1%, 3.3, 104.7ms, 0 | 20260507 06:52:22.197 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 06:52:22.197 | INFO | Uploaded 36GiB to GPU in 2.05976s (17.4778GiB/s) | 20260507 06:52:22.197 | INFO | Downloaded 38GiB to GPU in 2.17798s (17.4474GiB/s) | 20260507 06:52:27.335 | INFO | Analyzing chunk frames = [7954944,7955456) | 20260507 06:52:27.335 | INFO | MemInfo: { | 20260507 06:52:27.335 | INFO | "gpu" : " 17 GB", | 20260507 06:52:27.335 | INFO | "huge" : "", | 20260507 06:52:27.335 | INFO | "huge_shared" : " 27 GB", | 20260507 06:52:27.335 | INFO | "malloc" : " 14 GB", | 20260507 06:52:27.335 | INFO | "pinned" : " 29 GB" | 20260507 06:52:27.335 | INFO | } | 20260507 06:52:27.335 | INFO | Perf report for the last 5.13871 seconds: | 20260507 06:52:27.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:52:27.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:27.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:27.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:27.335 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:52:27.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:27.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:27.335 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 06:52:27.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:27.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:27.335 | INFO | Uploaded 36GiB to GPU in 1.97345s (18.2422GiB/s) | 20260507 06:52:27.335 | INFO | Downloaded 34GiB to GPU in 1.93184s (17.5998GiB/s) | 20260507 06:52:32.470 | INFO | Analyzing chunk frames = [7955456,7955968) | 20260507 06:52:32.470 | INFO | MemInfo: { | 20260507 06:52:32.470 | INFO | "gpu" : " 17 GB", | 20260507 06:52:32.470 | INFO | "huge" : "", | 20260507 06:52:32.470 | INFO | "huge_shared" : " 27 GB", | 20260507 06:52:32.470 | INFO | "malloc" : " 14 GB", | 20260507 06:52:32.470 | INFO | "pinned" : " 29 GB" | 20260507 06:52:32.470 | INFO | } | 20260507 06:52:32.470 | INFO | Perf report for the last 5.13468 seconds: | 20260507 06:52:32.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:52:32.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:32.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:32.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:32.470 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 06:52:32.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:32.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:32.470 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:52:32.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:32.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:32.470 | INFO | Uploaded 36GiB to GPU in 2.04583s (17.5968GiB/s) | 20260507 06:52:32.470 | INFO | Downloaded 34GiB to GPU in 1.95406s (17.3997GiB/s) | 20260507 06:52:37.615 | INFO | Analyzing chunk frames = [7955968,7956480) | 20260507 06:52:37.615 | INFO | MemInfo: { | 20260507 06:52:37.615 | INFO | "gpu" : " 17 GB", | 20260507 06:52:37.615 | INFO | "huge" : "", | 20260507 06:52:37.615 | INFO | "huge_shared" : " 27 GB", | 20260507 06:52:37.615 | INFO | "malloc" : " 14 GB", | 20260507 06:52:37.615 | INFO | "pinned" : " 29 GB" | 20260507 06:52:37.615 | INFO | } | 20260507 06:52:37.615 | INFO | Perf report for the last 5.14542 seconds: | 20260507 06:52:37.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:52:37.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:37.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:37.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:37.615 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 06:52:37.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:37.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:37.615 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:52:37.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:37.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:37.616 | INFO | Uploaded 36GiB to GPU in 1.98514s (18.1347GiB/s) | 20260507 06:52:37.616 | INFO | Downloaded 34GiB to GPU in 1.92422s (17.6695GiB/s) | 20260507 06:52:42.754 | INFO | Analyzing chunk frames = [7956480,7956992) | 20260507 06:52:42.754 | INFO | MemInfo: { | 20260507 06:52:42.754 | INFO | "gpu" : " 17 GB", | 20260507 06:52:42.754 | INFO | "huge" : "", | 20260507 06:52:42.754 | INFO | "huge_shared" : " 27 GB", | 20260507 06:52:42.754 | INFO | "malloc" : " 14 GB", | 20260507 06:52:42.754 | INFO | "pinned" : " 29 GB" | 20260507 06:52:42.754 | INFO | } | 20260507 06:52:42.754 | INFO | Perf report for the last 5.13867 seconds: | 20260507 06:52:42.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:52:42.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:42.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:42.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:42.754 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 06:52:42.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:42.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:42.754 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:52:42.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:42.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:42.754 | INFO | Uploaded 36GiB to GPU in 2.0509s (17.5533GiB/s) | 20260507 06:52:42.754 | INFO | Downloaded 34GiB to GPU in 1.95349s (17.4047GiB/s) | 20260507 06:52:47.895 | INFO | Analyzing chunk frames = [7956992,7957504) | 20260507 06:52:47.895 | INFO | MemInfo: { | 20260507 06:52:47.895 | INFO | "gpu" : " 17 GB", | 20260507 06:52:47.895 | INFO | "huge" : "", | 20260507 06:52:47.895 | INFO | "huge_shared" : " 27 GB", | 20260507 06:52:47.895 | INFO | "malloc" : " 14 GB", | 20260507 06:52:47.895 | INFO | "pinned" : " 29 GB" | 20260507 06:52:47.895 | INFO | } | 20260507 06:52:47.895 | INFO | Perf report for the last 5.14083 seconds: | 20260507 06:52:47.895 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:52:47.895 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:47.895 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:47.895 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:47.895 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 06:52:47.895 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:47.895 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:47.895 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 06:52:47.895 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:47.895 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:47.895 | INFO | Uploaded 36GiB to GPU in 2.04554s (17.5993GiB/s) | 20260507 06:52:47.895 | INFO | Downloaded 34GiB to GPU in 1.91067s (17.7948GiB/s) | 20260507 06:52:53.032 | INFO | Analyzing chunk frames = [7957504,7958016) | 20260507 06:52:53.033 | INFO | MemInfo: { | 20260507 06:52:53.033 | INFO | "gpu" : " 17 GB", | 20260507 06:52:53.033 | INFO | "huge" : "", | 20260507 06:52:53.033 | INFO | "huge_shared" : " 27 GB", | 20260507 06:52:53.033 | INFO | "malloc" : " 14 GB", | 20260507 06:52:53.033 | INFO | "pinned" : " 29 GB" | 20260507 06:52:53.033 | INFO | } | 20260507 06:52:53.033 | INFO | Perf report for the last 5.13802 seconds: | 20260507 06:52:53.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:52:53.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:53.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:53.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:53.033 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:52:53.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:53.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:53.033 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:52:53.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:53.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:53.033 | INFO | Uploaded 36GiB to GPU in 1.95869s (18.3797GiB/s) | 20260507 06:52:53.033 | INFO | Downloaded 34GiB to GPU in 1.98279s (17.1476GiB/s) | 20260507 06:52:58.170 | INFO | Analyzing chunk frames = [7958016,7958528) | 20260507 06:52:58.170 | INFO | MemInfo: { | 20260507 06:52:58.170 | INFO | "gpu" : " 17 GB", | 20260507 06:52:58.170 | INFO | "huge" : "", | 20260507 06:52:58.170 | INFO | "huge_shared" : " 27 GB", | 20260507 06:52:58.170 | INFO | "malloc" : " 14 GB", | 20260507 06:52:58.170 | INFO | "pinned" : " 29 GB" | 20260507 06:52:58.170 | INFO | } | 20260507 06:52:58.170 | INFO | Perf report for the last 5.13733 seconds: | 20260507 06:52:58.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:52:58.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:58.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:58.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:58.170 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 06:52:58.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:58.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:58.170 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 06:52:58.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:58.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:52:58.170 | INFO | Uploaded 36GiB to GPU in 1.97289s (18.2474GiB/s) | 20260507 06:52:58.170 | INFO | Downloaded 34GiB to GPU in 1.96101s (17.338GiB/s) | 20260507 06:52:58.267 | INFO | Starting a round of Baz IO | 20260507 06:53:01.382 | INFO | Populating RTMetrics report | 20260507 06:53:01.383 | INFO | RTMetrics report populated | 20260507 06:53:01.383 | INFO | Opening tmp RTMetrics json file | 20260507 06:53:01.383 | INFO | Serializing RTMetrics to json | 20260507 06:53:01.384 | INFO | Serialization done, swapping tmp file now... | 20260507 06:53:01.385 | INFO | RTMetrics done | 20260507 06:53:01.471 | INFO | Finished a round of Baz IO: Wrote 1.656 GiB in 3.203 seconds | 20260507 06:53:01.471 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.60% / 0.00% / 67.40% | 20260507 06:53:01.471 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.114644% / 0% / 5.96046e-05% | 20260507 06:53:01.471 | INFO | Baz IO overall: | 20260507 06:53:01.471 | INFO | BytesWritten : 4003115170242 | 20260507 06:53:01.471 | INFO | NumEvents : 1917598912993 | 20260507 06:53:03.310 | INFO | Analyzing chunk frames = [7958528,7959040) | 20260507 06:53:03.310 | INFO | MemInfo: { | 20260507 06:53:03.310 | INFO | "gpu" : " 17 GB", | 20260507 06:53:03.310 | INFO | "huge" : "", | 20260507 06:53:03.310 | INFO | "huge_shared" : " 27 GB", | 20260507 06:53:03.310 | INFO | "malloc" : " 14 GB", | 20260507 06:53:03.310 | INFO | "pinned" : " 29 GB" | 20260507 06:53:03.310 | INFO | } | 20260507 06:53:03.310 | INFO | Perf report for the last 5.13989 seconds: | 20260507 06:53:03.310 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:53:03.310 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:03.310 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:03.310 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:03.310 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 06:53:03.310 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:03.310 | INFO | RT_METRICS : 44.5%, 1.0, 23.4ms, 0 | 20260507 06:53:03.310 | INFO | PRE_HQ : 59.8%, 1.9, 59.6ms, 0 | 20260507 06:53:03.310 | INFO | BAZWRITER : 61.9%, 3.3, 104.7ms, 0 | 20260507 06:53:03.310 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:53:03.310 | INFO | Uploaded 36GiB to GPU in 2.03807s (17.6638GiB/s) | 20260507 06:53:03.310 | INFO | Downloaded 38GiB to GPU in 2.17464s (17.4741GiB/s) | 20260507 06:53:08.455 | INFO | Analyzing chunk frames = [7959040,7959552) | 20260507 06:53:08.455 | INFO | MemInfo: { | 20260507 06:53:08.455 | INFO | "gpu" : " 17 GB", | 20260507 06:53:08.455 | INFO | "huge" : "", | 20260507 06:53:08.455 | INFO | "huge_shared" : " 27 GB", | 20260507 06:53:08.455 | INFO | "malloc" : " 14 GB", | 20260507 06:53:08.455 | INFO | "pinned" : " 29 GB" | 20260507 06:53:08.455 | INFO | } | 20260507 06:53:08.455 | INFO | Perf report for the last 5.145 seconds: | 20260507 06:53:08.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:53:08.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:08.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:08.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:08.455 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 06:53:08.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:08.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:08.455 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 06:53:08.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:08.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:08.455 | INFO | Uploaded 36GiB to GPU in 1.9838s (18.147GiB/s) | 20260507 06:53:08.455 | INFO | Downloaded 34GiB to GPU in 1.93428s (17.5776GiB/s) | 20260507 06:53:13.590 | INFO | Analyzing chunk frames = [7959552,7960064) | 20260507 06:53:13.590 | INFO | MemInfo: { | 20260507 06:53:13.590 | INFO | "gpu" : " 17 GB", | 20260507 06:53:13.590 | INFO | "huge" : "", | 20260507 06:53:13.590 | INFO | "huge_shared" : " 27 GB", | 20260507 06:53:13.590 | INFO | "malloc" : " 14 GB", | 20260507 06:53:13.590 | INFO | "pinned" : " 29 GB" | 20260507 06:53:13.590 | INFO | } | 20260507 06:53:13.590 | INFO | Perf report for the last 5.13486 seconds: | 20260507 06:53:13.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:53:13.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:13.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:13.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:13.590 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 06:53:13.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:13.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:13.590 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 06:53:13.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:13.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:13.590 | INFO | Uploaded 36GiB to GPU in 2.02137s (17.8097GiB/s) | 20260507 06:53:13.590 | INFO | Downloaded 34GiB to GPU in 1.92279s (17.6826GiB/s) | 20260507 06:53:18.724 | INFO | Analyzing chunk frames = [7960064,7960576) | 20260507 06:53:18.724 | INFO | MemInfo: { | 20260507 06:53:18.724 | INFO | "gpu" : " 17 GB", | 20260507 06:53:18.724 | INFO | "huge" : "", | 20260507 06:53:18.724 | INFO | "huge_shared" : " 27 GB", | 20260507 06:53:18.724 | INFO | "malloc" : " 14 GB", | 20260507 06:53:18.724 | INFO | "pinned" : " 29 GB" | 20260507 06:53:18.724 | INFO | } | 20260507 06:53:18.725 | INFO | Perf report for the last 5.13475 seconds: | 20260507 06:53:18.725 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:53:18.725 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:18.725 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:18.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:18.725 | INFO | ANALYSIS : 60.3%, 2.9, 93.1ms, 0 | 20260507 06:53:18.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:18.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:18.725 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 06:53:18.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:18.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:18.725 | INFO | Uploaded 36GiB to GPU in 2.02856s (17.7466GiB/s) | 20260507 06:53:18.725 | INFO | Downloaded 34GiB to GPU in 1.95532s (17.3885GiB/s) | 20260507 06:53:23.870 | INFO | Analyzing chunk frames = [7960576,7961088) | 20260507 06:53:23.870 | INFO | MemInfo: { | 20260507 06:53:23.870 | INFO | "gpu" : " 17 GB", | 20260507 06:53:23.870 | INFO | "huge" : "", | 20260507 06:53:23.870 | INFO | "huge_shared" : " 27 GB", | 20260507 06:53:23.870 | INFO | "malloc" : " 14 GB", | 20260507 06:53:23.870 | INFO | "pinned" : " 29 GB" | 20260507 06:53:23.870 | INFO | } | 20260507 06:53:23.870 | INFO | Perf report for the last 5.14545 seconds: | 20260507 06:53:23.870 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:53:23.870 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:23.870 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:23.870 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:23.870 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 06:53:23.870 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:23.870 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:23.870 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 06:53:23.870 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:23.870 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:23.870 | INFO | Uploaded 36GiB to GPU in 2.04942s (17.566GiB/s) | 20260507 06:53:23.870 | INFO | Downloaded 34GiB to GPU in 1.95017s (17.4344GiB/s) | 20260507 06:53:29.012 | INFO | Analyzing chunk frames = [7961088,7961600) | 20260507 06:53:29.012 | INFO | MemInfo: { | 20260507 06:53:29.012 | INFO | "gpu" : " 17 GB", | 20260507 06:53:29.012 | INFO | "huge" : "", | 20260507 06:53:29.012 | INFO | "huge_shared" : " 27 GB", | 20260507 06:53:29.012 | INFO | "malloc" : " 14 GB", | 20260507 06:53:29.012 | INFO | "pinned" : " 29 GB" | 20260507 06:53:29.012 | INFO | } | 20260507 06:53:29.012 | INFO | Perf report for the last 5.14221 seconds: | 20260507 06:53:29.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:53:29.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:29.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:29.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:29.012 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 06:53:29.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:29.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:29.012 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:53:29.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:29.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:29.013 | INFO | Uploaded 36GiB to GPU in 2.0274s (17.7567GiB/s) | 20260507 06:53:29.013 | INFO | Downloaded 34GiB to GPU in 1.90683s (17.8306GiB/s) | 20260507 06:53:34.152 | INFO | Analyzing chunk frames = [7961600,7962112) | 20260507 06:53:34.152 | INFO | MemInfo: { | 20260507 06:53:34.152 | INFO | "gpu" : " 17 GB", | 20260507 06:53:34.152 | INFO | "huge" : "", | 20260507 06:53:34.152 | INFO | "huge_shared" : " 27 GB", | 20260507 06:53:34.152 | INFO | "malloc" : " 14 GB", | 20260507 06:53:34.152 | INFO | "pinned" : " 29 GB" | 20260507 06:53:34.152 | INFO | } | 20260507 06:53:34.152 | INFO | Perf report for the last 5.13934 seconds: | 20260507 06:53:34.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:53:34.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:34.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:34.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:34.152 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:53:34.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:34.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:34.152 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:53:34.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:34.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:34.152 | INFO | Uploaded 36GiB to GPU in 1.90016s (18.9457GiB/s) | 20260507 06:53:34.152 | INFO | Downloaded 34GiB to GPU in 1.97569s (17.2092GiB/s) | 20260507 06:53:39.281 | INFO | Analyzing chunk frames = [7962112,7962624) | 20260507 06:53:39.281 | INFO | MemInfo: { | 20260507 06:53:39.281 | INFO | "gpu" : " 17 GB", | 20260507 06:53:39.281 | INFO | "huge" : "", | 20260507 06:53:39.281 | INFO | "huge_shared" : " 27 GB", | 20260507 06:53:39.281 | INFO | "malloc" : " 14 GB", | 20260507 06:53:39.281 | INFO | "pinned" : " 29 GB" | 20260507 06:53:39.281 | INFO | } | 20260507 06:53:39.281 | INFO | Perf report for the last 5.12944 seconds: | 20260507 06:53:39.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:53:39.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:39.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:39.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:39.281 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 06:53:39.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:39.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:39.281 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:53:39.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:39.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:39.281 | INFO | Uploaded 36GiB to GPU in 1.98535s (18.1328GiB/s) | 20260507 06:53:39.281 | INFO | Downloaded 34GiB to GPU in 1.95599s (17.3825GiB/s) | 20260507 06:53:39.281 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:53:39.281 | INFO | Total mem usage: 18 GiB | 20260507 06:53:39.281 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:53:39.281 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.7%) | 20260507 06:53:39.281 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:53:39.281 | INFO | High Water Breakdown: | 20260507 06:53:39.281 | INFO | 60.2% : 8.63 GiB ( 75.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:53:39.282 | INFO | 39.6% : 5.68 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:53:39.282 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:53:39.282 | INFO | ------------------------------------------ | 20260507 06:53:39.282 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:53:39.282 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:53:39.282 | INFO | Total mem usage: 26.5 GiB | 20260507 06:53:39.282 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:53:39.282 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:53:39.282 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:53:39.282 | INFO | High Water Breakdown: | 20260507 06:53:39.282 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:53:39.282 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:53:39.282 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:53:39.282 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:53:39.282 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:53:39.282 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:53:39.282 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:53:39.282 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:53:39.282 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:53:39.282 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:53:39.282 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:53:39.282 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:53:39.282 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:53:39.282 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:53:39.282 | INFO | ------------------------------------------ | 20260507 06:53:39.282 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:53:39.282 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:53:39.282 | INFO | Total mem usage: 25.6 GiB | 20260507 06:53:39.282 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:53:39.282 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:53:39.282 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:53:39.282 | INFO | High Water Breakdown: | 20260507 06:53:39.282 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:53:39.282 | INFO | ------------------------------------------ | 20260507 06:53:39.282 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:53:39.282 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:53:39.282 | INFO | Total mem usage: 15.7 GiB | 20260507 06:53:39.282 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:53:39.282 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:53:39.282 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:53:39.282 | INFO | High Water Breakdown: | 20260507 06:53:39.282 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:53:39.282 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:53:39.282 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:53:39.282 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:53:39.282 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:53:39.282 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:53:39.282 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:53:39.282 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:53:39.282 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:53:39.282 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:53:39.282 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:53:39.282 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:53:39.282 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:53:39.282 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:53:39.282 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:53:39.282 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:53:39.282 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:53:39.282 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:53:39.282 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:53:39.283 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:53:39.283 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:53:39.283 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:53:39.283 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:53:39.283 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:53:39.283 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:53:39.283 | INFO | ------------------------------------------ | 20260507 06:53:39.283 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:53:39.283 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:53:39.283 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:53:39.283 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:53:39.283 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.50e+01, 1.35e+01 / 1.10e+00 | 20260507 06:53:39.283 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.02e-01, 9.98e-03 / 3.65e-03 | 20260507 06:53:39.283 | INFO | PerfReport: 11.68%, Baseline -- 3.58e+00 / 4.47e+00, 5.09e-01 / 2.38e-01 | 20260507 06:53:39.283 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.49e+00, 1.69e-01 / 9.06e-02 | 20260507 06:53:39.283 | INFO | PerfReport: 16.30%, DME -- 4.00e+01 / 4.51e+01, 3.94e+00 / 1.37e+00 | 20260507 06:53:39.283 | INFO | PerfReport: 57.64%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.25e+00 / 1.49e+00 | 20260507 06:53:39.283 | INFO | PerfReport: 6.91%, PulseAccumulating -- 2.12e+00 / 2.70e+00, 3.87e-01 / 2.05e-01 | 20260507 06:53:39.283 | INFO | PerfReport: 2.95%, Metrics -- 9.05e-01 / 1.21e+00, 2.42e-01 / 1.35e-01 | 20260507 06:53:39.283 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.63e-01, 5.07e-02 / 2.46e-02 | 20260507 06:53:39.283 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:53:39.283 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:53:39.283 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:53:39.283 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 06:53:39.283 | INFO | PerfReport: 21.78%, Upload -- 2.09e+01 / 0.00e+00, 7.06e+00 / 0.00e+00 | 20260507 06:53:39.283 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260507 06:53:39.283 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 06:53:39.283 | INFO | PerfReport: 1.47%, LockUpload -- 1.41e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 06:53:39.283 | INFO | PerfReport: 1.36%, LockDownload -- 1.31e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260507 06:53:39.283 | INFO | PerfReport: 22.03%, LockCompute -- 2.12e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 06:53:39.283 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260507 06:53:39.378 | INFO | Starting a round of Baz IO | 20260507 06:53:42.478 | INFO | Populating RTMetrics report | 20260507 06:53:42.478 | INFO | RTMetrics report populated | 20260507 06:53:42.478 | INFO | Opening tmp RTMetrics json file | 20260507 06:53:42.496 | INFO | Serializing RTMetrics to json | 20260507 06:53:42.497 | INFO | Serialization done, swapping tmp file now... | 20260507 06:53:42.497 | INFO | RTMetrics done | 20260507 06:53:42.563 | INFO | Finished a round of Baz IO: Wrote 1.654 GiB in 3.185 seconds | 20260507 06:53:42.563 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.60% / 0.00% / 67.40% | 20260507 06:53:42.563 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.114357% / 0% / 7.15256e-05% | 20260507 06:53:42.563 | INFO | Baz IO overall: | 20260507 06:53:42.563 | INFO | BytesWritten : 4004891578520 | 20260507 06:53:42.563 | INFO | NumEvents : 1918429276986 | 20260507 06:53:44.423 | INFO | Analyzing chunk frames = [7962624,7963136) | 20260507 06:53:44.423 | INFO | MemInfo: { | 20260507 06:53:44.423 | INFO | "gpu" : " 17 GB", | 20260507 06:53:44.423 | INFO | "huge" : "", | 20260507 06:53:44.423 | INFO | "huge_shared" : " 27 GB", | 20260507 06:53:44.423 | INFO | "malloc" : " 14 GB", | 20260507 06:53:44.423 | INFO | "pinned" : " 29 GB" | 20260507 06:53:44.423 | INFO | } | 20260507 06:53:44.423 | INFO | Perf report for the last 5.14211 seconds: | 20260507 06:53:44.423 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:53:44.423 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:44.423 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:44.423 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:44.423 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 06:53:44.423 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:44.423 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260507 06:53:44.423 | INFO | PRE_HQ : 58.8%, 1.8, 54.5ms, 0 | 20260507 06:53:44.423 | INFO | BAZWRITER : 61.5%, 3.1, 98.2ms, 0 | 20260507 06:53:44.423 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:53:44.424 | INFO | Uploaded 36GiB to GPU in 2.05023s (17.559GiB/s) | 20260507 06:53:44.424 | INFO | Downloaded 38GiB to GPU in 2.18367s (17.4019GiB/s) | 20260507 06:53:49.567 | INFO | Analyzing chunk frames = [7963136,7963648) | 20260507 06:53:49.567 | INFO | MemInfo: { | 20260507 06:53:49.567 | INFO | "gpu" : " 17 GB", | 20260507 06:53:49.567 | INFO | "huge" : "", | 20260507 06:53:49.567 | INFO | "huge_shared" : " 27 GB", | 20260507 06:53:49.567 | INFO | "malloc" : " 14 GB", | 20260507 06:53:49.567 | INFO | "pinned" : " 29 GB" | 20260507 06:53:49.568 | INFO | } | 20260507 06:53:49.568 | INFO | Perf report for the last 5.14434 seconds: | 20260507 06:53:49.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:53:49.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:49.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:49.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:49.568 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 06:53:49.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:49.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:49.568 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 06:53:49.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:49.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:49.568 | INFO | Uploaded 36GiB to GPU in 2.01003s (17.9102GiB/s) | 20260507 06:53:49.568 | INFO | Downloaded 34GiB to GPU in 1.93032s (17.6136GiB/s) | 20260507 06:53:54.702 | INFO | Analyzing chunk frames = [7963648,7964160) | 20260507 06:53:54.702 | INFO | MemInfo: { | 20260507 06:53:54.702 | INFO | "gpu" : " 17 GB", | 20260507 06:53:54.702 | INFO | "huge" : "", | 20260507 06:53:54.702 | INFO | "huge_shared" : " 27 GB", | 20260507 06:53:54.702 | INFO | "malloc" : " 14 GB", | 20260507 06:53:54.702 | INFO | "pinned" : " 29 GB" | 20260507 06:53:54.702 | INFO | } | 20260507 06:53:54.702 | INFO | Perf report for the last 5.13454 seconds: | 20260507 06:53:54.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:53:54.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:54.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:54.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:54.702 | INFO | ANALYSIS : 60.7%, 3.0, 93.8ms, 0 | 20260507 06:53:54.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:54.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:54.702 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 06:53:54.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:54.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:54.703 | INFO | Uploaded 36GiB to GPU in 2.05196s (17.5442GiB/s) | 20260507 06:53:54.703 | INFO | Downloaded 34GiB to GPU in 1.95062s (17.4303GiB/s) | 20260507 06:53:59.846 | INFO | Analyzing chunk frames = [7964160,7964672) | 20260507 06:53:59.846 | INFO | MemInfo: { | 20260507 06:53:59.846 | INFO | "gpu" : " 17 GB", | 20260507 06:53:59.846 | INFO | "huge" : "", | 20260507 06:53:59.846 | INFO | "huge_shared" : " 27 GB", | 20260507 06:53:59.846 | INFO | "malloc" : " 14 GB", | 20260507 06:53:59.846 | INFO | "pinned" : " 29 GB" | 20260507 06:53:59.846 | INFO | } | 20260507 06:53:59.846 | INFO | Perf report for the last 5.14425 seconds: | 20260507 06:53:59.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:53:59.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:59.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:59.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:59.846 | INFO | ANALYSIS : 60.2%, 2.9, 93.1ms, 0 | 20260507 06:53:59.847 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:59.847 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:59.847 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:53:59.847 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:59.847 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:53:59.847 | INFO | Uploaded 36GiB to GPU in 2.0239s (17.7875GiB/s) | 20260507 06:53:59.847 | INFO | Downloaded 34GiB to GPU in 1.94608s (17.471GiB/s) | 20260507 06:54:04.980 | INFO | Analyzing chunk frames = [7964672,7965184) | 20260507 06:54:04.980 | INFO | MemInfo: { | 20260507 06:54:04.980 | INFO | "gpu" : " 17 GB", | 20260507 06:54:04.980 | INFO | "huge" : "", | 20260507 06:54:04.980 | INFO | "huge_shared" : " 27 GB", | 20260507 06:54:04.980 | INFO | "malloc" : " 14 GB", | 20260507 06:54:04.980 | INFO | "pinned" : " 29 GB" | 20260507 06:54:04.980 | INFO | } | 20260507 06:54:04.980 | INFO | Perf report for the last 5.13348 seconds: | 20260507 06:54:04.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:54:04.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:04.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:04.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:04.980 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 06:54:04.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:04.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:04.980 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 06:54:04.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:04.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:04.980 | INFO | Uploaded 36GiB to GPU in 2.05268s (17.5381GiB/s) | 20260507 06:54:04.980 | INFO | Downloaded 34GiB to GPU in 1.90896s (17.8108GiB/s) | 20260507 06:54:10.127 | INFO | Analyzing chunk frames = [7965184,7965696) | 20260507 06:54:10.127 | INFO | MemInfo: { | 20260507 06:54:10.127 | INFO | "gpu" : " 17 GB", | 20260507 06:54:10.127 | INFO | "huge" : "", | 20260507 06:54:10.127 | INFO | "huge_shared" : " 27 GB", | 20260507 06:54:10.127 | INFO | "malloc" : " 14 GB", | 20260507 06:54:10.127 | INFO | "pinned" : " 29 GB" | 20260507 06:54:10.127 | INFO | } | 20260507 06:54:10.127 | INFO | Perf report for the last 5.14717 seconds: | 20260507 06:54:10.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:54:10.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:10.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:10.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:10.127 | INFO | ANALYSIS : 60.2%, 3.0, 94.2ms, 0 | 20260507 06:54:10.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:10.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:10.127 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 06:54:10.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:10.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:10.127 | INFO | Uploaded 36GiB to GPU in 2.03767s (17.6673GiB/s) | 20260507 06:54:10.127 | INFO | Downloaded 34GiB to GPU in 1.91145s (17.7875GiB/s) | 20260507 06:54:15.265 | INFO | Analyzing chunk frames = [7965696,7966208) | 20260507 06:54:15.265 | INFO | MemInfo: { | 20260507 06:54:15.265 | INFO | "gpu" : " 17 GB", | 20260507 06:54:15.265 | INFO | "huge" : "", | 20260507 06:54:15.265 | INFO | "huge_shared" : " 27 GB", | 20260507 06:54:15.265 | INFO | "malloc" : " 14 GB", | 20260507 06:54:15.265 | INFO | "pinned" : " 29 GB" | 20260507 06:54:15.265 | INFO | } | 20260507 06:54:15.265 | INFO | Perf report for the last 5.13775 seconds: | 20260507 06:54:15.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:54:15.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:15.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:15.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:15.265 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:54:15.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:15.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:15.265 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 06:54:15.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:15.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:15.265 | INFO | Uploaded 36GiB to GPU in 1.93812s (18.5747GiB/s) | 20260507 06:54:15.265 | INFO | Downloaded 34GiB to GPU in 1.9757s (17.2091GiB/s) | 20260507 06:54:20.407 | INFO | Analyzing chunk frames = [7966208,7966720) | 20260507 06:54:20.407 | INFO | MemInfo: { | 20260507 06:54:20.407 | INFO | "gpu" : " 17 GB", | 20260507 06:54:20.407 | INFO | "huge" : "", | 20260507 06:54:20.407 | INFO | "huge_shared" : " 27 GB", | 20260507 06:54:20.407 | INFO | "malloc" : " 14 GB", | 20260507 06:54:20.407 | INFO | "pinned" : " 29 GB" | 20260507 06:54:20.407 | INFO | } | 20260507 06:54:20.407 | INFO | Perf report for the last 5.14187 seconds: | 20260507 06:54:20.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:54:20.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:20.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:20.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:20.407 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 06:54:20.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:20.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:20.407 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 06:54:20.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:20.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:20.407 | INFO | Uploaded 36GiB to GPU in 1.978s (18.2002GiB/s) | 20260507 06:54:20.407 | INFO | Downloaded 34GiB to GPU in 1.96648s (17.2898GiB/s) | 20260507 06:54:20.504 | INFO | Starting a round of Baz IO | 20260507 06:54:23.617 | INFO | Populating RTMetrics report | 20260507 06:54:23.617 | INFO | RTMetrics report populated | 20260507 06:54:23.617 | INFO | Opening tmp RTMetrics json file | 20260507 06:54:23.618 | INFO | Serializing RTMetrics to json | 20260507 06:54:23.619 | INFO | Serialization done, swapping tmp file now... | 20260507 06:54:23.620 | INFO | RTMetrics done | 20260507 06:54:23.706 | INFO | Finished a round of Baz IO: Wrote 1.656 GiB in 3.202 seconds | 20260507 06:54:23.706 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.59% / 0.00% / 67.40% | 20260507 06:54:23.706 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.114155% / 0% / 7.15256e-05% | 20260507 06:54:23.706 | INFO | Baz IO overall: | 20260507 06:54:23.706 | INFO | BytesWritten : 4006670085907 | 20260507 06:54:23.706 | INFO | NumEvents : 1919261288682 | 20260507 06:54:25.535 | INFO | Analyzing chunk frames = [7966720,7967232) | 20260507 06:54:25.535 | INFO | MemInfo: { | 20260507 06:54:25.535 | INFO | "gpu" : " 17 GB", | 20260507 06:54:25.535 | INFO | "huge" : "", | 20260507 06:54:25.535 | INFO | "huge_shared" : " 27 GB", | 20260507 06:54:25.535 | INFO | "malloc" : " 14 GB", | 20260507 06:54:25.535 | INFO | "pinned" : " 29 GB" | 20260507 06:54:25.535 | INFO | } | 20260507 06:54:25.535 | INFO | Perf report for the last 5.12849 seconds: | 20260507 06:54:25.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:54:25.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:25.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:25.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:25.535 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260507 06:54:25.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:25.535 | INFO | RT_METRICS : 44.2%, 1.0, 23.1ms, 0 | 20260507 06:54:25.535 | INFO | PRE_HQ : 60.0%, 1.9, 59.1ms, 0 | 20260507 06:54:25.535 | INFO | BAZWRITER : 62.0%, 3.2, 103.1ms, 0 | 20260507 06:54:25.535 | INFO | BAZ_SAMPLING : 0.8%, 1.0, 0.4ms, 0 | 20260507 06:54:25.535 | INFO | Uploaded 36GiB to GPU in 2.07583s (17.3424GiB/s) | 20260507 06:54:25.535 | INFO | Downloaded 38GiB to GPU in 2.18502s (17.3912GiB/s) | 20260507 06:54:30.676 | INFO | Analyzing chunk frames = [7967232,7967744) | 20260507 06:54:30.676 | INFO | MemInfo: { | 20260507 06:54:30.676 | INFO | "gpu" : " 17 GB", | 20260507 06:54:30.676 | INFO | "huge" : "", | 20260507 06:54:30.676 | INFO | "huge_shared" : " 27 GB", | 20260507 06:54:30.676 | INFO | "malloc" : " 14 GB", | 20260507 06:54:30.676 | INFO | "pinned" : " 29 GB" | 20260507 06:54:30.676 | INFO | } | 20260507 06:54:30.676 | INFO | Perf report for the last 5.14069 seconds: | 20260507 06:54:30.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:54:30.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:30.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:30.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:30.676 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:54:30.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:30.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:30.676 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 06:54:30.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:30.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:30.676 | INFO | Uploaded 36GiB to GPU in 1.98797s (18.1089GiB/s) | 20260507 06:54:30.676 | INFO | Downloaded 34GiB to GPU in 1.93208s (17.5976GiB/s) | 20260507 06:54:35.823 | INFO | Analyzing chunk frames = [7967744,7968256) | 20260507 06:54:35.823 | INFO | MemInfo: { | 20260507 06:54:35.823 | INFO | "gpu" : " 17 GB", | 20260507 06:54:35.823 | INFO | "huge" : "", | 20260507 06:54:35.823 | INFO | "huge_shared" : " 27 GB", | 20260507 06:54:35.823 | INFO | "malloc" : " 14 GB", | 20260507 06:54:35.823 | INFO | "pinned" : " 29 GB" | 20260507 06:54:35.823 | INFO | } | 20260507 06:54:35.823 | INFO | Perf report for the last 5.14703 seconds: | 20260507 06:54:35.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:54:35.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:35.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:35.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:35.823 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 06:54:35.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:35.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:35.823 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 06:54:35.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:35.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:35.823 | INFO | Uploaded 36GiB to GPU in 1.98646s (18.1227GiB/s) | 20260507 06:54:35.823 | INFO | Downloaded 34GiB to GPU in 1.9463s (17.4691GiB/s) | 20260507 06:54:40.962 | INFO | Analyzing chunk frames = [7968256,7968768) | 20260507 06:54:40.962 | INFO | MemInfo: { | 20260507 06:54:40.962 | INFO | "gpu" : " 17 GB", | 20260507 06:54:40.962 | INFO | "huge" : "", | 20260507 06:54:40.962 | INFO | "huge_shared" : " 27 GB", | 20260507 06:54:40.962 | INFO | "malloc" : " 14 GB", | 20260507 06:54:40.962 | INFO | "pinned" : " 29 GB" | 20260507 06:54:40.962 | INFO | } | 20260507 06:54:40.962 | INFO | Perf report for the last 5.13884 seconds: | 20260507 06:54:40.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:54:40.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:40.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:40.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:40.962 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 06:54:40.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:40.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:40.962 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:54:40.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:40.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:40.962 | INFO | Uploaded 36GiB to GPU in 2.03548s (17.6863GiB/s) | 20260507 06:54:40.962 | INFO | Downloaded 34GiB to GPU in 1.95665s (17.3766GiB/s) | 20260507 06:54:46.102 | INFO | Analyzing chunk frames = [7968768,7969280) | 20260507 06:54:46.102 | INFO | MemInfo: { | 20260507 06:54:46.102 | INFO | "gpu" : " 17 GB", | 20260507 06:54:46.102 | INFO | "huge" : "", | 20260507 06:54:46.102 | INFO | "huge_shared" : " 27 GB", | 20260507 06:54:46.102 | INFO | "malloc" : " 14 GB", | 20260507 06:54:46.102 | INFO | "pinned" : " 29 GB" | 20260507 06:54:46.102 | INFO | } | 20260507 06:54:46.102 | INFO | Perf report for the last 5.14063 seconds: | 20260507 06:54:46.102 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:54:46.102 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:46.103 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:46.103 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:46.103 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 06:54:46.103 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:46.103 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:46.103 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:54:46.103 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:46.103 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:46.103 | INFO | Uploaded 36GiB to GPU in 2.04413s (17.6114GiB/s) | 20260507 06:54:46.103 | INFO | Downloaded 34GiB to GPU in 1.90662s (17.8326GiB/s) | 20260507 06:54:51.238 | INFO | Analyzing chunk frames = [7969280,7969792) | 20260507 06:54:51.238 | INFO | MemInfo: { | 20260507 06:54:51.238 | INFO | "gpu" : " 17 GB", | 20260507 06:54:51.238 | INFO | "huge" : "", | 20260507 06:54:51.238 | INFO | "huge_shared" : " 27 GB", | 20260507 06:54:51.238 | INFO | "malloc" : " 14 GB", | 20260507 06:54:51.238 | INFO | "pinned" : " 29 GB" | 20260507 06:54:51.238 | INFO | } | 20260507 06:54:51.238 | INFO | Perf report for the last 5.13567 seconds: | 20260507 06:54:51.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:54:51.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:51.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:51.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:51.238 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 06:54:51.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:51.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:51.238 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 06:54:51.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:51.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:51.239 | INFO | Uploaded 36GiB to GPU in 2.06859s (17.4032GiB/s) | 20260507 06:54:51.239 | INFO | Downloaded 34GiB to GPU in 1.94162s (17.5112GiB/s) | 20260507 06:54:56.379 | INFO | Analyzing chunk frames = [7969792,7970304) | 20260507 06:54:56.379 | INFO | MemInfo: { | 20260507 06:54:56.379 | INFO | "gpu" : " 17 GB", | 20260507 06:54:56.379 | INFO | "huge" : "", | 20260507 06:54:56.379 | INFO | "huge_shared" : " 27 GB", | 20260507 06:54:56.379 | INFO | "malloc" : " 14 GB", | 20260507 06:54:56.379 | INFO | "pinned" : " 29 GB" | 20260507 06:54:56.379 | INFO | } | 20260507 06:54:56.379 | INFO | Perf report for the last 5.14073 seconds: | 20260507 06:54:56.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:54:56.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:56.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:56.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:56.379 | INFO | ANALYSIS : 61.1%, 3.0, 95.1ms, 0 | 20260507 06:54:56.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:56.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:56.379 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 06:54:56.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:56.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:54:56.379 | INFO | Uploaded 36GiB to GPU in 2.04251s (17.6254GiB/s) | 20260507 06:54:56.379 | INFO | Downloaded 34GiB to GPU in 1.98411s (17.1362GiB/s) | 20260507 06:55:01.512 | INFO | Analyzing chunk frames = [7970304,7970816) | 20260507 06:55:01.512 | INFO | MemInfo: { | 20260507 06:55:01.512 | INFO | "gpu" : " 17 GB", | 20260507 06:55:01.512 | INFO | "huge" : "", | 20260507 06:55:01.512 | INFO | "huge_shared" : " 27 GB", | 20260507 06:55:01.512 | INFO | "malloc" : " 14 GB", | 20260507 06:55:01.512 | INFO | "pinned" : " 29 GB" | 20260507 06:55:01.512 | INFO | } | 20260507 06:55:01.512 | INFO | Perf report for the last 5.13333 seconds: | 20260507 06:55:01.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:55:01.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:01.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:01.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:01.512 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:55:01.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:01.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:01.512 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:55:01.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:01.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:01.513 | INFO | Uploaded 36GiB to GPU in 1.9789s (18.192GiB/s) | 20260507 06:55:01.513 | INFO | Downloaded 34GiB to GPU in 1.95604s (17.3821GiB/s) | 20260507 06:55:01.513 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:55:01.513 | INFO | Total mem usage: 18 GiB | 20260507 06:55:01.513 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:55:01.513 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 06:55:01.513 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:55:01.513 | INFO | High Water Breakdown: | 20260507 06:55:01.513 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:55:01.513 | INFO | 39.7% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:55:01.513 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:55:01.513 | INFO | ------------------------------------------ | 20260507 06:55:01.513 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:55:01.513 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:55:01.513 | INFO | Total mem usage: 26.5 GiB | 20260507 06:55:01.513 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:55:01.513 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:55:01.513 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:55:01.513 | INFO | High Water Breakdown: | 20260507 06:55:01.513 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:55:01.513 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:55:01.513 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:55:01.513 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:55:01.513 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:55:01.513 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:55:01.513 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:55:01.513 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:55:01.513 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:55:01.513 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:55:01.514 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:55:01.514 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:55:01.514 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:55:01.514 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:55:01.514 | INFO | ------------------------------------------ | 20260507 06:55:01.514 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:55:01.514 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:55:01.514 | INFO | Total mem usage: 25.6 GiB | 20260507 06:55:01.514 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:55:01.514 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:55:01.514 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:55:01.514 | INFO | High Water Breakdown: | 20260507 06:55:01.514 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:55:01.514 | INFO | ------------------------------------------ | 20260507 06:55:01.514 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:55:01.514 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:55:01.514 | INFO | Total mem usage: 15.7 GiB | 20260507 06:55:01.514 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:55:01.514 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:55:01.514 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:55:01.514 | INFO | High Water Breakdown: | 20260507 06:55:01.514 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:55:01.514 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:55:01.514 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:55:01.514 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:55:01.514 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:55:01.514 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:55:01.514 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:55:01.514 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:55:01.514 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:55:01.514 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:55:01.514 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:55:01.514 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:55:01.514 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:55:01.514 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:55:01.514 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:55:01.514 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:55:01.514 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:55:01.514 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:55:01.514 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:55:01.514 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:55:01.514 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:55:01.514 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:55:01.514 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:55:01.514 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:55:01.514 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:55:01.514 | INFO | ------------------------------------------ | 20260507 06:55:01.514 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:55:01.514 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:55:01.514 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:55:01.514 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:55:01.514 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.33e+01 / 1.29e+00 | 20260507 06:55:01.514 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.01e-01, 8.98e-03 / 6.34e-03 | 20260507 06:55:01.514 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.34e+00, 4.98e-01 / 3.77e-01 | 20260507 06:55:01.514 | INFO | PerfReport: 3.21%, Histogram -- 9.88e-01 / 1.46e+00, 1.64e-01 / 1.33e-01 | 20260507 06:55:01.514 | INFO | PerfReport: 16.25%, DME -- 3.99e+01 / 4.47e+01, 3.90e+00 / 1.87e+00 | 20260507 06:55:01.514 | INFO | PerfReport: 57.76%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.34e+00 / 1.36e+00 | 20260507 06:55:01.514 | INFO | PerfReport: 6.89%, PulseAccumulating -- 2.12e+00 / 2.68e+00, 3.81e-01 / 2.09e-01 | 20260507 06:55:01.514 | INFO | PerfReport: 2.94%, Metrics -- 9.03e-01 / 1.23e+00, 2.41e-01 / 1.39e-01 | 20260507 06:55:01.514 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.60e-01, 5.02e-02 / 1.51e-02 | 20260507 06:55:01.514 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:55:01.514 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:55:01.514 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:55:01.514 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 06:55:01.514 | INFO | PerfReport: 21.91%, Upload -- 2.11e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260507 06:55:01.515 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260507 06:55:01.515 | INFO | PerfReport: 31.95%, Compute -- 3.07e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 06:55:01.515 | INFO | PerfReport: 1.54%, LockUpload -- 1.48e+00 / 0.00e+00, 4.86e+00 / 0.00e+00 | 20260507 06:55:01.515 | INFO | PerfReport: 1.24%, LockDownload -- 1.20e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260507 06:55:01.515 | INFO | PerfReport: 21.97%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 06:55:01.515 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260507 06:55:01.609 | INFO | Starting a round of Baz IO | 20260507 06:55:04.699 | INFO | Populating RTMetrics report | 20260507 06:55:04.700 | INFO | RTMetrics report populated | 20260507 06:55:04.700 | INFO | Opening tmp RTMetrics json file | 20260507 06:55:04.716 | INFO | Serializing RTMetrics to json | 20260507 06:55:04.717 | INFO | Serialization done, swapping tmp file now... | 20260507 06:55:04.717 | INFO | RTMetrics done | 20260507 06:55:04.783 | INFO | Finished a round of Baz IO: Wrote 1.659 GiB in 3.173 seconds | 20260507 06:55:04.783 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.59% / 0.00% / 67.41% | 20260507 06:55:04.783 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.114107% / 0% / 6.75519e-05% | 20260507 06:55:04.783 | INFO | Baz IO overall: | 20260507 06:55:04.783 | INFO | BytesWritten : 4008451313617 | 20260507 06:55:04.783 | INFO | NumEvents : 1920094935813 | 20260507 06:55:06.659 | INFO | Analyzing chunk frames = [7970816,7971328) | 20260507 06:55:06.659 | INFO | MemInfo: { | 20260507 06:55:06.659 | INFO | "gpu" : " 17 GB", | 20260507 06:55:06.659 | INFO | "huge" : "", | 20260507 06:55:06.659 | INFO | "huge_shared" : " 27 GB", | 20260507 06:55:06.659 | INFO | "malloc" : " 14 GB", | 20260507 06:55:06.659 | INFO | "pinned" : " 29 GB" | 20260507 06:55:06.659 | INFO | } | 20260507 06:55:06.659 | INFO | Perf report for the last 5.147 seconds: | 20260507 06:55:06.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:55:06.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:06.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:06.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:06.659 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260507 06:55:06.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:06.660 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 06:55:06.660 | INFO | PRE_HQ : 58.7%, 1.8, 54.5ms, 0 | 20260507 06:55:06.660 | INFO | BAZWRITER : 61.3%, 3.1, 98.3ms, 0 | 20260507 06:55:06.660 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:55:06.660 | INFO | Uploaded 36GiB to GPU in 2.04119s (17.6368GiB/s) | 20260507 06:55:06.660 | INFO | Downloaded 38GiB to GPU in 2.16979s (17.5132GiB/s) | 20260507 06:55:11.790 | INFO | Analyzing chunk frames = [7971328,7971840) | 20260507 06:55:11.790 | INFO | MemInfo: { | 20260507 06:55:11.790 | INFO | "gpu" : " 17 GB", | 20260507 06:55:11.790 | INFO | "huge" : "", | 20260507 06:55:11.790 | INFO | "huge_shared" : " 27 GB", | 20260507 06:55:11.790 | INFO | "malloc" : " 14 GB", | 20260507 06:55:11.790 | INFO | "pinned" : " 29 GB" | 20260507 06:55:11.790 | INFO | } | 20260507 06:55:11.790 | INFO | Perf report for the last 5.13059 seconds: | 20260507 06:55:11.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:55:11.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:11.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:11.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:11.790 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 06:55:11.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:11.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:11.790 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 06:55:11.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:11.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:11.790 | INFO | Uploaded 36GiB to GPU in 1.98647s (18.1226GiB/s) | 20260507 06:55:11.790 | INFO | Downloaded 34GiB to GPU in 1.92563s (17.6566GiB/s) | 20260507 06:55:16.928 | INFO | Analyzing chunk frames = [7971840,7972352) | 20260507 06:55:16.928 | INFO | MemInfo: { | 20260507 06:55:16.928 | INFO | "gpu" : " 17 GB", | 20260507 06:55:16.928 | INFO | "huge" : "", | 20260507 06:55:16.928 | INFO | "huge_shared" : " 27 GB", | 20260507 06:55:16.928 | INFO | "malloc" : " 14 GB", | 20260507 06:55:16.928 | INFO | "pinned" : " 29 GB" | 20260507 06:55:16.928 | INFO | } | 20260507 06:55:16.928 | INFO | Perf report for the last 5.13805 seconds: | 20260507 06:55:16.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:55:16.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:16.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:16.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:16.928 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 06:55:16.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:16.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:16.928 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:55:16.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:16.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:16.928 | INFO | Uploaded 36GiB to GPU in 1.99008s (18.0897GiB/s) | 20260507 06:55:16.928 | INFO | Downloaded 34GiB to GPU in 1.92184s (17.6914GiB/s) | 20260507 06:55:22.076 | INFO | Analyzing chunk frames = [7972352,7972864) | 20260507 06:55:22.076 | INFO | MemInfo: { | 20260507 06:55:22.076 | INFO | "gpu" : " 17 GB", | 20260507 06:55:22.076 | INFO | "huge" : "", | 20260507 06:55:22.076 | INFO | "huge_shared" : " 27 GB", | 20260507 06:55:22.076 | INFO | "malloc" : " 14 GB", | 20260507 06:55:22.076 | INFO | "pinned" : " 29 GB" | 20260507 06:55:22.076 | INFO | } | 20260507 06:55:22.076 | INFO | Perf report for the last 5.14784 seconds: | 20260507 06:55:22.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:55:22.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:22.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:22.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:22.076 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 06:55:22.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:22.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:22.076 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:55:22.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:22.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:22.076 | INFO | Uploaded 36GiB to GPU in 2.01417s (17.8733GiB/s) | 20260507 06:55:22.076 | INFO | Downloaded 34GiB to GPU in 1.93233s (17.5953GiB/s) | 20260507 06:55:27.216 | INFO | Analyzing chunk frames = [7972864,7973376) | 20260507 06:55:27.216 | INFO | MemInfo: { | 20260507 06:55:27.216 | INFO | "gpu" : " 17 GB", | 20260507 06:55:27.216 | INFO | "huge" : "", | 20260507 06:55:27.216 | INFO | "huge_shared" : " 27 GB", | 20260507 06:55:27.216 | INFO | "malloc" : " 14 GB", | 20260507 06:55:27.216 | INFO | "pinned" : " 29 GB" | 20260507 06:55:27.216 | INFO | } | 20260507 06:55:27.216 | INFO | Perf report for the last 5.1403 seconds: | 20260507 06:55:27.216 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:55:27.216 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:27.216 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:27.216 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:27.216 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 06:55:27.216 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:27.216 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:27.216 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:55:27.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:27.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:27.216 | INFO | Uploaded 36GiB to GPU in 2.04315s (17.6199GiB/s) | 20260507 06:55:27.217 | INFO | Downloaded 34GiB to GPU in 1.9169s (17.737GiB/s) | 20260507 06:55:32.349 | INFO | Analyzing chunk frames = [7973376,7973888) | 20260507 06:55:32.349 | INFO | MemInfo: { | 20260507 06:55:32.349 | INFO | "gpu" : " 17 GB", | 20260507 06:55:32.349 | INFO | "huge" : "", | 20260507 06:55:32.349 | INFO | "huge_shared" : " 27 GB", | 20260507 06:55:32.349 | INFO | "malloc" : " 14 GB", | 20260507 06:55:32.349 | INFO | "pinned" : " 29 GB" | 20260507 06:55:32.349 | INFO | } | 20260507 06:55:32.349 | INFO | Perf report for the last 5.13315 seconds: | 20260507 06:55:32.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:55:32.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:32.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:32.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:32.349 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 06:55:32.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:32.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:32.349 | INFO | PRE_HQ : 27.5%, 1.0, 14.2ms, 0 | 20260507 06:55:32.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:32.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:32.350 | INFO | Uploaded 36GiB to GPU in 2.04637s (17.5921GiB/s) | 20260507 06:55:32.350 | INFO | Downloaded 34GiB to GPU in 1.9185s (17.7222GiB/s) | 20260507 06:55:37.483 | INFO | Analyzing chunk frames = [7973888,7974400) | 20260507 06:55:37.483 | INFO | MemInfo: { | 20260507 06:55:37.483 | INFO | "gpu" : " 17 GB", | 20260507 06:55:37.483 | INFO | "huge" : "", | 20260507 06:55:37.483 | INFO | "huge_shared" : " 27 GB", | 20260507 06:55:37.483 | INFO | "malloc" : " 14 GB", | 20260507 06:55:37.483 | INFO | "pinned" : " 29 GB" | 20260507 06:55:37.483 | INFO | } | 20260507 06:55:37.483 | INFO | Perf report for the last 5.13372 seconds: | 20260507 06:55:37.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:55:37.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:37.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:37.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:37.483 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 06:55:37.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:37.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:37.483 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:55:37.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:37.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:37.483 | INFO | Uploaded 36GiB to GPU in 2.00723s (17.9352GiB/s) | 20260507 06:55:37.483 | INFO | Downloaded 34GiB to GPU in 1.9871s (17.1103GiB/s) | 20260507 06:55:42.630 | INFO | Analyzing chunk frames = [7974400,7974912) | 20260507 06:55:42.630 | INFO | MemInfo: { | 20260507 06:55:42.630 | INFO | "gpu" : " 17 GB", | 20260507 06:55:42.630 | INFO | "huge" : "", | 20260507 06:55:42.630 | INFO | "huge_shared" : " 27 GB", | 20260507 06:55:42.630 | INFO | "malloc" : " 14 GB", | 20260507 06:55:42.630 | INFO | "pinned" : " 29 GB" | 20260507 06:55:42.630 | INFO | } | 20260507 06:55:42.630 | INFO | Perf report for the last 5.14739 seconds: | 20260507 06:55:42.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:55:42.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:42.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:42.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:42.630 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:55:42.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:42.630 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:42.630 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 06:55:42.630 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:42.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:42.631 | INFO | Uploaded 36GiB to GPU in 1.96777s (18.2948GiB/s) | 20260507 06:55:42.631 | INFO | Downloaded 34GiB to GPU in 1.9567s (17.3762GiB/s) | 20260507 06:55:42.728 | INFO | Starting a round of Baz IO | 20260507 06:55:45.830 | INFO | Populating RTMetrics report | 20260507 06:55:45.831 | INFO | RTMetrics report populated | 20260507 06:55:45.831 | INFO | Opening tmp RTMetrics json file | 20260507 06:55:45.835 | INFO | Serializing RTMetrics to json | 20260507 06:55:45.848 | INFO | Serialization done, swapping tmp file now... | 20260507 06:55:45.849 | INFO | RTMetrics done | 20260507 06:55:45.920 | INFO | Finished a round of Baz IO: Wrote 1.658 GiB in 3.192 seconds | 20260507 06:55:45.920 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.59% / 0.00% / 67.41% | 20260507 06:55:45.920 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.113757% / 0% / 5.5631e-05% | 20260507 06:55:45.920 | INFO | Baz IO overall: | 20260507 06:55:45.920 | INFO | BytesWritten : 4010231822306 | 20260507 06:55:45.920 | INFO | NumEvents : 1920927634115 | 20260507 06:55:47.774 | INFO | Analyzing chunk frames = [7974912,7975424) | 20260507 06:55:47.774 | INFO | MemInfo: { | 20260507 06:55:47.774 | INFO | "gpu" : " 17 GB", | 20260507 06:55:47.774 | INFO | "huge" : "", | 20260507 06:55:47.774 | INFO | "huge_shared" : " 27 GB", | 20260507 06:55:47.774 | INFO | "malloc" : " 14 GB", | 20260507 06:55:47.774 | INFO | "pinned" : " 29 GB" | 20260507 06:55:47.774 | INFO | } | 20260507 06:55:47.774 | INFO | Perf report for the last 5.14405 seconds: | 20260507 06:55:47.774 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:55:47.774 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:47.774 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:47.774 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:47.774 | INFO | ANALYSIS : 61.7%, 3.0, 95.6ms, 0 | 20260507 06:55:47.774 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:47.774 | INFO | RT_METRICS : 44.7%, 1.0, 23.5ms, 0 | 20260507 06:55:47.775 | INFO | PRE_HQ : 59.8%, 1.9, 60.2ms, 0 | 20260507 06:55:47.775 | INFO | BAZWRITER : 61.6%, 3.3, 104.9ms, 0 | 20260507 06:55:47.775 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:55:47.775 | INFO | Uploaded 36GiB to GPU in 2.06585s (17.4262GiB/s) | 20260507 06:55:47.775 | INFO | Downloaded 38GiB to GPU in 2.1853s (17.3889GiB/s) | 20260507 06:55:52.909 | INFO | Analyzing chunk frames = [7975424,7975936) | 20260507 06:55:52.909 | INFO | MemInfo: { | 20260507 06:55:52.909 | INFO | "gpu" : " 17 GB", | 20260507 06:55:52.909 | INFO | "huge" : "", | 20260507 06:55:52.909 | INFO | "huge_shared" : " 27 GB", | 20260507 06:55:52.909 | INFO | "malloc" : " 14 GB", | 20260507 06:55:52.909 | INFO | "pinned" : " 29 GB" | 20260507 06:55:52.909 | INFO | } | 20260507 06:55:52.909 | INFO | Perf report for the last 5.13487 seconds: | 20260507 06:55:52.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:55:52.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:52.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:52.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:52.910 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:55:52.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:52.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:52.910 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 06:55:52.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:52.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:52.910 | INFO | Uploaded 36GiB to GPU in 1.99457s (18.049GiB/s) | 20260507 06:55:52.910 | INFO | Downloaded 34GiB to GPU in 1.93789s (17.5448GiB/s) | 20260507 06:55:58.043 | INFO | Analyzing chunk frames = [7975936,7976448) | 20260507 06:55:58.043 | INFO | MemInfo: { | 20260507 06:55:58.043 | INFO | "gpu" : " 17 GB", | 20260507 06:55:58.043 | INFO | "huge" : "", | 20260507 06:55:58.043 | INFO | "huge_shared" : " 27 GB", | 20260507 06:55:58.043 | INFO | "malloc" : " 14 GB", | 20260507 06:55:58.043 | INFO | "pinned" : " 29 GB" | 20260507 06:55:58.043 | INFO | } | 20260507 06:55:58.043 | INFO | Perf report for the last 5.13363 seconds: | 20260507 06:55:58.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:55:58.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:58.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:58.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:58.043 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 06:55:58.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:58.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:58.043 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 06:55:58.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:58.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:55:58.043 | INFO | Uploaded 36GiB to GPU in 2.02344s (17.7915GiB/s) | 20260507 06:55:58.043 | INFO | Downloaded 34GiB to GPU in 1.92584s (17.6546GiB/s) | 20260507 06:56:03.189 | INFO | Analyzing chunk frames = [7976448,7976960) | 20260507 06:56:03.189 | INFO | MemInfo: { | 20260507 06:56:03.189 | INFO | "gpu" : " 17 GB", | 20260507 06:56:03.189 | INFO | "huge" : "", | 20260507 06:56:03.189 | INFO | "huge_shared" : " 27 GB", | 20260507 06:56:03.189 | INFO | "malloc" : " 14 GB", | 20260507 06:56:03.189 | INFO | "pinned" : " 29 GB" | 20260507 06:56:03.189 | INFO | } | 20260507 06:56:03.189 | INFO | Perf report for the last 5.14635 seconds: | 20260507 06:56:03.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:56:03.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:03.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:03.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:03.189 | INFO | ANALYSIS : 59.5%, 3.0, 92.5ms, 0 | 20260507 06:56:03.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:03.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:03.189 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:56:03.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:03.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:03.190 | INFO | Uploaded 36GiB to GPU in 2.00586s (17.9474GiB/s) | 20260507 06:56:03.190 | INFO | Downloaded 34GiB to GPU in 1.94354s (17.4939GiB/s) | 20260507 06:56:08.327 | INFO | Analyzing chunk frames = [7976960,7977472) | 20260507 06:56:08.327 | INFO | MemInfo: { | 20260507 06:56:08.327 | INFO | "gpu" : " 17 GB", | 20260507 06:56:08.327 | INFO | "huge" : "", | 20260507 06:56:08.327 | INFO | "huge_shared" : " 27 GB", | 20260507 06:56:08.327 | INFO | "malloc" : " 14 GB", | 20260507 06:56:08.327 | INFO | "pinned" : " 29 GB" | 20260507 06:56:08.328 | INFO | } | 20260507 06:56:08.328 | INFO | Perf report for the last 5.13823 seconds: | 20260507 06:56:08.328 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:56:08.328 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:08.328 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:08.328 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:08.328 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 06:56:08.328 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:08.328 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:08.328 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:56:08.328 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:08.328 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:08.328 | INFO | Uploaded 36GiB to GPU in 2.05083s (17.5539GiB/s) | 20260507 06:56:08.328 | INFO | Downloaded 34GiB to GPU in 1.95141s (17.4233GiB/s) | 20260507 06:56:13.466 | INFO | Analyzing chunk frames = [7977472,7977984) | 20260507 06:56:13.466 | INFO | MemInfo: { | 20260507 06:56:13.466 | INFO | "gpu" : " 17 GB", | 20260507 06:56:13.466 | INFO | "huge" : "", | 20260507 06:56:13.466 | INFO | "huge_shared" : " 27 GB", | 20260507 06:56:13.466 | INFO | "malloc" : " 14 GB", | 20260507 06:56:13.466 | INFO | "pinned" : " 29 GB" | 20260507 06:56:13.466 | INFO | } | 20260507 06:56:13.466 | INFO | Perf report for the last 5.13824 seconds: | 20260507 06:56:13.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:56:13.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:13.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:13.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:13.466 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 06:56:13.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:13.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:13.466 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:56:13.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:13.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:13.466 | INFO | Uploaded 36GiB to GPU in 2.02164s (17.8073GiB/s) | 20260507 06:56:13.466 | INFO | Downloaded 34GiB to GPU in 1.91385s (17.7652GiB/s) | 20260507 06:56:18.608 | INFO | Analyzing chunk frames = [7977984,7978496) | 20260507 06:56:18.608 | INFO | MemInfo: { | 20260507 06:56:18.608 | INFO | "gpu" : " 17 GB", | 20260507 06:56:18.608 | INFO | "huge" : "", | 20260507 06:56:18.608 | INFO | "huge_shared" : " 27 GB", | 20260507 06:56:18.608 | INFO | "malloc" : " 14 GB", | 20260507 06:56:18.608 | INFO | "pinned" : " 29 GB" | 20260507 06:56:18.608 | INFO | } | 20260507 06:56:18.608 | INFO | Perf report for the last 5.14193 seconds: | 20260507 06:56:18.608 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:56:18.608 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:18.608 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:18.608 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:18.608 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 06:56:18.608 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:18.608 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:18.608 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:56:18.608 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:18.608 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:18.608 | INFO | Uploaded 36GiB to GPU in 1.92433s (18.7078GiB/s) | 20260507 06:56:18.608 | INFO | Downloaded 34GiB to GPU in 1.97143s (17.2464GiB/s) | 20260507 06:56:23.742 | INFO | Analyzing chunk frames = [7978496,7979008) | 20260507 06:56:23.742 | INFO | MemInfo: { | 20260507 06:56:23.742 | INFO | "gpu" : " 17 GB", | 20260507 06:56:23.742 | INFO | "huge" : "", | 20260507 06:56:23.742 | INFO | "huge_shared" : " 27 GB", | 20260507 06:56:23.742 | INFO | "malloc" : " 14 GB", | 20260507 06:56:23.742 | INFO | "pinned" : " 29 GB" | 20260507 06:56:23.742 | INFO | } | 20260507 06:56:23.742 | INFO | Perf report for the last 5.13465 seconds: | 20260507 06:56:23.742 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:56:23.742 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:23.742 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:23.742 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:23.742 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 06:56:23.742 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:23.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:23.743 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:56:23.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:23.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:23.743 | INFO | Uploaded 36GiB to GPU in 1.98197s (18.1637GiB/s) | 20260507 06:56:23.743 | INFO | Downloaded 34GiB to GPU in 1.95503s (17.391GiB/s) | 20260507 06:56:23.743 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:56:23.743 | INFO | Total mem usage: 18 GiB | 20260507 06:56:23.743 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:56:23.743 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.9%) | 20260507 06:56:23.743 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:56:23.743 | INFO | High Water Breakdown: | 20260507 06:56:23.743 | INFO | 60.2% : 8.62 GiB ( 75.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:56:23.743 | INFO | 39.6% : 5.67 GiB ( 84.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:56:23.743 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:56:23.743 | INFO | ------------------------------------------ | 20260507 06:56:23.743 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:56:23.743 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:56:23.743 | INFO | Total mem usage: 26.5 GiB | 20260507 06:56:23.743 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:56:23.743 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -3.85%) | 20260507 06:56:23.743 | INFO | Unused Cache: 1.71 GiB (Waste: 6.43%) | 20260507 06:56:23.743 | INFO | High Water Breakdown: | 20260507 06:56:23.743 | INFO | 56.2% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:56:23.743 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:56:23.743 | INFO | 3.67% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:56:23.743 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:56:23.743 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:56:23.743 | INFO | 0.698% : 182 MiB ( 83.3% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:56:23.743 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:56:23.743 | INFO | 9.38e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:56:23.743 | INFO | 8.76e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:56:23.743 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:56:23.743 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:56:23.743 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:56:23.743 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:56:23.743 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:56:23.743 | INFO | ------------------------------------------ | 20260507 06:56:23.743 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:56:23.743 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:56:23.743 | INFO | Total mem usage: 25.6 GiB | 20260507 06:56:23.743 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:56:23.743 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:56:23.743 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:56:23.743 | INFO | High Water Breakdown: | 20260507 06:56:23.743 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:56:23.743 | INFO | ------------------------------------------ | 20260507 06:56:23.743 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:56:23.743 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:56:23.743 | INFO | Total mem usage: 15.7 GiB | 20260507 06:56:23.743 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:56:23.743 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:56:23.743 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:56:23.743 | INFO | High Water Breakdown: | 20260507 06:56:23.743 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:56:23.743 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:56:23.743 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:56:23.744 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:56:23.744 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:56:23.744 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:56:23.744 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:56:23.744 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:56:23.744 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:56:23.744 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:56:23.744 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:56:23.744 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:56:23.744 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:56:23.744 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:56:23.744 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:56:23.744 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:56:23.744 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:56:23.744 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:56:23.744 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:56:23.744 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:56:23.744 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:56:23.744 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:56:23.744 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:56:23.744 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:56:23.744 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:56:23.744 | INFO | ------------------------------------------ | 20260507 06:56:23.744 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:56:23.744 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:56:23.744 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:56:23.744 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:56:23.744 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.34e+01 / 1.21e+00 | 20260507 06:56:23.744 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.02e-01, 9.25e-03 / 5.14e-03 | 20260507 06:56:23.744 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.44e+00, 5.16e-01 / 2.44e-01 | 20260507 06:56:23.744 | INFO | PerfReport: 3.24%, Histogram -- 9.96e-01 / 1.46e+00, 1.71e-01 / 1.16e-01 | 20260507 06:56:23.744 | INFO | PerfReport: 16.15%, DME -- 3.97e+01 / 4.41e+01, 3.71e+00 / 1.60e+00 | 20260507 06:56:23.744 | INFO | PerfReport: 57.82%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.40e+00 / 1.40e+00 | 20260507 06:56:23.744 | INFO | PerfReport: 6.87%, PulseAccumulating -- 2.11e+00 / 2.65e+00, 3.77e-01 / 2.22e-01 | 20260507 06:56:23.744 | INFO | PerfReport: 2.95%, Metrics -- 9.06e-01 / 1.21e+00, 2.47e-01 / 1.06e-01 | 20260507 06:56:23.744 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.56e-01, 5.10e-02 / 1.85e-02 | 20260507 06:56:23.744 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:56:23.744 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:56:23.744 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:56:23.744 | INFO | PerfReport: Full Execution -- 9.60e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 06:56:23.744 | INFO | PerfReport: 21.81%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260507 06:56:23.744 | INFO | PerfReport: 21.37%, Download -- 2.05e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260507 06:56:23.744 | INFO | PerfReport: 32.03%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:56:23.744 | INFO | PerfReport: 1.48%, LockUpload -- 1.42e+00 / 0.00e+00, 4.79e+00 / 0.00e+00 | 20260507 06:56:23.744 | INFO | PerfReport: 1.25%, LockDownload -- 1.20e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260507 06:56:23.744 | INFO | PerfReport: 22.04%, LockCompute -- 2.12e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 06:56:23.744 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.18e-02 / 0.00e+00 | 20260507 06:56:23.839 | INFO | Starting a round of Baz IO | 20260507 06:56:26.937 | INFO | Populating RTMetrics report | 20260507 06:56:26.937 | INFO | RTMetrics report populated | 20260507 06:56:26.937 | INFO | Opening tmp RTMetrics json file | 20260507 06:56:26.953 | INFO | Serializing RTMetrics to json | 20260507 06:56:26.954 | INFO | Serialization done, swapping tmp file now... | 20260507 06:56:26.955 | INFO | RTMetrics done | 20260507 06:56:27.019 | INFO | Finished a round of Baz IO: Wrote 1.658 GiB in 3.179 seconds | 20260507 06:56:27.019 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.59% / 0.00% / 67.41% | 20260507 06:56:27.019 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.11487% / 0% / 6.75519e-05% | 20260507 06:56:27.019 | INFO | Baz IO overall: | 20260507 06:56:27.019 | INFO | BytesWritten : 4012012001944 | 20260507 06:56:27.019 | INFO | NumEvents : 1921760227803 | 20260507 06:56:28.884 | INFO | Analyzing chunk frames = [7979008,7979520) | 20260507 06:56:28.884 | INFO | MemInfo: { | 20260507 06:56:28.884 | INFO | "gpu" : " 17 GB", | 20260507 06:56:28.884 | INFO | "huge" : "", | 20260507 06:56:28.884 | INFO | "huge_shared" : " 27 GB", | 20260507 06:56:28.884 | INFO | "malloc" : " 14 GB", | 20260507 06:56:28.884 | INFO | "pinned" : " 29 GB" | 20260507 06:56:28.884 | INFO | } | 20260507 06:56:28.884 | INFO | Perf report for the last 5.14173 seconds: | 20260507 06:56:28.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:56:28.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:28.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:28.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:28.884 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 06:56:28.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:28.884 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 06:56:28.884 | INFO | PRE_HQ : 58.9%, 1.8, 54.4ms, 0 | 20260507 06:56:28.884 | INFO | BAZWRITER : 61.4%, 3.1, 97.8ms, 0 | 20260507 06:56:28.884 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:56:28.884 | INFO | Uploaded 36GiB to GPU in 2.05367s (17.5296GiB/s) | 20260507 06:56:28.884 | INFO | Downloaded 38GiB to GPU in 2.17549s (17.4673GiB/s) | 20260507 06:56:34.022 | INFO | Analyzing chunk frames = [7979520,7980032) | 20260507 06:56:34.022 | INFO | MemInfo: { | 20260507 06:56:34.022 | INFO | "gpu" : " 17 GB", | 20260507 06:56:34.022 | INFO | "huge" : "", | 20260507 06:56:34.022 | INFO | "huge_shared" : " 27 GB", | 20260507 06:56:34.022 | INFO | "malloc" : " 14 GB", | 20260507 06:56:34.022 | INFO | "pinned" : " 29 GB" | 20260507 06:56:34.023 | INFO | } | 20260507 06:56:34.023 | INFO | Perf report for the last 5.13844 seconds: | 20260507 06:56:34.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:56:34.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:34.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:34.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:34.023 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:56:34.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:34.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:34.023 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:56:34.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:34.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:34.023 | INFO | Uploaded 36GiB to GPU in 1.99614s (18.0348GiB/s) | 20260507 06:56:34.023 | INFO | Downloaded 34GiB to GPU in 1.93328s (17.5866GiB/s) | 20260507 06:56:39.164 | INFO | Analyzing chunk frames = [7980032,7980544) | 20260507 06:56:39.164 | INFO | MemInfo: { | 20260507 06:56:39.164 | INFO | "gpu" : " 17 GB", | 20260507 06:56:39.164 | INFO | "huge" : "", | 20260507 06:56:39.164 | INFO | "huge_shared" : " 27 GB", | 20260507 06:56:39.164 | INFO | "malloc" : " 14 GB", | 20260507 06:56:39.164 | INFO | "pinned" : " 29 GB" | 20260507 06:56:39.164 | INFO | } | 20260507 06:56:39.164 | INFO | Perf report for the last 5.14174 seconds: | 20260507 06:56:39.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:56:39.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:39.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:39.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:39.165 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 06:56:39.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:39.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:39.165 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:56:39.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:39.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:39.165 | INFO | Uploaded 36GiB to GPU in 2.05869s (17.4869GiB/s) | 20260507 06:56:39.165 | INFO | Downloaded 34GiB to GPU in 1.94915s (17.4435GiB/s) | 20260507 06:56:44.296 | INFO | Analyzing chunk frames = [7980544,7981056) | 20260507 06:56:44.296 | INFO | MemInfo: { | 20260507 06:56:44.296 | INFO | "gpu" : " 17 GB", | 20260507 06:56:44.296 | INFO | "huge" : "", | 20260507 06:56:44.296 | INFO | "huge_shared" : " 27 GB", | 20260507 06:56:44.296 | INFO | "malloc" : " 14 GB", | 20260507 06:56:44.296 | INFO | "pinned" : " 29 GB" | 20260507 06:56:44.296 | INFO | } | 20260507 06:56:44.296 | INFO | Perf report for the last 5.13169 seconds: | 20260507 06:56:44.296 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:56:44.296 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:44.296 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:44.296 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:44.296 | INFO | ANALYSIS : 60.6%, 2.9, 93.4ms, 0 | 20260507 06:56:44.296 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:44.296 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:44.296 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 06:56:44.296 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:44.296 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:44.296 | INFO | Uploaded 36GiB to GPU in 2.05411s (17.5258GiB/s) | 20260507 06:56:44.297 | INFO | Downloaded 34GiB to GPU in 1.94561s (17.4752GiB/s) | 20260507 06:56:49.437 | INFO | Analyzing chunk frames = [7981056,7981568) | 20260507 06:56:49.437 | INFO | MemInfo: { | 20260507 06:56:49.437 | INFO | "gpu" : " 17 GB", | 20260507 06:56:49.437 | INFO | "huge" : "", | 20260507 06:56:49.437 | INFO | "huge_shared" : " 27 GB", | 20260507 06:56:49.437 | INFO | "malloc" : " 14 GB", | 20260507 06:56:49.437 | INFO | "pinned" : " 29 GB" | 20260507 06:56:49.437 | INFO | } | 20260507 06:56:49.437 | INFO | Perf report for the last 5.14089 seconds: | 20260507 06:56:49.437 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:56:49.437 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:49.437 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:49.437 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:49.437 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 06:56:49.437 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:49.437 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:49.437 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:56:49.437 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:49.437 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:49.437 | INFO | Uploaded 36GiB to GPU in 2.05931s (17.4816GiB/s) | 20260507 06:56:49.437 | INFO | Downloaded 34GiB to GPU in 1.94793s (17.4544GiB/s) | 20260507 06:56:54.584 | INFO | Analyzing chunk frames = [7981568,7982080) | 20260507 06:56:54.584 | INFO | MemInfo: { | 20260507 06:56:54.584 | INFO | "gpu" : " 17 GB", | 20260507 06:56:54.584 | INFO | "huge" : "", | 20260507 06:56:54.584 | INFO | "huge_shared" : " 27 GB", | 20260507 06:56:54.584 | INFO | "malloc" : " 14 GB", | 20260507 06:56:54.584 | INFO | "pinned" : " 29 GB" | 20260507 06:56:54.584 | INFO | } | 20260507 06:56:54.584 | INFO | Perf report for the last 5.14705 seconds: | 20260507 06:56:54.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:56:54.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:54.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:54.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:54.584 | INFO | ANALYSIS : 60.4%, 3.0, 94.4ms, 0 | 20260507 06:56:54.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:54.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:54.584 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 06:56:54.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:54.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:54.584 | INFO | Uploaded 36GiB to GPU in 2.03845s (17.6605GiB/s) | 20260507 06:56:54.584 | INFO | Downloaded 34GiB to GPU in 1.93685s (17.5542GiB/s) | 20260507 06:56:59.727 | INFO | Analyzing chunk frames = [7982080,7982592) | 20260507 06:56:59.727 | INFO | MemInfo: { | 20260507 06:56:59.727 | INFO | "gpu" : " 17 GB", | 20260507 06:56:59.727 | INFO | "huge" : "", | 20260507 06:56:59.727 | INFO | "huge_shared" : " 27 GB", | 20260507 06:56:59.727 | INFO | "malloc" : " 14 GB", | 20260507 06:56:59.727 | INFO | "pinned" : " 29 GB" | 20260507 06:56:59.727 | INFO | } | 20260507 06:56:59.727 | INFO | Perf report for the last 5.14342 seconds: | 20260507 06:56:59.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:56:59.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:59.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:59.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:59.728 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:56:59.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:59.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:59.728 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:56:59.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:59.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:56:59.728 | INFO | Uploaded 36GiB to GPU in 1.93332s (18.6208GiB/s) | 20260507 06:56:59.728 | INFO | Downloaded 34GiB to GPU in 1.97251s (17.2369GiB/s) | 20260507 06:57:04.858 | INFO | Analyzing chunk frames = [7982592,7983104) | 20260507 06:57:04.858 | INFO | MemInfo: { | 20260507 06:57:04.858 | INFO | "gpu" : " 17 GB", | 20260507 06:57:04.858 | INFO | "huge" : "", | 20260507 06:57:04.858 | INFO | "huge_shared" : " 27 GB", | 20260507 06:57:04.858 | INFO | "malloc" : " 14 GB", | 20260507 06:57:04.858 | INFO | "pinned" : " 29 GB" | 20260507 06:57:04.858 | INFO | } | 20260507 06:57:04.858 | INFO | Perf report for the last 5.13094 seconds: | 20260507 06:57:04.858 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:57:04.858 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:04.858 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:04.858 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:04.858 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 06:57:04.858 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:04.858 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:04.859 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 06:57:04.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:04.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:04.859 | INFO | Uploaded 36GiB to GPU in 1.99478s (18.0471GiB/s) | 20260507 06:57:04.859 | INFO | Downloaded 34GiB to GPU in 1.95638s (17.379GiB/s) | 20260507 06:57:04.956 | INFO | Starting a round of Baz IO | 20260507 06:57:08.048 | INFO | Populating RTMetrics report | 20260507 06:57:08.048 | INFO | RTMetrics report populated | 20260507 06:57:08.048 | INFO | Opening tmp RTMetrics json file | 20260507 06:57:08.057 | INFO | Serializing RTMetrics to json | 20260507 06:57:08.064 | INFO | Serialization done, swapping tmp file now... | 20260507 06:57:08.064 | INFO | RTMetrics done | 20260507 06:57:08.135 | INFO | Finished a round of Baz IO: Wrote 1.656 GiB in 3.179 seconds | 20260507 06:57:08.135 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.59% / 0.00% / 67.41% | 20260507 06:57:08.135 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.113869% / 0% / 9.53674e-05% | 20260507 06:57:08.135 | INFO | Baz IO overall: | 20260507 06:57:08.135 | INFO | BytesWritten : 4013790545914 | 20260507 06:57:08.135 | INFO | NumEvents : 1922592243952 | 20260507 06:57:09.994 | INFO | Analyzing chunk frames = [7983104,7983616) | 20260507 06:57:09.994 | INFO | MemInfo: { | 20260507 06:57:09.994 | INFO | "gpu" : " 17 GB", | 20260507 06:57:09.994 | INFO | "huge" : "", | 20260507 06:57:09.994 | INFO | "huge_shared" : " 27 GB", | 20260507 06:57:09.994 | INFO | "malloc" : " 14 GB", | 20260507 06:57:09.994 | INFO | "pinned" : " 29 GB" | 20260507 06:57:09.994 | INFO | } | 20260507 06:57:09.995 | INFO | Perf report for the last 5.13609 seconds: | 20260507 06:57:09.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:57:09.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:09.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:09.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:09.995 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 06:57:09.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:09.995 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 06:57:09.995 | INFO | PRE_HQ : 58.9%, 1.8, 54.5ms, 0 | 20260507 06:57:09.995 | INFO | BAZWRITER : 61.5%, 3.1, 98.3ms, 0 | 20260507 06:57:09.995 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 06:57:09.995 | INFO | Uploaded 36GiB to GPU in 2.06785s (17.4094GiB/s) | 20260507 06:57:09.995 | INFO | Downloaded 38GiB to GPU in 2.17875s (17.4412GiB/s) | 20260507 06:57:15.141 | INFO | Analyzing chunk frames = [7983616,7984128) | 20260507 06:57:15.141 | INFO | MemInfo: { | 20260507 06:57:15.141 | INFO | "gpu" : " 17 GB", | 20260507 06:57:15.141 | INFO | "huge" : "", | 20260507 06:57:15.141 | INFO | "huge_shared" : " 27 GB", | 20260507 06:57:15.141 | INFO | "malloc" : " 14 GB", | 20260507 06:57:15.141 | INFO | "pinned" : " 29 GB" | 20260507 06:57:15.142 | INFO | } | 20260507 06:57:15.142 | INFO | Perf report for the last 5.14706 seconds: | 20260507 06:57:15.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:57:15.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:15.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:15.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:15.142 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 06:57:15.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:15.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:15.142 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 06:57:15.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:15.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:15.142 | INFO | Uploaded 36GiB to GPU in 2.01021s (17.9086GiB/s) | 20260507 06:57:15.142 | INFO | Downloaded 34GiB to GPU in 1.93301s (17.5891GiB/s) | 20260507 06:57:20.274 | INFO | Analyzing chunk frames = [7984128,7984640) | 20260507 06:57:20.274 | INFO | MemInfo: { | 20260507 06:57:20.274 | INFO | "gpu" : " 17 GB", | 20260507 06:57:20.274 | INFO | "huge" : "", | 20260507 06:57:20.274 | INFO | "huge_shared" : " 27 GB", | 20260507 06:57:20.274 | INFO | "malloc" : " 14 GB", | 20260507 06:57:20.274 | INFO | "pinned" : " 29 GB" | 20260507 06:57:20.274 | INFO | } | 20260507 06:57:20.274 | INFO | Perf report for the last 5.13282 seconds: | 20260507 06:57:20.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:57:20.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:20.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:20.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:20.275 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 06:57:20.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:20.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:20.275 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:57:20.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:20.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:20.275 | INFO | Uploaded 36GiB to GPU in 2.0517s (17.5464GiB/s) | 20260507 06:57:20.275 | INFO | Downloaded 34GiB to GPU in 1.95068s (17.4298GiB/s) | 20260507 06:57:25.418 | INFO | Analyzing chunk frames = [7984640,7985152) | 20260507 06:57:25.418 | INFO | MemInfo: { | 20260507 06:57:25.418 | INFO | "gpu" : " 17 GB", | 20260507 06:57:25.418 | INFO | "huge" : "", | 20260507 06:57:25.418 | INFO | "huge_shared" : " 27 GB", | 20260507 06:57:25.418 | INFO | "malloc" : " 14 GB", | 20260507 06:57:25.418 | INFO | "pinned" : " 29 GB" | 20260507 06:57:25.418 | INFO | } | 20260507 06:57:25.418 | INFO | Perf report for the last 5.14394 seconds: | 20260507 06:57:25.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:57:25.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:25.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:25.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:25.418 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 06:57:25.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:25.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:25.419 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:57:25.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:25.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:25.419 | INFO | Uploaded 36GiB to GPU in 2.01505s (17.8656GiB/s) | 20260507 06:57:25.419 | INFO | Downloaded 34GiB to GPU in 1.94269s (17.5015GiB/s) | 20260507 06:57:30.553 | INFO | Analyzing chunk frames = [7985152,7985664) | 20260507 06:57:30.553 | INFO | MemInfo: { | 20260507 06:57:30.553 | INFO | "gpu" : " 17 GB", | 20260507 06:57:30.553 | INFO | "huge" : "", | 20260507 06:57:30.553 | INFO | "huge_shared" : " 27 GB", | 20260507 06:57:30.553 | INFO | "malloc" : " 14 GB", | 20260507 06:57:30.553 | INFO | "pinned" : " 29 GB" | 20260507 06:57:30.553 | INFO | } | 20260507 06:57:30.553 | INFO | Perf report for the last 5.13471 seconds: | 20260507 06:57:30.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:57:30.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:30.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:30.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:30.553 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 06:57:30.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:30.553 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:30.553 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:57:30.553 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:30.553 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:30.553 | INFO | Uploaded 36GiB to GPU in 2.03417s (17.6976GiB/s) | 20260507 06:57:30.553 | INFO | Downloaded 34GiB to GPU in 1.91884s (17.719GiB/s) | 20260507 06:57:35.696 | INFO | Analyzing chunk frames = [7985664,7986176) | 20260507 06:57:35.696 | INFO | MemInfo: { | 20260507 06:57:35.696 | INFO | "gpu" : " 17 GB", | 20260507 06:57:35.696 | INFO | "huge" : "", | 20260507 06:57:35.696 | INFO | "huge_shared" : " 27 GB", | 20260507 06:57:35.696 | INFO | "malloc" : " 14 GB", | 20260507 06:57:35.696 | INFO | "pinned" : " 29 GB" | 20260507 06:57:35.696 | INFO | } | 20260507 06:57:35.696 | INFO | Perf report for the last 5.14322 seconds: | 20260507 06:57:35.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:57:35.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:35.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:35.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:35.697 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 06:57:35.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:35.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:35.697 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 06:57:35.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:35.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:35.697 | INFO | Uploaded 36GiB to GPU in 2.03274s (17.7101GiB/s) | 20260507 06:57:35.697 | INFO | Downloaded 34GiB to GPU in 1.90113s (17.8841GiB/s) | 20260507 06:57:40.833 | INFO | Analyzing chunk frames = [7986176,7986688) | 20260507 06:57:40.833 | INFO | MemInfo: { | 20260507 06:57:40.833 | INFO | "gpu" : " 17 GB", | 20260507 06:57:40.833 | INFO | "huge" : "", | 20260507 06:57:40.833 | INFO | "huge_shared" : " 27 GB", | 20260507 06:57:40.833 | INFO | "malloc" : " 14 GB", | 20260507 06:57:40.833 | INFO | "pinned" : " 29 GB" | 20260507 06:57:40.834 | INFO | } | 20260507 06:57:40.834 | INFO | Perf report for the last 5.13721 seconds: | 20260507 06:57:40.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:57:40.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:40.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:40.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:40.834 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 06:57:40.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:40.834 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:40.834 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 06:57:40.834 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:40.834 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:40.834 | INFO | Uploaded 36GiB to GPU in 2.03617s (17.6803GiB/s) | 20260507 06:57:40.834 | INFO | Downloaded 34GiB to GPU in 1.97239s (17.238GiB/s) | 20260507 06:57:45.973 | INFO | Analyzing chunk frames = [7986688,7987200) | 20260507 06:57:45.973 | INFO | MemInfo: { | 20260507 06:57:45.973 | INFO | "gpu" : " 17 GB", | 20260507 06:57:45.973 | INFO | "huge" : "", | 20260507 06:57:45.973 | INFO | "huge_shared" : " 27 GB", | 20260507 06:57:45.973 | INFO | "malloc" : " 14 GB", | 20260507 06:57:45.973 | INFO | "pinned" : " 29 GB" | 20260507 06:57:45.973 | INFO | } | 20260507 06:57:45.973 | INFO | Perf report for the last 5.13926 seconds: | 20260507 06:57:45.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:57:45.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:45.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:45.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:45.973 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 06:57:45.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:45.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:45.973 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 06:57:45.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:45.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:45.973 | INFO | Uploaded 36GiB to GPU in 1.99855s (18.0131GiB/s) | 20260507 06:57:45.973 | INFO | Downloaded 34GiB to GPU in 1.9609s (17.339GiB/s) | 20260507 06:57:45.973 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:57:45.973 | INFO | Total mem usage: 18 GiB | 20260507 06:57:45.973 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:57:45.973 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 06:57:45.973 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:57:45.973 | INFO | High Water Breakdown: | 20260507 06:57:45.973 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:57:45.973 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:57:45.973 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:57:45.973 | INFO | ------------------------------------------ | 20260507 06:57:45.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:57:45.973 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:57:45.973 | INFO | Total mem usage: 26.5 GiB | 20260507 06:57:45.973 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:57:45.973 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:57:45.973 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 06:57:45.973 | INFO | High Water Breakdown: | 20260507 06:57:45.973 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:57:45.973 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:57:45.973 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:57:45.974 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:57:45.974 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:57:45.974 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:57:45.974 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:57:45.974 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:57:45.974 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:57:45.974 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:57:45.974 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:57:45.974 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:57:45.974 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:57:45.974 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:57:45.974 | INFO | ------------------------------------------ | 20260507 06:57:45.974 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:57:45.974 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:57:45.974 | INFO | Total mem usage: 25.6 GiB | 20260507 06:57:45.974 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:57:45.974 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:57:45.974 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:57:45.974 | INFO | High Water Breakdown: | 20260507 06:57:45.974 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:57:45.974 | INFO | ------------------------------------------ | 20260507 06:57:45.974 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:57:45.974 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:57:45.974 | INFO | Total mem usage: 15.7 GiB | 20260507 06:57:45.974 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:57:45.974 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:57:45.974 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:57:45.974 | INFO | High Water Breakdown: | 20260507 06:57:45.974 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:57:45.974 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:57:45.974 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:57:45.974 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:57:45.974 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:57:45.974 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:57:45.974 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:57:45.974 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:57:45.974 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:57:45.974 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:57:45.974 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:57:45.974 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:57:45.974 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:57:45.974 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:57:45.974 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:57:45.974 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:57:45.974 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:57:45.974 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:57:45.974 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:57:45.974 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:57:45.974 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:57:45.974 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:57:45.974 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:57:45.974 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:57:45.974 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:57:45.974 | INFO | ------------------------------------------ | 20260507 06:57:45.974 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:57:45.974 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:57:45.974 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:57:45.974 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:57:45.975 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.50e+01, 1.34e+01 / 1.14e+00 | 20260507 06:57:45.975 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.03e-01, 9.82e-03 / 1.20e-02 | 20260507 06:57:45.975 | INFO | PerfReport: 11.64%, Baseline -- 3.57e+00 / 4.42e+00, 4.94e-01 / 2.57e-01 | 20260507 06:57:45.975 | INFO | PerfReport: 3.22%, Histogram -- 9.87e-01 / 1.47e+00, 1.64e-01 / 1.05e-01 | 20260507 06:57:45.975 | INFO | PerfReport: 16.28%, DME -- 4.00e+01 / 4.49e+01, 3.85e+00 / 1.46e+00 | 20260507 06:57:45.975 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.77e+01 / 1.98e+01, 1.28e+00 / 1.22e+00 | 20260507 06:57:45.975 | INFO | PerfReport: 6.91%, PulseAccumulating -- 2.12e+00 / 2.69e+00, 3.83e-01 / 1.83e-01 | 20260507 06:57:45.975 | INFO | PerfReport: 2.95%, Metrics -- 9.04e-01 / 1.23e+00, 2.42e-01 / 1.59e-01 | 20260507 06:57:45.975 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.03e-01 / 3.62e-01, 5.04e-02 / 2.17e-02 | 20260507 06:57:45.975 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:57:45.975 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:57:45.975 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:57:45.975 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 06:57:45.975 | INFO | PerfReport: 21.94%, Upload -- 2.11e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 06:57:45.975 | INFO | PerfReport: 21.36%, Download -- 2.06e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 06:57:45.975 | INFO | PerfReport: 31.89%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 06:57:45.975 | INFO | PerfReport: 1.68%, LockUpload -- 1.62e+00 / 0.00e+00, 5.01e+00 / 0.00e+00 | 20260507 06:57:45.975 | INFO | PerfReport: 1.13%, LockDownload -- 1.09e+00 / 0.00e+00, 2.71e+00 / 0.00e+00 | 20260507 06:57:45.975 | INFO | PerfReport: 21.98%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 06:57:45.975 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260507 06:57:46.069 | INFO | Starting a round of Baz IO | 20260507 06:57:49.164 | INFO | Populating RTMetrics report | 20260507 06:57:49.164 | INFO | RTMetrics report populated | 20260507 06:57:49.164 | INFO | Opening tmp RTMetrics json file | 20260507 06:57:49.181 | INFO | Serializing RTMetrics to json | 20260507 06:57:49.182 | INFO | Serialization done, swapping tmp file now... | 20260507 06:57:49.182 | INFO | RTMetrics done | 20260507 06:57:49.249 | INFO | Finished a round of Baz IO: Wrote 1.655 GiB in 3.18 seconds | 20260507 06:57:49.249 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.59% / 0.00% / 67.41% | 20260507 06:57:49.249 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.114107% / 0% / 5.96046e-05% | 20260507 06:57:49.249 | INFO | Baz IO overall: | 20260507 06:57:49.249 | INFO | BytesWritten : 4015567959523 | 20260507 06:57:49.249 | INFO | NumEvents : 1923423261868 | 20260507 06:57:51.110 | INFO | Analyzing chunk frames = [7987200,7987712) | 20260507 06:57:51.110 | INFO | MemInfo: { | 20260507 06:57:51.110 | INFO | "gpu" : " 17 GB", | 20260507 06:57:51.110 | INFO | "huge" : "", | 20260507 06:57:51.110 | INFO | "huge_shared" : " 27 GB", | 20260507 06:57:51.110 | INFO | "malloc" : " 14 GB", | 20260507 06:57:51.110 | INFO | "pinned" : " 29 GB" | 20260507 06:57:51.110 | INFO | } | 20260507 06:57:51.110 | INFO | Perf report for the last 5.13709 seconds: | 20260507 06:57:51.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:57:51.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:51.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:51.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:51.110 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 06:57:51.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:51.110 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260507 06:57:51.110 | INFO | PRE_HQ : 59.4%, 1.9, 58.2ms, 0 | 20260507 06:57:51.110 | INFO | BAZWRITER : 61.5%, 3.2, 102.3ms, 0 | 20260507 06:57:51.110 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 06:57:51.110 | INFO | Uploaded 36GiB to GPU in 2.04614s (17.5941GiB/s) | 20260507 06:57:51.110 | INFO | Downloaded 38GiB to GPU in 2.17607s (17.4627GiB/s) | 20260507 06:57:56.246 | INFO | Analyzing chunk frames = [7987712,7988224) | 20260507 06:57:56.246 | INFO | MemInfo: { | 20260507 06:57:56.246 | INFO | "gpu" : " 17 GB", | 20260507 06:57:56.246 | INFO | "huge" : "", | 20260507 06:57:56.246 | INFO | "huge_shared" : " 27 GB", | 20260507 06:57:56.246 | INFO | "malloc" : " 14 GB", | 20260507 06:57:56.246 | INFO | "pinned" : " 29 GB" | 20260507 06:57:56.246 | INFO | } | 20260507 06:57:56.246 | INFO | Perf report for the last 5.13626 seconds: | 20260507 06:57:56.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:57:56.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:56.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:56.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:56.246 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 06:57:56.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:56.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:56.246 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 06:57:56.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:56.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:57:56.246 | INFO | Uploaded 36GiB to GPU in 1.99848s (18.0137GiB/s) | 20260507 06:57:56.246 | INFO | Downloaded 34GiB to GPU in 1.92868s (17.6286GiB/s) | 20260507 06:58:01.398 | INFO | Analyzing chunk frames = [7988224,7988736) | 20260507 06:58:01.398 | INFO | MemInfo: { | 20260507 06:58:01.398 | INFO | "gpu" : " 17 GB", | 20260507 06:58:01.398 | INFO | "huge" : "", | 20260507 06:58:01.398 | INFO | "huge_shared" : " 27 GB", | 20260507 06:58:01.398 | INFO | "malloc" : " 14 GB", | 20260507 06:58:01.398 | INFO | "pinned" : " 29 GB" | 20260507 06:58:01.398 | INFO | } | 20260507 06:58:01.398 | INFO | Perf report for the last 5.15217 seconds: | 20260507 06:58:01.398 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:58:01.398 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:01.398 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:01.398 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:01.398 | INFO | ANALYSIS : 60.0%, 3.0, 93.1ms, 0 | 20260507 06:58:01.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:01.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:01.399 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:58:01.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:01.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:01.399 | INFO | Uploaded 36GiB to GPU in 2.01137s (17.8983GiB/s) | 20260507 06:58:01.399 | INFO | Downloaded 34GiB to GPU in 1.94232s (17.5048GiB/s) | 20260507 06:58:06.538 | INFO | Analyzing chunk frames = [7988736,7989248) | 20260507 06:58:06.538 | INFO | MemInfo: { | 20260507 06:58:06.538 | INFO | "gpu" : " 17 GB", | 20260507 06:58:06.538 | INFO | "huge" : "", | 20260507 06:58:06.538 | INFO | "huge_shared" : " 27 GB", | 20260507 06:58:06.539 | INFO | "malloc" : " 14 GB", | 20260507 06:58:06.539 | INFO | "pinned" : " 29 GB" | 20260507 06:58:06.539 | INFO | } | 20260507 06:58:06.539 | INFO | Perf report for the last 5.14031 seconds: | 20260507 06:58:06.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:58:06.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:06.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:06.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:06.539 | INFO | ANALYSIS : 60.1%, 3.0, 93.2ms, 0 | 20260507 06:58:06.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:06.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:06.539 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 06:58:06.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:06.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:06.539 | INFO | Uploaded 36GiB to GPU in 2.04196s (17.6302GiB/s) | 20260507 06:58:06.539 | INFO | Downloaded 34GiB to GPU in 1.9505s (17.4314GiB/s) | 20260507 06:58:11.674 | INFO | Analyzing chunk frames = [7989248,7989760) | 20260507 06:58:11.674 | INFO | MemInfo: { | 20260507 06:58:11.674 | INFO | "gpu" : " 17 GB", | 20260507 06:58:11.674 | INFO | "huge" : "", | 20260507 06:58:11.674 | INFO | "huge_shared" : " 27 GB", | 20260507 06:58:11.674 | INFO | "malloc" : " 14 GB", | 20260507 06:58:11.674 | INFO | "pinned" : " 29 GB" | 20260507 06:58:11.674 | INFO | } | 20260507 06:58:11.674 | INFO | Perf report for the last 5.13525 seconds: | 20260507 06:58:11.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:58:11.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:11.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:11.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:11.674 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 06:58:11.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:11.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:11.674 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:58:11.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:11.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:11.674 | INFO | Uploaded 36GiB to GPU in 2.06596s (17.4253GiB/s) | 20260507 06:58:11.674 | INFO | Downloaded 34GiB to GPU in 1.90865s (17.8137GiB/s) | 20260507 06:58:16.804 | INFO | Analyzing chunk frames = [7989760,7990272) | 20260507 06:58:16.804 | INFO | MemInfo: { | 20260507 06:58:16.804 | INFO | "gpu" : " 17 GB", | 20260507 06:58:16.804 | INFO | "huge" : "", | 20260507 06:58:16.804 | INFO | "huge_shared" : " 27 GB", | 20260507 06:58:16.804 | INFO | "malloc" : " 14 GB", | 20260507 06:58:16.804 | INFO | "pinned" : " 29 GB" | 20260507 06:58:16.804 | INFO | } | 20260507 06:58:16.804 | INFO | Perf report for the last 5.13 seconds: | 20260507 06:58:16.804 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:58:16.804 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:16.804 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:16.804 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:16.804 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 06:58:16.804 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:16.804 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:16.804 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:58:16.804 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:16.804 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:16.804 | INFO | Uploaded 36GiB to GPU in 2.07091s (17.3837GiB/s) | 20260507 06:58:16.804 | INFO | Downloaded 34GiB to GPU in 1.93269s (17.592GiB/s) | 20260507 06:58:21.947 | INFO | Analyzing chunk frames = [7990272,7990784) | 20260507 06:58:21.947 | INFO | MemInfo: { | 20260507 06:58:21.947 | INFO | "gpu" : " 17 GB", | 20260507 06:58:21.947 | INFO | "huge" : "", | 20260507 06:58:21.947 | INFO | "huge_shared" : " 27 GB", | 20260507 06:58:21.947 | INFO | "malloc" : " 14 GB", | 20260507 06:58:21.947 | INFO | "pinned" : " 29 GB" | 20260507 06:58:21.947 | INFO | } | 20260507 06:58:21.947 | INFO | Perf report for the last 5.14285 seconds: | 20260507 06:58:21.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:58:21.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:21.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:21.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:21.947 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 06:58:21.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:21.947 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:21.947 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 06:58:21.947 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:21.947 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:21.947 | INFO | Uploaded 36GiB to GPU in 1.98044s (18.1778GiB/s) | 20260507 06:58:21.947 | INFO | Downloaded 34GiB to GPU in 1.96215s (17.3279GiB/s) | 20260507 06:58:27.088 | INFO | Analyzing chunk frames = [7990784,7991296) | 20260507 06:58:27.088 | INFO | MemInfo: { | 20260507 06:58:27.088 | INFO | "gpu" : " 17 GB", | 20260507 06:58:27.088 | INFO | "huge" : "", | 20260507 06:58:27.088 | INFO | "huge_shared" : " 27 GB", | 20260507 06:58:27.088 | INFO | "malloc" : " 14 GB", | 20260507 06:58:27.088 | INFO | "pinned" : " 29 GB" | 20260507 06:58:27.088 | INFO | } | 20260507 06:58:27.088 | INFO | Perf report for the last 5.14121 seconds: | 20260507 06:58:27.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:58:27.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:27.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:27.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:27.088 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 06:58:27.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:27.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:27.088 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:58:27.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:27.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:27.088 | INFO | Uploaded 36GiB to GPU in 1.95232s (18.4396GiB/s) | 20260507 06:58:27.088 | INFO | Downloaded 34GiB to GPU in 1.96497s (17.303GiB/s) | 20260507 06:58:27.185 | INFO | Starting a round of Baz IO | 20260507 06:58:30.278 | INFO | Populating RTMetrics report | 20260507 06:58:30.278 | INFO | RTMetrics report populated | 20260507 06:58:30.278 | INFO | Opening tmp RTMetrics json file | 20260507 06:58:30.289 | INFO | Serializing RTMetrics to json | 20260507 06:58:30.293 | INFO | Serialization done, swapping tmp file now... | 20260507 06:58:30.293 | INFO | RTMetrics done | 20260507 06:58:30.363 | INFO | Finished a round of Baz IO: Wrote 1.654 GiB in 3.178 seconds | 20260507 06:58:30.363 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.59% / 0.00% / 67.41% | 20260507 06:58:30.363 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.11367% / 0% / 6.75519e-05% | 20260507 06:58:30.363 | INFO | Baz IO overall: | 20260507 06:58:30.363 | INFO | BytesWritten : 4017344458662 | 20260507 06:58:30.363 | INFO | NumEvents : 1924253860851 | 20260507 06:58:32.234 | INFO | Analyzing chunk frames = [7991296,7991808) | 20260507 06:58:32.234 | INFO | MemInfo: { | 20260507 06:58:32.234 | INFO | "gpu" : " 17 GB", | 20260507 06:58:32.234 | INFO | "huge" : "", | 20260507 06:58:32.234 | INFO | "huge_shared" : " 27 GB", | 20260507 06:58:32.234 | INFO | "malloc" : " 14 GB", | 20260507 06:58:32.234 | INFO | "pinned" : " 29 GB" | 20260507 06:58:32.234 | INFO | } | 20260507 06:58:32.235 | INFO | Perf report for the last 5.14644 seconds: | 20260507 06:58:32.235 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:58:32.235 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:32.235 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:32.235 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:32.235 | INFO | ANALYSIS : 61.4%, 3.0, 95.8ms, 0 | 20260507 06:58:32.235 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:32.235 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 06:58:32.235 | INFO | PRE_HQ : 58.8%, 1.8, 54.7ms, 0 | 20260507 06:58:32.235 | INFO | BAZWRITER : 61.3%, 3.1, 98.6ms, 0 | 20260507 06:58:32.235 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 06:58:32.235 | INFO | Uploaded 36GiB to GPU in 2.04407s (17.6119GiB/s) | 20260507 06:58:32.235 | INFO | Downloaded 38GiB to GPU in 2.177s (17.4552GiB/s) | 20260507 06:58:37.373 | INFO | Analyzing chunk frames = [7991808,7992320) | 20260507 06:58:37.373 | INFO | MemInfo: { | 20260507 06:58:37.373 | INFO | "gpu" : " 17 GB", | 20260507 06:58:37.373 | INFO | "huge" : "", | 20260507 06:58:37.373 | INFO | "huge_shared" : " 27 GB", | 20260507 06:58:37.373 | INFO | "malloc" : " 14 GB", | 20260507 06:58:37.373 | INFO | "pinned" : " 29 GB" | 20260507 06:58:37.373 | INFO | } | 20260507 06:58:37.373 | INFO | Perf report for the last 5.13864 seconds: | 20260507 06:58:37.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:58:37.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:37.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:37.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:37.373 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 06:58:37.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:37.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:37.373 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 06:58:37.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:37.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:37.374 | INFO | Uploaded 36GiB to GPU in 2.00742s (17.9335GiB/s) | 20260507 06:58:37.374 | INFO | Downloaded 34GiB to GPU in 1.94351s (17.4941GiB/s) | 20260507 06:58:42.505 | INFO | Analyzing chunk frames = [7992320,7992832) | 20260507 06:58:42.505 | INFO | MemInfo: { | 20260507 06:58:42.505 | INFO | "gpu" : " 17 GB", | 20260507 06:58:42.505 | INFO | "huge" : "", | 20260507 06:58:42.505 | INFO | "huge_shared" : " 27 GB", | 20260507 06:58:42.505 | INFO | "malloc" : " 14 GB", | 20260507 06:58:42.505 | INFO | "pinned" : " 29 GB" | 20260507 06:58:42.505 | INFO | } | 20260507 06:58:42.505 | INFO | Perf report for the last 5.13149 seconds: | 20260507 06:58:42.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:58:42.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:42.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:42.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:42.505 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 06:58:42.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:42.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:42.505 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 06:58:42.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:42.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:42.505 | INFO | Uploaded 36GiB to GPU in 2.05133s (17.5496GiB/s) | 20260507 06:58:42.505 | INFO | Downloaded 34GiB to GPU in 1.95403s (17.3999GiB/s) | 20260507 06:58:47.646 | INFO | Analyzing chunk frames = [7992832,7993344) | 20260507 06:58:47.646 | INFO | MemInfo: { | 20260507 06:58:47.646 | INFO | "gpu" : " 17 GB", | 20260507 06:58:47.646 | INFO | "huge" : "", | 20260507 06:58:47.646 | INFO | "huge_shared" : " 27 GB", | 20260507 06:58:47.646 | INFO | "malloc" : " 14 GB", | 20260507 06:58:47.646 | INFO | "pinned" : " 29 GB" | 20260507 06:58:47.646 | INFO | } | 20260507 06:58:47.646 | INFO | Perf report for the last 5.14155 seconds: | 20260507 06:58:47.646 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:58:47.646 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:47.646 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:47.646 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:47.646 | INFO | ANALYSIS : 60.3%, 2.9, 93.2ms, 0 | 20260507 06:58:47.646 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:47.646 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:47.646 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:58:47.646 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:47.646 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:47.647 | INFO | Uploaded 36GiB to GPU in 2.02856s (17.7466GiB/s) | 20260507 06:58:47.647 | INFO | Downloaded 34GiB to GPU in 1.95522s (17.3893GiB/s) | 20260507 06:58:52.783 | INFO | Analyzing chunk frames = [7993344,7993856) | 20260507 06:58:52.783 | INFO | MemInfo: { | 20260507 06:58:52.783 | INFO | "gpu" : " 17 GB", | 20260507 06:58:52.783 | INFO | "huge" : "", | 20260507 06:58:52.783 | INFO | "huge_shared" : " 27 GB", | 20260507 06:58:52.783 | INFO | "malloc" : " 14 GB", | 20260507 06:58:52.783 | INFO | "pinned" : " 29 GB" | 20260507 06:58:52.783 | INFO | } | 20260507 06:58:52.783 | INFO | Perf report for the last 5.1369 seconds: | 20260507 06:58:52.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:58:52.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:52.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:52.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:52.783 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 06:58:52.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:52.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:52.783 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 06:58:52.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:52.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:52.783 | INFO | Uploaded 36GiB to GPU in 2.05096s (17.5528GiB/s) | 20260507 06:58:52.784 | INFO | Downloaded 34GiB to GPU in 1.95431s (17.3975GiB/s) | 20260507 06:58:57.920 | INFO | Analyzing chunk frames = [7993856,7994368) | 20260507 06:58:57.920 | INFO | MemInfo: { | 20260507 06:58:57.920 | INFO | "gpu" : " 17 GB", | 20260507 06:58:57.920 | INFO | "huge" : "", | 20260507 06:58:57.920 | INFO | "huge_shared" : " 27 GB", | 20260507 06:58:57.920 | INFO | "malloc" : " 14 GB", | 20260507 06:58:57.920 | INFO | "pinned" : " 29 GB" | 20260507 06:58:57.921 | INFO | } | 20260507 06:58:57.921 | INFO | Perf report for the last 5.13749 seconds: | 20260507 06:58:57.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:58:57.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:57.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:57.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:57.921 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 06:58:57.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:57.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:57.921 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 06:58:57.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:57.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:58:57.921 | INFO | Uploaded 36GiB to GPU in 2.05894s (17.4847GiB/s) | 20260507 06:58:57.921 | INFO | Downloaded 34GiB to GPU in 1.92432s (17.6686GiB/s) | 20260507 06:59:03.059 | INFO | Analyzing chunk frames = [7994368,7994880) | 20260507 06:59:03.059 | INFO | MemInfo: { | 20260507 06:59:03.059 | INFO | "gpu" : " 17 GB", | 20260507 06:59:03.059 | INFO | "huge" : "", | 20260507 06:59:03.059 | INFO | "huge_shared" : " 27 GB", | 20260507 06:59:03.059 | INFO | "malloc" : " 14 GB", | 20260507 06:59:03.059 | INFO | "pinned" : " 29 GB" | 20260507 06:59:03.059 | INFO | } | 20260507 06:59:03.059 | INFO | Perf report for the last 5.13819 seconds: | 20260507 06:59:03.059 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:59:03.059 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:03.059 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:03.059 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:03.059 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 06:59:03.059 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:03.059 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:03.059 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 06:59:03.059 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:03.059 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:03.059 | INFO | Uploaded 36GiB to GPU in 2.0301s (17.7331GiB/s) | 20260507 06:59:03.059 | INFO | Downloaded 34GiB to GPU in 1.98959s (17.0889GiB/s) | 20260507 06:59:08.201 | INFO | Analyzing chunk frames = [7994880,7995392) | 20260507 06:59:08.201 | INFO | MemInfo: { | 20260507 06:59:08.201 | INFO | "gpu" : " 17 GB", | 20260507 06:59:08.201 | INFO | "huge" : "", | 20260507 06:59:08.201 | INFO | "huge_shared" : " 27 GB", | 20260507 06:59:08.201 | INFO | "malloc" : " 14 GB", | 20260507 06:59:08.201 | INFO | "pinned" : " 29 GB" | 20260507 06:59:08.201 | INFO | } | 20260507 06:59:08.201 | INFO | Perf report for the last 5.14208 seconds: | 20260507 06:59:08.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:59:08.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:08.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:08.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:08.201 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:59:08.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:08.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:08.201 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 06:59:08.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:08.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:08.201 | INFO | Uploaded 36GiB to GPU in 1.97714s (18.2081GiB/s) | 20260507 06:59:08.201 | INFO | Downloaded 34GiB to GPU in 1.96327s (17.3181GiB/s) | 20260507 06:59:08.202 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 06:59:08.202 | INFO | Total mem usage: 18 GiB | 20260507 06:59:08.202 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 06:59:08.202 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -27.5%) | 20260507 06:59:08.202 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 06:59:08.202 | INFO | High Water Breakdown: | 20260507 06:59:08.202 | INFO | 60% : 8.47 GiB ( 74.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 06:59:08.202 | INFO | 39.9% : 5.63 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 06:59:08.202 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 06:59:08.202 | INFO | ------------------------------------------ | 20260507 06:59:08.202 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:59:08.202 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 06:59:08.202 | INFO | Total mem usage: 26.5 GiB | 20260507 06:59:08.202 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 06:59:08.202 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 06:59:08.202 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 06:59:08.202 | INFO | High Water Breakdown: | 20260507 06:59:08.202 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:59:08.202 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:59:08.202 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:59:08.202 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:59:08.202 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:59:08.202 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:59:08.202 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 06:59:08.202 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 06:59:08.202 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:59:08.202 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 06:59:08.202 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 06:59:08.202 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 06:59:08.202 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 06:59:08.202 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 06:59:08.202 | INFO | ------------------------------------------ | 20260507 06:59:08.202 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:59:08.202 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 06:59:08.202 | INFO | Total mem usage: 25.6 GiB | 20260507 06:59:08.202 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 06:59:08.202 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 06:59:08.202 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 06:59:08.202 | INFO | High Water Breakdown: | 20260507 06:59:08.202 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 06:59:08.202 | INFO | ------------------------------------------ | 20260507 06:59:08.202 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:59:08.202 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 06:59:08.202 | INFO | Total mem usage: 15.7 GiB | 20260507 06:59:08.202 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 06:59:08.202 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 06:59:08.202 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 06:59:08.202 | INFO | High Water Breakdown: | 20260507 06:59:08.202 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 06:59:08.202 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 06:59:08.202 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 06:59:08.202 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 06:59:08.202 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 06:59:08.202 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 06:59:08.202 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 06:59:08.202 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 06:59:08.202 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 06:59:08.202 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 06:59:08.202 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 06:59:08.202 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 06:59:08.202 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 06:59:08.202 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 06:59:08.202 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 06:59:08.202 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 06:59:08.202 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 06:59:08.202 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 06:59:08.203 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 06:59:08.203 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 06:59:08.203 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 06:59:08.203 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 06:59:08.203 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 06:59:08.203 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 06:59:08.203 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 06:59:08.203 | INFO | ------------------------------------------ | 20260507 06:59:08.203 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 06:59:08.203 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 06:59:08.203 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:59:08.203 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:59:08.203 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.46e+01, 1.33e+01 / 1.14e+00 | 20260507 06:59:08.203 | INFO | PerfReport: 0.30%, AggregateStats -- 9.05e-02 / 1.04e-01, 1.04e-02 / 1.04e-02 | 20260507 06:59:08.203 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.51e+00, 4.92e-01 / 2.04e-01 | 20260507 06:59:08.203 | INFO | PerfReport: 3.21%, Histogram -- 9.81e-01 / 1.49e+00, 1.61e-01 / 9.31e-02 | 20260507 06:59:08.203 | INFO | PerfReport: 16.28%, DME -- 3.99e+01 / 4.47e+01, 4.00e+00 / 1.22e+00 | 20260507 06:59:08.203 | INFO | PerfReport: 57.69%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.21e+00 / 1.25e+00 | 20260507 06:59:08.203 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.12e+00 / 2.70e+00, 3.88e-01 / 2.09e-01 | 20260507 06:59:08.203 | INFO | PerfReport: 2.95%, Metrics -- 9.04e-01 / 1.21e+00, 2.43e-01 / 1.33e-01 | 20260507 06:59:08.203 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.03e-01 / 3.61e-01, 5.02e-02 / 2.57e-02 | 20260507 06:59:08.203 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 06:59:08.203 | INFO | PerfReport: Gathered 1536 samples. | 20260507 06:59:08.203 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 06:59:08.203 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 06:59:08.203 | INFO | PerfReport: 21.94%, Upload -- 2.11e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260507 06:59:08.203 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 06:59:08.203 | INFO | PerfReport: 31.85%, Compute -- 3.06e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 06:59:08.203 | INFO | PerfReport: 1.57%, LockUpload -- 1.51e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 06:59:08.203 | INFO | PerfReport: 1.27%, LockDownload -- 1.22e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260507 06:59:08.203 | INFO | PerfReport: 21.94%, LockCompute -- 2.11e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 06:59:08.203 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.38e-02 / 0.00e+00 | 20260507 06:59:08.298 | INFO | Starting a round of Baz IO | 20260507 06:59:11.394 | INFO | Populating RTMetrics report | 20260507 06:59:11.394 | INFO | RTMetrics report populated | 20260507 06:59:11.394 | INFO | Opening tmp RTMetrics json file | 20260507 06:59:11.412 | INFO | Serializing RTMetrics to json | 20260507 06:59:11.413 | INFO | Serialization done, swapping tmp file now... | 20260507 06:59:11.414 | INFO | RTMetrics done | 20260507 06:59:11.480 | INFO | Finished a round of Baz IO: Wrote 1.654 GiB in 3.182 seconds | 20260507 06:59:11.480 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.59% / 0.00% / 67.41% | 20260507 06:59:11.480 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.112883% / 7.94729e-06% / 9.13938e-05% | 20260507 06:59:11.480 | INFO | Baz IO overall: | 20260507 06:59:11.480 | INFO | BytesWritten : 4019120238259 | 20260507 06:59:11.480 | INFO | NumEvents : 1925084005417 | 20260507 06:59:13.350 | INFO | Analyzing chunk frames = [7995392,7995904) | 20260507 06:59:13.350 | INFO | MemInfo: { | 20260507 06:59:13.350 | INFO | "gpu" : " 17 GB", | 20260507 06:59:13.350 | INFO | "huge" : "", | 20260507 06:59:13.350 | INFO | "huge_shared" : " 27 GB", | 20260507 06:59:13.350 | INFO | "malloc" : " 14 GB", | 20260507 06:59:13.350 | INFO | "pinned" : " 29 GB" | 20260507 06:59:13.350 | INFO | } | 20260507 06:59:13.350 | INFO | Perf report for the last 5.14898 seconds: | 20260507 06:59:13.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:59:13.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:13.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:13.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:13.350 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260507 06:59:13.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:13.350 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 06:59:13.350 | INFO | PRE_HQ : 58.8%, 1.8, 54.5ms, 0 | 20260507 06:59:13.350 | INFO | BAZWRITER : 61.4%, 3.1, 97.6ms, 0 | 20260507 06:59:13.350 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 06:59:13.350 | INFO | Uploaded 36GiB to GPU in 2.05469s (17.5209GiB/s) | 20260507 06:59:13.350 | INFO | Downloaded 38GiB to GPU in 2.18384s (17.4006GiB/s) | 20260507 06:59:18.487 | INFO | Analyzing chunk frames = [7995904,7996416) | 20260507 06:59:18.487 | INFO | MemInfo: { | 20260507 06:59:18.487 | INFO | "gpu" : " 17 GB", | 20260507 06:59:18.487 | INFO | "huge" : "", | 20260507 06:59:18.487 | INFO | "huge_shared" : " 27 GB", | 20260507 06:59:18.487 | INFO | "malloc" : " 14 GB", | 20260507 06:59:18.487 | INFO | "pinned" : " 29 GB" | 20260507 06:59:18.487 | INFO | } | 20260507 06:59:18.487 | INFO | Perf report for the last 5.13714 seconds: | 20260507 06:59:18.487 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:59:18.487 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:18.487 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:18.487 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:18.487 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 06:59:18.487 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:18.487 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:18.487 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 06:59:18.487 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:18.487 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:18.487 | INFO | Uploaded 36GiB to GPU in 2.00088s (17.9921GiB/s) | 20260507 06:59:18.487 | INFO | Downloaded 34GiB to GPU in 1.94052s (17.5211GiB/s) | 20260507 06:59:23.618 | INFO | Analyzing chunk frames = [7996416,7996928) | 20260507 06:59:23.618 | INFO | MemInfo: { | 20260507 06:59:23.618 | INFO | "gpu" : " 17 GB", | 20260507 06:59:23.618 | INFO | "huge" : "", | 20260507 06:59:23.618 | INFO | "huge_shared" : " 27 GB", | 20260507 06:59:23.618 | INFO | "malloc" : " 14 GB", | 20260507 06:59:23.618 | INFO | "pinned" : " 29 GB" | 20260507 06:59:23.618 | INFO | } | 20260507 06:59:23.618 | INFO | Perf report for the last 5.13135 seconds: | 20260507 06:59:23.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:59:23.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:23.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:23.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:23.619 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 06:59:23.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:23.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:23.619 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 06:59:23.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:23.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:23.619 | INFO | Uploaded 36GiB to GPU in 2.04518s (17.6024GiB/s) | 20260507 06:59:23.619 | INFO | Downloaded 34GiB to GPU in 1.91486s (17.7559GiB/s) | 20260507 06:59:28.756 | INFO | Analyzing chunk frames = [7996928,7997440) | 20260507 06:59:28.756 | INFO | MemInfo: { | 20260507 06:59:28.756 | INFO | "gpu" : " 17 GB", | 20260507 06:59:28.756 | INFO | "huge" : "", | 20260507 06:59:28.756 | INFO | "huge_shared" : " 27 GB", | 20260507 06:59:28.756 | INFO | "malloc" : " 14 GB", | 20260507 06:59:28.756 | INFO | "pinned" : " 29 GB" | 20260507 06:59:28.756 | INFO | } | 20260507 06:59:28.756 | INFO | Perf report for the last 5.13759 seconds: | 20260507 06:59:28.756 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:59:28.756 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:28.756 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:28.756 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:28.756 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 06:59:28.756 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:28.756 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:28.756 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 06:59:28.756 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:28.756 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:28.756 | INFO | Uploaded 36GiB to GPU in 2.02625s (17.7668GiB/s) | 20260507 06:59:28.756 | INFO | Downloaded 34GiB to GPU in 1.94s (17.5258GiB/s) | 20260507 06:59:33.898 | INFO | Analyzing chunk frames = [7997440,7997952) | 20260507 06:59:33.898 | INFO | MemInfo: { | 20260507 06:59:33.898 | INFO | "gpu" : " 17 GB", | 20260507 06:59:33.898 | INFO | "huge" : "", | 20260507 06:59:33.898 | INFO | "huge_shared" : " 27 GB", | 20260507 06:59:33.898 | INFO | "malloc" : " 14 GB", | 20260507 06:59:33.898 | INFO | "pinned" : " 29 GB" | 20260507 06:59:33.898 | INFO | } | 20260507 06:59:33.898 | INFO | Perf report for the last 5.14215 seconds: | 20260507 06:59:33.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:59:33.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:33.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:33.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:33.898 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 06:59:33.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:33.898 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:33.898 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 06:59:33.898 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:33.899 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:33.899 | INFO | Uploaded 36GiB to GPU in 2.04431s (17.6098GiB/s) | 20260507 06:59:33.899 | INFO | Downloaded 34GiB to GPU in 1.93397s (17.5804GiB/s) | 20260507 06:59:39.040 | INFO | Analyzing chunk frames = [7997952,7998464) | 20260507 06:59:39.040 | INFO | MemInfo: { | 20260507 06:59:39.040 | INFO | "gpu" : " 17 GB", | 20260507 06:59:39.040 | INFO | "huge" : "", | 20260507 06:59:39.040 | INFO | "huge_shared" : " 27 GB", | 20260507 06:59:39.041 | INFO | "malloc" : " 14 GB", | 20260507 06:59:39.041 | INFO | "pinned" : " 29 GB" | 20260507 06:59:39.041 | INFO | } | 20260507 06:59:39.041 | INFO | Perf report for the last 5.14245 seconds: | 20260507 06:59:39.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:59:39.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:39.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:39.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:39.041 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 06:59:39.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:39.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:39.041 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 06:59:39.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:39.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:39.041 | INFO | Uploaded 36GiB to GPU in 2.03812s (17.6634GiB/s) | 20260507 06:59:39.041 | INFO | Downloaded 34GiB to GPU in 1.9013s (17.8825GiB/s) | 20260507 06:59:44.187 | INFO | Analyzing chunk frames = [7998464,7998976) | 20260507 06:59:44.187 | INFO | MemInfo: { | 20260507 06:59:44.187 | INFO | "gpu" : " 17 GB", | 20260507 06:59:44.187 | INFO | "huge" : "", | 20260507 06:59:44.187 | INFO | "huge_shared" : " 27 GB", | 20260507 06:59:44.187 | INFO | "malloc" : " 14 GB", | 20260507 06:59:44.187 | INFO | "pinned" : " 29 GB" | 20260507 06:59:44.187 | INFO | } | 20260507 06:59:44.187 | INFO | Perf report for the last 5.14624 seconds: | 20260507 06:59:44.187 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:59:44.187 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:44.187 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:44.187 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:44.187 | INFO | ANALYSIS : 60.3%, 3.0, 93.9ms, 0 | 20260507 06:59:44.187 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:44.187 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:44.187 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 06:59:44.187 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:44.187 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:44.187 | INFO | Uploaded 36GiB to GPU in 1.89296s (19.0179GiB/s) | 20260507 06:59:44.187 | INFO | Downloaded 34GiB to GPU in 1.96157s (17.3331GiB/s) | 20260507 06:59:49.317 | INFO | Analyzing chunk frames = [7998976,7999488) | 20260507 06:59:49.317 | INFO | MemInfo: { | 20260507 06:59:49.317 | INFO | "gpu" : " 17 GB", | 20260507 06:59:49.317 | INFO | "huge" : "", | 20260507 06:59:49.317 | INFO | "huge_shared" : " 27 GB", | 20260507 06:59:49.317 | INFO | "malloc" : " 14 GB", | 20260507 06:59:49.317 | INFO | "pinned" : " 29 GB" | 20260507 06:59:49.317 | INFO | } | 20260507 06:59:49.317 | INFO | Perf report for the last 5.1299 seconds: | 20260507 06:59:49.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:59:49.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:49.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:49.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:49.317 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 06:59:49.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:49.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:49.317 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 06:59:49.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:49.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:49.317 | INFO | Uploaded 36GiB to GPU in 1.99473s (18.0476GiB/s) | 20260507 06:59:49.317 | INFO | Downloaded 34GiB to GPU in 1.95546s (17.3872GiB/s) | 20260507 06:59:49.414 | INFO | Starting a round of Baz IO | 20260507 06:59:52.512 | INFO | Populating RTMetrics report | 20260507 06:59:52.512 | INFO | RTMetrics report populated | 20260507 06:59:52.512 | INFO | Opening tmp RTMetrics json file | 20260507 06:59:52.513 | INFO | Serializing RTMetrics to json | 20260507 06:59:52.514 | INFO | Serialization done, swapping tmp file now... | 20260507 06:59:52.514 | INFO | RTMetrics done | 20260507 06:59:52.599 | INFO | Finished a round of Baz IO: Wrote 1.655 GiB in 3.185 seconds | 20260507 06:59:52.599 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.59% / 0.00% / 67.41% | 20260507 06:59:52.599 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.112748% / 0% / 7.94729e-05% | 20260507 06:59:52.599 | INFO | Baz IO overall: | 20260507 06:59:52.599 | INFO | BytesWritten : 4020896756666 | 20260507 06:59:52.599 | INFO | NumEvents : 1925914668587 | 20260507 06:59:54.460 | INFO | Analyzing chunk frames = [7999488,8000000) | 20260507 06:59:54.460 | INFO | MemInfo: { | 20260507 06:59:54.460 | INFO | "gpu" : " 17 GB", | 20260507 06:59:54.460 | INFO | "huge" : "", | 20260507 06:59:54.460 | INFO | "huge_shared" : " 27 GB", | 20260507 06:59:54.460 | INFO | "malloc" : " 14 GB", | 20260507 06:59:54.460 | INFO | "pinned" : " 29 GB" | 20260507 06:59:54.460 | INFO | } | 20260507 06:59:54.460 | INFO | Perf report for the last 5.14359 seconds: | 20260507 06:59:54.460 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:59:54.460 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:54.460 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:54.461 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:54.461 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 06:59:54.461 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:54.461 | INFO | RT_METRICS : 44.5%, 1.0, 23.3ms, 0 | 20260507 06:59:54.461 | INFO | PRE_HQ : 59.7%, 1.9, 60.2ms, 0 | 20260507 06:59:54.461 | INFO | BAZWRITER : 61.5%, 3.3, 104.9ms, 0 | 20260507 06:59:54.461 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 06:59:54.461 | INFO | Uploaded 36GiB to GPU in 2.0664s (17.4216GiB/s) | 20260507 06:59:54.461 | INFO | Downloaded 38GiB to GPU in 2.17903s (17.4389GiB/s) | 20260507 06:59:59.594 | INFO | Analyzing chunk frames = [8000000,8000512) | 20260507 06:59:59.594 | INFO | MemInfo: { | 20260507 06:59:59.594 | INFO | "gpu" : " 17 GB", | 20260507 06:59:59.594 | INFO | "huge" : "", | 20260507 06:59:59.594 | INFO | "huge_shared" : " 27 GB", | 20260507 06:59:59.594 | INFO | "malloc" : " 14 GB", | 20260507 06:59:59.594 | INFO | "pinned" : " 29 GB" | 20260507 06:59:59.594 | INFO | } | 20260507 06:59:59.594 | INFO | Perf report for the last 5.13372 seconds: | 20260507 06:59:59.594 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 06:59:59.594 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:59.594 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:59.594 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:59.594 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 06:59:59.594 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:59.594 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:59.594 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 06:59:59.594 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:59.594 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 06:59:59.594 | INFO | Uploaded 36GiB to GPU in 2.01194s (17.8932GiB/s) | 20260507 06:59:59.595 | INFO | Downloaded 34GiB to GPU in 1.93019s (17.6149GiB/s) | 20260507 07:00:04.735 | INFO | Analyzing chunk frames = [8000512,8001024) | 20260507 07:00:04.735 | INFO | MemInfo: { | 20260507 07:00:04.735 | INFO | "gpu" : " 17 GB", | 20260507 07:00:04.735 | INFO | "huge" : "", | 20260507 07:00:04.735 | INFO | "huge_shared" : " 27 GB", | 20260507 07:00:04.735 | INFO | "malloc" : " 14 GB", | 20260507 07:00:04.735 | INFO | "pinned" : " 29 GB" | 20260507 07:00:04.735 | INFO | } | 20260507 07:00:04.735 | INFO | Perf report for the last 5.14117 seconds: | 20260507 07:00:04.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:00:04.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:04.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:04.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:04.735 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 07:00:04.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:04.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:04.735 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:00:04.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:04.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:04.736 | INFO | Uploaded 36GiB to GPU in 2.05232s (17.5412GiB/s) | 20260507 07:00:04.736 | INFO | Downloaded 34GiB to GPU in 1.95061s (17.4305GiB/s) | 20260507 07:00:09.872 | INFO | Analyzing chunk frames = [8001024,8001536) | 20260507 07:00:09.872 | INFO | MemInfo: { | 20260507 07:00:09.872 | INFO | "gpu" : " 17 GB", | 20260507 07:00:09.872 | INFO | "huge" : "", | 20260507 07:00:09.872 | INFO | "huge_shared" : " 27 GB", | 20260507 07:00:09.872 | INFO | "malloc" : " 14 GB", | 20260507 07:00:09.872 | INFO | "pinned" : " 29 GB" | 20260507 07:00:09.872 | INFO | } | 20260507 07:00:09.872 | INFO | Perf report for the last 5.13678 seconds: | 20260507 07:00:09.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:00:09.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:09.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:09.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:09.872 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 07:00:09.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:09.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:09.872 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:00:09.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:09.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:09.873 | INFO | Uploaded 36GiB to GPU in 2.04344s (17.6174GiB/s) | 20260507 07:00:09.873 | INFO | Downloaded 34GiB to GPU in 1.94993s (17.4366GiB/s) | 20260507 07:00:15.011 | INFO | Analyzing chunk frames = [8001536,8002048) | 20260507 07:00:15.011 | INFO | MemInfo: { | 20260507 07:00:15.011 | INFO | "gpu" : " 17 GB", | 20260507 07:00:15.011 | INFO | "huge" : "", | 20260507 07:00:15.011 | INFO | "huge_shared" : " 27 GB", | 20260507 07:00:15.011 | INFO | "malloc" : " 14 GB", | 20260507 07:00:15.011 | INFO | "pinned" : " 29 GB" | 20260507 07:00:15.011 | INFO | } | 20260507 07:00:15.011 | INFO | Perf report for the last 5.13913 seconds: | 20260507 07:00:15.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:00:15.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:15.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:15.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:15.011 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 07:00:15.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:15.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:15.012 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:00:15.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:15.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:15.012 | INFO | Uploaded 36GiB to GPU in 2.0584s (17.4893GiB/s) | 20260507 07:00:15.012 | INFO | Downloaded 34GiB to GPU in 1.95102s (17.4268GiB/s) | 20260507 07:00:20.154 | INFO | Analyzing chunk frames = [8002048,8002560) | 20260507 07:00:20.154 | INFO | MemInfo: { | 20260507 07:00:20.154 | INFO | "gpu" : " 17 GB", | 20260507 07:00:20.154 | INFO | "huge" : "", | 20260507 07:00:20.154 | INFO | "huge_shared" : " 27 GB", | 20260507 07:00:20.154 | INFO | "malloc" : " 14 GB", | 20260507 07:00:20.154 | INFO | "pinned" : " 29 GB" | 20260507 07:00:20.154 | INFO | } | 20260507 07:00:20.154 | INFO | Perf report for the last 5.14268 seconds: | 20260507 07:00:20.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:00:20.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:20.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:20.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:20.154 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 07:00:20.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:20.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:20.154 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260507 07:00:20.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:20.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:20.154 | INFO | Uploaded 36GiB to GPU in 2.04605s (17.5949GiB/s) | 20260507 07:00:20.155 | INFO | Downloaded 34GiB to GPU in 1.90233s (17.8728GiB/s) | 20260507 07:00:25.298 | INFO | Analyzing chunk frames = [8002560,8003072) | 20260507 07:00:25.298 | INFO | MemInfo: { | 20260507 07:00:25.298 | INFO | "gpu" : " 17 GB", | 20260507 07:00:25.298 | INFO | "huge" : "", | 20260507 07:00:25.298 | INFO | "huge_shared" : " 27 GB", | 20260507 07:00:25.298 | INFO | "malloc" : " 14 GB", | 20260507 07:00:25.298 | INFO | "pinned" : " 29 GB" | 20260507 07:00:25.298 | INFO | } | 20260507 07:00:25.298 | INFO | Perf report for the last 5.14446 seconds: | 20260507 07:00:25.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:00:25.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:25.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:25.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:25.299 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 07:00:25.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:25.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:25.299 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 07:00:25.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:25.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:25.299 | INFO | Uploaded 36GiB to GPU in 2.02096s (17.8133GiB/s) | 20260507 07:00:25.299 | INFO | Downloaded 34GiB to GPU in 1.98886s (17.0952GiB/s) | 20260507 07:00:30.433 | INFO | Analyzing chunk frames = [8003072,8003584) | 20260507 07:00:30.433 | INFO | MemInfo: { | 20260507 07:00:30.433 | INFO | "gpu" : " 17 GB", | 20260507 07:00:30.433 | INFO | "huge" : "", | 20260507 07:00:30.433 | INFO | "huge_shared" : " 27 GB", | 20260507 07:00:30.433 | INFO | "malloc" : " 14 GB", | 20260507 07:00:30.434 | INFO | "pinned" : " 29 GB" | 20260507 07:00:30.434 | INFO | } | 20260507 07:00:30.434 | INFO | Perf report for the last 5.13515 seconds: | 20260507 07:00:30.434 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:00:30.434 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:30.434 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:30.434 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:30.434 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:00:30.434 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:30.434 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:30.434 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 07:00:30.434 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:30.434 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:30.434 | INFO | Uploaded 36GiB to GPU in 1.98666s (18.1209GiB/s) | 20260507 07:00:30.434 | INFO | Downloaded 34GiB to GPU in 1.96637s (17.2908GiB/s) | 20260507 07:00:30.434 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:00:30.434 | INFO | Total mem usage: 18 GiB | 20260507 07:00:30.434 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:00:30.434 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.7%) | 20260507 07:00:30.434 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:00:30.434 | INFO | High Water Breakdown: | 20260507 07:00:30.434 | INFO | 60.2% : 8.62 GiB ( 75.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:00:30.434 | INFO | 39.6% : 5.68 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:00:30.434 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:00:30.434 | INFO | ------------------------------------------ | 20260507 07:00:30.434 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:00:30.434 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:00:30.434 | INFO | Total mem usage: 26.5 GiB | 20260507 07:00:30.434 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:00:30.434 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:00:30.434 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:00:30.434 | INFO | High Water Breakdown: | 20260507 07:00:30.434 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:00:30.434 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:00:30.434 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:00:30.434 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:00:30.434 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:00:30.434 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:00:30.434 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:00:30.434 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:00:30.434 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:00:30.434 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:00:30.434 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:00:30.434 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:00:30.434 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:00:30.434 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:00:30.434 | INFO | ------------------------------------------ | 20260507 07:00:30.434 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:00:30.434 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:00:30.434 | INFO | Total mem usage: 25.6 GiB | 20260507 07:00:30.434 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:00:30.434 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:00:30.435 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:00:30.435 | INFO | High Water Breakdown: | 20260507 07:00:30.435 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:00:30.435 | INFO | ------------------------------------------ | 20260507 07:00:30.435 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:00:30.435 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:00:30.435 | INFO | Total mem usage: 15.7 GiB | 20260507 07:00:30.435 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:00:30.435 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:00:30.435 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:00:30.435 | INFO | High Water Breakdown: | 20260507 07:00:30.435 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:00:30.435 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:00:30.435 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:00:30.435 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:00:30.435 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:00:30.435 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:00:30.435 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:00:30.435 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:00:30.435 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:00:30.435 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:00:30.435 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:00:30.435 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:00:30.435 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:00:30.435 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:00:30.435 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:00:30.435 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:00:30.435 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:00:30.435 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:00:30.435 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:00:30.435 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:00:30.435 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:00:30.435 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:00:30.435 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:00:30.435 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:00:30.435 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:00:30.435 | INFO | ------------------------------------------ | 20260507 07:00:30.435 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:00:30.435 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:00:30.435 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:00:30.435 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:00:30.435 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.34e+01 / 1.25e+00 | 20260507 07:00:30.435 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.02e-01, 1.04e-02 / 6.56e-03 | 20260507 07:00:30.435 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.48e+00, 5.02e-01 / 2.37e-01 | 20260507 07:00:30.435 | INFO | PerfReport: 3.22%, Histogram -- 9.86e-01 / 1.49e+00, 1.65e-01 / 1.02e-01 | 20260507 07:00:30.435 | INFO | PerfReport: 16.28%, DME -- 3.99e+01 / 4.47e+01, 3.87e+00 / 1.55e+00 | 20260507 07:00:30.435 | INFO | PerfReport: 57.67%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.23e+00 / 1.16e+00 | 20260507 07:00:30.435 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.12e+00 / 2.72e+00, 3.89e-01 / 1.81e-01 | 20260507 07:00:30.435 | INFO | PerfReport: 2.95%, Metrics -- 9.04e-01 / 1.22e+00, 2.43e-01 / 1.54e-01 | 20260507 07:00:30.435 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.62e-01, 5.04e-02 / 2.27e-02 | 20260507 07:00:30.435 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:00:30.435 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:00:30.435 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:00:30.435 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 07:00:30.435 | INFO | PerfReport: 21.93%, Upload -- 2.11e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260507 07:00:30.435 | INFO | PerfReport: 21.36%, Download -- 2.05e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260507 07:00:30.435 | INFO | PerfReport: 31.89%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 07:00:30.435 | INFO | PerfReport: 1.65%, LockUpload -- 1.59e+00 / 0.00e+00, 5.03e+00 / 0.00e+00 | 20260507 07:00:30.435 | INFO | PerfReport: 1.21%, LockDownload -- 1.16e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260507 07:00:30.435 | INFO | PerfReport: 21.94%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 07:00:30.435 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260507 07:00:30.530 | INFO | Starting a round of Baz IO | 20260507 07:00:33.624 | INFO | Populating RTMetrics report | 20260507 07:00:33.624 | INFO | RTMetrics report populated | 20260507 07:00:33.624 | INFO | Opening tmp RTMetrics json file | 20260507 07:00:33.640 | INFO | Serializing RTMetrics to json | 20260507 07:00:33.641 | INFO | Serialization done, swapping tmp file now... | 20260507 07:00:33.642 | INFO | RTMetrics done | 20260507 07:00:33.714 | INFO | Finished a round of Baz IO: Wrote 1.652 GiB in 3.183 seconds | 20260507 07:00:33.714 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.59% / 0.00% / 67.41% | 20260507 07:00:33.714 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.112224% / 0% / 5.16574e-05% | 20260507 07:00:33.714 | INFO | Baz IO overall: | 20260507 07:00:33.714 | INFO | BytesWritten : 4022670958027 | 20260507 07:00:33.714 | INFO | NumEvents : 1926744070301 | 20260507 07:00:35.565 | INFO | Analyzing chunk frames = [8003584,8004096) | 20260507 07:00:35.565 | INFO | MemInfo: { | 20260507 07:00:35.565 | INFO | "gpu" : " 17 GB", | 20260507 07:00:35.565 | INFO | "huge" : "", | 20260507 07:00:35.565 | INFO | "huge_shared" : " 27 GB", | 20260507 07:00:35.565 | INFO | "malloc" : " 14 GB", | 20260507 07:00:35.565 | INFO | "pinned" : " 29 GB" | 20260507 07:00:35.565 | INFO | } | 20260507 07:00:35.565 | INFO | Perf report for the last 5.13119 seconds: | 20260507 07:00:35.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:00:35.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:35.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:35.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:35.565 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 07:00:35.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:35.565 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 07:00:35.565 | INFO | PRE_HQ : 59.5%, 1.9, 58.4ms, 0 | 20260507 07:00:35.565 | INFO | BAZWRITER : 61.6%, 3.3, 102.8ms, 0 | 20260507 07:00:35.565 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:00:35.565 | INFO | Uploaded 36GiB to GPU in 2.05592s (17.5105GiB/s) | 20260507 07:00:35.565 | INFO | Downloaded 38GiB to GPU in 2.18408s (17.3986GiB/s) | 20260507 07:00:40.713 | INFO | Analyzing chunk frames = [8004096,8004608) | 20260507 07:00:40.713 | INFO | MemInfo: { | 20260507 07:00:40.713 | INFO | "gpu" : " 17 GB", | 20260507 07:00:40.713 | INFO | "huge" : "", | 20260507 07:00:40.713 | INFO | "huge_shared" : " 27 GB", | 20260507 07:00:40.713 | INFO | "malloc" : " 14 GB", | 20260507 07:00:40.713 | INFO | "pinned" : " 29 GB" | 20260507 07:00:40.713 | INFO | } | 20260507 07:00:40.713 | INFO | Perf report for the last 5.14827 seconds: | 20260507 07:00:40.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:00:40.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:40.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:40.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:40.713 | INFO | ANALYSIS : 60.7%, 3.0, 94.8ms, 0 | 20260507 07:00:40.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:40.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:40.713 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:00:40.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:40.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:40.714 | INFO | Uploaded 36GiB to GPU in 2.04618s (17.5938GiB/s) | 20260507 07:00:40.714 | INFO | Downloaded 34GiB to GPU in 1.94168s (17.5106GiB/s) | 20260507 07:00:45.855 | INFO | Analyzing chunk frames = [8004608,8005120) | 20260507 07:00:45.855 | INFO | MemInfo: { | 20260507 07:00:45.855 | INFO | "gpu" : " 17 GB", | 20260507 07:00:45.855 | INFO | "huge" : "", | 20260507 07:00:45.855 | INFO | "huge_shared" : " 27 GB", | 20260507 07:00:45.855 | INFO | "malloc" : " 14 GB", | 20260507 07:00:45.855 | INFO | "pinned" : " 29 GB" | 20260507 07:00:45.855 | INFO | } | 20260507 07:00:45.855 | INFO | Perf report for the last 5.14216 seconds: | 20260507 07:00:45.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:00:45.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:45.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:45.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:45.855 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 07:00:45.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:45.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:45.855 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:00:45.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:45.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:45.856 | INFO | Uploaded 36GiB to GPU in 2.03746s (17.669GiB/s) | 20260507 07:00:45.856 | INFO | Downloaded 34GiB to GPU in 1.95214s (17.4168GiB/s) | 20260507 07:00:50.993 | INFO | Analyzing chunk frames = [8005120,8005632) | 20260507 07:00:50.993 | INFO | MemInfo: { | 20260507 07:00:50.993 | INFO | "gpu" : " 17 GB", | 20260507 07:00:50.993 | INFO | "huge" : "", | 20260507 07:00:50.993 | INFO | "huge_shared" : " 27 GB", | 20260507 07:00:50.993 | INFO | "malloc" : " 14 GB", | 20260507 07:00:50.993 | INFO | "pinned" : " 29 GB" | 20260507 07:00:50.993 | INFO | } | 20260507 07:00:50.993 | INFO | Perf report for the last 5.13772 seconds: | 20260507 07:00:50.993 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:00:50.993 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:50.993 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:50.993 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:50.993 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 07:00:50.993 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:50.993 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:50.993 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:00:50.993 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:50.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:50.993 | INFO | Uploaded 36GiB to GPU in 2.04532s (17.6011GiB/s) | 20260507 07:00:50.993 | INFO | Downloaded 34GiB to GPU in 1.95095s (17.4274GiB/s) | 20260507 07:00:56.131 | INFO | Analyzing chunk frames = [8005632,8006144) | 20260507 07:00:56.131 | INFO | MemInfo: { | 20260507 07:00:56.131 | INFO | "gpu" : " 17 GB", | 20260507 07:00:56.131 | INFO | "huge" : "", | 20260507 07:00:56.131 | INFO | "huge_shared" : " 27 GB", | 20260507 07:00:56.131 | INFO | "malloc" : " 14 GB", | 20260507 07:00:56.131 | INFO | "pinned" : " 29 GB" | 20260507 07:00:56.131 | INFO | } | 20260507 07:00:56.131 | INFO | Perf report for the last 5.13812 seconds: | 20260507 07:00:56.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:00:56.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:56.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:56.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:56.131 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 07:00:56.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:56.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:56.131 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:00:56.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:56.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:00:56.132 | INFO | Uploaded 36GiB to GPU in 2.05479s (17.52GiB/s) | 20260507 07:00:56.132 | INFO | Downloaded 34GiB to GPU in 1.9119s (17.7833GiB/s) | 20260507 07:01:01.265 | INFO | Analyzing chunk frames = [8006144,8006656) | 20260507 07:01:01.265 | INFO | MemInfo: { | 20260507 07:01:01.265 | INFO | "gpu" : " 17 GB", | 20260507 07:01:01.265 | INFO | "huge" : "", | 20260507 07:01:01.265 | INFO | "huge_shared" : " 27 GB", | 20260507 07:01:01.265 | INFO | "malloc" : " 14 GB", | 20260507 07:01:01.265 | INFO | "pinned" : " 29 GB" | 20260507 07:01:01.265 | INFO | } | 20260507 07:01:01.266 | INFO | Perf report for the last 5.13434 seconds: | 20260507 07:01:01.266 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:01:01.266 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:01.266 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:01.266 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:01.266 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 07:01:01.266 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:01.266 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:01.266 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 07:01:01.266 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:01.266 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:01.266 | INFO | Uploaded 36GiB to GPU in 2.05799s (17.4928GiB/s) | 20260507 07:01:01.266 | INFO | Downloaded 34GiB to GPU in 1.91143s (17.7877GiB/s) | 20260507 07:01:06.406 | INFO | Analyzing chunk frames = [8006656,8007168) | 20260507 07:01:06.406 | INFO | MemInfo: { | 20260507 07:01:06.406 | INFO | "gpu" : " 17 GB", | 20260507 07:01:06.406 | INFO | "huge" : "", | 20260507 07:01:06.406 | INFO | "huge_shared" : " 27 GB", | 20260507 07:01:06.406 | INFO | "malloc" : " 14 GB", | 20260507 07:01:06.406 | INFO | "pinned" : " 29 GB" | 20260507 07:01:06.406 | INFO | } | 20260507 07:01:06.406 | INFO | Perf report for the last 5.14089 seconds: | 20260507 07:01:06.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:01:06.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:06.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:06.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:06.407 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:01:06.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:06.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:06.407 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 07:01:06.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:06.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:06.407 | INFO | Uploaded 36GiB to GPU in 1.96144s (18.3539GiB/s) | 20260507 07:01:06.407 | INFO | Downloaded 34GiB to GPU in 1.98126s (17.1608GiB/s) | 20260507 07:01:11.545 | INFO | Analyzing chunk frames = [8007168,8007680) | 20260507 07:01:11.545 | INFO | MemInfo: { | 20260507 07:01:11.545 | INFO | "gpu" : " 17 GB", | 20260507 07:01:11.545 | INFO | "huge" : "", | 20260507 07:01:11.545 | INFO | "huge_shared" : " 27 GB", | 20260507 07:01:11.545 | INFO | "malloc" : " 14 GB", | 20260507 07:01:11.545 | INFO | "pinned" : " 29 GB" | 20260507 07:01:11.545 | INFO | } | 20260507 07:01:11.545 | INFO | Perf report for the last 5.13869 seconds: | 20260507 07:01:11.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:01:11.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:11.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:11.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:11.545 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:01:11.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:11.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:11.545 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 07:01:11.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:11.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:11.546 | INFO | Uploaded 36GiB to GPU in 1.99119s (18.0796GiB/s) | 20260507 07:01:11.546 | INFO | Downloaded 34GiB to GPU in 1.96182s (17.3309GiB/s) | 20260507 07:01:11.642 | INFO | Starting a round of Baz IO | 20260507 07:01:14.738 | INFO | Populating RTMetrics report | 20260507 07:01:14.738 | INFO | RTMetrics report populated | 20260507 07:01:14.738 | INFO | Opening tmp RTMetrics json file | 20260507 07:01:14.746 | INFO | Serializing RTMetrics to json | 20260507 07:01:14.754 | INFO | Serialization done, swapping tmp file now... | 20260507 07:01:14.755 | INFO | RTMetrics done | 20260507 07:01:14.821 | INFO | Finished a round of Baz IO: Wrote 1.654 GiB in 3.179 seconds | 20260507 07:01:14.821 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.59% / 0.00% / 67.41% | 20260507 07:01:14.821 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.112406% / 0% / 8.34465e-05% | 20260507 07:01:14.821 | INFO | Baz IO overall: | 20260507 07:01:14.821 | INFO | BytesWritten : 4024446763431 | 20260507 07:01:14.821 | INFO | NumEvents : 1927574406361 | 20260507 07:01:16.686 | INFO | Analyzing chunk frames = [8007680,8008192) | 20260507 07:01:16.686 | INFO | MemInfo: { | 20260507 07:01:16.686 | INFO | "gpu" : " 17 GB", | 20260507 07:01:16.686 | INFO | "huge" : "", | 20260507 07:01:16.686 | INFO | "huge_shared" : " 27 GB", | 20260507 07:01:16.686 | INFO | "malloc" : " 14 GB", | 20260507 07:01:16.686 | INFO | "pinned" : " 29 GB" | 20260507 07:01:16.686 | INFO | } | 20260507 07:01:16.686 | INFO | Perf report for the last 5.14136 seconds: | 20260507 07:01:16.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:01:16.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:16.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:16.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:16.687 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 07:01:16.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:16.687 | INFO | RT_METRICS : 44.0%, 1.0, 23.1ms, 0 | 20260507 07:01:16.687 | INFO | PRE_HQ : 59.5%, 1.9, 59.0ms, 0 | 20260507 07:01:16.687 | INFO | BAZWRITER : 61.4%, 3.3, 103.0ms, 0 | 20260507 07:01:16.687 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:01:16.687 | INFO | Uploaded 36GiB to GPU in 2.05144s (17.5487GiB/s) | 20260507 07:01:16.687 | INFO | Downloaded 38GiB to GPU in 2.17195s (17.4958GiB/s) | 20260507 07:01:21.820 | INFO | Analyzing chunk frames = [8008192,8008704) | 20260507 07:01:21.820 | INFO | MemInfo: { | 20260507 07:01:21.820 | INFO | "gpu" : " 17 GB", | 20260507 07:01:21.820 | INFO | "huge" : "", | 20260507 07:01:21.820 | INFO | "huge_shared" : " 27 GB", | 20260507 07:01:21.820 | INFO | "malloc" : " 14 GB", | 20260507 07:01:21.820 | INFO | "pinned" : " 29 GB" | 20260507 07:01:21.820 | INFO | } | 20260507 07:01:21.821 | INFO | Perf report for the last 5.13409 seconds: | 20260507 07:01:21.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:01:21.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:21.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:21.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:21.821 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 07:01:21.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:21.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:21.821 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:01:21.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:21.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:21.821 | INFO | Uploaded 36GiB to GPU in 1.99695s (18.0275GiB/s) | 20260507 07:01:21.821 | INFO | Downloaded 34GiB to GPU in 1.93719s (17.5512GiB/s) | 20260507 07:01:26.962 | INFO | Analyzing chunk frames = [8008704,8009216) | 20260507 07:01:26.962 | INFO | MemInfo: { | 20260507 07:01:26.962 | INFO | "gpu" : " 17 GB", | 20260507 07:01:26.962 | INFO | "huge" : "", | 20260507 07:01:26.962 | INFO | "huge_shared" : " 27 GB", | 20260507 07:01:26.962 | INFO | "malloc" : " 14 GB", | 20260507 07:01:26.962 | INFO | "pinned" : " 29 GB" | 20260507 07:01:26.962 | INFO | } | 20260507 07:01:26.962 | INFO | Perf report for the last 5.14155 seconds: | 20260507 07:01:26.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:01:26.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:26.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:26.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:26.962 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260507 07:01:26.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:26.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:26.962 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:01:26.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:26.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:26.962 | INFO | Uploaded 36GiB to GPU in 2.01571s (17.8597GiB/s) | 20260507 07:01:26.962 | INFO | Downloaded 34GiB to GPU in 1.94043s (17.5219GiB/s) | 20260507 07:01:32.105 | INFO | Analyzing chunk frames = [8009216,8009728) | 20260507 07:01:32.105 | INFO | MemInfo: { | 20260507 07:01:32.105 | INFO | "gpu" : " 17 GB", | 20260507 07:01:32.105 | INFO | "huge" : "", | 20260507 07:01:32.105 | INFO | "huge_shared" : " 27 GB", | 20260507 07:01:32.105 | INFO | "malloc" : " 14 GB", | 20260507 07:01:32.105 | INFO | "pinned" : " 29 GB" | 20260507 07:01:32.105 | INFO | } | 20260507 07:01:32.105 | INFO | Perf report for the last 5.1429 seconds: | 20260507 07:01:32.105 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:01:32.105 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:32.105 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:32.105 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:32.105 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 07:01:32.105 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:32.105 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:32.105 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:01:32.105 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:32.105 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:32.105 | INFO | Uploaded 36GiB to GPU in 2.01503s (17.8657GiB/s) | 20260507 07:01:32.105 | INFO | Downloaded 34GiB to GPU in 1.93576s (17.5642GiB/s) | 20260507 07:01:37.246 | INFO | Analyzing chunk frames = [8009728,8010240) | 20260507 07:01:37.246 | INFO | MemInfo: { | 20260507 07:01:37.246 | INFO | "gpu" : " 17 GB", | 20260507 07:01:37.246 | INFO | "huge" : "", | 20260507 07:01:37.246 | INFO | "huge_shared" : " 27 GB", | 20260507 07:01:37.246 | INFO | "malloc" : " 14 GB", | 20260507 07:01:37.246 | INFO | "pinned" : " 29 GB" | 20260507 07:01:37.246 | INFO | } | 20260507 07:01:37.246 | INFO | Perf report for the last 5.14112 seconds: | 20260507 07:01:37.246 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:01:37.246 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:37.246 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:37.246 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:37.246 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 07:01:37.246 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:37.246 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:37.246 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:01:37.246 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:37.246 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:37.246 | INFO | Uploaded 36GiB to GPU in 2.03824s (17.6623GiB/s) | 20260507 07:01:37.247 | INFO | Downloaded 34GiB to GPU in 1.91345s (17.7689GiB/s) | 20260507 07:01:42.381 | INFO | Analyzing chunk frames = [8010240,8010752) | 20260507 07:01:42.381 | INFO | MemInfo: { | 20260507 07:01:42.381 | INFO | "gpu" : " 17 GB", | 20260507 07:01:42.381 | INFO | "huge" : "", | 20260507 07:01:42.381 | INFO | "huge_shared" : " 27 GB", | 20260507 07:01:42.381 | INFO | "malloc" : " 14 GB", | 20260507 07:01:42.381 | INFO | "pinned" : " 29 GB" | 20260507 07:01:42.381 | INFO | } | 20260507 07:01:42.382 | INFO | Perf report for the last 5.13537 seconds: | 20260507 07:01:42.382 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:01:42.382 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:42.382 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:42.382 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:42.382 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:01:42.382 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:42.382 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:42.382 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:01:42.382 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:42.382 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:42.382 | INFO | Uploaded 36GiB to GPU in 2.07228s (17.3722GiB/s) | 20260507 07:01:42.382 | INFO | Downloaded 34GiB to GPU in 1.93793s (17.5445GiB/s) | 20260507 07:01:47.518 | INFO | Analyzing chunk frames = [8010752,8011264) | 20260507 07:01:47.518 | INFO | MemInfo: { | 20260507 07:01:47.518 | INFO | "gpu" : " 17 GB", | 20260507 07:01:47.518 | INFO | "huge" : "", | 20260507 07:01:47.518 | INFO | "huge_shared" : " 27 GB", | 20260507 07:01:47.518 | INFO | "malloc" : " 14 GB", | 20260507 07:01:47.518 | INFO | "pinned" : " 29 GB" | 20260507 07:01:47.518 | INFO | } | 20260507 07:01:47.518 | INFO | Perf report for the last 5.13654 seconds: | 20260507 07:01:47.518 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:01:47.518 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:47.518 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:47.518 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:47.518 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 07:01:47.518 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:47.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:47.518 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:01:47.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:47.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:47.518 | INFO | Uploaded 36GiB to GPU in 2.04932s (17.5668GiB/s) | 20260507 07:01:47.518 | INFO | Downloaded 34GiB to GPU in 1.9867s (17.1138GiB/s) | 20260507 07:01:52.666 | INFO | Analyzing chunk frames = [8011264,8011776) | 20260507 07:01:52.666 | INFO | MemInfo: { | 20260507 07:01:52.666 | INFO | "gpu" : " 17 GB", | 20260507 07:01:52.666 | INFO | "huge" : "", | 20260507 07:01:52.666 | INFO | "huge_shared" : " 27 GB", | 20260507 07:01:52.666 | INFO | "malloc" : " 14 GB", | 20260507 07:01:52.667 | INFO | "pinned" : " 29 GB" | 20260507 07:01:52.667 | INFO | } | 20260507 07:01:52.667 | INFO | Perf report for the last 5.1484 seconds: | 20260507 07:01:52.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:01:52.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:52.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:52.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:52.667 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:01:52.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:52.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:52.667 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:01:52.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:52.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:52.667 | INFO | Uploaded 36GiB to GPU in 1.97844s (18.1962GiB/s) | 20260507 07:01:52.667 | INFO | Downloaded 34GiB to GPU in 1.96568s (17.2968GiB/s) | 20260507 07:01:52.667 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:01:52.667 | INFO | Total mem usage: 18 GiB | 20260507 07:01:52.667 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:01:52.667 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 07:01:52.667 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:01:52.667 | INFO | High Water Breakdown: | 20260507 07:01:52.667 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:01:52.667 | INFO | 39.7% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:01:52.667 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:01:52.667 | INFO | ------------------------------------------ | 20260507 07:01:52.667 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:01:52.667 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:01:52.667 | INFO | Total mem usage: 26.5 GiB | 20260507 07:01:52.667 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:01:52.667 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:01:52.667 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:01:52.667 | INFO | High Water Breakdown: | 20260507 07:01:52.667 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:01:52.667 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:01:52.667 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:01:52.667 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:01:52.667 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:01:52.667 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:01:52.667 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:01:52.667 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:01:52.667 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:01:52.667 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:01:52.667 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:01:52.667 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:01:52.668 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:01:52.668 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:01:52.668 | INFO | ------------------------------------------ | 20260507 07:01:52.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:01:52.668 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:01:52.668 | INFO | Total mem usage: 25.6 GiB | 20260507 07:01:52.668 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:01:52.668 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:01:52.668 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:01:52.668 | INFO | High Water Breakdown: | 20260507 07:01:52.668 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:01:52.668 | INFO | ------------------------------------------ | 20260507 07:01:52.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:01:52.668 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:01:52.668 | INFO | Total mem usage: 15.7 GiB | 20260507 07:01:52.668 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:01:52.668 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:01:52.668 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:01:52.668 | INFO | High Water Breakdown: | 20260507 07:01:52.668 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:01:52.668 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:01:52.668 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:01:52.668 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:01:52.668 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:01:52.668 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:01:52.668 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:01:52.668 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:01:52.668 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:01:52.668 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:01:52.668 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:01:52.668 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:01:52.668 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:01:52.668 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:01:52.668 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:01:52.668 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:01:52.668 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:01:52.668 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:01:52.668 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:01:52.668 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:01:52.668 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:01:52.668 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:01:52.668 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:01:52.668 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:01:52.668 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:01:52.668 | INFO | ------------------------------------------ | 20260507 07:01:52.668 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:01:52.668 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:01:52.668 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:01:52.668 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:01:52.668 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.44e+01, 1.34e+01 / 1.12e+00 | 20260507 07:01:52.668 | INFO | PerfReport: 0.30%, AggregateStats -- 9.05e-02 / 1.01e-01, 9.61e-03 / 4.74e-03 | 20260507 07:01:52.668 | INFO | PerfReport: 11.66%, Baseline -- 3.57e+00 / 4.50e+00, 4.99e-01 / 2.12e-01 | 20260507 07:01:52.668 | INFO | PerfReport: 3.21%, Histogram -- 9.82e-01 / 1.46e+00, 1.60e-01 / 1.18e-01 | 20260507 07:01:52.668 | INFO | PerfReport: 16.22%, DME -- 3.98e+01 / 4.43e+01, 3.69e+00 / 1.26e+00 | 20260507 07:01:52.668 | INFO | PerfReport: 57.75%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.27e+00 / 1.32e+00 | 20260507 07:01:52.668 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.12e+00 / 2.70e+00, 3.88e-01 / 2.11e-01 | 20260507 07:01:52.668 | INFO | PerfReport: 2.95%, Metrics -- 9.03e-01 / 1.21e+00, 2.43e-01 / 1.24e-01 | 20260507 07:01:52.668 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.65e-01, 5.16e-02 / 3.18e-02 | 20260507 07:01:52.669 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:01:52.669 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:01:52.669 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:01:52.669 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 07:01:52.669 | INFO | PerfReport: 21.99%, Upload -- 2.11e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 07:01:52.669 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 07:01:52.669 | INFO | PerfReport: 31.88%, Compute -- 3.06e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 07:01:52.669 | INFO | PerfReport: 1.58%, LockUpload -- 1.52e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 07:01:52.669 | INFO | PerfReport: 1.24%, LockDownload -- 1.19e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260507 07:01:52.669 | INFO | PerfReport: 21.91%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 07:01:52.669 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.62e-02 / 0.00e+00 | 20260507 07:01:52.764 | INFO | Starting a round of Baz IO | 20260507 07:01:55.856 | INFO | Populating RTMetrics report | 20260507 07:01:55.856 | INFO | RTMetrics report populated | 20260507 07:01:55.856 | INFO | Opening tmp RTMetrics json file | 20260507 07:01:55.870 | INFO | Serializing RTMetrics to json | 20260507 07:01:55.871 | INFO | Serialization done, swapping tmp file now... | 20260507 07:01:55.872 | INFO | RTMetrics done | 20260507 07:01:55.939 | INFO | Finished a round of Baz IO: Wrote 1.651 GiB in 3.175 seconds | 20260507 07:01:55.939 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.59% / 0.00% / 67.41% | 20260507 07:01:55.939 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.110718% / 0% / 6.35783e-05% | 20260507 07:01:55.939 | INFO | Baz IO overall: | 20260507 07:01:55.939 | INFO | BytesWritten : 4026219010626 | 20260507 07:01:55.939 | INFO | NumEvents : 1928402827452 | 20260507 07:01:57.810 | INFO | Analyzing chunk frames = [8011776,8012288) | 20260507 07:01:57.810 | INFO | MemInfo: { | 20260507 07:01:57.810 | INFO | "gpu" : " 17 GB", | 20260507 07:01:57.810 | INFO | "huge" : "", | 20260507 07:01:57.810 | INFO | "huge_shared" : " 27 GB", | 20260507 07:01:57.810 | INFO | "malloc" : " 14 GB", | 20260507 07:01:57.810 | INFO | "pinned" : " 29 GB" | 20260507 07:01:57.810 | INFO | } | 20260507 07:01:57.810 | INFO | Perf report for the last 5.14344 seconds: | 20260507 07:01:57.810 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:01:57.810 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:57.810 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:57.810 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:57.810 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 07:01:57.810 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:01:57.810 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:01:57.810 | INFO | PRE_HQ : 58.7%, 1.8, 54.5ms, 0 | 20260507 07:01:57.810 | INFO | BAZWRITER : 61.3%, 3.1, 97.4ms, 0 | 20260507 07:01:57.810 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:01:57.810 | INFO | Uploaded 36GiB to GPU in 2.06289s (17.4512GiB/s) | 20260507 07:01:57.810 | INFO | Downloaded 38GiB to GPU in 2.18309s (17.4065GiB/s) | 20260507 07:02:02.942 | INFO | Analyzing chunk frames = [8012288,8012800) | 20260507 07:02:02.942 | INFO | MemInfo: { | 20260507 07:02:02.942 | INFO | "gpu" : " 17 GB", | 20260507 07:02:02.942 | INFO | "huge" : "", | 20260507 07:02:02.942 | INFO | "huge_shared" : " 27 GB", | 20260507 07:02:02.942 | INFO | "malloc" : " 14 GB", | 20260507 07:02:02.942 | INFO | "pinned" : " 29 GB" | 20260507 07:02:02.942 | INFO | } | 20260507 07:02:02.942 | INFO | Perf report for the last 5.13193 seconds: | 20260507 07:02:02.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:02:02.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:02.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:02.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:02.942 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 07:02:02.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:02.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:02.942 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 07:02:02.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:02.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:02.942 | INFO | Uploaded 36GiB to GPU in 1.99945s (18.0049GiB/s) | 20260507 07:02:02.942 | INFO | Downloaded 34GiB to GPU in 1.94132s (17.5138GiB/s) | 20260507 07:02:08.074 | INFO | Analyzing chunk frames = [8012800,8013312) | 20260507 07:02:08.074 | INFO | MemInfo: { | 20260507 07:02:08.074 | INFO | "gpu" : " 17 GB", | 20260507 07:02:08.074 | INFO | "huge" : "", | 20260507 07:02:08.074 | INFO | "huge_shared" : " 27 GB", | 20260507 07:02:08.074 | INFO | "malloc" : " 14 GB", | 20260507 07:02:08.074 | INFO | "pinned" : " 29 GB" | 20260507 07:02:08.074 | INFO | } | 20260507 07:02:08.074 | INFO | Perf report for the last 5.13219 seconds: | 20260507 07:02:08.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:02:08.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:08.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:08.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:08.074 | INFO | ANALYSIS : 60.5%, 3.0, 93.5ms, 0 | 20260507 07:02:08.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:08.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:08.074 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:02:08.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:08.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:08.075 | INFO | Uploaded 36GiB to GPU in 2.04163s (17.633GiB/s) | 20260507 07:02:08.075 | INFO | Downloaded 34GiB to GPU in 1.95331s (17.4064GiB/s) | 20260507 07:02:13.214 | INFO | Analyzing chunk frames = [8013312,8013824) | 20260507 07:02:13.214 | INFO | MemInfo: { | 20260507 07:02:13.214 | INFO | "gpu" : " 17 GB", | 20260507 07:02:13.214 | INFO | "huge" : "", | 20260507 07:02:13.214 | INFO | "huge_shared" : " 27 GB", | 20260507 07:02:13.214 | INFO | "malloc" : " 14 GB", | 20260507 07:02:13.214 | INFO | "pinned" : " 29 GB" | 20260507 07:02:13.214 | INFO | } | 20260507 07:02:13.214 | INFO | Perf report for the last 5.14002 seconds: | 20260507 07:02:13.214 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:02:13.214 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:13.214 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:13.214 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:13.214 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 07:02:13.214 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:13.214 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:13.214 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:02:13.214 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:13.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:13.215 | INFO | Uploaded 36GiB to GPU in 2.00488s (17.9562GiB/s) | 20260507 07:02:13.215 | INFO | Downloaded 34GiB to GPU in 1.93858s (17.5386GiB/s) | 20260507 07:02:18.361 | INFO | Analyzing chunk frames = [8013824,8014336) | 20260507 07:02:18.361 | INFO | MemInfo: { | 20260507 07:02:18.361 | INFO | "gpu" : " 17 GB", | 20260507 07:02:18.361 | INFO | "huge" : "", | 20260507 07:02:18.361 | INFO | "huge_shared" : " 27 GB", | 20260507 07:02:18.361 | INFO | "malloc" : " 14 GB", | 20260507 07:02:18.361 | INFO | "pinned" : " 29 GB" | 20260507 07:02:18.362 | INFO | } | 20260507 07:02:18.362 | INFO | Perf report for the last 5.14755 seconds: | 20260507 07:02:18.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:02:18.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:18.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:18.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:18.362 | INFO | ANALYSIS : 59.8%, 3.0, 93.5ms, 0 | 20260507 07:02:18.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:18.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:18.362 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260507 07:02:18.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:18.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:18.362 | INFO | Uploaded 36GiB to GPU in 2.02708s (17.7595GiB/s) | 20260507 07:02:18.362 | INFO | Downloaded 34GiB to GPU in 1.94297s (17.499GiB/s) | 20260507 07:02:23.503 | INFO | Analyzing chunk frames = [8014336,8014848) | 20260507 07:02:23.503 | INFO | MemInfo: { | 20260507 07:02:23.503 | INFO | "gpu" : " 17 GB", | 20260507 07:02:23.503 | INFO | "huge" : "", | 20260507 07:02:23.503 | INFO | "huge_shared" : " 27 GB", | 20260507 07:02:23.503 | INFO | "malloc" : " 14 GB", | 20260507 07:02:23.503 | INFO | "pinned" : " 29 GB" | 20260507 07:02:23.503 | INFO | } | 20260507 07:02:23.503 | INFO | Perf report for the last 5.1412 seconds: | 20260507 07:02:23.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:02:23.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:23.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:23.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:23.503 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 07:02:23.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:23.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:23.503 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 07:02:23.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:23.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:23.503 | INFO | Uploaded 36GiB to GPU in 2.06601s (17.4249GiB/s) | 20260507 07:02:23.503 | INFO | Downloaded 34GiB to GPU in 1.9365s (17.5575GiB/s) | 20260507 07:02:28.638 | INFO | Analyzing chunk frames = [8014848,8015360) | 20260507 07:02:28.638 | INFO | MemInfo: { | 20260507 07:02:28.638 | INFO | "gpu" : " 17 GB", | 20260507 07:02:28.638 | INFO | "huge" : "", | 20260507 07:02:28.638 | INFO | "huge_shared" : " 27 GB", | 20260507 07:02:28.638 | INFO | "malloc" : " 14 GB", | 20260507 07:02:28.638 | INFO | "pinned" : " 29 GB" | 20260507 07:02:28.638 | INFO | } | 20260507 07:02:28.638 | INFO | Perf report for the last 5.13513 seconds: | 20260507 07:02:28.638 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:02:28.638 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:28.638 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:28.638 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:28.638 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:02:28.638 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:28.638 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:28.638 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 07:02:28.638 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:28.638 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:28.639 | INFO | Uploaded 36GiB to GPU in 1.95589s (18.4059GiB/s) | 20260507 07:02:28.639 | INFO | Downloaded 34GiB to GPU in 1.9823s (17.1518GiB/s) | 20260507 07:02:33.778 | INFO | Analyzing chunk frames = [8015360,8015872) | 20260507 07:02:33.778 | INFO | MemInfo: { | 20260507 07:02:33.778 | INFO | "gpu" : " 17 GB", | 20260507 07:02:33.778 | INFO | "huge" : "", | 20260507 07:02:33.778 | INFO | "huge_shared" : " 27 GB", | 20260507 07:02:33.778 | INFO | "malloc" : " 14 GB", | 20260507 07:02:33.778 | INFO | "pinned" : " 29 GB" | 20260507 07:02:33.778 | INFO | } | 20260507 07:02:33.778 | INFO | Perf report for the last 5.13983 seconds: | 20260507 07:02:33.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:02:33.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:33.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:33.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:33.778 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:02:33.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:33.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:33.778 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:02:33.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:33.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:33.778 | INFO | Uploaded 36GiB to GPU in 1.98231s (18.1606GiB/s) | 20260507 07:02:33.778 | INFO | Downloaded 34GiB to GPU in 1.95565s (17.3855GiB/s) | 20260507 07:02:33.876 | INFO | Starting a round of Baz IO | 20260507 07:02:36.967 | INFO | Populating RTMetrics report | 20260507 07:02:36.967 | INFO | RTMetrics report populated | 20260507 07:02:36.967 | INFO | Opening tmp RTMetrics json file | 20260507 07:02:36.973 | INFO | Serializing RTMetrics to json | 20260507 07:02:36.984 | INFO | Serialization done, swapping tmp file now... | 20260507 07:02:36.985 | INFO | RTMetrics done | 20260507 07:02:37.050 | INFO | Finished a round of Baz IO: Wrote 1.65 GiB in 3.174 seconds | 20260507 07:02:37.050 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.59% / 0.00% / 67.41% | 20260507 07:02:37.050 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.110805% / 1.58946e-05% / 6.35783e-05% | 20260507 07:02:37.050 | INFO | Baz IO overall: | 20260507 07:02:37.050 | INFO | BytesWritten : 4027991029166 | 20260507 07:02:37.050 | INFO | NumEvents : 1929231319484 | 20260507 07:02:38.916 | INFO | Analyzing chunk frames = [8015872,8016384) | 20260507 07:02:38.916 | INFO | MemInfo: { | 20260507 07:02:38.916 | INFO | "gpu" : " 17 GB", | 20260507 07:02:38.916 | INFO | "huge" : "", | 20260507 07:02:38.916 | INFO | "huge_shared" : " 27 GB", | 20260507 07:02:38.916 | INFO | "malloc" : " 14 GB", | 20260507 07:02:38.916 | INFO | "pinned" : " 29 GB" | 20260507 07:02:38.916 | INFO | } | 20260507 07:02:38.916 | INFO | Perf report for the last 5.13845 seconds: | 20260507 07:02:38.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:02:38.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:38.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:38.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:38.916 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 07:02:38.917 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:38.917 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 07:02:38.917 | INFO | PRE_HQ : 59.1%, 1.8, 54.7ms, 0 | 20260507 07:02:38.917 | INFO | BAZWRITER : 61.4%, 3.1, 98.3ms, 0 | 20260507 07:02:38.917 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260507 07:02:38.917 | INFO | Uploaded 36GiB to GPU in 2.03875s (17.6579GiB/s) | 20260507 07:02:38.917 | INFO | Downloaded 38GiB to GPU in 2.17197s (17.4957GiB/s) | 20260507 07:02:44.053 | INFO | Analyzing chunk frames = [8016384,8016896) | 20260507 07:02:44.053 | INFO | MemInfo: { | 20260507 07:02:44.053 | INFO | "gpu" : " 17 GB", | 20260507 07:02:44.053 | INFO | "huge" : "", | 20260507 07:02:44.053 | INFO | "huge_shared" : " 27 GB", | 20260507 07:02:44.053 | INFO | "malloc" : " 14 GB", | 20260507 07:02:44.053 | INFO | "pinned" : " 29 GB" | 20260507 07:02:44.053 | INFO | } | 20260507 07:02:44.053 | INFO | Perf report for the last 5.13699 seconds: | 20260507 07:02:44.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:02:44.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:44.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:44.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:44.053 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 07:02:44.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:44.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:44.054 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:02:44.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:44.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:44.054 | INFO | Uploaded 36GiB to GPU in 2.0094s (17.9158GiB/s) | 20260507 07:02:44.054 | INFO | Downloaded 34GiB to GPU in 1.93516s (17.5696GiB/s) | 20260507 07:02:49.193 | INFO | Analyzing chunk frames = [8016896,8017408) | 20260507 07:02:49.193 | INFO | MemInfo: { | 20260507 07:02:49.193 | INFO | "gpu" : " 17 GB", | 20260507 07:02:49.193 | INFO | "huge" : "", | 20260507 07:02:49.193 | INFO | "huge_shared" : " 27 GB", | 20260507 07:02:49.193 | INFO | "malloc" : " 14 GB", | 20260507 07:02:49.193 | INFO | "pinned" : " 29 GB" | 20260507 07:02:49.193 | INFO | } | 20260507 07:02:49.193 | INFO | Perf report for the last 5.13957 seconds: | 20260507 07:02:49.193 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:02:49.193 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:49.193 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:49.193 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:49.193 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 07:02:49.193 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:49.193 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:49.193 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:02:49.193 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:49.193 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:49.193 | INFO | Uploaded 36GiB to GPU in 2.05255s (17.5392GiB/s) | 20260507 07:02:49.193 | INFO | Downloaded 34GiB to GPU in 1.95006s (17.4354GiB/s) | 20260507 07:02:54.331 | INFO | Analyzing chunk frames = [8017408,8017920) | 20260507 07:02:54.331 | INFO | MemInfo: { | 20260507 07:02:54.331 | INFO | "gpu" : " 17 GB", | 20260507 07:02:54.331 | INFO | "huge" : "", | 20260507 07:02:54.331 | INFO | "huge_shared" : " 27 GB", | 20260507 07:02:54.331 | INFO | "malloc" : " 14 GB", | 20260507 07:02:54.331 | INFO | "pinned" : " 29 GB" | 20260507 07:02:54.331 | INFO | } | 20260507 07:02:54.331 | INFO | Perf report for the last 5.13777 seconds: | 20260507 07:02:54.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:02:54.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:54.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:54.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:54.331 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 07:02:54.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:54.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:54.331 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 07:02:54.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:54.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:54.331 | INFO | Uploaded 36GiB to GPU in 2.02198s (17.8043GiB/s) | 20260507 07:02:54.331 | INFO | Downloaded 34GiB to GPU in 1.94036s (17.5225GiB/s) | 20260507 07:02:59.470 | INFO | Analyzing chunk frames = [8017920,8018432) | 20260507 07:02:59.470 | INFO | MemInfo: { | 20260507 07:02:59.470 | INFO | "gpu" : " 17 GB", | 20260507 07:02:59.470 | INFO | "huge" : "", | 20260507 07:02:59.470 | INFO | "huge_shared" : " 27 GB", | 20260507 07:02:59.470 | INFO | "malloc" : " 14 GB", | 20260507 07:02:59.470 | INFO | "pinned" : " 29 GB" | 20260507 07:02:59.470 | INFO | } | 20260507 07:02:59.470 | INFO | Perf report for the last 5.13936 seconds: | 20260507 07:02:59.470 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:02:59.470 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:59.470 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:59.470 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:59.470 | INFO | ANALYSIS : 60.2%, 3.0, 94.0ms, 0 | 20260507 07:02:59.470 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:59.470 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:59.470 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:02:59.470 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:59.470 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:02:59.471 | INFO | Uploaded 36GiB to GPU in 2.05893s (17.4848GiB/s) | 20260507 07:02:59.471 | INFO | Downloaded 34GiB to GPU in 1.94739s (17.4592GiB/s) | 20260507 07:03:04.619 | INFO | Analyzing chunk frames = [8018432,8018944) | 20260507 07:03:04.619 | INFO | MemInfo: { | 20260507 07:03:04.619 | INFO | "gpu" : " 17 GB", | 20260507 07:03:04.619 | INFO | "huge" : "", | 20260507 07:03:04.619 | INFO | "huge_shared" : " 27 GB", | 20260507 07:03:04.619 | INFO | "malloc" : " 14 GB", | 20260507 07:03:04.619 | INFO | "pinned" : " 29 GB" | 20260507 07:03:04.619 | INFO | } | 20260507 07:03:04.619 | INFO | Perf report for the last 5.14921 seconds: | 20260507 07:03:04.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:03:04.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:04.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:04.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:04.619 | INFO | ANALYSIS : 60.2%, 3.0, 94.2ms, 0 | 20260507 07:03:04.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:04.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:04.619 | INFO | PRE_HQ : 26.8%, 1.0, 14.0ms, 0 | 20260507 07:03:04.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:04.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:04.620 | INFO | Uploaded 36GiB to GPU in 2.04642s (17.5917GiB/s) | 20260507 07:03:04.620 | INFO | Downloaded 34GiB to GPU in 1.91268s (17.7761GiB/s) | 20260507 07:03:09.757 | INFO | Analyzing chunk frames = [8018944,8019456) | 20260507 07:03:09.757 | INFO | MemInfo: { | 20260507 07:03:09.757 | INFO | "gpu" : " 17 GB", | 20260507 07:03:09.757 | INFO | "huge" : "", | 20260507 07:03:09.757 | INFO | "huge_shared" : " 27 GB", | 20260507 07:03:09.757 | INFO | "malloc" : " 14 GB", | 20260507 07:03:09.757 | INFO | "pinned" : " 29 GB" | 20260507 07:03:09.757 | INFO | } | 20260507 07:03:09.757 | INFO | Perf report for the last 5.13751 seconds: | 20260507 07:03:09.757 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:03:09.757 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:09.757 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:09.757 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:09.757 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 07:03:09.757 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:09.757 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:09.757 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:03:09.757 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:09.757 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:09.757 | INFO | Uploaded 36GiB to GPU in 1.96951s (18.2786GiB/s) | 20260507 07:03:09.757 | INFO | Downloaded 34GiB to GPU in 1.97063s (17.2533GiB/s) | 20260507 07:03:14.888 | INFO | Analyzing chunk frames = [8019456,8019968) | 20260507 07:03:14.888 | INFO | MemInfo: { | 20260507 07:03:14.888 | INFO | "gpu" : " 17 GB", | 20260507 07:03:14.888 | INFO | "huge" : "", | 20260507 07:03:14.888 | INFO | "huge_shared" : " 27 GB", | 20260507 07:03:14.888 | INFO | "malloc" : " 14 GB", | 20260507 07:03:14.888 | INFO | "pinned" : " 29 GB" | 20260507 07:03:14.888 | INFO | } | 20260507 07:03:14.888 | INFO | Perf report for the last 5.13125 seconds: | 20260507 07:03:14.888 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:03:14.888 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:14.888 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:14.888 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:14.888 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 07:03:14.888 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:14.888 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:14.888 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:03:14.888 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:14.888 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:14.888 | INFO | Uploaded 36GiB to GPU in 1.99898s (18.0092GiB/s) | 20260507 07:03:14.888 | INFO | Downloaded 34GiB to GPU in 1.96105s (17.3377GiB/s) | 20260507 07:03:14.889 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:03:14.889 | INFO | Total mem usage: 18 GiB | 20260507 07:03:14.889 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:03:14.889 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:03:14.889 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:03:14.889 | INFO | High Water Breakdown: | 20260507 07:03:14.889 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:03:14.889 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:03:14.889 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:03:14.889 | INFO | ------------------------------------------ | 20260507 07:03:14.889 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:03:14.889 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:03:14.889 | INFO | Total mem usage: 26.5 GiB | 20260507 07:03:14.889 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:03:14.889 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:03:14.889 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:03:14.889 | INFO | High Water Breakdown: | 20260507 07:03:14.889 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:03:14.889 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:03:14.889 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:03:14.889 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:03:14.889 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:03:14.889 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:03:14.889 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:03:14.889 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:03:14.889 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:03:14.889 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:03:14.889 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:03:14.889 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:03:14.889 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:03:14.889 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:03:14.889 | INFO | ------------------------------------------ | 20260507 07:03:14.889 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:03:14.889 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:03:14.889 | INFO | Total mem usage: 25.6 GiB | 20260507 07:03:14.889 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:03:14.889 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 07:03:14.889 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 07:03:14.889 | INFO | High Water Breakdown: | 20260507 07:03:14.889 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 07:03:14.889 | INFO | ------------------------------------------ | 20260507 07:03:14.889 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:03:14.889 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:03:14.889 | INFO | Total mem usage: 15.7 GiB | 20260507 07:03:14.889 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:03:14.889 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:03:14.889 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:03:14.889 | INFO | High Water Breakdown: | 20260507 07:03:14.889 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:03:14.889 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:03:14.889 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:03:14.889 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:03:14.889 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:03:14.889 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:03:14.889 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:03:14.889 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:03:14.889 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:03:14.889 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:03:14.889 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:03:14.889 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:03:14.889 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:03:14.889 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:03:14.889 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:03:14.889 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:03:14.889 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:03:14.889 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:03:14.890 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:03:14.890 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:03:14.890 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:03:14.890 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:03:14.890 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:03:14.890 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:03:14.890 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:03:14.890 | INFO | ------------------------------------------ | 20260507 07:03:14.890 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:03:14.890 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:03:14.890 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:03:14.890 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:03:14.890 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.35e+01 / 1.32e+00 | 20260507 07:03:14.890 | INFO | PerfReport: 0.30%, AggregateStats -- 9.05e-02 / 1.02e-01, 9.86e-03 / 4.96e-03 | 20260507 07:03:14.890 | INFO | PerfReport: 11.68%, Baseline -- 3.58e+00 / 4.48e+00, 5.03e-01 / 2.91e-01 | 20260507 07:03:14.890 | INFO | PerfReport: 3.21%, Histogram -- 9.85e-01 / 1.49e+00, 1.64e-01 / 1.11e-01 | 20260507 07:03:14.890 | INFO | PerfReport: 16.27%, DME -- 3.99e+01 / 4.46e+01, 3.92e+00 / 1.41e+00 | 20260507 07:03:14.890 | INFO | PerfReport: 57.67%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.24e+00 / 1.38e+00 | 20260507 07:03:14.890 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.12e+00 / 2.73e+00, 3.87e-01 / 1.66e-01 | 20260507 07:03:14.890 | INFO | PerfReport: 2.95%, Metrics -- 9.06e-01 / 1.23e+00, 2.46e-01 / 1.36e-01 | 20260507 07:03:14.890 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.03e-01 / 3.60e-01, 5.02e-02 / 1.94e-02 | 20260507 07:03:14.890 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:03:14.890 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:03:14.890 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:03:14.890 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 07:03:14.890 | INFO | PerfReport: 21.91%, Upload -- 2.11e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260507 07:03:14.890 | INFO | PerfReport: 21.43%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 07:03:14.890 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 07:03:14.890 | INFO | PerfReport: 1.51%, LockUpload -- 1.45e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 07:03:14.890 | INFO | PerfReport: 1.27%, LockDownload -- 1.22e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260507 07:03:14.890 | INFO | PerfReport: 21.96%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 07:03:14.890 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260507 07:03:14.985 | INFO | Starting a round of Baz IO | 20260507 07:03:18.080 | INFO | Populating RTMetrics report | 20260507 07:03:18.081 | INFO | RTMetrics report populated | 20260507 07:03:18.081 | INFO | Opening tmp RTMetrics json file | 20260507 07:03:18.098 | INFO | Serializing RTMetrics to json | 20260507 07:03:18.099 | INFO | Serialization done, swapping tmp file now... | 20260507 07:03:18.099 | INFO | RTMetrics done | 20260507 07:03:18.163 | INFO | Finished a round of Baz IO: Wrote 1.652 GiB in 3.178 seconds | 20260507 07:03:18.163 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.58% / 0.00% / 67.41% | 20260507 07:03:18.163 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.111401% / 0% / 7.15256e-05% | 20260507 07:03:18.163 | INFO | Baz IO overall: | 20260507 07:03:18.163 | INFO | BytesWritten : 4029765371989 | 20260507 07:03:18.163 | INFO | NumEvents : 1930060933141 | 20260507 07:03:20.030 | INFO | Analyzing chunk frames = [8019968,8020480) | 20260507 07:03:20.030 | INFO | MemInfo: { | 20260507 07:03:20.030 | INFO | "gpu" : " 17 GB", | 20260507 07:03:20.030 | INFO | "huge" : "", | 20260507 07:03:20.030 | INFO | "huge_shared" : " 27 GB", | 20260507 07:03:20.030 | INFO | "malloc" : " 14 GB", | 20260507 07:03:20.030 | INFO | "pinned" : " 29 GB" | 20260507 07:03:20.030 | INFO | } | 20260507 07:03:20.030 | INFO | Perf report for the last 5.14231 seconds: | 20260507 07:03:20.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:03:20.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:20.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:20.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:20.031 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 07:03:20.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:20.031 | INFO | RT_METRICS : 44.9%, 1.0, 23.5ms, 0 | 20260507 07:03:20.031 | INFO | PRE_HQ : 60.0%, 1.9, 60.0ms, 0 | 20260507 07:03:20.031 | INFO | BAZWRITER : 61.3%, 3.3, 104.6ms, 0 | 20260507 07:03:20.031 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:03:20.031 | INFO | Uploaded 36GiB to GPU in 2.04786s (17.5793GiB/s) | 20260507 07:03:20.031 | INFO | Downloaded 38GiB to GPU in 2.17542s (17.4679GiB/s) | 20260507 07:03:25.168 | INFO | Analyzing chunk frames = [8020480,8020992) | 20260507 07:03:25.168 | INFO | MemInfo: { | 20260507 07:03:25.168 | INFO | "gpu" : " 17 GB", | 20260507 07:03:25.168 | INFO | "huge" : "", | 20260507 07:03:25.168 | INFO | "huge_shared" : " 27 GB", | 20260507 07:03:25.168 | INFO | "malloc" : " 14 GB", | 20260507 07:03:25.169 | INFO | "pinned" : " 29 GB" | 20260507 07:03:25.169 | INFO | } | 20260507 07:03:25.169 | INFO | Perf report for the last 5.13823 seconds: | 20260507 07:03:25.169 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:03:25.169 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:25.169 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:25.169 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:25.169 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:03:25.169 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:25.169 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:25.169 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 07:03:25.169 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:25.169 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:25.169 | INFO | Uploaded 36GiB to GPU in 1.99904s (18.0086GiB/s) | 20260507 07:03:25.169 | INFO | Downloaded 34GiB to GPU in 1.92958s (17.6204GiB/s) | 20260507 07:03:30.316 | INFO | Analyzing chunk frames = [8020992,8021504) | 20260507 07:03:30.316 | INFO | MemInfo: { | 20260507 07:03:30.316 | INFO | "gpu" : " 17 GB", | 20260507 07:03:30.316 | INFO | "huge" : "", | 20260507 07:03:30.316 | INFO | "huge_shared" : " 27 GB", | 20260507 07:03:30.316 | INFO | "malloc" : " 14 GB", | 20260507 07:03:30.316 | INFO | "pinned" : " 29 GB" | 20260507 07:03:30.316 | INFO | } | 20260507 07:03:30.316 | INFO | Perf report for the last 5.14689 seconds: | 20260507 07:03:30.316 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:03:30.316 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:30.316 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:30.316 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:30.316 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 07:03:30.316 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:30.316 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:30.316 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:03:30.316 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:30.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:30.316 | INFO | Uploaded 36GiB to GPU in 2.05256s (17.5391GiB/s) | 20260507 07:03:30.316 | INFO | Downloaded 34GiB to GPU in 1.94988s (17.4369GiB/s) | 20260507 07:03:35.456 | INFO | Analyzing chunk frames = [8021504,8022016) | 20260507 07:03:35.456 | INFO | MemInfo: { | 20260507 07:03:35.456 | INFO | "gpu" : " 17 GB", | 20260507 07:03:35.456 | INFO | "huge" : "", | 20260507 07:03:35.456 | INFO | "huge_shared" : " 27 GB", | 20260507 07:03:35.456 | INFO | "malloc" : " 14 GB", | 20260507 07:03:35.456 | INFO | "pinned" : " 29 GB" | 20260507 07:03:35.456 | INFO | } | 20260507 07:03:35.456 | INFO | Perf report for the last 5.1403 seconds: | 20260507 07:03:35.456 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:03:35.456 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:35.456 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:35.456 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:35.456 | INFO | ANALYSIS : 60.2%, 2.9, 93.0ms, 0 | 20260507 07:03:35.456 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:35.456 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:35.456 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:03:35.456 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:35.456 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:35.456 | INFO | Uploaded 36GiB to GPU in 2.01967s (17.8247GiB/s) | 20260507 07:03:35.456 | INFO | Downloaded 34GiB to GPU in 1.94379s (17.4916GiB/s) | 20260507 07:03:40.583 | INFO | Analyzing chunk frames = [8022016,8022528) | 20260507 07:03:40.583 | INFO | MemInfo: { | 20260507 07:03:40.583 | INFO | "gpu" : " 17 GB", | 20260507 07:03:40.583 | INFO | "huge" : "", | 20260507 07:03:40.583 | INFO | "huge_shared" : " 27 GB", | 20260507 07:03:40.583 | INFO | "malloc" : " 14 GB", | 20260507 07:03:40.583 | INFO | "pinned" : " 29 GB" | 20260507 07:03:40.583 | INFO | } | 20260507 07:03:40.583 | INFO | Perf report for the last 5.12721 seconds: | 20260507 07:03:40.583 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:03:40.583 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:40.583 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:40.583 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:40.583 | INFO | ANALYSIS : 60.3%, 3.0, 93.6ms, 0 | 20260507 07:03:40.583 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:40.583 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:40.583 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:03:40.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:40.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:40.583 | INFO | Uploaded 36GiB to GPU in 2.06311s (17.4494GiB/s) | 20260507 07:03:40.583 | INFO | Downloaded 34GiB to GPU in 1.9091s (17.8095GiB/s) | 20260507 07:03:45.730 | INFO | Analyzing chunk frames = [8022528,8023040) | 20260507 07:03:45.730 | INFO | MemInfo: { | 20260507 07:03:45.730 | INFO | "gpu" : " 17 GB", | 20260507 07:03:45.730 | INFO | "huge" : "", | 20260507 07:03:45.730 | INFO | "huge_shared" : " 27 GB", | 20260507 07:03:45.730 | INFO | "malloc" : " 14 GB", | 20260507 07:03:45.730 | INFO | "pinned" : " 29 GB" | 20260507 07:03:45.730 | INFO | } | 20260507 07:03:45.730 | INFO | Perf report for the last 5.14671 seconds: | 20260507 07:03:45.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:03:45.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:45.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:45.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:45.730 | INFO | ANALYSIS : 60.2%, 3.0, 94.1ms, 0 | 20260507 07:03:45.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:45.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:45.730 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:03:45.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:45.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:45.730 | INFO | Uploaded 36GiB to GPU in 2.03561s (17.6851GiB/s) | 20260507 07:03:45.730 | INFO | Downloaded 34GiB to GPU in 1.89957s (17.8988GiB/s) | 20260507 07:03:50.866 | INFO | Analyzing chunk frames = [8023040,8023552) | 20260507 07:03:50.866 | INFO | MemInfo: { | 20260507 07:03:50.866 | INFO | "gpu" : " 17 GB", | 20260507 07:03:50.866 | INFO | "huge" : "", | 20260507 07:03:50.866 | INFO | "huge_shared" : " 27 GB", | 20260507 07:03:50.866 | INFO | "malloc" : " 14 GB", | 20260507 07:03:50.866 | INFO | "pinned" : " 29 GB" | 20260507 07:03:50.866 | INFO | } | 20260507 07:03:50.866 | INFO | Perf report for the last 5.13611 seconds: | 20260507 07:03:50.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:03:50.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:50.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:50.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:50.866 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 07:03:50.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:50.866 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:50.866 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:03:50.866 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:50.866 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:50.866 | INFO | Uploaded 36GiB to GPU in 1.96955s (18.2783GiB/s) | 20260507 07:03:50.867 | INFO | Downloaded 34GiB to GPU in 1.98075s (17.1652GiB/s) | 20260507 07:03:56.005 | INFO | Analyzing chunk frames = [8023552,8024064) | 20260507 07:03:56.005 | INFO | MemInfo: { | 20260507 07:03:56.005 | INFO | "gpu" : " 17 GB", | 20260507 07:03:56.005 | INFO | "huge" : "", | 20260507 07:03:56.005 | INFO | "huge_shared" : " 27 GB", | 20260507 07:03:56.005 | INFO | "malloc" : " 14 GB", | 20260507 07:03:56.005 | INFO | "pinned" : " 29 GB" | 20260507 07:03:56.005 | INFO | } | 20260507 07:03:56.005 | INFO | Perf report for the last 5.13935 seconds: | 20260507 07:03:56.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:03:56.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:56.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:56.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:56.005 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:03:56.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:56.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:56.005 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 07:03:56.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:56.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:03:56.006 | INFO | Uploaded 36GiB to GPU in 1.98695s (18.1182GiB/s) | 20260507 07:03:56.006 | INFO | Downloaded 34GiB to GPU in 1.95965s (17.35GiB/s) | 20260507 07:03:56.102 | INFO | Starting a round of Baz IO | 20260507 07:03:59.197 | INFO | Populating RTMetrics report | 20260507 07:03:59.198 | INFO | RTMetrics report populated | 20260507 07:03:59.198 | INFO | Opening tmp RTMetrics json file | 20260507 07:03:59.198 | INFO | Serializing RTMetrics to json | 20260507 07:03:59.200 | INFO | Serialization done, swapping tmp file now... | 20260507 07:03:59.200 | INFO | RTMetrics done | 20260507 07:03:59.283 | INFO | Finished a round of Baz IO: Wrote 1.651 GiB in 3.18 seconds | 20260507 07:03:59.283 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.58% / 0.00% / 67.42% | 20260507 07:03:59.283 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.110777% / 0% / 7.94729e-05% | 20260507 07:03:59.283 | INFO | Baz IO overall: | 20260507 07:03:59.283 | INFO | BytesWritten : 4031537781608 | 20260507 07:03:59.283 | INFO | NumEvents : 1930889155039 | 20260507 07:04:01.140 | INFO | Analyzing chunk frames = [8024064,8024576) | 20260507 07:04:01.140 | INFO | MemInfo: { | 20260507 07:04:01.140 | INFO | "gpu" : " 17 GB", | 20260507 07:04:01.140 | INFO | "huge" : "", | 20260507 07:04:01.140 | INFO | "huge_shared" : " 27 GB", | 20260507 07:04:01.140 | INFO | "malloc" : " 14 GB", | 20260507 07:04:01.140 | INFO | "pinned" : " 29 GB" | 20260507 07:04:01.141 | INFO | } | 20260507 07:04:01.141 | INFO | Perf report for the last 5.13526 seconds: | 20260507 07:04:01.141 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:04:01.141 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:01.141 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:01.141 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:01.141 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 07:04:01.141 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:01.141 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260507 07:04:01.141 | INFO | PRE_HQ : 58.9%, 1.8, 54.3ms, 0 | 20260507 07:04:01.141 | INFO | BAZWRITER : 61.5%, 3.1, 98.8ms, 0 | 20260507 07:04:01.141 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:04:01.141 | INFO | Uploaded 36GiB to GPU in 2.05667s (17.504GiB/s) | 20260507 07:04:01.141 | INFO | Downloaded 38GiB to GPU in 2.17531s (17.4688GiB/s) | 20260507 07:04:06.285 | INFO | Analyzing chunk frames = [8024576,8025088) | 20260507 07:04:06.285 | INFO | MemInfo: { | 20260507 07:04:06.285 | INFO | "gpu" : " 17 GB", | 20260507 07:04:06.285 | INFO | "huge" : "", | 20260507 07:04:06.285 | INFO | "huge_shared" : " 27 GB", | 20260507 07:04:06.285 | INFO | "malloc" : " 14 GB", | 20260507 07:04:06.285 | INFO | "pinned" : " 29 GB" | 20260507 07:04:06.285 | INFO | } | 20260507 07:04:06.285 | INFO | Perf report for the last 5.1444 seconds: | 20260507 07:04:06.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:04:06.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:06.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:06.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:06.285 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 07:04:06.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:06.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:06.285 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:04:06.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:06.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:06.285 | INFO | Uploaded 36GiB to GPU in 2.00536s (17.9519GiB/s) | 20260507 07:04:06.285 | INFO | Downloaded 34GiB to GPU in 1.93589s (17.563GiB/s) | 20260507 07:04:11.428 | INFO | Analyzing chunk frames = [8025088,8025600) | 20260507 07:04:11.428 | INFO | MemInfo: { | 20260507 07:04:11.428 | INFO | "gpu" : " 17 GB", | 20260507 07:04:11.428 | INFO | "huge" : "", | 20260507 07:04:11.428 | INFO | "huge_shared" : " 27 GB", | 20260507 07:04:11.428 | INFO | "malloc" : " 14 GB", | 20260507 07:04:11.428 | INFO | "pinned" : " 29 GB" | 20260507 07:04:11.428 | INFO | } | 20260507 07:04:11.428 | INFO | Perf report for the last 5.14295 seconds: | 20260507 07:04:11.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:04:11.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:11.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:11.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:11.428 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 07:04:11.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:11.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:11.428 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:04:11.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:11.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:11.428 | INFO | Uploaded 36GiB to GPU in 2.02633s (17.7661GiB/s) | 20260507 07:04:11.428 | INFO | Downloaded 34GiB to GPU in 1.95105s (17.4265GiB/s) | 20260507 07:04:16.563 | INFO | Analyzing chunk frames = [8025600,8026112) | 20260507 07:04:16.563 | INFO | MemInfo: { | 20260507 07:04:16.563 | INFO | "gpu" : " 17 GB", | 20260507 07:04:16.563 | INFO | "huge" : "", | 20260507 07:04:16.563 | INFO | "huge_shared" : " 27 GB", | 20260507 07:04:16.563 | INFO | "malloc" : " 14 GB", | 20260507 07:04:16.563 | INFO | "pinned" : " 29 GB" | 20260507 07:04:16.563 | INFO | } | 20260507 07:04:16.563 | INFO | Perf report for the last 5.13493 seconds: | 20260507 07:04:16.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:04:16.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:16.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:16.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:16.563 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 07:04:16.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:16.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:16.563 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:04:16.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:16.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:16.563 | INFO | Uploaded 36GiB to GPU in 2.04229s (17.6273GiB/s) | 20260507 07:04:16.563 | INFO | Downloaded 34GiB to GPU in 1.94955s (17.4399GiB/s) | 20260507 07:04:21.702 | INFO | Analyzing chunk frames = [8026112,8026624) | 20260507 07:04:21.702 | INFO | MemInfo: { | 20260507 07:04:21.702 | INFO | "gpu" : " 17 GB", | 20260507 07:04:21.702 | INFO | "huge" : "", | 20260507 07:04:21.702 | INFO | "huge_shared" : " 27 GB", | 20260507 07:04:21.702 | INFO | "malloc" : " 14 GB", | 20260507 07:04:21.702 | INFO | "pinned" : " 29 GB" | 20260507 07:04:21.702 | INFO | } | 20260507 07:04:21.702 | INFO | Perf report for the last 5.13914 seconds: | 20260507 07:04:21.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:04:21.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:21.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:21.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:21.702 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 07:04:21.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:21.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:21.702 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:04:21.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:21.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:21.702 | INFO | Uploaded 36GiB to GPU in 2.04967s (17.5638GiB/s) | 20260507 07:04:21.702 | INFO | Downloaded 34GiB to GPU in 1.90245s (17.8717GiB/s) | 20260507 07:04:26.838 | INFO | Analyzing chunk frames = [8026624,8027136) | 20260507 07:04:26.838 | INFO | MemInfo: { | 20260507 07:04:26.838 | INFO | "gpu" : " 17 GB", | 20260507 07:04:26.838 | INFO | "huge" : "", | 20260507 07:04:26.838 | INFO | "huge_shared" : " 27 GB", | 20260507 07:04:26.838 | INFO | "malloc" : " 14 GB", | 20260507 07:04:26.838 | INFO | "pinned" : " 29 GB" | 20260507 07:04:26.838 | INFO | } | 20260507 07:04:26.838 | INFO | Perf report for the last 5.13633 seconds: | 20260507 07:04:26.838 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:04:26.838 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:26.838 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:26.838 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:26.838 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:04:26.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:26.839 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:26.839 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:04:26.839 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:26.839 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:26.839 | INFO | Uploaded 36GiB to GPU in 2.07076s (17.3849GiB/s) | 20260507 07:04:26.839 | INFO | Downloaded 34GiB to GPU in 1.92436s (17.6682GiB/s) | 20260507 07:04:31.981 | INFO | Analyzing chunk frames = [8027136,8027648) | 20260507 07:04:31.981 | INFO | MemInfo: { | 20260507 07:04:31.981 | INFO | "gpu" : " 17 GB", | 20260507 07:04:31.981 | INFO | "huge" : "", | 20260507 07:04:31.981 | INFO | "huge_shared" : " 27 GB", | 20260507 07:04:31.981 | INFO | "malloc" : " 14 GB", | 20260507 07:04:31.981 | INFO | "pinned" : " 29 GB" | 20260507 07:04:31.981 | INFO | } | 20260507 07:04:31.981 | INFO | Perf report for the last 5.14297 seconds: | 20260507 07:04:31.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:04:31.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:31.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:31.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:31.982 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 07:04:31.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:31.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:31.982 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 07:04:31.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:31.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:31.982 | INFO | Uploaded 36GiB to GPU in 1.94328s (18.5254GiB/s) | 20260507 07:04:31.982 | INFO | Downloaded 34GiB to GPU in 1.97557s (17.2102GiB/s) | 20260507 07:04:37.118 | INFO | Analyzing chunk frames = [8027648,8028160) | 20260507 07:04:37.118 | INFO | MemInfo: { | 20260507 07:04:37.118 | INFO | "gpu" : " 17 GB", | 20260507 07:04:37.118 | INFO | "huge" : "", | 20260507 07:04:37.118 | INFO | "huge_shared" : " 27 GB", | 20260507 07:04:37.118 | INFO | "malloc" : " 14 GB", | 20260507 07:04:37.118 | INFO | "pinned" : " 29 GB" | 20260507 07:04:37.118 | INFO | } | 20260507 07:04:37.119 | INFO | Perf report for the last 5.13746 seconds: | 20260507 07:04:37.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:04:37.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:37.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:37.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:37.119 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:04:37.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:37.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:37.119 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:04:37.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:37.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:37.119 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:04:37.119 | INFO | Total mem usage: 18 GiB | 20260507 07:04:37.119 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:04:37.119 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.7%) | 20260507 07:04:37.119 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:04:37.119 | INFO | High Water Breakdown: | 20260507 07:04:37.119 | INFO | 60.2% : 8.63 GiB ( 75.6% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:04:37.119 | INFO | 39.6% : 5.68 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:04:37.119 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:04:37.119 | INFO | ------------------------------------------ | 20260507 07:04:37.119 | INFO | Uploaded 36GiB to GPU in 1.98156s (18.1675GiB/s) | 20260507 07:04:37.119 | INFO | Downloaded 34GiB to GPU in 1.95953s (17.3511GiB/s) | 20260507 07:04:37.119 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:04:37.119 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:04:37.119 | INFO | Total mem usage: 26.5 GiB | 20260507 07:04:37.119 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:04:37.119 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:04:37.119 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:04:37.119 | INFO | High Water Breakdown: | 20260507 07:04:37.119 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:04:37.119 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:04:37.119 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:04:37.119 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:04:37.119 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:04:37.119 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:04:37.119 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:04:37.119 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:04:37.119 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:04:37.119 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:04:37.119 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:04:37.119 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:04:37.119 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:04:37.119 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:04:37.119 | INFO | ------------------------------------------ | 20260507 07:04:37.119 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:04:37.120 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:04:37.120 | INFO | Total mem usage: 25.6 GiB | 20260507 07:04:37.120 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:04:37.120 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:04:37.120 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:04:37.120 | INFO | High Water Breakdown: | 20260507 07:04:37.120 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:04:37.120 | INFO | ------------------------------------------ | 20260507 07:04:37.120 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:04:37.120 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:04:37.120 | INFO | Total mem usage: 15.7 GiB | 20260507 07:04:37.120 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:04:37.120 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:04:37.120 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:04:37.120 | INFO | High Water Breakdown: | 20260507 07:04:37.120 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:04:37.120 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:04:37.120 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:04:37.120 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:04:37.120 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:04:37.120 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:04:37.120 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:04:37.120 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:04:37.120 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:04:37.120 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:04:37.120 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:04:37.120 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:04:37.120 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:04:37.120 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:04:37.120 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:04:37.120 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:04:37.120 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:04:37.120 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:04:37.120 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:04:37.120 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:04:37.120 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:04:37.120 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:04:37.120 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:04:37.120 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:04:37.120 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:04:37.120 | INFO | ------------------------------------------ | 20260507 07:04:37.120 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:04:37.120 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:04:37.120 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:04:37.120 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:04:37.120 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.35e+01 / 1.36e+00 | 20260507 07:04:37.120 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.02e-01, 1.03e-02 / 4.86e-03 | 20260507 07:04:37.120 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.41e+00, 5.10e-01 / 2.97e-01 | 20260507 07:04:37.120 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.50e+00, 1.68e-01 / 1.04e-01 | 20260507 07:04:37.120 | INFO | PerfReport: 16.31%, DME -- 4.01e+01 / 4.52e+01, 3.99e+00 / 1.88e+00 | 20260507 07:04:37.120 | INFO | PerfReport: 57.65%, FrameLabeling -- 1.77e+01 / 1.93e+01, 1.26e+00 / 1.26e+00 | 20260507 07:04:37.120 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.73e+00, 3.86e-01 / 1.59e-01 | 20260507 07:04:37.120 | INFO | PerfReport: 2.94%, Metrics -- 9.02e-01 / 1.22e+00, 2.40e-01 / 1.15e-01 | 20260507 07:04:37.120 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.03e-01 / 3.57e-01, 5.05e-02 / 1.52e-02 | 20260507 07:04:37.120 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:04:37.120 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:04:37.120 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:04:37.120 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 07:04:37.120 | INFO | PerfReport: 21.90%, Upload -- 2.11e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260507 07:04:37.120 | INFO | PerfReport: 21.34%, Download -- 2.05e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260507 07:04:37.120 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 07:04:37.120 | INFO | PerfReport: 1.62%, LockUpload -- 1.56e+00 / 0.00e+00, 4.99e+00 / 0.00e+00 | 20260507 07:04:37.120 | INFO | PerfReport: 1.19%, LockDownload -- 1.14e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260507 07:04:37.120 | INFO | PerfReport: 22.01%, LockCompute -- 2.12e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 07:04:37.120 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260507 07:04:37.216 | INFO | Starting a round of Baz IO | 20260507 07:04:40.308 | INFO | Populating RTMetrics report | 20260507 07:04:40.308 | INFO | RTMetrics report populated | 20260507 07:04:40.308 | INFO | Opening tmp RTMetrics json file | 20260507 07:04:40.323 | INFO | Serializing RTMetrics to json | 20260507 07:04:40.324 | INFO | Serialization done, swapping tmp file now... | 20260507 07:04:40.324 | INFO | RTMetrics done | 20260507 07:04:40.396 | INFO | Finished a round of Baz IO: Wrote 1.651 GiB in 3.18 seconds | 20260507 07:04:40.396 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.58% / 0.00% / 67.42% | 20260507 07:04:40.396 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.110777% / 3.97364e-06% / 6.35783e-05% | 20260507 07:04:40.396 | INFO | Baz IO overall: | 20260507 07:04:40.397 | INFO | BytesWritten : 4033310515724 | 20260507 07:04:40.397 | INFO | NumEvents : 1931717872615 | 20260507 07:04:42.265 | INFO | Analyzing chunk frames = [8028160,8028672) | 20260507 07:04:42.265 | INFO | MemInfo: { | 20260507 07:04:42.265 | INFO | "gpu" : " 17 GB", | 20260507 07:04:42.265 | INFO | "huge" : "", | 20260507 07:04:42.265 | INFO | "huge_shared" : " 27 GB", | 20260507 07:04:42.265 | INFO | "malloc" : " 14 GB", | 20260507 07:04:42.265 | INFO | "pinned" : " 29 GB" | 20260507 07:04:42.265 | INFO | } | 20260507 07:04:42.265 | INFO | Perf report for the last 5.14603 seconds: | 20260507 07:04:42.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:04:42.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:42.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:42.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:42.265 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 07:04:42.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:42.265 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 07:04:42.265 | INFO | PRE_HQ : 58.8%, 1.8, 54.2ms, 0 | 20260507 07:04:42.265 | INFO | BAZWRITER : 61.4%, 3.1, 97.8ms, 0 | 20260507 07:04:42.265 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:04:42.265 | INFO | Uploaded 36GiB to GPU in 2.04993s (17.5616GiB/s) | 20260507 07:04:42.265 | INFO | Downloaded 38GiB to GPU in 2.18072s (17.4254GiB/s) | 20260507 07:04:47.399 | INFO | Analyzing chunk frames = [8028672,8029184) | 20260507 07:04:47.399 | INFO | MemInfo: { | 20260507 07:04:47.399 | INFO | "gpu" : " 17 GB", | 20260507 07:04:47.399 | INFO | "huge" : "", | 20260507 07:04:47.399 | INFO | "huge_shared" : " 27 GB", | 20260507 07:04:47.399 | INFO | "malloc" : " 14 GB", | 20260507 07:04:47.399 | INFO | "pinned" : " 29 GB" | 20260507 07:04:47.399 | INFO | } | 20260507 07:04:47.399 | INFO | Perf report for the last 5.13369 seconds: | 20260507 07:04:47.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:04:47.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:47.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:47.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:47.399 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:04:47.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:47.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:47.399 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:04:47.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:47.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:47.399 | INFO | Uploaded 36GiB to GPU in 2.00147s (17.9868GiB/s) | 20260507 07:04:47.399 | INFO | Downloaded 34GiB to GPU in 1.93483s (17.5726GiB/s) | 20260507 07:04:52.535 | INFO | Analyzing chunk frames = [8029184,8029696) | 20260507 07:04:52.535 | INFO | MemInfo: { | 20260507 07:04:52.535 | INFO | "gpu" : " 17 GB", | 20260507 07:04:52.535 | INFO | "huge" : "", | 20260507 07:04:52.535 | INFO | "huge_shared" : " 27 GB", | 20260507 07:04:52.535 | INFO | "malloc" : " 14 GB", | 20260507 07:04:52.535 | INFO | "pinned" : " 29 GB" | 20260507 07:04:52.535 | INFO | } | 20260507 07:04:52.535 | INFO | Perf report for the last 5.13655 seconds: | 20260507 07:04:52.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:04:52.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:52.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:52.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:52.535 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 07:04:52.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:52.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:52.535 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:04:52.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:52.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:52.535 | INFO | Uploaded 36GiB to GPU in 2.05232s (17.5412GiB/s) | 20260507 07:04:52.535 | INFO | Downloaded 34GiB to GPU in 1.94955s (17.4399GiB/s) | 20260507 07:04:57.678 | INFO | Analyzing chunk frames = [8029696,8030208) | 20260507 07:04:57.678 | INFO | MemInfo: { | 20260507 07:04:57.678 | INFO | "gpu" : " 17 GB", | 20260507 07:04:57.678 | INFO | "huge" : "", | 20260507 07:04:57.678 | INFO | "huge_shared" : " 27 GB", | 20260507 07:04:57.678 | INFO | "malloc" : " 14 GB", | 20260507 07:04:57.678 | INFO | "pinned" : " 29 GB" | 20260507 07:04:57.678 | INFO | } | 20260507 07:04:57.678 | INFO | Perf report for the last 5.14294 seconds: | 20260507 07:04:57.678 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:04:57.678 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:57.678 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:57.678 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:57.678 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 07:04:57.678 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:57.678 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:57.678 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:04:57.678 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:57.678 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:04:57.678 | INFO | Uploaded 36GiB to GPU in 1.99405s (18.0537GiB/s) | 20260507 07:04:57.679 | INFO | Downloaded 34GiB to GPU in 1.92149s (17.6946GiB/s) | 20260507 07:05:02.823 | INFO | Analyzing chunk frames = [8030208,8030720) | 20260507 07:05:02.823 | INFO | MemInfo: { | 20260507 07:05:02.823 | INFO | "gpu" : " 17 GB", | 20260507 07:05:02.823 | INFO | "huge" : "", | 20260507 07:05:02.823 | INFO | "huge_shared" : " 27 GB", | 20260507 07:05:02.823 | INFO | "malloc" : " 14 GB", | 20260507 07:05:02.823 | INFO | "pinned" : " 29 GB" | 20260507 07:05:02.823 | INFO | } | 20260507 07:05:02.823 | INFO | Perf report for the last 5.14481 seconds: | 20260507 07:05:02.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:05:02.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:02.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:02.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:02.823 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 07:05:02.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:02.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:02.823 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:05:02.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:02.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:02.823 | INFO | Uploaded 36GiB to GPU in 2.0589s (17.4851GiB/s) | 20260507 07:05:02.824 | INFO | Downloaded 34GiB to GPU in 1.95207s (17.4174GiB/s) | 20260507 07:05:07.955 | INFO | Analyzing chunk frames = [8030720,8031232) | 20260507 07:05:07.955 | INFO | MemInfo: { | 20260507 07:05:07.955 | INFO | "gpu" : " 17 GB", | 20260507 07:05:07.955 | INFO | "huge" : "", | 20260507 07:05:07.955 | INFO | "huge_shared" : " 27 GB", | 20260507 07:05:07.955 | INFO | "malloc" : " 14 GB", | 20260507 07:05:07.955 | INFO | "pinned" : " 29 GB" | 20260507 07:05:07.955 | INFO | } | 20260507 07:05:07.955 | INFO | Perf report for the last 5.1323 seconds: | 20260507 07:05:07.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:05:07.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:07.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:07.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:07.955 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:05:07.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:07.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:07.955 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 07:05:07.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:07.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:07.956 | INFO | Uploaded 36GiB to GPU in 2.04469s (17.6065GiB/s) | 20260507 07:05:07.956 | INFO | Downloaded 34GiB to GPU in 1.92738s (17.6406GiB/s) | 20260507 07:05:13.089 | INFO | Analyzing chunk frames = [8031232,8031744) | 20260507 07:05:13.089 | INFO | MemInfo: { | 20260507 07:05:13.089 | INFO | "gpu" : " 17 GB", | 20260507 07:05:13.089 | INFO | "huge" : "", | 20260507 07:05:13.089 | INFO | "huge_shared" : " 27 GB", | 20260507 07:05:13.089 | INFO | "malloc" : " 14 GB", | 20260507 07:05:13.089 | INFO | "pinned" : " 29 GB" | 20260507 07:05:13.089 | INFO | } | 20260507 07:05:13.089 | INFO | Perf report for the last 5.13361 seconds: | 20260507 07:05:13.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:05:13.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:13.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:13.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:13.089 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 07:05:13.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:13.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:13.089 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 07:05:13.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:13.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:13.089 | INFO | Uploaded 36GiB to GPU in 2.05078s (17.5543GiB/s) | 20260507 07:05:13.089 | INFO | Downloaded 34GiB to GPU in 1.99591s (17.0348GiB/s) | 20260507 07:05:18.235 | INFO | Analyzing chunk frames = [8031744,8032256) | 20260507 07:05:18.235 | INFO | MemInfo: { | 20260507 07:05:18.235 | INFO | "gpu" : " 17 GB", | 20260507 07:05:18.235 | INFO | "huge" : "", | 20260507 07:05:18.235 | INFO | "huge_shared" : " 27 GB", | 20260507 07:05:18.235 | INFO | "malloc" : " 14 GB", | 20260507 07:05:18.235 | INFO | "pinned" : " 29 GB" | 20260507 07:05:18.235 | INFO | } | 20260507 07:05:18.236 | INFO | Perf report for the last 5.1468 seconds: | 20260507 07:05:18.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:05:18.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:18.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:18.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:18.236 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:05:18.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:18.236 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:18.236 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 07:05:18.236 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:18.236 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:18.236 | INFO | Uploaded 36GiB to GPU in 1.96633s (18.3082GiB/s) | 20260507 07:05:18.236 | INFO | Downloaded 34GiB to GPU in 1.95513s (17.3901GiB/s) | 20260507 07:05:18.333 | INFO | Starting a round of Baz IO | 20260507 07:05:21.429 | INFO | Populating RTMetrics report | 20260507 07:05:21.430 | INFO | RTMetrics report populated | 20260507 07:05:21.430 | INFO | Opening tmp RTMetrics json file | 20260507 07:05:21.435 | INFO | Serializing RTMetrics to json | 20260507 07:05:21.447 | INFO | Serialization done, swapping tmp file now... | 20260507 07:05:21.447 | INFO | RTMetrics done | 20260507 07:05:21.515 | INFO | Finished a round of Baz IO: Wrote 1.65 GiB in 3.182 seconds | 20260507 07:05:21.515 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.58% / 0.00% / 67.42% | 20260507 07:05:21.515 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.110455% / 0% / 7.54992e-05% | 20260507 07:05:21.515 | INFO | Baz IO overall: | 20260507 07:05:21.515 | INFO | BytesWritten : 4035082253537 | 20260507 07:05:21.515 | INFO | NumEvents : 1932545957110 | 20260507 07:05:23.377 | INFO | Analyzing chunk frames = [8032256,8032768) | 20260507 07:05:23.377 | INFO | MemInfo: { | 20260507 07:05:23.377 | INFO | "gpu" : " 17 GB", | 20260507 07:05:23.377 | INFO | "huge" : "", | 20260507 07:05:23.377 | INFO | "huge_shared" : " 27 GB", | 20260507 07:05:23.377 | INFO | "malloc" : " 14 GB", | 20260507 07:05:23.377 | INFO | "pinned" : " 29 GB" | 20260507 07:05:23.377 | INFO | } | 20260507 07:05:23.377 | INFO | Perf report for the last 5.14112 seconds: | 20260507 07:05:23.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:05:23.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:23.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:23.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:23.377 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 07:05:23.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:23.377 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 07:05:23.377 | INFO | PRE_HQ : 59.0%, 1.8, 55.1ms, 0 | 20260507 07:05:23.377 | INFO | BAZWRITER : 61.5%, 3.1, 98.7ms, 0 | 20260507 07:05:23.377 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:05:23.377 | INFO | Uploaded 36GiB to GPU in 2.05202s (17.5437GiB/s) | 20260507 07:05:23.377 | INFO | Downloaded 38GiB to GPU in 2.17832s (17.4447GiB/s) | 20260507 07:05:28.511 | INFO | Analyzing chunk frames = [8032768,8033280) | 20260507 07:05:28.511 | INFO | MemInfo: { | 20260507 07:05:28.511 | INFO | "gpu" : " 17 GB", | 20260507 07:05:28.512 | INFO | "huge" : "", | 20260507 07:05:28.512 | INFO | "huge_shared" : " 27 GB", | 20260507 07:05:28.512 | INFO | "malloc" : " 14 GB", | 20260507 07:05:28.512 | INFO | "pinned" : " 29 GB" | 20260507 07:05:28.512 | INFO | } | 20260507 07:05:28.512 | INFO | Perf report for the last 5.1347 seconds: | 20260507 07:05:28.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:05:28.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:28.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:28.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:28.512 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:05:28.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:28.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:28.512 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 07:05:28.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:28.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:28.512 | INFO | Uploaded 36GiB to GPU in 1.99334s (18.0602GiB/s) | 20260507 07:05:28.512 | INFO | Downloaded 34GiB to GPU in 1.93526s (17.5687GiB/s) | 20260507 07:05:33.643 | INFO | Analyzing chunk frames = [8033280,8033792) | 20260507 07:05:33.643 | INFO | MemInfo: { | 20260507 07:05:33.643 | INFO | "gpu" : " 17 GB", | 20260507 07:05:33.643 | INFO | "huge" : "", | 20260507 07:05:33.643 | INFO | "huge_shared" : " 27 GB", | 20260507 07:05:33.643 | INFO | "malloc" : " 14 GB", | 20260507 07:05:33.643 | INFO | "pinned" : " 29 GB" | 20260507 07:05:33.643 | INFO | } | 20260507 07:05:33.643 | INFO | Perf report for the last 5.13192 seconds: | 20260507 07:05:33.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:05:33.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:33.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:33.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:33.644 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 07:05:33.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:33.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:33.644 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 07:05:33.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:33.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:33.644 | INFO | Uploaded 36GiB to GPU in 2.05295s (17.5357GiB/s) | 20260507 07:05:33.644 | INFO | Downloaded 34GiB to GPU in 1.94782s (17.4554GiB/s) | 20260507 07:05:38.791 | INFO | Analyzing chunk frames = [8033792,8034304) | 20260507 07:05:38.791 | INFO | MemInfo: { | 20260507 07:05:38.791 | INFO | "gpu" : " 17 GB", | 20260507 07:05:38.791 | INFO | "huge" : "", | 20260507 07:05:38.791 | INFO | "huge_shared" : " 27 GB", | 20260507 07:05:38.791 | INFO | "malloc" : " 14 GB", | 20260507 07:05:38.791 | INFO | "pinned" : " 29 GB" | 20260507 07:05:38.791 | INFO | } | 20260507 07:05:38.791 | INFO | Perf report for the last 5.14766 seconds: | 20260507 07:05:38.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:05:38.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:38.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:38.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:38.791 | INFO | ANALYSIS : 59.5%, 3.0, 92.5ms, 0 | 20260507 07:05:38.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:38.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:38.791 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:05:38.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:38.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:38.791 | INFO | Uploaded 36GiB to GPU in 2.00131s (17.9882GiB/s) | 20260507 07:05:38.791 | INFO | Downloaded 34GiB to GPU in 1.93089s (17.6085GiB/s) | 20260507 07:05:43.936 | INFO | Analyzing chunk frames = [8034304,8034816) | 20260507 07:05:43.936 | INFO | MemInfo: { | 20260507 07:05:43.936 | INFO | "gpu" : " 17 GB", | 20260507 07:05:43.936 | INFO | "huge" : "", | 20260507 07:05:43.936 | INFO | "huge_shared" : " 27 GB", | 20260507 07:05:43.936 | INFO | "malloc" : " 14 GB", | 20260507 07:05:43.936 | INFO | "pinned" : " 29 GB" | 20260507 07:05:43.936 | INFO | } | 20260507 07:05:43.936 | INFO | Perf report for the last 5.14505 seconds: | 20260507 07:05:43.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:05:43.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:43.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:43.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:43.936 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 07:05:43.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:43.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:43.936 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:05:43.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:43.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:43.936 | INFO | Uploaded 36GiB to GPU in 2.05862s (17.4874GiB/s) | 20260507 07:05:43.936 | INFO | Downloaded 34GiB to GPU in 1.94915s (17.4435GiB/s) | 20260507 07:05:49.076 | INFO | Analyzing chunk frames = [8034816,8035328) | 20260507 07:05:49.076 | INFO | MemInfo: { | 20260507 07:05:49.076 | INFO | "gpu" : " 17 GB", | 20260507 07:05:49.076 | INFO | "huge" : "", | 20260507 07:05:49.076 | INFO | "huge_shared" : " 27 GB", | 20260507 07:05:49.076 | INFO | "malloc" : " 14 GB", | 20260507 07:05:49.076 | INFO | "pinned" : " 29 GB" | 20260507 07:05:49.076 | INFO | } | 20260507 07:05:49.076 | INFO | Perf report for the last 5.14027 seconds: | 20260507 07:05:49.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:05:49.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:49.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:49.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:49.076 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 07:05:49.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:49.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:49.077 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:05:49.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:49.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:49.077 | INFO | Uploaded 36GiB to GPU in 2.06395s (17.4423GiB/s) | 20260507 07:05:49.077 | INFO | Downloaded 34GiB to GPU in 1.91813s (17.7256GiB/s) | 20260507 07:05:54.210 | INFO | Analyzing chunk frames = [8035328,8035840) | 20260507 07:05:54.210 | INFO | MemInfo: { | 20260507 07:05:54.210 | INFO | "gpu" : " 17 GB", | 20260507 07:05:54.210 | INFO | "huge" : "", | 20260507 07:05:54.210 | INFO | "huge_shared" : " 27 GB", | 20260507 07:05:54.210 | INFO | "malloc" : " 14 GB", | 20260507 07:05:54.210 | INFO | "pinned" : " 29 GB" | 20260507 07:05:54.210 | INFO | } | 20260507 07:05:54.210 | INFO | Perf report for the last 5.1336 seconds: | 20260507 07:05:54.210 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:05:54.210 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:54.210 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:54.210 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:54.210 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 07:05:54.210 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:54.210 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:54.210 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:05:54.210 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:54.210 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:54.210 | INFO | Uploaded 36GiB to GPU in 1.89676s (18.9797GiB/s) | 20260507 07:05:54.210 | INFO | Downloaded 34GiB to GPU in 1.97316s (17.2312GiB/s) | 20260507 07:05:59.349 | INFO | Analyzing chunk frames = [8035840,8036352) | 20260507 07:05:59.349 | INFO | MemInfo: { | 20260507 07:05:59.349 | INFO | "gpu" : " 17 GB", | 20260507 07:05:59.349 | INFO | "huge" : "", | 20260507 07:05:59.349 | INFO | "huge_shared" : " 27 GB", | 20260507 07:05:59.349 | INFO | "malloc" : " 14 GB", | 20260507 07:05:59.349 | INFO | "pinned" : " 29 GB" | 20260507 07:05:59.349 | INFO | } | 20260507 07:05:59.349 | INFO | Perf report for the last 5.13892 seconds: | 20260507 07:05:59.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:05:59.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:59.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:59.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:59.349 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:05:59.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:59.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:59.349 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 07:05:59.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:59.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:05:59.349 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:05:59.349 | INFO | Total mem usage: 18 GiB | 20260507 07:05:59.349 | INFO | Uploaded 36GiB to GPU in 1.99344s (18.0593GiB/s) | 20260507 07:05:59.349 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:05:59.349 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:05:59.349 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:05:59.349 | INFO | Downloaded 34GiB to GPU in 1.9665s (17.2896GiB/s) | 20260507 07:05:59.349 | INFO | High Water Breakdown: | 20260507 07:05:59.349 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:05:59.349 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:05:59.349 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:05:59.349 | INFO | ------------------------------------------ | 20260507 07:05:59.349 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:05:59.349 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:05:59.349 | INFO | Total mem usage: 26.5 GiB | 20260507 07:05:59.349 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:05:59.349 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:05:59.350 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:05:59.350 | INFO | High Water Breakdown: | 20260507 07:05:59.350 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:05:59.350 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:05:59.350 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:05:59.350 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:05:59.350 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:05:59.350 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:05:59.350 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:05:59.350 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:05:59.350 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:05:59.350 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:05:59.350 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:05:59.350 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:05:59.350 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:05:59.350 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:05:59.350 | INFO | ------------------------------------------ | 20260507 07:05:59.350 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:05:59.350 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:05:59.350 | INFO | Total mem usage: 25.6 GiB | 20260507 07:05:59.350 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:05:59.350 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:05:59.350 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:05:59.350 | INFO | High Water Breakdown: | 20260507 07:05:59.350 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:05:59.350 | INFO | ------------------------------------------ | 20260507 07:05:59.350 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:05:59.350 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:05:59.350 | INFO | Total mem usage: 15.7 GiB | 20260507 07:05:59.350 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:05:59.350 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:05:59.350 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:05:59.350 | INFO | High Water Breakdown: | 20260507 07:05:59.350 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:05:59.350 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:05:59.350 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:05:59.350 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:05:59.350 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:05:59.350 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:05:59.350 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:05:59.350 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:05:59.350 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:05:59.350 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:05:59.350 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:05:59.350 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:05:59.350 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:05:59.350 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:05:59.350 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:05:59.350 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:05:59.350 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:05:59.350 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:05:59.350 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:05:59.350 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:05:59.350 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:05:59.350 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:05:59.350 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:05:59.350 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:05:59.351 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:05:59.351 | INFO | ------------------------------------------ | 20260507 07:05:59.351 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:05:59.351 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:05:59.351 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:05:59.351 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:05:59.351 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.45e+01, 1.35e+01 / 9.73e-01 | 20260507 07:05:59.351 | INFO | PerfReport: 0.29%, AggregateStats -- 9.02e-02 / 1.01e-01, 9.15e-03 / 3.47e-03 | 20260507 07:05:59.351 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.48e+00, 4.99e-01 / 1.82e-01 | 20260507 07:05:59.351 | INFO | PerfReport: 3.21%, Histogram -- 9.85e-01 / 1.48e+00, 1.65e-01 / 1.04e-01 | 20260507 07:05:59.351 | INFO | PerfReport: 16.25%, DME -- 3.98e+01 / 4.44e+01, 3.87e+00 / 1.01e+00 | 20260507 07:05:59.351 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.77e+01 / 1.98e+01, 1.24e+00 / 1.52e+00 | 20260507 07:05:59.351 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.12e+00 / 2.70e+00, 3.86e-01 / 1.98e-01 | 20260507 07:05:59.351 | INFO | PerfReport: 2.95%, Metrics -- 9.04e-01 / 1.22e+00, 2.43e-01 / 1.35e-01 | 20260507 07:05:59.351 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.03e-01 / 3.65e-01, 5.02e-02 / 2.57e-02 | 20260507 07:05:59.351 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:05:59.351 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:05:59.351 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:05:59.351 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 07:05:59.351 | INFO | PerfReport: 21.90%, Upload -- 2.10e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260507 07:05:59.351 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 07:05:59.351 | INFO | PerfReport: 31.88%, Compute -- 3.06e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 07:05:59.351 | INFO | PerfReport: 1.60%, LockUpload -- 1.54e+00 / 0.00e+00, 4.98e+00 / 0.00e+00 | 20260507 07:05:59.351 | INFO | PerfReport: 1.19%, LockDownload -- 1.14e+00 / 0.00e+00, 2.85e+00 / 0.00e+00 | 20260507 07:05:59.351 | INFO | PerfReport: 22.00%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 07:05:59.351 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 07:05:59.446 | INFO | Starting a round of Baz IO | 20260507 07:06:02.539 | INFO | Populating RTMetrics report | 20260507 07:06:02.540 | INFO | RTMetrics report populated | 20260507 07:06:02.540 | INFO | Opening tmp RTMetrics json file | 20260507 07:06:02.554 | INFO | Serializing RTMetrics to json | 20260507 07:06:02.555 | INFO | Serialization done, swapping tmp file now... | 20260507 07:06:02.555 | INFO | RTMetrics done | 20260507 07:06:02.622 | INFO | Finished a round of Baz IO: Wrote 1.65 GiB in 3.176 seconds | 20260507 07:06:02.622 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.58% / 0.00% / 67.42% | 20260507 07:06:02.622 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.110034% / 0% / 6.75519e-05% | 20260507 07:06:02.622 | INFO | Baz IO overall: | 20260507 07:06:02.622 | INFO | BytesWritten : 4036853430886 | 20260507 07:06:02.623 | INFO | NumEvents : 1933373944500 | 20260507 07:06:04.491 | INFO | Analyzing chunk frames = [8036352,8036864) | 20260507 07:06:04.491 | INFO | MemInfo: { | 20260507 07:06:04.491 | INFO | "gpu" : " 17 GB", | 20260507 07:06:04.491 | INFO | "huge" : "", | 20260507 07:06:04.491 | INFO | "huge_shared" : " 27 GB", | 20260507 07:06:04.491 | INFO | "malloc" : " 14 GB", | 20260507 07:06:04.491 | INFO | "pinned" : " 29 GB" | 20260507 07:06:04.491 | INFO | } | 20260507 07:06:04.491 | INFO | Perf report for the last 5.14208 seconds: | 20260507 07:06:04.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:06:04.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:04.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:04.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:04.491 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 07:06:04.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:04.491 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:06:04.491 | INFO | PRE_HQ : 59.1%, 1.8, 54.7ms, 0 | 20260507 07:06:04.491 | INFO | BAZWRITER : 61.4%, 3.1, 98.3ms, 0 | 20260507 07:06:04.491 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:06:04.491 | INFO | Uploaded 36GiB to GPU in 2.04992s (17.5616GiB/s) | 20260507 07:06:04.492 | INFO | Downloaded 38GiB to GPU in 2.1773s (17.4528GiB/s) | 20260507 07:06:09.628 | INFO | Analyzing chunk frames = [8036864,8037376) | 20260507 07:06:09.628 | INFO | MemInfo: { | 20260507 07:06:09.628 | INFO | "gpu" : " 17 GB", | 20260507 07:06:09.628 | INFO | "huge" : "", | 20260507 07:06:09.628 | INFO | "huge_shared" : " 27 GB", | 20260507 07:06:09.628 | INFO | "malloc" : " 14 GB", | 20260507 07:06:09.628 | INFO | "pinned" : " 29 GB" | 20260507 07:06:09.628 | INFO | } | 20260507 07:06:09.628 | INFO | Perf report for the last 5.13704 seconds: | 20260507 07:06:09.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:06:09.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:09.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:09.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:09.628 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 07:06:09.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:09.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:09.628 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:06:09.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:09.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:09.628 | INFO | Uploaded 36GiB to GPU in 2.00005s (17.9995GiB/s) | 20260507 07:06:09.628 | INFO | Downloaded 34GiB to GPU in 1.93081s (17.6092GiB/s) | 20260507 07:06:14.779 | INFO | Analyzing chunk frames = [8037376,8037888) | 20260507 07:06:14.779 | INFO | MemInfo: { | 20260507 07:06:14.779 | INFO | "gpu" : " 17 GB", | 20260507 07:06:14.779 | INFO | "huge" : "", | 20260507 07:06:14.779 | INFO | "huge_shared" : " 27 GB", | 20260507 07:06:14.779 | INFO | "malloc" : " 14 GB", | 20260507 07:06:14.779 | INFO | "pinned" : " 29 GB" | 20260507 07:06:14.779 | INFO | } | 20260507 07:06:14.779 | INFO | Perf report for the last 5.15099 seconds: | 20260507 07:06:14.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:06:14.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:14.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:14.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:14.779 | INFO | ANALYSIS : 60.4%, 3.0, 93.8ms, 0 | 20260507 07:06:14.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:14.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:14.779 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:06:14.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:14.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:14.779 | INFO | Uploaded 36GiB to GPU in 2.05831s (17.4901GiB/s) | 20260507 07:06:14.779 | INFO | Downloaded 34GiB to GPU in 1.94995s (17.4363GiB/s) | 20260507 07:06:19.909 | INFO | Analyzing chunk frames = [8037888,8038400) | 20260507 07:06:19.909 | INFO | MemInfo: { | 20260507 07:06:19.909 | INFO | "gpu" : " 17 GB", | 20260507 07:06:19.909 | INFO | "huge" : "", | 20260507 07:06:19.909 | INFO | "huge_shared" : " 27 GB", | 20260507 07:06:19.909 | INFO | "malloc" : " 14 GB", | 20260507 07:06:19.909 | INFO | "pinned" : " 29 GB" | 20260507 07:06:19.909 | INFO | } | 20260507 07:06:19.909 | INFO | Perf report for the last 5.12991 seconds: | 20260507 07:06:19.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:06:19.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:19.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:19.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:19.909 | INFO | ANALYSIS : 60.4%, 2.9, 93.1ms, 0 | 20260507 07:06:19.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:19.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:19.909 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:06:19.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:19.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:19.909 | INFO | Uploaded 36GiB to GPU in 2.0361s (17.6809GiB/s) | 20260507 07:06:19.909 | INFO | Downloaded 34GiB to GPU in 1.90587s (17.8396GiB/s) | 20260507 07:06:25.043 | INFO | Analyzing chunk frames = [8038400,8038912) | 20260507 07:06:25.043 | INFO | MemInfo: { | 20260507 07:06:25.043 | INFO | "gpu" : " 17 GB", | 20260507 07:06:25.043 | INFO | "huge" : "", | 20260507 07:06:25.043 | INFO | "huge_shared" : " 27 GB", | 20260507 07:06:25.043 | INFO | "malloc" : " 14 GB", | 20260507 07:06:25.043 | INFO | "pinned" : " 29 GB" | 20260507 07:06:25.043 | INFO | } | 20260507 07:06:25.043 | INFO | Perf report for the last 5.13384 seconds: | 20260507 07:06:25.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:06:25.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:25.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:25.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:25.043 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 07:06:25.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:25.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:25.043 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:06:25.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:25.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:25.043 | INFO | Uploaded 36GiB to GPU in 2.04018s (17.6455GiB/s) | 20260507 07:06:25.043 | INFO | Downloaded 34GiB to GPU in 1.92077s (17.7012GiB/s) | 20260507 07:06:30.182 | INFO | Analyzing chunk frames = [8038912,8039424) | 20260507 07:06:30.182 | INFO | MemInfo: { | 20260507 07:06:30.182 | INFO | "gpu" : " 17 GB", | 20260507 07:06:30.182 | INFO | "huge" : "", | 20260507 07:06:30.182 | INFO | "huge_shared" : " 27 GB", | 20260507 07:06:30.182 | INFO | "malloc" : " 14 GB", | 20260507 07:06:30.182 | INFO | "pinned" : " 29 GB" | 20260507 07:06:30.182 | INFO | } | 20260507 07:06:30.182 | INFO | Perf report for the last 5.13917 seconds: | 20260507 07:06:30.182 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:06:30.182 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:30.182 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:30.182 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:30.182 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 07:06:30.182 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:30.182 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:30.182 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260507 07:06:30.182 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:30.182 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:30.182 | INFO | Uploaded 36GiB to GPU in 2.04546s (17.6GiB/s) | 20260507 07:06:30.182 | INFO | Downloaded 34GiB to GPU in 1.90456s (17.8519GiB/s) | 20260507 07:06:35.328 | INFO | Analyzing chunk frames = [8039424,8039936) | 20260507 07:06:35.328 | INFO | MemInfo: { | 20260507 07:06:35.328 | INFO | "gpu" : " 17 GB", | 20260507 07:06:35.328 | INFO | "huge" : "", | 20260507 07:06:35.328 | INFO | "huge_shared" : " 27 GB", | 20260507 07:06:35.328 | INFO | "malloc" : " 14 GB", | 20260507 07:06:35.328 | INFO | "pinned" : " 29 GB" | 20260507 07:06:35.328 | INFO | } | 20260507 07:06:35.329 | INFO | Perf report for the last 5.14638 seconds: | 20260507 07:06:35.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:06:35.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:35.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:35.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:35.329 | INFO | ANALYSIS : 61.0%, 3.0, 95.1ms, 0 | 20260507 07:06:35.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:35.329 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:35.329 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 07:06:35.329 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:35.329 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:35.329 | INFO | Uploaded 36GiB to GPU in 2.05067s (17.5553GiB/s) | 20260507 07:06:35.329 | INFO | Downloaded 34GiB to GPU in 1.99379s (17.053GiB/s) | 20260507 07:06:40.464 | INFO | Analyzing chunk frames = [8039936,8040448) | 20260507 07:06:40.464 | INFO | MemInfo: { | 20260507 07:06:40.464 | INFO | "gpu" : " 17 GB", | 20260507 07:06:40.464 | INFO | "huge" : "", | 20260507 07:06:40.464 | INFO | "huge_shared" : " 27 GB", | 20260507 07:06:40.464 | INFO | "malloc" : " 14 GB", | 20260507 07:06:40.464 | INFO | "pinned" : " 29 GB" | 20260507 07:06:40.464 | INFO | } | 20260507 07:06:40.464 | INFO | Perf report for the last 5.1351 seconds: | 20260507 07:06:40.464 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:06:40.464 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:40.464 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:40.464 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:40.464 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:06:40.464 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:40.464 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:40.464 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:06:40.464 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:40.464 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:40.464 | INFO | Uploaded 36GiB to GPU in 1.98088s (18.1737GiB/s) | 20260507 07:06:40.464 | INFO | Downloaded 34GiB to GPU in 1.96424s (17.3095GiB/s) | 20260507 07:06:40.561 | INFO | Starting a round of Baz IO | 20260507 07:06:43.656 | INFO | Populating RTMetrics report | 20260507 07:06:43.656 | INFO | RTMetrics report populated | 20260507 07:06:43.656 | INFO | Opening tmp RTMetrics json file | 20260507 07:06:43.662 | INFO | Serializing RTMetrics to json | 20260507 07:06:43.673 | INFO | Serialization done, swapping tmp file now... | 20260507 07:06:43.673 | INFO | RTMetrics done | 20260507 07:06:43.743 | INFO | Finished a round of Baz IO: Wrote 1.651 GiB in 3.181 seconds | 20260507 07:06:43.743 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.58% / 0.00% / 67.42% | 20260507 07:06:43.743 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.110408% / 0% / 5.96046e-05% | 20260507 07:06:43.743 | INFO | Baz IO overall: | 20260507 07:06:43.743 | INFO | BytesWritten : 4038626164584 | 20260507 07:06:43.743 | INFO | NumEvents : 1934202720140 | 20260507 07:06:45.598 | INFO | Analyzing chunk frames = [8040448,8040960) | 20260507 07:06:45.598 | INFO | MemInfo: { | 20260507 07:06:45.598 | INFO | "gpu" : " 17 GB", | 20260507 07:06:45.598 | INFO | "huge" : "", | 20260507 07:06:45.598 | INFO | "huge_shared" : " 27 GB", | 20260507 07:06:45.598 | INFO | "malloc" : " 14 GB", | 20260507 07:06:45.598 | INFO | "pinned" : " 29 GB" | 20260507 07:06:45.598 | INFO | } | 20260507 07:06:45.598 | INFO | Perf report for the last 5.13423 seconds: | 20260507 07:06:45.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:06:45.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:45.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:45.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:45.598 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260507 07:06:45.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:45.598 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260507 07:06:45.598 | INFO | PRE_HQ : 59.0%, 1.8, 54.6ms, 0 | 20260507 07:06:45.598 | INFO | BAZWRITER : 61.6%, 3.1, 97.4ms, 0 | 20260507 07:06:45.598 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:06:45.598 | INFO | Uploaded 36GiB to GPU in 2.06252s (17.4543GiB/s) | 20260507 07:06:45.598 | INFO | Downloaded 38GiB to GPU in 2.1788s (17.4408GiB/s) | 20260507 07:06:50.743 | INFO | Analyzing chunk frames = [8040960,8041472) | 20260507 07:06:50.743 | INFO | MemInfo: { | 20260507 07:06:50.743 | INFO | "gpu" : " 17 GB", | 20260507 07:06:50.743 | INFO | "huge" : "", | 20260507 07:06:50.743 | INFO | "huge_shared" : " 27 GB", | 20260507 07:06:50.743 | INFO | "malloc" : " 14 GB", | 20260507 07:06:50.743 | INFO | "pinned" : " 29 GB" | 20260507 07:06:50.743 | INFO | } | 20260507 07:06:50.743 | INFO | Perf report for the last 5.14499 seconds: | 20260507 07:06:50.743 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:06:50.743 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:50.743 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:50.743 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:50.743 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 07:06:50.743 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:50.743 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:50.743 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:06:50.743 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:50.743 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:50.743 | INFO | Uploaded 36GiB to GPU in 2.0007s (17.9937GiB/s) | 20260507 07:06:50.743 | INFO | Downloaded 34GiB to GPU in 1.93339s (17.5857GiB/s) | 20260507 07:06:55.891 | INFO | Analyzing chunk frames = [8041472,8041984) | 20260507 07:06:55.891 | INFO | MemInfo: { | 20260507 07:06:55.891 | INFO | "gpu" : " 17 GB", | 20260507 07:06:55.891 | INFO | "huge" : "", | 20260507 07:06:55.891 | INFO | "huge_shared" : " 27 GB", | 20260507 07:06:55.891 | INFO | "malloc" : " 14 GB", | 20260507 07:06:55.891 | INFO | "pinned" : " 29 GB" | 20260507 07:06:55.891 | INFO | } | 20260507 07:06:55.891 | INFO | Perf report for the last 5.14843 seconds: | 20260507 07:06:55.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:06:55.892 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:55.892 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:55.892 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:55.892 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 07:06:55.892 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:55.892 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:55.892 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:06:55.892 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:55.892 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:06:55.892 | INFO | Uploaded 36GiB to GPU in 2.04726s (17.5845GiB/s) | 20260507 07:06:55.892 | INFO | Downloaded 34GiB to GPU in 1.95217s (17.4165GiB/s) | 20260507 07:07:01.030 | INFO | Analyzing chunk frames = [8041984,8042496) | 20260507 07:07:01.030 | INFO | MemInfo: { | 20260507 07:07:01.030 | INFO | "gpu" : " 17 GB", | 20260507 07:07:01.030 | INFO | "huge" : "", | 20260507 07:07:01.030 | INFO | "huge_shared" : " 27 GB", | 20260507 07:07:01.030 | INFO | "malloc" : " 14 GB", | 20260507 07:07:01.030 | INFO | "pinned" : " 29 GB" | 20260507 07:07:01.030 | INFO | } | 20260507 07:07:01.030 | INFO | Perf report for the last 5.13873 seconds: | 20260507 07:07:01.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:07:01.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:01.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:01.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:01.030 | INFO | ANALYSIS : 60.3%, 2.9, 93.1ms, 0 | 20260507 07:07:01.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:01.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:01.030 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:07:01.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:01.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:01.030 | INFO | Uploaded 36GiB to GPU in 2.0298s (17.7357GiB/s) | 20260507 07:07:01.031 | INFO | Downloaded 34GiB to GPU in 1.91857s (17.7215GiB/s) | 20260507 07:07:06.159 | INFO | Analyzing chunk frames = [8042496,8043008) | 20260507 07:07:06.159 | INFO | MemInfo: { | 20260507 07:07:06.159 | INFO | "gpu" : " 17 GB", | 20260507 07:07:06.159 | INFO | "huge" : "", | 20260507 07:07:06.159 | INFO | "huge_shared" : " 27 GB", | 20260507 07:07:06.159 | INFO | "malloc" : " 14 GB", | 20260507 07:07:06.159 | INFO | "pinned" : " 29 GB" | 20260507 07:07:06.159 | INFO | } | 20260507 07:07:06.159 | INFO | Perf report for the last 5.12907 seconds: | 20260507 07:07:06.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:07:06.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:06.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:06.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:06.159 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 07:07:06.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:06.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:06.160 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:07:06.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:06.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:06.160 | INFO | Uploaded 36GiB to GPU in 2.05136s (17.5493GiB/s) | 20260507 07:07:06.160 | INFO | Downloaded 34GiB to GPU in 1.93118s (17.6058GiB/s) | 20260507 07:07:11.299 | INFO | Analyzing chunk frames = [8043008,8043520) | 20260507 07:07:11.299 | INFO | MemInfo: { | 20260507 07:07:11.299 | INFO | "gpu" : " 17 GB", | 20260507 07:07:11.299 | INFO | "huge" : "", | 20260507 07:07:11.299 | INFO | "huge_shared" : " 27 GB", | 20260507 07:07:11.299 | INFO | "malloc" : " 14 GB", | 20260507 07:07:11.299 | INFO | "pinned" : " 29 GB" | 20260507 07:07:11.299 | INFO | } | 20260507 07:07:11.299 | INFO | Perf report for the last 5.14024 seconds: | 20260507 07:07:11.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:07:11.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:11.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:11.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:11.300 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 07:07:11.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:11.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:11.300 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 07:07:11.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:11.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:11.300 | INFO | Uploaded 36GiB to GPU in 2.06381s (17.4435GiB/s) | 20260507 07:07:11.300 | INFO | Downloaded 34GiB to GPU in 1.93199s (17.5985GiB/s) | 20260507 07:07:16.441 | INFO | Analyzing chunk frames = [8043520,8044032) | 20260507 07:07:16.441 | INFO | MemInfo: { | 20260507 07:07:16.441 | INFO | "gpu" : " 17 GB", | 20260507 07:07:16.441 | INFO | "huge" : "", | 20260507 07:07:16.441 | INFO | "huge_shared" : " 27 GB", | 20260507 07:07:16.441 | INFO | "malloc" : " 14 GB", | 20260507 07:07:16.441 | INFO | "pinned" : " 29 GB" | 20260507 07:07:16.441 | INFO | } | 20260507 07:07:16.441 | INFO | Perf report for the last 5.14172 seconds: | 20260507 07:07:16.441 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:07:16.441 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:16.441 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:16.441 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:16.441 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 07:07:16.441 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:16.441 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:16.441 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 07:07:16.441 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:16.441 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:16.441 | INFO | Uploaded 36GiB to GPU in 1.9667s (18.3047GiB/s) | 20260507 07:07:16.441 | INFO | Downloaded 34GiB to GPU in 1.98s (17.1717GiB/s) | 20260507 07:07:21.581 | INFO | Analyzing chunk frames = [8044032,8044544) | 20260507 07:07:21.581 | INFO | MemInfo: { | 20260507 07:07:21.581 | INFO | "gpu" : " 17 GB", | 20260507 07:07:21.581 | INFO | "huge" : "", | 20260507 07:07:21.581 | INFO | "huge_shared" : " 27 GB", | 20260507 07:07:21.581 | INFO | "malloc" : " 14 GB", | 20260507 07:07:21.581 | INFO | "pinned" : " 29 GB" | 20260507 07:07:21.581 | INFO | } | 20260507 07:07:21.581 | INFO | Perf report for the last 5.14019 seconds: | 20260507 07:07:21.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:07:21.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:21.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:21.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:21.582 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:07:21.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:21.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:21.582 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:07:21.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:21.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:21.582 | INFO | Uploaded 36GiB to GPU in 1.97535s (18.2246GiB/s) | 20260507 07:07:21.582 | INFO | Downloaded 34GiB to GPU in 1.96406s (17.3111GiB/s) | 20260507 07:07:21.582 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:07:21.582 | INFO | Total mem usage: 18 GiB | 20260507 07:07:21.582 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:07:21.582 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:07:21.582 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:07:21.582 | INFO | High Water Breakdown: | 20260507 07:07:21.582 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:07:21.582 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:07:21.582 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:07:21.582 | INFO | ------------------------------------------ | 20260507 07:07:21.582 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:07:21.582 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:07:21.582 | INFO | Total mem usage: 26.5 GiB | 20260507 07:07:21.582 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:07:21.582 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:07:21.582 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:07:21.582 | INFO | High Water Breakdown: | 20260507 07:07:21.582 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:07:21.582 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:07:21.582 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:07:21.582 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:07:21.582 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:07:21.582 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:07:21.582 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:07:21.582 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:07:21.582 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:07:21.582 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:07:21.582 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:07:21.582 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:07:21.582 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:07:21.582 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:07:21.582 | INFO | ------------------------------------------ | 20260507 07:07:21.582 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:07:21.582 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:07:21.582 | INFO | Total mem usage: 25.6 GiB | 20260507 07:07:21.582 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:07:21.582 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:07:21.583 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:07:21.583 | INFO | High Water Breakdown: | 20260507 07:07:21.583 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:07:21.583 | INFO | ------------------------------------------ | 20260507 07:07:21.583 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:07:21.583 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:07:21.583 | INFO | Total mem usage: 15.7 GiB | 20260507 07:07:21.583 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:07:21.583 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:07:21.583 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:07:21.583 | INFO | High Water Breakdown: | 20260507 07:07:21.583 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:07:21.583 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:07:21.583 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:07:21.583 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:07:21.583 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:07:21.583 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:07:21.583 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:07:21.583 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:07:21.583 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:07:21.583 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:07:21.583 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:07:21.583 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:07:21.583 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:07:21.583 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:07:21.583 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:07:21.583 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:07:21.583 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:07:21.583 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:07:21.583 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:07:21.583 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:07:21.583 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:07:21.583 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:07:21.583 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:07:21.583 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:07:21.583 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:07:21.583 | INFO | ------------------------------------------ | 20260507 07:07:21.583 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:07:21.583 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:07:21.583 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:07:21.583 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:07:21.583 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.34e+01 / 1.24e+00 | 20260507 07:07:21.583 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.23e-03 / 4.28e-03 | 20260507 07:07:21.583 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.44e+00, 5.00e-01 / 2.51e-01 | 20260507 07:07:21.583 | INFO | PerfReport: 3.22%, Histogram -- 9.90e-01 / 1.49e+00, 1.67e-01 / 1.04e-01 | 20260507 07:07:21.583 | INFO | PerfReport: 16.25%, DME -- 3.99e+01 / 4.46e+01, 3.82e+00 / 1.33e+00 | 20260507 07:07:21.583 | INFO | PerfReport: 57.79%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.35e+00 / 1.29e+00 | 20260507 07:07:21.583 | INFO | PerfReport: 6.88%, PulseAccumulating -- 2.11e+00 / 2.68e+00, 3.81e-01 / 2.04e-01 | 20260507 07:07:21.583 | INFO | PerfReport: 2.93%, Metrics -- 9.00e-01 / 1.21e+00, 2.41e-01 / 1.31e-01 | 20260507 07:07:21.583 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.03e-01 / 3.64e-01, 5.08e-02 / 1.81e-02 | 20260507 07:07:21.583 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:07:21.583 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:07:21.583 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:07:21.583 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 07:07:21.583 | INFO | PerfReport: 21.96%, Upload -- 2.11e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 07:07:21.583 | INFO | PerfReport: 21.34%, Download -- 2.05e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260507 07:07:21.583 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 07:07:21.583 | INFO | PerfReport: 1.60%, LockUpload -- 1.54e+00 / 0.00e+00, 4.96e+00 / 0.00e+00 | 20260507 07:07:21.583 | INFO | PerfReport: 1.21%, LockDownload -- 1.16e+00 / 0.00e+00, 2.83e+00 / 0.00e+00 | 20260507 07:07:21.583 | INFO | PerfReport: 21.95%, LockCompute -- 2.11e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 07:07:21.583 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260507 07:07:21.679 | INFO | Starting a round of Baz IO | 20260507 07:07:24.768 | INFO | Populating RTMetrics report | 20260507 07:07:24.769 | INFO | RTMetrics report populated | 20260507 07:07:24.769 | INFO | Opening tmp RTMetrics json file | 20260507 07:07:24.784 | INFO | Serializing RTMetrics to json | 20260507 07:07:24.785 | INFO | Serialization done, swapping tmp file now... | 20260507 07:07:24.786 | INFO | RTMetrics done | 20260507 07:07:24.853 | INFO | Finished a round of Baz IO: Wrote 1.651 GiB in 3.174 seconds | 20260507 07:07:24.853 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.58% / 0.00% / 67.42% | 20260507 07:07:24.853 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.110026% / 0% / 6.75519e-05% | 20260507 07:07:24.853 | INFO | Baz IO overall: | 20260507 07:07:24.853 | INFO | BytesWritten : 4040399056494 | 20260507 07:07:24.853 | INFO | NumEvents : 1935031591263 | 20260507 07:07:26.721 | INFO | Analyzing chunk frames = [8044544,8045056) | 20260507 07:07:26.721 | INFO | MemInfo: { | 20260507 07:07:26.721 | INFO | "gpu" : " 17 GB", | 20260507 07:07:26.721 | INFO | "huge" : "", | 20260507 07:07:26.721 | INFO | "huge_shared" : " 27 GB", | 20260507 07:07:26.721 | INFO | "malloc" : " 14 GB", | 20260507 07:07:26.721 | INFO | "pinned" : " 29 GB" | 20260507 07:07:26.721 | INFO | } | 20260507 07:07:26.721 | INFO | Perf report for the last 5.13925 seconds: | 20260507 07:07:26.721 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:07:26.721 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:26.721 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:26.721 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:26.721 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 07:07:26.721 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:26.721 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 07:07:26.721 | INFO | PRE_HQ : 58.8%, 1.8, 54.7ms, 0 | 20260507 07:07:26.721 | INFO | BAZWRITER : 61.4%, 3.1, 98.0ms, 0 | 20260507 07:07:26.721 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:07:26.721 | INFO | Uploaded 36GiB to GPU in 2.05433s (17.524GiB/s) | 20260507 07:07:26.721 | INFO | Downloaded 38GiB to GPU in 2.18037s (17.4283GiB/s) | 20260507 07:07:31.851 | INFO | Analyzing chunk frames = [8045056,8045568) | 20260507 07:07:31.851 | INFO | MemInfo: { | 20260507 07:07:31.851 | INFO | "gpu" : " 17 GB", | 20260507 07:07:31.851 | INFO | "huge" : "", | 20260507 07:07:31.851 | INFO | "huge_shared" : " 27 GB", | 20260507 07:07:31.851 | INFO | "malloc" : " 14 GB", | 20260507 07:07:31.851 | INFO | "pinned" : " 29 GB" | 20260507 07:07:31.852 | INFO | } | 20260507 07:07:31.852 | INFO | Perf report for the last 5.13093 seconds: | 20260507 07:07:31.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:07:31.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:31.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:31.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:31.852 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 07:07:31.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:31.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:31.852 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:07:31.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:31.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:31.852 | INFO | Uploaded 36GiB to GPU in 1.98213s (18.1623GiB/s) | 20260507 07:07:31.852 | INFO | Downloaded 34GiB to GPU in 1.93479s (17.573GiB/s) | 20260507 07:07:36.998 | INFO | Analyzing chunk frames = [8045568,8046080) | 20260507 07:07:36.998 | INFO | MemInfo: { | 20260507 07:07:36.998 | INFO | "gpu" : " 17 GB", | 20260507 07:07:36.998 | INFO | "huge" : "", | 20260507 07:07:36.998 | INFO | "huge_shared" : " 27 GB", | 20260507 07:07:36.998 | INFO | "malloc" : " 14 GB", | 20260507 07:07:36.998 | INFO | "pinned" : " 29 GB" | 20260507 07:07:36.998 | INFO | } | 20260507 07:07:36.998 | INFO | Perf report for the last 5.14685 seconds: | 20260507 07:07:36.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:07:36.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:36.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:36.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:36.999 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 07:07:36.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:36.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:36.999 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:07:36.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:36.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:36.999 | INFO | Uploaded 36GiB to GPU in 2.04142s (17.6348GiB/s) | 20260507 07:07:36.999 | INFO | Downloaded 34GiB to GPU in 1.95388s (17.4013GiB/s) | 20260507 07:07:42.136 | INFO | Analyzing chunk frames = [8046080,8046592) | 20260507 07:07:42.136 | INFO | MemInfo: { | 20260507 07:07:42.136 | INFO | "gpu" : " 17 GB", | 20260507 07:07:42.136 | INFO | "huge" : "", | 20260507 07:07:42.136 | INFO | "huge_shared" : " 27 GB", | 20260507 07:07:42.136 | INFO | "malloc" : " 14 GB", | 20260507 07:07:42.136 | INFO | "pinned" : " 29 GB" | 20260507 07:07:42.136 | INFO | } | 20260507 07:07:42.136 | INFO | Perf report for the last 5.13761 seconds: | 20260507 07:07:42.136 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:07:42.136 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:42.136 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:42.136 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:42.136 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 07:07:42.136 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:42.136 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:42.136 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:07:42.136 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:42.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:42.136 | INFO | Uploaded 36GiB to GPU in 2.01063s (17.9048GiB/s) | 20260507 07:07:42.136 | INFO | Downloaded 34GiB to GPU in 1.94212s (17.5067GiB/s) | 20260507 07:07:47.273 | INFO | Analyzing chunk frames = [8046592,8047104) | 20260507 07:07:47.273 | INFO | MemInfo: { | 20260507 07:07:47.273 | INFO | "gpu" : " 17 GB", | 20260507 07:07:47.273 | INFO | "huge" : "", | 20260507 07:07:47.273 | INFO | "huge_shared" : " 27 GB", | 20260507 07:07:47.273 | INFO | "malloc" : " 14 GB", | 20260507 07:07:47.274 | INFO | "pinned" : " 29 GB" | 20260507 07:07:47.274 | INFO | } | 20260507 07:07:47.274 | INFO | Perf report for the last 5.1376 seconds: | 20260507 07:07:47.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:07:47.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:47.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:47.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:47.274 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 07:07:47.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:47.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:47.274 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:07:47.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:47.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:47.274 | INFO | Uploaded 36GiB to GPU in 2.04076s (17.6405GiB/s) | 20260507 07:07:47.274 | INFO | Downloaded 34GiB to GPU in 1.91737s (17.7326GiB/s) | 20260507 07:07:52.413 | INFO | Analyzing chunk frames = [8047104,8047616) | 20260507 07:07:52.413 | INFO | MemInfo: { | 20260507 07:07:52.413 | INFO | "gpu" : " 17 GB", | 20260507 07:07:52.413 | INFO | "huge" : "", | 20260507 07:07:52.413 | INFO | "huge_shared" : " 27 GB", | 20260507 07:07:52.413 | INFO | "malloc" : " 14 GB", | 20260507 07:07:52.413 | INFO | "pinned" : " 29 GB" | 20260507 07:07:52.413 | INFO | } | 20260507 07:07:52.413 | INFO | Perf report for the last 5.13922 seconds: | 20260507 07:07:52.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:07:52.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:52.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:52.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:52.413 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 07:07:52.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:52.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:52.413 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 07:07:52.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:52.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:52.413 | INFO | Uploaded 36GiB to GPU in 2.05895s (17.4847GiB/s) | 20260507 07:07:52.413 | INFO | Downloaded 34GiB to GPU in 1.93281s (17.591GiB/s) | 20260507 07:07:57.555 | INFO | Analyzing chunk frames = [8047616,8048128) | 20260507 07:07:57.555 | INFO | MemInfo: { | 20260507 07:07:57.555 | INFO | "gpu" : " 17 GB", | 20260507 07:07:57.555 | INFO | "huge" : "", | 20260507 07:07:57.555 | INFO | "huge_shared" : " 27 GB", | 20260507 07:07:57.555 | INFO | "malloc" : " 14 GB", | 20260507 07:07:57.555 | INFO | "pinned" : " 29 GB" | 20260507 07:07:57.555 | INFO | } | 20260507 07:07:57.556 | INFO | Perf report for the last 5.14257 seconds: | 20260507 07:07:57.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:07:57.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:57.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:57.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:57.556 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 07:07:57.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:57.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:57.556 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 07:07:57.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:57.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:07:57.556 | INFO | Uploaded 36GiB to GPU in 2.04214s (17.6286GiB/s) | 20260507 07:07:57.556 | INFO | Downloaded 34GiB to GPU in 1.97686s (17.199GiB/s) | 20260507 07:08:02.690 | INFO | Analyzing chunk frames = [8048128,8048640) | 20260507 07:08:02.690 | INFO | MemInfo: { | 20260507 07:08:02.690 | INFO | "gpu" : " 17 GB", | 20260507 07:08:02.690 | INFO | "huge" : "", | 20260507 07:08:02.690 | INFO | "huge_shared" : " 27 GB", | 20260507 07:08:02.690 | INFO | "malloc" : " 14 GB", | 20260507 07:08:02.690 | INFO | "pinned" : " 29 GB" | 20260507 07:08:02.690 | INFO | } | 20260507 07:08:02.690 | INFO | Perf report for the last 5.13432 seconds: | 20260507 07:08:02.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:08:02.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:02.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:02.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:02.690 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:08:02.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:02.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:02.690 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 07:08:02.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:02.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:02.690 | INFO | Uploaded 36GiB to GPU in 1.97834s (18.197GiB/s) | 20260507 07:08:02.690 | INFO | Downloaded 34GiB to GPU in 1.96022s (17.345GiB/s) | 20260507 07:08:02.787 | INFO | Starting a round of Baz IO | 20260507 07:08:05.879 | INFO | Populating RTMetrics report | 20260507 07:08:05.879 | INFO | RTMetrics report populated | 20260507 07:08:05.879 | INFO | Opening tmp RTMetrics json file | 20260507 07:08:05.888 | INFO | Serializing RTMetrics to json | 20260507 07:08:05.896 | INFO | Serialization done, swapping tmp file now... | 20260507 07:08:05.896 | INFO | RTMetrics done | 20260507 07:08:05.966 | INFO | Finished a round of Baz IO: Wrote 1.65 GiB in 3.178 seconds | 20260507 07:08:05.966 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.58% / 0.00% / 67.42% | 20260507 07:08:05.966 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.110527% / 0% / 8.34465e-05% | 20260507 07:08:05.966 | INFO | Baz IO overall: | 20260507 07:08:05.966 | INFO | BytesWritten : 4042170879665 | 20260507 07:08:05.966 | INFO | NumEvents : 1935859479578 | 20260507 07:08:07.836 | INFO | Analyzing chunk frames = [8048640,8049152) | 20260507 07:08:07.836 | INFO | MemInfo: { | 20260507 07:08:07.836 | INFO | "gpu" : " 17 GB", | 20260507 07:08:07.836 | INFO | "huge" : "", | 20260507 07:08:07.836 | INFO | "huge_shared" : " 27 GB", | 20260507 07:08:07.836 | INFO | "malloc" : " 14 GB", | 20260507 07:08:07.836 | INFO | "pinned" : " 29 GB" | 20260507 07:08:07.836 | INFO | } | 20260507 07:08:07.836 | INFO | Perf report for the last 5.1463 seconds: | 20260507 07:08:07.836 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:08:07.836 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:07.836 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:07.836 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:07.836 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 07:08:07.836 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:07.836 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:08:07.836 | INFO | PRE_HQ : 58.9%, 1.8, 54.8ms, 0 | 20260507 07:08:07.836 | INFO | BAZWRITER : 61.4%, 3.1, 97.1ms, 0 | 20260507 07:08:07.836 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:08:07.836 | INFO | Uploaded 36GiB to GPU in 2.04426s (17.6103GiB/s) | 20260507 07:08:07.836 | INFO | Downloaded 38GiB to GPU in 2.18168s (17.4178GiB/s) | 20260507 07:08:12.974 | INFO | Analyzing chunk frames = [8049152,8049664) | 20260507 07:08:12.974 | INFO | MemInfo: { | 20260507 07:08:12.974 | INFO | "gpu" : " 17 GB", | 20260507 07:08:12.974 | INFO | "huge" : "", | 20260507 07:08:12.974 | INFO | "huge_shared" : " 27 GB", | 20260507 07:08:12.974 | INFO | "malloc" : " 14 GB", | 20260507 07:08:12.974 | INFO | "pinned" : " 29 GB" | 20260507 07:08:12.974 | INFO | } | 20260507 07:08:12.974 | INFO | Perf report for the last 5.13755 seconds: | 20260507 07:08:12.974 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:08:12.974 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:12.974 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:12.974 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:12.974 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:08:12.974 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:12.974 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:12.974 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:08:12.974 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:12.974 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:12.974 | INFO | Uploaded 36GiB to GPU in 2.0087s (17.922GiB/s) | 20260507 07:08:12.974 | INFO | Downloaded 34GiB to GPU in 1.94893s (17.4455GiB/s) | 20260507 07:08:18.110 | INFO | Analyzing chunk frames = [8049664,8050176) | 20260507 07:08:18.110 | INFO | MemInfo: { | 20260507 07:08:18.110 | INFO | "gpu" : " 17 GB", | 20260507 07:08:18.110 | INFO | "huge" : "", | 20260507 07:08:18.110 | INFO | "huge_shared" : " 27 GB", | 20260507 07:08:18.110 | INFO | "malloc" : " 14 GB", | 20260507 07:08:18.110 | INFO | "pinned" : " 29 GB" | 20260507 07:08:18.110 | INFO | } | 20260507 07:08:18.110 | INFO | Perf report for the last 5.13659 seconds: | 20260507 07:08:18.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:08:18.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:18.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:18.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:18.110 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 07:08:18.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:18.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:18.110 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 07:08:18.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:18.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:18.111 | INFO | Uploaded 36GiB to GPU in 2.04992s (17.5617GiB/s) | 20260507 07:08:18.111 | INFO | Downloaded 34GiB to GPU in 1.95292s (17.4098GiB/s) | 20260507 07:08:23.249 | INFO | Analyzing chunk frames = [8050176,8050688) | 20260507 07:08:23.249 | INFO | MemInfo: { | 20260507 07:08:23.249 | INFO | "gpu" : " 17 GB", | 20260507 07:08:23.249 | INFO | "huge" : "", | 20260507 07:08:23.249 | INFO | "huge_shared" : " 27 GB", | 20260507 07:08:23.249 | INFO | "malloc" : " 14 GB", | 20260507 07:08:23.249 | INFO | "pinned" : " 29 GB" | 20260507 07:08:23.249 | INFO | } | 20260507 07:08:23.249 | INFO | Perf report for the last 5.13908 seconds: | 20260507 07:08:23.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:08:23.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:23.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:23.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:23.249 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 07:08:23.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:23.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:23.249 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:08:23.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:23.250 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:23.250 | INFO | Uploaded 36GiB to GPU in 1.99342s (18.0594GiB/s) | 20260507 07:08:23.250 | INFO | Downloaded 34GiB to GPU in 1.93875s (17.5371GiB/s) | 20260507 07:08:28.393 | INFO | Analyzing chunk frames = [8050688,8051200) | 20260507 07:08:28.393 | INFO | MemInfo: { | 20260507 07:08:28.393 | INFO | "gpu" : " 17 GB", | 20260507 07:08:28.393 | INFO | "huge" : "", | 20260507 07:08:28.393 | INFO | "huge_shared" : " 27 GB", | 20260507 07:08:28.393 | INFO | "malloc" : " 14 GB", | 20260507 07:08:28.393 | INFO | "pinned" : " 29 GB" | 20260507 07:08:28.393 | INFO | } | 20260507 07:08:28.393 | INFO | Perf report for the last 5.14366 seconds: | 20260507 07:08:28.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:08:28.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:28.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:28.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:28.393 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 07:08:28.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:28.393 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:28.393 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:08:28.393 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:28.393 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:28.393 | INFO | Uploaded 36GiB to GPU in 2.05055s (17.5563GiB/s) | 20260507 07:08:28.393 | INFO | Downloaded 34GiB to GPU in 1.95244s (17.4141GiB/s) | 20260507 07:08:33.531 | INFO | Analyzing chunk frames = [8051200,8051712) | 20260507 07:08:33.531 | INFO | MemInfo: { | 20260507 07:08:33.531 | INFO | "gpu" : " 17 GB", | 20260507 07:08:33.531 | INFO | "huge" : "", | 20260507 07:08:33.531 | INFO | "huge_shared" : " 27 GB", | 20260507 07:08:33.531 | INFO | "malloc" : " 14 GB", | 20260507 07:08:33.531 | INFO | "pinned" : " 29 GB" | 20260507 07:08:33.531 | INFO | } | 20260507 07:08:33.532 | INFO | Perf report for the last 5.1385 seconds: | 20260507 07:08:33.532 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:08:33.532 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:33.532 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:33.532 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:33.532 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 07:08:33.532 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:33.532 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:33.532 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:08:33.532 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:33.532 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:33.532 | INFO | Uploaded 36GiB to GPU in 2.04679s (17.5885GiB/s) | 20260507 07:08:33.532 | INFO | Downloaded 34GiB to GPU in 1.91457s (17.7586GiB/s) | 20260507 07:08:38.666 | INFO | Analyzing chunk frames = [8051712,8052224) | 20260507 07:08:38.666 | INFO | MemInfo: { | 20260507 07:08:38.666 | INFO | "gpu" : " 17 GB", | 20260507 07:08:38.666 | INFO | "huge" : "", | 20260507 07:08:38.666 | INFO | "huge_shared" : " 27 GB", | 20260507 07:08:38.666 | INFO | "malloc" : " 14 GB", | 20260507 07:08:38.666 | INFO | "pinned" : " 29 GB" | 20260507 07:08:38.666 | INFO | } | 20260507 07:08:38.666 | INFO | Perf report for the last 5.13474 seconds: | 20260507 07:08:38.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:08:38.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:38.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:38.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:38.666 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:08:38.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:38.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:38.666 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 07:08:38.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:38.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:38.666 | INFO | Uploaded 36GiB to GPU in 1.91814s (18.7682GiB/s) | 20260507 07:08:38.667 | INFO | Downloaded 34GiB to GPU in 1.96914s (17.2664GiB/s) | 20260507 07:08:43.805 | INFO | Analyzing chunk frames = [8052224,8052736) | 20260507 07:08:43.805 | INFO | MemInfo: { | 20260507 07:08:43.805 | INFO | "gpu" : " 17 GB", | 20260507 07:08:43.805 | INFO | "huge" : "", | 20260507 07:08:43.805 | INFO | "huge_shared" : " 27 GB", | 20260507 07:08:43.805 | INFO | "malloc" : " 14 GB", | 20260507 07:08:43.805 | INFO | "pinned" : " 29 GB" | 20260507 07:08:43.805 | INFO | } | 20260507 07:08:43.805 | INFO | Perf report for the last 5.13853 seconds: | 20260507 07:08:43.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:08:43.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:43.805 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:43.805 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:43.805 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:08:43.805 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:43.805 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:43.805 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:08:43.805 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:43.805 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:43.805 | INFO | Uploaded 36GiB to GPU in 1.99233s (18.0693GiB/s) | 20260507 07:08:43.805 | INFO | Downloaded 34GiB to GPU in 1.96102s (17.3379GiB/s) | 20260507 07:08:43.805 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:08:43.805 | INFO | Total mem usage: 18 GiB | 20260507 07:08:43.805 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:08:43.805 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:08:43.805 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:08:43.805 | INFO | High Water Breakdown: | 20260507 07:08:43.805 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:08:43.805 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:08:43.805 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:08:43.805 | INFO | ------------------------------------------ | 20260507 07:08:43.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:08:43.805 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:08:43.805 | INFO | Total mem usage: 26.5 GiB | 20260507 07:08:43.805 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:08:43.805 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:08:43.805 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:08:43.805 | INFO | High Water Breakdown: | 20260507 07:08:43.805 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:08:43.805 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:08:43.805 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:08:43.805 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:08:43.805 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:08:43.805 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:08:43.805 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:08:43.805 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:08:43.805 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:08:43.805 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:08:43.805 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:08:43.805 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:08:43.805 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:08:43.805 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:08:43.805 | INFO | ------------------------------------------ | 20260507 07:08:43.805 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:08:43.805 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:08:43.805 | INFO | Total mem usage: 25.6 GiB | 20260507 07:08:43.805 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:08:43.805 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:08:43.805 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:08:43.806 | INFO | High Water Breakdown: | 20260507 07:08:43.806 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:08:43.806 | INFO | ------------------------------------------ | 20260507 07:08:43.806 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:08:43.806 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:08:43.806 | INFO | Total mem usage: 15.7 GiB | 20260507 07:08:43.806 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:08:43.806 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:08:43.806 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:08:43.806 | INFO | High Water Breakdown: | 20260507 07:08:43.806 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:08:43.806 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:08:43.806 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:08:43.806 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:08:43.806 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:08:43.806 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:08:43.806 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:08:43.806 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:08:43.806 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:08:43.806 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:08:43.806 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:08:43.806 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:08:43.806 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:08:43.806 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:08:43.806 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:08:43.806 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:08:43.806 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:08:43.806 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:08:43.806 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:08:43.806 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:08:43.806 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:08:43.806 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:08:43.806 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:08:43.806 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:08:43.806 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:08:43.806 | INFO | ------------------------------------------ | 20260507 07:08:43.806 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:08:43.806 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:08:43.806 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:08:43.806 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:08:43.806 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.33e+01 / 1.04e+00 | 20260507 07:08:43.806 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.62e-03 / 5.27e-03 | 20260507 07:08:43.806 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.47e+00, 5.01e-01 / 1.96e-01 | 20260507 07:08:43.806 | INFO | PerfReport: 3.22%, Histogram -- 9.87e-01 / 1.47e+00, 1.62e-01 / 9.34e-02 | 20260507 07:08:43.806 | INFO | PerfReport: 16.23%, DME -- 3.98e+01 / 4.45e+01, 3.84e+00 / 1.19e+00 | 20260507 07:08:43.806 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.77e+01 / 1.98e+01, 1.28e+00 / 8.11e-01 | 20260507 07:08:43.806 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.12e+00 / 2.69e+00, 3.86e-01 / 1.73e-01 | 20260507 07:08:43.806 | INFO | PerfReport: 2.95%, Metrics -- 9.04e-01 / 1.23e+00, 2.44e-01 / 1.39e-01 | 20260507 07:08:43.806 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.03e-01 / 3.65e-01, 5.07e-02 / 2.37e-02 | 20260507 07:08:43.806 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:08:43.806 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:08:43.806 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:08:43.806 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 07:08:43.806 | INFO | PerfReport: 21.89%, Upload -- 2.10e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260507 07:08:43.806 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 07:08:43.806 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 07:08:43.806 | INFO | PerfReport: 1.45%, LockUpload -- 1.39e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 07:08:43.806 | INFO | PerfReport: 1.33%, LockDownload -- 1.28e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260507 07:08:43.806 | INFO | PerfReport: 21.96%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 07:08:43.806 | INFO | PerfReport: 0.01%, Misc -- 1.30e-02 / 0.00e+00, 1.49e-02 / 0.00e+00 | 20260507 07:08:43.906 | INFO | Starting a round of Baz IO | 20260507 07:08:46.993 | INFO | Populating RTMetrics report | 20260507 07:08:46.993 | INFO | RTMetrics report populated | 20260507 07:08:46.993 | INFO | Opening tmp RTMetrics json file | 20260507 07:08:47.013 | INFO | Serializing RTMetrics to json | 20260507 07:08:47.015 | INFO | Serialization done, swapping tmp file now... | 20260507 07:08:47.015 | INFO | RTMetrics done | 20260507 07:08:47.080 | INFO | Finished a round of Baz IO: Wrote 1.649 GiB in 3.173 seconds | 20260507 07:08:47.080 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.58% / 0.00% / 67.42% | 20260507 07:08:47.080 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.109764% / 1.19209e-05% / 3.57628e-05% | 20260507 07:08:47.080 | INFO | Baz IO overall: | 20260507 07:08:47.080 | INFO | BytesWritten : 4043941703075 | 20260507 07:08:47.080 | INFO | NumEvents : 1936687207547 | 20260507 07:08:48.945 | INFO | Analyzing chunk frames = [8052736,8053248) | 20260507 07:08:48.945 | INFO | MemInfo: { | 20260507 07:08:48.945 | INFO | "gpu" : " 17 GB", | 20260507 07:08:48.945 | INFO | "huge" : "", | 20260507 07:08:48.945 | INFO | "huge_shared" : " 27 GB", | 20260507 07:08:48.945 | INFO | "malloc" : " 14 GB", | 20260507 07:08:48.945 | INFO | "pinned" : " 29 GB" | 20260507 07:08:48.946 | INFO | } | 20260507 07:08:48.946 | INFO | Perf report for the last 5.14069 seconds: | 20260507 07:08:48.946 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:08:48.946 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:48.946 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:48.946 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:48.946 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 07:08:48.946 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:48.946 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 07:08:48.946 | INFO | PRE_HQ : 58.9%, 1.8, 55.1ms, 0 | 20260507 07:08:48.946 | INFO | BAZWRITER : 61.2%, 3.1, 98.5ms, 0 | 20260507 07:08:48.946 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:08:48.946 | INFO | Uploaded 36GiB to GPU in 2.03663s (17.6762GiB/s) | 20260507 07:08:48.946 | INFO | Downloaded 38GiB to GPU in 2.18305s (17.4068GiB/s) | 20260507 07:08:54.094 | INFO | Analyzing chunk frames = [8053248,8053760) | 20260507 07:08:54.094 | INFO | MemInfo: { | 20260507 07:08:54.094 | INFO | "gpu" : " 17 GB", | 20260507 07:08:54.094 | INFO | "huge" : "", | 20260507 07:08:54.094 | INFO | "huge_shared" : " 27 GB", | 20260507 07:08:54.094 | INFO | "malloc" : " 14 GB", | 20260507 07:08:54.094 | INFO | "pinned" : " 29 GB" | 20260507 07:08:54.094 | INFO | } | 20260507 07:08:54.094 | INFO | Perf report for the last 5.14874 seconds: | 20260507 07:08:54.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:08:54.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:54.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:54.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:54.094 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 07:08:54.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:54.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:54.094 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260507 07:08:54.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:54.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:54.095 | INFO | Uploaded 36GiB to GPU in 1.98645s (18.1228GiB/s) | 20260507 07:08:54.095 | INFO | Downloaded 34GiB to GPU in 1.92751s (17.6394GiB/s) | 20260507 07:08:59.222 | INFO | Analyzing chunk frames = [8053760,8054272) | 20260507 07:08:59.222 | INFO | MemInfo: { | 20260507 07:08:59.222 | INFO | "gpu" : " 17 GB", | 20260507 07:08:59.222 | INFO | "huge" : "", | 20260507 07:08:59.222 | INFO | "huge_shared" : " 27 GB", | 20260507 07:08:59.222 | INFO | "malloc" : " 14 GB", | 20260507 07:08:59.222 | INFO | "pinned" : " 29 GB" | 20260507 07:08:59.222 | INFO | } | 20260507 07:08:59.222 | INFO | Perf report for the last 5.12773 seconds: | 20260507 07:08:59.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:08:59.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:59.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:59.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:59.222 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:08:59.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:59.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:59.222 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:08:59.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:59.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:08:59.222 | INFO | Uploaded 36GiB to GPU in 2.03234s (17.7136GiB/s) | 20260507 07:08:59.222 | INFO | Downloaded 34GiB to GPU in 1.91568s (17.7483GiB/s) | 20260507 07:09:04.359 | INFO | Analyzing chunk frames = [8054272,8054784) | 20260507 07:09:04.359 | INFO | MemInfo: { | 20260507 07:09:04.359 | INFO | "gpu" : " 17 GB", | 20260507 07:09:04.359 | INFO | "huge" : "", | 20260507 07:09:04.359 | INFO | "huge_shared" : " 27 GB", | 20260507 07:09:04.359 | INFO | "malloc" : " 14 GB", | 20260507 07:09:04.359 | INFO | "pinned" : " 29 GB" | 20260507 07:09:04.359 | INFO | } | 20260507 07:09:04.359 | INFO | Perf report for the last 5.13692 seconds: | 20260507 07:09:04.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:09:04.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:04.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:04.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:04.359 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 07:09:04.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:04.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:04.359 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:09:04.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:04.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:04.359 | INFO | Uploaded 36GiB to GPU in 2.02444s (17.7827GiB/s) | 20260507 07:09:04.359 | INFO | Downloaded 34GiB to GPU in 1.93977s (17.5278GiB/s) | 20260507 07:09:09.508 | INFO | Analyzing chunk frames = [8054784,8055296) | 20260507 07:09:09.508 | INFO | MemInfo: { | 20260507 07:09:09.508 | INFO | "gpu" : " 17 GB", | 20260507 07:09:09.508 | INFO | "huge" : "", | 20260507 07:09:09.508 | INFO | "huge_shared" : " 27 GB", | 20260507 07:09:09.508 | INFO | "malloc" : " 14 GB", | 20260507 07:09:09.508 | INFO | "pinned" : " 29 GB" | 20260507 07:09:09.508 | INFO | } | 20260507 07:09:09.508 | INFO | Perf report for the last 5.14944 seconds: | 20260507 07:09:09.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:09:09.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:09.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:09.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:09.509 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 07:09:09.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:09.509 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:09.509 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 07:09:09.509 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:09.509 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:09.509 | INFO | Uploaded 36GiB to GPU in 2.04723s (17.5848GiB/s) | 20260507 07:09:09.509 | INFO | Downloaded 34GiB to GPU in 1.93861s (17.5384GiB/s) | 20260507 07:09:14.648 | INFO | Analyzing chunk frames = [8055296,8055808) | 20260507 07:09:14.648 | INFO | MemInfo: { | 20260507 07:09:14.648 | INFO | "gpu" : " 17 GB", | 20260507 07:09:14.648 | INFO | "huge" : "", | 20260507 07:09:14.648 | INFO | "huge_shared" : " 27 GB", | 20260507 07:09:14.648 | INFO | "malloc" : " 14 GB", | 20260507 07:09:14.648 | INFO | "pinned" : " 29 GB" | 20260507 07:09:14.648 | INFO | } | 20260507 07:09:14.648 | INFO | Perf report for the last 5.13984 seconds: | 20260507 07:09:14.648 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:09:14.648 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:14.648 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:14.648 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:14.648 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 07:09:14.648 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:14.648 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:14.648 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:09:14.648 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:14.648 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:14.649 | INFO | Uploaded 36GiB to GPU in 2.05715s (17.4999GiB/s) | 20260507 07:09:14.649 | INFO | Downloaded 34GiB to GPU in 1.9113s (17.7889GiB/s) | 20260507 07:09:19.779 | INFO | Analyzing chunk frames = [8055808,8056320) | 20260507 07:09:19.779 | INFO | MemInfo: { | 20260507 07:09:19.779 | INFO | "gpu" : " 17 GB", | 20260507 07:09:19.779 | INFO | "huge" : "", | 20260507 07:09:19.779 | INFO | "huge_shared" : " 27 GB", | 20260507 07:09:19.779 | INFO | "malloc" : " 14 GB", | 20260507 07:09:19.779 | INFO | "pinned" : " 29 GB" | 20260507 07:09:19.779 | INFO | } | 20260507 07:09:19.779 | INFO | Perf report for the last 5.13085 seconds: | 20260507 07:09:19.779 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:09:19.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:19.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:19.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:19.779 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260507 07:09:19.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:19.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:19.779 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 07:09:19.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:19.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:19.780 | INFO | Uploaded 36GiB to GPU in 1.90211s (18.9263GiB/s) | 20260507 07:09:19.780 | INFO | Downloaded 34GiB to GPU in 1.97598s (17.2067GiB/s) | 20260507 07:09:24.917 | INFO | Analyzing chunk frames = [8056320,8056832) | 20260507 07:09:24.917 | INFO | MemInfo: { | 20260507 07:09:24.917 | INFO | "gpu" : " 17 GB", | 20260507 07:09:24.917 | INFO | "huge" : "", | 20260507 07:09:24.917 | INFO | "huge_shared" : " 27 GB", | 20260507 07:09:24.917 | INFO | "malloc" : " 14 GB", | 20260507 07:09:24.917 | INFO | "pinned" : " 29 GB" | 20260507 07:09:24.917 | INFO | } | 20260507 07:09:24.917 | INFO | Perf report for the last 5.13834 seconds: | 20260507 07:09:24.917 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:09:24.917 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:24.917 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:24.918 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:24.918 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:09:24.918 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:24.918 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:24.918 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 07:09:24.918 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:24.918 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:24.918 | INFO | Uploaded 36GiB to GPU in 1.98222s (18.1614GiB/s) | 20260507 07:09:24.918 | INFO | Downloaded 34GiB to GPU in 1.96289s (17.3214GiB/s) | 20260507 07:09:25.014 | INFO | Starting a round of Baz IO | 20260507 07:09:28.108 | INFO | Populating RTMetrics report | 20260507 07:09:28.108 | INFO | RTMetrics report populated | 20260507 07:09:28.108 | INFO | Opening tmp RTMetrics json file | 20260507 07:09:28.112 | INFO | Serializing RTMetrics to json | 20260507 07:09:28.127 | INFO | Serialization done, swapping tmp file now... | 20260507 07:09:28.128 | INFO | RTMetrics done | 20260507 07:09:28.199 | INFO | Finished a round of Baz IO: Wrote 1.647 GiB in 3.184 seconds | 20260507 07:09:28.199 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.58% / 0.00% / 67.42% | 20260507 07:09:28.199 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.109283% / 0% / 5.16574e-05% | 20260507 07:09:28.199 | INFO | Baz IO overall: | 20260507 07:09:28.199 | INFO | BytesWritten : 4045710226078 | 20260507 07:09:28.199 | INFO | NumEvents : 1937513563797 | 20260507 07:09:30.061 | INFO | Analyzing chunk frames = [8056832,8057344) | 20260507 07:09:30.061 | INFO | MemInfo: { | 20260507 07:09:30.061 | INFO | "gpu" : " 17 GB", | 20260507 07:09:30.061 | INFO | "huge" : "", | 20260507 07:09:30.062 | INFO | "huge_shared" : " 27 GB", | 20260507 07:09:30.062 | INFO | "malloc" : " 14 GB", | 20260507 07:09:30.062 | INFO | "pinned" : " 29 GB" | 20260507 07:09:30.062 | INFO | } | 20260507 07:09:30.062 | INFO | Perf report for the last 5.14414 seconds: | 20260507 07:09:30.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:09:30.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:30.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:30.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:30.062 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 07:09:30.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:30.062 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 07:09:30.062 | INFO | PRE_HQ : 59.0%, 1.8, 54.9ms, 0 | 20260507 07:09:30.062 | INFO | BAZWRITER : 61.5%, 3.1, 97.6ms, 0 | 20260507 07:09:30.062 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 07:09:30.062 | INFO | Uploaded 36GiB to GPU in 2.04085s (17.6397GiB/s) | 20260507 07:09:30.062 | INFO | Downloaded 38GiB to GPU in 2.18063s (17.4261GiB/s) | 20260507 07:09:35.202 | INFO | Analyzing chunk frames = [8057344,8057856) | 20260507 07:09:35.202 | INFO | MemInfo: { | 20260507 07:09:35.202 | INFO | "gpu" : " 17 GB", | 20260507 07:09:35.202 | INFO | "huge" : "", | 20260507 07:09:35.202 | INFO | "huge_shared" : " 27 GB", | 20260507 07:09:35.202 | INFO | "malloc" : " 14 GB", | 20260507 07:09:35.202 | INFO | "pinned" : " 29 GB" | 20260507 07:09:35.202 | INFO | } | 20260507 07:09:35.202 | INFO | Perf report for the last 5.14076 seconds: | 20260507 07:09:35.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:09:35.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:35.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:35.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:35.203 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:09:35.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:35.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:35.203 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:09:35.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:35.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:35.203 | INFO | Uploaded 36GiB to GPU in 1.98856s (18.1036GiB/s) | 20260507 07:09:35.203 | INFO | Downloaded 34GiB to GPU in 1.92803s (17.6346GiB/s) | 20260507 07:09:40.343 | INFO | Analyzing chunk frames = [8057856,8058368) | 20260507 07:09:40.343 | INFO | MemInfo: { | 20260507 07:09:40.343 | INFO | "gpu" : " 17 GB", | 20260507 07:09:40.343 | INFO | "huge" : "", | 20260507 07:09:40.343 | INFO | "huge_shared" : " 27 GB", | 20260507 07:09:40.343 | INFO | "malloc" : " 14 GB", | 20260507 07:09:40.343 | INFO | "pinned" : " 29 GB" | 20260507 07:09:40.343 | INFO | } | 20260507 07:09:40.343 | INFO | Perf report for the last 5.14046 seconds: | 20260507 07:09:40.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:09:40.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:40.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:40.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:40.343 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 07:09:40.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:40.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:40.343 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:09:40.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:40.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:40.343 | INFO | Uploaded 36GiB to GPU in 2.04451s (17.6082GiB/s) | 20260507 07:09:40.343 | INFO | Downloaded 34GiB to GPU in 1.95235s (17.4149GiB/s) | 20260507 07:09:45.483 | INFO | Analyzing chunk frames = [8058368,8058880) | 20260507 07:09:45.483 | INFO | MemInfo: { | 20260507 07:09:45.483 | INFO | "gpu" : " 17 GB", | 20260507 07:09:45.483 | INFO | "huge" : "", | 20260507 07:09:45.483 | INFO | "huge_shared" : " 27 GB", | 20260507 07:09:45.483 | INFO | "malloc" : " 14 GB", | 20260507 07:09:45.483 | INFO | "pinned" : " 29 GB" | 20260507 07:09:45.483 | INFO | } | 20260507 07:09:45.483 | INFO | Perf report for the last 5.14019 seconds: | 20260507 07:09:45.483 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:09:45.483 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:45.483 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:45.483 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:45.483 | INFO | ANALYSIS : 60.2%, 2.9, 93.0ms, 0 | 20260507 07:09:45.483 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:45.483 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:45.483 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:09:45.483 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:45.483 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:45.483 | INFO | Uploaded 36GiB to GPU in 2.01315s (17.8824GiB/s) | 20260507 07:09:45.483 | INFO | Downloaded 34GiB to GPU in 1.95662s (17.3769GiB/s) | 20260507 07:09:50.619 | INFO | Analyzing chunk frames = [8058880,8059392) | 20260507 07:09:50.619 | INFO | MemInfo: { | 20260507 07:09:50.619 | INFO | "gpu" : " 17 GB", | 20260507 07:09:50.619 | INFO | "huge" : "", | 20260507 07:09:50.619 | INFO | "huge_shared" : " 27 GB", | 20260507 07:09:50.619 | INFO | "malloc" : " 14 GB", | 20260507 07:09:50.619 | INFO | "pinned" : " 29 GB" | 20260507 07:09:50.619 | INFO | } | 20260507 07:09:50.619 | INFO | Perf report for the last 5.13635 seconds: | 20260507 07:09:50.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:09:50.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:50.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:50.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:50.619 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260507 07:09:50.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:50.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:50.620 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:09:50.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:50.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:50.620 | INFO | Uploaded 36GiB to GPU in 2.03227s (17.7142GiB/s) | 20260507 07:09:50.620 | INFO | Downloaded 34GiB to GPU in 1.92318s (17.679GiB/s) | 20260507 07:09:55.761 | INFO | Analyzing chunk frames = [8059392,8059904) | 20260507 07:09:55.761 | INFO | MemInfo: { | 20260507 07:09:55.761 | INFO | "gpu" : " 17 GB", | 20260507 07:09:55.761 | INFO | "huge" : "", | 20260507 07:09:55.761 | INFO | "huge_shared" : " 27 GB", | 20260507 07:09:55.761 | INFO | "malloc" : " 14 GB", | 20260507 07:09:55.761 | INFO | "pinned" : " 29 GB" | 20260507 07:09:55.761 | INFO | } | 20260507 07:09:55.761 | INFO | Perf report for the last 5.14143 seconds: | 20260507 07:09:55.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:09:55.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:55.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:55.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:55.761 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 07:09:55.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:55.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:55.761 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:09:55.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:55.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:09:55.761 | INFO | Uploaded 36GiB to GPU in 2.03454s (17.6944GiB/s) | 20260507 07:09:55.761 | INFO | Downloaded 34GiB to GPU in 1.92131s (17.6962GiB/s) | 20260507 07:10:00.901 | INFO | Analyzing chunk frames = [8059904,8060416) | 20260507 07:10:00.901 | INFO | MemInfo: { | 20260507 07:10:00.901 | INFO | "gpu" : " 17 GB", | 20260507 07:10:00.901 | INFO | "huge" : "", | 20260507 07:10:00.901 | INFO | "huge_shared" : " 27 GB", | 20260507 07:10:00.901 | INFO | "malloc" : " 14 GB", | 20260507 07:10:00.901 | INFO | "pinned" : " 29 GB" | 20260507 07:10:00.901 | INFO | } | 20260507 07:10:00.901 | INFO | Perf report for the last 5.14048 seconds: | 20260507 07:10:00.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:10:00.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:00.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:00.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:00.901 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:10:00.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:00.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:00.901 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:10:00.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:00.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:00.902 | INFO | Uploaded 36GiB to GPU in 1.8833s (19.1154GiB/s) | 20260507 07:10:00.902 | INFO | Downloaded 34GiB to GPU in 1.97315s (17.2313GiB/s) | 20260507 07:10:06.046 | INFO | Analyzing chunk frames = [8060416,8060928) | 20260507 07:10:06.046 | INFO | MemInfo: { | 20260507 07:10:06.046 | INFO | "gpu" : " 17 GB", | 20260507 07:10:06.046 | INFO | "huge" : "", | 20260507 07:10:06.047 | INFO | "huge_shared" : " 27 GB", | 20260507 07:10:06.047 | INFO | "malloc" : " 14 GB", | 20260507 07:10:06.047 | INFO | "pinned" : " 29 GB" | 20260507 07:10:06.047 | INFO | } | 20260507 07:10:06.047 | INFO | Perf report for the last 5.14524 seconds: | 20260507 07:10:06.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:10:06.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:06.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:06.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:06.047 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 07:10:06.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:06.047 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:06.047 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:10:06.047 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:06.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:06.047 | INFO | Uploaded 36GiB to GPU in 1.97222s (18.2535GiB/s) | 20260507 07:10:06.047 | INFO | Downloaded 34GiB to GPU in 1.96373s (17.314GiB/s) | 20260507 07:10:06.047 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:10:06.047 | INFO | Total mem usage: 18 GiB | 20260507 07:10:06.047 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:10:06.047 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:10:06.047 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:10:06.047 | INFO | High Water Breakdown: | 20260507 07:10:06.047 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:10:06.047 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:10:06.047 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:10:06.047 | INFO | ------------------------------------------ | 20260507 07:10:06.047 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:10:06.047 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:10:06.047 | INFO | Total mem usage: 26.5 GiB | 20260507 07:10:06.047 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:10:06.047 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:10:06.047 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:10:06.047 | INFO | High Water Breakdown: | 20260507 07:10:06.047 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:10:06.047 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:10:06.047 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:10:06.047 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:10:06.047 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:10:06.047 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:10:06.047 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:10:06.047 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:10:06.047 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:10:06.047 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:10:06.047 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:10:06.047 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:10:06.047 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:10:06.047 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:10:06.047 | INFO | ------------------------------------------ | 20260507 07:10:06.047 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:10:06.047 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:10:06.047 | INFO | Total mem usage: 25.6 GiB | 20260507 07:10:06.047 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:10:06.047 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:10:06.047 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:10:06.047 | INFO | High Water Breakdown: | 20260507 07:10:06.047 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:10:06.047 | INFO | ------------------------------------------ | 20260507 07:10:06.047 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:10:06.047 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:10:06.047 | INFO | Total mem usage: 15.7 GiB | 20260507 07:10:06.047 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:10:06.047 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:10:06.047 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:10:06.048 | INFO | High Water Breakdown: | 20260507 07:10:06.048 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:10:06.048 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:10:06.048 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:10:06.048 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:10:06.048 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:10:06.048 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:10:06.048 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:10:06.048 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:10:06.048 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:10:06.048 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:10:06.048 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:10:06.048 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:10:06.048 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:10:06.048 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:10:06.048 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:10:06.048 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:10:06.048 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:10:06.048 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:10:06.048 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:10:06.048 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:10:06.048 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:10:06.048 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:10:06.048 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:10:06.048 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:10:06.048 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:10:06.048 | INFO | ------------------------------------------ | 20260507 07:10:06.048 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:10:06.048 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:10:06.048 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:10:06.048 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:10:06.048 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.50e+01, 1.33e+01 / 1.24e+00 | 20260507 07:10:06.048 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.02e-01, 1.05e-02 / 4.45e-03 | 20260507 07:10:06.048 | INFO | PerfReport: 11.70%, Baseline -- 3.59e+00 / 4.43e+00, 5.19e-01 / 2.73e-01 | 20260507 07:10:06.048 | INFO | PerfReport: 3.24%, Histogram -- 9.94e-01 / 1.49e+00, 1.71e-01 / 1.21e-01 | 20260507 07:10:06.048 | INFO | PerfReport: 16.30%, DME -- 4.00e+01 / 4.52e+01, 4.05e+00 / 1.74e+00 | 20260507 07:10:06.048 | INFO | PerfReport: 57.64%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.26e+00 / 1.31e+00 | 20260507 07:10:06.048 | INFO | PerfReport: 6.91%, PulseAccumulating -- 2.12e+00 / 2.68e+00, 3.87e-01 / 2.27e-01 | 20260507 07:10:06.048 | INFO | PerfReport: 2.94%, Metrics -- 9.03e-01 / 1.22e+00, 2.43e-01 / 1.33e-01 | 20260507 07:10:06.048 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.02e-01 / 3.55e-01, 5.01e-02 / 1.90e-02 | 20260507 07:10:06.048 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:10:06.048 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:10:06.048 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:10:06.048 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 07:10:06.048 | INFO | PerfReport: 21.74%, Upload -- 2.09e+01 / 0.00e+00, 7.02e+00 / 0.00e+00 | 20260507 07:10:06.048 | INFO | PerfReport: 21.38%, Download -- 2.05e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 07:10:06.048 | INFO | PerfReport: 31.97%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 07:10:06.048 | INFO | PerfReport: 1.49%, LockUpload -- 1.43e+00 / 0.00e+00, 4.87e+00 / 0.00e+00 | 20260507 07:10:06.048 | INFO | PerfReport: 1.34%, LockDownload -- 1.29e+00 / 0.00e+00, 3.15e+00 / 0.00e+00 | 20260507 07:10:06.048 | INFO | PerfReport: 22.07%, LockCompute -- 2.12e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 07:10:06.048 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.89e-02 / 0.00e+00 | 20260507 07:10:06.144 | INFO | Starting a round of Baz IO | 20260507 07:10:09.240 | INFO | Populating RTMetrics report | 20260507 07:10:09.240 | INFO | RTMetrics report populated | 20260507 07:10:09.240 | INFO | Opening tmp RTMetrics json file | 20260507 07:10:09.257 | INFO | Serializing RTMetrics to json | 20260507 07:10:09.259 | INFO | Serialization done, swapping tmp file now... | 20260507 07:10:09.259 | INFO | RTMetrics done | 20260507 07:10:09.324 | INFO | Finished a round of Baz IO: Wrote 1.647 GiB in 3.18 seconds | 20260507 07:10:09.324 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.58% / 0.00% / 67.42% | 20260507 07:10:09.324 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.108937% / 0% / 6.35783e-05% | 20260507 07:10:09.324 | INFO | Baz IO overall: | 20260507 07:10:09.324 | INFO | BytesWritten : 4047478612177 | 20260507 07:10:09.324 | INFO | NumEvents : 1938339813814 | 20260507 07:10:11.171 | INFO | Analyzing chunk frames = [8060928,8061440) | 20260507 07:10:11.171 | INFO | MemInfo: { | 20260507 07:10:11.171 | INFO | "gpu" : " 17 GB", | 20260507 07:10:11.171 | INFO | "huge" : "", | 20260507 07:10:11.171 | INFO | "huge_shared" : " 27 GB", | 20260507 07:10:11.171 | INFO | "malloc" : " 14 GB", | 20260507 07:10:11.171 | INFO | "pinned" : " 29 GB" | 20260507 07:10:11.171 | INFO | } | 20260507 07:10:11.171 | INFO | Perf report for the last 5.12398 seconds: | 20260507 07:10:11.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:10:11.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:11.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:11.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:11.171 | INFO | ANALYSIS : 61.8%, 3.0, 95.9ms, 0 | 20260507 07:10:11.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:11.171 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 07:10:11.171 | INFO | PRE_HQ : 58.9%, 1.8, 54.4ms, 0 | 20260507 07:10:11.171 | INFO | BAZWRITER : 61.6%, 3.1, 97.4ms, 0 | 20260507 07:10:11.171 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:10:11.171 | INFO | Uploaded 36GiB to GPU in 2.06799s (17.4082GiB/s) | 20260507 07:10:11.171 | INFO | Downloaded 38GiB to GPU in 2.16623s (17.542GiB/s) | 20260507 07:10:16.314 | INFO | Analyzing chunk frames = [8061440,8061952) | 20260507 07:10:16.314 | INFO | MemInfo: { | 20260507 07:10:16.314 | INFO | "gpu" : " 17 GB", | 20260507 07:10:16.314 | INFO | "huge" : "", | 20260507 07:10:16.314 | INFO | "huge_shared" : " 27 GB", | 20260507 07:10:16.314 | INFO | "malloc" : " 14 GB", | 20260507 07:10:16.314 | INFO | "pinned" : " 29 GB" | 20260507 07:10:16.314 | INFO | } | 20260507 07:10:16.314 | INFO | Perf report for the last 5.1438 seconds: | 20260507 07:10:16.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:10:16.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:16.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:16.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:16.314 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 07:10:16.314 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:16.314 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:16.315 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 07:10:16.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:16.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:16.315 | INFO | Uploaded 36GiB to GPU in 2.00288s (17.9741GiB/s) | 20260507 07:10:16.315 | INFO | Downloaded 34GiB to GPU in 1.94151s (17.5122GiB/s) | 20260507 07:10:21.455 | INFO | Analyzing chunk frames = [8061952,8062464) | 20260507 07:10:21.455 | INFO | MemInfo: { | 20260507 07:10:21.455 | INFO | "gpu" : " 17 GB", | 20260507 07:10:21.455 | INFO | "huge" : "", | 20260507 07:10:21.455 | INFO | "huge_shared" : " 27 GB", | 20260507 07:10:21.455 | INFO | "malloc" : " 14 GB", | 20260507 07:10:21.455 | INFO | "pinned" : " 29 GB" | 20260507 07:10:21.455 | INFO | } | 20260507 07:10:21.455 | INFO | Perf report for the last 5.14051 seconds: | 20260507 07:10:21.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:10:21.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:21.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:21.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:21.455 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 07:10:21.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:21.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:21.455 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:10:21.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:21.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:21.455 | INFO | Uploaded 36GiB to GPU in 2.02029s (17.8192GiB/s) | 20260507 07:10:21.455 | INFO | Downloaded 34GiB to GPU in 1.95305s (17.4087GiB/s) | 20260507 07:10:26.596 | INFO | Analyzing chunk frames = [8062464,8062976) | 20260507 07:10:26.597 | INFO | MemInfo: { | 20260507 07:10:26.597 | INFO | "gpu" : " 17 GB", | 20260507 07:10:26.597 | INFO | "huge" : "", | 20260507 07:10:26.597 | INFO | "huge_shared" : " 27 GB", | 20260507 07:10:26.597 | INFO | "malloc" : " 14 GB", | 20260507 07:10:26.597 | INFO | "pinned" : " 29 GB" | 20260507 07:10:26.597 | INFO | } | 20260507 07:10:26.597 | INFO | Perf report for the last 5.14182 seconds: | 20260507 07:10:26.597 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:10:26.597 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:26.597 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:26.597 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:26.597 | INFO | ANALYSIS : 60.3%, 2.9, 93.2ms, 0 | 20260507 07:10:26.597 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:26.597 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:26.597 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:10:26.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:26.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:26.597 | INFO | Uploaded 36GiB to GPU in 2.02977s (17.736GiB/s) | 20260507 07:10:26.597 | INFO | Downloaded 34GiB to GPU in 1.95381s (17.4019GiB/s) | 20260507 07:10:31.731 | INFO | Analyzing chunk frames = [8062976,8063488) | 20260507 07:10:31.731 | INFO | MemInfo: { | 20260507 07:10:31.731 | INFO | "gpu" : " 17 GB", | 20260507 07:10:31.731 | INFO | "huge" : "", | 20260507 07:10:31.731 | INFO | "huge_shared" : " 27 GB", | 20260507 07:10:31.731 | INFO | "malloc" : " 14 GB", | 20260507 07:10:31.731 | INFO | "pinned" : " 29 GB" | 20260507 07:10:31.731 | INFO | } | 20260507 07:10:31.731 | INFO | Perf report for the last 5.13398 seconds: | 20260507 07:10:31.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:10:31.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:31.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:31.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:31.731 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 07:10:31.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:31.731 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:31.731 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:10:31.731 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:31.731 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:31.731 | INFO | Uploaded 36GiB to GPU in 2.03199s (17.7166GiB/s) | 20260507 07:10:31.731 | INFO | Downloaded 34GiB to GPU in 1.91783s (17.7284GiB/s) | 20260507 07:10:36.865 | INFO | Analyzing chunk frames = [8063488,8064000) | 20260507 07:10:36.865 | INFO | MemInfo: { | 20260507 07:10:36.865 | INFO | "gpu" : " 17 GB", | 20260507 07:10:36.865 | INFO | "huge" : "", | 20260507 07:10:36.865 | INFO | "huge_shared" : " 27 GB", | 20260507 07:10:36.865 | INFO | "malloc" : " 14 GB", | 20260507 07:10:36.865 | INFO | "pinned" : " 29 GB" | 20260507 07:10:36.865 | INFO | } | 20260507 07:10:36.865 | INFO | Perf report for the last 5.13442 seconds: | 20260507 07:10:36.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:10:36.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:36.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:36.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:36.865 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 07:10:36.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:36.865 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:36.865 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 07:10:36.865 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:36.865 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:36.865 | INFO | Uploaded 36GiB to GPU in 2.0284s (17.7479GiB/s) | 20260507 07:10:36.866 | INFO | Downloaded 34GiB to GPU in 1.91679s (17.738GiB/s) | 20260507 07:10:42.013 | INFO | Analyzing chunk frames = [8064000,8064512) | 20260507 07:10:42.013 | INFO | MemInfo: { | 20260507 07:10:42.013 | INFO | "gpu" : " 17 GB", | 20260507 07:10:42.013 | INFO | "huge" : "", | 20260507 07:10:42.013 | INFO | "huge_shared" : " 27 GB", | 20260507 07:10:42.013 | INFO | "malloc" : " 14 GB", | 20260507 07:10:42.013 | INFO | "pinned" : " 29 GB" | 20260507 07:10:42.014 | INFO | } | 20260507 07:10:42.014 | INFO | Perf report for the last 5.14836 seconds: | 20260507 07:10:42.014 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:10:42.014 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:42.014 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:42.014 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:42.014 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 07:10:42.014 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:42.014 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:42.014 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 07:10:42.014 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:42.014 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:42.014 | INFO | Uploaded 36GiB to GPU in 1.98255s (18.1584GiB/s) | 20260507 07:10:42.014 | INFO | Downloaded 34GiB to GPU in 1.98434s (17.1341GiB/s) | 20260507 07:10:47.157 | INFO | Analyzing chunk frames = [8064512,8065024) | 20260507 07:10:47.157 | INFO | MemInfo: { | 20260507 07:10:47.157 | INFO | "gpu" : " 17 GB", | 20260507 07:10:47.157 | INFO | "huge" : "", | 20260507 07:10:47.157 | INFO | "huge_shared" : " 27 GB", | 20260507 07:10:47.157 | INFO | "malloc" : " 14 GB", | 20260507 07:10:47.157 | INFO | "pinned" : " 29 GB" | 20260507 07:10:47.157 | INFO | } | 20260507 07:10:47.157 | INFO | Perf report for the last 5.14349 seconds: | 20260507 07:10:47.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:10:47.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:47.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:47.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:47.157 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:10:47.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:47.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:47.157 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:10:47.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:47.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:47.157 | INFO | Uploaded 36GiB to GPU in 1.99473s (18.0475GiB/s) | 20260507 07:10:47.158 | INFO | Downloaded 34GiB to GPU in 1.96131s (17.3354GiB/s) | 20260507 07:10:47.254 | INFO | Starting a round of Baz IO | 20260507 07:10:50.343 | INFO | Populating RTMetrics report | 20260507 07:10:50.343 | INFO | RTMetrics report populated | 20260507 07:10:50.343 | INFO | Opening tmp RTMetrics json file | 20260507 07:10:50.348 | INFO | Serializing RTMetrics to json | 20260507 07:10:50.361 | INFO | Serialization done, swapping tmp file now... | 20260507 07:10:50.361 | INFO | RTMetrics done | 20260507 07:10:50.426 | INFO | Finished a round of Baz IO: Wrote 1.648 GiB in 3.172 seconds | 20260507 07:10:50.426 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.58% / 0.00% / 67.42% | 20260507 07:10:50.426 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.108918% / 0% / 4.76837e-05% | 20260507 07:10:50.426 | INFO | Baz IO overall: | 20260507 07:10:50.426 | INFO | BytesWritten : 4049248620126 | 20260507 07:10:50.426 | INFO | NumEvents : 1939167019298 | 20260507 07:10:52.299 | INFO | Analyzing chunk frames = [8065024,8065536) | 20260507 07:10:52.299 | INFO | MemInfo: { | 20260507 07:10:52.299 | INFO | "gpu" : " 17 GB", | 20260507 07:10:52.299 | INFO | "huge" : "", | 20260507 07:10:52.299 | INFO | "huge_shared" : " 27 GB", | 20260507 07:10:52.299 | INFO | "malloc" : " 14 GB", | 20260507 07:10:52.299 | INFO | "pinned" : " 29 GB" | 20260507 07:10:52.299 | INFO | } | 20260507 07:10:52.299 | INFO | Perf report for the last 5.14195 seconds: | 20260507 07:10:52.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:10:52.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:52.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:52.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:52.299 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260507 07:10:52.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:52.299 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 07:10:52.299 | INFO | PRE_HQ : 58.9%, 1.8, 54.7ms, 0 | 20260507 07:10:52.299 | INFO | BAZWRITER : 61.3%, 3.1, 97.8ms, 0 | 20260507 07:10:52.299 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:10:52.299 | INFO | Uploaded 36GiB to GPU in 2.06629s (17.4225GiB/s) | 20260507 07:10:52.299 | INFO | Downloaded 38GiB to GPU in 2.17469s (17.4738GiB/s) | 20260507 07:10:57.425 | INFO | Analyzing chunk frames = [8065536,8066048) | 20260507 07:10:57.425 | INFO | MemInfo: { | 20260507 07:10:57.425 | INFO | "gpu" : " 17 GB", | 20260507 07:10:57.425 | INFO | "huge" : "", | 20260507 07:10:57.425 | INFO | "huge_shared" : " 27 GB", | 20260507 07:10:57.425 | INFO | "malloc" : " 14 GB", | 20260507 07:10:57.425 | INFO | "pinned" : " 29 GB" | 20260507 07:10:57.425 | INFO | } | 20260507 07:10:57.425 | INFO | Perf report for the last 5.12578 seconds: | 20260507 07:10:57.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:10:57.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:57.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:57.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:57.425 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 07:10:57.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:57.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:57.425 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 07:10:57.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:57.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:10:57.425 | INFO | Uploaded 36GiB to GPU in 2.03297s (17.7081GiB/s) | 20260507 07:10:57.425 | INFO | Downloaded 34GiB to GPU in 1.94906s (17.4443GiB/s) | 20260507 07:11:02.571 | INFO | Analyzing chunk frames = [8066048,8066560) | 20260507 07:11:02.571 | INFO | MemInfo: { | 20260507 07:11:02.571 | INFO | "gpu" : " 17 GB", | 20260507 07:11:02.571 | INFO | "huge" : "", | 20260507 07:11:02.571 | INFO | "huge_shared" : " 27 GB", | 20260507 07:11:02.571 | INFO | "malloc" : " 14 GB", | 20260507 07:11:02.571 | INFO | "pinned" : " 29 GB" | 20260507 07:11:02.571 | INFO | } | 20260507 07:11:02.571 | INFO | Perf report for the last 5.14624 seconds: | 20260507 07:11:02.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:11:02.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:02.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:02.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:02.571 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 07:11:02.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:02.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:02.571 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:11:02.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:02.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:02.571 | INFO | Uploaded 36GiB to GPU in 2.04726s (17.5845GiB/s) | 20260507 07:11:02.571 | INFO | Downloaded 34GiB to GPU in 1.9531s (17.4082GiB/s) | 20260507 07:11:07.711 | INFO | Analyzing chunk frames = [8066560,8067072) | 20260507 07:11:07.711 | INFO | MemInfo: { | 20260507 07:11:07.711 | INFO | "gpu" : " 17 GB", | 20260507 07:11:07.711 | INFO | "huge" : "", | 20260507 07:11:07.711 | INFO | "huge_shared" : " 27 GB", | 20260507 07:11:07.711 | INFO | "malloc" : " 14 GB", | 20260507 07:11:07.711 | INFO | "pinned" : " 29 GB" | 20260507 07:11:07.711 | INFO | } | 20260507 07:11:07.711 | INFO | Perf report for the last 5.1399 seconds: | 20260507 07:11:07.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:11:07.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:07.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:07.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:07.711 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 07:11:07.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:07.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:07.711 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:11:07.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:07.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:07.711 | INFO | Uploaded 36GiB to GPU in 2.00218s (17.9804GiB/s) | 20260507 07:11:07.711 | INFO | Downloaded 34GiB to GPU in 1.93146s (17.6033GiB/s) | 20260507 07:11:12.847 | INFO | Analyzing chunk frames = [8067072,8067584) | 20260507 07:11:12.847 | INFO | MemInfo: { | 20260507 07:11:12.847 | INFO | "gpu" : " 17 GB", | 20260507 07:11:12.847 | INFO | "huge" : "", | 20260507 07:11:12.847 | INFO | "huge_shared" : " 27 GB", | 20260507 07:11:12.848 | INFO | "malloc" : " 14 GB", | 20260507 07:11:12.848 | INFO | "pinned" : " 29 GB" | 20260507 07:11:12.848 | INFO | } | 20260507 07:11:12.848 | INFO | Perf report for the last 5.1367 seconds: | 20260507 07:11:12.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:11:12.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:12.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:12.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:12.848 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 07:11:12.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:12.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:12.848 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:11:12.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:12.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:12.848 | INFO | Uploaded 36GiB to GPU in 2.05549s (17.5141GiB/s) | 20260507 07:11:12.848 | INFO | Downloaded 34GiB to GPU in 1.90642s (17.8345GiB/s) | 20260507 07:11:17.987 | INFO | Analyzing chunk frames = [8067584,8068096) | 20260507 07:11:17.987 | INFO | MemInfo: { | 20260507 07:11:17.987 | INFO | "gpu" : " 17 GB", | 20260507 07:11:17.987 | INFO | "huge" : "", | 20260507 07:11:17.987 | INFO | "huge_shared" : " 27 GB", | 20260507 07:11:17.987 | INFO | "malloc" : " 14 GB", | 20260507 07:11:17.987 | INFO | "pinned" : " 29 GB" | 20260507 07:11:17.987 | INFO | } | 20260507 07:11:17.987 | INFO | Perf report for the last 5.13965 seconds: | 20260507 07:11:17.988 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:11:17.988 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:17.988 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:17.988 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:17.988 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 07:11:17.988 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:17.988 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:17.988 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 07:11:17.988 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:17.988 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:17.988 | INFO | Uploaded 36GiB to GPU in 2.0761s (17.3402GiB/s) | 20260507 07:11:17.988 | INFO | Downloaded 34GiB to GPU in 1.93645s (17.5579GiB/s) | 20260507 07:11:23.127 | INFO | Analyzing chunk frames = [8068096,8068608) | 20260507 07:11:23.127 | INFO | MemInfo: { | 20260507 07:11:23.127 | INFO | "gpu" : " 17 GB", | 20260507 07:11:23.127 | INFO | "huge" : "", | 20260507 07:11:23.127 | INFO | "huge_shared" : " 27 GB", | 20260507 07:11:23.127 | INFO | "malloc" : " 14 GB", | 20260507 07:11:23.127 | INFO | "pinned" : " 29 GB" | 20260507 07:11:23.127 | INFO | } | 20260507 07:11:23.127 | INFO | Perf report for the last 5.13974 seconds: | 20260507 07:11:23.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:11:23.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:23.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:23.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:23.127 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 07:11:23.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:23.127 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:23.127 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:11:23.127 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:23.127 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:23.127 | INFO | Uploaded 36GiB to GPU in 2.00519s (17.9534GiB/s) | 20260507 07:11:23.128 | INFO | Downloaded 34GiB to GPU in 1.99316s (17.0584GiB/s) | 20260507 07:11:28.269 | INFO | Analyzing chunk frames = [8068608,8069120) | 20260507 07:11:28.269 | INFO | MemInfo: { | 20260507 07:11:28.269 | INFO | "gpu" : " 17 GB", | 20260507 07:11:28.269 | INFO | "huge" : "", | 20260507 07:11:28.269 | INFO | "huge_shared" : " 27 GB", | 20260507 07:11:28.269 | INFO | "malloc" : " 14 GB", | 20260507 07:11:28.269 | INFO | "pinned" : " 29 GB" | 20260507 07:11:28.269 | INFO | } | 20260507 07:11:28.269 | INFO | Perf report for the last 5.14167 seconds: | 20260507 07:11:28.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:11:28.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:28.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:28.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:28.269 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:11:28.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:28.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:28.269 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:11:28.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:28.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:28.269 | INFO | Uploaded 36GiB to GPU in 1.96815s (18.2913GiB/s) | 20260507 07:11:28.269 | INFO | Downloaded 34GiB to GPU in 1.96706s (17.2847GiB/s) | 20260507 07:11:28.269 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:11:28.269 | INFO | Total mem usage: 18 GiB | 20260507 07:11:28.269 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:11:28.269 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -27.5%) | 20260507 07:11:28.269 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:11:28.269 | INFO | High Water Breakdown: | 20260507 07:11:28.269 | INFO | 60% : 8.47 GiB ( 74.2% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:11:28.270 | INFO | 39.9% : 5.63 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:11:28.270 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:11:28.270 | INFO | ------------------------------------------ | 20260507 07:11:28.270 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:11:28.270 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:11:28.270 | INFO | Total mem usage: 26.5 GiB | 20260507 07:11:28.270 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:11:28.270 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:11:28.270 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:11:28.270 | INFO | High Water Breakdown: | 20260507 07:11:28.270 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:11:28.270 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:11:28.270 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:11:28.270 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:11:28.270 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:11:28.270 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:11:28.270 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:11:28.270 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:11:28.270 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:11:28.270 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:11:28.270 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:11:28.270 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:11:28.270 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:11:28.270 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:11:28.270 | INFO | ------------------------------------------ | 20260507 07:11:28.270 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:11:28.270 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:11:28.270 | INFO | Total mem usage: 25.6 GiB | 20260507 07:11:28.270 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:11:28.270 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:11:28.270 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:11:28.270 | INFO | High Water Breakdown: | 20260507 07:11:28.270 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:11:28.270 | INFO | ------------------------------------------ | 20260507 07:11:28.270 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:11:28.270 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:11:28.270 | INFO | Total mem usage: 15.7 GiB | 20260507 07:11:28.270 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:11:28.270 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:11:28.270 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:11:28.270 | INFO | High Water Breakdown: | 20260507 07:11:28.270 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:11:28.270 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:11:28.270 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:11:28.270 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:11:28.270 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:11:28.270 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:11:28.270 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:11:28.270 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:11:28.270 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:11:28.270 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:11:28.270 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:11:28.270 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:11:28.270 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:11:28.270 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:11:28.270 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:11:28.270 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:11:28.270 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:11:28.270 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:11:28.270 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:11:28.270 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:11:28.270 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:11:28.270 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:11:28.270 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:11:28.270 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:11:28.270 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:11:28.270 | INFO | ------------------------------------------ | 20260507 07:11:28.270 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:11:28.270 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:11:28.270 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:11:28.270 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:11:28.270 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.34e+01 / 1.64e+00 | 20260507 07:11:28.270 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.03e-01, 9.51e-03 / 6.68e-03 | 20260507 07:11:28.270 | INFO | PerfReport: 11.68%, Baseline -- 3.58e+00 / 4.37e+00, 5.03e-01 / 2.96e-01 | 20260507 07:11:28.270 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.45e+00, 1.67e-01 / 1.09e-01 | 20260507 07:11:28.270 | INFO | PerfReport: 16.27%, DME -- 3.99e+01 / 4.46e+01, 3.93e+00 / 2.01e+00 | 20260507 07:11:28.270 | INFO | PerfReport: 57.73%, FrameLabeling -- 1.77e+01 / 1.99e+01, 1.31e+00 / 1.19e+00 | 20260507 07:11:28.270 | INFO | PerfReport: 6.88%, PulseAccumulating -- 2.11e+00 / 2.62e+00, 3.81e-01 / 2.24e-01 | 20260507 07:11:28.270 | INFO | PerfReport: 2.93%, Metrics -- 9.00e-01 / 1.19e+00, 2.39e-01 / 1.11e-01 | 20260507 07:11:28.270 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.03e-01 / 3.56e-01, 5.04e-02 / 2.03e-02 | 20260507 07:11:28.270 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:11:28.270 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:11:28.270 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:11:28.271 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 07:11:28.271 | INFO | PerfReport: 21.95%, Upload -- 2.11e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260507 07:11:28.271 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 07:11:28.271 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 07:11:28.271 | INFO | PerfReport: 1.53%, LockUpload -- 1.47e+00 / 0.00e+00, 4.86e+00 / 0.00e+00 | 20260507 07:11:28.271 | INFO | PerfReport: 1.27%, LockDownload -- 1.22e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260507 07:11:28.271 | INFO | PerfReport: 21.90%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 07:11:28.271 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.80e-02 / 0.00e+00 | 20260507 07:11:28.366 | INFO | Starting a round of Baz IO | 20260507 07:11:31.457 | INFO | Populating RTMetrics report | 20260507 07:11:31.457 | INFO | RTMetrics report populated | 20260507 07:11:31.457 | INFO | Opening tmp RTMetrics json file | 20260507 07:11:31.473 | INFO | Serializing RTMetrics to json | 20260507 07:11:31.474 | INFO | Serialization done, swapping tmp file now... | 20260507 07:11:31.475 | INFO | RTMetrics done | 20260507 07:11:31.540 | INFO | Finished a round of Baz IO: Wrote 1.648 GiB in 3.174 seconds | 20260507 07:11:31.540 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.58% / 0.00% / 67.42% | 20260507 07:11:31.540 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.108647% / 0% / 5.96046e-05% | 20260507 07:11:31.540 | INFO | Baz IO overall: | 20260507 07:11:31.540 | INFO | BytesWritten : 4051017948354 | 20260507 07:11:31.540 | INFO | NumEvents : 1939993700182 | 20260507 07:11:33.406 | INFO | Analyzing chunk frames = [8069120,8069632) | 20260507 07:11:33.406 | INFO | MemInfo: { | 20260507 07:11:33.406 | INFO | "gpu" : " 17 GB", | 20260507 07:11:33.406 | INFO | "huge" : "", | 20260507 07:11:33.406 | INFO | "huge_shared" : " 27 GB", | 20260507 07:11:33.406 | INFO | "malloc" : " 14 GB", | 20260507 07:11:33.406 | INFO | "pinned" : " 29 GB" | 20260507 07:11:33.406 | INFO | } | 20260507 07:11:33.406 | INFO | Perf report for the last 5.13698 seconds: | 20260507 07:11:33.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:11:33.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:33.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:33.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:33.406 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 07:11:33.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:33.406 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 07:11:33.406 | INFO | PRE_HQ : 58.9%, 1.8, 54.6ms, 0 | 20260507 07:11:33.406 | INFO | BAZWRITER : 61.4%, 3.1, 96.9ms, 0 | 20260507 07:11:33.406 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:11:33.406 | INFO | Uploaded 36GiB to GPU in 2.04386s (17.6137GiB/s) | 20260507 07:11:33.406 | INFO | Downloaded 38GiB to GPU in 2.17798s (17.4473GiB/s) | 20260507 07:11:38.544 | INFO | Analyzing chunk frames = [8069632,8070144) | 20260507 07:11:38.544 | INFO | MemInfo: { | 20260507 07:11:38.544 | INFO | "gpu" : " 17 GB", | 20260507 07:11:38.544 | INFO | "huge" : "", | 20260507 07:11:38.544 | INFO | "huge_shared" : " 27 GB", | 20260507 07:11:38.544 | INFO | "malloc" : " 14 GB", | 20260507 07:11:38.544 | INFO | "pinned" : " 29 GB" | 20260507 07:11:38.544 | INFO | } | 20260507 07:11:38.544 | INFO | Perf report for the last 5.13863 seconds: | 20260507 07:11:38.545 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:11:38.545 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:38.545 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:38.545 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:38.545 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 07:11:38.545 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:38.545 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:38.545 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 07:11:38.545 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:38.545 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:38.545 | INFO | Uploaded 36GiB to GPU in 1.98232s (18.1605GiB/s) | 20260507 07:11:38.545 | INFO | Downloaded 34GiB to GPU in 1.936s (17.562GiB/s) | 20260507 07:11:43.677 | INFO | Analyzing chunk frames = [8070144,8070656) | 20260507 07:11:43.677 | INFO | MemInfo: { | 20260507 07:11:43.677 | INFO | "gpu" : " 17 GB", | 20260507 07:11:43.677 | INFO | "huge" : "", | 20260507 07:11:43.677 | INFO | "huge_shared" : " 27 GB", | 20260507 07:11:43.677 | INFO | "malloc" : " 14 GB", | 20260507 07:11:43.677 | INFO | "pinned" : " 29 GB" | 20260507 07:11:43.677 | INFO | } | 20260507 07:11:43.677 | INFO | Perf report for the last 5.13239 seconds: | 20260507 07:11:43.677 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:11:43.677 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:43.677 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:43.677 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:43.677 | INFO | ANALYSIS : 60.5%, 3.0, 93.5ms, 0 | 20260507 07:11:43.677 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:43.677 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:43.677 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:11:43.677 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:43.677 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:43.677 | INFO | Uploaded 36GiB to GPU in 2.04259s (17.6247GiB/s) | 20260507 07:11:43.677 | INFO | Downloaded 34GiB to GPU in 1.95616s (17.381GiB/s) | 20260507 07:11:48.819 | INFO | Analyzing chunk frames = [8070656,8071168) | 20260507 07:11:48.819 | INFO | MemInfo: { | 20260507 07:11:48.819 | INFO | "gpu" : " 17 GB", | 20260507 07:11:48.819 | INFO | "huge" : "", | 20260507 07:11:48.819 | INFO | "huge_shared" : " 27 GB", | 20260507 07:11:48.819 | INFO | "malloc" : " 14 GB", | 20260507 07:11:48.819 | INFO | "pinned" : " 29 GB" | 20260507 07:11:48.819 | INFO | } | 20260507 07:11:48.819 | INFO | Perf report for the last 5.14214 seconds: | 20260507 07:11:48.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:11:48.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:48.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:48.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:48.819 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 07:11:48.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:48.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:48.819 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:11:48.820 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:48.820 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:48.820 | INFO | Uploaded 36GiB to GPU in 1.97215s (18.2542GiB/s) | 20260507 07:11:48.820 | INFO | Downloaded 34GiB to GPU in 1.92747s (17.6397GiB/s) | 20260507 07:11:53.964 | INFO | Analyzing chunk frames = [8071168,8071680) | 20260507 07:11:53.964 | INFO | MemInfo: { | 20260507 07:11:53.964 | INFO | "gpu" : " 17 GB", | 20260507 07:11:53.964 | INFO | "huge" : "", | 20260507 07:11:53.964 | INFO | "huge_shared" : " 27 GB", | 20260507 07:11:53.964 | INFO | "malloc" : " 14 GB", | 20260507 07:11:53.964 | INFO | "pinned" : " 29 GB" | 20260507 07:11:53.964 | INFO | } | 20260507 07:11:53.964 | INFO | Perf report for the last 5.14497 seconds: | 20260507 07:11:53.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:11:53.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:53.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:53.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:53.964 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 07:11:53.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:53.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:53.964 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:11:53.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:53.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:53.964 | INFO | Uploaded 36GiB to GPU in 2.04699s (17.5868GiB/s) | 20260507 07:11:53.964 | INFO | Downloaded 34GiB to GPU in 1.94085s (17.5181GiB/s) | 20260507 07:11:59.110 | INFO | Analyzing chunk frames = [8071680,8072192) | 20260507 07:11:59.110 | INFO | MemInfo: { | 20260507 07:11:59.110 | INFO | "gpu" : " 17 GB", | 20260507 07:11:59.110 | INFO | "huge" : "", | 20260507 07:11:59.110 | INFO | "huge_shared" : " 27 GB", | 20260507 07:11:59.110 | INFO | "malloc" : " 14 GB", | 20260507 07:11:59.110 | INFO | "pinned" : " 29 GB" | 20260507 07:11:59.110 | INFO | } | 20260507 07:11:59.110 | INFO | Perf report for the last 5.14624 seconds: | 20260507 07:11:59.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:11:59.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:59.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:59.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:59.110 | INFO | ANALYSIS : 60.4%, 3.0, 94.4ms, 0 | 20260507 07:11:59.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:59.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:59.110 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 07:11:59.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:59.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:11:59.111 | INFO | Uploaded 36GiB to GPU in 2.06728s (17.4142GiB/s) | 20260507 07:11:59.111 | INFO | Downloaded 34GiB to GPU in 1.93879s (17.5367GiB/s) | 20260507 07:12:04.244 | INFO | Analyzing chunk frames = [8072192,8072704) | 20260507 07:12:04.244 | INFO | MemInfo: { | 20260507 07:12:04.244 | INFO | "gpu" : " 17 GB", | 20260507 07:12:04.244 | INFO | "huge" : "", | 20260507 07:12:04.244 | INFO | "huge_shared" : " 27 GB", | 20260507 07:12:04.244 | INFO | "malloc" : " 14 GB", | 20260507 07:12:04.244 | INFO | "pinned" : " 29 GB" | 20260507 07:12:04.244 | INFO | } | 20260507 07:12:04.244 | INFO | Perf report for the last 5.13397 seconds: | 20260507 07:12:04.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:12:04.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:04.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:04.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:04.244 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 07:12:04.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:04.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:04.244 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:12:04.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:04.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:04.244 | INFO | Uploaded 36GiB to GPU in 1.93742s (18.5815GiB/s) | 20260507 07:12:04.244 | INFO | Downloaded 34GiB to GPU in 1.96162s (17.3326GiB/s) | 20260507 07:12:09.378 | INFO | Analyzing chunk frames = [8072704,8073216) | 20260507 07:12:09.378 | INFO | MemInfo: { | 20260507 07:12:09.378 | INFO | "gpu" : " 17 GB", | 20260507 07:12:09.379 | INFO | "huge" : "", | 20260507 07:12:09.379 | INFO | "huge_shared" : " 27 GB", | 20260507 07:12:09.379 | INFO | "malloc" : " 14 GB", | 20260507 07:12:09.379 | INFO | "pinned" : " 29 GB" | 20260507 07:12:09.379 | INFO | } | 20260507 07:12:09.379 | INFO | Perf report for the last 5.13434 seconds: | 20260507 07:12:09.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:12:09.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:09.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:09.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:09.379 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:12:09.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:09.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:09.379 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 07:12:09.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:09.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:09.379 | INFO | Uploaded 36GiB to GPU in 1.99257s (18.0671GiB/s) | 20260507 07:12:09.379 | INFO | Downloaded 34GiB to GPU in 1.95679s (17.3754GiB/s) | 20260507 07:12:09.476 | INFO | Starting a round of Baz IO | 20260507 07:12:12.575 | INFO | Populating RTMetrics report | 20260507 07:12:12.575 | INFO | RTMetrics report populated | 20260507 07:12:12.575 | INFO | Opening tmp RTMetrics json file | 20260507 07:12:12.583 | INFO | Serializing RTMetrics to json | 20260507 07:12:12.591 | INFO | Serialization done, swapping tmp file now... | 20260507 07:12:12.592 | INFO | RTMetrics done | 20260507 07:12:12.661 | INFO | Finished a round of Baz IO: Wrote 1.648 GiB in 3.184 seconds | 20260507 07:12:12.661 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.57% / 0.00% / 67.42% | 20260507 07:12:12.661 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.109359% / 0% / 5.96046e-05% | 20260507 07:12:12.661 | INFO | Baz IO overall: | 20260507 07:12:12.661 | INFO | BytesWritten : 4052787334691 | 20260507 07:12:12.661 | INFO | NumEvents : 1940820724345 | 20260507 07:12:14.520 | INFO | Analyzing chunk frames = [8073216,8073728) | 20260507 07:12:14.520 | INFO | MemInfo: { | 20260507 07:12:14.520 | INFO | "gpu" : " 17 GB", | 20260507 07:12:14.520 | INFO | "huge" : "", | 20260507 07:12:14.520 | INFO | "huge_shared" : " 27 GB", | 20260507 07:12:14.520 | INFO | "malloc" : " 14 GB", | 20260507 07:12:14.520 | INFO | "pinned" : " 29 GB" | 20260507 07:12:14.520 | INFO | } | 20260507 07:12:14.520 | INFO | Perf report for the last 5.14173 seconds: | 20260507 07:12:14.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:12:14.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:14.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:14.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:14.520 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 07:12:14.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:14.520 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 07:12:14.520 | INFO | PRE_HQ : 58.8%, 1.8, 54.6ms, 0 | 20260507 07:12:14.521 | INFO | BAZWRITER : 61.5%, 3.1, 98.0ms, 0 | 20260507 07:12:14.521 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:12:14.521 | INFO | Uploaded 36GiB to GPU in 2.04018s (17.6455GiB/s) | 20260507 07:12:14.521 | INFO | Downloaded 38GiB to GPU in 2.18182s (17.4166GiB/s) | 20260507 07:12:19.666 | INFO | Analyzing chunk frames = [8073728,8074240) | 20260507 07:12:19.666 | INFO | MemInfo: { | 20260507 07:12:19.666 | INFO | "gpu" : " 17 GB", | 20260507 07:12:19.666 | INFO | "huge" : "", | 20260507 07:12:19.666 | INFO | "huge_shared" : " 27 GB", | 20260507 07:12:19.666 | INFO | "malloc" : " 14 GB", | 20260507 07:12:19.666 | INFO | "pinned" : " 29 GB" | 20260507 07:12:19.666 | INFO | } | 20260507 07:12:19.666 | INFO | Perf report for the last 5.14599 seconds: | 20260507 07:12:19.666 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:12:19.666 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:19.666 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:19.666 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:19.666 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 07:12:19.666 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:19.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:19.666 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:12:19.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:19.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:19.667 | INFO | Uploaded 36GiB to GPU in 2.00451s (17.9595GiB/s) | 20260507 07:12:19.667 | INFO | Downloaded 34GiB to GPU in 1.92794s (17.6354GiB/s) | 20260507 07:12:24.799 | INFO | Analyzing chunk frames = [8074240,8074752) | 20260507 07:12:24.799 | INFO | MemInfo: { | 20260507 07:12:24.799 | INFO | "gpu" : " 17 GB", | 20260507 07:12:24.799 | INFO | "huge" : "", | 20260507 07:12:24.799 | INFO | "huge_shared" : " 27 GB", | 20260507 07:12:24.799 | INFO | "malloc" : " 14 GB", | 20260507 07:12:24.799 | INFO | "pinned" : " 29 GB" | 20260507 07:12:24.799 | INFO | } | 20260507 07:12:24.799 | INFO | Perf report for the last 5.13298 seconds: | 20260507 07:12:24.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:12:24.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:24.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:24.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:24.799 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:12:24.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:24.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:24.799 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:12:24.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:24.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:24.800 | INFO | Uploaded 36GiB to GPU in 2.02477s (17.7798GiB/s) | 20260507 07:12:24.800 | INFO | Downloaded 34GiB to GPU in 1.91577s (17.7475GiB/s) | 20260507 07:12:29.935 | INFO | Analyzing chunk frames = [8074752,8075264) | 20260507 07:12:29.935 | INFO | MemInfo: { | 20260507 07:12:29.935 | INFO | "gpu" : " 17 GB", | 20260507 07:12:29.935 | INFO | "huge" : "", | 20260507 07:12:29.935 | INFO | "huge_shared" : " 27 GB", | 20260507 07:12:29.935 | INFO | "malloc" : " 14 GB", | 20260507 07:12:29.935 | INFO | "pinned" : " 29 GB" | 20260507 07:12:29.935 | INFO | } | 20260507 07:12:29.935 | INFO | Perf report for the last 5.13569 seconds: | 20260507 07:12:29.935 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:12:29.935 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:29.935 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:29.935 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:29.935 | INFO | ANALYSIS : 60.6%, 2.9, 93.5ms, 0 | 20260507 07:12:29.935 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:29.935 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:29.935 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:12:29.935 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:29.935 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:29.935 | INFO | Uploaded 36GiB to GPU in 2.06037s (17.4726GiB/s) | 20260507 07:12:29.935 | INFO | Downloaded 34GiB to GPU in 1.94781s (17.4555GiB/s) | 20260507 07:12:35.083 | INFO | Analyzing chunk frames = [8075264,8075776) | 20260507 07:12:35.083 | INFO | MemInfo: { | 20260507 07:12:35.083 | INFO | "gpu" : " 17 GB", | 20260507 07:12:35.083 | INFO | "huge" : "", | 20260507 07:12:35.083 | INFO | "huge_shared" : " 27 GB", | 20260507 07:12:35.083 | INFO | "malloc" : " 14 GB", | 20260507 07:12:35.083 | INFO | "pinned" : " 29 GB" | 20260507 07:12:35.083 | INFO | } | 20260507 07:12:35.083 | INFO | Perf report for the last 5.14811 seconds: | 20260507 07:12:35.083 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:12:35.083 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:35.083 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:35.083 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:35.083 | INFO | ANALYSIS : 60.1%, 3.0, 94.0ms, 0 | 20260507 07:12:35.083 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:35.083 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:35.083 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 07:12:35.083 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:35.083 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:35.084 | INFO | Uploaded 36GiB to GPU in 2.05795s (17.4931GiB/s) | 20260507 07:12:35.084 | INFO | Downloaded 34GiB to GPU in 1.94957s (17.4397GiB/s) | 20260507 07:12:40.222 | INFO | Analyzing chunk frames = [8075776,8076288) | 20260507 07:12:40.222 | INFO | MemInfo: { | 20260507 07:12:40.222 | INFO | "gpu" : " 17 GB", | 20260507 07:12:40.222 | INFO | "huge" : "", | 20260507 07:12:40.222 | INFO | "huge_shared" : " 27 GB", | 20260507 07:12:40.222 | INFO | "malloc" : " 14 GB", | 20260507 07:12:40.222 | INFO | "pinned" : " 29 GB" | 20260507 07:12:40.222 | INFO | } | 20260507 07:12:40.222 | INFO | Perf report for the last 5.13905 seconds: | 20260507 07:12:40.222 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:12:40.222 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:40.222 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:40.222 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:40.222 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 07:12:40.222 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:40.222 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:40.222 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:12:40.222 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:40.222 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:40.222 | INFO | Uploaded 36GiB to GPU in 2.0691s (17.3989GiB/s) | 20260507 07:12:40.223 | INFO | Downloaded 34GiB to GPU in 1.9213s (17.6963GiB/s) | 20260507 07:12:45.352 | INFO | Analyzing chunk frames = [8076288,8076800) | 20260507 07:12:45.352 | INFO | MemInfo: { | 20260507 07:12:45.352 | INFO | "gpu" : " 17 GB", | 20260507 07:12:45.352 | INFO | "huge" : "", | 20260507 07:12:45.352 | INFO | "huge_shared" : " 27 GB", | 20260507 07:12:45.352 | INFO | "malloc" : " 14 GB", | 20260507 07:12:45.352 | INFO | "pinned" : " 29 GB" | 20260507 07:12:45.352 | INFO | } | 20260507 07:12:45.352 | INFO | Perf report for the last 5.13002 seconds: | 20260507 07:12:45.352 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:12:45.352 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:45.352 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:45.352 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:45.352 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:12:45.352 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:45.352 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:45.352 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 07:12:45.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:45.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:45.353 | INFO | Uploaded 36GiB to GPU in 1.90249s (18.9226GiB/s) | 20260507 07:12:45.353 | INFO | Downloaded 34GiB to GPU in 1.97353s (17.228GiB/s) | 20260507 07:12:50.490 | INFO | Analyzing chunk frames = [8076800,8077312) | 20260507 07:12:50.490 | INFO | MemInfo: { | 20260507 07:12:50.490 | INFO | "gpu" : " 17 GB", | 20260507 07:12:50.490 | INFO | "huge" : "", | 20260507 07:12:50.490 | INFO | "huge_shared" : " 27 GB", | 20260507 07:12:50.490 | INFO | "malloc" : " 14 GB", | 20260507 07:12:50.490 | INFO | "pinned" : " 29 GB" | 20260507 07:12:50.490 | INFO | } | 20260507 07:12:50.490 | INFO | Perf report for the last 5.13821 seconds: | 20260507 07:12:50.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:12:50.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:50.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:50.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:50.490 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:12:50.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:50.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:50.491 | INFO | PRE_HQ : 29.3%, 1.0, 15.2ms, 0 | 20260507 07:12:50.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:50.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:50.491 | INFO | Uploaded 36GiB to GPU in 1.98382s (18.1468GiB/s) | 20260507 07:12:50.491 | INFO | Downloaded 34GiB to GPU in 1.95568s (17.3852GiB/s) | 20260507 07:12:50.491 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:12:50.491 | INFO | Total mem usage: 18 GiB | 20260507 07:12:50.491 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:12:50.491 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 07:12:50.491 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:12:50.491 | INFO | High Water Breakdown: | 20260507 07:12:50.491 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:12:50.491 | INFO | 39.7% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:12:50.491 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:12:50.491 | INFO | ------------------------------------------ | 20260507 07:12:50.491 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:12:50.491 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:12:50.491 | INFO | Total mem usage: 26.5 GiB | 20260507 07:12:50.491 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:12:50.491 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:12:50.491 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:12:50.491 | INFO | High Water Breakdown: | 20260507 07:12:50.491 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:12:50.491 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:12:50.491 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:12:50.491 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:12:50.491 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:12:50.491 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:12:50.491 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:12:50.491 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:12:50.491 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:12:50.491 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:12:50.491 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:12:50.491 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:12:50.491 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:12:50.491 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:12:50.491 | INFO | ------------------------------------------ | 20260507 07:12:50.491 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:12:50.491 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:12:50.491 | INFO | Total mem usage: 25.6 GiB | 20260507 07:12:50.491 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:12:50.491 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:12:50.491 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:12:50.491 | INFO | High Water Breakdown: | 20260507 07:12:50.491 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:12:50.491 | INFO | ------------------------------------------ | 20260507 07:12:50.491 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:12:50.491 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:12:50.491 | INFO | Total mem usage: 15.7 GiB | 20260507 07:12:50.491 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:12:50.492 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:12:50.492 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:12:50.492 | INFO | High Water Breakdown: | 20260507 07:12:50.492 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:12:50.492 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:12:50.492 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:12:50.492 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:12:50.492 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:12:50.492 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:12:50.492 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:12:50.492 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:12:50.492 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:12:50.492 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:12:50.492 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:12:50.492 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:12:50.492 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:12:50.492 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:12:50.492 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:12:50.492 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:12:50.492 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:12:50.492 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:12:50.492 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:12:50.492 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:12:50.492 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:12:50.492 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:12:50.492 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:12:50.492 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:12:50.492 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:12:50.492 | INFO | ------------------------------------------ | 20260507 07:12:50.492 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:12:50.492 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:12:50.492 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:12:50.492 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:12:50.492 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.36e+01 / 1.31e+00 | 20260507 07:12:50.492 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.02e-01, 1.01e-02 / 3.75e-03 | 20260507 07:12:50.492 | INFO | PerfReport: 11.68%, Baseline -- 3.58e+00 / 4.47e+00, 5.12e-01 / 2.16e-01 | 20260507 07:12:50.492 | INFO | PerfReport: 3.22%, Histogram -- 9.90e-01 / 1.52e+00, 1.70e-01 / 5.97e-02 | 20260507 07:12:50.492 | INFO | PerfReport: 16.29%, DME -- 4.00e+01 / 4.48e+01, 3.83e+00 / 1.50e+00 | 20260507 07:12:50.492 | INFO | PerfReport: 57.64%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.25e+00 / 1.29e+00 | 20260507 07:12:50.492 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.13e+00 / 2.72e+00, 3.90e-01 / 1.83e-01 | 20260507 07:12:50.492 | INFO | PerfReport: 2.95%, Metrics -- 9.05e-01 / 1.23e+00, 2.41e-01 / 1.35e-01 | 20260507 07:12:50.492 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.58e-01, 5.01e-02 / 2.03e-02 | 20260507 07:12:50.492 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:12:50.492 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:12:50.492 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:12:50.492 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 07:12:50.492 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.12e+00 / 0.00e+00 | 20260507 07:12:50.492 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 07:12:50.492 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260507 07:12:50.492 | INFO | PerfReport: 1.55%, LockUpload -- 1.49e+00 / 0.00e+00, 4.96e+00 / 0.00e+00 | 20260507 07:12:50.492 | INFO | PerfReport: 1.25%, LockDownload -- 1.20e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260507 07:12:50.492 | INFO | PerfReport: 22.06%, LockCompute -- 2.12e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 07:12:50.492 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 07:12:50.587 | INFO | Starting a round of Baz IO | 20260507 07:12:53.690 | INFO | Populating RTMetrics report | 20260507 07:12:53.691 | INFO | RTMetrics report populated | 20260507 07:12:53.691 | INFO | Opening tmp RTMetrics json file | 20260507 07:12:53.692 | INFO | Serializing RTMetrics to json | 20260507 07:12:53.693 | INFO | Serialization done, swapping tmp file now... | 20260507 07:12:53.693 | INFO | RTMetrics done | 20260507 07:12:53.788 | INFO | Finished a round of Baz IO: Wrote 1.664 GiB in 3.2 seconds | 20260507 07:12:53.788 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.57% / 0.00% / 67.43% | 20260507 07:12:53.788 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.111369% / 3.97364e-06% / 7.54992e-05% | 20260507 07:12:53.788 | INFO | Baz IO overall: | 20260507 07:12:53.788 | INFO | BytesWritten : 4054573824743 | 20260507 07:12:53.788 | INFO | NumEvents : 1941654389168 | 20260507 07:12:55.629 | INFO | Analyzing chunk frames = [8077312,8077824) | 20260507 07:12:55.629 | INFO | MemInfo: { | 20260507 07:12:55.629 | INFO | "gpu" : " 17 GB", | 20260507 07:12:55.629 | INFO | "huge" : "", | 20260507 07:12:55.629 | INFO | "huge_shared" : " 27 GB", | 20260507 07:12:55.629 | INFO | "malloc" : " 14 GB", | 20260507 07:12:55.629 | INFO | "pinned" : " 29 GB" | 20260507 07:12:55.629 | INFO | } | 20260507 07:12:55.629 | INFO | Perf report for the last 5.13902 seconds: | 20260507 07:12:55.630 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:12:55.630 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:55.630 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:55.630 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:55.630 | INFO | ANALYSIS : 61.7%, 3.0, 95.7ms, 0 | 20260507 07:12:55.630 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:12:55.630 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 07:12:55.630 | INFO | PRE_HQ : 59.6%, 1.9, 58.9ms, 0 | 20260507 07:12:55.630 | INFO | BAZWRITER : 61.8%, 3.3, 104.0ms, 0 | 20260507 07:12:55.630 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:12:55.630 | INFO | Uploaded 36GiB to GPU in 2.03403s (17.6989GiB/s) | 20260507 07:12:55.630 | INFO | Downloaded 38GiB to GPU in 2.16688s (17.5367GiB/s) | 20260507 07:13:00.775 | INFO | Analyzing chunk frames = [8077824,8078336) | 20260507 07:13:00.775 | INFO | MemInfo: { | 20260507 07:13:00.775 | INFO | "gpu" : " 17 GB", | 20260507 07:13:00.775 | INFO | "huge" : "", | 20260507 07:13:00.775 | INFO | "huge_shared" : " 27 GB", | 20260507 07:13:00.775 | INFO | "malloc" : " 14 GB", | 20260507 07:13:00.775 | INFO | "pinned" : " 29 GB" | 20260507 07:13:00.776 | INFO | } | 20260507 07:13:00.776 | INFO | Perf report for the last 5.14611 seconds: | 20260507 07:13:00.776 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:13:00.776 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:00.776 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:00.776 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:00.776 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 07:13:00.776 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:00.776 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:00.776 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:13:00.776 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:00.776 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:00.776 | INFO | Uploaded 36GiB to GPU in 2.00866s (17.9224GiB/s) | 20260507 07:13:00.776 | INFO | Downloaded 34GiB to GPU in 1.93388s (17.5812GiB/s) | 20260507 07:13:05.913 | INFO | Analyzing chunk frames = [8078336,8078848) | 20260507 07:13:05.913 | INFO | MemInfo: { | 20260507 07:13:05.913 | INFO | "gpu" : " 17 GB", | 20260507 07:13:05.913 | INFO | "huge" : "", | 20260507 07:13:05.914 | INFO | "huge_shared" : " 27 GB", | 20260507 07:13:05.914 | INFO | "malloc" : " 14 GB", | 20260507 07:13:05.914 | INFO | "pinned" : " 29 GB" | 20260507 07:13:05.914 | INFO | } | 20260507 07:13:05.914 | INFO | Perf report for the last 5.13814 seconds: | 20260507 07:13:05.914 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:13:05.914 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:05.914 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:05.914 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:05.914 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 07:13:05.914 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:05.914 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:05.914 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:13:05.914 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:05.914 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:05.914 | INFO | Uploaded 36GiB to GPU in 2.05219s (17.5422GiB/s) | 20260507 07:13:05.914 | INFO | Downloaded 34GiB to GPU in 1.9496s (17.4394GiB/s) | 20260507 07:13:11.054 | INFO | Analyzing chunk frames = [8078848,8079360) | 20260507 07:13:11.054 | INFO | MemInfo: { | 20260507 07:13:11.054 | INFO | "gpu" : " 17 GB", | 20260507 07:13:11.054 | INFO | "huge" : "", | 20260507 07:13:11.054 | INFO | "huge_shared" : " 27 GB", | 20260507 07:13:11.054 | INFO | "malloc" : " 14 GB", | 20260507 07:13:11.054 | INFO | "pinned" : " 29 GB" | 20260507 07:13:11.054 | INFO | } | 20260507 07:13:11.054 | INFO | Perf report for the last 5.14043 seconds: | 20260507 07:13:11.054 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:13:11.054 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:11.054 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:11.054 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:11.054 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 07:13:11.054 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:11.054 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:11.054 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:13:11.054 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:11.054 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:11.055 | INFO | Uploaded 36GiB to GPU in 2.05372s (17.5292GiB/s) | 20260507 07:13:11.055 | INFO | Downloaded 34GiB to GPU in 1.94457s (17.4846GiB/s) | 20260507 07:13:16.187 | INFO | Analyzing chunk frames = [8079360,8079872) | 20260507 07:13:16.187 | INFO | MemInfo: { | 20260507 07:13:16.187 | INFO | "gpu" : " 17 GB", | 20260507 07:13:16.187 | INFO | "huge" : "", | 20260507 07:13:16.187 | INFO | "huge_shared" : " 27 GB", | 20260507 07:13:16.187 | INFO | "malloc" : " 14 GB", | 20260507 07:13:16.187 | INFO | "pinned" : " 29 GB" | 20260507 07:13:16.187 | INFO | } | 20260507 07:13:16.188 | INFO | Perf report for the last 5.13331 seconds: | 20260507 07:13:16.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:13:16.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:16.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:16.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:16.188 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 07:13:16.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:16.188 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:16.188 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:13:16.188 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:16.188 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:16.188 | INFO | Uploaded 36GiB to GPU in 2.05783s (17.4942GiB/s) | 20260507 07:13:16.188 | INFO | Downloaded 34GiB to GPU in 1.90736s (17.8256GiB/s) | 20260507 07:13:21.324 | INFO | Analyzing chunk frames = [8079872,8080384) | 20260507 07:13:21.324 | INFO | MemInfo: { | 20260507 07:13:21.324 | INFO | "gpu" : " 17 GB", | 20260507 07:13:21.324 | INFO | "huge" : "", | 20260507 07:13:21.325 | INFO | "huge_shared" : " 27 GB", | 20260507 07:13:21.325 | INFO | "malloc" : " 14 GB", | 20260507 07:13:21.325 | INFO | "pinned" : " 29 GB" | 20260507 07:13:21.325 | INFO | } | 20260507 07:13:21.325 | INFO | Perf report for the last 5.13702 seconds: | 20260507 07:13:21.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:13:21.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:21.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:21.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:21.325 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 07:13:21.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:21.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:21.325 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:13:21.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:21.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:21.325 | INFO | Uploaded 36GiB to GPU in 2.03578s (17.6836GiB/s) | 20260507 07:13:21.325 | INFO | Downloaded 34GiB to GPU in 1.90973s (17.8036GiB/s) | 20260507 07:13:26.472 | INFO | Analyzing chunk frames = [8080384,8080896) | 20260507 07:13:26.472 | INFO | MemInfo: { | 20260507 07:13:26.472 | INFO | "gpu" : " 17 GB", | 20260507 07:13:26.472 | INFO | "huge" : "", | 20260507 07:13:26.472 | INFO | "huge_shared" : " 27 GB", | 20260507 07:13:26.472 | INFO | "malloc" : " 14 GB", | 20260507 07:13:26.472 | INFO | "pinned" : " 29 GB" | 20260507 07:13:26.472 | INFO | } | 20260507 07:13:26.472 | INFO | Perf report for the last 5.14788 seconds: | 20260507 07:13:26.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:13:26.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:26.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:26.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:26.472 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 07:13:26.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:26.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:26.473 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 07:13:26.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:26.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:26.473 | INFO | Uploaded 36GiB to GPU in 1.97787s (18.2014GiB/s) | 20260507 07:13:26.473 | INFO | Downloaded 34GiB to GPU in 1.96988s (17.2599GiB/s) | 20260507 07:13:31.620 | INFO | Analyzing chunk frames = [8080896,8081408) | 20260507 07:13:31.620 | INFO | MemInfo: { | 20260507 07:13:31.620 | INFO | "gpu" : " 17 GB", | 20260507 07:13:31.620 | INFO | "huge" : "", | 20260507 07:13:31.620 | INFO | "huge_shared" : " 27 GB", | 20260507 07:13:31.620 | INFO | "malloc" : " 14 GB", | 20260507 07:13:31.621 | INFO | "pinned" : " 29 GB" | 20260507 07:13:31.621 | INFO | } | 20260507 07:13:31.621 | INFO | Perf report for the last 5.14817 seconds: | 20260507 07:13:31.621 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:13:31.621 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:31.621 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:31.621 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:31.621 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 07:13:31.621 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:31.621 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:31.621 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 07:13:31.621 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:31.621 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:31.621 | INFO | Uploaded 36GiB to GPU in 1.99253s (18.0675GiB/s) | 20260507 07:13:31.621 | INFO | Downloaded 34GiB to GPU in 1.97072s (17.2526GiB/s) | 20260507 07:13:31.718 | INFO | Starting a round of Baz IO | 20260507 07:13:34.809 | INFO | Populating RTMetrics report | 20260507 07:13:34.809 | INFO | RTMetrics report populated | 20260507 07:13:34.809 | INFO | Opening tmp RTMetrics json file | 20260507 07:13:34.815 | INFO | Serializing RTMetrics to json | 20260507 07:13:34.827 | INFO | Serialization done, swapping tmp file now... | 20260507 07:13:34.828 | INFO | RTMetrics done | 20260507 07:13:34.899 | INFO | Finished a round of Baz IO: Wrote 1.658 GiB in 3.18 seconds | 20260507 07:13:34.899 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.57% / 0.00% / 67.43% | 20260507 07:13:34.899 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.110257% / 3.97364e-06% / 5.16574e-05% | 20260507 07:13:34.899 | INFO | Baz IO overall: | 20260507 07:13:34.899 | INFO | BytesWritten : 4056354535416 | 20260507 07:13:34.899 | INFO | NumEvents : 1942484884820 | 20260507 07:13:36.751 | INFO | Analyzing chunk frames = [8081408,8081920) | 20260507 07:13:36.751 | INFO | MemInfo: { | 20260507 07:13:36.751 | INFO | "gpu" : " 17 GB", | 20260507 07:13:36.751 | INFO | "huge" : "", | 20260507 07:13:36.751 | INFO | "huge_shared" : " 27 GB", | 20260507 07:13:36.751 | INFO | "malloc" : " 14 GB", | 20260507 07:13:36.751 | INFO | "pinned" : " 29 GB" | 20260507 07:13:36.751 | INFO | } | 20260507 07:13:36.751 | INFO | Perf report for the last 5.13086 seconds: | 20260507 07:13:36.751 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:13:36.752 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:36.752 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:36.752 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:36.752 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 07:13:36.752 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:36.752 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 07:13:36.752 | INFO | PRE_HQ : 58.7%, 1.8, 54.4ms, 0 | 20260507 07:13:36.752 | INFO | BAZWRITER : 61.6%, 3.1, 98.8ms, 0 | 20260507 07:13:36.752 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:13:36.752 | INFO | Uploaded 36GiB to GPU in 2.06442s (17.4383GiB/s) | 20260507 07:13:36.752 | INFO | Downloaded 38GiB to GPU in 2.16715s (17.5345GiB/s) | 20260507 07:13:41.884 | INFO | Analyzing chunk frames = [8081920,8082432) | 20260507 07:13:41.884 | INFO | MemInfo: { | 20260507 07:13:41.884 | INFO | "gpu" : " 17 GB", | 20260507 07:13:41.884 | INFO | "huge" : "", | 20260507 07:13:41.884 | INFO | "huge_shared" : " 27 GB", | 20260507 07:13:41.884 | INFO | "malloc" : " 14 GB", | 20260507 07:13:41.884 | INFO | "pinned" : " 29 GB" | 20260507 07:13:41.884 | INFO | } | 20260507 07:13:41.884 | INFO | Perf report for the last 5.13264 seconds: | 20260507 07:13:41.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:13:41.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:41.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:41.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:41.884 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:13:41.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:41.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:41.884 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:13:41.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:41.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:41.884 | INFO | Uploaded 36GiB to GPU in 1.97788s (18.2013GiB/s) | 20260507 07:13:41.884 | INFO | Downloaded 34GiB to GPU in 1.92749s (17.6395GiB/s) | 20260507 07:13:47.030 | INFO | Analyzing chunk frames = [8082432,8082944) | 20260507 07:13:47.030 | INFO | MemInfo: { | 20260507 07:13:47.030 | INFO | "gpu" : " 17 GB", | 20260507 07:13:47.030 | INFO | "huge" : "", | 20260507 07:13:47.030 | INFO | "huge_shared" : " 27 GB", | 20260507 07:13:47.030 | INFO | "malloc" : " 14 GB", | 20260507 07:13:47.030 | INFO | "pinned" : " 29 GB" | 20260507 07:13:47.030 | INFO | } | 20260507 07:13:47.030 | INFO | Perf report for the last 5.14613 seconds: | 20260507 07:13:47.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:13:47.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:47.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:47.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:47.030 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 07:13:47.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:47.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:47.030 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:13:47.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:47.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:47.031 | INFO | Uploaded 36GiB to GPU in 2.01488s (17.8671GiB/s) | 20260507 07:13:47.031 | INFO | Downloaded 34GiB to GPU in 1.95222s (17.4161GiB/s) | 20260507 07:13:52.167 | INFO | Analyzing chunk frames = [8082944,8083456) | 20260507 07:13:52.167 | INFO | MemInfo: { | 20260507 07:13:52.167 | INFO | "gpu" : " 17 GB", | 20260507 07:13:52.167 | INFO | "huge" : "", | 20260507 07:13:52.167 | INFO | "huge_shared" : " 27 GB", | 20260507 07:13:52.167 | INFO | "malloc" : " 14 GB", | 20260507 07:13:52.167 | INFO | "pinned" : " 29 GB" | 20260507 07:13:52.167 | INFO | } | 20260507 07:13:52.167 | INFO | Perf report for the last 5.13647 seconds: | 20260507 07:13:52.167 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:13:52.167 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:52.167 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:52.167 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:52.167 | INFO | ANALYSIS : 60.2%, 2.9, 93.0ms, 0 | 20260507 07:13:52.167 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:52.167 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:52.167 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:13:52.167 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:52.167 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:52.167 | INFO | Uploaded 36GiB to GPU in 1.97871s (18.1937GiB/s) | 20260507 07:13:52.167 | INFO | Downloaded 34GiB to GPU in 1.94147s (17.5125GiB/s) | 20260507 07:13:57.304 | INFO | Analyzing chunk frames = [8083456,8083968) | 20260507 07:13:57.304 | INFO | MemInfo: { | 20260507 07:13:57.304 | INFO | "gpu" : " 17 GB", | 20260507 07:13:57.304 | INFO | "huge" : "", | 20260507 07:13:57.304 | INFO | "huge_shared" : " 27 GB", | 20260507 07:13:57.304 | INFO | "malloc" : " 14 GB", | 20260507 07:13:57.304 | INFO | "pinned" : " 29 GB" | 20260507 07:13:57.304 | INFO | } | 20260507 07:13:57.304 | INFO | Perf report for the last 5.13697 seconds: | 20260507 07:13:57.304 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:13:57.304 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:57.304 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:57.304 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:57.304 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 07:13:57.304 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:57.304 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:57.304 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:13:57.304 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:57.304 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:13:57.304 | INFO | Uploaded 36GiB to GPU in 2.03402s (17.6989GiB/s) | 20260507 07:13:57.304 | INFO | Downloaded 34GiB to GPU in 1.92646s (17.6489GiB/s) | 20260507 07:14:02.439 | INFO | Analyzing chunk frames = [8083968,8084480) | 20260507 07:14:02.439 | INFO | MemInfo: { | 20260507 07:14:02.439 | INFO | "gpu" : " 17 GB", | 20260507 07:14:02.439 | INFO | "huge" : "", | 20260507 07:14:02.439 | INFO | "huge_shared" : " 27 GB", | 20260507 07:14:02.439 | INFO | "malloc" : " 14 GB", | 20260507 07:14:02.439 | INFO | "pinned" : " 29 GB" | 20260507 07:14:02.439 | INFO | } | 20260507 07:14:02.439 | INFO | Perf report for the last 5.13518 seconds: | 20260507 07:14:02.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:14:02.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:02.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:02.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:02.439 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 07:14:02.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:02.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:02.439 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 07:14:02.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:02.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:02.439 | INFO | Uploaded 36GiB to GPU in 2.03113s (17.7241GiB/s) | 20260507 07:14:02.439 | INFO | Downloaded 34GiB to GPU in 1.92301s (17.6806GiB/s) | 20260507 07:14:07.583 | INFO | Analyzing chunk frames = [8084480,8084992) | 20260507 07:14:07.583 | INFO | MemInfo: { | 20260507 07:14:07.583 | INFO | "gpu" : " 17 GB", | 20260507 07:14:07.583 | INFO | "huge" : "", | 20260507 07:14:07.583 | INFO | "huge_shared" : " 27 GB", | 20260507 07:14:07.583 | INFO | "malloc" : " 14 GB", | 20260507 07:14:07.583 | INFO | "pinned" : " 29 GB" | 20260507 07:14:07.584 | INFO | } | 20260507 07:14:07.584 | INFO | Perf report for the last 5.14468 seconds: | 20260507 07:14:07.584 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:14:07.584 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:07.584 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:07.584 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:07.584 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:14:07.584 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:07.584 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:07.584 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 07:14:07.584 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:07.584 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:07.584 | INFO | Uploaded 36GiB to GPU in 1.97656s (18.2135GiB/s) | 20260507 07:14:07.584 | INFO | Downloaded 34GiB to GPU in 1.98502s (17.1283GiB/s) | 20260507 07:14:12.726 | INFO | Analyzing chunk frames = [8084992,8085504) | 20260507 07:14:12.726 | INFO | MemInfo: { | 20260507 07:14:12.726 | INFO | "gpu" : " 17 GB", | 20260507 07:14:12.726 | INFO | "huge" : "", | 20260507 07:14:12.727 | INFO | "huge_shared" : " 27 GB", | 20260507 07:14:12.727 | INFO | "malloc" : " 14 GB", | 20260507 07:14:12.727 | INFO | "pinned" : " 29 GB" | 20260507 07:14:12.727 | INFO | } | 20260507 07:14:12.727 | INFO | Perf report for the last 5.143 seconds: | 20260507 07:14:12.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:14:12.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:12.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:12.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:12.727 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:14:12.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:12.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:12.727 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 07:14:12.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:12.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:12.727 | INFO | Uploaded 36GiB to GPU in 1.96904s (18.283GiB/s) | 20260507 07:14:12.727 | INFO | Downloaded 34GiB to GPU in 1.9657s (17.2966GiB/s) | 20260507 07:14:12.727 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:14:12.727 | INFO | Total mem usage: 18 GiB | 20260507 07:14:12.727 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:14:12.727 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.8%) | 20260507 07:14:12.727 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:14:12.727 | INFO | High Water Breakdown: | 20260507 07:14:12.727 | INFO | 60.2% : 8.62 GiB ( 75.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:14:12.727 | INFO | 39.7% : 5.68 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:14:12.727 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:14:12.727 | INFO | ------------------------------------------ | 20260507 07:14:12.727 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:14:12.727 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:14:12.727 | INFO | Total mem usage: 26.5 GiB | 20260507 07:14:12.727 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:14:12.727 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:14:12.727 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:14:12.727 | INFO | High Water Breakdown: | 20260507 07:14:12.727 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:14:12.727 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:14:12.727 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:14:12.727 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:14:12.727 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:14:12.727 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:14:12.727 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:14:12.727 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:14:12.727 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:14:12.727 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:14:12.727 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:14:12.727 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:14:12.727 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:14:12.727 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:14:12.727 | INFO | ------------------------------------------ | 20260507 07:14:12.727 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:14:12.727 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:14:12.727 | INFO | Total mem usage: 25.6 GiB | 20260507 07:14:12.727 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:14:12.727 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:14:12.727 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:14:12.727 | INFO | High Water Breakdown: | 20260507 07:14:12.727 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:14:12.727 | INFO | ------------------------------------------ | 20260507 07:14:12.727 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:14:12.728 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:14:12.728 | INFO | Total mem usage: 15.7 GiB | 20260507 07:14:12.728 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:14:12.728 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:14:12.728 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:14:12.728 | INFO | High Water Breakdown: | 20260507 07:14:12.728 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:14:12.728 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:14:12.728 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:14:12.728 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:14:12.728 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:14:12.728 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:14:12.728 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:14:12.728 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:14:12.728 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:14:12.728 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:14:12.728 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:14:12.728 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:14:12.728 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:14:12.728 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:14:12.728 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:14:12.728 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:14:12.728 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:14:12.728 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:14:12.728 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:14:12.728 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:14:12.728 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:14:12.728 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:14:12.728 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:14:12.728 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:14:12.728 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:14:12.728 | INFO | ------------------------------------------ | 20260507 07:14:12.728 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:14:12.728 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:14:12.728 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:14:12.728 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:14:12.728 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.35e+01 / 1.10e+00 | 20260507 07:14:12.728 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.03e-01, 9.84e-03 / 5.23e-03 | 20260507 07:14:12.728 | INFO | PerfReport: 11.68%, Baseline -- 3.58e+00 / 4.50e+00, 5.01e-01 / 1.76e-01 | 20260507 07:14:12.728 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.50e+00, 1.66e-01 / 8.06e-02 | 20260507 07:14:12.728 | INFO | PerfReport: 16.32%, DME -- 4.01e+01 / 4.50e+01, 3.97e+00 / 1.41e+00 | 20260507 07:14:12.728 | INFO | PerfReport: 57.62%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.22e+00 / 1.16e+00 | 20260507 07:14:12.728 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.12e+00 / 2.73e+00, 3.86e-01 / 1.89e-01 | 20260507 07:14:12.728 | INFO | PerfReport: 2.95%, Metrics -- 9.05e-01 / 1.21e+00, 2.40e-01 / 1.25e-01 | 20260507 07:14:12.728 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.05e-01 / 3.62e-01, 5.06e-02 / 2.07e-02 | 20260507 07:14:12.728 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:14:12.728 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:14:12.728 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:14:12.728 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 07:14:12.728 | INFO | PerfReport: 21.84%, Upload -- 2.10e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260507 07:14:12.728 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260507 07:14:12.728 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 07:14:12.728 | INFO | PerfReport: 1.59%, LockUpload -- 1.53e+00 / 0.00e+00, 4.96e+00 / 0.00e+00 | 20260507 07:14:12.728 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260507 07:14:12.728 | INFO | PerfReport: 21.99%, LockCompute -- 2.11e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 07:14:12.728 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260507 07:14:12.824 | INFO | Starting a round of Baz IO | 20260507 07:14:15.918 | INFO | Populating RTMetrics report | 20260507 07:14:15.918 | INFO | RTMetrics report populated | 20260507 07:14:15.918 | INFO | Opening tmp RTMetrics json file | 20260507 07:14:15.936 | INFO | Serializing RTMetrics to json | 20260507 07:14:15.937 | INFO | Serialization done, swapping tmp file now... | 20260507 07:14:15.938 | INFO | RTMetrics done | 20260507 07:14:16.010 | INFO | Finished a round of Baz IO: Wrote 1.653 GiB in 3.185 seconds | 20260507 07:14:16.010 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.57% / 0.00% / 67.43% | 20260507 07:14:16.010 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.110567% / 0% / 7.15256e-05% | 20260507 07:14:16.010 | INFO | Baz IO overall: | 20260507 07:14:16.010 | INFO | BytesWritten : 4058129312877 | 20260507 07:14:16.010 | INFO | NumEvents : 1943313725733 | 20260507 07:14:17.866 | INFO | Analyzing chunk frames = [8085504,8086016) | 20260507 07:14:17.866 | INFO | MemInfo: { | 20260507 07:14:17.866 | INFO | "gpu" : " 17 GB", | 20260507 07:14:17.866 | INFO | "huge" : "", | 20260507 07:14:17.866 | INFO | "huge_shared" : " 27 GB", | 20260507 07:14:17.866 | INFO | "malloc" : " 14 GB", | 20260507 07:14:17.866 | INFO | "pinned" : " 29 GB" | 20260507 07:14:17.866 | INFO | } | 20260507 07:14:17.866 | INFO | Perf report for the last 5.13972 seconds: | 20260507 07:14:17.866 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:14:17.866 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:17.866 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:17.866 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:17.866 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 07:14:17.866 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:17.866 | INFO | RT_METRICS : 43.9%, 1.0, 23.0ms, 0 | 20260507 07:14:17.866 | INFO | PRE_HQ : 59.6%, 1.9, 59.3ms, 0 | 20260507 07:14:17.866 | INFO | BAZWRITER : 61.6%, 3.2, 102.8ms, 0 | 20260507 07:14:17.866 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:14:17.867 | INFO | Uploaded 36GiB to GPU in 2.06069s (17.4699GiB/s) | 20260507 07:14:17.867 | INFO | Downloaded 38GiB to GPU in 2.18503s (17.3911GiB/s) | 20260507 07:14:23.000 | INFO | Analyzing chunk frames = [8086016,8086528) | 20260507 07:14:23.000 | INFO | MemInfo: { | 20260507 07:14:23.000 | INFO | "gpu" : " 17 GB", | 20260507 07:14:23.000 | INFO | "huge" : "", | 20260507 07:14:23.000 | INFO | "huge_shared" : " 27 GB", | 20260507 07:14:23.000 | INFO | "malloc" : " 14 GB", | 20260507 07:14:23.000 | INFO | "pinned" : " 29 GB" | 20260507 07:14:23.000 | INFO | } | 20260507 07:14:23.000 | INFO | Perf report for the last 5.13398 seconds: | 20260507 07:14:23.000 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:14:23.000 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:23.000 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:23.000 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:23.000 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 07:14:23.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:23.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:23.000 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 07:14:23.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:23.001 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:23.001 | INFO | Uploaded 36GiB to GPU in 1.98914s (18.0983GiB/s) | 20260507 07:14:23.001 | INFO | Downloaded 34GiB to GPU in 1.94259s (17.5024GiB/s) | 20260507 07:14:28.138 | INFO | Analyzing chunk frames = [8086528,8087040) | 20260507 07:14:28.138 | INFO | MemInfo: { | 20260507 07:14:28.138 | INFO | "gpu" : " 17 GB", | 20260507 07:14:28.138 | INFO | "huge" : "", | 20260507 07:14:28.138 | INFO | "huge_shared" : " 27 GB", | 20260507 07:14:28.138 | INFO | "malloc" : " 14 GB", | 20260507 07:14:28.138 | INFO | "pinned" : " 29 GB" | 20260507 07:14:28.138 | INFO | } | 20260507 07:14:28.138 | INFO | Perf report for the last 5.13743 seconds: | 20260507 07:14:28.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:14:28.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:28.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:28.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:28.138 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 07:14:28.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:28.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:28.138 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:14:28.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:28.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:28.138 | INFO | Uploaded 36GiB to GPU in 2.05059s (17.5559GiB/s) | 20260507 07:14:28.138 | INFO | Downloaded 34GiB to GPU in 1.95655s (17.3775GiB/s) | 20260507 07:14:33.282 | INFO | Analyzing chunk frames = [8087040,8087552) | 20260507 07:14:33.282 | INFO | MemInfo: { | 20260507 07:14:33.282 | INFO | "gpu" : " 17 GB", | 20260507 07:14:33.282 | INFO | "huge" : "", | 20260507 07:14:33.282 | INFO | "huge_shared" : " 27 GB", | 20260507 07:14:33.282 | INFO | "malloc" : " 14 GB", | 20260507 07:14:33.282 | INFO | "pinned" : " 29 GB" | 20260507 07:14:33.282 | INFO | } | 20260507 07:14:33.282 | INFO | Perf report for the last 5.14432 seconds: | 20260507 07:14:33.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:14:33.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:33.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:33.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:33.282 | INFO | ANALYSIS : 59.4%, 3.0, 92.2ms, 0 | 20260507 07:14:33.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:33.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:33.282 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:14:33.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:33.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:33.282 | INFO | Uploaded 36GiB to GPU in 1.96053s (18.3624GiB/s) | 20260507 07:14:33.282 | INFO | Downloaded 34GiB to GPU in 1.92352s (17.6759GiB/s) | 20260507 07:14:38.425 | INFO | Analyzing chunk frames = [8087552,8088064) | 20260507 07:14:38.425 | INFO | MemInfo: { | 20260507 07:14:38.425 | INFO | "gpu" : " 17 GB", | 20260507 07:14:38.425 | INFO | "huge" : "", | 20260507 07:14:38.425 | INFO | "huge_shared" : " 27 GB", | 20260507 07:14:38.425 | INFO | "malloc" : " 14 GB", | 20260507 07:14:38.425 | INFO | "pinned" : " 29 GB" | 20260507 07:14:38.425 | INFO | } | 20260507 07:14:38.425 | INFO | Perf report for the last 5.14283 seconds: | 20260507 07:14:38.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:14:38.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:38.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:38.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:38.425 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 07:14:38.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:38.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:38.425 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:14:38.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:38.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:38.425 | INFO | Uploaded 36GiB to GPU in 2.05014s (17.5598GiB/s) | 20260507 07:14:38.425 | INFO | Downloaded 34GiB to GPU in 1.95191s (17.4188GiB/s) | 20260507 07:14:43.566 | INFO | Analyzing chunk frames = [8088064,8088576) | 20260507 07:14:43.566 | INFO | MemInfo: { | 20260507 07:14:43.566 | INFO | "gpu" : " 17 GB", | 20260507 07:14:43.566 | INFO | "huge" : "", | 20260507 07:14:43.566 | INFO | "huge_shared" : " 27 GB", | 20260507 07:14:43.566 | INFO | "malloc" : " 14 GB", | 20260507 07:14:43.566 | INFO | "pinned" : " 29 GB" | 20260507 07:14:43.566 | INFO | } | 20260507 07:14:43.566 | INFO | Perf report for the last 5.14111 seconds: | 20260507 07:14:43.566 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:14:43.566 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:43.566 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:43.566 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:43.566 | INFO | ANALYSIS : 60.4%, 3.0, 94.4ms, 0 | 20260507 07:14:43.566 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:43.566 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:43.566 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:14:43.566 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:43.566 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:43.566 | INFO | Uploaded 36GiB to GPU in 2.06902s (17.3995GiB/s) | 20260507 07:14:43.566 | INFO | Downloaded 34GiB to GPU in 1.9413s (17.5141GiB/s) | 20260507 07:14:48.698 | INFO | Analyzing chunk frames = [8088576,8089088) | 20260507 07:14:48.698 | INFO | MemInfo: { | 20260507 07:14:48.698 | INFO | "gpu" : " 17 GB", | 20260507 07:14:48.698 | INFO | "huge" : "", | 20260507 07:14:48.698 | INFO | "huge_shared" : " 27 GB", | 20260507 07:14:48.698 | INFO | "malloc" : " 14 GB", | 20260507 07:14:48.698 | INFO | "pinned" : " 29 GB" | 20260507 07:14:48.698 | INFO | } | 20260507 07:14:48.698 | INFO | Perf report for the last 5.13215 seconds: | 20260507 07:14:48.699 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:14:48.699 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:48.699 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:48.699 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:48.699 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:14:48.699 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:48.699 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:48.699 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 07:14:48.699 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:48.699 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:48.699 | INFO | Uploaded 36GiB to GPU in 1.88157s (19.133GiB/s) | 20260507 07:14:48.699 | INFO | Downloaded 34GiB to GPU in 1.97894s (17.1809GiB/s) | 20260507 07:14:53.840 | INFO | Analyzing chunk frames = [8089088,8089600) | 20260507 07:14:53.840 | INFO | MemInfo: { | 20260507 07:14:53.840 | INFO | "gpu" : " 17 GB", | 20260507 07:14:53.840 | INFO | "huge" : "", | 20260507 07:14:53.840 | INFO | "huge_shared" : " 27 GB", | 20260507 07:14:53.840 | INFO | "malloc" : " 14 GB", | 20260507 07:14:53.840 | INFO | "pinned" : " 29 GB" | 20260507 07:14:53.840 | INFO | } | 20260507 07:14:53.840 | INFO | Perf report for the last 5.14146 seconds: | 20260507 07:14:53.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:14:53.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:53.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:53.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:53.840 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:14:53.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:53.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:53.840 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:14:53.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:53.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:53.840 | INFO | Uploaded 36GiB to GPU in 1.97578s (18.2206GiB/s) | 20260507 07:14:53.840 | INFO | Downloaded 34GiB to GPU in 1.95764s (17.3678GiB/s) | 20260507 07:14:53.936 | INFO | Starting a round of Baz IO | 20260507 07:14:57.031 | INFO | Populating RTMetrics report | 20260507 07:14:57.031 | INFO | RTMetrics report populated | 20260507 07:14:57.031 | INFO | Opening tmp RTMetrics json file | 20260507 07:14:57.038 | INFO | Serializing RTMetrics to json | 20260507 07:14:57.047 | INFO | Serialization done, swapping tmp file now... | 20260507 07:14:57.048 | INFO | RTMetrics done | 20260507 07:14:57.120 | INFO | Finished a round of Baz IO: Wrote 1.652 GiB in 3.184 seconds | 20260507 07:14:57.120 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.57% / 0.00% / 67.43% | 20260507 07:14:57.120 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.11011% / 0% / 6.75519e-05% | 20260507 07:14:57.120 | INFO | Baz IO overall: | 20260507 07:14:57.120 | INFO | BytesWritten : 4059902775064 | 20260507 07:14:57.120 | INFO | NumEvents : 1944142534300 | 20260507 07:14:58.979 | INFO | Analyzing chunk frames = [8089600,8090112) | 20260507 07:14:58.979 | INFO | MemInfo: { | 20260507 07:14:58.979 | INFO | "gpu" : " 17 GB", | 20260507 07:14:58.979 | INFO | "huge" : "", | 20260507 07:14:58.979 | INFO | "huge_shared" : " 27 GB", | 20260507 07:14:58.979 | INFO | "malloc" : " 14 GB", | 20260507 07:14:58.979 | INFO | "pinned" : " 29 GB" | 20260507 07:14:58.979 | INFO | } | 20260507 07:14:58.979 | INFO | Perf report for the last 5.13924 seconds: | 20260507 07:14:58.979 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:14:58.979 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:58.979 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:58.979 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:58.979 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 07:14:58.979 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:14:58.979 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 07:14:58.979 | INFO | PRE_HQ : 58.9%, 1.8, 54.6ms, 0 | 20260507 07:14:58.979 | INFO | BAZWRITER : 61.5%, 3.1, 98.9ms, 0 | 20260507 07:14:58.979 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:14:58.979 | INFO | Uploaded 36GiB to GPU in 2.05907s (17.4837GiB/s) | 20260507 07:14:58.979 | INFO | Downloaded 38GiB to GPU in 2.17823s (17.4454GiB/s) | 20260507 07:15:04.117 | INFO | Analyzing chunk frames = [8090112,8090624) | 20260507 07:15:04.117 | INFO | MemInfo: { | 20260507 07:15:04.117 | INFO | "gpu" : " 17 GB", | 20260507 07:15:04.117 | INFO | "huge" : "", | 20260507 07:15:04.117 | INFO | "huge_shared" : " 27 GB", | 20260507 07:15:04.117 | INFO | "malloc" : " 14 GB", | 20260507 07:15:04.118 | INFO | "pinned" : " 29 GB" | 20260507 07:15:04.118 | INFO | } | 20260507 07:15:04.118 | INFO | Perf report for the last 5.1385 seconds: | 20260507 07:15:04.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:15:04.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:04.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:04.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:04.118 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 07:15:04.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:04.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:04.118 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:15:04.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:04.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:04.118 | INFO | Uploaded 36GiB to GPU in 1.99374s (18.0565GiB/s) | 20260507 07:15:04.118 | INFO | Downloaded 34GiB to GPU in 1.92739s (17.6405GiB/s) | 20260507 07:15:09.259 | INFO | Analyzing chunk frames = [8090624,8091136) | 20260507 07:15:09.259 | INFO | MemInfo: { | 20260507 07:15:09.259 | INFO | "gpu" : " 17 GB", | 20260507 07:15:09.259 | INFO | "huge" : "", | 20260507 07:15:09.259 | INFO | "huge_shared" : " 27 GB", | 20260507 07:15:09.259 | INFO | "malloc" : " 14 GB", | 20260507 07:15:09.259 | INFO | "pinned" : " 29 GB" | 20260507 07:15:09.259 | INFO | } | 20260507 07:15:09.259 | INFO | Perf report for the last 5.14113 seconds: | 20260507 07:15:09.259 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:15:09.259 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:09.259 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:09.259 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:09.259 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:15:09.259 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:09.259 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:09.259 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:15:09.259 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:09.259 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:09.259 | INFO | Uploaded 36GiB to GPU in 2.05718s (17.4997GiB/s) | 20260507 07:15:09.259 | INFO | Downloaded 34GiB to GPU in 1.94973s (17.4383GiB/s) | 20260507 07:15:14.395 | INFO | Analyzing chunk frames = [8091136,8091648) | 20260507 07:15:14.395 | INFO | MemInfo: { | 20260507 07:15:14.395 | INFO | "gpu" : " 17 GB", | 20260507 07:15:14.395 | INFO | "huge" : "", | 20260507 07:15:14.395 | INFO | "huge_shared" : " 27 GB", | 20260507 07:15:14.395 | INFO | "malloc" : " 14 GB", | 20260507 07:15:14.395 | INFO | "pinned" : " 29 GB" | 20260507 07:15:14.395 | INFO | } | 20260507 07:15:14.395 | INFO | Perf report for the last 5.13667 seconds: | 20260507 07:15:14.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:15:14.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:14.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:14.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:14.395 | INFO | ANALYSIS : 60.3%, 2.9, 93.0ms, 0 | 20260507 07:15:14.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:14.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:14.395 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:15:14.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:14.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:14.396 | INFO | Uploaded 36GiB to GPU in 1.99547s (18.0408GiB/s) | 20260507 07:15:14.396 | INFO | Downloaded 34GiB to GPU in 1.93786s (17.5451GiB/s) | 20260507 07:15:19.537 | INFO | Analyzing chunk frames = [8091648,8092160) | 20260507 07:15:19.537 | INFO | MemInfo: { | 20260507 07:15:19.537 | INFO | "gpu" : " 17 GB", | 20260507 07:15:19.537 | INFO | "huge" : "", | 20260507 07:15:19.537 | INFO | "huge_shared" : " 27 GB", | 20260507 07:15:19.537 | INFO | "malloc" : " 14 GB", | 20260507 07:15:19.537 | INFO | "pinned" : " 29 GB" | 20260507 07:15:19.537 | INFO | } | 20260507 07:15:19.537 | INFO | Perf report for the last 5.14145 seconds: | 20260507 07:15:19.537 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:15:19.537 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:19.537 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:19.537 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:19.537 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 07:15:19.537 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:19.537 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:19.537 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:15:19.537 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:19.537 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:19.537 | INFO | Uploaded 36GiB to GPU in 2.05744s (17.4975GiB/s) | 20260507 07:15:19.537 | INFO | Downloaded 34GiB to GPU in 1.9526s (17.4127GiB/s) | 20260507 07:15:24.679 | INFO | Analyzing chunk frames = [8092160,8092672) | 20260507 07:15:24.679 | INFO | MemInfo: { | 20260507 07:15:24.679 | INFO | "gpu" : " 17 GB", | 20260507 07:15:24.679 | INFO | "huge" : "", | 20260507 07:15:24.680 | INFO | "huge_shared" : " 27 GB", | 20260507 07:15:24.680 | INFO | "malloc" : " 14 GB", | 20260507 07:15:24.680 | INFO | "pinned" : " 29 GB" | 20260507 07:15:24.680 | INFO | } | 20260507 07:15:24.680 | INFO | Perf report for the last 5.14279 seconds: | 20260507 07:15:24.680 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:15:24.680 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:24.680 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:24.680 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:24.680 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 07:15:24.680 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:24.680 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:24.680 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 07:15:24.680 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:24.680 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:24.680 | INFO | Uploaded 36GiB to GPU in 2.07174s (17.3767GiB/s) | 20260507 07:15:24.680 | INFO | Downloaded 34GiB to GPU in 1.93022s (17.6145GiB/s) | 20260507 07:15:29.812 | INFO | Analyzing chunk frames = [8092672,8093184) | 20260507 07:15:29.812 | INFO | MemInfo: { | 20260507 07:15:29.812 | INFO | "gpu" : " 17 GB", | 20260507 07:15:29.812 | INFO | "huge" : "", | 20260507 07:15:29.812 | INFO | "huge_shared" : " 27 GB", | 20260507 07:15:29.812 | INFO | "malloc" : " 14 GB", | 20260507 07:15:29.812 | INFO | "pinned" : " 29 GB" | 20260507 07:15:29.812 | INFO | } | 20260507 07:15:29.813 | INFO | Perf report for the last 5.13285 seconds: | 20260507 07:15:29.813 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:15:29.813 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:29.813 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:29.813 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:29.813 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:15:29.813 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:29.813 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:29.813 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 07:15:29.813 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:29.813 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:29.813 | INFO | Uploaded 36GiB to GPU in 1.92903s (18.6622GiB/s) | 20260507 07:15:29.813 | INFO | Downloaded 34GiB to GPU in 1.97163s (17.2446GiB/s) | 20260507 07:15:34.948 | INFO | Analyzing chunk frames = [8093184,8093696) | 20260507 07:15:34.948 | INFO | MemInfo: { | 20260507 07:15:34.948 | INFO | "gpu" : " 17 GB", | 20260507 07:15:34.948 | INFO | "huge" : "", | 20260507 07:15:34.948 | INFO | "huge_shared" : " 27 GB", | 20260507 07:15:34.948 | INFO | "malloc" : " 14 GB", | 20260507 07:15:34.948 | INFO | "pinned" : " 29 GB" | 20260507 07:15:34.948 | INFO | } | 20260507 07:15:34.948 | INFO | Perf report for the last 5.13593 seconds: | 20260507 07:15:34.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:15:34.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:34.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:34.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:34.948 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:15:34.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:34.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:34.949 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 07:15:34.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:34.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:34.949 | INFO | Uploaded 36GiB to GPU in 1.97763s (18.2036GiB/s) | 20260507 07:15:34.949 | INFO | Downloaded 34GiB to GPU in 1.95885s (17.3572GiB/s) | 20260507 07:15:34.949 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:15:34.949 | INFO | Total mem usage: 18 GiB | 20260507 07:15:34.949 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:15:34.949 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 07:15:34.949 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:15:34.949 | INFO | High Water Breakdown: | 20260507 07:15:34.949 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:15:34.949 | INFO | 39.7% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:15:34.949 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:15:34.949 | INFO | ------------------------------------------ | 20260507 07:15:34.949 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:15:34.949 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:15:34.949 | INFO | Total mem usage: 26.5 GiB | 20260507 07:15:34.949 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:15:34.949 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:15:34.949 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:15:34.949 | INFO | High Water Breakdown: | 20260507 07:15:34.949 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:15:34.949 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:15:34.949 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:15:34.949 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:15:34.949 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:15:34.949 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:15:34.949 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:15:34.949 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:15:34.949 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:15:34.949 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:15:34.949 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:15:34.949 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:15:34.949 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:15:34.949 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:15:34.949 | INFO | ------------------------------------------ | 20260507 07:15:34.949 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:15:34.949 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:15:34.949 | INFO | Total mem usage: 25.6 GiB | 20260507 07:15:34.949 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:15:34.949 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:15:34.949 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:15:34.949 | INFO | High Water Breakdown: | 20260507 07:15:34.949 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:15:34.949 | INFO | ------------------------------------------ | 20260507 07:15:34.949 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:15:34.949 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:15:34.949 | INFO | Total mem usage: 15.7 GiB | 20260507 07:15:34.949 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:15:34.949 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:15:34.949 | INFO | Unused Cache: 869 MiB (Waste: 5.42%) | 20260507 07:15:34.949 | INFO | High Water Breakdown: | 20260507 07:15:34.949 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:15:34.949 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:15:34.949 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:15:34.949 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:15:34.949 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:15:34.949 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:15:34.949 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:15:34.949 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:15:34.949 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:15:34.949 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:15:34.949 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:15:34.949 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:15:34.949 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:15:34.949 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:15:34.949 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:15:34.949 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:15:34.949 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:15:34.949 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:15:34.949 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:15:34.949 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:15:34.949 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:15:34.949 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:15:34.949 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:15:34.949 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:15:34.949 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:15:34.949 | INFO | ------------------------------------------ | 20260507 07:15:34.949 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:15:34.949 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:15:34.949 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:15:34.949 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:15:34.949 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.46e+01, 1.32e+01 / 1.19e+00 | 20260507 07:15:34.949 | INFO | PerfReport: 0.29%, AggregateStats -- 9.01e-02 / 1.01e-01, 9.33e-03 / 3.10e-03 | 20260507 07:15:34.950 | INFO | PerfReport: 11.66%, Baseline -- 3.57e+00 / 4.52e+00, 4.95e-01 / 1.19e-01 | 20260507 07:15:34.950 | INFO | PerfReport: 3.22%, Histogram -- 9.85e-01 / 1.51e+00, 1.64e-01 / 5.70e-02 | 20260507 07:15:34.950 | INFO | PerfReport: 16.33%, DME -- 4.00e+01 / 4.49e+01, 3.88e+00 / 1.30e+00 | 20260507 07:15:34.950 | INFO | PerfReport: 57.57%, FrameLabeling -- 1.76e+01 / 1.92e+01, 1.12e+00 / 8.62e-01 | 20260507 07:15:34.950 | INFO | PerfReport: 6.97%, PulseAccumulating -- 2.13e+00 / 2.77e+00, 3.94e-01 / 1.08e-01 | 20260507 07:15:34.950 | INFO | PerfReport: 2.95%, Metrics -- 9.05e-01 / 1.21e+00, 2.41e-01 / 1.34e-01 | 20260507 07:15:34.950 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.66e-01, 5.04e-02 / 2.09e-02 | 20260507 07:15:34.950 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:15:34.950 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:15:34.950 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:15:34.950 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 07:15:34.950 | INFO | PerfReport: 21.80%, Upload -- 2.09e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260507 07:15:34.950 | INFO | PerfReport: 21.43%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 07:15:34.950 | INFO | PerfReport: 31.87%, Compute -- 3.06e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 07:15:34.950 | INFO | PerfReport: 1.55%, LockUpload -- 1.49e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 07:15:34.950 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260507 07:15:34.950 | INFO | PerfReport: 22.02%, LockCompute -- 2.12e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 07:15:34.950 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.26e-02 / 0.00e+00 | 20260507 07:15:35.045 | INFO | Starting a round of Baz IO | 20260507 07:15:38.142 | INFO | Populating RTMetrics report | 20260507 07:15:38.142 | INFO | RTMetrics report populated | 20260507 07:15:38.142 | INFO | Opening tmp RTMetrics json file | 20260507 07:15:38.159 | INFO | Serializing RTMetrics to json | 20260507 07:15:38.160 | INFO | Serialization done, swapping tmp file now... | 20260507 07:15:38.161 | INFO | RTMetrics done | 20260507 07:15:38.228 | INFO | Finished a round of Baz IO: Wrote 1.649 GiB in 3.182 seconds | 20260507 07:15:38.228 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.57% / 0.00% / 67.43% | 20260507 07:15:38.228 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.109828% / 3.97364e-06% / 5.96046e-05% | 20260507 07:15:38.228 | INFO | Baz IO overall: | 20260507 07:15:38.228 | INFO | BytesWritten : 4061673678258 | 20260507 07:15:38.228 | INFO | NumEvents : 1944969969232 | 20260507 07:15:40.092 | INFO | Analyzing chunk frames = [8093696,8094208) | 20260507 07:15:40.092 | INFO | MemInfo: { | 20260507 07:15:40.092 | INFO | "gpu" : " 17 GB", | 20260507 07:15:40.092 | INFO | "huge" : "", | 20260507 07:15:40.092 | INFO | "huge_shared" : " 27 GB", | 20260507 07:15:40.092 | INFO | "malloc" : " 14 GB", | 20260507 07:15:40.092 | INFO | "pinned" : " 29 GB" | 20260507 07:15:40.092 | INFO | } | 20260507 07:15:40.092 | INFO | Perf report for the last 5.14384 seconds: | 20260507 07:15:40.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:15:40.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:40.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:40.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:40.092 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 07:15:40.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:40.092 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 07:15:40.092 | INFO | PRE_HQ : 59.2%, 1.8, 56.8ms, 0 | 20260507 07:15:40.092 | INFO | BAZWRITER : 61.5%, 3.2, 101.2ms, 0 | 20260507 07:15:40.092 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:15:40.092 | INFO | Uploaded 36GiB to GPU in 2.02831s (17.7488GiB/s) | 20260507 07:15:40.093 | INFO | Downloaded 38GiB to GPU in 2.18326s (17.4051GiB/s) | 20260507 07:15:45.237 | INFO | Analyzing chunk frames = [8094208,8094720) | 20260507 07:15:45.237 | INFO | MemInfo: { | 20260507 07:15:45.237 | INFO | "gpu" : " 17 GB", | 20260507 07:15:45.237 | INFO | "huge" : "", | 20260507 07:15:45.237 | INFO | "huge_shared" : " 27 GB", | 20260507 07:15:45.237 | INFO | "malloc" : " 14 GB", | 20260507 07:15:45.237 | INFO | "pinned" : " 29 GB" | 20260507 07:15:45.237 | INFO | } | 20260507 07:15:45.238 | INFO | Perf report for the last 5.14521 seconds: | 20260507 07:15:45.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:15:45.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:45.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:45.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:45.238 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 07:15:45.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:45.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:45.238 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 07:15:45.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:45.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:45.238 | INFO | Uploaded 36GiB to GPU in 1.979s (18.191GiB/s) | 20260507 07:15:45.238 | INFO | Downloaded 34GiB to GPU in 1.93283s (17.5908GiB/s) | 20260507 07:15:50.379 | INFO | Analyzing chunk frames = [8094720,8095232) | 20260507 07:15:50.379 | INFO | MemInfo: { | 20260507 07:15:50.379 | INFO | "gpu" : " 17 GB", | 20260507 07:15:50.380 | INFO | "huge" : "", | 20260507 07:15:50.380 | INFO | "huge_shared" : " 27 GB", | 20260507 07:15:50.380 | INFO | "malloc" : " 14 GB", | 20260507 07:15:50.380 | INFO | "pinned" : " 29 GB" | 20260507 07:15:50.380 | INFO | } | 20260507 07:15:50.380 | INFO | Perf report for the last 5.14201 seconds: | 20260507 07:15:50.380 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:15:50.380 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:50.380 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:50.380 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:50.380 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 07:15:50.380 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:50.380 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:50.380 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:15:50.380 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:50.380 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:50.380 | INFO | Uploaded 36GiB to GPU in 2.04961s (17.5643GiB/s) | 20260507 07:15:50.380 | INFO | Downloaded 34GiB to GPU in 1.95264s (17.4124GiB/s) | 20260507 07:15:55.516 | INFO | Analyzing chunk frames = [8095232,8095744) | 20260507 07:15:55.516 | INFO | MemInfo: { | 20260507 07:15:55.516 | INFO | "gpu" : " 17 GB", | 20260507 07:15:55.516 | INFO | "huge" : "", | 20260507 07:15:55.516 | INFO | "huge_shared" : " 27 GB", | 20260507 07:15:55.516 | INFO | "malloc" : " 14 GB", | 20260507 07:15:55.516 | INFO | "pinned" : " 29 GB" | 20260507 07:15:55.516 | INFO | } | 20260507 07:15:55.516 | INFO | Perf report for the last 5.13616 seconds: | 20260507 07:15:55.516 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:15:55.516 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:55.516 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:55.516 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:55.516 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 07:15:55.516 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:55.516 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:55.516 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:15:55.516 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:55.516 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:15:55.516 | INFO | Uploaded 36GiB to GPU in 2.03478s (17.6923GiB/s) | 20260507 07:15:55.516 | INFO | Downloaded 34GiB to GPU in 1.91774s (17.7292GiB/s) | 20260507 07:16:00.653 | INFO | Analyzing chunk frames = [8095744,8096256) | 20260507 07:16:00.653 | INFO | MemInfo: { | 20260507 07:16:00.653 | INFO | "gpu" : " 17 GB", | 20260507 07:16:00.653 | INFO | "huge" : "", | 20260507 07:16:00.653 | INFO | "huge_shared" : " 27 GB", | 20260507 07:16:00.653 | INFO | "malloc" : " 14 GB", | 20260507 07:16:00.653 | INFO | "pinned" : " 29 GB" | 20260507 07:16:00.653 | INFO | } | 20260507 07:16:00.653 | INFO | Perf report for the last 5.13763 seconds: | 20260507 07:16:00.653 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:16:00.653 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:00.653 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:00.653 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:00.653 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 07:16:00.653 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:00.653 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:00.653 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:16:00.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:00.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:00.654 | INFO | Uploaded 36GiB to GPU in 2.02304s (17.795GiB/s) | 20260507 07:16:00.654 | INFO | Downloaded 34GiB to GPU in 1.92623s (17.651GiB/s) | 20260507 07:16:05.790 | INFO | Analyzing chunk frames = [8096256,8096768) | 20260507 07:16:05.790 | INFO | MemInfo: { | 20260507 07:16:05.790 | INFO | "gpu" : " 17 GB", | 20260507 07:16:05.790 | INFO | "huge" : "", | 20260507 07:16:05.790 | INFO | "huge_shared" : " 27 GB", | 20260507 07:16:05.790 | INFO | "malloc" : " 14 GB", | 20260507 07:16:05.790 | INFO | "pinned" : " 29 GB" | 20260507 07:16:05.790 | INFO | } | 20260507 07:16:05.790 | INFO | Perf report for the last 5.13675 seconds: | 20260507 07:16:05.790 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:16:05.790 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:05.790 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:05.790 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:05.790 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 07:16:05.790 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:05.790 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:05.790 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:16:05.790 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:05.790 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:05.790 | INFO | Uploaded 36GiB to GPU in 2.02615s (17.7677GiB/s) | 20260507 07:16:05.790 | INFO | Downloaded 34GiB to GPU in 1.91824s (17.7246GiB/s) | 20260507 07:16:10.925 | INFO | Analyzing chunk frames = [8096768,8097280) | 20260507 07:16:10.925 | INFO | MemInfo: { | 20260507 07:16:10.925 | INFO | "gpu" : " 17 GB", | 20260507 07:16:10.925 | INFO | "huge" : "", | 20260507 07:16:10.925 | INFO | "huge_shared" : " 27 GB", | 20260507 07:16:10.925 | INFO | "malloc" : " 14 GB", | 20260507 07:16:10.925 | INFO | "pinned" : " 29 GB" | 20260507 07:16:10.925 | INFO | } | 20260507 07:16:10.925 | INFO | Perf report for the last 5.13522 seconds: | 20260507 07:16:10.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:16:10.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:10.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:10.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:10.925 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 07:16:10.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:10.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:10.925 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:16:10.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:10.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:10.926 | INFO | Uploaded 36GiB to GPU in 1.99382s (18.0558GiB/s) | 20260507 07:16:10.926 | INFO | Downloaded 34GiB to GPU in 1.9888s (17.0957GiB/s) | 20260507 07:16:16.066 | INFO | Analyzing chunk frames = [8097280,8097792) | 20260507 07:16:16.066 | INFO | MemInfo: { | 20260507 07:16:16.066 | INFO | "gpu" : " 17 GB", | 20260507 07:16:16.066 | INFO | "huge" : "", | 20260507 07:16:16.066 | INFO | "huge_shared" : " 27 GB", | 20260507 07:16:16.066 | INFO | "malloc" : " 14 GB", | 20260507 07:16:16.066 | INFO | "pinned" : " 29 GB" | 20260507 07:16:16.066 | INFO | } | 20260507 07:16:16.066 | INFO | Perf report for the last 5.14076 seconds: | 20260507 07:16:16.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:16:16.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:16.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:16.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:16.066 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:16:16.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:16.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:16.066 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:16:16.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:16.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:16.066 | INFO | Uploaded 36GiB to GPU in 2.00711s (17.9363GiB/s) | 20260507 07:16:16.066 | INFO | Downloaded 34GiB to GPU in 1.97666s (17.2007GiB/s) | 20260507 07:16:16.163 | INFO | Starting a round of Baz IO | 20260507 07:16:19.255 | INFO | Populating RTMetrics report | 20260507 07:16:19.255 | INFO | RTMetrics report populated | 20260507 07:16:19.255 | INFO | Opening tmp RTMetrics json file | 20260507 07:16:19.264 | INFO | Serializing RTMetrics to json | 20260507 07:16:19.271 | INFO | Serialization done, swapping tmp file now... | 20260507 07:16:19.271 | INFO | RTMetrics done | 20260507 07:16:19.341 | INFO | Finished a round of Baz IO: Wrote 1.65 GiB in 3.178 seconds | 20260507 07:16:19.341 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.57% / 0.00% / 67.43% | 20260507 07:16:19.341 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.109565% / 7.94729e-06% / 7.94729e-05% | 20260507 07:16:19.341 | INFO | Baz IO overall: | 20260507 07:16:19.341 | INFO | BytesWritten : 4063445040537 | 20260507 07:16:19.341 | INFO | NumEvents : 1945797671114 | 20260507 07:16:21.204 | INFO | Analyzing chunk frames = [8097792,8098304) | 20260507 07:16:21.204 | INFO | MemInfo: { | 20260507 07:16:21.204 | INFO | "gpu" : " 17 GB", | 20260507 07:16:21.204 | INFO | "huge" : "", | 20260507 07:16:21.204 | INFO | "huge_shared" : " 27 GB", | 20260507 07:16:21.204 | INFO | "malloc" : " 14 GB", | 20260507 07:16:21.204 | INFO | "pinned" : " 29 GB" | 20260507 07:16:21.204 | INFO | } | 20260507 07:16:21.204 | INFO | Perf report for the last 5.13818 seconds: | 20260507 07:16:21.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:16:21.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:21.204 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:21.204 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:21.204 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 07:16:21.204 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:21.204 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:16:21.204 | INFO | PRE_HQ : 58.7%, 1.8, 54.4ms, 0 | 20260507 07:16:21.204 | INFO | BAZWRITER : 61.4%, 3.1, 98.1ms, 0 | 20260507 07:16:21.204 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:16:21.205 | INFO | Uploaded 36GiB to GPU in 2.05421s (17.525GiB/s) | 20260507 07:16:21.205 | INFO | Downloaded 38GiB to GPU in 2.17681s (17.4568GiB/s) | 20260507 07:16:26.349 | INFO | Analyzing chunk frames = [8098304,8098816) | 20260507 07:16:26.349 | INFO | MemInfo: { | 20260507 07:16:26.349 | INFO | "gpu" : " 17 GB", | 20260507 07:16:26.349 | INFO | "huge" : "", | 20260507 07:16:26.349 | INFO | "huge_shared" : " 27 GB", | 20260507 07:16:26.349 | INFO | "malloc" : " 14 GB", | 20260507 07:16:26.349 | INFO | "pinned" : " 29 GB" | 20260507 07:16:26.349 | INFO | } | 20260507 07:16:26.349 | INFO | Perf report for the last 5.14435 seconds: | 20260507 07:16:26.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:16:26.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:26.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:26.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:26.349 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 07:16:26.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:26.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:26.349 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:16:26.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:26.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:26.349 | INFO | Uploaded 36GiB to GPU in 2.00088s (17.9921GiB/s) | 20260507 07:16:26.349 | INFO | Downloaded 34GiB to GPU in 1.93813s (17.5427GiB/s) | 20260507 07:16:31.490 | INFO | Analyzing chunk frames = [8098816,8099328) | 20260507 07:16:31.490 | INFO | MemInfo: { | 20260507 07:16:31.490 | INFO | "gpu" : " 17 GB", | 20260507 07:16:31.490 | INFO | "huge" : "", | 20260507 07:16:31.490 | INFO | "huge_shared" : " 27 GB", | 20260507 07:16:31.490 | INFO | "malloc" : " 14 GB", | 20260507 07:16:31.490 | INFO | "pinned" : " 29 GB" | 20260507 07:16:31.490 | INFO | } | 20260507 07:16:31.490 | INFO | Perf report for the last 5.14144 seconds: | 20260507 07:16:31.490 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:16:31.490 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:31.490 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:31.490 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:31.490 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:16:31.490 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:31.490 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:31.490 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:16:31.490 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:31.490 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:31.490 | INFO | Uploaded 36GiB to GPU in 2.04595s (17.5957GiB/s) | 20260507 07:16:31.490 | INFO | Downloaded 34GiB to GPU in 1.95546s (17.3872GiB/s) | 20260507 07:16:36.632 | INFO | Analyzing chunk frames = [8099328,8099840) | 20260507 07:16:36.632 | INFO | MemInfo: { | 20260507 07:16:36.632 | INFO | "gpu" : " 17 GB", | 20260507 07:16:36.632 | INFO | "huge" : "", | 20260507 07:16:36.632 | INFO | "huge_shared" : " 27 GB", | 20260507 07:16:36.632 | INFO | "malloc" : " 14 GB", | 20260507 07:16:36.632 | INFO | "pinned" : " 29 GB" | 20260507 07:16:36.632 | INFO | } | 20260507 07:16:36.632 | INFO | Perf report for the last 5.1423 seconds: | 20260507 07:16:36.633 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:16:36.633 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:36.633 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:36.633 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:36.633 | INFO | ANALYSIS : 60.1%, 2.9, 92.8ms, 0 | 20260507 07:16:36.633 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:36.633 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:36.633 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:16:36.633 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:36.633 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:36.633 | INFO | Uploaded 36GiB to GPU in 1.969s (18.2834GiB/s) | 20260507 07:16:36.633 | INFO | Downloaded 34GiB to GPU in 1.94798s (17.454GiB/s) | 20260507 07:16:41.761 | INFO | Analyzing chunk frames = [8099840,8100352) | 20260507 07:16:41.761 | INFO | MemInfo: { | 20260507 07:16:41.761 | INFO | "gpu" : " 17 GB", | 20260507 07:16:41.761 | INFO | "huge" : "", | 20260507 07:16:41.761 | INFO | "huge_shared" : " 27 GB", | 20260507 07:16:41.761 | INFO | "malloc" : " 14 GB", | 20260507 07:16:41.761 | INFO | "pinned" : " 29 GB" | 20260507 07:16:41.761 | INFO | } | 20260507 07:16:41.761 | INFO | Perf report for the last 5.12862 seconds: | 20260507 07:16:41.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:16:41.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:41.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:41.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:41.761 | INFO | ANALYSIS : 60.2%, 3.0, 93.5ms, 0 | 20260507 07:16:41.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:41.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:41.761 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 07:16:41.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:41.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:41.761 | INFO | Uploaded 36GiB to GPU in 2.04275s (17.6233GiB/s) | 20260507 07:16:41.761 | INFO | Downloaded 34GiB to GPU in 1.90341s (17.8626GiB/s) | 20260507 07:16:46.901 | INFO | Analyzing chunk frames = [8100352,8100864) | 20260507 07:16:46.901 | INFO | MemInfo: { | 20260507 07:16:46.901 | INFO | "gpu" : " 17 GB", | 20260507 07:16:46.901 | INFO | "huge" : "", | 20260507 07:16:46.901 | INFO | "huge_shared" : " 27 GB", | 20260507 07:16:46.901 | INFO | "malloc" : " 14 GB", | 20260507 07:16:46.901 | INFO | "pinned" : " 29 GB" | 20260507 07:16:46.901 | INFO | } | 20260507 07:16:46.901 | INFO | Perf report for the last 5.14009 seconds: | 20260507 07:16:46.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:16:46.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:46.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:46.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:46.901 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 07:16:46.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:46.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:46.901 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 07:16:46.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:46.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:46.901 | INFO | Uploaded 36GiB to GPU in 2.04545s (17.6001GiB/s) | 20260507 07:16:46.901 | INFO | Downloaded 34GiB to GPU in 1.92084s (17.7006GiB/s) | 20260507 07:16:52.043 | INFO | Analyzing chunk frames = [8100864,8101376) | 20260507 07:16:52.043 | INFO | MemInfo: { | 20260507 07:16:52.043 | INFO | "gpu" : " 17 GB", | 20260507 07:16:52.043 | INFO | "huge" : "", | 20260507 07:16:52.043 | INFO | "huge_shared" : " 27 GB", | 20260507 07:16:52.043 | INFO | "malloc" : " 14 GB", | 20260507 07:16:52.043 | INFO | "pinned" : " 29 GB" | 20260507 07:16:52.043 | INFO | } | 20260507 07:16:52.043 | INFO | Perf report for the last 5.14203 seconds: | 20260507 07:16:52.043 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:16:52.043 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:52.043 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:52.043 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:52.043 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 07:16:52.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:52.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:52.043 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 07:16:52.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:52.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:52.043 | INFO | Uploaded 36GiB to GPU in 2.00679s (17.9391GiB/s) | 20260507 07:16:52.043 | INFO | Downloaded 34GiB to GPU in 1.97938s (17.1771GiB/s) | 20260507 07:16:57.191 | INFO | Analyzing chunk frames = [8101376,8101888) | 20260507 07:16:57.191 | INFO | MemInfo: { | 20260507 07:16:57.191 | INFO | "gpu" : " 17 GB", | 20260507 07:16:57.191 | INFO | "huge" : "", | 20260507 07:16:57.191 | INFO | "huge_shared" : " 27 GB", | 20260507 07:16:57.191 | INFO | "malloc" : " 14 GB", | 20260507 07:16:57.191 | INFO | "pinned" : " 29 GB" | 20260507 07:16:57.191 | INFO | } | 20260507 07:16:57.191 | INFO | Perf report for the last 5.14805 seconds: | 20260507 07:16:57.191 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:16:57.191 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:57.191 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:57.191 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:57.191 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 07:16:57.191 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:57.191 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:57.191 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 07:16:57.191 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:57.191 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:16:57.191 | INFO | Uploaded 36GiB to GPU in 1.97785s (18.2016GiB/s) | 20260507 07:16:57.192 | INFO | Downloaded 34GiB to GPU in 1.96436s (17.3084GiB/s) | 20260507 07:16:57.192 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:16:57.192 | INFO | Total mem usage: 18 GiB | 20260507 07:16:57.192 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:16:57.192 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:16:57.192 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:16:57.192 | INFO | High Water Breakdown: | 20260507 07:16:57.192 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:16:57.192 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:16:57.192 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:16:57.192 | INFO | ------------------------------------------ | 20260507 07:16:57.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:16:57.192 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:16:57.192 | INFO | Total mem usage: 26.5 GiB | 20260507 07:16:57.192 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:16:57.192 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:16:57.192 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:16:57.192 | INFO | High Water Breakdown: | 20260507 07:16:57.192 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:16:57.192 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:16:57.192 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:16:57.192 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:16:57.192 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:16:57.192 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:16:57.192 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:16:57.192 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:16:57.192 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:16:57.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:16:57.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:16:57.192 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:16:57.192 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:16:57.192 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:16:57.192 | INFO | ------------------------------------------ | 20260507 07:16:57.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:16:57.192 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:16:57.192 | INFO | Total mem usage: 25.6 GiB | 20260507 07:16:57.192 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:16:57.192 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:16:57.192 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:16:57.192 | INFO | High Water Breakdown: | 20260507 07:16:57.192 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:16:57.192 | INFO | ------------------------------------------ | 20260507 07:16:57.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:16:57.192 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:16:57.192 | INFO | Total mem usage: 15.7 GiB | 20260507 07:16:57.192 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:16:57.192 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:16:57.192 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:16:57.192 | INFO | High Water Breakdown: | 20260507 07:16:57.192 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:16:57.192 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:16:57.192 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:16:57.192 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:16:57.192 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:16:57.192 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:16:57.192 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:16:57.192 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:16:57.192 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:16:57.192 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:16:57.192 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:16:57.192 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:16:57.192 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:16:57.192 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:16:57.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:16:57.192 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:16:57.192 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:16:57.192 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:16:57.192 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:16:57.192 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:16:57.192 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:16:57.192 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:16:57.192 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:16:57.192 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:16:57.192 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:16:57.192 | INFO | ------------------------------------------ | 20260507 07:16:57.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:16:57.192 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:16:57.192 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:16:57.192 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:16:57.192 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.34e+01 / 1.33e+00 | 20260507 07:16:57.192 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 9.59e-03 / 4.95e-03 | 20260507 07:16:57.192 | INFO | PerfReport: 11.64%, Baseline -- 3.57e+00 / 4.47e+00, 4.99e-01 / 2.13e-01 | 20260507 07:16:57.192 | INFO | PerfReport: 3.21%, Histogram -- 9.87e-01 / 1.47e+00, 1.65e-01 / 1.21e-01 | 20260507 07:16:57.192 | INFO | PerfReport: 16.28%, DME -- 4.00e+01 / 4.47e+01, 3.82e+00 / 1.58e+00 | 20260507 07:16:57.192 | INFO | PerfReport: 57.73%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.29e+00 / 1.40e+00 | 20260507 07:16:57.192 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.69e+00, 3.86e-01 / 2.03e-01 | 20260507 07:16:57.192 | INFO | PerfReport: 2.94%, Metrics -- 9.03e-01 / 1.19e+00, 2.42e-01 / 1.29e-01 | 20260507 07:16:57.192 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.62e-01, 5.12e-02 / 2.01e-02 | 20260507 07:16:57.192 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:16:57.192 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:16:57.192 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:16:57.192 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 07:16:57.192 | INFO | PerfReport: 21.86%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260507 07:16:57.193 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 07:16:57.193 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 07:16:57.193 | INFO | PerfReport: 1.49%, LockUpload -- 1.43e+00 / 0.00e+00, 4.87e+00 / 0.00e+00 | 20260507 07:16:57.193 | INFO | PerfReport: 1.36%, LockDownload -- 1.31e+00 / 0.00e+00, 3.11e+00 / 0.00e+00 | 20260507 07:16:57.193 | INFO | PerfReport: 21.97%, LockCompute -- 2.11e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260507 07:16:57.193 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.54e-02 / 0.00e+00 | 20260507 07:16:57.288 | INFO | Starting a round of Baz IO | 20260507 07:17:00.375 | INFO | Populating RTMetrics report | 20260507 07:17:00.375 | INFO | RTMetrics report populated | 20260507 07:17:00.375 | INFO | Opening tmp RTMetrics json file | 20260507 07:17:00.392 | INFO | Serializing RTMetrics to json | 20260507 07:17:00.393 | INFO | Serialization done, swapping tmp file now... | 20260507 07:17:00.394 | INFO | RTMetrics done | 20260507 07:17:00.463 | INFO | Finished a round of Baz IO: Wrote 1.65 GiB in 3.175 seconds | 20260507 07:17:00.463 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.57% / 0.00% / 67.43% | 20260507 07:17:00.463 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.109319% / 0% / 4.76837e-05% | 20260507 07:17:00.463 | INFO | Baz IO overall: | 20260507 07:17:00.463 | INFO | BytesWritten : 4065216515386 | 20260507 07:17:00.463 | INFO | NumEvents : 1946626109679 | 20260507 07:17:02.329 | INFO | Analyzing chunk frames = [8101888,8102400) | 20260507 07:17:02.329 | INFO | MemInfo: { | 20260507 07:17:02.329 | INFO | "gpu" : " 17 GB", | 20260507 07:17:02.329 | INFO | "huge" : "", | 20260507 07:17:02.329 | INFO | "huge_shared" : " 27 GB", | 20260507 07:17:02.329 | INFO | "malloc" : " 14 GB", | 20260507 07:17:02.329 | INFO | "pinned" : " 29 GB" | 20260507 07:17:02.329 | INFO | } | 20260507 07:17:02.329 | INFO | Perf report for the last 5.13795 seconds: | 20260507 07:17:02.329 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:17:02.329 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:02.329 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:02.329 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:02.329 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260507 07:17:02.329 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:02.329 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 07:17:02.329 | INFO | PRE_HQ : 58.8%, 1.8, 54.7ms, 0 | 20260507 07:17:02.329 | INFO | BAZWRITER : 61.4%, 3.1, 98.2ms, 0 | 20260507 07:17:02.329 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 07:17:02.330 | INFO | Uploaded 36GiB to GPU in 2.05338s (17.5321GiB/s) | 20260507 07:17:02.330 | INFO | Downloaded 38GiB to GPU in 2.17467s (17.4739GiB/s) | 20260507 07:17:07.463 | INFO | Analyzing chunk frames = [8102400,8102912) | 20260507 07:17:07.463 | INFO | MemInfo: { | 20260507 07:17:07.463 | INFO | "gpu" : " 17 GB", | 20260507 07:17:07.463 | INFO | "huge" : "", | 20260507 07:17:07.463 | INFO | "huge_shared" : " 27 GB", | 20260507 07:17:07.463 | INFO | "malloc" : " 14 GB", | 20260507 07:17:07.463 | INFO | "pinned" : " 29 GB" | 20260507 07:17:07.463 | INFO | } | 20260507 07:17:07.463 | INFO | Perf report for the last 5.13383 seconds: | 20260507 07:17:07.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:17:07.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:07.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:07.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:07.463 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 07:17:07.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:07.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:07.463 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:17:07.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:07.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:07.463 | INFO | Uploaded 36GiB to GPU in 2.00861s (17.9228GiB/s) | 20260507 07:17:07.463 | INFO | Downloaded 34GiB to GPU in 1.94421s (17.4878GiB/s) | 20260507 07:17:12.604 | INFO | Analyzing chunk frames = [8102912,8103424) | 20260507 07:17:12.604 | INFO | MemInfo: { | 20260507 07:17:12.604 | INFO | "gpu" : " 17 GB", | 20260507 07:17:12.604 | INFO | "huge" : "", | 20260507 07:17:12.604 | INFO | "huge_shared" : " 27 GB", | 20260507 07:17:12.604 | INFO | "malloc" : " 14 GB", | 20260507 07:17:12.604 | INFO | "pinned" : " 29 GB" | 20260507 07:17:12.604 | INFO | } | 20260507 07:17:12.604 | INFO | Perf report for the last 5.14083 seconds: | 20260507 07:17:12.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:17:12.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:12.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:12.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:12.604 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:17:12.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:12.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:12.604 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:17:12.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:12.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:12.604 | INFO | Uploaded 36GiB to GPU in 2.04911s (17.5686GiB/s) | 20260507 07:17:12.604 | INFO | Downloaded 34GiB to GPU in 1.95346s (17.405GiB/s) | 20260507 07:17:17.739 | INFO | Analyzing chunk frames = [8103424,8103936) | 20260507 07:17:17.739 | INFO | MemInfo: { | 20260507 07:17:17.739 | INFO | "gpu" : " 17 GB", | 20260507 07:17:17.739 | INFO | "huge" : "", | 20260507 07:17:17.739 | INFO | "huge_shared" : " 27 GB", | 20260507 07:17:17.739 | INFO | "malloc" : " 14 GB", | 20260507 07:17:17.739 | INFO | "pinned" : " 29 GB" | 20260507 07:17:17.739 | INFO | } | 20260507 07:17:17.740 | INFO | Perf report for the last 5.13555 seconds: | 20260507 07:17:17.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:17:17.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:17.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:17.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:17.740 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 07:17:17.740 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:17.740 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:17.740 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:17:17.740 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:17.740 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:17.740 | INFO | Uploaded 36GiB to GPU in 1.99687s (18.0283GiB/s) | 20260507 07:17:17.740 | INFO | Downloaded 34GiB to GPU in 1.93787s (17.545GiB/s) | 20260507 07:17:22.880 | INFO | Analyzing chunk frames = [8103936,8104448) | 20260507 07:17:22.880 | INFO | MemInfo: { | 20260507 07:17:22.880 | INFO | "gpu" : " 17 GB", | 20260507 07:17:22.880 | INFO | "huge" : "", | 20260507 07:17:22.880 | INFO | "huge_shared" : " 27 GB", | 20260507 07:17:22.880 | INFO | "malloc" : " 14 GB", | 20260507 07:17:22.880 | INFO | "pinned" : " 29 GB" | 20260507 07:17:22.880 | INFO | } | 20260507 07:17:22.880 | INFO | Perf report for the last 5.14077 seconds: | 20260507 07:17:22.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:17:22.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:22.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:22.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:22.880 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 07:17:22.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:22.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:22.880 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:17:22.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:22.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:22.881 | INFO | Uploaded 36GiB to GPU in 2.05049s (17.5568GiB/s) | 20260507 07:17:22.881 | INFO | Downloaded 34GiB to GPU in 1.95137s (17.4237GiB/s) | 20260507 07:17:28.015 | INFO | Analyzing chunk frames = [8104448,8104960) | 20260507 07:17:28.015 | INFO | MemInfo: { | 20260507 07:17:28.015 | INFO | "gpu" : " 17 GB", | 20260507 07:17:28.015 | INFO | "huge" : "", | 20260507 07:17:28.015 | INFO | "huge_shared" : " 27 GB", | 20260507 07:17:28.015 | INFO | "malloc" : " 14 GB", | 20260507 07:17:28.015 | INFO | "pinned" : " 29 GB" | 20260507 07:17:28.015 | INFO | } | 20260507 07:17:28.015 | INFO | Perf report for the last 5.13514 seconds: | 20260507 07:17:28.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:17:28.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:28.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:28.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:28.015 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:17:28.016 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:28.016 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:28.016 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 07:17:28.016 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:28.016 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:28.016 | INFO | Uploaded 36GiB to GPU in 2.06617s (17.4235GiB/s) | 20260507 07:17:28.016 | INFO | Downloaded 34GiB to GPU in 1.93747s (17.5487GiB/s) | 20260507 07:17:33.159 | INFO | Analyzing chunk frames = [8104960,8105472) | 20260507 07:17:33.159 | INFO | MemInfo: { | 20260507 07:17:33.159 | INFO | "gpu" : " 17 GB", | 20260507 07:17:33.159 | INFO | "huge" : "", | 20260507 07:17:33.159 | INFO | "huge_shared" : " 27 GB", | 20260507 07:17:33.159 | INFO | "malloc" : " 14 GB", | 20260507 07:17:33.159 | INFO | "pinned" : " 29 GB" | 20260507 07:17:33.159 | INFO | } | 20260507 07:17:33.159 | INFO | Perf report for the last 5.14328 seconds: | 20260507 07:17:33.159 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:17:33.159 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:33.159 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:33.159 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:33.159 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 07:17:33.159 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:33.159 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:33.159 | INFO | PRE_HQ : 25.9%, 1.0, 13.5ms, 0 | 20260507 07:17:33.159 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:33.159 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:33.159 | INFO | Uploaded 36GiB to GPU in 1.97139s (18.2613GiB/s) | 20260507 07:17:33.159 | INFO | Downloaded 34GiB to GPU in 1.97775s (17.1913GiB/s) | 20260507 07:17:38.305 | INFO | Analyzing chunk frames = [8105472,8105984) | 20260507 07:17:38.305 | INFO | MemInfo: { | 20260507 07:17:38.305 | INFO | "gpu" : " 17 GB", | 20260507 07:17:38.305 | INFO | "huge" : "", | 20260507 07:17:38.305 | INFO | "huge_shared" : " 27 GB", | 20260507 07:17:38.305 | INFO | "malloc" : " 14 GB", | 20260507 07:17:38.305 | INFO | "pinned" : " 29 GB" | 20260507 07:17:38.305 | INFO | } | 20260507 07:17:38.305 | INFO | Perf report for the last 5.14669 seconds: | 20260507 07:17:38.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:17:38.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:38.305 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:38.305 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:38.305 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:17:38.305 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:38.305 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:38.306 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 07:17:38.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:38.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:38.306 | INFO | Uploaded 36GiB to GPU in 1.9836s (18.1489GiB/s) | 20260507 07:17:38.306 | INFO | Downloaded 34GiB to GPU in 1.9614s (17.3345GiB/s) | 20260507 07:17:38.402 | INFO | Starting a round of Baz IO | 20260507 07:17:41.490 | INFO | Populating RTMetrics report | 20260507 07:17:41.490 | INFO | RTMetrics report populated | 20260507 07:17:41.490 | INFO | Opening tmp RTMetrics json file | 20260507 07:17:41.501 | INFO | Serializing RTMetrics to json | 20260507 07:17:41.505 | INFO | Serialization done, swapping tmp file now... | 20260507 07:17:41.506 | INFO | RTMetrics done | 20260507 07:17:41.578 | INFO | Finished a round of Baz IO: Wrote 1.648 GiB in 3.175 seconds | 20260507 07:17:41.578 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.57% / 0.00% / 67.43% | 20260507 07:17:41.578 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.108906% / 0% / 5.96046e-05% | 20260507 07:17:41.578 | INFO | Baz IO overall: | 20260507 07:17:41.578 | INFO | BytesWritten : 4066985902700 | 20260507 07:17:41.578 | INFO | NumEvents : 1947452962765 | 20260507 07:17:43.442 | INFO | Analyzing chunk frames = [8105984,8106496) | 20260507 07:17:43.442 | INFO | MemInfo: { | 20260507 07:17:43.442 | INFO | "gpu" : " 17 GB", | 20260507 07:17:43.442 | INFO | "huge" : "", | 20260507 07:17:43.442 | INFO | "huge_shared" : " 27 GB", | 20260507 07:17:43.442 | INFO | "malloc" : " 14 GB", | 20260507 07:17:43.442 | INFO | "pinned" : " 29 GB" | 20260507 07:17:43.442 | INFO | } | 20260507 07:17:43.442 | INFO | Perf report for the last 5.13697 seconds: | 20260507 07:17:43.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:17:43.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:43.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:43.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:43.442 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260507 07:17:43.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:43.442 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:17:43.442 | INFO | PRE_HQ : 58.7%, 1.8, 54.6ms, 0 | 20260507 07:17:43.443 | INFO | BAZWRITER : 61.3%, 3.1, 97.4ms, 0 | 20260507 07:17:43.443 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:17:43.443 | INFO | Uploaded 36GiB to GPU in 2.05514s (17.517GiB/s) | 20260507 07:17:43.443 | INFO | Downloaded 38GiB to GPU in 2.17111s (17.5026GiB/s) | 20260507 07:17:48.576 | INFO | Analyzing chunk frames = [8106496,8107008) | 20260507 07:17:48.576 | INFO | MemInfo: { | 20260507 07:17:48.576 | INFO | "gpu" : " 17 GB", | 20260507 07:17:48.576 | INFO | "huge" : "", | 20260507 07:17:48.576 | INFO | "huge_shared" : " 27 GB", | 20260507 07:17:48.576 | INFO | "malloc" : " 14 GB", | 20260507 07:17:48.576 | INFO | "pinned" : " 29 GB" | 20260507 07:17:48.576 | INFO | } | 20260507 07:17:48.576 | INFO | Perf report for the last 5.13341 seconds: | 20260507 07:17:48.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:17:48.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:48.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:48.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:48.576 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 07:17:48.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:48.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:48.576 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:17:48.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:48.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:48.576 | INFO | Uploaded 36GiB to GPU in 2.00724s (17.9351GiB/s) | 20260507 07:17:48.576 | INFO | Downloaded 34GiB to GPU in 1.94559s (17.4754GiB/s) | 20260507 07:17:53.711 | INFO | Analyzing chunk frames = [8107008,8107520) | 20260507 07:17:53.711 | INFO | MemInfo: { | 20260507 07:17:53.711 | INFO | "gpu" : " 17 GB", | 20260507 07:17:53.711 | INFO | "huge" : "", | 20260507 07:17:53.711 | INFO | "huge_shared" : " 27 GB", | 20260507 07:17:53.711 | INFO | "malloc" : " 14 GB", | 20260507 07:17:53.711 | INFO | "pinned" : " 29 GB" | 20260507 07:17:53.711 | INFO | } | 20260507 07:17:53.711 | INFO | Perf report for the last 5.13486 seconds: | 20260507 07:17:53.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:17:53.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:53.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:53.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:53.711 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 07:17:53.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:53.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:53.711 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:17:53.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:53.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:53.711 | INFO | Uploaded 36GiB to GPU in 2.04557s (17.599GiB/s) | 20260507 07:17:53.711 | INFO | Downloaded 34GiB to GPU in 1.95282s (17.4107GiB/s) | 20260507 07:17:58.854 | INFO | Analyzing chunk frames = [8107520,8108032) | 20260507 07:17:58.854 | INFO | MemInfo: { | 20260507 07:17:58.854 | INFO | "gpu" : " 17 GB", | 20260507 07:17:58.854 | INFO | "huge" : "", | 20260507 07:17:58.854 | INFO | "huge_shared" : " 27 GB", | 20260507 07:17:58.854 | INFO | "malloc" : " 14 GB", | 20260507 07:17:58.854 | INFO | "pinned" : " 29 GB" | 20260507 07:17:58.854 | INFO | } | 20260507 07:17:58.854 | INFO | Perf report for the last 5.14336 seconds: | 20260507 07:17:58.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:17:58.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:58.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:58.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:58.854 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 07:17:58.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:58.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:58.854 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:17:58.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:58.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:17:58.854 | INFO | Uploaded 36GiB to GPU in 1.97398s (18.2373GiB/s) | 20260507 07:17:58.854 | INFO | Downloaded 34GiB to GPU in 1.92345s (17.6766GiB/s) | 20260507 07:18:03.996 | INFO | Analyzing chunk frames = [8108032,8108544) | 20260507 07:18:03.996 | INFO | MemInfo: { | 20260507 07:18:03.996 | INFO | "gpu" : " 17 GB", | 20260507 07:18:03.996 | INFO | "huge" : "", | 20260507 07:18:03.996 | INFO | "huge_shared" : " 27 GB", | 20260507 07:18:03.996 | INFO | "malloc" : " 14 GB", | 20260507 07:18:03.996 | INFO | "pinned" : " 29 GB" | 20260507 07:18:03.996 | INFO | } | 20260507 07:18:03.996 | INFO | Perf report for the last 5.14201 seconds: | 20260507 07:18:03.996 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:18:03.996 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:03.996 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:03.996 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:03.996 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 07:18:03.996 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:03.996 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:03.996 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:18:03.996 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:03.996 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:03.997 | INFO | Uploaded 36GiB to GPU in 2.05128s (17.5501GiB/s) | 20260507 07:18:03.997 | INFO | Downloaded 34GiB to GPU in 1.95222s (17.416GiB/s) | 20260507 07:18:09.142 | INFO | Analyzing chunk frames = [8108544,8109056) | 20260507 07:18:09.142 | INFO | MemInfo: { | 20260507 07:18:09.142 | INFO | "gpu" : " 17 GB", | 20260507 07:18:09.142 | INFO | "huge" : "", | 20260507 07:18:09.142 | INFO | "huge_shared" : " 27 GB", | 20260507 07:18:09.142 | INFO | "malloc" : " 14 GB", | 20260507 07:18:09.142 | INFO | "pinned" : " 29 GB" | 20260507 07:18:09.142 | INFO | } | 20260507 07:18:09.142 | INFO | Perf report for the last 5.14606 seconds: | 20260507 07:18:09.142 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:18:09.142 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:09.142 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:09.142 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:09.142 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 07:18:09.142 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:09.142 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:09.142 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:18:09.142 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:09.142 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:09.142 | INFO | Uploaded 36GiB to GPU in 2.0577s (17.4952GiB/s) | 20260507 07:18:09.142 | INFO | Downloaded 34GiB to GPU in 1.92568s (17.6561GiB/s) | 20260507 07:18:14.274 | INFO | Analyzing chunk frames = [8109056,8109568) | 20260507 07:18:14.274 | INFO | MemInfo: { | 20260507 07:18:14.274 | INFO | "gpu" : " 17 GB", | 20260507 07:18:14.274 | INFO | "huge" : "", | 20260507 07:18:14.274 | INFO | "huge_shared" : " 27 GB", | 20260507 07:18:14.275 | INFO | "malloc" : " 14 GB", | 20260507 07:18:14.275 | INFO | "pinned" : " 29 GB" | 20260507 07:18:14.275 | INFO | } | 20260507 07:18:14.275 | INFO | Perf report for the last 5.13248 seconds: | 20260507 07:18:14.275 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:18:14.275 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:14.275 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:14.275 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:14.275 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 07:18:14.275 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:14.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:14.275 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 07:18:14.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:14.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:14.275 | INFO | Uploaded 36GiB to GPU in 1.96436s (18.3266GiB/s) | 20260507 07:18:14.275 | INFO | Downloaded 34GiB to GPU in 1.96385s (17.3129GiB/s) | 20260507 07:18:19.414 | INFO | Analyzing chunk frames = [8109568,8110080) | 20260507 07:18:19.414 | INFO | MemInfo: { | 20260507 07:18:19.414 | INFO | "gpu" : " 17 GB", | 20260507 07:18:19.414 | INFO | "huge" : "", | 20260507 07:18:19.414 | INFO | "huge_shared" : " 27 GB", | 20260507 07:18:19.414 | INFO | "malloc" : " 14 GB", | 20260507 07:18:19.414 | INFO | "pinned" : " 29 GB" | 20260507 07:18:19.414 | INFO | } | 20260507 07:18:19.414 | INFO | Perf report for the last 5.13956 seconds: | 20260507 07:18:19.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:18:19.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:19.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:19.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:19.414 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:18:19.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:19.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:19.415 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 07:18:19.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:19.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:19.415 | INFO | Uploaded 36GiB to GPU in 1.98968s (18.0933GiB/s) | 20260507 07:18:19.415 | INFO | Downloaded 34GiB to GPU in 1.96051s (17.3424GiB/s) | 20260507 07:18:19.415 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:18:19.415 | INFO | Total mem usage: 18 GiB | 20260507 07:18:19.415 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:18:19.415 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:18:19.415 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:18:19.415 | INFO | High Water Breakdown: | 20260507 07:18:19.415 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:18:19.415 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:18:19.415 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:18:19.415 | INFO | ------------------------------------------ | 20260507 07:18:19.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:18:19.415 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:18:19.415 | INFO | Total mem usage: 26.5 GiB | 20260507 07:18:19.415 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:18:19.415 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:18:19.415 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:18:19.415 | INFO | High Water Breakdown: | 20260507 07:18:19.415 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:18:19.415 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:18:19.415 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:18:19.415 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:18:19.415 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:18:19.415 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:18:19.415 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:18:19.415 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:18:19.415 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:18:19.415 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:18:19.415 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:18:19.415 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:18:19.415 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:18:19.415 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:18:19.415 | INFO | ------------------------------------------ | 20260507 07:18:19.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:18:19.415 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:18:19.415 | INFO | Total mem usage: 25.6 GiB | 20260507 07:18:19.415 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:18:19.415 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:18:19.415 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:18:19.415 | INFO | High Water Breakdown: | 20260507 07:18:19.415 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:18:19.415 | INFO | ------------------------------------------ | 20260507 07:18:19.415 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:18:19.415 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:18:19.415 | INFO | Total mem usage: 15.7 GiB | 20260507 07:18:19.415 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:18:19.416 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:18:19.416 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:18:19.416 | INFO | High Water Breakdown: | 20260507 07:18:19.416 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:18:19.416 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:18:19.416 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:18:19.416 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:18:19.416 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:18:19.416 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:18:19.416 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:18:19.416 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:18:19.416 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:18:19.416 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:18:19.416 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:18:19.416 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:18:19.416 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:18:19.416 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:18:19.416 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:18:19.416 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:18:19.416 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:18:19.416 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:18:19.416 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:18:19.416 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:18:19.416 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:18:19.416 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:18:19.416 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:18:19.416 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:18:19.416 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:18:19.416 | INFO | ------------------------------------------ | 20260507 07:18:19.416 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:18:19.416 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:18:19.416 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:18:19.416 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:18:19.416 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 1.11e+00 | 20260507 07:18:19.416 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.02e-01, 1.05e-02 / 5.82e-03 | 20260507 07:18:19.416 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.45e+00, 4.96e-01 / 2.19e-01 | 20260507 07:18:19.416 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.49e+00, 1.66e-01 / 9.86e-02 | 20260507 07:18:19.416 | INFO | PerfReport: 16.26%, DME -- 3.99e+01 / 4.46e+01, 3.95e+00 / 1.56e+00 | 20260507 07:18:19.416 | INFO | PerfReport: 57.69%, FrameLabeling -- 1.77e+01 / 1.99e+01, 1.25e+00 / 1.45e+00 | 20260507 07:18:19.416 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.66e+00, 3.92e-01 / 2.50e-01 | 20260507 07:18:19.416 | INFO | PerfReport: 2.95%, Metrics -- 9.04e-01 / 1.17e+00, 2.42e-01 / 8.55e-02 | 20260507 07:18:19.416 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.62e-01, 5.10e-02 / 2.63e-02 | 20260507 07:18:19.416 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:18:19.416 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:18:19.416 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:18:19.416 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 07:18:19.416 | INFO | PerfReport: 21.90%, Upload -- 2.10e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260507 07:18:19.416 | INFO | PerfReport: 21.44%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260507 07:18:19.416 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 07:18:19.416 | INFO | PerfReport: 1.44%, LockUpload -- 1.39e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 07:18:19.416 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260507 07:18:19.416 | INFO | PerfReport: 21.96%, LockCompute -- 2.11e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 07:18:19.416 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260507 07:18:19.511 | INFO | Starting a round of Baz IO | 20260507 07:18:22.608 | INFO | Populating RTMetrics report | 20260507 07:18:22.608 | INFO | RTMetrics report populated | 20260507 07:18:22.608 | INFO | Opening tmp RTMetrics json file | 20260507 07:18:22.624 | INFO | Serializing RTMetrics to json | 20260507 07:18:22.625 | INFO | Serialization done, swapping tmp file now... | 20260507 07:18:22.625 | INFO | RTMetrics done | 20260507 07:18:22.694 | INFO | Finished a round of Baz IO: Wrote 1.647 GiB in 3.183 seconds | 20260507 07:18:22.695 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.57% / 0.00% / 67.43% | 20260507 07:18:22.695 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.108643% / 3.97364e-06% / 4.76837e-05% | 20260507 07:18:22.695 | INFO | Baz IO overall: | 20260507 07:18:22.695 | INFO | BytesWritten : 4068754475267 | 20260507 07:18:22.695 | INFO | NumEvents : 1948279380935 | 20260507 07:18:24.553 | INFO | Analyzing chunk frames = [8110080,8110592) | 20260507 07:18:24.553 | INFO | MemInfo: { | 20260507 07:18:24.553 | INFO | "gpu" : " 17 GB", | 20260507 07:18:24.553 | INFO | "huge" : "", | 20260507 07:18:24.553 | INFO | "huge_shared" : " 27 GB", | 20260507 07:18:24.553 | INFO | "malloc" : " 14 GB", | 20260507 07:18:24.553 | INFO | "pinned" : " 29 GB" | 20260507 07:18:24.553 | INFO | } | 20260507 07:18:24.553 | INFO | Perf report for the last 5.13883 seconds: | 20260507 07:18:24.553 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:18:24.553 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:24.553 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:24.553 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:24.553 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 07:18:24.553 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:24.553 | INFO | RT_METRICS : 42.9%, 1.0, 22.5ms, 0 | 20260507 07:18:24.553 | INFO | PRE_HQ : 59.4%, 1.9, 57.6ms, 0 | 20260507 07:18:24.553 | INFO | BAZWRITER : 61.5%, 3.2, 100.9ms, 0 | 20260507 07:18:24.553 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:18:24.553 | INFO | Uploaded 36GiB to GPU in 2.05454s (17.5221GiB/s) | 20260507 07:18:24.553 | INFO | Downloaded 38GiB to GPU in 2.18135s (17.4204GiB/s) | 20260507 07:18:29.690 | INFO | Analyzing chunk frames = [8110592,8111104) | 20260507 07:18:29.690 | INFO | MemInfo: { | 20260507 07:18:29.690 | INFO | "gpu" : " 17 GB", | 20260507 07:18:29.690 | INFO | "huge" : "", | 20260507 07:18:29.690 | INFO | "huge_shared" : " 27 GB", | 20260507 07:18:29.690 | INFO | "malloc" : " 14 GB", | 20260507 07:18:29.690 | INFO | "pinned" : " 29 GB" | 20260507 07:18:29.690 | INFO | } | 20260507 07:18:29.690 | INFO | Perf report for the last 5.13702 seconds: | 20260507 07:18:29.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:18:29.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:29.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:29.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:29.690 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:18:29.690 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:29.690 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:29.690 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 07:18:29.690 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:29.690 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:29.691 | INFO | Uploaded 36GiB to GPU in 1.97083s (18.2664GiB/s) | 20260507 07:18:29.691 | INFO | Downloaded 34GiB to GPU in 1.9314s (17.6038GiB/s) | 20260507 07:18:34.830 | INFO | Analyzing chunk frames = [8111104,8111616) | 20260507 07:18:34.830 | INFO | MemInfo: { | 20260507 07:18:34.830 | INFO | "gpu" : " 17 GB", | 20260507 07:18:34.830 | INFO | "huge" : "", | 20260507 07:18:34.830 | INFO | "huge_shared" : " 27 GB", | 20260507 07:18:34.830 | INFO | "malloc" : " 14 GB", | 20260507 07:18:34.830 | INFO | "pinned" : " 29 GB" | 20260507 07:18:34.830 | INFO | } | 20260507 07:18:34.830 | INFO | Perf report for the last 5.13974 seconds: | 20260507 07:18:34.830 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:18:34.830 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:34.830 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:34.830 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:34.830 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:18:34.830 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:34.830 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:34.830 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:18:34.830 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:34.830 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:34.830 | INFO | Uploaded 36GiB to GPU in 2.04554s (17.5992GiB/s) | 20260507 07:18:34.830 | INFO | Downloaded 34GiB to GPU in 1.95328s (17.4066GiB/s) | 20260507 07:18:39.963 | INFO | Analyzing chunk frames = [8111616,8112128) | 20260507 07:18:39.963 | INFO | MemInfo: { | 20260507 07:18:39.963 | INFO | "gpu" : " 17 GB", | 20260507 07:18:39.963 | INFO | "huge" : "", | 20260507 07:18:39.963 | INFO | "huge_shared" : " 27 GB", | 20260507 07:18:39.963 | INFO | "malloc" : " 14 GB", | 20260507 07:18:39.963 | INFO | "pinned" : " 29 GB" | 20260507 07:18:39.963 | INFO | } | 20260507 07:18:39.963 | INFO | Perf report for the last 5.1334 seconds: | 20260507 07:18:39.964 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:18:39.964 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:39.964 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:39.964 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:39.964 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 07:18:39.964 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:39.964 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:39.964 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:18:39.964 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:39.964 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:39.964 | INFO | Uploaded 36GiB to GPU in 2.00842s (17.9245GiB/s) | 20260507 07:18:39.964 | INFO | Downloaded 34GiB to GPU in 1.94666s (17.4658GiB/s) | 20260507 07:18:45.111 | INFO | Analyzing chunk frames = [8112128,8112640) | 20260507 07:18:45.111 | INFO | MemInfo: { | 20260507 07:18:45.111 | INFO | "gpu" : " 17 GB", | 20260507 07:18:45.111 | INFO | "huge" : "", | 20260507 07:18:45.111 | INFO | "huge_shared" : " 27 GB", | 20260507 07:18:45.111 | INFO | "malloc" : " 14 GB", | 20260507 07:18:45.111 | INFO | "pinned" : " 29 GB" | 20260507 07:18:45.111 | INFO | } | 20260507 07:18:45.111 | INFO | Perf report for the last 5.14786 seconds: | 20260507 07:18:45.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:18:45.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:45.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:45.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:45.111 | INFO | ANALYSIS : 59.8%, 3.0, 93.5ms, 0 | 20260507 07:18:45.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:45.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:45.111 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 07:18:45.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:45.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:45.111 | INFO | Uploaded 36GiB to GPU in 2.02193s (17.8047GiB/s) | 20260507 07:18:45.112 | INFO | Downloaded 34GiB to GPU in 1.94326s (17.4963GiB/s) | 20260507 07:18:50.254 | INFO | Analyzing chunk frames = [8112640,8113152) | 20260507 07:18:50.254 | INFO | MemInfo: { | 20260507 07:18:50.254 | INFO | "gpu" : " 17 GB", | 20260507 07:18:50.254 | INFO | "huge" : "", | 20260507 07:18:50.254 | INFO | "huge_shared" : " 27 GB", | 20260507 07:18:50.254 | INFO | "malloc" : " 14 GB", | 20260507 07:18:50.254 | INFO | "pinned" : " 29 GB" | 20260507 07:18:50.254 | INFO | } | 20260507 07:18:50.254 | INFO | Perf report for the last 5.14266 seconds: | 20260507 07:18:50.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:18:50.254 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:50.254 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:50.254 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:50.254 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 07:18:50.254 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:50.254 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:50.254 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:18:50.254 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:50.254 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:50.254 | INFO | Uploaded 36GiB to GPU in 2.06293s (17.4509GiB/s) | 20260507 07:18:50.254 | INFO | Downloaded 34GiB to GPU in 1.92615s (17.6518GiB/s) | 20260507 07:18:55.389 | INFO | Analyzing chunk frames = [8113152,8113664) | 20260507 07:18:55.389 | INFO | MemInfo: { | 20260507 07:18:55.389 | INFO | "gpu" : " 17 GB", | 20260507 07:18:55.389 | INFO | "huge" : "", | 20260507 07:18:55.389 | INFO | "huge_shared" : " 27 GB", | 20260507 07:18:55.389 | INFO | "malloc" : " 14 GB", | 20260507 07:18:55.389 | INFO | "pinned" : " 29 GB" | 20260507 07:18:55.389 | INFO | } | 20260507 07:18:55.389 | INFO | Perf report for the last 5.13564 seconds: | 20260507 07:18:55.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:18:55.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:55.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:55.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:55.390 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 07:18:55.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:55.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:55.390 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 07:18:55.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:55.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:18:55.390 | INFO | Uploaded 36GiB to GPU in 1.9052s (18.8957GiB/s) | 20260507 07:18:55.390 | INFO | Downloaded 34GiB to GPU in 1.98051s (17.1673GiB/s) | 20260507 07:19:00.520 | INFO | Analyzing chunk frames = [8113664,8114176) | 20260507 07:19:00.520 | INFO | MemInfo: { | 20260507 07:19:00.520 | INFO | "gpu" : " 17 GB", | 20260507 07:19:00.520 | INFO | "huge" : "", | 20260507 07:19:00.520 | INFO | "huge_shared" : " 27 GB", | 20260507 07:19:00.520 | INFO | "malloc" : " 14 GB", | 20260507 07:19:00.521 | INFO | "pinned" : " 29 GB" | 20260507 07:19:00.521 | INFO | } | 20260507 07:19:00.521 | INFO | Perf report for the last 5.13125 seconds: | 20260507 07:19:00.521 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:19:00.521 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:00.521 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:00.521 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:00.521 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 07:19:00.521 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:00.521 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:00.521 | INFO | PRE_HQ : 28.3%, 1.0, 14.6ms, 0 | 20260507 07:19:00.521 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:00.521 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:00.521 | INFO | Uploaded 36GiB to GPU in 1.99602s (18.0359GiB/s) | 20260507 07:19:00.521 | INFO | Downloaded 34GiB to GPU in 1.96059s (17.3417GiB/s) | 20260507 07:19:00.617 | INFO | Starting a round of Baz IO | 20260507 07:19:03.733 | INFO | Populating RTMetrics report | 20260507 07:19:03.734 | INFO | RTMetrics report populated | 20260507 07:19:03.734 | INFO | Opening tmp RTMetrics json file | 20260507 07:19:03.743 | INFO | Serializing RTMetrics to json | 20260507 07:19:03.748 | INFO | Serialization done, swapping tmp file now... | 20260507 07:19:03.748 | INFO | RTMetrics done | 20260507 07:19:03.817 | INFO | Finished a round of Baz IO: Wrote 1.647 GiB in 3.2 seconds | 20260507 07:19:03.817 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.57% / 0.00% / 67.43% | 20260507 07:19:03.818 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.108143% / 0% / 5.5631e-05% | 20260507 07:19:03.818 | INFO | Baz IO overall: | 20260507 07:19:03.818 | INFO | BytesWritten : 4070523227423 | 20260507 07:19:03.818 | INFO | NumEvents : 1949105868563 | 20260507 07:19:05.666 | INFO | Analyzing chunk frames = [8114176,8114688) | 20260507 07:19:05.666 | INFO | MemInfo: { | 20260507 07:19:05.666 | INFO | "gpu" : " 17 GB", | 20260507 07:19:05.666 | INFO | "huge" : "", | 20260507 07:19:05.666 | INFO | "huge_shared" : " 27 GB", | 20260507 07:19:05.666 | INFO | "malloc" : " 14 GB", | 20260507 07:19:05.666 | INFO | "pinned" : " 29 GB" | 20260507 07:19:05.666 | INFO | } | 20260507 07:19:05.667 | INFO | Perf report for the last 5.14576 seconds: | 20260507 07:19:05.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:19:05.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:05.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:05.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:05.667 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 07:19:05.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:05.667 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260507 07:19:05.667 | INFO | PRE_HQ : 60.2%, 1.9, 60.4ms, 0 | 20260507 07:19:05.667 | INFO | BAZWRITER : 61.8%, 3.3, 104.0ms, 0 | 20260507 07:19:05.667 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:19:05.667 | INFO | Uploaded 36GiB to GPU in 2.03476s (17.6925GiB/s) | 20260507 07:19:05.667 | INFO | Downloaded 38GiB to GPU in 2.17015s (17.5103GiB/s) | 20260507 07:19:10.805 | INFO | Analyzing chunk frames = [8114688,8115200) | 20260507 07:19:10.805 | INFO | MemInfo: { | 20260507 07:19:10.805 | INFO | "gpu" : " 17 GB", | 20260507 07:19:10.805 | INFO | "huge" : "", | 20260507 07:19:10.805 | INFO | "huge_shared" : " 27 GB", | 20260507 07:19:10.805 | INFO | "malloc" : " 14 GB", | 20260507 07:19:10.805 | INFO | "pinned" : " 29 GB" | 20260507 07:19:10.806 | INFO | } | 20260507 07:19:10.806 | INFO | Perf report for the last 5.13911 seconds: | 20260507 07:19:10.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:19:10.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:10.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:10.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:10.806 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 07:19:10.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:10.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:10.806 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:19:10.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:10.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:10.806 | INFO | Uploaded 36GiB to GPU in 2.00071s (17.9936GiB/s) | 20260507 07:19:10.806 | INFO | Downloaded 34GiB to GPU in 1.93136s (17.6042GiB/s) | 20260507 07:19:15.948 | INFO | Analyzing chunk frames = [8115200,8115712) | 20260507 07:19:15.948 | INFO | MemInfo: { | 20260507 07:19:15.948 | INFO | "gpu" : " 17 GB", | 20260507 07:19:15.948 | INFO | "huge" : "", | 20260507 07:19:15.948 | INFO | "huge_shared" : " 27 GB", | 20260507 07:19:15.948 | INFO | "malloc" : " 14 GB", | 20260507 07:19:15.948 | INFO | "pinned" : " 29 GB" | 20260507 07:19:15.948 | INFO | } | 20260507 07:19:15.948 | INFO | Perf report for the last 5.14203 seconds: | 20260507 07:19:15.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:19:15.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:15.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:15.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:15.948 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 07:19:15.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:15.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:15.948 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:19:15.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:15.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:15.948 | INFO | Uploaded 36GiB to GPU in 2.03617s (17.6802GiB/s) | 20260507 07:19:15.948 | INFO | Downloaded 34GiB to GPU in 1.94934s (17.4418GiB/s) | 20260507 07:19:21.092 | INFO | Analyzing chunk frames = [8115712,8116224) | 20260507 07:19:21.092 | INFO | MemInfo: { | 20260507 07:19:21.092 | INFO | "gpu" : " 17 GB", | 20260507 07:19:21.092 | INFO | "huge" : "", | 20260507 07:19:21.092 | INFO | "huge_shared" : " 27 GB", | 20260507 07:19:21.092 | INFO | "malloc" : " 14 GB", | 20260507 07:19:21.092 | INFO | "pinned" : " 29 GB" | 20260507 07:19:21.092 | INFO | } | 20260507 07:19:21.092 | INFO | Perf report for the last 5.14458 seconds: | 20260507 07:19:21.092 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:19:21.092 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:21.092 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:21.092 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:21.092 | INFO | ANALYSIS : 60.4%, 2.9, 93.4ms, 0 | 20260507 07:19:21.092 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:21.092 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:21.092 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:19:21.092 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:21.092 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:21.093 | INFO | Uploaded 36GiB to GPU in 2.06031s (17.4731GiB/s) | 20260507 07:19:21.093 | INFO | Downloaded 34GiB to GPU in 1.95186s (17.4193GiB/s) | 20260507 07:19:26.219 | INFO | Analyzing chunk frames = [8116224,8116736) | 20260507 07:19:26.219 | INFO | MemInfo: { | 20260507 07:19:26.219 | INFO | "gpu" : " 17 GB", | 20260507 07:19:26.219 | INFO | "huge" : "", | 20260507 07:19:26.219 | INFO | "huge_shared" : " 27 GB", | 20260507 07:19:26.219 | INFO | "malloc" : " 14 GB", | 20260507 07:19:26.219 | INFO | "pinned" : " 29 GB" | 20260507 07:19:26.219 | INFO | } | 20260507 07:19:26.219 | INFO | Perf report for the last 5.12652 seconds: | 20260507 07:19:26.219 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:19:26.219 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:26.219 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:26.219 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:26.219 | INFO | ANALYSIS : 60.3%, 3.0, 93.6ms, 0 | 20260507 07:19:26.219 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:26.219 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:26.219 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 07:19:26.219 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:26.219 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:26.219 | INFO | Uploaded 36GiB to GPU in 2.0564s (17.5063GiB/s) | 20260507 07:19:26.219 | INFO | Downloaded 34GiB to GPU in 1.90632s (17.8354GiB/s) | 20260507 07:19:31.359 | INFO | Analyzing chunk frames = [8116736,8117248) | 20260507 07:19:31.359 | INFO | MemInfo: { | 20260507 07:19:31.359 | INFO | "gpu" : " 17 GB", | 20260507 07:19:31.359 | INFO | "huge" : "", | 20260507 07:19:31.359 | INFO | "huge_shared" : " 27 GB", | 20260507 07:19:31.359 | INFO | "malloc" : " 14 GB", | 20260507 07:19:31.359 | INFO | "pinned" : " 29 GB" | 20260507 07:19:31.359 | INFO | } | 20260507 07:19:31.359 | INFO | Perf report for the last 5.14018 seconds: | 20260507 07:19:31.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:19:31.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:31.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:31.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:31.359 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 07:19:31.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:31.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:31.359 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:19:31.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:31.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:31.359 | INFO | Uploaded 36GiB to GPU in 2.04685s (17.588GiB/s) | 20260507 07:19:31.359 | INFO | Downloaded 34GiB to GPU in 1.91137s (17.7883GiB/s) | 20260507 07:19:36.503 | INFO | Analyzing chunk frames = [8117248,8117760) | 20260507 07:19:36.503 | INFO | MemInfo: { | 20260507 07:19:36.503 | INFO | "gpu" : " 17 GB", | 20260507 07:19:36.503 | INFO | "huge" : "", | 20260507 07:19:36.503 | INFO | "huge_shared" : " 27 GB", | 20260507 07:19:36.503 | INFO | "malloc" : " 14 GB", | 20260507 07:19:36.503 | INFO | "pinned" : " 29 GB" | 20260507 07:19:36.503 | INFO | } | 20260507 07:19:36.503 | INFO | Perf report for the last 5.14414 seconds: | 20260507 07:19:36.503 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:19:36.503 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:36.503 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:36.503 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:36.503 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 07:19:36.503 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:36.503 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:36.503 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 07:19:36.503 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:36.503 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:36.503 | INFO | Uploaded 36GiB to GPU in 1.99058s (18.0852GiB/s) | 20260507 07:19:36.503 | INFO | Downloaded 34GiB to GPU in 1.97544s (17.2114GiB/s) | 20260507 07:19:41.644 | INFO | Analyzing chunk frames = [8117760,8118272) | 20260507 07:19:41.644 | INFO | MemInfo: { | 20260507 07:19:41.644 | INFO | "gpu" : " 17 GB", | 20260507 07:19:41.644 | INFO | "huge" : "", | 20260507 07:19:41.644 | INFO | "huge_shared" : " 27 GB", | 20260507 07:19:41.644 | INFO | "malloc" : " 14 GB", | 20260507 07:19:41.644 | INFO | "pinned" : " 29 GB" | 20260507 07:19:41.644 | INFO | } | 20260507 07:19:41.644 | INFO | Perf report for the last 5.14077 seconds: | 20260507 07:19:41.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:19:41.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:41.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:41.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:41.644 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:19:41.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:41.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:41.644 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 07:19:41.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:41.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:41.644 | INFO | Uploaded 36GiB to GPU in 1.98563s (18.1303GiB/s) | 20260507 07:19:41.644 | INFO | Downloaded 34GiB to GPU in 1.96032s (17.3441GiB/s) | 20260507 07:19:41.644 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:19:41.644 | INFO | Total mem usage: 18 GiB | 20260507 07:19:41.644 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:19:41.644 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:19:41.644 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:19:41.644 | INFO | High Water Breakdown: | 20260507 07:19:41.644 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:19:41.644 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:19:41.644 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:19:41.644 | INFO | ------------------------------------------ | 20260507 07:19:41.644 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:19:41.644 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:19:41.644 | INFO | Total mem usage: 26.5 GiB | 20260507 07:19:41.644 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:19:41.644 | INFO | Recent High Water Marks Sum: 25.6 GiB (Savings: -3.59%) | 20260507 07:19:41.644 | INFO | Unused Cache: 1.64 GiB (Waste: 6.19%) | 20260507 07:19:41.644 | INFO | High Water Breakdown: | 20260507 07:19:41.644 | INFO | 56% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:19:41.645 | INFO | 36.3% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:19:41.645 | INFO | 3.66% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:19:41.645 | INFO | 2% : 523 MiB ( 99.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:19:41.645 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:19:41.645 | INFO | 0.557% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:19:41.645 | INFO | 0.00152% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:19:41.645 | INFO | 9.35e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:19:41.645 | INFO | 8.73e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:19:41.645 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:19:41.645 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:19:41.645 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:19:41.645 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:19:41.645 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:19:41.645 | INFO | ------------------------------------------ | 20260507 07:19:41.645 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:19:41.645 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:19:41.645 | INFO | Total mem usage: 25.6 GiB | 20260507 07:19:41.645 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:19:41.645 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:19:41.645 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:19:41.645 | INFO | High Water Breakdown: | 20260507 07:19:41.645 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:19:41.645 | INFO | ------------------------------------------ | 20260507 07:19:41.645 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:19:41.645 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:19:41.645 | INFO | Total mem usage: 15.7 GiB | 20260507 07:19:41.645 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:19:41.645 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:19:41.645 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:19:41.645 | INFO | High Water Breakdown: | 20260507 07:19:41.645 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:19:41.645 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:19:41.645 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:19:41.645 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:19:41.645 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:19:41.645 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:19:41.645 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:19:41.645 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:19:41.645 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:19:41.645 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:19:41.645 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:19:41.645 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:19:41.645 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:19:41.645 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:19:41.645 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:19:41.645 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:19:41.645 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:19:41.645 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:19:41.645 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:19:41.645 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:19:41.645 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:19:41.645 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:19:41.645 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:19:41.645 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:19:41.645 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:19:41.645 | INFO | ------------------------------------------ | 20260507 07:19:41.645 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:19:41.645 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:19:41.645 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:19:41.645 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:19:41.645 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.34e+01 / 9.82e-01 | 20260507 07:19:41.645 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.77e-03 / 5.57e-03 | 20260507 07:19:41.645 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.49e+00, 5.08e-01 / 2.55e-01 | 20260507 07:19:41.645 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.50e+00, 1.65e-01 / 9.57e-02 | 20260507 07:19:41.645 | INFO | PerfReport: 16.32%, DME -- 4.00e+01 / 4.50e+01, 3.93e+00 / 1.47e+00 | 20260507 07:19:41.645 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.21e+00 / 1.31e+00 | 20260507 07:19:41.645 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.12e+00 / 2.72e+00, 3.86e-01 / 1.70e-01 | 20260507 07:19:41.645 | INFO | PerfReport: 2.94%, Metrics -- 9.01e-01 / 1.21e+00, 2.38e-01 / 1.19e-01 | 20260507 07:19:41.645 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.04e-01 / 3.66e-01, 5.16e-02 / 2.43e-02 | 20260507 07:19:41.645 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:19:41.645 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:19:41.645 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:19:41.645 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 07:19:41.645 | INFO | PerfReport: 21.86%, Upload -- 2.10e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260507 07:19:41.645 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260507 07:19:41.645 | INFO | PerfReport: 31.90%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 07:19:41.645 | INFO | PerfReport: 1.55%, LockUpload -- 1.49e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 07:19:41.645 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260507 07:19:41.645 | INFO | PerfReport: 21.97%, LockCompute -- 2.11e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 07:19:41.645 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.14e-02 / 0.00e+00 | 20260507 07:19:41.741 | INFO | Starting a round of Baz IO | 20260507 07:19:44.832 | INFO | Populating RTMetrics report | 20260507 07:19:44.832 | INFO | RTMetrics report populated | 20260507 07:19:44.832 | INFO | Opening tmp RTMetrics json file | 20260507 07:19:44.848 | INFO | Serializing RTMetrics to json | 20260507 07:19:44.849 | INFO | Serialization done, swapping tmp file now... | 20260507 07:19:44.850 | INFO | RTMetrics done | 20260507 07:19:44.917 | INFO | Finished a round of Baz IO: Wrote 1.645 GiB in 3.175 seconds | 20260507 07:19:44.917 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.57% / 0.00% / 67.43% | 20260507 07:19:44.917 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.106847% / 0% / 6.35783e-05% | 20260507 07:19:44.917 | INFO | Baz IO overall: | 20260507 07:19:44.917 | INFO | BytesWritten : 4072289077734 | 20260507 07:19:44.917 | INFO | NumEvents : 1949930525992 | 20260507 07:19:46.780 | INFO | Analyzing chunk frames = [8118272,8118784) | 20260507 07:19:46.780 | INFO | MemInfo: { | 20260507 07:19:46.780 | INFO | "gpu" : " 17 GB", | 20260507 07:19:46.780 | INFO | "huge" : "", | 20260507 07:19:46.780 | INFO | "huge_shared" : " 27 GB", | 20260507 07:19:46.780 | INFO | "malloc" : " 14 GB", | 20260507 07:19:46.780 | INFO | "pinned" : " 29 GB" | 20260507 07:19:46.780 | INFO | } | 20260507 07:19:46.780 | INFO | Perf report for the last 5.13602 seconds: | 20260507 07:19:46.780 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:19:46.780 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:46.780 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:46.780 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:46.780 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 07:19:46.780 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:46.780 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260507 07:19:46.780 | INFO | PRE_HQ : 58.8%, 1.8, 54.6ms, 0 | 20260507 07:19:46.780 | INFO | BAZWRITER : 61.4%, 3.1, 98.0ms, 0 | 20260507 07:19:46.780 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:19:46.780 | INFO | Uploaded 36GiB to GPU in 2.05928s (17.4818GiB/s) | 20260507 07:19:46.780 | INFO | Downloaded 38GiB to GPU in 2.17015s (17.5103GiB/s) | 20260507 07:19:51.919 | INFO | Analyzing chunk frames = [8118784,8119296) | 20260507 07:19:51.919 | INFO | MemInfo: { | 20260507 07:19:51.919 | INFO | "gpu" : " 17 GB", | 20260507 07:19:51.919 | INFO | "huge" : "", | 20260507 07:19:51.919 | INFO | "huge_shared" : " 27 GB", | 20260507 07:19:51.919 | INFO | "malloc" : " 14 GB", | 20260507 07:19:51.919 | INFO | "pinned" : " 29 GB" | 20260507 07:19:51.919 | INFO | } | 20260507 07:19:51.919 | INFO | Perf report for the last 5.13882 seconds: | 20260507 07:19:51.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:19:51.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:51.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:51.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:51.919 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 07:19:51.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:51.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:51.919 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:19:51.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:51.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:51.919 | INFO | Uploaded 36GiB to GPU in 1.992s (18.0723GiB/s) | 20260507 07:19:51.919 | INFO | Downloaded 34GiB to GPU in 1.92961s (17.6201GiB/s) | 20260507 07:19:57.062 | INFO | Analyzing chunk frames = [8119296,8119808) | 20260507 07:19:57.062 | INFO | MemInfo: { | 20260507 07:19:57.062 | INFO | "gpu" : " 17 GB", | 20260507 07:19:57.062 | INFO | "huge" : "", | 20260507 07:19:57.062 | INFO | "huge_shared" : " 27 GB", | 20260507 07:19:57.062 | INFO | "malloc" : " 14 GB", | 20260507 07:19:57.062 | INFO | "pinned" : " 29 GB" | 20260507 07:19:57.062 | INFO | } | 20260507 07:19:57.062 | INFO | Perf report for the last 5.14322 seconds: | 20260507 07:19:57.062 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:19:57.062 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:57.062 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:57.062 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:57.062 | INFO | ANALYSIS : 60.2%, 3.0, 93.3ms, 0 | 20260507 07:19:57.062 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:57.062 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:57.062 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:19:57.062 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:57.062 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:19:57.062 | INFO | Uploaded 36GiB to GPU in 2.01213s (17.8915GiB/s) | 20260507 07:19:57.062 | INFO | Downloaded 34GiB to GPU in 1.954s (17.4002GiB/s) | 20260507 07:20:02.198 | INFO | Analyzing chunk frames = [8119808,8120320) | 20260507 07:20:02.198 | INFO | MemInfo: { | 20260507 07:20:02.198 | INFO | "gpu" : " 17 GB", | 20260507 07:20:02.198 | INFO | "huge" : "", | 20260507 07:20:02.198 | INFO | "huge_shared" : " 27 GB", | 20260507 07:20:02.198 | INFO | "malloc" : " 14 GB", | 20260507 07:20:02.198 | INFO | "pinned" : " 29 GB" | 20260507 07:20:02.198 | INFO | } | 20260507 07:20:02.198 | INFO | Perf report for the last 5.13624 seconds: | 20260507 07:20:02.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:20:02.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:02.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:02.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:02.198 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 07:20:02.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:02.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:02.198 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:20:02.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:02.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:02.198 | INFO | Uploaded 36GiB to GPU in 2.02953s (17.7381GiB/s) | 20260507 07:20:02.198 | INFO | Downloaded 34GiB to GPU in 1.95648s (17.3781GiB/s) | 20260507 07:20:07.335 | INFO | Analyzing chunk frames = [8120320,8120832) | 20260507 07:20:07.335 | INFO | MemInfo: { | 20260507 07:20:07.335 | INFO | "gpu" : " 17 GB", | 20260507 07:20:07.335 | INFO | "huge" : "", | 20260507 07:20:07.335 | INFO | "huge_shared" : " 27 GB", | 20260507 07:20:07.335 | INFO | "malloc" : " 14 GB", | 20260507 07:20:07.335 | INFO | "pinned" : " 29 GB" | 20260507 07:20:07.335 | INFO | } | 20260507 07:20:07.335 | INFO | Perf report for the last 5.13651 seconds: | 20260507 07:20:07.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:20:07.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:07.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:07.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:07.335 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 07:20:07.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:07.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:07.335 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:20:07.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:07.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:07.335 | INFO | Uploaded 36GiB to GPU in 2.03194s (17.7171GiB/s) | 20260507 07:20:07.335 | INFO | Downloaded 34GiB to GPU in 1.91004s (17.8007GiB/s) | 20260507 07:20:12.472 | INFO | Analyzing chunk frames = [8120832,8121344) | 20260507 07:20:12.472 | INFO | MemInfo: { | 20260507 07:20:12.472 | INFO | "gpu" : " 17 GB", | 20260507 07:20:12.472 | INFO | "huge" : "", | 20260507 07:20:12.472 | INFO | "huge_shared" : " 27 GB", | 20260507 07:20:12.472 | INFO | "malloc" : " 14 GB", | 20260507 07:20:12.472 | INFO | "pinned" : " 29 GB" | 20260507 07:20:12.472 | INFO | } | 20260507 07:20:12.472 | INFO | Perf report for the last 5.13737 seconds: | 20260507 07:20:12.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:20:12.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:12.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:12.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:12.472 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 07:20:12.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:12.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:12.472 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:20:12.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:12.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:12.472 | INFO | Uploaded 36GiB to GPU in 2.0443s (17.6099GiB/s) | 20260507 07:20:12.472 | INFO | Downloaded 34GiB to GPU in 1.91174s (17.7848GiB/s) | 20260507 07:20:17.620 | INFO | Analyzing chunk frames = [8121344,8121856) | 20260507 07:20:17.620 | INFO | MemInfo: { | 20260507 07:20:17.620 | INFO | "gpu" : " 17 GB", | 20260507 07:20:17.620 | INFO | "huge" : "", | 20260507 07:20:17.620 | INFO | "huge_shared" : " 27 GB", | 20260507 07:20:17.620 | INFO | "malloc" : " 14 GB", | 20260507 07:20:17.620 | INFO | "pinned" : " 29 GB" | 20260507 07:20:17.620 | INFO | } | 20260507 07:20:17.620 | INFO | Perf report for the last 5.14803 seconds: | 20260507 07:20:17.620 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:20:17.620 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:17.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:17.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:17.620 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:20:17.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:17.620 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:17.620 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 07:20:17.620 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:17.620 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:17.620 | INFO | Uploaded 36GiB to GPU in 1.95577s (18.4071GiB/s) | 20260507 07:20:17.621 | INFO | Downloaded 34GiB to GPU in 1.97911s (17.1794GiB/s) | 20260507 07:20:22.759 | INFO | Analyzing chunk frames = [8121856,8122368) | 20260507 07:20:22.759 | INFO | MemInfo: { | 20260507 07:20:22.759 | INFO | "gpu" : " 17 GB", | 20260507 07:20:22.759 | INFO | "huge" : "", | 20260507 07:20:22.759 | INFO | "huge_shared" : " 27 GB", | 20260507 07:20:22.759 | INFO | "malloc" : " 14 GB", | 20260507 07:20:22.759 | INFO | "pinned" : " 29 GB" | 20260507 07:20:22.759 | INFO | } | 20260507 07:20:22.759 | INFO | Perf report for the last 5.13895 seconds: | 20260507 07:20:22.759 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:20:22.759 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:22.759 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:22.759 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:22.759 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:20:22.759 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:22.759 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:22.759 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 07:20:22.759 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:22.759 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:22.759 | INFO | Uploaded 36GiB to GPU in 1.99071s (18.084GiB/s) | 20260507 07:20:22.759 | INFO | Downloaded 34GiB to GPU in 1.96064s (17.3413GiB/s) | 20260507 07:20:22.859 | INFO | Starting a round of Baz IO | 20260507 07:20:25.949 | INFO | Populating RTMetrics report | 20260507 07:20:25.949 | INFO | RTMetrics report populated | 20260507 07:20:25.949 | INFO | Opening tmp RTMetrics json file | 20260507 07:20:25.950 | INFO | Serializing RTMetrics to json | 20260507 07:20:25.953 | INFO | Serialization done, swapping tmp file now... | 20260507 07:20:25.959 | INFO | RTMetrics done | 20260507 07:20:26.037 | INFO | Finished a round of Baz IO: Wrote 1.645 GiB in 3.178 seconds | 20260507 07:20:26.037 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.57% / 0.00% / 67.43% | 20260507 07:20:26.037 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.107586% / 0% / 5.16574e-05% | 20260507 07:20:26.042 | INFO | Baz IO overall: | 20260507 07:20:26.042 | INFO | BytesWritten : 4074055666552 | 20260507 07:20:26.042 | INFO | NumEvents : 1950755973172 | 20260507 07:20:27.897 | INFO | Analyzing chunk frames = [8122368,8122880) | 20260507 07:20:27.897 | INFO | MemInfo: { | 20260507 07:20:27.897 | INFO | "gpu" : " 17 GB", | 20260507 07:20:27.897 | INFO | "huge" : "", | 20260507 07:20:27.898 | INFO | "huge_shared" : " 27 GB", | 20260507 07:20:27.898 | INFO | "malloc" : " 14 GB", | 20260507 07:20:27.898 | INFO | "pinned" : " 29 GB" | 20260507 07:20:27.898 | INFO | } | 20260507 07:20:27.898 | INFO | Perf report for the last 5.13845 seconds: | 20260507 07:20:27.898 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:20:27.898 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:27.898 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:27.898 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:27.898 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 07:20:27.898 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:27.898 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260507 07:20:27.898 | INFO | PRE_HQ : 58.8%, 1.8, 54.8ms, 0 | 20260507 07:20:27.898 | INFO | BAZWRITER : 61.5%, 3.1, 98.7ms, 0 | 20260507 07:20:27.898 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:20:27.898 | INFO | Uploaded 36GiB to GPU in 2.06078s (17.4691GiB/s) | 20260507 07:20:27.898 | INFO | Downloaded 38GiB to GPU in 2.16289s (17.5691GiB/s) | 20260507 07:20:33.031 | INFO | Analyzing chunk frames = [8122880,8123392) | 20260507 07:20:33.031 | INFO | MemInfo: { | 20260507 07:20:33.031 | INFO | "gpu" : " 17 GB", | 20260507 07:20:33.031 | INFO | "huge" : "", | 20260507 07:20:33.031 | INFO | "huge_shared" : " 27 GB", | 20260507 07:20:33.031 | INFO | "malloc" : " 14 GB", | 20260507 07:20:33.031 | INFO | "pinned" : " 29 GB" | 20260507 07:20:33.031 | INFO | } | 20260507 07:20:33.031 | INFO | Perf report for the last 5.13334 seconds: | 20260507 07:20:33.031 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:20:33.031 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:33.031 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:33.031 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:33.031 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:20:33.031 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:33.031 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:33.031 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:20:33.031 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:33.031 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:33.031 | INFO | Uploaded 36GiB to GPU in 1.98983s (18.092GiB/s) | 20260507 07:20:33.031 | INFO | Downloaded 34GiB to GPU in 1.93727s (17.5505GiB/s) | 20260507 07:20:38.172 | INFO | Analyzing chunk frames = [8123392,8123904) | 20260507 07:20:38.172 | INFO | MemInfo: { | 20260507 07:20:38.172 | INFO | "gpu" : " 17 GB", | 20260507 07:20:38.172 | INFO | "huge" : "", | 20260507 07:20:38.172 | INFO | "huge_shared" : " 27 GB", | 20260507 07:20:38.172 | INFO | "malloc" : " 14 GB", | 20260507 07:20:38.173 | INFO | "pinned" : " 29 GB" | 20260507 07:20:38.173 | INFO | } | 20260507 07:20:38.173 | INFO | Perf report for the last 5.14158 seconds: | 20260507 07:20:38.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:20:38.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:38.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:38.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:38.173 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:20:38.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:38.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:38.173 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:20:38.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:38.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:38.173 | INFO | Uploaded 36GiB to GPU in 2.0484s (17.5747GiB/s) | 20260507 07:20:38.173 | INFO | Downloaded 34GiB to GPU in 1.94751s (17.4582GiB/s) | 20260507 07:20:43.311 | INFO | Analyzing chunk frames = [8123904,8124416) | 20260507 07:20:43.311 | INFO | MemInfo: { | 20260507 07:20:43.311 | INFO | "gpu" : " 17 GB", | 20260507 07:20:43.311 | INFO | "huge" : "", | 20260507 07:20:43.311 | INFO | "huge_shared" : " 27 GB", | 20260507 07:20:43.311 | INFO | "malloc" : " 14 GB", | 20260507 07:20:43.311 | INFO | "pinned" : " 29 GB" | 20260507 07:20:43.311 | INFO | } | 20260507 07:20:43.311 | INFO | Perf report for the last 5.13881 seconds: | 20260507 07:20:43.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:20:43.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:43.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:43.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:43.311 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 07:20:43.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:43.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:43.312 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:20:43.312 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:43.312 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:43.312 | INFO | Uploaded 36GiB to GPU in 2.02675s (17.7625GiB/s) | 20260507 07:20:43.312 | INFO | Downloaded 34GiB to GPU in 1.93463s (17.5744GiB/s) | 20260507 07:20:48.457 | INFO | Analyzing chunk frames = [8124416,8124928) | 20260507 07:20:48.458 | INFO | MemInfo: { | 20260507 07:20:48.458 | INFO | "gpu" : " 17 GB", | 20260507 07:20:48.458 | INFO | "huge" : "", | 20260507 07:20:48.458 | INFO | "huge_shared" : " 27 GB", | 20260507 07:20:48.458 | INFO | "malloc" : " 14 GB", | 20260507 07:20:48.458 | INFO | "pinned" : " 29 GB" | 20260507 07:20:48.458 | INFO | } | 20260507 07:20:48.458 | INFO | Perf report for the last 5.14621 seconds: | 20260507 07:20:48.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:20:48.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:48.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:48.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:48.458 | INFO | ANALYSIS : 60.1%, 3.0, 94.0ms, 0 | 20260507 07:20:48.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:48.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:48.458 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:20:48.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:48.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:48.458 | INFO | Uploaded 36GiB to GPU in 2.05886s (17.4854GiB/s) | 20260507 07:20:48.458 | INFO | Downloaded 34GiB to GPU in 1.95193s (17.4186GiB/s) | 20260507 07:20:53.596 | INFO | Analyzing chunk frames = [8124928,8125440) | 20260507 07:20:53.596 | INFO | MemInfo: { | 20260507 07:20:53.596 | INFO | "gpu" : " 17 GB", | 20260507 07:20:53.596 | INFO | "huge" : "", | 20260507 07:20:53.596 | INFO | "huge_shared" : " 27 GB", | 20260507 07:20:53.596 | INFO | "malloc" : " 14 GB", | 20260507 07:20:53.596 | INFO | "pinned" : " 29 GB" | 20260507 07:20:53.596 | INFO | } | 20260507 07:20:53.596 | INFO | Perf report for the last 5.13876 seconds: | 20260507 07:20:53.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:20:53.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:53.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:53.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:53.596 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 07:20:53.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:53.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:53.596 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 07:20:53.597 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:53.597 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:53.597 | INFO | Uploaded 36GiB to GPU in 2.03976s (17.6491GiB/s) | 20260507 07:20:53.597 | INFO | Downloaded 34GiB to GPU in 1.91637s (17.7419GiB/s) | 20260507 07:20:58.728 | INFO | Analyzing chunk frames = [8125440,8125952) | 20260507 07:20:58.728 | INFO | MemInfo: { | 20260507 07:20:58.728 | INFO | "gpu" : " 17 GB", | 20260507 07:20:58.728 | INFO | "huge" : "", | 20260507 07:20:58.728 | INFO | "huge_shared" : " 27 GB", | 20260507 07:20:58.728 | INFO | "malloc" : " 14 GB", | 20260507 07:20:58.728 | INFO | "pinned" : " 29 GB" | 20260507 07:20:58.728 | INFO | } | 20260507 07:20:58.728 | INFO | Perf report for the last 5.13191 seconds: | 20260507 07:20:58.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:20:58.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:58.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:58.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:58.728 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260507 07:20:58.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:58.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:58.728 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 07:20:58.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:58.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:20:58.729 | INFO | Uploaded 36GiB to GPU in 1.92258s (18.7248GiB/s) | 20260507 07:20:58.729 | INFO | Downloaded 34GiB to GPU in 1.96988s (17.2599GiB/s) | 20260507 07:21:03.872 | INFO | Analyzing chunk frames = [8125952,8126464) | 20260507 07:21:03.872 | INFO | MemInfo: { | 20260507 07:21:03.872 | INFO | "gpu" : " 17 GB", | 20260507 07:21:03.872 | INFO | "huge" : "", | 20260507 07:21:03.872 | INFO | "huge_shared" : " 27 GB", | 20260507 07:21:03.872 | INFO | "malloc" : " 14 GB", | 20260507 07:21:03.872 | INFO | "pinned" : " 29 GB" | 20260507 07:21:03.872 | INFO | } | 20260507 07:21:03.872 | INFO | Perf report for the last 5.14368 seconds: | 20260507 07:21:03.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:21:03.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:03.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:03.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:03.872 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:21:03.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:03.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:03.872 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:21:03.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:03.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:03.872 | INFO | Uploaded 36GiB to GPU in 1.96809s (18.2919GiB/s) | 20260507 07:21:03.872 | INFO | Downloaded 34GiB to GPU in 1.95721s (17.3717GiB/s) | 20260507 07:21:03.872 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:21:03.872 | INFO | Total mem usage: 18 GiB | 20260507 07:21:03.872 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:21:03.872 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:21:03.872 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:21:03.873 | INFO | High Water Breakdown: | 20260507 07:21:03.873 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:21:03.873 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:21:03.873 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:21:03.873 | INFO | ------------------------------------------ | 20260507 07:21:03.873 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:21:03.873 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:21:03.873 | INFO | Total mem usage: 26.5 GiB | 20260507 07:21:03.873 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:21:03.873 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:21:03.873 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:21:03.873 | INFO | High Water Breakdown: | 20260507 07:21:03.873 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:21:03.873 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:21:03.873 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:21:03.873 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:21:03.873 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:21:03.873 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:21:03.873 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:21:03.873 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:21:03.873 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:21:03.873 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:21:03.873 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:21:03.873 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:21:03.873 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:21:03.873 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:21:03.873 | INFO | ------------------------------------------ | 20260507 07:21:03.873 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:21:03.873 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:21:03.873 | INFO | Total mem usage: 25.6 GiB | 20260507 07:21:03.873 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:21:03.873 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:21:03.873 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:21:03.873 | INFO | High Water Breakdown: | 20260507 07:21:03.873 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:21:03.873 | INFO | ------------------------------------------ | 20260507 07:21:03.873 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:21:03.873 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:21:03.873 | INFO | Total mem usage: 15.7 GiB | 20260507 07:21:03.873 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:21:03.873 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:21:03.873 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:21:03.873 | INFO | High Water Breakdown: | 20260507 07:21:03.873 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:21:03.873 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:21:03.873 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:21:03.873 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:21:03.873 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:21:03.873 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:21:03.873 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:21:03.873 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:21:03.873 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:21:03.873 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:21:03.873 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:21:03.873 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:21:03.873 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:21:03.873 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:21:03.873 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:21:03.873 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:21:03.873 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:21:03.873 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:21:03.873 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:21:03.873 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:21:03.873 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:21:03.873 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:21:03.873 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:21:03.873 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:21:03.873 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:21:03.873 | INFO | ------------------------------------------ | 20260507 07:21:03.873 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:21:03.873 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:21:03.873 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:21:03.873 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:21:03.873 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.35e+01 / 1.06e+00 | 20260507 07:21:03.873 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.02e-01, 9.58e-03 / 5.16e-03 | 20260507 07:21:03.874 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.46e+00, 5.11e-01 / 2.25e-01 | 20260507 07:21:03.874 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.49e+00, 1.69e-01 / 9.81e-02 | 20260507 07:21:03.874 | INFO | PerfReport: 16.31%, DME -- 4.01e+01 / 4.51e+01, 3.96e+00 / 1.75e+00 | 20260507 07:21:03.874 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.30e+00 / 1.51e+00 | 20260507 07:21:03.874 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.72e+00, 3.85e-01 / 1.86e-01 | 20260507 07:21:03.874 | INFO | PerfReport: 2.94%, Metrics -- 9.02e-01 / 1.20e+00, 2.42e-01 / 1.23e-01 | 20260507 07:21:03.874 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.03e-01 / 3.61e-01, 5.12e-02 / 2.25e-02 | 20260507 07:21:03.874 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:21:03.874 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:21:03.874 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:21:03.874 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 07:21:03.874 | INFO | PerfReport: 21.84%, Upload -- 2.10e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260507 07:21:03.874 | INFO | PerfReport: 21.38%, Download -- 2.05e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 07:21:03.874 | INFO | PerfReport: 31.98%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 07:21:03.874 | INFO | PerfReport: 1.52%, LockUpload -- 1.46e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 07:21:03.874 | INFO | PerfReport: 1.27%, LockDownload -- 1.22e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260507 07:21:03.874 | INFO | PerfReport: 21.99%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 07:21:03.874 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.98e-02 / 0.00e+00 | 20260507 07:21:03.969 | INFO | Starting a round of Baz IO | 20260507 07:21:07.067 | INFO | Populating RTMetrics report | 20260507 07:21:07.067 | INFO | RTMetrics report populated | 20260507 07:21:07.067 | INFO | Opening tmp RTMetrics json file | 20260507 07:21:07.081 | INFO | Serializing RTMetrics to json | 20260507 07:21:07.082 | INFO | Serialization done, swapping tmp file now... | 20260507 07:21:07.083 | INFO | RTMetrics done | 20260507 07:21:07.148 | INFO | Finished a round of Baz IO: Wrote 1.645 GiB in 3.179 seconds | 20260507 07:21:07.148 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.57% / 0.00% / 67.43% | 20260507 07:21:07.148 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.107781% / 3.97364e-06% / 6.75519e-05% | 20260507 07:21:07.148 | INFO | Baz IO overall: | 20260507 07:21:07.148 | INFO | BytesWritten : 4075821867096 | 20260507 07:21:07.148 | INFO | NumEvents : 1951580980108 | 20260507 07:21:09.011 | INFO | Analyzing chunk frames = [8126464,8126976) | 20260507 07:21:09.011 | INFO | MemInfo: { | 20260507 07:21:09.011 | INFO | "gpu" : " 17 GB", | 20260507 07:21:09.011 | INFO | "huge" : "", | 20260507 07:21:09.011 | INFO | "huge_shared" : " 27 GB", | 20260507 07:21:09.011 | INFO | "malloc" : " 14 GB", | 20260507 07:21:09.011 | INFO | "pinned" : " 29 GB" | 20260507 07:21:09.011 | INFO | } | 20260507 07:21:09.011 | INFO | Perf report for the last 5.13892 seconds: | 20260507 07:21:09.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:21:09.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:09.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:09.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:09.011 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 07:21:09.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:09.011 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260507 07:21:09.011 | INFO | PRE_HQ : 58.9%, 1.8, 54.4ms, 0 | 20260507 07:21:09.011 | INFO | BAZWRITER : 61.5%, 3.1, 97.7ms, 0 | 20260507 07:21:09.011 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:21:09.011 | INFO | Uploaded 36GiB to GPU in 2.05557s (17.5134GiB/s) | 20260507 07:21:09.011 | INFO | Downloaded 38GiB to GPU in 2.17511s (17.4704GiB/s) | 20260507 07:21:14.152 | INFO | Analyzing chunk frames = [8126976,8127488) | 20260507 07:21:14.152 | INFO | MemInfo: { | 20260507 07:21:14.152 | INFO | "gpu" : " 17 GB", | 20260507 07:21:14.152 | INFO | "huge" : "", | 20260507 07:21:14.152 | INFO | "huge_shared" : " 27 GB", | 20260507 07:21:14.152 | INFO | "malloc" : " 14 GB", | 20260507 07:21:14.152 | INFO | "pinned" : " 29 GB" | 20260507 07:21:14.152 | INFO | } | 20260507 07:21:14.152 | INFO | Perf report for the last 5.14068 seconds: | 20260507 07:21:14.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:21:14.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:14.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:14.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:14.152 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:21:14.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:14.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:14.152 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 07:21:14.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:14.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:14.152 | INFO | Uploaded 36GiB to GPU in 1.99212s (18.0712GiB/s) | 20260507 07:21:14.152 | INFO | Downloaded 34GiB to GPU in 1.93206s (17.5978GiB/s) | 20260507 07:21:19.283 | INFO | Analyzing chunk frames = [8127488,8128000) | 20260507 07:21:19.283 | INFO | MemInfo: { | 20260507 07:21:19.283 | INFO | "gpu" : " 17 GB", | 20260507 07:21:19.283 | INFO | "huge" : "", | 20260507 07:21:19.283 | INFO | "huge_shared" : " 27 GB", | 20260507 07:21:19.283 | INFO | "malloc" : " 14 GB", | 20260507 07:21:19.283 | INFO | "pinned" : " 29 GB" | 20260507 07:21:19.283 | INFO | } | 20260507 07:21:19.283 | INFO | Perf report for the last 5.13117 seconds: | 20260507 07:21:19.283 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:21:19.283 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:19.283 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:19.283 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:19.283 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:21:19.283 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:19.283 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:19.283 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 07:21:19.283 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:19.283 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:19.283 | INFO | Uploaded 36GiB to GPU in 2.02641s (17.7655GiB/s) | 20260507 07:21:19.283 | INFO | Downloaded 34GiB to GPU in 1.9142s (17.7619GiB/s) | 20260507 07:21:24.427 | INFO | Analyzing chunk frames = [8128000,8128512) | 20260507 07:21:24.427 | INFO | MemInfo: { | 20260507 07:21:24.427 | INFO | "gpu" : " 17 GB", | 20260507 07:21:24.427 | INFO | "huge" : "", | 20260507 07:21:24.427 | INFO | "huge_shared" : " 27 GB", | 20260507 07:21:24.427 | INFO | "malloc" : " 14 GB", | 20260507 07:21:24.427 | INFO | "pinned" : " 29 GB" | 20260507 07:21:24.427 | INFO | } | 20260507 07:21:24.427 | INFO | Perf report for the last 5.14387 seconds: | 20260507 07:21:24.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:21:24.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:24.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:24.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:24.427 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 07:21:24.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:24.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:24.427 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:21:24.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:24.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:24.427 | INFO | Uploaded 36GiB to GPU in 1.99187s (18.0735GiB/s) | 20260507 07:21:24.427 | INFO | Downloaded 34GiB to GPU in 1.91889s (17.7186GiB/s) | 20260507 07:21:29.572 | INFO | Analyzing chunk frames = [8128512,8129024) | 20260507 07:21:29.572 | INFO | MemInfo: { | 20260507 07:21:29.572 | INFO | "gpu" : " 17 GB", | 20260507 07:21:29.572 | INFO | "huge" : "", | 20260507 07:21:29.572 | INFO | "huge_shared" : " 27 GB", | 20260507 07:21:29.572 | INFO | "malloc" : " 14 GB", | 20260507 07:21:29.572 | INFO | "pinned" : " 29 GB" | 20260507 07:21:29.572 | INFO | } | 20260507 07:21:29.572 | INFO | Perf report for the last 5.14549 seconds: | 20260507 07:21:29.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:21:29.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:29.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:29.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:29.572 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 07:21:29.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:29.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:29.572 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:21:29.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:29.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:29.572 | INFO | Uploaded 36GiB to GPU in 2.05906s (17.4837GiB/s) | 20260507 07:21:29.573 | INFO | Downloaded 34GiB to GPU in 1.95136s (17.4237GiB/s) | 20260507 07:21:34.713 | INFO | Analyzing chunk frames = [8129024,8129536) | 20260507 07:21:34.713 | INFO | MemInfo: { | 20260507 07:21:34.713 | INFO | "gpu" : " 17 GB", | 20260507 07:21:34.713 | INFO | "huge" : "", | 20260507 07:21:34.713 | INFO | "huge_shared" : " 27 GB", | 20260507 07:21:34.713 | INFO | "malloc" : " 14 GB", | 20260507 07:21:34.713 | INFO | "pinned" : " 29 GB" | 20260507 07:21:34.713 | INFO | } | 20260507 07:21:34.713 | INFO | Perf report for the last 5.141 seconds: | 20260507 07:21:34.713 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:21:34.713 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:34.713 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:34.713 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:34.713 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 07:21:34.713 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:34.713 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:34.713 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 07:21:34.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:34.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:34.714 | INFO | Uploaded 36GiB to GPU in 2.07809s (17.3236GiB/s) | 20260507 07:21:34.714 | INFO | Downloaded 34GiB to GPU in 1.94165s (17.5109GiB/s) | 20260507 07:21:39.845 | INFO | Analyzing chunk frames = [8129536,8130048) | 20260507 07:21:39.845 | INFO | MemInfo: { | 20260507 07:21:39.845 | INFO | "gpu" : " 17 GB", | 20260507 07:21:39.845 | INFO | "huge" : "", | 20260507 07:21:39.845 | INFO | "huge_shared" : " 27 GB", | 20260507 07:21:39.845 | INFO | "malloc" : " 14 GB", | 20260507 07:21:39.845 | INFO | "pinned" : " 29 GB" | 20260507 07:21:39.845 | INFO | } | 20260507 07:21:39.845 | INFO | Perf report for the last 5.13157 seconds: | 20260507 07:21:39.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:21:39.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:39.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:39.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:39.845 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 07:21:39.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:39.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:39.845 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:21:39.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:39.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:39.845 | INFO | Uploaded 36GiB to GPU in 1.96447s (18.3256GiB/s) | 20260507 07:21:39.845 | INFO | Downloaded 34GiB to GPU in 1.98737s (17.1081GiB/s) | 20260507 07:21:44.980 | INFO | Analyzing chunk frames = [8130048,8130560) | 20260507 07:21:44.980 | INFO | MemInfo: { | 20260507 07:21:44.980 | INFO | "gpu" : " 17 GB", | 20260507 07:21:44.980 | INFO | "huge" : "", | 20260507 07:21:44.980 | INFO | "huge_shared" : " 27 GB", | 20260507 07:21:44.981 | INFO | "malloc" : " 14 GB", | 20260507 07:21:44.981 | INFO | "pinned" : " 29 GB" | 20260507 07:21:44.981 | INFO | } | 20260507 07:21:44.981 | INFO | Perf report for the last 5.13572 seconds: | 20260507 07:21:44.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:21:44.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:44.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:44.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:44.981 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:21:44.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:44.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:44.981 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:21:44.981 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:44.981 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:44.981 | INFO | Uploaded 36GiB to GPU in 1.98388s (18.1463GiB/s) | 20260507 07:21:44.981 | INFO | Downloaded 34GiB to GPU in 1.96097s (17.3383GiB/s) | 20260507 07:21:45.078 | INFO | Starting a round of Baz IO | 20260507 07:21:48.172 | INFO | Populating RTMetrics report | 20260507 07:21:48.172 | INFO | RTMetrics report populated | 20260507 07:21:48.172 | INFO | Opening tmp RTMetrics json file | 20260507 07:21:48.180 | INFO | Serializing RTMetrics to json | 20260507 07:21:48.189 | INFO | Serialization done, swapping tmp file now... | 20260507 07:21:48.189 | INFO | RTMetrics done | 20260507 07:21:48.254 | INFO | Finished a round of Baz IO: Wrote 1.644 GiB in 3.176 seconds | 20260507 07:21:48.254 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.56% / 0.00% / 67.43% | 20260507 07:21:48.254 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.106831% / 0% / 7.15256e-05% | 20260507 07:21:48.254 | INFO | Baz IO overall: | 20260507 07:21:48.255 | INFO | BytesWritten : 4077586612386 | 20260507 07:21:48.255 | INFO | NumEvents : 1952405063024 | 20260507 07:21:50.125 | INFO | Analyzing chunk frames = [8130560,8131072) | 20260507 07:21:50.125 | INFO | MemInfo: { | 20260507 07:21:50.125 | INFO | "gpu" : " 17 GB", | 20260507 07:21:50.125 | INFO | "huge" : "", | 20260507 07:21:50.125 | INFO | "huge_shared" : " 27 GB", | 20260507 07:21:50.125 | INFO | "malloc" : " 14 GB", | 20260507 07:21:50.125 | INFO | "pinned" : " 29 GB" | 20260507 07:21:50.125 | INFO | } | 20260507 07:21:50.125 | INFO | Perf report for the last 5.14412 seconds: | 20260507 07:21:50.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:21:50.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:50.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:50.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:50.125 | INFO | ANALYSIS : 61.2%, 3.0, 95.7ms, 0 | 20260507 07:21:50.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:50.125 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 07:21:50.125 | INFO | PRE_HQ : 58.7%, 1.8, 54.7ms, 0 | 20260507 07:21:50.125 | INFO | BAZWRITER : 61.3%, 3.1, 97.5ms, 0 | 20260507 07:21:50.125 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:21:50.125 | INFO | Uploaded 36GiB to GPU in 2.03973s (17.6494GiB/s) | 20260507 07:21:50.125 | INFO | Downloaded 38GiB to GPU in 2.16265s (17.571GiB/s) | 20260507 07:21:55.265 | INFO | Analyzing chunk frames = [8131072,8131584) | 20260507 07:21:55.265 | INFO | MemInfo: { | 20260507 07:21:55.265 | INFO | "gpu" : " 17 GB", | 20260507 07:21:55.265 | INFO | "huge" : "", | 20260507 07:21:55.265 | INFO | "huge_shared" : " 27 GB", | 20260507 07:21:55.265 | INFO | "malloc" : " 14 GB", | 20260507 07:21:55.265 | INFO | "pinned" : " 29 GB" | 20260507 07:21:55.265 | INFO | } | 20260507 07:21:55.265 | INFO | Perf report for the last 5.14061 seconds: | 20260507 07:21:55.265 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:21:55.265 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:55.265 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:55.265 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:55.265 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 07:21:55.265 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:55.265 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:55.265 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:21:55.265 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:55.265 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:21:55.266 | INFO | Uploaded 36GiB to GPU in 1.99218s (18.0707GiB/s) | 20260507 07:21:55.266 | INFO | Downloaded 34GiB to GPU in 1.92869s (17.6286GiB/s) | 20260507 07:22:00.408 | INFO | Analyzing chunk frames = [8131584,8132096) | 20260507 07:22:00.408 | INFO | MemInfo: { | 20260507 07:22:00.408 | INFO | "gpu" : " 17 GB", | 20260507 07:22:00.408 | INFO | "huge" : "", | 20260507 07:22:00.408 | INFO | "huge_shared" : " 27 GB", | 20260507 07:22:00.408 | INFO | "malloc" : " 14 GB", | 20260507 07:22:00.408 | INFO | "pinned" : " 29 GB" | 20260507 07:22:00.408 | INFO | } | 20260507 07:22:00.408 | INFO | Perf report for the last 5.14236 seconds: | 20260507 07:22:00.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:22:00.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:00.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:00.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:00.408 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 07:22:00.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:00.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:00.408 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:22:00.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:00.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:00.408 | INFO | Uploaded 36GiB to GPU in 2.05111s (17.5515GiB/s) | 20260507 07:22:00.408 | INFO | Downloaded 34GiB to GPU in 1.9512s (17.4251GiB/s) | 20260507 07:22:05.544 | INFO | Analyzing chunk frames = [8132096,8132608) | 20260507 07:22:05.544 | INFO | MemInfo: { | 20260507 07:22:05.544 | INFO | "gpu" : " 17 GB", | 20260507 07:22:05.544 | INFO | "huge" : "", | 20260507 07:22:05.544 | INFO | "huge_shared" : " 27 GB", | 20260507 07:22:05.544 | INFO | "malloc" : " 14 GB", | 20260507 07:22:05.544 | INFO | "pinned" : " 29 GB" | 20260507 07:22:05.544 | INFO | } | 20260507 07:22:05.544 | INFO | Perf report for the last 5.13662 seconds: | 20260507 07:22:05.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:22:05.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:05.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:05.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:05.544 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 07:22:05.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:05.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:05.544 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:22:05.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:05.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:05.545 | INFO | Uploaded 36GiB to GPU in 2.06009s (17.475GiB/s) | 20260507 07:22:05.545 | INFO | Downloaded 34GiB to GPU in 1.94901s (17.4447GiB/s) | 20260507 07:22:10.687 | INFO | Analyzing chunk frames = [8132608,8133120) | 20260507 07:22:10.687 | INFO | MemInfo: { | 20260507 07:22:10.687 | INFO | "gpu" : " 17 GB", | 20260507 07:22:10.687 | INFO | "huge" : "", | 20260507 07:22:10.687 | INFO | "huge_shared" : " 27 GB", | 20260507 07:22:10.687 | INFO | "malloc" : " 14 GB", | 20260507 07:22:10.687 | INFO | "pinned" : " 29 GB" | 20260507 07:22:10.687 | INFO | } | 20260507 07:22:10.688 | INFO | Perf report for the last 5.14317 seconds: | 20260507 07:22:10.688 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:22:10.688 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:10.688 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:10.688 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:10.688 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 07:22:10.688 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:10.688 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:10.688 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:22:10.688 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:10.688 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:10.688 | INFO | Uploaded 36GiB to GPU in 2.04361s (17.6159GiB/s) | 20260507 07:22:10.688 | INFO | Downloaded 34GiB to GPU in 1.9155s (17.7499GiB/s) | 20260507 07:22:15.825 | INFO | Analyzing chunk frames = [8133120,8133632) | 20260507 07:22:15.825 | INFO | MemInfo: { | 20260507 07:22:15.825 | INFO | "gpu" : " 17 GB", | 20260507 07:22:15.825 | INFO | "huge" : "", | 20260507 07:22:15.825 | INFO | "huge_shared" : " 27 GB", | 20260507 07:22:15.825 | INFO | "malloc" : " 14 GB", | 20260507 07:22:15.825 | INFO | "pinned" : " 29 GB" | 20260507 07:22:15.825 | INFO | } | 20260507 07:22:15.825 | INFO | Perf report for the last 5.13717 seconds: | 20260507 07:22:15.825 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:22:15.825 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:15.825 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:15.825 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:15.825 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 07:22:15.825 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:15.825 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:15.825 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 07:22:15.825 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:15.825 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:15.825 | INFO | Uploaded 36GiB to GPU in 2.06199s (17.4588GiB/s) | 20260507 07:22:15.825 | INFO | Downloaded 34GiB to GPU in 1.91747s (17.7317GiB/s) | 20260507 07:22:20.962 | INFO | Analyzing chunk frames = [8133632,8134144) | 20260507 07:22:20.962 | INFO | MemInfo: { | 20260507 07:22:20.962 | INFO | "gpu" : " 17 GB", | 20260507 07:22:20.962 | INFO | "huge" : "", | 20260507 07:22:20.962 | INFO | "huge_shared" : " 27 GB", | 20260507 07:22:20.962 | INFO | "malloc" : " 14 GB", | 20260507 07:22:20.962 | INFO | "pinned" : " 29 GB" | 20260507 07:22:20.962 | INFO | } | 20260507 07:22:20.962 | INFO | Perf report for the last 5.13706 seconds: | 20260507 07:22:20.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:22:20.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:20.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:20.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:20.962 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:22:20.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:20.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:20.962 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 07:22:20.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:20.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:20.962 | INFO | Uploaded 36GiB to GPU in 1.93887s (18.5675GiB/s) | 20260507 07:22:20.962 | INFO | Downloaded 34GiB to GPU in 1.96739s (17.2818GiB/s) | 20260507 07:22:26.099 | INFO | Analyzing chunk frames = [8134144,8134656) | 20260507 07:22:26.099 | INFO | MemInfo: { | 20260507 07:22:26.099 | INFO | "gpu" : " 17 GB", | 20260507 07:22:26.099 | INFO | "huge" : "", | 20260507 07:22:26.099 | INFO | "huge_shared" : " 27 GB", | 20260507 07:22:26.099 | INFO | "malloc" : " 14 GB", | 20260507 07:22:26.099 | INFO | "pinned" : " 29 GB" | 20260507 07:22:26.099 | INFO | } | 20260507 07:22:26.099 | INFO | Perf report for the last 5.1371 seconds: | 20260507 07:22:26.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:22:26.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:26.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:26.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:26.099 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:22:26.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:26.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:26.099 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:22:26.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:26.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:26.099 | INFO | Uploaded 36GiB to GPU in 1.97951s (18.1863GiB/s) | 20260507 07:22:26.099 | INFO | Downloaded 34GiB to GPU in 1.96096s (17.3385GiB/s) | 20260507 07:22:26.099 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:22:26.099 | INFO | Total mem usage: 18 GiB | 20260507 07:22:26.099 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:22:26.099 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:22:26.099 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:22:26.099 | INFO | High Water Breakdown: | 20260507 07:22:26.099 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:22:26.099 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:22:26.099 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:22:26.099 | INFO | ------------------------------------------ | 20260507 07:22:26.099 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:22:26.099 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:22:26.099 | INFO | Total mem usage: 26.5 GiB | 20260507 07:22:26.099 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:22:26.099 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:22:26.099 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:22:26.099 | INFO | High Water Breakdown: | 20260507 07:22:26.099 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:22:26.099 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:22:26.099 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:22:26.099 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:22:26.099 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:22:26.099 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:22:26.099 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:22:26.100 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:22:26.100 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:22:26.100 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:22:26.100 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:22:26.100 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:22:26.100 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:22:26.100 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:22:26.100 | INFO | ------------------------------------------ | 20260507 07:22:26.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:22:26.100 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:22:26.100 | INFO | Total mem usage: 25.6 GiB | 20260507 07:22:26.100 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:22:26.100 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:22:26.100 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:22:26.100 | INFO | High Water Breakdown: | 20260507 07:22:26.100 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:22:26.100 | INFO | ------------------------------------------ | 20260507 07:22:26.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:22:26.100 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:22:26.100 | INFO | Total mem usage: 15.7 GiB | 20260507 07:22:26.100 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:22:26.100 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:22:26.100 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:22:26.100 | INFO | High Water Breakdown: | 20260507 07:22:26.100 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:22:26.100 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:22:26.100 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:22:26.100 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:22:26.100 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:22:26.100 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:22:26.100 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:22:26.100 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:22:26.100 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:22:26.100 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:22:26.100 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:22:26.100 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:22:26.100 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:22:26.100 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:22:26.100 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:22:26.100 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:22:26.100 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:22:26.100 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:22:26.100 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:22:26.100 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:22:26.100 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:22:26.100 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:22:26.100 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:22:26.100 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:22:26.100 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:22:26.100 | INFO | ------------------------------------------ | 20260507 07:22:26.100 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:22:26.100 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:22:26.100 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:22:26.100 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:22:26.100 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.34e+01 / 1.09e+00 | 20260507 07:22:26.100 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 9.72e-03 / 3.52e-03 | 20260507 07:22:26.100 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.51e+00, 5.05e-01 / 1.65e-01 | 20260507 07:22:26.100 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.50e+00, 1.69e-01 / 8.74e-02 | 20260507 07:22:26.100 | INFO | PerfReport: 16.28%, DME -- 4.00e+01 / 4.46e+01, 3.87e+00 / 1.18e+00 | 20260507 07:22:26.100 | INFO | PerfReport: 57.71%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.28e+00 / 1.09e+00 | 20260507 07:22:26.100 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.68e+00, 3.83e-01 / 2.07e-01 | 20260507 07:22:26.100 | INFO | PerfReport: 2.94%, Metrics -- 9.02e-01 / 1.21e+00, 2.43e-01 / 1.39e-01 | 20260507 07:22:26.100 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.02e-01 / 3.64e-01, 5.07e-02 / 1.90e-02 | 20260507 07:22:26.100 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:22:26.100 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:22:26.100 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:22:26.100 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 07:22:26.100 | INFO | PerfReport: 21.88%, Upload -- 2.10e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260507 07:22:26.100 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 07:22:26.100 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 07:22:26.100 | INFO | PerfReport: 1.64%, LockUpload -- 1.58e+00 / 0.00e+00, 5.03e+00 / 0.00e+00 | 20260507 07:22:26.100 | INFO | PerfReport: 1.15%, LockDownload -- 1.11e+00 / 0.00e+00, 2.77e+00 / 0.00e+00 | 20260507 07:22:26.100 | INFO | PerfReport: 22.04%, LockCompute -- 2.12e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 07:22:26.100 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260507 07:22:26.198 | INFO | Starting a round of Baz IO | 20260507 07:22:29.293 | INFO | Populating RTMetrics report | 20260507 07:22:29.293 | INFO | RTMetrics report populated | 20260507 07:22:29.293 | INFO | Opening tmp RTMetrics json file | 20260507 07:22:29.308 | INFO | Serializing RTMetrics to json | 20260507 07:22:29.309 | INFO | Serialization done, swapping tmp file now... | 20260507 07:22:29.309 | INFO | RTMetrics done | 20260507 07:22:29.382 | INFO | Finished a round of Baz IO: Wrote 1.644 GiB in 3.183 seconds | 20260507 07:22:29.382 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.56% / 0.00% / 67.44% | 20260507 07:22:29.382 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.106831% / 0% / 5.96046e-05% | 20260507 07:22:29.382 | INFO | Baz IO overall: | 20260507 07:22:29.382 | INFO | BytesWritten : 4079351584354 | 20260507 07:22:29.382 | INFO | NumEvents : 1953229604873 | 20260507 07:22:31.235 | INFO | Analyzing chunk frames = [8134656,8135168) | 20260507 07:22:31.235 | INFO | MemInfo: { | 20260507 07:22:31.235 | INFO | "gpu" : " 17 GB", | 20260507 07:22:31.235 | INFO | "huge" : "", | 20260507 07:22:31.235 | INFO | "huge_shared" : " 27 GB", | 20260507 07:22:31.235 | INFO | "malloc" : " 14 GB", | 20260507 07:22:31.235 | INFO | "pinned" : " 29 GB" | 20260507 07:22:31.235 | INFO | } | 20260507 07:22:31.236 | INFO | Perf report for the last 5.13663 seconds: | 20260507 07:22:31.236 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:22:31.236 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:31.236 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:31.236 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:31.236 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 07:22:31.236 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:31.236 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 07:22:31.236 | INFO | PRE_HQ : 59.2%, 1.8, 55.0ms, 0 | 20260507 07:22:31.236 | INFO | BAZWRITER : 61.6%, 3.1, 98.3ms, 0 | 20260507 07:22:31.236 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:22:31.236 | INFO | Uploaded 36GiB to GPU in 2.05492s (17.5189GiB/s) | 20260507 07:22:31.236 | INFO | Downloaded 38GiB to GPU in 2.1839s (17.4001GiB/s) | 20260507 07:22:36.379 | INFO | Analyzing chunk frames = [8135168,8135680) | 20260507 07:22:36.379 | INFO | MemInfo: { | 20260507 07:22:36.379 | INFO | "gpu" : " 17 GB", | 20260507 07:22:36.379 | INFO | "huge" : "", | 20260507 07:22:36.379 | INFO | "huge_shared" : " 27 GB", | 20260507 07:22:36.379 | INFO | "malloc" : " 14 GB", | 20260507 07:22:36.379 | INFO | "pinned" : " 29 GB" | 20260507 07:22:36.379 | INFO | } | 20260507 07:22:36.379 | INFO | Perf report for the last 5.14385 seconds: | 20260507 07:22:36.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:22:36.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:36.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:36.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:36.379 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 07:22:36.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:36.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:36.379 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:22:36.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:36.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:36.380 | INFO | Uploaded 36GiB to GPU in 2.04659s (17.5903GiB/s) | 20260507 07:22:36.380 | INFO | Downloaded 34GiB to GPU in 1.94678s (17.4648GiB/s) | 20260507 07:22:41.522 | INFO | Analyzing chunk frames = [8135680,8136192) | 20260507 07:22:41.522 | INFO | MemInfo: { | 20260507 07:22:41.522 | INFO | "gpu" : " 17 GB", | 20260507 07:22:41.522 | INFO | "huge" : "", | 20260507 07:22:41.522 | INFO | "huge_shared" : " 27 GB", | 20260507 07:22:41.522 | INFO | "malloc" : " 14 GB", | 20260507 07:22:41.522 | INFO | "pinned" : " 29 GB" | 20260507 07:22:41.522 | INFO | } | 20260507 07:22:41.522 | INFO | Perf report for the last 5.14307 seconds: | 20260507 07:22:41.522 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:22:41.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:41.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:41.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:41.523 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 07:22:41.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:41.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:41.523 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:22:41.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:41.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:41.523 | INFO | Uploaded 36GiB to GPU in 2.0218s (17.8059GiB/s) | 20260507 07:22:41.523 | INFO | Downloaded 34GiB to GPU in 1.95675s (17.3758GiB/s) | 20260507 07:22:46.660 | INFO | Analyzing chunk frames = [8136192,8136704) | 20260507 07:22:46.660 | INFO | MemInfo: { | 20260507 07:22:46.660 | INFO | "gpu" : " 17 GB", | 20260507 07:22:46.660 | INFO | "huge" : "", | 20260507 07:22:46.660 | INFO | "huge_shared" : " 27 GB", | 20260507 07:22:46.660 | INFO | "malloc" : " 14 GB", | 20260507 07:22:46.660 | INFO | "pinned" : " 29 GB" | 20260507 07:22:46.660 | INFO | } | 20260507 07:22:46.660 | INFO | Perf report for the last 5.1376 seconds: | 20260507 07:22:46.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:22:46.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:46.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:46.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:46.660 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 07:22:46.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:46.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:46.660 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:22:46.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:46.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:46.660 | INFO | Uploaded 36GiB to GPU in 2.05113s (17.5513GiB/s) | 20260507 07:22:46.660 | INFO | Downloaded 34GiB to GPU in 1.95097s (17.4272GiB/s) | 20260507 07:22:51.793 | INFO | Analyzing chunk frames = [8136704,8137216) | 20260507 07:22:51.793 | INFO | MemInfo: { | 20260507 07:22:51.793 | INFO | "gpu" : " 17 GB", | 20260507 07:22:51.793 | INFO | "huge" : "", | 20260507 07:22:51.793 | INFO | "huge_shared" : " 27 GB", | 20260507 07:22:51.793 | INFO | "malloc" : " 14 GB", | 20260507 07:22:51.793 | INFO | "pinned" : " 29 GB" | 20260507 07:22:51.793 | INFO | } | 20260507 07:22:51.793 | INFO | Perf report for the last 5.13328 seconds: | 20260507 07:22:51.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:22:51.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:51.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:51.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:51.793 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 07:22:51.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:51.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:51.793 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:22:51.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:51.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:51.794 | INFO | Uploaded 36GiB to GPU in 2.03383s (17.7006GiB/s) | 20260507 07:22:51.794 | INFO | Downloaded 34GiB to GPU in 1.90859s (17.8142GiB/s) | 20260507 07:22:56.932 | INFO | Analyzing chunk frames = [8137216,8137728) | 20260507 07:22:56.932 | INFO | MemInfo: { | 20260507 07:22:56.932 | INFO | "gpu" : " 17 GB", | 20260507 07:22:56.932 | INFO | "huge" : "", | 20260507 07:22:56.932 | INFO | "huge_shared" : " 27 GB", | 20260507 07:22:56.932 | INFO | "malloc" : " 14 GB", | 20260507 07:22:56.932 | INFO | "pinned" : " 29 GB" | 20260507 07:22:56.932 | INFO | } | 20260507 07:22:56.932 | INFO | Perf report for the last 5.13883 seconds: | 20260507 07:22:56.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:22:56.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:56.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:56.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:56.932 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 07:22:56.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:56.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:56.932 | INFO | PRE_HQ : 26.1%, 1.0, 13.6ms, 0 | 20260507 07:22:56.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:56.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:22:56.932 | INFO | Uploaded 36GiB to GPU in 2.02808s (17.7508GiB/s) | 20260507 07:22:56.932 | INFO | Downloaded 34GiB to GPU in 1.92261s (17.6843GiB/s) | 20260507 07:23:02.076 | INFO | Analyzing chunk frames = [8137728,8138240) | 20260507 07:23:02.076 | INFO | MemInfo: { | 20260507 07:23:02.076 | INFO | "gpu" : " 17 GB", | 20260507 07:23:02.076 | INFO | "huge" : "", | 20260507 07:23:02.076 | INFO | "huge_shared" : " 27 GB", | 20260507 07:23:02.076 | INFO | "malloc" : " 14 GB", | 20260507 07:23:02.076 | INFO | "pinned" : " 29 GB" | 20260507 07:23:02.077 | INFO | } | 20260507 07:23:02.077 | INFO | Perf report for the last 5.14431 seconds: | 20260507 07:23:02.077 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:23:02.077 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:02.077 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:02.077 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:02.077 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 07:23:02.077 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:02.077 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:02.077 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 07:23:02.077 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:02.077 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:02.077 | INFO | Uploaded 36GiB to GPU in 1.97875s (18.1933GiB/s) | 20260507 07:23:02.077 | INFO | Downloaded 34GiB to GPU in 1.98025s (17.1695GiB/s) | 20260507 07:23:07.215 | INFO | Analyzing chunk frames = [8138240,8138752) | 20260507 07:23:07.215 | INFO | MemInfo: { | 20260507 07:23:07.215 | INFO | "gpu" : " 17 GB", | 20260507 07:23:07.215 | INFO | "huge" : "", | 20260507 07:23:07.215 | INFO | "huge_shared" : " 27 GB", | 20260507 07:23:07.215 | INFO | "malloc" : " 14 GB", | 20260507 07:23:07.215 | INFO | "pinned" : " 29 GB" | 20260507 07:23:07.215 | INFO | } | 20260507 07:23:07.215 | INFO | Perf report for the last 5.13855 seconds: | 20260507 07:23:07.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:23:07.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:07.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:07.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:07.215 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:23:07.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:07.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:07.215 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 07:23:07.215 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:07.215 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:07.215 | INFO | Uploaded 36GiB to GPU in 1.98639s (18.1233GiB/s) | 20260507 07:23:07.216 | INFO | Downloaded 34GiB to GPU in 1.96361s (17.315GiB/s) | 20260507 07:23:07.313 | INFO | Starting a round of Baz IO | 20260507 07:23:10.400 | INFO | Populating RTMetrics report | 20260507 07:23:10.401 | INFO | RTMetrics report populated | 20260507 07:23:10.401 | INFO | Opening tmp RTMetrics json file | 20260507 07:23:10.407 | INFO | Serializing RTMetrics to json | 20260507 07:23:10.418 | INFO | Serialization done, swapping tmp file now... | 20260507 07:23:10.418 | INFO | RTMetrics done | 20260507 07:23:10.486 | INFO | Finished a round of Baz IO: Wrote 1.643 GiB in 3.173 seconds | 20260507 07:23:10.486 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.56% / 0.00% / 67.44% | 20260507 07:23:10.486 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.106287% / 0% / 9.13938e-05% | 20260507 07:23:10.486 | INFO | Baz IO overall: | 20260507 07:23:10.486 | INFO | BytesWritten : 4081115430381 | 20260507 07:23:10.486 | INFO | NumEvents : 1954053247235 | 20260507 07:23:12.355 | INFO | Analyzing chunk frames = [8138752,8139264) | 20260507 07:23:12.355 | INFO | MemInfo: { | 20260507 07:23:12.355 | INFO | "gpu" : " 17 GB", | 20260507 07:23:12.355 | INFO | "huge" : "", | 20260507 07:23:12.355 | INFO | "huge_shared" : " 27 GB", | 20260507 07:23:12.355 | INFO | "malloc" : " 14 GB", | 20260507 07:23:12.355 | INFO | "pinned" : " 29 GB" | 20260507 07:23:12.355 | INFO | } | 20260507 07:23:12.355 | INFO | Perf report for the last 5.13979 seconds: | 20260507 07:23:12.355 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:23:12.355 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:12.355 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:12.355 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:12.355 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260507 07:23:12.355 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:12.355 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:23:12.355 | INFO | PRE_HQ : 58.6%, 1.8, 54.3ms, 0 | 20260507 07:23:12.355 | INFO | BAZWRITER : 61.3%, 3.1, 96.8ms, 0 | 20260507 07:23:12.355 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:23:12.355 | INFO | Uploaded 36GiB to GPU in 2.06152s (17.4628GiB/s) | 20260507 07:23:12.355 | INFO | Downloaded 38GiB to GPU in 2.18201s (17.4151GiB/s) | 20260507 07:23:17.491 | INFO | Analyzing chunk frames = [8139264,8139776) | 20260507 07:23:17.491 | INFO | MemInfo: { | 20260507 07:23:17.491 | INFO | "gpu" : " 17 GB", | 20260507 07:23:17.491 | INFO | "huge" : "", | 20260507 07:23:17.491 | INFO | "huge_shared" : " 27 GB", | 20260507 07:23:17.491 | INFO | "malloc" : " 14 GB", | 20260507 07:23:17.491 | INFO | "pinned" : " 29 GB" | 20260507 07:23:17.491 | INFO | } | 20260507 07:23:17.492 | INFO | Perf report for the last 5.13656 seconds: | 20260507 07:23:17.492 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:23:17.492 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:17.492 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:17.492 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:17.492 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:23:17.492 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:17.492 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:17.492 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:23:17.492 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:17.492 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:17.492 | INFO | Uploaded 36GiB to GPU in 1.98589s (18.1279GiB/s) | 20260507 07:23:17.492 | INFO | Downloaded 34GiB to GPU in 1.93785s (17.5453GiB/s) | 20260507 07:23:22.637 | INFO | Analyzing chunk frames = [8139776,8140288) | 20260507 07:23:22.637 | INFO | MemInfo: { | 20260507 07:23:22.637 | INFO | "gpu" : " 17 GB", | 20260507 07:23:22.637 | INFO | "huge" : "", | 20260507 07:23:22.637 | INFO | "huge_shared" : " 27 GB", | 20260507 07:23:22.637 | INFO | "malloc" : " 14 GB", | 20260507 07:23:22.637 | INFO | "pinned" : " 29 GB" | 20260507 07:23:22.637 | INFO | } | 20260507 07:23:22.637 | INFO | Perf report for the last 5.14586 seconds: | 20260507 07:23:22.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:23:22.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:22.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:22.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:22.637 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 07:23:22.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:22.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:22.637 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:23:22.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:22.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:22.638 | INFO | Uploaded 36GiB to GPU in 2.035s (17.6904GiB/s) | 20260507 07:23:22.638 | INFO | Downloaded 34GiB to GPU in 1.95529s (17.3887GiB/s) | 20260507 07:23:27.778 | INFO | Analyzing chunk frames = [8140288,8140800) | 20260507 07:23:27.778 | INFO | MemInfo: { | 20260507 07:23:27.778 | INFO | "gpu" : " 17 GB", | 20260507 07:23:27.778 | INFO | "huge" : "", | 20260507 07:23:27.778 | INFO | "huge_shared" : " 27 GB", | 20260507 07:23:27.778 | INFO | "malloc" : " 14 GB", | 20260507 07:23:27.778 | INFO | "pinned" : " 29 GB" | 20260507 07:23:27.778 | INFO | } | 20260507 07:23:27.778 | INFO | Perf report for the last 5.14106 seconds: | 20260507 07:23:27.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:23:27.779 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:27.779 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:27.779 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:27.779 | INFO | ANALYSIS : 60.3%, 2.9, 93.2ms, 0 | 20260507 07:23:27.779 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:27.779 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:27.779 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:23:27.779 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:27.779 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:27.779 | INFO | Uploaded 36GiB to GPU in 2.03807s (17.6637GiB/s) | 20260507 07:23:27.779 | INFO | Downloaded 34GiB to GPU in 1.95212s (17.4169GiB/s) | 20260507 07:23:32.910 | INFO | Analyzing chunk frames = [8140800,8141312) | 20260507 07:23:32.910 | INFO | MemInfo: { | 20260507 07:23:32.910 | INFO | "gpu" : " 17 GB", | 20260507 07:23:32.910 | INFO | "huge" : "", | 20260507 07:23:32.910 | INFO | "huge_shared" : " 27 GB", | 20260507 07:23:32.910 | INFO | "malloc" : " 14 GB", | 20260507 07:23:32.910 | INFO | "pinned" : " 29 GB" | 20260507 07:23:32.910 | INFO | } | 20260507 07:23:32.910 | INFO | Perf report for the last 5.13131 seconds: | 20260507 07:23:32.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:23:32.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:32.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:32.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:32.910 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 07:23:32.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:32.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:32.910 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:23:32.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:32.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:32.910 | INFO | Uploaded 36GiB to GPU in 2.03388s (17.7002GiB/s) | 20260507 07:23:32.910 | INFO | Downloaded 34GiB to GPU in 1.90967s (17.8041GiB/s) | 20260507 07:23:38.046 | INFO | Analyzing chunk frames = [8141312,8141824) | 20260507 07:23:38.046 | INFO | MemInfo: { | 20260507 07:23:38.046 | INFO | "gpu" : " 17 GB", | 20260507 07:23:38.046 | INFO | "huge" : "", | 20260507 07:23:38.046 | INFO | "huge_shared" : " 27 GB", | 20260507 07:23:38.046 | INFO | "malloc" : " 14 GB", | 20260507 07:23:38.046 | INFO | "pinned" : " 29 GB" | 20260507 07:23:38.046 | INFO | } | 20260507 07:23:38.046 | INFO | Perf report for the last 5.13663 seconds: | 20260507 07:23:38.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:23:38.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:38.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:38.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:38.046 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 07:23:38.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:38.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:38.046 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:23:38.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:38.047 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:38.047 | INFO | Uploaded 36GiB to GPU in 2.03748s (17.6689GiB/s) | 20260507 07:23:38.047 | INFO | Downloaded 34GiB to GPU in 1.91406s (17.7633GiB/s) | 20260507 07:23:43.192 | INFO | Analyzing chunk frames = [8141824,8142336) | 20260507 07:23:43.192 | INFO | MemInfo: { | 20260507 07:23:43.192 | INFO | "gpu" : " 17 GB", | 20260507 07:23:43.192 | INFO | "huge" : "", | 20260507 07:23:43.192 | INFO | "huge_shared" : " 27 GB", | 20260507 07:23:43.192 | INFO | "malloc" : " 14 GB", | 20260507 07:23:43.192 | INFO | "pinned" : " 29 GB" | 20260507 07:23:43.192 | INFO | } | 20260507 07:23:43.192 | INFO | Perf report for the last 5.14538 seconds: | 20260507 07:23:43.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:23:43.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:43.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:43.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:43.192 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 07:23:43.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:43.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:43.192 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 07:23:43.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:43.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:43.192 | INFO | Uploaded 36GiB to GPU in 1.96101s (18.3579GiB/s) | 20260507 07:23:43.192 | INFO | Downloaded 34GiB to GPU in 1.97613s (17.2053GiB/s) | 20260507 07:23:48.332 | INFO | Analyzing chunk frames = [8142336,8142848) | 20260507 07:23:48.332 | INFO | MemInfo: { | 20260507 07:23:48.332 | INFO | "gpu" : " 17 GB", | 20260507 07:23:48.332 | INFO | "huge" : "", | 20260507 07:23:48.333 | INFO | "huge_shared" : " 27 GB", | 20260507 07:23:48.333 | INFO | "malloc" : " 14 GB", | 20260507 07:23:48.333 | INFO | "pinned" : " 29 GB" | 20260507 07:23:48.333 | INFO | } | 20260507 07:23:48.333 | INFO | Perf report for the last 5.14077 seconds: | 20260507 07:23:48.333 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:23:48.333 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:48.333 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:48.333 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:48.333 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:23:48.333 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:48.333 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:48.333 | INFO | PRE_HQ : 29.1%, 1.0, 15.1ms, 0 | 20260507 07:23:48.333 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:48.333 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:48.333 | INFO | Uploaded 36GiB to GPU in 1.98914s (18.0983GiB/s) | 20260507 07:23:48.333 | INFO | Downloaded 34GiB to GPU in 1.95841s (17.3611GiB/s) | 20260507 07:23:48.333 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:23:48.333 | INFO | Total mem usage: 18 GiB | 20260507 07:23:48.333 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:23:48.333 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 07:23:48.333 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:23:48.333 | INFO | High Water Breakdown: | 20260507 07:23:48.333 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:23:48.333 | INFO | 39.7% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:23:48.333 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:23:48.333 | INFO | ------------------------------------------ | 20260507 07:23:48.333 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:23:48.333 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:23:48.333 | INFO | Total mem usage: 26.5 GiB | 20260507 07:23:48.333 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:23:48.333 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:23:48.333 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:23:48.333 | INFO | High Water Breakdown: | 20260507 07:23:48.333 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:23:48.333 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:23:48.333 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:23:48.333 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:23:48.333 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:23:48.333 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:23:48.333 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:23:48.333 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:23:48.333 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:23:48.333 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:23:48.333 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:23:48.333 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:23:48.333 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:23:48.333 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:23:48.333 | INFO | ------------------------------------------ | 20260507 07:23:48.333 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:23:48.333 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:23:48.333 | INFO | Total mem usage: 25.6 GiB | 20260507 07:23:48.333 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:23:48.333 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:23:48.333 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:23:48.333 | INFO | High Water Breakdown: | 20260507 07:23:48.333 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:23:48.333 | INFO | ------------------------------------------ | 20260507 07:23:48.333 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:23:48.333 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:23:48.333 | INFO | Total mem usage: 15.7 GiB | 20260507 07:23:48.333 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:23:48.333 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:23:48.333 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:23:48.333 | INFO | High Water Breakdown: | 20260507 07:23:48.333 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:23:48.333 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:23:48.333 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:23:48.333 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:23:48.333 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:23:48.333 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:23:48.333 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:23:48.333 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:23:48.333 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:23:48.333 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:23:48.333 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:23:48.333 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:23:48.333 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:23:48.333 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:23:48.333 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:23:48.333 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:23:48.334 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:23:48.334 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:23:48.334 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:23:48.334 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:23:48.334 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:23:48.334 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:23:48.334 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:23:48.334 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:23:48.334 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:23:48.334 | INFO | ------------------------------------------ | 20260507 07:23:48.334 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:23:48.334 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:23:48.334 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:23:48.334 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:23:48.334 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 1.23e+00 | 20260507 07:23:48.334 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.01e-01, 9.83e-03 / 3.91e-03 | 20260507 07:23:48.334 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.49e+00, 5.01e-01 / 2.19e-01 | 20260507 07:23:48.334 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.50e+00, 1.65e-01 / 8.64e-02 | 20260507 07:23:48.334 | INFO | PerfReport: 16.33%, DME -- 4.01e+01 / 4.50e+01, 3.97e+00 / 1.36e+00 | 20260507 07:23:48.334 | INFO | PerfReport: 57.63%, FrameLabeling -- 1.77e+01 / 1.93e+01, 1.23e+00 / 1.10e+00 | 20260507 07:23:48.334 | INFO | PerfReport: 6.91%, PulseAccumulating -- 2.12e+00 / 2.72e+00, 3.85e-01 / 1.39e-01 | 20260507 07:23:48.334 | INFO | PerfReport: 2.94%, Metrics -- 9.03e-01 / 1.21e+00, 2.44e-01 / 1.10e-01 | 20260507 07:23:48.334 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.03e-01 / 3.61e-01, 5.09e-02 / 1.99e-02 | 20260507 07:23:48.334 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:23:48.334 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:23:48.334 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:23:48.334 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 07:23:48.334 | INFO | PerfReport: 21.90%, Upload -- 2.11e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260507 07:23:48.334 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 07:23:48.334 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 07:23:48.334 | INFO | PerfReport: 1.50%, LockUpload -- 1.44e+00 / 0.00e+00, 4.86e+00 / 0.00e+00 | 20260507 07:23:48.334 | INFO | PerfReport: 1.39%, LockDownload -- 1.34e+00 / 0.00e+00, 3.13e+00 / 0.00e+00 | 20260507 07:23:48.334 | INFO | PerfReport: 21.89%, LockCompute -- 2.10e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 07:23:48.334 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.63e-02 / 0.00e+00 | 20260507 07:23:48.429 | INFO | Starting a round of Baz IO | 20260507 07:23:51.528 | INFO | Populating RTMetrics report | 20260507 07:23:51.528 | INFO | RTMetrics report populated | 20260507 07:23:51.528 | INFO | Opening tmp RTMetrics json file | 20260507 07:23:51.542 | INFO | Serializing RTMetrics to json | 20260507 07:23:51.543 | INFO | Serialization done, swapping tmp file now... | 20260507 07:23:51.544 | INFO | RTMetrics done | 20260507 07:23:51.610 | INFO | Finished a round of Baz IO: Wrote 1.643 GiB in 3.18 seconds | 20260507 07:23:51.610 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.56% / 0.00% / 67.44% | 20260507 07:23:51.610 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.106283% / 0% / 6.75519e-05% | 20260507 07:23:51.610 | INFO | Baz IO overall: | 20260507 07:23:51.610 | INFO | BytesWritten : 4082880030463 | 20260507 07:23:51.610 | INFO | NumEvents : 1954877610188 | 20260507 07:23:53.471 | INFO | Analyzing chunk frames = [8142848,8143360) | 20260507 07:23:53.471 | INFO | MemInfo: { | 20260507 07:23:53.471 | INFO | "gpu" : " 17 GB", | 20260507 07:23:53.471 | INFO | "huge" : "", | 20260507 07:23:53.471 | INFO | "huge_shared" : " 27 GB", | 20260507 07:23:53.471 | INFO | "malloc" : " 14 GB", | 20260507 07:23:53.471 | INFO | "pinned" : " 29 GB" | 20260507 07:23:53.471 | INFO | } | 20260507 07:23:53.472 | INFO | Perf report for the last 5.13888 seconds: | 20260507 07:23:53.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:23:53.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:53.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:53.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:53.472 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 07:23:53.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:53.472 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 07:23:53.472 | INFO | PRE_HQ : 59.4%, 1.8, 56.5ms, 0 | 20260507 07:23:53.472 | INFO | BAZWRITER : 61.5%, 3.1, 99.0ms, 0 | 20260507 07:23:53.472 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 07:23:53.472 | INFO | Uploaded 36GiB to GPU in 2.07353s (17.3617GiB/s) | 20260507 07:23:53.472 | INFO | Downloaded 38GiB to GPU in 2.18339s (17.4041GiB/s) | 20260507 07:23:58.610 | INFO | Analyzing chunk frames = [8143360,8143872) | 20260507 07:23:58.610 | INFO | MemInfo: { | 20260507 07:23:58.610 | INFO | "gpu" : " 17 GB", | 20260507 07:23:58.610 | INFO | "huge" : "", | 20260507 07:23:58.610 | INFO | "huge_shared" : " 27 GB", | 20260507 07:23:58.610 | INFO | "malloc" : " 14 GB", | 20260507 07:23:58.610 | INFO | "pinned" : " 29 GB" | 20260507 07:23:58.610 | INFO | } | 20260507 07:23:58.610 | INFO | Perf report for the last 5.1387 seconds: | 20260507 07:23:58.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:23:58.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:58.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:58.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:58.610 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 07:23:58.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:58.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:58.610 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 07:23:58.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:58.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:23:58.610 | INFO | Uploaded 36GiB to GPU in 2.01599s (17.8572GiB/s) | 20260507 07:23:58.611 | INFO | Downloaded 34GiB to GPU in 1.94369s (17.4925GiB/s) | 20260507 07:24:03.746 | INFO | Analyzing chunk frames = [8143872,8144384) | 20260507 07:24:03.746 | INFO | MemInfo: { | 20260507 07:24:03.746 | INFO | "gpu" : " 17 GB", | 20260507 07:24:03.746 | INFO | "huge" : "", | 20260507 07:24:03.746 | INFO | "huge_shared" : " 27 GB", | 20260507 07:24:03.747 | INFO | "malloc" : " 14 GB", | 20260507 07:24:03.747 | INFO | "pinned" : " 29 GB" | 20260507 07:24:03.747 | INFO | } | 20260507 07:24:03.747 | INFO | Perf report for the last 5.13639 seconds: | 20260507 07:24:03.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:24:03.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:03.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:03.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:03.747 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 07:24:03.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:03.747 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:03.747 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:24:03.747 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:03.747 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:03.747 | INFO | Uploaded 36GiB to GPU in 2.05427s (17.5244GiB/s) | 20260507 07:24:03.747 | INFO | Downloaded 34GiB to GPU in 1.95154s (17.4221GiB/s) | 20260507 07:24:08.884 | INFO | Analyzing chunk frames = [8144384,8144896) | 20260507 07:24:08.884 | INFO | MemInfo: { | 20260507 07:24:08.884 | INFO | "gpu" : " 17 GB", | 20260507 07:24:08.884 | INFO | "huge" : "", | 20260507 07:24:08.884 | INFO | "huge_shared" : " 27 GB", | 20260507 07:24:08.884 | INFO | "malloc" : " 14 GB", | 20260507 07:24:08.884 | INFO | "pinned" : " 29 GB" | 20260507 07:24:08.884 | INFO | } | 20260507 07:24:08.884 | INFO | Perf report for the last 5.13742 seconds: | 20260507 07:24:08.884 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:24:08.884 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:08.884 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:08.884 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:08.884 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 07:24:08.884 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:08.884 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:08.884 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:24:08.884 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:08.884 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:08.884 | INFO | Uploaded 36GiB to GPU in 2.02228s (17.8017GiB/s) | 20260507 07:24:08.884 | INFO | Downloaded 34GiB to GPU in 1.93257s (17.5932GiB/s) | 20260507 07:24:14.029 | INFO | Analyzing chunk frames = [8144896,8145408) | 20260507 07:24:14.029 | INFO | MemInfo: { | 20260507 07:24:14.029 | INFO | "gpu" : " 17 GB", | 20260507 07:24:14.029 | INFO | "huge" : "", | 20260507 07:24:14.029 | INFO | "huge_shared" : " 27 GB", | 20260507 07:24:14.029 | INFO | "malloc" : " 14 GB", | 20260507 07:24:14.029 | INFO | "pinned" : " 29 GB" | 20260507 07:24:14.029 | INFO | } | 20260507 07:24:14.030 | INFO | Perf report for the last 5.14546 seconds: | 20260507 07:24:14.030 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:24:14.030 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:14.030 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:14.030 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:14.030 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 07:24:14.030 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:14.030 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:14.030 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:24:14.030 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:14.030 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:14.030 | INFO | Uploaded 36GiB to GPU in 2.05441s (17.5233GiB/s) | 20260507 07:24:14.030 | INFO | Downloaded 34GiB to GPU in 1.94464s (17.484GiB/s) | 20260507 07:24:19.170 | INFO | Analyzing chunk frames = [8145408,8145920) | 20260507 07:24:19.170 | INFO | MemInfo: { | 20260507 07:24:19.170 | INFO | "gpu" : " 17 GB", | 20260507 07:24:19.170 | INFO | "huge" : "", | 20260507 07:24:19.170 | INFO | "huge_shared" : " 27 GB", | 20260507 07:24:19.170 | INFO | "malloc" : " 14 GB", | 20260507 07:24:19.170 | INFO | "pinned" : " 29 GB" | 20260507 07:24:19.170 | INFO | } | 20260507 07:24:19.170 | INFO | Perf report for the last 5.14032 seconds: | 20260507 07:24:19.170 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:24:19.170 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:19.170 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:19.170 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:19.170 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 07:24:19.170 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:19.170 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:19.170 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:24:19.170 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:19.170 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:19.170 | INFO | Uploaded 36GiB to GPU in 2.06884s (17.401GiB/s) | 20260507 07:24:19.170 | INFO | Downloaded 34GiB to GPU in 1.93009s (17.6157GiB/s) | 20260507 07:24:24.302 | INFO | Analyzing chunk frames = [8145920,8146432) | 20260507 07:24:24.302 | INFO | MemInfo: { | 20260507 07:24:24.302 | INFO | "gpu" : " 17 GB", | 20260507 07:24:24.302 | INFO | "huge" : "", | 20260507 07:24:24.302 | INFO | "huge_shared" : " 27 GB", | 20260507 07:24:24.302 | INFO | "malloc" : " 14 GB", | 20260507 07:24:24.302 | INFO | "pinned" : " 29 GB" | 20260507 07:24:24.302 | INFO | } | 20260507 07:24:24.302 | INFO | Perf report for the last 5.13179 seconds: | 20260507 07:24:24.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:24:24.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:24.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:24.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:24.302 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260507 07:24:24.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:24.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:24.302 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 07:24:24.302 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:24.302 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:24.302 | INFO | Uploaded 36GiB to GPU in 1.8839s (19.1093GiB/s) | 20260507 07:24:24.302 | INFO | Downloaded 34GiB to GPU in 1.97521s (17.2133GiB/s) | 20260507 07:24:29.441 | INFO | Analyzing chunk frames = [8146432,8146944) | 20260507 07:24:29.441 | INFO | MemInfo: { | 20260507 07:24:29.441 | INFO | "gpu" : " 17 GB", | 20260507 07:24:29.441 | INFO | "huge" : "", | 20260507 07:24:29.441 | INFO | "huge_shared" : " 27 GB", | 20260507 07:24:29.441 | INFO | "malloc" : " 14 GB", | 20260507 07:24:29.441 | INFO | "pinned" : " 29 GB" | 20260507 07:24:29.442 | INFO | } | 20260507 07:24:29.442 | INFO | Perf report for the last 5.13989 seconds: | 20260507 07:24:29.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:24:29.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:29.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:29.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:29.442 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:24:29.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:29.442 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:29.442 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 07:24:29.442 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:29.442 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:29.442 | INFO | Uploaded 36GiB to GPU in 1.96603s (18.311GiB/s) | 20260507 07:24:29.442 | INFO | Downloaded 34GiB to GPU in 1.9612s (17.3364GiB/s) | 20260507 07:24:29.539 | INFO | Starting a round of Baz IO | 20260507 07:24:32.632 | INFO | Populating RTMetrics report | 20260507 07:24:32.632 | INFO | RTMetrics report populated | 20260507 07:24:32.632 | INFO | Opening tmp RTMetrics json file | 20260507 07:24:32.639 | INFO | Serializing RTMetrics to json | 20260507 07:24:32.649 | INFO | Serialization done, swapping tmp file now... | 20260507 07:24:32.649 | INFO | RTMetrics done | 20260507 07:24:32.720 | INFO | Finished a round of Baz IO: Wrote 1.643 GiB in 3.18 seconds | 20260507 07:24:32.720 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.56% / 0.00% / 67.44% | 20260507 07:24:32.720 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.106061% / 0% / 7.15256e-05% | 20260507 07:24:32.720 | INFO | Baz IO overall: | 20260507 07:24:32.720 | INFO | BytesWritten : 4084644677628 | 20260507 07:24:32.720 | INFO | NumEvents : 1955702144267 | 20260507 07:24:34.585 | INFO | Analyzing chunk frames = [8146944,8147456) | 20260507 07:24:34.585 | INFO | MemInfo: { | 20260507 07:24:34.585 | INFO | "gpu" : " 17 GB", | 20260507 07:24:34.585 | INFO | "huge" : "", | 20260507 07:24:34.585 | INFO | "huge_shared" : " 27 GB", | 20260507 07:24:34.585 | INFO | "malloc" : " 14 GB", | 20260507 07:24:34.585 | INFO | "pinned" : " 29 GB" | 20260507 07:24:34.585 | INFO | } | 20260507 07:24:34.585 | INFO | Perf report for the last 5.14323 seconds: | 20260507 07:24:34.585 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:24:34.585 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:34.585 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:34.585 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:34.585 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 07:24:34.585 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:34.585 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:24:34.585 | INFO | PRE_HQ : 58.8%, 1.8, 54.6ms, 0 | 20260507 07:24:34.585 | INFO | BAZWRITER : 61.4%, 3.1, 98.4ms, 0 | 20260507 07:24:34.585 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 07:24:34.586 | INFO | Uploaded 36GiB to GPU in 2.04327s (17.6189GiB/s) | 20260507 07:24:34.586 | INFO | Downloaded 38GiB to GPU in 2.17446s (17.4756GiB/s) | 20260507 07:24:39.727 | INFO | Analyzing chunk frames = [8147456,8147968) | 20260507 07:24:39.727 | INFO | MemInfo: { | 20260507 07:24:39.727 | INFO | "gpu" : " 17 GB", | 20260507 07:24:39.727 | INFO | "huge" : "", | 20260507 07:24:39.727 | INFO | "huge_shared" : " 27 GB", | 20260507 07:24:39.727 | INFO | "malloc" : " 14 GB", | 20260507 07:24:39.727 | INFO | "pinned" : " 29 GB" | 20260507 07:24:39.727 | INFO | } | 20260507 07:24:39.727 | INFO | Perf report for the last 5.14246 seconds: | 20260507 07:24:39.727 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:24:39.727 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:39.727 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:39.727 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:39.727 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:24:39.727 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:39.727 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:39.727 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:24:39.727 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:39.727 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:39.727 | INFO | Uploaded 36GiB to GPU in 1.97523s (18.2257GiB/s) | 20260507 07:24:39.727 | INFO | Downloaded 34GiB to GPU in 1.93123s (17.6054GiB/s) | 20260507 07:24:44.861 | INFO | Analyzing chunk frames = [8147968,8148480) | 20260507 07:24:44.861 | INFO | MemInfo: { | 20260507 07:24:44.861 | INFO | "gpu" : " 17 GB", | 20260507 07:24:44.861 | INFO | "huge" : "", | 20260507 07:24:44.861 | INFO | "huge_shared" : " 27 GB", | 20260507 07:24:44.861 | INFO | "malloc" : " 14 GB", | 20260507 07:24:44.861 | INFO | "pinned" : " 29 GB" | 20260507 07:24:44.861 | INFO | } | 20260507 07:24:44.861 | INFO | Perf report for the last 5.13351 seconds: | 20260507 07:24:44.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:24:44.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:44.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:44.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:44.861 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:24:44.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:44.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:44.861 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:24:44.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:44.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:44.861 | INFO | Uploaded 36GiB to GPU in 2.02668s (17.7631GiB/s) | 20260507 07:24:44.861 | INFO | Downloaded 34GiB to GPU in 1.92208s (17.6892GiB/s) | 20260507 07:24:49.999 | INFO | Analyzing chunk frames = [8148480,8148992) | 20260507 07:24:49.999 | INFO | MemInfo: { | 20260507 07:24:49.999 | INFO | "gpu" : " 17 GB", | 20260507 07:24:49.999 | INFO | "huge" : "", | 20260507 07:24:49.999 | INFO | "huge_shared" : " 27 GB", | 20260507 07:24:49.999 | INFO | "malloc" : " 14 GB", | 20260507 07:24:49.999 | INFO | "pinned" : " 29 GB" | 20260507 07:24:49.999 | INFO | } | 20260507 07:24:49.999 | INFO | Perf report for the last 5.13866 seconds: | 20260507 07:24:49.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:24:49.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:49.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:49.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:50.000 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 07:24:50.000 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:50.000 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:50.000 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:24:50.000 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:50.000 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:50.000 | INFO | Uploaded 36GiB to GPU in 2.01131s (17.8988GiB/s) | 20260507 07:24:50.000 | INFO | Downloaded 34GiB to GPU in 1.94301s (17.4986GiB/s) | 20260507 07:24:55.147 | INFO | Analyzing chunk frames = [8148992,8149504) | 20260507 07:24:55.147 | INFO | MemInfo: { | 20260507 07:24:55.147 | INFO | "gpu" : " 17 GB", | 20260507 07:24:55.147 | INFO | "huge" : "", | 20260507 07:24:55.147 | INFO | "huge_shared" : " 27 GB", | 20260507 07:24:55.147 | INFO | "malloc" : " 14 GB", | 20260507 07:24:55.147 | INFO | "pinned" : " 29 GB" | 20260507 07:24:55.147 | INFO | } | 20260507 07:24:55.148 | INFO | Perf report for the last 5.14805 seconds: | 20260507 07:24:55.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:24:55.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:55.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:55.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:55.148 | INFO | ANALYSIS : 59.9%, 3.0, 93.7ms, 0 | 20260507 07:24:55.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:55.148 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:55.148 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:24:55.148 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:55.148 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:24:55.148 | INFO | Uploaded 36GiB to GPU in 2.04678s (17.5886GiB/s) | 20260507 07:24:55.148 | INFO | Downloaded 34GiB to GPU in 1.94927s (17.4424GiB/s) | 20260507 07:25:00.282 | INFO | Analyzing chunk frames = [8149504,8150016) | 20260507 07:25:00.282 | INFO | MemInfo: { | 20260507 07:25:00.282 | INFO | "gpu" : " 17 GB", | 20260507 07:25:00.282 | INFO | "huge" : "", | 20260507 07:25:00.282 | INFO | "huge_shared" : " 27 GB", | 20260507 07:25:00.282 | INFO | "malloc" : " 14 GB", | 20260507 07:25:00.282 | INFO | "pinned" : " 29 GB" | 20260507 07:25:00.282 | INFO | } | 20260507 07:25:00.282 | INFO | Perf report for the last 5.13468 seconds: | 20260507 07:25:00.282 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:25:00.282 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:00.282 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:00.282 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:00.282 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 07:25:00.282 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:00.282 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:00.282 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:25:00.282 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:00.282 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:00.282 | INFO | Uploaded 36GiB to GPU in 2.02889s (17.7437GiB/s) | 20260507 07:25:00.282 | INFO | Downloaded 34GiB to GPU in 1.90432s (17.8542GiB/s) | 20260507 07:25:05.418 | INFO | Analyzing chunk frames = [8150016,8150528) | 20260507 07:25:05.418 | INFO | MemInfo: { | 20260507 07:25:05.418 | INFO | "gpu" : " 17 GB", | 20260507 07:25:05.418 | INFO | "huge" : "", | 20260507 07:25:05.418 | INFO | "huge_shared" : " 27 GB", | 20260507 07:25:05.418 | INFO | "malloc" : " 14 GB", | 20260507 07:25:05.418 | INFO | "pinned" : " 29 GB" | 20260507 07:25:05.418 | INFO | } | 20260507 07:25:05.418 | INFO | Perf report for the last 5.1357 seconds: | 20260507 07:25:05.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:25:05.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:05.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:05.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:05.418 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:25:05.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:05.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:05.418 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 07:25:05.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:05.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:05.418 | INFO | Uploaded 36GiB to GPU in 1.89059s (19.0417GiB/s) | 20260507 07:25:05.418 | INFO | Downloaded 34GiB to GPU in 1.97836s (17.1859GiB/s) | 20260507 07:25:10.555 | INFO | Analyzing chunk frames = [8150528,8151040) | 20260507 07:25:10.555 | INFO | MemInfo: { | 20260507 07:25:10.555 | INFO | "gpu" : " 17 GB", | 20260507 07:25:10.555 | INFO | "huge" : "", | 20260507 07:25:10.555 | INFO | "huge_shared" : " 27 GB", | 20260507 07:25:10.555 | INFO | "malloc" : " 14 GB", | 20260507 07:25:10.555 | INFO | "pinned" : " 29 GB" | 20260507 07:25:10.555 | INFO | } | 20260507 07:25:10.555 | INFO | Perf report for the last 5.13698 seconds: | 20260507 07:25:10.555 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:25:10.555 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:10.555 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:10.555 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:10.555 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:25:10.555 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:10.555 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:10.555 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 07:25:10.555 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:10.555 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:10.555 | INFO | Uploaded 36GiB to GPU in 1.98937s (18.0961GiB/s) | 20260507 07:25:10.555 | INFO | Downloaded 34GiB to GPU in 1.95287s (17.4102GiB/s) | 20260507 07:25:10.555 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:25:10.555 | INFO | Total mem usage: 18 GiB | 20260507 07:25:10.555 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:25:10.555 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:25:10.555 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:25:10.555 | INFO | High Water Breakdown: | 20260507 07:25:10.555 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:25:10.555 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:25:10.555 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:25:10.555 | INFO | ------------------------------------------ | 20260507 07:25:10.555 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:25:10.555 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:25:10.555 | INFO | Total mem usage: 26.5 GiB | 20260507 07:25:10.555 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:25:10.555 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:25:10.555 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 07:25:10.555 | INFO | High Water Breakdown: | 20260507 07:25:10.555 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:25:10.555 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:25:10.555 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:25:10.555 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:25:10.555 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:25:10.555 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:25:10.555 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:25:10.555 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:25:10.556 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:25:10.556 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:25:10.556 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:25:10.556 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:25:10.556 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:25:10.556 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:25:10.556 | INFO | ------------------------------------------ | 20260507 07:25:10.556 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:25:10.556 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:25:10.556 | INFO | Total mem usage: 25.6 GiB | 20260507 07:25:10.556 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:25:10.556 | INFO | Recent High Water Marks Sum: 24.2 GiB (Savings: -5.67%) | 20260507 07:25:10.556 | INFO | Unused Cache: 1.38 GiB (Waste: 5.37%) | 20260507 07:25:10.556 | INFO | High Water Breakdown: | 20260507 07:25:10.556 | INFO | 100% : 24.2 GiB ( 94.6% Of Peak) : WXIPCDataSourceConfig | 20260507 07:25:10.556 | INFO | ------------------------------------------ | 20260507 07:25:10.556 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:25:10.556 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:25:10.556 | INFO | Total mem usage: 15.7 GiB | 20260507 07:25:10.556 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:25:10.556 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:25:10.556 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:25:10.556 | INFO | High Water Breakdown: | 20260507 07:25:10.556 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:25:10.556 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:25:10.556 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:25:10.556 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:25:10.556 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:25:10.556 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:25:10.556 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:25:10.556 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:25:10.556 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:25:10.556 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:25:10.556 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:25:10.556 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:25:10.556 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:25:10.556 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:25:10.556 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:25:10.556 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:25:10.556 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:25:10.556 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:25:10.556 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:25:10.556 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:25:10.556 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:25:10.556 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:25:10.556 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:25:10.556 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:25:10.556 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:25:10.556 | INFO | ------------------------------------------ | 20260507 07:25:10.556 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:25:10.556 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:25:10.556 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:25:10.556 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:25:10.556 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.35e+01 / 1.09e+00 | 20260507 07:25:10.556 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.02e-01, 9.77e-03 / 4.32e-03 | 20260507 07:25:10.556 | INFO | PerfReport: 11.68%, Baseline -- 3.58e+00 / 4.47e+00, 5.11e-01 / 2.19e-01 | 20260507 07:25:10.556 | INFO | PerfReport: 3.23%, Histogram -- 9.90e-01 / 1.51e+00, 1.68e-01 / 8.21e-02 | 20260507 07:25:10.556 | INFO | PerfReport: 16.27%, DME -- 3.99e+01 / 4.47e+01, 3.90e+00 / 1.33e+00 | 20260507 07:25:10.556 | INFO | PerfReport: 57.65%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.23e+00 / 1.08e+00 | 20260507 07:25:10.557 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.13e+00 / 2.72e+00, 3.90e-01 / 1.62e-01 | 20260507 07:25:10.557 | INFO | PerfReport: 2.95%, Metrics -- 9.04e-01 / 1.21e+00, 2.44e-01 / 1.40e-01 | 20260507 07:25:10.557 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.03e-01 / 3.65e-01, 5.12e-02 / 2.02e-02 | 20260507 07:25:10.557 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:25:10.557 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:25:10.557 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:25:10.557 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 07:25:10.557 | INFO | PerfReport: 21.79%, Upload -- 2.09e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260507 07:25:10.557 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 07:25:10.557 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 07:25:10.557 | INFO | PerfReport: 1.54%, LockUpload -- 1.48e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 07:25:10.557 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260507 07:25:10.557 | INFO | PerfReport: 22.03%, LockCompute -- 2.12e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 07:25:10.557 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 2.50e-02 / 0.00e+00 | 20260507 07:25:10.652 | INFO | Starting a round of Baz IO | 20260507 07:25:13.747 | INFO | Populating RTMetrics report | 20260507 07:25:13.748 | INFO | RTMetrics report populated | 20260507 07:25:13.748 | INFO | Opening tmp RTMetrics json file | 20260507 07:25:13.748 | INFO | Serializing RTMetrics to json | 20260507 07:25:13.749 | INFO | Serialization done, swapping tmp file now... | 20260507 07:25:13.750 | INFO | RTMetrics done | 20260507 07:25:13.830 | INFO | Finished a round of Baz IO: Wrote 1.643 GiB in 3.178 seconds | 20260507 07:25:13.830 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.56% / 0.00% / 67.44% | 20260507 07:25:13.830 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.10637% / 3.97364e-06% / 5.96046e-05% | 20260507 07:25:13.830 | INFO | Baz IO overall: | 20260507 07:25:13.830 | INFO | BytesWritten : 4086409358206 | 20260507 07:25:13.830 | INFO | NumEvents : 1956527056988 | 20260507 07:25:15.698 | INFO | Analyzing chunk frames = [8151040,8151552) | 20260507 07:25:15.698 | INFO | MemInfo: { | 20260507 07:25:15.698 | INFO | "gpu" : " 17 GB", | 20260507 07:25:15.698 | INFO | "huge" : "", | 20260507 07:25:15.698 | INFO | "huge_shared" : " 27 GB", | 20260507 07:25:15.698 | INFO | "malloc" : " 14 GB", | 20260507 07:25:15.698 | INFO | "pinned" : " 29 GB" | 20260507 07:25:15.698 | INFO | } | 20260507 07:25:15.698 | INFO | Perf report for the last 5.14323 seconds: | 20260507 07:25:15.698 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:25:15.698 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:15.698 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:15.698 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:15.698 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 07:25:15.698 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:15.698 | INFO | RT_METRICS : 40.7%, 1.0, 21.3ms, 0 | 20260507 07:25:15.698 | INFO | PRE_HQ : 58.9%, 1.8, 55.0ms, 0 | 20260507 07:25:15.698 | INFO | BAZWRITER : 61.3%, 3.1, 98.1ms, 0 | 20260507 07:25:15.698 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:25:15.698 | INFO | Uploaded 36GiB to GPU in 2.03328s (17.7054GiB/s) | 20260507 07:25:15.698 | INFO | Downloaded 38GiB to GPU in 2.16602s (17.5437GiB/s) | 20260507 07:25:20.840 | INFO | Analyzing chunk frames = [8151552,8152064) | 20260507 07:25:20.840 | INFO | MemInfo: { | 20260507 07:25:20.840 | INFO | "gpu" : " 17 GB", | 20260507 07:25:20.840 | INFO | "huge" : "", | 20260507 07:25:20.840 | INFO | "huge_shared" : " 27 GB", | 20260507 07:25:20.840 | INFO | "malloc" : " 14 GB", | 20260507 07:25:20.840 | INFO | "pinned" : " 29 GB" | 20260507 07:25:20.840 | INFO | } | 20260507 07:25:20.841 | INFO | Perf report for the last 5.14234 seconds: | 20260507 07:25:20.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:25:20.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:20.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:20.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:20.841 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 07:25:20.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:20.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:20.841 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:25:20.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:20.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:20.841 | INFO | Uploaded 36GiB to GPU in 1.99019s (18.0887GiB/s) | 20260507 07:25:20.841 | INFO | Downloaded 34GiB to GPU in 1.92873s (17.6282GiB/s) | 20260507 07:25:25.978 | INFO | Analyzing chunk frames = [8152064,8152576) | 20260507 07:25:25.978 | INFO | MemInfo: { | 20260507 07:25:25.978 | INFO | "gpu" : " 17 GB", | 20260507 07:25:25.978 | INFO | "huge" : "", | 20260507 07:25:25.978 | INFO | "huge_shared" : " 27 GB", | 20260507 07:25:25.978 | INFO | "malloc" : " 14 GB", | 20260507 07:25:25.978 | INFO | "pinned" : " 29 GB" | 20260507 07:25:25.978 | INFO | } | 20260507 07:25:25.978 | INFO | Perf report for the last 5.13779 seconds: | 20260507 07:25:25.978 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:25:25.978 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:25.978 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:25.978 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:25.978 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:25:25.978 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:25.978 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:25.978 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:25:25.978 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:25.978 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:25.979 | INFO | Uploaded 36GiB to GPU in 2.01073s (17.904GiB/s) | 20260507 07:25:25.979 | INFO | Downloaded 34GiB to GPU in 1.91568s (17.7483GiB/s) | 20260507 07:25:31.115 | INFO | Analyzing chunk frames = [8152576,8153088) | 20260507 07:25:31.115 | INFO | MemInfo: { | 20260507 07:25:31.115 | INFO | "gpu" : " 17 GB", | 20260507 07:25:31.115 | INFO | "huge" : "", | 20260507 07:25:31.115 | INFO | "huge_shared" : " 27 GB", | 20260507 07:25:31.115 | INFO | "malloc" : " 14 GB", | 20260507 07:25:31.115 | INFO | "pinned" : " 29 GB" | 20260507 07:25:31.115 | INFO | } | 20260507 07:25:31.115 | INFO | Perf report for the last 5.13689 seconds: | 20260507 07:25:31.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:25:31.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:31.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:31.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:31.115 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 07:25:31.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:31.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:31.115 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:25:31.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:31.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:31.115 | INFO | Uploaded 36GiB to GPU in 2.04069s (17.6411GiB/s) | 20260507 07:25:31.115 | INFO | Downloaded 34GiB to GPU in 1.9503s (17.4332GiB/s) | 20260507 07:25:36.253 | INFO | Analyzing chunk frames = [8153088,8153600) | 20260507 07:25:36.253 | INFO | MemInfo: { | 20260507 07:25:36.253 | INFO | "gpu" : " 17 GB", | 20260507 07:25:36.253 | INFO | "huge" : "", | 20260507 07:25:36.253 | INFO | "huge_shared" : " 27 GB", | 20260507 07:25:36.253 | INFO | "malloc" : " 14 GB", | 20260507 07:25:36.253 | INFO | "pinned" : " 29 GB" | 20260507 07:25:36.253 | INFO | } | 20260507 07:25:36.253 | INFO | Perf report for the last 5.13777 seconds: | 20260507 07:25:36.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:25:36.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:36.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:36.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:36.253 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 07:25:36.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:36.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:36.253 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:25:36.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:36.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:36.253 | INFO | Uploaded 36GiB to GPU in 2.02927s (17.7403GiB/s) | 20260507 07:25:36.253 | INFO | Downloaded 34GiB to GPU in 1.91313s (17.772GiB/s) | 20260507 07:25:41.395 | INFO | Analyzing chunk frames = [8153600,8154112) | 20260507 07:25:41.395 | INFO | MemInfo: { | 20260507 07:25:41.395 | INFO | "gpu" : " 17 GB", | 20260507 07:25:41.395 | INFO | "huge" : "", | 20260507 07:25:41.395 | INFO | "huge_shared" : " 27 GB", | 20260507 07:25:41.395 | INFO | "malloc" : " 14 GB", | 20260507 07:25:41.395 | INFO | "pinned" : " 29 GB" | 20260507 07:25:41.395 | INFO | } | 20260507 07:25:41.395 | INFO | Perf report for the last 5.14233 seconds: | 20260507 07:25:41.395 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:25:41.395 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:41.395 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:41.395 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:41.395 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 07:25:41.395 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:41.395 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:41.395 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:25:41.395 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:41.395 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:41.396 | INFO | Uploaded 36GiB to GPU in 2.04817s (17.5767GiB/s) | 20260507 07:25:41.396 | INFO | Downloaded 34GiB to GPU in 1.91402s (17.7636GiB/s) | 20260507 07:25:46.538 | INFO | Analyzing chunk frames = [8154112,8154624) | 20260507 07:25:46.538 | INFO | MemInfo: { | 20260507 07:25:46.538 | INFO | "gpu" : " 17 GB", | 20260507 07:25:46.538 | INFO | "huge" : "", | 20260507 07:25:46.538 | INFO | "huge_shared" : " 27 GB", | 20260507 07:25:46.538 | INFO | "malloc" : " 14 GB", | 20260507 07:25:46.539 | INFO | "pinned" : " 29 GB" | 20260507 07:25:46.539 | INFO | } | 20260507 07:25:46.539 | INFO | Perf report for the last 5.1433 seconds: | 20260507 07:25:46.539 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:25:46.539 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:46.539 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:46.539 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:46.539 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 07:25:46.539 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:46.539 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:46.539 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 07:25:46.539 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:46.539 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:46.539 | INFO | Uploaded 36GiB to GPU in 2.03236s (17.7134GiB/s) | 20260507 07:25:46.539 | INFO | Downloaded 34GiB to GPU in 1.99277s (17.0617GiB/s) | 20260507 07:25:51.676 | INFO | Analyzing chunk frames = [8154624,8155136) | 20260507 07:25:51.676 | INFO | MemInfo: { | 20260507 07:25:51.676 | INFO | "gpu" : " 17 GB", | 20260507 07:25:51.676 | INFO | "huge" : "", | 20260507 07:25:51.676 | INFO | "huge_shared" : " 27 GB", | 20260507 07:25:51.676 | INFO | "malloc" : " 14 GB", | 20260507 07:25:51.676 | INFO | "pinned" : " 29 GB" | 20260507 07:25:51.676 | INFO | } | 20260507 07:25:51.676 | INFO | Perf report for the last 5.13729 seconds: | 20260507 07:25:51.676 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:25:51.676 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:51.676 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:51.676 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:51.676 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:25:51.676 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:51.676 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:51.676 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 07:25:51.676 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:51.676 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:51.676 | INFO | Uploaded 36GiB to GPU in 1.98811s (18.1077GiB/s) | 20260507 07:25:51.676 | INFO | Downloaded 34GiB to GPU in 1.96355s (17.3156GiB/s) | 20260507 07:25:51.774 | INFO | Starting a round of Baz IO | 20260507 07:25:54.862 | INFO | Populating RTMetrics report | 20260507 07:25:54.862 | INFO | RTMetrics report populated | 20260507 07:25:54.862 | INFO | Opening tmp RTMetrics json file | 20260507 07:25:54.872 | INFO | Serializing RTMetrics to json | 20260507 07:25:54.878 | INFO | Serialization done, swapping tmp file now... | 20260507 07:25:54.878 | INFO | RTMetrics done | 20260507 07:25:54.947 | INFO | Finished a round of Baz IO: Wrote 1.642 GiB in 3.172 seconds | 20260507 07:25:54.947 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.56% / 0.00% / 67.44% | 20260507 07:25:54.947 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.105981% / 0% / 7.54992e-05% | 20260507 07:25:54.947 | INFO | Baz IO overall: | 20260507 07:25:54.947 | INFO | BytesWritten : 4088172911084 | 20260507 07:25:54.947 | INFO | NumEvents : 1957351244301 | 20260507 07:25:56.815 | INFO | Analyzing chunk frames = [8155136,8155648) | 20260507 07:25:56.815 | INFO | MemInfo: { | 20260507 07:25:56.815 | INFO | "gpu" : " 17 GB", | 20260507 07:25:56.815 | INFO | "huge" : "", | 20260507 07:25:56.815 | INFO | "huge_shared" : " 27 GB", | 20260507 07:25:56.815 | INFO | "malloc" : " 14 GB", | 20260507 07:25:56.815 | INFO | "pinned" : " 29 GB" | 20260507 07:25:56.815 | INFO | } | 20260507 07:25:56.815 | INFO | Perf report for the last 5.13882 seconds: | 20260507 07:25:56.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:25:56.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:56.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:56.815 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:56.815 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 07:25:56.815 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:25:56.815 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 07:25:56.815 | INFO | PRE_HQ : 59.3%, 1.9, 58.2ms, 0 | 20260507 07:25:56.815 | INFO | BAZWRITER : 61.4%, 3.2, 101.8ms, 0 | 20260507 07:25:56.815 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:25:56.815 | INFO | Uploaded 36GiB to GPU in 2.05747s (17.4972GiB/s) | 20260507 07:25:56.815 | INFO | Downloaded 38GiB to GPU in 2.1805s (17.4272GiB/s) | 20260507 07:26:01.953 | INFO | Analyzing chunk frames = [8155648,8156160) | 20260507 07:26:01.953 | INFO | MemInfo: { | 20260507 07:26:01.953 | INFO | "gpu" : " 17 GB", | 20260507 07:26:01.953 | INFO | "huge" : "", | 20260507 07:26:01.953 | INFO | "huge_shared" : " 27 GB", | 20260507 07:26:01.953 | INFO | "malloc" : " 14 GB", | 20260507 07:26:01.953 | INFO | "pinned" : " 29 GB" | 20260507 07:26:01.953 | INFO | } | 20260507 07:26:01.953 | INFO | Perf report for the last 5.13865 seconds: | 20260507 07:26:01.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:26:01.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:01.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:01.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:01.953 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:26:01.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:01.953 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:01.954 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:26:01.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:01.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:01.954 | INFO | Uploaded 36GiB to GPU in 1.97503s (18.2275GiB/s) | 20260507 07:26:01.954 | INFO | Downloaded 34GiB to GPU in 1.93438s (17.5767GiB/s) | 20260507 07:26:07.096 | INFO | Analyzing chunk frames = [8156160,8156672) | 20260507 07:26:07.096 | INFO | MemInfo: { | 20260507 07:26:07.096 | INFO | "gpu" : " 17 GB", | 20260507 07:26:07.096 | INFO | "huge" : "", | 20260507 07:26:07.096 | INFO | "huge_shared" : " 27 GB", | 20260507 07:26:07.096 | INFO | "malloc" : " 14 GB", | 20260507 07:26:07.096 | INFO | "pinned" : " 29 GB" | 20260507 07:26:07.096 | INFO | } | 20260507 07:26:07.096 | INFO | Perf report for the last 5.14276 seconds: | 20260507 07:26:07.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:26:07.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:07.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:07.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:07.096 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 07:26:07.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:07.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:07.096 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:26:07.096 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:07.096 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:07.096 | INFO | Uploaded 36GiB to GPU in 2.03526s (17.6881GiB/s) | 20260507 07:26:07.096 | INFO | Downloaded 34GiB to GPU in 1.95385s (17.4015GiB/s) | 20260507 07:26:12.231 | INFO | Analyzing chunk frames = [8156672,8157184) | 20260507 07:26:12.231 | INFO | MemInfo: { | 20260507 07:26:12.231 | INFO | "gpu" : " 17 GB", | 20260507 07:26:12.231 | INFO | "huge" : "", | 20260507 07:26:12.231 | INFO | "huge_shared" : " 27 GB", | 20260507 07:26:12.231 | INFO | "malloc" : " 14 GB", | 20260507 07:26:12.231 | INFO | "pinned" : " 29 GB" | 20260507 07:26:12.231 | INFO | } | 20260507 07:26:12.231 | INFO | Perf report for the last 5.13463 seconds: | 20260507 07:26:12.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:26:12.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:12.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:12.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:12.231 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 07:26:12.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:12.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:12.231 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 07:26:12.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:12.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:12.231 | INFO | Uploaded 36GiB to GPU in 2.01184s (17.8941GiB/s) | 20260507 07:26:12.231 | INFO | Downloaded 34GiB to GPU in 1.94707s (17.4621GiB/s) | 20260507 07:26:17.363 | INFO | Analyzing chunk frames = [8157184,8157696) | 20260507 07:26:17.363 | INFO | MemInfo: { | 20260507 07:26:17.363 | INFO | "gpu" : " 17 GB", | 20260507 07:26:17.363 | INFO | "huge" : "", | 20260507 07:26:17.363 | INFO | "huge_shared" : " 27 GB", | 20260507 07:26:17.363 | INFO | "malloc" : " 14 GB", | 20260507 07:26:17.363 | INFO | "pinned" : " 29 GB" | 20260507 07:26:17.363 | INFO | } | 20260507 07:26:17.363 | INFO | Perf report for the last 5.13207 seconds: | 20260507 07:26:17.363 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:26:17.363 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:17.363 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:17.363 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:17.363 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 07:26:17.363 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:17.363 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:17.363 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:26:17.363 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:17.363 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:17.363 | INFO | Uploaded 36GiB to GPU in 2.03478s (17.6923GiB/s) | 20260507 07:26:17.363 | INFO | Downloaded 34GiB to GPU in 1.923s (17.6807GiB/s) | 20260507 07:26:22.506 | INFO | Analyzing chunk frames = [8157696,8158208) | 20260507 07:26:22.506 | INFO | MemInfo: { | 20260507 07:26:22.506 | INFO | "gpu" : " 17 GB", | 20260507 07:26:22.506 | INFO | "huge" : "", | 20260507 07:26:22.506 | INFO | "huge_shared" : " 27 GB", | 20260507 07:26:22.506 | INFO | "malloc" : " 14 GB", | 20260507 07:26:22.506 | INFO | "pinned" : " 29 GB" | 20260507 07:26:22.506 | INFO | } | 20260507 07:26:22.506 | INFO | Perf report for the last 5.14278 seconds: | 20260507 07:26:22.506 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:26:22.506 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:22.506 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:22.506 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:22.506 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 07:26:22.506 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:22.506 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:22.506 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:26:22.506 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:22.506 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:22.506 | INFO | Uploaded 36GiB to GPU in 2.03387s (17.7002GiB/s) | 20260507 07:26:22.506 | INFO | Downloaded 34GiB to GPU in 1.90264s (17.8699GiB/s) | 20260507 07:26:27.650 | INFO | Analyzing chunk frames = [8158208,8158720) | 20260507 07:26:27.650 | INFO | MemInfo: { | 20260507 07:26:27.650 | INFO | "gpu" : " 17 GB", | 20260507 07:26:27.650 | INFO | "huge" : "", | 20260507 07:26:27.650 | INFO | "huge_shared" : " 27 GB", | 20260507 07:26:27.650 | INFO | "malloc" : " 14 GB", | 20260507 07:26:27.650 | INFO | "pinned" : " 29 GB" | 20260507 07:26:27.650 | INFO | } | 20260507 07:26:27.650 | INFO | Perf report for the last 5.14445 seconds: | 20260507 07:26:27.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:26:27.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:27.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:27.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:27.650 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:26:27.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:27.650 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:27.650 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 07:26:27.650 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:27.650 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:27.650 | INFO | Uploaded 36GiB to GPU in 1.98863s (18.1029GiB/s) | 20260507 07:26:27.650 | INFO | Downloaded 34GiB to GPU in 1.98263s (17.1489GiB/s) | 20260507 07:26:32.796 | INFO | Analyzing chunk frames = [8158720,8159232) | 20260507 07:26:32.796 | INFO | MemInfo: { | 20260507 07:26:32.796 | INFO | "gpu" : " 17 GB", | 20260507 07:26:32.796 | INFO | "huge" : "", | 20260507 07:26:32.796 | INFO | "huge_shared" : " 27 GB", | 20260507 07:26:32.796 | INFO | "malloc" : " 14 GB", | 20260507 07:26:32.796 | INFO | "pinned" : " 29 GB" | 20260507 07:26:32.796 | INFO | } | 20260507 07:26:32.796 | INFO | Perf report for the last 5.14604 seconds: | 20260507 07:26:32.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:26:32.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:32.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:32.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:32.796 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 07:26:32.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:32.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:32.796 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 07:26:32.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:32.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:32.796 | INFO | Uploaded 36GiB to GPU in 1.98895s (18.1GiB/s) | 20260507 07:26:32.797 | INFO | Downloaded 34GiB to GPU in 1.96923s (17.2657GiB/s) | 20260507 07:26:32.797 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:26:32.797 | INFO | Total mem usage: 18 GiB | 20260507 07:26:32.797 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:26:32.797 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:26:32.797 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:26:32.797 | INFO | High Water Breakdown: | 20260507 07:26:32.797 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:26:32.797 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:26:32.797 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:26:32.797 | INFO | ------------------------------------------ | 20260507 07:26:32.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:26:32.797 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:26:32.797 | INFO | Total mem usage: 26.5 GiB | 20260507 07:26:32.797 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:26:32.797 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:26:32.797 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:26:32.797 | INFO | High Water Breakdown: | 20260507 07:26:32.797 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:26:32.797 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:26:32.797 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:26:32.797 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:26:32.797 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:26:32.797 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:26:32.797 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:26:32.797 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:26:32.797 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:26:32.797 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:26:32.797 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:26:32.797 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:26:32.797 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:26:32.797 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:26:32.797 | INFO | ------------------------------------------ | 20260507 07:26:32.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:26:32.797 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:26:32.797 | INFO | Total mem usage: 25.6 GiB | 20260507 07:26:32.797 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:26:32.797 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:26:32.797 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:26:32.797 | INFO | High Water Breakdown: | 20260507 07:26:32.797 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:26:32.797 | INFO | ------------------------------------------ | 20260507 07:26:32.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:26:32.797 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:26:32.797 | INFO | Total mem usage: 15.7 GiB | 20260507 07:26:32.797 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:26:32.797 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:26:32.797 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:26:32.797 | INFO | High Water Breakdown: | 20260507 07:26:32.797 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:26:32.797 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:26:32.797 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:26:32.797 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:26:32.797 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:26:32.797 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:26:32.797 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:26:32.797 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:26:32.797 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:26:32.797 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:26:32.797 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:26:32.797 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:26:32.797 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:26:32.797 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:26:32.797 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:26:32.797 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:26:32.797 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:26:32.797 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:26:32.797 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:26:32.797 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:26:32.797 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:26:32.797 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:26:32.797 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:26:32.797 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:26:32.797 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:26:32.797 | INFO | ------------------------------------------ | 20260507 07:26:32.797 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:26:32.797 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:26:32.797 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:26:32.797 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:26:32.797 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.34e+01 / 1.30e+00 | 20260507 07:26:32.797 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.05e-01, 9.84e-03 / 1.12e-02 | 20260507 07:26:32.797 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.48e+00, 5.09e-01 / 2.06e-01 | 20260507 07:26:32.797 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.47e+00, 1.68e-01 / 1.21e-01 | 20260507 07:26:32.797 | INFO | PerfReport: 16.28%, DME -- 4.00e+01 / 4.49e+01, 3.90e+00 / 1.59e+00 | 20260507 07:26:32.797 | INFO | PerfReport: 57.71%, FrameLabeling -- 1.77e+01 / 1.98e+01, 1.32e+00 / 1.35e+00 | 20260507 07:26:32.797 | INFO | PerfReport: 6.87%, PulseAccumulating -- 2.11e+00 / 2.68e+00, 3.78e-01 / 1.75e-01 | 20260507 07:26:32.797 | INFO | PerfReport: 2.93%, Metrics -- 9.00e-01 / 1.21e+00, 2.40e-01 / 1.15e-01 | 20260507 07:26:32.797 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.03e-01 / 3.55e-01, 5.07e-02 / 1.47e-02 | 20260507 07:26:32.797 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:26:32.797 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:26:32.797 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:26:32.798 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 07:26:32.798 | INFO | PerfReport: 21.86%, Upload -- 2.10e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260507 07:26:32.798 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260507 07:26:32.798 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 07:26:32.798 | INFO | PerfReport: 1.55%, LockUpload -- 1.49e+00 / 0.00e+00, 4.87e+00 / 0.00e+00 | 20260507 07:26:32.798 | INFO | PerfReport: 1.27%, LockDownload -- 1.22e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260507 07:26:32.798 | INFO | PerfReport: 22.01%, LockCompute -- 2.12e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 07:26:32.798 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.48e-02 / 0.00e+00 | 20260507 07:26:32.892 | INFO | Starting a round of Baz IO | 20260507 07:26:35.981 | INFO | Populating RTMetrics report | 20260507 07:26:35.981 | INFO | RTMetrics report populated | 20260507 07:26:35.981 | INFO | Opening tmp RTMetrics json file | 20260507 07:26:35.999 | INFO | Serializing RTMetrics to json | 20260507 07:26:36.000 | INFO | Serialization done, swapping tmp file now... | 20260507 07:26:36.001 | INFO | RTMetrics done | 20260507 07:26:36.066 | INFO | Finished a round of Baz IO: Wrote 1.643 GiB in 3.173 seconds | 20260507 07:26:36.066 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.56% / 0.00% / 67.44% | 20260507 07:26:36.066 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.106124% / 3.97364e-06% / 7.15256e-05% | 20260507 07:26:36.066 | INFO | Baz IO overall: | 20260507 07:26:36.066 | INFO | BytesWritten : 4089936617264 | 20260507 07:26:36.066 | INFO | NumEvents : 1958175549615 | 20260507 07:26:37.926 | INFO | Analyzing chunk frames = [8159232,8159744) | 20260507 07:26:37.926 | INFO | MemInfo: { | 20260507 07:26:37.926 | INFO | "gpu" : " 17 GB", | 20260507 07:26:37.926 | INFO | "huge" : "", | 20260507 07:26:37.926 | INFO | "huge_shared" : " 27 GB", | 20260507 07:26:37.926 | INFO | "malloc" : " 14 GB", | 20260507 07:26:37.926 | INFO | "pinned" : " 29 GB" | 20260507 07:26:37.926 | INFO | } | 20260507 07:26:37.926 | INFO | Perf report for the last 5.12976 seconds: | 20260507 07:26:37.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:26:37.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:37.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:37.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:37.926 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260507 07:26:37.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:37.926 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 07:26:37.926 | INFO | PRE_HQ : 59.5%, 1.9, 58.7ms, 0 | 20260507 07:26:37.926 | INFO | BAZWRITER : 61.4%, 3.2, 102.0ms, 0 | 20260507 07:26:37.926 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:26:37.926 | INFO | Uploaded 36GiB to GPU in 2.05865s (17.4872GiB/s) | 20260507 07:26:37.926 | INFO | Downloaded 38GiB to GPU in 2.16956s (17.5151GiB/s) | 20260507 07:26:43.063 | INFO | Analyzing chunk frames = [8159744,8160256) | 20260507 07:26:43.063 | INFO | MemInfo: { | 20260507 07:26:43.063 | INFO | "gpu" : " 17 GB", | 20260507 07:26:43.063 | INFO | "huge" : "", | 20260507 07:26:43.063 | INFO | "huge_shared" : " 27 GB", | 20260507 07:26:43.063 | INFO | "malloc" : " 14 GB", | 20260507 07:26:43.063 | INFO | "pinned" : " 29 GB" | 20260507 07:26:43.063 | INFO | } | 20260507 07:26:43.063 | INFO | Perf report for the last 5.13688 seconds: | 20260507 07:26:43.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:26:43.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:43.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:43.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:43.063 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:26:43.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:43.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:43.063 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 07:26:43.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:43.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:43.064 | INFO | Uploaded 36GiB to GPU in 1.97386s (18.2383GiB/s) | 20260507 07:26:43.064 | INFO | Downloaded 34GiB to GPU in 1.94065s (17.5199GiB/s) | 20260507 07:26:48.203 | INFO | Analyzing chunk frames = [8160256,8160768) | 20260507 07:26:48.203 | INFO | MemInfo: { | 20260507 07:26:48.203 | INFO | "gpu" : " 17 GB", | 20260507 07:26:48.203 | INFO | "huge" : "", | 20260507 07:26:48.203 | INFO | "huge_shared" : " 27 GB", | 20260507 07:26:48.203 | INFO | "malloc" : " 14 GB", | 20260507 07:26:48.203 | INFO | "pinned" : " 29 GB" | 20260507 07:26:48.203 | INFO | } | 20260507 07:26:48.203 | INFO | Perf report for the last 5.14038 seconds: | 20260507 07:26:48.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:26:48.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:48.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:48.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:48.203 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 07:26:48.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:48.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:48.203 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:26:48.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:48.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:48.204 | INFO | Uploaded 36GiB to GPU in 2.04608s (17.5947GiB/s) | 20260507 07:26:48.204 | INFO | Downloaded 34GiB to GPU in 1.95267s (17.4121GiB/s) | 20260507 07:26:53.343 | INFO | Analyzing chunk frames = [8160768,8161280) | 20260507 07:26:53.343 | INFO | MemInfo: { | 20260507 07:26:53.343 | INFO | "gpu" : " 17 GB", | 20260507 07:26:53.343 | INFO | "huge" : "", | 20260507 07:26:53.343 | INFO | "huge_shared" : " 27 GB", | 20260507 07:26:53.343 | INFO | "malloc" : " 14 GB", | 20260507 07:26:53.343 | INFO | "pinned" : " 29 GB" | 20260507 07:26:53.343 | INFO | } | 20260507 07:26:53.343 | INFO | Perf report for the last 5.13984 seconds: | 20260507 07:26:53.343 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:26:53.343 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:53.343 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:53.343 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:53.343 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 07:26:53.343 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:53.343 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:53.343 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:26:53.343 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:53.343 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:53.343 | INFO | Uploaded 36GiB to GPU in 2.00967s (17.9134GiB/s) | 20260507 07:26:53.343 | INFO | Downloaded 34GiB to GPU in 1.94194s (17.5082GiB/s) | 20260507 07:26:58.486 | INFO | Analyzing chunk frames = [8161280,8161792) | 20260507 07:26:58.486 | INFO | MemInfo: { | 20260507 07:26:58.486 | INFO | "gpu" : " 17 GB", | 20260507 07:26:58.486 | INFO | "huge" : "", | 20260507 07:26:58.486 | INFO | "huge_shared" : " 27 GB", | 20260507 07:26:58.486 | INFO | "malloc" : " 14 GB", | 20260507 07:26:58.486 | INFO | "pinned" : " 29 GB" | 20260507 07:26:58.486 | INFO | } | 20260507 07:26:58.486 | INFO | Perf report for the last 5.14266 seconds: | 20260507 07:26:58.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:26:58.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:58.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:58.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:58.486 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 07:26:58.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:58.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:58.486 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:26:58.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:58.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:26:58.486 | INFO | Uploaded 36GiB to GPU in 2.04897s (17.5698GiB/s) | 20260507 07:26:58.486 | INFO | Downloaded 34GiB to GPU in 1.95133s (17.424GiB/s) | 20260507 07:27:03.627 | INFO | Analyzing chunk frames = [8161792,8162304) | 20260507 07:27:03.627 | INFO | MemInfo: { | 20260507 07:27:03.627 | INFO | "gpu" : " 17 GB", | 20260507 07:27:03.627 | INFO | "huge" : "", | 20260507 07:27:03.627 | INFO | "huge_shared" : " 27 GB", | 20260507 07:27:03.627 | INFO | "malloc" : " 14 GB", | 20260507 07:27:03.627 | INFO | "pinned" : " 29 GB" | 20260507 07:27:03.627 | INFO | } | 20260507 07:27:03.627 | INFO | Perf report for the last 5.1415 seconds: | 20260507 07:27:03.627 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:27:03.627 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:03.627 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:03.627 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:03.627 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 07:27:03.627 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:03.627 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:03.627 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:27:03.627 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:03.627 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:03.628 | INFO | Uploaded 36GiB to GPU in 2.06413s (17.4407GiB/s) | 20260507 07:27:03.628 | INFO | Downloaded 34GiB to GPU in 1.93687s (17.5541GiB/s) | 20260507 07:27:08.765 | INFO | Analyzing chunk frames = [8162304,8162816) | 20260507 07:27:08.765 | INFO | MemInfo: { | 20260507 07:27:08.765 | INFO | "gpu" : " 17 GB", | 20260507 07:27:08.765 | INFO | "huge" : "", | 20260507 07:27:08.765 | INFO | "huge_shared" : " 27 GB", | 20260507 07:27:08.765 | INFO | "malloc" : " 14 GB", | 20260507 07:27:08.765 | INFO | "pinned" : " 29 GB" | 20260507 07:27:08.765 | INFO | } | 20260507 07:27:08.765 | INFO | Perf report for the last 5.1379 seconds: | 20260507 07:27:08.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:27:08.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:08.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:08.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:08.765 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260507 07:27:08.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:08.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:08.765 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:27:08.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:08.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:08.766 | INFO | Uploaded 36GiB to GPU in 1.87595s (19.1903GiB/s) | 20260507 07:27:08.766 | INFO | Downloaded 34GiB to GPU in 1.97462s (17.2185GiB/s) | 20260507 07:27:13.903 | INFO | Analyzing chunk frames = [8162816,8163328) | 20260507 07:27:13.903 | INFO | MemInfo: { | 20260507 07:27:13.903 | INFO | "gpu" : " 17 GB", | 20260507 07:27:13.903 | INFO | "huge" : "", | 20260507 07:27:13.903 | INFO | "huge_shared" : " 27 GB", | 20260507 07:27:13.903 | INFO | "malloc" : " 14 GB", | 20260507 07:27:13.903 | INFO | "pinned" : " 29 GB" | 20260507 07:27:13.903 | INFO | } | 20260507 07:27:13.903 | INFO | Perf report for the last 5.1374 seconds: | 20260507 07:27:13.903 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:27:13.903 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:13.903 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:13.903 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:13.903 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:27:13.903 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:13.903 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:13.903 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 07:27:13.903 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:13.903 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:13.903 | INFO | Uploaded 36GiB to GPU in 1.98621s (18.125GiB/s) | 20260507 07:27:13.903 | INFO | Downloaded 34GiB to GPU in 1.95752s (17.3689GiB/s) | 20260507 07:27:14.000 | INFO | Starting a round of Baz IO | 20260507 07:27:17.094 | INFO | Populating RTMetrics report | 20260507 07:27:17.094 | INFO | RTMetrics report populated | 20260507 07:27:17.094 | INFO | Opening tmp RTMetrics json file | 20260507 07:27:17.105 | INFO | Serializing RTMetrics to json | 20260507 07:27:17.109 | INFO | Serialization done, swapping tmp file now... | 20260507 07:27:17.109 | INFO | RTMetrics done | 20260507 07:27:17.176 | INFO | Finished a round of Baz IO: Wrote 1.639 GiB in 3.176 seconds | 20260507 07:27:17.176 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.56% / 0.00% / 67.44% | 20260507 07:27:17.176 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.104717% / 0% / 7.54992e-05% | 20260507 07:27:17.176 | INFO | Baz IO overall: | 20260507 07:27:17.176 | INFO | BytesWritten : 4091696926016 | 20260507 07:27:17.176 | INFO | NumEvents : 1958997467856 | 20260507 07:27:19.048 | INFO | Analyzing chunk frames = [8163328,8163840) | 20260507 07:27:19.048 | INFO | MemInfo: { | 20260507 07:27:19.048 | INFO | "gpu" : " 17 GB", | 20260507 07:27:19.048 | INFO | "huge" : "", | 20260507 07:27:19.048 | INFO | "huge_shared" : " 27 GB", | 20260507 07:27:19.048 | INFO | "malloc" : " 14 GB", | 20260507 07:27:19.048 | INFO | "pinned" : " 29 GB" | 20260507 07:27:19.048 | INFO | } | 20260507 07:27:19.048 | INFO | Perf report for the last 5.14551 seconds: | 20260507 07:27:19.048 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:27:19.048 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:19.048 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:19.048 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:19.048 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 07:27:19.048 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:19.048 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 07:27:19.048 | INFO | PRE_HQ : 58.8%, 1.8, 54.5ms, 0 | 20260507 07:27:19.048 | INFO | BAZWRITER : 61.3%, 3.1, 97.5ms, 0 | 20260507 07:27:19.048 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:27:19.049 | INFO | Uploaded 36GiB to GPU in 2.06224s (17.4568GiB/s) | 20260507 07:27:19.049 | INFO | Downloaded 38GiB to GPU in 2.17721s (17.4535GiB/s) | 20260507 07:27:24.181 | INFO | Analyzing chunk frames = [8163840,8164352) | 20260507 07:27:24.181 | INFO | MemInfo: { | 20260507 07:27:24.181 | INFO | "gpu" : " 17 GB", | 20260507 07:27:24.181 | INFO | "huge" : "", | 20260507 07:27:24.181 | INFO | "huge_shared" : " 27 GB", | 20260507 07:27:24.181 | INFO | "malloc" : " 14 GB", | 20260507 07:27:24.181 | INFO | "pinned" : " 29 GB" | 20260507 07:27:24.181 | INFO | } | 20260507 07:27:24.181 | INFO | Perf report for the last 5.13257 seconds: | 20260507 07:27:24.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:27:24.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:24.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:24.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:24.181 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:27:24.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:24.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:24.181 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 07:27:24.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:24.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:24.181 | INFO | Uploaded 36GiB to GPU in 2.00428s (17.9616GiB/s) | 20260507 07:27:24.181 | INFO | Downloaded 34GiB to GPU in 1.94268s (17.5016GiB/s) | 20260507 07:27:29.314 | INFO | Analyzing chunk frames = [8164352,8164864) | 20260507 07:27:29.314 | INFO | MemInfo: { | 20260507 07:27:29.314 | INFO | "gpu" : " 17 GB", | 20260507 07:27:29.314 | INFO | "huge" : "", | 20260507 07:27:29.314 | INFO | "huge_shared" : " 27 GB", | 20260507 07:27:29.314 | INFO | "malloc" : " 14 GB", | 20260507 07:27:29.314 | INFO | "pinned" : " 29 GB" | 20260507 07:27:29.314 | INFO | } | 20260507 07:27:29.314 | INFO | Perf report for the last 5.13369 seconds: | 20260507 07:27:29.314 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:27:29.314 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:29.314 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:29.314 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:29.315 | INFO | ANALYSIS : 60.7%, 3.0, 93.8ms, 0 | 20260507 07:27:29.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:29.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:29.315 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:27:29.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:29.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:29.315 | INFO | Uploaded 36GiB to GPU in 2.0577s (17.4953GiB/s) | 20260507 07:27:29.315 | INFO | Downloaded 34GiB to GPU in 1.9506s (17.4306GiB/s) | 20260507 07:27:34.458 | INFO | Analyzing chunk frames = [8164864,8165376) | 20260507 07:27:34.458 | INFO | MemInfo: { | 20260507 07:27:34.458 | INFO | "gpu" : " 17 GB", | 20260507 07:27:34.458 | INFO | "huge" : "", | 20260507 07:27:34.458 | INFO | "huge_shared" : " 27 GB", | 20260507 07:27:34.458 | INFO | "malloc" : " 14 GB", | 20260507 07:27:34.458 | INFO | "pinned" : " 29 GB" | 20260507 07:27:34.458 | INFO | } | 20260507 07:27:34.458 | INFO | Perf report for the last 5.14349 seconds: | 20260507 07:27:34.458 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:27:34.458 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:34.458 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:34.458 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:34.458 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 07:27:34.458 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:34.458 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:34.458 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:27:34.458 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:34.458 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:34.459 | INFO | Uploaded 36GiB to GPU in 1.98789s (18.1096GiB/s) | 20260507 07:27:34.459 | INFO | Downloaded 34GiB to GPU in 1.92179s (17.6918GiB/s) | 20260507 07:27:39.607 | INFO | Analyzing chunk frames = [8165376,8165888) | 20260507 07:27:39.607 | INFO | MemInfo: { | 20260507 07:27:39.607 | INFO | "gpu" : " 17 GB", | 20260507 07:27:39.607 | INFO | "huge" : "", | 20260507 07:27:39.607 | INFO | "huge_shared" : " 27 GB", | 20260507 07:27:39.607 | INFO | "malloc" : " 14 GB", | 20260507 07:27:39.607 | INFO | "pinned" : " 29 GB" | 20260507 07:27:39.607 | INFO | } | 20260507 07:27:39.607 | INFO | Perf report for the last 5.14892 seconds: | 20260507 07:27:39.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:27:39.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:39.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:39.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:39.607 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 07:27:39.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:39.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:39.607 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:27:39.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:39.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:39.607 | INFO | Uploaded 36GiB to GPU in 2.05817s (17.4912GiB/s) | 20260507 07:27:39.607 | INFO | Downloaded 34GiB to GPU in 1.94974s (17.4382GiB/s) | 20260507 07:27:44.741 | INFO | Analyzing chunk frames = [8165888,8166400) | 20260507 07:27:44.741 | INFO | MemInfo: { | 20260507 07:27:44.741 | INFO | "gpu" : " 17 GB", | 20260507 07:27:44.741 | INFO | "huge" : "", | 20260507 07:27:44.741 | INFO | "huge_shared" : " 27 GB", | 20260507 07:27:44.741 | INFO | "malloc" : " 14 GB", | 20260507 07:27:44.741 | INFO | "pinned" : " 29 GB" | 20260507 07:27:44.741 | INFO | } | 20260507 07:27:44.741 | INFO | Perf report for the last 5.13388 seconds: | 20260507 07:27:44.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:27:44.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:44.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:44.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:44.741 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:27:44.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:44.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:44.741 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:27:44.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:44.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:44.741 | INFO | Uploaded 36GiB to GPU in 2.07123s (17.381GiB/s) | 20260507 07:27:44.741 | INFO | Downloaded 34GiB to GPU in 1.93675s (17.5552GiB/s) | 20260507 07:27:49.873 | INFO | Analyzing chunk frames = [8166400,8166912) | 20260507 07:27:49.873 | INFO | MemInfo: { | 20260507 07:27:49.873 | INFO | "gpu" : " 17 GB", | 20260507 07:27:49.873 | INFO | "huge" : "", | 20260507 07:27:49.873 | INFO | "huge_shared" : " 27 GB", | 20260507 07:27:49.873 | INFO | "malloc" : " 14 GB", | 20260507 07:27:49.873 | INFO | "pinned" : " 29 GB" | 20260507 07:27:49.873 | INFO | } | 20260507 07:27:49.873 | INFO | Perf report for the last 5.13236 seconds: | 20260507 07:27:49.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:27:49.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:49.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:49.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:49.873 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:27:49.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:49.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:49.873 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 07:27:49.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:49.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:49.873 | INFO | Uploaded 36GiB to GPU in 1.95968s (18.3704GiB/s) | 20260507 07:27:49.873 | INFO | Downloaded 34GiB to GPU in 1.98297s (17.146GiB/s) | 20260507 07:27:55.015 | INFO | Analyzing chunk frames = [8166912,8167424) | 20260507 07:27:55.015 | INFO | MemInfo: { | 20260507 07:27:55.015 | INFO | "gpu" : " 17 GB", | 20260507 07:27:55.015 | INFO | "huge" : "", | 20260507 07:27:55.015 | INFO | "huge_shared" : " 27 GB", | 20260507 07:27:55.015 | INFO | "malloc" : " 14 GB", | 20260507 07:27:55.015 | INFO | "pinned" : " 29 GB" | 20260507 07:27:55.015 | INFO | } | 20260507 07:27:55.015 | INFO | Perf report for the last 5.14147 seconds: | 20260507 07:27:55.015 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:27:55.015 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:55.015 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:55.015 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:55.015 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:27:55.015 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:55.015 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:55.015 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:27:55.015 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:55.015 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:27:55.015 | INFO | Uploaded 36GiB to GPU in 1.97366s (18.2402GiB/s) | 20260507 07:27:55.015 | INFO | Downloaded 34GiB to GPU in 1.95536s (17.3881GiB/s) | 20260507 07:27:55.015 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:27:55.015 | INFO | Total mem usage: 18 GiB | 20260507 07:27:55.015 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:27:55.015 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 07:27:55.015 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:27:55.015 | INFO | High Water Breakdown: | 20260507 07:27:55.015 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:27:55.015 | INFO | 39.7% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:27:55.015 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:27:55.015 | INFO | ------------------------------------------ | 20260507 07:27:55.015 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:27:55.015 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:27:55.015 | INFO | Total mem usage: 26.5 GiB | 20260507 07:27:55.015 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:27:55.015 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:27:55.015 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:27:55.015 | INFO | High Water Breakdown: | 20260507 07:27:55.015 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:27:55.015 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:27:55.015 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:27:55.015 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:27:55.015 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:27:55.015 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:27:55.015 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:27:55.015 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:27:55.015 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:27:55.015 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:27:55.015 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:27:55.015 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:27:55.015 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:27:55.015 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:27:55.015 | INFO | ------------------------------------------ | 20260507 07:27:55.015 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:27:55.015 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:27:55.015 | INFO | Total mem usage: 25.6 GiB | 20260507 07:27:55.015 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:27:55.015 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:27:55.015 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:27:55.015 | INFO | High Water Breakdown: | 20260507 07:27:55.015 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:27:55.015 | INFO | ------------------------------------------ | 20260507 07:27:55.015 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:27:55.015 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:27:55.015 | INFO | Total mem usage: 15.7 GiB | 20260507 07:27:55.015 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:27:55.015 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:27:55.015 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:27:55.015 | INFO | High Water Breakdown: | 20260507 07:27:55.015 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:27:55.015 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:27:55.015 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:27:55.015 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:27:55.015 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:27:55.016 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:27:55.016 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:27:55.016 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:27:55.016 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:27:55.016 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:27:55.016 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:27:55.016 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:27:55.016 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:27:55.016 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:27:55.016 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:27:55.016 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:27:55.016 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:27:55.016 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:27:55.016 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:27:55.016 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:27:55.016 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:27:55.016 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:27:55.016 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:27:55.016 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:27:55.016 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:27:55.016 | INFO | ------------------------------------------ | 20260507 07:27:55.016 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:27:55.016 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:27:55.016 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:27:55.016 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:27:55.016 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.43e+01, 1.33e+01 / 1.07e+00 | 20260507 07:27:55.016 | INFO | PerfReport: 0.30%, AggregateStats -- 9.05e-02 / 1.03e-01, 9.83e-03 / 4.36e-03 | 20260507 07:27:55.016 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.47e+00, 4.91e-01 / 2.21e-01 | 20260507 07:27:55.016 | INFO | PerfReport: 3.21%, Histogram -- 9.83e-01 / 1.49e+00, 1.61e-01 / 8.79e-02 | 20260507 07:27:55.016 | INFO | PerfReport: 16.27%, DME -- 3.99e+01 / 4.43e+01, 3.81e+00 / 1.07e+00 | 20260507 07:27:55.016 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.22e+00 / 1.50e+00 | 20260507 07:27:55.016 | INFO | PerfReport: 6.94%, PulseAccumulating -- 2.12e+00 / 2.66e+00, 3.90e-01 / 2.35e-01 | 20260507 07:27:55.016 | INFO | PerfReport: 2.94%, Metrics -- 9.02e-01 / 1.21e+00, 2.44e-01 / 1.01e-01 | 20260507 07:27:55.016 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.02e-01 / 3.62e-01, 5.08e-02 / 1.91e-02 | 20260507 07:27:55.016 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:27:55.016 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:27:55.016 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:27:55.016 | INFO | PerfReport: Full Execution -- 9.60e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 07:27:55.016 | INFO | PerfReport: 21.86%, Upload -- 2.10e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260507 07:27:55.016 | INFO | PerfReport: 21.45%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 07:27:55.016 | INFO | PerfReport: 31.89%, Compute -- 3.06e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 07:27:55.016 | INFO | PerfReport: 1.52%, LockUpload -- 1.46e+00 / 0.00e+00, 4.92e+00 / 0.00e+00 | 20260507 07:27:55.016 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260507 07:27:55.016 | INFO | PerfReport: 21.98%, LockCompute -- 2.11e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 07:27:55.016 | INFO | PerfReport: 0.01%, Misc -- 1.41e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260507 07:27:55.111 | INFO | Starting a round of Baz IO | 20260507 07:27:58.208 | INFO | Populating RTMetrics report | 20260507 07:27:58.208 | INFO | RTMetrics report populated | 20260507 07:27:58.208 | INFO | Opening tmp RTMetrics json file | 20260507 07:27:58.225 | INFO | Serializing RTMetrics to json | 20260507 07:27:58.226 | INFO | Serialization done, swapping tmp file now... | 20260507 07:27:58.226 | INFO | RTMetrics done | 20260507 07:27:58.291 | INFO | Finished a round of Baz IO: Wrote 1.64 GiB in 3.179 seconds | 20260507 07:27:58.291 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.56% / 0.00% / 67.44% | 20260507 07:27:58.291 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.105445% / 3.97364e-06% / 4.76837e-05% | 20260507 07:27:58.291 | INFO | Baz IO overall: | 20260507 07:27:58.291 | INFO | BytesWritten : 4093457813273 | 20260507 07:27:58.291 | INFO | NumEvents : 1959819906030 | 20260507 07:28:00.157 | INFO | Analyzing chunk frames = [8167424,8167936) | 20260507 07:28:00.157 | INFO | MemInfo: { | 20260507 07:28:00.157 | INFO | "gpu" : " 17 GB", | 20260507 07:28:00.157 | INFO | "huge" : "", | 20260507 07:28:00.157 | INFO | "huge_shared" : " 27 GB", | 20260507 07:28:00.157 | INFO | "malloc" : " 14 GB", | 20260507 07:28:00.157 | INFO | "pinned" : " 29 GB" | 20260507 07:28:00.157 | INFO | } | 20260507 07:28:00.157 | INFO | Perf report for the last 5.14249 seconds: | 20260507 07:28:00.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:28:00.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:00.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:00.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:00.157 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 07:28:00.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:00.157 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 07:28:00.157 | INFO | PRE_HQ : 58.8%, 1.8, 54.3ms, 0 | 20260507 07:28:00.157 | INFO | BAZWRITER : 61.4%, 3.1, 97.5ms, 0 | 20260507 07:28:00.157 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:28:00.158 | INFO | Uploaded 36GiB to GPU in 2.0506s (17.5559GiB/s) | 20260507 07:28:00.158 | INFO | Downloaded 38GiB to GPU in 2.17347s (17.4836GiB/s) | 20260507 07:28:05.297 | INFO | Analyzing chunk frames = [8167936,8168448) | 20260507 07:28:05.297 | INFO | MemInfo: { | 20260507 07:28:05.297 | INFO | "gpu" : " 17 GB", | 20260507 07:28:05.297 | INFO | "huge" : "", | 20260507 07:28:05.297 | INFO | "huge_shared" : " 27 GB", | 20260507 07:28:05.297 | INFO | "malloc" : " 14 GB", | 20260507 07:28:05.297 | INFO | "pinned" : " 29 GB" | 20260507 07:28:05.297 | INFO | } | 20260507 07:28:05.298 | INFO | Perf report for the last 5.14034 seconds: | 20260507 07:28:05.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:28:05.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:05.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:05.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:05.298 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:28:05.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:05.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:05.298 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:28:05.298 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:05.298 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:05.298 | INFO | Uploaded 36GiB to GPU in 1.98626s (18.1245GiB/s) | 20260507 07:28:05.298 | INFO | Downloaded 34GiB to GPU in 1.92897s (17.626GiB/s) | 20260507 07:28:10.440 | INFO | Analyzing chunk frames = [8168448,8168960) | 20260507 07:28:10.440 | INFO | MemInfo: { | 20260507 07:28:10.440 | INFO | "gpu" : " 17 GB", | 20260507 07:28:10.440 | INFO | "huge" : "", | 20260507 07:28:10.440 | INFO | "huge_shared" : " 27 GB", | 20260507 07:28:10.440 | INFO | "malloc" : " 14 GB", | 20260507 07:28:10.440 | INFO | "pinned" : " 29 GB" | 20260507 07:28:10.440 | INFO | } | 20260507 07:28:10.440 | INFO | Perf report for the last 5.14281 seconds: | 20260507 07:28:10.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:28:10.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:10.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:10.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:10.440 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 07:28:10.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:10.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:10.440 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:28:10.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:10.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:10.441 | INFO | Uploaded 36GiB to GPU in 2.05852s (17.4883GiB/s) | 20260507 07:28:10.441 | INFO | Downloaded 34GiB to GPU in 1.95034s (17.4329GiB/s) | 20260507 07:28:15.574 | INFO | Analyzing chunk frames = [8168960,8169472) | 20260507 07:28:15.574 | INFO | MemInfo: { | 20260507 07:28:15.574 | INFO | "gpu" : " 17 GB", | 20260507 07:28:15.574 | INFO | "huge" : "", | 20260507 07:28:15.574 | INFO | "huge_shared" : " 27 GB", | 20260507 07:28:15.574 | INFO | "malloc" : " 14 GB", | 20260507 07:28:15.574 | INFO | "pinned" : " 29 GB" | 20260507 07:28:15.574 | INFO | } | 20260507 07:28:15.574 | INFO | Perf report for the last 5.13332 seconds: | 20260507 07:28:15.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:28:15.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:15.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:15.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:15.574 | INFO | ANALYSIS : 60.4%, 2.9, 93.1ms, 0 | 20260507 07:28:15.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:15.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:15.574 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:28:15.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:15.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:15.574 | INFO | Uploaded 36GiB to GPU in 2.03716s (17.6717GiB/s) | 20260507 07:28:15.574 | INFO | Downloaded 34GiB to GPU in 1.94833s (17.4508GiB/s) | 20260507 07:28:20.719 | INFO | Analyzing chunk frames = [8169472,8169984) | 20260507 07:28:20.719 | INFO | MemInfo: { | 20260507 07:28:20.719 | INFO | "gpu" : " 17 GB", | 20260507 07:28:20.719 | INFO | "huge" : "", | 20260507 07:28:20.719 | INFO | "huge_shared" : " 27 GB", | 20260507 07:28:20.719 | INFO | "malloc" : " 14 GB", | 20260507 07:28:20.719 | INFO | "pinned" : " 29 GB" | 20260507 07:28:20.719 | INFO | } | 20260507 07:28:20.719 | INFO | Perf report for the last 5.145 seconds: | 20260507 07:28:20.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:28:20.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:20.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:20.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:20.719 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 07:28:20.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:20.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:20.719 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:28:20.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:20.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:20.719 | INFO | Uploaded 36GiB to GPU in 2.03886s (17.657GiB/s) | 20260507 07:28:20.719 | INFO | Downloaded 34GiB to GPU in 1.91211s (17.7814GiB/s) | 20260507 07:28:25.854 | INFO | Analyzing chunk frames = [8169984,8170496) | 20260507 07:28:25.854 | INFO | MemInfo: { | 20260507 07:28:25.854 | INFO | "gpu" : " 17 GB", | 20260507 07:28:25.854 | INFO | "huge" : "", | 20260507 07:28:25.854 | INFO | "huge_shared" : " 27 GB", | 20260507 07:28:25.854 | INFO | "malloc" : " 14 GB", | 20260507 07:28:25.854 | INFO | "pinned" : " 29 GB" | 20260507 07:28:25.854 | INFO | } | 20260507 07:28:25.854 | INFO | Perf report for the last 5.13556 seconds: | 20260507 07:28:25.854 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:28:25.854 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:25.854 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:25.854 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:25.854 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 07:28:25.854 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:25.854 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:25.854 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 07:28:25.854 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:25.854 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:25.854 | INFO | Uploaded 36GiB to GPU in 2.07491s (17.3502GiB/s) | 20260507 07:28:25.855 | INFO | Downloaded 34GiB to GPU in 1.9408s (17.5185GiB/s) | 20260507 07:28:30.995 | INFO | Analyzing chunk frames = [8170496,8171008) | 20260507 07:28:30.995 | INFO | MemInfo: { | 20260507 07:28:30.995 | INFO | "gpu" : " 17 GB", | 20260507 07:28:30.995 | INFO | "huge" : "", | 20260507 07:28:30.995 | INFO | "huge_shared" : " 27 GB", | 20260507 07:28:30.995 | INFO | "malloc" : " 14 GB", | 20260507 07:28:30.995 | INFO | "pinned" : " 29 GB" | 20260507 07:28:30.995 | INFO | } | 20260507 07:28:30.995 | INFO | Perf report for the last 5.14105 seconds: | 20260507 07:28:30.995 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:28:30.995 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:30.995 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:30.995 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:30.995 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:28:30.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:30.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:30.995 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:28:30.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:30.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:30.995 | INFO | Uploaded 36GiB to GPU in 1.90724s (18.8754GiB/s) | 20260507 07:28:30.995 | INFO | Downloaded 34GiB to GPU in 1.97177s (17.2434GiB/s) | 20260507 07:28:36.130 | INFO | Analyzing chunk frames = [8171008,8171520) | 20260507 07:28:36.130 | INFO | MemInfo: { | 20260507 07:28:36.130 | INFO | "gpu" : " 17 GB", | 20260507 07:28:36.130 | INFO | "huge" : "", | 20260507 07:28:36.130 | INFO | "huge_shared" : " 27 GB", | 20260507 07:28:36.130 | INFO | "malloc" : " 14 GB", | 20260507 07:28:36.130 | INFO | "pinned" : " 29 GB" | 20260507 07:28:36.130 | INFO | } | 20260507 07:28:36.131 | INFO | Perf report for the last 5.13517 seconds: | 20260507 07:28:36.131 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:28:36.131 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:36.131 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:36.131 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:36.131 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 07:28:36.131 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:36.131 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:36.131 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 07:28:36.131 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:36.131 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:36.131 | INFO | Uploaded 36GiB to GPU in 1.99388s (18.0553GiB/s) | 20260507 07:28:36.131 | INFO | Downloaded 34GiB to GPU in 1.95942s (17.3521GiB/s) | 20260507 07:28:36.228 | INFO | Starting a round of Baz IO | 20260507 07:28:39.323 | INFO | Populating RTMetrics report | 20260507 07:28:39.323 | INFO | RTMetrics report populated | 20260507 07:28:39.323 | INFO | Opening tmp RTMetrics json file | 20260507 07:28:39.334 | INFO | Serializing RTMetrics to json | 20260507 07:28:39.338 | INFO | Serialization done, swapping tmp file now... | 20260507 07:28:39.338 | INFO | RTMetrics done | 20260507 07:28:39.410 | INFO | Finished a round of Baz IO: Wrote 1.641 GiB in 3.182 seconds | 20260507 07:28:39.410 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.56% / 0.00% / 67.44% | 20260507 07:28:39.410 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.10521% / 0% / 6.35783e-05% | 20260507 07:28:39.410 | INFO | Baz IO overall: | 20260507 07:28:39.410 | INFO | BytesWritten : 4095219692968 | 20260507 07:28:39.410 | INFO | NumEvents : 1960642952410 | 20260507 07:28:41.267 | INFO | Analyzing chunk frames = [8171520,8172032) | 20260507 07:28:41.267 | INFO | MemInfo: { | 20260507 07:28:41.267 | INFO | "gpu" : " 17 GB", | 20260507 07:28:41.267 | INFO | "huge" : "", | 20260507 07:28:41.267 | INFO | "huge_shared" : " 27 GB", | 20260507 07:28:41.267 | INFO | "malloc" : " 14 GB", | 20260507 07:28:41.267 | INFO | "pinned" : " 29 GB" | 20260507 07:28:41.267 | INFO | } | 20260507 07:28:41.267 | INFO | Perf report for the last 5.13634 seconds: | 20260507 07:28:41.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:28:41.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:41.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:41.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:41.267 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 07:28:41.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:41.267 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 07:28:41.267 | INFO | PRE_HQ : 59.8%, 1.9, 58.3ms, 0 | 20260507 07:28:41.267 | INFO | BAZWRITER : 61.5%, 3.2, 100.9ms, 0 | 20260507 07:28:41.267 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:28:41.267 | INFO | Uploaded 36GiB to GPU in 2.07032s (17.3886GiB/s) | 20260507 07:28:41.267 | INFO | Downloaded 38GiB to GPU in 2.18593s (17.3839GiB/s) | 20260507 07:28:46.409 | INFO | Analyzing chunk frames = [8172032,8172544) | 20260507 07:28:46.409 | INFO | MemInfo: { | 20260507 07:28:46.409 | INFO | "gpu" : " 17 GB", | 20260507 07:28:46.409 | INFO | "huge" : "", | 20260507 07:28:46.409 | INFO | "huge_shared" : " 27 GB", | 20260507 07:28:46.409 | INFO | "malloc" : " 14 GB", | 20260507 07:28:46.409 | INFO | "pinned" : " 29 GB" | 20260507 07:28:46.409 | INFO | } | 20260507 07:28:46.409 | INFO | Perf report for the last 5.14258 seconds: | 20260507 07:28:46.409 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:28:46.409 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:46.409 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:46.409 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:46.409 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 07:28:46.409 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:46.410 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:46.410 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:28:46.410 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:46.410 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:46.410 | INFO | Uploaded 36GiB to GPU in 2.05226s (17.5416GiB/s) | 20260507 07:28:46.410 | INFO | Downloaded 34GiB to GPU in 1.94428s (17.4872GiB/s) | 20260507 07:28:51.554 | INFO | Analyzing chunk frames = [8172544,8173056) | 20260507 07:28:51.554 | INFO | MemInfo: { | 20260507 07:28:51.554 | INFO | "gpu" : " 17 GB", | 20260507 07:28:51.554 | INFO | "huge" : "", | 20260507 07:28:51.554 | INFO | "huge_shared" : " 27 GB", | 20260507 07:28:51.554 | INFO | "malloc" : " 14 GB", | 20260507 07:28:51.554 | INFO | "pinned" : " 29 GB" | 20260507 07:28:51.554 | INFO | } | 20260507 07:28:51.554 | INFO | Perf report for the last 5.14442 seconds: | 20260507 07:28:51.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:28:51.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:51.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:51.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:51.554 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 07:28:51.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:51.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:51.554 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:28:51.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:51.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:51.554 | INFO | Uploaded 36GiB to GPU in 2.05133s (17.5496GiB/s) | 20260507 07:28:51.554 | INFO | Downloaded 34GiB to GPU in 1.94918s (17.4433GiB/s) | 20260507 07:28:56.687 | INFO | Analyzing chunk frames = [8173056,8173568) | 20260507 07:28:56.687 | INFO | MemInfo: { | 20260507 07:28:56.687 | INFO | "gpu" : " 17 GB", | 20260507 07:28:56.687 | INFO | "huge" : "", | 20260507 07:28:56.687 | INFO | "huge_shared" : " 27 GB", | 20260507 07:28:56.687 | INFO | "malloc" : " 14 GB", | 20260507 07:28:56.687 | INFO | "pinned" : " 29 GB" | 20260507 07:28:56.687 | INFO | } | 20260507 07:28:56.687 | INFO | Perf report for the last 5.13285 seconds: | 20260507 07:28:56.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:28:56.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:56.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:56.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:56.687 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 07:28:56.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:56.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:56.687 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:28:56.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:56.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:28:56.687 | INFO | Uploaded 36GiB to GPU in 2.05861s (17.4875GiB/s) | 20260507 07:28:56.687 | INFO | Downloaded 34GiB to GPU in 1.94848s (17.4495GiB/s) | 20260507 07:29:01.824 | INFO | Analyzing chunk frames = [8173568,8174080) | 20260507 07:29:01.824 | INFO | MemInfo: { | 20260507 07:29:01.824 | INFO | "gpu" : " 17 GB", | 20260507 07:29:01.824 | INFO | "huge" : "", | 20260507 07:29:01.824 | INFO | "huge_shared" : " 27 GB", | 20260507 07:29:01.824 | INFO | "malloc" : " 14 GB", | 20260507 07:29:01.824 | INFO | "pinned" : " 29 GB" | 20260507 07:29:01.824 | INFO | } | 20260507 07:29:01.824 | INFO | Perf report for the last 5.13715 seconds: | 20260507 07:29:01.824 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:29:01.824 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:01.824 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:01.824 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:01.824 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 07:29:01.824 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:01.824 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:01.824 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:29:01.824 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:01.824 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:01.824 | INFO | Uploaded 36GiB to GPU in 2.03623s (17.6797GiB/s) | 20260507 07:29:01.824 | INFO | Downloaded 34GiB to GPU in 1.91426s (17.7615GiB/s) | 20260507 07:29:06.970 | INFO | Analyzing chunk frames = [8174080,8174592) | 20260507 07:29:06.970 | INFO | MemInfo: { | 20260507 07:29:06.970 | INFO | "gpu" : " 17 GB", | 20260507 07:29:06.970 | INFO | "huge" : "", | 20260507 07:29:06.970 | INFO | "huge_shared" : " 27 GB", | 20260507 07:29:06.970 | INFO | "malloc" : " 14 GB", | 20260507 07:29:06.970 | INFO | "pinned" : " 29 GB" | 20260507 07:29:06.970 | INFO | } | 20260507 07:29:06.970 | INFO | Perf report for the last 5.14658 seconds: | 20260507 07:29:06.971 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:29:06.971 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:06.971 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:06.971 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:06.971 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 07:29:06.971 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:06.971 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:06.971 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:29:06.971 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:06.971 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:06.971 | INFO | Uploaded 36GiB to GPU in 2.03331s (17.7051GiB/s) | 20260507 07:29:06.971 | INFO | Downloaded 34GiB to GPU in 1.90048s (17.8902GiB/s) | 20260507 07:29:12.110 | INFO | Analyzing chunk frames = [8174592,8175104) | 20260507 07:29:12.110 | INFO | MemInfo: { | 20260507 07:29:12.110 | INFO | "gpu" : " 17 GB", | 20260507 07:29:12.110 | INFO | "huge" : "", | 20260507 07:29:12.110 | INFO | "huge_shared" : " 27 GB", | 20260507 07:29:12.110 | INFO | "malloc" : " 14 GB", | 20260507 07:29:12.110 | INFO | "pinned" : " 29 GB" | 20260507 07:29:12.110 | INFO | } | 20260507 07:29:12.110 | INFO | Perf report for the last 5.13976 seconds: | 20260507 07:29:12.110 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:29:12.110 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:12.110 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:12.110 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:12.110 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:29:12.110 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:12.110 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:12.110 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 07:29:12.110 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:12.110 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:12.110 | INFO | Uploaded 36GiB to GPU in 1.92852s (18.6672GiB/s) | 20260507 07:29:12.110 | INFO | Downloaded 34GiB to GPU in 1.97013s (17.2577GiB/s) | 20260507 07:29:17.248 | INFO | Analyzing chunk frames = [8175104,8175616) | 20260507 07:29:17.248 | INFO | MemInfo: { | 20260507 07:29:17.248 | INFO | "gpu" : " 17 GB", | 20260507 07:29:17.248 | INFO | "huge" : "", | 20260507 07:29:17.248 | INFO | "huge_shared" : " 27 GB", | 20260507 07:29:17.248 | INFO | "malloc" : " 14 GB", | 20260507 07:29:17.248 | INFO | "pinned" : " 29 GB" | 20260507 07:29:17.248 | INFO | } | 20260507 07:29:17.248 | INFO | Perf report for the last 5.13816 seconds: | 20260507 07:29:17.248 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:29:17.248 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:17.248 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:17.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:17.249 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:29:17.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:17.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:17.249 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 07:29:17.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:17.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:17.249 | INFO | Uploaded 36GiB to GPU in 1.99282s (18.0649GiB/s) | 20260507 07:29:17.249 | INFO | Downloaded 34GiB to GPU in 1.95616s (17.381GiB/s) | 20260507 07:29:17.249 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:29:17.249 | INFO | Total mem usage: 18 GiB | 20260507 07:29:17.249 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:29:17.249 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -27%) | 20260507 07:29:17.249 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:29:17.249 | INFO | High Water Breakdown: | 20260507 07:29:17.249 | INFO | 60.1% : 8.52 GiB ( 74.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:29:17.249 | INFO | 39.7% : 5.63 GiB ( 84.2% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:29:17.249 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:29:17.249 | INFO | ------------------------------------------ | 20260507 07:29:17.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:29:17.249 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:29:17.249 | INFO | Total mem usage: 26.5 GiB | 20260507 07:29:17.249 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:29:17.249 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:29:17.249 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:29:17.249 | INFO | High Water Breakdown: | 20260507 07:29:17.249 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:29:17.249 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:29:17.249 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:29:17.249 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:29:17.249 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:29:17.249 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:29:17.249 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:29:17.249 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:29:17.249 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:29:17.249 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:29:17.249 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:29:17.249 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:29:17.249 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:29:17.249 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:29:17.249 | INFO | ------------------------------------------ | 20260507 07:29:17.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:29:17.249 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:29:17.249 | INFO | Total mem usage: 25.6 GiB | 20260507 07:29:17.249 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:29:17.249 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:29:17.249 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:29:17.249 | INFO | High Water Breakdown: | 20260507 07:29:17.249 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:29:17.249 | INFO | ------------------------------------------ | 20260507 07:29:17.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:29:17.249 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:29:17.249 | INFO | Total mem usage: 15.7 GiB | 20260507 07:29:17.249 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:29:17.249 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:29:17.249 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:29:17.249 | INFO | High Water Breakdown: | 20260507 07:29:17.249 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:29:17.249 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:29:17.249 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:29:17.249 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:29:17.249 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:29:17.249 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:29:17.249 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:29:17.249 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:29:17.249 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:29:17.249 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:29:17.249 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:29:17.249 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:29:17.249 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:29:17.249 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:29:17.249 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:29:17.249 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:29:17.249 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:29:17.249 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:29:17.249 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:29:17.249 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:29:17.249 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:29:17.249 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:29:17.249 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:29:17.249 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:29:17.249 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:29:17.249 | INFO | ------------------------------------------ | 20260507 07:29:17.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:29:17.250 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:29:17.250 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:29:17.250 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:29:17.250 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.35e+01 / 1.28e+00 | 20260507 07:29:17.250 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.02e-01, 9.20e-03 / 4.43e-03 | 20260507 07:29:17.250 | INFO | PerfReport: 11.66%, Baseline -- 3.57e+00 / 4.49e+00, 5.00e-01 / 1.87e-01 | 20260507 07:29:17.250 | INFO | PerfReport: 3.22%, Histogram -- 9.86e-01 / 1.49e+00, 1.66e-01 / 1.12e-01 | 20260507 07:29:17.250 | INFO | PerfReport: 16.37%, DME -- 4.01e+01 / 4.54e+01, 4.00e+00 / 1.56e+00 | 20260507 07:29:17.250 | INFO | PerfReport: 57.62%, FrameLabeling -- 1.77e+01 / 1.91e+01, 1.20e+00 / 8.03e-01 | 20260507 07:29:17.250 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.12e+00 / 2.72e+00, 3.86e-01 / 1.43e-01 | 20260507 07:29:17.250 | INFO | PerfReport: 2.94%, Metrics -- 9.01e-01 / 1.22e+00, 2.42e-01 / 1.33e-01 | 20260507 07:29:17.250 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.02e-01 / 3.68e-01, 5.12e-02 / 2.51e-02 | 20260507 07:29:17.250 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:29:17.250 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:29:17.250 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:29:17.250 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 07:29:17.250 | INFO | PerfReport: 21.90%, Upload -- 2.11e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 07:29:17.250 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 07:29:17.250 | INFO | PerfReport: 31.87%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 07:29:17.250 | INFO | PerfReport: 1.69%, LockUpload -- 1.63e+00 / 0.00e+00, 5.07e+00 / 0.00e+00 | 20260507 07:29:17.250 | INFO | PerfReport: 1.19%, LockDownload -- 1.14e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260507 07:29:17.250 | INFO | PerfReport: 21.99%, LockCompute -- 2.12e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 07:29:17.250 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260507 07:29:17.345 | INFO | Starting a round of Baz IO | 20260507 07:29:20.467 | INFO | Populating RTMetrics report | 20260507 07:29:20.467 | INFO | RTMetrics report populated | 20260507 07:29:20.467 | INFO | Opening tmp RTMetrics json file | 20260507 07:29:20.468 | INFO | Serializing RTMetrics to json | 20260507 07:29:20.469 | INFO | Serialization done, swapping tmp file now... | 20260507 07:29:20.470 | INFO | RTMetrics done | 20260507 07:29:20.550 | INFO | Finished a round of Baz IO: Wrote 1.641 GiB in 3.204 seconds | 20260507 07:29:20.550 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.56% / 0.00% / 67.44% | 20260507 07:29:20.550 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.104781% / 0% / 7.54992e-05% | 20260507 07:29:20.550 | INFO | Baz IO overall: | 20260507 07:29:20.550 | INFO | BytesWritten : 4096981930580 | 20260507 07:29:20.550 | INFO | NumEvents : 1961466034577 | 20260507 07:29:22.381 | INFO | Analyzing chunk frames = [8175616,8176128) | 20260507 07:29:22.381 | INFO | MemInfo: { | 20260507 07:29:22.381 | INFO | "gpu" : " 17 GB", | 20260507 07:29:22.381 | INFO | "huge" : "", | 20260507 07:29:22.381 | INFO | "huge_shared" : " 27 GB", | 20260507 07:29:22.381 | INFO | "malloc" : " 14 GB", | 20260507 07:29:22.381 | INFO | "pinned" : " 29 GB" | 20260507 07:29:22.381 | INFO | } | 20260507 07:29:22.381 | INFO | Perf report for the last 5.13255 seconds: | 20260507 07:29:22.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:29:22.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:22.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:22.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:22.381 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 07:29:22.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:22.381 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260507 07:29:22.381 | INFO | PRE_HQ : 59.8%, 1.9, 58.1ms, 0 | 20260507 07:29:22.381 | INFO | BAZWRITER : 62.0%, 3.2, 101.5ms, 0 | 20260507 07:29:22.381 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:29:22.381 | INFO | Uploaded 36GiB to GPU in 2.05525s (17.5161GiB/s) | 20260507 07:29:22.381 | INFO | Downloaded 38GiB to GPU in 2.17546s (17.4676GiB/s) | 20260507 07:29:27.524 | INFO | Analyzing chunk frames = [8176128,8176640) | 20260507 07:29:27.524 | INFO | MemInfo: { | 20260507 07:29:27.524 | INFO | "gpu" : " 17 GB", | 20260507 07:29:27.524 | INFO | "huge" : "", | 20260507 07:29:27.524 | INFO | "huge_shared" : " 27 GB", | 20260507 07:29:27.524 | INFO | "malloc" : " 14 GB", | 20260507 07:29:27.524 | INFO | "pinned" : " 29 GB" | 20260507 07:29:27.524 | INFO | } | 20260507 07:29:27.524 | INFO | Perf report for the last 5.1431 seconds: | 20260507 07:29:27.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:29:27.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:27.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:27.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:27.524 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 07:29:27.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:27.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:27.524 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:29:27.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:27.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:27.524 | INFO | Uploaded 36GiB to GPU in 1.98238s (18.16GiB/s) | 20260507 07:29:27.524 | INFO | Downloaded 34GiB to GPU in 1.92621s (17.6513GiB/s) | 20260507 07:29:32.665 | INFO | Analyzing chunk frames = [8176640,8177152) | 20260507 07:29:32.665 | INFO | MemInfo: { | 20260507 07:29:32.665 | INFO | "gpu" : " 17 GB", | 20260507 07:29:32.665 | INFO | "huge" : "", | 20260507 07:29:32.665 | INFO | "huge_shared" : " 27 GB", | 20260507 07:29:32.665 | INFO | "malloc" : " 14 GB", | 20260507 07:29:32.665 | INFO | "pinned" : " 29 GB" | 20260507 07:29:32.665 | INFO | } | 20260507 07:29:32.665 | INFO | Perf report for the last 5.14132 seconds: | 20260507 07:29:32.665 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:29:32.665 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:32.665 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:32.665 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:32.665 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:29:32.665 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:32.666 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:32.666 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:29:32.666 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:32.666 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:32.666 | INFO | Uploaded 36GiB to GPU in 2.04397s (17.6128GiB/s) | 20260507 07:29:32.666 | INFO | Downloaded 34GiB to GPU in 1.95693s (17.3741GiB/s) | 20260507 07:29:37.806 | INFO | Analyzing chunk frames = [8177152,8177664) | 20260507 07:29:37.806 | INFO | MemInfo: { | 20260507 07:29:37.806 | INFO | "gpu" : " 17 GB", | 20260507 07:29:37.806 | INFO | "huge" : "", | 20260507 07:29:37.806 | INFO | "huge_shared" : " 27 GB", | 20260507 07:29:37.806 | INFO | "malloc" : " 14 GB", | 20260507 07:29:37.806 | INFO | "pinned" : " 29 GB" | 20260507 07:29:37.806 | INFO | } | 20260507 07:29:37.806 | INFO | Perf report for the last 5.14067 seconds: | 20260507 07:29:37.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:29:37.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:37.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:37.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:37.806 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 07:29:37.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:37.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:37.806 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:29:37.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:37.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:37.806 | INFO | Uploaded 36GiB to GPU in 2.03796s (17.6647GiB/s) | 20260507 07:29:37.806 | INFO | Downloaded 34GiB to GPU in 1.9481s (17.4529GiB/s) | 20260507 07:29:42.945 | INFO | Analyzing chunk frames = [8177664,8178176) | 20260507 07:29:42.945 | INFO | MemInfo: { | 20260507 07:29:42.945 | INFO | "gpu" : " 17 GB", | 20260507 07:29:42.945 | INFO | "huge" : "", | 20260507 07:29:42.945 | INFO | "huge_shared" : " 27 GB", | 20260507 07:29:42.945 | INFO | "malloc" : " 14 GB", | 20260507 07:29:42.945 | INFO | "pinned" : " 29 GB" | 20260507 07:29:42.945 | INFO | } | 20260507 07:29:42.945 | INFO | Perf report for the last 5.13928 seconds: | 20260507 07:29:42.945 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:29:42.945 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:42.945 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:42.945 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:42.945 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 07:29:42.945 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:42.945 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:42.945 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:29:42.946 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:42.946 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:42.946 | INFO | Uploaded 36GiB to GPU in 2.0482s (17.5764GiB/s) | 20260507 07:29:42.946 | INFO | Downloaded 34GiB to GPU in 1.91053s (17.7962GiB/s) | 20260507 07:29:48.082 | INFO | Analyzing chunk frames = [8178176,8178688) | 20260507 07:29:48.082 | INFO | MemInfo: { | 20260507 07:29:48.082 | INFO | "gpu" : " 17 GB", | 20260507 07:29:48.082 | INFO | "huge" : "", | 20260507 07:29:48.082 | INFO | "huge_shared" : " 27 GB", | 20260507 07:29:48.082 | INFO | "malloc" : " 14 GB", | 20260507 07:29:48.082 | INFO | "pinned" : " 29 GB" | 20260507 07:29:48.082 | INFO | } | 20260507 07:29:48.082 | INFO | Perf report for the last 5.13672 seconds: | 20260507 07:29:48.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:29:48.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:48.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:48.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:48.082 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 07:29:48.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:48.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:48.082 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:29:48.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:48.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:48.082 | INFO | Uploaded 36GiB to GPU in 2.07416s (17.3564GiB/s) | 20260507 07:29:48.082 | INFO | Downloaded 34GiB to GPU in 1.93823s (17.5417GiB/s) | 20260507 07:29:53.224 | INFO | Analyzing chunk frames = [8178688,8179200) | 20260507 07:29:53.224 | INFO | MemInfo: { | 20260507 07:29:53.224 | INFO | "gpu" : " 17 GB", | 20260507 07:29:53.224 | INFO | "huge" : "", | 20260507 07:29:53.224 | INFO | "huge_shared" : " 27 GB", | 20260507 07:29:53.224 | INFO | "malloc" : " 14 GB", | 20260507 07:29:53.224 | INFO | "pinned" : " 29 GB" | 20260507 07:29:53.224 | INFO | } | 20260507 07:29:53.224 | INFO | Perf report for the last 5.14219 seconds: | 20260507 07:29:53.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:29:53.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:53.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:53.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:53.224 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:29:53.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:53.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:53.224 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:29:53.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:53.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:53.225 | INFO | Uploaded 36GiB to GPU in 1.94454s (18.5134GiB/s) | 20260507 07:29:53.225 | INFO | Downloaded 34GiB to GPU in 1.97096s (17.2504GiB/s) | 20260507 07:29:58.365 | INFO | Analyzing chunk frames = [8179200,8179712) | 20260507 07:29:58.365 | INFO | MemInfo: { | 20260507 07:29:58.365 | INFO | "gpu" : " 17 GB", | 20260507 07:29:58.365 | INFO | "huge" : "", | 20260507 07:29:58.365 | INFO | "huge_shared" : " 27 GB", | 20260507 07:29:58.365 | INFO | "malloc" : " 14 GB", | 20260507 07:29:58.365 | INFO | "pinned" : " 29 GB" | 20260507 07:29:58.365 | INFO | } | 20260507 07:29:58.365 | INFO | Perf report for the last 5.14077 seconds: | 20260507 07:29:58.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:29:58.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:58.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:58.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:58.365 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:29:58.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:58.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:58.365 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:29:58.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:58.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:29:58.365 | INFO | Uploaded 36GiB to GPU in 1.96874s (18.2858GiB/s) | 20260507 07:29:58.365 | INFO | Downloaded 34GiB to GPU in 1.96275s (17.3226GiB/s) | 20260507 07:29:58.462 | INFO | Starting a round of Baz IO | 20260507 07:30:01.555 | INFO | Populating RTMetrics report | 20260507 07:30:01.556 | INFO | RTMetrics report populated | 20260507 07:30:01.556 | INFO | Opening tmp RTMetrics json file | 20260507 07:30:01.565 | INFO | Serializing RTMetrics to json | 20260507 07:30:01.571 | INFO | Serialization done, swapping tmp file now... | 20260507 07:30:01.572 | INFO | RTMetrics done | 20260507 07:30:01.644 | INFO | Finished a round of Baz IO: Wrote 1.64 GiB in 3.182 seconds | 20260507 07:30:01.644 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.56% / 0.00% / 67.44% | 20260507 07:30:01.644 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.10463% / 3.97364e-06% / 7.54992e-05% | 20260507 07:30:01.645 | INFO | Baz IO overall: | 20260507 07:30:01.645 | INFO | BytesWritten : 4098742920549 | 20260507 07:30:01.645 | INFO | NumEvents : 1962288407268 | 20260507 07:30:03.501 | INFO | Analyzing chunk frames = [8179712,8180224) | 20260507 07:30:03.501 | INFO | MemInfo: { | 20260507 07:30:03.501 | INFO | "gpu" : " 17 GB", | 20260507 07:30:03.501 | INFO | "huge" : "", | 20260507 07:30:03.501 | INFO | "huge_shared" : " 27 GB", | 20260507 07:30:03.501 | INFO | "malloc" : " 14 GB", | 20260507 07:30:03.501 | INFO | "pinned" : " 29 GB" | 20260507 07:30:03.501 | INFO | } | 20260507 07:30:03.501 | INFO | Perf report for the last 5.13627 seconds: | 20260507 07:30:03.501 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:30:03.501 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:03.501 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:03.501 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:03.501 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 07:30:03.501 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:03.501 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260507 07:30:03.501 | INFO | PRE_HQ : 59.1%, 1.8, 55.1ms, 0 | 20260507 07:30:03.501 | INFO | BAZWRITER : 61.5%, 3.1, 97.2ms, 0 | 20260507 07:30:03.502 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:30:03.502 | INFO | Uploaded 36GiB to GPU in 2.05856s (17.4879GiB/s) | 20260507 07:30:03.502 | INFO | Downloaded 38GiB to GPU in 2.18663s (17.3783GiB/s) | 20260507 07:30:08.636 | INFO | Analyzing chunk frames = [8180224,8180736) | 20260507 07:30:08.636 | INFO | MemInfo: { | 20260507 07:30:08.636 | INFO | "gpu" : " 17 GB", | 20260507 07:30:08.636 | INFO | "huge" : "", | 20260507 07:30:08.636 | INFO | "huge_shared" : " 27 GB", | 20260507 07:30:08.636 | INFO | "malloc" : " 14 GB", | 20260507 07:30:08.636 | INFO | "pinned" : " 29 GB" | 20260507 07:30:08.636 | INFO | } | 20260507 07:30:08.637 | INFO | Perf report for the last 5.13509 seconds: | 20260507 07:30:08.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:30:08.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:08.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:08.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:08.637 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:30:08.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:08.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:08.637 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:30:08.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:08.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:08.637 | INFO | Uploaded 36GiB to GPU in 1.9906s (18.085GiB/s) | 20260507 07:30:08.637 | INFO | Downloaded 34GiB to GPU in 1.93937s (17.5314GiB/s) | 20260507 07:30:13.781 | INFO | Analyzing chunk frames = [8180736,8181248) | 20260507 07:30:13.782 | INFO | MemInfo: { | 20260507 07:30:13.782 | INFO | "gpu" : " 17 GB", | 20260507 07:30:13.782 | INFO | "huge" : "", | 20260507 07:30:13.782 | INFO | "huge_shared" : " 27 GB", | 20260507 07:30:13.782 | INFO | "malloc" : " 14 GB", | 20260507 07:30:13.782 | INFO | "pinned" : " 29 GB" | 20260507 07:30:13.782 | INFO | } | 20260507 07:30:13.782 | INFO | Perf report for the last 5.14511 seconds: | 20260507 07:30:13.782 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:30:13.782 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:13.782 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:13.782 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:13.782 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 07:30:13.782 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:13.782 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:13.782 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:30:13.782 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:13.782 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:13.782 | INFO | Uploaded 36GiB to GPU in 2.03363s (17.7023GiB/s) | 20260507 07:30:13.782 | INFO | Downloaded 34GiB to GPU in 1.95519s (17.3897GiB/s) | 20260507 07:30:18.919 | INFO | Analyzing chunk frames = [8181248,8181760) | 20260507 07:30:18.919 | INFO | MemInfo: { | 20260507 07:30:18.919 | INFO | "gpu" : " 17 GB", | 20260507 07:30:18.919 | INFO | "huge" : "", | 20260507 07:30:18.919 | INFO | "huge_shared" : " 27 GB", | 20260507 07:30:18.919 | INFO | "malloc" : " 14 GB", | 20260507 07:30:18.919 | INFO | "pinned" : " 29 GB" | 20260507 07:30:18.919 | INFO | } | 20260507 07:30:18.919 | INFO | Perf report for the last 5.13727 seconds: | 20260507 07:30:18.919 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:30:18.919 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:18.919 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:18.919 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:18.919 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 07:30:18.919 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:18.919 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:18.919 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:30:18.919 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:18.919 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:18.919 | INFO | Uploaded 36GiB to GPU in 2.02145s (17.809GiB/s) | 20260507 07:30:18.919 | INFO | Downloaded 34GiB to GPU in 1.94687s (17.4639GiB/s) | 20260507 07:30:24.063 | INFO | Analyzing chunk frames = [8181760,8182272) | 20260507 07:30:24.063 | INFO | MemInfo: { | 20260507 07:30:24.063 | INFO | "gpu" : " 17 GB", | 20260507 07:30:24.063 | INFO | "huge" : "", | 20260507 07:30:24.063 | INFO | "huge_shared" : " 27 GB", | 20260507 07:30:24.063 | INFO | "malloc" : " 14 GB", | 20260507 07:30:24.063 | INFO | "pinned" : " 29 GB" | 20260507 07:30:24.063 | INFO | } | 20260507 07:30:24.063 | INFO | Perf report for the last 5.14444 seconds: | 20260507 07:30:24.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:30:24.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:24.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:24.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:24.063 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 07:30:24.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:24.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:24.064 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:30:24.064 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:24.064 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:24.064 | INFO | Uploaded 36GiB to GPU in 2.02785s (17.7528GiB/s) | 20260507 07:30:24.064 | INFO | Downloaded 34GiB to GPU in 1.92072s (17.7017GiB/s) | 20260507 07:30:29.196 | INFO | Analyzing chunk frames = [8182272,8182784) | 20260507 07:30:29.196 | INFO | MemInfo: { | 20260507 07:30:29.196 | INFO | "gpu" : " 17 GB", | 20260507 07:30:29.196 | INFO | "huge" : "", | 20260507 07:30:29.196 | INFO | "huge_shared" : " 27 GB", | 20260507 07:30:29.196 | INFO | "malloc" : " 14 GB", | 20260507 07:30:29.196 | INFO | "pinned" : " 29 GB" | 20260507 07:30:29.196 | INFO | } | 20260507 07:30:29.196 | INFO | Perf report for the last 5.13293 seconds: | 20260507 07:30:29.196 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:30:29.196 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:29.196 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:29.196 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:29.196 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:30:29.196 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:29.196 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:29.196 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 07:30:29.196 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:29.196 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:29.197 | INFO | Uploaded 36GiB to GPU in 2.03923s (17.6537GiB/s) | 20260507 07:30:29.197 | INFO | Downloaded 34GiB to GPU in 1.92182s (17.6915GiB/s) | 20260507 07:30:34.330 | INFO | Analyzing chunk frames = [8182784,8183296) | 20260507 07:30:34.330 | INFO | MemInfo: { | 20260507 07:30:34.330 | INFO | "gpu" : " 17 GB", | 20260507 07:30:34.330 | INFO | "huge" : "", | 20260507 07:30:34.330 | INFO | "huge_shared" : " 27 GB", | 20260507 07:30:34.330 | INFO | "malloc" : " 14 GB", | 20260507 07:30:34.330 | INFO | "pinned" : " 29 GB" | 20260507 07:30:34.330 | INFO | } | 20260507 07:30:34.331 | INFO | Perf report for the last 5.13416 seconds: | 20260507 07:30:34.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:30:34.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:34.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:34.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:34.331 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 07:30:34.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:34.331 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:34.331 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:30:34.331 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:34.331 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:34.331 | INFO | Uploaded 36GiB to GPU in 1.99382s (18.0558GiB/s) | 20260507 07:30:34.331 | INFO | Downloaded 34GiB to GPU in 1.99024s (17.0834GiB/s) | 20260507 07:30:39.472 | INFO | Analyzing chunk frames = [8183296,8183808) | 20260507 07:30:39.472 | INFO | MemInfo: { | 20260507 07:30:39.472 | INFO | "gpu" : " 17 GB", | 20260507 07:30:39.472 | INFO | "huge" : "", | 20260507 07:30:39.472 | INFO | "huge_shared" : " 27 GB", | 20260507 07:30:39.472 | INFO | "malloc" : " 14 GB", | 20260507 07:30:39.472 | INFO | "pinned" : " 29 GB" | 20260507 07:30:39.472 | INFO | } | 20260507 07:30:39.472 | INFO | Perf report for the last 5.14112 seconds: | 20260507 07:30:39.472 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:30:39.472 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:39.472 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:39.472 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:39.472 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:30:39.472 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:39.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:39.472 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:30:39.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:39.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:39.472 | INFO | Uploaded 36GiB to GPU in 1.95835s (18.3828GiB/s) | 20260507 07:30:39.472 | INFO | Downloaded 34GiB to GPU in 1.95566s (17.3854GiB/s) | 20260507 07:30:39.472 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:30:39.472 | INFO | Total mem usage: 18 GiB | 20260507 07:30:39.472 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:30:39.472 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 07:30:39.472 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:30:39.472 | INFO | High Water Breakdown: | 20260507 07:30:39.472 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:30:39.472 | INFO | 39.7% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:30:39.472 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:30:39.472 | INFO | ------------------------------------------ | 20260507 07:30:39.472 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:30:39.472 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:30:39.472 | INFO | Total mem usage: 26.5 GiB | 20260507 07:30:39.472 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:30:39.472 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:30:39.472 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:30:39.472 | INFO | High Water Breakdown: | 20260507 07:30:39.472 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:30:39.472 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:30:39.472 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:30:39.472 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:30:39.472 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:30:39.472 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:30:39.472 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:30:39.472 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:30:39.472 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:30:39.472 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:30:39.472 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:30:39.472 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:30:39.472 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:30:39.472 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:30:39.472 | INFO | ------------------------------------------ | 20260507 07:30:39.472 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:30:39.472 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:30:39.472 | INFO | Total mem usage: 25.6 GiB | 20260507 07:30:39.472 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:30:39.472 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:30:39.472 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:30:39.472 | INFO | High Water Breakdown: | 20260507 07:30:39.472 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:30:39.472 | INFO | ------------------------------------------ | 20260507 07:30:39.472 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:30:39.472 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:30:39.472 | INFO | Total mem usage: 15.7 GiB | 20260507 07:30:39.472 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:30:39.472 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:30:39.472 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:30:39.472 | INFO | High Water Breakdown: | 20260507 07:30:39.472 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:30:39.472 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:30:39.473 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:30:39.473 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:30:39.473 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:30:39.473 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:30:39.473 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:30:39.473 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:30:39.473 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:30:39.473 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:30:39.473 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:30:39.473 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:30:39.473 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:30:39.473 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:30:39.473 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:30:39.473 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:30:39.473 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:30:39.473 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:30:39.473 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:30:39.473 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:30:39.473 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:30:39.473 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:30:39.473 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:30:39.473 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:30:39.473 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:30:39.473 | INFO | ------------------------------------------ | 20260507 07:30:39.473 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:30:39.473 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:30:39.473 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:30:39.473 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:30:39.473 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.50e+01, 1.34e+01 / 1.50e+00 | 20260507 07:30:39.473 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.02e-01, 9.61e-03 / 6.45e-03 | 20260507 07:30:39.473 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.42e+00, 5.18e-01 / 2.61e-01 | 20260507 07:30:39.473 | INFO | PerfReport: 3.24%, Histogram -- 9.95e-01 / 1.47e+00, 1.69e-01 / 1.27e-01 | 20260507 07:30:39.473 | INFO | PerfReport: 16.28%, DME -- 4.01e+01 / 4.50e+01, 3.98e+00 / 1.78e+00 | 20260507 07:30:39.473 | INFO | PerfReport: 57.74%, FrameLabeling -- 1.78e+01 / 1.97e+01, 1.37e+00 / 1.46e+00 | 20260507 07:30:39.473 | INFO | PerfReport: 6.86%, PulseAccumulating -- 2.11e+00 / 2.69e+00, 3.80e-01 / 2.00e-01 | 20260507 07:30:39.473 | INFO | PerfReport: 2.92%, Metrics -- 8.99e-01 / 1.22e+00, 2.44e-01 / 1.08e-01 | 20260507 07:30:39.473 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.02e-01 / 3.55e-01, 5.09e-02 / 1.56e-02 | 20260507 07:30:39.473 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:30:39.473 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:30:39.473 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:30:39.473 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 07:30:39.473 | INFO | PerfReport: 21.86%, Upload -- 2.10e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260507 07:30:39.473 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 07:30:39.473 | INFO | PerfReport: 32.00%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 07:30:39.473 | INFO | PerfReport: 1.48%, LockUpload -- 1.42e+00 / 0.00e+00, 4.79e+00 / 0.00e+00 | 20260507 07:30:39.473 | INFO | PerfReport: 1.23%, LockDownload -- 1.18e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260507 07:30:39.473 | INFO | PerfReport: 22.01%, LockCompute -- 2.12e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 07:30:39.473 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.87e-02 / 0.00e+00 | 20260507 07:30:39.569 | INFO | Starting a round of Baz IO | 20260507 07:30:42.664 | INFO | Populating RTMetrics report | 20260507 07:30:42.664 | INFO | RTMetrics report populated | 20260507 07:30:42.665 | INFO | Opening tmp RTMetrics json file | 20260507 07:30:42.679 | INFO | Serializing RTMetrics to json | 20260507 07:30:42.680 | INFO | Serialization done, swapping tmp file now... | 20260507 07:30:42.681 | INFO | RTMetrics done | 20260507 07:30:42.746 | INFO | Finished a round of Baz IO: Wrote 1.639 GiB in 3.177 seconds | 20260507 07:30:42.746 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.55% / 0.00% / 67.44% | 20260507 07:30:42.746 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.104296% / 0% / 5.5631e-05% | 20260507 07:30:42.746 | INFO | Baz IO overall: | 20260507 07:30:42.746 | INFO | BytesWritten : 4100503003890 | 20260507 07:30:42.746 | INFO | NumEvents : 1963110425475 | 20260507 07:30:44.619 | INFO | Analyzing chunk frames = [8183808,8184320) | 20260507 07:30:44.619 | INFO | MemInfo: { | 20260507 07:30:44.619 | INFO | "gpu" : " 17 GB", | 20260507 07:30:44.619 | INFO | "huge" : "", | 20260507 07:30:44.619 | INFO | "huge_shared" : " 27 GB", | 20260507 07:30:44.619 | INFO | "malloc" : " 14 GB", | 20260507 07:30:44.619 | INFO | "pinned" : " 29 GB" | 20260507 07:30:44.619 | INFO | } | 20260507 07:30:44.619 | INFO | Perf report for the last 5.14782 seconds: | 20260507 07:30:44.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:30:44.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:44.620 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:44.620 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:44.620 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 07:30:44.620 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:44.620 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 07:30:44.620 | INFO | PRE_HQ : 58.6%, 1.8, 54.5ms, 0 | 20260507 07:30:44.620 | INFO | BAZWRITER : 61.3%, 3.1, 97.0ms, 0 | 20260507 07:30:44.620 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:30:44.620 | INFO | Uploaded 36GiB to GPU in 2.03378s (17.701GiB/s) | 20260507 07:30:44.620 | INFO | Downloaded 38GiB to GPU in 2.18046s (17.4275GiB/s) | 20260507 07:30:49.759 | INFO | Analyzing chunk frames = [8184320,8184832) | 20260507 07:30:49.759 | INFO | MemInfo: { | 20260507 07:30:49.759 | INFO | "gpu" : " 17 GB", | 20260507 07:30:49.760 | INFO | "huge" : "", | 20260507 07:30:49.760 | INFO | "huge_shared" : " 27 GB", | 20260507 07:30:49.760 | INFO | "malloc" : " 14 GB", | 20260507 07:30:49.760 | INFO | "pinned" : " 29 GB" | 20260507 07:30:49.760 | INFO | } | 20260507 07:30:49.760 | INFO | Perf report for the last 5.14011 seconds: | 20260507 07:30:49.760 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:30:49.760 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:49.760 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:49.760 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:49.760 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:30:49.760 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:49.760 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:49.760 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:30:49.760 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:49.760 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:49.760 | INFO | Uploaded 36GiB to GPU in 1.97652s (18.2138GiB/s) | 20260507 07:30:49.760 | INFO | Downloaded 34GiB to GPU in 1.93777s (17.5459GiB/s) | 20260507 07:30:54.891 | INFO | Analyzing chunk frames = [8184832,8185344) | 20260507 07:30:54.891 | INFO | MemInfo: { | 20260507 07:30:54.891 | INFO | "gpu" : " 17 GB", | 20260507 07:30:54.891 | INFO | "huge" : "", | 20260507 07:30:54.891 | INFO | "huge_shared" : " 27 GB", | 20260507 07:30:54.891 | INFO | "malloc" : " 14 GB", | 20260507 07:30:54.891 | INFO | "pinned" : " 29 GB" | 20260507 07:30:54.891 | INFO | } | 20260507 07:30:54.891 | INFO | Perf report for the last 5.13172 seconds: | 20260507 07:30:54.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:30:54.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:54.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:54.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:54.891 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:30:54.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:54.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:54.891 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:30:54.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:54.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:30:54.892 | INFO | Uploaded 36GiB to GPU in 2.01184s (17.8941GiB/s) | 20260507 07:30:54.892 | INFO | Downloaded 34GiB to GPU in 1.92064s (17.7024GiB/s) | 20260507 07:31:00.032 | INFO | Analyzing chunk frames = [8185344,8185856) | 20260507 07:31:00.032 | INFO | MemInfo: { | 20260507 07:31:00.032 | INFO | "gpu" : " 17 GB", | 20260507 07:31:00.032 | INFO | "huge" : "", | 20260507 07:31:00.032 | INFO | "huge_shared" : " 27 GB", | 20260507 07:31:00.032 | INFO | "malloc" : " 14 GB", | 20260507 07:31:00.032 | INFO | "pinned" : " 29 GB" | 20260507 07:31:00.032 | INFO | } | 20260507 07:31:00.032 | INFO | Perf report for the last 5.14111 seconds: | 20260507 07:31:00.032 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:31:00.032 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:00.032 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:00.032 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:00.032 | INFO | ANALYSIS : 60.3%, 2.9, 93.1ms, 0 | 20260507 07:31:00.032 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:00.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:00.033 | INFO | PRE_HQ : 26.7%, 1.0, 13.9ms, 0 | 20260507 07:31:00.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:00.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:00.033 | INFO | Uploaded 36GiB to GPU in 2.03126s (17.723GiB/s) | 20260507 07:31:00.033 | INFO | Downloaded 34GiB to GPU in 1.954s (17.4002GiB/s) | 20260507 07:31:05.173 | INFO | Analyzing chunk frames = [8185856,8186368) | 20260507 07:31:05.173 | INFO | MemInfo: { | 20260507 07:31:05.173 | INFO | "gpu" : " 17 GB", | 20260507 07:31:05.173 | INFO | "huge" : "", | 20260507 07:31:05.173 | INFO | "huge_shared" : " 27 GB", | 20260507 07:31:05.173 | INFO | "malloc" : " 14 GB", | 20260507 07:31:05.173 | INFO | "pinned" : " 29 GB" | 20260507 07:31:05.173 | INFO | } | 20260507 07:31:05.173 | INFO | Perf report for the last 5.14088 seconds: | 20260507 07:31:05.173 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:31:05.173 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:05.173 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:05.173 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:05.173 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 07:31:05.173 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:05.173 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:05.173 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:31:05.173 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:05.173 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:05.174 | INFO | Uploaded 36GiB to GPU in 2.0268s (17.762GiB/s) | 20260507 07:31:05.174 | INFO | Downloaded 34GiB to GPU in 1.92002s (17.7082GiB/s) | 20260507 07:31:10.315 | INFO | Analyzing chunk frames = [8186368,8186880) | 20260507 07:31:10.315 | INFO | MemInfo: { | 20260507 07:31:10.315 | INFO | "gpu" : " 17 GB", | 20260507 07:31:10.315 | INFO | "huge" : "", | 20260507 07:31:10.315 | INFO | "huge_shared" : " 27 GB", | 20260507 07:31:10.315 | INFO | "malloc" : " 14 GB", | 20260507 07:31:10.315 | INFO | "pinned" : " 29 GB" | 20260507 07:31:10.315 | INFO | } | 20260507 07:31:10.315 | INFO | Perf report for the last 5.14154 seconds: | 20260507 07:31:10.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:31:10.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:10.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:10.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:10.315 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 07:31:10.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:10.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:10.315 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:31:10.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:10.315 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:10.315 | INFO | Uploaded 36GiB to GPU in 2.05825s (17.4906GiB/s) | 20260507 07:31:10.315 | INFO | Downloaded 34GiB to GPU in 1.9273s (17.6413GiB/s) | 20260507 07:31:15.453 | INFO | Analyzing chunk frames = [8186880,8187392) | 20260507 07:31:15.453 | INFO | MemInfo: { | 20260507 07:31:15.453 | INFO | "gpu" : " 17 GB", | 20260507 07:31:15.454 | INFO | "huge" : "", | 20260507 07:31:15.454 | INFO | "huge_shared" : " 27 GB", | 20260507 07:31:15.454 | INFO | "malloc" : " 14 GB", | 20260507 07:31:15.454 | INFO | "pinned" : " 29 GB" | 20260507 07:31:15.454 | INFO | } | 20260507 07:31:15.454 | INFO | Perf report for the last 5.13873 seconds: | 20260507 07:31:15.454 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:31:15.454 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:15.454 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:15.454 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:15.454 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260507 07:31:15.454 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:15.454 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:15.454 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:31:15.454 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:15.454 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:15.454 | INFO | Uploaded 36GiB to GPU in 1.88374s (19.1109GiB/s) | 20260507 07:31:15.454 | INFO | Downloaded 34GiB to GPU in 1.96978s (17.2608GiB/s) | 20260507 07:31:20.590 | INFO | Analyzing chunk frames = [8187392,8187904) | 20260507 07:31:20.590 | INFO | MemInfo: { | 20260507 07:31:20.590 | INFO | "gpu" : " 17 GB", | 20260507 07:31:20.590 | INFO | "huge" : "", | 20260507 07:31:20.590 | INFO | "huge_shared" : " 27 GB", | 20260507 07:31:20.590 | INFO | "malloc" : " 14 GB", | 20260507 07:31:20.590 | INFO | "pinned" : " 29 GB" | 20260507 07:31:20.590 | INFO | } | 20260507 07:31:20.590 | INFO | Perf report for the last 5.13643 seconds: | 20260507 07:31:20.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:31:20.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:20.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:20.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:20.590 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:31:20.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:20.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:20.590 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:31:20.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:20.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:20.590 | INFO | Uploaded 36GiB to GPU in 1.9951s (18.0442GiB/s) | 20260507 07:31:20.590 | INFO | Downloaded 34GiB to GPU in 1.96075s (17.3403GiB/s) | 20260507 07:31:20.687 | INFO | Starting a round of Baz IO | 20260507 07:31:23.779 | INFO | Populating RTMetrics report | 20260507 07:31:23.779 | INFO | RTMetrics report populated | 20260507 07:31:23.779 | INFO | Opening tmp RTMetrics json file | 20260507 07:31:23.789 | INFO | Serializing RTMetrics to json | 20260507 07:31:23.795 | INFO | Serialization done, swapping tmp file now... | 20260507 07:31:23.795 | INFO | RTMetrics done | 20260507 07:31:23.862 | INFO | Finished a round of Baz IO: Wrote 1.641 GiB in 3.174 seconds | 20260507 07:31:23.862 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.55% / 0.00% / 67.45% | 20260507 07:31:23.862 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.104312% / 7.94729e-06% / 5.5631e-05% | 20260507 07:31:23.862 | INFO | Baz IO overall: | 20260507 07:31:23.862 | INFO | BytesWritten : 4102264902009 | 20260507 07:31:23.862 | INFO | NumEvents : 1963933617621 | 20260507 07:31:25.731 | INFO | Analyzing chunk frames = [8187904,8188416) | 20260507 07:31:25.731 | INFO | MemInfo: { | 20260507 07:31:25.731 | INFO | "gpu" : " 17 GB", | 20260507 07:31:25.731 | INFO | "huge" : "", | 20260507 07:31:25.731 | INFO | "huge_shared" : " 27 GB", | 20260507 07:31:25.731 | INFO | "malloc" : " 14 GB", | 20260507 07:31:25.731 | INFO | "pinned" : " 29 GB" | 20260507 07:31:25.731 | INFO | } | 20260507 07:31:25.731 | INFO | Perf report for the last 5.14131 seconds: | 20260507 07:31:25.731 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:31:25.731 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:25.731 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:25.731 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:25.731 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 07:31:25.731 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:25.731 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:31:25.731 | INFO | PRE_HQ : 58.8%, 1.8, 54.2ms, 0 | 20260507 07:31:25.731 | INFO | BAZWRITER : 61.3%, 3.1, 96.7ms, 0 | 20260507 07:31:25.731 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:31:25.732 | INFO | Uploaded 36GiB to GPU in 2.0623s (17.4562GiB/s) | 20260507 07:31:25.732 | INFO | Downloaded 38GiB to GPU in 2.18521s (17.3896GiB/s) | 20260507 07:31:30.868 | INFO | Analyzing chunk frames = [8188416,8188928) | 20260507 07:31:30.868 | INFO | MemInfo: { | 20260507 07:31:30.868 | INFO | "gpu" : " 17 GB", | 20260507 07:31:30.868 | INFO | "huge" : "", | 20260507 07:31:30.868 | INFO | "huge_shared" : " 27 GB", | 20260507 07:31:30.868 | INFO | "malloc" : " 14 GB", | 20260507 07:31:30.868 | INFO | "pinned" : " 29 GB" | 20260507 07:31:30.868 | INFO | } | 20260507 07:31:30.869 | INFO | Perf report for the last 5.13715 seconds: | 20260507 07:31:30.869 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:31:30.869 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:30.869 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:30.869 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:30.869 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 07:31:30.869 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:30.869 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:30.869 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:31:30.869 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:30.869 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:30.869 | INFO | Uploaded 36GiB to GPU in 2.00252s (17.9773GiB/s) | 20260507 07:31:30.869 | INFO | Downloaded 34GiB to GPU in 1.92968s (17.6195GiB/s) | 20260507 07:31:36.008 | INFO | Analyzing chunk frames = [8188928,8189440) | 20260507 07:31:36.008 | INFO | MemInfo: { | 20260507 07:31:36.008 | INFO | "gpu" : " 17 GB", | 20260507 07:31:36.008 | INFO | "huge" : "", | 20260507 07:31:36.008 | INFO | "huge_shared" : " 27 GB", | 20260507 07:31:36.008 | INFO | "malloc" : " 14 GB", | 20260507 07:31:36.008 | INFO | "pinned" : " 29 GB" | 20260507 07:31:36.008 | INFO | } | 20260507 07:31:36.008 | INFO | Perf report for the last 5.13963 seconds: | 20260507 07:31:36.008 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:31:36.008 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:36.008 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:36.008 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:36.008 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 07:31:36.008 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:36.008 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:36.008 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:31:36.008 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:36.008 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:36.009 | INFO | Uploaded 36GiB to GPU in 2.05369s (17.5295GiB/s) | 20260507 07:31:36.009 | INFO | Downloaded 34GiB to GPU in 1.95158s (17.4218GiB/s) | 20260507 07:31:41.145 | INFO | Analyzing chunk frames = [8189440,8189952) | 20260507 07:31:41.145 | INFO | MemInfo: { | 20260507 07:31:41.145 | INFO | "gpu" : " 17 GB", | 20260507 07:31:41.145 | INFO | "huge" : "", | 20260507 07:31:41.145 | INFO | "huge_shared" : " 27 GB", | 20260507 07:31:41.145 | INFO | "malloc" : " 14 GB", | 20260507 07:31:41.145 | INFO | "pinned" : " 29 GB" | 20260507 07:31:41.145 | INFO | } | 20260507 07:31:41.145 | INFO | Perf report for the last 5.13701 seconds: | 20260507 07:31:41.145 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:31:41.145 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:41.145 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:41.145 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:41.145 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 07:31:41.145 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:41.145 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:41.145 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:31:41.145 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:41.145 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:41.145 | INFO | Uploaded 36GiB to GPU in 2.05288s (17.5364GiB/s) | 20260507 07:31:41.146 | INFO | Downloaded 34GiB to GPU in 1.9445s (17.4852GiB/s) | 20260507 07:31:46.288 | INFO | Analyzing chunk frames = [8189952,8190464) | 20260507 07:31:46.288 | INFO | MemInfo: { | 20260507 07:31:46.288 | INFO | "gpu" : " 17 GB", | 20260507 07:31:46.288 | INFO | "huge" : "", | 20260507 07:31:46.288 | INFO | "huge_shared" : " 27 GB", | 20260507 07:31:46.288 | INFO | "malloc" : " 14 GB", | 20260507 07:31:46.288 | INFO | "pinned" : " 29 GB" | 20260507 07:31:46.288 | INFO | } | 20260507 07:31:46.289 | INFO | Perf report for the last 5.14339 seconds: | 20260507 07:31:46.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:31:46.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:46.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:46.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:46.289 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 07:31:46.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:46.289 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:46.289 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:31:46.289 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:46.289 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:46.289 | INFO | Uploaded 36GiB to GPU in 2.04112s (17.6373GiB/s) | 20260507 07:31:46.289 | INFO | Downloaded 34GiB to GPU in 1.91926s (17.7151GiB/s) | 20260507 07:31:51.428 | INFO | Analyzing chunk frames = [8190464,8190976) | 20260507 07:31:51.428 | INFO | MemInfo: { | 20260507 07:31:51.428 | INFO | "gpu" : " 17 GB", | 20260507 07:31:51.428 | INFO | "huge" : "", | 20260507 07:31:51.428 | INFO | "huge_shared" : " 27 GB", | 20260507 07:31:51.428 | INFO | "malloc" : " 14 GB", | 20260507 07:31:51.428 | INFO | "pinned" : " 29 GB" | 20260507 07:31:51.428 | INFO | } | 20260507 07:31:51.428 | INFO | Perf report for the last 5.13971 seconds: | 20260507 07:31:51.428 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:31:51.428 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:51.428 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:51.428 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:51.428 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 07:31:51.428 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:51.428 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:51.428 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 07:31:51.428 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:51.428 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:51.429 | INFO | Uploaded 36GiB to GPU in 2.03353s (17.7032GiB/s) | 20260507 07:31:51.429 | INFO | Downloaded 34GiB to GPU in 1.91046s (17.7967GiB/s) | 20260507 07:31:56.570 | INFO | Analyzing chunk frames = [8190976,8191488) | 20260507 07:31:56.570 | INFO | MemInfo: { | 20260507 07:31:56.570 | INFO | "gpu" : " 17 GB", | 20260507 07:31:56.570 | INFO | "huge" : "", | 20260507 07:31:56.570 | INFO | "huge_shared" : " 27 GB", | 20260507 07:31:56.570 | INFO | "malloc" : " 14 GB", | 20260507 07:31:56.570 | INFO | "pinned" : " 29 GB" | 20260507 07:31:56.570 | INFO | } | 20260507 07:31:56.570 | INFO | Perf report for the last 5.1419 seconds: | 20260507 07:31:56.570 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:31:56.570 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:56.570 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:56.570 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:56.570 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:31:56.570 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:56.570 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:56.570 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:31:56.570 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:56.570 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:31:56.571 | INFO | Uploaded 36GiB to GPU in 1.92908s (18.6617GiB/s) | 20260507 07:31:56.571 | INFO | Downloaded 34GiB to GPU in 1.97832s (17.1863GiB/s) | 20260507 07:32:01.701 | INFO | Analyzing chunk frames = [8191488,8192000) | 20260507 07:32:01.701 | INFO | MemInfo: { | 20260507 07:32:01.701 | INFO | "gpu" : " 17 GB", | 20260507 07:32:01.701 | INFO | "huge" : "", | 20260507 07:32:01.701 | INFO | "huge_shared" : " 27 GB", | 20260507 07:32:01.701 | INFO | "malloc" : " 14 GB", | 20260507 07:32:01.701 | INFO | "pinned" : " 29 GB" | 20260507 07:32:01.701 | INFO | } | 20260507 07:32:01.701 | INFO | Perf report for the last 5.13106 seconds: | 20260507 07:32:01.701 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:32:01.701 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:01.701 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:01.701 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:01.701 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:32:01.701 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:01.701 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:01.701 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 07:32:01.701 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:01.701 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:01.702 | INFO | Uploaded 36GiB to GPU in 1.98705s (18.1173GiB/s) | 20260507 07:32:01.702 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:32:01.702 | INFO | Total mem usage: 18 GiB | 20260507 07:32:01.702 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:32:01.702 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:32:01.702 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:32:01.702 | INFO | High Water Breakdown: | 20260507 07:32:01.702 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:32:01.702 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:32:01.702 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:32:01.702 | INFO | ------------------------------------------ | 20260507 07:32:01.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:32:01.702 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:32:01.702 | INFO | Total mem usage: 26.5 GiB | 20260507 07:32:01.702 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:32:01.702 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:32:01.702 | INFO | Downloaded 34GiB to GPU in 1.95798s (17.3648GiB/s) | 20260507 07:32:01.702 | INFO | Unused Cache: 1.78 GiB (Waste: 6.7%) | 20260507 07:32:01.702 | INFO | High Water Breakdown: | 20260507 07:32:01.702 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:32:01.702 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:32:01.702 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:32:01.702 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:32:01.702 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:32:01.702 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:32:01.702 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:32:01.702 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:32:01.702 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:32:01.702 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:32:01.702 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:32:01.702 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:32:01.702 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:32:01.702 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:32:01.702 | INFO | ------------------------------------------ | 20260507 07:32:01.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:32:01.702 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:32:01.702 | INFO | Total mem usage: 25.6 GiB | 20260507 07:32:01.702 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:32:01.702 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:32:01.702 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:32:01.702 | INFO | High Water Breakdown: | 20260507 07:32:01.702 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:32:01.702 | INFO | ------------------------------------------ | 20260507 07:32:01.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:32:01.702 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:32:01.702 | INFO | Total mem usage: 15.7 GiB | 20260507 07:32:01.702 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:32:01.702 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:32:01.702 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:32:01.702 | INFO | High Water Breakdown: | 20260507 07:32:01.702 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:32:01.702 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:32:01.702 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:32:01.702 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:32:01.702 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:32:01.702 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:32:01.702 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:32:01.702 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:32:01.702 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:32:01.702 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:32:01.702 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:32:01.702 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:32:01.702 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:32:01.702 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:32:01.702 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:32:01.702 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:32:01.702 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:32:01.702 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:32:01.702 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:32:01.702 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:32:01.702 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:32:01.702 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:32:01.702 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:32:01.702 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:32:01.702 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:32:01.702 | INFO | ------------------------------------------ | 20260507 07:32:01.702 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:32:01.703 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:32:01.703 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:32:01.703 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:32:01.703 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.32e+01 / 1.17e+00 | 20260507 07:32:01.703 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.02e-01, 9.40e-03 / 4.24e-03 | 20260507 07:32:01.703 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.43e+00, 5.10e-01 / 2.35e-01 | 20260507 07:32:01.703 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.52e+00, 1.69e-01 / 6.82e-02 | 20260507 07:32:01.703 | INFO | PerfReport: 16.34%, DME -- 4.01e+01 / 4.52e+01, 4.00e+00 / 1.40e+00 | 20260507 07:32:01.703 | INFO | PerfReport: 57.64%, FrameLabeling -- 1.77e+01 / 1.93e+01, 1.25e+00 / 1.24e+00 | 20260507 07:32:01.703 | INFO | PerfReport: 6.91%, PulseAccumulating -- 2.12e+00 / 2.72e+00, 3.89e-01 / 1.74e-01 | 20260507 07:32:01.703 | INFO | PerfReport: 2.94%, Metrics -- 9.02e-01 / 1.22e+00, 2.43e-01 / 1.07e-01 | 20260507 07:32:01.703 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.02e-01 / 3.59e-01, 5.11e-02 / 1.84e-02 | 20260507 07:32:01.703 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:32:01.703 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:32:01.703 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:32:01.703 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 07:32:01.703 | INFO | PerfReport: 21.79%, Upload -- 2.10e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260507 07:32:01.703 | INFO | PerfReport: 21.37%, Download -- 2.05e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260507 07:32:01.703 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 07:32:01.703 | INFO | PerfReport: 1.57%, LockUpload -- 1.51e+00 / 0.00e+00, 4.93e+00 / 0.00e+00 | 20260507 07:32:01.703 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260507 07:32:01.703 | INFO | PerfReport: 22.04%, LockCompute -- 2.12e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260507 07:32:01.703 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.71e-02 / 0.00e+00 | 20260507 07:32:01.798 | INFO | Starting a round of Baz IO | 20260507 07:32:04.895 | INFO | Populating RTMetrics report | 20260507 07:32:04.895 | INFO | RTMetrics report populated | 20260507 07:32:04.895 | INFO | Opening tmp RTMetrics json file | 20260507 07:32:04.911 | INFO | Serializing RTMetrics to json | 20260507 07:32:04.912 | INFO | Serialization done, swapping tmp file now... | 20260507 07:32:04.913 | INFO | RTMetrics done | 20260507 07:32:04.978 | INFO | Finished a round of Baz IO: Wrote 1.641 GiB in 3.18 seconds | 20260507 07:32:04.978 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.55% / 0.00% / 67.45% | 20260507 07:32:04.978 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.104038% / 0% / 7.94729e-05% | 20260507 07:32:04.978 | INFO | Baz IO overall: | 20260507 07:32:04.978 | INFO | BytesWritten : 4104026410774 | 20260507 07:32:04.979 | INFO | NumEvents : 1964756192548 | 20260507 07:32:06.838 | INFO | Analyzing chunk frames = [8192000,8192512) | 20260507 07:32:06.838 | INFO | MemInfo: { | 20260507 07:32:06.838 | INFO | "gpu" : " 17 GB", | 20260507 07:32:06.838 | INFO | "huge" : "", | 20260507 07:32:06.838 | INFO | "huge_shared" : " 27 GB", | 20260507 07:32:06.839 | INFO | "malloc" : " 14 GB", | 20260507 07:32:06.839 | INFO | "pinned" : " 29 GB" | 20260507 07:32:06.839 | INFO | } | 20260507 07:32:06.839 | INFO | Perf report for the last 5.13751 seconds: | 20260507 07:32:06.839 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:32:06.839 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:06.839 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:06.839 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:06.839 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 07:32:06.839 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:06.839 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 07:32:06.839 | INFO | PRE_HQ : 58.8%, 1.8, 54.4ms, 0 | 20260507 07:32:06.839 | INFO | BAZWRITER : 61.5%, 3.1, 97.5ms, 0 | 20260507 07:32:06.839 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:32:06.839 | INFO | Uploaded 36GiB to GPU in 2.05467s (17.5211GiB/s) | 20260507 07:32:06.839 | INFO | Downloaded 38GiB to GPU in 2.17471s (17.4736GiB/s) | 20260507 07:32:11.982 | INFO | Analyzing chunk frames = [8192512,8193024) | 20260507 07:32:11.982 | INFO | MemInfo: { | 20260507 07:32:11.982 | INFO | "gpu" : " 17 GB", | 20260507 07:32:11.982 | INFO | "huge" : "", | 20260507 07:32:11.982 | INFO | "huge_shared" : " 27 GB", | 20260507 07:32:11.982 | INFO | "malloc" : " 14 GB", | 20260507 07:32:11.982 | INFO | "pinned" : " 29 GB" | 20260507 07:32:11.982 | INFO | } | 20260507 07:32:11.982 | INFO | Perf report for the last 5.14315 seconds: | 20260507 07:32:11.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:32:11.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:11.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:11.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:11.982 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 07:32:11.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:11.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:11.982 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:32:11.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:11.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:11.982 | INFO | Uploaded 36GiB to GPU in 2.05104s (17.552GiB/s) | 20260507 07:32:11.982 | INFO | Downloaded 34GiB to GPU in 1.94286s (17.5GiB/s) | 20260507 07:32:17.125 | INFO | Analyzing chunk frames = [8193024,8193536) | 20260507 07:32:17.125 | INFO | MemInfo: { | 20260507 07:32:17.125 | INFO | "gpu" : " 17 GB", | 20260507 07:32:17.125 | INFO | "huge" : "", | 20260507 07:32:17.125 | INFO | "huge_shared" : " 27 GB", | 20260507 07:32:17.125 | INFO | "malloc" : " 14 GB", | 20260507 07:32:17.125 | INFO | "pinned" : " 29 GB" | 20260507 07:32:17.125 | INFO | } | 20260507 07:32:17.125 | INFO | Perf report for the last 5.14287 seconds: | 20260507 07:32:17.125 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:32:17.125 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:17.125 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:17.125 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:17.125 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260507 07:32:17.125 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:17.125 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:17.125 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:32:17.125 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:17.125 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:17.125 | INFO | Uploaded 36GiB to GPU in 2.05199s (17.5439GiB/s) | 20260507 07:32:17.125 | INFO | Downloaded 34GiB to GPU in 1.94995s (17.4363GiB/s) | 20260507 07:32:22.267 | INFO | Analyzing chunk frames = [8193536,8194048) | 20260507 07:32:22.267 | INFO | MemInfo: { | 20260507 07:32:22.267 | INFO | "gpu" : " 17 GB", | 20260507 07:32:22.267 | INFO | "huge" : "", | 20260507 07:32:22.267 | INFO | "huge_shared" : " 27 GB", | 20260507 07:32:22.267 | INFO | "malloc" : " 14 GB", | 20260507 07:32:22.267 | INFO | "pinned" : " 29 GB" | 20260507 07:32:22.267 | INFO | } | 20260507 07:32:22.267 | INFO | Perf report for the last 5.14243 seconds: | 20260507 07:32:22.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:32:22.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:22.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:22.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:22.267 | INFO | ANALYSIS : 60.4%, 2.9, 93.4ms, 0 | 20260507 07:32:22.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:22.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:22.267 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:32:22.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:22.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:22.268 | INFO | Uploaded 36GiB to GPU in 2.05141s (17.5489GiB/s) | 20260507 07:32:22.268 | INFO | Downloaded 34GiB to GPU in 1.94324s (17.4966GiB/s) | 20260507 07:32:27.404 | INFO | Analyzing chunk frames = [8194048,8194560) | 20260507 07:32:27.404 | INFO | MemInfo: { | 20260507 07:32:27.404 | INFO | "gpu" : " 17 GB", | 20260507 07:32:27.404 | INFO | "huge" : "", | 20260507 07:32:27.404 | INFO | "huge_shared" : " 27 GB", | 20260507 07:32:27.404 | INFO | "malloc" : " 14 GB", | 20260507 07:32:27.404 | INFO | "pinned" : " 29 GB" | 20260507 07:32:27.404 | INFO | } | 20260507 07:32:27.404 | INFO | Perf report for the last 5.13707 seconds: | 20260507 07:32:27.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:32:27.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:27.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:27.405 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:27.405 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 07:32:27.405 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:27.405 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:27.405 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:32:27.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:27.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:27.405 | INFO | Uploaded 36GiB to GPU in 2.05268s (17.538GiB/s) | 20260507 07:32:27.405 | INFO | Downloaded 34GiB to GPU in 1.90139s (17.8816GiB/s) | 20260507 07:32:32.542 | INFO | Analyzing chunk frames = [8194560,8195072) | 20260507 07:32:32.542 | INFO | MemInfo: { | 20260507 07:32:32.542 | INFO | "gpu" : " 17 GB", | 20260507 07:32:32.542 | INFO | "huge" : "", | 20260507 07:32:32.542 | INFO | "huge_shared" : " 27 GB", | 20260507 07:32:32.542 | INFO | "malloc" : " 14 GB", | 20260507 07:32:32.542 | INFO | "pinned" : " 29 GB" | 20260507 07:32:32.542 | INFO | } | 20260507 07:32:32.542 | INFO | Perf report for the last 5.13786 seconds: | 20260507 07:32:32.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:32:32.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:32.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:32.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:32.542 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 07:32:32.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:32.542 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:32.542 | INFO | PRE_HQ : 27.8%, 1.0, 14.4ms, 0 | 20260507 07:32:32.542 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:32.542 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:32.542 | INFO | Uploaded 36GiB to GPU in 2.07572s (17.3434GiB/s) | 20260507 07:32:32.542 | INFO | Downloaded 34GiB to GPU in 1.94252s (17.503GiB/s) | 20260507 07:32:37.689 | INFO | Analyzing chunk frames = [8195072,8195584) | 20260507 07:32:37.689 | INFO | MemInfo: { | 20260507 07:32:37.689 | INFO | "gpu" : " 17 GB", | 20260507 07:32:37.689 | INFO | "huge" : "", | 20260507 07:32:37.689 | INFO | "huge_shared" : " 27 GB", | 20260507 07:32:37.689 | INFO | "malloc" : " 14 GB", | 20260507 07:32:37.689 | INFO | "pinned" : " 29 GB" | 20260507 07:32:37.689 | INFO | } | 20260507 07:32:37.689 | INFO | Perf report for the last 5.14671 seconds: | 20260507 07:32:37.689 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:32:37.689 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:37.689 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:37.689 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:37.689 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:32:37.689 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:37.689 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:37.689 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 07:32:37.689 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:37.689 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:37.689 | INFO | Uploaded 36GiB to GPU in 1.93589s (18.5961GiB/s) | 20260507 07:32:37.689 | INFO | Downloaded 34GiB to GPU in 1.97227s (17.2391GiB/s) | 20260507 07:32:42.818 | INFO | Analyzing chunk frames = [8195584,8196096) | 20260507 07:32:42.818 | INFO | MemInfo: { | 20260507 07:32:42.818 | INFO | "gpu" : " 17 GB", | 20260507 07:32:42.818 | INFO | "huge" : "", | 20260507 07:32:42.818 | INFO | "huge_shared" : " 27 GB", | 20260507 07:32:42.818 | INFO | "malloc" : " 14 GB", | 20260507 07:32:42.818 | INFO | "pinned" : " 29 GB" | 20260507 07:32:42.818 | INFO | } | 20260507 07:32:42.818 | INFO | Perf report for the last 5.129 seconds: | 20260507 07:32:42.818 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:32:42.818 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:42.818 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:42.818 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:42.818 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 07:32:42.818 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:42.818 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:42.818 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 07:32:42.818 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:42.818 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:42.819 | INFO | Uploaded 36GiB to GPU in 1.98384s (18.1467GiB/s) | 20260507 07:32:42.819 | INFO | Downloaded 34GiB to GPU in 1.96738s (17.2818GiB/s) | 20260507 07:32:42.915 | INFO | Starting a round of Baz IO | 20260507 07:32:46.009 | INFO | Populating RTMetrics report | 20260507 07:32:46.009 | INFO | RTMetrics report populated | 20260507 07:32:46.009 | INFO | Opening tmp RTMetrics json file | 20260507 07:32:46.017 | INFO | Serializing RTMetrics to json | 20260507 07:32:46.025 | INFO | Serialization done, swapping tmp file now... | 20260507 07:32:46.025 | INFO | RTMetrics done | 20260507 07:32:46.096 | INFO | Finished a round of Baz IO: Wrote 1.64 GiB in 3.18 seconds | 20260507 07:32:46.096 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.55% / 0.00% / 67.45% | 20260507 07:32:46.096 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.103533% / 0% / 8.74201e-05% | 20260507 07:32:46.096 | INFO | Baz IO overall: | 20260507 07:32:46.096 | INFO | BytesWritten : 4105786902956 | 20260507 07:32:46.096 | INFO | NumEvents : 1965577905716 | 20260507 07:32:47.957 | INFO | Analyzing chunk frames = [8196096,8196608) | 20260507 07:32:47.957 | INFO | MemInfo: { | 20260507 07:32:47.957 | INFO | "gpu" : " 17 GB", | 20260507 07:32:47.957 | INFO | "huge" : "", | 20260507 07:32:47.957 | INFO | "huge_shared" : " 27 GB", | 20260507 07:32:47.957 | INFO | "malloc" : " 14 GB", | 20260507 07:32:47.957 | INFO | "pinned" : " 29 GB" | 20260507 07:32:47.957 | INFO | } | 20260507 07:32:47.957 | INFO | Perf report for the last 5.13904 seconds: | 20260507 07:32:47.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:32:47.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:47.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:47.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:47.957 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 07:32:47.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:47.957 | INFO | RT_METRICS : 42.6%, 1.0, 22.3ms, 0 | 20260507 07:32:47.957 | INFO | PRE_HQ : 59.4%, 1.8, 56.7ms, 0 | 20260507 07:32:47.957 | INFO | BAZWRITER : 61.5%, 3.2, 101.1ms, 0 | 20260507 07:32:47.957 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:32:47.957 | INFO | Uploaded 36GiB to GPU in 2.04896s (17.5699GiB/s) | 20260507 07:32:47.957 | INFO | Downloaded 38GiB to GPU in 2.18091s (17.4239GiB/s) | 20260507 07:32:53.096 | INFO | Analyzing chunk frames = [8196608,8197120) | 20260507 07:32:53.096 | INFO | MemInfo: { | 20260507 07:32:53.096 | INFO | "gpu" : " 17 GB", | 20260507 07:32:53.096 | INFO | "huge" : "", | 20260507 07:32:53.096 | INFO | "huge_shared" : " 27 GB", | 20260507 07:32:53.096 | INFO | "malloc" : " 14 GB", | 20260507 07:32:53.096 | INFO | "pinned" : " 29 GB" | 20260507 07:32:53.096 | INFO | } | 20260507 07:32:53.096 | INFO | Perf report for the last 5.13932 seconds: | 20260507 07:32:53.096 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:32:53.096 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:53.096 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:53.096 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:53.096 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:32:53.096 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:53.096 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:53.097 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 07:32:53.097 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:53.097 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:53.097 | INFO | Uploaded 36GiB to GPU in 1.97773s (18.2027GiB/s) | 20260507 07:32:53.097 | INFO | Downloaded 34GiB to GPU in 1.93027s (17.6141GiB/s) | 20260507 07:32:58.237 | INFO | Analyzing chunk frames = [8197120,8197632) | 20260507 07:32:58.238 | INFO | MemInfo: { | 20260507 07:32:58.238 | INFO | "gpu" : " 17 GB", | 20260507 07:32:58.238 | INFO | "huge" : "", | 20260507 07:32:58.238 | INFO | "huge_shared" : " 27 GB", | 20260507 07:32:58.238 | INFO | "malloc" : " 14 GB", | 20260507 07:32:58.238 | INFO | "pinned" : " 29 GB" | 20260507 07:32:58.238 | INFO | } | 20260507 07:32:58.238 | INFO | Perf report for the last 5.14119 seconds: | 20260507 07:32:58.238 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:32:58.238 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:58.238 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:58.238 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:58.238 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 07:32:58.238 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:58.238 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:58.238 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:32:58.238 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:58.238 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:32:58.238 | INFO | Uploaded 36GiB to GPU in 2.04417s (17.6111GiB/s) | 20260507 07:32:58.238 | INFO | Downloaded 34GiB to GPU in 1.95269s (17.4119GiB/s) | 20260507 07:33:03.378 | INFO | Analyzing chunk frames = [8197632,8198144) | 20260507 07:33:03.378 | INFO | MemInfo: { | 20260507 07:33:03.378 | INFO | "gpu" : " 17 GB", | 20260507 07:33:03.378 | INFO | "huge" : "", | 20260507 07:33:03.378 | INFO | "huge_shared" : " 27 GB", | 20260507 07:33:03.378 | INFO | "malloc" : " 14 GB", | 20260507 07:33:03.378 | INFO | "pinned" : " 29 GB" | 20260507 07:33:03.378 | INFO | } | 20260507 07:33:03.378 | INFO | Perf report for the last 5.14062 seconds: | 20260507 07:33:03.378 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:33:03.378 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:03.378 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:03.378 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:03.378 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 07:33:03.378 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:03.378 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:03.378 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:33:03.378 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:03.378 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:03.378 | INFO | Uploaded 36GiB to GPU in 2.01105s (17.9011GiB/s) | 20260507 07:33:03.379 | INFO | Downloaded 34GiB to GPU in 1.94719s (17.461GiB/s) | 20260507 07:33:08.523 | INFO | Analyzing chunk frames = [8198144,8198656) | 20260507 07:33:08.523 | INFO | MemInfo: { | 20260507 07:33:08.523 | INFO | "gpu" : " 17 GB", | 20260507 07:33:08.523 | INFO | "huge" : "", | 20260507 07:33:08.523 | INFO | "huge_shared" : " 27 GB", | 20260507 07:33:08.523 | INFO | "malloc" : " 14 GB", | 20260507 07:33:08.523 | INFO | "pinned" : " 29 GB" | 20260507 07:33:08.523 | INFO | } | 20260507 07:33:08.523 | INFO | Perf report for the last 5.14474 seconds: | 20260507 07:33:08.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:33:08.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:08.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:08.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:08.523 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 07:33:08.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:08.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:08.523 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:33:08.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:08.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:08.523 | INFO | Uploaded 36GiB to GPU in 2.04142s (17.6348GiB/s) | 20260507 07:33:08.523 | INFO | Downloaded 34GiB to GPU in 1.90867s (17.8135GiB/s) | 20260507 07:33:13.652 | INFO | Analyzing chunk frames = [8198656,8199168) | 20260507 07:33:13.652 | INFO | MemInfo: { | 20260507 07:33:13.652 | INFO | "gpu" : " 17 GB", | 20260507 07:33:13.652 | INFO | "huge" : "", | 20260507 07:33:13.652 | INFO | "huge_shared" : " 27 GB", | 20260507 07:33:13.652 | INFO | "malloc" : " 14 GB", | 20260507 07:33:13.652 | INFO | "pinned" : " 29 GB" | 20260507 07:33:13.652 | INFO | } | 20260507 07:33:13.652 | INFO | Perf report for the last 5.12936 seconds: | 20260507 07:33:13.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:33:13.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:13.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:13.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:13.652 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:33:13.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:13.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:13.653 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:33:13.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:13.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:13.653 | INFO | Uploaded 36GiB to GPU in 2.06668s (17.4192GiB/s) | 20260507 07:33:13.653 | INFO | Downloaded 34GiB to GPU in 1.94443s (17.4859GiB/s) | 20260507 07:33:18.793 | INFO | Analyzing chunk frames = [8199168,8199680) | 20260507 07:33:18.793 | INFO | MemInfo: { | 20260507 07:33:18.793 | INFO | "gpu" : " 17 GB", | 20260507 07:33:18.793 | INFO | "huge" : "", | 20260507 07:33:18.793 | INFO | "huge_shared" : " 27 GB", | 20260507 07:33:18.793 | INFO | "malloc" : " 14 GB", | 20260507 07:33:18.793 | INFO | "pinned" : " 29 GB" | 20260507 07:33:18.793 | INFO | } | 20260507 07:33:18.793 | INFO | Perf report for the last 5.14049 seconds: | 20260507 07:33:18.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:33:18.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:18.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:18.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:18.793 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 07:33:18.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:18.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:18.793 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 07:33:18.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:18.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:18.793 | INFO | Uploaded 36GiB to GPU in 2.00417s (17.9625GiB/s) | 20260507 07:33:18.793 | INFO | Downloaded 34GiB to GPU in 1.98073s (17.1654GiB/s) | 20260507 07:33:23.933 | INFO | Analyzing chunk frames = [8199680,8200192) | 20260507 07:33:23.933 | INFO | MemInfo: { | 20260507 07:33:23.933 | INFO | "gpu" : " 17 GB", | 20260507 07:33:23.933 | INFO | "huge" : "", | 20260507 07:33:23.933 | INFO | "huge_shared" : " 27 GB", | 20260507 07:33:23.933 | INFO | "malloc" : " 14 GB", | 20260507 07:33:23.933 | INFO | "pinned" : " 29 GB" | 20260507 07:33:23.933 | INFO | } | 20260507 07:33:23.933 | INFO | Perf report for the last 5.14014 seconds: | 20260507 07:33:23.933 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:33:23.933 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:23.933 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:23.933 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:23.933 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:33:23.933 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:23.933 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:23.933 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 07:33:23.933 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:23.933 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:23.933 | INFO | Uploaded 36GiB to GPU in 1.97168s (18.2586GiB/s) | 20260507 07:33:23.933 | INFO | Downloaded 34GiB to GPU in 1.96215s (17.328GiB/s) | 20260507 07:33:23.933 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:33:23.933 | INFO | Total mem usage: 18 GiB | 20260507 07:33:23.933 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:33:23.933 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:33:23.933 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:33:23.933 | INFO | High Water Breakdown: | 20260507 07:33:23.933 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:33:23.933 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:33:23.933 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:33:23.933 | INFO | ------------------------------------------ | 20260507 07:33:23.933 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:33:23.933 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:33:23.934 | INFO | Total mem usage: 26.5 GiB | 20260507 07:33:23.934 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:33:23.934 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:33:23.934 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:33:23.934 | INFO | High Water Breakdown: | 20260507 07:33:23.934 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:33:23.934 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:33:23.934 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:33:23.934 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:33:23.934 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:33:23.934 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:33:23.934 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:33:23.934 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:33:23.934 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:33:23.934 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:33:23.934 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:33:23.934 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:33:23.934 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:33:23.934 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:33:23.934 | INFO | ------------------------------------------ | 20260507 07:33:23.934 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:33:23.934 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:33:23.934 | INFO | Total mem usage: 25.6 GiB | 20260507 07:33:23.934 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:33:23.934 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:33:23.934 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:33:23.934 | INFO | High Water Breakdown: | 20260507 07:33:23.934 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:33:23.934 | INFO | ------------------------------------------ | 20260507 07:33:23.934 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:33:23.934 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:33:23.934 | INFO | Total mem usage: 15.7 GiB | 20260507 07:33:23.934 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:33:23.934 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:33:23.934 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:33:23.934 | INFO | High Water Breakdown: | 20260507 07:33:23.934 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:33:23.934 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:33:23.934 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:33:23.934 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:33:23.934 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:33:23.934 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:33:23.934 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:33:23.934 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:33:23.934 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:33:23.934 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:33:23.934 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:33:23.934 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:33:23.934 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:33:23.934 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:33:23.934 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:33:23.934 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:33:23.934 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:33:23.934 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:33:23.934 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:33:23.934 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:33:23.934 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:33:23.934 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:33:23.934 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:33:23.934 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:33:23.934 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:33:23.934 | INFO | ------------------------------------------ | 20260507 07:33:23.934 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:33:23.934 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:33:23.934 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:33:23.934 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:33:23.934 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.47e+01, 1.34e+01 / 1.10e+00 | 20260507 07:33:23.934 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.01e-01, 1.35e-02 / 4.40e-03 | 20260507 07:33:23.934 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.53e+00, 4.99e-01 / 2.06e-01 | 20260507 07:33:23.934 | INFO | PerfReport: 3.21%, Histogram -- 9.84e-01 / 1.50e+00, 1.63e-01 / 9.77e-02 | 20260507 07:33:23.934 | INFO | PerfReport: 16.29%, DME -- 3.99e+01 / 4.46e+01, 3.82e+00 / 1.07e+00 | 20260507 07:33:23.934 | INFO | PerfReport: 57.69%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.21e+00 / 1.23e+00 | 20260507 07:33:23.934 | INFO | PerfReport: 6.93%, PulseAccumulating -- 2.12e+00 / 2.72e+00, 3.89e-01 / 1.58e-01 | 20260507 07:33:23.934 | INFO | PerfReport: 2.94%, Metrics -- 9.01e-01 / 1.20e+00, 2.41e-01 / 1.33e-01 | 20260507 07:33:23.934 | INFO | PerfReport: 0.99%, PulseSerializing -- 3.02e-01 / 3.64e-01, 5.13e-02 / 1.84e-02 | 20260507 07:33:23.934 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:33:23.934 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:33:23.934 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:33:23.934 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 07:33:23.934 | INFO | PerfReport: 21.94%, Upload -- 2.11e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 07:33:23.934 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 07:33:23.934 | INFO | PerfReport: 31.85%, Compute -- 3.06e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 07:33:23.934 | INFO | PerfReport: 1.62%, LockUpload -- 1.55e+00 / 0.00e+00, 4.99e+00 / 0.00e+00 | 20260507 07:33:23.934 | INFO | PerfReport: 1.24%, LockDownload -- 1.19e+00 / 0.00e+00, 2.87e+00 / 0.00e+00 | 20260507 07:33:23.934 | INFO | PerfReport: 21.95%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 07:33:23.934 | INFO | PerfReport: 0.02%, Misc -- 1.45e-02 / 0.00e+00, 2.06e-02 / 0.00e+00 | 20260507 07:33:24.031 | INFO | Starting a round of Baz IO | 20260507 07:33:27.123 | INFO | Populating RTMetrics report | 20260507 07:33:27.124 | INFO | RTMetrics report populated | 20260507 07:33:27.124 | INFO | Opening tmp RTMetrics json file | 20260507 07:33:27.138 | INFO | Serializing RTMetrics to json | 20260507 07:33:27.140 | INFO | Serialization done, swapping tmp file now... | 20260507 07:33:27.140 | INFO | RTMetrics done | 20260507 07:33:27.208 | INFO | Finished a round of Baz IO: Wrote 1.64 GiB in 3.176 seconds | 20260507 07:33:27.208 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.55% / 0.00% / 67.45% | 20260507 07:33:27.208 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.103382% / 0% / 5.5631e-05% | 20260507 07:33:27.208 | INFO | Baz IO overall: | 20260507 07:33:27.208 | INFO | BytesWritten : 4107547834751 | 20260507 07:33:27.208 | INFO | NumEvents : 1966400263694 | 20260507 07:33:29.079 | INFO | Analyzing chunk frames = [8200192,8200704) | 20260507 07:33:29.079 | INFO | MemInfo: { | 20260507 07:33:29.079 | INFO | "gpu" : " 17 GB", | 20260507 07:33:29.079 | INFO | "huge" : "", | 20260507 07:33:29.079 | INFO | "huge_shared" : " 27 GB", | 20260507 07:33:29.079 | INFO | "malloc" : " 14 GB", | 20260507 07:33:29.079 | INFO | "pinned" : " 29 GB" | 20260507 07:33:29.079 | INFO | } | 20260507 07:33:29.079 | INFO | Perf report for the last 5.14576 seconds: | 20260507 07:33:29.079 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:33:29.079 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:29.079 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:29.079 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:29.079 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 07:33:29.079 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:29.079 | INFO | RT_METRICS : 43.3%, 1.0, 22.7ms, 0 | 20260507 07:33:29.079 | INFO | PRE_HQ : 59.5%, 1.9, 58.3ms, 0 | 20260507 07:33:29.079 | INFO | BAZWRITER : 61.3%, 3.2, 100.8ms, 0 | 20260507 07:33:29.079 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:33:29.079 | INFO | Uploaded 36GiB to GPU in 2.05238s (17.5406GiB/s) | 20260507 07:33:29.079 | INFO | Downloaded 38GiB to GPU in 2.18379s (17.401GiB/s) | 20260507 07:33:34.215 | INFO | Analyzing chunk frames = [8200704,8201216) | 20260507 07:33:34.215 | INFO | MemInfo: { | 20260507 07:33:34.215 | INFO | "gpu" : " 17 GB", | 20260507 07:33:34.215 | INFO | "huge" : "", | 20260507 07:33:34.215 | INFO | "huge_shared" : " 27 GB", | 20260507 07:33:34.215 | INFO | "malloc" : " 14 GB", | 20260507 07:33:34.215 | INFO | "pinned" : " 29 GB" | 20260507 07:33:34.215 | INFO | } | 20260507 07:33:34.215 | INFO | Perf report for the last 5.13658 seconds: | 20260507 07:33:34.215 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:33:34.215 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:34.215 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:34.215 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:34.215 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 07:33:34.215 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:34.215 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:34.215 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 07:33:34.216 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:34.216 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:34.216 | INFO | Uploaded 36GiB to GPU in 2.0215s (17.8086GiB/s) | 20260507 07:33:34.216 | INFO | Downloaded 34GiB to GPU in 1.9529s (17.41GiB/s) | 20260507 07:33:39.347 | INFO | Analyzing chunk frames = [8201216,8201728) | 20260507 07:33:39.347 | INFO | MemInfo: { | 20260507 07:33:39.347 | INFO | "gpu" : " 17 GB", | 20260507 07:33:39.347 | INFO | "huge" : "", | 20260507 07:33:39.347 | INFO | "huge_shared" : " 27 GB", | 20260507 07:33:39.347 | INFO | "malloc" : " 14 GB", | 20260507 07:33:39.347 | INFO | "pinned" : " 29 GB" | 20260507 07:33:39.347 | INFO | } | 20260507 07:33:39.347 | INFO | Perf report for the last 5.13179 seconds: | 20260507 07:33:39.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:33:39.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:39.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:39.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:39.347 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 07:33:39.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:39.347 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:39.347 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:33:39.347 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:39.347 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:39.347 | INFO | Uploaded 36GiB to GPU in 2.05183s (17.5453GiB/s) | 20260507 07:33:39.347 | INFO | Downloaded 34GiB to GPU in 1.95485s (17.3926GiB/s) | 20260507 07:33:44.494 | INFO | Analyzing chunk frames = [8201728,8202240) | 20260507 07:33:44.494 | INFO | MemInfo: { | 20260507 07:33:44.494 | INFO | "gpu" : " 17 GB", | 20260507 07:33:44.494 | INFO | "huge" : "", | 20260507 07:33:44.494 | INFO | "huge_shared" : " 27 GB", | 20260507 07:33:44.494 | INFO | "malloc" : " 14 GB", | 20260507 07:33:44.494 | INFO | "pinned" : " 29 GB" | 20260507 07:33:44.494 | INFO | } | 20260507 07:33:44.494 | INFO | Perf report for the last 5.14711 seconds: | 20260507 07:33:44.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:33:44.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:44.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:44.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:44.494 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 07:33:44.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:44.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:44.494 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:33:44.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:44.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:44.495 | INFO | Uploaded 36GiB to GPU in 2.01429s (17.8723GiB/s) | 20260507 07:33:44.495 | INFO | Downloaded 34GiB to GPU in 1.94564s (17.475GiB/s) | 20260507 07:33:49.635 | INFO | Analyzing chunk frames = [8202240,8202752) | 20260507 07:33:49.635 | INFO | MemInfo: { | 20260507 07:33:49.635 | INFO | "gpu" : " 17 GB", | 20260507 07:33:49.636 | INFO | "huge" : "", | 20260507 07:33:49.636 | INFO | "huge_shared" : " 27 GB", | 20260507 07:33:49.636 | INFO | "malloc" : " 14 GB", | 20260507 07:33:49.636 | INFO | "pinned" : " 29 GB" | 20260507 07:33:49.636 | INFO | } | 20260507 07:33:49.636 | INFO | Perf report for the last 5.14164 seconds: | 20260507 07:33:49.636 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:33:49.636 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:49.636 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:49.636 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:49.636 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 07:33:49.636 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:49.636 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:49.636 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:33:49.636 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:49.636 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:49.636 | INFO | Uploaded 36GiB to GPU in 2.0269s (17.7611GiB/s) | 20260507 07:33:49.636 | INFO | Downloaded 34GiB to GPU in 1.9292s (17.6238GiB/s) | 20260507 07:33:54.771 | INFO | Analyzing chunk frames = [8202752,8203264) | 20260507 07:33:54.771 | INFO | MemInfo: { | 20260507 07:33:54.771 | INFO | "gpu" : " 17 GB", | 20260507 07:33:54.771 | INFO | "huge" : "", | 20260507 07:33:54.772 | INFO | "huge_shared" : " 27 GB", | 20260507 07:33:54.772 | INFO | "malloc" : " 14 GB", | 20260507 07:33:54.772 | INFO | "pinned" : " 29 GB" | 20260507 07:33:54.772 | INFO | } | 20260507 07:33:54.772 | INFO | Perf report for the last 5.13557 seconds: | 20260507 07:33:54.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:33:54.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:54.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:54.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:54.772 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 07:33:54.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:54.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:54.772 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:33:54.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:54.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:54.772 | INFO | Uploaded 36GiB to GPU in 2.02543s (17.774GiB/s) | 20260507 07:33:54.772 | INFO | Downloaded 34GiB to GPU in 1.92422s (17.6695GiB/s) | 20260507 07:33:59.904 | INFO | Analyzing chunk frames = [8203264,8203776) | 20260507 07:33:59.904 | INFO | MemInfo: { | 20260507 07:33:59.904 | INFO | "gpu" : " 17 GB", | 20260507 07:33:59.904 | INFO | "huge" : "", | 20260507 07:33:59.904 | INFO | "huge_shared" : " 27 GB", | 20260507 07:33:59.904 | INFO | "malloc" : " 14 GB", | 20260507 07:33:59.904 | INFO | "pinned" : " 29 GB" | 20260507 07:33:59.904 | INFO | } | 20260507 07:33:59.904 | INFO | Perf report for the last 5.13264 seconds: | 20260507 07:33:59.904 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:33:59.904 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:59.904 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:59.904 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:59.904 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:33:59.904 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:59.904 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:59.904 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 07:33:59.904 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:59.904 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:33:59.905 | INFO | Uploaded 36GiB to GPU in 1.87936s (19.1554GiB/s) | 20260507 07:33:59.905 | INFO | Downloaded 34GiB to GPU in 1.96932s (17.2649GiB/s) | 20260507 07:34:05.053 | INFO | Analyzing chunk frames = [8203776,8204288) | 20260507 07:34:05.053 | INFO | MemInfo: { | 20260507 07:34:05.053 | INFO | "gpu" : " 17 GB", | 20260507 07:34:05.053 | INFO | "huge" : "", | 20260507 07:34:05.053 | INFO | "huge_shared" : " 27 GB", | 20260507 07:34:05.053 | INFO | "malloc" : " 14 GB", | 20260507 07:34:05.053 | INFO | "pinned" : " 29 GB" | 20260507 07:34:05.053 | INFO | } | 20260507 07:34:05.053 | INFO | Perf report for the last 5.14856 seconds: | 20260507 07:34:05.053 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:34:05.053 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:05.053 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:05.053 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:05.053 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 07:34:05.053 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:05.053 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:05.053 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:34:05.053 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:05.053 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:05.053 | INFO | Uploaded 36GiB to GPU in 1.95724s (18.3932GiB/s) | 20260507 07:34:05.053 | INFO | Downloaded 34GiB to GPU in 1.96337s (17.3172GiB/s) | 20260507 07:34:05.149 | INFO | Starting a round of Baz IO | 20260507 07:34:08.242 | INFO | Populating RTMetrics report | 20260507 07:34:08.242 | INFO | RTMetrics report populated | 20260507 07:34:08.242 | INFO | Opening tmp RTMetrics json file | 20260507 07:34:08.243 | INFO | Serializing RTMetrics to json | 20260507 07:34:08.260 | INFO | Serialization done, swapping tmp file now... | 20260507 07:34:08.261 | INFO | RTMetrics done | 20260507 07:34:08.325 | INFO | Finished a round of Baz IO: Wrote 1.638 GiB in 3.175 seconds | 20260507 07:34:08.325 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.55% / 0.00% / 67.45% | 20260507 07:34:08.325 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.102691% / 0% / 5.5631e-05% | 20260507 07:34:08.325 | INFO | Baz IO overall: | 20260507 07:34:08.325 | INFO | BytesWritten : 4109307112291 | 20260507 07:34:08.325 | INFO | NumEvents : 1967221743082 | 20260507 07:34:10.189 | INFO | Analyzing chunk frames = [8204288,8204800) | 20260507 07:34:10.189 | INFO | MemInfo: { | 20260507 07:34:10.189 | INFO | "gpu" : " 17 GB", | 20260507 07:34:10.189 | INFO | "huge" : "", | 20260507 07:34:10.189 | INFO | "huge_shared" : " 27 GB", | 20260507 07:34:10.189 | INFO | "malloc" : " 14 GB", | 20260507 07:34:10.189 | INFO | "pinned" : " 29 GB" | 20260507 07:34:10.189 | INFO | } | 20260507 07:34:10.189 | INFO | Perf report for the last 5.13596 seconds: | 20260507 07:34:10.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:34:10.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:10.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:10.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:10.189 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 07:34:10.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:10.189 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:34:10.189 | INFO | PRE_HQ : 58.8%, 1.8, 54.5ms, 0 | 20260507 07:34:10.189 | INFO | BAZWRITER : 61.4%, 3.1, 97.4ms, 0 | 20260507 07:34:10.189 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:34:10.189 | INFO | Uploaded 36GiB to GPU in 2.03914s (17.6545GiB/s) | 20260507 07:34:10.189 | INFO | Downloaded 38GiB to GPU in 2.17305s (17.4869GiB/s) | 20260507 07:34:15.327 | INFO | Analyzing chunk frames = [8204800,8205312) | 20260507 07:34:15.327 | INFO | MemInfo: { | 20260507 07:34:15.327 | INFO | "gpu" : " 17 GB", | 20260507 07:34:15.327 | INFO | "huge" : "", | 20260507 07:34:15.327 | INFO | "huge_shared" : " 27 GB", | 20260507 07:34:15.327 | INFO | "malloc" : " 14 GB", | 20260507 07:34:15.327 | INFO | "pinned" : " 29 GB" | 20260507 07:34:15.327 | INFO | } | 20260507 07:34:15.327 | INFO | Perf report for the last 5.13836 seconds: | 20260507 07:34:15.327 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:34:15.327 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:15.327 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:15.327 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:15.327 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 07:34:15.327 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:15.327 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:15.327 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 07:34:15.327 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:15.327 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:15.327 | INFO | Uploaded 36GiB to GPU in 1.97327s (18.2438GiB/s) | 20260507 07:34:15.327 | INFO | Downloaded 34GiB to GPU in 1.93234s (17.5952GiB/s) | 20260507 07:34:20.471 | INFO | Analyzing chunk frames = [8205312,8205824) | 20260507 07:34:20.471 | INFO | MemInfo: { | 20260507 07:34:20.471 | INFO | "gpu" : " 17 GB", | 20260507 07:34:20.471 | INFO | "huge" : "", | 20260507 07:34:20.471 | INFO | "huge_shared" : " 27 GB", | 20260507 07:34:20.471 | INFO | "malloc" : " 14 GB", | 20260507 07:34:20.471 | INFO | "pinned" : " 29 GB" | 20260507 07:34:20.471 | INFO | } | 20260507 07:34:20.471 | INFO | Perf report for the last 5.14428 seconds: | 20260507 07:34:20.471 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:34:20.471 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:20.471 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:20.471 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:20.471 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 07:34:20.471 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:20.472 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:20.472 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:34:20.472 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:20.472 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:20.472 | INFO | Uploaded 36GiB to GPU in 2.0198s (17.8235GiB/s) | 20260507 07:34:20.472 | INFO | Downloaded 34GiB to GPU in 1.92093s (17.6998GiB/s) | 20260507 07:34:25.602 | INFO | Analyzing chunk frames = [8205824,8206336) | 20260507 07:34:25.602 | INFO | MemInfo: { | 20260507 07:34:25.602 | INFO | "gpu" : " 17 GB", | 20260507 07:34:25.602 | INFO | "huge" : "", | 20260507 07:34:25.602 | INFO | "huge_shared" : " 27 GB", | 20260507 07:34:25.602 | INFO | "malloc" : " 14 GB", | 20260507 07:34:25.602 | INFO | "pinned" : " 29 GB" | 20260507 07:34:25.602 | INFO | } | 20260507 07:34:25.602 | INFO | Perf report for the last 5.13078 seconds: | 20260507 07:34:25.602 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:34:25.602 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:25.602 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:25.602 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:25.602 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 07:34:25.602 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:25.602 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:25.602 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 07:34:25.602 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:25.602 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:25.602 | INFO | Uploaded 36GiB to GPU in 2.04685s (17.588GiB/s) | 20260507 07:34:25.602 | INFO | Downloaded 34GiB to GPU in 1.94687s (17.464GiB/s) | 20260507 07:34:30.744 | INFO | Analyzing chunk frames = [8206336,8206848) | 20260507 07:34:30.744 | INFO | MemInfo: { | 20260507 07:34:30.744 | INFO | "gpu" : " 17 GB", | 20260507 07:34:30.744 | INFO | "huge" : "", | 20260507 07:34:30.744 | INFO | "huge_shared" : " 27 GB", | 20260507 07:34:30.744 | INFO | "malloc" : " 14 GB", | 20260507 07:34:30.744 | INFO | "pinned" : " 29 GB" | 20260507 07:34:30.744 | INFO | } | 20260507 07:34:30.745 | INFO | Perf report for the last 5.14228 seconds: | 20260507 07:34:30.745 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:34:30.745 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:30.745 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:30.745 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:30.745 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 07:34:30.745 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:30.745 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:30.745 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 07:34:30.745 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:30.745 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:30.745 | INFO | Uploaded 36GiB to GPU in 2.05142s (17.5488GiB/s) | 20260507 07:34:30.745 | INFO | Downloaded 34GiB to GPU in 1.95322s (17.4071GiB/s) | 20260507 07:34:35.885 | INFO | Analyzing chunk frames = [8206848,8207360) | 20260507 07:34:35.885 | INFO | MemInfo: { | 20260507 07:34:35.885 | INFO | "gpu" : " 17 GB", | 20260507 07:34:35.885 | INFO | "huge" : "", | 20260507 07:34:35.885 | INFO | "huge_shared" : " 27 GB", | 20260507 07:34:35.885 | INFO | "malloc" : " 14 GB", | 20260507 07:34:35.885 | INFO | "pinned" : " 29 GB" | 20260507 07:34:35.885 | INFO | } | 20260507 07:34:35.885 | INFO | Perf report for the last 5.14013 seconds: | 20260507 07:34:35.885 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:34:35.885 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:35.885 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:35.885 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:35.885 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 07:34:35.885 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:35.885 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:35.885 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 07:34:35.885 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:35.885 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:35.885 | INFO | Uploaded 36GiB to GPU in 2.03165s (17.7196GiB/s) | 20260507 07:34:35.885 | INFO | Downloaded 34GiB to GPU in 1.9109s (17.7926GiB/s) | 20260507 07:34:41.027 | INFO | Analyzing chunk frames = [8207360,8207872) | 20260507 07:34:41.027 | INFO | MemInfo: { | 20260507 07:34:41.027 | INFO | "gpu" : " 17 GB", | 20260507 07:34:41.027 | INFO | "huge" : "", | 20260507 07:34:41.027 | INFO | "huge_shared" : " 27 GB", | 20260507 07:34:41.027 | INFO | "malloc" : " 14 GB", | 20260507 07:34:41.027 | INFO | "pinned" : " 29 GB" | 20260507 07:34:41.027 | INFO | } | 20260507 07:34:41.027 | INFO | Perf report for the last 5.14248 seconds: | 20260507 07:34:41.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:34:41.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:41.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:41.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:41.027 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:34:41.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:41.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:41.027 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 07:34:41.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:41.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:41.027 | INFO | Uploaded 36GiB to GPU in 1.92501s (18.7012GiB/s) | 20260507 07:34:41.027 | INFO | Downloaded 34GiB to GPU in 1.97271s (17.2352GiB/s) | 20260507 07:34:46.158 | INFO | Analyzing chunk frames = [8207872,8208384) | 20260507 07:34:46.158 | INFO | MemInfo: { | 20260507 07:34:46.158 | INFO | "gpu" : " 17 GB", | 20260507 07:34:46.158 | INFO | "huge" : "", | 20260507 07:34:46.158 | INFO | "huge_shared" : " 27 GB", | 20260507 07:34:46.158 | INFO | "malloc" : " 14 GB", | 20260507 07:34:46.158 | INFO | "pinned" : " 29 GB" | 20260507 07:34:46.158 | INFO | } | 20260507 07:34:46.158 | INFO | Perf report for the last 5.13127 seconds: | 20260507 07:34:46.158 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:34:46.158 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:46.158 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:46.158 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:46.158 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 07:34:46.158 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:46.158 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:46.158 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 07:34:46.158 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:46.158 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:46.159 | INFO | Uploaded 36GiB to GPU in 1.99079s (18.0832GiB/s) | 20260507 07:34:46.159 | INFO | Downloaded 34GiB to GPU in 1.95519s (17.3896GiB/s) | 20260507 07:34:46.159 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:34:46.159 | INFO | Total mem usage: 18 GiB | 20260507 07:34:46.159 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:34:46.159 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:34:46.159 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:34:46.159 | INFO | High Water Breakdown: | 20260507 07:34:46.159 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:34:46.159 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:34:46.159 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:34:46.159 | INFO | ------------------------------------------ | 20260507 07:34:46.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:34:46.159 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:34:46.159 | INFO | Total mem usage: 26.5 GiB | 20260507 07:34:46.159 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:34:46.159 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:34:46.159 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:34:46.159 | INFO | High Water Breakdown: | 20260507 07:34:46.159 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:34:46.159 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:34:46.159 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:34:46.159 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:34:46.159 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:34:46.159 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:34:46.159 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:34:46.159 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:34:46.159 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:34:46.159 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:34:46.159 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:34:46.159 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:34:46.159 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:34:46.159 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:34:46.159 | INFO | ------------------------------------------ | 20260507 07:34:46.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:34:46.159 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:34:46.159 | INFO | Total mem usage: 25.6 GiB | 20260507 07:34:46.159 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:34:46.159 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:34:46.159 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:34:46.159 | INFO | High Water Breakdown: | 20260507 07:34:46.159 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:34:46.159 | INFO | ------------------------------------------ | 20260507 07:34:46.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:34:46.159 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:34:46.159 | INFO | Total mem usage: 15.7 GiB | 20260507 07:34:46.159 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:34:46.159 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:34:46.159 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:34:46.159 | INFO | High Water Breakdown: | 20260507 07:34:46.159 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:34:46.159 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:34:46.159 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:34:46.159 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:34:46.159 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:34:46.159 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:34:46.159 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:34:46.159 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:34:46.159 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:34:46.159 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:34:46.159 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:34:46.159 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:34:46.159 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:34:46.159 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:34:46.159 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:34:46.159 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:34:46.159 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:34:46.159 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:34:46.159 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:34:46.159 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:34:46.159 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:34:46.159 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:34:46.159 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:34:46.159 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:34:46.159 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:34:46.159 | INFO | ------------------------------------------ | 20260507 07:34:46.159 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:34:46.160 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:34:46.160 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:34:46.160 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:34:46.160 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.35e+01 / 1.17e+00 | 20260507 07:34:46.160 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.03e-01, 9.64e-03 / 6.20e-03 | 20260507 07:34:46.160 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.48e+00, 5.08e-01 / 2.25e-01 | 20260507 07:34:46.160 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.49e+00, 1.70e-01 / 9.93e-02 | 20260507 07:34:46.160 | INFO | PerfReport: 16.31%, DME -- 4.01e+01 / 4.51e+01, 4.00e+00 / 1.51e+00 | 20260507 07:34:46.160 | INFO | PerfReport: 57.68%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.27e+00 / 1.19e+00 | 20260507 07:34:46.160 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.72e+00, 3.88e-01 / 1.93e-01 | 20260507 07:34:46.160 | INFO | PerfReport: 2.93%, Metrics -- 9.01e-01 / 1.20e+00, 2.43e-01 / 1.13e-01 | 20260507 07:34:46.160 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.02e-01 / 3.59e-01, 5.12e-02 / 1.89e-02 | 20260507 07:34:46.160 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:34:46.160 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:34:46.160 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:34:46.160 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 07:34:46.160 | INFO | PerfReport: 21.75%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260507 07:34:46.160 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 07:34:46.160 | INFO | PerfReport: 31.95%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 07:34:46.160 | INFO | PerfReport: 1.48%, LockUpload -- 1.42e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 07:34:46.160 | INFO | PerfReport: 1.34%, LockDownload -- 1.29e+00 / 0.00e+00, 3.16e+00 / 0.00e+00 | 20260507 07:34:46.160 | INFO | PerfReport: 22.07%, LockCompute -- 2.12e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 07:34:46.160 | INFO | PerfReport: 0.01%, Misc -- 1.38e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260507 07:34:46.255 | INFO | Starting a round of Baz IO | 20260507 07:34:49.352 | INFO | Populating RTMetrics report | 20260507 07:34:49.352 | INFO | RTMetrics report populated | 20260507 07:34:49.352 | INFO | Opening tmp RTMetrics json file | 20260507 07:34:49.369 | INFO | Serializing RTMetrics to json | 20260507 07:34:49.371 | INFO | Serialization done, swapping tmp file now... | 20260507 07:34:49.372 | INFO | RTMetrics done | 20260507 07:34:49.434 | INFO | Finished a round of Baz IO: Wrote 1.64 GiB in 3.179 seconds | 20260507 07:34:49.434 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.55% / 0.00% / 67.45% | 20260507 07:34:49.434 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.103494% / 3.97364e-06% / 7.15256e-05% | 20260507 07:34:49.434 | INFO | Baz IO overall: | 20260507 07:34:49.434 | INFO | BytesWritten : 4111068111198 | 20260507 07:34:49.434 | INFO | NumEvents : 1968044310788 | 20260507 07:34:51.303 | INFO | Analyzing chunk frames = [8208384,8208896) | 20260507 07:34:51.303 | INFO | MemInfo: { | 20260507 07:34:51.303 | INFO | "gpu" : " 17 GB", | 20260507 07:34:51.303 | INFO | "huge" : "", | 20260507 07:34:51.303 | INFO | "huge_shared" : " 27 GB", | 20260507 07:34:51.303 | INFO | "malloc" : " 14 GB", | 20260507 07:34:51.303 | INFO | "pinned" : " 29 GB" | 20260507 07:34:51.303 | INFO | } | 20260507 07:34:51.303 | INFO | Perf report for the last 5.14435 seconds: | 20260507 07:34:51.303 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:34:51.303 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:51.303 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:51.303 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:51.303 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 07:34:51.303 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:51.303 | INFO | RT_METRICS : 42.3%, 1.0, 22.2ms, 0 | 20260507 07:34:51.303 | INFO | PRE_HQ : 59.3%, 1.8, 56.5ms, 0 | 20260507 07:34:51.303 | INFO | BAZWRITER : 61.3%, 3.2, 99.9ms, 0 | 20260507 07:34:51.303 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:34:51.303 | INFO | Uploaded 36GiB to GPU in 2.0432s (17.6194GiB/s) | 20260507 07:34:51.303 | INFO | Downloaded 38GiB to GPU in 2.1805s (17.4272GiB/s) | 20260507 07:34:56.442 | INFO | Analyzing chunk frames = [8208896,8209408) | 20260507 07:34:56.442 | INFO | MemInfo: { | 20260507 07:34:56.442 | INFO | "gpu" : " 17 GB", | 20260507 07:34:56.442 | INFO | "huge" : "", | 20260507 07:34:56.442 | INFO | "huge_shared" : " 27 GB", | 20260507 07:34:56.442 | INFO | "malloc" : " 14 GB", | 20260507 07:34:56.442 | INFO | "pinned" : " 29 GB" | 20260507 07:34:56.442 | INFO | } | 20260507 07:34:56.442 | INFO | Perf report for the last 5.13957 seconds: | 20260507 07:34:56.442 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:34:56.442 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:56.442 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:56.442 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:56.442 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 07:34:56.442 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:56.443 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:56.443 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 07:34:56.443 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:56.443 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:34:56.443 | INFO | Uploaded 36GiB to GPU in 1.99442s (18.0503GiB/s) | 20260507 07:34:56.443 | INFO | Downloaded 34GiB to GPU in 1.93506s (17.5705GiB/s) | 20260507 07:35:01.590 | INFO | Analyzing chunk frames = [8209408,8209920) | 20260507 07:35:01.590 | INFO | MemInfo: { | 20260507 07:35:01.590 | INFO | "gpu" : " 17 GB", | 20260507 07:35:01.590 | INFO | "huge" : "", | 20260507 07:35:01.590 | INFO | "huge_shared" : " 27 GB", | 20260507 07:35:01.590 | INFO | "malloc" : " 14 GB", | 20260507 07:35:01.590 | INFO | "pinned" : " 29 GB" | 20260507 07:35:01.590 | INFO | } | 20260507 07:35:01.590 | INFO | Perf report for the last 5.14788 seconds: | 20260507 07:35:01.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:35:01.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:01.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:01.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:01.590 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 07:35:01.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:01.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:01.590 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:35:01.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:01.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:01.591 | INFO | Uploaded 36GiB to GPU in 2.0533s (17.5327GiB/s) | 20260507 07:35:01.591 | INFO | Downloaded 34GiB to GPU in 1.9502s (17.4342GiB/s) | 20260507 07:35:06.722 | INFO | Analyzing chunk frames = [8209920,8210432) | 20260507 07:35:06.722 | INFO | MemInfo: { | 20260507 07:35:06.722 | INFO | "gpu" : " 17 GB", | 20260507 07:35:06.722 | INFO | "huge" : "", | 20260507 07:35:06.722 | INFO | "huge_shared" : " 27 GB", | 20260507 07:35:06.722 | INFO | "malloc" : " 14 GB", | 20260507 07:35:06.722 | INFO | "pinned" : " 29 GB" | 20260507 07:35:06.722 | INFO | } | 20260507 07:35:06.723 | INFO | Perf report for the last 5.13224 seconds: | 20260507 07:35:06.723 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:35:06.723 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:06.723 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:06.723 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:06.723 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 07:35:06.723 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:06.723 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:06.723 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:35:06.723 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:06.723 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:06.723 | INFO | Uploaded 36GiB to GPU in 2.03832s (17.6616GiB/s) | 20260507 07:35:06.723 | INFO | Downloaded 34GiB to GPU in 1.91095s (17.7922GiB/s) | 20260507 07:35:11.857 | INFO | Analyzing chunk frames = [8210432,8210944) | 20260507 07:35:11.857 | INFO | MemInfo: { | 20260507 07:35:11.857 | INFO | "gpu" : " 17 GB", | 20260507 07:35:11.857 | INFO | "huge" : "", | 20260507 07:35:11.857 | INFO | "huge_shared" : " 27 GB", | 20260507 07:35:11.857 | INFO | "malloc" : " 14 GB", | 20260507 07:35:11.857 | INFO | "pinned" : " 29 GB" | 20260507 07:35:11.857 | INFO | } | 20260507 07:35:11.857 | INFO | Perf report for the last 5.13431 seconds: | 20260507 07:35:11.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:35:11.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:11.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:11.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:11.857 | INFO | ANALYSIS : 60.3%, 3.0, 93.7ms, 0 | 20260507 07:35:11.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:11.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:11.857 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:35:11.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:11.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:11.857 | INFO | Uploaded 36GiB to GPU in 2.05313s (17.5342GiB/s) | 20260507 07:35:11.857 | INFO | Downloaded 34GiB to GPU in 1.90675s (17.8314GiB/s) | 20260507 07:35:17.002 | INFO | Analyzing chunk frames = [8210944,8211456) | 20260507 07:35:17.002 | INFO | MemInfo: { | 20260507 07:35:17.002 | INFO | "gpu" : " 17 GB", | 20260507 07:35:17.002 | INFO | "huge" : "", | 20260507 07:35:17.002 | INFO | "huge_shared" : " 27 GB", | 20260507 07:35:17.002 | INFO | "malloc" : " 14 GB", | 20260507 07:35:17.002 | INFO | "pinned" : " 29 GB" | 20260507 07:35:17.002 | INFO | } | 20260507 07:35:17.002 | INFO | Perf report for the last 5.14554 seconds: | 20260507 07:35:17.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:35:17.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:17.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:17.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:17.002 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 07:35:17.003 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:17.003 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:17.003 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:35:17.003 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:17.003 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:17.003 | INFO | Uploaded 36GiB to GPU in 2.03032s (17.7312GiB/s) | 20260507 07:35:17.003 | INFO | Downloaded 34GiB to GPU in 1.91989s (17.7094GiB/s) | 20260507 07:35:22.140 | INFO | Analyzing chunk frames = [8211456,8211968) | 20260507 07:35:22.140 | INFO | MemInfo: { | 20260507 07:35:22.140 | INFO | "gpu" : " 17 GB", | 20260507 07:35:22.140 | INFO | "huge" : "", | 20260507 07:35:22.140 | INFO | "huge_shared" : " 27 GB", | 20260507 07:35:22.140 | INFO | "malloc" : " 14 GB", | 20260507 07:35:22.140 | INFO | "pinned" : " 29 GB" | 20260507 07:35:22.140 | INFO | } | 20260507 07:35:22.140 | INFO | Perf report for the last 5.13747 seconds: | 20260507 07:35:22.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:35:22.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:22.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:22.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:22.140 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:35:22.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:22.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:22.140 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:35:22.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:22.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:22.140 | INFO | Uploaded 36GiB to GPU in 1.92866s (18.6659GiB/s) | 20260507 07:35:22.140 | INFO | Downloaded 34GiB to GPU in 1.9601s (17.3461GiB/s) | 20260507 07:35:27.274 | INFO | Analyzing chunk frames = [8211968,8212480) | 20260507 07:35:27.274 | INFO | MemInfo: { | 20260507 07:35:27.274 | INFO | "gpu" : " 17 GB", | 20260507 07:35:27.274 | INFO | "huge" : "", | 20260507 07:35:27.274 | INFO | "huge_shared" : " 27 GB", | 20260507 07:35:27.274 | INFO | "malloc" : " 14 GB", | 20260507 07:35:27.274 | INFO | "pinned" : " 29 GB" | 20260507 07:35:27.274 | INFO | } | 20260507 07:35:27.274 | INFO | Perf report for the last 5.13385 seconds: | 20260507 07:35:27.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:35:27.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:27.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:27.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:27.274 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:35:27.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:27.274 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:27.274 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 07:35:27.274 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:27.274 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:27.274 | INFO | Uploaded 36GiB to GPU in 1.97873s (18.1935GiB/s) | 20260507 07:35:27.274 | INFO | Downloaded 34GiB to GPU in 1.96265s (17.3235GiB/s) | 20260507 07:35:27.371 | INFO | Starting a round of Baz IO | 20260507 07:35:30.465 | INFO | Populating RTMetrics report | 20260507 07:35:30.465 | INFO | RTMetrics report populated | 20260507 07:35:30.465 | INFO | Opening tmp RTMetrics json file | 20260507 07:35:30.471 | INFO | Serializing RTMetrics to json | 20260507 07:35:30.482 | INFO | Serialization done, swapping tmp file now... | 20260507 07:35:30.483 | INFO | RTMetrics done | 20260507 07:35:30.552 | INFO | Finished a round of Baz IO: Wrote 1.639 GiB in 3.18 seconds | 20260507 07:35:30.552 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.55% / 0.00% / 67.45% | 20260507 07:35:30.552 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.102739% / 3.97364e-06% / 7.15256e-05% | 20260507 07:35:30.552 | INFO | Baz IO overall: | 20260507 07:35:30.552 | INFO | BytesWritten : 4112828084802 | 20260507 07:35:30.552 | INFO | NumEvents : 1968866330474 | 20260507 07:35:32.414 | INFO | Analyzing chunk frames = [8212480,8212992) | 20260507 07:35:32.414 | INFO | MemInfo: { | 20260507 07:35:32.414 | INFO | "gpu" : " 17 GB", | 20260507 07:35:32.414 | INFO | "huge" : "", | 20260507 07:35:32.414 | INFO | "huge_shared" : " 27 GB", | 20260507 07:35:32.414 | INFO | "malloc" : " 14 GB", | 20260507 07:35:32.414 | INFO | "pinned" : " 29 GB" | 20260507 07:35:32.414 | INFO | } | 20260507 07:35:32.415 | INFO | Perf report for the last 5.14075 seconds: | 20260507 07:35:32.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:35:32.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:32.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:32.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:32.415 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 07:35:32.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:32.415 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 07:35:32.415 | INFO | PRE_HQ : 59.0%, 1.8, 54.6ms, 0 | 20260507 07:35:32.415 | INFO | BAZWRITER : 61.5%, 3.1, 97.2ms, 0 | 20260507 07:35:32.415 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:35:32.415 | INFO | Uploaded 36GiB to GPU in 2.04816s (17.5768GiB/s) | 20260507 07:35:32.415 | INFO | Downloaded 38GiB to GPU in 2.1746s (17.4745GiB/s) | 20260507 07:35:37.556 | INFO | Analyzing chunk frames = [8212992,8213504) | 20260507 07:35:37.556 | INFO | MemInfo: { | 20260507 07:35:37.556 | INFO | "gpu" : " 17 GB", | 20260507 07:35:37.556 | INFO | "huge" : "", | 20260507 07:35:37.556 | INFO | "huge_shared" : " 27 GB", | 20260507 07:35:37.556 | INFO | "malloc" : " 14 GB", | 20260507 07:35:37.556 | INFO | "pinned" : " 29 GB" | 20260507 07:35:37.556 | INFO | } | 20260507 07:35:37.556 | INFO | Perf report for the last 5.1418 seconds: | 20260507 07:35:37.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:35:37.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:37.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:37.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:37.556 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:35:37.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:37.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:37.556 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:35:37.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:37.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:37.557 | INFO | Uploaded 36GiB to GPU in 1.98017s (18.1803GiB/s) | 20260507 07:35:37.557 | INFO | Downloaded 34GiB to GPU in 1.93098s (17.6077GiB/s) | 20260507 07:35:42.699 | INFO | Analyzing chunk frames = [8213504,8214016) | 20260507 07:35:42.699 | INFO | MemInfo: { | 20260507 07:35:42.699 | INFO | "gpu" : " 17 GB", | 20260507 07:35:42.699 | INFO | "huge" : "", | 20260507 07:35:42.699 | INFO | "huge_shared" : " 27 GB", | 20260507 07:35:42.699 | INFO | "malloc" : " 14 GB", | 20260507 07:35:42.699 | INFO | "pinned" : " 29 GB" | 20260507 07:35:42.699 | INFO | } | 20260507 07:35:42.700 | INFO | Perf report for the last 5.14321 seconds: | 20260507 07:35:42.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:35:42.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:42.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:42.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:42.700 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 07:35:42.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:42.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:42.700 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:35:42.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:42.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:42.700 | INFO | Uploaded 36GiB to GPU in 2.04359s (17.6161GiB/s) | 20260507 07:35:42.700 | INFO | Downloaded 34GiB to GPU in 1.95444s (17.3962GiB/s) | 20260507 07:35:47.835 | INFO | Analyzing chunk frames = [8214016,8214528) | 20260507 07:35:47.835 | INFO | MemInfo: { | 20260507 07:35:47.835 | INFO | "gpu" : " 17 GB", | 20260507 07:35:47.835 | INFO | "huge" : "", | 20260507 07:35:47.835 | INFO | "huge_shared" : " 27 GB", | 20260507 07:35:47.835 | INFO | "malloc" : " 14 GB", | 20260507 07:35:47.835 | INFO | "pinned" : " 29 GB" | 20260507 07:35:47.835 | INFO | } | 20260507 07:35:47.835 | INFO | Perf report for the last 5.13578 seconds: | 20260507 07:35:47.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:35:47.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:47.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:47.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:47.835 | INFO | ANALYSIS : 60.3%, 2.9, 93.1ms, 0 | 20260507 07:35:47.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:47.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:47.836 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:35:47.836 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:47.836 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:47.836 | INFO | Uploaded 36GiB to GPU in 2.03421s (17.6973GiB/s) | 20260507 07:35:47.836 | INFO | Downloaded 34GiB to GPU in 1.9545s (17.3957GiB/s) | 20260507 07:35:52.971 | INFO | Analyzing chunk frames = [8214528,8215040) | 20260507 07:35:52.971 | INFO | MemInfo: { | 20260507 07:35:52.971 | INFO | "gpu" : " 17 GB", | 20260507 07:35:52.971 | INFO | "huge" : "", | 20260507 07:35:52.972 | INFO | "huge_shared" : " 27 GB", | 20260507 07:35:52.972 | INFO | "malloc" : " 14 GB", | 20260507 07:35:52.972 | INFO | "pinned" : " 29 GB" | 20260507 07:35:52.972 | INFO | } | 20260507 07:35:52.972 | INFO | Perf report for the last 5.13624 seconds: | 20260507 07:35:52.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:35:52.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:52.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:52.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:52.972 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 07:35:52.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:52.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:52.972 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 07:35:52.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:52.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:52.972 | INFO | Uploaded 36GiB to GPU in 2.03215s (17.7152GiB/s) | 20260507 07:35:52.972 | INFO | Downloaded 34GiB to GPU in 1.91528s (17.752GiB/s) | 20260507 07:35:58.114 | INFO | Analyzing chunk frames = [8215040,8215552) | 20260507 07:35:58.114 | INFO | MemInfo: { | 20260507 07:35:58.114 | INFO | "gpu" : " 17 GB", | 20260507 07:35:58.115 | INFO | "huge" : "", | 20260507 07:35:58.115 | INFO | "huge_shared" : " 27 GB", | 20260507 07:35:58.115 | INFO | "malloc" : " 14 GB", | 20260507 07:35:58.115 | INFO | "pinned" : " 29 GB" | 20260507 07:35:58.115 | INFO | } | 20260507 07:35:58.115 | INFO | Perf report for the last 5.14304 seconds: | 20260507 07:35:58.115 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:35:58.115 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:58.115 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:58.115 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:58.115 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 07:35:58.115 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:58.115 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:58.115 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:35:58.115 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:58.115 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:35:58.115 | INFO | Uploaded 36GiB to GPU in 2.0538s (17.5285GiB/s) | 20260507 07:35:58.115 | INFO | Downloaded 34GiB to GPU in 1.92028s (17.7057GiB/s) | 20260507 07:36:03.253 | INFO | Analyzing chunk frames = [8215552,8216064) | 20260507 07:36:03.253 | INFO | MemInfo: { | 20260507 07:36:03.253 | INFO | "gpu" : " 17 GB", | 20260507 07:36:03.253 | INFO | "huge" : "", | 20260507 07:36:03.253 | INFO | "huge_shared" : " 27 GB", | 20260507 07:36:03.253 | INFO | "malloc" : " 14 GB", | 20260507 07:36:03.253 | INFO | "pinned" : " 29 GB" | 20260507 07:36:03.253 | INFO | } | 20260507 07:36:03.253 | INFO | Perf report for the last 5.13869 seconds: | 20260507 07:36:03.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:36:03.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:03.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:03.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:03.253 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 07:36:03.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:03.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:03.253 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:36:03.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:03.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:03.254 | INFO | Uploaded 36GiB to GPU in 2.03778s (17.6663GiB/s) | 20260507 07:36:03.254 | INFO | Downloaded 34GiB to GPU in 1.98787s (17.1037GiB/s) | 20260507 07:36:08.397 | INFO | Analyzing chunk frames = [8216064,8216576) | 20260507 07:36:08.397 | INFO | MemInfo: { | 20260507 07:36:08.397 | INFO | "gpu" : " 17 GB", | 20260507 07:36:08.397 | INFO | "huge" : "", | 20260507 07:36:08.397 | INFO | "huge_shared" : " 27 GB", | 20260507 07:36:08.397 | INFO | "malloc" : " 14 GB", | 20260507 07:36:08.397 | INFO | "pinned" : " 29 GB" | 20260507 07:36:08.397 | INFO | } | 20260507 07:36:08.397 | INFO | Perf report for the last 5.1435 seconds: | 20260507 07:36:08.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:36:08.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:08.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:08.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:08.397 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:36:08.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:08.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:08.397 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 07:36:08.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:08.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:08.397 | INFO | Uploaded 36GiB to GPU in 1.97552s (18.2231GiB/s) | 20260507 07:36:08.397 | INFO | Downloaded 34GiB to GPU in 1.96014s (17.3457GiB/s) | 20260507 07:36:08.397 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:36:08.397 | INFO | Total mem usage: 18 GiB | 20260507 07:36:08.397 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:36:08.397 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:36:08.397 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:36:08.397 | INFO | High Water Breakdown: | 20260507 07:36:08.397 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:36:08.397 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:36:08.397 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:36:08.397 | INFO | ------------------------------------------ | 20260507 07:36:08.397 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:36:08.397 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:36:08.397 | INFO | Total mem usage: 26.5 GiB | 20260507 07:36:08.397 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:36:08.397 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:36:08.397 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:36:08.397 | INFO | High Water Breakdown: | 20260507 07:36:08.397 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:36:08.397 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:36:08.397 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:36:08.397 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:36:08.397 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:36:08.397 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:36:08.398 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:36:08.398 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:36:08.398 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:36:08.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:36:08.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:36:08.398 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:36:08.398 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:36:08.398 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:36:08.398 | INFO | ------------------------------------------ | 20260507 07:36:08.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:36:08.398 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:36:08.398 | INFO | Total mem usage: 25.6 GiB | 20260507 07:36:08.398 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:36:08.398 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:36:08.398 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:36:08.398 | INFO | High Water Breakdown: | 20260507 07:36:08.398 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:36:08.398 | INFO | ------------------------------------------ | 20260507 07:36:08.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:36:08.398 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:36:08.398 | INFO | Total mem usage: 15.7 GiB | 20260507 07:36:08.398 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:36:08.398 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:36:08.398 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:36:08.398 | INFO | High Water Breakdown: | 20260507 07:36:08.398 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:36:08.398 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:36:08.398 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:36:08.398 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:36:08.398 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:36:08.398 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:36:08.398 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:36:08.398 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:36:08.398 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:36:08.398 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:36:08.398 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:36:08.398 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:36:08.398 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:36:08.398 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:36:08.398 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:36:08.398 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:36:08.398 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:36:08.398 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:36:08.398 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:36:08.398 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:36:08.398 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:36:08.398 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:36:08.398 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:36:08.398 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:36:08.398 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:36:08.398 | INFO | ------------------------------------------ | 20260507 07:36:08.398 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:36:08.398 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:36:08.398 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:36:08.398 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:36:08.398 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.35e+01 / 9.43e-01 | 20260507 07:36:08.398 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.02e-01, 9.88e-03 / 4.49e-03 | 20260507 07:36:08.398 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.48e+00, 5.07e-01 / 2.15e-01 | 20260507 07:36:08.398 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.51e+00, 1.71e-01 / 8.26e-02 | 20260507 07:36:08.398 | INFO | PerfReport: 16.28%, DME -- 4.00e+01 / 4.48e+01, 3.86e+00 / 1.27e+00 | 20260507 07:36:08.398 | INFO | PerfReport: 57.74%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.34e+00 / 1.24e+00 | 20260507 07:36:08.398 | INFO | PerfReport: 6.88%, PulseAccumulating -- 2.11e+00 / 2.72e+00, 3.84e-01 / 1.79e-01 | 20260507 07:36:08.398 | INFO | PerfReport: 2.93%, Metrics -- 8.99e-01 / 1.20e+00, 2.44e-01 / 1.43e-01 | 20260507 07:36:08.398 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.02e-01 / 3.58e-01, 5.09e-02 / 1.87e-02 | 20260507 07:36:08.398 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:36:08.398 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:36:08.398 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:36:08.398 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 07:36:08.398 | INFO | PerfReport: 21.88%, Upload -- 2.10e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260507 07:36:08.398 | INFO | PerfReport: 21.34%, Download -- 2.05e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 07:36:08.398 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 07:36:08.398 | INFO | PerfReport: 1.54%, LockUpload -- 1.48e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 07:36:08.398 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260507 07:36:08.398 | INFO | PerfReport: 22.01%, LockCompute -- 2.12e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 07:36:08.398 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260507 07:36:08.493 | INFO | Starting a round of Baz IO | 20260507 07:36:11.581 | INFO | Populating RTMetrics report | 20260507 07:36:11.581 | INFO | RTMetrics report populated | 20260507 07:36:11.581 | INFO | Opening tmp RTMetrics json file | 20260507 07:36:11.597 | INFO | Serializing RTMetrics to json | 20260507 07:36:11.598 | INFO | Serialization done, swapping tmp file now... | 20260507 07:36:11.598 | INFO | RTMetrics done | 20260507 07:36:11.664 | INFO | Finished a round of Baz IO: Wrote 1.637 GiB in 3.17 seconds | 20260507 07:36:11.664 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.55% / 0.00% / 67.45% | 20260507 07:36:11.664 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.102564% / 0% / 4.76837e-05% | 20260507 07:36:11.664 | INFO | Baz IO overall: | 20260507 07:36:11.664 | INFO | BytesWritten : 4114586183112 | 20260507 07:36:11.664 | INFO | NumEvents : 1969687633670 | 20260507 07:36:13.538 | INFO | Analyzing chunk frames = [8216576,8217088) | 20260507 07:36:13.538 | INFO | MemInfo: { | 20260507 07:36:13.538 | INFO | "gpu" : " 17 GB", | 20260507 07:36:13.538 | INFO | "huge" : "", | 20260507 07:36:13.538 | INFO | "huge_shared" : " 27 GB", | 20260507 07:36:13.538 | INFO | "malloc" : " 14 GB", | 20260507 07:36:13.538 | INFO | "pinned" : " 29 GB" | 20260507 07:36:13.538 | INFO | } | 20260507 07:36:13.538 | INFO | Perf report for the last 5.14155 seconds: | 20260507 07:36:13.538 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:36:13.538 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:13.538 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:13.538 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:13.538 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260507 07:36:13.538 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:13.538 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 07:36:13.538 | INFO | PRE_HQ : 58.8%, 1.8, 54.6ms, 0 | 20260507 07:36:13.538 | INFO | BAZWRITER : 61.1%, 3.1, 98.0ms, 0 | 20260507 07:36:13.538 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:36:13.539 | INFO | Uploaded 36GiB to GPU in 2.05487s (17.5193GiB/s) | 20260507 07:36:13.539 | INFO | Downloaded 38GiB to GPU in 2.17301s (17.4872GiB/s) | 20260507 07:36:18.671 | INFO | Analyzing chunk frames = [8217088,8217600) | 20260507 07:36:18.671 | INFO | MemInfo: { | 20260507 07:36:18.671 | INFO | "gpu" : " 17 GB", | 20260507 07:36:18.671 | INFO | "huge" : "", | 20260507 07:36:18.671 | INFO | "huge_shared" : " 27 GB", | 20260507 07:36:18.671 | INFO | "malloc" : " 14 GB", | 20260507 07:36:18.671 | INFO | "pinned" : " 29 GB" | 20260507 07:36:18.671 | INFO | } | 20260507 07:36:18.671 | INFO | Perf report for the last 5.13227 seconds: | 20260507 07:36:18.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:36:18.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:18.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:18.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:18.671 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 07:36:18.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:18.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:18.671 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 07:36:18.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:18.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:18.671 | INFO | Uploaded 36GiB to GPU in 1.98375s (18.1474GiB/s) | 20260507 07:36:18.671 | INFO | Downloaded 34GiB to GPU in 1.9392s (17.533GiB/s) | 20260507 07:36:23.811 | INFO | Analyzing chunk frames = [8217600,8218112) | 20260507 07:36:23.811 | INFO | MemInfo: { | 20260507 07:36:23.811 | INFO | "gpu" : " 17 GB", | 20260507 07:36:23.811 | INFO | "huge" : "", | 20260507 07:36:23.812 | INFO | "huge_shared" : " 27 GB", | 20260507 07:36:23.812 | INFO | "malloc" : " 14 GB", | 20260507 07:36:23.812 | INFO | "pinned" : " 29 GB" | 20260507 07:36:23.812 | INFO | } | 20260507 07:36:23.812 | INFO | Perf report for the last 5.14088 seconds: | 20260507 07:36:23.812 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:36:23.812 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:23.812 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:23.812 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:23.812 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:36:23.812 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:23.812 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:23.812 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:36:23.812 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:23.812 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:23.812 | INFO | Uploaded 36GiB to GPU in 2.04408s (17.6119GiB/s) | 20260507 07:36:23.812 | INFO | Downloaded 34GiB to GPU in 1.95495s (17.3917GiB/s) | 20260507 07:36:28.951 | INFO | Analyzing chunk frames = [8218112,8218624) | 20260507 07:36:28.951 | INFO | MemInfo: { | 20260507 07:36:28.951 | INFO | "gpu" : " 17 GB", | 20260507 07:36:28.951 | INFO | "huge" : "", | 20260507 07:36:28.951 | INFO | "huge_shared" : " 27 GB", | 20260507 07:36:28.951 | INFO | "malloc" : " 14 GB", | 20260507 07:36:28.951 | INFO | "pinned" : " 29 GB" | 20260507 07:36:28.951 | INFO | } | 20260507 07:36:28.951 | INFO | Perf report for the last 5.13964 seconds: | 20260507 07:36:28.951 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:36:28.951 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:28.951 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:28.951 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:28.951 | INFO | ANALYSIS : 60.3%, 2.9, 93.1ms, 0 | 20260507 07:36:28.951 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:28.951 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:28.951 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:36:28.951 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:28.951 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:28.951 | INFO | Uploaded 36GiB to GPU in 1.99412s (18.0531GiB/s) | 20260507 07:36:28.951 | INFO | Downloaded 34GiB to GPU in 1.94558s (17.4755GiB/s) | 20260507 07:36:34.090 | INFO | Analyzing chunk frames = [8218624,8219136) | 20260507 07:36:34.090 | INFO | MemInfo: { | 20260507 07:36:34.090 | INFO | "gpu" : " 17 GB", | 20260507 07:36:34.090 | INFO | "huge" : "", | 20260507 07:36:34.090 | INFO | "huge_shared" : " 27 GB", | 20260507 07:36:34.090 | INFO | "malloc" : " 14 GB", | 20260507 07:36:34.090 | INFO | "pinned" : " 29 GB" | 20260507 07:36:34.090 | INFO | } | 20260507 07:36:34.090 | INFO | Perf report for the last 5.13889 seconds: | 20260507 07:36:34.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:36:34.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:34.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:34.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:34.090 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 07:36:34.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:34.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:34.090 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 07:36:34.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:34.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:34.091 | INFO | Uploaded 36GiB to GPU in 2.01676s (17.8504GiB/s) | 20260507 07:36:34.091 | INFO | Downloaded 34GiB to GPU in 1.91549s (17.75GiB/s) | 20260507 07:36:39.221 | INFO | Analyzing chunk frames = [8219136,8219648) | 20260507 07:36:39.221 | INFO | MemInfo: { | 20260507 07:36:39.221 | INFO | "gpu" : " 17 GB", | 20260507 07:36:39.221 | INFO | "huge" : "", | 20260507 07:36:39.221 | INFO | "huge_shared" : " 27 GB", | 20260507 07:36:39.221 | INFO | "malloc" : " 14 GB", | 20260507 07:36:39.221 | INFO | "pinned" : " 29 GB" | 20260507 07:36:39.221 | INFO | } | 20260507 07:36:39.221 | INFO | Perf report for the last 5.13098 seconds: | 20260507 07:36:39.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:36:39.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:39.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:39.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:39.221 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:36:39.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:39.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:39.221 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 07:36:39.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:39.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:39.221 | INFO | Uploaded 36GiB to GPU in 2.06502s (17.4333GiB/s) | 20260507 07:36:39.221 | INFO | Downloaded 34GiB to GPU in 1.936s (17.562GiB/s) | 20260507 07:36:44.367 | INFO | Analyzing chunk frames = [8219648,8220160) | 20260507 07:36:44.367 | INFO | MemInfo: { | 20260507 07:36:44.367 | INFO | "gpu" : " 17 GB", | 20260507 07:36:44.367 | INFO | "huge" : "", | 20260507 07:36:44.367 | INFO | "huge_shared" : " 27 GB", | 20260507 07:36:44.367 | INFO | "malloc" : " 14 GB", | 20260507 07:36:44.367 | INFO | "pinned" : " 29 GB" | 20260507 07:36:44.367 | INFO | } | 20260507 07:36:44.367 | INFO | Perf report for the last 5.14563 seconds: | 20260507 07:36:44.367 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:36:44.367 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:44.367 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:44.367 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:44.367 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 07:36:44.367 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:44.367 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:44.367 | INFO | PRE_HQ : 26.5%, 1.0, 13.8ms, 0 | 20260507 07:36:44.367 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:44.367 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:44.367 | INFO | Uploaded 36GiB to GPU in 1.97854s (18.1952GiB/s) | 20260507 07:36:44.367 | INFO | Downloaded 34GiB to GPU in 1.99001s (17.0853GiB/s) | 20260507 07:36:49.510 | INFO | Analyzing chunk frames = [8220160,8220672) | 20260507 07:36:49.510 | INFO | MemInfo: { | 20260507 07:36:49.511 | INFO | "gpu" : " 17 GB", | 20260507 07:36:49.511 | INFO | "huge" : "", | 20260507 07:36:49.511 | INFO | "huge_shared" : " 27 GB", | 20260507 07:36:49.511 | INFO | "malloc" : " 14 GB", | 20260507 07:36:49.511 | INFO | "pinned" : " 29 GB" | 20260507 07:36:49.511 | INFO | } | 20260507 07:36:49.511 | INFO | Perf report for the last 5.14389 seconds: | 20260507 07:36:49.511 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:36:49.511 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:49.511 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:49.511 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:49.511 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:36:49.511 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:49.511 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:49.511 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:36:49.511 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:49.511 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:49.511 | INFO | Uploaded 36GiB to GPU in 1.97567s (18.2217GiB/s) | 20260507 07:36:49.511 | INFO | Downloaded 34GiB to GPU in 1.96426s (17.3094GiB/s) | 20260507 07:36:49.608 | INFO | Starting a round of Baz IO | 20260507 07:36:52.701 | INFO | Populating RTMetrics report | 20260507 07:36:52.701 | INFO | RTMetrics report populated | 20260507 07:36:52.701 | INFO | Opening tmp RTMetrics json file | 20260507 07:36:52.707 | INFO | Serializing RTMetrics to json | 20260507 07:36:52.718 | INFO | Serialization done, swapping tmp file now... | 20260507 07:36:52.719 | INFO | RTMetrics done | 20260507 07:36:52.786 | INFO | Finished a round of Baz IO: Wrote 1.635 GiB in 3.178 seconds | 20260507 07:36:52.786 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.55% / 0.00% / 67.45% | 20260507 07:36:52.786 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.101888% / 0% / 7.15256e-05% | 20260507 07:36:52.786 | INFO | Baz IO overall: | 20260507 07:36:52.786 | INFO | BytesWritten : 4116341705682 | 20260507 07:36:52.786 | INFO | NumEvents : 1970506815379 | 20260507 07:36:54.650 | INFO | Analyzing chunk frames = [8220672,8221184) | 20260507 07:36:54.650 | INFO | MemInfo: { | 20260507 07:36:54.650 | INFO | "gpu" : " 17 GB", | 20260507 07:36:54.650 | INFO | "huge" : "", | 20260507 07:36:54.650 | INFO | "huge_shared" : " 27 GB", | 20260507 07:36:54.650 | INFO | "malloc" : " 14 GB", | 20260507 07:36:54.650 | INFO | "pinned" : " 29 GB" | 20260507 07:36:54.650 | INFO | } | 20260507 07:36:54.650 | INFO | Perf report for the last 5.13909 seconds: | 20260507 07:36:54.650 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:36:54.650 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:54.650 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:54.650 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:54.650 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 07:36:54.650 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:54.650 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:36:54.650 | INFO | PRE_HQ : 58.9%, 1.8, 54.3ms, 0 | 20260507 07:36:54.650 | INFO | BAZWRITER : 61.4%, 3.1, 98.6ms, 0 | 20260507 07:36:54.650 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:36:54.650 | INFO | Uploaded 36GiB to GPU in 2.05522s (17.5164GiB/s) | 20260507 07:36:54.650 | INFO | Downloaded 38GiB to GPU in 2.18194s (17.4157GiB/s) | 20260507 07:36:59.792 | INFO | Analyzing chunk frames = [8221184,8221696) | 20260507 07:36:59.792 | INFO | MemInfo: { | 20260507 07:36:59.792 | INFO | "gpu" : " 17 GB", | 20260507 07:36:59.792 | INFO | "huge" : "", | 20260507 07:36:59.792 | INFO | "huge_shared" : " 27 GB", | 20260507 07:36:59.793 | INFO | "malloc" : " 14 GB", | 20260507 07:36:59.793 | INFO | "pinned" : " 29 GB" | 20260507 07:36:59.793 | INFO | } | 20260507 07:36:59.793 | INFO | Perf report for the last 5.1428 seconds: | 20260507 07:36:59.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:36:59.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:59.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:59.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:59.793 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:36:59.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:59.793 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:59.793 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:36:59.793 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:59.793 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:36:59.793 | INFO | Uploaded 36GiB to GPU in 2.00808s (17.9275GiB/s) | 20260507 07:36:59.793 | INFO | Downloaded 34GiB to GPU in 1.94667s (17.4657GiB/s) | 20260507 07:37:04.920 | INFO | Analyzing chunk frames = [8221696,8222208) | 20260507 07:37:04.920 | INFO | MemInfo: { | 20260507 07:37:04.920 | INFO | "gpu" : " 17 GB", | 20260507 07:37:04.920 | INFO | "huge" : "", | 20260507 07:37:04.920 | INFO | "huge_shared" : " 27 GB", | 20260507 07:37:04.920 | INFO | "malloc" : " 14 GB", | 20260507 07:37:04.921 | INFO | "pinned" : " 29 GB" | 20260507 07:37:04.921 | INFO | } | 20260507 07:37:04.921 | INFO | Perf report for the last 5.12797 seconds: | 20260507 07:37:04.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:37:04.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:04.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:04.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:04.921 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 07:37:04.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:04.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:04.921 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:37:04.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:04.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:04.921 | INFO | Uploaded 36GiB to GPU in 2.02654s (17.7642GiB/s) | 20260507 07:37:04.921 | INFO | Downloaded 34GiB to GPU in 1.92362s (17.675GiB/s) | 20260507 07:37:10.063 | INFO | Analyzing chunk frames = [8222208,8222720) | 20260507 07:37:10.063 | INFO | MemInfo: { | 20260507 07:37:10.063 | INFO | "gpu" : " 17 GB", | 20260507 07:37:10.063 | INFO | "huge" : "", | 20260507 07:37:10.063 | INFO | "huge_shared" : " 27 GB", | 20260507 07:37:10.063 | INFO | "malloc" : " 14 GB", | 20260507 07:37:10.063 | INFO | "pinned" : " 29 GB" | 20260507 07:37:10.063 | INFO | } | 20260507 07:37:10.063 | INFO | Perf report for the last 5.14267 seconds: | 20260507 07:37:10.063 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:37:10.063 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:10.063 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:10.063 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:10.063 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 07:37:10.063 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:10.063 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:10.063 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:37:10.063 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:10.063 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:10.063 | INFO | Uploaded 36GiB to GPU in 2.00583s (17.9477GiB/s) | 20260507 07:37:10.064 | INFO | Downloaded 34GiB to GPU in 1.94413s (17.4886GiB/s) | 20260507 07:37:15.203 | INFO | Analyzing chunk frames = [8222720,8223232) | 20260507 07:37:15.203 | INFO | MemInfo: { | 20260507 07:37:15.203 | INFO | "gpu" : " 17 GB", | 20260507 07:37:15.203 | INFO | "huge" : "", | 20260507 07:37:15.203 | INFO | "huge_shared" : " 27 GB", | 20260507 07:37:15.203 | INFO | "malloc" : " 14 GB", | 20260507 07:37:15.203 | INFO | "pinned" : " 29 GB" | 20260507 07:37:15.203 | INFO | } | 20260507 07:37:15.203 | INFO | Perf report for the last 5.14005 seconds: | 20260507 07:37:15.203 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:37:15.203 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:15.203 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:15.203 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:15.203 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 07:37:15.203 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:15.203 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:15.203 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:37:15.203 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:15.203 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:15.204 | INFO | Uploaded 36GiB to GPU in 2.04261s (17.6245GiB/s) | 20260507 07:37:15.204 | INFO | Downloaded 34GiB to GPU in 1.95015s (17.4345GiB/s) | 20260507 07:37:20.349 | INFO | Analyzing chunk frames = [8223232,8223744) | 20260507 07:37:20.349 | INFO | MemInfo: { | 20260507 07:37:20.349 | INFO | "gpu" : " 17 GB", | 20260507 07:37:20.349 | INFO | "huge" : "", | 20260507 07:37:20.349 | INFO | "huge_shared" : " 27 GB", | 20260507 07:37:20.349 | INFO | "malloc" : " 14 GB", | 20260507 07:37:20.349 | INFO | "pinned" : " 29 GB" | 20260507 07:37:20.349 | INFO | } | 20260507 07:37:20.349 | INFO | Perf report for the last 5.14547 seconds: | 20260507 07:37:20.349 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:37:20.349 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:20.349 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:20.349 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:20.349 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 07:37:20.349 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:20.349 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:20.349 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 07:37:20.349 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:20.349 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:20.349 | INFO | Uploaded 36GiB to GPU in 2.04825s (17.576GiB/s) | 20260507 07:37:20.349 | INFO | Downloaded 34GiB to GPU in 1.91946s (17.7133GiB/s) | 20260507 07:37:25.484 | INFO | Analyzing chunk frames = [8223744,8224256) | 20260507 07:37:25.484 | INFO | MemInfo: { | 20260507 07:37:25.484 | INFO | "gpu" : " 17 GB", | 20260507 07:37:25.484 | INFO | "huge" : "", | 20260507 07:37:25.484 | INFO | "huge_shared" : " 27 GB", | 20260507 07:37:25.484 | INFO | "malloc" : " 14 GB", | 20260507 07:37:25.484 | INFO | "pinned" : " 29 GB" | 20260507 07:37:25.484 | INFO | } | 20260507 07:37:25.484 | INFO | Perf report for the last 5.13538 seconds: | 20260507 07:37:25.484 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:37:25.484 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:25.484 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:25.484 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:25.484 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 07:37:25.484 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:25.484 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:25.484 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:37:25.484 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:25.484 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:25.484 | INFO | Uploaded 36GiB to GPU in 1.96195s (18.3491GiB/s) | 20260507 07:37:25.485 | INFO | Downloaded 34GiB to GPU in 1.96401s (17.3115GiB/s) | 20260507 07:37:30.625 | INFO | Analyzing chunk frames = [8224256,8224768) | 20260507 07:37:30.625 | INFO | MemInfo: { | 20260507 07:37:30.625 | INFO | "gpu" : " 17 GB", | 20260507 07:37:30.625 | INFO | "huge" : "", | 20260507 07:37:30.625 | INFO | "huge_shared" : " 27 GB", | 20260507 07:37:30.625 | INFO | "malloc" : " 14 GB", | 20260507 07:37:30.625 | INFO | "pinned" : " 29 GB" | 20260507 07:37:30.625 | INFO | } | 20260507 07:37:30.625 | INFO | Perf report for the last 5.14065 seconds: | 20260507 07:37:30.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:37:30.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:30.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:30.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:30.625 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:37:30.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:30.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:30.625 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:37:30.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:30.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:30.625 | INFO | Uploaded 36GiB to GPU in 1.98999s (18.0905GiB/s) | 20260507 07:37:30.625 | INFO | Downloaded 34GiB to GPU in 1.96062s (17.3415GiB/s) | 20260507 07:37:30.625 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:37:30.625 | INFO | Total mem usage: 18 GiB | 20260507 07:37:30.625 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:37:30.625 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:37:30.625 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:37:30.625 | INFO | High Water Breakdown: | 20260507 07:37:30.625 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:37:30.625 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:37:30.625 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:37:30.625 | INFO | ------------------------------------------ | 20260507 07:37:30.625 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:37:30.625 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:37:30.625 | INFO | Total mem usage: 26.5 GiB | 20260507 07:37:30.625 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:37:30.625 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:37:30.625 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:37:30.625 | INFO | High Water Breakdown: | 20260507 07:37:30.625 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:37:30.625 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:37:30.625 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:37:30.625 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:37:30.625 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:37:30.625 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:37:30.625 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:37:30.625 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:37:30.625 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:37:30.625 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:37:30.625 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:37:30.625 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:37:30.625 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:37:30.625 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:37:30.626 | INFO | ------------------------------------------ | 20260507 07:37:30.626 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:37:30.626 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:37:30.626 | INFO | Total mem usage: 25.6 GiB | 20260507 07:37:30.626 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:37:30.626 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:37:30.626 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:37:30.626 | INFO | High Water Breakdown: | 20260507 07:37:30.626 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:37:30.626 | INFO | ------------------------------------------ | 20260507 07:37:30.626 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:37:30.626 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:37:30.626 | INFO | Total mem usage: 15.7 GiB | 20260507 07:37:30.626 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:37:30.626 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:37:30.626 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:37:30.626 | INFO | High Water Breakdown: | 20260507 07:37:30.626 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:37:30.626 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:37:30.626 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:37:30.626 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:37:30.626 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:37:30.626 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:37:30.626 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:37:30.626 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:37:30.626 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:37:30.626 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:37:30.626 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:37:30.626 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:37:30.626 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:37:30.626 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:37:30.626 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:37:30.626 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:37:30.626 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:37:30.626 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:37:30.626 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:37:30.626 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:37:30.626 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:37:30.626 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:37:30.626 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:37:30.626 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:37:30.626 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:37:30.626 | INFO | ------------------------------------------ | 20260507 07:37:30.626 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:37:30.626 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:37:30.626 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:37:30.626 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:37:30.626 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.34e+01 / 1.17e+00 | 20260507 07:37:30.626 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.01e-01, 9.75e-03 / 4.75e-03 | 20260507 07:37:30.626 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.47e+00, 5.12e-01 / 2.39e-01 | 20260507 07:37:30.626 | INFO | PerfReport: 3.23%, Histogram -- 9.90e-01 / 1.48e+00, 1.67e-01 / 1.13e-01 | 20260507 07:37:30.626 | INFO | PerfReport: 16.27%, DME -- 4.00e+01 / 4.46e+01, 3.82e+00 / 1.58e+00 | 20260507 07:37:30.626 | INFO | PerfReport: 57.73%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.30e+00 / 1.35e+00 | 20260507 07:37:30.626 | INFO | PerfReport: 6.89%, PulseAccumulating -- 2.12e+00 / 2.67e+00, 3.85e-01 / 2.04e-01 | 20260507 07:37:30.626 | INFO | PerfReport: 2.93%, Metrics -- 8.99e-01 / 1.21e+00, 2.42e-01 / 1.28e-01 | 20260507 07:37:30.626 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.01e-01 / 3.61e-01, 5.10e-02 / 2.29e-02 | 20260507 07:37:30.626 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:37:30.626 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:37:30.626 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:37:30.626 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 07:37:30.626 | INFO | PerfReport: 21.85%, Upload -- 2.10e+01 / 0.00e+00, 7.08e+00 / 0.00e+00 | 20260507 07:37:30.626 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 07:37:30.626 | INFO | PerfReport: 31.95%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 07:37:30.626 | INFO | PerfReport: 1.47%, LockUpload -- 1.42e+00 / 0.00e+00, 4.87e+00 / 0.00e+00 | 20260507 07:37:30.626 | INFO | PerfReport: 1.33%, LockDownload -- 1.28e+00 / 0.00e+00, 3.06e+00 / 0.00e+00 | 20260507 07:37:30.627 | INFO | PerfReport: 21.97%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 07:37:30.627 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.95e-02 / 0.00e+00 | 20260507 07:37:30.722 | INFO | Starting a round of Baz IO | 20260507 07:37:33.818 | INFO | Populating RTMetrics report | 20260507 07:37:33.819 | INFO | RTMetrics report populated | 20260507 07:37:33.819 | INFO | Opening tmp RTMetrics json file | 20260507 07:37:33.832 | INFO | Serializing RTMetrics to json | 20260507 07:37:33.833 | INFO | Serialization done, swapping tmp file now... | 20260507 07:37:33.834 | INFO | RTMetrics done | 20260507 07:37:33.901 | INFO | Finished a round of Baz IO: Wrote 1.636 GiB in 3.178 seconds | 20260507 07:37:33.901 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.55% / 0.00% / 67.45% | 20260507 07:37:33.901 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.101991% / 3.97364e-06% / 6.75519e-05% | 20260507 07:37:33.901 | INFO | Baz IO overall: | 20260507 07:37:33.901 | INFO | BytesWritten : 4118098818485 | 20260507 07:37:33.901 | INFO | NumEvents : 1971327235257 | 20260507 07:37:35.765 | INFO | Analyzing chunk frames = [8224768,8225280) | 20260507 07:37:35.765 | INFO | MemInfo: { | 20260507 07:37:35.765 | INFO | "gpu" : " 17 GB", | 20260507 07:37:35.765 | INFO | "huge" : "", | 20260507 07:37:35.765 | INFO | "huge_shared" : " 27 GB", | 20260507 07:37:35.765 | INFO | "malloc" : " 14 GB", | 20260507 07:37:35.765 | INFO | "pinned" : " 29 GB" | 20260507 07:37:35.765 | INFO | } | 20260507 07:37:35.765 | INFO | Perf report for the last 5.14049 seconds: | 20260507 07:37:35.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:37:35.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:35.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:35.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:35.765 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 07:37:35.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:35.765 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 07:37:35.765 | INFO | PRE_HQ : 58.8%, 1.8, 54.3ms, 0 | 20260507 07:37:35.765 | INFO | BAZWRITER : 61.4%, 3.1, 98.2ms, 0 | 20260507 07:37:35.765 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:37:35.766 | INFO | Uploaded 36GiB to GPU in 2.0707s (17.3855GiB/s) | 20260507 07:37:35.766 | INFO | Downloaded 38GiB to GPU in 2.18298s (17.4074GiB/s) | 20260507 07:37:40.901 | INFO | Analyzing chunk frames = [8225280,8225792) | 20260507 07:37:40.901 | INFO | MemInfo: { | 20260507 07:37:40.901 | INFO | "gpu" : " 17 GB", | 20260507 07:37:40.901 | INFO | "huge" : "", | 20260507 07:37:40.901 | INFO | "huge_shared" : " 27 GB", | 20260507 07:37:40.901 | INFO | "malloc" : " 14 GB", | 20260507 07:37:40.901 | INFO | "pinned" : " 29 GB" | 20260507 07:37:40.901 | INFO | } | 20260507 07:37:40.901 | INFO | Perf report for the last 5.1357 seconds: | 20260507 07:37:40.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:37:40.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:40.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:40.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:40.901 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:37:40.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:40.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:40.901 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 07:37:40.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:40.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:40.901 | INFO | Uploaded 36GiB to GPU in 1.98994s (18.091GiB/s) | 20260507 07:37:40.901 | INFO | Downloaded 34GiB to GPU in 1.93103s (17.6072GiB/s) | 20260507 07:37:46.039 | INFO | Analyzing chunk frames = [8225792,8226304) | 20260507 07:37:46.039 | INFO | MemInfo: { | 20260507 07:37:46.039 | INFO | "gpu" : " 17 GB", | 20260507 07:37:46.039 | INFO | "huge" : "", | 20260507 07:37:46.039 | INFO | "huge_shared" : " 27 GB", | 20260507 07:37:46.039 | INFO | "malloc" : " 14 GB", | 20260507 07:37:46.039 | INFO | "pinned" : " 29 GB" | 20260507 07:37:46.039 | INFO | } | 20260507 07:37:46.039 | INFO | Perf report for the last 5.1384 seconds: | 20260507 07:37:46.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:37:46.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:46.040 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:46.040 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:46.040 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 07:37:46.040 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:46.040 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:46.040 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:37:46.040 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:46.040 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:46.040 | INFO | Uploaded 36GiB to GPU in 2.0705s (17.3871GiB/s) | 20260507 07:37:46.040 | INFO | Downloaded 34GiB to GPU in 1.95654s (17.3777GiB/s) | 20260507 07:37:51.171 | INFO | Analyzing chunk frames = [8226304,8226816) | 20260507 07:37:51.171 | INFO | MemInfo: { | 20260507 07:37:51.171 | INFO | "gpu" : " 17 GB", | 20260507 07:37:51.172 | INFO | "huge" : "", | 20260507 07:37:51.172 | INFO | "huge_shared" : " 27 GB", | 20260507 07:37:51.172 | INFO | "malloc" : " 14 GB", | 20260507 07:37:51.172 | INFO | "pinned" : " 29 GB" | 20260507 07:37:51.172 | INFO | } | 20260507 07:37:51.172 | INFO | Perf report for the last 5.13211 seconds: | 20260507 07:37:51.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:37:51.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:51.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:51.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:51.172 | INFO | ANALYSIS : 60.4%, 2.9, 93.1ms, 0 | 20260507 07:37:51.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:51.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:51.172 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 07:37:51.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:51.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:51.172 | INFO | Uploaded 36GiB to GPU in 2.03532s (17.6877GiB/s) | 20260507 07:37:51.172 | INFO | Downloaded 34GiB to GPU in 1.94865s (17.448GiB/s) | 20260507 07:37:56.319 | INFO | Analyzing chunk frames = [8226816,8227328) | 20260507 07:37:56.319 | INFO | MemInfo: { | 20260507 07:37:56.319 | INFO | "gpu" : " 17 GB", | 20260507 07:37:56.319 | INFO | "huge" : "", | 20260507 07:37:56.319 | INFO | "huge_shared" : " 27 GB", | 20260507 07:37:56.319 | INFO | "malloc" : " 14 GB", | 20260507 07:37:56.319 | INFO | "pinned" : " 29 GB" | 20260507 07:37:56.319 | INFO | } | 20260507 07:37:56.319 | INFO | Perf report for the last 5.1473 seconds: | 20260507 07:37:56.319 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:37:56.319 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:56.319 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:56.319 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:56.319 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 07:37:56.319 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:56.319 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:56.319 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:37:56.319 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:56.319 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:37:56.319 | INFO | Uploaded 36GiB to GPU in 2.04176s (17.6318GiB/s) | 20260507 07:37:56.319 | INFO | Downloaded 34GiB to GPU in 1.94054s (17.5209GiB/s) | 20260507 07:38:01.461 | INFO | Analyzing chunk frames = [8227328,8227840) | 20260507 07:38:01.461 | INFO | MemInfo: { | 20260507 07:38:01.461 | INFO | "gpu" : " 17 GB", | 20260507 07:38:01.461 | INFO | "huge" : "", | 20260507 07:38:01.461 | INFO | "huge_shared" : " 27 GB", | 20260507 07:38:01.461 | INFO | "malloc" : " 14 GB", | 20260507 07:38:01.462 | INFO | "pinned" : " 29 GB" | 20260507 07:38:01.462 | INFO | } | 20260507 07:38:01.462 | INFO | Perf report for the last 5.14265 seconds: | 20260507 07:38:01.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:38:01.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:01.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:01.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:01.462 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 07:38:01.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:01.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:01.462 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 07:38:01.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:01.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:01.462 | INFO | Uploaded 36GiB to GPU in 2.04749s (17.5825GiB/s) | 20260507 07:38:01.462 | INFO | Downloaded 34GiB to GPU in 1.90768s (17.8227GiB/s) | 20260507 07:38:06.603 | INFO | Analyzing chunk frames = [8227840,8228352) | 20260507 07:38:06.603 | INFO | MemInfo: { | 20260507 07:38:06.603 | INFO | "gpu" : " 17 GB", | 20260507 07:38:06.604 | INFO | "huge" : "", | 20260507 07:38:06.604 | INFO | "huge_shared" : " 27 GB", | 20260507 07:38:06.604 | INFO | "malloc" : " 14 GB", | 20260507 07:38:06.604 | INFO | "pinned" : " 29 GB" | 20260507 07:38:06.604 | INFO | } | 20260507 07:38:06.604 | INFO | Perf report for the last 5.14199 seconds: | 20260507 07:38:06.604 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:38:06.604 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:06.604 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:06.604 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:06.604 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:38:06.604 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:06.604 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:06.604 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:38:06.604 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:06.604 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:06.604 | INFO | Uploaded 36GiB to GPU in 1.90115s (18.9359GiB/s) | 20260507 07:38:06.604 | INFO | Downloaded 34GiB to GPU in 1.97536s (17.2121GiB/s) | 20260507 07:38:11.737 | INFO | Analyzing chunk frames = [8228352,8228864) | 20260507 07:38:11.737 | INFO | MemInfo: { | 20260507 07:38:11.737 | INFO | "gpu" : " 17 GB", | 20260507 07:38:11.737 | INFO | "huge" : "", | 20260507 07:38:11.737 | INFO | "huge_shared" : " 27 GB", | 20260507 07:38:11.737 | INFO | "malloc" : " 14 GB", | 20260507 07:38:11.737 | INFO | "pinned" : " 29 GB" | 20260507 07:38:11.737 | INFO | } | 20260507 07:38:11.737 | INFO | Perf report for the last 5.13354 seconds: | 20260507 07:38:11.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:38:11.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:11.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:11.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:11.737 | INFO | ANALYSIS : 60.5%, 3.0, 94.0ms, 0 | 20260507 07:38:11.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:11.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:11.737 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:38:11.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:11.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:11.737 | INFO | Uploaded 36GiB to GPU in 1.98099s (18.1727GiB/s) | 20260507 07:38:11.737 | INFO | Downloaded 34GiB to GPU in 1.9568s (17.3753GiB/s) | 20260507 07:38:11.834 | INFO | Starting a round of Baz IO | 20260507 07:38:14.928 | INFO | Populating RTMetrics report | 20260507 07:38:14.928 | INFO | RTMetrics report populated | 20260507 07:38:14.928 | INFO | Opening tmp RTMetrics json file | 20260507 07:38:14.936 | INFO | Serializing RTMetrics to json | 20260507 07:38:14.944 | INFO | Serialization done, swapping tmp file now... | 20260507 07:38:14.945 | INFO | RTMetrics done | 20260507 07:38:15.011 | INFO | Finished a round of Baz IO: Wrote 1.635 GiB in 3.177 seconds | 20260507 07:38:15.012 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.55% / 0.00% / 67.45% | 20260507 07:38:15.012 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.101793% / 3.97364e-06% / 8.34465e-05% | 20260507 07:38:15.012 | INFO | Baz IO overall: | 20260507 07:38:15.012 | INFO | BytesWritten : 4119854713620 | 20260507 07:38:15.012 | INFO | NumEvents : 1972146862097 | 20260507 07:38:16.871 | INFO | Analyzing chunk frames = [8228864,8229376) | 20260507 07:38:16.871 | INFO | MemInfo: { | 20260507 07:38:16.871 | INFO | "gpu" : " 17 GB", | 20260507 07:38:16.871 | INFO | "huge" : "", | 20260507 07:38:16.871 | INFO | "huge_shared" : " 27 GB", | 20260507 07:38:16.871 | INFO | "malloc" : " 14 GB", | 20260507 07:38:16.871 | INFO | "pinned" : " 29 GB" | 20260507 07:38:16.871 | INFO | } | 20260507 07:38:16.871 | INFO | Perf report for the last 5.13429 seconds: | 20260507 07:38:16.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:38:16.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:16.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:16.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:16.871 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 07:38:16.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:16.871 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:38:16.871 | INFO | PRE_HQ : 59.0%, 1.8, 54.4ms, 0 | 20260507 07:38:16.872 | INFO | BAZWRITER : 61.5%, 3.1, 97.8ms, 0 | 20260507 07:38:16.872 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:38:16.872 | INFO | Uploaded 36GiB to GPU in 2.05427s (17.5245GiB/s) | 20260507 07:38:16.872 | INFO | Downloaded 38GiB to GPU in 2.17556s (17.4668GiB/s) | 20260507 07:38:22.012 | INFO | Analyzing chunk frames = [8229376,8229888) | 20260507 07:38:22.012 | INFO | MemInfo: { | 20260507 07:38:22.012 | INFO | "gpu" : " 17 GB", | 20260507 07:38:22.012 | INFO | "huge" : "", | 20260507 07:38:22.012 | INFO | "huge_shared" : " 27 GB", | 20260507 07:38:22.012 | INFO | "malloc" : " 14 GB", | 20260507 07:38:22.012 | INFO | "pinned" : " 29 GB" | 20260507 07:38:22.012 | INFO | } | 20260507 07:38:22.012 | INFO | Perf report for the last 5.14042 seconds: | 20260507 07:38:22.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:38:22.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:22.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:22.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:22.012 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 07:38:22.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:22.012 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:22.012 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:38:22.012 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:22.012 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:22.012 | INFO | Uploaded 36GiB to GPU in 2.00575s (17.9484GiB/s) | 20260507 07:38:22.012 | INFO | Downloaded 34GiB to GPU in 1.9346s (17.5747GiB/s) | 20260507 07:38:27.152 | INFO | Analyzing chunk frames = [8229888,8230400) | 20260507 07:38:27.152 | INFO | MemInfo: { | 20260507 07:38:27.152 | INFO | "gpu" : " 17 GB", | 20260507 07:38:27.152 | INFO | "huge" : "", | 20260507 07:38:27.152 | INFO | "huge_shared" : " 27 GB", | 20260507 07:38:27.152 | INFO | "malloc" : " 14 GB", | 20260507 07:38:27.152 | INFO | "pinned" : " 29 GB" | 20260507 07:38:27.152 | INFO | } | 20260507 07:38:27.152 | INFO | Perf report for the last 5.14035 seconds: | 20260507 07:38:27.152 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:38:27.152 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:27.152 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:27.152 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:27.152 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:38:27.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:27.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:27.152 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:38:27.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:27.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:27.153 | INFO | Uploaded 36GiB to GPU in 2.0427s (17.6237GiB/s) | 20260507 07:38:27.153 | INFO | Downloaded 34GiB to GPU in 1.95095s (17.4274GiB/s) | 20260507 07:38:32.296 | INFO | Analyzing chunk frames = [8230400,8230912) | 20260507 07:38:32.297 | INFO | MemInfo: { | 20260507 07:38:32.297 | INFO | "gpu" : " 17 GB", | 20260507 07:38:32.297 | INFO | "huge" : "", | 20260507 07:38:32.297 | INFO | "huge_shared" : " 27 GB", | 20260507 07:38:32.297 | INFO | "malloc" : " 14 GB", | 20260507 07:38:32.297 | INFO | "pinned" : " 29 GB" | 20260507 07:38:32.297 | INFO | } | 20260507 07:38:32.297 | INFO | Perf report for the last 5.14445 seconds: | 20260507 07:38:32.297 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:38:32.297 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:32.297 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:32.297 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:32.297 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 07:38:32.297 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:32.297 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:32.297 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:38:32.297 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:32.297 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:32.297 | INFO | Uploaded 36GiB to GPU in 2.01797s (17.8397GiB/s) | 20260507 07:38:32.297 | INFO | Downloaded 34GiB to GPU in 1.9395s (17.5302GiB/s) | 20260507 07:38:37.428 | INFO | Analyzing chunk frames = [8230912,8231424) | 20260507 07:38:37.428 | INFO | MemInfo: { | 20260507 07:38:37.428 | INFO | "gpu" : " 17 GB", | 20260507 07:38:37.428 | INFO | "huge" : "", | 20260507 07:38:37.428 | INFO | "huge_shared" : " 27 GB", | 20260507 07:38:37.429 | INFO | "malloc" : " 14 GB", | 20260507 07:38:37.429 | INFO | "pinned" : " 29 GB" | 20260507 07:38:37.429 | INFO | } | 20260507 07:38:37.429 | INFO | Perf report for the last 5.13185 seconds: | 20260507 07:38:37.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:38:37.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:37.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:37.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:37.429 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 07:38:37.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:37.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:37.429 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:38:37.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:37.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:37.429 | INFO | Uploaded 36GiB to GPU in 2.05313s (17.5342GiB/s) | 20260507 07:38:37.429 | INFO | Downloaded 34GiB to GPU in 1.90509s (17.8469GiB/s) | 20260507 07:38:42.572 | INFO | Analyzing chunk frames = [8231424,8231936) | 20260507 07:38:42.572 | INFO | MemInfo: { | 20260507 07:38:42.572 | INFO | "gpu" : " 17 GB", | 20260507 07:38:42.572 | INFO | "huge" : "", | 20260507 07:38:42.572 | INFO | "huge_shared" : " 27 GB", | 20260507 07:38:42.572 | INFO | "malloc" : " 14 GB", | 20260507 07:38:42.572 | INFO | "pinned" : " 29 GB" | 20260507 07:38:42.572 | INFO | } | 20260507 07:38:42.573 | INFO | Perf report for the last 5.14397 seconds: | 20260507 07:38:42.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:38:42.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:42.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:42.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:42.573 | INFO | ANALYSIS : 60.3%, 3.0, 94.3ms, 0 | 20260507 07:38:42.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:42.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:42.573 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 07:38:42.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:42.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:42.573 | INFO | Uploaded 36GiB to GPU in 2.0577s (17.4953GiB/s) | 20260507 07:38:42.573 | INFO | Downloaded 34GiB to GPU in 1.91861s (17.7212GiB/s) | 20260507 07:38:47.712 | INFO | Analyzing chunk frames = [8231936,8232448) | 20260507 07:38:47.712 | INFO | MemInfo: { | 20260507 07:38:47.712 | INFO | "gpu" : " 17 GB", | 20260507 07:38:47.712 | INFO | "huge" : "", | 20260507 07:38:47.712 | INFO | "huge_shared" : " 27 GB", | 20260507 07:38:47.712 | INFO | "malloc" : " 14 GB", | 20260507 07:38:47.712 | INFO | "pinned" : " 29 GB" | 20260507 07:38:47.712 | INFO | } | 20260507 07:38:47.712 | INFO | Perf report for the last 5.13956 seconds: | 20260507 07:38:47.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:38:47.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:47.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:47.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:47.712 | INFO | ANALYSIS : 61.0%, 3.0, 94.9ms, 0 | 20260507 07:38:47.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:47.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:47.712 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:38:47.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:47.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:47.712 | INFO | Uploaded 36GiB to GPU in 2.03838s (17.6611GiB/s) | 20260507 07:38:47.712 | INFO | Downloaded 34GiB to GPU in 1.99677s (17.0275GiB/s) | 20260507 07:38:52.851 | INFO | Analyzing chunk frames = [8232448,8232960) | 20260507 07:38:52.851 | INFO | MemInfo: { | 20260507 07:38:52.851 | INFO | "gpu" : " 17 GB", | 20260507 07:38:52.851 | INFO | "huge" : "", | 20260507 07:38:52.851 | INFO | "huge_shared" : " 27 GB", | 20260507 07:38:52.851 | INFO | "malloc" : " 14 GB", | 20260507 07:38:52.851 | INFO | "pinned" : " 29 GB" | 20260507 07:38:52.851 | INFO | } | 20260507 07:38:52.851 | INFO | Perf report for the last 5.13912 seconds: | 20260507 07:38:52.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:38:52.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:52.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:52.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:52.851 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:38:52.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:52.851 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:52.851 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:38:52.851 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:52.851 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:52.851 | INFO | Uploaded 36GiB to GPU in 1.98638s (18.1235GiB/s) | 20260507 07:38:52.852 | INFO | Downloaded 34GiB to GPU in 1.96111s (17.3372GiB/s) | 20260507 07:38:52.852 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:38:52.852 | INFO | Total mem usage: 18 GiB | 20260507 07:38:52.852 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:38:52.852 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:38:52.852 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:38:52.852 | INFO | High Water Breakdown: | 20260507 07:38:52.852 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:38:52.852 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:38:52.852 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:38:52.852 | INFO | ------------------------------------------ | 20260507 07:38:52.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:38:52.852 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:38:52.852 | INFO | Total mem usage: 26.5 GiB | 20260507 07:38:52.852 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:38:52.852 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:38:52.852 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 07:38:52.852 | INFO | High Water Breakdown: | 20260507 07:38:52.852 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:38:52.852 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:38:52.852 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:38:52.852 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:38:52.852 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:38:52.852 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:38:52.852 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:38:52.852 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:38:52.852 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:38:52.852 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:38:52.852 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:38:52.852 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:38:52.852 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:38:52.852 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:38:52.852 | INFO | ------------------------------------------ | 20260507 07:38:52.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:38:52.852 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:38:52.852 | INFO | Total mem usage: 25.6 GiB | 20260507 07:38:52.852 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:38:52.852 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:38:52.852 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:38:52.852 | INFO | High Water Breakdown: | 20260507 07:38:52.852 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:38:52.852 | INFO | ------------------------------------------ | 20260507 07:38:52.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:38:52.852 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:38:52.852 | INFO | Total mem usage: 15.7 GiB | 20260507 07:38:52.852 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:38:52.852 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:38:52.852 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:38:52.852 | INFO | High Water Breakdown: | 20260507 07:38:52.852 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:38:52.852 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:38:52.852 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:38:52.852 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:38:52.852 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:38:52.852 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:38:52.852 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:38:52.852 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:38:52.852 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:38:52.852 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:38:52.852 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:38:52.852 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:38:52.852 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:38:52.852 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:38:52.852 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:38:52.852 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:38:52.852 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:38:52.852 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:38:52.852 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:38:52.852 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:38:52.852 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:38:52.852 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:38:52.852 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:38:52.852 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:38:52.852 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:38:52.852 | INFO | ------------------------------------------ | 20260507 07:38:52.852 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:38:52.852 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:38:52.852 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:38:52.852 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:38:52.852 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.35e+01 / 1.04e+00 | 20260507 07:38:52.852 | INFO | PerfReport: 0.30%, AggregateStats -- 9.05e-02 / 1.03e-01, 9.43e-03 / 4.30e-03 | 20260507 07:38:52.852 | INFO | PerfReport: 11.70%, Baseline -- 3.59e+00 / 4.44e+00, 5.10e-01 / 2.57e-01 | 20260507 07:38:52.852 | INFO | PerfReport: 3.23%, Histogram -- 9.90e-01 / 1.48e+00, 1.66e-01 / 1.13e-01 | 20260507 07:38:52.852 | INFO | PerfReport: 16.31%, DME -- 4.00e+01 / 4.50e+01, 4.03e+00 / 1.42e+00 | 20260507 07:38:52.852 | INFO | PerfReport: 57.65%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.25e+00 / 1.20e+00 | 20260507 07:38:52.852 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.71e+00, 3.86e-01 / 1.65e-01 | 20260507 07:38:52.852 | INFO | PerfReport: 2.93%, Metrics -- 8.98e-01 / 1.22e+00, 2.41e-01 / 1.37e-01 | 20260507 07:38:52.852 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.01e-01 / 3.56e-01, 5.06e-02 / 1.46e-02 | 20260507 07:38:52.853 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:38:52.853 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:38:52.853 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:38:52.853 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 07:38:52.853 | INFO | PerfReport: 21.94%, Upload -- 2.11e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260507 07:38:52.853 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 07:38:52.853 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 07:38:52.853 | INFO | PerfReport: 1.56%, LockUpload -- 1.50e+00 / 0.00e+00, 4.89e+00 / 0.00e+00 | 20260507 07:38:52.853 | INFO | PerfReport: 1.25%, LockDownload -- 1.20e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260507 07:38:52.853 | INFO | PerfReport: 21.93%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 07:38:52.853 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260507 07:38:52.948 | INFO | Starting a round of Baz IO | 20260507 07:38:56.037 | INFO | Populating RTMetrics report | 20260507 07:38:56.037 | INFO | RTMetrics report populated | 20260507 07:38:56.037 | INFO | Opening tmp RTMetrics json file | 20260507 07:38:56.052 | INFO | Serializing RTMetrics to json | 20260507 07:38:56.054 | INFO | Serialization done, swapping tmp file now... | 20260507 07:38:56.054 | INFO | RTMetrics done | 20260507 07:38:56.121 | INFO | Finished a round of Baz IO: Wrote 1.637 GiB in 3.172 seconds | 20260507 07:38:56.121 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.55% / 0.00% / 67.45% | 20260507 07:38:56.121 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.102079% / 0% / 3.97364e-05% | 20260507 07:38:56.121 | INFO | Baz IO overall: | 20260507 07:38:56.121 | INFO | BytesWritten : 4121611990602 | 20260507 07:38:56.121 | INFO | NumEvents : 1972967619408 | 20260507 07:38:57.991 | INFO | Analyzing chunk frames = [8232960,8233472) | 20260507 07:38:57.991 | INFO | MemInfo: { | 20260507 07:38:57.991 | INFO | "gpu" : " 17 GB", | 20260507 07:38:57.991 | INFO | "huge" : "", | 20260507 07:38:57.991 | INFO | "huge_shared" : " 27 GB", | 20260507 07:38:57.991 | INFO | "malloc" : " 14 GB", | 20260507 07:38:57.991 | INFO | "pinned" : " 29 GB" | 20260507 07:38:57.991 | INFO | } | 20260507 07:38:57.991 | INFO | Perf report for the last 5.13957 seconds: | 20260507 07:38:57.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:38:57.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:57.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:57.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:57.991 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 07:38:57.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:38:57.991 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 07:38:57.991 | INFO | PRE_HQ : 58.8%, 1.8, 54.3ms, 0 | 20260507 07:38:57.991 | INFO | BAZWRITER : 61.3%, 3.1, 97.3ms, 0 | 20260507 07:38:57.991 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260507 07:38:57.991 | INFO | Uploaded 36GiB to GPU in 2.05962s (17.4789GiB/s) | 20260507 07:38:57.991 | INFO | Downloaded 38GiB to GPU in 2.17599s (17.4633GiB/s) | 20260507 07:39:03.126 | INFO | Analyzing chunk frames = [8233472,8233984) | 20260507 07:39:03.126 | INFO | MemInfo: { | 20260507 07:39:03.126 | INFO | "gpu" : " 17 GB", | 20260507 07:39:03.126 | INFO | "huge" : "", | 20260507 07:39:03.126 | INFO | "huge_shared" : " 27 GB", | 20260507 07:39:03.126 | INFO | "malloc" : " 14 GB", | 20260507 07:39:03.126 | INFO | "pinned" : " 29 GB" | 20260507 07:39:03.126 | INFO | } | 20260507 07:39:03.126 | INFO | Perf report for the last 5.1352 seconds: | 20260507 07:39:03.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:39:03.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:03.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:03.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:03.126 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:39:03.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:03.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:03.126 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:39:03.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:03.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:03.126 | INFO | Uploaded 36GiB to GPU in 1.98076s (18.1748GiB/s) | 20260507 07:39:03.126 | INFO | Downloaded 34GiB to GPU in 1.93027s (17.6141GiB/s) | 20260507 07:39:08.273 | INFO | Analyzing chunk frames = [8233984,8234496) | 20260507 07:39:08.273 | INFO | MemInfo: { | 20260507 07:39:08.273 | INFO | "gpu" : " 17 GB", | 20260507 07:39:08.273 | INFO | "huge" : "", | 20260507 07:39:08.273 | INFO | "huge_shared" : " 27 GB", | 20260507 07:39:08.273 | INFO | "malloc" : " 14 GB", | 20260507 07:39:08.273 | INFO | "pinned" : " 29 GB" | 20260507 07:39:08.273 | INFO | } | 20260507 07:39:08.273 | INFO | Perf report for the last 5.14733 seconds: | 20260507 07:39:08.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:39:08.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:08.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:08.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:08.273 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 07:39:08.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:08.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:08.273 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:39:08.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:08.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:08.274 | INFO | Uploaded 36GiB to GPU in 1.98787s (18.1098GiB/s) | 20260507 07:39:08.274 | INFO | Downloaded 34GiB to GPU in 1.94453s (17.4849GiB/s) | 20260507 07:39:13.412 | INFO | Analyzing chunk frames = [8234496,8235008) | 20260507 07:39:13.412 | INFO | MemInfo: { | 20260507 07:39:13.412 | INFO | "gpu" : " 17 GB", | 20260507 07:39:13.412 | INFO | "huge" : "", | 20260507 07:39:13.412 | INFO | "huge_shared" : " 27 GB", | 20260507 07:39:13.412 | INFO | "malloc" : " 14 GB", | 20260507 07:39:13.412 | INFO | "pinned" : " 29 GB" | 20260507 07:39:13.412 | INFO | } | 20260507 07:39:13.412 | INFO | Perf report for the last 5.13892 seconds: | 20260507 07:39:13.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:39:13.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:13.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:13.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:13.412 | INFO | ANALYSIS : 60.3%, 2.9, 93.2ms, 0 | 20260507 07:39:13.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:13.412 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:13.412 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:39:13.412 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:13.412 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:13.413 | INFO | Uploaded 36GiB to GPU in 2.04354s (17.6165GiB/s) | 20260507 07:39:13.413 | INFO | Downloaded 34GiB to GPU in 1.95308s (17.4084GiB/s) | 20260507 07:39:18.548 | INFO | Analyzing chunk frames = [8235008,8235520) | 20260507 07:39:18.548 | INFO | MemInfo: { | 20260507 07:39:18.548 | INFO | "gpu" : " 17 GB", | 20260507 07:39:18.548 | INFO | "huge" : "", | 20260507 07:39:18.548 | INFO | "huge_shared" : " 27 GB", | 20260507 07:39:18.548 | INFO | "malloc" : " 14 GB", | 20260507 07:39:18.548 | INFO | "pinned" : " 29 GB" | 20260507 07:39:18.548 | INFO | } | 20260507 07:39:18.548 | INFO | Perf report for the last 5.13593 seconds: | 20260507 07:39:18.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:39:18.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:18.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:18.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:18.548 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 07:39:18.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:18.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:18.548 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:39:18.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:18.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:18.548 | INFO | Uploaded 36GiB to GPU in 2.05783s (17.4942GiB/s) | 20260507 07:39:18.549 | INFO | Downloaded 34GiB to GPU in 1.91055s (17.7959GiB/s) | 20260507 07:39:23.687 | INFO | Analyzing chunk frames = [8235520,8236032) | 20260507 07:39:23.687 | INFO | MemInfo: { | 20260507 07:39:23.687 | INFO | "gpu" : " 17 GB", | 20260507 07:39:23.687 | INFO | "huge" : "", | 20260507 07:39:23.687 | INFO | "huge_shared" : " 27 GB", | 20260507 07:39:23.687 | INFO | "malloc" : " 14 GB", | 20260507 07:39:23.687 | INFO | "pinned" : " 29 GB" | 20260507 07:39:23.687 | INFO | } | 20260507 07:39:23.687 | INFO | Perf report for the last 5.13884 seconds: | 20260507 07:39:23.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:39:23.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:23.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:23.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:23.687 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 07:39:23.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:23.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:23.687 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 07:39:23.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:23.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:23.687 | INFO | Uploaded 36GiB to GPU in 2.06839s (17.4049GiB/s) | 20260507 07:39:23.687 | INFO | Downloaded 34GiB to GPU in 1.95386s (17.4015GiB/s) | 20260507 07:39:28.826 | INFO | Analyzing chunk frames = [8236032,8236544) | 20260507 07:39:28.827 | INFO | MemInfo: { | 20260507 07:39:28.827 | INFO | "gpu" : " 17 GB", | 20260507 07:39:28.827 | INFO | "huge" : "", | 20260507 07:39:28.827 | INFO | "huge_shared" : " 27 GB", | 20260507 07:39:28.827 | INFO | "malloc" : " 14 GB", | 20260507 07:39:28.827 | INFO | "pinned" : " 29 GB" | 20260507 07:39:28.827 | INFO | } | 20260507 07:39:28.827 | INFO | Perf report for the last 5.13954 seconds: | 20260507 07:39:28.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:39:28.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:28.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:28.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:28.827 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:39:28.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:28.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:28.827 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:39:28.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:28.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:28.827 | INFO | Uploaded 36GiB to GPU in 1.98507s (18.1354GiB/s) | 20260507 07:39:28.827 | INFO | Downloaded 34GiB to GPU in 1.98417s (17.1356GiB/s) | 20260507 07:39:33.966 | INFO | Analyzing chunk frames = [8236544,8237056) | 20260507 07:39:33.966 | INFO | MemInfo: { | 20260507 07:39:33.966 | INFO | "gpu" : " 17 GB", | 20260507 07:39:33.966 | INFO | "huge" : "", | 20260507 07:39:33.966 | INFO | "huge_shared" : " 27 GB", | 20260507 07:39:33.966 | INFO | "malloc" : " 14 GB", | 20260507 07:39:33.966 | INFO | "pinned" : " 29 GB" | 20260507 07:39:33.966 | INFO | } | 20260507 07:39:33.966 | INFO | Perf report for the last 5.13906 seconds: | 20260507 07:39:33.966 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:39:33.966 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:33.966 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:33.966 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:33.966 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:39:33.966 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:33.966 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:33.966 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:39:33.966 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:33.966 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:33.966 | INFO | Uploaded 36GiB to GPU in 1.98373s (18.1477GiB/s) | 20260507 07:39:33.966 | INFO | Downloaded 34GiB to GPU in 1.95391s (17.401GiB/s) | 20260507 07:39:34.063 | INFO | Starting a round of Baz IO | 20260507 07:39:37.155 | INFO | Populating RTMetrics report | 20260507 07:39:37.155 | INFO | RTMetrics report populated | 20260507 07:39:37.155 | INFO | Opening tmp RTMetrics json file | 20260507 07:39:37.166 | INFO | Serializing RTMetrics to json | 20260507 07:39:37.170 | INFO | Serialization done, swapping tmp file now... | 20260507 07:39:37.170 | INFO | RTMetrics done | 20260507 07:39:37.237 | INFO | Finished a round of Baz IO: Wrote 1.637 GiB in 3.174 seconds | 20260507 07:39:37.237 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.54% / 0.00% / 67.45% | 20260507 07:39:37.237 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.101594% / 3.97364e-06% / 6.35783e-05% | 20260507 07:39:37.237 | INFO | Baz IO overall: | 20260507 07:39:37.237 | INFO | BytesWritten : 4123369618910 | 20260507 07:39:37.237 | INFO | NumEvents : 1973788778258 | 20260507 07:39:39.107 | INFO | Analyzing chunk frames = [8237056,8237568) | 20260507 07:39:39.107 | INFO | MemInfo: { | 20260507 07:39:39.107 | INFO | "gpu" : " 17 GB", | 20260507 07:39:39.107 | INFO | "huge" : "", | 20260507 07:39:39.107 | INFO | "huge_shared" : " 27 GB", | 20260507 07:39:39.107 | INFO | "malloc" : " 14 GB", | 20260507 07:39:39.107 | INFO | "pinned" : " 29 GB" | 20260507 07:39:39.107 | INFO | } | 20260507 07:39:39.107 | INFO | Perf report for the last 5.14142 seconds: | 20260507 07:39:39.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:39:39.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:39.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:39.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:39.107 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 07:39:39.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:39.107 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:39:39.107 | INFO | PRE_HQ : 58.7%, 1.8, 54.5ms, 0 | 20260507 07:39:39.107 | INFO | BAZWRITER : 61.1%, 3.1, 97.3ms, 0 | 20260507 07:39:39.107 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:39:39.107 | INFO | Uploaded 36GiB to GPU in 2.02913s (17.7416GiB/s) | 20260507 07:39:39.107 | INFO | Downloaded 38GiB to GPU in 2.16688s (17.5367GiB/s) | 20260507 07:39:44.244 | INFO | Analyzing chunk frames = [8237568,8238080) | 20260507 07:39:44.244 | INFO | MemInfo: { | 20260507 07:39:44.244 | INFO | "gpu" : " 17 GB", | 20260507 07:39:44.244 | INFO | "huge" : "", | 20260507 07:39:44.244 | INFO | "huge_shared" : " 27 GB", | 20260507 07:39:44.244 | INFO | "malloc" : " 14 GB", | 20260507 07:39:44.244 | INFO | "pinned" : " 29 GB" | 20260507 07:39:44.244 | INFO | } | 20260507 07:39:44.244 | INFO | Perf report for the last 5.13687 seconds: | 20260507 07:39:44.244 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:39:44.244 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:44.244 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:44.244 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:44.244 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:39:44.244 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:44.244 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:44.244 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:39:44.244 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:44.244 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:44.244 | INFO | Uploaded 36GiB to GPU in 1.98974s (18.0928GiB/s) | 20260507 07:39:44.244 | INFO | Downloaded 34GiB to GPU in 1.92623s (17.651GiB/s) | 20260507 07:39:49.379 | INFO | Analyzing chunk frames = [8238080,8238592) | 20260507 07:39:49.379 | INFO | MemInfo: { | 20260507 07:39:49.379 | INFO | "gpu" : " 17 GB", | 20260507 07:39:49.379 | INFO | "huge" : "", | 20260507 07:39:49.379 | INFO | "huge_shared" : " 27 GB", | 20260507 07:39:49.379 | INFO | "malloc" : " 14 GB", | 20260507 07:39:49.379 | INFO | "pinned" : " 29 GB" | 20260507 07:39:49.379 | INFO | } | 20260507 07:39:49.379 | INFO | Perf report for the last 5.1346 seconds: | 20260507 07:39:49.379 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:39:49.379 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:49.379 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:49.379 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:49.379 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 07:39:49.379 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:49.379 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:49.379 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:39:49.379 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:49.379 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:49.379 | INFO | Uploaded 36GiB to GPU in 2.05443s (17.5231GiB/s) | 20260507 07:39:49.379 | INFO | Downloaded 34GiB to GPU in 1.95162s (17.4214GiB/s) | 20260507 07:39:54.524 | INFO | Analyzing chunk frames = [8238592,8239104) | 20260507 07:39:54.524 | INFO | MemInfo: { | 20260507 07:39:54.524 | INFO | "gpu" : " 17 GB", | 20260507 07:39:54.524 | INFO | "huge" : "", | 20260507 07:39:54.524 | INFO | "huge_shared" : " 27 GB", | 20260507 07:39:54.524 | INFO | "malloc" : " 14 GB", | 20260507 07:39:54.524 | INFO | "pinned" : " 29 GB" | 20260507 07:39:54.524 | INFO | } | 20260507 07:39:54.524 | INFO | Perf report for the last 5.14551 seconds: | 20260507 07:39:54.524 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:39:54.524 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:54.524 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:54.524 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:54.524 | INFO | ANALYSIS : 60.2%, 2.9, 93.1ms, 0 | 20260507 07:39:54.524 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:54.524 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:54.524 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:39:54.524 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:54.524 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:54.525 | INFO | Uploaded 36GiB to GPU in 2.01836s (17.8362GiB/s) | 20260507 07:39:54.525 | INFO | Downloaded 34GiB to GPU in 1.94621s (17.4699GiB/s) | 20260507 07:39:59.663 | INFO | Analyzing chunk frames = [8239104,8239616) | 20260507 07:39:59.663 | INFO | MemInfo: { | 20260507 07:39:59.663 | INFO | "gpu" : " 17 GB", | 20260507 07:39:59.663 | INFO | "huge" : "", | 20260507 07:39:59.663 | INFO | "huge_shared" : " 27 GB", | 20260507 07:39:59.663 | INFO | "malloc" : " 14 GB", | 20260507 07:39:59.663 | INFO | "pinned" : " 29 GB" | 20260507 07:39:59.663 | INFO | } | 20260507 07:39:59.663 | INFO | Perf report for the last 5.13875 seconds: | 20260507 07:39:59.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:39:59.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:59.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:59.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:59.663 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 07:39:59.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:59.663 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:59.663 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:39:59.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:59.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:39:59.663 | INFO | Uploaded 36GiB to GPU in 2.0394s (17.6522GiB/s) | 20260507 07:39:59.663 | INFO | Downloaded 34GiB to GPU in 1.91523s (17.7525GiB/s) | 20260507 07:40:04.802 | INFO | Analyzing chunk frames = [8239616,8240128) | 20260507 07:40:04.802 | INFO | MemInfo: { | 20260507 07:40:04.802 | INFO | "gpu" : " 17 GB", | 20260507 07:40:04.802 | INFO | "huge" : "", | 20260507 07:40:04.802 | INFO | "huge_shared" : " 27 GB", | 20260507 07:40:04.802 | INFO | "malloc" : " 14 GB", | 20260507 07:40:04.802 | INFO | "pinned" : " 29 GB" | 20260507 07:40:04.802 | INFO | } | 20260507 07:40:04.802 | INFO | Perf report for the last 5.13884 seconds: | 20260507 07:40:04.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:40:04.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:04.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:04.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:04.802 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 07:40:04.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:04.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:04.802 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 07:40:04.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:04.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:04.802 | INFO | Uploaded 36GiB to GPU in 2.03141s (17.7217GiB/s) | 20260507 07:40:04.802 | INFO | Downloaded 34GiB to GPU in 1.91799s (17.7268GiB/s) | 20260507 07:40:09.940 | INFO | Analyzing chunk frames = [8240128,8240640) | 20260507 07:40:09.940 | INFO | MemInfo: { | 20260507 07:40:09.940 | INFO | "gpu" : " 17 GB", | 20260507 07:40:09.940 | INFO | "huge" : "", | 20260507 07:40:09.940 | INFO | "huge_shared" : " 27 GB", | 20260507 07:40:09.940 | INFO | "malloc" : " 14 GB", | 20260507 07:40:09.940 | INFO | "pinned" : " 29 GB" | 20260507 07:40:09.940 | INFO | } | 20260507 07:40:09.940 | INFO | Perf report for the last 5.13849 seconds: | 20260507 07:40:09.940 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:40:09.940 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:09.940 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:09.940 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:09.940 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 07:40:09.940 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:09.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:09.940 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 07:40:09.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:09.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:09.941 | INFO | Uploaded 36GiB to GPU in 2.03701s (17.6729GiB/s) | 20260507 07:40:09.941 | INFO | Downloaded 34GiB to GPU in 2.00175s (16.9851GiB/s) | 20260507 07:40:15.082 | INFO | Analyzing chunk frames = [8240640,8241152) | 20260507 07:40:15.082 | INFO | MemInfo: { | 20260507 07:40:15.082 | INFO | "gpu" : " 17 GB", | 20260507 07:40:15.082 | INFO | "huge" : "", | 20260507 07:40:15.082 | INFO | "huge_shared" : " 27 GB", | 20260507 07:40:15.082 | INFO | "malloc" : " 14 GB", | 20260507 07:40:15.082 | INFO | "pinned" : " 29 GB" | 20260507 07:40:15.082 | INFO | } | 20260507 07:40:15.082 | INFO | Perf report for the last 5.14155 seconds: | 20260507 07:40:15.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:40:15.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:15.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:15.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:15.082 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:40:15.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:15.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:15.082 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 07:40:15.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:15.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:15.082 | INFO | Uploaded 36GiB to GPU in 1.98048s (18.1774GiB/s) | 20260507 07:40:15.082 | INFO | Downloaded 34GiB to GPU in 1.96335s (17.3173GiB/s) | 20260507 07:40:15.082 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:40:15.082 | INFO | Total mem usage: 18 GiB | 20260507 07:40:15.082 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:40:15.082 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:40:15.082 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:40:15.082 | INFO | High Water Breakdown: | 20260507 07:40:15.082 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:40:15.082 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:40:15.082 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:40:15.082 | INFO | ------------------------------------------ | 20260507 07:40:15.082 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:40:15.082 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:40:15.082 | INFO | Total mem usage: 26.5 GiB | 20260507 07:40:15.082 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:40:15.082 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:40:15.082 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:40:15.082 | INFO | High Water Breakdown: | 20260507 07:40:15.082 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:40:15.082 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:40:15.082 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:40:15.082 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:40:15.082 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:40:15.082 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:40:15.082 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:40:15.082 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:40:15.082 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:40:15.083 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:40:15.083 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:40:15.083 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:40:15.083 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:40:15.083 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:40:15.083 | INFO | ------------------------------------------ | 20260507 07:40:15.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:40:15.083 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:40:15.083 | INFO | Total mem usage: 25.6 GiB | 20260507 07:40:15.083 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:40:15.083 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:40:15.083 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:40:15.083 | INFO | High Water Breakdown: | 20260507 07:40:15.083 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:40:15.083 | INFO | ------------------------------------------ | 20260507 07:40:15.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:40:15.083 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:40:15.083 | INFO | Total mem usage: 15.7 GiB | 20260507 07:40:15.083 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:40:15.083 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:40:15.083 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:40:15.083 | INFO | High Water Breakdown: | 20260507 07:40:15.083 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:40:15.083 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:40:15.083 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:40:15.083 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:40:15.083 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:40:15.083 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:40:15.083 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:40:15.083 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:40:15.083 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:40:15.083 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:40:15.083 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:40:15.083 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:40:15.083 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:40:15.083 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:40:15.083 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:40:15.083 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:40:15.083 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:40:15.083 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:40:15.083 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:40:15.083 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:40:15.083 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:40:15.083 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:40:15.083 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:40:15.083 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:40:15.083 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:40:15.083 | INFO | ------------------------------------------ | 20260507 07:40:15.083 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:40:15.083 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:40:15.083 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:40:15.083 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:40:15.083 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.32e+01 / 1.03e+00 | 20260507 07:40:15.083 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 1.04e-01, 1.05e-02 / 1.03e-02 | 20260507 07:40:15.083 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.44e+00, 5.07e-01 / 2.74e-01 | 20260507 07:40:15.083 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.46e+00, 1.64e-01 / 1.04e-01 | 20260507 07:40:15.083 | INFO | PerfReport: 16.26%, DME -- 3.99e+01 / 4.45e+01, 3.82e+00 / 1.31e+00 | 20260507 07:40:15.083 | INFO | PerfReport: 57.77%, FrameLabeling -- 1.77e+01 / 1.98e+01, 1.34e+00 / 1.37e+00 | 20260507 07:40:15.083 | INFO | PerfReport: 6.88%, PulseAccumulating -- 2.11e+00 / 2.65e+00, 3.83e-01 / 2.39e-01 | 20260507 07:40:15.083 | INFO | PerfReport: 2.92%, Metrics -- 8.96e-01 / 1.20e+00, 2.39e-01 / 1.18e-01 | 20260507 07:40:15.083 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.02e-01 / 3.55e-01, 5.07e-02 / 1.51e-02 | 20260507 07:40:15.083 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:40:15.083 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:40:15.083 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:40:15.083 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 07:40:15.083 | INFO | PerfReport: 21.90%, Upload -- 2.11e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260507 07:40:15.083 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260507 07:40:15.083 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 07:40:15.083 | INFO | PerfReport: 1.54%, LockUpload -- 1.48e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 07:40:15.083 | INFO | PerfReport: 1.25%, LockDownload -- 1.20e+00 / 0.00e+00, 2.92e+00 / 0.00e+00 | 20260507 07:40:15.083 | INFO | PerfReport: 21.97%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 07:40:15.083 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.91e-02 / 0.00e+00 | 20260507 07:40:15.178 | INFO | Starting a round of Baz IO | 20260507 07:40:18.268 | INFO | Populating RTMetrics report | 20260507 07:40:18.268 | INFO | RTMetrics report populated | 20260507 07:40:18.268 | INFO | Opening tmp RTMetrics json file | 20260507 07:40:18.288 | INFO | Serializing RTMetrics to json | 20260507 07:40:18.289 | INFO | Serialization done, swapping tmp file now... | 20260507 07:40:18.290 | INFO | RTMetrics done | 20260507 07:40:18.358 | INFO | Finished a round of Baz IO: Wrote 1.633 GiB in 3.179 seconds | 20260507 07:40:18.358 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.54% / 0.00% / 67.46% | 20260507 07:40:18.358 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.101161% / 3.97364e-06% / 7.15256e-05% | 20260507 07:40:18.358 | INFO | Baz IO overall: | 20260507 07:40:18.358 | INFO | BytesWritten : 4125123551329 | 20260507 07:40:18.358 | INFO | NumEvents : 1974607132938 | 20260507 07:40:20.229 | INFO | Analyzing chunk frames = [8241152,8241664) | 20260507 07:40:20.229 | INFO | MemInfo: { | 20260507 07:40:20.229 | INFO | "gpu" : " 17 GB", | 20260507 07:40:20.229 | INFO | "huge" : "", | 20260507 07:40:20.229 | INFO | "huge_shared" : " 27 GB", | 20260507 07:40:20.229 | INFO | "malloc" : " 14 GB", | 20260507 07:40:20.229 | INFO | "pinned" : " 29 GB" | 20260507 07:40:20.229 | INFO | } | 20260507 07:40:20.229 | INFO | Perf report for the last 5.14677 seconds: | 20260507 07:40:20.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:40:20.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:20.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:20.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:20.229 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260507 07:40:20.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:20.229 | INFO | RT_METRICS : 42.5%, 1.0, 22.3ms, 0 | 20260507 07:40:20.229 | INFO | PRE_HQ : 59.2%, 1.8, 56.5ms, 0 | 20260507 07:40:20.229 | INFO | BAZWRITER : 61.4%, 3.3, 104.6ms, 0 | 20260507 07:40:20.229 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:40:20.229 | INFO | Uploaded 36GiB to GPU in 2.04261s (17.6245GiB/s) | 20260507 07:40:20.229 | INFO | Downloaded 38GiB to GPU in 2.17733s (17.4525GiB/s) | 20260507 07:40:25.361 | INFO | Analyzing chunk frames = [8241664,8242176) | 20260507 07:40:25.361 | INFO | MemInfo: { | 20260507 07:40:25.361 | INFO | "gpu" : " 17 GB", | 20260507 07:40:25.361 | INFO | "huge" : "", | 20260507 07:40:25.361 | INFO | "huge_shared" : " 27 GB", | 20260507 07:40:25.361 | INFO | "malloc" : " 14 GB", | 20260507 07:40:25.361 | INFO | "pinned" : " 29 GB" | 20260507 07:40:25.361 | INFO | } | 20260507 07:40:25.361 | INFO | Perf report for the last 5.13232 seconds: | 20260507 07:40:25.361 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:40:25.361 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:25.361 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:25.361 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:25.361 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 07:40:25.361 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:25.361 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:25.361 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 07:40:25.361 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:25.361 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:25.361 | INFO | Uploaded 36GiB to GPU in 2.01397s (17.8751GiB/s) | 20260507 07:40:25.361 | INFO | Downloaded 34GiB to GPU in 1.94688s (17.4638GiB/s) | 20260507 07:40:30.491 | INFO | Analyzing chunk frames = [8242176,8242688) | 20260507 07:40:30.491 | INFO | MemInfo: { | 20260507 07:40:30.491 | INFO | "gpu" : " 17 GB", | 20260507 07:40:30.491 | INFO | "huge" : "", | 20260507 07:40:30.491 | INFO | "huge_shared" : " 27 GB", | 20260507 07:40:30.491 | INFO | "malloc" : " 14 GB", | 20260507 07:40:30.491 | INFO | "pinned" : " 29 GB" | 20260507 07:40:30.491 | INFO | } | 20260507 07:40:30.491 | INFO | Perf report for the last 5.12986 seconds: | 20260507 07:40:30.491 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:40:30.491 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:30.491 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:30.491 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:30.491 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 07:40:30.491 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:30.491 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:30.491 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:40:30.491 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:30.491 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:30.491 | INFO | Uploaded 36GiB to GPU in 2.00894s (17.9199GiB/s) | 20260507 07:40:30.491 | INFO | Downloaded 34GiB to GPU in 1.92246s (17.6857GiB/s) | 20260507 07:40:35.637 | INFO | Analyzing chunk frames = [8242688,8243200) | 20260507 07:40:35.637 | INFO | MemInfo: { | 20260507 07:40:35.637 | INFO | "gpu" : " 17 GB", | 20260507 07:40:35.637 | INFO | "huge" : "", | 20260507 07:40:35.637 | INFO | "huge_shared" : " 27 GB", | 20260507 07:40:35.637 | INFO | "malloc" : " 14 GB", | 20260507 07:40:35.637 | INFO | "pinned" : " 29 GB" | 20260507 07:40:35.637 | INFO | } | 20260507 07:40:35.637 | INFO | Perf report for the last 5.14651 seconds: | 20260507 07:40:35.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:40:35.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:35.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:35.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:35.637 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 07:40:35.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:35.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:35.637 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:40:35.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:35.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:35.638 | INFO | Uploaded 36GiB to GPU in 1.99306s (18.0627GiB/s) | 20260507 07:40:35.638 | INFO | Downloaded 34GiB to GPU in 1.9458s (17.4736GiB/s) | 20260507 07:40:40.777 | INFO | Analyzing chunk frames = [8243200,8243712) | 20260507 07:40:40.777 | INFO | MemInfo: { | 20260507 07:40:40.777 | INFO | "gpu" : " 17 GB", | 20260507 07:40:40.778 | INFO | "huge" : "", | 20260507 07:40:40.778 | INFO | "huge_shared" : " 27 GB", | 20260507 07:40:40.778 | INFO | "malloc" : " 14 GB", | 20260507 07:40:40.778 | INFO | "pinned" : " 29 GB" | 20260507 07:40:40.778 | INFO | } | 20260507 07:40:40.778 | INFO | Perf report for the last 5.14021 seconds: | 20260507 07:40:40.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:40:40.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:40.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:40.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:40.778 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 07:40:40.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:40.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:40.778 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:40:40.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:40.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:40.778 | INFO | Uploaded 36GiB to GPU in 2.04971s (17.5635GiB/s) | 20260507 07:40:40.778 | INFO | Downloaded 34GiB to GPU in 1.95211s (17.417GiB/s) | 20260507 07:40:45.919 | INFO | Analyzing chunk frames = [8243712,8244224) | 20260507 07:40:45.919 | INFO | MemInfo: { | 20260507 07:40:45.919 | INFO | "gpu" : " 17 GB", | 20260507 07:40:45.920 | INFO | "huge" : "", | 20260507 07:40:45.920 | INFO | "huge_shared" : " 27 GB", | 20260507 07:40:45.920 | INFO | "malloc" : " 14 GB", | 20260507 07:40:45.920 | INFO | "pinned" : " 29 GB" | 20260507 07:40:45.920 | INFO | } | 20260507 07:40:45.920 | INFO | Perf report for the last 5.14201 seconds: | 20260507 07:40:45.920 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:40:45.920 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:45.920 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:45.920 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:45.920 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 07:40:45.920 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:45.920 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:45.920 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:40:45.920 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:45.920 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:45.920 | INFO | Uploaded 36GiB to GPU in 2.03432s (17.6963GiB/s) | 20260507 07:40:45.920 | INFO | Downloaded 34GiB to GPU in 1.91118s (17.7901GiB/s) | 20260507 07:40:51.061 | INFO | Analyzing chunk frames = [8244224,8244736) | 20260507 07:40:51.061 | INFO | MemInfo: { | 20260507 07:40:51.061 | INFO | "gpu" : " 17 GB", | 20260507 07:40:51.061 | INFO | "huge" : "", | 20260507 07:40:51.061 | INFO | "huge_shared" : " 27 GB", | 20260507 07:40:51.061 | INFO | "malloc" : " 14 GB", | 20260507 07:40:51.061 | INFO | "pinned" : " 29 GB" | 20260507 07:40:51.061 | INFO | } | 20260507 07:40:51.061 | INFO | Perf report for the last 5.1412 seconds: | 20260507 07:40:51.061 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:40:51.061 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:51.061 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:51.061 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:51.061 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 07:40:51.061 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:51.061 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:51.061 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 07:40:51.061 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:51.061 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:51.061 | INFO | Uploaded 36GiB to GPU in 1.90418s (18.9057GiB/s) | 20260507 07:40:51.061 | INFO | Downloaded 34GiB to GPU in 1.97929s (17.1779GiB/s) | 20260507 07:40:56.194 | INFO | Analyzing chunk frames = [8244736,8245248) | 20260507 07:40:56.194 | INFO | MemInfo: { | 20260507 07:40:56.194 | INFO | "gpu" : " 17 GB", | 20260507 07:40:56.194 | INFO | "huge" : "", | 20260507 07:40:56.194 | INFO | "huge_shared" : " 27 GB", | 20260507 07:40:56.194 | INFO | "malloc" : " 14 GB", | 20260507 07:40:56.194 | INFO | "pinned" : " 29 GB" | 20260507 07:40:56.194 | INFO | } | 20260507 07:40:56.194 | INFO | Perf report for the last 5.13325 seconds: | 20260507 07:40:56.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:40:56.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:56.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:56.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:56.194 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:40:56.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:56.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:56.194 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:40:56.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:56.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:40:56.194 | INFO | Uploaded 36GiB to GPU in 1.99738s (18.0236GiB/s) | 20260507 07:40:56.194 | INFO | Downloaded 34GiB to GPU in 1.96274s (17.3227GiB/s) | 20260507 07:40:56.291 | INFO | Starting a round of Baz IO | 20260507 07:40:59.383 | INFO | Populating RTMetrics report | 20260507 07:40:59.383 | INFO | RTMetrics report populated | 20260507 07:40:59.383 | INFO | Opening tmp RTMetrics json file | 20260507 07:40:59.393 | INFO | Serializing RTMetrics to json | 20260507 07:40:59.399 | INFO | Serialization done, swapping tmp file now... | 20260507 07:40:59.399 | INFO | RTMetrics done | 20260507 07:40:59.471 | INFO | Finished a round of Baz IO: Wrote 1.634 GiB in 3.179 seconds | 20260507 07:40:59.471 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.54% / 0.00% / 67.46% | 20260507 07:40:59.471 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.101574% / 3.97364e-06% / 3.97364e-05% | 20260507 07:40:59.471 | INFO | Baz IO overall: | 20260507 07:40:59.471 | INFO | BytesWritten : 4126878225649 | 20260507 07:40:59.471 | INFO | NumEvents : 1975426158095 | 20260507 07:41:01.331 | INFO | Analyzing chunk frames = [8245248,8245760) | 20260507 07:41:01.331 | INFO | MemInfo: { | 20260507 07:41:01.331 | INFO | "gpu" : " 17 GB", | 20260507 07:41:01.331 | INFO | "huge" : "", | 20260507 07:41:01.331 | INFO | "huge_shared" : " 27 GB", | 20260507 07:41:01.331 | INFO | "malloc" : " 14 GB", | 20260507 07:41:01.331 | INFO | "pinned" : " 29 GB" | 20260507 07:41:01.331 | INFO | } | 20260507 07:41:01.331 | INFO | Perf report for the last 5.13673 seconds: | 20260507 07:41:01.331 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:41:01.331 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:01.331 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:01.331 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:01.331 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 07:41:01.331 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:01.331 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:41:01.331 | INFO | PRE_HQ : 58.9%, 1.8, 54.5ms, 0 | 20260507 07:41:01.331 | INFO | BAZWRITER : 61.5%, 3.1, 98.5ms, 0 | 20260507 07:41:01.331 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:41:01.331 | INFO | Uploaded 36GiB to GPU in 2.04176s (17.6318GiB/s) | 20260507 07:41:01.331 | INFO | Downloaded 38GiB to GPU in 2.17516s (17.47GiB/s) | 20260507 07:41:06.477 | INFO | Analyzing chunk frames = [8245760,8246272) | 20260507 07:41:06.477 | INFO | MemInfo: { | 20260507 07:41:06.477 | INFO | "gpu" : " 17 GB", | 20260507 07:41:06.477 | INFO | "huge" : "", | 20260507 07:41:06.477 | INFO | "huge_shared" : " 27 GB", | 20260507 07:41:06.477 | INFO | "malloc" : " 14 GB", | 20260507 07:41:06.477 | INFO | "pinned" : " 29 GB" | 20260507 07:41:06.477 | INFO | } | 20260507 07:41:06.477 | INFO | Perf report for the last 5.14606 seconds: | 20260507 07:41:06.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:41:06.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:06.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:06.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:06.477 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 07:41:06.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:06.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:06.477 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 07:41:06.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:06.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:06.477 | INFO | Uploaded 36GiB to GPU in 1.99015s (18.0891GiB/s) | 20260507 07:41:06.477 | INFO | Downloaded 34GiB to GPU in 1.92834s (17.6318GiB/s) | 20260507 07:41:11.614 | INFO | Analyzing chunk frames = [8246272,8246784) | 20260507 07:41:11.614 | INFO | MemInfo: { | 20260507 07:41:11.614 | INFO | "gpu" : " 17 GB", | 20260507 07:41:11.614 | INFO | "huge" : "", | 20260507 07:41:11.614 | INFO | "huge_shared" : " 27 GB", | 20260507 07:41:11.614 | INFO | "malloc" : " 14 GB", | 20260507 07:41:11.614 | INFO | "pinned" : " 29 GB" | 20260507 07:41:11.614 | INFO | } | 20260507 07:41:11.614 | INFO | Perf report for the last 5.13683 seconds: | 20260507 07:41:11.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:41:11.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:11.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:11.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:11.614 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260507 07:41:11.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:11.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:11.614 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:41:11.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:11.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:11.614 | INFO | Uploaded 36GiB to GPU in 2.02888s (17.7437GiB/s) | 20260507 07:41:11.614 | INFO | Downloaded 34GiB to GPU in 1.91769s (17.7296GiB/s) | 20260507 07:41:16.750 | INFO | Analyzing chunk frames = [8246784,8247296) | 20260507 07:41:16.750 | INFO | MemInfo: { | 20260507 07:41:16.750 | INFO | "gpu" : " 17 GB", | 20260507 07:41:16.750 | INFO | "huge" : "", | 20260507 07:41:16.750 | INFO | "huge_shared" : " 27 GB", | 20260507 07:41:16.750 | INFO | "malloc" : " 14 GB", | 20260507 07:41:16.750 | INFO | "pinned" : " 29 GB" | 20260507 07:41:16.750 | INFO | } | 20260507 07:41:16.750 | INFO | Perf report for the last 5.13631 seconds: | 20260507 07:41:16.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:41:16.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:16.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:16.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:16.750 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 07:41:16.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:16.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:16.750 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:41:16.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:16.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:16.750 | INFO | Uploaded 36GiB to GPU in 2.05237s (17.5407GiB/s) | 20260507 07:41:16.750 | INFO | Downloaded 34GiB to GPU in 1.94549s (17.4763GiB/s) | 20260507 07:41:21.891 | INFO | Analyzing chunk frames = [8247296,8247808) | 20260507 07:41:21.891 | INFO | MemInfo: { | 20260507 07:41:21.891 | INFO | "gpu" : " 17 GB", | 20260507 07:41:21.891 | INFO | "huge" : "", | 20260507 07:41:21.891 | INFO | "huge_shared" : " 27 GB", | 20260507 07:41:21.891 | INFO | "malloc" : " 14 GB", | 20260507 07:41:21.891 | INFO | "pinned" : " 29 GB" | 20260507 07:41:21.891 | INFO | } | 20260507 07:41:21.891 | INFO | Perf report for the last 5.14072 seconds: | 20260507 07:41:21.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:41:21.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:21.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:21.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:21.891 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 07:41:21.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:21.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:21.891 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:41:21.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:21.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:21.891 | INFO | Uploaded 36GiB to GPU in 2.03338s (17.7045GiB/s) | 20260507 07:41:21.891 | INFO | Downloaded 34GiB to GPU in 1.91431s (17.761GiB/s) | 20260507 07:41:27.038 | INFO | Analyzing chunk frames = [8247808,8248320) | 20260507 07:41:27.038 | INFO | MemInfo: { | 20260507 07:41:27.038 | INFO | "gpu" : " 17 GB", | 20260507 07:41:27.038 | INFO | "huge" : "", | 20260507 07:41:27.038 | INFO | "huge_shared" : " 27 GB", | 20260507 07:41:27.038 | INFO | "malloc" : " 14 GB", | 20260507 07:41:27.038 | INFO | "pinned" : " 29 GB" | 20260507 07:41:27.038 | INFO | } | 20260507 07:41:27.038 | INFO | Perf report for the last 5.14758 seconds: | 20260507 07:41:27.038 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:41:27.038 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:27.038 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:27.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:27.039 | INFO | ANALYSIS : 60.3%, 3.0, 94.3ms, 0 | 20260507 07:41:27.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:27.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:27.039 | INFO | PRE_HQ : 28.9%, 1.0, 15.0ms, 0 | 20260507 07:41:27.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:27.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:27.039 | INFO | Uploaded 36GiB to GPU in 2.06682s (17.4181GiB/s) | 20260507 07:41:27.039 | INFO | Downloaded 34GiB to GPU in 1.92131s (17.6962GiB/s) | 20260507 07:41:32.177 | INFO | Analyzing chunk frames = [8248320,8248832) | 20260507 07:41:32.177 | INFO | MemInfo: { | 20260507 07:41:32.177 | INFO | "gpu" : " 17 GB", | 20260507 07:41:32.177 | INFO | "huge" : "", | 20260507 07:41:32.178 | INFO | "huge_shared" : " 27 GB", | 20260507 07:41:32.178 | INFO | "malloc" : " 14 GB", | 20260507 07:41:32.178 | INFO | "pinned" : " 29 GB" | 20260507 07:41:32.178 | INFO | } | 20260507 07:41:32.178 | INFO | Perf report for the last 5.13921 seconds: | 20260507 07:41:32.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:41:32.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:32.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:32.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:32.178 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 07:41:32.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:32.178 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:32.178 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 07:41:32.178 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:32.178 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:32.178 | INFO | Uploaded 36GiB to GPU in 1.98135s (18.1695GiB/s) | 20260507 07:41:32.178 | INFO | Downloaded 34GiB to GPU in 1.97169s (17.2441GiB/s) | 20260507 07:41:37.308 | INFO | Analyzing chunk frames = [8248832,8249344) | 20260507 07:41:37.308 | INFO | MemInfo: { | 20260507 07:41:37.308 | INFO | "gpu" : " 17 GB", | 20260507 07:41:37.308 | INFO | "huge" : "", | 20260507 07:41:37.308 | INFO | "huge_shared" : " 27 GB", | 20260507 07:41:37.308 | INFO | "malloc" : " 14 GB", | 20260507 07:41:37.308 | INFO | "pinned" : " 29 GB" | 20260507 07:41:37.308 | INFO | } | 20260507 07:41:37.308 | INFO | Perf report for the last 5.13045 seconds: | 20260507 07:41:37.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:41:37.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:37.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:37.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:37.308 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260507 07:41:37.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:37.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:37.308 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 07:41:37.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:37.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:37.308 | INFO | Uploaded 36GiB to GPU in 1.98459s (18.1398GiB/s) | 20260507 07:41:37.308 | INFO | Downloaded 34GiB to GPU in 1.96019s (17.3452GiB/s) | 20260507 07:41:37.308 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:41:37.308 | INFO | Total mem usage: 18 GiB | 20260507 07:41:37.308 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:41:37.308 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:41:37.308 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:41:37.308 | INFO | High Water Breakdown: | 20260507 07:41:37.309 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:41:37.309 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:41:37.309 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:41:37.309 | INFO | ------------------------------------------ | 20260507 07:41:37.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:41:37.309 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:41:37.309 | INFO | Total mem usage: 26.5 GiB | 20260507 07:41:37.309 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:41:37.309 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:41:37.309 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:41:37.309 | INFO | High Water Breakdown: | 20260507 07:41:37.309 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:41:37.309 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:41:37.309 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:41:37.309 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:41:37.309 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:41:37.309 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:41:37.309 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:41:37.309 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:41:37.309 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:41:37.309 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:41:37.309 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:41:37.309 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:41:37.309 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:41:37.309 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:41:37.309 | INFO | ------------------------------------------ | 20260507 07:41:37.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:41:37.309 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:41:37.309 | INFO | Total mem usage: 25.6 GiB | 20260507 07:41:37.309 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:41:37.309 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:41:37.309 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:41:37.309 | INFO | High Water Breakdown: | 20260507 07:41:37.309 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:41:37.309 | INFO | ------------------------------------------ | 20260507 07:41:37.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:41:37.309 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:41:37.309 | INFO | Total mem usage: 15.7 GiB | 20260507 07:41:37.309 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:41:37.309 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:41:37.309 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:41:37.309 | INFO | High Water Breakdown: | 20260507 07:41:37.309 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:41:37.309 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:41:37.309 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:41:37.309 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:41:37.309 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:41:37.309 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:41:37.309 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:41:37.309 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:41:37.309 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:41:37.309 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:41:37.309 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:41:37.309 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:41:37.309 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:41:37.309 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:41:37.309 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:41:37.309 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:41:37.309 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:41:37.309 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:41:37.309 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:41:37.309 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:41:37.309 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:41:37.309 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:41:37.309 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:41:37.309 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:41:37.309 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:41:37.309 | INFO | ------------------------------------------ | 20260507 07:41:37.309 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:41:37.309 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:41:37.309 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:41:37.309 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:41:37.309 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.34e+01 / 1.18e+00 | 20260507 07:41:37.309 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.02e-01, 1.03e-02 / 3.78e-03 | 20260507 07:41:37.309 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.48e+00, 5.12e-01 / 2.17e-01 | 20260507 07:41:37.309 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.51e+00, 1.69e-01 / 8.01e-02 | 20260507 07:41:37.309 | INFO | PerfReport: 16.28%, DME -- 4.00e+01 / 4.49e+01, 3.98e+00 / 1.53e+00 | 20260507 07:41:37.309 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.30e+00 / 1.28e+00 | 20260507 07:41:37.309 | INFO | PerfReport: 6.89%, PulseAccumulating -- 2.11e+00 / 2.67e+00, 3.84e-01 / 2.19e-01 | 20260507 07:41:37.309 | INFO | PerfReport: 2.93%, Metrics -- 8.99e-01 / 1.22e+00, 2.43e-01 / 1.05e-01 | 20260507 07:41:37.309 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.02e-01 / 3.63e-01, 5.11e-02 / 2.09e-02 | 20260507 07:41:37.309 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:41:37.309 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:41:37.309 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:41:37.309 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 07:41:37.309 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260507 07:41:37.310 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260507 07:41:37.310 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 07:41:37.310 | INFO | PerfReport: 1.55%, LockUpload -- 1.49e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 07:41:37.310 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260507 07:41:37.310 | INFO | PerfReport: 22.04%, LockCompute -- 2.12e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 07:41:37.310 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.07e-02 / 0.00e+00 | 20260507 07:41:37.416 | INFO | Starting a round of Baz IO | 20260507 07:41:40.498 | INFO | Populating RTMetrics report | 20260507 07:41:40.498 | INFO | RTMetrics report populated | 20260507 07:41:40.498 | INFO | Opening tmp RTMetrics json file | 20260507 07:41:40.514 | INFO | Serializing RTMetrics to json | 20260507 07:41:40.516 | INFO | Serialization done, swapping tmp file now... | 20260507 07:41:40.517 | INFO | RTMetrics done | 20260507 07:41:40.587 | INFO | Finished a round of Baz IO: Wrote 1.633 GiB in 3.17 seconds | 20260507 07:41:40.587 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.54% / 0.00% / 67.46% | 20260507 07:41:40.587 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.101161% / 7.94729e-06% / 5.5631e-05% | 20260507 07:41:40.587 | INFO | Baz IO overall: | 20260507 07:41:40.587 | INFO | BytesWritten : 4128632052563 | 20260507 07:41:40.587 | INFO | NumEvents : 1976244568110 | 20260507 07:41:42.444 | INFO | Analyzing chunk frames = [8249344,8249856) | 20260507 07:41:42.444 | INFO | MemInfo: { | 20260507 07:41:42.444 | INFO | "gpu" : " 17 GB", | 20260507 07:41:42.444 | INFO | "huge" : "", | 20260507 07:41:42.444 | INFO | "huge_shared" : " 27 GB", | 20260507 07:41:42.444 | INFO | "malloc" : " 14 GB", | 20260507 07:41:42.444 | INFO | "pinned" : " 29 GB" | 20260507 07:41:42.444 | INFO | } | 20260507 07:41:42.444 | INFO | Perf report for the last 5.13597 seconds: | 20260507 07:41:42.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:41:42.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:42.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:42.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:42.444 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 07:41:42.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:42.444 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 07:41:42.444 | INFO | PRE_HQ : 59.0%, 1.8, 54.6ms, 0 | 20260507 07:41:42.444 | INFO | BAZWRITER : 61.6%, 3.1, 98.3ms, 0 | 20260507 07:41:42.444 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:41:42.444 | INFO | Uploaded 36GiB to GPU in 2.0549s (17.5191GiB/s) | 20260507 07:41:42.444 | INFO | Downloaded 38GiB to GPU in 2.18021s (17.4295GiB/s) | 20260507 07:41:47.590 | INFO | Analyzing chunk frames = [8249856,8250368) | 20260507 07:41:47.590 | INFO | MemInfo: { | 20260507 07:41:47.590 | INFO | "gpu" : " 17 GB", | 20260507 07:41:47.590 | INFO | "huge" : "", | 20260507 07:41:47.590 | INFO | "huge_shared" : " 27 GB", | 20260507 07:41:47.590 | INFO | "malloc" : " 14 GB", | 20260507 07:41:47.590 | INFO | "pinned" : " 29 GB" | 20260507 07:41:47.590 | INFO | } | 20260507 07:41:47.590 | INFO | Perf report for the last 5.14558 seconds: | 20260507 07:41:47.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:41:47.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:47.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:47.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:47.590 | INFO | ANALYSIS : 60.7%, 3.0, 94.7ms, 0 | 20260507 07:41:47.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:47.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:47.590 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:41:47.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:47.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:47.590 | INFO | Uploaded 36GiB to GPU in 2.0441s (17.6117GiB/s) | 20260507 07:41:47.590 | INFO | Downloaded 34GiB to GPU in 1.94061s (17.5203GiB/s) | 20260507 07:41:52.728 | INFO | Analyzing chunk frames = [8250368,8250880) | 20260507 07:41:52.728 | INFO | MemInfo: { | 20260507 07:41:52.728 | INFO | "gpu" : " 17 GB", | 20260507 07:41:52.728 | INFO | "huge" : "", | 20260507 07:41:52.728 | INFO | "huge_shared" : " 27 GB", | 20260507 07:41:52.728 | INFO | "malloc" : " 14 GB", | 20260507 07:41:52.728 | INFO | "pinned" : " 29 GB" | 20260507 07:41:52.728 | INFO | } | 20260507 07:41:52.728 | INFO | Perf report for the last 5.13813 seconds: | 20260507 07:41:52.728 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:41:52.728 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:52.728 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:52.728 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:52.728 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:41:52.728 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:52.728 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:52.728 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:41:52.728 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:52.728 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:52.728 | INFO | Uploaded 36GiB to GPU in 2.04447s (17.6085GiB/s) | 20260507 07:41:52.728 | INFO | Downloaded 34GiB to GPU in 1.9522s (17.4162GiB/s) | 20260507 07:41:57.871 | INFO | Analyzing chunk frames = [8250880,8251392) | 20260507 07:41:57.871 | INFO | MemInfo: { | 20260507 07:41:57.871 | INFO | "gpu" : " 17 GB", | 20260507 07:41:57.871 | INFO | "huge" : "", | 20260507 07:41:57.871 | INFO | "huge_shared" : " 27 GB", | 20260507 07:41:57.871 | INFO | "malloc" : " 14 GB", | 20260507 07:41:57.871 | INFO | "pinned" : " 29 GB" | 20260507 07:41:57.871 | INFO | } | 20260507 07:41:57.871 | INFO | Perf report for the last 5.14291 seconds: | 20260507 07:41:57.871 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:41:57.871 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:57.871 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:57.871 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:57.871 | INFO | ANALYSIS : 60.1%, 2.9, 93.0ms, 0 | 20260507 07:41:57.871 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:57.871 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:57.871 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:41:57.871 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:57.871 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:41:57.871 | INFO | Uploaded 36GiB to GPU in 2.02028s (17.8193GiB/s) | 20260507 07:41:57.871 | INFO | Downloaded 34GiB to GPU in 1.95456s (17.3952GiB/s) | 20260507 07:42:03.006 | INFO | Analyzing chunk frames = [8251392,8251904) | 20260507 07:42:03.006 | INFO | MemInfo: { | 20260507 07:42:03.006 | INFO | "gpu" : " 17 GB", | 20260507 07:42:03.006 | INFO | "huge" : "", | 20260507 07:42:03.006 | INFO | "huge_shared" : " 27 GB", | 20260507 07:42:03.006 | INFO | "malloc" : " 14 GB", | 20260507 07:42:03.006 | INFO | "pinned" : " 29 GB" | 20260507 07:42:03.006 | INFO | } | 20260507 07:42:03.007 | INFO | Perf report for the last 5.13579 seconds: | 20260507 07:42:03.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:42:03.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:03.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:03.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:03.007 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 07:42:03.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:03.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:03.007 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 07:42:03.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:03.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:03.007 | INFO | Uploaded 36GiB to GPU in 2.06586s (17.4262GiB/s) | 20260507 07:42:03.007 | INFO | Downloaded 34GiB to GPU in 1.91574s (17.7477GiB/s) | 20260507 07:42:08.146 | INFO | Analyzing chunk frames = [8251904,8252416) | 20260507 07:42:08.146 | INFO | MemInfo: { | 20260507 07:42:08.146 | INFO | "gpu" : " 17 GB", | 20260507 07:42:08.146 | INFO | "huge" : "", | 20260507 07:42:08.146 | INFO | "huge_shared" : " 27 GB", | 20260507 07:42:08.146 | INFO | "malloc" : " 14 GB", | 20260507 07:42:08.146 | INFO | "pinned" : " 29 GB" | 20260507 07:42:08.146 | INFO | } | 20260507 07:42:08.146 | INFO | Perf report for the last 5.13936 seconds: | 20260507 07:42:08.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:42:08.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:08.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:08.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:08.146 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 07:42:08.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:08.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:08.146 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:42:08.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:08.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:08.146 | INFO | Uploaded 36GiB to GPU in 2.05707s (17.5006GiB/s) | 20260507 07:42:08.146 | INFO | Downloaded 34GiB to GPU in 1.9207s (17.7019GiB/s) | 20260507 07:42:13.284 | INFO | Analyzing chunk frames = [8252416,8252928) | 20260507 07:42:13.284 | INFO | MemInfo: { | 20260507 07:42:13.284 | INFO | "gpu" : " 17 GB", | 20260507 07:42:13.284 | INFO | "huge" : "", | 20260507 07:42:13.284 | INFO | "huge_shared" : " 27 GB", | 20260507 07:42:13.284 | INFO | "malloc" : " 14 GB", | 20260507 07:42:13.284 | INFO | "pinned" : " 29 GB" | 20260507 07:42:13.285 | INFO | } | 20260507 07:42:13.285 | INFO | Perf report for the last 5.1386 seconds: | 20260507 07:42:13.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:42:13.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:13.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:13.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:13.285 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 07:42:13.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:13.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:13.285 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:42:13.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:13.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:13.285 | INFO | Uploaded 36GiB to GPU in 2.02159s (17.8077GiB/s) | 20260507 07:42:13.285 | INFO | Downloaded 34GiB to GPU in 1.98794s (17.1031GiB/s) | 20260507 07:42:18.424 | INFO | Analyzing chunk frames = [8252928,8253440) | 20260507 07:42:18.424 | INFO | MemInfo: { | 20260507 07:42:18.424 | INFO | "gpu" : " 17 GB", | 20260507 07:42:18.424 | INFO | "huge" : "", | 20260507 07:42:18.424 | INFO | "huge_shared" : " 27 GB", | 20260507 07:42:18.424 | INFO | "malloc" : " 14 GB", | 20260507 07:42:18.424 | INFO | "pinned" : " 29 GB" | 20260507 07:42:18.424 | INFO | } | 20260507 07:42:18.425 | INFO | Perf report for the last 5.13998 seconds: | 20260507 07:42:18.425 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:42:18.425 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:18.425 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:18.425 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:18.425 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:42:18.425 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:18.425 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:18.425 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:42:18.425 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:18.425 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:18.425 | INFO | Uploaded 36GiB to GPU in 1.9883s (18.1059GiB/s) | 20260507 07:42:18.425 | INFO | Downloaded 34GiB to GPU in 1.95888s (17.3568GiB/s) | 20260507 07:42:18.522 | INFO | Starting a round of Baz IO | 20260507 07:42:21.615 | INFO | Populating RTMetrics report | 20260507 07:42:21.615 | INFO | RTMetrics report populated | 20260507 07:42:21.615 | INFO | Opening tmp RTMetrics json file | 20260507 07:42:21.635 | INFO | Serializing RTMetrics to json | 20260507 07:42:21.639 | INFO | Serialization done, swapping tmp file now... | 20260507 07:42:21.640 | INFO | RTMetrics done | 20260507 07:42:21.708 | INFO | Finished a round of Baz IO: Wrote 1.633 GiB in 3.186 seconds | 20260507 07:42:21.708 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.54% / 0.00% / 67.46% | 20260507 07:42:21.708 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.10045% / 0% / 5.5631e-05% | 20260507 07:42:21.708 | INFO | Baz IO overall: | 20260507 07:42:21.708 | INFO | BytesWritten : 4130385768952 | 20260507 07:42:21.708 | INFO | NumEvents : 1977062872121 | 20260507 07:42:23.565 | INFO | Analyzing chunk frames = [8253440,8253952) | 20260507 07:42:23.565 | INFO | MemInfo: { | 20260507 07:42:23.565 | INFO | "gpu" : " 17 GB", | 20260507 07:42:23.565 | INFO | "huge" : "", | 20260507 07:42:23.565 | INFO | "huge_shared" : " 27 GB", | 20260507 07:42:23.565 | INFO | "malloc" : " 14 GB", | 20260507 07:42:23.565 | INFO | "pinned" : " 29 GB" | 20260507 07:42:23.565 | INFO | } | 20260507 07:42:23.565 | INFO | Perf report for the last 5.14049 seconds: | 20260507 07:42:23.565 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:42:23.565 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:23.565 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:23.565 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:23.565 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 07:42:23.565 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:23.565 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:42:23.565 | INFO | PRE_HQ : 58.8%, 1.8, 54.3ms, 0 | 20260507 07:42:23.565 | INFO | BAZWRITER : 61.6%, 3.1, 98.1ms, 0 | 20260507 07:42:23.565 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 07:42:23.565 | INFO | Uploaded 36GiB to GPU in 2.06296s (17.4507GiB/s) | 20260507 07:42:23.565 | INFO | Downloaded 38GiB to GPU in 2.18201s (17.4151GiB/s) | 20260507 07:42:28.697 | INFO | Analyzing chunk frames = [8253952,8254464) | 20260507 07:42:28.697 | INFO | MemInfo: { | 20260507 07:42:28.697 | INFO | "gpu" : " 17 GB", | 20260507 07:42:28.697 | INFO | "huge" : "", | 20260507 07:42:28.697 | INFO | "huge_shared" : " 27 GB", | 20260507 07:42:28.697 | INFO | "malloc" : " 14 GB", | 20260507 07:42:28.697 | INFO | "pinned" : " 29 GB" | 20260507 07:42:28.697 | INFO | } | 20260507 07:42:28.697 | INFO | Perf report for the last 5.13223 seconds: | 20260507 07:42:28.697 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:42:28.697 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:28.697 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:28.697 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:28.697 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 07:42:28.697 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:28.697 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:28.697 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 07:42:28.697 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:28.697 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:28.698 | INFO | Uploaded 36GiB to GPU in 1.99501s (18.045GiB/s) | 20260507 07:42:28.698 | INFO | Downloaded 34GiB to GPU in 1.93966s (17.5288GiB/s) | 20260507 07:42:33.842 | INFO | Analyzing chunk frames = [8254464,8254976) | 20260507 07:42:33.842 | INFO | MemInfo: { | 20260507 07:42:33.842 | INFO | "gpu" : " 17 GB", | 20260507 07:42:33.842 | INFO | "huge" : "", | 20260507 07:42:33.842 | INFO | "huge_shared" : " 27 GB", | 20260507 07:42:33.842 | INFO | "malloc" : " 14 GB", | 20260507 07:42:33.842 | INFO | "pinned" : " 29 GB" | 20260507 07:42:33.842 | INFO | } | 20260507 07:42:33.842 | INFO | Perf report for the last 5.1448 seconds: | 20260507 07:42:33.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:42:33.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:33.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:33.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:33.842 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:42:33.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:33.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:33.842 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:42:33.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:33.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:33.842 | INFO | Uploaded 36GiB to GPU in 2.05195s (17.5443GiB/s) | 20260507 07:42:33.842 | INFO | Downloaded 34GiB to GPU in 1.95083s (17.4285GiB/s) | 20260507 07:42:38.983 | INFO | Analyzing chunk frames = [8254976,8255488) | 20260507 07:42:38.983 | INFO | MemInfo: { | 20260507 07:42:38.983 | INFO | "gpu" : " 17 GB", | 20260507 07:42:38.983 | INFO | "huge" : "", | 20260507 07:42:38.983 | INFO | "huge_shared" : " 27 GB", | 20260507 07:42:38.983 | INFO | "malloc" : " 14 GB", | 20260507 07:42:38.983 | INFO | "pinned" : " 29 GB" | 20260507 07:42:38.984 | INFO | } | 20260507 07:42:38.984 | INFO | Perf report for the last 5.14201 seconds: | 20260507 07:42:38.984 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:42:38.984 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:38.984 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:38.984 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:38.984 | INFO | ANALYSIS : 60.2%, 2.9, 93.0ms, 0 | 20260507 07:42:38.984 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:38.984 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:38.984 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:42:38.984 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:38.984 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:38.984 | INFO | Uploaded 36GiB to GPU in 2.01159s (17.8963GiB/s) | 20260507 07:42:38.984 | INFO | Downloaded 34GiB to GPU in 1.94039s (17.5222GiB/s) | 20260507 07:42:44.123 | INFO | Analyzing chunk frames = [8255488,8256000) | 20260507 07:42:44.123 | INFO | MemInfo: { | 20260507 07:42:44.123 | INFO | "gpu" : " 17 GB", | 20260507 07:42:44.123 | INFO | "huge" : "", | 20260507 07:42:44.123 | INFO | "huge_shared" : " 27 GB", | 20260507 07:42:44.123 | INFO | "malloc" : " 14 GB", | 20260507 07:42:44.123 | INFO | "pinned" : " 29 GB" | 20260507 07:42:44.123 | INFO | } | 20260507 07:42:44.123 | INFO | Perf report for the last 5.13929 seconds: | 20260507 07:42:44.123 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:42:44.123 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:44.123 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:44.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:44.124 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 07:42:44.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:44.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:44.124 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 07:42:44.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:44.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:44.124 | INFO | Uploaded 36GiB to GPU in 2.0582s (17.491GiB/s) | 20260507 07:42:44.124 | INFO | Downloaded 34GiB to GPU in 1.90584s (17.8399GiB/s) | 20260507 07:42:49.255 | INFO | Analyzing chunk frames = [8256000,8256512) | 20260507 07:42:49.255 | INFO | MemInfo: { | 20260507 07:42:49.255 | INFO | "gpu" : " 17 GB", | 20260507 07:42:49.255 | INFO | "huge" : "", | 20260507 07:42:49.255 | INFO | "huge_shared" : " 27 GB", | 20260507 07:42:49.255 | INFO | "malloc" : " 14 GB", | 20260507 07:42:49.255 | INFO | "pinned" : " 29 GB" | 20260507 07:42:49.255 | INFO | } | 20260507 07:42:49.255 | INFO | Perf report for the last 5.1314 seconds: | 20260507 07:42:49.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:42:49.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:49.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:49.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:49.255 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 07:42:49.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:49.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:49.255 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 07:42:49.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:49.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:49.255 | INFO | Uploaded 36GiB to GPU in 2.07562s (17.3442GiB/s) | 20260507 07:42:49.255 | INFO | Downloaded 34GiB to GPU in 1.94494s (17.4813GiB/s) | 20260507 07:42:54.397 | INFO | Analyzing chunk frames = [8256512,8257024) | 20260507 07:42:54.397 | INFO | MemInfo: { | 20260507 07:42:54.397 | INFO | "gpu" : " 17 GB", | 20260507 07:42:54.397 | INFO | "huge" : "", | 20260507 07:42:54.397 | INFO | "huge_shared" : " 27 GB", | 20260507 07:42:54.397 | INFO | "malloc" : " 14 GB", | 20260507 07:42:54.397 | INFO | "pinned" : " 29 GB" | 20260507 07:42:54.397 | INFO | } | 20260507 07:42:54.397 | INFO | Perf report for the last 5.1422 seconds: | 20260507 07:42:54.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:42:54.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:54.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:54.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:54.397 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 07:42:54.397 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:54.397 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:54.397 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:42:54.397 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:54.397 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:54.397 | INFO | Uploaded 36GiB to GPU in 1.96989s (18.2751GiB/s) | 20260507 07:42:54.397 | INFO | Downloaded 34GiB to GPU in 1.98103s (17.1627GiB/s) | 20260507 07:42:59.547 | INFO | Analyzing chunk frames = [8257024,8257536) | 20260507 07:42:59.547 | INFO | MemInfo: { | 20260507 07:42:59.547 | INFO | "gpu" : " 17 GB", | 20260507 07:42:59.547 | INFO | "huge" : "", | 20260507 07:42:59.547 | INFO | "huge_shared" : " 27 GB", | 20260507 07:42:59.547 | INFO | "malloc" : " 14 GB", | 20260507 07:42:59.547 | INFO | "pinned" : " 29 GB" | 20260507 07:42:59.547 | INFO | } | 20260507 07:42:59.547 | INFO | Perf report for the last 5.14995 seconds: | 20260507 07:42:59.547 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:42:59.547 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:59.547 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:59.547 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:59.547 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 07:42:59.547 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:59.547 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:59.547 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:42:59.547 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:59.547 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:42:59.547 | INFO | Uploaded 36GiB to GPU in 1.96672s (18.3046GiB/s) | 20260507 07:42:59.547 | INFO | Downloaded 34GiB to GPU in 1.96186s (17.3305GiB/s) | 20260507 07:42:59.547 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:42:59.547 | INFO | Total mem usage: 18 GiB | 20260507 07:42:59.547 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:42:59.547 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -27%) | 20260507 07:42:59.547 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:42:59.548 | INFO | High Water Breakdown: | 20260507 07:42:59.548 | INFO | 60.1% : 8.52 GiB ( 74.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:42:59.548 | INFO | 39.7% : 5.63 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:42:59.548 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:42:59.548 | INFO | ------------------------------------------ | 20260507 07:42:59.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:42:59.548 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:42:59.548 | INFO | Total mem usage: 26.5 GiB | 20260507 07:42:59.548 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:42:59.548 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:42:59.548 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 07:42:59.548 | INFO | High Water Breakdown: | 20260507 07:42:59.548 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:42:59.548 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:42:59.548 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:42:59.548 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:42:59.548 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:42:59.548 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:42:59.548 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:42:59.548 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:42:59.548 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:42:59.548 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:42:59.548 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:42:59.548 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:42:59.548 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:42:59.548 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:42:59.548 | INFO | ------------------------------------------ | 20260507 07:42:59.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:42:59.548 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:42:59.548 | INFO | Total mem usage: 25.6 GiB | 20260507 07:42:59.548 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:42:59.548 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:42:59.548 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:42:59.548 | INFO | High Water Breakdown: | 20260507 07:42:59.548 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:42:59.548 | INFO | ------------------------------------------ | 20260507 07:42:59.548 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:42:59.548 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:42:59.548 | INFO | Total mem usage: 15.7 GiB | 20260507 07:42:59.548 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:42:59.548 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:42:59.548 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:42:59.548 | INFO | High Water Breakdown: | 20260507 07:42:59.548 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:42:59.548 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:42:59.548 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:42:59.548 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:42:59.548 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:42:59.548 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:42:59.548 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:42:59.548 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:42:59.548 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:42:59.548 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:42:59.548 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:42:59.548 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:42:59.548 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:42:59.548 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:42:59.548 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:42:59.548 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:42:59.548 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:42:59.548 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:42:59.548 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:42:59.548 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:42:59.548 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:42:59.549 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:42:59.549 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:42:59.549 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:42:59.549 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:42:59.549 | INFO | ------------------------------------------ | 20260507 07:42:59.549 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:42:59.549 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:42:59.549 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:42:59.549 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:42:59.549 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.46e+01, 1.35e+01 / 1.28e+00 | 20260507 07:42:59.549 | INFO | PerfReport: 0.29%, AggregateStats -- 9.02e-02 / 1.03e-01, 9.64e-03 / 3.80e-03 | 20260507 07:42:59.549 | INFO | PerfReport: 11.67%, Baseline -- 3.57e+00 / 4.52e+00, 4.96e-01 / 1.44e-01 | 20260507 07:42:59.549 | INFO | PerfReport: 3.21%, Histogram -- 9.82e-01 / 1.51e+00, 1.62e-01 / 7.58e-02 | 20260507 07:42:59.549 | INFO | PerfReport: 16.33%, DME -- 4.00e+01 / 4.48e+01, 3.92e+00 / 1.23e+00 | 20260507 07:42:59.549 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.19e+00 / 1.07e+00 | 20260507 07:42:59.549 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.12e+00 / 2.72e+00, 3.89e-01 / 1.27e-01 | 20260507 07:42:59.549 | INFO | PerfReport: 2.93%, Metrics -- 8.97e-01 / 1.21e+00, 2.41e-01 / 1.18e-01 | 20260507 07:42:59.549 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.01e-01 / 3.62e-01, 4.97e-02 / 2.01e-02 | 20260507 07:42:59.549 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:42:59.549 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:42:59.549 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:42:59.549 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 07:42:59.549 | INFO | PerfReport: 21.99%, Upload -- 2.12e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 07:42:59.549 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 07:42:59.549 | INFO | PerfReport: 31.84%, Compute -- 3.06e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 07:42:59.549 | INFO | PerfReport: 1.58%, LockUpload -- 1.52e+00 / 0.00e+00, 4.94e+00 / 0.00e+00 | 20260507 07:42:59.549 | INFO | PerfReport: 1.29%, LockDownload -- 1.25e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260507 07:42:59.549 | INFO | PerfReport: 21.88%, LockCompute -- 2.11e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 07:42:59.549 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.31e-02 / 0.00e+00 | 20260507 07:42:59.644 | INFO | Starting a round of Baz IO | 20260507 07:43:02.722 | INFO | Populating RTMetrics report | 20260507 07:43:02.723 | INFO | RTMetrics report populated | 20260507 07:43:02.723 | INFO | Opening tmp RTMetrics json file | 20260507 07:43:02.740 | INFO | Serializing RTMetrics to json | 20260507 07:43:02.742 | INFO | Serialization done, swapping tmp file now... | 20260507 07:43:02.743 | INFO | RTMetrics done | 20260507 07:43:02.808 | INFO | Finished a round of Baz IO: Wrote 1.633 GiB in 3.163 seconds | 20260507 07:43:02.808 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.54% / 0.00% / 67.46% | 20260507 07:43:02.808 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.100426% / 0% / 7.54992e-05% | 20260507 07:43:02.808 | INFO | Baz IO overall: | 20260507 07:43:02.808 | INFO | BytesWritten : 4132139523484 | 20260507 07:43:02.808 | INFO | NumEvents : 1977880857777 | 20260507 07:43:04.684 | INFO | Analyzing chunk frames = [8257536,8258048) | 20260507 07:43:04.684 | INFO | MemInfo: { | 20260507 07:43:04.684 | INFO | "gpu" : " 17 GB", | 20260507 07:43:04.684 | INFO | "huge" : "", | 20260507 07:43:04.684 | INFO | "huge_shared" : " 27 GB", | 20260507 07:43:04.684 | INFO | "malloc" : " 14 GB", | 20260507 07:43:04.684 | INFO | "pinned" : " 29 GB" | 20260507 07:43:04.684 | INFO | } | 20260507 07:43:04.684 | INFO | Perf report for the last 5.13679 seconds: | 20260507 07:43:04.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:43:04.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:04.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:04.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:04.684 | INFO | ANALYSIS : 61.3%, 3.0, 95.3ms, 0 | 20260507 07:43:04.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:04.684 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260507 07:43:04.684 | INFO | PRE_HQ : 58.7%, 1.8, 54.2ms, 0 | 20260507 07:43:04.684 | INFO | BAZWRITER : 61.2%, 3.1, 97.4ms, 0 | 20260507 07:43:04.684 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:43:04.684 | INFO | Uploaded 36GiB to GPU in 2.05289s (17.5362GiB/s) | 20260507 07:43:04.684 | INFO | Downloaded 38GiB to GPU in 2.16516s (17.5507GiB/s) | 20260507 07:43:09.815 | INFO | Analyzing chunk frames = [8258048,8258560) | 20260507 07:43:09.815 | INFO | MemInfo: { | 20260507 07:43:09.815 | INFO | "gpu" : " 17 GB", | 20260507 07:43:09.815 | INFO | "huge" : "", | 20260507 07:43:09.815 | INFO | "huge_shared" : " 27 GB", | 20260507 07:43:09.815 | INFO | "malloc" : " 14 GB", | 20260507 07:43:09.815 | INFO | "pinned" : " 29 GB" | 20260507 07:43:09.815 | INFO | } | 20260507 07:43:09.816 | INFO | Perf report for the last 5.13169 seconds: | 20260507 07:43:09.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:43:09.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:09.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:09.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:09.816 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260507 07:43:09.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:09.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:09.816 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 07:43:09.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:09.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:09.816 | INFO | Uploaded 36GiB to GPU in 2.01287s (17.8849GiB/s) | 20260507 07:43:09.816 | INFO | Downloaded 34GiB to GPU in 1.94498s (17.4809GiB/s) | 20260507 07:43:14.953 | INFO | Analyzing chunk frames = [8258560,8259072) | 20260507 07:43:14.953 | INFO | MemInfo: { | 20260507 07:43:14.953 | INFO | "gpu" : " 17 GB", | 20260507 07:43:14.953 | INFO | "huge" : "", | 20260507 07:43:14.953 | INFO | "huge_shared" : " 27 GB", | 20260507 07:43:14.953 | INFO | "malloc" : " 14 GB", | 20260507 07:43:14.954 | INFO | "pinned" : " 29 GB" | 20260507 07:43:14.954 | INFO | } | 20260507 07:43:14.954 | INFO | Perf report for the last 5.13814 seconds: | 20260507 07:43:14.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:43:14.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:14.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:14.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:14.954 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 07:43:14.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:14.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:14.954 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:43:14.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:14.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:14.954 | INFO | Uploaded 36GiB to GPU in 2.01622s (17.8552GiB/s) | 20260507 07:43:14.954 | INFO | Downloaded 34GiB to GPU in 1.91876s (17.7198GiB/s) | 20260507 07:43:20.098 | INFO | Analyzing chunk frames = [8259072,8259584) | 20260507 07:43:20.098 | INFO | MemInfo: { | 20260507 07:43:20.098 | INFO | "gpu" : " 17 GB", | 20260507 07:43:20.098 | INFO | "huge" : "", | 20260507 07:43:20.098 | INFO | "huge_shared" : " 27 GB", | 20260507 07:43:20.098 | INFO | "malloc" : " 14 GB", | 20260507 07:43:20.098 | INFO | "pinned" : " 29 GB" | 20260507 07:43:20.098 | INFO | } | 20260507 07:43:20.098 | INFO | Perf report for the last 5.14404 seconds: | 20260507 07:43:20.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:43:20.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:20.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:20.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:20.098 | INFO | ANALYSIS : 60.2%, 2.9, 93.1ms, 0 | 20260507 07:43:20.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:20.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:20.098 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:43:20.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:20.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:20.098 | INFO | Uploaded 36GiB to GPU in 1.98905s (18.0991GiB/s) | 20260507 07:43:20.098 | INFO | Downloaded 34GiB to GPU in 1.94938s (17.4414GiB/s) | 20260507 07:43:25.232 | INFO | Analyzing chunk frames = [8259584,8260096) | 20260507 07:43:25.232 | INFO | MemInfo: { | 20260507 07:43:25.232 | INFO | "gpu" : " 17 GB", | 20260507 07:43:25.232 | INFO | "huge" : "", | 20260507 07:43:25.232 | INFO | "huge_shared" : " 27 GB", | 20260507 07:43:25.232 | INFO | "malloc" : " 14 GB", | 20260507 07:43:25.232 | INFO | "pinned" : " 29 GB" | 20260507 07:43:25.232 | INFO | } | 20260507 07:43:25.232 | INFO | Perf report for the last 5.13463 seconds: | 20260507 07:43:25.232 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:43:25.232 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:25.232 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:25.232 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:25.232 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 07:43:25.232 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:25.233 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:25.233 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:43:25.233 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:25.233 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:25.233 | INFO | Uploaded 36GiB to GPU in 2.05067s (17.5553GiB/s) | 20260507 07:43:25.233 | INFO | Downloaded 34GiB to GPU in 1.95233s (17.4151GiB/s) | 20260507 07:43:30.372 | INFO | Analyzing chunk frames = [8260096,8260608) | 20260507 07:43:30.372 | INFO | MemInfo: { | 20260507 07:43:30.372 | INFO | "gpu" : " 17 GB", | 20260507 07:43:30.372 | INFO | "huge" : "", | 20260507 07:43:30.372 | INFO | "huge_shared" : " 27 GB", | 20260507 07:43:30.372 | INFO | "malloc" : " 14 GB", | 20260507 07:43:30.372 | INFO | "pinned" : " 29 GB" | 20260507 07:43:30.372 | INFO | } | 20260507 07:43:30.373 | INFO | Perf report for the last 5.14021 seconds: | 20260507 07:43:30.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:43:30.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:30.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:30.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:30.373 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 07:43:30.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:30.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:30.373 | INFO | PRE_HQ : 27.4%, 1.0, 14.2ms, 0 | 20260507 07:43:30.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:30.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:30.373 | INFO | Uploaded 36GiB to GPU in 2.0361s (17.6808GiB/s) | 20260507 07:43:30.373 | INFO | Downloaded 34GiB to GPU in 1.91693s (17.7367GiB/s) | 20260507 07:43:35.514 | INFO | Analyzing chunk frames = [8260608,8261120) | 20260507 07:43:35.514 | INFO | MemInfo: { | 20260507 07:43:35.514 | INFO | "gpu" : " 17 GB", | 20260507 07:43:35.514 | INFO | "huge" : "", | 20260507 07:43:35.514 | INFO | "huge_shared" : " 27 GB", | 20260507 07:43:35.514 | INFO | "malloc" : " 14 GB", | 20260507 07:43:35.514 | INFO | "pinned" : " 29 GB" | 20260507 07:43:35.514 | INFO | } | 20260507 07:43:35.514 | INFO | Perf report for the last 5.14186 seconds: | 20260507 07:43:35.514 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:43:35.514 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:35.514 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:35.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:35.515 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 07:43:35.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:35.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:35.515 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:43:35.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:35.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:35.515 | INFO | Uploaded 36GiB to GPU in 2.03779s (17.6662GiB/s) | 20260507 07:43:35.515 | INFO | Downloaded 34GiB to GPU in 1.97788s (17.1901GiB/s) | 20260507 07:43:40.652 | INFO | Analyzing chunk frames = [8261120,8261632) | 20260507 07:43:40.652 | INFO | MemInfo: { | 20260507 07:43:40.652 | INFO | "gpu" : " 17 GB", | 20260507 07:43:40.652 | INFO | "huge" : "", | 20260507 07:43:40.652 | INFO | "huge_shared" : " 27 GB", | 20260507 07:43:40.652 | INFO | "malloc" : " 14 GB", | 20260507 07:43:40.652 | INFO | "pinned" : " 29 GB" | 20260507 07:43:40.652 | INFO | } | 20260507 07:43:40.652 | INFO | Perf report for the last 5.13795 seconds: | 20260507 07:43:40.652 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:43:40.652 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:40.652 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:40.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:40.652 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:43:40.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:40.652 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:40.652 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 07:43:40.653 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:40.653 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:40.653 | INFO | Uploaded 36GiB to GPU in 1.98294s (18.1549GiB/s) | 20260507 07:43:40.653 | INFO | Downloaded 34GiB to GPU in 1.96268s (17.3233GiB/s) | 20260507 07:43:40.749 | INFO | Starting a round of Baz IO | 20260507 07:43:43.843 | INFO | Populating RTMetrics report | 20260507 07:43:43.843 | INFO | RTMetrics report populated | 20260507 07:43:43.843 | INFO | Opening tmp RTMetrics json file | 20260507 07:43:43.850 | INFO | Serializing RTMetrics to json | 20260507 07:43:43.860 | INFO | Serialization done, swapping tmp file now... | 20260507 07:43:43.861 | INFO | RTMetrics done | 20260507 07:43:43.937 | INFO | Finished a round of Baz IO: Wrote 1.636 GiB in 3.188 seconds | 20260507 07:43:43.938 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.54% / 0.00% / 67.46% | 20260507 07:43:43.938 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.100791% / 0% / 5.96046e-05% | 20260507 07:43:43.938 | INFO | Baz IO overall: | 20260507 07:43:43.938 | INFO | BytesWritten : 4133895705394 | 20260507 07:43:43.938 | INFO | NumEvents : 1978700693673 | 20260507 07:43:45.793 | INFO | Analyzing chunk frames = [8261632,8262144) | 20260507 07:43:45.793 | INFO | MemInfo: { | 20260507 07:43:45.793 | INFO | "gpu" : " 17 GB", | 20260507 07:43:45.793 | INFO | "huge" : "", | 20260507 07:43:45.793 | INFO | "huge_shared" : " 27 GB", | 20260507 07:43:45.793 | INFO | "malloc" : " 14 GB", | 20260507 07:43:45.793 | INFO | "pinned" : " 29 GB" | 20260507 07:43:45.793 | INFO | } | 20260507 07:43:45.793 | INFO | Perf report for the last 5.14041 seconds: | 20260507 07:43:45.793 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:43:45.793 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:45.793 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:45.793 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:45.793 | INFO | ANALYSIS : 61.5%, 3.0, 95.4ms, 0 | 20260507 07:43:45.793 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:45.793 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:43:45.793 | INFO | PRE_HQ : 58.8%, 1.8, 54.4ms, 0 | 20260507 07:43:45.793 | INFO | BAZWRITER : 61.6%, 3.5, 109.4ms, 0 | 20260507 07:43:45.793 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:43:45.793 | INFO | Uploaded 36GiB to GPU in 2.04418s (17.611GiB/s) | 20260507 07:43:45.794 | INFO | Downloaded 38GiB to GPU in 2.18361s (17.4024GiB/s) | 20260507 07:43:50.932 | INFO | Analyzing chunk frames = [8262144,8262656) | 20260507 07:43:50.932 | INFO | MemInfo: { | 20260507 07:43:50.932 | INFO | "gpu" : " 17 GB", | 20260507 07:43:50.932 | INFO | "huge" : "", | 20260507 07:43:50.932 | INFO | "huge_shared" : " 27 GB", | 20260507 07:43:50.932 | INFO | "malloc" : " 14 GB", | 20260507 07:43:50.932 | INFO | "pinned" : " 29 GB" | 20260507 07:43:50.932 | INFO | } | 20260507 07:43:50.932 | INFO | Perf report for the last 5.13906 seconds: | 20260507 07:43:50.932 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:43:50.932 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:50.932 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:50.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:50.932 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:43:50.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:50.932 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:50.932 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:43:50.932 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:50.932 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:50.932 | INFO | Uploaded 36GiB to GPU in 1.96919s (18.2817GiB/s) | 20260507 07:43:50.932 | INFO | Downloaded 34GiB to GPU in 1.92694s (17.6445GiB/s) | 20260507 07:43:56.072 | INFO | Analyzing chunk frames = [8262656,8263168) | 20260507 07:43:56.072 | INFO | MemInfo: { | 20260507 07:43:56.072 | INFO | "gpu" : " 17 GB", | 20260507 07:43:56.072 | INFO | "huge" : "", | 20260507 07:43:56.072 | INFO | "huge_shared" : " 27 GB", | 20260507 07:43:56.072 | INFO | "malloc" : " 14 GB", | 20260507 07:43:56.072 | INFO | "pinned" : " 29 GB" | 20260507 07:43:56.072 | INFO | } | 20260507 07:43:56.072 | INFO | Perf report for the last 5.13997 seconds: | 20260507 07:43:56.072 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:43:56.072 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:56.072 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:56.072 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:56.072 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 07:43:56.072 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:56.072 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:56.072 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:43:56.072 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:56.072 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:43:56.072 | INFO | Uploaded 36GiB to GPU in 2.05274s (17.5375GiB/s) | 20260507 07:43:56.072 | INFO | Downloaded 34GiB to GPU in 1.95243s (17.4142GiB/s) | 20260507 07:44:01.207 | INFO | Analyzing chunk frames = [8263168,8263680) | 20260507 07:44:01.207 | INFO | MemInfo: { | 20260507 07:44:01.207 | INFO | "gpu" : " 17 GB", | 20260507 07:44:01.207 | INFO | "huge" : "", | 20260507 07:44:01.207 | INFO | "huge_shared" : " 27 GB", | 20260507 07:44:01.207 | INFO | "malloc" : " 14 GB", | 20260507 07:44:01.207 | INFO | "pinned" : " 29 GB" | 20260507 07:44:01.207 | INFO | } | 20260507 07:44:01.207 | INFO | Perf report for the last 5.13525 seconds: | 20260507 07:44:01.207 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:44:01.207 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:01.207 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:01.207 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:01.207 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 07:44:01.207 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:01.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:01.207 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:44:01.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:01.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:01.208 | INFO | Uploaded 36GiB to GPU in 2.00536s (17.9519GiB/s) | 20260507 07:44:01.208 | INFO | Downloaded 34GiB to GPU in 1.94364s (17.493GiB/s) | 20260507 07:44:06.350 | INFO | Analyzing chunk frames = [8263680,8264192) | 20260507 07:44:06.350 | INFO | MemInfo: { | 20260507 07:44:06.350 | INFO | "gpu" : " 17 GB", | 20260507 07:44:06.350 | INFO | "huge" : "", | 20260507 07:44:06.350 | INFO | "huge_shared" : " 27 GB", | 20260507 07:44:06.350 | INFO | "malloc" : " 14 GB", | 20260507 07:44:06.350 | INFO | "pinned" : " 29 GB" | 20260507 07:44:06.350 | INFO | } | 20260507 07:44:06.350 | INFO | Perf report for the last 5.14256 seconds: | 20260507 07:44:06.350 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:44:06.350 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:06.350 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:06.350 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:06.350 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 07:44:06.350 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:06.350 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:06.350 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:44:06.350 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:06.350 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:06.350 | INFO | Uploaded 36GiB to GPU in 2.05116s (17.551GiB/s) | 20260507 07:44:06.350 | INFO | Downloaded 34GiB to GPU in 1.95034s (17.4328GiB/s) | 20260507 07:44:11.499 | INFO | Analyzing chunk frames = [8264192,8264704) | 20260507 07:44:11.499 | INFO | MemInfo: { | 20260507 07:44:11.499 | INFO | "gpu" : " 17 GB", | 20260507 07:44:11.499 | INFO | "huge" : "", | 20260507 07:44:11.499 | INFO | "huge_shared" : " 27 GB", | 20260507 07:44:11.499 | INFO | "malloc" : " 14 GB", | 20260507 07:44:11.499 | INFO | "pinned" : " 29 GB" | 20260507 07:44:11.499 | INFO | } | 20260507 07:44:11.499 | INFO | Perf report for the last 5.14919 seconds: | 20260507 07:44:11.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:44:11.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:11.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:11.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:11.499 | INFO | ANALYSIS : 60.2%, 3.0, 94.2ms, 0 | 20260507 07:44:11.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:11.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:11.499 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:44:11.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:11.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:11.499 | INFO | Uploaded 36GiB to GPU in 2.04484s (17.6053GiB/s) | 20260507 07:44:11.499 | INFO | Downloaded 34GiB to GPU in 1.9133s (17.7704GiB/s) | 20260507 07:44:16.630 | INFO | Analyzing chunk frames = [8264704,8265216) | 20260507 07:44:16.630 | INFO | MemInfo: { | 20260507 07:44:16.630 | INFO | "gpu" : " 17 GB", | 20260507 07:44:16.630 | INFO | "huge" : "", | 20260507 07:44:16.630 | INFO | "huge_shared" : " 27 GB", | 20260507 07:44:16.630 | INFO | "malloc" : " 14 GB", | 20260507 07:44:16.631 | INFO | "pinned" : " 29 GB" | 20260507 07:44:16.631 | INFO | } | 20260507 07:44:16.631 | INFO | Perf report for the last 5.13166 seconds: | 20260507 07:44:16.631 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:44:16.631 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:16.631 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:16.631 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:16.631 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 07:44:16.631 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:16.631 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:16.631 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 07:44:16.631 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:16.631 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:16.631 | INFO | Uploaded 36GiB to GPU in 1.97025s (18.2718GiB/s) | 20260507 07:44:16.631 | INFO | Downloaded 34GiB to GPU in 1.97274s (17.235GiB/s) | 20260507 07:44:21.765 | INFO | Analyzing chunk frames = [8265216,8265728) | 20260507 07:44:21.765 | INFO | MemInfo: { | 20260507 07:44:21.765 | INFO | "gpu" : " 17 GB", | 20260507 07:44:21.765 | INFO | "huge" : "", | 20260507 07:44:21.765 | INFO | "huge_shared" : " 27 GB", | 20260507 07:44:21.765 | INFO | "malloc" : " 14 GB", | 20260507 07:44:21.765 | INFO | "pinned" : " 29 GB" | 20260507 07:44:21.765 | INFO | } | 20260507 07:44:21.765 | INFO | Perf report for the last 5.13448 seconds: | 20260507 07:44:21.765 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:44:21.765 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:21.765 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:21.765 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:21.765 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:44:21.765 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:21.765 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:21.765 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:44:21.765 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:21.765 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:21.765 | INFO | Uploaded 36GiB to GPU in 1.97906s (18.1905GiB/s) | 20260507 07:44:21.765 | INFO | Downloaded 34GiB to GPU in 1.96243s (17.3255GiB/s) | 20260507 07:44:21.765 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:44:21.765 | INFO | Total mem usage: 18 GiB | 20260507 07:44:21.765 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:44:21.765 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -21.1%) | 20260507 07:44:21.765 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:44:21.765 | INFO | High Water Breakdown: | 20260507 07:44:21.766 | INFO | 60.8% : 9.05 GiB ( 79.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:44:21.766 | INFO | 39% : 5.8 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:44:21.766 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:44:21.766 | INFO | ------------------------------------------ | 20260507 07:44:21.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:44:21.766 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:44:21.766 | INFO | Total mem usage: 26.5 GiB | 20260507 07:44:21.766 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:44:21.766 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:44:21.766 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 07:44:21.766 | INFO | High Water Breakdown: | 20260507 07:44:21.766 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:44:21.766 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:44:21.766 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:44:21.766 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:44:21.766 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:44:21.766 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:44:21.766 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:44:21.766 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:44:21.766 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:44:21.766 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:44:21.766 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:44:21.766 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:44:21.766 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:44:21.766 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:44:21.766 | INFO | ------------------------------------------ | 20260507 07:44:21.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:44:21.766 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:44:21.766 | INFO | Total mem usage: 25.6 GiB | 20260507 07:44:21.766 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:44:21.766 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:44:21.766 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:44:21.766 | INFO | High Water Breakdown: | 20260507 07:44:21.766 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:44:21.766 | INFO | ------------------------------------------ | 20260507 07:44:21.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:44:21.766 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:44:21.766 | INFO | Total mem usage: 15.7 GiB | 20260507 07:44:21.766 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:44:21.766 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:44:21.766 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:44:21.766 | INFO | High Water Breakdown: | 20260507 07:44:21.766 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:44:21.766 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:44:21.766 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:44:21.766 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:44:21.766 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:44:21.766 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:44:21.766 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:44:21.766 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:44:21.766 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:44:21.766 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:44:21.766 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:44:21.766 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:44:21.766 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:44:21.766 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:44:21.766 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:44:21.766 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:44:21.766 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:44:21.766 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:44:21.766 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:44:21.766 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:44:21.766 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:44:21.766 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:44:21.766 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:44:21.766 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:44:21.766 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:44:21.766 | INFO | ------------------------------------------ | 20260507 07:44:21.766 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:44:21.766 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:44:21.766 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:44:21.766 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:44:21.767 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 1.02e+00 | 20260507 07:44:21.767 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.01e-01, 1.06e-02 / 4.80e-03 | 20260507 07:44:21.767 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.49e+00, 5.03e-01 / 1.56e-01 | 20260507 07:44:21.767 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.49e+00, 1.66e-01 / 9.86e-02 | 20260507 07:44:21.767 | INFO | PerfReport: 16.30%, DME -- 4.00e+01 / 4.48e+01, 4.02e+00 / 1.29e+00 | 20260507 07:44:21.767 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.29e+00 / 9.92e-01 | 20260507 07:44:21.767 | INFO | PerfReport: 6.89%, PulseAccumulating -- 2.12e+00 / 2.64e+00, 3.82e-01 / 1.72e-01 | 20260507 07:44:21.767 | INFO | PerfReport: 2.93%, Metrics -- 8.98e-01 / 1.21e+00, 2.40e-01 / 1.27e-01 | 20260507 07:44:21.767 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.02e-01 / 3.66e-01, 5.13e-02 / 2.68e-02 | 20260507 07:44:21.767 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:44:21.767 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:44:21.767 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:44:21.767 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 07:44:21.767 | INFO | PerfReport: 21.88%, Upload -- 2.10e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260507 07:44:21.767 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 07:44:21.767 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 07:44:21.767 | INFO | PerfReport: 1.49%, LockUpload -- 1.44e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 07:44:21.767 | INFO | PerfReport: 1.35%, LockDownload -- 1.29e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260507 07:44:21.767 | INFO | PerfReport: 21.92%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 07:44:21.767 | INFO | PerfReport: 0.02%, Misc -- 1.45e-02 / 0.00e+00, 2.52e-02 / 0.00e+00 | 20260507 07:44:21.862 | INFO | Starting a round of Baz IO | 20260507 07:44:24.955 | INFO | Populating RTMetrics report | 20260507 07:44:24.955 | INFO | RTMetrics report populated | 20260507 07:44:24.955 | INFO | Opening tmp RTMetrics json file | 20260507 07:44:24.972 | INFO | Serializing RTMetrics to json | 20260507 07:44:24.974 | INFO | Serialization done, swapping tmp file now... | 20260507 07:44:24.975 | INFO | RTMetrics done | 20260507 07:44:25.042 | INFO | Finished a round of Baz IO: Wrote 1.633 GiB in 3.179 seconds | 20260507 07:44:25.042 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.54% / 0.00% / 67.46% | 20260507 07:44:25.042 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.100247% / 0% / 9.53674e-05% | 20260507 07:44:25.042 | INFO | Baz IO overall: | 20260507 07:44:25.042 | INFO | BytesWritten : 4135648938569 | 20260507 07:44:25.042 | INFO | NumEvents : 1979518603631 | 20260507 07:44:26.906 | INFO | Analyzing chunk frames = [8265728,8266240) | 20260507 07:44:26.906 | INFO | MemInfo: { | 20260507 07:44:26.906 | INFO | "gpu" : " 17 GB", | 20260507 07:44:26.906 | INFO | "huge" : "", | 20260507 07:44:26.906 | INFO | "huge_shared" : " 27 GB", | 20260507 07:44:26.906 | INFO | "malloc" : " 14 GB", | 20260507 07:44:26.906 | INFO | "pinned" : " 29 GB" | 20260507 07:44:26.906 | INFO | } | 20260507 07:44:26.906 | INFO | Perf report for the last 5.14096 seconds: | 20260507 07:44:26.906 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:44:26.906 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:26.906 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:26.906 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:26.906 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 07:44:26.906 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:26.906 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 07:44:26.906 | INFO | PRE_HQ : 58.8%, 1.8, 54.2ms, 0 | 20260507 07:44:26.906 | INFO | BAZWRITER : 61.4%, 3.1, 97.2ms, 0 | 20260507 07:44:26.906 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 07:44:26.906 | INFO | Uploaded 36GiB to GPU in 2.04555s (17.5992GiB/s) | 20260507 07:44:26.906 | INFO | Downloaded 38GiB to GPU in 2.17949s (17.4353GiB/s) | 20260507 07:44:32.046 | INFO | Analyzing chunk frames = [8266240,8266752) | 20260507 07:44:32.046 | INFO | MemInfo: { | 20260507 07:44:32.046 | INFO | "gpu" : " 17 GB", | 20260507 07:44:32.046 | INFO | "huge" : "", | 20260507 07:44:32.046 | INFO | "huge_shared" : " 27 GB", | 20260507 07:44:32.046 | INFO | "malloc" : " 14 GB", | 20260507 07:44:32.046 | INFO | "pinned" : " 29 GB" | 20260507 07:44:32.046 | INFO | } | 20260507 07:44:32.046 | INFO | Perf report for the last 5.13983 seconds: | 20260507 07:44:32.046 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:44:32.046 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:32.046 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:32.046 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:32.046 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:44:32.046 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:32.046 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:32.046 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:44:32.046 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:32.046 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:32.046 | INFO | Uploaded 36GiB to GPU in 1.98119s (18.1709GiB/s) | 20260507 07:44:32.046 | INFO | Downloaded 34GiB to GPU in 1.93262s (17.5927GiB/s) | 20260507 07:44:37.189 | INFO | Analyzing chunk frames = [8266752,8267264) | 20260507 07:44:37.189 | INFO | MemInfo: { | 20260507 07:44:37.189 | INFO | "gpu" : " 17 GB", | 20260507 07:44:37.189 | INFO | "huge" : "", | 20260507 07:44:37.189 | INFO | "huge_shared" : " 27 GB", | 20260507 07:44:37.189 | INFO | "malloc" : " 14 GB", | 20260507 07:44:37.189 | INFO | "pinned" : " 29 GB" | 20260507 07:44:37.189 | INFO | } | 20260507 07:44:37.189 | INFO | Perf report for the last 5.14278 seconds: | 20260507 07:44:37.189 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:44:37.189 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:37.189 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:37.189 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:37.189 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:44:37.189 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:37.189 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:37.189 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:44:37.189 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:37.189 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:37.189 | INFO | Uploaded 36GiB to GPU in 2.04493s (17.6045GiB/s) | 20260507 07:44:37.189 | INFO | Downloaded 34GiB to GPU in 1.95576s (17.3845GiB/s) | 20260507 07:44:42.324 | INFO | Analyzing chunk frames = [8267264,8267776) | 20260507 07:44:42.324 | INFO | MemInfo: { | 20260507 07:44:42.324 | INFO | "gpu" : " 17 GB", | 20260507 07:44:42.324 | INFO | "huge" : "", | 20260507 07:44:42.324 | INFO | "huge_shared" : " 27 GB", | 20260507 07:44:42.324 | INFO | "malloc" : " 14 GB", | 20260507 07:44:42.324 | INFO | "pinned" : " 29 GB" | 20260507 07:44:42.324 | INFO | } | 20260507 07:44:42.324 | INFO | Perf report for the last 5.13576 seconds: | 20260507 07:44:42.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:44:42.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:42.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:42.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:42.325 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 07:44:42.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:42.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:42.325 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:44:42.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:42.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:42.325 | INFO | Uploaded 36GiB to GPU in 2.04825s (17.576GiB/s) | 20260507 07:44:42.325 | INFO | Downloaded 34GiB to GPU in 1.94996s (17.4363GiB/s) | 20260507 07:44:47.461 | INFO | Analyzing chunk frames = [8267776,8268288) | 20260507 07:44:47.461 | INFO | MemInfo: { | 20260507 07:44:47.461 | INFO | "gpu" : " 17 GB", | 20260507 07:44:47.461 | INFO | "huge" : "", | 20260507 07:44:47.461 | INFO | "huge_shared" : " 27 GB", | 20260507 07:44:47.461 | INFO | "malloc" : " 14 GB", | 20260507 07:44:47.462 | INFO | "pinned" : " 29 GB" | 20260507 07:44:47.462 | INFO | } | 20260507 07:44:47.462 | INFO | Perf report for the last 5.13729 seconds: | 20260507 07:44:47.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:44:47.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:47.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:47.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:47.462 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 07:44:47.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:47.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:47.462 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:44:47.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:47.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:47.462 | INFO | Uploaded 36GiB to GPU in 2.02863s (17.7459GiB/s) | 20260507 07:44:47.462 | INFO | Downloaded 34GiB to GPU in 1.92229s (17.6872GiB/s) | 20260507 07:44:52.606 | INFO | Analyzing chunk frames = [8268288,8268800) | 20260507 07:44:52.606 | INFO | MemInfo: { | 20260507 07:44:52.606 | INFO | "gpu" : " 17 GB", | 20260507 07:44:52.606 | INFO | "huge" : "", | 20260507 07:44:52.606 | INFO | "huge_shared" : " 27 GB", | 20260507 07:44:52.606 | INFO | "malloc" : " 14 GB", | 20260507 07:44:52.606 | INFO | "pinned" : " 29 GB" | 20260507 07:44:52.606 | INFO | } | 20260507 07:44:52.606 | INFO | Perf report for the last 5.14444 seconds: | 20260507 07:44:52.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:44:52.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:52.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:52.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:52.606 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 07:44:52.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:52.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:52.606 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 07:44:52.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:52.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:52.606 | INFO | Uploaded 36GiB to GPU in 2.03537s (17.6872GiB/s) | 20260507 07:44:52.607 | INFO | Downloaded 34GiB to GPU in 1.91694s (17.7366GiB/s) | 20260507 07:44:57.748 | INFO | Analyzing chunk frames = [8268800,8269312) | 20260507 07:44:57.748 | INFO | MemInfo: { | 20260507 07:44:57.748 | INFO | "gpu" : " 17 GB", | 20260507 07:44:57.748 | INFO | "huge" : "", | 20260507 07:44:57.748 | INFO | "huge_shared" : " 27 GB", | 20260507 07:44:57.748 | INFO | "malloc" : " 14 GB", | 20260507 07:44:57.748 | INFO | "pinned" : " 29 GB" | 20260507 07:44:57.748 | INFO | } | 20260507 07:44:57.749 | INFO | Perf report for the last 5.14229 seconds: | 20260507 07:44:57.749 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:44:57.749 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:57.749 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:57.749 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:57.749 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:44:57.749 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:57.749 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:57.749 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 07:44:57.749 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:57.749 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:44:57.749 | INFO | Uploaded 36GiB to GPU in 1.91888s (18.7609GiB/s) | 20260507 07:44:57.749 | INFO | Downloaded 34GiB to GPU in 1.9728s (17.2344GiB/s) | 20260507 07:45:02.880 | INFO | Analyzing chunk frames = [8269312,8269824) | 20260507 07:45:02.880 | INFO | MemInfo: { | 20260507 07:45:02.880 | INFO | "gpu" : " 17 GB", | 20260507 07:45:02.880 | INFO | "huge" : "", | 20260507 07:45:02.880 | INFO | "huge_shared" : " 27 GB", | 20260507 07:45:02.880 | INFO | "malloc" : " 14 GB", | 20260507 07:45:02.880 | INFO | "pinned" : " 29 GB" | 20260507 07:45:02.880 | INFO | } | 20260507 07:45:02.880 | INFO | Perf report for the last 5.13182 seconds: | 20260507 07:45:02.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:45:02.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:02.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:02.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:02.880 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:45:02.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:02.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:02.880 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:45:02.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:02.881 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:02.881 | INFO | Uploaded 36GiB to GPU in 1.99209s (18.0715GiB/s) | 20260507 07:45:02.881 | INFO | Downloaded 34GiB to GPU in 1.95754s (17.3688GiB/s) | 20260507 07:45:02.977 | INFO | Starting a round of Baz IO | 20260507 07:45:06.071 | INFO | Populating RTMetrics report | 20260507 07:45:06.071 | INFO | RTMetrics report populated | 20260507 07:45:06.071 | INFO | Opening tmp RTMetrics json file | 20260507 07:45:06.082 | INFO | Serializing RTMetrics to json | 20260507 07:45:06.086 | INFO | Serialization done, swapping tmp file now... | 20260507 07:45:06.086 | INFO | RTMetrics done | 20260507 07:45:06.158 | INFO | Finished a round of Baz IO: Wrote 1.632 GiB in 3.181 seconds | 20260507 07:45:06.158 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.54% / 0.00% / 67.46% | 20260507 07:45:06.158 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0998298% / 0% / 7.94729e-05% | 20260507 07:45:06.158 | INFO | Baz IO overall: | 20260507 07:45:06.158 | INFO | BytesWritten : 4137401405540 | 20260507 07:45:06.158 | INFO | NumEvents : 1980336388134 | 20260507 07:45:08.018 | INFO | Analyzing chunk frames = [8269824,8270336) | 20260507 07:45:08.018 | INFO | MemInfo: { | 20260507 07:45:08.018 | INFO | "gpu" : " 17 GB", | 20260507 07:45:08.018 | INFO | "huge" : "", | 20260507 07:45:08.018 | INFO | "huge_shared" : " 27 GB", | 20260507 07:45:08.018 | INFO | "malloc" : " 14 GB", | 20260507 07:45:08.018 | INFO | "pinned" : " 29 GB" | 20260507 07:45:08.018 | INFO | } | 20260507 07:45:08.018 | INFO | Perf report for the last 5.13753 seconds: | 20260507 07:45:08.018 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:45:08.018 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:08.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:08.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:08.018 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 07:45:08.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:08.018 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 07:45:08.018 | INFO | PRE_HQ : 58.9%, 1.8, 54.3ms, 0 | 20260507 07:45:08.018 | INFO | BAZWRITER : 61.5%, 3.1, 97.5ms, 0 | 20260507 07:45:08.018 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:45:08.018 | INFO | Uploaded 36GiB to GPU in 2.05674s (17.5034GiB/s) | 20260507 07:45:08.018 | INFO | Downloaded 38GiB to GPU in 2.17813s (17.4461GiB/s) | 20260507 07:45:13.165 | INFO | Analyzing chunk frames = [8270336,8270848) | 20260507 07:45:13.165 | INFO | MemInfo: { | 20260507 07:45:13.165 | INFO | "gpu" : " 17 GB", | 20260507 07:45:13.165 | INFO | "huge" : "", | 20260507 07:45:13.165 | INFO | "huge_shared" : " 27 GB", | 20260507 07:45:13.165 | INFO | "malloc" : " 14 GB", | 20260507 07:45:13.165 | INFO | "pinned" : " 29 GB" | 20260507 07:45:13.165 | INFO | } | 20260507 07:45:13.165 | INFO | Perf report for the last 5.14725 seconds: | 20260507 07:45:13.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:45:13.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:13.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:13.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:13.165 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 07:45:13.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:13.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:13.165 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 07:45:13.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:13.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:13.165 | INFO | Uploaded 36GiB to GPU in 2.00331s (17.9702GiB/s) | 20260507 07:45:13.166 | INFO | Downloaded 34GiB to GPU in 1.93201s (17.5982GiB/s) | 20260507 07:45:18.302 | INFO | Analyzing chunk frames = [8270848,8271360) | 20260507 07:45:18.302 | INFO | MemInfo: { | 20260507 07:45:18.302 | INFO | "gpu" : " 17 GB", | 20260507 07:45:18.302 | INFO | "huge" : "", | 20260507 07:45:18.302 | INFO | "huge_shared" : " 27 GB", | 20260507 07:45:18.302 | INFO | "malloc" : " 14 GB", | 20260507 07:45:18.302 | INFO | "pinned" : " 29 GB" | 20260507 07:45:18.302 | INFO | } | 20260507 07:45:18.302 | INFO | Perf report for the last 5.13715 seconds: | 20260507 07:45:18.302 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:45:18.302 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:18.302 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:18.302 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:18.302 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260507 07:45:18.302 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:18.302 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:18.302 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:45:18.303 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:18.303 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:18.303 | INFO | Uploaded 36GiB to GPU in 2.0509s (17.5533GiB/s) | 20260507 07:45:18.303 | INFO | Downloaded 34GiB to GPU in 1.9505s (17.4314GiB/s) | 20260507 07:45:23.443 | INFO | Analyzing chunk frames = [8271360,8271872) | 20260507 07:45:23.443 | INFO | MemInfo: { | 20260507 07:45:23.443 | INFO | "gpu" : " 17 GB", | 20260507 07:45:23.443 | INFO | "huge" : "", | 20260507 07:45:23.443 | INFO | "huge_shared" : " 27 GB", | 20260507 07:45:23.443 | INFO | "malloc" : " 14 GB", | 20260507 07:45:23.443 | INFO | "pinned" : " 29 GB" | 20260507 07:45:23.443 | INFO | } | 20260507 07:45:23.444 | INFO | Perf report for the last 5.14118 seconds: | 20260507 07:45:23.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:45:23.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:23.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:23.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:23.444 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 07:45:23.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:23.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:23.444 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:45:23.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:23.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:23.444 | INFO | Uploaded 36GiB to GPU in 2.04854s (17.5735GiB/s) | 20260507 07:45:23.444 | INFO | Downloaded 34GiB to GPU in 1.94986s (17.4371GiB/s) | 20260507 07:45:28.582 | INFO | Analyzing chunk frames = [8271872,8272384) | 20260507 07:45:28.582 | INFO | MemInfo: { | 20260507 07:45:28.582 | INFO | "gpu" : " 17 GB", | 20260507 07:45:28.582 | INFO | "huge" : "", | 20260507 07:45:28.582 | INFO | "huge_shared" : " 27 GB", | 20260507 07:45:28.582 | INFO | "malloc" : " 14 GB", | 20260507 07:45:28.582 | INFO | "pinned" : " 29 GB" | 20260507 07:45:28.582 | INFO | } | 20260507 07:45:28.582 | INFO | Perf report for the last 5.1388 seconds: | 20260507 07:45:28.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:45:28.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:28.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:28.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:28.582 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 07:45:28.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:28.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:28.583 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:45:28.583 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:28.583 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:28.583 | INFO | Uploaded 36GiB to GPU in 2.06048s (17.4717GiB/s) | 20260507 07:45:28.583 | INFO | Downloaded 34GiB to GPU in 1.90365s (17.8604GiB/s) | 20260507 07:45:33.714 | INFO | Analyzing chunk frames = [8272384,8272896) | 20260507 07:45:33.714 | INFO | MemInfo: { | 20260507 07:45:33.714 | INFO | "gpu" : " 17 GB", | 20260507 07:45:33.715 | INFO | "huge" : "", | 20260507 07:45:33.715 | INFO | "huge_shared" : " 27 GB", | 20260507 07:45:33.715 | INFO | "malloc" : " 14 GB", | 20260507 07:45:33.715 | INFO | "pinned" : " 29 GB" | 20260507 07:45:33.715 | INFO | } | 20260507 07:45:33.715 | INFO | Perf report for the last 5.13237 seconds: | 20260507 07:45:33.715 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:45:33.715 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:33.715 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:33.715 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:33.715 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:45:33.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:33.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:33.715 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 07:45:33.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:33.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:33.715 | INFO | Uploaded 36GiB to GPU in 2.07341s (17.3627GiB/s) | 20260507 07:45:33.715 | INFO | Downloaded 34GiB to GPU in 1.93095s (17.6079GiB/s) | 20260507 07:45:38.855 | INFO | Analyzing chunk frames = [8272896,8273408) | 20260507 07:45:38.856 | INFO | MemInfo: { | 20260507 07:45:38.856 | INFO | "gpu" : " 17 GB", | 20260507 07:45:38.856 | INFO | "huge" : "", | 20260507 07:45:38.856 | INFO | "huge_shared" : " 27 GB", | 20260507 07:45:38.856 | INFO | "malloc" : " 14 GB", | 20260507 07:45:38.856 | INFO | "pinned" : " 29 GB" | 20260507 07:45:38.856 | INFO | } | 20260507 07:45:38.856 | INFO | Perf report for the last 5.14107 seconds: | 20260507 07:45:38.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:45:38.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:38.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:38.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:38.856 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:45:38.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:38.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:38.856 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 07:45:38.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:38.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:38.856 | INFO | Uploaded 36GiB to GPU in 1.98164s (18.1667GiB/s) | 20260507 07:45:38.856 | INFO | Downloaded 34GiB to GPU in 1.98181s (17.156GiB/s) | 20260507 07:45:44.002 | INFO | Analyzing chunk frames = [8273408,8273920) | 20260507 07:45:44.002 | INFO | MemInfo: { | 20260507 07:45:44.002 | INFO | "gpu" : " 17 GB", | 20260507 07:45:44.002 | INFO | "huge" : "", | 20260507 07:45:44.002 | INFO | "huge_shared" : " 27 GB", | 20260507 07:45:44.002 | INFO | "malloc" : " 14 GB", | 20260507 07:45:44.002 | INFO | "pinned" : " 29 GB" | 20260507 07:45:44.002 | INFO | } | 20260507 07:45:44.002 | INFO | Perf report for the last 5.14596 seconds: | 20260507 07:45:44.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:45:44.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:44.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:44.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:44.002 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:45:44.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:44.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:44.002 | INFO | PRE_HQ : 28.8%, 1.0, 15.0ms, 0 | 20260507 07:45:44.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:44.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:44.002 | INFO | Uploaded 36GiB to GPU in 1.99339s (18.0597GiB/s) | 20260507 07:45:44.002 | INFO | Downloaded 34GiB to GPU in 1.96641s (17.2904GiB/s) | 20260507 07:45:44.002 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:45:44.002 | INFO | Total mem usage: 18 GiB | 20260507 07:45:44.002 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:45:44.002 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:45:44.002 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:45:44.002 | INFO | High Water Breakdown: | 20260507 07:45:44.002 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:45:44.002 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:45:44.002 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:45:44.002 | INFO | ------------------------------------------ | 20260507 07:45:44.002 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:45:44.002 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:45:44.002 | INFO | Total mem usage: 26.5 GiB | 20260507 07:45:44.002 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:45:44.002 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:45:44.002 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 07:45:44.002 | INFO | High Water Breakdown: | 20260507 07:45:44.002 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:45:44.002 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:45:44.002 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:45:44.002 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:45:44.002 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:45:44.002 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:45:44.002 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:45:44.002 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:45:44.002 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:45:44.002 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:45:44.002 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:45:44.002 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:45:44.002 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:45:44.002 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:45:44.002 | INFO | ------------------------------------------ | 20260507 07:45:44.002 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:45:44.002 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:45:44.002 | INFO | Total mem usage: 25.6 GiB | 20260507 07:45:44.003 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:45:44.003 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:45:44.003 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:45:44.003 | INFO | High Water Breakdown: | 20260507 07:45:44.003 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:45:44.003 | INFO | ------------------------------------------ | 20260507 07:45:44.003 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:45:44.003 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:45:44.003 | INFO | Total mem usage: 15.7 GiB | 20260507 07:45:44.003 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:45:44.003 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:45:44.003 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:45:44.003 | INFO | High Water Breakdown: | 20260507 07:45:44.003 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:45:44.003 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:45:44.003 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:45:44.003 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:45:44.003 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:45:44.004 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:45:44.004 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:45:44.004 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:45:44.004 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:45:44.004 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:45:44.004 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:45:44.004 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:45:44.004 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:45:44.004 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:45:44.004 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:45:44.004 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:45:44.004 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:45:44.004 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:45:44.004 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:45:44.004 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:45:44.004 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:45:44.004 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:45:44.004 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:45:44.004 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:45:44.004 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:45:44.004 | INFO | ------------------------------------------ | 20260507 07:45:44.004 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:45:44.004 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:45:44.004 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:45:44.004 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:45:44.004 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.47e+01, 1.34e+01 / 1.14e+00 | 20260507 07:45:44.004 | INFO | PerfReport: 0.29%, AggregateStats -- 9.02e-02 / 9.98e-02, 9.10e-03 / 4.37e-03 | 20260507 07:45:44.004 | INFO | PerfReport: 11.66%, Baseline -- 3.57e+00 / 4.47e+00, 4.95e-01 / 2.15e-01 | 20260507 07:45:44.004 | INFO | PerfReport: 3.21%, Histogram -- 9.85e-01 / 1.49e+00, 1.64e-01 / 1.03e-01 | 20260507 07:45:44.004 | INFO | PerfReport: 16.35%, DME -- 4.01e+01 / 4.51e+01, 4.01e+00 / 1.22e+00 | 20260507 07:45:44.004 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.77e+01 / 1.92e+01, 1.19e+00 / 1.09e+00 | 20260507 07:45:44.004 | INFO | PerfReport: 6.91%, PulseAccumulating -- 2.12e+00 / 2.74e+00, 3.88e-01 / 1.51e-01 | 20260507 07:45:44.004 | INFO | PerfReport: 2.93%, Metrics -- 8.97e-01 / 1.20e+00, 2.39e-01 / 1.27e-01 | 20260507 07:45:44.004 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.01e-01 / 3.62e-01, 5.03e-02 / 1.91e-02 | 20260507 07:45:44.004 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:45:44.004 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:45:44.004 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:45:44.004 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 07:45:44.004 | INFO | PerfReport: 21.90%, Upload -- 2.11e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260507 07:45:44.004 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 07:45:44.004 | INFO | PerfReport: 31.86%, Compute -- 3.06e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 07:45:44.004 | INFO | PerfReport: 1.62%, LockUpload -- 1.56e+00 / 0.00e+00, 4.99e+00 / 0.00e+00 | 20260507 07:45:44.004 | INFO | PerfReport: 1.27%, LockDownload -- 1.22e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260507 07:45:44.005 | INFO | PerfReport: 21.96%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 07:45:44.005 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.72e-02 / 0.00e+00 | 20260507 07:45:44.099 | INFO | Starting a round of Baz IO | 20260507 07:45:47.203 | INFO | Populating RTMetrics report | 20260507 07:45:47.203 | INFO | RTMetrics report populated | 20260507 07:45:47.203 | INFO | Opening tmp RTMetrics json file | 20260507 07:45:47.220 | INFO | Serializing RTMetrics to json | 20260507 07:45:47.222 | INFO | Serialization done, swapping tmp file now... | 20260507 07:45:47.223 | INFO | RTMetrics done | 20260507 07:45:47.288 | INFO | Finished a round of Baz IO: Wrote 1.631 GiB in 3.188 seconds | 20260507 07:45:47.288 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.54% / 0.00% / 67.46% | 20260507 07:45:47.288 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0998855% / 0% / 7.54992e-05% | 20260507 07:45:47.288 | INFO | Baz IO overall: | 20260507 07:45:47.288 | INFO | BytesWritten : 4139152195068 | 20260507 07:45:47.288 | INFO | NumEvents : 1981153104783 | 20260507 07:45:49.144 | INFO | Analyzing chunk frames = [8273920,8274432) | 20260507 07:45:49.144 | INFO | MemInfo: { | 20260507 07:45:49.144 | INFO | "gpu" : " 17 GB", | 20260507 07:45:49.144 | INFO | "huge" : "", | 20260507 07:45:49.144 | INFO | "huge_shared" : " 27 GB", | 20260507 07:45:49.144 | INFO | "malloc" : " 14 GB", | 20260507 07:45:49.144 | INFO | "pinned" : " 29 GB" | 20260507 07:45:49.144 | INFO | } | 20260507 07:45:49.144 | INFO | Perf report for the last 5.14191 seconds: | 20260507 07:45:49.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:45:49.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:49.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:49.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:49.144 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260507 07:45:49.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:49.144 | INFO | RT_METRICS : 41.3%, 1.0, 21.7ms, 0 | 20260507 07:45:49.144 | INFO | PRE_HQ : 58.9%, 1.8, 54.8ms, 0 | 20260507 07:45:49.144 | INFO | BAZWRITER : 61.5%, 3.1, 99.0ms, 0 | 20260507 07:45:49.144 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:45:49.144 | INFO | Uploaded 36GiB to GPU in 2.03045s (17.7301GiB/s) | 20260507 07:45:49.144 | INFO | Downloaded 38GiB to GPU in 2.16546s (17.5482GiB/s) | 20260507 07:45:54.272 | INFO | Analyzing chunk frames = [8274432,8274944) | 20260507 07:45:54.272 | INFO | MemInfo: { | 20260507 07:45:54.272 | INFO | "gpu" : " 17 GB", | 20260507 07:45:54.272 | INFO | "huge" : "", | 20260507 07:45:54.272 | INFO | "huge_shared" : " 27 GB", | 20260507 07:45:54.272 | INFO | "malloc" : " 14 GB", | 20260507 07:45:54.272 | INFO | "pinned" : " 29 GB" | 20260507 07:45:54.272 | INFO | } | 20260507 07:45:54.272 | INFO | Perf report for the last 5.12868 seconds: | 20260507 07:45:54.272 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:45:54.272 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:54.272 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:54.272 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:54.272 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 07:45:54.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:54.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:54.273 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 07:45:54.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:54.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:54.273 | INFO | Uploaded 36GiB to GPU in 2.02798s (17.7517GiB/s) | 20260507 07:45:54.273 | INFO | Downloaded 34GiB to GPU in 1.94784s (17.4553GiB/s) | 20260507 07:45:59.413 | INFO | Analyzing chunk frames = [8274944,8275456) | 20260507 07:45:59.413 | INFO | MemInfo: { | 20260507 07:45:59.413 | INFO | "gpu" : " 17 GB", | 20260507 07:45:59.413 | INFO | "huge" : "", | 20260507 07:45:59.413 | INFO | "huge_shared" : " 27 GB", | 20260507 07:45:59.413 | INFO | "malloc" : " 14 GB", | 20260507 07:45:59.413 | INFO | "pinned" : " 29 GB" | 20260507 07:45:59.413 | INFO | } | 20260507 07:45:59.413 | INFO | Perf report for the last 5.14082 seconds: | 20260507 07:45:59.413 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:45:59.413 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:59.413 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:59.413 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:59.413 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:45:59.413 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:59.413 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:59.413 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:45:59.413 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:59.413 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:45:59.413 | INFO | Uploaded 36GiB to GPU in 2.04903s (17.5693GiB/s) | 20260507 07:45:59.413 | INFO | Downloaded 34GiB to GPU in 1.94988s (17.437GiB/s) | 20260507 07:46:04.556 | INFO | Analyzing chunk frames = [8275456,8275968) | 20260507 07:46:04.556 | INFO | MemInfo: { | 20260507 07:46:04.556 | INFO | "gpu" : " 17 GB", | 20260507 07:46:04.556 | INFO | "huge" : "", | 20260507 07:46:04.556 | INFO | "huge_shared" : " 27 GB", | 20260507 07:46:04.556 | INFO | "malloc" : " 14 GB", | 20260507 07:46:04.556 | INFO | "pinned" : " 29 GB" | 20260507 07:46:04.556 | INFO | } | 20260507 07:46:04.556 | INFO | Perf report for the last 5.14254 seconds: | 20260507 07:46:04.556 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:46:04.556 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:04.556 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:04.556 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:04.556 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 07:46:04.556 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:04.556 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:04.556 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:46:04.556 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:04.556 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:04.556 | INFO | Uploaded 36GiB to GPU in 2.01401s (17.8748GiB/s) | 20260507 07:46:04.556 | INFO | Downloaded 34GiB to GPU in 1.93656s (17.5569GiB/s) | 20260507 07:46:09.694 | INFO | Analyzing chunk frames = [8275968,8276480) | 20260507 07:46:09.694 | INFO | MemInfo: { | 20260507 07:46:09.694 | INFO | "gpu" : " 17 GB", | 20260507 07:46:09.694 | INFO | "huge" : "", | 20260507 07:46:09.694 | INFO | "huge_shared" : " 27 GB", | 20260507 07:46:09.694 | INFO | "malloc" : " 14 GB", | 20260507 07:46:09.694 | INFO | "pinned" : " 29 GB" | 20260507 07:46:09.694 | INFO | } | 20260507 07:46:09.694 | INFO | Perf report for the last 5.13805 seconds: | 20260507 07:46:09.694 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:46:09.694 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:09.694 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:09.694 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:09.694 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 07:46:09.694 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:09.694 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:09.694 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:46:09.694 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:09.694 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:09.694 | INFO | Uploaded 36GiB to GPU in 2.02559s (17.7726GiB/s) | 20260507 07:46:09.694 | INFO | Downloaded 34GiB to GPU in 1.9182s (17.725GiB/s) | 20260507 07:46:14.831 | INFO | Analyzing chunk frames = [8276480,8276992) | 20260507 07:46:14.831 | INFO | MemInfo: { | 20260507 07:46:14.831 | INFO | "gpu" : " 17 GB", | 20260507 07:46:14.831 | INFO | "huge" : "", | 20260507 07:46:14.831 | INFO | "huge_shared" : " 27 GB", | 20260507 07:46:14.831 | INFO | "malloc" : " 14 GB", | 20260507 07:46:14.831 | INFO | "pinned" : " 29 GB" | 20260507 07:46:14.831 | INFO | } | 20260507 07:46:14.831 | INFO | Perf report for the last 5.13709 seconds: | 20260507 07:46:14.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:46:14.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:14.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:14.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:14.831 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 07:46:14.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:14.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:14.831 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:46:14.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:14.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:14.831 | INFO | Uploaded 36GiB to GPU in 2.06796s (17.4085GiB/s) | 20260507 07:46:14.831 | INFO | Downloaded 34GiB to GPU in 1.92314s (17.6794GiB/s) | 20260507 07:46:19.970 | INFO | Analyzing chunk frames = [8276992,8277504) | 20260507 07:46:19.970 | INFO | MemInfo: { | 20260507 07:46:19.970 | INFO | "gpu" : " 17 GB", | 20260507 07:46:19.970 | INFO | "huge" : "", | 20260507 07:46:19.970 | INFO | "huge_shared" : " 27 GB", | 20260507 07:46:19.970 | INFO | "malloc" : " 14 GB", | 20260507 07:46:19.970 | INFO | "pinned" : " 29 GB" | 20260507 07:46:19.970 | INFO | } | 20260507 07:46:19.970 | INFO | Perf report for the last 5.13908 seconds: | 20260507 07:46:19.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:46:19.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:19.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:19.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:19.970 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:46:19.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:19.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:19.970 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 07:46:19.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:19.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:19.970 | INFO | Uploaded 36GiB to GPU in 1.95325s (18.4308GiB/s) | 20260507 07:46:19.970 | INFO | Downloaded 34GiB to GPU in 1.98089s (17.164GiB/s) | 20260507 07:46:25.119 | INFO | Analyzing chunk frames = [8277504,8278016) | 20260507 07:46:25.119 | INFO | MemInfo: { | 20260507 07:46:25.119 | INFO | "gpu" : " 17 GB", | 20260507 07:46:25.119 | INFO | "huge" : "", | 20260507 07:46:25.119 | INFO | "huge_shared" : " 27 GB", | 20260507 07:46:25.119 | INFO | "malloc" : " 14 GB", | 20260507 07:46:25.119 | INFO | "pinned" : " 29 GB" | 20260507 07:46:25.119 | INFO | } | 20260507 07:46:25.119 | INFO | Perf report for the last 5.14929 seconds: | 20260507 07:46:25.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:46:25.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:25.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:25.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:25.119 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 07:46:25.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:25.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:25.119 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 07:46:25.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:25.120 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:25.120 | INFO | Uploaded 36GiB to GPU in 1.96424s (18.3277GiB/s) | 20260507 07:46:25.120 | INFO | Downloaded 34GiB to GPU in 1.96005s (17.3465GiB/s) | 20260507 07:46:25.216 | INFO | Starting a round of Baz IO | 20260507 07:46:28.313 | INFO | Populating RTMetrics report | 20260507 07:46:28.313 | INFO | RTMetrics report populated | 20260507 07:46:28.313 | INFO | Opening tmp RTMetrics json file | 20260507 07:46:28.321 | INFO | Serializing RTMetrics to json | 20260507 07:46:28.329 | INFO | Serialization done, swapping tmp file now... | 20260507 07:46:28.330 | INFO | RTMetrics done | 20260507 07:46:28.397 | INFO | Finished a round of Baz IO: Wrote 1.633 GiB in 3.181 seconds | 20260507 07:46:28.397 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.54% / 0.00% / 67.46% | 20260507 07:46:28.397 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.100501% / 7.94729e-06% / 7.15256e-05% | 20260507 07:46:28.397 | INFO | Baz IO overall: | 20260507 07:46:28.397 | INFO | BytesWritten : 4140905168622 | 20260507 07:46:28.397 | INFO | NumEvents : 1981971571873 | 20260507 07:46:30.261 | INFO | Analyzing chunk frames = [8278016,8278528) | 20260507 07:46:30.261 | INFO | MemInfo: { | 20260507 07:46:30.261 | INFO | "gpu" : " 17 GB", | 20260507 07:46:30.261 | INFO | "huge" : "", | 20260507 07:46:30.262 | INFO | "huge_shared" : " 27 GB", | 20260507 07:46:30.262 | INFO | "malloc" : " 14 GB", | 20260507 07:46:30.262 | INFO | "pinned" : " 29 GB" | 20260507 07:46:30.262 | INFO | } | 20260507 07:46:30.262 | INFO | Perf report for the last 5.14232 seconds: | 20260507 07:46:30.262 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:46:30.262 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:30.262 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:30.262 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:30.262 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 07:46:30.262 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:30.262 | INFO | RT_METRICS : 43.8%, 1.0, 23.0ms, 0 | 20260507 07:46:30.262 | INFO | PRE_HQ : 59.6%, 1.9, 58.4ms, 0 | 20260507 07:46:30.262 | INFO | BAZWRITER : 61.5%, 3.2, 101.2ms, 0 | 20260507 07:46:30.262 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:46:30.262 | INFO | Uploaded 36GiB to GPU in 2.06286s (17.4515GiB/s) | 20260507 07:46:30.262 | INFO | Downloaded 38GiB to GPU in 2.18102s (17.423GiB/s) | 20260507 07:46:35.393 | INFO | Analyzing chunk frames = [8278528,8279040) | 20260507 07:46:35.393 | INFO | MemInfo: { | 20260507 07:46:35.393 | INFO | "gpu" : " 17 GB", | 20260507 07:46:35.393 | INFO | "huge" : "", | 20260507 07:46:35.393 | INFO | "huge_shared" : " 27 GB", | 20260507 07:46:35.393 | INFO | "malloc" : " 14 GB", | 20260507 07:46:35.393 | INFO | "pinned" : " 29 GB" | 20260507 07:46:35.393 | INFO | } | 20260507 07:46:35.393 | INFO | Perf report for the last 5.13168 seconds: | 20260507 07:46:35.393 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:46:35.393 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:35.393 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:35.393 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:35.393 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:46:35.393 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:35.394 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:35.394 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 07:46:35.394 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:35.394 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:35.394 | INFO | Uploaded 36GiB to GPU in 1.98168s (18.1664GiB/s) | 20260507 07:46:35.394 | INFO | Downloaded 34GiB to GPU in 1.94822s (17.4519GiB/s) | 20260507 07:46:40.525 | INFO | Analyzing chunk frames = [8279040,8279552) | 20260507 07:46:40.525 | INFO | MemInfo: { | 20260507 07:46:40.525 | INFO | "gpu" : " 17 GB", | 20260507 07:46:40.525 | INFO | "huge" : "", | 20260507 07:46:40.525 | INFO | "huge_shared" : " 27 GB", | 20260507 07:46:40.525 | INFO | "malloc" : " 14 GB", | 20260507 07:46:40.525 | INFO | "pinned" : " 29 GB" | 20260507 07:46:40.525 | INFO | } | 20260507 07:46:40.525 | INFO | Perf report for the last 5.13172 seconds: | 20260507 07:46:40.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:46:40.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:40.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:40.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:40.525 | INFO | ANALYSIS : 60.6%, 3.0, 93.6ms, 0 | 20260507 07:46:40.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:40.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:40.525 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 07:46:40.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:40.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:40.525 | INFO | Uploaded 36GiB to GPU in 2.05147s (17.5484GiB/s) | 20260507 07:46:40.525 | INFO | Downloaded 34GiB to GPU in 1.95578s (17.3844GiB/s) | 20260507 07:46:45.666 | INFO | Analyzing chunk frames = [8279552,8280064) | 20260507 07:46:45.666 | INFO | MemInfo: { | 20260507 07:46:45.666 | INFO | "gpu" : " 17 GB", | 20260507 07:46:45.666 | INFO | "huge" : "", | 20260507 07:46:45.666 | INFO | "huge_shared" : " 27 GB", | 20260507 07:46:45.667 | INFO | "malloc" : " 14 GB", | 20260507 07:46:45.667 | INFO | "pinned" : " 29 GB" | 20260507 07:46:45.667 | INFO | } | 20260507 07:46:45.667 | INFO | Perf report for the last 5.14153 seconds: | 20260507 07:46:45.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:46:45.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:45.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:45.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:45.667 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 07:46:45.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:45.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:45.667 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:46:45.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:45.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:45.667 | INFO | Uploaded 36GiB to GPU in 1.98597s (18.1272GiB/s) | 20260507 07:46:45.667 | INFO | Downloaded 34GiB to GPU in 1.92249s (17.6854GiB/s) | 20260507 07:46:50.811 | INFO | Analyzing chunk frames = [8280064,8280576) | 20260507 07:46:50.811 | INFO | MemInfo: { | 20260507 07:46:50.811 | INFO | "gpu" : " 17 GB", | 20260507 07:46:50.811 | INFO | "huge" : "", | 20260507 07:46:50.811 | INFO | "huge_shared" : " 27 GB", | 20260507 07:46:50.811 | INFO | "malloc" : " 14 GB", | 20260507 07:46:50.811 | INFO | "pinned" : " 29 GB" | 20260507 07:46:50.811 | INFO | } | 20260507 07:46:50.811 | INFO | Perf report for the last 5.14441 seconds: | 20260507 07:46:50.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:46:50.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:50.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:50.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:50.811 | INFO | ANALYSIS : 59.9%, 3.0, 93.7ms, 0 | 20260507 07:46:50.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:50.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:50.811 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:46:50.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:50.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:50.811 | INFO | Uploaded 36GiB to GPU in 2.03661s (17.6764GiB/s) | 20260507 07:46:50.811 | INFO | Downloaded 34GiB to GPU in 1.94422s (17.4878GiB/s) | 20260507 07:46:55.951 | INFO | Analyzing chunk frames = [8280576,8281088) | 20260507 07:46:55.951 | INFO | MemInfo: { | 20260507 07:46:55.951 | INFO | "gpu" : " 17 GB", | 20260507 07:46:55.951 | INFO | "huge" : "", | 20260507 07:46:55.951 | INFO | "huge_shared" : " 27 GB", | 20260507 07:46:55.951 | INFO | "malloc" : " 14 GB", | 20260507 07:46:55.951 | INFO | "pinned" : " 29 GB" | 20260507 07:46:55.951 | INFO | } | 20260507 07:46:55.952 | INFO | Perf report for the last 5.14044 seconds: | 20260507 07:46:55.952 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:46:55.952 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:55.952 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:55.952 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:55.952 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 07:46:55.952 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:55.952 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:55.952 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:46:55.952 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:55.952 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:46:55.952 | INFO | Uploaded 36GiB to GPU in 2.06718s (17.415GiB/s) | 20260507 07:46:55.952 | INFO | Downloaded 34GiB to GPU in 1.93878s (17.5368GiB/s) | 20260507 07:47:01.089 | INFO | Analyzing chunk frames = [8281088,8281600) | 20260507 07:47:01.089 | INFO | MemInfo: { | 20260507 07:47:01.089 | INFO | "gpu" : " 17 GB", | 20260507 07:47:01.089 | INFO | "huge" : "", | 20260507 07:47:01.089 | INFO | "huge_shared" : " 27 GB", | 20260507 07:47:01.089 | INFO | "malloc" : " 14 GB", | 20260507 07:47:01.089 | INFO | "pinned" : " 29 GB" | 20260507 07:47:01.089 | INFO | } | 20260507 07:47:01.089 | INFO | Perf report for the last 5.1376 seconds: | 20260507 07:47:01.089 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:47:01.089 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:01.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:01.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:01.089 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260507 07:47:01.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:01.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:01.089 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 07:47:01.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:01.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:01.089 | INFO | Uploaded 36GiB to GPU in 1.87933s (19.1558GiB/s) | 20260507 07:47:01.089 | INFO | Downloaded 34GiB to GPU in 1.97447s (17.2198GiB/s) | 20260507 07:47:06.228 | INFO | Analyzing chunk frames = [8281600,8282112) | 20260507 07:47:06.228 | INFO | MemInfo: { | 20260507 07:47:06.228 | INFO | "gpu" : " 17 GB", | 20260507 07:47:06.228 | INFO | "huge" : "", | 20260507 07:47:06.228 | INFO | "huge_shared" : " 27 GB", | 20260507 07:47:06.229 | INFO | "malloc" : " 14 GB", | 20260507 07:47:06.229 | INFO | "pinned" : " 29 GB" | 20260507 07:47:06.229 | INFO | } | 20260507 07:47:06.229 | INFO | Perf report for the last 5.13944 seconds: | 20260507 07:47:06.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:47:06.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:06.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:06.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:06.229 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:47:06.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:06.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:06.229 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:47:06.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:06.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:06.229 | INFO | Uploaded 36GiB to GPU in 1.99482s (18.0467GiB/s) | 20260507 07:47:06.229 | INFO | Downloaded 34GiB to GPU in 1.96179s (17.3311GiB/s) | 20260507 07:47:06.229 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:47:06.229 | INFO | Total mem usage: 18 GiB | 20260507 07:47:06.229 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:47:06.229 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:47:06.229 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:47:06.229 | INFO | High Water Breakdown: | 20260507 07:47:06.229 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:47:06.229 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:47:06.229 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:47:06.229 | INFO | ------------------------------------------ | 20260507 07:47:06.229 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:47:06.229 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:47:06.229 | INFO | Total mem usage: 26.5 GiB | 20260507 07:47:06.229 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:47:06.229 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:47:06.229 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:47:06.229 | INFO | High Water Breakdown: | 20260507 07:47:06.229 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:47:06.230 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:47:06.230 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:47:06.230 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:47:06.230 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:47:06.230 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:47:06.230 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:47:06.230 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:47:06.230 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:47:06.230 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:47:06.230 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:47:06.230 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:47:06.230 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:47:06.230 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:47:06.230 | INFO | ------------------------------------------ | 20260507 07:47:06.230 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:47:06.230 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:47:06.230 | INFO | Total mem usage: 25.6 GiB | 20260507 07:47:06.230 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:47:06.230 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:47:06.230 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:47:06.230 | INFO | High Water Breakdown: | 20260507 07:47:06.230 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:47:06.230 | INFO | ------------------------------------------ | 20260507 07:47:06.230 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:47:06.230 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:47:06.230 | INFO | Total mem usage: 15.7 GiB | 20260507 07:47:06.230 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:47:06.230 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:47:06.230 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:47:06.230 | INFO | High Water Breakdown: | 20260507 07:47:06.230 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:47:06.230 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:47:06.230 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:47:06.230 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:47:06.230 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:47:06.230 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:47:06.230 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:47:06.230 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:47:06.230 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:47:06.230 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:47:06.230 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:47:06.230 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:47:06.230 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:47:06.230 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:47:06.230 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:47:06.230 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:47:06.230 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:47:06.230 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:47:06.230 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:47:06.230 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:47:06.230 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:47:06.230 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:47:06.230 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:47:06.230 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:47:06.230 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:47:06.230 | INFO | ------------------------------------------ | 20260507 07:47:06.230 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:47:06.230 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:47:06.230 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:47:06.230 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:47:06.230 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.35e+01 / 1.18e+00 | 20260507 07:47:06.230 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.02e-01, 1.07e-02 / 4.85e-03 | 20260507 07:47:06.230 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.49e+00, 5.13e-01 / 2.22e-01 | 20260507 07:47:06.230 | INFO | PerfReport: 3.22%, Histogram -- 9.90e-01 / 1.49e+00, 1.68e-01 / 1.12e-01 | 20260507 07:47:06.230 | INFO | PerfReport: 16.26%, DME -- 3.99e+01 / 4.47e+01, 3.79e+00 / 1.36e+00 | 20260507 07:47:06.230 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.32e+00 / 1.49e+00 | 20260507 07:47:06.230 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.68e+00, 3.88e-01 / 2.19e-01 | 20260507 07:47:06.230 | INFO | PerfReport: 2.93%, Metrics -- 8.99e-01 / 1.20e+00, 2.42e-01 / 1.40e-01 | 20260507 07:47:06.230 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.01e-01 / 3.56e-01, 5.13e-02 / 1.55e-02 | 20260507 07:47:06.230 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:47:06.230 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:47:06.230 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:47:06.230 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 07:47:06.230 | INFO | PerfReport: 21.82%, Upload -- 2.10e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260507 07:47:06.230 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 07:47:06.230 | INFO | PerfReport: 31.97%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 07:47:06.230 | INFO | PerfReport: 1.49%, LockUpload -- 1.44e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 07:47:06.231 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.03e+00 / 0.00e+00 | 20260507 07:47:06.231 | INFO | PerfReport: 22.00%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 07:47:06.231 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260507 07:47:06.326 | INFO | Starting a round of Baz IO | 20260507 07:47:09.418 | INFO | Populating RTMetrics report | 20260507 07:47:09.419 | INFO | RTMetrics report populated | 20260507 07:47:09.419 | INFO | Opening tmp RTMetrics json file | 20260507 07:47:09.421 | INFO | Serializing RTMetrics to json | 20260507 07:47:09.437 | INFO | Serialization done, swapping tmp file now... | 20260507 07:47:09.438 | INFO | RTMetrics done | 20260507 07:47:09.505 | INFO | Finished a round of Baz IO: Wrote 1.632 GiB in 3.178 seconds | 20260507 07:47:09.505 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.54% / 0.00% / 67.46% | 20260507 07:47:09.505 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0999292% / 0% / 7.94729e-05% | 20260507 07:47:09.505 | INFO | Baz IO overall: | 20260507 07:47:09.505 | INFO | BytesWritten : 4142657909030 | 20260507 07:47:09.505 | INFO | NumEvents : 1982789889816 | 20260507 07:47:11.365 | INFO | Analyzing chunk frames = [8282112,8282624) | 20260507 07:47:11.365 | INFO | MemInfo: { | 20260507 07:47:11.365 | INFO | "gpu" : " 17 GB", | 20260507 07:47:11.365 | INFO | "huge" : "", | 20260507 07:47:11.365 | INFO | "huge_shared" : " 27 GB", | 20260507 07:47:11.365 | INFO | "malloc" : " 14 GB", | 20260507 07:47:11.365 | INFO | "pinned" : " 29 GB" | 20260507 07:47:11.365 | INFO | } | 20260507 07:47:11.365 | INFO | Perf report for the last 5.1366 seconds: | 20260507 07:47:11.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:47:11.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:11.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:11.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:11.365 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 07:47:11.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:11.365 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 07:47:11.365 | INFO | PRE_HQ : 59.0%, 1.8, 54.9ms, 0 | 20260507 07:47:11.365 | INFO | BAZWRITER : 61.5%, 3.1, 97.2ms, 0 | 20260507 07:47:11.365 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:47:11.366 | INFO | Uploaded 36GiB to GPU in 2.05438s (17.5236GiB/s) | 20260507 07:47:11.366 | INFO | Downloaded 38GiB to GPU in 2.18091s (17.4239GiB/s) | 20260507 07:47:16.501 | INFO | Analyzing chunk frames = [8282624,8283136) | 20260507 07:47:16.501 | INFO | MemInfo: { | 20260507 07:47:16.501 | INFO | "gpu" : " 17 GB", | 20260507 07:47:16.501 | INFO | "huge" : "", | 20260507 07:47:16.501 | INFO | "huge_shared" : " 27 GB", | 20260507 07:47:16.501 | INFO | "malloc" : " 14 GB", | 20260507 07:47:16.501 | INFO | "pinned" : " 29 GB" | 20260507 07:47:16.502 | INFO | } | 20260507 07:47:16.502 | INFO | Perf report for the last 5.13646 seconds: | 20260507 07:47:16.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:47:16.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:16.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:16.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:16.502 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 07:47:16.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:16.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:16.502 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:47:16.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:16.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:16.502 | INFO | Uploaded 36GiB to GPU in 1.99326s (18.0609GiB/s) | 20260507 07:47:16.502 | INFO | Downloaded 34GiB to GPU in 1.92762s (17.6383GiB/s) | 20260507 07:47:21.642 | INFO | Analyzing chunk frames = [8283136,8283648) | 20260507 07:47:21.642 | INFO | MemInfo: { | 20260507 07:47:21.642 | INFO | "gpu" : " 17 GB", | 20260507 07:47:21.642 | INFO | "huge" : "", | 20260507 07:47:21.642 | INFO | "huge_shared" : " 27 GB", | 20260507 07:47:21.642 | INFO | "malloc" : " 14 GB", | 20260507 07:47:21.642 | INFO | "pinned" : " 29 GB" | 20260507 07:47:21.642 | INFO | } | 20260507 07:47:21.642 | INFO | Perf report for the last 5.14033 seconds: | 20260507 07:47:21.642 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:47:21.642 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:21.642 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:21.642 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:21.642 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 07:47:21.642 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:21.642 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:21.642 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:47:21.642 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:21.642 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:21.642 | INFO | Uploaded 36GiB to GPU in 2.05253s (17.5393GiB/s) | 20260507 07:47:21.643 | INFO | Downloaded 34GiB to GPU in 1.94942s (17.441GiB/s) | 20260507 07:47:26.781 | INFO | Analyzing chunk frames = [8283648,8284160) | 20260507 07:47:26.781 | INFO | MemInfo: { | 20260507 07:47:26.781 | INFO | "gpu" : " 17 GB", | 20260507 07:47:26.781 | INFO | "huge" : "", | 20260507 07:47:26.781 | INFO | "huge_shared" : " 27 GB", | 20260507 07:47:26.781 | INFO | "malloc" : " 14 GB", | 20260507 07:47:26.781 | INFO | "pinned" : " 29 GB" | 20260507 07:47:26.781 | INFO | } | 20260507 07:47:26.781 | INFO | Perf report for the last 5.13876 seconds: | 20260507 07:47:26.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:47:26.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:26.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:26.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:26.781 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 07:47:26.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:26.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:26.781 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:47:26.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:26.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:26.781 | INFO | Uploaded 36GiB to GPU in 2.04147s (17.6344GiB/s) | 20260507 07:47:26.782 | INFO | Downloaded 34GiB to GPU in 1.94794s (17.4544GiB/s) | 20260507 07:47:31.921 | INFO | Analyzing chunk frames = [8284160,8284672) | 20260507 07:47:31.921 | INFO | MemInfo: { | 20260507 07:47:31.921 | INFO | "gpu" : " 17 GB", | 20260507 07:47:31.921 | INFO | "huge" : "", | 20260507 07:47:31.921 | INFO | "huge_shared" : " 27 GB", | 20260507 07:47:31.921 | INFO | "malloc" : " 14 GB", | 20260507 07:47:31.921 | INFO | "pinned" : " 29 GB" | 20260507 07:47:31.921 | INFO | } | 20260507 07:47:31.921 | INFO | Perf report for the last 5.14048 seconds: | 20260507 07:47:31.921 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:47:31.921 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:31.921 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:31.921 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:31.921 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 07:47:31.921 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:31.921 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:31.921 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:47:31.921 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:31.921 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:31.922 | INFO | Uploaded 36GiB to GPU in 2.05734s (17.4984GiB/s) | 20260507 07:47:31.922 | INFO | Downloaded 34GiB to GPU in 1.95263s (17.4124GiB/s) | 20260507 07:47:37.068 | INFO | Analyzing chunk frames = [8284672,8285184) | 20260507 07:47:37.068 | INFO | MemInfo: { | 20260507 07:47:37.068 | INFO | "gpu" : " 17 GB", | 20260507 07:47:37.068 | INFO | "huge" : "", | 20260507 07:47:37.068 | INFO | "huge_shared" : " 27 GB", | 20260507 07:47:37.068 | INFO | "malloc" : " 14 GB", | 20260507 07:47:37.068 | INFO | "pinned" : " 29 GB" | 20260507 07:47:37.068 | INFO | } | 20260507 07:47:37.068 | INFO | Perf report for the last 5.14711 seconds: | 20260507 07:47:37.068 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:47:37.068 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:37.068 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:37.068 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:37.068 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 07:47:37.068 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:37.068 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:37.068 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 07:47:37.068 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:37.068 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:37.069 | INFO | Uploaded 36GiB to GPU in 2.04031s (17.6443GiB/s) | 20260507 07:47:37.069 | INFO | Downloaded 34GiB to GPU in 1.90912s (17.8093GiB/s) | 20260507 07:47:42.206 | INFO | Analyzing chunk frames = [8285184,8285696) | 20260507 07:47:42.206 | INFO | MemInfo: { | 20260507 07:47:42.206 | INFO | "gpu" : " 17 GB", | 20260507 07:47:42.206 | INFO | "huge" : "", | 20260507 07:47:42.206 | INFO | "huge_shared" : " 27 GB", | 20260507 07:47:42.206 | INFO | "malloc" : " 14 GB", | 20260507 07:47:42.206 | INFO | "pinned" : " 29 GB" | 20260507 07:47:42.206 | INFO | } | 20260507 07:47:42.206 | INFO | Perf report for the last 5.13797 seconds: | 20260507 07:47:42.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:47:42.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:42.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:42.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:42.206 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 07:47:42.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:42.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:42.206 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 07:47:42.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:42.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:42.207 | INFO | Uploaded 36GiB to GPU in 1.9596s (18.3711GiB/s) | 20260507 07:47:42.207 | INFO | Downloaded 34GiB to GPU in 1.97834s (17.1861GiB/s) | 20260507 07:47:47.337 | INFO | Analyzing chunk frames = [8285696,8286208) | 20260507 07:47:47.337 | INFO | MemInfo: { | 20260507 07:47:47.337 | INFO | "gpu" : " 17 GB", | 20260507 07:47:47.337 | INFO | "huge" : "", | 20260507 07:47:47.337 | INFO | "huge_shared" : " 27 GB", | 20260507 07:47:47.337 | INFO | "malloc" : " 14 GB", | 20260507 07:47:47.337 | INFO | "pinned" : " 29 GB" | 20260507 07:47:47.337 | INFO | } | 20260507 07:47:47.337 | INFO | Perf report for the last 5.13093 seconds: | 20260507 07:47:47.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:47:47.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:47.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:47.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:47.337 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 07:47:47.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:47.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:47.337 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:47:47.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:47.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:47.338 | INFO | Uploaded 36GiB to GPU in 1.99893s (18.0096GiB/s) | 20260507 07:47:47.338 | INFO | Downloaded 34GiB to GPU in 1.96665s (17.2883GiB/s) | 20260507 07:47:47.434 | INFO | Starting a round of Baz IO | 20260507 07:47:50.528 | INFO | Populating RTMetrics report | 20260507 07:47:50.528 | INFO | RTMetrics report populated | 20260507 07:47:50.528 | INFO | Opening tmp RTMetrics json file | 20260507 07:47:50.537 | INFO | Serializing RTMetrics to json | 20260507 07:47:50.544 | INFO | Serialization done, swapping tmp file now... | 20260507 07:47:50.544 | INFO | RTMetrics done | 20260507 07:47:50.610 | INFO | Finished a round of Baz IO: Wrote 1.631 GiB in 3.175 seconds | 20260507 07:47:50.610 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.54% / 0.00% / 67.46% | 20260507 07:47:50.610 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0993371% / 7.94729e-06% / 4.76837e-05% | 20260507 07:47:50.610 | INFO | Baz IO overall: | 20260507 07:47:50.610 | INFO | BytesWritten : 4144409371558 | 20260507 07:47:50.610 | INFO | NumEvents : 1983607296898 | 20260507 07:47:52.479 | INFO | Analyzing chunk frames = [8286208,8286720) | 20260507 07:47:52.479 | INFO | MemInfo: { | 20260507 07:47:52.479 | INFO | "gpu" : " 17 GB", | 20260507 07:47:52.479 | INFO | "huge" : "", | 20260507 07:47:52.479 | INFO | "huge_shared" : " 27 GB", | 20260507 07:47:52.480 | INFO | "malloc" : " 14 GB", | 20260507 07:47:52.480 | INFO | "pinned" : " 29 GB" | 20260507 07:47:52.480 | INFO | } | 20260507 07:47:52.480 | INFO | Perf report for the last 5.14231 seconds: | 20260507 07:47:52.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:47:52.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:52.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:52.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:52.480 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 07:47:52.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:52.480 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 07:47:52.480 | INFO | PRE_HQ : 58.7%, 1.8, 54.4ms, 0 | 20260507 07:47:52.480 | INFO | BAZWRITER : 61.3%, 3.0, 94.3ms, 0 | 20260507 07:47:52.480 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:47:52.480 | INFO | Uploaded 36GiB to GPU in 2.05349s (17.5311GiB/s) | 20260507 07:47:52.480 | INFO | Downloaded 38GiB to GPU in 2.17993s (17.4318GiB/s) | 20260507 07:47:57.619 | INFO | Analyzing chunk frames = [8286720,8287232) | 20260507 07:47:57.619 | INFO | MemInfo: { | 20260507 07:47:57.619 | INFO | "gpu" : " 17 GB", | 20260507 07:47:57.619 | INFO | "huge" : "", | 20260507 07:47:57.619 | INFO | "huge_shared" : " 27 GB", | 20260507 07:47:57.619 | INFO | "malloc" : " 14 GB", | 20260507 07:47:57.619 | INFO | "pinned" : " 29 GB" | 20260507 07:47:57.619 | INFO | } | 20260507 07:47:57.619 | INFO | Perf report for the last 5.13949 seconds: | 20260507 07:47:57.619 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:47:57.619 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:57.619 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:57.619 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:57.619 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 07:47:57.619 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:57.619 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:57.619 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:47:57.619 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:57.619 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:47:57.619 | INFO | Uploaded 36GiB to GPU in 2.0025s (17.9775GiB/s) | 20260507 07:47:57.619 | INFO | Downloaded 34GiB to GPU in 1.9295s (17.6212GiB/s) | 20260507 07:48:02.764 | INFO | Analyzing chunk frames = [8287232,8287744) | 20260507 07:48:02.764 | INFO | MemInfo: { | 20260507 07:48:02.764 | INFO | "gpu" : " 17 GB", | 20260507 07:48:02.764 | INFO | "huge" : "", | 20260507 07:48:02.764 | INFO | "huge_shared" : " 27 GB", | 20260507 07:48:02.764 | INFO | "malloc" : " 14 GB", | 20260507 07:48:02.764 | INFO | "pinned" : " 29 GB" | 20260507 07:48:02.764 | INFO | } | 20260507 07:48:02.764 | INFO | Perf report for the last 5.14507 seconds: | 20260507 07:48:02.764 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:48:02.764 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:02.764 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:02.764 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:02.764 | INFO | ANALYSIS : 60.5%, 3.0, 93.9ms, 0 | 20260507 07:48:02.764 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:02.764 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:02.764 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:48:02.764 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:02.764 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:02.765 | INFO | Uploaded 36GiB to GPU in 2.05402s (17.5266GiB/s) | 20260507 07:48:02.765 | INFO | Downloaded 34GiB to GPU in 1.95144s (17.423GiB/s) | 20260507 07:48:07.900 | INFO | Analyzing chunk frames = [8287744,8288256) | 20260507 07:48:07.900 | INFO | MemInfo: { | 20260507 07:48:07.900 | INFO | "gpu" : " 17 GB", | 20260507 07:48:07.900 | INFO | "huge" : "", | 20260507 07:48:07.900 | INFO | "huge_shared" : " 27 GB", | 20260507 07:48:07.900 | INFO | "malloc" : " 14 GB", | 20260507 07:48:07.900 | INFO | "pinned" : " 29 GB" | 20260507 07:48:07.900 | INFO | } | 20260507 07:48:07.900 | INFO | Perf report for the last 5.13619 seconds: | 20260507 07:48:07.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:48:07.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:07.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:07.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:07.900 | INFO | ANALYSIS : 60.2%, 2.9, 92.9ms, 0 | 20260507 07:48:07.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:07.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:07.901 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:48:07.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:07.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:07.901 | INFO | Uploaded 36GiB to GPU in 2.01321s (17.8819GiB/s) | 20260507 07:48:07.901 | INFO | Downloaded 34GiB to GPU in 1.94119s (17.515GiB/s) | 20260507 07:48:13.037 | INFO | Analyzing chunk frames = [8288256,8288768) | 20260507 07:48:13.037 | INFO | MemInfo: { | 20260507 07:48:13.037 | INFO | "gpu" : " 17 GB", | 20260507 07:48:13.037 | INFO | "huge" : "", | 20260507 07:48:13.037 | INFO | "huge_shared" : " 27 GB", | 20260507 07:48:13.037 | INFO | "malloc" : " 14 GB", | 20260507 07:48:13.037 | INFO | "pinned" : " 29 GB" | 20260507 07:48:13.037 | INFO | } | 20260507 07:48:13.037 | INFO | Perf report for the last 5.13671 seconds: | 20260507 07:48:13.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:48:13.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:13.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:13.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:13.037 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 07:48:13.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:13.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:13.037 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:48:13.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:13.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:13.038 | INFO | Uploaded 36GiB to GPU in 2.04495s (17.6043GiB/s) | 20260507 07:48:13.038 | INFO | Downloaded 34GiB to GPU in 1.89424s (17.9491GiB/s) | 20260507 07:48:18.176 | INFO | Analyzing chunk frames = [8288768,8289280) | 20260507 07:48:18.176 | INFO | MemInfo: { | 20260507 07:48:18.176 | INFO | "gpu" : " 17 GB", | 20260507 07:48:18.176 | INFO | "huge" : "", | 20260507 07:48:18.176 | INFO | "huge_shared" : " 27 GB", | 20260507 07:48:18.176 | INFO | "malloc" : " 14 GB", | 20260507 07:48:18.176 | INFO | "pinned" : " 29 GB" | 20260507 07:48:18.176 | INFO | } | 20260507 07:48:18.176 | INFO | Perf report for the last 5.13887 seconds: | 20260507 07:48:18.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:48:18.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:18.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:18.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:18.176 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 07:48:18.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:18.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:18.176 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 07:48:18.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:18.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:18.176 | INFO | Uploaded 36GiB to GPU in 2.07915s (17.3147GiB/s) | 20260507 07:48:18.176 | INFO | Downloaded 34GiB to GPU in 1.94138s (17.5133GiB/s) | 20260507 07:48:23.316 | INFO | Analyzing chunk frames = [8289280,8289792) | 20260507 07:48:23.316 | INFO | MemInfo: { | 20260507 07:48:23.316 | INFO | "gpu" : " 17 GB", | 20260507 07:48:23.316 | INFO | "huge" : "", | 20260507 07:48:23.316 | INFO | "huge_shared" : " 27 GB", | 20260507 07:48:23.316 | INFO | "malloc" : " 14 GB", | 20260507 07:48:23.316 | INFO | "pinned" : " 29 GB" | 20260507 07:48:23.316 | INFO | } | 20260507 07:48:23.317 | INFO | Perf report for the last 5.14051 seconds: | 20260507 07:48:23.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:48:23.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:23.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:23.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:23.317 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 07:48:23.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:23.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:23.317 | INFO | PRE_HQ : 25.3%, 1.0, 13.2ms, 0 | 20260507 07:48:23.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:23.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:23.317 | INFO | Uploaded 36GiB to GPU in 2.03065s (17.7283GiB/s) | 20260507 07:48:23.317 | INFO | Downloaded 34GiB to GPU in 1.97368s (17.2267GiB/s) | 20260507 07:48:28.454 | INFO | Analyzing chunk frames = [8289792,8290304) | 20260507 07:48:28.455 | INFO | MemInfo: { | 20260507 07:48:28.455 | INFO | "gpu" : " 17 GB", | 20260507 07:48:28.455 | INFO | "huge" : "", | 20260507 07:48:28.455 | INFO | "huge_shared" : " 27 GB", | 20260507 07:48:28.455 | INFO | "malloc" : " 14 GB", | 20260507 07:48:28.455 | INFO | "pinned" : " 29 GB" | 20260507 07:48:28.455 | INFO | } | 20260507 07:48:28.455 | INFO | Perf report for the last 5.13821 seconds: | 20260507 07:48:28.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:48:28.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:28.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:28.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:28.455 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 07:48:28.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:28.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:28.455 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:48:28.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:28.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:28.455 | INFO | Uploaded 36GiB to GPU in 1.9809s (18.1736GiB/s) | 20260507 07:48:28.455 | INFO | Downloaded 34GiB to GPU in 1.95952s (17.3511GiB/s) | 20260507 07:48:28.455 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:48:28.455 | INFO | Total mem usage: 18 GiB | 20260507 07:48:28.455 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:48:28.455 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:48:28.455 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:48:28.455 | INFO | High Water Breakdown: | 20260507 07:48:28.455 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:48:28.455 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:48:28.455 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:48:28.455 | INFO | ------------------------------------------ | 20260507 07:48:28.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:48:28.455 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:48:28.455 | INFO | Total mem usage: 26.5 GiB | 20260507 07:48:28.455 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:48:28.455 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:48:28.455 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:48:28.455 | INFO | High Water Breakdown: | 20260507 07:48:28.455 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:48:28.455 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:48:28.455 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:48:28.455 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:48:28.455 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:48:28.455 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:48:28.455 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:48:28.455 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:48:28.455 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:48:28.455 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:48:28.455 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:48:28.455 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:48:28.455 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:48:28.455 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:48:28.455 | INFO | ------------------------------------------ | 20260507 07:48:28.455 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:48:28.456 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:48:28.456 | INFO | Total mem usage: 25.6 GiB | 20260507 07:48:28.456 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:48:28.456 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:48:28.456 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:48:28.456 | INFO | High Water Breakdown: | 20260507 07:48:28.456 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:48:28.456 | INFO | ------------------------------------------ | 20260507 07:48:28.456 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:48:28.456 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:48:28.456 | INFO | Total mem usage: 15.7 GiB | 20260507 07:48:28.456 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:48:28.456 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:48:28.456 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:48:28.456 | INFO | High Water Breakdown: | 20260507 07:48:28.456 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:48:28.456 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:48:28.456 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:48:28.456 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:48:28.456 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:48:28.456 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:48:28.456 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:48:28.456 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:48:28.456 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:48:28.456 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:48:28.456 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:48:28.456 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:48:28.456 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:48:28.456 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:48:28.456 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:48:28.456 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:48:28.456 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:48:28.456 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:48:28.456 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:48:28.456 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:48:28.456 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:48:28.456 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:48:28.456 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:48:28.456 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:48:28.456 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:48:28.456 | INFO | ------------------------------------------ | 20260507 07:48:28.456 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:48:28.456 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:48:28.456 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:48:28.456 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:48:28.456 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.33e+01 / 1.02e+00 | 20260507 07:48:28.456 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.02e-01, 9.28e-03 / 5.42e-03 | 20260507 07:48:28.456 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.50e+00, 5.00e-01 / 1.94e-01 | 20260507 07:48:28.456 | INFO | PerfReport: 3.20%, Histogram -- 9.82e-01 / 1.48e+00, 1.60e-01 / 1.03e-01 | 20260507 07:48:28.456 | INFO | PerfReport: 16.35%, DME -- 4.01e+01 / 4.51e+01, 3.92e+00 / 1.25e+00 | 20260507 07:48:28.456 | INFO | PerfReport: 57.69%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.24e+00 / 1.01e+00 | 20260507 07:48:28.456 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.70e+00, 3.88e-01 / 2.00e-01 | 20260507 07:48:28.456 | INFO | PerfReport: 2.93%, Metrics -- 8.97e-01 / 1.20e+00, 2.41e-01 / 7.98e-02 | 20260507 07:48:28.456 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.01e-01 / 3.62e-01, 5.11e-02 / 2.20e-02 | 20260507 07:48:28.456 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:48:28.456 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:48:28.456 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:48:28.456 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 07:48:28.456 | INFO | PerfReport: 21.96%, Upload -- 2.11e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260507 07:48:28.456 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 07:48:28.456 | INFO | PerfReport: 31.88%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 07:48:28.456 | INFO | PerfReport: 1.68%, LockUpload -- 1.61e+00 / 0.00e+00, 5.05e+00 / 0.00e+00 | 20260507 07:48:28.456 | INFO | PerfReport: 1.18%, LockDownload -- 1.13e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260507 07:48:28.456 | INFO | PerfReport: 21.93%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 07:48:28.456 | INFO | PerfReport: 0.02%, Misc -- 1.65e-02 / 0.00e+00, 3.68e-02 / 0.00e+00 | 20260507 07:48:28.552 | INFO | Starting a round of Baz IO | 20260507 07:48:31.646 | INFO | Populating RTMetrics report | 20260507 07:48:31.647 | INFO | RTMetrics report populated | 20260507 07:48:31.647 | INFO | Opening tmp RTMetrics json file | 20260507 07:48:31.662 | INFO | Serializing RTMetrics to json | 20260507 07:48:31.663 | INFO | Serialization done, swapping tmp file now... | 20260507 07:48:31.664 | INFO | RTMetrics done | 20260507 07:48:31.734 | INFO | Finished a round of Baz IO: Wrote 1.63 GiB in 3.182 seconds | 20260507 07:48:31.734 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.54% / 0.00% / 67.46% | 20260507 07:48:31.734 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0990351% / 3.97364e-06% / 6.75519e-05% | 20260507 07:48:31.734 | INFO | Baz IO overall: | 20260507 07:48:31.734 | INFO | BytesWritten : 4146159620293 | 20260507 07:48:31.734 | INFO | NumEvents : 1984424022822 | 20260507 07:48:33.591 | INFO | Analyzing chunk frames = [8290304,8290816) | 20260507 07:48:33.591 | INFO | MemInfo: { | 20260507 07:48:33.591 | INFO | "gpu" : " 17 GB", | 20260507 07:48:33.591 | INFO | "huge" : "", | 20260507 07:48:33.591 | INFO | "huge_shared" : " 27 GB", | 20260507 07:48:33.591 | INFO | "malloc" : " 14 GB", | 20260507 07:48:33.591 | INFO | "pinned" : " 29 GB" | 20260507 07:48:33.591 | INFO | } | 20260507 07:48:33.591 | INFO | Perf report for the last 5.13621 seconds: | 20260507 07:48:33.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:48:33.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:33.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:33.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:33.591 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 07:48:33.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:33.591 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260507 07:48:33.591 | INFO | PRE_HQ : 58.9%, 1.8, 55.2ms, 0 | 20260507 07:48:33.591 | INFO | BAZWRITER : 61.5%, 3.1, 99.1ms, 0 | 20260507 07:48:33.591 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 07:48:33.591 | INFO | Uploaded 36GiB to GPU in 2.05484s (17.5196GiB/s) | 20260507 07:48:33.591 | INFO | Downloaded 38GiB to GPU in 2.1764s (17.46GiB/s) | 20260507 07:48:38.735 | INFO | Analyzing chunk frames = [8290816,8291328) | 20260507 07:48:38.735 | INFO | MemInfo: { | 20260507 07:48:38.735 | INFO | "gpu" : " 17 GB", | 20260507 07:48:38.735 | INFO | "huge" : "", | 20260507 07:48:38.735 | INFO | "huge_shared" : " 27 GB", | 20260507 07:48:38.735 | INFO | "malloc" : " 14 GB", | 20260507 07:48:38.735 | INFO | "pinned" : " 29 GB" | 20260507 07:48:38.735 | INFO | } | 20260507 07:48:38.735 | INFO | Perf report for the last 5.14373 seconds: | 20260507 07:48:38.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:48:38.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:38.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:38.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:38.735 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 07:48:38.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:38.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:38.735 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:48:38.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:38.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:38.735 | INFO | Uploaded 36GiB to GPU in 1.98111s (18.1716GiB/s) | 20260507 07:48:38.735 | INFO | Downloaded 34GiB to GPU in 1.93486s (17.5723GiB/s) | 20260507 07:48:43.875 | INFO | Analyzing chunk frames = [8291328,8291840) | 20260507 07:48:43.875 | INFO | MemInfo: { | 20260507 07:48:43.875 | INFO | "gpu" : " 17 GB", | 20260507 07:48:43.875 | INFO | "huge" : "", | 20260507 07:48:43.875 | INFO | "huge_shared" : " 27 GB", | 20260507 07:48:43.875 | INFO | "malloc" : " 14 GB", | 20260507 07:48:43.875 | INFO | "pinned" : " 29 GB" | 20260507 07:48:43.875 | INFO | } | 20260507 07:48:43.875 | INFO | Perf report for the last 5.14 seconds: | 20260507 07:48:43.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:48:43.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:43.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:43.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:43.875 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 07:48:43.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:43.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:43.875 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:48:43.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:43.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:43.875 | INFO | Uploaded 36GiB to GPU in 2.024s (17.7866GiB/s) | 20260507 07:48:43.875 | INFO | Downloaded 34GiB to GPU in 1.95661s (17.377GiB/s) | 20260507 07:48:49.019 | INFO | Analyzing chunk frames = [8291840,8292352) | 20260507 07:48:49.019 | INFO | MemInfo: { | 20260507 07:48:49.019 | INFO | "gpu" : " 17 GB", | 20260507 07:48:49.019 | INFO | "huge" : "", | 20260507 07:48:49.019 | INFO | "huge_shared" : " 27 GB", | 20260507 07:48:49.019 | INFO | "malloc" : " 14 GB", | 20260507 07:48:49.019 | INFO | "pinned" : " 29 GB" | 20260507 07:48:49.019 | INFO | } | 20260507 07:48:49.019 | INFO | Perf report for the last 5.14423 seconds: | 20260507 07:48:49.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:48:49.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:49.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:49.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:49.019 | INFO | ANALYSIS : 60.1%, 2.9, 93.0ms, 0 | 20260507 07:48:49.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:49.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:49.019 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:48:49.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:49.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:49.019 | INFO | Uploaded 36GiB to GPU in 2.03412s (17.698GiB/s) | 20260507 07:48:49.019 | INFO | Downloaded 34GiB to GPU in 1.94643s (17.4678GiB/s) | 20260507 07:48:54.153 | INFO | Analyzing chunk frames = [8292352,8292864) | 20260507 07:48:54.153 | INFO | MemInfo: { | 20260507 07:48:54.154 | INFO | "gpu" : " 17 GB", | 20260507 07:48:54.154 | INFO | "huge" : "", | 20260507 07:48:54.154 | INFO | "huge_shared" : " 27 GB", | 20260507 07:48:54.154 | INFO | "malloc" : " 14 GB", | 20260507 07:48:54.154 | INFO | "pinned" : " 29 GB" | 20260507 07:48:54.154 | INFO | } | 20260507 07:48:54.154 | INFO | Perf report for the last 5.13491 seconds: | 20260507 07:48:54.154 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:48:54.154 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:54.154 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:54.154 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:54.154 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 07:48:54.154 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:54.154 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:54.154 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:48:54.154 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:54.154 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:54.154 | INFO | Uploaded 36GiB to GPU in 2.06453s (17.4374GiB/s) | 20260507 07:48:54.154 | INFO | Downloaded 34GiB to GPU in 1.91512s (17.7535GiB/s) | 20260507 07:48:59.287 | INFO | Analyzing chunk frames = [8292864,8293376) | 20260507 07:48:59.287 | INFO | MemInfo: { | 20260507 07:48:59.287 | INFO | "gpu" : " 17 GB", | 20260507 07:48:59.287 | INFO | "huge" : "", | 20260507 07:48:59.287 | INFO | "huge_shared" : " 27 GB", | 20260507 07:48:59.287 | INFO | "malloc" : " 14 GB", | 20260507 07:48:59.287 | INFO | "pinned" : " 29 GB" | 20260507 07:48:59.287 | INFO | } | 20260507 07:48:59.287 | INFO | Perf report for the last 5.13322 seconds: | 20260507 07:48:59.287 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:48:59.287 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:59.287 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:59.287 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:59.287 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:48:59.287 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:59.287 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:59.287 | INFO | PRE_HQ : 26.4%, 1.0, 13.7ms, 0 | 20260507 07:48:59.287 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:59.287 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:48:59.288 | INFO | Uploaded 36GiB to GPU in 2.02159s (17.8077GiB/s) | 20260507 07:48:59.288 | INFO | Downloaded 34GiB to GPU in 1.93634s (17.5589GiB/s) | 20260507 07:49:04.432 | INFO | Analyzing chunk frames = [8293376,8293888) | 20260507 07:49:04.432 | INFO | MemInfo: { | 20260507 07:49:04.432 | INFO | "gpu" : " 17 GB", | 20260507 07:49:04.432 | INFO | "huge" : "", | 20260507 07:49:04.432 | INFO | "huge_shared" : " 27 GB", | 20260507 07:49:04.432 | INFO | "malloc" : " 14 GB", | 20260507 07:49:04.432 | INFO | "pinned" : " 29 GB" | 20260507 07:49:04.432 | INFO | } | 20260507 07:49:04.432 | INFO | Perf report for the last 5.1451 seconds: | 20260507 07:49:04.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:49:04.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:04.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:04.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:04.432 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 07:49:04.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:04.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:04.432 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 07:49:04.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:04.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:04.433 | INFO | Uploaded 36GiB to GPU in 2.03559s (17.6853GiB/s) | 20260507 07:49:04.433 | INFO | Downloaded 34GiB to GPU in 1.98499s (17.1285GiB/s) | 20260507 07:49:09.572 | INFO | Analyzing chunk frames = [8293888,8294400) | 20260507 07:49:09.572 | INFO | MemInfo: { | 20260507 07:49:09.572 | INFO | "gpu" : " 17 GB", | 20260507 07:49:09.572 | INFO | "huge" : "", | 20260507 07:49:09.572 | INFO | "huge_shared" : " 27 GB", | 20260507 07:49:09.572 | INFO | "malloc" : " 14 GB", | 20260507 07:49:09.572 | INFO | "pinned" : " 29 GB" | 20260507 07:49:09.572 | INFO | } | 20260507 07:49:09.572 | INFO | Perf report for the last 5.13959 seconds: | 20260507 07:49:09.572 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:49:09.572 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:09.572 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:09.572 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:09.572 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:49:09.572 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:09.572 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:09.572 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:49:09.572 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:09.572 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:09.572 | INFO | Uploaded 36GiB to GPU in 1.98995s (18.0909GiB/s) | 20260507 07:49:09.572 | INFO | Downloaded 34GiB to GPU in 1.9585s (17.3603GiB/s) | 20260507 07:49:09.669 | INFO | Starting a round of Baz IO | 20260507 07:49:12.758 | INFO | Populating RTMetrics report | 20260507 07:49:12.759 | INFO | RTMetrics report populated | 20260507 07:49:12.759 | INFO | Opening tmp RTMetrics json file | 20260507 07:49:12.764 | INFO | Serializing RTMetrics to json | 20260507 07:49:12.777 | INFO | Serialization done, swapping tmp file now... | 20260507 07:49:12.777 | INFO | RTMetrics done | 20260507 07:49:12.846 | INFO | Finished a round of Baz IO: Wrote 1.629 GiB in 3.177 seconds | 20260507 07:49:12.846 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.53% / 0.00% / 67.46% | 20260507 07:49:12.846 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0990629% / 3.97364e-06% / 7.15256e-05% | 20260507 07:49:12.847 | INFO | Baz IO overall: | 20260507 07:49:12.847 | INFO | BytesWritten : 4147909116643 | 20260507 07:49:12.847 | INFO | NumEvents : 1985240268330 | 20260507 07:49:14.713 | INFO | Analyzing chunk frames = [8294400,8294912) | 20260507 07:49:14.714 | INFO | MemInfo: { | 20260507 07:49:14.714 | INFO | "gpu" : " 17 GB", | 20260507 07:49:14.714 | INFO | "huge" : "", | 20260507 07:49:14.714 | INFO | "huge_shared" : " 27 GB", | 20260507 07:49:14.714 | INFO | "malloc" : " 14 GB", | 20260507 07:49:14.714 | INFO | "pinned" : " 29 GB" | 20260507 07:49:14.714 | INFO | } | 20260507 07:49:14.714 | INFO | Perf report for the last 5.14194 seconds: | 20260507 07:49:14.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:49:14.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:14.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:14.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:14.714 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 07:49:14.714 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:14.714 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 07:49:14.714 | INFO | PRE_HQ : 58.9%, 1.8, 54.3ms, 0 | 20260507 07:49:14.714 | INFO | BAZWRITER : 61.4%, 3.1, 99.1ms, 0 | 20260507 07:49:14.714 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:49:14.714 | INFO | Uploaded 36GiB to GPU in 2.05653s (17.5052GiB/s) | 20260507 07:49:14.714 | INFO | Downloaded 38GiB to GPU in 2.17483s (17.4726GiB/s) | 20260507 07:49:19.852 | INFO | Analyzing chunk frames = [8294912,8295424) | 20260507 07:49:19.852 | INFO | MemInfo: { | 20260507 07:49:19.852 | INFO | "gpu" : " 17 GB", | 20260507 07:49:19.852 | INFO | "huge" : "", | 20260507 07:49:19.852 | INFO | "huge_shared" : " 27 GB", | 20260507 07:49:19.852 | INFO | "malloc" : " 14 GB", | 20260507 07:49:19.852 | INFO | "pinned" : " 29 GB" | 20260507 07:49:19.852 | INFO | } | 20260507 07:49:19.852 | INFO | Perf report for the last 5.13846 seconds: | 20260507 07:49:19.852 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:49:19.852 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:19.852 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:19.852 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:19.852 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:49:19.852 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:19.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:19.852 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:49:19.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:19.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:19.853 | INFO | Uploaded 36GiB to GPU in 1.99402s (18.0539GiB/s) | 20260507 07:49:19.853 | INFO | Downloaded 34GiB to GPU in 1.94086s (17.518GiB/s) | 20260507 07:49:24.986 | INFO | Analyzing chunk frames = [8295424,8295936) | 20260507 07:49:24.986 | INFO | MemInfo: { | 20260507 07:49:24.986 | INFO | "gpu" : " 17 GB", | 20260507 07:49:24.986 | INFO | "huge" : "", | 20260507 07:49:24.986 | INFO | "huge_shared" : " 27 GB", | 20260507 07:49:24.986 | INFO | "malloc" : " 14 GB", | 20260507 07:49:24.986 | INFO | "pinned" : " 29 GB" | 20260507 07:49:24.986 | INFO | } | 20260507 07:49:24.986 | INFO | Perf report for the last 5.13341 seconds: | 20260507 07:49:24.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:49:24.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:24.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:24.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:24.986 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:49:24.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:24.986 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:24.986 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:49:24.986 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:24.986 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:24.986 | INFO | Uploaded 36GiB to GPU in 2.03004s (17.7337GiB/s) | 20260507 07:49:24.986 | INFO | Downloaded 34GiB to GPU in 1.91656s (17.7401GiB/s) | 20260507 07:49:30.128 | INFO | Analyzing chunk frames = [8295936,8296448) | 20260507 07:49:30.128 | INFO | MemInfo: { | 20260507 07:49:30.128 | INFO | "gpu" : " 17 GB", | 20260507 07:49:30.128 | INFO | "huge" : "", | 20260507 07:49:30.128 | INFO | "huge_shared" : " 27 GB", | 20260507 07:49:30.128 | INFO | "malloc" : " 14 GB", | 20260507 07:49:30.128 | INFO | "pinned" : " 29 GB" | 20260507 07:49:30.128 | INFO | } | 20260507 07:49:30.128 | INFO | Perf report for the last 5.14213 seconds: | 20260507 07:49:30.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:49:30.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:30.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:30.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:30.128 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 07:49:30.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:30.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:30.128 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:49:30.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:30.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:30.128 | INFO | Uploaded 36GiB to GPU in 2.01459s (17.8697GiB/s) | 20260507 07:49:30.128 | INFO | Downloaded 34GiB to GPU in 1.94125s (17.5145GiB/s) | 20260507 07:49:35.277 | INFO | Analyzing chunk frames = [8296448,8296960) | 20260507 07:49:35.278 | INFO | MemInfo: { | 20260507 07:49:35.278 | INFO | "gpu" : " 17 GB", | 20260507 07:49:35.278 | INFO | "huge" : "", | 20260507 07:49:35.278 | INFO | "huge_shared" : " 27 GB", | 20260507 07:49:35.278 | INFO | "malloc" : " 14 GB", | 20260507 07:49:35.278 | INFO | "pinned" : " 29 GB" | 20260507 07:49:35.278 | INFO | } | 20260507 07:49:35.278 | INFO | Perf report for the last 5.15012 seconds: | 20260507 07:49:35.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:49:35.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:35.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:35.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:35.278 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 07:49:35.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:35.278 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:35.278 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:49:35.278 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:35.278 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:35.279 | INFO | Uploaded 36GiB to GPU in 2.04954s (17.565GiB/s) | 20260507 07:49:35.279 | INFO | Downloaded 34GiB to GPU in 1.90817s (17.8181GiB/s) | 20260507 07:49:40.407 | INFO | Analyzing chunk frames = [8296960,8297472) | 20260507 07:49:40.407 | INFO | MemInfo: { | 20260507 07:49:40.407 | INFO | "gpu" : " 17 GB", | 20260507 07:49:40.407 | INFO | "huge" : "", | 20260507 07:49:40.407 | INFO | "huge_shared" : " 27 GB", | 20260507 07:49:40.407 | INFO | "malloc" : " 14 GB", | 20260507 07:49:40.407 | INFO | "pinned" : " 29 GB" | 20260507 07:49:40.407 | INFO | } | 20260507 07:49:40.407 | INFO | Perf report for the last 5.12919 seconds: | 20260507 07:49:40.407 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:49:40.407 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:40.407 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:40.407 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:40.407 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260507 07:49:40.407 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:40.407 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:40.407 | INFO | PRE_HQ : 26.5%, 1.0, 13.7ms, 0 | 20260507 07:49:40.407 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:40.407 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:40.407 | INFO | Uploaded 36GiB to GPU in 2.03708s (17.6724GiB/s) | 20260507 07:49:40.407 | INFO | Downloaded 34GiB to GPU in 1.90813s (17.8185GiB/s) | 20260507 07:49:45.539 | INFO | Analyzing chunk frames = [8297472,8297984) | 20260507 07:49:45.539 | INFO | MemInfo: { | 20260507 07:49:45.539 | INFO | "gpu" : " 17 GB", | 20260507 07:49:45.539 | INFO | "huge" : "", | 20260507 07:49:45.539 | INFO | "huge_shared" : " 27 GB", | 20260507 07:49:45.539 | INFO | "malloc" : " 14 GB", | 20260507 07:49:45.539 | INFO | "pinned" : " 29 GB" | 20260507 07:49:45.539 | INFO | } | 20260507 07:49:45.540 | INFO | Perf report for the last 5.13237 seconds: | 20260507 07:49:45.540 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:49:45.540 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:45.540 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:45.540 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:45.540 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260507 07:49:45.540 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:45.540 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:45.540 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:49:45.540 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:45.540 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:45.540 | INFO | Uploaded 36GiB to GPU in 1.9024s (18.9235GiB/s) | 20260507 07:49:45.540 | INFO | Downloaded 34GiB to GPU in 1.97422s (17.222GiB/s) | 20260507 07:49:50.686 | INFO | Analyzing chunk frames = [8297984,8298496) | 20260507 07:49:50.686 | INFO | MemInfo: { | 20260507 07:49:50.686 | INFO | "gpu" : " 17 GB", | 20260507 07:49:50.686 | INFO | "huge" : "", | 20260507 07:49:50.686 | INFO | "huge_shared" : " 27 GB", | 20260507 07:49:50.686 | INFO | "malloc" : " 14 GB", | 20260507 07:49:50.686 | INFO | "pinned" : " 29 GB" | 20260507 07:49:50.686 | INFO | } | 20260507 07:49:50.686 | INFO | Perf report for the last 5.14627 seconds: | 20260507 07:49:50.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:49:50.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:50.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:50.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:50.686 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:49:50.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:50.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:50.686 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 07:49:50.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:50.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:50.686 | INFO | Uploaded 36GiB to GPU in 1.96327s (18.3367GiB/s) | 20260507 07:49:50.686 | INFO | Downloaded 34GiB to GPU in 1.96316s (17.319GiB/s) | 20260507 07:49:50.686 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:49:50.686 | INFO | Total mem usage: 18 GiB | 20260507 07:49:50.686 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:49:50.686 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:49:50.686 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:49:50.686 | INFO | High Water Breakdown: | 20260507 07:49:50.686 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:49:50.686 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:49:50.686 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:49:50.686 | INFO | ------------------------------------------ | 20260507 07:49:50.686 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:49:50.686 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:49:50.686 | INFO | Total mem usage: 26.5 GiB | 20260507 07:49:50.686 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:49:50.686 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:49:50.687 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:49:50.687 | INFO | High Water Breakdown: | 20260507 07:49:50.687 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:49:50.687 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:49:50.687 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:49:50.687 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:49:50.687 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:49:50.687 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:49:50.687 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:49:50.687 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:49:50.687 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:49:50.687 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:49:50.687 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:49:50.687 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:49:50.687 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:49:50.687 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:49:50.687 | INFO | ------------------------------------------ | 20260507 07:49:50.687 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:49:50.687 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:49:50.687 | INFO | Total mem usage: 25.6 GiB | 20260507 07:49:50.687 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:49:50.687 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:49:50.687 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:49:50.687 | INFO | High Water Breakdown: | 20260507 07:49:50.687 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:49:50.687 | INFO | ------------------------------------------ | 20260507 07:49:50.687 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:49:50.687 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:49:50.687 | INFO | Total mem usage: 15.7 GiB | 20260507 07:49:50.687 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:49:50.687 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:49:50.687 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:49:50.687 | INFO | High Water Breakdown: | 20260507 07:49:50.687 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:49:50.687 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:49:50.687 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:49:50.687 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:49:50.687 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:49:50.687 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:49:50.687 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:49:50.687 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:49:50.687 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:49:50.687 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:49:50.687 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:49:50.687 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:49:50.687 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:49:50.687 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:49:50.687 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:49:50.687 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:49:50.687 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:49:50.687 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:49:50.687 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:49:50.687 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:49:50.687 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:49:50.687 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:49:50.687 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:49:50.687 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:49:50.687 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:49:50.687 | INFO | ------------------------------------------ | 20260507 07:49:50.687 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:49:50.687 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:49:50.687 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:49:50.687 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:49:50.687 | INFO | PerfReport: Full Execution -- 3.08e+01 / 7.46e+01, 1.33e+01 / 1.12e+00 | 20260507 07:49:50.687 | INFO | PerfReport: 0.30%, AggregateStats -- 9.10e-02 / 1.02e-01, 9.81e-03 / 6.95e-03 | 20260507 07:49:50.687 | INFO | PerfReport: 11.70%, Baseline -- 3.60e+00 / 4.45e+00, 5.23e-01 / 2.70e-01 | 20260507 07:49:50.687 | INFO | PerfReport: 3.24%, Histogram -- 9.97e-01 / 1.48e+00, 1.70e-01 / 1.08e-01 | 20260507 07:49:50.687 | INFO | PerfReport: 16.23%, DME -- 3.99e+01 / 4.45e+01, 3.89e+00 / 1.26e+00 | 20260507 07:49:50.687 | INFO | PerfReport: 57.78%, FrameLabeling -- 1.78e+01 / 1.97e+01, 1.39e+00 / 1.17e+00 | 20260507 07:49:50.687 | INFO | PerfReport: 6.86%, PulseAccumulating -- 2.11e+00 / 2.68e+00, 3.81e-01 / 1.95e-01 | 20260507 07:49:50.687 | INFO | PerfReport: 2.91%, Metrics -- 8.96e-01 / 1.21e+00, 2.40e-01 / 1.33e-01 | 20260507 07:49:50.687 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.01e-01 / 3.56e-01, 5.05e-02 / 1.76e-02 | 20260507 07:49:50.687 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:49:50.687 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:49:50.687 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:49:50.687 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 07:49:50.687 | INFO | PerfReport: 21.85%, Upload -- 2.10e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260507 07:49:50.687 | INFO | PerfReport: 21.37%, Download -- 2.05e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260507 07:49:50.688 | INFO | PerfReport: 32.02%, Compute -- 3.08e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 07:49:50.688 | INFO | PerfReport: 1.50%, LockUpload -- 1.44e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 07:49:50.688 | INFO | PerfReport: 1.24%, LockDownload -- 1.19e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260507 07:49:50.688 | INFO | PerfReport: 22.01%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 07:49:50.688 | INFO | PerfReport: 0.02%, Misc -- 1.50e-02 / 0.00e+00, 2.55e-02 / 0.00e+00 | 20260507 07:49:50.782 | INFO | Starting a round of Baz IO | 20260507 07:49:53.877 | INFO | Populating RTMetrics report | 20260507 07:49:53.877 | INFO | RTMetrics report populated | 20260507 07:49:53.877 | INFO | Opening tmp RTMetrics json file | 20260507 07:49:53.894 | INFO | Serializing RTMetrics to json | 20260507 07:49:53.895 | INFO | Serialization done, swapping tmp file now... | 20260507 07:49:53.896 | INFO | RTMetrics done | 20260507 07:49:53.962 | INFO | Finished a round of Baz IO: Wrote 1.633 GiB in 3.179 seconds | 20260507 07:49:53.962 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.53% / 0.00% / 67.47% | 20260507 07:49:53.962 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0999888% / 0% / 9.53674e-05% | 20260507 07:49:53.962 | INFO | Baz IO overall: | 20260507 07:49:53.962 | INFO | BytesWritten : 4149662696701 | 20260507 07:49:53.962 | INFO | NumEvents : 1986059806930 | 20260507 07:49:55.827 | INFO | Analyzing chunk frames = [8298496,8299008) | 20260507 07:49:55.827 | INFO | MemInfo: { | 20260507 07:49:55.827 | INFO | "gpu" : " 17 GB", | 20260507 07:49:55.827 | INFO | "huge" : "", | 20260507 07:49:55.827 | INFO | "huge_shared" : " 27 GB", | 20260507 07:49:55.827 | INFO | "malloc" : " 14 GB", | 20260507 07:49:55.827 | INFO | "pinned" : " 29 GB" | 20260507 07:49:55.827 | INFO | } | 20260507 07:49:55.827 | INFO | Perf report for the last 5.14142 seconds: | 20260507 07:49:55.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:49:55.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:55.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:55.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:55.827 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 07:49:55.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:49:55.827 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 07:49:55.827 | INFO | PRE_HQ : 59.6%, 1.9, 58.1ms, 0 | 20260507 07:49:55.827 | INFO | BAZWRITER : 61.5%, 3.3, 104.3ms, 0 | 20260507 07:49:55.827 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:49:55.828 | INFO | Uploaded 36GiB to GPU in 2.03585s (17.6831GiB/s) | 20260507 07:49:55.828 | INFO | Downloaded 38GiB to GPU in 2.17491s (17.472GiB/s) | 20260507 07:50:00.960 | INFO | Analyzing chunk frames = [8299008,8299520) | 20260507 07:50:00.960 | INFO | MemInfo: { | 20260507 07:50:00.960 | INFO | "gpu" : " 17 GB", | 20260507 07:50:00.960 | INFO | "huge" : "", | 20260507 07:50:00.960 | INFO | "huge_shared" : " 27 GB", | 20260507 07:50:00.960 | INFO | "malloc" : " 14 GB", | 20260507 07:50:00.960 | INFO | "pinned" : " 29 GB" | 20260507 07:50:00.960 | INFO | } | 20260507 07:50:00.960 | INFO | Perf report for the last 5.13305 seconds: | 20260507 07:50:00.960 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:50:00.960 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:00.960 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:00.960 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:00.960 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:50:00.960 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:00.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:00.960 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 07:50:00.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:00.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:00.961 | INFO | Uploaded 36GiB to GPU in 1.99186s (18.0735GiB/s) | 20260507 07:50:00.961 | INFO | Downloaded 34GiB to GPU in 1.93921s (17.5329GiB/s) | 20260507 07:50:06.099 | INFO | Analyzing chunk frames = [8299520,8300032) | 20260507 07:50:06.099 | INFO | MemInfo: { | 20260507 07:50:06.099 | INFO | "gpu" : " 17 GB", | 20260507 07:50:06.099 | INFO | "huge" : "", | 20260507 07:50:06.099 | INFO | "huge_shared" : " 27 GB", | 20260507 07:50:06.099 | INFO | "malloc" : " 14 GB", | 20260507 07:50:06.099 | INFO | "pinned" : " 29 GB" | 20260507 07:50:06.099 | INFO | } | 20260507 07:50:06.099 | INFO | Perf report for the last 5.13908 seconds: | 20260507 07:50:06.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:50:06.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:06.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:06.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:06.099 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 07:50:06.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:06.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:06.099 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:50:06.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:06.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:06.100 | INFO | Uploaded 36GiB to GPU in 2.04792s (17.5788GiB/s) | 20260507 07:50:06.100 | INFO | Downloaded 34GiB to GPU in 1.95265s (17.4122GiB/s) | 20260507 07:50:11.239 | INFO | Analyzing chunk frames = [8300032,8300544) | 20260507 07:50:11.239 | INFO | MemInfo: { | 20260507 07:50:11.239 | INFO | "gpu" : " 17 GB", | 20260507 07:50:11.240 | INFO | "huge" : "", | 20260507 07:50:11.240 | INFO | "huge_shared" : " 27 GB", | 20260507 07:50:11.240 | INFO | "malloc" : " 14 GB", | 20260507 07:50:11.240 | INFO | "pinned" : " 29 GB" | 20260507 07:50:11.240 | INFO | } | 20260507 07:50:11.240 | INFO | Perf report for the last 5.14029 seconds: | 20260507 07:50:11.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:50:11.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:11.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:11.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:11.240 | INFO | ANALYSIS : 60.3%, 2.9, 93.2ms, 0 | 20260507 07:50:11.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:11.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:11.240 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:50:11.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:11.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:11.240 | INFO | Uploaded 36GiB to GPU in 2.02789s (17.7525GiB/s) | 20260507 07:50:11.240 | INFO | Downloaded 34GiB to GPU in 1.95584s (17.3838GiB/s) | 20260507 07:50:16.384 | INFO | Analyzing chunk frames = [8300544,8301056) | 20260507 07:50:16.384 | INFO | MemInfo: { | 20260507 07:50:16.384 | INFO | "gpu" : " 17 GB", | 20260507 07:50:16.384 | INFO | "huge" : "", | 20260507 07:50:16.384 | INFO | "huge_shared" : " 27 GB", | 20260507 07:50:16.384 | INFO | "malloc" : " 14 GB", | 20260507 07:50:16.384 | INFO | "pinned" : " 29 GB" | 20260507 07:50:16.384 | INFO | } | 20260507 07:50:16.384 | INFO | Perf report for the last 5.14411 seconds: | 20260507 07:50:16.384 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:50:16.384 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:16.384 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:16.384 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:16.384 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 07:50:16.384 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:16.384 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:16.384 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:50:16.384 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:16.384 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:16.384 | INFO | Uploaded 36GiB to GPU in 2.05289s (17.5363GiB/s) | 20260507 07:50:16.384 | INFO | Downloaded 34GiB to GPU in 1.95306s (17.4086GiB/s) | 20260507 07:50:21.528 | INFO | Analyzing chunk frames = [8301056,8301568) | 20260507 07:50:21.528 | INFO | MemInfo: { | 20260507 07:50:21.528 | INFO | "gpu" : " 17 GB", | 20260507 07:50:21.528 | INFO | "huge" : "", | 20260507 07:50:21.528 | INFO | "huge_shared" : " 27 GB", | 20260507 07:50:21.528 | INFO | "malloc" : " 14 GB", | 20260507 07:50:21.528 | INFO | "pinned" : " 29 GB" | 20260507 07:50:21.528 | INFO | } | 20260507 07:50:21.528 | INFO | Perf report for the last 5.14409 seconds: | 20260507 07:50:21.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:50:21.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:21.528 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:21.528 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:21.528 | INFO | ANALYSIS : 60.4%, 3.0, 94.4ms, 0 | 20260507 07:50:21.528 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:21.528 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:21.528 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:50:21.528 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:21.528 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:21.528 | INFO | Uploaded 36GiB to GPU in 2.06574s (17.4271GiB/s) | 20260507 07:50:21.528 | INFO | Downloaded 34GiB to GPU in 1.94604s (17.4713GiB/s) | 20260507 07:50:26.663 | INFO | Analyzing chunk frames = [8301568,8302080) | 20260507 07:50:26.663 | INFO | MemInfo: { | 20260507 07:50:26.663 | INFO | "gpu" : " 17 GB", | 20260507 07:50:26.663 | INFO | "huge" : "", | 20260507 07:50:26.663 | INFO | "huge_shared" : " 27 GB", | 20260507 07:50:26.663 | INFO | "malloc" : " 14 GB", | 20260507 07:50:26.663 | INFO | "pinned" : " 29 GB" | 20260507 07:50:26.663 | INFO | } | 20260507 07:50:26.663 | INFO | Perf report for the last 5.13541 seconds: | 20260507 07:50:26.663 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:50:26.663 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:26.663 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:26.663 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:26.663 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 07:50:26.663 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:26.664 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:26.664 | INFO | PRE_HQ : 25.2%, 1.0, 13.0ms, 0 | 20260507 07:50:26.664 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:26.664 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:26.664 | INFO | Uploaded 36GiB to GPU in 1.96558s (18.3152GiB/s) | 20260507 07:50:26.664 | INFO | Downloaded 34GiB to GPU in 1.97326s (17.2303GiB/s) | 20260507 07:50:31.800 | INFO | Analyzing chunk frames = [8302080,8302592) | 20260507 07:50:31.800 | INFO | MemInfo: { | 20260507 07:50:31.800 | INFO | "gpu" : " 17 GB", | 20260507 07:50:31.800 | INFO | "huge" : "", | 20260507 07:50:31.800 | INFO | "huge_shared" : " 27 GB", | 20260507 07:50:31.800 | INFO | "malloc" : " 14 GB", | 20260507 07:50:31.800 | INFO | "pinned" : " 29 GB" | 20260507 07:50:31.800 | INFO | } | 20260507 07:50:31.800 | INFO | Perf report for the last 5.13644 seconds: | 20260507 07:50:31.800 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:50:31.800 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:31.800 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:31.800 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:31.800 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:50:31.800 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:31.800 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:31.800 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 07:50:31.800 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:31.800 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:31.800 | INFO | Uploaded 36GiB to GPU in 1.97949s (18.1865GiB/s) | 20260507 07:50:31.800 | INFO | Downloaded 34GiB to GPU in 1.96071s (17.3407GiB/s) | 20260507 07:50:31.897 | INFO | Starting a round of Baz IO | 20260507 07:50:34.990 | INFO | Populating RTMetrics report | 20260507 07:50:34.991 | INFO | RTMetrics report populated | 20260507 07:50:34.991 | INFO | Opening tmp RTMetrics json file | 20260507 07:50:35.004 | INFO | Serializing RTMetrics to json | 20260507 07:50:35.009 | INFO | Serialization done, swapping tmp file now... | 20260507 07:50:35.010 | INFO | RTMetrics done | 20260507 07:50:35.077 | INFO | Finished a round of Baz IO: Wrote 1.63 GiB in 3.179 seconds | 20260507 07:50:35.077 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.53% / 0.00% / 67.47% | 20260507 07:50:35.077 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0990828% / 0% / 8.74201e-05% | 20260507 07:50:35.077 | INFO | Baz IO overall: | 20260507 07:50:35.077 | INFO | BytesWritten : 4151412906570 | 20260507 07:50:35.077 | INFO | NumEvents : 1986876594439 | 20260507 07:50:36.938 | INFO | Analyzing chunk frames = [8302592,8303104) | 20260507 07:50:36.938 | INFO | MemInfo: { | 20260507 07:50:36.938 | INFO | "gpu" : " 17 GB", | 20260507 07:50:36.938 | INFO | "huge" : "", | 20260507 07:50:36.938 | INFO | "huge_shared" : " 27 GB", | 20260507 07:50:36.938 | INFO | "malloc" : " 14 GB", | 20260507 07:50:36.938 | INFO | "pinned" : " 29 GB" | 20260507 07:50:36.938 | INFO | } | 20260507 07:50:36.938 | INFO | Perf report for the last 5.13792 seconds: | 20260507 07:50:36.938 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:50:36.938 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:36.938 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:36.938 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:36.938 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 07:50:36.938 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:36.938 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 07:50:36.938 | INFO | PRE_HQ : 59.5%, 1.9, 58.1ms, 0 | 20260507 07:50:36.938 | INFO | BAZWRITER : 61.5%, 3.3, 103.3ms, 0 | 20260507 07:50:36.938 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:50:36.938 | INFO | Uploaded 36GiB to GPU in 2.03388s (17.7001GiB/s) | 20260507 07:50:36.938 | INFO | Downloaded 38GiB to GPU in 2.17839s (17.444GiB/s) | 20260507 07:50:42.080 | INFO | Analyzing chunk frames = [8303104,8303616) | 20260507 07:50:42.080 | INFO | MemInfo: { | 20260507 07:50:42.080 | INFO | "gpu" : " 17 GB", | 20260507 07:50:42.080 | INFO | "huge" : "", | 20260507 07:50:42.080 | INFO | "huge_shared" : " 27 GB", | 20260507 07:50:42.080 | INFO | "malloc" : " 14 GB", | 20260507 07:50:42.080 | INFO | "pinned" : " 29 GB" | 20260507 07:50:42.080 | INFO | } | 20260507 07:50:42.080 | INFO | Perf report for the last 5.14248 seconds: | 20260507 07:50:42.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:50:42.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:42.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:42.080 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:42.080 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 07:50:42.080 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:42.080 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:42.080 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 07:50:42.080 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:42.080 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:42.081 | INFO | Uploaded 36GiB to GPU in 1.98089s (18.1736GiB/s) | 20260507 07:50:42.081 | INFO | Downloaded 34GiB to GPU in 1.92854s (17.6299GiB/s) | 20260507 07:50:47.228 | INFO | Analyzing chunk frames = [8303616,8304128) | 20260507 07:50:47.228 | INFO | MemInfo: { | 20260507 07:50:47.228 | INFO | "gpu" : " 17 GB", | 20260507 07:50:47.228 | INFO | "huge" : "", | 20260507 07:50:47.228 | INFO | "huge_shared" : " 27 GB", | 20260507 07:50:47.228 | INFO | "malloc" : " 14 GB", | 20260507 07:50:47.228 | INFO | "pinned" : " 29 GB" | 20260507 07:50:47.228 | INFO | } | 20260507 07:50:47.228 | INFO | Perf report for the last 5.14808 seconds: | 20260507 07:50:47.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:50:47.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:47.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:47.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:47.228 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 07:50:47.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:47.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:47.228 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:50:47.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:47.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:47.229 | INFO | Uploaded 36GiB to GPU in 2.05378s (17.5287GiB/s) | 20260507 07:50:47.229 | INFO | Downloaded 34GiB to GPU in 1.95468s (17.3941GiB/s) | 20260507 07:50:52.358 | INFO | Analyzing chunk frames = [8304128,8304640) | 20260507 07:50:52.358 | INFO | MemInfo: { | 20260507 07:50:52.358 | INFO | "gpu" : " 17 GB", | 20260507 07:50:52.358 | INFO | "huge" : "", | 20260507 07:50:52.358 | INFO | "huge_shared" : " 27 GB", | 20260507 07:50:52.358 | INFO | "malloc" : " 14 GB", | 20260507 07:50:52.358 | INFO | "pinned" : " 29 GB" | 20260507 07:50:52.358 | INFO | } | 20260507 07:50:52.359 | INFO | Perf report for the last 5.13019 seconds: | 20260507 07:50:52.359 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:50:52.359 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:52.359 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:52.359 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:52.359 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 07:50:52.359 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:52.359 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:52.359 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:50:52.359 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:52.359 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:52.359 | INFO | Uploaded 36GiB to GPU in 2.03618s (17.6802GiB/s) | 20260507 07:50:52.359 | INFO | Downloaded 34GiB to GPU in 1.91924s (17.7154GiB/s) | 20260507 07:50:57.495 | INFO | Analyzing chunk frames = [8304640,8305152) | 20260507 07:50:57.495 | INFO | MemInfo: { | 20260507 07:50:57.495 | INFO | "gpu" : " 17 GB", | 20260507 07:50:57.495 | INFO | "huge" : "", | 20260507 07:50:57.495 | INFO | "huge_shared" : " 27 GB", | 20260507 07:50:57.495 | INFO | "malloc" : " 14 GB", | 20260507 07:50:57.495 | INFO | "pinned" : " 29 GB" | 20260507 07:50:57.495 | INFO | } | 20260507 07:50:57.495 | INFO | Perf report for the last 5.13685 seconds: | 20260507 07:50:57.495 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:50:57.495 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:57.495 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:57.495 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:57.496 | INFO | ANALYSIS : 59.9%, 3.0, 93.3ms, 0 | 20260507 07:50:57.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:57.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:57.496 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:50:57.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:57.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:50:57.496 | INFO | Uploaded 36GiB to GPU in 2.0296s (17.7375GiB/s) | 20260507 07:50:57.496 | INFO | Downloaded 34GiB to GPU in 1.91562s (17.7488GiB/s) | 20260507 07:51:02.637 | INFO | Analyzing chunk frames = [8305152,8305664) | 20260507 07:51:02.637 | INFO | MemInfo: { | 20260507 07:51:02.637 | INFO | "gpu" : " 17 GB", | 20260507 07:51:02.637 | INFO | "huge" : "", | 20260507 07:51:02.637 | INFO | "huge_shared" : " 27 GB", | 20260507 07:51:02.637 | INFO | "malloc" : " 14 GB", | 20260507 07:51:02.637 | INFO | "pinned" : " 29 GB" | 20260507 07:51:02.637 | INFO | } | 20260507 07:51:02.637 | INFO | Perf report for the last 5.14175 seconds: | 20260507 07:51:02.637 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:51:02.637 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:02.637 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:02.637 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:02.637 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 07:51:02.637 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:02.637 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:02.637 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:51:02.637 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:02.637 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:02.637 | INFO | Uploaded 36GiB to GPU in 2.04024s (17.645GiB/s) | 20260507 07:51:02.638 | INFO | Downloaded 34GiB to GPU in 1.91105s (17.7913GiB/s) | 20260507 07:51:07.776 | INFO | Analyzing chunk frames = [8305664,8306176) | 20260507 07:51:07.776 | INFO | MemInfo: { | 20260507 07:51:07.777 | INFO | "gpu" : " 17 GB", | 20260507 07:51:07.777 | INFO | "huge" : "", | 20260507 07:51:07.777 | INFO | "huge_shared" : " 27 GB", | 20260507 07:51:07.777 | INFO | "malloc" : " 14 GB", | 20260507 07:51:07.777 | INFO | "pinned" : " 29 GB" | 20260507 07:51:07.777 | INFO | } | 20260507 07:51:07.777 | INFO | Perf report for the last 5.13947 seconds: | 20260507 07:51:07.777 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:51:07.777 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:07.777 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:07.777 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:07.777 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 07:51:07.777 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:07.777 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:07.777 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:51:07.777 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:07.777 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:07.777 | INFO | Uploaded 36GiB to GPU in 1.95056s (18.4562GiB/s) | 20260507 07:51:07.777 | INFO | Downloaded 34GiB to GPU in 1.96639s (17.2905GiB/s) | 20260507 07:51:12.911 | INFO | Analyzing chunk frames = [8306176,8306688) | 20260507 07:51:12.911 | INFO | MemInfo: { | 20260507 07:51:12.911 | INFO | "gpu" : " 17 GB", | 20260507 07:51:12.911 | INFO | "huge" : "", | 20260507 07:51:12.911 | INFO | "huge_shared" : " 27 GB", | 20260507 07:51:12.911 | INFO | "malloc" : " 14 GB", | 20260507 07:51:12.911 | INFO | "pinned" : " 29 GB" | 20260507 07:51:12.911 | INFO | } | 20260507 07:51:12.912 | INFO | Perf report for the last 5.13483 seconds: | 20260507 07:51:12.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:51:12.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:12.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:12.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:12.912 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:51:12.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:12.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:12.912 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:51:12.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:12.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:12.912 | INFO | Uploaded 36GiB to GPU in 1.9919s (18.0732GiB/s) | 20260507 07:51:12.912 | INFO | Downloaded 34GiB to GPU in 1.95762s (17.368GiB/s) | 20260507 07:51:12.912 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:51:12.912 | INFO | Total mem usage: 18 GiB | 20260507 07:51:12.912 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:51:12.912 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 07:51:12.912 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:51:12.912 | INFO | High Water Breakdown: | 20260507 07:51:12.912 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:51:12.912 | INFO | 39.7% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:51:12.912 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:51:12.912 | INFO | ------------------------------------------ | 20260507 07:51:12.912 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:51:12.912 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:51:12.912 | INFO | Total mem usage: 26.5 GiB | 20260507 07:51:12.912 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:51:12.912 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:51:12.912 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:51:12.912 | INFO | High Water Breakdown: | 20260507 07:51:12.912 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:51:12.912 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:51:12.912 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:51:12.912 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:51:12.912 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:51:12.912 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:51:12.912 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:51:12.912 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:51:12.912 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:51:12.912 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:51:12.912 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:51:12.912 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:51:12.912 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:51:12.912 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:51:12.912 | INFO | ------------------------------------------ | 20260507 07:51:12.912 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:51:12.912 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:51:12.912 | INFO | Total mem usage: 25.6 GiB | 20260507 07:51:12.912 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:51:12.912 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:51:12.912 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:51:12.912 | INFO | High Water Breakdown: | 20260507 07:51:12.912 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:51:12.912 | INFO | ------------------------------------------ | 20260507 07:51:12.912 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:51:12.912 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:51:12.912 | INFO | Total mem usage: 15.7 GiB | 20260507 07:51:12.912 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:51:12.912 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:51:12.912 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:51:12.912 | INFO | High Water Breakdown: | 20260507 07:51:12.912 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:51:12.912 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:51:12.912 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:51:12.912 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:51:12.912 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:51:12.912 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:51:12.912 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:51:12.912 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:51:12.912 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:51:12.912 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:51:12.913 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:51:12.913 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:51:12.913 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:51:12.913 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:51:12.913 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:51:12.913 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:51:12.913 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:51:12.913 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:51:12.913 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:51:12.913 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:51:12.913 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:51:12.913 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:51:12.913 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:51:12.913 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:51:12.913 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:51:12.913 | INFO | ------------------------------------------ | 20260507 07:51:12.913 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:51:12.913 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:51:12.913 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:51:12.913 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:51:12.913 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.35e+01 / 1.05e+00 | 20260507 07:51:12.913 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.03e-01, 1.02e-02 / 7.31e-03 | 20260507 07:51:12.913 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.50e+00, 5.00e-01 / 2.16e-01 | 20260507 07:51:12.913 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.50e+00, 1.67e-01 / 9.69e-02 | 20260507 07:51:12.913 | INFO | PerfReport: 16.35%, DME -- 4.01e+01 / 4.53e+01, 4.00e+00 / 1.32e+00 | 20260507 07:51:12.913 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.77e+01 / 1.92e+01, 1.26e+00 / 8.59e-01 | 20260507 07:51:12.913 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.69e+00, 3.85e-01 / 1.67e-01 | 20260507 07:51:12.913 | INFO | PerfReport: 2.92%, Metrics -- 8.97e-01 / 1.20e+00, 2.39e-01 / 1.17e-01 | 20260507 07:51:12.913 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.01e-01 / 3.58e-01, 5.11e-02 / 2.29e-02 | 20260507 07:51:12.913 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:51:12.913 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:51:12.913 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:51:12.913 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 07:51:12.913 | INFO | PerfReport: 21.86%, Upload -- 2.10e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260507 07:51:12.913 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260507 07:51:12.913 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 07:51:12.913 | INFO | PerfReport: 1.47%, LockUpload -- 1.41e+00 / 0.00e+00, 4.86e+00 / 0.00e+00 | 20260507 07:51:12.913 | INFO | PerfReport: 1.34%, LockDownload -- 1.28e+00 / 0.00e+00, 3.08e+00 / 0.00e+00 | 20260507 07:51:12.913 | INFO | PerfReport: 22.01%, LockCompute -- 2.12e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 07:51:12.913 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 1.95e-02 / 0.00e+00 | 20260507 07:51:13.010 | INFO | Starting a round of Baz IO | 20260507 07:51:16.101 | INFO | Populating RTMetrics report | 20260507 07:51:16.101 | INFO | RTMetrics report populated | 20260507 07:51:16.101 | INFO | Opening tmp RTMetrics json file | 20260507 07:51:16.120 | INFO | Serializing RTMetrics to json | 20260507 07:51:16.121 | INFO | Serialization done, swapping tmp file now... | 20260507 07:51:16.122 | INFO | RTMetrics done | 20260507 07:51:16.190 | INFO | Finished a round of Baz IO: Wrote 1.63 GiB in 3.179 seconds | 20260507 07:51:16.190 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.53% / 0.00% / 67.47% | 20260507 07:51:16.190 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0988404% / 0% / 9.53674e-05% | 20260507 07:51:16.190 | INFO | Baz IO overall: | 20260507 07:51:16.190 | INFO | BytesWritten : 4153162854880 | 20260507 07:51:16.190 | INFO | NumEvents : 1987693141440 | 20260507 07:51:18.047 | INFO | Analyzing chunk frames = [8306688,8307200) | 20260507 07:51:18.047 | INFO | MemInfo: { | 20260507 07:51:18.047 | INFO | "gpu" : " 17 GB", | 20260507 07:51:18.047 | INFO | "huge" : "", | 20260507 07:51:18.047 | INFO | "huge_shared" : " 27 GB", | 20260507 07:51:18.047 | INFO | "malloc" : " 14 GB", | 20260507 07:51:18.047 | INFO | "pinned" : " 29 GB" | 20260507 07:51:18.047 | INFO | } | 20260507 07:51:18.047 | INFO | Perf report for the last 5.13568 seconds: | 20260507 07:51:18.047 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:51:18.047 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:18.047 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:18.047 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:18.047 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 07:51:18.047 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:18.047 | INFO | RT_METRICS : 41.3%, 1.0, 21.6ms, 0 | 20260507 07:51:18.047 | INFO | PRE_HQ : 58.9%, 1.8, 54.5ms, 0 | 20260507 07:51:18.047 | INFO | BAZWRITER : 61.5%, 3.1, 98.9ms, 0 | 20260507 07:51:18.047 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 07:51:18.048 | INFO | Uploaded 36GiB to GPU in 2.06481s (17.435GiB/s) | 20260507 07:51:18.048 | INFO | Downloaded 38GiB to GPU in 2.18511s (17.3904GiB/s) | 20260507 07:51:23.192 | INFO | Analyzing chunk frames = [8307200,8307712) | 20260507 07:51:23.192 | INFO | MemInfo: { | 20260507 07:51:23.192 | INFO | "gpu" : " 17 GB", | 20260507 07:51:23.192 | INFO | "huge" : "", | 20260507 07:51:23.192 | INFO | "huge_shared" : " 27 GB", | 20260507 07:51:23.192 | INFO | "malloc" : " 14 GB", | 20260507 07:51:23.192 | INFO | "pinned" : " 29 GB" | 20260507 07:51:23.192 | INFO | } | 20260507 07:51:23.192 | INFO | Perf report for the last 5.14458 seconds: | 20260507 07:51:23.192 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:51:23.192 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:23.192 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:23.192 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:23.192 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 07:51:23.192 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:23.192 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:23.192 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 07:51:23.192 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:23.192 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:23.192 | INFO | Uploaded 36GiB to GPU in 2.0049s (17.956GiB/s) | 20260507 07:51:23.192 | INFO | Downloaded 34GiB to GPU in 1.92933s (17.6227GiB/s) | 20260507 07:51:28.338 | INFO | Analyzing chunk frames = [8307712,8308224) | 20260507 07:51:28.338 | INFO | MemInfo: { | 20260507 07:51:28.338 | INFO | "gpu" : " 17 GB", | 20260507 07:51:28.338 | INFO | "huge" : "", | 20260507 07:51:28.338 | INFO | "huge_shared" : " 27 GB", | 20260507 07:51:28.338 | INFO | "malloc" : " 14 GB", | 20260507 07:51:28.338 | INFO | "pinned" : " 29 GB" | 20260507 07:51:28.338 | INFO | } | 20260507 07:51:28.338 | INFO | Perf report for the last 5.14657 seconds: | 20260507 07:51:28.338 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:51:28.338 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:28.338 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:28.338 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:28.338 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 07:51:28.338 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:28.338 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:28.338 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:51:28.338 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:28.338 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:28.339 | INFO | Uploaded 36GiB to GPU in 2.05043s (17.5573GiB/s) | 20260507 07:51:28.339 | INFO | Downloaded 34GiB to GPU in 1.94925s (17.4426GiB/s) | 20260507 07:51:33.476 | INFO | Analyzing chunk frames = [8308224,8308736) | 20260507 07:51:33.476 | INFO | MemInfo: { | 20260507 07:51:33.476 | INFO | "gpu" : " 17 GB", | 20260507 07:51:33.476 | INFO | "huge" : "", | 20260507 07:51:33.476 | INFO | "huge_shared" : " 27 GB", | 20260507 07:51:33.476 | INFO | "malloc" : " 14 GB", | 20260507 07:51:33.476 | INFO | "pinned" : " 29 GB" | 20260507 07:51:33.476 | INFO | } | 20260507 07:51:33.476 | INFO | Perf report for the last 5.13806 seconds: | 20260507 07:51:33.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:51:33.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:33.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:33.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:33.477 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 07:51:33.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:33.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:33.477 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:51:33.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:33.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:33.477 | INFO | Uploaded 36GiB to GPU in 2.04129s (17.6359GiB/s) | 20260507 07:51:33.477 | INFO | Downloaded 34GiB to GPU in 1.91473s (17.757GiB/s) | 20260507 07:51:38.607 | INFO | Analyzing chunk frames = [8308736,8309248) | 20260507 07:51:38.607 | INFO | MemInfo: { | 20260507 07:51:38.607 | INFO | "gpu" : " 17 GB", | 20260507 07:51:38.607 | INFO | "huge" : "", | 20260507 07:51:38.607 | INFO | "huge_shared" : " 27 GB", | 20260507 07:51:38.607 | INFO | "malloc" : " 14 GB", | 20260507 07:51:38.607 | INFO | "pinned" : " 29 GB" | 20260507 07:51:38.607 | INFO | } | 20260507 07:51:38.607 | INFO | Perf report for the last 5.13057 seconds: | 20260507 07:51:38.607 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:51:38.607 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:38.607 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:38.607 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:38.607 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 07:51:38.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:38.607 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:38.607 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:51:38.607 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:38.607 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:38.607 | INFO | Uploaded 36GiB to GPU in 2.06451s (17.4376GiB/s) | 20260507 07:51:38.607 | INFO | Downloaded 34GiB to GPU in 1.9074s (17.8253GiB/s) | 20260507 07:51:43.754 | INFO | Analyzing chunk frames = [8309248,8309760) | 20260507 07:51:43.754 | INFO | MemInfo: { | 20260507 07:51:43.754 | INFO | "gpu" : " 17 GB", | 20260507 07:51:43.754 | INFO | "huge" : "", | 20260507 07:51:43.754 | INFO | "huge_shared" : " 27 GB", | 20260507 07:51:43.754 | INFO | "malloc" : " 14 GB", | 20260507 07:51:43.754 | INFO | "pinned" : " 29 GB" | 20260507 07:51:43.754 | INFO | } | 20260507 07:51:43.754 | INFO | Perf report for the last 5.14729 seconds: | 20260507 07:51:43.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:51:43.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:43.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:43.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:43.754 | INFO | ANALYSIS : 60.2%, 3.0, 94.1ms, 0 | 20260507 07:51:43.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:43.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:43.754 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:51:43.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:43.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:43.755 | INFO | Uploaded 36GiB to GPU in 2.03966s (17.65GiB/s) | 20260507 07:51:43.755 | INFO | Downloaded 34GiB to GPU in 1.90863s (17.8138GiB/s) | 20260507 07:51:48.886 | INFO | Analyzing chunk frames = [8309760,8310272) | 20260507 07:51:48.886 | INFO | MemInfo: { | 20260507 07:51:48.886 | INFO | "gpu" : " 17 GB", | 20260507 07:51:48.886 | INFO | "huge" : "", | 20260507 07:51:48.886 | INFO | "huge_shared" : " 27 GB", | 20260507 07:51:48.886 | INFO | "malloc" : " 14 GB", | 20260507 07:51:48.886 | INFO | "pinned" : " 29 GB" | 20260507 07:51:48.886 | INFO | } | 20260507 07:51:48.887 | INFO | Perf report for the last 5.13225 seconds: | 20260507 07:51:48.887 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:51:48.887 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:48.887 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:48.887 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:48.887 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 07:51:48.887 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:48.887 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:48.887 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 07:51:48.887 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:48.887 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:48.887 | INFO | Uploaded 36GiB to GPU in 1.91589s (18.7902GiB/s) | 20260507 07:51:48.887 | INFO | Downloaded 34GiB to GPU in 1.9782s (17.1873GiB/s) | 20260507 07:51:54.033 | INFO | Analyzing chunk frames = [8310272,8310784) | 20260507 07:51:54.033 | INFO | MemInfo: { | 20260507 07:51:54.033 | INFO | "gpu" : " 17 GB", | 20260507 07:51:54.033 | INFO | "huge" : "", | 20260507 07:51:54.033 | INFO | "huge_shared" : " 27 GB", | 20260507 07:51:54.033 | INFO | "malloc" : " 14 GB", | 20260507 07:51:54.033 | INFO | "pinned" : " 29 GB" | 20260507 07:51:54.033 | INFO | } | 20260507 07:51:54.033 | INFO | Perf report for the last 5.14676 seconds: | 20260507 07:51:54.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:51:54.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:54.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:54.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:54.033 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 07:51:54.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:54.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:54.033 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 07:51:54.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:54.034 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:54.034 | INFO | Uploaded 36GiB to GPU in 1.9707s (18.2676GiB/s) | 20260507 07:51:54.034 | INFO | Downloaded 34GiB to GPU in 1.96232s (17.3264GiB/s) | 20260507 07:51:54.131 | INFO | Starting a round of Baz IO | 20260507 07:51:57.221 | INFO | Populating RTMetrics report | 20260507 07:51:57.221 | INFO | RTMetrics report populated | 20260507 07:51:57.221 | INFO | Opening tmp RTMetrics json file | 20260507 07:51:57.231 | INFO | Serializing RTMetrics to json | 20260507 07:51:57.244 | INFO | Serialization done, swapping tmp file now... | 20260507 07:51:57.245 | INFO | RTMetrics done | 20260507 07:51:57.312 | INFO | Finished a round of Baz IO: Wrote 1.628 GiB in 3.181 seconds | 20260507 07:51:57.312 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.53% / 0.00% / 67.47% | 20260507 07:51:57.312 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0984629% / 3.97364e-06% / 9.53674e-05% | 20260507 07:51:57.312 | INFO | Baz IO overall: | 20260507 07:51:57.312 | INFO | BytesWritten : 4154911130752 | 20260507 07:51:57.312 | INFO | NumEvents : 1988508711861 | 20260507 07:51:59.172 | INFO | Analyzing chunk frames = [8310784,8311296) | 20260507 07:51:59.172 | INFO | MemInfo: { | 20260507 07:51:59.172 | INFO | "gpu" : " 17 GB", | 20260507 07:51:59.172 | INFO | "huge" : "", | 20260507 07:51:59.172 | INFO | "huge_shared" : " 27 GB", | 20260507 07:51:59.172 | INFO | "malloc" : " 14 GB", | 20260507 07:51:59.172 | INFO | "pinned" : " 29 GB" | 20260507 07:51:59.172 | INFO | } | 20260507 07:51:59.172 | INFO | Perf report for the last 5.13854 seconds: | 20260507 07:51:59.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:51:59.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:59.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:59.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:59.172 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 07:51:59.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:51:59.172 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:51:59.172 | INFO | PRE_HQ : 58.9%, 1.8, 54.6ms, 0 | 20260507 07:51:59.172 | INFO | BAZWRITER : 61.5%, 3.1, 99.2ms, 0 | 20260507 07:51:59.172 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 07:51:59.172 | INFO | Uploaded 36GiB to GPU in 2.06874s (17.4019GiB/s) | 20260507 07:51:59.172 | INFO | Downloaded 38GiB to GPU in 2.17606s (17.4628GiB/s) | 20260507 07:52:04.306 | INFO | Analyzing chunk frames = [8311296,8311808) | 20260507 07:52:04.306 | INFO | MemInfo: { | 20260507 07:52:04.306 | INFO | "gpu" : " 17 GB", | 20260507 07:52:04.306 | INFO | "huge" : "", | 20260507 07:52:04.306 | INFO | "huge_shared" : " 27 GB", | 20260507 07:52:04.306 | INFO | "malloc" : " 14 GB", | 20260507 07:52:04.307 | INFO | "pinned" : " 29 GB" | 20260507 07:52:04.307 | INFO | } | 20260507 07:52:04.307 | INFO | Perf report for the last 5.13466 seconds: | 20260507 07:52:04.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:52:04.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:04.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:04.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:04.307 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:52:04.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:04.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:04.307 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 07:52:04.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:04.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:04.307 | INFO | Uploaded 36GiB to GPU in 1.98574s (18.1293GiB/s) | 20260507 07:52:04.307 | INFO | Downloaded 34GiB to GPU in 1.94086s (17.518GiB/s) | 20260507 07:52:09.446 | INFO | Analyzing chunk frames = [8311808,8312320) | 20260507 07:52:09.446 | INFO | MemInfo: { | 20260507 07:52:09.446 | INFO | "gpu" : " 17 GB", | 20260507 07:52:09.446 | INFO | "huge" : "", | 20260507 07:52:09.446 | INFO | "huge_shared" : " 27 GB", | 20260507 07:52:09.446 | INFO | "malloc" : " 14 GB", | 20260507 07:52:09.446 | INFO | "pinned" : " 29 GB" | 20260507 07:52:09.446 | INFO | } | 20260507 07:52:09.447 | INFO | Perf report for the last 5.13999 seconds: | 20260507 07:52:09.447 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:52:09.447 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:09.447 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:09.447 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:09.447 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 07:52:09.447 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:09.447 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:09.447 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:52:09.447 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:09.447 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:09.447 | INFO | Uploaded 36GiB to GPU in 2.03952s (17.6512GiB/s) | 20260507 07:52:09.447 | INFO | Downloaded 34GiB to GPU in 1.95429s (17.3976GiB/s) | 20260507 07:52:14.587 | INFO | Analyzing chunk frames = [8312320,8312832) | 20260507 07:52:14.587 | INFO | MemInfo: { | 20260507 07:52:14.587 | INFO | "gpu" : " 17 GB", | 20260507 07:52:14.587 | INFO | "huge" : "", | 20260507 07:52:14.587 | INFO | "huge_shared" : " 27 GB", | 20260507 07:52:14.588 | INFO | "malloc" : " 14 GB", | 20260507 07:52:14.588 | INFO | "pinned" : " 29 GB" | 20260507 07:52:14.588 | INFO | } | 20260507 07:52:14.588 | INFO | Perf report for the last 5.141 seconds: | 20260507 07:52:14.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:52:14.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:14.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:14.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:14.588 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 07:52:14.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:14.588 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:14.588 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:52:14.588 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:14.588 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:14.588 | INFO | Uploaded 36GiB to GPU in 2.00303s (17.9728GiB/s) | 20260507 07:52:14.588 | INFO | Downloaded 34GiB to GPU in 1.93796s (17.5443GiB/s) | 20260507 07:52:19.730 | INFO | Analyzing chunk frames = [8312832,8313344) | 20260507 07:52:19.730 | INFO | MemInfo: { | 20260507 07:52:19.730 | INFO | "gpu" : " 17 GB", | 20260507 07:52:19.730 | INFO | "huge" : "", | 20260507 07:52:19.730 | INFO | "huge_shared" : " 27 GB", | 20260507 07:52:19.730 | INFO | "malloc" : " 14 GB", | 20260507 07:52:19.730 | INFO | "pinned" : " 29 GB" | 20260507 07:52:19.730 | INFO | } | 20260507 07:52:19.730 | INFO | Perf report for the last 5.14244 seconds: | 20260507 07:52:19.730 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:52:19.730 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:19.730 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:19.730 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:19.730 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 07:52:19.730 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:19.730 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:19.730 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:52:19.730 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:19.730 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:19.731 | INFO | Uploaded 36GiB to GPU in 2.02595s (17.7694GiB/s) | 20260507 07:52:19.731 | INFO | Downloaded 34GiB to GPU in 1.92184s (17.6914GiB/s) | 20260507 07:52:24.857 | INFO | Analyzing chunk frames = [8313344,8313856) | 20260507 07:52:24.857 | INFO | MemInfo: { | 20260507 07:52:24.857 | INFO | "gpu" : " 17 GB", | 20260507 07:52:24.857 | INFO | "huge" : "", | 20260507 07:52:24.857 | INFO | "huge_shared" : " 27 GB", | 20260507 07:52:24.857 | INFO | "malloc" : " 14 GB", | 20260507 07:52:24.857 | INFO | "pinned" : " 29 GB" | 20260507 07:52:24.857 | INFO | } | 20260507 07:52:24.857 | INFO | Perf report for the last 5.12677 seconds: | 20260507 07:52:24.857 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:52:24.857 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:24.857 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:24.857 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:24.857 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:52:24.857 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:24.857 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:24.857 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 07:52:24.857 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:24.857 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:24.857 | INFO | Uploaded 36GiB to GPU in 2.07033s (17.3885GiB/s) | 20260507 07:52:24.857 | INFO | Downloaded 34GiB to GPU in 1.94997s (17.4361GiB/s) | 20260507 07:52:30.002 | INFO | Analyzing chunk frames = [8313856,8314368) | 20260507 07:52:30.002 | INFO | MemInfo: { | 20260507 07:52:30.002 | INFO | "gpu" : " 17 GB", | 20260507 07:52:30.002 | INFO | "huge" : "", | 20260507 07:52:30.002 | INFO | "huge_shared" : " 27 GB", | 20260507 07:52:30.002 | INFO | "malloc" : " 14 GB", | 20260507 07:52:30.002 | INFO | "pinned" : " 29 GB" | 20260507 07:52:30.002 | INFO | } | 20260507 07:52:30.002 | INFO | Perf report for the last 5.14495 seconds: | 20260507 07:52:30.002 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:52:30.002 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:30.002 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:30.002 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:30.002 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 07:52:30.002 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:30.002 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:30.002 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:52:30.002 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:30.002 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:30.002 | INFO | Uploaded 36GiB to GPU in 1.96392s (18.3307GiB/s) | 20260507 07:52:30.002 | INFO | Downloaded 34GiB to GPU in 1.98133s (17.1602GiB/s) | 20260507 07:52:35.144 | INFO | Analyzing chunk frames = [8314368,8314880) | 20260507 07:52:35.144 | INFO | MemInfo: { | 20260507 07:52:35.144 | INFO | "gpu" : " 17 GB", | 20260507 07:52:35.144 | INFO | "huge" : "", | 20260507 07:52:35.144 | INFO | "huge_shared" : " 27 GB", | 20260507 07:52:35.144 | INFO | "malloc" : " 14 GB", | 20260507 07:52:35.144 | INFO | "pinned" : " 29 GB" | 20260507 07:52:35.144 | INFO | } | 20260507 07:52:35.144 | INFO | Perf report for the last 5.14215 seconds: | 20260507 07:52:35.144 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:52:35.144 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:35.144 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:35.144 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:35.144 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:52:35.144 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:35.144 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:35.144 | INFO | PRE_HQ : 29.0%, 1.0, 15.1ms, 0 | 20260507 07:52:35.144 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:35.144 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:35.144 | INFO | Uploaded 36GiB to GPU in 1.98107s (18.172GiB/s) | 20260507 07:52:35.144 | INFO | Downloaded 34GiB to GPU in 1.9634s (17.3169GiB/s) | 20260507 07:52:35.145 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:52:35.145 | INFO | Total mem usage: 18 GiB | 20260507 07:52:35.145 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:52:35.145 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 07:52:35.145 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:52:35.145 | INFO | High Water Breakdown: | 20260507 07:52:35.145 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:52:35.145 | INFO | 39.7% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:52:35.145 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:52:35.145 | INFO | ------------------------------------------ | 20260507 07:52:35.145 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:52:35.145 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:52:35.145 | INFO | Total mem usage: 26.5 GiB | 20260507 07:52:35.145 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:52:35.145 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:52:35.145 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:52:35.145 | INFO | High Water Breakdown: | 20260507 07:52:35.145 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:52:35.145 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:52:35.145 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:52:35.145 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:52:35.145 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:52:35.145 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:52:35.145 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:52:35.145 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:52:35.145 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:52:35.145 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:52:35.145 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:52:35.145 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:52:35.145 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:52:35.145 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:52:35.145 | INFO | ------------------------------------------ | 20260507 07:52:35.145 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:52:35.145 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:52:35.145 | INFO | Total mem usage: 25.6 GiB | 20260507 07:52:35.145 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:52:35.145 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:52:35.145 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:52:35.145 | INFO | High Water Breakdown: | 20260507 07:52:35.145 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:52:35.145 | INFO | ------------------------------------------ | 20260507 07:52:35.145 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:52:35.145 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:52:35.145 | INFO | Total mem usage: 15.7 GiB | 20260507 07:52:35.145 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:52:35.145 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:52:35.145 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:52:35.145 | INFO | High Water Breakdown: | 20260507 07:52:35.145 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:52:35.145 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:52:35.145 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:52:35.145 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:52:35.145 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:52:35.145 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:52:35.145 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:52:35.145 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:52:35.145 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:52:35.145 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:52:35.145 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:52:35.145 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:52:35.145 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:52:35.145 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:52:35.145 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:52:35.145 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:52:35.145 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:52:35.145 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:52:35.145 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:52:35.145 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:52:35.145 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:52:35.145 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:52:35.145 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:52:35.145 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:52:35.145 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:52:35.145 | INFO | ------------------------------------------ | 20260507 07:52:35.145 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:52:35.145 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:52:35.146 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:52:35.146 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:52:35.146 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.35e+01 / 1.32e+00 | 20260507 07:52:35.146 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.82e-03 / 4.98e-03 | 20260507 07:52:35.146 | INFO | PerfReport: 11.69%, Baseline -- 3.59e+00 / 4.43e+00, 5.11e-01 / 2.67e-01 | 20260507 07:52:35.146 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.48e+00, 1.66e-01 / 1.13e-01 | 20260507 07:52:35.146 | INFO | PerfReport: 16.29%, DME -- 4.00e+01 / 4.48e+01, 3.88e+00 / 1.59e+00 | 20260507 07:52:35.146 | INFO | PerfReport: 57.71%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.28e+00 / 1.49e+00 | 20260507 07:52:35.146 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.70e+00, 3.86e-01 / 2.04e-01 | 20260507 07:52:35.146 | INFO | PerfReport: 2.92%, Metrics -- 8.95e-01 / 1.21e+00, 2.39e-01 / 1.27e-01 | 20260507 07:52:35.146 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.01e-01 / 3.55e-01, 5.09e-02 / 1.87e-02 | 20260507 07:52:35.146 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:52:35.146 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:52:35.146 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:52:35.146 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 07:52:35.146 | INFO | PerfReport: 21.88%, Upload -- 2.10e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260507 07:52:35.146 | INFO | PerfReport: 21.39%, Download -- 2.05e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 07:52:35.146 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 07:52:35.146 | INFO | PerfReport: 1.53%, LockUpload -- 1.47e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 07:52:35.146 | INFO | PerfReport: 1.30%, LockDownload -- 1.25e+00 / 0.00e+00, 3.05e+00 / 0.00e+00 | 20260507 07:52:35.146 | INFO | PerfReport: 21.95%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 07:52:35.146 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.78e-02 / 0.00e+00 | 20260507 07:52:35.241 | INFO | Starting a round of Baz IO | 20260507 07:52:38.334 | INFO | Populating RTMetrics report | 20260507 07:52:38.334 | INFO | RTMetrics report populated | 20260507 07:52:38.334 | INFO | Opening tmp RTMetrics json file | 20260507 07:52:38.356 | INFO | Serializing RTMetrics to json | 20260507 07:52:38.357 | INFO | Serialization done, swapping tmp file now... | 20260507 07:52:38.358 | INFO | RTMetrics done | 20260507 07:52:38.421 | INFO | Finished a round of Baz IO: Wrote 1.628 GiB in 3.18 seconds | 20260507 07:52:38.421 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.53% / 0.00% / 67.47% | 20260507 07:52:38.421 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0989437% / 0% / 7.15256e-05% | 20260507 07:52:38.421 | INFO | Baz IO overall: | 20260507 07:52:38.421 | INFO | BytesWritten : 4156659047936 | 20260507 07:52:38.421 | INFO | NumEvents : 1989324036512 | 20260507 07:52:40.284 | INFO | Analyzing chunk frames = [8314880,8315392) | 20260507 07:52:40.284 | INFO | MemInfo: { | 20260507 07:52:40.284 | INFO | "gpu" : " 17 GB", | 20260507 07:52:40.284 | INFO | "huge" : "", | 20260507 07:52:40.284 | INFO | "huge_shared" : " 27 GB", | 20260507 07:52:40.284 | INFO | "malloc" : " 14 GB", | 20260507 07:52:40.284 | INFO | "pinned" : " 29 GB" | 20260507 07:52:40.284 | INFO | } | 20260507 07:52:40.284 | INFO | Perf report for the last 5.14025 seconds: | 20260507 07:52:40.284 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:52:40.284 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:40.284 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:40.284 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:40.284 | INFO | ANALYSIS : 61.5%, 3.0, 95.4ms, 0 | 20260507 07:52:40.284 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:40.284 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 07:52:40.284 | INFO | PRE_HQ : 58.7%, 1.8, 54.3ms, 0 | 20260507 07:52:40.284 | INFO | BAZWRITER : 61.5%, 3.7, 115.4ms, 0 | 20260507 07:52:40.284 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:52:40.285 | INFO | Uploaded 36GiB to GPU in 2.06333s (17.4475GiB/s) | 20260507 07:52:40.285 | INFO | Downloaded 38GiB to GPU in 2.18926s (17.3574GiB/s) | 20260507 07:52:45.417 | INFO | Analyzing chunk frames = [8315392,8315904) | 20260507 07:52:45.417 | INFO | MemInfo: { | 20260507 07:52:45.417 | INFO | "gpu" : " 17 GB", | 20260507 07:52:45.417 | INFO | "huge" : "", | 20260507 07:52:45.417 | INFO | "huge_shared" : " 27 GB", | 20260507 07:52:45.417 | INFO | "malloc" : " 14 GB", | 20260507 07:52:45.418 | INFO | "pinned" : " 29 GB" | 20260507 07:52:45.418 | INFO | } | 20260507 07:52:45.418 | INFO | Perf report for the last 5.13334 seconds: | 20260507 07:52:45.418 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:52:45.418 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:45.418 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:45.418 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:45.418 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:52:45.418 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:45.418 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:45.418 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260507 07:52:45.418 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:45.418 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:45.418 | INFO | Uploaded 36GiB to GPU in 1.98185s (18.1649GiB/s) | 20260507 07:52:45.418 | INFO | Downloaded 34GiB to GPU in 1.93261s (17.5928GiB/s) | 20260507 07:52:50.559 | INFO | Analyzing chunk frames = [8315904,8316416) | 20260507 07:52:50.559 | INFO | MemInfo: { | 20260507 07:52:50.559 | INFO | "gpu" : " 17 GB", | 20260507 07:52:50.559 | INFO | "huge" : "", | 20260507 07:52:50.559 | INFO | "huge_shared" : " 27 GB", | 20260507 07:52:50.559 | INFO | "malloc" : " 14 GB", | 20260507 07:52:50.559 | INFO | "pinned" : " 29 GB" | 20260507 07:52:50.559 | INFO | } | 20260507 07:52:50.559 | INFO | Perf report for the last 5.14136 seconds: | 20260507 07:52:50.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:52:50.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:50.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:50.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:50.559 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:52:50.559 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:50.559 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:50.559 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:52:50.559 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:50.559 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:50.559 | INFO | Uploaded 36GiB to GPU in 2.04228s (17.6274GiB/s) | 20260507 07:52:50.559 | INFO | Downloaded 34GiB to GPU in 1.95299s (17.4092GiB/s) | 20260507 07:52:55.711 | INFO | Analyzing chunk frames = [8316416,8316928) | 20260507 07:52:55.711 | INFO | MemInfo: { | 20260507 07:52:55.711 | INFO | "gpu" : " 17 GB", | 20260507 07:52:55.711 | INFO | "huge" : "", | 20260507 07:52:55.711 | INFO | "huge_shared" : " 27 GB", | 20260507 07:52:55.711 | INFO | "malloc" : " 14 GB", | 20260507 07:52:55.711 | INFO | "pinned" : " 29 GB" | 20260507 07:52:55.711 | INFO | } | 20260507 07:52:55.711 | INFO | Perf report for the last 5.15203 seconds: | 20260507 07:52:55.711 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:52:55.711 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:55.711 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:55.711 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:55.711 | INFO | ANALYSIS : 59.4%, 3.0, 92.3ms, 0 | 20260507 07:52:55.711 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:55.711 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:55.711 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:52:55.711 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:55.711 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:52:55.711 | INFO | Uploaded 36GiB to GPU in 2.00073s (17.9934GiB/s) | 20260507 07:52:55.711 | INFO | Downloaded 34GiB to GPU in 1.9407s (17.5195GiB/s) | 20260507 07:53:00.845 | INFO | Analyzing chunk frames = [8316928,8317440) | 20260507 07:53:00.845 | INFO | MemInfo: { | 20260507 07:53:00.845 | INFO | "gpu" : " 17 GB", | 20260507 07:53:00.845 | INFO | "huge" : "", | 20260507 07:53:00.845 | INFO | "huge_shared" : " 27 GB", | 20260507 07:53:00.845 | INFO | "malloc" : " 14 GB", | 20260507 07:53:00.845 | INFO | "pinned" : " 29 GB" | 20260507 07:53:00.845 | INFO | } | 20260507 07:53:00.845 | INFO | Perf report for the last 5.13383 seconds: | 20260507 07:53:00.845 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:53:00.845 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:00.845 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:00.845 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:00.845 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 07:53:00.845 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:00.845 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:00.845 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:53:00.845 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:00.845 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:00.845 | INFO | Uploaded 36GiB to GPU in 2.03885s (17.657GiB/s) | 20260507 07:53:00.845 | INFO | Downloaded 34GiB to GPU in 1.90279s (17.8685GiB/s) | 20260507 07:53:05.981 | INFO | Analyzing chunk frames = [8317440,8317952) | 20260507 07:53:05.981 | INFO | MemInfo: { | 20260507 07:53:05.981 | INFO | "gpu" : " 17 GB", | 20260507 07:53:05.981 | INFO | "huge" : "", | 20260507 07:53:05.981 | INFO | "huge_shared" : " 27 GB", | 20260507 07:53:05.981 | INFO | "malloc" : " 14 GB", | 20260507 07:53:05.981 | INFO | "pinned" : " 29 GB" | 20260507 07:53:05.981 | INFO | } | 20260507 07:53:05.981 | INFO | Perf report for the last 5.13653 seconds: | 20260507 07:53:05.981 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:53:05.981 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:05.981 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:05.981 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:05.981 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 07:53:05.981 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:05.981 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:05.981 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:53:05.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:05.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:05.982 | INFO | Uploaded 36GiB to GPU in 2.06658s (17.4201GiB/s) | 20260507 07:53:05.982 | INFO | Downloaded 34GiB to GPU in 1.94206s (17.5072GiB/s) | 20260507 07:53:11.116 | INFO | Analyzing chunk frames = [8317952,8318464) | 20260507 07:53:11.116 | INFO | MemInfo: { | 20260507 07:53:11.116 | INFO | "gpu" : " 17 GB", | 20260507 07:53:11.116 | INFO | "huge" : "", | 20260507 07:53:11.116 | INFO | "huge_shared" : " 27 GB", | 20260507 07:53:11.116 | INFO | "malloc" : " 14 GB", | 20260507 07:53:11.116 | INFO | "pinned" : " 29 GB" | 20260507 07:53:11.116 | INFO | } | 20260507 07:53:11.116 | INFO | Perf report for the last 5.13485 seconds: | 20260507 07:53:11.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:53:11.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:11.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:11.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:11.116 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:53:11.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:11.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:11.116 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 07:53:11.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:11.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:11.117 | INFO | Uploaded 36GiB to GPU in 1.88751s (19.0728GiB/s) | 20260507 07:53:11.117 | INFO | Downloaded 34GiB to GPU in 1.97899s (17.1805GiB/s) | 20260507 07:53:16.256 | INFO | Analyzing chunk frames = [8318464,8318976) | 20260507 07:53:16.256 | INFO | MemInfo: { | 20260507 07:53:16.256 | INFO | "gpu" : " 17 GB", | 20260507 07:53:16.256 | INFO | "huge" : "", | 20260507 07:53:16.256 | INFO | "huge_shared" : " 27 GB", | 20260507 07:53:16.257 | INFO | "malloc" : " 14 GB", | 20260507 07:53:16.257 | INFO | "pinned" : " 29 GB" | 20260507 07:53:16.257 | INFO | } | 20260507 07:53:16.257 | INFO | Perf report for the last 5.14041 seconds: | 20260507 07:53:16.257 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:53:16.257 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:16.257 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:16.257 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:16.257 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:53:16.257 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:16.257 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:16.257 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 07:53:16.257 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:16.257 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:16.257 | INFO | Uploaded 36GiB to GPU in 1.97732s (18.2065GiB/s) | 20260507 07:53:16.257 | INFO | Downloaded 34GiB to GPU in 1.96169s (17.332GiB/s) | 20260507 07:53:16.354 | INFO | Starting a round of Baz IO | 20260507 07:53:19.444 | INFO | Populating RTMetrics report | 20260507 07:53:19.444 | INFO | RTMetrics report populated | 20260507 07:53:19.444 | INFO | Opening tmp RTMetrics json file | 20260507 07:53:19.457 | INFO | Serializing RTMetrics to json | 20260507 07:53:19.466 | INFO | Serialization done, swapping tmp file now... | 20260507 07:53:19.467 | INFO | RTMetrics done | 20260507 07:53:19.538 | INFO | Finished a round of Baz IO: Wrote 1.628 GiB in 3.184 seconds | 20260507 07:53:19.538 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.53% / 0.00% / 67.47% | 20260507 07:53:19.538 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0979185% / 0% / 4.37101e-05% | 20260507 07:53:19.538 | INFO | Baz IO overall: | 20260507 07:53:19.538 | INFO | BytesWritten : 4158406733757 | 20260507 07:53:19.538 | INFO | NumEvents : 1990139412648 | 20260507 07:53:21.397 | INFO | Analyzing chunk frames = [8318976,8319488) | 20260507 07:53:21.397 | INFO | MemInfo: { | 20260507 07:53:21.397 | INFO | "gpu" : " 17 GB", | 20260507 07:53:21.397 | INFO | "huge" : "", | 20260507 07:53:21.397 | INFO | "huge_shared" : " 27 GB", | 20260507 07:53:21.397 | INFO | "malloc" : " 14 GB", | 20260507 07:53:21.397 | INFO | "pinned" : " 29 GB" | 20260507 07:53:21.397 | INFO | } | 20260507 07:53:21.397 | INFO | Perf report for the last 5.14078 seconds: | 20260507 07:53:21.397 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:53:21.397 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:21.397 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:21.397 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:21.398 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 07:53:21.398 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:21.398 | INFO | RT_METRICS : 43.5%, 1.0, 22.8ms, 0 | 20260507 07:53:21.398 | INFO | PRE_HQ : 59.7%, 1.9, 58.0ms, 0 | 20260507 07:53:21.398 | INFO | BAZWRITER : 61.6%, 3.3, 103.3ms, 0 | 20260507 07:53:21.398 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 07:53:21.398 | INFO | Uploaded 36GiB to GPU in 2.03604s (17.6814GiB/s) | 20260507 07:53:21.398 | INFO | Downloaded 38GiB to GPU in 2.17454s (17.4749GiB/s) | 20260507 07:53:26.543 | INFO | Analyzing chunk frames = [8319488,8320000) | 20260507 07:53:26.544 | INFO | MemInfo: { | 20260507 07:53:26.544 | INFO | "gpu" : " 17 GB", | 20260507 07:53:26.544 | INFO | "huge" : "", | 20260507 07:53:26.544 | INFO | "huge_shared" : " 27 GB", | 20260507 07:53:26.544 | INFO | "malloc" : " 14 GB", | 20260507 07:53:26.544 | INFO | "pinned" : " 29 GB" | 20260507 07:53:26.544 | INFO | } | 20260507 07:53:26.544 | INFO | Perf report for the last 5.14633 seconds: | 20260507 07:53:26.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:53:26.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:26.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:26.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:26.544 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:53:26.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:26.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:26.544 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 07:53:26.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:26.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:26.544 | INFO | Uploaded 36GiB to GPU in 1.9864s (18.1233GiB/s) | 20260507 07:53:26.544 | INFO | Downloaded 34GiB to GPU in 1.9391s (17.5339GiB/s) | 20260507 07:53:31.686 | INFO | Analyzing chunk frames = [8320000,8320512) | 20260507 07:53:31.686 | INFO | MemInfo: { | 20260507 07:53:31.686 | INFO | "gpu" : " 17 GB", | 20260507 07:53:31.686 | INFO | "huge" : "", | 20260507 07:53:31.686 | INFO | "huge_shared" : " 27 GB", | 20260507 07:53:31.686 | INFO | "malloc" : " 14 GB", | 20260507 07:53:31.686 | INFO | "pinned" : " 29 GB" | 20260507 07:53:31.686 | INFO | } | 20260507 07:53:31.686 | INFO | Perf report for the last 5.14222 seconds: | 20260507 07:53:31.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:53:31.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:31.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:31.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:31.686 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 07:53:31.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:31.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:31.686 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:53:31.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:31.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:31.686 | INFO | Uploaded 36GiB to GPU in 2.03164s (17.7197GiB/s) | 20260507 07:53:31.686 | INFO | Downloaded 34GiB to GPU in 1.9226s (17.6844GiB/s) | 20260507 07:53:36.815 | INFO | Analyzing chunk frames = [8320512,8321024) | 20260507 07:53:36.815 | INFO | MemInfo: { | 20260507 07:53:36.815 | INFO | "gpu" : " 17 GB", | 20260507 07:53:36.815 | INFO | "huge" : "", | 20260507 07:53:36.815 | INFO | "huge_shared" : " 27 GB", | 20260507 07:53:36.815 | INFO | "malloc" : " 14 GB", | 20260507 07:53:36.815 | INFO | "pinned" : " 29 GB" | 20260507 07:53:36.815 | INFO | } | 20260507 07:53:36.815 | INFO | Perf report for the last 5.12946 seconds: | 20260507 07:53:36.815 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:53:36.815 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:36.815 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:36.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:36.816 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 07:53:36.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:36.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:36.816 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 07:53:36.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:36.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:36.816 | INFO | Uploaded 36GiB to GPU in 2.05465s (17.5213GiB/s) | 20260507 07:53:36.816 | INFO | Downloaded 34GiB to GPU in 1.95426s (17.3979GiB/s) | 20260507 07:53:41.953 | INFO | Analyzing chunk frames = [8321024,8321536) | 20260507 07:53:41.953 | INFO | MemInfo: { | 20260507 07:53:41.953 | INFO | "gpu" : " 17 GB", | 20260507 07:53:41.953 | INFO | "huge" : "", | 20260507 07:53:41.954 | INFO | "huge_shared" : " 27 GB", | 20260507 07:53:41.954 | INFO | "malloc" : " 14 GB", | 20260507 07:53:41.954 | INFO | "pinned" : " 29 GB" | 20260507 07:53:41.954 | INFO | } | 20260507 07:53:41.954 | INFO | Perf report for the last 5.13824 seconds: | 20260507 07:53:41.954 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:53:41.954 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:41.954 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:41.954 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:41.954 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 07:53:41.954 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:41.954 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:41.954 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:53:41.954 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:41.954 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:41.954 | INFO | Uploaded 36GiB to GPU in 2.0234s (17.7918GiB/s) | 20260507 07:53:41.954 | INFO | Downloaded 34GiB to GPU in 1.91863s (17.721GiB/s) | 20260507 07:53:47.094 | INFO | Analyzing chunk frames = [8321536,8322048) | 20260507 07:53:47.094 | INFO | MemInfo: { | 20260507 07:53:47.094 | INFO | "gpu" : " 17 GB", | 20260507 07:53:47.094 | INFO | "huge" : "", | 20260507 07:53:47.094 | INFO | "huge_shared" : " 27 GB", | 20260507 07:53:47.094 | INFO | "malloc" : " 14 GB", | 20260507 07:53:47.094 | INFO | "pinned" : " 29 GB" | 20260507 07:53:47.094 | INFO | } | 20260507 07:53:47.094 | INFO | Perf report for the last 5.14051 seconds: | 20260507 07:53:47.094 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:53:47.094 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:47.094 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:47.094 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:47.094 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 07:53:47.094 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:47.094 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:47.094 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:53:47.094 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:47.094 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:47.095 | INFO | Uploaded 36GiB to GPU in 2.02608s (17.7683GiB/s) | 20260507 07:53:47.095 | INFO | Downloaded 34GiB to GPU in 1.91178s (17.7845GiB/s) | 20260507 07:53:52.231 | INFO | Analyzing chunk frames = [8322048,8322560) | 20260507 07:53:52.231 | INFO | MemInfo: { | 20260507 07:53:52.231 | INFO | "gpu" : " 17 GB", | 20260507 07:53:52.231 | INFO | "huge" : "", | 20260507 07:53:52.231 | INFO | "huge_shared" : " 27 GB", | 20260507 07:53:52.231 | INFO | "malloc" : " 14 GB", | 20260507 07:53:52.231 | INFO | "pinned" : " 29 GB" | 20260507 07:53:52.231 | INFO | } | 20260507 07:53:52.231 | INFO | Perf report for the last 5.13681 seconds: | 20260507 07:53:52.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:53:52.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:52.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:52.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:52.231 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 07:53:52.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:52.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:52.231 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:53:52.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:52.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:52.231 | INFO | Uploaded 36GiB to GPU in 1.8916s (19.0315GiB/s) | 20260507 07:53:52.231 | INFO | Downloaded 34GiB to GPU in 1.97754s (17.1931GiB/s) | 20260507 07:53:57.364 | INFO | Analyzing chunk frames = [8322560,8323072) | 20260507 07:53:57.364 | INFO | MemInfo: { | 20260507 07:53:57.364 | INFO | "gpu" : " 17 GB", | 20260507 07:53:57.364 | INFO | "huge" : "", | 20260507 07:53:57.364 | INFO | "huge_shared" : " 27 GB", | 20260507 07:53:57.365 | INFO | "malloc" : " 14 GB", | 20260507 07:53:57.365 | INFO | "pinned" : " 29 GB" | 20260507 07:53:57.365 | INFO | } | 20260507 07:53:57.365 | INFO | Perf report for the last 5.13365 seconds: | 20260507 07:53:57.365 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:53:57.365 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:57.365 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:57.365 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:57.365 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:53:57.365 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:57.365 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:57.365 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:53:57.365 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:57.365 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:53:57.365 | INFO | Uploaded 36GiB to GPU in 1.97595s (18.2191GiB/s) | 20260507 07:53:57.365 | INFO | Downloaded 34GiB to GPU in 1.96229s (17.3267GiB/s) | 20260507 07:53:57.365 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:53:57.365 | INFO | Total mem usage: 18 GiB | 20260507 07:53:57.365 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:53:57.365 | INFO | Recent High Water Marks Sum: 15 GiB (Savings: -20.2%) | 20260507 07:53:57.365 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:53:57.365 | INFO | High Water Breakdown: | 20260507 07:53:57.365 | INFO | 61% : 9.13 GiB ( 80% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:53:57.365 | INFO | 38.9% : 5.83 GiB ( 87.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:53:57.365 | INFO | 0.133% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:53:57.365 | INFO | ------------------------------------------ | 20260507 07:53:57.365 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:53:57.365 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:53:57.365 | INFO | Total mem usage: 26.5 GiB | 20260507 07:53:57.365 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:53:57.365 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:53:57.365 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:53:57.365 | INFO | High Water Breakdown: | 20260507 07:53:57.365 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:53:57.365 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:53:57.365 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:53:57.365 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:53:57.365 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:53:57.365 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:53:57.365 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:53:57.365 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:53:57.365 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:53:57.365 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:53:57.365 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:53:57.365 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:53:57.365 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:53:57.365 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:53:57.365 | INFO | ------------------------------------------ | 20260507 07:53:57.365 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:53:57.365 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:53:57.365 | INFO | Total mem usage: 25.6 GiB | 20260507 07:53:57.365 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:53:57.365 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:53:57.365 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:53:57.365 | INFO | High Water Breakdown: | 20260507 07:53:57.365 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:53:57.365 | INFO | ------------------------------------------ | 20260507 07:53:57.365 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:53:57.365 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:53:57.365 | INFO | Total mem usage: 15.7 GiB | 20260507 07:53:57.365 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:53:57.365 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:53:57.365 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:53:57.365 | INFO | High Water Breakdown: | 20260507 07:53:57.365 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:53:57.365 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:53:57.365 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:53:57.365 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:53:57.365 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:53:57.365 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:53:57.365 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:53:57.366 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:53:57.366 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:53:57.366 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:53:57.366 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:53:57.366 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:53:57.366 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:53:57.366 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:53:57.366 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:53:57.366 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:53:57.366 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:53:57.366 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:53:57.366 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:53:57.366 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:53:57.366 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:53:57.366 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:53:57.366 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:53:57.366 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:53:57.366 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:53:57.366 | INFO | ------------------------------------------ | 20260507 07:53:57.366 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:53:57.366 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:53:57.366 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:53:57.366 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:53:57.366 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.44e+01, 1.34e+01 / 8.82e-01 | 20260507 07:53:57.366 | INFO | PerfReport: 0.29%, AggregateStats -- 9.02e-02 / 1.02e-01, 9.71e-03 / 5.61e-03 | 20260507 07:53:57.366 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.49e+00, 5.08e-01 / 1.81e-01 | 20260507 07:53:57.366 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.49e+00, 1.67e-01 / 9.76e-02 | 20260507 07:53:57.366 | INFO | PerfReport: 16.29%, DME -- 4.00e+01 / 4.47e+01, 3.94e+00 / 1.21e+00 | 20260507 07:53:57.366 | INFO | PerfReport: 57.69%, FrameLabeling -- 1.77e+01 / 1.93e+01, 1.24e+00 / 9.31e-01 | 20260507 07:53:57.366 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.12e+00 / 2.74e+00, 3.91e-01 / 1.20e-01 | 20260507 07:53:57.366 | INFO | PerfReport: 2.92%, Metrics -- 8.96e-01 / 1.21e+00, 2.41e-01 / 1.12e-01 | 20260507 07:53:57.366 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.60e-01, 5.04e-02 / 1.96e-02 | 20260507 07:53:57.366 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:53:57.366 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:53:57.366 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:53:57.366 | INFO | PerfReport: Full Execution -- 9.60e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 07:53:57.366 | INFO | PerfReport: 21.75%, Upload -- 2.09e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260507 07:53:57.366 | INFO | PerfReport: 21.40%, Download -- 2.05e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260507 07:53:57.366 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 07:53:57.366 | INFO | PerfReport: 1.48%, LockUpload -- 1.43e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 07:53:57.366 | INFO | PerfReport: 1.38%, LockDownload -- 1.33e+00 / 0.00e+00, 3.19e+00 / 0.00e+00 | 20260507 07:53:57.366 | INFO | PerfReport: 22.03%, LockCompute -- 2.11e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260507 07:53:57.366 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260507 07:53:57.462 | INFO | Starting a round of Baz IO | 20260507 07:54:00.553 | INFO | Populating RTMetrics report | 20260507 07:54:00.553 | INFO | RTMetrics report populated | 20260507 07:54:00.553 | INFO | Opening tmp RTMetrics json file | 20260507 07:54:00.574 | INFO | Serializing RTMetrics to json | 20260507 07:54:00.576 | INFO | Serialization done, swapping tmp file now... | 20260507 07:54:00.576 | INFO | RTMetrics done | 20260507 07:54:00.646 | INFO | Finished a round of Baz IO: Wrote 1.628 GiB in 3.184 seconds | 20260507 07:54:00.646 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.53% / 0.00% / 67.47% | 20260507 07:54:00.646 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0982046% / 0% / 5.5631e-05% | 20260507 07:54:00.646 | INFO | Baz IO overall: | 20260507 07:54:00.646 | INFO | BytesWritten : 4160154651709 | 20260507 07:54:00.646 | INFO | NumEvents : 1990955073695 | 20260507 07:54:02.509 | INFO | Analyzing chunk frames = [8323072,8323584) | 20260507 07:54:02.509 | INFO | MemInfo: { | 20260507 07:54:02.509 | INFO | "gpu" : " 17 GB", | 20260507 07:54:02.509 | INFO | "huge" : "", | 20260507 07:54:02.509 | INFO | "huge_shared" : " 27 GB", | 20260507 07:54:02.509 | INFO | "malloc" : " 14 GB", | 20260507 07:54:02.509 | INFO | "pinned" : " 29 GB" | 20260507 07:54:02.509 | INFO | } | 20260507 07:54:02.509 | INFO | Perf report for the last 5.14392 seconds: | 20260507 07:54:02.509 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:54:02.509 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:02.509 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:02.509 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:02.509 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 07:54:02.509 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:02.509 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:54:02.509 | INFO | PRE_HQ : 58.6%, 1.8, 54.4ms, 0 | 20260507 07:54:02.509 | INFO | BAZWRITER : 61.5%, 3.2, 100.1ms, 0 | 20260507 07:54:02.509 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:54:02.509 | INFO | Uploaded 36GiB to GPU in 2.02927s (17.7404GiB/s) | 20260507 07:54:02.509 | INFO | Downloaded 38GiB to GPU in 2.17596s (17.4635GiB/s) | 20260507 07:54:07.659 | INFO | Analyzing chunk frames = [8323584,8324096) | 20260507 07:54:07.659 | INFO | MemInfo: { | 20260507 07:54:07.659 | INFO | "gpu" : " 17 GB", | 20260507 07:54:07.659 | INFO | "huge" : "", | 20260507 07:54:07.659 | INFO | "huge_shared" : " 27 GB", | 20260507 07:54:07.659 | INFO | "malloc" : " 14 GB", | 20260507 07:54:07.659 | INFO | "pinned" : " 29 GB" | 20260507 07:54:07.659 | INFO | } | 20260507 07:54:07.660 | INFO | Perf report for the last 5.15084 seconds: | 20260507 07:54:07.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:54:07.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:07.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:07.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:07.660 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 07:54:07.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:07.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:07.660 | INFO | PRE_HQ : 23.9%, 1.0, 12.5ms, 0 | 20260507 07:54:07.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:07.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:07.660 | INFO | Uploaded 36GiB to GPU in 1.9969s (18.0279GiB/s) | 20260507 07:54:07.660 | INFO | Downloaded 34GiB to GPU in 1.93344s (17.5852GiB/s) | 20260507 07:54:12.799 | INFO | Analyzing chunk frames = [8324096,8324608) | 20260507 07:54:12.799 | INFO | MemInfo: { | 20260507 07:54:12.799 | INFO | "gpu" : " 17 GB", | 20260507 07:54:12.799 | INFO | "huge" : "", | 20260507 07:54:12.799 | INFO | "huge_shared" : " 27 GB", | 20260507 07:54:12.799 | INFO | "malloc" : " 14 GB", | 20260507 07:54:12.799 | INFO | "pinned" : " 29 GB" | 20260507 07:54:12.799 | INFO | } | 20260507 07:54:12.799 | INFO | Perf report for the last 5.13971 seconds: | 20260507 07:54:12.799 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:54:12.799 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:12.799 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:12.799 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:12.799 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 07:54:12.799 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:12.799 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:12.799 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 07:54:12.799 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:12.799 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:12.799 | INFO | Uploaded 36GiB to GPU in 2.02604s (17.7686GiB/s) | 20260507 07:54:12.800 | INFO | Downloaded 34GiB to GPU in 1.92056s (17.7032GiB/s) | 20260507 07:54:17.927 | INFO | Analyzing chunk frames = [8324608,8325120) | 20260507 07:54:17.927 | INFO | MemInfo: { | 20260507 07:54:17.927 | INFO | "gpu" : " 17 GB", | 20260507 07:54:17.927 | INFO | "huge" : "", | 20260507 07:54:17.927 | INFO | "huge_shared" : " 27 GB", | 20260507 07:54:17.927 | INFO | "malloc" : " 14 GB", | 20260507 07:54:17.927 | INFO | "pinned" : " 29 GB" | 20260507 07:54:17.927 | INFO | } | 20260507 07:54:17.927 | INFO | Perf report for the last 5.1277 seconds: | 20260507 07:54:17.927 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:54:17.927 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:17.927 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:17.927 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:17.927 | INFO | ANALYSIS : 60.5%, 2.9, 93.2ms, 0 | 20260507 07:54:17.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:17.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:17.927 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 07:54:17.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:17.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:17.927 | INFO | Uploaded 36GiB to GPU in 2.04021s (17.6452GiB/s) | 20260507 07:54:17.927 | INFO | Downloaded 34GiB to GPU in 1.92018s (17.7067GiB/s) | 20260507 07:54:23.066 | INFO | Analyzing chunk frames = [8325120,8325632) | 20260507 07:54:23.066 | INFO | MemInfo: { | 20260507 07:54:23.066 | INFO | "gpu" : " 17 GB", | 20260507 07:54:23.066 | INFO | "huge" : "", | 20260507 07:54:23.066 | INFO | "huge_shared" : " 27 GB", | 20260507 07:54:23.066 | INFO | "malloc" : " 14 GB", | 20260507 07:54:23.066 | INFO | "pinned" : " 29 GB" | 20260507 07:54:23.066 | INFO | } | 20260507 07:54:23.066 | INFO | Perf report for the last 5.13944 seconds: | 20260507 07:54:23.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:54:23.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:23.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:23.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:23.066 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 07:54:23.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:23.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:23.067 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:54:23.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:23.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:23.067 | INFO | Uploaded 36GiB to GPU in 2.02897s (17.743GiB/s) | 20260507 07:54:23.067 | INFO | Downloaded 34GiB to GPU in 1.91739s (17.7325GiB/s) | 20260507 07:54:28.209 | INFO | Analyzing chunk frames = [8325632,8326144) | 20260507 07:54:28.209 | INFO | MemInfo: { | 20260507 07:54:28.209 | INFO | "gpu" : " 17 GB", | 20260507 07:54:28.209 | INFO | "huge" : "", | 20260507 07:54:28.209 | INFO | "huge_shared" : " 27 GB", | 20260507 07:54:28.209 | INFO | "malloc" : " 14 GB", | 20260507 07:54:28.209 | INFO | "pinned" : " 29 GB" | 20260507 07:54:28.209 | INFO | } | 20260507 07:54:28.209 | INFO | Perf report for the last 5.14246 seconds: | 20260507 07:54:28.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:54:28.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:28.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:28.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:28.209 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 07:54:28.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:28.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:28.209 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 07:54:28.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:28.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:28.209 | INFO | Uploaded 36GiB to GPU in 2.04981s (17.5626GiB/s) | 20260507 07:54:28.209 | INFO | Downloaded 34GiB to GPU in 1.91427s (17.7613GiB/s) | 20260507 07:54:33.351 | INFO | Analyzing chunk frames = [8326144,8326656) | 20260507 07:54:33.351 | INFO | MemInfo: { | 20260507 07:54:33.351 | INFO | "gpu" : " 17 GB", | 20260507 07:54:33.351 | INFO | "huge" : "", | 20260507 07:54:33.351 | INFO | "huge_shared" : " 27 GB", | 20260507 07:54:33.351 | INFO | "malloc" : " 14 GB", | 20260507 07:54:33.351 | INFO | "pinned" : " 29 GB" | 20260507 07:54:33.351 | INFO | } | 20260507 07:54:33.351 | INFO | Perf report for the last 5.1425 seconds: | 20260507 07:54:33.351 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:54:33.351 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:33.351 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:33.351 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:33.351 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 07:54:33.351 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:33.351 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:33.351 | INFO | PRE_HQ : 25.1%, 1.0, 13.1ms, 0 | 20260507 07:54:33.352 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:33.352 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:33.352 | INFO | Uploaded 36GiB to GPU in 2.0331s (17.7069GiB/s) | 20260507 07:54:33.352 | INFO | Downloaded 34GiB to GPU in 1.98338s (17.1424GiB/s) | 20260507 07:54:38.493 | INFO | Analyzing chunk frames = [8326656,8327168) | 20260507 07:54:38.493 | INFO | MemInfo: { | 20260507 07:54:38.493 | INFO | "gpu" : " 17 GB", | 20260507 07:54:38.493 | INFO | "huge" : "", | 20260507 07:54:38.493 | INFO | "huge_shared" : " 27 GB", | 20260507 07:54:38.493 | INFO | "malloc" : " 14 GB", | 20260507 07:54:38.493 | INFO | "pinned" : " 29 GB" | 20260507 07:54:38.493 | INFO | } | 20260507 07:54:38.493 | INFO | Perf report for the last 5.14191 seconds: | 20260507 07:54:38.493 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:54:38.493 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:38.493 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:38.493 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:38.493 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:54:38.493 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:38.493 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:38.493 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:54:38.493 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:38.493 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:38.494 | INFO | Uploaded 36GiB to GPU in 1.99051s (18.0858GiB/s) | 20260507 07:54:38.494 | INFO | Downloaded 34GiB to GPU in 1.96285s (17.3218GiB/s) | 20260507 07:54:38.590 | INFO | Starting a round of Baz IO | 20260507 07:54:41.685 | INFO | Populating RTMetrics report | 20260507 07:54:41.685 | INFO | RTMetrics report populated | 20260507 07:54:41.685 | INFO | Opening tmp RTMetrics json file | 20260507 07:54:41.702 | INFO | Serializing RTMetrics to json | 20260507 07:54:41.704 | INFO | Serialization done, swapping tmp file now... | 20260507 07:54:41.705 | INFO | RTMetrics done | 20260507 07:54:41.777 | INFO | Finished a round of Baz IO: Wrote 1.627 GiB in 3.186 seconds | 20260507 07:54:41.777 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.53% / 0.00% / 67.47% | 20260507 07:54:41.777 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0984033% / 3.97364e-06% / 4.76837e-05% | 20260507 07:54:41.777 | INFO | Baz IO overall: | 20260507 07:54:41.777 | INFO | BytesWritten : 4161902073049 | 20260507 07:54:41.777 | INFO | NumEvents : 1991770341907 | 20260507 07:54:43.624 | INFO | Analyzing chunk frames = [8327168,8327680) | 20260507 07:54:43.624 | INFO | MemInfo: { | 20260507 07:54:43.624 | INFO | "gpu" : " 17 GB", | 20260507 07:54:43.624 | INFO | "huge" : "", | 20260507 07:54:43.624 | INFO | "huge_shared" : " 27 GB", | 20260507 07:54:43.624 | INFO | "malloc" : " 14 GB", | 20260507 07:54:43.624 | INFO | "pinned" : " 29 GB" | 20260507 07:54:43.624 | INFO | } | 20260507 07:54:43.624 | INFO | Perf report for the last 5.13103 seconds: | 20260507 07:54:43.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:54:43.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:43.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:43.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:43.624 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260507 07:54:43.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:43.624 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:54:43.624 | INFO | PRE_HQ : 59.0%, 1.8, 54.3ms, 0 | 20260507 07:54:43.625 | INFO | BAZWRITER : 61.7%, 3.2, 100.0ms, 0 | 20260507 07:54:43.625 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:54:43.625 | INFO | Uploaded 36GiB to GPU in 2.06219s (17.4572GiB/s) | 20260507 07:54:43.625 | INFO | Downloaded 38GiB to GPU in 2.17683s (17.4566GiB/s) | 20260507 07:54:48.763 | INFO | Analyzing chunk frames = [8327680,8328192) | 20260507 07:54:48.763 | INFO | MemInfo: { | 20260507 07:54:48.763 | INFO | "gpu" : " 17 GB", | 20260507 07:54:48.763 | INFO | "huge" : "", | 20260507 07:54:48.763 | INFO | "huge_shared" : " 27 GB", | 20260507 07:54:48.763 | INFO | "malloc" : " 14 GB", | 20260507 07:54:48.763 | INFO | "pinned" : " 29 GB" | 20260507 07:54:48.763 | INFO | } | 20260507 07:54:48.763 | INFO | Perf report for the last 5.13877 seconds: | 20260507 07:54:48.763 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:54:48.763 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:48.763 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:48.763 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:48.763 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:54:48.763 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:48.763 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:48.763 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:54:48.763 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:48.763 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:48.763 | INFO | Uploaded 36GiB to GPU in 1.99534s (18.042GiB/s) | 20260507 07:54:48.763 | INFO | Downloaded 34GiB to GPU in 1.93415s (17.5788GiB/s) | 20260507 07:54:53.901 | INFO | Analyzing chunk frames = [8328192,8328704) | 20260507 07:54:53.901 | INFO | MemInfo: { | 20260507 07:54:53.901 | INFO | "gpu" : " 17 GB", | 20260507 07:54:53.901 | INFO | "huge" : "", | 20260507 07:54:53.901 | INFO | "huge_shared" : " 27 GB", | 20260507 07:54:53.901 | INFO | "malloc" : " 14 GB", | 20260507 07:54:53.901 | INFO | "pinned" : " 29 GB" | 20260507 07:54:53.901 | INFO | } | 20260507 07:54:53.901 | INFO | Perf report for the last 5.13823 seconds: | 20260507 07:54:53.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:54:53.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:53.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:53.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:53.901 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 07:54:53.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:53.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:53.901 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:54:53.902 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:53.902 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:53.902 | INFO | Uploaded 36GiB to GPU in 2.03518s (17.6888GiB/s) | 20260507 07:54:53.902 | INFO | Downloaded 34GiB to GPU in 1.9589s (17.3567GiB/s) | 20260507 07:54:59.044 | INFO | Analyzing chunk frames = [8328704,8329216) | 20260507 07:54:59.044 | INFO | MemInfo: { | 20260507 07:54:59.044 | INFO | "gpu" : " 17 GB", | 20260507 07:54:59.044 | INFO | "huge" : "", | 20260507 07:54:59.044 | INFO | "huge_shared" : " 27 GB", | 20260507 07:54:59.044 | INFO | "malloc" : " 14 GB", | 20260507 07:54:59.044 | INFO | "pinned" : " 29 GB" | 20260507 07:54:59.044 | INFO | } | 20260507 07:54:59.044 | INFO | Perf report for the last 5.14277 seconds: | 20260507 07:54:59.044 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:54:59.044 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:59.044 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:59.044 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:59.044 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 07:54:59.044 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:59.044 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:59.044 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:54:59.044 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:59.044 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:54:59.044 | INFO | Uploaded 36GiB to GPU in 2.01252s (17.888GiB/s) | 20260507 07:54:59.044 | INFO | Downloaded 34GiB to GPU in 1.94238s (17.5043GiB/s) | 20260507 07:55:04.182 | INFO | Analyzing chunk frames = [8329216,8329728) | 20260507 07:55:04.182 | INFO | MemInfo: { | 20260507 07:55:04.182 | INFO | "gpu" : " 17 GB", | 20260507 07:55:04.182 | INFO | "huge" : "", | 20260507 07:55:04.183 | INFO | "huge_shared" : " 27 GB", | 20260507 07:55:04.183 | INFO | "malloc" : " 14 GB", | 20260507 07:55:04.183 | INFO | "pinned" : " 29 GB" | 20260507 07:55:04.183 | INFO | } | 20260507 07:55:04.183 | INFO | Perf report for the last 5.13845 seconds: | 20260507 07:55:04.183 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:55:04.183 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:04.183 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:04.183 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:04.183 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 07:55:04.183 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:04.183 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:04.183 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:55:04.183 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:04.183 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:04.183 | INFO | Uploaded 36GiB to GPU in 2.04037s (17.6438GiB/s) | 20260507 07:55:04.183 | INFO | Downloaded 34GiB to GPU in 1.90248s (17.8714GiB/s) | 20260507 07:55:09.317 | INFO | Analyzing chunk frames = [8329728,8330240) | 20260507 07:55:09.317 | INFO | MemInfo: { | 20260507 07:55:09.317 | INFO | "gpu" : " 17 GB", | 20260507 07:55:09.317 | INFO | "huge" : "", | 20260507 07:55:09.317 | INFO | "huge_shared" : " 27 GB", | 20260507 07:55:09.317 | INFO | "malloc" : " 14 GB", | 20260507 07:55:09.317 | INFO | "pinned" : " 29 GB" | 20260507 07:55:09.317 | INFO | } | 20260507 07:55:09.317 | INFO | Perf report for the last 5.13421 seconds: | 20260507 07:55:09.317 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:55:09.317 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:09.317 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:09.317 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:09.317 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 07:55:09.317 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:09.317 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:09.317 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 07:55:09.317 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:09.317 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:09.317 | INFO | Uploaded 36GiB to GPU in 2.06784s (17.4095GiB/s) | 20260507 07:55:09.317 | INFO | Downloaded 34GiB to GPU in 1.93828s (17.5414GiB/s) | 20260507 07:55:14.462 | INFO | Analyzing chunk frames = [8330240,8330752) | 20260507 07:55:14.462 | INFO | MemInfo: { | 20260507 07:55:14.462 | INFO | "gpu" : " 17 GB", | 20260507 07:55:14.462 | INFO | "huge" : "", | 20260507 07:55:14.462 | INFO | "huge_shared" : " 27 GB", | 20260507 07:55:14.462 | INFO | "malloc" : " 14 GB", | 20260507 07:55:14.462 | INFO | "pinned" : " 29 GB" | 20260507 07:55:14.462 | INFO | } | 20260507 07:55:14.462 | INFO | Perf report for the last 5.14562 seconds: | 20260507 07:55:14.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:55:14.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:14.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:14.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:14.463 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:55:14.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:14.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:14.463 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:55:14.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:14.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:14.463 | INFO | Uploaded 36GiB to GPU in 1.95202s (18.4424GiB/s) | 20260507 07:55:14.463 | INFO | Downloaded 34GiB to GPU in 1.98134s (17.1601GiB/s) | 20260507 07:55:19.605 | INFO | Analyzing chunk frames = [8330752,8331264) | 20260507 07:55:19.605 | INFO | MemInfo: { | 20260507 07:55:19.605 | INFO | "gpu" : " 17 GB", | 20260507 07:55:19.605 | INFO | "huge" : "", | 20260507 07:55:19.605 | INFO | "huge_shared" : " 27 GB", | 20260507 07:55:19.605 | INFO | "malloc" : " 14 GB", | 20260507 07:55:19.605 | INFO | "pinned" : " 29 GB" | 20260507 07:55:19.605 | INFO | } | 20260507 07:55:19.605 | INFO | Perf report for the last 5.14248 seconds: | 20260507 07:55:19.605 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:55:19.605 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:19.605 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:19.605 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:19.605 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:55:19.605 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:19.605 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:19.605 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 07:55:19.605 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:19.605 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:19.605 | INFO | Uploaded 36GiB to GPU in 1.97935s (18.1878GiB/s) | 20260507 07:55:19.605 | INFO | Downloaded 34GiB to GPU in 1.96279s (17.3223GiB/s) | 20260507 07:55:19.605 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:55:19.605 | INFO | Total mem usage: 18 GiB | 20260507 07:55:19.605 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:55:19.606 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:55:19.606 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:55:19.606 | INFO | High Water Breakdown: | 20260507 07:55:19.606 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:55:19.606 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:55:19.606 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:55:19.606 | INFO | ------------------------------------------ | 20260507 07:55:19.606 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:55:19.606 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:55:19.606 | INFO | Total mem usage: 26.5 GiB | 20260507 07:55:19.606 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:55:19.606 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:55:19.606 | INFO | Unused Cache: 1.78 GiB (Waste: 6.7%) | 20260507 07:55:19.606 | INFO | High Water Breakdown: | 20260507 07:55:19.606 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:55:19.606 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:55:19.606 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:55:19.606 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:55:19.606 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:55:19.606 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:55:19.606 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:55:19.606 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:55:19.606 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:55:19.606 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:55:19.606 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:55:19.606 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:55:19.606 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:55:19.606 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:55:19.606 | INFO | ------------------------------------------ | 20260507 07:55:19.606 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:55:19.606 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:55:19.606 | INFO | Total mem usage: 25.6 GiB | 20260507 07:55:19.606 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:55:19.606 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:55:19.606 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:55:19.606 | INFO | High Water Breakdown: | 20260507 07:55:19.606 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:55:19.606 | INFO | ------------------------------------------ | 20260507 07:55:19.606 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:55:19.606 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:55:19.606 | INFO | Total mem usage: 15.7 GiB | 20260507 07:55:19.606 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:55:19.606 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:55:19.606 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:55:19.606 | INFO | High Water Breakdown: | 20260507 07:55:19.606 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:55:19.606 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:55:19.606 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:55:19.606 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:55:19.606 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:55:19.606 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:55:19.606 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:55:19.606 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:55:19.606 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:55:19.606 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:55:19.606 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:55:19.606 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:55:19.606 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:55:19.606 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:55:19.606 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:55:19.606 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:55:19.606 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:55:19.606 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:55:19.606 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:55:19.606 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:55:19.606 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:55:19.606 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:55:19.606 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:55:19.606 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:55:19.606 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:55:19.606 | INFO | ------------------------------------------ | 20260507 07:55:19.606 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:55:19.606 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:55:19.606 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:55:19.606 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:55:19.606 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.32e+01 / 1.50e+00 | 20260507 07:55:19.606 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.89e-03 / 5.11e-03 | 20260507 07:55:19.606 | INFO | PerfReport: 11.67%, Baseline -- 3.59e+00 / 4.40e+00, 5.06e-01 / 3.01e-01 | 20260507 07:55:19.606 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.46e+00, 1.66e-01 / 1.34e-01 | 20260507 07:55:19.606 | INFO | PerfReport: 16.24%, DME -- 3.99e+01 / 4.45e+01, 3.96e+00 / 1.77e+00 | 20260507 07:55:19.606 | INFO | PerfReport: 57.81%, FrameLabeling -- 1.78e+01 / 1.98e+01, 1.36e+00 / 1.48e+00 | 20260507 07:55:19.606 | INFO | PerfReport: 6.86%, PulseAccumulating -- 2.11e+00 / 2.69e+00, 3.76e-01 / 2.10e-01 | 20260507 07:55:19.606 | INFO | PerfReport: 2.91%, Metrics -- 8.95e-01 / 1.20e+00, 2.39e-01 / 1.11e-01 | 20260507 07:55:19.606 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.54e-01, 5.11e-02 / 1.99e-02 | 20260507 07:55:19.607 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:55:19.607 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:55:19.607 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:55:19.607 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 07:55:19.607 | INFO | PerfReport: 21.90%, Upload -- 2.11e+01 / 0.00e+00, 7.14e+00 / 0.00e+00 | 20260507 07:55:19.607 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260507 07:55:19.607 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 07:55:19.607 | INFO | PerfReport: 1.53%, LockUpload -- 1.47e+00 / 0.00e+00, 4.90e+00 / 0.00e+00 | 20260507 07:55:19.607 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260507 07:55:19.607 | INFO | PerfReport: 21.95%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 07:55:19.607 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.88e-02 / 0.00e+00 | 20260507 07:55:19.702 | INFO | Starting a round of Baz IO | 20260507 07:55:22.790 | INFO | Populating RTMetrics report | 20260507 07:55:22.790 | INFO | RTMetrics report populated | 20260507 07:55:22.790 | INFO | Opening tmp RTMetrics json file | 20260507 07:55:22.812 | INFO | Serializing RTMetrics to json | 20260507 07:55:22.813 | INFO | Serialization done, swapping tmp file now... | 20260507 07:55:22.814 | INFO | RTMetrics done | 20260507 07:55:22.885 | INFO | Finished a round of Baz IO: Wrote 1.628 GiB in 3.182 seconds | 20260507 07:55:22.885 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.53% / 0.00% / 67.47% | 20260507 07:55:22.885 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0979503% / 3.97364e-06% / 7.94729e-05% | 20260507 07:55:22.885 | INFO | Baz IO overall: | 20260507 07:55:22.885 | INFO | BytesWritten : 4163649774278 | 20260507 07:55:22.885 | INFO | NumEvents : 1992586130244 | 20260507 07:55:24.740 | INFO | Analyzing chunk frames = [8331264,8331776) | 20260507 07:55:24.740 | INFO | MemInfo: { | 20260507 07:55:24.740 | INFO | "gpu" : " 17 GB", | 20260507 07:55:24.740 | INFO | "huge" : "", | 20260507 07:55:24.740 | INFO | "huge_shared" : " 27 GB", | 20260507 07:55:24.740 | INFO | "malloc" : " 14 GB", | 20260507 07:55:24.740 | INFO | "pinned" : " 29 GB" | 20260507 07:55:24.740 | INFO | } | 20260507 07:55:24.740 | INFO | Perf report for the last 5.13543 seconds: | 20260507 07:55:24.740 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:55:24.740 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:24.740 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:24.740 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:24.741 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260507 07:55:24.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:24.741 | INFO | RT_METRICS : 43.1%, 1.0, 22.6ms, 0 | 20260507 07:55:24.741 | INFO | PRE_HQ : 59.4%, 1.9, 57.3ms, 0 | 20260507 07:55:24.741 | INFO | BAZWRITER : 61.6%, 3.3, 104.4ms, 0 | 20260507 07:55:24.741 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:55:24.741 | INFO | Uploaded 36GiB to GPU in 2.05962s (17.4789GiB/s) | 20260507 07:55:24.741 | INFO | Downloaded 38GiB to GPU in 2.17809s (17.4465GiB/s) | 20260507 07:55:29.875 | INFO | Analyzing chunk frames = [8331776,8332288) | 20260507 07:55:29.875 | INFO | MemInfo: { | 20260507 07:55:29.875 | INFO | "gpu" : " 17 GB", | 20260507 07:55:29.875 | INFO | "huge" : "", | 20260507 07:55:29.875 | INFO | "huge_shared" : " 27 GB", | 20260507 07:55:29.875 | INFO | "malloc" : " 14 GB", | 20260507 07:55:29.875 | INFO | "pinned" : " 29 GB" | 20260507 07:55:29.875 | INFO | } | 20260507 07:55:29.875 | INFO | Perf report for the last 5.13494 seconds: | 20260507 07:55:29.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:55:29.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:29.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:29.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:29.875 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:55:29.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:29.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:29.875 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260507 07:55:29.876 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:29.876 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:29.876 | INFO | Uploaded 36GiB to GPU in 1.98814s (18.1074GiB/s) | 20260507 07:55:29.876 | INFO | Downloaded 34GiB to GPU in 1.94066s (17.5198GiB/s) | 20260507 07:55:35.021 | INFO | Analyzing chunk frames = [8332288,8332800) | 20260507 07:55:35.021 | INFO | MemInfo: { | 20260507 07:55:35.021 | INFO | "gpu" : " 17 GB", | 20260507 07:55:35.021 | INFO | "huge" : "", | 20260507 07:55:35.021 | INFO | "huge_shared" : " 27 GB", | 20260507 07:55:35.021 | INFO | "malloc" : " 14 GB", | 20260507 07:55:35.021 | INFO | "pinned" : " 29 GB" | 20260507 07:55:35.021 | INFO | } | 20260507 07:55:35.021 | INFO | Perf report for the last 5.14581 seconds: | 20260507 07:55:35.021 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:55:35.021 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:35.021 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:35.021 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:35.021 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 07:55:35.021 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:35.021 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:35.021 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:55:35.021 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:35.021 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:35.022 | INFO | Uploaded 36GiB to GPU in 2.03033s (17.7311GiB/s) | 20260507 07:55:35.022 | INFO | Downloaded 34GiB to GPU in 1.95452s (17.3956GiB/s) | 20260507 07:55:40.162 | INFO | Analyzing chunk frames = [8332800,8333312) | 20260507 07:55:40.162 | INFO | MemInfo: { | 20260507 07:55:40.162 | INFO | "gpu" : " 17 GB", | 20260507 07:55:40.162 | INFO | "huge" : "", | 20260507 07:55:40.162 | INFO | "huge_shared" : " 27 GB", | 20260507 07:55:40.162 | INFO | "malloc" : " 14 GB", | 20260507 07:55:40.162 | INFO | "pinned" : " 29 GB" | 20260507 07:55:40.162 | INFO | } | 20260507 07:55:40.162 | INFO | Perf report for the last 5.1408 seconds: | 20260507 07:55:40.162 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:55:40.162 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:40.162 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:40.162 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:40.162 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 07:55:40.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:40.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:40.162 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:55:40.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:40.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:40.162 | INFO | Uploaded 36GiB to GPU in 2.01314s (17.8825GiB/s) | 20260507 07:55:40.162 | INFO | Downloaded 34GiB to GPU in 1.94635s (17.4686GiB/s) | 20260507 07:55:45.300 | INFO | Analyzing chunk frames = [8333312,8333824) | 20260507 07:55:45.300 | INFO | MemInfo: { | 20260507 07:55:45.300 | INFO | "gpu" : " 17 GB", | 20260507 07:55:45.300 | INFO | "huge" : "", | 20260507 07:55:45.300 | INFO | "huge_shared" : " 27 GB", | 20260507 07:55:45.300 | INFO | "malloc" : " 14 GB", | 20260507 07:55:45.300 | INFO | "pinned" : " 29 GB" | 20260507 07:55:45.300 | INFO | } | 20260507 07:55:45.300 | INFO | Perf report for the last 5.13792 seconds: | 20260507 07:55:45.300 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:55:45.300 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:45.300 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:45.300 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:45.300 | INFO | ANALYSIS : 59.9%, 3.0, 93.4ms, 0 | 20260507 07:55:45.300 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:45.300 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:45.300 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:55:45.300 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:45.300 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:45.300 | INFO | Uploaded 36GiB to GPU in 2.03275s (17.71GiB/s) | 20260507 07:55:45.300 | INFO | Downloaded 34GiB to GPU in 1.90979s (17.803GiB/s) | 20260507 07:55:50.439 | INFO | Analyzing chunk frames = [8333824,8334336) | 20260507 07:55:50.439 | INFO | MemInfo: { | 20260507 07:55:50.439 | INFO | "gpu" : " 17 GB", | 20260507 07:55:50.439 | INFO | "huge" : "", | 20260507 07:55:50.439 | INFO | "huge_shared" : " 27 GB", | 20260507 07:55:50.439 | INFO | "malloc" : " 14 GB", | 20260507 07:55:50.439 | INFO | "pinned" : " 29 GB" | 20260507 07:55:50.439 | INFO | } | 20260507 07:55:50.439 | INFO | Perf report for the last 5.13906 seconds: | 20260507 07:55:50.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:55:50.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:50.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:50.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:50.439 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 07:55:50.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:50.439 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:50.439 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:55:50.439 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:50.439 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:50.439 | INFO | Uploaded 36GiB to GPU in 2.06651s (17.4207GiB/s) | 20260507 07:55:50.439 | INFO | Downloaded 34GiB to GPU in 1.94414s (17.4884GiB/s) | 20260507 07:55:55.573 | INFO | Analyzing chunk frames = [8334336,8334848) | 20260507 07:55:55.573 | INFO | MemInfo: { | 20260507 07:55:55.573 | INFO | "gpu" : " 17 GB", | 20260507 07:55:55.573 | INFO | "huge" : "", | 20260507 07:55:55.573 | INFO | "huge_shared" : " 27 GB", | 20260507 07:55:55.573 | INFO | "malloc" : " 14 GB", | 20260507 07:55:55.573 | INFO | "pinned" : " 29 GB" | 20260507 07:55:55.573 | INFO | } | 20260507 07:55:55.573 | INFO | Perf report for the last 5.13395 seconds: | 20260507 07:55:55.573 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:55:55.573 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:55.573 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:55.573 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:55.573 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:55:55.573 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:55.573 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:55.573 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:55:55.573 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:55.573 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:55:55.573 | INFO | Uploaded 36GiB to GPU in 1.92342s (18.7167GiB/s) | 20260507 07:55:55.573 | INFO | Downloaded 34GiB to GPU in 1.98335s (17.1427GiB/s) | 20260507 07:56:00.718 | INFO | Analyzing chunk frames = [8334848,8335360) | 20260507 07:56:00.718 | INFO | MemInfo: { | 20260507 07:56:00.718 | INFO | "gpu" : " 17 GB", | 20260507 07:56:00.718 | INFO | "huge" : "", | 20260507 07:56:00.718 | INFO | "huge_shared" : " 27 GB", | 20260507 07:56:00.718 | INFO | "malloc" : " 14 GB", | 20260507 07:56:00.719 | INFO | "pinned" : " 29 GB" | 20260507 07:56:00.719 | INFO | } | 20260507 07:56:00.719 | INFO | Perf report for the last 5.14561 seconds: | 20260507 07:56:00.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:56:00.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:00.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:00.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:00.719 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:56:00.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:00.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:00.719 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:56:00.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:00.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:00.719 | INFO | Uploaded 36GiB to GPU in 1.96461s (18.3242GiB/s) | 20260507 07:56:00.719 | INFO | Downloaded 34GiB to GPU in 1.95506s (17.3908GiB/s) | 20260507 07:56:00.816 | INFO | Starting a round of Baz IO | 20260507 07:56:03.911 | INFO | Populating RTMetrics report | 20260507 07:56:03.912 | INFO | RTMetrics report populated | 20260507 07:56:03.912 | INFO | Opening tmp RTMetrics json file | 20260507 07:56:03.925 | INFO | Serializing RTMetrics to json | 20260507 07:56:03.933 | INFO | Serialization done, swapping tmp file now... | 20260507 07:56:03.934 | INFO | RTMetrics done | 20260507 07:56:04.005 | INFO | Finished a round of Baz IO: Wrote 1.626 GiB in 3.188 seconds | 20260507 07:56:04.005 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.53% / 0.00% / 67.47% | 20260507 07:56:04.005 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.097692% / 3.97364e-06% / 7.54992e-05% | 20260507 07:56:04.005 | INFO | Baz IO overall: | 20260507 07:56:04.005 | INFO | BytesWritten : 4165395161759 | 20260507 07:56:04.005 | INFO | NumEvents : 1993400205461 | 20260507 07:56:05.855 | INFO | Analyzing chunk frames = [8335360,8335872) | 20260507 07:56:05.855 | INFO | MemInfo: { | 20260507 07:56:05.855 | INFO | "gpu" : " 17 GB", | 20260507 07:56:05.855 | INFO | "huge" : "", | 20260507 07:56:05.855 | INFO | "huge_shared" : " 27 GB", | 20260507 07:56:05.855 | INFO | "malloc" : " 14 GB", | 20260507 07:56:05.855 | INFO | "pinned" : " 29 GB" | 20260507 07:56:05.855 | INFO | } | 20260507 07:56:05.855 | INFO | Perf report for the last 5.13666 seconds: | 20260507 07:56:05.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:56:05.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:05.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:05.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:05.855 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 07:56:05.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:05.855 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 07:56:05.855 | INFO | PRE_HQ : 58.8%, 1.8, 54.3ms, 0 | 20260507 07:56:05.855 | INFO | BAZWRITER : 61.7%, 3.2, 101.2ms, 0 | 20260507 07:56:05.855 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 07:56:05.856 | INFO | Uploaded 36GiB to GPU in 2.0696s (17.3947GiB/s) | 20260507 07:56:05.856 | INFO | Downloaded 38GiB to GPU in 2.18159s (17.4185GiB/s) | 20260507 07:56:10.994 | INFO | Analyzing chunk frames = [8335872,8336384) | 20260507 07:56:10.994 | INFO | MemInfo: { | 20260507 07:56:10.994 | INFO | "gpu" : " 17 GB", | 20260507 07:56:10.994 | INFO | "huge" : "", | 20260507 07:56:10.994 | INFO | "huge_shared" : " 27 GB", | 20260507 07:56:10.994 | INFO | "malloc" : " 14 GB", | 20260507 07:56:10.994 | INFO | "pinned" : " 29 GB" | 20260507 07:56:10.994 | INFO | } | 20260507 07:56:10.994 | INFO | Perf report for the last 5.13913 seconds: | 20260507 07:56:10.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:56:10.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:10.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:10.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:10.995 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 07:56:10.995 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:10.995 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:10.995 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 07:56:10.995 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:10.995 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:10.995 | INFO | Uploaded 36GiB to GPU in 1.98428s (18.1426GiB/s) | 20260507 07:56:10.995 | INFO | Downloaded 34GiB to GPU in 1.92999s (17.6166GiB/s) | 20260507 07:56:16.129 | INFO | Analyzing chunk frames = [8336384,8336896) | 20260507 07:56:16.129 | INFO | MemInfo: { | 20260507 07:56:16.129 | INFO | "gpu" : " 17 GB", | 20260507 07:56:16.129 | INFO | "huge" : "", | 20260507 07:56:16.129 | INFO | "huge_shared" : " 27 GB", | 20260507 07:56:16.129 | INFO | "malloc" : " 14 GB", | 20260507 07:56:16.129 | INFO | "pinned" : " 29 GB" | 20260507 07:56:16.129 | INFO | } | 20260507 07:56:16.130 | INFO | Perf report for the last 5.13511 seconds: | 20260507 07:56:16.130 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:56:16.130 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:16.130 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:16.130 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:16.130 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 07:56:16.130 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:16.130 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:16.130 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:56:16.130 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:16.130 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:16.130 | INFO | Uploaded 36GiB to GPU in 2.05834s (17.4898GiB/s) | 20260507 07:56:16.130 | INFO | Downloaded 34GiB to GPU in 1.95121s (17.425GiB/s) | 20260507 07:56:21.273 | INFO | Analyzing chunk frames = [8336896,8337408) | 20260507 07:56:21.273 | INFO | MemInfo: { | 20260507 07:56:21.273 | INFO | "gpu" : " 17 GB", | 20260507 07:56:21.273 | INFO | "huge" : "", | 20260507 07:56:21.273 | INFO | "huge_shared" : " 27 GB", | 20260507 07:56:21.273 | INFO | "malloc" : " 14 GB", | 20260507 07:56:21.273 | INFO | "pinned" : " 29 GB" | 20260507 07:56:21.273 | INFO | } | 20260507 07:56:21.273 | INFO | Perf report for the last 5.14346 seconds: | 20260507 07:56:21.273 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:56:21.273 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:21.273 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:21.273 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:21.273 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 07:56:21.273 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:21.273 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:21.273 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 07:56:21.273 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:21.273 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:21.273 | INFO | Uploaded 36GiB to GPU in 2.01879s (17.8325GiB/s) | 20260507 07:56:21.273 | INFO | Downloaded 34GiB to GPU in 1.93767s (17.5469GiB/s) | 20260507 07:56:26.414 | INFO | Analyzing chunk frames = [8337408,8337920) | 20260507 07:56:26.415 | INFO | MemInfo: { | 20260507 07:56:26.415 | INFO | "gpu" : " 17 GB", | 20260507 07:56:26.415 | INFO | "huge" : "", | 20260507 07:56:26.415 | INFO | "huge_shared" : " 27 GB", | 20260507 07:56:26.415 | INFO | "malloc" : " 14 GB", | 20260507 07:56:26.415 | INFO | "pinned" : " 29 GB" | 20260507 07:56:26.415 | INFO | } | 20260507 07:56:26.415 | INFO | Perf report for the last 5.14176 seconds: | 20260507 07:56:26.415 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:56:26.415 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:26.415 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:26.415 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:26.415 | INFO | ANALYSIS : 60.2%, 3.0, 93.9ms, 0 | 20260507 07:56:26.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:26.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:26.415 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:56:26.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:26.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:26.415 | INFO | Uploaded 36GiB to GPU in 2.0581s (17.4918GiB/s) | 20260507 07:56:26.415 | INFO | Downloaded 34GiB to GPU in 1.95157s (17.4219GiB/s) | 20260507 07:56:31.553 | INFO | Analyzing chunk frames = [8337920,8338432) | 20260507 07:56:31.553 | INFO | MemInfo: { | 20260507 07:56:31.553 | INFO | "gpu" : " 17 GB", | 20260507 07:56:31.553 | INFO | "huge" : "", | 20260507 07:56:31.553 | INFO | "huge_shared" : " 27 GB", | 20260507 07:56:31.554 | INFO | "malloc" : " 14 GB", | 20260507 07:56:31.554 | INFO | "pinned" : " 29 GB" | 20260507 07:56:31.554 | INFO | } | 20260507 07:56:31.554 | INFO | Perf report for the last 5.13886 seconds: | 20260507 07:56:31.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:56:31.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:31.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:31.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:31.554 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 07:56:31.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:31.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:31.554 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 07:56:31.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:31.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:31.554 | INFO | Uploaded 36GiB to GPU in 2.07373s (17.36GiB/s) | 20260507 07:56:31.554 | INFO | Downloaded 34GiB to GPU in 1.93324s (17.5871GiB/s) | 20260507 07:56:36.690 | INFO | Analyzing chunk frames = [8338432,8338944) | 20260507 07:56:36.690 | INFO | MemInfo: { | 20260507 07:56:36.690 | INFO | "gpu" : " 17 GB", | 20260507 07:56:36.690 | INFO | "huge" : "", | 20260507 07:56:36.690 | INFO | "huge_shared" : " 27 GB", | 20260507 07:56:36.690 | INFO | "malloc" : " 14 GB", | 20260507 07:56:36.690 | INFO | "pinned" : " 29 GB" | 20260507 07:56:36.690 | INFO | } | 20260507 07:56:36.691 | INFO | Perf report for the last 5.13688 seconds: | 20260507 07:56:36.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:56:36.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:36.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:36.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:36.691 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 07:56:36.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:36.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:36.691 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:56:36.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:36.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:36.691 | INFO | Uploaded 36GiB to GPU in 1.91886s (18.7612GiB/s) | 20260507 07:56:36.691 | INFO | Downloaded 34GiB to GPU in 1.9808s (17.1648GiB/s) | 20260507 07:56:41.827 | INFO | Analyzing chunk frames = [8338944,8339456) | 20260507 07:56:41.828 | INFO | MemInfo: { | 20260507 07:56:41.828 | INFO | "gpu" : " 17 GB", | 20260507 07:56:41.828 | INFO | "huge" : "", | 20260507 07:56:41.828 | INFO | "huge_shared" : " 27 GB", | 20260507 07:56:41.828 | INFO | "malloc" : " 14 GB", | 20260507 07:56:41.828 | INFO | "pinned" : " 29 GB" | 20260507 07:56:41.828 | INFO | } | 20260507 07:56:41.828 | INFO | Perf report for the last 5.13718 seconds: | 20260507 07:56:41.828 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:56:41.828 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:41.828 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:41.828 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:41.828 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:56:41.828 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:41.828 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:41.828 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:56:41.828 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:41.828 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:41.828 | INFO | Uploaded 36GiB to GPU in 1.98668s (18.1207GiB/s) | 20260507 07:56:41.828 | INFO | Downloaded 34GiB to GPU in 1.95513s (17.3902GiB/s) | 20260507 07:56:41.828 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:56:41.828 | INFO | Total mem usage: 18 GiB | 20260507 07:56:41.828 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:56:41.828 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:56:41.828 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:56:41.828 | INFO | High Water Breakdown: | 20260507 07:56:41.828 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:56:41.828 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:56:41.828 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:56:41.828 | INFO | ------------------------------------------ | 20260507 07:56:41.828 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:56:41.829 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:56:41.829 | INFO | Total mem usage: 26.5 GiB | 20260507 07:56:41.829 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:56:41.829 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:56:41.829 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:56:41.829 | INFO | High Water Breakdown: | 20260507 07:56:41.829 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:56:41.829 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:56:41.829 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:56:41.829 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:56:41.829 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:56:41.829 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:56:41.829 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:56:41.829 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:56:41.829 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:56:41.829 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:56:41.829 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:56:41.829 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:56:41.829 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:56:41.829 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:56:41.829 | INFO | ------------------------------------------ | 20260507 07:56:41.829 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:56:41.829 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:56:41.829 | INFO | Total mem usage: 25.6 GiB | 20260507 07:56:41.829 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:56:41.829 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:56:41.829 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:56:41.829 | INFO | High Water Breakdown: | 20260507 07:56:41.829 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:56:41.829 | INFO | ------------------------------------------ | 20260507 07:56:41.829 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:56:41.829 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:56:41.829 | INFO | Total mem usage: 15.7 GiB | 20260507 07:56:41.829 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:56:41.829 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:56:41.829 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:56:41.829 | INFO | High Water Breakdown: | 20260507 07:56:41.829 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:56:41.829 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:56:41.829 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:56:41.829 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:56:41.829 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:56:41.829 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:56:41.829 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:56:41.829 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:56:41.829 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:56:41.829 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:56:41.829 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:56:41.829 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:56:41.829 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:56:41.829 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:56:41.829 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:56:41.829 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:56:41.829 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:56:41.829 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:56:41.829 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:56:41.829 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:56:41.829 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:56:41.829 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:56:41.829 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:56:41.829 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:56:41.829 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:56:41.829 | INFO | ------------------------------------------ | 20260507 07:56:41.829 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:56:41.829 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:56:41.829 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:56:41.829 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:56:41.829 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.45e+01, 1.32e+01 / 1.01e+00 | 20260507 07:56:41.829 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.02e-01, 9.90e-03 / 3.83e-03 | 20260507 07:56:41.829 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.47e+00, 4.95e-01 / 2.44e-01 | 20260507 07:56:41.829 | INFO | PerfReport: 3.22%, Histogram -- 9.86e-01 / 1.49e+00, 1.65e-01 / 9.32e-02 | 20260507 07:56:41.829 | INFO | PerfReport: 16.27%, DME -- 3.99e+01 / 4.45e+01, 3.88e+00 / 1.34e+00 | 20260507 07:56:41.829 | INFO | PerfReport: 57.73%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.23e+00 / 9.66e-01 | 20260507 07:56:41.829 | INFO | PerfReport: 6.92%, PulseAccumulating -- 2.12e+00 / 2.69e+00, 3.88e-01 / 1.77e-01 | 20260507 07:56:41.829 | INFO | PerfReport: 2.93%, Metrics -- 8.97e-01 / 1.20e+00, 2.41e-01 / 1.53e-01 | 20260507 07:56:41.829 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.56e-01, 5.01e-02 / 2.50e-02 | 20260507 07:56:41.829 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:56:41.829 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:56:41.829 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:56:41.829 | INFO | PerfReport: Full Execution -- 9.60e+01 / -nan, 2.37e+01 / 0.00e+00 | 20260507 07:56:41.829 | INFO | PerfReport: 21.86%, Upload -- 2.10e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 07:56:41.829 | INFO | PerfReport: 21.45%, Download -- 2.06e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260507 07:56:41.829 | INFO | PerfReport: 31.91%, Compute -- 3.06e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 07:56:41.829 | INFO | PerfReport: 1.52%, LockUpload -- 1.46e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 07:56:41.829 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260507 07:56:41.829 | INFO | PerfReport: 21.95%, LockCompute -- 2.11e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 07:56:41.829 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260507 07:56:41.925 | INFO | Starting a round of Baz IO | 20260507 07:56:45.017 | INFO | Populating RTMetrics report | 20260507 07:56:45.017 | INFO | RTMetrics report populated | 20260507 07:56:45.017 | INFO | Opening tmp RTMetrics json file | 20260507 07:56:45.038 | INFO | Serializing RTMetrics to json | 20260507 07:56:45.039 | INFO | Serialization done, swapping tmp file now... | 20260507 07:56:45.039 | INFO | RTMetrics done | 20260507 07:56:45.107 | INFO | Finished a round of Baz IO: Wrote 1.625 GiB in 3.181 seconds | 20260507 07:56:45.107 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.53% / 0.00% / 67.47% | 20260507 07:56:45.107 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0983397% / 0% / 5.5631e-05% | 20260507 07:56:45.107 | INFO | Baz IO overall: | 20260507 07:56:45.107 | INFO | BytesWritten : 4167140235391 | 20260507 07:56:45.107 | INFO | NumEvents : 1994213907552 | 20260507 07:56:46.976 | INFO | Analyzing chunk frames = [8339456,8339968) | 20260507 07:56:46.976 | INFO | MemInfo: { | 20260507 07:56:46.976 | INFO | "gpu" : " 17 GB", | 20260507 07:56:46.976 | INFO | "huge" : "", | 20260507 07:56:46.976 | INFO | "huge_shared" : " 27 GB", | 20260507 07:56:46.976 | INFO | "malloc" : " 14 GB", | 20260507 07:56:46.976 | INFO | "pinned" : " 29 GB" | 20260507 07:56:46.976 | INFO | } | 20260507 07:56:46.976 | INFO | Perf report for the last 5.14803 seconds: | 20260507 07:56:46.976 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:56:46.976 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:46.976 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:46.976 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:46.976 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260507 07:56:46.976 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:46.976 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 07:56:46.976 | INFO | PRE_HQ : 58.9%, 1.8, 54.7ms, 0 | 20260507 07:56:46.976 | INFO | BAZWRITER : 61.4%, 3.1, 99.1ms, 0 | 20260507 07:56:46.976 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:56:46.976 | INFO | Uploaded 36GiB to GPU in 2.04319s (17.6195GiB/s) | 20260507 07:56:46.976 | INFO | Downloaded 38GiB to GPU in 2.17327s (17.4851GiB/s) | 20260507 07:56:52.118 | INFO | Analyzing chunk frames = [8339968,8340480) | 20260507 07:56:52.118 | INFO | MemInfo: { | 20260507 07:56:52.118 | INFO | "gpu" : " 17 GB", | 20260507 07:56:52.118 | INFO | "huge" : "", | 20260507 07:56:52.118 | INFO | "huge_shared" : " 27 GB", | 20260507 07:56:52.118 | INFO | "malloc" : " 14 GB", | 20260507 07:56:52.118 | INFO | "pinned" : " 29 GB" | 20260507 07:56:52.118 | INFO | } | 20260507 07:56:52.118 | INFO | Perf report for the last 5.14204 seconds: | 20260507 07:56:52.118 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:56:52.118 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:52.118 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:52.118 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:52.118 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:56:52.118 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:52.118 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:52.118 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:56:52.118 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:52.118 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:52.118 | INFO | Uploaded 36GiB to GPU in 2.00502s (17.955GiB/s) | 20260507 07:56:52.118 | INFO | Downloaded 34GiB to GPU in 1.93536s (17.5678GiB/s) | 20260507 07:56:57.252 | INFO | Analyzing chunk frames = [8340480,8340992) | 20260507 07:56:57.252 | INFO | MemInfo: { | 20260507 07:56:57.252 | INFO | "gpu" : " 17 GB", | 20260507 07:56:57.252 | INFO | "huge" : "", | 20260507 07:56:57.252 | INFO | "huge_shared" : " 27 GB", | 20260507 07:56:57.252 | INFO | "malloc" : " 14 GB", | 20260507 07:56:57.252 | INFO | "pinned" : " 29 GB" | 20260507 07:56:57.252 | INFO | } | 20260507 07:56:57.252 | INFO | Perf report for the last 5.13401 seconds: | 20260507 07:56:57.252 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:56:57.252 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:57.252 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:57.252 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:57.252 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 07:56:57.252 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:57.252 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:57.252 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:56:57.252 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:57.252 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:56:57.252 | INFO | Uploaded 36GiB to GPU in 2.03351s (17.7034GiB/s) | 20260507 07:56:57.252 | INFO | Downloaded 34GiB to GPU in 1.91552s (17.7498GiB/s) | 20260507 07:57:02.385 | INFO | Analyzing chunk frames = [8340992,8341504) | 20260507 07:57:02.385 | INFO | MemInfo: { | 20260507 07:57:02.385 | INFO | "gpu" : " 17 GB", | 20260507 07:57:02.385 | INFO | "huge" : "", | 20260507 07:57:02.385 | INFO | "huge_shared" : " 27 GB", | 20260507 07:57:02.385 | INFO | "malloc" : " 14 GB", | 20260507 07:57:02.385 | INFO | "pinned" : " 29 GB" | 20260507 07:57:02.385 | INFO | } | 20260507 07:57:02.386 | INFO | Perf report for the last 5.13429 seconds: | 20260507 07:57:02.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:57:02.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:02.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:02.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:02.386 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 07:57:02.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:02.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:02.386 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:57:02.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:02.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:02.386 | INFO | Uploaded 36GiB to GPU in 2.05516s (17.5168GiB/s) | 20260507 07:57:02.386 | INFO | Downloaded 34GiB to GPU in 1.94996s (17.4363GiB/s) | 20260507 07:57:07.525 | INFO | Analyzing chunk frames = [8341504,8342016) | 20260507 07:57:07.525 | INFO | MemInfo: { | 20260507 07:57:07.525 | INFO | "gpu" : " 17 GB", | 20260507 07:57:07.525 | INFO | "huge" : "", | 20260507 07:57:07.525 | INFO | "huge_shared" : " 27 GB", | 20260507 07:57:07.525 | INFO | "malloc" : " 14 GB", | 20260507 07:57:07.525 | INFO | "pinned" : " 29 GB" | 20260507 07:57:07.525 | INFO | } | 20260507 07:57:07.525 | INFO | Perf report for the last 5.13912 seconds: | 20260507 07:57:07.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:57:07.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:07.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:07.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:07.525 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 07:57:07.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:07.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:07.526 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 07:57:07.526 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:07.526 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:07.526 | INFO | Uploaded 36GiB to GPU in 2.05836s (17.4896GiB/s) | 20260507 07:57:07.526 | INFO | Downloaded 34GiB to GPU in 1.94841s (17.4501GiB/s) | 20260507 07:57:12.669 | INFO | Analyzing chunk frames = [8342016,8342528) | 20260507 07:57:12.669 | INFO | MemInfo: { | 20260507 07:57:12.669 | INFO | "gpu" : " 17 GB", | 20260507 07:57:12.669 | INFO | "huge" : "", | 20260507 07:57:12.669 | INFO | "huge_shared" : " 27 GB", | 20260507 07:57:12.669 | INFO | "malloc" : " 14 GB", | 20260507 07:57:12.669 | INFO | "pinned" : " 29 GB" | 20260507 07:57:12.669 | INFO | } | 20260507 07:57:12.669 | INFO | Perf report for the last 5.14403 seconds: | 20260507 07:57:12.669 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:57:12.669 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:12.669 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:12.669 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:12.669 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 07:57:12.669 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:12.669 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:12.669 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 07:57:12.669 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:12.669 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:12.670 | INFO | Uploaded 36GiB to GPU in 2.0479s (17.579GiB/s) | 20260507 07:57:12.670 | INFO | Downloaded 34GiB to GPU in 1.91263s (17.7766GiB/s) | 20260507 07:57:17.811 | INFO | Analyzing chunk frames = [8342528,8343040) | 20260507 07:57:17.811 | INFO | MemInfo: { | 20260507 07:57:17.811 | INFO | "gpu" : " 17 GB", | 20260507 07:57:17.811 | INFO | "huge" : "", | 20260507 07:57:17.811 | INFO | "huge_shared" : " 27 GB", | 20260507 07:57:17.811 | INFO | "malloc" : " 14 GB", | 20260507 07:57:17.811 | INFO | "pinned" : " 29 GB" | 20260507 07:57:17.811 | INFO | } | 20260507 07:57:17.811 | INFO | Perf report for the last 5.1415 seconds: | 20260507 07:57:17.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:57:17.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:17.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:17.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:17.811 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 07:57:17.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:17.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:17.811 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 07:57:17.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:17.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:17.811 | INFO | Uploaded 36GiB to GPU in 1.89552s (18.9921GiB/s) | 20260507 07:57:17.811 | INFO | Downloaded 34GiB to GPU in 1.9732s (17.2309GiB/s) | 20260507 07:57:22.948 | INFO | Analyzing chunk frames = [8343040,8343552) | 20260507 07:57:22.948 | INFO | MemInfo: { | 20260507 07:57:22.948 | INFO | "gpu" : " 17 GB", | 20260507 07:57:22.948 | INFO | "huge" : "", | 20260507 07:57:22.948 | INFO | "huge_shared" : " 27 GB", | 20260507 07:57:22.948 | INFO | "malloc" : " 14 GB", | 20260507 07:57:22.948 | INFO | "pinned" : " 29 GB" | 20260507 07:57:22.948 | INFO | } | 20260507 07:57:22.948 | INFO | Perf report for the last 5.13709 seconds: | 20260507 07:57:22.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:57:22.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:22.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:22.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:22.948 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 07:57:22.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:22.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:22.948 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 07:57:22.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:22.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:22.948 | INFO | Uploaded 36GiB to GPU in 1.98096s (18.173GiB/s) | 20260507 07:57:22.948 | INFO | Downloaded 34GiB to GPU in 1.96197s (17.3295GiB/s) | 20260507 07:57:23.045 | INFO | Starting a round of Baz IO | 20260507 07:57:26.133 | INFO | Populating RTMetrics report | 20260507 07:57:26.133 | INFO | RTMetrics report populated | 20260507 07:57:26.133 | INFO | Opening tmp RTMetrics json file | 20260507 07:57:26.147 | INFO | Serializing RTMetrics to json | 20260507 07:57:26.152 | INFO | Serialization done, swapping tmp file now... | 20260507 07:57:26.152 | INFO | RTMetrics done | 20260507 07:57:26.225 | INFO | Finished a round of Baz IO: Wrote 1.625 GiB in 3.18 seconds | 20260507 07:57:26.225 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.53% / 0.00% / 67.47% | 20260507 07:57:26.225 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0964244% / 7.94729e-06% / 6.75519e-05% | 20260507 07:57:26.225 | INFO | Baz IO overall: | 20260507 07:57:26.225 | INFO | BytesWritten : 4168884862036 | 20260507 07:57:26.225 | INFO | NumEvents : 1995027516738 | 20260507 07:57:28.084 | INFO | Analyzing chunk frames = [8343552,8344064) | 20260507 07:57:28.084 | INFO | MemInfo: { | 20260507 07:57:28.084 | INFO | "gpu" : " 17 GB", | 20260507 07:57:28.084 | INFO | "huge" : "", | 20260507 07:57:28.084 | INFO | "huge_shared" : " 27 GB", | 20260507 07:57:28.086 | INFO | "malloc" : " 14 GB", | 20260507 07:57:28.086 | INFO | "pinned" : " 29 GB" | 20260507 07:57:28.086 | INFO | } | 20260507 07:57:28.086 | INFO | Perf report for the last 5.13822 seconds: | 20260507 07:57:28.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:57:28.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:28.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:28.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:28.086 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 07:57:28.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:28.086 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 07:57:28.086 | INFO | PRE_HQ : 58.9%, 1.8, 54.6ms, 0 | 20260507 07:57:28.086 | INFO | BAZWRITER : 61.5%, 3.2, 100.5ms, 0 | 20260507 07:57:28.086 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:57:28.086 | INFO | Uploaded 36GiB to GPU in 2.0522s (17.5421GiB/s) | 20260507 07:57:28.087 | INFO | Downloaded 38GiB to GPU in 2.18005s (17.4308GiB/s) | 20260507 07:57:33.223 | INFO | Analyzing chunk frames = [8344064,8344576) | 20260507 07:57:33.223 | INFO | MemInfo: { | 20260507 07:57:33.223 | INFO | "gpu" : " 17 GB", | 20260507 07:57:33.223 | INFO | "huge" : "", | 20260507 07:57:33.223 | INFO | "huge_shared" : " 27 GB", | 20260507 07:57:33.223 | INFO | "malloc" : " 14 GB", | 20260507 07:57:33.223 | INFO | "pinned" : " 29 GB" | 20260507 07:57:33.223 | INFO | } | 20260507 07:57:33.224 | INFO | Perf report for the last 5.13734 seconds: | 20260507 07:57:33.224 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:57:33.224 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:33.224 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:33.224 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:33.224 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 07:57:33.224 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:33.224 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:33.224 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 07:57:33.224 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:33.224 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:33.224 | INFO | Uploaded 36GiB to GPU in 1.98174s (18.1659GiB/s) | 20260507 07:57:33.224 | INFO | Downloaded 34GiB to GPU in 1.93575s (17.5643GiB/s) | 20260507 07:57:38.362 | INFO | Analyzing chunk frames = [8344576,8345088) | 20260507 07:57:38.362 | INFO | MemInfo: { | 20260507 07:57:38.362 | INFO | "gpu" : " 17 GB", | 20260507 07:57:38.362 | INFO | "huge" : "", | 20260507 07:57:38.362 | INFO | "huge_shared" : " 27 GB", | 20260507 07:57:38.362 | INFO | "malloc" : " 14 GB", | 20260507 07:57:38.362 | INFO | "pinned" : " 29 GB" | 20260507 07:57:38.362 | INFO | } | 20260507 07:57:38.362 | INFO | Perf report for the last 5.13852 seconds: | 20260507 07:57:38.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:57:38.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:38.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:38.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:38.362 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:57:38.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:38.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:38.362 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:57:38.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:38.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:38.362 | INFO | Uploaded 36GiB to GPU in 2.04609s (17.5945GiB/s) | 20260507 07:57:38.362 | INFO | Downloaded 34GiB to GPU in 1.95644s (17.3785GiB/s) | 20260507 07:57:43.502 | INFO | Analyzing chunk frames = [8345088,8345600) | 20260507 07:57:43.502 | INFO | MemInfo: { | 20260507 07:57:43.502 | INFO | "gpu" : " 17 GB", | 20260507 07:57:43.502 | INFO | "huge" : "", | 20260507 07:57:43.502 | INFO | "huge_shared" : " 27 GB", | 20260507 07:57:43.502 | INFO | "malloc" : " 14 GB", | 20260507 07:57:43.502 | INFO | "pinned" : " 29 GB" | 20260507 07:57:43.502 | INFO | } | 20260507 07:57:43.502 | INFO | Perf report for the last 5.13995 seconds: | 20260507 07:57:43.502 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:57:43.502 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:43.502 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:43.502 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:43.502 | INFO | ANALYSIS : 60.2%, 2.9, 93.1ms, 0 | 20260507 07:57:43.502 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:43.502 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:43.502 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:57:43.502 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:43.502 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:43.502 | INFO | Uploaded 36GiB to GPU in 2.02401s (17.7864GiB/s) | 20260507 07:57:43.502 | INFO | Downloaded 34GiB to GPU in 1.9562s (17.3806GiB/s) | 20260507 07:57:48.640 | INFO | Analyzing chunk frames = [8345600,8346112) | 20260507 07:57:48.640 | INFO | MemInfo: { | 20260507 07:57:48.640 | INFO | "gpu" : " 17 GB", | 20260507 07:57:48.640 | INFO | "huge" : "", | 20260507 07:57:48.640 | INFO | "huge_shared" : " 27 GB", | 20260507 07:57:48.640 | INFO | "malloc" : " 14 GB", | 20260507 07:57:48.640 | INFO | "pinned" : " 29 GB" | 20260507 07:57:48.640 | INFO | } | 20260507 07:57:48.641 | INFO | Perf report for the last 5.13849 seconds: | 20260507 07:57:48.641 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:57:48.641 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:48.641 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:48.641 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:48.641 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 07:57:48.641 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:48.641 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:48.641 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:57:48.641 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:48.641 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:48.641 | INFO | Uploaded 36GiB to GPU in 2.03409s (17.6984GiB/s) | 20260507 07:57:48.641 | INFO | Downloaded 34GiB to GPU in 1.92703s (17.6437GiB/s) | 20260507 07:57:53.778 | INFO | Analyzing chunk frames = [8346112,8346624) | 20260507 07:57:53.778 | INFO | MemInfo: { | 20260507 07:57:53.778 | INFO | "gpu" : " 17 GB", | 20260507 07:57:53.778 | INFO | "huge" : "", | 20260507 07:57:53.778 | INFO | "huge_shared" : " 27 GB", | 20260507 07:57:53.778 | INFO | "malloc" : " 14 GB", | 20260507 07:57:53.778 | INFO | "pinned" : " 29 GB" | 20260507 07:57:53.778 | INFO | } | 20260507 07:57:53.778 | INFO | Perf report for the last 5.13789 seconds: | 20260507 07:57:53.778 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:57:53.778 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:53.778 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:53.778 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:53.778 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:57:53.778 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:53.778 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:53.778 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 07:57:53.778 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:53.778 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:53.779 | INFO | Uploaded 36GiB to GPU in 2.05766s (17.4956GiB/s) | 20260507 07:57:53.779 | INFO | Downloaded 34GiB to GPU in 1.92284s (17.6822GiB/s) | 20260507 07:57:58.925 | INFO | Analyzing chunk frames = [8346624,8347136) | 20260507 07:57:58.925 | INFO | MemInfo: { | 20260507 07:57:58.925 | INFO | "gpu" : " 17 GB", | 20260507 07:57:58.925 | INFO | "huge" : "", | 20260507 07:57:58.925 | INFO | "huge_shared" : " 27 GB", | 20260507 07:57:58.925 | INFO | "malloc" : " 14 GB", | 20260507 07:57:58.925 | INFO | "pinned" : " 29 GB" | 20260507 07:57:58.925 | INFO | } | 20260507 07:57:58.925 | INFO | Perf report for the last 5.14656 seconds: | 20260507 07:57:58.925 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:57:58.925 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:58.925 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:58.925 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:58.925 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 07:57:58.925 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:58.925 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:58.925 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:57:58.925 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:58.925 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:57:58.925 | INFO | Uploaded 36GiB to GPU in 1.99733s (18.024GiB/s) | 20260507 07:57:58.925 | INFO | Downloaded 34GiB to GPU in 1.98539s (17.1251GiB/s) | 20260507 07:58:04.065 | INFO | Analyzing chunk frames = [8347136,8347648) | 20260507 07:58:04.065 | INFO | MemInfo: { | 20260507 07:58:04.065 | INFO | "gpu" : " 17 GB", | 20260507 07:58:04.065 | INFO | "huge" : "", | 20260507 07:58:04.065 | INFO | "huge_shared" : " 27 GB", | 20260507 07:58:04.065 | INFO | "malloc" : " 14 GB", | 20260507 07:58:04.066 | INFO | "pinned" : " 29 GB" | 20260507 07:58:04.066 | INFO | } | 20260507 07:58:04.066 | INFO | Perf report for the last 5.14057 seconds: | 20260507 07:58:04.066 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:58:04.066 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:04.066 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:04.066 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:04.066 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:58:04.066 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:04.066 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:04.066 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:58:04.066 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:04.066 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:04.066 | INFO | Uploaded 36GiB to GPU in 1.98671s (18.1204GiB/s) | 20260507 07:58:04.066 | INFO | Downloaded 34GiB to GPU in 1.96725s (17.2831GiB/s) | 20260507 07:58:04.066 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:58:04.066 | INFO | Total mem usage: 18 GiB | 20260507 07:58:04.066 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:58:04.066 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:58:04.066 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:58:04.066 | INFO | High Water Breakdown: | 20260507 07:58:04.066 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:58:04.066 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:58:04.066 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:58:04.066 | INFO | ------------------------------------------ | 20260507 07:58:04.066 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:58:04.066 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:58:04.066 | INFO | Total mem usage: 26.5 GiB | 20260507 07:58:04.066 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:58:04.066 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:58:04.066 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:58:04.066 | INFO | High Water Breakdown: | 20260507 07:58:04.066 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:58:04.066 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:58:04.066 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:58:04.066 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:58:04.066 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:58:04.066 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:58:04.066 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:58:04.066 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:58:04.066 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:58:04.066 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:58:04.066 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:58:04.066 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:58:04.066 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:58:04.066 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:58:04.066 | INFO | ------------------------------------------ | 20260507 07:58:04.066 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:58:04.066 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:58:04.066 | INFO | Total mem usage: 25.6 GiB | 20260507 07:58:04.066 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:58:04.066 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:58:04.066 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:58:04.067 | INFO | High Water Breakdown: | 20260507 07:58:04.067 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:58:04.067 | INFO | ------------------------------------------ | 20260507 07:58:04.067 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:58:04.067 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:58:04.067 | INFO | Total mem usage: 15.7 GiB | 20260507 07:58:04.067 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:58:04.067 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:58:04.067 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:58:04.067 | INFO | High Water Breakdown: | 20260507 07:58:04.067 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:58:04.067 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:58:04.067 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:58:04.067 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:58:04.067 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:58:04.067 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:58:04.067 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:58:04.067 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:58:04.067 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:58:04.067 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:58:04.067 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:58:04.067 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:58:04.067 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:58:04.067 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:58:04.067 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:58:04.067 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:58:04.067 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:58:04.067 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:58:04.067 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:58:04.067 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:58:04.067 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:58:04.067 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:58:04.067 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:58:04.067 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:58:04.067 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:58:04.067 | INFO | ------------------------------------------ | 20260507 07:58:04.067 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:58:04.067 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:58:04.067 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:58:04.067 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:58:04.067 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.35e+01 / 1.14e+00 | 20260507 07:58:04.067 | INFO | PerfReport: 0.30%, AggregateStats -- 9.05e-02 / 1.03e-01, 9.58e-03 / 4.12e-03 | 20260507 07:58:04.067 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.46e+00, 4.97e-01 / 2.20e-01 | 20260507 07:58:04.067 | INFO | PerfReport: 3.21%, Histogram -- 9.85e-01 / 1.49e+00, 1.64e-01 / 1.19e-01 | 20260507 07:58:04.067 | INFO | PerfReport: 16.36%, DME -- 4.01e+01 / 4.52e+01, 3.94e+00 / 1.37e+00 | 20260507 07:58:04.067 | INFO | PerfReport: 57.68%, FrameLabeling -- 1.77e+01 / 1.93e+01, 1.25e+00 / 1.11e+00 | 20260507 07:58:04.067 | INFO | PerfReport: 6.91%, PulseAccumulating -- 2.12e+00 / 2.69e+00, 3.89e-01 / 2.12e-01 | 20260507 07:58:04.067 | INFO | PerfReport: 2.92%, Metrics -- 8.95e-01 / 1.19e+00, 2.40e-01 / 8.01e-02 | 20260507 07:58:04.067 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.57e-01, 5.04e-02 / 2.05e-02 | 20260507 07:58:04.067 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:58:04.067 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:58:04.067 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:58:04.067 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260507 07:58:04.067 | INFO | PerfReport: 21.87%, Upload -- 2.10e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260507 07:58:04.067 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260507 07:58:04.067 | INFO | PerfReport: 31.90%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 07:58:04.067 | INFO | PerfReport: 1.57%, LockUpload -- 1.51e+00 / 0.00e+00, 4.96e+00 / 0.00e+00 | 20260507 07:58:04.067 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260507 07:58:04.067 | INFO | PerfReport: 21.97%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 07:58:04.067 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.63e-02 / 0.00e+00 | 20260507 07:58:04.163 | INFO | Starting a round of Baz IO | 20260507 07:58:07.278 | INFO | Populating RTMetrics report | 20260507 07:58:07.278 | INFO | RTMetrics report populated | 20260507 07:58:07.278 | INFO | Opening tmp RTMetrics json file | 20260507 07:58:07.293 | INFO | Serializing RTMetrics to json | 20260507 07:58:07.294 | INFO | Serialization done, swapping tmp file now... | 20260507 07:58:07.294 | INFO | RTMetrics done | 20260507 07:58:07.358 | INFO | Finished a round of Baz IO: Wrote 1.626 GiB in 3.195 seconds | 20260507 07:58:07.358 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.53% / 0.00% / 67.47% | 20260507 07:58:07.358 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0971119% / 0% / 6.75519e-05% | 20260507 07:58:07.358 | INFO | Baz IO overall: | 20260507 07:58:07.358 | INFO | BytesWritten : 4170631197637 | 20260507 07:58:07.358 | INFO | NumEvents : 1995842510303 | 20260507 07:58:09.195 | INFO | Analyzing chunk frames = [8347648,8348160) | 20260507 07:58:09.195 | INFO | MemInfo: { | 20260507 07:58:09.195 | INFO | "gpu" : " 17 GB", | 20260507 07:58:09.195 | INFO | "huge" : "", | 20260507 07:58:09.195 | INFO | "huge_shared" : " 27 GB", | 20260507 07:58:09.195 | INFO | "malloc" : " 14 GB", | 20260507 07:58:09.195 | INFO | "pinned" : " 29 GB" | 20260507 07:58:09.195 | INFO | } | 20260507 07:58:09.195 | INFO | Perf report for the last 5.1292 seconds: | 20260507 07:58:09.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:58:09.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:09.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:09.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:09.195 | INFO | ANALYSIS : 61.5%, 3.0, 96.0ms, 0 | 20260507 07:58:09.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:09.195 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260507 07:58:09.195 | INFO | PRE_HQ : 58.6%, 1.8, 54.4ms, 0 | 20260507 07:58:09.195 | INFO | BAZWRITER : 61.9%, 3.1, 99.1ms, 0 | 20260507 07:58:09.195 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 07:58:09.195 | INFO | Uploaded 36GiB to GPU in 2.03516s (17.689GiB/s) | 20260507 07:58:09.195 | INFO | Downloaded 38GiB to GPU in 2.15801s (17.6088GiB/s) | 20260507 07:58:14.337 | INFO | Analyzing chunk frames = [8348160,8348672) | 20260507 07:58:14.337 | INFO | MemInfo: { | 20260507 07:58:14.337 | INFO | "gpu" : " 17 GB", | 20260507 07:58:14.337 | INFO | "huge" : "", | 20260507 07:58:14.337 | INFO | "huge_shared" : " 27 GB", | 20260507 07:58:14.337 | INFO | "malloc" : " 14 GB", | 20260507 07:58:14.337 | INFO | "pinned" : " 29 GB" | 20260507 07:58:14.337 | INFO | } | 20260507 07:58:14.337 | INFO | Perf report for the last 5.14247 seconds: | 20260507 07:58:14.337 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:58:14.337 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:14.337 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:14.337 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:14.337 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 07:58:14.337 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:14.337 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:14.337 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 07:58:14.337 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:14.337 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:14.338 | INFO | Uploaded 36GiB to GPU in 1.99688s (18.0282GiB/s) | 20260507 07:58:14.338 | INFO | Downloaded 34GiB to GPU in 1.93816s (17.5424GiB/s) | 20260507 07:58:19.479 | INFO | Analyzing chunk frames = [8348672,8349184) | 20260507 07:58:19.479 | INFO | MemInfo: { | 20260507 07:58:19.479 | INFO | "gpu" : " 17 GB", | 20260507 07:58:19.479 | INFO | "huge" : "", | 20260507 07:58:19.479 | INFO | "huge_shared" : " 27 GB", | 20260507 07:58:19.479 | INFO | "malloc" : " 14 GB", | 20260507 07:58:19.479 | INFO | "pinned" : " 29 GB" | 20260507 07:58:19.479 | INFO | } | 20260507 07:58:19.479 | INFO | Perf report for the last 5.14142 seconds: | 20260507 07:58:19.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:58:19.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:19.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:19.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:19.479 | INFO | ANALYSIS : 60.2%, 3.0, 93.4ms, 0 | 20260507 07:58:19.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:19.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:19.479 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:58:19.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:19.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:19.479 | INFO | Uploaded 36GiB to GPU in 2.00915s (17.9181GiB/s) | 20260507 07:58:19.479 | INFO | Downloaded 34GiB to GPU in 1.95297s (17.4094GiB/s) | 20260507 07:58:24.624 | INFO | Analyzing chunk frames = [8349184,8349696) | 20260507 07:58:24.624 | INFO | MemInfo: { | 20260507 07:58:24.624 | INFO | "gpu" : " 17 GB", | 20260507 07:58:24.624 | INFO | "huge" : "", | 20260507 07:58:24.624 | INFO | "huge_shared" : " 27 GB", | 20260507 07:58:24.624 | INFO | "malloc" : " 14 GB", | 20260507 07:58:24.624 | INFO | "pinned" : " 29 GB" | 20260507 07:58:24.624 | INFO | } | 20260507 07:58:24.624 | INFO | Perf report for the last 5.14509 seconds: | 20260507 07:58:24.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:58:24.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:24.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:24.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:24.624 | INFO | ANALYSIS : 60.3%, 2.9, 93.2ms, 0 | 20260507 07:58:24.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:24.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:24.624 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:58:24.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:24.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:24.624 | INFO | Uploaded 36GiB to GPU in 2.03159s (17.7201GiB/s) | 20260507 07:58:24.624 | INFO | Downloaded 34GiB to GPU in 1.95695s (17.374GiB/s) | 20260507 07:58:29.761 | INFO | Analyzing chunk frames = [8349696,8350208) | 20260507 07:58:29.761 | INFO | MemInfo: { | 20260507 07:58:29.761 | INFO | "gpu" : " 17 GB", | 20260507 07:58:29.761 | INFO | "huge" : "", | 20260507 07:58:29.761 | INFO | "huge_shared" : " 27 GB", | 20260507 07:58:29.761 | INFO | "malloc" : " 14 GB", | 20260507 07:58:29.761 | INFO | "pinned" : " 29 GB" | 20260507 07:58:29.761 | INFO | } | 20260507 07:58:29.761 | INFO | Perf report for the last 5.13735 seconds: | 20260507 07:58:29.761 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:58:29.761 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:29.761 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:29.761 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:29.761 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 07:58:29.761 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:29.761 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:29.761 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:58:29.761 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:29.761 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:29.761 | INFO | Uploaded 36GiB to GPU in 2.04859s (17.5731GiB/s) | 20260507 07:58:29.762 | INFO | Downloaded 34GiB to GPU in 1.90472s (17.8504GiB/s) | 20260507 07:58:34.892 | INFO | Analyzing chunk frames = [8350208,8350720) | 20260507 07:58:34.892 | INFO | MemInfo: { | 20260507 07:58:34.892 | INFO | "gpu" : " 17 GB", | 20260507 07:58:34.892 | INFO | "huge" : "", | 20260507 07:58:34.892 | INFO | "huge_shared" : " 27 GB", | 20260507 07:58:34.892 | INFO | "malloc" : " 14 GB", | 20260507 07:58:34.892 | INFO | "pinned" : " 29 GB" | 20260507 07:58:34.892 | INFO | } | 20260507 07:58:34.893 | INFO | Perf report for the last 5.1313 seconds: | 20260507 07:58:34.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:58:34.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:34.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:34.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:34.893 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:58:34.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:34.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:34.893 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:58:34.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:34.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:34.893 | INFO | Uploaded 36GiB to GPU in 2.06644s (17.4213GiB/s) | 20260507 07:58:34.893 | INFO | Downloaded 34GiB to GPU in 1.94431s (17.4869GiB/s) | 20260507 07:58:40.035 | INFO | Analyzing chunk frames = [8350720,8351232) | 20260507 07:58:40.035 | INFO | MemInfo: { | 20260507 07:58:40.035 | INFO | "gpu" : " 17 GB", | 20260507 07:58:40.035 | INFO | "huge" : "", | 20260507 07:58:40.035 | INFO | "huge_shared" : " 27 GB", | 20260507 07:58:40.035 | INFO | "malloc" : " 14 GB", | 20260507 07:58:40.035 | INFO | "pinned" : " 29 GB" | 20260507 07:58:40.035 | INFO | } | 20260507 07:58:40.035 | INFO | Perf report for the last 5.14295 seconds: | 20260507 07:58:40.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:58:40.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:40.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:40.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:40.035 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 07:58:40.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:40.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:40.036 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:58:40.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:40.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:40.036 | INFO | Uploaded 36GiB to GPU in 1.9189s (18.7608GiB/s) | 20260507 07:58:40.036 | INFO | Downloaded 34GiB to GPU in 1.97224s (17.2393GiB/s) | 20260507 07:58:45.176 | INFO | Analyzing chunk frames = [8351232,8351744) | 20260507 07:58:45.176 | INFO | MemInfo: { | 20260507 07:58:45.176 | INFO | "gpu" : " 17 GB", | 20260507 07:58:45.176 | INFO | "huge" : "", | 20260507 07:58:45.176 | INFO | "huge_shared" : " 27 GB", | 20260507 07:58:45.176 | INFO | "malloc" : " 14 GB", | 20260507 07:58:45.176 | INFO | "pinned" : " 29 GB" | 20260507 07:58:45.176 | INFO | } | 20260507 07:58:45.176 | INFO | Perf report for the last 5.14053 seconds: | 20260507 07:58:45.176 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:58:45.176 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:45.176 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:45.176 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:45.176 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 07:58:45.176 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:45.176 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:45.176 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 07:58:45.176 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:45.176 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:45.176 | INFO | Uploaded 36GiB to GPU in 1.97353s (18.2414GiB/s) | 20260507 07:58:45.176 | INFO | Downloaded 34GiB to GPU in 1.96742s (17.2815GiB/s) | 20260507 07:58:45.275 | INFO | Starting a round of Baz IO | 20260507 07:58:48.398 | INFO | Populating RTMetrics report | 20260507 07:58:48.398 | INFO | RTMetrics report populated | 20260507 07:58:48.398 | INFO | Opening tmp RTMetrics json file | 20260507 07:58:48.399 | INFO | Serializing RTMetrics to json | 20260507 07:58:48.400 | INFO | Serialization done, swapping tmp file now... | 20260507 07:58:48.401 | INFO | RTMetrics done | 20260507 07:58:48.479 | INFO | Finished a round of Baz IO: Wrote 1.627 GiB in 3.204 seconds | 20260507 07:58:48.479 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.52% / 0.00% / 67.47% | 20260507 07:58:48.479 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0971834% / 0% / 7.15256e-05% | 20260507 07:58:48.479 | INFO | Baz IO overall: | 20260507 07:58:48.479 | INFO | BytesWritten : 4172377762737 | 20260507 07:58:48.479 | INFO | NumEvents : 1996657553260 | 20260507 07:58:50.313 | INFO | Analyzing chunk frames = [8351744,8352256) | 20260507 07:58:50.313 | INFO | MemInfo: { | 20260507 07:58:50.313 | INFO | "gpu" : " 17 GB", | 20260507 07:58:50.313 | INFO | "huge" : "", | 20260507 07:58:50.313 | INFO | "huge_shared" : " 27 GB", | 20260507 07:58:50.313 | INFO | "malloc" : " 14 GB", | 20260507 07:58:50.313 | INFO | "pinned" : " 29 GB" | 20260507 07:58:50.313 | INFO | } | 20260507 07:58:50.313 | INFO | Perf report for the last 5.13701 seconds: | 20260507 07:58:50.313 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:58:50.313 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:50.313 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:50.313 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:50.313 | INFO | ANALYSIS : 61.6%, 3.0, 95.6ms, 0 | 20260507 07:58:50.313 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:50.313 | INFO | RT_METRICS : 44.3%, 1.0, 23.2ms, 0 | 20260507 07:58:50.313 | INFO | PRE_HQ : 60.0%, 1.9, 59.3ms, 0 | 20260507 07:58:50.313 | INFO | BAZWRITER : 62.0%, 3.3, 104.7ms, 0 | 20260507 07:58:50.313 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.3ms, 0 | 20260507 07:58:50.313 | INFO | Uploaded 36GiB to GPU in 2.056s (17.5097GiB/s) | 20260507 07:58:50.313 | INFO | Downloaded 38GiB to GPU in 2.18285s (17.4085GiB/s) | 20260507 07:58:55.453 | INFO | Analyzing chunk frames = [8352256,8352768) | 20260507 07:58:55.453 | INFO | MemInfo: { | 20260507 07:58:55.453 | INFO | "gpu" : " 17 GB", | 20260507 07:58:55.453 | INFO | "huge" : "", | 20260507 07:58:55.453 | INFO | "huge_shared" : " 27 GB", | 20260507 07:58:55.453 | INFO | "malloc" : " 14 GB", | 20260507 07:58:55.453 | INFO | "pinned" : " 29 GB" | 20260507 07:58:55.453 | INFO | } | 20260507 07:58:55.453 | INFO | Perf report for the last 5.13976 seconds: | 20260507 07:58:55.453 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:58:55.453 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:55.453 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:55.453 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:55.453 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 07:58:55.453 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:55.453 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:55.453 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 07:58:55.453 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:55.453 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:58:55.453 | INFO | Uploaded 36GiB to GPU in 1.99489s (18.0461GiB/s) | 20260507 07:58:55.453 | INFO | Downloaded 34GiB to GPU in 1.93523s (17.569GiB/s) | 20260507 07:59:00.592 | INFO | Analyzing chunk frames = [8352768,8353280) | 20260507 07:59:00.592 | INFO | MemInfo: { | 20260507 07:59:00.592 | INFO | "gpu" : " 17 GB", | 20260507 07:59:00.592 | INFO | "huge" : "", | 20260507 07:59:00.592 | INFO | "huge_shared" : " 27 GB", | 20260507 07:59:00.592 | INFO | "malloc" : " 14 GB", | 20260507 07:59:00.592 | INFO | "pinned" : " 29 GB" | 20260507 07:59:00.592 | INFO | } | 20260507 07:59:00.592 | INFO | Perf report for the last 5.13933 seconds: | 20260507 07:59:00.592 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:59:00.592 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:00.592 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:00.592 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:00.592 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 07:59:00.592 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:00.592 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:00.592 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:59:00.592 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:00.592 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:00.592 | INFO | Uploaded 36GiB to GPU in 2.04659s (17.5903GiB/s) | 20260507 07:59:00.592 | INFO | Downloaded 34GiB to GPU in 1.95351s (17.4045GiB/s) | 20260507 07:59:05.734 | INFO | Analyzing chunk frames = [8353280,8353792) | 20260507 07:59:05.734 | INFO | MemInfo: { | 20260507 07:59:05.734 | INFO | "gpu" : " 17 GB", | 20260507 07:59:05.734 | INFO | "huge" : "", | 20260507 07:59:05.734 | INFO | "huge_shared" : " 27 GB", | 20260507 07:59:05.734 | INFO | "malloc" : " 14 GB", | 20260507 07:59:05.734 | INFO | "pinned" : " 29 GB" | 20260507 07:59:05.734 | INFO | } | 20260507 07:59:05.734 | INFO | Perf report for the last 5.14172 seconds: | 20260507 07:59:05.734 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:59:05.734 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:05.734 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:05.734 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:05.734 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 07:59:05.734 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:05.734 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:05.734 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:59:05.734 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:05.734 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:05.734 | INFO | Uploaded 36GiB to GPU in 2.02263s (17.7986GiB/s) | 20260507 07:59:05.734 | INFO | Downloaded 34GiB to GPU in 1.9381s (17.543GiB/s) | 20260507 07:59:10.873 | INFO | Analyzing chunk frames = [8353792,8354304) | 20260507 07:59:10.873 | INFO | MemInfo: { | 20260507 07:59:10.873 | INFO | "gpu" : " 17 GB", | 20260507 07:59:10.873 | INFO | "huge" : "", | 20260507 07:59:10.873 | INFO | "huge_shared" : " 27 GB", | 20260507 07:59:10.873 | INFO | "malloc" : " 14 GB", | 20260507 07:59:10.873 | INFO | "pinned" : " 29 GB" | 20260507 07:59:10.873 | INFO | } | 20260507 07:59:10.873 | INFO | Perf report for the last 5.1393 seconds: | 20260507 07:59:10.873 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:59:10.873 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:10.873 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:10.873 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:10.873 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 07:59:10.873 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:10.873 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:10.873 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 07:59:10.873 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:10.873 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:10.873 | INFO | Uploaded 36GiB to GPU in 2.04191s (17.6306GiB/s) | 20260507 07:59:10.873 | INFO | Downloaded 34GiB to GPU in 1.91168s (17.7854GiB/s) | 20260507 07:59:16.011 | INFO | Analyzing chunk frames = [8354304,8354816) | 20260507 07:59:16.011 | INFO | MemInfo: { | 20260507 07:59:16.011 | INFO | "gpu" : " 17 GB", | 20260507 07:59:16.011 | INFO | "huge" : "", | 20260507 07:59:16.011 | INFO | "huge_shared" : " 27 GB", | 20260507 07:59:16.011 | INFO | "malloc" : " 14 GB", | 20260507 07:59:16.011 | INFO | "pinned" : " 29 GB" | 20260507 07:59:16.011 | INFO | } | 20260507 07:59:16.011 | INFO | Perf report for the last 5.13793 seconds: | 20260507 07:59:16.011 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:59:16.011 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:16.011 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:16.011 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:16.011 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 07:59:16.011 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:16.011 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:16.011 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:59:16.011 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:16.011 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:16.011 | INFO | Uploaded 36GiB to GPU in 2.06618s (17.4235GiB/s) | 20260507 07:59:16.011 | INFO | Downloaded 34GiB to GPU in 1.94076s (17.5189GiB/s) | 20260507 07:59:21.146 | INFO | Analyzing chunk frames = [8354816,8355328) | 20260507 07:59:21.146 | INFO | MemInfo: { | 20260507 07:59:21.146 | INFO | "gpu" : " 17 GB", | 20260507 07:59:21.146 | INFO | "huge" : "", | 20260507 07:59:21.146 | INFO | "huge_shared" : " 27 GB", | 20260507 07:59:21.146 | INFO | "malloc" : " 14 GB", | 20260507 07:59:21.146 | INFO | "pinned" : " 29 GB" | 20260507 07:59:21.146 | INFO | } | 20260507 07:59:21.146 | INFO | Perf report for the last 5.13473 seconds: | 20260507 07:59:21.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:59:21.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:21.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:21.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:21.146 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 07:59:21.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:21.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:21.146 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 07:59:21.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:21.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:21.146 | INFO | Uploaded 36GiB to GPU in 1.89698s (18.9775GiB/s) | 20260507 07:59:21.146 | INFO | Downloaded 34GiB to GPU in 1.97709s (17.197GiB/s) | 20260507 07:59:26.292 | INFO | Analyzing chunk frames = [8355328,8355840) | 20260507 07:59:26.292 | INFO | MemInfo: { | 20260507 07:59:26.292 | INFO | "gpu" : " 17 GB", | 20260507 07:59:26.292 | INFO | "huge" : "", | 20260507 07:59:26.292 | INFO | "huge_shared" : " 27 GB", | 20260507 07:59:26.292 | INFO | "malloc" : " 14 GB", | 20260507 07:59:26.293 | INFO | "pinned" : " 29 GB" | 20260507 07:59:26.293 | INFO | } | 20260507 07:59:26.293 | INFO | Perf report for the last 5.14673 seconds: | 20260507 07:59:26.293 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:59:26.293 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:26.293 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:26.293 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:26.293 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 07:59:26.293 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:26.293 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:26.293 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 07:59:26.293 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:26.293 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:26.293 | INFO | Uploaded 36GiB to GPU in 1.96003s (18.3671GiB/s) | 20260507 07:59:26.293 | INFO | Downloaded 34GiB to GPU in 1.95784s (17.3661GiB/s) | 20260507 07:59:26.293 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 07:59:26.293 | INFO | Total mem usage: 18 GiB | 20260507 07:59:26.293 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 07:59:26.293 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 07:59:26.293 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 07:59:26.293 | INFO | High Water Breakdown: | 20260507 07:59:26.293 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 07:59:26.293 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 07:59:26.293 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 07:59:26.293 | INFO | ------------------------------------------ | 20260507 07:59:26.293 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:59:26.293 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 07:59:26.293 | INFO | Total mem usage: 26.5 GiB | 20260507 07:59:26.293 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 07:59:26.293 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 07:59:26.293 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 07:59:26.293 | INFO | High Water Breakdown: | 20260507 07:59:26.293 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:59:26.293 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:59:26.293 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:59:26.293 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:59:26.293 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:59:26.293 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:59:26.293 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 07:59:26.293 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 07:59:26.293 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:59:26.293 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 07:59:26.293 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 07:59:26.294 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 07:59:26.294 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 07:59:26.294 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 07:59:26.294 | INFO | ------------------------------------------ | 20260507 07:59:26.294 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:59:26.294 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 07:59:26.294 | INFO | Total mem usage: 25.6 GiB | 20260507 07:59:26.294 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 07:59:26.294 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 07:59:26.294 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 07:59:26.294 | INFO | High Water Breakdown: | 20260507 07:59:26.294 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 07:59:26.294 | INFO | ------------------------------------------ | 20260507 07:59:26.294 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:59:26.294 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 07:59:26.294 | INFO | Total mem usage: 15.7 GiB | 20260507 07:59:26.294 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 07:59:26.294 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 07:59:26.294 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 07:59:26.294 | INFO | High Water Breakdown: | 20260507 07:59:26.294 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 07:59:26.294 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 07:59:26.294 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 07:59:26.294 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 07:59:26.294 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 07:59:26.294 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 07:59:26.294 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 07:59:26.294 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 07:59:26.294 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 07:59:26.294 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 07:59:26.294 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 07:59:26.294 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 07:59:26.294 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 07:59:26.294 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 07:59:26.294 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 07:59:26.294 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 07:59:26.294 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 07:59:26.294 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 07:59:26.294 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 07:59:26.294 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 07:59:26.294 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 07:59:26.294 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 07:59:26.294 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 07:59:26.294 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 07:59:26.294 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 07:59:26.294 | INFO | ------------------------------------------ | 20260507 07:59:26.294 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 07:59:26.294 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 07:59:26.294 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:59:26.294 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:59:26.294 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.35e+01 / 1.52e+00 | 20260507 07:59:26.294 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.03e-01, 9.71e-03 / 5.73e-03 | 20260507 07:59:26.294 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.46e+00, 5.06e-01 / 2.15e-01 | 20260507 07:59:26.294 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.49e+00, 1.69e-01 / 8.87e-02 | 20260507 07:59:26.294 | INFO | PerfReport: 16.33%, DME -- 4.01e+01 / 4.52e+01, 4.10e+00 / 1.69e+00 | 20260507 07:59:26.294 | INFO | PerfReport: 57.66%, FrameLabeling -- 1.77e+01 / 1.93e+01, 1.27e+00 / 1.23e+00 | 20260507 07:59:26.294 | INFO | PerfReport: 6.89%, PulseAccumulating -- 2.12e+00 / 2.69e+00, 3.84e-01 / 1.85e-01 | 20260507 07:59:26.294 | INFO | PerfReport: 2.92%, Metrics -- 8.95e-01 / 1.20e+00, 2.40e-01 / 1.21e-01 | 20260507 07:59:26.294 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.55e-01, 5.02e-02 / 1.55e-02 | 20260507 07:59:26.294 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 07:59:26.294 | INFO | PerfReport: Gathered 1536 samples. | 20260507 07:59:26.294 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 07:59:26.294 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 07:59:26.294 | INFO | PerfReport: 21.80%, Upload -- 2.09e+01 / 0.00e+00, 7.03e+00 / 0.00e+00 | 20260507 07:59:26.294 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 07:59:26.294 | INFO | PerfReport: 31.96%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 07:59:26.294 | INFO | PerfReport: 1.43%, LockUpload -- 1.38e+00 / 0.00e+00, 4.81e+00 / 0.00e+00 | 20260507 07:59:26.294 | INFO | PerfReport: 1.39%, LockDownload -- 1.34e+00 / 0.00e+00, 3.20e+00 / 0.00e+00 | 20260507 07:59:26.294 | INFO | PerfReport: 21.99%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 07:59:26.294 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260507 07:59:26.391 | INFO | Starting a round of Baz IO | 20260507 07:59:29.494 | INFO | Populating RTMetrics report | 20260507 07:59:29.494 | INFO | RTMetrics report populated | 20260507 07:59:29.494 | INFO | Opening tmp RTMetrics json file | 20260507 07:59:29.495 | INFO | Serializing RTMetrics to json | 20260507 07:59:29.497 | INFO | Serialization done, swapping tmp file now... | 20260507 07:59:29.497 | INFO | RTMetrics done | 20260507 07:59:29.598 | INFO | Finished a round of Baz IO: Wrote 1.625 GiB in 3.207 seconds | 20260507 07:59:29.598 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.52% / 0.00% / 67.48% | 20260507 07:59:29.598 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0970483% / 0% / 7.15256e-05% | 20260507 07:59:29.598 | INFO | Baz IO overall: | 20260507 07:59:29.598 | INFO | BytesWritten : 4174123058570 | 20260507 07:59:29.598 | INFO | NumEvents : 1997471772432 | 20260507 07:59:31.435 | INFO | Analyzing chunk frames = [8355840,8356352) | 20260507 07:59:31.435 | INFO | MemInfo: { | 20260507 07:59:31.435 | INFO | "gpu" : " 17 GB", | 20260507 07:59:31.435 | INFO | "huge" : "", | 20260507 07:59:31.435 | INFO | "huge_shared" : " 27 GB", | 20260507 07:59:31.435 | INFO | "malloc" : " 14 GB", | 20260507 07:59:31.435 | INFO | "pinned" : " 29 GB" | 20260507 07:59:31.435 | INFO | } | 20260507 07:59:31.435 | INFO | Perf report for the last 5.14242 seconds: | 20260507 07:59:31.435 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:59:31.435 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:31.435 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:31.435 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:31.435 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260507 07:59:31.435 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:31.435 | INFO | RT_METRICS : 45.1%, 1.0, 23.7ms, 0 | 20260507 07:59:31.435 | INFO | PRE_HQ : 60.3%, 1.9, 60.6ms, 0 | 20260507 07:59:31.435 | INFO | BAZWRITER : 62.0%, 3.3, 106.7ms, 0 | 20260507 07:59:31.435 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 07:59:31.435 | INFO | Uploaded 36GiB to GPU in 2.06747s (17.4126GiB/s) | 20260507 07:59:31.436 | INFO | Downloaded 38GiB to GPU in 2.17941s (17.4359GiB/s) | 20260507 07:59:36.574 | INFO | Analyzing chunk frames = [8356352,8356864) | 20260507 07:59:36.574 | INFO | MemInfo: { | 20260507 07:59:36.574 | INFO | "gpu" : " 17 GB", | 20260507 07:59:36.574 | INFO | "huge" : "", | 20260507 07:59:36.574 | INFO | "huge_shared" : " 27 GB", | 20260507 07:59:36.574 | INFO | "malloc" : " 14 GB", | 20260507 07:59:36.574 | INFO | "pinned" : " 29 GB" | 20260507 07:59:36.574 | INFO | } | 20260507 07:59:36.574 | INFO | Perf report for the last 5.13868 seconds: | 20260507 07:59:36.574 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:59:36.574 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:36.574 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:36.574 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:36.574 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 07:59:36.574 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:36.574 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:36.574 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 07:59:36.574 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:36.574 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:36.574 | INFO | Uploaded 36GiB to GPU in 1.98706s (18.1173GiB/s) | 20260507 07:59:36.574 | INFO | Downloaded 34GiB to GPU in 1.93947s (17.5306GiB/s) | 20260507 07:59:41.707 | INFO | Analyzing chunk frames = [8356864,8357376) | 20260507 07:59:41.707 | INFO | MemInfo: { | 20260507 07:59:41.707 | INFO | "gpu" : " 17 GB", | 20260507 07:59:41.707 | INFO | "huge" : "", | 20260507 07:59:41.707 | INFO | "huge_shared" : " 27 GB", | 20260507 07:59:41.707 | INFO | "malloc" : " 14 GB", | 20260507 07:59:41.707 | INFO | "pinned" : " 29 GB" | 20260507 07:59:41.708 | INFO | } | 20260507 07:59:41.708 | INFO | Perf report for the last 5.13389 seconds: | 20260507 07:59:41.708 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:59:41.708 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:41.708 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:41.708 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:41.708 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 07:59:41.708 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:41.708 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:41.708 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 07:59:41.708 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:41.708 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:41.708 | INFO | Uploaded 36GiB to GPU in 2.00939s (17.9159GiB/s) | 20260507 07:59:41.708 | INFO | Downloaded 34GiB to GPU in 1.92082s (17.7007GiB/s) | 20260507 07:59:46.842 | INFO | Analyzing chunk frames = [8357376,8357888) | 20260507 07:59:46.842 | INFO | MemInfo: { | 20260507 07:59:46.842 | INFO | "gpu" : " 17 GB", | 20260507 07:59:46.842 | INFO | "huge" : "", | 20260507 07:59:46.842 | INFO | "huge_shared" : " 27 GB", | 20260507 07:59:46.842 | INFO | "malloc" : " 14 GB", | 20260507 07:59:46.842 | INFO | "pinned" : " 29 GB" | 20260507 07:59:46.842 | INFO | } | 20260507 07:59:46.842 | INFO | Perf report for the last 5.13444 seconds: | 20260507 07:59:46.842 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:59:46.842 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:46.842 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:46.842 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:46.842 | INFO | ANALYSIS : 60.3%, 2.9, 93.1ms, 0 | 20260507 07:59:46.842 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:46.842 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:46.842 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 07:59:46.842 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:46.842 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:46.842 | INFO | Uploaded 36GiB to GPU in 2.02033s (17.8189GiB/s) | 20260507 07:59:46.842 | INFO | Downloaded 34GiB to GPU in 1.95345s (17.4051GiB/s) | 20260507 07:59:51.986 | INFO | Analyzing chunk frames = [8357888,8358400) | 20260507 07:59:51.986 | INFO | MemInfo: { | 20260507 07:59:51.986 | INFO | "gpu" : " 17 GB", | 20260507 07:59:51.986 | INFO | "huge" : "", | 20260507 07:59:51.986 | INFO | "huge_shared" : " 27 GB", | 20260507 07:59:51.986 | INFO | "malloc" : " 14 GB", | 20260507 07:59:51.987 | INFO | "pinned" : " 29 GB" | 20260507 07:59:51.987 | INFO | } | 20260507 07:59:51.987 | INFO | Perf report for the last 5.14454 seconds: | 20260507 07:59:51.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:59:51.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:51.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:51.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:51.987 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 07:59:51.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:51.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:51.987 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 07:59:51.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:51.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:51.987 | INFO | Uploaded 36GiB to GPU in 2.05009s (17.5602GiB/s) | 20260507 07:59:51.987 | INFO | Downloaded 34GiB to GPU in 1.95116s (17.4255GiB/s) | 20260507 07:59:57.135 | INFO | Analyzing chunk frames = [8358400,8358912) | 20260507 07:59:57.135 | INFO | MemInfo: { | 20260507 07:59:57.135 | INFO | "gpu" : " 17 GB", | 20260507 07:59:57.135 | INFO | "huge" : "", | 20260507 07:59:57.135 | INFO | "huge_shared" : " 27 GB", | 20260507 07:59:57.135 | INFO | "malloc" : " 14 GB", | 20260507 07:59:57.135 | INFO | "pinned" : " 29 GB" | 20260507 07:59:57.135 | INFO | } | 20260507 07:59:57.135 | INFO | Perf report for the last 5.1482 seconds: | 20260507 07:59:57.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 07:59:57.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:57.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:57.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:57.135 | INFO | ANALYSIS : 60.3%, 3.0, 94.3ms, 0 | 20260507 07:59:57.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:57.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:57.135 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 07:59:57.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:57.135 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 07:59:57.135 | INFO | Uploaded 36GiB to GPU in 2.0638s (17.4435GiB/s) | 20260507 07:59:57.135 | INFO | Downloaded 34GiB to GPU in 1.92466s (17.6654GiB/s) | 20260507 08:00:02.269 | INFO | Analyzing chunk frames = [8358912,8359424) | 20260507 08:00:02.269 | INFO | MemInfo: { | 20260507 08:00:02.269 | INFO | "gpu" : " 17 GB", | 20260507 08:00:02.269 | INFO | "huge" : "", | 20260507 08:00:02.269 | INFO | "huge_shared" : " 27 GB", | 20260507 08:00:02.269 | INFO | "malloc" : " 14 GB", | 20260507 08:00:02.269 | INFO | "pinned" : " 29 GB" | 20260507 08:00:02.269 | INFO | } | 20260507 08:00:02.269 | INFO | Perf report for the last 5.13423 seconds: | 20260507 08:00:02.269 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:00:02.269 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:02.269 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:02.269 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:02.269 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 08:00:02.269 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:02.269 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:02.269 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 08:00:02.269 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:02.269 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:02.270 | INFO | Uploaded 36GiB to GPU in 1.95803s (18.3859GiB/s) | 20260507 08:00:02.270 | INFO | Downloaded 34GiB to GPU in 1.97124s (17.248GiB/s) | 20260507 08:00:07.401 | INFO | Analyzing chunk frames = [8359424,8359936) | 20260507 08:00:07.401 | INFO | MemInfo: { | 20260507 08:00:07.401 | INFO | "gpu" : " 17 GB", | 20260507 08:00:07.401 | INFO | "huge" : "", | 20260507 08:00:07.401 | INFO | "huge_shared" : " 27 GB", | 20260507 08:00:07.401 | INFO | "malloc" : " 14 GB", | 20260507 08:00:07.401 | INFO | "pinned" : " 29 GB" | 20260507 08:00:07.401 | INFO | } | 20260507 08:00:07.401 | INFO | Perf report for the last 5.13192 seconds: | 20260507 08:00:07.401 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:00:07.401 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:07.401 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:07.401 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:07.401 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 08:00:07.401 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:07.401 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:07.401 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 08:00:07.401 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:07.401 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:07.401 | INFO | Uploaded 36GiB to GPU in 1.99092s (18.0821GiB/s) | 20260507 08:00:07.401 | INFO | Downloaded 34GiB to GPU in 1.95644s (17.3785GiB/s) | 20260507 08:00:07.498 | INFO | Starting a round of Baz IO | 20260507 08:00:10.592 | INFO | Populating RTMetrics report | 20260507 08:00:10.592 | INFO | RTMetrics report populated | 20260507 08:00:10.592 | INFO | Opening tmp RTMetrics json file | 20260507 08:00:10.598 | INFO | Serializing RTMetrics to json | 20260507 08:00:10.615 | INFO | Serialization done, swapping tmp file now... | 20260507 08:00:10.615 | INFO | RTMetrics done | 20260507 08:00:10.684 | INFO | Finished a round of Baz IO: Wrote 1.626 GiB in 3.186 seconds | 20260507 08:00:10.684 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.52% / 0.00% / 67.48% | 20260507 08:00:10.684 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0966867% / 0% / 8.74201e-05% | 20260507 08:00:10.684 | INFO | Baz IO overall: | 20260507 08:00:10.684 | INFO | BytesWritten : 4175868443078 | 20260507 08:00:10.684 | INFO | NumEvents : 1998286176673 | 20260507 08:00:12.542 | INFO | Analyzing chunk frames = [8359936,8360448) | 20260507 08:00:12.542 | INFO | MemInfo: { | 20260507 08:00:12.542 | INFO | "gpu" : " 17 GB", | 20260507 08:00:12.542 | INFO | "huge" : "", | 20260507 08:00:12.542 | INFO | "huge_shared" : " 27 GB", | 20260507 08:00:12.542 | INFO | "malloc" : " 14 GB", | 20260507 08:00:12.542 | INFO | "pinned" : " 29 GB" | 20260507 08:00:12.542 | INFO | } | 20260507 08:00:12.542 | INFO | Perf report for the last 5.14113 seconds: | 20260507 08:00:12.542 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:00:12.542 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:12.542 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:12.542 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:12.542 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260507 08:00:12.542 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:12.542 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 08:00:12.542 | INFO | PRE_HQ : 59.6%, 1.9, 58.5ms, 0 | 20260507 08:00:12.542 | INFO | BAZWRITER : 61.5%, 3.3, 105.4ms, 0 | 20260507 08:00:12.542 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:00:12.542 | INFO | Uploaded 36GiB to GPU in 2.05863s (17.4874GiB/s) | 20260507 08:00:12.543 | INFO | Downloaded 38GiB to GPU in 2.17491s (17.472GiB/s) | 20260507 08:00:17.684 | INFO | Analyzing chunk frames = [8360448,8360960) | 20260507 08:00:17.684 | INFO | MemInfo: { | 20260507 08:00:17.684 | INFO | "gpu" : " 17 GB", | 20260507 08:00:17.684 | INFO | "huge" : "", | 20260507 08:00:17.684 | INFO | "huge_shared" : " 27 GB", | 20260507 08:00:17.684 | INFO | "malloc" : " 14 GB", | 20260507 08:00:17.684 | INFO | "pinned" : " 29 GB" | 20260507 08:00:17.684 | INFO | } | 20260507 08:00:17.684 | INFO | Perf report for the last 5.14184 seconds: | 20260507 08:00:17.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:00:17.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:17.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:17.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:17.684 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 08:00:17.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:17.684 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:17.684 | INFO | PRE_HQ : 25.8%, 1.0, 13.4ms, 0 | 20260507 08:00:17.684 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:17.684 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:17.684 | INFO | Uploaded 36GiB to GPU in 1.99903s (18.0088GiB/s) | 20260507 08:00:17.684 | INFO | Downloaded 34GiB to GPU in 1.93536s (17.5678GiB/s) | 20260507 08:00:22.821 | INFO | Analyzing chunk frames = [8360960,8361472) | 20260507 08:00:22.821 | INFO | MemInfo: { | 20260507 08:00:22.821 | INFO | "gpu" : " 17 GB", | 20260507 08:00:22.821 | INFO | "huge" : "", | 20260507 08:00:22.821 | INFO | "huge_shared" : " 27 GB", | 20260507 08:00:22.821 | INFO | "malloc" : " 14 GB", | 20260507 08:00:22.821 | INFO | "pinned" : " 29 GB" | 20260507 08:00:22.821 | INFO | } | 20260507 08:00:22.821 | INFO | Perf report for the last 5.13673 seconds: | 20260507 08:00:22.821 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:00:22.821 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:22.821 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:22.821 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:22.821 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 08:00:22.821 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:22.821 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:22.821 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:00:22.821 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:22.821 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:22.821 | INFO | Uploaded 36GiB to GPU in 2.05162s (17.5471GiB/s) | 20260507 08:00:22.821 | INFO | Downloaded 34GiB to GPU in 1.95092s (17.4277GiB/s) | 20260507 08:00:27.962 | INFO | Analyzing chunk frames = [8361472,8361984) | 20260507 08:00:27.962 | INFO | MemInfo: { | 20260507 08:00:27.962 | INFO | "gpu" : " 17 GB", | 20260507 08:00:27.962 | INFO | "huge" : "", | 20260507 08:00:27.962 | INFO | "huge_shared" : " 27 GB", | 20260507 08:00:27.962 | INFO | "malloc" : " 14 GB", | 20260507 08:00:27.962 | INFO | "pinned" : " 29 GB" | 20260507 08:00:27.962 | INFO | } | 20260507 08:00:27.962 | INFO | Perf report for the last 5.14141 seconds: | 20260507 08:00:27.962 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:00:27.962 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:27.962 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:27.962 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:27.962 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 08:00:27.962 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:27.962 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:27.962 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:00:27.962 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:27.962 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:27.963 | INFO | Uploaded 36GiB to GPU in 2.05102s (17.5523GiB/s) | 20260507 08:00:27.963 | INFO | Downloaded 34GiB to GPU in 1.94505s (17.4802GiB/s) | 20260507 08:00:33.098 | INFO | Analyzing chunk frames = [8361984,8362496) | 20260507 08:00:33.098 | INFO | MemInfo: { | 20260507 08:00:33.098 | INFO | "gpu" : " 17 GB", | 20260507 08:00:33.098 | INFO | "huge" : "", | 20260507 08:00:33.098 | INFO | "huge_shared" : " 27 GB", | 20260507 08:00:33.099 | INFO | "malloc" : " 14 GB", | 20260507 08:00:33.099 | INFO | "pinned" : " 29 GB" | 20260507 08:00:33.099 | INFO | } | 20260507 08:00:33.099 | INFO | Perf report for the last 5.13642 seconds: | 20260507 08:00:33.099 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:00:33.099 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:33.099 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:33.099 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:33.099 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 08:00:33.099 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:33.099 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:33.099 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:00:33.099 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:33.099 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:33.099 | INFO | Uploaded 36GiB to GPU in 2.0597s (17.4782GiB/s) | 20260507 08:00:33.099 | INFO | Downloaded 34GiB to GPU in 1.90799s (17.8198GiB/s) | 20260507 08:00:38.243 | INFO | Analyzing chunk frames = [8362496,8363008) | 20260507 08:00:38.243 | INFO | MemInfo: { | 20260507 08:00:38.243 | INFO | "gpu" : " 17 GB", | 20260507 08:00:38.243 | INFO | "huge" : "", | 20260507 08:00:38.243 | INFO | "huge_shared" : " 27 GB", | 20260507 08:00:38.243 | INFO | "malloc" : " 14 GB", | 20260507 08:00:38.243 | INFO | "pinned" : " 29 GB" | 20260507 08:00:38.243 | INFO | } | 20260507 08:00:38.243 | INFO | Perf report for the last 5.14438 seconds: | 20260507 08:00:38.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:00:38.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:38.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:38.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:38.243 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 08:00:38.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:38.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:38.243 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:00:38.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:38.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:38.243 | INFO | Uploaded 36GiB to GPU in 2.04373s (17.6148GiB/s) | 20260507 08:00:38.243 | INFO | Downloaded 34GiB to GPU in 1.90703s (17.8287GiB/s) | 20260507 08:00:43.390 | INFO | Analyzing chunk frames = [8363008,8363520) | 20260507 08:00:43.390 | INFO | MemInfo: { | 20260507 08:00:43.390 | INFO | "gpu" : " 17 GB", | 20260507 08:00:43.390 | INFO | "huge" : "", | 20260507 08:00:43.390 | INFO | "huge_shared" : " 27 GB", | 20260507 08:00:43.390 | INFO | "malloc" : " 14 GB", | 20260507 08:00:43.390 | INFO | "pinned" : " 29 GB" | 20260507 08:00:43.390 | INFO | } | 20260507 08:00:43.390 | INFO | Perf report for the last 5.14691 seconds: | 20260507 08:00:43.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:00:43.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:43.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:43.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:43.390 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 08:00:43.390 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:43.390 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:43.390 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 08:00:43.390 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:43.390 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:43.390 | INFO | Uploaded 36GiB to GPU in 1.95419s (18.422GiB/s) | 20260507 08:00:43.390 | INFO | Downloaded 34GiB to GPU in 1.98659s (17.1147GiB/s) | 20260507 08:00:48.520 | INFO | Analyzing chunk frames = [8363520,8364032) | 20260507 08:00:48.520 | INFO | MemInfo: { | 20260507 08:00:48.520 | INFO | "gpu" : " 17 GB", | 20260507 08:00:48.520 | INFO | "huge" : "", | 20260507 08:00:48.520 | INFO | "huge_shared" : " 27 GB", | 20260507 08:00:48.520 | INFO | "malloc" : " 14 GB", | 20260507 08:00:48.520 | INFO | "pinned" : " 29 GB" | 20260507 08:00:48.520 | INFO | } | 20260507 08:00:48.520 | INFO | Perf report for the last 5.13018 seconds: | 20260507 08:00:48.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:00:48.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:48.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:48.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:48.520 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 08:00:48.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:48.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:48.520 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 08:00:48.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:48.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:48.520 | INFO | Uploaded 36GiB to GPU in 1.93761s (18.5796GiB/s) | 20260507 08:00:48.520 | INFO | Downloaded 34GiB to GPU in 1.9548s (17.3931GiB/s) | 20260507 08:00:48.521 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:00:48.521 | INFO | Total mem usage: 18 GiB | 20260507 08:00:48.521 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:00:48.521 | INFO | Recent High Water Marks Sum: 14.3 GiB (Savings: -25.8%) | 20260507 08:00:48.521 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:00:48.521 | INFO | High Water Breakdown: | 20260507 08:00:48.521 | INFO | 60.2% : 8.61 GiB ( 75.5% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:00:48.521 | INFO | 39.7% : 5.68 GiB ( 84.8% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:00:48.521 | INFO | 0.139% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:00:48.521 | INFO | ------------------------------------------ | 20260507 08:00:48.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:00:48.521 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:00:48.521 | INFO | Total mem usage: 26.5 GiB | 20260507 08:00:48.521 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:00:48.521 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:00:48.521 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:00:48.521 | INFO | High Water Breakdown: | 20260507 08:00:48.521 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:00:48.521 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:00:48.521 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:00:48.521 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:00:48.521 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:00:48.521 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:00:48.521 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:00:48.521 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:00:48.521 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:00:48.521 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:00:48.521 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:00:48.521 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:00:48.521 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:00:48.521 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:00:48.521 | INFO | ------------------------------------------ | 20260507 08:00:48.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:00:48.521 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:00:48.521 | INFO | Total mem usage: 25.6 GiB | 20260507 08:00:48.521 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:00:48.521 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:00:48.521 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:00:48.521 | INFO | High Water Breakdown: | 20260507 08:00:48.521 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:00:48.521 | INFO | ------------------------------------------ | 20260507 08:00:48.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:00:48.521 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:00:48.521 | INFO | Total mem usage: 15.7 GiB | 20260507 08:00:48.521 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:00:48.521 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:00:48.521 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:00:48.521 | INFO | High Water Breakdown: | 20260507 08:00:48.521 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:00:48.521 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:00:48.521 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:00:48.521 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:00:48.521 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:00:48.521 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:00:48.521 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:00:48.521 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:00:48.521 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:00:48.521 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:00:48.521 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:00:48.521 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:00:48.521 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:00:48.521 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:00:48.521 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:00:48.521 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:00:48.521 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:00:48.521 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:00:48.521 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:00:48.521 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:00:48.521 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:00:48.521 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:00:48.521 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:00:48.521 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:00:48.521 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:00:48.521 | INFO | ------------------------------------------ | 20260507 08:00:48.521 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:00:48.522 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:00:48.522 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:00:48.522 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:00:48.522 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.35e+01 / 1.31e+00 | 20260507 08:00:48.522 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.02e-01, 1.03e-02 / 4.54e-03 | 20260507 08:00:48.522 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.43e+00, 5.00e-01 / 3.16e-01 | 20260507 08:00:48.522 | INFO | PerfReport: 3.21%, Histogram -- 9.86e-01 / 1.48e+00, 1.65e-01 / 1.14e-01 | 20260507 08:00:48.522 | INFO | PerfReport: 16.33%, DME -- 4.01e+01 / 4.49e+01, 3.98e+00 / 1.59e+00 | 20260507 08:00:48.522 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.30e+00 / 1.19e+00 | 20260507 08:00:48.522 | INFO | PerfReport: 6.88%, PulseAccumulating -- 2.11e+00 / 2.65e+00, 3.86e-01 / 2.37e-01 | 20260507 08:00:48.522 | INFO | PerfReport: 2.92%, Metrics -- 8.96e-01 / 1.19e+00, 2.41e-01 / 1.05e-01 | 20260507 08:00:48.522 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.57e-01, 5.05e-02 / 2.01e-02 | 20260507 08:00:48.522 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:00:48.522 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:00:48.522 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:00:48.522 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 08:00:48.522 | INFO | PerfReport: 21.88%, Upload -- 2.10e+01 / 0.00e+00, 7.16e+00 / 0.00e+00 | 20260507 08:00:48.522 | INFO | PerfReport: 21.38%, Download -- 2.05e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 08:00:48.522 | INFO | PerfReport: 31.95%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 08:00:48.522 | INFO | PerfReport: 1.59%, LockUpload -- 1.52e+00 / 0.00e+00, 4.98e+00 / 0.00e+00 | 20260507 08:00:48.522 | INFO | PerfReport: 1.25%, LockDownload -- 1.20e+00 / 0.00e+00, 2.93e+00 / 0.00e+00 | 20260507 08:00:48.522 | INFO | PerfReport: 21.93%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 08:00:48.522 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 2.02e-02 / 0.00e+00 | 20260507 08:00:48.617 | INFO | Starting a round of Baz IO | 20260507 08:00:51.707 | INFO | Populating RTMetrics report | 20260507 08:00:51.708 | INFO | RTMetrics report populated | 20260507 08:00:51.708 | INFO | Opening tmp RTMetrics json file | 20260507 08:00:51.723 | INFO | Serializing RTMetrics to json | 20260507 08:00:51.725 | INFO | Serialization done, swapping tmp file now... | 20260507 08:00:51.725 | INFO | RTMetrics done | 20260507 08:00:51.793 | INFO | Finished a round of Baz IO: Wrote 1.624 GiB in 3.175 seconds | 20260507 08:00:51.793 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.52% / 0.00% / 67.48% | 20260507 08:00:51.793 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.096635% / 0% / 5.5631e-05% | 20260507 08:00:51.793 | INFO | Baz IO overall: | 20260507 08:00:51.793 | INFO | BytesWritten : 4177611968319 | 20260507 08:00:51.793 | INFO | NumEvents : 1999099334100 | 20260507 08:00:53.655 | INFO | Analyzing chunk frames = [8364032,8364544) | 20260507 08:00:53.656 | INFO | MemInfo: { | 20260507 08:00:53.656 | INFO | "gpu" : " 17 GB", | 20260507 08:00:53.656 | INFO | "huge" : "", | 20260507 08:00:53.656 | INFO | "huge_shared" : " 27 GB", | 20260507 08:00:53.656 | INFO | "malloc" : " 14 GB", | 20260507 08:00:53.656 | INFO | "pinned" : " 29 GB" | 20260507 08:00:53.656 | INFO | } | 20260507 08:00:53.656 | INFO | Perf report for the last 5.13544 seconds: | 20260507 08:00:53.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:00:53.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:53.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:53.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:53.656 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 08:00:53.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:53.656 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260507 08:00:53.656 | INFO | PRE_HQ : 58.9%, 1.8, 54.2ms, 0 | 20260507 08:00:53.656 | INFO | BAZWRITER : 61.4%, 3.1, 97.5ms, 0 | 20260507 08:00:53.656 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:00:53.656 | INFO | Uploaded 36GiB to GPU in 2.05473s (17.5205GiB/s) | 20260507 08:00:53.656 | INFO | Downloaded 38GiB to GPU in 2.17992s (17.4318GiB/s) | 20260507 08:00:58.796 | INFO | Analyzing chunk frames = [8364544,8365056) | 20260507 08:00:58.796 | INFO | MemInfo: { | 20260507 08:00:58.796 | INFO | "gpu" : " 17 GB", | 20260507 08:00:58.796 | INFO | "huge" : "", | 20260507 08:00:58.796 | INFO | "huge_shared" : " 27 GB", | 20260507 08:00:58.796 | INFO | "malloc" : " 14 GB", | 20260507 08:00:58.797 | INFO | "pinned" : " 29 GB" | 20260507 08:00:58.797 | INFO | } | 20260507 08:00:58.797 | INFO | Perf report for the last 5.14116 seconds: | 20260507 08:00:58.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:00:58.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:58.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:58.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:58.797 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 08:00:58.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:58.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:58.797 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 08:00:58.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:58.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:00:58.797 | INFO | Uploaded 36GiB to GPU in 1.98733s (18.1148GiB/s) | 20260507 08:00:58.797 | INFO | Downloaded 34GiB to GPU in 1.93165s (17.6016GiB/s) | 20260507 08:01:03.941 | INFO | Analyzing chunk frames = [8365056,8365568) | 20260507 08:01:03.941 | INFO | MemInfo: { | 20260507 08:01:03.941 | INFO | "gpu" : " 17 GB", | 20260507 08:01:03.941 | INFO | "huge" : "", | 20260507 08:01:03.941 | INFO | "huge_shared" : " 27 GB", | 20260507 08:01:03.941 | INFO | "malloc" : " 14 GB", | 20260507 08:01:03.941 | INFO | "pinned" : " 29 GB" | 20260507 08:01:03.941 | INFO | } | 20260507 08:01:03.941 | INFO | Perf report for the last 5.1445 seconds: | 20260507 08:01:03.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:01:03.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:03.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:03.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:03.941 | INFO | ANALYSIS : 60.3%, 3.0, 93.4ms, 0 | 20260507 08:01:03.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:03.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:03.941 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:01:03.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:03.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:03.941 | INFO | Uploaded 36GiB to GPU in 2.02957s (17.7377GiB/s) | 20260507 08:01:03.941 | INFO | Downloaded 34GiB to GPU in 1.95217s (17.4166GiB/s) | 20260507 08:01:09.080 | INFO | Analyzing chunk frames = [8365568,8366080) | 20260507 08:01:09.080 | INFO | MemInfo: { | 20260507 08:01:09.080 | INFO | "gpu" : " 17 GB", | 20260507 08:01:09.080 | INFO | "huge" : "", | 20260507 08:01:09.080 | INFO | "huge_shared" : " 27 GB", | 20260507 08:01:09.080 | INFO | "malloc" : " 14 GB", | 20260507 08:01:09.080 | INFO | "pinned" : " 29 GB" | 20260507 08:01:09.080 | INFO | } | 20260507 08:01:09.080 | INFO | Perf report for the last 5.13919 seconds: | 20260507 08:01:09.080 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:01:09.080 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:09.080 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:09.081 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:09.081 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 08:01:09.081 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:09.081 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:09.081 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:01:09.081 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:09.081 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:09.081 | INFO | Uploaded 36GiB to GPU in 2.05325s (17.5331GiB/s) | 20260507 08:01:09.081 | INFO | Downloaded 34GiB to GPU in 1.9566s (17.377GiB/s) | 20260507 08:01:14.211 | INFO | Analyzing chunk frames = [8366080,8366592) | 20260507 08:01:14.211 | INFO | MemInfo: { | 20260507 08:01:14.211 | INFO | "gpu" : " 17 GB", | 20260507 08:01:14.211 | INFO | "huge" : "", | 20260507 08:01:14.211 | INFO | "huge_shared" : " 27 GB", | 20260507 08:01:14.211 | INFO | "malloc" : " 14 GB", | 20260507 08:01:14.211 | INFO | "pinned" : " 29 GB" | 20260507 08:01:14.211 | INFO | } | 20260507 08:01:14.211 | INFO | Perf report for the last 5.13083 seconds: | 20260507 08:01:14.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:01:14.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:14.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:14.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:14.211 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 08:01:14.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:14.211 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:14.211 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:01:14.211 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:14.211 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:14.212 | INFO | Uploaded 36GiB to GPU in 2.05031s (17.5583GiB/s) | 20260507 08:01:14.212 | INFO | Downloaded 34GiB to GPU in 1.89707s (17.9224GiB/s) | 20260507 08:01:19.352 | INFO | Analyzing chunk frames = [8366592,8367104) | 20260507 08:01:19.352 | INFO | MemInfo: { | 20260507 08:01:19.352 | INFO | "gpu" : " 17 GB", | 20260507 08:01:19.352 | INFO | "huge" : "", | 20260507 08:01:19.352 | INFO | "huge_shared" : " 27 GB", | 20260507 08:01:19.353 | INFO | "malloc" : " 14 GB", | 20260507 08:01:19.353 | INFO | "pinned" : " 29 GB" | 20260507 08:01:19.353 | INFO | } | 20260507 08:01:19.353 | INFO | Perf report for the last 5.14124 seconds: | 20260507 08:01:19.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:01:19.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:19.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:19.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:19.353 | INFO | ANALYSIS : 60.4%, 3.0, 94.4ms, 0 | 20260507 08:01:19.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:19.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:19.353 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 08:01:19.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:19.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:19.353 | INFO | Uploaded 36GiB to GPU in 2.06306s (17.4498GiB/s) | 20260507 08:01:19.353 | INFO | Downloaded 34GiB to GPU in 1.93864s (17.538GiB/s) | 20260507 08:01:24.494 | INFO | Analyzing chunk frames = [8367104,8367616) | 20260507 08:01:24.494 | INFO | MemInfo: { | 20260507 08:01:24.494 | INFO | "gpu" : " 17 GB", | 20260507 08:01:24.494 | INFO | "huge" : "", | 20260507 08:01:24.494 | INFO | "huge_shared" : " 27 GB", | 20260507 08:01:24.494 | INFO | "malloc" : " 14 GB", | 20260507 08:01:24.494 | INFO | "pinned" : " 29 GB" | 20260507 08:01:24.494 | INFO | } | 20260507 08:01:24.494 | INFO | Perf report for the last 5.14172 seconds: | 20260507 08:01:24.494 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:01:24.494 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:24.494 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:24.494 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:24.494 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:01:24.494 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:24.494 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:24.494 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:01:24.494 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:24.494 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:24.495 | INFO | Uploaded 36GiB to GPU in 1.95336s (18.4298GiB/s) | 20260507 08:01:24.495 | INFO | Downloaded 34GiB to GPU in 1.97728s (17.1953GiB/s) | 20260507 08:01:29.635 | INFO | Analyzing chunk frames = [8367616,8368128) | 20260507 08:01:29.635 | INFO | MemInfo: { | 20260507 08:01:29.635 | INFO | "gpu" : " 17 GB", | 20260507 08:01:29.635 | INFO | "huge" : "", | 20260507 08:01:29.635 | INFO | "huge_shared" : " 27 GB", | 20260507 08:01:29.635 | INFO | "malloc" : " 14 GB", | 20260507 08:01:29.635 | INFO | "pinned" : " 29 GB" | 20260507 08:01:29.635 | INFO | } | 20260507 08:01:29.635 | INFO | Perf report for the last 5.14073 seconds: | 20260507 08:01:29.635 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:01:29.635 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:29.635 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:29.635 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:29.635 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:01:29.635 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:29.635 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:29.635 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:01:29.635 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:29.635 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:29.635 | INFO | Uploaded 36GiB to GPU in 1.9867s (18.1205GiB/s) | 20260507 08:01:29.635 | INFO | Downloaded 34GiB to GPU in 1.96145s (17.3341GiB/s) | 20260507 08:01:29.732 | INFO | Starting a round of Baz IO | 20260507 08:01:32.823 | INFO | Populating RTMetrics report | 20260507 08:01:32.824 | INFO | RTMetrics report populated | 20260507 08:01:32.824 | INFO | Opening tmp RTMetrics json file | 20260507 08:01:32.834 | INFO | Serializing RTMetrics to json | 20260507 08:01:32.839 | INFO | Serialization done, swapping tmp file now... | 20260507 08:01:32.839 | INFO | RTMetrics done | 20260507 08:01:32.907 | INFO | Finished a round of Baz IO: Wrote 1.626 GiB in 3.174 seconds | 20260507 08:01:32.907 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.52% / 0.00% / 67.48% | 20260507 08:01:32.907 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0963847% / 0% / 4.37101e-05% | 20260507 08:01:32.907 | INFO | Baz IO overall: | 20260507 08:01:32.907 | INFO | BytesWritten : 4179357530654 | 20260507 08:01:32.907 | INFO | NumEvents : 1999913682373 | 20260507 08:01:34.770 | INFO | Analyzing chunk frames = [8368128,8368640) | 20260507 08:01:34.770 | INFO | MemInfo: { | 20260507 08:01:34.770 | INFO | "gpu" : " 17 GB", | 20260507 08:01:34.770 | INFO | "huge" : "", | 20260507 08:01:34.770 | INFO | "huge_shared" : " 27 GB", | 20260507 08:01:34.770 | INFO | "malloc" : " 14 GB", | 20260507 08:01:34.770 | INFO | "pinned" : " 29 GB" | 20260507 08:01:34.770 | INFO | } | 20260507 08:01:34.771 | INFO | Perf report for the last 5.13545 seconds: | 20260507 08:01:34.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:01:34.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:34.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:34.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:34.771 | INFO | ANALYSIS : 61.6%, 3.0, 95.7ms, 0 | 20260507 08:01:34.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:34.771 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260507 08:01:34.771 | INFO | PRE_HQ : 58.9%, 1.8, 54.1ms, 0 | 20260507 08:01:34.771 | INFO | BAZWRITER : 61.4%, 3.1, 99.2ms, 0 | 20260507 08:01:34.771 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:01:34.771 | INFO | Uploaded 36GiB to GPU in 2.06694s (17.417GiB/s) | 20260507 08:01:34.771 | INFO | Downloaded 38GiB to GPU in 2.18267s (17.4098GiB/s) | 20260507 08:01:39.908 | INFO | Analyzing chunk frames = [8368640,8369152) | 20260507 08:01:39.908 | INFO | MemInfo: { | 20260507 08:01:39.908 | INFO | "gpu" : " 17 GB", | 20260507 08:01:39.908 | INFO | "huge" : "", | 20260507 08:01:39.908 | INFO | "huge_shared" : " 27 GB", | 20260507 08:01:39.908 | INFO | "malloc" : " 14 GB", | 20260507 08:01:39.908 | INFO | "pinned" : " 29 GB" | 20260507 08:01:39.908 | INFO | } | 20260507 08:01:39.908 | INFO | Perf report for the last 5.13775 seconds: | 20260507 08:01:39.908 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:01:39.908 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:39.908 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:39.908 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:39.908 | INFO | ANALYSIS : 60.8%, 3.0, 94.6ms, 0 | 20260507 08:01:39.908 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:39.908 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:39.908 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260507 08:01:39.908 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:39.908 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:39.909 | INFO | Uploaded 36GiB to GPU in 2.01311s (17.8828GiB/s) | 20260507 08:01:39.909 | INFO | Downloaded 34GiB to GPU in 1.93008s (17.6159GiB/s) | 20260507 08:01:45.052 | INFO | Analyzing chunk frames = [8369152,8369664) | 20260507 08:01:45.052 | INFO | MemInfo: { | 20260507 08:01:45.052 | INFO | "gpu" : " 17 GB", | 20260507 08:01:45.052 | INFO | "huge" : "", | 20260507 08:01:45.052 | INFO | "huge_shared" : " 27 GB", | 20260507 08:01:45.052 | INFO | "malloc" : " 14 GB", | 20260507 08:01:45.052 | INFO | "pinned" : " 29 GB" | 20260507 08:01:45.052 | INFO | } | 20260507 08:01:45.052 | INFO | Perf report for the last 5.14353 seconds: | 20260507 08:01:45.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:01:45.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:45.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:45.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:45.052 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 08:01:45.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:45.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:45.052 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:01:45.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:45.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:45.052 | INFO | Uploaded 36GiB to GPU in 2.03712s (17.672GiB/s) | 20260507 08:01:45.052 | INFO | Downloaded 34GiB to GPU in 1.95226s (17.4157GiB/s) | 20260507 08:01:50.193 | INFO | Analyzing chunk frames = [8369664,8370176) | 20260507 08:01:50.193 | INFO | MemInfo: { | 20260507 08:01:50.193 | INFO | "gpu" : " 17 GB", | 20260507 08:01:50.193 | INFO | "huge" : "", | 20260507 08:01:50.193 | INFO | "huge_shared" : " 27 GB", | 20260507 08:01:50.194 | INFO | "malloc" : " 14 GB", | 20260507 08:01:50.194 | INFO | "pinned" : " 29 GB" | 20260507 08:01:50.194 | INFO | } | 20260507 08:01:50.194 | INFO | Perf report for the last 5.14176 seconds: | 20260507 08:01:50.194 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:01:50.194 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:50.194 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:50.194 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:50.194 | INFO | ANALYSIS : 60.3%, 2.9, 93.1ms, 0 | 20260507 08:01:50.194 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:50.194 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:50.194 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:01:50.194 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:50.194 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:50.194 | INFO | Uploaded 36GiB to GPU in 2.02229s (17.8016GiB/s) | 20260507 08:01:50.194 | INFO | Downloaded 34GiB to GPU in 1.94521s (17.4789GiB/s) | 20260507 08:01:55.330 | INFO | Analyzing chunk frames = [8370176,8370688) | 20260507 08:01:55.330 | INFO | MemInfo: { | 20260507 08:01:55.330 | INFO | "gpu" : " 17 GB", | 20260507 08:01:55.330 | INFO | "huge" : "", | 20260507 08:01:55.330 | INFO | "huge_shared" : " 27 GB", | 20260507 08:01:55.330 | INFO | "malloc" : " 14 GB", | 20260507 08:01:55.330 | INFO | "pinned" : " 29 GB" | 20260507 08:01:55.330 | INFO | } | 20260507 08:01:55.330 | INFO | Perf report for the last 5.1365 seconds: | 20260507 08:01:55.330 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:01:55.330 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:55.330 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:55.330 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:55.330 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 08:01:55.330 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:55.330 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:55.330 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:01:55.330 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:55.330 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:01:55.330 | INFO | Uploaded 36GiB to GPU in 2.05855s (17.488GiB/s) | 20260507 08:01:55.330 | INFO | Downloaded 34GiB to GPU in 1.90702s (17.8289GiB/s) | 20260507 08:02:00.463 | INFO | Analyzing chunk frames = [8370688,8371200) | 20260507 08:02:00.463 | INFO | MemInfo: { | 20260507 08:02:00.463 | INFO | "gpu" : " 17 GB", | 20260507 08:02:00.463 | INFO | "huge" : "", | 20260507 08:02:00.463 | INFO | "huge_shared" : " 27 GB", | 20260507 08:02:00.463 | INFO | "malloc" : " 14 GB", | 20260507 08:02:00.463 | INFO | "pinned" : " 29 GB" | 20260507 08:02:00.463 | INFO | } | 20260507 08:02:00.463 | INFO | Perf report for the last 5.1329 seconds: | 20260507 08:02:00.463 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:02:00.463 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:00.463 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:00.463 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:00.463 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:02:00.463 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:00.463 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:00.463 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:02:00.463 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:00.463 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:00.463 | INFO | Uploaded 36GiB to GPU in 2.02612s (17.768GiB/s) | 20260507 08:02:00.463 | INFO | Downloaded 34GiB to GPU in 1.92613s (17.652GiB/s) | 20260507 08:02:05.614 | INFO | Analyzing chunk frames = [8371200,8371712) | 20260507 08:02:05.614 | INFO | MemInfo: { | 20260507 08:02:05.614 | INFO | "gpu" : " 17 GB", | 20260507 08:02:05.614 | INFO | "huge" : "", | 20260507 08:02:05.614 | INFO | "huge_shared" : " 27 GB", | 20260507 08:02:05.614 | INFO | "malloc" : " 14 GB", | 20260507 08:02:05.614 | INFO | "pinned" : " 29 GB" | 20260507 08:02:05.614 | INFO | } | 20260507 08:02:05.614 | INFO | Perf report for the last 5.15128 seconds: | 20260507 08:02:05.614 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:02:05.614 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:05.614 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:05.614 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:05.614 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 08:02:05.614 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:05.614 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:05.614 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 08:02:05.614 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:05.614 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:05.615 | INFO | Uploaded 36GiB to GPU in 1.99042s (18.0866GiB/s) | 20260507 08:02:05.615 | INFO | Downloaded 34GiB to GPU in 1.97012s (17.2579GiB/s) | 20260507 08:02:10.747 | INFO | Analyzing chunk frames = [8371712,8372224) | 20260507 08:02:10.747 | INFO | MemInfo: { | 20260507 08:02:10.747 | INFO | "gpu" : " 17 GB", | 20260507 08:02:10.747 | INFO | "huge" : "", | 20260507 08:02:10.747 | INFO | "huge_shared" : " 27 GB", | 20260507 08:02:10.747 | INFO | "malloc" : " 14 GB", | 20260507 08:02:10.747 | INFO | "pinned" : " 29 GB" | 20260507 08:02:10.747 | INFO | } | 20260507 08:02:10.747 | INFO | Perf report for the last 5.1331 seconds: | 20260507 08:02:10.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:02:10.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:10.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:10.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:10.748 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 08:02:10.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:10.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:10.748 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 08:02:10.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:10.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:10.748 | INFO | Uploaded 36GiB to GPU in 1.99312s (18.0621GiB/s) | 20260507 08:02:10.748 | INFO | Downloaded 34GiB to GPU in 1.95661s (17.377GiB/s) | 20260507 08:02:10.748 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:02:10.748 | INFO | Total mem usage: 18 GiB | 20260507 08:02:10.748 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:02:10.748 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:02:10.748 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:02:10.748 | INFO | High Water Breakdown: | 20260507 08:02:10.748 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:02:10.748 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:02:10.748 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:02:10.748 | INFO | ------------------------------------------ | 20260507 08:02:10.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:02:10.748 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:02:10.748 | INFO | Total mem usage: 26.5 GiB | 20260507 08:02:10.748 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:02:10.748 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:02:10.748 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:02:10.748 | INFO | High Water Breakdown: | 20260507 08:02:10.748 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:02:10.748 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:02:10.748 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:02:10.748 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:02:10.748 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:02:10.748 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:02:10.748 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:02:10.748 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:02:10.748 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:02:10.748 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:02:10.748 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:02:10.748 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:02:10.748 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:02:10.748 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:02:10.748 | INFO | ------------------------------------------ | 20260507 08:02:10.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:02:10.748 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:02:10.748 | INFO | Total mem usage: 25.6 GiB | 20260507 08:02:10.748 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:02:10.748 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:02:10.748 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:02:10.748 | INFO | High Water Breakdown: | 20260507 08:02:10.748 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:02:10.748 | INFO | ------------------------------------------ | 20260507 08:02:10.748 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:02:10.748 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:02:10.748 | INFO | Total mem usage: 15.7 GiB | 20260507 08:02:10.748 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:02:10.748 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:02:10.748 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:02:10.748 | INFO | High Water Breakdown: | 20260507 08:02:10.748 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:02:10.748 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:02:10.748 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:02:10.748 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:02:10.748 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:02:10.748 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:02:10.748 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:02:10.749 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:02:10.749 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:02:10.749 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:02:10.749 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:02:10.749 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:02:10.749 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:02:10.749 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:02:10.749 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:02:10.749 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:02:10.749 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:02:10.749 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:02:10.749 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:02:10.749 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:02:10.749 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:02:10.749 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:02:10.749 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:02:10.749 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:02:10.749 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:02:10.749 | INFO | ------------------------------------------ | 20260507 08:02:10.749 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:02:10.749 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:02:10.749 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:02:10.749 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:02:10.749 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.33e+01 / 1.42e+00 | 20260507 08:02:10.749 | INFO | PerfReport: 0.30%, AggregateStats -- 9.05e-02 / 1.01e-01, 9.48e-03 / 5.45e-03 | 20260507 08:02:10.749 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.44e+00, 4.97e-01 / 2.37e-01 | 20260507 08:02:10.749 | INFO | PerfReport: 3.23%, Histogram -- 9.90e-01 / 1.47e+00, 1.67e-01 / 1.26e-01 | 20260507 08:02:10.749 | INFO | PerfReport: 16.34%, DME -- 4.01e+01 / 4.49e+01, 4.01e+00 / 1.87e+00 | 20260507 08:02:10.749 | INFO | PerfReport: 57.71%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.27e+00 / 1.16e+00 | 20260507 08:02:10.749 | INFO | PerfReport: 6.88%, PulseAccumulating -- 2.11e+00 / 2.66e+00, 3.82e-01 / 2.05e-01 | 20260507 08:02:10.749 | INFO | PerfReport: 2.92%, Metrics -- 8.95e-01 / 1.20e+00, 2.41e-01 / 7.42e-02 | 20260507 08:02:10.749 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.52e-01, 5.04e-02 / 1.69e-02 | 20260507 08:02:10.749 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:02:10.749 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:02:10.749 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:02:10.749 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 08:02:10.749 | INFO | PerfReport: 21.93%, Upload -- 2.11e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260507 08:02:10.749 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 08:02:10.749 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 08:02:10.749 | INFO | PerfReport: 1.55%, LockUpload -- 1.49e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 08:02:10.749 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260507 08:02:10.749 | INFO | PerfReport: 21.91%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 08:02:10.749 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.83e-02 / 0.00e+00 | 20260507 08:02:10.845 | INFO | Starting a round of Baz IO | 20260507 08:02:13.944 | INFO | Populating RTMetrics report | 20260507 08:02:13.945 | INFO | RTMetrics report populated | 20260507 08:02:13.945 | INFO | Opening tmp RTMetrics json file | 20260507 08:02:13.945 | INFO | Serializing RTMetrics to json | 20260507 08:02:13.947 | INFO | Serialization done, swapping tmp file now... | 20260507 08:02:13.947 | INFO | RTMetrics done | 20260507 08:02:14.033 | INFO | Finished a round of Baz IO: Wrote 1.625 GiB in 3.188 seconds | 20260507 08:02:14.033 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.52% / 0.00% / 67.48% | 20260507 08:02:14.033 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0963648% / 3.97364e-06% / 5.5631e-05% | 20260507 08:02:14.033 | INFO | Baz IO overall: | 20260507 08:02:14.033 | INFO | BytesWritten : 4181102607276 | 20260507 08:02:14.033 | INFO | NumEvents : 2000727743455 | 20260507 08:02:15.893 | INFO | Analyzing chunk frames = [8372224,8372736) | 20260507 08:02:15.893 | INFO | MemInfo: { | 20260507 08:02:15.893 | INFO | "gpu" : " 17 GB", | 20260507 08:02:15.893 | INFO | "huge" : "", | 20260507 08:02:15.893 | INFO | "huge_shared" : " 27 GB", | 20260507 08:02:15.893 | INFO | "malloc" : " 14 GB", | 20260507 08:02:15.893 | INFO | "pinned" : " 29 GB" | 20260507 08:02:15.893 | INFO | } | 20260507 08:02:15.893 | INFO | Perf report for the last 5.14568 seconds: | 20260507 08:02:15.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:02:15.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:15.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:15.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:15.893 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260507 08:02:15.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:15.893 | INFO | RT_METRICS : 42.4%, 1.0, 22.2ms, 0 | 20260507 08:02:15.893 | INFO | PRE_HQ : 59.3%, 1.8, 56.8ms, 0 | 20260507 08:02:15.893 | INFO | BAZWRITER : 61.5%, 3.2, 100.8ms, 0 | 20260507 08:02:15.893 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 08:02:15.893 | INFO | Uploaded 36GiB to GPU in 2.06265s (17.4533GiB/s) | 20260507 08:02:15.894 | INFO | Downloaded 38GiB to GPU in 2.17428s (17.4771GiB/s) | 20260507 08:02:21.026 | INFO | Analyzing chunk frames = [8372736,8373248) | 20260507 08:02:21.027 | INFO | MemInfo: { | 20260507 08:02:21.027 | INFO | "gpu" : " 17 GB", | 20260507 08:02:21.027 | INFO | "huge" : "", | 20260507 08:02:21.027 | INFO | "huge_shared" : " 27 GB", | 20260507 08:02:21.027 | INFO | "malloc" : " 14 GB", | 20260507 08:02:21.027 | INFO | "pinned" : " 29 GB" | 20260507 08:02:21.027 | INFO | } | 20260507 08:02:21.027 | INFO | Perf report for the last 5.13362 seconds: | 20260507 08:02:21.027 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:02:21.027 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:21.027 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:21.027 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:21.027 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 08:02:21.027 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:21.027 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:21.027 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 08:02:21.027 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:21.027 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:21.027 | INFO | Uploaded 36GiB to GPU in 2.02388s (17.7877GiB/s) | 20260507 08:02:21.027 | INFO | Downloaded 34GiB to GPU in 1.94412s (17.4886GiB/s) | 20260507 08:02:26.160 | INFO | Analyzing chunk frames = [8373248,8373760) | 20260507 08:02:26.160 | INFO | MemInfo: { | 20260507 08:02:26.160 | INFO | "gpu" : " 17 GB", | 20260507 08:02:26.160 | INFO | "huge" : "", | 20260507 08:02:26.160 | INFO | "huge_shared" : " 27 GB", | 20260507 08:02:26.160 | INFO | "malloc" : " 14 GB", | 20260507 08:02:26.160 | INFO | "pinned" : " 29 GB" | 20260507 08:02:26.160 | INFO | } | 20260507 08:02:26.160 | INFO | Perf report for the last 5.13326 seconds: | 20260507 08:02:26.160 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:02:26.160 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:26.160 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:26.160 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:26.160 | INFO | ANALYSIS : 60.7%, 3.0, 93.8ms, 0 | 20260507 08:02:26.160 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:26.160 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:26.160 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:02:26.160 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:26.160 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:26.160 | INFO | Uploaded 36GiB to GPU in 2.05037s (17.5578GiB/s) | 20260507 08:02:26.160 | INFO | Downloaded 34GiB to GPU in 1.94941s (17.4412GiB/s) | 20260507 08:02:31.305 | INFO | Analyzing chunk frames = [8373760,8374272) | 20260507 08:02:31.305 | INFO | MemInfo: { | 20260507 08:02:31.305 | INFO | "gpu" : " 17 GB", | 20260507 08:02:31.305 | INFO | "huge" : "", | 20260507 08:02:31.305 | INFO | "huge_shared" : " 27 GB", | 20260507 08:02:31.305 | INFO | "malloc" : " 14 GB", | 20260507 08:02:31.305 | INFO | "pinned" : " 29 GB" | 20260507 08:02:31.305 | INFO | } | 20260507 08:02:31.305 | INFO | Perf report for the last 5.14543 seconds: | 20260507 08:02:31.305 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:02:31.305 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:31.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:31.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:31.306 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 08:02:31.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:31.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:31.306 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:02:31.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:31.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:31.306 | INFO | Uploaded 36GiB to GPU in 2.02837s (17.7482GiB/s) | 20260507 08:02:31.306 | INFO | Downloaded 34GiB to GPU in 1.9393s (17.5321GiB/s) | 20260507 08:02:36.452 | INFO | Analyzing chunk frames = [8374272,8374784) | 20260507 08:02:36.452 | INFO | MemInfo: { | 20260507 08:02:36.452 | INFO | "gpu" : " 17 GB", | 20260507 08:02:36.452 | INFO | "huge" : "", | 20260507 08:02:36.452 | INFO | "huge_shared" : " 27 GB", | 20260507 08:02:36.452 | INFO | "malloc" : " 14 GB", | 20260507 08:02:36.452 | INFO | "pinned" : " 29 GB" | 20260507 08:02:36.452 | INFO | } | 20260507 08:02:36.452 | INFO | Perf report for the last 5.14631 seconds: | 20260507 08:02:36.452 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:02:36.452 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:36.452 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:36.452 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:36.452 | INFO | ANALYSIS : 60.1%, 3.0, 93.9ms, 0 | 20260507 08:02:36.452 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:36.452 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:36.452 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:02:36.452 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:36.452 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:36.452 | INFO | Uploaded 36GiB to GPU in 2.05339s (17.532GiB/s) | 20260507 08:02:36.452 | INFO | Downloaded 34GiB to GPU in 1.91549s (17.7501GiB/s) | 20260507 08:02:41.589 | INFO | Analyzing chunk frames = [8374784,8375296) | 20260507 08:02:41.589 | INFO | MemInfo: { | 20260507 08:02:41.589 | INFO | "gpu" : " 17 GB", | 20260507 08:02:41.589 | INFO | "huge" : "", | 20260507 08:02:41.590 | INFO | "huge_shared" : " 27 GB", | 20260507 08:02:41.590 | INFO | "malloc" : " 14 GB", | 20260507 08:02:41.590 | INFO | "pinned" : " 29 GB" | 20260507 08:02:41.590 | INFO | } | 20260507 08:02:41.590 | INFO | Perf report for the last 5.13791 seconds: | 20260507 08:02:41.590 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:02:41.590 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:41.590 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:41.590 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:41.590 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:02:41.590 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:41.590 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:41.590 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:02:41.590 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:41.590 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:41.590 | INFO | Uploaded 36GiB to GPU in 2.07604s (17.3407GiB/s) | 20260507 08:02:41.590 | INFO | Downloaded 34GiB to GPU in 1.94427s (17.4873GiB/s) | 20260507 08:02:46.718 | INFO | Analyzing chunk frames = [8375296,8375808) | 20260507 08:02:46.718 | INFO | MemInfo: { | 20260507 08:02:46.718 | INFO | "gpu" : " 17 GB", | 20260507 08:02:46.718 | INFO | "huge" : "", | 20260507 08:02:46.718 | INFO | "huge_shared" : " 27 GB", | 20260507 08:02:46.718 | INFO | "malloc" : " 14 GB", | 20260507 08:02:46.718 | INFO | "pinned" : " 29 GB" | 20260507 08:02:46.718 | INFO | } | 20260507 08:02:46.718 | INFO | Perf report for the last 5.12867 seconds: | 20260507 08:02:46.718 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:02:46.718 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:46.718 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:46.718 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:46.718 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 08:02:46.718 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:46.718 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:46.719 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 08:02:46.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:46.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:46.719 | INFO | Uploaded 36GiB to GPU in 1.97941s (18.1873GiB/s) | 20260507 08:02:46.719 | INFO | Downloaded 34GiB to GPU in 1.98381s (17.1388GiB/s) | 20260507 08:02:51.859 | INFO | Analyzing chunk frames = [8375808,8376320) | 20260507 08:02:51.859 | INFO | MemInfo: { | 20260507 08:02:51.859 | INFO | "gpu" : " 17 GB", | 20260507 08:02:51.859 | INFO | "huge" : "", | 20260507 08:02:51.859 | INFO | "huge_shared" : " 27 GB", | 20260507 08:02:51.859 | INFO | "malloc" : " 14 GB", | 20260507 08:02:51.859 | INFO | "pinned" : " 29 GB" | 20260507 08:02:51.859 | INFO | } | 20260507 08:02:51.859 | INFO | Perf report for the last 5.14044 seconds: | 20260507 08:02:51.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:02:51.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:51.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:51.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:51.859 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:02:51.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:51.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:51.859 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:02:51.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:51.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:51.859 | INFO | Uploaded 36GiB to GPU in 1.9856s (18.1305GiB/s) | 20260507 08:02:51.859 | INFO | Downloaded 34GiB to GPU in 1.95957s (17.3508GiB/s) | 20260507 08:02:51.956 | INFO | Starting a round of Baz IO | 20260507 08:02:55.054 | INFO | Populating RTMetrics report | 20260507 08:02:55.055 | INFO | RTMetrics report populated | 20260507 08:02:55.055 | INFO | Opening tmp RTMetrics json file | 20260507 08:02:55.066 | INFO | Serializing RTMetrics to json | 20260507 08:02:55.069 | INFO | Serialization done, swapping tmp file now... | 20260507 08:02:55.069 | INFO | RTMetrics done | 20260507 08:02:55.139 | INFO | Finished a round of Baz IO: Wrote 1.624 GiB in 3.182 seconds | 20260507 08:02:55.139 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.52% / 0.00% / 67.48% | 20260507 08:02:55.139 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0959555% / 0% / 4.37101e-05% | 20260507 08:02:55.139 | INFO | Baz IO overall: | 20260507 08:02:55.139 | INFO | BytesWritten : 4182846854018 | 20260507 08:02:55.139 | INFO | NumEvents : 2001540788645 | 20260507 08:02:57.001 | INFO | Analyzing chunk frames = [8376320,8376832) | 20260507 08:02:57.001 | INFO | MemInfo: { | 20260507 08:02:57.001 | INFO | "gpu" : " 17 GB", | 20260507 08:02:57.001 | INFO | "huge" : "", | 20260507 08:02:57.001 | INFO | "huge_shared" : " 27 GB", | 20260507 08:02:57.001 | INFO | "malloc" : " 14 GB", | 20260507 08:02:57.001 | INFO | "pinned" : " 29 GB" | 20260507 08:02:57.001 | INFO | } | 20260507 08:02:57.001 | INFO | Perf report for the last 5.14222 seconds: | 20260507 08:02:57.001 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:02:57.001 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:57.001 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:57.001 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:57.001 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 08:02:57.001 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:02:57.001 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260507 08:02:57.001 | INFO | PRE_HQ : 58.9%, 1.8, 54.4ms, 0 | 20260507 08:02:57.001 | INFO | BAZWRITER : 61.3%, 3.1, 98.1ms, 0 | 20260507 08:02:57.001 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:02:57.001 | INFO | Uploaded 36GiB to GPU in 2.06798s (17.4083GiB/s) | 20260507 08:02:57.001 | INFO | Downloaded 38GiB to GPU in 2.18528s (17.3891GiB/s) | 20260507 08:03:02.137 | INFO | Analyzing chunk frames = [8376832,8377344) | 20260507 08:03:02.137 | INFO | MemInfo: { | 20260507 08:03:02.137 | INFO | "gpu" : " 17 GB", | 20260507 08:03:02.137 | INFO | "huge" : "", | 20260507 08:03:02.137 | INFO | "huge_shared" : " 27 GB", | 20260507 08:03:02.137 | INFO | "malloc" : " 14 GB", | 20260507 08:03:02.138 | INFO | "pinned" : " 29 GB" | 20260507 08:03:02.138 | INFO | } | 20260507 08:03:02.138 | INFO | Perf report for the last 5.13664 seconds: | 20260507 08:03:02.138 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:03:02.138 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:02.138 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:02.138 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:02.138 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 08:03:02.138 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:02.138 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:02.138 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 08:03:02.138 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:02.138 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:02.138 | INFO | Uploaded 36GiB to GPU in 1.98955s (18.0945GiB/s) | 20260507 08:03:02.138 | INFO | Downloaded 34GiB to GPU in 1.93214s (17.597GiB/s) | 20260507 08:03:07.280 | INFO | Analyzing chunk frames = [8377344,8377856) | 20260507 08:03:07.280 | INFO | MemInfo: { | 20260507 08:03:07.280 | INFO | "gpu" : " 17 GB", | 20260507 08:03:07.280 | INFO | "huge" : "", | 20260507 08:03:07.280 | INFO | "huge_shared" : " 27 GB", | 20260507 08:03:07.280 | INFO | "malloc" : " 14 GB", | 20260507 08:03:07.280 | INFO | "pinned" : " 29 GB" | 20260507 08:03:07.280 | INFO | } | 20260507 08:03:07.280 | INFO | Perf report for the last 5.142 seconds: | 20260507 08:03:07.280 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:03:07.280 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:07.280 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:07.280 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:07.280 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 08:03:07.280 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:07.280 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:07.280 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:03:07.280 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:07.280 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:07.280 | INFO | Uploaded 36GiB to GPU in 2.0563s (17.5071GiB/s) | 20260507 08:03:07.280 | INFO | Downloaded 34GiB to GPU in 1.94978s (17.4379GiB/s) | 20260507 08:03:12.416 | INFO | Analyzing chunk frames = [8377856,8378368) | 20260507 08:03:12.416 | INFO | MemInfo: { | 20260507 08:03:12.416 | INFO | "gpu" : " 17 GB", | 20260507 08:03:12.416 | INFO | "huge" : "", | 20260507 08:03:12.417 | INFO | "huge_shared" : " 27 GB", | 20260507 08:03:12.417 | INFO | "malloc" : " 14 GB", | 20260507 08:03:12.417 | INFO | "pinned" : " 29 GB" | 20260507 08:03:12.417 | INFO | } | 20260507 08:03:12.417 | INFO | Perf report for the last 5.1369 seconds: | 20260507 08:03:12.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:03:12.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:12.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:12.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:12.417 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 08:03:12.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:12.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:12.417 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 08:03:12.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:12.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:12.417 | INFO | Uploaded 36GiB to GPU in 2.04222s (17.6279GiB/s) | 20260507 08:03:12.417 | INFO | Downloaded 34GiB to GPU in 1.95145s (17.4229GiB/s) | 20260507 08:03:17.570 | INFO | Analyzing chunk frames = [8378368,8378880) | 20260507 08:03:17.570 | INFO | MemInfo: { | 20260507 08:03:17.570 | INFO | "gpu" : " 17 GB", | 20260507 08:03:17.570 | INFO | "huge" : "", | 20260507 08:03:17.570 | INFO | "huge_shared" : " 27 GB", | 20260507 08:03:17.570 | INFO | "malloc" : " 14 GB", | 20260507 08:03:17.570 | INFO | "pinned" : " 29 GB" | 20260507 08:03:17.571 | INFO | } | 20260507 08:03:17.571 | INFO | Perf report for the last 5.15389 seconds: | 20260507 08:03:17.571 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:03:17.571 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:17.571 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:17.571 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:17.571 | INFO | ANALYSIS : 60.0%, 3.0, 93.9ms, 0 | 20260507 08:03:17.571 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:17.571 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:17.571 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:03:17.571 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:17.571 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:17.571 | INFO | Uploaded 36GiB to GPU in 2.05742s (17.4977GiB/s) | 20260507 08:03:17.571 | INFO | Downloaded 34GiB to GPU in 1.95104s (17.4266GiB/s) | 20260507 08:03:22.703 | INFO | Analyzing chunk frames = [8378880,8379392) | 20260507 08:03:22.703 | INFO | MemInfo: { | 20260507 08:03:22.703 | INFO | "gpu" : " 17 GB", | 20260507 08:03:22.703 | INFO | "huge" : "", | 20260507 08:03:22.703 | INFO | "huge_shared" : " 27 GB", | 20260507 08:03:22.703 | INFO | "malloc" : " 14 GB", | 20260507 08:03:22.703 | INFO | "pinned" : " 29 GB" | 20260507 08:03:22.703 | INFO | } | 20260507 08:03:22.703 | INFO | Perf report for the last 5.13263 seconds: | 20260507 08:03:22.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:03:22.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:22.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:22.703 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:22.703 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 08:03:22.703 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:22.703 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:22.703 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:03:22.703 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:22.703 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:22.704 | INFO | Uploaded 36GiB to GPU in 2.0281s (17.7506GiB/s) | 20260507 08:03:22.704 | INFO | Downloaded 34GiB to GPU in 1.92899s (17.6258GiB/s) | 20260507 08:03:27.840 | INFO | Analyzing chunk frames = [8379392,8379904) | 20260507 08:03:27.840 | INFO | MemInfo: { | 20260507 08:03:27.840 | INFO | "gpu" : " 17 GB", | 20260507 08:03:27.840 | INFO | "huge" : "", | 20260507 08:03:27.840 | INFO | "huge_shared" : " 27 GB", | 20260507 08:03:27.840 | INFO | "malloc" : " 14 GB", | 20260507 08:03:27.840 | INFO | "pinned" : " 29 GB" | 20260507 08:03:27.840 | INFO | } | 20260507 08:03:27.840 | INFO | Perf report for the last 5.13656 seconds: | 20260507 08:03:27.840 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:03:27.840 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:27.840 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:27.840 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:27.840 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 08:03:27.840 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:27.840 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:27.840 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 08:03:27.840 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:27.840 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:27.840 | INFO | Uploaded 36GiB to GPU in 1.95947s (18.3723GiB/s) | 20260507 08:03:27.840 | INFO | Downloaded 34GiB to GPU in 1.9716s (17.2448GiB/s) | 20260507 08:03:32.973 | INFO | Analyzing chunk frames = [8379904,8380416) | 20260507 08:03:32.973 | INFO | MemInfo: { | 20260507 08:03:32.973 | INFO | "gpu" : " 17 GB", | 20260507 08:03:32.973 | INFO | "huge" : "", | 20260507 08:03:32.973 | INFO | "huge_shared" : " 27 GB", | 20260507 08:03:32.973 | INFO | "malloc" : " 14 GB", | 20260507 08:03:32.973 | INFO | "pinned" : " 29 GB" | 20260507 08:03:32.973 | INFO | } | 20260507 08:03:32.973 | INFO | Perf report for the last 5.13293 seconds: | 20260507 08:03:32.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:03:32.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:32.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:32.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:32.973 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 08:03:32.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:32.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:32.973 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:03:32.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:32.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:32.973 | INFO | Uploaded 36GiB to GPU in 1.98585s (18.1282GiB/s) | 20260507 08:03:32.973 | INFO | Downloaded 34GiB to GPU in 1.96341s (17.3168GiB/s) | 20260507 08:03:32.973 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:03:32.973 | INFO | Total mem usage: 18 GiB | 20260507 08:03:32.973 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:03:32.973 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:03:32.973 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:03:32.973 | INFO | High Water Breakdown: | 20260507 08:03:32.973 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:03:32.973 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:03:32.973 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:03:32.973 | INFO | ------------------------------------------ | 20260507 08:03:32.973 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:03:32.973 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:03:32.973 | INFO | Total mem usage: 26.5 GiB | 20260507 08:03:32.973 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:03:32.973 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:03:32.973 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 08:03:32.973 | INFO | High Water Breakdown: | 20260507 08:03:32.973 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:03:32.973 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:03:32.973 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:03:32.973 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:03:32.973 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:03:32.973 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:03:32.973 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:03:32.973 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:03:32.973 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:03:32.973 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:03:32.974 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:03:32.974 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:03:32.974 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:03:32.974 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:03:32.974 | INFO | ------------------------------------------ | 20260507 08:03:32.974 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:03:32.974 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:03:32.974 | INFO | Total mem usage: 25.6 GiB | 20260507 08:03:32.974 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:03:32.974 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:03:32.974 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:03:32.974 | INFO | High Water Breakdown: | 20260507 08:03:32.974 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:03:32.974 | INFO | ------------------------------------------ | 20260507 08:03:32.974 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:03:32.974 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:03:32.974 | INFO | Total mem usage: 15.7 GiB | 20260507 08:03:32.974 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:03:32.974 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:03:32.974 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:03:32.974 | INFO | High Water Breakdown: | 20260507 08:03:32.974 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:03:32.974 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:03:32.974 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:03:32.974 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:03:32.974 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:03:32.974 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:03:32.974 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:03:32.974 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:03:32.974 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:03:32.974 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:03:32.974 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:03:32.974 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:03:32.974 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:03:32.974 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:03:32.974 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:03:32.974 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:03:32.974 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:03:32.974 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:03:32.974 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:03:32.974 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:03:32.974 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:03:32.974 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:03:32.974 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:03:32.974 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:03:32.974 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:03:32.974 | INFO | ------------------------------------------ | 20260507 08:03:32.974 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:03:32.974 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:03:32.974 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:03:32.974 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:03:32.974 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.50e+01, 1.34e+01 / 1.24e+00 | 20260507 08:03:32.974 | INFO | PerfReport: 0.29%, AggregateStats -- 9.01e-02 / 1.02e-01, 9.79e-03 / 3.99e-03 | 20260507 08:03:32.974 | INFO | PerfReport: 11.64%, Baseline -- 3.57e+00 / 4.50e+00, 4.97e-01 / 1.56e-01 | 20260507 08:03:32.974 | INFO | PerfReport: 3.21%, Histogram -- 9.86e-01 / 1.49e+00, 1.65e-01 / 1.01e-01 | 20260507 08:03:32.974 | INFO | PerfReport: 16.30%, DME -- 4.00e+01 / 4.50e+01, 3.90e+00 / 1.14e+00 | 20260507 08:03:32.974 | INFO | PerfReport: 57.77%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.30e+00 / 1.15e+00 | 20260507 08:03:32.974 | INFO | PerfReport: 6.88%, PulseAccumulating -- 2.11e+00 / 2.66e+00, 3.84e-01 / 1.62e-01 | 20260507 08:03:32.974 | INFO | PerfReport: 2.91%, Metrics -- 8.94e-01 / 1.19e+00, 2.43e-01 / 9.44e-02 | 20260507 08:03:32.974 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.59e-01, 5.09e-02 / 2.42e-02 | 20260507 08:03:32.974 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:03:32.974 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:03:32.974 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:03:32.974 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 08:03:32.974 | INFO | PerfReport: 21.96%, Upload -- 2.11e+01 / 0.00e+00, 7.33e+00 / 0.00e+00 | 20260507 08:03:32.974 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 08:03:32.974 | INFO | PerfReport: 31.90%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 08:03:32.974 | INFO | PerfReport: 1.63%, LockUpload -- 1.56e+00 / 0.00e+00, 4.98e+00 / 0.00e+00 | 20260507 08:03:32.974 | INFO | PerfReport: 1.16%, LockDownload -- 1.11e+00 / 0.00e+00, 2.73e+00 / 0.00e+00 | 20260507 08:03:32.974 | INFO | PerfReport: 21.92%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 08:03:32.974 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260507 08:03:33.069 | INFO | Starting a round of Baz IO | 20260507 08:03:36.160 | INFO | Populating RTMetrics report | 20260507 08:03:36.160 | INFO | RTMetrics report populated | 20260507 08:03:36.160 | INFO | Opening tmp RTMetrics json file | 20260507 08:03:36.177 | INFO | Serializing RTMetrics to json | 20260507 08:03:36.178 | INFO | Serialization done, swapping tmp file now... | 20260507 08:03:36.179 | INFO | RTMetrics done | 20260507 08:03:36.243 | INFO | Finished a round of Baz IO: Wrote 1.625 GiB in 3.173 seconds | 20260507 08:03:36.243 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.52% / 0.00% / 67.48% | 20260507 08:03:36.243 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0958522% / 3.97364e-06% / 7.94729e-05% | 20260507 08:03:36.243 | INFO | Baz IO overall: | 20260507 08:03:36.243 | INFO | BytesWritten : 4184591477525 | 20260507 08:03:36.243 | INFO | NumEvents : 2002354471204 | 20260507 08:03:38.112 | INFO | Analyzing chunk frames = [8380416,8380928) | 20260507 08:03:38.112 | INFO | MemInfo: { | 20260507 08:03:38.112 | INFO | "gpu" : " 17 GB", | 20260507 08:03:38.112 | INFO | "huge" : "", | 20260507 08:03:38.112 | INFO | "huge_shared" : " 27 GB", | 20260507 08:03:38.112 | INFO | "malloc" : " 14 GB", | 20260507 08:03:38.112 | INFO | "pinned" : " 29 GB" | 20260507 08:03:38.112 | INFO | } | 20260507 08:03:38.113 | INFO | Perf report for the last 5.13977 seconds: | 20260507 08:03:38.113 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:03:38.113 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:38.113 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:38.113 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:38.113 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 08:03:38.113 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:38.113 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 08:03:38.113 | INFO | PRE_HQ : 58.7%, 1.8, 54.3ms, 0 | 20260507 08:03:38.113 | INFO | BAZWRITER : 61.3%, 3.1, 97.5ms, 0 | 20260507 08:03:38.113 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:03:38.113 | INFO | Uploaded 36GiB to GPU in 2.03548s (17.6863GiB/s) | 20260507 08:03:38.113 | INFO | Downloaded 38GiB to GPU in 2.17726s (17.4532GiB/s) | 20260507 08:03:43.254 | INFO | Analyzing chunk frames = [8380928,8381440) | 20260507 08:03:43.254 | INFO | MemInfo: { | 20260507 08:03:43.254 | INFO | "gpu" : " 17 GB", | 20260507 08:03:43.254 | INFO | "huge" : "", | 20260507 08:03:43.254 | INFO | "huge_shared" : " 27 GB", | 20260507 08:03:43.254 | INFO | "malloc" : " 14 GB", | 20260507 08:03:43.254 | INFO | "pinned" : " 29 GB" | 20260507 08:03:43.254 | INFO | } | 20260507 08:03:43.254 | INFO | Perf report for the last 5.14197 seconds: | 20260507 08:03:43.254 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:03:43.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:43.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:43.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:43.255 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 08:03:43.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:43.255 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:43.255 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260507 08:03:43.255 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:43.255 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:43.255 | INFO | Uploaded 36GiB to GPU in 1.98294s (18.1549GiB/s) | 20260507 08:03:43.255 | INFO | Downloaded 34GiB to GPU in 1.93204s (17.598GiB/s) | 20260507 08:03:48.404 | INFO | Analyzing chunk frames = [8381440,8381952) | 20260507 08:03:48.404 | INFO | MemInfo: { | 20260507 08:03:48.404 | INFO | "gpu" : " 17 GB", | 20260507 08:03:48.404 | INFO | "huge" : "", | 20260507 08:03:48.404 | INFO | "huge_shared" : " 27 GB", | 20260507 08:03:48.404 | INFO | "malloc" : " 14 GB", | 20260507 08:03:48.404 | INFO | "pinned" : " 29 GB" | 20260507 08:03:48.404 | INFO | } | 20260507 08:03:48.404 | INFO | Perf report for the last 5.14988 seconds: | 20260507 08:03:48.404 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:03:48.404 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:48.404 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:48.404 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:48.404 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 08:03:48.404 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:48.404 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:48.405 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:03:48.405 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:48.405 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:48.405 | INFO | Uploaded 36GiB to GPU in 2.04635s (17.5923GiB/s) | 20260507 08:03:48.405 | INFO | Downloaded 34GiB to GPU in 1.95633s (17.3794GiB/s) | 20260507 08:03:53.545 | INFO | Analyzing chunk frames = [8381952,8382464) | 20260507 08:03:53.545 | INFO | MemInfo: { | 20260507 08:03:53.545 | INFO | "gpu" : " 17 GB", | 20260507 08:03:53.545 | INFO | "huge" : "", | 20260507 08:03:53.545 | INFO | "huge_shared" : " 27 GB", | 20260507 08:03:53.546 | INFO | "malloc" : " 14 GB", | 20260507 08:03:53.546 | INFO | "pinned" : " 29 GB" | 20260507 08:03:53.546 | INFO | } | 20260507 08:03:53.546 | INFO | Perf report for the last 5.14124 seconds: | 20260507 08:03:53.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:03:53.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:53.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:53.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:53.546 | INFO | ANALYSIS : 60.3%, 2.9, 93.2ms, 0 | 20260507 08:03:53.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:53.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:53.546 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:03:53.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:53.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:53.546 | INFO | Uploaded 36GiB to GPU in 2.02902s (17.7425GiB/s) | 20260507 08:03:53.546 | INFO | Downloaded 34GiB to GPU in 1.91115s (17.7904GiB/s) | 20260507 08:03:58.673 | INFO | Analyzing chunk frames = [8382464,8382976) | 20260507 08:03:58.673 | INFO | MemInfo: { | 20260507 08:03:58.673 | INFO | "gpu" : " 17 GB", | 20260507 08:03:58.673 | INFO | "huge" : "", | 20260507 08:03:58.674 | INFO | "huge_shared" : " 27 GB", | 20260507 08:03:58.674 | INFO | "malloc" : " 14 GB", | 20260507 08:03:58.674 | INFO | "pinned" : " 29 GB" | 20260507 08:03:58.674 | INFO | } | 20260507 08:03:58.674 | INFO | Perf report for the last 5.12808 seconds: | 20260507 08:03:58.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:03:58.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:58.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:58.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:58.674 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 08:03:58.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:58.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:58.674 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:03:58.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:58.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:03:58.674 | INFO | Uploaded 36GiB to GPU in 2.05238s (17.5406GiB/s) | 20260507 08:03:58.674 | INFO | Downloaded 34GiB to GPU in 1.91026s (17.7987GiB/s) | 20260507 08:04:03.810 | INFO | Analyzing chunk frames = [8382976,8383488) | 20260507 08:04:03.810 | INFO | MemInfo: { | 20260507 08:04:03.810 | INFO | "gpu" : " 17 GB", | 20260507 08:04:03.810 | INFO | "huge" : "", | 20260507 08:04:03.810 | INFO | "huge_shared" : " 27 GB", | 20260507 08:04:03.810 | INFO | "malloc" : " 14 GB", | 20260507 08:04:03.810 | INFO | "pinned" : " 29 GB" | 20260507 08:04:03.811 | INFO | } | 20260507 08:04:03.811 | INFO | Perf report for the last 5.13682 seconds: | 20260507 08:04:03.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:04:03.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:03.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:03.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:03.811 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 08:04:03.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:03.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:03.811 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 08:04:03.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:03.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:03.811 | INFO | Uploaded 36GiB to GPU in 2.04376s (17.6146GiB/s) | 20260507 08:04:03.811 | INFO | Downloaded 34GiB to GPU in 1.92237s (17.6865GiB/s) | 20260507 08:04:08.948 | INFO | Analyzing chunk frames = [8383488,8384000) | 20260507 08:04:08.948 | INFO | MemInfo: { | 20260507 08:04:08.948 | INFO | "gpu" : " 17 GB", | 20260507 08:04:08.948 | INFO | "huge" : "", | 20260507 08:04:08.948 | INFO | "huge_shared" : " 27 GB", | 20260507 08:04:08.948 | INFO | "malloc" : " 14 GB", | 20260507 08:04:08.948 | INFO | "pinned" : " 29 GB" | 20260507 08:04:08.948 | INFO | } | 20260507 08:04:08.948 | INFO | Perf report for the last 5.13782 seconds: | 20260507 08:04:08.948 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:04:08.948 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:08.948 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:08.948 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:08.948 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 08:04:08.948 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:08.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:08.949 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 08:04:08.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:08.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:08.949 | INFO | Uploaded 36GiB to GPU in 1.9679s (18.2936GiB/s) | 20260507 08:04:08.949 | INFO | Downloaded 34GiB to GPU in 1.98046s (17.1677GiB/s) | 20260507 08:04:14.089 | INFO | Analyzing chunk frames = [8384000,8384512) | 20260507 08:04:14.089 | INFO | MemInfo: { | 20260507 08:04:14.089 | INFO | "gpu" : " 17 GB", | 20260507 08:04:14.089 | INFO | "huge" : "", | 20260507 08:04:14.090 | INFO | "huge_shared" : " 27 GB", | 20260507 08:04:14.090 | INFO | "malloc" : " 14 GB", | 20260507 08:04:14.090 | INFO | "pinned" : " 29 GB" | 20260507 08:04:14.090 | INFO | } | 20260507 08:04:14.090 | INFO | Perf report for the last 5.14124 seconds: | 20260507 08:04:14.090 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:04:14.090 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:14.090 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:14.090 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:14.090 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:04:14.090 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:14.090 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:14.090 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:04:14.090 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:14.090 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:14.090 | INFO | Uploaded 36GiB to GPU in 1.98851s (18.104GiB/s) | 20260507 08:04:14.090 | INFO | Downloaded 34GiB to GPU in 1.95594s (17.383GiB/s) | 20260507 08:04:14.187 | INFO | Starting a round of Baz IO | 20260507 08:04:17.280 | INFO | Populating RTMetrics report | 20260507 08:04:17.280 | INFO | RTMetrics report populated | 20260507 08:04:17.280 | INFO | Opening tmp RTMetrics json file | 20260507 08:04:17.288 | INFO | Serializing RTMetrics to json | 20260507 08:04:17.297 | INFO | Serialization done, swapping tmp file now... | 20260507 08:04:17.297 | INFO | RTMetrics done | 20260507 08:04:17.363 | INFO | Finished a round of Baz IO: Wrote 1.623 GiB in 3.176 seconds | 20260507 08:04:17.363 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.52% / 0.00% / 67.48% | 20260507 08:04:17.363 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0950694% / 0% / 5.96046e-05% | 20260507 08:04:17.363 | INFO | Baz IO overall: | 20260507 08:04:17.364 | INFO | BytesWritten : 4186333868993 | 20260507 08:04:17.364 | INFO | NumEvents : 2003166876368 | 20260507 08:04:19.223 | INFO | Analyzing chunk frames = [8384512,8385024) | 20260507 08:04:19.223 | INFO | MemInfo: { | 20260507 08:04:19.223 | INFO | "gpu" : " 17 GB", | 20260507 08:04:19.223 | INFO | "huge" : "", | 20260507 08:04:19.223 | INFO | "huge_shared" : " 27 GB", | 20260507 08:04:19.223 | INFO | "malloc" : " 14 GB", | 20260507 08:04:19.223 | INFO | "pinned" : " 29 GB" | 20260507 08:04:19.223 | INFO | } | 20260507 08:04:19.223 | INFO | Perf report for the last 5.13314 seconds: | 20260507 08:04:19.223 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:04:19.223 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:19.223 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:19.223 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:19.223 | INFO | ANALYSIS : 61.6%, 3.0, 95.3ms, 0 | 20260507 08:04:19.223 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:19.223 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260507 08:04:19.223 | INFO | PRE_HQ : 58.8%, 1.8, 54.1ms, 0 | 20260507 08:04:19.223 | INFO | BAZWRITER : 61.5%, 3.8, 119.3ms, 0 | 20260507 08:04:19.223 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:04:19.223 | INFO | Uploaded 36GiB to GPU in 2.08013s (17.3066GiB/s) | 20260507 08:04:19.223 | INFO | Downloaded 38GiB to GPU in 2.17998s (17.4313GiB/s) | 20260507 08:04:24.368 | INFO | Analyzing chunk frames = [8385024,8385536) | 20260507 08:04:24.368 | INFO | MemInfo: { | 20260507 08:04:24.368 | INFO | "gpu" : " 17 GB", | 20260507 08:04:24.368 | INFO | "huge" : "", | 20260507 08:04:24.368 | INFO | "huge_shared" : " 27 GB", | 20260507 08:04:24.369 | INFO | "malloc" : " 14 GB", | 20260507 08:04:24.369 | INFO | "pinned" : " 29 GB" | 20260507 08:04:24.369 | INFO | } | 20260507 08:04:24.369 | INFO | Perf report for the last 5.14584 seconds: | 20260507 08:04:24.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:04:24.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:24.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:24.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:24.369 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 08:04:24.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:24.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:24.369 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 08:04:24.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:24.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:24.369 | INFO | Uploaded 36GiB to GPU in 1.9978s (18.0198GiB/s) | 20260507 08:04:24.369 | INFO | Downloaded 34GiB to GPU in 1.93149s (17.603GiB/s) | 20260507 08:04:29.517 | INFO | Analyzing chunk frames = [8385536,8386048) | 20260507 08:04:29.517 | INFO | MemInfo: { | 20260507 08:04:29.517 | INFO | "gpu" : " 17 GB", | 20260507 08:04:29.517 | INFO | "huge" : "", | 20260507 08:04:29.517 | INFO | "huge_shared" : " 27 GB", | 20260507 08:04:29.517 | INFO | "malloc" : " 14 GB", | 20260507 08:04:29.517 | INFO | "pinned" : " 29 GB" | 20260507 08:04:29.517 | INFO | } | 20260507 08:04:29.517 | INFO | Perf report for the last 5.14875 seconds: | 20260507 08:04:29.517 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:04:29.517 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:29.517 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:29.517 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:29.517 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 08:04:29.517 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:29.518 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:29.518 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:04:29.518 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:29.518 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:29.518 | INFO | Uploaded 36GiB to GPU in 2.03672s (17.6755GiB/s) | 20260507 08:04:29.518 | INFO | Downloaded 34GiB to GPU in 1.95123s (17.4249GiB/s) | 20260507 08:04:34.658 | INFO | Analyzing chunk frames = [8386048,8386560) | 20260507 08:04:34.658 | INFO | MemInfo: { | 20260507 08:04:34.658 | INFO | "gpu" : " 17 GB", | 20260507 08:04:34.658 | INFO | "huge" : "", | 20260507 08:04:34.658 | INFO | "huge_shared" : " 27 GB", | 20260507 08:04:34.658 | INFO | "malloc" : " 14 GB", | 20260507 08:04:34.659 | INFO | "pinned" : " 29 GB" | 20260507 08:04:34.659 | INFO | } | 20260507 08:04:34.659 | INFO | Perf report for the last 5.1412 seconds: | 20260507 08:04:34.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:04:34.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:34.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:34.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:34.659 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 08:04:34.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:34.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:34.659 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:04:34.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:34.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:34.659 | INFO | Uploaded 36GiB to GPU in 2.05733s (17.4984GiB/s) | 20260507 08:04:34.659 | INFO | Downloaded 34GiB to GPU in 1.95043s (17.432GiB/s) | 20260507 08:04:39.788 | INFO | Analyzing chunk frames = [8386560,8387072) | 20260507 08:04:39.788 | INFO | MemInfo: { | 20260507 08:04:39.788 | INFO | "gpu" : " 17 GB", | 20260507 08:04:39.788 | INFO | "huge" : "", | 20260507 08:04:39.788 | INFO | "huge_shared" : " 27 GB", | 20260507 08:04:39.788 | INFO | "malloc" : " 14 GB", | 20260507 08:04:39.788 | INFO | "pinned" : " 29 GB" | 20260507 08:04:39.788 | INFO | } | 20260507 08:04:39.788 | INFO | Perf report for the last 5.12916 seconds: | 20260507 08:04:39.788 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:04:39.788 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:39.788 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:39.788 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:39.788 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 08:04:39.788 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:39.788 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:39.788 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:04:39.788 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:39.788 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:39.788 | INFO | Uploaded 36GiB to GPU in 2.06722s (17.4147GiB/s) | 20260507 08:04:39.788 | INFO | Downloaded 34GiB to GPU in 1.90828s (17.8171GiB/s) | 20260507 08:04:44.927 | INFO | Analyzing chunk frames = [8387072,8387584) | 20260507 08:04:44.928 | INFO | MemInfo: { | 20260507 08:04:44.928 | INFO | "gpu" : " 17 GB", | 20260507 08:04:44.928 | INFO | "huge" : "", | 20260507 08:04:44.928 | INFO | "huge_shared" : " 27 GB", | 20260507 08:04:44.928 | INFO | "malloc" : " 14 GB", | 20260507 08:04:44.928 | INFO | "pinned" : " 29 GB" | 20260507 08:04:44.928 | INFO | } | 20260507 08:04:44.928 | INFO | Perf report for the last 5.13997 seconds: | 20260507 08:04:44.928 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:04:44.928 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:44.928 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:44.928 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:44.928 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 08:04:44.928 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:44.928 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:44.928 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:04:44.928 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:44.928 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:44.928 | INFO | Uploaded 36GiB to GPU in 2.06697s (17.4168GiB/s) | 20260507 08:04:44.928 | INFO | Downloaded 34GiB to GPU in 1.92036s (17.705GiB/s) | 20260507 08:04:50.067 | INFO | Analyzing chunk frames = [8387584,8388096) | 20260507 08:04:50.067 | INFO | MemInfo: { | 20260507 08:04:50.067 | INFO | "gpu" : " 17 GB", | 20260507 08:04:50.067 | INFO | "huge" : "", | 20260507 08:04:50.067 | INFO | "huge_shared" : " 27 GB", | 20260507 08:04:50.067 | INFO | "malloc" : " 14 GB", | 20260507 08:04:50.067 | INFO | "pinned" : " 29 GB" | 20260507 08:04:50.067 | INFO | } | 20260507 08:04:50.067 | INFO | Perf report for the last 5.1393 seconds: | 20260507 08:04:50.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:04:50.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:50.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:50.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:50.067 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 08:04:50.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:50.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:50.067 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 08:04:50.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:50.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:50.067 | INFO | Uploaded 36GiB to GPU in 1.95812s (18.385GiB/s) | 20260507 08:04:50.067 | INFO | Downloaded 34GiB to GPU in 1.97701s (17.1977GiB/s) | 20260507 08:04:55.205 | INFO | Analyzing chunk frames = [8388096,8388608) | 20260507 08:04:55.205 | INFO | MemInfo: { | 20260507 08:04:55.205 | INFO | "gpu" : " 17 GB", | 20260507 08:04:55.205 | INFO | "huge" : "", | 20260507 08:04:55.205 | INFO | "huge_shared" : " 27 GB", | 20260507 08:04:55.205 | INFO | "malloc" : " 14 GB", | 20260507 08:04:55.205 | INFO | "pinned" : " 29 GB" | 20260507 08:04:55.205 | INFO | } | 20260507 08:04:55.205 | INFO | Perf report for the last 5.13804 seconds: | 20260507 08:04:55.205 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:04:55.205 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:55.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:55.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:55.205 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:04:55.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:55.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:55.205 | INFO | PRE_HQ : 28.6%, 1.0, 14.8ms, 0 | 20260507 08:04:55.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:55.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:04:55.205 | INFO | Uploaded 36GiB to GPU in 1.97789s (18.2012GiB/s) | 20260507 08:04:55.205 | INFO | Downloaded 34GiB to GPU in 1.96457s (17.3066GiB/s) | 20260507 08:04:55.206 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:04:55.206 | INFO | Total mem usage: 18 GiB | 20260507 08:04:55.206 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:04:55.206 | INFO | Recent High Water Marks Sum: 15.6 GiB (Savings: -15.1%) | 20260507 08:04:55.206 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:04:55.206 | INFO | High Water Breakdown: | 20260507 08:04:55.206 | INFO | 61.6% : 9.64 GiB ( 84.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:04:55.206 | INFO | 38.2% : 5.98 GiB ( 89.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:04:55.206 | INFO | 0.127% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:04:55.206 | INFO | ------------------------------------------ | 20260507 08:04:55.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:04:55.206 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:04:55.206 | INFO | Total mem usage: 26.5 GiB | 20260507 08:04:55.206 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:04:55.206 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:04:55.206 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 08:04:55.206 | INFO | High Water Breakdown: | 20260507 08:04:55.206 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:04:55.206 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:04:55.206 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:04:55.206 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:04:55.206 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:04:55.206 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:04:55.206 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:04:55.206 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:04:55.206 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:04:55.206 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:04:55.206 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:04:55.206 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:04:55.206 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:04:55.206 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:04:55.206 | INFO | ------------------------------------------ | 20260507 08:04:55.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:04:55.206 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:04:55.206 | INFO | Total mem usage: 25.6 GiB | 20260507 08:04:55.206 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:04:55.206 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:04:55.206 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:04:55.206 | INFO | High Water Breakdown: | 20260507 08:04:55.206 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:04:55.206 | INFO | ------------------------------------------ | 20260507 08:04:55.206 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:04:55.206 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:04:55.206 | INFO | Total mem usage: 15.7 GiB | 20260507 08:04:55.206 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:04:55.206 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:04:55.206 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:04:55.206 | INFO | High Water Breakdown: | 20260507 08:04:55.206 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:04:55.206 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:04:55.206 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:04:55.206 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:04:55.206 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:04:55.207 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:04:55.207 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:04:55.207 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:04:55.207 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:04:55.207 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:04:55.207 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:04:55.207 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:04:55.207 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:04:55.207 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:04:55.207 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:04:55.207 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:04:55.207 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:04:55.207 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:04:55.207 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:04:55.207 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:04:55.207 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:04:55.207 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:04:55.207 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:04:55.207 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:04:55.207 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:04:55.207 | INFO | ------------------------------------------ | 20260507 08:04:55.207 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:04:55.207 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:04:55.207 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:04:55.207 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:04:55.207 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.34e+01 / 1.24e+00 | 20260507 08:04:55.207 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.03e-01, 9.68e-03 / 4.39e-03 | 20260507 08:04:55.207 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.45e+00, 5.07e-01 / 2.78e-01 | 20260507 08:04:55.207 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.48e+00, 1.66e-01 / 1.14e-01 | 20260507 08:04:55.207 | INFO | PerfReport: 16.30%, DME -- 4.00e+01 / 4.49e+01, 3.94e+00 / 1.46e+00 | 20260507 08:04:55.207 | INFO | PerfReport: 57.75%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.33e+00 / 1.13e+00 | 20260507 08:04:55.207 | INFO | PerfReport: 6.88%, PulseAccumulating -- 2.11e+00 / 2.67e+00, 3.83e-01 / 2.07e-01 | 20260507 08:04:55.207 | INFO | PerfReport: 2.91%, Metrics -- 8.93e-01 / 1.18e+00, 2.38e-01 / 9.28e-02 | 20260507 08:04:55.207 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.58e-01, 5.06e-02 / 1.92e-02 | 20260507 08:04:55.207 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:04:55.207 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:04:55.207 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:04:55.207 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 08:04:55.207 | INFO | PerfReport: 21.94%, Upload -- 2.11e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260507 08:04:55.207 | INFO | PerfReport: 21.36%, Download -- 2.05e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260507 08:04:55.207 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 08:04:55.207 | INFO | PerfReport: 1.60%, LockUpload -- 1.54e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 08:04:55.207 | INFO | PerfReport: 1.25%, LockDownload -- 1.20e+00 / 0.00e+00, 2.95e+00 / 0.00e+00 | 20260507 08:04:55.207 | INFO | PerfReport: 21.90%, LockCompute -- 2.10e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 08:04:55.207 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.74e-02 / 0.00e+00 | 20260507 08:04:55.302 | INFO | Starting a round of Baz IO | 20260507 08:04:58.389 | INFO | Populating RTMetrics report | 20260507 08:04:58.389 | INFO | RTMetrics report populated | 20260507 08:04:58.389 | INFO | Opening tmp RTMetrics json file | 20260507 08:04:58.406 | INFO | Serializing RTMetrics to json | 20260507 08:04:58.407 | INFO | Serialization done, swapping tmp file now... | 20260507 08:04:58.408 | INFO | RTMetrics done | 20260507 08:04:58.471 | INFO | Finished a round of Baz IO: Wrote 1.624 GiB in 3.168 seconds | 20260507 08:04:58.471 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.52% / 0.00% / 67.48% | 20260507 08:04:58.471 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0952323% / 0% / 5.16574e-05% | 20260507 08:04:58.471 | INFO | Baz IO overall: | 20260507 08:04:58.471 | INFO | BytesWritten : 4188077990441 | 20260507 08:04:58.471 | INFO | NumEvents : 2003980609517 | 20260507 08:05:00.353 | INFO | Analyzing chunk frames = [8388608,8389120) | 20260507 08:05:00.353 | INFO | MemInfo: { | 20260507 08:05:00.353 | INFO | "gpu" : " 17 GB", | 20260507 08:05:00.353 | INFO | "huge" : "", | 20260507 08:05:00.353 | INFO | "huge_shared" : " 27 GB", | 20260507 08:05:00.353 | INFO | "malloc" : " 14 GB", | 20260507 08:05:00.353 | INFO | "pinned" : " 29 GB" | 20260507 08:05:00.353 | INFO | } | 20260507 08:05:00.353 | INFO | Perf report for the last 5.14814 seconds: | 20260507 08:05:00.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:05:00.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:00.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:00.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:00.353 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260507 08:05:00.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:00.354 | INFO | RT_METRICS : 42.8%, 1.0, 22.5ms, 0 | 20260507 08:05:00.354 | INFO | PRE_HQ : 59.1%, 1.9, 57.8ms, 0 | 20260507 08:05:00.354 | INFO | BAZWRITER : 61.1%, 3.2, 100.3ms, 0 | 20260507 08:05:00.354 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:05:00.354 | INFO | Uploaded 36GiB to GPU in 2.04269s (17.6238GiB/s) | 20260507 08:05:00.354 | INFO | Downloaded 38GiB to GPU in 2.17583s (17.4646GiB/s) | 20260507 08:05:05.486 | INFO | Analyzing chunk frames = [8389120,8389632) | 20260507 08:05:05.486 | INFO | MemInfo: { | 20260507 08:05:05.486 | INFO | "gpu" : " 17 GB", | 20260507 08:05:05.486 | INFO | "huge" : "", | 20260507 08:05:05.486 | INFO | "huge_shared" : " 27 GB", | 20260507 08:05:05.486 | INFO | "malloc" : " 14 GB", | 20260507 08:05:05.486 | INFO | "pinned" : " 29 GB" | 20260507 08:05:05.486 | INFO | } | 20260507 08:05:05.486 | INFO | Perf report for the last 5.13251 seconds: | 20260507 08:05:05.486 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:05:05.486 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:05.486 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:05.486 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:05.486 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 08:05:05.486 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:05.486 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:05.486 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 08:05:05.486 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:05.486 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:05.486 | INFO | Uploaded 36GiB to GPU in 2.0342s (17.6973GiB/s) | 20260507 08:05:05.486 | INFO | Downloaded 34GiB to GPU in 1.95803s (17.3644GiB/s) | 20260507 08:05:10.624 | INFO | Analyzing chunk frames = [8389632,8390144) | 20260507 08:05:10.624 | INFO | MemInfo: { | 20260507 08:05:10.624 | INFO | "gpu" : " 17 GB", | 20260507 08:05:10.624 | INFO | "huge" : "", | 20260507 08:05:10.624 | INFO | "huge_shared" : " 27 GB", | 20260507 08:05:10.624 | INFO | "malloc" : " 14 GB", | 20260507 08:05:10.624 | INFO | "pinned" : " 29 GB" | 20260507 08:05:10.624 | INFO | } | 20260507 08:05:10.624 | INFO | Perf report for the last 5.1384 seconds: | 20260507 08:05:10.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:05:10.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:10.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:10.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:10.624 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 08:05:10.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:10.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:10.624 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:05:10.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:10.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:10.624 | INFO | Uploaded 36GiB to GPU in 2.05098s (17.5526GiB/s) | 20260507 08:05:10.625 | INFO | Downloaded 34GiB to GPU in 1.9525s (17.4135GiB/s) | 20260507 08:05:15.765 | INFO | Analyzing chunk frames = [8390144,8390656) | 20260507 08:05:15.765 | INFO | MemInfo: { | 20260507 08:05:15.765 | INFO | "gpu" : " 17 GB", | 20260507 08:05:15.765 | INFO | "huge" : "", | 20260507 08:05:15.765 | INFO | "huge_shared" : " 27 GB", | 20260507 08:05:15.765 | INFO | "malloc" : " 14 GB", | 20260507 08:05:15.766 | INFO | "pinned" : " 29 GB" | 20260507 08:05:15.766 | INFO | } | 20260507 08:05:15.766 | INFO | Perf report for the last 5.14133 seconds: | 20260507 08:05:15.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:05:15.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:15.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:15.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:15.766 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 08:05:15.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:15.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:15.766 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:05:15.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:15.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:15.766 | INFO | Uploaded 36GiB to GPU in 2.01168s (17.8955GiB/s) | 20260507 08:05:15.766 | INFO | Downloaded 34GiB to GPU in 1.94754s (17.4579GiB/s) | 20260507 08:05:20.900 | INFO | Analyzing chunk frames = [8390656,8391168) | 20260507 08:05:20.900 | INFO | MemInfo: { | 20260507 08:05:20.900 | INFO | "gpu" : " 17 GB", | 20260507 08:05:20.900 | INFO | "huge" : "", | 20260507 08:05:20.900 | INFO | "huge_shared" : " 27 GB", | 20260507 08:05:20.900 | INFO | "malloc" : " 14 GB", | 20260507 08:05:20.900 | INFO | "pinned" : " 29 GB" | 20260507 08:05:20.900 | INFO | } | 20260507 08:05:20.901 | INFO | Perf report for the last 5.13495 seconds: | 20260507 08:05:20.901 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:05:20.901 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:20.901 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:20.901 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:20.901 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 08:05:20.901 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:20.901 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:20.901 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:05:20.901 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:20.901 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:20.901 | INFO | Uploaded 36GiB to GPU in 2.01702s (17.8481GiB/s) | 20260507 08:05:20.901 | INFO | Downloaded 34GiB to GPU in 1.91847s (17.7224GiB/s) | 20260507 08:05:26.041 | INFO | Analyzing chunk frames = [8391168,8391680) | 20260507 08:05:26.041 | INFO | MemInfo: { | 20260507 08:05:26.041 | INFO | "gpu" : " 17 GB", | 20260507 08:05:26.041 | INFO | "huge" : "", | 20260507 08:05:26.041 | INFO | "huge_shared" : " 27 GB", | 20260507 08:05:26.041 | INFO | "malloc" : " 14 GB", | 20260507 08:05:26.041 | INFO | "pinned" : " 29 GB" | 20260507 08:05:26.041 | INFO | } | 20260507 08:05:26.041 | INFO | Perf report for the last 5.14037 seconds: | 20260507 08:05:26.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:05:26.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:26.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:26.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:26.041 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 08:05:26.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:26.041 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:26.041 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:05:26.041 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:26.041 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:26.041 | INFO | Uploaded 36GiB to GPU in 2.06559s (17.4284GiB/s) | 20260507 08:05:26.041 | INFO | Downloaded 34GiB to GPU in 1.93402s (17.58GiB/s) | 20260507 08:05:31.174 | INFO | Analyzing chunk frames = [8391680,8392192) | 20260507 08:05:31.174 | INFO | MemInfo: { | 20260507 08:05:31.174 | INFO | "gpu" : " 17 GB", | 20260507 08:05:31.174 | INFO | "huge" : "", | 20260507 08:05:31.174 | INFO | "huge_shared" : " 27 GB", | 20260507 08:05:31.174 | INFO | "malloc" : " 14 GB", | 20260507 08:05:31.174 | INFO | "pinned" : " 29 GB" | 20260507 08:05:31.174 | INFO | } | 20260507 08:05:31.174 | INFO | Perf report for the last 5.13348 seconds: | 20260507 08:05:31.174 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:05:31.174 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:31.174 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:31.174 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:31.174 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 08:05:31.174 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:31.174 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:31.174 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 08:05:31.174 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:31.174 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:31.175 | INFO | Uploaded 36GiB to GPU in 1.93009s (18.652GiB/s) | 20260507 08:05:31.175 | INFO | Downloaded 34GiB to GPU in 1.9767s (17.2004GiB/s) | 20260507 08:05:36.321 | INFO | Analyzing chunk frames = [8392192,8392704) | 20260507 08:05:36.321 | INFO | MemInfo: { | 20260507 08:05:36.321 | INFO | "gpu" : " 17 GB", | 20260507 08:05:36.321 | INFO | "huge" : "", | 20260507 08:05:36.321 | INFO | "huge_shared" : " 27 GB", | 20260507 08:05:36.321 | INFO | "malloc" : " 14 GB", | 20260507 08:05:36.321 | INFO | "pinned" : " 29 GB" | 20260507 08:05:36.321 | INFO | } | 20260507 08:05:36.321 | INFO | Perf report for the last 5.14668 seconds: | 20260507 08:05:36.321 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:05:36.321 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:36.321 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:36.321 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:36.321 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 08:05:36.321 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:36.321 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:36.321 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 08:05:36.321 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:36.321 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:36.321 | INFO | Uploaded 36GiB to GPU in 1.94994s (18.4621GiB/s) | 20260507 08:05:36.321 | INFO | Downloaded 34GiB to GPU in 1.96175s (17.3315GiB/s) | 20260507 08:05:36.418 | INFO | Starting a round of Baz IO | 20260507 08:05:39.510 | INFO | Populating RTMetrics report | 20260507 08:05:39.510 | INFO | RTMetrics report populated | 20260507 08:05:39.510 | INFO | Opening tmp RTMetrics json file | 20260507 08:05:39.518 | INFO | Serializing RTMetrics to json | 20260507 08:05:39.526 | INFO | Serialization done, swapping tmp file now... | 20260507 08:05:39.526 | INFO | RTMetrics done | 20260507 08:05:39.594 | INFO | Finished a round of Baz IO: Wrote 1.623 GiB in 3.175 seconds | 20260507 08:05:39.594 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.52% / 0.00% / 67.48% | 20260507 08:05:39.594 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0951648% / 0% / 5.5631e-05% | 20260507 08:05:39.594 | INFO | Baz IO overall: | 20260507 08:05:39.594 | INFO | BytesWritten : 4189820550546 | 20260507 08:05:39.594 | INFO | NumEvents : 2004792982507 | 20260507 08:05:41.465 | INFO | Analyzing chunk frames = [8392704,8393216) | 20260507 08:05:41.465 | INFO | MemInfo: { | 20260507 08:05:41.465 | INFO | "gpu" : " 17 GB", | 20260507 08:05:41.465 | INFO | "huge" : "", | 20260507 08:05:41.465 | INFO | "huge_shared" : " 27 GB", | 20260507 08:05:41.466 | INFO | "malloc" : " 14 GB", | 20260507 08:05:41.466 | INFO | "pinned" : " 29 GB" | 20260507 08:05:41.466 | INFO | } | 20260507 08:05:41.466 | INFO | Perf report for the last 5.14449 seconds: | 20260507 08:05:41.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:05:41.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:41.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:41.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:41.466 | INFO | ANALYSIS : 61.5%, 3.0, 94.8ms, 0 | 20260507 08:05:41.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:41.466 | INFO | RT_METRICS : 40.9%, 1.0, 21.2ms, 0 | 20260507 08:05:41.466 | INFO | PRE_HQ : 58.8%, 1.8, 53.8ms, 0 | 20260507 08:05:41.466 | INFO | BAZWRITER : 61.2%, 3.1, 95.6ms, 0 | 20260507 08:05:41.466 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:05:41.466 | INFO | Uploaded 36GiB to GPU in 2.05546s (17.5143GiB/s) | 20260507 08:05:41.466 | INFO | Downloaded 38GiB to GPU in 2.18109s (17.4225GiB/s) | 20260507 08:05:46.606 | INFO | Analyzing chunk frames = [8393216,8393728) | 20260507 08:05:46.606 | INFO | MemInfo: { | 20260507 08:05:46.606 | INFO | "gpu" : " 17 GB", | 20260507 08:05:46.606 | INFO | "huge" : "", | 20260507 08:05:46.606 | INFO | "huge_shared" : " 27 GB", | 20260507 08:05:46.606 | INFO | "malloc" : " 14 GB", | 20260507 08:05:46.606 | INFO | "pinned" : " 29 GB" | 20260507 08:05:46.606 | INFO | } | 20260507 08:05:46.606 | INFO | Perf report for the last 5.14043 seconds: | 20260507 08:05:46.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:05:46.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:46.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:46.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:46.606 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:05:46.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:46.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:46.606 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:05:46.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:46.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:46.606 | INFO | Uploaded 36GiB to GPU in 2.00726s (17.9349GiB/s) | 20260507 08:05:46.606 | INFO | Downloaded 34GiB to GPU in 1.94504s (17.4803GiB/s) | 20260507 08:05:51.736 | INFO | Analyzing chunk frames = [8393728,8394240) | 20260507 08:05:51.736 | INFO | MemInfo: { | 20260507 08:05:51.736 | INFO | "gpu" : " 17 GB", | 20260507 08:05:51.736 | INFO | "huge" : "", | 20260507 08:05:51.736 | INFO | "huge_shared" : " 27 GB", | 20260507 08:05:51.737 | INFO | "malloc" : " 14 GB", | 20260507 08:05:51.737 | INFO | "pinned" : " 29 GB" | 20260507 08:05:51.737 | INFO | } | 20260507 08:05:51.737 | INFO | Perf report for the last 5.13055 seconds: | 20260507 08:05:51.737 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:05:51.737 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:51.737 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:51.737 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:51.737 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 08:05:51.737 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:51.737 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:51.737 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:05:51.737 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:51.737 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:51.737 | INFO | Uploaded 36GiB to GPU in 2.02295s (17.7958GiB/s) | 20260507 08:05:51.737 | INFO | Downloaded 34GiB to GPU in 1.92027s (17.7059GiB/s) | 20260507 08:05:56.878 | INFO | Analyzing chunk frames = [8394240,8394752) | 20260507 08:05:56.878 | INFO | MemInfo: { | 20260507 08:05:56.878 | INFO | "gpu" : " 17 GB", | 20260507 08:05:56.878 | INFO | "huge" : "", | 20260507 08:05:56.878 | INFO | "huge_shared" : " 27 GB", | 20260507 08:05:56.878 | INFO | "malloc" : " 14 GB", | 20260507 08:05:56.878 | INFO | "pinned" : " 29 GB" | 20260507 08:05:56.878 | INFO | } | 20260507 08:05:56.878 | INFO | Perf report for the last 5.14156 seconds: | 20260507 08:05:56.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:05:56.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:56.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:56.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:56.878 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 08:05:56.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:56.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:56.878 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:05:56.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:56.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:05:56.878 | INFO | Uploaded 36GiB to GPU in 2.00641s (17.9425GiB/s) | 20260507 08:05:56.878 | INFO | Downloaded 34GiB to GPU in 1.94525s (17.4785GiB/s) | 20260507 08:06:02.019 | INFO | Analyzing chunk frames = [8394752,8395264) | 20260507 08:06:02.019 | INFO | MemInfo: { | 20260507 08:06:02.019 | INFO | "gpu" : " 17 GB", | 20260507 08:06:02.019 | INFO | "huge" : "", | 20260507 08:06:02.019 | INFO | "huge_shared" : " 27 GB", | 20260507 08:06:02.019 | INFO | "malloc" : " 14 GB", | 20260507 08:06:02.019 | INFO | "pinned" : " 29 GB" | 20260507 08:06:02.019 | INFO | } | 20260507 08:06:02.019 | INFO | Perf report for the last 5.14041 seconds: | 20260507 08:06:02.019 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:06:02.019 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:02.019 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:02.019 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:02.019 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 08:06:02.019 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:02.019 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:02.019 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:06:02.019 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:02.019 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:02.019 | INFO | Uploaded 36GiB to GPU in 2.03474s (17.6927GiB/s) | 20260507 08:06:02.019 | INFO | Downloaded 34GiB to GPU in 1.92137s (17.6957GiB/s) | 20260507 08:06:07.161 | INFO | Analyzing chunk frames = [8395264,8395776) | 20260507 08:06:07.161 | INFO | MemInfo: { | 20260507 08:06:07.161 | INFO | "gpu" : " 17 GB", | 20260507 08:06:07.161 | INFO | "huge" : "", | 20260507 08:06:07.161 | INFO | "huge_shared" : " 27 GB", | 20260507 08:06:07.161 | INFO | "malloc" : " 14 GB", | 20260507 08:06:07.161 | INFO | "pinned" : " 29 GB" | 20260507 08:06:07.161 | INFO | } | 20260507 08:06:07.161 | INFO | Perf report for the last 5.14282 seconds: | 20260507 08:06:07.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:06:07.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:07.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:07.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:07.162 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 08:06:07.162 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:07.162 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:07.162 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:06:07.162 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:07.162 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:07.162 | INFO | Uploaded 36GiB to GPU in 2.06233s (17.4559GiB/s) | 20260507 08:06:07.162 | INFO | Downloaded 34GiB to GPU in 1.92786s (17.6361GiB/s) | 20260507 08:06:12.299 | INFO | Analyzing chunk frames = [8395776,8396288) | 20260507 08:06:12.299 | INFO | MemInfo: { | 20260507 08:06:12.299 | INFO | "gpu" : " 17 GB", | 20260507 08:06:12.299 | INFO | "huge" : "", | 20260507 08:06:12.299 | INFO | "huge_shared" : " 27 GB", | 20260507 08:06:12.299 | INFO | "malloc" : " 14 GB", | 20260507 08:06:12.299 | INFO | "pinned" : " 29 GB" | 20260507 08:06:12.299 | INFO | } | 20260507 08:06:12.299 | INFO | Perf report for the last 5.13748 seconds: | 20260507 08:06:12.299 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:06:12.299 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:12.299 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:12.299 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:12.299 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 08:06:12.299 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:12.299 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:12.299 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:06:12.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:12.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:12.299 | INFO | Uploaded 36GiB to GPU in 1.96021s (18.3654GiB/s) | 20260507 08:06:12.299 | INFO | Downloaded 34GiB to GPU in 1.96594s (17.2945GiB/s) | 20260507 08:06:17.429 | INFO | Analyzing chunk frames = [8396288,8396800) | 20260507 08:06:17.429 | INFO | MemInfo: { | 20260507 08:06:17.429 | INFO | "gpu" : " 17 GB", | 20260507 08:06:17.429 | INFO | "huge" : "", | 20260507 08:06:17.429 | INFO | "huge_shared" : " 27 GB", | 20260507 08:06:17.429 | INFO | "malloc" : " 14 GB", | 20260507 08:06:17.429 | INFO | "pinned" : " 29 GB" | 20260507 08:06:17.429 | INFO | } | 20260507 08:06:17.429 | INFO | Perf report for the last 5.12968 seconds: | 20260507 08:06:17.429 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:06:17.429 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:17.429 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:17.429 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:17.429 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 08:06:17.429 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:17.429 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:17.429 | INFO | PRE_HQ : 29.0%, 1.0, 15.0ms, 0 | 20260507 08:06:17.429 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:17.429 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:17.429 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:06:17.429 | INFO | Total mem usage: 18 GiB | 20260507 08:06:17.429 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:06:17.429 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:06:17.429 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:06:17.429 | INFO | High Water Breakdown: | 20260507 08:06:17.429 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:06:17.429 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:06:17.429 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:06:17.429 | INFO | ------------------------------------------ | 20260507 08:06:17.429 | INFO | Uploaded 36GiB to GPU in 1.98949s (18.0951GiB/s) | 20260507 08:06:17.429 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:06:17.430 | INFO | Downloaded 34GiB to GPU in 1.95736s (17.3703GiB/s) | 20260507 08:06:17.430 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:06:17.430 | INFO | Total mem usage: 26.5 GiB | 20260507 08:06:17.430 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:06:17.430 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:06:17.430 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:06:17.430 | INFO | High Water Breakdown: | 20260507 08:06:17.430 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:06:17.430 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:06:17.430 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:06:17.430 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:06:17.430 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:06:17.430 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:06:17.430 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:06:17.430 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:06:17.430 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:06:17.430 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:06:17.430 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:06:17.430 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:06:17.430 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:06:17.430 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:06:17.430 | INFO | ------------------------------------------ | 20260507 08:06:17.430 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:06:17.430 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:06:17.430 | INFO | Total mem usage: 25.6 GiB | 20260507 08:06:17.430 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:06:17.430 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:06:17.430 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:06:17.430 | INFO | High Water Breakdown: | 20260507 08:06:17.430 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:06:17.430 | INFO | ------------------------------------------ | 20260507 08:06:17.430 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:06:17.431 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:06:17.431 | INFO | Total mem usage: 15.7 GiB | 20260507 08:06:17.431 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:06:17.431 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:06:17.431 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:06:17.431 | INFO | High Water Breakdown: | 20260507 08:06:17.431 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:06:17.431 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:06:17.431 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:06:17.431 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:06:17.431 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:06:17.431 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:06:17.431 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:06:17.431 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:06:17.431 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:06:17.431 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:06:17.431 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:06:17.431 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:06:17.431 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:06:17.431 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:06:17.431 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:06:17.431 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:06:17.431 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:06:17.431 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:06:17.431 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:06:17.431 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:06:17.431 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:06:17.431 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:06:17.431 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:06:17.431 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:06:17.431 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:06:17.431 | INFO | ------------------------------------------ | 20260507 08:06:17.431 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:06:17.431 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:06:17.431 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:06:17.431 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:06:17.431 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.44e+01, 1.32e+01 / 1.04e+00 | 20260507 08:06:17.431 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.02e-01, 9.88e-03 / 5.42e-03 | 20260507 08:06:17.431 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.46e+00, 5.09e-01 / 2.13e-01 | 20260507 08:06:17.431 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.49e+00, 1.70e-01 / 1.07e-01 | 20260507 08:06:17.431 | INFO | PerfReport: 16.23%, DME -- 3.98e+01 / 4.44e+01, 3.79e+00 / 1.26e+00 | 20260507 08:06:17.431 | INFO | PerfReport: 57.78%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.31e+00 / 8.00e-01 | 20260507 08:06:17.431 | INFO | PerfReport: 6.88%, PulseAccumulating -- 2.11e+00 / 2.68e+00, 3.83e-01 / 1.67e-01 | 20260507 08:06:17.431 | INFO | PerfReport: 2.92%, Metrics -- 8.96e-01 / 1.21e+00, 2.44e-01 / 9.94e-02 | 20260507 08:06:17.431 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.58e-01, 5.08e-02 / 1.63e-02 | 20260507 08:06:17.431 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:06:17.431 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:06:17.431 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:06:17.431 | INFO | PerfReport: Full Execution -- 9.60e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 08:06:17.431 | INFO | PerfReport: 21.85%, Upload -- 2.10e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260507 08:06:17.431 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260507 08:06:17.431 | INFO | PerfReport: 31.95%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 08:06:17.431 | INFO | PerfReport: 1.45%, LockUpload -- 1.40e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 08:06:17.431 | INFO | PerfReport: 1.31%, LockDownload -- 1.26e+00 / 0.00e+00, 3.07e+00 / 0.00e+00 | 20260507 08:06:17.431 | INFO | PerfReport: 22.00%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 08:06:17.431 | INFO | PerfReport: 0.02%, Misc -- 1.53e-02 / 0.00e+00, 2.64e-02 / 0.00e+00 | 20260507 08:06:17.526 | INFO | Starting a round of Baz IO | 20260507 08:06:20.622 | INFO | Populating RTMetrics report | 20260507 08:06:20.622 | INFO | RTMetrics report populated | 20260507 08:06:20.622 | INFO | Opening tmp RTMetrics json file | 20260507 08:06:20.637 | INFO | Serializing RTMetrics to json | 20260507 08:06:20.639 | INFO | Serialization done, swapping tmp file now... | 20260507 08:06:20.639 | INFO | RTMetrics done | 20260507 08:06:20.704 | INFO | Finished a round of Baz IO: Wrote 1.623 GiB in 3.178 seconds | 20260507 08:06:20.704 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.52% / 0.00% / 67.48% | 20260507 08:06:20.704 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.094827% / 0% / 5.96046e-05% | 20260507 08:06:20.704 | INFO | Baz IO overall: | 20260507 08:06:20.704 | INFO | BytesWritten : 4191562725301 | 20260507 08:06:20.705 | INFO | NumEvents : 2005605147512 | 20260507 08:06:22.576 | INFO | Analyzing chunk frames = [8396800,8397312) | 20260507 08:06:22.576 | INFO | MemInfo: { | 20260507 08:06:22.576 | INFO | "gpu" : " 17 GB", | 20260507 08:06:22.576 | INFO | "huge" : "", | 20260507 08:06:22.576 | INFO | "huge_shared" : " 27 GB", | 20260507 08:06:22.576 | INFO | "malloc" : " 14 GB", | 20260507 08:06:22.576 | INFO | "pinned" : " 29 GB" | 20260507 08:06:22.576 | INFO | } | 20260507 08:06:22.576 | INFO | Perf report for the last 5.14722 seconds: | 20260507 08:06:22.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:06:22.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:22.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:22.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:22.576 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260507 08:06:22.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:22.576 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 08:06:22.576 | INFO | PRE_HQ : 58.8%, 1.8, 54.6ms, 0 | 20260507 08:06:22.576 | INFO | BAZWRITER : 61.4%, 3.0, 95.8ms, 0 | 20260507 08:06:22.576 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:06:22.576 | INFO | Uploaded 36GiB to GPU in 2.04656s (17.5905GiB/s) | 20260507 08:06:22.576 | INFO | Downloaded 38GiB to GPU in 2.18165s (17.418GiB/s) | 20260507 08:06:27.715 | INFO | Analyzing chunk frames = [8397312,8397824) | 20260507 08:06:27.715 | INFO | MemInfo: { | 20260507 08:06:27.715 | INFO | "gpu" : " 17 GB", | 20260507 08:06:27.715 | INFO | "huge" : "", | 20260507 08:06:27.715 | INFO | "huge_shared" : " 27 GB", | 20260507 08:06:27.715 | INFO | "malloc" : " 14 GB", | 20260507 08:06:27.715 | INFO | "pinned" : " 29 GB" | 20260507 08:06:27.716 | INFO | } | 20260507 08:06:27.716 | INFO | Perf report for the last 5.13974 seconds: | 20260507 08:06:27.716 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:06:27.716 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:27.716 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:27.716 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:27.716 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 08:06:27.716 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:27.716 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:27.716 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260507 08:06:27.716 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:27.716 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:27.716 | INFO | Uploaded 36GiB to GPU in 2.00387s (17.9652GiB/s) | 20260507 08:06:27.716 | INFO | Downloaded 34GiB to GPU in 1.93019s (17.6148GiB/s) | 20260507 08:06:32.851 | INFO | Analyzing chunk frames = [8397824,8398336) | 20260507 08:06:32.851 | INFO | MemInfo: { | 20260507 08:06:32.851 | INFO | "gpu" : " 17 GB", | 20260507 08:06:32.851 | INFO | "huge" : "", | 20260507 08:06:32.851 | INFO | "huge_shared" : " 27 GB", | 20260507 08:06:32.851 | INFO | "malloc" : " 14 GB", | 20260507 08:06:32.851 | INFO | "pinned" : " 29 GB" | 20260507 08:06:32.851 | INFO | } | 20260507 08:06:32.851 | INFO | Perf report for the last 5.13578 seconds: | 20260507 08:06:32.851 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:06:32.851 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:32.851 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:32.851 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:32.851 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 08:06:32.851 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:32.852 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:32.852 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 08:06:32.852 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:32.852 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:32.852 | INFO | Uploaded 36GiB to GPU in 2.02524s (17.7757GiB/s) | 20260507 08:06:32.852 | INFO | Downloaded 34GiB to GPU in 1.91361s (17.7675GiB/s) | 20260507 08:06:37.986 | INFO | Analyzing chunk frames = [8398336,8398848) | 20260507 08:06:37.986 | INFO | MemInfo: { | 20260507 08:06:37.986 | INFO | "gpu" : " 17 GB", | 20260507 08:06:37.986 | INFO | "huge" : "", | 20260507 08:06:37.986 | INFO | "huge_shared" : " 27 GB", | 20260507 08:06:37.986 | INFO | "malloc" : " 14 GB", | 20260507 08:06:37.986 | INFO | "pinned" : " 29 GB" | 20260507 08:06:37.986 | INFO | } | 20260507 08:06:37.986 | INFO | Perf report for the last 5.13505 seconds: | 20260507 08:06:37.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:06:37.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:37.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:37.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:37.986 | INFO | ANALYSIS : 59.7%, 3.0, 92.5ms, 0 | 20260507 08:06:37.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:37.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:37.987 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:06:37.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:37.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:37.987 | INFO | Uploaded 36GiB to GPU in 2.00784s (17.9297GiB/s) | 20260507 08:06:37.987 | INFO | Downloaded 34GiB to GPU in 1.9436s (17.4933GiB/s) | 20260507 08:06:43.133 | INFO | Analyzing chunk frames = [8398848,8399360) | 20260507 08:06:43.133 | INFO | MemInfo: { | 20260507 08:06:43.133 | INFO | "gpu" : " 17 GB", | 20260507 08:06:43.133 | INFO | "huge" : "", | 20260507 08:06:43.133 | INFO | "huge_shared" : " 27 GB", | 20260507 08:06:43.133 | INFO | "malloc" : " 14 GB", | 20260507 08:06:43.133 | INFO | "pinned" : " 29 GB" | 20260507 08:06:43.133 | INFO | } | 20260507 08:06:43.133 | INFO | Perf report for the last 5.14675 seconds: | 20260507 08:06:43.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:06:43.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:43.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:43.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:43.133 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 08:06:43.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:43.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:43.133 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:06:43.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:43.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:43.134 | INFO | Uploaded 36GiB to GPU in 2.05274s (17.5375GiB/s) | 20260507 08:06:43.134 | INFO | Downloaded 34GiB to GPU in 1.94495s (17.4811GiB/s) | 20260507 08:06:48.271 | INFO | Analyzing chunk frames = [8399360,8399872) | 20260507 08:06:48.271 | INFO | MemInfo: { | 20260507 08:06:48.271 | INFO | "gpu" : " 17 GB", | 20260507 08:06:48.271 | INFO | "huge" : "", | 20260507 08:06:48.271 | INFO | "huge_shared" : " 27 GB", | 20260507 08:06:48.271 | INFO | "malloc" : " 14 GB", | 20260507 08:06:48.271 | INFO | "pinned" : " 29 GB" | 20260507 08:06:48.271 | INFO | } | 20260507 08:06:48.271 | INFO | Perf report for the last 5.13783 seconds: | 20260507 08:06:48.271 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:06:48.271 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:48.271 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:48.271 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:48.271 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 08:06:48.271 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:48.271 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:48.271 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:06:48.271 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:48.271 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:48.271 | INFO | Uploaded 36GiB to GPU in 2.05014s (17.5597GiB/s) | 20260507 08:06:48.271 | INFO | Downloaded 34GiB to GPU in 1.90838s (17.8161GiB/s) | 20260507 08:06:53.414 | INFO | Analyzing chunk frames = [8399872,8400384) | 20260507 08:06:53.414 | INFO | MemInfo: { | 20260507 08:06:53.414 | INFO | "gpu" : " 17 GB", | 20260507 08:06:53.414 | INFO | "huge" : "", | 20260507 08:06:53.414 | INFO | "huge_shared" : " 27 GB", | 20260507 08:06:53.414 | INFO | "malloc" : " 14 GB", | 20260507 08:06:53.414 | INFO | "pinned" : " 29 GB" | 20260507 08:06:53.414 | INFO | } | 20260507 08:06:53.414 | INFO | Perf report for the last 5.1427 seconds: | 20260507 08:06:53.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:06:53.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:53.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:53.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:53.414 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 08:06:53.415 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:53.415 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:53.415 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 08:06:53.415 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:53.415 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:53.415 | INFO | Uploaded 36GiB to GPU in 1.91246s (18.8239GiB/s) | 20260507 08:06:53.415 | INFO | Downloaded 34GiB to GPU in 1.96562s (17.2973GiB/s) | 20260507 08:06:58.554 | INFO | Analyzing chunk frames = [8400384,8400896) | 20260507 08:06:58.554 | INFO | MemInfo: { | 20260507 08:06:58.554 | INFO | "gpu" : " 17 GB", | 20260507 08:06:58.554 | INFO | "huge" : "", | 20260507 08:06:58.554 | INFO | "huge_shared" : " 27 GB", | 20260507 08:06:58.554 | INFO | "malloc" : " 14 GB", | 20260507 08:06:58.554 | INFO | "pinned" : " 29 GB" | 20260507 08:06:58.554 | INFO | } | 20260507 08:06:58.554 | INFO | Perf report for the last 5.14044 seconds: | 20260507 08:06:58.554 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:06:58.554 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:58.554 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:58.554 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:58.554 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:06:58.554 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:58.554 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:58.554 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:06:58.554 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:58.554 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:06:58.555 | INFO | Uploaded 36GiB to GPU in 1.9806s (18.1763GiB/s) | 20260507 08:06:58.555 | INFO | Downloaded 34GiB to GPU in 1.9641s (17.3107GiB/s) | 20260507 08:06:58.651 | INFO | Starting a round of Baz IO | 20260507 08:07:01.746 | INFO | Populating RTMetrics report | 20260507 08:07:01.747 | INFO | RTMetrics report populated | 20260507 08:07:01.747 | INFO | Opening tmp RTMetrics json file | 20260507 08:07:01.756 | INFO | Serializing RTMetrics to json | 20260507 08:07:01.761 | INFO | Serialization done, swapping tmp file now... | 20260507 08:07:01.762 | INFO | RTMetrics done | 20260507 08:07:01.830 | INFO | Finished a round of Baz IO: Wrote 1.621 GiB in 3.178 seconds | 20260507 08:07:01.830 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.52% / 0.00% / 67.48% | 20260507 08:07:01.830 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0943621% / 0% / 6.35783e-05% | 20260507 08:07:01.830 | INFO | Baz IO overall: | 20260507 08:07:01.830 | INFO | BytesWritten : 4193303546248 | 20260507 08:07:01.830 | INFO | NumEvents : 2006416751129 | 20260507 08:07:03.684 | INFO | Analyzing chunk frames = [8400896,8401408) | 20260507 08:07:03.684 | INFO | MemInfo: { | 20260507 08:07:03.684 | INFO | "gpu" : " 17 GB", | 20260507 08:07:03.684 | INFO | "huge" : "", | 20260507 08:07:03.684 | INFO | "huge_shared" : " 27 GB", | 20260507 08:07:03.684 | INFO | "malloc" : " 14 GB", | 20260507 08:07:03.684 | INFO | "pinned" : " 29 GB" | 20260507 08:07:03.684 | INFO | } | 20260507 08:07:03.684 | INFO | Perf report for the last 5.12987 seconds: | 20260507 08:07:03.684 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:07:03.684 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:03.684 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:03.684 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:03.684 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260507 08:07:03.684 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:03.684 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260507 08:07:03.684 | INFO | PRE_HQ : 59.0%, 1.8, 54.7ms, 0 | 20260507 08:07:03.684 | INFO | BAZWRITER : 61.5%, 3.1, 97.1ms, 0 | 20260507 08:07:03.684 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 08:07:03.684 | INFO | Uploaded 36GiB to GPU in 2.06082s (17.4687GiB/s) | 20260507 08:07:03.684 | INFO | Downloaded 38GiB to GPU in 2.18348s (17.4034GiB/s) | 20260507 08:07:08.829 | INFO | Analyzing chunk frames = [8401408,8401920) | 20260507 08:07:08.829 | INFO | MemInfo: { | 20260507 08:07:08.829 | INFO | "gpu" : " 17 GB", | 20260507 08:07:08.829 | INFO | "huge" : "", | 20260507 08:07:08.829 | INFO | "huge_shared" : " 27 GB", | 20260507 08:07:08.829 | INFO | "malloc" : " 14 GB", | 20260507 08:07:08.829 | INFO | "pinned" : " 29 GB" | 20260507 08:07:08.829 | INFO | } | 20260507 08:07:08.829 | INFO | Perf report for the last 5.14487 seconds: | 20260507 08:07:08.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:07:08.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:08.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:08.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:08.829 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:07:08.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:08.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:08.829 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 08:07:08.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:08.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:08.829 | INFO | Uploaded 36GiB to GPU in 2.00216s (17.9806GiB/s) | 20260507 08:07:08.829 | INFO | Downloaded 34GiB to GPU in 1.93749s (17.5485GiB/s) | 20260507 08:07:13.970 | INFO | Analyzing chunk frames = [8401920,8402432) | 20260507 08:07:13.970 | INFO | MemInfo: { | 20260507 08:07:13.970 | INFO | "gpu" : " 17 GB", | 20260507 08:07:13.970 | INFO | "huge" : "", | 20260507 08:07:13.970 | INFO | "huge_shared" : " 27 GB", | 20260507 08:07:13.970 | INFO | "malloc" : " 14 GB", | 20260507 08:07:13.970 | INFO | "pinned" : " 29 GB" | 20260507 08:07:13.970 | INFO | } | 20260507 08:07:13.970 | INFO | Perf report for the last 5.14067 seconds: | 20260507 08:07:13.970 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:07:13.970 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:13.970 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:13.970 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:13.970 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 08:07:13.970 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:13.970 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:13.970 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 08:07:13.970 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:13.970 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:13.970 | INFO | Uploaded 36GiB to GPU in 2.04006s (17.6465GiB/s) | 20260507 08:07:13.970 | INFO | Downloaded 34GiB to GPU in 1.9561s (17.3815GiB/s) | 20260507 08:07:19.111 | INFO | Analyzing chunk frames = [8402432,8402944) | 20260507 08:07:19.111 | INFO | MemInfo: { | 20260507 08:07:19.111 | INFO | "gpu" : " 17 GB", | 20260507 08:07:19.111 | INFO | "huge" : "", | 20260507 08:07:19.111 | INFO | "huge_shared" : " 27 GB", | 20260507 08:07:19.111 | INFO | "malloc" : " 14 GB", | 20260507 08:07:19.111 | INFO | "pinned" : " 29 GB" | 20260507 08:07:19.111 | INFO | } | 20260507 08:07:19.111 | INFO | Perf report for the last 5.14149 seconds: | 20260507 08:07:19.111 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:07:19.111 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:19.111 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:19.111 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:19.111 | INFO | ANALYSIS : 60.3%, 2.9, 93.2ms, 0 | 20260507 08:07:19.111 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:19.111 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:19.111 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:07:19.111 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:19.111 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:19.111 | INFO | Uploaded 36GiB to GPU in 2.03872s (17.6582GiB/s) | 20260507 08:07:19.112 | INFO | Downloaded 34GiB to GPU in 1.95443s (17.3964GiB/s) | 20260507 08:07:24.243 | INFO | Analyzing chunk frames = [8402944,8403456) | 20260507 08:07:24.243 | INFO | MemInfo: { | 20260507 08:07:24.243 | INFO | "gpu" : " 17 GB", | 20260507 08:07:24.243 | INFO | "huge" : "", | 20260507 08:07:24.243 | INFO | "huge_shared" : " 27 GB", | 20260507 08:07:24.243 | INFO | "malloc" : " 14 GB", | 20260507 08:07:24.243 | INFO | "pinned" : " 29 GB" | 20260507 08:07:24.243 | INFO | } | 20260507 08:07:24.243 | INFO | Perf report for the last 5.13191 seconds: | 20260507 08:07:24.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:07:24.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:24.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:24.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:24.243 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 08:07:24.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:24.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:24.243 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:07:24.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:24.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:24.243 | INFO | Uploaded 36GiB to GPU in 2.03472s (17.6929GiB/s) | 20260507 08:07:24.244 | INFO | Downloaded 34GiB to GPU in 1.9156s (17.749GiB/s) | 20260507 08:07:29.385 | INFO | Analyzing chunk frames = [8403456,8403968) | 20260507 08:07:29.385 | INFO | MemInfo: { | 20260507 08:07:29.385 | INFO | "gpu" : " 17 GB", | 20260507 08:07:29.385 | INFO | "huge" : "", | 20260507 08:07:29.385 | INFO | "huge_shared" : " 27 GB", | 20260507 08:07:29.385 | INFO | "malloc" : " 14 GB", | 20260507 08:07:29.385 | INFO | "pinned" : " 29 GB" | 20260507 08:07:29.385 | INFO | } | 20260507 08:07:29.385 | INFO | Perf report for the last 5.14169 seconds: | 20260507 08:07:29.385 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:07:29.385 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:29.385 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:29.385 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:29.385 | INFO | ANALYSIS : 60.2%, 3.0, 94.1ms, 0 | 20260507 08:07:29.385 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:29.385 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:29.385 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:07:29.385 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:29.385 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:29.385 | INFO | Uploaded 36GiB to GPU in 2.03899s (17.6558GiB/s) | 20260507 08:07:29.385 | INFO | Downloaded 34GiB to GPU in 1.91575s (17.7476GiB/s) | 20260507 08:07:34.529 | INFO | Analyzing chunk frames = [8403968,8404480) | 20260507 08:07:34.529 | INFO | MemInfo: { | 20260507 08:07:34.529 | INFO | "gpu" : " 17 GB", | 20260507 08:07:34.529 | INFO | "huge" : "", | 20260507 08:07:34.529 | INFO | "huge_shared" : " 27 GB", | 20260507 08:07:34.529 | INFO | "malloc" : " 14 GB", | 20260507 08:07:34.529 | INFO | "pinned" : " 29 GB" | 20260507 08:07:34.530 | INFO | } | 20260507 08:07:34.530 | INFO | Perf report for the last 5.14472 seconds: | 20260507 08:07:34.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:07:34.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:34.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:34.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:34.530 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 08:07:34.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:34.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:34.530 | INFO | PRE_HQ : 26.8%, 1.0, 13.9ms, 0 | 20260507 08:07:34.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:34.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:34.530 | INFO | Uploaded 36GiB to GPU in 2.03852s (17.6599GiB/s) | 20260507 08:07:34.530 | INFO | Downloaded 34GiB to GPU in 1.98803s (17.1024GiB/s) | 20260507 08:07:39.673 | INFO | Analyzing chunk frames = [8404480,8404992) | 20260507 08:07:39.673 | INFO | MemInfo: { | 20260507 08:07:39.673 | INFO | "gpu" : " 17 GB", | 20260507 08:07:39.673 | INFO | "huge" : "", | 20260507 08:07:39.673 | INFO | "huge_shared" : " 27 GB", | 20260507 08:07:39.673 | INFO | "malloc" : " 14 GB", | 20260507 08:07:39.673 | INFO | "pinned" : " 29 GB" | 20260507 08:07:39.673 | INFO | } | 20260507 08:07:39.673 | INFO | Perf report for the last 5.14379 seconds: | 20260507 08:07:39.673 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:07:39.673 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:39.673 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:39.673 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:39.673 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:07:39.673 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:39.673 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:39.674 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:07:39.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:39.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:39.674 | INFO | Uploaded 36GiB to GPU in 1.98964s (18.0938GiB/s) | 20260507 08:07:39.674 | INFO | Downloaded 34GiB to GPU in 1.96202s (17.3291GiB/s) | 20260507 08:07:39.674 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:07:39.674 | INFO | Total mem usage: 18 GiB | 20260507 08:07:39.674 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:07:39.674 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 08:07:39.674 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:07:39.674 | INFO | High Water Breakdown: | 20260507 08:07:39.674 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:07:39.674 | INFO | 39.7% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:07:39.674 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:07:39.674 | INFO | ------------------------------------------ | 20260507 08:07:39.674 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:07:39.674 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:07:39.674 | INFO | Total mem usage: 26.5 GiB | 20260507 08:07:39.674 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:07:39.674 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:07:39.674 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:07:39.674 | INFO | High Water Breakdown: | 20260507 08:07:39.674 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:07:39.674 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:07:39.674 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:07:39.674 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:07:39.674 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:07:39.674 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:07:39.674 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:07:39.674 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:07:39.674 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:07:39.674 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:07:39.674 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:07:39.674 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:07:39.674 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:07:39.674 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:07:39.674 | INFO | ------------------------------------------ | 20260507 08:07:39.674 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:07:39.674 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:07:39.674 | INFO | Total mem usage: 25.6 GiB | 20260507 08:07:39.674 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:07:39.674 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:07:39.674 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:07:39.674 | INFO | High Water Breakdown: | 20260507 08:07:39.674 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:07:39.674 | INFO | ------------------------------------------ | 20260507 08:07:39.674 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:07:39.674 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:07:39.674 | INFO | Total mem usage: 15.7 GiB | 20260507 08:07:39.674 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:07:39.674 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:07:39.674 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:07:39.674 | INFO | High Water Breakdown: | 20260507 08:07:39.674 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:07:39.674 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:07:39.674 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:07:39.674 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:07:39.674 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:07:39.674 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:07:39.674 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:07:39.674 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:07:39.674 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:07:39.674 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:07:39.674 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:07:39.674 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:07:39.674 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:07:39.674 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:07:39.674 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:07:39.674 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:07:39.674 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:07:39.674 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:07:39.674 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:07:39.674 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:07:39.674 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:07:39.674 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:07:39.674 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:07:39.674 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:07:39.674 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:07:39.674 | INFO | ------------------------------------------ | 20260507 08:07:39.675 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:07:39.675 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:07:39.675 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:07:39.675 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:07:39.675 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 1.02e+00 | 20260507 08:07:39.675 | INFO | PerfReport: 0.29%, AggregateStats -- 9.01e-02 / 9.98e-02, 9.19e-03 / 5.12e-03 | 20260507 08:07:39.675 | INFO | PerfReport: 11.68%, Baseline -- 3.58e+00 / 4.43e+00, 5.05e-01 / 2.86e-01 | 20260507 08:07:39.675 | INFO | PerfReport: 3.21%, Histogram -- 9.86e-01 / 1.47e+00, 1.62e-01 / 1.35e-01 | 20260507 08:07:39.675 | INFO | PerfReport: 16.31%, DME -- 4.00e+01 / 4.49e+01, 4.04e+00 / 1.32e+00 | 20260507 08:07:39.675 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.27e+00 / 1.28e+00 | 20260507 08:07:39.675 | INFO | PerfReport: 6.88%, PulseAccumulating -- 2.11e+00 / 2.68e+00, 3.82e-01 / 2.05e-01 | 20260507 08:07:39.675 | INFO | PerfReport: 2.91%, Metrics -- 8.94e-01 / 1.21e+00, 2.39e-01 / 1.22e-01 | 20260507 08:07:39.675 | INFO | PerfReport: 0.98%, PulseSerializing -- 2.99e-01 / 3.52e-01, 5.06e-02 / 1.71e-02 | 20260507 08:07:39.675 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:07:39.675 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:07:39.675 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:07:39.675 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 08:07:39.675 | INFO | PerfReport: 21.90%, Upload -- 2.10e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260507 08:07:39.675 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.30e+00 / 0.00e+00 | 20260507 08:07:39.675 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 08:07:39.675 | INFO | PerfReport: 1.52%, LockUpload -- 1.46e+00 / 0.00e+00, 4.87e+00 / 0.00e+00 | 20260507 08:07:39.675 | INFO | PerfReport: 1.29%, LockDownload -- 1.24e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260507 08:07:39.675 | INFO | PerfReport: 21.97%, LockCompute -- 2.11e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 08:07:39.675 | INFO | PerfReport: 0.02%, Misc -- 1.48e-02 / 0.00e+00, 3.49e-02 / 0.00e+00 | 20260507 08:07:39.770 | INFO | Starting a round of Baz IO | 20260507 08:07:42.866 | INFO | Populating RTMetrics report | 20260507 08:07:42.866 | INFO | RTMetrics report populated | 20260507 08:07:42.866 | INFO | Opening tmp RTMetrics json file | 20260507 08:07:42.878 | INFO | Serializing RTMetrics to json | 20260507 08:07:42.880 | INFO | Serialization done, swapping tmp file now... | 20260507 08:07:42.880 | INFO | RTMetrics done | 20260507 08:07:42.945 | INFO | Finished a round of Baz IO: Wrote 1.62 GiB in 3.175 seconds | 20260507 08:07:42.945 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.52% / 0.00% / 67.48% | 20260507 08:07:42.945 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0939449% / 7.94729e-06% / 4.76837e-05% | 20260507 08:07:42.945 | INFO | Baz IO overall: | 20260507 08:07:42.945 | INFO | BytesWritten : 4195043395973 | 20260507 08:07:42.945 | INFO | NumEvents : 2007227703821 | 20260507 08:07:44.801 | INFO | Analyzing chunk frames = [8404992,8405504) | 20260507 08:07:44.801 | INFO | MemInfo: { | 20260507 08:07:44.801 | INFO | "gpu" : " 17 GB", | 20260507 08:07:44.801 | INFO | "huge" : "", | 20260507 08:07:44.801 | INFO | "huge_shared" : " 27 GB", | 20260507 08:07:44.801 | INFO | "malloc" : " 14 GB", | 20260507 08:07:44.801 | INFO | "pinned" : " 29 GB" | 20260507 08:07:44.801 | INFO | } | 20260507 08:07:44.801 | INFO | Perf report for the last 5.12793 seconds: | 20260507 08:07:44.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:07:44.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:44.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:44.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:44.801 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260507 08:07:44.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:44.801 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260507 08:07:44.802 | INFO | PRE_HQ : 58.7%, 1.8, 54.6ms, 0 | 20260507 08:07:44.802 | INFO | BAZWRITER : 61.5%, 3.1, 97.1ms, 0 | 20260507 08:07:44.802 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 08:07:44.802 | INFO | Uploaded 36GiB to GPU in 2.03422s (17.6972GiB/s) | 20260507 08:07:44.802 | INFO | Downloaded 38GiB to GPU in 2.15615s (17.624GiB/s) | 20260507 08:07:49.935 | INFO | Analyzing chunk frames = [8405504,8406016) | 20260507 08:07:49.935 | INFO | MemInfo: { | 20260507 08:07:49.935 | INFO | "gpu" : " 17 GB", | 20260507 08:07:49.935 | INFO | "huge" : "", | 20260507 08:07:49.935 | INFO | "huge_shared" : " 27 GB", | 20260507 08:07:49.935 | INFO | "malloc" : " 14 GB", | 20260507 08:07:49.935 | INFO | "pinned" : " 29 GB" | 20260507 08:07:49.935 | INFO | } | 20260507 08:07:49.936 | INFO | Perf report for the last 5.13429 seconds: | 20260507 08:07:49.936 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:07:49.936 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:49.936 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:49.936 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:49.936 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 08:07:49.936 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:49.936 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:49.936 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260507 08:07:49.936 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:49.936 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:49.936 | INFO | Uploaded 36GiB to GPU in 1.98197s (18.1637GiB/s) | 20260507 08:07:49.936 | INFO | Downloaded 34GiB to GPU in 1.92704s (17.6436GiB/s) | 20260507 08:07:55.081 | INFO | Analyzing chunk frames = [8406016,8406528) | 20260507 08:07:55.081 | INFO | MemInfo: { | 20260507 08:07:55.081 | INFO | "gpu" : " 17 GB", | 20260507 08:07:55.081 | INFO | "huge" : "", | 20260507 08:07:55.081 | INFO | "huge_shared" : " 27 GB", | 20260507 08:07:55.081 | INFO | "malloc" : " 14 GB", | 20260507 08:07:55.081 | INFO | "pinned" : " 29 GB" | 20260507 08:07:55.082 | INFO | } | 20260507 08:07:55.082 | INFO | Perf report for the last 5.14599 seconds: | 20260507 08:07:55.082 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:07:55.082 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:55.082 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:55.082 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:55.082 | INFO | ANALYSIS : 59.9%, 3.0, 93.0ms, 0 | 20260507 08:07:55.082 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:55.082 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:55.082 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:07:55.082 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:55.082 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:07:55.082 | INFO | Uploaded 36GiB to GPU in 2.01353s (17.879GiB/s) | 20260507 08:07:55.082 | INFO | Downloaded 34GiB to GPU in 1.94111s (17.5157GiB/s) | 20260507 08:08:00.225 | INFO | Analyzing chunk frames = [8406528,8407040) | 20260507 08:08:00.225 | INFO | MemInfo: { | 20260507 08:08:00.225 | INFO | "gpu" : " 17 GB", | 20260507 08:08:00.225 | INFO | "huge" : "", | 20260507 08:08:00.225 | INFO | "huge_shared" : " 27 GB", | 20260507 08:08:00.225 | INFO | "malloc" : " 14 GB", | 20260507 08:08:00.225 | INFO | "pinned" : " 29 GB" | 20260507 08:08:00.225 | INFO | } | 20260507 08:08:00.225 | INFO | Perf report for the last 5.1432 seconds: | 20260507 08:08:00.225 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:08:00.225 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:00.225 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:00.225 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:00.225 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 08:08:00.225 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:00.225 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:00.225 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:08:00.225 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:00.225 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:00.225 | INFO | Uploaded 36GiB to GPU in 2.02346s (17.7913GiB/s) | 20260507 08:08:00.225 | INFO | Downloaded 34GiB to GPU in 1.94321s (17.4968GiB/s) | 20260507 08:08:05.362 | INFO | Analyzing chunk frames = [8407040,8407552) | 20260507 08:08:05.362 | INFO | MemInfo: { | 20260507 08:08:05.362 | INFO | "gpu" : " 17 GB", | 20260507 08:08:05.362 | INFO | "huge" : "", | 20260507 08:08:05.362 | INFO | "huge_shared" : " 27 GB", | 20260507 08:08:05.362 | INFO | "malloc" : " 14 GB", | 20260507 08:08:05.362 | INFO | "pinned" : " 29 GB" | 20260507 08:08:05.362 | INFO | } | 20260507 08:08:05.362 | INFO | Perf report for the last 5.1374 seconds: | 20260507 08:08:05.362 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:08:05.362 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:05.362 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:05.362 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:05.362 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 08:08:05.362 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:05.362 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:05.362 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 08:08:05.362 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:05.362 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:05.362 | INFO | Uploaded 36GiB to GPU in 2.0606s (17.4707GiB/s) | 20260507 08:08:05.363 | INFO | Downloaded 34GiB to GPU in 1.9071s (17.8281GiB/s) | 20260507 08:08:10.497 | INFO | Analyzing chunk frames = [8407552,8408064) | 20260507 08:08:10.497 | INFO | MemInfo: { | 20260507 08:08:10.497 | INFO | "gpu" : " 17 GB", | 20260507 08:08:10.497 | INFO | "huge" : "", | 20260507 08:08:10.497 | INFO | "huge_shared" : " 27 GB", | 20260507 08:08:10.497 | INFO | "malloc" : " 14 GB", | 20260507 08:08:10.497 | INFO | "pinned" : " 29 GB" | 20260507 08:08:10.498 | INFO | } | 20260507 08:08:10.498 | INFO | Perf report for the last 5.13535 seconds: | 20260507 08:08:10.498 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:08:10.498 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:10.498 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:10.498 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:10.498 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 08:08:10.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:10.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:10.498 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 08:08:10.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:10.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:10.498 | INFO | Uploaded 36GiB to GPU in 2.07591s (17.3418GiB/s) | 20260507 08:08:10.498 | INFO | Downloaded 34GiB to GPU in 1.9443s (17.487GiB/s) | 20260507 08:08:15.639 | INFO | Analyzing chunk frames = [8408064,8408576) | 20260507 08:08:15.639 | INFO | MemInfo: { | 20260507 08:08:15.639 | INFO | "gpu" : " 17 GB", | 20260507 08:08:15.639 | INFO | "huge" : "", | 20260507 08:08:15.639 | INFO | "huge_shared" : " 27 GB", | 20260507 08:08:15.639 | INFO | "malloc" : " 14 GB", | 20260507 08:08:15.639 | INFO | "pinned" : " 29 GB" | 20260507 08:08:15.639 | INFO | } | 20260507 08:08:15.639 | INFO | Perf report for the last 5.14185 seconds: | 20260507 08:08:15.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:08:15.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:15.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:15.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:15.640 | INFO | ANALYSIS : 60.9%, 3.0, 94.8ms, 0 | 20260507 08:08:15.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:15.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:15.640 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 08:08:15.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:15.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:15.640 | INFO | Uploaded 36GiB to GPU in 2.04323s (17.6192GiB/s) | 20260507 08:08:15.640 | INFO | Downloaded 34GiB to GPU in 1.98762s (17.1059GiB/s) | 20260507 08:08:20.784 | INFO | Analyzing chunk frames = [8408576,8409088) | 20260507 08:08:20.784 | INFO | MemInfo: { | 20260507 08:08:20.784 | INFO | "gpu" : " 17 GB", | 20260507 08:08:20.784 | INFO | "huge" : "", | 20260507 08:08:20.784 | INFO | "huge_shared" : " 27 GB", | 20260507 08:08:20.784 | INFO | "malloc" : " 14 GB", | 20260507 08:08:20.785 | INFO | "pinned" : " 29 GB" | 20260507 08:08:20.785 | INFO | } | 20260507 08:08:20.785 | INFO | Perf report for the last 5.14524 seconds: | 20260507 08:08:20.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:08:20.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:20.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:20.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:20.785 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:08:20.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:20.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:20.785 | INFO | PRE_HQ : 27.8%, 1.0, 14.5ms, 0 | 20260507 08:08:20.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:20.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:20.785 | INFO | Uploaded 36GiB to GPU in 1.99257s (18.0671GiB/s) | 20260507 08:08:20.785 | INFO | Downloaded 34GiB to GPU in 1.95801s (17.3646GiB/s) | 20260507 08:08:20.882 | INFO | Starting a round of Baz IO | 20260507 08:08:23.965 | INFO | Populating RTMetrics report | 20260507 08:08:23.965 | INFO | RTMetrics report populated | 20260507 08:08:23.965 | INFO | Opening tmp RTMetrics json file | 20260507 08:08:23.976 | INFO | Serializing RTMetrics to json | 20260507 08:08:23.980 | INFO | Serialization done, swapping tmp file now... | 20260507 08:08:23.980 | INFO | RTMetrics done | 20260507 08:08:24.050 | INFO | Finished a round of Baz IO: Wrote 1.621 GiB in 3.167 seconds | 20260507 08:08:24.050 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.51% / 0.00% / 67.48% | 20260507 08:08:24.050 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0940641% / 0% / 6.75519e-05% | 20260507 08:08:24.050 | INFO | Baz IO overall: | 20260507 08:08:24.050 | INFO | BytesWritten : 4196784161237 | 20260507 08:08:24.050 | INFO | NumEvents : 2008039246893 | 20260507 08:08:25.922 | INFO | Analyzing chunk frames = [8409088,8409600) | 20260507 08:08:25.922 | INFO | MemInfo: { | 20260507 08:08:25.922 | INFO | "gpu" : " 17 GB", | 20260507 08:08:25.922 | INFO | "huge" : "", | 20260507 08:08:25.922 | INFO | "huge_shared" : " 27 GB", | 20260507 08:08:25.922 | INFO | "malloc" : " 14 GB", | 20260507 08:08:25.922 | INFO | "pinned" : " 29 GB" | 20260507 08:08:25.922 | INFO | } | 20260507 08:08:25.922 | INFO | Perf report for the last 5.13726 seconds: | 20260507 08:08:25.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:08:25.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:25.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:25.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:25.922 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260507 08:08:25.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:25.922 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 08:08:25.922 | INFO | PRE_HQ : 58.6%, 1.8, 54.5ms, 0 | 20260507 08:08:25.922 | INFO | BAZWRITER : 61.2%, 3.1, 97.1ms, 0 | 20260507 08:08:25.922 | INFO | BAZ_SAMPLING : 0.7%, 1.0, 0.4ms, 0 | 20260507 08:08:25.922 | INFO | Uploaded 36GiB to GPU in 2.05346s (17.5314GiB/s) | 20260507 08:08:25.922 | INFO | Downloaded 38GiB to GPU in 2.16598s (17.544GiB/s) | 20260507 08:08:31.058 | INFO | Analyzing chunk frames = [8409600,8410112) | 20260507 08:08:31.058 | INFO | MemInfo: { | 20260507 08:08:31.058 | INFO | "gpu" : " 17 GB", | 20260507 08:08:31.058 | INFO | "huge" : "", | 20260507 08:08:31.058 | INFO | "huge_shared" : " 27 GB", | 20260507 08:08:31.058 | INFO | "malloc" : " 14 GB", | 20260507 08:08:31.058 | INFO | "pinned" : " 29 GB" | 20260507 08:08:31.058 | INFO | } | 20260507 08:08:31.058 | INFO | Perf report for the last 5.13596 seconds: | 20260507 08:08:31.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:08:31.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:31.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:31.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:31.058 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 08:08:31.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:31.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:31.058 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 08:08:31.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:31.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:31.058 | INFO | Uploaded 36GiB to GPU in 2.00269s (17.9758GiB/s) | 20260507 08:08:31.058 | INFO | Downloaded 34GiB to GPU in 1.93873s (17.5373GiB/s) | 20260507 08:08:36.197 | INFO | Analyzing chunk frames = [8410112,8410624) | 20260507 08:08:36.197 | INFO | MemInfo: { | 20260507 08:08:36.197 | INFO | "gpu" : " 17 GB", | 20260507 08:08:36.197 | INFO | "huge" : "", | 20260507 08:08:36.197 | INFO | "huge_shared" : " 27 GB", | 20260507 08:08:36.197 | INFO | "malloc" : " 14 GB", | 20260507 08:08:36.197 | INFO | "pinned" : " 29 GB" | 20260507 08:08:36.197 | INFO | } | 20260507 08:08:36.197 | INFO | Perf report for the last 5.13915 seconds: | 20260507 08:08:36.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:08:36.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:36.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:36.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:36.197 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 08:08:36.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:36.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:36.197 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:08:36.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:36.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:36.197 | INFO | Uploaded 36GiB to GPU in 2.05107s (17.5518GiB/s) | 20260507 08:08:36.198 | INFO | Downloaded 34GiB to GPU in 1.95068s (17.4298GiB/s) | 20260507 08:08:41.335 | INFO | Analyzing chunk frames = [8410624,8411136) | 20260507 08:08:41.335 | INFO | MemInfo: { | 20260507 08:08:41.335 | INFO | "gpu" : " 17 GB", | 20260507 08:08:41.335 | INFO | "huge" : "", | 20260507 08:08:41.335 | INFO | "huge_shared" : " 27 GB", | 20260507 08:08:41.335 | INFO | "malloc" : " 14 GB", | 20260507 08:08:41.335 | INFO | "pinned" : " 29 GB" | 20260507 08:08:41.335 | INFO | } | 20260507 08:08:41.335 | INFO | Perf report for the last 5.13797 seconds: | 20260507 08:08:41.335 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:08:41.335 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:41.335 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:41.335 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:41.335 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 08:08:41.335 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:41.335 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:41.335 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:08:41.335 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:41.335 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:41.335 | INFO | Uploaded 36GiB to GPU in 2.01517s (17.8645GiB/s) | 20260507 08:08:41.335 | INFO | Downloaded 34GiB to GPU in 1.94237s (17.5044GiB/s) | 20260507 08:08:46.481 | INFO | Analyzing chunk frames = [8411136,8411648) | 20260507 08:08:46.481 | INFO | MemInfo: { | 20260507 08:08:46.481 | INFO | "gpu" : " 17 GB", | 20260507 08:08:46.481 | INFO | "huge" : "", | 20260507 08:08:46.481 | INFO | "huge_shared" : " 27 GB", | 20260507 08:08:46.481 | INFO | "malloc" : " 14 GB", | 20260507 08:08:46.481 | INFO | "pinned" : " 29 GB" | 20260507 08:08:46.481 | INFO | } | 20260507 08:08:46.481 | INFO | Perf report for the last 5.14574 seconds: | 20260507 08:08:46.481 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:08:46.481 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:46.481 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:46.481 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:46.481 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 08:08:46.481 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:46.481 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:46.481 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:08:46.481 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:46.481 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:46.481 | INFO | Uploaded 36GiB to GPU in 2.05851s (17.4884GiB/s) | 20260507 08:08:46.481 | INFO | Downloaded 34GiB to GPU in 1.94999s (17.436GiB/s) | 20260507 08:08:51.625 | INFO | Analyzing chunk frames = [8411648,8412160) | 20260507 08:08:51.625 | INFO | MemInfo: { | 20260507 08:08:51.625 | INFO | "gpu" : " 17 GB", | 20260507 08:08:51.625 | INFO | "huge" : "", | 20260507 08:08:51.626 | INFO | "huge_shared" : " 27 GB", | 20260507 08:08:51.626 | INFO | "malloc" : " 14 GB", | 20260507 08:08:51.626 | INFO | "pinned" : " 29 GB" | 20260507 08:08:51.626 | INFO | } | 20260507 08:08:51.626 | INFO | Perf report for the last 5.14478 seconds: | 20260507 08:08:51.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:08:51.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:51.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:51.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:51.626 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 08:08:51.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:51.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:51.626 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:08:51.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:51.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:51.626 | INFO | Uploaded 36GiB to GPU in 2.04042s (17.6434GiB/s) | 20260507 08:08:51.626 | INFO | Downloaded 34GiB to GPU in 1.92438s (17.668GiB/s) | 20260507 08:08:56.754 | INFO | Analyzing chunk frames = [8412160,8412672) | 20260507 08:08:56.754 | INFO | MemInfo: { | 20260507 08:08:56.754 | INFO | "gpu" : " 17 GB", | 20260507 08:08:56.754 | INFO | "huge" : "", | 20260507 08:08:56.754 | INFO | "huge_shared" : " 27 GB", | 20260507 08:08:56.754 | INFO | "malloc" : " 14 GB", | 20260507 08:08:56.754 | INFO | "pinned" : " 29 GB" | 20260507 08:08:56.754 | INFO | } | 20260507 08:08:56.754 | INFO | Perf report for the last 5.12871 seconds: | 20260507 08:08:56.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:08:56.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:56.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:56.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:56.754 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:08:56.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:56.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:56.755 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 08:08:56.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:56.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:08:56.755 | INFO | Uploaded 36GiB to GPU in 1.94623s (18.4973GiB/s) | 20260507 08:08:56.755 | INFO | Downloaded 34GiB to GPU in 1.97467s (17.2181GiB/s) | 20260507 08:09:01.892 | INFO | Analyzing chunk frames = [8412672,8413184) | 20260507 08:09:01.892 | INFO | MemInfo: { | 20260507 08:09:01.892 | INFO | "gpu" : " 17 GB", | 20260507 08:09:01.892 | INFO | "huge" : "", | 20260507 08:09:01.892 | INFO | "huge_shared" : " 27 GB", | 20260507 08:09:01.892 | INFO | "malloc" : " 14 GB", | 20260507 08:09:01.892 | INFO | "pinned" : " 29 GB" | 20260507 08:09:01.892 | INFO | } | 20260507 08:09:01.893 | INFO | Perf report for the last 5.13816 seconds: | 20260507 08:09:01.893 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:09:01.893 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:01.893 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:01.893 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:01.893 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 08:09:01.893 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:01.893 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:01.893 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:09:01.893 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:01.893 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:01.893 | INFO | Uploaded 36GiB to GPU in 1.97599s (18.2187GiB/s) | 20260507 08:09:01.893 | INFO | Downloaded 34GiB to GPU in 1.96597s (17.2943GiB/s) | 20260507 08:09:01.893 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:09:01.893 | INFO | Total mem usage: 18 GiB | 20260507 08:09:01.893 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:09:01.893 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:09:01.893 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:09:01.893 | INFO | High Water Breakdown: | 20260507 08:09:01.893 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:09:01.893 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:09:01.893 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:09:01.893 | INFO | ------------------------------------------ | 20260507 08:09:01.893 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:09:01.893 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:09:01.893 | INFO | Total mem usage: 26.5 GiB | 20260507 08:09:01.893 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:09:01.893 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:09:01.893 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:09:01.893 | INFO | High Water Breakdown: | 20260507 08:09:01.893 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:09:01.893 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:09:01.893 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:09:01.893 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:09:01.893 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:09:01.893 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:09:01.893 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:09:01.893 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:09:01.893 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:09:01.893 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:09:01.893 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:09:01.893 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:09:01.893 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:09:01.893 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:09:01.893 | INFO | ------------------------------------------ | 20260507 08:09:01.893 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:09:01.893 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:09:01.893 | INFO | Total mem usage: 25.6 GiB | 20260507 08:09:01.893 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:09:01.893 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:09:01.894 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:09:01.894 | INFO | High Water Breakdown: | 20260507 08:09:01.894 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:09:01.894 | INFO | ------------------------------------------ | 20260507 08:09:01.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:09:01.894 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:09:01.894 | INFO | Total mem usage: 15.7 GiB | 20260507 08:09:01.894 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:09:01.894 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:09:01.894 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:09:01.894 | INFO | High Water Breakdown: | 20260507 08:09:01.894 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:09:01.894 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:09:01.894 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:09:01.894 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:09:01.894 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:09:01.894 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:09:01.894 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:09:01.894 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:09:01.894 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:09:01.894 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:09:01.894 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:09:01.894 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:09:01.894 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:09:01.894 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:09:01.894 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:09:01.894 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:09:01.894 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:09:01.894 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:09:01.894 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:09:01.894 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:09:01.894 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:09:01.894 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:09:01.894 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:09:01.894 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:09:01.894 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:09:01.894 | INFO | ------------------------------------------ | 20260507 08:09:01.894 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:09:01.894 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:09:01.894 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:09:01.894 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:09:01.894 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.43e+01, 1.33e+01 / 1.15e+00 | 20260507 08:09:01.894 | INFO | PerfReport: 0.29%, AggregateStats -- 9.03e-02 / 1.01e-01, 9.25e-03 / 5.54e-03 | 20260507 08:09:01.894 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.42e+00, 5.00e-01 / 3.04e-01 | 20260507 08:09:01.894 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.47e+00, 1.66e-01 / 1.17e-01 | 20260507 08:09:01.894 | INFO | PerfReport: 16.24%, DME -- 3.98e+01 / 4.42e+01, 3.68e+00 / 1.30e+00 | 20260507 08:09:01.894 | INFO | PerfReport: 57.82%, FrameLabeling -- 1.77e+01 / 2.00e+01, 1.33e+00 / 1.52e+00 | 20260507 08:09:01.894 | INFO | PerfReport: 6.88%, PulseAccumulating -- 2.11e+00 / 2.64e+00, 3.84e-01 / 2.41e-01 | 20260507 08:09:01.894 | INFO | PerfReport: 2.91%, Metrics -- 8.92e-01 / 1.20e+00, 2.39e-01 / 1.30e-01 | 20260507 08:09:01.894 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.56e-01, 5.06e-02 / 2.67e-02 | 20260507 08:09:01.894 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:09:01.894 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:09:01.894 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:09:01.894 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.39e+01 / 0.00e+00 | 20260507 08:09:01.894 | INFO | PerfReport: 21.94%, Upload -- 2.11e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 08:09:01.894 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 08:09:01.894 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 08:09:01.894 | INFO | PerfReport: 1.59%, LockUpload -- 1.52e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 08:09:01.894 | INFO | PerfReport: 1.17%, LockDownload -- 1.12e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260507 08:09:01.894 | INFO | PerfReport: 21.96%, LockCompute -- 2.11e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 08:09:01.894 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.94e-02 / 0.00e+00 | 20260507 08:09:01.993 | INFO | Starting a round of Baz IO | 20260507 08:09:05.080 | INFO | Populating RTMetrics report | 20260507 08:09:05.080 | INFO | RTMetrics report populated | 20260507 08:09:05.080 | INFO | Opening tmp RTMetrics json file | 20260507 08:09:05.095 | INFO | Serializing RTMetrics to json | 20260507 08:09:05.097 | INFO | Serialization done, swapping tmp file now... | 20260507 08:09:05.097 | INFO | RTMetrics done | 20260507 08:09:05.166 | INFO | Finished a round of Baz IO: Wrote 1.621 GiB in 3.172 seconds | 20260507 08:09:05.166 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.51% / 0.00% / 67.49% | 20260507 08:09:05.166 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0940124% / 0% / 7.54992e-05% | 20260507 08:09:05.166 | INFO | Baz IO overall: | 20260507 08:09:05.166 | INFO | BytesWritten : 4198524267051 | 20260507 08:09:05.166 | INFO | NumEvents : 2008850219169 | 20260507 08:09:07.035 | INFO | Analyzing chunk frames = [8413184,8413696) | 20260507 08:09:07.035 | INFO | MemInfo: { | 20260507 08:09:07.035 | INFO | "gpu" : " 17 GB", | 20260507 08:09:07.035 | INFO | "huge" : "", | 20260507 08:09:07.035 | INFO | "huge_shared" : " 27 GB", | 20260507 08:09:07.035 | INFO | "malloc" : " 14 GB", | 20260507 08:09:07.035 | INFO | "pinned" : " 29 GB" | 20260507 08:09:07.035 | INFO | } | 20260507 08:09:07.035 | INFO | Perf report for the last 5.14206 seconds: | 20260507 08:09:07.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:09:07.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:07.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:07.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:07.035 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 08:09:07.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:07.035 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 08:09:07.035 | INFO | PRE_HQ : 58.7%, 1.8, 54.7ms, 0 | 20260507 08:09:07.035 | INFO | BAZWRITER : 61.2%, 3.1, 97.3ms, 0 | 20260507 08:09:07.035 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:09:07.035 | INFO | Uploaded 36GiB to GPU in 2.03599s (17.6818GiB/s) | 20260507 08:09:07.035 | INFO | Downloaded 38GiB to GPU in 2.17174s (17.4975GiB/s) | 20260507 08:09:12.172 | INFO | Analyzing chunk frames = [8413696,8414208) | 20260507 08:09:12.172 | INFO | MemInfo: { | 20260507 08:09:12.172 | INFO | "gpu" : " 17 GB", | 20260507 08:09:12.172 | INFO | "huge" : "", | 20260507 08:09:12.172 | INFO | "huge_shared" : " 27 GB", | 20260507 08:09:12.172 | INFO | "malloc" : " 14 GB", | 20260507 08:09:12.172 | INFO | "pinned" : " 29 GB" | 20260507 08:09:12.172 | INFO | } | 20260507 08:09:12.172 | INFO | Perf report for the last 5.13748 seconds: | 20260507 08:09:12.172 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:09:12.172 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:12.172 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:12.172 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:12.172 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:09:12.172 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:12.172 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:12.172 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:09:12.172 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:12.172 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:12.173 | INFO | Uploaded 36GiB to GPU in 1.9693s (18.2806GiB/s) | 20260507 08:09:12.173 | INFO | Downloaded 34GiB to GPU in 1.93242s (17.5946GiB/s) | 20260507 08:09:17.307 | INFO | Analyzing chunk frames = [8414208,8414720) | 20260507 08:09:17.307 | INFO | MemInfo: { | 20260507 08:09:17.307 | INFO | "gpu" : " 17 GB", | 20260507 08:09:17.307 | INFO | "huge" : "", | 20260507 08:09:17.307 | INFO | "huge_shared" : " 27 GB", | 20260507 08:09:17.308 | INFO | "malloc" : " 14 GB", | 20260507 08:09:17.308 | INFO | "pinned" : " 29 GB" | 20260507 08:09:17.308 | INFO | } | 20260507 08:09:17.308 | INFO | Perf report for the last 5.13559 seconds: | 20260507 08:09:17.308 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:09:17.308 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:17.308 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:17.308 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:17.308 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 08:09:17.308 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:17.308 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:17.308 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:09:17.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:17.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:17.308 | INFO | Uploaded 36GiB to GPU in 2.01262s (17.8871GiB/s) | 20260507 08:09:17.308 | INFO | Downloaded 34GiB to GPU in 1.92071s (17.7018GiB/s) | 20260507 08:09:22.448 | INFO | Analyzing chunk frames = [8414720,8415232) | 20260507 08:09:22.448 | INFO | MemInfo: { | 20260507 08:09:22.449 | INFO | "gpu" : " 17 GB", | 20260507 08:09:22.449 | INFO | "huge" : "", | 20260507 08:09:22.449 | INFO | "huge_shared" : " 27 GB", | 20260507 08:09:22.449 | INFO | "malloc" : " 14 GB", | 20260507 08:09:22.449 | INFO | "pinned" : " 29 GB" | 20260507 08:09:22.449 | INFO | } | 20260507 08:09:22.449 | INFO | Perf report for the last 5.14102 seconds: | 20260507 08:09:22.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:09:22.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:22.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:22.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:22.449 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 08:09:22.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:22.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:22.449 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:09:22.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:22.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:22.449 | INFO | Uploaded 36GiB to GPU in 2.01495s (17.8665GiB/s) | 20260507 08:09:22.449 | INFO | Downloaded 34GiB to GPU in 1.9399s (17.5266GiB/s) | 20260507 08:09:27.593 | INFO | Analyzing chunk frames = [8415232,8415744) | 20260507 08:09:27.593 | INFO | MemInfo: { | 20260507 08:09:27.593 | INFO | "gpu" : " 17 GB", | 20260507 08:09:27.593 | INFO | "huge" : "", | 20260507 08:09:27.593 | INFO | "huge_shared" : " 27 GB", | 20260507 08:09:27.593 | INFO | "malloc" : " 14 GB", | 20260507 08:09:27.593 | INFO | "pinned" : " 29 GB" | 20260507 08:09:27.593 | INFO | } | 20260507 08:09:27.593 | INFO | Perf report for the last 5.14415 seconds: | 20260507 08:09:27.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:09:27.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:27.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:27.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:27.593 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 08:09:27.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:27.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:27.593 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:09:27.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:27.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:27.593 | INFO | Uploaded 36GiB to GPU in 2.05119s (17.5508GiB/s) | 20260507 08:09:27.593 | INFO | Downloaded 34GiB to GPU in 1.95172s (17.4206GiB/s) | 20260507 08:09:32.735 | INFO | Analyzing chunk frames = [8415744,8416256) | 20260507 08:09:32.735 | INFO | MemInfo: { | 20260507 08:09:32.735 | INFO | "gpu" : " 17 GB", | 20260507 08:09:32.735 | INFO | "huge" : "", | 20260507 08:09:32.736 | INFO | "huge_shared" : " 27 GB", | 20260507 08:09:32.736 | INFO | "malloc" : " 14 GB", | 20260507 08:09:32.736 | INFO | "pinned" : " 29 GB" | 20260507 08:09:32.736 | INFO | } | 20260507 08:09:32.736 | INFO | Perf report for the last 5.14282 seconds: | 20260507 08:09:32.736 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:09:32.736 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:32.736 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:32.736 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:32.736 | INFO | ANALYSIS : 60.4%, 3.0, 94.4ms, 0 | 20260507 08:09:32.736 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:32.736 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:32.736 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 08:09:32.736 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:32.736 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:32.736 | INFO | Uploaded 36GiB to GPU in 2.06621s (17.4232GiB/s) | 20260507 08:09:32.736 | INFO | Downloaded 34GiB to GPU in 1.93317s (17.5877GiB/s) | 20260507 08:09:37.871 | INFO | Analyzing chunk frames = [8416256,8416768) | 20260507 08:09:37.871 | INFO | MemInfo: { | 20260507 08:09:37.871 | INFO | "gpu" : " 17 GB", | 20260507 08:09:37.871 | INFO | "huge" : "", | 20260507 08:09:37.871 | INFO | "huge_shared" : " 27 GB", | 20260507 08:09:37.871 | INFO | "malloc" : " 14 GB", | 20260507 08:09:37.872 | INFO | "pinned" : " 29 GB" | 20260507 08:09:37.872 | INFO | } | 20260507 08:09:37.872 | INFO | Perf report for the last 5.13591 seconds: | 20260507 08:09:37.872 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:09:37.872 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:37.872 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:37.872 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:37.872 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 08:09:37.872 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:37.872 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:37.872 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 08:09:37.872 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:37.872 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:37.872 | INFO | Uploaded 36GiB to GPU in 1.95313s (18.432GiB/s) | 20260507 08:09:37.872 | INFO | Downloaded 34GiB to GPU in 1.96898s (17.2678GiB/s) | 20260507 08:09:43.005 | INFO | Analyzing chunk frames = [8416768,8417280) | 20260507 08:09:43.005 | INFO | MemInfo: { | 20260507 08:09:43.005 | INFO | "gpu" : " 17 GB", | 20260507 08:09:43.005 | INFO | "huge" : "", | 20260507 08:09:43.005 | INFO | "huge_shared" : " 27 GB", | 20260507 08:09:43.005 | INFO | "malloc" : " 14 GB", | 20260507 08:09:43.005 | INFO | "pinned" : " 29 GB" | 20260507 08:09:43.005 | INFO | } | 20260507 08:09:43.006 | INFO | Perf report for the last 5.13383 seconds: | 20260507 08:09:43.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:09:43.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:43.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:43.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:43.006 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 08:09:43.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:43.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:43.006 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:09:43.006 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:43.006 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:43.006 | INFO | Uploaded 36GiB to GPU in 1.99525s (18.0429GiB/s) | 20260507 08:09:43.006 | INFO | Downloaded 34GiB to GPU in 1.96217s (17.3278GiB/s) | 20260507 08:09:43.102 | INFO | Starting a round of Baz IO | 20260507 08:09:46.198 | INFO | Populating RTMetrics report | 20260507 08:09:46.198 | INFO | RTMetrics report populated | 20260507 08:09:46.198 | INFO | Opening tmp RTMetrics json file | 20260507 08:09:46.208 | INFO | Serializing RTMetrics to json | 20260507 08:09:46.213 | INFO | Serialization done, swapping tmp file now... | 20260507 08:09:46.213 | INFO | RTMetrics done | 20260507 08:09:46.281 | INFO | Finished a round of Baz IO: Wrote 1.62 GiB in 3.179 seconds | 20260507 08:09:46.282 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.51% / 0.00% / 67.49% | 20260507 08:09:46.282 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0937303% / 3.97364e-06% / 5.16574e-05% | 20260507 08:09:46.282 | INFO | Baz IO overall: | 20260507 08:09:46.282 | INFO | BytesWritten : 4200263834537 | 20260507 08:09:46.282 | INFO | NumEvents : 2009660836876 | 20260507 08:09:48.148 | INFO | Analyzing chunk frames = [8417280,8417792) | 20260507 08:09:48.148 | INFO | MemInfo: { | 20260507 08:09:48.148 | INFO | "gpu" : " 17 GB", | 20260507 08:09:48.148 | INFO | "huge" : "", | 20260507 08:09:48.148 | INFO | "huge_shared" : " 27 GB", | 20260507 08:09:48.148 | INFO | "malloc" : " 14 GB", | 20260507 08:09:48.148 | INFO | "pinned" : " 29 GB" | 20260507 08:09:48.148 | INFO | } | 20260507 08:09:48.148 | INFO | Perf report for the last 5.1428 seconds: | 20260507 08:09:48.148 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:09:48.148 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:48.148 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:48.148 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:48.148 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 08:09:48.148 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:48.148 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260507 08:09:48.148 | INFO | PRE_HQ : 58.8%, 1.8, 54.2ms, 0 | 20260507 08:09:48.149 | INFO | BAZWRITER : 61.4%, 3.1, 96.9ms, 0 | 20260507 08:09:48.149 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:09:48.149 | INFO | Uploaded 36GiB to GPU in 2.06276s (17.4524GiB/s) | 20260507 08:09:48.149 | INFO | Downloaded 38GiB to GPU in 2.18338s (17.4042GiB/s) | 20260507 08:09:53.285 | INFO | Analyzing chunk frames = [8417792,8418304) | 20260507 08:09:53.285 | INFO | MemInfo: { | 20260507 08:09:53.285 | INFO | "gpu" : " 17 GB", | 20260507 08:09:53.285 | INFO | "huge" : "", | 20260507 08:09:53.285 | INFO | "huge_shared" : " 27 GB", | 20260507 08:09:53.285 | INFO | "malloc" : " 14 GB", | 20260507 08:09:53.285 | INFO | "pinned" : " 29 GB" | 20260507 08:09:53.285 | INFO | } | 20260507 08:09:53.285 | INFO | Perf report for the last 5.13696 seconds: | 20260507 08:09:53.285 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:09:53.285 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:53.285 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:53.285 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:53.285 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 08:09:53.285 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:53.285 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:53.285 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 08:09:53.285 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:53.285 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:53.286 | INFO | Uploaded 36GiB to GPU in 1.98768s (18.1116GiB/s) | 20260507 08:09:53.286 | INFO | Downloaded 34GiB to GPU in 1.93287s (17.5904GiB/s) | 20260507 08:09:58.430 | INFO | Analyzing chunk frames = [8418304,8418816) | 20260507 08:09:58.430 | INFO | MemInfo: { | 20260507 08:09:58.430 | INFO | "gpu" : " 17 GB", | 20260507 08:09:58.430 | INFO | "huge" : "", | 20260507 08:09:58.430 | INFO | "huge_shared" : " 27 GB", | 20260507 08:09:58.430 | INFO | "malloc" : " 14 GB", | 20260507 08:09:58.430 | INFO | "pinned" : " 29 GB" | 20260507 08:09:58.430 | INFO | } | 20260507 08:09:58.430 | INFO | Perf report for the last 5.14508 seconds: | 20260507 08:09:58.430 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:09:58.430 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:58.430 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:58.430 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:58.430 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 08:09:58.430 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:58.430 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:58.431 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:09:58.431 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:58.431 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:09:58.431 | INFO | Uploaded 36GiB to GPU in 2.0511s (17.5516GiB/s) | 20260507 08:09:58.431 | INFO | Downloaded 34GiB to GPU in 1.94954s (17.44GiB/s) | 20260507 08:10:03.568 | INFO | Analyzing chunk frames = [8418816,8419328) | 20260507 08:10:03.568 | INFO | MemInfo: { | 20260507 08:10:03.568 | INFO | "gpu" : " 17 GB", | 20260507 08:10:03.568 | INFO | "huge" : "", | 20260507 08:10:03.568 | INFO | "huge_shared" : " 27 GB", | 20260507 08:10:03.568 | INFO | "malloc" : " 14 GB", | 20260507 08:10:03.568 | INFO | "pinned" : " 29 GB" | 20260507 08:10:03.568 | INFO | } | 20260507 08:10:03.568 | INFO | Perf report for the last 5.13786 seconds: | 20260507 08:10:03.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:10:03.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:03.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:03.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:03.568 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 08:10:03.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:03.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:03.568 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:10:03.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:03.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:03.569 | INFO | Uploaded 36GiB to GPU in 2.0602s (17.474GiB/s) | 20260507 08:10:03.569 | INFO | Downloaded 34GiB to GPU in 1.94478s (17.4827GiB/s) | 20260507 08:10:08.703 | INFO | Analyzing chunk frames = [8419328,8419840) | 20260507 08:10:08.703 | INFO | MemInfo: { | 20260507 08:10:08.703 | INFO | "gpu" : " 17 GB", | 20260507 08:10:08.703 | INFO | "huge" : "", | 20260507 08:10:08.704 | INFO | "huge_shared" : " 27 GB", | 20260507 08:10:08.704 | INFO | "malloc" : " 14 GB", | 20260507 08:10:08.704 | INFO | "pinned" : " 29 GB" | 20260507 08:10:08.704 | INFO | } | 20260507 08:10:08.704 | INFO | Perf report for the last 5.13541 seconds: | 20260507 08:10:08.704 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:10:08.704 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:08.704 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:08.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:08.704 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 08:10:08.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:08.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:08.704 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:10:08.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:08.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:08.704 | INFO | Uploaded 36GiB to GPU in 2.05033s (17.5582GiB/s) | 20260507 08:10:08.704 | INFO | Downloaded 34GiB to GPU in 1.91536s (17.7512GiB/s) | 20260507 08:10:13.848 | INFO | Analyzing chunk frames = [8419840,8420352) | 20260507 08:10:13.848 | INFO | MemInfo: { | 20260507 08:10:13.848 | INFO | "gpu" : " 17 GB", | 20260507 08:10:13.848 | INFO | "huge" : "", | 20260507 08:10:13.848 | INFO | "huge_shared" : " 27 GB", | 20260507 08:10:13.848 | INFO | "malloc" : " 14 GB", | 20260507 08:10:13.848 | INFO | "pinned" : " 29 GB" | 20260507 08:10:13.848 | INFO | } | 20260507 08:10:13.848 | INFO | Perf report for the last 5.14439 seconds: | 20260507 08:10:13.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:10:13.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:13.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:13.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:13.848 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 08:10:13.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:13.848 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:13.848 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 08:10:13.848 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:13.848 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:13.848 | INFO | Uploaded 36GiB to GPU in 2.05989s (17.4767GiB/s) | 20260507 08:10:13.848 | INFO | Downloaded 34GiB to GPU in 1.91499s (17.7547GiB/s) | 20260507 08:10:18.986 | INFO | Analyzing chunk frames = [8420352,8420864) | 20260507 08:10:18.986 | INFO | MemInfo: { | 20260507 08:10:18.986 | INFO | "gpu" : " 17 GB", | 20260507 08:10:18.986 | INFO | "huge" : "", | 20260507 08:10:18.986 | INFO | "huge_shared" : " 27 GB", | 20260507 08:10:18.987 | INFO | "malloc" : " 14 GB", | 20260507 08:10:18.987 | INFO | "pinned" : " 29 GB" | 20260507 08:10:18.987 | INFO | } | 20260507 08:10:18.987 | INFO | Perf report for the last 5.13852 seconds: | 20260507 08:10:18.987 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:10:18.987 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:18.987 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:18.987 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:18.987 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 08:10:18.987 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:18.987 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:18.987 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 08:10:18.987 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:18.987 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:18.987 | INFO | Uploaded 36GiB to GPU in 1.89719s (18.9755GiB/s) | 20260507 08:10:18.987 | INFO | Downloaded 34GiB to GPU in 1.97523s (17.2132GiB/s) | 20260507 08:10:24.119 | INFO | Analyzing chunk frames = [8420864,8421376) | 20260507 08:10:24.119 | INFO | MemInfo: { | 20260507 08:10:24.119 | INFO | "gpu" : " 17 GB", | 20260507 08:10:24.119 | INFO | "huge" : "", | 20260507 08:10:24.119 | INFO | "huge_shared" : " 27 GB", | 20260507 08:10:24.119 | INFO | "malloc" : " 14 GB", | 20260507 08:10:24.119 | INFO | "pinned" : " 29 GB" | 20260507 08:10:24.119 | INFO | } | 20260507 08:10:24.119 | INFO | Perf report for the last 5.13231 seconds: | 20260507 08:10:24.119 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:10:24.119 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:24.119 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:24.119 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:24.119 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 08:10:24.119 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:24.119 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:24.119 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 08:10:24.119 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:24.119 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:24.119 | INFO | Uploaded 36GiB to GPU in 1.97614s (18.2173GiB/s) | 20260507 08:10:24.119 | INFO | Downloaded 34GiB to GPU in 1.96128s (17.3356GiB/s) | 20260507 08:10:24.119 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:10:24.119 | INFO | Total mem usage: 18 GiB | 20260507 08:10:24.119 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:10:24.119 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:10:24.119 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:10:24.119 | INFO | High Water Breakdown: | 20260507 08:10:24.120 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:10:24.120 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:10:24.120 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:10:24.120 | INFO | ------------------------------------------ | 20260507 08:10:24.120 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:10:24.120 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:10:24.120 | INFO | Total mem usage: 26.5 GiB | 20260507 08:10:24.120 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:10:24.120 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:10:24.120 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:10:24.120 | INFO | High Water Breakdown: | 20260507 08:10:24.120 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:10:24.120 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:10:24.120 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:10:24.120 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:10:24.120 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:10:24.120 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:10:24.120 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:10:24.120 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:10:24.120 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:10:24.120 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:10:24.120 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:10:24.120 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:10:24.120 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:10:24.120 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:10:24.120 | INFO | ------------------------------------------ | 20260507 08:10:24.120 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:10:24.120 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:10:24.120 | INFO | Total mem usage: 25.6 GiB | 20260507 08:10:24.120 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:10:24.120 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:10:24.120 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:10:24.120 | INFO | High Water Breakdown: | 20260507 08:10:24.120 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:10:24.120 | INFO | ------------------------------------------ | 20260507 08:10:24.120 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:10:24.120 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:10:24.120 | INFO | Total mem usage: 15.7 GiB | 20260507 08:10:24.120 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:10:24.120 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:10:24.120 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:10:24.120 | INFO | High Water Breakdown: | 20260507 08:10:24.120 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:10:24.120 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:10:24.120 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:10:24.120 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:10:24.120 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:10:24.120 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:10:24.120 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:10:24.120 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:10:24.120 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:10:24.120 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:10:24.120 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:10:24.120 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:10:24.120 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:10:24.120 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:10:24.120 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:10:24.120 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:10:24.120 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:10:24.120 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:10:24.120 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:10:24.120 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:10:24.120 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:10:24.120 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:10:24.120 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:10:24.120 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:10:24.120 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:10:24.120 | INFO | ------------------------------------------ | 20260507 08:10:24.120 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:10:24.120 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:10:24.120 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:10:24.120 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:10:24.120 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.35e+01 / 9.78e-01 | 20260507 08:10:24.120 | INFO | PerfReport: 0.30%, AggregateStats -- 9.05e-02 / 1.04e-01, 9.07e-03 / 5.34e-03 | 20260507 08:10:24.120 | INFO | PerfReport: 11.68%, Baseline -- 3.58e+00 / 4.52e+00, 5.01e-01 / 1.48e-01 | 20260507 08:10:24.120 | INFO | PerfReport: 3.22%, Histogram -- 9.87e-01 / 1.53e+00, 1.66e-01 / 4.57e-02 | 20260507 08:10:24.120 | INFO | PerfReport: 16.31%, DME -- 4.00e+01 / 4.49e+01, 3.85e+00 / 1.30e+00 | 20260507 08:10:24.120 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.26e+00 / 1.27e+00 | 20260507 08:10:24.120 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.69e+00, 3.86e-01 / 1.57e-01 | 20260507 08:10:24.120 | INFO | PerfReport: 2.92%, Metrics -- 8.94e-01 / 1.20e+00, 2.41e-01 / 1.51e-01 | 20260507 08:10:24.120 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.60e-01, 5.01e-02 / 2.08e-02 | 20260507 08:10:24.121 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:10:24.121 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:10:24.121 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:10:24.121 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 08:10:24.121 | INFO | PerfReport: 21.85%, Upload -- 2.10e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 08:10:24.121 | INFO | PerfReport: 21.38%, Download -- 2.05e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260507 08:10:24.121 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 08:10:24.121 | INFO | PerfReport: 1.59%, LockUpload -- 1.52e+00 / 0.00e+00, 5.01e+00 / 0.00e+00 | 20260507 08:10:24.121 | INFO | PerfReport: 1.25%, LockDownload -- 1.21e+00 / 0.00e+00, 2.96e+00 / 0.00e+00 | 20260507 08:10:24.121 | INFO | PerfReport: 22.00%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 08:10:24.121 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260507 08:10:24.217 | INFO | Starting a round of Baz IO | 20260507 08:10:27.303 | INFO | Populating RTMetrics report | 20260507 08:10:27.303 | INFO | RTMetrics report populated | 20260507 08:10:27.303 | INFO | Opening tmp RTMetrics json file | 20260507 08:10:27.320 | INFO | Serializing RTMetrics to json | 20260507 08:10:27.321 | INFO | Serialization done, swapping tmp file now... | 20260507 08:10:27.322 | INFO | RTMetrics done | 20260507 08:10:27.389 | INFO | Finished a round of Baz IO: Wrote 1.619 GiB in 3.172 seconds | 20260507 08:10:27.389 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.51% / 0.00% / 67.49% | 20260507 08:10:27.389 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0933727% / 0% / 5.96046e-05% | 20260507 08:10:27.389 | INFO | Baz IO overall: | 20260507 08:10:27.389 | INFO | BytesWritten : 4202002104152 | 20260507 08:10:27.389 | INFO | NumEvents : 2010470732171 | 20260507 08:10:29.255 | INFO | Analyzing chunk frames = [8421376,8421888) | 20260507 08:10:29.255 | INFO | MemInfo: { | 20260507 08:10:29.255 | INFO | "gpu" : " 17 GB", | 20260507 08:10:29.255 | INFO | "huge" : "", | 20260507 08:10:29.255 | INFO | "huge_shared" : " 27 GB", | 20260507 08:10:29.255 | INFO | "malloc" : " 14 GB", | 20260507 08:10:29.255 | INFO | "pinned" : " 29 GB" | 20260507 08:10:29.255 | INFO | } | 20260507 08:10:29.255 | INFO | Perf report for the last 5.13577 seconds: | 20260507 08:10:29.255 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:10:29.255 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:29.255 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:29.255 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:29.255 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260507 08:10:29.255 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:29.255 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 08:10:29.255 | INFO | PRE_HQ : 58.9%, 1.8, 54.5ms, 0 | 20260507 08:10:29.255 | INFO | BAZWRITER : 61.4%, 3.1, 98.4ms, 0 | 20260507 08:10:29.255 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:10:29.255 | INFO | Uploaded 36GiB to GPU in 2.04916s (17.5682GiB/s) | 20260507 08:10:29.255 | INFO | Downloaded 38GiB to GPU in 2.17771s (17.4495GiB/s) | 20260507 08:10:34.403 | INFO | Analyzing chunk frames = [8421888,8422400) | 20260507 08:10:34.403 | INFO | MemInfo: { | 20260507 08:10:34.403 | INFO | "gpu" : " 17 GB", | 20260507 08:10:34.403 | INFO | "huge" : "", | 20260507 08:10:34.403 | INFO | "huge_shared" : " 27 GB", | 20260507 08:10:34.403 | INFO | "malloc" : " 14 GB", | 20260507 08:10:34.403 | INFO | "pinned" : " 29 GB" | 20260507 08:10:34.403 | INFO | } | 20260507 08:10:34.403 | INFO | Perf report for the last 5.14823 seconds: | 20260507 08:10:34.403 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:10:34.403 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:34.403 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:34.403 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:34.403 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:10:34.403 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:34.403 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:34.403 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:10:34.403 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:34.403 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:34.403 | INFO | Uploaded 36GiB to GPU in 1.99181s (18.074GiB/s) | 20260507 08:10:34.403 | INFO | Downloaded 34GiB to GPU in 1.93598s (17.5622GiB/s) | 20260507 08:10:39.546 | INFO | Analyzing chunk frames = [8422400,8422912) | 20260507 08:10:39.546 | INFO | MemInfo: { | 20260507 08:10:39.546 | INFO | "gpu" : " 17 GB", | 20260507 08:10:39.546 | INFO | "huge" : "", | 20260507 08:10:39.546 | INFO | "huge_shared" : " 27 GB", | 20260507 08:10:39.546 | INFO | "malloc" : " 14 GB", | 20260507 08:10:39.546 | INFO | "pinned" : " 29 GB" | 20260507 08:10:39.546 | INFO | } | 20260507 08:10:39.546 | INFO | Perf report for the last 5.14329 seconds: | 20260507 08:10:39.546 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:10:39.546 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:39.546 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:39.546 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:39.546 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 08:10:39.546 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:39.546 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:39.546 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:10:39.546 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:39.546 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:39.547 | INFO | Uploaded 36GiB to GPU in 2.04715s (17.5855GiB/s) | 20260507 08:10:39.547 | INFO | Downloaded 34GiB to GPU in 1.95226s (17.4157GiB/s) | 20260507 08:10:44.681 | INFO | Analyzing chunk frames = [8422912,8423424) | 20260507 08:10:44.681 | INFO | MemInfo: { | 20260507 08:10:44.681 | INFO | "gpu" : " 17 GB", | 20260507 08:10:44.681 | INFO | "huge" : "", | 20260507 08:10:44.681 | INFO | "huge_shared" : " 27 GB", | 20260507 08:10:44.681 | INFO | "malloc" : " 14 GB", | 20260507 08:10:44.681 | INFO | "pinned" : " 29 GB" | 20260507 08:10:44.681 | INFO | } | 20260507 08:10:44.681 | INFO | Perf report for the last 5.13503 seconds: | 20260507 08:10:44.681 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:10:44.681 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:44.681 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:44.681 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:44.681 | INFO | ANALYSIS : 60.2%, 2.9, 92.9ms, 0 | 20260507 08:10:44.681 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:44.681 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:44.681 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:10:44.681 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:44.681 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:44.682 | INFO | Uploaded 36GiB to GPU in 1.98642s (18.123GiB/s) | 20260507 08:10:44.682 | INFO | Downloaded 34GiB to GPU in 1.94712s (17.4617GiB/s) | 20260507 08:10:49.816 | INFO | Analyzing chunk frames = [8423424,8423936) | 20260507 08:10:49.816 | INFO | MemInfo: { | 20260507 08:10:49.816 | INFO | "gpu" : " 17 GB", | 20260507 08:10:49.816 | INFO | "huge" : "", | 20260507 08:10:49.816 | INFO | "huge_shared" : " 27 GB", | 20260507 08:10:49.816 | INFO | "malloc" : " 14 GB", | 20260507 08:10:49.816 | INFO | "pinned" : " 29 GB" | 20260507 08:10:49.816 | INFO | } | 20260507 08:10:49.816 | INFO | Perf report for the last 5.13475 seconds: | 20260507 08:10:49.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:10:49.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:49.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:49.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:49.816 | INFO | ANALYSIS : 60.1%, 3.0, 93.5ms, 0 | 20260507 08:10:49.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:49.816 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:49.816 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:10:49.816 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:49.816 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:49.816 | INFO | Uploaded 36GiB to GPU in 2.05506s (17.5178GiB/s) | 20260507 08:10:49.816 | INFO | Downloaded 34GiB to GPU in 1.90868s (17.8133GiB/s) | 20260507 08:10:54.958 | INFO | Analyzing chunk frames = [8423936,8424448) | 20260507 08:10:54.958 | INFO | MemInfo: { | 20260507 08:10:54.958 | INFO | "gpu" : " 17 GB", | 20260507 08:10:54.958 | INFO | "huge" : "", | 20260507 08:10:54.958 | INFO | "huge_shared" : " 27 GB", | 20260507 08:10:54.958 | INFO | "malloc" : " 14 GB", | 20260507 08:10:54.958 | INFO | "pinned" : " 29 GB" | 20260507 08:10:54.958 | INFO | } | 20260507 08:10:54.958 | INFO | Perf report for the last 5.14229 seconds: | 20260507 08:10:54.958 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:10:54.958 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:54.958 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:54.958 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:54.958 | INFO | ANALYSIS : 60.2%, 3.0, 94.1ms, 0 | 20260507 08:10:54.958 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:54.958 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:54.958 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:10:54.958 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:54.958 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:10:54.959 | INFO | Uploaded 36GiB to GPU in 2.03755s (17.6683GiB/s) | 20260507 08:10:54.959 | INFO | Downloaded 34GiB to GPU in 1.91636s (17.742GiB/s) | 20260507 08:11:00.098 | INFO | Analyzing chunk frames = [8424448,8424960) | 20260507 08:11:00.098 | INFO | MemInfo: { | 20260507 08:11:00.098 | INFO | "gpu" : " 17 GB", | 20260507 08:11:00.098 | INFO | "huge" : "", | 20260507 08:11:00.098 | INFO | "huge_shared" : " 27 GB", | 20260507 08:11:00.098 | INFO | "malloc" : " 14 GB", | 20260507 08:11:00.098 | INFO | "pinned" : " 29 GB" | 20260507 08:11:00.098 | INFO | } | 20260507 08:11:00.098 | INFO | Perf report for the last 5.13927 seconds: | 20260507 08:11:00.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:11:00.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:00.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:00.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:00.098 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:11:00.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:00.098 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:00.098 | INFO | PRE_HQ : 25.5%, 1.0, 13.3ms, 0 | 20260507 08:11:00.098 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:00.098 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:00.098 | INFO | Uploaded 36GiB to GPU in 1.97501s (18.2278GiB/s) | 20260507 08:11:00.098 | INFO | Downloaded 34GiB to GPU in 1.97082s (17.2517GiB/s) | 20260507 08:11:05.240 | INFO | Analyzing chunk frames = [8424960,8425472) | 20260507 08:11:05.240 | INFO | MemInfo: { | 20260507 08:11:05.240 | INFO | "gpu" : " 17 GB", | 20260507 08:11:05.240 | INFO | "huge" : "", | 20260507 08:11:05.240 | INFO | "huge_shared" : " 27 GB", | 20260507 08:11:05.240 | INFO | "malloc" : " 14 GB", | 20260507 08:11:05.240 | INFO | "pinned" : " 29 GB" | 20260507 08:11:05.240 | INFO | } | 20260507 08:11:05.240 | INFO | Perf report for the last 5.14217 seconds: | 20260507 08:11:05.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:11:05.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:05.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:05.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:05.240 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:11:05.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:05.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:05.240 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:11:05.240 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:05.240 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:05.240 | INFO | Uploaded 36GiB to GPU in 1.98455s (18.1401GiB/s) | 20260507 08:11:05.240 | INFO | Downloaded 34GiB to GPU in 1.96542s (17.2991GiB/s) | 20260507 08:11:05.337 | INFO | Starting a round of Baz IO | 20260507 08:11:08.425 | INFO | Populating RTMetrics report | 20260507 08:11:08.426 | INFO | RTMetrics report populated | 20260507 08:11:08.426 | INFO | Opening tmp RTMetrics json file | 20260507 08:11:08.432 | INFO | Serializing RTMetrics to json | 20260507 08:11:08.442 | INFO | Serialization done, swapping tmp file now... | 20260507 08:11:08.442 | INFO | RTMetrics done | 20260507 08:11:08.509 | INFO | Finished a round of Baz IO: Wrote 1.619 GiB in 3.171 seconds | 20260507 08:11:08.509 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.51% / 0.00% / 67.49% | 20260507 08:11:08.509 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0936111% / 3.97364e-06% / 7.54992e-05% | 20260507 08:11:08.509 | INFO | Baz IO overall: | 20260507 08:11:08.509 | INFO | BytesWritten : 4203740572626 | 20260507 08:11:08.509 | INFO | NumEvents : 2011280980984 | 20260507 08:11:10.377 | INFO | Analyzing chunk frames = [8425472,8425984) | 20260507 08:11:10.377 | INFO | MemInfo: { | 20260507 08:11:10.377 | INFO | "gpu" : " 17 GB", | 20260507 08:11:10.377 | INFO | "huge" : "", | 20260507 08:11:10.377 | INFO | "huge_shared" : " 27 GB", | 20260507 08:11:10.377 | INFO | "malloc" : " 14 GB", | 20260507 08:11:10.377 | INFO | "pinned" : " 29 GB" | 20260507 08:11:10.377 | INFO | } | 20260507 08:11:10.377 | INFO | Perf report for the last 5.13738 seconds: | 20260507 08:11:10.377 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:11:10.377 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:10.377 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:10.377 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:10.377 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 08:11:10.377 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:10.378 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 08:11:10.378 | INFO | PRE_HQ : 58.8%, 1.8, 54.5ms, 0 | 20260507 08:11:10.378 | INFO | BAZWRITER : 61.2%, 3.1, 96.6ms, 0 | 20260507 08:11:10.378 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:11:10.378 | INFO | Uploaded 36GiB to GPU in 2.05263s (17.5384GiB/s) | 20260507 08:11:10.378 | INFO | Downloaded 38GiB to GPU in 2.17824s (17.4453GiB/s) | 20260507 08:11:15.514 | INFO | Analyzing chunk frames = [8425984,8426496) | 20260507 08:11:15.514 | INFO | MemInfo: { | 20260507 08:11:15.514 | INFO | "gpu" : " 17 GB", | 20260507 08:11:15.514 | INFO | "huge" : "", | 20260507 08:11:15.514 | INFO | "huge_shared" : " 27 GB", | 20260507 08:11:15.514 | INFO | "malloc" : " 14 GB", | 20260507 08:11:15.514 | INFO | "pinned" : " 29 GB" | 20260507 08:11:15.515 | INFO | } | 20260507 08:11:15.515 | INFO | Perf report for the last 5.13736 seconds: | 20260507 08:11:15.515 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:11:15.515 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:15.515 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:15.515 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:15.515 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:11:15.515 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:15.515 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:15.515 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:11:15.515 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:15.515 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:15.515 | INFO | Uploaded 36GiB to GPU in 1.99409s (18.0533GiB/s) | 20260507 08:11:15.515 | INFO | Downloaded 34GiB to GPU in 1.93946s (17.5306GiB/s) | 20260507 08:11:20.654 | INFO | Analyzing chunk frames = [8426496,8427008) | 20260507 08:11:20.654 | INFO | MemInfo: { | 20260507 08:11:20.654 | INFO | "gpu" : " 17 GB", | 20260507 08:11:20.654 | INFO | "huge" : "", | 20260507 08:11:20.654 | INFO | "huge_shared" : " 27 GB", | 20260507 08:11:20.654 | INFO | "malloc" : " 14 GB", | 20260507 08:11:20.654 | INFO | "pinned" : " 29 GB" | 20260507 08:11:20.654 | INFO | } | 20260507 08:11:20.654 | INFO | Perf report for the last 5.1393 seconds: | 20260507 08:11:20.654 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:11:20.654 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:20.654 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:20.654 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:20.654 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 08:11:20.654 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:20.654 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:20.654 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:11:20.654 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:20.654 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:20.654 | INFO | Uploaded 36GiB to GPU in 2.04522s (17.602GiB/s) | 20260507 08:11:20.654 | INFO | Downloaded 34GiB to GPU in 1.95226s (17.4157GiB/s) | 20260507 08:11:25.796 | INFO | Analyzing chunk frames = [8427008,8427520) | 20260507 08:11:25.796 | INFO | MemInfo: { | 20260507 08:11:25.796 | INFO | "gpu" : " 17 GB", | 20260507 08:11:25.796 | INFO | "huge" : "", | 20260507 08:11:25.796 | INFO | "huge_shared" : " 27 GB", | 20260507 08:11:25.796 | INFO | "malloc" : " 14 GB", | 20260507 08:11:25.796 | INFO | "pinned" : " 29 GB" | 20260507 08:11:25.796 | INFO | } | 20260507 08:11:25.796 | INFO | Perf report for the last 5.14217 seconds: | 20260507 08:11:25.796 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:11:25.796 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:25.796 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:25.796 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:25.796 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 08:11:25.796 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:25.796 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:25.796 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:11:25.796 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:25.796 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:25.796 | INFO | Uploaded 36GiB to GPU in 2.01053s (17.9058GiB/s) | 20260507 08:11:25.796 | INFO | Downloaded 34GiB to GPU in 1.9465s (17.4672GiB/s) | 20260507 08:11:30.942 | INFO | Analyzing chunk frames = [8427520,8428032) | 20260507 08:11:30.942 | INFO | MemInfo: { | 20260507 08:11:30.942 | INFO | "gpu" : " 17 GB", | 20260507 08:11:30.942 | INFO | "huge" : "", | 20260507 08:11:30.942 | INFO | "huge_shared" : " 27 GB", | 20260507 08:11:30.942 | INFO | "malloc" : " 14 GB", | 20260507 08:11:30.942 | INFO | "pinned" : " 29 GB" | 20260507 08:11:30.942 | INFO | } | 20260507 08:11:30.942 | INFO | Perf report for the last 5.14625 seconds: | 20260507 08:11:30.942 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:11:30.942 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:30.942 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:30.942 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:30.942 | INFO | ANALYSIS : 59.9%, 3.0, 93.5ms, 0 | 20260507 08:11:30.942 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:30.942 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:30.942 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:11:30.942 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:30.942 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:30.943 | INFO | Uploaded 36GiB to GPU in 2.02121s (17.8111GiB/s) | 20260507 08:11:30.943 | INFO | Downloaded 34GiB to GPU in 1.92529s (17.6596GiB/s) | 20260507 08:11:36.071 | INFO | Analyzing chunk frames = [8428032,8428544) | 20260507 08:11:36.071 | INFO | MemInfo: { | 20260507 08:11:36.071 | INFO | "gpu" : " 17 GB", | 20260507 08:11:36.071 | INFO | "huge" : "", | 20260507 08:11:36.071 | INFO | "huge_shared" : " 27 GB", | 20260507 08:11:36.071 | INFO | "malloc" : " 14 GB", | 20260507 08:11:36.071 | INFO | "pinned" : " 29 GB" | 20260507 08:11:36.071 | INFO | } | 20260507 08:11:36.071 | INFO | Perf report for the last 5.12888 seconds: | 20260507 08:11:36.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:11:36.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:36.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:36.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:36.071 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:11:36.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:36.071 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:36.071 | INFO | PRE_HQ : 26.9%, 1.0, 13.9ms, 0 | 20260507 08:11:36.071 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:36.071 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:36.071 | INFO | Uploaded 36GiB to GPU in 2.02603s (17.7687GiB/s) | 20260507 08:11:36.071 | INFO | Downloaded 34GiB to GPU in 1.93178s (17.6003GiB/s) | 20260507 08:11:41.206 | INFO | Analyzing chunk frames = [8428544,8429056) | 20260507 08:11:41.206 | INFO | MemInfo: { | 20260507 08:11:41.206 | INFO | "gpu" : " 17 GB", | 20260507 08:11:41.206 | INFO | "huge" : "", | 20260507 08:11:41.206 | INFO | "huge_shared" : " 27 GB", | 20260507 08:11:41.206 | INFO | "malloc" : " 14 GB", | 20260507 08:11:41.206 | INFO | "pinned" : " 29 GB" | 20260507 08:11:41.206 | INFO | } | 20260507 08:11:41.206 | INFO | Perf report for the last 5.13504 seconds: | 20260507 08:11:41.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:11:41.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:41.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:41.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:41.206 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 08:11:41.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:41.206 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:41.206 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 08:11:41.206 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:41.206 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:41.207 | INFO | Uploaded 36GiB to GPU in 1.96864s (18.2868GiB/s) | 20260507 08:11:41.207 | INFO | Downloaded 34GiB to GPU in 1.98074s (17.1653GiB/s) | 20260507 08:11:46.353 | INFO | Analyzing chunk frames = [8429056,8429568) | 20260507 08:11:46.353 | INFO | MemInfo: { | 20260507 08:11:46.353 | INFO | "gpu" : " 17 GB", | 20260507 08:11:46.353 | INFO | "huge" : "", | 20260507 08:11:46.353 | INFO | "huge_shared" : " 27 GB", | 20260507 08:11:46.353 | INFO | "malloc" : " 14 GB", | 20260507 08:11:46.353 | INFO | "pinned" : " 29 GB" | 20260507 08:11:46.353 | INFO | } | 20260507 08:11:46.353 | INFO | Perf report for the last 5.14675 seconds: | 20260507 08:11:46.353 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:11:46.353 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:46.353 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:46.353 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:46.353 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 08:11:46.353 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:46.353 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:46.353 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 08:11:46.353 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:46.353 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:46.353 | INFO | Uploaded 36GiB to GPU in 1.97664s (18.2127GiB/s) | 20260507 08:11:46.353 | INFO | Downloaded 34GiB to GPU in 1.94387s (17.4909GiB/s) | 20260507 08:11:46.353 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:11:46.353 | INFO | Total mem usage: 18 GiB | 20260507 08:11:46.353 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:11:46.353 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:11:46.353 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:11:46.353 | INFO | High Water Breakdown: | 20260507 08:11:46.353 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:11:46.353 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:11:46.353 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:11:46.353 | INFO | ------------------------------------------ | 20260507 08:11:46.353 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:11:46.353 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:11:46.353 | INFO | Total mem usage: 26.5 GiB | 20260507 08:11:46.353 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:11:46.354 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:11:46.354 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:11:46.354 | INFO | High Water Breakdown: | 20260507 08:11:46.354 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:11:46.354 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:11:46.354 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:11:46.354 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:11:46.354 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:11:46.354 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:11:46.354 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:11:46.354 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:11:46.354 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:11:46.354 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:11:46.354 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:11:46.354 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:11:46.354 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:11:46.354 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:11:46.354 | INFO | ------------------------------------------ | 20260507 08:11:46.354 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:11:46.354 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:11:46.354 | INFO | Total mem usage: 25.6 GiB | 20260507 08:11:46.354 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:11:46.354 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:11:46.354 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:11:46.354 | INFO | High Water Breakdown: | 20260507 08:11:46.354 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:11:46.354 | INFO | ------------------------------------------ | 20260507 08:11:46.354 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:11:46.354 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:11:46.354 | INFO | Total mem usage: 15.7 GiB | 20260507 08:11:46.354 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:11:46.354 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:11:46.354 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:11:46.354 | INFO | High Water Breakdown: | 20260507 08:11:46.354 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:11:46.354 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:11:46.354 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:11:46.354 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:11:46.354 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:11:46.354 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:11:46.354 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:11:46.354 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:11:46.354 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:11:46.354 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:11:46.354 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:11:46.354 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:11:46.354 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:11:46.354 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:11:46.354 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:11:46.354 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:11:46.354 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:11:46.354 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:11:46.354 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:11:46.354 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:11:46.354 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:11:46.354 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:11:46.354 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:11:46.354 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:11:46.354 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:11:46.354 | INFO | ------------------------------------------ | 20260507 08:11:46.354 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:11:46.354 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:11:46.354 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:11:46.354 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:11:46.354 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.35e+01 / 1.21e+00 | 20260507 08:11:46.354 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 1.04e-02 / 7.11e-03 | 20260507 08:11:46.354 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.45e+00, 5.09e-01 / 2.77e-01 | 20260507 08:11:46.354 | INFO | PerfReport: 3.23%, Histogram -- 9.90e-01 / 1.48e+00, 1.67e-01 / 1.16e-01 | 20260507 08:11:46.354 | INFO | PerfReport: 16.28%, DME -- 4.00e+01 / 4.48e+01, 3.91e+00 / 1.62e+00 | 20260507 08:11:46.354 | INFO | PerfReport: 57.76%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.32e+00 / 1.54e+00 | 20260507 08:11:46.354 | INFO | PerfReport: 6.87%, PulseAccumulating -- 2.11e+00 / 2.69e+00, 3.84e-01 / 1.93e-01 | 20260507 08:11:46.354 | INFO | PerfReport: 2.91%, Metrics -- 8.94e-01 / 1.20e+00, 2.42e-01 / 1.36e-01 | 20260507 08:11:46.354 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.99e-01 / 3.55e-01, 5.07e-02 / 1.96e-02 | 20260507 08:11:46.354 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:11:46.354 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:11:46.354 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:11:46.354 | INFO | PerfReport: Full Execution -- 9.60e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 08:11:46.354 | INFO | PerfReport: 21.84%, Upload -- 2.10e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260507 08:11:46.354 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.35e+00 / 0.00e+00 | 20260507 08:11:46.354 | INFO | PerfReport: 31.97%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 08:11:46.354 | INFO | PerfReport: 1.44%, LockUpload -- 1.38e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 08:11:46.354 | INFO | PerfReport: 1.36%, LockDownload -- 1.31e+00 / 0.00e+00, 3.19e+00 / 0.00e+00 | 20260507 08:11:46.354 | INFO | PerfReport: 21.97%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 08:11:46.354 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.77e-02 / 0.00e+00 | 20260507 08:11:46.457 | INFO | Starting a round of Baz IO | 20260507 08:11:49.540 | INFO | Populating RTMetrics report | 20260507 08:11:49.540 | INFO | RTMetrics report populated | 20260507 08:11:49.540 | INFO | Opening tmp RTMetrics json file | 20260507 08:11:49.557 | INFO | Serializing RTMetrics to json | 20260507 08:11:49.558 | INFO | Serialization done, swapping tmp file now... | 20260507 08:11:49.559 | INFO | RTMetrics done | 20260507 08:11:49.622 | INFO | Finished a round of Baz IO: Wrote 1.62 GiB in 3.165 seconds | 20260507 08:11:49.623 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.51% / 0.00% / 67.49% | 20260507 08:11:49.623 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0932773% / 3.97364e-06% / 5.5631e-05% | 20260507 08:11:49.623 | INFO | Baz IO overall: | 20260507 08:11:49.623 | INFO | BytesWritten : 4205479915187 | 20260507 08:11:49.623 | INFO | NumEvents : 2012092174734 | 20260507 08:11:51.499 | INFO | Analyzing chunk frames = [8429568,8430080) | 20260507 08:11:51.499 | INFO | MemInfo: { | 20260507 08:11:51.499 | INFO | "gpu" : " 17 GB", | 20260507 08:11:51.499 | INFO | "huge" : "", | 20260507 08:11:51.499 | INFO | "huge_shared" : " 27 GB", | 20260507 08:11:51.499 | INFO | "malloc" : " 14 GB", | 20260507 08:11:51.499 | INFO | "pinned" : " 29 GB" | 20260507 08:11:51.499 | INFO | } | 20260507 08:11:51.499 | INFO | Perf report for the last 5.14627 seconds: | 20260507 08:11:51.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:11:51.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:51.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:51.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:51.499 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260507 08:11:51.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:51.499 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 08:11:51.499 | INFO | PRE_HQ : 58.7%, 1.8, 54.6ms, 0 | 20260507 08:11:51.499 | INFO | BAZWRITER : 61.1%, 3.2, 99.4ms, 0 | 20260507 08:11:51.500 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:11:51.500 | INFO | Uploaded 36GiB to GPU in 2.05584s (17.5111GiB/s) | 20260507 08:11:51.500 | INFO | Downloaded 38GiB to GPU in 2.16088s (17.5854GiB/s) | 20260507 08:11:56.628 | INFO | Analyzing chunk frames = [8430080,8430592) | 20260507 08:11:56.628 | INFO | MemInfo: { | 20260507 08:11:56.628 | INFO | "gpu" : " 17 GB", | 20260507 08:11:56.628 | INFO | "huge" : "", | 20260507 08:11:56.628 | INFO | "huge_shared" : " 27 GB", | 20260507 08:11:56.628 | INFO | "malloc" : " 14 GB", | 20260507 08:11:56.628 | INFO | "pinned" : " 29 GB" | 20260507 08:11:56.628 | INFO | } | 20260507 08:11:56.628 | INFO | Perf report for the last 5.12829 seconds: | 20260507 08:11:56.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:11:56.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:56.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:56.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:56.628 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 08:11:56.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:56.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:56.628 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:11:56.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:56.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:11:56.628 | INFO | Uploaded 36GiB to GPU in 2.03322s (17.7059GiB/s) | 20260507 08:11:56.628 | INFO | Downloaded 34GiB to GPU in 1.95352s (17.4045GiB/s) | 20260507 08:12:01.768 | INFO | Analyzing chunk frames = [8430592,8431104) | 20260507 08:12:01.768 | INFO | MemInfo: { | 20260507 08:12:01.768 | INFO | "gpu" : " 17 GB", | 20260507 08:12:01.768 | INFO | "huge" : "", | 20260507 08:12:01.769 | INFO | "huge_shared" : " 27 GB", | 20260507 08:12:01.769 | INFO | "malloc" : " 14 GB", | 20260507 08:12:01.769 | INFO | "pinned" : " 29 GB" | 20260507 08:12:01.769 | INFO | } | 20260507 08:12:01.769 | INFO | Perf report for the last 5.14099 seconds: | 20260507 08:12:01.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:12:01.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:01.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:01.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:01.769 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 08:12:01.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:01.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:01.769 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:12:01.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:01.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:01.769 | INFO | Uploaded 36GiB to GPU in 2.05627s (17.5075GiB/s) | 20260507 08:12:01.769 | INFO | Downloaded 34GiB to GPU in 1.95005s (17.4355GiB/s) | 20260507 08:12:06.909 | INFO | Analyzing chunk frames = [8431104,8431616) | 20260507 08:12:06.909 | INFO | MemInfo: { | 20260507 08:12:06.909 | INFO | "gpu" : " 17 GB", | 20260507 08:12:06.909 | INFO | "huge" : "", | 20260507 08:12:06.909 | INFO | "huge_shared" : " 27 GB", | 20260507 08:12:06.909 | INFO | "malloc" : " 14 GB", | 20260507 08:12:06.909 | INFO | "pinned" : " 29 GB" | 20260507 08:12:06.909 | INFO | } | 20260507 08:12:06.909 | INFO | Perf report for the last 5.14038 seconds: | 20260507 08:12:06.909 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:12:06.909 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:06.909 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:06.909 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:06.909 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 08:12:06.909 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:06.909 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:06.909 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:12:06.909 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:06.909 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:06.909 | INFO | Uploaded 36GiB to GPU in 2.01879s (17.8325GiB/s) | 20260507 08:12:06.909 | INFO | Downloaded 34GiB to GPU in 1.94313s (17.4975GiB/s) | 20260507 08:12:12.055 | INFO | Analyzing chunk frames = [8431616,8432128) | 20260507 08:12:12.055 | INFO | MemInfo: { | 20260507 08:12:12.055 | INFO | "gpu" : " 17 GB", | 20260507 08:12:12.056 | INFO | "huge" : "", | 20260507 08:12:12.056 | INFO | "huge_shared" : " 27 GB", | 20260507 08:12:12.056 | INFO | "malloc" : " 14 GB", | 20260507 08:12:12.056 | INFO | "pinned" : " 29 GB" | 20260507 08:12:12.056 | INFO | } | 20260507 08:12:12.056 | INFO | Perf report for the last 5.14653 seconds: | 20260507 08:12:12.056 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:12:12.056 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:12.056 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:12.056 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:12.056 | INFO | ANALYSIS : 60.0%, 3.0, 93.8ms, 0 | 20260507 08:12:12.056 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:12.056 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:12.056 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:12:12.056 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:12.056 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:12.056 | INFO | Uploaded 36GiB to GPU in 2.05844s (17.489GiB/s) | 20260507 08:12:12.056 | INFO | Downloaded 34GiB to GPU in 1.94722s (17.4608GiB/s) | 20260507 08:12:17.194 | INFO | Analyzing chunk frames = [8432128,8432640) | 20260507 08:12:17.194 | INFO | MemInfo: { | 20260507 08:12:17.194 | INFO | "gpu" : " 17 GB", | 20260507 08:12:17.194 | INFO | "huge" : "", | 20260507 08:12:17.195 | INFO | "huge_shared" : " 27 GB", | 20260507 08:12:17.195 | INFO | "malloc" : " 14 GB", | 20260507 08:12:17.195 | INFO | "pinned" : " 29 GB" | 20260507 08:12:17.195 | INFO | } | 20260507 08:12:17.195 | INFO | Perf report for the last 5.139 seconds: | 20260507 08:12:17.195 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:12:17.195 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:17.195 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:17.195 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:17.195 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:12:17.195 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:17.195 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:17.195 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:12:17.195 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:17.195 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:17.195 | INFO | Uploaded 36GiB to GPU in 2.04216s (17.6284GiB/s) | 20260507 08:12:17.195 | INFO | Downloaded 34GiB to GPU in 1.93437s (17.5767GiB/s) | 20260507 08:12:22.324 | INFO | Analyzing chunk frames = [8432640,8433152) | 20260507 08:12:22.324 | INFO | MemInfo: { | 20260507 08:12:22.324 | INFO | "gpu" : " 17 GB", | 20260507 08:12:22.324 | INFO | "huge" : "", | 20260507 08:12:22.324 | INFO | "huge_shared" : " 27 GB", | 20260507 08:12:22.324 | INFO | "malloc" : " 14 GB", | 20260507 08:12:22.324 | INFO | "pinned" : " 29 GB" | 20260507 08:12:22.324 | INFO | } | 20260507 08:12:22.324 | INFO | Perf report for the last 5.12987 seconds: | 20260507 08:12:22.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:12:22.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:22.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:22.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:22.325 | INFO | ANALYSIS : 60.5%, 3.0, 94.0ms, 0 | 20260507 08:12:22.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:22.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:22.325 | INFO | PRE_HQ : 26.3%, 1.0, 13.6ms, 0 | 20260507 08:12:22.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:22.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:22.325 | INFO | Uploaded 36GiB to GPU in 1.88964s (19.0513GiB/s) | 20260507 08:12:22.325 | INFO | Downloaded 34GiB to GPU in 1.97204s (17.241GiB/s) | 20260507 08:12:27.468 | INFO | Analyzing chunk frames = [8433152,8433664) | 20260507 08:12:27.468 | INFO | MemInfo: { | 20260507 08:12:27.468 | INFO | "gpu" : " 17 GB", | 20260507 08:12:27.468 | INFO | "huge" : "", | 20260507 08:12:27.468 | INFO | "huge_shared" : " 27 GB", | 20260507 08:12:27.468 | INFO | "malloc" : " 14 GB", | 20260507 08:12:27.468 | INFO | "pinned" : " 29 GB" | 20260507 08:12:27.468 | INFO | } | 20260507 08:12:27.468 | INFO | Perf report for the last 5.14334 seconds: | 20260507 08:12:27.468 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:12:27.468 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:27.468 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:27.468 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:27.468 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:12:27.468 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:27.468 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:27.468 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 08:12:27.468 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:27.468 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:27.468 | INFO | Uploaded 36GiB to GPU in 1.99019s (18.0887GiB/s) | 20260507 08:12:27.468 | INFO | Downloaded 34GiB to GPU in 1.95566s (17.3855GiB/s) | 20260507 08:12:27.565 | INFO | Starting a round of Baz IO | 20260507 08:12:30.655 | INFO | Populating RTMetrics report | 20260507 08:12:30.655 | INFO | RTMetrics report populated | 20260507 08:12:30.655 | INFO | Opening tmp RTMetrics json file | 20260507 08:12:30.664 | INFO | Serializing RTMetrics to json | 20260507 08:12:30.671 | INFO | Serialization done, swapping tmp file now... | 20260507 08:12:30.671 | INFO | RTMetrics done | 20260507 08:12:30.738 | INFO | Finished a round of Baz IO: Wrote 1.623 GiB in 3.172 seconds | 20260507 08:12:30.738 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.51% / 0.00% / 67.49% | 20260507 08:12:30.738 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0940959% / 3.97364e-06% / 6.75519e-05% | 20260507 08:12:30.738 | INFO | Baz IO overall: | 20260507 08:12:30.738 | INFO | BytesWritten : 4207222338892 | 20260507 08:12:30.738 | INFO | NumEvents : 2012904080995 | 20260507 08:12:32.606 | INFO | Analyzing chunk frames = [8433664,8434176) | 20260507 08:12:32.606 | INFO | MemInfo: { | 20260507 08:12:32.606 | INFO | "gpu" : " 17 GB", | 20260507 08:12:32.606 | INFO | "huge" : "", | 20260507 08:12:32.606 | INFO | "huge_shared" : " 27 GB", | 20260507 08:12:32.606 | INFO | "malloc" : " 14 GB", | 20260507 08:12:32.606 | INFO | "pinned" : " 29 GB" | 20260507 08:12:32.606 | INFO | } | 20260507 08:12:32.606 | INFO | Perf report for the last 5.13856 seconds: | 20260507 08:12:32.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:12:32.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:32.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:32.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:32.606 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 08:12:32.607 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:32.607 | INFO | RT_METRICS : 41.4%, 1.0, 21.7ms, 0 | 20260507 08:12:32.607 | INFO | PRE_HQ : 58.9%, 1.8, 54.7ms, 0 | 20260507 08:12:32.607 | INFO | BAZWRITER : 61.4%, 3.2, 99.6ms, 0 | 20260507 08:12:32.607 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:12:32.607 | INFO | Uploaded 36GiB to GPU in 2.04521s (17.6021GiB/s) | 20260507 08:12:32.607 | INFO | Downloaded 38GiB to GPU in 2.1699s (17.5123GiB/s) | 20260507 08:12:37.748 | INFO | Analyzing chunk frames = [8434176,8434688) | 20260507 08:12:37.748 | INFO | MemInfo: { | 20260507 08:12:37.748 | INFO | "gpu" : " 17 GB", | 20260507 08:12:37.748 | INFO | "huge" : "", | 20260507 08:12:37.748 | INFO | "huge_shared" : " 27 GB", | 20260507 08:12:37.748 | INFO | "malloc" : " 14 GB", | 20260507 08:12:37.748 | INFO | "pinned" : " 29 GB" | 20260507 08:12:37.748 | INFO | } | 20260507 08:12:37.748 | INFO | Perf report for the last 5.14132 seconds: | 20260507 08:12:37.748 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:12:37.748 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:37.748 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:37.748 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:37.748 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:12:37.748 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:37.748 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:37.748 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:12:37.748 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:37.748 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:37.748 | INFO | Uploaded 36GiB to GPU in 1.98578s (18.1289GiB/s) | 20260507 08:12:37.748 | INFO | Downloaded 34GiB to GPU in 1.92703s (17.6437GiB/s) | 20260507 08:12:42.889 | INFO | Analyzing chunk frames = [8434688,8435200) | 20260507 08:12:42.889 | INFO | MemInfo: { | 20260507 08:12:42.889 | INFO | "gpu" : " 17 GB", | 20260507 08:12:42.889 | INFO | "huge" : "", | 20260507 08:12:42.889 | INFO | "huge_shared" : " 27 GB", | 20260507 08:12:42.889 | INFO | "malloc" : " 14 GB", | 20260507 08:12:42.889 | INFO | "pinned" : " 29 GB" | 20260507 08:12:42.889 | INFO | } | 20260507 08:12:42.889 | INFO | Perf report for the last 5.14172 seconds: | 20260507 08:12:42.889 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:12:42.889 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:42.889 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:42.890 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:42.890 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 08:12:42.890 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:42.890 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:42.890 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:12:42.890 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:42.890 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:42.890 | INFO | Uploaded 36GiB to GPU in 2.03639s (17.6784GiB/s) | 20260507 08:12:42.890 | INFO | Downloaded 34GiB to GPU in 1.91684s (17.7375GiB/s) | 20260507 08:12:48.022 | INFO | Analyzing chunk frames = [8435200,8435712) | 20260507 08:12:48.022 | INFO | MemInfo: { | 20260507 08:12:48.022 | INFO | "gpu" : " 17 GB", | 20260507 08:12:48.022 | INFO | "huge" : "", | 20260507 08:12:48.022 | INFO | "huge_shared" : " 27 GB", | 20260507 08:12:48.022 | INFO | "malloc" : " 14 GB", | 20260507 08:12:48.022 | INFO | "pinned" : " 29 GB" | 20260507 08:12:48.022 | INFO | } | 20260507 08:12:48.023 | INFO | Perf report for the last 5.13304 seconds: | 20260507 08:12:48.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:12:48.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:48.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:48.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:48.023 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 08:12:48.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:48.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:48.023 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:12:48.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:48.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:48.023 | INFO | Uploaded 36GiB to GPU in 2.05946s (17.4803GiB/s) | 20260507 08:12:48.023 | INFO | Downloaded 34GiB to GPU in 1.9488s (17.4466GiB/s) | 20260507 08:12:53.164 | INFO | Analyzing chunk frames = [8435712,8436224) | 20260507 08:12:53.164 | INFO | MemInfo: { | 20260507 08:12:53.164 | INFO | "gpu" : " 17 GB", | 20260507 08:12:53.164 | INFO | "huge" : "", | 20260507 08:12:53.164 | INFO | "huge_shared" : " 27 GB", | 20260507 08:12:53.164 | INFO | "malloc" : " 14 GB", | 20260507 08:12:53.164 | INFO | "pinned" : " 29 GB" | 20260507 08:12:53.164 | INFO | } | 20260507 08:12:53.164 | INFO | Perf report for the last 5.14174 seconds: | 20260507 08:12:53.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:12:53.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:53.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:53.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:53.164 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 08:12:53.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:53.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:53.164 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:12:53.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:53.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:53.164 | INFO | Uploaded 36GiB to GPU in 2.01666s (17.8513GiB/s) | 20260507 08:12:53.165 | INFO | Downloaded 34GiB to GPU in 1.9109s (17.7927GiB/s) | 20260507 08:12:58.307 | INFO | Analyzing chunk frames = [8436224,8436736) | 20260507 08:12:58.307 | INFO | MemInfo: { | 20260507 08:12:58.307 | INFO | "gpu" : " 17 GB", | 20260507 08:12:58.307 | INFO | "huge" : "", | 20260507 08:12:58.307 | INFO | "huge_shared" : " 27 GB", | 20260507 08:12:58.307 | INFO | "malloc" : " 14 GB", | 20260507 08:12:58.307 | INFO | "pinned" : " 29 GB" | 20260507 08:12:58.307 | INFO | } | 20260507 08:12:58.307 | INFO | Perf report for the last 5.14262 seconds: | 20260507 08:12:58.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:12:58.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:58.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:58.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:58.307 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 08:12:58.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:58.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:58.307 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:12:58.307 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:58.307 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:12:58.307 | INFO | Uploaded 36GiB to GPU in 2.05774s (17.4949GiB/s) | 20260507 08:12:58.307 | INFO | Downloaded 34GiB to GPU in 1.91037s (17.7976GiB/s) | 20260507 08:13:03.443 | INFO | Analyzing chunk frames = [8436736,8437248) | 20260507 08:13:03.443 | INFO | MemInfo: { | 20260507 08:13:03.443 | INFO | "gpu" : " 17 GB", | 20260507 08:13:03.443 | INFO | "huge" : "", | 20260507 08:13:03.443 | INFO | "huge_shared" : " 27 GB", | 20260507 08:13:03.443 | INFO | "malloc" : " 14 GB", | 20260507 08:13:03.444 | INFO | "pinned" : " 29 GB" | 20260507 08:13:03.444 | INFO | } | 20260507 08:13:03.444 | INFO | Perf report for the last 5.13662 seconds: | 20260507 08:13:03.444 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:13:03.444 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:03.444 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:03.444 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:03.444 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260507 08:13:03.444 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:03.444 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:03.444 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 08:13:03.444 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:03.444 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:03.444 | INFO | Uploaded 36GiB to GPU in 1.88035s (19.1454GiB/s) | 20260507 08:13:03.444 | INFO | Downloaded 34GiB to GPU in 1.96446s (17.3076GiB/s) | 20260507 08:13:08.576 | INFO | Analyzing chunk frames = [8437248,8437760) | 20260507 08:13:08.576 | INFO | MemInfo: { | 20260507 08:13:08.576 | INFO | "gpu" : " 17 GB", | 20260507 08:13:08.576 | INFO | "huge" : "", | 20260507 08:13:08.576 | INFO | "huge_shared" : " 27 GB", | 20260507 08:13:08.576 | INFO | "malloc" : " 14 GB", | 20260507 08:13:08.576 | INFO | "pinned" : " 29 GB" | 20260507 08:13:08.576 | INFO | } | 20260507 08:13:08.576 | INFO | Perf report for the last 5.13221 seconds: | 20260507 08:13:08.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:13:08.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:08.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:08.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:08.576 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 08:13:08.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:08.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:08.576 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:13:08.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:08.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:08.576 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:13:08.576 | INFO | Total mem usage: 18 GiB | 20260507 08:13:08.576 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:13:08.576 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 08:13:08.576 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:13:08.576 | INFO | High Water Breakdown: | 20260507 08:13:08.576 | INFO | Uploaded 36GiB to GPU in 1.9848s (18.1379GiB/s) | 20260507 08:13:08.576 | INFO | Downloaded 34GiB to GPU in 1.9654s (17.2993GiB/s) | 20260507 08:13:08.576 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:13:08.576 | INFO | 39.7% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:13:08.576 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:13:08.576 | INFO | ------------------------------------------ | 20260507 08:13:08.576 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:13:08.576 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:13:08.576 | INFO | Total mem usage: 26.5 GiB | 20260507 08:13:08.576 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:13:08.576 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:13:08.576 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:13:08.576 | INFO | High Water Breakdown: | 20260507 08:13:08.576 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:13:08.576 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:13:08.576 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:13:08.576 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:13:08.576 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:13:08.576 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:13:08.576 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:13:08.576 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:13:08.576 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:13:08.576 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:13:08.576 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:13:08.577 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:13:08.577 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:13:08.577 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:13:08.577 | INFO | ------------------------------------------ | 20260507 08:13:08.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:13:08.577 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:13:08.577 | INFO | Total mem usage: 25.6 GiB | 20260507 08:13:08.577 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:13:08.577 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:13:08.577 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:13:08.577 | INFO | High Water Breakdown: | 20260507 08:13:08.577 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:13:08.577 | INFO | ------------------------------------------ | 20260507 08:13:08.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:13:08.577 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:13:08.577 | INFO | Total mem usage: 15.7 GiB | 20260507 08:13:08.577 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:13:08.577 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:13:08.577 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:13:08.577 | INFO | High Water Breakdown: | 20260507 08:13:08.577 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:13:08.577 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:13:08.577 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:13:08.577 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:13:08.577 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:13:08.577 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:13:08.577 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:13:08.577 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:13:08.577 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:13:08.577 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:13:08.577 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:13:08.577 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:13:08.577 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:13:08.577 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:13:08.577 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:13:08.577 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:13:08.577 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:13:08.577 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:13:08.577 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:13:08.577 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:13:08.577 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:13:08.577 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:13:08.577 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:13:08.577 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:13:08.577 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:13:08.577 | INFO | ------------------------------------------ | 20260507 08:13:08.577 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:13:08.577 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:13:08.577 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:13:08.577 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:13:08.577 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.36e+01 / 1.06e+00 | 20260507 08:13:08.577 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 9.30e-03 / 4.81e-03 | 20260507 08:13:08.577 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.53e+00, 5.04e-01 / 1.26e-01 | 20260507 08:13:08.577 | INFO | PerfReport: 3.23%, Histogram -- 9.89e-01 / 1.51e+00, 1.69e-01 / 9.24e-02 | 20260507 08:13:08.577 | INFO | PerfReport: 16.32%, DME -- 4.00e+01 / 4.49e+01, 3.90e+00 / 1.17e+00 | 20260507 08:13:08.577 | INFO | PerfReport: 57.69%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.24e+00 / 1.20e+00 | 20260507 08:13:08.577 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.12e+00 / 2.70e+00, 3.86e-01 / 1.92e-01 | 20260507 08:13:08.577 | INFO | PerfReport: 2.92%, Metrics -- 8.96e-01 / 1.20e+00, 2.42e-01 / 1.22e-01 | 20260507 08:13:08.577 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.59e-01, 5.13e-02 / 1.90e-02 | 20260507 08:13:08.577 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:13:08.577 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:13:08.577 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:13:08.577 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 08:13:08.578 | INFO | PerfReport: 21.83%, Upload -- 2.10e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260507 08:13:08.578 | INFO | PerfReport: 21.37%, Download -- 2.05e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 08:13:08.578 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.36e+01 / 0.00e+00 | 20260507 08:13:08.578 | INFO | PerfReport: 1.64%, LockUpload -- 1.57e+00 / 0.00e+00, 5.03e+00 / 0.00e+00 | 20260507 08:13:08.578 | INFO | PerfReport: 1.24%, LockDownload -- 1.19e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260507 08:13:08.578 | INFO | PerfReport: 22.00%, LockCompute -- 2.11e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 08:13:08.578 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.70e-02 / 0.00e+00 | 20260507 08:13:08.673 | INFO | Starting a round of Baz IO | 20260507 08:13:11.766 | INFO | Populating RTMetrics report | 20260507 08:13:11.766 | INFO | RTMetrics report populated | 20260507 08:13:11.766 | INFO | Opening tmp RTMetrics json file | 20260507 08:13:11.781 | INFO | Serializing RTMetrics to json | 20260507 08:13:11.782 | INFO | Serialization done, swapping tmp file now... | 20260507 08:13:11.783 | INFO | RTMetrics done | 20260507 08:13:11.849 | INFO | Finished a round of Baz IO: Wrote 1.624 GiB in 3.175 seconds | 20260507 08:13:11.849 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.51% / 0.00% / 67.49% | 20260507 08:13:11.849 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0943065% / 0% / 5.96046e-05% | 20260507 08:13:11.849 | INFO | Baz IO overall: | 20260507 08:13:11.849 | INFO | BytesWritten : 4208966039040 | 20260507 08:13:11.849 | INFO | NumEvents : 2013715652085 | 20260507 08:13:13.719 | INFO | Analyzing chunk frames = [8437760,8438272) | 20260507 08:13:13.719 | INFO | MemInfo: { | 20260507 08:13:13.719 | INFO | "gpu" : " 17 GB", | 20260507 08:13:13.719 | INFO | "huge" : "", | 20260507 08:13:13.719 | INFO | "huge_shared" : " 27 GB", | 20260507 08:13:13.719 | INFO | "malloc" : " 14 GB", | 20260507 08:13:13.719 | INFO | "pinned" : " 29 GB" | 20260507 08:13:13.719 | INFO | } | 20260507 08:13:13.719 | INFO | Perf report for the last 5.14358 seconds: | 20260507 08:13:13.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:13:13.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:13.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:13.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:13.719 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 08:13:13.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:13.719 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 08:13:13.719 | INFO | PRE_HQ : 58.8%, 1.8, 54.4ms, 0 | 20260507 08:13:13.719 | INFO | BAZWRITER : 61.3%, 3.1, 96.8ms, 0 | 20260507 08:13:13.719 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:13:13.720 | INFO | Uploaded 36GiB to GPU in 2.04997s (17.5612GiB/s) | 20260507 08:13:13.720 | INFO | Downloaded 38GiB to GPU in 2.17949s (17.4353GiB/s) | 20260507 08:13:18.859 | INFO | Analyzing chunk frames = [8438272,8438784) | 20260507 08:13:18.859 | INFO | MemInfo: { | 20260507 08:13:18.859 | INFO | "gpu" : " 17 GB", | 20260507 08:13:18.859 | INFO | "huge" : "", | 20260507 08:13:18.859 | INFO | "huge_shared" : " 27 GB", | 20260507 08:13:18.859 | INFO | "malloc" : " 14 GB", | 20260507 08:13:18.859 | INFO | "pinned" : " 29 GB" | 20260507 08:13:18.859 | INFO | } | 20260507 08:13:18.859 | INFO | Perf report for the last 5.13942 seconds: | 20260507 08:13:18.859 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:13:18.859 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:18.859 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:18.859 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:18.859 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 08:13:18.859 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:18.859 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:18.859 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:13:18.859 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:18.859 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:18.859 | INFO | Uploaded 36GiB to GPU in 1.98896s (18.0999GiB/s) | 20260507 08:13:18.859 | INFO | Downloaded 34GiB to GPU in 1.93354s (17.5843GiB/s) | 20260507 08:13:24.005 | INFO | Analyzing chunk frames = [8438784,8439296) | 20260507 08:13:24.005 | INFO | MemInfo: { | 20260507 08:13:24.005 | INFO | "gpu" : " 17 GB", | 20260507 08:13:24.005 | INFO | "huge" : "", | 20260507 08:13:24.005 | INFO | "huge_shared" : " 27 GB", | 20260507 08:13:24.005 | INFO | "malloc" : " 14 GB", | 20260507 08:13:24.005 | INFO | "pinned" : " 29 GB" | 20260507 08:13:24.005 | INFO | } | 20260507 08:13:24.005 | INFO | Perf report for the last 5.14589 seconds: | 20260507 08:13:24.005 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:13:24.005 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:24.005 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:24.005 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:24.005 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 08:13:24.005 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:24.005 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:24.005 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:13:24.005 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:24.005 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:24.005 | INFO | Uploaded 36GiB to GPU in 2.05082s (17.554GiB/s) | 20260507 08:13:24.005 | INFO | Downloaded 34GiB to GPU in 1.95272s (17.4116GiB/s) | 20260507 08:13:29.146 | INFO | Analyzing chunk frames = [8439296,8439808) | 20260507 08:13:29.146 | INFO | MemInfo: { | 20260507 08:13:29.146 | INFO | "gpu" : " 17 GB", | 20260507 08:13:29.146 | INFO | "huge" : "", | 20260507 08:13:29.146 | INFO | "huge_shared" : " 27 GB", | 20260507 08:13:29.146 | INFO | "malloc" : " 14 GB", | 20260507 08:13:29.146 | INFO | "pinned" : " 29 GB" | 20260507 08:13:29.146 | INFO | } | 20260507 08:13:29.146 | INFO | Perf report for the last 5.14164 seconds: | 20260507 08:13:29.146 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:13:29.146 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:29.146 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:29.146 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:29.146 | INFO | ANALYSIS : 60.3%, 2.9, 93.1ms, 0 | 20260507 08:13:29.146 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:29.146 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:29.146 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:13:29.146 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:29.146 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:29.147 | INFO | Uploaded 36GiB to GPU in 2.0296s (17.7375GiB/s) | 20260507 08:13:29.147 | INFO | Downloaded 34GiB to GPU in 1.91801s (17.7267GiB/s) | 20260507 08:13:34.274 | INFO | Analyzing chunk frames = [8439808,8440320) | 20260507 08:13:34.274 | INFO | MemInfo: { | 20260507 08:13:34.274 | INFO | "gpu" : " 17 GB", | 20260507 08:13:34.274 | INFO | "huge" : "", | 20260507 08:13:34.274 | INFO | "huge_shared" : " 27 GB", | 20260507 08:13:34.274 | INFO | "malloc" : " 14 GB", | 20260507 08:13:34.274 | INFO | "pinned" : " 29 GB" | 20260507 08:13:34.274 | INFO | } | 20260507 08:13:34.274 | INFO | Perf report for the last 5.12809 seconds: | 20260507 08:13:34.274 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:13:34.274 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:34.274 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:34.274 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:34.274 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 08:13:34.274 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:34.275 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:34.275 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:13:34.275 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:34.275 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:34.275 | INFO | Uploaded 36GiB to GPU in 2.06002s (17.4756GiB/s) | 20260507 08:13:34.275 | INFO | Downloaded 34GiB to GPU in 1.9105s (17.7964GiB/s) | 20260507 08:13:39.419 | INFO | Analyzing chunk frames = [8440320,8440832) | 20260507 08:13:39.419 | INFO | MemInfo: { | 20260507 08:13:39.419 | INFO | "gpu" : " 17 GB", | 20260507 08:13:39.419 | INFO | "huge" : "", | 20260507 08:13:39.419 | INFO | "huge_shared" : " 27 GB", | 20260507 08:13:39.419 | INFO | "malloc" : " 14 GB", | 20260507 08:13:39.419 | INFO | "pinned" : " 29 GB" | 20260507 08:13:39.419 | INFO | } | 20260507 08:13:39.419 | INFO | Perf report for the last 5.14427 seconds: | 20260507 08:13:39.419 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:13:39.419 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:39.419 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:39.419 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:39.419 | INFO | ANALYSIS : 60.2%, 3.0, 94.1ms, 0 | 20260507 08:13:39.419 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:39.419 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:39.419 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:13:39.419 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:39.419 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:39.419 | INFO | Uploaded 36GiB to GPU in 2.03931s (17.653GiB/s) | 20260507 08:13:39.419 | INFO | Downloaded 34GiB to GPU in 1.90634s (17.8352GiB/s) | 20260507 08:13:44.563 | INFO | Analyzing chunk frames = [8440832,8441344) | 20260507 08:13:44.563 | INFO | MemInfo: { | 20260507 08:13:44.563 | INFO | "gpu" : " 17 GB", | 20260507 08:13:44.563 | INFO | "huge" : "", | 20260507 08:13:44.563 | INFO | "huge_shared" : " 27 GB", | 20260507 08:13:44.563 | INFO | "malloc" : " 14 GB", | 20260507 08:13:44.563 | INFO | "pinned" : " 29 GB" | 20260507 08:13:44.563 | INFO | } | 20260507 08:13:44.563 | INFO | Perf report for the last 5.14447 seconds: | 20260507 08:13:44.563 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:13:44.563 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:44.563 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:44.563 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:44.563 | INFO | ANALYSIS : 60.8%, 3.0, 94.8ms, 0 | 20260507 08:13:44.563 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:44.563 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:44.563 | INFO | PRE_HQ : 26.2%, 1.0, 13.6ms, 0 | 20260507 08:13:44.563 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:44.563 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:44.563 | INFO | Uploaded 36GiB to GPU in 2.02898s (17.7429GiB/s) | 20260507 08:13:44.563 | INFO | Downloaded 34GiB to GPU in 1.98846s (17.0987GiB/s) | 20260507 08:13:49.700 | INFO | Analyzing chunk frames = [8441344,8441856) | 20260507 08:13:49.700 | INFO | MemInfo: { | 20260507 08:13:49.700 | INFO | "gpu" : " 17 GB", | 20260507 08:13:49.700 | INFO | "huge" : "", | 20260507 08:13:49.700 | INFO | "huge_shared" : " 27 GB", | 20260507 08:13:49.700 | INFO | "malloc" : " 14 GB", | 20260507 08:13:49.700 | INFO | "pinned" : " 29 GB" | 20260507 08:13:49.700 | INFO | } | 20260507 08:13:49.700 | INFO | Perf report for the last 5.13704 seconds: | 20260507 08:13:49.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:13:49.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:49.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:49.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:49.700 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 08:13:49.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:49.700 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:49.700 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 08:13:49.700 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:49.700 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:49.700 | INFO | Uploaded 36GiB to GPU in 2.00092s (17.9917GiB/s) | 20260507 08:13:49.701 | INFO | Downloaded 34GiB to GPU in 1.94883s (17.4463GiB/s) | 20260507 08:13:49.797 | INFO | Starting a round of Baz IO | 20260507 08:13:52.885 | INFO | Populating RTMetrics report | 20260507 08:13:52.886 | INFO | RTMetrics report populated | 20260507 08:13:52.886 | INFO | Opening tmp RTMetrics json file | 20260507 08:13:52.888 | INFO | Serializing RTMetrics to json | 20260507 08:13:52.903 | INFO | Serialization done, swapping tmp file now... | 20260507 08:13:52.903 | INFO | RTMetrics done | 20260507 08:13:52.973 | INFO | Finished a round of Baz IO: Wrote 1.623 GiB in 3.176 seconds | 20260507 08:13:52.973 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.51% / 0.00% / 67.49% | 20260507 08:13:52.973 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0943502% / 3.97364e-06% / 5.96046e-05% | 20260507 08:13:52.973 | INFO | Baz IO overall: | 20260507 08:13:52.973 | INFO | BytesWritten : 4210708902172 | 20260507 08:13:52.973 | INFO | NumEvents : 2014527643406 | 20260507 08:13:54.834 | INFO | Analyzing chunk frames = [8441856,8442368) | 20260507 08:13:54.834 | INFO | MemInfo: { | 20260507 08:13:54.834 | INFO | "gpu" : " 17 GB", | 20260507 08:13:54.834 | INFO | "huge" : "", | 20260507 08:13:54.834 | INFO | "huge_shared" : " 27 GB", | 20260507 08:13:54.834 | INFO | "malloc" : " 14 GB", | 20260507 08:13:54.834 | INFO | "pinned" : " 29 GB" | 20260507 08:13:54.834 | INFO | } | 20260507 08:13:54.834 | INFO | Perf report for the last 5.13386 seconds: | 20260507 08:13:54.834 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:13:54.834 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:54.834 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:54.834 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:54.834 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260507 08:13:54.834 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:54.834 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260507 08:13:54.834 | INFO | PRE_HQ : 58.9%, 1.8, 54.3ms, 0 | 20260507 08:13:54.834 | INFO | BAZWRITER : 61.5%, 3.1, 96.3ms, 0 | 20260507 08:13:54.834 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 08:13:54.834 | INFO | Uploaded 36GiB to GPU in 2.07795s (17.3247GiB/s) | 20260507 08:13:54.834 | INFO | Downloaded 38GiB to GPU in 2.17141s (17.5002GiB/s) | 20260507 08:13:59.973 | INFO | Analyzing chunk frames = [8442368,8442880) | 20260507 08:13:59.973 | INFO | MemInfo: { | 20260507 08:13:59.973 | INFO | "gpu" : " 17 GB", | 20260507 08:13:59.973 | INFO | "huge" : "", | 20260507 08:13:59.973 | INFO | "huge_shared" : " 27 GB", | 20260507 08:13:59.973 | INFO | "malloc" : " 14 GB", | 20260507 08:13:59.973 | INFO | "pinned" : " 29 GB" | 20260507 08:13:59.973 | INFO | } | 20260507 08:13:59.973 | INFO | Perf report for the last 5.1388 seconds: | 20260507 08:13:59.973 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:13:59.973 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:59.973 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:59.973 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:59.973 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 08:13:59.973 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:59.973 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:59.973 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:13:59.973 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:59.973 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:13:59.973 | INFO | Uploaded 36GiB to GPU in 1.98455s (18.1402GiB/s) | 20260507 08:13:59.973 | INFO | Downloaded 34GiB to GPU in 1.92859s (17.6295GiB/s) | 20260507 08:14:05.116 | INFO | Analyzing chunk frames = [8442880,8443392) | 20260507 08:14:05.116 | INFO | MemInfo: { | 20260507 08:14:05.116 | INFO | "gpu" : " 17 GB", | 20260507 08:14:05.116 | INFO | "huge" : "", | 20260507 08:14:05.116 | INFO | "huge_shared" : " 27 GB", | 20260507 08:14:05.116 | INFO | "malloc" : " 14 GB", | 20260507 08:14:05.116 | INFO | "pinned" : " 29 GB" | 20260507 08:14:05.116 | INFO | } | 20260507 08:14:05.116 | INFO | Perf report for the last 5.14288 seconds: | 20260507 08:14:05.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:14:05.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:05.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:05.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:05.116 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260507 08:14:05.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:05.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:05.116 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:14:05.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:05.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:05.116 | INFO | Uploaded 36GiB to GPU in 2.05378s (17.5287GiB/s) | 20260507 08:14:05.116 | INFO | Downloaded 34GiB to GPU in 1.95181s (17.4198GiB/s) | 20260507 08:14:10.256 | INFO | Analyzing chunk frames = [8443392,8443904) | 20260507 08:14:10.256 | INFO | MemInfo: { | 20260507 08:14:10.256 | INFO | "gpu" : " 17 GB", | 20260507 08:14:10.256 | INFO | "huge" : "", | 20260507 08:14:10.256 | INFO | "huge_shared" : " 27 GB", | 20260507 08:14:10.256 | INFO | "malloc" : " 14 GB", | 20260507 08:14:10.256 | INFO | "pinned" : " 29 GB" | 20260507 08:14:10.256 | INFO | } | 20260507 08:14:10.256 | INFO | Perf report for the last 5.14055 seconds: | 20260507 08:14:10.256 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:14:10.256 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:10.256 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:10.256 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:10.256 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 08:14:10.256 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:10.256 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:10.256 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:14:10.256 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:10.256 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:10.257 | INFO | Uploaded 36GiB to GPU in 2.05851s (17.4884GiB/s) | 20260507 08:14:10.257 | INFO | Downloaded 34GiB to GPU in 1.94877s (17.4469GiB/s) | 20260507 08:14:15.389 | INFO | Analyzing chunk frames = [8443904,8444416) | 20260507 08:14:15.389 | INFO | MemInfo: { | 20260507 08:14:15.389 | INFO | "gpu" : " 17 GB", | 20260507 08:14:15.389 | INFO | "huge" : "", | 20260507 08:14:15.389 | INFO | "huge_shared" : " 27 GB", | 20260507 08:14:15.389 | INFO | "malloc" : " 14 GB", | 20260507 08:14:15.389 | INFO | "pinned" : " 29 GB" | 20260507 08:14:15.389 | INFO | } | 20260507 08:14:15.389 | INFO | Perf report for the last 5.13236 seconds: | 20260507 08:14:15.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:14:15.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:15.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:15.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:15.389 | INFO | ANALYSIS : 60.2%, 3.0, 93.5ms, 0 | 20260507 08:14:15.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:15.389 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:15.389 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:14:15.389 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:15.389 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:15.389 | INFO | Uploaded 36GiB to GPU in 2.05199s (17.5439GiB/s) | 20260507 08:14:15.389 | INFO | Downloaded 34GiB to GPU in 1.90425s (17.8548GiB/s) | 20260507 08:14:20.528 | INFO | Analyzing chunk frames = [8444416,8444928) | 20260507 08:14:20.528 | INFO | MemInfo: { | 20260507 08:14:20.528 | INFO | "gpu" : " 17 GB", | 20260507 08:14:20.528 | INFO | "huge" : "", | 20260507 08:14:20.528 | INFO | "huge_shared" : " 27 GB", | 20260507 08:14:20.528 | INFO | "malloc" : " 14 GB", | 20260507 08:14:20.528 | INFO | "pinned" : " 29 GB" | 20260507 08:14:20.528 | INFO | } | 20260507 08:14:20.528 | INFO | Perf report for the last 5.1397 seconds: | 20260507 08:14:20.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:14:20.528 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:20.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:20.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:20.529 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 08:14:20.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:20.529 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:20.529 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:14:20.529 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:20.529 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:20.529 | INFO | Uploaded 36GiB to GPU in 2.05002s (17.5608GiB/s) | 20260507 08:14:20.529 | INFO | Downloaded 34GiB to GPU in 1.90694s (17.8296GiB/s) | 20260507 08:14:25.670 | INFO | Analyzing chunk frames = [8444928,8445440) | 20260507 08:14:25.670 | INFO | MemInfo: { | 20260507 08:14:25.670 | INFO | "gpu" : " 17 GB", | 20260507 08:14:25.670 | INFO | "huge" : "", | 20260507 08:14:25.670 | INFO | "huge_shared" : " 27 GB", | 20260507 08:14:25.670 | INFO | "malloc" : " 14 GB", | 20260507 08:14:25.670 | INFO | "pinned" : " 29 GB" | 20260507 08:14:25.670 | INFO | } | 20260507 08:14:25.671 | INFO | Perf report for the last 5.14202 seconds: | 20260507 08:14:25.671 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:14:25.671 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:25.671 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:25.671 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:25.671 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 08:14:25.671 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:25.671 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:25.671 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 08:14:25.671 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:25.671 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:25.671 | INFO | Uploaded 36GiB to GPU in 1.97726s (18.2071GiB/s) | 20260507 08:14:25.671 | INFO | Downloaded 34GiB to GPU in 1.97112s (17.2491GiB/s) | 20260507 08:14:30.811 | INFO | Analyzing chunk frames = [8445440,8445952) | 20260507 08:14:30.811 | INFO | MemInfo: { | 20260507 08:14:30.811 | INFO | "gpu" : " 17 GB", | 20260507 08:14:30.811 | INFO | "huge" : "", | 20260507 08:14:30.811 | INFO | "huge_shared" : " 27 GB", | 20260507 08:14:30.811 | INFO | "malloc" : " 14 GB", | 20260507 08:14:30.811 | INFO | "pinned" : " 29 GB" | 20260507 08:14:30.811 | INFO | } | 20260507 08:14:30.811 | INFO | Perf report for the last 5.1407 seconds: | 20260507 08:14:30.811 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:14:30.811 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:30.811 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:30.811 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:30.811 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:14:30.811 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:30.811 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:30.811 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:14:30.811 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:30.811 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:30.812 | INFO | Uploaded 36GiB to GPU in 1.9821s (18.1625GiB/s) | 20260507 08:14:30.812 | INFO | Downloaded 34GiB to GPU in 1.96121s (17.3362GiB/s) | 20260507 08:14:30.812 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:14:30.812 | INFO | Total mem usage: 18 GiB | 20260507 08:14:30.812 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:14:30.812 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -27%) | 20260507 08:14:30.812 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:14:30.812 | INFO | High Water Breakdown: | 20260507 08:14:30.812 | INFO | 60.1% : 8.52 GiB ( 74.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:14:30.812 | INFO | 39.7% : 5.63 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:14:30.812 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:14:30.812 | INFO | ------------------------------------------ | 20260507 08:14:30.812 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:14:30.812 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:14:30.812 | INFO | Total mem usage: 26.5 GiB | 20260507 08:14:30.812 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:14:30.812 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:14:30.812 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:14:30.812 | INFO | High Water Breakdown: | 20260507 08:14:30.812 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:14:30.812 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:14:30.812 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:14:30.812 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:14:30.812 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:14:30.812 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:14:30.812 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:14:30.812 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:14:30.812 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:14:30.812 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:14:30.812 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:14:30.812 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:14:30.812 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:14:30.812 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:14:30.812 | INFO | ------------------------------------------ | 20260507 08:14:30.812 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:14:30.812 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:14:30.812 | INFO | Total mem usage: 25.6 GiB | 20260507 08:14:30.812 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:14:30.812 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:14:30.812 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:14:30.812 | INFO | High Water Breakdown: | 20260507 08:14:30.812 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:14:30.812 | INFO | ------------------------------------------ | 20260507 08:14:30.812 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:14:30.812 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:14:30.812 | INFO | Total mem usage: 15.7 GiB | 20260507 08:14:30.812 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:14:30.812 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:14:30.812 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:14:30.812 | INFO | High Water Breakdown: | 20260507 08:14:30.812 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:14:30.812 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:14:30.812 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:14:30.812 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:14:30.812 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:14:30.812 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:14:30.812 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:14:30.812 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:14:30.812 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:14:30.812 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:14:30.812 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:14:30.812 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:14:30.812 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:14:30.812 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:14:30.812 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:14:30.812 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:14:30.812 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:14:30.812 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:14:30.812 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:14:30.812 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:14:30.812 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:14:30.812 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:14:30.812 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:14:30.812 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:14:30.812 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:14:30.812 | INFO | ------------------------------------------ | 20260507 08:14:30.812 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:14:30.812 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:14:30.812 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:14:30.812 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:14:30.812 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.44e+01, 1.35e+01 / 1.03e+00 | 20260507 08:14:30.812 | INFO | PerfReport: 0.30%, AggregateStats -- 9.04e-02 / 1.02e-01, 9.92e-03 / 5.00e-03 | 20260507 08:14:30.813 | INFO | PerfReport: 11.66%, Baseline -- 3.57e+00 / 4.46e+00, 4.92e-01 / 2.57e-01 | 20260507 08:14:30.813 | INFO | PerfReport: 3.21%, Histogram -- 9.85e-01 / 1.49e+00, 1.63e-01 / 9.46e-02 | 20260507 08:14:30.813 | INFO | PerfReport: 16.33%, DME -- 4.00e+01 / 4.47e+01, 3.89e+00 / 1.27e+00 | 20260507 08:14:30.813 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.25e+00 / 1.12e+00 | 20260507 08:14:30.813 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.11e+00 / 2.68e+00, 3.82e-01 / 2.17e-01 | 20260507 08:14:30.813 | INFO | PerfReport: 2.92%, Metrics -- 8.94e-01 / 1.21e+00, 2.38e-01 / 1.19e-01 | 20260507 08:14:30.813 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.63e-01, 5.17e-02 / 2.48e-02 | 20260507 08:14:30.813 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:14:30.813 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:14:30.813 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:14:30.813 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260507 08:14:30.813 | INFO | PerfReport: 21.99%, Upload -- 2.11e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 08:14:30.813 | INFO | PerfReport: 21.31%, Download -- 2.05e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260507 08:14:30.813 | INFO | PerfReport: 31.87%, Compute -- 3.06e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 08:14:30.813 | INFO | PerfReport: 1.67%, LockUpload -- 1.60e+00 / 0.00e+00, 5.08e+00 / 0.00e+00 | 20260507 08:14:30.813 | INFO | PerfReport: 1.23%, LockDownload -- 1.19e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260507 08:14:30.813 | INFO | PerfReport: 21.91%, LockCompute -- 2.11e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260507 08:14:30.813 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.24e-02 / 0.00e+00 | 20260507 08:14:30.908 | INFO | Starting a round of Baz IO | 20260507 08:14:34.008 | INFO | Populating RTMetrics report | 20260507 08:14:34.009 | INFO | RTMetrics report populated | 20260507 08:14:34.009 | INFO | Opening tmp RTMetrics json file | 20260507 08:14:34.023 | INFO | Serializing RTMetrics to json | 20260507 08:14:34.024 | INFO | Serialization done, swapping tmp file now... | 20260507 08:14:34.025 | INFO | RTMetrics done | 20260507 08:14:34.093 | INFO | Finished a round of Baz IO: Wrote 1.621 GiB in 3.184 seconds | 20260507 08:14:34.093 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.51% / 0.00% / 67.49% | 20260507 08:14:34.093 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0932813% / 3.97364e-06% / 6.35783e-05% | 20260507 08:14:34.093 | INFO | Baz IO overall: | 20260507 08:14:34.093 | INFO | BytesWritten : 4212448995158 | 20260507 08:14:34.093 | INFO | NumEvents : 2015338066781 | 20260507 08:14:35.953 | INFO | Analyzing chunk frames = [8445952,8446464) | 20260507 08:14:35.953 | INFO | MemInfo: { | 20260507 08:14:35.953 | INFO | "gpu" : " 17 GB", | 20260507 08:14:35.953 | INFO | "huge" : "", | 20260507 08:14:35.953 | INFO | "huge_shared" : " 27 GB", | 20260507 08:14:35.953 | INFO | "malloc" : " 14 GB", | 20260507 08:14:35.953 | INFO | "pinned" : " 29 GB" | 20260507 08:14:35.953 | INFO | } | 20260507 08:14:35.953 | INFO | Perf report for the last 5.1415 seconds: | 20260507 08:14:35.953 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:14:35.953 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:35.953 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:35.953 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:35.953 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 08:14:35.953 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:35.953 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 08:14:35.953 | INFO | PRE_HQ : 58.8%, 1.8, 54.4ms, 0 | 20260507 08:14:35.953 | INFO | BAZWRITER : 61.5%, 3.1, 97.6ms, 0 | 20260507 08:14:35.953 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 08:14:35.953 | INFO | Uploaded 36GiB to GPU in 2.07061s (17.3862GiB/s) | 20260507 08:14:35.953 | INFO | Downloaded 38GiB to GPU in 2.18473s (17.3934GiB/s) | 20260507 08:14:41.088 | INFO | Analyzing chunk frames = [8446464,8446976) | 20260507 08:14:41.088 | INFO | MemInfo: { | 20260507 08:14:41.088 | INFO | "gpu" : " 17 GB", | 20260507 08:14:41.088 | INFO | "huge" : "", | 20260507 08:14:41.088 | INFO | "huge_shared" : " 27 GB", | 20260507 08:14:41.088 | INFO | "malloc" : " 14 GB", | 20260507 08:14:41.088 | INFO | "pinned" : " 29 GB" | 20260507 08:14:41.088 | INFO | } | 20260507 08:14:41.088 | INFO | Perf report for the last 5.13574 seconds: | 20260507 08:14:41.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:14:41.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:41.089 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:41.089 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:41.089 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 08:14:41.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:41.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:41.089 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 08:14:41.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:41.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:41.089 | INFO | Uploaded 36GiB to GPU in 2.00172s (17.9845GiB/s) | 20260507 08:14:41.089 | INFO | Downloaded 34GiB to GPU in 1.939s (17.5348GiB/s) | 20260507 08:14:46.228 | INFO | Analyzing chunk frames = [8446976,8447488) | 20260507 08:14:46.228 | INFO | MemInfo: { | 20260507 08:14:46.228 | INFO | "gpu" : " 17 GB", | 20260507 08:14:46.228 | INFO | "huge" : "", | 20260507 08:14:46.228 | INFO | "huge_shared" : " 27 GB", | 20260507 08:14:46.228 | INFO | "malloc" : " 14 GB", | 20260507 08:14:46.228 | INFO | "pinned" : " 29 GB" | 20260507 08:14:46.228 | INFO | } | 20260507 08:14:46.228 | INFO | Perf report for the last 5.13938 seconds: | 20260507 08:14:46.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:14:46.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:46.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:46.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:46.228 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 08:14:46.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:46.228 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:46.228 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:14:46.228 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:46.228 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:46.228 | INFO | Uploaded 36GiB to GPU in 2.05306s (17.5348GiB/s) | 20260507 08:14:46.228 | INFO | Downloaded 34GiB to GPU in 1.95517s (17.3898GiB/s) | 20260507 08:14:51.369 | INFO | Analyzing chunk frames = [8447488,8448000) | 20260507 08:14:51.369 | INFO | MemInfo: { | 20260507 08:14:51.369 | INFO | "gpu" : " 17 GB", | 20260507 08:14:51.369 | INFO | "huge" : "", | 20260507 08:14:51.369 | INFO | "huge_shared" : " 27 GB", | 20260507 08:14:51.369 | INFO | "malloc" : " 14 GB", | 20260507 08:14:51.369 | INFO | "pinned" : " 29 GB" | 20260507 08:14:51.369 | INFO | } | 20260507 08:14:51.369 | INFO | Perf report for the last 5.14114 seconds: | 20260507 08:14:51.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:14:51.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:51.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:51.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:51.369 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 08:14:51.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:51.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:51.369 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:14:51.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:51.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:51.369 | INFO | Uploaded 36GiB to GPU in 2.01003s (17.9102GiB/s) | 20260507 08:14:51.369 | INFO | Downloaded 34GiB to GPU in 1.94737s (17.4594GiB/s) | 20260507 08:14:56.513 | INFO | Analyzing chunk frames = [8448000,8448512) | 20260507 08:14:56.513 | INFO | MemInfo: { | 20260507 08:14:56.513 | INFO | "gpu" : " 17 GB", | 20260507 08:14:56.513 | INFO | "huge" : "", | 20260507 08:14:56.513 | INFO | "huge_shared" : " 27 GB", | 20260507 08:14:56.513 | INFO | "malloc" : " 14 GB", | 20260507 08:14:56.513 | INFO | "pinned" : " 29 GB" | 20260507 08:14:56.513 | INFO | } | 20260507 08:14:56.513 | INFO | Perf report for the last 5.14418 seconds: | 20260507 08:14:56.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:14:56.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:56.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:56.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:56.513 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 08:14:56.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:56.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:56.513 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:14:56.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:56.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:14:56.513 | INFO | Uploaded 36GiB to GPU in 2.04897s (17.5698GiB/s) | 20260507 08:14:56.513 | INFO | Downloaded 34GiB to GPU in 1.95444s (17.3963GiB/s) | 20260507 08:15:01.648 | INFO | Analyzing chunk frames = [8448512,8449024) | 20260507 08:15:01.648 | INFO | MemInfo: { | 20260507 08:15:01.648 | INFO | "gpu" : " 17 GB", | 20260507 08:15:01.648 | INFO | "huge" : "", | 20260507 08:15:01.648 | INFO | "huge_shared" : " 27 GB", | 20260507 08:15:01.648 | INFO | "malloc" : " 14 GB", | 20260507 08:15:01.649 | INFO | "pinned" : " 29 GB" | 20260507 08:15:01.649 | INFO | } | 20260507 08:15:01.649 | INFO | Perf report for the last 5.13531 seconds: | 20260507 08:15:01.649 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:15:01.649 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:01.649 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:01.649 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:01.649 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:15:01.649 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:01.649 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:01.649 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 08:15:01.649 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:01.649 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:01.649 | INFO | Uploaded 36GiB to GPU in 2.02913s (17.7416GiB/s) | 20260507 08:15:01.649 | INFO | Downloaded 34GiB to GPU in 1.93272s (17.5918GiB/s) | 20260507 08:15:06.785 | INFO | Analyzing chunk frames = [8449024,8449536) | 20260507 08:15:06.785 | INFO | MemInfo: { | 20260507 08:15:06.785 | INFO | "gpu" : " 17 GB", | 20260507 08:15:06.785 | INFO | "huge" : "", | 20260507 08:15:06.785 | INFO | "huge_shared" : " 27 GB", | 20260507 08:15:06.785 | INFO | "malloc" : " 14 GB", | 20260507 08:15:06.785 | INFO | "pinned" : " 29 GB" | 20260507 08:15:06.785 | INFO | } | 20260507 08:15:06.785 | INFO | Perf report for the last 5.13673 seconds: | 20260507 08:15:06.785 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:15:06.785 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:06.785 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:06.785 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:06.785 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 08:15:06.785 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:06.785 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:06.785 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 08:15:06.785 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:06.785 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:06.786 | INFO | Uploaded 36GiB to GPU in 1.96138s (18.3544GiB/s) | 20260507 08:15:06.786 | INFO | Downloaded 34GiB to GPU in 1.97046s (17.2549GiB/s) | 20260507 08:15:11.922 | INFO | Analyzing chunk frames = [8449536,8450048) | 20260507 08:15:11.922 | INFO | MemInfo: { | 20260507 08:15:11.922 | INFO | "gpu" : " 17 GB", | 20260507 08:15:11.922 | INFO | "huge" : "", | 20260507 08:15:11.922 | INFO | "huge_shared" : " 27 GB", | 20260507 08:15:11.922 | INFO | "malloc" : " 14 GB", | 20260507 08:15:11.922 | INFO | "pinned" : " 29 GB" | 20260507 08:15:11.922 | INFO | } | 20260507 08:15:11.922 | INFO | Perf report for the last 5.1369 seconds: | 20260507 08:15:11.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:15:11.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:11.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:11.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:11.922 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 08:15:11.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:11.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:11.922 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:15:11.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:11.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:11.922 | INFO | Uploaded 36GiB to GPU in 1.97723s (18.2073GiB/s) | 20260507 08:15:11.922 | INFO | Downloaded 34GiB to GPU in 1.96367s (17.3145GiB/s) | 20260507 08:15:12.019 | INFO | Starting a round of Baz IO | 20260507 08:15:15.110 | INFO | Populating RTMetrics report | 20260507 08:15:15.110 | INFO | RTMetrics report populated | 20260507 08:15:15.110 | INFO | Opening tmp RTMetrics json file | 20260507 08:15:15.122 | INFO | Serializing RTMetrics to json | 20260507 08:15:15.125 | INFO | Serialization done, swapping tmp file now... | 20260507 08:15:15.125 | INFO | RTMetrics done | 20260507 08:15:15.193 | INFO | Finished a round of Baz IO: Wrote 1.62 GiB in 3.173 seconds | 20260507 08:15:15.193 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.51% / 0.00% / 67.49% | 20260507 08:15:15.193 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0930707% / 0% / 5.16574e-05% | 20260507 08:15:15.193 | INFO | Baz IO overall: | 20260507 08:15:15.193 | INFO | BytesWritten : 4214188969446 | 20260507 08:15:15.193 | INFO | NumEvents : 2016148757753 | 20260507 08:15:17.066 | INFO | Analyzing chunk frames = [8450048,8450560) | 20260507 08:15:17.066 | INFO | MemInfo: { | 20260507 08:15:17.066 | INFO | "gpu" : " 17 GB", | 20260507 08:15:17.067 | INFO | "huge" : "", | 20260507 08:15:17.067 | INFO | "huge_shared" : " 27 GB", | 20260507 08:15:17.067 | INFO | "malloc" : " 14 GB", | 20260507 08:15:17.067 | INFO | "pinned" : " 29 GB" | 20260507 08:15:17.067 | INFO | } | 20260507 08:15:17.067 | INFO | Perf report for the last 5.14441 seconds: | 20260507 08:15:17.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:15:17.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:17.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:17.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:17.067 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 08:15:17.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:17.067 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 08:15:17.067 | INFO | PRE_HQ : 58.8%, 1.8, 55.0ms, 0 | 20260507 08:15:17.067 | INFO | BAZWRITER : 61.3%, 3.1, 98.5ms, 0 | 20260507 08:15:17.067 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:15:17.067 | INFO | Uploaded 36GiB to GPU in 2.03959s (17.6506GiB/s) | 20260507 08:15:17.067 | INFO | Downloaded 38GiB to GPU in 2.1828s (17.4088GiB/s) | 20260507 08:15:22.206 | INFO | Analyzing chunk frames = [8450560,8451072) | 20260507 08:15:22.206 | INFO | MemInfo: { | 20260507 08:15:22.206 | INFO | "gpu" : " 17 GB", | 20260507 08:15:22.206 | INFO | "huge" : "", | 20260507 08:15:22.206 | INFO | "huge_shared" : " 27 GB", | 20260507 08:15:22.206 | INFO | "malloc" : " 14 GB", | 20260507 08:15:22.206 | INFO | "pinned" : " 29 GB" | 20260507 08:15:22.206 | INFO | } | 20260507 08:15:22.206 | INFO | Perf report for the last 5.13976 seconds: | 20260507 08:15:22.206 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:15:22.206 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:22.206 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:22.206 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:22.206 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:15:22.206 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:22.207 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:22.207 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:15:22.207 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:22.207 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:22.207 | INFO | Uploaded 36GiB to GPU in 1.99661s (18.0306GiB/s) | 20260507 08:15:22.207 | INFO | Downloaded 34GiB to GPU in 1.93837s (17.5405GiB/s) | 20260507 08:15:27.336 | INFO | Analyzing chunk frames = [8451072,8451584) | 20260507 08:15:27.336 | INFO | MemInfo: { | 20260507 08:15:27.336 | INFO | "gpu" : " 17 GB", | 20260507 08:15:27.336 | INFO | "huge" : "", | 20260507 08:15:27.336 | INFO | "huge_shared" : " 27 GB", | 20260507 08:15:27.336 | INFO | "malloc" : " 14 GB", | 20260507 08:15:27.336 | INFO | "pinned" : " 29 GB" | 20260507 08:15:27.336 | INFO | } | 20260507 08:15:27.336 | INFO | Perf report for the last 5.12963 seconds: | 20260507 08:15:27.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:15:27.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:27.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:27.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:27.336 | INFO | ANALYSIS : 60.6%, 3.0, 93.6ms, 0 | 20260507 08:15:27.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:27.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:27.336 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 08:15:27.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:27.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:27.336 | INFO | Uploaded 36GiB to GPU in 2.05269s (17.538GiB/s) | 20260507 08:15:27.336 | INFO | Downloaded 34GiB to GPU in 1.95356s (17.4041GiB/s) | 20260507 08:15:32.477 | INFO | Analyzing chunk frames = [8451584,8452096) | 20260507 08:15:32.477 | INFO | MemInfo: { | 20260507 08:15:32.477 | INFO | "gpu" : " 17 GB", | 20260507 08:15:32.477 | INFO | "huge" : "", | 20260507 08:15:32.477 | INFO | "huge_shared" : " 27 GB", | 20260507 08:15:32.477 | INFO | "malloc" : " 14 GB", | 20260507 08:15:32.477 | INFO | "pinned" : " 29 GB" | 20260507 08:15:32.477 | INFO | } | 20260507 08:15:32.477 | INFO | Perf report for the last 5.14093 seconds: | 20260507 08:15:32.477 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:15:32.477 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:32.477 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:32.477 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:32.477 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 08:15:32.477 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:32.477 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:32.477 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:15:32.477 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:32.477 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:32.477 | INFO | Uploaded 36GiB to GPU in 1.99797s (18.0182GiB/s) | 20260507 08:15:32.477 | INFO | Downloaded 34GiB to GPU in 1.93984s (17.5272GiB/s) | 20260507 08:15:37.624 | INFO | Analyzing chunk frames = [8452096,8452608) | 20260507 08:15:37.624 | INFO | MemInfo: { | 20260507 08:15:37.624 | INFO | "gpu" : " 17 GB", | 20260507 08:15:37.624 | INFO | "huge" : "", | 20260507 08:15:37.624 | INFO | "huge_shared" : " 27 GB", | 20260507 08:15:37.624 | INFO | "malloc" : " 14 GB", | 20260507 08:15:37.624 | INFO | "pinned" : " 29 GB" | 20260507 08:15:37.624 | INFO | } | 20260507 08:15:37.624 | INFO | Perf report for the last 5.14664 seconds: | 20260507 08:15:37.624 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:15:37.624 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:37.624 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:37.624 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:37.624 | INFO | ANALYSIS : 59.8%, 3.0, 93.5ms, 0 | 20260507 08:15:37.624 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:37.624 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:37.624 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:15:37.624 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:37.624 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:37.624 | INFO | Uploaded 36GiB to GPU in 2.01713s (17.8471GiB/s) | 20260507 08:15:37.624 | INFO | Downloaded 34GiB to GPU in 1.94331s (17.4959GiB/s) | 20260507 08:15:42.768 | INFO | Analyzing chunk frames = [8452608,8453120) | 20260507 08:15:42.768 | INFO | MemInfo: { | 20260507 08:15:42.768 | INFO | "gpu" : " 17 GB", | 20260507 08:15:42.768 | INFO | "huge" : "", | 20260507 08:15:42.768 | INFO | "huge_shared" : " 27 GB", | 20260507 08:15:42.768 | INFO | "malloc" : " 14 GB", | 20260507 08:15:42.768 | INFO | "pinned" : " 29 GB" | 20260507 08:15:42.768 | INFO | } | 20260507 08:15:42.768 | INFO | Perf report for the last 5.14475 seconds: | 20260507 08:15:42.768 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:15:42.768 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:42.768 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:42.768 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:42.768 | INFO | ANALYSIS : 60.4%, 3.0, 94.4ms, 0 | 20260507 08:15:42.768 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:42.768 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:42.769 | INFO | PRE_HQ : 28.2%, 1.0, 14.7ms, 0 | 20260507 08:15:42.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:42.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:42.769 | INFO | Uploaded 36GiB to GPU in 2.0682s (17.4065GiB/s) | 20260507 08:15:42.769 | INFO | Downloaded 34GiB to GPU in 1.94315s (17.4974GiB/s) | 20260507 08:15:47.896 | INFO | Analyzing chunk frames = [8453120,8453632) | 20260507 08:15:47.896 | INFO | MemInfo: { | 20260507 08:15:47.896 | INFO | "gpu" : " 17 GB", | 20260507 08:15:47.896 | INFO | "huge" : "", | 20260507 08:15:47.896 | INFO | "huge_shared" : " 27 GB", | 20260507 08:15:47.896 | INFO | "malloc" : " 14 GB", | 20260507 08:15:47.896 | INFO | "pinned" : " 29 GB" | 20260507 08:15:47.896 | INFO | } | 20260507 08:15:47.896 | INFO | Perf report for the last 5.12742 seconds: | 20260507 08:15:47.896 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:15:47.896 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:47.896 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:47.896 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:47.896 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:15:47.896 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:47.896 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:47.896 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 08:15:47.896 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:47.896 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:47.896 | INFO | Uploaded 36GiB to GPU in 1.95457s (18.4184GiB/s) | 20260507 08:15:47.896 | INFO | Downloaded 34GiB to GPU in 1.96613s (17.2928GiB/s) | 20260507 08:15:53.035 | INFO | Analyzing chunk frames = [8453632,8454144) | 20260507 08:15:53.035 | INFO | MemInfo: { | 20260507 08:15:53.035 | INFO | "gpu" : " 17 GB", | 20260507 08:15:53.035 | INFO | "huge" : "", | 20260507 08:15:53.035 | INFO | "huge_shared" : " 27 GB", | 20260507 08:15:53.035 | INFO | "malloc" : " 14 GB", | 20260507 08:15:53.035 | INFO | "pinned" : " 29 GB" | 20260507 08:15:53.035 | INFO | } | 20260507 08:15:53.035 | INFO | Perf report for the last 5.13941 seconds: | 20260507 08:15:53.035 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:15:53.035 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:53.035 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:53.035 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:53.035 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:15:53.035 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:53.035 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:53.035 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:15:53.035 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:53.035 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:53.036 | INFO | Uploaded 36GiB to GPU in 1.98645s (18.1228GiB/s) | 20260507 08:15:53.036 | INFO | Downloaded 34GiB to GPU in 1.9649s (17.3037GiB/s) | 20260507 08:15:53.036 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:15:53.036 | INFO | Total mem usage: 18 GiB | 20260507 08:15:53.036 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:15:53.036 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:15:53.036 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:15:53.036 | INFO | High Water Breakdown: | 20260507 08:15:53.036 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:15:53.036 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:15:53.036 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:15:53.036 | INFO | ------------------------------------------ | 20260507 08:15:53.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:15:53.036 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:15:53.036 | INFO | Total mem usage: 26.5 GiB | 20260507 08:15:53.036 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:15:53.036 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:15:53.036 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:15:53.036 | INFO | High Water Breakdown: | 20260507 08:15:53.036 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:15:53.036 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:15:53.036 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:15:53.036 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:15:53.036 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:15:53.036 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:15:53.036 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:15:53.036 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:15:53.036 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:15:53.036 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:15:53.036 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:15:53.036 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:15:53.036 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:15:53.036 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:15:53.036 | INFO | ------------------------------------------ | 20260507 08:15:53.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:15:53.036 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:15:53.036 | INFO | Total mem usage: 25.6 GiB | 20260507 08:15:53.036 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:15:53.036 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:15:53.036 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:15:53.036 | INFO | High Water Breakdown: | 20260507 08:15:53.036 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:15:53.036 | INFO | ------------------------------------------ | 20260507 08:15:53.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:15:53.036 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:15:53.036 | INFO | Total mem usage: 15.7 GiB | 20260507 08:15:53.036 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:15:53.036 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:15:53.036 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:15:53.036 | INFO | High Water Breakdown: | 20260507 08:15:53.036 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:15:53.036 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:15:53.036 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:15:53.036 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:15:53.036 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:15:53.036 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:15:53.036 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:15:53.036 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:15:53.036 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:15:53.036 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:15:53.036 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:15:53.036 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:15:53.036 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:15:53.036 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:15:53.036 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:15:53.036 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:15:53.036 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:15:53.036 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:15:53.036 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:15:53.036 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:15:53.036 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:15:53.036 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:15:53.036 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:15:53.036 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:15:53.036 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:15:53.036 | INFO | ------------------------------------------ | 20260507 08:15:53.036 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:15:53.036 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:15:53.036 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:15:53.036 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:15:53.036 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.31e+01 / 1.14e+00 | 20260507 08:15:53.036 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.02e-01, 1.00e-02 / 6.22e-03 | 20260507 08:15:53.036 | INFO | PerfReport: 11.64%, Baseline -- 3.57e+00 / 4.51e+00, 4.97e-01 / 1.96e-01 | 20260507 08:15:53.036 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.49e+00, 1.66e-01 / 9.48e-02 | 20260507 08:15:53.037 | INFO | PerfReport: 16.29%, DME -- 4.00e+01 / 4.46e+01, 3.86e+00 / 1.51e+00 | 20260507 08:15:53.037 | INFO | PerfReport: 57.75%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.29e+00 / 1.41e+00 | 20260507 08:15:53.037 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.11e+00 / 2.64e+00, 3.85e-01 / 2.13e-01 | 20260507 08:15:53.037 | INFO | PerfReport: 2.92%, Metrics -- 8.94e-01 / 1.17e+00, 2.38e-01 / 1.23e-01 | 20260507 08:15:53.037 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.61e-01, 5.19e-02 / 2.82e-02 | 20260507 08:15:53.037 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:15:53.037 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:15:53.037 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:15:53.037 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 08:15:53.037 | INFO | PerfReport: 21.87%, Upload -- 2.10e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260507 08:15:53.037 | INFO | PerfReport: 21.47%, Download -- 2.06e+01 / 0.00e+00, 7.37e+00 / 0.00e+00 | 20260507 08:15:53.037 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 08:15:53.037 | INFO | PerfReport: 1.44%, LockUpload -- 1.38e+00 / 0.00e+00, 4.85e+00 / 0.00e+00 | 20260507 08:15:53.037 | INFO | PerfReport: 1.34%, LockDownload -- 1.29e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260507 08:15:53.037 | INFO | PerfReport: 21.94%, LockCompute -- 2.11e+01 / 0.00e+00, 1.80e+01 / 0.00e+00 | 20260507 08:15:53.037 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260507 08:15:53.132 | INFO | Starting a round of Baz IO | 20260507 08:15:56.224 | INFO | Populating RTMetrics report | 20260507 08:15:56.224 | INFO | RTMetrics report populated | 20260507 08:15:56.224 | INFO | Opening tmp RTMetrics json file | 20260507 08:15:56.239 | INFO | Serializing RTMetrics to json | 20260507 08:15:56.240 | INFO | Serialization done, swapping tmp file now... | 20260507 08:15:56.240 | INFO | RTMetrics done | 20260507 08:15:56.307 | INFO | Finished a round of Baz IO: Wrote 1.62 GiB in 3.174 seconds | 20260507 08:15:56.307 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.51% / 0.00% / 67.49% | 20260507 08:15:56.307 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0934362% / 3.97364e-06% / 5.96046e-05% | 20260507 08:15:56.307 | INFO | Baz IO overall: | 20260507 08:15:56.307 | INFO | BytesWritten : 4215928652647 | 20260507 08:15:56.307 | INFO | NumEvents : 2016959681618 | 20260507 08:15:58.178 | INFO | Analyzing chunk frames = [8454144,8454656) | 20260507 08:15:58.178 | INFO | MemInfo: { | 20260507 08:15:58.178 | INFO | "gpu" : " 17 GB", | 20260507 08:15:58.178 | INFO | "huge" : "", | 20260507 08:15:58.178 | INFO | "huge_shared" : " 27 GB", | 20260507 08:15:58.178 | INFO | "malloc" : " 14 GB", | 20260507 08:15:58.178 | INFO | "pinned" : " 29 GB" | 20260507 08:15:58.178 | INFO | } | 20260507 08:15:58.178 | INFO | Perf report for the last 5.14291 seconds: | 20260507 08:15:58.178 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:15:58.178 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:58.178 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:58.178 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:58.178 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 08:15:58.178 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:15:58.178 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 08:15:58.178 | INFO | PRE_HQ : 58.8%, 1.8, 54.1ms, 0 | 20260507 08:15:58.178 | INFO | BAZWRITER : 61.3%, 3.1, 97.1ms, 0 | 20260507 08:15:58.178 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:15:58.178 | INFO | Uploaded 36GiB to GPU in 2.0533s (17.5328GiB/s) | 20260507 08:15:58.178 | INFO | Downloaded 38GiB to GPU in 2.17886s (17.4403GiB/s) | 20260507 08:16:03.322 | INFO | Analyzing chunk frames = [8454656,8455168) | 20260507 08:16:03.322 | INFO | MemInfo: { | 20260507 08:16:03.322 | INFO | "gpu" : " 17 GB", | 20260507 08:16:03.322 | INFO | "huge" : "", | 20260507 08:16:03.322 | INFO | "huge_shared" : " 27 GB", | 20260507 08:16:03.322 | INFO | "malloc" : " 14 GB", | 20260507 08:16:03.322 | INFO | "pinned" : " 29 GB" | 20260507 08:16:03.322 | INFO | } | 20260507 08:16:03.322 | INFO | Perf report for the last 5.14363 seconds: | 20260507 08:16:03.322 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:16:03.322 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:03.322 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:03.322 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:03.322 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 08:16:03.322 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:03.322 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:03.322 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260507 08:16:03.322 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:03.322 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:03.322 | INFO | Uploaded 36GiB to GPU in 1.9893s (18.0968GiB/s) | 20260507 08:16:03.322 | INFO | Downloaded 34GiB to GPU in 1.93168s (17.6012GiB/s) | 20260507 08:16:08.462 | INFO | Analyzing chunk frames = [8455168,8455680) | 20260507 08:16:08.462 | INFO | MemInfo: { | 20260507 08:16:08.462 | INFO | "gpu" : " 17 GB", | 20260507 08:16:08.462 | INFO | "huge" : "", | 20260507 08:16:08.462 | INFO | "huge_shared" : " 27 GB", | 20260507 08:16:08.462 | INFO | "malloc" : " 14 GB", | 20260507 08:16:08.462 | INFO | "pinned" : " 29 GB" | 20260507 08:16:08.462 | INFO | } | 20260507 08:16:08.462 | INFO | Perf report for the last 5.14013 seconds: | 20260507 08:16:08.462 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:16:08.462 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:08.462 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:08.462 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:08.462 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 08:16:08.462 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:08.462 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:08.462 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:16:08.462 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:08.462 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:08.463 | INFO | Uploaded 36GiB to GPU in 2.03869s (17.6584GiB/s) | 20260507 08:16:08.463 | INFO | Downloaded 34GiB to GPU in 1.91295s (17.7736GiB/s) | 20260507 08:16:13.595 | INFO | Analyzing chunk frames = [8455680,8456192) | 20260507 08:16:13.595 | INFO | MemInfo: { | 20260507 08:16:13.595 | INFO | "gpu" : " 17 GB", | 20260507 08:16:13.595 | INFO | "huge" : "", | 20260507 08:16:13.595 | INFO | "huge_shared" : " 27 GB", | 20260507 08:16:13.595 | INFO | "malloc" : " 14 GB", | 20260507 08:16:13.595 | INFO | "pinned" : " 29 GB" | 20260507 08:16:13.595 | INFO | } | 20260507 08:16:13.595 | INFO | Perf report for the last 5.1332 seconds: | 20260507 08:16:13.595 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:16:13.595 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:13.595 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:13.595 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:13.595 | INFO | ANALYSIS : 60.5%, 2.9, 93.2ms, 0 | 20260507 08:16:13.595 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:13.595 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:13.595 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 08:16:13.595 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:13.595 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:13.595 | INFO | Uploaded 36GiB to GPU in 2.05214s (17.5427GiB/s) | 20260507 08:16:13.595 | INFO | Downloaded 34GiB to GPU in 1.94697s (17.4631GiB/s) | 20260507 08:16:18.739 | INFO | Analyzing chunk frames = [8456192,8456704) | 20260507 08:16:18.739 | INFO | MemInfo: { | 20260507 08:16:18.739 | INFO | "gpu" : " 17 GB", | 20260507 08:16:18.739 | INFO | "huge" : "", | 20260507 08:16:18.739 | INFO | "huge_shared" : " 27 GB", | 20260507 08:16:18.739 | INFO | "malloc" : " 14 GB", | 20260507 08:16:18.739 | INFO | "pinned" : " 29 GB" | 20260507 08:16:18.739 | INFO | } | 20260507 08:16:18.739 | INFO | Perf report for the last 5.14387 seconds: | 20260507 08:16:18.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:16:18.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:18.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:18.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:18.739 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 08:16:18.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:18.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:18.739 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:16:18.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:18.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:18.739 | INFO | Uploaded 36GiB to GPU in 2.03854s (17.6597GiB/s) | 20260507 08:16:18.739 | INFO | Downloaded 34GiB to GPU in 1.91535s (17.7513GiB/s) | 20260507 08:16:23.876 | INFO | Analyzing chunk frames = [8456704,8457216) | 20260507 08:16:23.876 | INFO | MemInfo: { | 20260507 08:16:23.876 | INFO | "gpu" : " 17 GB", | 20260507 08:16:23.876 | INFO | "huge" : "", | 20260507 08:16:23.876 | INFO | "huge_shared" : " 27 GB", | 20260507 08:16:23.876 | INFO | "malloc" : " 14 GB", | 20260507 08:16:23.876 | INFO | "pinned" : " 29 GB" | 20260507 08:16:23.876 | INFO | } | 20260507 08:16:23.877 | INFO | Perf report for the last 5.13744 seconds: | 20260507 08:16:23.877 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:16:23.877 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:23.877 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:23.877 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:23.877 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 08:16:23.877 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:23.877 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:23.877 | INFO | PRE_HQ : 26.6%, 1.0, 13.8ms, 0 | 20260507 08:16:23.877 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:23.877 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:23.877 | INFO | Uploaded 36GiB to GPU in 2.07059s (17.3864GiB/s) | 20260507 08:16:23.877 | INFO | Downloaded 34GiB to GPU in 1.9322s (17.5965GiB/s) | 20260507 08:16:29.017 | INFO | Analyzing chunk frames = [8457216,8457728) | 20260507 08:16:29.017 | INFO | MemInfo: { | 20260507 08:16:29.017 | INFO | "gpu" : " 17 GB", | 20260507 08:16:29.017 | INFO | "huge" : "", | 20260507 08:16:29.017 | INFO | "huge_shared" : " 27 GB", | 20260507 08:16:29.017 | INFO | "malloc" : " 14 GB", | 20260507 08:16:29.017 | INFO | "pinned" : " 29 GB" | 20260507 08:16:29.017 | INFO | } | 20260507 08:16:29.017 | INFO | Perf report for the last 5.14095 seconds: | 20260507 08:16:29.017 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:16:29.017 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:29.018 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:29.018 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:29.018 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:16:29.018 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:29.018 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:29.018 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 08:16:29.018 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:29.018 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:29.018 | INFO | Uploaded 36GiB to GPU in 1.91809s (18.7687GiB/s) | 20260507 08:16:29.018 | INFO | Downloaded 34GiB to GPU in 1.97571s (17.209GiB/s) | 20260507 08:16:34.160 | INFO | Analyzing chunk frames = [8457728,8458240) | 20260507 08:16:34.160 | INFO | MemInfo: { | 20260507 08:16:34.160 | INFO | "gpu" : " 17 GB", | 20260507 08:16:34.160 | INFO | "huge" : "", | 20260507 08:16:34.160 | INFO | "huge_shared" : " 27 GB", | 20260507 08:16:34.160 | INFO | "malloc" : " 14 GB", | 20260507 08:16:34.160 | INFO | "pinned" : " 29 GB" | 20260507 08:16:34.160 | INFO | } | 20260507 08:16:34.161 | INFO | Perf report for the last 5.14305 seconds: | 20260507 08:16:34.161 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:16:34.161 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:34.161 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:34.161 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:34.161 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:16:34.161 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:34.161 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:34.161 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:16:34.161 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:34.161 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:34.161 | INFO | Uploaded 36GiB to GPU in 1.99049s (18.086GiB/s) | 20260507 08:16:34.161 | INFO | Downloaded 34GiB to GPU in 1.9607s (17.3407GiB/s) | 20260507 08:16:34.257 | INFO | Starting a round of Baz IO | 20260507 08:16:37.347 | INFO | Populating RTMetrics report | 20260507 08:16:37.347 | INFO | RTMetrics report populated | 20260507 08:16:37.347 | INFO | Opening tmp RTMetrics json file | 20260507 08:16:37.357 | INFO | Serializing RTMetrics to json | 20260507 08:16:37.364 | INFO | Serialization done, swapping tmp file now... | 20260507 08:16:37.365 | INFO | RTMetrics done | 20260507 08:16:37.435 | INFO | Finished a round of Baz IO: Wrote 1.618 GiB in 3.178 seconds | 20260507 08:16:37.435 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.51% / 0.00% / 67.49% | 20260507 08:16:37.435 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0928243% / 0% / 5.5631e-05% | 20260507 08:16:37.435 | INFO | Baz IO overall: | 20260507 08:16:37.435 | INFO | BytesWritten : 4217665516094 | 20260507 08:16:37.435 | INFO | NumEvents : 2017768597354 | 20260507 08:16:39.289 | INFO | Analyzing chunk frames = [8458240,8458752) | 20260507 08:16:39.289 | INFO | MemInfo: { | 20260507 08:16:39.289 | INFO | "gpu" : " 17 GB", | 20260507 08:16:39.289 | INFO | "huge" : "", | 20260507 08:16:39.289 | INFO | "huge_shared" : " 27 GB", | 20260507 08:16:39.289 | INFO | "malloc" : " 14 GB", | 20260507 08:16:39.289 | INFO | "pinned" : " 29 GB" | 20260507 08:16:39.289 | INFO | } | 20260507 08:16:39.289 | INFO | Perf report for the last 5.12861 seconds: | 20260507 08:16:39.289 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:16:39.289 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:39.289 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:39.289 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:39.289 | INFO | ANALYSIS : 61.6%, 3.0, 95.5ms, 0 | 20260507 08:16:39.289 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:39.289 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260507 08:16:39.289 | INFO | PRE_HQ : 58.7%, 1.8, 54.2ms, 0 | 20260507 08:16:39.289 | INFO | BAZWRITER : 61.6%, 3.5, 111.3ms, 0 | 20260507 08:16:39.289 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 08:16:39.289 | INFO | Uploaded 36GiB to GPU in 2.06566s (17.4279GiB/s) | 20260507 08:16:39.289 | INFO | Downloaded 38GiB to GPU in 2.17551s (17.4672GiB/s) | 20260507 08:16:44.431 | INFO | Analyzing chunk frames = [8458752,8459264) | 20260507 08:16:44.431 | INFO | MemInfo: { | 20260507 08:16:44.431 | INFO | "gpu" : " 17 GB", | 20260507 08:16:44.431 | INFO | "huge" : "", | 20260507 08:16:44.431 | INFO | "huge_shared" : " 27 GB", | 20260507 08:16:44.431 | INFO | "malloc" : " 14 GB", | 20260507 08:16:44.431 | INFO | "pinned" : " 29 GB" | 20260507 08:16:44.431 | INFO | } | 20260507 08:16:44.432 | INFO | Perf report for the last 5.14233 seconds: | 20260507 08:16:44.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:16:44.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:44.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:44.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:44.432 | INFO | ANALYSIS : 60.9%, 3.0, 94.9ms, 0 | 20260507 08:16:44.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:44.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:44.432 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 08:16:44.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:44.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:44.432 | INFO | Uploaded 36GiB to GPU in 2.05021s (17.5592GiB/s) | 20260507 08:16:44.432 | INFO | Downloaded 34GiB to GPU in 1.94308s (17.498GiB/s) | 20260507 08:16:49.576 | INFO | Analyzing chunk frames = [8459264,8459776) | 20260507 08:16:49.576 | INFO | MemInfo: { | 20260507 08:16:49.576 | INFO | "gpu" : " 17 GB", | 20260507 08:16:49.576 | INFO | "huge" : "", | 20260507 08:16:49.576 | INFO | "huge_shared" : " 27 GB", | 20260507 08:16:49.576 | INFO | "malloc" : " 14 GB", | 20260507 08:16:49.576 | INFO | "pinned" : " 29 GB" | 20260507 08:16:49.576 | INFO | } | 20260507 08:16:49.576 | INFO | Perf report for the last 5.14479 seconds: | 20260507 08:16:49.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:16:49.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:49.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:49.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:49.576 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 08:16:49.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:49.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:49.576 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:16:49.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:49.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:49.577 | INFO | Uploaded 36GiB to GPU in 2.04003s (17.6468GiB/s) | 20260507 08:16:49.577 | INFO | Downloaded 34GiB to GPU in 1.95029s (17.4333GiB/s) | 20260507 08:16:54.714 | INFO | Analyzing chunk frames = [8459776,8460288) | 20260507 08:16:54.714 | INFO | MemInfo: { | 20260507 08:16:54.714 | INFO | "gpu" : " 17 GB", | 20260507 08:16:54.714 | INFO | "huge" : "", | 20260507 08:16:54.714 | INFO | "huge_shared" : " 27 GB", | 20260507 08:16:54.714 | INFO | "malloc" : " 14 GB", | 20260507 08:16:54.714 | INFO | "pinned" : " 29 GB" | 20260507 08:16:54.714 | INFO | } | 20260507 08:16:54.714 | INFO | Perf report for the last 5.13816 seconds: | 20260507 08:16:54.714 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:16:54.714 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:54.714 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:54.714 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:54.715 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 08:16:54.715 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:54.715 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:54.715 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:16:54.715 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:54.715 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:54.715 | INFO | Uploaded 36GiB to GPU in 2.05747s (17.4972GiB/s) | 20260507 08:16:54.715 | INFO | Downloaded 34GiB to GPU in 1.94423s (17.4877GiB/s) | 20260507 08:16:59.844 | INFO | Analyzing chunk frames = [8460288,8460800) | 20260507 08:16:59.844 | INFO | MemInfo: { | 20260507 08:16:59.844 | INFO | "gpu" : " 17 GB", | 20260507 08:16:59.844 | INFO | "huge" : "", | 20260507 08:16:59.844 | INFO | "huge_shared" : " 27 GB", | 20260507 08:16:59.844 | INFO | "malloc" : " 14 GB", | 20260507 08:16:59.844 | INFO | "pinned" : " 29 GB" | 20260507 08:16:59.844 | INFO | } | 20260507 08:16:59.844 | INFO | Perf report for the last 5.12976 seconds: | 20260507 08:16:59.844 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:16:59.844 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:59.844 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:59.844 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:59.844 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 08:16:59.844 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:59.844 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:59.844 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:16:59.844 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:59.844 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:16:59.845 | INFO | Uploaded 36GiB to GPU in 2.05379s (17.5286GiB/s) | 20260507 08:16:59.845 | INFO | Downloaded 34GiB to GPU in 1.90387s (17.8584GiB/s) | 20260507 08:17:04.990 | INFO | Analyzing chunk frames = [8460800,8461312) | 20260507 08:17:04.990 | INFO | MemInfo: { | 20260507 08:17:04.990 | INFO | "gpu" : " 17 GB", | 20260507 08:17:04.990 | INFO | "huge" : "", | 20260507 08:17:04.990 | INFO | "huge_shared" : " 27 GB", | 20260507 08:17:04.990 | INFO | "malloc" : " 14 GB", | 20260507 08:17:04.990 | INFO | "pinned" : " 29 GB" | 20260507 08:17:04.990 | INFO | } | 20260507 08:17:04.990 | INFO | Perf report for the last 5.14584 seconds: | 20260507 08:17:04.990 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:17:04.990 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:04.990 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:04.990 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:04.990 | INFO | ANALYSIS : 60.2%, 3.0, 94.1ms, 0 | 20260507 08:17:04.990 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:04.990 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:04.990 | INFO | PRE_HQ : 27.6%, 1.0, 14.4ms, 0 | 20260507 08:17:04.990 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:04.990 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:04.990 | INFO | Uploaded 36GiB to GPU in 2.04697s (17.5869GiB/s) | 20260507 08:17:04.990 | INFO | Downloaded 34GiB to GPU in 1.91021s (17.7991GiB/s) | 20260507 08:17:10.128 | INFO | Analyzing chunk frames = [8461312,8461824) | 20260507 08:17:10.128 | INFO | MemInfo: { | 20260507 08:17:10.128 | INFO | "gpu" : " 17 GB", | 20260507 08:17:10.128 | INFO | "huge" : "", | 20260507 08:17:10.128 | INFO | "huge_shared" : " 27 GB", | 20260507 08:17:10.128 | INFO | "malloc" : " 14 GB", | 20260507 08:17:10.128 | INFO | "pinned" : " 29 GB" | 20260507 08:17:10.128 | INFO | } | 20260507 08:17:10.128 | INFO | Perf report for the last 5.1377 seconds: | 20260507 08:17:10.128 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:17:10.128 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:10.128 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:10.128 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:10.128 | INFO | ANALYSIS : 61.1%, 3.0, 95.0ms, 0 | 20260507 08:17:10.128 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:10.128 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:10.128 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 08:17:10.128 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:10.128 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:10.128 | INFO | Uploaded 36GiB to GPU in 2.03687s (17.6742GiB/s) | 20260507 08:17:10.128 | INFO | Downloaded 34GiB to GPU in 1.99901s (17.0084GiB/s) | 20260507 08:17:15.277 | INFO | Analyzing chunk frames = [8461824,8462336) | 20260507 08:17:15.277 | INFO | MemInfo: { | 20260507 08:17:15.277 | INFO | "gpu" : " 17 GB", | 20260507 08:17:15.277 | INFO | "huge" : "", | 20260507 08:17:15.277 | INFO | "huge_shared" : " 27 GB", | 20260507 08:17:15.277 | INFO | "malloc" : " 14 GB", | 20260507 08:17:15.277 | INFO | "pinned" : " 29 GB" | 20260507 08:17:15.277 | INFO | } | 20260507 08:17:15.277 | INFO | Perf report for the last 5.14947 seconds: | 20260507 08:17:15.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:17:15.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:15.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:15.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:15.277 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 08:17:15.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:15.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:15.277 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:17:15.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:15.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:15.278 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:17:15.278 | INFO | Total mem usage: 18 GiB | 20260507 08:17:15.277 | INFO | Uploaded 36GiB to GPU in 1.9856s (18.1306GiB/s) | 20260507 08:17:15.278 | INFO | Downloaded 34GiB to GPU in 1.95842s (17.3609GiB/s) | 20260507 08:17:15.278 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:17:15.278 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -21.2%) | 20260507 08:17:15.278 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:17:15.278 | INFO | High Water Breakdown: | 20260507 08:17:15.278 | INFO | 60.8% : 9.03 GiB ( 79.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:17:15.278 | INFO | 39.1% : 5.8 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:17:15.278 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:17:15.278 | INFO | ------------------------------------------ | 20260507 08:17:15.278 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:17:15.278 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:17:15.278 | INFO | Total mem usage: 26.5 GiB | 20260507 08:17:15.278 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:17:15.278 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:17:15.278 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 08:17:15.278 | INFO | High Water Breakdown: | 20260507 08:17:15.278 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:17:15.278 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:17:15.278 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:17:15.278 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:17:15.278 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:17:15.278 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:17:15.278 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:17:15.278 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:17:15.278 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:17:15.278 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:17:15.278 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:17:15.278 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:17:15.278 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:17:15.278 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:17:15.278 | INFO | ------------------------------------------ | 20260507 08:17:15.278 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:17:15.278 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:17:15.278 | INFO | Total mem usage: 25.6 GiB | 20260507 08:17:15.278 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:17:15.278 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:17:15.278 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:17:15.278 | INFO | High Water Breakdown: | 20260507 08:17:15.278 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:17:15.278 | INFO | ------------------------------------------ | 20260507 08:17:15.278 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:17:15.278 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:17:15.278 | INFO | Total mem usage: 15.7 GiB | 20260507 08:17:15.278 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:17:15.278 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:17:15.278 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:17:15.278 | INFO | High Water Breakdown: | 20260507 08:17:15.278 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:17:15.278 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:17:15.278 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:17:15.278 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:17:15.278 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:17:15.278 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:17:15.278 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:17:15.278 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:17:15.278 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:17:15.278 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:17:15.278 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:17:15.278 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:17:15.278 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:17:15.278 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:17:15.278 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:17:15.278 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:17:15.278 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:17:15.278 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:17:15.278 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:17:15.278 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:17:15.278 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:17:15.278 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:17:15.278 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:17:15.278 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:17:15.278 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:17:15.278 | INFO | ------------------------------------------ | 20260507 08:17:15.278 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:17:15.278 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:17:15.278 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:17:15.278 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:17:15.278 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.35e+01 / 1.33e+00 | 20260507 08:17:15.278 | INFO | PerfReport: 0.29%, AggregateStats -- 9.01e-02 / 1.03e-01, 9.20e-03 / 5.14e-03 | 20260507 08:17:15.278 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.45e+00, 4.95e-01 / 1.98e-01 | 20260507 08:17:15.278 | INFO | PerfReport: 3.22%, Histogram -- 9.86e-01 / 1.48e+00, 1.63e-01 / 9.35e-02 | 20260507 08:17:15.278 | INFO | PerfReport: 16.34%, DME -- 4.01e+01 / 4.49e+01, 3.84e+00 / 1.69e+00 | 20260507 08:17:15.278 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.25e+00 / 1.31e+00 | 20260507 08:17:15.278 | INFO | PerfReport: 6.88%, PulseAccumulating -- 2.11e+00 / 2.68e+00, 3.82e-01 / 1.91e-01 | 20260507 08:17:15.278 | INFO | PerfReport: 2.91%, Metrics -- 8.94e-01 / 1.19e+00, 2.38e-01 / 9.04e-02 | 20260507 08:17:15.278 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.51e-01, 4.92e-02 / 1.70e-02 | 20260507 08:17:15.278 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:17:15.278 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:17:15.278 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:17:15.278 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 08:17:15.278 | INFO | PerfReport: 21.98%, Upload -- 2.12e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260507 08:17:15.278 | INFO | PerfReport: 21.34%, Download -- 2.05e+01 / 0.00e+00, 7.22e+00 / 0.00e+00 | 20260507 08:17:15.278 | INFO | PerfReport: 31.87%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 08:17:15.278 | INFO | PerfReport: 1.71%, LockUpload -- 1.65e+00 / 0.00e+00, 5.13e+00 / 0.00e+00 | 20260507 08:17:15.278 | INFO | PerfReport: 1.16%, LockDownload -- 1.12e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260507 08:17:15.279 | INFO | PerfReport: 21.91%, LockCompute -- 2.11e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260507 08:17:15.279 | INFO | PerfReport: 0.01%, Misc -- 1.39e-02 / 0.00e+00, 2.00e-02 / 0.00e+00 | 20260507 08:17:15.375 | INFO | Starting a round of Baz IO | 20260507 08:17:18.465 | INFO | Populating RTMetrics report | 20260507 08:17:18.466 | INFO | RTMetrics report populated | 20260507 08:17:18.466 | INFO | Opening tmp RTMetrics json file | 20260507 08:17:18.481 | INFO | Serializing RTMetrics to json | 20260507 08:17:18.482 | INFO | Serialization done, swapping tmp file now... | 20260507 08:17:18.482 | INFO | RTMetrics done | 20260507 08:17:18.548 | INFO | Finished a round of Baz IO: Wrote 1.618 GiB in 3.173 seconds | 20260507 08:17:18.548 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.51% / 0.00% / 67.49% | 20260507 08:17:18.548 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.093329% / 0% / 7.54992e-05% | 20260507 08:17:18.548 | INFO | Baz IO overall: | 20260507 08:17:18.548 | INFO | BytesWritten : 4219402811600 | 20260507 08:17:18.548 | INFO | NumEvents : 2018578206672 | 20260507 08:17:20.412 | INFO | Analyzing chunk frames = [8462336,8462848) | 20260507 08:17:20.412 | INFO | MemInfo: { | 20260507 08:17:20.412 | INFO | "gpu" : " 17 GB", | 20260507 08:17:20.412 | INFO | "huge" : "", | 20260507 08:17:20.412 | INFO | "huge_shared" : " 27 GB", | 20260507 08:17:20.412 | INFO | "malloc" : " 14 GB", | 20260507 08:17:20.412 | INFO | "pinned" : " 29 GB" | 20260507 08:17:20.412 | INFO | } | 20260507 08:17:20.412 | INFO | Perf report for the last 5.13446 seconds: | 20260507 08:17:20.412 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:17:20.412 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:20.412 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:20.412 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:20.412 | INFO | ANALYSIS : 61.3%, 3.0, 95.6ms, 0 | 20260507 08:17:20.412 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:20.412 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260507 08:17:20.412 | INFO | PRE_HQ : 58.5%, 1.8, 54.2ms, 0 | 20260507 08:17:20.412 | INFO | BAZWRITER : 61.4%, 3.2, 100.1ms, 0 | 20260507 08:17:20.412 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 08:17:20.412 | INFO | Uploaded 36GiB to GPU in 2.04013s (17.6459GiB/s) | 20260507 08:17:20.412 | INFO | Downloaded 38GiB to GPU in 2.17274s (17.4895GiB/s) | 20260507 08:17:25.544 | INFO | Analyzing chunk frames = [8462848,8463360) | 20260507 08:17:25.544 | INFO | MemInfo: { | 20260507 08:17:25.544 | INFO | "gpu" : " 17 GB", | 20260507 08:17:25.544 | INFO | "huge" : "", | 20260507 08:17:25.544 | INFO | "huge_shared" : " 27 GB", | 20260507 08:17:25.544 | INFO | "malloc" : " 14 GB", | 20260507 08:17:25.544 | INFO | "pinned" : " 29 GB" | 20260507 08:17:25.544 | INFO | } | 20260507 08:17:25.544 | INFO | Perf report for the last 5.13238 seconds: | 20260507 08:17:25.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:17:25.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:25.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:25.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:25.544 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:17:25.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:25.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:25.544 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260507 08:17:25.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:25.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:25.545 | INFO | Uploaded 36GiB to GPU in 1.98729s (18.1151GiB/s) | 20260507 08:17:25.545 | INFO | Downloaded 34GiB to GPU in 1.93733s (17.5499GiB/s) | 20260507 08:17:30.691 | INFO | Analyzing chunk frames = [8463360,8463872) | 20260507 08:17:30.691 | INFO | MemInfo: { | 20260507 08:17:30.691 | INFO | "gpu" : " 17 GB", | 20260507 08:17:30.691 | INFO | "huge" : "", | 20260507 08:17:30.691 | INFO | "huge_shared" : " 27 GB", | 20260507 08:17:30.691 | INFO | "malloc" : " 14 GB", | 20260507 08:17:30.691 | INFO | "pinned" : " 29 GB" | 20260507 08:17:30.691 | INFO | } | 20260507 08:17:30.691 | INFO | Perf report for the last 5.14661 seconds: | 20260507 08:17:30.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:17:30.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:30.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:30.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:30.691 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 08:17:30.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:30.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:30.691 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:17:30.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:30.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:30.691 | INFO | Uploaded 36GiB to GPU in 2.02795s (17.7519GiB/s) | 20260507 08:17:30.691 | INFO | Downloaded 34GiB to GPU in 1.95394s (17.4007GiB/s) | 20260507 08:17:35.829 | INFO | Analyzing chunk frames = [8463872,8464384) | 20260507 08:17:35.829 | INFO | MemInfo: { | 20260507 08:17:35.829 | INFO | "gpu" : " 17 GB", | 20260507 08:17:35.829 | INFO | "huge" : "", | 20260507 08:17:35.829 | INFO | "huge_shared" : " 27 GB", | 20260507 08:17:35.829 | INFO | "malloc" : " 14 GB", | 20260507 08:17:35.829 | INFO | "pinned" : " 29 GB" | 20260507 08:17:35.829 | INFO | } | 20260507 08:17:35.829 | INFO | Perf report for the last 5.13797 seconds: | 20260507 08:17:35.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:17:35.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:35.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:35.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:35.829 | INFO | ANALYSIS : 60.2%, 2.9, 93.0ms, 0 | 20260507 08:17:35.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:35.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:35.829 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:17:35.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:35.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:35.829 | INFO | Uploaded 36GiB to GPU in 2.01917s (17.8291GiB/s) | 20260507 08:17:35.829 | INFO | Downloaded 34GiB to GPU in 1.95661s (17.377GiB/s) | 20260507 08:17:40.969 | INFO | Analyzing chunk frames = [8464384,8464896) | 20260507 08:17:40.969 | INFO | MemInfo: { | 20260507 08:17:40.969 | INFO | "gpu" : " 17 GB", | 20260507 08:17:40.969 | INFO | "huge" : "", | 20260507 08:17:40.969 | INFO | "huge_shared" : " 27 GB", | 20260507 08:17:40.969 | INFO | "malloc" : " 14 GB", | 20260507 08:17:40.969 | INFO | "pinned" : " 29 GB" | 20260507 08:17:40.969 | INFO | } | 20260507 08:17:40.969 | INFO | Perf report for the last 5.14029 seconds: | 20260507 08:17:40.969 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:17:40.969 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:40.969 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:40.969 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:40.969 | INFO | ANALYSIS : 59.9%, 3.0, 93.4ms, 0 | 20260507 08:17:40.969 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:40.969 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:40.969 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:17:40.969 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:40.969 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:40.969 | INFO | Uploaded 36GiB to GPU in 2.0327s (17.7104GiB/s) | 20260507 08:17:40.969 | INFO | Downloaded 34GiB to GPU in 1.90592s (17.8391GiB/s) | 20260507 08:17:46.106 | INFO | Analyzing chunk frames = [8464896,8465408) | 20260507 08:17:46.106 | INFO | MemInfo: { | 20260507 08:17:46.106 | INFO | "gpu" : " 17 GB", | 20260507 08:17:46.106 | INFO | "huge" : "", | 20260507 08:17:46.106 | INFO | "huge_shared" : " 27 GB", | 20260507 08:17:46.106 | INFO | "malloc" : " 14 GB", | 20260507 08:17:46.106 | INFO | "pinned" : " 29 GB" | 20260507 08:17:46.106 | INFO | } | 20260507 08:17:46.106 | INFO | Perf report for the last 5.13711 seconds: | 20260507 08:17:46.106 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:17:46.106 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:46.106 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:46.106 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:46.106 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 08:17:46.106 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:46.106 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:46.106 | INFO | PRE_HQ : 27.7%, 1.0, 14.4ms, 0 | 20260507 08:17:46.106 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:46.106 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:46.106 | INFO | Uploaded 36GiB to GPU in 2.06813s (17.407GiB/s) | 20260507 08:17:46.106 | INFO | Downloaded 34GiB to GPU in 1.9441s (17.4888GiB/s) | 20260507 08:17:51.243 | INFO | Analyzing chunk frames = [8465408,8465920) | 20260507 08:17:51.243 | INFO | MemInfo: { | 20260507 08:17:51.243 | INFO | "gpu" : " 17 GB", | 20260507 08:17:51.243 | INFO | "huge" : "", | 20260507 08:17:51.243 | INFO | "huge_shared" : " 27 GB", | 20260507 08:17:51.243 | INFO | "malloc" : " 14 GB", | 20260507 08:17:51.243 | INFO | "pinned" : " 29 GB" | 20260507 08:17:51.243 | INFO | } | 20260507 08:17:51.243 | INFO | Perf report for the last 5.13659 seconds: | 20260507 08:17:51.243 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:17:51.243 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:51.243 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:51.243 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:51.243 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 08:17:51.243 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:51.243 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:51.243 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 08:17:51.243 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:51.243 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:51.243 | INFO | Uploaded 36GiB to GPU in 1.91671s (18.7822GiB/s) | 20260507 08:17:51.243 | INFO | Downloaded 34GiB to GPU in 1.97183s (17.2429GiB/s) | 20260507 08:17:56.388 | INFO | Analyzing chunk frames = [8465920,8466432) | 20260507 08:17:56.388 | INFO | MemInfo: { | 20260507 08:17:56.388 | INFO | "gpu" : " 17 GB", | 20260507 08:17:56.388 | INFO | "huge" : "", | 20260507 08:17:56.388 | INFO | "huge_shared" : " 27 GB", | 20260507 08:17:56.388 | INFO | "malloc" : " 14 GB", | 20260507 08:17:56.388 | INFO | "pinned" : " 29 GB" | 20260507 08:17:56.388 | INFO | } | 20260507 08:17:56.388 | INFO | Perf report for the last 5.14523 seconds: | 20260507 08:17:56.388 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:17:56.388 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:56.388 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:56.388 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:56.388 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:17:56.388 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:56.388 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:56.388 | INFO | PRE_HQ : 28.0%, 1.0, 14.6ms, 0 | 20260507 08:17:56.388 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:56.388 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:17:56.388 | INFO | Uploaded 36GiB to GPU in 1.97462s (18.2313GiB/s) | 20260507 08:17:56.388 | INFO | Downloaded 34GiB to GPU in 1.95517s (17.3898GiB/s) | 20260507 08:17:56.485 | INFO | Starting a round of Baz IO | 20260507 08:17:59.578 | INFO | Populating RTMetrics report | 20260507 08:17:59.579 | INFO | RTMetrics report populated | 20260507 08:17:59.579 | INFO | Opening tmp RTMetrics json file | 20260507 08:17:59.587 | INFO | Serializing RTMetrics to json | 20260507 08:17:59.594 | INFO | Serialization done, swapping tmp file now... | 20260507 08:17:59.595 | INFO | RTMetrics done | 20260507 08:17:59.661 | INFO | Finished a round of Baz IO: Wrote 1.618 GiB in 3.176 seconds | 20260507 08:17:59.661 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.50% / 0.00% / 67.49% | 20260507 08:17:59.661 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0934482% / 3.97364e-06% / 5.96046e-05% | 20260507 08:17:59.661 | INFO | Baz IO overall: | 20260507 08:17:59.661 | INFO | BytesWritten : 4221140246611 | 20260507 08:17:59.661 | INFO | NumEvents : 2019388087537 | 20260507 08:18:01.528 | INFO | Analyzing chunk frames = [8466432,8466944) | 20260507 08:18:01.528 | INFO | MemInfo: { | 20260507 08:18:01.528 | INFO | "gpu" : " 17 GB", | 20260507 08:18:01.528 | INFO | "huge" : "", | 20260507 08:18:01.528 | INFO | "huge_shared" : " 27 GB", | 20260507 08:18:01.528 | INFO | "malloc" : " 14 GB", | 20260507 08:18:01.528 | INFO | "pinned" : " 29 GB" | 20260507 08:18:01.528 | INFO | } | 20260507 08:18:01.528 | INFO | Perf report for the last 5.14042 seconds: | 20260507 08:18:01.528 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:18:01.529 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:01.529 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:01.529 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:01.529 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 08:18:01.529 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:01.529 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 08:18:01.529 | INFO | PRE_HQ : 58.7%, 1.8, 54.3ms, 0 | 20260507 08:18:01.529 | INFO | BAZWRITER : 61.4%, 3.1, 98.3ms, 0 | 20260507 08:18:01.529 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:18:01.529 | INFO | Uploaded 36GiB to GPU in 2.0678s (17.4098GiB/s) | 20260507 08:18:01.529 | INFO | Downloaded 38GiB to GPU in 2.1764s (17.46GiB/s) | 20260507 08:18:06.659 | INFO | Analyzing chunk frames = [8466944,8467456) | 20260507 08:18:06.659 | INFO | MemInfo: { | 20260507 08:18:06.659 | INFO | "gpu" : " 17 GB", | 20260507 08:18:06.659 | INFO | "huge" : "", | 20260507 08:18:06.659 | INFO | "huge_shared" : " 27 GB", | 20260507 08:18:06.659 | INFO | "malloc" : " 14 GB", | 20260507 08:18:06.659 | INFO | "pinned" : " 29 GB" | 20260507 08:18:06.659 | INFO | } | 20260507 08:18:06.659 | INFO | Perf report for the last 5.13026 seconds: | 20260507 08:18:06.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:18:06.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:06.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:06.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:06.659 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 08:18:06.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:06.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:06.659 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260507 08:18:06.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:06.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:06.659 | INFO | Uploaded 36GiB to GPU in 2.0252s (17.776GiB/s) | 20260507 08:18:06.659 | INFO | Downloaded 34GiB to GPU in 1.94579s (17.4736GiB/s) | 20260507 08:18:11.797 | INFO | Analyzing chunk frames = [8467456,8467968) | 20260507 08:18:11.797 | INFO | MemInfo: { | 20260507 08:18:11.797 | INFO | "gpu" : " 17 GB", | 20260507 08:18:11.797 | INFO | "huge" : "", | 20260507 08:18:11.797 | INFO | "huge_shared" : " 27 GB", | 20260507 08:18:11.797 | INFO | "malloc" : " 14 GB", | 20260507 08:18:11.797 | INFO | "pinned" : " 29 GB" | 20260507 08:18:11.797 | INFO | } | 20260507 08:18:11.797 | INFO | Perf report for the last 5.13803 seconds: | 20260507 08:18:11.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:18:11.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:11.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:11.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:11.797 | INFO | ANALYSIS : 60.5%, 3.0, 93.6ms, 0 | 20260507 08:18:11.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:11.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:11.797 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:18:11.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:11.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:11.797 | INFO | Uploaded 36GiB to GPU in 2.05271s (17.5378GiB/s) | 20260507 08:18:11.797 | INFO | Downloaded 34GiB to GPU in 1.94936s (17.4417GiB/s) | 20260507 08:18:16.941 | INFO | Analyzing chunk frames = [8467968,8468480) | 20260507 08:18:16.941 | INFO | MemInfo: { | 20260507 08:18:16.941 | INFO | "gpu" : " 17 GB", | 20260507 08:18:16.941 | INFO | "huge" : "", | 20260507 08:18:16.941 | INFO | "huge_shared" : " 27 GB", | 20260507 08:18:16.941 | INFO | "malloc" : " 14 GB", | 20260507 08:18:16.941 | INFO | "pinned" : " 29 GB" | 20260507 08:18:16.941 | INFO | } | 20260507 08:18:16.941 | INFO | Perf report for the last 5.14414 seconds: | 20260507 08:18:16.941 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:18:16.941 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:16.941 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:16.941 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:16.941 | INFO | ANALYSIS : 59.6%, 3.0, 92.6ms, 0 | 20260507 08:18:16.941 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:16.941 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:16.941 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:18:16.941 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:16.941 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:16.941 | INFO | Uploaded 36GiB to GPU in 2.01967s (17.8247GiB/s) | 20260507 08:18:16.941 | INFO | Downloaded 34GiB to GPU in 1.94115s (17.5154GiB/s) | 20260507 08:18:22.087 | INFO | Analyzing chunk frames = [8468480,8468992) | 20260507 08:18:22.087 | INFO | MemInfo: { | 20260507 08:18:22.087 | INFO | "gpu" : " 17 GB", | 20260507 08:18:22.087 | INFO | "huge" : "", | 20260507 08:18:22.087 | INFO | "huge_shared" : " 27 GB", | 20260507 08:18:22.087 | INFO | "malloc" : " 14 GB", | 20260507 08:18:22.087 | INFO | "pinned" : " 29 GB" | 20260507 08:18:22.087 | INFO | } | 20260507 08:18:22.087 | INFO | Perf report for the last 5.14575 seconds: | 20260507 08:18:22.087 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:18:22.087 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:22.087 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:22.087 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:22.087 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 08:18:22.087 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:22.087 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:22.087 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:18:22.087 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:22.087 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:22.087 | INFO | Uploaded 36GiB to GPU in 2.03993s (17.6476GiB/s) | 20260507 08:18:22.087 | INFO | Downloaded 34GiB to GPU in 1.91222s (17.7804GiB/s) | 20260507 08:18:27.226 | INFO | Analyzing chunk frames = [8468992,8469504) | 20260507 08:18:27.226 | INFO | MemInfo: { | 20260507 08:18:27.226 | INFO | "gpu" : " 17 GB", | 20260507 08:18:27.226 | INFO | "huge" : "", | 20260507 08:18:27.226 | INFO | "huge_shared" : " 27 GB", | 20260507 08:18:27.226 | INFO | "malloc" : " 14 GB", | 20260507 08:18:27.226 | INFO | "pinned" : " 29 GB" | 20260507 08:18:27.226 | INFO | } | 20260507 08:18:27.226 | INFO | Perf report for the last 5.13954 seconds: | 20260507 08:18:27.226 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:18:27.226 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:27.226 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:27.226 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:27.226 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 08:18:27.226 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:27.226 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:27.226 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:18:27.226 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:27.226 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:27.226 | INFO | Uploaded 36GiB to GPU in 2.03907s (17.6551GiB/s) | 20260507 08:18:27.227 | INFO | Downloaded 34GiB to GPU in 1.9213s (17.6964GiB/s) | 20260507 08:18:32.360 | INFO | Analyzing chunk frames = [8469504,8470016) | 20260507 08:18:32.360 | INFO | MemInfo: { | 20260507 08:18:32.360 | INFO | "gpu" : " 17 GB", | 20260507 08:18:32.360 | INFO | "huge" : "", | 20260507 08:18:32.360 | INFO | "huge_shared" : " 27 GB", | 20260507 08:18:32.360 | INFO | "malloc" : " 14 GB", | 20260507 08:18:32.360 | INFO | "pinned" : " 29 GB" | 20260507 08:18:32.360 | INFO | } | 20260507 08:18:32.360 | INFO | Perf report for the last 5.13377 seconds: | 20260507 08:18:32.360 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:18:32.360 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:32.360 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:32.360 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:32.360 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:18:32.360 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:32.360 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:32.360 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 08:18:32.360 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:32.360 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:32.360 | INFO | Uploaded 36GiB to GPU in 1.97978s (18.1838GiB/s) | 20260507 08:18:32.360 | INFO | Downloaded 34GiB to GPU in 1.97978s (17.1736GiB/s) | 20260507 08:18:37.499 | INFO | Analyzing chunk frames = [8470016,8470528) | 20260507 08:18:37.499 | INFO | MemInfo: { | 20260507 08:18:37.499 | INFO | "gpu" : " 17 GB", | 20260507 08:18:37.499 | INFO | "huge" : "", | 20260507 08:18:37.499 | INFO | "huge_shared" : " 27 GB", | 20260507 08:18:37.499 | INFO | "malloc" : " 14 GB", | 20260507 08:18:37.499 | INFO | "pinned" : " 29 GB" | 20260507 08:18:37.499 | INFO | } | 20260507 08:18:37.499 | INFO | Perf report for the last 5.13897 seconds: | 20260507 08:18:37.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:18:37.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:37.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:37.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:37.499 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 08:18:37.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:37.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:37.499 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 08:18:37.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:37.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:37.499 | INFO | Uploaded 36GiB to GPU in 1.98916s (18.0981GiB/s) | 20260507 08:18:37.499 | INFO | Downloaded 34GiB to GPU in 1.95758s (17.3684GiB/s) | 20260507 08:18:37.499 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:18:37.499 | INFO | Total mem usage: 18 GiB | 20260507 08:18:37.499 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:18:37.499 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:18:37.499 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:18:37.499 | INFO | High Water Breakdown: | 20260507 08:18:37.499 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:18:37.499 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:18:37.499 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:18:37.499 | INFO | ------------------------------------------ | 20260507 08:18:37.499 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:18:37.499 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:18:37.499 | INFO | Total mem usage: 26.5 GiB | 20260507 08:18:37.499 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:18:37.499 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:18:37.499 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 08:18:37.499 | INFO | High Water Breakdown: | 20260507 08:18:37.499 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:18:37.499 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:18:37.500 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:18:37.500 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:18:37.500 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:18:37.500 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:18:37.500 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:18:37.500 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:18:37.500 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:18:37.500 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:18:37.500 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:18:37.500 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:18:37.500 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:18:37.500 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:18:37.500 | INFO | ------------------------------------------ | 20260507 08:18:37.500 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:18:37.500 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:18:37.500 | INFO | Total mem usage: 25.6 GiB | 20260507 08:18:37.500 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:18:37.500 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:18:37.500 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:18:37.500 | INFO | High Water Breakdown: | 20260507 08:18:37.500 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:18:37.500 | INFO | ------------------------------------------ | 20260507 08:18:37.500 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:18:37.500 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:18:37.500 | INFO | Total mem usage: 15.7 GiB | 20260507 08:18:37.500 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:18:37.500 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:18:37.500 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:18:37.500 | INFO | High Water Breakdown: | 20260507 08:18:37.500 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:18:37.500 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:18:37.500 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:18:37.500 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:18:37.500 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:18:37.500 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:18:37.500 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:18:37.500 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:18:37.500 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:18:37.500 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:18:37.500 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:18:37.500 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:18:37.500 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:18:37.500 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:18:37.500 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:18:37.500 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:18:37.500 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:18:37.500 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:18:37.500 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:18:37.500 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:18:37.500 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:18:37.500 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:18:37.500 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:18:37.500 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:18:37.500 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:18:37.500 | INFO | ------------------------------------------ | 20260507 08:18:37.500 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:18:37.500 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:18:37.500 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:18:37.500 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:18:37.500 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.31e+01 / 1.07e+00 | 20260507 08:18:37.500 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.02e-01, 9.63e-03 / 4.20e-03 | 20260507 08:18:37.500 | INFO | PerfReport: 11.71%, Baseline -- 3.60e+00 / 4.45e+00, 5.21e-01 / 2.66e-01 | 20260507 08:18:37.500 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.48e+00, 1.67e-01 / 1.14e-01 | 20260507 08:18:37.500 | INFO | PerfReport: 16.30%, DME -- 4.01e+01 / 4.50e+01, 3.88e+00 / 1.57e+00 | 20260507 08:18:37.500 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.32e+00 / 1.30e+00 | 20260507 08:18:37.500 | INFO | PerfReport: 6.87%, PulseAccumulating -- 2.11e+00 / 2.68e+00, 3.84e-01 / 1.89e-01 | 20260507 08:18:37.500 | INFO | PerfReport: 2.91%, Metrics -- 8.93e-01 / 1.21e+00, 2.42e-01 / 1.28e-01 | 20260507 08:18:37.500 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.58e-01, 5.11e-02 / 1.83e-02 | 20260507 08:18:37.500 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:18:37.500 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:18:37.500 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:18:37.500 | INFO | PerfReport: Full Execution -- 9.60e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 08:18:37.500 | INFO | PerfReport: 21.88%, Upload -- 2.10e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260507 08:18:37.501 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 08:18:37.501 | INFO | PerfReport: 31.98%, Compute -- 3.07e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 08:18:37.501 | INFO | PerfReport: 1.51%, LockUpload -- 1.45e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 08:18:37.501 | INFO | PerfReport: 1.26%, LockDownload -- 1.21e+00 / 0.00e+00, 2.98e+00 / 0.00e+00 | 20260507 08:18:37.501 | INFO | PerfReport: 21.95%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 08:18:37.501 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260507 08:18:37.597 | INFO | Starting a round of Baz IO | 20260507 08:18:40.691 | INFO | Populating RTMetrics report | 20260507 08:18:40.692 | INFO | RTMetrics report populated | 20260507 08:18:40.692 | INFO | Opening tmp RTMetrics json file | 20260507 08:18:40.708 | INFO | Serializing RTMetrics to json | 20260507 08:18:40.709 | INFO | Serialization done, swapping tmp file now... | 20260507 08:18:40.710 | INFO | RTMetrics done | 20260507 08:18:40.777 | INFO | Finished a round of Baz IO: Wrote 1.616 GiB in 3.179 seconds | 20260507 08:18:40.777 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.50% / 0.00% / 67.50% | 20260507 08:18:40.777 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.092864% / 0% / 4.76837e-05% | 20260507 08:18:40.777 | INFO | Baz IO overall: | 20260507 08:18:40.777 | INFO | BytesWritten : 4222875783796 | 20260507 08:18:40.777 | INFO | NumEvents : 2020196400662 | 20260507 08:18:42.638 | INFO | Analyzing chunk frames = [8470528,8471040) | 20260507 08:18:42.638 | INFO | MemInfo: { | 20260507 08:18:42.638 | INFO | "gpu" : " 17 GB", | 20260507 08:18:42.638 | INFO | "huge" : "", | 20260507 08:18:42.638 | INFO | "huge_shared" : " 27 GB", | 20260507 08:18:42.638 | INFO | "malloc" : " 14 GB", | 20260507 08:18:42.638 | INFO | "pinned" : " 29 GB" | 20260507 08:18:42.638 | INFO | } | 20260507 08:18:42.639 | INFO | Perf report for the last 5.13947 seconds: | 20260507 08:18:42.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:18:42.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:42.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:42.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:42.639 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 08:18:42.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:42.639 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 08:18:42.639 | INFO | PRE_HQ : 58.8%, 1.8, 54.4ms, 0 | 20260507 08:18:42.639 | INFO | BAZWRITER : 61.5%, 3.1, 98.3ms, 0 | 20260507 08:18:42.639 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:18:42.639 | INFO | Uploaded 36GiB to GPU in 2.06049s (17.4716GiB/s) | 20260507 08:18:42.639 | INFO | Downloaded 38GiB to GPU in 2.17001s (17.5115GiB/s) | 20260507 08:18:47.782 | INFO | Analyzing chunk frames = [8471040,8471552) | 20260507 08:18:47.782 | INFO | MemInfo: { | 20260507 08:18:47.782 | INFO | "gpu" : " 17 GB", | 20260507 08:18:47.782 | INFO | "huge" : "", | 20260507 08:18:47.782 | INFO | "huge_shared" : " 27 GB", | 20260507 08:18:47.783 | INFO | "malloc" : " 14 GB", | 20260507 08:18:47.783 | INFO | "pinned" : " 29 GB" | 20260507 08:18:47.783 | INFO | } | 20260507 08:18:47.783 | INFO | Perf report for the last 5.14419 seconds: | 20260507 08:18:47.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:18:47.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:47.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:47.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:47.783 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:18:47.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:47.783 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:47.783 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:18:47.783 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:47.783 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:47.783 | INFO | Uploaded 36GiB to GPU in 1.99183s (18.0738GiB/s) | 20260507 08:18:47.783 | INFO | Downloaded 34GiB to GPU in 1.93406s (17.5796GiB/s) | 20260507 08:18:52.916 | INFO | Analyzing chunk frames = [8471552,8472064) | 20260507 08:18:52.916 | INFO | MemInfo: { | 20260507 08:18:52.916 | INFO | "gpu" : " 17 GB", | 20260507 08:18:52.916 | INFO | "huge" : "", | 20260507 08:18:52.916 | INFO | "huge_shared" : " 27 GB", | 20260507 08:18:52.916 | INFO | "malloc" : " 14 GB", | 20260507 08:18:52.916 | INFO | "pinned" : " 29 GB" | 20260507 08:18:52.916 | INFO | } | 20260507 08:18:52.916 | INFO | Perf report for the last 5.13346 seconds: | 20260507 08:18:52.916 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:18:52.916 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:52.916 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:52.916 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:52.916 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 08:18:52.916 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:52.916 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:52.916 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 08:18:52.916 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:52.916 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:52.916 | INFO | Uploaded 36GiB to GPU in 2.02537s (17.7745GiB/s) | 20260507 08:18:52.916 | INFO | Downloaded 34GiB to GPU in 1.91668s (17.739GiB/s) | 20260507 08:18:58.057 | INFO | Analyzing chunk frames = [8472064,8472576) | 20260507 08:18:58.057 | INFO | MemInfo: { | 20260507 08:18:58.057 | INFO | "gpu" : " 17 GB", | 20260507 08:18:58.057 | INFO | "huge" : "", | 20260507 08:18:58.057 | INFO | "huge_shared" : " 27 GB", | 20260507 08:18:58.057 | INFO | "malloc" : " 14 GB", | 20260507 08:18:58.057 | INFO | "pinned" : " 29 GB" | 20260507 08:18:58.057 | INFO | } | 20260507 08:18:58.057 | INFO | Perf report for the last 5.14047 seconds: | 20260507 08:18:58.057 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:18:58.057 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:58.057 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:58.057 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:58.057 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 08:18:58.057 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:58.057 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:58.057 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:18:58.057 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:58.057 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:18:58.057 | INFO | Uploaded 36GiB to GPU in 2.04466s (17.6068GiB/s) | 20260507 08:18:58.057 | INFO | Downloaded 34GiB to GPU in 1.95072s (17.4295GiB/s) | 20260507 08:19:03.198 | INFO | Analyzing chunk frames = [8472576,8473088) | 20260507 08:19:03.198 | INFO | MemInfo: { | 20260507 08:19:03.198 | INFO | "gpu" : " 17 GB", | 20260507 08:19:03.198 | INFO | "huge" : "", | 20260507 08:19:03.198 | INFO | "huge_shared" : " 27 GB", | 20260507 08:19:03.198 | INFO | "malloc" : " 14 GB", | 20260507 08:19:03.198 | INFO | "pinned" : " 29 GB" | 20260507 08:19:03.198 | INFO | } | 20260507 08:19:03.198 | INFO | Perf report for the last 5.14134 seconds: | 20260507 08:19:03.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:19:03.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:03.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:03.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:03.198 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 08:19:03.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:03.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:03.198 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:19:03.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:03.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:03.198 | INFO | Uploaded 36GiB to GPU in 2.03643s (17.678GiB/s) | 20260507 08:19:03.198 | INFO | Downloaded 34GiB to GPU in 1.91632s (17.7424GiB/s) | 20260507 08:19:08.336 | INFO | Analyzing chunk frames = [8473088,8473600) | 20260507 08:19:08.336 | INFO | MemInfo: { | 20260507 08:19:08.336 | INFO | "gpu" : " 17 GB", | 20260507 08:19:08.336 | INFO | "huge" : "", | 20260507 08:19:08.336 | INFO | "huge_shared" : " 27 GB", | 20260507 08:19:08.336 | INFO | "malloc" : " 14 GB", | 20260507 08:19:08.336 | INFO | "pinned" : " 29 GB" | 20260507 08:19:08.336 | INFO | } | 20260507 08:19:08.336 | INFO | Perf report for the last 5.13818 seconds: | 20260507 08:19:08.336 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:19:08.336 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:08.336 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:08.336 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:08.336 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 08:19:08.336 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:08.336 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:08.336 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:19:08.336 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:08.336 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:08.336 | INFO | Uploaded 36GiB to GPU in 2.06792s (17.4088GiB/s) | 20260507 08:19:08.336 | INFO | Downloaded 34GiB to GPU in 1.92429s (17.6689GiB/s) | 20260507 08:19:13.473 | INFO | Analyzing chunk frames = [8473600,8474112) | 20260507 08:19:13.473 | INFO | MemInfo: { | 20260507 08:19:13.473 | INFO | "gpu" : " 17 GB", | 20260507 08:19:13.473 | INFO | "huge" : "", | 20260507 08:19:13.473 | INFO | "huge_shared" : " 27 GB", | 20260507 08:19:13.473 | INFO | "malloc" : " 14 GB", | 20260507 08:19:13.473 | INFO | "pinned" : " 29 GB" | 20260507 08:19:13.473 | INFO | } | 20260507 08:19:13.473 | INFO | Perf report for the last 5.13701 seconds: | 20260507 08:19:13.473 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:19:13.473 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:13.473 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:13.473 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:13.473 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 08:19:13.473 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:13.473 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:13.473 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 08:19:13.473 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:13.473 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:13.473 | INFO | Uploaded 36GiB to GPU in 1.95236s (18.4392GiB/s) | 20260507 08:19:13.474 | INFO | Downloaded 34GiB to GPU in 1.99374s (17.0534GiB/s) | 20260507 08:19:18.603 | INFO | Analyzing chunk frames = [8474112,8474624) | 20260507 08:19:18.603 | INFO | MemInfo: { | 20260507 08:19:18.603 | INFO | "gpu" : " 17 GB", | 20260507 08:19:18.603 | INFO | "huge" : "", | 20260507 08:19:18.603 | INFO | "huge_shared" : " 27 GB", | 20260507 08:19:18.603 | INFO | "malloc" : " 14 GB", | 20260507 08:19:18.603 | INFO | "pinned" : " 29 GB" | 20260507 08:19:18.603 | INFO | } | 20260507 08:19:18.603 | INFO | Perf report for the last 5.13007 seconds: | 20260507 08:19:18.603 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:19:18.603 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:18.603 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:18.603 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:18.603 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 08:19:18.603 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:18.603 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:18.603 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 08:19:18.603 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:18.603 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:18.603 | INFO | Uploaded 36GiB to GPU in 1.99621s (18.0341GiB/s) | 20260507 08:19:18.603 | INFO | Downloaded 34GiB to GPU in 1.96177s (17.3313GiB/s) | 20260507 08:19:18.700 | INFO | Starting a round of Baz IO | 20260507 08:19:21.800 | INFO | Populating RTMetrics report | 20260507 08:19:21.801 | INFO | RTMetrics report populated | 20260507 08:19:21.801 | INFO | Opening tmp RTMetrics json file | 20260507 08:19:21.812 | INFO | Serializing RTMetrics to json | 20260507 08:19:21.819 | INFO | Serialization done, swapping tmp file now... | 20260507 08:19:21.819 | INFO | RTMetrics done | 20260507 08:19:21.889 | INFO | Finished a round of Baz IO: Wrote 1.617 GiB in 3.189 seconds | 20260507 08:19:21.889 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.50% / 0.00% / 67.50% | 20260507 08:19:21.889 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0931899% / 0% / 7.15256e-05% | 20260507 08:19:21.889 | INFO | Baz IO overall: | 20260507 08:19:21.889 | INFO | BytesWritten : 4224612415256 | 20260507 08:19:21.889 | INFO | NumEvents : 2021005765988 | 20260507 08:19:23.747 | INFO | Analyzing chunk frames = [8474624,8475136) | 20260507 08:19:23.747 | INFO | MemInfo: { | 20260507 08:19:23.747 | INFO | "gpu" : " 17 GB", | 20260507 08:19:23.747 | INFO | "huge" : "", | 20260507 08:19:23.747 | INFO | "huge_shared" : " 27 GB", | 20260507 08:19:23.747 | INFO | "malloc" : " 14 GB", | 20260507 08:19:23.747 | INFO | "pinned" : " 29 GB" | 20260507 08:19:23.747 | INFO | } | 20260507 08:19:23.747 | INFO | Perf report for the last 5.14371 seconds: | 20260507 08:19:23.747 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:19:23.747 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:23.747 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:23.747 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:23.747 | INFO | ANALYSIS : 61.6%, 3.0, 96.0ms, 0 | 20260507 08:19:23.747 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:23.747 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 08:19:23.747 | INFO | PRE_HQ : 58.7%, 1.8, 54.3ms, 0 | 20260507 08:19:23.747 | INFO | BAZWRITER : 61.6%, 3.1, 98.0ms, 0 | 20260507 08:19:23.747 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:19:23.748 | INFO | Uploaded 36GiB to GPU in 2.05393s (17.5274GiB/s) | 20260507 08:19:23.748 | INFO | Downloaded 38GiB to GPU in 2.16796s (17.528GiB/s) | 20260507 08:19:28.894 | INFO | Analyzing chunk frames = [8475136,8475648) | 20260507 08:19:28.894 | INFO | MemInfo: { | 20260507 08:19:28.894 | INFO | "gpu" : " 17 GB", | 20260507 08:19:28.894 | INFO | "huge" : "", | 20260507 08:19:28.894 | INFO | "huge_shared" : " 27 GB", | 20260507 08:19:28.894 | INFO | "malloc" : " 14 GB", | 20260507 08:19:28.894 | INFO | "pinned" : " 29 GB" | 20260507 08:19:28.894 | INFO | } | 20260507 08:19:28.894 | INFO | Perf report for the last 5.14724 seconds: | 20260507 08:19:28.894 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:19:28.894 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:28.894 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:28.894 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:28.894 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 08:19:28.894 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:28.894 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:28.894 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260507 08:19:28.894 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:28.894 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:28.894 | INFO | Uploaded 36GiB to GPU in 1.99926s (18.0067GiB/s) | 20260507 08:19:28.894 | INFO | Downloaded 34GiB to GPU in 1.93371s (17.5828GiB/s) | 20260507 08:19:34.036 | INFO | Analyzing chunk frames = [8475648,8476160) | 20260507 08:19:34.036 | INFO | MemInfo: { | 20260507 08:19:34.036 | INFO | "gpu" : " 17 GB", | 20260507 08:19:34.036 | INFO | "huge" : "", | 20260507 08:19:34.036 | INFO | "huge_shared" : " 27 GB", | 20260507 08:19:34.036 | INFO | "malloc" : " 14 GB", | 20260507 08:19:34.036 | INFO | "pinned" : " 29 GB" | 20260507 08:19:34.036 | INFO | } | 20260507 08:19:34.036 | INFO | Perf report for the last 5.14194 seconds: | 20260507 08:19:34.036 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:19:34.036 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:34.036 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:34.036 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:34.036 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 08:19:34.036 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:34.036 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:34.036 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:19:34.036 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:34.036 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:34.036 | INFO | Uploaded 36GiB to GPU in 2.00747s (17.933GiB/s) | 20260507 08:19:34.036 | INFO | Downloaded 34GiB to GPU in 1.91205s (17.7819GiB/s) | 20260507 08:19:39.171 | INFO | Analyzing chunk frames = [8476160,8476672) | 20260507 08:19:39.171 | INFO | MemInfo: { | 20260507 08:19:39.171 | INFO | "gpu" : " 17 GB", | 20260507 08:19:39.171 | INFO | "huge" : "", | 20260507 08:19:39.171 | INFO | "huge_shared" : " 27 GB", | 20260507 08:19:39.171 | INFO | "malloc" : " 14 GB", | 20260507 08:19:39.171 | INFO | "pinned" : " 29 GB" | 20260507 08:19:39.171 | INFO | } | 20260507 08:19:39.171 | INFO | Perf report for the last 5.13507 seconds: | 20260507 08:19:39.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:19:39.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:39.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:39.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:39.171 | INFO | ANALYSIS : 60.4%, 2.9, 93.1ms, 0 | 20260507 08:19:39.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:39.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:39.171 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:19:39.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:39.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:39.172 | INFO | Uploaded 36GiB to GPU in 2.0379s (17.6653GiB/s) | 20260507 08:19:39.172 | INFO | Downloaded 34GiB to GPU in 1.91229s (17.7797GiB/s) | 20260507 08:19:44.309 | INFO | Analyzing chunk frames = [8476672,8477184) | 20260507 08:19:44.309 | INFO | MemInfo: { | 20260507 08:19:44.309 | INFO | "gpu" : " 17 GB", | 20260507 08:19:44.309 | INFO | "huge" : "", | 20260507 08:19:44.309 | INFO | "huge_shared" : " 27 GB", | 20260507 08:19:44.309 | INFO | "malloc" : " 14 GB", | 20260507 08:19:44.309 | INFO | "pinned" : " 29 GB" | 20260507 08:19:44.309 | INFO | } | 20260507 08:19:44.309 | INFO | Perf report for the last 5.13796 seconds: | 20260507 08:19:44.309 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:19:44.309 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:44.309 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:44.309 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:44.309 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 08:19:44.309 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:44.309 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:44.309 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:19:44.309 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:44.309 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:44.309 | INFO | Uploaded 36GiB to GPU in 2.06079s (17.469GiB/s) | 20260507 08:19:44.310 | INFO | Downloaded 34GiB to GPU in 1.90842s (17.8158GiB/s) | 20260507 08:19:49.451 | INFO | Analyzing chunk frames = [8477184,8477696) | 20260507 08:19:49.451 | INFO | MemInfo: { | 20260507 08:19:49.451 | INFO | "gpu" : " 17 GB", | 20260507 08:19:49.451 | INFO | "huge" : "", | 20260507 08:19:49.451 | INFO | "huge_shared" : " 27 GB", | 20260507 08:19:49.451 | INFO | "malloc" : " 14 GB", | 20260507 08:19:49.451 | INFO | "pinned" : " 29 GB" | 20260507 08:19:49.451 | INFO | } | 20260507 08:19:49.451 | INFO | Perf report for the last 5.14184 seconds: | 20260507 08:19:49.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:19:49.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:49.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:49.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:49.451 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 08:19:49.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:49.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:49.451 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:19:49.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:49.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:49.451 | INFO | Uploaded 36GiB to GPU in 2.06177s (17.4607GiB/s) | 20260507 08:19:49.451 | INFO | Downloaded 34GiB to GPU in 1.91224s (17.7802GiB/s) | 20260507 08:19:54.591 | INFO | Analyzing chunk frames = [8477696,8478208) | 20260507 08:19:54.591 | INFO | MemInfo: { | 20260507 08:19:54.591 | INFO | "gpu" : " 17 GB", | 20260507 08:19:54.591 | INFO | "huge" : "", | 20260507 08:19:54.591 | INFO | "huge_shared" : " 27 GB", | 20260507 08:19:54.591 | INFO | "malloc" : " 14 GB", | 20260507 08:19:54.591 | INFO | "pinned" : " 29 GB" | 20260507 08:19:54.591 | INFO | } | 20260507 08:19:54.591 | INFO | Perf report for the last 5.13965 seconds: | 20260507 08:19:54.591 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:19:54.591 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:54.591 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:54.591 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:54.591 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:19:54.591 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:54.591 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:54.591 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 08:19:54.591 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:54.591 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:54.591 | INFO | Uploaded 36GiB to GPU in 1.94661s (18.4937GiB/s) | 20260507 08:19:54.591 | INFO | Downloaded 34GiB to GPU in 1.98758s (17.1062GiB/s) | 20260507 08:19:59.735 | INFO | Analyzing chunk frames = [8478208,8478720) | 20260507 08:19:59.735 | INFO | MemInfo: { | 20260507 08:19:59.735 | INFO | "gpu" : " 17 GB", | 20260507 08:19:59.735 | INFO | "huge" : "", | 20260507 08:19:59.735 | INFO | "huge_shared" : " 27 GB", | 20260507 08:19:59.735 | INFO | "malloc" : " 14 GB", | 20260507 08:19:59.735 | INFO | "pinned" : " 29 GB" | 20260507 08:19:59.735 | INFO | } | 20260507 08:19:59.735 | INFO | Perf report for the last 5.14401 seconds: | 20260507 08:19:59.735 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:19:59.735 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:59.735 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:59.735 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:59.735 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:19:59.735 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:59.735 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:59.735 | INFO | PRE_HQ : 28.7%, 1.0, 14.9ms, 0 | 20260507 08:19:59.735 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:59.735 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:19:59.735 | INFO | Uploaded 36GiB to GPU in 1.99555s (18.0401GiB/s) | 20260507 08:19:59.735 | INFO | Downloaded 34GiB to GPU in 1.96027s (17.3446GiB/s) | 20260507 08:19:59.735 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:19:59.735 | INFO | Total mem usage: 18 GiB | 20260507 08:19:59.735 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:19:59.735 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:19:59.735 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:19:59.735 | INFO | High Water Breakdown: | 20260507 08:19:59.735 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:19:59.735 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:19:59.735 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:19:59.735 | INFO | ------------------------------------------ | 20260507 08:19:59.735 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:19:59.735 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:19:59.735 | INFO | Total mem usage: 26.5 GiB | 20260507 08:19:59.735 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:19:59.735 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:19:59.735 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 08:19:59.735 | INFO | High Water Breakdown: | 20260507 08:19:59.735 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:19:59.735 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:19:59.735 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:19:59.735 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:19:59.735 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:19:59.735 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:19:59.735 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:19:59.735 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:19:59.735 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:19:59.735 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:19:59.735 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:19:59.735 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:19:59.735 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:19:59.735 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:19:59.735 | INFO | ------------------------------------------ | 20260507 08:19:59.735 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:19:59.735 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:19:59.735 | INFO | Total mem usage: 25.6 GiB | 20260507 08:19:59.735 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:19:59.735 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:19:59.736 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:19:59.736 | INFO | High Water Breakdown: | 20260507 08:19:59.736 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:19:59.736 | INFO | ------------------------------------------ | 20260507 08:19:59.736 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:19:59.736 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:19:59.736 | INFO | Total mem usage: 15.7 GiB | 20260507 08:19:59.736 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:19:59.736 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:19:59.736 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:19:59.736 | INFO | High Water Breakdown: | 20260507 08:19:59.736 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:19:59.736 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:19:59.736 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:19:59.736 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:19:59.736 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:19:59.736 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:19:59.736 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:19:59.736 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:19:59.736 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:19:59.736 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:19:59.736 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:19:59.736 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:19:59.736 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:19:59.736 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:19:59.736 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:19:59.736 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:19:59.736 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:19:59.736 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:19:59.736 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:19:59.736 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:19:59.736 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:19:59.736 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:19:59.736 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:19:59.736 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:19:59.736 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:19:59.736 | INFO | ------------------------------------------ | 20260507 08:19:59.736 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:19:59.736 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:19:59.736 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:19:59.736 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:19:59.736 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.33e+01 / 1.31e+00 | 20260507 08:19:59.736 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.04e-01, 1.04e-02 / 1.39e-02 | 20260507 08:19:59.736 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.40e+00, 5.07e-01 / 2.89e-01 | 20260507 08:19:59.736 | INFO | PerfReport: 3.24%, Histogram -- 9.94e-01 / 1.48e+00, 1.69e-01 / 1.26e-01 | 20260507 08:19:59.736 | INFO | PerfReport: 16.29%, DME -- 4.00e+01 / 4.48e+01, 3.98e+00 / 1.60e+00 | 20260507 08:19:59.736 | INFO | PerfReport: 57.77%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.34e+00 / 1.19e+00 | 20260507 08:19:59.736 | INFO | PerfReport: 6.86%, PulseAccumulating -- 2.11e+00 / 2.68e+00, 3.81e-01 / 1.85e-01 | 20260507 08:19:59.736 | INFO | PerfReport: 2.90%, Metrics -- 8.92e-01 / 1.21e+00, 2.41e-01 / 1.35e-01 | 20260507 08:19:59.736 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.57e-01, 5.15e-02 / 1.90e-02 | 20260507 08:19:59.736 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:19:59.736 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:19:59.736 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:19:59.736 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260507 08:19:59.736 | INFO | PerfReport: 21.89%, Upload -- 2.11e+01 / 0.00e+00, 7.19e+00 / 0.00e+00 | 20260507 08:19:59.736 | INFO | PerfReport: 21.30%, Download -- 2.05e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260507 08:19:59.736 | INFO | PerfReport: 31.94%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 08:19:59.736 | INFO | PerfReport: 1.66%, LockUpload -- 1.60e+00 / 0.00e+00, 5.05e+00 / 0.00e+00 | 20260507 08:19:59.736 | INFO | PerfReport: 1.17%, LockDownload -- 1.12e+00 / 0.00e+00, 2.79e+00 / 0.00e+00 | 20260507 08:19:59.736 | INFO | PerfReport: 22.03%, LockCompute -- 2.12e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 08:19:59.736 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.76e-02 / 0.00e+00 | 20260507 08:19:59.832 | INFO | Starting a round of Baz IO | 20260507 08:20:02.928 | INFO | Populating RTMetrics report | 20260507 08:20:02.929 | INFO | RTMetrics report populated | 20260507 08:20:02.929 | INFO | Opening tmp RTMetrics json file | 20260507 08:20:02.946 | INFO | Serializing RTMetrics to json | 20260507 08:20:02.947 | INFO | Serialization done, swapping tmp file now... | 20260507 08:20:02.948 | INFO | RTMetrics done | 20260507 08:20:03.012 | INFO | Finished a round of Baz IO: Wrote 1.617 GiB in 3.179 seconds | 20260507 08:20:03.012 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.50% / 0.00% / 67.50% | 20260507 08:20:03.012 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0924587% / 3.97364e-06% / 7.54992e-05% | 20260507 08:20:03.012 | INFO | Baz IO overall: | 20260507 08:20:03.012 | INFO | BytesWritten : 4226348716825 | 20260507 08:20:03.012 | INFO | NumEvents : 2021814867591 | 20260507 08:20:04.865 | INFO | Analyzing chunk frames = [8478720,8479232) | 20260507 08:20:04.865 | INFO | MemInfo: { | 20260507 08:20:04.865 | INFO | "gpu" : " 17 GB", | 20260507 08:20:04.865 | INFO | "huge" : "", | 20260507 08:20:04.865 | INFO | "huge_shared" : " 27 GB", | 20260507 08:20:04.865 | INFO | "malloc" : " 14 GB", | 20260507 08:20:04.865 | INFO | "pinned" : " 29 GB" | 20260507 08:20:04.865 | INFO | } | 20260507 08:20:04.865 | INFO | Perf report for the last 5.13039 seconds: | 20260507 08:20:04.865 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:20:04.865 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:04.865 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:04.865 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:04.865 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260507 08:20:04.865 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:04.865 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 08:20:04.865 | INFO | PRE_HQ : 58.8%, 1.8, 54.9ms, 0 | 20260507 08:20:04.865 | INFO | BAZWRITER : 61.5%, 3.0, 96.1ms, 0 | 20260507 08:20:04.865 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:20:04.865 | INFO | Uploaded 36GiB to GPU in 2.05025s (17.5588GiB/s) | 20260507 08:20:04.865 | INFO | Downloaded 38GiB to GPU in 2.16222s (17.5745GiB/s) | 20260507 08:20:10.007 | INFO | Analyzing chunk frames = [8479232,8479744) | 20260507 08:20:10.007 | INFO | MemInfo: { | 20260507 08:20:10.007 | INFO | "gpu" : " 17 GB", | 20260507 08:20:10.007 | INFO | "huge" : "", | 20260507 08:20:10.007 | INFO | "huge_shared" : " 27 GB", | 20260507 08:20:10.007 | INFO | "malloc" : " 14 GB", | 20260507 08:20:10.007 | INFO | "pinned" : " 29 GB" | 20260507 08:20:10.007 | INFO | } | 20260507 08:20:10.007 | INFO | Perf report for the last 5.14212 seconds: | 20260507 08:20:10.007 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:20:10.007 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:10.007 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:10.007 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:10.007 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:20:10.007 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:10.007 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:10.007 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:20:10.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:10.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:10.008 | INFO | Uploaded 36GiB to GPU in 1.99895s (18.0095GiB/s) | 20260507 08:20:10.008 | INFO | Downloaded 34GiB to GPU in 1.92977s (17.6186GiB/s) | 20260507 08:20:15.149 | INFO | Analyzing chunk frames = [8479744,8480256) | 20260507 08:20:15.149 | INFO | MemInfo: { | 20260507 08:20:15.149 | INFO | "gpu" : " 17 GB", | 20260507 08:20:15.149 | INFO | "huge" : "", | 20260507 08:20:15.149 | INFO | "huge_shared" : " 27 GB", | 20260507 08:20:15.149 | INFO | "malloc" : " 14 GB", | 20260507 08:20:15.149 | INFO | "pinned" : " 29 GB" | 20260507 08:20:15.149 | INFO | } | 20260507 08:20:15.149 | INFO | Perf report for the last 5.14135 seconds: | 20260507 08:20:15.149 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:20:15.149 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:15.149 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:15.149 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:15.149 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 08:20:15.149 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:15.149 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:15.149 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:20:15.149 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:15.149 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:15.149 | INFO | Uploaded 36GiB to GPU in 2.05745s (17.4974GiB/s) | 20260507 08:20:15.149 | INFO | Downloaded 34GiB to GPU in 1.94902s (17.4447GiB/s) | 20260507 08:20:20.285 | INFO | Analyzing chunk frames = [8480256,8480768) | 20260507 08:20:20.285 | INFO | MemInfo: { | 20260507 08:20:20.285 | INFO | "gpu" : " 17 GB", | 20260507 08:20:20.285 | INFO | "huge" : "", | 20260507 08:20:20.285 | INFO | "huge_shared" : " 27 GB", | 20260507 08:20:20.285 | INFO | "malloc" : " 14 GB", | 20260507 08:20:20.285 | INFO | "pinned" : " 29 GB" | 20260507 08:20:20.286 | INFO | } | 20260507 08:20:20.286 | INFO | Perf report for the last 5.13679 seconds: | 20260507 08:20:20.286 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:20:20.286 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:20.286 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:20.286 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:20.286 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 08:20:20.286 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:20.286 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:20.286 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 08:20:20.286 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:20.286 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:20.286 | INFO | Uploaded 36GiB to GPU in 2.05268s (17.538GiB/s) | 20260507 08:20:20.286 | INFO | Downloaded 34GiB to GPU in 1.94488s (17.4818GiB/s) | 20260507 08:20:25.420 | INFO | Analyzing chunk frames = [8480768,8481280) | 20260507 08:20:25.420 | INFO | MemInfo: { | 20260507 08:20:25.420 | INFO | "gpu" : " 17 GB", | 20260507 08:20:25.420 | INFO | "huge" : "", | 20260507 08:20:25.420 | INFO | "huge_shared" : " 27 GB", | 20260507 08:20:25.420 | INFO | "malloc" : " 14 GB", | 20260507 08:20:25.420 | INFO | "pinned" : " 29 GB" | 20260507 08:20:25.420 | INFO | } | 20260507 08:20:25.420 | INFO | Perf report for the last 5.13466 seconds: | 20260507 08:20:25.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:20:25.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:25.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:25.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:25.420 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 08:20:25.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:25.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:25.420 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 08:20:25.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:25.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:25.420 | INFO | Uploaded 36GiB to GPU in 2.06066s (17.4701GiB/s) | 20260507 08:20:25.420 | INFO | Downloaded 34GiB to GPU in 1.90311s (17.8655GiB/s) | 20260507 08:20:30.557 | INFO | Analyzing chunk frames = [8481280,8481792) | 20260507 08:20:30.557 | INFO | MemInfo: { | 20260507 08:20:30.557 | INFO | "gpu" : " 17 GB", | 20260507 08:20:30.557 | INFO | "huge" : "", | 20260507 08:20:30.557 | INFO | "huge_shared" : " 27 GB", | 20260507 08:20:30.557 | INFO | "malloc" : " 14 GB", | 20260507 08:20:30.557 | INFO | "pinned" : " 29 GB" | 20260507 08:20:30.557 | INFO | } | 20260507 08:20:30.557 | INFO | Perf report for the last 5.13694 seconds: | 20260507 08:20:30.557 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:20:30.557 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:30.557 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:30.557 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:30.557 | INFO | ANALYSIS : 60.3%, 3.0, 94.0ms, 0 | 20260507 08:20:30.557 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:30.557 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:30.557 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:20:30.557 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:30.557 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:30.557 | INFO | Uploaded 36GiB to GPU in 2.03846s (17.6604GiB/s) | 20260507 08:20:30.558 | INFO | Downloaded 34GiB to GPU in 1.9066s (17.8328GiB/s) | 20260507 08:20:35.703 | INFO | Analyzing chunk frames = [8481792,8482304) | 20260507 08:20:35.703 | INFO | MemInfo: { | 20260507 08:20:35.703 | INFO | "gpu" : " 17 GB", | 20260507 08:20:35.703 | INFO | "huge" : "", | 20260507 08:20:35.703 | INFO | "huge_shared" : " 27 GB", | 20260507 08:20:35.703 | INFO | "malloc" : " 14 GB", | 20260507 08:20:35.703 | INFO | "pinned" : " 29 GB" | 20260507 08:20:35.703 | INFO | } | 20260507 08:20:35.703 | INFO | Perf report for the last 5.14629 seconds: | 20260507 08:20:35.703 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:20:35.703 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:35.703 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:35.704 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:35.704 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 08:20:35.704 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:35.704 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:35.704 | INFO | PRE_HQ : 25.5%, 1.0, 13.2ms, 0 | 20260507 08:20:35.704 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:35.704 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:35.704 | INFO | Uploaded 36GiB to GPU in 1.97243s (18.2516GiB/s) | 20260507 08:20:35.704 | INFO | Downloaded 34GiB to GPU in 1.97309s (17.2319GiB/s) | 20260507 08:20:40.846 | INFO | Analyzing chunk frames = [8482304,8482816) | 20260507 08:20:40.846 | INFO | MemInfo: { | 20260507 08:20:40.846 | INFO | "gpu" : " 17 GB", | 20260507 08:20:40.846 | INFO | "huge" : "", | 20260507 08:20:40.846 | INFO | "huge_shared" : " 27 GB", | 20260507 08:20:40.846 | INFO | "malloc" : " 14 GB", | 20260507 08:20:40.846 | INFO | "pinned" : " 29 GB" | 20260507 08:20:40.846 | INFO | } | 20260507 08:20:40.846 | INFO | Perf report for the last 5.14241 seconds: | 20260507 08:20:40.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:20:40.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:40.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:40.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:40.846 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:20:40.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:40.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:40.846 | INFO | PRE_HQ : 28.4%, 1.0, 14.7ms, 0 | 20260507 08:20:40.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:40.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:40.846 | INFO | Uploaded 36GiB to GPU in 1.99495s (18.0455GiB/s) | 20260507 08:20:40.846 | INFO | Downloaded 34GiB to GPU in 1.95712s (17.3725GiB/s) | 20260507 08:20:40.944 | INFO | Starting a round of Baz IO | 20260507 08:20:44.037 | INFO | Populating RTMetrics report | 20260507 08:20:44.037 | INFO | RTMetrics report populated | 20260507 08:20:44.037 | INFO | Opening tmp RTMetrics json file | 20260507 08:20:44.041 | INFO | Serializing RTMetrics to json | 20260507 08:20:44.055 | INFO | Serialization done, swapping tmp file now... | 20260507 08:20:44.055 | INFO | RTMetrics done | 20260507 08:20:44.118 | INFO | Finished a round of Baz IO: Wrote 1.615 GiB in 3.173 seconds | 20260507 08:20:44.118 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.50% / 0.00% / 67.50% | 20260507 08:20:44.118 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0920931% / 3.97364e-06% / 5.96046e-05% | 20260507 08:20:44.118 | INFO | Baz IO overall: | 20260507 08:20:44.118 | INFO | BytesWritten : 4228083212441 | 20260507 08:20:44.118 | INFO | NumEvents : 2022623052267 | 20260507 08:20:45.985 | INFO | Analyzing chunk frames = [8482816,8483328) | 20260507 08:20:45.985 | INFO | MemInfo: { | 20260507 08:20:45.985 | INFO | "gpu" : " 17 GB", | 20260507 08:20:45.985 | INFO | "huge" : "", | 20260507 08:20:45.985 | INFO | "huge_shared" : " 27 GB", | 20260507 08:20:45.985 | INFO | "malloc" : " 14 GB", | 20260507 08:20:45.986 | INFO | "pinned" : " 29 GB" | 20260507 08:20:45.986 | INFO | } | 20260507 08:20:45.986 | INFO | Perf report for the last 5.13962 seconds: | 20260507 08:20:45.986 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:20:45.986 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:45.986 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:45.986 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:45.986 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 08:20:45.986 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:45.986 | INFO | RT_METRICS : 41.1%, 1.0, 21.6ms, 0 | 20260507 08:20:45.986 | INFO | PRE_HQ : 59.1%, 1.8, 54.4ms, 0 | 20260507 08:20:45.986 | INFO | BAZWRITER : 61.3%, 3.1, 97.2ms, 0 | 20260507 08:20:45.986 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:20:45.986 | INFO | Uploaded 36GiB to GPU in 2.07983s (17.3091GiB/s) | 20260507 08:20:45.986 | INFO | Downloaded 38GiB to GPU in 2.16911s (17.5187GiB/s) | 20260507 08:20:51.117 | INFO | Analyzing chunk frames = [8483328,8483840) | 20260507 08:20:51.117 | INFO | MemInfo: { | 20260507 08:20:51.117 | INFO | "gpu" : " 17 GB", | 20260507 08:20:51.117 | INFO | "huge" : "", | 20260507 08:20:51.117 | INFO | "huge_shared" : " 27 GB", | 20260507 08:20:51.117 | INFO | "malloc" : " 14 GB", | 20260507 08:20:51.117 | INFO | "pinned" : " 29 GB" | 20260507 08:20:51.117 | INFO | } | 20260507 08:20:51.117 | INFO | Perf report for the last 5.13152 seconds: | 20260507 08:20:51.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:20:51.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:51.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:51.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:51.117 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 08:20:51.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:51.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:51.117 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 08:20:51.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:51.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:51.117 | INFO | Uploaded 36GiB to GPU in 2.01229s (17.8901GiB/s) | 20260507 08:20:51.117 | INFO | Downloaded 34GiB to GPU in 1.93904s (17.5344GiB/s) | 20260507 08:20:56.261 | INFO | Analyzing chunk frames = [8483840,8484352) | 20260507 08:20:56.261 | INFO | MemInfo: { | 20260507 08:20:56.261 | INFO | "gpu" : " 17 GB", | 20260507 08:20:56.261 | INFO | "huge" : "", | 20260507 08:20:56.261 | INFO | "huge_shared" : " 27 GB", | 20260507 08:20:56.261 | INFO | "malloc" : " 14 GB", | 20260507 08:20:56.261 | INFO | "pinned" : " 29 GB" | 20260507 08:20:56.261 | INFO | } | 20260507 08:20:56.261 | INFO | Perf report for the last 5.14367 seconds: | 20260507 08:20:56.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:20:56.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:56.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:56.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:56.261 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 08:20:56.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:56.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:56.261 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:20:56.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:56.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:20:56.261 | INFO | Uploaded 36GiB to GPU in 2.05082s (17.554GiB/s) | 20260507 08:20:56.261 | INFO | Downloaded 34GiB to GPU in 1.95143s (17.4231GiB/s) | 20260507 08:21:01.402 | INFO | Analyzing chunk frames = [8484352,8484864) | 20260507 08:21:01.402 | INFO | MemInfo: { | 20260507 08:21:01.402 | INFO | "gpu" : " 17 GB", | 20260507 08:21:01.402 | INFO | "huge" : "", | 20260507 08:21:01.402 | INFO | "huge_shared" : " 27 GB", | 20260507 08:21:01.402 | INFO | "malloc" : " 14 GB", | 20260507 08:21:01.402 | INFO | "pinned" : " 29 GB" | 20260507 08:21:01.402 | INFO | } | 20260507 08:21:01.402 | INFO | Perf report for the last 5.14145 seconds: | 20260507 08:21:01.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:21:01.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:01.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:01.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:01.402 | INFO | ANALYSIS : 59.7%, 3.0, 92.6ms, 0 | 20260507 08:21:01.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:01.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:01.402 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:21:01.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:01.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:01.402 | INFO | Uploaded 36GiB to GPU in 2.02893s (17.7433GiB/s) | 20260507 08:21:01.402 | INFO | Downloaded 34GiB to GPU in 1.94486s (17.482GiB/s) | 20260507 08:21:06.544 | INFO | Analyzing chunk frames = [8484864,8485376) | 20260507 08:21:06.544 | INFO | MemInfo: { | 20260507 08:21:06.544 | INFO | "gpu" : " 17 GB", | 20260507 08:21:06.544 | INFO | "huge" : "", | 20260507 08:21:06.544 | INFO | "huge_shared" : " 27 GB", | 20260507 08:21:06.544 | INFO | "malloc" : " 14 GB", | 20260507 08:21:06.544 | INFO | "pinned" : " 29 GB" | 20260507 08:21:06.544 | INFO | } | 20260507 08:21:06.544 | INFO | Perf report for the last 5.14195 seconds: | 20260507 08:21:06.544 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:21:06.544 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:06.544 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:06.544 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:06.544 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 08:21:06.544 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:06.544 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:06.544 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:21:06.544 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:06.544 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:06.544 | INFO | Uploaded 36GiB to GPU in 2.04317s (17.6197GiB/s) | 20260507 08:21:06.545 | INFO | Downloaded 34GiB to GPU in 1.90789s (17.8208GiB/s) | 20260507 08:21:11.687 | INFO | Analyzing chunk frames = [8485376,8485888) | 20260507 08:21:11.687 | INFO | MemInfo: { | 20260507 08:21:11.687 | INFO | "gpu" : " 17 GB", | 20260507 08:21:11.687 | INFO | "huge" : "", | 20260507 08:21:11.687 | INFO | "huge_shared" : " 27 GB", | 20260507 08:21:11.687 | INFO | "malloc" : " 14 GB", | 20260507 08:21:11.687 | INFO | "pinned" : " 29 GB" | 20260507 08:21:11.687 | INFO | } | 20260507 08:21:11.687 | INFO | Perf report for the last 5.14283 seconds: | 20260507 08:21:11.687 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:21:11.687 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:11.687 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:11.687 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:11.687 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 08:21:11.687 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:11.687 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:11.687 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:21:11.687 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:11.687 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:11.687 | INFO | Uploaded 36GiB to GPU in 2.05105s (17.552GiB/s) | 20260507 08:21:11.687 | INFO | Downloaded 34GiB to GPU in 1.92286s (17.682GiB/s) | 20260507 08:21:16.816 | INFO | Analyzing chunk frames = [8485888,8486400) | 20260507 08:21:16.816 | INFO | MemInfo: { | 20260507 08:21:16.816 | INFO | "gpu" : " 17 GB", | 20260507 08:21:16.816 | INFO | "huge" : "", | 20260507 08:21:16.816 | INFO | "huge_shared" : " 27 GB", | 20260507 08:21:16.816 | INFO | "malloc" : " 14 GB", | 20260507 08:21:16.816 | INFO | "pinned" : " 29 GB" | 20260507 08:21:16.816 | INFO | } | 20260507 08:21:16.816 | INFO | Perf report for the last 5.12947 seconds: | 20260507 08:21:16.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:21:16.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:16.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:16.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:16.816 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 08:21:16.817 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:16.817 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:16.817 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 08:21:16.817 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:16.817 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:16.817 | INFO | Uploaded 36GiB to GPU in 1.97486s (18.2292GiB/s) | 20260507 08:21:16.817 | INFO | Downloaded 34GiB to GPU in 1.97443s (17.2201GiB/s) | 20260507 08:21:21.957 | INFO | Analyzing chunk frames = [8486400,8486912) | 20260507 08:21:21.957 | INFO | MemInfo: { | 20260507 08:21:21.957 | INFO | "gpu" : " 17 GB", | 20260507 08:21:21.957 | INFO | "huge" : "", | 20260507 08:21:21.957 | INFO | "huge_shared" : " 27 GB", | 20260507 08:21:21.957 | INFO | "malloc" : " 14 GB", | 20260507 08:21:21.957 | INFO | "pinned" : " 29 GB" | 20260507 08:21:21.957 | INFO | } | 20260507 08:21:21.957 | INFO | Perf report for the last 5.14027 seconds: | 20260507 08:21:21.957 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:21:21.957 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:21.957 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:21.957 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:21.957 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:21:21.957 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:21.957 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:21.957 | INFO | PRE_HQ : 28.6%, 1.0, 14.9ms, 0 | 20260507 08:21:21.957 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:21.957 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:21.957 | INFO | Uploaded 36GiB to GPU in 1.96755s (18.2969GiB/s) | 20260507 08:21:21.957 | INFO | Downloaded 34GiB to GPU in 1.96408s (17.3109GiB/s) | 20260507 08:21:21.957 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:21:21.957 | INFO | Total mem usage: 18 GiB | 20260507 08:21:21.957 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:21:21.957 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:21:21.957 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:21:21.957 | INFO | High Water Breakdown: | 20260507 08:21:21.957 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:21:21.957 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:21:21.957 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:21:21.957 | INFO | ------------------------------------------ | 20260507 08:21:21.957 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:21:21.957 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:21:21.957 | INFO | Total mem usage: 26.5 GiB | 20260507 08:21:21.957 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:21:21.957 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:21:21.957 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:21:21.957 | INFO | High Water Breakdown: | 20260507 08:21:21.957 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:21:21.957 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:21:21.957 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:21:21.957 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:21:21.957 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:21:21.957 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:21:21.957 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:21:21.957 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:21:21.957 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:21:21.957 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:21:21.957 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:21:21.957 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:21:21.957 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:21:21.957 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:21:21.958 | INFO | ------------------------------------------ | 20260507 08:21:21.958 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:21:21.958 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:21:21.958 | INFO | Total mem usage: 25.6 GiB | 20260507 08:21:21.958 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:21:21.958 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:21:21.958 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:21:21.958 | INFO | High Water Breakdown: | 20260507 08:21:21.958 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:21:21.958 | INFO | ------------------------------------------ | 20260507 08:21:21.958 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:21:21.958 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:21:21.958 | INFO | Total mem usage: 15.7 GiB | 20260507 08:21:21.958 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:21:21.958 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:21:21.958 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:21:21.958 | INFO | High Water Breakdown: | 20260507 08:21:21.958 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:21:21.958 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:21:21.958 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:21:21.958 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:21:21.958 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:21:21.958 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:21:21.958 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:21:21.958 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:21:21.958 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:21:21.958 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:21:21.958 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:21:21.958 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:21:21.958 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:21:21.958 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:21:21.958 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:21:21.958 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:21:21.958 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:21:21.958 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:21:21.958 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:21:21.958 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:21:21.958 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:21:21.958 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:21:21.958 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:21:21.958 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:21:21.958 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:21:21.958 | INFO | ------------------------------------------ | 20260507 08:21:21.958 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:21:21.958 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:21:21.958 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:21:21.958 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:21:21.958 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.44e+01, 1.33e+01 / 9.20e-01 | 20260507 08:21:21.958 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.03e-01, 9.38e-03 / 5.61e-03 | 20260507 08:21:21.958 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.44e+00, 5.05e-01 / 1.88e-01 | 20260507 08:21:21.958 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.47e+00, 1.65e-01 / 9.19e-02 | 20260507 08:21:21.958 | INFO | PerfReport: 16.29%, DME -- 4.00e+01 / 4.46e+01, 3.77e+00 / 1.14e+00 | 20260507 08:21:21.958 | INFO | PerfReport: 57.78%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.35e+00 / 9.42e-01 | 20260507 08:21:21.958 | INFO | PerfReport: 6.86%, PulseAccumulating -- 2.11e+00 / 2.64e+00, 3.82e-01 / 2.29e-01 | 20260507 08:21:21.958 | INFO | PerfReport: 2.91%, Metrics -- 8.92e-01 / 1.19e+00, 2.39e-01 / 1.38e-01 | 20260507 08:21:21.958 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.99e-01 / 3.53e-01, 5.04e-02 / 1.57e-02 | 20260507 08:21:21.958 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:21:21.958 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:21:21.958 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:21:21.958 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 08:21:21.958 | INFO | PerfReport: 21.96%, Upload -- 2.11e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 08:21:21.958 | INFO | PerfReport: 21.33%, Download -- 2.05e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260507 08:21:21.958 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 08:21:21.958 | INFO | PerfReport: 1.68%, LockUpload -- 1.61e+00 / 0.00e+00, 5.01e+00 / 0.00e+00 | 20260507 08:21:21.958 | INFO | PerfReport: 1.17%, LockDownload -- 1.12e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260507 08:21:21.958 | INFO | PerfReport: 21.92%, LockCompute -- 2.11e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 08:21:21.958 | INFO | PerfReport: 0.01%, Misc -- 1.43e-02 / 0.00e+00, 2.03e-02 / 0.00e+00 | 20260507 08:21:22.054 | INFO | Starting a round of Baz IO | 20260507 08:21:25.152 | INFO | Populating RTMetrics report | 20260507 08:21:25.153 | INFO | RTMetrics report populated | 20260507 08:21:25.153 | INFO | Opening tmp RTMetrics json file | 20260507 08:21:25.154 | INFO | Serializing RTMetrics to json | 20260507 08:21:25.155 | INFO | Serialization done, swapping tmp file now... | 20260507 08:21:25.155 | INFO | RTMetrics done | 20260507 08:21:25.236 | INFO | Finished a round of Baz IO: Wrote 1.615 GiB in 3.182 seconds | 20260507 08:21:25.236 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.50% / 0.00% / 67.50% | 20260507 08:21:25.237 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0919739% / 0% / 7.94729e-05% | 20260507 08:21:25.237 | INFO | Baz IO overall: | 20260507 08:21:25.237 | INFO | BytesWritten : 4229817343359 | 20260507 08:21:25.237 | INFO | NumEvents : 2023430769821 | 20260507 08:21:27.098 | INFO | Analyzing chunk frames = [8486912,8487424) | 20260507 08:21:27.098 | INFO | MemInfo: { | 20260507 08:21:27.098 | INFO | "gpu" : " 17 GB", | 20260507 08:21:27.098 | INFO | "huge" : "", | 20260507 08:21:27.098 | INFO | "huge_shared" : " 27 GB", | 20260507 08:21:27.098 | INFO | "malloc" : " 14 GB", | 20260507 08:21:27.098 | INFO | "pinned" : " 29 GB" | 20260507 08:21:27.098 | INFO | } | 20260507 08:21:27.098 | INFO | Perf report for the last 5.14134 seconds: | 20260507 08:21:27.098 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:21:27.098 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:27.098 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:27.098 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:27.098 | INFO | ANALYSIS : 61.5%, 2.9, 95.2ms, 0 | 20260507 08:21:27.098 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:27.098 | INFO | RT_METRICS : 42.7%, 1.0, 22.4ms, 0 | 20260507 08:21:27.098 | INFO | PRE_HQ : 59.6%, 1.9, 57.4ms, 0 | 20260507 08:21:27.098 | INFO | BAZWRITER : 61.4%, 3.5, 110.5ms, 0 | 20260507 08:21:27.098 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 08:21:27.098 | INFO | Uploaded 36GiB to GPU in 2.06018s (17.4742GiB/s) | 20260507 08:21:27.098 | INFO | Downloaded 38GiB to GPU in 2.18153s (17.419GiB/s) | 20260507 08:21:32.231 | INFO | Analyzing chunk frames = [8487424,8487936) | 20260507 08:21:32.231 | INFO | MemInfo: { | 20260507 08:21:32.231 | INFO | "gpu" : " 17 GB", | 20260507 08:21:32.231 | INFO | "huge" : "", | 20260507 08:21:32.231 | INFO | "huge_shared" : " 27 GB", | 20260507 08:21:32.231 | INFO | "malloc" : " 14 GB", | 20260507 08:21:32.231 | INFO | "pinned" : " 29 GB" | 20260507 08:21:32.231 | INFO | } | 20260507 08:21:32.231 | INFO | Perf report for the last 5.13315 seconds: | 20260507 08:21:32.231 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:21:32.231 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:32.231 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:32.231 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:32.231 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:21:32.231 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:32.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:32.231 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:21:32.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:32.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:32.231 | INFO | Uploaded 36GiB to GPU in 1.9865s (18.1223GiB/s) | 20260507 08:21:32.231 | INFO | Downloaded 34GiB to GPU in 1.92861s (17.6293GiB/s) | 20260507 08:21:37.373 | INFO | Analyzing chunk frames = [8487936,8488448) | 20260507 08:21:37.373 | INFO | MemInfo: { | 20260507 08:21:37.373 | INFO | "gpu" : " 17 GB", | 20260507 08:21:37.373 | INFO | "huge" : "", | 20260507 08:21:37.373 | INFO | "huge_shared" : " 27 GB", | 20260507 08:21:37.373 | INFO | "malloc" : " 14 GB", | 20260507 08:21:37.373 | INFO | "pinned" : " 29 GB" | 20260507 08:21:37.373 | INFO | } | 20260507 08:21:37.373 | INFO | Perf report for the last 5.14173 seconds: | 20260507 08:21:37.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:21:37.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:37.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:37.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:37.373 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 08:21:37.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:37.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:37.373 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 08:21:37.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:37.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:37.373 | INFO | Uploaded 36GiB to GPU in 2.0463s (17.5927GiB/s) | 20260507 08:21:37.373 | INFO | Downloaded 34GiB to GPU in 1.95557s (17.3862GiB/s) | 20260507 08:21:42.513 | INFO | Analyzing chunk frames = [8488448,8488960) | 20260507 08:21:42.513 | INFO | MemInfo: { | 20260507 08:21:42.513 | INFO | "gpu" : " 17 GB", | 20260507 08:21:42.513 | INFO | "huge" : "", | 20260507 08:21:42.513 | INFO | "huge_shared" : " 27 GB", | 20260507 08:21:42.513 | INFO | "malloc" : " 14 GB", | 20260507 08:21:42.513 | INFO | "pinned" : " 29 GB" | 20260507 08:21:42.513 | INFO | } | 20260507 08:21:42.513 | INFO | Perf report for the last 5.1401 seconds: | 20260507 08:21:42.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:21:42.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:42.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:42.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:42.513 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 08:21:42.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:42.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:42.513 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:21:42.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:42.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:42.513 | INFO | Uploaded 36GiB to GPU in 2.01058s (17.9053GiB/s) | 20260507 08:21:42.513 | INFO | Downloaded 34GiB to GPU in 1.9477s (17.4565GiB/s) | 20260507 08:21:47.659 | INFO | Analyzing chunk frames = [8488960,8489472) | 20260507 08:21:47.659 | INFO | MemInfo: { | 20260507 08:21:47.659 | INFO | "gpu" : " 17 GB", | 20260507 08:21:47.659 | INFO | "huge" : "", | 20260507 08:21:47.659 | INFO | "huge_shared" : " 27 GB", | 20260507 08:21:47.659 | INFO | "malloc" : " 14 GB", | 20260507 08:21:47.659 | INFO | "pinned" : " 29 GB" | 20260507 08:21:47.659 | INFO | } | 20260507 08:21:47.659 | INFO | Perf report for the last 5.14612 seconds: | 20260507 08:21:47.659 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:21:47.659 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:47.659 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:47.659 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:47.659 | INFO | ANALYSIS : 59.9%, 3.0, 93.4ms, 0 | 20260507 08:21:47.659 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:47.659 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:47.659 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:21:47.659 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:47.659 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:47.659 | INFO | Uploaded 36GiB to GPU in 2.02355s (17.7905GiB/s) | 20260507 08:21:47.660 | INFO | Downloaded 34GiB to GPU in 1.91432s (17.7609GiB/s) | 20260507 08:21:52.797 | INFO | Analyzing chunk frames = [8489472,8489984) | 20260507 08:21:52.797 | INFO | MemInfo: { | 20260507 08:21:52.797 | INFO | "gpu" : " 17 GB", | 20260507 08:21:52.797 | INFO | "huge" : "", | 20260507 08:21:52.797 | INFO | "huge_shared" : " 27 GB", | 20260507 08:21:52.797 | INFO | "malloc" : " 14 GB", | 20260507 08:21:52.797 | INFO | "pinned" : " 29 GB" | 20260507 08:21:52.797 | INFO | } | 20260507 08:21:52.797 | INFO | Perf report for the last 5.13751 seconds: | 20260507 08:21:52.797 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:21:52.797 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:52.797 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:52.797 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:52.797 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 08:21:52.797 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:52.797 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:52.797 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 08:21:52.797 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:52.797 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:52.797 | INFO | Uploaded 36GiB to GPU in 2.03423s (17.6972GiB/s) | 20260507 08:21:52.797 | INFO | Downloaded 34GiB to GPU in 1.9214s (17.6955GiB/s) | 20260507 08:21:57.926 | INFO | Analyzing chunk frames = [8489984,8490496) | 20260507 08:21:57.926 | INFO | MemInfo: { | 20260507 08:21:57.926 | INFO | "gpu" : " 17 GB", | 20260507 08:21:57.926 | INFO | "huge" : "", | 20260507 08:21:57.926 | INFO | "huge_shared" : " 27 GB", | 20260507 08:21:57.926 | INFO | "malloc" : " 14 GB", | 20260507 08:21:57.926 | INFO | "pinned" : " 29 GB" | 20260507 08:21:57.926 | INFO | } | 20260507 08:21:57.926 | INFO | Perf report for the last 5.12898 seconds: | 20260507 08:21:57.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:21:57.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:57.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:57.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:57.926 | INFO | ANALYSIS : 60.6%, 3.0, 94.1ms, 0 | 20260507 08:21:57.926 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:57.926 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:57.926 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 08:21:57.926 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:57.926 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:21:57.926 | INFO | Uploaded 36GiB to GPU in 1.88618s (19.0862GiB/s) | 20260507 08:21:57.926 | INFO | Downloaded 34GiB to GPU in 1.97766s (17.192GiB/s) | 20260507 08:22:03.064 | INFO | Analyzing chunk frames = [8490496,8491008) | 20260507 08:22:03.064 | INFO | MemInfo: { | 20260507 08:22:03.064 | INFO | "gpu" : " 17 GB", | 20260507 08:22:03.064 | INFO | "huge" : "", | 20260507 08:22:03.064 | INFO | "huge_shared" : " 27 GB", | 20260507 08:22:03.065 | INFO | "malloc" : " 14 GB", | 20260507 08:22:03.065 | INFO | "pinned" : " 29 GB" | 20260507 08:22:03.065 | INFO | } | 20260507 08:22:03.065 | INFO | Perf report for the last 5.13888 seconds: | 20260507 08:22:03.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:22:03.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:03.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:03.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:03.065 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:22:03.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:03.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:03.065 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 08:22:03.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:03.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:03.065 | INFO | Uploaded 36GiB to GPU in 1.98562s (18.1304GiB/s) | 20260507 08:22:03.065 | INFO | Downloaded 34GiB to GPU in 1.95539s (17.3878GiB/s) | 20260507 08:22:03.162 | INFO | Starting a round of Baz IO | 20260507 08:22:06.254 | INFO | Populating RTMetrics report | 20260507 08:22:06.254 | INFO | RTMetrics report populated | 20260507 08:22:06.254 | INFO | Opening tmp RTMetrics json file | 20260507 08:22:06.262 | INFO | Serializing RTMetrics to json | 20260507 08:22:06.270 | INFO | Serialization done, swapping tmp file now... | 20260507 08:22:06.270 | INFO | RTMetrics done | 20260507 08:22:06.342 | INFO | Finished a round of Baz IO: Wrote 1.616 GiB in 3.18 seconds | 20260507 08:22:06.342 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.50% / 0.00% / 67.50% | 20260507 08:22:06.342 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0917951% / 0% / 5.5631e-05% | 20260507 08:22:06.342 | INFO | Baz IO overall: | 20260507 08:22:06.342 | INFO | BytesWritten : 4231552748522 | 20260507 08:22:06.342 | INFO | NumEvents : 2024239570651 | 20260507 08:22:08.211 | INFO | Analyzing chunk frames = [8491008,8491520) | 20260507 08:22:08.211 | INFO | MemInfo: { | 20260507 08:22:08.211 | INFO | "gpu" : " 17 GB", | 20260507 08:22:08.211 | INFO | "huge" : "", | 20260507 08:22:08.211 | INFO | "huge_shared" : " 27 GB", | 20260507 08:22:08.211 | INFO | "malloc" : " 14 GB", | 20260507 08:22:08.211 | INFO | "pinned" : " 29 GB" | 20260507 08:22:08.211 | INFO | } | 20260507 08:22:08.211 | INFO | Perf report for the last 5.14633 seconds: | 20260507 08:22:08.211 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:22:08.211 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:08.211 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:08.211 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:08.211 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 08:22:08.211 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:08.211 | INFO | RT_METRICS : 41.0%, 1.0, 21.6ms, 0 | 20260507 08:22:08.211 | INFO | PRE_HQ : 58.6%, 1.8, 54.5ms, 0 | 20260507 08:22:08.211 | INFO | BAZWRITER : 61.3%, 3.1, 97.3ms, 0 | 20260507 08:22:08.211 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 08:22:08.211 | INFO | Uploaded 36GiB to GPU in 2.03823s (17.6624GiB/s) | 20260507 08:22:08.211 | INFO | Downloaded 38GiB to GPU in 2.16968s (17.5141GiB/s) | 20260507 08:22:13.355 | INFO | Analyzing chunk frames = [8491520,8492032) | 20260507 08:22:13.355 | INFO | MemInfo: { | 20260507 08:22:13.355 | INFO | "gpu" : " 17 GB", | 20260507 08:22:13.356 | INFO | "huge" : "", | 20260507 08:22:13.356 | INFO | "huge_shared" : " 27 GB", | 20260507 08:22:13.356 | INFO | "malloc" : " 14 GB", | 20260507 08:22:13.356 | INFO | "pinned" : " 29 GB" | 20260507 08:22:13.356 | INFO | } | 20260507 08:22:13.356 | INFO | Perf report for the last 5.14462 seconds: | 20260507 08:22:13.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:22:13.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:13.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:13.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:13.356 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:22:13.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:13.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:13.356 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:22:13.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:13.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:13.356 | INFO | Uploaded 36GiB to GPU in 1.98576s (18.1291GiB/s) | 20260507 08:22:13.356 | INFO | Downloaded 34GiB to GPU in 1.9339s (17.5811GiB/s) | 20260507 08:22:18.496 | INFO | Analyzing chunk frames = [8492032,8492544) | 20260507 08:22:18.496 | INFO | MemInfo: { | 20260507 08:22:18.496 | INFO | "gpu" : " 17 GB", | 20260507 08:22:18.496 | INFO | "huge" : "", | 20260507 08:22:18.496 | INFO | "huge_shared" : " 27 GB", | 20260507 08:22:18.496 | INFO | "malloc" : " 14 GB", | 20260507 08:22:18.496 | INFO | "pinned" : " 29 GB" | 20260507 08:22:18.496 | INFO | } | 20260507 08:22:18.496 | INFO | Perf report for the last 5.14071 seconds: | 20260507 08:22:18.496 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:22:18.496 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:18.496 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:18.496 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:18.496 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 08:22:18.496 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:18.496 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:18.496 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:22:18.496 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:18.496 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:18.497 | INFO | Uploaded 36GiB to GPU in 2.03721s (17.6712GiB/s) | 20260507 08:22:18.497 | INFO | Downloaded 34GiB to GPU in 1.91871s (17.7202GiB/s) | 20260507 08:22:23.631 | INFO | Analyzing chunk frames = [8492544,8493056) | 20260507 08:22:23.631 | INFO | MemInfo: { | 20260507 08:22:23.631 | INFO | "gpu" : " 17 GB", | 20260507 08:22:23.631 | INFO | "huge" : "", | 20260507 08:22:23.631 | INFO | "huge_shared" : " 27 GB", | 20260507 08:22:23.631 | INFO | "malloc" : " 14 GB", | 20260507 08:22:23.631 | INFO | "pinned" : " 29 GB" | 20260507 08:22:23.631 | INFO | } | 20260507 08:22:23.631 | INFO | Perf report for the last 5.13477 seconds: | 20260507 08:22:23.632 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:22:23.632 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:23.632 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:23.632 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:23.632 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 08:22:23.632 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:23.632 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:23.632 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:22:23.632 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:23.632 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:23.632 | INFO | Uploaded 36GiB to GPU in 2.05294s (17.5358GiB/s) | 20260507 08:22:23.632 | INFO | Downloaded 34GiB to GPU in 1.91365s (17.7671GiB/s) | 20260507 08:22:28.772 | INFO | Analyzing chunk frames = [8493056,8493568) | 20260507 08:22:28.772 | INFO | MemInfo: { | 20260507 08:22:28.772 | INFO | "gpu" : " 17 GB", | 20260507 08:22:28.772 | INFO | "huge" : "", | 20260507 08:22:28.772 | INFO | "huge_shared" : " 27 GB", | 20260507 08:22:28.772 | INFO | "malloc" : " 14 GB", | 20260507 08:22:28.772 | INFO | "pinned" : " 29 GB" | 20260507 08:22:28.772 | INFO | } | 20260507 08:22:28.772 | INFO | Perf report for the last 5.14098 seconds: | 20260507 08:22:28.772 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:22:28.772 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:28.772 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:28.772 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:28.772 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 08:22:28.772 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:28.772 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:28.772 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:22:28.772 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:28.772 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:28.773 | INFO | Uploaded 36GiB to GPU in 2.03772s (17.6668GiB/s) | 20260507 08:22:28.773 | INFO | Downloaded 34GiB to GPU in 1.92074s (17.7015GiB/s) | 20260507 08:22:33.906 | INFO | Analyzing chunk frames = [8493568,8494080) | 20260507 08:22:33.906 | INFO | MemInfo: { | 20260507 08:22:33.906 | INFO | "gpu" : " 17 GB", | 20260507 08:22:33.906 | INFO | "huge" : "", | 20260507 08:22:33.906 | INFO | "huge_shared" : " 27 GB", | 20260507 08:22:33.906 | INFO | "malloc" : " 14 GB", | 20260507 08:22:33.906 | INFO | "pinned" : " 29 GB" | 20260507 08:22:33.906 | INFO | } | 20260507 08:22:33.906 | INFO | Perf report for the last 5.13433 seconds: | 20260507 08:22:33.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:22:33.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:33.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:33.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:33.907 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:22:33.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:33.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:33.907 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 08:22:33.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:33.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:33.907 | INFO | Uploaded 36GiB to GPU in 2.06676s (17.4186GiB/s) | 20260507 08:22:33.907 | INFO | Downloaded 34GiB to GPU in 1.92508s (17.6616GiB/s) | 20260507 08:22:39.051 | INFO | Analyzing chunk frames = [8494080,8494592) | 20260507 08:22:39.051 | INFO | MemInfo: { | 20260507 08:22:39.051 | INFO | "gpu" : " 17 GB", | 20260507 08:22:39.051 | INFO | "huge" : "", | 20260507 08:22:39.051 | INFO | "huge_shared" : " 27 GB", | 20260507 08:22:39.051 | INFO | "malloc" : " 14 GB", | 20260507 08:22:39.051 | INFO | "pinned" : " 29 GB" | 20260507 08:22:39.051 | INFO | } | 20260507 08:22:39.051 | INFO | Perf report for the last 5.14496 seconds: | 20260507 08:22:39.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:22:39.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:39.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:39.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:39.052 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 08:22:39.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:39.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:39.052 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 08:22:39.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:39.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:39.052 | INFO | Uploaded 36GiB to GPU in 2.0305s (17.7296GiB/s) | 20260507 08:22:39.052 | INFO | Downloaded 34GiB to GPU in 1.98671s (17.1137GiB/s) | 20260507 08:22:44.181 | INFO | Analyzing chunk frames = [8494592,8495104) | 20260507 08:22:44.181 | INFO | MemInfo: { | 20260507 08:22:44.181 | INFO | "gpu" : " 17 GB", | 20260507 08:22:44.181 | INFO | "huge" : "", | 20260507 08:22:44.181 | INFO | "huge_shared" : " 27 GB", | 20260507 08:22:44.181 | INFO | "malloc" : " 14 GB", | 20260507 08:22:44.181 | INFO | "pinned" : " 29 GB" | 20260507 08:22:44.181 | INFO | } | 20260507 08:22:44.181 | INFO | Perf report for the last 5.12979 seconds: | 20260507 08:22:44.181 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:22:44.181 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:44.181 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:44.181 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:44.181 | INFO | ANALYSIS : 60.7%, 3.0, 94.2ms, 0 | 20260507 08:22:44.181 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:44.181 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:44.181 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:22:44.181 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:44.181 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:44.181 | INFO | Uploaded 36GiB to GPU in 1.98393s (18.1458GiB/s) | 20260507 08:22:44.181 | INFO | Downloaded 34GiB to GPU in 1.96705s (17.2848GiB/s) | 20260507 08:22:44.182 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:22:44.182 | INFO | Total mem usage: 18 GiB | 20260507 08:22:44.182 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:22:44.182 | INFO | Recent High Water Marks Sum: 14.9 GiB (Savings: -21.1%) | 20260507 08:22:44.182 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:22:44.182 | INFO | High Water Breakdown: | 20260507 08:22:44.182 | INFO | 60.8% : 9.05 GiB ( 79.3% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:22:44.182 | INFO | 39% : 5.8 GiB ( 86.7% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:22:44.182 | INFO | 0.134% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:22:44.182 | INFO | ------------------------------------------ | 20260507 08:22:44.182 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:22:44.182 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:22:44.182 | INFO | Total mem usage: 26.5 GiB | 20260507 08:22:44.182 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:22:44.182 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:22:44.182 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:22:44.182 | INFO | High Water Breakdown: | 20260507 08:22:44.182 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:22:44.182 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:22:44.182 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:22:44.182 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:22:44.182 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:22:44.182 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:22:44.182 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:22:44.182 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:22:44.182 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:22:44.182 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:22:44.182 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:22:44.182 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:22:44.182 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:22:44.182 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:22:44.182 | INFO | ------------------------------------------ | 20260507 08:22:44.182 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:22:44.182 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:22:44.182 | INFO | Total mem usage: 25.6 GiB | 20260507 08:22:44.182 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:22:44.182 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:22:44.182 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:22:44.182 | INFO | High Water Breakdown: | 20260507 08:22:44.182 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:22:44.182 | INFO | ------------------------------------------ | 20260507 08:22:44.182 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:22:44.182 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:22:44.182 | INFO | Total mem usage: 15.7 GiB | 20260507 08:22:44.182 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:22:44.182 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:22:44.182 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:22:44.182 | INFO | High Water Breakdown: | 20260507 08:22:44.182 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:22:44.182 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:22:44.182 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:22:44.182 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:22:44.182 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:22:44.182 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:22:44.182 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:22:44.182 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:22:44.182 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:22:44.182 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:22:44.182 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:22:44.182 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:22:44.182 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:22:44.182 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:22:44.182 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:22:44.182 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:22:44.182 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:22:44.182 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:22:44.182 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:22:44.182 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:22:44.183 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:22:44.183 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:22:44.183 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:22:44.183 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:22:44.183 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:22:44.183 | INFO | ------------------------------------------ | 20260507 08:22:44.183 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:22:44.183 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:22:44.183 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:22:44.183 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:22:44.183 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.44e+01, 1.31e+01 / 1.17e+00 | 20260507 08:22:44.183 | INFO | PerfReport: 0.30%, AggregateStats -- 9.11e-02 / 1.04e-01, 1.02e-02 / 9.64e-03 | 20260507 08:22:44.183 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.48e+00, 5.12e-01 / 2.10e-01 | 20260507 08:22:44.183 | INFO | PerfReport: 3.24%, Histogram -- 9.94e-01 / 1.47e+00, 1.67e-01 / 1.23e-01 | 20260507 08:22:44.183 | INFO | PerfReport: 16.25%, DME -- 3.99e+01 / 4.44e+01, 3.77e+00 / 1.25e+00 | 20260507 08:22:44.183 | INFO | PerfReport: 57.78%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.35e+00 / 1.50e+00 | 20260507 08:22:44.183 | INFO | PerfReport: 6.87%, PulseAccumulating -- 2.11e+00 / 2.68e+00, 3.82e-01 / 2.10e-01 | 20260507 08:22:44.183 | INFO | PerfReport: 2.91%, Metrics -- 8.93e-01 / 1.20e+00, 2.40e-01 / 1.01e-01 | 20260507 08:22:44.183 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.99e-01 / 3.53e-01, 5.04e-02 / 1.87e-02 | 20260507 08:22:44.183 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:22:44.183 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:22:44.183 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:22:44.183 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 08:22:44.183 | INFO | PerfReport: 21.87%, Upload -- 2.10e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 08:22:44.183 | INFO | PerfReport: 21.36%, Download -- 2.05e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260507 08:22:44.183 | INFO | PerfReport: 31.98%, Compute -- 3.07e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 08:22:44.183 | INFO | PerfReport: 1.55%, LockUpload -- 1.49e+00 / 0.00e+00, 4.91e+00 / 0.00e+00 | 20260507 08:22:44.183 | INFO | PerfReport: 1.25%, LockDownload -- 1.20e+00 / 0.00e+00, 2.99e+00 / 0.00e+00 | 20260507 08:22:44.183 | INFO | PerfReport: 21.98%, LockCompute -- 2.11e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 08:22:44.183 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.46e-02 / 0.00e+00 | 20260507 08:22:44.278 | INFO | Starting a round of Baz IO | 20260507 08:22:47.367 | INFO | Populating RTMetrics report | 20260507 08:22:47.367 | INFO | RTMetrics report populated | 20260507 08:22:47.367 | INFO | Opening tmp RTMetrics json file | 20260507 08:22:47.382 | INFO | Serializing RTMetrics to json | 20260507 08:22:47.384 | INFO | Serialization done, swapping tmp file now... | 20260507 08:22:47.384 | INFO | RTMetrics done | 20260507 08:22:47.452 | INFO | Finished a round of Baz IO: Wrote 1.615 GiB in 3.173 seconds | 20260507 08:22:47.452 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.50% / 0.00% / 67.50% | 20260507 08:22:47.452 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0917276% / 0% / 6.75519e-05% | 20260507 08:22:47.452 | INFO | Baz IO overall: | 20260507 08:22:47.452 | INFO | BytesWritten : 4233286687778 | 20260507 08:22:47.452 | INFO | NumEvents : 2025047477982 | 20260507 08:22:49.323 | INFO | Analyzing chunk frames = [8495104,8495616) | 20260507 08:22:49.323 | INFO | MemInfo: { | 20260507 08:22:49.323 | INFO | "gpu" : " 17 GB", | 20260507 08:22:49.323 | INFO | "huge" : "", | 20260507 08:22:49.323 | INFO | "huge_shared" : " 27 GB", | 20260507 08:22:49.323 | INFO | "malloc" : " 14 GB", | 20260507 08:22:49.323 | INFO | "pinned" : " 29 GB" | 20260507 08:22:49.323 | INFO | } | 20260507 08:22:49.323 | INFO | Perf report for the last 5.14213 seconds: | 20260507 08:22:49.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:22:49.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:49.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:49.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:49.324 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260507 08:22:49.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:49.324 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 08:22:49.324 | INFO | PRE_HQ : 58.8%, 1.8, 54.7ms, 0 | 20260507 08:22:49.324 | INFO | BAZWRITER : 61.3%, 3.1, 96.2ms, 0 | 20260507 08:22:49.324 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 08:22:49.324 | INFO | Uploaded 36GiB to GPU in 2.04745s (17.5828GiB/s) | 20260507 08:22:49.324 | INFO | Downloaded 38GiB to GPU in 2.18127s (17.421GiB/s) | 20260507 08:22:54.466 | INFO | Analyzing chunk frames = [8495616,8496128) | 20260507 08:22:54.466 | INFO | MemInfo: { | 20260507 08:22:54.466 | INFO | "gpu" : " 17 GB", | 20260507 08:22:54.466 | INFO | "huge" : "", | 20260507 08:22:54.466 | INFO | "huge_shared" : " 27 GB", | 20260507 08:22:54.466 | INFO | "malloc" : " 14 GB", | 20260507 08:22:54.466 | INFO | "pinned" : " 29 GB" | 20260507 08:22:54.466 | INFO | } | 20260507 08:22:54.467 | INFO | Perf report for the last 5.14312 seconds: | 20260507 08:22:54.467 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:22:54.467 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:54.467 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:54.467 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:54.467 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:22:54.467 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:54.467 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:54.467 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:22:54.467 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:54.467 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:54.467 | INFO | Uploaded 36GiB to GPU in 1.97592s (18.2194GiB/s) | 20260507 08:22:54.467 | INFO | Downloaded 34GiB to GPU in 1.93073s (17.61GiB/s) | 20260507 08:22:59.612 | INFO | Analyzing chunk frames = [8496128,8496640) | 20260507 08:22:59.612 | INFO | MemInfo: { | 20260507 08:22:59.612 | INFO | "gpu" : " 17 GB", | 20260507 08:22:59.612 | INFO | "huge" : "", | 20260507 08:22:59.612 | INFO | "huge_shared" : " 27 GB", | 20260507 08:22:59.612 | INFO | "malloc" : " 14 GB", | 20260507 08:22:59.612 | INFO | "pinned" : " 29 GB" | 20260507 08:22:59.612 | INFO | } | 20260507 08:22:59.612 | INFO | Perf report for the last 5.14518 seconds: | 20260507 08:22:59.612 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:22:59.612 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:59.612 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:59.612 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:59.612 | INFO | ANALYSIS : 60.3%, 3.0, 93.6ms, 0 | 20260507 08:22:59.612 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:59.612 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:59.612 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:22:59.612 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:59.612 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:22:59.612 | INFO | Uploaded 36GiB to GPU in 2.0141s (17.874GiB/s) | 20260507 08:22:59.612 | INFO | Downloaded 34GiB to GPU in 1.917s (17.7361GiB/s) | 20260507 08:23:04.744 | INFO | Analyzing chunk frames = [8496640,8497152) | 20260507 08:23:04.744 | INFO | MemInfo: { | 20260507 08:23:04.744 | INFO | "gpu" : " 17 GB", | 20260507 08:23:04.744 | INFO | "huge" : "", | 20260507 08:23:04.744 | INFO | "huge_shared" : " 27 GB", | 20260507 08:23:04.744 | INFO | "malloc" : " 14 GB", | 20260507 08:23:04.744 | INFO | "pinned" : " 29 GB" | 20260507 08:23:04.744 | INFO | } | 20260507 08:23:04.744 | INFO | Perf report for the last 5.13219 seconds: | 20260507 08:23:04.744 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:23:04.744 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:04.744 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:04.744 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:04.744 | INFO | ANALYSIS : 60.4%, 2.9, 93.1ms, 0 | 20260507 08:23:04.744 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:04.744 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:04.744 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 08:23:04.744 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:04.744 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:04.744 | INFO | Uploaded 36GiB to GPU in 2.02908s (17.742GiB/s) | 20260507 08:23:04.744 | INFO | Downloaded 34GiB to GPU in 1.91395s (17.7643GiB/s) | 20260507 08:23:09.875 | INFO | Analyzing chunk frames = [8497152,8497664) | 20260507 08:23:09.875 | INFO | MemInfo: { | 20260507 08:23:09.875 | INFO | "gpu" : " 17 GB", | 20260507 08:23:09.875 | INFO | "huge" : "", | 20260507 08:23:09.875 | INFO | "huge_shared" : " 27 GB", | 20260507 08:23:09.875 | INFO | "malloc" : " 14 GB", | 20260507 08:23:09.875 | INFO | "pinned" : " 29 GB" | 20260507 08:23:09.875 | INFO | } | 20260507 08:23:09.875 | INFO | Perf report for the last 5.13102 seconds: | 20260507 08:23:09.875 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:23:09.875 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:09.875 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:09.875 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:09.875 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 08:23:09.875 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:09.875 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:09.875 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:23:09.875 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:09.875 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:09.875 | INFO | Uploaded 36GiB to GPU in 2.04028s (17.6447GiB/s) | 20260507 08:23:09.875 | INFO | Downloaded 34GiB to GPU in 1.91441s (17.76GiB/s) | 20260507 08:23:15.019 | INFO | Analyzing chunk frames = [8497664,8498176) | 20260507 08:23:15.019 | INFO | MemInfo: { | 20260507 08:23:15.019 | INFO | "gpu" : " 17 GB", | 20260507 08:23:15.019 | INFO | "huge" : "", | 20260507 08:23:15.020 | INFO | "huge_shared" : " 27 GB", | 20260507 08:23:15.020 | INFO | "malloc" : " 14 GB", | 20260507 08:23:15.020 | INFO | "pinned" : " 29 GB" | 20260507 08:23:15.020 | INFO | } | 20260507 08:23:15.020 | INFO | Perf report for the last 5.14466 seconds: | 20260507 08:23:15.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:23:15.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:15.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:15.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:15.020 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 08:23:15.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:15.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:15.020 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 08:23:15.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:15.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:15.020 | INFO | Uploaded 36GiB to GPU in 2.02239s (17.8007GiB/s) | 20260507 08:23:15.020 | INFO | Downloaded 34GiB to GPU in 1.94232s (17.5049GiB/s) | 20260507 08:23:20.165 | INFO | Analyzing chunk frames = [8498176,8498688) | 20260507 08:23:20.165 | INFO | MemInfo: { | 20260507 08:23:20.165 | INFO | "gpu" : " 17 GB", | 20260507 08:23:20.165 | INFO | "huge" : "", | 20260507 08:23:20.165 | INFO | "huge_shared" : " 27 GB", | 20260507 08:23:20.165 | INFO | "malloc" : " 14 GB", | 20260507 08:23:20.165 | INFO | "pinned" : " 29 GB" | 20260507 08:23:20.165 | INFO | } | 20260507 08:23:20.165 | INFO | Perf report for the last 5.1452 seconds: | 20260507 08:23:20.165 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:23:20.165 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:20.165 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:20.165 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:20.165 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 08:23:20.165 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:20.165 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:20.165 | INFO | PRE_HQ : 25.7%, 1.0, 13.4ms, 0 | 20260507 08:23:20.165 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:20.165 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:20.165 | INFO | Uploaded 36GiB to GPU in 2.01774s (17.8417GiB/s) | 20260507 08:23:20.165 | INFO | Downloaded 34GiB to GPU in 1.98122s (17.1611GiB/s) | 20260507 08:23:25.306 | INFO | Analyzing chunk frames = [8498688,8499200) | 20260507 08:23:25.306 | INFO | MemInfo: { | 20260507 08:23:25.306 | INFO | "gpu" : " 17 GB", | 20260507 08:23:25.306 | INFO | "huge" : "", | 20260507 08:23:25.306 | INFO | "huge_shared" : " 27 GB", | 20260507 08:23:25.306 | INFO | "malloc" : " 14 GB", | 20260507 08:23:25.306 | INFO | "pinned" : " 29 GB" | 20260507 08:23:25.306 | INFO | } | 20260507 08:23:25.306 | INFO | Perf report for the last 5.14139 seconds: | 20260507 08:23:25.306 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:23:25.306 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:25.306 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:25.306 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:25.306 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:23:25.306 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:25.306 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:25.306 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:23:25.306 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:25.306 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:25.307 | INFO | Uploaded 36GiB to GPU in 1.99452s (18.0494GiB/s) | 20260507 08:23:25.307 | INFO | Downloaded 34GiB to GPU in 1.95586s (17.3837GiB/s) | 20260507 08:23:25.404 | INFO | Starting a round of Baz IO | 20260507 08:23:28.489 | INFO | Populating RTMetrics report | 20260507 08:23:28.489 | INFO | RTMetrics report populated | 20260507 08:23:28.489 | INFO | Opening tmp RTMetrics json file | 20260507 08:23:28.496 | INFO | Serializing RTMetrics to json | 20260507 08:23:28.506 | INFO | Serialization done, swapping tmp file now... | 20260507 08:23:28.506 | INFO | RTMetrics done | 20260507 08:23:28.572 | INFO | Finished a round of Baz IO: Wrote 1.614 GiB in 3.168 seconds | 20260507 08:23:28.572 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.50% / 0.00% / 67.50% | 20260507 08:23:28.572 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0911355% / 0% / 5.96046e-05% | 20260507 08:23:28.572 | INFO | Baz IO overall: | 20260507 08:23:28.572 | INFO | BytesWritten : 4235020088787 | 20260507 08:23:28.572 | INFO | NumEvents : 2025854994446 | 20260507 08:23:30.438 | INFO | Analyzing chunk frames = [8499200,8499712) | 20260507 08:23:30.438 | INFO | MemInfo: { | 20260507 08:23:30.438 | INFO | "gpu" : " 17 GB", | 20260507 08:23:30.438 | INFO | "huge" : "", | 20260507 08:23:30.439 | INFO | "huge_shared" : " 27 GB", | 20260507 08:23:30.439 | INFO | "malloc" : " 14 GB", | 20260507 08:23:30.439 | INFO | "pinned" : " 29 GB" | 20260507 08:23:30.439 | INFO | } | 20260507 08:23:30.439 | INFO | Perf report for the last 5.13236 seconds: | 20260507 08:23:30.439 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:23:30.439 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:30.439 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:30.439 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:30.439 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260507 08:23:30.439 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:30.439 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 08:23:30.439 | INFO | PRE_HQ : 58.7%, 1.8, 54.1ms, 0 | 20260507 08:23:30.439 | INFO | BAZWRITER : 61.3%, 3.0, 95.8ms, 0 | 20260507 08:23:30.439 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:23:30.439 | INFO | Uploaded 36GiB to GPU in 2.0522s (17.5421GiB/s) | 20260507 08:23:30.439 | INFO | Downloaded 38GiB to GPU in 2.16684s (17.5371GiB/s) | 20260507 08:23:35.582 | INFO | Analyzing chunk frames = [8499712,8500224) | 20260507 08:23:35.582 | INFO | MemInfo: { | 20260507 08:23:35.582 | INFO | "gpu" : " 17 GB", | 20260507 08:23:35.582 | INFO | "huge" : "", | 20260507 08:23:35.582 | INFO | "huge_shared" : " 27 GB", | 20260507 08:23:35.582 | INFO | "malloc" : " 14 GB", | 20260507 08:23:35.582 | INFO | "pinned" : " 29 GB" | 20260507 08:23:35.582 | INFO | } | 20260507 08:23:35.582 | INFO | Perf report for the last 5.14359 seconds: | 20260507 08:23:35.582 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:23:35.582 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:35.582 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:35.582 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:35.582 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:23:35.582 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:35.582 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:35.582 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:23:35.582 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:35.582 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:35.583 | INFO | Uploaded 36GiB to GPU in 1.9956s (18.0397GiB/s) | 20260507 08:23:35.583 | INFO | Downloaded 34GiB to GPU in 1.93423s (17.5781GiB/s) | 20260507 08:23:40.724 | INFO | Analyzing chunk frames = [8500224,8500736) | 20260507 08:23:40.724 | INFO | MemInfo: { | 20260507 08:23:40.724 | INFO | "gpu" : " 17 GB", | 20260507 08:23:40.724 | INFO | "huge" : "", | 20260507 08:23:40.724 | INFO | "huge_shared" : " 27 GB", | 20260507 08:23:40.724 | INFO | "malloc" : " 14 GB", | 20260507 08:23:40.724 | INFO | "pinned" : " 29 GB" | 20260507 08:23:40.724 | INFO | } | 20260507 08:23:40.724 | INFO | Perf report for the last 5.14226 seconds: | 20260507 08:23:40.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:23:40.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:40.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:40.725 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:40.725 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 08:23:40.725 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:40.725 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:40.725 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:23:40.725 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:40.725 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:40.725 | INFO | Uploaded 36GiB to GPU in 2.02048s (17.8175GiB/s) | 20260507 08:23:40.725 | INFO | Downloaded 34GiB to GPU in 1.91657s (17.74GiB/s) | 20260507 08:23:45.855 | INFO | Analyzing chunk frames = [8500736,8501248) | 20260507 08:23:45.855 | INFO | MemInfo: { | 20260507 08:23:45.855 | INFO | "gpu" : " 17 GB", | 20260507 08:23:45.855 | INFO | "huge" : "", | 20260507 08:23:45.855 | INFO | "huge_shared" : " 27 GB", | 20260507 08:23:45.855 | INFO | "malloc" : " 14 GB", | 20260507 08:23:45.855 | INFO | "pinned" : " 29 GB" | 20260507 08:23:45.855 | INFO | } | 20260507 08:23:45.856 | INFO | Perf report for the last 5.13101 seconds: | 20260507 08:23:45.856 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:23:45.856 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:45.856 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:45.856 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:45.856 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 08:23:45.856 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:45.856 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:45.856 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 08:23:45.856 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:45.856 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:45.856 | INFO | Uploaded 36GiB to GPU in 2.05117s (17.551GiB/s) | 20260507 08:23:45.856 | INFO | Downloaded 34GiB to GPU in 1.9452s (17.479GiB/s) | 20260507 08:23:50.999 | INFO | Analyzing chunk frames = [8501248,8501760) | 20260507 08:23:50.999 | INFO | MemInfo: { | 20260507 08:23:50.999 | INFO | "gpu" : " 17 GB", | 20260507 08:23:50.999 | INFO | "huge" : "", | 20260507 08:23:50.999 | INFO | "huge_shared" : " 27 GB", | 20260507 08:23:50.999 | INFO | "malloc" : " 14 GB", | 20260507 08:23:50.999 | INFO | "pinned" : " 29 GB" | 20260507 08:23:50.999 | INFO | } | 20260507 08:23:50.999 | INFO | Perf report for the last 5.14375 seconds: | 20260507 08:23:50.999 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:23:50.999 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:50.999 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:50.999 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:50.999 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 08:23:50.999 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:50.999 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:50.999 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:23:50.999 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:50.999 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:51.000 | INFO | Uploaded 36GiB to GPU in 2.04348s (17.617GiB/s) | 20260507 08:23:51.000 | INFO | Downloaded 34GiB to GPU in 1.92258s (17.6845GiB/s) | 20260507 08:23:56.133 | INFO | Analyzing chunk frames = [8501760,8502272) | 20260507 08:23:56.133 | INFO | MemInfo: { | 20260507 08:23:56.133 | INFO | "gpu" : " 17 GB", | 20260507 08:23:56.133 | INFO | "huge" : "", | 20260507 08:23:56.133 | INFO | "huge_shared" : " 27 GB", | 20260507 08:23:56.133 | INFO | "malloc" : " 14 GB", | 20260507 08:23:56.133 | INFO | "pinned" : " 29 GB" | 20260507 08:23:56.133 | INFO | } | 20260507 08:23:56.133 | INFO | Perf report for the last 5.13382 seconds: | 20260507 08:23:56.133 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:23:56.133 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:56.133 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:56.133 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:56.133 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:23:56.133 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:56.133 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:56.133 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 08:23:56.133 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:56.133 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:23:56.133 | INFO | Uploaded 36GiB to GPU in 2.07345s (17.3624GiB/s) | 20260507 08:23:56.133 | INFO | Downloaded 34GiB to GPU in 1.93507s (17.5704GiB/s) | 20260507 08:24:01.277 | INFO | Analyzing chunk frames = [8502272,8502784) | 20260507 08:24:01.277 | INFO | MemInfo: { | 20260507 08:24:01.277 | INFO | "gpu" : " 17 GB", | 20260507 08:24:01.277 | INFO | "huge" : "", | 20260507 08:24:01.277 | INFO | "huge_shared" : " 27 GB", | 20260507 08:24:01.277 | INFO | "malloc" : " 14 GB", | 20260507 08:24:01.277 | INFO | "pinned" : " 29 GB" | 20260507 08:24:01.277 | INFO | } | 20260507 08:24:01.277 | INFO | Perf report for the last 5.14416 seconds: | 20260507 08:24:01.277 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:24:01.277 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:01.277 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:01.277 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:01.277 | INFO | ANALYSIS : 60.6%, 3.0, 94.5ms, 0 | 20260507 08:24:01.277 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:01.277 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:01.277 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 08:24:01.277 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:01.277 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:01.278 | INFO | Uploaded 36GiB to GPU in 1.93749s (18.5808GiB/s) | 20260507 08:24:01.278 | INFO | Downloaded 34GiB to GPU in 1.97361s (17.2273GiB/s) | 20260507 08:24:06.417 | INFO | Analyzing chunk frames = [8502784,8503296) | 20260507 08:24:06.417 | INFO | MemInfo: { | 20260507 08:24:06.417 | INFO | "gpu" : " 17 GB", | 20260507 08:24:06.417 | INFO | "huge" : "", | 20260507 08:24:06.417 | INFO | "huge_shared" : " 27 GB", | 20260507 08:24:06.417 | INFO | "malloc" : " 14 GB", | 20260507 08:24:06.417 | INFO | "pinned" : " 29 GB" | 20260507 08:24:06.417 | INFO | } | 20260507 08:24:06.417 | INFO | Perf report for the last 5.13995 seconds: | 20260507 08:24:06.417 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:24:06.417 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:06.417 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:06.417 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:06.417 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:24:06.417 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:06.417 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:06.417 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 08:24:06.417 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:06.417 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:06.417 | INFO | Uploaded 36GiB to GPU in 1.9816s (18.1672GiB/s) | 20260507 08:24:06.417 | INFO | Downloaded 34GiB to GPU in 1.96209s (17.3285GiB/s) | 20260507 08:24:06.418 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:24:06.418 | INFO | Total mem usage: 18 GiB | 20260507 08:24:06.418 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:24:06.418 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:24:06.418 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:24:06.418 | INFO | High Water Breakdown: | 20260507 08:24:06.418 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:24:06.418 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:24:06.418 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:24:06.418 | INFO | ------------------------------------------ | 20260507 08:24:06.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:24:06.418 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:24:06.418 | INFO | Total mem usage: 26.5 GiB | 20260507 08:24:06.418 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:24:06.418 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:24:06.418 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:24:06.418 | INFO | High Water Breakdown: | 20260507 08:24:06.418 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:24:06.418 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:24:06.418 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:24:06.418 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:24:06.418 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:24:06.418 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:24:06.418 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:24:06.418 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:24:06.418 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:24:06.418 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:24:06.418 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:24:06.418 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:24:06.418 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:24:06.418 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:24:06.418 | INFO | ------------------------------------------ | 20260507 08:24:06.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:24:06.418 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:24:06.418 | INFO | Total mem usage: 25.6 GiB | 20260507 08:24:06.418 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:24:06.418 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:24:06.418 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:24:06.418 | INFO | High Water Breakdown: | 20260507 08:24:06.418 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:24:06.418 | INFO | ------------------------------------------ | 20260507 08:24:06.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:24:06.418 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:24:06.418 | INFO | Total mem usage: 15.7 GiB | 20260507 08:24:06.418 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:24:06.418 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:24:06.418 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:24:06.418 | INFO | High Water Breakdown: | 20260507 08:24:06.418 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:24:06.418 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:24:06.418 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:24:06.418 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:24:06.418 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:24:06.418 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:24:06.418 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:24:06.418 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:24:06.418 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:24:06.418 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:24:06.418 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:24:06.418 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:24:06.418 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:24:06.418 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:24:06.418 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:24:06.418 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:24:06.418 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:24:06.418 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:24:06.418 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:24:06.418 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:24:06.418 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:24:06.418 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:24:06.418 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:24:06.418 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:24:06.418 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:24:06.418 | INFO | ------------------------------------------ | 20260507 08:24:06.418 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:24:06.418 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:24:06.418 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:24:06.418 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:24:06.418 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.50e+01, 1.35e+01 / 1.22e+00 | 20260507 08:24:06.418 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.02e-01, 9.63e-03 / 4.27e-03 | 20260507 08:24:06.418 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.46e+00, 5.09e-01 / 2.04e-01 | 20260507 08:24:06.418 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.46e+00, 1.68e-01 / 1.25e-01 | 20260507 08:24:06.418 | INFO | PerfReport: 16.31%, DME -- 4.01e+01 / 4.50e+01, 3.95e+00 / 1.38e+00 | 20260507 08:24:06.418 | INFO | PerfReport: 57.76%, FrameLabeling -- 1.77e+01 / 1.97e+01, 1.35e+00 / 1.30e+00 | 20260507 08:24:06.418 | INFO | PerfReport: 6.86%, PulseAccumulating -- 2.11e+00 / 2.68e+00, 3.81e-01 / 2.24e-01 | 20260507 08:24:06.418 | INFO | PerfReport: 2.90%, Metrics -- 8.90e-01 / 1.21e+00, 2.38e-01 / 1.45e-01 | 20260507 08:24:06.418 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.99e-01 / 3.52e-01, 5.01e-02 / 1.90e-02 | 20260507 08:24:06.418 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:24:06.418 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:24:06.418 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:24:06.418 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 08:24:06.418 | INFO | PerfReport: 21.87%, Upload -- 2.10e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260507 08:24:06.418 | INFO | PerfReport: 21.32%, Download -- 2.05e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260507 08:24:06.418 | INFO | PerfReport: 31.96%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 08:24:06.418 | INFO | PerfReport: 1.62%, LockUpload -- 1.56e+00 / 0.00e+00, 4.97e+00 / 0.00e+00 | 20260507 08:24:06.418 | INFO | PerfReport: 1.22%, LockDownload -- 1.17e+00 / 0.00e+00, 2.90e+00 / 0.00e+00 | 20260507 08:24:06.418 | INFO | PerfReport: 22.00%, LockCompute -- 2.12e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260507 08:24:06.418 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260507 08:24:06.514 | INFO | Starting a round of Baz IO | 20260507 08:24:09.602 | INFO | Populating RTMetrics report | 20260507 08:24:09.603 | INFO | RTMetrics report populated | 20260507 08:24:09.603 | INFO | Opening tmp RTMetrics json file | 20260507 08:24:09.620 | INFO | Serializing RTMetrics to json | 20260507 08:24:09.621 | INFO | Serialization done, swapping tmp file now... | 20260507 08:24:09.621 | INFO | RTMetrics done | 20260507 08:24:09.686 | INFO | Finished a round of Baz IO: Wrote 1.614 GiB in 3.171 seconds | 20260507 08:24:09.686 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.50% / 0.00% / 67.50% | 20260507 08:24:09.686 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0912944% / 3.97364e-06% / 4.76837e-05% | 20260507 08:24:09.686 | INFO | Baz IO overall: | 20260507 08:24:09.686 | INFO | BytesWritten : 4236752962639 | 20260507 08:24:09.686 | INFO | NumEvents : 2026661990529 | 20260507 08:24:11.559 | INFO | Analyzing chunk frames = [8503296,8503808) | 20260507 08:24:11.559 | INFO | MemInfo: { | 20260507 08:24:11.559 | INFO | "gpu" : " 17 GB", | 20260507 08:24:11.559 | INFO | "huge" : "", | 20260507 08:24:11.559 | INFO | "huge_shared" : " 27 GB", | 20260507 08:24:11.559 | INFO | "malloc" : " 14 GB", | 20260507 08:24:11.559 | INFO | "pinned" : " 29 GB" | 20260507 08:24:11.559 | INFO | } | 20260507 08:24:11.559 | INFO | Perf report for the last 5.14219 seconds: | 20260507 08:24:11.559 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:24:11.559 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:11.559 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:11.559 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:11.559 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260507 08:24:11.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:11.560 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 08:24:11.560 | INFO | PRE_HQ : 58.9%, 1.8, 54.9ms, 0 | 20260507 08:24:11.560 | INFO | BAZWRITER : 61.3%, 3.2, 100.8ms, 0 | 20260507 08:24:11.560 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:24:11.560 | INFO | Uploaded 36GiB to GPU in 2.05056s (17.5561GiB/s) | 20260507 08:24:11.560 | INFO | Downloaded 38GiB to GPU in 2.17338s (17.4843GiB/s) | 20260507 08:24:16.691 | INFO | Analyzing chunk frames = [8503808,8504320) | 20260507 08:24:16.692 | INFO | MemInfo: { | 20260507 08:24:16.692 | INFO | "gpu" : " 17 GB", | 20260507 08:24:16.692 | INFO | "huge" : "", | 20260507 08:24:16.692 | INFO | "huge_shared" : " 27 GB", | 20260507 08:24:16.692 | INFO | "malloc" : " 14 GB", | 20260507 08:24:16.692 | INFO | "pinned" : " 29 GB" | 20260507 08:24:16.692 | INFO | } | 20260507 08:24:16.692 | INFO | Perf report for the last 5.13218 seconds: | 20260507 08:24:16.692 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:24:16.692 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:16.692 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:16.692 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:16.692 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260507 08:24:16.692 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:16.692 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:16.692 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:24:16.692 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:16.692 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:16.692 | INFO | Uploaded 36GiB to GPU in 2.01613s (17.856GiB/s) | 20260507 08:24:16.692 | INFO | Downloaded 34GiB to GPU in 1.95004s (17.4356GiB/s) | 20260507 08:24:21.828 | INFO | Analyzing chunk frames = [8504320,8504832) | 20260507 08:24:21.828 | INFO | MemInfo: { | 20260507 08:24:21.828 | INFO | "gpu" : " 17 GB", | 20260507 08:24:21.828 | INFO | "huge" : "", | 20260507 08:24:21.829 | INFO | "huge_shared" : " 27 GB", | 20260507 08:24:21.829 | INFO | "malloc" : " 14 GB", | 20260507 08:24:21.829 | INFO | "pinned" : " 29 GB" | 20260507 08:24:21.829 | INFO | } | 20260507 08:24:21.829 | INFO | Perf report for the last 5.13692 seconds: | 20260507 08:24:21.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:24:21.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:21.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:21.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:21.829 | INFO | ANALYSIS : 60.6%, 3.0, 93.7ms, 0 | 20260507 08:24:21.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:21.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:21.829 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:24:21.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:21.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:21.829 | INFO | Uploaded 36GiB to GPU in 2.04619s (17.5937GiB/s) | 20260507 08:24:21.829 | INFO | Downloaded 34GiB to GPU in 1.95438s (17.3968GiB/s) | 20260507 08:24:26.972 | INFO | Analyzing chunk frames = [8504832,8505344) | 20260507 08:24:26.972 | INFO | MemInfo: { | 20260507 08:24:26.972 | INFO | "gpu" : " 17 GB", | 20260507 08:24:26.972 | INFO | "huge" : "", | 20260507 08:24:26.972 | INFO | "huge_shared" : " 27 GB", | 20260507 08:24:26.972 | INFO | "malloc" : " 14 GB", | 20260507 08:24:26.972 | INFO | "pinned" : " 29 GB" | 20260507 08:24:26.972 | INFO | } | 20260507 08:24:26.972 | INFO | Perf report for the last 5.14314 seconds: | 20260507 08:24:26.972 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:24:26.972 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:26.972 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:26.972 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:26.972 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 08:24:26.972 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:26.972 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:26.972 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:24:26.972 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:26.972 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:26.972 | INFO | Uploaded 36GiB to GPU in 2.00094s (17.9915GiB/s) | 20260507 08:24:26.972 | INFO | Downloaded 34GiB to GPU in 1.93463s (17.5744GiB/s) | 20260507 08:24:32.117 | INFO | Analyzing chunk frames = [8505344,8505856) | 20260507 08:24:32.117 | INFO | MemInfo: { | 20260507 08:24:32.117 | INFO | "gpu" : " 17 GB", | 20260507 08:24:32.117 | INFO | "huge" : "", | 20260507 08:24:32.117 | INFO | "huge_shared" : " 27 GB", | 20260507 08:24:32.117 | INFO | "malloc" : " 14 GB", | 20260507 08:24:32.117 | INFO | "pinned" : " 29 GB" | 20260507 08:24:32.117 | INFO | } | 20260507 08:24:32.117 | INFO | Perf report for the last 5.14518 seconds: | 20260507 08:24:32.117 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:24:32.117 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:32.117 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:32.117 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:32.117 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 08:24:32.117 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:32.117 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:32.117 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:24:32.117 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:32.117 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:32.117 | INFO | Uploaded 36GiB to GPU in 2.02687s (17.7614GiB/s) | 20260507 08:24:32.117 | INFO | Downloaded 34GiB to GPU in 1.92071s (17.7018GiB/s) | 20260507 08:24:37.253 | INFO | Analyzing chunk frames = [8505856,8506368) | 20260507 08:24:37.253 | INFO | MemInfo: { | 20260507 08:24:37.253 | INFO | "gpu" : " 17 GB", | 20260507 08:24:37.253 | INFO | "huge" : "", | 20260507 08:24:37.253 | INFO | "huge_shared" : " 27 GB", | 20260507 08:24:37.253 | INFO | "malloc" : " 14 GB", | 20260507 08:24:37.253 | INFO | "pinned" : " 29 GB" | 20260507 08:24:37.253 | INFO | } | 20260507 08:24:37.253 | INFO | Perf report for the last 5.13577 seconds: | 20260507 08:24:37.253 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:24:37.253 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:37.253 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:37.253 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:37.253 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 08:24:37.253 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:37.253 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:37.253 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:24:37.253 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:37.253 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:37.253 | INFO | Uploaded 36GiB to GPU in 2.04081s (17.6401GiB/s) | 20260507 08:24:37.253 | INFO | Downloaded 34GiB to GPU in 1.9219s (17.6908GiB/s) | 20260507 08:24:42.385 | INFO | Analyzing chunk frames = [8506368,8506880) | 20260507 08:24:42.385 | INFO | MemInfo: { | 20260507 08:24:42.385 | INFO | "gpu" : " 17 GB", | 20260507 08:24:42.385 | INFO | "huge" : "", | 20260507 08:24:42.385 | INFO | "huge_shared" : " 27 GB", | 20260507 08:24:42.385 | INFO | "malloc" : " 14 GB", | 20260507 08:24:42.385 | INFO | "pinned" : " 29 GB" | 20260507 08:24:42.385 | INFO | } | 20260507 08:24:42.386 | INFO | Perf report for the last 5.13281 seconds: | 20260507 08:24:42.386 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:24:42.386 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:42.386 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:42.386 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:42.386 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 08:24:42.386 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:42.386 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:42.386 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 08:24:42.386 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:42.386 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:42.386 | INFO | Uploaded 36GiB to GPU in 1.89343s (19.0131GiB/s) | 20260507 08:24:42.386 | INFO | Downloaded 34GiB to GPU in 1.97905s (17.18GiB/s) | 20260507 08:24:47.533 | INFO | Analyzing chunk frames = [8506880,8507392) | 20260507 08:24:47.533 | INFO | MemInfo: { | 20260507 08:24:47.533 | INFO | "gpu" : " 17 GB", | 20260507 08:24:47.533 | INFO | "huge" : "", | 20260507 08:24:47.533 | INFO | "huge_shared" : " 27 GB", | 20260507 08:24:47.533 | INFO | "malloc" : " 14 GB", | 20260507 08:24:47.533 | INFO | "pinned" : " 29 GB" | 20260507 08:24:47.533 | INFO | } | 20260507 08:24:47.533 | INFO | Perf report for the last 5.14725 seconds: | 20260507 08:24:47.533 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:24:47.533 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:47.533 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:47.533 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:47.533 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:24:47.533 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:47.533 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:47.533 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 08:24:47.533 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:47.533 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:47.533 | INFO | Uploaded 36GiB to GPU in 1.97742s (18.2056GiB/s) | 20260507 08:24:47.533 | INFO | Downloaded 34GiB to GPU in 1.96285s (17.3218GiB/s) | 20260507 08:24:47.630 | INFO | Starting a round of Baz IO | 20260507 08:24:50.717 | INFO | Populating RTMetrics report | 20260507 08:24:50.717 | INFO | RTMetrics report populated | 20260507 08:24:50.717 | INFO | Opening tmp RTMetrics json file | 20260507 08:24:50.727 | INFO | Serializing RTMetrics to json | 20260507 08:24:50.731 | INFO | Serialization done, swapping tmp file now... | 20260507 08:24:50.732 | INFO | RTMetrics done | 20260507 08:24:50.799 | INFO | Finished a round of Baz IO: Wrote 1.615 GiB in 3.168 seconds | 20260507 08:24:50.799 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.50% / 0.00% / 67.50% | 20260507 08:24:50.799 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0911593% / 0% / 4.37101e-05% | 20260507 08:24:50.799 | INFO | Baz IO overall: | 20260507 08:24:50.799 | INFO | BytesWritten : 4238486891954 | 20260507 08:24:50.799 | INFO | NumEvents : 2027469771229 | 20260507 08:24:52.672 | INFO | Analyzing chunk frames = [8507392,8507904) | 20260507 08:24:52.672 | INFO | MemInfo: { | 20260507 08:24:52.672 | INFO | "gpu" : " 17 GB", | 20260507 08:24:52.672 | INFO | "huge" : "", | 20260507 08:24:52.672 | INFO | "huge_shared" : " 27 GB", | 20260507 08:24:52.672 | INFO | "malloc" : " 14 GB", | 20260507 08:24:52.672 | INFO | "pinned" : " 29 GB" | 20260507 08:24:52.672 | INFO | } | 20260507 08:24:52.672 | INFO | Perf report for the last 5.13889 seconds: | 20260507 08:24:52.672 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:24:52.672 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:52.672 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:52.672 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:52.672 | INFO | ANALYSIS : 61.4%, 3.0, 95.6ms, 0 | 20260507 08:24:52.672 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:52.672 | INFO | RT_METRICS : 43.7%, 1.0, 22.9ms, 0 | 20260507 08:24:52.672 | INFO | PRE_HQ : 59.2%, 1.9, 58.4ms, 0 | 20260507 08:24:52.672 | INFO | BAZWRITER : 61.3%, 3.3, 102.4ms, 0 | 20260507 08:24:52.672 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:24:52.672 | INFO | Uploaded 36GiB to GPU in 2.03602s (17.6815GiB/s) | 20260507 08:24:52.672 | INFO | Downloaded 38GiB to GPU in 2.16815s (17.5265GiB/s) | 20260507 08:24:57.806 | INFO | Analyzing chunk frames = [8507904,8508416) | 20260507 08:24:57.806 | INFO | MemInfo: { | 20260507 08:24:57.806 | INFO | "gpu" : " 17 GB", | 20260507 08:24:57.806 | INFO | "huge" : "", | 20260507 08:24:57.806 | INFO | "huge_shared" : " 27 GB", | 20260507 08:24:57.806 | INFO | "malloc" : " 14 GB", | 20260507 08:24:57.806 | INFO | "pinned" : " 29 GB" | 20260507 08:24:57.806 | INFO | } | 20260507 08:24:57.806 | INFO | Perf report for the last 5.13401 seconds: | 20260507 08:24:57.806 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:24:57.806 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:57.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:57.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:57.806 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 08:24:57.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:57.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:57.806 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 08:24:57.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:57.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:24:57.806 | INFO | Uploaded 36GiB to GPU in 1.99883s (18.0106GiB/s) | 20260507 08:24:57.806 | INFO | Downloaded 34GiB to GPU in 1.93876s (17.5369GiB/s) | 20260507 08:25:02.949 | INFO | Analyzing chunk frames = [8508416,8508928) | 20260507 08:25:02.949 | INFO | MemInfo: { | 20260507 08:25:02.949 | INFO | "gpu" : " 17 GB", | 20260507 08:25:02.949 | INFO | "huge" : "", | 20260507 08:25:02.949 | INFO | "huge_shared" : " 27 GB", | 20260507 08:25:02.949 | INFO | "malloc" : " 14 GB", | 20260507 08:25:02.949 | INFO | "pinned" : " 29 GB" | 20260507 08:25:02.949 | INFO | } | 20260507 08:25:02.949 | INFO | Perf report for the last 5.14321 seconds: | 20260507 08:25:02.949 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:25:02.949 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:02.949 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:02.949 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:02.949 | INFO | ANALYSIS : 60.3%, 3.0, 93.5ms, 0 | 20260507 08:25:02.949 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:02.949 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:02.949 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:25:02.949 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:02.949 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:02.949 | INFO | Uploaded 36GiB to GPU in 2.03854s (17.6597GiB/s) | 20260507 08:25:02.949 | INFO | Downloaded 34GiB to GPU in 1.95352s (17.4044GiB/s) | 20260507 08:25:08.086 | INFO | Analyzing chunk frames = [8508928,8509440) | 20260507 08:25:08.086 | INFO | MemInfo: { | 20260507 08:25:08.086 | INFO | "gpu" : " 17 GB", | 20260507 08:25:08.086 | INFO | "huge" : "", | 20260507 08:25:08.086 | INFO | "huge_shared" : " 27 GB", | 20260507 08:25:08.086 | INFO | "malloc" : " 14 GB", | 20260507 08:25:08.086 | INFO | "pinned" : " 29 GB" | 20260507 08:25:08.086 | INFO | } | 20260507 08:25:08.086 | INFO | Perf report for the last 5.13667 seconds: | 20260507 08:25:08.086 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:25:08.086 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:08.086 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:08.086 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:08.086 | INFO | ANALYSIS : 60.3%, 2.9, 93.2ms, 0 | 20260507 08:25:08.086 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:08.086 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:08.086 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:25:08.086 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:08.086 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:08.086 | INFO | Uploaded 36GiB to GPU in 2.0437s (17.6151GiB/s) | 20260507 08:25:08.086 | INFO | Downloaded 34GiB to GPU in 1.95279s (17.411GiB/s) | 20260507 08:25:13.227 | INFO | Analyzing chunk frames = [8509440,8509952) | 20260507 08:25:13.227 | INFO | MemInfo: { | 20260507 08:25:13.227 | INFO | "gpu" : " 17 GB", | 20260507 08:25:13.227 | INFO | "huge" : "", | 20260507 08:25:13.227 | INFO | "huge_shared" : " 27 GB", | 20260507 08:25:13.227 | INFO | "malloc" : " 14 GB", | 20260507 08:25:13.227 | INFO | "pinned" : " 29 GB" | 20260507 08:25:13.227 | INFO | } | 20260507 08:25:13.227 | INFO | Perf report for the last 5.1414 seconds: | 20260507 08:25:13.227 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:25:13.227 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:13.227 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:13.227 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:13.227 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 08:25:13.227 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:13.227 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:13.227 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:25:13.227 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:13.227 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:13.227 | INFO | Uploaded 36GiB to GPU in 2.02446s (17.7825GiB/s) | 20260507 08:25:13.227 | INFO | Downloaded 34GiB to GPU in 1.92008s (17.7076GiB/s) | 20260507 08:25:18.369 | INFO | Analyzing chunk frames = [8509952,8510464) | 20260507 08:25:18.369 | INFO | MemInfo: { | 20260507 08:25:18.369 | INFO | "gpu" : " 17 GB", | 20260507 08:25:18.369 | INFO | "huge" : "", | 20260507 08:25:18.369 | INFO | "huge_shared" : " 27 GB", | 20260507 08:25:18.369 | INFO | "malloc" : " 14 GB", | 20260507 08:25:18.369 | INFO | "pinned" : " 29 GB" | 20260507 08:25:18.369 | INFO | } | 20260507 08:25:18.369 | INFO | Perf report for the last 5.14171 seconds: | 20260507 08:25:18.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:25:18.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:18.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:18.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:18.369 | INFO | ANALYSIS : 60.4%, 3.0, 94.4ms, 0 | 20260507 08:25:18.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:18.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:18.369 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:25:18.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:18.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:18.369 | INFO | Uploaded 36GiB to GPU in 2.06614s (17.4238GiB/s) | 20260507 08:25:18.369 | INFO | Downloaded 34GiB to GPU in 1.93826s (17.5415GiB/s) | 20260507 08:25:23.504 | INFO | Analyzing chunk frames = [8510464,8510976) | 20260507 08:25:23.504 | INFO | MemInfo: { | 20260507 08:25:23.504 | INFO | "gpu" : " 17 GB", | 20260507 08:25:23.504 | INFO | "huge" : "", | 20260507 08:25:23.504 | INFO | "huge_shared" : " 27 GB", | 20260507 08:25:23.504 | INFO | "malloc" : " 14 GB", | 20260507 08:25:23.504 | INFO | "pinned" : " 29 GB" | 20260507 08:25:23.504 | INFO | } | 20260507 08:25:23.504 | INFO | Perf report for the last 5.13552 seconds: | 20260507 08:25:23.504 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:25:23.504 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:23.504 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:23.504 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:23.504 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 08:25:23.504 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:23.504 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:23.504 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:25:23.504 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:23.504 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:23.504 | INFO | Uploaded 36GiB to GPU in 1.91674s (18.7819GiB/s) | 20260507 08:25:23.505 | INFO | Downloaded 34GiB to GPU in 1.9811s (17.1622GiB/s) | 20260507 08:25:28.639 | INFO | Analyzing chunk frames = [8510976,8511488) | 20260507 08:25:28.639 | INFO | MemInfo: { | 20260507 08:25:28.639 | INFO | "gpu" : " 17 GB", | 20260507 08:25:28.639 | INFO | "huge" : "", | 20260507 08:25:28.639 | INFO | "huge_shared" : " 27 GB", | 20260507 08:25:28.639 | INFO | "malloc" : " 14 GB", | 20260507 08:25:28.639 | INFO | "pinned" : " 29 GB" | 20260507 08:25:28.639 | INFO | } | 20260507 08:25:28.639 | INFO | Perf report for the last 5.13481 seconds: | 20260507 08:25:28.639 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:25:28.639 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:28.639 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:28.639 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:28.639 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 08:25:28.639 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:28.639 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:28.639 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 08:25:28.639 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:28.639 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:28.639 | INFO | Uploaded 36GiB to GPU in 1.98163s (18.1668GiB/s) | 20260507 08:25:28.639 | INFO | Downloaded 34GiB to GPU in 1.95746s (17.3694GiB/s) | 20260507 08:25:28.639 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:25:28.639 | INFO | Total mem usage: 18 GiB | 20260507 08:25:28.639 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:25:28.639 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.6%) | 20260507 08:25:28.639 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:25:28.639 | INFO | High Water Breakdown: | 20260507 08:25:28.639 | INFO | 60.1% : 8.55 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:25:28.639 | INFO | 39.7% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:25:28.639 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:25:28.639 | INFO | ------------------------------------------ | 20260507 08:25:28.639 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:25:28.639 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:25:28.639 | INFO | Total mem usage: 26.5 GiB | 20260507 08:25:28.640 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:25:28.640 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:25:28.640 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:25:28.640 | INFO | High Water Breakdown: | 20260507 08:25:28.640 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:25:28.640 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:25:28.640 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:25:28.640 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:25:28.640 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:25:28.640 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:25:28.640 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:25:28.640 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:25:28.640 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:25:28.640 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:25:28.640 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:25:28.640 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:25:28.640 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:25:28.640 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:25:28.640 | INFO | ------------------------------------------ | 20260507 08:25:28.640 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:25:28.640 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:25:28.640 | INFO | Total mem usage: 25.6 GiB | 20260507 08:25:28.640 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:25:28.640 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:25:28.640 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:25:28.640 | INFO | High Water Breakdown: | 20260507 08:25:28.640 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:25:28.640 | INFO | ------------------------------------------ | 20260507 08:25:28.640 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:25:28.640 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:25:28.640 | INFO | Total mem usage: 15.7 GiB | 20260507 08:25:28.640 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:25:28.640 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:25:28.640 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:25:28.640 | INFO | High Water Breakdown: | 20260507 08:25:28.640 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:25:28.640 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:25:28.640 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:25:28.640 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:25:28.640 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:25:28.640 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:25:28.640 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:25:28.640 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:25:28.640 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:25:28.640 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:25:28.640 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:25:28.640 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:25:28.640 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:25:28.640 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:25:28.640 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:25:28.640 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:25:28.640 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:25:28.640 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:25:28.640 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:25:28.640 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:25:28.640 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:25:28.640 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:25:28.640 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:25:28.640 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:25:28.640 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:25:28.640 | INFO | ------------------------------------------ | 20260507 08:25:28.640 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:25:28.640 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:25:28.640 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:25:28.640 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:25:28.640 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.48e+01, 1.35e+01 / 1.14e+00 | 20260507 08:25:28.640 | INFO | PerfReport: 0.30%, AggregateStats -- 9.06e-02 / 1.02e-01, 1.04e-02 / 6.05e-03 | 20260507 08:25:28.640 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.44e+00, 5.05e-01 / 2.45e-01 | 20260507 08:25:28.640 | INFO | PerfReport: 3.22%, Histogram -- 9.88e-01 / 1.49e+00, 1.67e-01 / 1.08e-01 | 20260507 08:25:28.640 | INFO | PerfReport: 16.32%, DME -- 4.01e+01 / 4.49e+01, 4.00e+00 / 1.26e+00 | 20260507 08:25:28.640 | INFO | PerfReport: 57.71%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.28e+00 / 1.15e+00 | 20260507 08:25:28.640 | INFO | PerfReport: 6.89%, PulseAccumulating -- 2.11e+00 / 2.68e+00, 3.87e-01 / 1.51e-01 | 20260507 08:25:28.640 | INFO | PerfReport: 2.91%, Metrics -- 8.93e-01 / 1.19e+00, 2.40e-01 / 1.22e-01 | 20260507 08:25:28.640 | INFO | PerfReport: 0.98%, PulseSerializing -- 3.00e-01 / 3.55e-01, 5.02e-02 / 1.92e-02 | 20260507 08:25:28.640 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:25:28.640 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:25:28.640 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:25:28.640 | INFO | PerfReport: Full Execution -- 9.60e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 08:25:28.640 | INFO | PerfReport: 21.80%, Upload -- 2.09e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260507 08:25:28.640 | INFO | PerfReport: 21.43%, Download -- 2.06e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 08:25:28.640 | INFO | PerfReport: 31.95%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 08:25:28.640 | INFO | PerfReport: 1.44%, LockUpload -- 1.38e+00 / 0.00e+00, 4.82e+00 / 0.00e+00 | 20260507 08:25:28.640 | INFO | PerfReport: 1.37%, LockDownload -- 1.32e+00 / 0.00e+00, 3.16e+00 / 0.00e+00 | 20260507 08:25:28.640 | INFO | PerfReport: 21.99%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 08:25:28.640 | INFO | PerfReport: 0.01%, Misc -- 1.37e-02 / 0.00e+00, 1.79e-02 / 0.00e+00 | 20260507 08:25:28.736 | INFO | Starting a round of Baz IO | 20260507 08:25:31.832 | INFO | Populating RTMetrics report | 20260507 08:25:31.832 | INFO | RTMetrics report populated | 20260507 08:25:31.832 | INFO | Opening tmp RTMetrics json file | 20260507 08:25:31.847 | INFO | Serializing RTMetrics to json | 20260507 08:25:31.849 | INFO | Serialization done, swapping tmp file now... | 20260507 08:25:31.849 | INFO | RTMetrics done | 20260507 08:25:31.915 | INFO | Finished a round of Baz IO: Wrote 1.614 GiB in 3.178 seconds | 20260507 08:25:31.915 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.50% / 0.00% / 67.50% | 20260507 08:25:31.915 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0908852% / 3.97364e-06% / 7.54992e-05% | 20260507 08:25:31.915 | INFO | Baz IO overall: | 20260507 08:25:31.915 | INFO | BytesWritten : 4240219842731 | 20260507 08:25:31.915 | INFO | NumEvents : 2028276952564 | 20260507 08:25:33.783 | INFO | Analyzing chunk frames = [8511488,8512000) | 20260507 08:25:33.783 | INFO | MemInfo: { | 20260507 08:25:33.783 | INFO | "gpu" : " 17 GB", | 20260507 08:25:33.783 | INFO | "huge" : "", | 20260507 08:25:33.783 | INFO | "huge_shared" : " 27 GB", | 20260507 08:25:33.783 | INFO | "malloc" : " 14 GB", | 20260507 08:25:33.783 | INFO | "pinned" : " 29 GB" | 20260507 08:25:33.783 | INFO | } | 20260507 08:25:33.783 | INFO | Perf report for the last 5.144 seconds: | 20260507 08:25:33.783 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:25:33.783 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:33.783 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:33.783 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:33.783 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 08:25:33.783 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:33.783 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 08:25:33.783 | INFO | PRE_HQ : 58.8%, 1.8, 54.4ms, 0 | 20260507 08:25:33.783 | INFO | BAZWRITER : 61.4%, 3.1, 99.1ms, 0 | 20260507 08:25:33.783 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:25:33.783 | INFO | Uploaded 36GiB to GPU in 2.04371s (17.615GiB/s) | 20260507 08:25:33.783 | INFO | Downloaded 38GiB to GPU in 2.18316s (17.406GiB/s) | 20260507 08:25:38.926 | INFO | Analyzing chunk frames = [8512000,8512512) | 20260507 08:25:38.926 | INFO | MemInfo: { | 20260507 08:25:38.926 | INFO | "gpu" : " 17 GB", | 20260507 08:25:38.926 | INFO | "huge" : "", | 20260507 08:25:38.926 | INFO | "huge_shared" : " 27 GB", | 20260507 08:25:38.926 | INFO | "malloc" : " 14 GB", | 20260507 08:25:38.926 | INFO | "pinned" : " 29 GB" | 20260507 08:25:38.926 | INFO | } | 20260507 08:25:38.926 | INFO | Perf report for the last 5.14329 seconds: | 20260507 08:25:38.926 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:25:38.926 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:38.926 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:38.926 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:38.927 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 08:25:38.927 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:38.927 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:38.927 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:25:38.927 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:38.927 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:38.927 | INFO | Uploaded 36GiB to GPU in 1.99395s (18.0546GiB/s) | 20260507 08:25:38.927 | INFO | Downloaded 34GiB to GPU in 1.93451s (17.5755GiB/s) | 20260507 08:25:44.067 | INFO | Analyzing chunk frames = [8512512,8513024) | 20260507 08:25:44.067 | INFO | MemInfo: { | 20260507 08:25:44.067 | INFO | "gpu" : " 17 GB", | 20260507 08:25:44.067 | INFO | "huge" : "", | 20260507 08:25:44.067 | INFO | "huge_shared" : " 27 GB", | 20260507 08:25:44.067 | INFO | "malloc" : " 14 GB", | 20260507 08:25:44.067 | INFO | "pinned" : " 29 GB" | 20260507 08:25:44.067 | INFO | } | 20260507 08:25:44.067 | INFO | Perf report for the last 5.14074 seconds: | 20260507 08:25:44.067 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:25:44.067 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:44.067 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:44.067 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:44.067 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 08:25:44.067 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:44.067 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:44.067 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:25:44.067 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:44.067 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:44.067 | INFO | Uploaded 36GiB to GPU in 2.02295s (17.7958GiB/s) | 20260507 08:25:44.068 | INFO | Downloaded 34GiB to GPU in 1.92483s (17.6639GiB/s) | 20260507 08:25:49.198 | INFO | Analyzing chunk frames = [8513024,8513536) | 20260507 08:25:49.198 | INFO | MemInfo: { | 20260507 08:25:49.198 | INFO | "gpu" : " 17 GB", | 20260507 08:25:49.198 | INFO | "huge" : "", | 20260507 08:25:49.198 | INFO | "huge_shared" : " 27 GB", | 20260507 08:25:49.198 | INFO | "malloc" : " 14 GB", | 20260507 08:25:49.198 | INFO | "pinned" : " 29 GB" | 20260507 08:25:49.198 | INFO | } | 20260507 08:25:49.198 | INFO | Perf report for the last 5.13104 seconds: | 20260507 08:25:49.198 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:25:49.198 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:49.198 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:49.198 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:49.198 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 08:25:49.198 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:49.198 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:49.198 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:25:49.198 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:49.198 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:49.198 | INFO | Uploaded 36GiB to GPU in 2.03924s (17.6536GiB/s) | 20260507 08:25:49.198 | INFO | Downloaded 34GiB to GPU in 1.92004s (17.7079GiB/s) | 20260507 08:25:54.339 | INFO | Analyzing chunk frames = [8513536,8514048) | 20260507 08:25:54.339 | INFO | MemInfo: { | 20260507 08:25:54.339 | INFO | "gpu" : " 17 GB", | 20260507 08:25:54.339 | INFO | "huge" : "", | 20260507 08:25:54.339 | INFO | "huge_shared" : " 27 GB", | 20260507 08:25:54.339 | INFO | "malloc" : " 14 GB", | 20260507 08:25:54.339 | INFO | "pinned" : " 29 GB" | 20260507 08:25:54.339 | INFO | } | 20260507 08:25:54.339 | INFO | Perf report for the last 5.14099 seconds: | 20260507 08:25:54.339 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:25:54.339 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:54.339 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:54.340 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:54.340 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 08:25:54.340 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:54.340 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:54.340 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:25:54.340 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:54.340 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:54.340 | INFO | Uploaded 36GiB to GPU in 2.05049s (17.5568GiB/s) | 20260507 08:25:54.340 | INFO | Downloaded 34GiB to GPU in 1.95402s (17.4GiB/s) | 20260507 08:25:59.480 | INFO | Analyzing chunk frames = [8514048,8514560) | 20260507 08:25:59.480 | INFO | MemInfo: { | 20260507 08:25:59.480 | INFO | "gpu" : " 17 GB", | 20260507 08:25:59.480 | INFO | "huge" : "", | 20260507 08:25:59.480 | INFO | "huge_shared" : " 27 GB", | 20260507 08:25:59.480 | INFO | "malloc" : " 14 GB", | 20260507 08:25:59.480 | INFO | "pinned" : " 29 GB" | 20260507 08:25:59.480 | INFO | } | 20260507 08:25:59.480 | INFO | Perf report for the last 5.14095 seconds: | 20260507 08:25:59.480 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:25:59.480 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:59.480 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:59.480 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:59.480 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 08:25:59.480 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:59.480 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:59.480 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 08:25:59.480 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:59.480 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:25:59.481 | INFO | Uploaded 36GiB to GPU in 2.06129s (17.4648GiB/s) | 20260507 08:25:59.481 | INFO | Downloaded 34GiB to GPU in 1.92692s (17.6447GiB/s) | 20260507 08:26:04.625 | INFO | Analyzing chunk frames = [8514560,8515072) | 20260507 08:26:04.625 | INFO | MemInfo: { | 20260507 08:26:04.625 | INFO | "gpu" : " 17 GB", | 20260507 08:26:04.625 | INFO | "huge" : "", | 20260507 08:26:04.625 | INFO | "huge_shared" : " 27 GB", | 20260507 08:26:04.625 | INFO | "malloc" : " 14 GB", | 20260507 08:26:04.625 | INFO | "pinned" : " 29 GB" | 20260507 08:26:04.625 | INFO | } | 20260507 08:26:04.625 | INFO | Perf report for the last 5.14481 seconds: | 20260507 08:26:04.625 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:26:04.625 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:04.625 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:04.625 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:04.625 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:26:04.625 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:04.625 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:04.625 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 08:26:04.625 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:04.625 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:04.625 | INFO | Uploaded 36GiB to GPU in 1.92605s (18.6911GiB/s) | 20260507 08:26:04.625 | INFO | Downloaded 34GiB to GPU in 1.97462s (17.2185GiB/s) | 20260507 08:26:09.761 | INFO | Analyzing chunk frames = [8515072,8515584) | 20260507 08:26:09.761 | INFO | MemInfo: { | 20260507 08:26:09.761 | INFO | "gpu" : " 17 GB", | 20260507 08:26:09.761 | INFO | "huge" : "", | 20260507 08:26:09.761 | INFO | "huge_shared" : " 27 GB", | 20260507 08:26:09.761 | INFO | "malloc" : " 14 GB", | 20260507 08:26:09.761 | INFO | "pinned" : " 29 GB" | 20260507 08:26:09.761 | INFO | } | 20260507 08:26:09.762 | INFO | Perf report for the last 5.13649 seconds: | 20260507 08:26:09.762 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:26:09.762 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:09.762 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:09.762 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:09.762 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 08:26:09.762 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:09.762 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:09.762 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:26:09.762 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:09.762 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:09.762 | INFO | Uploaded 36GiB to GPU in 1.99074s (18.0837GiB/s) | 20260507 08:26:09.762 | INFO | Downloaded 34GiB to GPU in 1.95581s (17.3841GiB/s) | 20260507 08:26:09.859 | INFO | Starting a round of Baz IO | 20260507 08:26:12.952 | INFO | Populating RTMetrics report | 20260507 08:26:12.953 | INFO | RTMetrics report populated | 20260507 08:26:12.953 | INFO | Opening tmp RTMetrics json file | 20260507 08:26:12.960 | INFO | Serializing RTMetrics to json | 20260507 08:26:12.969 | INFO | Serialization done, swapping tmp file now... | 20260507 08:26:12.969 | INFO | RTMetrics done | 20260507 08:26:13.036 | INFO | Finished a round of Baz IO: Wrote 1.614 GiB in 3.177 seconds | 20260507 08:26:13.036 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.50% / 0.00% / 67.50% | 20260507 08:26:13.036 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0910163% / 7.94729e-06% / 5.16574e-05% | 20260507 08:26:13.036 | INFO | Baz IO overall: | 20260507 08:26:13.036 | INFO | BytesWritten : 4241953110423 | 20260507 08:26:13.036 | INFO | NumEvents : 2029084508126 | 20260507 08:26:14.899 | INFO | Analyzing chunk frames = [8515584,8516096) | 20260507 08:26:14.900 | INFO | MemInfo: { | 20260507 08:26:14.900 | INFO | "gpu" : " 17 GB", | 20260507 08:26:14.900 | INFO | "huge" : "", | 20260507 08:26:14.900 | INFO | "huge_shared" : " 27 GB", | 20260507 08:26:14.900 | INFO | "malloc" : " 14 GB", | 20260507 08:26:14.900 | INFO | "pinned" : " 29 GB" | 20260507 08:26:14.900 | INFO | } | 20260507 08:26:14.900 | INFO | Perf report for the last 5.1381 seconds: | 20260507 08:26:14.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:26:14.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:14.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:14.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:14.900 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 08:26:14.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:14.900 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 08:26:14.900 | INFO | PRE_HQ : 58.8%, 1.8, 54.3ms, 0 | 20260507 08:26:14.900 | INFO | BAZWRITER : 61.4%, 3.1, 97.6ms, 0 | 20260507 08:26:14.900 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:26:14.900 | INFO | Uploaded 36GiB to GPU in 2.07063s (17.386GiB/s) | 20260507 08:26:14.900 | INFO | Downloaded 38GiB to GPU in 2.17366s (17.4821GiB/s) | 20260507 08:26:20.037 | INFO | Analyzing chunk frames = [8516096,8516608) | 20260507 08:26:20.037 | INFO | MemInfo: { | 20260507 08:26:20.037 | INFO | "gpu" : " 17 GB", | 20260507 08:26:20.037 | INFO | "huge" : "", | 20260507 08:26:20.037 | INFO | "huge_shared" : " 27 GB", | 20260507 08:26:20.037 | INFO | "malloc" : " 14 GB", | 20260507 08:26:20.037 | INFO | "pinned" : " 29 GB" | 20260507 08:26:20.037 | INFO | } | 20260507 08:26:20.037 | INFO | Perf report for the last 5.1377 seconds: | 20260507 08:26:20.037 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:26:20.037 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:20.037 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:20.037 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:20.037 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:26:20.037 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:20.037 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:20.037 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260507 08:26:20.037 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:20.037 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:20.038 | INFO | Uploaded 36GiB to GPU in 1.98743s (18.1139GiB/s) | 20260507 08:26:20.038 | INFO | Downloaded 34GiB to GPU in 1.92724s (17.6418GiB/s) | 20260507 08:26:25.180 | INFO | Analyzing chunk frames = [8516608,8517120) | 20260507 08:26:25.180 | INFO | MemInfo: { | 20260507 08:26:25.180 | INFO | "gpu" : " 17 GB", | 20260507 08:26:25.180 | INFO | "huge" : "", | 20260507 08:26:25.180 | INFO | "huge_shared" : " 27 GB", | 20260507 08:26:25.180 | INFO | "malloc" : " 14 GB", | 20260507 08:26:25.180 | INFO | "pinned" : " 29 GB" | 20260507 08:26:25.180 | INFO | } | 20260507 08:26:25.180 | INFO | Perf report for the last 5.14294 seconds: | 20260507 08:26:25.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:26:25.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:25.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:25.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:25.180 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 08:26:25.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:25.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:25.180 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 08:26:25.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:25.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:25.180 | INFO | Uploaded 36GiB to GPU in 2.05309s (17.5345GiB/s) | 20260507 08:26:25.181 | INFO | Downloaded 34GiB to GPU in 1.95114s (17.4257GiB/s) | 20260507 08:26:30.315 | INFO | Analyzing chunk frames = [8517120,8517632) | 20260507 08:26:30.315 | INFO | MemInfo: { | 20260507 08:26:30.315 | INFO | "gpu" : " 17 GB", | 20260507 08:26:30.315 | INFO | "huge" : "", | 20260507 08:26:30.315 | INFO | "huge_shared" : " 27 GB", | 20260507 08:26:30.315 | INFO | "malloc" : " 14 GB", | 20260507 08:26:30.315 | INFO | "pinned" : " 29 GB" | 20260507 08:26:30.315 | INFO | } | 20260507 08:26:30.315 | INFO | Perf report for the last 5.13504 seconds: | 20260507 08:26:30.315 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:26:30.315 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:30.315 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:30.315 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:30.315 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 08:26:30.315 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:30.315 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:30.315 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 08:26:30.315 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:30.316 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:30.316 | INFO | Uploaded 36GiB to GPU in 2.05413s (17.5256GiB/s) | 20260507 08:26:30.316 | INFO | Downloaded 34GiB to GPU in 1.94435s (17.4866GiB/s) | 20260507 08:26:35.451 | INFO | Analyzing chunk frames = [8517632,8518144) | 20260507 08:26:35.451 | INFO | MemInfo: { | 20260507 08:26:35.451 | INFO | "gpu" : " 17 GB", | 20260507 08:26:35.451 | INFO | "huge" : "", | 20260507 08:26:35.451 | INFO | "huge_shared" : " 27 GB", | 20260507 08:26:35.451 | INFO | "malloc" : " 14 GB", | 20260507 08:26:35.451 | INFO | "pinned" : " 29 GB" | 20260507 08:26:35.451 | INFO | } | 20260507 08:26:35.451 | INFO | Perf report for the last 5.1356 seconds: | 20260507 08:26:35.451 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:26:35.451 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:35.451 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:35.451 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:35.451 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 08:26:35.451 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:35.451 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:35.451 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:26:35.451 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:35.451 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:35.451 | INFO | Uploaded 36GiB to GPU in 2.05905s (17.4838GiB/s) | 20260507 08:26:35.451 | INFO | Downloaded 34GiB to GPU in 1.90439s (17.8535GiB/s) | 20260507 08:26:40.596 | INFO | Analyzing chunk frames = [8518144,8518656) | 20260507 08:26:40.596 | INFO | MemInfo: { | 20260507 08:26:40.596 | INFO | "gpu" : " 17 GB", | 20260507 08:26:40.596 | INFO | "huge" : "", | 20260507 08:26:40.596 | INFO | "huge_shared" : " 27 GB", | 20260507 08:26:40.596 | INFO | "malloc" : " 14 GB", | 20260507 08:26:40.596 | INFO | "pinned" : " 29 GB" | 20260507 08:26:40.596 | INFO | } | 20260507 08:26:40.596 | INFO | Perf report for the last 5.14514 seconds: | 20260507 08:26:40.596 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:26:40.596 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:40.596 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:40.596 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:40.596 | INFO | ANALYSIS : 60.3%, 3.0, 94.3ms, 0 | 20260507 08:26:40.596 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:40.596 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:40.596 | INFO | PRE_HQ : 27.1%, 1.0, 14.1ms, 0 | 20260507 08:26:40.596 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:40.596 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:40.596 | INFO | Uploaded 36GiB to GPU in 2.06002s (17.4756GiB/s) | 20260507 08:26:40.597 | INFO | Downloaded 34GiB to GPU in 1.91961s (17.7119GiB/s) | 20260507 08:26:45.739 | INFO | Analyzing chunk frames = [8518656,8519168) | 20260507 08:26:45.739 | INFO | MemInfo: { | 20260507 08:26:45.739 | INFO | "gpu" : " 17 GB", | 20260507 08:26:45.739 | INFO | "huge" : "", | 20260507 08:26:45.739 | INFO | "huge_shared" : " 27 GB", | 20260507 08:26:45.739 | INFO | "malloc" : " 14 GB", | 20260507 08:26:45.739 | INFO | "pinned" : " 29 GB" | 20260507 08:26:45.739 | INFO | } | 20260507 08:26:45.739 | INFO | Perf report for the last 5.14279 seconds: | 20260507 08:26:45.739 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:26:45.739 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:45.739 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:45.739 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:45.739 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 08:26:45.739 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:45.739 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:45.739 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 08:26:45.739 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:45.739 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:45.739 | INFO | Uploaded 36GiB to GPU in 1.98665s (18.121GiB/s) | 20260507 08:26:45.739 | INFO | Downloaded 34GiB to GPU in 1.98591s (17.1206GiB/s) | 20260507 08:26:50.878 | INFO | Analyzing chunk frames = [8519168,8519680) | 20260507 08:26:50.878 | INFO | MemInfo: { | 20260507 08:26:50.878 | INFO | "gpu" : " 17 GB", | 20260507 08:26:50.878 | INFO | "huge" : "", | 20260507 08:26:50.878 | INFO | "huge_shared" : " 27 GB", | 20260507 08:26:50.878 | INFO | "malloc" : " 14 GB", | 20260507 08:26:50.878 | INFO | "pinned" : " 29 GB" | 20260507 08:26:50.878 | INFO | } | 20260507 08:26:50.878 | INFO | Perf report for the last 5.13919 seconds: | 20260507 08:26:50.878 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:26:50.878 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:50.878 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:50.878 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:50.878 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 08:26:50.878 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:50.878 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:50.878 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 08:26:50.878 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:50.878 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:50.878 | INFO | Uploaded 36GiB to GPU in 1.98355s (18.1493GiB/s) | 20260507 08:26:50.878 | INFO | Downloaded 34GiB to GPU in 1.96559s (17.2976GiB/s) | 20260507 08:26:50.879 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:26:50.879 | INFO | Total mem usage: 18 GiB | 20260507 08:26:50.879 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:26:50.879 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:26:50.879 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:26:50.879 | INFO | High Water Breakdown: | 20260507 08:26:50.879 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:26:50.879 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:26:50.879 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:26:50.879 | INFO | ------------------------------------------ | 20260507 08:26:50.879 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:26:50.879 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:26:50.879 | INFO | Total mem usage: 26.5 GiB | 20260507 08:26:50.879 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:26:50.879 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:26:50.879 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:26:50.879 | INFO | High Water Breakdown: | 20260507 08:26:50.879 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:26:50.879 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:26:50.879 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:26:50.879 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:26:50.879 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:26:50.879 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:26:50.879 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:26:50.879 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:26:50.879 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:26:50.879 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:26:50.879 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:26:50.879 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:26:50.879 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:26:50.879 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:26:50.879 | INFO | ------------------------------------------ | 20260507 08:26:50.879 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:26:50.879 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:26:50.879 | INFO | Total mem usage: 25.6 GiB | 20260507 08:26:50.879 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:26:50.879 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:26:50.879 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:26:50.879 | INFO | High Water Breakdown: | 20260507 08:26:50.879 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:26:50.879 | INFO | ------------------------------------------ | 20260507 08:26:50.879 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:26:50.879 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:26:50.879 | INFO | Total mem usage: 15.7 GiB | 20260507 08:26:50.879 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:26:50.879 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:26:50.879 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:26:50.879 | INFO | High Water Breakdown: | 20260507 08:26:50.879 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:26:50.879 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:26:50.879 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:26:50.879 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:26:50.879 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:26:50.879 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:26:50.879 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:26:50.879 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:26:50.879 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:26:50.879 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:26:50.879 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:26:50.879 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:26:50.879 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:26:50.879 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:26:50.879 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:26:50.879 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:26:50.879 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:26:50.879 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:26:50.879 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:26:50.879 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:26:50.879 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:26:50.879 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:26:50.879 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:26:50.879 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:26:50.879 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:26:50.879 | INFO | ------------------------------------------ | 20260507 08:26:50.879 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:26:50.880 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:26:50.880 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:26:50.880 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:26:50.880 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 9.20e-01 | 20260507 08:26:50.880 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.02e-01, 9.77e-03 / 4.73e-03 | 20260507 08:26:50.880 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.48e+00, 5.04e-01 / 2.19e-01 | 20260507 08:26:50.880 | INFO | PerfReport: 3.22%, Histogram -- 9.87e-01 / 1.50e+00, 1.66e-01 / 1.03e-01 | 20260507 08:26:50.880 | INFO | PerfReport: 16.33%, DME -- 4.01e+01 / 4.50e+01, 3.89e+00 / 1.01e+00 | 20260507 08:26:50.880 | INFO | PerfReport: 57.71%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.28e+00 / 1.15e+00 | 20260507 08:26:50.880 | INFO | PerfReport: 6.89%, PulseAccumulating -- 2.11e+00 / 2.70e+00, 3.86e-01 / 1.97e-01 | 20260507 08:26:50.880 | INFO | PerfReport: 2.91%, Metrics -- 8.92e-01 / 1.20e+00, 2.38e-01 / 1.21e-01 | 20260507 08:26:50.880 | INFO | PerfReport: 0.98%, PulseSerializing -- 2.99e-01 / 3.58e-01, 4.96e-02 / 2.27e-02 | 20260507 08:26:50.880 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:26:50.880 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:26:50.880 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:26:50.880 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260507 08:26:50.880 | INFO | PerfReport: 21.92%, Upload -- 2.11e+01 / 0.00e+00, 7.26e+00 / 0.00e+00 | 20260507 08:26:50.880 | INFO | PerfReport: 21.35%, Download -- 2.05e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260507 08:26:50.880 | INFO | PerfReport: 31.89%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 08:26:50.880 | INFO | PerfReport: 1.63%, LockUpload -- 1.57e+00 / 0.00e+00, 5.02e+00 / 0.00e+00 | 20260507 08:26:50.880 | INFO | PerfReport: 1.22%, LockDownload -- 1.18e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260507 08:26:50.880 | INFO | PerfReport: 21.97%, LockCompute -- 2.11e+01 / 0.00e+00, 1.87e+01 / 0.00e+00 | 20260507 08:26:50.880 | INFO | PerfReport: 0.01%, Misc -- 1.42e-02 / 0.00e+00, 1.96e-02 / 0.00e+00 | 20260507 08:26:50.975 | INFO | Starting a round of Baz IO | 20260507 08:26:54.067 | INFO | Populating RTMetrics report | 20260507 08:26:54.067 | INFO | RTMetrics report populated | 20260507 08:26:54.067 | INFO | Opening tmp RTMetrics json file | 20260507 08:26:54.083 | INFO | Serializing RTMetrics to json | 20260507 08:26:54.084 | INFO | Serialization done, swapping tmp file now... | 20260507 08:26:54.084 | INFO | RTMetrics done | 20260507 08:26:54.152 | INFO | Finished a round of Baz IO: Wrote 1.613 GiB in 3.176 seconds | 20260507 08:26:54.152 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.50% / 0.00% / 67.50% | 20260507 08:26:54.152 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0902136% / 0% / 8.34465e-05% | 20260507 08:26:54.152 | INFO | Baz IO overall: | 20260507 08:26:54.152 | INFO | BytesWritten : 4243684878094 | 20260507 08:26:54.152 | INFO | NumEvents : 2029890925099 | 20260507 08:26:56.012 | INFO | Analyzing chunk frames = [8519680,8520192) | 20260507 08:26:56.012 | INFO | MemInfo: { | 20260507 08:26:56.012 | INFO | "gpu" : " 17 GB", | 20260507 08:26:56.012 | INFO | "huge" : "", | 20260507 08:26:56.012 | INFO | "huge_shared" : " 27 GB", | 20260507 08:26:56.012 | INFO | "malloc" : " 14 GB", | 20260507 08:26:56.012 | INFO | "pinned" : " 29 GB" | 20260507 08:26:56.012 | INFO | } | 20260507 08:26:56.012 | INFO | Perf report for the last 5.13409 seconds: | 20260507 08:26:56.012 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:26:56.012 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:56.012 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:56.012 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:56.012 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 08:26:56.012 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:26:56.012 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 08:26:56.012 | INFO | PRE_HQ : 58.7%, 1.8, 54.5ms, 0 | 20260507 08:26:56.012 | INFO | BAZWRITER : 61.5%, 3.1, 96.5ms, 0 | 20260507 08:26:56.012 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:26:56.013 | INFO | Uploaded 36GiB to GPU in 2.06587s (17.4261GiB/s) | 20260507 08:26:56.013 | INFO | Downloaded 38GiB to GPU in 2.17783s (17.4485GiB/s) | 20260507 08:27:01.151 | INFO | Analyzing chunk frames = [8520192,8520704) | 20260507 08:27:01.151 | INFO | MemInfo: { | 20260507 08:27:01.151 | INFO | "gpu" : " 17 GB", | 20260507 08:27:01.151 | INFO | "huge" : "", | 20260507 08:27:01.151 | INFO | "huge_shared" : " 27 GB", | 20260507 08:27:01.151 | INFO | "malloc" : " 14 GB", | 20260507 08:27:01.151 | INFO | "pinned" : " 29 GB" | 20260507 08:27:01.151 | INFO | } | 20260507 08:27:01.151 | INFO | Perf report for the last 5.13869 seconds: | 20260507 08:27:01.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:27:01.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:01.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:01.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:01.151 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:27:01.151 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:01.151 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:01.151 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 08:27:01.151 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:01.151 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:01.151 | INFO | Uploaded 36GiB to GPU in 1.97251s (18.2509GiB/s) | 20260507 08:27:01.151 | INFO | Downloaded 34GiB to GPU in 1.93511s (17.57GiB/s) | 20260507 08:27:06.291 | INFO | Analyzing chunk frames = [8520704,8521216) | 20260507 08:27:06.291 | INFO | MemInfo: { | 20260507 08:27:06.291 | INFO | "gpu" : " 17 GB", | 20260507 08:27:06.291 | INFO | "huge" : "", | 20260507 08:27:06.291 | INFO | "huge_shared" : " 27 GB", | 20260507 08:27:06.291 | INFO | "malloc" : " 14 GB", | 20260507 08:27:06.291 | INFO | "pinned" : " 29 GB" | 20260507 08:27:06.291 | INFO | } | 20260507 08:27:06.291 | INFO | Perf report for the last 5.14041 seconds: | 20260507 08:27:06.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:27:06.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:06.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:06.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:06.291 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 08:27:06.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:06.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:06.291 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:27:06.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:06.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:06.292 | INFO | Uploaded 36GiB to GPU in 2.03572s (17.6842GiB/s) | 20260507 08:27:06.292 | INFO | Downloaded 34GiB to GPU in 1.95717s (17.372GiB/s) | 20260507 08:27:11.427 | INFO | Analyzing chunk frames = [8521216,8521728) | 20260507 08:27:11.427 | INFO | MemInfo: { | 20260507 08:27:11.427 | INFO | "gpu" : " 17 GB", | 20260507 08:27:11.427 | INFO | "huge" : "", | 20260507 08:27:11.427 | INFO | "huge_shared" : " 27 GB", | 20260507 08:27:11.427 | INFO | "malloc" : " 14 GB", | 20260507 08:27:11.427 | INFO | "pinned" : " 29 GB" | 20260507 08:27:11.427 | INFO | } | 20260507 08:27:11.427 | INFO | Perf report for the last 5.13581 seconds: | 20260507 08:27:11.427 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:27:11.427 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:11.427 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:11.427 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:11.427 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 08:27:11.427 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:11.427 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:11.427 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:27:11.427 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:11.427 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:11.427 | INFO | Uploaded 36GiB to GPU in 2.00401s (17.964GiB/s) | 20260507 08:27:11.428 | INFO | Downloaded 34GiB to GPU in 1.94027s (17.5233GiB/s) | 20260507 08:27:16.576 | INFO | Analyzing chunk frames = [8521728,8522240) | 20260507 08:27:16.576 | INFO | MemInfo: { | 20260507 08:27:16.576 | INFO | "gpu" : " 17 GB", | 20260507 08:27:16.576 | INFO | "huge" : "", | 20260507 08:27:16.576 | INFO | "huge_shared" : " 27 GB", | 20260507 08:27:16.576 | INFO | "malloc" : " 14 GB", | 20260507 08:27:16.576 | INFO | "pinned" : " 29 GB" | 20260507 08:27:16.576 | INFO | } | 20260507 08:27:16.576 | INFO | Perf report for the last 5.14904 seconds: | 20260507 08:27:16.576 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:27:16.576 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:16.576 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:16.576 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:16.576 | INFO | ANALYSIS : 60.0%, 3.0, 93.7ms, 0 | 20260507 08:27:16.576 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:16.576 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:16.576 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260507 08:27:16.576 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:16.576 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:16.576 | INFO | Uploaded 36GiB to GPU in 2.0283s (17.7489GiB/s) | 20260507 08:27:16.577 | INFO | Downloaded 34GiB to GPU in 1.92469s (17.6651GiB/s) | 20260507 08:27:21.712 | INFO | Analyzing chunk frames = [8522240,8522752) | 20260507 08:27:21.712 | INFO | MemInfo: { | 20260507 08:27:21.712 | INFO | "gpu" : " 17 GB", | 20260507 08:27:21.712 | INFO | "huge" : "", | 20260507 08:27:21.712 | INFO | "huge_shared" : " 27 GB", | 20260507 08:27:21.712 | INFO | "malloc" : " 14 GB", | 20260507 08:27:21.712 | INFO | "pinned" : " 29 GB" | 20260507 08:27:21.712 | INFO | } | 20260507 08:27:21.712 | INFO | Perf report for the last 5.13579 seconds: | 20260507 08:27:21.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:27:21.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:21.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:21.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:21.712 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 08:27:21.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:21.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:21.712 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:27:21.712 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:21.712 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:21.712 | INFO | Uploaded 36GiB to GPU in 2.06091s (17.468GiB/s) | 20260507 08:27:21.712 | INFO | Downloaded 34GiB to GPU in 1.94152s (17.512GiB/s) | 20260507 08:27:26.850 | INFO | Analyzing chunk frames = [8522752,8523264) | 20260507 08:27:26.850 | INFO | MemInfo: { | 20260507 08:27:26.850 | INFO | "gpu" : " 17 GB", | 20260507 08:27:26.850 | INFO | "huge" : "", | 20260507 08:27:26.850 | INFO | "huge_shared" : " 27 GB", | 20260507 08:27:26.850 | INFO | "malloc" : " 14 GB", | 20260507 08:27:26.850 | INFO | "pinned" : " 29 GB" | 20260507 08:27:26.850 | INFO | } | 20260507 08:27:26.850 | INFO | Perf report for the last 5.13774 seconds: | 20260507 08:27:26.850 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:27:26.850 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:26.850 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:26.850 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:26.850 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 08:27:26.850 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:26.850 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:26.850 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 08:27:26.850 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:26.850 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:26.850 | INFO | Uploaded 36GiB to GPU in 1.89815s (18.9659GiB/s) | 20260507 08:27:26.850 | INFO | Downloaded 34GiB to GPU in 1.97069s (17.2529GiB/s) | 20260507 08:27:31.988 | INFO | Analyzing chunk frames = [8523264,8523776) | 20260507 08:27:31.988 | INFO | MemInfo: { | 20260507 08:27:31.988 | INFO | "gpu" : " 17 GB", | 20260507 08:27:31.989 | INFO | "huge" : "", | 20260507 08:27:31.989 | INFO | "huge_shared" : " 27 GB", | 20260507 08:27:31.989 | INFO | "malloc" : " 14 GB", | 20260507 08:27:31.989 | INFO | "pinned" : " 29 GB" | 20260507 08:27:31.989 | INFO | } | 20260507 08:27:31.989 | INFO | Perf report for the last 5.13884 seconds: | 20260507 08:27:31.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:27:31.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:31.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:31.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:31.989 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:27:31.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:31.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:31.989 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:27:31.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:31.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:31.989 | INFO | Uploaded 36GiB to GPU in 1.98105s (18.1722GiB/s) | 20260507 08:27:31.989 | INFO | Downloaded 34GiB to GPU in 1.96451s (17.3071GiB/s) | 20260507 08:27:32.087 | INFO | Starting a round of Baz IO | 20260507 08:27:35.179 | INFO | Populating RTMetrics report | 20260507 08:27:35.180 | INFO | RTMetrics report populated | 20260507 08:27:35.180 | INFO | Opening tmp RTMetrics json file | 20260507 08:27:35.190 | INFO | Serializing RTMetrics to json | 20260507 08:27:35.194 | INFO | Serialization done, swapping tmp file now... | 20260507 08:27:35.195 | INFO | RTMetrics done | 20260507 08:27:35.261 | INFO | Finished a round of Baz IO: Wrote 1.613 GiB in 3.173 seconds | 20260507 08:27:35.261 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.49% / 0.00% / 67.50% | 20260507 08:27:35.261 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0908057% / 3.97364e-06% / 5.96046e-05% | 20260507 08:27:35.261 | INFO | Baz IO overall: | 20260507 08:27:35.261 | INFO | BytesWritten : 4245417067063 | 20260507 08:27:35.261 | INFO | NumEvents : 2030697740131 | 20260507 08:27:37.127 | INFO | Analyzing chunk frames = [8523776,8524288) | 20260507 08:27:37.127 | INFO | MemInfo: { | 20260507 08:27:37.127 | INFO | "gpu" : " 17 GB", | 20260507 08:27:37.127 | INFO | "huge" : "", | 20260507 08:27:37.127 | INFO | "huge_shared" : " 27 GB", | 20260507 08:27:37.127 | INFO | "malloc" : " 14 GB", | 20260507 08:27:37.127 | INFO | "pinned" : " 29 GB" | 20260507 08:27:37.127 | INFO | } | 20260507 08:27:37.127 | INFO | Perf report for the last 5.13813 seconds: | 20260507 08:27:37.127 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:27:37.127 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:37.127 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:37.127 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:37.127 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 08:27:37.127 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:37.127 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 08:27:37.127 | INFO | PRE_HQ : 58.9%, 1.8, 54.4ms, 0 | 20260507 08:27:37.127 | INFO | BAZWRITER : 61.4%, 3.1, 97.2ms, 0 | 20260507 08:27:37.127 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 08:27:37.127 | INFO | Uploaded 36GiB to GPU in 2.05927s (17.4819GiB/s) | 20260507 08:27:37.127 | INFO | Downloaded 38GiB to GPU in 2.18174s (17.4173GiB/s) | 20260507 08:27:42.267 | INFO | Analyzing chunk frames = [8524288,8524800) | 20260507 08:27:42.267 | INFO | MemInfo: { | 20260507 08:27:42.267 | INFO | "gpu" : " 17 GB", | 20260507 08:27:42.267 | INFO | "huge" : "", | 20260507 08:27:42.267 | INFO | "huge_shared" : " 27 GB", | 20260507 08:27:42.267 | INFO | "malloc" : " 14 GB", | 20260507 08:27:42.267 | INFO | "pinned" : " 29 GB" | 20260507 08:27:42.267 | INFO | } | 20260507 08:27:42.267 | INFO | Perf report for the last 5.14012 seconds: | 20260507 08:27:42.267 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:27:42.267 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:42.267 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:42.267 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:42.267 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:27:42.267 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:42.267 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:42.267 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260507 08:27:42.267 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:42.267 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:42.267 | INFO | Uploaded 36GiB to GPU in 1.99607s (18.0355GiB/s) | 20260507 08:27:42.267 | INFO | Downloaded 34GiB to GPU in 1.93745s (17.5488GiB/s) | 20260507 08:27:47.402 | INFO | Analyzing chunk frames = [8524800,8525312) | 20260507 08:27:47.402 | INFO | MemInfo: { | 20260507 08:27:47.402 | INFO | "gpu" : " 17 GB", | 20260507 08:27:47.402 | INFO | "huge" : "", | 20260507 08:27:47.402 | INFO | "huge_shared" : " 27 GB", | 20260507 08:27:47.402 | INFO | "malloc" : " 14 GB", | 20260507 08:27:47.402 | INFO | "pinned" : " 29 GB" | 20260507 08:27:47.402 | INFO | } | 20260507 08:27:47.402 | INFO | Perf report for the last 5.13479 seconds: | 20260507 08:27:47.402 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:27:47.402 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:47.402 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:47.402 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:47.402 | INFO | ANALYSIS : 60.7%, 3.0, 93.8ms, 0 | 20260507 08:27:47.402 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:47.402 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:47.402 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:27:47.402 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:47.402 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:47.402 | INFO | Uploaded 36GiB to GPU in 2.05871s (17.4867GiB/s) | 20260507 08:27:47.402 | INFO | Downloaded 34GiB to GPU in 1.95597s (17.3827GiB/s) | 20260507 08:27:52.547 | INFO | Analyzing chunk frames = [8525312,8525824) | 20260507 08:27:52.547 | INFO | MemInfo: { | 20260507 08:27:52.547 | INFO | "gpu" : " 17 GB", | 20260507 08:27:52.547 | INFO | "huge" : "", | 20260507 08:27:52.547 | INFO | "huge_shared" : " 27 GB", | 20260507 08:27:52.547 | INFO | "malloc" : " 14 GB", | 20260507 08:27:52.547 | INFO | "pinned" : " 29 GB" | 20260507 08:27:52.547 | INFO | } | 20260507 08:27:52.548 | INFO | Perf report for the last 5.14581 seconds: | 20260507 08:27:52.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:27:52.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:52.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:52.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:52.548 | INFO | ANALYSIS : 59.5%, 3.0, 92.5ms, 0 | 20260507 08:27:52.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:52.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:52.548 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:27:52.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:52.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:52.548 | INFO | Uploaded 36GiB to GPU in 2.01892s (17.8313GiB/s) | 20260507 08:27:52.548 | INFO | Downloaded 34GiB to GPU in 1.94753s (17.458GiB/s) | 20260507 08:27:57.691 | INFO | Analyzing chunk frames = [8525824,8526336) | 20260507 08:27:57.691 | INFO | MemInfo: { | 20260507 08:27:57.691 | INFO | "gpu" : " 17 GB", | 20260507 08:27:57.691 | INFO | "huge" : "", | 20260507 08:27:57.691 | INFO | "huge_shared" : " 27 GB", | 20260507 08:27:57.691 | INFO | "malloc" : " 14 GB", | 20260507 08:27:57.691 | INFO | "pinned" : " 29 GB" | 20260507 08:27:57.691 | INFO | } | 20260507 08:27:57.691 | INFO | Perf report for the last 5.14315 seconds: | 20260507 08:27:57.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:27:57.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:57.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:57.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:57.691 | INFO | ANALYSIS : 60.0%, 3.0, 93.6ms, 0 | 20260507 08:27:57.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:57.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:57.691 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:27:57.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:57.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:27:57.691 | INFO | Uploaded 36GiB to GPU in 2.03107s (17.7246GiB/s) | 20260507 08:27:57.691 | INFO | Downloaded 34GiB to GPU in 1.91411s (17.7628GiB/s) | 20260507 08:28:02.828 | INFO | Analyzing chunk frames = [8526336,8526848) | 20260507 08:28:02.828 | INFO | MemInfo: { | 20260507 08:28:02.828 | INFO | "gpu" : " 17 GB", | 20260507 08:28:02.829 | INFO | "huge" : "", | 20260507 08:28:02.829 | INFO | "huge_shared" : " 27 GB", | 20260507 08:28:02.829 | INFO | "malloc" : " 14 GB", | 20260507 08:28:02.829 | INFO | "pinned" : " 29 GB" | 20260507 08:28:02.829 | INFO | } | 20260507 08:28:02.829 | INFO | Perf report for the last 5.13793 seconds: | 20260507 08:28:02.829 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:28:02.829 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:02.829 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:02.829 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:02.829 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 08:28:02.829 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:02.829 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:02.829 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:28:02.829 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:02.829 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:02.829 | INFO | Uploaded 36GiB to GPU in 2.06329s (17.4479GiB/s) | 20260507 08:28:02.829 | INFO | Downloaded 34GiB to GPU in 1.94585s (17.4731GiB/s) | 20260507 08:28:07.955 | INFO | Analyzing chunk frames = [8526848,8527360) | 20260507 08:28:07.955 | INFO | MemInfo: { | 20260507 08:28:07.955 | INFO | "gpu" : " 17 GB", | 20260507 08:28:07.955 | INFO | "huge" : "", | 20260507 08:28:07.955 | INFO | "huge_shared" : " 27 GB", | 20260507 08:28:07.955 | INFO | "malloc" : " 14 GB", | 20260507 08:28:07.955 | INFO | "pinned" : " 29 GB" | 20260507 08:28:07.955 | INFO | } | 20260507 08:28:07.955 | INFO | Perf report for the last 5.12674 seconds: | 20260507 08:28:07.955 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:28:07.955 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:07.955 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:07.955 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:07.955 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 08:28:07.955 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:07.955 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:07.955 | INFO | PRE_HQ : 26.1%, 1.0, 13.5ms, 0 | 20260507 08:28:07.955 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:07.955 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:07.956 | INFO | Uploaded 36GiB to GPU in 1.9478s (18.4824GiB/s) | 20260507 08:28:07.956 | INFO | Downloaded 34GiB to GPU in 1.98789s (17.1035GiB/s) | 20260507 08:28:13.101 | INFO | Analyzing chunk frames = [8527360,8527872) | 20260507 08:28:13.101 | INFO | MemInfo: { | 20260507 08:28:13.101 | INFO | "gpu" : " 17 GB", | 20260507 08:28:13.101 | INFO | "huge" : "", | 20260507 08:28:13.101 | INFO | "huge_shared" : " 27 GB", | 20260507 08:28:13.101 | INFO | "malloc" : " 14 GB", | 20260507 08:28:13.101 | INFO | "pinned" : " 29 GB" | 20260507 08:28:13.101 | INFO | } | 20260507 08:28:13.101 | INFO | Perf report for the last 5.14564 seconds: | 20260507 08:28:13.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:28:13.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:13.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:13.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:13.101 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 08:28:13.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:13.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:13.101 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:28:13.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:13.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:13.101 | INFO | Uploaded 36GiB to GPU in 1.94289s (18.5291GiB/s) | 20260507 08:28:13.101 | INFO | Downloaded 34GiB to GPU in 1.9595s (17.3513GiB/s) | 20260507 08:28:13.101 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:28:13.101 | INFO | Total mem usage: 18 GiB | 20260507 08:28:13.101 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:28:13.101 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -27%) | 20260507 08:28:13.101 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:28:13.101 | INFO | High Water Breakdown: | 20260507 08:28:13.101 | INFO | 60.1% : 8.52 GiB ( 74.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:28:13.101 | INFO | 39.7% : 5.63 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:28:13.101 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:28:13.101 | INFO | ------------------------------------------ | 20260507 08:28:13.101 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:28:13.101 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:28:13.101 | INFO | Total mem usage: 26.5 GiB | 20260507 08:28:13.101 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:28:13.101 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:28:13.101 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 08:28:13.101 | INFO | High Water Breakdown: | 20260507 08:28:13.101 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:28:13.101 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:28:13.101 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:28:13.102 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:28:13.102 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:28:13.102 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:28:13.102 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:28:13.102 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:28:13.102 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:28:13.102 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:28:13.102 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:28:13.102 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:28:13.102 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:28:13.102 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:28:13.102 | INFO | ------------------------------------------ | 20260507 08:28:13.102 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:28:13.102 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:28:13.102 | INFO | Total mem usage: 25.6 GiB | 20260507 08:28:13.102 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:28:13.102 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:28:13.102 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:28:13.102 | INFO | High Water Breakdown: | 20260507 08:28:13.102 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:28:13.102 | INFO | ------------------------------------------ | 20260507 08:28:13.102 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:28:13.102 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:28:13.102 | INFO | Total mem usage: 15.7 GiB | 20260507 08:28:13.102 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:28:13.102 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:28:13.102 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:28:13.102 | INFO | High Water Breakdown: | 20260507 08:28:13.102 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:28:13.102 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:28:13.102 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:28:13.102 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:28:13.102 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:28:13.102 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:28:13.102 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:28:13.102 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:28:13.102 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:28:13.102 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:28:13.102 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:28:13.102 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:28:13.102 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:28:13.102 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:28:13.102 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:28:13.102 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:28:13.102 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:28:13.102 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:28:13.102 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:28:13.102 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:28:13.102 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:28:13.102 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:28:13.102 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:28:13.102 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:28:13.102 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:28:13.102 | INFO | ------------------------------------------ | 20260507 08:28:13.102 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:28:13.102 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:28:13.102 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:28:13.102 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:28:13.102 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.44e+01, 1.31e+01 / 1.11e+00 | 20260507 08:28:13.102 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 9.71e-03 / 5.14e-03 | 20260507 08:28:13.102 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.46e+00, 4.99e-01 / 2.20e-01 | 20260507 08:28:13.102 | INFO | PerfReport: 3.23%, Histogram -- 9.90e-01 / 1.49e+00, 1.67e-01 / 1.01e-01 | 20260507 08:28:13.102 | INFO | PerfReport: 16.27%, DME -- 3.99e+01 / 4.43e+01, 3.79e+00 / 1.39e+00 | 20260507 08:28:13.102 | INFO | PerfReport: 57.78%, FrameLabeling -- 1.77e+01 / 1.98e+01, 1.29e+00 / 1.44e+00 | 20260507 08:28:13.102 | INFO | PerfReport: 6.89%, PulseAccumulating -- 2.11e+00 / 2.66e+00, 3.88e-01 / 2.03e-01 | 20260507 08:28:13.102 | INFO | PerfReport: 2.91%, Metrics -- 8.91e-01 / 1.17e+00, 2.36e-01 / 1.48e-01 | 20260507 08:28:13.102 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.99e-01 / 3.51e-01, 4.94e-02 / 1.64e-02 | 20260507 08:28:13.102 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:28:13.102 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:28:13.102 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:28:13.102 | INFO | PerfReport: Full Execution -- 9.60e+01 / -nan, 2.38e+01 / 0.00e+00 | 20260507 08:28:13.102 | INFO | PerfReport: 21.81%, Upload -- 2.09e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260507 08:28:13.102 | INFO | PerfReport: 21.45%, Download -- 2.06e+01 / 0.00e+00, 7.39e+00 / 0.00e+00 | 20260507 08:28:13.102 | INFO | PerfReport: 31.93%, Compute -- 3.07e+01 / 0.00e+00, 1.31e+01 / 0.00e+00 | 20260507 08:28:13.102 | INFO | PerfReport: 1.45%, LockUpload -- 1.39e+00 / 0.00e+00, 4.80e+00 / 0.00e+00 | 20260507 08:28:13.102 | INFO | PerfReport: 1.37%, LockDownload -- 1.32e+00 / 0.00e+00, 3.17e+00 / 0.00e+00 | 20260507 08:28:13.102 | INFO | PerfReport: 21.97%, LockCompute -- 2.11e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 08:28:13.102 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.85e-02 / 0.00e+00 | 20260507 08:28:13.198 | INFO | Starting a round of Baz IO | 20260507 08:28:16.289 | INFO | Populating RTMetrics report | 20260507 08:28:16.290 | INFO | RTMetrics report populated | 20260507 08:28:16.290 | INFO | Opening tmp RTMetrics json file | 20260507 08:28:16.307 | INFO | Serializing RTMetrics to json | 20260507 08:28:16.308 | INFO | Serialization done, swapping tmp file now... | 20260507 08:28:16.309 | INFO | RTMetrics done | 20260507 08:28:16.376 | INFO | Finished a round of Baz IO: Wrote 1.612 GiB in 3.177 seconds | 20260507 08:28:16.376 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.49% / 0.00% / 67.51% | 20260507 08:28:16.376 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0903408% / 3.97364e-06% / 6.75519e-05% | 20260507 08:28:16.376 | INFO | Baz IO overall: | 20260507 08:28:16.376 | INFO | BytesWritten : 4247148104715 | 20260507 08:28:16.376 | INFO | NumEvents : 2031503860446 | 20260507 08:28:18.242 | INFO | Analyzing chunk frames = [8527872,8528384) | 20260507 08:28:18.242 | INFO | MemInfo: { | 20260507 08:28:18.242 | INFO | "gpu" : " 17 GB", | 20260507 08:28:18.242 | INFO | "huge" : "", | 20260507 08:28:18.242 | INFO | "huge_shared" : " 27 GB", | 20260507 08:28:18.242 | INFO | "malloc" : " 14 GB", | 20260507 08:28:18.242 | INFO | "pinned" : " 29 GB" | 20260507 08:28:18.242 | INFO | } | 20260507 08:28:18.242 | INFO | Perf report for the last 5.14113 seconds: | 20260507 08:28:18.242 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:28:18.242 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:18.242 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:18.242 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:18.242 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 08:28:18.242 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:18.242 | INFO | RT_METRICS : 40.9%, 1.0, 21.5ms, 0 | 20260507 08:28:18.242 | INFO | PRE_HQ : 58.8%, 1.8, 54.5ms, 0 | 20260507 08:28:18.242 | INFO | BAZWRITER : 61.4%, 3.1, 97.0ms, 0 | 20260507 08:28:18.242 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:28:18.243 | INFO | Uploaded 36GiB to GPU in 2.04164s (17.6329GiB/s) | 20260507 08:28:18.243 | INFO | Downloaded 38GiB to GPU in 2.18018s (17.4298GiB/s) | 20260507 08:28:23.382 | INFO | Analyzing chunk frames = [8528384,8528896) | 20260507 08:28:23.382 | INFO | MemInfo: { | 20260507 08:28:23.382 | INFO | "gpu" : " 17 GB", | 20260507 08:28:23.382 | INFO | "huge" : "", | 20260507 08:28:23.382 | INFO | "huge_shared" : " 27 GB", | 20260507 08:28:23.382 | INFO | "malloc" : " 14 GB", | 20260507 08:28:23.382 | INFO | "pinned" : " 29 GB" | 20260507 08:28:23.382 | INFO | } | 20260507 08:28:23.383 | INFO | Perf report for the last 5.14033 seconds: | 20260507 08:28:23.383 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:28:23.383 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:23.383 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:23.383 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:23.383 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 08:28:23.383 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:23.383 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:23.383 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 08:28:23.383 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:23.383 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:23.383 | INFO | Uploaded 36GiB to GPU in 1.97821s (18.1983GiB/s) | 20260507 08:28:23.383 | INFO | Downloaded 34GiB to GPU in 1.93587s (17.5632GiB/s) | 20260507 08:28:28.522 | INFO | Analyzing chunk frames = [8528896,8529408) | 20260507 08:28:28.522 | INFO | MemInfo: { | 20260507 08:28:28.522 | INFO | "gpu" : " 17 GB", | 20260507 08:28:28.523 | INFO | "huge" : "", | 20260507 08:28:28.523 | INFO | "huge_shared" : " 27 GB", | 20260507 08:28:28.523 | INFO | "malloc" : " 14 GB", | 20260507 08:28:28.523 | INFO | "pinned" : " 29 GB" | 20260507 08:28:28.523 | INFO | } | 20260507 08:28:28.523 | INFO | Perf report for the last 5.14012 seconds: | 20260507 08:28:28.523 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:28:28.523 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:28.523 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:28.523 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:28.523 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 08:28:28.523 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:28.523 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:28.523 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:28:28.523 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:28.523 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:28.523 | INFO | Uploaded 36GiB to GPU in 2.02659s (17.7639GiB/s) | 20260507 08:28:28.523 | INFO | Downloaded 34GiB to GPU in 1.92485s (17.6637GiB/s) | 20260507 08:28:33.660 | INFO | Analyzing chunk frames = [8529408,8529920) | 20260507 08:28:33.660 | INFO | MemInfo: { | 20260507 08:28:33.660 | INFO | "gpu" : " 17 GB", | 20260507 08:28:33.660 | INFO | "huge" : "", | 20260507 08:28:33.660 | INFO | "huge_shared" : " 27 GB", | 20260507 08:28:33.660 | INFO | "malloc" : " 14 GB", | 20260507 08:28:33.660 | INFO | "pinned" : " 29 GB" | 20260507 08:28:33.660 | INFO | } | 20260507 08:28:33.660 | INFO | Perf report for the last 5.1371 seconds: | 20260507 08:28:33.660 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:28:33.660 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:33.660 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:33.660 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:33.660 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 08:28:33.660 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:33.660 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:33.660 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:28:33.660 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:33.660 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:33.660 | INFO | Uploaded 36GiB to GPU in 2.04784s (17.5795GiB/s) | 20260507 08:28:33.660 | INFO | Downloaded 34GiB to GPU in 1.95113s (17.4258GiB/s) | 20260507 08:28:38.805 | INFO | Analyzing chunk frames = [8529920,8530432) | 20260507 08:28:38.805 | INFO | MemInfo: { | 20260507 08:28:38.805 | INFO | "gpu" : " 17 GB", | 20260507 08:28:38.805 | INFO | "huge" : "", | 20260507 08:28:38.805 | INFO | "huge_shared" : " 27 GB", | 20260507 08:28:38.805 | INFO | "malloc" : " 14 GB", | 20260507 08:28:38.805 | INFO | "pinned" : " 29 GB" | 20260507 08:28:38.805 | INFO | } | 20260507 08:28:38.805 | INFO | Perf report for the last 5.14572 seconds: | 20260507 08:28:38.805 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:28:38.805 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:38.806 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:38.806 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:38.806 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 08:28:38.806 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:38.806 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:38.806 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:28:38.806 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:38.806 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:38.806 | INFO | Uploaded 36GiB to GPU in 2.03931s (17.653GiB/s) | 20260507 08:28:38.806 | INFO | Downloaded 34GiB to GPU in 1.92304s (17.6803GiB/s) | 20260507 08:28:43.939 | INFO | Analyzing chunk frames = [8530432,8530944) | 20260507 08:28:43.939 | INFO | MemInfo: { | 20260507 08:28:43.939 | INFO | "gpu" : " 17 GB", | 20260507 08:28:43.939 | INFO | "huge" : "", | 20260507 08:28:43.939 | INFO | "huge_shared" : " 27 GB", | 20260507 08:28:43.939 | INFO | "malloc" : " 14 GB", | 20260507 08:28:43.939 | INFO | "pinned" : " 29 GB" | 20260507 08:28:43.939 | INFO | } | 20260507 08:28:43.939 | INFO | Perf report for the last 5.13353 seconds: | 20260507 08:28:43.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:28:43.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:43.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:43.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:43.939 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:28:43.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:43.939 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:43.939 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 08:28:43.939 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:43.939 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:43.939 | INFO | Uploaded 36GiB to GPU in 2.02515s (17.7765GiB/s) | 20260507 08:28:43.939 | INFO | Downloaded 34GiB to GPU in 1.9334s (17.5856GiB/s) | 20260507 08:28:49.076 | INFO | Analyzing chunk frames = [8530944,8531456) | 20260507 08:28:49.076 | INFO | MemInfo: { | 20260507 08:28:49.076 | INFO | "gpu" : " 17 GB", | 20260507 08:28:49.076 | INFO | "huge" : "", | 20260507 08:28:49.076 | INFO | "huge_shared" : " 27 GB", | 20260507 08:28:49.076 | INFO | "malloc" : " 14 GB", | 20260507 08:28:49.076 | INFO | "pinned" : " 29 GB" | 20260507 08:28:49.076 | INFO | } | 20260507 08:28:49.076 | INFO | Perf report for the last 5.13686 seconds: | 20260507 08:28:49.076 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:28:49.076 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:49.076 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:49.076 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:49.076 | INFO | ANALYSIS : 60.9%, 3.0, 94.7ms, 0 | 20260507 08:28:49.076 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:49.076 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:49.076 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 08:28:49.076 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:49.076 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:49.076 | INFO | Uploaded 36GiB to GPU in 2.01276s (17.8859GiB/s) | 20260507 08:28:49.076 | INFO | Downloaded 34GiB to GPU in 2.00306s (16.9741GiB/s) | 20260507 08:28:54.221 | INFO | Analyzing chunk frames = [8531456,8531968) | 20260507 08:28:54.221 | INFO | MemInfo: { | 20260507 08:28:54.221 | INFO | "gpu" : " 17 GB", | 20260507 08:28:54.221 | INFO | "huge" : "", | 20260507 08:28:54.221 | INFO | "huge_shared" : " 27 GB", | 20260507 08:28:54.221 | INFO | "malloc" : " 14 GB", | 20260507 08:28:54.221 | INFO | "pinned" : " 29 GB" | 20260507 08:28:54.221 | INFO | } | 20260507 08:28:54.221 | INFO | Perf report for the last 5.14509 seconds: | 20260507 08:28:54.221 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:28:54.221 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:54.221 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:54.221 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:54.221 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:28:54.221 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:54.221 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:54.221 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:28:54.221 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:54.221 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:54.221 | INFO | Uploaded 36GiB to GPU in 1.98342s (18.1505GiB/s) | 20260507 08:28:54.221 | INFO | Downloaded 34GiB to GPU in 1.95783s (17.3661GiB/s) | 20260507 08:28:54.318 | INFO | Starting a round of Baz IO | 20260507 08:28:57.407 | INFO | Populating RTMetrics report | 20260507 08:28:57.407 | INFO | RTMetrics report populated | 20260507 08:28:57.407 | INFO | Opening tmp RTMetrics json file | 20260507 08:28:57.416 | INFO | Serializing RTMetrics to json | 20260507 08:28:57.422 | INFO | Serialization done, swapping tmp file now... | 20260507 08:28:57.423 | INFO | RTMetrics done | 20260507 08:28:57.490 | INFO | Finished a round of Baz IO: Wrote 1.611 GiB in 3.171 seconds | 20260507 08:28:57.490 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.49% / 0.00% / 67.51% | 20260507 08:28:57.490 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0898163% / 0% / 7.94729e-05% | 20260507 08:28:57.490 | INFO | Baz IO overall: | 20260507 08:28:57.490 | INFO | BytesWritten : 4248877693473 | 20260507 08:28:57.490 | INFO | NumEvents : 2032309057017 | 20260507 08:28:59.356 | INFO | Analyzing chunk frames = [8531968,8532480) | 20260507 08:28:59.356 | INFO | MemInfo: { | 20260507 08:28:59.356 | INFO | "gpu" : " 17 GB", | 20260507 08:28:59.356 | INFO | "huge" : "", | 20260507 08:28:59.356 | INFO | "huge_shared" : " 27 GB", | 20260507 08:28:59.356 | INFO | "malloc" : " 14 GB", | 20260507 08:28:59.356 | INFO | "pinned" : " 29 GB" | 20260507 08:28:59.356 | INFO | } | 20260507 08:28:59.356 | INFO | Perf report for the last 5.13542 seconds: | 20260507 08:28:59.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:28:59.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:59.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:59.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:59.356 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 08:28:59.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:28:59.356 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260507 08:28:59.356 | INFO | PRE_HQ : 58.9%, 1.8, 54.2ms, 0 | 20260507 08:28:59.357 | INFO | BAZWRITER : 61.3%, 3.1, 97.5ms, 0 | 20260507 08:28:59.357 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:28:59.357 | INFO | Uploaded 36GiB to GPU in 2.05858s (17.4878GiB/s) | 20260507 08:28:59.357 | INFO | Downloaded 38GiB to GPU in 2.16937s (17.5166GiB/s) | 20260507 08:29:04.497 | INFO | Analyzing chunk frames = [8532480,8532992) | 20260507 08:29:04.497 | INFO | MemInfo: { | 20260507 08:29:04.497 | INFO | "gpu" : " 17 GB", | 20260507 08:29:04.497 | INFO | "huge" : "", | 20260507 08:29:04.497 | INFO | "huge_shared" : " 27 GB", | 20260507 08:29:04.497 | INFO | "malloc" : " 14 GB", | 20260507 08:29:04.497 | INFO | "pinned" : " 29 GB" | 20260507 08:29:04.497 | INFO | } | 20260507 08:29:04.497 | INFO | Perf report for the last 5.14101 seconds: | 20260507 08:29:04.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:29:04.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:04.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:04.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:04.497 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:29:04.498 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:04.498 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:04.498 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260507 08:29:04.498 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:04.498 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:04.498 | INFO | Uploaded 36GiB to GPU in 1.981s (18.1727GiB/s) | 20260507 08:29:04.498 | INFO | Downloaded 34GiB to GPU in 1.9311s (17.6065GiB/s) | 20260507 08:29:09.643 | INFO | Analyzing chunk frames = [8532992,8533504) | 20260507 08:29:09.643 | INFO | MemInfo: { | 20260507 08:29:09.643 | INFO | "gpu" : " 17 GB", | 20260507 08:29:09.643 | INFO | "huge" : "", | 20260507 08:29:09.643 | INFO | "huge_shared" : " 27 GB", | 20260507 08:29:09.643 | INFO | "malloc" : " 14 GB", | 20260507 08:29:09.643 | INFO | "pinned" : " 29 GB" | 20260507 08:29:09.643 | INFO | } | 20260507 08:29:09.643 | INFO | Perf report for the last 5.14577 seconds: | 20260507 08:29:09.643 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:29:09.643 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:09.643 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:09.643 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:09.643 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 08:29:09.643 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:09.643 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:09.643 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:29:09.643 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:09.643 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:09.643 | INFO | Uploaded 36GiB to GPU in 2.00803s (17.928GiB/s) | 20260507 08:29:09.644 | INFO | Downloaded 34GiB to GPU in 1.92357s (17.6755GiB/s) | 20260507 08:29:14.771 | INFO | Analyzing chunk frames = [8533504,8534016) | 20260507 08:29:14.771 | INFO | MemInfo: { | 20260507 08:29:14.771 | INFO | "gpu" : " 17 GB", | 20260507 08:29:14.771 | INFO | "huge" : "", | 20260507 08:29:14.771 | INFO | "huge_shared" : " 27 GB", | 20260507 08:29:14.771 | INFO | "malloc" : " 14 GB", | 20260507 08:29:14.771 | INFO | "pinned" : " 29 GB" | 20260507 08:29:14.771 | INFO | } | 20260507 08:29:14.771 | INFO | Perf report for the last 5.12747 seconds: | 20260507 08:29:14.771 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:29:14.771 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:14.771 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:14.771 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:14.771 | INFO | ANALYSIS : 60.4%, 2.9, 93.1ms, 0 | 20260507 08:29:14.771 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:14.771 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:14.771 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 08:29:14.771 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:14.771 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:14.771 | INFO | Uploaded 36GiB to GPU in 2.03198s (17.7167GiB/s) | 20260507 08:29:14.771 | INFO | Downloaded 34GiB to GPU in 1.91865s (17.7208GiB/s) | 20260507 08:29:19.907 | INFO | Analyzing chunk frames = [8534016,8534528) | 20260507 08:29:19.907 | INFO | MemInfo: { | 20260507 08:29:19.907 | INFO | "gpu" : " 17 GB", | 20260507 08:29:19.907 | INFO | "huge" : "", | 20260507 08:29:19.907 | INFO | "huge_shared" : " 27 GB", | 20260507 08:29:19.907 | INFO | "malloc" : " 14 GB", | 20260507 08:29:19.907 | INFO | "pinned" : " 29 GB" | 20260507 08:29:19.907 | INFO | } | 20260507 08:29:19.907 | INFO | Perf report for the last 5.13642 seconds: | 20260507 08:29:19.907 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:29:19.907 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:19.907 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:19.907 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:19.907 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 08:29:19.907 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:19.907 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:19.907 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:29:19.907 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:19.907 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:19.907 | INFO | Uploaded 36GiB to GPU in 2.02342s (17.7917GiB/s) | 20260507 08:29:19.907 | INFO | Downloaded 34GiB to GPU in 1.91856s (17.7216GiB/s) | 20260507 08:29:25.052 | INFO | Analyzing chunk frames = [8534528,8535040) | 20260507 08:29:25.052 | INFO | MemInfo: { | 20260507 08:29:25.052 | INFO | "gpu" : " 17 GB", | 20260507 08:29:25.052 | INFO | "huge" : "", | 20260507 08:29:25.052 | INFO | "huge_shared" : " 27 GB", | 20260507 08:29:25.052 | INFO | "malloc" : " 14 GB", | 20260507 08:29:25.052 | INFO | "pinned" : " 29 GB" | 20260507 08:29:25.052 | INFO | } | 20260507 08:29:25.052 | INFO | Perf report for the last 5.14513 seconds: | 20260507 08:29:25.052 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:29:25.052 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:25.052 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:25.052 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:25.052 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 08:29:25.052 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:25.052 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:25.052 | INFO | PRE_HQ : 27.2%, 1.0, 14.1ms, 0 | 20260507 08:29:25.052 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:25.052 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:25.052 | INFO | Uploaded 36GiB to GPU in 2.01569s (17.8599GiB/s) | 20260507 08:29:25.053 | INFO | Downloaded 34GiB to GPU in 1.90311s (17.8655GiB/s) | 20260507 08:29:30.196 | INFO | Analyzing chunk frames = [8535040,8535552) | 20260507 08:29:30.196 | INFO | MemInfo: { | 20260507 08:29:30.196 | INFO | "gpu" : " 17 GB", | 20260507 08:29:30.196 | INFO | "huge" : "", | 20260507 08:29:30.196 | INFO | "huge_shared" : " 27 GB", | 20260507 08:29:30.196 | INFO | "malloc" : " 14 GB", | 20260507 08:29:30.196 | INFO | "pinned" : " 29 GB" | 20260507 08:29:30.196 | INFO | } | 20260507 08:29:30.197 | INFO | Perf report for the last 5.14419 seconds: | 20260507 08:29:30.197 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:29:30.197 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:30.197 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:30.197 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:30.197 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 08:29:30.197 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:30.197 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:30.197 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:29:30.197 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:30.197 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:30.197 | INFO | Uploaded 36GiB to GPU in 2.03856s (17.6595GiB/s) | 20260507 08:29:30.197 | INFO | Downloaded 34GiB to GPU in 1.98869s (17.0966GiB/s) | 20260507 08:29:35.332 | INFO | Analyzing chunk frames = [8535552,8536064) | 20260507 08:29:35.332 | INFO | MemInfo: { | 20260507 08:29:35.332 | INFO | "gpu" : " 17 GB", | 20260507 08:29:35.332 | INFO | "huge" : "", | 20260507 08:29:35.332 | INFO | "huge_shared" : " 27 GB", | 20260507 08:29:35.332 | INFO | "malloc" : " 14 GB", | 20260507 08:29:35.332 | INFO | "pinned" : " 29 GB" | 20260507 08:29:35.332 | INFO | } | 20260507 08:29:35.332 | INFO | Perf report for the last 5.13517 seconds: | 20260507 08:29:35.332 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:29:35.332 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:35.332 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:35.332 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:35.332 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 08:29:35.332 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:35.332 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:35.332 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:29:35.332 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:35.332 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:35.332 | INFO | Uploaded 36GiB to GPU in 1.99687s (18.0282GiB/s) | 20260507 08:29:35.332 | INFO | Downloaded 34GiB to GPU in 1.96187s (17.3304GiB/s) | 20260507 08:29:35.332 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:29:35.332 | INFO | Total mem usage: 18 GiB | 20260507 08:29:35.332 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:29:35.332 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:29:35.332 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:29:35.332 | INFO | High Water Breakdown: | 20260507 08:29:35.332 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:29:35.332 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:29:35.332 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:29:35.332 | INFO | ------------------------------------------ | 20260507 08:29:35.332 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:29:35.332 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:29:35.332 | INFO | Total mem usage: 26.5 GiB | 20260507 08:29:35.332 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:29:35.332 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:29:35.332 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:29:35.332 | INFO | High Water Breakdown: | 20260507 08:29:35.332 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:29:35.332 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:29:35.332 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:29:35.332 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:29:35.332 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:29:35.332 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:29:35.332 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:29:35.332 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:29:35.332 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:29:35.332 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:29:35.332 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:29:35.332 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:29:35.332 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:29:35.332 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:29:35.333 | INFO | ------------------------------------------ | 20260507 08:29:35.333 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:29:35.333 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:29:35.333 | INFO | Total mem usage: 25.6 GiB | 20260507 08:29:35.333 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:29:35.333 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:29:35.333 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:29:35.333 | INFO | High Water Breakdown: | 20260507 08:29:35.333 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:29:35.333 | INFO | ------------------------------------------ | 20260507 08:29:35.333 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:29:35.333 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:29:35.333 | INFO | Total mem usage: 15.7 GiB | 20260507 08:29:35.333 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:29:35.333 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:29:35.333 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:29:35.333 | INFO | High Water Breakdown: | 20260507 08:29:35.333 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:29:35.333 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:29:35.333 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:29:35.333 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:29:35.333 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:29:35.333 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:29:35.333 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:29:35.333 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:29:35.333 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:29:35.333 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:29:35.333 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:29:35.333 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:29:35.333 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:29:35.333 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:29:35.333 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:29:35.333 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:29:35.333 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:29:35.333 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:29:35.333 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:29:35.333 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:29:35.333 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:29:35.333 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:29:35.333 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:29:35.333 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:29:35.333 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:29:35.333 | INFO | ------------------------------------------ | 20260507 08:29:35.333 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:29:35.333 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:29:35.333 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:29:35.333 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:29:35.333 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 1.19e+00 | 20260507 08:29:35.333 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.04e-01, 9.71e-03 / 9.49e-03 | 20260507 08:29:35.333 | INFO | PerfReport: 11.65%, Baseline -- 3.58e+00 / 4.42e+00, 5.07e-01 / 2.35e-01 | 20260507 08:29:35.333 | INFO | PerfReport: 3.24%, Histogram -- 9.96e-01 / 1.49e+00, 1.71e-01 / 9.95e-02 | 20260507 08:29:35.333 | INFO | PerfReport: 16.26%, DME -- 4.00e+01 / 4.46e+01, 3.77e+00 / 1.33e+00 | 20260507 08:29:35.333 | INFO | PerfReport: 57.87%, FrameLabeling -- 1.78e+01 / 1.99e+01, 1.41e+00 / 1.42e+00 | 20260507 08:29:35.333 | INFO | PerfReport: 6.82%, PulseAccumulating -- 2.10e+00 / 2.64e+00, 3.75e-01 / 2.33e-01 | 20260507 08:29:35.333 | INFO | PerfReport: 2.89%, Metrics -- 8.89e-01 / 1.21e+00, 2.40e-01 / 1.48e-01 | 20260507 08:29:35.333 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.99e-01 / 3.54e-01, 5.02e-02 / 2.13e-02 | 20260507 08:29:35.333 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:29:35.333 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:29:35.333 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:29:35.333 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260507 08:29:35.333 | INFO | PerfReport: 21.86%, Upload -- 2.10e+01 / 0.00e+00, 7.15e+00 / 0.00e+00 | 20260507 08:29:35.333 | INFO | PerfReport: 21.33%, Download -- 2.05e+01 / 0.00e+00, 7.31e+00 / 0.00e+00 | 20260507 08:29:35.333 | INFO | PerfReport: 31.96%, Compute -- 3.08e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 08:29:35.333 | INFO | PerfReport: 1.59%, LockUpload -- 1.53e+00 / 0.00e+00, 5.00e+00 / 0.00e+00 | 20260507 08:29:35.333 | INFO | PerfReport: 1.25%, LockDownload -- 1.20e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260507 08:29:35.333 | INFO | PerfReport: 21.99%, LockCompute -- 2.12e+01 / 0.00e+00, 1.88e+01 / 0.00e+00 | 20260507 08:29:35.333 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260507 08:29:35.428 | INFO | Starting a round of Baz IO | 20260507 08:29:38.519 | INFO | Populating RTMetrics report | 20260507 08:29:38.519 | INFO | RTMetrics report populated | 20260507 08:29:38.519 | INFO | Opening tmp RTMetrics json file | 20260507 08:29:38.535 | INFO | Serializing RTMetrics to json | 20260507 08:29:38.537 | INFO | Serialization done, swapping tmp file now... | 20260507 08:29:38.537 | INFO | RTMetrics done | 20260507 08:29:38.606 | INFO | Finished a round of Baz IO: Wrote 1.611 GiB in 3.177 seconds | 20260507 08:29:38.606 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.49% / 0.00% / 67.51% | 20260507 08:29:38.606 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0901937% / 0% / 5.5631e-05% | 20260507 08:29:38.606 | INFO | Baz IO overall: | 20260507 08:29:38.606 | INFO | BytesWritten : 4250607990857 | 20260507 08:29:38.606 | INFO | NumEvents : 2033115105034 | 20260507 08:29:40.469 | INFO | Analyzing chunk frames = [8536064,8536576) | 20260507 08:29:40.469 | INFO | MemInfo: { | 20260507 08:29:40.469 | INFO | "gpu" : " 17 GB", | 20260507 08:29:40.469 | INFO | "huge" : "", | 20260507 08:29:40.469 | INFO | "huge_shared" : " 27 GB", | 20260507 08:29:40.469 | INFO | "malloc" : " 14 GB", | 20260507 08:29:40.469 | INFO | "pinned" : " 29 GB" | 20260507 08:29:40.469 | INFO | } | 20260507 08:29:40.469 | INFO | Perf report for the last 5.13769 seconds: | 20260507 08:29:40.469 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:29:40.469 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:40.469 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:40.469 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:40.469 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 08:29:40.469 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:40.469 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 08:29:40.469 | INFO | PRE_HQ : 59.2%, 1.8, 54.7ms, 0 | 20260507 08:29:40.469 | INFO | BAZWRITER : 61.4%, 3.1, 98.7ms, 0 | 20260507 08:29:40.469 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:29:40.470 | INFO | Uploaded 36GiB to GPU in 2.07678s (17.3346GiB/s) | 20260507 08:29:40.470 | INFO | Downloaded 38GiB to GPU in 2.16799s (17.5277GiB/s) | 20260507 08:29:45.610 | INFO | Analyzing chunk frames = [8536576,8537088) | 20260507 08:29:45.610 | INFO | MemInfo: { | 20260507 08:29:45.610 | INFO | "gpu" : " 17 GB", | 20260507 08:29:45.610 | INFO | "huge" : "", | 20260507 08:29:45.610 | INFO | "huge_shared" : " 27 GB", | 20260507 08:29:45.610 | INFO | "malloc" : " 14 GB", | 20260507 08:29:45.610 | INFO | "pinned" : " 29 GB" | 20260507 08:29:45.610 | INFO | } | 20260507 08:29:45.610 | INFO | Perf report for the last 5.14086 seconds: | 20260507 08:29:45.610 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:29:45.610 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:45.610 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:45.610 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:45.610 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 08:29:45.610 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:45.610 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:45.610 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:29:45.610 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:45.610 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:45.610 | INFO | Uploaded 36GiB to GPU in 1.99004s (18.0901GiB/s) | 20260507 08:29:45.610 | INFO | Downloaded 34GiB to GPU in 1.929s (17.6257GiB/s) | 20260507 08:29:50.750 | INFO | Analyzing chunk frames = [8537088,8537600) | 20260507 08:29:50.750 | INFO | MemInfo: { | 20260507 08:29:50.750 | INFO | "gpu" : " 17 GB", | 20260507 08:29:50.750 | INFO | "huge" : "", | 20260507 08:29:50.750 | INFO | "huge_shared" : " 27 GB", | 20260507 08:29:50.750 | INFO | "malloc" : " 14 GB", | 20260507 08:29:50.750 | INFO | "pinned" : " 29 GB" | 20260507 08:29:50.750 | INFO | } | 20260507 08:29:50.750 | INFO | Perf report for the last 5.13982 seconds: | 20260507 08:29:50.750 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:29:50.750 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:50.750 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:50.750 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:50.750 | INFO | ANALYSIS : 60.7%, 3.0, 94.0ms, 0 | 20260507 08:29:50.750 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:50.750 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:50.750 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:29:50.750 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:50.750 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:50.750 | INFO | Uploaded 36GiB to GPU in 2.06298s (17.4505GiB/s) | 20260507 08:29:50.750 | INFO | Downloaded 34GiB to GPU in 1.94781s (17.4555GiB/s) | 20260507 08:29:55.891 | INFO | Analyzing chunk frames = [8537600,8538112) | 20260507 08:29:55.891 | INFO | MemInfo: { | 20260507 08:29:55.891 | INFO | "gpu" : " 17 GB", | 20260507 08:29:55.891 | INFO | "huge" : "", | 20260507 08:29:55.891 | INFO | "huge_shared" : " 27 GB", | 20260507 08:29:55.891 | INFO | "malloc" : " 14 GB", | 20260507 08:29:55.891 | INFO | "pinned" : " 29 GB" | 20260507 08:29:55.891 | INFO | } | 20260507 08:29:55.891 | INFO | Perf report for the last 5.14104 seconds: | 20260507 08:29:55.891 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:29:55.891 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:55.891 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:55.891 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:55.891 | INFO | ANALYSIS : 60.4%, 2.9, 93.3ms, 0 | 20260507 08:29:55.891 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:55.891 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:55.891 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:29:55.891 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:55.891 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:29:55.891 | INFO | Uploaded 36GiB to GPU in 2.04608s (17.5946GiB/s) | 20260507 08:29:55.891 | INFO | Downloaded 34GiB to GPU in 1.95086s (17.4282GiB/s) | 20260507 08:30:01.033 | INFO | Analyzing chunk frames = [8538112,8538624) | 20260507 08:30:01.033 | INFO | MemInfo: { | 20260507 08:30:01.033 | INFO | "gpu" : " 17 GB", | 20260507 08:30:01.033 | INFO | "huge" : "", | 20260507 08:30:01.033 | INFO | "huge_shared" : " 27 GB", | 20260507 08:30:01.033 | INFO | "malloc" : " 14 GB", | 20260507 08:30:01.033 | INFO | "pinned" : " 29 GB" | 20260507 08:30:01.033 | INFO | } | 20260507 08:30:01.033 | INFO | Perf report for the last 5.14171 seconds: | 20260507 08:30:01.033 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:30:01.033 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:01.033 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:01.033 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:01.033 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 08:30:01.033 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:01.033 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:01.033 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:30:01.033 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:01.033 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:01.033 | INFO | Uploaded 36GiB to GPU in 2.04911s (17.5686GiB/s) | 20260507 08:30:01.033 | INFO | Downloaded 34GiB to GPU in 1.90145s (17.8811GiB/s) | 20260507 08:30:06.171 | INFO | Analyzing chunk frames = [8538624,8539136) | 20260507 08:30:06.171 | INFO | MemInfo: { | 20260507 08:30:06.171 | INFO | "gpu" : " 17 GB", | 20260507 08:30:06.171 | INFO | "huge" : "", | 20260507 08:30:06.171 | INFO | "huge_shared" : " 27 GB", | 20260507 08:30:06.171 | INFO | "malloc" : " 14 GB", | 20260507 08:30:06.171 | INFO | "pinned" : " 29 GB" | 20260507 08:30:06.171 | INFO | } | 20260507 08:30:06.171 | INFO | Perf report for the last 5.13853 seconds: | 20260507 08:30:06.171 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:30:06.171 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:06.171 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:06.171 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:06.171 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:30:06.171 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:06.171 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:06.171 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 08:30:06.171 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:06.171 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:06.172 | INFO | Uploaded 36GiB to GPU in 2.02722s (17.7583GiB/s) | 20260507 08:30:06.172 | INFO | Downloaded 34GiB to GPU in 1.92998s (17.6167GiB/s) | 20260507 08:30:11.307 | INFO | Analyzing chunk frames = [8539136,8539648) | 20260507 08:30:11.307 | INFO | MemInfo: { | 20260507 08:30:11.307 | INFO | "gpu" : " 17 GB", | 20260507 08:30:11.307 | INFO | "huge" : "", | 20260507 08:30:11.307 | INFO | "huge_shared" : " 27 GB", | 20260507 08:30:11.307 | INFO | "malloc" : " 14 GB", | 20260507 08:30:11.307 | INFO | "pinned" : " 29 GB" | 20260507 08:30:11.307 | INFO | } | 20260507 08:30:11.307 | INFO | Perf report for the last 5.13595 seconds: | 20260507 08:30:11.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:30:11.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:11.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:11.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:11.307 | INFO | ANALYSIS : 60.7%, 3.0, 94.3ms, 0 | 20260507 08:30:11.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:11.307 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:11.307 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 08:30:11.308 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:11.308 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:11.308 | INFO | Uploaded 36GiB to GPU in 1.92878s (18.6646GiB/s) | 20260507 08:30:11.308 | INFO | Downloaded 34GiB to GPU in 1.97602s (17.2063GiB/s) | 20260507 08:30:16.449 | INFO | Analyzing chunk frames = [8539648,8540160) | 20260507 08:30:16.449 | INFO | MemInfo: { | 20260507 08:30:16.449 | INFO | "gpu" : " 17 GB", | 20260507 08:30:16.449 | INFO | "huge" : "", | 20260507 08:30:16.449 | INFO | "huge_shared" : " 27 GB", | 20260507 08:30:16.449 | INFO | "malloc" : " 14 GB", | 20260507 08:30:16.449 | INFO | "pinned" : " 29 GB" | 20260507 08:30:16.449 | INFO | } | 20260507 08:30:16.449 | INFO | Perf report for the last 5.14173 seconds: | 20260507 08:30:16.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:30:16.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:16.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:16.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:16.449 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:30:16.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:16.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:16.449 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:30:16.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:16.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:16.449 | INFO | Uploaded 36GiB to GPU in 1.99192s (18.073GiB/s) | 20260507 08:30:16.449 | INFO | Downloaded 34GiB to GPU in 1.96423s (17.3096GiB/s) | 20260507 08:30:16.547 | INFO | Starting a round of Baz IO | 20260507 08:30:19.635 | INFO | Populating RTMetrics report | 20260507 08:30:19.635 | INFO | RTMetrics report populated | 20260507 08:30:19.635 | INFO | Opening tmp RTMetrics json file | 20260507 08:30:19.644 | INFO | Serializing RTMetrics to json | 20260507 08:30:19.650 | INFO | Serialization done, swapping tmp file now... | 20260507 08:30:19.651 | INFO | RTMetrics done | 20260507 08:30:19.719 | INFO | Finished a round of Baz IO: Wrote 1.611 GiB in 3.172 seconds | 20260507 08:30:19.719 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.49% / 0.00% / 67.51% | 20260507 08:30:19.719 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0905196% / 0% / 6.35783e-05% | 20260507 08:30:19.719 | INFO | Baz IO overall: | 20260507 08:30:19.719 | INFO | BytesWritten : 4252338164230 | 20260507 08:30:19.719 | INFO | NumEvents : 2033921208689 | 20260507 08:30:21.587 | INFO | Analyzing chunk frames = [8540160,8540672) | 20260507 08:30:21.587 | INFO | MemInfo: { | 20260507 08:30:21.587 | INFO | "gpu" : " 17 GB", | 20260507 08:30:21.587 | INFO | "huge" : "", | 20260507 08:30:21.587 | INFO | "huge_shared" : " 27 GB", | 20260507 08:30:21.587 | INFO | "malloc" : " 14 GB", | 20260507 08:30:21.587 | INFO | "pinned" : " 29 GB" | 20260507 08:30:21.587 | INFO | } | 20260507 08:30:21.588 | INFO | Perf report for the last 5.13837 seconds: | 20260507 08:30:21.588 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:30:21.588 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:21.588 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:21.588 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:21.588 | INFO | ANALYSIS : 61.5%, 3.0, 95.6ms, 0 | 20260507 08:30:21.588 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:21.588 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260507 08:30:21.588 | INFO | PRE_HQ : 58.6%, 1.8, 54.1ms, 0 | 20260507 08:30:21.588 | INFO | BAZWRITER : 61.3%, 3.0, 95.7ms, 0 | 20260507 08:30:21.588 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 08:30:21.588 | INFO | Uploaded 36GiB to GPU in 2.07371s (17.3602GiB/s) | 20260507 08:30:21.588 | INFO | Downloaded 38GiB to GPU in 2.17393s (17.4798GiB/s) | 20260507 08:30:26.722 | INFO | Analyzing chunk frames = [8540672,8541184) | 20260507 08:30:26.722 | INFO | MemInfo: { | 20260507 08:30:26.722 | INFO | "gpu" : " 17 GB", | 20260507 08:30:26.722 | INFO | "huge" : "", | 20260507 08:30:26.722 | INFO | "huge_shared" : " 27 GB", | 20260507 08:30:26.722 | INFO | "malloc" : " 14 GB", | 20260507 08:30:26.722 | INFO | "pinned" : " 29 GB" | 20260507 08:30:26.722 | INFO | } | 20260507 08:30:26.722 | INFO | Perf report for the last 5.13439 seconds: | 20260507 08:30:26.722 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:30:26.722 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:26.722 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:26.722 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:26.722 | INFO | ANALYSIS : 60.8%, 3.0, 94.5ms, 0 | 20260507 08:30:26.722 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:26.722 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:26.722 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260507 08:30:26.722 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:26.722 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:26.722 | INFO | Uploaded 36GiB to GPU in 2.01965s (17.8249GiB/s) | 20260507 08:30:26.722 | INFO | Downloaded 34GiB to GPU in 1.94133s (17.5138GiB/s) | 20260507 08:30:31.864 | INFO | Analyzing chunk frames = [8541184,8541696) | 20260507 08:30:31.864 | INFO | MemInfo: { | 20260507 08:30:31.864 | INFO | "gpu" : " 17 GB", | 20260507 08:30:31.864 | INFO | "huge" : "", | 20260507 08:30:31.864 | INFO | "huge_shared" : " 27 GB", | 20260507 08:30:31.864 | INFO | "malloc" : " 14 GB", | 20260507 08:30:31.864 | INFO | "pinned" : " 29 GB" | 20260507 08:30:31.864 | INFO | } | 20260507 08:30:31.864 | INFO | Perf report for the last 5.14197 seconds: | 20260507 08:30:31.864 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:30:31.864 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:31.864 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:31.864 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:31.864 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 08:30:31.864 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:31.864 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:31.864 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:30:31.864 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:31.864 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:31.864 | INFO | Uploaded 36GiB to GPU in 2.05038s (17.5578GiB/s) | 20260507 08:30:31.864 | INFO | Downloaded 34GiB to GPU in 1.95036s (17.4327GiB/s) | 20260507 08:30:37.010 | INFO | Analyzing chunk frames = [8541696,8542208) | 20260507 08:30:37.010 | INFO | MemInfo: { | 20260507 08:30:37.010 | INFO | "gpu" : " 17 GB", | 20260507 08:30:37.010 | INFO | "huge" : "", | 20260507 08:30:37.010 | INFO | "huge_shared" : " 27 GB", | 20260507 08:30:37.010 | INFO | "malloc" : " 14 GB", | 20260507 08:30:37.010 | INFO | "pinned" : " 29 GB" | 20260507 08:30:37.010 | INFO | } | 20260507 08:30:37.010 | INFO | Perf report for the last 5.1458 seconds: | 20260507 08:30:37.010 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:30:37.010 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:37.010 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:37.010 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:37.010 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 08:30:37.010 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:37.010 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:37.010 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:30:37.010 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:37.010 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:37.010 | INFO | Uploaded 36GiB to GPU in 2.01445s (17.8709GiB/s) | 20260507 08:30:37.010 | INFO | Downloaded 34GiB to GPU in 1.9358s (17.5638GiB/s) | 20260507 08:30:42.150 | INFO | Analyzing chunk frames = [8542208,8542720) | 20260507 08:30:42.150 | INFO | MemInfo: { | 20260507 08:30:42.150 | INFO | "gpu" : " 17 GB", | 20260507 08:30:42.150 | INFO | "huge" : "", | 20260507 08:30:42.150 | INFO | "huge_shared" : " 27 GB", | 20260507 08:30:42.150 | INFO | "malloc" : " 14 GB", | 20260507 08:30:42.150 | INFO | "pinned" : " 29 GB" | 20260507 08:30:42.150 | INFO | } | 20260507 08:30:42.150 | INFO | Perf report for the last 5.14005 seconds: | 20260507 08:30:42.150 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:30:42.150 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:42.150 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:42.150 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:42.150 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 08:30:42.150 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:42.150 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:42.150 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:30:42.150 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:42.150 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:42.150 | INFO | Uploaded 36GiB to GPU in 2.05037s (17.5578GiB/s) | 20260507 08:30:42.150 | INFO | Downloaded 34GiB to GPU in 1.91187s (17.7836GiB/s) | 20260507 08:30:47.281 | INFO | Analyzing chunk frames = [8542720,8543232) | 20260507 08:30:47.281 | INFO | MemInfo: { | 20260507 08:30:47.281 | INFO | "gpu" : " 17 GB", | 20260507 08:30:47.281 | INFO | "huge" : "", | 20260507 08:30:47.281 | INFO | "huge_shared" : " 27 GB", | 20260507 08:30:47.281 | INFO | "malloc" : " 14 GB", | 20260507 08:30:47.281 | INFO | "pinned" : " 29 GB" | 20260507 08:30:47.281 | INFO | } | 20260507 08:30:47.281 | INFO | Perf report for the last 5.13132 seconds: | 20260507 08:30:47.281 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:30:47.281 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:47.281 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:47.281 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:47.281 | INFO | ANALYSIS : 60.4%, 3.0, 94.0ms, 0 | 20260507 08:30:47.281 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:47.281 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:47.281 | INFO | PRE_HQ : 28.3%, 1.0, 14.7ms, 0 | 20260507 08:30:47.281 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:47.281 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:47.281 | INFO | Uploaded 36GiB to GPU in 2.02694s (17.7607GiB/s) | 20260507 08:30:47.281 | INFO | Downloaded 34GiB to GPU in 1.91769s (17.7296GiB/s) | 20260507 08:30:52.420 | INFO | Analyzing chunk frames = [8543232,8543744) | 20260507 08:30:52.420 | INFO | MemInfo: { | 20260507 08:30:52.420 | INFO | "gpu" : " 17 GB", | 20260507 08:30:52.420 | INFO | "huge" : "", | 20260507 08:30:52.420 | INFO | "huge_shared" : " 27 GB", | 20260507 08:30:52.420 | INFO | "malloc" : " 14 GB", | 20260507 08:30:52.420 | INFO | "pinned" : " 29 GB" | 20260507 08:30:52.420 | INFO | } | 20260507 08:30:52.420 | INFO | Perf report for the last 5.13872 seconds: | 20260507 08:30:52.420 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:30:52.420 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:52.420 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:52.420 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:52.420 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:30:52.420 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:52.420 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:52.420 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:30:52.420 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:52.420 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:52.420 | INFO | Uploaded 36GiB to GPU in 1.91906s (18.7592GiB/s) | 20260507 08:30:52.420 | INFO | Downloaded 34GiB to GPU in 1.97434s (17.2209GiB/s) | 20260507 08:30:57.561 | INFO | Analyzing chunk frames = [8543744,8544256) | 20260507 08:30:57.561 | INFO | MemInfo: { | 20260507 08:30:57.561 | INFO | "gpu" : " 17 GB", | 20260507 08:30:57.561 | INFO | "huge" : "", | 20260507 08:30:57.561 | INFO | "huge_shared" : " 27 GB", | 20260507 08:30:57.561 | INFO | "malloc" : " 14 GB", | 20260507 08:30:57.561 | INFO | "pinned" : " 29 GB" | 20260507 08:30:57.561 | INFO | } | 20260507 08:30:57.561 | INFO | Perf report for the last 5.14096 seconds: | 20260507 08:30:57.561 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:30:57.561 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:57.561 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:57.561 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:57.561 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:30:57.561 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:57.561 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:57.561 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:30:57.561 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:57.561 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:30:57.561 | INFO | Uploaded 36GiB to GPU in 1.96878s (18.2854GiB/s) | 20260507 08:30:57.561 | INFO | Downloaded 34GiB to GPU in 1.96584s (17.2954GiB/s) | 20260507 08:30:57.561 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:30:57.561 | INFO | Total mem usage: 18 GiB | 20260507 08:30:57.561 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:30:57.561 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:30:57.561 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:30:57.561 | INFO | High Water Breakdown: | 20260507 08:30:57.561 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:30:57.561 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:30:57.561 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:30:57.561 | INFO | ------------------------------------------ | 20260507 08:30:57.561 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:30:57.561 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:30:57.561 | INFO | Total mem usage: 26.5 GiB | 20260507 08:30:57.561 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:30:57.561 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:30:57.561 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:30:57.561 | INFO | High Water Breakdown: | 20260507 08:30:57.561 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:30:57.561 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:30:57.562 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:30:57.562 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:30:57.562 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:30:57.562 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:30:57.562 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:30:57.562 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:30:57.562 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:30:57.562 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:30:57.562 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:30:57.562 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:30:57.562 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:30:57.562 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:30:57.562 | INFO | ------------------------------------------ | 20260507 08:30:57.562 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:30:57.562 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:30:57.562 | INFO | Total mem usage: 25.6 GiB | 20260507 08:30:57.562 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:30:57.562 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:30:57.562 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:30:57.562 | INFO | High Water Breakdown: | 20260507 08:30:57.562 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:30:57.562 | INFO | ------------------------------------------ | 20260507 08:30:57.562 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:30:57.562 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:30:57.562 | INFO | Total mem usage: 15.7 GiB | 20260507 08:30:57.562 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:30:57.562 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:30:57.562 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:30:57.562 | INFO | High Water Breakdown: | 20260507 08:30:57.562 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:30:57.562 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:30:57.562 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:30:57.562 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:30:57.562 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:30:57.562 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:30:57.562 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:30:57.562 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:30:57.562 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:30:57.562 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:30:57.562 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:30:57.562 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:30:57.562 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:30:57.562 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:30:57.562 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:30:57.562 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:30:57.562 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:30:57.562 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:30:57.562 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:30:57.562 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:30:57.562 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:30:57.562 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:30:57.562 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:30:57.562 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:30:57.562 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:30:57.562 | INFO | ------------------------------------------ | 20260507 08:30:57.562 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:30:57.562 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:30:57.562 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:30:57.562 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:30:57.562 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.50e+01, 1.34e+01 / 7.50e-01 | 20260507 08:30:57.562 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 9.32e-03 / 6.38e-03 | 20260507 08:30:57.562 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.41e+00, 5.06e-01 / 2.50e-01 | 20260507 08:30:57.562 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.49e+00, 1.69e-01 / 1.01e-01 | 20260507 08:30:57.562 | INFO | PerfReport: 16.33%, DME -- 4.01e+01 / 4.53e+01, 4.07e+00 / 1.31e+00 | 20260507 08:30:57.562 | INFO | PerfReport: 57.75%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.34e+00 / 1.31e+00 | 20260507 08:30:57.562 | INFO | PerfReport: 6.86%, PulseAccumulating -- 2.11e+00 / 2.67e+00, 3.82e-01 / 2.21e-01 | 20260507 08:30:57.562 | INFO | PerfReport: 2.91%, Metrics -- 8.92e-01 / 1.19e+00, 2.41e-01 / 1.12e-01 | 20260507 08:30:57.562 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.99e-01 / 3.58e-01, 5.00e-02 / 2.12e-02 | 20260507 08:30:57.562 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:30:57.562 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:30:57.562 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:30:57.562 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.41e+01 / 0.00e+00 | 20260507 08:30:57.562 | INFO | PerfReport: 21.88%, Upload -- 2.10e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260507 08:30:57.562 | INFO | PerfReport: 21.37%, Download -- 2.05e+01 / 0.00e+00, 7.27e+00 / 0.00e+00 | 20260507 08:30:57.562 | INFO | PerfReport: 31.97%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 08:30:57.562 | INFO | PerfReport: 1.62%, LockUpload -- 1.56e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 08:30:57.562 | INFO | PerfReport: 1.16%, LockDownload -- 1.11e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260507 08:30:57.562 | INFO | PerfReport: 21.98%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 08:30:57.562 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.49e-02 / 0.00e+00 | 20260507 08:30:57.658 | INFO | Starting a round of Baz IO | 20260507 08:31:00.749 | INFO | Populating RTMetrics report | 20260507 08:31:00.749 | INFO | RTMetrics report populated | 20260507 08:31:00.749 | INFO | Opening tmp RTMetrics json file | 20260507 08:31:00.762 | INFO | Serializing RTMetrics to json | 20260507 08:31:00.763 | INFO | Serialization done, swapping tmp file now... | 20260507 08:31:00.764 | INFO | RTMetrics done | 20260507 08:31:00.834 | INFO | Finished a round of Baz IO: Wrote 1.61 GiB in 3.176 seconds | 20260507 08:31:00.834 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.49% / 0.00% / 67.51% | 20260507 08:31:00.834 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0896335% / 0% / 6.35783e-05% | 20260507 08:31:00.834 | INFO | Baz IO overall: | 20260507 08:31:00.834 | INFO | BytesWritten : 4254066873749 | 20260507 08:31:00.834 | INFO | NumEvents : 2034726036135 | 20260507 08:31:02.700 | INFO | Analyzing chunk frames = [8544256,8544768) | 20260507 08:31:02.700 | INFO | MemInfo: { | 20260507 08:31:02.700 | INFO | "gpu" : " 17 GB", | 20260507 08:31:02.700 | INFO | "huge" : "", | 20260507 08:31:02.700 | INFO | "huge_shared" : " 27 GB", | 20260507 08:31:02.700 | INFO | "malloc" : " 14 GB", | 20260507 08:31:02.700 | INFO | "pinned" : " 29 GB" | 20260507 08:31:02.700 | INFO | } | 20260507 08:31:02.700 | INFO | Perf report for the last 5.13948 seconds: | 20260507 08:31:02.700 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:31:02.700 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:02.700 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:02.700 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:02.700 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 08:31:02.700 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:02.700 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 08:31:02.700 | INFO | PRE_HQ : 58.7%, 1.8, 54.2ms, 0 | 20260507 08:31:02.700 | INFO | BAZWRITER : 61.4%, 3.1, 96.4ms, 0 | 20260507 08:31:02.700 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:31:02.701 | INFO | Uploaded 36GiB to GPU in 2.06499s (17.4335GiB/s) | 20260507 08:31:02.701 | INFO | Downloaded 38GiB to GPU in 2.17764s (17.45GiB/s) | 20260507 08:31:07.841 | INFO | Analyzing chunk frames = [8544768,8545280) | 20260507 08:31:07.841 | INFO | MemInfo: { | 20260507 08:31:07.841 | INFO | "gpu" : " 17 GB", | 20260507 08:31:07.841 | INFO | "huge" : "", | 20260507 08:31:07.841 | INFO | "huge_shared" : " 27 GB", | 20260507 08:31:07.841 | INFO | "malloc" : " 14 GB", | 20260507 08:31:07.841 | INFO | "pinned" : " 29 GB" | 20260507 08:31:07.841 | INFO | } | 20260507 08:31:07.841 | INFO | Perf report for the last 5.14086 seconds: | 20260507 08:31:07.841 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:31:07.841 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:07.841 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:07.841 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:07.841 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:31:07.841 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:07.841 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:07.841 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260507 08:31:07.841 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:07.841 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:07.841 | INFO | Uploaded 36GiB to GPU in 1.99817s (18.0165GiB/s) | 20260507 08:31:07.841 | INFO | Downloaded 34GiB to GPU in 1.9384s (17.5403GiB/s) | 20260507 08:31:12.981 | INFO | Analyzing chunk frames = [8545280,8545792) | 20260507 08:31:12.981 | INFO | MemInfo: { | 20260507 08:31:12.981 | INFO | "gpu" : " 17 GB", | 20260507 08:31:12.981 | INFO | "huge" : "", | 20260507 08:31:12.982 | INFO | "huge_shared" : " 27 GB", | 20260507 08:31:12.982 | INFO | "malloc" : " 14 GB", | 20260507 08:31:12.982 | INFO | "pinned" : " 29 GB" | 20260507 08:31:12.982 | INFO | } | 20260507 08:31:12.982 | INFO | Perf report for the last 5.14036 seconds: | 20260507 08:31:12.982 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:31:12.982 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:12.982 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:12.982 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:12.982 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 08:31:12.982 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:12.982 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:12.982 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:31:12.982 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:12.982 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:12.982 | INFO | Uploaded 36GiB to GPU in 2.0224s (17.8007GiB/s) | 20260507 08:31:12.982 | INFO | Downloaded 34GiB to GPU in 1.91455s (17.7587GiB/s) | 20260507 08:31:18.126 | INFO | Analyzing chunk frames = [8545792,8546304) | 20260507 08:31:18.126 | INFO | MemInfo: { | 20260507 08:31:18.126 | INFO | "gpu" : " 17 GB", | 20260507 08:31:18.126 | INFO | "huge" : "", | 20260507 08:31:18.126 | INFO | "huge_shared" : " 27 GB", | 20260507 08:31:18.126 | INFO | "malloc" : " 14 GB", | 20260507 08:31:18.126 | INFO | "pinned" : " 29 GB" | 20260507 08:31:18.126 | INFO | } | 20260507 08:31:18.126 | INFO | Perf report for the last 5.14441 seconds: | 20260507 08:31:18.126 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:31:18.126 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:18.126 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:18.126 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:18.126 | INFO | ANALYSIS : 60.3%, 2.9, 93.2ms, 0 | 20260507 08:31:18.126 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:18.126 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:18.126 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:31:18.126 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:18.126 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:18.126 | INFO | Uploaded 36GiB to GPU in 2.05025s (17.5589GiB/s) | 20260507 08:31:18.126 | INFO | Downloaded 34GiB to GPU in 1.94999s (17.4359GiB/s) | 20260507 08:31:23.263 | INFO | Analyzing chunk frames = [8546304,8546816) | 20260507 08:31:23.263 | INFO | MemInfo: { | 20260507 08:31:23.263 | INFO | "gpu" : " 17 GB", | 20260507 08:31:23.263 | INFO | "huge" : "", | 20260507 08:31:23.263 | INFO | "huge_shared" : " 27 GB", | 20260507 08:31:23.263 | INFO | "malloc" : " 14 GB", | 20260507 08:31:23.263 | INFO | "pinned" : " 29 GB" | 20260507 08:31:23.263 | INFO | } | 20260507 08:31:23.263 | INFO | Perf report for the last 5.1368 seconds: | 20260507 08:31:23.263 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:31:23.263 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:23.263 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:23.263 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:23.263 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 08:31:23.263 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:23.263 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:23.263 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:31:23.263 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:23.263 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:23.263 | INFO | Uploaded 36GiB to GPU in 2.04881s (17.5712GiB/s) | 20260507 08:31:23.263 | INFO | Downloaded 34GiB to GPU in 1.91547s (17.7502GiB/s) | 20260507 08:31:28.398 | INFO | Analyzing chunk frames = [8546816,8547328) | 20260507 08:31:28.398 | INFO | MemInfo: { | 20260507 08:31:28.398 | INFO | "gpu" : " 17 GB", | 20260507 08:31:28.398 | INFO | "huge" : "", | 20260507 08:31:28.399 | INFO | "huge_shared" : " 27 GB", | 20260507 08:31:28.399 | INFO | "malloc" : " 14 GB", | 20260507 08:31:28.399 | INFO | "pinned" : " 29 GB" | 20260507 08:31:28.399 | INFO | } | 20260507 08:31:28.399 | INFO | Perf report for the last 5.13579 seconds: | 20260507 08:31:28.399 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:31:28.399 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:28.399 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:28.399 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:28.399 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:31:28.399 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:28.399 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:28.399 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 08:31:28.399 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:28.399 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:28.399 | INFO | Uploaded 36GiB to GPU in 2.0697s (17.3938GiB/s) | 20260507 08:31:28.399 | INFO | Downloaded 34GiB to GPU in 1.93552s (17.5664GiB/s) | 20260507 08:31:33.535 | INFO | Analyzing chunk frames = [8547328,8547840) | 20260507 08:31:33.535 | INFO | MemInfo: { | 20260507 08:31:33.535 | INFO | "gpu" : " 17 GB", | 20260507 08:31:33.535 | INFO | "huge" : "", | 20260507 08:31:33.535 | INFO | "huge_shared" : " 27 GB", | 20260507 08:31:33.535 | INFO | "malloc" : " 14 GB", | 20260507 08:31:33.535 | INFO | "pinned" : " 29 GB" | 20260507 08:31:33.535 | INFO | } | 20260507 08:31:33.535 | INFO | Perf report for the last 5.13625 seconds: | 20260507 08:31:33.535 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:31:33.535 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:33.535 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:33.535 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:33.535 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 08:31:33.535 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:33.535 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:33.535 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 08:31:33.535 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:33.535 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:33.535 | INFO | Uploaded 36GiB to GPU in 1.89965s (18.9508GiB/s) | 20260507 08:31:33.535 | INFO | Downloaded 34GiB to GPU in 1.97177s (17.2434GiB/s) | 20260507 08:31:38.673 | INFO | Analyzing chunk frames = [8547840,8548352) | 20260507 08:31:38.673 | INFO | MemInfo: { | 20260507 08:31:38.673 | INFO | "gpu" : " 17 GB", | 20260507 08:31:38.673 | INFO | "huge" : "", | 20260507 08:31:38.674 | INFO | "huge_shared" : " 27 GB", | 20260507 08:31:38.674 | INFO | "malloc" : " 14 GB", | 20260507 08:31:38.674 | INFO | "pinned" : " 29 GB" | 20260507 08:31:38.674 | INFO | } | 20260507 08:31:38.674 | INFO | Perf report for the last 5.13871 seconds: | 20260507 08:31:38.674 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:31:38.674 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:38.674 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:38.674 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:38.674 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:31:38.674 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:38.674 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:38.674 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:31:38.674 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:38.674 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:38.674 | INFO | Uploaded 36GiB to GPU in 1.98168s (18.1664GiB/s) | 20260507 08:31:38.674 | INFO | Downloaded 34GiB to GPU in 1.96289s (17.3214GiB/s) | 20260507 08:31:38.771 | INFO | Starting a round of Baz IO | 20260507 08:31:41.860 | INFO | Populating RTMetrics report | 20260507 08:31:41.860 | INFO | RTMetrics report populated | 20260507 08:31:41.860 | INFO | Opening tmp RTMetrics json file | 20260507 08:31:41.869 | INFO | Serializing RTMetrics to json | 20260507 08:31:41.875 | INFO | Serialization done, swapping tmp file now... | 20260507 08:31:41.876 | INFO | RTMetrics done | 20260507 08:31:41.946 | INFO | Finished a round of Baz IO: Wrote 1.61 GiB in 3.175 seconds | 20260507 08:31:41.946 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.49% / 0.00% / 67.51% | 20260507 08:31:41.946 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0897408% / 0% / 6.35783e-05% | 20260507 08:31:41.946 | INFO | Baz IO overall: | 20260507 08:31:41.946 | INFO | BytesWritten : 4255795454935 | 20260507 08:31:41.946 | INFO | NumEvents : 2035531091995 | 20260507 08:31:43.816 | INFO | Analyzing chunk frames = [8548352,8548864) | 20260507 08:31:43.816 | INFO | MemInfo: { | 20260507 08:31:43.816 | INFO | "gpu" : " 17 GB", | 20260507 08:31:43.816 | INFO | "huge" : "", | 20260507 08:31:43.816 | INFO | "huge_shared" : " 27 GB", | 20260507 08:31:43.816 | INFO | "malloc" : " 14 GB", | 20260507 08:31:43.816 | INFO | "pinned" : " 29 GB" | 20260507 08:31:43.816 | INFO | } | 20260507 08:31:43.816 | INFO | Perf report for the last 5.14256 seconds: | 20260507 08:31:43.816 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:31:43.816 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:43.816 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:43.816 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:43.816 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260507 08:31:43.816 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:43.816 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260507 08:31:43.816 | INFO | PRE_HQ : 58.6%, 1.8, 53.8ms, 0 | 20260507 08:31:43.816 | INFO | BAZWRITER : 61.3%, 3.0, 96.2ms, 0 | 20260507 08:31:43.816 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 08:31:43.816 | INFO | Uploaded 36GiB to GPU in 2.04271s (17.6236GiB/s) | 20260507 08:31:43.816 | INFO | Downloaded 38GiB to GPU in 2.17908s (17.4385GiB/s) | 20260507 08:31:48.967 | INFO | Analyzing chunk frames = [8548864,8549376) | 20260507 08:31:48.967 | INFO | MemInfo: { | 20260507 08:31:48.967 | INFO | "gpu" : " 17 GB", | 20260507 08:31:48.967 | INFO | "huge" : "", | 20260507 08:31:48.967 | INFO | "huge_shared" : " 27 GB", | 20260507 08:31:48.967 | INFO | "malloc" : " 14 GB", | 20260507 08:31:48.967 | INFO | "pinned" : " 29 GB" | 20260507 08:31:48.967 | INFO | } | 20260507 08:31:48.967 | INFO | Perf report for the last 5.15055 seconds: | 20260507 08:31:48.967 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:31:48.967 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:48.967 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:48.967 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:48.967 | INFO | ANALYSIS : 60.5%, 3.0, 94.5ms, 0 | 20260507 08:31:48.967 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:48.967 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:48.967 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260507 08:31:48.967 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:48.967 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:48.967 | INFO | Uploaded 36GiB to GPU in 1.977s (18.2094GiB/s) | 20260507 08:31:48.967 | INFO | Downloaded 34GiB to GPU in 1.92829s (17.6322GiB/s) | 20260507 08:31:54.101 | INFO | Analyzing chunk frames = [8549376,8549888) | 20260507 08:31:54.101 | INFO | MemInfo: { | 20260507 08:31:54.101 | INFO | "gpu" : " 17 GB", | 20260507 08:31:54.101 | INFO | "huge" : "", | 20260507 08:31:54.101 | INFO | "huge_shared" : " 27 GB", | 20260507 08:31:54.101 | INFO | "malloc" : " 14 GB", | 20260507 08:31:54.101 | INFO | "pinned" : " 29 GB" | 20260507 08:31:54.101 | INFO | } | 20260507 08:31:54.101 | INFO | Perf report for the last 5.13437 seconds: | 20260507 08:31:54.101 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:31:54.101 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:54.101 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:54.101 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:54.101 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 08:31:54.101 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:54.101 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:54.101 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:31:54.101 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:54.101 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:54.101 | INFO | Uploaded 36GiB to GPU in 2.04694s (17.5872GiB/s) | 20260507 08:31:54.101 | INFO | Downloaded 34GiB to GPU in 1.92748s (17.6396GiB/s) | 20260507 08:31:59.229 | INFO | Analyzing chunk frames = [8549888,8550400) | 20260507 08:31:59.229 | INFO | MemInfo: { | 20260507 08:31:59.229 | INFO | "gpu" : " 17 GB", | 20260507 08:31:59.229 | INFO | "huge" : "", | 20260507 08:31:59.229 | INFO | "huge_shared" : " 27 GB", | 20260507 08:31:59.229 | INFO | "malloc" : " 14 GB", | 20260507 08:31:59.229 | INFO | "pinned" : " 29 GB" | 20260507 08:31:59.229 | INFO | } | 20260507 08:31:59.229 | INFO | Perf report for the last 5.12797 seconds: | 20260507 08:31:59.229 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:31:59.229 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:59.229 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:59.229 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:59.229 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 08:31:59.229 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:59.229 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:59.229 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:31:59.229 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:59.229 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:31:59.229 | INFO | Uploaded 36GiB to GPU in 2.05574s (17.5119GiB/s) | 20260507 08:31:59.229 | INFO | Downloaded 34GiB to GPU in 1.94878s (17.4468GiB/s) | 20260507 08:32:04.370 | INFO | Analyzing chunk frames = [8550400,8550912) | 20260507 08:32:04.370 | INFO | MemInfo: { | 20260507 08:32:04.370 | INFO | "gpu" : " 17 GB", | 20260507 08:32:04.371 | INFO | "huge" : "", | 20260507 08:32:04.371 | INFO | "huge_shared" : " 27 GB", | 20260507 08:32:04.371 | INFO | "malloc" : " 14 GB", | 20260507 08:32:04.371 | INFO | "pinned" : " 29 GB" | 20260507 08:32:04.371 | INFO | } | 20260507 08:32:04.371 | INFO | Perf report for the last 5.14157 seconds: | 20260507 08:32:04.371 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:32:04.371 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:04.371 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:04.371 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:04.371 | INFO | ANALYSIS : 59.9%, 3.0, 93.5ms, 0 | 20260507 08:32:04.371 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:04.371 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:04.371 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:32:04.371 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:04.371 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:04.371 | INFO | Uploaded 36GiB to GPU in 2.02998s (17.7342GiB/s) | 20260507 08:32:04.371 | INFO | Downloaded 34GiB to GPU in 1.93404s (17.5798GiB/s) | 20260507 08:32:09.512 | INFO | Analyzing chunk frames = [8550912,8551424) | 20260507 08:32:09.512 | INFO | MemInfo: { | 20260507 08:32:09.512 | INFO | "gpu" : " 17 GB", | 20260507 08:32:09.512 | INFO | "huge" : "", | 20260507 08:32:09.512 | INFO | "huge_shared" : " 27 GB", | 20260507 08:32:09.512 | INFO | "malloc" : " 14 GB", | 20260507 08:32:09.512 | INFO | "pinned" : " 29 GB" | 20260507 08:32:09.512 | INFO | } | 20260507 08:32:09.512 | INFO | Perf report for the last 5.14111 seconds: | 20260507 08:32:09.512 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:32:09.512 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:09.512 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:09.512 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:09.512 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 08:32:09.512 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:09.512 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:09.512 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 08:32:09.512 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:09.512 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:09.512 | INFO | Uploaded 36GiB to GPU in 2.05088s (17.5535GiB/s) | 20260507 08:32:09.512 | INFO | Downloaded 34GiB to GPU in 1.91242s (17.7785GiB/s) | 20260507 08:32:14.656 | INFO | Analyzing chunk frames = [8551424,8551936) | 20260507 08:32:14.656 | INFO | MemInfo: { | 20260507 08:32:14.656 | INFO | "gpu" : " 17 GB", | 20260507 08:32:14.656 | INFO | "huge" : "", | 20260507 08:32:14.656 | INFO | "huge_shared" : " 27 GB", | 20260507 08:32:14.656 | INFO | "malloc" : " 14 GB", | 20260507 08:32:14.656 | INFO | "pinned" : " 29 GB" | 20260507 08:32:14.656 | INFO | } | 20260507 08:32:14.656 | INFO | Perf report for the last 5.1444 seconds: | 20260507 08:32:14.656 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:32:14.656 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:14.656 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:14.656 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:14.656 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 08:32:14.656 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:14.656 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:14.656 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 08:32:14.656 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:14.656 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:14.656 | INFO | Uploaded 36GiB to GPU in 1.88861s (19.0617GiB/s) | 20260507 08:32:14.656 | INFO | Downloaded 34GiB to GPU in 1.96955s (17.2628GiB/s) | 20260507 08:32:19.791 | INFO | Analyzing chunk frames = [8551936,8552448) | 20260507 08:32:19.791 | INFO | MemInfo: { | 20260507 08:32:19.791 | INFO | "gpu" : " 17 GB", | 20260507 08:32:19.791 | INFO | "huge" : "", | 20260507 08:32:19.791 | INFO | "huge_shared" : " 27 GB", | 20260507 08:32:19.791 | INFO | "malloc" : " 14 GB", | 20260507 08:32:19.791 | INFO | "pinned" : " 29 GB" | 20260507 08:32:19.791 | INFO | } | 20260507 08:32:19.791 | INFO | Perf report for the last 5.13492 seconds: | 20260507 08:32:19.791 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:32:19.791 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:19.791 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:19.791 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:19.791 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 08:32:19.791 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:19.791 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:19.791 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:32:19.791 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:19.791 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:19.791 | INFO | Uploaded 36GiB to GPU in 2.00954s (17.9145GiB/s) | 20260507 08:32:19.791 | INFO | Downloaded 34GiB to GPU in 1.96711s (17.2842GiB/s) | 20260507 08:32:19.791 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:32:19.791 | INFO | Total mem usage: 18 GiB | 20260507 08:32:19.791 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:32:19.791 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:32:19.791 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:32:19.792 | INFO | High Water Breakdown: | 20260507 08:32:19.792 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:32:19.792 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:32:19.792 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:32:19.792 | INFO | ------------------------------------------ | 20260507 08:32:19.792 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:32:19.792 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:32:19.792 | INFO | Total mem usage: 26.5 GiB | 20260507 08:32:19.792 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:32:19.792 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:32:19.792 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:32:19.792 | INFO | High Water Breakdown: | 20260507 08:32:19.792 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:32:19.792 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:32:19.792 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:32:19.792 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:32:19.792 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:32:19.792 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:32:19.792 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:32:19.792 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:32:19.792 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:32:19.792 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:32:19.792 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:32:19.792 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:32:19.792 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:32:19.792 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:32:19.792 | INFO | ------------------------------------------ | 20260507 08:32:19.792 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:32:19.792 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:32:19.792 | INFO | Total mem usage: 25.6 GiB | 20260507 08:32:19.792 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:32:19.792 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:32:19.792 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:32:19.792 | INFO | High Water Breakdown: | 20260507 08:32:19.792 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:32:19.792 | INFO | ------------------------------------------ | 20260507 08:32:19.792 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:32:19.792 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:32:19.792 | INFO | Total mem usage: 15.7 GiB | 20260507 08:32:19.792 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:32:19.792 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:32:19.792 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:32:19.792 | INFO | High Water Breakdown: | 20260507 08:32:19.792 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:32:19.792 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:32:19.792 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:32:19.792 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:32:19.792 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:32:19.792 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:32:19.792 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:32:19.792 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:32:19.792 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:32:19.792 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:32:19.792 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:32:19.792 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:32:19.792 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:32:19.792 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:32:19.792 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:32:19.792 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:32:19.792 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:32:19.792 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:32:19.792 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:32:19.792 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:32:19.792 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:32:19.792 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:32:19.792 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:32:19.792 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:32:19.792 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:32:19.792 | INFO | ------------------------------------------ | 20260507 08:32:19.792 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:32:19.792 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:32:19.792 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:32:19.792 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:32:19.792 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.46e+01, 1.33e+01 / 1.19e+00 | 20260507 08:32:19.792 | INFO | PerfReport: 0.29%, AggregateStats -- 9.02e-02 / 1.01e-01, 9.07e-03 / 4.16e-03 | 20260507 08:32:19.792 | INFO | PerfReport: 11.70%, Baseline -- 3.58e+00 / 4.54e+00, 5.06e-01 / 1.61e-01 | 20260507 08:32:19.792 | INFO | PerfReport: 3.22%, Histogram -- 9.87e-01 / 1.53e+00, 1.66e-01 / 5.82e-02 | 20260507 08:32:19.792 | INFO | PerfReport: 16.35%, DME -- 4.01e+01 / 4.50e+01, 4.00e+00 / 1.31e+00 | 20260507 08:32:19.792 | INFO | PerfReport: 57.65%, FrameLabeling -- 1.77e+01 / 1.91e+01, 1.21e+00 / 1.11e+00 | 20260507 08:32:19.792 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.11e+00 / 2.74e+00, 3.88e-01 / 1.40e-01 | 20260507 08:32:19.792 | INFO | PerfReport: 2.91%, Metrics -- 8.92e-01 / 1.20e+00, 2.38e-01 / 1.13e-01 | 20260507 08:32:19.792 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.99e-01 / 3.65e-01, 4.93e-02 / 2.19e-02 | 20260507 08:32:19.792 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:32:19.792 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:32:19.792 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:32:19.792 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 08:32:19.792 | INFO | PerfReport: 21.84%, Upload -- 2.10e+01 / 0.00e+00, 7.11e+00 / 0.00e+00 | 20260507 08:32:19.792 | INFO | PerfReport: 21.36%, Download -- 2.05e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260507 08:32:19.792 | INFO | PerfReport: 31.89%, Compute -- 3.06e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 08:32:19.792 | INFO | PerfReport: 1.59%, LockUpload -- 1.53e+00 / 0.00e+00, 4.96e+00 / 0.00e+00 | 20260507 08:32:19.792 | INFO | PerfReport: 1.32%, LockDownload -- 1.27e+00 / 0.00e+00, 3.10e+00 / 0.00e+00 | 20260507 08:32:19.792 | INFO | PerfReport: 21.97%, LockCompute -- 2.11e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 08:32:19.792 | INFO | PerfReport: 0.01%, Misc -- 1.44e-02 / 0.00e+00, 2.19e-02 / 0.00e+00 | 20260507 08:32:19.888 | INFO | Starting a round of Baz IO | 20260507 08:32:22.981 | INFO | Populating RTMetrics report | 20260507 08:32:22.981 | INFO | RTMetrics report populated | 20260507 08:32:22.981 | INFO | Opening tmp RTMetrics json file | 20260507 08:32:22.996 | INFO | Serializing RTMetrics to json | 20260507 08:32:22.997 | INFO | Serialization done, swapping tmp file now... | 20260507 08:32:22.998 | INFO | RTMetrics done | 20260507 08:32:23.068 | INFO | Finished a round of Baz IO: Wrote 1.61 GiB in 3.179 seconds | 20260507 08:32:23.068 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.49% / 0.00% / 67.51% | 20260507 08:32:23.068 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0896215% / 0% / 4.76837e-05% | 20260507 08:32:23.068 | INFO | Baz IO overall: | 20260507 08:32:23.068 | INFO | BytesWritten : 4257523754744 | 20260507 08:32:23.068 | INFO | NumEvents : 2036335864365 | 20260507 08:32:24.931 | INFO | Analyzing chunk frames = [8552448,8552960) | 20260507 08:32:24.931 | INFO | MemInfo: { | 20260507 08:32:24.931 | INFO | "gpu" : " 17 GB", | 20260507 08:32:24.931 | INFO | "huge" : "", | 20260507 08:32:24.931 | INFO | "huge_shared" : " 27 GB", | 20260507 08:32:24.931 | INFO | "malloc" : " 14 GB", | 20260507 08:32:24.931 | INFO | "pinned" : " 29 GB" | 20260507 08:32:24.931 | INFO | } | 20260507 08:32:24.931 | INFO | Perf report for the last 5.14021 seconds: | 20260507 08:32:24.931 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:32:24.931 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:24.931 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:24.932 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:24.932 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 08:32:24.932 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:24.932 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 08:32:24.932 | INFO | PRE_HQ : 58.8%, 1.8, 54.1ms, 0 | 20260507 08:32:24.932 | INFO | BAZWRITER : 61.5%, 3.1, 97.0ms, 0 | 20260507 08:32:24.932 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:32:24.932 | INFO | Uploaded 36GiB to GPU in 2.0687s (17.4022GiB/s) | 20260507 08:32:24.932 | INFO | Downloaded 38GiB to GPU in 2.17595s (17.4636GiB/s) | 20260507 08:32:30.074 | INFO | Analyzing chunk frames = [8552960,8553472) | 20260507 08:32:30.074 | INFO | MemInfo: { | 20260507 08:32:30.074 | INFO | "gpu" : " 17 GB", | 20260507 08:32:30.074 | INFO | "huge" : "", | 20260507 08:32:30.074 | INFO | "huge_shared" : " 27 GB", | 20260507 08:32:30.074 | INFO | "malloc" : " 14 GB", | 20260507 08:32:30.074 | INFO | "pinned" : " 29 GB" | 20260507 08:32:30.074 | INFO | } | 20260507 08:32:30.074 | INFO | Perf report for the last 5.14302 seconds: | 20260507 08:32:30.074 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:32:30.074 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:30.074 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:30.074 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:30.074 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:32:30.074 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:30.074 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:30.074 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260507 08:32:30.074 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:30.074 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:30.075 | INFO | Uploaded 36GiB to GPU in 1.99736s (18.0238GiB/s) | 20260507 08:32:30.075 | INFO | Downloaded 34GiB to GPU in 1.93554s (17.5661GiB/s) | 20260507 08:32:35.209 | INFO | Analyzing chunk frames = [8553472,8553984) | 20260507 08:32:35.209 | INFO | MemInfo: { | 20260507 08:32:35.209 | INFO | "gpu" : " 17 GB", | 20260507 08:32:35.209 | INFO | "huge" : "", | 20260507 08:32:35.209 | INFO | "huge_shared" : " 27 GB", | 20260507 08:32:35.209 | INFO | "malloc" : " 14 GB", | 20260507 08:32:35.209 | INFO | "pinned" : " 29 GB" | 20260507 08:32:35.209 | INFO | } | 20260507 08:32:35.209 | INFO | Perf report for the last 5.13466 seconds: | 20260507 08:32:35.209 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:32:35.209 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:35.209 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:35.209 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:35.209 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 08:32:35.209 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:35.209 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:35.209 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:32:35.209 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:35.209 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:35.209 | INFO | Uploaded 36GiB to GPU in 2.02781s (17.7531GiB/s) | 20260507 08:32:35.209 | INFO | Downloaded 34GiB to GPU in 1.9236s (17.6752GiB/s) | 20260507 08:32:40.347 | INFO | Analyzing chunk frames = [8553984,8554496) | 20260507 08:32:40.347 | INFO | MemInfo: { | 20260507 08:32:40.347 | INFO | "gpu" : " 17 GB", | 20260507 08:32:40.347 | INFO | "huge" : "", | 20260507 08:32:40.347 | INFO | "huge_shared" : " 27 GB", | 20260507 08:32:40.347 | INFO | "malloc" : " 14 GB", | 20260507 08:32:40.347 | INFO | "pinned" : " 29 GB" | 20260507 08:32:40.347 | INFO | } | 20260507 08:32:40.347 | INFO | Perf report for the last 5.13834 seconds: | 20260507 08:32:40.347 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:32:40.347 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:40.347 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:40.347 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:40.347 | INFO | ANALYSIS : 60.6%, 2.9, 93.5ms, 0 | 20260507 08:32:40.347 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:40.348 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:40.348 | INFO | PRE_HQ : 26.0%, 1.0, 13.5ms, 0 | 20260507 08:32:40.348 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:40.348 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:40.348 | INFO | Uploaded 36GiB to GPU in 2.05503s (17.518GiB/s) | 20260507 08:32:40.348 | INFO | Downloaded 34GiB to GPU in 1.95037s (17.4326GiB/s) | 20260507 08:32:45.485 | INFO | Analyzing chunk frames = [8554496,8555008) | 20260507 08:32:45.485 | INFO | MemInfo: { | 20260507 08:32:45.485 | INFO | "gpu" : " 17 GB", | 20260507 08:32:45.485 | INFO | "huge" : "", | 20260507 08:32:45.485 | INFO | "huge_shared" : " 27 GB", | 20260507 08:32:45.485 | INFO | "malloc" : " 14 GB", | 20260507 08:32:45.485 | INFO | "pinned" : " 29 GB" | 20260507 08:32:45.485 | INFO | } | 20260507 08:32:45.485 | INFO | Perf report for the last 5.13764 seconds: | 20260507 08:32:45.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:32:45.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:45.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:45.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:45.485 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 08:32:45.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:45.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:45.485 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:32:45.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:45.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:45.485 | INFO | Uploaded 36GiB to GPU in 2.05065s (17.5554GiB/s) | 20260507 08:32:45.485 | INFO | Downloaded 34GiB to GPU in 1.91112s (17.7906GiB/s) | 20260507 08:32:50.626 | INFO | Analyzing chunk frames = [8555008,8555520) | 20260507 08:32:50.626 | INFO | MemInfo: { | 20260507 08:32:50.626 | INFO | "gpu" : " 17 GB", | 20260507 08:32:50.626 | INFO | "huge" : "", | 20260507 08:32:50.626 | INFO | "huge_shared" : " 27 GB", | 20260507 08:32:50.626 | INFO | "malloc" : " 14 GB", | 20260507 08:32:50.626 | INFO | "pinned" : " 29 GB" | 20260507 08:32:50.626 | INFO | } | 20260507 08:32:50.626 | INFO | Perf report for the last 5.14083 seconds: | 20260507 08:32:50.626 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:32:50.626 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:50.626 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:50.626 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:50.626 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 08:32:50.626 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:50.626 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:50.626 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:32:50.626 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:50.626 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:50.626 | INFO | Uploaded 36GiB to GPU in 2.05035s (17.558GiB/s) | 20260507 08:32:50.626 | INFO | Downloaded 34GiB to GPU in 1.9093s (17.8076GiB/s) | 20260507 08:32:55.765 | INFO | Analyzing chunk frames = [8555520,8556032) | 20260507 08:32:55.765 | INFO | MemInfo: { | 20260507 08:32:55.765 | INFO | "gpu" : " 17 GB", | 20260507 08:32:55.765 | INFO | "huge" : "", | 20260507 08:32:55.766 | INFO | "huge_shared" : " 27 GB", | 20260507 08:32:55.766 | INFO | "malloc" : " 14 GB", | 20260507 08:32:55.766 | INFO | "pinned" : " 29 GB" | 20260507 08:32:55.766 | INFO | } | 20260507 08:32:55.766 | INFO | Perf report for the last 5.13967 seconds: | 20260507 08:32:55.766 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:32:55.766 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:55.766 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:55.766 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:55.766 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:32:55.766 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:55.766 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:55.766 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 08:32:55.766 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:55.766 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:32:55.766 | INFO | Uploaded 36GiB to GPU in 1.99546s (18.041GiB/s) | 20260507 08:32:55.766 | INFO | Downloaded 34GiB to GPU in 1.97061s (17.2535GiB/s) | 20260507 08:33:00.910 | INFO | Analyzing chunk frames = [8556032,8556544) | 20260507 08:33:00.910 | INFO | MemInfo: { | 20260507 08:33:00.910 | INFO | "gpu" : " 17 GB", | 20260507 08:33:00.910 | INFO | "huge" : "", | 20260507 08:33:00.910 | INFO | "huge_shared" : " 27 GB", | 20260507 08:33:00.910 | INFO | "malloc" : " 14 GB", | 20260507 08:33:00.910 | INFO | "pinned" : " 29 GB" | 20260507 08:33:00.910 | INFO | } | 20260507 08:33:00.910 | INFO | Perf report for the last 5.14434 seconds: | 20260507 08:33:00.910 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:33:00.910 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:00.910 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:00.910 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:00.910 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 08:33:00.910 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:00.910 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:00.910 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 08:33:00.910 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:00.910 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:00.910 | INFO | Uploaded 36GiB to GPU in 2.01094s (17.9021GiB/s) | 20260507 08:33:00.910 | INFO | Downloaded 34GiB to GPU in 1.96602s (17.2938GiB/s) | 20260507 08:33:01.007 | INFO | Starting a round of Baz IO | 20260507 08:33:04.088 | INFO | Populating RTMetrics report | 20260507 08:33:04.088 | INFO | RTMetrics report populated | 20260507 08:33:04.088 | INFO | Opening tmp RTMetrics json file | 20260507 08:33:04.098 | INFO | Serializing RTMetrics to json | 20260507 08:33:04.103 | INFO | Serialization done, swapping tmp file now... | 20260507 08:33:04.104 | INFO | RTMetrics done | 20260507 08:33:04.171 | INFO | Finished a round of Baz IO: Wrote 1.611 GiB in 3.164 seconds | 20260507 08:33:04.171 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.49% / 0.00% / 67.51% | 20260507 08:33:04.171 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0893394% / 0% / 5.96046e-05% | 20260507 08:33:04.171 | INFO | Baz IO overall: | 20260507 08:33:04.171 | INFO | BytesWritten : 4259253145150 | 20260507 08:33:04.171 | INFO | NumEvents : 2037141436995 | 20260507 08:33:06.041 | INFO | Analyzing chunk frames = [8556544,8557056) | 20260507 08:33:06.041 | INFO | MemInfo: { | 20260507 08:33:06.041 | INFO | "gpu" : " 17 GB", | 20260507 08:33:06.041 | INFO | "huge" : "", | 20260507 08:33:06.041 | INFO | "huge_shared" : " 27 GB", | 20260507 08:33:06.041 | INFO | "malloc" : " 14 GB", | 20260507 08:33:06.041 | INFO | "pinned" : " 29 GB" | 20260507 08:33:06.041 | INFO | } | 20260507 08:33:06.041 | INFO | Perf report for the last 5.13131 seconds: | 20260507 08:33:06.041 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:33:06.041 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:06.041 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:06.041 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:06.041 | INFO | ANALYSIS : 61.4%, 3.0, 95.4ms, 0 | 20260507 08:33:06.041 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:06.041 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260507 08:33:06.041 | INFO | PRE_HQ : 58.7%, 1.8, 54.4ms, 0 | 20260507 08:33:06.041 | INFO | BAZWRITER : 61.2%, 3.1, 97.6ms, 0 | 20260507 08:33:06.041 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:33:06.042 | INFO | Uploaded 36GiB to GPU in 2.05895s (17.4846GiB/s) | 20260507 08:33:06.042 | INFO | Downloaded 38GiB to GPU in 2.17039s (17.5084GiB/s) | 20260507 08:33:11.179 | INFO | Analyzing chunk frames = [8557056,8557568) | 20260507 08:33:11.179 | INFO | MemInfo: { | 20260507 08:33:11.179 | INFO | "gpu" : " 17 GB", | 20260507 08:33:11.179 | INFO | "huge" : "", | 20260507 08:33:11.179 | INFO | "huge_shared" : " 27 GB", | 20260507 08:33:11.179 | INFO | "malloc" : " 14 GB", | 20260507 08:33:11.179 | INFO | "pinned" : " 29 GB" | 20260507 08:33:11.179 | INFO | } | 20260507 08:33:11.179 | INFO | Perf report for the last 5.13819 seconds: | 20260507 08:33:11.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:33:11.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:11.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:11.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:11.180 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 08:33:11.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:11.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:11.180 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260507 08:33:11.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:11.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:11.180 | INFO | Uploaded 36GiB to GPU in 1.99626s (18.0338GiB/s) | 20260507 08:33:11.180 | INFO | Downloaded 34GiB to GPU in 1.93612s (17.5609GiB/s) | 20260507 08:33:16.324 | INFO | Analyzing chunk frames = [8557568,8558080) | 20260507 08:33:16.324 | INFO | MemInfo: { | 20260507 08:33:16.324 | INFO | "gpu" : " 17 GB", | 20260507 08:33:16.324 | INFO | "huge" : "", | 20260507 08:33:16.324 | INFO | "huge_shared" : " 27 GB", | 20260507 08:33:16.324 | INFO | "malloc" : " 14 GB", | 20260507 08:33:16.324 | INFO | "pinned" : " 29 GB" | 20260507 08:33:16.324 | INFO | } | 20260507 08:33:16.324 | INFO | Perf report for the last 5.14446 seconds: | 20260507 08:33:16.324 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:33:16.324 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:16.324 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:16.324 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:16.324 | INFO | ANALYSIS : 60.0%, 3.0, 93.0ms, 0 | 20260507 08:33:16.324 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:16.324 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:16.324 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:33:16.324 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:16.324 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:16.324 | INFO | Uploaded 36GiB to GPU in 2.02372s (17.789GiB/s) | 20260507 08:33:16.324 | INFO | Downloaded 34GiB to GPU in 1.93933s (17.5318GiB/s) | 20260507 08:33:21.466 | INFO | Analyzing chunk frames = [8558080,8558592) | 20260507 08:33:21.466 | INFO | MemInfo: { | 20260507 08:33:21.466 | INFO | "gpu" : " 17 GB", | 20260507 08:33:21.466 | INFO | "huge" : "", | 20260507 08:33:21.466 | INFO | "huge_shared" : " 27 GB", | 20260507 08:33:21.466 | INFO | "malloc" : " 14 GB", | 20260507 08:33:21.466 | INFO | "pinned" : " 29 GB" | 20260507 08:33:21.466 | INFO | } | 20260507 08:33:21.466 | INFO | Perf report for the last 5.14194 seconds: | 20260507 08:33:21.466 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:33:21.466 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:21.466 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:21.466 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:21.466 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 08:33:21.466 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:21.466 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:21.466 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:33:21.466 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:21.466 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:21.466 | INFO | Uploaded 36GiB to GPU in 2.04454s (17.6079GiB/s) | 20260507 08:33:21.466 | INFO | Downloaded 34GiB to GPU in 1.95238s (17.4147GiB/s) | 20260507 08:33:26.606 | INFO | Analyzing chunk frames = [8558592,8559104) | 20260507 08:33:26.606 | INFO | MemInfo: { | 20260507 08:33:26.606 | INFO | "gpu" : " 17 GB", | 20260507 08:33:26.606 | INFO | "huge" : "", | 20260507 08:33:26.606 | INFO | "huge_shared" : " 27 GB", | 20260507 08:33:26.606 | INFO | "malloc" : " 14 GB", | 20260507 08:33:26.606 | INFO | "pinned" : " 29 GB" | 20260507 08:33:26.606 | INFO | } | 20260507 08:33:26.606 | INFO | Perf report for the last 5.1401 seconds: | 20260507 08:33:26.606 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:33:26.606 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:26.606 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:26.606 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:26.606 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 08:33:26.606 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:26.606 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:26.606 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:33:26.606 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:26.606 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:26.606 | INFO | Uploaded 36GiB to GPU in 2.0543s (17.5242GiB/s) | 20260507 08:33:26.606 | INFO | Downloaded 34GiB to GPU in 1.90666s (17.8322GiB/s) | 20260507 08:33:31.740 | INFO | Analyzing chunk frames = [8559104,8559616) | 20260507 08:33:31.740 | INFO | MemInfo: { | 20260507 08:33:31.740 | INFO | "gpu" : " 17 GB", | 20260507 08:33:31.740 | INFO | "huge" : "", | 20260507 08:33:31.740 | INFO | "huge_shared" : " 27 GB", | 20260507 08:33:31.740 | INFO | "malloc" : " 14 GB", | 20260507 08:33:31.740 | INFO | "pinned" : " 29 GB" | 20260507 08:33:31.740 | INFO | } | 20260507 08:33:31.740 | INFO | Perf report for the last 5.13442 seconds: | 20260507 08:33:31.741 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:33:31.741 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:31.741 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:31.741 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:31.741 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 08:33:31.741 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:31.741 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:31.741 | INFO | PRE_HQ : 24.8%, 1.0, 12.8ms, 0 | 20260507 08:33:31.741 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:31.741 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:31.741 | INFO | Uploaded 36GiB to GPU in 2.06998s (17.3915GiB/s) | 20260507 08:33:31.741 | INFO | Downloaded 34GiB to GPU in 1.93862s (17.5382GiB/s) | 20260507 08:33:36.880 | INFO | Analyzing chunk frames = [8559616,8560128) | 20260507 08:33:36.880 | INFO | MemInfo: { | 20260507 08:33:36.880 | INFO | "gpu" : " 17 GB", | 20260507 08:33:36.880 | INFO | "huge" : "", | 20260507 08:33:36.880 | INFO | "huge_shared" : " 27 GB", | 20260507 08:33:36.880 | INFO | "malloc" : " 14 GB", | 20260507 08:33:36.880 | INFO | "pinned" : " 29 GB" | 20260507 08:33:36.880 | INFO | } | 20260507 08:33:36.880 | INFO | Perf report for the last 5.13989 seconds: | 20260507 08:33:36.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:33:36.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:36.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:36.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:36.880 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 08:33:36.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:36.880 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:36.880 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 08:33:36.880 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:36.880 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:36.881 | INFO | Uploaded 36GiB to GPU in 1.94596s (18.4999GiB/s) | 20260507 08:33:36.881 | INFO | Downloaded 34GiB to GPU in 1.97093s (17.2507GiB/s) | 20260507 08:33:42.019 | INFO | Analyzing chunk frames = [8560128,8560640) | 20260507 08:33:42.019 | INFO | MemInfo: { | 20260507 08:33:42.019 | INFO | "gpu" : " 17 GB", | 20260507 08:33:42.019 | INFO | "huge" : "", | 20260507 08:33:42.019 | INFO | "huge_shared" : " 27 GB", | 20260507 08:33:42.019 | INFO | "malloc" : " 14 GB", | 20260507 08:33:42.019 | INFO | "pinned" : " 29 GB" | 20260507 08:33:42.019 | INFO | } | 20260507 08:33:42.019 | INFO | Perf report for the last 5.13916 seconds: | 20260507 08:33:42.020 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:33:42.020 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:42.020 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:42.020 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:42.020 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:33:42.020 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:42.020 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:42.020 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 08:33:42.020 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:42.020 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:42.020 | INFO | Uploaded 36GiB to GPU in 1.99178s (18.0743GiB/s) | 20260507 08:33:42.020 | INFO | Downloaded 34GiB to GPU in 1.95808s (17.364GiB/s) | 20260507 08:33:42.020 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:33:42.020 | INFO | Total mem usage: 18 GiB | 20260507 08:33:42.020 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:33:42.020 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:33:42.020 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:33:42.020 | INFO | High Water Breakdown: | 20260507 08:33:42.020 | INFO | 60.1% : 8.54 GiB ( 74.9% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:33:42.020 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:33:42.020 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:33:42.020 | INFO | ------------------------------------------ | 20260507 08:33:42.020 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:33:42.020 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:33:42.020 | INFO | Total mem usage: 26.5 GiB | 20260507 08:33:42.020 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:33:42.020 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:33:42.020 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:33:42.020 | INFO | High Water Breakdown: | 20260507 08:33:42.020 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:33:42.020 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:33:42.020 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:33:42.020 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:33:42.020 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:33:42.020 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:33:42.020 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:33:42.020 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:33:42.020 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:33:42.020 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:33:42.021 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:33:42.021 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:33:42.021 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:33:42.021 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:33:42.021 | INFO | ------------------------------------------ | 20260507 08:33:42.021 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:33:42.021 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:33:42.021 | INFO | Total mem usage: 25.6 GiB | 20260507 08:33:42.021 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:33:42.021 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:33:42.021 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:33:42.021 | INFO | High Water Breakdown: | 20260507 08:33:42.021 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:33:42.021 | INFO | ------------------------------------------ | 20260507 08:33:42.021 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:33:42.021 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:33:42.021 | INFO | Total mem usage: 15.7 GiB | 20260507 08:33:42.021 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:33:42.021 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:33:42.021 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:33:42.021 | INFO | High Water Breakdown: | 20260507 08:33:42.021 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:33:42.021 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:33:42.021 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:33:42.021 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:33:42.021 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:33:42.021 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:33:42.021 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:33:42.021 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:33:42.021 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:33:42.021 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:33:42.021 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:33:42.021 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:33:42.021 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:33:42.021 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:33:42.021 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:33:42.021 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:33:42.021 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:33:42.021 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:33:42.021 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:33:42.021 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:33:42.021 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:33:42.021 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:33:42.021 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:33:42.021 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:33:42.021 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:33:42.021 | INFO | ------------------------------------------ | 20260507 08:33:42.021 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:33:42.021 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:33:42.021 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:33:42.021 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:33:42.021 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.34e+01 / 1.37e+00 | 20260507 08:33:42.021 | INFO | PerfReport: 0.30%, AggregateStats -- 9.06e-02 / 1.02e-01, 9.62e-03 / 5.66e-03 | 20260507 08:33:42.021 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.45e+00, 5.01e-01 / 2.43e-01 | 20260507 08:33:42.021 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.48e+00, 1.66e-01 / 1.17e-01 | 20260507 08:33:42.021 | INFO | PerfReport: 16.32%, DME -- 4.00e+01 / 4.49e+01, 3.93e+00 / 1.61e+00 | 20260507 08:33:42.021 | INFO | PerfReport: 57.75%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.31e+00 / 1.13e+00 | 20260507 08:33:42.021 | INFO | PerfReport: 6.87%, PulseAccumulating -- 2.11e+00 / 2.65e+00, 3.82e-01 / 2.30e-01 | 20260507 08:33:42.021 | INFO | PerfReport: 2.90%, Metrics -- 8.89e-01 / 1.19e+00, 2.38e-01 / 1.11e-01 | 20260507 08:33:42.021 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.99e-01 / 3.58e-01, 4.95e-02 / 2.36e-02 | 20260507 08:33:42.021 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:33:42.021 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:33:42.021 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:33:42.021 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.42e+01 / 0.00e+00 | 20260507 08:33:42.021 | INFO | PerfReport: 21.97%, Upload -- 2.11e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 08:33:42.021 | INFO | PerfReport: 21.34%, Download -- 2.05e+01 / 0.00e+00, 7.25e+00 / 0.00e+00 | 20260507 08:33:42.021 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 08:33:42.021 | INFO | PerfReport: 1.67%, LockUpload -- 1.60e+00 / 0.00e+00, 5.04e+00 / 0.00e+00 | 20260507 08:33:42.021 | INFO | PerfReport: 1.16%, LockDownload -- 1.12e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260507 08:33:42.021 | INFO | PerfReport: 21.93%, LockCompute -- 2.11e+01 / 0.00e+00, 1.84e+01 / 0.00e+00 | 20260507 08:33:42.021 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.84e-02 / 0.00e+00 | 20260507 08:33:42.117 | INFO | Starting a round of Baz IO | 20260507 08:33:45.221 | INFO | Populating RTMetrics report | 20260507 08:33:45.221 | INFO | RTMetrics report populated | 20260507 08:33:45.221 | INFO | Opening tmp RTMetrics json file | 20260507 08:33:45.238 | INFO | Serializing RTMetrics to json | 20260507 08:33:45.239 | INFO | Serialization done, swapping tmp file now... | 20260507 08:33:45.240 | INFO | RTMetrics done | 20260507 08:33:45.306 | INFO | Finished a round of Baz IO: Wrote 1.608 GiB in 3.189 seconds | 20260507 08:33:45.306 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.49% / 0.00% / 67.51% | 20260507 08:33:45.306 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.08895% / 0% / 0.000103315% | 20260507 08:33:45.306 | INFO | Baz IO overall: | 20260507 08:33:45.306 | INFO | BytesWritten : 4260979956751 | 20260507 08:33:45.306 | INFO | NumEvents : 2037944986241 | 20260507 08:33:47.164 | INFO | Analyzing chunk frames = [8560640,8561152) | 20260507 08:33:47.164 | INFO | MemInfo: { | 20260507 08:33:47.164 | INFO | "gpu" : " 17 GB", | 20260507 08:33:47.164 | INFO | "huge" : "", | 20260507 08:33:47.164 | INFO | "huge_shared" : " 27 GB", | 20260507 08:33:47.164 | INFO | "malloc" : " 14 GB", | 20260507 08:33:47.164 | INFO | "pinned" : " 29 GB" | 20260507 08:33:47.164 | INFO | } | 20260507 08:33:47.164 | INFO | Perf report for the last 5.14427 seconds: | 20260507 08:33:47.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:33:47.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:47.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:47.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:47.164 | INFO | ANALYSIS : 61.4%, 3.0, 96.0ms, 0 | 20260507 08:33:47.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:47.164 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 08:33:47.164 | INFO | PRE_HQ : 58.8%, 1.8, 54.8ms, 0 | 20260507 08:33:47.164 | INFO | BAZWRITER : 61.6%, 3.1, 97.8ms, 0 | 20260507 08:33:47.164 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 08:33:47.164 | INFO | Uploaded 36GiB to GPU in 2.05296s (17.5357GiB/s) | 20260507 08:33:47.164 | INFO | Downloaded 38GiB to GPU in 2.17183s (17.4967GiB/s) | 20260507 08:33:52.298 | INFO | Analyzing chunk frames = [8561152,8561664) | 20260507 08:33:52.298 | INFO | MemInfo: { | 20260507 08:33:52.298 | INFO | "gpu" : " 17 GB", | 20260507 08:33:52.298 | INFO | "huge" : "", | 20260507 08:33:52.298 | INFO | "huge_shared" : " 27 GB", | 20260507 08:33:52.298 | INFO | "malloc" : " 14 GB", | 20260507 08:33:52.298 | INFO | "pinned" : " 29 GB" | 20260507 08:33:52.298 | INFO | } | 20260507 08:33:52.298 | INFO | Perf report for the last 5.13459 seconds: | 20260507 08:33:52.298 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:33:52.298 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:52.298 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:52.298 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:52.298 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 08:33:52.298 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:52.298 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:52.298 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260507 08:33:52.299 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:52.299 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:52.299 | INFO | Uploaded 36GiB to GPU in 2.01417s (17.8734GiB/s) | 20260507 08:33:52.299 | INFO | Downloaded 34GiB to GPU in 1.94298s (17.4989GiB/s) | 20260507 08:33:57.436 | INFO | Analyzing chunk frames = [8561664,8562176) | 20260507 08:33:57.436 | INFO | MemInfo: { | 20260507 08:33:57.436 | INFO | "gpu" : " 17 GB", | 20260507 08:33:57.436 | INFO | "huge" : "", | 20260507 08:33:57.436 | INFO | "huge_shared" : " 27 GB", | 20260507 08:33:57.436 | INFO | "malloc" : " 14 GB", | 20260507 08:33:57.436 | INFO | "pinned" : " 29 GB" | 20260507 08:33:57.436 | INFO | } | 20260507 08:33:57.436 | INFO | Perf report for the last 5.13745 seconds: | 20260507 08:33:57.436 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:33:57.436 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:57.436 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:57.436 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:57.436 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 08:33:57.436 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:57.436 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:57.436 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 08:33:57.436 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:57.436 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:33:57.436 | INFO | Uploaded 36GiB to GPU in 2.05433s (17.5239GiB/s) | 20260507 08:33:57.436 | INFO | Downloaded 34GiB to GPU in 1.94987s (17.437GiB/s) | 20260507 08:34:02.580 | INFO | Analyzing chunk frames = [8562176,8562688) | 20260507 08:34:02.580 | INFO | MemInfo: { | 20260507 08:34:02.580 | INFO | "gpu" : " 17 GB", | 20260507 08:34:02.580 | INFO | "huge" : "", | 20260507 08:34:02.580 | INFO | "huge_shared" : " 27 GB", | 20260507 08:34:02.580 | INFO | "malloc" : " 14 GB", | 20260507 08:34:02.580 | INFO | "pinned" : " 29 GB" | 20260507 08:34:02.580 | INFO | } | 20260507 08:34:02.580 | INFO | Perf report for the last 5.14458 seconds: | 20260507 08:34:02.580 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:34:02.580 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:02.580 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:02.581 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:02.581 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 08:34:02.581 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:02.581 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:02.581 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:34:02.581 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:02.581 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:02.581 | INFO | Uploaded 36GiB to GPU in 2.02036s (17.8186GiB/s) | 20260507 08:34:02.581 | INFO | Downloaded 34GiB to GPU in 1.94022s (17.5238GiB/s) | 20260507 08:34:07.719 | INFO | Analyzing chunk frames = [8562688,8563200) | 20260507 08:34:07.719 | INFO | MemInfo: { | 20260507 08:34:07.719 | INFO | "gpu" : " 17 GB", | 20260507 08:34:07.719 | INFO | "huge" : "", | 20260507 08:34:07.719 | INFO | "huge_shared" : " 27 GB", | 20260507 08:34:07.719 | INFO | "malloc" : " 14 GB", | 20260507 08:34:07.719 | INFO | "pinned" : " 29 GB" | 20260507 08:34:07.719 | INFO | } | 20260507 08:34:07.719 | INFO | Perf report for the last 5.13848 seconds: | 20260507 08:34:07.719 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:34:07.719 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:07.719 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:07.719 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:07.719 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 08:34:07.719 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:07.719 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:07.719 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:34:07.719 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:07.719 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:07.719 | INFO | Uploaded 36GiB to GPU in 2.04852s (17.5737GiB/s) | 20260507 08:34:07.719 | INFO | Downloaded 34GiB to GPU in 1.92034s (17.7052GiB/s) | 20260507 08:34:12.855 | INFO | Analyzing chunk frames = [8563200,8563712) | 20260507 08:34:12.855 | INFO | MemInfo: { | 20260507 08:34:12.855 | INFO | "gpu" : " 17 GB", | 20260507 08:34:12.855 | INFO | "huge" : "", | 20260507 08:34:12.855 | INFO | "huge_shared" : " 27 GB", | 20260507 08:34:12.855 | INFO | "malloc" : " 14 GB", | 20260507 08:34:12.855 | INFO | "pinned" : " 29 GB" | 20260507 08:34:12.855 | INFO | } | 20260507 08:34:12.855 | INFO | Perf report for the last 5.13612 seconds: | 20260507 08:34:12.855 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:34:12.855 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:12.855 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:12.855 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:12.855 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 08:34:12.855 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:12.855 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:12.855 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 08:34:12.855 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:12.855 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:12.855 | INFO | Uploaded 36GiB to GPU in 2.0701s (17.3904GiB/s) | 20260507 08:34:12.856 | INFO | Downloaded 34GiB to GPU in 1.94304s (17.4984GiB/s) | 20260507 08:34:17.989 | INFO | Analyzing chunk frames = [8563712,8564224) | 20260507 08:34:17.989 | INFO | MemInfo: { | 20260507 08:34:17.989 | INFO | "gpu" : " 17 GB", | 20260507 08:34:17.989 | INFO | "huge" : "", | 20260507 08:34:17.989 | INFO | "huge_shared" : " 27 GB", | 20260507 08:34:17.989 | INFO | "malloc" : " 14 GB", | 20260507 08:34:17.989 | INFO | "pinned" : " 29 GB" | 20260507 08:34:17.989 | INFO | } | 20260507 08:34:17.989 | INFO | Perf report for the last 5.13409 seconds: | 20260507 08:34:17.989 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:34:17.989 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:17.989 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:17.989 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:17.989 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 08:34:17.989 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:17.989 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:17.989 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 08:34:17.989 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:17.989 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:17.990 | INFO | Uploaded 36GiB to GPU in 1.88735s (19.0743GiB/s) | 20260507 08:34:17.990 | INFO | Downloaded 34GiB to GPU in 1.97578s (17.2084GiB/s) | 20260507 08:34:23.135 | INFO | Analyzing chunk frames = [8564224,8564736) | 20260507 08:34:23.135 | INFO | MemInfo: { | 20260507 08:34:23.135 | INFO | "gpu" : " 17 GB", | 20260507 08:34:23.135 | INFO | "huge" : "", | 20260507 08:34:23.135 | INFO | "huge_shared" : " 27 GB", | 20260507 08:34:23.135 | INFO | "malloc" : " 14 GB", | 20260507 08:34:23.135 | INFO | "pinned" : " 29 GB" | 20260507 08:34:23.135 | INFO | } | 20260507 08:34:23.135 | INFO | Perf report for the last 5.14622 seconds: | 20260507 08:34:23.135 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:34:23.135 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:23.135 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:23.135 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:23.135 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 08:34:23.135 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:23.135 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:23.135 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:34:23.135 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:23.136 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:23.136 | INFO | Uploaded 36GiB to GPU in 1.95663s (18.399GiB/s) | 20260507 08:34:23.136 | INFO | Downloaded 34GiB to GPU in 1.96162s (17.3326GiB/s) | 20260507 08:34:23.232 | INFO | Starting a round of Baz IO | 20260507 08:34:26.322 | INFO | Populating RTMetrics report | 20260507 08:34:26.322 | INFO | RTMetrics report populated | 20260507 08:34:26.322 | INFO | Opening tmp RTMetrics json file | 20260507 08:34:26.331 | INFO | Serializing RTMetrics to json | 20260507 08:34:26.337 | INFO | Serialization done, swapping tmp file now... | 20260507 08:34:26.338 | INFO | RTMetrics done | 20260507 08:34:26.406 | INFO | Finished a round of Baz IO: Wrote 1.609 GiB in 3.173 seconds | 20260507 08:34:26.406 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.49% / 0.00% / 67.51% | 20260507 08:34:26.406 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0891209% / 0% / 6.35783e-05% | 20260507 08:34:26.406 | INFO | Baz IO overall: | 20260507 08:34:26.406 | INFO | BytesWritten : 4262707766807 | 20260507 08:34:26.406 | INFO | NumEvents : 2038749453081 | 20260507 08:34:28.278 | INFO | Analyzing chunk frames = [8564736,8565248) | 20260507 08:34:28.278 | INFO | MemInfo: { | 20260507 08:34:28.278 | INFO | "gpu" : " 17 GB", | 20260507 08:34:28.278 | INFO | "huge" : "", | 20260507 08:34:28.278 | INFO | "huge_shared" : " 27 GB", | 20260507 08:34:28.278 | INFO | "malloc" : " 14 GB", | 20260507 08:34:28.278 | INFO | "pinned" : " 29 GB" | 20260507 08:34:28.278 | INFO | } | 20260507 08:34:28.278 | INFO | Perf report for the last 5.14285 seconds: | 20260507 08:34:28.278 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:34:28.278 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:28.278 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:28.278 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:28.278 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 08:34:28.278 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:28.278 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260507 08:34:28.278 | INFO | PRE_HQ : 58.8%, 1.8, 54.1ms, 0 | 20260507 08:34:28.278 | INFO | BAZWRITER : 61.3%, 3.1, 98.2ms, 0 | 20260507 08:34:28.278 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:34:28.279 | INFO | Uploaded 36GiB to GPU in 2.04833s (17.5753GiB/s) | 20260507 08:34:28.279 | INFO | Downloaded 38GiB to GPU in 2.17611s (17.4623GiB/s) | 20260507 08:34:33.414 | INFO | Analyzing chunk frames = [8565248,8565760) | 20260507 08:34:33.414 | INFO | MemInfo: { | 20260507 08:34:33.414 | INFO | "gpu" : " 17 GB", | 20260507 08:34:33.414 | INFO | "huge" : "", | 20260507 08:34:33.414 | INFO | "huge_shared" : " 27 GB", | 20260507 08:34:33.414 | INFO | "malloc" : " 14 GB", | 20260507 08:34:33.414 | INFO | "pinned" : " 29 GB" | 20260507 08:34:33.414 | INFO | } | 20260507 08:34:33.414 | INFO | Perf report for the last 5.13576 seconds: | 20260507 08:34:33.414 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:34:33.414 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:33.414 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:33.414 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:33.414 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 08:34:33.414 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:33.414 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:33.414 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:34:33.414 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:33.414 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:33.414 | INFO | Uploaded 36GiB to GPU in 1.99041s (18.0867GiB/s) | 20260507 08:34:33.414 | INFO | Downloaded 34GiB to GPU in 1.94242s (17.504GiB/s) | 20260507 08:34:38.551 | INFO | Analyzing chunk frames = [8565760,8566272) | 20260507 08:34:38.551 | INFO | MemInfo: { | 20260507 08:34:38.551 | INFO | "gpu" : " 17 GB", | 20260507 08:34:38.551 | INFO | "huge" : "", | 20260507 08:34:38.551 | INFO | "huge_shared" : " 27 GB", | 20260507 08:34:38.551 | INFO | "malloc" : " 14 GB", | 20260507 08:34:38.551 | INFO | "pinned" : " 29 GB" | 20260507 08:34:38.551 | INFO | } | 20260507 08:34:38.551 | INFO | Perf report for the last 5.13675 seconds: | 20260507 08:34:38.551 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:34:38.551 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:38.551 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:38.551 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:38.551 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 08:34:38.551 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:38.551 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:38.551 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:34:38.551 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:38.551 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:38.551 | INFO | Uploaded 36GiB to GPU in 2.02037s (17.8185GiB/s) | 20260507 08:34:38.551 | INFO | Downloaded 34GiB to GPU in 1.92446s (17.6673GiB/s) | 20260507 08:34:43.690 | INFO | Analyzing chunk frames = [8566272,8566784) | 20260507 08:34:43.690 | INFO | MemInfo: { | 20260507 08:34:43.690 | INFO | "gpu" : " 17 GB", | 20260507 08:34:43.690 | INFO | "huge" : "", | 20260507 08:34:43.690 | INFO | "huge_shared" : " 27 GB", | 20260507 08:34:43.690 | INFO | "malloc" : " 14 GB", | 20260507 08:34:43.690 | INFO | "pinned" : " 29 GB" | 20260507 08:34:43.690 | INFO | } | 20260507 08:34:43.690 | INFO | Perf report for the last 5.13965 seconds: | 20260507 08:34:43.690 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:34:43.690 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:43.690 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:43.690 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:43.691 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 08:34:43.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:43.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:43.691 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:34:43.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:43.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:43.691 | INFO | Uploaded 36GiB to GPU in 2.008s (17.9282GiB/s) | 20260507 08:34:43.691 | INFO | Downloaded 34GiB to GPU in 1.94523s (17.4786GiB/s) | 20260507 08:34:48.831 | INFO | Analyzing chunk frames = [8566784,8567296) | 20260507 08:34:48.831 | INFO | MemInfo: { | 20260507 08:34:48.831 | INFO | "gpu" : " 17 GB", | 20260507 08:34:48.831 | INFO | "huge" : "", | 20260507 08:34:48.831 | INFO | "huge_shared" : " 27 GB", | 20260507 08:34:48.831 | INFO | "malloc" : " 14 GB", | 20260507 08:34:48.831 | INFO | "pinned" : " 29 GB" | 20260507 08:34:48.831 | INFO | } | 20260507 08:34:48.831 | INFO | Perf report for the last 5.14057 seconds: | 20260507 08:34:48.831 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:34:48.831 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:48.831 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:48.831 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:48.831 | INFO | ANALYSIS : 60.2%, 3.0, 93.8ms, 0 | 20260507 08:34:48.831 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:48.831 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:48.831 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:34:48.831 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:48.831 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:48.831 | INFO | Uploaded 36GiB to GPU in 2.03069s (17.7279GiB/s) | 20260507 08:34:48.831 | INFO | Downloaded 34GiB to GPU in 1.9235s (17.6761GiB/s) | 20260507 08:34:53.968 | INFO | Analyzing chunk frames = [8567296,8567808) | 20260507 08:34:53.968 | INFO | MemInfo: { | 20260507 08:34:53.968 | INFO | "gpu" : " 17 GB", | 20260507 08:34:53.968 | INFO | "huge" : "", | 20260507 08:34:53.968 | INFO | "huge_shared" : " 27 GB", | 20260507 08:34:53.968 | INFO | "malloc" : " 14 GB", | 20260507 08:34:53.968 | INFO | "pinned" : " 29 GB" | 20260507 08:34:53.968 | INFO | } | 20260507 08:34:53.968 | INFO | Perf report for the last 5.13727 seconds: | 20260507 08:34:53.968 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:34:53.968 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:53.968 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:53.968 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:53.968 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 08:34:53.968 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:53.968 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:53.968 | INFO | PRE_HQ : 27.0%, 1.0, 14.0ms, 0 | 20260507 08:34:53.968 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:53.968 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:53.969 | INFO | Uploaded 36GiB to GPU in 2.0602s (17.4741GiB/s) | 20260507 08:34:53.969 | INFO | Downloaded 34GiB to GPU in 1.9353s (17.5683GiB/s) | 20260507 08:34:59.120 | INFO | Analyzing chunk frames = [8567808,8568320) | 20260507 08:34:59.120 | INFO | MemInfo: { | 20260507 08:34:59.120 | INFO | "gpu" : " 17 GB", | 20260507 08:34:59.120 | INFO | "huge" : "", | 20260507 08:34:59.120 | INFO | "huge_shared" : " 27 GB", | 20260507 08:34:59.120 | INFO | "malloc" : " 14 GB", | 20260507 08:34:59.121 | INFO | "pinned" : " 29 GB" | 20260507 08:34:59.121 | INFO | } | 20260507 08:34:59.121 | INFO | Perf report for the last 5.15313 seconds: | 20260507 08:34:59.121 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:34:59.121 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:59.121 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:59.122 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:59.122 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 08:34:59.122 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:59.122 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:59.122 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 08:34:59.122 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:59.122 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:34:59.122 | INFO | Uploaded 36GiB to GPU in 1.90967s (18.8514GiB/s) | 20260507 08:34:59.122 | INFO | Downloaded 34GiB to GPU in 1.97885s (17.1817GiB/s) | 20260507 08:35:04.249 | INFO | Analyzing chunk frames = [8568320,8568832) | 20260507 08:35:04.249 | INFO | MemInfo: { | 20260507 08:35:04.249 | INFO | "gpu" : " 17 GB", | 20260507 08:35:04.249 | INFO | "huge" : "", | 20260507 08:35:04.249 | INFO | "huge_shared" : " 27 GB", | 20260507 08:35:04.249 | INFO | "malloc" : " 14 GB", | 20260507 08:35:04.249 | INFO | "pinned" : " 29 GB" | 20260507 08:35:04.249 | INFO | } | 20260507 08:35:04.249 | INFO | Perf report for the last 5.12742 seconds: | 20260507 08:35:04.249 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:35:04.249 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:04.249 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:04.249 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:04.249 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 08:35:04.249 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:04.249 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:04.249 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 08:35:04.249 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:04.249 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:04.249 | INFO | Uploaded 36GiB to GPU in 1.96583s (18.3129GiB/s) | 20260507 08:35:04.249 | INFO | Downloaded 34GiB to GPU in 1.95038s (17.4325GiB/s) | 20260507 08:35:04.249 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:35:04.249 | INFO | Total mem usage: 18 GiB | 20260507 08:35:04.249 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:35:04.249 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:35:04.249 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:35:04.249 | INFO | High Water Breakdown: | 20260507 08:35:04.249 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:35:04.249 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:35:04.249 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:35:04.249 | INFO | ------------------------------------------ | 20260507 08:35:04.249 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:35:04.249 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:35:04.249 | INFO | Total mem usage: 26.5 GiB | 20260507 08:35:04.249 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:35:04.249 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:35:04.250 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:35:04.250 | INFO | High Water Breakdown: | 20260507 08:35:04.250 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:35:04.250 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:35:04.250 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:35:04.250 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:35:04.250 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:35:04.250 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:35:04.250 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:35:04.250 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:35:04.250 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:35:04.250 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:35:04.250 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:35:04.250 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:35:04.250 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:35:04.250 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:35:04.250 | INFO | ------------------------------------------ | 20260507 08:35:04.250 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:35:04.250 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:35:04.250 | INFO | Total mem usage: 25.6 GiB | 20260507 08:35:04.250 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:35:04.250 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:35:04.250 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:35:04.250 | INFO | High Water Breakdown: | 20260507 08:35:04.250 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:35:04.250 | INFO | ------------------------------------------ | 20260507 08:35:04.250 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:35:04.250 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:35:04.250 | INFO | Total mem usage: 15.7 GiB | 20260507 08:35:04.250 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:35:04.250 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:35:04.250 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:35:04.250 | INFO | High Water Breakdown: | 20260507 08:35:04.250 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:35:04.250 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:35:04.250 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:35:04.250 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:35:04.250 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:35:04.250 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:35:04.250 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:35:04.250 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:35:04.250 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:35:04.250 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:35:04.250 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:35:04.250 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:35:04.250 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:35:04.250 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:35:04.250 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:35:04.250 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:35:04.250 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:35:04.250 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:35:04.250 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:35:04.250 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:35:04.250 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:35:04.250 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:35:04.250 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:35:04.250 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:35:04.250 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:35:04.250 | INFO | ------------------------------------------ | 20260507 08:35:04.250 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:35:04.250 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:35:04.250 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:35:04.250 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:35:04.250 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.46e+01, 1.34e+01 / 1.24e+00 | 20260507 08:35:04.250 | INFO | PerfReport: 0.29%, AggregateStats -- 9.03e-02 / 1.02e-01, 9.31e-03 / 4.83e-03 | 20260507 08:35:04.250 | INFO | PerfReport: 11.67%, Baseline -- 3.58e+00 / 4.45e+00, 5.12e-01 / 2.45e-01 | 20260507 08:35:04.250 | INFO | PerfReport: 3.23%, Histogram -- 9.93e-01 / 1.49e+00, 1.71e-01 / 1.06e-01 | 20260507 08:35:04.250 | INFO | PerfReport: 16.25%, DME -- 3.99e+01 / 4.45e+01, 3.91e+00 / 1.47e+00 | 20260507 08:35:04.250 | INFO | PerfReport: 57.83%, FrameLabeling -- 1.78e+01 / 1.98e+01, 1.35e+00 / 1.54e+00 | 20260507 08:35:04.250 | INFO | PerfReport: 6.85%, PulseAccumulating -- 2.10e+00 / 2.68e+00, 3.83e-01 / 1.96e-01 | 20260507 08:35:04.250 | INFO | PerfReport: 2.90%, Metrics -- 8.90e-01 / 1.19e+00, 2.39e-01 / 1.50e-01 | 20260507 08:35:04.250 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.99e-01 / 3.58e-01, 5.05e-02 / 2.03e-02 | 20260507 08:35:04.250 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:35:04.250 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:35:04.250 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:35:04.250 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.40e+01 / 0.00e+00 | 20260507 08:35:04.250 | INFO | PerfReport: 21.78%, Upload -- 2.09e+01 / 0.00e+00, 7.09e+00 / 0.00e+00 | 20260507 08:35:04.251 | INFO | PerfReport: 21.41%, Download -- 2.06e+01 / 0.00e+00, 7.36e+00 / 0.00e+00 | 20260507 08:35:04.251 | INFO | PerfReport: 31.96%, Compute -- 3.07e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 08:35:04.251 | INFO | PerfReport: 1.53%, LockUpload -- 1.47e+00 / 0.00e+00, 4.88e+00 / 0.00e+00 | 20260507 08:35:04.251 | INFO | PerfReport: 1.28%, LockDownload -- 1.23e+00 / 0.00e+00, 3.09e+00 / 0.00e+00 | 20260507 08:35:04.251 | INFO | PerfReport: 22.02%, LockCompute -- 2.12e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 08:35:04.251 | INFO | PerfReport: 0.01%, Misc -- 1.29e-02 / 0.00e+00, 1.47e-02 / 0.00e+00 | 20260507 08:35:04.346 | INFO | Starting a round of Baz IO | 20260507 08:35:07.438 | INFO | Populating RTMetrics report | 20260507 08:35:07.438 | INFO | RTMetrics report populated | 20260507 08:35:07.438 | INFO | Opening tmp RTMetrics json file | 20260507 08:35:07.452 | INFO | Serializing RTMetrics to json | 20260507 08:35:07.453 | INFO | Serialization done, swapping tmp file now... | 20260507 08:35:07.454 | INFO | RTMetrics done | 20260507 08:35:07.525 | INFO | Finished a round of Baz IO: Wrote 1.611 GiB in 3.179 seconds | 20260507 08:35:07.526 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.49% / 0.00% / 67.51% | 20260507 08:35:07.526 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0891288% / 0% / 4.76837e-05% | 20260507 08:35:07.526 | INFO | Baz IO overall: | 20260507 08:35:07.526 | INFO | BytesWritten : 4264437520463 | 20260507 08:35:07.526 | INFO | NumEvents : 2039555360846 | 20260507 08:35:09.389 | INFO | Analyzing chunk frames = [8568832,8569344) | 20260507 08:35:09.389 | INFO | MemInfo: { | 20260507 08:35:09.389 | INFO | "gpu" : " 17 GB", | 20260507 08:35:09.389 | INFO | "huge" : "", | 20260507 08:35:09.389 | INFO | "huge_shared" : " 27 GB", | 20260507 08:35:09.389 | INFO | "malloc" : " 14 GB", | 20260507 08:35:09.389 | INFO | "pinned" : " 29 GB" | 20260507 08:35:09.389 | INFO | } | 20260507 08:35:09.389 | INFO | Perf report for the last 5.1401 seconds: | 20260507 08:35:09.389 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:35:09.389 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:09.389 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:09.389 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:09.389 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 08:35:09.389 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:09.389 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260507 08:35:09.389 | INFO | PRE_HQ : 58.9%, 1.8, 54.6ms, 0 | 20260507 08:35:09.389 | INFO | BAZWRITER : 61.4%, 3.1, 96.7ms, 0 | 20260507 08:35:09.389 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:35:09.389 | INFO | Uploaded 36GiB to GPU in 2.06722s (17.4147GiB/s) | 20260507 08:35:09.389 | INFO | Downloaded 38GiB to GPU in 2.17318s (17.4859GiB/s) | 20260507 08:35:14.530 | INFO | Analyzing chunk frames = [8569344,8569856) | 20260507 08:35:14.530 | INFO | MemInfo: { | 20260507 08:35:14.530 | INFO | "gpu" : " 17 GB", | 20260507 08:35:14.530 | INFO | "huge" : "", | 20260507 08:35:14.530 | INFO | "huge_shared" : " 27 GB", | 20260507 08:35:14.530 | INFO | "malloc" : " 14 GB", | 20260507 08:35:14.530 | INFO | "pinned" : " 29 GB" | 20260507 08:35:14.530 | INFO | } | 20260507 08:35:14.530 | INFO | Perf report for the last 5.14103 seconds: | 20260507 08:35:14.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:35:14.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:14.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:14.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:14.530 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 08:35:14.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:14.530 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:14.530 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:35:14.530 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:14.530 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:14.530 | INFO | Uploaded 36GiB to GPU in 1.99074s (18.0837GiB/s) | 20260507 08:35:14.530 | INFO | Downloaded 34GiB to GPU in 1.93485s (17.5724GiB/s) | 20260507 08:35:19.666 | INFO | Analyzing chunk frames = [8569856,8570368) | 20260507 08:35:19.666 | INFO | MemInfo: { | 20260507 08:35:19.666 | INFO | "gpu" : " 17 GB", | 20260507 08:35:19.666 | INFO | "huge" : "", | 20260507 08:35:19.666 | INFO | "huge_shared" : " 27 GB", | 20260507 08:35:19.667 | INFO | "malloc" : " 14 GB", | 20260507 08:35:19.667 | INFO | "pinned" : " 29 GB" | 20260507 08:35:19.667 | INFO | } | 20260507 08:35:19.667 | INFO | Perf report for the last 5.13653 seconds: | 20260507 08:35:19.667 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:35:19.667 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:19.667 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:19.667 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:19.667 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 08:35:19.667 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:19.667 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:19.667 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:35:19.667 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:19.667 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:19.667 | INFO | Uploaded 36GiB to GPU in 2.05658s (17.5048GiB/s) | 20260507 08:35:19.667 | INFO | Downloaded 34GiB to GPU in 1.9511s (17.426GiB/s) | 20260507 08:35:24.803 | INFO | Analyzing chunk frames = [8570368,8570880) | 20260507 08:35:24.803 | INFO | MemInfo: { | 20260507 08:35:24.803 | INFO | "gpu" : " 17 GB", | 20260507 08:35:24.803 | INFO | "huge" : "", | 20260507 08:35:24.803 | INFO | "huge_shared" : " 27 GB", | 20260507 08:35:24.803 | INFO | "malloc" : " 14 GB", | 20260507 08:35:24.803 | INFO | "pinned" : " 29 GB" | 20260507 08:35:24.803 | INFO | } | 20260507 08:35:24.803 | INFO | Perf report for the last 5.13633 seconds: | 20260507 08:35:24.803 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:35:24.803 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:24.803 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:24.803 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:24.803 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 08:35:24.803 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:24.803 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:24.803 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:35:24.803 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:24.803 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:24.803 | INFO | Uploaded 36GiB to GPU in 2.0546s (17.5217GiB/s) | 20260507 08:35:24.803 | INFO | Downloaded 34GiB to GPU in 1.94572s (17.4742GiB/s) | 20260507 08:35:29.944 | INFO | Analyzing chunk frames = [8570880,8571392) | 20260507 08:35:29.944 | INFO | MemInfo: { | 20260507 08:35:29.944 | INFO | "gpu" : " 17 GB", | 20260507 08:35:29.944 | INFO | "huge" : "", | 20260507 08:35:29.944 | INFO | "huge_shared" : " 27 GB", | 20260507 08:35:29.944 | INFO | "malloc" : " 14 GB", | 20260507 08:35:29.944 | INFO | "pinned" : " 29 GB" | 20260507 08:35:29.944 | INFO | } | 20260507 08:35:29.944 | INFO | Perf report for the last 5.14124 seconds: | 20260507 08:35:29.944 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:35:29.944 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:29.944 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:29.944 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:29.944 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 08:35:29.944 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:29.944 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:29.944 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 08:35:29.944 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:29.944 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:29.944 | INFO | Uploaded 36GiB to GPU in 2.04174s (17.6321GiB/s) | 20260507 08:35:29.944 | INFO | Downloaded 34GiB to GPU in 1.91417s (17.7622GiB/s) | 20260507 08:35:35.088 | INFO | Analyzing chunk frames = [8571392,8571904) | 20260507 08:35:35.088 | INFO | MemInfo: { | 20260507 08:35:35.088 | INFO | "gpu" : " 17 GB", | 20260507 08:35:35.088 | INFO | "huge" : "", | 20260507 08:35:35.088 | INFO | "huge_shared" : " 27 GB", | 20260507 08:35:35.088 | INFO | "malloc" : " 14 GB", | 20260507 08:35:35.088 | INFO | "pinned" : " 29 GB" | 20260507 08:35:35.088 | INFO | } | 20260507 08:35:35.088 | INFO | Perf report for the last 5.14407 seconds: | 20260507 08:35:35.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:35:35.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:35.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:35.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:35.088 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 08:35:35.088 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:35.088 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:35.088 | INFO | PRE_HQ : 26.3%, 1.0, 13.7ms, 0 | 20260507 08:35:35.088 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:35.088 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:35.088 | INFO | Uploaded 36GiB to GPU in 2.06879s (17.4015GiB/s) | 20260507 08:35:35.089 | INFO | Downloaded 34GiB to GPU in 1.92487s (17.6636GiB/s) | 20260507 08:35:40.230 | INFO | Analyzing chunk frames = [8571904,8572416) | 20260507 08:35:40.230 | INFO | MemInfo: { | 20260507 08:35:40.230 | INFO | "gpu" : " 17 GB", | 20260507 08:35:40.230 | INFO | "huge" : "", | 20260507 08:35:40.230 | INFO | "huge_shared" : " 27 GB", | 20260507 08:35:40.230 | INFO | "malloc" : " 14 GB", | 20260507 08:35:40.230 | INFO | "pinned" : " 29 GB" | 20260507 08:35:40.230 | INFO | } | 20260507 08:35:40.230 | INFO | Perf report for the last 5.14213 seconds: | 20260507 08:35:40.230 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:35:40.230 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:40.230 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:40.230 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:40.230 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 08:35:40.230 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:40.231 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:40.231 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 08:35:40.231 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:40.231 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:40.231 | INFO | Uploaded 36GiB to GPU in 1.89438s (19.0036GiB/s) | 20260507 08:35:40.231 | INFO | Downloaded 34GiB to GPU in 1.97614s (17.2053GiB/s) | 20260507 08:35:45.369 | INFO | Analyzing chunk frames = [8572416,8572928) | 20260507 08:35:45.369 | INFO | MemInfo: { | 20260507 08:35:45.369 | INFO | "gpu" : " 17 GB", | 20260507 08:35:45.369 | INFO | "huge" : "", | 20260507 08:35:45.369 | INFO | "huge_shared" : " 27 GB", | 20260507 08:35:45.369 | INFO | "malloc" : " 14 GB", | 20260507 08:35:45.369 | INFO | "pinned" : " 29 GB" | 20260507 08:35:45.369 | INFO | } | 20260507 08:35:45.369 | INFO | Perf report for the last 5.13844 seconds: | 20260507 08:35:45.369 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:35:45.369 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:45.369 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:45.369 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:45.369 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 08:35:45.369 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:45.369 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:45.369 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:35:45.369 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:45.369 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:45.369 | INFO | Uploaded 36GiB to GPU in 2.0078s (17.9301GiB/s) | 20260507 08:35:45.369 | INFO | Downloaded 34GiB to GPU in 1.96852s (17.2719GiB/s) | 20260507 08:35:45.466 | INFO | Starting a round of Baz IO | 20260507 08:35:48.560 | INFO | Populating RTMetrics report | 20260507 08:35:48.561 | INFO | RTMetrics report populated | 20260507 08:35:48.561 | INFO | Opening tmp RTMetrics json file | 20260507 08:35:48.569 | INFO | Serializing RTMetrics to json | 20260507 08:35:48.577 | INFO | Serialization done, swapping tmp file now... | 20260507 08:35:48.578 | INFO | RTMetrics done | 20260507 08:35:48.647 | INFO | Finished a round of Baz IO: Wrote 1.61 GiB in 3.18 seconds | 20260507 08:35:48.647 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.49% / 0.00% / 67.51% | 20260507 08:35:48.647 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0888268% / 3.97364e-06% / 7.54992e-05% | 20260507 08:35:48.647 | INFO | Baz IO overall: | 20260507 08:35:48.647 | INFO | BytesWritten : 4266166438057 | 20260507 08:35:48.647 | INFO | NumEvents : 2040360903417 | 20260507 08:35:50.497 | INFO | Analyzing chunk frames = [8572928,8573440) | 20260507 08:35:50.497 | INFO | MemInfo: { | 20260507 08:35:50.497 | INFO | "gpu" : " 17 GB", | 20260507 08:35:50.497 | INFO | "huge" : "", | 20260507 08:35:50.497 | INFO | "huge_shared" : " 27 GB", | 20260507 08:35:50.497 | INFO | "malloc" : " 14 GB", | 20260507 08:35:50.497 | INFO | "pinned" : " 29 GB" | 20260507 08:35:50.497 | INFO | } | 20260507 08:35:50.497 | INFO | Perf report for the last 5.12832 seconds: | 20260507 08:35:50.497 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:35:50.497 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:50.497 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:50.497 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:50.497 | INFO | ANALYSIS : 61.4%, 3.0, 95.7ms, 0 | 20260507 08:35:50.497 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:50.497 | INFO | RT_METRICS : 40.9%, 1.0, 21.4ms, 0 | 20260507 08:35:50.497 | INFO | PRE_HQ : 58.6%, 1.8, 54.4ms, 0 | 20260507 08:35:50.497 | INFO | BAZWRITER : 61.5%, 3.1, 96.5ms, 0 | 20260507 08:35:50.497 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 08:35:50.497 | INFO | Uploaded 36GiB to GPU in 2.05313s (17.5342GiB/s) | 20260507 08:35:50.498 | INFO | Downloaded 38GiB to GPU in 2.16641s (17.5405GiB/s) | 20260507 08:35:55.643 | INFO | Analyzing chunk frames = [8573440,8573952) | 20260507 08:35:55.643 | INFO | MemInfo: { | 20260507 08:35:55.643 | INFO | "gpu" : " 17 GB", | 20260507 08:35:55.643 | INFO | "huge" : "", | 20260507 08:35:55.643 | INFO | "huge_shared" : " 27 GB", | 20260507 08:35:55.644 | INFO | "malloc" : " 14 GB", | 20260507 08:35:55.644 | INFO | "pinned" : " 29 GB" | 20260507 08:35:55.644 | INFO | } | 20260507 08:35:55.644 | INFO | Perf report for the last 5.14652 seconds: | 20260507 08:35:55.644 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:35:55.644 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:55.644 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:55.644 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:55.644 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 08:35:55.644 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:55.644 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:55.644 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260507 08:35:55.644 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:55.644 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:35:55.644 | INFO | Uploaded 36GiB to GPU in 1.9976s (18.0216GiB/s) | 20260507 08:35:55.644 | INFO | Downloaded 34GiB to GPU in 1.93034s (17.6135GiB/s) | 20260507 08:36:00.780 | INFO | Analyzing chunk frames = [8573952,8574464) | 20260507 08:36:00.780 | INFO | MemInfo: { | 20260507 08:36:00.780 | INFO | "gpu" : " 17 GB", | 20260507 08:36:00.780 | INFO | "huge" : "", | 20260507 08:36:00.780 | INFO | "huge_shared" : " 27 GB", | 20260507 08:36:00.780 | INFO | "malloc" : " 14 GB", | 20260507 08:36:00.780 | INFO | "pinned" : " 29 GB" | 20260507 08:36:00.780 | INFO | } | 20260507 08:36:00.781 | INFO | Perf report for the last 5.13677 seconds: | 20260507 08:36:00.781 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:36:00.781 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:00.781 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:00.781 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:00.781 | INFO | ANALYSIS : 60.7%, 3.0, 93.9ms, 0 | 20260507 08:36:00.781 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:00.781 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:00.781 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:36:00.781 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:00.781 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:00.781 | INFO | Uploaded 36GiB to GPU in 2.0558s (17.5114GiB/s) | 20260507 08:36:00.781 | INFO | Downloaded 34GiB to GPU in 1.95095s (17.4274GiB/s) | 20260507 08:36:05.923 | INFO | Analyzing chunk frames = [8574464,8574976) | 20260507 08:36:05.923 | INFO | MemInfo: { | 20260507 08:36:05.923 | INFO | "gpu" : " 17 GB", | 20260507 08:36:05.924 | INFO | "huge" : "", | 20260507 08:36:05.924 | INFO | "huge_shared" : " 27 GB", | 20260507 08:36:05.924 | INFO | "malloc" : " 14 GB", | 20260507 08:36:05.924 | INFO | "pinned" : " 29 GB" | 20260507 08:36:05.924 | INFO | } | 20260507 08:36:05.924 | INFO | Perf report for the last 5.14323 seconds: | 20260507 08:36:05.924 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:36:05.924 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:05.924 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:05.924 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:05.924 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 08:36:05.924 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:05.924 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:05.924 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 08:36:05.924 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:05.924 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:05.924 | INFO | Uploaded 36GiB to GPU in 2.04393s (17.6131GiB/s) | 20260507 08:36:05.924 | INFO | Downloaded 34GiB to GPU in 1.94935s (17.4417GiB/s) | 20260507 08:36:11.065 | INFO | Analyzing chunk frames = [8574976,8575488) | 20260507 08:36:11.065 | INFO | MemInfo: { | 20260507 08:36:11.065 | INFO | "gpu" : " 17 GB", | 20260507 08:36:11.065 | INFO | "huge" : "", | 20260507 08:36:11.065 | INFO | "huge_shared" : " 27 GB", | 20260507 08:36:11.065 | INFO | "malloc" : " 14 GB", | 20260507 08:36:11.065 | INFO | "pinned" : " 29 GB" | 20260507 08:36:11.065 | INFO | } | 20260507 08:36:11.065 | INFO | Perf report for the last 5.14147 seconds: | 20260507 08:36:11.065 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:36:11.065 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:11.065 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:11.065 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:11.065 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 08:36:11.065 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:11.065 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:11.065 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 08:36:11.065 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:11.065 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:11.066 | INFO | Uploaded 36GiB to GPU in 2.04717s (17.5852GiB/s) | 20260507 08:36:11.066 | INFO | Downloaded 34GiB to GPU in 1.9216s (17.6936GiB/s) | 20260507 08:36:16.200 | INFO | Analyzing chunk frames = [8575488,8576000) | 20260507 08:36:16.200 | INFO | MemInfo: { | 20260507 08:36:16.200 | INFO | "gpu" : " 17 GB", | 20260507 08:36:16.200 | INFO | "huge" : "", | 20260507 08:36:16.200 | INFO | "huge_shared" : " 27 GB", | 20260507 08:36:16.200 | INFO | "malloc" : " 14 GB", | 20260507 08:36:16.201 | INFO | "pinned" : " 29 GB" | 20260507 08:36:16.201 | INFO | } | 20260507 08:36:16.201 | INFO | Perf report for the last 5.13534 seconds: | 20260507 08:36:16.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:36:16.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:16.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:16.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:16.201 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 08:36:16.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:16.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:16.201 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:36:16.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:16.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:16.201 | INFO | Uploaded 36GiB to GPU in 2.07053s (17.3869GiB/s) | 20260507 08:36:16.201 | INFO | Downloaded 34GiB to GPU in 1.94344s (17.4948GiB/s) | 20260507 08:36:21.345 | INFO | Analyzing chunk frames = [8576000,8576512) | 20260507 08:36:21.345 | INFO | MemInfo: { | 20260507 08:36:21.345 | INFO | "gpu" : " 17 GB", | 20260507 08:36:21.345 | INFO | "huge" : "", | 20260507 08:36:21.345 | INFO | "huge_shared" : " 27 GB", | 20260507 08:36:21.345 | INFO | "malloc" : " 14 GB", | 20260507 08:36:21.345 | INFO | "pinned" : " 29 GB" | 20260507 08:36:21.345 | INFO | } | 20260507 08:36:21.345 | INFO | Perf report for the last 5.14448 seconds: | 20260507 08:36:21.345 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:36:21.345 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:21.345 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:21.345 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:21.345 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 08:36:21.345 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:21.345 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:21.345 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:36:21.345 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:21.345 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:21.345 | INFO | Uploaded 36GiB to GPU in 1.90633s (18.8844GiB/s) | 20260507 08:36:21.345 | INFO | Downloaded 34GiB to GPU in 1.97092s (17.2508GiB/s) | 20260507 08:36:26.482 | INFO | Analyzing chunk frames = [8576512,8577024) | 20260507 08:36:26.482 | INFO | MemInfo: { | 20260507 08:36:26.482 | INFO | "gpu" : " 17 GB", | 20260507 08:36:26.482 | INFO | "huge" : "", | 20260507 08:36:26.482 | INFO | "huge_shared" : " 27 GB", | 20260507 08:36:26.482 | INFO | "malloc" : " 14 GB", | 20260507 08:36:26.482 | INFO | "pinned" : " 29 GB" | 20260507 08:36:26.482 | INFO | } | 20260507 08:36:26.482 | INFO | Perf report for the last 5.13724 seconds: | 20260507 08:36:26.482 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:36:26.482 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:26.482 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:26.482 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:26.482 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:36:26.482 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:26.482 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:26.482 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:36:26.482 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:26.482 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:26.483 | INFO | Uploaded 36GiB to GPU in 1.98647s (18.1226GiB/s) | 20260507 08:36:26.483 | INFO | Downloaded 34GiB to GPU in 1.96684s (17.2866GiB/s) | 20260507 08:36:26.483 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:36:26.483 | INFO | Total mem usage: 18 GiB | 20260507 08:36:26.483 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:36:26.483 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:36:26.483 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:36:26.483 | INFO | High Water Breakdown: | 20260507 08:36:26.483 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:36:26.483 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:36:26.483 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:36:26.483 | INFO | ------------------------------------------ | 20260507 08:36:26.483 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:36:26.483 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:36:26.483 | INFO | Total mem usage: 26.5 GiB | 20260507 08:36:26.483 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:36:26.483 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:36:26.483 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:36:26.483 | INFO | High Water Breakdown: | 20260507 08:36:26.483 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:36:26.483 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:36:26.483 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:36:26.483 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:36:26.483 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:36:26.483 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:36:26.483 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:36:26.483 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:36:26.483 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:36:26.483 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:36:26.483 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:36:26.483 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:36:26.483 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:36:26.483 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:36:26.483 | INFO | ------------------------------------------ | 20260507 08:36:26.483 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:36:26.483 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:36:26.483 | INFO | Total mem usage: 25.6 GiB | 20260507 08:36:26.483 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:36:26.483 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:36:26.483 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:36:26.483 | INFO | High Water Breakdown: | 20260507 08:36:26.483 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:36:26.483 | INFO | ------------------------------------------ | 20260507 08:36:26.483 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:36:26.483 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:36:26.483 | INFO | Total mem usage: 15.7 GiB | 20260507 08:36:26.483 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:36:26.483 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:36:26.483 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:36:26.483 | INFO | High Water Breakdown: | 20260507 08:36:26.483 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:36:26.484 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:36:26.484 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:36:26.484 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:36:26.484 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:36:26.484 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:36:26.484 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:36:26.484 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:36:26.484 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:36:26.484 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:36:26.484 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:36:26.484 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:36:26.484 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:36:26.484 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:36:26.484 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:36:26.484 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:36:26.484 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:36:26.484 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:36:26.484 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:36:26.484 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:36:26.484 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:36:26.484 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:36:26.484 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:36:26.484 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:36:26.484 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:36:26.484 | INFO | ------------------------------------------ | 20260507 08:36:26.484 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:36:26.484 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:36:26.484 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:36:26.484 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:36:26.484 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.48e+01, 1.32e+01 / 1.27e+00 | 20260507 08:36:26.484 | INFO | PerfReport: 0.29%, AggregateStats -- 9.04e-02 / 1.01e-01, 1.00e-02 / 4.33e-03 | 20260507 08:36:26.484 | INFO | PerfReport: 11.63%, Baseline -- 3.56e+00 / 4.43e+00, 4.91e-01 / 3.00e-01 | 20260507 08:36:26.484 | INFO | PerfReport: 3.20%, Histogram -- 9.82e-01 / 1.51e+00, 1.62e-01 / 8.92e-02 | 20260507 08:36:26.484 | INFO | PerfReport: 16.40%, DME -- 4.02e+01 / 4.52e+01, 4.03e+00 / 1.50e+00 | 20260507 08:36:26.484 | INFO | PerfReport: 57.70%, FrameLabeling -- 1.77e+01 / 1.93e+01, 1.22e+00 / 1.12e+00 | 20260507 08:36:26.484 | INFO | PerfReport: 6.90%, PulseAccumulating -- 2.11e+00 / 2.72e+00, 3.88e-01 / 1.82e-01 | 20260507 08:36:26.484 | INFO | PerfReport: 2.90%, Metrics -- 8.90e-01 / 1.20e+00, 2.35e-01 / 1.07e-01 | 20260507 08:36:26.484 | INFO | PerfReport: 0.98%, PulseSerializing -- 2.99e-01 / 3.63e-01, 4.95e-02 / 2.17e-02 | 20260507 08:36:26.484 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:36:26.484 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:36:26.484 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:36:26.484 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.45e+01 / 0.00e+00 | 20260507 08:36:26.484 | INFO | PerfReport: 21.89%, Upload -- 2.11e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260507 08:36:26.484 | INFO | PerfReport: 21.38%, Download -- 2.06e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 08:36:26.484 | INFO | PerfReport: 31.86%, Compute -- 3.06e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 08:36:26.484 | INFO | PerfReport: 1.73%, LockUpload -- 1.66e+00 / 0.00e+00, 5.12e+00 / 0.00e+00 | 20260507 08:36:26.484 | INFO | PerfReport: 1.18%, LockDownload -- 1.13e+00 / 0.00e+00, 2.82e+00 / 0.00e+00 | 20260507 08:36:26.484 | INFO | PerfReport: 21.95%, LockCompute -- 2.11e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 08:36:26.484 | INFO | PerfReport: 0.01%, Misc -- 1.33e-02 / 0.00e+00, 1.75e-02 / 0.00e+00 | 20260507 08:36:26.580 | INFO | Starting a round of Baz IO | 20260507 08:36:29.677 | INFO | Populating RTMetrics report | 20260507 08:36:29.677 | INFO | RTMetrics report populated | 20260507 08:36:29.677 | INFO | Opening tmp RTMetrics json file | 20260507 08:36:29.678 | INFO | Serializing RTMetrics to json | 20260507 08:36:29.679 | INFO | Serialization done, swapping tmp file now... | 20260507 08:36:29.680 | INFO | RTMetrics done | 20260507 08:36:29.763 | INFO | Finished a round of Baz IO: Wrote 1.609 GiB in 3.182 seconds | 20260507 08:36:29.763 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.49% / 0.00% / 67.51% | 20260507 08:36:29.763 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0886917% / 3.97364e-06% / 8.34465e-05% | 20260507 08:36:29.763 | INFO | Baz IO overall: | 20260507 08:36:29.763 | INFO | BytesWritten : 4267893649020 | 20260507 08:36:29.763 | INFO | NumEvents : 2041165384081 | 20260507 08:36:31.616 | INFO | Analyzing chunk frames = [8577024,8577536) | 20260507 08:36:31.616 | INFO | MemInfo: { | 20260507 08:36:31.616 | INFO | "gpu" : " 17 GB", | 20260507 08:36:31.616 | INFO | "huge" : "", | 20260507 08:36:31.616 | INFO | "huge_shared" : " 27 GB", | 20260507 08:36:31.616 | INFO | "malloc" : " 14 GB", | 20260507 08:36:31.616 | INFO | "pinned" : " 29 GB" | 20260507 08:36:31.616 | INFO | } | 20260507 08:36:31.617 | INFO | Perf report for the last 5.13417 seconds: | 20260507 08:36:31.617 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:36:31.617 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:31.617 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:31.617 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:31.617 | INFO | ANALYSIS : 61.7%, 3.0, 95.9ms, 0 | 20260507 08:36:31.617 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:31.617 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 08:36:31.617 | INFO | PRE_HQ : 58.9%, 1.8, 54.3ms, 0 | 20260507 08:36:31.617 | INFO | BAZWRITER : 61.6%, 3.1, 97.3ms, 0 | 20260507 08:36:31.617 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 08:36:31.617 | INFO | Uploaded 36GiB to GPU in 2.06567s (17.4277GiB/s) | 20260507 08:36:31.617 | INFO | Downloaded 38GiB to GPU in 2.17025s (17.5095GiB/s) | 20260507 08:36:36.754 | INFO | Analyzing chunk frames = [8577536,8578048) | 20260507 08:36:36.754 | INFO | MemInfo: { | 20260507 08:36:36.754 | INFO | "gpu" : " 17 GB", | 20260507 08:36:36.754 | INFO | "huge" : "", | 20260507 08:36:36.754 | INFO | "huge_shared" : " 27 GB", | 20260507 08:36:36.754 | INFO | "malloc" : " 14 GB", | 20260507 08:36:36.754 | INFO | "pinned" : " 29 GB" | 20260507 08:36:36.754 | INFO | } | 20260507 08:36:36.754 | INFO | Perf report for the last 5.1377 seconds: | 20260507 08:36:36.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:36:36.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:36.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:36.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:36.754 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 08:36:36.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:36.754 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:36.754 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:36:36.754 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:36.754 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:36.755 | INFO | Uploaded 36GiB to GPU in 1.97251s (18.2508GiB/s) | 20260507 08:36:36.755 | INFO | Downloaded 34GiB to GPU in 1.93155s (17.6025GiB/s) | 20260507 08:36:41.896 | INFO | Analyzing chunk frames = [8578048,8578560) | 20260507 08:36:41.896 | INFO | MemInfo: { | 20260507 08:36:41.896 | INFO | "gpu" : " 17 GB", | 20260507 08:36:41.896 | INFO | "huge" : "", | 20260507 08:36:41.897 | INFO | "huge_shared" : " 27 GB", | 20260507 08:36:41.897 | INFO | "malloc" : " 14 GB", | 20260507 08:36:41.897 | INFO | "pinned" : " 29 GB" | 20260507 08:36:41.897 | INFO | } | 20260507 08:36:41.897 | INFO | Perf report for the last 5.1424 seconds: | 20260507 08:36:41.897 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:36:41.897 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:41.897 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:41.897 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:41.897 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 08:36:41.897 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:41.897 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:41.897 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:36:41.897 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:41.897 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:41.897 | INFO | Uploaded 36GiB to GPU in 2.03676s (17.6751GiB/s) | 20260507 08:36:41.897 | INFO | Downloaded 34GiB to GPU in 1.95663s (17.3768GiB/s) | 20260507 08:36:47.042 | INFO | Analyzing chunk frames = [8578560,8579072) | 20260507 08:36:47.042 | INFO | MemInfo: { | 20260507 08:36:47.042 | INFO | "gpu" : " 17 GB", | 20260507 08:36:47.042 | INFO | "huge" : "", | 20260507 08:36:47.042 | INFO | "huge_shared" : " 27 GB", | 20260507 08:36:47.042 | INFO | "malloc" : " 14 GB", | 20260507 08:36:47.042 | INFO | "pinned" : " 29 GB" | 20260507 08:36:47.042 | INFO | } | 20260507 08:36:47.042 | INFO | Perf report for the last 5.14575 seconds: | 20260507 08:36:47.042 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:36:47.042 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:47.042 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:47.042 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:47.043 | INFO | ANALYSIS : 60.3%, 2.9, 93.3ms, 0 | 20260507 08:36:47.043 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:47.043 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:47.043 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:36:47.043 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:47.043 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:47.043 | INFO | Uploaded 36GiB to GPU in 2.03238s (17.7132GiB/s) | 20260507 08:36:47.043 | INFO | Downloaded 34GiB to GPU in 1.95772s (17.3672GiB/s) | 20260507 08:36:52.180 | INFO | Analyzing chunk frames = [8579072,8579584) | 20260507 08:36:52.180 | INFO | MemInfo: { | 20260507 08:36:52.180 | INFO | "gpu" : " 17 GB", | 20260507 08:36:52.180 | INFO | "huge" : "", | 20260507 08:36:52.180 | INFO | "huge_shared" : " 27 GB", | 20260507 08:36:52.180 | INFO | "malloc" : " 14 GB", | 20260507 08:36:52.180 | INFO | "pinned" : " 29 GB" | 20260507 08:36:52.180 | INFO | } | 20260507 08:36:52.180 | INFO | Perf report for the last 5.13784 seconds: | 20260507 08:36:52.180 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:36:52.180 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:52.180 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:52.180 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:52.180 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 08:36:52.180 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:52.180 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:52.180 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:36:52.180 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:52.180 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:52.180 | INFO | Uploaded 36GiB to GPU in 2.05989s (17.4767GiB/s) | 20260507 08:36:52.181 | INFO | Downloaded 34GiB to GPU in 1.91852s (17.722GiB/s) | 20260507 08:36:57.311 | INFO | Analyzing chunk frames = [8579584,8580096) | 20260507 08:36:57.311 | INFO | MemInfo: { | 20260507 08:36:57.311 | INFO | "gpu" : " 17 GB", | 20260507 08:36:57.311 | INFO | "huge" : "", | 20260507 08:36:57.311 | INFO | "huge_shared" : " 27 GB", | 20260507 08:36:57.311 | INFO | "malloc" : " 14 GB", | 20260507 08:36:57.311 | INFO | "pinned" : " 29 GB" | 20260507 08:36:57.311 | INFO | } | 20260507 08:36:57.311 | INFO | Perf report for the last 5.13082 seconds: | 20260507 08:36:57.311 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:36:57.311 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:57.311 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:57.311 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:57.311 | INFO | ANALYSIS : 60.4%, 3.0, 93.9ms, 0 | 20260507 08:36:57.311 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:57.311 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:57.311 | INFO | PRE_HQ : 26.9%, 1.0, 14.0ms, 0 | 20260507 08:36:57.311 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:57.311 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:36:57.311 | INFO | Uploaded 36GiB to GPU in 2.02137s (17.8097GiB/s) | 20260507 08:36:57.311 | INFO | Downloaded 34GiB to GPU in 1.92193s (17.6906GiB/s) | 20260507 08:37:02.449 | INFO | Analyzing chunk frames = [8580096,8580608) | 20260507 08:37:02.449 | INFO | MemInfo: { | 20260507 08:37:02.449 | INFO | "gpu" : " 17 GB", | 20260507 08:37:02.449 | INFO | "huge" : "", | 20260507 08:37:02.449 | INFO | "huge_shared" : " 27 GB", | 20260507 08:37:02.449 | INFO | "malloc" : " 14 GB", | 20260507 08:37:02.449 | INFO | "pinned" : " 29 GB" | 20260507 08:37:02.449 | INFO | } | 20260507 08:37:02.449 | INFO | Perf report for the last 5.13815 seconds: | 20260507 08:37:02.449 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:37:02.449 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:02.449 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:02.449 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:02.449 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:37:02.449 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:02.449 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:02.449 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 08:37:02.449 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:02.449 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:02.450 | INFO | Uploaded 36GiB to GPU in 1.93251s (18.6287GiB/s) | 20260507 08:37:02.450 | INFO | Downloaded 34GiB to GPU in 1.96921s (17.2658GiB/s) | 20260507 08:37:07.592 | INFO | Analyzing chunk frames = [8580608,8581120) | 20260507 08:37:07.592 | INFO | MemInfo: { | 20260507 08:37:07.592 | INFO | "gpu" : " 17 GB", | 20260507 08:37:07.592 | INFO | "huge" : "", | 20260507 08:37:07.592 | INFO | "huge_shared" : " 27 GB", | 20260507 08:37:07.592 | INFO | "malloc" : " 14 GB", | 20260507 08:37:07.592 | INFO | "pinned" : " 29 GB" | 20260507 08:37:07.593 | INFO | } | 20260507 08:37:07.593 | INFO | Perf report for the last 5.1433 seconds: | 20260507 08:37:07.593 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:37:07.593 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:07.593 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:07.593 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:07.593 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:37:07.593 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:07.593 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:07.593 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 08:37:07.593 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:07.593 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:07.593 | INFO | Uploaded 36GiB to GPU in 1.9682s (18.2908GiB/s) | 20260507 08:37:07.593 | INFO | Downloaded 34GiB to GPU in 1.96569s (17.2968GiB/s) | 20260507 08:37:07.690 | INFO | Starting a round of Baz IO | 20260507 08:37:10.784 | INFO | Populating RTMetrics report | 20260507 08:37:10.785 | INFO | RTMetrics report populated | 20260507 08:37:10.785 | INFO | Opening tmp RTMetrics json file | 20260507 08:37:10.797 | INFO | Serializing RTMetrics to json | 20260507 08:37:10.799 | INFO | Serialization done, swapping tmp file now... | 20260507 08:37:10.799 | INFO | RTMetrics done | 20260507 08:37:10.868 | INFO | Finished a round of Baz IO: Wrote 1.609 GiB in 3.178 seconds | 20260507 08:37:10.868 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.49% / 0.00% / 67.51% | 20260507 08:37:10.868 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0888228% / 0% / 6.35783e-05% | 20260507 08:37:10.868 | INFO | Baz IO overall: | 20260507 08:37:10.869 | INFO | BytesWritten : 4269620952010 | 20260507 08:37:10.869 | INFO | NumEvents : 2041969813411 | 20260507 08:37:12.733 | INFO | Analyzing chunk frames = [8581120,8581632) | 20260507 08:37:12.733 | INFO | MemInfo: { | 20260507 08:37:12.733 | INFO | "gpu" : " 17 GB", | 20260507 08:37:12.733 | INFO | "huge" : "", | 20260507 08:37:12.733 | INFO | "huge_shared" : " 27 GB", | 20260507 08:37:12.733 | INFO | "malloc" : " 14 GB", | 20260507 08:37:12.733 | INFO | "pinned" : " 29 GB" | 20260507 08:37:12.733 | INFO | } | 20260507 08:37:12.733 | INFO | Perf report for the last 5.14052 seconds: | 20260507 08:37:12.733 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:37:12.733 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:12.733 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:12.733 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:12.733 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 08:37:12.733 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:12.733 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260507 08:37:12.733 | INFO | PRE_HQ : 58.8%, 1.8, 54.0ms, 0 | 20260507 08:37:12.733 | INFO | BAZWRITER : 61.4%, 3.0, 96.0ms, 0 | 20260507 08:37:12.733 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:37:12.733 | INFO | Uploaded 36GiB to GPU in 2.07382s (17.3593GiB/s) | 20260507 08:37:12.733 | INFO | Downloaded 38GiB to GPU in 2.18603s (17.3831GiB/s) | 20260507 08:37:17.873 | INFO | Analyzing chunk frames = [8581632,8582144) | 20260507 08:37:17.873 | INFO | MemInfo: { | 20260507 08:37:17.873 | INFO | "gpu" : " 17 GB", | 20260507 08:37:17.873 | INFO | "huge" : "", | 20260507 08:37:17.873 | INFO | "huge_shared" : " 27 GB", | 20260507 08:37:17.873 | INFO | "malloc" : " 14 GB", | 20260507 08:37:17.874 | INFO | "pinned" : " 29 GB" | 20260507 08:37:17.874 | INFO | } | 20260507 08:37:17.874 | INFO | Perf report for the last 5.14047 seconds: | 20260507 08:37:17.874 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:37:17.874 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:17.874 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:17.874 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:17.874 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:37:17.874 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:17.874 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:17.874 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:37:17.874 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:17.874 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:17.874 | INFO | Uploaded 36GiB to GPU in 1.9936s (18.0578GiB/s) | 20260507 08:37:17.874 | INFO | Downloaded 34GiB to GPU in 1.93687s (17.5541GiB/s) | 20260507 08:37:23.012 | INFO | Analyzing chunk frames = [8582144,8582656) | 20260507 08:37:23.012 | INFO | MemInfo: { | 20260507 08:37:23.012 | INFO | "gpu" : " 17 GB", | 20260507 08:37:23.012 | INFO | "huge" : "", | 20260507 08:37:23.012 | INFO | "huge_shared" : " 27 GB", | 20260507 08:37:23.012 | INFO | "malloc" : " 14 GB", | 20260507 08:37:23.012 | INFO | "pinned" : " 29 GB" | 20260507 08:37:23.012 | INFO | } | 20260507 08:37:23.012 | INFO | Perf report for the last 5.13873 seconds: | 20260507 08:37:23.013 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:37:23.013 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:23.013 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:23.013 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:23.013 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 08:37:23.013 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:23.013 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:23.013 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:37:23.013 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:23.013 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:23.013 | INFO | Uploaded 36GiB to GPU in 2.0338s (17.7008GiB/s) | 20260507 08:37:23.013 | INFO | Downloaded 34GiB to GPU in 1.91494s (17.7551GiB/s) | 20260507 08:37:28.151 | INFO | Analyzing chunk frames = [8582656,8583168) | 20260507 08:37:28.151 | INFO | MemInfo: { | 20260507 08:37:28.151 | INFO | "gpu" : " 17 GB", | 20260507 08:37:28.151 | INFO | "huge" : "", | 20260507 08:37:28.151 | INFO | "huge_shared" : " 27 GB", | 20260507 08:37:28.151 | INFO | "malloc" : " 14 GB", | 20260507 08:37:28.151 | INFO | "pinned" : " 29 GB" | 20260507 08:37:28.151 | INFO | } | 20260507 08:37:28.151 | INFO | Perf report for the last 5.13908 seconds: | 20260507 08:37:28.151 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:37:28.151 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:28.151 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:28.151 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:28.151 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 08:37:28.152 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:28.152 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:28.152 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 08:37:28.152 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:28.152 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:28.152 | INFO | Uploaded 36GiB to GPU in 2.05765s (17.4957GiB/s) | 20260507 08:37:28.152 | INFO | Downloaded 34GiB to GPU in 1.94644s (17.4678GiB/s) | 20260507 08:37:33.290 | INFO | Analyzing chunk frames = [8583168,8583680) | 20260507 08:37:33.291 | INFO | MemInfo: { | 20260507 08:37:33.291 | INFO | "gpu" : " 17 GB", | 20260507 08:37:33.291 | INFO | "huge" : "", | 20260507 08:37:33.291 | INFO | "huge_shared" : " 27 GB", | 20260507 08:37:33.291 | INFO | "malloc" : " 14 GB", | 20260507 08:37:33.291 | INFO | "pinned" : " 29 GB" | 20260507 08:37:33.291 | INFO | } | 20260507 08:37:33.291 | INFO | Perf report for the last 5.13916 seconds: | 20260507 08:37:33.291 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:37:33.291 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:33.291 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:33.291 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:33.291 | INFO | ANALYSIS : 60.3%, 3.0, 93.8ms, 0 | 20260507 08:37:33.291 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:33.291 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:33.291 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:37:33.291 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:33.291 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:33.291 | INFO | Uploaded 36GiB to GPU in 2.04898s (17.5697GiB/s) | 20260507 08:37:33.291 | INFO | Downloaded 34GiB to GPU in 1.92652s (17.6484GiB/s) | 20260507 08:37:38.431 | INFO | Analyzing chunk frames = [8583680,8584192) | 20260507 08:37:38.431 | INFO | MemInfo: { | 20260507 08:37:38.431 | INFO | "gpu" : " 17 GB", | 20260507 08:37:38.431 | INFO | "huge" : "", | 20260507 08:37:38.431 | INFO | "huge_shared" : " 27 GB", | 20260507 08:37:38.431 | INFO | "malloc" : " 14 GB", | 20260507 08:37:38.431 | INFO | "pinned" : " 29 GB" | 20260507 08:37:38.432 | INFO | } | 20260507 08:37:38.432 | INFO | Perf report for the last 5.14105 seconds: | 20260507 08:37:38.432 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:37:38.432 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:38.432 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:38.432 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:38.432 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 08:37:38.432 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:38.432 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:38.432 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:37:38.432 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:38.432 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:38.432 | INFO | Uploaded 36GiB to GPU in 2.06977s (17.3932GiB/s) | 20260507 08:37:38.432 | INFO | Downloaded 34GiB to GPU in 1.93913s (17.5337GiB/s) | 20260507 08:37:43.567 | INFO | Analyzing chunk frames = [8584192,8584704) | 20260507 08:37:43.567 | INFO | MemInfo: { | 20260507 08:37:43.567 | INFO | "gpu" : " 17 GB", | 20260507 08:37:43.567 | INFO | "huge" : "", | 20260507 08:37:43.567 | INFO | "huge_shared" : " 27 GB", | 20260507 08:37:43.567 | INFO | "malloc" : " 14 GB", | 20260507 08:37:43.567 | INFO | "pinned" : " 29 GB" | 20260507 08:37:43.567 | INFO | } | 20260507 08:37:43.567 | INFO | Perf report for the last 5.1357 seconds: | 20260507 08:37:43.568 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:37:43.568 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:43.568 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:43.568 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:43.568 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:37:43.568 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:43.568 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:43.568 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 08:37:43.568 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:43.568 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:43.568 | INFO | Uploaded 36GiB to GPU in 1.89748s (18.9725GiB/s) | 20260507 08:37:43.568 | INFO | Downloaded 34GiB to GPU in 1.97818s (17.1875GiB/s) | 20260507 08:37:48.706 | INFO | Analyzing chunk frames = [8584704,8585216) | 20260507 08:37:48.706 | INFO | MemInfo: { | 20260507 08:37:48.706 | INFO | "gpu" : " 17 GB", | 20260507 08:37:48.706 | INFO | "huge" : "", | 20260507 08:37:48.706 | INFO | "huge_shared" : " 27 GB", | 20260507 08:37:48.706 | INFO | "malloc" : " 14 GB", | 20260507 08:37:48.706 | INFO | "pinned" : " 29 GB" | 20260507 08:37:48.706 | INFO | } | 20260507 08:37:48.706 | INFO | Perf report for the last 5.13853 seconds: | 20260507 08:37:48.706 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:37:48.706 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:48.706 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:48.706 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:48.706 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:37:48.706 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:48.706 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:48.706 | INFO | PRE_HQ : 25.3%, 1.0, 13.1ms, 0 | 20260507 08:37:48.706 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:48.706 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:48.706 | INFO | Uploaded 36GiB to GPU in 1.98373s (18.1476GiB/s) | 20260507 08:37:48.706 | INFO | Downloaded 34GiB to GPU in 1.96634s (17.291GiB/s) | 20260507 08:37:48.706 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:37:48.706 | INFO | Total mem usage: 18 GiB | 20260507 08:37:48.706 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:37:48.706 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:37:48.706 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:37:48.706 | INFO | High Water Breakdown: | 20260507 08:37:48.706 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:37:48.706 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:37:48.706 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:37:48.706 | INFO | ------------------------------------------ | 20260507 08:37:48.706 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:37:48.706 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:37:48.706 | INFO | Total mem usage: 26.5 GiB | 20260507 08:37:48.706 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:37:48.706 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:37:48.706 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 08:37:48.706 | INFO | High Water Breakdown: | 20260507 08:37:48.706 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:37:48.706 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:37:48.707 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:37:48.707 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:37:48.707 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:37:48.707 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:37:48.707 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:37:48.707 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:37:48.707 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:37:48.707 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:37:48.707 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:37:48.707 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:37:48.707 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:37:48.707 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:37:48.707 | INFO | ------------------------------------------ | 20260507 08:37:48.707 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:37:48.707 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:37:48.707 | INFO | Total mem usage: 25.6 GiB | 20260507 08:37:48.707 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:37:48.707 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:37:48.707 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:37:48.707 | INFO | High Water Breakdown: | 20260507 08:37:48.707 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:37:48.707 | INFO | ------------------------------------------ | 20260507 08:37:48.707 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:37:48.707 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:37:48.707 | INFO | Total mem usage: 15.7 GiB | 20260507 08:37:48.707 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:37:48.707 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:37:48.707 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:37:48.707 | INFO | High Water Breakdown: | 20260507 08:37:48.707 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:37:48.707 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:37:48.707 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:37:48.707 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:37:48.707 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:37:48.707 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:37:48.707 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:37:48.707 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:37:48.707 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:37:48.707 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:37:48.707 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:37:48.707 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:37:48.707 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:37:48.707 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:37:48.707 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:37:48.707 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:37:48.707 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:37:48.707 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:37:48.707 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:37:48.707 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:37:48.707 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:37:48.707 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:37:48.707 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:37:48.707 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:37:48.707 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:37:48.707 | INFO | ------------------------------------------ | 20260507 08:37:48.707 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:37:48.707 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:37:48.707 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:37:48.707 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:37:48.707 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.50e+01, 1.35e+01 / 1.01e+00 | 20260507 08:37:48.707 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.02e-01, 9.71e-03 / 4.78e-03 | 20260507 08:37:48.707 | INFO | PerfReport: 11.68%, Baseline -- 3.58e+00 / 4.48e+00, 5.10e-01 / 1.91e-01 | 20260507 08:37:48.707 | INFO | PerfReport: 3.23%, Histogram -- 9.92e-01 / 1.52e+00, 1.71e-01 / 8.35e-02 | 20260507 08:37:48.707 | INFO | PerfReport: 16.36%, DME -- 4.02e+01 / 4.54e+01, 4.08e+00 / 1.19e+00 | 20260507 08:37:48.707 | INFO | PerfReport: 57.67%, FrameLabeling -- 1.77e+01 / 1.93e+01, 1.27e+00 / 1.12e+00 | 20260507 08:37:48.707 | INFO | PerfReport: 6.88%, PulseAccumulating -- 2.11e+00 / 2.70e+00, 3.89e-01 / 1.77e-01 | 20260507 08:37:48.707 | INFO | PerfReport: 2.91%, Metrics -- 8.93e-01 / 1.20e+00, 2.41e-01 / 1.31e-01 | 20260507 08:37:48.707 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.99e-01 / 3.59e-01, 5.03e-02 / 1.96e-02 | 20260507 08:37:48.707 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:37:48.707 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:37:48.707 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:37:48.707 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 08:37:48.707 | INFO | PerfReport: 21.84%, Upload -- 2.10e+01 / 0.00e+00, 7.17e+00 / 0.00e+00 | 20260507 08:37:48.707 | INFO | PerfReport: 21.39%, Download -- 2.06e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 08:37:48.707 | INFO | PerfReport: 31.92%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 08:37:48.707 | INFO | PerfReport: 1.58%, LockUpload -- 1.52e+00 / 0.00e+00, 4.95e+00 / 0.00e+00 | 20260507 08:37:48.707 | INFO | PerfReport: 1.27%, LockDownload -- 1.22e+00 / 0.00e+00, 2.97e+00 / 0.00e+00 | 20260507 08:37:48.707 | INFO | PerfReport: 21.99%, LockCompute -- 2.11e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 08:37:48.707 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.42e-02 / 0.00e+00 | 20260507 08:37:48.814 | INFO | Starting a round of Baz IO | 20260507 08:37:51.894 | INFO | Populating RTMetrics report | 20260507 08:37:51.894 | INFO | RTMetrics report populated | 20260507 08:37:51.894 | INFO | Opening tmp RTMetrics json file | 20260507 08:37:51.912 | INFO | Serializing RTMetrics to json | 20260507 08:37:51.913 | INFO | Serialization done, swapping tmp file now... | 20260507 08:37:51.914 | INFO | RTMetrics done | 20260507 08:37:51.978 | INFO | Finished a round of Baz IO: Wrote 1.608 GiB in 3.163 seconds | 20260507 08:37:51.978 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.49% / 0.00% / 67.51% | 20260507 08:37:51.978 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0879288% / 3.97364e-06% / 7.15256e-05% | 20260507 08:37:51.978 | INFO | Baz IO overall: | 20260507 08:37:51.978 | INFO | BytesWritten : 4271347058084 | 20260507 08:37:51.978 | INFO | NumEvents : 2042773430820 | 20260507 08:37:53.848 | INFO | Analyzing chunk frames = [8585216,8585728) | 20260507 08:37:53.848 | INFO | MemInfo: { | 20260507 08:37:53.848 | INFO | "gpu" : " 17 GB", | 20260507 08:37:53.848 | INFO | "huge" : "", | 20260507 08:37:53.848 | INFO | "huge_shared" : " 27 GB", | 20260507 08:37:53.848 | INFO | "malloc" : " 14 GB", | 20260507 08:37:53.848 | INFO | "pinned" : " 29 GB" | 20260507 08:37:53.848 | INFO | } | 20260507 08:37:53.848 | INFO | Perf report for the last 5.14235 seconds: | 20260507 08:37:53.848 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:37:53.848 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:53.848 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:53.848 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:53.848 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 08:37:53.848 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:53.848 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 08:37:53.848 | INFO | PRE_HQ : 58.8%, 1.8, 55.1ms, 0 | 20260507 08:37:53.848 | INFO | BAZWRITER : 61.2%, 3.1, 98.4ms, 0 | 20260507 08:37:53.848 | INFO | BAZ_SAMPLING : 0.4%, 1.0, 0.2ms, 0 | 20260507 08:37:53.849 | INFO | Uploaded 36GiB to GPU in 2.04366s (17.6155GiB/s) | 20260507 08:37:53.849 | INFO | Downloaded 38GiB to GPU in 2.16785s (17.5289GiB/s) | 20260507 08:37:58.991 | INFO | Analyzing chunk frames = [8585728,8586240) | 20260507 08:37:58.991 | INFO | MemInfo: { | 20260507 08:37:58.991 | INFO | "gpu" : " 17 GB", | 20260507 08:37:58.991 | INFO | "huge" : "", | 20260507 08:37:58.991 | INFO | "huge_shared" : " 27 GB", | 20260507 08:37:58.991 | INFO | "malloc" : " 14 GB", | 20260507 08:37:58.991 | INFO | "pinned" : " 29 GB" | 20260507 08:37:58.991 | INFO | } | 20260507 08:37:58.991 | INFO | Perf report for the last 5.14299 seconds: | 20260507 08:37:58.991 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:37:58.991 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:58.991 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:58.991 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:58.991 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:37:58.991 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:58.991 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:58.991 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:37:58.991 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:58.993 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:37:58.993 | INFO | Uploaded 36GiB to GPU in 1.98318s (18.1526GiB/s) | 20260507 08:37:58.993 | INFO | Downloaded 34GiB to GPU in 1.93654s (17.5571GiB/s) | 20260507 08:38:04.123 | INFO | Analyzing chunk frames = [8586240,8586752) | 20260507 08:38:04.123 | INFO | MemInfo: { | 20260507 08:38:04.123 | INFO | "gpu" : " 17 GB", | 20260507 08:38:04.123 | INFO | "huge" : "", | 20260507 08:38:04.123 | INFO | "huge_shared" : " 27 GB", | 20260507 08:38:04.123 | INFO | "malloc" : " 14 GB", | 20260507 08:38:04.123 | INFO | "pinned" : " 29 GB" | 20260507 08:38:04.124 | INFO | } | 20260507 08:38:04.124 | INFO | Perf report for the last 5.13244 seconds: | 20260507 08:38:04.124 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:38:04.124 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:04.124 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:04.124 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:04.124 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 08:38:04.124 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:04.124 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:04.124 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:38:04.124 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:04.124 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:04.124 | INFO | Uploaded 36GiB to GPU in 2.02775s (17.7536GiB/s) | 20260507 08:38:04.124 | INFO | Downloaded 34GiB to GPU in 1.91987s (17.7095GiB/s) | 20260507 08:38:09.261 | INFO | Analyzing chunk frames = [8586752,8587264) | 20260507 08:38:09.261 | INFO | MemInfo: { | 20260507 08:38:09.261 | INFO | "gpu" : " 17 GB", | 20260507 08:38:09.261 | INFO | "huge" : "", | 20260507 08:38:09.261 | INFO | "huge_shared" : " 27 GB", | 20260507 08:38:09.261 | INFO | "malloc" : " 14 GB", | 20260507 08:38:09.261 | INFO | "pinned" : " 29 GB" | 20260507 08:38:09.261 | INFO | } | 20260507 08:38:09.261 | INFO | Perf report for the last 5.13758 seconds: | 20260507 08:38:09.261 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:38:09.261 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:09.261 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:09.261 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:09.261 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 08:38:09.261 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:09.261 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:09.261 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:38:09.261 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:09.261 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:09.262 | INFO | Uploaded 36GiB to GPU in 2.05251s (17.5395GiB/s) | 20260507 08:38:09.262 | INFO | Downloaded 34GiB to GPU in 1.94595s (17.4722GiB/s) | 20260507 08:38:14.406 | INFO | Analyzing chunk frames = [8587264,8587776) | 20260507 08:38:14.406 | INFO | MemInfo: { | 20260507 08:38:14.406 | INFO | "gpu" : " 17 GB", | 20260507 08:38:14.406 | INFO | "huge" : "", | 20260507 08:38:14.406 | INFO | "huge_shared" : " 27 GB", | 20260507 08:38:14.406 | INFO | "malloc" : " 14 GB", | 20260507 08:38:14.406 | INFO | "pinned" : " 29 GB" | 20260507 08:38:14.406 | INFO | } | 20260507 08:38:14.406 | INFO | Perf report for the last 5.14475 seconds: | 20260507 08:38:14.406 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:38:14.406 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:14.406 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:14.406 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:14.406 | INFO | ANALYSIS : 60.1%, 3.0, 93.8ms, 0 | 20260507 08:38:14.406 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:14.406 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:14.406 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:38:14.406 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:14.406 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:14.406 | INFO | Uploaded 36GiB to GPU in 2.03018s (17.7324GiB/s) | 20260507 08:38:14.406 | INFO | Downloaded 34GiB to GPU in 1.91424s (17.7616GiB/s) | 20260507 08:38:19.549 | INFO | Analyzing chunk frames = [8587776,8588288) | 20260507 08:38:19.549 | INFO | MemInfo: { | 20260507 08:38:19.549 | INFO | "gpu" : " 17 GB", | 20260507 08:38:19.549 | INFO | "huge" : "", | 20260507 08:38:19.549 | INFO | "huge_shared" : " 27 GB", | 20260507 08:38:19.549 | INFO | "malloc" : " 14 GB", | 20260507 08:38:19.549 | INFO | "pinned" : " 29 GB" | 20260507 08:38:19.549 | INFO | } | 20260507 08:38:19.549 | INFO | Perf report for the last 5.1428 seconds: | 20260507 08:38:19.549 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:38:19.549 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:19.549 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:19.549 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:19.549 | INFO | ANALYSIS : 60.3%, 3.0, 94.3ms, 0 | 20260507 08:38:19.549 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:19.549 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:19.549 | INFO | PRE_HQ : 28.5%, 1.0, 14.8ms, 0 | 20260507 08:38:19.549 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:19.549 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:19.549 | INFO | Uploaded 36GiB to GPU in 2.05987s (17.4768GiB/s) | 20260507 08:38:19.549 | INFO | Downloaded 34GiB to GPU in 1.94064s (17.52GiB/s) | 20260507 08:38:24.686 | INFO | Analyzing chunk frames = [8588288,8588800) | 20260507 08:38:24.686 | INFO | MemInfo: { | 20260507 08:38:24.686 | INFO | "gpu" : " 17 GB", | 20260507 08:38:24.686 | INFO | "huge" : "", | 20260507 08:38:24.686 | INFO | "huge_shared" : " 27 GB", | 20260507 08:38:24.686 | INFO | "malloc" : " 14 GB", | 20260507 08:38:24.686 | INFO | "pinned" : " 29 GB" | 20260507 08:38:24.686 | INFO | } | 20260507 08:38:24.686 | INFO | Perf report for the last 5.13704 seconds: | 20260507 08:38:24.686 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:38:24.686 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:24.686 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:24.686 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:24.686 | INFO | ANALYSIS : 60.6%, 3.0, 94.6ms, 0 | 20260507 08:38:24.686 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:24.686 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:24.686 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 08:38:24.686 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:24.686 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:24.686 | INFO | Uploaded 36GiB to GPU in 1.96425s (18.3276GiB/s) | 20260507 08:38:24.686 | INFO | Downloaded 34GiB to GPU in 1.96884s (17.269GiB/s) | 20260507 08:38:29.819 | INFO | Analyzing chunk frames = [8588800,8589312) | 20260507 08:38:29.819 | INFO | MemInfo: { | 20260507 08:38:29.819 | INFO | "gpu" : " 17 GB", | 20260507 08:38:29.819 | INFO | "huge" : "", | 20260507 08:38:29.819 | INFO | "huge_shared" : " 27 GB", | 20260507 08:38:29.819 | INFO | "malloc" : " 14 GB", | 20260507 08:38:29.819 | INFO | "pinned" : " 29 GB" | 20260507 08:38:29.819 | INFO | } | 20260507 08:38:29.819 | INFO | Perf report for the last 5.13318 seconds: | 20260507 08:38:29.819 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:38:29.819 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:29.819 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:29.819 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:29.819 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:38:29.819 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:29.819 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:29.819 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:38:29.819 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:29.819 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:29.820 | INFO | Uploaded 36GiB to GPU in 1.98875s (18.1019GiB/s) | 20260507 08:38:29.820 | INFO | Downloaded 34GiB to GPU in 1.95652s (17.3778GiB/s) | 20260507 08:38:29.916 | INFO | Starting a round of Baz IO | 20260507 08:38:33.012 | INFO | Populating RTMetrics report | 20260507 08:38:33.013 | INFO | RTMetrics report populated | 20260507 08:38:33.013 | INFO | Opening tmp RTMetrics json file | 20260507 08:38:33.016 | INFO | Serializing RTMetrics to json | 20260507 08:38:33.030 | INFO | Serialization done, swapping tmp file now... | 20260507 08:38:33.030 | INFO | RTMetrics done | 20260507 08:38:33.098 | INFO | Finished a round of Baz IO: Wrote 1.607 GiB in 3.181 seconds | 20260507 08:38:33.098 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.48% / 0.00% / 67.51% | 20260507 08:38:33.098 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.087897% / 7.94729e-06% / 7.15256e-05% | 20260507 08:38:33.098 | INFO | Baz IO overall: | 20260507 08:38:33.098 | INFO | BytesWritten : 4273072747099 | 20260507 08:38:33.098 | INFO | NumEvents : 2043576726549 | 20260507 08:38:34.960 | INFO | Analyzing chunk frames = [8589312,8589824) | 20260507 08:38:34.960 | INFO | MemInfo: { | 20260507 08:38:34.960 | INFO | "gpu" : " 17 GB", | 20260507 08:38:34.960 | INFO | "huge" : "", | 20260507 08:38:34.960 | INFO | "huge_shared" : " 27 GB", | 20260507 08:38:34.961 | INFO | "malloc" : " 14 GB", | 20260507 08:38:34.961 | INFO | "pinned" : " 29 GB" | 20260507 08:38:34.961 | INFO | } | 20260507 08:38:34.961 | INFO | Perf report for the last 5.14141 seconds: | 20260507 08:38:34.961 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:38:34.961 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:34.961 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:34.961 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:34.961 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 08:38:34.961 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:34.961 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 08:38:34.961 | INFO | PRE_HQ : 58.9%, 1.8, 54.4ms, 0 | 20260507 08:38:34.961 | INFO | BAZWRITER : 61.5%, 3.1, 97.5ms, 0 | 20260507 08:38:34.961 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:38:34.961 | INFO | Uploaded 36GiB to GPU in 2.05591s (17.5105GiB/s) | 20260507 08:38:34.961 | INFO | Downloaded 38GiB to GPU in 2.17656s (17.4588GiB/s) | 20260507 08:38:40.104 | INFO | Analyzing chunk frames = [8589824,8590336) | 20260507 08:38:40.104 | INFO | MemInfo: { | 20260507 08:38:40.104 | INFO | "gpu" : " 17 GB", | 20260507 08:38:40.104 | INFO | "huge" : "", | 20260507 08:38:40.104 | INFO | "huge_shared" : " 27 GB", | 20260507 08:38:40.104 | INFO | "malloc" : " 14 GB", | 20260507 08:38:40.104 | INFO | "pinned" : " 29 GB" | 20260507 08:38:40.104 | INFO | } | 20260507 08:38:40.104 | INFO | Perf report for the last 5.14348 seconds: | 20260507 08:38:40.104 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:38:40.104 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:40.104 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:40.104 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:40.104 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:38:40.104 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:40.104 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:40.104 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:38:40.104 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:40.104 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:40.104 | INFO | Uploaded 36GiB to GPU in 1.9902s (18.0887GiB/s) | 20260507 08:38:40.104 | INFO | Downloaded 34GiB to GPU in 1.92589s (17.6542GiB/s) | 20260507 08:38:45.240 | INFO | Analyzing chunk frames = [8590336,8590848) | 20260507 08:38:45.240 | INFO | MemInfo: { | 20260507 08:38:45.240 | INFO | "gpu" : " 17 GB", | 20260507 08:38:45.240 | INFO | "huge" : "", | 20260507 08:38:45.240 | INFO | "huge_shared" : " 27 GB", | 20260507 08:38:45.240 | INFO | "malloc" : " 14 GB", | 20260507 08:38:45.240 | INFO | "pinned" : " 29 GB" | 20260507 08:38:45.240 | INFO | } | 20260507 08:38:45.240 | INFO | Perf report for the last 5.13627 seconds: | 20260507 08:38:45.240 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:38:45.240 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:45.240 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:45.240 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:45.240 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 08:38:45.240 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:45.240 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:45.240 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:38:45.241 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:45.241 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:45.241 | INFO | Uploaded 36GiB to GPU in 2.02513s (17.7767GiB/s) | 20260507 08:38:45.241 | INFO | Downloaded 34GiB to GPU in 1.91682s (17.7377GiB/s) | 20260507 08:38:50.381 | INFO | Analyzing chunk frames = [8590848,8591360) | 20260507 08:38:50.381 | INFO | MemInfo: { | 20260507 08:38:50.381 | INFO | "gpu" : " 17 GB", | 20260507 08:38:50.381 | INFO | "huge" : "", | 20260507 08:38:50.381 | INFO | "huge_shared" : " 27 GB", | 20260507 08:38:50.381 | INFO | "malloc" : " 14 GB", | 20260507 08:38:50.381 | INFO | "pinned" : " 29 GB" | 20260507 08:38:50.381 | INFO | } | 20260507 08:38:50.381 | INFO | Perf report for the last 5.14075 seconds: | 20260507 08:38:50.381 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:38:50.381 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:50.381 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:50.381 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:50.381 | INFO | ANALYSIS : 60.5%, 2.9, 93.5ms, 0 | 20260507 08:38:50.381 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:50.381 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:50.381 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:38:50.381 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:50.381 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:50.381 | INFO | Uploaded 36GiB to GPU in 2.06221s (17.457GiB/s) | 20260507 08:38:50.382 | INFO | Downloaded 34GiB to GPU in 1.94788s (17.4549GiB/s) | 20260507 08:38:55.520 | INFO | Analyzing chunk frames = [8591360,8591872) | 20260507 08:38:55.520 | INFO | MemInfo: { | 20260507 08:38:55.520 | INFO | "gpu" : " 17 GB", | 20260507 08:38:55.520 | INFO | "huge" : "", | 20260507 08:38:55.520 | INFO | "huge_shared" : " 27 GB", | 20260507 08:38:55.520 | INFO | "malloc" : " 14 GB", | 20260507 08:38:55.520 | INFO | "pinned" : " 29 GB" | 20260507 08:38:55.520 | INFO | } | 20260507 08:38:55.520 | INFO | Perf report for the last 5.13911 seconds: | 20260507 08:38:55.520 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:38:55.520 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:55.520 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:55.520 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:55.520 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 08:38:55.520 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:55.520 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:55.520 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 08:38:55.520 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:55.520 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:38:55.521 | INFO | Uploaded 36GiB to GPU in 2.04369s (17.6152GiB/s) | 20260507 08:38:55.521 | INFO | Downloaded 34GiB to GPU in 1.92219s (17.6881GiB/s) | 20260507 08:39:00.661 | INFO | Analyzing chunk frames = [8591872,8592384) | 20260507 08:39:00.661 | INFO | MemInfo: { | 20260507 08:39:00.661 | INFO | "gpu" : " 17 GB", | 20260507 08:39:00.662 | INFO | "huge" : "", | 20260507 08:39:00.662 | INFO | "huge_shared" : " 27 GB", | 20260507 08:39:00.662 | INFO | "malloc" : " 14 GB", | 20260507 08:39:00.662 | INFO | "pinned" : " 29 GB" | 20260507 08:39:00.662 | INFO | } | 20260507 08:39:00.662 | INFO | Perf report for the last 5.14151 seconds: | 20260507 08:39:00.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:39:00.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:00.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:00.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:00.662 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 08:39:00.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:00.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:00.662 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 08:39:00.662 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:00.662 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:00.662 | INFO | Uploaded 36GiB to GPU in 2.07019s (17.3897GiB/s) | 20260507 08:39:00.662 | INFO | Downloaded 34GiB to GPU in 1.93662s (17.5563GiB/s) | 20260507 08:39:05.802 | INFO | Analyzing chunk frames = [8592384,8592896) | 20260507 08:39:05.802 | INFO | MemInfo: { | 20260507 08:39:05.802 | INFO | "gpu" : " 17 GB", | 20260507 08:39:05.802 | INFO | "huge" : "", | 20260507 08:39:05.802 | INFO | "huge_shared" : " 27 GB", | 20260507 08:39:05.802 | INFO | "malloc" : " 14 GB", | 20260507 08:39:05.802 | INFO | "pinned" : " 29 GB" | 20260507 08:39:05.802 | INFO | } | 20260507 08:39:05.802 | INFO | Perf report for the last 5.14021 seconds: | 20260507 08:39:05.802 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:39:05.802 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:05.802 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:05.802 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:05.802 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 08:39:05.802 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:05.802 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:05.802 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 08:39:05.802 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:05.802 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:05.802 | INFO | Uploaded 36GiB to GPU in 1.90617s (18.8861GiB/s) | 20260507 08:39:05.802 | INFO | Downloaded 34GiB to GPU in 1.96554s (17.298GiB/s) | 20260507 08:39:10.939 | INFO | Analyzing chunk frames = [8592896,8593408) | 20260507 08:39:10.939 | INFO | MemInfo: { | 20260507 08:39:10.939 | INFO | "gpu" : " 17 GB", | 20260507 08:39:10.939 | INFO | "huge" : "", | 20260507 08:39:10.939 | INFO | "huge_shared" : " 27 GB", | 20260507 08:39:10.939 | INFO | "malloc" : " 14 GB", | 20260507 08:39:10.939 | INFO | "pinned" : " 29 GB" | 20260507 08:39:10.939 | INFO | } | 20260507 08:39:10.939 | INFO | Perf report for the last 5.13739 seconds: | 20260507 08:39:10.939 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:39:10.939 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:10.939 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:10.939 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:10.939 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:39:10.939 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:10.940 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:10.940 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:39:10.940 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:10.940 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:10.940 | INFO | Uploaded 36GiB to GPU in 1.99084s (18.0828GiB/s) | 20260507 08:39:10.940 | INFO | Downloaded 34GiB to GPU in 1.95491s (17.3921GiB/s) | 20260507 08:39:10.940 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:39:10.940 | INFO | Total mem usage: 18 GiB | 20260507 08:39:10.940 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:39:10.940 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:39:10.940 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:39:10.940 | INFO | High Water Breakdown: | 20260507 08:39:10.940 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:39:10.940 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:39:10.940 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:39:10.940 | INFO | ------------------------------------------ | 20260507 08:39:10.940 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:39:10.940 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:39:10.940 | INFO | Total mem usage: 26.5 GiB | 20260507 08:39:10.940 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:39:10.940 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:39:10.940 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:39:10.940 | INFO | High Water Breakdown: | 20260507 08:39:10.940 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:39:10.940 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:39:10.940 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:39:10.940 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:39:10.940 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:39:10.940 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:39:10.940 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:39:10.940 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:39:10.940 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:39:10.940 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:39:10.940 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:39:10.940 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:39:10.940 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:39:10.940 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:39:10.940 | INFO | ------------------------------------------ | 20260507 08:39:10.940 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:39:10.940 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:39:10.940 | INFO | Total mem usage: 25.6 GiB | 20260507 08:39:10.940 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:39:10.940 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:39:10.940 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:39:10.940 | INFO | High Water Breakdown: | 20260507 08:39:10.940 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:39:10.940 | INFO | ------------------------------------------ | 20260507 08:39:10.941 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:39:10.941 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:39:10.941 | INFO | Total mem usage: 15.7 GiB | 20260507 08:39:10.941 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:39:10.941 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:39:10.941 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:39:10.941 | INFO | High Water Breakdown: | 20260507 08:39:10.941 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:39:10.941 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:39:10.941 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:39:10.941 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:39:10.941 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:39:10.941 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:39:10.941 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:39:10.941 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:39:10.941 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:39:10.941 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:39:10.941 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:39:10.941 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:39:10.941 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:39:10.941 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:39:10.941 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:39:10.941 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:39:10.941 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:39:10.941 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:39:10.941 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:39:10.941 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:39:10.941 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:39:10.941 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:39:10.941 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:39:10.941 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:39:10.941 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:39:10.941 | INFO | ------------------------------------------ | 20260507 08:39:10.941 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:39:10.941 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:39:10.941 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:39:10.941 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:39:10.941 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.49e+01, 1.34e+01 / 1.20e+00 | 20260507 08:39:10.941 | INFO | PerfReport: 0.30%, AggregateStats -- 9.09e-02 / 1.02e-01, 9.85e-03 / 5.38e-03 | 20260507 08:39:10.941 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.49e+00, 5.08e-01 / 1.79e-01 | 20260507 08:39:10.941 | INFO | PerfReport: 3.23%, Histogram -- 9.91e-01 / 1.51e+00, 1.69e-01 / 9.87e-02 | 20260507 08:39:10.941 | INFO | PerfReport: 16.34%, DME -- 4.01e+01 / 4.51e+01, 4.00e+00 / 1.10e+00 | 20260507 08:39:10.941 | INFO | PerfReport: 57.72%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.29e+00 / 1.04e+00 | 20260507 08:39:10.941 | INFO | PerfReport: 6.86%, PulseAccumulating -- 2.11e+00 / 2.67e+00, 3.79e-01 / 1.83e-01 | 20260507 08:39:10.941 | INFO | PerfReport: 2.90%, Metrics -- 8.89e-01 / 1.19e+00, 2.38e-01 / 1.10e-01 | 20260507 08:39:10.941 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.99e-01 / 3.60e-01, 4.98e-02 / 2.33e-02 | 20260507 08:39:10.941 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:39:10.941 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:39:10.941 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:39:10.941 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260507 08:39:10.941 | INFO | PerfReport: 21.86%, Upload -- 2.10e+01 / 0.00e+00, 7.13e+00 / 0.00e+00 | 20260507 08:39:10.941 | INFO | PerfReport: 21.32%, Download -- 2.05e+01 / 0.00e+00, 7.20e+00 / 0.00e+00 | 20260507 08:39:10.941 | INFO | PerfReport: 31.91%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 08:39:10.941 | INFO | PerfReport: 1.66%, LockUpload -- 1.60e+00 / 0.00e+00, 5.07e+00 / 0.00e+00 | 20260507 08:39:10.941 | INFO | PerfReport: 1.21%, LockDownload -- 1.17e+00 / 0.00e+00, 2.91e+00 / 0.00e+00 | 20260507 08:39:10.941 | INFO | PerfReport: 22.03%, LockCompute -- 2.12e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 08:39:10.941 | INFO | PerfReport: 0.01%, Misc -- 1.34e-02 / 0.00e+00, 1.67e-02 / 0.00e+00 | 20260507 08:39:11.037 | INFO | Starting a round of Baz IO | 20260507 08:39:14.129 | INFO | Populating RTMetrics report | 20260507 08:39:14.130 | INFO | RTMetrics report populated | 20260507 08:39:14.130 | INFO | Opening tmp RTMetrics json file | 20260507 08:39:14.146 | INFO | Serializing RTMetrics to json | 20260507 08:39:14.147 | INFO | Serialization done, swapping tmp file now... | 20260507 08:39:14.148 | INFO | RTMetrics done | 20260507 08:39:14.213 | INFO | Finished a round of Baz IO: Wrote 1.606 GiB in 3.176 seconds | 20260507 08:39:14.213 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.48% / 0.00% / 67.52% | 20260507 08:39:14.213 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0877301% / 0% / 5.96046e-05% | 20260507 08:39:14.213 | INFO | Baz IO overall: | 20260507 08:39:14.213 | INFO | BytesWritten : 4274797266421 | 20260507 08:39:14.213 | INFO | NumEvents : 2044379501438 | 20260507 08:39:16.071 | INFO | Analyzing chunk frames = [8593408,8593920) | 20260507 08:39:16.071 | INFO | MemInfo: { | 20260507 08:39:16.071 | INFO | "gpu" : " 17 GB", | 20260507 08:39:16.071 | INFO | "huge" : "", | 20260507 08:39:16.071 | INFO | "huge_shared" : " 27 GB", | 20260507 08:39:16.071 | INFO | "malloc" : " 14 GB", | 20260507 08:39:16.071 | INFO | "pinned" : " 29 GB" | 20260507 08:39:16.071 | INFO | } | 20260507 08:39:16.071 | INFO | Perf report for the last 5.13119 seconds: | 20260507 08:39:16.071 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:39:16.071 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:16.071 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:16.071 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:16.071 | INFO | ANALYSIS : 61.7%, 3.0, 95.8ms, 0 | 20260507 08:39:16.071 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:16.071 | INFO | RT_METRICS : 41.1%, 1.0, 21.5ms, 0 | 20260507 08:39:16.071 | INFO | PRE_HQ : 58.9%, 1.8, 54.4ms, 0 | 20260507 08:39:16.071 | INFO | BAZWRITER : 61.5%, 3.1, 97.1ms, 0 | 20260507 08:39:16.071 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:39:16.071 | INFO | Uploaded 36GiB to GPU in 2.07456s (17.3531GiB/s) | 20260507 08:39:16.071 | INFO | Downloaded 38GiB to GPU in 2.18536s (17.3884GiB/s) | 20260507 08:39:21.213 | INFO | Analyzing chunk frames = [8593920,8594432) | 20260507 08:39:21.213 | INFO | MemInfo: { | 20260507 08:39:21.213 | INFO | "gpu" : " 17 GB", | 20260507 08:39:21.213 | INFO | "huge" : "", | 20260507 08:39:21.213 | INFO | "huge_shared" : " 27 GB", | 20260507 08:39:21.213 | INFO | "malloc" : " 14 GB", | 20260507 08:39:21.213 | INFO | "pinned" : " 29 GB" | 20260507 08:39:21.213 | INFO | } | 20260507 08:39:21.213 | INFO | Perf report for the last 5.14244 seconds: | 20260507 08:39:21.213 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:39:21.213 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:21.213 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:21.213 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:21.213 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 08:39:21.213 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:21.213 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:21.213 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:39:21.213 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:21.213 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:21.213 | INFO | Uploaded 36GiB to GPU in 1.99736s (18.0238GiB/s) | 20260507 08:39:21.213 | INFO | Downloaded 34GiB to GPU in 1.93173s (17.6008GiB/s) | 20260507 08:39:26.356 | INFO | Analyzing chunk frames = [8594432,8594944) | 20260507 08:39:26.356 | INFO | MemInfo: { | 20260507 08:39:26.356 | INFO | "gpu" : " 17 GB", | 20260507 08:39:26.356 | INFO | "huge" : "", | 20260507 08:39:26.356 | INFO | "huge_shared" : " 27 GB", | 20260507 08:39:26.356 | INFO | "malloc" : " 14 GB", | 20260507 08:39:26.356 | INFO | "pinned" : " 29 GB" | 20260507 08:39:26.356 | INFO | } | 20260507 08:39:26.356 | INFO | Perf report for the last 5.1428 seconds: | 20260507 08:39:26.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:39:26.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:26.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:26.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:26.356 | INFO | ANALYSIS : 60.4%, 3.0, 93.5ms, 0 | 20260507 08:39:26.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:26.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:26.356 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:39:26.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:26.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:26.356 | INFO | Uploaded 36GiB to GPU in 2.03525s (17.6883GiB/s) | 20260507 08:39:26.356 | INFO | Downloaded 34GiB to GPU in 1.95121s (17.425GiB/s) | 20260507 08:39:31.504 | INFO | Analyzing chunk frames = [8594944,8595456) | 20260507 08:39:31.504 | INFO | MemInfo: { | 20260507 08:39:31.504 | INFO | "gpu" : " 17 GB", | 20260507 08:39:31.505 | INFO | "huge" : "", | 20260507 08:39:31.505 | INFO | "huge_shared" : " 27 GB", | 20260507 08:39:31.505 | INFO | "malloc" : " 14 GB", | 20260507 08:39:31.505 | INFO | "pinned" : " 29 GB" | 20260507 08:39:31.505 | INFO | } | 20260507 08:39:31.505 | INFO | Perf report for the last 5.14879 seconds: | 20260507 08:39:31.505 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:39:31.505 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:31.505 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:31.505 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:31.505 | INFO | ANALYSIS : 60.4%, 2.9, 93.5ms, 0 | 20260507 08:39:31.505 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:31.505 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:31.505 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:39:31.505 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:31.505 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:31.505 | INFO | Uploaded 36GiB to GPU in 2.05764s (17.4958GiB/s) | 20260507 08:39:31.505 | INFO | Downloaded 34GiB to GPU in 1.94492s (17.4815GiB/s) | 20260507 08:39:36.634 | INFO | Analyzing chunk frames = [8595456,8595968) | 20260507 08:39:36.634 | INFO | MemInfo: { | 20260507 08:39:36.634 | INFO | "gpu" : " 17 GB", | 20260507 08:39:36.634 | INFO | "huge" : "", | 20260507 08:39:36.634 | INFO | "huge_shared" : " 27 GB", | 20260507 08:39:36.634 | INFO | "malloc" : " 14 GB", | 20260507 08:39:36.634 | INFO | "pinned" : " 29 GB" | 20260507 08:39:36.634 | INFO | } | 20260507 08:39:36.634 | INFO | Perf report for the last 5.12966 seconds: | 20260507 08:39:36.634 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:39:36.634 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:36.634 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:36.634 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:36.634 | INFO | ANALYSIS : 60.2%, 3.0, 93.6ms, 0 | 20260507 08:39:36.634 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:36.634 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:36.634 | INFO | PRE_HQ : 24.6%, 1.0, 12.7ms, 0 | 20260507 08:39:36.634 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:36.634 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:36.635 | INFO | Uploaded 36GiB to GPU in 2.06948s (17.3957GiB/s) | 20260507 08:39:36.635 | INFO | Downloaded 34GiB to GPU in 1.9281s (17.634GiB/s) | 20260507 08:39:41.769 | INFO | Analyzing chunk frames = [8595968,8596480) | 20260507 08:39:41.769 | INFO | MemInfo: { | 20260507 08:39:41.769 | INFO | "gpu" : " 17 GB", | 20260507 08:39:41.769 | INFO | "huge" : "", | 20260507 08:39:41.769 | INFO | "huge_shared" : " 27 GB", | 20260507 08:39:41.769 | INFO | "malloc" : " 14 GB", | 20260507 08:39:41.769 | INFO | "pinned" : " 29 GB" | 20260507 08:39:41.769 | INFO | } | 20260507 08:39:41.769 | INFO | Perf report for the last 5.13476 seconds: | 20260507 08:39:41.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:39:41.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:41.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:41.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:41.769 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:39:41.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:41.769 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:41.769 | INFO | PRE_HQ : 28.0%, 1.0, 14.5ms, 0 | 20260507 08:39:41.769 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:41.769 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:41.770 | INFO | Uploaded 36GiB to GPU in 2.06953s (17.3952GiB/s) | 20260507 08:39:41.770 | INFO | Downloaded 34GiB to GPU in 1.93579s (17.5639GiB/s) | 20260507 08:39:46.912 | INFO | Analyzing chunk frames = [8596480,8596992) | 20260507 08:39:46.912 | INFO | MemInfo: { | 20260507 08:39:46.912 | INFO | "gpu" : " 17 GB", | 20260507 08:39:46.912 | INFO | "huge" : "", | 20260507 08:39:46.912 | INFO | "huge_shared" : " 27 GB", | 20260507 08:39:46.912 | INFO | "malloc" : " 14 GB", | 20260507 08:39:46.912 | INFO | "pinned" : " 29 GB" | 20260507 08:39:46.912 | INFO | } | 20260507 08:39:46.912 | INFO | Perf report for the last 5.14291 seconds: | 20260507 08:39:46.912 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:39:46.912 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:46.912 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:46.912 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:46.912 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:39:46.912 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:46.912 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:46.912 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 08:39:46.912 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:46.912 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:46.912 | INFO | Uploaded 36GiB to GPU in 1.95518s (18.4126GiB/s) | 20260507 08:39:46.912 | INFO | Downloaded 34GiB to GPU in 1.9797s (17.1743GiB/s) | 20260507 08:39:52.050 | INFO | Analyzing chunk frames = [8596992,8597504) | 20260507 08:39:52.050 | INFO | MemInfo: { | 20260507 08:39:52.050 | INFO | "gpu" : " 17 GB", | 20260507 08:39:52.050 | INFO | "huge" : "", | 20260507 08:39:52.051 | INFO | "huge_shared" : " 27 GB", | 20260507 08:39:52.051 | INFO | "malloc" : " 14 GB", | 20260507 08:39:52.051 | INFO | "pinned" : " 29 GB" | 20260507 08:39:52.051 | INFO | } | 20260507 08:39:52.051 | INFO | Perf report for the last 5.13867 seconds: | 20260507 08:39:52.051 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:39:52.051 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:52.051 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:52.051 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:52.051 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:39:52.051 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:52.051 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:52.051 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:39:52.051 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:52.051 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:52.051 | INFO | Uploaded 36GiB to GPU in 1.99028s (18.0879GiB/s) | 20260507 08:39:52.051 | INFO | Downloaded 34GiB to GPU in 1.96582s (17.2956GiB/s) | 20260507 08:39:52.148 | INFO | Starting a round of Baz IO | 20260507 08:39:55.238 | INFO | Populating RTMetrics report | 20260507 08:39:55.238 | INFO | RTMetrics report populated | 20260507 08:39:55.238 | INFO | Opening tmp RTMetrics json file | 20260507 08:39:55.240 | INFO | Serializing RTMetrics to json | 20260507 08:39:55.256 | INFO | Serialization done, swapping tmp file now... | 20260507 08:39:55.256 | INFO | RTMetrics done | 20260507 08:39:55.321 | INFO | Finished a round of Baz IO: Wrote 1.606 GiB in 3.173 seconds | 20260507 08:39:55.321 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.48% / 0.00% / 67.52% | 20260507 08:39:55.321 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0874956% / 0% / 6.75519e-05% | 20260507 08:39:55.321 | INFO | Baz IO overall: | 20260507 08:39:55.321 | INFO | BytesWritten : 4276521565988 | 20260507 08:39:55.321 | INFO | NumEvents : 2045182137903 | 20260507 08:39:57.188 | INFO | Analyzing chunk frames = [8597504,8598016) | 20260507 08:39:57.188 | INFO | MemInfo: { | 20260507 08:39:57.188 | INFO | "gpu" : " 17 GB", | 20260507 08:39:57.188 | INFO | "huge" : "", | 20260507 08:39:57.188 | INFO | "huge_shared" : " 27 GB", | 20260507 08:39:57.188 | INFO | "malloc" : " 14 GB", | 20260507 08:39:57.188 | INFO | "pinned" : " 29 GB" | 20260507 08:39:57.188 | INFO | } | 20260507 08:39:57.188 | INFO | Perf report for the last 5.13764 seconds: | 20260507 08:39:57.188 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:39:57.188 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:57.188 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:57.188 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:57.188 | INFO | ANALYSIS : 61.5%, 3.0, 95.7ms, 0 | 20260507 08:39:57.188 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:39:57.188 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 08:39:57.189 | INFO | PRE_HQ : 58.7%, 1.8, 54.2ms, 0 | 20260507 08:39:57.189 | INFO | BAZWRITER : 61.4%, 3.1, 97.0ms, 0 | 20260507 08:39:57.189 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:39:57.189 | INFO | Uploaded 36GiB to GPU in 2.06677s (17.4185GiB/s) | 20260507 08:39:57.189 | INFO | Downloaded 38GiB to GPU in 2.1765s (17.4592GiB/s) | 20260507 08:40:02.325 | INFO | Analyzing chunk frames = [8598016,8598528) | 20260507 08:40:02.325 | INFO | MemInfo: { | 20260507 08:40:02.325 | INFO | "gpu" : " 17 GB", | 20260507 08:40:02.325 | INFO | "huge" : "", | 20260507 08:40:02.325 | INFO | "huge_shared" : " 27 GB", | 20260507 08:40:02.325 | INFO | "malloc" : " 14 GB", | 20260507 08:40:02.325 | INFO | "pinned" : " 29 GB" | 20260507 08:40:02.325 | INFO | } | 20260507 08:40:02.326 | INFO | Perf report for the last 5.13718 seconds: | 20260507 08:40:02.326 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:40:02.326 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:02.326 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:02.326 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:02.326 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 08:40:02.326 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:02.326 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:02.326 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260507 08:40:02.326 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:02.326 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:02.326 | INFO | Uploaded 36GiB to GPU in 1.98752s (18.1131GiB/s) | 20260507 08:40:02.326 | INFO | Downloaded 34GiB to GPU in 1.94119s (17.515GiB/s) | 20260507 08:40:07.475 | INFO | Analyzing chunk frames = [8598528,8599040) | 20260507 08:40:07.475 | INFO | MemInfo: { | 20260507 08:40:07.475 | INFO | "gpu" : " 17 GB", | 20260507 08:40:07.475 | INFO | "huge" : "", | 20260507 08:40:07.475 | INFO | "huge_shared" : " 27 GB", | 20260507 08:40:07.475 | INFO | "malloc" : " 14 GB", | 20260507 08:40:07.475 | INFO | "pinned" : " 29 GB" | 20260507 08:40:07.475 | INFO | } | 20260507 08:40:07.476 | INFO | Perf report for the last 5.14993 seconds: | 20260507 08:40:07.476 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:40:07.476 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:07.476 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:07.476 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:07.476 | INFO | ANALYSIS : 60.3%, 3.0, 93.6ms, 0 | 20260507 08:40:07.476 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:07.476 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:07.476 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:40:07.476 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:07.476 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:07.476 | INFO | Uploaded 36GiB to GPU in 2.04413s (17.6114GiB/s) | 20260507 08:40:07.476 | INFO | Downloaded 34GiB to GPU in 1.95476s (17.3934GiB/s) | 20260507 08:40:12.614 | INFO | Analyzing chunk frames = [8599040,8599552) | 20260507 08:40:12.614 | INFO | MemInfo: { | 20260507 08:40:12.614 | INFO | "gpu" : " 17 GB", | 20260507 08:40:12.614 | INFO | "huge" : "", | 20260507 08:40:12.614 | INFO | "huge_shared" : " 27 GB", | 20260507 08:40:12.614 | INFO | "malloc" : " 14 GB", | 20260507 08:40:12.614 | INFO | "pinned" : " 29 GB" | 20260507 08:40:12.614 | INFO | } | 20260507 08:40:12.615 | INFO | Perf report for the last 5.13898 seconds: | 20260507 08:40:12.615 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:40:12.615 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:12.615 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:12.615 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:12.615 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 08:40:12.615 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:12.615 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:12.615 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:40:12.615 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:12.615 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:12.615 | INFO | Uploaded 36GiB to GPU in 2.05769s (17.4953GiB/s) | 20260507 08:40:12.615 | INFO | Downloaded 34GiB to GPU in 1.95107s (17.4263GiB/s) | 20260507 08:40:17.754 | INFO | Analyzing chunk frames = [8599552,8600064) | 20260507 08:40:17.754 | INFO | MemInfo: { | 20260507 08:40:17.754 | INFO | "gpu" : " 17 GB", | 20260507 08:40:17.754 | INFO | "huge" : "", | 20260507 08:40:17.754 | INFO | "huge_shared" : " 27 GB", | 20260507 08:40:17.754 | INFO | "malloc" : " 14 GB", | 20260507 08:40:17.754 | INFO | "pinned" : " 29 GB" | 20260507 08:40:17.754 | INFO | } | 20260507 08:40:17.754 | INFO | Perf report for the last 5.1398 seconds: | 20260507 08:40:17.754 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:40:17.754 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:17.754 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:17.754 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:17.754 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 08:40:17.754 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:17.755 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:17.755 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:40:17.755 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:17.755 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:17.755 | INFO | Uploaded 36GiB to GPU in 2.0507s (17.555GiB/s) | 20260507 08:40:17.755 | INFO | Downloaded 34GiB to GPU in 1.90988s (17.8022GiB/s) | 20260507 08:40:22.881 | INFO | Analyzing chunk frames = [8600064,8600576) | 20260507 08:40:22.881 | INFO | MemInfo: { | 20260507 08:40:22.881 | INFO | "gpu" : " 17 GB", | 20260507 08:40:22.881 | INFO | "huge" : "", | 20260507 08:40:22.881 | INFO | "huge_shared" : " 27 GB", | 20260507 08:40:22.881 | INFO | "malloc" : " 14 GB", | 20260507 08:40:22.881 | INFO | "pinned" : " 29 GB" | 20260507 08:40:22.881 | INFO | } | 20260507 08:40:22.881 | INFO | Perf report for the last 5.12704 seconds: | 20260507 08:40:22.881 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:40:22.881 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:22.881 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:22.881 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:22.881 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 08:40:22.881 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:22.881 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:22.882 | INFO | PRE_HQ : 27.5%, 1.0, 14.3ms, 0 | 20260507 08:40:22.882 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:22.882 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:22.882 | INFO | Uploaded 36GiB to GPU in 2.04495s (17.6044GiB/s) | 20260507 08:40:22.882 | INFO | Downloaded 34GiB to GPU in 1.92884s (17.6272GiB/s) | 20260507 08:40:28.022 | INFO | Analyzing chunk frames = [8600576,8601088) | 20260507 08:40:28.022 | INFO | MemInfo: { | 20260507 08:40:28.022 | INFO | "gpu" : " 17 GB", | 20260507 08:40:28.022 | INFO | "huge" : "", | 20260507 08:40:28.022 | INFO | "huge_shared" : " 27 GB", | 20260507 08:40:28.022 | INFO | "malloc" : " 14 GB", | 20260507 08:40:28.022 | INFO | "pinned" : " 29 GB" | 20260507 08:40:28.022 | INFO | } | 20260507 08:40:28.023 | INFO | Perf report for the last 5.14113 seconds: | 20260507 08:40:28.023 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:40:28.023 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:28.023 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:28.023 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:28.023 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 08:40:28.023 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:28.023 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:28.023 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 08:40:28.023 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:28.023 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:28.023 | INFO | Uploaded 36GiB to GPU in 2.01045s (17.9064GiB/s) | 20260507 08:40:28.023 | INFO | Downloaded 34GiB to GPU in 1.97547s (17.2111GiB/s) | 20260507 08:40:33.164 | INFO | Analyzing chunk frames = [8601088,8601600) | 20260507 08:40:33.164 | INFO | MemInfo: { | 20260507 08:40:33.164 | INFO | "gpu" : " 17 GB", | 20260507 08:40:33.164 | INFO | "huge" : "", | 20260507 08:40:33.164 | INFO | "huge_shared" : " 27 GB", | 20260507 08:40:33.164 | INFO | "malloc" : " 14 GB", | 20260507 08:40:33.164 | INFO | "pinned" : " 29 GB" | 20260507 08:40:33.164 | INFO | } | 20260507 08:40:33.164 | INFO | Perf report for the last 5.14164 seconds: | 20260507 08:40:33.164 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:40:33.164 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:33.164 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:33.164 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:33.164 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:40:33.164 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:33.164 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:33.164 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:40:33.164 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:33.164 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:33.165 | INFO | Uploaded 36GiB to GPU in 1.98716s (18.1163GiB/s) | 20260507 08:40:33.165 | INFO | Downloaded 34GiB to GPU in 1.95596s (17.3828GiB/s) | 20260507 08:40:33.165 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:40:33.165 | INFO | Total mem usage: 18 GiB | 20260507 08:40:33.165 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:40:33.165 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -27.1%) | 20260507 08:40:33.165 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:40:33.165 | INFO | High Water Breakdown: | 20260507 08:40:33.165 | INFO | 60.1% : 8.52 GiB ( 74.7% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:40:33.165 | INFO | 39.7% : 5.63 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:40:33.165 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:40:33.165 | INFO | ------------------------------------------ | 20260507 08:40:33.165 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:40:33.165 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:40:33.165 | INFO | Total mem usage: 26.5 GiB | 20260507 08:40:33.165 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:40:33.165 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:40:33.165 | INFO | Unused Cache: 1.78 GiB (Waste: 6.71%) | 20260507 08:40:33.165 | INFO | High Water Breakdown: | 20260507 08:40:33.165 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:40:33.165 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:40:33.165 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:40:33.165 | INFO | 1.6% : 418 MiB ( 79.6% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:40:33.165 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:40:33.165 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:40:33.165 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:40:33.165 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:40:33.165 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:40:33.165 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:40:33.165 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:40:33.165 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:40:33.165 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:40:33.165 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:40:33.165 | INFO | ------------------------------------------ | 20260507 08:40:33.165 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:40:33.165 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:40:33.165 | INFO | Total mem usage: 25.6 GiB | 20260507 08:40:33.165 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:40:33.165 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:40:33.165 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:40:33.165 | INFO | High Water Breakdown: | 20260507 08:40:33.165 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:40:33.165 | INFO | ------------------------------------------ | 20260507 08:40:33.165 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:40:33.165 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:40:33.165 | INFO | Total mem usage: 15.7 GiB | 20260507 08:40:33.165 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:40:33.165 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:40:33.165 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:40:33.165 | INFO | High Water Breakdown: | 20260507 08:40:33.165 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:40:33.165 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:40:33.165 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:40:33.165 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:40:33.165 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:40:33.165 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:40:33.165 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:40:33.165 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:40:33.165 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:40:33.165 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:40:33.165 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:40:33.165 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:40:33.165 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:40:33.165 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:40:33.165 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:40:33.165 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:40:33.165 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:40:33.165 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:40:33.165 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:40:33.165 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:40:33.165 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:40:33.165 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:40:33.165 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:40:33.165 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:40:33.165 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:40:33.165 | INFO | ------------------------------------------ | 20260507 08:40:33.165 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:40:33.165 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:40:33.165 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:40:33.165 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:40:33.165 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.45e+01, 1.34e+01 / 9.68e-01 | 20260507 08:40:33.165 | INFO | PerfReport: 0.29%, AggregateStats -- 9.00e-02 / 1.00e-01, 9.62e-03 / 4.18e-03 | 20260507 08:40:33.165 | INFO | PerfReport: 11.64%, Baseline -- 3.57e+00 / 4.50e+00, 4.88e-01 / 1.95e-01 | 20260507 08:40:33.165 | INFO | PerfReport: 3.20%, Histogram -- 9.79e-01 / 1.47e+00, 1.56e-01 / 1.16e-01 | 20260507 08:40:33.165 | INFO | PerfReport: 16.36%, DME -- 4.01e+01 / 4.49e+01, 3.77e+00 / 1.15e+00 | 20260507 08:40:33.165 | INFO | PerfReport: 57.74%, FrameLabeling -- 1.77e+01 / 1.93e+01, 1.25e+00 / 1.08e+00 | 20260507 08:40:33.165 | INFO | PerfReport: 6.89%, PulseAccumulating -- 2.11e+00 / 2.69e+00, 3.87e-01 / 2.09e-01 | 20260507 08:40:33.165 | INFO | PerfReport: 2.90%, Metrics -- 8.89e-01 / 1.20e+00, 2.36e-01 / 1.48e-01 | 20260507 08:40:33.165 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.98e-01 / 3.62e-01, 4.93e-02 / 2.31e-02 | 20260507 08:40:33.165 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:40:33.165 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:40:33.165 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:40:33.166 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 08:40:33.166 | INFO | PerfReport: 22.00%, Upload -- 2.12e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 08:40:33.166 | INFO | PerfReport: 21.40%, Download -- 2.06e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 08:40:33.166 | INFO | PerfReport: 31.85%, Compute -- 3.06e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 08:40:33.166 | INFO | PerfReport: 1.63%, LockUpload -- 1.57e+00 / 0.00e+00, 5.02e+00 / 0.00e+00 | 20260507 08:40:33.166 | INFO | PerfReport: 1.26%, LockDownload -- 1.21e+00 / 0.00e+00, 2.89e+00 / 0.00e+00 | 20260507 08:40:33.166 | INFO | PerfReport: 21.84%, LockCompute -- 2.10e+01 / 0.00e+00, 1.85e+01 / 0.00e+00 | 20260507 08:40:33.166 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.69e-02 / 0.00e+00 | 20260507 08:40:33.261 | INFO | Starting a round of Baz IO | 20260507 08:40:36.356 | INFO | Populating RTMetrics report | 20260507 08:40:36.356 | INFO | RTMetrics report populated | 20260507 08:40:36.356 | INFO | Opening tmp RTMetrics json file | 20260507 08:40:36.370 | INFO | Serializing RTMetrics to json | 20260507 08:40:36.371 | INFO | Serialization done, swapping tmp file now... | 20260507 08:40:36.372 | INFO | RTMetrics done | 20260507 08:40:36.445 | INFO | Finished a round of Baz IO: Wrote 1.606 GiB in 3.184 seconds | 20260507 08:40:36.446 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.48% / 0.00% / 67.52% | 20260507 08:40:36.446 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0875592% / 3.97364e-06% / 7.94729e-05% | 20260507 08:40:36.446 | INFO | Baz IO overall: | 20260507 08:40:36.446 | INFO | BytesWritten : 4278245957829 | 20260507 08:40:36.446 | INFO | NumEvents : 2045985180603 | 20260507 08:40:38.307 | INFO | Analyzing chunk frames = [8601600,8602112) | 20260507 08:40:38.307 | INFO | MemInfo: { | 20260507 08:40:38.307 | INFO | "gpu" : " 17 GB", | 20260507 08:40:38.307 | INFO | "huge" : "", | 20260507 08:40:38.307 | INFO | "huge_shared" : " 27 GB", | 20260507 08:40:38.307 | INFO | "malloc" : " 14 GB", | 20260507 08:40:38.307 | INFO | "pinned" : " 29 GB" | 20260507 08:40:38.307 | INFO | } | 20260507 08:40:38.307 | INFO | Perf report for the last 5.14284 seconds: | 20260507 08:40:38.307 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:40:38.307 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:38.307 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:38.307 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:38.307 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 08:40:38.307 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:38.307 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260507 08:40:38.307 | INFO | PRE_HQ : 58.7%, 1.8, 54.3ms, 0 | 20260507 08:40:38.307 | INFO | BAZWRITER : 61.5%, 3.1, 97.4ms, 0 | 20260507 08:40:38.307 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 08:40:38.307 | INFO | Uploaded 36GiB to GPU in 2.06564s (17.428GiB/s) | 20260507 08:40:38.307 | INFO | Downloaded 38GiB to GPU in 2.17809s (17.4465GiB/s) | 20260507 08:40:43.455 | INFO | Analyzing chunk frames = [8602112,8602624) | 20260507 08:40:43.455 | INFO | MemInfo: { | 20260507 08:40:43.455 | INFO | "gpu" : " 17 GB", | 20260507 08:40:43.455 | INFO | "huge" : "", | 20260507 08:40:43.455 | INFO | "huge_shared" : " 27 GB", | 20260507 08:40:43.455 | INFO | "malloc" : " 14 GB", | 20260507 08:40:43.455 | INFO | "pinned" : " 29 GB" | 20260507 08:40:43.455 | INFO | } | 20260507 08:40:43.455 | INFO | Perf report for the last 5.1481 seconds: | 20260507 08:40:43.455 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:40:43.455 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:43.455 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:43.455 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:43.455 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 08:40:43.455 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:43.455 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:43.455 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260507 08:40:43.455 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:43.455 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:43.455 | INFO | Uploaded 36GiB to GPU in 2.00625s (17.9439GiB/s) | 20260507 08:40:43.455 | INFO | Downloaded 34GiB to GPU in 1.93788s (17.5449GiB/s) | 20260507 08:40:48.578 | INFO | Analyzing chunk frames = [8602624,8603136) | 20260507 08:40:48.578 | INFO | MemInfo: { | 20260507 08:40:48.578 | INFO | "gpu" : " 17 GB", | 20260507 08:40:48.578 | INFO | "huge" : "", | 20260507 08:40:48.578 | INFO | "huge_shared" : " 27 GB", | 20260507 08:40:48.578 | INFO | "malloc" : " 14 GB", | 20260507 08:40:48.578 | INFO | "pinned" : " 29 GB" | 20260507 08:40:48.578 | INFO | } | 20260507 08:40:48.578 | INFO | Perf report for the last 5.12325 seconds: | 20260507 08:40:48.578 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:40:48.578 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:48.578 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:48.578 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:48.578 | INFO | ANALYSIS : 60.6%, 3.0, 93.6ms, 0 | 20260507 08:40:48.578 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:48.578 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:48.578 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:40:48.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:48.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:48.579 | INFO | Uploaded 36GiB to GPU in 2.03919s (17.6541GiB/s) | 20260507 08:40:48.579 | INFO | Downloaded 34GiB to GPU in 1.91913s (17.7164GiB/s) | 20260507 08:40:53.724 | INFO | Analyzing chunk frames = [8603136,8603648) | 20260507 08:40:53.724 | INFO | MemInfo: { | 20260507 08:40:53.724 | INFO | "gpu" : " 17 GB", | 20260507 08:40:53.724 | INFO | "huge" : "", | 20260507 08:40:53.724 | INFO | "huge_shared" : " 27 GB", | 20260507 08:40:53.724 | INFO | "malloc" : " 14 GB", | 20260507 08:40:53.724 | INFO | "pinned" : " 29 GB" | 20260507 08:40:53.724 | INFO | } | 20260507 08:40:53.724 | INFO | Perf report for the last 5.14544 seconds: | 20260507 08:40:53.724 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:40:53.724 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:53.724 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:53.724 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:53.724 | INFO | ANALYSIS : 59.6%, 3.0, 92.5ms, 0 | 20260507 08:40:53.724 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:53.724 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:53.724 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:40:53.724 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:53.724 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:53.724 | INFO | Uploaded 36GiB to GPU in 2.01591s (17.8579GiB/s) | 20260507 08:40:53.724 | INFO | Downloaded 34GiB to GPU in 1.93694s (17.5534GiB/s) | 20260507 08:40:58.861 | INFO | Analyzing chunk frames = [8603648,8604160) | 20260507 08:40:58.861 | INFO | MemInfo: { | 20260507 08:40:58.861 | INFO | "gpu" : " 17 GB", | 20260507 08:40:58.861 | INFO | "huge" : "", | 20260507 08:40:58.861 | INFO | "huge_shared" : " 27 GB", | 20260507 08:40:58.861 | INFO | "malloc" : " 14 GB", | 20260507 08:40:58.861 | INFO | "pinned" : " 29 GB" | 20260507 08:40:58.861 | INFO | } | 20260507 08:40:58.861 | INFO | Perf report for the last 5.13713 seconds: | 20260507 08:40:58.861 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:40:58.861 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:58.861 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:58.861 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:58.861 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 08:40:58.861 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:58.861 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:58.861 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:40:58.861 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:58.861 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:40:58.861 | INFO | Uploaded 36GiB to GPU in 2.06348s (17.4463GiB/s) | 20260507 08:40:58.861 | INFO | Downloaded 34GiB to GPU in 1.94927s (17.4425GiB/s) | 20260507 08:41:04.006 | INFO | Analyzing chunk frames = [8604160,8604672) | 20260507 08:41:04.006 | INFO | MemInfo: { | 20260507 08:41:04.006 | INFO | "gpu" : " 17 GB", | 20260507 08:41:04.006 | INFO | "huge" : "", | 20260507 08:41:04.006 | INFO | "huge_shared" : " 27 GB", | 20260507 08:41:04.006 | INFO | "malloc" : " 14 GB", | 20260507 08:41:04.006 | INFO | "pinned" : " 29 GB" | 20260507 08:41:04.006 | INFO | } | 20260507 08:41:04.006 | INFO | Perf report for the last 5.14539 seconds: | 20260507 08:41:04.006 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:41:04.006 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:04.006 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:04.006 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:04.006 | INFO | ANALYSIS : 60.3%, 3.0, 94.3ms, 0 | 20260507 08:41:04.006 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:04.006 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:04.006 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 08:41:04.007 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:04.007 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:04.007 | INFO | Uploaded 36GiB to GPU in 2.0633s (17.4478GiB/s) | 20260507 08:41:04.007 | INFO | Downloaded 34GiB to GPU in 1.92643s (17.6492GiB/s) | 20260507 08:41:09.139 | INFO | Analyzing chunk frames = [8604672,8605184) | 20260507 08:41:09.139 | INFO | MemInfo: { | 20260507 08:41:09.139 | INFO | "gpu" : " 17 GB", | 20260507 08:41:09.139 | INFO | "huge" : "", | 20260507 08:41:09.139 | INFO | "huge_shared" : " 27 GB", | 20260507 08:41:09.139 | INFO | "malloc" : " 14 GB", | 20260507 08:41:09.139 | INFO | "pinned" : " 29 GB" | 20260507 08:41:09.139 | INFO | } | 20260507 08:41:09.139 | INFO | Perf report for the last 5.13303 seconds: | 20260507 08:41:09.139 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:41:09.139 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:09.139 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:09.139 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:09.140 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 08:41:09.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:09.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:09.140 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 08:41:09.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:09.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:09.140 | INFO | Uploaded 36GiB to GPU in 1.97073s (18.2673GiB/s) | 20260507 08:41:09.140 | INFO | Downloaded 34GiB to GPU in 1.99022s (17.0836GiB/s) | 20260507 08:41:14.279 | INFO | Analyzing chunk frames = [8605184,8605696) | 20260507 08:41:14.279 | INFO | MemInfo: { | 20260507 08:41:14.279 | INFO | "gpu" : " 17 GB", | 20260507 08:41:14.279 | INFO | "huge" : "", | 20260507 08:41:14.279 | INFO | "huge_shared" : " 27 GB", | 20260507 08:41:14.279 | INFO | "malloc" : " 14 GB", | 20260507 08:41:14.279 | INFO | "pinned" : " 29 GB" | 20260507 08:41:14.279 | INFO | } | 20260507 08:41:14.279 | INFO | Perf report for the last 5.13965 seconds: | 20260507 08:41:14.279 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:41:14.279 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:14.279 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:14.279 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:14.279 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:41:14.279 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:14.279 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:14.279 | INFO | PRE_HQ : 28.4%, 1.0, 14.8ms, 0 | 20260507 08:41:14.279 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:14.279 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:14.279 | INFO | Uploaded 36GiB to GPU in 1.98306s (18.1537GiB/s) | 20260507 08:41:14.279 | INFO | Downloaded 34GiB to GPU in 1.96081s (17.3398GiB/s) | 20260507 08:41:14.377 | INFO | Starting a round of Baz IO | 20260507 08:41:17.469 | INFO | Populating RTMetrics report | 20260507 08:41:17.469 | INFO | RTMetrics report populated | 20260507 08:41:17.469 | INFO | Opening tmp RTMetrics json file | 20260507 08:41:17.482 | INFO | Serializing RTMetrics to json | 20260507 08:41:17.483 | INFO | Serialization done, swapping tmp file now... | 20260507 08:41:17.484 | INFO | RTMetrics done | 20260507 08:41:17.551 | INFO | Finished a round of Baz IO: Wrote 1.605 GiB in 3.174 seconds | 20260507 08:41:17.551 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.48% / 0.00% / 67.52% | 20260507 08:41:17.551 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0871142% / 3.97364e-06% / 7.54992e-05% | 20260507 08:41:17.551 | INFO | Baz IO overall: | 20260507 08:41:17.551 | INFO | BytesWritten : 4279969513568 | 20260507 08:41:17.551 | INFO | NumEvents : 2046787611695 | 20260507 08:41:19.421 | INFO | Analyzing chunk frames = [8605696,8606208) | 20260507 08:41:19.421 | INFO | MemInfo: { | 20260507 08:41:19.421 | INFO | "gpu" : " 17 GB", | 20260507 08:41:19.421 | INFO | "huge" : "", | 20260507 08:41:19.421 | INFO | "huge_shared" : " 27 GB", | 20260507 08:41:19.421 | INFO | "malloc" : " 14 GB", | 20260507 08:41:19.421 | INFO | "pinned" : " 29 GB" | 20260507 08:41:19.421 | INFO | } | 20260507 08:41:19.421 | INFO | Perf report for the last 5.14205 seconds: | 20260507 08:41:19.421 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:41:19.421 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:19.421 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:19.421 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:19.421 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 08:41:19.421 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:19.421 | INFO | RT_METRICS : 41.2%, 1.0, 21.6ms, 0 | 20260507 08:41:19.421 | INFO | PRE_HQ : 58.9%, 1.8, 54.7ms, 0 | 20260507 08:41:19.421 | INFO | BAZWRITER : 61.3%, 3.1, 97.7ms, 0 | 20260507 08:41:19.421 | INFO | BAZ_SAMPLING : 0.6%, 1.0, 0.3ms, 0 | 20260507 08:41:19.421 | INFO | Uploaded 36GiB to GPU in 2.04551s (17.5996GiB/s) | 20260507 08:41:19.421 | INFO | Downloaded 38GiB to GPU in 2.18103s (17.4229GiB/s) | 20260507 08:41:24.559 | INFO | Analyzing chunk frames = [8606208,8606720) | 20260507 08:41:24.559 | INFO | MemInfo: { | 20260507 08:41:24.559 | INFO | "gpu" : " 17 GB", | 20260507 08:41:24.559 | INFO | "huge" : "", | 20260507 08:41:24.559 | INFO | "huge_shared" : " 27 GB", | 20260507 08:41:24.559 | INFO | "malloc" : " 14 GB", | 20260507 08:41:24.559 | INFO | "pinned" : " 29 GB" | 20260507 08:41:24.559 | INFO | } | 20260507 08:41:24.560 | INFO | Perf report for the last 5.13835 seconds: | 20260507 08:41:24.560 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:41:24.560 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:24.560 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:24.560 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:24.560 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 08:41:24.560 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:24.560 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:24.560 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260507 08:41:24.560 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:24.560 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:24.560 | INFO | Uploaded 36GiB to GPU in 1.9875s (18.1132GiB/s) | 20260507 08:41:24.560 | INFO | Downloaded 34GiB to GPU in 1.93265s (17.5924GiB/s) | 20260507 08:41:29.702 | INFO | Analyzing chunk frames = [8606720,8607232) | 20260507 08:41:29.702 | INFO | MemInfo: { | 20260507 08:41:29.702 | INFO | "gpu" : " 17 GB", | 20260507 08:41:29.702 | INFO | "huge" : "", | 20260507 08:41:29.702 | INFO | "huge_shared" : " 27 GB", | 20260507 08:41:29.702 | INFO | "malloc" : " 14 GB", | 20260507 08:41:29.702 | INFO | "pinned" : " 29 GB" | 20260507 08:41:29.702 | INFO | } | 20260507 08:41:29.702 | INFO | Perf report for the last 5.14219 seconds: | 20260507 08:41:29.702 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:41:29.702 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:29.702 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:29.702 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:29.702 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 08:41:29.702 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:29.702 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:29.702 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:41:29.702 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:29.702 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:29.702 | INFO | Uploaded 36GiB to GPU in 2.02555s (17.7729GiB/s) | 20260507 08:41:29.702 | INFO | Downloaded 34GiB to GPU in 1.92369s (17.6744GiB/s) | 20260507 08:41:34.834 | INFO | Analyzing chunk frames = [8607232,8607744) | 20260507 08:41:34.834 | INFO | MemInfo: { | 20260507 08:41:34.834 | INFO | "gpu" : " 17 GB", | 20260507 08:41:34.834 | INFO | "huge" : "", | 20260507 08:41:34.834 | INFO | "huge_shared" : " 27 GB", | 20260507 08:41:34.834 | INFO | "malloc" : " 14 GB", | 20260507 08:41:34.834 | INFO | "pinned" : " 29 GB" | 20260507 08:41:34.834 | INFO | } | 20260507 08:41:34.835 | INFO | Perf report for the last 5.13279 seconds: | 20260507 08:41:34.835 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:41:34.835 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:34.835 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:34.835 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:34.835 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 08:41:34.835 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:34.835 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:34.835 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:41:34.835 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:34.835 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:34.835 | INFO | Uploaded 36GiB to GPU in 2.05416s (17.5254GiB/s) | 20260507 08:41:34.835 | INFO | Downloaded 34GiB to GPU in 1.94912s (17.4438GiB/s) | 20260507 08:41:39.980 | INFO | Analyzing chunk frames = [8607744,8608256) | 20260507 08:41:39.980 | INFO | MemInfo: { | 20260507 08:41:39.980 | INFO | "gpu" : " 17 GB", | 20260507 08:41:39.980 | INFO | "huge" : "", | 20260507 08:41:39.980 | INFO | "huge_shared" : " 27 GB", | 20260507 08:41:39.980 | INFO | "malloc" : " 14 GB", | 20260507 08:41:39.980 | INFO | "pinned" : " 29 GB" | 20260507 08:41:39.980 | INFO | } | 20260507 08:41:39.980 | INFO | Perf report for the last 5.14547 seconds: | 20260507 08:41:39.980 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:41:39.980 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:39.980 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:39.980 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:39.980 | INFO | ANALYSIS : 59.9%, 3.0, 93.6ms, 0 | 20260507 08:41:39.980 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:39.980 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:39.980 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:41:39.980 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:39.980 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:39.980 | INFO | Uploaded 36GiB to GPU in 2.04296s (17.6215GiB/s) | 20260507 08:41:39.980 | INFO | Downloaded 34GiB to GPU in 1.91294s (17.7737GiB/s) | 20260507 08:41:45.116 | INFO | Analyzing chunk frames = [8608256,8608768) | 20260507 08:41:45.116 | INFO | MemInfo: { | 20260507 08:41:45.116 | INFO | "gpu" : " 17 GB", | 20260507 08:41:45.116 | INFO | "huge" : "", | 20260507 08:41:45.116 | INFO | "huge_shared" : " 27 GB", | 20260507 08:41:45.116 | INFO | "malloc" : " 14 GB", | 20260507 08:41:45.116 | INFO | "pinned" : " 29 GB" | 20260507 08:41:45.116 | INFO | } | 20260507 08:41:45.116 | INFO | Perf report for the last 5.13579 seconds: | 20260507 08:41:45.116 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:41:45.116 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:45.116 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:45.116 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:45.116 | INFO | ANALYSIS : 60.3%, 3.0, 94.1ms, 0 | 20260507 08:41:45.116 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:45.116 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:45.116 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:41:45.116 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:45.116 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:45.116 | INFO | Uploaded 36GiB to GPU in 2.05777s (17.4946GiB/s) | 20260507 08:41:45.116 | INFO | Downloaded 34GiB to GPU in 1.92896s (17.6261GiB/s) | 20260507 08:41:50.258 | INFO | Analyzing chunk frames = [8608768,8609280) | 20260507 08:41:50.258 | INFO | MemInfo: { | 20260507 08:41:50.258 | INFO | "gpu" : " 17 GB", | 20260507 08:41:50.258 | INFO | "huge" : "", | 20260507 08:41:50.258 | INFO | "huge_shared" : " 27 GB", | 20260507 08:41:50.258 | INFO | "malloc" : " 14 GB", | 20260507 08:41:50.258 | INFO | "pinned" : " 29 GB" | 20260507 08:41:50.258 | INFO | } | 20260507 08:41:50.258 | INFO | Perf report for the last 5.1421 seconds: | 20260507 08:41:50.258 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:41:50.258 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:50.258 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:50.258 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:50.258 | INFO | ANALYSIS : 61.0%, 3.0, 95.0ms, 0 | 20260507 08:41:50.258 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:50.258 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:50.258 | INFO | PRE_HQ : 24.9%, 1.0, 13.0ms, 0 | 20260507 08:41:50.258 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:50.258 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:50.258 | INFO | Uploaded 36GiB to GPU in 2.03778s (17.6663GiB/s) | 20260507 08:41:50.258 | INFO | Downloaded 34GiB to GPU in 1.98881s (17.0956GiB/s) | 20260507 08:41:55.392 | INFO | Analyzing chunk frames = [8609280,8609792) | 20260507 08:41:55.392 | INFO | MemInfo: { | 20260507 08:41:55.392 | INFO | "gpu" : " 17 GB", | 20260507 08:41:55.392 | INFO | "huge" : "", | 20260507 08:41:55.392 | INFO | "huge_shared" : " 27 GB", | 20260507 08:41:55.392 | INFO | "malloc" : " 14 GB", | 20260507 08:41:55.392 | INFO | "pinned" : " 29 GB" | 20260507 08:41:55.392 | INFO | } | 20260507 08:41:55.392 | INFO | Perf report for the last 5.13441 seconds: | 20260507 08:41:55.392 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:41:55.392 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:55.392 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:55.392 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:55.392 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 08:41:55.392 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:55.392 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:55.392 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:41:55.392 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:55.392 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:41:55.393 | INFO | Uploaded 36GiB to GPU in 1.99167s (18.0753GiB/s) | 20260507 08:41:55.393 | INFO | Downloaded 34GiB to GPU in 1.95813s (17.3635GiB/s) | 20260507 08:41:55.393 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:41:55.393 | INFO | Total mem usage: 18 GiB | 20260507 08:41:55.393 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:41:55.393 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -27.6%) | 20260507 08:41:55.393 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:41:55.393 | INFO | High Water Breakdown: | 20260507 08:41:55.393 | INFO | 60% : 8.46 GiB ( 74.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:41:55.393 | INFO | 39.9% : 5.62 GiB ( 84% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:41:55.393 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:41:55.393 | INFO | ------------------------------------------ | 20260507 08:41:55.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:41:55.393 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:41:55.393 | INFO | Total mem usage: 26.5 GiB | 20260507 08:41:55.393 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:41:55.393 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:41:55.393 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:41:55.393 | INFO | High Water Breakdown: | 20260507 08:41:55.393 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:41:55.393 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:41:55.393 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:41:55.393 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:41:55.393 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:41:55.393 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:41:55.393 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:41:55.393 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:41:55.393 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:41:55.393 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:41:55.393 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:41:55.393 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:41:55.393 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:41:55.393 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:41:55.393 | INFO | ------------------------------------------ | 20260507 08:41:55.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:41:55.393 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:41:55.393 | INFO | Total mem usage: 25.6 GiB | 20260507 08:41:55.393 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:41:55.393 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:41:55.393 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:41:55.393 | INFO | High Water Breakdown: | 20260507 08:41:55.393 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:41:55.393 | INFO | ------------------------------------------ | 20260507 08:41:55.393 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:41:55.393 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:41:55.394 | INFO | Total mem usage: 15.7 GiB | 20260507 08:41:55.394 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:41:55.394 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:41:55.394 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:41:55.394 | INFO | High Water Breakdown: | 20260507 08:41:55.394 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:41:55.394 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:41:55.394 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:41:55.394 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:41:55.394 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:41:55.394 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:41:55.394 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:41:55.394 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:41:55.394 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:41:55.394 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:41:55.394 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:41:55.394 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:41:55.394 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:41:55.394 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:41:55.394 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:41:55.394 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:41:55.394 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:41:55.394 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:41:55.394 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:41:55.394 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:41:55.394 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:41:55.394 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:41:55.394 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:41:55.394 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:41:55.394 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:41:55.394 | INFO | ------------------------------------------ | 20260507 08:41:55.394 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:41:55.394 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:41:55.394 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:41:55.394 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:41:55.394 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.42e+01, 1.35e+01 / 1.05e+00 | 20260507 08:41:55.394 | INFO | PerfReport: 0.30%, AggregateStats -- 9.07e-02 / 1.01e-01, 9.68e-03 / 4.15e-03 | 20260507 08:41:55.394 | INFO | PerfReport: 11.65%, Baseline -- 3.57e+00 / 4.49e+00, 4.98e-01 / 1.90e-01 | 20260507 08:41:55.394 | INFO | PerfReport: 3.22%, Histogram -- 9.86e-01 / 1.48e+00, 1.65e-01 / 1.13e-01 | 20260507 08:41:55.394 | INFO | PerfReport: 16.27%, DME -- 3.99e+01 / 4.43e+01, 3.76e+00 / 1.12e+00 | 20260507 08:41:55.394 | INFO | PerfReport: 57.81%, FrameLabeling -- 1.77e+01 / 1.96e+01, 1.33e+00 / 1.12e+00 | 20260507 08:41:55.394 | INFO | PerfReport: 6.88%, PulseAccumulating -- 2.11e+00 / 2.70e+00, 3.85e-01 / 1.53e-01 | 20260507 08:41:55.394 | INFO | PerfReport: 2.90%, Metrics -- 8.90e-01 / 1.21e+00, 2.41e-01 / 1.47e-01 | 20260507 08:41:55.394 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.98e-01 / 3.64e-01, 4.90e-02 / 2.39e-02 | 20260507 08:41:55.394 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:41:55.394 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:41:55.394 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:41:55.394 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 08:41:55.394 | INFO | PerfReport: 21.97%, Upload -- 2.11e+01 / 0.00e+00, 7.29e+00 / 0.00e+00 | 20260507 08:41:55.394 | INFO | PerfReport: 21.37%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 08:41:55.394 | INFO | PerfReport: 31.88%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 08:41:55.394 | INFO | PerfReport: 1.62%, LockUpload -- 1.55e+00 / 0.00e+00, 5.03e+00 / 0.00e+00 | 20260507 08:41:55.394 | INFO | PerfReport: 1.22%, LockDownload -- 1.17e+00 / 0.00e+00, 2.86e+00 / 0.00e+00 | 20260507 08:41:55.394 | INFO | PerfReport: 21.93%, LockCompute -- 2.11e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 08:41:55.394 | INFO | PerfReport: 0.01%, Misc -- 1.36e-02 / 0.00e+00, 1.66e-02 / 0.00e+00 | 20260507 08:41:55.489 | INFO | Starting a round of Baz IO | 20260507 08:41:58.582 | INFO | Populating RTMetrics report | 20260507 08:41:58.582 | INFO | RTMetrics report populated | 20260507 08:41:58.582 | INFO | Opening tmp RTMetrics json file | 20260507 08:41:58.596 | INFO | Serializing RTMetrics to json | 20260507 08:41:58.597 | INFO | Serialization done, swapping tmp file now... | 20260507 08:41:58.598 | INFO | RTMetrics done | 20260507 08:41:58.664 | INFO | Finished a round of Baz IO: Wrote 1.606 GiB in 3.174 seconds | 20260507 08:41:58.664 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.48% / 0.00% / 67.52% | 20260507 08:41:58.664 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0871857% / 0% / 6.75519e-05% | 20260507 08:41:58.664 | INFO | Baz IO overall: | 20260507 08:41:58.664 | INFO | BytesWritten : 4281694343576 | 20260507 08:41:58.664 | INFO | NumEvents : 2047590976298 | 20260507 08:42:00.530 | INFO | Analyzing chunk frames = [8609792,8610304) | 20260507 08:42:00.530 | INFO | MemInfo: { | 20260507 08:42:00.530 | INFO | "gpu" : " 17 GB", | 20260507 08:42:00.530 | INFO | "huge" : "", | 20260507 08:42:00.530 | INFO | "huge_shared" : " 27 GB", | 20260507 08:42:00.530 | INFO | "malloc" : " 14 GB", | 20260507 08:42:00.530 | INFO | "pinned" : " 29 GB" | 20260507 08:42:00.530 | INFO | } | 20260507 08:42:00.530 | INFO | Perf report for the last 5.13794 seconds: | 20260507 08:42:00.530 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:42:00.530 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:00.530 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:00.530 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:00.530 | INFO | ANALYSIS : 61.6%, 3.0, 95.8ms, 0 | 20260507 08:42:00.530 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:00.530 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260507 08:42:00.530 | INFO | PRE_HQ : 58.8%, 1.8, 54.0ms, 0 | 20260507 08:42:00.530 | INFO | BAZWRITER : 61.4%, 3.0, 96.0ms, 0 | 20260507 08:42:00.530 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:42:00.531 | INFO | Uploaded 36GiB to GPU in 2.06752s (17.4122GiB/s) | 20260507 08:42:00.531 | INFO | Downloaded 38GiB to GPU in 2.16681s (17.5373GiB/s) | 20260507 08:42:05.674 | INFO | Analyzing chunk frames = [8610304,8610816) | 20260507 08:42:05.674 | INFO | MemInfo: { | 20260507 08:42:05.674 | INFO | "gpu" : " 17 GB", | 20260507 08:42:05.674 | INFO | "huge" : "", | 20260507 08:42:05.674 | INFO | "huge_shared" : " 27 GB", | 20260507 08:42:05.674 | INFO | "malloc" : " 14 GB", | 20260507 08:42:05.674 | INFO | "pinned" : " 29 GB" | 20260507 08:42:05.675 | INFO | } | 20260507 08:42:05.675 | INFO | Perf report for the last 5.14424 seconds: | 20260507 08:42:05.675 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:42:05.675 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:05.675 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:05.675 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:05.675 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 08:42:05.675 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:05.675 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:05.675 | INFO | PRE_HQ : 23.9%, 1.0, 12.4ms, 0 | 20260507 08:42:05.675 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:05.675 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:05.675 | INFO | Uploaded 36GiB to GPU in 2.00479s (17.957GiB/s) | 20260507 08:42:05.675 | INFO | Downloaded 34GiB to GPU in 1.93068s (17.6103GiB/s) | 20260507 08:42:10.823 | INFO | Analyzing chunk frames = [8610816,8611328) | 20260507 08:42:10.823 | INFO | MemInfo: { | 20260507 08:42:10.823 | INFO | "gpu" : " 17 GB", | 20260507 08:42:10.823 | INFO | "huge" : "", | 20260507 08:42:10.823 | INFO | "huge_shared" : " 27 GB", | 20260507 08:42:10.823 | INFO | "malloc" : " 14 GB", | 20260507 08:42:10.823 | INFO | "pinned" : " 29 GB" | 20260507 08:42:10.823 | INFO | } | 20260507 08:42:10.823 | INFO | Perf report for the last 5.14817 seconds: | 20260507 08:42:10.823 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:42:10.823 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:10.823 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:10.823 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:10.823 | INFO | ANALYSIS : 60.6%, 3.0, 93.9ms, 0 | 20260507 08:42:10.823 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:10.823 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:10.823 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 08:42:10.823 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:10.823 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:10.823 | INFO | Uploaded 36GiB to GPU in 2.06305s (17.4499GiB/s) | 20260507 08:42:10.823 | INFO | Downloaded 34GiB to GPU in 1.94938s (17.4415GiB/s) | 20260507 08:42:15.959 | INFO | Analyzing chunk frames = [8611328,8611840) | 20260507 08:42:15.959 | INFO | MemInfo: { | 20260507 08:42:15.959 | INFO | "gpu" : " 17 GB", | 20260507 08:42:15.959 | INFO | "huge" : "", | 20260507 08:42:15.959 | INFO | "huge_shared" : " 27 GB", | 20260507 08:42:15.959 | INFO | "malloc" : " 14 GB", | 20260507 08:42:15.959 | INFO | "pinned" : " 29 GB" | 20260507 08:42:15.959 | INFO | } | 20260507 08:42:15.959 | INFO | Perf report for the last 5.13674 seconds: | 20260507 08:42:15.959 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:42:15.959 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:15.959 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:15.959 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:15.959 | INFO | ANALYSIS : 60.4%, 2.9, 93.2ms, 0 | 20260507 08:42:15.959 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:15.960 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:15.960 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:42:15.960 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:15.960 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:15.960 | INFO | Uploaded 36GiB to GPU in 2.05577s (17.5117GiB/s) | 20260507 08:42:15.960 | INFO | Downloaded 34GiB to GPU in 1.90981s (17.8028GiB/s) | 20260507 08:42:21.085 | INFO | Analyzing chunk frames = [8611840,8612352) | 20260507 08:42:21.085 | INFO | MemInfo: { | 20260507 08:42:21.085 | INFO | "gpu" : " 17 GB", | 20260507 08:42:21.085 | INFO | "huge" : "", | 20260507 08:42:21.085 | INFO | "huge_shared" : " 27 GB", | 20260507 08:42:21.085 | INFO | "malloc" : " 14 GB", | 20260507 08:42:21.085 | INFO | "pinned" : " 29 GB" | 20260507 08:42:21.085 | INFO | } | 20260507 08:42:21.085 | INFO | Perf report for the last 5.12565 seconds: | 20260507 08:42:21.085 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:42:21.085 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:21.085 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:21.085 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:21.085 | INFO | ANALYSIS : 60.3%, 3.0, 93.6ms, 0 | 20260507 08:42:21.085 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:21.085 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:21.085 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 08:42:21.085 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:21.085 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:21.085 | INFO | Uploaded 36GiB to GPU in 2.06847s (17.4042GiB/s) | 20260507 08:42:21.085 | INFO | Downloaded 34GiB to GPU in 1.90755s (17.8239GiB/s) | 20260507 08:42:26.234 | INFO | Analyzing chunk frames = [8612352,8612864) | 20260507 08:42:26.234 | INFO | MemInfo: { | 20260507 08:42:26.234 | INFO | "gpu" : " 17 GB", | 20260507 08:42:26.234 | INFO | "huge" : "", | 20260507 08:42:26.234 | INFO | "huge_shared" : " 27 GB", | 20260507 08:42:26.234 | INFO | "malloc" : " 14 GB", | 20260507 08:42:26.234 | INFO | "pinned" : " 29 GB" | 20260507 08:42:26.234 | INFO | } | 20260507 08:42:26.234 | INFO | Perf report for the last 5.14857 seconds: | 20260507 08:42:26.234 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:42:26.234 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:26.234 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:26.234 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:26.234 | INFO | ANALYSIS : 60.3%, 3.0, 94.2ms, 0 | 20260507 08:42:26.234 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:26.234 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:26.234 | INFO | PRE_HQ : 28.1%, 1.0, 14.6ms, 0 | 20260507 08:42:26.234 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:26.234 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:26.234 | INFO | Uploaded 36GiB to GPU in 2.04687s (17.5879GiB/s) | 20260507 08:42:26.234 | INFO | Downloaded 34GiB to GPU in 1.90721s (17.8271GiB/s) | 20260507 08:42:31.373 | INFO | Analyzing chunk frames = [8612864,8613376) | 20260507 08:42:31.373 | INFO | MemInfo: { | 20260507 08:42:31.373 | INFO | "gpu" : " 17 GB", | 20260507 08:42:31.373 | INFO | "huge" : "", | 20260507 08:42:31.373 | INFO | "huge_shared" : " 27 GB", | 20260507 08:42:31.373 | INFO | "malloc" : " 14 GB", | 20260507 08:42:31.373 | INFO | "pinned" : " 29 GB" | 20260507 08:42:31.373 | INFO | } | 20260507 08:42:31.373 | INFO | Perf report for the last 5.13943 seconds: | 20260507 08:42:31.373 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:42:31.373 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:31.373 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:31.373 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:31.373 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:42:31.373 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:31.373 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:31.373 | INFO | PRE_HQ : 25.6%, 1.0, 13.3ms, 0 | 20260507 08:42:31.373 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:31.373 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:31.374 | INFO | Uploaded 36GiB to GPU in 1.93456s (18.6089GiB/s) | 20260507 08:42:31.374 | INFO | Downloaded 34GiB to GPU in 1.96944s (17.2638GiB/s) | 20260507 08:42:36.512 | INFO | Analyzing chunk frames = [8613376,8613888) | 20260507 08:42:36.512 | INFO | MemInfo: { | 20260507 08:42:36.512 | INFO | "gpu" : " 17 GB", | 20260507 08:42:36.512 | INFO | "huge" : "", | 20260507 08:42:36.512 | INFO | "huge_shared" : " 27 GB", | 20260507 08:42:36.512 | INFO | "malloc" : " 14 GB", | 20260507 08:42:36.512 | INFO | "pinned" : " 29 GB" | 20260507 08:42:36.512 | INFO | } | 20260507 08:42:36.513 | INFO | Perf report for the last 5.13931 seconds: | 20260507 08:42:36.513 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:42:36.513 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:36.513 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:36.513 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:36.513 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 08:42:36.513 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:36.513 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:36.513 | INFO | PRE_HQ : 25.4%, 1.0, 13.2ms, 0 | 20260507 08:42:36.513 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:36.513 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:36.513 | INFO | Uploaded 36GiB to GPU in 1.9792s (18.1892GiB/s) | 20260507 08:42:36.513 | INFO | Downloaded 34GiB to GPU in 1.96114s (17.3369GiB/s) | 20260507 08:42:36.609 | INFO | Starting a round of Baz IO | 20260507 08:42:39.694 | INFO | Populating RTMetrics report | 20260507 08:42:39.694 | INFO | RTMetrics report populated | 20260507 08:42:39.694 | INFO | Opening tmp RTMetrics json file | 20260507 08:42:39.703 | INFO | Serializing RTMetrics to json | 20260507 08:42:39.709 | INFO | Serialization done, swapping tmp file now... | 20260507 08:42:39.709 | INFO | RTMetrics done | 20260507 08:42:39.774 | INFO | Finished a round of Baz IO: Wrote 1.606 GiB in 3.164 seconds | 20260507 08:42:39.774 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.48% / 0.00% / 67.52% | 20260507 08:42:39.774 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0873248% / 7.94729e-06% / 9.53674e-05% | 20260507 08:42:39.774 | INFO | Baz IO overall: | 20260507 08:42:39.774 | INFO | BytesWritten : 4283418458703 | 20260507 08:42:39.774 | INFO | NumEvents : 2048393755169 | 20260507 08:42:41.651 | INFO | Analyzing chunk frames = [8613888,8614400) | 20260507 08:42:41.651 | INFO | MemInfo: { | 20260507 08:42:41.651 | INFO | "gpu" : " 17 GB", | 20260507 08:42:41.651 | INFO | "huge" : "", | 20260507 08:42:41.651 | INFO | "huge_shared" : " 27 GB", | 20260507 08:42:41.651 | INFO | "malloc" : " 14 GB", | 20260507 08:42:41.651 | INFO | "pinned" : " 29 GB" | 20260507 08:42:41.651 | INFO | } | 20260507 08:42:41.651 | INFO | Perf report for the last 5.13892 seconds: | 20260507 08:42:41.651 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:42:41.651 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:41.651 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:41.652 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:41.652 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260507 08:42:41.652 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:41.652 | INFO | RT_METRICS : 41.0%, 1.0, 21.5ms, 0 | 20260507 08:42:41.652 | INFO | PRE_HQ : 58.8%, 1.8, 54.6ms, 0 | 20260507 08:42:41.652 | INFO | BAZWRITER : 61.1%, 3.0, 94.9ms, 0 | 20260507 08:42:41.652 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:42:41.652 | INFO | Uploaded 36GiB to GPU in 2.06039s (17.4724GiB/s) | 20260507 08:42:41.652 | INFO | Downloaded 38GiB to GPU in 2.17146s (17.4998GiB/s) | 20260507 08:42:46.789 | INFO | Analyzing chunk frames = [8614400,8614912) | 20260507 08:42:46.789 | INFO | MemInfo: { | 20260507 08:42:46.789 | INFO | "gpu" : " 17 GB", | 20260507 08:42:46.789 | INFO | "huge" : "", | 20260507 08:42:46.789 | INFO | "huge_shared" : " 27 GB", | 20260507 08:42:46.789 | INFO | "malloc" : " 14 GB", | 20260507 08:42:46.789 | INFO | "pinned" : " 29 GB" | 20260507 08:42:46.789 | INFO | } | 20260507 08:42:46.789 | INFO | Perf report for the last 5.13786 seconds: | 20260507 08:42:46.789 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:42:46.789 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:46.789 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:46.789 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:46.789 | INFO | ANALYSIS : 60.7%, 3.0, 94.4ms, 0 | 20260507 08:42:46.789 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:46.789 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:46.789 | INFO | PRE_HQ : 24.4%, 1.0, 12.6ms, 0 | 20260507 08:42:46.789 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:46.789 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:46.790 | INFO | Uploaded 36GiB to GPU in 1.97981s (18.1835GiB/s) | 20260507 08:42:46.790 | INFO | Downloaded 34GiB to GPU in 1.93912s (17.5337GiB/s) | 20260507 08:42:51.937 | INFO | Analyzing chunk frames = [8614912,8615424) | 20260507 08:42:51.937 | INFO | MemInfo: { | 20260507 08:42:51.937 | INFO | "gpu" : " 17 GB", | 20260507 08:42:51.937 | INFO | "huge" : "", | 20260507 08:42:51.937 | INFO | "huge_shared" : " 27 GB", | 20260507 08:42:51.937 | INFO | "malloc" : " 14 GB", | 20260507 08:42:51.937 | INFO | "pinned" : " 29 GB" | 20260507 08:42:51.937 | INFO | } | 20260507 08:42:51.937 | INFO | Perf report for the last 5.14784 seconds: | 20260507 08:42:51.937 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:42:51.937 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:51.937 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:51.937 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:51.937 | INFO | ANALYSIS : 60.5%, 3.0, 93.8ms, 0 | 20260507 08:42:51.937 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:51.937 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:51.937 | INFO | PRE_HQ : 24.2%, 1.0, 12.6ms, 0 | 20260507 08:42:51.937 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:51.937 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:51.937 | INFO | Uploaded 36GiB to GPU in 2.05763s (17.4959GiB/s) | 20260507 08:42:51.938 | INFO | Downloaded 34GiB to GPU in 1.95598s (17.3826GiB/s) | 20260507 08:42:57.069 | INFO | Analyzing chunk frames = [8615424,8615936) | 20260507 08:42:57.069 | INFO | MemInfo: { | 20260507 08:42:57.069 | INFO | "gpu" : " 17 GB", | 20260507 08:42:57.069 | INFO | "huge" : "", | 20260507 08:42:57.069 | INFO | "huge_shared" : " 27 GB", | 20260507 08:42:57.069 | INFO | "malloc" : " 14 GB", | 20260507 08:42:57.069 | INFO | "pinned" : " 29 GB" | 20260507 08:42:57.069 | INFO | } | 20260507 08:42:57.069 | INFO | Perf report for the last 5.13193 seconds: | 20260507 08:42:57.069 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:42:57.069 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:57.069 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:57.069 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:57.069 | INFO | ANALYSIS : 60.6%, 2.9, 93.4ms, 0 | 20260507 08:42:57.069 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:57.069 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:57.069 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 08:42:57.069 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:57.069 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:42:57.069 | INFO | Uploaded 36GiB to GPU in 2.06069s (17.4698GiB/s) | 20260507 08:42:57.069 | INFO | Downloaded 34GiB to GPU in 1.95485s (17.3926GiB/s) | 20260507 08:43:02.204 | INFO | Analyzing chunk frames = [8615936,8616448) | 20260507 08:43:02.204 | INFO | MemInfo: { | 20260507 08:43:02.204 | INFO | "gpu" : " 17 GB", | 20260507 08:43:02.204 | INFO | "huge" : "", | 20260507 08:43:02.204 | INFO | "huge_shared" : " 27 GB", | 20260507 08:43:02.204 | INFO | "malloc" : " 14 GB", | 20260507 08:43:02.204 | INFO | "pinned" : " 29 GB" | 20260507 08:43:02.204 | INFO | } | 20260507 08:43:02.204 | INFO | Perf report for the last 5.1353 seconds: | 20260507 08:43:02.204 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:43:02.204 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:02.205 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:02.205 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:02.205 | INFO | ANALYSIS : 60.0%, 3.0, 93.4ms, 0 | 20260507 08:43:02.205 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:02.205 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:02.205 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:43:02.205 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:02.205 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:02.205 | INFO | Uploaded 36GiB to GPU in 2.0352s (17.6887GiB/s) | 20260507 08:43:02.205 | INFO | Downloaded 34GiB to GPU in 1.90516s (17.8462GiB/s) | 20260507 08:43:07.341 | INFO | Analyzing chunk frames = [8616448,8616960) | 20260507 08:43:07.341 | INFO | MemInfo: { | 20260507 08:43:07.341 | INFO | "gpu" : " 17 GB", | 20260507 08:43:07.341 | INFO | "huge" : "", | 20260507 08:43:07.342 | INFO | "huge_shared" : " 27 GB", | 20260507 08:43:07.342 | INFO | "malloc" : " 14 GB", | 20260507 08:43:07.342 | INFO | "pinned" : " 29 GB" | 20260507 08:43:07.342 | INFO | } | 20260507 08:43:07.342 | INFO | Perf report for the last 5.13721 seconds: | 20260507 08:43:07.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:43:07.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:07.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:07.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:07.342 | INFO | ANALYSIS : 60.4%, 3.0, 94.3ms, 0 | 20260507 08:43:07.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:07.342 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:07.342 | INFO | PRE_HQ : 28.2%, 1.0, 14.6ms, 0 | 20260507 08:43:07.342 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:07.342 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:07.342 | INFO | Uploaded 36GiB to GPU in 2.06081s (17.4689GiB/s) | 20260507 08:43:07.342 | INFO | Downloaded 34GiB to GPU in 1.93829s (17.5412GiB/s) | 20260507 08:43:12.484 | INFO | Analyzing chunk frames = [8616960,8617472) | 20260507 08:43:12.484 | INFO | MemInfo: { | 20260507 08:43:12.484 | INFO | "gpu" : " 17 GB", | 20260507 08:43:12.484 | INFO | "huge" : "", | 20260507 08:43:12.484 | INFO | "huge_shared" : " 27 GB", | 20260507 08:43:12.485 | INFO | "malloc" : " 14 GB", | 20260507 08:43:12.485 | INFO | "pinned" : " 29 GB" | 20260507 08:43:12.485 | INFO | } | 20260507 08:43:12.485 | INFO | Perf report for the last 5.143 seconds: | 20260507 08:43:12.485 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:43:12.485 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:12.485 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:12.485 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:12.485 | INFO | ANALYSIS : 60.8%, 3.0, 94.7ms, 0 | 20260507 08:43:12.485 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:12.485 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:12.485 | INFO | PRE_HQ : 25.7%, 1.0, 13.3ms, 0 | 20260507 08:43:12.485 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:12.485 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:12.485 | INFO | Uploaded 36GiB to GPU in 2.03398s (17.6993GiB/s) | 20260507 08:43:12.485 | INFO | Downloaded 34GiB to GPU in 1.98599s (17.1199GiB/s) | 20260507 08:43:17.628 | INFO | Analyzing chunk frames = [8617472,8617984) | 20260507 08:43:17.628 | INFO | MemInfo: { | 20260507 08:43:17.628 | INFO | "gpu" : " 17 GB", | 20260507 08:43:17.628 | INFO | "huge" : "", | 20260507 08:43:17.628 | INFO | "huge_shared" : " 27 GB", | 20260507 08:43:17.628 | INFO | "malloc" : " 14 GB", | 20260507 08:43:17.628 | INFO | "pinned" : " 29 GB" | 20260507 08:43:17.628 | INFO | } | 20260507 08:43:17.628 | INFO | Perf report for the last 5.14355 seconds: | 20260507 08:43:17.628 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:43:17.628 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:17.628 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:17.628 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:17.628 | INFO | ANALYSIS : 60.5%, 3.0, 94.3ms, 0 | 20260507 08:43:17.628 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:17.628 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:17.628 | INFO | PRE_HQ : 25.1%, 1.0, 13.0ms, 0 | 20260507 08:43:17.628 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:17.628 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:17.629 | INFO | Uploaded 36GiB to GPU in 1.98572s (18.1295GiB/s) | 20260507 08:43:17.629 | INFO | Downloaded 34GiB to GPU in 1.96149s (17.3338GiB/s) | 20260507 08:43:17.629 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:43:17.629 | INFO | Total mem usage: 18 GiB | 20260507 08:43:17.629 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:43:17.629 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -27.6%) | 20260507 08:43:17.629 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:43:17.629 | INFO | High Water Breakdown: | 20260507 08:43:17.629 | INFO | 60% : 8.46 GiB ( 74.1% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:43:17.629 | INFO | 39.9% : 5.62 GiB ( 84% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:43:17.629 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:43:17.629 | INFO | ------------------------------------------ | 20260507 08:43:17.629 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:43:17.629 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:43:17.629 | INFO | Total mem usage: 26.5 GiB | 20260507 08:43:17.629 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:43:17.629 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:43:17.629 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:43:17.629 | INFO | High Water Breakdown: | 20260507 08:43:17.629 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:43:17.629 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:43:17.629 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:43:17.629 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:43:17.629 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:43:17.629 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:43:17.629 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:43:17.629 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:43:17.629 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:43:17.629 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:43:17.629 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:43:17.629 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:43:17.629 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:43:17.629 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:43:17.629 | INFO | ------------------------------------------ | 20260507 08:43:17.629 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:43:17.629 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:43:17.629 | INFO | Total mem usage: 25.6 GiB | 20260507 08:43:17.629 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:43:17.629 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:43:17.629 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:43:17.629 | INFO | High Water Breakdown: | 20260507 08:43:17.629 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:43:17.629 | INFO | ------------------------------------------ | 20260507 08:43:17.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:43:17.630 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:43:17.630 | INFO | Total mem usage: 15.7 GiB | 20260507 08:43:17.630 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:43:17.630 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:43:17.630 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:43:17.630 | INFO | High Water Breakdown: | 20260507 08:43:17.630 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:43:17.630 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:43:17.630 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:43:17.630 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:43:17.630 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:43:17.630 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:43:17.630 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:43:17.630 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:43:17.630 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:43:17.630 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:43:17.630 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:43:17.630 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:43:17.630 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:43:17.630 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:43:17.630 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:43:17.630 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:43:17.630 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:43:17.630 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:43:17.630 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:43:17.630 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:43:17.630 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:43:17.630 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:43:17.630 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:43:17.630 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:43:17.630 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:43:17.630 | INFO | ------------------------------------------ | 20260507 08:43:17.630 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:43:17.630 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:43:17.630 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:43:17.630 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:43:17.630 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.43e+01, 1.35e+01 / 8.73e-01 | 20260507 08:43:17.630 | INFO | PerfReport: 0.30%, AggregateStats -- 9.08e-02 / 1.02e-01, 9.34e-03 / 4.61e-03 | 20260507 08:43:17.630 | INFO | PerfReport: 11.66%, Baseline -- 3.57e+00 / 4.48e+00, 4.97e-01 / 2.27e-01 | 20260507 08:43:17.630 | INFO | PerfReport: 3.22%, Histogram -- 9.86e-01 / 1.49e+00, 1.65e-01 / 1.09e-01 | 20260507 08:43:17.630 | INFO | PerfReport: 16.33%, DME -- 4.00e+01 / 4.46e+01, 3.88e+00 / 8.98e-01 | 20260507 08:43:17.630 | INFO | PerfReport: 57.73%, FrameLabeling -- 1.77e+01 / 1.94e+01, 1.27e+00 / 1.17e+00 | 20260507 08:43:17.630 | INFO | PerfReport: 6.89%, PulseAccumulating -- 2.11e+00 / 2.68e+00, 3.85e-01 / 2.13e-01 | 20260507 08:43:17.630 | INFO | PerfReport: 2.90%, Metrics -- 8.89e-01 / 1.19e+00, 2.38e-01 / 1.23e-01 | 20260507 08:43:17.630 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.98e-01 / 3.56e-01, 4.96e-02 / 2.14e-02 | 20260507 08:43:17.630 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:43:17.630 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:43:17.630 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:43:17.630 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260507 08:43:17.630 | INFO | PerfReport: 22.00%, Upload -- 2.12e+01 / 0.00e+00, 7.28e+00 / 0.00e+00 | 20260507 08:43:17.630 | INFO | PerfReport: 21.34%, Download -- 2.05e+01 / 0.00e+00, 7.24e+00 / 0.00e+00 | 20260507 08:43:17.630 | INFO | PerfReport: 31.88%, Compute -- 3.07e+01 / 0.00e+00, 1.35e+01 / 0.00e+00 | 20260507 08:43:17.630 | INFO | PerfReport: 1.65%, LockUpload -- 1.58e+00 / 0.00e+00, 5.01e+00 / 0.00e+00 | 20260507 08:43:17.630 | INFO | PerfReport: 1.21%, LockDownload -- 1.16e+00 / 0.00e+00, 2.80e+00 / 0.00e+00 | 20260507 08:43:17.630 | INFO | PerfReport: 21.92%, LockCompute -- 2.11e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 08:43:17.630 | INFO | PerfReport: 0.01%, Misc -- 1.32e-02 / 0.00e+00, 1.56e-02 / 0.00e+00 | 20260507 08:43:17.725 | INFO | Starting a round of Baz IO | 20260507 08:43:20.811 | INFO | Populating RTMetrics report | 20260507 08:43:20.811 | INFO | RTMetrics report populated | 20260507 08:43:20.811 | INFO | Opening tmp RTMetrics json file | 20260507 08:43:20.825 | INFO | Serializing RTMetrics to json | 20260507 08:43:20.826 | INFO | Serialization done, swapping tmp file now... | 20260507 08:43:20.826 | INFO | RTMetrics done | 20260507 08:43:20.894 | INFO | Finished a round of Baz IO: Wrote 1.608 GiB in 3.169 seconds | 20260507 08:43:20.894 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.48% / 0.00% / 67.52% | 20260507 08:43:20.894 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0872691% / 0% / 6.35783e-05% | 20260507 08:43:20.894 | INFO | Baz IO overall: | 20260507 08:43:20.894 | INFO | BytesWritten : 4285144926417 | 20260507 08:43:20.894 | INFO | NumEvents : 2049198352950 | 20260507 08:43:22.769 | INFO | Analyzing chunk frames = [8617984,8618496) | 20260507 08:43:22.769 | INFO | MemInfo: { | 20260507 08:43:22.769 | INFO | "gpu" : " 17 GB", | 20260507 08:43:22.769 | INFO | "huge" : "", | 20260507 08:43:22.769 | INFO | "huge_shared" : " 27 GB", | 20260507 08:43:22.769 | INFO | "malloc" : " 14 GB", | 20260507 08:43:22.769 | INFO | "pinned" : " 29 GB" | 20260507 08:43:22.769 | INFO | } | 20260507 08:43:22.769 | INFO | Perf report for the last 5.14082 seconds: | 20260507 08:43:22.769 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:43:22.769 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:22.769 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:22.769 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:22.769 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260507 08:43:22.769 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:22.769 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260507 08:43:22.769 | INFO | PRE_HQ : 58.6%, 1.8, 54.3ms, 0 | 20260507 08:43:22.769 | INFO | BAZWRITER : 61.2%, 3.1, 96.4ms, 0 | 20260507 08:43:22.769 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 08:43:22.769 | INFO | Uploaded 36GiB to GPU in 2.06137s (17.4641GiB/s) | 20260507 08:43:22.769 | INFO | Downloaded 38GiB to GPU in 2.17979s (17.4329GiB/s) | 20260507 08:43:27.900 | INFO | Analyzing chunk frames = [8618496,8619008) | 20260507 08:43:27.900 | INFO | MemInfo: { | 20260507 08:43:27.900 | INFO | "gpu" : " 17 GB", | 20260507 08:43:27.900 | INFO | "huge" : "", | 20260507 08:43:27.900 | INFO | "huge_shared" : " 27 GB", | 20260507 08:43:27.900 | INFO | "malloc" : " 14 GB", | 20260507 08:43:27.900 | INFO | "pinned" : " 29 GB" | 20260507 08:43:27.900 | INFO | } | 20260507 08:43:27.900 | INFO | Perf report for the last 5.13126 seconds: | 20260507 08:43:27.900 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:43:27.900 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:27.900 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:27.900 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:27.900 | INFO | ANALYSIS : 60.8%, 3.0, 94.4ms, 0 | 20260507 08:43:27.900 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:27.900 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:27.900 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:43:27.900 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:27.900 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:27.901 | INFO | Uploaded 36GiB to GPU in 1.97989s (18.1829GiB/s) | 20260507 08:43:27.901 | INFO | Downloaded 34GiB to GPU in 1.94922s (17.4428GiB/s) | 20260507 08:43:33.038 | INFO | Analyzing chunk frames = [8619008,8619520) | 20260507 08:43:33.038 | INFO | MemInfo: { | 20260507 08:43:33.038 | INFO | "gpu" : " 17 GB", | 20260507 08:43:33.038 | INFO | "huge" : "", | 20260507 08:43:33.038 | INFO | "huge_shared" : " 27 GB", | 20260507 08:43:33.039 | INFO | "malloc" : " 14 GB", | 20260507 08:43:33.039 | INFO | "pinned" : " 29 GB" | 20260507 08:43:33.039 | INFO | } | 20260507 08:43:33.039 | INFO | Perf report for the last 5.1389 seconds: | 20260507 08:43:33.039 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:43:33.039 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:33.039 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:33.039 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:33.039 | INFO | ANALYSIS : 60.6%, 3.0, 93.8ms, 0 | 20260507 08:43:33.039 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:33.039 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:33.039 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:43:33.039 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:33.039 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:33.039 | INFO | Uploaded 36GiB to GPU in 2.04913s (17.5684GiB/s) | 20260507 08:43:33.040 | INFO | Downloaded 34GiB to GPU in 1.95708s (17.3728GiB/s) | 20260507 08:43:38.183 | INFO | Analyzing chunk frames = [8619520,8620032) | 20260507 08:43:38.183 | INFO | MemInfo: { | 20260507 08:43:38.183 | INFO | "gpu" : " 17 GB", | 20260507 08:43:38.183 | INFO | "huge" : "", | 20260507 08:43:38.184 | INFO | "huge_shared" : " 27 GB", | 20260507 08:43:38.184 | INFO | "malloc" : " 14 GB", | 20260507 08:43:38.184 | INFO | "pinned" : " 29 GB" | 20260507 08:43:38.184 | INFO | } | 20260507 08:43:38.184 | INFO | Perf report for the last 5.14435 seconds: | 20260507 08:43:38.184 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:43:38.184 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:38.184 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:38.184 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:38.184 | INFO | ANALYSIS : 59.5%, 3.0, 92.4ms, 0 | 20260507 08:43:38.184 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:38.184 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:38.184 | INFO | PRE_HQ : 24.5%, 1.0, 12.8ms, 0 | 20260507 08:43:38.184 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:38.184 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:38.184 | INFO | Uploaded 36GiB to GPU in 2.00627s (17.9438GiB/s) | 20260507 08:43:38.184 | INFO | Downloaded 34GiB to GPU in 1.94207s (17.5071GiB/s) | 20260507 08:43:43.325 | INFO | Analyzing chunk frames = [8620032,8620544) | 20260507 08:43:43.325 | INFO | MemInfo: { | 20260507 08:43:43.325 | INFO | "gpu" : " 17 GB", | 20260507 08:43:43.325 | INFO | "huge" : "", | 20260507 08:43:43.325 | INFO | "huge_shared" : " 27 GB", | 20260507 08:43:43.325 | INFO | "malloc" : " 14 GB", | 20260507 08:43:43.325 | INFO | "pinned" : " 29 GB" | 20260507 08:43:43.325 | INFO | } | 20260507 08:43:43.325 | INFO | Perf report for the last 5.14146 seconds: | 20260507 08:43:43.325 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:43:43.325 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:43.325 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:43.325 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:43.325 | INFO | ANALYSIS : 60.1%, 3.0, 93.7ms, 0 | 20260507 08:43:43.325 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:43.325 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:43.325 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:43:43.325 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:43.325 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:43.325 | INFO | Uploaded 36GiB to GPU in 2.02788s (17.7526GiB/s) | 20260507 08:43:43.325 | INFO | Downloaded 34GiB to GPU in 1.92017s (17.7067GiB/s) | 20260507 08:43:48.465 | INFO | Analyzing chunk frames = [8620544,8621056) | 20260507 08:43:48.465 | INFO | MemInfo: { | 20260507 08:43:48.465 | INFO | "gpu" : " 17 GB", | 20260507 08:43:48.465 | INFO | "huge" : "", | 20260507 08:43:48.465 | INFO | "huge_shared" : " 27 GB", | 20260507 08:43:48.465 | INFO | "malloc" : " 14 GB", | 20260507 08:43:48.465 | INFO | "pinned" : " 29 GB" | 20260507 08:43:48.465 | INFO | } | 20260507 08:43:48.465 | INFO | Perf report for the last 5.13987 seconds: | 20260507 08:43:48.465 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:43:48.465 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:48.465 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:48.465 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:48.465 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 08:43:48.465 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:48.465 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:48.465 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:43:48.465 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:48.465 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:48.465 | INFO | Uploaded 36GiB to GPU in 2.06292s (17.451GiB/s) | 20260507 08:43:48.465 | INFO | Downloaded 34GiB to GPU in 1.93598s (17.5622GiB/s) | 20260507 08:43:53.598 | INFO | Analyzing chunk frames = [8621056,8621568) | 20260507 08:43:53.598 | INFO | MemInfo: { | 20260507 08:43:53.598 | INFO | "gpu" : " 17 GB", | 20260507 08:43:53.598 | INFO | "huge" : "", | 20260507 08:43:53.598 | INFO | "huge_shared" : " 27 GB", | 20260507 08:43:53.598 | INFO | "malloc" : " 14 GB", | 20260507 08:43:53.598 | INFO | "pinned" : " 29 GB" | 20260507 08:43:53.598 | INFO | } | 20260507 08:43:53.598 | INFO | Perf report for the last 5.133 seconds: | 20260507 08:43:53.598 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:43:53.598 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:53.598 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:53.598 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:53.598 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 08:43:53.598 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:53.598 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:53.598 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 08:43:53.598 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:53.598 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:53.598 | INFO | Uploaded 36GiB to GPU in 1.90172s (18.9303GiB/s) | 20260507 08:43:53.598 | INFO | Downloaded 34GiB to GPU in 1.9777s (17.1917GiB/s) | 20260507 08:43:58.738 | INFO | Analyzing chunk frames = [8621568,8622080) | 20260507 08:43:58.738 | INFO | MemInfo: { | 20260507 08:43:58.738 | INFO | "gpu" : " 17 GB", | 20260507 08:43:58.738 | INFO | "huge" : "", | 20260507 08:43:58.738 | INFO | "huge_shared" : " 27 GB", | 20260507 08:43:58.738 | INFO | "malloc" : " 14 GB", | 20260507 08:43:58.738 | INFO | "pinned" : " 29 GB" | 20260507 08:43:58.738 | INFO | } | 20260507 08:43:58.738 | INFO | Perf report for the last 5.13975 seconds: | 20260507 08:43:58.738 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:43:58.738 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:58.738 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:58.738 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:58.738 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:43:58.738 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:58.738 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:58.738 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:43:58.738 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:58.738 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:43:58.738 | INFO | Uploaded 36GiB to GPU in 1.9719s (18.2565GiB/s) | 20260507 08:43:58.738 | INFO | Downloaded 34GiB to GPU in 1.96396s (17.312GiB/s) | 20260507 08:43:58.837 | INFO | Starting a round of Baz IO | 20260507 08:44:01.927 | INFO | Populating RTMetrics report | 20260507 08:44:01.928 | INFO | RTMetrics report populated | 20260507 08:44:01.928 | INFO | Opening tmp RTMetrics json file | 20260507 08:44:01.935 | INFO | Serializing RTMetrics to json | 20260507 08:44:01.944 | INFO | Serialization done, swapping tmp file now... | 20260507 08:44:01.944 | INFO | RTMetrics done | 20260507 08:44:02.012 | INFO | Finished a round of Baz IO: Wrote 1.606 GiB in 3.175 seconds | 20260507 08:44:02.012 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.48% / 0.00% / 67.52% | 20260507 08:44:02.012 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0868758% / 0% / 7.54992e-05% | 20260507 08:44:02.012 | INFO | Baz IO overall: | 20260507 08:44:02.012 | INFO | BytesWritten : 4286869183546 | 20260507 08:44:02.012 | INFO | NumEvents : 2050001209338 | 20260507 08:44:03.880 | INFO | Analyzing chunk frames = [8622080,8622592) | 20260507 08:44:03.880 | INFO | MemInfo: { | 20260507 08:44:03.880 | INFO | "gpu" : " 17 GB", | 20260507 08:44:03.880 | INFO | "huge" : "", | 20260507 08:44:03.880 | INFO | "huge_shared" : " 27 GB", | 20260507 08:44:03.880 | INFO | "malloc" : " 14 GB", | 20260507 08:44:03.880 | INFO | "pinned" : " 29 GB" | 20260507 08:44:03.880 | INFO | } | 20260507 08:44:03.880 | INFO | Perf report for the last 5.14199 seconds: | 20260507 08:44:03.880 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:44:03.880 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:03.880 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:03.880 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:03.880 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 08:44:03.880 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:03.880 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 08:44:03.880 | INFO | PRE_HQ : 58.7%, 1.8, 54.5ms, 0 | 20260507 08:44:03.880 | INFO | BAZWRITER : 61.1%, 3.1, 97.3ms, 0 | 20260507 08:44:03.880 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:44:03.880 | INFO | Uploaded 36GiB to GPU in 2.0702s (17.3897GiB/s) | 20260507 08:44:03.880 | INFO | Downloaded 38GiB to GPU in 2.18108s (17.4225GiB/s) | 20260507 08:44:09.022 | INFO | Analyzing chunk frames = [8622592,8623104) | 20260507 08:44:09.022 | INFO | MemInfo: { | 20260507 08:44:09.022 | INFO | "gpu" : " 17 GB", | 20260507 08:44:09.022 | INFO | "huge" : "", | 20260507 08:44:09.022 | INFO | "huge_shared" : " 27 GB", | 20260507 08:44:09.022 | INFO | "malloc" : " 14 GB", | 20260507 08:44:09.022 | INFO | "pinned" : " 29 GB" | 20260507 08:44:09.022 | INFO | } | 20260507 08:44:09.022 | INFO | Perf report for the last 5.14239 seconds: | 20260507 08:44:09.022 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:44:09.022 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:09.022 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:09.022 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:09.022 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:44:09.022 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:09.022 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:09.022 | INFO | PRE_HQ : 24.0%, 1.0, 12.5ms, 0 | 20260507 08:44:09.022 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:09.022 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:09.022 | INFO | Uploaded 36GiB to GPU in 2.00144s (17.987GiB/s) | 20260507 08:44:09.022 | INFO | Downloaded 34GiB to GPU in 1.94151s (17.5121GiB/s) | 20260507 08:44:14.156 | INFO | Analyzing chunk frames = [8623104,8623616) | 20260507 08:44:14.156 | INFO | MemInfo: { | 20260507 08:44:14.156 | INFO | "gpu" : " 17 GB", | 20260507 08:44:14.156 | INFO | "huge" : "", | 20260507 08:44:14.156 | INFO | "huge_shared" : " 27 GB", | 20260507 08:44:14.157 | INFO | "malloc" : " 14 GB", | 20260507 08:44:14.157 | INFO | "pinned" : " 29 GB" | 20260507 08:44:14.157 | INFO | } | 20260507 08:44:14.157 | INFO | Perf report for the last 5.13446 seconds: | 20260507 08:44:14.157 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:44:14.157 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:14.157 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:14.157 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:14.157 | INFO | ANALYSIS : 60.5%, 3.0, 93.7ms, 0 | 20260507 08:44:14.157 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:14.157 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:14.157 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:44:14.157 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:14.157 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:14.157 | INFO | Uploaded 36GiB to GPU in 2.03682s (17.6746GiB/s) | 20260507 08:44:14.157 | INFO | Downloaded 34GiB to GPU in 1.91684s (17.7375GiB/s) | 20260507 08:44:19.295 | INFO | Analyzing chunk frames = [8623616,8624128) | 20260507 08:44:19.295 | INFO | MemInfo: { | 20260507 08:44:19.295 | INFO | "gpu" : " 17 GB", | 20260507 08:44:19.295 | INFO | "huge" : "", | 20260507 08:44:19.295 | INFO | "huge_shared" : " 27 GB", | 20260507 08:44:19.295 | INFO | "malloc" : " 14 GB", | 20260507 08:44:19.295 | INFO | "pinned" : " 29 GB" | 20260507 08:44:19.295 | INFO | } | 20260507 08:44:19.295 | INFO | Perf report for the last 5.1385 seconds: | 20260507 08:44:19.295 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:44:19.295 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:19.295 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:19.295 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:19.295 | INFO | ANALYSIS : 60.5%, 2.9, 93.4ms, 0 | 20260507 08:44:19.295 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:19.295 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:19.295 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:44:19.295 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:19.295 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:19.295 | INFO | Uploaded 36GiB to GPU in 2.05589s (17.5107GiB/s) | 20260507 08:44:19.295 | INFO | Downloaded 34GiB to GPU in 1.94575s (17.474GiB/s) | 20260507 08:44:24.440 | INFO | Analyzing chunk frames = [8624128,8624640) | 20260507 08:44:24.440 | INFO | MemInfo: { | 20260507 08:44:24.440 | INFO | "gpu" : " 17 GB", | 20260507 08:44:24.440 | INFO | "huge" : "", | 20260507 08:44:24.440 | INFO | "huge_shared" : " 27 GB", | 20260507 08:44:24.440 | INFO | "malloc" : " 14 GB", | 20260507 08:44:24.440 | INFO | "pinned" : " 29 GB" | 20260507 08:44:24.440 | INFO | } | 20260507 08:44:24.440 | INFO | Perf report for the last 5.14518 seconds: | 20260507 08:44:24.440 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:44:24.440 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:24.440 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:24.440 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:24.440 | INFO | ANALYSIS : 60.1%, 3.0, 94.0ms, 0 | 20260507 08:44:24.440 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:24.440 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:24.440 | INFO | PRE_HQ : 24.7%, 1.0, 12.8ms, 0 | 20260507 08:44:24.440 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:24.440 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:24.441 | INFO | Uploaded 36GiB to GPU in 2.06157s (17.4624GiB/s) | 20260507 08:44:24.441 | INFO | Downloaded 34GiB to GPU in 1.95041s (17.4323GiB/s) | 20260507 08:44:29.579 | INFO | Analyzing chunk frames = [8624640,8625152) | 20260507 08:44:29.579 | INFO | MemInfo: { | 20260507 08:44:29.579 | INFO | "gpu" : " 17 GB", | 20260507 08:44:29.579 | INFO | "huge" : "", | 20260507 08:44:29.579 | INFO | "huge_shared" : " 27 GB", | 20260507 08:44:29.579 | INFO | "malloc" : " 14 GB", | 20260507 08:44:29.579 | INFO | "pinned" : " 29 GB" | 20260507 08:44:29.579 | INFO | } | 20260507 08:44:29.579 | INFO | Perf report for the last 5.13862 seconds: | 20260507 08:44:29.579 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:44:29.579 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:29.579 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:29.579 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:29.579 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 08:44:29.579 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:29.579 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:29.579 | INFO | PRE_HQ : 27.3%, 1.0, 14.2ms, 0 | 20260507 08:44:29.579 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:29.579 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:29.579 | INFO | Uploaded 36GiB to GPU in 2.06601s (17.4249GiB/s) | 20260507 08:44:29.579 | INFO | Downloaded 34GiB to GPU in 1.91852s (17.722GiB/s) | 20260507 08:44:34.712 | INFO | Analyzing chunk frames = [8625152,8625664) | 20260507 08:44:34.712 | INFO | MemInfo: { | 20260507 08:44:34.712 | INFO | "gpu" : " 17 GB", | 20260507 08:44:34.712 | INFO | "huge" : "", | 20260507 08:44:34.712 | INFO | "huge_shared" : " 27 GB", | 20260507 08:44:34.712 | INFO | "malloc" : " 14 GB", | 20260507 08:44:34.712 | INFO | "pinned" : " 29 GB" | 20260507 08:44:34.712 | INFO | } | 20260507 08:44:34.712 | INFO | Perf report for the last 5.13308 seconds: | 20260507 08:44:34.712 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:44:34.712 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:34.712 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:34.712 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:34.712 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:44:34.712 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:34.712 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:34.713 | INFO | PRE_HQ : 25.2%, 1.0, 13.1ms, 0 | 20260507 08:44:34.713 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:34.713 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:34.713 | INFO | Uploaded 36GiB to GPU in 1.96616s (18.3098GiB/s) | 20260507 08:44:34.713 | INFO | Downloaded 34GiB to GPU in 1.97103s (17.2498GiB/s) | 20260507 08:44:39.846 | INFO | Analyzing chunk frames = [8625664,8626176) | 20260507 08:44:39.846 | INFO | MemInfo: { | 20260507 08:44:39.846 | INFO | "gpu" : " 17 GB", | 20260507 08:44:39.846 | INFO | "huge" : "", | 20260507 08:44:39.846 | INFO | "huge_shared" : " 27 GB", | 20260507 08:44:39.846 | INFO | "malloc" : " 14 GB", | 20260507 08:44:39.846 | INFO | "pinned" : " 29 GB" | 20260507 08:44:39.846 | INFO | } | 20260507 08:44:39.846 | INFO | Perf report for the last 5.13372 seconds: | 20260507 08:44:39.846 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:44:39.846 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:39.846 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:39.846 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:39.846 | INFO | ANALYSIS : 60.6%, 3.0, 94.2ms, 0 | 20260507 08:44:39.846 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:39.846 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:39.846 | INFO | PRE_HQ : 25.0%, 1.0, 12.9ms, 0 | 20260507 08:44:39.846 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:39.846 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:39.846 | INFO | Uploaded 36GiB to GPU in 1.9847s (18.1387GiB/s) | 20260507 08:44:39.846 | INFO | Downloaded 34GiB to GPU in 1.95962s (17.3503GiB/s) | 20260507 08:44:39.846 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:44:39.846 | INFO | Total mem usage: 18 GiB | 20260507 08:44:39.846 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:44:39.846 | INFO | Recent High Water Marks Sum: 14.1 GiB (Savings: -27.3%) | 20260507 08:44:39.846 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:44:39.846 | INFO | High Water Breakdown: | 20260507 08:44:39.846 | INFO | 60.1% : 8.5 GiB ( 74.4% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:44:39.846 | INFO | 39.8% : 5.63 GiB ( 84.1% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:44:39.846 | INFO | 0.141% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:44:39.846 | INFO | ------------------------------------------ | 20260507 08:44:39.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:44:39.846 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:44:39.846 | INFO | Total mem usage: 26.5 GiB | 20260507 08:44:39.846 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:44:39.846 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:44:39.846 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:44:39.846 | INFO | High Water Breakdown: | 20260507 08:44:39.846 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:44:39.846 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:44:39.846 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:44:39.846 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:44:39.846 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:44:39.846 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:44:39.846 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:44:39.846 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:44:39.846 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:44:39.846 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:44:39.846 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:44:39.846 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:44:39.846 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:44:39.846 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:44:39.846 | INFO | ------------------------------------------ | 20260507 08:44:39.846 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:44:39.847 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:44:39.847 | INFO | Total mem usage: 25.6 GiB | 20260507 08:44:39.847 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:44:39.847 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:44:39.847 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:44:39.847 | INFO | High Water Breakdown: | 20260507 08:44:39.847 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:44:39.847 | INFO | ------------------------------------------ | 20260507 08:44:39.847 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:44:39.847 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:44:39.847 | INFO | Total mem usage: 15.7 GiB | 20260507 08:44:39.847 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:44:39.847 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:44:39.847 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:44:39.847 | INFO | High Water Breakdown: | 20260507 08:44:39.847 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:44:39.847 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:44:39.847 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:44:39.847 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:44:39.847 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:44:39.847 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:44:39.847 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:44:39.847 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:44:39.847 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:44:39.847 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:44:39.847 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:44:39.847 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:44:39.847 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:44:39.847 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:44:39.847 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:44:39.847 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:44:39.847 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:44:39.847 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:44:39.847 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:44:39.847 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:44:39.847 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:44:39.847 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:44:39.847 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:44:39.847 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:44:39.847 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:44:39.847 | INFO | ------------------------------------------ | 20260507 08:44:39.847 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:44:39.847 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:44:39.847 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:44:39.847 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:44:39.847 | INFO | PerfReport: Full Execution -- 3.06e+01 / 7.45e+01, 1.32e+01 / 1.15e+00 | 20260507 08:44:39.847 | INFO | PerfReport: 0.29%, AggregateStats -- 9.01e-02 / 1.01e-01, 9.74e-03 / 4.65e-03 | 20260507 08:44:39.847 | INFO | PerfReport: 11.66%, Baseline -- 3.57e+00 / 4.47e+00, 5.03e-01 / 2.46e-01 | 20260507 08:44:39.847 | INFO | PerfReport: 3.22%, Histogram -- 9.87e-01 / 1.48e+00, 1.65e-01 / 1.11e-01 | 20260507 08:44:39.847 | INFO | PerfReport: 16.31%, DME -- 4.00e+01 / 4.47e+01, 3.96e+00 / 1.38e+00 | 20260507 08:44:39.847 | INFO | PerfReport: 57.74%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.27e+00 / 1.24e+00 | 20260507 08:44:39.847 | INFO | PerfReport: 6.89%, PulseAccumulating -- 2.11e+00 / 2.66e+00, 3.87e-01 / 2.31e-01 | 20260507 08:44:39.847 | INFO | PerfReport: 2.91%, Metrics -- 8.91e-01 / 1.20e+00, 2.39e-01 / 1.20e-01 | 20260507 08:44:39.847 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.98e-01 / 3.60e-01, 5.06e-02 / 2.21e-02 | 20260507 08:44:39.847 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:44:39.847 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:44:39.847 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:44:39.847 | INFO | PerfReport: Full Execution -- 9.61e+01 / -nan, 2.44e+01 / 0.00e+00 | 20260507 08:44:39.847 | INFO | PerfReport: 21.88%, Upload -- 2.10e+01 / 0.00e+00, 7.21e+00 / 0.00e+00 | 20260507 08:44:39.847 | INFO | PerfReport: 21.42%, Download -- 2.06e+01 / 0.00e+00, 7.32e+00 / 0.00e+00 | 20260507 08:44:39.847 | INFO | PerfReport: 31.89%, Compute -- 3.07e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 08:44:39.847 | INFO | PerfReport: 1.64%, LockUpload -- 1.57e+00 / 0.00e+00, 5.06e+00 / 0.00e+00 | 20260507 08:44:39.847 | INFO | PerfReport: 1.23%, LockDownload -- 1.19e+00 / 0.00e+00, 2.94e+00 / 0.00e+00 | 20260507 08:44:39.847 | INFO | PerfReport: 21.92%, LockCompute -- 2.11e+01 / 0.00e+00, 1.83e+01 / 0.00e+00 | 20260507 08:44:39.847 | INFO | PerfReport: 0.01%, Misc -- 1.31e-02 / 0.00e+00, 1.81e-02 / 0.00e+00 | 20260507 08:44:39.942 | INFO | Starting a round of Baz IO | 20260507 08:44:43.039 | INFO | Populating RTMetrics report | 20260507 08:44:43.039 | INFO | RTMetrics report populated | 20260507 08:44:43.039 | INFO | Opening tmp RTMetrics json file | 20260507 08:44:43.053 | INFO | Serializing RTMetrics to json | 20260507 08:44:43.054 | INFO | Serialization done, swapping tmp file now... | 20260507 08:44:43.055 | INFO | RTMetrics done | 20260507 08:44:43.125 | INFO | Finished a round of Baz IO: Wrote 1.607 GiB in 3.182 seconds | 20260507 08:44:43.125 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.48% / 0.00% / 67.52% | 20260507 08:44:43.125 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0869672% / 3.97364e-06% / 8.34465e-05% | 20260507 08:44:43.125 | INFO | Baz IO overall: | 20260507 08:44:43.125 | INFO | BytesWritten : 4288594326466 | 20260507 08:44:43.125 | INFO | NumEvents : 2050804958068 | 20260507 08:44:44.994 | INFO | Analyzing chunk frames = [8626176,8626688) | 20260507 08:44:44.994 | INFO | MemInfo: { | 20260507 08:44:44.994 | INFO | "gpu" : " 17 GB", | 20260507 08:44:44.994 | INFO | "huge" : "", | 20260507 08:44:44.994 | INFO | "huge_shared" : " 27 GB", | 20260507 08:44:44.994 | INFO | "malloc" : " 14 GB", | 20260507 08:44:44.994 | INFO | "pinned" : " 29 GB" | 20260507 08:44:44.994 | INFO | } | 20260507 08:44:44.994 | INFO | Perf report for the last 5.14842 seconds: | 20260507 08:44:44.994 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:44:44.994 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:44.994 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:44.994 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:44.994 | INFO | ANALYSIS : 61.5%, 3.0, 95.9ms, 0 | 20260507 08:44:44.994 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:44.994 | INFO | RT_METRICS : 40.7%, 1.0, 21.4ms, 0 | 20260507 08:44:44.994 | INFO | PRE_HQ : 58.7%, 1.8, 54.5ms, 0 | 20260507 08:44:44.995 | INFO | BAZWRITER : 61.4%, 3.0, 95.5ms, 0 | 20260507 08:44:44.995 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:44:44.995 | INFO | Uploaded 36GiB to GPU in 2.03272s (17.7103GiB/s) | 20260507 08:44:44.995 | INFO | Downloaded 38GiB to GPU in 2.17095s (17.5038GiB/s) | 20260507 08:44:50.140 | INFO | Analyzing chunk frames = [8626688,8627200) | 20260507 08:44:50.140 | INFO | MemInfo: { | 20260507 08:44:50.140 | INFO | "gpu" : " 17 GB", | 20260507 08:44:50.140 | INFO | "huge" : "", | 20260507 08:44:50.140 | INFO | "huge_shared" : " 27 GB", | 20260507 08:44:50.140 | INFO | "malloc" : " 14 GB", | 20260507 08:44:50.140 | INFO | "pinned" : " 29 GB" | 20260507 08:44:50.140 | INFO | } | 20260507 08:44:50.140 | INFO | Perf report for the last 5.1457 seconds: | 20260507 08:44:50.140 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:44:50.140 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:50.140 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:50.140 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:50.140 | INFO | ANALYSIS : 60.6%, 3.0, 94.4ms, 0 | 20260507 08:44:50.140 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:50.140 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:50.140 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:44:50.140 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:50.140 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:50.140 | INFO | Uploaded 36GiB to GPU in 1.9839s (18.1461GiB/s) | 20260507 08:44:50.140 | INFO | Downloaded 34GiB to GPU in 1.93428s (17.5776GiB/s) | 20260507 08:44:55.276 | INFO | Analyzing chunk frames = [8627200,8627712) | 20260507 08:44:55.276 | INFO | MemInfo: { | 20260507 08:44:55.276 | INFO | "gpu" : " 17 GB", | 20260507 08:44:55.276 | INFO | "huge" : "", | 20260507 08:44:55.276 | INFO | "huge_shared" : " 27 GB", | 20260507 08:44:55.276 | INFO | "malloc" : " 14 GB", | 20260507 08:44:55.276 | INFO | "pinned" : " 29 GB" | 20260507 08:44:55.276 | INFO | } | 20260507 08:44:55.276 | INFO | Perf report for the last 5.13618 seconds: | 20260507 08:44:55.276 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:44:55.276 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:55.276 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:55.276 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:55.276 | INFO | ANALYSIS : 60.4%, 3.0, 93.6ms, 0 | 20260507 08:44:55.276 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:55.276 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:55.276 | INFO | PRE_HQ : 24.6%, 1.0, 12.8ms, 0 | 20260507 08:44:55.276 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:55.276 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:44:55.276 | INFO | Uploaded 36GiB to GPU in 2.03325s (17.7057GiB/s) | 20260507 08:44:55.276 | INFO | Downloaded 34GiB to GPU in 1.92521s (17.6604GiB/s) | 20260507 08:45:00.408 | INFO | Analyzing chunk frames = [8627712,8628224) | 20260507 08:45:00.408 | INFO | MemInfo: { | 20260507 08:45:00.408 | INFO | "gpu" : " 17 GB", | 20260507 08:45:00.408 | INFO | "huge" : "", | 20260507 08:45:00.408 | INFO | "huge_shared" : " 27 GB", | 20260507 08:45:00.408 | INFO | "malloc" : " 14 GB", | 20260507 08:45:00.408 | INFO | "pinned" : " 29 GB" | 20260507 08:45:00.408 | INFO | } | 20260507 08:45:00.408 | INFO | Perf report for the last 5.13182 seconds: | 20260507 08:45:00.408 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:45:00.408 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:00.408 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:00.408 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:00.408 | INFO | ANALYSIS : 60.5%, 2.9, 93.3ms, 0 | 20260507 08:45:00.408 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:00.408 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:00.408 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:45:00.408 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:00.408 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:00.408 | INFO | Uploaded 36GiB to GPU in 2.05394s (17.5273GiB/s) | 20260507 08:45:00.408 | INFO | Downloaded 34GiB to GPU in 1.9498s (17.4377GiB/s) | 20260507 08:45:05.548 | INFO | Analyzing chunk frames = [8628224,8628736) | 20260507 08:45:05.548 | INFO | MemInfo: { | 20260507 08:45:05.548 | INFO | "gpu" : " 17 GB", | 20260507 08:45:05.548 | INFO | "huge" : "", | 20260507 08:45:05.548 | INFO | "huge_shared" : " 27 GB", | 20260507 08:45:05.548 | INFO | "malloc" : " 14 GB", | 20260507 08:45:05.548 | INFO | "pinned" : " 29 GB" | 20260507 08:45:05.548 | INFO | } | 20260507 08:45:05.548 | INFO | Perf report for the last 5.13965 seconds: | 20260507 08:45:05.548 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:45:05.548 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:05.548 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:05.548 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:05.548 | INFO | ANALYSIS : 60.2%, 3.0, 93.7ms, 0 | 20260507 08:45:05.548 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:05.548 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:05.548 | INFO | PRE_HQ : 24.3%, 1.0, 12.6ms, 0 | 20260507 08:45:05.548 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:05.548 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:05.548 | INFO | Uploaded 36GiB to GPU in 2.02847s (17.7474GiB/s) | 20260507 08:45:05.548 | INFO | Downloaded 34GiB to GPU in 1.9197s (17.7111GiB/s) | 20260507 08:45:10.690 | INFO | Analyzing chunk frames = [8628736,8629248) | 20260507 08:45:10.690 | INFO | MemInfo: { | 20260507 08:45:10.690 | INFO | "gpu" : " 17 GB", | 20260507 08:45:10.690 | INFO | "huge" : "", | 20260507 08:45:10.690 | INFO | "huge_shared" : " 27 GB", | 20260507 08:45:10.690 | INFO | "malloc" : " 14 GB", | 20260507 08:45:10.690 | INFO | "pinned" : " 29 GB" | 20260507 08:45:10.690 | INFO | } | 20260507 08:45:10.690 | INFO | Perf report for the last 5.14279 seconds: | 20260507 08:45:10.691 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:45:10.691 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:10.691 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:10.691 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:10.691 | INFO | ANALYSIS : 60.2%, 3.0, 94.1ms, 0 | 20260507 08:45:10.691 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:10.691 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:10.691 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:45:10.691 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:10.691 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:10.691 | INFO | Uploaded 36GiB to GPU in 2.04497s (17.6042GiB/s) | 20260507 08:45:10.691 | INFO | Downloaded 34GiB to GPU in 1.91119s (17.79GiB/s) | 20260507 08:45:15.827 | INFO | Analyzing chunk frames = [8629248,8629760) | 20260507 08:45:15.827 | INFO | MemInfo: { | 20260507 08:45:15.827 | INFO | "gpu" : " 17 GB", | 20260507 08:45:15.827 | INFO | "huge" : "", | 20260507 08:45:15.827 | INFO | "huge_shared" : " 27 GB", | 20260507 08:45:15.827 | INFO | "malloc" : " 14 GB", | 20260507 08:45:15.827 | INFO | "pinned" : " 29 GB" | 20260507 08:45:15.827 | INFO | } | 20260507 08:45:15.827 | INFO | Perf report for the last 5.13634 seconds: | 20260507 08:45:15.827 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:45:15.827 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:15.827 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:15.827 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:15.827 | INFO | ANALYSIS : 60.5%, 3.0, 94.1ms, 0 | 20260507 08:45:15.827 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:15.827 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:15.827 | INFO | PRE_HQ : 25.0%, 1.0, 13.0ms, 0 | 20260507 08:45:15.827 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:15.827 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:15.827 | INFO | Uploaded 36GiB to GPU in 1.8869s (19.0789GiB/s) | 20260507 08:45:15.827 | INFO | Downloaded 34GiB to GPU in 1.97518s (17.2136GiB/s) | 20260507 08:45:20.977 | INFO | Analyzing chunk frames = [8629760,8630272) | 20260507 08:45:20.977 | INFO | MemInfo: { | 20260507 08:45:20.977 | INFO | "gpu" : " 17 GB", | 20260507 08:45:20.977 | INFO | "huge" : "", | 20260507 08:45:20.977 | INFO | "huge_shared" : " 27 GB", | 20260507 08:45:20.977 | INFO | "malloc" : " 14 GB", | 20260507 08:45:20.977 | INFO | "pinned" : " 29 GB" | 20260507 08:45:20.977 | INFO | } | 20260507 08:45:20.977 | INFO | Perf report for the last 5.1502 seconds: | 20260507 08:45:20.977 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:45:20.977 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:20.977 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:20.977 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:20.977 | INFO | ANALYSIS : 60.4%, 3.0, 94.2ms, 0 | 20260507 08:45:20.977 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:20.977 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:20.977 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:45:20.977 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:20.977 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:20.977 | INFO | Uploaded 36GiB to GPU in 1.99115s (18.08GiB/s) | 20260507 08:45:20.977 | INFO | Downloaded 34GiB to GPU in 1.9605s (17.3426GiB/s) | 20260507 08:45:21.076 | INFO | Starting a round of Baz IO | 20260507 08:45:24.171 | INFO | Populating RTMetrics report | 20260507 08:45:24.171 | INFO | RTMetrics report populated | 20260507 08:45:24.171 | INFO | Opening tmp RTMetrics json file | 20260507 08:45:24.176 | INFO | Serializing RTMetrics to json | 20260507 08:45:24.189 | INFO | Serialization done, swapping tmp file now... | 20260507 08:45:24.189 | INFO | RTMetrics done | 20260507 08:45:24.253 | INFO | Finished a round of Baz IO: Wrote 1.605 GiB in 3.176 seconds | 20260507 08:45:24.253 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.48% / 0.00% / 67.52% | 20260507 08:45:24.253 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0867367% / 0% / 7.15256e-05% | 20260507 08:45:24.253 | INFO | Baz IO overall: | 20260507 08:45:24.253 | INFO | BytesWritten : 4290317202152 | 20260507 08:45:24.253 | INFO | NumEvents : 2051606870959 | 20260507 08:45:26.107 | INFO | Analyzing chunk frames = [8630272,8630784) | 20260507 08:45:26.107 | INFO | MemInfo: { | 20260507 08:45:26.107 | INFO | "gpu" : " 17 GB", | 20260507 08:45:26.107 | INFO | "huge" : "", | 20260507 08:45:26.107 | INFO | "huge_shared" : " 27 GB", | 20260507 08:45:26.107 | INFO | "malloc" : " 14 GB", | 20260507 08:45:26.107 | INFO | "pinned" : " 29 GB" | 20260507 08:45:26.107 | INFO | } | 20260507 08:45:26.107 | INFO | Perf report for the last 5.13034 seconds: | 20260507 08:45:26.107 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:45:26.107 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:26.107 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:26.107 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:26.107 | INFO | ANALYSIS : 61.5%, 3.0, 95.8ms, 0 | 20260507 08:45:26.107 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:26.107 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 08:45:26.107 | INFO | PRE_HQ : 58.6%, 1.8, 54.3ms, 0 | 20260507 08:45:26.107 | INFO | BAZWRITER : 61.6%, 3.0, 96.2ms, 0 | 20260507 08:45:26.107 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:45:26.108 | INFO | Uploaded 36GiB to GPU in 2.04416s (17.6112GiB/s) | 20260507 08:45:26.108 | INFO | Downloaded 38GiB to GPU in 2.15756s (17.6125GiB/s) | 20260507 08:45:31.251 | INFO | Analyzing chunk frames = [8630784,8631296) | 20260507 08:45:31.251 | INFO | MemInfo: { | 20260507 08:45:31.251 | INFO | "gpu" : " 17 GB", | 20260507 08:45:31.251 | INFO | "huge" : "", | 20260507 08:45:31.251 | INFO | "huge_shared" : " 27 GB", | 20260507 08:45:31.251 | INFO | "malloc" : " 14 GB", | 20260507 08:45:31.251 | INFO | "pinned" : " 29 GB" | 20260507 08:45:31.251 | INFO | } | 20260507 08:45:31.251 | INFO | Perf report for the last 5.14384 seconds: | 20260507 08:45:31.251 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:45:31.251 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:31.251 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:31.251 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:31.251 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 08:45:31.251 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:31.251 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:31.251 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:45:31.251 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:31.251 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:31.251 | INFO | Uploaded 36GiB to GPU in 1.99121s (18.0794GiB/s) | 20260507 08:45:31.252 | INFO | Downloaded 34GiB to GPU in 1.9315s (17.6029GiB/s) | 20260507 08:45:36.390 | INFO | Analyzing chunk frames = [8631296,8631808) | 20260507 08:45:36.390 | INFO | MemInfo: { | 20260507 08:45:36.390 | INFO | "gpu" : " 17 GB", | 20260507 08:45:36.390 | INFO | "huge" : "", | 20260507 08:45:36.390 | INFO | "huge_shared" : " 27 GB", | 20260507 08:45:36.390 | INFO | "malloc" : " 14 GB", | 20260507 08:45:36.390 | INFO | "pinned" : " 29 GB" | 20260507 08:45:36.390 | INFO | } | 20260507 08:45:36.390 | INFO | Perf report for the last 5.13924 seconds: | 20260507 08:45:36.390 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:45:36.390 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:36.390 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:36.390 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:36.390 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 08:45:36.391 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:36.391 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:36.391 | INFO | PRE_HQ : 24.1%, 1.0, 12.5ms, 0 | 20260507 08:45:36.391 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:36.391 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:36.391 | INFO | Uploaded 36GiB to GPU in 2.02671s (17.7628GiB/s) | 20260507 08:45:36.391 | INFO | Downloaded 34GiB to GPU in 1.91795s (17.7272GiB/s) | 20260507 08:45:41.525 | INFO | Analyzing chunk frames = [8631808,8632320) | 20260507 08:45:41.525 | INFO | MemInfo: { | 20260507 08:45:41.525 | INFO | "gpu" : " 17 GB", | 20260507 08:45:41.525 | INFO | "huge" : "", | 20260507 08:45:41.525 | INFO | "huge_shared" : " 27 GB", | 20260507 08:45:41.525 | INFO | "malloc" : " 14 GB", | 20260507 08:45:41.525 | INFO | "pinned" : " 29 GB" | 20260507 08:45:41.525 | INFO | } | 20260507 08:45:41.525 | INFO | Perf report for the last 5.13484 seconds: | 20260507 08:45:41.525 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:45:41.525 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:41.525 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:41.525 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:41.525 | INFO | ANALYSIS : 60.6%, 2.9, 93.4ms, 0 | 20260507 08:45:41.525 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:41.525 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:41.525 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 08:45:41.525 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:41.525 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:41.526 | INFO | Uploaded 36GiB to GPU in 2.05778s (17.4946GiB/s) | 20260507 08:45:41.526 | INFO | Downloaded 34GiB to GPU in 1.94303s (17.4984GiB/s) | 20260507 08:45:46.662 | INFO | Analyzing chunk frames = [8632320,8632832) | 20260507 08:45:46.662 | INFO | MemInfo: { | 20260507 08:45:46.662 | INFO | "gpu" : " 17 GB", | 20260507 08:45:46.662 | INFO | "huge" : "", | 20260507 08:45:46.662 | INFO | "huge_shared" : " 27 GB", | 20260507 08:45:46.662 | INFO | "malloc" : " 14 GB", | 20260507 08:45:46.662 | INFO | "pinned" : " 29 GB" | 20260507 08:45:46.662 | INFO | } | 20260507 08:45:46.662 | INFO | Perf report for the last 5.13706 seconds: | 20260507 08:45:46.662 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:45:46.662 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:46.662 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:46.662 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:46.662 | INFO | ANALYSIS : 60.1%, 3.0, 93.6ms, 0 | 20260507 08:45:46.662 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:46.662 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:46.663 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:45:46.663 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:46.663 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:46.663 | INFO | Uploaded 36GiB to GPU in 2.05384s (17.5282GiB/s) | 20260507 08:45:46.663 | INFO | Downloaded 34GiB to GPU in 1.9127s (17.776GiB/s) | 20260507 08:45:51.800 | INFO | Analyzing chunk frames = [8632832,8633344) | 20260507 08:45:51.800 | INFO | MemInfo: { | 20260507 08:45:51.800 | INFO | "gpu" : " 17 GB", | 20260507 08:45:51.800 | INFO | "huge" : "", | 20260507 08:45:51.800 | INFO | "huge_shared" : " 27 GB", | 20260507 08:45:51.800 | INFO | "malloc" : " 14 GB", | 20260507 08:45:51.800 | INFO | "pinned" : " 29 GB" | 20260507 08:45:51.801 | INFO | } | 20260507 08:45:51.801 | INFO | Perf report for the last 5.13832 seconds: | 20260507 08:45:51.801 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:45:51.801 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:51.801 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:51.801 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:51.801 | INFO | ANALYSIS : 60.5%, 3.0, 94.4ms, 0 | 20260507 08:45:51.801 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:51.801 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:51.801 | INFO | PRE_HQ : 27.9%, 1.0, 14.5ms, 0 | 20260507 08:45:51.801 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:51.801 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:51.801 | INFO | Uploaded 36GiB to GPU in 2.06947s (17.3958GiB/s) | 20260507 08:45:51.801 | INFO | Downloaded 34GiB to GPU in 1.93226s (17.596GiB/s) | 20260507 08:45:56.947 | INFO | Analyzing chunk frames = [8633344,8633856) | 20260507 08:45:56.947 | INFO | MemInfo: { | 20260507 08:45:56.947 | INFO | "gpu" : " 17 GB", | 20260507 08:45:56.947 | INFO | "huge" : "", | 20260507 08:45:56.947 | INFO | "huge_shared" : " 27 GB", | 20260507 08:45:56.947 | INFO | "malloc" : " 14 GB", | 20260507 08:45:56.947 | INFO | "pinned" : " 29 GB" | 20260507 08:45:56.947 | INFO | } | 20260507 08:45:56.947 | INFO | Perf report for the last 5.14674 seconds: | 20260507 08:45:56.947 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:45:56.947 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:56.947 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:56.947 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:56.947 | INFO | ANALYSIS : 60.7%, 3.0, 94.6ms, 0 | 20260507 08:45:56.947 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:56.948 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:56.948 | INFO | PRE_HQ : 24.8%, 1.0, 12.9ms, 0 | 20260507 08:45:56.948 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:56.948 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:45:56.948 | INFO | Uploaded 36GiB to GPU in 2.03122s (17.7234GiB/s) | 20260507 08:45:56.948 | INFO | Downloaded 34GiB to GPU in 1.97159s (17.2449GiB/s) | 20260507 08:46:02.088 | INFO | Analyzing chunk frames = [8633856,8634368) | 20260507 08:46:02.088 | INFO | MemInfo: { | 20260507 08:46:02.088 | INFO | "gpu" : " 17 GB", | 20260507 08:46:02.088 | INFO | "huge" : "", | 20260507 08:46:02.088 | INFO | "huge_shared" : " 27 GB", | 20260507 08:46:02.088 | INFO | "malloc" : " 14 GB", | 20260507 08:46:02.088 | INFO | "pinned" : " 29 GB" | 20260507 08:46:02.088 | INFO | } | 20260507 08:46:02.088 | INFO | Perf report for the last 5.141 seconds: | 20260507 08:46:02.088 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:46:02.088 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:02.088 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:02.088 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:02.088 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:46:02.089 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:02.089 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:02.089 | INFO | PRE_HQ : 27.6%, 1.0, 14.3ms, 0 | 20260507 08:46:02.089 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:02.089 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:02.089 | INFO | Uploaded 36GiB to GPU in 1.98252s (18.1587GiB/s) | 20260507 08:46:02.089 | INFO | Downloaded 34GiB to GPU in 1.95844s (17.3608GiB/s) | 20260507 08:46:02.089 | INFO | ----Memory usage report for Standard Host Memory----- | 20260507 08:46:02.089 | INFO | Total mem usage: 18 GiB | 20260507 08:46:02.089 | INFO | All Time High Water Marks sum: 18.1 GiB (Savings: 0.67%) | 20260507 08:46:02.089 | INFO | Recent High Water Marks Sum: 14.2 GiB (Savings: -26.7%) | 20260507 08:46:02.089 | INFO | Unused Cache: 0B (Waste: 0%) | 20260507 08:46:02.089 | INFO | High Water Breakdown: | 20260507 08:46:02.089 | INFO | 60.1% : 8.54 GiB ( 74.8% Of Peak) : PrelimHQFilter_packetsAllocator | 20260507 08:46:02.089 | INFO | 39.8% : 5.65 GiB ( 84.4% Of Peak) : PrelimHQFilter_metricsAllocator | 20260507 08:46:02.089 | INFO | 0.14% : 20.4 MiB ( 100% Of Peak) : SampledBazData | 20260507 08:46:02.089 | INFO | ------------------------------------------ | 20260507 08:46:02.089 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:46:02.089 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:46:02.089 | INFO | Total mem usage: 26.5 GiB | 20260507 08:46:02.089 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:46:02.089 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:46:02.089 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:46:02.089 | INFO | High Water Breakdown: | 20260507 08:46:02.089 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:46:02.089 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:46:02.089 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:46:02.089 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:46:02.089 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:46:02.089 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:46:02.089 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:46:02.089 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:46:02.089 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:46:02.089 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:46:02.089 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:46:02.089 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:46:02.089 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:46:02.089 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:46:02.089 | INFO | ------------------------------------------ | 20260507 08:46:02.089 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:46:02.089 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:46:02.089 | INFO | Total mem usage: 25.6 GiB | 20260507 08:46:02.089 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:46:02.089 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:46:02.089 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:46:02.089 | INFO | High Water Breakdown: | 20260507 08:46:02.089 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:46:02.089 | INFO | ------------------------------------------ | 20260507 08:46:02.089 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:46:02.090 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:46:02.090 | INFO | Total mem usage: 15.7 GiB | 20260507 08:46:02.090 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:46:02.090 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:46:02.090 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:46:02.090 | INFO | High Water Breakdown: | 20260507 08:46:02.090 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:46:02.090 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:46:02.090 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:46:02.090 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:46:02.090 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:46:02.090 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:46:02.090 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:46:02.090 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:46:02.090 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:46:02.090 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:46:02.090 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:46:02.090 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:46:02.090 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:46:02.090 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:46:02.090 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:46:02.090 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:46:02.090 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:46:02.090 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:46:02.090 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:46:02.090 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:46:02.090 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:46:02.090 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:46:02.090 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:46:02.090 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:46:02.090 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:46:02.090 | INFO | ------------------------------------------ | 20260507 08:46:02.090 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:46:02.090 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:46:02.090 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:46:02.090 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:46:02.090 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.47e+01, 1.34e+01 / 1.13e+00 | 20260507 08:46:02.090 | INFO | PerfReport: 0.29%, AggregateStats -- 9.05e-02 / 1.03e-01, 9.39e-03 / 6.50e-03 | 20260507 08:46:02.090 | INFO | PerfReport: 11.66%, Baseline -- 3.58e+00 / 4.50e+00, 5.01e-01 / 1.82e-01 | 20260507 08:46:02.090 | INFO | PerfReport: 3.22%, Histogram -- 9.89e-01 / 1.49e+00, 1.67e-01 / 1.06e-01 | 20260507 08:46:02.090 | INFO | PerfReport: 16.32%, DME -- 4.01e+01 / 4.49e+01, 3.87e+00 / 1.29e+00 | 20260507 08:46:02.090 | INFO | PerfReport: 57.77%, FrameLabeling -- 1.77e+01 / 1.95e+01, 1.30e+00 / 1.16e+00 | 20260507 08:46:02.090 | INFO | PerfReport: 6.86%, PulseAccumulating -- 2.11e+00 / 2.67e+00, 3.83e-01 / 2.13e-01 | 20260507 08:46:02.090 | INFO | PerfReport: 2.89%, Metrics -- 8.87e-01 / 1.19e+00, 2.38e-01 / 1.13e-01 | 20260507 08:46:02.090 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.97e-01 / 3.59e-01, 5.02e-02 / 2.54e-02 | 20260507 08:46:02.090 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:46:02.090 | INFO | PerfReport: Gathered 1536 samples. | 20260507 08:46:02.090 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:46:02.090 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.43e+01 / 0.00e+00 | 20260507 08:46:02.090 | INFO | PerfReport: 21.87%, Upload -- 2.10e+01 / 0.00e+00, 7.18e+00 / 0.00e+00 | 20260507 08:46:02.090 | INFO | PerfReport: 21.30%, Download -- 2.05e+01 / 0.00e+00, 7.23e+00 / 0.00e+00 | 20260507 08:46:02.090 | INFO | PerfReport: 31.90%, Compute -- 3.07e+01 / 0.00e+00, 1.34e+01 / 0.00e+00 | 20260507 08:46:02.090 | INFO | PerfReport: 1.68%, LockUpload -- 1.62e+00 / 0.00e+00, 5.11e+00 / 0.00e+00 | 20260507 08:46:02.090 | INFO | PerfReport: 1.26%, LockDownload -- 1.21e+00 / 0.00e+00, 3.01e+00 / 0.00e+00 | 20260507 08:46:02.090 | INFO | PerfReport: 21.97%, LockCompute -- 2.11e+01 / 0.00e+00, 1.86e+01 / 0.00e+00 | 20260507 08:46:02.090 | INFO | PerfReport: 0.01%, Misc -- 1.35e-02 / 0.00e+00, 1.60e-02 / 0.00e+00 | 20260507 08:46:02.186 | INFO | Starting a round of Baz IO | 20260507 08:46:05.274 | INFO | Populating RTMetrics report | 20260507 08:46:05.275 | INFO | RTMetrics report populated | 20260507 08:46:05.275 | INFO | Opening tmp RTMetrics json file | 20260507 08:46:05.289 | INFO | Serializing RTMetrics to json | 20260507 08:46:05.290 | INFO | Serialization done, swapping tmp file now... | 20260507 08:46:05.290 | INFO | RTMetrics done | 20260507 08:46:05.358 | INFO | Finished a round of Baz IO: Wrote 1.603 GiB in 3.172 seconds | 20260507 08:46:05.358 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.48% / 0.00% / 67.52% | 20260507 08:46:05.358 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0861367% / 7.94729e-06% / 8.74201e-05% | 20260507 08:46:05.358 | INFO | Baz IO overall: | 20260507 08:46:05.358 | INFO | BytesWritten : 4292038919073 | 20260507 08:46:05.358 | INFO | NumEvents : 2052407863592 | 20260507 08:46:07.228 | INFO | Analyzing chunk frames = [8634368,8634880) | 20260507 08:46:07.228 | INFO | MemInfo: { | 20260507 08:46:07.228 | INFO | "gpu" : " 17 GB", | 20260507 08:46:07.228 | INFO | "huge" : "", | 20260507 08:46:07.228 | INFO | "huge_shared" : " 27 GB", | 20260507 08:46:07.228 | INFO | "malloc" : " 14 GB", | 20260507 08:46:07.228 | INFO | "pinned" : " 29 GB" | 20260507 08:46:07.228 | INFO | } | 20260507 08:46:07.228 | INFO | Perf report for the last 5.13947 seconds: | 20260507 08:46:07.228 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:46:07.228 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:07.228 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:07.228 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:07.228 | INFO | ANALYSIS : 61.4%, 3.0, 95.5ms, 0 | 20260507 08:46:07.228 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:07.228 | INFO | RT_METRICS : 43.4%, 1.0, 22.7ms, 0 | 20260507 08:46:07.228 | INFO | PRE_HQ : 59.5%, 1.9, 58.3ms, 0 | 20260507 08:46:07.228 | INFO | BAZWRITER : 61.3%, 3.2, 100.8ms, 0 | 20260507 08:46:07.228 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.2ms, 0 | 20260507 08:46:07.228 | INFO | Uploaded 36GiB to GPU in 2.05146s (17.5485GiB/s) | 20260507 08:46:07.228 | INFO | Downloaded 38GiB to GPU in 2.17454s (17.475GiB/s) | 20260507 08:46:12.355 | INFO | Analyzing chunk frames = [8634880,8635392) | 20260507 08:46:12.355 | INFO | MemInfo: { | 20260507 08:46:12.355 | INFO | "gpu" : " 17 GB", | 20260507 08:46:12.355 | INFO | "huge" : "", | 20260507 08:46:12.355 | INFO | "huge_shared" : " 27 GB", | 20260507 08:46:12.355 | INFO | "malloc" : " 14 GB", | 20260507 08:46:12.356 | INFO | "pinned" : " 29 GB" | 20260507 08:46:12.356 | INFO | } | 20260507 08:46:12.356 | INFO | Perf report for the last 5.1277 seconds: | 20260507 08:46:12.356 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:46:12.356 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:12.356 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:12.356 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:12.356 | INFO | ANALYSIS : 60.9%, 3.0, 94.6ms, 0 | 20260507 08:46:12.356 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:12.356 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:12.356 | INFO | PRE_HQ : 25.9%, 1.0, 13.4ms, 0 | 20260507 08:46:12.356 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:12.356 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:12.356 | INFO | Uploaded 36GiB to GPU in 2.02463s (17.781GiB/s) | 20260507 08:46:12.356 | INFO | Downloaded 34GiB to GPU in 1.94891s (17.4457GiB/s) | 20260507 08:46:17.499 | INFO | Analyzing chunk frames = [8635392,8635904) | 20260507 08:46:17.499 | INFO | MemInfo: { | 20260507 08:46:17.499 | INFO | "gpu" : " 17 GB", | 20260507 08:46:17.499 | INFO | "huge" : "", | 20260507 08:46:17.499 | INFO | "huge_shared" : " 27 GB", | 20260507 08:46:17.499 | INFO | "malloc" : " 14 GB", | 20260507 08:46:17.499 | INFO | "pinned" : " 29 GB" | 20260507 08:46:17.499 | INFO | } | 20260507 08:46:17.499 | INFO | Perf report for the last 5.14366 seconds: | 20260507 08:46:17.499 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:46:17.499 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:17.499 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:17.499 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:17.499 | INFO | ANALYSIS : 59.8%, 3.0, 92.8ms, 0 | 20260507 08:46:17.499 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:17.499 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:17.499 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:46:17.499 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:17.499 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:17.500 | INFO | Uploaded 36GiB to GPU in 1.99571s (18.0387GiB/s) | 20260507 08:46:17.500 | INFO | Downloaded 34GiB to GPU in 1.94577s (17.4738GiB/s) | 20260507 08:46:22.640 | INFO | Analyzing chunk frames = [8635904,8636416) | 20260507 08:46:22.640 | INFO | MemInfo: { | 20260507 08:46:22.640 | INFO | "gpu" : " 17 GB", | 20260507 08:46:22.640 | INFO | "huge" : "", | 20260507 08:46:22.640 | INFO | "huge_shared" : " 27 GB", | 20260507 08:46:22.640 | INFO | "malloc" : " 14 GB", | 20260507 08:46:22.640 | INFO | "pinned" : " 29 GB" | 20260507 08:46:22.640 | INFO | } | 20260507 08:46:22.640 | INFO | Perf report for the last 5.14074 seconds: | 20260507 08:46:22.640 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:46:22.640 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:22.640 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:22.640 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:22.640 | INFO | ANALYSIS : 59.6%, 3.0, 92.4ms, 0 | 20260507 08:46:22.640 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:22.640 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:22.640 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:46:22.640 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:22.640 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:22.640 | INFO | Uploaded 36GiB to GPU in 2.01829s (17.8368GiB/s) | 20260507 08:46:22.640 | INFO | Downloaded 34GiB to GPU in 1.94781s (17.4555GiB/s) | 20260507 08:46:27.783 | INFO | Analyzing chunk frames = [8636416,8636928) | 20260507 08:46:27.783 | INFO | MemInfo: { | 20260507 08:46:27.783 | INFO | "gpu" : " 17 GB", | 20260507 08:46:27.783 | INFO | "huge" : "", | 20260507 08:46:27.783 | INFO | "huge_shared" : " 27 GB", | 20260507 08:46:27.783 | INFO | "malloc" : " 14 GB", | 20260507 08:46:27.783 | INFO | "pinned" : " 29 GB" | 20260507 08:46:27.783 | INFO | } | 20260507 08:46:27.783 | INFO | Perf report for the last 5.14328 seconds: | 20260507 08:46:27.784 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:46:27.784 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:27.784 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:27.784 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:27.784 | INFO | ANALYSIS : 60.0%, 3.0, 93.5ms, 0 | 20260507 08:46:27.784 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:27.784 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:27.784 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 08:46:27.784 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:27.784 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:27.784 | INFO | Uploaded 36GiB to GPU in 2.04341s (17.6176GiB/s) | 20260507 08:46:27.784 | INFO | Downloaded 34GiB to GPU in 1.91812s (17.7257GiB/s) | 20260507 08:46:32.922 | INFO | Analyzing chunk frames = [8636928,8637440) | 20260507 08:46:32.922 | INFO | MemInfo: { | 20260507 08:46:32.922 | INFO | "gpu" : " 17 GB", | 20260507 08:46:32.922 | INFO | "huge" : "", | 20260507 08:46:32.922 | INFO | "huge_shared" : " 27 GB", | 20260507 08:46:32.922 | INFO | "malloc" : " 14 GB", | 20260507 08:46:32.922 | INFO | "pinned" : " 29 GB" | 20260507 08:46:32.922 | INFO | } | 20260507 08:46:32.922 | INFO | Perf report for the last 5.13847 seconds: | 20260507 08:46:32.922 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:46:32.922 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:32.922 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:32.922 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:32.922 | INFO | ANALYSIS : 60.4%, 3.0, 94.1ms, 0 | 20260507 08:46:32.922 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:32.922 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:32.922 | INFO | PRE_HQ : 24.4%, 1.0, 12.7ms, 0 | 20260507 08:46:32.922 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:32.922 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:32.922 | INFO | Uploaded 36GiB to GPU in 2.02366s (17.7896GiB/s) | 20260507 08:46:32.922 | INFO | Downloaded 34GiB to GPU in 1.93566s (17.565GiB/s) | 20260507 08:46:38.057 | INFO | Analyzing chunk frames = [8637440,8637952) | 20260507 08:46:38.057 | INFO | MemInfo: { | 20260507 08:46:38.057 | INFO | "gpu" : " 17 GB", | 20260507 08:46:38.057 | INFO | "huge" : "", | 20260507 08:46:38.057 | INFO | "huge_shared" : " 27 GB", | 20260507 08:46:38.057 | INFO | "malloc" : " 14 GB", | 20260507 08:46:38.057 | INFO | "pinned" : " 29 GB" | 20260507 08:46:38.057 | INFO | } | 20260507 08:46:38.058 | INFO | Perf report for the last 5.13562 seconds: | 20260507 08:46:38.058 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:46:38.058 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:38.058 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:38.058 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:38.058 | INFO | ANALYSIS : 60.5%, 3.0, 94.2ms, 0 | 20260507 08:46:38.058 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:38.058 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:38.058 | INFO | PRE_HQ : 24.9%, 1.0, 12.9ms, 0 | 20260507 08:46:38.058 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:38.058 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:38.058 | INFO | Uploaded 36GiB to GPU in 1.90666s (18.8812GiB/s) | 20260507 08:46:38.058 | INFO | Downloaded 34GiB to GPU in 1.97467s (17.218GiB/s) | 20260507 08:46:43.201 | INFO | Analyzing chunk frames = [8637952,8638464) | 20260507 08:46:43.201 | INFO | MemInfo: { | 20260507 08:46:43.201 | INFO | "gpu" : " 17 GB", | 20260507 08:46:43.201 | INFO | "huge" : "", | 20260507 08:46:43.201 | INFO | "huge_shared" : " 27 GB", | 20260507 08:46:43.201 | INFO | "malloc" : " 14 GB", | 20260507 08:46:43.201 | INFO | "pinned" : " 29 GB" | 20260507 08:46:43.201 | INFO | } | 20260507 08:46:43.201 | INFO | Perf report for the last 5.14336 seconds: | 20260507 08:46:43.201 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:46:43.201 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:43.201 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:43.201 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:43.201 | INFO | ANALYSIS : 60.6%, 3.0, 94.3ms, 0 | 20260507 08:46:43.201 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:43.201 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:43.201 | INFO | PRE_HQ : 24.5%, 1.0, 12.7ms, 0 | 20260507 08:46:43.201 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:43.201 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:43.201 | INFO | Uploaded 36GiB to GPU in 1.99702s (18.0268GiB/s) | 20260507 08:46:43.202 | INFO | Downloaded 34GiB to GPU in 1.96017s (17.3454GiB/s) | 20260507 08:46:43.298 | INFO | Starting a round of Baz IO | 20260507 08:46:46.393 | INFO | Populating RTMetrics report | 20260507 08:46:46.393 | INFO | RTMetrics report populated | 20260507 08:46:46.393 | INFO | Opening tmp RTMetrics json file | 20260507 08:46:46.403 | INFO | Serializing RTMetrics to json | 20260507 08:46:46.407 | INFO | Serialization done, swapping tmp file now... | 20260507 08:46:46.408 | INFO | RTMetrics done | 20260507 08:46:46.478 | INFO | Finished a round of Baz IO: Wrote 1.602 GiB in 3.18 seconds | 20260507 08:46:46.478 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.48% / 0.00% / 67.52% | 20260507 08:46:46.478 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 0.0859459% / 0% / 7.15256e-05% | 20260507 08:46:46.478 | INFO | Baz IO overall: | 20260507 08:46:46.478 | INFO | BytesWritten : 4293759540590 | 20260507 08:46:46.479 | INFO | NumEvents : 2053208074127 | 20260507 08:46:48.342 | INFO | Analyzing chunk frames = [8638464,8638976) | 20260507 08:46:48.342 | INFO | MemInfo: { | 20260507 08:46:48.342 | INFO | "gpu" : " 17 GB", | 20260507 08:46:48.342 | INFO | "huge" : "", | 20260507 08:46:48.342 | INFO | "huge_shared" : " 27 GB", | 20260507 08:46:48.342 | INFO | "malloc" : " 14 GB", | 20260507 08:46:48.342 | INFO | "pinned" : " 29 GB" | 20260507 08:46:48.342 | INFO | } | 20260507 08:46:48.342 | INFO | Perf report for the last 5.1415 seconds: | 20260507 08:46:48.342 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:46:48.342 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:48.342 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:48.342 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:48.342 | INFO | ANALYSIS : 61.6%, 3.0, 95.9ms, 0 | 20260507 08:46:48.342 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:48.343 | INFO | RT_METRICS : 40.8%, 1.0, 21.4ms, 0 | 20260507 08:46:48.343 | INFO | PRE_HQ : 58.8%, 1.8, 54.2ms, 0 | 20260507 08:46:48.343 | INFO | BAZWRITER : 61.4%, 3.1, 96.7ms, 0 | 20260507 08:46:48.343 | INFO | BAZ_SAMPLING : 0.5%, 1.0, 0.3ms, 0 | 20260507 08:46:48.343 | INFO | Uploaded 36GiB to GPU in 2.07854s (17.3199GiB/s) | 20260507 08:46:48.343 | INFO | Downloaded 38GiB to GPU in 2.17544s (17.4677GiB/s) | 20260507 08:46:53.478 | INFO | Analyzing chunk frames = [8638976,8639488) | 20260507 08:46:53.478 | INFO | MemInfo: { | 20260507 08:46:53.478 | INFO | "gpu" : " 17 GB", | 20260507 08:46:53.478 | INFO | "huge" : "", | 20260507 08:46:53.478 | INFO | "huge_shared" : " 27 GB", | 20260507 08:46:53.479 | INFO | "malloc" : " 14 GB", | 20260507 08:46:53.479 | INFO | "pinned" : " 29 GB" | 20260507 08:46:53.479 | INFO | } | 20260507 08:46:53.479 | INFO | Perf report for the last 5.13622 seconds: | 20260507 08:46:53.479 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:46:53.479 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:53.479 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:53.479 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:53.479 | INFO | ANALYSIS : 60.7%, 3.0, 94.5ms, 0 | 20260507 08:46:53.479 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:53.479 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:53.479 | INFO | PRE_HQ : 24.0%, 1.0, 12.4ms, 0 | 20260507 08:46:53.479 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:53.479 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:53.479 | INFO | Uploaded 36GiB to GPU in 2.01847s (17.8353GiB/s) | 20260507 08:46:53.479 | INFO | Downloaded 34GiB to GPU in 1.9425s (17.5032GiB/s) | 20260507 08:46:58.616 | INFO | Reached end of movie, endFrame:8640000 expectedEndFrame:8640000 | 20260507 08:46:58.616 | INFO | DataSourceRunner thread exiting | 20260507 08:46:58.616 | INFO | Analyzing chunk frames = [8639488,8640000) | 20260507 08:46:58.616 | INFO | MemInfo: { | 20260507 08:46:58.616 | INFO | "gpu" : " 17 GB", | 20260507 08:46:58.616 | INFO | "huge" : "", | 20260507 08:46:58.616 | INFO | "huge_shared" : " 27 GB", | 20260507 08:46:58.616 | INFO | "malloc" : " 14 GB", | 20260507 08:46:58.616 | INFO | "pinned" : " 29 GB" | 20260507 08:46:58.616 | INFO | } | 20260507 08:46:58.616 | INFO | Perf report for the last 5.13765 seconds: | 20260507 08:46:58.616 | INFO | (Active%, Avg Active Occupancy, Avg Duration, QueueSize) | 20260507 08:46:58.616 | INFO | REPACKER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:58.616 | INFO | PREP_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:58.616 | INFO | SAVE_TRACE : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:58.616 | INFO | ANALYSIS : 60.4%, 3.0, 93.7ms, 0 | 20260507 08:46:58.616 | INFO | PREP_RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:58.616 | INFO | RT_METRICS : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:58.616 | INFO | PRE_HQ : 24.2%, 1.0, 12.5ms, 0 | 20260507 08:46:58.616 | INFO | BAZWRITER : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:58.616 | INFO | BAZ_SAMPLING : 0.0%, 1.0, 0.0ms, 0 | 20260507 08:46:58.617 | INFO | Uploaded 36GiB to GPU in 2.03243s (17.7128GiB/s) | 20260507 08:46:58.617 | INFO | Downloaded 34GiB to GPU in 1.91938s (17.714GiB/s) | 20260507 08:46:58.617 | INFO | Flushing graph node REPACKER with 0 jobs. | 20260507 08:47:01.703 | INFO | Flushing graph node PREP_TRACE with 0 jobs. | 20260507 08:47:01.703 | INFO | Flushing graph node SAVE_TRACE with 0 jobs. | 20260507 08:47:01.703 | INFO | Flushing graph node SAVE_TRACE done. | 20260507 08:47:01.703 | INFO | Flushing graph node PREP_TRACE done. | 20260507 08:47:01.703 | INFO | Flushing graph node ANALYSIS with 0 jobs. | 20260507 08:47:01.703 | INFO | Flushing graph node PREP_RT_METRICS with 0 jobs. | 20260507 08:47:01.703 | INFO | Flushing graph node RT_METRICS with 0 jobs. | 20260507 08:47:01.703 | INFO | Flushing graph node RT_METRICS done. | 20260507 08:47:01.703 | INFO | Flushing graph node PREP_RT_METRICS done. | 20260507 08:47:01.703 | INFO | Flushing graph node PRE_HQ with 96 jobs. | 20260507 08:47:01.715 | INFO | Starting a round of Baz IO | 20260507 08:47:05.924 | INFO | Finished a round of Baz IO: Wrote 1.951 GiB in 4.209 seconds | 20260507 08:47:05.924 | INFO | ZMW percent preHQ/newHQ/alreadyHQ: 32.48% / 0.00% / 67.52% | 20260507 08:47:05.924 | INFO | ZMW percent with data loss expired/newHQ/alreadyHQ: 2.78155e-05% / 0% / 5.5631e-05% | 20260507 08:47:05.924 | INFO | Baz IO overall: | 20260507 08:47:05.924 | INFO | BytesWritten : 4295854321204 | 20260507 08:47:05.924 | INFO | NumEvents : 2053508955049 | 20260507 08:47:05.924 | INFO | Flushing graph node BAZWRITER with 0 jobs. | 20260507 08:47:05.924 | INFO | Flushing graph node BAZWRITER done. | 20260507 08:47:05.924 | INFO | Flushing graph node BAZ_SAMPLING with 0 jobs. | 20260507 08:47:05.924 | INFO | Flushing graph node BAZ_SAMPLING done. | 20260507 08:47:05.924 | INFO | Flushing graph node PRE_HQ done. | 20260507 08:47:05.924 | INFO | Flushing graph node ANALYSIS done. | 20260507 08:47:05.924 | INFO | Flushing graph node REPACKER done. | 20260507 08:47:05.924 | INFO | Exited chunk analysis loop. | 20260507 08:47:05.924 | INFO | framesAnalyzed:8640000 framesRequested:8640000 | 20260507 08:47:05.924 | INFO | All frames analyzed = 8640000 | 20260507 08:47:05.925 | INFO | Analyzed 16875 chunks at 0.194557 chunks/sec (4.8962e+06 zmws/sec) | 20260507 08:47:06.107 | INFO | Intermediate Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:47:06.107 | INFO | PerfReport: Gathered 1152 samples. | 20260507 08:47:06.107 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:47:06.107 | INFO | PerfReport: Full Execution -- 3.07e+01 / 7.45e+01, 1.32e+01 / 1.05e+00 | 20260507 08:47:06.107 | INFO | PerfReport: 0.29%, AggregateStats -- 9.06e-02 / 1.02e-01, 9.74e-03 / 5.79e-03 | 20260507 08:47:06.107 | INFO | PerfReport: 11.68%, Baseline -- 3.59e+00 / 4.41e+00, 5.11e-01 / 3.22e-01 | 20260507 08:47:06.107 | INFO | PerfReport: 3.23%, Histogram -- 9.94e-01 / 1.48e+00, 1.69e-01 / 1.21e-01 | 20260507 08:47:06.107 | INFO | PerfReport: 16.19%, DME -- 3.98e+01 / 4.41e+01, 3.58e+00 / 1.04e+00 | 20260507 08:47:06.107 | INFO | PerfReport: 57.80%, FrameLabeling -- 1.78e+01 / 2.02e+01, 1.34e+00 / 1.23e+00 | 20260507 08:47:06.107 | INFO | PerfReport: 6.85%, PulseAccumulating -- 2.10e+00 / 2.62e+00, 3.80e-01 / 1.81e-01 | 20260507 08:47:06.107 | INFO | PerfReport: 2.97%, Metrics -- 9.14e-01 / 1.21e+00, 2.44e-01 / 1.25e-01 | 20260507 08:47:06.107 | INFO | PerfReport: 0.97%, PulseSerializing -- 2.99e-01 / 3.57e-01, 4.94e-02 / 2.04e-02 | 20260507 08:47:06.107 | INFO | Final Performance Profiling Results for AnalysisStages: (reported in milliseconds) | 20260507 08:47:06.107 | INFO | PerfReport: Gathered 1615776 samples. | 20260507 08:47:06.107 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:47:06.107 | INFO | PerfReport: Full Execution -- 3.10e+01 / 7.55e+01, 1.33e+01 / 2.04e+00 | 20260507 08:47:06.107 | INFO | PerfReport: 0.29%, AggregateStats -- 9.08e-02 / 1.02e-01, 2.85e-02 / 1.18e-01 | 20260507 08:47:06.107 | INFO | PerfReport: 11.56%, Baseline -- 3.59e+00 / 4.42e+00, 5.16e-01 / 3.14e-01 | 20260507 08:47:06.107 | INFO | PerfReport: 3.22%, Histogram -- 1.00e+00 / 1.46e+00, 1.70e-01 / 1.29e-01 | 20260507 08:47:06.107 | INFO | PerfReport: 16.04%, DME -- 3.98e+01 / 4.46e+01, 3.91e+00 / 2.27e+00 | 20260507 08:47:06.107 | INFO | PerfReport: 57.19%, FrameLabeling -- 1.77e+01 / 2.04e+01, 1.30e+00 / 1.66e+00 | 20260507 08:47:06.107 | INFO | PerfReport: 7.19%, PulseAccumulating -- 2.23e+00 / 2.81e+00, 3.71e-01 / 2.20e-01 | 20260507 08:47:06.107 | INFO | PerfReport: 3.34%, Metrics -- 1.03e+00 / 1.37e+00, 2.61e-01 / 1.54e-01 | 20260507 08:47:06.107 | INFO | PerfReport: 1.16%, PulseSerializing -- 3.61e-01 / 4.06e-01, 5.78e-02 / 3.69e-02 | 20260507 08:47:06.107 | INFO | Intermediate Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:47:06.107 | INFO | PerfReport: Gathered 1152 samples. | 20260507 08:47:06.107 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:47:06.107 | INFO | PerfReport: Full Execution -- 9.60e+01 / -nan, 2.36e+01 / 0.00e+00 | 20260507 08:47:06.107 | INFO | PerfReport: 21.91%, Upload -- 2.10e+01 / 0.00e+00, 7.10e+00 / 0.00e+00 | 20260507 08:47:06.107 | INFO | PerfReport: 21.54%, Download -- 2.07e+01 / 0.00e+00, 7.34e+00 / 0.00e+00 | 20260507 08:47:06.107 | INFO | PerfReport: 32.02%, Compute -- 3.07e+01 / 0.00e+00, 1.32e+01 / 0.00e+00 | 20260507 08:47:06.107 | INFO | PerfReport: 1.48%, LockUpload -- 1.42e+00 / 0.00e+00, 4.84e+00 / 0.00e+00 | 20260507 08:47:06.107 | INFO | PerfReport: 1.26%, LockDownload -- 1.21e+00 / 0.00e+00, 3.02e+00 / 0.00e+00 | 20260507 08:47:06.107 | INFO | PerfReport: 21.77%, LockCompute -- 2.09e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 08:47:06.107 | INFO | PerfReport: 0.02%, Misc -- 1.47e-02 / 0.00e+00, 2.58e-02 / 0.00e+00 | 20260507 08:47:06.107 | INFO | Final Performance Profiling Results for IOStages: (reported in milliseconds) | 20260507 08:47:06.107 | INFO | PerfReport: Gathered 1620000 samples. | 20260507 08:47:06.107 | INFO | PerfReport: % Compute, FullMean/OutlierMean FullDev/OutlierDev | 20260507 08:47:06.108 | INFO | PerfReport: Full Execution -- 9.62e+01 / -nan, 2.33e+01 / 0.00e+00 | 20260507 08:47:06.108 | INFO | PerfReport: 21.33%, Upload -- 2.05e+01 / 0.00e+00, 6.71e+00 / 0.00e+00 | 20260507 08:47:06.108 | INFO | PerfReport: 20.91%, Download -- 2.01e+01 / 0.00e+00, 6.99e+00 / 0.00e+00 | 20260507 08:47:06.108 | INFO | PerfReport: 32.23%, Compute -- 3.10e+01 / 0.00e+00, 1.33e+01 / 0.00e+00 | 20260507 08:47:06.108 | INFO | PerfReport: 1.39%, LockUpload -- 1.34e+00 / 0.00e+00, 5.30e+00 / 0.00e+00 | 20260507 08:47:06.108 | INFO | PerfReport: 1.07%, LockDownload -- 1.03e+00 / 0.00e+00, 2.76e+00 / 0.00e+00 | 20260507 08:47:06.108 | INFO | PerfReport: 23.05%, LockCompute -- 2.22e+01 / 0.00e+00, 1.82e+01 / 0.00e+00 | 20260507 08:47:06.108 | INFO | PerfReport: 0.01%, Misc -- 1.40e-02 / 0.00e+00, 2.15e-02 / 0.00e+00 | 20260507 08:47:06.167 | INFO | Closing BAZ file: /data/nrta/0/m84031_260506_033729_s1/pb_internal/baz/m84031_260506_033729_s1.bazfilelist.txt | 20260507 08:47:06.172 | INFO | 90 baz files left to close | 20260507 08:47:06.172 | INFO | 80 baz files left to close | 20260507 08:47:06.173 | INFO | 70 baz files left to close | 20260507 08:47:06.174 | INFO | 60 baz files left to close | 20260507 08:47:06.175 | INFO | 50 baz files left to close | 20260507 08:47:06.176 | INFO | 40 baz files left to close | 20260507 08:47:06.176 | INFO | 30 baz files left to close | 20260507 08:47:06.177 | INFO | 20 baz files left to close | 20260507 08:47:06.178 | INFO | 10 baz files left to close | 20260507 08:47:06.178 | INFO | 0 baz files left to close | 20260507 08:47:06.178 | INFO | Header/Footer Bytes : 2.395 GiB | 20260507 08:47:06.178 | INFO | OverheadBytes : 117 GiB | 20260507 08:47:06.178 | INFO | EventBytes : 3.625 TiB | 20260507 08:47:06.178 | INFO | MetricsBytes : 169.1 GiB | 20260507 08:47:06.179 | INFO | NumEvents : 2053508955049 | 20260507 08:47:06.179 | INFO | NumMetrics : 2193969213 | 20260507 08:47:06.179 | INFO | ------------------------------------------ | 20260507 08:47:06.179 | INFO | Total BytesWritten : 3.907 TiB | 20260507 08:47:06.179 | INFO | ------------------------------------------ | 20260507 08:47:06.179 | INFO | UncompressedEventBytes : 4.105 TiB | 20260507 08:47:06.179 | INFO | UncompressedMetricsBytes : 192.1 GiB | 20260507 08:47:06.179 | INFO | Data compression ratio : 1.13 | 20260507 08:47:06.179 | INFO | ------------------------------------------ | 20260507 08:47:06.179 | INFO | 0.000287% of recorded slices had data loss, with 0.000187% from the start of a preliminary HQ regions | 20260507 08:47:06.179 | INFO | Closing baz file "/data/nrta/0/m84031_260506_033729_s1/pb_internal/baz/m84031_260506_033729_s1_basic_preview.baz": | 20260507 08:47:06.179 | INFO | Header/Footer Bytes : 2.464 MiB | 20260507 08:47:06.179 | INFO | OverheadBytes : 118.5 MiB | 20260507 08:47:06.179 | INFO | EventBytes : 3.665 GiB | 20260507 08:47:06.179 | INFO | MetricsBytes : 171.8 MiB | 20260507 08:47:06.179 | INFO | NumEvents : 2028202273 | 20260507 08:47:06.179 | INFO | NumMetrics : 2176844 | 20260507 08:47:06.179 | INFO | ------------------------------------------ | 20260507 08:47:06.179 | INFO | Total BytesWritten : 3.951 GiB | 20260507 08:47:06.179 | INFO | ------------------------------------------ | 20260507 08:47:06.179 | INFO | UncompressedEventBytes : 4.15 GiB | 20260507 08:47:06.179 | INFO | UncompressedMetricsBytes : 195.1 MiB | 20260507 08:47:06.179 | INFO | Data compression ratio : 1.13 | 20260507 08:47:06.179 | INFO | ------------------------------------------ | 20260507 08:47:06.179 | INFO | 0.000139% of recorded slices had data loss, with 0% from the start of a preliminary HQ regions | 20260507 08:47:06.179 | INFO | Closing baz file "/data/nrta/0/m84031_260506_033729_s1/pb_internal/baz/m84031_260506_033729_s1_full_preview.baz": | 20260507 08:47:06.179 | INFO | Header/Footer Bytes : 305.9 KiB | 20260507 08:47:06.179 | INFO | OverheadBytes : 13.13 MiB | 20260507 08:47:06.179 | INFO | EventBytes : 415.8 MiB | 20260507 08:47:06.180 | INFO | MetricsBytes : 18.24 MiB | 20260507 08:47:06.180 | INFO | NumEvents : 224949030 | 20260507 08:47:06.180 | INFO | NumMetrics : 230840 | 20260507 08:47:06.180 | INFO | ------------------------------------------ | 20260507 08:47:06.180 | INFO | Total BytesWritten : 447.5 MiB | 20260507 08:47:06.180 | INFO | ------------------------------------------ | 20260507 08:47:06.180 | INFO | UncompressedEventBytes : 470.5 MiB | 20260507 08:47:06.180 | INFO | UncompressedMetricsBytes : 20.69 MiB | 20260507 08:47:06.180 | INFO | Data compression ratio : 1.13 | 20260507 08:47:06.180 | INFO | ------------------------------------------ | 20260507 08:47:06.180 | INFO | 0.000167% of recorded slices had data loss, with 0% from the start of a preliminary HQ regions | 20260507 08:47:06.183 | NOTICE | ~WXIPCDataSource | 20260507 08:47:06.185 | NOTICE | Packet History start: now:220387.469 1970-01-03T13:13:07.469Z | 20260507 08:47:06.185 | NOTICE | packetHistory[0] 220368.282 2026-05-07T08:46:46.999Z 0x680000000 66963317089 free:0 | 20260507 08:47:06.185 | NOTICE | packetHistory[1] 220368.282 2026-05-07T08:46:46.999Z BURST 1 pointers, free:0 | 20260507 08:47:06.185 | NOTICE | packetHistory[2] 220368.326 2026-05-07T08:46:47.043Z 0x7d8000000 67097534817 free:0 | 20260507 08:47:06.185 | NOTICE | packetHistory[3] 220368.326 2026-05-07T08:46:47.043Z BURST 1 pointers, free:0 | 20260507 08:47:06.185 | NOTICE | packetHistory[4] 220368.368 2026-05-07T08:46:47.085Z 0x6e0000000 67231752545 free:0 | 20260507 08:47:06.185 | NOTICE | packetHistory[5] 220368.368 2026-05-07T08:46:47.085Z BURST 1 pointers, free:0 | 20260507 08:47:06.185 | NOTICE | packetHistory[6] 220368.410 2026-05-07T08:46:47.127Z 0x930000000 67365970273 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[7] 220368.410 2026-05-07T08:46:47.127Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[8] 220368.453 2026-05-07T08:46:47.170Z 0xa28000000 67500188001 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[9] 220368.453 2026-05-07T08:46:47.170Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[10] 220368.496 2026-05-07T08:46:47.213Z 0xa78000000 67634405729 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[11] 220368.496 2026-05-07T08:46:47.213Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[12] 220368.538 2026-05-07T08:46:47.255Z 0xae0000000 67768623457 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[13] 220368.538 2026-05-07T08:46:47.255Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[14] 220368.581 2026-05-07T08:46:47.298Z 0x550000000 67902841185 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[15] 220368.581 2026-05-07T08:46:47.298Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[16] 220369.670 2026-05-07T08:46:48.387Z 0x598000000 68145062241 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[17] 220369.670 2026-05-07T08:46:48.387Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[18] 220369.712 2026-05-07T08:46:48.429Z 0x5b0000000 68012941669 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[19] 220369.712 2026-05-07T08:46:48.429Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[20] 220369.755 2026-05-07T08:46:48.472Z 0xa38000000 68012941669 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[21] 220369.755 2026-05-07T08:46:48.472Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[22] 220369.797 2026-05-07T08:46:48.514Z 0xa30000000 67768623481 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[23] 220369.797 2026-05-07T08:46:48.514Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[24] 220369.841 2026-05-07T08:46:48.558Z 0x7e0000000 68012941669 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[25] 220369.841 2026-05-07T08:46:48.558Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[26] 220369.883 2026-05-07T08:46:48.600Z 0x600000000 67878723941 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[27] 220369.883 2026-05-07T08:46:48.600Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[28] 220369.925 2026-05-07T08:46:48.642Z 0x610000000 67744506213 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[29] 220369.925 2026-05-07T08:46:48.642Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[30] 220369.968 2026-05-07T08:46:48.685Z 0xad0000000 67610288485 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[31] 220369.968 2026-05-07T08:46:48.685Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[32] 220370.011 2026-05-07T08:46:48.728Z 0x720000000 67610288509 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[33] 220370.011 2026-05-07T08:46:48.728Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[34] 220370.054 2026-05-07T08:46:48.771Z 0xb30000000 67500188001 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[35] 220370.054 2026-05-07T08:46:48.771Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[36] 220370.096 2026-05-07T08:46:48.813Z 0x898000000 67610288485 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[37] 220370.096 2026-05-07T08:46:48.813Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[38] 220370.138 2026-05-07T08:46:48.855Z 0x540000000 67476070757 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[39] 220370.138 2026-05-07T08:46:48.855Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[40] 220370.182 2026-05-07T08:46:48.899Z 0xab8000000 67341853029 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[41] 220370.182 2026-05-07T08:46:48.899Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[42] 220370.224 2026-05-07T08:46:48.941Z 0x960000000 67341853029 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[43] 220370.224 2026-05-07T08:46:48.941Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[44] 220370.267 2026-05-07T08:46:48.984Z 0x8e0000000 67097534841 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[45] 220370.267 2026-05-07T08:46:48.984Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[46] 220370.309 2026-05-07T08:46:49.026Z 0xa98000000 67341853029 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[47] 220370.309 2026-05-07T08:46:49.026Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[48] 220370.353 2026-05-07T08:46:49.070Z 0x950000000 67315638629 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[49] 220370.353 2026-05-07T08:46:49.070Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[50] 220370.395 2026-05-07T08:46:49.112Z 0x850000000 67073417573 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[51] 220370.395 2026-05-07T08:46:49.112Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[52] 220370.437 2026-05-07T08:46:49.154Z 0x8f0000000 67073417573 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[53] 220370.437 2026-05-07T08:46:49.154Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[54] 220370.480 2026-05-07T08:46:49.197Z 0x7f8000000 66939199869 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[55] 220370.480 2026-05-07T08:46:49.197Z BURST 1 pointers, free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[56] 220370.523 2026-05-07T08:46:49.240Z 0x910000000 66829099361 free:0 | 20260507 08:47:06.186 | NOTICE | packetHistory[57] 220370.523 2026-05-07T08:46:49.240Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[58] 220370.566 2026-05-07T08:46:49.283Z 0x9b8000000 67047203173 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[59] 220370.566 2026-05-07T08:46:49.283Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[60] 220370.609 2026-05-07T08:46:49.326Z 0x920000000 66804982117 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[61] 220370.609 2026-05-07T08:46:49.326Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[62] 220370.651 2026-05-07T08:46:49.368Z 0xaa8000000 66804982117 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[63] 220370.651 2026-05-07T08:46:49.368Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[64] 220370.693 2026-05-07T08:46:49.410Z 0x858000000 66670764389 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[65] 220370.693 2026-05-07T08:46:49.410Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[66] 220370.736 2026-05-07T08:46:49.453Z 0x888000000 66536546685 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[67] 220370.736 2026-05-07T08:46:49.453Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[68] 220370.779 2026-05-07T08:46:49.496Z 0x8d0000000 66668667233 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[69] 220370.779 2026-05-07T08:46:49.496Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[70] 220370.822 2026-05-07T08:46:49.539Z 0x620000000 66536546661 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[71] 220370.822 2026-05-07T08:46:49.539Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[72] 220370.864 2026-05-07T08:46:49.581Z 0x6e8000000 66536546661 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[73] 220370.864 2026-05-07T08:46:49.581Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[74] 220370.907 2026-05-07T08:46:49.624Z 0xb00000000 66402328933 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[75] 220370.907 2026-05-07T08:46:49.624Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[76] 220370.949 2026-05-07T08:46:49.666Z 0xb10000000 66376114533 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[77] 220370.949 2026-05-07T08:46:49.666Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[78] 220370.993 2026-05-07T08:46:49.710Z 0x5c0000000 66158010721 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[79] 220370.993 2026-05-07T08:46:49.710Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[80] 220371.035 2026-05-07T08:46:49.752Z 0x9f0000000 66268111205 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[81] 220371.035 2026-05-07T08:46:49.752Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[82] 220371.077 2026-05-07T08:46:49.794Z 0x688000000 66133893477 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[83] 220371.077 2026-05-07T08:46:49.794Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[84] 220371.120 2026-05-07T08:46:49.837Z 0xa08000000 66133893477 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[85] 220371.120 2026-05-07T08:46:49.837Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[86] 220371.163 2026-05-07T08:46:49.880Z 0x5f8000000 65999675749 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[87] 220371.163 2026-05-07T08:46:49.880Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[88] 220371.206 2026-05-07T08:46:49.923Z 0x828000000 65865458045 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[89] 220371.206 2026-05-07T08:46:49.923Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[90] 220371.248 2026-05-07T08:46:49.965Z 0x588000000 65997578593 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[91] 220371.248 2026-05-07T08:46:49.965Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[92] 220371.291 2026-05-07T08:46:50.008Z 0xae8000000 65865458021 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[93] 220371.291 2026-05-07T08:46:50.008Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[94] 220371.334 2026-05-07T08:46:50.051Z 0x9e0000000 65731240293 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[95] 220371.334 2026-05-07T08:46:50.051Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[96] 220371.376 2026-05-07T08:46:50.093Z 0x998000000 65731240293 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[97] 220371.376 2026-05-07T08:46:50.093Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[98] 220371.419 2026-05-07T08:46:50.136Z 0x7c0000000 65705025893 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[99] 220371.419 2026-05-07T08:46:50.136Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[100] 220371.461 2026-05-07T08:46:50.178Z 0x860000000 65486922081 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[101] 220371.461 2026-05-07T08:46:50.178Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[102] 220371.505 2026-05-07T08:46:50.222Z 0xa18000000 65705025893 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[103] 220371.505 2026-05-07T08:46:50.222Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[104] 220371.547 2026-05-07T08:46:50.264Z 0x500000000 65462804837 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[105] 220371.547 2026-05-07T08:46:50.264Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[106] 220371.590 2026-05-07T08:46:50.307Z 0xa58000000 65462804837 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[107] 220371.590 2026-05-07T08:46:50.307Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[108] 220371.632 2026-05-07T08:46:50.349Z 0x810000000 65438687593 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[109] 220371.632 2026-05-07T08:46:50.349Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[110] 220371.675 2026-05-07T08:46:50.392Z 0x580000000 65328587133 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[111] 220371.675 2026-05-07T08:46:50.392Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[112] 220371.717 2026-05-07T08:46:50.434Z 0x548000000 65218486625 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[113] 220371.717 2026-05-07T08:46:50.434Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[114] 220371.761 2026-05-07T08:46:50.478Z 0x698000000 65436590437 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[115] 220371.761 2026-05-07T08:46:50.478Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[116] 220371.803 2026-05-07T08:46:50.520Z 0xb38000000 65438687593 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[117] 220371.803 2026-05-07T08:46:50.520Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[118] 220371.845 2026-05-07T08:46:50.562Z 0x650000000 65328587109 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[119] 220371.845 2026-05-07T08:46:50.562Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[120] 220371.888 2026-05-07T08:46:50.605Z 0xa60000000 65328587109 free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[121] 220371.888 2026-05-07T08:46:50.605Z BURST 1 pointers, free:0 | 20260507 08:47:06.187 | NOTICE | packetHistory[122] 220371.931 2026-05-07T08:46:50.648Z 0x668000000 65084268897 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[123] 220371.931 2026-05-07T08:46:50.648Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[124] 220371.974 2026-05-07T08:46:50.691Z 0x618000000 65194369405 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[125] 220371.974 2026-05-07T08:46:50.691Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[126] 220372.016 2026-05-07T08:46:50.733Z 0x970000000 65084268897 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[127] 220372.016 2026-05-07T08:46:50.733Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[128] 220372.059 2026-05-07T08:46:50.776Z 0xac8000000 65328587109 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[129] 220372.059 2026-05-07T08:46:50.776Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[130] 220372.102 2026-05-07T08:46:50.819Z 0x8c0000000 65302372709 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[131] 220372.102 2026-05-07T08:46:50.819Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[132] 220372.144 2026-05-07T08:46:50.861Z 0x608000000 65304469865 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[133] 220372.144 2026-05-07T08:46:50.861Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[134] 220372.187 2026-05-07T08:46:50.904Z 0xab0000000 65194369381 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[135] 220372.187 2026-05-07T08:46:50.904Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[136] 220372.230 2026-05-07T08:46:50.947Z 0x8e8000000 65060151653 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[137] 220372.230 2026-05-07T08:46:50.947Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[138] 220372.272 2026-05-07T08:46:50.989Z 0x678000000 65060151653 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[139] 220372.272 2026-05-07T08:46:50.989Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[140] 220372.315 2026-05-07T08:46:51.032Z 0x880000000 65060151677 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[141] 220372.315 2026-05-07T08:46:51.032Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[142] 220372.357 2026-05-07T08:46:51.074Z 0x900000000 65194369381 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[143] 220372.357 2026-05-07T08:46:51.074Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[144] 220372.401 2026-05-07T08:46:51.118Z 0x9d0000000 65194369381 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[145] 220372.401 2026-05-07T08:46:51.118Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[146] 220372.443 2026-05-07T08:46:51.160Z 0xa88000000 65194369381 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[147] 220372.443 2026-05-07T08:46:51.160Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[148] 220372.486 2026-05-07T08:46:51.203Z 0x670000000 65168154981 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[149] 220372.486 2026-05-07T08:46:51.203Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[150] 220372.528 2026-05-07T08:46:51.245Z 0x988000000 65060151653 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[151] 220372.528 2026-05-07T08:46:51.245Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[152] 220372.571 2026-05-07T08:46:51.288Z 0x8f8000000 65060151653 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[153] 220372.571 2026-05-07T08:46:51.288Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[154] 220372.613 2026-05-07T08:46:51.330Z 0x538000000 65060151677 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[155] 220372.613 2026-05-07T08:46:51.330Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[156] 220372.656 2026-05-07T08:46:51.373Z 0x728000000 65058054497 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[157] 220372.656 2026-05-07T08:46:51.373Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[158] 220372.699 2026-05-07T08:46:51.416Z 0xaf8000000 65060151653 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[159] 220372.699 2026-05-07T08:46:51.416Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[160] 220372.742 2026-05-07T08:46:51.459Z 0x9a0000000 65060151653 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[161] 220372.742 2026-05-07T08:46:51.459Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[162] 220372.784 2026-05-07T08:46:51.501Z 0x6b8000000 64950051169 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[163] 220372.784 2026-05-07T08:46:51.501Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[164] 220372.827 2026-05-07T08:46:51.544Z 0x6c0000000 65084268897 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[165] 220372.827 2026-05-07T08:46:51.544Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[166] 220372.869 2026-05-07T08:46:51.586Z 0x870000000 65218486625 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[167] 220372.869 2026-05-07T08:46:51.586Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[168] 220372.913 2026-05-07T08:46:51.630Z 0x718000000 65352704353 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[169] 220372.913 2026-05-07T08:46:51.630Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[170] 220372.955 2026-05-07T08:46:51.672Z 0x878000000 65486922081 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[171] 220372.955 2026-05-07T08:46:51.672Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[172] 220372.997 2026-05-07T08:46:51.714Z 0x7f0000000 65621139809 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[173] 220372.997 2026-05-07T08:46:51.714Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[174] 220373.040 2026-05-07T08:46:51.757Z 0x918000000 65755357537 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[175] 220373.040 2026-05-07T08:46:51.757Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[176] 220373.082 2026-05-07T08:46:51.799Z 0xb08000000 65889575265 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[177] 220373.082 2026-05-07T08:46:51.799Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[178] 220373.126 2026-05-07T08:46:51.843Z 0x800000000 66023792993 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[179] 220373.126 2026-05-07T08:46:51.843Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[180] 220373.168 2026-05-07T08:46:51.885Z 0x700000000 66158010721 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[181] 220373.168 2026-05-07T08:46:51.885Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[182] 220373.211 2026-05-07T08:46:51.928Z 0x928000000 66292228449 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[183] 220373.211 2026-05-07T08:46:51.928Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[184] 220373.253 2026-05-07T08:46:51.970Z 0x890000000 66426446177 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[185] 220373.253 2026-05-07T08:46:51.970Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[186] 220373.296 2026-05-07T08:46:52.013Z 0x690000000 66560663905 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[187] 220373.296 2026-05-07T08:46:52.013Z BURST 1 pointers, free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[188] 220373.339 2026-05-07T08:46:52.056Z 0x520000000 66694881633 free:0 | 20260507 08:47:06.188 | NOTICE | packetHistory[189] 220373.339 2026-05-07T08:46:52.056Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[190] 220373.381 2026-05-07T08:46:52.098Z 0x6d0000000 66829099361 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[191] 220373.381 2026-05-07T08:46:52.098Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[192] 220373.424 2026-05-07T08:46:52.141Z 0x7e8000000 66963317089 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[193] 220373.424 2026-05-07T08:46:52.141Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[194] 220373.467 2026-05-07T08:46:52.184Z 0x840000000 67097534817 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[195] 220373.467 2026-05-07T08:46:52.184Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[196] 220373.510 2026-05-07T08:46:52.227Z 0xa20000000 67231752545 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[197] 220373.510 2026-05-07T08:46:52.227Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[198] 220373.552 2026-05-07T08:46:52.269Z 0x5f0000000 67365970273 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[199] 220373.552 2026-05-07T08:46:52.269Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[200] 220373.594 2026-05-07T08:46:52.311Z 0x9c8000000 67500188001 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[201] 220373.594 2026-05-07T08:46:52.311Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[202] 220373.637 2026-05-07T08:46:52.354Z 0xb48000000 67634405729 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[203] 220373.637 2026-05-07T08:46:52.354Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[204] 220373.681 2026-05-07T08:46:52.398Z 0x830000000 67768623457 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[205] 220373.681 2026-05-07T08:46:52.398Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[206] 220373.723 2026-05-07T08:46:52.440Z 0x868000000 67902841185 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[207] 220373.723 2026-05-07T08:46:52.440Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[208] 220374.805 2026-05-07T08:46:53.522Z 0xa70000000 68145062241 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[209] 220374.805 2026-05-07T08:46:53.522Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[210] 220374.847 2026-05-07T08:46:53.564Z 0x638000000 68012941669 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[211] 220374.847 2026-05-07T08:46:53.564Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[212] 220374.890 2026-05-07T08:46:53.607Z 0x8a0000000 68012941669 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[213] 220374.890 2026-05-07T08:46:53.607Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[214] 220374.933 2026-05-07T08:46:53.650Z 0x948000000 67878723965 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[215] 220374.933 2026-05-07T08:46:53.650Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[216] 220374.976 2026-05-07T08:46:53.693Z 0xb18000000 67768623457 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[217] 220374.976 2026-05-07T08:46:53.693Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[218] 220375.018 2026-05-07T08:46:53.735Z 0x8b0000000 67878723941 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[219] 220375.018 2026-05-07T08:46:53.735Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[220] 220375.061 2026-05-07T08:46:53.778Z 0xa68000000 67852509541 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[221] 220375.061 2026-05-07T08:46:53.778Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[222] 220375.104 2026-05-07T08:46:53.821Z 0x6c8000000 67610288485 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[223] 220375.104 2026-05-07T08:46:53.821Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[224] 220375.146 2026-05-07T08:46:53.863Z 0x8d8000000 67610288485 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[225] 220375.146 2026-05-07T08:46:53.863Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[226] 220375.189 2026-05-07T08:46:53.906Z 0x7d0000000 67365970297 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[227] 220375.189 2026-05-07T08:46:53.906Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[228] 220375.231 2026-05-07T08:46:53.948Z 0x510000000 67610288485 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[229] 220375.231 2026-05-07T08:46:53.948Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[230] 220375.275 2026-05-07T08:46:53.992Z 0x5d0000000 67584074085 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[231] 220375.275 2026-05-07T08:46:53.992Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[232] 220375.317 2026-05-07T08:46:54.034Z 0x508000000 67341853029 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[233] 220375.317 2026-05-07T08:46:54.034Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[234] 220375.360 2026-05-07T08:46:54.077Z 0x818000000 67207635301 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[235] 220375.360 2026-05-07T08:46:54.077Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[236] 220375.402 2026-05-07T08:46:54.119Z 0xa00000000 67207635325 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[237] 220375.402 2026-05-07T08:46:54.119Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[238] 220375.444 2026-05-07T08:46:54.161Z 0x750000000 67097534817 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[239] 220375.444 2026-05-07T08:46:54.161Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[240] 220375.488 2026-05-07T08:46:54.205Z 0x570000000 67207635301 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[241] 220375.488 2026-05-07T08:46:54.205Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[242] 220375.530 2026-05-07T08:46:54.247Z 0x838000000 67181420901 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[243] 220375.530 2026-05-07T08:46:54.247Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[244] 220375.573 2026-05-07T08:46:54.290Z 0xa48000000 66939199845 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[245] 220375.573 2026-05-07T08:46:54.290Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[246] 220375.616 2026-05-07T08:46:54.333Z 0x938000000 66939199845 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[247] 220375.616 2026-05-07T08:46:54.333Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[248] 220375.658 2026-05-07T08:46:54.375Z 0x940000000 66694881657 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[249] 220375.658 2026-05-07T08:46:54.375Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[250] 220375.700 2026-05-07T08:46:54.417Z 0x6d8000000 66939199845 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[251] 220375.700 2026-05-07T08:46:54.417Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[252] 220375.744 2026-05-07T08:46:54.461Z 0x9b0000000 66912985445 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[253] 220375.744 2026-05-07T08:46:54.461Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[254] 220375.786 2026-05-07T08:46:54.503Z 0xa40000000 66670764389 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[255] 220375.786 2026-05-07T08:46:54.503Z BURST 1 pointers, free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[256] 220375.829 2026-05-07T08:46:54.546Z 0x5a0000000 66670764389 free:0 | 20260507 08:47:06.189 | NOTICE | packetHistory[257] 220375.829 2026-05-07T08:46:54.546Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[258] 220375.871 2026-05-07T08:46:54.588Z 0xb58000000 66536546685 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[259] 220375.871 2026-05-07T08:46:54.588Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[260] 220375.915 2026-05-07T08:46:54.632Z 0x5d8000000 66426446177 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[261] 220375.915 2026-05-07T08:46:54.632Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[262] 220375.957 2026-05-07T08:46:54.674Z 0x978000000 66536546661 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[263] 220375.957 2026-05-07T08:46:54.674Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[264] 220375.999 2026-05-07T08:46:54.716Z 0x958000000 66402328933 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[265] 220375.999 2026-05-07T08:46:54.716Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[266] 220376.042 2026-05-07T08:46:54.759Z 0x9a8000000 66268111205 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[267] 220376.042 2026-05-07T08:46:54.759Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[268] 220376.085 2026-05-07T08:46:54.802Z 0x6b0000000 66268111205 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[269] 220376.085 2026-05-07T08:46:54.802Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[270] 220376.128 2026-05-07T08:46:54.845Z 0x8c8000000 66023793017 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[271] 220376.128 2026-05-07T08:46:54.845Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[272] 220376.170 2026-05-07T08:46:54.887Z 0x9f8000000 66268111205 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[273] 220376.170 2026-05-07T08:46:54.887Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[274] 220376.213 2026-05-07T08:46:54.930Z 0x628000000 66133893477 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[275] 220376.213 2026-05-07T08:46:54.930Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[276] 220376.256 2026-05-07T08:46:54.973Z 0xac0000000 65999675749 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[277] 220376.256 2026-05-07T08:46:54.973Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[278] 220376.298 2026-05-07T08:46:55.015Z 0xa80000000 65973461349 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[279] 220376.298 2026-05-07T08:46:55.015Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[280] 220376.341 2026-05-07T08:46:55.058Z 0xb28000000 65731240317 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[281] 220376.341 2026-05-07T08:46:55.058Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[282] 220376.383 2026-05-07T08:46:55.100Z 0x530000000 65755357537 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[283] 220376.383 2026-05-07T08:46:55.100Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[284] 220376.427 2026-05-07T08:46:55.144Z 0xb50000000 65731240293 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[285] 220376.427 2026-05-07T08:46:55.144Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[286] 220376.469 2026-05-07T08:46:55.186Z 0xad8000000 65731240293 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[287] 220376.469 2026-05-07T08:46:55.186Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[288] 220376.511 2026-05-07T08:46:55.228Z 0x740000000 65705025893 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[289] 220376.511 2026-05-07T08:46:55.228Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[290] 220376.554 2026-05-07T08:46:55.271Z 0x6a8000000 65462804837 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[291] 220376.554 2026-05-07T08:46:55.271Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[292] 220376.597 2026-05-07T08:46:55.314Z 0x730000000 65352704377 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[293] 220376.597 2026-05-07T08:46:55.314Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[294] 220376.639 2026-05-07T08:46:55.356Z 0x708000000 65462804837 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[295] 220376.639 2026-05-07T08:46:55.356Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[296] 220376.683 2026-05-07T08:46:55.400Z 0x648000000 65462804837 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[297] 220376.683 2026-05-07T08:46:55.400Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[298] 220376.725 2026-05-07T08:46:55.442Z 0x990000000 65328587109 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[299] 220376.725 2026-05-07T08:46:55.442Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[300] 220376.768 2026-05-07T08:46:55.485Z 0x9e8000000 65302372709 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[301] 220376.768 2026-05-07T08:46:55.485Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[302] 220376.810 2026-05-07T08:46:55.527Z 0x8b8000000 65060151677 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[303] 220376.810 2026-05-07T08:46:55.527Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[304] 220376.852 2026-05-07T08:46:55.569Z 0x5e0000000 65084268897 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[305] 220376.852 2026-05-07T08:46:55.569Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[306] 220376.896 2026-05-07T08:46:55.613Z 0xa10000000 65328587109 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[307] 220376.896 2026-05-07T08:46:55.613Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[308] 220376.938 2026-05-07T08:46:55.655Z 0x590000000 65302372709 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[309] 220376.938 2026-05-07T08:46:55.655Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[310] 220376.981 2026-05-07T08:46:55.698Z 0x790000000 65302372709 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[311] 220376.981 2026-05-07T08:46:55.698Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[312] 220377.023 2026-05-07T08:46:55.740Z 0x660000000 65194369381 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[313] 220377.023 2026-05-07T08:46:55.740Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[314] 220377.067 2026-05-07T08:46:55.784Z 0xb40000000 65194369381 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[315] 220377.067 2026-05-07T08:46:55.784Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[316] 220377.109 2026-05-07T08:46:55.826Z 0x7a0000000 65060151653 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[317] 220377.109 2026-05-07T08:46:55.826Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[318] 220377.151 2026-05-07T08:46:55.868Z 0x5c8000000 64950051193 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[319] 220377.151 2026-05-07T08:46:55.868Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[320] 220377.194 2026-05-07T08:46:55.911Z 0x518000000 65192272225 free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[321] 220377.194 2026-05-07T08:46:55.911Z BURST 1 pointers, free:0 | 20260507 08:47:06.190 | NOTICE | packetHistory[322] 220377.237 2026-05-07T08:46:55.954Z 0x560000000 65194369381 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[323] 220377.237 2026-05-07T08:46:55.954Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[324] 220377.280 2026-05-07T08:46:55.997Z 0x8a8000000 65194369381 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[325] 220377.280 2026-05-07T08:46:55.997Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[326] 220377.322 2026-05-07T08:46:56.039Z 0xaa0000000 65194369381 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[327] 220377.322 2026-05-07T08:46:56.039Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[328] 220377.365 2026-05-07T08:46:56.082Z 0x558000000 65168154981 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[329] 220377.365 2026-05-07T08:46:56.082Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[330] 220377.408 2026-05-07T08:46:56.125Z 0x808000000 65060151653 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[331] 220377.408 2026-05-07T08:46:56.125Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[332] 220377.450 2026-05-07T08:46:56.167Z 0x6f8000000 65060151677 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[333] 220377.450 2026-05-07T08:46:56.167Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[334] 220377.493 2026-05-07T08:46:56.210Z 0x9c0000000 64950051169 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[335] 220377.493 2026-05-07T08:46:56.210Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[336] 220377.535 2026-05-07T08:46:56.252Z 0xa50000000 65168154981 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[337] 220377.535 2026-05-07T08:46:56.252Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[338] 220377.578 2026-05-07T08:46:56.295Z 0xa90000000 65170252137 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[339] 220377.578 2026-05-07T08:46:56.295Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[340] 220377.621 2026-05-07T08:46:56.338Z 0x680000000 65060151653 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[341] 220377.621 2026-05-07T08:46:56.338Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[342] 220377.663 2026-05-07T08:46:56.380Z 0x7d8000000 65060151653 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[343] 220377.663 2026-05-07T08:46:56.380Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[344] 220377.707 2026-05-07T08:46:56.424Z 0x930000000 65033937253 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[345] 220377.707 2026-05-07T08:46:56.424Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[346] 220377.749 2026-05-07T08:46:56.466Z 0xa28000000 64925933925 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[347] 220377.749 2026-05-07T08:46:56.466Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[348] 220377.791 2026-05-07T08:46:56.508Z 0xa78000000 64815833441 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[349] 220377.791 2026-05-07T08:46:56.508Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[350] 220377.834 2026-05-07T08:46:56.551Z 0x6e0000000 65060151653 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[351] 220377.834 2026-05-07T08:46:56.551Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[352] 220377.877 2026-05-07T08:46:56.594Z 0x550000000 64815833441 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[353] 220377.877 2026-05-07T08:46:56.594Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[354] 220377.919 2026-05-07T08:46:56.636Z 0xae0000000 64950051169 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[355] 220377.919 2026-05-07T08:46:56.636Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[356] 220377.963 2026-05-07T08:46:56.680Z 0x780000000 65084268897 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[357] 220377.963 2026-05-07T08:46:56.680Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[358] 220378.005 2026-05-07T08:46:56.722Z 0x6f0000000 65218486625 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[359] 220378.005 2026-05-07T08:46:56.722Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[360] 220378.047 2026-05-07T08:46:56.764Z 0x640000000 65352704353 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[361] 220378.047 2026-05-07T08:46:56.764Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[362] 220378.091 2026-05-07T08:46:56.808Z 0x5b8000000 65486922081 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[363] 220378.091 2026-05-07T08:46:56.808Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[364] 220378.133 2026-05-07T08:46:56.850Z 0x710000000 65621139809 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[365] 220378.133 2026-05-07T08:46:56.850Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[366] 220378.175 2026-05-07T08:46:56.892Z 0x5a8000000 65755357537 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[367] 220378.175 2026-05-07T08:46:56.892Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[368] 220378.218 2026-05-07T08:46:56.935Z 0x5e8000000 65889575265 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[369] 220378.218 2026-05-07T08:46:56.935Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[370] 220378.261 2026-05-07T08:46:56.978Z 0x658000000 66023792993 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[371] 220378.261 2026-05-07T08:46:56.978Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[372] 220378.304 2026-05-07T08:46:57.021Z 0xaf0000000 66158010721 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[373] 220378.304 2026-05-07T08:46:57.021Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[374] 220378.346 2026-05-07T08:46:57.063Z 0x760000000 66292228449 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[375] 220378.346 2026-05-07T08:46:57.063Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[376] 220378.389 2026-05-07T08:46:57.106Z 0x6a0000000 66426446177 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[377] 220378.389 2026-05-07T08:46:57.106Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[378] 220378.431 2026-05-07T08:46:57.148Z 0xb60000000 66560663905 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[379] 220378.431 2026-05-07T08:46:57.148Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[380] 220378.474 2026-05-07T08:46:57.191Z 0x578000000 66694881633 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[381] 220378.474 2026-05-07T08:46:57.191Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[382] 220378.517 2026-05-07T08:46:57.234Z 0xb20000000 66829099361 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[383] 220378.517 2026-05-07T08:46:57.234Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[384] 220378.559 2026-05-07T08:46:57.276Z 0x7b0000000 66963317089 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[385] 220378.559 2026-05-07T08:46:57.276Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[386] 220378.603 2026-05-07T08:46:57.320Z 0x528000000 67097534817 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[387] 220378.603 2026-05-07T08:46:57.320Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[388] 220378.644 2026-05-07T08:46:57.361Z 0x980000000 67231752545 free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[389] 220378.644 2026-05-07T08:46:57.361Z BURST 1 pointers, free:0 | 20260507 08:47:06.191 | NOTICE | packetHistory[390] 220378.688 2026-05-07T08:46:57.405Z 0x968000000 67365970273 free:0 | 20260507 08:47:06.192 | NOTICE | packetHistory[391] 220378.688 2026-05-07T08:46:57.405Z BURST 1 pointers, free:0 | 20260507 08:47:06.192 | NOTICE | packetHistory[392] 220378.730 2026-05-07T08:46:57.447Z 0x568000000 67500188001 free:0 | 20260507 08:47:06.192 | NOTICE | packetHistory[393] 220378.730 2026-05-07T08:46:57.447Z BURST 1 pointers, free:0 | 20260507 08:47:06.192 | NOTICE | packetHistory[394] 220378.773 2026-05-07T08:46:57.490Z 0x820000000 67634405729 free:0 | 20260507 08:47:06.192 | NOTICE | packetHistory[395] 220378.773 2026-05-07T08:46:57.490Z BURST 1 pointers, free:0 | 20260507 08:47:06.192 | NOTICE | packetHistory[396] 220378.815 2026-05-07T08:46:57.532Z 0x848000000 67768623457 free:0 | 20260507 08:47:06.192 | NOTICE | packetHistory[397] 220378.815 2026-05-07T08:46:57.532Z BURST 1 pointers, free:0 | 20260507 08:47:06.192 | NOTICE | packetHistory[398] 220378.858 2026-05-07T08:46:57.575Z 0x770000000 67902841185 free:0 | 20260507 08:47:06.192 | NOTICE | packetHistory[399] 220378.858 2026-05-07T08:46:57.575Z BURST 1 pointers, free:0 | 20260507 08:47:06.192 | NOTICE | std::size(inFlight):96 freequeue:0 | 20260507 08:47:06.192 | NOTICE | Packet History end | 20260507 08:47:06.192 | INFO | ----Memory usage report for Shared Pinned HugePage Host Memory----- | 20260507 08:47:06.192 | INFO | Total mem usage: 25.6 GiB | 20260507 08:47:06.192 | INFO | All Time High Water Marks sum: 25.6 GiB (Savings: 0%) | 20260507 08:47:06.192 | INFO | Recent High Water Marks Sum: 24.1 GiB (Savings: -6.22%) | 20260507 08:47:06.192 | INFO | Unused Cache: 1.5 GiB (Waste: 5.85%) | 20260507 08:47:06.192 | INFO | High Water Breakdown: | 20260507 08:47:06.192 | INFO | 100% : 24.1 GiB ( 94.1% Of Peak) : WXIPCDataSourceConfig | 20260507 08:47:06.192 | INFO | ------------------------------------------ | 20260507 08:47:06.192 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:47:06.319 | INFO | CpuSchedulerCanary: done | 20260507 08:47:09.636 | INFO | ----Memory usage report for Gpu Memory----- | 20260507 08:47:09.636 | INFO | Total mem usage: 15.7 GiB | 20260507 08:47:09.636 | INFO | All Time High Water Marks sum: 17.6 GiB (Savings: 11.1%) | 20260507 08:47:09.636 | INFO | Recent High Water Marks Sum: 16.5 GiB (Savings: 4.8%) | 20260507 08:47:09.636 | INFO | Unused Cache: 854 MiB (Waste: 5.32%) | 20260507 08:47:09.636 | INFO | High Water Breakdown: | 20260507 08:47:09.636 | INFO | 32.7% : 5.39 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:866 | 20260507 08:47:09.636 | INFO | 13.4% : 2.2 GiB ( 100% Of Peak) : /BaselineStatsAggregatorDevice.cu:131 | 20260507 08:47:09.636 | INFO | 8.54% : 1.41 GiB ( 100% Of Peak) : /DmeEmDevice.cu:1903 | 20260507 08:47:09.636 | INFO | 8.42% : 1.39 GiB ( 100% Of Peak) : /LabelsBatch.h:99 | 20260507 08:47:09.636 | INFO | 6.26% : 1.03 GiB ( 100% Of Peak) : /FrameLabelerDeviceDataStructures.cuh:182 | 20260507 08:47:09.636 | INFO | 6.12% : 1.01 GiB ( 100% Of Peak) : /HFMetricsFilterDevice.cu:907 | 20260507 08:47:09.636 | INFO | 4.84% : 816 MiB ( 99% Of Peak) : /DmeEmDevice.cu:1973 | 20260507 08:47:09.636 | INFO | 3.42% : 576 MiB ( 100% Of Peak) : /DevicePulseAccumulator.cu:316 | 20260507 08:47:09.636 | INFO | 3.07% : 517 MiB ( 100% Of Peak) : /PulseBatch.h:67 | 20260507 08:47:09.636 | INFO | 2.72% : 459 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:47:09.636 | INFO | 2.28% : 384 MiB ( 100% Of Peak) : /TrivialRepacker.h:114 | 20260507 08:47:09.636 | INFO | 1.76% : 297 MiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:47:09.636 | INFO | 1.66% : 280 MiB ( 100% Of Peak) : /CameraTraceBatch.h:31 | 20260507 08:47:09.636 | INFO | 1.58% : 267 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:122 | 20260507 08:47:09.636 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:739 | 20260507 08:47:09.636 | INFO | 0.759% : 128 MiB ( 100% Of Peak) : /BaselineFilterKernel.cuh:737 | 20260507 08:47:09.636 | INFO | 0.623% : 105 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:47:09.636 | INFO | 0.569% : 96 MiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:499 | 20260507 08:47:09.636 | INFO | 0.216% : 36.5 MiB ( 50% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:47:09.636 | INFO | 0.178% : 30 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:47:09.636 | INFO | 0.0712% : 12 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:47:09.636 | INFO | 0.0237% : 4 MiB ( 100% Of Peak) : /PulseBatch.h:68 | 20260507 08:47:09.636 | INFO | 0.00593% : 1 MiB ( 100% Of Peak) : /LabelsBatch.h:106 | 20260507 08:47:09.636 | INFO | 1.36e-07% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:47:09.636 | INFO | 0% : 0B ( 0% Of Peak) : /FrameLabelerDevice.cu:93 | 20260507 08:47:09.636 | INFO | ------------------------------------------ | 20260507 08:47:09.636 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:47:17.307 | INFO | ----Memory usage report for Pinned Host Memory----- | 20260507 08:47:17.307 | INFO | Total mem usage: 26.5 GiB | 20260507 08:47:17.307 | INFO | All Time High Water Marks sum: 27.2 GiB (Savings: 2.67%) | 20260507 08:47:17.307 | INFO | Recent High Water Marks Sum: 25.5 GiB (Savings: -4%) | 20260507 08:47:17.307 | INFO | Unused Cache: 1.74 GiB (Waste: 6.57%) | 20260507 08:47:17.307 | INFO | High Water Breakdown: | 20260507 08:47:17.307 | INFO | 56.3% : 14.3 GiB ( 100% Of Peak) : /TraceHistogramAccumDevice.cu:498 | 20260507 08:47:17.307 | INFO | 36.4% : 9.28 GiB ( 100% Of Peak) : /DeviceMultiScaleBaseliner.cu:62 | 20260507 08:47:17.307 | INFO | 3.68% : 960 MiB ( 100% Of Peak) : /FrameLabelerDevice.cu:92 | 20260507 08:47:17.307 | INFO | 1.61% : 420 MiB ( 80% Of Peak) : /PulseSerializerDevice.cu:107 | 20260507 08:47:17.307 | INFO | 1.47% : 384 MiB ( 100% Of Peak) : /PulseSerializerDevice.cu:88 | 20260507 08:47:17.307 | INFO | 0.559% : 146 MiB ( 66.7% Of Peak) : /BasecallingMetrics.h:66 | 20260507 08:47:17.307 | INFO | 0.00153% : 408 KiB ( 100% Of Peak) : /BatchAnalyzer.cpp:46 | 20260507 08:47:17.307 | INFO | 9.39e-07% : 257B ( 100% Of Peak) : /LaneErrors.cpp:33 | 20260507 08:47:17.307 | INFO | 8.77e-08% : 24B ( 100% Of Peak) : /UnifiedCudaArrayQueue.h:133 | 20260507 08:47:17.307 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:45 | 20260507 08:47:17.307 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:67 | 20260507 08:47:17.307 | INFO | 0% : 0B ( 0% Of Peak) : /PulseBatch.h:68 | 20260507 08:47:17.307 | INFO | 0% : 0B ( 0% Of Peak) : /BatchAnalyzer.cpp:315 | 20260507 08:47:17.307 | INFO | 0% : 0B ( 0% Of Peak) : /LabelsBatch.h:99 | 20260507 08:47:17.307 | INFO | ------------------------------------------ | 20260507 08:47:17.307 | INFO | Note: Different filter stages consume memory at different times. The sum of high water marks for individual high watermarks is not expected to be quite the same as the high watermark for the application as a whole | 20260507 08:47:17.307 | NOTICE | Because ALLOCATION_HISTORY is defined, dumping CUDA allocation history for IA-849 | 20260507 08:47:17.307 | NOTICE | ----- PBCudaRuntime DumpAllocationHistoryToLog ---- (begin) | 20260507 08:47:17.307 | NOTICE | [0]: 1970-01-02T13:06:43.862Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.307 | NOTICE | [1]: 1970-01-02T13:06:43.869Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.307 | NOTICE | [2]: 1970-01-02T13:06:43.876Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.307 | NOTICE | [3]: 1970-01-02T13:06:43.883Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.307 | NOTICE | [4]: 1970-01-02T13:06:43.887Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.307 | NOTICE | [5]: 1970-01-02T13:06:43.892Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.307 | NOTICE | [6]: 1970-01-02T13:06:43.894Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.307 | NOTICE | [7]: 1970-01-02T13:06:43.897Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.307 | NOTICE | [8]: 1970-01-02T13:06:43.897Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.307 | NOTICE | [9]: 1970-01-02T13:06:43.898Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.308 | NOTICE | [10]: 1970-01-02T13:06:43.898Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.308 | NOTICE | [11]: 1970-01-02T13:06:43.898Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.308 | NOTICE | [12]: 1970-01-02T13:06:44.001Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.308 | NOTICE | [13]: 1970-01-02T13:06:44.023Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.308 | NOTICE | [14]: 1970-01-02T13:06:44.045Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.308 | NOTICE | [15]: 1970-01-02T13:06:44.052Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.308 | NOTICE | [16]: 1970-01-02T13:06:44.058Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.308 | NOTICE | [17]: 1970-01-02T13:06:44.065Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.308 | NOTICE | [18]: 1970-01-02T13:06:44.069Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.308 | NOTICE | [19]: 1970-01-02T13:06:44.074Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.308 | NOTICE | [20]: 1970-01-02T13:06:44.077Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.308 | NOTICE | [21]: 1970-01-02T13:06:44.080Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.308 | NOTICE | [22]: 1970-01-02T13:06:44.080Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.308 | NOTICE | [23]: 1970-01-02T13:06:44.080Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.308 | NOTICE | [24]: 1970-01-02T13:06:44.080Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.308 | NOTICE | [25]: 1970-01-02T13:06:44.080Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.308 | NOTICE | [26]: 1970-01-02T13:06:44.184Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.308 | NOTICE | [27]: 1970-01-02T13:06:44.206Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.308 | NOTICE | [28]: 1970-01-02T13:06:44.227Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.308 | NOTICE | [29]: 1970-01-02T13:06:44.234Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.308 | NOTICE | [30]: 1970-01-02T13:06:44.241Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.308 | NOTICE | [31]: 1970-01-02T13:06:44.248Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.308 | NOTICE | [32]: 1970-01-02T13:06:44.252Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.308 | NOTICE | [33]: 1970-01-02T13:06:44.257Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.308 | NOTICE | [34]: 1970-01-02T13:06:44.260Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.308 | NOTICE | [35]: 1970-01-02T13:06:44.262Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.308 | NOTICE | [36]: 1970-01-02T13:06:44.263Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.308 | NOTICE | [37]: 1970-01-02T13:06:44.263Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.308 | NOTICE | [38]: 1970-01-02T13:06:44.263Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.308 | NOTICE | [39]: 1970-01-02T13:06:44.263Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.308 | NOTICE | [40]: 1970-01-02T13:06:44.366Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.308 | NOTICE | [41]: 1970-01-02T13:06:44.388Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.308 | NOTICE | [42]: 1970-01-02T13:06:44.410Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.308 | NOTICE | [43]: 1970-01-02T13:06:44.417Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.308 | NOTICE | [44]: 1970-01-02T13:06:44.424Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.308 | NOTICE | [45]: 1970-01-02T13:06:44.430Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.308 | NOTICE | [46]: 1970-01-02T13:06:44.435Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.308 | NOTICE | [47]: 1970-01-02T13:06:44.439Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.308 | NOTICE | [48]: 1970-01-02T13:06:44.442Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.308 | NOTICE | [49]: 1970-01-02T13:06:44.445Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.308 | NOTICE | [50]: 1970-01-02T13:06:44.445Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.308 | NOTICE | [51]: 1970-01-02T13:06:44.445Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.308 | NOTICE | [52]: 1970-01-02T13:06:44.446Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.308 | NOTICE | [53]: 1970-01-02T13:06:44.446Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.308 | NOTICE | [54]: 1970-01-02T13:06:44.549Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.308 | NOTICE | [55]: 1970-01-02T13:06:44.571Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.308 | NOTICE | [56]: 1970-01-02T13:06:44.593Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.308 | NOTICE | [57]: 1970-01-02T13:06:44.600Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.308 | NOTICE | [58]: 1970-01-02T13:06:44.607Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.308 | NOTICE | [59]: 1970-01-02T13:06:44.613Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.308 | NOTICE | [60]: 1970-01-02T13:06:44.618Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.308 | NOTICE | [61]: 1970-01-02T13:06:44.622Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.308 | NOTICE | [62]: 1970-01-02T13:06:44.625Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.308 | NOTICE | [63]: 1970-01-02T13:06:44.628Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.308 | NOTICE | [64]: 1970-01-02T13:06:44.628Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.308 | NOTICE | [65]: 1970-01-02T13:06:44.628Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.308 | NOTICE | [66]: 1970-01-02T13:06:44.628Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.308 | NOTICE | [67]: 1970-01-02T13:06:44.628Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.308 | NOTICE | [68]: 1970-01-02T13:06:44.732Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.308 | NOTICE | [69]: 1970-01-02T13:06:44.754Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.308 | NOTICE | [70]: 1970-01-02T13:06:44.776Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.308 | NOTICE | [71]: 1970-01-02T13:06:44.782Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.309 | NOTICE | [72]: 1970-01-02T13:06:44.789Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.309 | NOTICE | [73]: 1970-01-02T13:06:44.796Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.309 | NOTICE | [74]: 1970-01-02T13:06:44.800Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.309 | NOTICE | [75]: 1970-01-02T13:06:44.805Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.309 | NOTICE | [76]: 1970-01-02T13:06:44.808Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.309 | NOTICE | [77]: 1970-01-02T13:06:44.810Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.309 | NOTICE | [78]: 1970-01-02T13:06:44.811Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.309 | NOTICE | [79]: 1970-01-02T13:06:44.811Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.309 | NOTICE | [80]: 1970-01-02T13:06:44.811Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.309 | NOTICE | [81]: 1970-01-02T13:06:44.811Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.309 | NOTICE | [82]: 1970-01-02T13:06:44.914Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.309 | NOTICE | [83]: 1970-01-02T13:06:44.936Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.309 | NOTICE | [84]: 1970-01-02T13:06:44.958Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.309 | NOTICE | [85]: 1970-01-02T13:06:44.965Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.309 | NOTICE | [86]: 1970-01-02T13:06:44.972Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.309 | NOTICE | [87]: 1970-01-02T13:06:44.978Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.309 | NOTICE | [88]: 1970-01-02T13:06:44.983Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.309 | NOTICE | [89]: 1970-01-02T13:06:44.987Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.309 | NOTICE | [90]: 1970-01-02T13:06:44.990Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.309 | NOTICE | [91]: 1970-01-02T13:06:44.993Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.309 | NOTICE | [92]: 1970-01-02T13:06:44.993Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.309 | NOTICE | [93]: 1970-01-02T13:06:44.993Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.309 | NOTICE | [94]: 1970-01-02T13:06:44.993Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.309 | NOTICE | [95]: 1970-01-02T13:06:44.993Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.309 | NOTICE | [96]: 1970-01-02T13:06:45.097Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.309 | NOTICE | [97]: 1970-01-02T13:06:45.119Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.309 | NOTICE | [98]: 1970-01-02T13:06:45.143Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.309 | NOTICE | [99]: 1970-01-02T13:06:45.150Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.309 | NOTICE | [100]: 1970-01-02T13:06:45.157Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.309 | NOTICE | [101]: 1970-01-02T13:06:45.165Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.309 | NOTICE | [102]: 1970-01-02T13:06:45.170Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.309 | NOTICE | [103]: 1970-01-02T13:06:45.174Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.309 | NOTICE | [104]: 1970-01-02T13:06:45.177Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.309 | NOTICE | [105]: 1970-01-02T13:06:45.180Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.309 | NOTICE | [106]: 1970-01-02T13:06:45.180Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.309 | NOTICE | [107]: 1970-01-02T13:06:45.180Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.309 | NOTICE | [108]: 1970-01-02T13:06:45.180Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.309 | NOTICE | [109]: 1970-01-02T13:06:45.180Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.309 | NOTICE | [110]: 1970-01-02T13:06:45.282Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.309 | NOTICE | [111]: 1970-01-02T13:06:45.304Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.309 | NOTICE | [112]: 1970-01-02T13:06:45.326Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.309 | NOTICE | [113]: 1970-01-02T13:06:45.333Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.309 | NOTICE | [114]: 1970-01-02T13:06:45.340Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.309 | NOTICE | [115]: 1970-01-02T13:06:45.346Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.309 | NOTICE | [116]: 1970-01-02T13:06:45.351Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.309 | NOTICE | [117]: 1970-01-02T13:06:45.355Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.309 | NOTICE | [118]: 1970-01-02T13:06:45.358Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.309 | NOTICE | [119]: 1970-01-02T13:06:45.361Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.309 | NOTICE | [120]: 1970-01-02T13:06:45.361Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.309 | NOTICE | [121]: 1970-01-02T13:06:45.361Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.309 | NOTICE | [122]: 1970-01-02T13:06:45.361Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.309 | NOTICE | [123]: 1970-01-02T13:06:45.361Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.309 | NOTICE | [124]: 1970-01-02T13:06:45.464Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.309 | NOTICE | [125]: 1970-01-02T13:06:45.486Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.309 | NOTICE | [126]: 1970-01-02T13:06:45.508Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.309 | NOTICE | [127]: 1970-01-02T13:06:45.515Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.309 | NOTICE | [128]: 1970-01-02T13:06:45.522Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.309 | NOTICE | [129]: 1970-01-02T13:06:45.528Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.309 | NOTICE | [130]: 1970-01-02T13:06:45.533Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.309 | NOTICE | [131]: 1970-01-02T13:06:45.537Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.309 | NOTICE | [132]: 1970-01-02T13:06:45.540Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.309 | NOTICE | [133]: 1970-01-02T13:06:45.543Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.309 | NOTICE | [134]: 1970-01-02T13:06:45.543Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.309 | NOTICE | [135]: 1970-01-02T13:06:45.543Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.309 | NOTICE | [136]: 1970-01-02T13:06:45.543Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.309 | NOTICE | [137]: 1970-01-02T13:06:45.543Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.309 | NOTICE | [138]: 1970-01-02T13:06:45.648Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.309 | NOTICE | [139]: 1970-01-02T13:06:45.670Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.309 | NOTICE | [140]: 1970-01-02T13:06:45.692Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.309 | NOTICE | [141]: 1970-01-02T13:06:45.698Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.310 | NOTICE | [142]: 1970-01-02T13:06:45.705Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.310 | NOTICE | [143]: 1970-01-02T13:06:45.712Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.310 | NOTICE | [144]: 1970-01-02T13:06:45.716Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.310 | NOTICE | [145]: 1970-01-02T13:06:45.721Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.310 | NOTICE | [146]: 1970-01-02T13:06:45.724Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.310 | NOTICE | [147]: 1970-01-02T13:06:45.726Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.310 | NOTICE | [148]: 1970-01-02T13:06:45.727Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.310 | NOTICE | [149]: 1970-01-02T13:06:45.727Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.310 | NOTICE | [150]: 1970-01-02T13:06:45.727Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.310 | NOTICE | [151]: 1970-01-02T13:06:45.727Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.310 | NOTICE | [152]: 1970-01-02T13:06:45.831Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.310 | NOTICE | [153]: 1970-01-02T13:06:45.853Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.310 | NOTICE | [154]: 1970-01-02T13:06:45.875Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.310 | NOTICE | [155]: 1970-01-02T13:06:45.882Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.310 | NOTICE | [156]: 1970-01-02T13:06:45.889Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.310 | NOTICE | [157]: 1970-01-02T13:06:45.895Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.310 | NOTICE | [158]: 1970-01-02T13:06:45.900Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.310 | NOTICE | [159]: 1970-01-02T13:06:45.904Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.310 | NOTICE | [160]: 1970-01-02T13:06:45.907Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.310 | NOTICE | [161]: 1970-01-02T13:06:45.910Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.310 | NOTICE | [162]: 1970-01-02T13:06:45.910Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.310 | NOTICE | [163]: 1970-01-02T13:06:45.910Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.310 | NOTICE | [164]: 1970-01-02T13:06:45.910Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.310 | NOTICE | [165]: 1970-01-02T13:06:45.910Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.310 | NOTICE | [166]: 1970-01-02T13:06:46.014Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.310 | NOTICE | [167]: 1970-01-02T13:06:46.036Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.310 | NOTICE | [168]: 1970-01-02T13:06:46.058Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.310 | NOTICE | [169]: 1970-01-02T13:06:46.065Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.310 | NOTICE | [170]: 1970-01-02T13:06:46.072Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.310 | NOTICE | [171]: 1970-01-02T13:06:46.078Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.310 | NOTICE | [172]: 1970-01-02T13:06:46.083Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.310 | NOTICE | [173]: 1970-01-02T13:06:46.087Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.310 | NOTICE | [174]: 1970-01-02T13:06:46.090Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.310 | NOTICE | [175]: 1970-01-02T13:06:46.093Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.310 | NOTICE | [176]: 1970-01-02T13:06:46.093Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.310 | NOTICE | [177]: 1970-01-02T13:06:46.093Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.310 | NOTICE | [178]: 1970-01-02T13:06:46.093Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.310 | NOTICE | [179]: 1970-01-02T13:06:46.093Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.310 | NOTICE | [180]: 1970-01-02T13:06:46.197Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.310 | NOTICE | [181]: 1970-01-02T13:06:46.219Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.310 | NOTICE | [182]: 1970-01-02T13:06:46.241Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.310 | NOTICE | [183]: 1970-01-02T13:06:46.248Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.310 | NOTICE | [184]: 1970-01-02T13:06:46.254Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.310 | NOTICE | [185]: 1970-01-02T13:06:46.261Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.310 | NOTICE | [186]: 1970-01-02T13:06:46.265Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.310 | NOTICE | [187]: 1970-01-02T13:06:46.270Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.310 | NOTICE | [188]: 1970-01-02T13:06:46.273Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.310 | NOTICE | [189]: 1970-01-02T13:06:46.276Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.310 | NOTICE | [190]: 1970-01-02T13:06:46.276Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.310 | NOTICE | [191]: 1970-01-02T13:06:46.276Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.310 | NOTICE | [192]: 1970-01-02T13:06:46.276Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.310 | NOTICE | [193]: 1970-01-02T13:06:46.276Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.310 | NOTICE | [194]: 1970-01-02T13:06:46.379Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.310 | NOTICE | [195]: 1970-01-02T13:06:46.401Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.310 | NOTICE | [196]: 1970-01-02T13:06:46.423Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.310 | NOTICE | [197]: 1970-01-02T13:06:46.430Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.310 | NOTICE | [198]: 1970-01-02T13:06:46.436Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.310 | NOTICE | [199]: 1970-01-02T13:06:46.443Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.310 | NOTICE | [200]: 1970-01-02T13:06:46.447Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.310 | NOTICE | [201]: 1970-01-02T13:06:46.452Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.310 | NOTICE | [202]: 1970-01-02T13:06:46.455Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.310 | NOTICE | [203]: 1970-01-02T13:06:46.457Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.310 | NOTICE | [204]: 1970-01-02T13:06:46.458Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.310 | NOTICE | [205]: 1970-01-02T13:06:46.458Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.310 | NOTICE | [206]: 1970-01-02T13:06:46.458Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.310 | NOTICE | [207]: 1970-01-02T13:06:46.458Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.310 | NOTICE | [208]: 1970-01-02T13:06:46.560Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.310 | NOTICE | [209]: 1970-01-02T13:06:46.581Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.310 | NOTICE | [210]: 1970-01-02T13:06:46.603Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.310 | NOTICE | [211]: 1970-01-02T13:06:46.610Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.310 | NOTICE | [212]: 1970-01-02T13:06:46.616Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.310 | NOTICE | [213]: 1970-01-02T13:06:46.623Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.310 | NOTICE | [214]: 1970-01-02T13:06:46.627Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.310 | NOTICE | [215]: 1970-01-02T13:06:46.632Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.311 | NOTICE | [216]: 1970-01-02T13:06:46.635Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.311 | NOTICE | [217]: 1970-01-02T13:06:46.637Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.311 | NOTICE | [218]: 1970-01-02T13:06:46.638Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.311 | NOTICE | [219]: 1970-01-02T13:06:46.638Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.311 | NOTICE | [220]: 1970-01-02T13:06:46.638Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.311 | NOTICE | [221]: 1970-01-02T13:06:46.638Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.311 | NOTICE | [222]: 1970-01-02T13:06:46.740Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.311 | NOTICE | [223]: 1970-01-02T13:06:46.761Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.311 | NOTICE | [224]: 1970-01-02T13:06:46.783Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.311 | NOTICE | [225]: 1970-01-02T13:06:46.790Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.311 | NOTICE | [226]: 1970-01-02T13:06:46.796Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.311 | NOTICE | [227]: 1970-01-02T13:06:46.803Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.311 | NOTICE | [228]: 1970-01-02T13:06:46.807Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.311 | NOTICE | [229]: 1970-01-02T13:06:46.812Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.311 | NOTICE | [230]: 1970-01-02T13:06:46.815Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.311 | NOTICE | [231]: 1970-01-02T13:06:46.817Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.311 | NOTICE | [232]: 1970-01-02T13:06:46.818Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.311 | NOTICE | [233]: 1970-01-02T13:06:46.818Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.311 | NOTICE | [234]: 1970-01-02T13:06:46.818Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.311 | NOTICE | [235]: 1970-01-02T13:06:46.818Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.311 | NOTICE | [236]: 1970-01-02T13:06:46.920Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.311 | NOTICE | [237]: 1970-01-02T13:06:46.941Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.311 | NOTICE | [238]: 1970-01-02T13:06:46.963Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.311 | NOTICE | [239]: 1970-01-02T13:06:46.970Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.311 | NOTICE | [240]: 1970-01-02T13:06:46.976Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.311 | NOTICE | [241]: 1970-01-02T13:06:46.983Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.311 | NOTICE | [242]: 1970-01-02T13:06:46.987Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.311 | NOTICE | [243]: 1970-01-02T13:06:46.992Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.311 | NOTICE | [244]: 1970-01-02T13:06:46.995Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.311 | NOTICE | [245]: 1970-01-02T13:06:46.997Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.311 | NOTICE | [246]: 1970-01-02T13:06:46.998Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.311 | NOTICE | [247]: 1970-01-02T13:06:46.998Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.311 | NOTICE | [248]: 1970-01-02T13:06:46.998Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.311 | NOTICE | [249]: 1970-01-02T13:06:46.998Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.311 | NOTICE | [250]: 1970-01-02T13:06:47.100Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.311 | NOTICE | [251]: 1970-01-02T13:06:47.121Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.311 | NOTICE | [252]: 1970-01-02T13:06:47.143Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.311 | NOTICE | [253]: 1970-01-02T13:06:47.150Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.311 | NOTICE | [254]: 1970-01-02T13:06:47.157Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.311 | NOTICE | [255]: 1970-01-02T13:06:47.163Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.311 | NOTICE | [256]: 1970-01-02T13:06:47.168Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.311 | NOTICE | [257]: 1970-01-02T13:06:47.172Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.311 | NOTICE | [258]: 1970-01-02T13:06:47.175Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.311 | NOTICE | [259]: 1970-01-02T13:06:47.178Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.311 | NOTICE | [260]: 1970-01-02T13:06:47.178Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.311 | NOTICE | [261]: 1970-01-02T13:06:47.178Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.311 | NOTICE | [262]: 1970-01-02T13:06:47.178Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.311 | NOTICE | [263]: 1970-01-02T13:06:47.178Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.311 | NOTICE | [264]: 1970-01-02T13:06:47.280Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.311 | NOTICE | [265]: 1970-01-02T13:06:47.302Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.311 | NOTICE | [266]: 1970-01-02T13:06:47.323Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.311 | NOTICE | [267]: 1970-01-02T13:06:47.330Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.311 | NOTICE | [268]: 1970-01-02T13:06:47.337Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.311 | NOTICE | [269]: 1970-01-02T13:06:47.343Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.311 | NOTICE | [270]: 1970-01-02T13:06:47.348Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.311 | NOTICE | [271]: 1970-01-02T13:06:47.352Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.311 | NOTICE | [272]: 1970-01-02T13:06:47.355Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.311 | NOTICE | [273]: 1970-01-02T13:06:47.358Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.311 | NOTICE | [274]: 1970-01-02T13:06:47.358Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.311 | NOTICE | [275]: 1970-01-02T13:06:47.358Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.311 | NOTICE | [276]: 1970-01-02T13:06:47.358Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.311 | NOTICE | [277]: 1970-01-02T13:06:47.358Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.311 | NOTICE | [278]: 1970-01-02T13:06:47.460Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.311 | NOTICE | [279]: 1970-01-02T13:06:47.482Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.311 | NOTICE | [280]: 1970-01-02T13:06:47.503Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.311 | NOTICE | [281]: 1970-01-02T13:06:47.510Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.311 | NOTICE | [282]: 1970-01-02T13:06:47.516Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.311 | NOTICE | [283]: 1970-01-02T13:06:47.523Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.311 | NOTICE | [284]: 1970-01-02T13:06:47.527Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.311 | NOTICE | [285]: 1970-01-02T13:06:47.532Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.311 | NOTICE | [286]: 1970-01-02T13:06:47.535Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.311 | NOTICE | [287]: 1970-01-02T13:06:47.537Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.312 | NOTICE | [288]: 1970-01-02T13:06:47.538Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.312 | NOTICE | [289]: 1970-01-02T13:06:47.538Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.312 | NOTICE | [290]: 1970-01-02T13:06:47.538Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.312 | NOTICE | [291]: 1970-01-02T13:06:47.538Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.312 | NOTICE | [292]: 1970-01-02T13:06:47.640Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.312 | NOTICE | [293]: 1970-01-02T13:06:47.662Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.312 | NOTICE | [294]: 1970-01-02T13:06:47.683Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.312 | NOTICE | [295]: 1970-01-02T13:06:47.690Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.312 | NOTICE | [296]: 1970-01-02T13:06:47.697Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.312 | NOTICE | [297]: 1970-01-02T13:06:47.703Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.312 | NOTICE | [298]: 1970-01-02T13:06:47.707Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.312 | NOTICE | [299]: 1970-01-02T13:06:47.712Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.312 | NOTICE | [300]: 1970-01-02T13:06:47.715Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.312 | NOTICE | [301]: 1970-01-02T13:06:47.717Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.312 | NOTICE | [302]: 1970-01-02T13:06:47.718Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.312 | NOTICE | [303]: 1970-01-02T13:06:47.718Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.312 | NOTICE | [304]: 1970-01-02T13:06:47.718Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.312 | NOTICE | [305]: 1970-01-02T13:06:47.718Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.312 | NOTICE | [306]: 1970-01-02T13:06:47.820Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.312 | NOTICE | [307]: 1970-01-02T13:06:47.842Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.312 | NOTICE | [308]: 1970-01-02T13:06:47.864Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.312 | NOTICE | [309]: 1970-01-02T13:06:47.871Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.312 | NOTICE | [310]: 1970-01-02T13:06:47.877Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.312 | NOTICE | [311]: 1970-01-02T13:06:47.884Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.312 | NOTICE | [312]: 1970-01-02T13:06:47.888Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.312 | NOTICE | [313]: 1970-01-02T13:06:47.893Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.312 | NOTICE | [314]: 1970-01-02T13:06:47.896Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.312 | NOTICE | [315]: 1970-01-02T13:06:47.899Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.312 | NOTICE | [316]: 1970-01-02T13:06:47.899Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.312 | NOTICE | [317]: 1970-01-02T13:06:47.899Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.312 | NOTICE | [318]: 1970-01-02T13:06:47.899Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.312 | NOTICE | [319]: 1970-01-02T13:06:47.899Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.312 | NOTICE | [320]: 1970-01-02T13:06:48.002Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.312 | NOTICE | [321]: 1970-01-02T13:06:48.024Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.312 | NOTICE | [322]: 1970-01-02T13:06:48.046Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.312 | NOTICE | [323]: 1970-01-02T13:06:48.053Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.312 | NOTICE | [324]: 1970-01-02T13:06:48.060Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.312 | NOTICE | [325]: 1970-01-02T13:06:48.066Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.312 | NOTICE | [326]: 1970-01-02T13:06:48.071Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.312 | NOTICE | [327]: 1970-01-02T13:06:48.075Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.312 | NOTICE | [328]: 1970-01-02T13:06:48.078Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.312 | NOTICE | [329]: 1970-01-02T13:06:48.081Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.312 | NOTICE | [330]: 1970-01-02T13:06:48.081Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.312 | NOTICE | [331]: 1970-01-02T13:06:48.081Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.312 | NOTICE | [332]: 1970-01-02T13:06:48.081Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.312 | NOTICE | [333]: 1970-01-02T13:06:48.081Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.312 | NOTICE | [334]: 1970-01-02T13:06:48.185Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.312 | NOTICE | [335]: 1970-01-02T13:06:48.207Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.312 | NOTICE | [336]: 1970-01-02T13:06:48.229Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.312 | NOTICE | [337]: 1970-01-02T13:06:48.236Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.312 | NOTICE | [338]: 1970-01-02T13:06:48.243Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.312 | NOTICE | [339]: 1970-01-02T13:06:48.249Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.312 | NOTICE | [340]: 1970-01-02T13:06:48.254Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.312 | NOTICE | [341]: 1970-01-02T13:06:48.258Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.312 | NOTICE | [342]: 1970-01-02T13:06:48.261Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.312 | NOTICE | [343]: 1970-01-02T13:06:48.264Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.312 | NOTICE | [344]: 1970-01-02T13:06:48.264Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.312 | NOTICE | [345]: 1970-01-02T13:06:48.264Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.312 | NOTICE | [346]: 1970-01-02T13:06:48.264Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.312 | NOTICE | [347]: 1970-01-02T13:06:48.264Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.312 | NOTICE | [348]: 1970-01-02T13:06:48.368Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.312 | NOTICE | [349]: 1970-01-02T13:06:48.390Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.312 | NOTICE | [350]: 1970-01-02T13:06:48.411Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.312 | NOTICE | [351]: 1970-01-02T13:06:48.418Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.312 | NOTICE | [352]: 1970-01-02T13:06:48.425Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.312 | NOTICE | [353]: 1970-01-02T13:06:48.431Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.312 | NOTICE | [354]: 1970-01-02T13:06:48.436Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.312 | NOTICE | [355]: 1970-01-02T13:06:48.441Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.312 | NOTICE | [356]: 1970-01-02T13:06:48.443Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.312 | NOTICE | [357]: 1970-01-02T13:06:48.446Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.312 | NOTICE | [358]: 1970-01-02T13:06:48.446Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.312 | NOTICE | [359]: 1970-01-02T13:06:48.446Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.312 | NOTICE | [360]: 1970-01-02T13:06:48.446Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.313 | NOTICE | [361]: 1970-01-02T13:06:48.446Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.313 | NOTICE | [362]: 1970-01-02T13:06:48.550Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.313 | NOTICE | [363]: 1970-01-02T13:06:48.572Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.313 | NOTICE | [364]: 1970-01-02T13:06:48.594Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.313 | NOTICE | [365]: 1970-01-02T13:06:48.601Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.313 | NOTICE | [366]: 1970-01-02T13:06:48.608Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.313 | NOTICE | [367]: 1970-01-02T13:06:48.615Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.313 | NOTICE | [368]: 1970-01-02T13:06:48.619Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.313 | NOTICE | [369]: 1970-01-02T13:06:48.623Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.313 | NOTICE | [370]: 1970-01-02T13:06:48.626Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.313 | NOTICE | [371]: 1970-01-02T13:06:48.629Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.313 | NOTICE | [372]: 1970-01-02T13:06:48.629Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.313 | NOTICE | [373]: 1970-01-02T13:06:48.629Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.313 | NOTICE | [374]: 1970-01-02T13:06:48.629Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.313 | NOTICE | [375]: 1970-01-02T13:06:48.629Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.313 | NOTICE | [376]: 1970-01-02T13:06:48.733Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.313 | NOTICE | [377]: 1970-01-02T13:06:48.755Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.313 | NOTICE | [378]: 1970-01-02T13:06:48.777Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.313 | NOTICE | [379]: 1970-01-02T13:06:48.784Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.313 | NOTICE | [380]: 1970-01-02T13:06:48.791Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.313 | NOTICE | [381]: 1970-01-02T13:06:48.798Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.313 | NOTICE | [382]: 1970-01-02T13:06:48.802Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.313 | NOTICE | [383]: 1970-01-02T13:06:48.807Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.313 | NOTICE | [384]: 1970-01-02T13:06:48.809Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.313 | NOTICE | [385]: 1970-01-02T13:06:48.812Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.313 | NOTICE | [386]: 1970-01-02T13:06:48.812Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.313 | NOTICE | [387]: 1970-01-02T13:06:48.812Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.313 | NOTICE | [388]: 1970-01-02T13:06:48.812Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.313 | NOTICE | [389]: 1970-01-02T13:06:48.812Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.313 | NOTICE | [390]: 1970-01-02T13:06:48.917Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.313 | NOTICE | [391]: 1970-01-02T13:06:48.939Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.313 | NOTICE | [392]: 1970-01-02T13:06:48.961Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.313 | NOTICE | [393]: 1970-01-02T13:06:48.968Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.313 | NOTICE | [394]: 1970-01-02T13:06:48.974Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.313 | NOTICE | [395]: 1970-01-02T13:06:48.981Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.313 | NOTICE | [396]: 1970-01-02T13:06:48.985Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.313 | NOTICE | [397]: 1970-01-02T13:06:48.990Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.313 | NOTICE | [398]: 1970-01-02T13:06:48.993Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.313 | NOTICE | [399]: 1970-01-02T13:06:48.996Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.313 | NOTICE | [400]: 1970-01-02T13:06:48.996Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.313 | NOTICE | [401]: 1970-01-02T13:06:48.996Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.313 | NOTICE | [402]: 1970-01-02T13:06:48.996Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.313 | NOTICE | [403]: 1970-01-02T13:06:48.996Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.313 | NOTICE | [404]: 1970-01-02T13:06:49.100Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.313 | NOTICE | [405]: 1970-01-02T13:06:49.122Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.313 | NOTICE | [406]: 1970-01-02T13:06:49.144Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.313 | NOTICE | [407]: 1970-01-02T13:06:49.151Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.313 | NOTICE | [408]: 1970-01-02T13:06:49.157Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.313 | NOTICE | [409]: 1970-01-02T13:06:49.164Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.313 | NOTICE | [410]: 1970-01-02T13:06:49.168Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.313 | NOTICE | [411]: 1970-01-02T13:06:49.173Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.313 | NOTICE | [412]: 1970-01-02T13:06:49.176Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.313 | NOTICE | [413]: 1970-01-02T13:06:49.179Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.313 | NOTICE | [414]: 1970-01-02T13:06:49.179Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.313 | NOTICE | [415]: 1970-01-02T13:06:49.179Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.313 | NOTICE | [416]: 1970-01-02T13:06:49.179Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.313 | NOTICE | [417]: 1970-01-02T13:06:49.179Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.313 | NOTICE | [418]: 1970-01-02T13:06:49.282Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.313 | NOTICE | [419]: 1970-01-02T13:06:49.305Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.313 | NOTICE | [420]: 1970-01-02T13:06:49.327Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.313 | NOTICE | [421]: 1970-01-02T13:06:49.334Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.313 | NOTICE | [422]: 1970-01-02T13:06:49.340Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.313 | NOTICE | [423]: 1970-01-02T13:06:49.347Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.313 | NOTICE | [424]: 1970-01-02T13:06:49.352Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.313 | NOTICE | [425]: 1970-01-02T13:06:49.356Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.313 | NOTICE | [426]: 1970-01-02T13:06:49.359Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.313 | NOTICE | [427]: 1970-01-02T13:06:49.362Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.313 | NOTICE | [428]: 1970-01-02T13:06:49.362Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.313 | NOTICE | [429]: 1970-01-02T13:06:49.362Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.313 | NOTICE | [430]: 1970-01-02T13:06:49.362Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.313 | NOTICE | [431]: 1970-01-02T13:06:49.362Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.314 | NOTICE | [432]: 1970-01-02T13:06:49.466Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.314 | NOTICE | [433]: 1970-01-02T13:06:49.488Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.314 | NOTICE | [434]: 1970-01-02T13:06:49.510Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.314 | NOTICE | [435]: 1970-01-02T13:06:49.517Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.314 | NOTICE | [436]: 1970-01-02T13:06:49.523Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.314 | NOTICE | [437]: 1970-01-02T13:06:49.530Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.314 | NOTICE | [438]: 1970-01-02T13:06:49.535Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.314 | NOTICE | [439]: 1970-01-02T13:06:49.539Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.314 | NOTICE | [440]: 1970-01-02T13:06:49.542Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.314 | NOTICE | [441]: 1970-01-02T13:06:49.545Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.314 | NOTICE | [442]: 1970-01-02T13:06:49.545Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.314 | NOTICE | [443]: 1970-01-02T13:06:49.545Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.314 | NOTICE | [444]: 1970-01-02T13:06:49.545Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.314 | NOTICE | [445]: 1970-01-02T13:06:49.545Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.314 | NOTICE | [446]: 1970-01-02T13:06:49.650Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.314 | NOTICE | [447]: 1970-01-02T13:06:49.672Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.314 | NOTICE | [448]: 1970-01-02T13:06:49.694Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.314 | NOTICE | [449]: 1970-01-02T13:06:49.701Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.314 | NOTICE | [450]: 1970-01-02T13:06:49.708Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.314 | NOTICE | [451]: 1970-01-02T13:06:49.715Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.314 | NOTICE | [452]: 1970-01-02T13:06:49.719Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.314 | NOTICE | [453]: 1970-01-02T13:06:49.724Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.314 | NOTICE | [454]: 1970-01-02T13:06:49.727Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.314 | NOTICE | [455]: 1970-01-02T13:06:49.730Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.314 | NOTICE | [456]: 1970-01-02T13:06:49.730Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.314 | NOTICE | [457]: 1970-01-02T13:06:49.730Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.314 | NOTICE | [458]: 1970-01-02T13:06:49.730Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.314 | NOTICE | [459]: 1970-01-02T13:06:49.730Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.314 | NOTICE | [460]: 1970-01-02T13:06:49.834Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.314 | NOTICE | [461]: 1970-01-02T13:06:49.856Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.314 | NOTICE | [462]: 1970-01-02T13:06:49.878Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.314 | NOTICE | [463]: 1970-01-02T13:06:49.885Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.314 | NOTICE | [464]: 1970-01-02T13:06:49.891Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.314 | NOTICE | [465]: 1970-01-02T13:06:49.898Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.314 | NOTICE | [466]: 1970-01-02T13:06:49.902Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.314 | NOTICE | [467]: 1970-01-02T13:06:49.907Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.314 | NOTICE | [468]: 1970-01-02T13:06:49.910Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.314 | NOTICE | [469]: 1970-01-02T13:06:49.913Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.314 | NOTICE | [470]: 1970-01-02T13:06:49.913Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.314 | NOTICE | [471]: 1970-01-02T13:06:49.913Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.314 | NOTICE | [472]: 1970-01-02T13:06:49.913Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.314 | NOTICE | [473]: 1970-01-02T13:06:49.913Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.314 | NOTICE | [474]: 1970-01-02T13:06:50.017Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.314 | NOTICE | [475]: 1970-01-02T13:06:50.039Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.314 | NOTICE | [476]: 1970-01-02T13:06:50.061Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.314 | NOTICE | [477]: 1970-01-02T13:06:50.068Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.314 | NOTICE | [478]: 1970-01-02T13:06:50.075Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.314 | NOTICE | [479]: 1970-01-02T13:06:50.082Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.314 | NOTICE | [480]: 1970-01-02T13:06:50.086Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.314 | NOTICE | [481]: 1970-01-02T13:06:50.091Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.314 | NOTICE | [482]: 1970-01-02T13:06:50.094Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.314 | NOTICE | [483]: 1970-01-02T13:06:50.096Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.314 | NOTICE | [484]: 1970-01-02T13:06:50.097Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.314 | NOTICE | [485]: 1970-01-02T13:06:50.097Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.314 | NOTICE | [486]: 1970-01-02T13:06:50.097Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.314 | NOTICE | [487]: 1970-01-02T13:06:50.097Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.314 | NOTICE | [488]: 1970-01-02T13:06:50.201Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.314 | NOTICE | [489]: 1970-01-02T13:06:50.223Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.314 | NOTICE | [490]: 1970-01-02T13:06:50.245Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.314 | NOTICE | [491]: 1970-01-02T13:06:50.252Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.314 | NOTICE | [492]: 1970-01-02T13:06:50.258Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.314 | NOTICE | [493]: 1970-01-02T13:06:50.265Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.314 | NOTICE | [494]: 1970-01-02T13:06:50.269Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.314 | NOTICE | [495]: 1970-01-02T13:06:50.274Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.314 | NOTICE | [496]: 1970-01-02T13:06:50.277Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.314 | NOTICE | [497]: 1970-01-02T13:06:50.280Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.314 | NOTICE | [498]: 1970-01-02T13:06:50.280Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.314 | NOTICE | [499]: 1970-01-02T13:06:50.280Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.314 | NOTICE | [500]: 1970-01-02T13:06:50.280Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.314 | NOTICE | [501]: 1970-01-02T13:06:50.280Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.314 | NOTICE | [502]: 1970-01-02T13:06:50.384Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.315 | NOTICE | [503]: 1970-01-02T13:06:50.406Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.315 | NOTICE | [504]: 1970-01-02T13:06:50.428Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.315 | NOTICE | [505]: 1970-01-02T13:06:50.434Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.315 | NOTICE | [506]: 1970-01-02T13:06:50.441Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.315 | NOTICE | [507]: 1970-01-02T13:06:50.448Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.315 | NOTICE | [508]: 1970-01-02T13:06:50.452Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.315 | NOTICE | [509]: 1970-01-02T13:06:50.457Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.315 | NOTICE | [510]: 1970-01-02T13:06:50.460Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.315 | NOTICE | [511]: 1970-01-02T13:06:50.462Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.315 | NOTICE | [512]: 1970-01-02T13:06:50.463Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.315 | NOTICE | [513]: 1970-01-02T13:06:50.463Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.315 | NOTICE | [514]: 1970-01-02T13:06:50.463Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.315 | NOTICE | [515]: 1970-01-02T13:06:50.463Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.315 | NOTICE | [516]: 1970-01-02T13:06:50.567Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.315 | NOTICE | [517]: 1970-01-02T13:06:50.589Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.315 | NOTICE | [518]: 1970-01-02T13:06:50.611Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.315 | NOTICE | [519]: 1970-01-02T13:06:50.618Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.315 | NOTICE | [520]: 1970-01-02T13:06:50.625Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.315 | NOTICE | [521]: 1970-01-02T13:06:50.631Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.315 | NOTICE | [522]: 1970-01-02T13:06:50.636Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.315 | NOTICE | [523]: 1970-01-02T13:06:50.640Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.315 | NOTICE | [524]: 1970-01-02T13:06:50.643Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.315 | NOTICE | [525]: 1970-01-02T13:06:50.646Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.315 | NOTICE | [526]: 1970-01-02T13:06:50.646Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.315 | NOTICE | [527]: 1970-01-02T13:06:50.646Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.315 | NOTICE | [528]: 1970-01-02T13:06:50.646Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.315 | NOTICE | [529]: 1970-01-02T13:06:50.646Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.315 | NOTICE | [530]: 1970-01-02T13:06:50.750Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.315 | NOTICE | [531]: 1970-01-02T13:06:50.772Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.315 | NOTICE | [532]: 1970-01-02T13:06:50.794Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.315 | NOTICE | [533]: 1970-01-02T13:06:50.801Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.315 | NOTICE | [534]: 1970-01-02T13:06:50.808Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.315 | NOTICE | [535]: 1970-01-02T13:06:50.814Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.315 | NOTICE | [536]: 1970-01-02T13:06:50.819Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.315 | NOTICE | [537]: 1970-01-02T13:06:50.823Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.315 | NOTICE | [538]: 1970-01-02T13:06:50.826Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.315 | NOTICE | [539]: 1970-01-02T13:06:50.829Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.315 | NOTICE | [540]: 1970-01-02T13:06:50.829Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.315 | NOTICE | [541]: 1970-01-02T13:06:50.829Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.315 | NOTICE | [542]: 1970-01-02T13:06:50.829Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.315 | NOTICE | [543]: 1970-01-02T13:06:50.829Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.315 | NOTICE | [544]: 1970-01-02T13:06:50.934Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.315 | NOTICE | [545]: 1970-01-02T13:06:50.956Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.315 | NOTICE | [546]: 1970-01-02T13:06:50.978Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.315 | NOTICE | [547]: 1970-01-02T13:06:50.985Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.315 | NOTICE | [548]: 1970-01-02T13:06:50.991Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.315 | NOTICE | [549]: 1970-01-02T13:06:50.998Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.315 | NOTICE | [550]: 1970-01-02T13:06:51.002Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.315 | NOTICE | [551]: 1970-01-02T13:06:51.007Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.315 | NOTICE | [552]: 1970-01-02T13:06:51.010Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.315 | NOTICE | [553]: 1970-01-02T13:06:51.013Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.315 | NOTICE | [554]: 1970-01-02T13:06:51.013Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.315 | NOTICE | [555]: 1970-01-02T13:06:51.013Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.315 | NOTICE | [556]: 1970-01-02T13:06:51.013Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.315 | NOTICE | [557]: 1970-01-02T13:06:51.013Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.315 | NOTICE | [558]: 1970-01-02T13:06:51.117Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.315 | NOTICE | [559]: 1970-01-02T13:06:51.139Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.315 | NOTICE | [560]: 1970-01-02T13:06:51.161Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.315 | NOTICE | [561]: 1970-01-02T13:06:51.168Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.315 | NOTICE | [562]: 1970-01-02T13:06:51.174Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.315 | NOTICE | [563]: 1970-01-02T13:06:51.181Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.315 | NOTICE | [564]: 1970-01-02T13:06:51.185Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.315 | NOTICE | [565]: 1970-01-02T13:06:51.190Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.315 | NOTICE | [566]: 1970-01-02T13:06:51.193Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.315 | NOTICE | [567]: 1970-01-02T13:06:51.196Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.315 | NOTICE | [568]: 1970-01-02T13:06:51.196Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.315 | NOTICE | [569]: 1970-01-02T13:06:51.196Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.315 | NOTICE | [570]: 1970-01-02T13:06:51.196Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.315 | NOTICE | [571]: 1970-01-02T13:06:51.196Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.315 | NOTICE | [572]: 1970-01-02T13:06:51.300Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.315 | NOTICE | [573]: 1970-01-02T13:06:51.322Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.315 | NOTICE | [574]: 1970-01-02T13:06:51.344Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.315 | NOTICE | [575]: 1970-01-02T13:06:51.351Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.315 | NOTICE | [576]: 1970-01-02T13:06:51.358Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.315 | NOTICE | [577]: 1970-01-02T13:06:51.364Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.316 | NOTICE | [578]: 1970-01-02T13:06:51.369Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.316 | NOTICE | [579]: 1970-01-02T13:06:51.373Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.316 | NOTICE | [580]: 1970-01-02T13:06:51.376Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.316 | NOTICE | [581]: 1970-01-02T13:06:51.379Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.316 | NOTICE | [582]: 1970-01-02T13:06:51.379Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.316 | NOTICE | [583]: 1970-01-02T13:06:51.379Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.316 | NOTICE | [584]: 1970-01-02T13:06:51.379Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.316 | NOTICE | [585]: 1970-01-02T13:06:51.379Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.316 | NOTICE | [586]: 1970-01-02T13:06:51.483Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.316 | NOTICE | [587]: 1970-01-02T13:06:51.505Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.316 | NOTICE | [588]: 1970-01-02T13:06:51.526Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.316 | NOTICE | [589]: 1970-01-02T13:06:51.533Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.316 | NOTICE | [590]: 1970-01-02T13:06:51.540Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.316 | NOTICE | [591]: 1970-01-02T13:06:51.546Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.316 | NOTICE | [592]: 1970-01-02T13:06:51.551Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.316 | NOTICE | [593]: 1970-01-02T13:06:51.555Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.316 | NOTICE | [594]: 1970-01-02T13:06:51.558Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.316 | NOTICE | [595]: 1970-01-02T13:06:51.561Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.316 | NOTICE | [596]: 1970-01-02T13:06:51.561Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.316 | NOTICE | [597]: 1970-01-02T13:06:51.561Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.316 | NOTICE | [598]: 1970-01-02T13:06:51.561Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.316 | NOTICE | [599]: 1970-01-02T13:06:51.561Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.316 | NOTICE | [600]: 1970-01-02T13:06:51.663Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.316 | NOTICE | [601]: 1970-01-02T13:06:51.685Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.316 | NOTICE | [602]: 1970-01-02T13:06:51.707Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.316 | NOTICE | [603]: 1970-01-02T13:06:51.713Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.316 | NOTICE | [604]: 1970-01-02T13:06:51.720Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.316 | NOTICE | [605]: 1970-01-02T13:06:51.726Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.316 | NOTICE | [606]: 1970-01-02T13:06:51.731Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.316 | NOTICE | [607]: 1970-01-02T13:06:51.735Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.316 | NOTICE | [608]: 1970-01-02T13:06:51.738Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.316 | NOTICE | [609]: 1970-01-02T13:06:51.741Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.316 | NOTICE | [610]: 1970-01-02T13:06:51.741Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.316 | NOTICE | [611]: 1970-01-02T13:06:51.741Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.316 | NOTICE | [612]: 1970-01-02T13:06:51.741Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.316 | NOTICE | [613]: 1970-01-02T13:06:51.741Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.316 | NOTICE | [614]: 1970-01-02T13:06:51.844Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.316 | NOTICE | [615]: 1970-01-02T13:06:51.866Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.316 | NOTICE | [616]: 1970-01-02T13:06:51.888Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.316 | NOTICE | [617]: 1970-01-02T13:06:51.894Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.316 | NOTICE | [618]: 1970-01-02T13:06:51.901Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.316 | NOTICE | [619]: 1970-01-02T13:06:51.907Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.316 | NOTICE | [620]: 1970-01-02T13:06:51.912Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.316 | NOTICE | [621]: 1970-01-02T13:06:51.916Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.316 | NOTICE | [622]: 1970-01-02T13:06:51.919Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.316 | NOTICE | [623]: 1970-01-02T13:06:51.922Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.316 | NOTICE | [624]: 1970-01-02T13:06:51.922Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.316 | NOTICE | [625]: 1970-01-02T13:06:51.922Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.316 | NOTICE | [626]: 1970-01-02T13:06:51.922Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.316 | NOTICE | [627]: 1970-01-02T13:06:51.922Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.316 | NOTICE | [628]: 1970-01-02T13:06:52.024Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.316 | NOTICE | [629]: 1970-01-02T13:06:52.046Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.316 | NOTICE | [630]: 1970-01-02T13:06:52.067Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.316 | NOTICE | [631]: 1970-01-02T13:06:52.074Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.316 | NOTICE | [632]: 1970-01-02T13:06:52.081Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.316 | NOTICE | [633]: 1970-01-02T13:06:52.087Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.316 | NOTICE | [634]: 1970-01-02T13:06:52.092Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.316 | NOTICE | [635]: 1970-01-02T13:06:52.096Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.316 | NOTICE | [636]: 1970-01-02T13:06:52.099Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.316 | NOTICE | [637]: 1970-01-02T13:06:52.102Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.316 | NOTICE | [638]: 1970-01-02T13:06:52.102Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.316 | NOTICE | [639]: 1970-01-02T13:06:52.102Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.316 | NOTICE | [640]: 1970-01-02T13:06:52.102Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.316 | NOTICE | [641]: 1970-01-02T13:06:52.102Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.316 | NOTICE | [642]: 1970-01-02T13:06:52.204Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.316 | NOTICE | [643]: 1970-01-02T13:06:52.226Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.316 | NOTICE | [644]: 1970-01-02T13:06:52.247Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.316 | NOTICE | [645]: 1970-01-02T13:06:52.254Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.316 | NOTICE | [646]: 1970-01-02T13:06:52.261Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.316 | NOTICE | [647]: 1970-01-02T13:06:52.267Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.316 | NOTICE | [648]: 1970-01-02T13:06:52.272Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.316 | NOTICE | [649]: 1970-01-02T13:06:52.276Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.316 | NOTICE | [650]: 1970-01-02T13:06:52.279Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.316 | NOTICE | [651]: 1970-01-02T13:06:52.282Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.317 | NOTICE | [652]: 1970-01-02T13:06:52.282Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.317 | NOTICE | [653]: 1970-01-02T13:06:52.282Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.317 | NOTICE | [654]: 1970-01-02T13:06:52.282Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.317 | NOTICE | [655]: 1970-01-02T13:06:52.282Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.317 | NOTICE | [656]: 1970-01-02T13:06:52.383Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.317 | NOTICE | [657]: 1970-01-02T13:06:52.405Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.317 | NOTICE | [658]: 1970-01-02T13:06:52.427Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.317 | NOTICE | [659]: 1970-01-02T13:06:52.433Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.317 | NOTICE | [660]: 1970-01-02T13:06:52.440Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.317 | NOTICE | [661]: 1970-01-02T13:06:52.447Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.317 | NOTICE | [662]: 1970-01-02T13:06:52.451Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.317 | NOTICE | [663]: 1970-01-02T13:06:52.456Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.317 | NOTICE | [664]: 1970-01-02T13:06:52.458Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.317 | NOTICE | [665]: 1970-01-02T13:06:52.461Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.317 | NOTICE | [666]: 1970-01-02T13:06:52.461Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.317 | NOTICE | [667]: 1970-01-02T13:06:52.461Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.317 | NOTICE | [668]: 1970-01-02T13:06:52.461Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.317 | NOTICE | [669]: 1970-01-02T13:06:52.461Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.317 | NOTICE | [670]: 1970-01-02T13:06:52.563Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.317 | NOTICE | [671]: 1970-01-02T13:06:52.585Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.317 | NOTICE | [672]: 1970-01-02T13:06:52.606Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.317 | NOTICE | [673]: 1970-01-02T13:06:52.613Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.317 | NOTICE | [674]: 1970-01-02T13:06:52.620Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.317 | NOTICE | [675]: 1970-01-02T13:06:52.626Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.317 | NOTICE | [676]: 1970-01-02T13:06:52.631Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.317 | NOTICE | [677]: 1970-01-02T13:06:52.635Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.317 | NOTICE | [678]: 1970-01-02T13:06:52.638Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.317 | NOTICE | [679]: 1970-01-02T13:06:52.641Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.317 | NOTICE | [680]: 1970-01-02T13:06:52.641Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.317 | NOTICE | [681]: 1970-01-02T13:06:52.641Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.317 | NOTICE | [682]: 1970-01-02T13:06:52.641Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.317 | NOTICE | [683]: 1970-01-02T13:06:52.641Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.317 | NOTICE | [684]: 1970-01-02T13:06:52.743Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.317 | NOTICE | [685]: 1970-01-02T13:06:52.765Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.317 | NOTICE | [686]: 1970-01-02T13:06:52.786Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.317 | NOTICE | [687]: 1970-01-02T13:06:52.793Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.317 | NOTICE | [688]: 1970-01-02T13:06:52.800Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.317 | NOTICE | [689]: 1970-01-02T13:06:52.806Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.317 | NOTICE | [690]: 1970-01-02T13:06:52.810Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.317 | NOTICE | [691]: 1970-01-02T13:06:52.815Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.317 | NOTICE | [692]: 1970-01-02T13:06:52.818Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.317 | NOTICE | [693]: 1970-01-02T13:06:52.821Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.317 | NOTICE | [694]: 1970-01-02T13:06:52.821Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.317 | NOTICE | [695]: 1970-01-02T13:06:52.821Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.317 | NOTICE | [696]: 1970-01-02T13:06:52.821Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.317 | NOTICE | [697]: 1970-01-02T13:06:52.821Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.317 | NOTICE | [698]: 1970-01-02T13:06:52.922Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.317 | NOTICE | [699]: 1970-01-02T13:06:52.944Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.317 | NOTICE | [700]: 1970-01-02T13:06:52.966Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.317 | NOTICE | [701]: 1970-01-02T13:06:52.973Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.317 | NOTICE | [702]: 1970-01-02T13:06:52.979Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.317 | NOTICE | [703]: 1970-01-02T13:06:52.986Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.317 | NOTICE | [704]: 1970-01-02T13:06:52.990Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.317 | NOTICE | [705]: 1970-01-02T13:06:52.994Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.317 | NOTICE | [706]: 1970-01-02T13:06:52.997Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.317 | NOTICE | [707]: 1970-01-02T13:06:52.999Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.317 | NOTICE | [708]: 1970-01-02T13:06:53.000Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.317 | NOTICE | [709]: 1970-01-02T13:06:53.000Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.317 | NOTICE | [710]: 1970-01-02T13:06:53.000Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.317 | NOTICE | [711]: 1970-01-02T13:06:53.000Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.317 | NOTICE | [712]: 1970-01-02T13:06:53.102Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.317 | NOTICE | [713]: 1970-01-02T13:06:53.123Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.317 | NOTICE | [714]: 1970-01-02T13:06:53.145Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.317 | NOTICE | [715]: 1970-01-02T13:06:53.152Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.317 | NOTICE | [716]: 1970-01-02T13:06:53.158Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.317 | NOTICE | [717]: 1970-01-02T13:06:53.165Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.317 | NOTICE | [718]: 1970-01-02T13:06:53.169Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.317 | NOTICE | [719]: 1970-01-02T13:06:53.174Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.317 | NOTICE | [720]: 1970-01-02T13:06:53.176Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.317 | NOTICE | [721]: 1970-01-02T13:06:53.179Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.317 | NOTICE | [722]: 1970-01-02T13:06:53.179Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.317 | NOTICE | [723]: 1970-01-02T13:06:53.179Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.317 | NOTICE | [724]: 1970-01-02T13:06:53.179Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.317 | NOTICE | [725]: 1970-01-02T13:06:53.179Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.317 | NOTICE | [726]: 1970-01-02T13:06:53.281Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.318 | NOTICE | [727]: 1970-01-02T13:06:53.303Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.318 | NOTICE | [728]: 1970-01-02T13:06:53.325Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.318 | NOTICE | [729]: 1970-01-02T13:06:53.332Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.318 | NOTICE | [730]: 1970-01-02T13:06:53.339Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.318 | NOTICE | [731]: 1970-01-02T13:06:53.345Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.318 | NOTICE | [732]: 1970-01-02T13:06:53.350Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.318 | NOTICE | [733]: 1970-01-02T13:06:53.354Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.318 | NOTICE | [734]: 1970-01-02T13:06:53.357Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.318 | NOTICE | [735]: 1970-01-02T13:06:53.360Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.318 | NOTICE | [736]: 1970-01-02T13:06:53.360Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.318 | NOTICE | [737]: 1970-01-02T13:06:53.360Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.318 | NOTICE | [738]: 1970-01-02T13:06:53.360Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.318 | NOTICE | [739]: 1970-01-02T13:06:53.360Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.318 | NOTICE | [740]: 1970-01-02T13:06:53.462Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.318 | NOTICE | [741]: 1970-01-02T13:06:53.484Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.318 | NOTICE | [742]: 1970-01-02T13:06:53.506Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.318 | NOTICE | [743]: 1970-01-02T13:06:53.512Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.318 | NOTICE | [744]: 1970-01-02T13:06:53.519Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.318 | NOTICE | [745]: 1970-01-02T13:06:53.525Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.318 | NOTICE | [746]: 1970-01-02T13:06:53.530Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.318 | NOTICE | [747]: 1970-01-02T13:06:53.534Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.318 | NOTICE | [748]: 1970-01-02T13:06:53.537Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.318 | NOTICE | [749]: 1970-01-02T13:06:53.540Z GPU thread:140016204685312 size:60293120 | 20260507 08:47:17.318 | NOTICE | [750]: 1970-01-02T13:06:53.540Z GPU thread:140016204685312 size:15728640 | 20260507 08:47:17.318 | NOTICE | [751]: 1970-01-02T13:06:53.540Z GPU thread:140016204685312 size:6291456 | 20260507 08:47:17.318 | NOTICE | [752]: 1970-01-02T13:06:53.540Z GPU thread:140016204685312 size:1048576 | 20260507 08:47:17.318 | NOTICE | [753]: 1970-01-02T13:06:53.540Z HOST thread:140016204685312 size:160432128 | 20260507 08:47:17.318 | NOTICE | [754]: 1970-01-02T13:06:53.643Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.318 | NOTICE | [755]: 1970-01-02T13:06:53.665Z HOST thread:140016204685312 size:33030144 | 20260507 08:47:17.318 | NOTICE | [756]: 1970-01-02T13:06:53.686Z HOST thread:140016204685312 size:10485760 | 20260507 08:47:17.318 | NOTICE | [757]: 1970-01-02T13:06:53.693Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.318 | NOTICE | [758]: 1970-01-02T13:06:53.700Z HOST thread:140016204685312 size:9961472 | 20260507 08:47:17.318 | NOTICE | [759]: 1970-01-02T13:06:53.706Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.318 | NOTICE | [760]: 1970-01-02T13:06:53.711Z HOST thread:140016204685312 size:6815744 | 20260507 08:47:17.318 | NOTICE | [761]: 1970-01-02T13:06:53.715Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.318 | NOTICE | [762]: 1970-01-02T13:06:53.718Z HOST thread:140016204685312 size:4194304 | 20260507 08:47:17.318 | NOTICE | [763]: 1970-01-02T13:07:38.476Z GPU thread:139974759010304 size:134217728 | 20260507 08:47:17.318 | NOTICE | [764]: 1970-01-02T13:07:38.493Z HOST thread:139974759010304 size:1 | 20260507 08:47:17.318 | NOTICE | [765]: 1970-01-02T13:07:38.493Z HOST thread:139974759010304 size:256 | 20260507 08:47:17.318 | NOTICE | [766]: 1970-01-02T13:07:38.493Z GPU thread:139974721224704 size:134217728 | 20260507 08:47:17.318 | NOTICE | [767]: 1970-01-02T13:07:38.494Z GPU thread:139974759010304 size:268435456 | 20260507 08:47:17.318 | NOTICE | [768]: 1970-01-02T13:07:38.499Z GPU thread:139974759010304 size:24641536 | 20260507 08:47:17.318 | NOTICE | [769]: 1970-01-02T13:07:38.499Z GPU thread:139974759010304 size:134217728 | 20260507 08:47:17.318 | NOTICE | [770]: 1970-01-02T13:07:38.500Z GPU thread:139974759010304 size:134217728 | 20260507 08:47:17.318 | NOTICE | [771]: 1970-01-02T13:07:38.500Z GPU thread:139974721224704 size:160432128 | 20260507 08:47:17.318 | NOTICE | [772]: 1970-01-02T13:07:38.506Z GPU thread:139974721224704 size:33030144 | 20260507 08:47:17.318 | NOTICE | [773]: 1970-01-02T13:07:38.508Z GPU thread:139974721224704 size:33030144 | 20260507 08:47:17.318 | NOTICE | [774]: 1970-01-02T13:07:38.509Z GPU thread:139974721224704 size:10485760 | 20260507 08:47:17.318 | NOTICE | [775]: 1970-01-02T13:07:38.511Z GPU thread:139974721224704 size:9961472 | 20260507 08:47:17.318 | NOTICE | [776]: 1970-01-02T13:07:38.512Z GPU thread:139974721224704 size:9961472 | 20260507 08:47:17.318 | NOTICE | [777]: 1970-01-02T13:07:38.514Z GPU thread:139974721224704 size:6815744 | 20260507 08:47:17.318 | NOTICE | [778]: 1970-01-02T13:07:38.522Z GPU thread:139974721224704 size:6815744 | 20260507 08:47:17.318 | NOTICE | [779]: 1970-01-02T13:07:38.529Z GPU thread:139974721224704 size:4194304 | 20260507 08:47:17.318 | NOTICE | [780]: 1970-01-02T13:07:38.529Z GPU thread:139974721224704 size:4194304 | 20260507 08:47:17.318 | NOTICE | [781]: 1970-01-02T13:07:38.529Z GPU thread:139974750613504 size:134217728 | 20260507 08:47:17.318 | NOTICE | [782]: 1970-01-02T13:07:38.530Z HOST thread:139974759010304 size:268435456 | 20260507 08:47:17.318 | NOTICE | [783]: 1970-01-02T13:07:38.535Z GPU thread:139974750613504 size:160432128 | 20260507 08:47:17.318 | NOTICE | [784]: 1970-01-02T13:07:38.691Z HOST thread:139974759010304 size:12582912 | 20260507 08:47:17.318 | NOTICE | [785]: 1970-01-02T13:07:38.706Z GPU thread:139974750613504 size:33030144 | 20260507 08:47:17.318 | NOTICE | [786]: 1970-01-02T13:07:38.707Z GPU thread:139974750613504 size:33030144 | 20260507 08:47:17.318 | NOTICE | [787]: 1970-01-02T13:07:38.709Z GPU thread:139974750613504 size:10485760 | 20260507 08:47:17.318 | NOTICE | [788]: 1970-01-02T13:07:38.709Z GPU thread:139974750613504 size:9961472 | 20260507 08:47:17.318 | NOTICE | [789]: 1970-01-02T13:07:38.710Z GPU thread:139974750613504 size:9961472 | 20260507 08:47:17.318 | NOTICE | [790]: 1970-01-02T13:07:38.710Z GPU thread:139974750613504 size:6815744 | 20260507 08:47:17.318 | NOTICE | [791]: 1970-01-02T13:07:38.710Z GPU thread:139974750613504 size:6815744 | 20260507 08:47:17.318 | NOTICE | [792]: 1970-01-02T13:07:38.711Z GPU thread:139974750613504 size:4194304 | 20260507 08:47:17.318 | NOTICE | [793]: 1970-01-02T13:07:38.711Z GPU thread:139974750613504 size:4194304 | 20260507 08:47:17.318 | NOTICE | [794]: 1970-01-02T13:07:38.717Z HOST thread:139974759010304 size:540016640 | 20260507 08:47:17.318 | NOTICE | [795]: 1970-01-02T13:07:39.038Z HOST thread:139974759010304 size:1048576 | 20260507 08:47:17.318 | NOTICE | [796]: 1970-01-02T13:07:39.038Z HOST thread:139974759010304 size:1048576 | 20260507 08:47:17.318 | NOTICE | [797]: 1970-01-02T13:07:39.039Z HOST thread:139974759010304 size:4194304 | 20260507 08:47:17.318 | NOTICE | [798]: 1970-01-02T13:07:39.042Z HOST thread:139974759010304 size:1 | 20260507 08:47:17.318 | NOTICE | [799]: 1970-01-02T13:07:39.042Z HOST thread:139974759010304 size:2097156 | 20260507 08:47:17.319 | NOTICE | [800]: 1970-01-02T13:08:14.512Z HOST thread:139974725423104 size:1 | 20260507 08:47:17.319 | NOTICE | [801]: 1970-01-02T13:08:14.512Z HOST thread:139974725423104 size:2097156 | 20260507 08:47:17.319 | NOTICE | [802]: 1970-01-02T13:09:11.006Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [803]: 1970-01-02T13:09:11.192Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [804]: 1970-01-02T13:09:11.371Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [805]: 1970-01-02T13:09:11.551Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [806]: 1970-01-02T13:09:11.731Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [807]: 1970-01-02T13:09:11.911Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [808]: 1970-01-02T13:09:12.092Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [809]: 1970-01-02T13:09:12.272Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [810]: 1970-01-02T13:09:12.451Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [811]: 1970-01-02T13:09:12.630Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [812]: 1970-01-02T13:09:12.809Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [813]: 1970-01-02T13:09:12.989Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [814]: 1970-01-02T13:09:16.142Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [815]: 1970-01-02T13:09:16.332Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [816]: 1970-01-02T13:09:16.512Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [817]: 1970-01-02T13:09:16.692Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [818]: 1970-01-02T13:09:16.872Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [819]: 1970-01-02T13:09:17.053Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [820]: 1970-01-02T13:09:17.235Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [821]: 1970-01-02T13:09:17.415Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [822]: 1970-01-02T13:09:17.595Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [823]: 1970-01-02T13:09:17.775Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [824]: 1970-01-02T13:09:17.956Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [825]: 1970-01-02T13:09:18.137Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [826]: 1970-01-02T13:09:21.285Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [827]: 1970-01-02T13:09:21.468Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [828]: 1970-01-02T13:09:21.650Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [829]: 1970-01-02T13:09:21.833Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [830]: 1970-01-02T13:09:22.017Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [831]: 1970-01-02T13:09:22.201Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [832]: 1970-01-02T13:09:22.383Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [833]: 1970-01-02T13:09:22.565Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [834]: 1970-01-02T13:09:22.748Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [835]: 1970-01-02T13:09:22.930Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [836]: 1970-01-02T13:09:23.113Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [837]: 1970-01-02T13:09:23.296Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [838]: 1970-01-02T13:09:26.422Z GPU thread:139974754811904 size:8912896 | 20260507 08:47:17.319 | NOTICE | [839]: 1970-01-02T13:09:26.605Z GPU thread:139974754811904 size:8912896 | 20260507 08:47:17.319 | NOTICE | [840]: 1970-01-02T13:09:26.788Z GPU thread:139974754811904 size:8912896 | 20260507 08:47:17.319 | NOTICE | [841]: 1970-01-02T13:09:26.971Z GPU thread:139974754811904 size:8912896 | 20260507 08:47:17.319 | NOTICE | [842]: 1970-01-02T13:09:27.154Z GPU thread:139974754811904 size:8912896 | 20260507 08:47:17.319 | NOTICE | [843]: 1970-01-02T13:09:27.337Z GPU thread:139974754811904 size:8912896 | 20260507 08:47:17.319 | NOTICE | [844]: 1970-01-02T13:09:27.521Z GPU thread:139974754811904 size:8912896 | 20260507 08:47:17.319 | NOTICE | [845]: 1970-01-02T13:09:27.705Z GPU thread:139974754811904 size:8912896 | 20260507 08:47:17.319 | NOTICE | [846]: 1970-01-02T13:09:27.888Z GPU thread:139974754811904 size:8912896 | 20260507 08:47:17.319 | NOTICE | [847]: 1970-01-02T13:09:28.071Z GPU thread:139974754811904 size:8912896 | 20260507 08:47:17.319 | NOTICE | [848]: 1970-01-02T13:09:28.254Z GPU thread:139974754811904 size:8912896 | 20260507 08:47:17.319 | NOTICE | [849]: 1970-01-02T13:09:28.438Z GPU thread:139974754811904 size:8912896 | 20260507 08:47:17.319 | NOTICE | [850]: 1970-01-02T13:09:31.562Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [851]: 1970-01-02T13:09:31.748Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [852]: 1970-01-02T13:09:31.933Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [853]: 1970-01-02T13:09:32.119Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [854]: 1970-01-02T13:09:32.307Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [855]: 1970-01-02T13:09:32.493Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [856]: 1970-01-02T13:09:32.679Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [857]: 1970-01-02T13:09:32.865Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [858]: 1970-01-02T13:09:33.050Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [859]: 1970-01-02T13:09:33.235Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [860]: 1970-01-02T13:09:33.420Z GPU thread:139974733819904 size:8912896 | 20260507 08:47:17.319 | NOTICE | [861]: 1970-01-02T13:09:33.606Z GPU thread:139974733819904 size:8912896 | 20260507 08:47:17.319 | NOTICE | [862]: 1970-01-02T13:09:36.701Z GPU thread:139974750613504 size:8912896 | 20260507 08:47:17.319 | NOTICE | [863]: 1970-01-02T13:09:36.884Z GPU thread:139974759010304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [864]: 1970-01-02T13:09:37.069Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.319 | NOTICE | [865]: 1970-01-02T13:09:37.253Z GPU thread:139974729621504 size:8912896 | 20260507 08:47:17.319 | NOTICE | [866]: 1970-01-02T13:09:37.439Z GPU thread:139974738018304 size:8912896 | 20260507 08:47:17.319 | NOTICE | [867]: 1970-01-02T13:09:37.627Z GPU thread:139974729621504 size:8912896 | 20260507 08:47:17.319 | NOTICE | [868]: 1970-01-02T13:09:37.812Z GPU thread:139974733819904 size:8912896 | 20260507 08:47:17.319 | NOTICE | [869]: 1970-01-02T13:09:37.997Z GPU thread:139974725423104 size:8912896 | 20260507 08:47:17.319 | NOTICE | [870]: 1970-01-02T13:09:38.185Z GPU thread:139974746415104 size:8912896 | 20260507 08:47:17.319 | NOTICE | [871]: 1970-01-02T13:09:38.369Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [872]: 1970-01-02T13:09:38.555Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [873]: 1970-01-02T13:09:38.740Z GPU thread:139974729621504 size:8912896 | 20260507 08:47:17.320 | NOTICE | [874]: 1970-01-02T13:09:41.841Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [875]: 1970-01-02T13:09:42.029Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [876]: 1970-01-02T13:09:42.216Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [877]: 1970-01-02T13:09:42.406Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [878]: 1970-01-02T13:09:42.594Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [879]: 1970-01-02T13:09:42.782Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [880]: 1970-01-02T13:09:42.970Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [881]: 1970-01-02T13:09:43.158Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [882]: 1970-01-02T13:09:43.345Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [883]: 1970-01-02T13:09:43.532Z GPU thread:139974742216704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [884]: 1970-01-02T13:09:43.720Z GPU thread:139974754811904 size:8912896 | 20260507 08:47:17.320 | NOTICE | [885]: 1970-01-02T13:09:43.908Z GPU thread:139974754811904 size:8912896 | 20260507 08:47:17.320 | NOTICE | [886]: 1970-01-02T13:09:46.980Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [887]: 1970-01-02T13:09:47.169Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [888]: 1970-01-02T13:09:47.356Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [889]: 1970-01-02T13:09:47.543Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [890]: 1970-01-02T13:09:47.732Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [891]: 1970-01-02T13:09:47.920Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [892]: 1970-01-02T13:09:48.108Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [893]: 1970-01-02T13:09:48.295Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [894]: 1970-01-02T13:09:48.484Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [895]: 1970-01-02T13:09:48.671Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [896]: 1970-01-02T13:09:48.859Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [897]: 1970-01-02T13:09:49.048Z GPU thread:139974721224704 size:8912896 | 20260507 08:47:17.320 | NOTICE | [898]: 1970-01-02T13:09:52.119Z GPU thread:139974733819904 size:8912896 | 20260507 08:47:17.320 | NOTICE | [899]: 1970-01-02T13:09:57.296Z GPU thread:139974759010304 size:268435456 | 20260507 08:47:17.320 | NOTICE | [900]: 1970-01-02T13:09:57.297Z GPU thread:139974759010304 size:12582912 | 20260507 08:47:17.320 | NOTICE | [901]: 1970-01-02T13:09:57.297Z GPU thread:139974759010304 size:1048576 | 20260507 08:47:17.320 | NOTICE | [902]: 1970-01-02T13:09:57.297Z GPU thread:139974759010304 size:1107296256 | 20260507 08:47:17.320 | NOTICE | [903]: 1970-01-02T13:09:57.299Z GPU thread:139974759010304 size:279969792 | 20260507 08:47:17.320 | NOTICE | [904]: 1970-01-02T13:09:57.318Z GPU thread:139974759010304 size:540016640 | 20260507 08:47:17.320 | NOTICE | [905]: 1970-01-02T13:09:57.319Z GPU thread:139974759010304 size:1048576 | 20260507 08:47:17.320 | NOTICE | [906]: 1970-01-02T13:09:57.319Z GPU thread:139974759010304 size:1048576 | 20260507 08:47:17.320 | NOTICE | [907]: 1970-01-02T13:09:57.319Z GPU thread:139974759010304 size:4194304 | 20260507 08:47:17.320 | NOTICE | [908]: 1970-01-02T13:09:57.328Z GPU thread:139974759010304 size:108003328 | 20260507 08:47:17.320 | NOTICE | [909]: 1970-01-02T13:09:57.328Z GPU thread:139974759010304 size:2097156 | 20260507 08:47:17.320 | NOTICE | [910]: 1970-01-02T13:09:57.331Z HOST thread:139974759010304 size:108003328 | 20260507 08:47:17.320 | NOTICE | [911]: 1970-01-02T13:10:07.843Z HOST thread:139974754811904 size:108003328 | 20260507 08:47:17.320 | NOTICE | [912]: 1970-01-02T13:10:17.834Z GPU thread:139974725423104 size:38273024 | 20260507 08:47:17.320 | NOTICE | [913]: 1970-01-02T13:10:17.836Z GPU thread:139974725423104 size:11272192 | 20260507 08:47:17.320 | NOTICE | [914]: 1970-01-02T13:10:17.844Z HOST thread:139974725423104 size:38273024 | 20260507 08:47:17.320 | NOTICE | [915]: 1970-01-02T13:10:17.870Z GPU thread:139974733819904 size:11272192 | 20260507 08:47:17.320 | NOTICE | [916]: 1970-01-02T13:10:17.885Z HOST thread:139974733819904 size:38273024 | 20260507 08:47:17.320 | NOTICE | [917]: 1970-01-02T13:10:17.911Z GPU thread:139974759010304 size:11272192 | 20260507 08:47:17.320 | NOTICE | [918]: 1970-01-02T13:10:17.923Z HOST thread:139974759010304 size:108003328 | 20260507 08:47:17.320 | NOTICE | [919]: 1970-01-02T13:10:18.011Z GPU thread:139974754811904 size:11272192 | 20260507 08:47:17.320 | NOTICE | [920]: 1970-01-02T13:10:18.108Z GPU thread:139974763208704 size:11272192 | 20260507 08:47:17.320 | NOTICE | [921]: 1970-01-02T13:10:18.158Z HOST thread:139974738018304 size:2097156 | 20260507 08:47:17.320 | NOTICE | [922]: 1970-01-02T13:10:18.207Z GPU thread:139974721224704 size:11272192 | 20260507 08:47:17.320 | NOTICE | [923]: 1970-01-02T13:10:18.233Z GPU thread:139974729621504 size:11272192 | 20260507 08:47:17.320 | NOTICE | [924]: 1970-01-02T13:10:18.260Z GPU thread:139974750613504 size:11272192 | 20260507 08:47:17.320 | NOTICE | [925]: 1970-01-02T13:10:18.267Z HOST thread:139974750613504 size:38273024 | 20260507 08:47:17.320 | NOTICE | [926]: 1970-01-02T13:10:18.293Z GPU thread:139974738018304 size:11272192 | 20260507 08:47:17.320 | NOTICE | [927]: 1970-01-02T13:10:18.387Z GPU thread:139974742216704 size:11272192 | 20260507 08:47:17.320 | NOTICE | [928]: 1970-01-02T13:10:18.485Z GPU thread:139974754811904 size:11272192 | 20260507 08:47:17.320 | NOTICE | [929]: 1970-01-02T13:10:18.511Z GPU thread:139974738018304 size:11272192 | 20260507 08:47:17.320 | NOTICE | [930]: 1970-01-02T13:10:18.537Z GPU thread:139974763208704 size:11272192 | 20260507 08:47:17.320 | NOTICE | [931]: 1970-01-02T13:10:18.563Z GPU thread:139974725423104 size:11272192 | 20260507 08:47:17.320 | NOTICE | [932]: 1970-01-02T13:10:18.658Z GPU thread:139974742216704 size:11272192 | 20260507 08:47:17.320 | NOTICE | [933]: 1970-01-02T13:10:18.755Z GPU thread:139974738018304 size:11272192 | 20260507 08:47:17.320 | NOTICE | [934]: 1970-01-02T13:10:18.781Z GPU thread:139974763208704 size:11272192 | 20260507 08:47:17.320 | NOTICE | [935]: 1970-01-02T13:10:18.807Z GPU thread:139974721224704 size:11272192 | 20260507 08:47:17.320 | NOTICE | [936]: 1970-01-02T13:10:18.832Z GPU thread:139974725423104 size:11272192 | 20260507 08:47:17.320 | NOTICE | [937]: 1970-01-02T13:10:18.926Z GPU thread:139974759010304 size:11272192 | 20260507 08:47:17.320 | NOTICE | [938]: 1970-01-02T13:10:19.023Z GPU thread:139974746415104 size:11272192 | 20260507 08:47:17.320 | NOTICE | [939]: 1970-01-02T13:10:19.049Z GPU thread:139974729621504 size:11272192 | 20260507 08:47:17.320 | NOTICE | [940]: 1970-01-02T13:10:19.076Z GPU thread:139974750613504 size:11272192 | 20260507 08:47:17.320 | NOTICE | [941]: 1970-01-02T13:10:19.102Z GPU thread:139974721224704 size:11272192 | 20260507 08:47:17.320 | NOTICE | [942]: 1970-01-02T13:10:19.198Z GPU thread:139974763208704 size:11272192 | 20260507 08:47:17.320 | NOTICE | [943]: 1970-01-02T13:10:19.295Z GPU thread:139974733819904 size:11272192 | 20260507 08:47:17.321 | NOTICE | [944]: 1970-01-02T13:10:19.321Z GPU thread:139974721224704 size:11272192 | 20260507 08:47:17.321 | NOTICE | [945]: 1970-01-02T13:10:19.347Z GPU thread:139974738018304 size:11272192 | 20260507 08:47:17.321 | NOTICE | [946]: 1970-01-02T13:10:19.373Z GPU thread:139974746415104 size:11272192 | 20260507 08:47:17.321 | NOTICE | [947]: 1970-01-02T13:10:19.463Z GPU thread:139974759010304 size:11272192 | 20260507 08:47:17.321 | NOTICE | [948]: 1970-01-02T13:10:19.559Z GPU thread:139974750613504 size:11272192 | 20260507 08:47:17.321 | NOTICE | [949]: 1970-01-02T13:10:19.583Z GPU thread:139974763208704 size:11272192 | 20260507 08:47:17.321 | NOTICE | [950]: 1970-01-02T13:10:19.608Z GPU thread:139974733819904 size:11272192 | 20260507 08:47:17.321 | NOTICE | [951]: 1970-01-02T13:10:19.634Z GPU thread:139974738018304 size:11272192 | 20260507 08:47:17.321 | NOTICE | [952]: 1970-01-02T13:10:19.729Z GPU thread:139974729621504 size:11272192 | 20260507 08:47:17.321 | NOTICE | [953]: 1970-01-02T13:10:19.827Z GPU thread:139974733819904 size:11272192 | 20260507 08:47:17.321 | NOTICE | [954]: 1970-01-02T13:10:19.852Z GPU thread:139974763208704 size:11272192 | 20260507 08:47:17.321 | NOTICE | [955]: 1970-01-02T13:10:19.878Z GPU thread:139974754811904 size:11272192 | 20260507 08:47:17.321 | NOTICE | [956]: 1970-01-02T13:10:19.904Z GPU thread:139974746415104 size:11272192 | 20260507 08:47:17.321 | NOTICE | [957]: 1970-01-02T13:10:19.997Z GPU thread:139974742216704 size:11272192 | 20260507 08:47:17.321 | NOTICE | [958]: 1970-01-02T13:10:20.097Z GPU thread:139974750613504 size:11272192 | 20260507 08:47:17.321 | NOTICE | [959]: 1970-01-02T13:10:20.122Z GPU thread:139974738018304 size:11272192 | 20260507 08:47:17.321 | NOTICE | [960]: 1970-01-02T13:10:20.148Z GPU thread:139974725423104 size:11272192 | 20260507 08:47:17.321 | NOTICE | [961]: 1970-01-02T13:10:20.174Z GPU thread:139974729621504 size:11272192 | 20260507 08:47:17.321 | NOTICE | [962]: 1970-01-02T13:10:20.176Z HOST thread:139974729621504 size:108003328 | 20260507 08:47:17.321 | NOTICE | [963]: 1970-01-02T13:10:20.331Z GPU thread:139974721224704 size:11272192 | 20260507 08:47:17.321 | NOTICE | [964]: 1970-01-02T13:10:20.431Z GPU thread:139974746415104 size:11272192 | 20260507 08:47:17.321 | NOTICE | [965]: 1970-01-02T13:10:20.456Z GPU thread:139974759010304 size:11272192 | 20260507 08:47:17.321 | NOTICE | [966]: 1970-01-02T13:10:20.482Z GPU thread:139974742216704 size:11272192 | 20260507 08:47:17.321 | NOTICE | [967]: 1970-01-02T13:10:20.509Z GPU thread:139974721224704 size:11272192 | 20260507 08:47:17.321 | NOTICE | [968]: 1970-01-02T13:10:20.602Z GPU thread:139974750613504 size:11272192 | 20260507 08:47:17.321 | NOTICE | [969]: 1970-01-02T13:10:20.699Z GPU thread:139974754811904 size:11272192 | 20260507 08:47:17.321 | NOTICE | [970]: 1970-01-02T13:10:20.725Z GPU thread:139974733819904 size:11272192 | 20260507 08:47:17.321 | NOTICE | [971]: 1970-01-02T13:10:20.751Z GPU thread:139974738018304 size:11272192 | 20260507 08:47:17.321 | NOTICE | [972]: 1970-01-02T13:10:20.776Z GPU thread:139974750613504 size:11272192 | 20260507 08:47:17.321 | NOTICE | [973]: 1970-01-02T13:10:20.870Z GPU thread:139974721224704 size:11272192 | 20260507 08:47:17.321 | NOTICE | [974]: 1970-01-02T13:10:20.968Z GPU thread:139974759010304 size:11272192 | 20260507 08:47:17.321 | NOTICE | [975]: 1970-01-02T13:10:20.994Z GPU thread:139974742216704 size:11272192 | 20260507 08:47:17.321 | NOTICE | [976]: 1970-01-02T13:10:21.020Z GPU thread:139974746415104 size:11272192 | 20260507 08:47:17.321 | NOTICE | [977]: 1970-01-02T13:10:21.046Z GPU thread:139974750613504 size:11272192 | 20260507 08:47:17.321 | NOTICE | [978]: 1970-01-02T13:10:21.140Z GPU thread:139974733819904 size:11272192 | 20260507 08:47:17.321 | NOTICE | [979]: 1970-01-02T13:10:38.374Z HOST thread:139974742216704 size:24 | 20260507 08:47:17.321 | NOTICE | [980]: 1970-01-02T13:10:38.374Z GPU thread:139974742216704 size:24 | 20260507 08:47:17.321 | NOTICE | [981]: 1970-01-02T13:11:00.776Z GPU thread:139974729621504 size:11272192 | 20260507 08:47:17.321 | NOTICE | [982]: 1970-01-02T13:11:00.802Z GPU thread:139974746415104 size:11272192 | 20260507 08:47:17.321 | NOTICE | [983]: 1970-01-02T13:11:00.827Z GPU thread:139974750613504 size:11272192 | 20260507 08:47:17.321 | NOTICE | [984]: 1970-01-02T13:11:00.854Z GPU thread:139974733819904 size:11272192 | 20260507 08:47:17.321 | NOTICE | [985]: 1970-01-02T13:11:00.879Z GPU thread:139974763208704 size:11272192 | 20260507 08:47:17.321 | NOTICE | [986]: 1970-01-02T13:11:00.988Z GPU thread:139974754811904 size:11272192 | 20260507 08:47:17.321 | NOTICE | [987]: 1970-01-02T13:11:01.014Z GPU thread:139974746415104 size:11272192 | 20260507 08:47:17.321 | NOTICE | [988]: 1970-01-02T13:11:01.039Z GPU thread:139974729621504 size:11272192 | 20260507 08:47:17.321 | NOTICE | [989]: 1970-01-02T13:11:01.065Z GPU thread:139974725423104 size:11272192 | 20260507 08:47:17.321 | NOTICE | [990]: 1970-01-02T13:11:01.091Z GPU thread:139974759010304 size:11272192 | 20260507 08:47:17.321 | NOTICE | [991]: 1970-01-02T13:11:01.117Z GPU thread:139974742216704 size:11272192 | 20260507 08:47:17.321 | NOTICE | [992]: 1970-01-02T13:11:01.142Z GPU thread:139974733819904 size:11272192 | 20260507 08:47:17.321 | NOTICE | [993]: 1970-01-02T13:11:01.252Z GPU thread:139974729621504 size:11272192 | 20260507 08:47:17.321 | NOTICE | [994]: 1970-01-02T13:11:01.278Z GPU thread:139974763208704 size:11272192 | 20260507 08:47:17.321 | NOTICE | [995]: 1970-01-02T13:11:01.304Z GPU thread:139974759010304 size:11272192 | 20260507 08:47:17.321 | NOTICE | [996]: 1970-01-02T13:11:01.330Z GPU thread:139974746415104 size:11272192 | 20260507 08:47:17.321 | NOTICE | [997]: 1970-01-02T13:11:01.356Z GPU thread:139974733819904 size:11272192 | 20260507 08:47:17.321 | NOTICE | [998]: 1970-01-02T13:11:01.381Z GPU thread:139974750613504 size:11272192 | 20260507 08:47:17.321 | NOTICE | [999]: 1970-01-02T13:11:01.408Z GPU thread:139974729621504 size:11272192 | 20260507 08:47:17.321 | NOTICE | [1000]: 1970-01-02T13:11:01.519Z GPU thread:139974763208704 size:11272192 | 20260507 08:47:17.321 | NOTICE | [1001]: 1970-01-02T13:11:01.545Z GPU thread:139974754811904 size:11272192 | 20260507 08:47:17.321 | NOTICE | [1002]: 1970-01-02T13:11:01.571Z GPU thread:139974742216704 size:11272192 | 20260507 08:47:17.321 | NOTICE | [1003]: 1970-01-02T13:11:01.597Z GPU thread:139974729621504 size:11272192 | 20260507 08:47:17.321 | NOTICE | [1004]: 1970-01-02T13:11:01.623Z GPU thread:139974746415104 size:11272192 | 20260507 08:47:17.321 | NOTICE | [1005]: 1970-01-02T13:11:01.649Z GPU thread:139974725423104 size:11272192 | 20260507 08:47:17.321 | NOTICE | [1006]: 1970-01-02T13:11:01.675Z GPU thread:139974733819904 size:11272192 | 20260507 08:47:17.321 | NOTICE | [1007]: 1970-01-02T13:11:01.784Z GPU thread:139974738018304 size:11272192 | 20260507 08:47:17.321 | NOTICE | [1008]: 1970-01-02T13:11:01.811Z GPU thread:139974754811904 size:11272192 | 20260507 08:47:17.321 | NOTICE | [1009]: 1970-01-02T13:11:01.837Z GPU thread:139974742216704 size:11272192 | 20260507 08:47:17.321 | NOTICE | [1010]: 1970-01-02T13:11:01.863Z GPU thread:139974725423104 size:11272192 | 20260507 08:47:17.321 | NOTICE | [1011]: 1970-01-02T13:11:01.889Z GPU thread:139974746415104 size:11272192 | 20260507 08:47:17.321 | NOTICE | [1012]: 1970-01-02T13:11:01.915Z GPU thread:139974763208704 size:11272192 | 20260507 08:47:17.321 | NOTICE | [1013]: 1970-01-02T13:11:01.940Z GPU thread:139974738018304 size:11272192 | 20260507 08:47:17.321 | NOTICE | [1014]: 1970-01-02T13:11:02.049Z GPU thread:139974729621504 size:11272192 | 20260507 08:47:17.321 | NOTICE | [1015]: 1970-01-02T13:11:02.075Z GPU thread:139974733819904 size:11272192 | 20260507 08:47:17.322 | NOTICE | [1016]: 1970-01-02T13:11:02.099Z GPU thread:139974763208704 size:11272192 | 20260507 08:47:17.322 | NOTICE | [1017]: 1970-01-02T13:11:40.588Z HOST thread:139974729621504 size:2097156 | 20260507 08:47:17.322 | NOTICE | [1018]: 1970-01-02T13:11:40.590Z HOST thread:139974729621504 size:38273024 | 20260507 08:47:17.322 | NOTICE | [1019]: 1970-01-02T15:39:00.476Z HOST thread:139974754811904 size:108003328 | 20260507 08:47:17.322 | NOTICE | [1020]: 1970-01-02T16:18:45.250Z HOST thread:139974725423104 size:38273024 | 20260507 08:47:17.322 | NOTICE | [1021]: 1970-01-02T18:36:30.203Z HOST thread:139974729621504 size:2097156 | 20260507 08:47:17.322 | NOTICE | [1022]: 1970-01-02T19:57:20.937Z HOST thread:139974733819904 size:38273024 | 20260507 08:47:17.322 | NOTICE | [1023]: 1970-01-02T19:57:20.974Z GPU thread:139974763208704 size:38273024 | 20260507 08:47:17.322 | NOTICE | ----- PBCudaRuntime DumpAllocationHistoryToLog ---- (end) | 20260507 08:47:17.322 | INFO | Application done | 20260507 08:47:17.322 | INFO | Run Time : 1d 54s | 20260507 08:47:17.322 | INFO | CPU Time : 1d 23h | 20260507 08:47:17.322 | INFO | Peak RSS : 61.441 GB | 20260507 08:47:17.322 | INFO | Exit code : 0